From 63045e7cc835efff49d27fdf49e27ea5fbc49bac Mon Sep 17 00:00:00 2001 From: yang_hongliang Date: Wed, 5 Jan 2022 16:07:05 +0800 Subject: [PATCH] add rk3568 patchs Signed-off-by: yang_hongliang --- linux-5.10/rk3568_patch/hdf.patch | 327 + linux-5.10/rk3568_patch/kernel.patch | 1230512 +++++++++++++++++++++++ 2 files changed, 1230839 insertions(+) create mode 100644 linux-5.10/rk3568_patch/hdf.patch create mode 100644 linux-5.10/rk3568_patch/kernel.patch diff --git a/linux-5.10/rk3568_patch/hdf.patch b/linux-5.10/rk3568_patch/hdf.patch new file mode 100644 index 0000000..a118ba5 --- /dev/null +++ b/linux-5.10/rk3568_patch/hdf.patch @@ -0,0 +1,327 @@ +diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S +index 30c102978942..1d8b8de34f1c 100644 +--- a/arch/arm64/kernel/vmlinux.lds.S ++++ b/arch/arm64/kernel/vmlinux.lds.S +@@ -201,6 +201,15 @@ SECTIONS + INIT_RAM_FS + *(.init.rodata.* .init.bss) /* from the EFI stub */ + } ++ ++#ifdef CONFIG_DRIVERS_HDF ++ .init.hdf_table : { ++ _hdf_drivers_start = .; ++ *(.hdf.driver) ++ _hdf_drivers_end = .; ++ } ++#endif ++ + .exit.data : { + EXIT_DATA + } +diff --git a/drivers/Kconfig b/drivers/Kconfig +index 8c97ea01aabd..340987ef1f17 100644 +--- a/drivers/Kconfig ++++ b/drivers/Kconfig +@@ -234,6 +234,8 @@ source "drivers/interconnect/Kconfig" + + source "drivers/counter/Kconfig" + ++source "drivers/hdf/khdf/Kconfig" ++ + source "drivers/most/Kconfig" + + source "drivers/rkflash/Kconfig" +diff --git a/drivers/Makefile b/drivers/Makefile +index 18cc78f5385c..f9cc4fa69f3b 100644 +--- a/drivers/Makefile ++++ b/drivers/Makefile +@@ -188,6 +188,7 @@ obj-$(CONFIG_SIOX) += siox/ + obj-$(CONFIG_GNSS) += gnss/ + obj-$(CONFIG_INTERCONNECT) += interconnect/ + obj-$(CONFIG_COUNTER) += counter/ ++obj-$(CONFIG_DRIVERS_HDF) += hdf/ + obj-$(CONFIG_MOST) += most/ + obj-$(CONFIG_RK_FLASH) += rkflash/ + obj-$(CONFIG_RK_NAND) += rk_nand/ +diff --git a/drivers/hdf/Makefile b/drivers/hdf/Makefile +new file mode 100644 +index 000000000000..5c5e1911c4f7 +--- /dev/null ++++ b/drivers/hdf/Makefile +@@ -0,0 +1,2 @@ ++export PROJECT_ROOT := ../../../../../ ++obj-$(CONFIG_DRIVERS_HDF) += khdf/ +diff --git a/drivers/hid/Makefile b/drivers/hid/Makefile +index 4acb583c92a6..ddcaf4cdcb9c 100644 +--- a/drivers/hid/Makefile ++++ b/drivers/hid/Makefile +@@ -2,6 +2,15 @@ + # + # Makefile for the HID driver + # ++HDF_ROOT_DIR = -I$(srctree)/drivers/hdf ++ccflags-$(CONFIG_DRIVERS_HDF_INPUT) += $(HDF_ROOT_DIR)/framework/model/input/driver \ ++ $(HDF_ROOT_DIR)/framework/include/core \ ++ $(HDF_ROOT_DIR)/framework/core/common/include/host \ ++ $(HDF_ROOT_DIR)/framework/include/utils \ ++ $(HDF_ROOT_DIR)/framework/include/osal \ ++ $(HDF_ROOT_DIR)/framework/ability/sbuf/include \ ++ $(HDF_ROOT_DIR)/khdf/osal/include \ ++ $(HDF_ROOT_DIR)/evdev + hid-y := hid-core.o hid-input.o hid-quirks.o + hid-$(CONFIG_DEBUG_FS) += hid-debug.o + +diff --git a/drivers/hid/hid-core.c b/drivers/hid/hid-core.c +index 5550c943f985..f0503e2327a3 100644 +--- a/drivers/hid/hid-core.c ++++ b/drivers/hid/hid-core.c +@@ -33,6 +33,9 @@ + #include + #include + ++#if defined(CONFIG_DRIVERS_HDF_INPUT) ++#include "hdf_hid_adapter.h" ++#endif + #include "hid-ids.h" + + /* +@@ -1522,6 +1525,11 @@ static void hid_process_event(struct hid_device *hid, struct hid_field *field, + hidinput_hid_event(hid, field, usage, value); + if (hid->claimed & HID_CLAIMED_HIDDEV && interrupt && hid->hiddev_hid_event) + hid->hiddev_hid_event(hid, field, usage, value); ++#if defined(CONFIG_DRIVERS_HDF_INPUT) ++ if (hid->input_dev) { ++ HidReportEvent(hid->input_dev, usage->type, usage->code, value); ++ } ++#endif + } + + /* +@@ -1928,6 +1936,85 @@ static const struct device_attribute dev_attr_country = { + .show = show_country, + }; + ++#if defined(CONFIG_DRIVERS_HDF_INPUT) ++static bool check_mouse(char *name) ++{ ++ int i; ++ static char *option[]={"Mouse", "mouse", "MOUSE", "Razer"}; ++ for (i = 0; i < 4; i++) { ++ if (strstr(name, option[i])) ++ return true; ++ } ++ return false; ++} ++static bool check_kbd(char *name) ++{ ++ int i; ++ static char *option[]={"Keyboard", "keyboard"}; ++ for (i = 0; i < 2; i++) { ++ if (strstr(name, option[i])) ++ return true; ++ } ++ return false; ++} ++static bool check_rocker(char *name) ++{ ++ int i; ++ static char *option[]={"Thrustmaster"}; ++ for (i = 0; i < 1; i++) { ++ if (strstr(name, option[i])) ++ return true; ++ } ++ return false; ++} ++static bool check_encoder(char *name) ++{ ++ if (strcmp(name, "Wired KeyBoard") == 0) { ++ return true; ++ } ++ return false; ++} ++static bool check_trackball(char *name) ++{ ++ int i; ++ static char *option[]={"Trackball"}; ++ for (i = 0; i < 1; i++) { ++ if (strstr(name, option[i])) ++ return true; ++ } ++ return false; ++} ++static void notify_connect_event(struct hid_device *hdev) ++{ ++ bool check; ++ int type = -1; ++ HidInfo *dev = (HidInfo *)kmalloc(sizeof(HidInfo), GFP_KERNEL); ++ if (dev == NULL) { ++ printk("%s: malloc failed", __func__); ++ return; ++ } ++ type = check_mouse(hdev->name)?HID_TYPE_MOUSE:type; ++ type = check_kbd(hdev->name)?HID_TYPE_KEYBOARD:type; ++ type = check_rocker(hdev->name)?HID_TYPE_ROCKER:type; ++ type = check_encoder(hdev->name)?HID_TYPE_ENCODER:type; ++ type = check_trackball(hdev->name)?HID_TYPE_TRACKBALL:type; ++ if ( type < 0) { ++ kfree(dev); ++ dev = NULL; ++ return; ++ } ++ ++ dev->devType = type; ++ dev->devName = hdev->name; ++ hdev->input_dev = HidRegisterHdfInputDev(dev); ++ if (hdev->input_dev == NULL) { ++ printk("%s: RegisterInputDevice failed\n", __func__); ++ } ++ kfree(dev); ++ dev = NULL; ++} ++#endif ++ + int hid_connect(struct hid_device *hdev, unsigned int connect_mask) + { + static const char *types[] = { "Device", "Pointer", "Mouse", "Device", +@@ -2020,6 +2107,9 @@ int hid_connect(struct hid_device *hdev, unsigned int connect_mask) + hid_info(hdev, "%s: %s HID v%x.%02x %s [%s] on %s\n", + buf, bus, hdev->version >> 8, hdev->version & 0xff, + type, hdev->name, hdev->phys); ++#if defined(CONFIG_DRIVERS_HDF_INPUT) ++ notify_connect_event(hdev); ++#endif + + return 0; + } +@@ -2035,6 +2125,10 @@ void hid_disconnect(struct hid_device *hdev) + if (hdev->claimed & HID_CLAIMED_HIDRAW) + hidraw_disconnect(hdev); + hdev->claimed = 0; ++#if defined(CONFIG_DRIVERS_HDF_INPUT) ++ if (hdev->input_dev) ++ HidUnregisterHdfInputDev(hdev->input_dev); ++#endif + } + EXPORT_SYMBOL_GPL(hid_disconnect); + +@@ -2119,6 +2213,11 @@ EXPORT_SYMBOL_GPL(hid_hw_open); + */ + void hid_hw_close(struct hid_device *hdev) + { ++#if defined(CONFIG_DRIVERS_HDF_INPUT) ++ if (hdev->input_dev) { ++ return; ++ } ++#endif + mutex_lock(&hdev->ll_open_lock); + if (!--hdev->ll_open_count) + hdev->ll_driver->close(hdev); +diff --git a/drivers/hid/hid-input.c b/drivers/hid/hid-input.c +index 580d378342c4..fb945f4ce1ff 100644 +--- a/drivers/hid/hid-input.c ++++ b/drivers/hid/hid-input.c +@@ -20,6 +20,10 @@ + #include + #include + ++#if defined(CONFIG_DRIVERS_HDF_INPUT) ++#include "hdf_hid_adapter.h" ++#endif ++ + #include "hid-ids.h" + + #define unk KEY_UNKNOWN +@@ -1416,7 +1420,15 @@ void hidinput_report_event(struct hid_device *hid, struct hid_report *report) + return; + + list_for_each_entry(hidinput, &hid->inputs, list) ++#if defined(CONFIG_DRIVERS_HDF_INPUT) ++ { ++#endif + input_sync(hidinput->input); ++#if defined(CONFIG_DRIVERS_HDF_INPUT) ++ if(hid->input_dev) ++ HidReportEvent(hid->input_dev, EV_SYN, SYN_REPORT, 0); ++ } ++#endif + } + EXPORT_SYMBOL_GPL(hidinput_report_event); + +@@ -1867,6 +1879,42 @@ static inline void hidinput_configure_usages(struct hid_input *hidinput, + report->field[i]->usage + j); + } + ++#if defined(CONFIG_DRIVERS_HDF_INPUT) ++static void transfer_info(struct input_dev *dev) ++{ ++ int i; ++ HidInfo *info = (HidInfo *)kmalloc(sizeof(HidInfo),GFP_KERNEL); ++ if (info == NULL) { ++ printk("%s: malloc failed\n",__func__); ++ return; ++ } ++ info->devName = dev->name; ++ memcpy(info->devProp, dev->propbit, sizeof(unsigned long) * BITS_TO_LONGS(INPUT_PROP_CNT)); ++ memcpy(info->eventType, dev->evbit, sizeof(unsigned long) * BITS_TO_LONGS(EV_CNT)); ++ memcpy(info->keyCode, dev->keybit, sizeof(unsigned long) * BITS_TO_LONGS(KEY_CNT)); ++ memcpy(info->relCode, dev->relbit, sizeof(unsigned long) * BITS_TO_LONGS(REL_CNT)); ++ memcpy(info->absCode, dev->absbit, sizeof(unsigned long) * BITS_TO_LONGS(ABS_CNT)); ++ memcpy(info->miscCode, dev->mscbit, sizeof(unsigned long) * BITS_TO_LONGS(MSC_CNT)); ++ memcpy(info->ledCode, dev->ledbit, sizeof(unsigned long) * BITS_TO_LONGS(LED_CNT)); ++ memcpy(info->soundCode, dev->sndbit, sizeof(unsigned long) * BITS_TO_LONGS(SND_CNT)); ++ memcpy(info->forceCode, dev->ffbit, sizeof(unsigned long) * BITS_TO_LONGS(FF_CNT)); ++ memcpy(info->switchCode, dev->swbit, sizeof(unsigned long) * BITS_TO_LONGS(SW_CNT)); ++ for (i = 0; i < BITS_TO_LONGS(ABS_CNT); i++) { ++ if (dev->absbit[i] != 0) { ++ memcpy(info->axisInfo, dev->absinfo, sizeof(struct input_absinfo) * ABS_CNT); ++ break; ++ } ++ } ++ info->bustype = dev->id.bustype; ++ info->vendor = dev->id.vendor; ++ info->product = dev->id.product; ++ info->version = dev->id.version; ++ SendInfoToHdf(info); ++ kfree(info); ++ info = NULL; ++} ++#endif ++ + /* + * Register the input device; print a message. + * Configure the input layer interface +@@ -1952,6 +2000,9 @@ int hidinput_connect(struct hid_device *hid, unsigned int force) + continue; + } + ++#if defined(CONFIG_DRIVERS_HDF_INPUT) ++ transfer_info(hidinput->input); ++#endif + if (input_register_device(hidinput->input)) + goto out_unwind; + hidinput->registered = true; +diff --git a/drivers/input/mousedev.c b/drivers/input/mousedev.c +index 505c562a5daa..67d451beba08 100644 +--- a/drivers/input/mousedev.c ++++ b/drivers/input/mousedev.c +@@ -869,7 +869,7 @@ static struct mousedev *mousedev_create(struct input_dev *dev, + + if (mixdev) { + dev_set_name(&mousedev->dev, "mice"); +- ++ mousedev->open = 1; + mousedev->open_device = mixdev_open_devices; + mousedev->close_device = mixdev_close_devices; + } else { +diff --git a/include/linux/hid.h b/include/linux/hid.h +index 6ed2a97eb55f..1d1445a23967 100644 +--- a/include/linux/hid.h ++++ b/include/linux/hid.h +@@ -622,6 +622,7 @@ struct hid_device { /* device report descriptor */ + struct list_head debug_list; + spinlock_t debug_list_lock; + wait_queue_head_t debug_wait; ++ void *input_dev; + }; + + #define to_hid_device(pdev) \ diff --git a/linux-5.10/rk3568_patch/kernel.patch b/linux-5.10/rk3568_patch/kernel.patch new file mode 100644 index 0000000..7c412c2 --- /dev/null +++ b/linux-5.10/rk3568_patch/kernel.patch @@ -0,0 +1,1230512 @@ +diff --git a/Documentation/devicetree/bindings/devfreq/event/rockchip-dfi.txt b/Documentation/devicetree/bindings/devfreq/event/rockchip-dfi.txt +index 148191b0fc15..eee2a7f7cb92 100644 +--- a/Documentation/devicetree/bindings/devfreq/event/rockchip-dfi.txt ++++ b/Documentation/devicetree/bindings/devfreq/event/rockchip-dfi.txt +@@ -1,8 +1,22 @@ + +-* Rockchip rk3399 DFI device ++* Rockchip DFI device + + Required properties: +-- compatible: Must be "rockchip,rk3399-dfi". ++- compatible: Should be one of the following. ++ - "rockchip,px30-dfi" - for PX30 SoCs. ++ - "rockchip,rk1808-dfi" - for RK1808 SoCs. ++ - "rockchip,rk3128-dfi" - for RK3128 SoCs. ++ - "rockchip,rk3288-dfi" - for RK3288 SoCs. ++ - "rockchip,rk3328-dfi" - for RK3328 SoCs. ++ - "rockchip,rk3368-dfi" - for RK3368 SoCs. ++ - "rockchip,rk3399-dfi" - for RK3399 SoCs. ++ - "rockchip,rk3568-dfi" - for RK3568 SoCs. ++ - "rockchip,rv1126-dfi" - for RV1126 SoCs. ++ ++Required properties for RK3368: ++- rockchip,grf: phandle to the syscon managing the "general register files" ++ ++Required properties for RK3399: + - reg: physical base address of each DFI and length of memory mapped region + - rockchip,pmu: phandle to the syscon managing the "pmu general register files" + - clocks: phandles for clock specified in "clock-names" property +diff --git a/Documentation/devicetree/bindings/display/bridge/analogix_dp.txt b/Documentation/devicetree/bindings/display/bridge/analogix_dp.txt +index 027d76c27a41..a548f404033c 100644 +--- a/Documentation/devicetree/bindings/display/bridge/analogix_dp.txt ++++ b/Documentation/devicetree/bindings/display/bridge/analogix_dp.txt +@@ -21,17 +21,23 @@ Required properties for dp-controller: + from general PHY binding: Should be "dp". + + Optional properties for dp-controller: ++ -analogix,video-bist-enable: ++ Enable video bist pattern for DP_TX debugging. + -force-hpd: + Indicate driver need force hpd when hpd detect failed, this + is used for some eDP screen which don't have hpd signal. + -hpd-gpios: + Hotplug detect GPIO. + Indicates which GPIO should be used for hotplug detection ++ -panel-self-test: ++ Enable optional LCD Panel Self Test. + -port@[X]: SoC specific port nodes with endpoint definitions as defined + in Documentation/devicetree/bindings/media/video-interfaces.txt, + please refer to the SoC specific binding document: + * Documentation/devicetree/bindings/display/exynos/exynos_dp.txt + * Documentation/devicetree/bindings/display/rockchip/analogix_dp-rockchip.txt ++ -support-psr: ++ Enable Source's PSR capability. + + [1]: Documentation/devicetree/bindings/media/video-interfaces.txt + ------------------------------------------------------------------------------- +diff --git a/Documentation/devicetree/bindings/iio/adc/rockchip-saradc.yaml b/Documentation/devicetree/bindings/iio/adc/rockchip-saradc.yaml +index 1bb76197787b..de8e78d56dea 100644 +--- a/Documentation/devicetree/bindings/iio/adc/rockchip-saradc.yaml ++++ b/Documentation/devicetree/bindings/iio/adc/rockchip-saradc.yaml +@@ -18,6 +18,7 @@ properties: + - items: + - enum: + - rockchip,px30-saradc ++ - rockchip,rk1808-saradc + - rockchip,rk3308-saradc + - rockchip,rk3328-saradc + - rockchip,rv1108-saradc +diff --git a/Documentation/devicetree/bindings/iommu/rockchip,iommu.txt b/Documentation/devicetree/bindings/iommu/rockchip,iommu.txt +index 6ecefea1c6f9..0b906fb630e2 100644 +--- a/Documentation/devicetree/bindings/iommu/rockchip,iommu.txt ++++ b/Documentation/devicetree/bindings/iommu/rockchip,iommu.txt +@@ -24,6 +24,10 @@ Optional properties: + - rockchip,disable-mmu-reset : Don't use the mmu reset operation. + Some mmu instances may produce unexpected results + when the reset operation is used. ++- rk_iommu,disable_reset_quirk : Same with above, for compatible with previous code ++ ++- rockchip,skip-mmu-read : Some iommu instances are not able to be read, skip ++ reading operation to make iommu work as normal + + Example: + +diff --git a/Documentation/devicetree/bindings/mtd/rockchip,nand-controller.yaml b/Documentation/devicetree/bindings/mtd/rockchip,nand-controller.yaml +new file mode 100644 +index 000000000000..0922536b1811 +--- /dev/null ++++ b/Documentation/devicetree/bindings/mtd/rockchip,nand-controller.yaml +@@ -0,0 +1,161 @@ ++# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) ++%YAML 1.2 ++--- ++$id: http://devicetree.org/schemas/mtd/rockchip,nand-controller.yaml# ++$schema: http://devicetree.org/meta-schemas/core.yaml# ++ ++title: Rockchip SoCs NAND FLASH Controller (NFC) ++ ++allOf: ++ - $ref: "nand-controller.yaml#" ++ ++maintainers: ++ - Heiko Stuebner ++ ++properties: ++ compatible: ++ oneOf: ++ - const: rockchip,px30-nfc ++ - const: rockchip,rk2928-nfc ++ - const: rockchip,rv1108-nfc ++ - items: ++ - const: rockchip,rk3036-nfc ++ - const: rockchip,rk2928-nfc ++ - items: ++ - const: rockchip,rk3308-nfc ++ - const: rockchip,rv1108-nfc ++ ++ reg: ++ maxItems: 1 ++ ++ interrupts: ++ maxItems: 1 ++ ++ clocks: ++ minItems: 1 ++ items: ++ - description: Bus Clock ++ - description: Module Clock ++ ++ clock-names: ++ minItems: 1 ++ items: ++ - const: ahb ++ - const: nfc ++ ++ assigned-clocks: ++ maxItems: 1 ++ ++ assigned-clock-rates: ++ maxItems: 1 ++ ++ power-domains: ++ maxItems: 1 ++ ++patternProperties: ++ "^nand@[0-7]$": ++ type: object ++ properties: ++ reg: ++ minimum: 0 ++ maximum: 7 ++ ++ nand-ecc-mode: ++ const: hw ++ ++ nand-ecc-step-size: ++ const: 1024 ++ ++ nand-ecc-strength: ++ enum: [16, 24, 40, 60, 70] ++ description: | ++ The ECC configurations that can be supported are as follows. ++ NFC v600 ECC 16, 24, 40, 60 ++ RK2928, RK3066, RK3188 ++ ++ NFC v622 ECC 16, 24, 40, 60 ++ RK3036, RK3128 ++ ++ NFC v800 ECC 16 ++ RK3308, RV1108 ++ ++ NFC v900 ECC 16, 40, 60, 70 ++ RK3326, PX30 ++ ++ nand-bus-width: ++ const: 8 ++ ++ rockchip,boot-blks: ++ $ref: /schemas/types.yaml#/definitions/uint32 ++ minimum: 2 ++ default: 16 ++ description: ++ The NFC driver need this information to select ECC ++ algorithms supported by the boot ROM. ++ Only used in combination with 'nand-is-boot-medium'. ++ ++ rockchip,boot-ecc-strength: ++ enum: [16, 24, 40, 60, 70] ++ allOf: ++ - $ref: /schemas/types.yaml#/definitions/uint32 ++ description: | ++ If specified it indicates that a different BCH/ECC setting is ++ supported by the boot ROM. ++ NFC v600 ECC 16, 24 ++ RK2928, RK3066, RK3188 ++ ++ NFC v622 ECC 16, 24, 40, 60 ++ RK3036, RK3128 ++ ++ NFC v800 ECC 16 ++ RK3308, RV1108 ++ ++ NFC v900 ECC 16, 70 ++ RK3326, PX30 ++ ++ Only used in combination with 'nand-is-boot-medium'. ++ ++required: ++ - compatible ++ - reg ++ - interrupts ++ - clocks ++ - clock-names ++ ++unevaluatedProperties: false ++ ++examples: ++ - | ++ #include ++ #include ++ nfc: nand-controller@ff4b0000 { ++ compatible = "rockchip,rk3308-nfc", ++ "rockchip,rv1108-nfc"; ++ reg = <0xff4b0000 0x4000>; ++ interrupts = ; ++ clocks = <&cru HCLK_NANDC>, <&cru SCLK_NANDC>; ++ clock-names = "ahb", "nfc"; ++ assigned-clocks = <&clks SCLK_NANDC>; ++ assigned-clock-rates = <150000000>; ++ ++ pinctrl-0 = <&flash_ale &flash_bus8 &flash_cle &flash_csn0 ++ &flash_rdn &flash_rdy &flash_wrn>; ++ pinctrl-names = "default"; ++ ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ nand@0 { ++ reg = <0>; ++ label = "rk-nand"; ++ nand-bus-width = <8>; ++ nand-ecc-mode = "hw"; ++ nand-ecc-step-size = <1024>; ++ nand-ecc-strength = <16>; ++ nand-is-boot-medium; ++ rockchip,boot-blks = <8>; ++ rockchip,boot-ecc-strength = <16>; ++ }; ++ }; ++ ++... +diff --git a/Documentation/devicetree/bindings/nvmem/rockchip-efuse.yaml b/Documentation/devicetree/bindings/nvmem/rockchip-efuse.yaml +index 104dd508565e..af1b9c31862a 100644 +--- a/Documentation/devicetree/bindings/nvmem/rockchip-efuse.yaml ++++ b/Documentation/devicetree/bindings/nvmem/rockchip-efuse.yaml +@@ -15,10 +15,13 @@ allOf: + properties: + compatible: + enum: ++ - rockchip,rk1808-efuse + - rockchip,rk3066a-efuse ++ - rockchip,rk3128-efuse + - rockchip,rk3188-efuse + - rockchip,rk3228-efuse + - rockchip,rk3288-efuse ++ - rockchip,rk3288-secure-efuse + - rockchip,rk3328-efuse + - rockchip,rk3368-efuse + - rockchip,rk3399-efuse +diff --git a/Documentation/devicetree/bindings/nvmem/rockchip-otp.txt b/Documentation/devicetree/bindings/nvmem/rockchip-otp.txt +index 40f649f7c2e5..6e61345829e7 100644 +--- a/Documentation/devicetree/bindings/nvmem/rockchip-otp.txt ++++ b/Documentation/devicetree/bindings/nvmem/rockchip-otp.txt +@@ -7,6 +7,8 @@ Required properties: + - reg: Should contain the registers location and size + - clocks: Must contain an entry for each entry in clock-names. + - clock-names: Should be "otp", "apb_pclk" and "phy". ++ ++Optional properties: + - resets: Must contain an entry for each entry in reset-names. + See ../../reset/reset.txt for details. + - reset-names: Should be "phy". +diff --git a/Documentation/devicetree/bindings/phy/phy-rockchip-typec.txt b/Documentation/devicetree/bindings/phy/phy-rockchip-typec.txt +index 960da7fcaa9e..ac82f7b4dd58 100644 +--- a/Documentation/devicetree/bindings/phy/phy-rockchip-typec.txt ++++ b/Documentation/devicetree/bindings/phy/phy-rockchip-typec.txt +@@ -17,6 +17,11 @@ Required properties: + + Optional properties: + - extcon : extcon specifier for the Power Delivery ++ - rockchip,phy-config : A list of voltage swing(mV) and pre-emphasis ++ (dB) pairs. They are 3 blocks of 4 entries and ++ correspond to s0p0 ~ s0p3, s1p0 ~ s1p3, ++ s2p0 ~ s2p3, s3p0 ~ s2p3 swing and pre-emphasis ++ values. + + Required nodes : a sub-node is required for each port the phy provides. + The sub-node name is used to identify dp or usb3 port, +@@ -50,6 +55,21 @@ Example: + <&cru SRST_P_UPHY0_TCPHY>; + reset-names = "uphy", "uphy-pipe", "uphy-tcphy"; + ++ rockchip,phy-config = <0x2a 0x00>, ++ <0x1f 0x15>, ++ <0x14 0x22>, ++ <0x02 0x2b>, ++ ++ <0x21 0x00>, ++ <0x12 0x15>, ++ <0x02 0x22>, ++ <0 0>, ++ ++ <0x15 0x00>, ++ <0x00 0x15>, ++ <0 0>, ++ <0 0>; ++ + tcphy0_dp: dp-port { + #phy-cells = <0>; + }; +@@ -74,6 +94,21 @@ Example: + <&cru SRST_P_UPHY1_TCPHY>; + reset-names = "uphy", "uphy-pipe", "uphy-tcphy"; + ++ rockchip,phy-config = <0x2a 0x00>, ++ <0x1f 0x15>, ++ <0x14 0x22>, ++ <0x02 0x2b>, ++ ++ <0x21 0x00>, ++ <0x12 0x15>, ++ <0x02 0x22>, ++ <0 0>, ++ ++ <0x15 0x00>, ++ <0x00 0x15>, ++ <0 0>, ++ <0 0>; ++ + tcphy1_dp: dp-port { + #phy-cells = <0>; + }; +diff --git a/Documentation/devicetree/bindings/power/rockchip-io-domain.txt b/Documentation/devicetree/bindings/power/rockchip-io-domain.txt +index e66fd4eab71c..09824bca248e 100644 +--- a/Documentation/devicetree/bindings/power/rockchip-io-domain.txt ++++ b/Documentation/devicetree/bindings/power/rockchip-io-domain.txt +@@ -41,8 +41,10 @@ Required properties: + - "rockchip,rk3368-pmu-io-voltage-domain" for rk3368 pmu-domains + - "rockchip,rk3399-io-voltage-domain" for rk3399 + - "rockchip,rk3399-pmu-io-voltage-domain" for rk3399 pmu-domains ++ - "rockchip,rk3568-pmu-io-voltage-domain" for rk3568 pmu-domains + - "rockchip,rv1108-io-voltage-domain" for rv1108 + - "rockchip,rv1108-pmu-io-voltage-domain" for rv1108 pmu-domains ++ - "rockchip,rv1126-pmu-io-voltage-domain" for rv1126 pmu-domains + + Deprecated properties: + - rockchip,grf: phandle to the syscon managing the "general register files" +@@ -112,11 +114,33 @@ Possible supplies for rk3399: + - bt656-supply: The supply connected to APIO2_VDD. + - audio-supply: The supply connected to APIO5_VDD. + - sdmmc-supply: The supply connected to SDMMC0_VDD. +-- gpio1830 The supply connected to APIO4_VDD. ++- gpio1830-supply: The supply connected to APIO4_VDD. + + Possible supplies for rk3399 pmu-domains: + - pmu1830-supply:The supply connected to PMUIO2_VDD. + ++Possible supplies for rk3568 pmu-domains: ++- vccio1-supply:The supply connected to VCCIO1. ++- vccio2-supply:The supply connected to VCCIO2, can be reserved since ignored by driver. ++- vccio3-supply:The supply connected to VCCIO3. ++- vccio4-supply:The supply connected to VCCIO4. ++- vccio5-supply:The supply connected to VCCIO5. ++- vccio6-supply:The supply connected to VCCIO6. ++- vccio7-supply:The supply connected to VCCIO7. ++- pmuio1-supply:The supply connected to PMUIO1, 3.3v only can be reserved ++- pmuio2-supply:The supply connected to PMUIO2. ++ ++Possible supplies for rv1126 pmu-domains: ++- vccio1-supply:The supply connected to VCCIO1_VDD. ++- vccio2-supply:The supply connected to VCCIO2_VDD. ++- vccio3-supply:The supply connected to VCCIO3_VDD. ++- vccio4-supply:The supply connected to VCCIO4_VDD. ++- vccio5-supply:The supply connected to VCCIO5_VDD. ++- vccio6-supply:The supply connected to VCCIO6_VDD. ++- vccio7-supply:The supply connected to VCCIO7_VDD. ++- pmuio1-supply:The supply connected to PMUIO1_VDD. ++- pmuio2-supply:The supply connected to PMUIO2_VDD. ++ + Example: + + io-domains { +diff --git a/Documentation/devicetree/bindings/soc/rockchip/power_domain.txt b/Documentation/devicetree/bindings/soc/rockchip/power_domain.txt +index 8304eceb62e4..e1920b15c86f 100644 +--- a/Documentation/devicetree/bindings/soc/rockchip/power_domain.txt ++++ b/Documentation/devicetree/bindings/soc/rockchip/power_domain.txt +@@ -16,6 +16,7 @@ Required properties for power domain controller: + "rockchip,rk3366-power-controller" - for RK3366 SoCs. + "rockchip,rk3368-power-controller" - for RK3368 SoCs. + "rockchip,rk3399-power-controller" - for RK3399 SoCs. ++ "rockchip,rk3568-power-controller" - for RK3568 SoCs. + - #power-domain-cells: Number of cells in a power-domain specifier. + Should be 1 for multiple PM domains. + - #address-cells: Should be 1. +@@ -34,6 +35,7 @@ Required properties for power domain sub nodes: + "include/dt-bindings/power/rk3366-power.h" - for RK3366 type power domain. + "include/dt-bindings/power/rk3368-power.h" - for RK3368 type power domain. + "include/dt-bindings/power/rk3399-power.h" - for RK3399 type power domain. ++ "include/dt-bindings/power/rk3568-power.h" - for RK3568 type power domain. + - clocks (optional): phandles to clocks which need to be enabled while power domain + switches state. + - pm_qos (optional): phandles to qos blocks which need to be saved and restored +@@ -114,6 +116,7 @@ The index should use macros in: + "include/dt-bindings/power/rk3366-power.h" - for rk3366 type power domain. + "include/dt-bindings/power/rk3368-power.h" - for rk3368 type power domain. + "include/dt-bindings/power/rk3399-power.h" - for rk3399 type power domain. ++ "include/dt-bindings/power/rk3568-power.h" - for rk3568 type power domain. + + Example of the node using power domain: + +diff --git a/Documentation/devicetree/bindings/sound/rt5651.txt b/Documentation/devicetree/bindings/sound/rt5651.txt +index 56e736a1cba9..e7cd338d2b59 100644 +--- a/Documentation/devicetree/bindings/sound/rt5651.txt ++++ b/Documentation/devicetree/bindings/sound/rt5651.txt +@@ -38,6 +38,8 @@ Optional properties: + 2: Scale current by 1.0 + 3: Scale current by 1.5 + ++- spk-con-gpio: speaker amplifier enable/disable control ++ + Pins on the device (for linking into audio routes) for RT5651: + + * DMIC L1 +diff --git a/Documentation/devicetree/bindings/spi/spi-rockchip.yaml b/Documentation/devicetree/bindings/spi/spi-rockchip.yaml +index 1e6cf29e6388..7f987e79337c 100644 +--- a/Documentation/devicetree/bindings/spi/spi-rockchip.yaml ++++ b/Documentation/devicetree/bindings/spi/spi-rockchip.yaml +@@ -33,6 +33,7 @@ properties: + - rockchip,rk3328-spi + - rockchip,rk3368-spi + - rockchip,rk3399-spi ++ - rockchip,rv1126-spi + - const: rockchip,rk3066-spi + + reg: +diff --git a/Documentation/devicetree/bindings/thermal/rockchip-thermal.txt b/Documentation/devicetree/bindings/thermal/rockchip-thermal.txt +index 7f94669e9ebe..346e466c2006 100644 +--- a/Documentation/devicetree/bindings/thermal/rockchip-thermal.txt ++++ b/Documentation/devicetree/bindings/thermal/rockchip-thermal.txt +@@ -9,6 +9,7 @@ Required properties: + "rockchip,rk3328-tsadc": found on RK3328 SoCs + "rockchip,rk3368-tsadc": found on RK3368 SoCs + "rockchip,rk3399-tsadc": found on RK3399 SoCs ++ "rockchip,rk3568-tsadc": found on RK3568 SoCs + - reg : physical base address of the controller and length of memory mapped + region. + - interrupts : The interrupt number to the cpu. The interrupt specifier format +diff --git a/Makefile b/Makefile +index 756479b101f8..2ad9ef45fafd 100644 +--- a/Makefile ++++ b/Makefile +@@ -493,7 +493,7 @@ LINUXINCLUDE := \ + $(USERINCLUDE) + + KBUILD_AFLAGS := -D__ASSEMBLY__ -fno-PIE +-KBUILD_CFLAGS := -Wall -Wundef -Werror=strict-prototypes -Wno-trigraphs \ ++KBUILD_CFLAGS := -Wall -Wundef -Wno-trigraphs \ + -fno-strict-aliasing -fno-common -fshort-wchar -fno-PIE \ + -Werror=implicit-function-declaration -Werror=implicit-int \ + -Werror=return-type -Wno-format-security \ +@@ -944,7 +944,7 @@ KBUILD_CFLAGS += $(call cc-option,-fconserve-stack) + KBUILD_CFLAGS += -Werror=date-time + + # enforce correct pointer usage +-KBUILD_CFLAGS += $(call cc-option,-Werror=incompatible-pointer-types) ++#KBUILD_CFLAGS += $(call cc-option,-Werror=incompatible-pointer-types) + + # Require designated initializers for all marked structures + KBUILD_CFLAGS += $(call cc-option,-Werror=designated-init) +diff --git a/arch/arm/boot/dts/rk3036.dtsi b/arch/arm/boot/dts/rk3036.dtsi +index 093567022386..dc4c26a8a196 100644 +--- a/arch/arm/boot/dts/rk3036.dtsi ++++ b/arch/arm/boot/dts/rk3036.dtsi +@@ -85,6 +85,11 @@ display-subsystem { + ports = <&vop_out>; + }; + ++ psci { ++ compatible = "arm,psci-1.0"; ++ method = "smc"; ++ }; ++ + timer { + compatible = "arm,armv7-timer"; + arm,cpu-registers-not-fw-configured; +@@ -360,7 +365,7 @@ pwm0: pwm@20050000 { + #pwm-cells = <3>; + clocks = <&cru PCLK_PWM>; + clock-names = "pwm"; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm0_pin>; + status = "disabled"; + }; +@@ -371,7 +376,7 @@ pwm1: pwm@20050010 { + #pwm-cells = <3>; + clocks = <&cru PCLK_PWM>; + clock-names = "pwm"; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm1_pin>; + status = "disabled"; + }; +@@ -382,7 +387,7 @@ pwm2: pwm@20050020 { + #pwm-cells = <3>; + clocks = <&cru PCLK_PWM>; + clock-names = "pwm"; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm2_pin>; + status = "disabled"; + }; +@@ -393,7 +398,7 @@ pwm3: pwm@20050030 { + #pwm-cells = <2>; + clocks = <&cru PCLK_PWM>; + clock-names = "pwm"; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm3_pin>; + status = "disabled"; + }; +diff --git a/arch/arm/boot/dts/rk3066a-rayeager.dts b/arch/arm/boot/dts/rk3066a-rayeager.dts +index 309518403d86..1a9891f802f8 100644 +--- a/arch/arm/boot/dts/rk3066a-rayeager.dts ++++ b/arch/arm/boot/dts/rk3066a-rayeager.dts +@@ -23,6 +23,20 @@ ir: ir-receiver { + pinctrl-0 = <&ir_int>; + }; + ++ vdd_logic: vdd-logic { ++ compatible = "pwm-regulator"; ++ ++ pwms = <&pwm3 0 25000>; ++ pwm-dutycycle-range = <100 0>; ++ ++ regulator-name = "vdd_logic"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <925000>; ++ regulator-max-microvolt = <1400000>; ++ status = "okay"; ++ }; ++ + keys: gpio-keys { + compatible = "gpio-keys"; + +diff --git a/arch/arm/boot/dts/rk3066a.dtsi b/arch/arm/boot/dts/rk3066a.dtsi +index bbc3bff50856..49b8652da097 100644 +--- a/arch/arm/boot/dts/rk3066a.dtsi ++++ b/arch/arm/boot/dts/rk3066a.dtsi +@@ -800,22 +800,22 @@ power-domain@RK3066_PD_GPU { + }; + + &pwm0 { +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm0_out>; + }; + + &pwm1 { +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm1_out>; + }; + + &pwm2 { +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm2_out>; + }; + + &pwm3 { +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm3_out>; + }; + +diff --git a/arch/arm/boot/dts/rk3188.dtsi b/arch/arm/boot/dts/rk3188.dtsi +index b6bde9d12c2b..c1a87fbe1c92 100644 +--- a/arch/arm/boot/dts/rk3188.dtsi ++++ b/arch/arm/boot/dts/rk3188.dtsi +@@ -739,22 +739,22 @@ power-domain@RK3188_PD_GPU { + }; + + &pwm0 { +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm0_out>; + }; + + &pwm1 { +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm1_out>; + }; + + &pwm2 { +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm2_out>; + }; + + &pwm3 { +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm3_out>; + }; + +diff --git a/arch/arm/boot/dts/rk322x.dtsi b/arch/arm/boot/dts/rk322x.dtsi +index 7de8b006ca13..4b2b7f027c23 100644 +--- a/arch/arm/boot/dts/rk322x.dtsi ++++ b/arch/arm/boot/dts/rk322x.dtsi +@@ -398,7 +398,7 @@ pwm0: pwm@110b0000 { + #pwm-cells = <3>; + clocks = <&cru PCLK_PWM>; + clock-names = "pwm"; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm0_pin>; + status = "disabled"; + }; +@@ -409,7 +409,7 @@ pwm1: pwm@110b0010 { + #pwm-cells = <3>; + clocks = <&cru PCLK_PWM>; + clock-names = "pwm"; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm1_pin>; + status = "disabled"; + }; +@@ -420,7 +420,7 @@ pwm2: pwm@110b0020 { + #pwm-cells = <3>; + clocks = <&cru PCLK_PWM>; + clock-names = "pwm"; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm2_pin>; + status = "disabled"; + }; +@@ -431,7 +431,7 @@ pwm3: pwm@110b0030 { + #pwm-cells = <2>; + clocks = <&cru PCLK_PWM>; + clock-names = "pwm"; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm3_pin>; + status = "disabled"; + }; +diff --git a/arch/arm/boot/dts/rk3288.dtsi b/arch/arm/boot/dts/rk3288.dtsi +index 0d89ad274268..8683c9c13a6a 100644 +--- a/arch/arm/boot/dts/rk3288.dtsi ++++ b/arch/arm/boot/dts/rk3288.dtsi +@@ -695,7 +695,7 @@ pwm0: pwm@ff680000 { + compatible = "rockchip,rk3288-pwm"; + reg = <0x0 0xff680000 0x0 0x10>; + #pwm-cells = <3>; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm0_pin>; + clocks = <&cru PCLK_RKPWM>; + clock-names = "pwm"; +@@ -706,7 +706,7 @@ pwm1: pwm@ff680010 { + compatible = "rockchip,rk3288-pwm"; + reg = <0x0 0xff680010 0x0 0x10>; + #pwm-cells = <3>; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm1_pin>; + clocks = <&cru PCLK_RKPWM>; + clock-names = "pwm"; +@@ -717,7 +717,7 @@ pwm2: pwm@ff680020 { + compatible = "rockchip,rk3288-pwm"; + reg = <0x0 0xff680020 0x0 0x10>; + #pwm-cells = <3>; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm2_pin>; + clocks = <&cru PCLK_RKPWM>; + clock-names = "pwm"; +@@ -728,7 +728,7 @@ pwm3: pwm@ff680030 { + compatible = "rockchip,rk3288-pwm"; + reg = <0x0 0xff680030 0x0 0x10>; + #pwm-cells = <3>; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm3_pin>; + clocks = <&cru PCLK_RKPWM>; + clock-names = "pwm"; +diff --git a/arch/arm/boot/dts/rv1108.dtsi b/arch/arm/boot/dts/rv1108.dtsi +index a1a08cb9364e..b079c0008032 100644 +--- a/arch/arm/boot/dts/rv1108.dtsi ++++ b/arch/arm/boot/dts/rv1108.dtsi +@@ -217,7 +217,7 @@ pwm4: pwm@10280000 { + interrupts = ; + clocks = <&cru SCLK_PWM>, <&cru PCLK_PWM>; + clock-names = "pwm", "pclk"; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm4_pin>; + #pwm-cells = <3>; + status = "disabled"; +@@ -229,7 +229,7 @@ pwm5: pwm@10280010 { + interrupts = ; + clocks = <&cru SCLK_PWM>, <&cru PCLK_PWM>; + clock-names = "pwm", "pclk"; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm5_pin>; + #pwm-cells = <3>; + status = "disabled"; +@@ -241,7 +241,7 @@ pwm6: pwm@10280020 { + interrupts = ; + clocks = <&cru SCLK_PWM>, <&cru PCLK_PWM>; + clock-names = "pwm", "pclk"; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm6_pin>; + #pwm-cells = <3>; + status = "disabled"; +@@ -253,7 +253,7 @@ pwm7: pwm@10280030 { + interrupts = ; + clocks = <&cru SCLK_PWM>, <&cru PCLK_PWM>; + clock-names = "pwm", "pclk"; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm7_pin>; + #pwm-cells = <3>; + status = "disabled"; +@@ -392,7 +392,7 @@ pwm0: pwm@20040000 { + interrupts = ; + clocks = <&cru SCLK_PWM0_PMU>, <&cru PCLK_PWM0_PMU>; + clock-names = "pwm", "pclk"; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm0_pin>; + #pwm-cells = <3>; + status = "disabled"; +@@ -404,7 +404,7 @@ pwm1: pwm@20040010 { + interrupts = ; + clocks = <&cru SCLK_PWM0_PMU>, <&cru PCLK_PWM0_PMU>; + clock-names = "pwm", "pclk"; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm1_pin>; + #pwm-cells = <3>; + status = "disabled"; +@@ -416,7 +416,7 @@ pwm2: pwm@20040020 { + interrupts = ; + clocks = <&cru SCLK_PWM0_PMU>, <&cru PCLK_PWM0_PMU>; + clock-names = "pwm", "pclk"; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm2_pin>; + #pwm-cells = <3>; + status = "disabled"; +@@ -428,7 +428,7 @@ pwm3: pwm@20040030 { + interrupts = ; + clocks = <&cru SCLK_PWM0_PMU>, <&cru PCLK_PWM0_PMU>; + clock-names = "pwm", "pclk"; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm3_pin>; + #pwm-cells = <3>; + status = "disabled"; +diff --git a/arch/arm/kernel/psci_smp.c b/arch/arm/kernel/psci_smp.c +index d4392e177484..059db0b1c1de 100644 +--- a/arch/arm/kernel/psci_smp.c ++++ b/arch/arm/kernel/psci_smp.c +@@ -107,11 +107,12 @@ static int psci_cpu_kill(unsigned int cpu) + + #endif + +-bool __init psci_smp_available(void) ++bool psci_smp_available(void) + { + /* is cpu_on available at least? */ + return (psci_ops.cpu_on != NULL); + } ++EXPORT_SYMBOL(psci_smp_available); + + const struct smp_operations psci_smp_ops __initconst = { + .smp_boot_secondary = psci_boot_secondary, +diff --git a/arch/arm/kernel/reboot.c b/arch/arm/kernel/reboot.c +index 0ce388f15422..6f874647b63b 100644 +--- a/arch/arm/kernel/reboot.c ++++ b/arch/arm/kernel/reboot.c +@@ -18,7 +18,6 @@ typedef void (*phys_reset_t)(unsigned long, bool); + /* + * Function pointers to optional machine specific functions + */ +-void (*arm_pm_restart)(enum reboot_mode reboot_mode, const char *cmd); + void (*pm_power_off)(void); + EXPORT_SYMBOL(pm_power_off); + +@@ -138,10 +137,8 @@ void machine_restart(char *cmd) + local_irq_disable(); + smp_send_stop(); + +- if (arm_pm_restart) +- arm_pm_restart(reboot_mode, cmd); +- else +- do_kernel_restart(cmd); ++ do_kernel_pre_restart(cmd); ++ do_kernel_restart(cmd); + + /* Give a grace period for failure to restart of 1s */ + mdelay(1000); +diff --git a/arch/arm64/Kconfig.platforms b/arch/arm64/Kconfig.platforms +index 5c4ac1c9f4e0..8c0cda1d28d2 100644 +--- a/arch/arm64/Kconfig.platforms ++++ b/arch/arm64/Kconfig.platforms +@@ -224,11 +224,8 @@ config ARCH_RENESAS + config ARCH_ROCKCHIP + bool "Rockchip Platforms" + select ARCH_HAS_RESET_CONTROLLER +- select GPIOLIB + select PINCTRL +- select PINCTRL_ROCKCHIP + select PM +- select ROCKCHIP_TIMER + help + This enables support for the ARMv8 based Rockchip chipsets, + like the RK3368. +diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile +index 485b7dbd4f9e..06633207b0cf 100644 +--- a/arch/arm64/Makefile ++++ b/arch/arm64/Makefile +@@ -44,7 +44,7 @@ ifeq ($(CONFIG_BROKEN_GAS_INST),y) + $(warning Detected assembler with broken .inst; disassembly will be unreliable) + endif + +-KBUILD_CFLAGS += -mgeneral-regs-only \ ++KBUILD_CFLAGS += \ + $(compat_vdso) $(cc_has_k_constraint) + KBUILD_CFLAGS += $(call cc-disable-warning, psabi) + KBUILD_AFLAGS += $(compat_vdso) +@@ -198,3 +198,16 @@ define archhelp + echo ' (distribution) /sbin/installkernel or' + echo ' install to $$(INSTALL_PATH) and run lilo' + endef ++ ++MAKE_MODULES ?= y ++ ++%.img: ++ifeq ("$(CONFIG_MODULES)$(MAKE_MODULES)$(srctree)","yy$(objtree)") ++ $(Q)$(MAKE) rockchip/$*.dtb Image.lz4 modules ++else ++ $(Q)$(MAKE) rockchip/$*.dtb Image.lz4 ++endif ++ $(Q)$(srctree)/scripts/mkimg --dtb $*.dtb ++ ++CLEAN_DIRS += out ++CLEAN_FILES += boot.img kernel.img resource.img zboot.img +diff --git a/arch/arm64/boot/dts/rockchip/Makefile b/arch/arm64/boot/dts/rockchip/Makefile +index 26661c7b736b..a2386c005926 100644 +--- a/arch/arm64/boot/dts/rockchip/Makefile ++++ b/arch/arm64/boot/dts/rockchip/Makefile +@@ -44,3 +44,43 @@ dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-rockpro64.dtb + dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-sapphire.dtb + dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-sapphire-excavator.dtb + dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399pro-rock-pi-n10.dtb ++dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-box-demo-v10.dtb ++dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-evb-mipitest-v10.dtb ++dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-evb1-ddr4-v10.dtb ++dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-evb1-ddr4-v10-linux.dtb ++dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-evb1-ddr4-v10-lvds.dtb ++dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-evb2-lp4x-v10.dtb ++dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-evb2-lp4x-v10-eink.dtb ++dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-evb2-lp4x-v10-i2s-mic-array.dtb ++dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-evb2-lp4x-v10-linux.dtb ++dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-evb2-lp4x-v10-pdm-mic-array.dtb ++dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-evb3-ddr3-v10.dtb ++dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-evb3-ddr3-v10-linux.dtb ++dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-evb5-lp4x-v10.dtb ++dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-rk817-eink.dtb ++dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-rk817-eink-w6.dtb ++dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-rk817-eink-w103.dtb ++dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-rk817-tablet.dtb ++dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-rk817-tablet-k108.dtb ++dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-rk817-tablet-rkg11.dtb ++dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-rk817-tablet-v10.dtb ++dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-evb1-ddr4-v10.dtb ++dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-evb1-ddr4-v10-linux.dtb ++dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-evb1-ddr4-v10-linux-spi-nor.dtb ++dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-evb2-lp4x-v10.dtb ++dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-evb2-lp4x-v10-bt1120-to-hdmi.dtb ++dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-evb4-lp3-v10.dtb ++dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-evb5-ddr4-v10.dtb ++dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-evb6-ddr3-v10.dtb ++dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-evb6-ddr3-v10-linux.dtb ++dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-evb6-ddr3-v10-rk628-bt1120-to-hdmi.dtb ++dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-evb6-ddr3-v10-rk628-rgb2hdmi.dtb ++dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-evb6-ddr3-v10-rk630-bt656-to-cvbs.dtb ++dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-evb7-ddr4-v10.dtb ++dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-iotest-ddr3-v10.dtb ++dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-iotest-ddr3-v10-linux.dtb ++dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-nvr-demo-v10.dtb ++dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-nvr-demo-v10-linux.dtb ++dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-nvr-demo-v10-linux-spi-nand.dtb ++dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-nvr-demo-v12-linux.dtb ++dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-nvr-demo-v12-linux-spi-nand.dtb +diff --git a/arch/arm64/boot/dts/rockchip/px30-ad-d6-anx6345.dts b/arch/arm64/boot/dts/rockchip/px30-ad-d6-anx6345.dts +new file mode 100755 +index 000000000000..2f258286286a +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/px30-ad-d6-anx6345.dts +@@ -0,0 +1,759 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++#include ++#include ++#include ++#include ++#include ++#include "px30.dtsi" ++#include "px30-android.dtsi" ++ ++/ { ++ model = "Rockchip PX30 AD D6 board"; ++ compatible = "rockchip,px30-ad-d6", "rockchip,px30"; ++ ++ dvdd12_anx: dvdd12-anx { ++ compatible = "regulator-fixed"; ++ regulator-name = "dvdd12-anx"; ++ regulator-boot-on; ++ gpio = <&gpio1 RK_PB5 GPIO_ACTIVE_HIGH>; ++ enable-active-high; ++ ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 2>; ++ io-channel-names = "buttons"; ++ poll-interval = <100>; ++ keyup-threshold-microvolt = <1800000>; ++ ++ esc-key { ++ linux,code = ; ++ label = "esc"; ++ press-threshold-microvolt = <1310000>; ++ }; ++ ++ home-key { ++ linux,code = ; ++ label = "home"; ++ press-threshold-microvolt = <624000>; ++ }; ++ ++ menu-key { ++ linux,code = ; ++ label = "menu"; ++ press-threshold-microvolt = <987000>; ++ }; ++ ++ vol-down-key { ++ linux,code = ; ++ label = "volume down"; ++ press-threshold-microvolt = <300000>; ++ }; ++ ++ vol-up-key { ++ linux,code = ; ++ label = "volume up"; ++ press-threshold-microvolt = <17000>; ++ }; ++ }; ++ ++ backlight: backlight { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm1 0 25000 0>; ++ brightness-levels = < ++ 0 1 2 3 4 5 6 7 ++ 8 9 10 11 12 13 14 15 ++ 16 17 18 19 20 21 22 23 ++ 24 25 26 27 28 29 30 31 ++ 32 33 34 35 36 37 38 39 ++ 40 41 42 43 44 45 46 47 ++ 48 49 50 51 52 53 54 55 ++ 56 57 58 59 60 61 62 63 ++ 64 65 66 67 68 69 70 71 ++ 72 73 74 75 76 77 78 79 ++ 80 81 82 83 84 85 86 87 ++ 88 89 90 91 92 93 94 95 ++ 96 97 98 99 100 101 102 103 ++ 104 105 106 107 108 109 110 111 ++ 112 113 114 115 116 117 118 119 ++ 120 121 122 123 124 125 126 127 ++ 128 129 130 131 132 133 134 135 ++ 136 137 138 139 140 141 142 143 ++ 144 145 146 147 148 149 150 151 ++ 152 153 154 155 156 157 158 159 ++ 160 161 162 163 164 165 166 167 ++ 168 169 170 171 172 173 174 175 ++ 176 177 178 179 180 181 182 183 ++ 184 185 186 187 188 189 190 191 ++ 192 193 194 195 196 197 198 199 ++ 200 201 202 203 204 205 206 207 ++ 208 209 210 211 212 213 214 215 ++ 216 217 218 219 220 221 222 223 ++ 224 225 226 227 228 229 230 231 ++ 232 233 234 235 236 237 238 239 ++ 240 241 242 243 244 245 246 247 ++ 248 249 250 251 252 253 254 255>; ++ default-brightness-level = <200>; ++ enable-gpios = <&gpio0 RK_PB7 GPIO_ACTIVE_HIGH>; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_LOW>; ++ }; ++ ++ vcc_phy: vcc-phy-regulator { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_phy"; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ vcc5v0_sys: vccsys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc5v0_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ }; ++}; ++ ++&bus_apll { ++ bus-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&cpu0 { ++ cpu-supply = <&vdd_arm>; ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&dmc { ++ auto-freq-en = <0>; ++ center-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&emmc { ++ bus-width = <8>; ++ cap-mmc-highspeed; ++ mmc-hs200-1_8v; ++ no-sdio; ++ no-sd; ++ disable-wp; ++ non-removable; ++ num-slots = <1>; ++ status = "okay"; ++}; ++ ++&gpu { ++ mali-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ ++ rk809: pmic@20 { ++ compatible = "rockchip,rk809"; ++ reg = <0x20>; ++ interrupt-parent = <&gpio0>; ++ interrupts = <7 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default", "pmic-sleep", ++ "pmic-power-off", "pmic-reset"; ++ pinctrl-0 = <&pmic_int>; ++ pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; ++ pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; ++ pinctrl-3 = <&soc_slppin_rst>, <&rk817_slppin_rst>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ pmic-reset-func = <1>; ++ ++ vcc1-supply = <&vcc5v0_sys>; ++ vcc2-supply = <&vcc5v0_sys>; ++ vcc3-supply = <&vcc5v0_sys>; ++ vcc4-supply = <&vcc5v0_sys>; ++ vcc5-supply = <&vcc3v3_sys>; ++ vcc6-supply = <&vcc3v3_sys>; ++ vcc7-supply = <&vcc3v3_sys>; ++ vcc8-supply = <&vcc3v3_sys>; ++ vcc9-supply = <&vcc5v0_sys>; ++ ++ pwrkey { ++ status = "okay"; ++ }; ++ ++ pinctrl_rk8xx: pinctrl_rk8xx { ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ rk817_slppin_null: rk817_slppin_null { ++ pins = "gpio_slp"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_slppin_slp: rk817_slppin_slp { ++ pins = "gpio_slp"; ++ function = "pin_fun1"; ++ }; ++ ++ rk817_slppin_pwrdn: rk817_slppin_pwrdn { ++ pins = "gpio_slp"; ++ function = "pin_fun2"; ++ }; ++ ++ rk817_slppin_rst: rk817_slppin_rst { ++ pins = "gpio_slp"; ++ function = "pin_fun3"; ++ }; ++ }; ++ ++ regulators { ++ vdd_logic: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <950000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_logic"; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <950000>; ++ }; ++ }; ++ ++ vdd_arm: DCDC_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <950000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_arm"; ++ ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <950000>; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc_ddr"; ++ regulator-initial-mode = <0x2>; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_3v0: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_3v0"; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ ++ vcc_1v0: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ regulator-name = "vcc_1v0"; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc1v8_soc: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc1v8_soc"; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd1v0_soc: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ regulator-name = "vcc1v0_soc"; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc3v0_pmu: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcc3v0_pmu"; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ ++ }; ++ }; ++ ++ vccio_sd: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vccio_sd"; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc_sd: LDO_REG6 { ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcc_sd"; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ ++ }; ++ }; ++ ++ vcc2v8_dvp: LDO_REG7 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <2800000>; ++ regulator-max-microvolt = <2800000>; ++ regulator-name = "vcc2v8_dvp"; ++ ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <2800000>; ++ }; ++ }; ++ ++ vcc1v8_dvp: LDO_REG8 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc1v8_dvp"; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd1v5_dvp: LDO_REG9 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ regulator-name = "vdd1v5_dvp"; ++ ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <1500000>; ++ }; ++ }; ++ ++ vcc3v3_sys: DCDC_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcc3v3_sys"; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc5v0_host: SWITCH_REG1 { ++ regulator-name = "vcc5v0_host"; ++ }; ++ ++ vcc3v3_lcd: SWITCH_REG2 { ++ regulator-boot-on; ++ regulator-name = "vcc3v3_lcd"; ++ ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ }; ++ }; ++}; ++ ++&io_domains { ++ vccio1-supply = <&vcc1v8_soc>; ++ vccio2-supply = <&vccio_sd>; ++ vccio3-supply = <&vcc_3v0>; ++ vccio4-supply = <&vcc3v0_pmu>; ++ vccio5-supply = <&vcc_3v0>; ++ status = "okay"; ++}; ++ ++&nandc0 { ++ status = "okay"; ++}; ++ ++&pmu_io_domains { ++ pmuio1-supply = <&vcc3v0_pmu>; ++ pmuio2-supply = <&vcc3v0_pmu>; ++ status = "okay"; ++}; ++ ++&pwm1 { ++ status = "okay"; ++}; ++ ++&rk_rga { ++ status = "okay"; ++}; ++ ++&rockchip_suspend { ++ rockchip,sleep-debug-en = <1>; ++ status = "okay"; ++}; ++ ++&saradc { ++ vref-supply = <&vcc1v8_soc>; ++ status = "okay"; ++}; ++ ++&sdmmc { ++ bus-width = <4>; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ no-sdio; ++ no-mmc; ++ card-detect-delay = <800>; ++ ignore-pm-notify; ++ sd-uhs-sdr12; ++ sd-uhs-sdr25; ++ sd-uhs-sdr50; ++ sd-uhs-sdr104; ++ vqmmc-supply = <&vccio_sd>; ++ vmmc-supply = <&vcc_sd>; ++ status = "okay"; ++}; ++ ++&sdio { ++ bus-width = <4>; ++ cap-sd-highspeed; ++ no-sd; ++ no-mmc; ++ ignore-pm-notify; ++ keep-power-in-suspend; ++ non-removable; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&tsadc { ++ pinctrl-names = "init", "default"; ++ pinctrl-0 = <&tsadc_otp_gpio>; ++ pinctrl-1 = <&tsadc_otp_out>; ++ status = "okay"; ++}; ++ ++&uart1 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart1_xfer &uart1_cts>; ++ status = "okay"; ++}; ++ ++&u2phy { ++ status = "okay"; ++ ++ u2phy_host: host-port { ++ status = "okay"; ++ }; ++ ++ u2phy_otg: otg-port { ++ status = "okay"; ++ }; ++}; ++ ++&usb20_otg { ++ status = "okay"; ++}; ++ ++&usb_host0_ehci { ++ status = "okay"; ++}; ++ ++&usb_host0_ohci { ++ status = "okay"; ++}; ++ ++&display_subsystem { ++ status = "okay"; ++}; ++ ++&vopb { ++ status = "okay"; ++}; ++ ++&vopb_mmu { ++ status = "okay"; ++}; ++ ++&vopl { ++ status = "okay"; ++}; ++ ++&vopl_mmu { ++ status = "okay"; ++}; ++ ++&mpp_srv { ++ status = "okay"; ++}; ++ ++&vdpu { ++ status = "okay"; ++}; ++ ++&vepu { ++ status = "okay"; ++}; ++ ++&vpu_mmu { ++ status = "okay"; ++}; ++ ++&hevc { ++ status = "okay"; ++}; ++ ++&hevc_mmu { ++ status = "okay"; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ ++ anx6345@38 { ++ compatible = "analogix,anx6345"; ++ reg = <0x38>; ++ reset-gpios = <&gpio1 RK_PB0 GPIO_ACTIVE_LOW>; ++ panel-supply = <&vcc3v3_lcd>; ++ dvdd25-supply = <&vcc3v3_lcd>; ++ dvdd12-supply = <&dvdd12_anx>; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ ++ anx6345_in_rgb: endpoint { ++ remote-endpoint = <&rgb_out_anx6345>; ++ }; ++ }; ++ }; ++ }; ++}; ++ ++&rgb { ++ pinctrl-names = "default", "sleep"; ++ pinctrl-0 = <&lcdc_rgb_pins>; ++ pinctrl-1 = <&lcdc_sleep_pins>; ++ status = "okay"; ++ ++ ports { ++ port@1 { ++ reg = <1>; ++ ++ rgb_out_anx6345: endpoint { ++ remote-endpoint = <&anx6345_in_rgb>; ++ }; ++ }; ++ }; ++}; ++ ++&rgb_in_vopb { ++ status = "okay"; ++}; ++ ++&rgb_in_vopl { ++ status = "disabled"; ++}; ++ ++&route_rgb { ++ connect = <&vopb_out_rgb>; ++ status = "okay"; ++}; ++ ++&pinctrl { ++ lcdc { ++ lcdc_rgb_pins: lcdc-rgb-pins { ++ rockchip,pins = ++ <3 RK_PA0 1 &pcfg_pull_none_8ma>, /* LCDC_DCLK */ ++ <3 RK_PA1 1 &pcfg_pull_none_8ma>, /* LCDC_HSYNC */ ++ <3 RK_PA2 1 &pcfg_pull_none_8ma>, /* LCDC_VSYNC */ ++ <3 RK_PA3 1 &pcfg_pull_none_8ma>, /* LCDC_DEN */ ++ <3 RK_PD3 1 &pcfg_pull_none_8ma>, /* LCDC_D23 */ ++ <3 RK_PD2 1 &pcfg_pull_none_8ma>, /* LCDC_D22 */ ++ <3 RK_PD1 1 &pcfg_pull_none_8ma>, /* LCDC_D21 */ ++ <3 RK_PD0 1 &pcfg_pull_none_8ma>, /* LCDC_D20 */ ++ <3 RK_PC7 1 &pcfg_pull_none_8ma>, /* LCDC_D19 */ ++ <3 RK_PC6 1 &pcfg_pull_none_8ma>, /* LCDC_D18 */ ++ <3 RK_PC5 1 &pcfg_pull_none_8ma>, /* LCDC_D17 */ ++ <3 RK_PC4 1 &pcfg_pull_none_8ma>, /* LCDC_D16 */ ++ <3 RK_PC3 1 &pcfg_pull_none_8ma>, /* LCDC_D15 */ ++ <3 RK_PC2 1 &pcfg_pull_none_8ma>, /* LCDC_D14 */ ++ <3 RK_PC1 1 &pcfg_pull_none_8ma>, /* LCDC_D13 */ ++ <3 RK_PC0 1 &pcfg_pull_none_8ma>, /* LCDC_D12 */ ++ <3 RK_PB7 1 &pcfg_pull_none_8ma>, /* LCDC_D11 */ ++ <3 RK_PB6 1 &pcfg_pull_none_8ma>, /* LCDC_D10 */ ++ <3 RK_PB5 1 &pcfg_pull_none_8ma>, /* LCDC_D9 */ ++ <3 RK_PB4 1 &pcfg_pull_none_8ma>, /* LCDC_D8 */ ++ <3 RK_PB3 1 &pcfg_pull_none_8ma>, /* LCDC_D7 */ ++ <3 RK_PB2 1 &pcfg_pull_none_8ma>, /* LCDC_D6 */ ++ <3 RK_PB1 1 &pcfg_pull_none_8ma>, /* LCDC_D5 */ ++ <3 RK_PB0 1 &pcfg_pull_none_8ma>, /* LCDC_D4 */ ++ <3 RK_PA7 1 &pcfg_pull_none_8ma>, /* LCDC_D3 */ ++ <3 RK_PA6 1 &pcfg_pull_none_8ma>, /* LCDC_D2 */ ++ <3 RK_PA5 1 &pcfg_pull_none_8ma>, /* LCDC_D1 */ ++ <3 RK_PA4 1 &pcfg_pull_none_8ma>; /* LCDC_D0 */ ++ }; ++ ++ lcdc_sleep_pins: lcdc-sleep-pins { ++ rockchip,pins = ++ <3 RK_PA0 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_DCLK */ ++ <3 RK_PA1 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_HSYNC */ ++ <3 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_VSYNC */ ++ <3 RK_PA3 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_DEN */ ++ <3 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D23 */ ++ <3 RK_PD2 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D22 */ ++ <3 RK_PD1 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D21 */ ++ <3 RK_PD0 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D20 */ ++ <3 RK_PC7 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D19 */ ++ <3 RK_PC6 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D18 */ ++ <3 RK_PC5 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D17 */ ++ <3 RK_PC4 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D16 */ ++ <3 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D15 */ ++ <3 RK_PC2 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D14 */ ++ <3 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D13 */ ++ <3 RK_PC0 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D12 */ ++ <3 RK_PB7 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D11 */ ++ <3 RK_PB6 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D10 */ ++ <3 RK_PB5 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D9 */ ++ <3 RK_PB4 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D8 */ ++ <3 RK_PB3 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D7 */ ++ <3 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D6 */ ++ <3 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D5 */ ++ <3 RK_PB0 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D4 */ ++ <3 RK_PA7 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D3 */ ++ <3 RK_PA6 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D2 */ ++ <3 RK_PA5 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D1 */ ++ <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; /* LCDC_D0 */ ++ }; ++ }; ++ ++ pmic { ++ pmic_int: pmic_int { ++ rockchip,pins = ++ <0 RK_PA7 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ soc_slppin_gpio: soc_slppin_gpio { ++ rockchip,pins = ++ <0 RK_PA4 RK_FUNC_GPIO &pcfg_output_low>; ++ }; ++ ++ soc_slppin_slp: soc_slppin_slp { ++ rockchip,pins = ++ <0 RK_PA4 1 &pcfg_pull_none>; ++ }; ++ ++ soc_slppin_rst: soc_slppin_rst { ++ rockchip,pins = ++ <0 RK_PA4 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++&firmware_android { ++ compatible = "android,firmware"; ++ ++ fstab { ++ compatible = "android,fstab"; ++ ++ system { ++ compatible = "android,system"; ++ dev = "/dev/block/by-name/system"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait"; ++ }; ++ ++ vendor { ++ compatible = "android,vendor"; ++ dev = "/dev/block/by-name/vendor"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait"; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/px30-ad-r35-mb-rk618-dual-lvds.dts b/arch/arm64/boot/dts/rockchip/px30-ad-r35-mb-rk618-dual-lvds.dts +new file mode 100755 +index 000000000000..09fc265a8d7a +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/px30-ad-r35-mb-rk618-dual-lvds.dts +@@ -0,0 +1,147 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++#include ++#include "px30-ad-r35-mb.dtsi" ++ ++/ { ++ panel { ++ compatible = "lg,lm215wf3", "simple-panel"; ++ backlight = <&backlight>; ++ power-supply = <&vcc3v3_lcd>; ++ enable-gpios = <&gpio0 RK_PB5 GPIO_ACTIVE_LOW>; ++ prepare-delay-ms = <120>; ++ enable-delay-ms = <120>; ++ disable-delay-ms = <120>; ++ unprepare-delay-ms = <120>; ++ bus-format = ; ++ width-mm = <476>; ++ height-mm = <268>; ++ ++ display-timings { ++ native-mode = <&timing1>; ++ ++ timing1: timing1 { ++ clock-frequency = <144000000>; ++ hactive = <1920>; ++ vactive = <1080>; ++ hback-porch = <96>; ++ hfront-porch = <96>; ++ vback-porch = <8>; ++ vfront-porch = <8>; ++ hsync-len = <64>; ++ vsync-len = <4>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ port { ++ panel_in_lvds: endpoint { ++ remote-endpoint = <&lvds_out_panel>; ++ }; ++ }; ++ }; ++}; ++ ++&dmc { ++ auto-freq-en = <0>; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ ++ rk618@50 { ++ compatible = "rockchip,rk618"; ++ reg = <0x50>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1_2ch_mclk>; ++ clocks = <&cru SCLK_I2S1_OUT>; ++ clock-names = "clkin"; ++ assigned-clocks = <&cru SCLK_I2S1_OUT>; ++ assigned-clock-rates = <11289600>; ++ reset-gpios = <&gpio0 RK_PA0 GPIO_ACTIVE_LOW>; ++ status = "okay"; ++ ++ clock: cru { ++ compatible = "rockchip,rk618-cru"; ++ clocks = <&cru SCLK_I2S1_OUT>, <&cru DCLK_VOPL>; ++ clock-names = "clkin", "lcdc0_dclkp"; ++ assigned-clocks = <&clock SCALER_PLLIN_CLK>, ++ <&clock VIF_PLLIN_CLK>, ++ <&clock SCALER_CLK>, ++ <&clock VIF0_PRE_CLK>, ++ <&clock CODEC_CLK>, ++ <&clock DITHER_CLK>; ++ assigned-clock-parents = <&cru SCLK_I2S1_OUT>, ++ <&clock LCDC0_CLK>, ++ <&clock SCALER_PLL_CLK>, ++ <&clock VIF_PLL_CLK>, ++ <&cru SCLK_I2S1_OUT>, ++ <&clock VIF0_CLK>; ++ #clock-cells = <1>; ++ status = "okay"; ++ }; ++ ++ lvds { ++ compatible = "rockchip,rk618-lvds"; ++ clocks = <&clock LVDS_CLK>; ++ clock-names = "lvds"; ++ dual-channel; ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ ++ lvds_in_rgb: endpoint { ++ remote-endpoint = <&rgb_out_lvds>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ ++ lvds_out_panel: endpoint { ++ remote-endpoint = <&panel_in_lvds>; ++ }; ++ }; ++ }; ++ }; ++ }; ++}; ++ ++&rgb { ++ status = "okay"; ++ ++ ports { ++ port@1 { ++ reg = <1>; ++ ++ rgb_out_lvds: endpoint { ++ remote-endpoint = <&lvds_in_rgb>; ++ }; ++ }; ++ }; ++}; ++ ++&rgb_in_vopb { ++ status = "disabled"; ++}; ++ ++&rgb_in_vopl { ++ status = "okay"; ++}; ++ ++&route_rgb { ++ connect = <&vopl_out_rgb>; ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/px30-ad-r35-mb-rk618-hdmi-lvds.dts b/arch/arm64/boot/dts/rockchip/px30-ad-r35-mb-rk618-hdmi-lvds.dts +new file mode 100755 +index 000000000000..d4f202189250 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/px30-ad-r35-mb-rk618-hdmi-lvds.dts +@@ -0,0 +1,241 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++#include ++#include ++#include "px30-ad-r35-mb.dtsi" ++ ++/ { ++ panel { ++ compatible = "simple-panel"; ++ backlight = <&backlight>; ++ power-supply = <&vcc3v3_lcd>; ++ enable-gpios = <&gpio0 RK_PB5 GPIO_ACTIVE_LOW>; ++ prepare-delay-ms = <120>; ++ enable-delay-ms = <120>; ++ disable-delay-ms = <120>; ++ unprepare-delay-ms = <120>; ++ bus-format = ; ++ width-mm = <231>; ++ height-mm = <154>; ++ ++ display-timings { ++ native-mode = <&timing1>; ++ ++ timing1: timing1 { ++ clock-frequency = <72000000>; ++ hactive = <1280>; ++ vactive = <800>; ++ hback-porch = <60>; ++ hfront-porch = <60>; ++ vback-porch = <16>; ++ vfront-porch = <16>; ++ hsync-len = <40>; ++ vsync-len = <6>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ port { ++ panel_in_lvds: endpoint { ++ remote-endpoint = <&lvds_out_panel>; ++ }; ++ }; ++ }; ++}; ++ ++&dmc { ++ auto-freq-en = <0>; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ ++ rk618@50 { ++ compatible = "rockchip,rk618"; ++ reg = <0x50>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1_2ch_mclk>; ++ clocks = <&cru SCLK_I2S1_OUT>; ++ clock-names = "clkin"; ++ assigned-clocks = <&cru SCLK_I2S1_OUT>; ++ assigned-clock-rates = <11289600>; ++ reset-gpios = <&gpio0 RK_PA0 GPIO_ACTIVE_LOW>; ++ status = "okay"; ++ ++ clock: cru { ++ compatible = "rockchip,rk618-cru"; ++ clocks = <&cru SCLK_I2S1_OUT>, <&cru DCLK_VOPL>; ++ clock-names = "clkin", "lcdc0_dclkp"; ++ assigned-clocks = <&clock SCALER_PLLIN_CLK>, ++ <&clock VIF_PLLIN_CLK>, ++ <&clock SCALER_CLK>, ++ <&clock VIF0_PRE_CLK>, ++ <&clock CODEC_CLK>, ++ <&clock DITHER_CLK>; ++ assigned-clock-parents = <&cru SCLK_I2S1_OUT>, ++ <&clock LCDC0_CLK>, ++ <&clock SCALER_PLL_CLK>, ++ <&clock VIF_PLL_CLK>, ++ <&cru SCLK_I2S1_OUT>, ++ <&clock VIF0_CLK>; ++ #clock-cells = <1>; ++ status = "okay"; ++ }; ++ ++ hdmi { ++ compatible = "rockchip,rk618-hdmi"; ++ clocks = <&clock HDMI_CLK>; ++ clock-names = "hdmi"; ++ assigned-clocks = <&clock HDMI_CLK>; ++ assigned-clock-parents = <&clock VIF0_CLK>; ++ interrupt-parent = <&gpio2>; ++ interrupts = <12 IRQ_TYPE_LEVEL_HIGH>; ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ ++ hdmi_in_vif: endpoint { ++ remote-endpoint = <&vif_out_hdmi>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ ++ hdmi_out_scaler: endpoint { ++ remote-endpoint = <&scaler_in_hdmi>; ++ }; ++ }; ++ }; ++ }; ++ ++ lvds { ++ compatible = "rockchip,rk618-lvds"; ++ clocks = <&clock LVDS_CLK>; ++ clock-names = "lvds"; ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ ++ lvds_in_scaler: endpoint { ++ remote-endpoint = <&scaler_out_lvds>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ ++ lvds_out_panel: endpoint { ++ remote-endpoint = <&panel_in_lvds>; ++ }; ++ }; ++ }; ++ }; ++ ++ scaler { ++ compatible = "rockchip,rk618-scaler"; ++ clocks = <&clock SCALER_CLK>, <&clock VIF0_CLK>, ++ <&clock DITHER_CLK>; ++ clock-names = "scaler", "vif", "dither"; ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ ++ scaler_in_hdmi: endpoint { ++ remote-endpoint = <&hdmi_out_scaler>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ ++ scaler_out_lvds: endpoint { ++ remote-endpoint = <&lvds_in_scaler>; ++ }; ++ }; ++ }; ++ }; ++ ++ vif { ++ compatible = "rockchip,rk618-vif"; ++ clocks = <&clock VIF0_CLK>, <&clock VIF0_PRE_CLK>; ++ clock-names = "vif", "vif_pre"; ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ ++ vif_in_rgb: endpoint { ++ remote-endpoint = <&rgb_out_vif>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ ++ vif_out_hdmi: endpoint { ++ remote-endpoint = <&hdmi_in_vif>; ++ }; ++ }; ++ }; ++ }; ++ }; ++}; ++ ++&vopl { ++ assigned-clocks = <&cru PLL_NPLL>; ++ assigned-clock-rates = <1188000000>; ++}; ++ ++&rgb { ++ status = "okay"; ++ ++ ports { ++ port@1 { ++ reg = <1>; ++ ++ rgb_out_vif: endpoint { ++ remote-endpoint = <&vif_in_rgb>; ++ }; ++ }; ++ }; ++}; ++ ++&rgb_in_vopb { ++ status = "disabled"; ++}; ++ ++&rgb_in_vopl { ++ status = "okay"; ++}; ++ ++&route_rgb { ++ connect = <&vopl_out_rgb>; ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/px30-ad-r35-mb-rk618-hdmi.dts b/arch/arm64/boot/dts/rockchip/px30-ad-r35-mb-rk618-hdmi.dts +new file mode 100755 +index 000000000000..0ea056682e57 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/px30-ad-r35-mb-rk618-hdmi.dts +@@ -0,0 +1,105 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++#include ++#include "px30-ad-r35-mb.dtsi" ++ ++&dmc { ++ auto-freq-en = <0>; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ ++ rk618@50 { ++ compatible = "rockchip,rk618"; ++ reg = <0x50>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1_2ch_mclk>; ++ clocks = <&cru SCLK_I2S1_OUT>; ++ clock-names = "clkin"; ++ assigned-clocks = <&cru SCLK_I2S1_OUT>; ++ assigned-clock-rates = <11289600>; ++ reset-gpios = <&gpio0 RK_PA0 GPIO_ACTIVE_LOW>; ++ status = "okay"; ++ ++ clock: cru { ++ compatible = "rockchip,rk618-cru"; ++ clocks = <&cru SCLK_I2S1_OUT>, <&cru DCLK_VOPL>; ++ clock-names = "clkin", "lcdc0_dclkp"; ++ assigned-clocks = <&clock SCALER_PLLIN_CLK>, ++ <&clock VIF_PLLIN_CLK>, ++ <&clock SCALER_CLK>, ++ <&clock VIF0_PRE_CLK>, ++ <&clock CODEC_CLK>, ++ <&clock DITHER_CLK>; ++ assigned-clock-parents = <&cru SCLK_I2S1_OUT>, ++ <&clock LCDC0_CLK>, ++ <&clock SCALER_PLL_CLK>, ++ <&clock VIF_PLL_CLK>, ++ <&cru SCLK_I2S1_OUT>, ++ <&clock VIF0_CLK>; ++ #clock-cells = <1>; ++ status = "okay"; ++ }; ++ ++ hdmi { ++ compatible = "rockchip,rk618-hdmi"; ++ clocks = <&clock HDMI_CLK>; ++ clock-names = "hdmi"; ++ assigned-clocks = <&clock HDMI_CLK>; ++ assigned-clock-parents = <&clock VIF0_CLK>; ++ interrupt-parent = <&gpio2>; ++ interrupts = <12 IRQ_TYPE_LEVEL_HIGH>; ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ ++ hdmi_in_rgb: endpoint { ++ remote-endpoint = <&rgb_out_hdmi>; ++ }; ++ }; ++ }; ++ }; ++ }; ++}; ++ ++&vopl { ++ assigned-clocks = <&cru PLL_NPLL>; ++ assigned-clock-rates = <1188000000>; ++}; ++ ++&rgb { ++ status = "okay"; ++ ++ ports { ++ port@1 { ++ reg = <1>; ++ ++ rgb_out_hdmi: endpoint { ++ remote-endpoint = <&hdmi_in_rgb>; ++ }; ++ }; ++ }; ++}; ++ ++&rgb_in_vopb { ++ status = "disabled"; ++}; ++ ++&rgb_in_vopl { ++ status = "okay"; ++}; ++ ++&route_rgb { ++ connect = <&vopl_out_rgb>; ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/px30-ad-r35-mb-rk618-lvds.dts b/arch/arm64/boot/dts/rockchip/px30-ad-r35-mb-rk618-lvds.dts +new file mode 100755 +index 000000000000..8b54a9a18fcc +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/px30-ad-r35-mb-rk618-lvds.dts +@@ -0,0 +1,146 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++#include ++#include "px30-ad-r35-mb.dtsi" ++ ++/ { ++ panel { ++ compatible = "chunghwa,claa101wh31-cw", "simple-panel"; ++ backlight = <&backlight>; ++ power-supply = <&vcc3v3_lcd>; ++ enable-gpios = <&gpio0 RK_PB5 GPIO_ACTIVE_LOW>; ++ prepare-delay-ms = <120>; ++ enable-delay-ms = <120>; ++ disable-delay-ms = <120>; ++ unprepare-delay-ms = <120>; ++ bus-format = ; ++ width-mm = <231>; ++ height-mm = <154>; ++ ++ display-timings { ++ native-mode = <&timing1>; ++ ++ timing1: timing1 { ++ clock-frequency = <72000000>; ++ hactive = <1280>; ++ vactive = <800>; ++ hback-porch = <60>; ++ hfront-porch = <60>; ++ vback-porch = <16>; ++ vfront-porch = <16>; ++ hsync-len = <40>; ++ vsync-len = <6>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ port { ++ panel_in_lvds: endpoint { ++ remote-endpoint = <&lvds_out_panel>; ++ }; ++ }; ++ }; ++}; ++ ++&dmc { ++ auto-freq-en = <0>; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ ++ rk618@50 { ++ compatible = "rockchip,rk618"; ++ reg = <0x50>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1_2ch_mclk>; ++ clocks = <&cru SCLK_I2S1_OUT>; ++ clock-names = "clkin"; ++ assigned-clocks = <&cru SCLK_I2S1_OUT>; ++ assigned-clock-rates = <12000000>; ++ reset-gpios = <&gpio0 RK_PA0 GPIO_ACTIVE_LOW>; ++ status = "okay"; ++ ++ clock: cru { ++ compatible = "rockchip,rk618-cru"; ++ clocks = <&cru SCLK_I2S1_OUT>, <&cru DCLK_VOPL>; ++ clock-names = "clkin", "lcdc0_dclkp"; ++ assigned-clocks = <&clock SCALER_PLLIN_CLK>, ++ <&clock VIF_PLLIN_CLK>, ++ <&clock SCALER_CLK>, ++ <&clock VIF0_PRE_CLK>, ++ <&clock CODEC_CLK>, ++ <&clock DITHER_CLK>; ++ assigned-clock-parents = <&cru SCLK_I2S1_OUT>, ++ <&clock LCDC0_CLK>, ++ <&clock SCALER_PLL_CLK>, ++ <&clock VIF_PLL_CLK>, ++ <&cru SCLK_I2S1_OUT>, ++ <&clock VIF0_CLK>; ++ #clock-cells = <1>; ++ status = "okay"; ++ }; ++ ++ lvds { ++ compatible = "rockchip,rk618-lvds"; ++ clocks = <&clock LVDS_CLK>; ++ clock-names = "lvds"; ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ ++ lvds_in_rgb: endpoint { ++ remote-endpoint = <&rgb_out_lvds>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ ++ lvds_out_panel: endpoint { ++ remote-endpoint = <&panel_in_lvds>; ++ }; ++ }; ++ }; ++ }; ++ }; ++}; ++ ++&rgb { ++ status = "okay"; ++ ++ ports { ++ port@1 { ++ reg = <1>; ++ ++ rgb_out_lvds: endpoint { ++ remote-endpoint = <&lvds_in_rgb>; ++ }; ++ }; ++ }; ++}; ++ ++&rgb_in_vopb { ++ status = "disabled"; ++}; ++ ++&rgb_in_vopl { ++ status = "okay"; ++}; ++ ++&route_rgb { ++ connect = <&vopl_out_rgb>; ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/px30-ad-r35-mb.dtsi b/arch/arm64/boot/dts/rockchip/px30-ad-r35-mb.dtsi +new file mode 100755 +index 000000000000..bbcd18959dc4 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/px30-ad-r35-mb.dtsi +@@ -0,0 +1,823 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include "px30.dtsi" ++#include "px30-android.dtsi" ++ ++/ { ++ model = "Rockchip PX30 AD R35 MB board"; ++ compatible = "rockchip,px30-ad-r35-mb", "rockchip,px30"; ++ ++ adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 2>; ++ io-channel-names = "buttons"; ++ poll-interval = <100>; ++ keyup-threshold-microvolt = <1800000>; ++ ++ esc-key { ++ linux,code = ; ++ label = "esc"; ++ press-threshold-microvolt = <1270000>; ++ }; ++ ++ home-key { ++ linux,code = ; ++ label = "home"; ++ press-threshold-microvolt = <602000>; ++ }; ++ ++ menu-key { ++ linux,code = ; ++ label = "menu"; ++ press-threshold-microvolt = <952000>; ++ }; ++ ++ vol-down-key { ++ linux,code = ; ++ label = "volume down"; ++ press-threshold-microvolt = <290000>; ++ }; ++ ++ vol-up-key { ++ linux,code = ; ++ label = "volume up"; ++ press-threshold-microvolt = <17000>; ++ }; ++ }; ++ ++ backlight: backlight { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm1 0 25000 0>; ++ brightness-levels = < ++ 0 1 2 3 4 5 6 7 ++ 8 9 10 11 12 13 14 15 ++ 16 17 18 19 20 21 22 23 ++ 24 25 26 27 28 29 30 31 ++ 32 33 34 35 36 37 38 39 ++ 40 41 42 43 44 45 46 47 ++ 48 49 50 51 52 53 54 55 ++ 56 57 58 59 60 61 62 63 ++ 64 65 66 67 68 69 70 71 ++ 72 73 74 75 76 77 78 79 ++ 80 81 82 83 84 85 86 87 ++ 88 89 90 91 92 93 94 95 ++ 96 97 98 99 100 101 102 103 ++ 104 105 106 107 108 109 110 111 ++ 112 113 114 115 116 117 118 119 ++ 120 121 122 123 124 125 126 127 ++ 128 129 130 131 132 133 134 135 ++ 136 137 138 139 140 141 142 143 ++ 144 145 146 147 148 149 150 151 ++ 152 153 154 155 156 157 158 159 ++ 160 161 162 163 164 165 166 167 ++ 168 169 170 171 172 173 174 175 ++ 176 177 178 179 180 181 182 183 ++ 184 185 186 187 188 189 190 191 ++ 192 193 194 195 196 197 198 199 ++ 200 201 202 203 204 205 206 207 ++ 208 209 210 211 212 213 214 215 ++ 216 217 218 219 220 221 222 223 ++ 224 225 226 227 228 229 230 231 ++ 232 233 234 235 236 237 238 239 ++ 240 241 242 243 244 245 246 247 ++ 248 249 250 251 252 253 254 255>; ++ default-brightness-level = <200>; ++ enable-gpios = <&gpio0 RK_PB7 GPIO_ACTIVE_HIGH>; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_LOW>; ++ }; ++ ++ vcc_phy: vcc-phy-regulator { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_phy"; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ vcc5v0_sys: vccsys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc5v0_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ }; ++}; ++ ++&cpu0 { ++ cpu-supply = <&vdd_arm>; ++}; ++ ++&display_subsystem { ++ status = "okay"; ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&dmc { ++ center-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&emmc { ++ bus-width = <8>; ++ cap-mmc-highspeed; ++ mmc-hs200-1_8v; ++ no-sdio; ++ no-sd; ++ disable-wp; ++ non-removable; ++ num-slots = <1>; ++ status = "okay"; ++}; ++ ++&gpu { ++ mali-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ ++ rk809: pmic@20 { ++ compatible = "rockchip,rk809"; ++ reg = <0x20>; ++ interrupt-parent = <&gpio0>; ++ interrupts = <7 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pmic_int>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ ++ vcc1-supply = <&vcc5v0_sys>; ++ vcc2-supply = <&vcc5v0_sys>; ++ vcc3-supply = <&vcc5v0_sys>; ++ vcc4-supply = <&vcc5v0_sys>; ++ vcc5-supply = <&vcc3v3_sys>; ++ vcc6-supply = <&vcc3v3_sys>; ++ vcc7-supply = <&vcc3v3_sys>; ++ vcc8-supply = <&vcc3v3_sys>; ++ vcc9-supply = <&vcc5v0_sys>; ++ ++ regulators { ++ vdd_logic: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <950000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x1>; ++ regulator-name = "vdd_logic"; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <950000>; ++ }; ++ }; ++ ++ vdd_arm: DCDC_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <950000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x1>; ++ regulator-name = "vdd_arm"; ++ ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <950000>; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc_ddr"; ++ regulator-initial-mode = <0x1>; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_3v0: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-initial-mode = <0x1>; ++ regulator-name = "vcc_3v0"; ++ ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ ++ vcc_1v0: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ regulator-name = "vcc_1v0"; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc1v8_soc: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc1v8_soc"; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd1v0_soc: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ regulator-name = "vcc1v0_soc"; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc3v0_pmu: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcc3v0_pmu"; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ ++ }; ++ }; ++ ++ vccio_sd: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vccio_sd"; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc_sd: LDO_REG6 { ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcc_sd"; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ ++ }; ++ }; ++ ++ vcc2v8_dvp: LDO_REG7 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <2800000>; ++ regulator-max-microvolt = <2800000>; ++ regulator-name = "vcc2v8_dvp"; ++ ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <2800000>; ++ }; ++ }; ++ ++ vcc1v8_dvp: LDO_REG8 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc1v8_dvp"; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd1v5_dvp: LDO_REG9 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ regulator-name = "vdd1v5_dvp"; ++ ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <1500000>; ++ }; ++ }; ++ ++ vcc3v3_sys: DCDC_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcc3v3_sys"; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc3v3_lcd: SWITCH_REG1 { ++ regulator-boot-on; ++ regulator-name = "vcc3v3_lcd"; ++ }; ++ ++ vcc5v0_host: SWITCH_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc5v0_host"; ++ }; ++ }; ++ }; ++}; ++ ++&io_domains { ++ vccio1-supply = <&vcc1v8_soc>; ++ vccio2-supply = <&vccio_sd>; ++ vccio3-supply = <&vcc_3v0>; ++ vccio4-supply = <&vcc3v0_pmu>; ++ vccio5-supply = <&vcc_3v0>; ++ status = "okay"; ++}; ++ ++&nandc0 { ++ status = "okay"; ++}; ++ ++&pmu_io_domains { ++ pmuio1-supply = <&vcc3v0_pmu>; ++ pmuio2-supply = <&vcc3v0_pmu>; ++ status = "okay"; ++}; ++ ++&pwm1 { ++ status = "okay"; ++}; ++ ++&saradc { ++ vref-supply = <&vcc1v8_soc>; ++ status = "okay"; ++}; ++ ++&sdmmc { ++ bus-width = <4>; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ no-sdio; ++ no-mmc; ++ card-detect-delay = <800>; ++ ignore-pm-notify; ++ sd-uhs-sdr12; ++ sd-uhs-sdr25; ++ sd-uhs-sdr50; ++ sd-uhs-sdr104; ++ vqmmc-supply = <&vccio_sd>; ++ vmmc-supply = <&vcc_sd>; ++ status = "okay"; ++}; ++ ++&sdio { ++ bus-width = <4>; ++ cap-sd-highspeed; ++ no-sd; ++ no-mmc; ++ ignore-pm-notify; ++ keep-power-in-suspend; ++ non-removable; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&tsadc { ++ pinctrl-names = "init", "default"; ++ pinctrl-0 = <&tsadc_otp_gpio>; ++ pinctrl-1 = <&tsadc_otp_out>; ++ status = "okay"; ++}; ++ ++&uart1 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart1_xfer &uart1_cts>; ++ status = "okay"; ++}; ++ ++&u2phy { ++ status = "okay"; ++ ++ u2phy_host: host-port { ++ status = "okay"; ++ }; ++ ++ u2phy_otg: otg-port { ++ status = "okay"; ++ }; ++}; ++ ++&usb20_otg { ++ status = "okay"; ++}; ++ ++&usb_host0_ehci { ++ status = "okay"; ++}; ++ ++&usb_host0_ohci { ++ status = "okay"; ++}; ++ ++&vopb { ++ status = "okay"; ++}; ++ ++&vopb_mmu { ++ status = "okay"; ++}; ++ ++&vopl { ++ status = "okay"; ++}; ++ ++&vopl_mmu { ++ status = "okay"; ++}; ++ ++&mpp_srv { ++ status = "okay"; ++}; ++ ++&vdpu { ++ status = "okay"; ++}; ++ ++&vepu { ++ status = "okay"; ++}; ++ ++&vpu_mmu { ++ status = "okay"; ++}; ++ ++&hevc { ++ status = "okay"; ++}; ++ ++&hevc_mmu { ++ status = "okay"; ++}; ++ ++&dsi { ++ status = "okay"; ++ ++ panel@0 { ++ compatible = "pvo,p101nwwbp-01g", "simple-panel-dsi"; ++ reg = <0>; ++ power-supply = <&vcc3v3_lcd>; ++ backlight = <&backlight>; ++ prepare-delay-ms = <20>; ++ reset-delay-ms = <20>; ++ init-delay-ms = <20>; ++ enable-delay-ms = <20>; ++ disable-delay-ms = <20>; ++ unprepare-delay-ms = <20>; ++ ++ width-mm = <135>; ++ height-mm = <216>; ++ ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | ++ MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ ++ panel-init-sequence = [ ++ 15 00 02 E0 00 ++ 15 00 02 E1 93 ++ 15 00 02 E2 65 ++ 15 00 02 E3 F8 ++ 15 00 02 80 03 ++ 15 00 02 E0 04 ++ 15 00 02 2B 2B ++ 15 00 02 2D 03 ++ 15 00 02 2E 44 ++ 15 00 02 E0 01 ++ 15 00 02 00 00 ++ 15 00 02 01 6D ++ 15 00 02 0C 74 ++ 15 00 02 17 00 ++ 15 00 02 18 A7 ++ 15 00 02 19 01 ++ 15 00 02 1A 00 ++ 15 00 02 1B A7 ++ 15 00 02 1C 01 ++ 15 00 02 1F 6A ++ 15 00 02 20 23 ++ 15 00 02 21 23 ++ 15 00 02 22 0E ++ 15 00 02 35 28 ++ 15 00 02 37 59 ++ 15 00 02 38 05 ++ 15 00 02 39 04 ++ 15 00 02 3A 08 ++ 15 00 02 3B 08 ++ 15 00 02 3C 7C ++ 15 00 02 3D FF ++ 15 00 02 3E FF ++ 15 00 02 3F FF ++ 15 00 02 40 06 ++ 15 00 02 41 A0 ++ 15 00 02 43 08 ++ 15 00 02 44 0B ++ 15 00 02 45 88 ++ 15 00 02 4B 04 ++ 15 00 02 55 01 ++ 15 00 02 56 01 ++ 15 00 02 57 8D ++ 15 00 02 58 0A ++ 15 00 02 59 0A ++ 15 00 02 5A 28 ++ 15 00 02 5B 1E ++ 15 00 02 5C 16 ++ 15 00 02 5D 76 ++ 15 00 02 5E 58 ++ 15 00 02 5F 46 ++ 15 00 02 60 39 ++ 15 00 02 61 37 ++ 15 00 02 62 2A ++ 15 00 02 63 2F ++ 15 00 02 64 18 ++ 15 00 02 65 39 ++ 15 00 02 66 38 ++ 15 00 02 67 3A ++ 15 00 02 68 5A ++ 15 00 02 69 46 ++ 15 00 02 6A 4C ++ 15 00 02 6B 3F ++ 15 00 02 6C 3D ++ 15 00 02 6D 2F ++ 15 00 02 6E 1E ++ 15 00 02 6F 00 ++ 15 00 02 70 76 ++ 15 00 02 71 58 ++ 15 00 02 72 46 ++ 15 00 02 73 39 ++ 15 00 02 74 33 ++ 15 00 02 75 22 ++ 15 00 02 76 27 ++ 15 00 02 77 14 ++ 15 00 02 78 29 ++ 15 00 02 79 2A ++ 15 00 02 7A 28 ++ 15 00 02 7B 46 ++ 15 00 02 7C 38 ++ 15 00 02 7D 3E ++ 15 00 02 7E 31 ++ 15 00 02 7F 29 ++ 15 00 02 80 1B ++ 15 00 02 81 0A ++ 15 00 02 82 00 ++ 15 00 02 E0 02 ++ 15 00 02 00 44 ++ 15 00 02 01 44 ++ 15 00 02 02 45 ++ 15 00 02 03 45 ++ 15 00 02 04 46 ++ 15 00 02 05 46 ++ 15 00 02 06 47 ++ 15 00 02 07 47 ++ 15 00 02 08 1D ++ 15 00 02 09 1D ++ 15 00 02 0A 1D ++ 15 00 02 0B 1D ++ 15 00 02 0C 1D ++ 15 00 02 0D 1D ++ 15 00 02 0E 1D ++ 15 00 02 0F 57 ++ 15 00 02 10 57 ++ 15 00 02 11 58 ++ 15 00 02 12 58 ++ 15 00 02 13 40 ++ 15 00 02 14 55 ++ 15 00 02 15 55 ++ 15 00 02 16 44 ++ 15 00 02 17 44 ++ 15 00 02 18 45 ++ 15 00 02 19 45 ++ 15 00 02 1A 46 ++ 15 00 02 1B 46 ++ 15 00 02 1C 47 ++ 15 00 02 1D 47 ++ 15 00 02 1E 1D ++ 15 00 02 1F 1D ++ 15 00 02 20 1D ++ 15 00 02 21 1D ++ 15 00 02 22 1D ++ 15 00 02 23 1D ++ 15 00 02 24 1D ++ 15 00 02 25 57 ++ 15 00 02 26 57 ++ 15 00 02 27 58 ++ 15 00 02 28 58 ++ 15 00 02 29 40 ++ 15 00 02 2A 55 ++ 15 00 02 2B 55 ++ 15 00 02 58 40 ++ 15 00 02 59 00 ++ 15 00 02 5A 00 ++ 15 00 02 5B 00 ++ 15 00 02 5C 0A ++ 15 00 02 5D 10 ++ 15 00 02 5E 01 ++ 15 00 02 5F 02 ++ 15 00 02 60 00 ++ 15 00 02 61 01 ++ 15 00 02 62 02 ++ 15 00 02 63 0B ++ 15 00 02 64 6A ++ 15 00 02 65 00 ++ 15 00 02 66 00 ++ 15 00 02 67 31 ++ 15 00 02 68 0B ++ 15 00 02 69 1E ++ 15 00 02 6A 6A ++ 15 00 02 6B 04 ++ 15 00 02 6C 00 ++ 15 00 02 6D 04 ++ 15 00 02 6E 00 ++ 15 00 02 6F 88 ++ 15 00 02 70 00 ++ 15 00 02 71 00 ++ 15 00 02 72 06 ++ 15 00 02 73 7B ++ 15 00 02 74 00 ++ 15 00 02 75 F8 ++ 15 00 02 76 00 ++ 15 00 02 77 0D ++ 15 00 02 78 14 ++ 15 00 02 79 00 ++ 15 00 02 7A 00 ++ 15 00 02 7B 00 ++ 15 00 02 7C 00 ++ 15 00 02 7D 03 ++ 15 00 02 7E 7B ++ 15 00 02 E0 00 ++ 15 00 02 E6 02 ++ 15 00 02 E7 06 ++ 15 80 01 11 ++ 15 16 01 29 ++ ]; ++ ++ panel-exit-sequence = [ ++ 05 00 01 28 ++ 05 00 01 10 ++ ]; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <68500000>; ++ hactive = <800>; ++ hfront-porch = <16>; ++ hsync-len = <16>; ++ hback-porch = <48>; ++ vactive = <1280>; ++ vfront-porch = <8>; ++ vsync-len = <4>; ++ vback-porch = <4>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ panel_in_dsi: endpoint { ++ remote-endpoint = <&dsi_out_panel>; ++ }; ++ }; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@1 { ++ reg = <1>; ++ dsi_out_panel: endpoint { ++ remote-endpoint = <&panel_in_dsi>; ++ }; ++ }; ++ }; ++}; ++ ++&dsi_in_vopl { ++ status = "disabled"; ++}; ++ ++&dsi_in_vopb { ++ status = "okay"; ++}; ++ ++&route_dsi { ++ connect = <&vopb_out_dsi>; ++ status = "okay"; ++}; ++ ++&pinctrl { ++ pmic { ++ pmic_int: pmic_int { ++ rockchip,pins = ++ <0 RK_PA7 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++&firmware_android { ++ compatible = "android,firmware"; ++ ++ fstab { ++ compatible = "android,fstab"; ++ ++ system { ++ compatible = "android,system"; ++ dev = "/dev/block/by-name/system"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait"; ++ }; ++ ++ vendor { ++ compatible = "android,vendor"; ++ dev = "/dev/block/by-name/vendor"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait"; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/px30-android.dtsi b/arch/arm64/boot/dts/rockchip/px30-android.dtsi +new file mode 100755 +index 000000000000..aecb6344f01f +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/px30-android.dtsi +@@ -0,0 +1,131 @@ ++/* ++ * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++ */ ++ ++/ { ++ chosen: chosen { ++ bootargs = "earlycon=uart8250,mmio32,0xff160000 console=ttyFIQ0 androidboot.baseband=N/A androidboot.veritymode=enforcing androidboot.hardware=rk30board androidboot.console=ttyFIQ0 init=/init kpti=0"; ++ }; ++ ++ fiq-debugger { ++ compatible = "rockchip,fiq-debugger"; ++ rockchip,serial-id = <2>; ++ rockchip,wake-irq = <0>; ++ /* If enable uart uses irq instead of fiq */ ++ rockchip,irq-mode-enable = <0>; ++ rockchip,baudrate = <1500000>; /* Only 115200 and 1500000 */ ++ interrupts = ; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart2m0_xfer>; ++ status = "okay"; ++ }; ++ ++ firmware { ++ firmware_android: android {}; ++ }; ++ ++ reserved-memory { ++ #address-cells = <2>; ++ #size-cells = <2>; ++ ranges; ++ ++ drm_logo: drm-logo@00000000 { ++ compatible = "rockchip,drm-logo"; ++ reg = <0x0 0x0 0x0 0x0>; ++ }; ++ ++ ramoops: ramoops@110000 { ++ compatible = "ramoops"; ++ reg = <0x0 0x110000 0x0 0xf0000>; ++ record-size = <0x20000>; ++ console-size = <0x80000>; ++ ftrace-size = <0x00000>; ++ pmsg-size = <0x50000>; ++ }; ++ }; ++}; ++ ++&cpu0_opp_table { ++ rockchip,avs = <1>; ++}; ++ ++&display_subsystem { ++ status = "disabled"; ++ ports = <&vopb_out>, <&vopl_out>; ++ logo-memory-region = <&drm_logo>; ++ ++ route { ++ route_lvds: route-lvds { ++ status = "disabled"; ++ logo,uboot = "logo.bmp"; ++ logo,kernel = "logo_kernel.bmp"; ++ logo,mode = "center"; ++ charge_logo,mode = "center"; ++ connect = <&vopb_out_lvds>; ++ }; ++ ++ route_dsi: route-dsi { ++ status = "disabled"; ++ logo,uboot = "logo.bmp"; ++ logo,kernel = "logo_kernel.bmp"; ++ logo,mode = "center"; ++ charge_logo,mode = "center"; ++ connect = <&vopb_out_dsi>; ++ }; ++ ++ route_rgb: route-rgb { ++ status = "disabled"; ++ logo,uboot = "logo.bmp"; ++ logo,kernel = "logo_kernel.bmp"; ++ logo,mode = "center"; ++ charge_logo,mode = "center"; ++ connect = <&vopb_out_rgb>; ++ }; ++ }; ++}; ++ ++&dsi { ++ panel@0 { ++ reg = <0>; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ ++ panel_in_dsi: endpoint { ++ remote-endpoint = <&dsi_out_panel>; ++ }; ++ }; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@1 { ++ reg = <1>; ++ ++ dsi_out_panel: endpoint { ++ remote-endpoint = <&panel_in_dsi>; ++ }; ++ }; ++ }; ++}; ++ ++&rng { ++ status = "okay"; ++}; ++ ++&video_phy { ++ status = "okay"; ++}; ++ ++&vopb { ++ support-multi-area; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/px30-ddr4p416dd6-timing.dtsi b/arch/arm64/boot/dts/rockchip/px30-ddr4p416dd6-timing.dtsi +new file mode 100755 +index 000000000000..fde5895db94a +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/px30-ddr4p416dd6-timing.dtsi +@@ -0,0 +1,216 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++// Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd. ++ ++&ddr_timing { ++ /* CA de-skew, one step is 47.8ps, range 0-15 */ ++ ddr3a1_ddr4a9_de-skew = <0>; ++ ddr3a0_ddr4a10_de-skew = <5>; ++ ddr3a3_ddr4a6_de-skew = <2>; ++ ddr3a2_ddr4a4_de-skew = <4>; ++ ddr3a5_ddr4a8_de-skew = <2>; ++ ddr3a4_ddr4a5_de-skew = <1>; ++ ddr3a7_ddr4a11_de-skew = <2>; ++ ddr3a6_ddr4a7_de-skew = <0>; ++ ddr3a9_ddr4a0_de-skew = <4>; ++ ddr3a8_ddr4a13_de-skew = <0>; ++ ddr3a11_ddr4a3_de-skew = <4>; ++ ddr3a10_ddr4cs0_de-skew = <5>; ++ ddr3a13_ddr4a2_de-skew = <2>; ++ ddr3a12_ddr4ba1_de-skew = <2>; ++ ddr3a15_ddr4odt0_de-skew = <7>; ++ ddr3a14_ddr4a1_de-skew = <4>; ++ ddr3ba1_ddr4a15_de-skew = <3>; ++ ddr3ba0_ddr4bg0_de-skew = <4>; ++ ddr3ras_ddr4cke_de-skew = <6>; ++ ddr3ba2_ddr4ba0_de-skew = <3>; ++ ddr3we_ddr4bg1_de-skew = <3>; ++ ddr3cas_ddr4a12_de-skew = <6>; ++ ddr3ckn_ddr4ckn_de-skew = <8>; ++ ddr3ckp_ddr4ckp_de-skew = <8>; ++ ddr3cke_ddr4a16_de-skew = <4>; ++ ddr3odt0_ddr4a14_de-skew = <4>; ++ ddr3cs0_ddr4act_de-skew = <7>; ++ ddr3reset_ddr4reset_de-skew = <3>; ++ ddr3cs1_ddr4cs1_de-skew = <5>; ++ ddr3odt1_ddr4odt1_de-skew = <6>; ++ ++ /* DATA de-skew ++ * RX one step is 25.1ps, range 0-15 ++ * TX one step is 47.8ps, range 0-15 ++ */ ++ cs0_dm0_rx_de-skew = <7>; ++ cs0_dm0_tx_de-skew = <9>; ++ cs0_dq0_rx_de-skew = <10>; ++ cs0_dq0_tx_de-skew = <10>; ++ cs0_dq1_rx_de-skew = <11>; ++ cs0_dq1_tx_de-skew = <10>; ++ cs0_dq2_rx_de-skew = <8>; ++ cs0_dq2_tx_de-skew = <10>; ++ cs0_dq3_rx_de-skew = <9>; ++ cs0_dq3_tx_de-skew = <9>; ++ cs0_dq4_rx_de-skew = <10>; ++ cs0_dq4_tx_de-skew = <10>; ++ cs0_dq5_rx_de-skew = <10>; ++ cs0_dq5_tx_de-skew = <10>; ++ cs0_dq6_rx_de-skew = <10>; ++ cs0_dq6_tx_de-skew = <10>; ++ cs0_dq7_rx_de-skew = <10>; ++ cs0_dq7_tx_de-skew = <10>; ++ cs0_dqs0_rx_de-skew = <5>; ++ cs0_dqs0p_tx_de-skew = <11>; ++ cs0_dqs0n_tx_de-skew = <11>; ++ ++ cs0_dm1_rx_de-skew = <7>; ++ cs0_dm1_tx_de-skew = <9>; ++ cs0_dq8_rx_de-skew = <8>; ++ cs0_dq8_tx_de-skew = <8>; ++ cs0_dq9_rx_de-skew = <10>; ++ cs0_dq9_tx_de-skew = <9>; ++ cs0_dq10_rx_de-skew = <8>; ++ cs0_dq10_tx_de-skew = <8>; ++ cs0_dq11_rx_de-skew = <9>; ++ cs0_dq11_tx_de-skew = <10>; ++ cs0_dq12_rx_de-skew = <9>; ++ cs0_dq12_tx_de-skew = <8>; ++ cs0_dq13_rx_de-skew = <8>; ++ cs0_dq13_tx_de-skew = <9>; ++ cs0_dq14_rx_de-skew = <9>; ++ cs0_dq14_tx_de-skew = <8>; ++ cs0_dq15_rx_de-skew = <9>; ++ cs0_dq15_tx_de-skew = <9>; ++ cs0_dqs1_rx_de-skew = <5>; ++ cs0_dqs1p_tx_de-skew = <11>; ++ cs0_dqs1n_tx_de-skew = <11>; ++ ++ cs0_dm2_rx_de-skew = <7>; ++ cs0_dm2_tx_de-skew = <10>; ++ cs0_dq16_rx_de-skew = <10>; ++ cs0_dq16_tx_de-skew = <9>; ++ cs0_dq17_rx_de-skew = <10>; ++ cs0_dq17_tx_de-skew = <9>; ++ cs0_dq18_rx_de-skew = <9>; ++ cs0_dq18_tx_de-skew = <8>; ++ cs0_dq19_rx_de-skew = <10>; ++ cs0_dq19_tx_de-skew = <9>; ++ cs0_dq20_rx_de-skew = <10>; ++ cs0_dq20_tx_de-skew = <10>; ++ cs0_dq21_rx_de-skew = <10>; ++ cs0_dq21_tx_de-skew = <9>; ++ cs0_dq22_rx_de-skew = <9>; ++ cs0_dq22_tx_de-skew = <8>; ++ cs0_dq23_rx_de-skew = <10>; ++ cs0_dq23_tx_de-skew = <9>; ++ cs0_dqs2_rx_de-skew = <5>; ++ cs0_dqs2p_tx_de-skew = <10>; ++ cs0_dqs2n_tx_de-skew = <10>; ++ ++ cs0_dm3_rx_de-skew = <7>; ++ cs0_dm3_tx_de-skew = <7>; ++ cs0_dq24_rx_de-skew = <9>; ++ cs0_dq24_tx_de-skew = <8>; ++ cs0_dq25_rx_de-skew = <9>; ++ cs0_dq25_tx_de-skew = <8>; ++ cs0_dq26_rx_de-skew = <9>; ++ cs0_dq26_tx_de-skew = <8>; ++ cs0_dq27_rx_de-skew = <9>; ++ cs0_dq27_tx_de-skew = <9>; ++ cs0_dq28_rx_de-skew = <9>; ++ cs0_dq28_tx_de-skew = <9>; ++ cs0_dq29_rx_de-skew = <9>; ++ cs0_dq29_tx_de-skew = <9>; ++ cs0_dq30_rx_de-skew = <10>; ++ cs0_dq30_tx_de-skew = <9>; ++ cs0_dq31_rx_de-skew = <10>; ++ cs0_dq31_tx_de-skew = <9>; ++ cs0_dqs3_rx_de-skew = <6>; ++ cs0_dqs3p_tx_de-skew = <10>; ++ cs0_dqs3n_tx_de-skew = <10>; ++ ++ cs1_dm0_rx_de-skew = <7>; ++ cs1_dm0_tx_de-skew = <9>; ++ cs1_dq0_rx_de-skew = <10>; ++ cs1_dq0_tx_de-skew = <10>; ++ cs1_dq1_rx_de-skew = <11>; ++ cs1_dq1_tx_de-skew = <10>; ++ cs1_dq2_rx_de-skew = <8>; ++ cs1_dq2_tx_de-skew = <10>; ++ cs1_dq3_rx_de-skew = <9>; ++ cs1_dq3_tx_de-skew = <9>; ++ cs1_dq4_rx_de-skew = <10>; ++ cs1_dq4_tx_de-skew = <10>; ++ cs1_dq5_rx_de-skew = <10>; ++ cs1_dq5_tx_de-skew = <10>; ++ cs1_dq6_rx_de-skew = <10>; ++ cs1_dq6_tx_de-skew = <10>; ++ cs1_dq7_rx_de-skew = <10>; ++ cs1_dq7_tx_de-skew = <10>; ++ cs1_dqs0_rx_de-skew = <5>; ++ cs1_dqs0p_tx_de-skew = <11>; ++ cs1_dqs0n_tx_de-skew = <11>; ++ ++ cs1_dm1_rx_de-skew = <7>; ++ cs1_dm1_tx_de-skew = <9>; ++ cs1_dq8_rx_de-skew = <8>; ++ cs1_dq8_tx_de-skew = <8>; ++ cs1_dq9_rx_de-skew = <10>; ++ cs1_dq9_tx_de-skew = <9>; ++ cs1_dq10_rx_de-skew = <8>; ++ cs1_dq10_tx_de-skew = <8>; ++ cs1_dq11_rx_de-skew = <9>; ++ cs1_dq11_tx_de-skew = <10>; ++ cs1_dq12_rx_de-skew = <9>; ++ cs1_dq12_tx_de-skew = <8>; ++ cs1_dq13_rx_de-skew = <8>; ++ cs1_dq13_tx_de-skew = <9>; ++ cs1_dq14_rx_de-skew = <9>; ++ cs1_dq14_tx_de-skew = <8>; ++ cs1_dq15_rx_de-skew = <9>; ++ cs1_dq15_tx_de-skew = <9>; ++ cs1_dqs1_rx_de-skew = <5>; ++ cs1_dqs1p_tx_de-skew = <11>; ++ cs1_dqs1n_tx_de-skew = <11>; ++ ++ cs1_dm2_rx_de-skew = <7>; ++ cs1_dm2_tx_de-skew = <10>; ++ cs1_dq16_rx_de-skew = <10>; ++ cs1_dq16_tx_de-skew = <9>; ++ cs1_dq17_rx_de-skew = <10>; ++ cs1_dq17_tx_de-skew = <9>; ++ cs1_dq18_rx_de-skew = <9>; ++ cs1_dq18_tx_de-skew = <8>; ++ cs1_dq19_rx_de-skew = <10>; ++ cs1_dq19_tx_de-skew = <9>; ++ cs1_dq20_rx_de-skew = <10>; ++ cs1_dq20_tx_de-skew = <10>; ++ cs1_dq21_rx_de-skew = <10>; ++ cs1_dq21_tx_de-skew = <9>; ++ cs1_dq22_rx_de-skew = <9>; ++ cs1_dq22_tx_de-skew = <8>; ++ cs1_dq23_rx_de-skew = <10>; ++ cs1_dq23_tx_de-skew = <9>; ++ cs1_dqs2_rx_de-skew = <5>; ++ cs1_dqs2p_tx_de-skew = <10>; ++ cs1_dqs2n_tx_de-skew = <10>; ++ ++ cs1_dm3_rx_de-skew = <7>; ++ cs1_dm3_tx_de-skew = <7>; ++ cs1_dq24_rx_de-skew = <9>; ++ cs1_dq24_tx_de-skew = <8>; ++ cs1_dq25_rx_de-skew = <9>; ++ cs1_dq25_tx_de-skew = <8>; ++ cs1_dq26_rx_de-skew = <9>; ++ cs1_dq26_tx_de-skew = <8>; ++ cs1_dq27_rx_de-skew = <9>; ++ cs1_dq27_tx_de-skew = <9>; ++ cs1_dq28_rx_de-skew = <9>; ++ cs1_dq28_tx_de-skew = <9>; ++ cs1_dq29_rx_de-skew = <9>; ++ cs1_dq29_tx_de-skew = <9>; ++ cs1_dq30_rx_de-skew = <10>; ++ cs1_dq30_tx_de-skew = <9>; ++ cs1_dq31_rx_de-skew = <10>; ++ cs1_dq31_tx_de-skew = <9>; ++ cs1_dqs3_rx_de-skew = <6>; ++ cs1_dqs3p_tx_de-skew = <10>; ++ cs1_dqs3n_tx_de-skew = <10>; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/px30-dram-default-timing.dtsi b/arch/arm64/boot/dts/rockchip/px30-dram-default-timing.dtsi +new file mode 100755 +index 000000000000..c75c5ef4ef2a +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/px30-dram-default-timing.dtsi +@@ -0,0 +1,294 @@ ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++ */ ++ ++#include ++#include ++ ++/ { ++ ddr_timing: ddr_timing { ++ compatible = "rockchip,ddr-timing"; ++ ddr2_speed_bin = ; ++ ddr3_speed_bin = ; ++ ddr4_speed_bin = ; ++ pd_idle = <13>; ++ sr_idle = <93>; ++ sr_mc_gate_idle = <0>; ++ srpd_lite_idle = <0>; ++ standby_idle = <0>; ++ ++ auto_pd_dis_freq = <1066>; ++ auto_sr_dis_freq = <800>; ++ ddr2_dll_dis_freq = <300>; ++ ddr3_dll_dis_freq = <300>; ++ ddr4_dll_dis_freq = <625>; ++ phy_dll_dis_freq = <400>; ++ ++ ddr2_odt_dis_freq = <100>; ++ phy_ddr2_odt_dis_freq = <100>; ++ ddr2_drv = ; ++ ddr2_odt = ; ++ phy_ddr2_ca_drv = ; ++ phy_ddr2_ck_drv = ; ++ phy_ddr2_dq_drv = ; ++ phy_ddr2_odt = ; ++ ++ ddr3_odt_dis_freq = <400>; ++ phy_ddr3_odt_dis_freq = <400>; ++ ddr3_drv = ; ++ ddr3_odt = ; ++ phy_ddr3_ca_drv = ; ++ phy_ddr3_ck_drv = ; ++ phy_ddr3_dq_drv = ; ++ phy_ddr3_odt = ; ++ ++ phy_lpddr2_odt_dis_freq = <666>; ++ lpddr2_drv = ; ++ phy_lpddr2_ca_drv = ; ++ phy_lpddr2_ck_drv = ; ++ phy_lpddr2_dq_drv = ; ++ phy_lpddr2_odt = ; ++ ++ lpddr3_odt_dis_freq = <400>; ++ phy_lpddr3_odt_dis_freq = <400>; ++ lpddr3_drv = ; ++ lpddr3_odt = ; ++ phy_lpddr3_ca_drv = ; ++ phy_lpddr3_ck_drv = ; ++ phy_lpddr3_dq_drv = ; ++ phy_lpddr3_odt = ; ++ ++ lpddr4_odt_dis_freq = <800>; ++ phy_lpddr4_odt_dis_freq = <800>; ++ lpddr4_drv = ; ++ lpddr4_dq_odt = ; ++ lpddr4_ca_odt = ; ++ phy_lpddr4_ca_drv = ; ++ phy_lpddr4_ck_cs_drv = ; ++ phy_lpddr4_dq_drv = ; ++ phy_lpddr4_odt = ; ++ ++ ddr4_odt_dis_freq = <666>; ++ phy_ddr4_odt_dis_freq = <666>; ++ ddr4_drv = ; ++ ddr4_odt = ; ++ phy_ddr4_ca_drv = ; ++ phy_ddr4_ck_drv = ; ++ phy_ddr4_dq_drv = ; ++ phy_ddr4_odt = ; ++ ++ /* CA de-skew, one step is 47.8ps, range 0-15 */ ++ ddr3a1_ddr4a9_de-skew = <6>; ++ ddr3a0_ddr4a10_de-skew = <7>; ++ ddr3a3_ddr4a6_de-skew = <7>; ++ ddr3a2_ddr4a4_de-skew = <7>; ++ ddr3a5_ddr4a8_de-skew = <7>; ++ ddr3a4_ddr4a5_de-skew = <7>; ++ ddr3a7_ddr4a11_de-skew = <7>; ++ ddr3a6_ddr4a7_de-skew = <6>; ++ ddr3a9_ddr4a0_de-skew = <7>; ++ ddr3a8_ddr4a13_de-skew = <7>; ++ ddr3a11_ddr4a3_de-skew = <7>; ++ ddr3a10_ddr4cs0_de-skew = <7>; ++ ddr3a13_ddr4a2_de-skew = <7>; ++ ddr3a12_ddr4ba1_de-skew = <7>; ++ ddr3a15_ddr4odt0_de-skew = <7>; ++ ddr3a14_ddr4a1_de-skew = <7>; ++ ddr3ba1_ddr4a15_de-skew = <7>; ++ ddr3ba0_ddr4bg0_de-skew = <7>; ++ ddr3ras_ddr4cke_de-skew = <7>; ++ ddr3ba2_ddr4ba0_de-skew = <7>; ++ ddr3we_ddr4bg1_de-skew = <7>; ++ ddr3cas_ddr4a12_de-skew = <7>; ++ ddr3ckn_ddr4ckn_de-skew = <7>; ++ ddr3ckp_ddr4ckp_de-skew = <7>; ++ ddr3cke_ddr4a16_de-skew = <7>; ++ ddr3odt0_ddr4a14_de-skew = <7>; ++ ddr3cs0_ddr4act_de-skew = <6>; ++ ddr3reset_ddr4reset_de-skew = <7>; ++ ddr3cs1_ddr4cs1_de-skew = <6>; ++ ddr3odt1_ddr4odt1_de-skew = <7>; ++ ++ /* DATA de-skew ++ * RX one step is 25.1ps, range 0-15 ++ * TX one step is 47.8ps, range 0-15 ++ */ ++ cs0_dm0_rx_de-skew = <7>; ++ cs0_dm0_tx_de-skew = <7>; ++ cs0_dq0_rx_de-skew = <8>; ++ cs0_dq0_tx_de-skew = <8>; ++ cs0_dq1_rx_de-skew = <9>; ++ cs0_dq1_tx_de-skew = <8>; ++ cs0_dq2_rx_de-skew = <8>; ++ cs0_dq2_tx_de-skew = <8>; ++ cs0_dq3_rx_de-skew = <8>; ++ cs0_dq3_tx_de-skew = <8>; ++ cs0_dq4_rx_de-skew = <9>; ++ cs0_dq4_tx_de-skew = <8>; ++ cs0_dq5_rx_de-skew = <9>; ++ cs0_dq5_tx_de-skew = <8>; ++ cs0_dq6_rx_de-skew = <9>; ++ cs0_dq6_tx_de-skew = <8>; ++ cs0_dq7_rx_de-skew = <8>; ++ cs0_dq7_tx_de-skew = <8>; ++ cs0_dqs0_rx_de-skew = <6>; ++ cs0_dqs0p_tx_de-skew = <9>; ++ cs0_dqs0n_tx_de-skew = <9>; ++ ++ cs0_dm1_rx_de-skew = <7>; ++ cs0_dm1_tx_de-skew = <6>; ++ cs0_dq8_rx_de-skew = <8>; ++ cs0_dq8_tx_de-skew = <7>; ++ cs0_dq9_rx_de-skew = <9>; ++ cs0_dq9_tx_de-skew = <7>; ++ cs0_dq10_rx_de-skew = <8>; ++ cs0_dq10_tx_de-skew = <8>; ++ cs0_dq11_rx_de-skew = <8>; ++ cs0_dq11_tx_de-skew = <7>; ++ cs0_dq12_rx_de-skew = <8>; ++ cs0_dq12_tx_de-skew = <8>; ++ cs0_dq13_rx_de-skew = <9>; ++ cs0_dq13_tx_de-skew = <7>; ++ cs0_dq14_rx_de-skew = <9>; ++ cs0_dq14_tx_de-skew = <8>; ++ cs0_dq15_rx_de-skew = <9>; ++ cs0_dq15_tx_de-skew = <7>; ++ cs0_dqs1_rx_de-skew = <7>; ++ cs0_dqs1p_tx_de-skew = <9>; ++ cs0_dqs1n_tx_de-skew = <9>; ++ ++ cs0_dm2_rx_de-skew = <7>; ++ cs0_dm2_tx_de-skew = <7>; ++ cs0_dq16_rx_de-skew = <9>; ++ cs0_dq16_tx_de-skew = <9>; ++ cs0_dq17_rx_de-skew = <7>; ++ cs0_dq17_tx_de-skew = <9>; ++ cs0_dq18_rx_de-skew = <7>; ++ cs0_dq18_tx_de-skew = <8>; ++ cs0_dq19_rx_de-skew = <7>; ++ cs0_dq19_tx_de-skew = <9>; ++ cs0_dq20_rx_de-skew = <9>; ++ cs0_dq20_tx_de-skew = <9>; ++ cs0_dq21_rx_de-skew = <9>; ++ cs0_dq21_tx_de-skew = <9>; ++ cs0_dq22_rx_de-skew = <8>; ++ cs0_dq22_tx_de-skew = <9>; ++ cs0_dq23_rx_de-skew = <8>; ++ cs0_dq23_tx_de-skew = <9>; ++ cs0_dqs2_rx_de-skew = <6>; ++ cs0_dqs2p_tx_de-skew = <9>; ++ cs0_dqs2n_tx_de-skew = <9>; ++ ++ cs0_dm3_rx_de-skew = <7>; ++ cs0_dm3_tx_de-skew = <7>; ++ cs0_dq24_rx_de-skew = <8>; ++ cs0_dq24_tx_de-skew = <8>; ++ cs0_dq25_rx_de-skew = <9>; ++ cs0_dq25_tx_de-skew = <9>; ++ cs0_dq26_rx_de-skew = <9>; ++ cs0_dq26_tx_de-skew = <8>; ++ cs0_dq27_rx_de-skew = <9>; ++ cs0_dq27_tx_de-skew = <8>; ++ cs0_dq28_rx_de-skew = <9>; ++ cs0_dq28_tx_de-skew = <9>; ++ cs0_dq29_rx_de-skew = <9>; ++ cs0_dq29_tx_de-skew = <9>; ++ cs0_dq30_rx_de-skew = <8>; ++ cs0_dq30_tx_de-skew = <8>; ++ cs0_dq31_rx_de-skew = <8>; ++ cs0_dq31_tx_de-skew = <8>; ++ cs0_dqs3_rx_de-skew = <7>; ++ cs0_dqs3p_tx_de-skew = <9>; ++ cs0_dqs3n_tx_de-skew = <9>; ++ ++ cs1_dm0_rx_de-skew = <7>; ++ cs1_dm0_tx_de-skew = <7>; ++ cs1_dq0_rx_de-skew = <8>; ++ cs1_dq0_tx_de-skew = <8>; ++ cs1_dq1_rx_de-skew = <9>; ++ cs1_dq1_tx_de-skew = <8>; ++ cs1_dq2_rx_de-skew = <8>; ++ cs1_dq2_tx_de-skew = <8>; ++ cs1_dq3_rx_de-skew = <8>; ++ cs1_dq3_tx_de-skew = <8>; ++ cs1_dq4_rx_de-skew = <8>; ++ cs1_dq4_tx_de-skew = <8>; ++ cs1_dq5_rx_de-skew = <9>; ++ cs1_dq5_tx_de-skew = <8>; ++ cs1_dq6_rx_de-skew = <9>; ++ cs1_dq6_tx_de-skew = <8>; ++ cs1_dq7_rx_de-skew = <8>; ++ cs1_dq7_tx_de-skew = <8>; ++ cs1_dqs0_rx_de-skew = <6>; ++ cs1_dqs0p_tx_de-skew = <9>; ++ cs1_dqs0n_tx_de-skew = <9>; ++ ++ cs1_dm1_rx_de-skew = <7>; ++ cs1_dm1_tx_de-skew = <7>; ++ cs1_dq8_rx_de-skew = <8>; ++ cs1_dq8_tx_de-skew = <8>; ++ cs1_dq9_rx_de-skew = <8>; ++ cs1_dq9_tx_de-skew = <7>; ++ cs1_dq10_rx_de-skew = <7>; ++ cs1_dq10_tx_de-skew = <8>; ++ cs1_dq11_rx_de-skew = <8>; ++ cs1_dq11_tx_de-skew = <8>; ++ cs1_dq12_rx_de-skew = <8>; ++ cs1_dq12_tx_de-skew = <7>; ++ cs1_dq13_rx_de-skew = <8>; ++ cs1_dq13_tx_de-skew = <8>; ++ cs1_dq14_rx_de-skew = <8>; ++ cs1_dq14_tx_de-skew = <8>; ++ cs1_dq15_rx_de-skew = <8>; ++ cs1_dq15_tx_de-skew = <7>; ++ cs1_dqs1_rx_de-skew = <7>; ++ cs1_dqs1p_tx_de-skew = <9>; ++ cs1_dqs1n_tx_de-skew = <9>; ++ ++ cs1_dm2_rx_de-skew = <7>; ++ cs1_dm2_tx_de-skew = <8>; ++ cs1_dq16_rx_de-skew = <8>; ++ cs1_dq16_tx_de-skew = <9>; ++ cs1_dq17_rx_de-skew = <8>; ++ cs1_dq17_tx_de-skew = <9>; ++ cs1_dq18_rx_de-skew = <7>; ++ cs1_dq18_tx_de-skew = <8>; ++ cs1_dq19_rx_de-skew = <8>; ++ cs1_dq19_tx_de-skew = <9>; ++ cs1_dq20_rx_de-skew = <9>; ++ cs1_dq20_tx_de-skew = <9>; ++ cs1_dq21_rx_de-skew = <9>; ++ cs1_dq21_tx_de-skew = <9>; ++ cs1_dq22_rx_de-skew = <8>; ++ cs1_dq22_tx_de-skew = <9>; ++ cs1_dq23_rx_de-skew = <8>; ++ cs1_dq23_tx_de-skew = <9>; ++ cs1_dqs2_rx_de-skew = <6>; ++ cs1_dqs2p_tx_de-skew = <9>; ++ cs1_dqs2n_tx_de-skew = <9>; ++ ++ cs1_dm3_rx_de-skew = <7>; ++ cs1_dm3_tx_de-skew = <7>; ++ cs1_dq24_rx_de-skew = <8>; ++ cs1_dq24_tx_de-skew = <9>; ++ cs1_dq25_rx_de-skew = <9>; ++ cs1_dq25_tx_de-skew = <9>; ++ cs1_dq26_rx_de-skew = <9>; ++ cs1_dq26_tx_de-skew = <8>; ++ cs1_dq27_rx_de-skew = <8>; ++ cs1_dq27_tx_de-skew = <8>; ++ cs1_dq28_rx_de-skew = <9>; ++ cs1_dq28_tx_de-skew = <9>; ++ cs1_dq29_rx_de-skew = <9>; ++ cs1_dq29_tx_de-skew = <9>; ++ cs1_dq30_rx_de-skew = <9>; ++ cs1_dq30_tx_de-skew = <8>; ++ cs1_dq31_rx_de-skew = <8>; ++ cs1_dq31_tx_de-skew = <8>; ++ cs1_dqs3_rx_de-skew = <7>; ++ cs1_dqs3p_tx_de-skew = <9>; ++ cs1_dqs3n_tx_de-skew = <9>; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-lvds-v10.dts b/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-lvds-v10.dts +new file mode 100755 +index 000000000000..eb60e71b77f3 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-lvds-v10.dts +@@ -0,0 +1,689 @@ ++/* ++ * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++ */ ++ ++/dts-v1/; ++#include ++#include ++#include ++#include ++#include "px30.dtsi" ++#include "px30-android.dtsi" ++ ++/ { ++ model = "Rockchip PX30 evb ddr3 lvds board"; ++ compatible = "rockchip,px30-evb-ddr3-lvds-v10", "rockchip,px30"; ++ ++ adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 2>; ++ io-channel-names = "buttons"; ++ poll-interval = <100>; ++ keyup-threshold-microvolt = <1800000>; ++ ++ esc-key { ++ linux,code = ; ++ label = "esc"; ++ press-threshold-microvolt = <1270000>; ++ }; ++ ++ home-key { ++ linux,code = ; ++ label = "home"; ++ press-threshold-microvolt = <602000>; ++ }; ++ ++ menu-key { ++ linux,code = ; ++ label = "menu"; ++ press-threshold-microvolt = <952000>; ++ }; ++ ++ vol-down-key { ++ linux,code = ; ++ label = "volume down"; ++ press-threshold-microvolt = <290000>; ++ }; ++ ++ vol-up-key { ++ linux,code = ; ++ label = "volume up"; ++ press-threshold-microvolt = <17000>; ++ }; ++ }; ++ ++ backlight: backlight { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm1 0 25000 0>; ++ brightness-levels = < ++ 0 1 2 3 4 5 6 7 ++ 8 9 10 11 12 13 14 15 ++ 16 17 18 19 20 21 22 23 ++ 24 25 26 27 28 29 30 31 ++ 32 33 34 35 36 37 38 39 ++ 40 41 42 43 44 45 46 47 ++ 48 49 50 51 52 53 54 55 ++ 56 57 58 59 60 61 62 63 ++ 64 65 66 67 68 69 70 71 ++ 72 73 74 75 76 77 78 79 ++ 80 81 82 83 84 85 86 87 ++ 88 89 90 91 92 93 94 95 ++ 96 97 98 99 100 101 102 103 ++ 104 105 106 107 108 109 110 111 ++ 112 113 114 115 116 117 118 119 ++ 120 121 122 123 124 125 126 127 ++ 128 129 130 131 132 133 134 135 ++ 136 137 138 139 140 141 142 143 ++ 144 145 146 147 148 149 150 151 ++ 152 153 154 155 156 157 158 159 ++ 160 161 162 163 164 165 166 167 ++ 168 169 170 171 172 173 174 175 ++ 176 177 178 179 180 181 182 183 ++ 184 185 186 187 188 189 190 191 ++ 192 193 194 195 196 197 198 199 ++ 200 201 202 203 204 205 206 207 ++ 208 209 210 211 212 213 214 215 ++ 216 217 218 219 220 221 222 223 ++ 224 225 226 227 228 229 230 231 ++ 232 233 234 235 236 237 238 239 ++ 240 241 242 243 244 245 246 247 ++ 248 249 250 251 252 253 254 255>; ++ default-brightness-level = <200>; ++ }; ++ ++ panel { ++ compatible = "samsung,lsl070nl01", "simple-panel"; ++ backlight = <&backlight>; ++ power-supply = <&vcc3v3_lcd>; ++ enable-delay-ms = <20>; ++ prepare-delay-ms = <20>; ++ unprepare-delay-ms = <20>; ++ disable-delay-ms = <20>; ++ bus-format = ; ++ ++ width-mm = <217>; ++ height-mm = <136>; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <49500000>; ++ hactive = <1024>; ++ vactive = <600>; ++ hback-porch = <90>; ++ hfront-porch = <90>; ++ vback-porch = <10>; ++ vfront-porch = <10>; ++ hsync-len = <90>; ++ vsync-len = <10>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ port { ++ panel_in_lvds: endpoint { ++ remote-endpoint = <&lvds_out_panel>; ++ }; ++ }; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ /*clocks = <&rk809 1>;*/ ++ /*clock-names = "ext_clock";*/ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_LOW>; /* GPIO3_A4 */ ++ }; ++ ++ vcc_phy: vcc-phy-regulator { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_phy"; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ vcc5v0_sys: vccsys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc5v0_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ wifi_chip_type = "AP6210"; ++ WIFI,host_wake_irq = <&gpio0 RK_PB2 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ /*clocks = <&rk809 1>;*/ ++ /*clock-names = "ext_clock";*/ ++ uart_rts_gpios = <&gpio1 RK_PC3 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default","rts_gpio"; ++ pinctrl-0 = <&uart1_rts>; ++ pinctrl-1 = <&uart1_rts_gpio>; ++ BT,reset_gpio = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio0 RK_PA1 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio0 RK_PB3 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++}; ++ ++&cpu0 { ++ cpu-supply = <&vdd_arm>; ++}; ++ ++&display_subsystem { ++ status = "okay"; ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&dmc { ++ center-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&emmc { ++ bus-width = <8>; ++ cap-mmc-highspeed; ++ mmc-hs200-1_8v; ++ no-sdio; ++ no-sd; ++ disable-wp; ++ non-removable; ++ num-slots = <1>; ++ status = "okay"; ++}; ++ ++&gmac { ++ phy-supply = <&vcc_phy>; ++ clock_in_out = "output"; ++ snps,reset-gpio = <&gpio2 13 GPIO_ACTIVE_LOW>; ++ snps,reset-active-low; ++ snps,reset-delays-us = <0 50000 50000>; ++ status = "okay"; ++}; ++ ++&gpu { ++ mali-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ ++ rk809: pmic@20 { ++ compatible = "rockchip,rk809"; ++ reg = <0x20>; ++ interrupt-parent = <&gpio0>; ++ interrupts = <7 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pmic_int>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ ++ vcc1-supply = <&vcc5v0_sys>; ++ vcc2-supply = <&vcc5v0_sys>; ++ vcc3-supply = <&vcc5v0_sys>; ++ vcc4-supply = <&vcc5v0_sys>; ++ vcc5-supply = <&vcc3v3_sys>; ++ vcc6-supply = <&vcc3v3_sys>; ++ vcc7-supply = <&vcc3v3_sys>; ++ vcc8-supply = <&vcc3v3_sys>; ++ vcc9-supply = <&vcc5v0_sys>; ++ ++ regulators { ++ vdd_logic: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <950000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x1>; ++ regulator-name = "vdd_logic"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <950000>; ++ }; ++ }; ++ ++ vdd_arm: DCDC_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <950000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x1>; ++ regulator-name = "vdd_arm"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <950000>; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc_ddr"; ++ regulator-initial-mode = <0x1>; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_3v0: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-initial-mode = <0x1>; ++ regulator-name = "vcc_3v0"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ ++ vcc_1v0: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ regulator-name = "vcc_1v0"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc1v8_soc: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcc1v8_soc"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd1v0_soc: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ ++ regulator-name = "vcc1v0_soc"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc3v0_pmu: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ ++ regulator-name = "vcc3v0_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ ++ }; ++ }; ++ ++ vccio_sd: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vccio_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc_sd: LDO_REG6 { ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vcc_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ ++ }; ++ }; ++ ++ vcc2v8_dvp: LDO_REG7 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <2800000>; ++ regulator-max-microvolt = <2800000>; ++ ++ regulator-name = "vcc2v8_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <2800000>; ++ }; ++ }; ++ ++ vcc1v8_dvp: LDO_REG8 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcc1v8_dvp"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd1v5_dvp: LDO_REG9 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ ++ regulator-name = "vdd1v5_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <1500000>; ++ }; ++ }; ++ ++ vcc3v3_sys: DCDC_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcc3v3_sys"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc3v3_lcd: SWITCH_REG1 { ++ regulator-boot-on; ++ regulator-name = "vcc3v3_lcd"; ++ }; ++ ++ vcc5v0_host: SWITCH_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc5v0_host"; ++ }; ++ }; ++ }; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ ++ sensor@f { ++ status = "okay"; ++ compatible = "ak8963"; ++ reg = <0x0f>; ++ type = ; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ layout = <1>; ++ reprobe_en = <1>; ++ }; ++ ++ gt1x: gt1x@14 { ++ compatible = "goodix,gt1x"; ++ reg = <0x14>; ++ goodix,rst-gpio = <&gpio0 RK_PB4 GPIO_ACTIVE_HIGH>; ++ goodix,irq-gpio = <&gpio0 RK_PA5 IRQ_TYPE_LEVEL_LOW>; ++ }; ++ ++ sensor@4c { ++ status = "okay"; ++ compatible = "gs_mma7660"; ++ reg = <0x4c>; ++ type = ; ++ irq-gpio = <&gpio0 RK_PB7 IRQ_TYPE_LEVEL_LOW>; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ layout = <2>; ++ reprobe_en = <1>; ++ }; ++}; ++ ++&io_domains { ++ status = "okay"; ++ ++ vccio1-supply = <&vcc1v8_soc>; ++ vccio2-supply = <&vccio_sd>; ++ vccio3-supply = <&vcc_3v0>; ++ vccio4-supply = <&vcc3v0_pmu>; ++ vccio5-supply = <&vcc_3v0>; ++}; ++ ++&lvds { ++ status = "okay"; ++ ++ ports { ++ port@1 { ++ reg = <1>; ++ ++ lvds_out_panel: endpoint { ++ remote-endpoint = <&panel_in_lvds>; ++ }; ++ }; ++ }; ++}; ++ ++&lvds_in_vopb { ++ status = "okay"; ++}; ++ ++&lvds_in_vopl { ++ status = "disabled"; ++}; ++ ++&route_lvds { ++ connect = <&vopb_out_lvds>; ++ status = "okay"; ++}; ++ ++&nandc0 { ++ status = "okay"; ++}; ++ ++&pinctrl { ++ pmic { ++ pmic_int: pmic_int { ++ rockchip,pins = ++ <0 RK_PA7 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ ++ pmuio1-supply = <&vcc3v0_pmu>; ++ pmuio2-supply = <&vcc3v0_pmu>; ++}; ++ ++&pwm1 { ++ status = "okay"; ++}; ++ ++&rk_rga { ++ status = "okay"; ++}; ++ ++&saradc { ++ status = "okay"; ++ vref-supply = <&vcc1v8_soc>; ++}; ++ ++&sdmmc { ++ bus-width = <4>; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ no-sdio; ++ no-mmc; ++ card-detect-delay = <800>; ++ ignore-pm-notify; ++ /*cd-gpios = <&gpio2 4 GPIO_ACTIVE_HIGH>; [> CD GPIO <]*/ ++ sd-uhs-sdr12; ++ sd-uhs-sdr25; ++ sd-uhs-sdr50; ++ sd-uhs-sdr104; ++ vqmmc-supply = <&vccio_sd>; ++ vmmc-supply = <&vcc_sd>; ++ status = "okay"; ++}; ++ ++&sdio { ++ bus-width = <4>; ++ cap-sd-highspeed; ++ no-sd; ++ no-mmc; ++ ignore-pm-notify; ++ keep-power-in-suspend; ++ non-removable; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&tsadc { ++ pinctrl-names = "gpio", "otpout"; ++ pinctrl-0 = <&tsadc_otp_gpio>; ++ pinctrl-1 = <&tsadc_otp_out>; ++ status = "okay"; ++}; ++ ++&uart1 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart1_xfer &uart1_cts>; ++ status = "okay"; ++}; ++ ++&u2phy { ++ status = "okay"; ++ ++ u2phy_host: host-port { ++ status = "okay"; ++ }; ++ ++ u2phy_otg: otg-port { ++ status = "okay"; ++ }; ++}; ++ ++&usb20_otg { ++ status = "okay"; ++}; ++ ++&usb_host0_ehci { ++ status = "okay"; ++}; ++ ++&usb_host0_ohci { ++ status = "okay"; ++}; ++ ++&vopb { ++ status = "okay"; ++}; ++ ++&vopb_mmu { ++ status = "okay"; ++}; ++ ++&vopl { ++ status = "okay"; ++}; ++ ++&vopl_mmu { ++ status = "okay"; ++}; ++ ++&mpp_srv { ++ status = "okay"; ++}; ++ ++&vdpu { ++ status = "okay"; ++}; ++ ++&vepu { ++ status = "okay"; ++}; ++ ++&vpu_mmu { ++ status = "okay"; ++}; ++ ++&hevc { ++ status = "okay"; ++}; ++ ++&hevc_mmu { ++ status = "okay"; ++}; ++ ++&firmware_android { ++ compatible = "android,firmware"; ++ fstab { ++ compatible = "android,fstab"; ++ system { ++ compatible = "android,system"; ++ dev = "/dev/block/by-name/system"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait"; ++ }; ++ vendor { ++ compatible = "android,vendor"; ++ dev = "/dev/block/by-name/vendor"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait"; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v10-avb.dts b/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v10-avb.dts +new file mode 100755 +index 000000000000..fca982485b16 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v10-avb.dts +@@ -0,0 +1,109 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2017-2019 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++#include "px30-evb-ddr3-v10.dtsi" ++ ++/ { ++ model = "Rockchip PX30 evb ddr3 board"; ++ compatible = "rockchip,px30-evb-ddr3-v10-avb", "rockchip,px30"; ++}; ++ ++&chosen { ++ bootargs_ext = "androidboot.boot_devices=ff390000.dwmmc,ff3b0000.nandc"; ++}; ++ ++&dsi { ++ status = "okay"; ++ ++ panel@0 { ++ compatible = "sitronix,st7703", "simple-panel-dsi"; ++ reg = <0>; ++ power-supply = <&vcc3v3_lcd>; ++ backlight = <&backlight>; ++ prepare-delay-ms = <2>; ++ reset-delay-ms = <1>; ++ init-delay-ms = <20>; ++ enable-delay-ms = <120>; ++ disable-delay-ms = <50>; ++ unprepare-delay-ms = <20>; ++ ++ width-mm = <68>; ++ height-mm = <121>; ++ ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | ++ MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ ++ panel-init-sequence = [ ++ 05 fa 01 11 ++ 39 00 04 b9 f1 12 83 ++ 39 00 1c ba 33 81 05 f9 0e 0e 00 00 00 ++ 00 00 00 00 00 44 25 00 91 0a ++ 00 00 02 4f 01 00 00 37 ++ 15 00 02 b8 25 ++ 39 00 04 bf 02 11 00 ++ 39 00 0b b3 0c 10 0a 50 03 ff 00 00 00 ++ 00 ++ 39 00 0a c0 73 73 50 50 00 00 08 70 00 ++ 15 00 02 bc 46 ++ 15 00 02 cc 0b ++ 15 00 02 b4 80 ++ 39 00 04 b2 c8 12 30 ++ 39 00 0f e3 07 07 0b 0b 03 0b 00 00 00 ++ 00 ff 00 c0 10 ++ 39 00 0d c1 53 00 1e 1e 77 e1 cc dd 67 ++ 77 33 33 ++ 39 00 07 c6 00 00 ff ff 01 ff ++ 39 00 03 b5 09 09 ++ 39 00 03 b6 87 95 ++ 39 00 40 e9 c2 10 05 05 10 05 a0 12 31 ++ 23 3f 81 0a a0 37 18 00 80 01 ++ 00 00 00 00 80 01 00 00 00 48 ++ f8 86 42 08 88 88 80 88 88 88 ++ 58 f8 87 53 18 88 88 81 88 88 ++ 88 00 00 00 01 00 00 00 00 00 ++ 00 00 00 00 ++ 39 00 3e ea 00 1a 00 00 00 00 02 00 00 ++ 00 00 00 1f 88 81 35 78 88 88 ++ 85 88 88 88 0f 88 80 24 68 88 ++ 88 84 88 88 88 23 10 00 00 1c ++ 00 00 00 00 00 00 00 00 00 00 ++ 00 00 00 00 00 30 05 a0 00 00 ++ 00 00 ++ 39 00 23 e0 00 06 08 2a 31 3f 38 36 07 ++ 0c 0d 11 13 12 13 11 18 00 06 ++ 08 2a 31 3f 38 36 07 0c 0d 11 ++ 13 12 13 11 18 ++ 05 32 01 29 ++ ]; ++ ++ panel-exit-sequence = [ ++ 05 00 01 28 ++ 05 00 01 10 ++ ]; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <64000000>; ++ hactive = <720>; ++ vactive = <1280>; ++ hfront-porch = <40>; ++ hsync-len = <10>; ++ hback-porch = <40>; ++ vfront-porch = <22>; ++ vsync-len = <4>; ++ vback-porch = <11>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v10-linux.dts b/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v10-linux.dts +new file mode 100755 +index 000000000000..ff24bdd46aea +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v10-linux.dts +@@ -0,0 +1,939 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++#include ++#include ++#include ++#include ++#include ++#include "px30.dtsi" ++#include "rk3326-linux.dtsi" ++ ++/ { ++ model = "Rockchip linux PX30 evb ddr3 board"; ++ compatible = "rockchip,px30-evb-ddr3-v10-linux", "rockchip,px30"; ++ ++ adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 2>; ++ io-channel-names = "buttons"; ++ poll-interval = <100>; ++ keyup-threshold-microvolt = <1800000>; ++ ++ esc-key { ++ linux,code = ; ++ label = "esc"; ++ press-threshold-microvolt = <1310000>; ++ }; ++ ++ home-key { ++ linux,code = ; ++ label = "home"; ++ press-threshold-microvolt = <624000>; ++ }; ++ ++ menu-key { ++ linux,code = ; ++ label = "menu"; ++ press-threshold-microvolt = <987000>; ++ }; ++ ++ vol-down-key { ++ linux,code = ; ++ label = "volume down"; ++ press-threshold-microvolt = <300000>; ++ }; ++ ++ vol-up-key { ++ linux,code = ; ++ label = "volume up"; ++ press-threshold-microvolt = <17000>; ++ }; ++ }; ++ ++ backlight: backlight { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm1 0 25000 0>; ++ brightness-levels = < ++ 0 1 2 3 4 5 6 7 ++ 8 9 10 11 12 13 14 15 ++ 16 17 18 19 20 21 22 23 ++ 24 25 26 27 28 29 30 31 ++ 32 33 34 35 36 37 38 39 ++ 40 41 42 43 44 45 46 47 ++ 48 49 50 51 52 53 54 55 ++ 56 57 58 59 60 61 62 63 ++ 64 65 66 67 68 69 70 71 ++ 72 73 74 75 76 77 78 79 ++ 80 81 82 83 84 85 86 87 ++ 88 89 90 91 92 93 94 95 ++ 96 97 98 99 100 101 102 103 ++ 104 105 106 107 108 109 110 111 ++ 112 113 114 115 116 117 118 119 ++ 120 121 122 123 124 125 126 127 ++ 128 129 130 131 132 133 134 135 ++ 136 137 138 139 140 141 142 143 ++ 144 145 146 147 148 149 150 151 ++ 152 153 154 155 156 157 158 159 ++ 160 161 162 163 164 165 166 167 ++ 168 169 170 171 172 173 174 175 ++ 176 177 178 179 180 181 182 183 ++ 184 185 186 187 188 189 190 191 ++ 192 193 194 195 196 197 198 199 ++ 200 201 202 203 204 205 206 207 ++ 208 209 210 211 212 213 214 215 ++ 216 217 218 219 220 221 222 223 ++ 224 225 226 227 228 229 230 231 ++ 232 233 234 235 236 237 238 239 ++ 240 241 242 243 244 245 246 247 ++ 248 249 250 251 252 253 254 255>; ++ default-brightness-level = <200>; ++ }; ++ ++ charge-animation { ++ compatible = "rockchip,uboot-charge"; ++ rockchip,uboot-charge-on = <0>; ++ rockchip,android-charge-on = <1>; ++ rockchip,uboot-low-power-voltage = <3500>; ++ rockchip,screen-on-voltage = <3600>; ++ status = "okay"; ++ }; ++ ++ rk809-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,rk809-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,widgets = ++ "Microphone", "Mic Jack", ++ "Headphone", "Headphone Jack"; ++ simple-audio-card,routing = ++ "Mic Jack", "MICBIAS1", ++ "IN1P", "Mic Jack", ++ "Headphone Jack", "HPOL", ++ "Headphone Jack", "HPOR"; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s1_2ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&rk809_codec>; ++ }; ++ }; ++ ++ rk_headset: rk-headset { ++ compatible = "rockchip_headset"; ++ headset_gpio = <&gpio2 RK_PB0 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hp_det>; ++ io-channels = <&saradc 1>; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ /*clocks = <&rk809 1>;*/ ++ /*clock-names = "ext_clock";*/ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_LOW>; /* GPIO3_A4 */ ++ }; ++ ++ vcc_phy: vcc-phy-regulator { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_phy"; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ vcc5v0_sys: vccsys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc5v0_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ wifi_chip_type = "AP6210"; ++ WIFI,host_wake_irq = <&gpio0 RK_PB2 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&rk809 1>; ++ clock-names = "ext_clock"; ++ uart_rts_gpios = <&gpio1 RK_PC3 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default","rts_gpio"; ++ pinctrl-0 = <&uart1_rts>; ++ pinctrl-1 = <&uart1_rts_gpio>; ++ BT,reset_gpio = <&gpio0 RK_PA1 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio2 RK_PB0 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio0 RK_PB3 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++}; ++ ++&display_subsystem { ++ status = "okay"; ++}; ++ ++&dsi { ++ status = "okay"; ++ ++ panel@0 { ++ compatible = "sitronix,st7703", "simple-panel-dsi"; ++ reg = <0>; ++ power-supply = <&vcc3v3_lcd>; ++ backlight = <&backlight>; ++ prepare-delay-ms = <2>; ++ reset-delay-ms = <1>; ++ init-delay-ms = <20>; ++ enable-delay-ms = <120>; ++ disable-delay-ms = <50>; ++ unprepare-delay-ms = <20>; ++ ++ width-mm = <68>; ++ height-mm = <121>; ++ ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | ++ MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ ++ panel-init-sequence = [ ++ 05 fa 01 11 ++ 39 00 04 b9 f1 12 83 ++ 39 00 1c ba 33 81 05 f9 0e 0e 00 00 00 ++ 00 00 00 00 00 44 25 00 91 0a ++ 00 00 02 4f 01 00 00 37 ++ 15 00 02 b8 25 ++ 39 00 04 bf 02 11 00 ++ 39 00 0b b3 0c 10 0a 50 03 ff 00 00 00 ++ 00 ++ 39 00 0a c0 73 73 50 50 00 00 08 70 00 ++ 15 00 02 bc 46 ++ 15 00 02 cc 0b ++ 15 00 02 b4 80 ++ 39 00 04 b2 c8 12 30 ++ 39 00 0f e3 07 07 0b 0b 03 0b 00 00 00 ++ 00 ff 00 c0 10 ++ 39 00 0d c1 53 00 1e 1e 77 e1 cc dd 67 ++ 77 33 33 ++ 39 00 07 c6 00 00 ff ff 01 ff ++ 39 00 03 b5 09 09 ++ 39 00 03 b6 87 95 ++ 39 00 40 e9 c2 10 05 05 10 05 a0 12 31 ++ 23 3f 81 0a a0 37 18 00 80 01 ++ 00 00 00 00 80 01 00 00 00 48 ++ f8 86 42 08 88 88 80 88 88 88 ++ 58 f8 87 53 18 88 88 81 88 88 ++ 88 00 00 00 01 00 00 00 00 00 ++ 00 00 00 00 ++ 39 00 3e ea 00 1a 00 00 00 00 02 00 00 ++ 00 00 00 1f 88 81 35 78 88 88 ++ 85 88 88 88 0f 88 80 24 68 88 ++ 88 84 88 88 88 23 10 00 00 1c ++ 00 00 00 00 00 00 00 00 00 00 ++ 00 00 00 00 00 30 05 a0 00 00 ++ 00 00 ++ 39 00 23 e0 00 06 08 2a 31 3f 38 36 07 ++ 0c 0d 11 13 12 13 11 18 00 06 ++ 08 2a 31 3f 38 36 07 0c 0d 11 ++ 13 12 13 11 18 ++ 05 32 01 29 ++ ]; ++ ++ panel-exit-sequence = [ ++ 05 00 01 28 ++ 05 00 01 10 ++ ]; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <64000000>; ++ hactive = <720>; ++ vactive = <1280>; ++ hfront-porch = <40>; ++ hsync-len = <10>; ++ hback-porch = <40>; ++ vfront-porch = <22>; ++ vsync-len = <4>; ++ vback-porch = <11>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ panel_in_dsi: endpoint { ++ remote-endpoint = <&dsi_out_panel>; ++ }; ++ }; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@1 { ++ reg = <1>; ++ dsi_out_panel: endpoint { ++ remote-endpoint = <&panel_in_dsi>; ++ }; ++ }; ++ }; ++}; ++ ++&dsi_in_vopb { ++ status = "okay"; ++}; ++ ++&dsi_in_vopl { ++ status = "disabled"; ++}; ++ ++&route_dsi { ++ connect = <&vopb_out_dsi>; ++ status = "okay"; ++}; ++ ++&bus_apll { ++ bus-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&cpu0 { ++ cpu-supply = <&vdd_arm>; ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&dmc { ++ center-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&emmc { ++ bus-width = <8>; ++ cap-mmc-highspeed; ++ mmc-hs200-1_8v; ++ no-sdio; ++ no-sd; ++ disable-wp; ++ non-removable; ++ num-slots = <1>; ++ status = "okay"; ++}; ++ ++&gmac { ++ phy-supply = <&vcc_phy>; ++ clock_in_out = "output"; ++ snps,reset-gpio = <&gpio2 13 GPIO_ACTIVE_LOW>; ++ snps,reset-active-low; ++ snps,reset-delays-us = <0 50000 50000>; ++ status = "okay"; ++}; ++ ++&gpu { ++ mali-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ ++ rk809: pmic@20 { ++ compatible = "rockchip,rk809"; ++ reg = <0x20>; ++ interrupt-parent = <&gpio0>; ++ interrupts = <7 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default", "pmic-sleep", ++ "pmic-power-off", "pmic-reset"; ++ pinctrl-0 = <&pmic_int>; ++ pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; ++ pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; ++ pinctrl-3 = <&soc_slppin_rst>, <&rk817_slppin_rst>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ //fb-inner-reg-idxs = <2>; ++ /* 1: rst regs (default in codes), 0: rst the pmic */ ++ pmic-reset-func = <1>; ++ ++ vcc1-supply = <&vcc5v0_sys>; ++ vcc2-supply = <&vcc5v0_sys>; ++ vcc3-supply = <&vcc5v0_sys>; ++ vcc4-supply = <&vcc5v0_sys>; ++ vcc5-supply = <&vcc3v3_sys>; ++ vcc6-supply = <&vcc3v3_sys>; ++ vcc7-supply = <&vcc3v3_sys>; ++ vcc8-supply = <&vcc3v3_sys>; ++ vcc9-supply = <&vcc5v0_sys>; ++ ++ pwrkey { ++ status = "okay"; ++ }; ++ ++ pinctrl_rk8xx: pinctrl_rk8xx { ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ rk817_slppin_null: rk817_slppin_null { ++ pins = "gpio_slp"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_slppin_slp: rk817_slppin_slp { ++ pins = "gpio_slp"; ++ function = "pin_fun1"; ++ }; ++ ++ rk817_slppin_pwrdn: rk817_slppin_pwrdn { ++ pins = "gpio_slp"; ++ function = "pin_fun2"; ++ }; ++ ++ rk817_slppin_rst: rk817_slppin_rst { ++ pins = "gpio_slp"; ++ function = "pin_fun3"; ++ }; ++ }; ++ ++ regulators { ++ vdd_logic: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <950000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_logic"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <950000>; ++ }; ++ }; ++ ++ vdd_arm: DCDC_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <950000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_arm"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <950000>; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc_ddr"; ++ regulator-initial-mode = <0x2>; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_3v0: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_3v0"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ ++ vcc_1v0: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ regulator-name = "vcc_1v0"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc1v8_soc: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcc1v8_soc"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd1v0_soc: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ ++ regulator-name = "vcc1v0_soc"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc3v0_pmu: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ ++ regulator-name = "vcc3v0_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ ++ }; ++ }; ++ ++ vccio_sd: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vccio_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc_sd: LDO_REG6 { ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-boot-on; ++ ++ regulator-name = "vcc_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ ++ }; ++ }; ++ ++ vcc2v8_dvp: LDO_REG7 { ++ regulator-boot-on; ++ regulator-min-microvolt = <2800000>; ++ regulator-max-microvolt = <2800000>; ++ ++ regulator-name = "vcc2v8_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <2800000>; ++ }; ++ }; ++ ++ vcc1v8_dvp: LDO_REG8 { ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcc1v8_dvp"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd1v5_dvp: LDO_REG9 { ++ regulator-boot-on; ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ ++ regulator-name = "vdd1v5_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <1500000>; ++ }; ++ }; ++ ++ vcc3v3_sys: DCDC_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcc3v3_sys"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc5v0_host: SWITCH_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc5v0_host"; ++ }; ++ ++ vcc3v3_lcd: SWITCH_REG2 { ++ regulator-boot-on; ++ regulator-name = "vcc3v3_lcd"; ++ }; ++ }; ++ ++ rk809_codec: codec { ++ #sound-dai-cells = <0>; ++ compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; ++ clocks = <&cru SCLK_I2S1_OUT>; ++ clock-names = "mclk"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1_2ch_mclk>; ++ hp-volume = <20>; ++ spk-volume = <3>; ++ status = "okay"; ++ }; ++ }; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ ++ sensor@f { ++ status = "okay"; ++ compatible = "ak8963"; ++ reg = <0x0f>; ++ type = ; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ layout = <1>; ++ reprobe_en = <1>; ++ }; ++ ++ gt1x: gt1x@14 { ++ compatible = "goodix,gt1x"; ++ reg = <0x14>; ++ power-supply = <&vcc3v3_lcd>; ++ goodix,rst-gpio = <&gpio0 RK_PB4 GPIO_ACTIVE_HIGH>; ++ goodix,irq-gpio = <&gpio0 RK_PA5 IRQ_TYPE_LEVEL_LOW>; ++ }; ++ ++ sensor@4c { ++ status = "okay"; ++ compatible = "gs_mma7660"; ++ reg = <0x4c>; ++ type = ; ++ irq-gpio = <&gpio0 RK_PB7 IRQ_TYPE_LEVEL_LOW>; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ layout = <2>; ++ reprobe_en = <1>; ++ }; ++}; ++ ++&i2c2 { ++ status = "okay"; ++ ++ clock-frequency = <100000>; ++ ++ /* These are relatively safe rise/fall times; TODO: measure */ ++ i2c-scl-falling-time-ns = <50>; ++ i2c-scl-rising-time-ns = <300>; ++ ++ ov5695: ov5695@36 { ++ compatible = "ovti,ov5695"; ++ reg = <0x36>; ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ ++ avdd-supply = <&vcc2v8_dvp>; ++ dovdd-supply = <&vcc1v8_dvp>; ++ dvdd-supply = <&vdd1v5_dvp>; ++ ++ /*reset-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>;*/ ++ pwdn-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_clkout_m0>; ++ port { ++ ucam_out: endpoint { ++ remote-endpoint = <&mipi_in_ucam>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++}; ++ ++&i2s1_2ch { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++}; ++ ++&io_domains { ++ status = "okay"; ++ ++ vccio1-supply = <&vcc_3v0>; ++ vccio2-supply = <&vccio_sd>; ++ vccio3-supply = <&vcc_3v0>; ++ vccio4-supply = <&vcc3v0_pmu>; ++ vccio5-supply = <&vcc_3v0>; ++}; ++ ++&isp_mmu { ++ status = "okay"; ++}; ++ ++&mipi_dphy_rx0 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ucam_out>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_rx0_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp0_mipi_in>; ++ }; ++ }; ++ }; ++}; ++ ++&nandc0 { ++ status = "okay"; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ ++ pmuio1-supply = <&vcc3v0_pmu>; ++ pmuio2-supply = <&vcc3v0_pmu>; ++}; ++ ++&pwm1 { ++ status = "okay"; ++}; ++ ++&rk_rga { ++ status = "okay"; ++}; ++ ++&rkisp1 { ++ status = "okay"; ++ ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp0_mipi_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dphy_rx0_out>; ++ }; ++ }; ++}; ++ ++&rockchip_suspend { ++ status = "okay"; ++ rockchip,sleep-debug-en = <1>; ++}; ++ ++&saradc { ++ status = "okay"; ++ vref-supply = <&vcc1v8_soc>; ++}; ++ ++&sdmmc { ++ bus-width = <4>; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ no-sdio; ++ no-mmc; ++ card-detect-delay = <800>; ++ ignore-pm-notify; ++ /*cd-gpios = <&gpio2 4 GPIO_ACTIVE_HIGH>; [> CD GPIO <]*/ ++ sd-uhs-sdr12; ++ sd-uhs-sdr25; ++ sd-uhs-sdr50; ++ sd-uhs-sdr104; ++ vqmmc-supply = <&vccio_sd>; ++ vmmc-supply = <&vcc_sd>; ++ status = "okay"; ++}; ++ ++&sdio { ++ bus-width = <4>; ++ cap-sd-highspeed; ++ no-sd; ++ no-mmc; ++ ignore-pm-notify; ++ keep-power-in-suspend; ++ non-removable; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&tsadc { ++ pinctrl-names = "gpio", "otpout"; ++ pinctrl-0 = <&tsadc_otp_gpio>; ++ pinctrl-1 = <&tsadc_otp_out>; ++ status = "okay"; ++}; ++ ++&uart1 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart1_xfer &uart1_cts>; ++ status = "okay"; ++}; ++ ++&u2phy { ++ status = "okay"; ++ ++ u2phy_host: host-port { ++ status = "okay"; ++ }; ++ ++ u2phy_otg: otg-port { ++ status = "okay"; ++ }; ++}; ++ ++&usb20_otg { ++ status = "okay"; ++}; ++ ++&usb_host0_ehci { ++ status = "okay"; ++}; ++ ++&usb_host0_ohci { ++ status = "okay"; ++}; ++ ++&vopb { ++ status = "okay"; ++}; ++ ++&vopb_mmu { ++ status = "okay"; ++}; ++ ++&vopl { ++ status = "okay"; ++}; ++ ++&vopl_mmu { ++ status = "okay"; ++}; ++ ++&mpp_srv { ++ status = "okay"; ++}; ++ ++&vdpu { ++ status = "okay"; ++}; ++ ++&vepu { ++ status = "okay"; ++}; ++ ++&vpu_mmu { ++ status = "okay"; ++}; ++ ++&hevc { ++ status = "okay"; ++}; ++ ++&hevc_mmu { ++ status = "okay"; ++}; ++ ++&pinctrl { ++ headphone { ++ hp_det: hp-det { ++ rockchip,pins = <2 RK_PB0 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ pmic { ++ pmic_int: pmic_int { ++ rockchip,pins = ++ <0 RK_PA7 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ soc_slppin_gpio: soc_slppin_gpio { ++ rockchip,pins = ++ <0 RK_PA4 RK_FUNC_GPIO &pcfg_output_low>; ++ }; ++ ++ soc_slppin_slp: soc_slppin_slp { ++ rockchip,pins = ++ <0 RK_PA4 1 &pcfg_pull_none>; ++ }; ++ ++ soc_slppin_rst: soc_slppin_rst { ++ rockchip,pins = ++ <0 RK_PA4 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ ++/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ ++/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ ++ +diff --git a/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v10-robot-linux.dts b/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v10-robot-linux.dts +new file mode 100755 +index 000000000000..317b53b2a1eb +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v10-robot-linux.dts +@@ -0,0 +1,627 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++#include ++#include ++#include ++#include "px30-robot.dtsi" ++ ++/ { ++ model = "Rockchip linux PX30 evb ddr3 board"; ++ compatible = "rockchip,px30-evb-ddr3-v10-linux", "rockchip,px30"; ++ ++ adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 2>; ++ io-channel-names = "buttons"; ++ poll-interval = <100>; ++ keyup-threshold-microvolt = <1800000>; ++ ++ esc-key { ++ linux,code = ; ++ label = "esc"; ++ press-threshold-microvolt = <1310000>; ++ }; ++ ++ home-key { ++ linux,code = ; ++ label = "home"; ++ press-threshold-microvolt = <624000>; ++ }; ++ ++ menu-key { ++ linux,code = ; ++ label = "menu"; ++ press-threshold-microvolt = <987000>; ++ }; ++ ++ vol-down-key { ++ linux,code = ; ++ label = "volume down"; ++ press-threshold-microvolt = <300000>; ++ }; ++ ++ vol-up-key { ++ linux,code = ; ++ label = "volume up"; ++ press-threshold-microvolt = <17000>; ++ }; ++ }; ++ ++ rk809-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,rk809-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s1_2ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&rk809_codec>; ++ }; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ /*clocks = <&rk809 1>;*/ ++ /*clock-names = "ext_clock";*/ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_LOW>; /* GPIO3_A4 */ ++ }; ++ ++ vcc_phy: vcc-phy-regulator { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_phy"; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ vcc5v0_sys: vccsys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc5v0_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ wifi_chip_type = "AP6210"; ++ WIFI,host_wake_irq = <&gpio0 RK_PB2 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++}; ++ ++&bus_apll { ++ bus-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&cpu0 { ++ cpu-supply = <&vdd_arm>; ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&dmc { ++ center-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&emmc { ++ bus-width = <8>; ++ cap-mmc-highspeed; ++ mmc-hs200-1_8v; ++ no-sdio; ++ no-sd; ++ disable-wp; ++ non-removable; ++ num-slots = <1>; ++ status = "okay"; ++}; ++ ++&gpu { ++ mali-supply = <&vdd_logic>; ++ status = "disabled"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ ++ rk809: pmic@20 { ++ compatible = "rockchip,rk809"; ++ reg = <0x20>; ++ interrupt-parent = <&gpio0>; ++ interrupts = <7 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default", "pmic-sleep", ++ "pmic-power-off", "pmic-reset"; ++ pinctrl-0 = <&pmic_int>; ++ pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; ++ pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; ++ pinctrl-3 = <&soc_slppin_rst>, <&rk817_slppin_rst>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ //fb-inner-reg-idxs = <2>; ++ /* 1: rst regs (default in codes), 0: rst the pmic */ ++ pmic-reset-func = <1>; ++ ++ vcc1-supply = <&vcc5v0_sys>; ++ vcc2-supply = <&vcc5v0_sys>; ++ vcc3-supply = <&vcc5v0_sys>; ++ vcc4-supply = <&vcc5v0_sys>; ++ vcc5-supply = <&vcc3v3_sys>; ++ vcc6-supply = <&vcc3v3_sys>; ++ vcc7-supply = <&vcc3v3_sys>; ++ vcc8-supply = <&vcc3v3_sys>; ++ vcc9-supply = <&vcc5v0_sys>; ++ ++ pwrkey { ++ status = "okay"; ++ }; ++ ++ pinctrl_rk8xx: pinctrl_rk8xx { ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ rk817_slppin_null: rk817_slppin_null { ++ pins = "gpio_slp"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_slppin_slp: rk817_slppin_slp { ++ pins = "gpio_slp"; ++ function = "pin_fun1"; ++ }; ++ ++ rk817_slppin_pwrdn: rk817_slppin_pwrdn { ++ pins = "gpio_slp"; ++ function = "pin_fun2"; ++ }; ++ ++ rk817_slppin_rst: rk817_slppin_rst { ++ pins = "gpio_slp"; ++ function = "pin_fun3"; ++ }; ++ }; ++ ++ regulators { ++ vdd_logic: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <950000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_logic"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <950000>; ++ }; ++ }; ++ ++ vdd_arm: DCDC_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <950000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_arm"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <950000>; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc_ddr"; ++ regulator-initial-mode = <0x2>; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_3v0: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_3v0"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ ++ vcc_1v0: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ regulator-name = "vcc_1v0"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc1v8_soc: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcc1v8_soc"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd1v0_soc: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ ++ regulator-name = "vcc1v0_soc"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc3v0_pmu: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ ++ regulator-name = "vcc3v0_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ ++ }; ++ }; ++ ++ vccio_sd: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vccio_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc_sd: LDO_REG6 { ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-boot-on; ++ ++ regulator-name = "vcc_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ ++ }; ++ }; ++ ++ vcc2v8_dvp: LDO_REG7 { ++ regulator-min-microvolt = <2800000>; ++ regulator-max-microvolt = <2800000>; ++ ++ regulator-name = "vcc2v8_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <2800000>; ++ }; ++ }; ++ ++ vcc1v8_dvp: LDO_REG8 { ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcc1v8_dvp"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd1v5_dvp: LDO_REG9 { ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ ++ regulator-name = "vdd1v5_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <1500000>; ++ }; ++ }; ++ ++ vcc3v3_sys: DCDC_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcc3v3_sys"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc5v0_host: SWITCH_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc5v0_host"; ++ }; ++ ++ vcc3v3_lcd: SWITCH_REG2 { ++ regulator-boot-on; ++ regulator-name = "vcc3v3_lcd"; ++ }; ++ }; ++ ++ rk809_codec: codec { ++ #sound-dai-cells = <0>; ++ compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; ++ clocks = <&cru SCLK_I2S1_OUT>; ++ clock-names = "mclk"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1_2ch_mclk>; ++ hp-volume = <20>; ++ spk-volume = <3>; ++ status = "okay"; ++ }; ++ }; ++}; ++ ++&i2c2 { ++ status = "okay"; ++ ++ clock-frequency = <100000>; ++ pinctrl-0 = <&i2c2_xfer>; ++ ++ /* These are relatively safe rise/fall times; TODO: measure */ ++ i2c-scl-falling-time-ns = <50>; ++ i2c-scl-rising-time-ns = <300>; ++ ++ ov5695: ov5695@36 { ++ compatible = "ovti,ov5695"; ++ reg = <0x36>; ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ /*reset-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>;*/ ++ ++ avdd-supply = <&vcc2v8_dvp>; ++ dovdd-supply = <&vcc1v8_dvp>; ++ dvdd-supply = <&vdd1v5_dvp>; ++ ++ pwdn-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_clkout_m0>; ++ port { ++ ucam_out: endpoint { ++ remote-endpoint = <&mipi_in_ucam>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++}; ++ ++&i2s1_2ch { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++}; ++ ++&io_domains { ++ status = "okay"; ++ ++ vccio1-supply = <&vcc_3v0>; ++ vccio2-supply = <&vccio_sd>; ++ vccio3-supply = <&vcc_3v0>; ++ vccio4-supply = <&vcc3v0_pmu>; ++ vccio5-supply = <&vcc_3v0>; ++}; ++ ++&isp_mmu { ++ status = "okay"; ++}; ++ ++&mipi_dphy_rx0 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ucam_out>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_rx0_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp0_mipi_in>; ++ }; ++ }; ++ }; ++}; ++ ++&nandc0 { ++ status = "okay"; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ ++ pmuio1-supply = <&vcc3v0_pmu>; ++ pmuio2-supply = <&vcc3v0_pmu>; ++}; ++ ++&rk_rga { ++ status = "okay"; ++}; ++ ++&rkisp1 { ++ status = "okay"; ++ ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp0_mipi_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dphy_rx0_out>; ++ }; ++ }; ++}; ++ ++&saradc { ++ status = "okay"; ++ vref-supply = <&vcc1v8_soc>; ++}; ++ ++&sdmmc { ++ bus-width = <4>; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ no-sdio; ++ no-mmc; ++ card-detect-delay = <800>; ++ ignore-pm-notify; ++ /*cd-gpios = <&gpio2 4 GPIO_ACTIVE_HIGH>; [> CD GPIO <]*/ ++ sd-uhs-sdr12; ++ sd-uhs-sdr25; ++ sd-uhs-sdr50; ++ sd-uhs-sdr104; ++ vqmmc-supply = <&vccio_sd>; ++ vmmc-supply = <&vcc_sd>; ++ status = "disabled"; ++}; ++ ++&sdio { ++ bus-width = <4>; ++ cap-sd-highspeed; ++ no-sd; ++ no-mmc; ++ ignore-pm-notify; ++ keep-power-in-suspend; ++ non-removable; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&mpp_srv { ++ status = "okay"; ++}; ++ ++&vdpu { ++ status = "okay"; ++}; ++ ++&vepu { ++ status = "okay"; ++}; ++ ++&vpu_mmu { ++ status = "okay"; ++}; ++ ++&hevc { ++ status = "okay"; ++}; ++ ++&hevc_mmu { ++ status = "okay"; ++}; ++ ++&uart1 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart1_xfer &uart1_cts>; ++ status = "okay"; ++}; ++ ++&u2phy { ++ status = "okay"; ++ ++ u2phy_otg: otg-port { ++ status = "okay"; ++ }; ++}; ++ ++&usb20_otg { ++ status = "okay"; ++}; ++ ++&pinctrl { ++ pmic { ++ pmic_int: pmic_int { ++ rockchip,pins = ++ <0 RK_PA7 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ soc_slppin_gpio: soc_slppin_gpio { ++ rockchip,pins = ++ <0 RK_PA4 RK_FUNC_GPIO &pcfg_output_low>; ++ }; ++ ++ soc_slppin_slp: soc_slppin_slp { ++ rockchip,pins = ++ <0 RK_PA4 1 &pcfg_pull_none>; ++ }; ++ ++ soc_slppin_rst: soc_slppin_rst { ++ rockchip,pins = ++ <0 RK_PA4 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ ++/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ ++/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ +diff --git a/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v10-robot-no-gpu-linux.dts b/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v10-robot-no-gpu-linux.dts +new file mode 100755 +index 000000000000..81f44c36686c +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v10-robot-no-gpu-linux.dts +@@ -0,0 +1,627 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++#include ++#include ++#include ++#include "px30-robot-no-gpu.dtsi" ++ ++/ { ++ model = "Rockchip linux PX30 evb ddr3 board"; ++ compatible = "rockchip,px30-evb-ddr3-v10-linux", "rockchip,px30"; ++ ++ adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 2>; ++ io-channel-names = "buttons"; ++ poll-interval = <100>; ++ keyup-threshold-microvolt = <1800000>; ++ ++ esc-key { ++ linux,code = ; ++ label = "esc"; ++ press-threshold-microvolt = <1310000>; ++ }; ++ ++ home-key { ++ linux,code = ; ++ label = "home"; ++ press-threshold-microvolt = <624000>; ++ }; ++ ++ menu-key { ++ linux,code = ; ++ label = "menu"; ++ press-threshold-microvolt = <987000>; ++ }; ++ ++ vol-down-key { ++ linux,code = ; ++ label = "volume down"; ++ press-threshold-microvolt = <300000>; ++ }; ++ ++ vol-up-key { ++ linux,code = ; ++ label = "volume up"; ++ press-threshold-microvolt = <17000>; ++ }; ++ }; ++ ++ rk809-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,rk809-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s1_2ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&rk809_codec>; ++ }; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ /*clocks = <&rk809 1>;*/ ++ /*clock-names = "ext_clock";*/ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_LOW>; /* GPIO3_A4 */ ++ }; ++ ++ vcc_phy: vcc-phy-regulator { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_phy"; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ vcc5v0_sys: vccsys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc5v0_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ wifi_chip_type = "AP6210"; ++ WIFI,host_wake_irq = <&gpio0 RK_PB2 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++}; ++ ++&bus_apll { ++ bus-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&cpu0 { ++ cpu-supply = <&vdd_arm>; ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&dmc { ++ center-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&emmc { ++ bus-width = <8>; ++ cap-mmc-highspeed; ++ mmc-hs200-1_8v; ++ no-sdio; ++ no-sd; ++ disable-wp; ++ non-removable; ++ num-slots = <1>; ++ status = "okay"; ++}; ++ ++&gpu { ++ mali-supply = <&vdd_logic>; ++ status = "disabled"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ ++ rk809: pmic@20 { ++ compatible = "rockchip,rk809"; ++ reg = <0x20>; ++ interrupt-parent = <&gpio0>; ++ interrupts = <7 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default", "pmic-sleep", ++ "pmic-power-off", "pmic-reset"; ++ pinctrl-0 = <&pmic_int>; ++ pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; ++ pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; ++ pinctrl-3 = <&soc_slppin_rst>, <&rk817_slppin_rst>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ //fb-inner-reg-idxs = <2>; ++ /* 1: rst regs (default in codes), 0: rst the pmic */ ++ pmic-reset-func = <1>; ++ ++ vcc1-supply = <&vcc5v0_sys>; ++ vcc2-supply = <&vcc5v0_sys>; ++ vcc3-supply = <&vcc5v0_sys>; ++ vcc4-supply = <&vcc5v0_sys>; ++ vcc5-supply = <&vcc3v3_sys>; ++ vcc6-supply = <&vcc3v3_sys>; ++ vcc7-supply = <&vcc3v3_sys>; ++ vcc8-supply = <&vcc3v3_sys>; ++ vcc9-supply = <&vcc5v0_sys>; ++ ++ pwrkey { ++ status = "okay"; ++ }; ++ ++ pinctrl_rk8xx: pinctrl_rk8xx { ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ rk817_slppin_null: rk817_slppin_null { ++ pins = "gpio_slp"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_slppin_slp: rk817_slppin_slp { ++ pins = "gpio_slp"; ++ function = "pin_fun1"; ++ }; ++ ++ rk817_slppin_pwrdn: rk817_slppin_pwrdn { ++ pins = "gpio_slp"; ++ function = "pin_fun2"; ++ }; ++ ++ rk817_slppin_rst: rk817_slppin_rst { ++ pins = "gpio_slp"; ++ function = "pin_fun3"; ++ }; ++ }; ++ ++ regulators { ++ vdd_logic: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <950000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_logic"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <950000>; ++ }; ++ }; ++ ++ vdd_arm: DCDC_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <950000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_arm"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <950000>; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc_ddr"; ++ regulator-initial-mode = <0x2>; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_3v0: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_3v0"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ ++ vcc_1v0: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ regulator-name = "vcc_1v0"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc1v8_soc: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcc1v8_soc"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd1v0_soc: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ ++ regulator-name = "vcc1v0_soc"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc3v0_pmu: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ ++ regulator-name = "vcc3v0_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ ++ }; ++ }; ++ ++ vccio_sd: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vccio_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc_sd: LDO_REG6 { ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-boot-on; ++ ++ regulator-name = "vcc_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ ++ }; ++ }; ++ ++ vcc2v8_dvp: LDO_REG7 { ++ regulator-min-microvolt = <2800000>; ++ regulator-max-microvolt = <2800000>; ++ ++ regulator-name = "vcc2v8_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <2800000>; ++ }; ++ }; ++ ++ vcc1v8_dvp: LDO_REG8 { ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcc1v8_dvp"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd1v5_dvp: LDO_REG9 { ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ ++ regulator-name = "vdd1v5_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <1500000>; ++ }; ++ }; ++ ++ vcc3v3_sys: DCDC_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcc3v3_sys"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc5v0_host: SWITCH_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc5v0_host"; ++ }; ++ ++ vcc3v3_lcd: SWITCH_REG2 { ++ regulator-boot-on; ++ regulator-name = "vcc3v3_lcd"; ++ }; ++ }; ++ ++ rk809_codec: codec { ++ #sound-dai-cells = <0>; ++ compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; ++ clocks = <&cru SCLK_I2S1_OUT>; ++ clock-names = "mclk"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1_2ch_mclk>; ++ hp-volume = <20>; ++ spk-volume = <3>; ++ status = "okay"; ++ }; ++ }; ++}; ++ ++&i2c2 { ++ status = "okay"; ++ ++ clock-frequency = <100000>; ++ pinctrl-0 = <&i2c2_xfer>; ++ ++ /* These are relatively safe rise/fall times; TODO: measure */ ++ i2c-scl-falling-time-ns = <50>; ++ i2c-scl-rising-time-ns = <300>; ++ ++ ov5695: ov5695@36 { ++ compatible = "ovti,ov5695"; ++ reg = <0x36>; ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ /*reset-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>;*/ ++ ++ avdd-supply = <&vcc2v8_dvp>; ++ dovdd-supply = <&vcc1v8_dvp>; ++ dvdd-supply = <&vdd1v5_dvp>; ++ ++ pwdn-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_clkout_m0>; ++ port { ++ ucam_out: endpoint { ++ remote-endpoint = <&mipi_in_ucam>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++}; ++ ++&i2s1_2ch { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++}; ++ ++&io_domains { ++ status = "okay"; ++ ++ vccio1-supply = <&vcc_3v0>; ++ vccio2-supply = <&vccio_sd>; ++ vccio3-supply = <&vcc_3v0>; ++ vccio4-supply = <&vcc3v0_pmu>; ++ vccio5-supply = <&vcc_3v0>; ++}; ++ ++&isp_mmu { ++ status = "okay"; ++}; ++ ++&mipi_dphy_rx0 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ucam_out>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_rx0_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp0_mipi_in>; ++ }; ++ }; ++ }; ++}; ++ ++&nandc0 { ++ status = "okay"; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ ++ pmuio1-supply = <&vcc3v0_pmu>; ++ pmuio2-supply = <&vcc3v0_pmu>; ++}; ++ ++&rk_rga { ++ status = "okay"; ++}; ++ ++&rkisp1 { ++ status = "okay"; ++ ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp0_mipi_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dphy_rx0_out>; ++ }; ++ }; ++}; ++ ++&saradc { ++ status = "okay"; ++ vref-supply = <&vcc1v8_soc>; ++}; ++ ++&sdmmc { ++ bus-width = <4>; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ no-sdio; ++ no-mmc; ++ card-detect-delay = <800>; ++ ignore-pm-notify; ++ /*cd-gpios = <&gpio2 4 GPIO_ACTIVE_HIGH>; [> CD GPIO <]*/ ++ sd-uhs-sdr12; ++ sd-uhs-sdr25; ++ sd-uhs-sdr50; ++ sd-uhs-sdr104; ++ vqmmc-supply = <&vccio_sd>; ++ vmmc-supply = <&vcc_sd>; ++ status = "disabled"; ++}; ++ ++&sdio { ++ bus-width = <4>; ++ cap-sd-highspeed; ++ no-sd; ++ no-mmc; ++ ignore-pm-notify; ++ keep-power-in-suspend; ++ non-removable; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&mpp_srv { ++ status = "okay"; ++}; ++ ++&vdpu { ++ status = "okay"; ++}; ++ ++&vepu { ++ status = "okay"; ++}; ++ ++&vpu_mmu { ++ status = "okay"; ++}; ++ ++&hevc { ++ status = "okay"; ++}; ++ ++&hevc_mmu { ++ status = "okay"; ++}; ++ ++&uart1 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart1_xfer &uart1_cts>; ++ status = "okay"; ++}; ++ ++&u2phy { ++ status = "okay"; ++ ++ u2phy_otg: otg-port { ++ status = "okay"; ++ }; ++}; ++ ++&usb20_otg { ++ status = "okay"; ++}; ++ ++&pinctrl { ++ pmic { ++ pmic_int: pmic_int { ++ rockchip,pins = ++ <0 RK_PA7 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ soc_slppin_gpio: soc_slppin_gpio { ++ rockchip,pins = ++ <0 RK_PA4 RK_FUNC_GPIO &pcfg_output_low>; ++ }; ++ ++ soc_slppin_slp: soc_slppin_slp { ++ rockchip,pins = ++ <0 RK_PA4 1 &pcfg_pull_none>; ++ }; ++ ++ soc_slppin_rst: soc_slppin_rst { ++ rockchip,pins = ++ <0 RK_PA4 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ ++/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ ++/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ +diff --git a/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v10.dts b/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v10.dts +new file mode 100755 +index 000000000000..b814812edbe9 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v10.dts +@@ -0,0 +1,127 @@ ++/* ++ * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++ */ ++ ++/dts-v1/; ++#include "px30-evb-ddr3-v10.dtsi" ++ ++/ { ++ model = "Rockchip PX30 evb ddr3 board"; ++ compatible = "rockchip,px30-evb-ddr3-v10", "rockchip,px30"; ++}; ++ ++&dsi { ++ status = "okay"; ++ ++ panel@0 { ++ compatible = "sitronix,st7703", "simple-panel-dsi"; ++ reg = <0>; ++ power-supply = <&vcc3v3_lcd>; ++ backlight = <&backlight>; ++ prepare-delay-ms = <2>; ++ reset-delay-ms = <1>; ++ init-delay-ms = <20>; ++ enable-delay-ms = <120>; ++ disable-delay-ms = <50>; ++ unprepare-delay-ms = <20>; ++ ++ width-mm = <68>; ++ height-mm = <121>; ++ ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | ++ MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ ++ panel-init-sequence = [ ++ 05 fa 01 11 ++ 39 00 04 b9 f1 12 83 ++ 39 00 1c ba 33 81 05 f9 0e 0e 00 00 00 ++ 00 00 00 00 00 44 25 00 91 0a ++ 00 00 02 4f 01 00 00 37 ++ 15 00 02 b8 25 ++ 39 00 04 bf 02 11 00 ++ 39 00 0b b3 0c 10 0a 50 03 ff 00 00 00 ++ 00 ++ 39 00 0a c0 73 73 50 50 00 00 08 70 00 ++ 15 00 02 bc 46 ++ 15 00 02 cc 0b ++ 15 00 02 b4 80 ++ 39 00 04 b2 c8 12 30 ++ 39 00 0f e3 07 07 0b 0b 03 0b 00 00 00 ++ 00 ff 00 c0 10 ++ 39 00 0d c1 53 00 1e 1e 77 e1 cc dd 67 ++ 77 33 33 ++ 39 00 07 c6 00 00 ff ff 01 ff ++ 39 00 03 b5 09 09 ++ 39 00 03 b6 87 95 ++ 39 00 40 e9 c2 10 05 05 10 05 a0 12 31 ++ 23 3f 81 0a a0 37 18 00 80 01 ++ 00 00 00 00 80 01 00 00 00 48 ++ f8 86 42 08 88 88 80 88 88 88 ++ 58 f8 87 53 18 88 88 81 88 88 ++ 88 00 00 00 01 00 00 00 00 00 ++ 00 00 00 00 ++ 39 00 3e ea 00 1a 00 00 00 00 02 00 00 ++ 00 00 00 1f 88 81 35 78 88 88 ++ 85 88 88 88 0f 88 80 24 68 88 ++ 88 84 88 88 88 23 10 00 00 1c ++ 00 00 00 00 00 00 00 00 00 00 ++ 00 00 00 00 00 30 05 a0 00 00 ++ 00 00 ++ 39 00 23 e0 00 06 08 2a 31 3f 38 36 07 ++ 0c 0d 11 13 12 13 11 18 00 06 ++ 08 2a 31 3f 38 36 07 0c 0d 11 ++ 13 12 13 11 18 ++ 05 32 01 29 ++ ]; ++ ++ panel-exit-sequence = [ ++ 05 00 01 28 ++ 05 00 01 10 ++ ]; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <64000000>; ++ hactive = <720>; ++ vactive = <1280>; ++ hfront-porch = <40>; ++ hsync-len = <10>; ++ hback-porch = <40>; ++ vfront-porch = <22>; ++ vsync-len = <4>; ++ vback-porch = <11>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ }; ++}; ++ ++&firmware_android { ++ compatible = "android,firmware"; ++ fstab { ++ compatible = "android,fstab"; ++ system { ++ compatible = "android,system"; ++ dev = "/dev/block/by-name/system"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait"; ++ }; ++ vendor { ++ compatible = "android,vendor"; ++ dev = "/dev/block/by-name/vendor"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait"; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v10.dtsi b/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v10.dtsi +new file mode 100755 +index 000000000000..3d8e81009bef +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v10.dtsi +@@ -0,0 +1,815 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2017-2019 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++#include ++#include ++#include ++#include ++#include ++#include "px30.dtsi" ++#include "px30-android.dtsi" ++ ++/ { ++ adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 2>; ++ io-channel-names = "buttons"; ++ poll-interval = <100>; ++ keyup-threshold-microvolt = <1800000>; ++ ++ esc-key { ++ linux,code = ; ++ label = "esc"; ++ press-threshold-microvolt = <1310000>; ++ }; ++ ++ home-key { ++ linux,code = ; ++ label = "home"; ++ press-threshold-microvolt = <624000>; ++ }; ++ ++ menu-key { ++ linux,code = ; ++ label = "menu"; ++ press-threshold-microvolt = <987000>; ++ }; ++ ++ vol-down-key { ++ linux,code = ; ++ label = "volume down"; ++ press-threshold-microvolt = <300000>; ++ }; ++ ++ vol-up-key { ++ linux,code = ; ++ label = "volume up"; ++ press-threshold-microvolt = <17000>; ++ }; ++ }; ++ ++ backlight: backlight { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm1 0 25000 0>; ++ brightness-levels = < ++ 0 1 2 3 4 5 6 7 ++ 8 9 10 11 12 13 14 15 ++ 16 17 18 19 20 21 22 23 ++ 24 25 26 27 28 29 30 31 ++ 32 33 34 35 36 37 38 39 ++ 40 41 42 43 44 45 46 47 ++ 48 49 50 51 52 53 54 55 ++ 56 57 58 59 60 61 62 63 ++ 64 65 66 67 68 69 70 71 ++ 72 73 74 75 76 77 78 79 ++ 80 81 82 83 84 85 86 87 ++ 88 89 90 91 92 93 94 95 ++ 96 97 98 99 100 101 102 103 ++ 104 105 106 107 108 109 110 111 ++ 112 113 114 115 116 117 118 119 ++ 120 121 122 123 124 125 126 127 ++ 128 129 130 131 132 133 134 135 ++ 136 137 138 139 140 141 142 143 ++ 144 145 146 147 148 149 150 151 ++ 152 153 154 155 156 157 158 159 ++ 160 161 162 163 164 165 166 167 ++ 168 169 170 171 172 173 174 175 ++ 176 177 178 179 180 181 182 183 ++ 184 185 186 187 188 189 190 191 ++ 192 193 194 195 196 197 198 199 ++ 200 201 202 203 204 205 206 207 ++ 208 209 210 211 212 213 214 215 ++ 216 217 218 219 220 221 222 223 ++ 224 225 226 227 228 229 230 231 ++ 232 233 234 235 236 237 238 239 ++ 240 241 242 243 244 245 246 247 ++ 248 249 250 251 252 253 254 255>; ++ default-brightness-level = <200>; ++ }; ++ ++ charge-animation { ++ compatible = "rockchip,uboot-charge"; ++ rockchip,uboot-charge-on = <0>; ++ rockchip,android-charge-on = <1>; ++ rockchip,uboot-low-power-voltage = <3500>; ++ rockchip,screen-on-voltage = <3600>; ++ status = "okay"; ++ }; ++ ++ rk809-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,rk809-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,widgets = ++ "Microphone", "Mic Jack", ++ "Headphone", "Headphone Jack"; ++ simple-audio-card,routing = ++ "Mic Jack", "MICBIAS1", ++ "IN1P", "Mic Jack", ++ "Headphone Jack", "HPOL", ++ "Headphone Jack", "HPOR"; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s1_2ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&rk809_codec>; ++ }; ++ }; ++ ++ rk_headset: rk-headset { ++ compatible = "rockchip_headset"; ++ headset_gpio = <&gpio3 RK_PA4 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hp_det>; ++ io-channels = <&saradc 1>; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ /*clocks = <&rk809 1>;*/ ++ /*clock-names = "ext_clock";*/ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_LOW>; /* GPIO3_A4 */ ++ }; ++ ++ vcc_phy: vcc-phy-regulator { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_phy"; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ vcc5v0_sys: vccsys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc5v0_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ wifi_chip_type = "AP6210"; ++ WIFI,host_wake_irq = <&gpio0 RK_PB2 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&rk809 1>; ++ clock-names = "ext_clock"; ++ uart_rts_gpios = <&gpio1 RK_PC3 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default","rts_gpio"; ++ pinctrl-0 = <&uart1_rts>; ++ pinctrl-1 = <&uart1_rts_gpio>; ++ BT,reset_gpio = <&gpio0 RK_PA1 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio2 RK_PB0 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio0 RK_PB3 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++}; ++ ++&display_subsystem { ++ status = "okay"; ++}; ++ ++&dsi_in_vopb { ++ status = "okay"; ++}; ++ ++&dsi_in_vopl { ++ status = "disabled"; ++}; ++ ++&route_dsi { ++ connect = <&vopb_out_dsi>; ++ status = "okay"; ++}; ++ ++&bus_apll { ++ bus-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&cpu0 { ++ cpu-supply = <&vdd_arm>; ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&dmc { ++ center-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&emmc { ++ bus-width = <8>; ++ cap-mmc-highspeed; ++ mmc-hs200-1_8v; ++ no-sdio; ++ no-sd; ++ disable-wp; ++ non-removable; ++ num-slots = <1>; ++ status = "okay"; ++}; ++ ++&gmac { ++ phy-supply = <&vcc_phy>; ++ clock_in_out = "output"; ++ snps,reset-gpio = <&gpio2 13 GPIO_ACTIVE_LOW>; ++ snps,reset-active-low; ++ snps,reset-delays-us = <0 50000 50000>; ++ status = "okay"; ++}; ++ ++&gpu { ++ mali-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ ++ rk809: pmic@20 { ++ compatible = "rockchip,rk809"; ++ reg = <0x20>; ++ interrupt-parent = <&gpio0>; ++ interrupts = <7 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default", "pmic-sleep", ++ "pmic-power-off", "pmic-reset"; ++ pinctrl-0 = <&pmic_int>; ++ pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; ++ pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; ++ pinctrl-3 = <&soc_slppin_rst>, <&rk817_slppin_rst>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ //fb-inner-reg-idxs = <2>; ++ /* 1: rst regs (default in codes), 0: rst the pmic */ ++ pmic-reset-func = <1>; ++ ++ vcc1-supply = <&vcc5v0_sys>; ++ vcc2-supply = <&vcc5v0_sys>; ++ vcc3-supply = <&vcc5v0_sys>; ++ vcc4-supply = <&vcc5v0_sys>; ++ vcc5-supply = <&vcc3v3_sys>; ++ vcc6-supply = <&vcc3v3_sys>; ++ vcc7-supply = <&vcc3v3_sys>; ++ vcc8-supply = <&vcc3v3_sys>; ++ vcc9-supply = <&vcc5v0_sys>; ++ ++ pwrkey { ++ status = "okay"; ++ }; ++ ++ pinctrl_rk8xx: pinctrl_rk8xx { ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ rk817_slppin_null: rk817_slppin_null { ++ pins = "gpio_slp"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_slppin_slp: rk817_slppin_slp { ++ pins = "gpio_slp"; ++ function = "pin_fun1"; ++ }; ++ ++ rk817_slppin_pwrdn: rk817_slppin_pwrdn { ++ pins = "gpio_slp"; ++ function = "pin_fun2"; ++ }; ++ ++ rk817_slppin_rst: rk817_slppin_rst { ++ pins = "gpio_slp"; ++ function = "pin_fun3"; ++ }; ++ }; ++ ++ regulators { ++ vdd_logic: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <950000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_logic"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <950000>; ++ }; ++ }; ++ ++ vdd_arm: DCDC_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <950000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_arm"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <950000>; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc_ddr"; ++ regulator-initial-mode = <0x2>; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_3v0: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_3v0"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ ++ vcc_1v0: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ regulator-name = "vcc_1v0"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc1v8_soc: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcc1v8_soc"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd1v0_soc: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ ++ regulator-name = "vcc1v0_soc"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc3v0_pmu: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ ++ regulator-name = "vcc3v0_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ ++ }; ++ }; ++ ++ vccio_sd: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vccio_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc_sd: LDO_REG6 { ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vcc_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ ++ }; ++ }; ++ ++ vcc2v8_dvp: LDO_REG7 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <2800000>; ++ regulator-max-microvolt = <2800000>; ++ ++ regulator-name = "vcc2v8_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <2800000>; ++ }; ++ }; ++ ++ vcc1v8_dvp: LDO_REG8 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcc1v8_dvp"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd1v5_dvp: LDO_REG9 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ ++ regulator-name = "vdd1v5_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <1500000>; ++ }; ++ }; ++ ++ vcc3v3_sys: DCDC_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcc3v3_sys"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc5v0_host: SWITCH_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc5v0_host"; ++ }; ++ ++ vcc3v3_lcd: SWITCH_REG1 { ++ regulator-boot-on; ++ regulator-name = "vcc3v3_lcd"; ++ }; ++ }; ++ ++ rk809_codec: codec { ++ #sound-dai-cells = <0>; ++ compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; ++ clocks = <&cru SCLK_I2S1_OUT>; ++ clock-names = "mclk"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1_2ch_mclk>; ++ hp-volume = <20>; ++ spk-volume = <3>; ++ status = "okay"; ++ }; ++ }; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ ++ sensor@f { ++ status = "okay"; ++ compatible = "ak8963"; ++ reg = <0x0f>; ++ type = ; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ layout = <1>; ++ reprobe_en = <1>; ++ }; ++ ++ gt1x: gt1x@14 { ++ compatible = "goodix,gt1x"; ++ reg = <0x14>; ++ power-supply = <&vcc3v3_lcd>; ++ goodix,rst-gpio = <&gpio0 RK_PB4 GPIO_ACTIVE_HIGH>; ++ goodix,irq-gpio = <&gpio0 RK_PA5 IRQ_TYPE_LEVEL_LOW>; ++ }; ++ ++ sensor@4c { ++ status = "okay"; ++ compatible = "gs_mma7660"; ++ reg = <0x4c>; ++ type = ; ++ irq-gpio = <&gpio0 RK_PB7 IRQ_TYPE_LEVEL_LOW>; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ layout = <8>; ++ reprobe_en = <1>; ++ }; ++}; ++ ++&i2c2 { ++ status = "okay"; ++ ++ clock-frequency = <100000>; ++ ++ /* These are relatively safe rise/fall times; TODO: measure */ ++ i2c-scl-falling-time-ns = <50>; ++ i2c-scl-rising-time-ns = <300>; ++ ++ ov5695: ov5695@36 { ++ compatible = "ovti,ov5695"; ++ reg = <0x36>; ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ /*reset-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>;*/ ++ pwdn-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_clkout_m0>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "TongJu"; ++ rockchip,camera-module-lens-name = "CHT842-MD"; ++ port { ++ ucam_out: endpoint { ++ remote-endpoint = <&mipi_in_ucam>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++}; ++ ++&i2s1_2ch { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++}; ++ ++&io_domains { ++ status = "okay"; ++ ++ vccio1-supply = <&vcc1v8_soc>; ++ vccio2-supply = <&vccio_sd>; ++ vccio3-supply = <&vcc_3v0>; ++ vccio4-supply = <&vcc3v0_pmu>; ++ vccio5-supply = <&vcc_3v0>; ++}; ++ ++&isp_mmu { ++ status = "okay"; ++}; ++ ++&mipi_dphy_rx0 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ucam_out>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_rx0_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp0_mipi_in>; ++ }; ++ }; ++ }; ++}; ++ ++&nandc0 { ++ status = "okay"; ++}; ++ ++&pinctrl { ++ headphone { ++ hp_det: hp-det { ++ rockchip,pins = <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ pmic { ++ pmic_int: pmic_int { ++ rockchip,pins = ++ <0 RK_PA7 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ soc_slppin_gpio: soc_slppin_gpio { ++ rockchip,pins = ++ <0 RK_PA4 RK_FUNC_GPIO &pcfg_output_low>; ++ }; ++ ++ soc_slppin_slp: soc_slppin_slp { ++ rockchip,pins = ++ <0 RK_PA4 1 &pcfg_pull_none>; ++ }; ++ ++ soc_slppin_rst: soc_slppin_rst { ++ rockchip,pins = ++ <0 RK_PA4 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ ++ pmuio1-supply = <&vcc3v0_pmu>; ++ pmuio2-supply = <&vcc3v0_pmu>; ++}; ++ ++&pwm1 { ++ status = "okay"; ++}; ++ ++&rk_rga { ++ status = "okay"; ++}; ++ ++&rkisp1 { ++ status = "okay"; ++ ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp0_mipi_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dphy_rx0_out>; ++ }; ++ }; ++}; ++ ++&rockchip_suspend { ++ status = "okay"; ++ rockchip,sleep-debug-en = <1>; ++}; ++ ++&saradc { ++ status = "okay"; ++ vref-supply = <&vcc1v8_soc>; ++}; ++ ++&sdmmc { ++ bus-width = <4>; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ no-sdio; ++ no-mmc; ++ card-detect-delay = <800>; ++ ignore-pm-notify; ++ /*cd-gpios = <&gpio2 4 GPIO_ACTIVE_HIGH>; [> CD GPIO <]*/ ++ sd-uhs-sdr12; ++ sd-uhs-sdr25; ++ sd-uhs-sdr50; ++ sd-uhs-sdr104; ++ vqmmc-supply = <&vccio_sd>; ++ vmmc-supply = <&vcc_sd>; ++ status = "okay"; ++}; ++ ++&sdio { ++ bus-width = <4>; ++ cap-sd-highspeed; ++ no-sd; ++ no-mmc; ++ ignore-pm-notify; ++ keep-power-in-suspend; ++ non-removable; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&tsadc { ++ pinctrl-names = "gpio", "otpout"; ++ pinctrl-0 = <&tsadc_otp_gpio>; ++ pinctrl-1 = <&tsadc_otp_out>; ++ status = "okay"; ++}; ++ ++&uart1 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart1_xfer &uart1_cts>; ++ status = "okay"; ++}; ++ ++&u2phy { ++ status = "okay"; ++ ++ u2phy_host: host-port { ++ status = "okay"; ++ }; ++ ++ u2phy_otg: otg-port { ++ status = "okay"; ++ }; ++}; ++ ++&usb20_otg { ++ status = "okay"; ++}; ++ ++&usb_host0_ehci { ++ status = "okay"; ++}; ++ ++&usb_host0_ohci { ++ status = "okay"; ++}; ++ ++&vopb { ++ status = "okay"; ++}; ++ ++&vopb_mmu { ++ status = "okay"; ++}; ++ ++&vopl { ++ status = "okay"; ++}; ++ ++&vopl_mmu { ++ status = "okay"; ++}; ++ ++&mpp_srv { ++ status = "okay"; ++}; ++ ++&vdpu { ++ status = "okay"; ++}; ++ ++&vepu { ++ status = "okay"; ++}; ++ ++&vpu_mmu { ++ status = "okay"; ++}; ++ ++&hevc { ++ status = "okay"; ++}; ++ ++&hevc_mmu { ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v11-avb.dts b/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v11-avb.dts +new file mode 100755 +index 000000000000..f2659a7f524e +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v11-avb.dts +@@ -0,0 +1,276 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2017-2020 Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++#include "px30-evb-ddr3-v10.dtsi" ++ ++/ { ++ model = "Rockchip PX30 evb ddr3 board"; ++ compatible = "rockchip,px30-evb-ddr3-v11-avb", "rockchip,px30"; ++}; ++ ++&chosen { ++ bootargs_ext = "androidboot.boot_devices=ff390000.dwmmc,ff3b0000.nandc"; ++}; ++ ++&dsi { ++ status = "okay"; ++ ++ panel@0 { ++ compatible = "simple-panel-dsi"; ++ reg = <0>; ++ power-supply = <&vcc3v3_lcd>; ++ backlight = <&backlight>; ++ prepare-delay-ms = <0>; ++ reset-delay-ms = <0>; ++ init-delay-ms = <80>; ++ enable-delay-ms = <0>; ++ disable-delay-ms = <10>; ++ unprepare-delay-ms = <60>; ++ ++ width-mm = <68>; ++ height-mm = <121>; ++ ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | ++ MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ ++ panel-init-sequence = [ ++ 39 00 04 ff 98 81 03 ++ 15 00 02 01 00 ++ 15 00 02 02 00 ++ 15 00 02 03 53 ++ 15 00 02 04 53 ++ 15 00 02 05 13 ++ 15 00 02 06 04 ++ 15 00 02 07 02 ++ 15 00 02 08 02 ++ 15 00 02 09 00 ++ 15 00 02 0a 00 ++ 15 00 02 0b 00 ++ 15 00 02 0c 00 ++ 15 00 02 0d 00 ++ 15 00 02 0e 00 ++ 15 00 02 0f 00 ++ ++ 15 00 02 10 00 ++ 15 00 02 11 00 ++ 15 00 02 12 00 ++ 15 00 02 13 00 ++ 15 00 02 14 00 ++ 15 00 02 15 08 ++ 15 00 02 16 10 ++ 15 00 02 17 00 ++ 15 00 02 18 08 ++ 15 00 02 19 00 ++ 15 00 02 1a 00 ++ 15 00 02 1b 00 ++ 15 00 02 1c 00 ++ 15 00 02 1d 00 ++ 15 00 02 1e c0 ++ 15 00 02 1f 80 ++ ++ 15 00 02 20 02 ++ 15 00 02 21 09 ++ 15 00 02 22 00 ++ 15 00 02 23 00 ++ 15 00 02 24 00 ++ 15 00 02 25 00 ++ 15 00 02 26 00 ++ 15 00 02 27 00 ++ 15 00 02 28 55 ++ 15 00 02 29 03 ++ 15 00 02 2a 00 ++ 15 00 02 2b 00 ++ 15 00 02 2c 00 ++ 15 00 02 2d 00 ++ 15 00 02 2e 00 ++ 15 00 02 2f 00 ++ ++ 15 00 02 30 00 ++ 15 00 02 31 00 ++ 15 00 02 32 00 ++ 15 00 02 33 00 ++ 15 00 02 34 04 ++ 15 00 02 35 05 ++ 15 00 02 36 05 ++ 15 00 02 37 00 ++ 15 00 02 38 3c ++ 15 00 02 39 35 ++ 15 00 02 3a 00 ++ 15 00 02 3b 40 ++ 15 00 02 3c 00 ++ 15 00 02 3d 00 ++ 15 00 02 3e 00 ++ 15 00 02 3f 00 ++ ++ 15 00 02 40 00 ++ 15 00 02 41 88 ++ 15 00 02 42 00 ++ 15 00 02 43 00 ++ 15 00 02 44 1f ++ ++ 15 00 02 50 01 ++ 15 00 02 51 23 ++ 15 00 02 52 45 ++ 15 00 02 53 67 ++ 15 00 02 54 89 ++ 15 00 02 55 ab ++ 15 00 02 56 01 ++ 15 00 02 57 23 ++ 15 00 02 58 45 ++ 15 00 02 59 67 ++ 15 00 02 5a 89 ++ 15 00 02 5b ab ++ 15 00 02 5c cd ++ 15 00 02 5d ef ++ 15 00 02 5e 03 ++ 15 00 02 5f 14 ++ ++ 15 00 02 60 15 ++ 15 00 02 61 0c ++ 15 00 02 62 0d ++ 15 00 02 63 0e ++ 15 00 02 64 0f ++ 15 00 02 65 10 ++ 15 00 02 66 11 ++ 15 00 02 67 08 ++ 15 00 02 68 02 ++ 15 00 02 69 0a ++ 15 00 02 6a 02 ++ 15 00 02 6b 02 ++ 15 00 02 6c 02 ++ 15 00 02 6d 02 ++ 15 00 02 6e 02 ++ 15 00 02 6f 02 ++ ++ 15 00 02 70 02 ++ 15 00 02 71 02 ++ 15 00 02 72 06 ++ 15 00 02 73 02 ++ 15 00 02 74 02 ++ 15 00 02 75 14 ++ 15 00 02 76 15 ++ 15 00 02 77 0f ++ 15 00 02 78 0e ++ 15 00 02 79 0d ++ 15 00 02 7a 0c ++ 15 00 02 7b 11 ++ 15 00 02 7c 10 ++ 15 00 02 7d 06 ++ 15 00 02 7e 02 ++ 15 00 02 7f 0a ++ ++ 15 00 02 80 02 ++ 15 00 02 81 02 ++ 15 00 02 82 02 ++ 15 00 02 83 02 ++ 15 00 02 84 02 ++ 15 00 02 85 02 ++ 15 00 02 86 02 ++ 15 00 02 87 02 ++ 15 00 02 88 08 ++ 15 00 02 89 02 ++ 15 00 02 8a 02 ++ ++ 39 00 04 ff 98 81 04 ++ 15 00 02 00 80 ++ 15 00 02 70 00 ++ 15 00 02 71 00 ++ 15 00 02 66 fe ++ 15 00 02 82 15 ++ 15 00 02 84 15 ++ 15 00 02 85 15 ++ 15 00 02 3a 24 ++ 15 00 02 32 ac ++ 15 00 02 8c 80 ++ 15 00 02 3c f5 ++ 15 00 02 88 33 ++ ++ 39 00 04 ff 98 81 01 ++ 15 00 02 22 0a ++ 15 00 02 31 00 ++ 15 00 02 53 78 ++ 15 00 02 50 5b ++ 15 00 02 51 5b ++ 15 00 02 60 20 ++ 15 00 02 61 00 ++ 15 00 02 62 0d ++ 15 00 02 63 00 ++ ++ 15 00 02 a0 00 ++ 15 00 02 a1 10 ++ 15 00 02 a2 1c ++ 15 00 02 a3 13 ++ 15 00 02 a4 15 ++ 15 00 02 a5 26 ++ 15 00 02 a6 1a ++ 15 00 02 a7 1d ++ 15 00 02 a8 67 ++ 15 00 02 a9 1c ++ 15 00 02 aa 29 ++ 15 00 02 ab 5b ++ 15 00 02 ac 26 ++ 15 00 02 ad 28 ++ 15 00 02 ae 5c ++ 15 00 02 af 30 ++ 15 00 02 b0 31 ++ 15 00 02 b1 2e ++ 15 00 02 b2 32 ++ 15 00 02 b3 00 ++ ++ 15 00 02 c0 00 ++ 15 00 02 c1 10 ++ 15 00 02 c2 1c ++ 15 00 02 c3 13 ++ 15 00 02 c4 15 ++ 15 00 02 c5 26 ++ 15 00 02 c6 1a ++ 15 00 02 c7 1d ++ 15 00 02 c8 67 ++ 15 00 02 c9 1c ++ 15 00 02 ca 29 ++ 15 00 02 cb 5b ++ 15 00 02 cc 26 ++ 15 00 02 cd 28 ++ 15 00 02 ce 5c ++ 15 00 02 cf 30 ++ 15 00 02 d0 31 ++ 15 00 02 d1 2e ++ 15 00 02 d2 32 ++ 15 00 02 d3 00 ++ 39 00 04 ff 98 81 00 ++ 05 00 01 11 ++ 05 01 01 29 ++ ]; ++ ++ panel-exit-sequence = [ ++ 05 00 01 28 ++ 05 00 01 10 ++ ]; ++ ++ display-timings { ++ native-mode = <&timing1>; ++ ++ timing1: timing1 { ++ clock-frequency = <64000000>; ++ hactive = <720>; ++ vactive = <1280>; ++ hfront-porch = <40>; ++ hsync-len = <10>; ++ hback-porch = <40>; ++ vfront-porch = <22>; ++ vsync-len = <4>; ++ vback-porch = <11>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ }; ++ ++}; +diff --git a/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v11-linux.dts b/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v11-linux.dts +new file mode 100755 +index 000000000000..54a4c19d11b3 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v11-linux.dts +@@ -0,0 +1,296 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++#include "px30-evb-ddr3-v10-linux.dts" ++ ++/ { ++ model = "Rockchip linux PX30 evb ddr3 board"; ++ compatible = "rockchip,px30-evb-ddr3-v11-linux", "rockchip,px30"; ++}; ++ ++&dsi { ++ status = "okay"; ++ ++ panel@0 { ++ compatible = "sitronix,st7703", "simple-panel-dsi"; ++ reg = <0>; ++ power-supply = <&vcc3v3_lcd>; ++ backlight = <&backlight>; ++ prepare-delay-ms = <0>; ++ reset-delay-ms = <0>; ++ init-delay-ms = <80>; ++ enable-delay-ms = <0>; ++ disable-delay-ms = <10>; ++ unprepare-delay-ms = <60>; ++ ++ width-mm = <68>; ++ height-mm = <121>; ++ ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | ++ MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ ++ panel-init-sequence = [ ++ 39 00 04 ff 98 81 03 ++ 15 00 02 01 00 ++ 15 00 02 02 00 ++ 15 00 02 03 53 ++ 15 00 02 04 53 ++ 15 00 02 05 13 ++ 15 00 02 06 04 ++ 15 00 02 07 02 ++ 15 00 02 08 02 ++ 15 00 02 09 00 ++ 15 00 02 0a 00 ++ 15 00 02 0b 00 ++ 15 00 02 0c 00 ++ 15 00 02 0d 00 ++ 15 00 02 0e 00 ++ 15 00 02 0f 00 ++ ++ 15 00 02 10 00 ++ 15 00 02 11 00 ++ 15 00 02 12 00 ++ 15 00 02 13 00 ++ 15 00 02 14 00 ++ 15 00 02 15 08 ++ 15 00 02 16 10 ++ 15 00 02 17 00 ++ 15 00 02 18 08 ++ 15 00 02 19 00 ++ 15 00 02 1a 00 ++ 15 00 02 1b 00 ++ 15 00 02 1c 00 ++ 15 00 02 1d 00 ++ 15 00 02 1e c0 ++ 15 00 02 1f 80 ++ ++ 15 00 02 20 02 ++ 15 00 02 21 09 ++ 15 00 02 22 00 ++ 15 00 02 23 00 ++ 15 00 02 24 00 ++ 15 00 02 25 00 ++ 15 00 02 26 00 ++ 15 00 02 27 00 ++ 15 00 02 28 55 ++ 15 00 02 29 03 ++ 15 00 02 2a 00 ++ 15 00 02 2b 00 ++ 15 00 02 2c 00 ++ 15 00 02 2d 00 ++ 15 00 02 2e 00 ++ 15 00 02 2f 00 ++ ++ 15 00 02 30 00 ++ 15 00 02 31 00 ++ 15 00 02 32 00 ++ 15 00 02 33 00 ++ 15 00 02 34 04 ++ 15 00 02 35 05 ++ 15 00 02 36 05 ++ 15 00 02 37 00 ++ 15 00 02 38 3c ++ 15 00 02 39 35 ++ 15 00 02 3a 00 ++ 15 00 02 3b 40 ++ 15 00 02 3c 00 ++ 15 00 02 3d 00 ++ 15 00 02 3e 00 ++ 15 00 02 3f 00 ++ ++ 15 00 02 40 00 ++ 15 00 02 41 88 ++ 15 00 02 42 00 ++ 15 00 02 43 00 ++ 15 00 02 44 1f ++ ++ 15 00 02 50 01 ++ 15 00 02 51 23 ++ 15 00 02 52 45 ++ 15 00 02 53 67 ++ 15 00 02 54 89 ++ 15 00 02 55 ab ++ 15 00 02 56 01 ++ 15 00 02 57 23 ++ 15 00 02 58 45 ++ 15 00 02 59 67 ++ 15 00 02 5a 89 ++ 15 00 02 5b ab ++ 15 00 02 5c cd ++ 15 00 02 5d ef ++ 15 00 02 5e 03 ++ 15 00 02 5f 14 ++ ++ 15 00 02 60 15 ++ 15 00 02 61 0c ++ 15 00 02 62 0d ++ 15 00 02 63 0e ++ 15 00 02 64 0f ++ 15 00 02 65 10 ++ 15 00 02 66 11 ++ 15 00 02 67 08 ++ 15 00 02 68 02 ++ 15 00 02 69 0a ++ 15 00 02 6a 02 ++ 15 00 02 6b 02 ++ 15 00 02 6c 02 ++ 15 00 02 6d 02 ++ 15 00 02 6e 02 ++ 15 00 02 6f 02 ++ ++ 15 00 02 70 02 ++ 15 00 02 71 02 ++ 15 00 02 72 06 ++ 15 00 02 73 02 ++ 15 00 02 74 02 ++ 15 00 02 75 14 ++ 15 00 02 76 15 ++ 15 00 02 77 0f ++ 15 00 02 78 0e ++ 15 00 02 79 0d ++ 15 00 02 7a 0c ++ 15 00 02 7b 11 ++ 15 00 02 7c 10 ++ 15 00 02 7d 06 ++ 15 00 02 7e 02 ++ 15 00 02 7f 0a ++ ++ 15 00 02 80 02 ++ 15 00 02 81 02 ++ 15 00 02 82 02 ++ 15 00 02 83 02 ++ 15 00 02 84 02 ++ 15 00 02 85 02 ++ 15 00 02 86 02 ++ 15 00 02 87 02 ++ 15 00 02 88 08 ++ 15 00 02 89 02 ++ 15 00 02 8a 02 ++ ++ 39 00 04 ff 98 81 04 ++ 15 00 02 00 80 ++ 15 00 02 70 00 ++ 15 00 02 71 00 ++ 15 00 02 66 fe ++ 15 00 02 82 15 ++ 15 00 02 84 15 ++ 15 00 02 85 15 ++ 15 00 02 3a 24 ++ 15 00 02 32 ac ++ 15 00 02 8c 80 ++ 15 00 02 3c f5 ++ 15 00 02 88 33 ++ ++ 39 00 04 ff 98 81 01 ++ 15 00 02 22 0a ++ 15 00 02 31 00 ++ 15 00 02 53 78 ++ 15 00 02 50 5b ++ 15 00 02 51 5b ++ 15 00 02 60 20 ++ 15 00 02 61 00 ++ 15 00 02 62 0d ++ 15 00 02 63 00 ++ ++ 15 00 02 a0 00 ++ 15 00 02 a1 10 ++ 15 00 02 a2 1c ++ 15 00 02 a3 13 ++ 15 00 02 a4 15 ++ 15 00 02 a5 26 ++ 15 00 02 a6 1a ++ 15 00 02 a7 1d ++ 15 00 02 a8 67 ++ 15 00 02 a9 1c ++ 15 00 02 aa 29 ++ 15 00 02 ab 5b ++ 15 00 02 ac 26 ++ 15 00 02 ad 28 ++ 15 00 02 ae 5c ++ 15 00 02 af 30 ++ 15 00 02 b0 31 ++ 15 00 02 b1 2e ++ 15 00 02 b2 32 ++ 15 00 02 b3 00 ++ ++ 15 00 02 c0 00 ++ 15 00 02 c1 10 ++ 15 00 02 c2 1c ++ 15 00 02 c3 13 ++ 15 00 02 c4 15 ++ 15 00 02 c5 26 ++ 15 00 02 c6 1a ++ 15 00 02 c7 1d ++ 15 00 02 c8 67 ++ 15 00 02 c9 1c ++ 15 00 02 ca 29 ++ 15 00 02 cb 5b ++ 15 00 02 cc 26 ++ 15 00 02 cd 28 ++ 15 00 02 ce 5c ++ 15 00 02 cf 30 ++ 15 00 02 d0 31 ++ 15 00 02 d1 2e ++ 15 00 02 d2 32 ++ 15 00 02 d3 00 ++ 39 00 04 ff 98 81 00 ++ 05 00 01 11 ++ 05 01 01 29 ++ ]; ++ ++ panel-exit-sequence = [ ++ 05 00 01 28 ++ 05 00 01 10 ++ ]; ++ ++ display-timings { ++ native-mode = <&timing1>; ++ ++ timing1: timing1 { ++ clock-frequency = <64000000>; ++ hactive = <720>; ++ vactive = <1280>; ++ hfront-porch = <40>; ++ hsync-len = <10>; ++ hback-porch = <40>; ++ vfront-porch = <22>; ++ vsync-len = <4>; ++ vback-porch = <11>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ panel_in_dsi: endpoint { ++ remote-endpoint = <&dsi_out_panel>; ++ }; ++ }; ++ }; ++ ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@1 { ++ reg = <1>; ++ dsi_out_panel: endpoint { ++ remote-endpoint = <&panel_in_dsi>; ++ }; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v11.dts b/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v11.dts +new file mode 100755 +index 000000000000..2e59a09d3c5c +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v11.dts +@@ -0,0 +1,293 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++#include "px30-evb-ddr3-v10.dtsi" ++ ++/ { ++ model = "Rockchip PX30 evb ddr3 board"; ++ compatible = "rockchip,px30-evb-ddr3-v11", "rockchip,px30"; ++}; ++ ++&dsi { ++ status = "okay"; ++ ++ panel@0 { ++ compatible = "simple-panel-dsi"; ++ reg = <0>; ++ power-supply = <&vcc3v3_lcd>; ++ backlight = <&backlight>; ++ prepare-delay-ms = <0>; ++ reset-delay-ms = <0>; ++ init-delay-ms = <80>; ++ enable-delay-ms = <0>; ++ disable-delay-ms = <10>; ++ unprepare-delay-ms = <60>; ++ ++ width-mm = <68>; ++ height-mm = <121>; ++ ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | ++ MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ ++ panel-init-sequence = [ ++ 39 00 04 ff 98 81 03 ++ 15 00 02 01 00 ++ 15 00 02 02 00 ++ 15 00 02 03 53 ++ 15 00 02 04 53 ++ 15 00 02 05 13 ++ 15 00 02 06 04 ++ 15 00 02 07 02 ++ 15 00 02 08 02 ++ 15 00 02 09 00 ++ 15 00 02 0a 00 ++ 15 00 02 0b 00 ++ 15 00 02 0c 00 ++ 15 00 02 0d 00 ++ 15 00 02 0e 00 ++ 15 00 02 0f 00 ++ ++ 15 00 02 10 00 ++ 15 00 02 11 00 ++ 15 00 02 12 00 ++ 15 00 02 13 00 ++ 15 00 02 14 00 ++ 15 00 02 15 08 ++ 15 00 02 16 10 ++ 15 00 02 17 00 ++ 15 00 02 18 08 ++ 15 00 02 19 00 ++ 15 00 02 1a 00 ++ 15 00 02 1b 00 ++ 15 00 02 1c 00 ++ 15 00 02 1d 00 ++ 15 00 02 1e c0 ++ 15 00 02 1f 80 ++ ++ 15 00 02 20 02 ++ 15 00 02 21 09 ++ 15 00 02 22 00 ++ 15 00 02 23 00 ++ 15 00 02 24 00 ++ 15 00 02 25 00 ++ 15 00 02 26 00 ++ 15 00 02 27 00 ++ 15 00 02 28 55 ++ 15 00 02 29 03 ++ 15 00 02 2a 00 ++ 15 00 02 2b 00 ++ 15 00 02 2c 00 ++ 15 00 02 2d 00 ++ 15 00 02 2e 00 ++ 15 00 02 2f 00 ++ ++ 15 00 02 30 00 ++ 15 00 02 31 00 ++ 15 00 02 32 00 ++ 15 00 02 33 00 ++ 15 00 02 34 04 ++ 15 00 02 35 05 ++ 15 00 02 36 05 ++ 15 00 02 37 00 ++ 15 00 02 38 3c ++ 15 00 02 39 35 ++ 15 00 02 3a 00 ++ 15 00 02 3b 40 ++ 15 00 02 3c 00 ++ 15 00 02 3d 00 ++ 15 00 02 3e 00 ++ 15 00 02 3f 00 ++ ++ 15 00 02 40 00 ++ 15 00 02 41 88 ++ 15 00 02 42 00 ++ 15 00 02 43 00 ++ 15 00 02 44 1f ++ ++ 15 00 02 50 01 ++ 15 00 02 51 23 ++ 15 00 02 52 45 ++ 15 00 02 53 67 ++ 15 00 02 54 89 ++ 15 00 02 55 ab ++ 15 00 02 56 01 ++ 15 00 02 57 23 ++ 15 00 02 58 45 ++ 15 00 02 59 67 ++ 15 00 02 5a 89 ++ 15 00 02 5b ab ++ 15 00 02 5c cd ++ 15 00 02 5d ef ++ 15 00 02 5e 03 ++ 15 00 02 5f 14 ++ ++ 15 00 02 60 15 ++ 15 00 02 61 0c ++ 15 00 02 62 0d ++ 15 00 02 63 0e ++ 15 00 02 64 0f ++ 15 00 02 65 10 ++ 15 00 02 66 11 ++ 15 00 02 67 08 ++ 15 00 02 68 02 ++ 15 00 02 69 0a ++ 15 00 02 6a 02 ++ 15 00 02 6b 02 ++ 15 00 02 6c 02 ++ 15 00 02 6d 02 ++ 15 00 02 6e 02 ++ 15 00 02 6f 02 ++ ++ 15 00 02 70 02 ++ 15 00 02 71 02 ++ 15 00 02 72 06 ++ 15 00 02 73 02 ++ 15 00 02 74 02 ++ 15 00 02 75 14 ++ 15 00 02 76 15 ++ 15 00 02 77 0f ++ 15 00 02 78 0e ++ 15 00 02 79 0d ++ 15 00 02 7a 0c ++ 15 00 02 7b 11 ++ 15 00 02 7c 10 ++ 15 00 02 7d 06 ++ 15 00 02 7e 02 ++ 15 00 02 7f 0a ++ ++ 15 00 02 80 02 ++ 15 00 02 81 02 ++ 15 00 02 82 02 ++ 15 00 02 83 02 ++ 15 00 02 84 02 ++ 15 00 02 85 02 ++ 15 00 02 86 02 ++ 15 00 02 87 02 ++ 15 00 02 88 08 ++ 15 00 02 89 02 ++ 15 00 02 8a 02 ++ ++ 39 00 04 ff 98 81 04 ++ 15 00 02 00 80 ++ 15 00 02 70 00 ++ 15 00 02 71 00 ++ 15 00 02 66 fe ++ 15 00 02 82 15 ++ 15 00 02 84 15 ++ 15 00 02 85 15 ++ 15 00 02 3a 24 ++ 15 00 02 32 ac ++ 15 00 02 8c 80 ++ 15 00 02 3c f5 ++ 15 00 02 88 33 ++ ++ 39 00 04 ff 98 81 01 ++ 15 00 02 22 0a ++ 15 00 02 31 00 ++ 15 00 02 53 78 ++ 15 00 02 50 5b ++ 15 00 02 51 5b ++ 15 00 02 60 20 ++ 15 00 02 61 00 ++ 15 00 02 62 0d ++ 15 00 02 63 00 ++ ++ 15 00 02 a0 00 ++ 15 00 02 a1 10 ++ 15 00 02 a2 1c ++ 15 00 02 a3 13 ++ 15 00 02 a4 15 ++ 15 00 02 a5 26 ++ 15 00 02 a6 1a ++ 15 00 02 a7 1d ++ 15 00 02 a8 67 ++ 15 00 02 a9 1c ++ 15 00 02 aa 29 ++ 15 00 02 ab 5b ++ 15 00 02 ac 26 ++ 15 00 02 ad 28 ++ 15 00 02 ae 5c ++ 15 00 02 af 30 ++ 15 00 02 b0 31 ++ 15 00 02 b1 2e ++ 15 00 02 b2 32 ++ 15 00 02 b3 00 ++ ++ 15 00 02 c0 00 ++ 15 00 02 c1 10 ++ 15 00 02 c2 1c ++ 15 00 02 c3 13 ++ 15 00 02 c4 15 ++ 15 00 02 c5 26 ++ 15 00 02 c6 1a ++ 15 00 02 c7 1d ++ 15 00 02 c8 67 ++ 15 00 02 c9 1c ++ 15 00 02 ca 29 ++ 15 00 02 cb 5b ++ 15 00 02 cc 26 ++ 15 00 02 cd 28 ++ 15 00 02 ce 5c ++ 15 00 02 cf 30 ++ 15 00 02 d0 31 ++ 15 00 02 d1 2e ++ 15 00 02 d2 32 ++ 15 00 02 d3 00 ++ 39 00 04 ff 98 81 00 ++ 05 00 01 11 ++ 05 01 01 29 ++ ]; ++ ++ panel-exit-sequence = [ ++ 05 00 01 28 ++ 05 00 01 10 ++ ]; ++ ++ display-timings { ++ native-mode = <&timing1>; ++ ++ timing1: timing1 { ++ clock-frequency = <64000000>; ++ hactive = <720>; ++ vactive = <1280>; ++ hfront-porch = <40>; ++ hsync-len = <10>; ++ hback-porch = <40>; ++ vfront-porch = <22>; ++ vsync-len = <4>; ++ vback-porch = <11>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ }; ++ ++}; ++ ++&firmware_android { ++ compatible = "android,firmware"; ++ fstab { ++ compatible = "android,fstab"; ++ system { ++ compatible = "android,system"; ++ dev = "/dev/block/by-name/system"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait"; ++ }; ++ vendor { ++ compatible = "android,vendor"; ++ dev = "/dev/block/by-name/vendor"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait"; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/px30-evb-ddr4-v10.dts b/arch/arm64/boot/dts/rockchip/px30-evb-ddr4-v10.dts +new file mode 100755 +index 000000000000..7bfe640d32de +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/px30-evb-ddr4-v10.dts +@@ -0,0 +1,853 @@ ++/* ++ * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++ */ ++ ++/dts-v1/; ++#include ++#include ++#include ++#include ++#include ++#include "px30.dtsi" ++#include "px30-android.dtsi" ++#include "px30-ddr4p416dd6-timing.dtsi" ++/ { ++ model = "Rockchip PX30 evb ddr4 board"; ++ compatible = "rockchip,px30-evb-ddr4-v10", "rockchip,px30"; ++ ++ adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 2>; ++ io-channel-names = "buttons"; ++ poll-interval = <100>; ++ keyup-threshold-microvolt = <1800000>; ++ ++ esc-key { ++ linux,code = ; ++ label = "esc"; ++ press-threshold-microvolt = <1310000>; ++ }; ++ ++ home-key { ++ linux,code = ; ++ label = "home"; ++ press-threshold-microvolt = <624000>; ++ }; ++ ++ menu-key { ++ linux,code = ; ++ label = "menu"; ++ press-threshold-microvolt = <987000>; ++ }; ++ ++ vol-down-key { ++ linux,code = ; ++ label = "volume down"; ++ press-threshold-microvolt = <300000>; ++ }; ++ ++ vol-up-key { ++ linux,code = ; ++ label = "volume up"; ++ press-threshold-microvolt = <17000>; ++ }; ++ }; ++ ++ backlight: backlight { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm1 0 25000 0>; ++ brightness-levels = < ++ 0 1 2 3 4 5 6 7 ++ 8 9 10 11 12 13 14 15 ++ 16 17 18 19 20 21 22 23 ++ 24 25 26 27 28 29 30 31 ++ 32 33 34 35 36 37 38 39 ++ 40 41 42 43 44 45 46 47 ++ 48 49 50 51 52 53 54 55 ++ 56 57 58 59 60 61 62 63 ++ 64 65 66 67 68 69 70 71 ++ 72 73 74 75 76 77 78 79 ++ 80 81 82 83 84 85 86 87 ++ 88 89 90 91 92 93 94 95 ++ 96 97 98 99 100 101 102 103 ++ 104 105 106 107 108 109 110 111 ++ 112 113 114 115 116 117 118 119 ++ 120 121 122 123 124 125 126 127 ++ 128 129 130 131 132 133 134 135 ++ 136 137 138 139 140 141 142 143 ++ 144 145 146 147 148 149 150 151 ++ 152 153 154 155 156 157 158 159 ++ 160 161 162 163 164 165 166 167 ++ 168 169 170 171 172 173 174 175 ++ 176 177 178 179 180 181 182 183 ++ 184 185 186 187 188 189 190 191 ++ 192 193 194 195 196 197 198 199 ++ 200 201 202 203 204 205 206 207 ++ 208 209 210 211 212 213 214 215 ++ 216 217 218 219 220 221 222 223 ++ 224 225 226 227 228 229 230 231 ++ 232 233 234 235 236 237 238 239 ++ 240 241 242 243 244 245 246 247 ++ 248 249 250 251 252 253 254 255>; ++ default-brightness-level = <200>; ++ }; ++ ++ rk809-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,rk809-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,widgets = ++ "Microphone", "Mic Jack", ++ "Headphone", "Headphone Jack"; ++ simple-audio-card,routing = ++ "Mic Jack", "MICBIAS1", ++ "IN1P", "Mic Jack", ++ "Headphone Jack", "HPOL", ++ "Headphone Jack", "HPOR"; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s1_2ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&rk809_codec>; ++ }; ++ }; ++ ++ rk_headset: rk-headset { ++ compatible = "rockchip_headset"; ++ headset_gpio = <&gpio2 RK_PB0 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hp_det>; ++ io-channels = <&saradc 1>; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ /*clocks = <&rk809 1>;*/ ++ /*clock-names = "ext_clock";*/ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_LOW>; /* GPIO3_A4 */ ++ }; ++ ++ vcc_phy: vcc-phy-regulator { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_phy"; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ vcc5v0_sys: vccsys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc5v0_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ wifi_chip_type = "AP6210"; ++ WIFI,host_wake_irq = <&gpio0 RK_PB2 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ /*clocks = <&rk809 1>;*/ ++ /*clock-names = "ext_clock";*/ ++ uart_rts_gpios = <&gpio1 RK_PC3 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default","rts_gpio"; ++ pinctrl-0 = <&uart1_rts>; ++ pinctrl-1 = <&uart1_rts_gpio>; ++ BT,reset_gpio = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio0 RK_PA1 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio0 RK_PB3 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++}; ++ ++&display_subsystem { ++ status = "okay"; ++}; ++ ++&dsi { ++ status = "okay"; ++ ++ panel@0 { ++ compatible = "sitronix,st7703", "simple-panel-dsi"; ++ reg = <0>; ++ power-supply = <&vcc3v3_lcd>; ++ backlight = <&backlight>; ++ prepare-delay-ms = <2>; ++ reset-delay-ms = <1>; ++ init-delay-ms = <20>; ++ enable-delay-ms = <120>; ++ disable-delay-ms = <50>; ++ unprepare-delay-ms = <20>; ++ ++ width-mm = <68>; ++ height-mm = <121>; ++ ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | ++ MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ ++ panel-init-sequence = [ ++ 05 fa 01 11 ++ 39 00 04 b9 f1 12 83 ++ 39 00 1c ba 33 81 05 f9 0e 0e 00 00 00 ++ 00 00 00 00 00 44 25 00 91 0a ++ 00 00 02 4f 01 00 00 37 ++ 15 00 02 b8 25 ++ 39 00 04 bf 02 11 00 ++ 39 00 0b b3 0c 10 0a 50 03 ff 00 00 00 ++ 00 ++ 39 00 0a c0 73 73 50 50 00 00 08 70 00 ++ 15 00 02 bc 46 ++ 15 00 02 cc 0b ++ 15 00 02 b4 80 ++ 39 00 04 b2 c8 12 30 ++ 39 00 0f e3 07 07 0b 0b 03 0b 00 00 00 ++ 00 ff 00 c0 10 ++ 39 00 0d c1 53 00 1e 1e 77 e1 cc dd 67 ++ 77 33 33 ++ 39 00 07 c6 00 00 ff ff 01 ff ++ 39 00 03 b5 09 09 ++ 39 00 03 b6 87 95 ++ 39 00 40 e9 c2 10 05 05 10 05 a0 12 31 ++ 23 3f 81 0a a0 37 18 00 80 01 ++ 00 00 00 00 80 01 00 00 00 48 ++ f8 86 42 08 88 88 80 88 88 88 ++ 58 f8 87 53 18 88 88 81 88 88 ++ 88 00 00 00 01 00 00 00 00 00 ++ 00 00 00 00 ++ 39 00 3e ea 00 1a 00 00 00 00 02 00 00 ++ 00 00 00 1f 88 81 35 78 88 88 ++ 85 88 88 88 0f 88 80 24 68 88 ++ 88 84 88 88 88 23 10 00 00 1c ++ 00 00 00 00 00 00 00 00 00 00 ++ 00 00 00 00 00 30 05 a0 00 00 ++ 00 00 ++ 39 00 23 e0 00 06 08 2a 31 3f 38 36 07 ++ 0c 0d 11 13 12 13 11 18 00 06 ++ 08 2a 31 3f 38 36 07 0c 0d 11 ++ 13 12 13 11 18 ++ 05 32 01 29 ++ ]; ++ ++ panel-exit-sequence = [ ++ 05 00 01 28 ++ 05 00 01 10 ++ ]; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <64000000>; ++ hactive = <720>; ++ vactive = <1280>; ++ hfront-porch = <40>; ++ hsync-len = <10>; ++ hback-porch = <40>; ++ vfront-porch = <22>; ++ vsync-len = <4>; ++ vback-porch = <11>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ panel_in_dsi: endpoint { ++ remote-endpoint = <&dsi_out_panel>; ++ }; ++ }; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@1 { ++ reg = <1>; ++ dsi_out_panel: endpoint { ++ remote-endpoint = <&panel_in_dsi>; ++ }; ++ }; ++ }; ++}; ++ ++&chosen { ++ bootargs_ext = "androidboot.boot_devices=ff390000.dwmmc,ff3b0000.nandc"; ++}; ++ ++&dsi_in_vopb { ++ status = "okay"; ++}; ++ ++&dsi_in_vopl { ++ status = "disabled"; ++}; ++ ++&route_dsi { ++ connect = <&vopb_out_dsi>; ++ status = "okay"; ++}; ++ ++&bus_apll { ++ bus-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&cpu0 { ++ cpu-supply = <&vdd_arm>; ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&dmc { ++ center-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&emmc { ++ bus-width = <8>; ++ cap-mmc-highspeed; ++ mmc-hs200-1_8v; ++ no-sdio; ++ no-sd; ++ disable-wp; ++ non-removable; ++ num-slots = <1>; ++ status = "okay"; ++}; ++ ++&gmac { ++ phy-supply = <&vcc_phy>; ++ assigned-clocks = <&cru SCLK_GMAC>; ++ assigned-clock-parents = <&gmac_clkin>; ++ clock_in_out = "input"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&rmii_pins &mac_refclk>; ++ snps,reset-gpio = <&gpio2 13 GPIO_ACTIVE_LOW>; ++ snps,reset-active-low; ++ snps,reset-delays-us = <0 50000 50000>; ++ status = "okay"; ++}; ++ ++&gpu { ++ mali-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ ++ rk809: pmic@20 { ++ compatible = "rockchip,rk809"; ++ reg = <0x20>; ++ interrupt-parent = <&gpio0>; ++ interrupts = <7 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default", "pmic-sleep", ++ "pmic-power-off", "pmic-reset"; ++ pinctrl-0 = <&pmic_int>; ++ pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; ++ pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; ++ pinctrl-3 = <&soc_slppin_rst>, <&rk817_slppin_rst>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ //fb-inner-reg-idxs = <2>; ++ /* 1: rst regs (default in codes), 0: rst the pmic */ ++ pmic-reset-func = <1>; ++ ++ vcc1-supply = <&vcc5v0_sys>; ++ vcc2-supply = <&vcc5v0_sys>; ++ vcc3-supply = <&vcc5v0_sys>; ++ vcc4-supply = <&vcc5v0_sys>; ++ vcc5-supply = <&vcc3v3_sys>; ++ vcc6-supply = <&vcc3v3_sys>; ++ vcc7-supply = <&vcc3v3_sys>; ++ vcc8-supply = <&vcc3v3_sys>; ++ vcc9-supply = <&vcc5v0_sys>; ++ ++ pwrkey { ++ status = "okay"; ++ }; ++ ++ pinctrl_rk8xx: pinctrl_rk8xx { ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ rk817_slppin_null: rk817_slppin_null { ++ pins = "gpio_slp"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_slppin_slp: rk817_slppin_slp { ++ pins = "gpio_slp"; ++ function = "pin_fun1"; ++ }; ++ ++ rk817_slppin_pwrdn: rk817_slppin_pwrdn { ++ pins = "gpio_slp"; ++ function = "pin_fun2"; ++ }; ++ ++ rk817_slppin_rst: rk817_slppin_rst { ++ pins = "gpio_slp"; ++ function = "pin_fun3"; ++ }; ++ }; ++ ++ regulators { ++ vdd_logic: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <950000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_logic"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <950000>; ++ }; ++ }; ++ ++ vdd_arm: DCDC_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <950000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_arm"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <950000>; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_ddr"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_3v0: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_3v0"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ ++ vcc_1v0: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <2500000>; ++ regulator-max-microvolt = <2500000>; ++ regulator-initial-mode = <0x1>; ++ regulator-name = "vcc_1v0"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <2500000>; ++ }; ++ }; ++ ++ vcc1v8_soc: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcc1v8_soc"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd1v0_soc: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ ++ regulator-name = "vcc1v0_soc"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc3v0_pmu: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ ++ regulator-name = "vcc3v0_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ ++ }; ++ }; ++ ++ vccio_sd: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vccio_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc_sd: LDO_REG6 { ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vcc_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ ++ }; ++ }; ++ ++ vcc2v8_dvp: LDO_REG7 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <2800000>; ++ regulator-max-microvolt = <2800000>; ++ ++ regulator-name = "vcc2v8_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <2800000>; ++ }; ++ }; ++ ++ vcc1v8_dvp: LDO_REG8 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcc1v8_dvp"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd1v5_dvp: LDO_REG9 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ ++ regulator-name = "vdd1v5_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <1500000>; ++ }; ++ }; ++ ++ vcc3v3_sys: DCDC_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcc3v3_sys"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc5v0_host: SWITCH_REG1 { ++ regulator-name = "vcc5v0_host"; ++ }; ++ ++ vcc3v3_lcd: SWITCH_REG2 { ++ regulator-boot-on; ++ regulator-name = "vcc3v3_lcd"; ++ }; ++ }; ++ ++ rk809_codec: codec { ++ #sound-dai-cells = <0>; ++ compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; ++ clocks = <&cru SCLK_I2S1_OUT>; ++ clock-names = "mclk"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1_2ch_mclk>; ++ hp-volume = <20>; ++ spk-volume = <3>; ++ status = "okay"; ++ }; ++ }; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ ++ sensor@f { ++ status = "okay"; ++ compatible = "ak8963"; ++ reg = <0x0f>; ++ type = ; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ layout = <1>; ++ reprobe_en = <1>; ++ }; ++ ++ gt1x: gt1x@14 { ++ compatible = "goodix,gt1x"; ++ reg = <0x14>; ++ power-supply = <&vcc3v3_lcd>; ++ goodix,rst-gpio = <&gpio0 RK_PB4 GPIO_ACTIVE_HIGH>; ++ goodix,irq-gpio = <&gpio0 RK_PA5 IRQ_TYPE_LEVEL_LOW>; ++ }; ++ ++ sensor@4c { ++ status = "okay"; ++ compatible = "gs_mma7660"; ++ reg = <0x4c>; ++ type = ; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ layout = <6>; ++ reprobe_en = <1>; ++ }; ++}; ++ ++&io_domains { ++ status = "okay"; ++ ++ vccio1-supply = <&vcc1v8_soc>; ++ vccio2-supply = <&vccio_sd>; ++ vccio3-supply = <&vcc_3v0>; ++ vccio4-supply = <&vcc3v0_pmu>; ++ vccio5-supply = <&vcc_3v0>; ++}; ++ ++&i2s1_2ch { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++}; ++ ++&nandc0 { ++ status = "okay"; ++}; ++ ++&pinctrl { ++ headphone { ++ hp_det: hp-det { ++ rockchip,pins = <2 RK_PB0 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ pmic { ++ pmic_int: pmic_int { ++ rockchip,pins = ++ <0 RK_PA7 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ soc_slppin_gpio: soc_slppin_gpio { ++ rockchip,pins = ++ <0 RK_PA4 RK_FUNC_GPIO &pcfg_output_low>; ++ }; ++ ++ soc_slppin_slp: soc_slppin_slp { ++ rockchip,pins = ++ <0 RK_PA4 1 &pcfg_pull_none>; ++ }; ++ ++ soc_slppin_rst: soc_slppin_rst { ++ rockchip,pins = ++ <0 RK_PA4 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ ++ pmuio1-supply = <&vcc3v0_pmu>; ++ pmuio2-supply = <&vcc3v0_pmu>; ++}; ++ ++&pwm1 { ++ status = "okay"; ++}; ++ ++&rk_rga { ++ status = "okay"; ++}; ++ ++&rockchip_suspend { ++ status = "okay"; ++ rockchip,sleep-debug-en = <1>; ++}; ++ ++&saradc { ++ status = "okay"; ++ vref-supply = <&vcc1v8_soc>; ++}; ++ ++&sdmmc { ++ bus-width = <4>; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ no-sdio; ++ no-mmc; ++ card-detect-delay = <800>; ++ ignore-pm-notify; ++ /*cd-gpios = <&gpio2 4 GPIO_ACTIVE_HIGH>; [> CD GPIO <]*/ ++ sd-uhs-sdr12; ++ sd-uhs-sdr25; ++ sd-uhs-sdr50; ++ sd-uhs-sdr104; ++ vqmmc-supply = <&vccio_sd>; ++ vmmc-supply = <&vcc_sd>; ++ status = "disabled"; ++}; ++ ++&sdio { ++ bus-width = <4>; ++ cap-sd-highspeed; ++ no-sd; ++ no-mmc; ++ ignore-pm-notify; ++ keep-power-in-suspend; ++ non-removable; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&tsadc { ++ pinctrl-names = "gpio", "otpout"; ++ pinctrl-0 = <&tsadc_otp_gpio>; ++ pinctrl-1 = <&tsadc_otp_out>; ++ status = "okay"; ++}; ++ ++&uart1 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart1_xfer &uart1_cts>; ++ status = "okay"; ++}; ++ ++&u2phy { ++ status = "okay"; ++ ++ u2phy_host: host-port { ++ status = "okay"; ++ }; ++ ++ u2phy_otg: otg-port { ++ status = "okay"; ++ }; ++}; ++ ++&usb20_otg { ++ status = "okay"; ++}; ++ ++&usb_host0_ehci { ++ status = "okay"; ++}; ++ ++&usb_host0_ohci { ++ status = "okay"; ++}; ++ ++&vopb { ++ status = "okay"; ++}; ++ ++&vopb_mmu { ++ status = "okay"; ++}; ++ ++&vopl { ++ status = "okay"; ++}; ++ ++&vopl_mmu { ++ status = "okay"; ++}; ++ ++&mpp_srv { ++ status = "okay"; ++}; ++ ++&vdpu { ++ status = "okay"; ++}; ++ ++&vepu { ++ status = "okay"; ++}; ++ ++&vpu_mmu { ++ status = "okay"; ++}; ++ ++&hevc { ++ status = "okay"; ++}; ++ ++&hevc_mmu { ++ status = "okay"; ++}; ++ +diff --git a/arch/arm64/boot/dts/rockchip/px30-evb-ext-rk618-avb.dts b/arch/arm64/boot/dts/rockchip/px30-evb-ext-rk618-avb.dts +new file mode 100755 +index 000000000000..ea44da5066af +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/px30-evb-ext-rk618-avb.dts +@@ -0,0 +1,11 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++#include "px30-evb-ext-rk618.dtsi" ++ ++/ { ++ model = "Rockchip PX30 EVB EXT RK618 board"; ++ compatible = "rockchip,px30-evb-ext-rk618-avb", "rockchip,px30"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/px30-evb-ext-rk618.dts b/arch/arm64/boot/dts/rockchip/px30-evb-ext-rk618.dts +new file mode 100755 +index 000000000000..69f44b4d115c +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/px30-evb-ext-rk618.dts +@@ -0,0 +1,33 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++#include "px30-evb-ext-rk618.dtsi" ++ ++/ { ++ model = "Rockchip PX30 EVB EXT RK618 board"; ++ compatible = "rockchip,px30-evb-ext-rk618", "rockchip,px30"; ++}; ++ ++&firmware_android { ++ compatible = "android,firmware"; ++ fstab { ++ compatible = "android,fstab"; ++ system { ++ compatible = "android,system"; ++ dev = "/dev/block/by-name/system"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait"; ++ }; ++ vendor { ++ compatible = "android,vendor"; ++ dev = "/dev/block/by-name/vendor"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait"; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/px30-evb-ext-rk618.dtsi b/arch/arm64/boot/dts/rockchip/px30-evb-ext-rk618.dtsi +new file mode 100755 +index 000000000000..0eac63181407 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/px30-evb-ext-rk618.dtsi +@@ -0,0 +1,202 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++#include ++#include ++#include "px30-evb-ddr3-v10.dtsi" ++ ++&dsi { ++ status = "okay"; ++ ++ panel@0 { ++ compatible = "sitronix,st7703", "simple-panel-dsi"; ++ reg = <0>; ++ power-supply = <&vcc3v3_lcd>; ++ backlight = <&backlight>; ++ prepare-delay-ms = <2>; ++ reset-delay-ms = <1>; ++ init-delay-ms = <20>; ++ enable-delay-ms = <120>; ++ disable-delay-ms = <50>; ++ unprepare-delay-ms = <20>; ++ ++ width-mm = <68>; ++ height-mm = <121>; ++ ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | ++ MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ ++ panel-init-sequence = [ ++ 05 fa 01 11 ++ 39 00 04 b9 f1 12 83 ++ 39 00 1c ba 33 81 05 f9 0e 0e 00 00 00 ++ 00 00 00 00 00 44 25 00 91 0a ++ 00 00 02 4f 01 00 00 37 ++ 15 00 02 b8 25 ++ 39 00 04 bf 02 11 00 ++ 39 00 0b b3 0c 10 0a 50 03 ff 00 00 00 ++ 00 ++ 39 00 0a c0 73 73 50 50 00 00 08 70 00 ++ 15 00 02 bc 46 ++ 15 00 02 cc 0b ++ 15 00 02 b4 80 ++ 39 00 04 b2 c8 12 30 ++ 39 00 0f e3 07 07 0b 0b 03 0b 00 00 00 ++ 00 ff 00 c0 10 ++ 39 00 0d c1 53 00 1e 1e 77 e1 cc dd 67 ++ 77 33 33 ++ 39 00 07 c6 00 00 ff ff 01 ff ++ 39 00 03 b5 09 09 ++ 39 00 03 b6 87 95 ++ 39 00 40 e9 c2 10 05 05 10 05 a0 12 31 ++ 23 3f 81 0a a0 37 18 00 80 01 ++ 00 00 00 00 80 01 00 00 00 48 ++ f8 86 42 08 88 88 80 88 88 88 ++ 58 f8 87 53 18 88 88 81 88 88 ++ 88 00 00 00 01 00 00 00 00 00 ++ 00 00 00 00 ++ 39 00 3e ea 00 1a 00 00 00 00 02 00 00 ++ 00 00 00 1f 88 81 35 78 88 88 ++ 85 88 88 88 0f 88 80 24 68 88 ++ 88 84 88 88 88 23 10 00 00 1c ++ 00 00 00 00 00 00 00 00 00 00 ++ 00 00 00 00 00 30 05 a0 00 00 ++ 00 00 ++ 39 00 23 e0 00 06 08 2a 31 3f 38 36 07 ++ 0c 0d 11 13 12 13 11 18 00 06 ++ 08 2a 31 3f 38 36 07 0c 0d 11 ++ 13 12 13 11 18 ++ 05 32 01 29 ++ ]; ++ ++ panel-exit-sequence = [ ++ 05 00 01 28 ++ 05 00 01 10 ++ ]; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <64000000>; ++ hactive = <720>; ++ vactive = <1280>; ++ hfront-porch = <40>; ++ hsync-len = <10>; ++ hback-porch = <40>; ++ vfront-porch = <22>; ++ vsync-len = <4>; ++ vback-porch = <11>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ }; ++}; ++ ++&dmc { ++ auto-freq-en = <0>; ++}; ++ ++&vcc3v0_pmu { ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-state-mem { ++ regulator-suspend-microvolt = <3300000>; ++ }; ++}; ++ ++&i2c1 { ++ ++ rk618@50 { ++ compatible = "rockchip,rk618"; ++ reg = <0x50>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1_2ch_mclk>; ++ clocks = <&cru SCLK_I2S1_OUT>; ++ clock-names = "clkin"; ++ assigned-clocks = <&cru SCLK_I2S1_OUT>; ++ assigned-clock-rates = <11289600>; ++ reset-gpios = <&gpio0 RK_PA0 GPIO_ACTIVE_LOW>; ++ status = "okay"; ++ ++ clock: cru { ++ compatible = "rockchip,rk618-cru"; ++ clocks = <&cru SCLK_I2S1_OUT>, <&cru DCLK_VOPL>; ++ clock-names = "clkin", "lcdc0_dclkp"; ++ assigned-clocks = <&clock SCALER_PLLIN_CLK>, ++ <&clock VIF_PLLIN_CLK>, ++ <&clock SCALER_CLK>, ++ <&clock VIF0_PRE_CLK>, ++ <&clock CODEC_CLK>, ++ <&clock DITHER_CLK>; ++ assigned-clock-parents = <&cru SCLK_I2S1_OUT>, ++ <&clock LCDC0_CLK>, ++ <&clock SCALER_PLL_CLK>, ++ <&clock VIF_PLL_CLK>, ++ <&cru SCLK_I2S1_OUT>, ++ <&clock VIF0_CLK>; ++ #clock-cells = <1>; ++ status = "okay"; ++ }; ++ ++ hdmi { ++ compatible = "rockchip,rk618-hdmi"; ++ clocks = <&clock HDMI_CLK>; ++ clock-names = "hdmi"; ++ assigned-clocks = <&clock HDMI_CLK>; ++ assigned-clock-parents = <&clock VIF0_CLK>; ++ interrupt-parent = <&gpio2>; ++ interrupts = <12 IRQ_TYPE_LEVEL_HIGH>; ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ ++ hdmi_in_rgb: endpoint { ++ remote-endpoint = <&rgb_out_hdmi>; ++ }; ++ }; ++ }; ++ }; ++ }; ++}; ++ ++&rgb { ++ status = "okay"; ++ ++ ports { ++ port@1 { ++ reg = <1>; ++ ++ rgb_out_hdmi: endpoint { ++ remote-endpoint = <&hdmi_in_rgb>; ++ }; ++ }; ++ }; ++}; ++ ++&rgb_in_vopb { ++ status = "disabled"; ++}; ++ ++&rgb_in_vopl { ++ status = "okay"; ++}; ++ ++&route_rgb { ++ connect = <&vopl_out_rgb>; ++ status = "disabled"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/px30-robot-no-gpu.dtsi b/arch/arm64/boot/dts/rockchip/px30-robot-no-gpu.dtsi +new file mode 100755 +index 000000000000..e3f4274b2309 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/px30-robot-no-gpu.dtsi +@@ -0,0 +1,57 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++#include "px30-robot.dtsi" ++ ++/ { ++ compatible = "rockchip,linux", "rockchip,px30-robot-no-gpu"; ++ ++ /* Remove gpu thermal and gpu cooling map */ ++ /delete-node/ thermal-zones; ++ ++ thermal_zones: thermal-zones { ++ soc_thermal: soc-thermal { ++ polling-delay-passive = <20>; ++ polling-delay = <1000>; ++ sustainable-power = <252>; ++ ++ thermal-sensors = <&tsadc 0>; ++ trips { ++ threshold: trip-point-0 { ++ temperature = <75000>; ++ hysteresis = <2000>; ++ type = "passive"; ++ }; ++ target: trip-point-1 { ++ temperature = <90000>; ++ hysteresis = <2000>; ++ type = "passive"; ++ }; ++ soc_crit: soc-crit { ++ temperature = <115000>; ++ hysteresis = <2000>; ++ type = "critical"; ++ }; ++ }; ++ ++ cooling-maps { ++ map0 { ++ trip = <&target>; ++ cooling-device = <&cpu0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; ++ }; ++ }; ++ }; ++ ++ gpu_thermal: gpu-thermal { ++ polling-delay-passive = <100>; /* milliseconds */ ++ polling-delay = <1000>; /* milliseconds */ ++ ++ thermal-sensors = <&tsadc 1>; ++ }; ++ }; ++}; ++ ++&gpu { ++ status = "disabled"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/px30-robot.dtsi b/arch/arm64/boot/dts/rockchip/px30-robot.dtsi +new file mode 100755 +index 000000000000..f48a753e6453 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/px30-robot.dtsi +@@ -0,0 +1,93 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++#include "px30.dtsi" ++ ++/ { ++ compatible = "rockchip,linux", "rockchip,px30-robot"; ++ ++ chosen { ++ bootargs = "console=ttyFIQ0 root=PARTUUID=614e0000-0000 rootwait"; ++ }; ++ ++ fiq-debugger { ++ compatible = "rockchip,fiq-debugger"; ++ rockchip,serial-id = <2>; ++ rockchip,wake-irq = <0>; ++ /* If enable uart uses irq instead of fiq */ ++ rockchip,irq-mode-enable = <0>; ++ rockchip,baudrate = <1500000>; /* Only 115200 and 1500000 */ ++ interrupts = ; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart2m0_xfer>; ++ status = "okay"; ++ }; ++ ++ reserved-memory { ++ #address-cells = <2>; ++ #size-cells = <2>; ++ ranges; ++ ++ ramoops: ramoops@8000000 { ++ compatible = "ramoops"; ++ reg = <0x0 0x8000000 0x0 0xa0000>; ++ record-size = <0x20000>; ++ console-size = <0x80000>; ++ ftrace-size = <0x00000>; ++ pmsg-size = <0x00000>; ++ }; ++ }; ++}; ++ ++&cpu0_opp_table { ++ /delete-node/ opp-1248000000; ++ /delete-node/ opp-1296000000; ++ /delete-node/ opp-1416000000; ++ /delete-node/ opp-1512000000; ++}; ++ ++&dmc_opp_table { ++ /delete-node/ opp-666000000; ++ /delete-node/ opp-768000000; ++}; ++ ++&i2s1_2ch { ++ rockchip,playback-only; ++}; ++ ++&rng { ++ status = "okay"; ++}; ++ ++&soc_thermal { ++ trips { ++ threshold: trip-point-0 { ++ temperature = <75000>; ++ hysteresis = <2000>; ++ type = "passive"; ++ }; ++ target: trip-point-1 { ++ temperature = <90000>; ++ hysteresis = <2000>; ++ type = "passive"; ++ }; ++ soc_crit: soc-crit { ++ temperature = <115000>; ++ hysteresis = <2000>; ++ type = "critical"; ++ }; ++ }; ++}; ++ ++&tsadc { ++ pinctrl-names = "gpio", "otpout"; ++ pinctrl-0 = <&tsadc_otp_gpio>; ++ pinctrl-1 = <&tsadc_otp_out>; ++ status = "okay"; ++}; ++ ++&uart2 { ++ status = "disabled"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/px30-z7-a0-rk618-dsi.dts b/arch/arm64/boot/dts/rockchip/px30-z7-a0-rk618-dsi.dts +new file mode 100755 +index 000000000000..1657151c988f +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/px30-z7-a0-rk618-dsi.dts +@@ -0,0 +1,875 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++#include ++#include ++#include ++#include ++#include ++#include ++#include "px30.dtsi" ++#include "px30-android.dtsi" ++ ++/ { ++ model = "Rockchip PX30 Z7 A0 board"; ++ compatible = "rockchip,px30-z7-a0", "rockchip,px30"; ++ ++ adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 2>; ++ io-channel-names = "buttons"; ++ poll-interval = <100>; ++ keyup-threshold-microvolt = <1800000>; ++ ++ esc-key { ++ linux,code = ; ++ label = "esc"; ++ press-threshold-microvolt = <1310000>; ++ }; ++ ++ home-key { ++ linux,code = ; ++ label = "home"; ++ press-threshold-microvolt = <624000>; ++ }; ++ ++ menu-key { ++ linux,code = ; ++ label = "menu"; ++ press-threshold-microvolt = <987000>; ++ }; ++ ++ vol-down-key { ++ linux,code = ; ++ label = "volume down"; ++ press-threshold-microvolt = <300000>; ++ }; ++ ++ vol-up-key { ++ linux,code = ; ++ label = "volume up"; ++ press-threshold-microvolt = <17000>; ++ }; ++ }; ++ ++ backlight: backlight { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm0 0 25000 0>; ++ brightness-levels = < ++ 0 1 2 3 4 5 6 7 ++ 8 9 10 11 12 13 14 15 ++ 16 17 18 19 20 21 22 23 ++ 24 25 26 27 28 29 30 31 ++ 32 33 34 35 36 37 38 39 ++ 40 41 42 43 44 45 46 47 ++ 48 49 50 51 52 53 54 55 ++ 56 57 58 59 60 61 62 63 ++ 64 65 66 67 68 69 70 71 ++ 72 73 74 75 76 77 78 79 ++ 80 81 82 83 84 85 86 87 ++ 88 89 90 91 92 93 94 95 ++ 96 97 98 99 100 101 102 103 ++ 104 105 106 107 108 109 110 111 ++ 112 113 114 115 116 117 118 119 ++ 120 121 122 123 124 125 126 127 ++ 128 129 130 131 132 133 134 135 ++ 136 137 138 139 140 141 142 143 ++ 144 145 146 147 148 149 150 151 ++ 152 153 154 155 156 157 158 159 ++ 160 161 162 163 164 165 166 167 ++ 168 169 170 171 172 173 174 175 ++ 176 177 178 179 180 181 182 183 ++ 184 185 186 187 188 189 190 191 ++ 192 193 194 195 196 197 198 199 ++ 200 201 202 203 204 205 206 207 ++ 208 209 210 211 212 213 214 215 ++ 216 217 218 219 220 221 222 223 ++ 224 225 226 227 228 229 230 231 ++ 232 233 234 235 236 237 238 239 ++ 240 241 242 243 244 245 246 247 ++ 248 249 250 251 252 253 254 255>; ++ default-brightness-level = <200>; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio2 RK_PB1 GPIO_ACTIVE_LOW>; ++ }; ++ ++ vcc_phy: vcc-phy-regulator { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_phy"; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ vcc5v0_sys: vccsys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc5v0_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ }; ++}; ++ ++&display_subsystem { ++ status = "okay"; ++}; ++ ++&dsi { ++ status = "okay"; ++ ++ panel@0 { ++ compatible = "simple-panel-dsi"; ++ reg = <0>; ++ power-supply = <&vcc3v3_lcd>; ++ backlight = <&backlight>; ++ reset-gpios = <&gpio2 RK_PA1 GPIO_ACTIVE_LOW>; ++ prepare-delay-ms = <20>; ++ reset-delay-ms = <20>; ++ init-delay-ms = <20>; ++ enable-delay-ms = <120>; ++ disable-delay-ms = <20>; ++ unprepare-delay-ms = <20>; ++ ++ width-mm = <95>; ++ height-mm = <151>; ++ ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | ++ MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ ++ panel-init-sequence = [ ++ 15 00 02 b0 00 ++ 15 00 02 d6 01 ++ 39 00 06 b3 14 08 00 22 00 ++ 15 00 02 b4 0c ++ 15 00 02 de 00 ++ 39 00 03 b6 3a d3 ++ 15 00 02 51 e0 ++ 15 00 02 53 04 ++ 15 00 02 3a 77 ++ 15 00 02 35 01 ++ 39 00 05 2a 00 00 04 af ++ 39 00 05 2b 00 00 07 7f ++ 05 96 01 29 ++ 05 14 01 11 ++ ]; ++ ++ panel-exit-sequence = [ ++ 05 00 01 28 ++ 05 00 01 10 ++ ]; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <156000000>; ++ hactive = <1200>; ++ vactive = <1920>; ++ hback-porch = <60>; ++ hfront-porch = <80>; ++ vback-porch = <4>; ++ vfront-porch = <4>; ++ hsync-len = <10>; ++ vsync-len = <1>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ panel_in_dsi: endpoint { ++ remote-endpoint = <&dsi_out_panel>; ++ }; ++ }; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@1 { ++ reg = <1>; ++ dsi_out_panel: endpoint { ++ remote-endpoint = <&panel_in_dsi>; ++ }; ++ }; ++ }; ++}; ++ ++&dsi_in_vopb { ++ status = "okay"; ++}; ++ ++&dsi_in_vopl { ++ status = "disabled"; ++}; ++ ++&route_dsi { ++ connect = <&vopb_out_dsi>; ++ status = "okay"; ++}; ++ ++&bus_apll { ++ bus-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&cpu0 { ++ cpu-supply = <&vdd_arm>; ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&dmc { ++ center-supply = <&vdd_logic>; ++ auto-freq-en = <0>; ++ status = "okay"; ++}; ++ ++&emmc { ++ bus-width = <8>; ++ cap-mmc-highspeed; ++ mmc-hs200-1_8v; ++ no-sdio; ++ no-sd; ++ disable-wp; ++ non-removable; ++ num-slots = <1>; ++ status = "okay"; ++}; ++ ++&gpu { ++ mali-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ ++ rk809: pmic@20 { ++ compatible = "rockchip,rk809"; ++ reg = <0x20>; ++ interrupt-parent = <&gpio0>; ++ interrupts = <7 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default", "pmic-sleep", ++ "pmic-power-off", "pmic-reset"; ++ pinctrl-0 = <&pmic_int>; ++ pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; ++ pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; ++ pinctrl-3 = <&soc_slppin_rst>, <&rk817_slppin_rst>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ pmic-reset-func = <1>; ++ ++ vcc1-supply = <&vcc5v0_sys>; ++ vcc2-supply = <&vcc5v0_sys>; ++ vcc3-supply = <&vcc5v0_sys>; ++ vcc4-supply = <&vcc5v0_sys>; ++ vcc5-supply = <&vcc3v3_sys>; ++ vcc6-supply = <&vcc3v3_sys>; ++ vcc7-supply = <&vcc3v3_sys>; ++ vcc8-supply = <&vcc3v3_sys>; ++ vcc9-supply = <&vcc5v0_sys>; ++ ++ pwrkey { ++ status = "okay"; ++ }; ++ ++ pinctrl_rk8xx: pinctrl_rk8xx { ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ rk817_slppin_null: rk817_slppin_null { ++ pins = "gpio_slp"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_slppin_slp: rk817_slppin_slp { ++ pins = "gpio_slp"; ++ function = "pin_fun1"; ++ }; ++ ++ rk817_slppin_pwrdn: rk817_slppin_pwrdn { ++ pins = "gpio_slp"; ++ function = "pin_fun2"; ++ }; ++ ++ rk817_slppin_rst: rk817_slppin_rst { ++ pins = "gpio_slp"; ++ function = "pin_fun3"; ++ }; ++ }; ++ ++ regulators { ++ vdd_logic: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <950000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_logic"; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <950000>; ++ }; ++ }; ++ ++ vdd_arm: DCDC_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <950000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_arm"; ++ ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <950000>; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc_ddr"; ++ regulator-initial-mode = <0x2>; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_3v0: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_3v0"; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ ++ vcc_1v0: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ regulator-name = "vcc_1v0"; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc1v8_soc: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc1v8_soc"; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd1v0_soc: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ regulator-name = "vcc1v0_soc"; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc3v0_pmu: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcc3v0_pmu"; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vccio_sd: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vccio_sd"; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc_sd: LDO_REG6 { ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcc_sd"; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc2v8_dvp: LDO_REG7 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <2800000>; ++ regulator-max-microvolt = <2800000>; ++ regulator-name = "vcc2v8_dvp"; ++ ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <2800000>; ++ }; ++ }; ++ ++ vcc1v8_dvp: LDO_REG8 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc1v8_dvp"; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd1v5_dvp: LDO_REG9 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ regulator-name = "vdd1v5_dvp"; ++ ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <1500000>; ++ }; ++ }; ++ ++ vcc3v3_sys: DCDC_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcc3v3_sys"; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc5v0_host: SWITCH_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc5v0_host"; ++ }; ++ ++ vcc3v3_lcd: SWITCH_REG2 { ++ regulator-boot-on; ++ regulator-name = "vcc3v3_lcd"; ++ }; ++ }; ++ }; ++ ++ rk618@50 { ++ compatible = "rockchip,rk618"; ++ reg = <0x50>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1_2ch_mclk>; ++ clocks = <&cru SCLK_I2S1_OUT>; ++ clock-names = "clkin"; ++ assigned-clocks = <&cru SCLK_I2S1_OUT>; ++ assigned-clock-rates = <12000000>; ++ reset-gpios = <&gpio0 RK_PA0 GPIO_ACTIVE_LOW>; ++ status = "okay"; ++ ++ clock: cru { ++ compatible = "rockchip,rk618-cru"; ++ clocks = <&cru SCLK_I2S1_OUT>, <&cru DCLK_VOPL>; ++ clock-names = "clkin", "lcdc0_dclkp"; ++ assigned-clocks = <&clock SCALER_PLLIN_CLK>, ++ <&clock VIF_PLLIN_CLK>, ++ <&clock SCALER_CLK>, ++ <&clock VIF0_PRE_CLK>, ++ <&clock CODEC_CLK>, ++ <&clock DITHER_CLK>; ++ assigned-clock-parents = <&cru SCLK_I2S1_OUT>, ++ <&clock LCDC0_CLK>, ++ <&clock SCALER_PLL_CLK>, ++ <&clock VIF_PLL_CLK>, ++ <&cru SCLK_I2S1_OUT>, ++ <&clock VIF0_CLK>; ++ #clock-cells = <1>; ++ status = "okay"; ++ }; ++ ++ dsi { ++ compatible = "rockchip,rk618-dsi"; ++ clocks = <&clock MIPI_CLK>; ++ clock-names = "dsi"; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ ++ dsi_in_rgb: endpoint { ++ remote-endpoint = <&rgb_out_dsi>; ++ }; ++ }; ++ }; ++ ++ panel@0 { ++ compatible = "simple-panel-dsi"; ++ reg = <0>; ++ power-supply = <&vcc3v3_lcd>; ++ backlight = <&backlight>; ++ reset-gpios = <&gpio2 RK_PA0 GPIO_ACTIVE_LOW>; ++ prepare-delay-ms = <20>; ++ reset-delay-ms = <20>; ++ init-delay-ms = <20>; ++ enable-delay-ms = <120>; ++ disable-delay-ms = <20>; ++ unprepare-delay-ms = <20>; ++ ++ width-mm = <95>; ++ height-mm = <151>; ++ ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | ++ MIPI_DSI_MODE_VIDEO_BURST | ++ MIPI_DSI_MODE_LPM | ++ MIPI_DSI_MODE_EOT_PACKET)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ ++ panel-init-sequence = [ ++ 15 00 02 b0 00 ++ 15 00 02 d6 01 ++ 39 00 06 b3 14 08 00 22 00 ++ 15 00 02 b4 0c ++ 15 00 02 DE 00 ++ 39 00 03 b6 3a d3 ++ 15 00 02 51 E0 ++ 15 00 02 53 04 ++ 15 00 02 3a 77 ++ 15 00 02 35 01 ++ 39 00 05 2A 00 00 04 AF ++ 39 00 05 2B 00 00 07 7F ++ 05 96 01 29 ++ 05 14 01 11 ++ ]; ++ ++ panel-exit-sequence = [ ++ 05 00 01 28 ++ 05 00 01 10 ++ ]; ++ ++ display-timings { ++ native-mode = <&timing1>; ++ ++ timing1: timing1 { ++ clock-frequency = <156000000>; ++ hactive = <1200>; ++ vactive = <1920>; ++ hback-porch = <60>; ++ hfront-porch = <80>; ++ vback-porch = <4>; ++ vfront-porch = <4>; ++ hsync-len = <10>; ++ vsync-len = <1>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ }; ++ }; ++ }; ++}; ++ ++&io_domains { ++ vccio1-supply = <&vcc1v8_soc>; ++ vccio2-supply = <&vccio_sd>; ++ vccio3-supply = <&vcc_3v0>; ++ vccio4-supply = <&vcc3v0_pmu>; ++ vccio5-supply = <&vcc_3v0>; ++ status = "okay"; ++}; ++ ++&nandc0 { ++ status = "okay"; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ ++ pmuio1-supply = <&vcc3v0_pmu>; ++ pmuio2-supply = <&vcc3v0_pmu>; ++}; ++ ++&pwm0 { ++ status = "okay"; ++}; ++ ++&rk_rga { ++ status = "okay"; ++}; ++ ++&rockchip_suspend { ++ rockchip,sleep-debug-en = <1>; ++ status = "okay"; ++}; ++ ++&saradc { ++ vref-supply = <&vcc1v8_soc>; ++ status = "okay"; ++}; ++ ++&sdmmc { ++ bus-width = <4>; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ no-sdio; ++ no-mmc; ++ card-detect-delay = <800>; ++ ignore-pm-notify; ++ sd-uhs-sdr12; ++ sd-uhs-sdr25; ++ sd-uhs-sdr50; ++ sd-uhs-sdr104; ++ vqmmc-supply = <&vccio_sd>; ++ vmmc-supply = <&vcc_sd>; ++ status = "okay"; ++}; ++ ++&sdio { ++ bus-width = <4>; ++ cap-sd-highspeed; ++ no-sd; ++ no-mmc; ++ ignore-pm-notify; ++ keep-power-in-suspend; ++ non-removable; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&tsadc { ++ pinctrl-names = "init", "default"; ++ pinctrl-0 = <&tsadc_otp_gpio>; ++ pinctrl-1 = <&tsadc_otp_out>; ++ status = "okay"; ++}; ++ ++&uart1 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart1_xfer &uart1_cts>; ++ status = "okay"; ++}; ++ ++&u2phy { ++ status = "okay"; ++ ++ u2phy_host: host-port { ++ status = "okay"; ++ }; ++ ++ u2phy_otg: otg-port { ++ status = "okay"; ++ }; ++}; ++ ++&usb20_otg { ++ status = "okay"; ++}; ++ ++&usb_host0_ehci { ++ status = "okay"; ++}; ++ ++&usb_host0_ohci { ++ status = "okay"; ++}; ++ ++&vopb { ++ status = "okay"; ++}; ++ ++&vopb_mmu { ++ status = "okay"; ++}; ++ ++&vopl { ++ status = "okay"; ++}; ++ ++&vopl_mmu { ++ status = "okay"; ++}; ++ ++&mpp_srv { ++ status = "okay"; ++}; ++ ++&vdpu { ++ status = "okay"; ++}; ++ ++&vepu { ++ status = "okay"; ++}; ++ ++&vpu_mmu { ++ status = "okay"; ++}; ++ ++&hevc { ++ status = "okay"; ++}; ++ ++&hevc_mmu { ++ status = "okay"; ++}; ++ ++&rgb { ++ status = "okay"; ++ ++ ports { ++ port@1 { ++ reg = <1>; ++ ++ rgb_out_dsi: endpoint { ++ remote-endpoint = <&dsi_in_rgb>; ++ }; ++ }; ++ }; ++}; ++ ++&rgb_in_vopl { ++ status = "okay"; ++}; ++ ++&rgb_in_vopb { ++ status = "disabled"; ++}; ++ ++&route_rgb { ++ connect = <&vopl_out_rgb>; ++ status = "okay"; ++}; ++ ++&pinctrl { ++ pmic { ++ pmic_int: pmic_int { ++ rockchip,pins = ++ <0 RK_PA7 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ soc_slppin_gpio: soc_slppin_gpio { ++ rockchip,pins = ++ <0 RK_PA4 RK_FUNC_GPIO &pcfg_output_low>; ++ }; ++ ++ soc_slppin_slp: soc_slppin_slp { ++ rockchip,pins = ++ <0 RK_PA4 1 &pcfg_pull_none>; ++ }; ++ ++ soc_slppin_rst: soc_slppin_rst { ++ rockchip,pins = ++ <0 RK_PA4 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <2 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++&firmware_android { ++ compatible = "android,firmware"; ++ ++ fstab { ++ compatible = "android,fstab"; ++ ++ system { ++ compatible = "android,system"; ++ dev = "/dev/block/by-name/system"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait"; ++ }; ++ ++ vendor { ++ compatible = "android,vendor"; ++ dev = "/dev/block/by-name/vendor"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait"; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/px30.dtsi b/arch/arm64/boot/dts/rockchip/px30.dtsi +index 0d6761074b11..237c0d9af325 100644 +--- a/arch/arm64/boot/dts/rockchip/px30.dtsi ++++ b/arch/arm64/boot/dts/rockchip/px30.dtsi +@@ -25,6 +25,9 @@ aliases { + i2c1 = &i2c1; + i2c2 = &i2c2; + i2c3 = &i2c3; ++ mmc0 = &sdmmc; ++ mmc1 = &sdio; ++ mmc2 = &emmc; + serial0 = &uart0; + serial1 = &uart1; + serial2 = &uart2; +@@ -244,20 +247,20 @@ power: power-controller { + #size-cells = <0>; + + /* These power domains are grouped by VD_LOGIC */ +- power-domain@PX30_PD_USB { ++ pd_usb@PX30_PD_USB { + reg = ; + clocks = <&cru HCLK_HOST>, + <&cru HCLK_OTG>, + <&cru SCLK_OTG_ADP>; + pm_qos = <&qos_usb_host>, <&qos_usb_otg>; + }; +- power-domain@PX30_PD_SDCARD { ++ pd_sdcard@PX30_PD_SDCARD { + reg = ; + clocks = <&cru HCLK_SDMMC>, + <&cru SCLK_SDMMC>; + pm_qos = <&qos_sdmmc>; + }; +- power-domain@PX30_PD_GMAC { ++ pd_gmac@PX30_PD_GMAC { + reg = ; + clocks = <&cru ACLK_GMAC>, + <&cru PCLK_GMAC>, +@@ -265,7 +268,7 @@ power-domain@PX30_PD_GMAC { + <&cru SCLK_GMAC_RX_TX>; + pm_qos = <&qos_gmac>; + }; +- power-domain@PX30_PD_MMC_NAND { ++ pd_mmc_nand@PX30_PD_MMC_NAND { + reg = ; + clocks = <&cru HCLK_NANDC>, + <&cru HCLK_EMMC>, +@@ -278,14 +281,14 @@ power-domain@PX30_PD_MMC_NAND { + pm_qos = <&qos_emmc>, <&qos_nand>, + <&qos_sdio>, <&qos_sfc>; + }; +- power-domain@PX30_PD_VPU { ++ pd_vpu@PX30_PD_VPU { + reg = ; + clocks = <&cru ACLK_VPU>, + <&cru HCLK_VPU>, + <&cru SCLK_CORE_VPU>; + pm_qos = <&qos_vpu>, <&qos_vpu_r128>; + }; +- power-domain@PX30_PD_VO { ++ pd_vo@PX30_PD_VO { + reg = ; + clocks = <&cru ACLK_RGA>, + <&cru ACLK_VOPB>, +@@ -301,7 +304,7 @@ power-domain@PX30_PD_VO { + pm_qos = <&qos_rga_rd>, <&qos_rga_wr>, + <&qos_vop_m0>, <&qos_vop_m1>; + }; +- power-domain@PX30_PD_VI { ++ pd_vi@PX30_PD_VI { + reg = ; + clocks = <&cru ACLK_CIF>, + <&cru ACLK_ISP>, +@@ -312,7 +315,7 @@ power-domain@PX30_PD_VI { + <&qos_isp_wr>, <&qos_isp_m1>, + <&qos_vip>; + }; +- power-domain@PX30_PD_GPU { ++ pd_gpu@PX30_PD_GPU { + reg = ; + clocks = <&cru SCLK_GPU>; + pm_qos = <&qos_gpu>; +@@ -612,7 +615,7 @@ pwm0: pwm@ff200000 { + reg = <0x0 0xff200000 0x0 0x10>; + clocks = <&cru SCLK_PWM0>, <&cru PCLK_PWM0>; + clock-names = "pwm", "pclk"; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm0_pin>; + #pwm-cells = <3>; + status = "disabled"; +@@ -623,7 +626,7 @@ pwm1: pwm@ff200010 { + reg = <0x0 0xff200010 0x0 0x10>; + clocks = <&cru SCLK_PWM0>, <&cru PCLK_PWM0>; + clock-names = "pwm", "pclk"; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm1_pin>; + #pwm-cells = <3>; + status = "disabled"; +@@ -634,7 +637,7 @@ pwm2: pwm@ff200020 { + reg = <0x0 0xff200020 0x0 0x10>; + clocks = <&cru SCLK_PWM0>, <&cru PCLK_PWM0>; + clock-names = "pwm", "pclk"; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm2_pin>; + #pwm-cells = <3>; + status = "disabled"; +@@ -645,7 +648,7 @@ pwm3: pwm@ff200030 { + reg = <0x0 0xff200030 0x0 0x10>; + clocks = <&cru SCLK_PWM0>, <&cru PCLK_PWM0>; + clock-names = "pwm", "pclk"; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm3_pin>; + #pwm-cells = <3>; + status = "disabled"; +@@ -656,7 +659,7 @@ pwm4: pwm@ff208000 { + reg = <0x0 0xff208000 0x0 0x10>; + clocks = <&cru SCLK_PWM1>, <&cru PCLK_PWM1>; + clock-names = "pwm", "pclk"; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm4_pin>; + #pwm-cells = <3>; + status = "disabled"; +@@ -667,7 +670,7 @@ pwm5: pwm@ff208010 { + reg = <0x0 0xff208010 0x0 0x10>; + clocks = <&cru SCLK_PWM1>, <&cru PCLK_PWM1>; + clock-names = "pwm", "pclk"; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm5_pin>; + #pwm-cells = <3>; + status = "disabled"; +@@ -678,7 +681,7 @@ pwm6: pwm@ff208020 { + reg = <0x0 0xff208020 0x0 0x10>; + clocks = <&cru SCLK_PWM1>, <&cru PCLK_PWM1>; + clock-names = "pwm", "pclk"; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm6_pin>; + #pwm-cells = <3>; + status = "disabled"; +@@ -689,7 +692,7 @@ pwm7: pwm@ff208030 { + reg = <0x0 0xff208030 0x0 0x10>; + clocks = <&cru SCLK_PWM1>, <&cru PCLK_PWM1>; + clock-names = "pwm", "pclk"; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm7_pin>; + #pwm-cells = <3>; + status = "disabled"; +diff --git a/arch/arm64/boot/dts/rockchip/rk1808-dram-default-timing.dtsi b/arch/arm64/boot/dts/rockchip/rk1808-dram-default-timing.dtsi +new file mode 100755 +index 000000000000..0fa79e2f05c0 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk1808-dram-default-timing.dtsi +@@ -0,0 +1,302 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++#include ++#include ++ ++/ { ++ ddr_timing: ddr_timing { ++ compatible = "rockchip,ddr-timing"; ++ ddr2_speed_bin = ; ++ ddr3_speed_bin = ; ++ ddr4_speed_bin = ; ++ pd_idle = <0>; ++ sr_idle = <0>; ++ sr_mc_gate_idle = <0>; ++ srpd_lite_idle = <0>; ++ standby_idle = <0>; ++ ++ auto_pd_dis_freq = <1066>; ++ auto_sr_dis_freq = <800>; ++ ddr2_dll_dis_freq = <300>; ++ ddr3_dll_dis_freq = <300>; ++ ddr4_dll_dis_freq = <625>; ++ phy_dll_dis_freq = <400>; ++ ++ ddr2_odt_dis_freq = <100>; ++ phy_ddr2_odt_dis_freq = <100>; ++ ddr2_drv = ; ++ ddr2_odt = ; ++ phy_ddr2_ca_drv = ; ++ phy_ddr2_ck_drv = ; ++ phy_ddr2_dq_drv = ; ++ phy_ddr2_odt = ; ++ ++ ddr3_odt_dis_freq = <400>; ++ phy_ddr3_odt_dis_freq = <400>; ++ ddr3_drv = ; ++ ddr3_odt = ; ++ phy_ddr3_ca_drv = ; ++ phy_ddr3_ck_drv = ; ++ phy_ddr3_dq_drv = ; ++ phy_ddr3_odt = ; ++ ++ phy_lpddr2_odt_dis_freq = <666>; ++ lpddr2_drv = ; ++ phy_lpddr2_ca_drv = ; ++ phy_lpddr2_ck_drv = ; ++ phy_lpddr2_dq_drv = ; ++ phy_lpddr2_odt = ; ++ ++ lpddr3_odt_dis_freq = <400>; ++ phy_lpddr3_odt_dis_freq = <400>; ++ lpddr3_drv = ; ++ lpddr3_odt = ; ++ phy_lpddr3_ca_drv = ; ++ phy_lpddr3_ck_drv = ; ++ phy_lpddr3_dq_drv = ; ++ phy_lpddr3_odt = ; ++ ++ lpddr4_odt_dis_freq = <800>; ++ phy_lpddr4_odt_dis_freq = <800>; ++ lpddr4_drv = ; ++ lpddr4_dq_odt = ; ++ lpddr4_ca_odt = ; ++ phy_lpddr4_ca_drv = ; ++ phy_lpddr4_ck_cs_drv = ; ++ phy_lpddr4_dq_drv = ; ++ phy_lpddr4_odt = ; ++ ++ ddr4_odt_dis_freq = <666>; ++ phy_ddr4_odt_dis_freq = <666>; ++ ddr4_drv = ; ++ ddr4_odt = ; ++ phy_ddr4_ca_drv = ; ++ phy_ddr4_ck_drv = ; ++ phy_ddr4_dq_drv = ; ++ phy_ddr4_odt = ; ++ ++ /* ++ * CA de-skew, one step is 15ps, range 0-31 ++ * DDR3 CA define is different from others(DDR4/LPDDR2/LPDDR3). ++ */ ++ a0_ddr3a9_de-skew = <7>; ++ a1_ddr3a14_de-skew = <7>; ++ a2_ddr3a13_de-skew = <7>; ++ a3_ddr3a11_de-skew = <7>; ++ a4_ddr3a2_de-skew = <7>; ++ a5_ddr3a4_de-skew = <7>; ++ a6_ddr3a3_de-skew = <7>; ++ a7_ddr3a6_de-skew = <7>; ++ a8_ddr3a5_de-skew = <7>; ++ a9_ddr3a1_de-skew = <7>; ++ a10_ddr3a0_de-skew = <7>; ++ a11_ddr3a7_de-skew = <7>; ++ a12_ddr3casb_de-skew = <7>; ++ a13_ddr3a8_de-skew = <7>; ++ a14_ddr3odt0_de-skew = <7>; ++ a15_ddr3ba1_de-skew = <7>; ++ a16_ddr3rasb_de-skew = <7>; ++ a17_ddr3null_de-skew = <7>; ++ ba0_ddr3ba2_de-skew = <7>; ++ ba1_ddr3a12_de-skew = <7>; ++ bg0_ddr3ba0_de-skew = <7>; ++ bg1_ddr3web_de-skew = <7>; ++ cke_ddr3cke_de-skew = <7>; ++ ck_ddr3ck_de-skew = <7>; ++ ckb_ddr3ckb_de-skew = <7>; ++ csb0_ddr3a10_de-skew = <7>; ++ odt0_ddr3a15_de-skew = <7>; ++ resetn_ddr3resetn_de-skew = <7>; ++ actn_ddr3csb0_de-skew = <7>; ++ csb1_ddr3csb1_de-skew = <7>; ++ odt1_ddr3odt1_de-skew = <7>; ++ ++ /* DATA de-skew, one step is 15ps, range 0-31 */ ++ /* cs0_skew_a */ ++ cs0_dm0_rx_de-skew = <7>; ++ cs0_dm0_tx_de-skew = <7>; ++ cs0_dq0_rx_de-skew = <7>; ++ cs0_dq0_tx_de-skew = <7>; ++ cs0_dq1_rx_de-skew = <7>; ++ cs0_dq1_tx_de-skew = <7>; ++ cs0_dq2_rx_de-skew = <7>; ++ cs0_dq2_tx_de-skew = <7>; ++ cs0_dq3_rx_de-skew = <7>; ++ cs0_dq3_tx_de-skew = <7>; ++ cs0_dq4_rx_de-skew = <7>; ++ cs0_dq4_tx_de-skew = <7>; ++ cs0_dq5_rx_de-skew = <7>; ++ cs0_dq5_tx_de-skew = <7>; ++ cs0_dq6_rx_de-skew = <7>; ++ cs0_dq6_tx_de-skew = <7>; ++ cs0_dq7_rx_de-skew = <7>; ++ cs0_dq7_tx_de-skew = <7>; ++ cs0_dqs0p_rx_de-skew = <14>; ++ cs0_dqs0p_tx_de-skew = <9>; ++ cs0_dqs0n_tx_de-skew = <9>; ++ cs0_dm1_rx_de-skew = <7>; ++ cs0_dm1_tx_de-skew = <7>; ++ cs0_dq8_rx_de-skew = <7>; ++ cs0_dq8_tx_de-skew = <7>; ++ cs0_dq9_rx_de-skew = <7>; ++ cs0_dq9_tx_de-skew = <7>; ++ cs0_dq10_rx_de-skew = <7>; ++ cs0_dq10_tx_de-skew = <7>; ++ cs0_dq11_rx_de-skew = <7>; ++ cs0_dq11_tx_de-skew = <7>; ++ cs0_dq12_rx_de-skew = <7>; ++ cs0_dq12_tx_de-skew = <7>; ++ cs0_dq13_rx_de-skew = <7>; ++ cs0_dq13_tx_de-skew = <7>; ++ cs0_dq14_rx_de-skew = <7>; ++ cs0_dq14_tx_de-skew = <7>; ++ cs0_dq15_rx_de-skew = <7>; ++ cs0_dq15_tx_de-skew = <7>; ++ cs0_dqs1p_rx_de-skew = <14>; ++ cs0_dqs1p_tx_de-skew = <9>; ++ cs0_dqs1n_tx_de-skew = <9>; ++ cs0_dqs0n_rx_de-skew = <14>; ++ cs0_dqs1n_rx_de-skew = <14>; ++ ++ /* cs0_skew_b */ ++ cs0_dm2_rx_de-skew = <7>; ++ cs0_dm2_tx_de-skew = <7>; ++ cs0_dq16_rx_de-skew = <7>; ++ cs0_dq16_tx_de-skew = <7>; ++ cs0_dq17_rx_de-skew = <7>; ++ cs0_dq17_tx_de-skew = <7>; ++ cs0_dq18_rx_de-skew = <7>; ++ cs0_dq18_tx_de-skew = <7>; ++ cs0_dq19_rx_de-skew = <7>; ++ cs0_dq19_tx_de-skew = <7>; ++ cs0_dq20_rx_de-skew = <7>; ++ cs0_dq20_tx_de-skew = <7>; ++ cs0_dq21_rx_de-skew = <7>; ++ cs0_dq21_tx_de-skew = <7>; ++ cs0_dq22_rx_de-skew = <7>; ++ cs0_dq22_tx_de-skew = <7>; ++ cs0_dq23_rx_de-skew = <7>; ++ cs0_dq23_tx_de-skew = <7>; ++ cs0_dqs2p_rx_de-skew = <14>; ++ cs0_dqs2p_tx_de-skew = <9>; ++ cs0_dqs2n_tx_de-skew = <9>; ++ cs0_dm3_rx_de-skew = <7>; ++ cs0_dm3_tx_de-skew = <7>; ++ cs0_dq24_rx_de-skew = <7>; ++ cs0_dq24_tx_de-skew = <7>; ++ cs0_dq25_rx_de-skew = <7>; ++ cs0_dq25_tx_de-skew = <7>; ++ cs0_dq26_rx_de-skew = <7>; ++ cs0_dq26_tx_de-skew = <7>; ++ cs0_dq27_rx_de-skew = <7>; ++ cs0_dq27_tx_de-skew = <7>; ++ cs0_dq28_rx_de-skew = <7>; ++ cs0_dq28_tx_de-skew = <7>; ++ cs0_dq29_rx_de-skew = <7>; ++ cs0_dq29_tx_de-skew = <7>; ++ cs0_dq30_rx_de-skew = <7>; ++ cs0_dq30_tx_de-skew = <7>; ++ cs0_dq31_rx_de-skew = <7>; ++ cs0_dq31_tx_de-skew = <7>; ++ cs0_dqs3p_rx_de-skew = <14>; ++ cs0_dqs3p_tx_de-skew = <9>; ++ cs0_dqs3n_tx_de-skew = <9>; ++ cs0_dqs2n_rx_de-skew = <14>; ++ cs0_dqs3n_rx_de-skew = <14>; ++ ++ /* cs1_skew_a */ ++ cs1_dm0_rx_de-skew = <7>; ++ cs1_dm0_tx_de-skew = <7>; ++ cs1_dq0_rx_de-skew = <7>; ++ cs1_dq0_tx_de-skew = <7>; ++ cs1_dq1_rx_de-skew = <7>; ++ cs1_dq1_tx_de-skew = <7>; ++ cs1_dq2_rx_de-skew = <7>; ++ cs1_dq2_tx_de-skew = <7>; ++ cs1_dq3_rx_de-skew = <7>; ++ cs1_dq3_tx_de-skew = <7>; ++ cs1_dq4_rx_de-skew = <7>; ++ cs1_dq4_tx_de-skew = <7>; ++ cs1_dq5_rx_de-skew = <7>; ++ cs1_dq5_tx_de-skew = <7>; ++ cs1_dq6_rx_de-skew = <7>; ++ cs1_dq6_tx_de-skew = <7>; ++ cs1_dq7_rx_de-skew = <7>; ++ cs1_dq7_tx_de-skew = <7>; ++ cs1_dqs0p_rx_de-skew = <14>; ++ cs1_dqs0p_tx_de-skew = <9>; ++ cs1_dqs0n_tx_de-skew = <9>; ++ cs1_dm1_rx_de-skew = <7>; ++ cs1_dm1_tx_de-skew = <7>; ++ cs1_dq8_rx_de-skew = <7>; ++ cs1_dq8_tx_de-skew = <7>; ++ cs1_dq9_rx_de-skew = <7>; ++ cs1_dq9_tx_de-skew = <7>; ++ cs1_dq10_rx_de-skew = <7>; ++ cs1_dq10_tx_de-skew = <7>; ++ cs1_dq11_rx_de-skew = <7>; ++ cs1_dq11_tx_de-skew = <7>; ++ cs1_dq12_rx_de-skew = <7>; ++ cs1_dq12_tx_de-skew = <7>; ++ cs1_dq13_rx_de-skew = <7>; ++ cs1_dq13_tx_de-skew = <7>; ++ cs1_dq14_rx_de-skew = <7>; ++ cs1_dq14_tx_de-skew = <7>; ++ cs1_dq15_rx_de-skew = <7>; ++ cs1_dq15_tx_de-skew = <7>; ++ cs1_dqs1p_rx_de-skew = <14>; ++ cs1_dqs1p_tx_de-skew = <9>; ++ cs1_dqs1n_tx_de-skew = <9>; ++ cs1_dqs0n_rx_de-skew = <14>; ++ cs1_dqs1n_rx_de-skew = <14>; ++ ++ /* cs1_skew_b */ ++ cs1_dm2_rx_de-skew = <7>; ++ cs1_dm2_tx_de-skew = <7>; ++ cs1_dq16_rx_de-skew = <7>; ++ cs1_dq16_tx_de-skew = <7>; ++ cs1_dq17_rx_de-skew = <7>; ++ cs1_dq17_tx_de-skew = <7>; ++ cs1_dq18_rx_de-skew = <7>; ++ cs1_dq18_tx_de-skew = <7>; ++ cs1_dq19_rx_de-skew = <7>; ++ cs1_dq19_tx_de-skew = <7>; ++ cs1_dq20_rx_de-skew = <7>; ++ cs1_dq20_tx_de-skew = <7>; ++ cs1_dq21_rx_de-skew = <7>; ++ cs1_dq21_tx_de-skew = <7>; ++ cs1_dq22_rx_de-skew = <7>; ++ cs1_dq22_tx_de-skew = <7>; ++ cs1_dq23_rx_de-skew = <7>; ++ cs1_dq23_tx_de-skew = <7>; ++ cs1_dqs2p_rx_de-skew = <14>; ++ cs1_dqs2p_tx_de-skew = <9>; ++ cs1_dqs2n_tx_de-skew = <9>; ++ cs1_dm3_rx_de-skew = <7>; ++ cs1_dm3_tx_de-skew = <7>; ++ cs1_dq24_rx_de-skew = <7>; ++ cs1_dq24_tx_de-skew = <7>; ++ cs1_dq25_rx_de-skew = <7>; ++ cs1_dq25_tx_de-skew = <7>; ++ cs1_dq26_rx_de-skew = <7>; ++ cs1_dq26_tx_de-skew = <7>; ++ cs1_dq27_rx_de-skew = <7>; ++ cs1_dq27_tx_de-skew = <7>; ++ cs1_dq28_rx_de-skew = <7>; ++ cs1_dq28_tx_de-skew = <7>; ++ cs1_dq29_rx_de-skew = <7>; ++ cs1_dq29_tx_de-skew = <7>; ++ cs1_dq30_rx_de-skew = <7>; ++ cs1_dq30_tx_de-skew = <7>; ++ cs1_dq31_rx_de-skew = <7>; ++ cs1_dq31_tx_de-skew = <7>; ++ cs1_dqs3p_rx_de-skew = <14>; ++ cs1_dqs3p_tx_de-skew = <9>; ++ cs1_dqs3n_tx_de-skew = <9>; ++ cs1_dqs2n_rx_de-skew = <14>; ++ cs1_dqs3n_rx_de-skew = <14>; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk1808-evb-v10.dts b/arch/arm64/boot/dts/rockchip/rk1808-evb-v10.dts +new file mode 100755 +index 000000000000..a09824816188 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk1808-evb-v10.dts +@@ -0,0 +1,305 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++#include ++#include "rk1808-evb.dtsi" ++ ++/ { ++ model = "Rockchip RK1808 EVB V10 Board"; ++ compatible = "rockchip,rk1808-evb-v10", "rockchip,rk1808"; ++ ++ chosen { ++ bootargs = "earlycon=uart8250,mmio32,0xff550000 console=ttyFIQ0 root=PARTUUID=614e0000-0000 rootfstype=ext4 rootwait kpti=0 snd_aloop.index=7"; ++ }; ++ ++ vad-sound { ++ status = "okay"; ++ compatible = "rockchip,multicodecs-card"; ++ rockchip,card-name = "rockchip,rk1808-vad"; ++ rockchip,cpu = <&i2s0>; ++ rockchip,codec = <&vad>; ++ }; ++}; ++ ++&adc_key { ++ vol-down-key { ++ linux,code = ; ++ label = "volume down"; ++ press-threshold-microvolt = <300000>; ++ }; ++ ++ vol-up-key { ++ linux,code = ; ++ label = "volume up"; ++ press-threshold-microvolt = <18000>; ++ }; ++}; ++ ++&display_subsystem { ++ status = "okay"; ++}; ++ ++&dsi { ++ status = "okay"; ++ ++ panel@0 { ++ compatible = "sitronix,st7703", "simple-panel-dsi"; ++ reg = <0>; ++ backlight = <&backlight>; ++ enable-gpios = <&gpio0 RK_PC4 GPIO_ACTIVE_HIGH>; ++ power-supply = <&vcc5v0_sys>; ++ prepare-delay-ms = <2>; ++ reset-delay-ms = <1>; ++ init-delay-ms = <20>; ++ enable-delay-ms = <120>; ++ disable-delay-ms = <50>; ++ unprepare-delay-ms = <20>; ++ ++ width-mm = <68>; ++ height-mm = <121>; ++ ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | ++ MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ ++ panel-init-sequence = [ ++ 05 fa 01 11 ++ 39 00 04 b9 f1 12 83 ++ 39 00 1c ba 33 81 05 f9 0e 0e 00 00 00 ++ 00 00 00 00 00 44 25 00 91 0a ++ 00 00 02 4f 01 00 00 37 ++ 15 00 02 b8 25 ++ 39 00 04 bf 02 11 00 ++ 39 00 0b b3 0c 10 0a 50 03 ff 00 00 00 ++ 00 ++ 39 00 0a c0 73 73 50 50 00 00 08 70 00 ++ 15 00 02 bc 46 ++ 15 00 02 cc 0b ++ 15 00 02 b4 80 ++ 39 00 04 b2 c8 12 30 ++ 39 00 0f e3 07 07 0b 0b 03 0b 00 00 00 ++ 00 ff 00 c0 10 ++ 39 00 0d c1 53 00 1e 1e 77 e1 cc dd 67 ++ 77 33 33 ++ 39 00 07 c6 00 00 ff ff 01 ff ++ 39 00 03 b5 09 09 ++ 39 00 03 b6 87 95 ++ 39 00 40 e9 c2 10 05 05 10 05 a0 12 31 ++ 23 3f 81 0a a0 37 18 00 80 01 ++ 00 00 00 00 80 01 00 00 00 48 ++ f8 86 42 08 88 88 80 88 88 88 ++ 58 f8 87 53 18 88 88 81 88 88 ++ 88 00 00 00 01 00 00 00 00 00 ++ 00 00 00 00 ++ 39 00 3e ea 00 1a 00 00 00 00 02 00 00 ++ 00 00 00 1f 88 81 35 78 88 88 ++ 85 88 88 88 0f 88 80 24 68 88 ++ 88 84 88 88 88 23 10 00 00 1c ++ 00 00 00 00 00 00 00 00 00 00 ++ 00 00 00 00 00 30 05 a0 00 00 ++ 00 00 ++ 39 00 23 e0 00 06 08 2a 31 3f 38 36 07 ++ 0c 0d 11 13 12 13 11 18 00 06 ++ 08 2a 31 3f 38 36 07 0c 0d 11 ++ 13 12 13 11 18 ++ 05 32 01 29 ++ ]; ++ ++ panel-exit-sequence = [ ++ 05 00 01 28 ++ 05 00 01 10 ++ ]; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <64000000>; ++ hactive = <720>; ++ vactive = <1280>; ++ hfront-porch = <40>; ++ hsync-len = <10>; ++ hback-porch = <40>; ++ vfront-porch = <22>; ++ vsync-len = <4>; ++ vback-porch = <11>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ panel_in_dsi: endpoint { ++ remote-endpoint = <&dsi_out_panel>; ++ }; ++ }; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@1 { ++ reg = <1>; ++ dsi_out_panel: endpoint { ++ remote-endpoint = <&panel_in_dsi>; ++ }; ++ }; ++ }; ++}; ++ ++&i2c3 { ++ status = "okay"; ++ ++ clock-frequency = <100000>; ++ ++ ov5695: ov5695@36 { ++ compatible = "ovti,ov5695"; ++ reg = <0x36>; ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ avdd-supply = <&vcc2v8_dvp>; ++ dovdd-supply = <&vdd1v5_dvp>; ++ dvdd-supply = <&vcc1v8_dvp>; ++ pwdn-gpios = <&gpio0 23 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_clkout_m0>; ++ port { ++ ucam_out: endpoint { ++ remote-endpoint = <&mipi_in_ucam>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++}; ++ ++&i2s0 { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++}; ++ ++&i2s1 { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++}; ++ ++&isp_mmu { ++ status = "okay"; ++}; ++ ++&mipi_dphy { ++ status = "okay"; ++}; ++ ++&mipi_dphy_rx { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ucam_out>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_rx0_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp0_mipi_in>; ++ }; ++ }; ++ }; ++}; ++ ++&rk_rga { ++ status = "okay"; ++}; ++ ++&rk809_sound { ++ status = "okay"; ++}; ++ ++&rkisp1 { ++ status = "okay"; ++ ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp0_mipi_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dphy_rx0_out>; ++ }; ++ }; ++}; ++ ++&rng { ++ status = "okay"; ++}; ++ ++&rockchip_suspend { ++ status = "okay"; ++ rockchip,sleep-debug-en = <1>; ++}; ++ ++&route_dsi { ++ status = "disabled"; ++}; ++ ++&tsadc { ++ rockchip,hw-tshut-mode = <1>; /* tshut mode 0:CRU 1:GPIO */ ++ rockchip,hw-tshut-polarity = <0>; /* tshut polarity 0:LOW 1:HIGH */ ++ pinctrl-names = "gpio", "otpout"; ++ pinctrl-0 = <&tsadc_otp_gpio>; ++ pinctrl-1 = <&tsadc_otp_out>; ++ status = "okay"; ++}; ++ ++&vad { ++ status = "okay"; ++ rockchip,audio-src = <&i2s0>; ++ rockchip,buffer-time-ms = <200>; ++ rockchip,det-channel = <0>; ++ rockchip,mode = <1>; ++ #sound-dai-cells = <0>; ++}; ++ ++&vop_lite { ++ status = "okay"; ++}; ++ ++&vopl_mmu { ++ status = "okay"; ++}; ++ ++&vpu_mmu { ++ status = "okay"; ++}; ++ ++&vpu_service { ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk1808-evb-x4-second.dts b/arch/arm64/boot/dts/rockchip/rk1808-evb-x4-second.dts +new file mode 100755 +index 000000000000..413d4f6fa29d +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk1808-evb-x4-second.dts +@@ -0,0 +1,272 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++#include ++#include "rk1808-evb.dtsi" ++ ++/ { ++ model = "Rockchip RK1808 EVB X4 Board"; ++ compatible = "rockchip,rk1808-evb-x4", "rockchip,rk1808"; ++ ++ chosen { ++ bootargs = "earlycon=uart8250,mmio32,0xff550000 console=ttyFIQ0 dump_initrd init=/init kpti=0"; ++ }; ++}; ++ ++&adc_key { ++ power-key { ++ linux,code = ; ++ label = "power key"; ++ press-threshold-microvolt = <18000>; ++ }; ++}; ++ ++/delete-node/ &backlight; ++/delete-node/ &vcc1v8_dvp; ++/delete-node/ &vdd1v5_dvp; ++/delete-node/ &vcc2v8_dvp; ++ ++&cif { ++ status = "okay"; ++ ++ port { ++ cif_in: endpoint@0 { ++ remote-endpoint = <&dphy_rx_out>; ++ data-lanes = <1 2 3 4>; ++ }; ++ }; ++}; ++ ++&cif_mmu { ++ status = "okay"; ++}; ++ ++&cru { ++ assigned-clocks = ++ <&cru PLL_GPLL>, <&cru PLL_CPLL>, ++ <&cru PLL_PPLL>, <&cru ARMCLK>, ++ <&cru MSCLK_PERI>, <&cru LSCLK_PERI>, ++ <&cru HSCLK_BUS_PRE>, <&cru MSCLK_BUS_PRE>, ++ <&cru LSCLK_BUS_PRE>, <&cru DCLK_VOPRAW>; ++ assigned-clock-rates = ++ <1188000000>, <1000000000>, ++ <100000000>, <816000000>, ++ <200000000>, <100000000>, ++ <300000000>, <200000000>, ++ <100000000>, <80000000>; ++}; ++ ++&csi_tx { ++ status = "okay"; ++ csi-tx-bypass-mode = <1>; ++ ++ panel@0 { ++ compatible = "simple-panel-dsi"; ++ reg = <0>; ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | ++ MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET | ++ MIPI_DSI_CLOCK_NON_CONTINUOUS)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ ++ display-timings { ++ native-mode = <&timing_1280x3_720>; ++ ++ timing_1280x3_720: timing-1280x3-720 { ++ clock-frequency = <80000000>; ++ hactive = <3840>; ++ vactive = <720>; ++ hfront-porch = <1200>; ++ hsync-len = <500>; ++ hback-porch = <30>; ++ vfront-porch = <40>; ++ vsync-len = <20>; ++ vback-porch = <40>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ timing_4k: timing-4k { ++ clock-frequency = <250000000>; ++ hactive = <3840>; ++ vactive = <2160>; ++ hfront-porch = <1500>; ++ hsync-len = <500>; ++ hback-porch = <30>; ++ vfront-porch = <40>; ++ vsync-len = <20>; ++ vback-porch = <40>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ timing_4096: timing-4096 { ++ clock-frequency = <100000000>; ++ hactive = <4096>; ++ vactive = <2048>; ++ hfront-porch = <1500>; ++ hsync-len = <500>; ++ hback-porch = <30>; ++ vfront-porch = <40>; ++ vsync-len = <20>; ++ vback-porch = <40>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ timing_1920x3_1080: timing-1920x3-1080 { ++ clock-frequency = <250000000>; ++ hactive = <5760>; ++ vactive = <1080>; ++ hfront-porch = <1500>; ++ hsync-len = <70>; ++ hback-porch = <30>; ++ vfront-porch = <40>; ++ vsync-len = <20>; ++ vback-porch = <40>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ }; ++}; ++ ++&display_subsystem { ++ status = "okay"; ++}; ++ ++&emmc { ++ status = "disabled"; ++}; ++ ++&gmac { ++ status = "disabled"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ ++ vcamera@30 { ++ compatible = "rockchip,virtual-camera"; ++ reg = <0x30>; ++ width = <3840>; ++ height = <720>; ++ bus-format = ; ++ ++ port { ++ vcamera_out: endpoint { ++ remote-endpoint = <&dphy_rx_in>; ++ link-frequencies = /bits/ 64 <320000000>; ++ }; ++ }; ++ }; ++}; ++ ++&i2c1 { ++ status = "disabled"; ++}; ++ ++&i2c4 { ++ status = "disabled"; ++}; ++ ++&mipi_dphy { ++ status = "okay"; ++}; ++ ++&mipi_dphy_rx { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_rx_in: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&vcamera_out>; ++ data-lanes = <1 2 3 4>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_rx_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&cif_in>; ++ }; ++ }; ++ }; ++}; ++ ++&rk809_codec { ++ status = "disabled"; ++}; ++ ++&rk_rga { ++ status = "okay"; ++}; ++ ++&route_csi { ++ status = "disabled"; ++}; ++ ++&sdmmc { ++ status = "disabled"; ++}; ++ ++&sdio { ++ status = "disabled"; ++}; ++ ++&sfc { ++ status = "okay"; ++}; ++ ++&uart4 { ++ status = "disabled"; ++}; ++ ++&wireless_bluetooth { ++ status = "disabled"; ++}; ++ ++&wireless_wlan { ++ status = "disabled"; ++}; ++ ++&tsadc { ++ rockchip,hw-tshut-mode = <1>; /* tshut mode 0:CRU 1:GPIO */ ++ rockchip,hw-tshut-polarity = <0>; /* tshut polarity 0:LOW 1:HIGH */ ++ pinctrl-names = "init", "default"; ++ pinctrl-0 = <&tsadc_otp_gpio>; ++ pinctrl-1 = <&tsadc_otp_out>; ++ status = "okay"; ++}; ++ ++&vop_raw { ++ status = "okay"; ++}; ++ ++&vopr_mmu { ++ status = "okay"; ++}; ++ ++&vpu_mmu { ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk1808-evb-x4.dts b/arch/arm64/boot/dts/rockchip/rk1808-evb-x4.dts +new file mode 100755 +index 000000000000..17993d1ff44a +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk1808-evb-x4.dts +@@ -0,0 +1,271 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++#include ++#include "rk1808-evb.dtsi" ++ ++/ { ++ model = "Rockchip RK1808 EVB X4 Board"; ++ compatible = "rockchip,rk1808-evb-x4", "rockchip,rk1808"; ++ ++ chosen { ++ bootargs = "earlycon=uart8250,mmio32,0xff550000 console=ttyFIQ0 dump_initrd init=/init kpti=0"; ++ }; ++}; ++ ++&adc_key { ++ power-key { ++ linux,code = ; ++ label = "power key"; ++ press-threshold-microvolt = <18000>; ++ }; ++}; ++ ++/delete-node/ &backlight; ++/delete-node/ &vcc1v8_dvp; ++/delete-node/ &vdd1v5_dvp; ++/delete-node/ &vcc2v8_dvp; ++ ++&cif { ++ status = "okay"; ++ ++ port { ++ cif_in: endpoint@0 { ++ remote-endpoint = <&dphy_rx_out>; ++ data-lanes = <1 2 3 4>; ++ }; ++ }; ++}; ++ ++&cif_mmu { ++ status = "okay"; ++}; ++ ++&cru { ++ assigned-clocks = ++ <&cru PLL_GPLL>, <&cru PLL_CPLL>, ++ <&cru PLL_PPLL>, <&cru ARMCLK>, ++ <&cru MSCLK_PERI>, <&cru LSCLK_PERI>, ++ <&cru HSCLK_BUS_PRE>, <&cru MSCLK_BUS_PRE>, ++ <&cru LSCLK_BUS_PRE>, <&cru DCLK_VOPRAW>; ++ assigned-clock-rates = ++ <1188000000>, <1000000000>, ++ <100000000>, <816000000>, ++ <200000000>, <100000000>, ++ <300000000>, <200000000>, ++ <100000000>, <80000000>; ++}; ++ ++&csi_tx { ++ status = "okay"; ++ ++ panel@0 { ++ compatible = "simple-panel-dsi"; ++ reg = <0>; ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | ++ MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET | ++ MIPI_DSI_CLOCK_NON_CONTINUOUS)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ ++ display-timings { ++ native-mode = <&timing_1280x3_720>; ++ ++ timing_1280x3_720: timing-1280x3-720 { ++ clock-frequency = <80000000>; ++ hactive = <3840>; ++ vactive = <720>; ++ hfront-porch = <1200>; ++ hsync-len = <500>; ++ hback-porch = <30>; ++ vfront-porch = <40>; ++ vsync-len = <20>; ++ vback-porch = <40>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ timing_4k: timing-4k { ++ clock-frequency = <250000000>; ++ hactive = <3840>; ++ vactive = <2160>; ++ hfront-porch = <1500>; ++ hsync-len = <500>; ++ hback-porch = <30>; ++ vfront-porch = <40>; ++ vsync-len = <20>; ++ vback-porch = <40>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ timing_4096: timing-4096 { ++ clock-frequency = <190000000>; ++ hactive = <4096>; ++ vactive = <2048>; ++ hfront-porch = <1500>; ++ hsync-len = <500>; ++ hback-porch = <30>; ++ vfront-porch = <40>; ++ vsync-len = <20>; ++ vback-porch = <40>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ timing_1920x3_1080: timing-1920x3-1080 { ++ clock-frequency = <250000000>; ++ hactive = <5760>; ++ vactive = <1080>; ++ hfront-porch = <1500>; ++ hsync-len = <70>; ++ hback-porch = <30>; ++ vfront-porch = <40>; ++ vsync-len = <20>; ++ vback-porch = <40>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ }; ++}; ++ ++&display_subsystem { ++ status = "okay"; ++}; ++ ++&emmc { ++ status = "disabled"; ++}; ++ ++&gmac { ++ status = "disabled"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ ++ vcamera@30 { ++ compatible = "rockchip,virtual-camera"; ++ reg = <0x30>; ++ width = <1280>; ++ height = <720>; ++ bus-format = ; ++ ++ port { ++ vcamera_out: endpoint { ++ remote-endpoint = <&dphy_rx_in>; ++ link-frequencies = /bits/ 64 <320000000>; ++ }; ++ }; ++ }; ++}; ++ ++&i2c1 { ++ status = "disabled"; ++}; ++ ++&i2c4 { ++ status = "disabled"; ++}; ++ ++&mipi_dphy { ++ status = "okay"; ++}; ++ ++&mipi_dphy_rx { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_rx_in: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&vcamera_out>; ++ data-lanes = <1 2 3 4>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_rx_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&cif_in>; ++ }; ++ }; ++ }; ++}; ++ ++&rk809_codec { ++ status = "disabled"; ++}; ++ ++&rk_rga { ++ status = "okay"; ++}; ++ ++&route_csi { ++ status = "disabled"; ++}; ++ ++&sdmmc { ++ status = "disabled"; ++}; ++ ++&sdio { ++ status = "disabled"; ++}; ++ ++&sfc { ++ status = "okay"; ++}; ++ ++&uart4 { ++ status = "disabled"; ++}; ++ ++&wireless_bluetooth { ++ status = "disabled"; ++}; ++ ++&wireless_wlan { ++ status = "disabled"; ++}; ++ ++&tsadc { ++ rockchip,hw-tshut-mode = <1>; /* tshut mode 0:CRU 1:GPIO */ ++ rockchip,hw-tshut-polarity = <0>; /* tshut polarity 0:LOW 1:HIGH */ ++ pinctrl-names = "init", "default"; ++ pinctrl-0 = <&tsadc_otp_gpio>; ++ pinctrl-1 = <&tsadc_otp_out>; ++ status = "okay"; ++}; ++ ++&vop_raw { ++ status = "okay"; ++}; ++ ++&vopr_mmu { ++ status = "okay"; ++}; ++ ++&vpu_mmu { ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk1808-evb.dtsi b/arch/arm64/boot/dts/rockchip/rk1808-evb.dtsi +new file mode 100755 +index 000000000000..3b9c88660701 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk1808-evb.dtsi +@@ -0,0 +1,717 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++// Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd. ++ ++#include ++#include ++#include ++#include ++#include ++#include "rk1808.dtsi" ++ ++/ { ++ model = "Rockchip RK1808 EVB"; ++ compatible = "rockchip,rk1808-evb", "rockchip,rk1808"; ++ ++ adc_key: adc-keys { ++ compatible = "adc-keys"; ++ autorepeat; ++ io-channels = <&saradc 2>; ++ io-channel-names = "buttons"; ++ keyup-threshold-microvolt = <1800000>; ++ poll-interval = <100>; ++ }; ++ ++ backlight: backlight { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm1 0 25000 0>; ++ brightness-levels = < ++ 0 1 2 3 4 5 6 7 ++ 8 9 10 11 12 13 14 15 ++ 16 17 18 19 20 21 22 23 ++ 24 25 26 27 28 29 30 31 ++ 32 33 34 35 36 37 38 39 ++ 40 41 42 43 44 45 46 47 ++ 48 49 50 51 52 53 54 55 ++ 56 57 58 59 60 61 62 63 ++ 64 65 66 67 68 69 70 71 ++ 72 73 74 75 76 77 78 79 ++ 80 81 82 83 84 85 86 87 ++ 88 89 90 91 92 93 94 95 ++ 96 97 98 99 100 101 102 103 ++ 104 105 106 107 108 109 110 111 ++ 112 113 114 115 116 117 118 119 ++ 120 121 122 123 124 125 126 127 ++ 128 129 130 131 132 133 134 135 ++ 136 137 138 139 140 141 142 143 ++ 144 145 146 147 148 149 150 151 ++ 152 153 154 155 156 157 158 159 ++ 160 161 162 163 164 165 166 167 ++ 168 169 170 171 172 173 174 175 ++ 176 177 178 179 180 181 182 183 ++ 184 185 186 187 188 189 190 191 ++ 192 193 194 195 196 197 198 199 ++ 200 201 202 203 204 205 206 207 ++ 208 209 210 211 212 213 214 215 ++ 216 217 218 219 220 221 222 223 ++ 224 225 226 227 228 229 230 231 ++ 232 233 234 235 236 237 238 239 ++ 240 241 242 243 244 245 246 247 ++ 248 249 250 251 252 253 254 255>; ++ default-brightness-level = <200>; ++ }; ++ ++ display_subsystem: display-subsystem { ++ compatible = "rockchip,display-subsystem"; ++ ports = <&vop_lite_out>, <&vop_raw_out>; ++ logo-memory-region = <&drm_logo>; ++ status = "disabled"; ++ ++ route { ++ route_csi: route-csi { ++ status = "disabled"; ++ logo,uboot = "logo.bmp"; ++ logo,kernel = "logo_kernel.bmp"; ++ logo,mode = "center"; ++ charge_logo,mode = "center"; ++ connect = <&vop_raw_out_csi>; ++ }; ++ ++ route_dsi: route-dsi { ++ status = "disabled"; ++ logo,uboot = "logo.bmp"; ++ logo,kernel = "logo_kernel.bmp"; ++ logo,mode = "center"; ++ charge_logo,mode = "center"; ++ connect = <&vop_lite_out_dsi>; ++ }; ++ ++ route_rgb: route-rgb { ++ status = "disabled"; ++ logo,uboot = "logo.bmp"; ++ logo,kernel = "logo_kernel.bmp"; ++ logo,mode = "center"; ++ charge_logo,mode = "center"; ++ connect = <&vop_lite_out_rgb>; ++ }; ++ }; ++ }; ++ ++ fiq-debugger { ++ compatible = "rockchip,fiq-debugger"; ++ rockchip,serial-id = <2>; ++ rockchip,wake-irq = <0>; ++ /* If enable uart uses irq instead of fiq */ ++ rockchip,irq-mode-enable = <0>; ++ rockchip,baudrate = <1500000>; /* Only 115200 and 1500000 */ ++ interrupts = ; ++ status = "okay"; ++ }; ++ ++ reserved-memory { ++ #address-cells = <2>; ++ #size-cells = <2>; ++ ranges; ++ ++ drm_logo: drm-logo@00000000 { ++ compatible = "rockchip,drm-logo"; ++ reg = <0x0 0x0 0x0 0x0>; ++ }; ++ ++ ramoops: ramoops@110000 { ++ compatible = "ramoops"; ++ reg = <0x0 0x110000 0x0 0xf0000>; ++ record-size = <0x30000>; ++ console-size = <0xc0000>; ++ ftrace-size = <0x00000>; ++ pmsg-size = <0x00000>; ++ }; ++ }; ++ ++ rk809_sound: rk809-sound { ++ status = "disabled"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,rk809-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,widgets = ++ "Microphone", "Mic Jack", ++ "Headphone", "Headphone Jack"; ++ simple-audio-card,routing = ++ "Mic Jack", "MICBIAS1", ++ "IN1P", "Mic Jack", ++ "Headphone Jack", "HPOL", ++ "Headphone Jack", "HPOR"; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s1>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&rk809_codec>; ++ }; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio4 RK_PC0 GPIO_ACTIVE_LOW>; ++ }; ++ ++ vcc_otg_vbus: otg-vbus-regulator { ++ compatible = "regulator-fixed"; ++ gpio = <&gpio0 RK_PB7 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&otg_vbus_drv>; ++ regulator-name = "vcc_otg_vbus"; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ enable-active-high; ++ }; ++ ++ vcc5v0_sys: vcc5v0-sys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc5v0_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ }; ++ ++ vcc_phy: vcc-phy-regulator { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_phy"; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ wireless_bluetooth: wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ uart_rts_gpios = <&gpio4 RK_PB7 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart4_rts>; ++ pinctrl-1 = <&uart4_rts_gpio>; ++ BT,power_gpio = <&gpio4 RK_PC3 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio4 RK_PC2 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ wireless_wlan: wireless-wlan { ++ compatible = "wlan-platdata"; ++ rockchip,grf = <&grf>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_wake_host>; ++ wifi_chip_type = "ap6212"; ++ WIFI,host_wake_irq = <&gpio4 RK_PC1 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++}; ++ ++&cpu0 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu1 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&combphy { ++ status = "okay"; ++}; ++ ++&emmc { ++ bus-width = <8>; ++ cap-mmc-highspeed; ++ max-frequency = <200000000>; ++ mmc-hs200-1_8v; ++ no-sdio; ++ no-sd; ++ non-removable; ++ num-slots = <1>; ++ status = "okay"; ++}; ++ ++&gmac { ++ phy-supply = <&vcc_phy>; ++ phy-mode = "rgmii"; ++ clock_in_out = "input"; ++ snps,reset-gpio = <&gpio0 10 GPIO_ACTIVE_LOW>; ++ snps,reset-active-low; ++ /* Reset time is 20ms, 100ms for rtl8211f */ ++ snps,reset-delays-us = <0 20000 100000>; ++ assigned-clocks = <&cru SCLK_GMAC>; ++ assigned-clock-parents = <&gmac_clkin>; ++ tx_delay = <0x50>; ++ rx_delay = <0x3a>; ++ status = "okay"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ clock-frequency = <400000>; ++ ++ vdd_npu: tcs4525@1c { ++ compatible = "tcs,tcs452x"; ++ reg = <0x1c>; ++ vin-supply = <&vcc5v0_sys>; ++ regulator-compatible = "fan53555-reg"; ++ pinctrl-0 = <&vsel_gpio>; ++ vsel-gpios = <&gpio0 5 GPIO_ACTIVE_HIGH>; ++ regulator-name = "vdd_npu"; ++ regulator-min-microvolt = <750000>; ++ regulator-max-microvolt = <950000>; ++ regulator-ramp-delay = <2300>; ++ fcs,suspend-voltage-selector = <0>; ++ regulator-boot-on; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ rk809: pmic@20 { ++ compatible = "rockchip,rk809"; ++ reg = <0x20>; ++ interrupt-parent = <&gpio0>; ++ interrupts = <1 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default", "pmic-sleep", ++ "pmic-power-off", "pmic-reset"; ++ pinctrl-0 = <&pmic_int>; ++ pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; ++ pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; ++ pinctrl-3 = <&soc_slppin_gpio>, <&rk817_slppin_null>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ //fb-inner-reg-idxs = <2>; ++ /* 1: rst regs (default in codes), 0: rst the pmic */ ++ pmic-reset-func = <0>; ++ ++ vcc1-supply = <&vcc5v0_sys>; ++ vcc2-supply = <&vcc5v0_sys>; ++ vcc3-supply = <&vcc5v0_sys>; ++ vcc4-supply = <&vcc5v0_sys>; ++ vcc5-supply = <&vcc_buck5>; ++ vcc6-supply = <&vcc_buck5>; ++ vcc7-supply = <&vcc5v0_sys>; ++ vcc8-supply = <&vcc_3v3>; ++ vcc9-supply = <&vcc5v0_sys>; ++ ++ pwrkey { ++ status = "okay"; ++ }; ++ ++ rtc { ++ status = "okay"; ++ }; ++ ++ pinctrl_rk8xx: pinctrl_rk8xx { ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ rk817_slppin_null: rk817_slppin_null { ++ pins = "gpio_slp"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_slppin_slp: rk817_slppin_slp { ++ pins = "gpio_slp"; ++ function = "pin_fun1"; ++ }; ++ ++ rk817_slppin_pwrdn: rk817_slppin_pwrdn { ++ pins = "gpio_slp"; ++ function = "pin_fun2"; ++ }; ++ ++ rk817_slppin_rst: rk817_slppin_rst { ++ pins = "gpio_slp"; ++ function = "pin_fun3"; ++ }; ++ }; ++ ++ regulators { ++ vdd_log: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <750000>; ++ regulator-max-microvolt = <950000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_log"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <800000>; ++ }; ++ }; ++ ++ vdd_cpu: DCDC_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <750000>; ++ regulator-max-microvolt = <950000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_cpu"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc_ddr"; ++ regulator-initial-mode = <0x2>; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_3v3: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_3v3"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vdda_0v8: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <800000>; ++ regulator-max-microvolt = <800000>; ++ regulator-name = "vdda_0v8"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <800000>; ++ }; ++ }; ++ ++ vcc_1v8: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcc_1v8"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd_0v8: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <800000>; ++ regulator-max-microvolt = <800000>; ++ ++ regulator-name = "vdd_0v8"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <800000>; ++ }; ++ }; ++ ++ vcca_1v8: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcca_1v8"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ ++ }; ++ }; ++ ++ vcc1v8_dvp: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcc1v8_dvp"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd1v5_dvp: LDO_REG6 { ++ regulator-min-microvolt = <1200000>; ++ regulator-max-microvolt = <1500000>; ++ ++ regulator-name = "vdd1v5_dvp"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1500000>; ++ ++ }; ++ }; ++ ++ vcc2v8_dvp: LDO_REG7 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <2800000>; ++ regulator-max-microvolt = <2800000>; ++ ++ regulator-name = "vcc2v8_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <2800000>; ++ }; ++ }; ++ ++ vccio_sd: LDO_REG8 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vccio_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc3v3_sd: LDO_REG9 { ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vcc3v3_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc_buck5: DCDC_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <2200000>; ++ regulator-max-microvolt = <2200000>; ++ regulator-name = "vcc_buck5"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <2200000>; ++ }; ++ }; ++ ++ vcc5v0_host: SWITCH_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc5v0_host"; ++ }; ++ ++ vccio_3v3: SWITCH_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vccio_3v3"; ++ }; ++ ++ }; ++ ++ rk809_codec: codec { ++ #sound-dai-cells = <0>; ++ compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; ++ clocks = <&cru SCLK_I2S1_2CH_OUT>; ++ clock-names = "mclk"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1_2ch_mclk>; ++ hp-volume = <20>; ++ spk-volume = <3>; ++ status = "okay"; ++ }; ++ }; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ ++ gt1x: gt1x@14 { ++ compatible = "goodix,gt1x"; ++ reg = <0x14>; ++ goodix,rst-gpio = <&gpio0 RK_PB4 GPIO_ACTIVE_HIGH>; ++ goodix,irq-gpio = <&gpio0 RK_PB5 IRQ_TYPE_LEVEL_LOW>; ++ }; ++}; ++ ++&i2c4 { ++ status = "okay"; ++ ++ sensor@d { ++ status = "okay"; ++ compatible = "ak8963"; ++ reg = <0x0d>; ++ type = ; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ layout = <1>; ++ reprobe_en = <1>; ++ }; ++ ++ sensor@4c { ++ status = "okay"; ++ compatible = "gs_mma7660"; ++ reg = <0x4c>; ++ type = ; ++ irq-gpio = <&gpio0 RK_PC6 IRQ_TYPE_LEVEL_LOW>; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ layout = <2>; ++ reprobe_en = <1>; ++ }; ++}; ++ ++&npu { ++ npu-supply = <&vdd_npu>; ++ status = "okay"; ++}; ++ ++&pcie0 { ++ reset-gpios = <&gpio0 RK_PB6 GPIO_ACTIVE_HIGH>; ++ /* Disable usbdrd_dwc3 and usbdrd3 if using pcie0 */ ++ status = "disabled"; ++}; ++ ++&power { ++ pd_npu-supply = <&vdd_npu>; ++}; ++ ++&pwm1 { ++ status = "okay"; ++}; ++ ++&saradc { ++ status = "okay"; ++ vref-supply = <&vcc_1v8>; ++}; ++ ++&sdio { ++ bus-width = <4>; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ no-sd; ++ no-mmc; ++ keep-power-in-suspend; ++ non-removable; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&sdmmc { ++ bus-width = <4>; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ no-sdio; ++ no-mmc; ++ card-detect-delay = <300>; ++ sd-uhs-sdr25; ++ sd-uhs-sdr50; ++ sd-uhs-sdr104; ++ vmmc-supply = <&vcc3v3_sd>; ++ vqmmc-supply = <&vccio_sd>; ++ status = "okay"; ++}; ++ ++&uart4 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart4_xfer &uart4_cts>; ++ status = "okay"; ++}; ++ ++&u2phy { ++ status = "okay"; ++}; ++ ++&u2phy_host { ++ status = "okay"; ++}; ++ ++&u2phy_otg { ++ status = "okay"; ++ vbus-supply = <&vcc_otg_vbus>; ++}; ++ ++&usb_host0_ehci { ++ status = "okay"; ++}; ++ ++&usb_host0_ohci { ++ status = "okay"; ++}; ++ ++&usbdrd3 { ++ status = "okay"; ++ extcon = <&u2phy>; ++}; ++ ++&usbdrd_dwc3 { ++ status = "okay"; ++}; ++ ++&pinctrl { ++ pmic { ++ pmic_int: pmic_int { ++ rockchip,pins = ++ <0 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ soc_slppin_gpio: soc_slppin_gpio { ++ rockchip,pins = ++ <0 RK_PA4 RK_FUNC_GPIO &pcfg_output_low>; ++ }; ++ ++ soc_slppin_slp: soc_slppin_slp { ++ rockchip,pins = ++ <0 RK_PA4 1 &pcfg_pull_none>; ++ }; ++ ++ vsel_gpio: vsel-gpio { ++ rockchip,pins = ++ <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = ++ <4 RK_PC0 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ usb2 { ++ otg_vbus_drv: otg-vbus-drv { ++ rockchip,pins = <0 RK_PB7 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart4_rts_gpio: uart4-rts-gpio { ++ rockchip,pins = <4 RK_PB7 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ wireless-wlan { ++ wifi_wake_host: wifi-wake-host { ++ rockchip,pins = ++ <4 RK_PC1 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk1808-fpga.dts b/arch/arm64/boot/dts/rockchip/rk1808-fpga.dts +new file mode 100755 +index 000000000000..d021918cacd1 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk1808-fpga.dts +@@ -0,0 +1,58 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++// Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ ++/dts-v1/; ++#include "rk1808.dtsi" ++ ++/ { ++ model = "Rockchip rk1808 fpga board"; ++ compatible = "rockchip,fpga", "rockchip,rk1808"; ++ ++ chosen { ++ bootargs = "earlycon=uart8250,mmio32,0xff550000 console=ttyFIQ0 root=/dev/mmcblk1p8 rootfstype=ext4 rootwait clk_ignore_unused"; ++ }; ++ ++ memory@200000 { ++ device_type = "memory"; ++ reg = <0x0 0x00200000 0x0 0x0FE00000>; ++ }; ++ ++ fiq_debugger: fiq-debugger { ++ compatible = "rockchip,fiq-debugger"; ++ rockchip,serial-id = <2>; ++ rockchip,wake-irq = <0>; ++ rockchip,irq-mode-enable = <1>; ++ rockchip,baudrate = <1500000>; /* Only 115200 and 1500000 */ ++ interrupts = ; ++ status = "okay"; ++ }; ++}; ++ ++&emmc { ++ max-frequency = <400000>; ++ clocks = <&xin24m>, <&xin24m>, <&xin24m>, <&xin24m>; ++ clock-names = "biu", "ciu", "ciu-drv", "ciu-sample"; ++ mmc-hs200-1_8v; ++ no-sdio; ++ no-sd; ++ num-slots = <1>; ++ status = "okay"; ++}; ++ ++&npu { ++ status = "okay"; ++}; ++ ++&sdmmc { ++ max-frequency = <400000>; ++ clocks = <&xin24m>, <&xin24m>, <&xin24m>, <&xin24m>; ++ clock-names = "biu", "ciu", "ciu-drv", "ciu-sample"; ++ no-sdio; ++ no-mmc; ++ status = "okay"; ++}; ++ ++/* If fiq_debugger set okay, need to define uart2 and to be disabled */ ++&uart2 { ++ status = "disabled"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk1808.dtsi b/arch/arm64/boot/dts/rockchip/rk1808.dtsi +new file mode 100755 +index 000000000000..3469abac4f34 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk1808.dtsi +@@ -0,0 +1,3040 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++// Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd. ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include "rk1808-dram-default-timing.dtsi" ++ ++/ { ++ compatible = "rockchip,rk1808"; ++ ++ interrupt-parent = <&gic>; ++ #address-cells = <2>; ++ #size-cells = <2>; ++ ++ aliases { ++ i2c0 = &i2c0; ++ i2c1 = &i2c1; ++ i2c2 = &i2c2; ++ i2c3 = &i2c3; ++ i2c4 = &i2c4; ++ i2c5 = &i2c5; ++ serial0 = &uart0; ++ serial1 = &uart1; ++ serial2 = &uart2; ++ serial3 = &uart3; ++ serial4 = &uart4; ++ serial5 = &uart5; ++ serial6 = &uart6; ++ serial7 = &uart7; ++ spi0 = &spi0; ++ spi1 = &spi1; ++ spi2 = &spi2; ++ }; ++ ++ cpus { ++ #address-cells = <2>; ++ #size-cells = <0>; ++ ++ cpu0: cpu@0 { ++ device_type = "cpu"; ++ compatible = "arm,cortex-a35", "arm,armv8"; ++ reg = <0x0 0x0>; ++ enable-method = "psci"; ++ clocks = <&cru ARMCLK>; ++ operating-points-v2 = <&cpu0_opp_table>; ++ dynamic-power-coefficient = <74>; ++ #cooling-cells = <2>; ++ cpu-idle-states = <&CPU_SLEEP>; ++ power-model { ++ compatible = "simple-power-model"; ++ ref-leakage = <31>; ++ static-coefficient = <100000>; ++ ts = <597400 241050 (-2450) 70>; ++ thermal-zone = "soc-thermal"; ++ }; ++ }; ++ ++ cpu1: cpu@1 { ++ device_type = "cpu"; ++ compatible = "arm,cortex-a35", "arm,armv8"; ++ reg = <0x0 0x1>; ++ enable-method = "psci"; ++ clocks = <&cru ARMCLK>; ++ operating-points-v2 = <&cpu0_opp_table>; ++ dynamic-power-coefficient = <74>; ++ cpu-idle-states = <&CPU_SLEEP>; ++ }; ++ ++ idle-states { ++ entry-method = "psci"; ++ ++ CPU_SLEEP: cpu-sleep { ++ compatible = "arm,idle-state"; ++ local-timer-stop; ++ arm,psci-suspend-param = <0x0010000>; ++ entry-latency-us = <120>; ++ exit-latency-us = <250>; ++ min-residency-us = <900>; ++ }; ++ ++ CLUSTER_SLEEP: cluster-sleep { ++ compatible = "arm,idle-state"; ++ local-timer-stop; ++ arm,psci-suspend-param = <0x1010000>; ++ entry-latency-us = <400>; ++ exit-latency-us = <500>; ++ min-residency-us = <2000>; ++ }; ++ }; ++ }; ++ ++ cpu0_opp_table: cpu0-opp-table { ++ compatible = "operating-points-v2"; ++ opp-shared; ++ ++ rockchip,temp-hysteresis = <5000>; ++ rockchip,low-temp = <0>; ++ rockchip,low-temp-min-volt = <800000>; ++ rockchip,low-temp-adjust-volt = < ++ /* MHz MHz uV */ ++ 0 1608 50000 ++ >; ++ ++ rockchip,max-volt = <950000>; ++ rockchip,evb-irdrop = <25000>; ++ nvmem-cells = <&cpu_leakage>; ++ nvmem-cell-names = "leakage"; ++ ++ rockchip,pvtm-voltage-sel = < ++ 0 69000 0 ++ 69001 74000 1 ++ 74001 99999 2 ++ >; ++ rockchip,pvtm-freq = <408000>; ++ rockchip,pvtm-volt = <800000>; ++ rockchip,pvtm-ch = <0 0>; ++ rockchip,pvtm-sample-time = <1000>; ++ rockchip,pvtm-number = <10>; ++ rockchip,pvtm-error = <1000>; ++ rockchip,pvtm-ref-temp = <25>; ++ rockchip,pvtm-temp-prop = <(-20) (-26)>; ++ rockchip,thermal-zone = "soc-thermal"; ++ ++ opp-408000000 { ++ opp-hz = /bits/ 64 <408000000>; ++ opp-microvolt = <750000 750000 950000>; ++ clock-latency-ns = <40000>; ++ opp-suspend; ++ }; ++ opp-600000000 { ++ opp-hz = /bits/ 64 <600000000>; ++ opp-microvolt = <750000 750000 950000>; ++ clock-latency-ns = <40000>; ++ }; ++ opp-816000000 { ++ opp-hz = /bits/ 64 <816000000>; ++ opp-microvolt = <750000 750000 950000>; ++ clock-latency-ns = <40000>; ++ }; ++ opp-1008000000 { ++ opp-hz = /bits/ 64 <1008000000>; ++ opp-microvolt = <750000 750000 950000>; ++ clock-latency-ns = <40000>; ++ }; ++ opp-1200000000 { ++ opp-hz = /bits/ 64 <1200000000>; ++ opp-microvolt = <800000 800000 950000>; ++ opp-microvolt-L0 = <800000 800000 950000>; ++ opp-microvolt-L1 = <750000 750000 950000>; ++ opp-microvolt-L2 = <750000 750000 950000>; ++ clock-latency-ns = <40000>; ++ }; ++ opp-1296000000 { ++ opp-hz = /bits/ 64 <1296000000>; ++ opp-microvolt = <825000 825000 950000>; ++ opp-microvolt-L0 = <825000 825000 950000>; ++ opp-microvolt-L1 = <775000 775000 950000>; ++ opp-microvolt-L2 = <750000 750000 950000>; ++ clock-latency-ns = <40000>; ++ }; ++ opp-1416000000 { ++ opp-hz = /bits/ 64 <1416000000>; ++ opp-microvolt = <850000 850000 950000>; ++ opp-microvolt-L0 = <850000 850000 950000>; ++ opp-microvolt-L1 = <800000 800000 950000>; ++ opp-microvolt-L2 = <775000 775000 950000>; ++ clock-latency-ns = <40000>; ++ }; ++ opp-1512000000 { ++ opp-hz = /bits/ 64 <1512000000>; ++ opp-microvolt = <875000 875000 950000>; ++ opp-microvolt-L0 = <875000 875000 950000>; ++ opp-microvolt-L1 = <825000 825000 950000>; ++ opp-microvolt-L2 = <800000 800000 950000>; ++ clock-latency-ns = <40000>; ++ }; ++ opp-1608000000 { ++ opp-hz = /bits/ 64 <1608000000>; ++ opp-microvolt = <900000 900000 950000>; ++ opp-microvolt-L0 = <900000 900000 950000>; ++ opp-microvolt-L1 = <850000 850000 950000>; ++ opp-microvolt-L2 = <825000 825000 950000>; ++ clock-latency-ns = <40000>; ++ }; ++ }; ++ ++ arm-pmu { ++ compatible = "arm,cortex-a53-pmu"; ++ interrupts = , ++ ; ++ interrupt-affinity = <&cpu0>, <&cpu1>; ++ }; ++ ++ cpuinfo { ++ compatible = "rockchip,cpuinfo"; ++ nvmem-cells = <&efuse_id>, <&efuse_cpu_version>; ++ nvmem-cell-names = "id", "cpu-version"; ++ }; ++ ++ bus_soc: bus-soc { ++ compatible = "rockchip,rk1808-bus"; ++ rockchip,busfreq-policy = "smc"; ++ soc-bus0 { ++ bus-id = <0>; ++ cfg-val = <0x1e0>; ++ enable-msk = <0x407f>; ++ status = "okay"; ++ }; ++ soc-bus1 { ++ bus-id = <1>; ++ cfg-val = <0x12c0>; ++ enable-msk = <0x41ff>; ++ status = "okay"; ++ }; ++ soc-bus2 { ++ bus-id = <2>; ++ cfg-val = <0x12c0>; ++ enable-msk = <0x4005>; ++ status = "okay"; ++ }; ++ soc-bus3 { ++ bus-id = <3>; ++ cfg-val = <0x12c0>; ++ enable-msk = <0x4001>; ++ status = "okay"; ++ }; ++ soc-bus4 { ++ bus-id = <4>; ++ cfg-val = <0x12c0>; ++ enable-msk = <0x4001>; ++ status = "disabled"; ++ }; ++ }; ++ ++ firmware { ++ optee: optee { ++ compatible = "linaro,optee-tz"; ++ method = "smc"; ++ status = "disabled"; ++ }; ++ }; ++ ++ gmac_clkin: external-gmac-clock { ++ compatible = "fixed-clock"; ++ clock-frequency = <125000000>; ++ clock-output-names = "gmac_clkin"; ++ #clock-cells = <0>; ++ }; ++ ++ psci { ++ compatible = "arm,psci-1.0"; ++ method = "smc"; ++ }; ++ ++ rockchip_suspend: rockchip-suspend { ++ compatible = "rockchip,pm-rk1808"; ++ status = "disabled"; ++ rockchip,sleep-debug-en = <0>; ++ rockchip,sleep-mode-config = < ++ (0 ++ | RKPM_SLP_ARMOFF ++ | RKPM_SLP_PMU_PMUALIVE_32K ++ | RKPM_SLP_PMU_DIS_OSC ++ | RKPM_SLP_PMIC_LP ++ | RKPM_SLP_32K_EXT ++ ) ++ >; ++ rockchip,wakeup-config = < ++ (0 ++ | RKPM_GPIO_WKUP_EN ++ ) ++ >; ++ }; ++ ++ timer { ++ compatible = "arm,armv8-timer"; ++ interrupts = , ++ , ++ , ++ ; ++ arm,no-tick-in-suspend; ++ }; ++ ++ xin24m: xin24m { ++ compatible = "fixed-clock"; ++ clock-frequency = <24000000>; ++ clock-output-names = "xin24m"; ++ #clock-cells = <0>; ++ }; ++ ++ xin32k: xin32k { ++ compatible = "fixed-clock"; ++ clock-frequency = <32768>; ++ clock-output-names = "xin32k"; ++ #clock-cells = <0>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&clkin_32k>; ++ }; ++ ++ pcie0: pcie@fc400000 { ++ compatible = "rockchip,rk1808-pcie", "snps,dw-pcie"; ++ #address-cells = <3>; ++ #size-cells = <2>; ++ bus-range = <0x0 0x1f>; ++ clocks = <&cru HSCLK_PCIE>, <&cru LSCLK_PCIE>, ++ <&cru ACLK_PCIE>, <&cru PCLK_PCIE>, ++ <&cru SCLK_PCIE_AUX>; ++ clock-names = "hsclk", "lsclk", ++ "aclk", "pclk", ++ "sclk-aux"; ++ interrupts = , ++ , ++ , ++ ; ++ interrupt-names = "sys", "legacy", "msg", "err"; ++ linux,pci-domain = <0>; ++ num-ib-windows = <6>; ++ num-ob-windows = <2>; ++ msi-map = <0x0 &its 0x0 0x1000>; ++ num-lanes = <2>; ++ phys = <&combphy PHY_TYPE_PCIE>; ++ phy-names = "pcie-phy"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pcie_clkreq>; ++ power-domains = <&power RK1808_PD_PCIE>; ++ ranges = <0x00000800 0x0 0xf8000000 0x0 0xf8000000 0x0 0x800000 ++ 0x83000000 0x0 0xf8800000 0x0 0xf8800000 0x0 0x3700000 ++ 0x81000000 0x0 0xfbf00000 0x0 0xfbf00000 0x0 0x100000>; ++ reg = <0x0 0xfc000000 0x0 0x400000>, ++ <0x0 0xfc400000 0x0 0x10000>; ++ reg-names = "pcie-dbi", "pcie-apb"; ++ resets = <&cru SRST_PCIE_NIU_H>, <&cru SRST_PCIE_NIU_L>, ++ <&cru SRST_PCIEGRF_P>, <&cru SRST_PCIECTL_P>, ++ <&cru SRST_PCIECTL_POWERUP>, <&cru SRST_PCIECTL_MST_A>, ++ <&cru SRST_PCIECTL_SLV_A>, <&cru SRST_PCIECTL_DBI_A>, ++ <&cru SRST_PCIECTL_BUTTON>, <&cru SRST_PCIECTL_PE>, ++ <&cru SRST_PCIECTL_CORE>, <&cru SRST_PCIECTL_NSTICKY>, ++ <&cru SRST_PCIECTL_STICKY>, <&cru SRST_PCIECTL_PWR>, ++ <&cru SRST_PCIE_NIU_A>, <&cru SRST_PCIE_NIU_P>; ++ reset-names = "niu-h", "niu-l", "grf-p", "ctl-p", ++ "ctl-powerup", "ctl-mst-a", "ctl-slv-a", ++ "ctl-dbi-a", "ctl-button", "ctl-pe", ++ "ctl-core", "ctl-nsticky", "ctl-sticky", ++ "ctl-pwr", "ctl-niu-a", "ctl-niu-p"; ++ rockchip,usbpciegrf = <&usb_pcie_grf>; ++ rockchip,pmugrf = <&pmugrf>; ++ status = "disabled"; ++ }; ++ ++ usbdrd3: usb { ++ compatible = "rockchip,rk1808-dwc3", "rockchip,rk3399-dwc3"; ++ clocks = <&cru SCLK_USB3_OTG0_REF>, <&cru ACLK_USB3OTG>, ++ <&cru SCLK_USB3_OTG0_SUSPEND>; ++ clock-names = "ref_clk", "bus_clk", ++ "suspend_clk"; ++ assigned-clocks = <&cru SCLK_USB3_OTG0_SUSPEND>; ++ assigned-clock-rates = <24000000>; ++ power-domains = <&power RK1808_PD_PCIE>; ++ resets = <&cru SRST_USB3_OTG_A>; ++ reset-names = "usb3-otg"; ++ #address-cells = <2>; ++ #size-cells = <2>; ++ ranges; ++ status = "disabled"; ++ ++ usbdrd_dwc3: dwc3@fd000000 { ++ compatible = "snps,dwc3"; ++ reg = <0x0 0xfd000000 0x0 0x200000>; ++ interrupts = ; ++ dr_mode = "otg"; ++ phys = <&u2phy_otg>, <&combphy PHY_TYPE_USB3>; ++ phy-names = "usb2-phy", "usb3-phy"; ++ phy_type = "utmi_wide"; ++ snps,dis_enblslpm_quirk; ++ snps,dis-u1u2-quirk; ++ snps,dis-u2-freeclk-exists-quirk; ++ snps,dis_u2_susphy_quirk; ++ snps,dis_u3_susphy_quirk; ++ snps,dis-del-phy-power-chg-quirk; ++ snps,tx-ipgap-linecheck-dis-quirk; ++ snps,xhci-trb-ent-quirk; ++ status = "disabled"; ++ }; ++ }; ++ ++ grf: syscon@fe000000 { ++ compatible = "rockchip,rk1808-grf", "syscon", "simple-mfd"; ++ reg = <0x0 0xfe000000 0x0 0x1000>; ++ #address-cells = <1>; ++ #size-cells = <1>; ++ ++ npu_pvtm: npu-pvtm { ++ compatible = "rockchip,rk1808-npu-pvtm"; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ status = "okay"; ++ ++ pvtm@2 { ++ reg = <2>; ++ clocks = <&cru SCLK_PVTM_NPU>; ++ clock-names = "clk"; ++ }; ++ }; ++ ++ rgb: rgb { ++ compatible = "rockchip,rk1808-rgb"; ++ status = "disabled"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ ++ rgb_in_vop_lite: endpoint { ++ remote-endpoint = <&vop_lite_out_rgb>; ++ }; ++ }; ++ }; ++ }; ++ }; ++ ++ usb2phy_grf: syscon@fe010000 { ++ compatible = "rockchip,rk1808-usb2phy-grf", "syscon", ++ "simple-mfd"; ++ reg = <0x0 0xfe010000 0x0 0x8000>; ++ #address-cells = <1>; ++ #size-cells = <1>; ++ ++ u2phy: usb2-phy@100 { ++ compatible = "rockchip,rk1808-usb2phy"; ++ reg = <0x100 0x10>; ++ clocks = <&cru SCLK_USBPHY_REF>; ++ clock-names = "phyclk"; ++ #clock-cells = <0>; ++ assigned-clocks = <&cru USB480M>; ++ assigned-clock-parents = <&u2phy>; ++ clock-output-names = "usb480m_phy"; ++ status = "disabled"; ++ ++ u2phy_host: host-port { ++ #phy-cells = <0>; ++ interrupts = ; ++ interrupt-names = "linestate"; ++ status = "disabled"; ++ }; ++ ++ u2phy_otg: otg-port { ++ #phy-cells = <0>; ++ interrupts = , ++ , ++ ; ++ interrupt-names = "otg-bvalid", "otg-id", ++ "linestate"; ++ status = "disabled"; ++ }; ++ }; ++ }; ++ ++ combphy_grf: syscon@fe018000 { ++ compatible = "rockchip,usb3phy-grf", "syscon"; ++ reg = <0x0 0xfe018000 0x0 0x8000>; ++ }; ++ ++ pmugrf: syscon@fe020000 { ++ compatible = "rockchip,rk1808-pmugrf", "syscon", "simple-mfd"; ++ reg = <0x0 0xfe020000 0x0 0x1000>; ++ #address-cells = <1>; ++ #size-cells = <1>; ++ ++ pmu_pvtm: pmu-pvtm { ++ compatible = "rockchip,rk1808-pmu-pvtm"; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ status = "okay"; ++ ++ pvtm@1 { ++ reg = <1>; ++ clocks = <&cru SCLK_PVTM_PMU>; ++ clock-names = "clk"; ++ }; ++ }; ++ ++ reboot-mode { ++ compatible = "syscon-reboot-mode"; ++ offset = <0x200>; ++ mode-bootloader = ; ++ mode-charge = ; ++ mode-fastboot = ; ++ mode-loader = ; ++ mode-normal = ; ++ mode-recovery = ; ++ mode-ums = ; ++ }; ++ }; ++ ++ usb_pcie_grf: syscon@fe040000 { ++ compatible = "rockchip,usb-pcie-grf", "syscon"; ++ reg = <0x0 0xfe040000 0x0 0x1000>; ++ }; ++ ++ coregrf: syscon@fe050000 { ++ compatible = "rockchip,rk1808-coregrf", "syscon", "simple-mfd"; ++ reg = <0x0 0xfe050000 0x0 0x1000>; ++ #address-cells = <1>; ++ #size-cells = <1>; ++ ++ pvtm: pvtm { ++ compatible = "rockchip,rk1808-pvtm"; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ status = "okay"; ++ ++ pvtm@0 { ++ reg = <0>; ++ clocks = <&cru SCLK_PVTM_CORE>; ++ clock-names = "clk"; ++ }; ++ }; ++ }; ++ ++ qos_npu: qos@fe850000 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe850000 0x0 0x20>; ++ }; ++ ++ qos_pcie: qos@fe880000 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe880000 0x0 0x20>; ++ status = "disabled"; ++ }; ++ ++ qos_usb2: qos@fe890000 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe890000 0x0 0x20>; ++ status = "disabled"; ++ }; ++ ++ qos_usb3: qos@fe890080 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe890080 0x0 0x20>; ++ status = "disabled"; ++ }; ++ ++ qos_isp: qos@fe8a0000 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe8a0000 0x0 0x20>; ++ }; ++ ++ qos_rga_rd: qos@fe8a0080 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe8a0080 0x0 0x20>; ++ }; ++ ++ qos_rga_wr: qos@fe8a0100 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe8a0100 0x0 0x20>; ++ }; ++ ++ qos_cif: qos@fe8a0180 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe8a0180 0x0 0x20>; ++ }; ++ ++ qos_vop_raw: qos@fe8b0000 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe8b0000 0x0 0x20>; ++ }; ++ ++ qos_vop_lite: qos@fe8b0080 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe8b0080 0x0 0x20>; ++ }; ++ ++ qos_vpu: qos@fe8c0000 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe8c0000 0x0 0x20>; ++ }; ++ ++ sram: sram@fec00000 { ++ compatible = "mmio-sram"; ++ reg = <0x0 0xfec00000 0x0 0x200000>; ++ #address-cells = <1>; ++ #size-cells = <1>; ++ ranges = <0 0x0 0xfec00000 0x200000>; ++ /* reserved for ddr dvfs and system suspend/resume */ ++ ddr-sram@0 { ++ reg = <0x0 0x8000>; ++ }; ++ /* reserved for vad audio buffer */ ++ vad_sram: vad-sram@1c0000 { ++ reg = <0x1c0000 0x40000>; ++ }; ++ }; ++ ++ hwlock: hwspinlock@ff040000 { ++ compatible = "rockchip,hwspinlock"; ++ reg = <0 0xff040000 0 0x10000>; ++ #hwlock-cells = <1>; ++ }; ++ ++ gic: interrupt-controller@ff100000 { ++ compatible = "arm,gic-v3"; ++ #interrupt-cells = <3>; ++ #address-cells = <2>; ++ #size-cells = <2>; ++ ranges; ++ interrupt-controller; ++ ++ reg = <0x0 0xff100000 0 0x10000>, /* GICD */ ++ <0x0 0xff140000 0 0xc0000>, /* GICR */ ++ <0x0 0xff300000 0 0x10000>, /* GICC */ ++ <0x0 0xff310000 0 0x10000>, /* GICH */ ++ <0x0 0xff320000 0 0x10000>; /* GICV */ ++ interrupts = ; ++ its: interrupt-controller@ff120000 { ++ compatible = "arm,gic-v3-its"; ++ msi-controller; ++ reg = <0x0 0xff120000 0x0 0x20000>; ++ }; ++ }; ++ ++ efuse: efuse@ff260000 { ++ compatible = "rockchip,rk1808-efuse"; ++ reg = <0x0 0xff3b0000 0x0 0x50>; ++ #address-cells = <1>; ++ #size-cells = <1>; ++ clocks = <&cru SCLK_EFUSE_NS>, <&cru PCLK_EFUSE>; ++ clock-names = "sclk_efuse", "pclk_efuse"; ++ assigned-clocks = <&cru SCLK_EFUSE_NS>; ++ assigned-clock-rates = <24000000>; ++ rockchip,efuse-size = <0x20>; ++ ++ /* Data cells */ ++ efuse_id: id@7 { ++ reg = <0x07 0x10>; ++ }; ++ cpu_leakage: cpu-leakage@17 { ++ reg = <0x17 0x1>; ++ }; ++ logic_leakage: logic-leakage@18 { ++ reg = <0x18 0x1>; ++ }; ++ npu_leakage: npu-leakage@19 { ++ reg = <0x19 0x1>; ++ }; ++ efuse_cpu_version: cpu-version@1c { ++ reg = <0x1c 0x1>; ++ bits = <3 3>; ++ }; ++ }; ++ ++ cru: clock-controller@ff350000 { ++ compatible = "rockchip,rk1808-cru"; ++ reg = <0x0 0xff350000 0x0 0x5000>; ++ rockchip,grf = <&grf>; ++ rockchip,pmugrf = <&pmugrf>; ++ #clock-cells = <1>; ++ #reset-cells = <1>; ++ ++ assigned-clocks = ++ <&cru SCLK_32K_IOE>, ++ <&cru PLL_GPLL>, <&cru PLL_CPLL>, ++ <&cru PLL_PPLL>, <&cru ARMCLK>, ++ <&cru MSCLK_PERI>, <&cru LSCLK_PERI>, ++ <&cru HSCLK_BUS_PRE>, <&cru MSCLK_BUS_PRE>, ++ <&cru LSCLK_BUS_PRE>; ++ assigned-clock-parents = <&xin32k>; ++ assigned-clock-rates = ++ <32768>, ++ <1188000000>, <1000000000>, ++ <100000000>, <816000000>, ++ <200000000>, <100000000>, ++ <300000000>, <200000000>, ++ <100000000>; ++ }; ++ ++ mipi_dphy_rx: mipi-dphy-rx@ff360000 { ++ compatible = "rockchip,rk1808-mipi-dphy-rx"; ++ reg = <0x0 0xff360000 0x0 0x4000>; ++ clocks = <&cru PCLK_MIPICSIPHY>; ++ clock-names = "pclk"; ++ power-domains = <&power RK1808_PD_VIO>; ++ rockchip,grf = <&grf>; ++ status = "disabled"; ++ }; ++ ++ mipi_dphy: mipi-dphy@ff370000 { ++ compatible = "rockchip,rk1808-mipi-dphy"; ++ reg = <0x0 0xff370000 0x0 0x500>; ++ clocks = <&cru SCLK_MIPIDSIPHY_REF>, <&cru PCLK_MIPIDSIPHY>; ++ clock-names = "ref", "pclk"; ++ clock-output-names = "mipi_dphy_pll"; ++ #clock-cells = <0>; ++ resets = <&cru SRST_MIPIDSIPHY_P>; ++ reset-names = "apb"; ++ #phy-cells = <0>; ++ rockchip,grf = <&grf>; ++ status = "disabled"; ++ }; ++ ++ combphy: phy@ff380000 { ++ compatible = "rockchip,rk1808-combphy"; ++ reg = <0x0 0xff380000 0x0 0x10000>; ++ #phy-cells = <1>; ++ clocks = <&cru SCLK_PCIEPHY_REF>; ++ clock-names = "refclk"; ++ assigned-clocks = <&cru SCLK_PCIEPHY_REF>; ++ assigned-clock-rates = <25000000>; ++ resets = <&cru SRST_USB3_OTG_A>, <&cru SRST_PCIEPHY_POR>, ++ <&cru SRST_PCIEPHY_P>, <&cru SRST_PCIEPHY_PIPE>, ++ <&cru SRST_USB3PHY_GRF_P>; ++ reset-names = "otg-rst", "combphy-por", ++ "combphy-apb", "combphy-pipe", ++ "usb3phy_grf_p"; ++ rockchip,combphygrf = <&combphy_grf>; ++ rockchip,usbpciegrf = <&usb_pcie_grf>; ++ status = "disabled"; ++ }; ++ ++ thermal_zones: thermal-zones { ++ soc_thermal: soc-thermal { ++ polling-delay-passive = <20>; /* milliseconds */ ++ polling-delay = <1000>; /* milliseconds */ ++ sustainable-power = <977>; /* milliwatts */ ++ ++ thermal-sensors = <&tsadc 0>; ++ ++ trips { ++ threshold: trip-point-0 { ++ /* millicelsius */ ++ temperature = <75000>; ++ /* millicelsius */ ++ hysteresis = <2000>; ++ type = "passive"; ++ }; ++ target: trip-point-1 { ++ /* millicelsius */ ++ temperature = <85000>; ++ /* millicelsius */ ++ hysteresis = <2000>; ++ type = "passive"; ++ }; ++ soc_crit: soc-crit { ++ /* millicelsius */ ++ temperature = <115000>; ++ /* millicelsius */ ++ hysteresis = <2000>; ++ type = "critical"; ++ }; ++ }; ++ ++ cooling-maps { ++ map0 { ++ trip = <&target>; ++ cooling-device = ++ <&cpu0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; ++ contribution = <4096>; ++ }; ++ map1 { ++ trip = <&target>; ++ cooling-device = ++ <&npu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; ++ contribution = <1024>; ++ }; ++ }; ++ }; ++ }; ++ ++ tsadc: tsadc@ff3a0000 { ++ compatible = "rockchip,rk1808-tsadc"; ++ reg = <0x0 0xff3a0000 0x0 0x100>; ++ interrupts = ; ++ rockchip,grf = <&grf>; ++ clocks = <&cru SCLK_TSADC>, <&cru PCLK_TSADC>; ++ clock-names = "tsadc", "apb_pclk"; ++ assigned-clocks = <&cru SCLK_TSADC>; ++ assigned-clock-rates = <650000>; ++ resets = <&cru SRST_TSADC>; ++ reset-names = "tsadc-apb"; ++ #thermal-sensor-cells = <1>; ++ rockchip,hw-tshut-temp = <120000>; ++ status = "disabled"; ++ }; ++ ++ saradc: saradc@ff3c0000 { ++ compatible = "rockchip,rk1808-saradc", "rockchip,rk3399-saradc"; ++ reg = <0x0 0xff3c0000 0x0 0x100>; ++ interrupts = ; ++ #io-channel-cells = <1>; ++ clocks = <&cru SCLK_SARADC>, <&cru PCLK_SARADC>; ++ clock-names = "saradc", "apb_pclk"; ++ resets = <&cru SRST_SARADC_P>; ++ reset-names = "saradc-apb"; ++ status = "disabled"; ++ }; ++ ++ pwm0: pwm@ff3d0000 { ++ compatible = "rockchip,rk1808-pwm", "rockchip,rk3328-pwm"; ++ reg = <0x0 0xff3d0000 0x0 0x10>; ++ #pwm-cells = <3>; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm0_pin>; ++ clocks = <&cru SCLK_PWM0>, <&cru PCLK_PWM0>; ++ clock-names = "pwm", "pclk"; ++ status = "disabled"; ++ }; ++ ++ pwm1: pwm@ff3d0010 { ++ compatible = "rockchip,rk1808-pwm", "rockchip,rk3328-pwm"; ++ reg = <0x0 0xff3d0010 0x0 0x10>; ++ #pwm-cells = <3>; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm1_pin>; ++ clocks = <&cru SCLK_PWM0>, <&cru PCLK_PWM0>; ++ clock-names = "pwm", "pclk"; ++ status = "disabled"; ++ }; ++ ++ pwm2: pwm@ff3d0020 { ++ compatible = "rockchip,rk1808-pwm", "rockchip,rk3328-pwm"; ++ reg = <0x0 0xff3d0020 0x0 0x10>; ++ #pwm-cells = <3>; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm2_pin>; ++ clocks = <&cru SCLK_PWM0>, <&cru PCLK_PWM0>; ++ clock-names = "pwm", "pclk"; ++ status = "disabled"; ++ }; ++ ++ pwm3: pwm@ff3d0030 { ++ compatible = "rockchip,rk1808-pwm", "rockchip,rk3328-pwm"; ++ reg = <0x0 0xff3d0030 0x0 0x10>; ++ #pwm-cells = <3>; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm3_pin>; ++ clocks = <&cru SCLK_PWM0>, <&cru PCLK_PWM0>; ++ clock-names = "pwm", "pclk"; ++ status = "disabled"; ++ }; ++ ++ pwm4: pwm@ff3d8000 { ++ compatible = "rockchip,rk1808-pwm", "rockchip,rk3328-pwm"; ++ reg = <0x0 0xff3d8000 0x0 0x10>; ++ #pwm-cells = <3>; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm4_pin>; ++ clocks = <&cru SCLK_PWM1>, <&cru PCLK_PWM1>; ++ clock-names = "pwm", "pclk"; ++ status = "disabled"; ++ }; ++ ++ pwm5: pwm@ff3d8010 { ++ compatible = "rockchip,rk1808-pwm", "rockchip,rk3328-pwm"; ++ reg = <0x0 0xff3d8010 0x0 0x10>; ++ #pwm-cells = <3>; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm5_pin>; ++ clocks = <&cru SCLK_PWM1>, <&cru PCLK_PWM1>; ++ clock-names = "pwm", "pclk"; ++ status = "disabled"; ++ }; ++ ++ pwm6: pwm@ff3d8020 { ++ compatible = "rockchip,rk1808-pwm", "rockchip,rk3328-pwm"; ++ reg = <0x0 0xff3d8020 0x0 0x10>; ++ #pwm-cells = <3>; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm6_pin>; ++ clocks = <&cru SCLK_PWM1>, <&cru PCLK_PWM1>; ++ clock-names = "pwm", "pclk"; ++ status = "disabled"; ++ }; ++ ++ pwm7: pwm@ff3d8030 { ++ compatible = "rockchip,rk1808-pwm", "rockchip,rk3328-pwm"; ++ reg = <0x0 0xff3d8030 0x0 0x10>; ++ #pwm-cells = <3>; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm7_pin>; ++ clocks = <&cru SCLK_PWM1>, <&cru PCLK_PWM1>; ++ clock-names = "pwm", "pclk"; ++ status = "disabled"; ++ }; ++ ++ pmu: power-management@ff3e0000 { ++ compatible = "rockchip,rk1808-pmu", "syscon", "simple-mfd"; ++ reg = <0x0 0xff3e0000 0x0 0x1000>; ++ ++ power: power-controller { ++ compatible = "rockchip,rk1808-power-controller"; ++ #power-domain-cells = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ status = "okay"; ++ ++ /* These power domains are grouped by VD_NPU */ ++ pd_npu@RK1808_VD_NPU { ++ reg = ; ++ clocks = <&cru SCLK_NPU>, ++ <&cru ACLK_NPU>, ++ <&cru HCLK_NPU>; ++ pm_qos = <&qos_npu>; ++ }; ++ ++ /* These power domains are grouped by VD_LOGIC */ ++ pd_pcie@RK1808_PD_PCIE { ++ reg = ; ++ clocks = <&cru HSCLK_PCIE>, ++ <&cru LSCLK_PCIE>, ++ <&cru ACLK_PCIE>, ++ <&cru ACLK_PCIE_MST>, ++ <&cru ACLK_PCIE_SLV>, ++ <&cru PCLK_PCIE>, ++ <&cru SCLK_PCIE_AUX>, ++ <&cru SCLK_PCIE_AUX>, ++ <&cru ACLK_USB3OTG>, ++ <&cru HCLK_HOST>, ++ <&cru HCLK_HOST_ARB>, ++ <&cru SCLK_USB3_OTG0_REF>, ++ <&cru SCLK_USB3_OTG0_SUSPEND>; ++ pm_qos = <&qos_pcie>, ++ <&qos_usb2>, ++ <&qos_usb3>; ++ }; ++ pd_vpu@RK1808_PD_VPU { ++ reg = ; ++ clocks = <&cru ACLK_VPU>, ++ <&cru HCLK_VPU>; ++ pm_qos = <&qos_vpu>; ++ }; ++ pd_vio@RK1808_PD_VIO { ++ reg = ; ++ clocks = <&cru HSCLK_VIO>, ++ <&cru LSCLK_VIO>, ++ <&cru ACLK_VOPRAW>, ++ <&cru HCLK_VOPRAW>, ++ <&cru ACLK_VOPLITE>, ++ <&cru HCLK_VOPLITE>, ++ <&cru PCLK_DSI_TX>, ++ <&cru PCLK_CSI_TX>, ++ <&cru ACLK_RGA>, ++ <&cru HCLK_RGA>, ++ <&cru ACLK_ISP>, ++ <&cru HCLK_ISP>, ++ <&cru ACLK_CIF>, ++ <&cru HCLK_CIF>, ++ <&cru PCLK_CSI2HOST>, ++ <&cru DCLK_VOPRAW>, ++ <&cru DCLK_VOPLITE>; ++ pm_qos = <&qos_rga_rd>, <&qos_rga_wr>, ++ <&qos_isp>, <&qos_cif>, ++ <&qos_vop_raw>, <&qos_vop_lite>; ++ }; ++ }; ++ }; ++ ++ i2c0: i2c@ff410000 { ++ compatible = "rockchip,rk3399-i2c"; ++ reg = <0x0 0xff410000 0x0 0x1000>; ++ clocks = <&cru SCLK_PMU_I2C0>, <&cru PCLK_I2C0_PMU>; ++ clock-names = "i2c", "pclk"; ++ interrupts = ; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2c0_xfer>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ status = "disabled"; ++ }; ++ ++ dmac: dmac@ff4e0000 { ++ compatible = "arm,pl330", "arm,primecell"; ++ reg = <0x0 0xff4e0000 0x0 0x4000>; ++ interrupts = ; ++ clocks = <&cru ACLK_DMAC>; ++ clock-names = "apb_pclk"; ++ #dma-cells = <1>; ++ arm,pl330-periph-burst; ++ }; ++ ++ uart0: serial@ff430000 { ++ compatible = "rockchip,rk1808-uart", "snps,dw-apb-uart"; ++ reg = <0x0 0xff430000 0x0 0x100>; ++ interrupts = ; ++ clocks = <&cru SCLK_UART0_PMU>, <&cru PCLK_UART0_PMU>; ++ clock-names = "baudclk", "apb_pclk"; ++ reg-shift = <2>; ++ reg-io-width = <4>; ++ dmas = <&dmac 0>, <&dmac 1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart0_xfer &uart0_cts &uart0_rts>; ++ status = "disabled"; ++ }; ++ ++ i2c1: i2c@ff500000 { ++ compatible = "rockchip,rk3399-i2c"; ++ reg = <0x0 0xff500000 0x0 0x1000>; ++ clocks = <&cru SCLK_I2C1>, <&cru PCLK_I2C1>; ++ clock-names = "i2c", "pclk"; ++ interrupts = ; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2c1_xfer>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ status = "disabled"; ++ }; ++ ++ i2c2: i2c@ff504000 { ++ compatible = "rockchip,rk3399-i2c"; ++ reg = <0x0 0xff504000 0x0 0x1000>; ++ clocks = <&cru SCLK_I2C2>, <&cru PCLK_I2C2>; ++ clock-names = "i2c", "pclk"; ++ interrupts = ; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2c2m0_xfer>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ status = "disabled"; ++ }; ++ ++ i2c3: i2c@ff508000 { ++ compatible = "rockchip,rk3399-i2c"; ++ reg = <0x0 0xff508000 0x0 0x1000>; ++ clocks = <&cru SCLK_I2C3>, <&cru PCLK_I2C3>; ++ clock-names = "i2c", "pclk"; ++ interrupts = ; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2c3_xfer>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ status = "disabled"; ++ }; ++ ++ i2c4: i2c@ff50c000 { ++ compatible = "rockchip,rk3399-i2c"; ++ reg = <0x0 0xff50c000 0x0 0x1000>; ++ clocks = <&cru SCLK_I2C4>, <&cru PCLK_I2C4>; ++ clock-names = "i2c", "pclk"; ++ interrupts = ; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2c4_xfer>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ status = "disabled"; ++ }; ++ ++ i2c5: i2c@ff510000 { ++ compatible = "rockchip,rk3399-i2c"; ++ reg = <0x0 0xff510000 0x0 0x1000>; ++ clocks = <&cru SCLK_I2C5>, <&cru PCLK_I2C5>; ++ clock-names = "i2c", "pclk"; ++ interrupts = ; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2c5_xfer>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ status = "disabled"; ++ }; ++ ++ spi0: spi@ff520000 { ++ compatible = "rockchip,rk1808-spi", "rockchip,rk3066-spi"; ++ reg = <0x0 0xff520000 0x0 0x1000>; ++ interrupts = ; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ clocks = <&cru SCLK_SPI0>, <&cru PCLK_SPI0>; ++ clock-names = "spiclk", "apb_pclk"; ++ dmas = <&dmac 10>, <&dmac 11>; ++ pinctrl-names = "default", "high_speed"; ++ pinctrl-0 = <&spi0_clk &spi0_csn &spi0_miso &spi0_mosi>; ++ pinctrl-1 = <&spi0_clk_hs &spi0_csn &spi0_miso_hs &spi0_mosi_hs>; ++ status = "disabled"; ++ }; ++ ++ spi1: spi@ff530000 { ++ compatible = "rockchip,rk1808-spi", "rockchip,rk3066-spi"; ++ reg = <0x0 0xff530000 0x0 0x1000>; ++ interrupts = ; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ clocks = <&cru SCLK_SPI1>, <&cru PCLK_SPI1>; ++ clock-names = "spiclk", "apb_pclk"; ++ dmas = <&dmac 12>, <&dmac 13>; ++ pinctrl-names = "default", "high_speed"; ++ pinctrl-0 = <&spi1_clk &spi1_csn0 &spi1_csn1 &spi1_miso &spi1_mosi>; ++ pinctrl-1 = <&spi1_clk_hs &spi1_csn0 &spi1_csn1 &spi1_miso_hs &spi1_mosi_hs>; ++ status = "disabled"; ++ }; ++ ++ uart1: serial@ff540000 { ++ compatible = "rockchip,rk1808-uart", "snps,dw-apb-uart"; ++ reg = <0x0 0xff540000 0x0 0x100>; ++ interrupts = ; ++ clocks = <&cru SCLK_UART1>, <&cru PCLK_UART1>; ++ clock-names = "baudclk", "apb_pclk"; ++ reg-shift = <2>; ++ reg-io-width = <4>; ++ dmas = <&dmac 2>, <&dmac 3>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart1m0_xfer &uart1_cts &uart1_rts>; ++ status = "disabled"; ++ }; ++ ++ uart2: serial@ff550000 { ++ compatible = "rockchip,rk1808-uart", "snps,dw-apb-uart"; ++ reg = <0x0 0xff550000 0x0 0x100>; ++ interrupts = ; ++ clocks = <&cru SCLK_UART2>, <&cru PCLK_UART2>; ++ clock-names = "baudclk", "apb_pclk"; ++ reg-shift = <2>; ++ reg-io-width = <4>; ++ dmas = <&dmac 4>, <&dmac 5>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart2m0_xfer>; ++ status = "disabled"; ++ }; ++ ++ uart3: serial@ff560000 { ++ compatible = "rockchip,rk1808-uart", "snps,dw-apb-uart"; ++ reg = <0x0 0xff560000 0x0 0x100>; ++ interrupts = ; ++ clocks = <&cru SCLK_UART3>, <&cru PCLK_UART3>; ++ clock-names = "baudclk", "apb_pclk"; ++ reg-shift = <2>; ++ reg-io-width = <4>; ++ dmas = <&dmac 6>, <&dmac 7>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart3m0_xfer &uart3_ctsm0 &uart3_rtsm0>; ++ status = "disabled"; ++ }; ++ ++ uart4: serial@ff570000 { ++ compatible = "rockchip,rk1808-uart", "snps,dw-apb-uart"; ++ reg = <0x0 0xff570000 0x0 0x100>; ++ interrupts = ; ++ clocks = <&cru SCLK_UART4>, <&cru PCLK_UART4>; ++ clock-names = "baudclk", "apb_pclk"; ++ reg-shift = <2>; ++ reg-io-width = <4>; ++ dmas = <&dmac 8>, <&dmac 9>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart4_xfer &uart4_cts &uart4_rts>; ++ status = "disabled"; ++ }; ++ ++ spi2: spi@ff580000 { ++ compatible = "rockchip,rk1808-spi", "rockchip,rk3066-spi"; ++ reg = <0x0 0xff580000 0x0 0x1000>; ++ interrupts = ; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ clocks = <&cru SCLK_SPI2>, <&cru PCLK_SPI2>; ++ clock-names = "spiclk", "apb_pclk"; ++ dmas = <&dmac 14>, <&dmac 15>; ++ pinctrl-names = "default", "high_speed"; ++ pinctrl-0 = <&spi2m0_clk &spi2m0_csn &spi2m0_miso &spi2m0_mosi>; ++ pinctrl-1 = <&spi2m0_clk_hs &spi2m0_csn &spi2m0_miso_hs &spi2m0_mosi_hs>; ++ status = "disabled"; ++ }; ++ ++ uart5: serial@ff5a0000 { ++ compatible = "rockchip,rk1808-uart", "snps,dw-apb-uart"; ++ reg = <0x0 0xff5a0000 0x0 0x100>; ++ interrupts = ; ++ clocks = <&cru SCLK_UART5>, <&cru PCLK_UART5>; ++ clock-names = "baudclk", "apb_pclk"; ++ reg-shift = <2>; ++ reg-io-width = <4>; ++ dmas = <&dmac 25>, <&dmac 26>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart5_xfer>; ++ status = "disabled"; ++ }; ++ ++ uart6: serial@ff5b0000 { ++ compatible = "rockchip,rk1808-uart", "snps,dw-apb-uart"; ++ reg = <0x0 0xff5b0000 0x0 0x100>; ++ interrupts = ; ++ clocks = <&cru SCLK_UART6>, <&cru PCLK_UART6>; ++ clock-names = "baudclk", "apb_pclk"; ++ reg-shift = <2>; ++ reg-io-width = <4>; ++ dmas = <&dmac 27>, <&dmac 28>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart6_xfer>; ++ status = "disabled"; ++ }; ++ ++ uart7: serial@ff5c0000 { ++ compatible = "rockchip,rk1808-uart", "snps,dw-apb-uart"; ++ reg = <0x0 0xff5c0000 0x0 0x100>; ++ interrupts = ; ++ clocks = <&cru SCLK_UART7>, <&cru PCLK_UART7>; ++ clock-names = "baudclk", "apb_pclk"; ++ reg-shift = <2>; ++ reg-io-width = <4>; ++ dmas = <&dmac 29>, <&dmac 30>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart7_xfer>; ++ status = "disabled"; ++ }; ++ ++ pwm8: pwm@ff5d0000 { ++ compatible = "rockchip,rk1808-pwm", "rockchip,rk3328-pwm"; ++ reg = <0x0 0xff5d0000 0x0 0x10>; ++ #pwm-cells = <3>; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm8_pin>; ++ clocks = <&cru SCLK_PWM2>, <&cru PCLK_PWM2>; ++ clock-names = "pwm", "pclk"; ++ status = "disabled"; ++ }; ++ ++ pwm9: pwm@fff5d0010 { ++ compatible = "rockchip,rk1808-pwm", "rockchip,rk3328-pwm"; ++ reg = <0x0 0xff5d0010 0x0 0x10>; ++ #pwm-cells = <3>; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm9_pin>; ++ clocks = <&cru SCLK_PWM2>, <&cru PCLK_PWM2>; ++ clock-names = "pwm", "pclk"; ++ status = "disabled"; ++ }; ++ ++ pwm10: pwm@ff5d0020 { ++ compatible = "rockchip,rk1808-pwm", "rockchip,rk3328-pwm"; ++ reg = <0x0 0xff5d0020 0x0 0x10>; ++ #pwm-cells = <3>; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm10_pin>; ++ clocks = <&cru SCLK_PWM2>, <&cru PCLK_PWM2>; ++ clock-names = "pwm", "pclk"; ++ status = "disabled"; ++ }; ++ ++ pwm11: pwm@ff5d0030 { ++ compatible = "rockchip,rk1808-pwm", "rockchip,rk3328-pwm"; ++ reg = <0x0 0xff5d0030 0x0 0x10>; ++ #pwm-cells = <3>; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm11_pin>; ++ clocks = <&cru SCLK_PWM2>, <&cru PCLK_PWM2>; ++ clock-names = "pwm", "pclk"; ++ status = "disabled"; ++ }; ++ ++ rng: rng@ff630000 { ++ compatible = "rockchip,cryptov2-rng"; ++ reg = <0x0 0xff630000 0x0 0x4000>; ++ clocks = <&cru SCLK_CRYPTO>, <&cru SCLK_CRYPTO_APK>, ++ <&cru ACLK_CRYPTO>, <&cru HCLK_CRYPTO>; ++ clock-names = "clk_crypto", "clk_crypto_apk", ++ "aclk_crypto", "hclk_crypto"; ++ resets = <&cru SRST_CRYPTO_CORE>; ++ reset-names = "reset"; ++ status = "disabled"; ++ }; ++ ++ dcf: dcf@ff640000 { ++ compatible = "syscon"; ++ reg = <0x0 0xff640000 0x0 0x1000>; ++ }; ++ ++ rktimer: rktimer@ff700000 { ++ compatible = "rockchip,rk3288-timer"; ++ reg = <0x0 0xff700000 0x0 0x1000>; ++ interrupts = ; ++ clocks = <&cru PCLK_TIMER>, <&cru SCLK_TIMER0>; ++ clock-names = "pclk", "timer"; ++ }; ++ ++ wdt: watchdog@ff720000 { ++ compatible = "snps,dw-wdt"; ++ reg = <0x0 0xff720000 0x0 0x100>; ++ clocks = <&cru PCLK_WDT>; ++ interrupts = ; ++ status = "okay"; ++ }; ++ ++ i2s0: i2s@ff7e0000 { ++ compatible = "rockchip,rk1808-i2s-tdm"; ++ reg = <0x0 0xff7e0000 0x0 0x1000>; ++ interrupts = ; ++ clocks = <&cru SCLK_I2S0_8CH_TX>, <&cru SCLK_I2S0_8CH_RX>, <&cru HCLK_I2S0_8CH>; ++ clock-names = "mclk_tx", "mclk_rx", "hclk"; ++ dmas = <&dmac 16>, <&dmac 17>; ++ dma-names = "tx", "rx"; ++ resets = <&cru SRST_I2S0_TX>, <&cru SRST_I2S0_RX>; ++ reset-names = "tx-m", "rx-m"; ++ rockchip,cru = <&cru>; ++ rockchip,grf = <&grf>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s0_8ch_sclktx ++ &i2s0_8ch_sclkrx ++ &i2s0_8ch_lrcktx ++ &i2s0_8ch_lrckrx ++ &i2s0_8ch_sdi0 ++ &i2s0_8ch_sdi1 ++ &i2s0_8ch_sdi2 ++ &i2s0_8ch_sdi3 ++ &i2s0_8ch_sdo0 ++ &i2s0_8ch_sdo1 ++ &i2s0_8ch_sdo2 ++ &i2s0_8ch_sdo3 ++ &i2s0_8ch_mclk>; ++ status = "disabled"; ++ }; ++ ++ i2s1: i2s@ff7f0000 { ++ compatible = "rockchip,rk1808-i2s", "rockchip,rk3066-i2s"; ++ reg = <0x0 0xff7f0000 0x0 0x1000>; ++ interrupts = ; ++ clocks = <&cru SCLK_I2S1_2CH>, <&cru HCLK_I2S1_2CH>; ++ clock-names = "i2s_clk", "i2s_hclk"; ++ dmas = <&dmac 18>, <&dmac 19>; ++ dma-names = "tx", "rx"; ++ resets = <&cru SRST_I2S1>, <&cru SRST_I2S1_H>; ++ reset-names = "reset-m", "reset-h"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1_2ch_sclk ++ &i2s1_2ch_lrck ++ &i2s1_2ch_sdi ++ &i2s1_2ch_sdo>; ++ status = "disabled"; ++ }; ++ ++ pdm: pdm@ff800000 { ++ compatible = "rockchip,rk1808-pdm", "rockchip,pdm"; ++ reg = <0x0 0xff800000 0x0 0x1000>; ++ clocks = <&cru SCLK_PDM>, <&cru HCLK_PDM>; ++ clock-names = "pdm_clk", "pdm_hclk"; ++ dmas = <&dmac 24>; ++ dma-names = "rx"; ++ resets = <&cru SRST_PDM>; ++ reset-names = "pdm-m"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pdm_clk ++ &pdm_clk1 ++ &pdm_sdi0 ++ &pdm_sdi1 ++ &pdm_sdi2 ++ &pdm_sdi3>; ++ status = "disabled"; ++ }; ++ ++ vad: vad@ff810000 { ++ compatible = "rockchip,rk1808-vad"; ++ reg = <0x0 0xff810000 0x0 0x10000>; ++ reg-names = "vad"; ++ clocks = <&cru HCLK_VAD>; ++ clock-names = "hclk"; ++ interrupts = ; ++ rockchip,audio-sram = <&vad_sram>; ++ rockchip,audio-src = <0>; ++ rockchip,det-channel = <0>; ++ rockchip,mode = <1>; ++ status = "disabled"; ++ }; ++ ++ dfi: dfi@ff9c0000 { ++ reg = <0x00 0xff9c0000 0x00 0x400>; ++ compatible = "rockchip,rk1808-dfi"; ++ rockchip,pmugrf = <&pmugrf>; ++ status = "disabled"; ++ }; ++ ++ dmc: dmc { ++ compatible = "rockchip,rk1808-dmc"; ++ dcf_reg = <&dcf>; ++ interrupts = ; ++ interrupt-names = "complete_irq"; ++ devfreq-events = <&dfi>; ++ clocks = <&cru SCLK_DDRCLK>; ++ clock-names = "dmc_clk"; ++ operating-points-v2 = <&dmc_opp_table>; ++ ddr_timing = <&ddr_timing>; ++ upthreshold = <40>; ++ downdifferential = <20>; ++ system-status-freq = < ++ /*system status freq(KHz)*/ ++ SYS_STATUS_NORMAL 924000 ++ SYS_STATUS_REBOOT 450000 ++ SYS_STATUS_SUSPEND 328000 ++ SYS_STATUS_VIDEO_1080P 924000 ++ SYS_STATUS_BOOST 924000 ++ SYS_STATUS_ISP 924000 ++ SYS_STATUS_PERFORMANCE 924000 ++ >; ++ auto-min-freq = <328000>; ++ auto-freq-en = <0>; ++ #cooling-cells = <2>; ++ status = "disabled"; ++ }; ++ ++ dmc_opp_table: dmc-opp-table { ++ compatible = "operating-points-v2"; ++ ++ rockchip,max-volt = <950000>; ++ rockchip,evb-irdrop = <12500>; ++ nvmem-cells = <&logic_leakage>; ++ nvmem-cell-names = "leakage"; ++ rockchip,temp-hysteresis = <5000>; ++ rockchip,low-temp = <0>; ++ rockchip,low-temp-min-volt = <800000>; ++ ++ opp-192000000 { ++ opp-hz = /bits/ 64 <192000000>; ++ opp-microvolt = <800000>; ++ }; ++ opp-324000000 { ++ opp-hz = /bits/ 64 <324000000>; ++ opp-microvolt = <800000>; ++ }; ++ opp-450000000 { ++ opp-hz = /bits/ 64 <450000000>; ++ opp-microvolt = <800000>; ++ }; ++ opp-528000000 { ++ opp-hz = /bits/ 64 <528000000>; ++ opp-microvolt = <800000>; ++ }; ++ opp-664000000 { ++ opp-hz = /bits/ 64 <664000000>; ++ opp-microvolt = <800000>; ++ }; ++ opp-784000000 { ++ opp-hz = /bits/ 64 <784000000>; ++ opp-microvolt = <800000>; ++ }; ++ opp-924000000 { ++ opp-hz = /bits/ 64 <924000000>; ++ opp-microvolt = <800000>; ++ }; ++ /* 1066M is only for ddr4 */ ++ opp-1066000000 { ++ opp-hz = /bits/ 64 <1066000000>; ++ opp-microvolt = <800000>; ++ status = "disabled"; ++ }; ++ }; ++ ++ rk_rga: rk_rga@ffaf0000 { ++ compatible = "rockchip,rga2"; ++ dev_mode = <0>; ++ reg = <0x0 0xffaf0000 0x0 0x1000>; ++ interrupts = ; ++ clocks = <&cru ACLK_RGA>, <&cru HCLK_RGA>, <&cru SCLK_RGA>; ++ clock-names = "aclk_rga", "hclk_rga", "clk_rga"; ++ power-domains = <&power RK1808_PD_VIO>; ++ status = "disabled"; ++ }; ++ ++ cif: cif@ffae0000 { ++ compatible = "rockchip,rk1808-cif"; ++ reg = <0x0 0xffae0000 0x0 0x200>; ++ reg-names = "cif_regs"; ++ interrupts = ; ++ interrupt-names = "cif-intr"; ++ clocks = <&cru ACLK_CIF>, <&cru DCLK_CIF>, ++ <&cru HCLK_CIF>, <&cru SCLK_CIF_OUT>; ++ clock-names = "aclk_cif", "dclk_cif", ++ "hclk_cif", "sclk_cif_out"; ++ resets = <&cru SRST_CIF_A>, <&cru SRST_CIF_H>, ++ <&cru SRST_CIF_I>, <&cru SRST_CIF_D>, ++ <&cru SRST_CIF_PCLKIN>; ++ reset-names = "rst_cif_a", "rst_cif_h", ++ "rst_cif_i", "rst_cif_d", ++ "rst_cif_pclkin"; ++ power-domains = <&power RK1808_PD_VIO>; ++ iommus = <&cif_mmu>; ++ status = "disabled"; ++ }; ++ ++ cif_mmu: iommu@ffae0800 { ++ compatible = "rockchip,iommu"; ++ reg = <0x0 0xffae0800 0x0 0x100>; ++ interrupts = ; ++ interrupt-names = "cif_mmu"; ++ clocks = <&cru ACLK_CIF>, <&cru HCLK_CIF>; ++ clock-names = "aclk", "iface"; ++ power-domains = <&power RK1808_PD_VIO>; ++ #iommu-cells = <0>; ++ status = "disabled"; ++ }; ++ ++ vop_lite: vop@ffb00000 { ++ compatible = "rockchip,rk1808-vop-lit"; ++ reg = <0x0 0xffb00000 0x0 0x200>; ++ reg-names = "regs"; ++ interrupts = ; ++ clocks = <&cru ACLK_VOPLITE>, <&cru DCLK_VOPLITE>, ++ <&cru HCLK_VOPLITE>; ++ clock-names = "aclk_vop", "dclk_vop", "hclk_vop"; ++ power-domains = <&power RK1808_PD_VIO>; ++ iommus = <&vopl_mmu>; ++ status = "disabled"; ++ ++ vop_lite_out: port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ vop_lite_out_dsi: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dsi_in_vop_lite>; ++ }; ++ ++ vop_lite_out_rgb: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&rgb_in_vop_lite>; ++ }; ++ }; ++ }; ++ ++ vopl_mmu: iommu@ffb00f00 { ++ compatible = "rockchip,iommu"; ++ reg = <0x0 0xffb00f00 0x0 0x100>; ++ interrupts = ; ++ interrupt-names = "vopl_mmu"; ++ clocks = <&cru ACLK_VOPLITE>, <&cru HCLK_VOPLITE>; ++ clock-names = "aclk", "iface"; ++ power-domains = <&power RK1808_PD_VIO>; ++ #iommu-cells = <0>; ++ status = "disabled"; ++ }; ++ ++ mipi_csi2: mipi-csi2@ffb10000 { ++ compatible = "rockchip,rk1808-mipi-csi2"; ++ reg = <0x0 0xffb10000 0x0 0x100>; ++ reg-names = "csihost_regs"; ++ interrupts = , ++ ; ++ interrupt-names = "csi-intr1", "csi-intr2"; ++ clocks = <&cru PCLK_CSI2HOST>; ++ clock-names = "pclk_csi2host"; ++ status = "disabled"; ++ }; ++ ++ csi_tx: csi@ffb20000 { ++ compatible = "rockchip,rk1808-mipi-csi"; ++ reg = <0x0 0xffb20000 0x0 0x500>; ++ reg-names = "csi_regs"; ++ interrupts = ; ++ clocks = <&cru PCLK_CSI_TX>, <&mipi_dphy>; ++ clock-names = "pclk", "hs_clk"; ++ resets = <&cru SRST_CSITX_P>, ++ <&cru SRST_CSITX_TXBYTEHS>, ++ <&cru SRST_CSITX_TXESC>, ++ <&cru SRST_CSITX_CAM>, ++ <&cru SRST_CSITX_I>; ++ reset-names = "tx_apb", "tx_bytehs", "tx_esc", "tx_cam", "tx_i"; ++ phys = <&mipi_dphy>; ++ phy-names = "mipi_dphy"; ++ power-domains = <&power RK1808_PD_VIO>; ++ rockchip,grf = <&grf>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ status = "disabled"; ++ ++ ports { ++ ++ port { ++ csi_in_vop_raw: endpoint { ++ remote-endpoint = <&vop_raw_out_csi>; ++ }; ++ }; ++ }; ++ }; ++ ++ dsi: dsi@ffb30000 { ++ compatible = "rockchip,rk1808-mipi-dsi"; ++ reg = <0x0 0xffb30000 0x0 0x500>; ++ interrupts = ; ++ clocks = <&cru PCLK_DSI_TX>, <&mipi_dphy>; ++ clock-names = "pclk", "hs_clk"; ++ resets = <&cru SRST_MIPIDSI_HOST_P>; ++ reset-names = "apb"; ++ phys = <&mipi_dphy>; ++ phy-names = "mipi_dphy"; ++ power-domains = <&power RK1808_PD_VIO>; ++ rockchip,grf = <&grf>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ status = "disabled"; ++ ++ ports { ++ port { ++ dsi_in_vop_lite: endpoint { ++ remote-endpoint = <&vop_lite_out_dsi>; ++ }; ++ }; ++ }; ++ }; ++ ++ vop_raw: vop@ffb40000 { ++ compatible = "rockchip,rk1808-vop-raw"; ++ reg = <0x0 0xffb40000 0x0 0x500>; ++ reg-names = "regs"; ++ interrupts = ; ++ clocks = <&cru ACLK_VOPRAW>, <&cru DCLK_VOPRAW>, ++ <&cru HCLK_VOPRAW>; ++ clock-names = "aclk_vop", "dclk_vop", "hclk_vop"; ++ power-domains = <&power RK1808_PD_VIO>; ++ iommus = <&vopr_mmu>; ++ status = "disabled"; ++ ++ vop_raw_out: port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ vop_raw_out_csi: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&csi_in_vop_raw>; ++ }; ++ }; ++ }; ++ ++ vopr_mmu: iommu@ffb40f00 { ++ compatible = "rockchip,iommu"; ++ reg = <0x0 0xffb40f00 0x0 0x100>; ++ interrupts = ; ++ interrupt-names = "vopr_mmu"; ++ clocks = <&cru ACLK_VOPRAW>, <&cru HCLK_VOPRAW>; ++ clock-names = "aclk", "iface"; ++ power-domains = <&power RK1808_PD_VIO>; ++ #iommu-cells = <0>; ++ status = "disabled"; ++ }; ++ ++ rkisp1: rkisp1@ffb50000 { ++ compatible = "rockchip,rk1808-rkisp1"; ++ reg = <0x0 0xffb50000 0x0 0x8000>; ++ interrupts = , ++ , ++ ; ++ interrupt-names = "isp_irq", "mi_irq", "mipi_irq"; ++ clocks = <&cru ACLK_ISP>, <&cru HCLK_ISP>, ++ <&cru SCLK_ISP>, <&cru DCLK_CIF>; ++ clock-names = "aclk_isp", "hclk_isp", ++ "clk_isp", "pclk_isp"; ++ power-domains = <&power RK1808_PD_VIO>; ++ iommus = <&isp_mmu>; ++ rockchip,grf = <&grf>; ++ status = "disabled"; ++ }; ++ ++ isp_mmu: iommu@ffb58000 { ++ compatible = "rockchip,iommu"; ++ reg = <0x0 0xffb58000 0x0 0x100>; ++ interrupts = ; ++ interrupt-names = "isp_mmu"; ++ clocks = <&cru ACLK_ISP>, <&cru HCLK_ISP>, ++ <&cru SCLK_ISP>; ++ clock-names = "aclk", "iface", "sclk"; ++ power-domains = <&power RK1808_PD_VIO>; ++ rk_iommu,disable_reset_quirk; ++ #iommu-cells = <0>; ++ status = "disabled"; ++ }; ++ ++ vpu_service: vpu_service@ffb80000 { ++ compatible = "rockchip,vpu_service"; ++ reg = <0x0 0xffb80000 0x0 0x800>; ++ interrupts = , ++ ; ++ interrupt-names = "irq_enc", "irq_dec"; ++ clocks = <&cru ACLK_VPU>, <&cru HCLK_VPU>; ++ clock-names = "aclk_vcodec", "hclk_vcodec"; ++ power-domains = <&power RK1808_PD_VPU>; ++ resets = <&cru SRST_VPU_A>, <&cru SRST_VPU_H>; ++ reset-names = "video_a", "video_h"; ++ iommus = <&vpu_mmu>; ++ iommu_enabled = <1>; ++ allocator = <1>; /* 0 means ion, 1 means drm */ ++ status = "disabled"; ++ }; ++ ++ vpu_mmu: iommu@ffb80800 { ++ compatible = "rockchip,iommu"; ++ reg = <0x0 0xffb80800 0x0 0x100>; ++ interrupts = ; ++ interrupt-names = "vpu_mmu"; ++ clocks = <&cru ACLK_VPU>, <&cru HCLK_VPU>; ++ clock-names = "aclk", "iface"; ++ power-domains = <&power RK1808_PD_VPU>; ++ #iommu-cells = <0>; ++ status = "disabled"; ++ }; ++ ++ sdio: dwmmc@ffc60000 { ++ compatible = "rockchip,rk1808-dw-mshc", "rockchip,rk3288-dw-mshc"; ++ reg = <0x0 0xffc60000 0x0 0x4000>; ++ clocks = <&cru HCLK_SDIO>, <&cru SCLK_SDIO>, ++ <&cru SCLK_SDIO_DRV>, <&cru SCLK_SDIO_SAMPLE>; ++ clock-names = "biu", "ciu", "ciu-drv", "ciu-sample"; ++ max-frequency = <150000000>; ++ fifo-depth = <0x100>; ++ interrupts = ; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc1_clk &sdmmc1_cmd &sdmmc1_bus4>; ++ status = "disabled"; ++ }; ++ ++ npu: npu@ffbc0000 { ++ compatible = "rockchip,npu"; ++ reg = <0x0 0xffbc0000 0x0 0x1000>; ++ clocks = <&cru SCLK_NPU>, <&cru ACLK_NPU>, <&cru HCLK_NPU>; ++ clock-names = "sclk_npu", "aclk_npu", "hclk_npu"; ++ assigned-clocks = <&cru SCLK_NPU>; ++ assigned-clock-rates = <800000000>; ++ interrupts = ; ++ power-domains = <&power RK1808_VD_NPU>; ++ operating-points-v2 = <&npu_opp_table>; ++ #cooling-cells = <2>; ++ status = "disabled"; ++ ++ npu_power_model: power-model { ++ compatible = "simple-power-model"; ++ ref-leakage = <31>; ++ static-coefficient = <100000>; ++ dynamic-coefficient = <3080>; ++ ts = <88610 303120 (-5000) 100>; ++ thermal-zone = "soc-thermal"; ++ }; ++ }; ++ ++ npu_opp_table: npu-opp-table { ++ compatible = "operating-points-v2"; ++ ++ rockchip,thermal-zone = "soc-thermal"; ++ rockchip,temp-hysteresis = <5000>; ++ rockchip,low-temp = <0>; ++ rockchip,low-temp-min-volt = <800000>; ++ rockchip,low-temp-adjust-volt = < ++ /* MHz MHz uV */ ++ 0 792 50000 ++ >; ++ ++ rockchip,max-volt = <880000>; ++ rockchip,evb-irdrop = <37500>; ++ nvmem-cells = <&npu_leakage>; ++ nvmem-cell-names = "leakage"; ++ ++ rockchip,pvtm-voltage-sel = < ++ 0 69000 0 ++ 69001 74000 1 ++ 74001 99999 2 ++ >; ++ rockchip,pvtm-ch = <0 0>; ++ ++ opp-200000000 { ++ opp-hz = /bits/ 64 <200000000>; ++ opp-microvolt = <750000 750000 880000>; ++ }; ++ opp-297000000 { ++ opp-hz = /bits/ 64 <297000000>; ++ opp-microvolt = <750000 750000 880000>; ++ }; ++ opp-400000000 { ++ opp-hz = /bits/ 64 <400000000>; ++ opp-microvolt = <750000 750000 880000>; ++ }; ++ opp-594000000 { ++ opp-hz = /bits/ 64 <594000000>; ++ opp-microvolt = <750000 750000 880000>; ++ }; ++ opp-792000000 { ++ opp-hz = /bits/ 64 <792000000>; ++ opp-microvolt = <850000 850000 880000>; ++ opp-microvolt-L0 = <850000 850000 880000>; ++ opp-microvolt-L1 = <825000 825000 880000>; ++ opp-microvolt-L2 = <800000 800000 880000>; ++ }; ++ }; ++ ++ sfc: sfc@ffc50000 { ++ compatible = "rockchip,sfc"; ++ reg = <0x0 0xffc50000 0x0 0x4000>; ++ interrupts = ; ++ clocks = <&cru SCLK_SFC>, <&cru HCLK_SFC>; ++ clock-names = "clk_sfc", "hclk_sfc"; ++ assigned-clocks = <&cru SCLK_SFC>; ++ assigned-clock-rates = <100000000>; ++ status = "disabled"; ++ }; ++ ++ sdmmc: dwmmc@ffcf0000 { ++ compatible = "rockchip,rk1808-dw-mshc", "rockchip,rk3288-dw-mshc"; ++ reg = <0x0 0xffcf0000 0x0 0x4000>; ++ clocks = <&cru HCLK_SDMMC>, <&cru SCLK_SDMMC>, ++ <&cru SCLK_SDMMC_DRV>, <&cru SCLK_SDMMC_SAMPLE>; ++ clock-names = "biu", "ciu", "ciu-drv", "ciu-sample"; ++ max-frequency = <150000000>; ++ fifo-depth = <0x100>; ++ interrupts = ; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc0_clk &sdmmc0_cmd &sdmmc0_bus4 &sdmmc0_detn>; ++ status = "disabled"; ++ }; ++ ++ emmc: dwmmc@ffd00000 { ++ compatible = "rockchip,rk1808-dw-mshc", "rockchip,rk3288-dw-mshc"; ++ reg = <0x0 0xffd00000 0x0 0x4000>; ++ clocks = <&cru HCLK_EMMC>, <&cru SCLK_EMMC>, ++ <&cru SCLK_EMMC_DRV>, <&cru SCLK_EMMC_SAMPLE>; ++ clock-names = "biu", "ciu", "ciu-drv", "ciu-sample"; ++ max-frequency = <150000000>; ++ fifo-depth = <0x100>; ++ interrupts = ; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&emmc_clk &emmc_cmd &emmc_bus8>; ++ status = "disabled"; ++ }; ++ ++ usb_host0_ehci: usb@ffd80000 { ++ compatible = "generic-ehci"; ++ reg = <0x0 0xffd80000 0x0 0x10000>; ++ interrupts = ; ++ clocks = <&cru HCLK_HOST>, <&cru HCLK_HOST_ARB>, ++ <&u2phy>; ++ clock-names = "usbhost", "arbiter", "utmi"; ++ phys = <&u2phy_host>; ++ phy-names = "usb"; ++ status = "disabled"; ++ power-domains = <&power RK1808_PD_PCIE>; ++ }; ++ ++ usb_host0_ohci: usb@ffd90000 { ++ compatible = "generic-ohci"; ++ reg = <0x0 0xffd90000 0x0 0x10000>; ++ interrupts = ; ++ clocks = <&cru HCLK_HOST>, <&cru HCLK_HOST_ARB>, ++ <&u2phy>; ++ clock-names = "usbhost", "arbiter", "utmi"; ++ phys = <&u2phy_host>; ++ phy-names = "usb"; ++ status = "disabled"; ++ power-domains = <&power RK1808_PD_PCIE>; ++ }; ++ ++ gmac: ethernet@ffdd0000 { ++ compatible = "rockchip,rk1808-gmac"; ++ reg = <0x0 0xffdd0000 0x0 0x10000>; ++ rockchip,grf = <&grf>; ++ interrupts = ; ++ interrupt-names = "macirq"; ++ clocks = <&cru SCLK_GMAC>, <&cru SCLK_GMAC_RX_TX>, ++ <&cru SCLK_GMAC_RX_TX>, <&cru SCLK_GMAC_REF>, ++ <&cru SCLK_GMAC_REFOUT>, <&cru ACLK_GMAC>, ++ <&cru PCLK_GMAC>, <&cru SCLK_GMAC_RGMII_SPEED>; ++ clock-names = "stmmaceth", "mac_clk_rx", ++ "mac_clk_tx", "clk_mac_ref", ++ "clk_mac_refout", "aclk_mac", ++ "pclk_mac", "clk_mac_speed"; ++ phy-mode = "rgmii"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&rgmii_pins>; ++ resets = <&cru SRST_GAMC_A>; ++ reset-names = "stmmaceth"; ++ /* power-domains = <&power RK1808_PD_GMAC>; */ ++ status = "disabled"; ++ }; ++ ++ rockchip_system_monitor: rockchip-system-monitor { ++ compatible = "rockchip,system-monitor"; ++ ++ rockchip,thermal-zone = "soc-thermal"; ++ rockchip,polling-delay = <200>; /* milliseconds */ ++ }; ++ ++ pinctrl: pinctrl { ++ compatible = "rockchip,rk1808-pinctrl"; ++ rockchip,grf = <&grf>; ++ rockchip,pmu = <&pmugrf>; ++ #address-cells = <2>; ++ #size-cells = <2>; ++ ranges; ++ ++ gpio0: gpio0@ff4c0000 { ++ compatible = "rockchip,gpio-bank"; ++ reg = <0x0 0xff4c0000 0x0 0x100>; ++ interrupts = ; ++ clocks = <&cru PCLK_GPIO0_PMU>, <&cru DBCLK_PMU_GPIO0>; ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ interrupt-controller; ++ #interrupt-cells = <2>; ++ }; ++ ++ gpio1: gpio1@ff690000 { ++ compatible = "rockchip,gpio-bank"; ++ reg = <0x0 0xff690000 0x0 0x100>; ++ interrupts = ; ++ clocks = <&cru PCLK_GPIO1>, <&cru DBCLK_GPIO1>; ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ interrupt-controller; ++ #interrupt-cells = <2>; ++ }; ++ ++ gpio2: gpio2@ff6a0000 { ++ compatible = "rockchip,gpio-bank"; ++ reg = <0x0 0xff6a0000 0x0 0x100>; ++ interrupts = ; ++ clocks = <&cru PCLK_GPIO2>, <&cru DBCLK_GPIO2>; ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ interrupt-controller; ++ #interrupt-cells = <2>; ++ }; ++ ++ gpio3: gpio3@ff6b0000 { ++ compatible = "rockchip,gpio-bank"; ++ reg = <0x0 0xff6b0000 0x0 0x100>; ++ interrupts = ; ++ clocks = <&cru PCLK_GPIO3>, <&cru DBCLK_GPIO3>; ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ interrupt-controller; ++ #interrupt-cells = <2>; ++ }; ++ ++ gpio4: gpio4@ff6c0000 { ++ compatible = "rockchip,gpio-bank"; ++ reg = <0x0 0xff6c0000 0x0 0x100>; ++ interrupts = ; ++ clocks = <&cru PCLK_GPIO4>, <&cru DBCLK_GPIO4>; ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ interrupt-controller; ++ #interrupt-cells = <2>; ++ }; ++ ++ pcfg_pull_up: pcfg-pull-up { ++ bias-pull-up; ++ }; ++ ++ pcfg_pull_down: pcfg-pull-down { ++ bias-pull-down; ++ }; ++ ++ pcfg_pull_none: pcfg-pull-none { ++ bias-disable; ++ }; ++ ++ pcfg_pull_none_2ma: pcfg-pull-none-2ma { ++ bias-disable; ++ drive-strength = <2>; ++ }; ++ ++ pcfg_pull_up_2ma: pcfg-pull-up-2ma { ++ bias-pull-up; ++ drive-strength = <2>; ++ }; ++ ++ pcfg_pull_up_4ma: pcfg-pull-up-4ma { ++ bias-pull-up; ++ drive-strength = <4>; ++ }; ++ ++ pcfg_pull_none_4ma: pcfg-pull-none-4ma { ++ bias-disable; ++ drive-strength = <4>; ++ }; ++ ++ pcfg_pull_down_4ma: pcfg-pull-down-4ma { ++ bias-pull-down; ++ drive-strength = <4>; ++ }; ++ ++ pcfg_pull_none_8ma: pcfg-pull-none-8ma { ++ bias-disable; ++ drive-strength = <8>; ++ }; ++ ++ pcfg_pull_up_8ma: pcfg-pull-up-8ma { ++ bias-pull-up; ++ drive-strength = <8>; ++ }; ++ ++ pcfg_pull_none_12ma: pcfg-pull-none-12ma { ++ bias-disable; ++ drive-strength = <12>; ++ }; ++ ++ pcfg_pull_up_12ma: pcfg-pull-up-12ma { ++ bias-pull-up; ++ drive-strength = <12>; ++ }; ++ ++ pcfg_pull_none_smt: pcfg-pull-none-smt { ++ bias-disable; ++ input-schmitt-enable; ++ }; ++ ++ pcfg_pull_none_2ma_smt: pcfg-pull-none-2ma-smt { ++ bias-disable; ++ drive-strength = <2>; ++ input-schmitt-enable; ++ }; ++ ++ pcfg_output_high: pcfg-output-high { ++ output-high; ++ }; ++ ++ pcfg_output_low: pcfg-output-low { ++ output-low; ++ }; ++ ++ pcfg_input_high: pcfg-input-high { ++ bias-pull-up; ++ input-enable; ++ }; ++ ++ pcfg_input: pcfg-input { ++ input-enable; ++ }; ++ ++ pcfg_input_smt: pcfg-input-smt { ++ input-enable; ++ input-schmitt-enable; ++ }; ++ ++ cif-m0 { ++ cif_clkout_m0: cif-clkout-m0 { ++ rockchip,pins = ++ <2 RK_PB7 1 &pcfg_pull_none>; ++ }; ++ ++ cif_d12d15_m0: cif-d12d15-m0 { ++ rockchip,pins = ++ <2 RK_PA0 1 &pcfg_pull_none>,/* cif_d12 */ ++ <2 RK_PA1 1 &pcfg_pull_none>,/* cif_d13 */ ++ <2 RK_PA2 1 &pcfg_pull_none>,/* cif_d14 */ ++ <2 RK_PA3 1 &pcfg_pull_none>;/* cif_d15 */ ++ }; ++ ++ cif_d10d11_m0: cif-d10d11-m0 { ++ rockchip,pins = ++ <2 RK_PC2 1 &pcfg_pull_none>,/* cif_d10 */ ++ <2 RK_PC3 1 &pcfg_pull_none>;/* cif_d11 */ ++ }; ++ ++ cif_d2d9_m0: cif-d2d9-m0 { ++ rockchip,pins = ++ <2 RK_PA4 1 &pcfg_pull_none>,/* cif_d2 */ ++ <2 RK_PA5 1 &pcfg_pull_none>,/* cif_d3 */ ++ <2 RK_PA6 1 &pcfg_pull_none>,/* cif_d4 */ ++ <2 RK_PA7 1 &pcfg_pull_none>,/* cif_d5 */ ++ <2 RK_PB0 1 &pcfg_pull_none>,/* cif_d6 */ ++ <2 RK_PB1 1 &pcfg_pull_none>,/* cif_d7 */ ++ <2 RK_PB2 1 &pcfg_pull_none>,/* cif_d8 */ ++ <2 RK_PB3 1 &pcfg_pull_none>,/* cif_d9 */ ++ <2 RK_PB4 1 &pcfg_pull_none>,/* cif_vsync */ ++ <2 RK_PB5 1 &pcfg_pull_none>,/* cif_href */ ++ <2 RK_PB6 1 &pcfg_pull_none>;/* cif_clkin */ ++ }; ++ ++ cif_d0d1_m0: cif-d0d1-m0 { ++ rockchip,pins = ++ <2 RK_PC0 1 &pcfg_pull_none>,/* cif_d0 */ ++ <2 RK_PC1 1 &pcfg_pull_none>;/* cif_d1 */ ++ }; ++ }; ++ ++ emmc { ++ emmc_clk: emmc-clk { ++ rockchip,pins = ++ /* emmc_clkout */ ++ <1 RK_PB1 1 &pcfg_pull_up_4ma>; ++ }; ++ ++ emmc_rstnout: emmc-rstnout { ++ rockchip,pins = ++ /* emmc_rstn */ ++ <1 RK_PB3 1 &pcfg_pull_none>; ++ }; ++ ++ emmc_bus8: emmc-bus8 { ++ rockchip,pins = ++ /* emmc_d0 */ ++ <1 RK_PA0 1 &pcfg_pull_up_4ma>, ++ /* emmc_d1 */ ++ <1 RK_PA1 1 &pcfg_pull_up_4ma>, ++ /* emmc_d2 */ ++ <1 RK_PA2 1 &pcfg_pull_up_4ma>, ++ /* emmc_d3 */ ++ <1 RK_PA3 1 &pcfg_pull_up_4ma>, ++ /* emmc_d4 */ ++ <1 RK_PA4 1 &pcfg_pull_up_4ma>, ++ /* emmc_d5 */ ++ <1 RK_PA5 1 &pcfg_pull_up_4ma>, ++ /* emmc_d6 */ ++ <1 RK_PA6 1 &pcfg_pull_up_4ma>, ++ /* emmc_d7 */ ++ <1 RK_PA7 1 &pcfg_pull_up_4ma>; ++ }; ++ ++ emmc_pwren: emmc-pwren { ++ rockchip,pins = ++ <1 RK_PB0 1 &pcfg_pull_none>; ++ }; ++ ++ emmc_cmd: emmc-cmd { ++ rockchip,pins = ++ <1 RK_PB2 1 &pcfg_pull_up_4ma>; ++ }; ++ }; ++ ++ gmac { ++ rgmii_pins: rgmii-pins { ++ rockchip,pins = ++ /* rgmii_txen */ ++ <2 RK_PA1 2 &pcfg_pull_none_4ma>, ++ /* rgmii_txd1 */ ++ <2 RK_PA2 2 &pcfg_pull_none_4ma>, ++ /* rgmii_txd0 */ ++ <2 RK_PA3 2 &pcfg_pull_none_4ma>, ++ /* rgmii_rxd0 */ ++ <2 RK_PA4 2 &pcfg_pull_none>, ++ /* rgmii_rxd1 */ ++ <2 RK_PA5 2 &pcfg_pull_none>, ++ /* rgmii_rxdv */ ++ <2 RK_PA7 2 &pcfg_pull_none>, ++ /* rgmii_mdio */ ++ <2 RK_PB0 2 &pcfg_pull_none_2ma>, ++ /* rgmii_mdc */ ++ <2 RK_PB2 2 &pcfg_pull_none_2ma>, ++ /* rgmii_txd3 */ ++ <2 RK_PB3 2 &pcfg_pull_none_4ma>, ++ /* rgmii_txd2 */ ++ <2 RK_PB4 2 &pcfg_pull_none_4ma>, ++ /* rgmii_rxd2 */ ++ <2 RK_PB5 2 &pcfg_pull_none>, ++ /* rgmii_rxd3 */ ++ <2 RK_PB6 2 &pcfg_pull_none>, ++ /* rgmii_clk */ ++ <2 RK_PB7 2 &pcfg_pull_none>, ++ /* rgmii_txclk */ ++ <2 RK_PC1 2 &pcfg_pull_none_4ma>, ++ /* rgmii_rxclk */ ++ <2 RK_PC2 2 &pcfg_pull_none>; ++ }; ++ ++ rmii_pins: rmii-pins { ++ rockchip,pins = ++ /* rmii_txen */ ++ <2 RK_PA1 2 &pcfg_pull_none_4ma>, ++ /* rmii_txd1 */ ++ <2 RK_PA2 2 &pcfg_pull_none_4ma>, ++ /* rmii_txd0 */ ++ <2 RK_PA3 2 &pcfg_pull_none_4ma>, ++ /* rmii_rxd0 */ ++ <2 RK_PA4 2 &pcfg_pull_none>, ++ /* rmii_rxd1 */ ++ <2 RK_PA5 2 &pcfg_pull_none>, ++ /* rmii_rxer */ ++ <2 RK_PA6 2 &pcfg_pull_none>, ++ /* rmii_rxdv */ ++ <2 RK_PA7 2 &pcfg_pull_none>, ++ /* rmii_mdio */ ++ <2 RK_PB0 2 &pcfg_pull_none_2ma>, ++ /* rmii_mdc */ ++ <2 RK_PB2 2 &pcfg_pull_none_2ma>, ++ /* rmii_clk */ ++ <2 RK_PB7 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ i2c0 { ++ i2c0_xfer: i2c0-xfer { ++ rockchip,pins = ++ /* i2c0_sda */ ++ <0 RK_PB1 1 &pcfg_pull_none_2ma_smt>, ++ /* i2c0_scl */ ++ <0 RK_PB0 1 &pcfg_pull_none_2ma_smt>; ++ }; ++ }; ++ ++ i2c1 { ++ i2c1_xfer: i2c1-xfer { ++ rockchip,pins = ++ /* i2c1_sda */ ++ <0 RK_PC1 1 &pcfg_pull_none_2ma_smt>, ++ /* i2c1_scl */ ++ <0 RK_PC0 1 &pcfg_pull_none_2ma_smt>; ++ }; ++ }; ++ ++ i2c2m0 { ++ i2c2m0_xfer: i2c2m0-xfer { ++ rockchip,pins = ++ /* i2c2m0_sda */ ++ <3 RK_PB4 2 &pcfg_pull_none_2ma_smt>, ++ /* i2c2m0_scl */ ++ <3 RK_PB3 2 &pcfg_pull_none_2ma_smt>; ++ }; ++ }; ++ ++ i2c2m1 { ++ i2c2m1_xfer: i2c2m1-xfer { ++ rockchip,pins = ++ /* i2c2m1_sda */ ++ <1 RK_PB5 2 &pcfg_pull_none_2ma_smt>, ++ /* i2c2m1_scl */ ++ <1 RK_PB4 2 &pcfg_pull_none_2ma_smt>; ++ }; ++ }; ++ ++ i2c3 { ++ i2c3_xfer: i2c3-xfer { ++ rockchip,pins = ++ /* i2c3_sda */ ++ <2 RK_PD1 1 &pcfg_pull_none_2ma_smt>, ++ /* i2c3_scl */ ++ <2 RK_PD0 1 &pcfg_pull_none_2ma_smt>; ++ }; ++ }; ++ ++ i2c4 { ++ i2c4_xfer: i2c4-xfer { ++ rockchip,pins = ++ /* i2c4_sda */ ++ <3 RK_PC3 3 &pcfg_pull_none_2ma_smt>, ++ /* i2c4_scl */ ++ <3 RK_PC2 3 &pcfg_pull_none_2ma_smt>; ++ }; ++ }; ++ ++ i2c5 { ++ i2c5_xfer: i2c5-xfer { ++ rockchip,pins = ++ /* i2c5_sda */ ++ <4 RK_PC2 1 &pcfg_pull_none_2ma_smt>, ++ /* i2c5_scl */ ++ <4 RK_PC1 1 &pcfg_pull_none_2ma_smt>; ++ }; ++ }; ++ ++ i2s1 { ++ i2s1_2ch_lrck: i2s1-2ch-lrck { ++ rockchip,pins = ++ <3 RK_PA0 1 &pcfg_pull_none_2ma>; ++ }; ++ i2s1_2ch_sclk: i2s1-2ch-sclk { ++ rockchip,pins = ++ <3 RK_PA1 1 &pcfg_pull_none_2ma>; ++ }; ++ i2s1_2ch_mclk: i2s1-2ch-mclk { ++ rockchip,pins = ++ <3 RK_PA2 1 &pcfg_pull_none_2ma>; ++ }; ++ i2s1_2ch_sdo: i2s1-2ch-sdo { ++ rockchip,pins = ++ <3 RK_PA3 1 &pcfg_pull_none_2ma>; ++ }; ++ i2s1_2ch_sdi: i2s1-2ch-sdi { ++ rockchip,pins = ++ <3 RK_PA4 1 &pcfg_pull_none_2ma>; ++ }; ++ }; ++ ++ i2s0 { ++ i2s0_8ch_sdi3: i2s0-8ch-sdi3 { ++ rockchip,pins = ++ <3 RK_PA5 1 &pcfg_pull_none_2ma>; ++ }; ++ i2s0_8ch_sdi2: i2s0-8ch-sdi2 { ++ rockchip,pins = ++ <3 RK_PA6 1 &pcfg_pull_none_2ma>; ++ }; ++ i2s0_8ch_sdi1: i2s0-8ch-sdi1 { ++ rockchip,pins = ++ <3 RK_PA7 1 &pcfg_pull_none_2ma>; ++ }; ++ i2s0_8ch_sclkrx: i2s0-8ch-sclkrx { ++ rockchip,pins = ++ <3 RK_PB0 1 &pcfg_pull_none_2ma>; ++ }; ++ i2s0_8ch_lrckrx: i2s0-8ch-lrckrx { ++ rockchip,pins = ++ <3 RK_PB1 1 &pcfg_pull_none_2ma>; ++ }; ++ i2s0_8ch_sdo3: i2s0-8ch-sdo3 { ++ rockchip,pins = ++ <3 RK_PB2 1 &pcfg_pull_none_2ma>; ++ }; ++ i2s0_8ch_sdo2: i2s0-8ch-sdo2 { ++ rockchip,pins = ++ <3 RK_PB3 1 &pcfg_pull_none_2ma>; ++ }; ++ i2s0_8ch_sdo1: i2s0-8ch-sdo1 { ++ rockchip,pins = ++ <3 RK_PB4 1 &pcfg_pull_none_2ma>; ++ }; ++ i2s0_8ch_mclk: i2s0-8ch-mclk { ++ rockchip,pins = ++ <3 RK_PB5 1 &pcfg_pull_none_2ma>; ++ }; ++ i2s0_8ch_lrcktx: i2s0-8ch-lrcktx { ++ rockchip,pins = ++ <3 RK_PB6 1 &pcfg_pull_none_2ma>; ++ }; ++ i2s0_8ch_sclktx: i2s0-8ch-sclktx { ++ rockchip,pins = ++ <3 RK_PB7 1 &pcfg_pull_none_2ma>; ++ }; ++ i2s0_8ch_sdo0: i2s0-8ch-sdo0 { ++ rockchip,pins = ++ <3 RK_PC0 1 &pcfg_pull_none_2ma>; ++ }; ++ i2s0_8ch_sdi0: i2s0-8ch-sdi0 { ++ rockchip,pins = ++ <3 RK_PC1 1 &pcfg_pull_none_2ma>; ++ }; ++ }; ++ ++ lcdc { ++ lcdc_rgb_dclk_pin: lcdc-rgb-dclk-pin { ++ rockchip,pins = ++ /* lcdc_clkm0 */ ++ <2 RK_PC6 3 &pcfg_pull_none>; ++ }; ++ ++ lcdc_rgb_den_pin: lcdc-rgb-den-pin { ++ rockchip,pins = ++ /* lcdc_denm0 */ ++ <2 RK_PC7 3 &pcfg_pull_none>; ++ }; ++ ++ lcdc_rgb_m0_hsync_pin: lcdc-rgb-m0-hsync-pin { ++ rockchip,pins = ++ /* lcdc_hsyncm0 */ ++ <2 RK_PB2 3 &pcfg_pull_none>; ++ }; ++ ++ lcdc_rgb_m0_vsync_pin: lcdc-rgb-m0-vsync-pin { ++ rockchip,pins = ++ /* lcdc_vsyncm0 */ ++ <2 RK_PB3 3 &pcfg_pull_none>; ++ }; ++ ++ lcdc_rgb_m1_hsync_pin: lcdc-rgb-m1-hsync-pin { ++ rockchip,pins = ++ /* lcdc_hsyncm1 */ ++ <3 RK_PB2 3 &pcfg_pull_none>; ++ }; ++ ++ lcdc_rgb_m1_vsync_pin: lcdc-rgb-m1-vsync-pin { ++ rockchip,pins = ++ /* lcdc_vsyncm1 */ ++ <3 RK_PB3 3 &pcfg_pull_none>; ++ }; ++ ++ lcdc_rgb666_data_pins: lcdc-rgb666-data-pins { ++ rockchip,pins = ++ /* lcdc_d0m0 */ ++ <2 RK_PA2 3 &pcfg_pull_none>, ++ /* lcdc_d1m0 */ ++ <2 RK_PA3 3 &pcfg_pull_none>, ++ /* lcdc_d2m0 */ ++ <2 RK_PC2 3 &pcfg_pull_none>, ++ /* lcdc_d3m0 */ ++ <2 RK_PC3 3 &pcfg_pull_none>, ++ /* lcdc_d4m0 */ ++ <2 RK_PC4 3 &pcfg_pull_none>, ++ /* lcdc_d5m0 */ ++ <2 RK_PC5 3 &pcfg_pull_none>, ++ /* lcdc_d6m0 */ ++ <2 RK_PA0 3 &pcfg_pull_none>, ++ /* lcdc_d7m0 */ ++ <2 RK_PA1 3 &pcfg_pull_none>, ++ /* lcdc_d8 */ ++ <3 RK_PC2 1 &pcfg_pull_none>, ++ /* lcdc_d9 */ ++ <3 RK_PC3 1 &pcfg_pull_none>, ++ /* lcdc_d10 */ ++ <3 RK_PC4 1 &pcfg_pull_none>, ++ /* lcdc_d11 */ ++ <3 RK_PC5 1 &pcfg_pull_none>, ++ /* lcdc_d12 */ ++ <3 RK_PC6 1 &pcfg_pull_none>, ++ /* lcdc_d13 */ ++ <3 RK_PC7 1 &pcfg_pull_none>, ++ /* lcdc_d14 */ ++ <3 RK_PD0 1 &pcfg_pull_none>, ++ /* lcdc_d15 */ ++ <3 RK_PD1 1 &pcfg_pull_none>, ++ /* lcdc_d16 */ ++ <3 RK_PD2 1 &pcfg_pull_none>, ++ /* lcdc_d17 */ ++ <3 RK_PD3 1 &pcfg_pull_none>; ++ }; ++ ++ lcdc_rgb565_data_pins: lcdc-rgb565-data-pins { ++ rockchip,pins = ++ /* lcdc_d0m0 */ ++ <2 RK_PA2 3 &pcfg_pull_none>, ++ /* lcdc_d1m0 */ ++ <2 RK_PA3 3 &pcfg_pull_none>, ++ /* lcdc_d2m0 */ ++ <2 RK_PC2 3 &pcfg_pull_none>, ++ /* lcdc_d3m0 */ ++ <2 RK_PC3 3 &pcfg_pull_none>, ++ /* lcdc_d4m0 */ ++ <2 RK_PC4 3 &pcfg_pull_none>, ++ /* lcdc_d5m0 */ ++ <2 RK_PC5 3 &pcfg_pull_none>, ++ /* lcdc_d6m0 */ ++ <2 RK_PA0 3 &pcfg_pull_none>, ++ /* lcdc_d7m0 */ ++ <2 RK_PA1 3 &pcfg_pull_none>, ++ /* lcdc_d8 */ ++ <3 RK_PC2 1 &pcfg_pull_none>, ++ /* lcdc_d9 */ ++ <3 RK_PC3 1 &pcfg_pull_none>, ++ /* lcdc_d10 */ ++ <3 RK_PC4 1 &pcfg_pull_none>, ++ /* lcdc_d11 */ ++ <3 RK_PC5 1 &pcfg_pull_none>, ++ /* lcdc_d12 */ ++ <3 RK_PC6 1 &pcfg_pull_none>, ++ /* lcdc_d13 */ ++ <3 RK_PC7 1 &pcfg_pull_none>, ++ /* lcdc_d14 */ ++ <3 RK_PD0 1 &pcfg_pull_none>, ++ /* lcdc_d15 */ ++ <3 RK_PD1 1 &pcfg_pull_none>; ++ }; ++ }; ++ ++ pciusb { ++ pciusb_pins: pciusb-pins { ++ rockchip,pins = ++ /* pciusb_debug0 */ ++ <4 RK_PB4 3 &pcfg_pull_none>, ++ /* pciusb_debug1 */ ++ <4 RK_PB5 3 &pcfg_pull_none>, ++ /* pciusb_debug2 */ ++ <4 RK_PB6 3 &pcfg_pull_none>, ++ /* pciusb_debug3 */ ++ <4 RK_PB7 3 &pcfg_pull_none>, ++ /* pciusb_debug4 */ ++ <4 RK_PC0 3 &pcfg_pull_none>, ++ /* pciusb_debug5 */ ++ <4 RK_PC1 3 &pcfg_pull_none>, ++ /* pciusb_debug6 */ ++ <4 RK_PC2 3 &pcfg_pull_none>, ++ /* pciusb_debug7 */ ++ <4 RK_PC3 3 &pcfg_pull_none>; ++ }; ++ ++ pcie_clkreq: pcie-clkreq { ++ rockchip,pins = ++ /* pcie_clkreqn_m1 */ ++ <0 RK_PC6 1 &pcfg_pull_none >; ++ }; ++ }; ++ ++ pdm { ++ pdm_clk: pdm-clk { ++ rockchip,pins = ++ /* pdm_clk0 */ ++ <3 RK_PB0 2 &pcfg_pull_none_2ma>; ++ }; ++ ++ pdm_sdi3: pdm-sdi3 { ++ rockchip,pins = ++ <3 RK_PA5 2 &pcfg_pull_none_2ma>; ++ }; ++ ++ pdm_sdi2: pdm-sdi2 { ++ rockchip,pins = ++ <3 RK_PA6 2 &pcfg_pull_none_2ma>; ++ }; ++ ++ pdm_sdi1: pdm-sdi1 { ++ rockchip,pins = ++ <3 RK_PA7 2 &pcfg_pull_none_2ma>; ++ }; ++ ++ pdm_clk1: pdm-clk1 { ++ rockchip,pins = ++ <3 RK_PB1 2 &pcfg_pull_none_2ma>; ++ }; ++ ++ pdm_sdi0: pdm-sdi0 { ++ rockchip,pins = ++ <3 RK_PC1 2 &pcfg_pull_none_2ma>; ++ }; ++ }; ++ ++ pwm0 { ++ pwm0_pin: pwm0-pin { ++ rockchip,pins = ++ <0 RK_PB7 1 &pcfg_pull_none_2ma>; ++ }; ++ }; ++ ++ pwm1 { ++ pwm1_pin: pwm1-pin { ++ rockchip,pins = ++ <0 RK_PC3 1 &pcfg_pull_none_2ma>; ++ }; ++ }; ++ ++ pwm2 { ++ pwm2_pin: pwm2-pin { ++ rockchip,pins = ++ <0 RK_PC5 1 &pcfg_pull_none_2ma>; ++ }; ++ }; ++ ++ pwm3 { ++ pwm3_pin: pwm3-pin { ++ rockchip,pins = ++ <0 RK_PC4 1 &pcfg_pull_none_2ma>; ++ }; ++ }; ++ ++ pwm4 { ++ pwm4_pin: pwm4-pin { ++ rockchip,pins = ++ <1 RK_PB6 2 &pcfg_pull_none_2ma>; ++ }; ++ }; ++ ++ pwm5 { ++ pwm5_pin: pwm5-pin { ++ rockchip,pins = ++ <1 RK_PB7 2 &pcfg_pull_none_2ma>; ++ }; ++ }; ++ pwm6 { ++ pwm6_pin: pwm6-pin { ++ rockchip,pins = ++ <3 RK_PA1 2 &pcfg_pull_none_2ma>; ++ }; ++ }; ++ ++ pwm7 { ++ pwm7_pin: pwm7-pin { ++ rockchip,pins = ++ <3 RK_PA2 2 &pcfg_pull_none_2ma>; ++ }; ++ }; ++ ++ pwm8 { ++ pwm8_pin: pwm8-pin { ++ rockchip,pins = ++ <3 RK_PD0 2 &pcfg_pull_none_2ma>; ++ }; ++ }; ++ ++ pwm9 { ++ pwm9_pin: pwm9-pin { ++ rockchip,pins = ++ <3 RK_PD1 2 &pcfg_pull_none_2ma>; ++ }; ++ }; ++ ++ pwm10 { ++ pwm10_pin: pwm10-pin { ++ rockchip,pins = ++ <3 RK_PD2 2 &pcfg_pull_none_2ma>; ++ }; ++ }; ++ ++ pwm11 { ++ pwm11_pin: pwm11-pin { ++ rockchip,pins = ++ <3 RK_PD3 2 &pcfg_pull_none_2ma>; ++ }; ++ }; ++ ++ sdmmc0 { ++ sdmmc0_bus4: sdmmc0-bus4 { ++ rockchip,pins = ++ /* sdmmc0_d0 */ ++ <4 RK_PA2 1 &pcfg_pull_up_8ma>, ++ /* sdmmc0_d1 */ ++ <4 RK_PA3 1 &pcfg_pull_up_8ma>, ++ /* sdmmc0_d2 */ ++ <4 RK_PA4 1 &pcfg_pull_up_8ma>, ++ /* sdmmc0_d3 */ ++ <4 RK_PA5 1 &pcfg_pull_up_8ma>; ++ }; ++ ++ sdmmc0_cmd: sdmmc0-cmd { ++ rockchip,pins = ++ <4 RK_PA0 1 &pcfg_pull_up_8ma>; ++ }; ++ ++ sdmmc0_clk: sdmmc0-clk { ++ rockchip,pins = ++ <4 RK_PA1 1 &pcfg_pull_up_8ma>; ++ }; ++ ++ sdmmc0_detn: sdmmc0-detn { ++ rockchip,pins = ++ <0 RK_PA3 1 &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdmmc1 { ++ sdmmc1_bus4: sdmmc1-bus4 { ++ rockchip,pins = ++ /* sdmmc1_d0 */ ++ <4 RK_PB0 1 &pcfg_pull_up_4ma>, ++ /* sdmmc1_d1 */ ++ <4 RK_PB1 1 &pcfg_pull_up_4ma>, ++ /* sdmmc1_d2 */ ++ <4 RK_PB2 1 &pcfg_pull_up_4ma>, ++ /* sdmmc1_d3 */ ++ <4 RK_PB3 1 &pcfg_pull_up_4ma>; ++ }; ++ ++ sdmmc1_cmd: sdmmc1-cmd { ++ rockchip,pins = ++ <4 RK_PA6 1 &pcfg_pull_up_4ma>; ++ }; ++ ++ sdmmc1_clk: sdmmc1-clk { ++ rockchip,pins = ++ <4 RK_PA7 1 &pcfg_pull_up_4ma>; ++ }; ++ }; ++ ++ spi0 { ++ spi0_mosi: spi0-mosi { ++ rockchip,pins = ++ <1 RK_PB4 1 &pcfg_pull_up_2ma>; ++ }; ++ ++ spi0_miso: spi0-miso { ++ rockchip,pins = ++ <1 RK_PB5 1 &pcfg_pull_up_2ma>; ++ }; ++ ++ spi0_csn: spi0-csn { ++ rockchip,pins = ++ <1 RK_PB6 1 &pcfg_pull_up_2ma>; ++ }; ++ ++ spi0_clk: spi0-clk { ++ rockchip,pins = ++ <1 RK_PB7 1 &pcfg_pull_up_2ma>; ++ }; ++ ++ spi0_mosi_hs: spi0-mosi-hs { ++ rockchip,pins = ++ <1 RK_PB4 1 &pcfg_pull_up_2ma>; ++ }; ++ ++ spi0_miso_hs: spi0-miso-hs { ++ rockchip,pins = ++ <1 RK_PB5 1 &pcfg_pull_up_2ma>; ++ }; ++ ++ spi0_csn_hs: spi0-csn-hs { ++ rockchip,pins = ++ <1 RK_PB6 1 &pcfg_pull_up_2ma>; ++ }; ++ ++ spi0_clk_hs: spi0-clk-hs { ++ rockchip,pins = ++ <1 RK_PB7 1 &pcfg_pull_up_2ma>; ++ }; ++ }; ++ ++ spi1m0 { ++ spi1_clk: spi1-clk { ++ rockchip,pins = ++ <4 RK_PB4 2 &pcfg_pull_up_2ma>; ++ }; ++ ++ spi1_mosi: spi1-mosi { ++ rockchip,pins = ++ <4 RK_PB5 2 &pcfg_pull_up_2ma>; ++ }; ++ ++ spi1_csn0: spi1-csn0 { ++ rockchip,pins = ++ <4 RK_PB6 2 &pcfg_pull_up_2ma>; ++ }; ++ ++ spi1_miso: spi1-miso { ++ rockchip,pins = ++ <4 RK_PB7 2 &pcfg_pull_up_2ma>; ++ }; ++ ++ spi1_csn1: spi1-csn1 { ++ rockchip,pins = ++ <4 RK_PC0 2 &pcfg_pull_up_2ma>; ++ }; ++ ++ spi1_clk_hs: spi1-clk-hs { ++ rockchip,pins = ++ <4 RK_PB4 2 &pcfg_pull_up_2ma>; ++ }; ++ ++ spi1_mosi_hs: spi1-mosi-hs { ++ rockchip,pins = ++ <4 RK_PB5 2 &pcfg_pull_up_2ma>; ++ }; ++ ++ spi1_csn0_hs: spi1-csn0-hs { ++ rockchip,pins = ++ <4 RK_PB6 2 &pcfg_pull_up_2ma>; ++ }; ++ ++ spi1_miso_hs: spi1-miso-hs { ++ rockchip,pins = ++ <4 RK_PB7 2 &pcfg_pull_up_2ma>; ++ }; ++ ++ spi1_csn1_hs: spi1-csn1-hs { ++ rockchip,pins = ++ <4 RK_PC0 2 &pcfg_pull_up_2ma>; ++ }; ++ }; ++ ++ spi1m1 { ++ spi1m1_clk: spi1m1-clk { ++ rockchip,pins = ++ <3 RK_PC7 3 &pcfg_pull_up_2ma>; ++ }; ++ ++ spi1m1_mosi: spi1m1-mosi { ++ rockchip,pins = ++ <3 RK_PD0 3 &pcfg_pull_up_2ma>; ++ }; ++ ++ spi1m1_csn0: spi1m1-csn0 { ++ rockchip,pins = ++ <3 RK_PD1 3 &pcfg_pull_up_2ma>; ++ }; ++ ++ spi1m1_miso: spi1m1-miso { ++ rockchip,pins = ++ <3 RK_PD2 3 &pcfg_pull_up_2ma>; ++ }; ++ ++ spi1m1_csn1: spi1m1-csn1 { ++ rockchip,pins = ++ <3 RK_PD3 3 &pcfg_pull_up_2ma>; ++ }; ++ ++ spi1m1_clk_hs: spi1m1-clk-hs { ++ rockchip,pins = ++ <3 RK_PC7 3 &pcfg_pull_up_2ma>; ++ }; ++ ++ spi1m1_mosi_hs: spi1m1-mosi-hs { ++ rockchip,pins = ++ <3 RK_PD0 3 &pcfg_pull_up_2ma>; ++ }; ++ ++ spi1m1_csn0_hs: spi1m1-csn0-hs { ++ rockchip,pins = ++ <3 RK_PD1 3 &pcfg_pull_up_2ma>; ++ }; ++ ++ spi1m1_miso_hs: spi1m1-miso-hs { ++ rockchip,pins = ++ <3 RK_PD2 3 &pcfg_pull_up_2ma>; ++ }; ++ ++ spi1m1_csn1_hs: spi1m1-csn1-hs { ++ rockchip,pins = ++ <3 RK_PD3 3 &pcfg_pull_up_2ma>; ++ }; ++ }; ++ ++ spi2m0 { ++ spi2m0_miso: spi2m0-miso { ++ rockchip,pins = ++ <1 RK_PA6 2 &pcfg_pull_up_2ma>; ++ }; ++ ++ spi2m0_clk: spi2m0-clk { ++ rockchip,pins = ++ <1 RK_PA7 2 &pcfg_pull_up_2ma>; ++ }; ++ ++ spi2m0_mosi: spi2m0-mosi { ++ rockchip,pins = ++ <1 RK_PB0 2 &pcfg_pull_up_2ma>; ++ }; ++ ++ spi2m0_csn: spi2m0-csn { ++ rockchip,pins = ++ <1 RK_PB1 2 &pcfg_pull_up_2ma>; ++ }; ++ ++ spi2m0_miso_hs: spi2m0-miso-hs { ++ rockchip,pins = ++ <1 RK_PA6 2 &pcfg_pull_up_2ma>; ++ }; ++ ++ spi2m0_clk_hs: spi2m0-clk-hs { ++ rockchip,pins = ++ <1 RK_PA7 2 &pcfg_pull_up_2ma>; ++ }; ++ ++ spi2m0_mosi_hs: spi2m0-mosi-hs { ++ rockchip,pins = ++ <1 RK_PB0 2 &pcfg_pull_up_2ma>; ++ }; ++ ++ spi2m0_csn_hs: spi2m0-csn-hs { ++ rockchip,pins = ++ <1 RK_PB1 2 &pcfg_pull_up_2ma>; ++ }; ++ }; ++ ++ spi2m1 { ++ spi2m1_miso: spi2m1-miso { ++ rockchip,pins = ++ <2 RK_PA4 3 &pcfg_pull_up_2ma>; ++ }; ++ ++ spi2m1_clk: spi2m1-clk { ++ rockchip,pins = ++ <2 RK_PA5 3 &pcfg_pull_up_2ma>; ++ }; ++ ++ spi2m1_mosi: spi2m1-mosi { ++ rockchip,pins = ++ <2 RK_PA6 3 &pcfg_pull_up_2ma>; ++ }; ++ ++ spi2m1_csn: spi2m1-csn { ++ rockchip,pins = ++ <2 RK_PA7 3 &pcfg_pull_up_2ma>; ++ }; ++ ++ spi2m1_miso_hs: spi2m1-miso-hs { ++ rockchip,pins = ++ <2 RK_PA4 3 &pcfg_pull_up_2ma>; ++ }; ++ ++ spi2m1_clk_hs: spi2m1-clk-hs { ++ rockchip,pins = ++ <2 RK_PA5 3 &pcfg_pull_up_2ma>; ++ }; ++ ++ spi2m1_mosi_hs: spi2m1-mosi-hs { ++ rockchip,pins = ++ <2 RK_PA6 3 &pcfg_pull_up_2ma>; ++ }; ++ ++ spi2m1_csn_hs: spi2m1-csn-hs { ++ rockchip,pins = ++ <2 RK_PA7 3 &pcfg_pull_up_2ma>; ++ }; ++ }; ++ ++ uart0 { ++ uart0_xfer: uart0-xfer { ++ rockchip,pins = ++ /* uart0_rx */ ++ <0 RK_PB3 1 &pcfg_pull_up_2ma>, ++ /* uart0_tx */ ++ <0 RK_PB2 1 &pcfg_pull_up_2ma>; ++ }; ++ ++ uart0_cts: uart0-cts { ++ rockchip,pins = ++ <0 RK_PB4 1 &pcfg_pull_none>; ++ }; ++ ++ uart0_rts: uart0-rts { ++ rockchip,pins = ++ <0 RK_PB5 1 &pcfg_pull_none>; ++ }; ++ }; ++ ++ uart1 { ++ uart1m0_xfer: uart1m0-xfer { ++ rockchip,pins = ++ /* uart1_rxm0 */ ++ <4 RK_PB0 2 &pcfg_pull_up_2ma>, ++ /* uart1_txm0 */ ++ <4 RK_PB1 2 &pcfg_pull_up_2ma>; ++ }; ++ ++ uart1m1_xfer: uart1m1-xfer { ++ rockchip,pins = ++ /* uart1_rxm1 */ ++ <1 RK_PB4 3 &pcfg_pull_up_2ma>, ++ /* uart1_txm1 */ ++ <1 RK_PB5 3 &pcfg_pull_up_2ma>; ++ }; ++ ++ uart1_cts: uart1-cts { ++ rockchip,pins = ++ <4 RK_PB2 2 &pcfg_pull_none>; ++ }; ++ ++ uart1_rts: uart1-rts { ++ rockchip,pins = ++ <4 RK_PB3 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ uart2 { ++ uart2m0_xfer: uart2m0-xfer { ++ rockchip,pins = ++ /* uart2_rxm0 */ ++ <4 RK_PA3 2 &pcfg_pull_up_2ma>, ++ /* uart2_txm0 */ ++ <4 RK_PA2 2 &pcfg_pull_up_2ma>; ++ }; ++ ++ uart2m1_xfer: uart2m1-xfer { ++ rockchip,pins = ++ /* uart2_rxm1 */ ++ <2 RK_PD1 2 &pcfg_pull_up_2ma>, ++ /* uart2_txm1 */ ++ <2 RK_PD0 2 &pcfg_pull_up_2ma>; ++ }; ++ ++ uart2m2_xfer: uart2m2-xfer { ++ rockchip,pins = ++ /* uart2_rxm2 */ ++ <3 RK_PA4 2 &pcfg_pull_up_2ma>, ++ /* uart2_txm2 */ ++ <3 RK_PA3 2 &pcfg_pull_up_2ma>; ++ }; ++ }; ++ ++ uart3 { ++ uart3m0_xfer: uart3m0-xfer { ++ rockchip,pins = ++ /* uart3_rxm0 */ ++ <0 RK_PC4 2 &pcfg_pull_up_2ma>, ++ /* uart3_txm0 */ ++ <0 RK_PC3 2 &pcfg_pull_up_2ma>; ++ }; ++ ++ uart3_ctsm0: uart3-ctsm0 { ++ rockchip,pins = ++ <0 RK_PC6 2 &pcfg_pull_none>; ++ }; ++ ++ uart3_rtsm0: uart3-rtsm0 { ++ rockchip,pins = ++ <0 RK_PC7 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ uart4 { ++ uart4_xfer: uart4-xfer { ++ rockchip,pins = ++ /* uart4_rx */ ++ <4 RK_PB4 1 &pcfg_pull_up_2ma>, ++ /* uart4_tx */ ++ <4 RK_PB5 1 &pcfg_pull_up_2ma>; ++ }; ++ ++ uart4_cts: uart4-cts { ++ rockchip,pins = ++ <4 RK_PB6 1 &pcfg_pull_none>; ++ }; ++ ++ uart4_rts: uart4-rts { ++ rockchip,pins = ++ <4 RK_PB7 1 &pcfg_pull_none>; ++ }; ++ }; ++ ++ uart5 { ++ uart5_xfer: uart5-xfer { ++ rockchip,pins = ++ /* uart5_rx */ ++ <3 RK_PC3 2 &pcfg_pull_up_2ma>, ++ /* uart5_tx */ ++ <3 RK_PC2 2 &pcfg_pull_up_2ma>; ++ }; ++ }; ++ ++ uart6 { ++ uart6_xfer: uart6-xfer { ++ rockchip,pins = ++ /* uart6_rx */ ++ <3 RK_PC5 2 &pcfg_pull_up_2ma>, ++ /* uart6_tx */ ++ <3 RK_PC4 2 &pcfg_pull_up_2ma>; ++ }; ++ }; ++ ++ uart7 { ++ uart7_xfer: uart7-xfer { ++ rockchip,pins = ++ /* uart7_rx */ ++ <3 RK_PC7 2 &pcfg_pull_up_2ma>, ++ /* uart7_tx */ ++ <3 RK_PC6 2 &pcfg_pull_up_2ma>; ++ }; ++ }; ++ ++ tsadc { ++ tsadc_otp_gpio: tsadc-otp-gpio { ++ rockchip,pins = ++ <0 RK_PA6 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ tsadc_otp_out: tsadc-otp-out { ++ rockchip,pins = ++ <0 RK_PA6 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ xin32k { ++ clkin_32k: clkin-32k { ++ rockchip,pins = ++ <0 RK_PC2 1 &pcfg_pull_none>; ++ }; ++ ++ clkout_32k: clkout-32k { ++ rockchip,pins = ++ <0 RK_PC2 1 &pcfg_pull_none>; ++ }; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk1808k.dtsi b/arch/arm64/boot/dts/rockchip/rk1808k.dtsi +new file mode 100755 +index 000000000000..78bd92e4c2c6 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk1808k.dtsi +@@ -0,0 +1,51 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++&cpu0_opp_table { ++ rockchip,high-temp = <85000>; ++ rockchip,high-temp-max-freq = <1008000>; ++}; ++ ++&dmc { ++ status = "okay"; ++ center-supply = <&vdd_log>; ++ system-status-freq = < ++ /*system status freq(KHz)*/ ++ SYS_STATUS_NORMAL 924000 ++ SYS_STATUS_REBOOT 924000 ++ >; ++}; ++ ++&dmc_opp_table { ++ rockchip,high-temp = <85000>; ++ rockchip,high-temp-max-freq = <664000>; ++ rockchip,thermal-zone = "soc-thermal"; ++}; ++ ++&thermal_zones { ++ soc-thermal { ++ sustainable-power = <1224>; ++ k_pu = <27>; ++ k_po = <55>; ++ k_i = <0>; ++ ++ trips { ++ trip-point-0 { ++ temperature = <85000>; ++ }; ++ trip-point-1 { ++ temperature = <100000>; ++ }; ++ }; ++ /delete-node/ cooling-maps; ++ cooling-maps { ++ map0 { ++ trip = <&target>; ++ cooling-device = ++ <&npu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; ++ }; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3308-ai-va-v10.dts b/arch/arm64/boot/dts/rockchip/rk3308-ai-va-v10.dts +new file mode 100755 +index 000000000000..c77d2e963266 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3308-ai-va-v10.dts +@@ -0,0 +1,681 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++ ++#include ++#include "rk3308.dtsi" ++ ++/ { ++ model = "Rockchip RK3308 voice assistant v10 board"; ++ compatible = "rockchip,rk3308-ai-va-v10", "rockchip,rk3308"; ++ ++ chosen { ++ bootargs = "earlycon=uart8250,mmio32,0xff0c0000 console=ttyFIQ0 root=PARTUUID=614e0000-0000 rootfstype=squashfs rootwait snd_aloop.index=7 snd_aloop.use_raw_jiffies=1"; ++ }; ++ ++ adc-keys0 { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 0>; ++ io-channel-names = "buttons"; ++ poll-interval = <100>; ++ keyup-threshold-microvolt = <1800000>; ++ ++ func-key { ++ linux,code = ; ++ label = "function"; ++ press-threshold-microvolt = <17000>; ++ }; ++ }; ++ ++ adc-keys1 { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 1>; ++ io-channel-names = "buttons"; ++ poll-interval = <100>; ++ keyup-threshold-microvolt = <1800000>; ++ ++ play-key { ++ linux,code = ; ++ label = "play"; ++ press-threshold-microvolt = <625000>; ++ }; ++ }; ++ ++ gpio-keys { ++ compatible = "gpio-keys"; ++ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&mic_mute>; ++ ++ mute { ++ gpios = <&gpio0 RK_PC0 GPIO_ACTIVE_LOW>; ++ linux,code = ; ++ label = "GPIO Mic Mute"; ++ debounce-interval = <100>; ++ }; ++ }; ++ ++ rotary { ++ compatible = "rotary-encoder"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&rotary_gpio>; ++ gpios = <&gpio2 RK_PB3 GPIO_ACTIVE_LOW>, ++ <&gpio2 RK_PB4 GPIO_ACTIVE_LOW>; ++ linux,axis = <0>; /* REL_X */ ++ rotary-encoder,relative-axis; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_LOW>; ++ }; ++ ++ sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,name = "i2s_8ch_0"; ++ ++ simple-audio-card,dai-link@0 { ++ format = "i2s"; ++ cpu { ++ sound-dai = <&i2s_8ch_0>; ++ }; ++ ++ codec { ++ sound-dai = <&dummy_codec>; ++ }; ++ }; ++ ++ simple-audio-card,dai-link@1 { ++ format = "i2s"; ++ cpu { ++ sound-dai = <&i2s_8ch_0>; ++ }; ++ ++ codec { ++ sound-dai = <&tas5711>; ++ }; ++ }; ++ }; ++ ++ dummy_codec: dummy-codec { ++ compatible = "rockchip,dummy-codec"; ++ #sound-dai-cells = <0>; ++ }; ++ ++ vdd_log: vdd_core: vdd-core { ++ compatible = "pwm-regulator"; ++ pwms = <&pwm0 0 5000 1>; ++ regulator-name = "vdd_core"; ++ regulator-min-microvolt = <827000>; ++ regulator-max-microvolt = <1340000>; ++ regulator-init-microvolt = <1015000>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-settling-time-up-us = <250>; ++ status = "okay"; ++ }; ++ ++ vdd_1v0: vdd-1v0 { ++ compatible = "regulator-fixed"; ++ regulator-name = "vdd_1v0"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ }; ++ ++ vccio_sdio: vcc_1v8: vcc-1v8 { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_1v8"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ vin-supply = <&vcc_io>; ++ }; ++ ++ vcc_1v8_codec: vcc-1v8-codec { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_1v8_codec"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ vin-supply = <&vcc_io>; ++ }; ++ ++ vcc_ddr: vcc-ddr { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_ddr"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ }; ++ ++ vcc_3v3_codec: vcc_io: vcc-io { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_io"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ }; ++ ++ vccio_flash: vccio-flash { ++ compatible = "regulator-fixed"; ++ regulator-name = "vccio_flash"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ uart_rts_gpios = <&gpio4 RK_PA7 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart4_rts>; ++ pinctrl-1 = <&uart4_rts_gpio>; ++ BT,power_gpio = <&gpio4 RK_PB3 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio4 RK_PB4 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ rockchip,grf = <&grf>; ++ wifi_chip_type = "ap6255"; ++ WIFI,host_wake_irq = <&gpio0 RK_PA0 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++}; ++ ++&cpu0 { ++ cpu-supply = <&vdd_core>; ++}; ++ ++&dmc { ++ center-supply = <&vdd_core>; ++ status = "okay"; ++}; ++ ++&emmc { ++ bus-width = <8>; ++ cap-mmc-highspeed; ++ mmc-hs200-1_8v; ++ no-sdio; ++ no-sd; ++ disable-wp; ++ non-removable; ++ num-slots = <1>; ++ status = "okay"; ++}; ++ ++&fiq_debugger { ++ status = "okay"; ++}; ++ ++&io_domains { ++ status = "okay"; ++ ++ vccio0-supply = <&vcc_io>; ++ vccio1-supply = <&vcc_io>; ++ vccio2-supply = <&vcc_1v8>; ++ vccio3-supply = <&vccio_flash>; ++ vccio4-supply = <&vccio_sdio>; ++ vccio5-supply = <&vcc_io>; ++}; ++ ++&i2c1 { ++ clock-frequency = <400000>; ++ status = "okay"; ++ ++ is31fl3236: led-controller@3c { ++ compatible = "issi,is31fl3236"; ++ reg = <0x3c>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ reset-gpios = <&gpio0 RK_PB0 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ ++ led1: led@1 { ++ label = "led1"; ++ reg = <1>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <0>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led2: led@2 { ++ label = "led2"; ++ reg = <2>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <0>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led3: led@3 { ++ label = "led3"; ++ reg = <3>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led4: led@4 { ++ label = "led4"; ++ reg = <4>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <100>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led5: led@5 { ++ label = "led5"; ++ reg = <5>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <100>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led6: led@6 { ++ label = "led6"; ++ reg = <6>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led7: led@7 { ++ label = "led7"; ++ reg = <7>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <200>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led8: led@8 { ++ label = "led8"; ++ reg = <8>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <200>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led9: led@9 { ++ label = "led9"; ++ reg = <9>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led10: led@10 { ++ label = "led10"; ++ reg = <10>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <300>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led11: led@11 { ++ label = "led11"; ++ reg = <11>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <300>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led12: led@12 { ++ label = "led12"; ++ reg = <12>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led13: led@13 { ++ label = "led13"; ++ reg = <13>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <400>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led14: led@14 { ++ label = "led14"; ++ reg = <14>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <400>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led15: led@15 { ++ label = "led15"; ++ reg = <15>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led16: led@16 { ++ label = "led16"; ++ reg = <16>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <500>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led17: led@17 { ++ label = "led17"; ++ reg = <17>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <500>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led18: led@18 { ++ label = "led18"; ++ reg = <18>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led19: led@19 { ++ label = "led19"; ++ reg = <19>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <600>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led20: led@20 { ++ label = "led20"; ++ reg = <20>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <600>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led21: led@21 { ++ label = "led21"; ++ reg = <21>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led22: led@22 { ++ label = "led22"; ++ reg = <22>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <700>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led23: led@23 { ++ label = "led23"; ++ reg = <23>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <700>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led124: led@24 { ++ label = "led24"; ++ reg = <24>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led25: led@25 { ++ label = "led25"; ++ reg = <25>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <800>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led26: led@26 { ++ label = "led26"; ++ reg = <26>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <800>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led27: led@27 { ++ label = "led27"; ++ reg = <27>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led28: led@28 { ++ label = "led28"; ++ reg = <28>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <900>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led29: led@29 { ++ label = "led29"; ++ reg = <29>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <900>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led30: led@30 { ++ label = "led30"; ++ reg = <30>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led31: led@31 { ++ label = "led31"; ++ reg = <31>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <1000>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led32: led@32 { ++ label = "led32"; ++ reg = <32>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <1000>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led33: led@33 { ++ label = "led33"; ++ reg = <33>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led34: led@34 { ++ label = "led34"; ++ reg = <34>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <1100>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led35: led@35 { ++ label = "led35"; ++ reg = <35>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <1100>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led36: led@36 { ++ label = "led36"; ++ reg = <36>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ }; ++ ++ tas5711: tas5711@1b { ++ #sound-dai-cells = <0>; ++ compatible = "ti,tas5711"; ++ reg = <0x1b>; ++ clocks = <&cru SCLK_I2S0_8CH_TX_OUT>; ++ clock-names = "mclk"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s_8ch_0_mclk>; ++ pdn-gpios = <&gpio0 RK_PA5 GPIO_ACTIVE_LOW>; ++ reset-gpios = <&gpio0 RK_PC4 GPIO_ACTIVE_LOW>; ++ }; ++}; ++ ++&i2s_8ch_0 { ++ status = "okay"; ++ assigned-clocks = <&cru SCLK_I2S0_8CH_RX>; ++ assigned-clock-parents = <&cru SCLK_I2S0_8CH_TX_MUX>; ++ rockchip,clk-trcm = <1>; ++ #sound-dai-cells = <0>; ++}; ++ ++&pinctrl { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&rtc_32k>; ++ ++ buttons { ++ mic_mute: mic-mute { ++ rockchip,pins = <0 RK_PC0 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ rotary { ++ rotary_gpio: rotary-gpio { ++ rockchip,pins = ++ <2 RK_PB3 RK_FUNC_GPIO &pcfg_pull_none>, ++ <2 RK_PB4 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++&pwm0 { ++ status = "okay"; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm0_pin_pull_down>; ++}; ++ ++&rockchip_suspend { ++ rockchip,pwm-regulator-config = < ++ (0 ++ | RKPM_PWM_REGULATOR ++ ) ++ >; ++ ++ status = "okay"; ++}; ++ ++&saradc { ++ status = "okay"; ++ vref-supply = <&vcc_1v8>; ++}; ++ ++&sdio { ++ bus-width = <4>; ++ cap-sd-highspeed; ++ no-sd; ++ no-mmc; ++ ignore-pm-notify; ++ keep-power-in-suspend; ++ non-removable; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&tsadc { ++ rockchip,hw-tshut-mode = <0>; /* tshut mode 0:CRU 1:GPIO */ ++ rockchip,hw-tshut-polarity = <1>; /* tshut polarity 0:LOW 1:HIGH */ ++ status = "okay"; ++}; ++ ++&uart4 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart4_xfer &uart4_cts>; ++ status = "okay"; ++}; ++ ++&u2phy { ++ status = "okay"; ++ ++ u2phy_otg: otg-port { ++ status = "okay"; ++ }; ++}; ++ ++&usb20_otg { ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3308-evb-amic-v10.dts b/arch/arm64/boot/dts/rockchip/rk3308-evb-amic-v10.dts +new file mode 100755 +index 000000000000..557daa856132 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3308-evb-amic-v10.dts +@@ -0,0 +1,55 @@ ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++ */ ++ ++/dts-v1/; ++ ++#include "rk3308-evb-v10.dtsi" ++ ++/ { ++ model = "Rockchip RK3308 evb analog mic board"; ++ compatible = "rockchip,rk3308-evb-amic-v10", "rockchip,rk3308"; ++ ++ vad_acodec_sound: vad-acodec-sound { ++ status = "okay"; ++ compatible = "rockchip,multicodecs-card"; ++ rockchip,card-name = "rockchip,rk3308-vad"; ++ rockchip,codec-hp-det; ++ rockchip,mclk-fs = <256>; ++ rockchip,cpu = <&i2s_8ch_2>; ++ rockchip,codec = <&acodec>, <&vad>; ++ }; ++}; ++ ++&acodec { ++ rockchip,micbias1; ++ rockchip,micbias2; ++ rockchip,en-always-grps = <0 1 2>; ++}; ++ ++&acodec_sound { ++ status = "disabled"; ++}; ++ ++&bluetooth_sound { ++ status = "okay"; ++}; ++ ++&i2s_2ch_0 { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++}; ++ ++&is31fl3236 { ++ reg = <0x3f>; ++}; ++ ++&vad { ++ status = "okay"; ++ rockchip,audio-src = <&i2s_8ch_2>; ++ rockchip,buffer-time-ms = <200>; ++ rockchip,mode = <1>; ++ #sound-dai-cells = <0>; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3308-evb-amic-v11.dts b/arch/arm64/boot/dts/rockchip/rk3308-evb-amic-v11.dts +new file mode 100755 +index 000000000000..90a29a903545 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3308-evb-amic-v11.dts +@@ -0,0 +1,56 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++ ++#include "rk3308-evb-v11.dtsi" ++ ++/ { ++ model = "Rockchip RK3308 evb analog mic v11 board"; ++ compatible = "rockchip,rk3308-evb-amic-v11", "rockchip,rk3308"; ++ ++ vad_acodec_sound: vad-acodec-sound { ++ status = "okay"; ++ compatible = "rockchip,multicodecs-card"; ++ rockchip,card-name = "rockchip,rk3308-vad"; ++ rockchip,codec-hp-det; ++ rockchip,mclk-fs = <256>; ++ rockchip,cpu = <&i2s_8ch_2>; ++ rockchip,codec = <&acodec>, <&vad>; ++ }; ++}; ++ ++&acodec { ++ rockchip,micbias1; ++ rockchip,micbias2; ++ rockchip,en-always-grps = <1 2 3>; ++ rockchip,adc-grps-route = <1 2 3 0>; ++}; ++ ++&acodec_sound { ++ status = "disabled"; ++}; ++ ++&bluetooth_sound { ++ status = "okay"; ++}; ++ ++&i2s_2ch_0 { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++}; ++ ++&is31fl3236 { ++ reg = <0x3f>; ++}; ++ ++&vad { ++ status = "okay"; ++ rockchip,audio-src = <&i2s_8ch_2>; ++ rockchip,det-channel = <0>; ++ rockchip,buffer-time-ms = <200>; ++ rockchip,mode = <1>; ++ #sound-dai-cells = <0>; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3308-evb-dmic-i2s-v10.dts b/arch/arm64/boot/dts/rockchip/rk3308-evb-dmic-i2s-v10.dts +new file mode 100755 +index 000000000000..88c1e9c6c8ac +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3308-evb-dmic-i2s-v10.dts +@@ -0,0 +1,101 @@ ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++ */ ++ ++/dts-v1/; ++ ++#include "rk3308-evb-v10.dtsi" ++ ++/ { ++ model = "Rockchip RK3308 evb digital-i2s mic board"; ++ compatible = "rockchip,rk3308-evb-dmic-i2s-v10", "rockchip,rk3308"; ++ ++ i2s_16ch_dais: i2s-16ch-dais { ++ status = "disabled"; ++ compatible = "rockchip,rk3308-multi-dais", "rockchip,multi-dais"; ++ dais = <&i2s_8ch_0>, <&i2s_8ch_1>; ++ capture,channel-mapping = <8 8>; ++ playback,channel-mapping = <0 0>; ++ bitclock-master = <1 0>; ++ frame-master = <1 0>; ++ rockchip,grf = <&grf>; ++ }; ++ ++ i2s_8ch_0_2_dais: i2s-8ch-0-2-dais { ++ status = "okay"; ++ compatible = "rockchip,rk3308-multi-dais", "rockchip,multi-dais"; ++ dais = <&i2s_8ch_0>, <&i2s_8ch_2>; ++ capture,channel-mapping = <6 2>; ++ playback,channel-mapping = <0 2>; ++ }; ++ ++ i2s-dmic-array { ++ status = "disabled"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,i2s-dmic-array"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s_8ch_0>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&dummy_codec>; ++ }; ++ }; ++ ++ vad-sound { ++ status = "okay"; ++ compatible = "rockchip,multicodecs-card"; ++ rockchip,card-name = "rockchip,rk3308-vad"; ++ rockchip,cpu = <&i2s_8ch_0_2_dais>; ++ rockchip,codec = <&acodec>, <&vad>; ++ }; ++}; ++ ++&acodec_sound { ++ status = "disabled"; ++}; ++ ++&bluetooth_sound { ++ status = "okay"; ++}; ++ ++&i2s_2ch_0 { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++}; ++ ++&i2s_8ch_0 { ++ status = "okay"; ++ rockchip,no-dmaengine; ++ #sound-dai-cells = <0>; ++}; ++ ++&i2s_8ch_1 { ++ status = "disabled"; ++ #sound-dai-cells = <0>; ++ rockchip,no-dmaengine; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s_8ch_1_m0_sdo0 ++ &i2s_8ch_1_m0_sdo1_sdi3 ++ &i2s_8ch_1_m0_sdo2_sdi2 ++ &i2s_8ch_1_m0_sdo3_sdi1 ++ &i2s_8ch_1_m0_sdi0>; ++}; ++ ++&i2s_8ch_2 { ++ status = "okay"; ++ rockchip,no-dmaengine; ++ #sound-dai-cells = <0>; ++}; ++ ++&vad { ++ status = "okay"; ++ rockchip,audio-src = <&i2s_8ch_0>; ++ rockchip,buffer-time-ms = <200>; ++ rockchip,det-channel = <0>; ++ rockchip,mode = <1>; ++ #sound-dai-cells = <0>; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3308-evb-dmic-i2s-v11.dts b/arch/arm64/boot/dts/rockchip/rk3308-evb-dmic-i2s-v11.dts +new file mode 100755 +index 000000000000..364f74a40d9d +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3308-evb-dmic-i2s-v11.dts +@@ -0,0 +1,77 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++ ++#include "rk3308-evb-v11.dtsi" ++ ++/ { ++ model = "Rockchip RK3308 evb digital-i2s mic v11 board"; ++ compatible = "rockchip,rk3308-evb-dmic-i2s-v11", "rockchip,rk3308"; ++ ++ i2s_8ch_0_2_dais: i2s-8ch-0-2-dais { ++ status = "okay"; ++ compatible = "rockchip,rk3308-multi-dais", "rockchip,multi-dais"; ++ dais = <&i2s_8ch_0>, <&i2s_8ch_2>; ++ capture,channel-mapping = <6 2>; ++ playback,channel-mapping = <0 2>; ++ }; ++ ++ i2s-dmic-array { ++ status = "disabled"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,i2s-dmic-array"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s_8ch_0>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&dummy_codec>; ++ }; ++ }; ++ ++ vad-sound { ++ status = "okay"; ++ compatible = "rockchip,multicodecs-card"; ++ rockchip,card-name = "rockchip,rk3308-vad"; ++ rockchip,cpu = <&i2s_8ch_0_2_dais>; ++ rockchip,codec = <&acodec>, <&vad>; ++ }; ++}; ++ ++&acodec_sound { ++ status = "disabled"; ++}; ++ ++&bluetooth_sound { ++ status = "okay"; ++}; ++ ++&i2s_2ch_0 { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++}; ++ ++&i2s_8ch_0 { ++ status = "okay"; ++ rockchip,no-dmaengine; ++ #sound-dai-cells = <0>; ++}; ++ ++&i2s_8ch_2 { ++ status = "okay"; ++ rockchip,no-dmaengine; ++ #sound-dai-cells = <0>; ++}; ++ ++&vad { ++ status = "okay"; ++ rockchip,audio-src = <&i2s_8ch_0>; ++ rockchip,buffer-time-ms = <200>; ++ rockchip,det-channel = <0>; ++ rockchip,mode = <1>; ++ #sound-dai-cells = <0>; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3308-evb-dmic-pdm-v10.dts b/arch/arm64/boot/dts/rockchip/rk3308-evb-dmic-pdm-v10.dts +new file mode 100755 +index 000000000000..e2891ce1092c +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3308-evb-dmic-pdm-v10.dts +@@ -0,0 +1,88 @@ ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++ */ ++ ++/dts-v1/; ++ ++#include "rk3308-evb-v10.dtsi" ++ ++/ { ++ model = "Rockchip RK3308 evb digital-pdm mic board"; ++ compatible = "rockchip,rk3308-evb-dmic-pdm-v10", "rockchip,rk3308"; ++ ++ pdm_i2s_dais: pdm-i2s-dais { ++ status = "okay"; ++ compatible = "rockchip,rk3308-multi-dais", "rockchip,multi-dais"; ++ dais = <&pdm_8ch>, <&i2s_8ch_2>; ++ capture,channel-mapping = <6 2>; ++ playback,channel-mapping = <0 2>; ++ }; ++ ++ pdm-mic-array { ++ status = "disabled"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,name = "rockchip,pdm-mic-array"; ++ simple-audio-card,cpu { ++ sound-dai = <&pdm_8ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&dummy_codec>; ++ }; ++ }; ++ ++ vad-sound { ++ status = "okay"; ++ compatible = "rockchip,multicodecs-card"; ++ rockchip,card-name = "rockchip,rk3308-vad"; ++ rockchip,cpu = <&pdm_i2s_dais>; ++ rockchip,codec = <&acodec>, <&vad>; ++ }; ++}; ++ ++&acodec_sound { ++ status = "disabled"; ++}; ++ ++&bluetooth_sound { ++ status = "okay"; ++}; ++ ++&i2s_2ch_0 { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++}; ++ ++&pdm_8ch { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++ rockchip,no-dmaengine; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pdm_m2_clk ++ &pdm_m2_clkm ++ &pdm_m2_sdi0 ++ &pdm_m2_sdi1 ++ &pdm_m2_sdi2 ++ &pdm_m2_sdi3>; ++}; ++ ++&vad { ++ status = "okay"; ++ rockchip,audio-src = <&pdm_8ch>; ++ rockchip,buffer-time-ms = <200>; ++ rockchip,det-channel = <2>; ++ rockchip,mode = <1>; ++ #sound-dai-cells = <0>; ++}; ++ ++&pdm_i2s_dais { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++}; ++ ++&i2s_8ch_2 { ++ status = "okay"; ++ rockchip,no-dmaengine; ++ #sound-dai-cells = <0>; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3308-evb-dmic-pdm-v11.dts b/arch/arm64/boot/dts/rockchip/rk3308-evb-dmic-pdm-v11.dts +new file mode 100755 +index 000000000000..c4a7178f1dc8 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3308-evb-dmic-pdm-v11.dts +@@ -0,0 +1,92 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++ ++#include "rk3308-evb-v11.dtsi" ++ ++/ { ++ model = "Rockchip RK3308 evb digital-pdm mic v11 board"; ++ compatible = "rockchip,rk3308-evb-dmic-pdm-v11", "rockchip,rk3308"; ++ ++ pdm_i2s_dais: pdm-i2s-dais { ++ status = "okay"; ++ compatible = "rockchip,rk3308-multi-dais", "rockchip,multi-dais"; ++ dais = <&pdm_8ch>, <&i2s_8ch_2>; ++ capture,channel-mapping = <6 2>; ++ playback,channel-mapping = <0 2>; ++ bitclock-inversion = <1 0>; ++ }; ++ ++ pdm-mic-array { ++ status = "disabled"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,name = "rockchip,pdm-mic-array"; ++ simple-audio-card,cpu { ++ sound-dai = <&pdm_8ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&dummy_codec>; ++ }; ++ }; ++ ++ vad-sound { ++ status = "okay"; ++ compatible = "rockchip,multicodecs-card"; ++ rockchip,card-name = "rockchip,rk3308-vad"; ++ rockchip,cpu = <&pdm_i2s_dais>; ++ rockchip,codec = <&acodec>, <&vad>; ++ }; ++}; ++ ++&rk_timer_rtc { ++ status = "okay"; ++}; ++ ++&acodec_sound { ++ status = "disabled"; ++}; ++ ++&bluetooth_sound { ++ status = "okay"; ++}; ++ ++&i2s_2ch_0 { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++}; ++ ++&pdm_8ch { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++ rockchip,no-dmaengine; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pdm_m2_clk ++ &pdm_m2_clkm ++ &pdm_m2_sdi0 ++ &pdm_m2_sdi1 ++ &pdm_m2_sdi2 ++ &pdm_m2_sdi3>; ++}; ++ ++&vad { ++ status = "okay"; ++ rockchip,audio-src = <&pdm_8ch>; ++ rockchip,det-channel = <0>; ++ rockchip,mode = <1>; ++ rockchip,buffer-time-ms = <200>; ++ #sound-dai-cells = <0>; ++}; ++ ++&i2s_8ch_2 { ++ status = "okay"; ++ rockchip,no-dmaengine; ++ #sound-dai-cells = <0>; ++}; ++ ++&pdm_i2s_dais { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3308-evb-ext-v10.dtsi b/arch/arm64/boot/dts/rockchip/rk3308-evb-ext-v10.dtsi +new file mode 100755 +index 000000000000..1c0e66384c16 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3308-evb-ext-v10.dtsi +@@ -0,0 +1,235 @@ ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++ */ ++ ++/ { ++ backlight: backlight { ++ status = "okay"; ++ compatible = "pwm-backlight"; ++ pwms = <&pwm1 0 25000 0>; ++ brightness-levels = < ++ 0 1 2 3 4 5 6 7 ++ 8 9 10 11 12 13 14 15 ++ 16 17 18 19 20 21 22 23 ++ 24 25 26 27 28 29 30 31 ++ 32 33 34 35 36 37 38 39 ++ 40 41 42 43 44 45 46 47 ++ 48 49 50 51 52 53 54 55 ++ 56 57 58 59 60 61 62 63 ++ 64 65 66 67 68 69 70 71 ++ 72 73 74 75 76 77 78 79 ++ 80 81 82 83 84 85 86 87 ++ 88 89 90 91 92 93 94 95 ++ 96 97 98 99 100 101 102 103 ++ 104 105 106 107 108 109 110 111 ++ 112 113 114 115 116 117 118 119 ++ 120 121 122 123 124 125 126 127 ++ 128 129 130 131 132 133 134 135 ++ 136 137 138 139 140 141 142 143 ++ 144 145 146 147 148 149 150 151 ++ 152 153 154 155 156 157 158 159 ++ 160 161 162 163 164 165 166 167 ++ 168 169 170 171 172 173 174 175 ++ 176 177 178 179 180 181 182 183 ++ 184 185 186 187 188 189 190 191 ++ 192 193 194 195 196 197 198 199 ++ 200 201 202 203 204 205 206 207 ++ 208 209 210 211 212 213 214 215 ++ 216 217 218 219 220 221 222 223 ++ 224 225 226 227 228 229 230 231 ++ 232 233 234 235 236 237 238 239 ++ 240 241 242 243 244 245 246 247 ++ 248 249 250 251 252 253 254 255>; ++ default-brightness-level = <200>; ++ }; ++ ++ panel: panel { ++ compatible = "simple-panel"; ++ bus-format = ; ++ backlight = <&backlight>; ++ enable-gpios = <&gpio0 RK_PA4 GPIO_ACTIVE_LOW>; ++ enable-delay-ms = <20>; ++ reset-gpios = <&gpio0 RK_PC4 GPIO_ACTIVE_LOW>; ++ reset-delay-ms = <10>; ++ prepare-delay-ms = <20>; ++ unprepare-delay-ms = <20>; ++ disable-delay-ms = <20>; ++ /* spi-sdo-gpios = <&gpio3 RK_PD2 GPIO_ACTIVE_HIGH>; */ ++ spi-sdi-gpios = <&gpio1 RK_PC7 GPIO_ACTIVE_HIGH>; ++ spi-scl-gpios = <&gpio1 RK_PD0 GPIO_ACTIVE_HIGH>; ++ spi-cs-gpios = <&gpio1 RK_PD1 GPIO_ACTIVE_HIGH>; ++ width-mm = <217>; ++ height-mm = <136>; ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&spi_init_cmd>; ++ rockchip,cmd-type = "spi"; ++ ++ /* type:0 is cmd, 1 is data */ ++ panel-init-sequence = [ ++ /* type delay num val1 val2 val3 */ ++ 00 00 01 e0 ++ 01 00 01 00 ++ 01 00 01 07 ++ 01 00 01 0f ++ 01 00 01 0d ++ 01 00 01 1b ++ 01 00 01 0a ++ 01 00 01 3c ++ 01 00 01 78 ++ 01 00 01 4a ++ 01 00 01 07 ++ 01 00 01 0e ++ 01 00 01 09 ++ 01 00 01 1b ++ 01 00 01 1e ++ 01 00 01 0f ++ 00 00 01 e1 ++ 01 00 01 00 ++ 01 00 01 22 ++ 01 00 01 24 ++ 01 00 01 06 ++ 01 00 01 12 ++ 01 00 01 07 ++ 01 00 01 36 ++ 01 00 01 47 ++ 01 00 01 47 ++ 01 00 01 06 ++ 01 00 01 0a ++ 01 00 01 07 ++ 01 00 01 30 ++ 01 00 01 37 ++ 01 00 01 0f ++ ++ 00 00 01 c0 ++ 01 00 01 10 ++ 01 00 01 10 ++ ++ 00 00 01 c1 ++ 01 00 01 41 ++ ++ 00 00 01 c5 ++ 01 00 01 00 ++ 01 00 01 22 ++ 01 00 01 80 ++ ++ 00 00 01 36 ++ 01 00 01 48 ++ ++ 00 00 01 3a /* interface mode control */ ++ 01 00 01 66 ++ ++ 00 00 01 b0 /* interface mode control */ ++ 01 00 01 00 ++ ++ 00 00 01 b1 /* frame rate 70hz */ ++ 01 00 01 b0 ++ 01 00 01 11 ++ 00 00 01 b4 ++ 01 00 01 02 ++ 00 00 01 B6 /* RGB/MCU Interface Control */ ++ 01 00 01 32 /* 02 mcu, 32 rgb */ ++ 01 00 01 02 ++ ++ 00 00 01 b7 ++ 01 00 01 c6 ++ ++ 00 00 01 be ++ 01 00 01 00 ++ 01 00 01 04 ++ ++ 00 00 01 e9 ++ 01 00 01 00 ++ ++ 00 00 01 f7 ++ 01 00 01 a9 ++ 01 00 01 51 ++ 01 00 01 2c ++ 01 00 01 82 ++ ++ 00 78 01 11 ++ 00 00 01 29 ++ ]; ++ ++ panel-exit-sequence = [ ++ /* type delay num val1 val2 val3 */ ++ 00 0a 01 28 ++ 00 78 01 10 ++ ]; ++ ++ display-timings { ++ native-mode = <&kd050fwfba002_timing>; ++ ++ kd050fwfba002_timing: timing0 { ++ clock-frequency = <12000000>; ++ hactive = <320>; ++ vactive = <480>; ++ hback-porch = <10>; ++ hfront-porch = <5>; ++ vback-porch = <10>; ++ vfront-porch = <5>; ++ hsync-len = <10>; ++ vsync-len = <10>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ port { ++ panel_in_rgb: endpoint { ++ remote-endpoint = <&rgb_out_panel>; ++ }; ++ }; ++ }; ++}; ++ ++&display_subsystem { ++ status = "okay"; ++}; ++ ++&pinctrl { ++ spi_panel { ++ spi_init_cmd: spi-init-cmd { ++ rockchip,pins = ++ /* spi sdi */ ++ <1 RK_PC7 RK_FUNC_GPIO &pcfg_pull_none>, ++ /* spi scl */ ++ <1 RK_PD0 RK_FUNC_GPIO &pcfg_pull_none>, ++ /* spi cs */ ++ <1 RK_PD1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++&pwm1 { ++ status = "okay"; ++}; ++ ++&rgb { ++ status = "okay"; ++ ++ ports { ++ rgb_out: port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ rgb_out_panel: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&panel_in_rgb>; ++ }; ++ }; ++ }; ++}; ++ ++&route_rgb { ++ status = "okay"; ++}; ++ ++&vop { ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3308-evb-v10.dtsi b/arch/arm64/boot/dts/rockchip/rk3308-evb-v10.dtsi +new file mode 100755 +index 000000000000..3d052d0da477 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3308-evb-v10.dtsi +@@ -0,0 +1,780 @@ ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++ */ ++ ++#include ++#include "rk3308.dtsi" ++ ++/ { ++ model = "Rockchip RK3308 EVB"; ++ compatible = "rockchip,rk3308-evb", "rockchip,rk3308"; ++ ++ chosen { ++ bootargs = "earlycon=uart8250,mmio32,0xff0c0000 console=ttyFIQ0 root=PARTUUID=614e0000-0000 rootfstype=squashfs rootwait snd_aloop.index=7 snd_aloop.use_raw_jiffies=1"; ++ }; ++ ++ adc-keys0 { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 0>; ++ io-channel-names = "buttons"; ++ poll-interval = <100>; ++ keyup-threshold-microvolt = <1800000>; ++ ++ func-key { ++ linux,code = ; ++ label = "function"; ++ press-threshold-microvolt = <18000>; ++ }; ++ }; ++ ++ adc-keys1 { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 1>; ++ io-channel-names = "buttons"; ++ poll-interval = <100>; ++ keyup-threshold-microvolt = <1800000>; ++ ++ esc-key { ++ linux,code = ; ++ label = "micmute"; ++ press-threshold-microvolt = <1130000>; ++ }; ++ ++ home-key { ++ linux,code = ; ++ label = "mode"; ++ press-threshold-microvolt = <901000>; ++ }; ++ ++ menu-key { ++ linux,code = ; ++ label = "play"; ++ press-threshold-microvolt = <624000>; ++ }; ++ ++ vol-down-key { ++ linux,code = ; ++ label = "volume down"; ++ press-threshold-microvolt = <300000>; ++ }; ++ ++ vol-up-key { ++ linux,code = ; ++ label = "volume up"; ++ press-threshold-microvolt = <18000>; ++ }; ++ }; ++ ++ dummy_codec: dummy-codec { ++ compatible = "rockchip,dummy-codec"; ++ #sound-dai-cells = <0>; ++ }; ++ ++ gpio-keys { ++ compatible = "gpio-keys"; ++ autorepeat; ++ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pwr_key>; ++ ++ power { ++ gpios = <&gpio0 RK_PA6 GPIO_ACTIVE_LOW>; ++ linux,code = ; ++ label = "GPIO Key Power"; ++ wakeup-source; ++ debounce-interval = <100>; ++ }; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_LOW>; ++ }; ++ ++ acodec_sound: acodec-sound { ++ compatible = "rockchip,multicodecs-card"; ++ rockchip,card-name = "rockchip,rk3308-acodec"; ++ rockchip,codec-hp-det; ++ rockchip,mclk-fs = <256>; ++ rockchip,cpu = <&i2s_8ch_2>; ++ rockchip,codec = <&acodec>; ++ }; ++ ++ bluetooth_sound: bluetooth-sound { ++ status = "disabled"; ++ compatible = "rockchip,multicodecs-card"; ++ rockchip,card-name = "rockchip,rk3308-pcm"; ++ rockchip,mclk-fs = <128>; ++ rockchip,cpu = <&i2s_2ch_0>; ++ rockchip,codec = <&dummy_codec>; ++ rockchip,format = "dsp_b"; ++ rockchip,bitclock-inversion = <0>; ++ rockchip,wait-card-locked = <0>; ++ }; ++ ++ spdif_rx_sound: spdif-rx-sound { ++ status = "disabled"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,name = "rockchip,spdif-rx-sound"; ++ simple-audio-card,cpu { ++ sound-dai = <&spdif_rx>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&dummy_codec>; ++ }; ++ }; ++ ++ spdif_tx_sound: spdif-tx-sound { ++ status = "disabled"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,name = "rockchip,spdif-tx-sound"; ++ simple-audio-card,cpu { ++ sound-dai = <&spdif_tx>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&dummy_codec>; ++ }; ++ }; ++ ++ vdd_log: vdd_core: vdd-core { ++ compatible = "pwm-regulator"; ++ pwms = <&pwm0 0 5000 1>; ++ regulator-name = "vdd_core"; ++ regulator-min-microvolt = <827000>; ++ regulator-max-microvolt = <1340000>; ++ regulator-init-microvolt = <1015000>; ++ regulator-early-min-microvolt = <1015000>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-settling-time-up-us = <250>; ++ status = "okay"; ++ }; ++ ++ vdd_1v0: vdd-1v0 { ++ compatible = "regulator-fixed"; ++ regulator-name = "vdd_1v0"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ }; ++ ++ vccio_sdio: vcc_1v8: vcc-1v8 { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_1v8"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ vin-supply = <&vcc_io>; ++ }; ++ ++ vcc_sd: vcc-sd { ++ compatible = "regulator-fixed"; ++ gpio = <&gpio4 RK_PD6 GPIO_ACTIVE_LOW>; ++ regulator-name = "vcc_sd"; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ }; ++ ++ vcc_1v8_codec: vcc-1v8-codec { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_1v8_codec"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ vin-supply = <&vcc_io>; ++ }; ++ ++ vcc_ddr: vcc-ddr { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_ddr"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ }; ++ ++ vcc_3v3_codec: vcc_io: vcc-io { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_io"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ }; ++ ++ vccio_flash: vccio-flash { ++ compatible = "regulator-fixed"; ++ regulator-name = "vccio_flash"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ }; ++ ++ vcc_phy: vcc-phy-regulator { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_phy"; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ vbus_host: vbus-host-regulator { ++ compatible = "regulator-fixed"; ++ enable-active-high; ++ gpio = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&usb_drv>; ++ regulator-name = "vbus_host"; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ uart_rts_gpios = <&gpio4 RK_PA7 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart4_rts>; ++ pinctrl-1 = <&uart4_rts_gpio>; ++ BT,power_gpio = <&gpio4 RK_PB3 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio4 RK_PB4 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ rockchip,grf = <&grf>; ++ wifi_chip_type = "ap6255"; ++ WIFI,host_wake_irq = <&gpio0 RK_PA0 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++}; ++ ++&acodec { ++ status = "okay"; ++ ++ rockchip,no-deep-low-power; ++ rockchip,loopback-grp = <3>; ++ hp-ctl-gpios = <&gpio0 RK_PA1 GPIO_ACTIVE_HIGH>; ++ spk-ctl-gpios = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>; ++}; ++ ++&cpu0 { ++ cpu-supply = <&vdd_core>; ++}; ++ ++&dmc { ++ center-supply = <&vdd_core>; ++ status = "okay"; ++}; ++ ++&emmc { ++ bus-width = <8>; ++ cap-mmc-highspeed; ++ mmc-hs200-1_8v; ++ no-sdio; ++ no-sd; ++ disable-wp; ++ non-removable; ++ num-slots = <1>; ++ status = "okay"; ++}; ++ ++&fiq_debugger { ++ status = "okay"; ++}; ++ ++&mac { ++ phy-supply = <&vcc_phy>; ++ assigned-clocks = <&cru SCLK_MAC>; ++ assigned-clock-parents = <&mac_clkin>; ++ clock_in_out = "input"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&rmii_pins &mac_refclk>; ++ snps,reset-gpio = <&gpio0 RK_PB2 GPIO_ACTIVE_LOW>; ++ snps,reset-active-low; ++ snps,reset-delays-us = <0 50000 50000>; ++ status = "disable"; ++}; ++ ++&io_domains { ++ status = "okay"; ++ ++ vccio0-supply = <&vcc_io>; ++ vccio1-supply = <&vcc_io>; ++ vccio2-supply = <&vcc_1v8>; ++ vccio3-supply = <&vccio_flash>; ++ vccio4-supply = <&vccio_sdio>; ++ vccio5-supply = <&vcc_io>; ++}; ++ ++&i2c1 { ++ clock-frequency = <400000>; ++ status = "okay"; ++ ++ is31fl3236: led-controller@3c { ++ compatible = "issi,is31fl3236"; ++ reg = <0x3c>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ reset-gpios = <&gpio0 RK_PB0 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ ++ led1: led@1 { ++ label = "led1"; ++ reg = <1>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <0>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led2: led@2 { ++ label = "led2"; ++ reg = <2>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <0>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led3: led@3 { ++ label = "led3"; ++ reg = <3>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led4: led@4 { ++ label = "led4"; ++ reg = <4>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <100>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led5: led@5 { ++ label = "led5"; ++ reg = <5>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <100>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led6: led@6 { ++ label = "led6"; ++ reg = <6>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led7: led@7 { ++ label = "led7"; ++ reg = <7>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <200>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led8: led@8 { ++ label = "led8"; ++ reg = <8>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <200>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led9: led@9 { ++ label = "led9"; ++ reg = <9>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led10: led@10 { ++ label = "led10"; ++ reg = <10>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <300>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led11: led@11 { ++ label = "led11"; ++ reg = <11>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <300>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led12: led@12 { ++ label = "led12"; ++ reg = <12>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led13: led@13 { ++ label = "led13"; ++ reg = <13>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <400>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led14: led@14 { ++ label = "led14"; ++ reg = <14>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <400>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led15: led@15 { ++ label = "led15"; ++ reg = <15>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led16: led@16 { ++ label = "led16"; ++ reg = <16>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <500>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led17: led@17 { ++ label = "led17"; ++ reg = <17>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <500>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led18: led@18 { ++ label = "led18"; ++ reg = <18>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led19: led@19 { ++ label = "led19"; ++ reg = <19>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <600>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led20: led@20 { ++ label = "led20"; ++ reg = <20>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <600>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led21: led@21 { ++ label = "led21"; ++ reg = <21>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led22: led@22 { ++ label = "led22"; ++ reg = <22>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <700>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led23: led@23 { ++ label = "led23"; ++ reg = <23>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <700>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led124: led@24 { ++ label = "led24"; ++ reg = <24>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led25: led@25 { ++ label = "led25"; ++ reg = <25>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <800>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led26: led@26 { ++ label = "led26"; ++ reg = <26>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <800>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led27: led@27 { ++ label = "led27"; ++ reg = <27>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led28: led@28 { ++ label = "led28"; ++ reg = <28>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <900>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led29: led@29 { ++ label = "led29"; ++ reg = <29>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <900>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led30: led@30 { ++ label = "led30"; ++ reg = <30>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led31: led@31 { ++ label = "led31"; ++ reg = <31>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <1000>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led32: led@32 { ++ label = "led32"; ++ reg = <32>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <1000>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led33: led@33 { ++ label = "led33"; ++ reg = <33>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led34: led@34 { ++ label = "led34"; ++ reg = <34>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <1100>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led35: led@35 { ++ label = "led35"; ++ reg = <35>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <1100>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led36: led@36 { ++ label = "led36"; ++ reg = <36>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ }; ++}; ++ ++&i2s_8ch_2 { ++ status = "okay"; ++}; ++ ++&nandc { ++ status = "okay"; ++}; ++ ++&rockchip_suspend { ++ rockchip,pwm-regulator-config = < ++ (0 ++ | RKPM_PWM_REGULATOR ++ ) ++ >; ++ ++ status = "okay"; ++}; ++ ++&rng { ++ status = "okay"; ++}; ++ ++&saradc { ++ status = "okay"; ++ vref-supply = <&vcc_1v8>; ++}; ++ ++&sdio { ++ bus-width = <4>; ++ cap-sd-highspeed; ++ no-sd; ++ no-mmc; ++ ignore-pm-notify; ++ keep-power-in-suspend; ++ non-removable; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&sdmmc { ++ bus-width = <4>; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ no-sdio; ++ no-mmc; ++ card-detect-delay = <300>; ++ vmmc-supply = <&vcc_sd>; ++ status = "disabled"; ++}; ++ ++&sfc { ++ status = "okay"; ++}; ++ ++&tsadc { ++ rockchip,hw-tshut-mode = <0>; /* tshut mode 0:CRU 1:GPIO */ ++ rockchip,hw-tshut-polarity = <1>; /* tshut polarity 0:LOW 1:HIGH */ ++ status = "okay"; ++}; ++ ++&pinctrl { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&rtc_32k>; ++ ++ buttons { ++ pwr_key: pwr-key { ++ rockchip,pins = <0 RK_PA6 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ usb { ++ usb_drv: usb-drv { ++ rockchip,pins = ++ <0 RK_PC5 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++&pwm0 { ++ status = "okay"; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm0_pin_pull_down>; ++}; ++ ++&u2phy { ++ status = "okay"; ++ ++ u2phy_host: host-port { ++ phy-supply = <&vbus_host>; ++ status = "okay"; ++ }; ++ ++ u2phy_otg: otg-port { ++ status = "okay"; ++ }; ++}; ++ ++&uart4 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart4_xfer &uart4_cts>; ++ status = "okay"; ++}; ++ ++&usb20_otg { ++ status = "okay"; ++}; ++ ++&usb_host0_ehci { ++ status = "okay"; ++}; ++ ++&usb_host0_ohci{ ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3308-evb-v11.dtsi b/arch/arm64/boot/dts/rockchip/rk3308-evb-v11.dtsi +new file mode 100755 +index 000000000000..7b692a810333 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3308-evb-v11.dtsi +@@ -0,0 +1,835 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++#include ++#include "rk3308.dtsi" ++ ++/ { ++ model = "Rockchip RK3308 EVB V11"; ++ compatible = "rockchip,rk3308-evb-v11", "rockchip,rk3308"; ++ ++ chosen { ++ bootargs = "earlycon=uart8250,mmio32,0xff0c0000 console=ttyFIQ0 root=PARTUUID=614e0000-0000 rootfstype=squashfs rootwait snd_aloop.index=7 snd_aloop.use_raw_jiffies=1"; ++ }; ++ ++ adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 1>; ++ io-channel-names = "buttons"; ++ poll-interval = <100>; ++ keyup-threshold-microvolt = <1800000>; ++ ++ esc-key { ++ linux,code = ; ++ label = "micmute"; ++ press-threshold-microvolt = <1130000>; ++ }; ++ ++ home-key { ++ linux,code = ; ++ label = "mode"; ++ press-threshold-microvolt = <901000>; ++ }; ++ ++ menu-key { ++ linux,code = ; ++ label = "play"; ++ press-threshold-microvolt = <624000>; ++ }; ++ ++ vol-down-key { ++ linux,code = ; ++ label = "volume down"; ++ press-threshold-microvolt = <300000>; ++ }; ++ ++ vol-up-key { ++ linux,code = ; ++ label = "volume up"; ++ press-threshold-microvolt = <18000>; ++ }; ++ }; ++ ++ dummy_codec: dummy-codec { ++ compatible = "rockchip,dummy-codec"; ++ #sound-dai-cells = <0>; ++ }; ++ ++ gpio-keys { ++ compatible = "gpio-keys"; ++ autorepeat; ++ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pwr_key>; ++ ++ power { ++ gpios = <&gpio0 RK_PA6 GPIO_ACTIVE_LOW>; ++ linux,code = ; ++ label = "GPIO Key Power"; ++ wakeup-source; ++ debounce-interval = <100>; ++ }; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_LOW>; ++ }; ++ ++ acodec_sound: acodec-sound { ++ compatible = "rockchip,multicodecs-card"; ++ rockchip,card-name = "rockchip,rk3308-acodec"; ++ rockchip,codec-hp-det; ++ rockchip,mclk-fs = <256>; ++ rockchip,cpu = <&i2s_8ch_2>; ++ rockchip,codec = <&acodec>; ++ }; ++ ++ bluetooth_sound: bluetooth-sound { ++ status = "disabled"; ++ compatible = "rockchip,multicodecs-card"; ++ rockchip,card-name = "rockchip,rk3308-pcm"; ++ rockchip,mclk-fs = <128>; ++ rockchip,cpu = <&i2s_2ch_0>; ++ rockchip,codec = <&dummy_codec>; ++ rockchip,format = "dsp_b"; ++ rockchip,bitclock-inversion = <0>; ++ rockchip,wait-card-locked = <0>; ++ }; ++ ++ spdif_rx_sound: spdif-rx-sound { ++ status = "disabled"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,name = "rockchip,spdif-rx-sound"; ++ simple-audio-card,cpu { ++ sound-dai = <&spdif_rx>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&dummy_codec>; ++ }; ++ }; ++ ++ spdif_tx_sound: spdif-tx-sound { ++ status = "disabled"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,name = "rockchip,spdif-tx-sound"; ++ simple-audio-card,cpu { ++ sound-dai = <&spdif_tx>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&dummy_codec>; ++ }; ++ }; ++ ++ tas5731_sound: tas5731-sound { ++ status = "disabled"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,tas5731"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s_8ch_1>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&tas5731>; ++ }; ++ }; ++ ++ vdd_core: vdd-core { ++ compatible = "pwm-regulator"; ++ pwms = <&pwm0 0 5000 1>; ++ regulator-name = "vdd_core"; ++ regulator-min-microvolt = <827000>; ++ regulator-max-microvolt = <1340000>; ++ regulator-init-microvolt = <1015000>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-settling-time-up-us = <250>; ++ status = "okay"; ++ }; ++ ++ vdd_log: vdd-log { ++ compatible = "regulator-fixed"; ++ regulator-name = "vdd_log"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1050000>; ++ regulator-max-microvolt = <1050000>; ++ }; ++ ++ vdd_1v0: vdd-1v0 { ++ compatible = "regulator-fixed"; ++ regulator-name = "vdd_1v0"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ }; ++ ++ vccio_sdio: vcc_1v8: vcc-1v8 { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_1v8"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ vin-supply = <&vcc_io>; ++ }; ++ ++ vccio_sd: vccio-sd { ++ compatible = "regulator-gpio"; ++ regulator-name = "vccio_sd"; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ gpios = <&gpio0 RK_PB7 GPIO_ACTIVE_HIGH>; ++ states = <1800000 0x0 ++ 3300000 0x1>; ++ }; ++ ++ vcc_sd: vcc-sd { ++ compatible = "regulator-fixed"; ++ gpio = <&gpio4 RK_PD6 GPIO_ACTIVE_LOW>; ++ regulator-name = "vcc_sd"; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ }; ++ ++ vcc_1v8_codec: vcc-1v8-codec { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_1v8_codec"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ vin-supply = <&vcc_io>; ++ }; ++ ++ vcc_ddr: vcc-ddr { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_ddr"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ }; ++ ++ vcc_3v3_codec: vcc_io: vcc-io { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_io"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ }; ++ ++ vccio_flash: vccio-flash { ++ compatible = "regulator-fixed"; ++ regulator-name = "vccio_flash"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ }; ++ ++ vcc_phy: vcc-phy-regulator { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_phy"; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ vbus_host: vbus-host-regulator { ++ compatible = "regulator-fixed"; ++ enable-active-high; ++ gpio = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&usb_drv>; ++ regulator-name = "vbus_host"; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ uart_rts_gpios = <&gpio4 RK_PA7 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart4_rts>; ++ pinctrl-1 = <&uart4_rts_gpio>; ++ BT,power_gpio = <&gpio4 RK_PB3 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio4 RK_PB4 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ rockchip,grf = <&grf>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_wake_host>; ++ wifi_chip_type = "ap6255"; ++ WIFI,host_wake_irq = <&gpio0 RK_PA0 GPIO_ACTIVE_LOW>; ++ status = "okay"; ++ }; ++}; ++ ++&acodec { ++ status = "okay"; ++ ++ rockchip,no-deep-low-power; ++ rockchip,loopback-grp = <0>; ++ hp-ctl-gpios = <&gpio0 RK_PA1 GPIO_ACTIVE_HIGH>; ++ spk-ctl-gpios = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>; ++}; ++ ++&cpu0 { ++ cpu-supply = <&vdd_core>; ++}; ++ ++&cpu0_opp_table { ++ opp-1200000000 { ++ status = "okay"; ++ }; ++ opp-1296000000 { ++ status = "okay"; ++ }; ++}; ++ ++&dmc { ++ center-supply = <&vdd_log>; ++ status = "okay"; ++}; ++ ++&emmc { ++ bus-width = <8>; ++ cap-mmc-highspeed; ++ mmc-hs200-1_8v; ++ no-sdio; ++ no-sd; ++ disable-wp; ++ non-removable; ++ num-slots = <1>; ++ status = "disabled"; ++}; ++ ++&fiq_debugger { ++ status = "okay"; ++}; ++ ++&mac { ++ phy-supply = <&vcc_phy>; ++ assigned-clocks = <&cru SCLK_MAC>; ++ assigned-clock-parents = <&mac_clkin>; ++ clock_in_out = "input"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&rmii_pins &mac_refclk>; ++ snps,reset-gpio = <&gpio0 RK_PB2 GPIO_ACTIVE_LOW>; ++ snps,reset-active-low; ++ snps,reset-delays-us = <0 50000 50000>; ++ status = "disable"; ++}; ++ ++&io_domains { ++ status = "okay"; ++ ++ vccio0-supply = <&vcc_io>; ++ vccio1-supply = <&vcc_io>; ++ vccio2-supply = <&vcc_1v8>; ++ vccio3-supply = <&vccio_flash>; ++ vccio4-supply = <&vccio_sdio>; ++ vccio5-supply = <&vccio_sd>; ++}; ++ ++&i2c1 { ++ clock-frequency = <400000>; ++ status = "okay"; ++ ++ tas5731: tas5731@1a { ++ #sound-dai-cells = <0>; ++ compatible = "ti,tas5731"; ++ reg = <0x1a>; ++ clocks = <&cru SCLK_I2S1_8CH_TX_OUT>; ++ clock-names = "mclk"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s_8ch_1_m0_mclk>; ++ pdn-gpios = <&gpio0 RK_PA5 GPIO_ACTIVE_LOW>; ++ reset-gpios = <&gpio1 RK_PA1 GPIO_ACTIVE_LOW>; ++ }; ++ ++ is31fl3236: led-controller@3c { ++ compatible = "issi,is31fl3236"; ++ reg = <0x3c>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ reset-gpios = <&gpio0 RK_PB0 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ ++ led1: led@1 { ++ label = "led1"; ++ reg = <1>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <0>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led2: led@2 { ++ label = "led2"; ++ reg = <2>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <0>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led3: led@3 { ++ label = "led3"; ++ reg = <3>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led4: led@4 { ++ label = "led4"; ++ reg = <4>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <100>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led5: led@5 { ++ label = "led5"; ++ reg = <5>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <100>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led6: led@6 { ++ label = "led6"; ++ reg = <6>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led7: led@7 { ++ label = "led7"; ++ reg = <7>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <200>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led8: led@8 { ++ label = "led8"; ++ reg = <8>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <200>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led9: led@9 { ++ label = "led9"; ++ reg = <9>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led10: led@10 { ++ label = "led10"; ++ reg = <10>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <300>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led11: led@11 { ++ label = "led11"; ++ reg = <11>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <300>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led12: led@12 { ++ label = "led12"; ++ reg = <12>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led13: led@13 { ++ label = "led13"; ++ reg = <13>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <400>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led14: led@14 { ++ label = "led14"; ++ reg = <14>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <400>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led15: led@15 { ++ label = "led15"; ++ reg = <15>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led16: led@16 { ++ label = "led16"; ++ reg = <16>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <500>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led17: led@17 { ++ label = "led17"; ++ reg = <17>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <500>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led18: led@18 { ++ label = "led18"; ++ reg = <18>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led19: led@19 { ++ label = "led19"; ++ reg = <19>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <600>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led20: led@20 { ++ label = "led20"; ++ reg = <20>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <600>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led21: led@21 { ++ label = "led21"; ++ reg = <21>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led22: led@22 { ++ label = "led22"; ++ reg = <22>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <700>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led23: led@23 { ++ label = "led23"; ++ reg = <23>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <700>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led124: led@24 { ++ label = "led24"; ++ reg = <24>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led25: led@25 { ++ label = "led25"; ++ reg = <25>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <800>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led26: led@26 { ++ label = "led26"; ++ reg = <26>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <800>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led27: led@27 { ++ label = "led27"; ++ reg = <27>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led28: led@28 { ++ label = "led28"; ++ reg = <28>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <900>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led29: led@29 { ++ label = "led29"; ++ reg = <29>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <900>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led30: led@30 { ++ label = "led30"; ++ reg = <30>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led31: led@31 { ++ label = "led31"; ++ reg = <31>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <1000>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led32: led@32 { ++ label = "led32"; ++ reg = <32>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <1000>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led33: led@33 { ++ label = "led33"; ++ reg = <33>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led34: led@34 { ++ label = "led34"; ++ reg = <34>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <1100>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led35: led@35 { ++ label = "led35"; ++ reg = <35>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <1100>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led36: led@36 { ++ label = "led36"; ++ reg = <36>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ }; ++}; ++ ++&i2s_8ch_1 { ++ status = "disabled"; ++ #sound-dai-cells = <0>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s_8ch_1_m0_sclktx ++ &i2s_8ch_1_m0_lrcktx ++ &i2s_8ch_1_m0_sdo0 ++ &i2s_8ch_1_m0_mclk>; ++}; ++ ++&i2s_8ch_2 { ++ status = "okay"; ++}; ++ ++&nandc { ++ status = "okay"; ++}; ++ ++&rockchip_suspend { ++ rockchip,pwm-regulator-config = < ++ (0 ++ | RKPM_PWM_REGULATOR ++ ) ++ >; ++ ++ status = "okay"; ++}; ++ ++&rng { ++ status = "okay"; ++}; ++ ++&saradc { ++ status = "okay"; ++ vref-supply = <&vcc_1v8>; ++}; ++ ++&sdio { ++ bus-width = <4>; ++ cap-sd-highspeed; ++ no-sd; ++ no-mmc; ++ ignore-pm-notify; ++ keep-power-in-suspend; ++ non-removable; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&sdmmc { ++ bus-width = <4>; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ no-sdio; ++ no-mmc; ++ card-detect-delay = <300>; ++ sd-uhs-sdr25; ++ sd-uhs-sdr50; ++ sd-uhs-sdr104; ++ vmmc-supply = <&vcc_sd>; ++ vqmmc-supply = <&vccio_sd>; ++ status = "disabled"; ++}; ++ ++&tsadc { ++ rockchip,hw-tshut-mode = <0>; /* tshut mode 0:CRU 1:GPIO */ ++ rockchip,hw-tshut-polarity = <1>; /* tshut polarity 0:LOW 1:HIGH */ ++ status = "okay"; ++}; ++ ++&pinctrl { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&rtc_32k>; ++ ++ buttons { ++ pwr_key: pwr-key { ++ rockchip,pins = <0 RK_PA6 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ usb { ++ usb_drv: usb-drv { ++ rockchip,pins = <0 RK_PC5 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ wireless-wlan { ++ wifi_wake_host: wifi-wake-host { ++ rockchip,pins = <0 RK_PA0 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++}; ++ ++&pwm0 { ++ status = "okay"; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm0_pin_pull_down>; ++}; ++ ++&u2phy { ++ status = "okay"; ++ ++ u2phy_host: host-port { ++ phy-supply = <&vbus_host>; ++ status = "okay"; ++ }; ++ ++ u2phy_otg: otg-port { ++ status = "okay"; ++ }; ++}; ++ ++&uart4 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart4_xfer &uart4_cts>; ++ status = "okay"; ++}; ++ ++&usb20_otg { ++ status = "okay"; ++}; ++ ++&usb_host0_ehci { ++ status = "okay"; ++}; ++ ++&usb_host0_ohci{ ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3308-fpga.dts b/arch/arm64/boot/dts/rockchip/rk3308-fpga.dts +new file mode 100755 +index 000000000000..225aca323f4d +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3308-fpga.dts +@@ -0,0 +1,51 @@ ++/* ++ * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++ */ ++ ++/dts-v1/; ++ ++#include "rk3308.dtsi" ++ ++/ { ++ model = "Rockchip RK3308 FPGA Platform"; ++ compatible = "rockchip,rk3308-fpga", "rockchip,rk3308"; ++ ++ chosen { ++ bootargs = "earlycon=uart8250,mmio32,0xff0b0000 console=ttyFIQ0 init=/init initrd=0x9000000,0x18bfc0"; ++ }; ++ ++ memory@200000 { ++ device_type = "memory"; ++ reg = <0x0 0x00200000 0x0 0x0FE00000>; ++ }; ++}; ++ ++&fiq_debugger { ++ rockchip,serial-id = <1>; ++ rockchip,irq-mode-enable = <1>; ++ status = "ok"; ++}; ++ ++&cpu1 { ++ /delete-property/enable-method; ++}; ++ ++&cpu2 { ++ /delete-property/enable-method; ++}; ++ ++&cpu3 { ++ /delete-property/enable-method; ++}; ++ ++&emmc { ++ cap-mmc-highspeed; ++ mmc-hs200-1_8v; ++ no-sdio; ++ no-sd; ++ non-removable; ++ num-slots = <1>; ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3308-roc-cc.dts b/arch/arm64/boot/dts/rockchip/rk3308-roc-cc.dts +index bce6f8b7db43..7a96be10eaf0 100644 +--- a/arch/arm64/boot/dts/rockchip/rk3308-roc-cc.dts ++++ b/arch/arm64/boot/dts/rockchip/rk3308-roc-cc.dts +@@ -78,8 +78,8 @@ vcc_sdmmc: vcc-sdmmc { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + gpios = <&gpio0 RK_PA7 GPIO_ACTIVE_HIGH>; +- states = <1800000 0x0>, +- <3300000 0x1>; ++ states = <1800000 0x0 ++ 3300000 0x1>; + vin-supply = <&vcc5v0_sys>; + }; + +diff --git a/arch/arm64/boot/dts/rockchip/rk3308-voice-module-board-v10.dts b/arch/arm64/boot/dts/rockchip/rk3308-voice-module-board-v10.dts +new file mode 100755 +index 000000000000..2586d9905072 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3308-voice-module-board-v10.dts +@@ -0,0 +1,19 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++ ++#include "arm/rk3308-voice-module-board-v10-aarch32.dts" ++ ++/ { ++ model = "Rockchip RK3308 Voice Module Board V10"; ++ compatible = "rockchip,rk3308-voice-module-board-v10", "rockchip,rk3308"; ++}; ++ ++&ramoops { ++ reg = <0x0 0x110000 0x0 0xf0000>; ++ record-size = <0x30000>; ++ console-size = <0xc0000>; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3308.dtsi b/arch/arm64/boot/dts/rockchip/rk3308.dtsi +index 2560b98771ca..cc86c0880cec 100644 +--- a/arch/arm64/boot/dts/rockchip/rk3308.dtsi ++++ b/arch/arm64/boot/dts/rockchip/rk3308.dtsi +@@ -10,6 +10,7 @@ + #include + #include + #include ++#include + #include + + / { +@@ -147,6 +148,21 @@ psci { + method = "smc"; + }; + ++ rockchip_suspend: rockchip-suspend { ++ compatible = "rockchip,pm-rk3308"; ++ status = "disabled"; ++ rockchip,sleep-mode-config = < ++ (0 ++ | RKPM_PMU_HW_PLLS_PD ++ ) ++ >; ++ rockchip,wakeup-config = < ++ (0 ++ | RKPM_GPIO0_WAKEUP_EN ++ ) ++ >; ++ }; ++ + timer { + compatible = "arm,armv8-timer"; + interrupts = , +@@ -366,7 +382,7 @@ pwm8: pwm@ff160000 { + reg = <0x0 0xff160000 0x0 0x10>; + clocks = <&cru SCLK_PWM2>, <&cru PCLK_PWM2>; + clock-names = "pwm", "pclk"; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm8_pin>; + #pwm-cells = <3>; + status = "disabled"; +@@ -377,7 +393,7 @@ pwm9: pwm@ff160010 { + reg = <0x0 0xff160010 0x0 0x10>; + clocks = <&cru SCLK_PWM2>, <&cru PCLK_PWM2>; + clock-names = "pwm", "pclk"; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm9_pin>; + #pwm-cells = <3>; + status = "disabled"; +@@ -388,7 +404,7 @@ pwm10: pwm@ff160020 { + reg = <0x0 0xff160020 0x0 0x10>; + clocks = <&cru SCLK_PWM2>, <&cru PCLK_PWM2>; + clock-names = "pwm", "pclk"; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm10_pin>; + #pwm-cells = <3>; + status = "disabled"; +@@ -399,7 +415,7 @@ pwm11: pwm@ff160030 { + reg = <0x0 0xff160030 0x0 0x10>; + clocks = <&cru SCLK_PWM2>, <&cru PCLK_PWM2>; + clock-names = "pwm", "pclk"; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm11_pin>; + #pwm-cells = <3>; + status = "disabled"; +@@ -410,7 +426,7 @@ pwm4: pwm@ff170000 { + reg = <0x0 0xff170000 0x0 0x10>; + clocks = <&cru SCLK_PWM1>, <&cru PCLK_PWM1>; + clock-names = "pwm", "pclk"; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm4_pin>; + #pwm-cells = <3>; + status = "disabled"; +@@ -421,7 +437,7 @@ pwm5: pwm@ff170010 { + reg = <0x0 0xff170010 0x0 0x10>; + clocks = <&cru SCLK_PWM1>, <&cru PCLK_PWM1>; + clock-names = "pwm", "pclk"; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm5_pin>; + #pwm-cells = <3>; + status = "disabled"; +@@ -432,7 +448,7 @@ pwm6: pwm@ff170020 { + reg = <0x0 0xff170020 0x0 0x10>; + clocks = <&cru SCLK_PWM1>, <&cru PCLK_PWM1>; + clock-names = "pwm", "pclk"; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm6_pin>; + #pwm-cells = <3>; + status = "disabled"; +@@ -443,7 +459,7 @@ pwm7: pwm@ff170030 { + reg = <0x0 0xff170030 0x0 0x10>; + clocks = <&cru SCLK_PWM1>, <&cru PCLK_PWM1>; + clock-names = "pwm", "pclk"; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm7_pin>; + #pwm-cells = <3>; + status = "disabled"; +@@ -454,7 +470,7 @@ pwm0: pwm@ff180000 { + reg = <0x0 0xff180000 0x0 0x10>; + clocks = <&cru SCLK_PWM0>, <&cru PCLK_PWM0>; + clock-names = "pwm", "pclk"; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm0_pin>; + #pwm-cells = <3>; + status = "disabled"; +@@ -465,7 +481,7 @@ pwm1: pwm@ff180010 { + reg = <0x0 0xff180010 0x0 0x10>; + clocks = <&cru SCLK_PWM0>, <&cru PCLK_PWM0>; + clock-names = "pwm", "pclk"; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm1_pin>; + #pwm-cells = <3>; + status = "disabled"; +@@ -476,7 +492,7 @@ pwm2: pwm@ff180020 { + reg = <0x0 0xff180020 0x0 0x10>; + clocks = <&cru SCLK_PWM0>, <&cru PCLK_PWM0>; + clock-names = "pwm", "pclk"; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm2_pin>; + #pwm-cells = <3>; + status = "disabled"; +@@ -487,7 +503,7 @@ pwm3: pwm@ff180030 { + reg = <0x0 0xff180030 0x0 0x10>; + clocks = <&cru SCLK_PWM0>, <&cru PCLK_PWM0>; + clock-names = "pwm", "pclk"; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm3_pin>; + #pwm-cells = <3>; + status = "disabled"; +diff --git a/arch/arm64/boot/dts/rockchip/rk3308b-evb-amic-v10.dts b/arch/arm64/boot/dts/rockchip/rk3308b-evb-amic-v10.dts +new file mode 100755 +index 000000000000..d5e2d9ebd4aa +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3308b-evb-amic-v10.dts +@@ -0,0 +1,62 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++ ++#include "rk3308b-evb-v10.dtsi" ++ ++/ { ++ model = "Rockchip RK3308b evb analog mic v10 board"; ++ compatible = "rockchip,rk3308b-evb-amic-v10", "rockchip,rk3308"; ++ ++ vad_acodec_sound: vad-acodec-sound { ++ status = "okay"; ++ compatible = "rockchip,multicodecs-card"; ++ rockchip,card-name = "rockchip,rk3308-vad"; ++ rockchip,codec-hp-det; ++ rockchip,mclk-fs = <256>; ++ rockchip,cpu = <&i2s_8ch_2>; ++ rockchip,codec = <&acodec>, <&vad>; ++ }; ++}; ++ ++&acodec { ++ rockchip,micbias1; ++ rockchip,micbias2; ++ rockchip,en-always-grps = <1 2 3>; ++ rockchip,adc-grps-route = <1 2 3 0>; ++}; ++ ++&acodec_sound { ++ status = "disabled"; ++}; ++ ++&bluetooth_sound { ++ status = "okay"; ++}; ++ ++&i2s_8ch_0 { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++ rockchip,clk-trcm = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s_8ch_0_sclktx ++ &i2s_8ch_0_lrcktx ++ &i2s_8ch_0_sdi0 ++ &i2s_8ch_0_sdo2>; ++}; ++ ++&is31fl3236 { ++ reg = <0x3f>; ++}; ++ ++&vad { ++ status = "okay"; ++ rockchip,audio-src = <&i2s_8ch_2>; ++ rockchip,det-channel = <0>; ++ rockchip,buffer-time-ms = <200>; ++ rockchip,mode = <1>; ++ #sound-dai-cells = <0>; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3308b-evb-ext-v10.dtsi b/arch/arm64/boot/dts/rockchip/rk3308b-evb-ext-v10.dtsi +new file mode 100755 +index 000000000000..e2414b67f0d1 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3308b-evb-ext-v10.dtsi +@@ -0,0 +1,124 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ */ ++ ++/ { ++ backlight: backlight { ++ status = "okay"; ++ compatible = "pwm-backlight"; ++ pwms = <&pwm1 0 25000 0>; ++ brightness-levels = < ++ 0 1 2 3 4 5 6 7 ++ 8 9 10 11 12 13 14 15 ++ 16 17 18 19 20 21 22 23 ++ 24 25 26 27 28 29 30 31 ++ 32 33 34 35 36 37 38 39 ++ 40 41 42 43 44 45 46 47 ++ 48 49 50 51 52 53 54 55 ++ 56 57 58 59 60 61 62 63 ++ 64 65 66 67 68 69 70 71 ++ 72 73 74 75 76 77 78 79 ++ 80 81 82 83 84 85 86 87 ++ 88 89 90 91 92 93 94 95 ++ 96 97 98 99 100 101 102 103 ++ 104 105 106 107 108 109 110 111 ++ 112 113 114 115 116 117 118 119 ++ 120 121 122 123 124 125 126 127 ++ 128 129 130 131 132 133 134 135 ++ 136 137 138 139 140 141 142 143 ++ 144 145 146 147 148 149 150 151 ++ 152 153 154 155 156 157 158 159 ++ 160 161 162 163 164 165 166 167 ++ 168 169 170 171 172 173 174 175 ++ 176 177 178 179 180 181 182 183 ++ 184 185 186 187 188 189 190 191 ++ 192 193 194 195 196 197 198 199 ++ 200 201 202 203 204 205 206 207 ++ 208 209 210 211 212 213 214 215 ++ 216 217 218 219 220 221 222 223 ++ 224 225 226 227 228 229 230 231 ++ 232 233 234 235 236 237 238 239 ++ 240 241 242 243 244 245 246 247 ++ 248 249 250 251 252 253 254 255>; ++ default-brightness-level = <200>; ++ }; ++ ++ panel: panel { ++ compatible = "simple-panel"; ++ bus-format = ; ++ backlight = <&backlight>; ++ enable-gpios = <&gpio0 RK_PA4 GPIO_ACTIVE_LOW>; ++ enable-delay-ms = <20>; ++ reset-gpios = <&gpio0 RK_PC4 GPIO_ACTIVE_LOW>; ++ reset-delay-ms = <10>; ++ prepare-delay-ms = <20>; ++ unprepare-delay-ms = <20>; ++ disable-delay-ms = <20>; ++ width-mm = <95>; ++ height-mm = <54>; ++ status = "okay"; ++ ++ display-timings { ++ native-mode = <&stt0430_enl2c_timing>; ++ ++ stt0430_enl2c_timing: timing0 { ++ clock-frequency = <12000000>; ++ hactive = <480>; ++ vactive = <272>; ++ hback-porch = <60>; ++ hfront-porch = <20>; ++ vback-porch = <28>; ++ vfront-porch = <20>; ++ hsync-len = <20>; ++ vsync-len = <20>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ port { ++ panel_in_rgb: endpoint { ++ remote-endpoint = <&rgb_out_panel>; ++ }; ++ }; ++ }; ++}; ++ ++&display_subsystem { ++ status = "okay"; ++}; ++ ++&pwm1 { ++ status = "okay"; ++}; ++ ++&rgb { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&lcdc_ctl &lcdc_rgb888_m1>; ++ ++ ports { ++ rgb_out: port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ rgb_out_panel: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&panel_in_rgb>; ++ }; ++ }; ++ }; ++}; ++ ++&route_rgb { ++ status = "okay"; ++}; ++ ++&vop { ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3308b-evb-v10.dtsi b/arch/arm64/boot/dts/rockchip/rk3308b-evb-v10.dtsi +new file mode 100755 +index 000000000000..902ae3544235 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3308b-evb-v10.dtsi +@@ -0,0 +1,784 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++#include ++#include "rk3308.dtsi" ++ ++/ { ++ model = "Rockchip RK3308B EVB V10"; ++ compatible = "rockchip,rk3308b-evb-v10", "rockchip,rk3308"; ++ ++ chosen { ++ bootargs = "earlycon=uart8250,mmio32,0xff0e0000 console=ttyFIQ0 root=PARTUUID=614e0000-0000 rootfstype=squashfs rootwait snd_aloop.index=7 snd_aloop.use_raw_jiffies=1"; ++ }; ++ ++ acodec_sound: acodec-sound { ++ compatible = "rockchip,multicodecs-card"; ++ rockchip,card-name = "rockchip,rk3308-acodec"; ++ rockchip,codec-hp-det; ++ rockchip,mclk-fs = <256>; ++ rockchip,cpu = <&i2s_8ch_2>; ++ rockchip,codec = <&acodec>; ++ }; ++ ++ adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 1>; ++ io-channel-names = "buttons"; ++ poll-interval = <100>; ++ keyup-threshold-microvolt = <1800000>; ++ ++ esc-key { ++ linux,code = ; ++ label = "micmute"; ++ press-threshold-microvolt = <1130000>; ++ }; ++ ++ home-key { ++ linux,code = ; ++ label = "mode"; ++ press-threshold-microvolt = <901000>; ++ }; ++ ++ menu-key { ++ linux,code = ; ++ label = "play"; ++ press-threshold-microvolt = <624000>; ++ }; ++ ++ vol-down-key { ++ linux,code = ; ++ label = "volume down"; ++ press-threshold-microvolt = <300000>; ++ }; ++ ++ vol-up-key { ++ linux,code = ; ++ label = "volume up"; ++ press-threshold-microvolt = <18000>; ++ }; ++ }; ++ ++ bluetooth_sound: bluetooth-sound { ++ status = "disabled"; ++ compatible = "rockchip,multicodecs-card"; ++ rockchip,card-name = "rockchip,rk3308-pcm"; ++ rockchip,mclk-fs = <128>; ++ rockchip,cpu = <&i2s_8ch_0>; ++ rockchip,codec = <&dummy_codec>; ++ rockchip,format = "dsp_b"; ++ rockchip,bitclock-inversion = <0>; ++ rockchip,wait-card-locked = <0>; ++ }; ++ ++ dummy_codec: dummy-codec { ++ compatible = "rockchip,dummy-codec"; ++ #sound-dai-cells = <0>; ++ }; ++ ++ gpio-keys { ++ compatible = "gpio-keys"; ++ autorepeat; ++ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pwr_key>; ++ ++ power { ++ gpios = <&gpio0 RK_PA6 GPIO_ACTIVE_LOW>; ++ linux,code = ; ++ label = "GPIO Key Power"; ++ wakeup-source; ++ debounce-interval = <100>; ++ }; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio4 RK_PD6 GPIO_ACTIVE_LOW>; ++ }; ++ ++ spdif_rx_sound: spdif-rx-sound { ++ status = "disabled"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,name = "rockchip,spdif-rx-sound"; ++ simple-audio-card,cpu { ++ sound-dai = <&spdif_rx>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&dummy_codec>; ++ }; ++ }; ++ ++ spdif_tx_sound: spdif-tx-sound { ++ status = "disabled"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,name = "rockchip,spdif-tx-sound"; ++ simple-audio-card,cpu { ++ sound-dai = <&spdif_tx>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&dummy_codec>; ++ }; ++ }; ++ ++ vdd_core: vdd-core { ++ compatible = "pwm-regulator"; ++ pwms = <&pwm0 0 5000 1>; ++ regulator-name = "vdd_core"; ++ regulator-min-microvolt = <827000>; ++ regulator-max-microvolt = <1340000>; ++ regulator-init-microvolt = <1015000>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-settling-time-up-us = <250>; ++ status = "okay"; ++ }; ++ ++ vdd_log: vdd-log { ++ compatible = "regulator-fixed"; ++ regulator-name = "vdd_log"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1050000>; ++ regulator-max-microvolt = <1050000>; ++ }; ++ ++ vdd_1v0: vdd-1v0 { ++ compatible = "regulator-fixed"; ++ regulator-name = "vdd_1v0"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ }; ++ ++ vccio_sdio: vcc_1v8: vcc-1v8 { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_1v8"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ vin-supply = <&vcc_io>; ++ }; ++ ++ vcc_1v8_codec: vcc-1v8-codec { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_1v8_codec"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ vin-supply = <&vcc_io>; ++ }; ++ ++ vcc_ddr: vcc-ddr { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_ddr"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ }; ++ ++ vcc_3v3_codec: vcc_io: vcc-io { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_io"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ }; ++ ++ vccio_flash: vccio-flash { ++ compatible = "regulator-fixed"; ++ regulator-name = "vccio_flash"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ }; ++ ++ vcc_phy: vcc-phy-regulator { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_phy"; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ vbus_host: vbus-host-regulator { ++ compatible = "regulator-fixed"; ++ enable-active-high; ++ gpio = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&usb_drv>; ++ regulator-name = "vbus_host"; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ uart_rts_gpios = <&gpio2 RK_PA3 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart0_rts>; ++ pinctrl-1 = <&uart0_rts_gpio>; ++ BT,power_gpio = <&gpio2 RK_PA6 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio2 RK_PB0 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ rockchip,grf = <&grf>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_wake_host>; ++ wifi_chip_type = "ap6255"; ++ WIFI,host_wake_irq = <&gpio2 RK_PB4 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++}; ++ ++&acodec { ++ status = "okay"; ++ ++ rockchip,no-deep-low-power; ++ hp-ctl-gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_HIGH>; ++ spk-ctl-gpios = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>; ++}; ++ ++&cpu0 { ++ cpu-supply = <&vdd_core>; ++}; ++ ++&cpu0_opp_table { ++ opp-1200000000 { ++ status = "okay"; ++ }; ++ opp-1296000000 { ++ status = "okay"; ++ }; ++}; ++ ++&dmc { ++ center-supply = <&vdd_log>; ++ status = "okay"; ++}; ++ ++&emmc { ++ bus-width = <8>; ++ cap-mmc-highspeed; ++ mmc-hs200-1_8v; ++ no-sdio; ++ no-sd; ++ disable-wp; ++ non-removable; ++ num-slots = <1>; ++ status = "disabled"; ++}; ++ ++&fiq_debugger { ++ rockchip,serial-id = <4>; ++ status = "okay"; ++}; ++ ++&io_domains { ++ status = "okay"; ++ ++ vccio0-supply = <&vcc_io>; ++ vccio1-supply = <&vcc_io>; ++ vccio2-supply = <&vccio_sdio>; ++ vccio3-supply = <&vccio_flash>; ++ vccio4-supply = <&vcc_io>; ++ vccio5-supply = <&vccio_sdio>; ++}; ++ ++&i2c1 { ++ clock-frequency = <400000>; ++ status = "okay"; ++ ++ is31fl3236: led-controller@3c { ++ compatible = "issi,is31fl3236"; ++ reg = <0x3c>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ reset-gpios = <&gpio0 RK_PB0 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ ++ led1: led@1 { ++ label = "led1"; ++ reg = <1>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <0>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led2: led@2 { ++ label = "led2"; ++ reg = <2>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <0>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led3: led@3 { ++ label = "led3"; ++ reg = <3>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led4: led@4 { ++ label = "led4"; ++ reg = <4>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <100>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led5: led@5 { ++ label = "led5"; ++ reg = <5>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <100>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led6: led@6 { ++ label = "led6"; ++ reg = <6>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led7: led@7 { ++ label = "led7"; ++ reg = <7>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <200>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led8: led@8 { ++ label = "led8"; ++ reg = <8>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <200>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led9: led@9 { ++ label = "led9"; ++ reg = <9>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led10: led@10 { ++ label = "led10"; ++ reg = <10>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <300>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led11: led@11 { ++ label = "led11"; ++ reg = <11>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <300>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led12: led@12 { ++ label = "led12"; ++ reg = <12>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led13: led@13 { ++ label = "led13"; ++ reg = <13>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <400>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led14: led@14 { ++ label = "led14"; ++ reg = <14>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <400>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led15: led@15 { ++ label = "led15"; ++ reg = <15>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led16: led@16 { ++ label = "led16"; ++ reg = <16>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <500>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led17: led@17 { ++ label = "led17"; ++ reg = <17>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <500>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led18: led@18 { ++ label = "led18"; ++ reg = <18>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led19: led@19 { ++ label = "led19"; ++ reg = <19>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <600>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led20: led@20 { ++ label = "led20"; ++ reg = <20>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <600>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led21: led@21 { ++ label = "led21"; ++ reg = <21>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led22: led@22 { ++ label = "led22"; ++ reg = <22>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <700>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led23: led@23 { ++ label = "led23"; ++ reg = <23>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <700>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led124: led@24 { ++ label = "led24"; ++ reg = <24>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led25: led@25 { ++ label = "led25"; ++ reg = <25>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <800>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led26: led@26 { ++ label = "led26"; ++ reg = <26>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <800>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led27: led@27 { ++ label = "led27"; ++ reg = <27>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led28: led@28 { ++ label = "led28"; ++ reg = <28>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <900>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led29: led@29 { ++ label = "led29"; ++ reg = <29>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <900>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led30: led@30 { ++ label = "led30"; ++ reg = <30>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led31: led@31 { ++ label = "led31"; ++ reg = <31>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <1000>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led32: led@32 { ++ label = "led32"; ++ reg = <32>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <1000>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led33: led@33 { ++ label = "led33"; ++ reg = <33>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led34: led@34 { ++ label = "led34"; ++ reg = <34>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <1100>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led35: led@35 { ++ label = "led35"; ++ reg = <35>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <1100>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led36: led@36 { ++ label = "led36"; ++ reg = <36>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ }; ++}; ++ ++&i2s_8ch_1 { ++ status = "disabled"; ++ #sound-dai-cells = <0>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s_8ch_1_m0_sclktx ++ &i2s_8ch_1_m0_lrcktx ++ &i2s_8ch_1_m0_sdo0 ++ &i2s_8ch_1_m0_mclk>; ++}; ++ ++&i2s_8ch_2 { ++ status = "okay"; ++}; ++ ++&mac { ++ phy-supply = <&vcc_phy>; ++ assigned-clocks = <&cru SCLK_MAC>; ++ assigned-clock-parents = <&mac_clkin>; ++ clock_in_out = "input"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&rmiim1_pins &macm1_refclk>; ++ snps,reset-gpio = <&gpio4 RK_PC0 GPIO_ACTIVE_LOW>; ++ snps,reset-active-low; ++ snps,reset-delays-us = <0 50000 50000>; ++ status = "disable"; ++}; ++ ++&nandc { ++ status = "okay"; ++}; ++ ++&pinctrl { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&rtc_32k>; ++ ++ buttons { ++ pwr_key: pwr-key { ++ rockchip,pins = <0 RK_PA6 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ usb { ++ usb_drv: usb-drv { ++ rockchip,pins = <0 RK_PC5 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <4 RK_PD6 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ wireless-wlan { ++ wifi_wake_host: wifi-wake-host { ++ rockchip,pins = <2 RK_PB4 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++}; ++ ++&pwm0 { ++ status = "okay"; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm0_pin_pull_down>; ++}; ++ ++&rng { ++ status = "okay"; ++}; ++ ++&rockchip_suspend { ++ rockchip,pwm-regulator-config = < ++ (0 ++ | RKPM_PWM_REGULATOR ++ ) ++ >; ++ ++ status = "okay"; ++}; ++ ++&saradc { ++ status = "okay"; ++ vref-supply = <&vcc_1v8>; ++}; ++ ++&sdmmc { ++ bus-width = <4>; ++ cap-sd-highspeed; ++ no-sd; ++ no-mmc; ++ ignore-pm-notify; ++ keep-power-in-suspend; ++ non-removable; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&sfc { ++ status = "okay"; ++}; ++ ++&tsadc { ++ rockchip,hw-tshut-mode = <0>; /* tshut mode 0:CRU 1:GPIO */ ++ rockchip,hw-tshut-polarity = <1>; /* tshut polarity 0:LOW 1:HIGH */ ++ pinctrl-names = "init", "default", "sleep"; ++ pinctrl-0 = <&tsadc_otp_gpio>; ++ pinctrl-1 = <&tsadc_otp_gpio>; ++ pinctrl-2 = <&tsadc_otp_gpio>; ++ status = "okay"; ++}; ++ ++&u2phy { ++ status = "okay"; ++ ++ u2phy_host: host-port { ++ phy-supply = <&vbus_host>; ++ status = "okay"; ++ }; ++ ++ u2phy_otg: otg-port { ++ status = "okay"; ++ }; ++}; ++ ++&uart0 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart0_xfer &uart0_cts>; ++ status = "okay"; ++}; ++ ++&usb20_otg { ++ status = "okay"; ++}; ++ ++&usb_host0_ehci { ++ status = "okay"; ++}; ++ ++&usb_host0_ohci{ ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3308k.dtsi b/arch/arm64/boot/dts/rockchip/rk3308k.dtsi +new file mode 100755 +index 000000000000..ffd53fd158c5 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3308k.dtsi +@@ -0,0 +1,46 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++#include ++#include "rk3308.dtsi" ++ ++/ { ++ uboot-wide-temperature { ++ status = "okay"; ++ compatible = "rockchip,uboot-wide-temperature"; ++ }; ++}; ++ ++&cpu0_opp_table { ++ rockchip,high-temp = <55000>; ++ rockchip,high-temp-max-volt = <1125000>; ++}; ++ ++&rockchip_suspend { ++ rockchip,sleep-mode-config = < ++ (0 ++ | RKPM_PMU_HW_PLLS_PD ++ | RKPM_PWM_VOLTAGE_DEFAULT ++ ) ++ >; ++}; ++ ++&thermal_zones { ++ soc-thermal { ++ sustainable-power = <422>; ++ k_pu = <6>; ++ k_po = <1024>; ++ k_i = <0>; ++ ++ trips { ++ trip-point-0 { ++ temperature = <55000>; ++ }; ++ trip-point-1 { ++ temperature = <90000>; ++ }; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3326-863-cif-sensor.dtsi b/arch/arm64/boot/dts/rockchip/rk3326-863-cif-sensor.dtsi +new file mode 100755 +index 000000000000..c01f4d014a1b +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3326-863-cif-sensor.dtsi +@@ -0,0 +1,54 @@ ++/* ++ * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++ */ ++ ++#include "../../../../../drivers/soc/rockchip/rk_camera_sensor_info.h" ++/{ ++ cif_sensor: cif_sensor { ++ compatible = "rockchip,sensor"; ++ status = "disabled"; ++ ++ gc2145_b { ++ is_front = <0>; ++ powerdown-gpios = <&gpio2 RK_PB5 GPIO_ACTIVE_HIGH>; ++ pwdn_active = ; ++ pwr_active = ; ++ rockchip,power_pmu_name1 = "vcc2v8_dvp"; ++ rockchip,power_pmu_voltage1 = <2800000>; ++ rockchip,power_pmu_name2 = "vcc1v8_dvp"; ++ rockchip,power_pmu_voltage2 = <1800000>; ++ mir = <0>; ++ flash_attach = <0>; ++ resolution = ; ++ powerup_sequence = ; ++ orientation = <90>; ++ i2c_add = ; ++ i2c_chl = <2>; ++ cif_chl = <0>; ++ mclk_rate = <24>; ++ }; ++ ++ gc0312_f { ++ is_front = <1>; ++ powerdown-gpios = <&gpio2 RK_PB6 GPIO_ACTIVE_HIGH>; ++ pwdn_active = ; ++ pwr_active = ; ++ rockchip,power_pmu_name1 = "vcc2v8_dvp"; ++ rockchip,power_pmu_voltage1 = <2800000>; ++ rockchip,power_pmu_name2 = "vcc1v8_dvp"; ++ rockchip,power_pmu_voltage2 = <1800000>; ++ mir = <0>; ++ flash_attach = <0>; ++ resolution = ; ++ powerup_sequence = ; ++ orientation = <270>; ++ i2c_add = ; ++ i2c_chl = <2>; ++ cif_chl = <0>; ++ mclk_rate = <24>; ++ }; ++ }; ++}; ++ +diff --git a/arch/arm64/boot/dts/rockchip/rk3326-863-lp3-v10-avb.dts b/arch/arm64/boot/dts/rockchip/rk3326-863-lp3-v10-avb.dts +new file mode 100755 +index 000000000000..174d05e37ccb +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3326-863-lp3-v10-avb.dts +@@ -0,0 +1,112 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++#include "rk3326-863-lp3-v10.dtsi" ++ ++/ { ++ model = "Rockchip rk3326 863 avb board"; ++ compatible = "rockchip,rk3326-863-lp3-v10-avb", "rockchip,rk3326"; ++}; ++ ++&firmware_android { ++ compatible = "android,firmware"; ++ boot_devices = "ff390000.dwmmc,ff3b0000.nandc"; ++ vbmeta { ++ compatible = "android,vbmeta"; ++ parts = "vbmeta,boot,system,vendor,dtbo"; ++ }; ++ fstab { ++ compatible = "android,fstab"; ++ vendor { ++ compatible = "android,vendor"; ++ dev = "/dev/block/by-name/vendor"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,avb"; ++ }; ++ }; ++}; ++ ++&i2c2 { ++ status = "okay"; ++ ++ gc0312@21 { ++ status = "okay"; ++ compatible = "galaxycore,gc0312"; ++ reg = <0x21>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_clkout_m0>; ++ ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ ++ avdd-supply = <&vcc2v8_dvp>; ++ dovdd-supply = <&vcc1v8_dvp>; ++ dvdd-supply = <&vcc1v8_dvp>; ++ ++ pwdn-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>; ++ ++ port { ++ gc0312_out: endpoint { ++ remote-endpoint = <&dvp_in_fcam>; ++ }; ++ }; ++ }; ++ ++ gc2145@3c { ++ status = "okay"; ++ compatible = "galaxycore,gc2145"; ++ reg = <0x3c>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_clkout_m0>; ++ ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ ++ avdd-supply = <&vcc2v8_dvp>; ++ dovdd-supply = <&vcc1v8_dvp>; ++ dvdd-supply = <&vcc1v8_dvp>; ++ ++ pwdn-gpios = <&gpio2 13 GPIO_ACTIVE_HIGH>; ++ ++ port { ++ gc2145_out: endpoint { ++ remote-endpoint = <&dvp_in_bcam>; ++ }; ++ }; ++ }; ++}; ++ ++&isp_mmu { ++ status = "okay"; ++}; ++ ++&rkisp1 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_clkout_m0 &dvp_d0d1_m0 &dvp_d2d9_m0 &dvp_d10d11_m0>; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dvp_in_fcam: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&gc0312_out>; ++ }; ++ ++ dvp_in_bcam: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&gc2145_out>; ++ }; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3326-863-lp3-v10-rkisp1.dts b/arch/arm64/boot/dts/rockchip/rk3326-863-lp3-v10-rkisp1.dts +new file mode 100755 +index 000000000000..d80dad694c1d +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3326-863-lp3-v10-rkisp1.dts +@@ -0,0 +1,103 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++#include "rk3326-863-lp3-v10.dtsi" ++ ++/ { ++ model = "Rockchip rk3326 863 rkisp1 board"; ++ compatible = "rockchip,rk3326-863-lp3-v10-rkisp1", "rockchip,rk3326"; ++}; ++ ++&chosen { ++ bootargs_ext = "androidboot.boot_devices=ff390000.dwmmc,ff3b0000.nandc"; ++}; ++ ++&i2c2 { ++ status = "okay"; ++ ++ gc0312: gc0312@21 { ++ status = "okay"; ++ compatible = "galaxycore,gc0312"; ++ reg = <0x21>; ++ //pinctrl-names = "default"; ++ //pinctrl-0 = <&cif_clkout_m0>; ++ ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ ++ avdd-supply = <&vcc2v8_dvp>; ++ dovdd-supply = <&vcc1v8_dvp>; ++ dvdd-supply = <&vcc1v8_dvp>; ++ ++ pwdn-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>; ++ rockchip,camera-module-index = <1>; ++ rockchip,camera-module-facing = "front"; ++ rockchip,camera-module-name = "CameraKing"; ++ rockchip,camera-module-lens-name = "Largan"; ++ port { ++ gc0312_out: endpoint { ++ remote-endpoint = <&dvp_in_fcam>; ++ }; ++ }; ++ }; ++ ++ gc2145: gc2145@3c { ++ status = "okay"; ++ compatible = "galaxycore,gc2145"; ++ reg = <0x3c>; ++ //pinctrl-names = "default"; ++ //pinctrl-0 = <&cif_clkout_m0>; ++ ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ ++ avdd-supply = <&vcc2v8_dvp>; ++ dovdd-supply = <&vcc1v8_dvp>; ++ dvdd-supply = <&vcc1v8_dvp>; ++ ++ pwdn-gpios = <&gpio2 13 GPIO_ACTIVE_HIGH>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "CameraKing"; ++ rockchip,camera-module-lens-name = "Largan"; ++ port { ++ gc2145_out: endpoint { ++ remote-endpoint = <&dvp_in_bcam>; ++ }; ++ }; ++ }; ++}; ++ ++&isp_mmu { ++ status = "okay"; ++}; ++ ++&rkisp1 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&dvp_d0d1_m0 &dvp_d2d9_m0>; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dvp_in_fcam: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&gc0312_out>; ++ }; ++ ++ dvp_in_bcam: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&gc2145_out>; ++ }; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3326-863-lp3-v10.dts b/arch/arm64/boot/dts/rockchip/rk3326-863-lp3-v10.dts +new file mode 100755 +index 000000000000..7a399b3356ac +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3326-863-lp3-v10.dts +@@ -0,0 +1,42 @@ ++/* ++ * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++ */ ++ ++/dts-v1/; ++#include "rk3326-863-lp3-v10.dtsi" ++ ++/ { ++ model = "Rockchip rk3326 863 board"; ++ compatible = "rockchip,rk3326-863-lp3-v10", "rockchip,rk3326"; ++}; ++ ++&cif { ++ status = "okay"; ++}; ++ ++&cif_sensor { ++ status = "okay"; ++}; ++ ++&firmware_android { ++ compatible = "android,firmware"; ++ fstab { ++ compatible = "android,fstab"; ++ system { ++ compatible = "android,system"; ++ dev = "/dev/block/by-name/system"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait"; ++ }; ++ vendor { ++ compatible = "android,vendor"; ++ dev = "/dev/block/by-name/vendor"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait"; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3326-863-lp3-v10.dtsi b/arch/arm64/boot/dts/rockchip/rk3326-863-lp3-v10.dtsi +new file mode 100755 +index 000000000000..6e6e3be2f104 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3326-863-lp3-v10.dtsi +@@ -0,0 +1,833 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++#include ++#include ++#include ++#include ++#include ++#include "rk3326.dtsi" ++#include "rk3326-863-cif-sensor.dtsi" ++#include "px30-android.dtsi" ++ ++/ { ++ adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 2>; ++ io-channel-names = "buttons"; ++ poll-interval = <100>; ++ keyup-threshold-microvolt = <1800000>; ++ ++ vol-down-key { ++ linux,code = ; ++ label = "volume down"; ++ press-threshold-microvolt = <300000>; ++ }; ++ ++ vol-up-key { ++ linux,code = ; ++ label = "volume up"; ++ press-threshold-microvolt = <17000>; ++ }; ++ }; ++ ++ backlight: backlight { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm1 0 25000 0>; ++ brightness-levels = < ++ 0 10 10 11 11 12 12 13 ++ 13 14 14 15 15 16 16 17 ++ 17 18 18 19 20 21 22 23 ++ 24 25 26 27 28 29 30 31 ++ 32 33 34 35 36 37 38 39 ++ 40 41 42 43 44 45 46 47 ++ 48 49 50 51 52 53 54 55 ++ 56 57 58 59 60 61 62 63 ++ 64 65 66 67 68 69 70 71 ++ 72 73 74 75 76 77 78 79 ++ 80 81 82 83 84 85 86 87 ++ 88 89 90 91 92 93 94 95 ++ 96 97 98 99 100 101 102 103 ++ 104 105 106 107 108 109 110 111 ++ 112 113 114 115 116 117 118 119 ++ 120 121 122 123 124 125 126 127 ++ 128 129 130 131 132 133 134 135 ++ 136 137 138 139 140 141 142 143 ++ 144 145 146 147 148 149 150 151 ++ 152 153 154 155 156 157 158 159 ++ 160 161 162 163 164 165 166 167 ++ 168 169 170 171 172 173 174 175 ++ 176 177 178 179 180 181 182 183 ++ 184 185 186 187 188 189 190 191 ++ 192 193 194 195 196 197 198 199 ++ 200 201 202 203 204 205 206 207 ++ 208 209 210 211 212 213 214 215 ++ 216 217 218 219 220 221 222 223 ++ 224 225 226 227 228 229 230 231 ++ 232 233 234 235 236 237 238 239 ++ 240 241 242 243 244 245 246 247 ++ 248 249 250 251 252 253 254 255>; ++ default-brightness-level = <200>; ++ }; ++ ++ charge-animation { ++ compatible = "rockchip,uboot-charge"; ++ rockchip,uboot-charge-on = <1>; ++ rockchip,android-charge-on = <0>; ++ rockchip,uboot-low-power-voltage = <3500>; ++ rockchip,screen-on-voltage = <3600>; ++ status = "okay"; ++ }; ++ ++ rk817-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip-rk817-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,widgets = ++ "Microphone", "Mic Jack", ++ "Headphone", "Headphone Jack"; ++ simple-audio-card,routing = ++ "Mic Jack", "MICBIAS1", ++ "IN1P", "Mic Jack", ++ "Headphone Jack", "HPOL", ++ "Headphone Jack", "HPOR"; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s1_2ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&rk817_codec>; ++ }; ++ }; ++ ++ rk_headset: rk-headset { ++ compatible = "rockchip_headset"; ++ headset_gpio = <&gpio2 RK_PC6 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hp_det>; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ clocks = <&cru SCLK_WIFI_PMU>; ++ clock-names = "clk_wifi_pmu"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_LOW>; /* GPIO3_A4 */ ++ }; ++ ++ vccsys: vccsys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc3v8_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3800000>; ++ regulator-max-microvolt = <3800000>; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ wifi_chip_type = "rtl8723cs"; ++ WIFI,host_wake_irq = <&gpio0 RK_PB2 GPIO_ACTIVE_HIGH>; ++ WIFI,vbat_gpio = <&gpio3 RK_PB6 GPIO_ACTIVE_LOW>; ++ status = "okay"; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ uart_rts_gpios = <&gpio1 RK_PC3 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default","rts_gpio"; ++ pinctrl-0 = <&uart1_rts>; ++ pinctrl-1 = <&uart1_rts_gpio>; ++ BT,reset_gpio = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio0 RK_PA1 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio0 RK_PB3 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++}; ++ ++ ++&display_subsystem { ++ status = "okay"; ++}; ++ ++&dsi { ++ status = "okay"; ++ ++ panel@0 { ++ compatible = "aoly,sl008pa21y1285-b00", "simple-panel-dsi"; ++ reg = <0>; ++ backlight = <&backlight>; ++ enable-gpios = <&gpio0 RK_PB7 GPIO_ACTIVE_LOW>; ++ reset-gpios = <&gpio3 RK_PB7 GPIO_ACTIVE_LOW>; ++ prepare-delay-ms = <20>; ++ reset-delay-ms = <20>; ++ init-delay-ms = <20>; ++ enable-delay-ms = <120>; ++ disable-delay-ms = <20>; ++ unprepare-delay-ms = <20>; ++ ++ width-mm = <108>; ++ height-mm = <172>; ++ ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | ++ MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ ++ panel-init-sequence = [ ++ 05 78 01 11 ++ 05 14 01 29 ++ ]; ++ ++ panel-exit-sequence = [ ++ 05 00 01 28 ++ 05 00 01 10 ++ ]; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <66000000>; ++ hactive = <800>; ++ vactive = <1280>; ++ hfront-porch = <2>; ++ hsync-len = <18>; ++ hback-porch = <18>; ++ vfront-porch = <4>; ++ vsync-len = <4>; ++ vback-porch = <16>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ panel_in_dsi: endpoint { ++ remote-endpoint = <&dsi_out_panel>; ++ }; ++ }; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@1 { ++ reg = <1>; ++ dsi_out_panel: endpoint { ++ remote-endpoint = <&panel_in_dsi>; ++ }; ++ }; ++ }; ++}; ++ ++&dsi_in_vopb { ++ status = "okay"; ++}; ++ ++&dsi_in_vopl { ++ status = "disabled"; ++}; ++ ++&route_dsi { ++ connect = <&vopb_out_dsi>; ++ status = "okay"; ++}; ++ ++&bus_apll { ++ bus-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&cpu0 { ++ cpu-supply = <&vdd_arm>; ++}; ++ ++&cpu0_opp_table { ++ /* ++ * max IR-drop values on different freq condition for this board! ++ */ ++ rockchip,board-irdrop = < ++ /*MHz MHz uV */ ++ 0 815 37500 ++ 816 1119 50000 ++ 1200 1512 75000 ++ >; ++}; ++ ++&dmc_opp_table { ++ /* ++ * max IR-drop values on different freq condition for this board! ++ */ ++ rockchip,board-irdrop = < ++ /*MHz MHz uV */ ++ 451 800 75000 ++ >; ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&dmc { ++ center-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&emmc { ++ bus-width = <8>; ++ cap-mmc-highspeed; ++ mmc-hs200-1_8v; ++ no-sdio; ++ no-sd; ++ disable-wp; ++ non-removable; ++ num-slots = <1>; ++ status = "okay"; ++}; ++ ++&gpu { ++ mali-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ ++ rk817: pmic@20 { ++ compatible = "rockchip,rk817"; ++ reg = <0x20>; ++ interrupt-parent = <&gpio0>; ++ interrupts = <7 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default", "pmic-sleep", ++ "pmic-power-off", "pmic-reset"; ++ pinctrl-0 = <&pmic_int>; ++ pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; ++ pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; ++ pinctrl-3 = <&soc_slppin_rst>, <&rk817_slppin_rst>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ //fb-inner-reg-idxs = <2>; ++ /* 1: rst regs (default in codes), 0: rst the pmic */ ++ pmic-reset-func = <1>; ++ ++ vcc1-supply = <&vccsys>; ++ vcc2-supply = <&vccsys>; ++ vcc3-supply = <&vccsys>; ++ vcc4-supply = <&vccsys>; ++ vcc5-supply = <&vccsys>; ++ vcc6-supply = <&vccsys>; ++ vcc7-supply = <&vcc_3v0>; ++ vcc8-supply = <&vccsys>; ++ vcc9-supply = <&dcdc_boost>; ++ ++ pwrkey { ++ status = "okay"; ++ }; ++ ++ pinctrl_rk8xx: pinctrl_rk8xx { ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ rk817_ts_gpio1: rk817_ts_gpio1 { ++ pins = "gpio_ts"; ++ function = "pin_fun1"; ++ /* output-low; */ ++ /* input-enable; */ ++ }; ++ ++ rk817_gt_gpio2: rk817_gt_gpio2 { ++ pins = "gpio_gt"; ++ function = "pin_fun1"; ++ }; ++ ++ rk817_pin_ts: rk817_pin_ts { ++ pins = "gpio_ts"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_pin_gt: rk817_pin_gt { ++ pins = "gpio_gt"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_slppin_null: rk817_slppin_null { ++ pins = "gpio_slp"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_slppin_slp: rk817_slppin_slp { ++ pins = "gpio_slp"; ++ function = "pin_fun1"; ++ }; ++ ++ rk817_slppin_pwrdn: rk817_slppin_pwrdn { ++ pins = "gpio_slp"; ++ function = "pin_fun2"; ++ }; ++ ++ rk817_slppin_rst: rk817_slppin_rst { ++ pins = "gpio_slp"; ++ function = "pin_fun3"; ++ }; ++ }; ++ ++ regulators { ++ vdd_logic: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <950000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_logic"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <950000>; ++ }; ++ }; ++ ++ vdd_arm: DCDC_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <950000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_arm"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <950000>; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_ddr"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_3v0: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_3v0"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ ++ vcc_1v0: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ regulator-name = "vcc_1v0"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc1v8_soc: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcc1v8_soc"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd1v0_soc: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ ++ regulator-name = "vcc1v0_soc"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc3v0_pmu: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ ++ regulator-name = "vcc3v0_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ ++ }; ++ }; ++ ++ vccio_sd: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vccio_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc_sd: LDO_REG6 { ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vcc_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ ++ }; ++ }; ++ ++ vcc2v8_dvp: LDO_REG7 { ++ regulator-boot-on; ++ regulator-min-microvolt = <2800000>; ++ regulator-max-microvolt = <2800000>; ++ ++ regulator-name = "vcc2v8_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <2800000>; ++ }; ++ }; ++ ++ vcc1v8_dvp: LDO_REG8 { ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcc1v8_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd1v5_dvp: LDO_REG9 { ++ regulator-boot-on; ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ ++ regulator-name = "vdd1v5_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <1500000>; ++ }; ++ }; ++ ++ dcdc_boost: BOOST { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <4700000>; ++ regulator-max-microvolt = <5400000>; ++ regulator-name = "boost"; ++ }; ++ ++ otg_switch: OTG_SWITCH { ++ regulator-name = "otg_switch"; ++ }; ++ }; ++ ++ battery { ++ compatible = "rk817,battery"; ++ ocv_table = <3500 3548 3592 3636 3687 3740 3780 ++ 3806 3827 3846 3864 3889 3929 3964 ++ 3993 4015 4030 4041 4056 4076 4148>; ++ design_capacity = <4000>; ++ design_qmax = <4200>; ++ bat_res = <100>; ++ sleep_enter_current = <150>; ++ sleep_exit_current = <180>; ++ sleep_filter_current = <100>; ++ power_off_thresd = <3500>; ++ zero_algorithm_vol = <3850>; ++ max_soc_offset = <60>; ++ monitor_sec = <5>; ++ sample_res = <10>; ++ virtual_power = <0>; ++ }; ++ ++ charger { ++ compatible = "rk817,charger"; ++ min_input_voltage = <4500>; ++ max_input_current = <1500>; ++ max_chrg_current = <2000>; ++ max_chrg_voltage = <4200>; ++ chrg_term_mode = <0>; ++ chrg_finish_cur = <300>; ++ virtual_power = <0>; ++ dc_det_adc = <0>; ++ extcon = <&u2phy>; ++ }; ++ ++ rk817_codec: codec { ++ #sound-dai-cells = <0>; ++ compatible = "rockchip,rk817-codec"; ++ clocks = <&cru SCLK_I2S1_OUT>; ++ clock-names = "mclk"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1_2ch_mclk>; ++ hp-volume = <20>; ++ spk-volume = <3>; ++ mic-in-differential; ++ status = "okay"; ++ }; ++ }; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ ++ ts@40 { ++ status = "okay"; ++ compatible = "GSL,GSL3673_800X1280"; ++ reg = <0x40>; ++ irq_gpio_number = <&gpio0 RK_PA5 IRQ_TYPE_LEVEL_LOW>; ++ rst_gpio_number = <&gpio0 RK_PB4 GPIO_ACTIVE_HIGH>; ++ }; ++ ++ sensor@19 { ++ status = "okay"; ++ compatible = "gs_lis3dh"; ++ reg = <0x19>; ++ type = ; ++ irq-gpio = <&gpio0 RK_PB5 IRQ_TYPE_LEVEL_LOW>; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ layout = <7>; ++ reprobe_en = <1>; ++ }; ++}; ++ ++&i2c2 { ++ status = "okay"; ++}; ++ ++&i2s1_2ch { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++}; ++ ++&io_domains { ++ status = "okay"; ++ ++ vccio1-supply = <&vcc_3v0>; ++ vccio2-supply = <&vccio_sd>; ++ vccio3-supply = <&vcc2v8_dvp>; ++ vccio4-supply = <&vcc_3v0>; ++ vccio5-supply = <&vcc_3v0>; ++}; ++ ++&nandc0 { ++ status = "okay"; ++}; ++ ++&pinctrl { ++ headphone { ++ hp_det: hp-det { ++ rockchip,pins = <2 RK_PC6 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ pmic { ++ pmic_int: pmic_int { ++ rockchip,pins = ++ <0 RK_PA7 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ soc_slppin_gpio: soc_slppin_gpio { ++ rockchip,pins = ++ <0 RK_PA4 RK_FUNC_GPIO &pcfg_output_low>; ++ }; ++ ++ soc_slppin_slp: soc_slppin_slp { ++ rockchip,pins = ++ <0 RK_PA4 1 &pcfg_pull_none>; ++ }; ++ ++ soc_slppin_rst: soc_slppin_rst { ++ rockchip,pins = ++ <0 RK_PA4 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ ++ pmuio1-supply = <&vcc3v0_pmu>; ++ pmuio2-supply = <&vcc3v0_pmu>; ++}; ++ ++&pwm1 { ++ status = "okay"; ++}; ++ ++&rk_rga { ++ status = "okay"; ++}; ++ ++&rockchip_suspend { ++ status = "okay"; ++ rockchip,sleep-debug-en = <1>; ++}; ++ ++&saradc { ++ status = "okay"; ++ vref-supply = <&vcc1v8_soc>; ++}; ++ ++&sdmmc { ++ bus-width = <4>; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ no-sdio; ++ no-mmc; ++ card-detect-delay = <800>; ++ ignore-pm-notify; ++ sd-uhs-sdr12; ++ sd-uhs-sdr25; ++ sd-uhs-sdr50; ++ sd-uhs-sdr104; ++ vqmmc-supply = <&vccio_sd>; ++ vmmc-supply = <&vcc_sd>; ++ status = "disabled"; ++}; ++ ++&sdio { ++ bus-width = <4>; ++ cap-sd-highspeed; ++ no-sd; ++ no-mmc; ++ ignore-pm-notify; ++ keep-power-in-suspend; ++ non-removable; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&tsadc { ++ pinctrl-names = "gpio", "otpout"; ++ pinctrl-0 = <&tsadc_otp_gpio>; ++ pinctrl-1 = <&tsadc_otp_out>; ++ status = "okay"; ++}; ++ ++&u2phy { ++ status = "okay"; ++ ++ u2phy_host: host-port { ++ rockchip,low-power-mode; ++ status = "okay"; ++ }; ++ ++ u2phy_otg: otg-port { ++ rockchip,low-power-mode; ++ status = "okay"; ++ }; ++}; ++ ++&usb20_otg { ++ status = "okay"; ++}; ++ ++&uart1 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart1_xfer &uart1_cts>; ++ status = "okay"; ++}; ++ ++&vip_mmu { ++ status = "okay"; ++}; ++ ++&vopb { ++ status = "okay"; ++}; ++ ++&vopb_mmu { ++ status = "okay"; ++}; ++ ++&vopl { ++ status = "okay"; ++}; ++ ++&vopl_mmu { ++ status = "okay"; ++}; ++ ++&mpp_srv { ++ status = "okay"; ++}; ++ ++&vdpu { ++ status = "okay"; ++}; ++ ++&vepu { ++ status = "okay"; ++}; ++ ++&vpu_mmu { ++ status = "okay"; ++}; ++ ++&hevc { ++ status = "okay"; ++}; ++ ++&hevc_mmu { ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3326-86v-v10.dts b/arch/arm64/boot/dts/rockchip/rk3326-86v-v10.dts +new file mode 100755 +index 000000000000..86d048c0d731 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3326-86v-v10.dts +@@ -0,0 +1,840 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++#include ++#include ++#include ++#include ++#include ++#include "rk3326.dtsi" ++#include "rk3326-863-cif-sensor.dtsi" ++#include "px30-android.dtsi" ++ ++/ { ++ model = "Rockchip rk3326 86v board"; ++ compatible = "rockchip,rk3326-86v-v10", "rockchip,rk3326"; ++ ++ adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 2>; ++ io-channel-names = "buttons"; ++ poll-interval = <100>; ++ keyup-threshold-microvolt = <617000>; ++ ++ vol-down-key { ++ linux,code = ; ++ label = "volume down"; ++ press-threshold-microvolt = <300000>; ++ }; ++ ++ vol-up-key { ++ linux,code = ; ++ label = "volume up"; ++ press-threshold-microvolt = <17000>; ++ }; ++ }; ++ ++ backlight: backlight { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm1 0 25000 0>; ++ brightness-levels = < ++ 0 10 10 11 11 12 12 13 ++ 13 14 14 15 15 16 16 17 ++ 17 18 18 19 20 21 22 23 ++ 24 25 26 27 28 29 30 31 ++ 32 33 34 35 36 37 38 39 ++ 40 41 42 43 44 45 46 47 ++ 48 49 50 51 52 53 54 55 ++ 56 57 58 59 60 61 62 63 ++ 64 65 66 67 68 69 70 71 ++ 72 73 74 75 76 77 78 79 ++ 80 81 82 83 84 85 86 87 ++ 88 89 90 91 92 93 94 95 ++ 96 97 98 99 100 101 102 103 ++ 104 105 106 107 108 109 110 111 ++ 112 113 114 115 116 117 118 119 ++ 120 121 122 123 124 125 126 127 ++ 128 129 130 131 132 133 134 135 ++ 136 137 138 139 140 141 142 143 ++ 144 145 146 147 148 149 150 151 ++ 152 153 154 155 156 157 158 159 ++ 160 161 162 163 164 165 166 167 ++ 168 169 170 171 172 173 174 175 ++ 176 177 178 179 180 181 182 183 ++ 184 185 186 187 188 189 190 191 ++ 192 193 194 195 196 197 198 199 ++ 200 201 202 203 204 205 206 207 ++ 208 209 210 211 212 213 214 215 ++ 216 217 218 219 220 221 222 223 ++ 224 225 226 227 228 229 230 231 ++ 232 233 234 235 236 237 238 239 ++ 240 241 242 243 244 245 246 247 ++ 248 249 250 251 252 253 254 255>; ++ default-brightness-level = <200>; ++ }; ++ ++ charge-animation { ++ compatible = "rockchip,uboot-charge"; ++ rockchip,uboot-charge-on = <0>; ++ rockchip,android-charge-on = <0>; ++ rockchip,uboot-low-power-voltage = <3500>; ++ rockchip,screen-on-voltage = <3600>; ++ status = "okay"; ++ }; ++ ++ panel { ++ compatible ="simple-panel"; ++ enable-gpios = <&gpio0 RK_PB5 GPIO_ACTIVE_LOW>; ++ reset-gpios = <&gpio3 RK_PB7 GPIO_ACTIVE_LOW>; ++ bus-format = ; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <51200000>; ++ hactive = <1024>; ++ vactive = <600>; ++ hback-porch = <100>; ++ hfront-porch = <120>; ++ vback-porch = <10>; ++ vfront-porch = <15>; ++ hsync-len = <100>; ++ vsync-len = <10>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ port { ++ panel_in_rgb: endpoint { ++ remote-endpoint = <&rgb_out_panel>; ++ }; ++ }; ++ }; ++ ++ rk817-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,rk817-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,widgets = ++ "Microphone", "Mic Jack", ++ "Headphone", "Headphone Jack"; ++ simple-audio-card,routing = ++ "Mic Jack", "MICBIAS1", ++ "IN1P", "Mic Jack", ++ "Headphone Jack", "HPOL", ++ "Headphone Jack", "HPOR"; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s1_2ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&rk817_codec>; ++ }; ++ }; ++ ++ rk_headset: rk-headset { ++ compatible = "rockchip_headset"; ++ headset_gpio = <&gpio2 RK_PC6 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hp_det>; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ clocks = <&cru SCLK_WIFI_PMU>; ++ clock-names = "clk_wifi_pmu"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_LOW>; /* GPIO3_A4 */ ++ }; ++ ++ vccsys: vccsys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc3v8_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3800000>; ++ regulator-max-microvolt = <3800000>; ++ }; ++}; ++ ++&cif { ++ status = "okay"; ++}; ++ ++&cif_sensor { ++ status = "okay"; ++}; ++ ++&display_subsystem { ++ status = "okay"; ++}; ++ ++&bus_apll { ++ bus-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&cpu0 { ++ cpu-supply = <&vdd_arm>; ++}; ++ ++&cpu0_opp_table { ++ /* ++ * max IR-drop values on different freq condition for this board! ++ */ ++ rockchip,board-irdrop = < ++ /*MHz MHz uV */ ++ 0 815 75000 ++ 816 1119 75000 ++ 1200 1512 75000 ++ >; ++}; ++ ++&dmc_opp_table { ++ /* ++ * max IR-drop values on different freq condition for this board! ++ */ ++ rockchip,board-irdrop = < ++ /*MHz MHz uV */ ++ 451 800 75000 ++ >; ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&dmc { ++ center-supply = <&vdd_logic>; ++ status = "disabled"; ++}; ++ ++&emmc { ++ bus-width = <8>; ++ cap-mmc-highspeed; ++ /*mmc-hs200-1_8v;*/ ++ no-sdio; ++ no-sd; ++ disable-wp; ++ non-removable; ++ num-slots = <1>; ++ status = "okay"; ++}; ++ ++&gpu { ++ mali-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ ++ rk817: pmic@20 { ++ compatible = "rockchip,rk817"; ++ reg = <0x20>; ++ interrupt-parent = <&gpio0>; ++ interrupts = <10 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default", "pmic-sleep", ++ "pmic-power-off", "pmic-reset"; ++ pinctrl-0 = <&pmic_int>; ++ pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; ++ pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; ++ pinctrl-3 = <&soc_slppin_rst>, <&rk817_slppin_rst>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ //fb-inner-reg-idxs = <2>; ++ /* 1: rst regs (default in codes), 0: rst the pmic */ ++ pmic-reset-func = <1>; ++ ++ vcc1-supply = <&vccsys>; ++ vcc2-supply = <&vccsys>; ++ vcc3-supply = <&vccsys>; ++ vcc4-supply = <&vccsys>; ++ vcc5-supply = <&vccsys>; ++ vcc6-supply = <&vccsys>; ++ vcc7-supply = <&vcc_3v0>; ++ vcc8-supply = <&vccsys>; ++ vcc9-supply = <&dcdc_boost>; ++ ++ pwrkey { ++ status = "okay"; ++ }; ++ ++ pinctrl_rk8xx: pinctrl_rk8xx { ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ rk817_ts_gpio1: rk817_ts_gpio1 { ++ pins = "gpio_ts"; ++ function = "pin_fun1"; ++ /* output-low; */ ++ /* input-enable; */ ++ }; ++ ++ rk817_gt_gpio2: rk817_gt_gpio2 { ++ pins = "gpio_gt"; ++ function = "pin_fun1"; ++ }; ++ ++ rk817_pin_ts: rk817_pin_ts { ++ pins = "gpio_ts"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_pin_gt: rk817_pin_gt { ++ pins = "gpio_gt"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_slppin_null: rk817_slppin_null { ++ pins = "gpio_slp"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_slppin_slp: rk817_slppin_slp { ++ pins = "gpio_slp"; ++ function = "pin_fun1"; ++ }; ++ ++ rk817_slppin_pwrdn: rk817_slppin_pwrdn { ++ pins = "gpio_slp"; ++ function = "pin_fun2"; ++ }; ++ ++ rk817_slppin_rst: rk817_slppin_rst { ++ pins = "gpio_slp"; ++ function = "pin_fun3"; ++ }; ++ }; ++ ++ regulators { ++ vdd_logic: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <950000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_logic"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <950000>; ++ }; ++ }; ++ ++ vdd_arm: DCDC_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <950000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_arm"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <950000>; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_ddr"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_3v0: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_3v0"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ ++ vcc_1v0: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ regulator-name = "vcc_1v0"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc1v8_soc: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcc1v8_soc"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd1v0_soc: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ ++ regulator-name = "vcc1v0_soc"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc3v0_pmu: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ ++ regulator-name = "vcc3v0_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ ++ }; ++ }; ++ ++ vccio_sd: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vccio_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc_sd: LDO_REG6 { ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vcc_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ ++ }; ++ }; ++ ++ vcc2v8_dvp: LDO_REG7 { ++ regulator-boot-on; ++ regulator-min-microvolt = <2800000>; ++ regulator-max-microvolt = <2800000>; ++ ++ regulator-name = "vcc2v8_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <2800000>; ++ }; ++ }; ++ ++ vcc1v8_dvp: LDO_REG8 { ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcc1v8_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd1v5_dvp: LDO_REG9 { ++ regulator-boot-on; ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ ++ regulator-name = "vdd1v5_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <1500000>; ++ }; ++ }; ++ ++ dcdc_boost: BOOST { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <4700000>; ++ regulator-max-microvolt = <5400000>; ++ regulator-name = "boost"; ++ }; ++ ++ otg_switch: OTG_SWITCH { ++ regulator-name = "otg_switch"; ++ }; ++ }; ++ ++ battery { ++ compatible = "rk817,battery"; ++ ocv_table = <3500 3548 3592 3636 3687 3740 3780 ++ 3806 3827 3846 3864 3889 3929 3964 ++ 3993 4015 4030 4041 4056 4076 4148>; ++ design_capacity = <4000>; ++ design_qmax = <4200>; ++ bat_res = <100>; ++ sleep_enter_current = <150>; ++ sleep_exit_current = <180>; ++ sleep_filter_current = <100>; ++ power_off_thresd = <3500>; ++ zero_algorithm_vol = <3850>; ++ max_soc_offset = <60>; ++ monitor_sec = <5>; ++ sample_res = <10>; ++ virtual_power = <0>; ++ }; ++ ++ charger { ++ compatible = "rk817,charger"; ++ min_input_voltage = <4500>; ++ max_input_current = <1500>; ++ max_chrg_current = <2000>; ++ max_chrg_voltage = <4200>; ++ chrg_term_mode = <0>; ++ chrg_finish_cur = <300>; ++ virtual_power = <0>; ++ dc_det_adc = <0>; ++ extcon = <&u2phy>; ++ }; ++ ++ rk817_codec: codec { ++ #sound-dai-cells = <0>; ++ compatible = "rockchip,rk817-codec"; ++ clocks = <&cru SCLK_I2S1_OUT>; ++ clock-names = "mclk"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1_2ch_mclk>; ++ hp-volume = <20>; ++ spk-volume = <3>; ++ mic-in-differential; ++ status = "okay"; ++ }; ++ }; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ ++ ts@40 { ++ compatible = "gslX680-d708"; ++ reg = <0x40>; ++ touch-gpio = <&gpio0 RK_PB3 IRQ_TYPE_LEVEL_LOW>; ++ wake-gpio = <&gpio0 RK_PC1 IRQ_TYPE_LEVEL_LOW>; ++ screen_max_x = <1024>; ++ screen_max_y = <600>; ++ revert_x = <1>; ++ status = "okay"; ++ }; ++ ++ sensor@1d { ++ status = "okay"; ++ compatible = "gs_lsm303d"; ++ reg = <0x1d>; ++ type = ; ++ irq-gpio = <&gpio0 RK_PA1 IRQ_TYPE_LEVEL_LOW>; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ layout = <5>; ++ reprobe_en = <1>; ++ }; ++}; ++ ++&i2c2 { ++ status = "okay"; ++}; ++ ++&i2s1_2ch { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++}; ++ ++&io_domains { ++ status = "okay"; ++ ++ vccio1-supply = <&vcc_3v0>; ++ vccio2-supply = <&vccio_sd>; ++ vccio3-supply = <&vcc2v8_dvp>; ++ vccio4-supply = <&vcc_3v0>; ++ vccio5-supply = <&vcc_3v0>; ++}; ++ ++&rgb { ++ status = "okay"; ++ ++ ports { ++ port@1 { ++ reg = <1>; ++ ++ rgb_out_panel: endpoint { ++ remote-endpoint = <&panel_in_rgb>; ++ }; ++ }; ++ }; ++}; ++ ++&nandc0 { ++ status = "okay"; ++}; ++ ++&pinctrl { ++ headphone { ++ hp_det: hp-det { ++ rockchip,pins = <2 RK_PC6 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ lcdc { ++ lcdc_m1_rgb_pins: lcdc-m1-rgb-pins { ++ rockchip,pins = ++ <3 RK_PA0 1 &pcfg_pull_none>, /* LCDC_DCLK */ ++ <3 RK_PA4 1 &pcfg_pull_none_8ma>, /* LCDC_D0 */ ++ <3 RK_PA6 1 &pcfg_pull_none_8ma>, /* LCDC_D2 */ ++ <3 RK_PB2 1 &pcfg_pull_none_8ma>, /* LCDC_D6 */ ++ <3 RK_PB3 1 &pcfg_pull_none_8ma>, /* LCDC_D7 */ ++ <3 RK_PB5 1 &pcfg_pull_none_8ma>, /* LCDC_D9 */ ++ <3 RK_PC0 1 &pcfg_pull_none_8ma>, /* LCDC_D12 */ ++ <3 RK_PC1 1 &pcfg_pull_none_8ma>, /* LCDC_D13 */ ++ <3 RK_PC2 1 &pcfg_pull_none_8ma>, /* LCDC_D14 */ ++ <3 RK_PC3 1 &pcfg_pull_none_8ma>, /* LCDC_D15 */ ++ <3 RK_PC4 1 &pcfg_pull_none_8ma>, /* LCDC_D16 */ ++ <3 RK_PC5 1 &pcfg_pull_none_8ma>; /* LCDC_D17 */ ++ }; ++ ++ lcdc_m1_sleep_pins: lcdc-m1-sleep-pins { ++ rockchip,pins = ++ <3 RK_PA0 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_DCLK */ ++ <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D0 */ ++ <3 RK_PA6 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D2 */ ++ <3 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D6 */ ++ <3 RK_PB3 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D7 */ ++ <3 RK_PB5 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D9 */ ++ <3 RK_PC0 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D12 */ ++ <3 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D13 */ ++ <3 RK_PC2 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D14 */ ++ <3 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D15 */ ++ <3 RK_PC4 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D16 */ ++ <3 RK_PC5 RK_FUNC_GPIO &pcfg_pull_none>; /* LCDC_D17 */ ++ }; ++ }; ++ ++ pmic { ++ pmic_int: pmic_int { ++ rockchip,pins = ++ <0 RK_PB2 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ soc_slppin_gpio: soc_slppin_gpio { ++ rockchip,pins = ++ <0 RK_PA4 RK_FUNC_GPIO &pcfg_output_low>; ++ }; ++ ++ soc_slppin_slp: soc_slppin_slp { ++ rockchip,pins = ++ <0 RK_PA4 1 &pcfg_pull_none>; ++ }; ++ ++ soc_slppin_rst: soc_slppin_rst { ++ rockchip,pins = ++ <0 RK_PA4 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ ++ pmuio1-supply = <&vcc3v0_pmu>; ++ pmuio2-supply = <&vcc3v0_pmu>; ++}; ++ ++&pwm1 { ++ status = "okay"; ++}; ++ ++&rk_rga { ++ status = "okay"; ++}; ++ ++&rockchip_suspend { ++ status = "okay"; ++ rockchip,sleep-debug-en = <1>; ++}; ++ ++&route_rgb { ++ connect = <&vopb_out_rgb>; ++ status = "okay"; ++}; ++ ++&saradc { ++ status = "okay"; ++ vref-supply = <&vcc1v8_soc>; ++}; ++ ++&sdmmc { ++ bus-width = <4>; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ no-sdio; ++ no-mmc; ++ card-detect-delay = <800>; ++ ignore-pm-notify; ++ sd-uhs-sdr12; ++ sd-uhs-sdr25; ++ sd-uhs-sdr50; ++ sd-uhs-sdr104; ++ vqmmc-supply = <&vccio_sd>; ++ vmmc-supply = <&vcc_sd>; ++ status = "disabled"; ++}; ++ ++&sdio { ++ bus-width = <4>; ++ cap-sd-highspeed; ++ no-sd; ++ no-mmc; ++ ignore-pm-notify; ++ keep-power-in-suspend; ++ non-removable; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ sd-uhs-sdr104; ++ status = "disabled"; ++}; ++ ++&tsadc { ++ pinctrl-names = "gpio", "otpout"; ++ pinctrl-0 = <&tsadc_otp_gpio>; ++ pinctrl-1 = <&tsadc_otp_out>; ++ status = "okay"; ++}; ++ ++&u2phy { ++ status = "okay"; ++ ++ u2phy_host: host-port { ++ rockchip,low-power-mode; ++ status = "okay"; ++ }; ++ ++ u2phy_otg: otg-port { ++ rockchip,low-power-mode; ++ status = "okay"; ++ }; ++}; ++ ++&usb20_otg { ++ status = "okay"; ++}; ++ ++&uart1 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart1_xfer &uart1_cts>; ++ status = "okay"; ++}; ++ ++&vip_mmu { ++ status = "okay"; ++}; ++ ++&vopb { ++ status = "okay"; ++}; ++ ++&vopb_mmu { ++ status = "okay"; ++}; ++ ++&vopl { ++ status = "okay"; ++}; ++ ++&vopl_mmu { ++ status = "okay"; ++}; ++ ++&mpp_srv { ++ status = "okay"; ++}; ++ ++&vdpu { ++ status = "okay"; ++}; ++ ++&vepu { ++ status = "okay"; ++}; ++ ++&vpu_mmu { ++ status = "okay"; ++}; ++ ++&hevc { ++ status = "okay"; ++}; ++ ++&hevc_mmu { ++ status = "okay"; ++}; ++ ++&firmware_android { ++ compatible = "android,firmware"; ++ fstab { ++ compatible = "android,fstab"; ++ system { ++ compatible = "android,system"; ++ dev = "/dev/block/by-name/system"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait"; ++ }; ++ vendor { ++ compatible = "android,vendor"; ++ dev = "/dev/block/by-name/vendor"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait"; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3326-evb-ai-va-v10.dts b/arch/arm64/boot/dts/rockchip/rk3326-evb-ai-va-v10.dts +new file mode 100755 +index 000000000000..02308ebae99e +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3326-evb-ai-va-v10.dts +@@ -0,0 +1,1308 @@ ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++ */ ++ ++/dts-v1/; ++#include ++#include ++#include ++#include ++#include ++#include "rk3326.dtsi" ++#include "rk3326-863-cif-sensor.dtsi" ++#include "px30-android.dtsi" ++ ++/ { ++ model = "Rockchip rk3326 ai voice assistant evb board"; ++ compatible = "rockchip,rk3326-evb-ai-va-v10", "rockchip,rk3326"; ++ ++ adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 2>; ++ io-channel-names = "buttons"; ++ poll-interval = <100>; ++ keyup-threshold-microvolt = <1800000>; ++ ++ mute-key { ++ linux,code = ; ++ label = "mute"; ++ press-threshold-microvolt = <1119000>; ++ }; ++ ++ mode-key { ++ linux,code = ; ++ label = "mode"; ++ press-threshold-microvolt = <892000>; ++ }; ++ ++ media-key { ++ linux,code = ; ++ label = "media"; ++ press-threshold-microvolt = <616000>; ++ }; ++ ++ vol-down-key { ++ linux,code = ; ++ label = "volume down"; ++ press-threshold-microvolt = <300000>; ++ }; ++ ++ vol-up-key { ++ linux,code = ; ++ label = "volume up"; ++ press-threshold-microvolt = <15000>; ++ }; ++ }; ++ ++ backlight: backlight { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm1 0 25000 0>; ++ brightness-levels = < ++ 0 1 2 3 4 5 6 7 ++ 8 9 10 11 12 13 14 15 ++ 16 17 18 19 20 21 22 23 ++ 24 25 26 27 28 29 30 31 ++ 32 33 34 35 36 37 38 39 ++ 40 41 42 43 44 45 46 47 ++ 48 49 50 51 52 53 54 55 ++ 56 57 58 59 60 61 62 63 ++ 64 65 66 67 68 69 70 71 ++ 72 73 74 75 76 77 78 79 ++ 80 81 82 83 84 85 86 87 ++ 88 89 90 91 92 93 94 95 ++ 96 97 98 99 100 101 102 103 ++ 104 105 106 107 108 109 110 111 ++ 112 113 114 115 116 117 118 119 ++ 120 121 122 123 124 125 126 127 ++ 128 129 130 131 132 133 134 135 ++ 136 137 138 139 140 141 142 143 ++ 144 145 146 147 148 149 150 151 ++ 152 153 154 155 156 157 158 159 ++ 160 161 162 163 164 165 166 167 ++ 168 169 170 171 172 173 174 175 ++ 176 177 178 179 180 181 182 183 ++ 184 185 186 187 188 189 190 191 ++ 192 193 194 195 196 197 198 199 ++ 200 201 202 203 204 205 206 207 ++ 208 209 210 211 212 213 214 215 ++ 216 217 218 219 220 221 222 223 ++ 224 225 226 227 228 229 230 231 ++ 232 233 234 235 236 237 238 239 ++ 240 241 242 243 244 245 246 247 ++ 248 249 250 251 252 253 254 255>; ++ default-brightness-level = <200>; ++ }; ++ ++ rk809-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,name = "rockchip,rk809-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,widgets = ++ "Microphone", "Mic Jack", ++ "Headphone", "Headphone Jack"; ++ simple-audio-card,routing = ++ "Mic Jack", "MICBIAS1", ++ "IN1P", "Mic Jack", ++ "Headphone Jack", "HPOL", ++ "Headphone Jack", "HPOR"; ++ simple-audio-card,dai-link@0 { ++ format = "i2s"; ++ cpu { ++ sound-dai = <&i2s1_2ch>; ++ }; ++ codec { ++ sound-dai = <&rk809_codec 0>; ++ }; ++ }; ++ simple-audio-card,dai-link@1 { ++ format = "pdm"; ++ cpu { ++ sound-dai = <&pdm>; ++ }; ++ codec { ++ sound-dai = <&rk809_codec 1>; ++ }; ++ }; ++ }; ++ ++ bt-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "dsp_a"; ++ simple-audio-card,bitclock-inversion = <1>; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,name = "rockchip,bt"; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s2_2ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&bt_sco>; ++ }; ++ }; ++ ++ bt_sco: bt-sco { ++ compatible = "delta,dfbmcs320"; ++ #sound-dai-cells = <0>; ++ status = "okay"; ++ }; ++ ++ rk_headset: rk-headset { ++ compatible = "rockchip_headset"; ++ headset_gpio = <&gpio3 RK_PA4 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hp_det>; ++ io-channels = <&saradc 1>; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ clocks = <&rk809 1>; ++ clock-names = "ext_clock"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio0 RK_PB3 GPIO_ACTIVE_LOW>; /* GPIO3_A4 */ ++ }; ++ ++ test-power { ++ status = "okay"; ++ }; ++ ++ vcc5v0_sys: vccsys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc5v0_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ wifi_chip_type = "AP6210"; ++ WIFI,host_wake_irq = <&gpio0 RK_PA1 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&rk809 1>; ++ clock-names = "ext_clock"; ++ uart_rts_gpios = <&gpio1 RK_PC3 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default","rts_gpio"; ++ pinctrl-0 = <&uart1_rts>; ++ pinctrl-1 = <&uart1_rts_gpio>; ++ BT,reset_gpio = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio0 RK_PB2 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio0 RK_PA2 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++}; ++ ++&bus_apll { ++ bus-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&cpu0 { ++ cpu-supply = <&vdd_arm>; ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&display_subsystem { ++ status = "okay"; ++}; ++ ++&dmc { ++ center-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&dsi { ++ status = "okay"; ++ ++ panel@0 { ++ compatible = "sitronix,st7703", "simple-panel-dsi"; ++ reg = <0>; ++ backlight = <&backlight>; ++ enable-gpios = <&gpio0 RK_PB5 GPIO_ACTIVE_HIGH>; ++ prepare-delay-ms = <2>; ++ reset-delay-ms = <1>; ++ init-delay-ms = <20>; ++ enable-delay-ms = <120>; ++ disable-delay-ms = <50>; ++ unprepare-delay-ms = <40>; ++ ++ width-mm = <68>; ++ height-mm = <121>; ++ ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | ++ MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ ++ panel-init-sequence = [ ++ 05 fa 01 11 ++ 39 00 04 b9 f1 12 83 ++ 39 00 1c ba 33 81 05 f9 0e 0e 00 00 00 ++ 00 00 00 00 00 44 25 00 91 0a ++ 00 00 02 4f 01 00 00 37 ++ 15 00 02 b8 25 ++ 39 00 04 bf 02 11 00 ++ 39 00 0b b3 0c 10 0a 50 03 ff 00 00 00 ++ 00 ++ 39 00 0a c0 73 73 50 50 00 00 08 70 00 ++ 15 00 02 bc 46 ++ 15 00 02 cc 0b ++ 15 00 02 b4 80 ++ 39 00 04 b2 c8 12 30 ++ 39 00 0f e3 07 07 0b 0b 03 0b 00 00 00 ++ 00 ff 00 c0 10 ++ 39 00 0d c1 53 00 1e 1e 77 e1 cc dd 67 ++ 77 33 33 ++ 39 00 07 c6 00 00 ff ff 01 ff ++ 39 00 03 b5 09 09 ++ 39 00 03 b6 87 95 ++ 39 00 40 e9 c2 10 05 05 10 05 a0 12 31 ++ 23 3f 81 0a a0 37 18 00 80 01 ++ 00 00 00 00 80 01 00 00 00 48 ++ f8 86 42 08 88 88 80 88 88 88 ++ 58 f8 87 53 18 88 88 81 88 88 ++ 88 00 00 00 01 00 00 00 00 00 ++ 00 00 00 00 ++ 39 00 3e ea 00 1a 00 00 00 00 02 00 00 ++ 00 00 00 1f 88 81 35 78 88 88 ++ 85 88 88 88 0f 88 80 24 68 88 ++ 88 84 88 88 88 23 10 00 00 1c ++ 00 00 00 00 00 00 00 00 00 00 ++ 00 00 00 00 00 30 05 a0 00 00 ++ 00 00 ++ 39 00 23 e0 00 06 08 2a 31 3f 38 36 07 ++ 0c 0d 11 13 12 13 11 18 00 06 ++ 08 2a 31 3f 38 36 07 0c 0d 11 ++ 13 12 13 11 18 ++ 05 32 01 29 ++ ]; ++ ++ panel-exit-sequence = [ ++ 05 00 01 28 ++ 05 00 01 10 ++ ]; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <66000000>; ++ hactive = <720>; ++ vactive = <1280>; ++ hfront-porch = <40>; ++ hsync-len = <10>; ++ hback-porch = <40>; ++ vfront-porch = <22>; ++ vsync-len = <4>; ++ vback-porch = <11>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ panel_in_dsi: endpoint { ++ remote-endpoint = <&dsi_out_panel>; ++ }; ++ }; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@1 { ++ reg = <1>; ++ dsi_out_panel: endpoint { ++ remote-endpoint = <&panel_in_dsi>; ++ }; ++ }; ++ }; ++}; ++ ++&dsi_in_vopb { ++ status = "okay"; ++}; ++ ++&dsi_in_vopl { ++ status = "disabled"; ++}; ++ ++&route_dsi { ++ connect = <&vopb_out_dsi>; ++ status = "okay"; ++}; ++ ++&emmc { ++ bus-width = <8>; ++ cap-mmc-highspeed; ++ mmc-hs200-1_8v; ++ no-sdio; ++ no-sd; ++ disable-wp; ++ non-removable; ++ num-slots = <1>; ++ status = "okay"; ++}; ++ ++&gpu { ++ mali-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ clock-frequency = <400000>; ++ i2c-scl-rising-time-ns = <280>; ++ i2c-scl-falling-time-ns = <16>; ++ ++ rk809: pmic@20 { ++ compatible = "rockchip,rk809"; ++ reg = <0x20>; ++ interrupt-parent = <&gpio0>; ++ interrupts = <7 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default", "pmic-sleep", ++ "pmic-power-off", "pmic-reset"; ++ pinctrl-0 = <&pmic_int>; ++ pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; ++ pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; ++ pinctrl-3 = <&soc_slppin_rst>, <&rk817_slppin_rst>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ //fb-inner-reg-idxs = <2>; ++ /* 1: rst regs (default in codes), 0: rst the pmic */ ++ pmic-reset-func = <1>; ++ ++ vcc1-supply = <&vcc5v0_sys>; ++ vcc2-supply = <&vcc5v0_sys>; ++ vcc3-supply = <&vcc5v0_sys>; ++ vcc4-supply = <&vcc5v0_sys>; ++ vcc5-supply = <&vcc3v3_sys>; ++ vcc6-supply = <&vcc3v3_sys>; ++ vcc7-supply = <&vcc3v3_sys>; ++ vcc8-supply = <&vcc3v3_sys>; ++ vcc9-supply = <&vcc5v0_sys>; ++ ++ pwrkey { ++ status = "okay"; ++ }; ++ ++ pinctrl_rk8xx: pinctrl_rk8xx { ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ rk817_slppin_null: rk817_slppin_null { ++ pins = "gpio_slp"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_slppin_slp: rk817_slppin_slp { ++ pins = "gpio_slp"; ++ function = "pin_fun1"; ++ }; ++ ++ rk817_slppin_pwrdn: rk817_slppin_pwrdn { ++ pins = "gpio_slp"; ++ function = "pin_fun2"; ++ }; ++ ++ rk817_slppin_rst: rk817_slppin_rst { ++ pins = "gpio_slp"; ++ function = "pin_fun3"; ++ }; ++ }; ++ ++ regulators { ++ vdd_logic: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <950000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_logic"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <950000>; ++ }; ++ }; ++ ++ vdd_arm: DCDC_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <950000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_arm"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <950000>; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_ddr"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_3v0: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_3v0"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ ++ vcc_1v0: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ regulator-name = "vcc_1v0"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc1v8_soc: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcc1v8_soc"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd1v0_soc: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ ++ regulator-name = "vcc1v0_soc"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc3v0_pmu: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ ++ regulator-name = "vcc3v0_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ ++ }; ++ }; ++ ++ vccio_sd: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vccio_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc_sd: LDO_REG6 { ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vcc_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ ++ }; ++ }; ++ ++ vcc2v8_dvp: LDO_REG7 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <2800000>; ++ regulator-max-microvolt = <2800000>; ++ ++ regulator-name = "vcc2v8_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <2800000>; ++ }; ++ }; ++ ++ vcc1v8_dvp: LDO_REG8 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcc1v8_dvp"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd1v5_dvp: LDO_REG9 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ ++ regulator-name = "vdd1v5_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <1500000>; ++ }; ++ }; ++ ++ vcc3v3_sys: DCDC_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcc3v3_sys"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ ++ vcc5v0_host: SWITCH_REG1 { ++ regulator-name = "vcc5v0_host"; ++ }; ++ ++ vcc3v3_lcd: SWITCH_REG2 { ++ regulator-boot-on; ++ regulator-name = "vcc3v3_lcd"; ++ }; ++ }; ++ ++ rk809_codec: codec { ++ #sound-dai-cells = <1>; ++ compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; ++ clocks = <&cru SCLK_I2S1_OUT>; ++ clock-names = "mclk"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1_2ch_mclk>; ++ pdmdata-out-enable; ++ use-ext-amplifier; ++ adc-for-loopback; ++ spk-ctl-gpios = <&gpio3 RK_PB2 GPIO_ACTIVE_HIGH>; ++ hp-volume = <20>; ++ spk-volume = <20>; ++ }; ++ }; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ clock-frequency = <400000>; ++ i2c-scl-rising-time-ns = <275>; ++ i2c-scl-falling-time-ns = <16>; ++ ++ gt1x: gt1x@14 { ++ compatible = "goodix,gt1x"; ++ reg = <0x14>; ++ goodix,rst-gpio = <&gpio0 RK_PB4 GPIO_ACTIVE_HIGH>; ++ goodix,irq-gpio = <&gpio0 RK_PA5 IRQ_TYPE_LEVEL_LOW>; ++ }; ++ ++ is31fl3236: led-controller@3c { ++ compatible = "issi,is31fl3236"; ++ reg = <0x3c>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ reset-gpios = <&gpio3 RK_PA6 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ ++ led1: led@1 { ++ label = "led1"; ++ reg = <1>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <0>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led2: led@2 { ++ label = "led2"; ++ reg = <2>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <0>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led3: led@3 { ++ label = "led3"; ++ reg = <3>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led4: led@4 { ++ label = "led4"; ++ reg = <4>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <100>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led5: led@5 { ++ label = "led5"; ++ reg = <5>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <100>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led6: led@6 { ++ label = "led6"; ++ reg = <6>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led7: led@7 { ++ label = "led7"; ++ reg = <7>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <200>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led8: led@8 { ++ label = "led8"; ++ reg = <8>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <200>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led9: led@9 { ++ label = "led9"; ++ reg = <9>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led10: led@10 { ++ label = "led10"; ++ reg = <10>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <300>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led11: led@11 { ++ label = "led11"; ++ reg = <11>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <300>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led12: led@12 { ++ label = "led12"; ++ reg = <12>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led13: led@13 { ++ label = "led13"; ++ reg = <13>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <400>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led14: led@14 { ++ label = "led14"; ++ reg = <14>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <400>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led15: led@15 { ++ label = "led15"; ++ reg = <15>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led16: led@16 { ++ label = "led16"; ++ reg = <16>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <500>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led17: led@17 { ++ label = "led17"; ++ reg = <17>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <500>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led18: led@18 { ++ label = "led18"; ++ reg = <18>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led19: led@19 { ++ label = "led19"; ++ reg = <19>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <600>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led20: led@20 { ++ label = "led20"; ++ reg = <20>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <600>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led21: led@21 { ++ label = "led21"; ++ reg = <21>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led22: led@22 { ++ label = "led22"; ++ reg = <22>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <700>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led23: led@23 { ++ label = "led23"; ++ reg = <23>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <700>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led124: led@24 { ++ label = "led24"; ++ reg = <24>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led25: led@25 { ++ label = "led25"; ++ reg = <25>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <800>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led26: led@26 { ++ label = "led26"; ++ reg = <26>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <800>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led27: led@27 { ++ label = "led27"; ++ reg = <27>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led28: led@28 { ++ label = "led28"; ++ reg = <28>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <900>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led29: led@29 { ++ label = "led29"; ++ reg = <29>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <900>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led30: led@30 { ++ label = "led30"; ++ reg = <30>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led31: led@31 { ++ label = "led31"; ++ reg = <31>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <1000>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led32: led@32 { ++ label = "led32"; ++ reg = <32>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <1000>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led33: led@33 { ++ label = "led33"; ++ reg = <33>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led34: led@34 { ++ label = "led34"; ++ reg = <34>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <1100>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led35: led@35 { ++ label = "led35"; ++ reg = <35>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <1100>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led36: led@36 { ++ label = "led36"; ++ reg = <36>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ }; ++ ++ ls_stk3410: light@48 { ++ compatible = "ls_stk3410"; ++ status = "okay"; ++ reg = <0x48>; ++ type = ; ++ irq_enable = <0>; ++ als_threshold_high = <100>; ++ als_threshold_low = <10>; ++ als_ctrl_gain = <2>; /* 0:x1 1:x4 2:x16 3:x64 */ ++ poll_delay_ms = <100>; ++ }; ++ ++ ps_stk3410: proximity@48 { ++ compatible = "ps_stk3410"; ++ status = "okay"; ++ reg = <0x48>; ++ type = ; ++ //pinctrl-names = "default"; ++ //pinctrl-0 = <&gpio2_c3>; ++ //irq-gpio = <&gpio0 RK_PB7 IRQ_TYPE_LEVEL_LOW>; ++ //irq_enable = <1>; ++ ps_threshold_high = <0x200>; ++ ps_threshold_low = <0x100>; ++ ps_ctrl_gain = <3>; /* 0:x1 1:x4 2:x16 3:x64 */ ++ ps_led_current = <3>; /* 0:12.5mA 1:25mA 2:50mA 3:100mA */ ++ poll_delay_ms = <100>; ++ }; ++ ++}; ++ ++&i2c2 { ++ status = "okay"; ++ ++ clock-frequency = <100000>; ++ ++ /* These are relatively safe rise/fall times; TODO: measure */ ++ i2c-scl-falling-time-ns = <50>; ++ i2c-scl-rising-time-ns = <300>; ++ ++ ov5695: ov5695@36 { ++ compatible = "ovti,ov5695"; ++ reg = <0x36>; ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ /*reset-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>;*/ ++ pwdn-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>; ++ //pinctrl-names = "default"; ++ //pinctrl-0 = <&cif_clkout_m0>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "TongJu"; ++ rockchip,camera-module-lens-name = "CHT842-MD"; ++ port { ++ ov5695_out: endpoint { ++ remote-endpoint = <&mipi_in>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++}; ++ ++&i2s1_2ch { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++ pinctrl-0 = <&i2s1_2ch_sclk ++ &i2s1_2ch_lrck ++ &i2s1_2ch_sdo>; ++}; ++ ++&i2s2_2ch { ++ status = "okay"; ++ rockchip,bclk-fs = <64>; ++ #sound-dai-cells = <0>; ++}; ++ ++&io_domains { ++ status = "okay"; ++ ++ vccio1-supply = <&vcc1v8_soc>; ++ vccio2-supply = <&vccio_sd>; ++ vccio3-supply = <&vcc1v8_dvp>; ++ vccio4-supply = <&vcc1v8_soc>; ++ vccio5-supply = <&vcc_3v0>; ++}; ++ ++&isp_mmu { ++ status = "okay"; ++}; ++ ++&nandc0 { ++ status = "okay"; ++}; ++ ++&pdm { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pdm_clk0m1 ++ &pdm_clk1 ++ &pdm_sdi0m1 ++ &pdm_sdi1 ++ &pdm_sdi2 ++ &pdm_sdi3>; ++}; ++ ++&pinctrl { ++ headphone { ++ hp_det: hp-det { ++ rockchip,pins = <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ pmic { ++ pmic_int: pmic_int { ++ rockchip,pins = ++ <0 RK_PA7 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ soc_slppin_gpio: soc_slppin_gpio { ++ rockchip,pins = ++ <0 RK_PA4 RK_FUNC_GPIO &pcfg_output_low>; ++ }; ++ ++ soc_slppin_slp: soc_slppin_slp { ++ rockchip,pins = ++ <0 RK_PA4 1 &pcfg_pull_none>; ++ }; ++ ++ soc_slppin_rst: soc_slppin_rst { ++ rockchip,pins = ++ <0 RK_PA4 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <0 RK_PB3 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ ++ pmuio1-supply = <&vcc3v0_pmu>; ++ pmuio2-supply = <&vcc3v0_pmu>; ++}; ++ ++&pwm1 { ++ status = "okay"; ++}; ++ ++&rk_rga { ++ status = "okay"; ++}; ++ ++&rockchip_suspend { ++ status = "okay"; ++ rockchip,sleep-debug-en = <1>; ++}; ++ ++&saradc { ++ status = "okay"; ++ vref-supply = <&vcc1v8_soc>; ++}; ++ ++&sdmmc { ++ bus-width = <4>; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ no-sdio; ++ no-mmc; ++ card-detect-delay = <800>; ++ ignore-pm-notify; ++ /*cd-gpios = <&gpio2 4 GPIO_ACTIVE_HIGH>; [> CD GPIO <]*/ ++ sd-uhs-sdr12; ++ sd-uhs-sdr25; ++ sd-uhs-sdr50; ++ sd-uhs-sdr104; ++ vqmmc-supply = <&vccio_sd>; ++ vmmc-supply = <&vcc_sd>; ++ status = "okay"; ++}; ++ ++&sdio { ++ bus-width = <4>; ++ cap-sd-highspeed; ++ no-sd; ++ no-mmc; ++ ignore-pm-notify; ++ keep-power-in-suspend; ++ non-removable; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&tsadc { ++ pinctrl-names = "gpio", "otpout"; ++ pinctrl-0 = <&tsadc_otp_gpio>; ++ pinctrl-1 = <&tsadc_otp_out>; ++ status = "okay"; ++}; ++ ++&u2phy { ++ status = "okay"; ++ ++ u2phy_host: host-port { ++ status = "okay"; ++ }; ++ ++ u2phy_otg: otg-port { ++ status = "okay"; ++ }; ++}; ++ ++&usb20_otg { ++ status = "okay"; ++}; ++ ++&uart1 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart1_xfer &uart1_cts>; ++ status = "okay"; ++}; ++ ++&vopb { ++ status = "okay"; ++}; ++ ++&vopb_mmu { ++ status = "okay"; ++}; ++ ++&vopl { ++ status = "okay"; ++}; ++ ++&vopl_mmu { ++ status = "okay"; ++}; ++ ++&mpp_srv { ++ status = "okay"; ++}; ++ ++&vdpu { ++ status = "okay"; ++}; ++ ++&vepu { ++ status = "okay"; ++}; ++ ++&vpu_mmu { ++ status = "okay"; ++}; ++ ++&hevc { ++ status = "okay"; ++}; ++ ++&hevc_mmu { ++ status = "okay"; ++}; ++ ++&mipi_dphy_rx0 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ov5695_out>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_rx_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp_mipi_in>; ++ }; ++ }; ++ }; ++}; ++ ++&rkisp1 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_clkout_m0 &dvp_d0d1_m0 &dvp_d2d9_m0>; ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp_mipi_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dphy_rx_out>; ++ }; ++ ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3326-evb-ai-va-v11-i2s-dmic.dts b/arch/arm64/boot/dts/rockchip/rk3326-evb-ai-va-v11-i2s-dmic.dts +new file mode 100755 +index 000000000000..0af03d1ec2a8 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3326-evb-ai-va-v11-i2s-dmic.dts +@@ -0,0 +1,1330 @@ ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++ */ ++ ++/dts-v1/; ++#include ++#include ++#include ++#include ++#include ++#include "rk3326.dtsi" ++#include "rk3326-863-cif-sensor.dtsi" ++#include "px30-android.dtsi" ++ ++/ { ++ model = "Rockchip rk3326 ai voice assistant evb v11 i2s-dmic board"; ++ compatible = "rockchip,rk3326-evb-ai-va-v11-i2s-dmic", "rockchip,rk3326"; ++ ++ adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 2>; ++ io-channel-names = "buttons"; ++ poll-interval = <100>; ++ keyup-threshold-microvolt = <1800000>; ++ ++ mute-key { ++ linux,code = ; ++ label = "mute"; ++ press-threshold-microvolt = <1119000>; ++ }; ++ ++ mode-key { ++ linux,code = ; ++ label = "mode"; ++ press-threshold-microvolt = <892000>; ++ }; ++ ++ media-key { ++ linux,code = ; ++ label = "media"; ++ press-threshold-microvolt = <616000>; ++ }; ++ ++ vol-down-key { ++ linux,code = ; ++ label = "volume down"; ++ press-threshold-microvolt = <300000>; ++ }; ++ ++ vol-up-key { ++ linux,code = ; ++ label = "volume up"; ++ press-threshold-microvolt = <15000>; ++ }; ++ }; ++ ++ backlight: backlight { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm1 0 25000 0>; ++ brightness-levels = < ++ 0 1 2 3 4 5 6 7 ++ 8 9 10 11 12 13 14 15 ++ 16 17 18 19 20 21 22 23 ++ 24 25 26 27 28 29 30 31 ++ 32 33 34 35 36 37 38 39 ++ 40 41 42 43 44 45 46 47 ++ 48 49 50 51 52 53 54 55 ++ 56 57 58 59 60 61 62 63 ++ 64 65 66 67 68 69 70 71 ++ 72 73 74 75 76 77 78 79 ++ 80 81 82 83 84 85 86 87 ++ 88 89 90 91 92 93 94 95 ++ 96 97 98 99 100 101 102 103 ++ 104 105 106 107 108 109 110 111 ++ 112 113 114 115 116 117 118 119 ++ 120 121 122 123 124 125 126 127 ++ 128 129 130 131 132 133 134 135 ++ 136 137 138 139 140 141 142 143 ++ 144 145 146 147 148 149 150 151 ++ 152 153 154 155 156 157 158 159 ++ 160 161 162 163 164 165 166 167 ++ 168 169 170 171 172 173 174 175 ++ 176 177 178 179 180 181 182 183 ++ 184 185 186 187 188 189 190 191 ++ 192 193 194 195 196 197 198 199 ++ 200 201 202 203 204 205 206 207 ++ 208 209 210 211 212 213 214 215 ++ 216 217 218 219 220 221 222 223 ++ 224 225 226 227 228 229 230 231 ++ 232 233 234 235 236 237 238 239 ++ 240 241 242 243 244 245 246 247 ++ 248 249 250 251 252 253 254 255>; ++ default-brightness-level = <200>; ++ }; ++ ++ multi_dais: multi-dais { ++ status = "okay"; ++ compatible = "rockchip,multi-dais"; ++ dais = <&pdm>, <&i2s0_8ch>; ++ capture,channel-mapping = <2 6>; ++ playback,channel-mapping = <0 0>; ++ #sound-dai-cells = <0>; ++ }; ++ ++ rk809-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,name = "rockchip,rk809-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,widgets = ++ "Microphone", "Mic Jack", ++ "Headphone", "Headphone Jack"; ++ simple-audio-card,routing = ++ "Mic Jack", "MICBIAS1", ++ "IN1P", "Mic Jack", ++ "Headphone Jack", "HPOL", ++ "Headphone Jack", "HPOR"; ++ simple-audio-card,dai-link@0 { ++ format = "i2s"; ++ cpu { ++ sound-dai = <&i2s1_2ch>; ++ }; ++ codec { ++ sound-dai = <&rk809_codec 0>; ++ }; ++ }; ++ simple-audio-card,dai-link@1 { ++ format = "i2s"; ++ cpu { ++ sound-dai = <&multi_dais>; ++ }; ++ codec { ++ sound-dai = <&rk809_codec 1>; ++ }; ++ }; ++ }; ++ ++ bt-sound { ++ compatible = "simple-audio-card"; ++ status = "disabled"; ++ simple-audio-card,format = "dsp_a"; ++ simple-audio-card,bitclock-inversion = <1>; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,name = "rockchip,bt"; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s2_2ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&bt_sco>; ++ }; ++ }; ++ ++ bt_sco: bt-sco { ++ compatible = "delta,dfbmcs320"; ++ #sound-dai-cells = <0>; ++ status = "okay"; ++ }; ++ ++ rk_headset: rk-headset { ++ compatible = "rockchip_headset"; ++ headset_gpio = <&gpio3 RK_PA4 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hp_det>; ++ io-channels = <&saradc 1>; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ clocks = <&rk809 1>; ++ clock-names = "ext_clock"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_LOW>; /* GPIO3_A4 */ ++ }; ++ ++ test-power { ++ status = "okay"; ++ }; ++ ++ vcc5v0_sys: vccsys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc5v0_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ wifi_chip_type = "AP6255"; ++ WIFI,host_wake_irq = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&rk809 1>; ++ clock-names = "ext_clock"; ++ uart_rts_gpios = <&gpio1 RK_PC3 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default","rts_gpio"; ++ pinctrl-0 = <&uart1_rts>; ++ pinctrl-1 = <&uart1_rts_gpio>; ++ BT,reset_gpio = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio0 RK_PA1 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio0 RK_PA7 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++}; ++ ++&bus_apll { ++ bus-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&cpu0 { ++ cpu-supply = <&vdd_arm>; ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&display_subsystem { ++ status = "okay"; ++}; ++ ++&dmc { ++ center-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&dsi { ++ status = "okay"; ++ ++ panel@0 { ++ compatible = "sitronix,st7703", "simple-panel-dsi"; ++ reg = <0>; ++ backlight = <&backlight>; ++ enable-gpios = <&gpio0 RK_PB5 GPIO_ACTIVE_HIGH>; ++ prepare-delay-ms = <2>; ++ reset-delay-ms = <1>; ++ init-delay-ms = <20>; ++ enable-delay-ms = <120>; ++ disable-delay-ms = <50>; ++ unprepare-delay-ms = <40>; ++ ++ width-mm = <68>; ++ height-mm = <121>; ++ ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | ++ MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ ++ panel-init-sequence = [ ++ 05 fa 01 11 ++ 39 00 04 b9 f1 12 83 ++ 39 00 1c ba 33 81 05 f9 0e 0e 00 00 00 ++ 00 00 00 00 00 44 25 00 91 0a ++ 00 00 02 4f 01 00 00 37 ++ 15 00 02 b8 25 ++ 39 00 04 bf 02 11 00 ++ 39 00 0b b3 0c 10 0a 50 03 ff 00 00 00 ++ 00 ++ 39 00 0a c0 73 73 50 50 00 00 08 70 00 ++ 15 00 02 bc 46 ++ 15 00 02 cc 0b ++ 15 00 02 b4 80 ++ 39 00 04 b2 c8 12 30 ++ 39 00 0f e3 07 07 0b 0b 03 0b 00 00 00 ++ 00 ff 00 c0 10 ++ 39 00 0d c1 53 00 1e 1e 77 e1 cc dd 67 ++ 77 33 33 ++ 39 00 07 c6 00 00 ff ff 01 ff ++ 39 00 03 b5 09 09 ++ 39 00 03 b6 87 95 ++ 39 00 40 e9 c2 10 05 05 10 05 a0 12 31 ++ 23 3f 81 0a a0 37 18 00 80 01 ++ 00 00 00 00 80 01 00 00 00 48 ++ f8 86 42 08 88 88 80 88 88 88 ++ 58 f8 87 53 18 88 88 81 88 88 ++ 88 00 00 00 01 00 00 00 00 00 ++ 00 00 00 00 ++ 39 00 3e ea 00 1a 00 00 00 00 02 00 00 ++ 00 00 00 1f 88 81 35 78 88 88 ++ 85 88 88 88 0f 88 80 24 68 88 ++ 88 84 88 88 88 23 10 00 00 1c ++ 00 00 00 00 00 00 00 00 00 00 ++ 00 00 00 00 00 30 05 a0 00 00 ++ 00 00 ++ 39 00 23 e0 00 06 08 2a 31 3f 38 36 07 ++ 0c 0d 11 13 12 13 11 18 00 06 ++ 08 2a 31 3f 38 36 07 0c 0d 11 ++ 13 12 13 11 18 ++ 05 32 01 29 ++ ]; ++ ++ panel-exit-sequence = [ ++ 05 00 01 28 ++ 05 00 01 10 ++ ]; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <66000000>; ++ hactive = <720>; ++ vactive = <1280>; ++ hfront-porch = <40>; ++ hsync-len = <10>; ++ hback-porch = <40>; ++ vfront-porch = <22>; ++ vsync-len = <4>; ++ vback-porch = <11>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ panel_in_dsi: endpoint { ++ remote-endpoint = <&dsi_out_panel>; ++ }; ++ }; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@1 { ++ reg = <1>; ++ dsi_out_panel: endpoint { ++ remote-endpoint = <&panel_in_dsi>; ++ }; ++ }; ++ }; ++}; ++ ++&dsi_in_vopb { ++ status = "okay"; ++}; ++ ++&dsi_in_vopl { ++ status = "disabled"; ++}; ++ ++&route_dsi { ++ connect = <&vopb_out_dsi>; ++ status = "okay"; ++}; ++ ++&emmc { ++ bus-width = <8>; ++ cap-mmc-highspeed; ++ mmc-hs200-1_8v; ++ no-sdio; ++ no-sd; ++ disable-wp; ++ non-removable; ++ num-slots = <1>; ++ status = "okay"; ++}; ++ ++&gpu { ++ mali-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ clock-frequency = <400000>; ++ i2c-scl-rising-time-ns = <280>; ++ i2c-scl-falling-time-ns = <16>; ++ ++ rk809: pmic@20 { ++ compatible = "rockchip,rk809"; ++ reg = <0x20>; ++ interrupt-parent = <&gpio0>; ++ interrupts = <10 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default", "pmic-sleep", ++ "pmic-power-off", "pmic-reset"; ++ pinctrl-0 = <&pmic_int>; ++ pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; ++ pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; ++ pinctrl-3 = <&soc_slppin_rst>, <&rk817_slppin_rst>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ //fb-inner-reg-idxs = <2>; ++ /* 1: rst regs (default in codes), 0: rst the pmic */ ++ pmic-reset-func = <1>; ++ ++ vcc1-supply = <&vcc5v0_sys>; ++ vcc2-supply = <&vcc5v0_sys>; ++ vcc3-supply = <&vcc5v0_sys>; ++ vcc4-supply = <&vcc5v0_sys>; ++ vcc5-supply = <&vcc3v3_sys>; ++ vcc6-supply = <&vcc3v3_sys>; ++ vcc7-supply = <&vcc3v3_sys>; ++ vcc8-supply = <&vcc3v3_sys>; ++ vcc9-supply = <&vcc5v0_sys>; ++ ++ pwrkey { ++ status = "okay"; ++ }; ++ ++ pinctrl_rk8xx: pinctrl_rk8xx { ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ rk817_slppin_null: rk817_slppin_null { ++ pins = "gpio_slp"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_slppin_slp: rk817_slppin_slp { ++ pins = "gpio_slp"; ++ function = "pin_fun1"; ++ }; ++ ++ rk817_slppin_pwrdn: rk817_slppin_pwrdn { ++ pins = "gpio_slp"; ++ function = "pin_fun2"; ++ }; ++ ++ rk817_slppin_rst: rk817_slppin_rst { ++ pins = "gpio_slp"; ++ function = "pin_fun3"; ++ }; ++ }; ++ ++ regulators { ++ vdd_logic: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <950000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_logic"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <950000>; ++ }; ++ }; ++ ++ vdd_arm: DCDC_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <950000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_arm"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <950000>; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_ddr"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_3v0: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_3v0"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ ++ vcc_1v0: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ regulator-name = "vcc_1v0"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc1v8_soc: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcc1v8_soc"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd1v0_soc: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ ++ regulator-name = "vcc1v0_soc"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc3v0_pmu: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ ++ regulator-name = "vcc3v0_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ ++ }; ++ }; ++ ++ vccio_sd: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vccio_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc_sd: LDO_REG6 { ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vcc_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ ++ }; ++ }; ++ ++ vcc2v8_dvp: LDO_REG7 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <2800000>; ++ regulator-max-microvolt = <2800000>; ++ ++ regulator-name = "vcc2v8_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <2800000>; ++ }; ++ }; ++ ++ vcc1v8_dvp: LDO_REG8 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcc1v8_dvp"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd1v5_dvp: LDO_REG9 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ ++ regulator-name = "vdd1v5_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <1500000>; ++ }; ++ }; ++ ++ vcc3v3_sys: DCDC_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcc3v3_sys"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc5v0_host: SWITCH_REG1 { ++ regulator-name = "vcc5v0_host"; ++ }; ++ ++ vcc3v3_lcd: SWITCH_REG2 { ++ regulator-boot-on; ++ regulator-name = "vcc3v3_lcd"; ++ }; ++ }; ++ ++ rk809_codec: codec { ++ #sound-dai-cells = <1>; ++ compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; ++ clocks = <&cru SCLK_I2S1_OUT>; ++ clock-names = "mclk"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1_2ch_mclk>; ++ pdmdata-out-enable; ++ use-ext-amplifier; ++ adc-for-loopback; ++ spk-ctl-gpios = <&gpio3 RK_PB2 GPIO_ACTIVE_HIGH>; ++ hp-volume = <20>; ++ spk-volume = <20>; ++ }; ++ }; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ clock-frequency = <400000>; ++ i2c-scl-rising-time-ns = <275>; ++ i2c-scl-falling-time-ns = <16>; ++ ++ gt1x: gt1x@14 { ++ compatible = "goodix,gt1x"; ++ reg = <0x14>; ++ goodix,rst-gpio = <&gpio0 RK_PB4 GPIO_ACTIVE_HIGH>; ++ goodix,irq-gpio = <&gpio0 RK_PB3 IRQ_TYPE_LEVEL_LOW>; ++ }; ++ ++ is31fl3236: led-controller@3c { ++ compatible = "issi,is31fl3236"; ++ reg = <0x3c>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ reset-gpios = <&gpio3 RK_PA6 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ ++ led1: led@1 { ++ label = "led1"; ++ reg = <1>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <0>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led2: led@2 { ++ label = "led2"; ++ reg = <2>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <0>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led3: led@3 { ++ label = "led3"; ++ reg = <3>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led4: led@4 { ++ label = "led4"; ++ reg = <4>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <100>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led5: led@5 { ++ label = "led5"; ++ reg = <5>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <100>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led6: led@6 { ++ label = "led6"; ++ reg = <6>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led7: led@7 { ++ label = "led7"; ++ reg = <7>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <200>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led8: led@8 { ++ label = "led8"; ++ reg = <8>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <200>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led9: led@9 { ++ label = "led9"; ++ reg = <9>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led10: led@10 { ++ label = "led10"; ++ reg = <10>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <300>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led11: led@11 { ++ label = "led11"; ++ reg = <11>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <300>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led12: led@12 { ++ label = "led12"; ++ reg = <12>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led13: led@13 { ++ label = "led13"; ++ reg = <13>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <400>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led14: led@14 { ++ label = "led14"; ++ reg = <14>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <400>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led15: led@15 { ++ label = "led15"; ++ reg = <15>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led16: led@16 { ++ label = "led16"; ++ reg = <16>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <500>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led17: led@17 { ++ label = "led17"; ++ reg = <17>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <500>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led18: led@18 { ++ label = "led18"; ++ reg = <18>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led19: led@19 { ++ label = "led19"; ++ reg = <19>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <600>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led20: led@20 { ++ label = "led20"; ++ reg = <20>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <600>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led21: led@21 { ++ label = "led21"; ++ reg = <21>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led22: led@22 { ++ label = "led22"; ++ reg = <22>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <700>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led23: led@23 { ++ label = "led23"; ++ reg = <23>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <700>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led124: led@24 { ++ label = "led24"; ++ reg = <24>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led25: led@25 { ++ label = "led25"; ++ reg = <25>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <800>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led26: led@26 { ++ label = "led26"; ++ reg = <26>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <800>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led27: led@27 { ++ label = "led27"; ++ reg = <27>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led28: led@28 { ++ label = "led28"; ++ reg = <28>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <900>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led29: led@29 { ++ label = "led29"; ++ reg = <29>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <900>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led30: led@30 { ++ label = "led30"; ++ reg = <30>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led31: led@31 { ++ label = "led31"; ++ reg = <31>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <1000>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led32: led@32 { ++ label = "led32"; ++ reg = <32>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <1000>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led33: led@33 { ++ label = "led33"; ++ reg = <33>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led34: led@34 { ++ label = "led34"; ++ reg = <34>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <1100>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led35: led@35 { ++ label = "led35"; ++ reg = <35>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <1100>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led36: led@36 { ++ label = "led36"; ++ reg = <36>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ }; ++ ++ ls_stk3410: light@48 { ++ compatible = "ls_stk3410"; ++ status = "okay"; ++ reg = <0x48>; ++ type = ; ++ irq_enable = <0>; ++ als_threshold_high = <100>; ++ als_threshold_low = <10>; ++ als_ctrl_gain = <2>; /* 0:x1 1:x4 2:x16 3:x64 */ ++ poll_delay_ms = <100>; ++ }; ++ ++ ps_stk3410: proximity@48 { ++ compatible = "ps_stk3410"; ++ status = "okay"; ++ reg = <0x48>; ++ type = ; ++ //pinctrl-names = "default"; ++ //pinctrl-0 = <&gpio2_c3>; ++ //irq-gpio = <&gpio0 RK_PB7 IRQ_TYPE_LEVEL_LOW>; ++ //irq_enable = <1>; ++ ps_threshold_high = <0x200>; ++ ps_threshold_low = <0x100>; ++ ps_ctrl_gain = <3>; /* 0:x1 1:x4 2:x16 3:x64 */ ++ ps_led_current = <3>; /* 0:12.5mA 1:25mA 2:50mA 3:100mA */ ++ poll_delay_ms = <100>; ++ }; ++ ++}; ++ ++&i2c2 { ++ status = "okay"; ++ ++ clock-frequency = <100000>; ++ ++ /* These are relatively safe rise/fall times; TODO: measure */ ++ i2c-scl-falling-time-ns = <50>; ++ i2c-scl-rising-time-ns = <300>; ++ ++ ov5695: ov5695@36 { ++ compatible = "ovti,ov5695"; ++ reg = <0x36>; ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ /*reset-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>;*/ ++ pwdn-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>; ++ //pinctrl-names = "default"; ++ //pinctrl-0 = <&cif_clkout_m0>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "TongJu"; ++ rockchip,camera-module-lens-name = "CHT842-MD"; ++ port { ++ ov5695_out: endpoint { ++ remote-endpoint = <&mipi_in>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++}; ++ ++&i2s0_8ch { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++ rockchip,no-dmaengine; ++}; ++ ++&i2s1_2ch { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++ pinctrl-0 = <&i2s1_2ch_sclk ++ &i2s1_2ch_lrck ++ &i2s1_2ch_sdo>; ++}; ++ ++&i2s2_2ch { ++ status = "okay"; ++ rockchip,bclk-fs = <64>; ++ #sound-dai-cells = <0>; ++}; ++ ++&io_domains { ++ status = "okay"; ++ ++ vccio1-supply = <&vcc1v8_soc>; ++ vccio2-supply = <&vccio_sd>; ++ vccio3-supply = <&vcc1v8_dvp>; ++ vccio4-supply = <&vcc1v8_soc>; ++ vccio5-supply = <&vcc_3v0>; ++}; ++ ++&isp_mmu { ++ status = "okay"; ++}; ++ ++&nandc0 { ++ status = "okay"; ++}; ++ ++&pdm { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++ rockchip,no-dmaengine; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pdm_clk0m1 ++ &pdm_sdi0m1>; ++}; ++ ++&pinctrl { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&tp_int>; ++ ++ headphone { ++ hp_det: hp-det { ++ rockchip,pins = <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ pmic { ++ pmic_int: pmic_int { ++ rockchip,pins = ++ <0 RK_PB2 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ soc_slppin_gpio: soc_slppin_gpio { ++ rockchip,pins = ++ <0 RK_PA4 RK_FUNC_GPIO &pcfg_output_low>; ++ }; ++ ++ soc_slppin_slp: soc_slppin_slp { ++ rockchip,pins = ++ <0 RK_PA4 1 &pcfg_pull_none>; ++ }; ++ ++ soc_slppin_rst: soc_slppin_rst { ++ rockchip,pins = ++ <0 RK_PA4 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ touchscreen-int { ++ tp_int: tp-int { ++ rockchip,pins = <0 RK_PB3 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ ++ pmuio1-supply = <&vcc3v0_pmu>; ++ pmuio2-supply = <&vcc3v0_pmu>; ++}; ++ ++&pwm1 { ++ status = "okay"; ++}; ++ ++&rk_rga { ++ status = "okay"; ++}; ++ ++&rockchip_suspend { ++ status = "okay"; ++ rockchip,sleep-debug-en = <1>; ++}; ++ ++&saradc { ++ status = "okay"; ++ vref-supply = <&vcc1v8_soc>; ++}; ++ ++&sdmmc { ++ bus-width = <4>; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ no-sdio; ++ no-mmc; ++ card-detect-delay = <800>; ++ ignore-pm-notify; ++ /*cd-gpios = <&gpio2 4 GPIO_ACTIVE_HIGH>; [> CD GPIO <]*/ ++ sd-uhs-sdr12; ++ sd-uhs-sdr25; ++ sd-uhs-sdr50; ++ sd-uhs-sdr104; ++ vqmmc-supply = <&vccio_sd>; ++ vmmc-supply = <&vcc_sd>; ++ status = "okay"; ++}; ++ ++&sdio { ++ bus-width = <4>; ++ cap-sd-highspeed; ++ no-sd; ++ no-mmc; ++ ignore-pm-notify; ++ keep-power-in-suspend; ++ non-removable; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&tsadc { ++ pinctrl-names = "gpio", "otpout"; ++ pinctrl-0 = <&tsadc_otp_gpio>; ++ pinctrl-1 = <&tsadc_otp_out>; ++ status = "okay"; ++}; ++ ++&u2phy { ++ status = "okay"; ++ ++ u2phy_host: host-port { ++ status = "okay"; ++ }; ++ ++ u2phy_otg: otg-port { ++ vbus-supply = <&vcc5v0_host>; ++ status = "okay"; ++ }; ++}; ++ ++&usb20_otg { ++ status = "okay"; ++}; ++ ++&uart1 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart1_xfer &uart1_cts>; ++ status = "okay"; ++}; ++ ++&vopb { ++ status = "okay"; ++}; ++ ++&vopb_mmu { ++ status = "okay"; ++}; ++ ++&vopl { ++ status = "okay"; ++}; ++ ++&vopl_mmu { ++ status = "okay"; ++}; ++ ++&mpp_srv { ++ status = "okay"; ++}; ++ ++&vdpu { ++ status = "okay"; ++}; ++ ++&vepu { ++ status = "okay"; ++}; ++ ++&vpu_mmu { ++ status = "okay"; ++}; ++ ++&hevc { ++ status = "okay"; ++}; ++ ++&hevc_mmu { ++ status = "okay"; ++}; ++ ++&mipi_dphy_rx0 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ov5695_out>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_rx_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp_mipi_in>; ++ }; ++ }; ++ }; ++}; ++ ++&rkisp1 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_clkout_m0 &dvp_d0d1_m0 &dvp_d2d9_m0>; ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp_mipi_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dphy_rx_out>; ++ }; ++ ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3326-evb-ai-va-v11.dts b/arch/arm64/boot/dts/rockchip/rk3326-evb-ai-va-v11.dts +new file mode 100755 +index 000000000000..fa8b7cff619f +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3326-evb-ai-va-v11.dts +@@ -0,0 +1,1317 @@ ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++ */ ++ ++/dts-v1/; ++#include ++#include ++#include ++#include ++#include ++#include "rk3326.dtsi" ++#include "rk3326-863-cif-sensor.dtsi" ++#include "px30-android.dtsi" ++ ++/ { ++ model = "Rockchip rk3326 ai voice assistant evb v11 board"; ++ compatible = "rockchip,rk3326-evb-ai-va-v11", "rockchip,rk3326"; ++ ++ adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 2>; ++ io-channel-names = "buttons"; ++ poll-interval = <100>; ++ keyup-threshold-microvolt = <1800000>; ++ ++ mute-key { ++ linux,code = ; ++ label = "mute"; ++ press-threshold-microvolt = <1119000>; ++ }; ++ ++ mode-key { ++ linux,code = ; ++ label = "mode"; ++ press-threshold-microvolt = <892000>; ++ }; ++ ++ media-key { ++ linux,code = ; ++ label = "media"; ++ press-threshold-microvolt = <616000>; ++ }; ++ ++ vol-down-key { ++ linux,code = ; ++ label = "volume down"; ++ press-threshold-microvolt = <300000>; ++ }; ++ ++ vol-up-key { ++ linux,code = ; ++ label = "volume up"; ++ press-threshold-microvolt = <15000>; ++ }; ++ }; ++ ++ backlight: backlight { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm1 0 25000 0>; ++ brightness-levels = < ++ 0 1 2 3 4 5 6 7 ++ 8 9 10 11 12 13 14 15 ++ 16 17 18 19 20 21 22 23 ++ 24 25 26 27 28 29 30 31 ++ 32 33 34 35 36 37 38 39 ++ 40 41 42 43 44 45 46 47 ++ 48 49 50 51 52 53 54 55 ++ 56 57 58 59 60 61 62 63 ++ 64 65 66 67 68 69 70 71 ++ 72 73 74 75 76 77 78 79 ++ 80 81 82 83 84 85 86 87 ++ 88 89 90 91 92 93 94 95 ++ 96 97 98 99 100 101 102 103 ++ 104 105 106 107 108 109 110 111 ++ 112 113 114 115 116 117 118 119 ++ 120 121 122 123 124 125 126 127 ++ 128 129 130 131 132 133 134 135 ++ 136 137 138 139 140 141 142 143 ++ 144 145 146 147 148 149 150 151 ++ 152 153 154 155 156 157 158 159 ++ 160 161 162 163 164 165 166 167 ++ 168 169 170 171 172 173 174 175 ++ 176 177 178 179 180 181 182 183 ++ 184 185 186 187 188 189 190 191 ++ 192 193 194 195 196 197 198 199 ++ 200 201 202 203 204 205 206 207 ++ 208 209 210 211 212 213 214 215 ++ 216 217 218 219 220 221 222 223 ++ 224 225 226 227 228 229 230 231 ++ 232 233 234 235 236 237 238 239 ++ 240 241 242 243 244 245 246 247 ++ 248 249 250 251 252 253 254 255>; ++ default-brightness-level = <200>; ++ }; ++ ++ rk809-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,name = "rockchip,rk809-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,widgets = ++ "Microphone", "Mic Jack", ++ "Headphone", "Headphone Jack"; ++ simple-audio-card,routing = ++ "Mic Jack", "MICBIAS1", ++ "IN1P", "Mic Jack", ++ "Headphone Jack", "HPOL", ++ "Headphone Jack", "HPOR"; ++ simple-audio-card,dai-link@0 { ++ format = "i2s"; ++ cpu { ++ sound-dai = <&i2s1_2ch>; ++ }; ++ codec { ++ sound-dai = <&rk809_codec 0>; ++ }; ++ }; ++ simple-audio-card,dai-link@1 { ++ format = "pdm"; ++ cpu { ++ sound-dai = <&pdm>; ++ }; ++ codec { ++ sound-dai = <&rk809_codec 1>; ++ }; ++ }; ++ }; ++ ++ bt-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "dsp_a"; ++ simple-audio-card,bitclock-inversion = <1>; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,name = "rockchip,bt"; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s2_2ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&bt_sco>; ++ }; ++ }; ++ ++ bt_sco: bt-sco { ++ compatible = "delta,dfbmcs320"; ++ #sound-dai-cells = <0>; ++ status = "okay"; ++ }; ++ ++ rk_headset: rk-headset { ++ compatible = "rockchip_headset"; ++ headset_gpio = <&gpio3 RK_PA4 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hp_det>; ++ io-channels = <&saradc 1>; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ clocks = <&rk809 1>; ++ clock-names = "ext_clock"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_LOW>; /* GPIO3_A4 */ ++ }; ++ ++ test-power { ++ status = "okay"; ++ }; ++ ++ vcc5v0_sys: vccsys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc5v0_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ wifi_chip_type = "AP6255"; ++ WIFI,host_wake_irq = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&rk809 1>; ++ clock-names = "ext_clock"; ++ uart_rts_gpios = <&gpio1 RK_PC3 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default","rts_gpio"; ++ pinctrl-0 = <&uart1_rts>; ++ pinctrl-1 = <&uart1_rts_gpio>; ++ BT,reset_gpio = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio0 RK_PA1 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio0 RK_PA7 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++}; ++ ++&bus_apll { ++ bus-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&cpu0 { ++ cpu-supply = <&vdd_arm>; ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&display_subsystem { ++ status = "okay"; ++}; ++ ++&dmc { ++ center-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&dsi { ++ status = "okay"; ++ ++ panel@0 { ++ compatible = "sitronix,st7703", "simple-panel-dsi"; ++ reg = <0>; ++ backlight = <&backlight>; ++ enable-gpios = <&gpio0 RK_PB5 GPIO_ACTIVE_HIGH>; ++ prepare-delay-ms = <2>; ++ reset-delay-ms = <1>; ++ init-delay-ms = <20>; ++ enable-delay-ms = <120>; ++ disable-delay-ms = <50>; ++ unprepare-delay-ms = <40>; ++ ++ width-mm = <68>; ++ height-mm = <121>; ++ ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | ++ MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ ++ panel-init-sequence = [ ++ 05 fa 01 11 ++ 39 00 04 b9 f1 12 83 ++ 39 00 1c ba 33 81 05 f9 0e 0e 00 00 00 ++ 00 00 00 00 00 44 25 00 91 0a ++ 00 00 02 4f 01 00 00 37 ++ 15 00 02 b8 25 ++ 39 00 04 bf 02 11 00 ++ 39 00 0b b3 0c 10 0a 50 03 ff 00 00 00 ++ 00 ++ 39 00 0a c0 73 73 50 50 00 00 08 70 00 ++ 15 00 02 bc 46 ++ 15 00 02 cc 0b ++ 15 00 02 b4 80 ++ 39 00 04 b2 c8 12 30 ++ 39 00 0f e3 07 07 0b 0b 03 0b 00 00 00 ++ 00 ff 00 c0 10 ++ 39 00 0d c1 53 00 1e 1e 77 e1 cc dd 67 ++ 77 33 33 ++ 39 00 07 c6 00 00 ff ff 01 ff ++ 39 00 03 b5 09 09 ++ 39 00 03 b6 87 95 ++ 39 00 40 e9 c2 10 05 05 10 05 a0 12 31 ++ 23 3f 81 0a a0 37 18 00 80 01 ++ 00 00 00 00 80 01 00 00 00 48 ++ f8 86 42 08 88 88 80 88 88 88 ++ 58 f8 87 53 18 88 88 81 88 88 ++ 88 00 00 00 01 00 00 00 00 00 ++ 00 00 00 00 ++ 39 00 3e ea 00 1a 00 00 00 00 02 00 00 ++ 00 00 00 1f 88 81 35 78 88 88 ++ 85 88 88 88 0f 88 80 24 68 88 ++ 88 84 88 88 88 23 10 00 00 1c ++ 00 00 00 00 00 00 00 00 00 00 ++ 00 00 00 00 00 30 05 a0 00 00 ++ 00 00 ++ 39 00 23 e0 00 06 08 2a 31 3f 38 36 07 ++ 0c 0d 11 13 12 13 11 18 00 06 ++ 08 2a 31 3f 38 36 07 0c 0d 11 ++ 13 12 13 11 18 ++ 05 32 01 29 ++ ]; ++ ++ panel-exit-sequence = [ ++ 05 00 01 28 ++ 05 00 01 10 ++ ]; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <66000000>; ++ hactive = <720>; ++ vactive = <1280>; ++ hfront-porch = <40>; ++ hsync-len = <10>; ++ hback-porch = <40>; ++ vfront-porch = <22>; ++ vsync-len = <4>; ++ vback-porch = <11>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ panel_in_dsi: endpoint { ++ remote-endpoint = <&dsi_out_panel>; ++ }; ++ }; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@1 { ++ reg = <1>; ++ dsi_out_panel: endpoint { ++ remote-endpoint = <&panel_in_dsi>; ++ }; ++ }; ++ }; ++}; ++ ++&dsi_in_vopb { ++ status = "okay"; ++}; ++ ++&dsi_in_vopl { ++ status = "disabled"; ++}; ++ ++&route_dsi { ++ connect = <&vopb_out_dsi>; ++ status = "okay"; ++}; ++ ++&emmc { ++ bus-width = <8>; ++ cap-mmc-highspeed; ++ mmc-hs200-1_8v; ++ no-sdio; ++ no-sd; ++ disable-wp; ++ non-removable; ++ num-slots = <1>; ++ status = "okay"; ++}; ++ ++&gpu { ++ mali-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ clock-frequency = <400000>; ++ i2c-scl-rising-time-ns = <280>; ++ i2c-scl-falling-time-ns = <16>; ++ ++ rk809: pmic@20 { ++ compatible = "rockchip,rk809"; ++ reg = <0x20>; ++ interrupt-parent = <&gpio0>; ++ interrupts = <10 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default", "pmic-sleep", ++ "pmic-power-off", "pmic-reset"; ++ pinctrl-0 = <&pmic_int>; ++ pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; ++ pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; ++ pinctrl-3 = <&soc_slppin_rst>, <&rk817_slppin_rst>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ //fb-inner-reg-idxs = <2>; ++ /* 1: rst regs (default in codes), 0: rst the pmic */ ++ pmic-reset-func = <1>; ++ ++ vcc1-supply = <&vcc5v0_sys>; ++ vcc2-supply = <&vcc5v0_sys>; ++ vcc3-supply = <&vcc5v0_sys>; ++ vcc4-supply = <&vcc5v0_sys>; ++ vcc5-supply = <&vcc3v3_sys>; ++ vcc6-supply = <&vcc3v3_sys>; ++ vcc7-supply = <&vcc3v3_sys>; ++ vcc8-supply = <&vcc3v3_sys>; ++ vcc9-supply = <&vcc5v0_sys>; ++ ++ pwrkey { ++ status = "okay"; ++ }; ++ ++ pinctrl_rk8xx: pinctrl_rk8xx { ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ rk817_slppin_null: rk817_slppin_null { ++ pins = "gpio_slp"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_slppin_slp: rk817_slppin_slp { ++ pins = "gpio_slp"; ++ function = "pin_fun1"; ++ }; ++ ++ rk817_slppin_pwrdn: rk817_slppin_pwrdn { ++ pins = "gpio_slp"; ++ function = "pin_fun2"; ++ }; ++ ++ rk817_slppin_rst: rk817_slppin_rst { ++ pins = "gpio_slp"; ++ function = "pin_fun3"; ++ }; ++ }; ++ ++ regulators { ++ vdd_logic: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <950000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_logic"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <950000>; ++ }; ++ }; ++ ++ vdd_arm: DCDC_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <950000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_arm"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <950000>; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_ddr"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_3v0: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_3v0"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ ++ vcc_1v0: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ regulator-name = "vcc_1v0"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc1v8_soc: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcc1v8_soc"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd1v0_soc: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ ++ regulator-name = "vcc1v0_soc"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc3v0_pmu: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ ++ regulator-name = "vcc3v0_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ ++ }; ++ }; ++ ++ vccio_sd: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vccio_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc_sd: LDO_REG6 { ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vcc_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ ++ }; ++ }; ++ ++ vcc2v8_dvp: LDO_REG7 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <2800000>; ++ regulator-max-microvolt = <2800000>; ++ ++ regulator-name = "vcc2v8_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <2800000>; ++ }; ++ }; ++ ++ vcc1v8_dvp: LDO_REG8 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcc1v8_dvp"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd1v5_dvp: LDO_REG9 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ ++ regulator-name = "vdd1v5_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <1500000>; ++ }; ++ }; ++ ++ vcc3v3_sys: DCDC_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcc3v3_sys"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc5v0_host: SWITCH_REG1 { ++ regulator-name = "vcc5v0_host"; ++ }; ++ ++ vcc3v3_lcd: SWITCH_REG2 { ++ regulator-boot-on; ++ regulator-name = "vcc3v3_lcd"; ++ }; ++ }; ++ ++ rk809_codec: codec { ++ #sound-dai-cells = <1>; ++ compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; ++ clocks = <&cru SCLK_I2S1_OUT>; ++ clock-names = "mclk"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1_2ch_mclk>; ++ pdmdata-out-enable; ++ use-ext-amplifier; ++ adc-for-loopback; ++ spk-ctl-gpios = <&gpio3 RK_PB2 GPIO_ACTIVE_HIGH>; ++ hp-volume = <20>; ++ spk-volume = <20>; ++ }; ++ }; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ clock-frequency = <400000>; ++ i2c-scl-rising-time-ns = <275>; ++ i2c-scl-falling-time-ns = <16>; ++ ++ gt1x: gt1x@14 { ++ compatible = "goodix,gt1x"; ++ reg = <0x14>; ++ goodix,rst-gpio = <&gpio0 RK_PB4 GPIO_ACTIVE_HIGH>; ++ goodix,irq-gpio = <&gpio0 RK_PB3 IRQ_TYPE_LEVEL_LOW>; ++ }; ++ ++ is31fl3236: led-controller@3c { ++ compatible = "issi,is31fl3236"; ++ reg = <0x3c>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ reset-gpios = <&gpio3 RK_PA6 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ ++ led1: led@1 { ++ label = "led1"; ++ reg = <1>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <0>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led2: led@2 { ++ label = "led2"; ++ reg = <2>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <0>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led3: led@3 { ++ label = "led3"; ++ reg = <3>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led4: led@4 { ++ label = "led4"; ++ reg = <4>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <100>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led5: led@5 { ++ label = "led5"; ++ reg = <5>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <100>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led6: led@6 { ++ label = "led6"; ++ reg = <6>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led7: led@7 { ++ label = "led7"; ++ reg = <7>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <200>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led8: led@8 { ++ label = "led8"; ++ reg = <8>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <200>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led9: led@9 { ++ label = "led9"; ++ reg = <9>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led10: led@10 { ++ label = "led10"; ++ reg = <10>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <300>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led11: led@11 { ++ label = "led11"; ++ reg = <11>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <300>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led12: led@12 { ++ label = "led12"; ++ reg = <12>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led13: led@13 { ++ label = "led13"; ++ reg = <13>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <400>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led14: led@14 { ++ label = "led14"; ++ reg = <14>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <400>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led15: led@15 { ++ label = "led15"; ++ reg = <15>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led16: led@16 { ++ label = "led16"; ++ reg = <16>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <500>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led17: led@17 { ++ label = "led17"; ++ reg = <17>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <500>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led18: led@18 { ++ label = "led18"; ++ reg = <18>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led19: led@19 { ++ label = "led19"; ++ reg = <19>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <600>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led20: led@20 { ++ label = "led20"; ++ reg = <20>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <600>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led21: led@21 { ++ label = "led21"; ++ reg = <21>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led22: led@22 { ++ label = "led22"; ++ reg = <22>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <700>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led23: led@23 { ++ label = "led23"; ++ reg = <23>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <700>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led124: led@24 { ++ label = "led24"; ++ reg = <24>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led25: led@25 { ++ label = "led25"; ++ reg = <25>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <800>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led26: led@26 { ++ label = "led26"; ++ reg = <26>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <800>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led27: led@27 { ++ label = "led27"; ++ reg = <27>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led28: led@28 { ++ label = "led28"; ++ reg = <28>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <900>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led29: led@29 { ++ label = "led29"; ++ reg = <29>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <900>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led30: led@30 { ++ label = "led30"; ++ reg = <30>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led31: led@31 { ++ label = "led31"; ++ reg = <31>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <1000>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led32: led@32 { ++ label = "led32"; ++ reg = <32>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <1000>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led33: led@33 { ++ label = "led33"; ++ reg = <33>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led34: led@34 { ++ label = "led34"; ++ reg = <34>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <1100>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led35: led@35 { ++ label = "led35"; ++ reg = <35>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <1100>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led36: led@36 { ++ label = "led36"; ++ reg = <36>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ }; ++ ++ ls_stk3410: light@48 { ++ compatible = "ls_stk3410"; ++ status = "okay"; ++ reg = <0x48>; ++ type = ; ++ irq_enable = <0>; ++ als_threshold_high = <100>; ++ als_threshold_low = <10>; ++ als_ctrl_gain = <2>; /* 0:x1 1:x4 2:x16 3:x64 */ ++ poll_delay_ms = <100>; ++ }; ++ ++ ps_stk3410: proximity@48 { ++ compatible = "ps_stk3410"; ++ status = "okay"; ++ reg = <0x48>; ++ type = ; ++ //pinctrl-names = "default"; ++ //pinctrl-0 = <&gpio2_c3>; ++ //irq-gpio = <&gpio0 RK_PB7 IRQ_TYPE_LEVEL_LOW>; ++ //irq_enable = <1>; ++ ps_threshold_high = <0x200>; ++ ps_threshold_low = <0x100>; ++ ps_ctrl_gain = <3>; /* 0:x1 1:x4 2:x16 3:x64 */ ++ ps_led_current = <3>; /* 0:12.5mA 1:25mA 2:50mA 3:100mA */ ++ poll_delay_ms = <100>; ++ }; ++ ++}; ++ ++&i2c2 { ++ status = "okay"; ++ ++ clock-frequency = <100000>; ++ ++ /* These are relatively safe rise/fall times; TODO: measure */ ++ i2c-scl-falling-time-ns = <50>; ++ i2c-scl-rising-time-ns = <300>; ++ ++ ov5695: ov5695@36 { ++ compatible = "ovti,ov5695"; ++ reg = <0x36>; ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ /*reset-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>;*/ ++ pwdn-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>; ++ //pinctrl-names = "default"; ++ //pinctrl-0 = <&cif_clkout_m0>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "TongJu"; ++ rockchip,camera-module-lens-name = "CHT842-MD"; ++ port { ++ ov5695_out: endpoint { ++ remote-endpoint = <&mipi_in>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++}; ++ ++&i2s1_2ch { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++ pinctrl-0 = <&i2s1_2ch_sclk ++ &i2s1_2ch_lrck ++ &i2s1_2ch_sdo>; ++}; ++ ++&i2s2_2ch { ++ status = "okay"; ++ rockchip,bclk-fs = <64>; ++ #sound-dai-cells = <0>; ++}; ++ ++&io_domains { ++ status = "okay"; ++ ++ vccio1-supply = <&vcc1v8_soc>; ++ vccio2-supply = <&vccio_sd>; ++ vccio3-supply = <&vcc1v8_dvp>; ++ vccio4-supply = <&vcc1v8_soc>; ++ vccio5-supply = <&vcc_3v0>; ++}; ++ ++&isp_mmu { ++ status = "okay"; ++}; ++ ++&nandc0 { ++ status = "okay"; ++}; ++ ++&pdm { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pdm_clk0m1 ++ &pdm_clk1 ++ &pdm_sdi0m1 ++ &pdm_sdi1 ++ &pdm_sdi2 ++ &pdm_sdi3>; ++}; ++ ++&pinctrl { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&tp_int>; ++ ++ headphone { ++ hp_det: hp-det { ++ rockchip,pins = <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ pmic { ++ pmic_int: pmic_int { ++ rockchip,pins = ++ <0 RK_PB2 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ soc_slppin_gpio: soc_slppin_gpio { ++ rockchip,pins = ++ <0 RK_PA4 RK_FUNC_GPIO &pcfg_output_low>; ++ }; ++ ++ soc_slppin_slp: soc_slppin_slp { ++ rockchip,pins = ++ <0 RK_PA4 1 &pcfg_pull_none>; ++ }; ++ ++ soc_slppin_rst: soc_slppin_rst { ++ rockchip,pins = ++ <0 RK_PA4 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ touchscreen-int { ++ tp_int: tp-int { ++ rockchip,pins = <0 RK_PB3 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ ++ pmuio1-supply = <&vcc3v0_pmu>; ++ pmuio2-supply = <&vcc3v0_pmu>; ++}; ++ ++&pwm1 { ++ status = "okay"; ++}; ++ ++&rk_rga { ++ status = "okay"; ++}; ++ ++&rockchip_suspend { ++ status = "okay"; ++ rockchip,sleep-debug-en = <1>; ++}; ++ ++&saradc { ++ status = "okay"; ++ vref-supply = <&vcc1v8_soc>; ++}; ++ ++&sdmmc { ++ bus-width = <4>; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ no-sdio; ++ no-mmc; ++ card-detect-delay = <800>; ++ ignore-pm-notify; ++ /*cd-gpios = <&gpio2 4 GPIO_ACTIVE_HIGH>; [> CD GPIO <]*/ ++ sd-uhs-sdr12; ++ sd-uhs-sdr25; ++ sd-uhs-sdr50; ++ sd-uhs-sdr104; ++ vqmmc-supply = <&vccio_sd>; ++ vmmc-supply = <&vcc_sd>; ++ status = "okay"; ++}; ++ ++&sdio { ++ bus-width = <4>; ++ cap-sd-highspeed; ++ no-sd; ++ no-mmc; ++ ignore-pm-notify; ++ keep-power-in-suspend; ++ non-removable; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&tsadc { ++ pinctrl-names = "gpio", "otpout"; ++ pinctrl-0 = <&tsadc_otp_gpio>; ++ pinctrl-1 = <&tsadc_otp_out>; ++ status = "okay"; ++}; ++ ++&u2phy { ++ status = "okay"; ++ ++ u2phy_host: host-port { ++ status = "okay"; ++ }; ++ ++ u2phy_otg: otg-port { ++ vbus-supply = <&vcc5v0_host>; ++ status = "okay"; ++ }; ++}; ++ ++&usb20_otg { ++ status = "okay"; ++}; ++ ++&uart1 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart1_xfer &uart1_cts>; ++ status = "okay"; ++}; ++ ++&vopb { ++ status = "okay"; ++}; ++ ++&vopb_mmu { ++ status = "okay"; ++}; ++ ++&vopl { ++ status = "okay"; ++}; ++ ++&vopl_mmu { ++ status = "okay"; ++}; ++ ++&mpp_srv { ++ status = "okay"; ++}; ++ ++&vdpu { ++ status = "okay"; ++}; ++ ++&vepu { ++ status = "okay"; ++}; ++ ++&vpu_mmu { ++ status = "okay"; ++}; ++ ++&hevc { ++ status = "okay"; ++}; ++ ++&hevc_mmu { ++ status = "okay"; ++}; ++ ++&mipi_dphy_rx0 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ov5695_out>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_rx_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp_mipi_in>; ++ }; ++ }; ++ }; ++}; ++ ++&rkisp1 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_clkout_m0 &dvp_d0d1_m0 &dvp_d2d9_m0>; ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp_mipi_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dphy_rx_out>; ++ }; ++ ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3326-evb-ai-va-v12.dts b/arch/arm64/boot/dts/rockchip/rk3326-evb-ai-va-v12.dts +new file mode 100755 +index 000000000000..9669993499e0 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3326-evb-ai-va-v12.dts +@@ -0,0 +1,1317 @@ ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++ */ ++ ++/dts-v1/; ++#include ++#include ++#include ++#include ++#include ++#include "rk3326.dtsi" ++#include "rk3326-863-cif-sensor.dtsi" ++#include "px30-android.dtsi" ++ ++/ { ++ model = "Rockchip rk3326 ai voice assistant evb v12 board"; ++ compatible = "rockchip,rk3326-evb-ai-va-v12", "rockchip,rk3326"; ++ ++ adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 2>; ++ io-channel-names = "buttons"; ++ poll-interval = <100>; ++ keyup-threshold-microvolt = <1800000>; ++ ++ mute-key { ++ linux,code = ; ++ label = "mute"; ++ press-threshold-microvolt = <1119000>; ++ }; ++ ++ mode-key { ++ linux,code = ; ++ label = "mode"; ++ press-threshold-microvolt = <892000>; ++ }; ++ ++ media-key { ++ linux,code = ; ++ label = "media"; ++ press-threshold-microvolt = <616000>; ++ }; ++ ++ vol-down-key { ++ linux,code = ; ++ label = "volume down"; ++ press-threshold-microvolt = <300000>; ++ }; ++ ++ vol-up-key { ++ linux,code = ; ++ label = "volume up"; ++ press-threshold-microvolt = <15000>; ++ }; ++ }; ++ ++ backlight: backlight { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm1 0 25000 0>; ++ brightness-levels = < ++ 0 1 2 3 4 5 6 7 ++ 8 9 10 11 12 13 14 15 ++ 16 17 18 19 20 21 22 23 ++ 24 25 26 27 28 29 30 31 ++ 32 33 34 35 36 37 38 39 ++ 40 41 42 43 44 45 46 47 ++ 48 49 50 51 52 53 54 55 ++ 56 57 58 59 60 61 62 63 ++ 64 65 66 67 68 69 70 71 ++ 72 73 74 75 76 77 78 79 ++ 80 81 82 83 84 85 86 87 ++ 88 89 90 91 92 93 94 95 ++ 96 97 98 99 100 101 102 103 ++ 104 105 106 107 108 109 110 111 ++ 112 113 114 115 116 117 118 119 ++ 120 121 122 123 124 125 126 127 ++ 128 129 130 131 132 133 134 135 ++ 136 137 138 139 140 141 142 143 ++ 144 145 146 147 148 149 150 151 ++ 152 153 154 155 156 157 158 159 ++ 160 161 162 163 164 165 166 167 ++ 168 169 170 171 172 173 174 175 ++ 176 177 178 179 180 181 182 183 ++ 184 185 186 187 188 189 190 191 ++ 192 193 194 195 196 197 198 199 ++ 200 201 202 203 204 205 206 207 ++ 208 209 210 211 212 213 214 215 ++ 216 217 218 219 220 221 222 223 ++ 224 225 226 227 228 229 230 231 ++ 232 233 234 235 236 237 238 239 ++ 240 241 242 243 244 245 246 247 ++ 248 249 250 251 252 253 254 255>; ++ default-brightness-level = <200>; ++ }; ++ ++ rk809-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,name = "rockchip,rk809-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,widgets = ++ "Microphone", "Mic Jack", ++ "Headphone", "Headphone Jack"; ++ simple-audio-card,routing = ++ "Mic Jack", "MICBIAS1", ++ "IN1P", "Mic Jack", ++ "Headphone Jack", "HPOL", ++ "Headphone Jack", "HPOR"; ++ simple-audio-card,dai-link@0 { ++ format = "i2s"; ++ cpu { ++ sound-dai = <&i2s1_2ch>; ++ }; ++ codec { ++ sound-dai = <&rk809_codec 0>; ++ }; ++ }; ++ simple-audio-card,dai-link@1 { ++ format = "pdm"; ++ cpu { ++ sound-dai = <&pdm>; ++ }; ++ codec { ++ sound-dai = <&rk809_codec 1>; ++ }; ++ }; ++ }; ++ ++ bt-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "dsp_a"; ++ simple-audio-card,bitclock-inversion = <1>; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,name = "rockchip,bt"; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s2_2ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&bt_sco>; ++ }; ++ }; ++ ++ bt_sco: bt-sco { ++ compatible = "delta,dfbmcs320"; ++ #sound-dai-cells = <0>; ++ status = "okay"; ++ }; ++ ++ rk_headset: rk-headset { ++ compatible = "rockchip_headset"; ++ headset_gpio = <&gpio3 RK_PA4 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hp_det>; ++ io-channels = <&saradc 1>; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ clocks = <&rk809 1>; ++ clock-names = "ext_clock"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_LOW>; /* GPIO3_A4 */ ++ }; ++ ++ test-power { ++ status = "okay"; ++ }; ++ ++ vcc5v0_sys: vccsys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc5v0_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ wifi_chip_type = "AP6256"; ++ WIFI,host_wake_irq = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&rk809 1>; ++ clock-names = "ext_clock"; ++ uart_rts_gpios = <&gpio1 RK_PC3 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default","rts_gpio"; ++ pinctrl-0 = <&uart1_rts>; ++ pinctrl-1 = <&uart1_rts_gpio>; ++ BT,reset_gpio = <&gpio0 RK_PA0 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio0 RK_PA1 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio0 RK_PA7 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++}; ++ ++&bus_apll { ++ bus-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&cpu0 { ++ cpu-supply = <&vdd_arm>; ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&display_subsystem { ++ status = "okay"; ++}; ++ ++&dmc { ++ center-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&dsi { ++ status = "okay"; ++ ++ panel@0 { ++ compatible = "sitronix,st7703", "simple-panel-dsi"; ++ reg = <0>; ++ backlight = <&backlight>; ++ enable-gpios = <&gpio0 RK_PB5 GPIO_ACTIVE_HIGH>; ++ prepare-delay-ms = <2>; ++ reset-delay-ms = <1>; ++ init-delay-ms = <20>; ++ enable-delay-ms = <120>; ++ disable-delay-ms = <50>; ++ unprepare-delay-ms = <40>; ++ ++ width-mm = <68>; ++ height-mm = <121>; ++ ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | ++ MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ ++ panel-init-sequence = [ ++ 05 fa 01 11 ++ 39 00 04 b9 f1 12 83 ++ 39 00 1c ba 33 81 05 f9 0e 0e 00 00 00 ++ 00 00 00 00 00 44 25 00 91 0a ++ 00 00 02 4f 01 00 00 37 ++ 15 00 02 b8 25 ++ 39 00 04 bf 02 11 00 ++ 39 00 0b b3 0c 10 0a 50 03 ff 00 00 00 ++ 00 ++ 39 00 0a c0 73 73 50 50 00 00 08 70 00 ++ 15 00 02 bc 46 ++ 15 00 02 cc 0b ++ 15 00 02 b4 80 ++ 39 00 04 b2 c8 12 30 ++ 39 00 0f e3 07 07 0b 0b 03 0b 00 00 00 ++ 00 ff 00 c0 10 ++ 39 00 0d c1 53 00 1e 1e 77 e1 cc dd 67 ++ 77 33 33 ++ 39 00 07 c6 00 00 ff ff 01 ff ++ 39 00 03 b5 09 09 ++ 39 00 03 b6 87 95 ++ 39 00 40 e9 c2 10 05 05 10 05 a0 12 31 ++ 23 3f 81 0a a0 37 18 00 80 01 ++ 00 00 00 00 80 01 00 00 00 48 ++ f8 86 42 08 88 88 80 88 88 88 ++ 58 f8 87 53 18 88 88 81 88 88 ++ 88 00 00 00 01 00 00 00 00 00 ++ 00 00 00 00 ++ 39 00 3e ea 00 1a 00 00 00 00 02 00 00 ++ 00 00 00 1f 88 81 35 78 88 88 ++ 85 88 88 88 0f 88 80 24 68 88 ++ 88 84 88 88 88 23 10 00 00 1c ++ 00 00 00 00 00 00 00 00 00 00 ++ 00 00 00 00 00 30 05 a0 00 00 ++ 00 00 ++ 39 00 23 e0 00 06 08 2a 31 3f 38 36 07 ++ 0c 0d 11 13 12 13 11 18 00 06 ++ 08 2a 31 3f 38 36 07 0c 0d 11 ++ 13 12 13 11 18 ++ 05 32 01 29 ++ ]; ++ ++ panel-exit-sequence = [ ++ 05 00 01 28 ++ 05 00 01 10 ++ ]; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <66000000>; ++ hactive = <720>; ++ vactive = <1280>; ++ hfront-porch = <40>; ++ hsync-len = <10>; ++ hback-porch = <40>; ++ vfront-porch = <22>; ++ vsync-len = <4>; ++ vback-porch = <11>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ panel_in_dsi: endpoint { ++ remote-endpoint = <&dsi_out_panel>; ++ }; ++ }; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@1 { ++ reg = <1>; ++ dsi_out_panel: endpoint { ++ remote-endpoint = <&panel_in_dsi>; ++ }; ++ }; ++ }; ++}; ++ ++&dsi_in_vopb { ++ status = "okay"; ++}; ++ ++&dsi_in_vopl { ++ status = "disabled"; ++}; ++ ++&route_dsi { ++ connect = <&vopb_out_dsi>; ++ status = "okay"; ++}; ++ ++&emmc { ++ bus-width = <8>; ++ cap-mmc-highspeed; ++ mmc-hs200-1_8v; ++ no-sdio; ++ no-sd; ++ disable-wp; ++ non-removable; ++ num-slots = <1>; ++ status = "okay"; ++}; ++ ++&gpu { ++ mali-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ clock-frequency = <400000>; ++ i2c-scl-rising-time-ns = <280>; ++ i2c-scl-falling-time-ns = <16>; ++ ++ rk809: pmic@20 { ++ compatible = "rockchip,rk809"; ++ reg = <0x20>; ++ interrupt-parent = <&gpio0>; ++ interrupts = <10 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default", "pmic-sleep", ++ "pmic-power-off", "pmic-reset"; ++ pinctrl-0 = <&pmic_int>; ++ pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; ++ pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; ++ pinctrl-3 = <&soc_slppin_rst>, <&rk817_slppin_rst>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ //fb-inner-reg-idxs = <2>; ++ /* 1: rst regs (default in codes), 0: rst the pmic */ ++ pmic-reset-func = <1>; ++ ++ vcc1-supply = <&vcc5v0_sys>; ++ vcc2-supply = <&vcc5v0_sys>; ++ vcc3-supply = <&vcc5v0_sys>; ++ vcc4-supply = <&vcc5v0_sys>; ++ vcc5-supply = <&vcc3v3_sys>; ++ vcc6-supply = <&vcc3v3_sys>; ++ vcc7-supply = <&vcc3v3_sys>; ++ vcc8-supply = <&vcc3v3_sys>; ++ vcc9-supply = <&vcc5v0_sys>; ++ ++ pwrkey { ++ status = "okay"; ++ }; ++ ++ pinctrl_rk8xx: pinctrl_rk8xx { ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ rk817_slppin_null: rk817_slppin_null { ++ pins = "gpio_slp"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_slppin_slp: rk817_slppin_slp { ++ pins = "gpio_slp"; ++ function = "pin_fun1"; ++ }; ++ ++ rk817_slppin_pwrdn: rk817_slppin_pwrdn { ++ pins = "gpio_slp"; ++ function = "pin_fun2"; ++ }; ++ ++ rk817_slppin_rst: rk817_slppin_rst { ++ pins = "gpio_slp"; ++ function = "pin_fun3"; ++ }; ++ }; ++ ++ regulators { ++ vdd_logic: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <950000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_logic"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <950000>; ++ }; ++ }; ++ ++ vdd_arm: DCDC_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <950000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_arm"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <950000>; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_ddr"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_3v0: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_3v0"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ ++ vcc_1v0: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ regulator-name = "vcc_1v0"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc1v8_soc: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcc1v8_soc"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd1v0_soc: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ ++ regulator-name = "vcc1v0_soc"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc3v0_pmu: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ ++ regulator-name = "vcc3v0_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ ++ }; ++ }; ++ ++ vccio_sd: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vccio_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc_sd: LDO_REG6 { ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vcc_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ ++ }; ++ }; ++ ++ vcc2v8_dvp: LDO_REG7 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <2800000>; ++ regulator-max-microvolt = <2800000>; ++ ++ regulator-name = "vcc2v8_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <2800000>; ++ }; ++ }; ++ ++ vcc1v8_dvp: LDO_REG8 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcc1v8_dvp"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd1v5_dvp: LDO_REG9 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ ++ regulator-name = "vdd1v5_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <1500000>; ++ }; ++ }; ++ ++ vcc3v3_sys: DCDC_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcc3v3_sys"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc5v0_host: SWITCH_REG1 { ++ regulator-name = "vcc5v0_host"; ++ }; ++ ++ vcc3v3_lcd: SWITCH_REG2 { ++ regulator-boot-on; ++ regulator-name = "vcc3v3_lcd"; ++ }; ++ }; ++ ++ rk809_codec: codec { ++ #sound-dai-cells = <1>; ++ compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; ++ clocks = <&cru SCLK_I2S1_OUT>; ++ clock-names = "mclk"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1_2ch_mclk>; ++ pdmdata-out-enable; ++ use-ext-amplifier; ++ adc-for-loopback; ++ spk-ctl-gpios = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; ++ hp-volume = <20>; ++ spk-volume = <20>; ++ }; ++ }; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ clock-frequency = <400000>; ++ i2c-scl-rising-time-ns = <275>; ++ i2c-scl-falling-time-ns = <16>; ++ ++ gt1x: gt1x@14 { ++ compatible = "goodix,gt1x"; ++ reg = <0x14>; ++ goodix,rst-gpio = <&gpio0 RK_PB4 GPIO_ACTIVE_HIGH>; ++ goodix,irq-gpio = <&gpio0 RK_PB3 IRQ_TYPE_LEVEL_LOW>; ++ }; ++ ++ is31fl3236: led-controller@3c { ++ compatible = "issi,is31fl3236"; ++ reg = <0x3c>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ reset-gpios = <&gpio3 RK_PA6 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ ++ led1: led@1 { ++ label = "led1"; ++ reg = <1>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <0>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led2: led@2 { ++ label = "led2"; ++ reg = <2>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <0>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led3: led@3 { ++ label = "led3"; ++ reg = <3>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led4: led@4 { ++ label = "led4"; ++ reg = <4>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <100>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led5: led@5 { ++ label = "led5"; ++ reg = <5>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <100>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led6: led@6 { ++ label = "led6"; ++ reg = <6>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led7: led@7 { ++ label = "led7"; ++ reg = <7>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <200>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led8: led@8 { ++ label = "led8"; ++ reg = <8>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <200>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led9: led@9 { ++ label = "led9"; ++ reg = <9>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led10: led@10 { ++ label = "led10"; ++ reg = <10>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <300>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led11: led@11 { ++ label = "led11"; ++ reg = <11>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <300>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led12: led@12 { ++ label = "led12"; ++ reg = <12>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led13: led@13 { ++ label = "led13"; ++ reg = <13>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <400>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led14: led@14 { ++ label = "led14"; ++ reg = <14>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <400>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led15: led@15 { ++ label = "led15"; ++ reg = <15>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led16: led@16 { ++ label = "led16"; ++ reg = <16>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <500>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led17: led@17 { ++ label = "led17"; ++ reg = <17>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <500>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led18: led@18 { ++ label = "led18"; ++ reg = <18>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led19: led@19 { ++ label = "led19"; ++ reg = <19>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <600>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led20: led@20 { ++ label = "led20"; ++ reg = <20>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <600>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led21: led@21 { ++ label = "led21"; ++ reg = <21>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led22: led@22 { ++ label = "led22"; ++ reg = <22>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <700>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led23: led@23 { ++ label = "led23"; ++ reg = <23>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <700>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led124: led@24 { ++ label = "led24"; ++ reg = <24>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led25: led@25 { ++ label = "led25"; ++ reg = <25>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <800>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led26: led@26 { ++ label = "led26"; ++ reg = <26>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <800>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led27: led@27 { ++ label = "led27"; ++ reg = <27>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led28: led@28 { ++ label = "led28"; ++ reg = <28>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <900>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led29: led@29 { ++ label = "led29"; ++ reg = <29>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <900>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led30: led@30 { ++ label = "led30"; ++ reg = <30>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led31: led@31 { ++ label = "led31"; ++ reg = <31>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <1000>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led32: led@32 { ++ label = "led32"; ++ reg = <32>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <1000>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led33: led@33 { ++ label = "led33"; ++ reg = <33>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ ++ led34: led@34 { ++ label = "led34"; ++ reg = <34>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <1100>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led35: led@35 { ++ label = "led35"; ++ reg = <35>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "timer"; ++ linux,default-trigger-delay-ms = <1100>; ++ linux,blink-delay-on-ms = <100>; ++ linux,blink-delay-off-ms = <1200>; ++ }; ++ ++ led36: led@36 { ++ label = "led36"; ++ reg = <36>; ++ led-max-microamp = <10000>; ++ linux,default-trigger = "default-on"; ++ }; ++ }; ++ ++ ls_stk3410: light@48 { ++ compatible = "ls_stk3410"; ++ status = "okay"; ++ reg = <0x48>; ++ type = ; ++ irq_enable = <0>; ++ als_threshold_high = <100>; ++ als_threshold_low = <10>; ++ als_ctrl_gain = <2>; /* 0:x1 1:x4 2:x16 3:x64 */ ++ poll_delay_ms = <100>; ++ }; ++ ++ ps_stk3410: proximity@48 { ++ compatible = "ps_stk3410"; ++ status = "okay"; ++ reg = <0x48>; ++ type = ; ++ //pinctrl-names = "default"; ++ //pinctrl-0 = <&gpio2_c3>; ++ //irq-gpio = <&gpio0 RK_PB7 IRQ_TYPE_LEVEL_LOW>; ++ //irq_enable = <1>; ++ ps_threshold_high = <0x200>; ++ ps_threshold_low = <0x100>; ++ ps_ctrl_gain = <3>; /* 0:x1 1:x4 2:x16 3:x64 */ ++ ps_led_current = <3>; /* 0:12.5mA 1:25mA 2:50mA 3:100mA */ ++ poll_delay_ms = <100>; ++ }; ++ ++}; ++ ++&i2c2 { ++ status = "okay"; ++ ++ clock-frequency = <100000>; ++ ++ /* These are relatively safe rise/fall times; TODO: measure */ ++ i2c-scl-falling-time-ns = <50>; ++ i2c-scl-rising-time-ns = <300>; ++ ++ ov5695: ov5695@36 { ++ compatible = "ovti,ov5695"; ++ reg = <0x36>; ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ /*reset-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>;*/ ++ pwdn-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>; ++ //pinctrl-names = "default"; ++ //pinctrl-0 = <&cif_clkout_m0>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "TongJu"; ++ rockchip,camera-module-lens-name = "CHT842-MD"; ++ port { ++ ov5695_out: endpoint { ++ remote-endpoint = <&mipi_in>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++}; ++ ++&i2s1_2ch { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++ pinctrl-0 = <&i2s1_2ch_sclk ++ &i2s1_2ch_lrck ++ &i2s1_2ch_sdo>; ++}; ++ ++&i2s2_2ch { ++ status = "okay"; ++ rockchip,bclk-fs = <64>; ++ #sound-dai-cells = <0>; ++}; ++ ++&io_domains { ++ status = "okay"; ++ ++ vccio1-supply = <&vcc1v8_soc>; ++ vccio2-supply = <&vccio_sd>; ++ vccio3-supply = <&vcc1v8_dvp>; ++ vccio4-supply = <&vcc1v8_soc>; ++ vccio5-supply = <&vcc_3v0>; ++}; ++ ++&isp_mmu { ++ status = "okay"; ++}; ++ ++&nandc0 { ++ status = "okay"; ++}; ++ ++&pdm { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pdm_clk0m1 ++ &pdm_clk1 ++ &pdm_sdi0m1 ++ &pdm_sdi1 ++ &pdm_sdi2 ++ &pdm_sdi3>; ++}; ++ ++&pinctrl { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&tp_int>; ++ ++ headphone { ++ hp_det: hp-det { ++ rockchip,pins = <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ pmic { ++ pmic_int: pmic_int { ++ rockchip,pins = ++ <0 RK_PB2 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ soc_slppin_gpio: soc_slppin_gpio { ++ rockchip,pins = ++ <0 RK_PA4 RK_FUNC_GPIO &pcfg_output_low>; ++ }; ++ ++ soc_slppin_slp: soc_slppin_slp { ++ rockchip,pins = ++ <0 RK_PA4 1 &pcfg_pull_none>; ++ }; ++ ++ soc_slppin_rst: soc_slppin_rst { ++ rockchip,pins = ++ <0 RK_PA4 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ touchscreen-int { ++ tp_int: tp-int { ++ rockchip,pins = <0 RK_PB3 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ ++ pmuio1-supply = <&vcc3v0_pmu>; ++ pmuio2-supply = <&vcc3v0_pmu>; ++}; ++ ++&pwm1 { ++ status = "okay"; ++}; ++ ++&rk_rga { ++ status = "okay"; ++}; ++ ++&rockchip_suspend { ++ status = "okay"; ++ rockchip,sleep-debug-en = <1>; ++}; ++ ++&saradc { ++ status = "okay"; ++ vref-supply = <&vcc1v8_soc>; ++}; ++ ++&sdmmc { ++ bus-width = <4>; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ no-sdio; ++ no-mmc; ++ card-detect-delay = <800>; ++ ignore-pm-notify; ++ /*cd-gpios = <&gpio2 4 GPIO_ACTIVE_HIGH>; [> CD GPIO <]*/ ++ sd-uhs-sdr12; ++ sd-uhs-sdr25; ++ sd-uhs-sdr50; ++ sd-uhs-sdr104; ++ vqmmc-supply = <&vccio_sd>; ++ vmmc-supply = <&vcc_sd>; ++ status = "okay"; ++}; ++ ++&sdio { ++ bus-width = <4>; ++ cap-sd-highspeed; ++ no-sd; ++ no-mmc; ++ ignore-pm-notify; ++ keep-power-in-suspend; ++ non-removable; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&tsadc { ++ pinctrl-names = "gpio", "otpout"; ++ pinctrl-0 = <&tsadc_otp_gpio>; ++ pinctrl-1 = <&tsadc_otp_out>; ++ status = "okay"; ++}; ++ ++&u2phy { ++ status = "okay"; ++ ++ u2phy_host: host-port { ++ status = "okay"; ++ }; ++ ++ u2phy_otg: otg-port { ++ vbus-supply = <&vcc5v0_host>; ++ status = "okay"; ++ }; ++}; ++ ++&usb20_otg { ++ status = "okay"; ++}; ++ ++&uart1 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart1_xfer &uart1_cts>; ++ status = "okay"; ++}; ++ ++&vopb { ++ status = "okay"; ++}; ++ ++&vopb_mmu { ++ status = "okay"; ++}; ++ ++&vopl { ++ status = "okay"; ++}; ++ ++&vopl_mmu { ++ status = "okay"; ++}; ++ ++&mpp_srv { ++ status = "okay"; ++}; ++ ++&vdpu { ++ status = "okay"; ++}; ++ ++&vepu { ++ status = "okay"; ++}; ++ ++&vpu_mmu { ++ status = "okay"; ++}; ++ ++&hevc { ++ status = "okay"; ++}; ++ ++&hevc_mmu { ++ status = "okay"; ++}; ++ ++&mipi_dphy_rx0 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ov5695_out>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_rx_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp_mipi_in>; ++ }; ++ }; ++ }; ++}; ++ ++&rkisp1 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_clkout_m0 &dvp_d0d1_m0 &dvp_d2d9_m0>; ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp_mipi_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dphy_rx_out>; ++ }; ++ ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v10-avb.dts b/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v10-avb.dts +new file mode 100755 +index 000000000000..4c12a79f814b +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v10-avb.dts +@@ -0,0 +1,91 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++#include "rk3326-evb-lp3-v10.dtsi" ++ ++/ { ++ model = "Rockchip rk3326 evb board"; ++ compatible = "rockchip,rk3326-evb-lp3-v10-avb", "rockchip,rk3326"; ++}; ++ ++&i2c2 { ++ status = "okay"; ++ ++ clock-frequency = <100000>; ++ ++ /* These are relatively safe rise/fall times; TODO: measure */ ++ i2c-scl-falling-time-ns = <50>; ++ i2c-scl-rising-time-ns = <300>; ++ ++ ov5695: ov5695@36 { ++ compatible = "ovti,ov5695"; ++ reg = <0x36>; ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ /*reset-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>;*/ ++ pwdn-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>; ++ //pinctrl-names = "default"; ++ //pinctrl-0 = <&cif_clkout_m0>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "TongJu"; ++ rockchip,camera-module-lens-name = "CHT842-MD"; ++ port { ++ ov5695_out: endpoint { ++ remote-endpoint = <&mipi_in>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++}; ++ ++&mipi_dphy_rx0 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ov5695_out>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_rx_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp_mipi_in>; ++ }; ++ }; ++ }; ++}; ++ ++&rkisp1 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_clkout_m0 &dvp_d0d1_m0 &dvp_d2d9_m0>; ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp_mipi_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dphy_rx_out>; ++ }; ++ ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v10-linux.dts b/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v10-linux.dts +new file mode 100755 +index 000000000000..67c131c9618a +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v10-linux.dts +@@ -0,0 +1,1024 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++#include ++#include ++#include ++#include ++#include ++#include "rk3326.dtsi" ++#include "rk3326-linux.dtsi" ++ ++/ { ++ model = "Rockchip rk3326 evb lpddr3 v10 board for linux"; ++ compatible = "rockchip,rk3326-evb-lp3-v10-linux", "rockchip,rk3326"; ++ ++ chosen { ++ bootargs = "earlycon=uart8250,mmio32,0xff160000 console=ttyFIQ0 rw root=PARTUUID=614e0000-0000 rootfstype=ext4 rootwait"; ++ }; ++ ++ adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 2>; ++ io-channel-names = "buttons"; ++ poll-interval = <100>; ++ keyup-threshold-microvolt = <1800000>; ++ ++ esc-key { ++ linux,code = ; ++ label = "esc"; ++ press-threshold-microvolt = <1310000>; ++ }; ++ ++ home-key { ++ linux,code = ; ++ label = "home"; ++ press-threshold-microvolt = <624000>; ++ }; ++ ++ menu-key { ++ linux,code = ; ++ label = "menu"; ++ press-threshold-microvolt = <987000>; ++ }; ++ ++ vol-down-key { ++ linux,code = ; ++ label = "volume down"; ++ press-threshold-microvolt = <300000>; ++ }; ++ ++ vol-up-key { ++ linux,code = ; ++ label = "volume up"; ++ press-threshold-microvolt = <17000>; ++ }; ++ }; ++ ++ backlight: backlight { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm1 0 25000 0>; ++ brightness-levels = < ++ 0 1 2 3 4 5 6 7 ++ 8 9 10 11 12 13 14 15 ++ 16 17 18 19 20 21 22 23 ++ 24 25 26 27 28 29 30 31 ++ 32 33 34 35 36 37 38 39 ++ 40 41 42 43 44 45 46 47 ++ 48 49 50 51 52 53 54 55 ++ 56 57 58 59 60 61 62 63 ++ 64 65 66 67 68 69 70 71 ++ 72 73 74 75 76 77 78 79 ++ 80 81 82 83 84 85 86 87 ++ 88 89 90 91 92 93 94 95 ++ 96 97 98 99 100 101 102 103 ++ 104 105 106 107 108 109 110 111 ++ 112 113 114 115 116 117 118 119 ++ 120 121 122 123 124 125 126 127 ++ 128 129 130 131 132 133 134 135 ++ 136 137 138 139 140 141 142 143 ++ 144 145 146 147 148 149 150 151 ++ 152 153 154 155 156 157 158 159 ++ 160 161 162 163 164 165 166 167 ++ 168 169 170 171 172 173 174 175 ++ 176 177 178 179 180 181 182 183 ++ 184 185 186 187 188 189 190 191 ++ 192 193 194 195 196 197 198 199 ++ 200 201 202 203 204 205 206 207 ++ 208 209 210 211 212 213 214 215 ++ 216 217 218 219 220 221 222 223 ++ 224 225 226 227 228 229 230 231 ++ 232 233 234 235 236 237 238 239 ++ 240 241 242 243 244 245 246 247 ++ 248 249 250 251 252 253 254 255>; ++ default-brightness-level = <200>; ++ }; ++ ++ rk817-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,rk817-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,widgets = ++ "Microphone", "Mic Jack", ++ "Headphone", "Headphone Jack"; ++ simple-audio-card,routing = ++ "MIC_IN", "Microphone Jack", ++ "IN1P", "Mic Jack", ++ "Headphone Jack", "HPOL", ++ "Headphone Jack", "HPOR"; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s1_2ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&rk817_codec>; ++ }; ++ }; ++ ++ rk_headset { ++ compatible = "rockchip_headset"; ++ headset_gpio = <&gpio2 RK_PC6 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hp_det>; ++ io-channels = <&saradc 1>; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ /*clocks = <&rk817 1>;*/ ++ /*clock-names = "ext_clock";*/ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_LOW>; /* GPIO3_A4 */ ++ }; ++ ++ vccsys: vccsys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc3v8_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3800000>; ++ regulator-max-microvolt = <3800000>; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ wifi_chip_type = "AP6210"; ++ WIFI,host_wake_irq = <&gpio0 RK_PB2 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&rk817 1>; ++ clock-names = "ext_clock"; ++ uart_rts_gpios = <&gpio1 RK_PC3 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default","rts_gpio"; ++ pinctrl-0 = <&uart1_rts>; ++ pinctrl-1 = <&uart1_rts_gpio>; ++ BT,reset_gpio = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio0 RK_PA1 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio0 RK_PB3 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ vcc18_lcd_n: vcc18-lcd-n { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc18_lcd_n"; ++ regulator-boot-on; ++ gpio = <&gpio0 RK_PB5 GPIO_ACTIVE_HIGH>; ++ enable-active-high; ++ }; ++}; ++ ++&bus_apll { ++ bus-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&cif_new { ++ status = "okay"; ++ ++ port { ++ cif_in: endpoint { ++ remote-endpoint = <&gc2155_out>; ++ vsync-active = <0>; ++ hsync-active = <1>; ++ }; ++ }; ++}; ++ ++&cpu0 { ++ cpu-supply = <&vdd_arm>; ++}; ++ ++&display_subsystem { ++ status = "okay"; ++}; ++ ++&dsi { ++ status = "okay"; ++ ++ panel@0 { ++ compatible = "sitronix,st7703", "simple-panel-dsi"; ++ reg = <0>; ++ backlight = <&backlight>; ++ power-supply = <&vcc18_lcd_n>; ++ prepare-delay-ms = <2>; ++ reset-delay-ms = <1>; ++ init-delay-ms = <20>; ++ enable-delay-ms = <120>; ++ disable-delay-ms = <50>; ++ unprepare-delay-ms = <20>; ++ ++ width-mm = <68>; ++ height-mm = <121>; ++ ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | ++ MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ ++ panel-init-sequence = [ ++ 05 fa 01 11 ++ 39 00 04 b9 f1 12 83 ++ 39 00 1c ba 33 81 05 f9 0e 0e 00 00 00 ++ 00 00 00 00 00 44 25 00 91 0a ++ 00 00 02 4f 01 00 00 37 ++ 15 00 02 b8 25 ++ 39 00 04 bf 02 11 00 ++ 39 00 0b b3 0c 10 0a 50 03 ff 00 00 00 ++ 00 ++ 39 00 0a c0 73 73 50 50 00 00 08 70 00 ++ 15 00 02 bc 46 ++ 15 00 02 cc 0b ++ 15 00 02 b4 80 ++ 39 00 04 b2 c8 12 30 ++ 39 00 0f e3 07 07 0b 0b 03 0b 00 00 00 ++ 00 ff 00 c0 10 ++ 39 00 0d c1 53 00 1e 1e 77 e1 cc dd 67 ++ 77 33 33 ++ 39 00 07 c6 00 00 ff ff 01 ff ++ 39 00 03 b5 09 09 ++ 39 00 03 b6 87 95 ++ 39 00 40 e9 c2 10 05 05 10 05 a0 12 31 ++ 23 3f 81 0a a0 37 18 00 80 01 ++ 00 00 00 00 80 01 00 00 00 48 ++ f8 86 42 08 88 88 80 88 88 88 ++ 58 f8 87 53 18 88 88 81 88 88 ++ 88 00 00 00 01 00 00 00 00 00 ++ 00 00 00 00 ++ 39 00 3e ea 00 1a 00 00 00 00 02 00 00 ++ 00 00 00 1f 88 81 35 78 88 88 ++ 85 88 88 88 0f 88 80 24 68 88 ++ 88 84 88 88 88 23 10 00 00 1c ++ 00 00 00 00 00 00 00 00 00 00 ++ 00 00 00 00 00 30 05 a0 00 00 ++ 00 00 ++ 39 00 23 e0 00 06 08 2a 31 3f 38 36 07 ++ 0c 0d 11 13 12 13 11 18 00 06 ++ 08 2a 31 3f 38 36 07 0c 0d 11 ++ 13 12 13 11 18 ++ 05 32 01 29 ++ ]; ++ ++ panel-exit-sequence = [ ++ 05 00 01 28 ++ 05 00 01 10 ++ ]; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <66000000>; ++ hactive = <720>; ++ vactive = <1280>; ++ hfront-porch = <40>; ++ hsync-len = <10>; ++ hback-porch = <40>; ++ vfront-porch = <22>; ++ vsync-len = <4>; ++ vback-porch = <11>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ panel_in_dsi: endpoint { ++ remote-endpoint = <&dsi_out_panel>; ++ }; ++ }; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@1 { ++ reg = <1>; ++ dsi_out_panel: endpoint { ++ remote-endpoint = <&panel_in_dsi>; ++ }; ++ }; ++ }; ++}; ++ ++&dsi_in_vopb { ++ status = "okay"; ++}; ++ ++&dsi_in_vopl { ++ status = "disabled"; ++}; ++ ++&route_dsi { ++ connect = <&vopb_out_dsi>; ++ status = "okay"; ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&dmc { ++ center-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&emmc { ++ bus-width = <8>; ++ cap-mmc-highspeed; ++ mmc-hs200-1_8v; ++ no-sdio; ++ no-sd; ++ disable-wp; ++ non-removable; ++ num-slots = <1>; ++ status = "okay"; ++}; ++ ++&gpu { ++ mali-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ clock-frequency = <400000>; ++ i2c-scl-rising-time-ns = <280>; ++ i2c-scl-falling-time-ns = <16>; ++ ++ rk817: pmic@20 { ++ compatible = "rockchip,rk817"; ++ reg = <0x20>; ++ interrupt-parent = <&gpio0>; ++ interrupts = <7 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default", "pmic-sleep", ++ "pmic-power-off", "pmic-reset"; ++ pinctrl-0 = <&pmic_int>; ++ pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; ++ pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; ++ pinctrl-3 = <&soc_slppin_rst>, <&rk817_slppin_rst>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ //fb-inner-reg-idxs = <2>; ++ /* 1: rst regs (default in codes), 0: rst the pmic */ ++ pmic-reset-func = <1>; ++ ++ vcc1-supply = <&vccsys>; ++ vcc2-supply = <&vccsys>; ++ vcc3-supply = <&vccsys>; ++ vcc4-supply = <&vccsys>; ++ vcc5-supply = <&vccsys>; ++ vcc6-supply = <&vccsys>; ++ vcc7-supply = <&vcc_3v0>; ++ vcc8-supply = <&vccsys>; ++ vcc9-supply = <&dcdc_boost>; ++ ++ pwrkey { ++ status = "okay"; ++ }; ++ ++ pinctrl_rk8xx: pinctrl_rk8xx { ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ rk817_ts_gpio1: rk817_ts_gpio1 { ++ pins = "gpio_ts"; ++ function = "pin_fun1"; ++ /* output-low; */ ++ /* input-enable; */ ++ }; ++ ++ rk817_gt_gpio2: rk817_gt_gpio2 { ++ pins = "gpio_gt"; ++ function = "pin_fun1"; ++ }; ++ ++ rk817_pin_ts: rk817_pin_ts { ++ pins = "gpio_ts"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_pin_gt: rk817_pin_gt { ++ pins = "gpio_gt"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_slppin_null: rk817_slppin_null { ++ pins = "gpio_slp"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_slppin_slp: rk817_slppin_slp { ++ pins = "gpio_slp"; ++ function = "pin_fun1"; ++ }; ++ ++ rk817_slppin_pwrdn: rk817_slppin_pwrdn { ++ pins = "gpio_slp"; ++ function = "pin_fun2"; ++ }; ++ ++ rk817_slppin_rst: rk817_slppin_rst { ++ pins = "gpio_slp"; ++ function = "pin_fun3"; ++ }; ++ }; ++ ++ regulators { ++ vdd_logic: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <950000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_logic"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <950000>; ++ }; ++ }; ++ ++ vdd_arm: DCDC_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <950000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_arm"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <950000>; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_ddr"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_3v0: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_3v0"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ ++ vcc_1v0: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ regulator-name = "vcc_1v0"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc1v8_soc: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcc1v8_soc"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd1v0_soc: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ ++ regulator-name = "vcc1v0_soc"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc3v0_pmu: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ ++ regulator-name = "vcc3v0_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ ++ }; ++ }; ++ ++ vccio_sd: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vccio_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc_sd: LDO_REG6 { ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-boot-on; ++ ++ regulator-name = "vcc_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ ++ }; ++ }; ++ ++ vcc2v8_dvp: LDO_REG7 { ++ regulator-boot-on; ++ regulator-min-microvolt = <2800000>; ++ regulator-max-microvolt = <2800000>; ++ ++ regulator-name = "vcc2v8_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <2800000>; ++ }; ++ }; ++ ++ vcc1v8_dvp: LDO_REG8 { ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcc1v8_dvp"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd1v5_dvp: LDO_REG9 { ++ regulator-boot-on; ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ ++ regulator-name = "vdd1v5_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <1500000>; ++ }; ++ }; ++ ++ dcdc_boost: BOOST { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <4700000>; ++ regulator-max-microvolt = <5400000>; ++ regulator-name = "boost"; ++ }; ++ ++ otg_switch: OTG_SWITCH { ++ regulator-name = "otg_switch"; ++ }; ++ }; ++ ++ battery { ++ compatible = "rk817,battery"; ++ ocv_table = <3500 3625 3685 3697 3718 3735 3748 ++ 3760 3774 3788 3802 3816 3834 3853 ++ 3877 3908 3946 3975 4018 4071 4106>; ++ design_capacity = <2500>; ++ design_qmax = <2750>; ++ bat_res = <100>; ++ sleep_enter_current = <300>; ++ sleep_exit_current = <300>; ++ sleep_filter_current = <100>; ++ power_off_thresd = <3500>; ++ zero_algorithm_vol = <3850>; ++ max_soc_offset = <60>; ++ monitor_sec = <5>; ++ sample_res = <10>; ++ virtual_power = <1>; ++ }; ++ ++ charger { ++ compatible = "rk817,charger"; ++ min_input_voltage = <4500>; ++ max_input_current = <1500>; ++ max_chrg_current = <2000>; ++ max_chrg_voltage = <4200>; ++ chrg_term_mode = <0>; ++ chrg_finish_cur = <300>; ++ virtual_power = <0>; ++ dc_det_adc = <0>; ++ extcon = <&u2phy>; ++ }; ++ ++ rk817_codec: codec { ++ #sound-dai-cells = <0>; ++ compatible = "rockchip,rk817-codec"; ++ clocks = <&cru SCLK_I2S1_OUT>; ++ clock-names = "mclk"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1_2ch_mclk>; ++ hp-volume = <20>; ++ spk-volume = <3>; ++ status = "okay"; ++ }; ++ }; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ clock-frequency = <400000>; ++ i2c-scl-rising-time-ns = <275>; ++ i2c-scl-falling-time-ns = <16>; ++ ++ sensor@f { ++ status = "okay"; ++ compatible = "ak8963"; ++ reg = <0x0f>; ++ type = ; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ layout = <1>; ++ reprobe_en = <1>; ++ }; ++ ++ gt1x: gt1x@14 { ++ compatible = "goodix,gt1x"; ++ reg = <0x14>; ++ power-supply = <&vcc18_lcd_n>; ++ goodix,rst-gpio = <&gpio0 RK_PB4 GPIO_ACTIVE_HIGH>; ++ goodix,irq-gpio = <&gpio0 RK_PA5 IRQ_TYPE_LEVEL_LOW>; ++ }; ++ ++ sensor@4c { ++ status = "okay"; ++ compatible = "gs_mma7660"; ++ reg = <0x4c>; ++ type = ; ++ irq-gpio = <&gpio0 RK_PB5 IRQ_TYPE_LEVEL_LOW>; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ layout = <1>; ++ reprobe_en = <1>; ++ }; ++}; ++ ++&i2c2 { ++ status = "okay"; ++ clock-frequency = <400000>; ++ ++ /* 24M mclk is shared for multiple cameras */ ++ pinctrl-0 = <&i2c2_xfer &cif_clkout_m0>; ++ ++ /* These are relatively safe rise/fall times; TODO: measure */ ++ i2c-scl-falling-time-ns = <50>; ++ i2c-scl-rising-time-ns = <300>; ++ ++ gc2155: gc2155@3c { ++ compatible = "gc,gc2155"; ++ reg = <0x3c>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_pin_m0>; ++ ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ ++ avdd-supply = <&vcc2v8_dvp>; ++ dovdd-supply = <&vcc1v8_dvp>; ++ dvdd-supply = <&vcc1v8_dvp>; ++ ++ /* hw changed the pwdn to gpio2_b5 */ ++ pwdn-gpios = <&gpio2 13 GPIO_ACTIVE_HIGH>; ++ ++ port { ++ gc2155_out: endpoint { ++ remote-endpoint = <&cif_in>; ++ }; ++ }; ++ }; ++ ++ ov5695: ov5695@36 { ++ compatible = "ovti,ov5695"; ++ reg = <0x36>; ++ ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ ++ avdd-supply = <&vcc2v8_dvp>; ++ dovdd-supply = <&vcc1v8_dvp>; ++ dvdd-supply = <&vdd1v5_dvp>; ++ ++ /*reset-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>;*/ ++ pwdn-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>; ++ ++ port { ++ ucam_out: endpoint { ++ remote-endpoint = <&mipi_in_ucam>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++}; ++ ++&i2s1_2ch { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++}; ++ ++&io_domains { ++ status = "okay"; ++ ++ vccio1-supply = <&vcc1v8_soc>; ++ vccio2-supply = <&vccio_sd>; ++ vccio3-supply = <&vcc1v8_dvp>; ++ vccio4-supply = <&vcc_3v0>; ++ vccio5-supply = <&vcc_3v0>; ++}; ++ ++&isp_mmu { ++ status = "okay"; ++}; ++ ++&mipi_dphy_rx0 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ucam_out>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_rx0_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp0_mipi_in>; ++ }; ++ }; ++ }; ++}; ++ ++&nandc0 { ++ status = "okay"; ++}; ++ ++&rkisp1 { ++ status = "okay"; ++ ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp0_mipi_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dphy_rx0_out>; ++ }; ++ }; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ ++ pmuio1-supply = <&vcc3v0_pmu>; ++ pmuio2-supply = <&vcc3v0_pmu>; ++}; ++ ++&pwm1 { ++ status = "okay"; ++}; ++ ++&rk_rga { ++ status = "okay"; ++}; ++ ++&rockchip_suspend { ++ status = "okay"; ++ rockchip,sleep-debug-en = <1>; ++}; ++ ++&saradc { ++ status = "okay"; ++ vref-supply = <&vcc1v8_soc>; ++}; ++ ++&sdmmc { ++ bus-width = <4>; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ no-sdio; ++ no-mmc; ++ card-detect-delay = <800>; ++ ignore-pm-notify; ++ /*cd-gpios = <&gpio2 4 GPIO_ACTIVE_HIGH>; [> CD GPIO <]*/ ++ sd-uhs-sdr12; ++ sd-uhs-sdr25; ++ sd-uhs-sdr50; ++ sd-uhs-sdr104; ++ vqmmc-supply = <&vccio_sd>; ++ vmmc-supply = <&vcc_sd>; ++ status = "disabled"; ++}; ++ ++&sdio { ++ bus-width = <4>; ++ cap-sd-highspeed; ++ no-sd; ++ no-mmc; ++ ignore-pm-notify; ++ keep-power-in-suspend; ++ non-removable; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&tsadc { ++ pinctrl-names = "gpio", "otpout"; ++ pinctrl-0 = <&tsadc_otp_gpio>; ++ pinctrl-1 = <&tsadc_otp_out>; ++ status = "okay"; ++}; ++ ++&u2phy { ++ status = "okay"; ++ ++ u2phy_host: host-port { ++ status = "okay"; ++ }; ++ ++ u2phy_otg: otg-port { ++ status = "okay"; ++ }; ++}; ++ ++&usb20_otg { ++ status = "okay"; ++}; ++ ++&uart1 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart1_xfer &uart1_cts>; ++ status = "okay"; ++}; ++ ++&vip_mmu { ++ status = "okay"; ++}; ++ ++&vopb { ++ status = "okay"; ++}; ++ ++&vopb_mmu { ++ status = "okay"; ++}; ++ ++&vopl { ++ status = "okay"; ++}; ++ ++&vopl_mmu { ++ status = "okay"; ++}; ++ ++&mpp_srv { ++ status = "okay"; ++}; ++ ++&vdpu { ++ status = "okay"; ++}; ++ ++&vepu { ++ status = "okay"; ++}; ++ ++&vpu_mmu { ++ status = "okay"; ++}; ++ ++&hevc { ++ status = "okay"; ++}; ++ ++&hevc_mmu { ++ status = "okay"; ++}; ++ ++&pinctrl { ++ cif-pin-m0 { ++ cif_pin_m0: cif-pin-m0 { ++ rockchip,pins = ++ <2 RK_PA0 1 &pcfg_pull_none>,/* cif_data2 */ ++ <2 RK_PA1 1 &pcfg_pull_none>,/* cif_data3 */ ++ <2 RK_PA2 1 &pcfg_pull_none>,/* cif_data4 */ ++ <2 RK_PA3 1 &pcfg_pull_none>,/* cif_data5 */ ++ <2 RK_PA4 1 &pcfg_pull_none>,/* cif_data6 */ ++ <2 RK_PA5 1 &pcfg_pull_none>,/* cif_data7 */ ++ <2 RK_PA6 1 &pcfg_pull_none>,/* cif_data8 */ ++ <2 RK_PA7 1 &pcfg_pull_none>,/* cif_data9 */ ++ <2 RK_PB0 1 &pcfg_pull_none>,/* cif_sync */ ++ <2 RK_PB1 1 &pcfg_pull_none>,/* cif_href */ ++ <2 RK_PB2 1 &pcfg_pull_none>;/* cif_clkin */ ++ }; ++ }; ++ ++ headphone { ++ hp_det: hp-det { ++ rockchip,pins = <2 RK_PC6 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ pmic { ++ pmic_int: pmic_int { ++ rockchip,pins = ++ <0 RK_PA7 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ soc_slppin_gpio: soc_slppin_gpio { ++ rockchip,pins = ++ <0 RK_PA4 RK_FUNC_GPIO &pcfg_output_low>; ++ }; ++ ++ soc_slppin_slp: soc_slppin_slp { ++ rockchip,pins = ++ <0 RK_PA4 1 &pcfg_pull_none>; ++ }; ++ ++ soc_slppin_rst: soc_slppin_rst { ++ rockchip,pins = ++ <0 RK_PA4 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ ++/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ ++/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ +diff --git a/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v10-robot-linux.dts b/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v10-robot-linux.dts +new file mode 100755 +index 000000000000..6966806150f5 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v10-robot-linux.dts +@@ -0,0 +1,748 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++#include ++#include ++#include ++#include "px30-robot.dtsi" ++ ++/ { ++ model = "Rockchip rk3326 evb lpddr3 v10 board for robot linux"; ++ compatible = "rockchip,rk3326-evb-lp3-v10-robot-linux", "rockchip,rk3326"; ++ ++ adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 2>; ++ io-channel-names = "buttons"; ++ poll-interval = <100>; ++ keyup-threshold-microvolt = <1800000>; ++ ++ esc-key { ++ linux,code = ; ++ label = "esc"; ++ press-threshold-microvolt = <1310000>; ++ }; ++ ++ home-key { ++ linux,code = ; ++ label = "home"; ++ press-threshold-microvolt = <624000>; ++ }; ++ ++ menu-key { ++ linux,code = ; ++ label = "menu"; ++ press-threshold-microvolt = <987000>; ++ }; ++ ++ vol-down-key { ++ linux,code = ; ++ label = "volume down"; ++ press-threshold-microvolt = <300000>; ++ }; ++ ++ vol-up-key { ++ linux,code = ; ++ label = "volume up"; ++ press-threshold-microvolt = <17000>; ++ }; ++ }; ++ ++ rk817-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,rk817-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s1_2ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&rk817_codec>; ++ }; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ /*clocks = <&rk817 1>;*/ ++ /*clock-names = "ext_clock";*/ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_LOW>; /* GPIO3_A4 */ ++ }; ++ ++ vccsys: vccsys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc3v8_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3800000>; ++ regulator-max-microvolt = <3800000>; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ wifi_chip_type = "AP6210"; ++ WIFI,host_wake_irq = <&gpio0 RK_PB2 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++}; ++ ++&bus_apll { ++ bus-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&cif_new { ++ status = "okay"; ++ ++ port { ++ cif_in: endpoint { ++ remote-endpoint = <&gc2155_out>; ++ vsync-active = <0>; ++ hsync-active = <1>; ++ }; ++ }; ++}; ++ ++&cpu0 { ++ cpu-supply = <&vdd_arm>; ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&dmc { ++ center-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&emmc { ++ bus-width = <8>; ++ cap-mmc-highspeed; ++ mmc-hs200-1_8v; ++ no-sdio; ++ no-sd; ++ disable-wp; ++ non-removable; ++ num-slots = <1>; ++ status = "okay"; ++}; ++ ++&gpu { ++ mali-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ clock-frequency = <400000>; ++ i2c-scl-rising-time-ns = <280>; ++ i2c-scl-falling-time-ns = <16>; ++ ++ rk817: pmic@20 { ++ compatible = "rockchip,rk817"; ++ reg = <0x20>; ++ interrupt-parent = <&gpio0>; ++ interrupts = <7 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default", "pmic-sleep", ++ "pmic-power-off", "pmic-reset"; ++ pinctrl-0 = <&pmic_int>; ++ pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; ++ pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; ++ pinctrl-3 = <&soc_slppin_rst>, <&rk817_slppin_rst>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ //fb-inner-reg-idxs = <2>; ++ /* 1: rst regs (default in codes), 0: rst the pmic */ ++ pmic-reset-func = <1>; ++ ++ vcc1-supply = <&vccsys>; ++ vcc2-supply = <&vccsys>; ++ vcc3-supply = <&vccsys>; ++ vcc4-supply = <&vccsys>; ++ vcc5-supply = <&vccsys>; ++ vcc6-supply = <&vccsys>; ++ vcc7-supply = <&vcc_3v0>; ++ vcc8-supply = <&vccsys>; ++ vcc9-supply = <&dcdc_boost>; ++ ++ pwrkey { ++ status = "okay"; ++ }; ++ ++ pinctrl_rk8xx: pinctrl_rk8xx { ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ rk817_ts_gpio1: rk817_ts_gpio1 { ++ pins = "gpio_ts"; ++ function = "pin_fun1"; ++ /* output-low; */ ++ /* input-enable; */ ++ }; ++ ++ rk817_gt_gpio2: rk817_gt_gpio2 { ++ pins = "gpio_gt"; ++ function = "pin_fun1"; ++ }; ++ ++ rk817_pin_ts: rk817_pin_ts { ++ pins = "gpio_ts"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_pin_gt: rk817_pin_gt { ++ pins = "gpio_gt"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_slppin_null: rk817_slppin_null { ++ pins = "gpio_slp"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_slppin_slp: rk817_slppin_slp { ++ pins = "gpio_slp"; ++ function = "pin_fun1"; ++ }; ++ ++ rk817_slppin_pwrdn: rk817_slppin_pwrdn { ++ pins = "gpio_slp"; ++ function = "pin_fun2"; ++ }; ++ ++ rk817_slppin_rst: rk817_slppin_rst { ++ pins = "gpio_slp"; ++ function = "pin_fun3"; ++ }; ++ }; ++ ++ regulators { ++ vdd_logic: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <950000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_logic"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <950000>; ++ }; ++ }; ++ ++ vdd_arm: DCDC_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <950000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_arm"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <950000>; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_ddr"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_3v0: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_3v0"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ ++ vcc_1v0: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ regulator-name = "vcc_1v0"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc1v8_soc: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcc1v8_soc"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd1v0_soc: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ ++ regulator-name = "vcc1v0_soc"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc3v0_pmu: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ ++ regulator-name = "vcc3v0_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ ++ }; ++ }; ++ ++ vccio_sd: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vccio_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc_sd: LDO_REG6 { ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vcc_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ ++ }; ++ }; ++ ++ vcc2v8_dvp: LDO_REG7 { ++ regulator-min-microvolt = <2800000>; ++ regulator-max-microvolt = <2800000>; ++ ++ regulator-name = "vcc2v8_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <2800000>; ++ }; ++ }; ++ ++ vcc1v8_dvp: LDO_REG8 { ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcc1v8_dvp"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd1v5_dvp: LDO_REG9 { ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ ++ regulator-name = "vdd1v5_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <1500000>; ++ }; ++ }; ++ ++ dcdc_boost: BOOST { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <4700000>; ++ regulator-max-microvolt = <5400000>; ++ regulator-name = "boost"; ++ }; ++ ++ otg_switch: OTG_SWITCH { ++ regulator-name = "otg_switch"; ++ }; ++ }; ++ ++ battery { ++ compatible = "rk817,battery"; ++ ocv_table = <3500 3625 3685 3697 3718 3735 3748 ++ 3760 3774 3788 3802 3816 3834 3853 ++ 3877 3908 3946 3975 4018 4071 4106>; ++ design_capacity = <2500>; ++ design_qmax = <2750>; ++ bat_res = <100>; ++ sleep_enter_current = <300>; ++ sleep_exit_current = <300>; ++ sleep_filter_current = <100>; ++ power_off_thresd = <3500>; ++ zero_algorithm_vol = <3850>; ++ max_soc_offset = <60>; ++ monitor_sec = <5>; ++ sample_res = <10>; ++ virtual_power = <1>; ++ }; ++ ++ charger { ++ compatible = "rk817,charger"; ++ min_input_voltage = <4500>; ++ max_input_current = <1500>; ++ max_chrg_current = <2000>; ++ max_chrg_voltage = <4200>; ++ chrg_term_mode = <0>; ++ chrg_finish_cur = <300>; ++ virtual_power = <0>; ++ dc_det_adc = <0>; ++ extcon = <&u2phy>; ++ }; ++ ++ rk817_codec: codec { ++ #sound-dai-cells = <0>; ++ compatible = "rockchip,rk817-codec"; ++ clocks = <&cru SCLK_I2S1_OUT>; ++ clock-names = "mclk"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1_2ch_mclk>; ++ hp-volume = <20>; ++ spk-volume = <3>; ++ status = "okay"; ++ }; ++ }; ++}; ++ ++&i2c2 { ++ status = "okay"; ++ clock-frequency = <400000>; ++ ++ /* 24M mclk is shared for multiple cameras */ ++ pinctrl-0 = <&i2c2_xfer &cif_clkout_m0>; ++ ++ /* These are relatively safe rise/fall times; TODO: measure */ ++ i2c-scl-falling-time-ns = <50>; ++ i2c-scl-rising-time-ns = <300>; ++ ++ gc2155: gc2155@3c { ++ compatible = "gc,gc2155"; ++ reg = <0x3c>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_pin_m0>; ++ ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ ++ avdd-supply = <&vcc2v8_dvp>; ++ dovdd-supply = <&vcc1v8_dvp>; ++ dvdd-supply = <&vcc1v8_dvp>; ++ ++ /* hw changed the pwdn to gpio2_b5 */ ++ pwdn-gpios = <&gpio2 13 GPIO_ACTIVE_HIGH>; ++ ++ port { ++ gc2155_out: endpoint { ++ remote-endpoint = <&cif_in>; ++ }; ++ }; ++ }; ++ ++ ov5695: ov5695@36 { ++ compatible = "ovti,ov5695"; ++ reg = <0x36>; ++ ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ ++ avdd-supply = <&vcc2v8_dvp>; ++ dovdd-supply = <&vcc1v8_dvp>; ++ dvdd-supply = <&vcc1v8_dvp>; ++ ++ /*reset-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>;*/ ++ pwdn-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>; ++ ++ port { ++ ucam_out: endpoint { ++ remote-endpoint = <&mipi_in_ucam>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++}; ++ ++&i2s1_2ch { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++}; ++ ++&io_domains { ++ status = "okay"; ++ ++ vccio1-supply = <&vcc1v8_soc>; ++ vccio2-supply = <&vccio_sd>; ++ vccio3-supply = <&vcc1v8_dvp>; ++ vccio4-supply = <&vcc_3v0>; ++ vccio5-supply = <&vcc_3v0>; ++}; ++ ++&isp_mmu { ++ status = "okay"; ++}; ++ ++&mipi_dphy_rx0 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ucam_out>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_rx0_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp0_mipi_in>; ++ }; ++ }; ++ }; ++}; ++ ++&nandc0 { ++ status = "okay"; ++}; ++ ++&rkisp1 { ++ status = "okay"; ++ ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp0_mipi_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dphy_rx0_out>; ++ }; ++ }; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ ++ pmuio1-supply = <&vcc3v0_pmu>; ++ pmuio2-supply = <&vcc3v0_pmu>; ++}; ++ ++&rk_rga { ++ status = "okay"; ++}; ++ ++&saradc { ++ status = "okay"; ++ vref-supply = <&vcc1v8_soc>; ++}; ++ ++&sdmmc { ++ bus-width = <4>; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ no-sdio; ++ no-mmc; ++ card-detect-delay = <800>; ++ ignore-pm-notify; ++ /*cd-gpios = <&gpio2 4 GPIO_ACTIVE_HIGH>; [> CD GPIO <]*/ ++ sd-uhs-sdr12; ++ sd-uhs-sdr25; ++ sd-uhs-sdr50; ++ sd-uhs-sdr104; ++ vqmmc-supply = <&vccio_sd>; ++ vmmc-supply = <&vcc_sd>; ++ status = "disabled"; ++}; ++ ++&sdio { ++ bus-width = <4>; ++ cap-sd-highspeed; ++ no-sd; ++ no-mmc; ++ ignore-pm-notify; ++ keep-power-in-suspend; ++ non-removable; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&tsadc { ++ pinctrl-names = "init", "default"; ++ pinctrl-0 = <&tsadc_otp_gpio>; ++ pinctrl-1 = <&tsadc_otp_out>; ++ status = "okay"; ++}; ++ ++&u2phy { ++ status = "okay"; ++ ++ u2phy_host: host-port { ++ status = "okay"; ++ }; ++ ++ u2phy_otg: otg-port { ++ status = "okay"; ++ }; ++}; ++ ++&usb20_otg { ++ status = "okay"; ++}; ++ ++&uart1 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart1_xfer &uart1_cts>; ++ status = "okay"; ++}; ++ ++&vip_mmu { ++ status = "okay"; ++}; ++ ++&mpp_srv { ++ status = "okay"; ++}; ++ ++&vdpu { ++ status = "okay"; ++}; ++ ++&vepu { ++ status = "okay"; ++}; ++ ++&vpu_mmu { ++ status = "okay"; ++}; ++ ++&hevc { ++ status = "okay"; ++}; ++ ++&hevc_mmu { ++ status = "okay"; ++}; ++ ++&pinctrl { ++ cif-pin-m0 { ++ cif_pin_m0: cif-pin-m0 { ++ rockchip,pins = ++ <2 RK_PA0 1 &pcfg_pull_none>,/* cif_data2 */ ++ <2 RK_PA1 1 &pcfg_pull_none>,/* cif_data3 */ ++ <2 RK_PA2 1 &pcfg_pull_none>,/* cif_data4 */ ++ <2 RK_PA3 1 &pcfg_pull_none>,/* cif_data5 */ ++ <2 RK_PA4 1 &pcfg_pull_none>,/* cif_data6 */ ++ <2 RK_PA5 1 &pcfg_pull_none>,/* cif_data7 */ ++ <2 RK_PA6 1 &pcfg_pull_none>,/* cif_data8 */ ++ <2 RK_PA7 1 &pcfg_pull_none>,/* cif_data9 */ ++ <2 RK_PB0 1 &pcfg_pull_none>,/* cif_sync */ ++ <2 RK_PB1 1 &pcfg_pull_none>,/* cif_href */ ++ <2 RK_PB2 1 &pcfg_pull_none>;/* cif_clkin */ ++ }; ++ ++ cif_pin_m1: cif-pin-m1 { ++ rockchip,pins = ++ <3 RK_PA3 3 &pcfg_pull_none>,/* cif_data2 */ ++ <3 RK_PA5 3 &pcfg_pull_none>,/* cif_data3 */ ++ <3 RK_PA7 3 &pcfg_pull_none>,/* cif_data4 */ ++ <3 RK_PB0 3 &pcfg_pull_none>,/* cif_data5 */ ++ <3 RK_PB1 3 &pcfg_pull_none>,/* cif_data6 */ ++ <3 RK_PB4 3 &pcfg_pull_none>,/* cif_data7 */ ++ <3 RK_PB6 3 &pcfg_pull_none>,/* cif_data8 */ ++ <3 RK_PB7 3 &pcfg_pull_none>,/* cif_data9 */ ++ <3 RK_PD1 3 &pcfg_pull_none>,/* cif_sync */ ++ <3 RK_PD2 3 &pcfg_pull_none>,/* cif_href */ ++ <3 RK_PD3 3 &pcfg_pull_none>;/* cif_clkin */ ++ }; ++ }; ++ ++ pmic { ++ pmic_int: pmic_int { ++ rockchip,pins = ++ <0 RK_PA7 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ soc_slppin_gpio: soc_slppin_gpio { ++ rockchip,pins = ++ <0 RK_PA4 RK_FUNC_GPIO &pcfg_output_low>; ++ }; ++ ++ soc_slppin_slp: soc_slppin_slp { ++ rockchip,pins = ++ <0 RK_PA4 1 &pcfg_pull_none>; ++ }; ++ ++ soc_slppin_rst: soc_slppin_rst { ++ rockchip,pins = ++ <0 RK_PA4 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ ++/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ ++/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ +diff --git a/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v10-robot-no-gpu-linux.dts b/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v10-robot-no-gpu-linux.dts +new file mode 100755 +index 000000000000..b3b9efec92e4 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v10-robot-no-gpu-linux.dts +@@ -0,0 +1,728 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++#include ++#include ++#include ++#include "px30-robot-no-gpu.dtsi" ++ ++/ { ++ model = "Rockchip rk3326 evb lpddr3 v10 board for robot linux"; ++ compatible = "rockchip,rk3326-evb-lp3-v10-robot-linux", "rockchip,rk3326"; ++ ++ adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 2>; ++ io-channel-names = "buttons"; ++ poll-interval = <100>; ++ keyup-threshold-microvolt = <1800000>; ++ ++ esc-key { ++ linux,code = ; ++ label = "esc"; ++ press-threshold-microvolt = <1310000>; ++ }; ++ ++ home-key { ++ linux,code = ; ++ label = "home"; ++ press-threshold-microvolt = <624000>; ++ }; ++ ++ menu-key { ++ linux,code = ; ++ label = "menu"; ++ press-threshold-microvolt = <987000>; ++ }; ++ ++ vol-down-key { ++ linux,code = ; ++ label = "volume down"; ++ press-threshold-microvolt = <300000>; ++ }; ++ ++ vol-up-key { ++ linux,code = ; ++ label = "volume up"; ++ press-threshold-microvolt = <17000>; ++ }; ++ }; ++ ++ rk817-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,rk817-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s1_2ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&rk817_codec>; ++ }; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ /*clocks = <&rk817 1>;*/ ++ /*clock-names = "ext_clock";*/ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_LOW>; /* GPIO3_A4 */ ++ }; ++ ++ vccsys: vccsys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc3v8_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3800000>; ++ regulator-max-microvolt = <3800000>; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ wifi_chip_type = "AP6210"; ++ WIFI,host_wake_irq = <&gpio0 RK_PB2 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++}; ++ ++&bus_apll { ++ bus-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&cif_new { ++ status = "okay"; ++ ++ port { ++ cif_in: endpoint { ++ remote-endpoint = <&gc2155_out>; ++ vsync-active = <0>; ++ hsync-active = <1>; ++ }; ++ }; ++}; ++ ++&cpu0 { ++ cpu-supply = <&vdd_arm>; ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&dmc { ++ center-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&emmc { ++ bus-width = <8>; ++ cap-mmc-highspeed; ++ mmc-hs200-1_8v; ++ no-sdio; ++ no-sd; ++ disable-wp; ++ non-removable; ++ num-slots = <1>; ++ status = "okay"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ clock-frequency = <400000>; ++ i2c-scl-rising-time-ns = <280>; ++ i2c-scl-falling-time-ns = <16>; ++ ++ rk817: pmic@20 { ++ compatible = "rockchip,rk817"; ++ reg = <0x20>; ++ interrupt-parent = <&gpio0>; ++ interrupts = <7 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default", "pmic-sleep", ++ "pmic-power-off", "pmic-reset"; ++ pinctrl-0 = <&pmic_int>; ++ pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; ++ pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; ++ pinctrl-3 = <&soc_slppin_rst>, <&rk817_slppin_rst>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ //fb-inner-reg-idxs = <2>; ++ /* 1: rst regs (default in codes), 0: rst the pmic */ ++ pmic-reset-func = <1>; ++ ++ vcc1-supply = <&vccsys>; ++ vcc2-supply = <&vccsys>; ++ vcc3-supply = <&vccsys>; ++ vcc4-supply = <&vccsys>; ++ vcc5-supply = <&vccsys>; ++ vcc6-supply = <&vccsys>; ++ vcc7-supply = <&vcc_3v0>; ++ vcc8-supply = <&vccsys>; ++ vcc9-supply = <&dcdc_boost>; ++ ++ pwrkey { ++ status = "okay"; ++ }; ++ ++ pinctrl_rk8xx: pinctrl_rk8xx { ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ rk817_ts_gpio1: rk817_ts_gpio1 { ++ pins = "gpio_ts"; ++ function = "pin_fun1"; ++ /* output-low; */ ++ /* input-enable; */ ++ }; ++ ++ rk817_gt_gpio2: rk817_gt_gpio2 { ++ pins = "gpio_gt"; ++ function = "pin_fun1"; ++ }; ++ ++ rk817_pin_ts: rk817_pin_ts { ++ pins = "gpio_ts"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_pin_gt: rk817_pin_gt { ++ pins = "gpio_gt"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_slppin_null: rk817_slppin_null { ++ pins = "gpio_slp"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_slppin_slp: rk817_slppin_slp { ++ pins = "gpio_slp"; ++ function = "pin_fun1"; ++ }; ++ ++ rk817_slppin_pwrdn: rk817_slppin_pwrdn { ++ pins = "gpio_slp"; ++ function = "pin_fun2"; ++ }; ++ ++ rk817_slppin_rst: rk817_slppin_rst { ++ pins = "gpio_slp"; ++ function = "pin_fun3"; ++ }; ++ }; ++ ++ regulators { ++ vdd_logic: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <950000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_logic"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <950000>; ++ }; ++ }; ++ ++ vdd_arm: DCDC_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <950000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_arm"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <950000>; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_ddr"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_3v0: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_3v0"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ ++ vcc_1v0: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ regulator-name = "vcc_1v0"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc1v8_soc: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcc1v8_soc"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd1v0_soc: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ ++ regulator-name = "vcc1v0_soc"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc3v0_pmu: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ ++ regulator-name = "vcc3v0_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ ++ }; ++ }; ++ ++ vccio_sd: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vccio_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc_sd: LDO_REG6 { ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vcc_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ ++ }; ++ }; ++ ++ vcc2v8_dvp: LDO_REG7 { ++ regulator-min-microvolt = <2800000>; ++ regulator-max-microvolt = <2800000>; ++ ++ regulator-name = "vcc2v8_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <2800000>; ++ }; ++ }; ++ ++ vcc1v8_dvp: LDO_REG8 { ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcc1v8_dvp"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd1v5_dvp: LDO_REG9 { ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ ++ regulator-name = "vdd1v5_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <1500000>; ++ }; ++ }; ++ ++ dcdc_boost: BOOST { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <4700000>; ++ regulator-max-microvolt = <5400000>; ++ regulator-name = "boost"; ++ }; ++ ++ otg_switch: OTG_SWITCH { ++ regulator-name = "otg_switch"; ++ }; ++ }; ++ ++ battery { ++ compatible = "rk817,battery"; ++ ocv_table = <3500 3625 3685 3697 3718 3735 3748 ++ 3760 3774 3788 3802 3816 3834 3853 ++ 3877 3908 3946 3975 4018 4071 4106>; ++ design_capacity = <2500>; ++ design_qmax = <2750>; ++ bat_res = <100>; ++ sleep_enter_current = <300>; ++ sleep_exit_current = <300>; ++ sleep_filter_current = <100>; ++ power_off_thresd = <3500>; ++ zero_algorithm_vol = <3850>; ++ max_soc_offset = <60>; ++ monitor_sec = <5>; ++ sample_res = <10>; ++ virtual_power = <1>; ++ }; ++ ++ charger { ++ compatible = "rk817,charger"; ++ min_input_voltage = <4500>; ++ max_input_current = <1500>; ++ max_chrg_current = <2000>; ++ max_chrg_voltage = <4200>; ++ chrg_term_mode = <0>; ++ chrg_finish_cur = <300>; ++ virtual_power = <0>; ++ dc_det_adc = <0>; ++ extcon = <&u2phy>; ++ }; ++ ++ rk817_codec: codec { ++ #sound-dai-cells = <0>; ++ compatible = "rockchip,rk817-codec"; ++ clocks = <&cru SCLK_I2S1_OUT>; ++ clock-names = "mclk"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1_2ch_mclk>; ++ hp-volume = <20>; ++ spk-volume = <3>; ++ status = "okay"; ++ }; ++ }; ++}; ++ ++&i2c2 { ++ status = "okay"; ++ clock-frequency = <400000>; ++ ++ /* 24M mclk is shared for multiple cameras */ ++ pinctrl-0 = <&i2c2_xfer &cif_clkout_m0>; ++ ++ /* These are relatively safe rise/fall times; TODO: measure */ ++ i2c-scl-falling-time-ns = <50>; ++ i2c-scl-rising-time-ns = <300>; ++ ++ gc2155: gc2155@3c { ++ compatible = "gc,gc2155"; ++ reg = <0x3c>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_pin_m0>; ++ ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ ++ avdd-supply = <&vcc2v8_dvp>; ++ dovdd-supply = <&vcc1v8_dvp>; ++ dvdd-supply = <&vcc1v8_dvp>; ++ ++ /* hw changed the pwdn to gpio2_b5 */ ++ pwdn-gpios = <&gpio2 13 GPIO_ACTIVE_HIGH>; ++ ++ port { ++ gc2155_out: endpoint { ++ remote-endpoint = <&cif_in>; ++ }; ++ }; ++ }; ++ ++ ov5695: ov5695@36 { ++ compatible = "ovti,ov5695"; ++ reg = <0x36>; ++ ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ ++ avdd-supply = <&vcc2v8_dvp>; ++ dovdd-supply = <&vcc1v8_dvp>; ++ dvdd-supply = <&vcc1v8_dvp>; ++ ++ /*reset-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>;*/ ++ pwdn-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>; ++ ++ port { ++ ucam_out: endpoint { ++ remote-endpoint = <&mipi_in_ucam>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++}; ++ ++&i2s1_2ch { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++}; ++ ++&io_domains { ++ status = "okay"; ++ ++ vccio1-supply = <&vcc1v8_soc>; ++ vccio2-supply = <&vccio_sd>; ++ vccio3-supply = <&vcc1v8_dvp>; ++ vccio4-supply = <&vcc_3v0>; ++ vccio5-supply = <&vcc_3v0>; ++}; ++ ++&isp_mmu { ++ status = "okay"; ++}; ++ ++&mipi_dphy_rx0 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ucam_out>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_rx0_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp0_mipi_in>; ++ }; ++ }; ++ }; ++}; ++ ++&nandc0 { ++ status = "okay"; ++}; ++ ++&rkisp1 { ++ status = "okay"; ++ ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp0_mipi_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dphy_rx0_out>; ++ }; ++ }; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ ++ pmuio1-supply = <&vcc3v0_pmu>; ++ pmuio2-supply = <&vcc3v0_pmu>; ++}; ++ ++&rk_rga { ++ status = "okay"; ++}; ++ ++&saradc { ++ status = "okay"; ++ vref-supply = <&vcc1v8_soc>; ++}; ++ ++&sdmmc { ++ bus-width = <4>; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ no-sdio; ++ no-mmc; ++ card-detect-delay = <800>; ++ ignore-pm-notify; ++ /*cd-gpios = <&gpio2 4 GPIO_ACTIVE_HIGH>; [> CD GPIO <]*/ ++ sd-uhs-sdr12; ++ sd-uhs-sdr25; ++ sd-uhs-sdr50; ++ sd-uhs-sdr104; ++ vqmmc-supply = <&vccio_sd>; ++ vmmc-supply = <&vcc_sd>; ++ status = "disabled"; ++}; ++ ++&sdio { ++ bus-width = <4>; ++ cap-sd-highspeed; ++ no-sd; ++ no-mmc; ++ ignore-pm-notify; ++ keep-power-in-suspend; ++ non-removable; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&tsadc { ++ pinctrl-names = "init", "default"; ++ pinctrl-0 = <&tsadc_otp_gpio>; ++ pinctrl-1 = <&tsadc_otp_out>; ++ status = "okay"; ++}; ++ ++&u2phy { ++ status = "okay"; ++ ++ u2phy_host: host-port { ++ status = "okay"; ++ }; ++ ++ u2phy_otg: otg-port { ++ status = "okay"; ++ }; ++}; ++ ++&usb20_otg { ++ status = "okay"; ++}; ++ ++&uart1 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart1_xfer &uart1_cts>; ++ status = "okay"; ++}; ++ ++&vip_mmu { ++ status = "okay"; ++}; ++ ++&mpp_srv { ++ status = "okay"; ++}; ++ ++&vdpu { ++ status = "okay"; ++}; ++ ++&vepu { ++ status = "okay"; ++}; ++ ++&vpu_mmu { ++ status = "okay"; ++}; ++ ++&hevc { ++ status = "okay"; ++}; ++ ++&hevc_mmu { ++ status = "okay"; ++}; ++ ++&pinctrl { ++ cif-pin-m0 { ++ cif_pin_m0: cif-pin-m0 { ++ rockchip,pins = ++ <2 RK_PA0 1 &pcfg_pull_none>,/* cif_data2 */ ++ <2 RK_PA1 1 &pcfg_pull_none>,/* cif_data3 */ ++ <2 RK_PA2 1 &pcfg_pull_none>,/* cif_data4 */ ++ <2 RK_PA3 1 &pcfg_pull_none>,/* cif_data5 */ ++ <2 RK_PA4 1 &pcfg_pull_none>,/* cif_data6 */ ++ <2 RK_PA5 1 &pcfg_pull_none>,/* cif_data7 */ ++ <2 RK_PA6 1 &pcfg_pull_none>,/* cif_data8 */ ++ <2 RK_PA7 1 &pcfg_pull_none>,/* cif_data9 */ ++ <2 RK_PB0 1 &pcfg_pull_none>,/* cif_sync */ ++ <2 RK_PB1 1 &pcfg_pull_none>,/* cif_href */ ++ <2 RK_PB2 1 &pcfg_pull_none>;/* cif_clkin */ ++ }; ++ }; ++ ++ pmic { ++ pmic_int: pmic_int { ++ rockchip,pins = ++ <0 RK_PA7 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ soc_slppin_gpio: soc_slppin_gpio { ++ rockchip,pins = ++ <0 RK_PA4 RK_FUNC_GPIO &pcfg_output_low>; ++ }; ++ ++ soc_slppin_slp: soc_slppin_slp { ++ rockchip,pins = ++ <0 RK_PA4 1 &pcfg_pull_none>; ++ }; ++ ++ soc_slppin_rst: soc_slppin_rst { ++ rockchip,pins = ++ <0 RK_PA4 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ ++/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ ++/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ +diff --git a/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v10.dts b/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v10.dts +new file mode 100755 +index 000000000000..58bbfdafb489 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v10.dts +@@ -0,0 +1,37 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++#include "rk3326-evb-lp3-v10.dtsi" ++ ++/ { ++ model = "Rockchip rk3326 evb board"; ++ compatible = "rockchip,rk3326-evb-lp3-v10", "rockchip,rk3326"; ++}; ++ ++&firmware_android { ++ compatible = "android,firmware"; ++ fstab { ++ compatible = "android,fstab"; ++ system { ++ compatible = "android,system"; ++ dev = "/dev/block/by-name/system"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait"; ++ }; ++ vendor { ++ compatible = "android,vendor"; ++ dev = "/dev/block/by-name/vendor"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait"; ++ }; ++ }; ++}; ++ ++&rk_isp { ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v10.dtsi b/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v10.dtsi +new file mode 100755 +index 000000000000..940c05df8304 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v10.dtsi +@@ -0,0 +1,887 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++#include ++#include ++#include ++#include ++#include ++#include "rk3326.dtsi" ++#include "rk3326-863-cif-sensor.dtsi" ++#include "px30-android.dtsi" ++ ++/ { ++ adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 2>; ++ io-channel-names = "buttons"; ++ poll-interval = <100>; ++ keyup-threshold-microvolt = <1800000>; ++ ++ esc-key { ++ linux,code = ; ++ label = "esc"; ++ press-threshold-microvolt = <1310000>; ++ }; ++ ++ home-key { ++ linux,code = ; ++ label = "home"; ++ press-threshold-microvolt = <624000>; ++ }; ++ ++ menu-key { ++ linux,code = ; ++ label = "menu"; ++ press-threshold-microvolt = <987000>; ++ }; ++ ++ vol-down-key { ++ linux,code = ; ++ label = "volume down"; ++ press-threshold-microvolt = <300000>; ++ }; ++ ++ vol-up-key { ++ linux,code = ; ++ label = "volume up"; ++ press-threshold-microvolt = <17000>; ++ }; ++ }; ++ ++ backlight: backlight { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm1 0 25000 0>; ++ brightness-levels = < ++ 0 1 2 3 4 5 6 7 ++ 8 9 10 11 12 13 14 15 ++ 16 17 18 19 20 21 22 23 ++ 24 25 26 27 28 29 30 31 ++ 32 33 34 35 36 37 38 39 ++ 40 41 42 43 44 45 46 47 ++ 48 49 50 51 52 53 54 55 ++ 56 57 58 59 60 61 62 63 ++ 64 65 66 67 68 69 70 71 ++ 72 73 74 75 76 77 78 79 ++ 80 81 82 83 84 85 86 87 ++ 88 89 90 91 92 93 94 95 ++ 96 97 98 99 100 101 102 103 ++ 104 105 106 107 108 109 110 111 ++ 112 113 114 115 116 117 118 119 ++ 120 121 122 123 124 125 126 127 ++ 128 129 130 131 132 133 134 135 ++ 136 137 138 139 140 141 142 143 ++ 144 145 146 147 148 149 150 151 ++ 152 153 154 155 156 157 158 159 ++ 160 161 162 163 164 165 166 167 ++ 168 169 170 171 172 173 174 175 ++ 176 177 178 179 180 181 182 183 ++ 184 185 186 187 188 189 190 191 ++ 192 193 194 195 196 197 198 199 ++ 200 201 202 203 204 205 206 207 ++ 208 209 210 211 212 213 214 215 ++ 216 217 218 219 220 221 222 223 ++ 224 225 226 227 228 229 230 231 ++ 232 233 234 235 236 237 238 239 ++ 240 241 242 243 244 245 246 247 ++ 248 249 250 251 252 253 254 255>; ++ default-brightness-level = <200>; ++ }; ++ ++ rk817-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,rk817-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,widgets = ++ "Microphone", "Mic Jack", ++ "Headphone", "Headphone Jack"; ++ simple-audio-card,routing = ++ "Mic Jack", "MICBIAS1", ++ "IN1P", "Mic Jack", ++ "Headphone Jack", "HPOL", ++ "Headphone Jack", "HPOR"; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s1_2ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&rk817_codec>; ++ }; ++ }; ++ ++ rk_headset: rk-headset { ++ compatible = "rockchip_headset"; ++ headset_gpio = <&gpio2 RK_PC6 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hp_det>; ++ io-channels = <&saradc 1>; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ /*clocks = <&rk817 1>;*/ ++ /*clock-names = "ext_clock";*/ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_LOW>; /* GPIO3_A4 */ ++ }; ++ ++ vccsys: vccsys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc3v8_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3800000>; ++ regulator-max-microvolt = <3800000>; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ wifi_chip_type = "AP6210"; ++ WIFI,host_wake_irq = <&gpio0 RK_PB2 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&rk817 1>; ++ clock-names = "ext_clock"; ++ uart_rts_gpios = <&gpio1 RK_PC3 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default","rts_gpio"; ++ pinctrl-0 = <&uart1_rts>; ++ pinctrl-1 = <&uart1_rts_gpio>; ++ BT,reset_gpio = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio0 RK_PA1 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio0 RK_PB3 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ vcc18_lcd_n: vcc18-lcd-n { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc18_lcd_n"; ++ regulator-boot-on; ++ gpio = <&gpio0 RK_PB5 GPIO_ACTIVE_HIGH>; ++ enable-active-high; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++}; ++ ++&bus_apll { ++ bus-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&cpu0 { ++ cpu-supply = <&vdd_arm>; ++}; ++ ++&display_subsystem { ++ status = "okay"; ++}; ++ ++&dsi { ++ status = "okay"; ++ ++ panel@0 { ++ compatible = "sitronix,st7703", "simple-panel-dsi"; ++ reg = <0>; ++ backlight = <&backlight>; ++ power-supply = <&vcc18_lcd_n>; ++ prepare-delay-ms = <2>; ++ reset-delay-ms = <1>; ++ init-delay-ms = <20>; ++ enable-delay-ms = <120>; ++ disable-delay-ms = <50>; ++ unprepare-delay-ms = <20>; ++ ++ width-mm = <68>; ++ height-mm = <121>; ++ ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | ++ MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ ++ panel-init-sequence = [ ++ 05 fa 01 11 ++ 39 00 04 b9 f1 12 83 ++ 39 00 1c ba 33 81 05 f9 0e 0e 00 00 00 ++ 00 00 00 00 00 44 25 00 91 0a ++ 00 00 02 4f 01 00 00 37 ++ 15 00 02 b8 25 ++ 39 00 04 bf 02 11 00 ++ 39 00 0b b3 0c 10 0a 50 03 ff 00 00 00 ++ 00 ++ 39 00 0a c0 73 73 50 50 00 00 08 70 00 ++ 15 00 02 bc 46 ++ 15 00 02 cc 0b ++ 15 00 02 b4 80 ++ 39 00 04 b2 c8 12 30 ++ 39 00 0f e3 07 07 0b 0b 03 0b 00 00 00 ++ 00 ff 00 c0 10 ++ 39 00 0d c1 53 00 1e 1e 77 e1 cc dd 67 ++ 77 33 33 ++ 39 00 07 c6 00 00 ff ff 01 ff ++ 39 00 03 b5 09 09 ++ 39 00 03 b6 87 95 ++ 39 00 40 e9 c2 10 05 05 10 05 a0 12 31 ++ 23 3f 81 0a a0 37 18 00 80 01 ++ 00 00 00 00 80 01 00 00 00 48 ++ f8 86 42 08 88 88 80 88 88 88 ++ 58 f8 87 53 18 88 88 81 88 88 ++ 88 00 00 00 01 00 00 00 00 00 ++ 00 00 00 00 ++ 39 00 3e ea 00 1a 00 00 00 00 02 00 00 ++ 00 00 00 1f 88 81 35 78 88 88 ++ 85 88 88 88 0f 88 80 24 68 88 ++ 88 84 88 88 88 23 10 00 00 1c ++ 00 00 00 00 00 00 00 00 00 00 ++ 00 00 00 00 00 30 05 a0 00 00 ++ 00 00 ++ 39 00 23 e0 00 06 08 2a 31 3f 38 36 07 ++ 0c 0d 11 13 12 13 11 18 00 06 ++ 08 2a 31 3f 38 36 07 0c 0d 11 ++ 13 12 13 11 18 ++ 05 32 01 29 ++ ]; ++ ++ panel-exit-sequence = [ ++ 05 00 01 28 ++ 05 00 01 10 ++ ]; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <66000000>; ++ hactive = <720>; ++ vactive = <1280>; ++ hfront-porch = <40>; ++ hsync-len = <10>; ++ hback-porch = <40>; ++ vfront-porch = <22>; ++ vsync-len = <4>; ++ vback-porch = <11>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ panel_in_dsi: endpoint { ++ remote-endpoint = <&dsi_out_panel>; ++ }; ++ }; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@1 { ++ reg = <1>; ++ dsi_out_panel: endpoint { ++ remote-endpoint = <&panel_in_dsi>; ++ }; ++ }; ++ }; ++}; ++ ++&dsi_in_vopb { ++ status = "okay"; ++}; ++ ++&dsi_in_vopl { ++ status = "disabled"; ++}; ++ ++&route_dsi { ++ connect = <&vopb_out_dsi>; ++ status = "okay"; ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&dmc { ++ center-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&emmc { ++ bus-width = <8>; ++ cap-mmc-highspeed; ++ mmc-hs200-1_8v; ++ no-sdio; ++ no-sd; ++ disable-wp; ++ non-removable; ++ num-slots = <1>; ++ status = "okay"; ++}; ++ ++&gpu { ++ mali-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ clock-frequency = <400000>; ++ i2c-scl-rising-time-ns = <280>; ++ i2c-scl-falling-time-ns = <16>; ++ ++ rk817: pmic@20 { ++ compatible = "rockchip,rk817"; ++ reg = <0x20>; ++ interrupt-parent = <&gpio0>; ++ interrupts = <7 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default", "pmic-sleep", ++ "pmic-power-off", "pmic-reset"; ++ pinctrl-0 = <&pmic_int>; ++ pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; ++ pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; ++ pinctrl-3 = <&soc_slppin_rst>, <&rk817_slppin_rst>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ //fb-inner-reg-idxs = <2>; ++ /* 1: rst regs (default in codes), 0: rst the pmic */ ++ pmic-reset-func = <1>; ++ ++ vcc1-supply = <&vccsys>; ++ vcc2-supply = <&vccsys>; ++ vcc3-supply = <&vccsys>; ++ vcc4-supply = <&vccsys>; ++ vcc5-supply = <&vccsys>; ++ vcc6-supply = <&vccsys>; ++ vcc7-supply = <&vcc_3v0>; ++ vcc8-supply = <&vccsys>; ++ vcc9-supply = <&dcdc_boost>; ++ ++ pwrkey { ++ status = "okay"; ++ }; ++ ++ pinctrl_rk8xx: pinctrl_rk8xx { ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ rk817_ts_gpio1: rk817_ts_gpio1 { ++ pins = "gpio_ts"; ++ function = "pin_fun1"; ++ /* output-low; */ ++ /* input-enable; */ ++ }; ++ ++ rk817_gt_gpio2: rk817_gt_gpio2 { ++ pins = "gpio_gt"; ++ function = "pin_fun1"; ++ }; ++ ++ rk817_pin_ts: rk817_pin_ts { ++ pins = "gpio_ts"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_pin_gt: rk817_pin_gt { ++ pins = "gpio_gt"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_slppin_null: rk817_slppin_null { ++ pins = "gpio_slp"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_slppin_slp: rk817_slppin_slp { ++ pins = "gpio_slp"; ++ function = "pin_fun1"; ++ }; ++ ++ rk817_slppin_pwrdn: rk817_slppin_pwrdn { ++ pins = "gpio_slp"; ++ function = "pin_fun2"; ++ }; ++ ++ rk817_slppin_rst: rk817_slppin_rst { ++ pins = "gpio_slp"; ++ function = "pin_fun3"; ++ }; ++ }; ++ ++ regulators { ++ vdd_logic: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <950000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_logic"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <950000>; ++ }; ++ }; ++ ++ vdd_arm: DCDC_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <950000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_arm"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <950000>; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_ddr"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_3v0: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_3v0"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ ++ vcc_1v0: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ regulator-name = "vcc_1v0"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc1v8_soc: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcc1v8_soc"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd1v0_soc: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ ++ regulator-name = "vcc1v0_soc"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc3v0_pmu: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ ++ regulator-name = "vcc3v0_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ ++ }; ++ }; ++ ++ vccio_sd: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vccio_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc_sd: LDO_REG6 { ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vcc_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ ++ }; ++ }; ++ ++ vcc2v8_dvp: LDO_REG7 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <2800000>; ++ regulator-max-microvolt = <2800000>; ++ ++ regulator-name = "vcc2v8_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <2800000>; ++ }; ++ }; ++ ++ vcc1v8_dvp: LDO_REG8 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcc1v8_dvp"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd1v5_dvp: LDO_REG9 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ ++ regulator-name = "vdd1v5_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <1500000>; ++ }; ++ }; ++ ++ dcdc_boost: BOOST { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <4700000>; ++ regulator-max-microvolt = <5400000>; ++ regulator-name = "boost"; ++ }; ++ ++ otg_switch: OTG_SWITCH { ++ regulator-name = "otg_switch"; ++ }; ++ }; ++ ++ battery { ++ compatible = "rk817,battery"; ++ ocv_table = <3500 3625 3685 3697 3718 3735 3748 ++ 3760 3774 3788 3802 3816 3834 3853 ++ 3877 3908 3946 3975 4018 4071 4106>; ++ design_capacity = <2500>; ++ design_qmax = <2750>; ++ bat_res = <100>; ++ sleep_enter_current = <300>; ++ sleep_exit_current = <300>; ++ sleep_filter_current = <100>; ++ power_off_thresd = <3500>; ++ zero_algorithm_vol = <3850>; ++ max_soc_offset = <60>; ++ monitor_sec = <5>; ++ sample_res = <10>; ++ virtual_power = <1>; ++ }; ++ ++ charger { ++ compatible = "rk817,charger"; ++ min_input_voltage = <4500>; ++ max_input_current = <1500>; ++ max_chrg_current = <2000>; ++ max_chrg_voltage = <4200>; ++ chrg_term_mode = <0>; ++ chrg_finish_cur = <300>; ++ virtual_power = <0>; ++ dc_det_adc = <0>; ++ extcon = <&u2phy>; ++ }; ++ ++ rk817_codec: codec { ++ #sound-dai-cells = <0>; ++ compatible = "rockchip,rk817-codec"; ++ clocks = <&cru SCLK_I2S1_OUT>; ++ clock-names = "mclk"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1_2ch_mclk>; ++ hp-volume = <20>; ++ spk-volume = <3>; ++ status = "okay"; ++ }; ++ }; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ clock-frequency = <400000>; ++ i2c-scl-rising-time-ns = <275>; ++ i2c-scl-falling-time-ns = <16>; ++ ++ sensor@f { ++ status = "okay"; ++ compatible = "ak8963"; ++ reg = <0x0f>; ++ type = ; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ layout = <1>; ++ reprobe_en = <1>; ++ }; ++ ++ gt1x: gt1x@14 { ++ compatible = "goodix,gt1x"; ++ reg = <0x14>; ++ power-supply = <&vcc18_lcd_n>; ++ goodix,rst-gpio = <&gpio0 RK_PB4 GPIO_ACTIVE_HIGH>; ++ goodix,irq-gpio = <&gpio0 RK_PA5 IRQ_TYPE_LEVEL_LOW>; ++ }; ++ ++ sensor@4c { ++ status = "okay"; ++ compatible = "gs_mma7660"; ++ reg = <0x4c>; ++ type = ; ++ irq-gpio = <&gpio0 RK_PB5 IRQ_TYPE_LEVEL_LOW>; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ layout = <1>; ++ reprobe_en = <1>; ++ }; ++}; ++ ++&i2c2 { ++ status = "okay"; ++}; ++ ++&i2s1_2ch { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++}; ++ ++&io_domains { ++ status = "okay"; ++ ++ vccio1-supply = <&vcc1v8_soc>; ++ vccio2-supply = <&vccio_sd>; ++ vccio3-supply = <&vcc1v8_dvp>; ++ vccio4-supply = <&vcc_3v0>; ++ vccio5-supply = <&vcc_3v0>; ++}; ++ ++&isp_mmu { ++ status = "okay"; ++}; ++ ++&nandc0 { ++ status = "okay"; ++}; ++ ++&pinctrl { ++ headphone { ++ hp_det: hp-det { ++ rockchip,pins = <2 RK_PC6 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ pmic { ++ pmic_int: pmic_int { ++ rockchip,pins = ++ <0 RK_PA7 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ soc_slppin_gpio: soc_slppin_gpio { ++ rockchip,pins = ++ <0 RK_PA4 RK_FUNC_GPIO &pcfg_output_low>; ++ }; ++ ++ soc_slppin_slp: soc_slppin_slp { ++ rockchip,pins = ++ <0 RK_PA4 1 &pcfg_pull_none>; ++ }; ++ ++ soc_slppin_rst: soc_slppin_rst { ++ rockchip,pins = ++ <0 RK_PA4 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ ++ pmuio1-supply = <&vcc3v0_pmu>; ++ pmuio2-supply = <&vcc3v0_pmu>; ++}; ++ ++&pwm1 { ++ status = "okay"; ++}; ++ ++&rk_rga { ++ status = "okay"; ++}; ++ ++&rockchip_suspend { ++ status = "okay"; ++ rockchip,sleep-debug-en = <1>; ++}; ++ ++&saradc { ++ status = "okay"; ++ vref-supply = <&vcc1v8_soc>; ++}; ++ ++&sdmmc { ++ bus-width = <4>; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ no-sdio; ++ no-mmc; ++ card-detect-delay = <800>; ++ ignore-pm-notify; ++ /*cd-gpios = <&gpio2 4 GPIO_ACTIVE_HIGH>; [> CD GPIO <]*/ ++ sd-uhs-sdr12; ++ sd-uhs-sdr25; ++ sd-uhs-sdr50; ++ sd-uhs-sdr104; ++ vqmmc-supply = <&vccio_sd>; ++ vmmc-supply = <&vcc_sd>; ++ status = "disabled"; ++}; ++ ++&sdio { ++ bus-width = <4>; ++ cap-sd-highspeed; ++ no-sd; ++ no-mmc; ++ ignore-pm-notify; ++ keep-power-in-suspend; ++ non-removable; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&tsadc { ++ pinctrl-names = "gpio", "otpout"; ++ pinctrl-0 = <&tsadc_otp_gpio>; ++ pinctrl-1 = <&tsadc_otp_out>; ++ status = "okay"; ++}; ++ ++&u2phy { ++ status = "okay"; ++ ++ u2phy_host: host-port { ++ status = "okay"; ++ }; ++ ++ u2phy_otg: otg-port { ++ status = "okay"; ++ }; ++}; ++ ++&usb20_otg { ++ status = "okay"; ++}; ++ ++&uart1 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart1_xfer &uart1_cts>; ++ status = "okay"; ++}; ++ ++&vopb { ++ status = "okay"; ++}; ++ ++&vopb_mmu { ++ status = "okay"; ++}; ++ ++&vopl { ++ status = "okay"; ++}; ++ ++&vopl_mmu { ++ status = "okay"; ++}; ++ ++&mpp_srv { ++ status = "okay"; ++}; ++ ++&vdpu { ++ status = "okay"; ++}; ++ ++&vepu { ++ status = "okay"; ++}; ++ ++&vpu_mmu { ++ status = "okay"; ++}; ++ ++&hevc { ++ status = "okay"; ++}; ++ ++&hevc_mmu { ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v11-avb.dts b/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v11-avb.dts +new file mode 100755 +index 000000000000..2f0c3fc3a38d +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v11-avb.dts +@@ -0,0 +1,351 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2021 Rockchip Electronics Co., Ltd. ++ */ ++ ++/dts-v1/; ++#include "rk3326-evb-lp3-v10.dtsi" ++ ++/ { ++ model = "Rockchip rk3326 evb board"; ++ compatible = "rockchip,rk3326-evb-lp3-v11-avb", "rockchip,rk3326"; ++}; ++ ++&dsi { ++ status = "okay"; ++ ++ panel@0 { ++ compatible = "simple-panel-dsi"; ++ reg = <0>; ++ power-supply = <&vcc18_lcd_n>; ++ backlight = <&backlight>; ++ prepare-delay-ms = <0>; ++ reset-delay-ms = <0>; ++ init-delay-ms = <80>; ++ enable-delay-ms = <0>; ++ disable-delay-ms = <10>; ++ unprepare-delay-ms = <60>; ++ ++ width-mm = <68>; ++ height-mm = <121>; ++ ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | ++ MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ ++ panel-init-sequence = [ ++ 39 00 04 ff 98 81 03 ++ 15 00 02 01 00 ++ 15 00 02 02 00 ++ 15 00 02 03 53 ++ 15 00 02 04 53 ++ 15 00 02 05 13 ++ 15 00 02 06 04 ++ 15 00 02 07 02 ++ 15 00 02 08 02 ++ 15 00 02 09 00 ++ 15 00 02 0a 00 ++ 15 00 02 0b 00 ++ 15 00 02 0c 00 ++ 15 00 02 0d 00 ++ 15 00 02 0e 00 ++ 15 00 02 0f 00 ++ ++ 15 00 02 10 00 ++ 15 00 02 11 00 ++ 15 00 02 12 00 ++ 15 00 02 13 00 ++ 15 00 02 14 00 ++ 15 00 02 15 08 ++ 15 00 02 16 10 ++ 15 00 02 17 00 ++ 15 00 02 18 08 ++ 15 00 02 19 00 ++ 15 00 02 1a 00 ++ 15 00 02 1b 00 ++ 15 00 02 1c 00 ++ 15 00 02 1d 00 ++ 15 00 02 1e c0 ++ 15 00 02 1f 80 ++ ++ 15 00 02 20 02 ++ 15 00 02 21 09 ++ 15 00 02 22 00 ++ 15 00 02 23 00 ++ 15 00 02 24 00 ++ 15 00 02 25 00 ++ 15 00 02 26 00 ++ 15 00 02 27 00 ++ 15 00 02 28 55 ++ 15 00 02 29 03 ++ 15 00 02 2a 00 ++ 15 00 02 2b 00 ++ 15 00 02 2c 00 ++ 15 00 02 2d 00 ++ 15 00 02 2e 00 ++ 15 00 02 2f 00 ++ ++ 15 00 02 30 00 ++ 15 00 02 31 00 ++ 15 00 02 32 00 ++ 15 00 02 33 00 ++ 15 00 02 34 04 ++ 15 00 02 35 05 ++ 15 00 02 36 05 ++ 15 00 02 37 00 ++ 15 00 02 38 3c ++ 15 00 02 39 35 ++ 15 00 02 3a 00 ++ 15 00 02 3b 40 ++ 15 00 02 3c 00 ++ 15 00 02 3d 00 ++ 15 00 02 3e 00 ++ 15 00 02 3f 00 ++ ++ 15 00 02 40 00 ++ 15 00 02 41 88 ++ 15 00 02 42 00 ++ 15 00 02 43 00 ++ 15 00 02 44 1f ++ ++ 15 00 02 50 01 ++ 15 00 02 51 23 ++ 15 00 02 52 45 ++ 15 00 02 53 67 ++ 15 00 02 54 89 ++ 15 00 02 55 ab ++ 15 00 02 56 01 ++ 15 00 02 57 23 ++ 15 00 02 58 45 ++ 15 00 02 59 67 ++ 15 00 02 5a 89 ++ 15 00 02 5b ab ++ 15 00 02 5c cd ++ 15 00 02 5d ef ++ 15 00 02 5e 03 ++ 15 00 02 5f 14 ++ ++ 15 00 02 60 15 ++ 15 00 02 61 0c ++ 15 00 02 62 0d ++ 15 00 02 63 0e ++ 15 00 02 64 0f ++ 15 00 02 65 10 ++ 15 00 02 66 11 ++ 15 00 02 67 08 ++ 15 00 02 68 02 ++ 15 00 02 69 0a ++ 15 00 02 6a 02 ++ 15 00 02 6b 02 ++ 15 00 02 6c 02 ++ 15 00 02 6d 02 ++ 15 00 02 6e 02 ++ 15 00 02 6f 02 ++ ++ 15 00 02 70 02 ++ 15 00 02 71 02 ++ 15 00 02 72 06 ++ 15 00 02 73 02 ++ 15 00 02 74 02 ++ 15 00 02 75 14 ++ 15 00 02 76 15 ++ 15 00 02 77 0f ++ 15 00 02 78 0e ++ 15 00 02 79 0d ++ 15 00 02 7a 0c ++ 15 00 02 7b 11 ++ 15 00 02 7c 10 ++ 15 00 02 7d 06 ++ 15 00 02 7e 02 ++ 15 00 02 7f 0a ++ ++ 15 00 02 80 02 ++ 15 00 02 81 02 ++ 15 00 02 82 02 ++ 15 00 02 83 02 ++ 15 00 02 84 02 ++ 15 00 02 85 02 ++ 15 00 02 86 02 ++ 15 00 02 87 02 ++ 15 00 02 88 08 ++ 15 00 02 89 02 ++ 15 00 02 8a 02 ++ ++ 39 00 04 ff 98 81 04 ++ 15 00 02 00 80 ++ 15 00 02 70 00 ++ 15 00 02 71 00 ++ 15 00 02 66 fe ++ 15 00 02 82 15 ++ 15 00 02 84 15 ++ 15 00 02 85 15 ++ 15 00 02 3a 24 ++ 15 00 02 32 ac ++ 15 00 02 8c 80 ++ 15 00 02 3c f5 ++ 15 00 02 88 33 ++ ++ 39 00 04 ff 98 81 01 ++ 15 00 02 22 0a ++ 15 00 02 31 00 ++ 15 00 02 53 78 ++ 15 00 02 50 5b ++ 15 00 02 51 5b ++ 15 00 02 60 20 ++ 15 00 02 61 00 ++ 15 00 02 62 0d ++ 15 00 02 63 00 ++ ++ 15 00 02 a0 00 ++ 15 00 02 a1 10 ++ 15 00 02 a2 1c ++ 15 00 02 a3 13 ++ 15 00 02 a4 15 ++ 15 00 02 a5 26 ++ 15 00 02 a6 1a ++ 15 00 02 a7 1d ++ 15 00 02 a8 67 ++ 15 00 02 a9 1c ++ 15 00 02 aa 29 ++ 15 00 02 ab 5b ++ 15 00 02 ac 26 ++ 15 00 02 ad 28 ++ 15 00 02 ae 5c ++ 15 00 02 af 30 ++ 15 00 02 b0 31 ++ 15 00 02 b1 2e ++ 15 00 02 b2 32 ++ 15 00 02 b3 00 ++ ++ 15 00 02 c0 00 ++ 15 00 02 c1 10 ++ 15 00 02 c2 1c ++ 15 00 02 c3 13 ++ 15 00 02 c4 15 ++ 15 00 02 c5 26 ++ 15 00 02 c6 1a ++ 15 00 02 c7 1d ++ 15 00 02 c8 67 ++ 15 00 02 c9 1c ++ 15 00 02 ca 29 ++ 15 00 02 cb 5b ++ 15 00 02 cc 26 ++ 15 00 02 cd 28 ++ 15 00 02 ce 5c ++ 15 00 02 cf 30 ++ 15 00 02 d0 31 ++ 15 00 02 d1 2e ++ 15 00 02 d2 32 ++ 15 00 02 d3 00 ++ 39 00 04 ff 98 81 00 ++ 05 00 01 11 ++ 05 01 01 29 ++ ]; ++ ++ panel-exit-sequence = [ ++ 05 00 01 28 ++ 05 00 01 10 ++ ]; ++ ++ display-timings { ++ native-mode = <&timing1>; ++ ++ timing1: timing1 { ++ clock-frequency = <64000000>; ++ hactive = <720>; ++ vactive = <1280>; ++ hfront-porch = <40>; ++ hsync-len = <10>; ++ hback-porch = <40>; ++ vfront-porch = <22>; ++ vsync-len = <4>; ++ vback-porch = <11>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ }; ++ ++}; ++ ++&i2c2 { ++ status = "okay"; ++ ++ clock-frequency = <100000>; ++ ++ /* These are relatively safe rise/fall times; TODO: measure */ ++ i2c-scl-falling-time-ns = <50>; ++ i2c-scl-rising-time-ns = <300>; ++ ++ ov5695: ov5695@36 { ++ compatible = "ovti,ov5695"; ++ reg = <0x36>; ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ /*reset-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>;*/ ++ pwdn-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>; ++ //pinctrl-names = "default"; ++ //pinctrl-0 = <&cif_clkout_m0>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "TongJu"; ++ rockchip,camera-module-lens-name = "CHT842-MD"; ++ port { ++ ov5695_out: endpoint { ++ remote-endpoint = <&mipi_in>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++}; ++ ++&mipi_dphy_rx0 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ov5695_out>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_rx_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp_mipi_in>; ++ }; ++ }; ++ }; ++}; ++ ++&rkisp1 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_clkout_m0 &dvp_d0d1_m0 &dvp_d2d9_m0>; ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp_mipi_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dphy_rx_out>; ++ }; ++ ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v11.dts b/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v11.dts +new file mode 100755 +index 000000000000..139efd9d1332 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v11.dts +@@ -0,0 +1,297 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2021 Rockchip Electronics Co., Ltd. ++ */ ++ ++/dts-v1/; ++#include "rk3326-evb-lp3-v10.dtsi" ++ ++/ { ++ model = "Rockchip rk3326 evb board"; ++ compatible = "rockchip,rk3326-evb-lp3-v11", "rockchip,rk3326"; ++}; ++ ++&dsi { ++ status = "okay"; ++ ++ panel@0 { ++ compatible = "simple-panel-dsi"; ++ reg = <0>; ++ power-supply = <&vcc18_lcd_n>; ++ backlight = <&backlight>; ++ prepare-delay-ms = <0>; ++ reset-delay-ms = <0>; ++ init-delay-ms = <80>; ++ enable-delay-ms = <0>; ++ disable-delay-ms = <10>; ++ unprepare-delay-ms = <60>; ++ ++ width-mm = <68>; ++ height-mm = <121>; ++ ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | ++ MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ ++ panel-init-sequence = [ ++ 39 00 04 ff 98 81 03 ++ 15 00 02 01 00 ++ 15 00 02 02 00 ++ 15 00 02 03 53 ++ 15 00 02 04 53 ++ 15 00 02 05 13 ++ 15 00 02 06 04 ++ 15 00 02 07 02 ++ 15 00 02 08 02 ++ 15 00 02 09 00 ++ 15 00 02 0a 00 ++ 15 00 02 0b 00 ++ 15 00 02 0c 00 ++ 15 00 02 0d 00 ++ 15 00 02 0e 00 ++ 15 00 02 0f 00 ++ ++ 15 00 02 10 00 ++ 15 00 02 11 00 ++ 15 00 02 12 00 ++ 15 00 02 13 00 ++ 15 00 02 14 00 ++ 15 00 02 15 08 ++ 15 00 02 16 10 ++ 15 00 02 17 00 ++ 15 00 02 18 08 ++ 15 00 02 19 00 ++ 15 00 02 1a 00 ++ 15 00 02 1b 00 ++ 15 00 02 1c 00 ++ 15 00 02 1d 00 ++ 15 00 02 1e c0 ++ 15 00 02 1f 80 ++ ++ 15 00 02 20 02 ++ 15 00 02 21 09 ++ 15 00 02 22 00 ++ 15 00 02 23 00 ++ 15 00 02 24 00 ++ 15 00 02 25 00 ++ 15 00 02 26 00 ++ 15 00 02 27 00 ++ 15 00 02 28 55 ++ 15 00 02 29 03 ++ 15 00 02 2a 00 ++ 15 00 02 2b 00 ++ 15 00 02 2c 00 ++ 15 00 02 2d 00 ++ 15 00 02 2e 00 ++ 15 00 02 2f 00 ++ ++ 15 00 02 30 00 ++ 15 00 02 31 00 ++ 15 00 02 32 00 ++ 15 00 02 33 00 ++ 15 00 02 34 04 ++ 15 00 02 35 05 ++ 15 00 02 36 05 ++ 15 00 02 37 00 ++ 15 00 02 38 3c ++ 15 00 02 39 35 ++ 15 00 02 3a 00 ++ 15 00 02 3b 40 ++ 15 00 02 3c 00 ++ 15 00 02 3d 00 ++ 15 00 02 3e 00 ++ 15 00 02 3f 00 ++ ++ 15 00 02 40 00 ++ 15 00 02 41 88 ++ 15 00 02 42 00 ++ 15 00 02 43 00 ++ 15 00 02 44 1f ++ ++ 15 00 02 50 01 ++ 15 00 02 51 23 ++ 15 00 02 52 45 ++ 15 00 02 53 67 ++ 15 00 02 54 89 ++ 15 00 02 55 ab ++ 15 00 02 56 01 ++ 15 00 02 57 23 ++ 15 00 02 58 45 ++ 15 00 02 59 67 ++ 15 00 02 5a 89 ++ 15 00 02 5b ab ++ 15 00 02 5c cd ++ 15 00 02 5d ef ++ 15 00 02 5e 03 ++ 15 00 02 5f 14 ++ ++ 15 00 02 60 15 ++ 15 00 02 61 0c ++ 15 00 02 62 0d ++ 15 00 02 63 0e ++ 15 00 02 64 0f ++ 15 00 02 65 10 ++ 15 00 02 66 11 ++ 15 00 02 67 08 ++ 15 00 02 68 02 ++ 15 00 02 69 0a ++ 15 00 02 6a 02 ++ 15 00 02 6b 02 ++ 15 00 02 6c 02 ++ 15 00 02 6d 02 ++ 15 00 02 6e 02 ++ 15 00 02 6f 02 ++ ++ 15 00 02 70 02 ++ 15 00 02 71 02 ++ 15 00 02 72 06 ++ 15 00 02 73 02 ++ 15 00 02 74 02 ++ 15 00 02 75 14 ++ 15 00 02 76 15 ++ 15 00 02 77 0f ++ 15 00 02 78 0e ++ 15 00 02 79 0d ++ 15 00 02 7a 0c ++ 15 00 02 7b 11 ++ 15 00 02 7c 10 ++ 15 00 02 7d 06 ++ 15 00 02 7e 02 ++ 15 00 02 7f 0a ++ ++ 15 00 02 80 02 ++ 15 00 02 81 02 ++ 15 00 02 82 02 ++ 15 00 02 83 02 ++ 15 00 02 84 02 ++ 15 00 02 85 02 ++ 15 00 02 86 02 ++ 15 00 02 87 02 ++ 15 00 02 88 08 ++ 15 00 02 89 02 ++ 15 00 02 8a 02 ++ ++ 39 00 04 ff 98 81 04 ++ 15 00 02 00 80 ++ 15 00 02 70 00 ++ 15 00 02 71 00 ++ 15 00 02 66 fe ++ 15 00 02 82 15 ++ 15 00 02 84 15 ++ 15 00 02 85 15 ++ 15 00 02 3a 24 ++ 15 00 02 32 ac ++ 15 00 02 8c 80 ++ 15 00 02 3c f5 ++ 15 00 02 88 33 ++ ++ 39 00 04 ff 98 81 01 ++ 15 00 02 22 0a ++ 15 00 02 31 00 ++ 15 00 02 53 78 ++ 15 00 02 50 5b ++ 15 00 02 51 5b ++ 15 00 02 60 20 ++ 15 00 02 61 00 ++ 15 00 02 62 0d ++ 15 00 02 63 00 ++ ++ 15 00 02 a0 00 ++ 15 00 02 a1 10 ++ 15 00 02 a2 1c ++ 15 00 02 a3 13 ++ 15 00 02 a4 15 ++ 15 00 02 a5 26 ++ 15 00 02 a6 1a ++ 15 00 02 a7 1d ++ 15 00 02 a8 67 ++ 15 00 02 a9 1c ++ 15 00 02 aa 29 ++ 15 00 02 ab 5b ++ 15 00 02 ac 26 ++ 15 00 02 ad 28 ++ 15 00 02 ae 5c ++ 15 00 02 af 30 ++ 15 00 02 b0 31 ++ 15 00 02 b1 2e ++ 15 00 02 b2 32 ++ 15 00 02 b3 00 ++ ++ 15 00 02 c0 00 ++ 15 00 02 c1 10 ++ 15 00 02 c2 1c ++ 15 00 02 c3 13 ++ 15 00 02 c4 15 ++ 15 00 02 c5 26 ++ 15 00 02 c6 1a ++ 15 00 02 c7 1d ++ 15 00 02 c8 67 ++ 15 00 02 c9 1c ++ 15 00 02 ca 29 ++ 15 00 02 cb 5b ++ 15 00 02 cc 26 ++ 15 00 02 cd 28 ++ 15 00 02 ce 5c ++ 15 00 02 cf 30 ++ 15 00 02 d0 31 ++ 15 00 02 d1 2e ++ 15 00 02 d2 32 ++ 15 00 02 d3 00 ++ 39 00 04 ff 98 81 00 ++ 05 00 01 11 ++ 05 01 01 29 ++ ]; ++ ++ panel-exit-sequence = [ ++ 05 00 01 28 ++ 05 00 01 10 ++ ]; ++ ++ display-timings { ++ native-mode = <&timing1>; ++ ++ timing1: timing1 { ++ clock-frequency = <64000000>; ++ hactive = <720>; ++ vactive = <1280>; ++ hfront-porch = <40>; ++ hsync-len = <10>; ++ hback-porch = <40>; ++ vfront-porch = <22>; ++ vsync-len = <4>; ++ vback-porch = <11>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ }; ++ ++}; ++ ++&firmware_android { ++ compatible = "android,firmware"; ++ fstab { ++ compatible = "android,fstab"; ++ system { ++ compatible = "android,system"; ++ dev = "/dev/block/by-name/system"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait"; ++ }; ++ vendor { ++ compatible = "android,vendor"; ++ dev = "/dev/block/by-name/vendor"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait"; ++ }; ++ }; ++}; ++ ++&rk_isp { ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3326-linux.dtsi b/arch/arm64/boot/dts/rockchip/rk3326-linux.dtsi +new file mode 100755 +index 000000000000..a386cd291cf3 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3326-linux.dtsi +@@ -0,0 +1,120 @@ ++/* ++ * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This file is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This file is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++/ { ++ compatible = "rockchip,linux", "rockchip,rk3326"; ++ ++ chosen { ++ bootargs = "earlycon=uart8250,mmio32,0xff160000 console=ttyFIQ0 rw root=PARTUUID=614e0000-0000 rootwait"; ++ }; ++ ++ fiq-debugger { ++ compatible = "rockchip,fiq-debugger"; ++ rockchip,serial-id = <2>; ++ rockchip,wake-irq = <0>; ++ /* If enable uart uses irq instead of fiq */ ++ rockchip,irq-mode-enable = <0>; ++ rockchip,baudrate = <1500000>; /* Only 115200 and 1500000 */ ++ interrupts = ; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart2m0_xfer>; ++ status = "okay"; ++ }; ++ ++ reserved-memory { ++ #address-cells = <2>; ++ #size-cells = <2>; ++ ranges; ++ ++ drm_logo: drm-logo@00000000 { ++ compatible = "rockchip,drm-logo"; ++ reg = <0x0 0x0 0x0 0x0>; ++ }; ++ }; ++}; ++ ++&cpu0_opp_table { ++ rockchip,avs = <1>; ++}; ++ ++&display_subsystem { ++ status = "disabled"; ++ ports = <&vopb_out>, <&vopl_out>; ++ logo-memory-region = <&drm_logo>; ++ ++ route { ++ route_lvds: route-lvds { ++ status = "disabled"; ++ logo,uboot = "logo.bmp"; ++ logo,kernel = "logo_kernel.bmp"; ++ logo,mode = "center"; ++ charge_logo,mode = "center"; ++ connect = <&vopb_out_lvds>; ++ }; ++ ++ route_dsi: route-dsi { ++ status = "disabled"; ++ logo,uboot = "logo.bmp"; ++ logo,kernel = "logo_kernel.bmp"; ++ logo,mode = "center"; ++ charge_logo,mode = "center"; ++ connect = <&vopb_out_dsi>; ++ }; ++ ++ route_rgb: route-rgb { ++ status = "disabled"; ++ logo,uboot = "logo.bmp"; ++ logo,kernel = "logo_kernel.bmp"; ++ logo,mode = "center"; ++ charge_logo,mode = "center"; ++ connect = <&vopb_out_rgb>; ++ }; ++ }; ++}; ++ ++&rng { ++ status = "okay"; ++}; ++ ++&video_phy { ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3328-android.dtsi b/arch/arm64/boot/dts/rockchip/rk3328-android.dtsi +new file mode 100755 +index 000000000000..809b6501b78f +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3328-android.dtsi +@@ -0,0 +1,88 @@ ++/* ++ * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++ */ ++ ++/ { ++ chosen: chosen { ++ bootargs = "earlycon=uart8250,mmio32,0xff130000 kpti=0 coherent_pool=1m"; ++ }; ++ ++ fiq-debugger { ++ compatible = "rockchip,fiq-debugger"; ++ rockchip,serial-id = <2>; ++ rockchip,signal-irq = <159>; ++ rockchip,wake-irq = <0>; ++ /* If enable uart uses irq instead of fiq */ ++ rockchip,irq-mode-enable = <0>; ++ rockchip,baudrate = <1500000>; /* Only 115200 and 1500000 */ ++ interrupts = ; ++ status = "okay"; ++ }; ++ ++ firmware { ++ firmware_android: android {}; ++ }; ++ ++ reserved-memory { ++ #address-cells = <2>; ++ #size-cells = <2>; ++ ranges; ++ ++ drm_logo: drm-logo@00000000 { ++ compatible = "rockchip,drm-logo"; ++ reg = <0x0 0x0 0x0 0x0>; ++ }; ++ ++ ramoops: ramoops@110000 { ++ compatible = "ramoops"; ++ reg = <0x0 0x110000 0x0 0xf0000>; ++ record-size = <0x20000>; ++ console-size = <0x80000>; ++ ftrace-size = <0x00000>; ++ pmsg-size = <0x50000>; ++ }; ++ ++ secure_memory: secure-memory@20000000 { ++ compatible = "rockchip,secure-memory"; ++ reg = <0x0 0x20000000 0x0 0x0>; ++ }; ++ ++ /* global autoconfigured region for contiguous allocations */ ++ linux,cma { ++ compatible = "shared-dma-pool"; ++ reusable; ++ size = <0x0 0x2000000>; ++ linux,cma-default; ++ }; ++ }; ++}; ++ ++&display_subsystem { ++ logo-memory-region = <&drm_logo>; ++ status = "okay"; ++ secure-memory-region = <&secure_memory>; ++ route { ++ route_hdmi: route-hdmi { ++ status = "okay"; ++ logo,uboot = "logo.bmp"; ++ logo,kernel = "logo_kernel.bmp"; ++ logo,mode = "fullscreen"; ++ charge_logo,mode = "fullscreen"; ++ connect = <&vop_out_hdmi>; ++ }; ++ route_tve: route-tve { ++ status = "okay"; ++ logo,uboot = "logo.bmp"; ++ logo,kernel = "logo_kernel.bmp"; ++ logo,mode = "fullscreen"; ++ charge_logo,mode = "fullscreen"; ++ connect = <&vop_out_tve>; ++ }; ++ }; ++}; ++ ++&rng { ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3328-box-liantong-avb.dts b/arch/arm64/boot/dts/rockchip/rk3328-box-liantong-avb.dts +new file mode 100755 +index 000000000000..ee04d8988fbb +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3328-box-liantong-avb.dts +@@ -0,0 +1,13 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ */ ++ ++/dts-v1/; ++#include "rk3328-box-liantong.dtsi" ++ ++/ { ++ model = "Rockchip RK3328 box liantong avb"; ++ compatible = "rockchip,rk3328-box-liantong-avb", "rockchip,rk3328"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3328-box-liantong.dts b/arch/arm64/boot/dts/rockchip/rk3328-box-liantong.dts +new file mode 100755 +index 000000000000..dcff87208d39 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3328-box-liantong.dts +@@ -0,0 +1,34 @@ ++/* ++ * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++ */ ++ ++/dts-v1/; ++#include "rk3328-box-liantong.dtsi" ++ ++/ { ++ model = "Rockchip RK3328 box liantong"; ++ compatible = "rockchip,rk3328-box-liantong", "rockchip,rk3328"; ++}; ++ ++&firmware_android{ ++ compatible = "android,firmware"; ++ fstab { ++ compatible = "android,fstab"; ++ system { ++ compatible = "android,system"; ++ dev = "/dev/block/platform/ff520000.dwmmc/by-name/system"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,verify"; ++ }; ++ vendor { ++ compatible = "android,vendor"; ++ dev = "/dev/block/platform/ff520000.dwmmc/by-name/vendor"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,verify"; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3328-box-liantong.dtsi b/arch/arm64/boot/dts/rockchip/rk3328-box-liantong.dtsi +new file mode 100755 +index 000000000000..ad7ef6be49fa +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3328-box-liantong.dtsi +@@ -0,0 +1,673 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ */ ++ ++#include "rk3328.dtsi" ++#include "rk3328-android.dtsi" ++#include "rk3328-box-plus-dram-timing.dtsi" ++#include ++ ++/ { ++ gmac_clkin: external-gmac-clock { ++ compatible = "fixed-clock"; ++ clock-frequency = <125000000>; ++ clock-output-names = "gmac_clkin"; ++ #clock-cells = <0>; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio1 18 GPIO_ACTIVE_LOW>; ++ }; ++ ++ sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,name = "rockchip-rk3328"; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s1>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&codec>; ++ }; ++ }; ++ ++ hdmi-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,mclk-fs = <128>; ++ simple-audio-card,name = "rockchip-hdmi"; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s0>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&hdmi>; ++ }; ++ }; ++ ++ regulators { ++ compatible = "simple-bus"; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ vccio_1v8_reg: regulator@0 { ++ compatible = "regulator-fixed"; ++ regulator-name = "vccio_1v8"; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-always-on; ++ }; ++ ++ vccio_3v3_reg: regulator@1 { ++ compatible = "regulator-fixed"; ++ regulator-name = "vccio_3v3"; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-always-on; ++ }; ++ }; ++ ++ rtc-fake { ++ compatible = "rtc-fake"; ++ status = "okay"; ++ }; ++ ++ spdif-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,name = "rockchip-spdif"; ++ simple-audio-card,cpu { ++ sound-dai = <&spdif>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&spdif_out>; ++ }; ++ }; ++ ++ spdif_out: spdif-out { ++ compatible = "linux,spdif-dit"; ++ #sound-dai-cells = <0>; ++ }; ++ ++ vcc_host_vbus: host-vbus-regulator { ++ compatible = "regulator-fixed"; ++ gpio = <&gpio0 RK_PA0 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&host_vbus_drv>; ++ regulator-name = "vcc_host_vbus"; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ enable-active-high; ++ }; ++ ++ vcc_otg_vbus: otg-vbus-regulator { ++ compatible = "regulator-fixed"; ++ gpio = <&gpio0 RK_PA2 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&otg_vbus_drv>; ++ regulator-name = "vcc_otg_vbus"; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ enable-active-high; ++ }; ++ ++ vcc_phy: vcc-phy-regulator { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_phy"; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ vcc_sd: sdmmc-regulator { ++ compatible = "regulator-fixed"; ++ gpio = <&gpio0 30 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc0m1_gpio>; ++ regulator-name = "vcc_sd"; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ vin-supply = <&vccio_3v3_reg>; ++ }; ++ ++ vdd_arm: vdd-center { ++ compatible = "pwm-regulator"; ++ rockchip,pwm_id = <0>; ++ rockchip,pwm_voltage = <1250000>; ++ pwms = <&pwm0 0 5000 1>; ++ regulator-name = "vcc_arm"; ++ regulator-min-microvolt = <950000>; ++ regulator-max-microvolt = <1400000>; ++ regulator-settling-time-up-us = <250>; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ vdd_logic: vdd-log { ++ compatible = "pwm-regulator"; ++ rockchip,pwm_id = <1>; ++ rockchip,pwm_voltage = <1100000>; ++ pwms = <&pwm1 0 5000 1>; ++ regulator-name = "vcc_log"; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <1300000>; ++ regulator-settling-time-up-us = <250>; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ uart_rts_gpios = <&gpio1 10 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart0_rts>; ++ pinctrl-1 = <&uart0_gpios>; ++ BT,power_gpio = <&gpio1 24 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio1 26 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ rockchip,grf = <&grf>; ++ wifi_chip_type = "rtl8822bs"; ++ sdio_vref = <1800>; ++ WIFI,host_wake_irq = <&gpio1 19 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++}; ++ ++&avsd { ++ status = "okay"; ++}; ++ ++&codec { ++ #sound-dai-cells = <0>; ++ status = "okay"; ++}; ++ ++&cpu0 { ++ cpu-supply = <&vdd_arm>; ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&dmc { ++ center-supply = <&vdd_logic>; ++ status = "okay"; ++ system-status-freq = < ++ /*system status freq(KHz)*/ ++ SYS_STATUS_NORMAL 786000 ++ SYS_STATUS_REBOOT 786000 ++ SYS_STATUS_SUSPEND 786000 ++ SYS_STATUS_VIDEO_1080P 786000 ++ SYS_STATUS_VIDEO_4K 786000 ++ SYS_STATUS_VIDEO_4K_10B 786000 ++ SYS_STATUS_PERFORMANCE 786000 ++ SYS_STATUS_BOOST 786000 ++ >; ++}; ++ ++&dmc_opp_table { ++ opp-800000000 { ++ status = "disabled"; ++ }; ++ ++ opp-850000000 { ++ status = "disabled"; ++ }; ++ ++ opp-933000000 { ++ status = "disabled"; ++ }; ++ ++ opp-1066000000 { ++ status = "disabled"; ++ }; ++}; ++ ++&emmc { ++ bus-width = <8>; ++ cap-mmc-highspeed; ++ no-sdio; ++ no-sd; ++ disable-wp; ++ non-removable; ++ num-slots = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&emmc_clk &emmc_cmd &emmc_bus8>; ++ status = "okay"; ++}; ++ ++&gmac2io { ++ phy-supply = <&vcc_phy>; ++ phy-mode = "rgmii"; ++ clock_in_out = "input"; ++ snps,reset-gpio = <&gpio1 RK_PC2 GPIO_ACTIVE_LOW>; ++ snps,reset-active-low; ++ snps,reset-delays-us = <0 10000 50000>; ++ assigned-clocks = <&cru SCLK_MAC2IO>, <&cru SCLK_MAC2IO_EXT>; ++ assigned-clock-parents = <&gmac_clkin>, <&gmac_clkin>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&rgmiim1_pins>; ++ tx_delay = <0x26>; ++ rx_delay = <0x11>; ++ status = "disabled"; ++}; ++ ++&gmac2phy { ++ phy-supply = <&vcc_phy>; ++ clock_in_out = "output"; ++ assigned-clocks = <&cru SCLK_MAC2PHY_SRC>; ++ assigned-clock-rate = <50000000>; ++ assigned-clocks = <&cru SCLK_MAC2PHY>; ++ assigned-clock-parents = <&cru SCLK_MAC2PHY_SRC>; ++ status = "okay"; ++}; ++ ++&gpu { ++ status = "okay"; ++ mali-supply = <&vdd_logic>; ++}; ++ ++&hdmi { ++ #sound-dai-cells = <0>; ++ ddc-i2c-scl-high-time-ns = <9625>; ++ ddc-i2c-scl-low-time-ns = <10000>; ++ status = "okay"; ++}; ++ ++&hdmiphy { ++ rockchip,phy-table = ++ <165000000 0x07 0x0a 0x0a 0x0a 0x00 0x00 0x08 ++ 0x08 0x08 0x00 0xac 0xcc 0xcc 0xcc>, ++ <340000000 0x0b 0x0d 0x0d 0x0d 0x07 0x15 0x08 ++ 0x08 0x08 0x3f 0xac 0xcc 0xcd 0xdd>, ++ <594000000 0x10 0x1a 0x1a 0x1a 0x07 0x15 0x08 ++ 0x08 0x08 0x00 0xac 0xcc 0xcc 0xcc>; ++ status = "okay"; ++}; ++ ++&secure_memory { ++ /* ++ * enable like this: ++ * reg = <0x0 0x20000000 0x0 0x10000000>; ++ */ ++ reg = <0x0 0x20000000 0x0 0x0>; ++}; ++ ++&i2s0 { ++ #sound-dai-cells = <0>; ++ rockchip,bclk-fs = <128>; ++ status = "okay"; ++}; ++ ++&i2s1 { ++ #sound-dai-cells = <0>; ++ status = "okay"; ++}; ++ ++&iep { ++ status = "okay"; ++}; ++ ++&iep_mmu { ++ status = "okay"; ++}; ++ ++&io_domains { ++ status = "okay"; ++ ++ vccio1-supply = <&vccio_3v3_reg>; ++ vccio2-supply = <&vccio_1v8_reg>; ++ vccio3-supply = <&vccio_3v3_reg>; ++ vccio4-supply = <&vccio_1v8_reg>; ++ vccio5-supply = <&vccio_3v3_reg>; ++ vccio6-supply = <&vccio_3v3_reg>; ++ pmuio-supply = <&vccio_3v3_reg>; ++}; ++ ++&mpp_srv { ++ status = "okay"; ++}; ++ ++&pinctrl { ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = ++ <1 RK_PC2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ usb { ++ host_vbus_drv: host-vbus-drv { ++ rockchip,pins = ++ <0 RK_PA0 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ otg_vbus_drv: otg-vbus-drv { ++ rockchip,pins = ++ <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart0_gpios: uart0-gpios { ++ rockchip,pins = ++ <1 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++&pwm0 { ++ status = "okay"; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm0_pin_pull_up>; ++}; ++ ++&pwm1 { ++ status = "okay"; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm1_pin_pull_up>; ++}; ++ ++&pwm3 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pwmir_pin>; ++ compatible = "rockchip,remotectl-pwm"; ++ remote_pwm_id = <3>; ++ handle_cpu_id = <1>; ++ remote_support_psci = <1>; ++ ++ ir_key1 { ++ rockchip,usercode = <0x4040>; ++ rockchip,key_table = ++ <0xf2 KEY_REPLY>, ++ <0xba KEY_BACK>, ++ <0xf4 KEY_UP>, ++ <0xf1 KEY_DOWN>, ++ <0xef KEY_LEFT>, ++ <0xee KEY_RIGHT>, ++ <0xbd KEY_HOME>, ++ <0xea KEY_VOLUMEUP>, ++ <0xe3 KEY_VOLUMEDOWN>, ++ <0xe2 KEY_SEARCH>, ++ <0xb2 KEY_POWER>, ++ <0xbc KEY_MUTE>, ++ <0xec KEY_MENU>, ++ <0xbf 0x190>, ++ <0xe0 0x191>, ++ <0xe1 0x192>, ++ <0xe9 183>, ++ <0xe6 248>, ++ <0xe8 185>, ++ <0xe7 186>, ++ <0xf0 388>, ++ <0xbe 0x175>; ++ }; ++ ++ ir_key2 { ++ rockchip,usercode = <0xff00>; ++ rockchip,key_table = ++ <0xf9 KEY_HOME>, ++ <0xbf KEY_BACK>, ++ <0xfb KEY_MENU>, ++ <0xaa KEY_REPLY>, ++ <0xb9 KEY_UP>, ++ <0xe9 KEY_DOWN>, ++ <0xb8 KEY_LEFT>, ++ <0xea KEY_RIGHT>, ++ <0xeb KEY_VOLUMEDOWN>, ++ <0xef KEY_VOLUMEUP>, ++ <0xf7 KEY_MUTE>, ++ <0xe7 KEY_POWER>, ++ <0xfc KEY_POWER>, ++ <0xa9 KEY_VOLUMEDOWN>, ++ <0xa8 KEY_PLAYPAUSE>, ++ <0xe0 KEY_VOLUMEDOWN>, ++ <0xa5 KEY_VOLUMEDOWN>, ++ <0xab 183>, ++ <0xb7 388>, ++ <0xe8 388>, ++ <0xf8 184>, ++ <0xaf 185>, ++ <0xed KEY_VOLUMEDOWN>, ++ <0xee 186>, ++ <0xb3 KEY_VOLUMEDOWN>, ++ <0xf1 KEY_VOLUMEDOWN>, ++ <0xf2 KEY_VOLUMEDOWN>, ++ <0xf3 KEY_SEARCH>, ++ <0xb4 KEY_VOLUMEDOWN>, ++ <0xa4 KEY_SETUP>, ++ <0xbe KEY_SEARCH>; ++ }; ++ ++ ir_key3 { ++ rockchip,usercode = <0x1dcc>; ++ rockchip,key_table = ++ <0xee KEY_REPLY>, ++ <0xf0 KEY_BACK>, ++ <0xf8 KEY_UP>, ++ <0xbb KEY_DOWN>, ++ <0xef KEY_LEFT>, ++ <0xed KEY_RIGHT>, ++ <0xfc KEY_HOME>, ++ <0xf1 KEY_VOLUMEUP>, ++ <0xfd KEY_VOLUMEDOWN>, ++ <0xb7 KEY_SEARCH>, ++ <0xff KEY_POWER>, ++ <0xf3 KEY_MUTE>, ++ <0xbf KEY_MENU>, ++ <0xf9 0x191>, ++ <0xf5 0x192>, ++ <0xb3 388>, ++ <0xbe KEY_1>, ++ <0xba KEY_2>, ++ <0xb2 KEY_3>, ++ <0xbd KEY_4>, ++ <0xf9 KEY_5>, ++ <0xb1 KEY_6>, ++ <0xfc KEY_7>, ++ <0xf8 KEY_8>, ++ <0xb0 KEY_9>, ++ <0xb6 KEY_0>, ++ <0xb5 KEY_BACKSPACE>; ++ }; ++}; ++ ++&rga { ++ status = "okay"; ++}; ++ ++&rkvdec { ++ status = "okay"; ++ vcodec-supply = <&vdd_logic>; ++}; ++ ++&rkvdec_mmu { ++ status = "okay"; ++}; ++ ++&rockchip_suspend { ++ status = "okay"; ++ rockchip,virtual-poweroff = <1>; ++ rockchip,sleep-mode-config = < ++ (0 ++ |RKPM_SLP_CTR_VOL_PWM0 ++ |RKPM_SLP_CTR_VOL_PWM1 ++ ) ++ >; ++}; ++ ++&uart0 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart0_xfer &uart0_cts>; ++ status = "okay"; ++}; ++ ++&sdio { ++ bus-width = <4>; ++ cap-sd-highspeed; ++ cap-sdio-irq; ++ disable-wp; ++ keep-power-in-suspend; ++ max-frequency = <125000000>; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ non-removable; ++ num-slots = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc1_bus4 &sdmmc1_cmd &sdmmc1_clk>; ++ no-sd; ++ no-mmc; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&sdmmc_ext { ++ bus-width = <4>; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ disable-wp; ++ keep-power-in-suspend; ++ max-frequency = <150000000>; ++ num-slots = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc0ext_clk &sdmmc0ext_cmd &sdmmc0ext_dectn &sdmmc0ext_bus4>; ++ no-sdio; ++ no-mmc; ++ status = "okay"; ++}; ++ ++&sdmmc { ++ bus-width = <4>; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ disable-wp; ++ max-frequency = <150000000>; ++ num-slots = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc0_clk &sdmmc0_cmd &sdmmc0_dectn &sdmmc0_bus4>; ++ no-sdio; ++ no-mmc; ++ status = "okay"; ++ vmmc-supply = <&vcc_sd>; ++}; ++ ++&spdif { ++ #sound-dai-cells = <0>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&spdifm0_tx>; ++ status = "okay"; ++}; ++ ++&threshold { ++ temperature = <90000>; /* millicelsius */ ++}; ++ ++&target { ++ temperature = <105000>; /* millicelsius */ ++}; ++ ++&soc_crit { ++ temperature = <115000>; /* millicelsius */ ++}; ++ ++&tsadc { ++ rockchip,hw-tshut-temp = <120000>; ++ status = "okay"; ++}; ++ ++&tve { ++ status = "okay"; ++}; ++ ++&u2phy { ++ status = "okay"; ++ ++ u2phy_host: host-port { ++ status = "okay"; ++ }; ++ ++ u2phy_otg: otg-port { ++ vbus-supply = <&vcc_otg_vbus>; ++ status = "okay"; ++ }; ++}; ++ ++&u3phy { ++ vbus-supply = <&vcc_host_vbus>; ++ status = "okay"; ++}; ++ ++&u3phy_utmi { ++ status = "okay"; ++}; ++ ++&u3phy_pipe { ++ status = "okay"; ++}; ++ ++&usb20_otg { ++ status = "okay"; ++}; ++ ++&usb_host0_ehci { ++ status = "okay"; ++}; ++ ++&usb_host0_ohci { ++ status = "okay"; ++}; ++ ++&usbdrd3 { ++ status = "okay"; ++}; ++ ++&usbdrd_dwc3 { ++ status = "okay"; ++}; ++ ++&vdpu { ++ status = "okay"; ++}; ++ ++&vpu_mmu { ++ status = "okay"; ++}; ++ ++&vepu { ++ status = "okay"; ++}; ++ ++&vepu_mmu { ++ status = "okay"; ++}; ++ ++&vepu22 { ++ status = "okay"; ++}; ++ ++&vepu22_mmu { ++ status = "okay"; ++}; ++ ++&vop { ++ status = "okay"; ++}; ++ ++&vop_mmu { ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3328-box-plus-dram-timing.dtsi b/arch/arm64/boot/dts/rockchip/rk3328-box-plus-dram-timing.dtsi +new file mode 100755 +index 000000000000..0ea270539a23 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3328-box-plus-dram-timing.dtsi +@@ -0,0 +1,221 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ */ ++#include ++#include ++ ++&ddr_timing { ++ /* CA de-skew, one step is 47.8ps, range 0-15 */ ++ ddr3a1_ddr4a9_de-skew = <0>; ++ ddr3a0_ddr4a10_de-skew = <0>; ++ ddr3a3_ddr4a6_de-skew = <1>; ++ ddr3a2_ddr4a4_de-skew = <1>; ++ ddr3a5_ddr4a8_de-skew = <0>; ++ ddr3a4_ddr4a5_de-skew = <2>; ++ ddr3a7_ddr4a11_de-skew = <0>; ++ ddr3a6_ddr4a7_de-skew = <2>; ++ ddr3a9_ddr4a0_de-skew = <1>; ++ ddr3a8_ddr4a13_de-skew = <0>; ++ ddr3a11_ddr4a3_de-skew = <2>; ++ ddr3a10_ddr4cs0_de-skew = <0>; ++ ddr3a13_ddr4a2_de-skew = <1>; ++ ddr3a12_ddr4ba1_de-skew = <0>; ++ ddr3a15_ddr4odt0_de-skew = <0>; ++ ddr3a14_ddr4a1_de-skew = <1>; ++ ddr3ba1_ddr4a15_de-skew = <0>; ++ ddr3ba0_ddr4bg0_de-skew = <0>; ++ ddr3ras_ddr4cke_de-skew = <0>; ++ ddr3ba2_ddr4ba0_de-skew = <1>; ++ ddr3we_ddr4bg1_de-skew = <1>; ++ ddr3cas_ddr4a12_de-skew = <0>; ++ ddr3ckn_ddr4ckn_de-skew = <5>; ++ ddr3ckp_ddr4ckp_de-skew = <5>; ++ ddr3cke_ddr4a16_de-skew = <1>; ++ ddr3odt0_ddr4a14_de-skew = <0>; ++ ddr3cs0_ddr4act_de-skew = <1>; ++ ddr3reset_ddr4reset_de-skew = <0>; ++ ddr3cs1_ddr4cs1_de-skew = <0>; ++ ddr3odt1_ddr4odt1_de-skew = <0>; ++ ++ /* DATA de-skew ++ * RX one step is 25.1ps, range 0-15 ++ * TX one step is 47.8ps, range 0-15 ++ */ ++ cs0_dm0_rx_de-skew = <7>; ++ cs0_dm0_tx_de-skew = <8>; ++ cs0_dq0_rx_de-skew = <7>; ++ cs0_dq0_tx_de-skew = <8>; ++ cs0_dq1_rx_de-skew = <7>; ++ cs0_dq1_tx_de-skew = <8>; ++ cs0_dq2_rx_de-skew = <7>; ++ cs0_dq2_tx_de-skew = <8>; ++ cs0_dq3_rx_de-skew = <7>; ++ cs0_dq3_tx_de-skew = <8>; ++ cs0_dq4_rx_de-skew = <7>; ++ cs0_dq4_tx_de-skew = <8>; ++ cs0_dq5_rx_de-skew = <7>; ++ cs0_dq5_tx_de-skew = <8>; ++ cs0_dq6_rx_de-skew = <7>; ++ cs0_dq6_tx_de-skew = <8>; ++ cs0_dq7_rx_de-skew = <7>; ++ cs0_dq7_tx_de-skew = <8>; ++ cs0_dqs0_rx_de-skew = <6>; ++ cs0_dqs0p_tx_de-skew = <9>; ++ cs0_dqs0n_tx_de-skew = <9>; ++ ++ cs0_dm1_rx_de-skew = <7>; ++ cs0_dm1_tx_de-skew = <7>; ++ cs0_dq8_rx_de-skew = <7>; ++ cs0_dq8_tx_de-skew = <8>; ++ cs0_dq9_rx_de-skew = <7>; ++ cs0_dq9_tx_de-skew = <7>; ++ cs0_dq10_rx_de-skew = <7>; ++ cs0_dq10_tx_de-skew = <8>; ++ cs0_dq11_rx_de-skew = <7>; ++ cs0_dq11_tx_de-skew = <7>; ++ cs0_dq12_rx_de-skew = <7>; ++ cs0_dq12_tx_de-skew = <8>; ++ cs0_dq13_rx_de-skew = <7>; ++ cs0_dq13_tx_de-skew = <7>; ++ cs0_dq14_rx_de-skew = <7>; ++ cs0_dq14_tx_de-skew = <8>; ++ cs0_dq15_rx_de-skew = <7>; ++ cs0_dq15_tx_de-skew = <7>; ++ cs0_dqs1_rx_de-skew = <7>; ++ cs0_dqs1p_tx_de-skew = <9>; ++ cs0_dqs1n_tx_de-skew = <9>; ++ ++ cs0_dm2_rx_de-skew = <7>; ++ cs0_dm2_tx_de-skew = <8>; ++ cs0_dq16_rx_de-skew = <7>; ++ cs0_dq16_tx_de-skew = <8>; ++ cs0_dq17_rx_de-skew = <7>; ++ cs0_dq17_tx_de-skew = <8>; ++ cs0_dq18_rx_de-skew = <7>; ++ cs0_dq18_tx_de-skew = <8>; ++ cs0_dq19_rx_de-skew = <7>; ++ cs0_dq19_tx_de-skew = <8>; ++ cs0_dq20_rx_de-skew = <7>; ++ cs0_dq20_tx_de-skew = <8>; ++ cs0_dq21_rx_de-skew = <7>; ++ cs0_dq21_tx_de-skew = <8>; ++ cs0_dq22_rx_de-skew = <7>; ++ cs0_dq22_tx_de-skew = <8>; ++ cs0_dq23_rx_de-skew = <7>; ++ cs0_dq23_tx_de-skew = <8>; ++ cs0_dqs2_rx_de-skew = <6>; ++ cs0_dqs2p_tx_de-skew = <9>; ++ cs0_dqs2n_tx_de-skew = <9>; ++ ++ cs0_dm3_rx_de-skew = <7>; ++ cs0_dm3_tx_de-skew = <7>; ++ cs0_dq24_rx_de-skew = <7>; ++ cs0_dq24_tx_de-skew = <8>; ++ cs0_dq25_rx_de-skew = <7>; ++ cs0_dq25_tx_de-skew = <7>; ++ cs0_dq26_rx_de-skew = <7>; ++ cs0_dq26_tx_de-skew = <7>; ++ cs0_dq27_rx_de-skew = <7>; ++ cs0_dq27_tx_de-skew = <7>; ++ cs0_dq28_rx_de-skew = <7>; ++ cs0_dq28_tx_de-skew = <7>; ++ cs0_dq29_rx_de-skew = <7>; ++ cs0_dq29_tx_de-skew = <7>; ++ cs0_dq30_rx_de-skew = <7>; ++ cs0_dq30_tx_de-skew = <7>; ++ cs0_dq31_rx_de-skew = <7>; ++ cs0_dq31_tx_de-skew = <7>; ++ cs0_dqs3_rx_de-skew = <7>; ++ cs0_dqs3p_tx_de-skew = <9>; ++ cs0_dqs3n_tx_de-skew = <9>; ++ ++ cs1_dm0_rx_de-skew = <7>; ++ cs1_dm0_tx_de-skew = <8>; ++ cs1_dq0_rx_de-skew = <7>; ++ cs1_dq0_tx_de-skew = <8>; ++ cs1_dq1_rx_de-skew = <7>; ++ cs1_dq1_tx_de-skew = <8>; ++ cs1_dq2_rx_de-skew = <7>; ++ cs1_dq2_tx_de-skew = <8>; ++ cs1_dq3_rx_de-skew = <7>; ++ cs1_dq3_tx_de-skew = <8>; ++ cs1_dq4_rx_de-skew = <7>; ++ cs1_dq4_tx_de-skew = <8>; ++ cs1_dq5_rx_de-skew = <7>; ++ cs1_dq5_tx_de-skew = <8>; ++ cs1_dq6_rx_de-skew = <7>; ++ cs1_dq6_tx_de-skew = <8>; ++ cs1_dq7_rx_de-skew = <7>; ++ cs1_dq7_tx_de-skew = <8>; ++ cs1_dqs0_rx_de-skew = <6>; ++ cs1_dqs0p_tx_de-skew = <9>; ++ cs1_dqs0n_tx_de-skew = <9>; ++ ++ cs1_dm1_rx_de-skew = <7>; ++ cs1_dm1_tx_de-skew = <7>; ++ cs1_dq8_rx_de-skew = <7>; ++ cs1_dq8_tx_de-skew = <8>; ++ cs1_dq9_rx_de-skew = <7>; ++ cs1_dq9_tx_de-skew = <7>; ++ cs1_dq10_rx_de-skew = <7>; ++ cs1_dq10_tx_de-skew = <8>; ++ cs1_dq11_rx_de-skew = <7>; ++ cs1_dq11_tx_de-skew = <7>; ++ cs1_dq12_rx_de-skew = <7>; ++ cs1_dq12_tx_de-skew = <8>; ++ cs1_dq13_rx_de-skew = <7>; ++ cs1_dq13_tx_de-skew = <7>; ++ cs1_dq14_rx_de-skew = <7>; ++ cs1_dq14_tx_de-skew = <8>; ++ cs1_dq15_rx_de-skew = <7>; ++ cs1_dq15_tx_de-skew = <7>; ++ cs1_dqs1_rx_de-skew = <7>; ++ cs1_dqs1p_tx_de-skew = <9>; ++ cs1_dqs1n_tx_de-skew = <9>; ++ ++ cs1_dm2_rx_de-skew = <7>; ++ cs1_dm2_tx_de-skew = <8>; ++ cs1_dq16_rx_de-skew = <7>; ++ cs1_dq16_tx_de-skew = <8>; ++ cs1_dq17_rx_de-skew = <7>; ++ cs1_dq17_tx_de-skew = <8>; ++ cs1_dq18_rx_de-skew = <7>; ++ cs1_dq18_tx_de-skew = <8>; ++ cs1_dq19_rx_de-skew = <7>; ++ cs1_dq19_tx_de-skew = <8>; ++ cs1_dq20_rx_de-skew = <7>; ++ cs1_dq20_tx_de-skew = <8>; ++ cs1_dq21_rx_de-skew = <7>; ++ cs1_dq21_tx_de-skew = <8>; ++ cs1_dq22_rx_de-skew = <7>; ++ cs1_dq22_tx_de-skew = <8>; ++ cs1_dq23_rx_de-skew = <7>; ++ cs1_dq23_tx_de-skew = <8>; ++ cs1_dqs2_rx_de-skew = <6>; ++ cs1_dqs2p_tx_de-skew = <9>; ++ cs1_dqs2n_tx_de-skew = <9>; ++ ++ cs1_dm3_rx_de-skew = <7>; ++ cs1_dm3_tx_de-skew = <7>; ++ cs1_dq24_rx_de-skew = <7>; ++ cs1_dq24_tx_de-skew = <8>; ++ cs1_dq25_rx_de-skew = <7>; ++ cs1_dq25_tx_de-skew = <7>; ++ cs1_dq26_rx_de-skew = <7>; ++ cs1_dq26_tx_de-skew = <7>; ++ cs1_dq27_rx_de-skew = <7>; ++ cs1_dq27_tx_de-skew = <7>; ++ cs1_dq28_rx_de-skew = <7>; ++ cs1_dq28_tx_de-skew = <7>; ++ cs1_dq29_rx_de-skew = <7>; ++ cs1_dq29_tx_de-skew = <7>; ++ cs1_dq30_rx_de-skew = <7>; ++ cs1_dq30_tx_de-skew = <7>; ++ cs1_dq31_rx_de-skew = <7>; ++ cs1_dq31_tx_de-skew = <7>; ++ cs1_dqs3_rx_de-skew = <7>; ++ cs1_dqs3p_tx_de-skew = <9>; ++ cs1_dqs3n_tx_de-skew = <9>; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3328-dram-2layer-timing.dtsi b/arch/arm64/boot/dts/rockchip/rk3328-dram-2layer-timing.dtsi +new file mode 100755 +index 000000000000..940024920b5d +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3328-dram-2layer-timing.dtsi +@@ -0,0 +1,257 @@ ++/* ++ * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This library is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This library is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++#include ++#include ++ ++&ddr_timing { ++ /* CA de-skew, one step is 47.8ps, range 0-15 */ ++ ddr3a1_ddr4a9_de-skew = <2>; ++ ddr3a0_ddr4a10_de-skew = <4>; ++ ddr3a3_ddr4a6_de-skew = <6>; ++ ddr3a2_ddr4a4_de-skew = <5>; ++ ddr3a5_ddr4a8_de-skew = <7>; ++ ddr3a4_ddr4a5_de-skew = <7>; ++ ddr3a7_ddr4a11_de-skew = <7>; ++ ddr3a6_ddr4a7_de-skew = <7>; ++ ddr3a9_ddr4a0_de-skew = <7>; ++ ddr3a8_ddr4a13_de-skew = <4>; ++ ddr3a11_ddr4a3_de-skew = <4>; ++ ddr3a10_ddr4cs0_de-skew = <4>; ++ ddr3a13_ddr4a2_de-skew = <7>; ++ ddr3a12_ddr4ba1_de-skew = <5>; ++ ddr3a15_ddr4odt0_de-skew = <7>; ++ ddr3a14_ddr4a1_de-skew = <6>; ++ ddr3ba1_ddr4a15_de-skew = <3>; ++ ddr3ba0_ddr4bg0_de-skew = <9>; ++ ddr3ras_ddr4cke_de-skew = <6>; ++ ddr3ba2_ddr4ba0_de-skew = <8>; ++ ddr3we_ddr4bg1_de-skew = <4>; ++ ddr3cas_ddr4a12_de-skew = <4>; ++ ddr3ckn_ddr4ckn_de-skew = <14>; ++ ddr3ckp_ddr4ckp_de-skew = <14>; ++ ddr3cke_ddr4a16_de-skew = <5>; ++ ddr3odt0_ddr4a14_de-skew = <9>; ++ ddr3cs0_ddr4act_de-skew = <9>; ++ ddr3reset_ddr4reset_de-skew = <10>; ++ ddr3cs1_ddr4cs1_de-skew = <7>; ++ ddr3odt1_ddr4odt1_de-skew = <7>; ++ ++ /* DATA de-skew ++ * RX one step is 25.1ps, range 0-15 ++ * TX one step is 47.8ps, range 0-15 ++ */ ++ cs0_dm0_rx_de-skew = <15>; ++ cs0_dm0_tx_de-skew = <14>; ++ cs0_dq0_rx_de-skew = <13>; ++ cs0_dq0_tx_de-skew = <13>; ++ cs0_dq1_rx_de-skew = <14>; ++ cs0_dq1_tx_de-skew = <14>; ++ cs0_dq2_rx_de-skew = <13>; ++ cs0_dq2_tx_de-skew = <13>; ++ cs0_dq3_rx_de-skew = <15>; ++ cs0_dq3_tx_de-skew = <14>; ++ cs0_dq4_rx_de-skew = <15>; ++ cs0_dq4_tx_de-skew = <14>; ++ cs0_dq5_rx_de-skew = <15>; ++ cs0_dq5_tx_de-skew = <14>; ++ cs0_dq6_rx_de-skew = <15>; ++ cs0_dq6_tx_de-skew = <14>; ++ cs0_dq7_rx_de-skew = <15>; ++ cs0_dq7_tx_de-skew = <14>; ++ cs0_dqs0_rx_de-skew = <13>; ++ cs0_dqs0p_tx_de-skew = <15>; ++ cs0_dqs0n_tx_de-skew = <15>; ++ ++ cs0_dm1_rx_de-skew = <11>; ++ cs0_dm1_tx_de-skew = <11>; ++ cs0_dq8_rx_de-skew = <12>; ++ cs0_dq8_tx_de-skew = <13>; ++ cs0_dq9_rx_de-skew = <13>; ++ cs0_dq9_tx_de-skew = <12>; ++ cs0_dq10_rx_de-skew = <12>; ++ cs0_dq10_tx_de-skew = <13>; ++ cs0_dq11_rx_de-skew = <15>; ++ cs0_dq11_tx_de-skew = <13>; ++ cs0_dq12_rx_de-skew = <9>; ++ cs0_dq12_tx_de-skew = <11>; ++ cs0_dq13_rx_de-skew = <12>; ++ cs0_dq13_tx_de-skew = <12>; ++ cs0_dq14_rx_de-skew = <9>; ++ cs0_dq14_tx_de-skew = <11>; ++ cs0_dq15_rx_de-skew = <13>; ++ cs0_dq15_tx_de-skew = <12>; ++ cs0_dqs1_rx_de-skew = <14>; ++ cs0_dqs1p_tx_de-skew = <14>; ++ cs0_dqs1n_tx_de-skew = <14>; ++ ++ cs0_dm2_rx_de-skew = <10>; ++ cs0_dm2_tx_de-skew = <12>; ++ cs0_dq16_rx_de-skew = <11>; ++ cs0_dq16_tx_de-skew = <12>; ++ cs0_dq17_rx_de-skew = <11>; ++ cs0_dq17_tx_de-skew = <12>; ++ cs0_dq18_rx_de-skew = <11>; ++ cs0_dq18_tx_de-skew = <12>; ++ cs0_dq19_rx_de-skew = <13>; ++ cs0_dq19_tx_de-skew = <13>; ++ cs0_dq20_rx_de-skew = <12>; ++ cs0_dq20_tx_de-skew = <13>; ++ cs0_dq21_rx_de-skew = <11>; ++ cs0_dq21_tx_de-skew = <12>; ++ cs0_dq22_rx_de-skew = <12>; ++ cs0_dq22_tx_de-skew = <12>; ++ cs0_dq23_rx_de-skew = <12>; ++ cs0_dq23_tx_de-skew = <12>; ++ cs0_dqs2_rx_de-skew = <11>; ++ cs0_dqs2p_tx_de-skew = <14>; ++ cs0_dqs2n_tx_de-skew = <14>; ++ ++ cs0_dm3_rx_de-skew = <10>; ++ cs0_dm3_tx_de-skew = <11>; ++ cs0_dq24_rx_de-skew = <9>; ++ cs0_dq24_tx_de-skew = <12>; ++ cs0_dq25_rx_de-skew = <10>; ++ cs0_dq25_tx_de-skew = <12>; ++ cs0_dq26_rx_de-skew = <13>; ++ cs0_dq26_tx_de-skew = <13>; ++ cs0_dq27_rx_de-skew = <14>; ++ cs0_dq27_tx_de-skew = <13>; ++ cs0_dq28_rx_de-skew = <8>; ++ cs0_dq28_tx_de-skew = <10>; ++ cs0_dq29_rx_de-skew = <10>; ++ cs0_dq29_tx_de-skew = <12>; ++ cs0_dq30_rx_de-skew = <14>; ++ cs0_dq30_tx_de-skew = <13>; ++ cs0_dq31_rx_de-skew = <15>; ++ cs0_dq31_tx_de-skew = <14>; ++ cs0_dqs3_rx_de-skew = <12>; ++ cs0_dqs3p_tx_de-skew = <15>; ++ cs0_dqs3n_tx_de-skew = <15>; ++ ++ cs1_dm0_rx_de-skew = <11>; ++ cs1_dm0_tx_de-skew = <10>; ++ cs1_dq0_rx_de-skew = <9>; ++ cs1_dq0_tx_de-skew = <9>; ++ cs1_dq1_rx_de-skew = <10>; ++ cs1_dq1_tx_de-skew = <10>; ++ cs1_dq2_rx_de-skew = <9>; ++ cs1_dq2_tx_de-skew = <9>; ++ cs1_dq3_rx_de-skew = <11>; ++ cs1_dq3_tx_de-skew = <10>; ++ cs1_dq4_rx_de-skew = <11>; ++ cs1_dq4_tx_de-skew = <10>; ++ cs1_dq5_rx_de-skew = <11>; ++ cs1_dq5_tx_de-skew = <10>; ++ cs1_dq6_rx_de-skew = <11>; ++ cs1_dq6_tx_de-skew = <10>; ++ cs1_dq7_rx_de-skew = <11>; ++ cs1_dq7_tx_de-skew = <10>; ++ cs1_dqs0_rx_de-skew = <9>; ++ cs1_dqs0p_tx_de-skew = <10>; ++ cs1_dqs0n_tx_de-skew = <10>; ++ ++ cs1_dm1_rx_de-skew = <7>; ++ cs1_dm1_tx_de-skew = <7>; ++ cs1_dq8_rx_de-skew = <8>; ++ cs1_dq8_tx_de-skew = <9>; ++ cs1_dq9_rx_de-skew = <9>; ++ cs1_dq9_tx_de-skew = <8>; ++ cs1_dq10_rx_de-skew = <8>; ++ cs1_dq10_tx_de-skew = <9>; ++ cs1_dq11_rx_de-skew = <11>; ++ cs1_dq11_tx_de-skew = <9>; ++ cs1_dq12_rx_de-skew = <5>; ++ cs1_dq12_tx_de-skew = <7>; ++ cs1_dq13_rx_de-skew = <8>; ++ cs1_dq13_tx_de-skew = <8>; ++ cs1_dq14_rx_de-skew = <5>; ++ cs1_dq14_tx_de-skew = <7>; ++ cs1_dq15_rx_de-skew = <9>; ++ cs1_dq15_tx_de-skew = <8>; ++ cs1_dqs1_rx_de-skew = <9>; ++ cs1_dqs1p_tx_de-skew = <10>; ++ cs1_dqs1n_tx_de-skew = <10>; ++ ++ cs1_dm2_rx_de-skew = <6>; ++ cs1_dm2_tx_de-skew = <8>; ++ cs1_dq16_rx_de-skew = <7>; ++ cs1_dq16_tx_de-skew = <8>; ++ cs1_dq17_rx_de-skew = <7>; ++ cs1_dq17_tx_de-skew = <8>; ++ cs1_dq18_rx_de-skew = <7>; ++ cs1_dq18_tx_de-skew = <8>; ++ cs1_dq19_rx_de-skew = <9>; ++ cs1_dq19_tx_de-skew = <9>; ++ cs1_dq20_rx_de-skew = <8>; ++ cs1_dq20_tx_de-skew = <9>; ++ cs1_dq21_rx_de-skew = <7>; ++ cs1_dq21_tx_de-skew = <8>; ++ cs1_dq22_rx_de-skew = <8>; ++ cs1_dq22_tx_de-skew = <8>; ++ cs1_dq23_rx_de-skew = <8>; ++ cs1_dq23_tx_de-skew = <8>; ++ cs1_dqs2_rx_de-skew = <7>; ++ cs1_dqs2p_tx_de-skew = <9>; ++ cs1_dqs2n_tx_de-skew = <9>; ++ ++ cs1_dm3_rx_de-skew = <4>; ++ cs1_dm3_tx_de-skew = <5>; ++ cs1_dq24_rx_de-skew = <3>; ++ cs1_dq24_tx_de-skew = <6>; ++ cs1_dq25_rx_de-skew = <4>; ++ cs1_dq25_tx_de-skew = <6>; ++ cs1_dq26_rx_de-skew = <7>; ++ cs1_dq26_tx_de-skew = <7>; ++ cs1_dq27_rx_de-skew = <8>; ++ cs1_dq27_tx_de-skew = <7>; ++ cs1_dq28_rx_de-skew = <2>; ++ cs1_dq28_tx_de-skew = <4>; ++ cs1_dq29_rx_de-skew = <4>; ++ cs1_dq29_tx_de-skew = <6>; ++ cs1_dq30_rx_de-skew = <8>; ++ cs1_dq30_tx_de-skew = <7>; ++ cs1_dq31_rx_de-skew = <9>; ++ cs1_dq31_tx_de-skew = <8>; ++ cs1_dqs3_rx_de-skew = <6>; ++ cs1_dqs3p_tx_de-skew = <8>; ++ cs1_dqs3n_tx_de-skew = <8>; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3328-dram-default-timing.dtsi b/arch/arm64/boot/dts/rockchip/rk3328-dram-default-timing.dtsi +new file mode 100755 +index 000000000000..a3f5ff4bdc47 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3328-dram-default-timing.dtsi +@@ -0,0 +1,311 @@ ++/* ++ * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This library is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This library is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++#include ++#include ++ ++/ { ++ ddr_timing: ddr_timing { ++ compatible = "rockchip,ddr-timing"; ++ ddr3_speed_bin = ; ++ ddr4_speed_bin = ; ++ pd_idle = <0>; ++ sr_idle = <0>; ++ sr_mc_gate_idle = <0>; ++ srpd_lite_idle = <0>; ++ standby_idle = <0>; ++ ++ auto_pd_dis_freq = <1066>; ++ auto_sr_dis_freq = <800>; ++ ddr3_dll_dis_freq = <300>; ++ ddr4_dll_dis_freq = <625>; ++ phy_dll_dis_freq = <400>; ++ ++ ddr3_odt_dis_freq = <100>; ++ phy_ddr3_odt_dis_freq = <100>; ++ ddr3_drv = ; ++ ddr3_odt = ; ++ phy_ddr3_ca_drv = ; ++ phy_ddr3_ck_drv = ; ++ phy_ddr3_dq_drv = ; ++ phy_ddr3_odt = ; ++ ++ lpddr3_odt_dis_freq = <666>; ++ phy_lpddr3_odt_dis_freq = <666>; ++ lpddr3_drv = ; ++ lpddr3_odt = ; ++ phy_lpddr3_ca_drv = ; ++ phy_lpddr3_ck_drv = ; ++ phy_lpddr3_dq_drv = ; ++ phy_lpddr3_odt = ; ++ ++ lpddr4_odt_dis_freq = <800>; ++ phy_lpddr4_odt_dis_freq = <800>; ++ lpddr4_drv = ; ++ lpddr4_dq_odt = ; ++ lpddr4_ca_odt = ; ++ phy_lpddr4_ca_drv = ; ++ phy_lpddr4_ck_cs_drv = ; ++ phy_lpddr4_dq_drv = ; ++ phy_lpddr4_odt = ; ++ ++ ddr4_odt_dis_freq = <666>; ++ phy_ddr4_odt_dis_freq = <666>; ++ ddr4_drv = ; ++ ddr4_odt = ; ++ phy_ddr4_ca_drv = ; ++ phy_ddr4_ck_drv = ; ++ phy_ddr4_dq_drv = ; ++ phy_ddr4_odt = ; ++ ++ /* CA de-skew, one step is 47.8ps, range 0-15 */ ++ ddr3a1_ddr4a9_de-skew = <7>; ++ ddr3a0_ddr4a10_de-skew = <7>; ++ ddr3a3_ddr4a6_de-skew = <8>; ++ ddr3a2_ddr4a4_de-skew = <8>; ++ ddr3a5_ddr4a8_de-skew = <7>; ++ ddr3a4_ddr4a5_de-skew = <9>; ++ ddr3a7_ddr4a11_de-skew = <7>; ++ ddr3a6_ddr4a7_de-skew = <9>; ++ ddr3a9_ddr4a0_de-skew = <8>; ++ ddr3a8_ddr4a13_de-skew = <7>; ++ ddr3a11_ddr4a3_de-skew = <9>; ++ ddr3a10_ddr4cs0_de-skew = <7>; ++ ddr3a13_ddr4a2_de-skew = <8>; ++ ddr3a12_ddr4ba1_de-skew = <7>; ++ ddr3a15_ddr4odt0_de-skew = <7>; ++ ddr3a14_ddr4a1_de-skew = <8>; ++ ddr3ba1_ddr4a15_de-skew = <7>; ++ ddr3ba0_ddr4bg0_de-skew = <7>; ++ ddr3ras_ddr4cke_de-skew = <7>; ++ ddr3ba2_ddr4ba0_de-skew = <8>; ++ ddr3we_ddr4bg1_de-skew = <8>; ++ ddr3cas_ddr4a12_de-skew = <7>; ++ ddr3ckn_ddr4ckn_de-skew = <8>; ++ ddr3ckp_ddr4ckp_de-skew = <8>; ++ ddr3cke_ddr4a16_de-skew = <8>; ++ ddr3odt0_ddr4a14_de-skew = <7>; ++ ddr3cs0_ddr4act_de-skew = <8>; ++ ddr3reset_ddr4reset_de-skew = <7>; ++ ddr3cs1_ddr4cs1_de-skew = <7>; ++ ddr3odt1_ddr4odt1_de-skew = <7>; ++ ++ /* DATA de-skew ++ * RX one step is 25.1ps, range 0-15 ++ * TX one step is 47.8ps, range 0-15 ++ */ ++ cs0_dm0_rx_de-skew = <7>; ++ cs0_dm0_tx_de-skew = <8>; ++ cs0_dq0_rx_de-skew = <7>; ++ cs0_dq0_tx_de-skew = <8>; ++ cs0_dq1_rx_de-skew = <7>; ++ cs0_dq1_tx_de-skew = <8>; ++ cs0_dq2_rx_de-skew = <7>; ++ cs0_dq2_tx_de-skew = <8>; ++ cs0_dq3_rx_de-skew = <7>; ++ cs0_dq3_tx_de-skew = <8>; ++ cs0_dq4_rx_de-skew = <7>; ++ cs0_dq4_tx_de-skew = <8>; ++ cs0_dq5_rx_de-skew = <7>; ++ cs0_dq5_tx_de-skew = <8>; ++ cs0_dq6_rx_de-skew = <7>; ++ cs0_dq6_tx_de-skew = <8>; ++ cs0_dq7_rx_de-skew = <7>; ++ cs0_dq7_tx_de-skew = <8>; ++ cs0_dqs0_rx_de-skew = <6>; ++ cs0_dqs0p_tx_de-skew = <9>; ++ cs0_dqs0n_tx_de-skew = <9>; ++ ++ cs0_dm1_rx_de-skew = <7>; ++ cs0_dm1_tx_de-skew = <7>; ++ cs0_dq8_rx_de-skew = <7>; ++ cs0_dq8_tx_de-skew = <8>; ++ cs0_dq9_rx_de-skew = <7>; ++ cs0_dq9_tx_de-skew = <7>; ++ cs0_dq10_rx_de-skew = <7>; ++ cs0_dq10_tx_de-skew = <8>; ++ cs0_dq11_rx_de-skew = <7>; ++ cs0_dq11_tx_de-skew = <7>; ++ cs0_dq12_rx_de-skew = <7>; ++ cs0_dq12_tx_de-skew = <8>; ++ cs0_dq13_rx_de-skew = <7>; ++ cs0_dq13_tx_de-skew = <7>; ++ cs0_dq14_rx_de-skew = <7>; ++ cs0_dq14_tx_de-skew = <8>; ++ cs0_dq15_rx_de-skew = <7>; ++ cs0_dq15_tx_de-skew = <7>; ++ cs0_dqs1_rx_de-skew = <7>; ++ cs0_dqs1p_tx_de-skew = <9>; ++ cs0_dqs1n_tx_de-skew = <9>; ++ ++ cs0_dm2_rx_de-skew = <7>; ++ cs0_dm2_tx_de-skew = <8>; ++ cs0_dq16_rx_de-skew = <7>; ++ cs0_dq16_tx_de-skew = <8>; ++ cs0_dq17_rx_de-skew = <7>; ++ cs0_dq17_tx_de-skew = <8>; ++ cs0_dq18_rx_de-skew = <7>; ++ cs0_dq18_tx_de-skew = <8>; ++ cs0_dq19_rx_de-skew = <7>; ++ cs0_dq19_tx_de-skew = <8>; ++ cs0_dq20_rx_de-skew = <7>; ++ cs0_dq20_tx_de-skew = <8>; ++ cs0_dq21_rx_de-skew = <7>; ++ cs0_dq21_tx_de-skew = <8>; ++ cs0_dq22_rx_de-skew = <7>; ++ cs0_dq22_tx_de-skew = <8>; ++ cs0_dq23_rx_de-skew = <7>; ++ cs0_dq23_tx_de-skew = <8>; ++ cs0_dqs2_rx_de-skew = <6>; ++ cs0_dqs2p_tx_de-skew = <9>; ++ cs0_dqs2n_tx_de-skew = <9>; ++ ++ cs0_dm3_rx_de-skew = <7>; ++ cs0_dm3_tx_de-skew = <7>; ++ cs0_dq24_rx_de-skew = <7>; ++ cs0_dq24_tx_de-skew = <8>; ++ cs0_dq25_rx_de-skew = <7>; ++ cs0_dq25_tx_de-skew = <7>; ++ cs0_dq26_rx_de-skew = <7>; ++ cs0_dq26_tx_de-skew = <7>; ++ cs0_dq27_rx_de-skew = <7>; ++ cs0_dq27_tx_de-skew = <7>; ++ cs0_dq28_rx_de-skew = <7>; ++ cs0_dq28_tx_de-skew = <7>; ++ cs0_dq29_rx_de-skew = <7>; ++ cs0_dq29_tx_de-skew = <7>; ++ cs0_dq30_rx_de-skew = <7>; ++ cs0_dq30_tx_de-skew = <7>; ++ cs0_dq31_rx_de-skew = <7>; ++ cs0_dq31_tx_de-skew = <7>; ++ cs0_dqs3_rx_de-skew = <7>; ++ cs0_dqs3p_tx_de-skew = <9>; ++ cs0_dqs3n_tx_de-skew = <9>; ++ ++ cs1_dm0_rx_de-skew = <7>; ++ cs1_dm0_tx_de-skew = <8>; ++ cs1_dq0_rx_de-skew = <7>; ++ cs1_dq0_tx_de-skew = <8>; ++ cs1_dq1_rx_de-skew = <7>; ++ cs1_dq1_tx_de-skew = <8>; ++ cs1_dq2_rx_de-skew = <7>; ++ cs1_dq2_tx_de-skew = <8>; ++ cs1_dq3_rx_de-skew = <7>; ++ cs1_dq3_tx_de-skew = <8>; ++ cs1_dq4_rx_de-skew = <7>; ++ cs1_dq4_tx_de-skew = <8>; ++ cs1_dq5_rx_de-skew = <7>; ++ cs1_dq5_tx_de-skew = <8>; ++ cs1_dq6_rx_de-skew = <7>; ++ cs1_dq6_tx_de-skew = <8>; ++ cs1_dq7_rx_de-skew = <7>; ++ cs1_dq7_tx_de-skew = <8>; ++ cs1_dqs0_rx_de-skew = <6>; ++ cs1_dqs0p_tx_de-skew = <9>; ++ cs1_dqs0n_tx_de-skew = <9>; ++ ++ cs1_dm1_rx_de-skew = <7>; ++ cs1_dm1_tx_de-skew = <7>; ++ cs1_dq8_rx_de-skew = <7>; ++ cs1_dq8_tx_de-skew = <8>; ++ cs1_dq9_rx_de-skew = <7>; ++ cs1_dq9_tx_de-skew = <7>; ++ cs1_dq10_rx_de-skew = <7>; ++ cs1_dq10_tx_de-skew = <8>; ++ cs1_dq11_rx_de-skew = <7>; ++ cs1_dq11_tx_de-skew = <7>; ++ cs1_dq12_rx_de-skew = <7>; ++ cs1_dq12_tx_de-skew = <8>; ++ cs1_dq13_rx_de-skew = <7>; ++ cs1_dq13_tx_de-skew = <7>; ++ cs1_dq14_rx_de-skew = <7>; ++ cs1_dq14_tx_de-skew = <8>; ++ cs1_dq15_rx_de-skew = <7>; ++ cs1_dq15_tx_de-skew = <7>; ++ cs1_dqs1_rx_de-skew = <7>; ++ cs1_dqs1p_tx_de-skew = <9>; ++ cs1_dqs1n_tx_de-skew = <9>; ++ ++ cs1_dm2_rx_de-skew = <7>; ++ cs1_dm2_tx_de-skew = <8>; ++ cs1_dq16_rx_de-skew = <7>; ++ cs1_dq16_tx_de-skew = <8>; ++ cs1_dq17_rx_de-skew = <7>; ++ cs1_dq17_tx_de-skew = <8>; ++ cs1_dq18_rx_de-skew = <7>; ++ cs1_dq18_tx_de-skew = <8>; ++ cs1_dq19_rx_de-skew = <7>; ++ cs1_dq19_tx_de-skew = <8>; ++ cs1_dq20_rx_de-skew = <7>; ++ cs1_dq20_tx_de-skew = <8>; ++ cs1_dq21_rx_de-skew = <7>; ++ cs1_dq21_tx_de-skew = <8>; ++ cs1_dq22_rx_de-skew = <7>; ++ cs1_dq22_tx_de-skew = <8>; ++ cs1_dq23_rx_de-skew = <7>; ++ cs1_dq23_tx_de-skew = <8>; ++ cs1_dqs2_rx_de-skew = <6>; ++ cs1_dqs2p_tx_de-skew = <9>; ++ cs1_dqs2n_tx_de-skew = <9>; ++ ++ cs1_dm3_rx_de-skew = <7>; ++ cs1_dm3_tx_de-skew = <7>; ++ cs1_dq24_rx_de-skew = <7>; ++ cs1_dq24_tx_de-skew = <8>; ++ cs1_dq25_rx_de-skew = <7>; ++ cs1_dq25_tx_de-skew = <7>; ++ cs1_dq26_rx_de-skew = <7>; ++ cs1_dq26_tx_de-skew = <7>; ++ cs1_dq27_rx_de-skew = <7>; ++ cs1_dq27_tx_de-skew = <7>; ++ cs1_dq28_rx_de-skew = <7>; ++ cs1_dq28_tx_de-skew = <7>; ++ cs1_dq29_rx_de-skew = <7>; ++ cs1_dq29_tx_de-skew = <7>; ++ cs1_dq30_rx_de-skew = <7>; ++ cs1_dq30_tx_de-skew = <7>; ++ cs1_dq31_rx_de-skew = <7>; ++ cs1_dq31_tx_de-skew = <7>; ++ cs1_dqs3_rx_de-skew = <7>; ++ cs1_dqs3p_tx_de-skew = <9>; ++ cs1_dqs3n_tx_de-skew = <9>; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3328-evb-android-avb.dts b/arch/arm64/boot/dts/rockchip/rk3328-evb-android-avb.dts +new file mode 100755 +index 000000000000..0d72e05e2d59 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3328-evb-android-avb.dts +@@ -0,0 +1,13 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ */ ++ ++/dts-v1/; ++#include "rk3328-evb-android.dtsi" ++ ++/ { ++ model = "Rockchip RK3328 EVB avb"; ++ compatible = "rockchip,rk3328-evb-avb", "rockchip,rk3328"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3328-evb-android.dts b/arch/arm64/boot/dts/rockchip/rk3328-evb-android.dts +new file mode 100755 +index 000000000000..6d50444a5a1d +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3328-evb-android.dts +@@ -0,0 +1,34 @@ ++/* ++ * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++ */ ++ ++/dts-v1/; ++#include "rk3328-evb-android.dtsi" ++ ++/ { ++ model = "Rockchip RK3328 EVB"; ++ compatible = "rockchip,rk3328-evb", "rockchip,rk3328"; ++}; ++ ++&firmware_android{ ++ compatible = "android,firmware"; ++ fstab { ++ compatible = "android,fstab"; ++ system { ++ compatible = "android,system"; ++ dev = "/dev/block/platform/ff520000.dwmmc/by-name/system"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,verify"; ++ }; ++ vendor { ++ compatible = "android,vendor"; ++ dev = "/dev/block/platform/ff520000.dwmmc/by-name/vendor"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,verify"; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3328-evb-android.dtsi b/arch/arm64/boot/dts/rockchip/rk3328-evb-android.dtsi +new file mode 100755 +index 000000000000..e050047420a6 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3328-evb-android.dtsi +@@ -0,0 +1,715 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ */ ++ ++#include "rk3328.dtsi" ++#include "rk3328-android.dtsi" ++#include ++ ++/ { ++ gmac_clkin: external-gmac-clock { ++ compatible = "fixed-clock"; ++ clock-frequency = <125000000>; ++ clock-output-names = "gmac_clkin"; ++ #clock-cells = <0>; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio1 18 GPIO_ACTIVE_LOW>; ++ }; ++ ++ sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,name = "rockchip-rk3328"; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s1>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&codec>; ++ }; ++ }; ++ ++ hdmi-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,mclk-fs = <128>; ++ simple-audio-card,name = "rockchip-hdmi"; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s0>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&hdmi>; ++ }; ++ }; ++ ++ spdif-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,name = "rockchip-spdif"; ++ simple-audio-card,cpu { ++ sound-dai = <&spdif>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&spdif_out>; ++ }; ++ }; ++ ++ spdif_out: spdif-out { ++ compatible = "linux,spdif-dit"; ++ #sound-dai-cells = <0>; ++ }; ++ ++ vcc_host_vbus: host-vbus-regulator { ++ compatible = "regulator-fixed"; ++ gpio = <&gpio0 RK_PA0 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&host_vbus_drv>; ++ regulator-name = "vcc_host_vbus"; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ enable-active-high; ++ }; ++ ++ vcc_otg_vbus: otg-vbus-regulator { ++ compatible = "regulator-fixed"; ++ gpio = <&gpio0 RK_PD3 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&otg_vbus_drv>; ++ regulator-name = "vcc_otg_vbus"; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ enable-active-high; ++ }; ++ ++ vcc_phy: vcc-phy-regulator { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_phy"; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ vcc_sd: sdmmc-regulator { ++ compatible = "regulator-fixed"; ++ gpio = <&gpio0 30 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc0m1_gpio>; ++ regulator-name = "vcc_sd"; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ vin-supply = <&vcc_io>; ++ }; ++ ++ vcc_sys: vcc-sys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_sys"; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ regulator-always-on; ++ }; ++ ++ xin32k: xin32k { ++ compatible = "fixed-clock"; ++ clock-frequency = <32768>; ++ clock-output-names = "xin32k"; ++ #clock-cells = <0>; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&rk805 1>; ++ clock-names = "ext_clock"; ++ uart_rts_gpios = <&gpio1 10 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart0_rts>; ++ pinctrl-1 = <&uart0_gpios>; ++ BT,power_gpio = <&gpio1 21 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio1 26 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ rockchip,grf = <&grf>; ++ wifi_chip_type = "ap6354"; ++ sdio_vref = <1800>; ++ WIFI,host_wake_irq = <&gpio1 19 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++}; ++ ++&avsd { ++ status = "okay"; ++}; ++ ++&codec { ++ #sound-dai-cells = <0>; ++ status = "okay"; ++}; ++ ++&cpu0 { ++ cpu-supply = <&vdd_arm>; ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&dmc { ++ center-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&emmc { ++ bus-width = <8>; ++ cap-mmc-highspeed; ++ mmc-hs200-1_8v; ++ no-sdio; ++ no-sd; ++ disable-wp; ++ non-removable; ++ num-slots = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&emmc_clk &emmc_cmd &emmc_bus8>; ++ rockchip,default-sample-phase = <90>; ++ status = "okay"; ++}; ++ ++&gmac2io { ++ phy-supply = <&vcc_phy>; ++ phy-mode = "rgmii"; ++ clock_in_out = "input"; ++ snps,reset-gpio = <&gpio1 RK_PC2 GPIO_ACTIVE_LOW>; ++ snps,reset-active-low; ++ snps,reset-delays-us = <0 10000 50000>; ++ assigned-clocks = <&cru SCLK_MAC2IO>, <&cru SCLK_MAC2IO_EXT>; ++ assigned-clock-parents = <&gmac_clkin>, <&gmac_clkin>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&rgmiim1_pins>; ++ tx_delay = <0x26>; ++ rx_delay = <0x11>; ++ status = "disabled"; ++}; ++ ++&gmac2phy { ++ phy-supply = <&vcc_phy>; ++ clock_in_out = "output"; ++ assigned-clocks = <&cru SCLK_MAC2PHY_SRC>; ++ assigned-clock-rate = <50000000>; ++ assigned-clocks = <&cru SCLK_MAC2PHY>; ++ assigned-clock-parents = <&cru SCLK_MAC2PHY_SRC>; ++ status = "okay"; ++}; ++ ++&gpu { ++ status = "okay"; ++ mali-supply = <&vdd_logic>; ++}; ++ ++&hdmi { ++ #sound-dai-cells = <0>; ++ ddc-i2c-scl-high-time-ns = <9625>; ++ ddc-i2c-scl-low-time-ns = <10000>; ++ status = "okay"; ++}; ++ ++&hdmiphy { ++ rockchip,phy-table = ++ <165000000 0x07 0x0a 0x0a 0x0a 0x00 0x00 0x08 ++ 0x08 0x08 0x00 0xac 0xcc 0xcc 0xcc>, ++ <340000000 0x0b 0x0d 0x0d 0x0d 0x07 0x15 0x08 ++ 0x08 0x08 0x3f 0xac 0xcc 0xcd 0xdd>, ++ <594000000 0x10 0x1a 0x1a 0x1a 0x07 0x15 0x08 ++ 0x08 0x08 0x00 0xac 0xcc 0xcc 0xcc>; ++ status = "okay"; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ ++ rk805: rk805@18 { ++ compatible = "rockchip,rk805"; ++ status = "okay"; ++ reg = <0x18>; ++ interrupt-parent = <&gpio2>; ++ interrupts = <6 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pmic_int_l>; ++ wakeup-source; ++ gpio-controller; ++ #gpio-cells = <2>; ++ #clock-cells = <1>; ++ clock-output-names = "rk805-clkout1", "rk805-clkout2"; ++ ++ vcc1-supply = <&vcc_sys>; ++ vcc2-supply = <&vcc_sys>; ++ vcc3-supply = <&vcc_sys>; ++ vcc4-supply = <&vcc_sys>; ++ vcc5-supply = <&vcc_io>; ++ vcc6-supply = <&vcc_io>; ++ ++ rtc { ++ status = "okay"; ++ }; ++ ++ pwrkey { ++ status = "disabled"; ++ }; ++ ++ gpio { ++ status = "okay"; ++ }; ++ ++ regulators { ++ vdd_logic: DCDC_REG1 { ++ regulator-name = "vdd_logic"; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1450000>; ++ regulator-initial-mode = <0x1>; ++ regulator-ramp-delay = <12500>; ++ regulator-boot-on; ++ regulator-always-on; ++ regulator-state-mem { ++ regulator-mode = <0x2>; ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vdd_arm: DCDC_REG2 { ++ regulator-name = "vdd_arm"; ++ regulator-init-microvolt = <1225000>; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1450000>; ++ regulator-initial-mode = <0x1>; ++ regulator-ramp-delay = <12500>; ++ regulator-boot-on; ++ regulator-always-on; ++ regulator-state-mem { ++ regulator-mode = <0x2>; ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <950000>; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-name = "vcc_ddr"; ++ regulator-initial-mode = <0x1>; ++ regulator-boot-on; ++ regulator-always-on; ++ regulator-state-mem { ++ regulator-mode = <0x2>; ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_io: DCDC_REG4 { ++ regulator-name = "vcc_io"; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-initial-mode = <0x1>; ++ regulator-boot-on; ++ regulator-always-on; ++ regulator-state-mem { ++ regulator-mode = <0x2>; ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vdd_18: LDO_REG1 { ++ regulator-name = "vdd_18"; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-boot-on; ++ regulator-always-on; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vcc_18emmc: LDO_REG2 { ++ regulator-name = "vcc_18emmc"; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-boot-on; ++ regulator-always-on; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd_11: LDO_REG3 { ++ regulator-name = "vdd_11"; ++ regulator-min-microvolt = <1100000>; ++ regulator-max-microvolt = <1100000>; ++ regulator-boot-on; ++ regulator-always-on; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1100000>; ++ }; ++ }; ++ }; ++ }; ++}; ++ ++&secure_memory { ++ /* ++ * enable like this: ++ * reg = <0x0 0x20000000 0x0 0x10000000>; ++ */ ++ reg = <0x0 0x20000000 0x0 0x0>; ++}; ++ ++&i2s0 { ++ #sound-dai-cells = <0>; ++ rockchip,bclk-fs = <128>; ++ status = "okay"; ++}; ++ ++&i2s1 { ++ #sound-dai-cells = <0>; ++ status = "okay"; ++}; ++ ++&iep { ++ status = "okay"; ++}; ++ ++&iep_mmu { ++ status = "okay"; ++}; ++ ++&io_domains { ++ status = "okay"; ++ ++ vccio1-supply = <&vcc_io>; ++ vccio2-supply = <&vcc_18emmc>; ++ vccio3-supply = <&vcc_io>; ++ vccio4-supply = <&vdd_18>; ++ vccio5-supply = <&vcc_io>; ++ vccio6-supply = <&vcc_io>; ++ pmuio-supply = <&vcc_io>; ++}; ++ ++&mpp_srv { ++ status = "okay"; ++}; ++ ++&pinctrl { ++ pmic { ++ pmic_int_l: pmic-int-l { ++ rockchip,pins = ++ <2 RK_PA6 RK_FUNC_GPIO &pcfg_pull_up>; /* gpio2_a6 */ ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = ++ <1 RK_PC2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ usb { ++ host_vbus_drv: host-vbus-drv { ++ rockchip,pins = ++ <0 RK_PA0 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ otg_vbus_drv: otg-vbus-drv { ++ rockchip,pins = ++ <0 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart0_gpios: uart0-gpios { ++ rockchip,pins = ++ <1 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++&pwm3 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pwmir_pin>; ++ compatible = "rockchip,remotectl-pwm"; ++ remote_pwm_id = <3>; ++ handle_cpu_id = <1>; ++ remote_support_psci = <1>; ++ ++ ir_key1 { ++ rockchip,usercode = <0x4040>; ++ rockchip,key_table = ++ <0xf2 KEY_REPLY>, ++ <0xba KEY_BACK>, ++ <0xf4 KEY_UP>, ++ <0xf1 KEY_DOWN>, ++ <0xef KEY_LEFT>, ++ <0xee KEY_RIGHT>, ++ <0xbd KEY_HOME>, ++ <0xea KEY_VOLUMEUP>, ++ <0xe3 KEY_VOLUMEDOWN>, ++ <0xe2 KEY_SEARCH>, ++ <0xb2 KEY_POWER>, ++ <0xbc KEY_MUTE>, ++ <0xec KEY_MENU>, ++ <0xbf 0x190>, ++ <0xe0 0x191>, ++ <0xe1 0x192>, ++ <0xe9 183>, ++ <0xe6 248>, ++ <0xe8 185>, ++ <0xe7 186>, ++ <0xf0 388>, ++ <0xbe 0x175>; ++ }; ++ ++ ir_key2 { ++ rockchip,usercode = <0xff00>; ++ rockchip,key_table = ++ <0xf9 KEY_HOME>, ++ <0xbf KEY_BACK>, ++ <0xfb KEY_MENU>, ++ <0xaa KEY_REPLY>, ++ <0xb9 KEY_UP>, ++ <0xe9 KEY_DOWN>, ++ <0xb8 KEY_LEFT>, ++ <0xea KEY_RIGHT>, ++ <0xeb KEY_VOLUMEDOWN>, ++ <0xef KEY_VOLUMEUP>, ++ <0xf7 KEY_MUTE>, ++ <0xe7 KEY_POWER>, ++ <0xfc KEY_POWER>, ++ <0xa9 KEY_VOLUMEDOWN>, ++ <0xa8 KEY_PLAYPAUSE>, ++ <0xe0 KEY_VOLUMEDOWN>, ++ <0xa5 KEY_VOLUMEDOWN>, ++ <0xab 183>, ++ <0xb7 388>, ++ <0xe8 388>, ++ <0xf8 184>, ++ <0xaf 185>, ++ <0xed KEY_VOLUMEDOWN>, ++ <0xee 186>, ++ <0xb3 KEY_VOLUMEDOWN>, ++ <0xf1 KEY_VOLUMEDOWN>, ++ <0xf2 KEY_VOLUMEDOWN>, ++ <0xf3 KEY_SEARCH>, ++ <0xb4 KEY_VOLUMEDOWN>, ++ <0xa4 KEY_SETUP>, ++ <0xbe KEY_SEARCH>; ++ }; ++ ++ ir_key3 { ++ rockchip,usercode = <0x1dcc>; ++ rockchip,key_table = ++ <0xee KEY_REPLY>, ++ <0xf0 KEY_BACK>, ++ <0xf8 KEY_UP>, ++ <0xbb KEY_DOWN>, ++ <0xef KEY_LEFT>, ++ <0xed KEY_RIGHT>, ++ <0xfc KEY_HOME>, ++ <0xf1 KEY_VOLUMEUP>, ++ <0xfd KEY_VOLUMEDOWN>, ++ <0xb7 KEY_SEARCH>, ++ <0xff KEY_POWER>, ++ <0xf3 KEY_MUTE>, ++ <0xbf KEY_MENU>, ++ <0xf9 0x191>, ++ <0xf5 0x192>, ++ <0xb3 388>, ++ <0xbe KEY_1>, ++ <0xba KEY_2>, ++ <0xb2 KEY_3>, ++ <0xbd KEY_4>, ++ <0xf9 KEY_5>, ++ <0xb1 KEY_6>, ++ <0xfc KEY_7>, ++ <0xf8 KEY_8>, ++ <0xb0 KEY_9>, ++ <0xb6 KEY_0>, ++ <0xb5 KEY_BACKSPACE>; ++ }; ++}; ++ ++&rga { ++ status = "okay"; ++}; ++ ++&rkvdec { ++ status = "okay"; ++ vcodec-supply = <&vdd_logic>; ++}; ++ ++&rkvdec_mmu { ++ status = "okay"; ++}; ++ ++&rockchip_suspend { ++ status = "okay"; ++ rockchip,virtual-poweroff = <1>; ++}; ++ ++&uart0 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart0_xfer &uart0_cts>; ++ status = "okay"; ++}; ++ ++&sdio { ++ bus-width = <4>; ++ cap-sd-highspeed; ++ cap-sdio-irq; ++ disable-wp; ++ keep-power-in-suspend; ++ max-frequency = <125000000>; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ non-removable; ++ num-slots = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc1_bus4 &sdmmc1_cmd &sdmmc1_clk>; ++ no-sd; ++ no-mmc; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&sdmmc { ++ bus-width = <4>; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ disable-wp; ++ max-frequency = <150000000>; ++ num-slots = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc0_clk &sdmmc0_cmd &sdmmc0_dectn &sdmmc0_bus4>; ++ no-sdio; ++ no-mmc; ++ status = "okay"; ++ vmmc-supply = <&vcc_sd>; ++}; ++ ++&spdif { ++ #sound-dai-cells = <0>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&spdifm2_tx>; ++ status = "okay"; ++}; ++ ++&threshold { ++ temperature = <90000>; /* millicelsius */ ++}; ++ ++&target { ++ temperature = <105000>; /* millicelsius */ ++}; ++ ++&soc_crit { ++ temperature = <115000>; /* millicelsius */ ++}; ++ ++&tsadc { ++ rockchip,hw-tshut-temp = <120000>; ++ status = "okay"; ++}; ++ ++&tve { ++ status = "okay"; ++}; ++ ++&u2phy { ++ status = "okay"; ++ ++ u2phy_host: host-port { ++ status = "okay"; ++ }; ++ ++ u2phy_otg: otg-port { ++ vbus-supply = <&vcc_otg_vbus>; ++ status = "okay"; ++ }; ++}; ++ ++&u3phy { ++ vbus-supply = <&vcc_host_vbus>; ++ status = "okay"; ++}; ++ ++&u3phy_utmi { ++ status = "okay"; ++}; ++ ++&u3phy_pipe { ++ status = "okay"; ++}; ++ ++&usb20_otg { ++ status = "okay"; ++}; ++ ++&usb_host0_ehci { ++ status = "okay"; ++}; ++ ++&usb_host0_ohci { ++ status = "okay"; ++}; ++ ++&usbdrd3 { ++ status = "okay"; ++}; ++ ++&usbdrd_dwc3 { ++ status = "okay"; ++}; ++ ++&vdpu { ++ status = "okay"; ++}; ++ ++&vpu_mmu { ++ status = "okay"; ++}; ++ ++&vepu { ++ status = "okay"; ++}; ++ ++&vepu_mmu { ++ status = "okay"; ++}; ++ ++&vepu22 { ++ status = "okay"; ++}; ++ ++&vepu22_mmu { ++ status = "okay"; ++}; ++ ++&vop { ++ status = "okay"; ++}; ++ ++&vop_mmu { ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3328-evb.dts b/arch/arm64/boot/dts/rockchip/rk3328-evb.dts +index a48767931af6..82d2f01a8be6 100644 +--- a/arch/arm64/boot/dts/rockchip/rk3328-evb.dts ++++ b/arch/arm64/boot/dts/rockchip/rk3328-evb.dts +@@ -37,6 +37,18 @@ sdio_pwrseq: sdio-pwrseq { + reset-gpios = <&gpio1 18 GPIO_ACTIVE_LOW>; + }; + ++ fiq-debugger { ++ compatible = "rockchip,fiq-debugger"; ++ rockchip,serial-id = <2>; ++ rockchip,signal-irq = <159>; ++ rockchip,wake-irq = <0>; ++ /* If enable uart uses irq instead of fiq */ ++ rockchip,irq-mode-enable = <0>; ++ rockchip,baudrate = <1500000>; /* Only 115200 and 1500000 */ ++ interrupts = ; ++ status = "okay"; ++ }; ++ + vcc_sd: sdmmc-regulator { + compatible = "regulator-fixed"; + gpio = <&gpio0 30 GPIO_ACTIVE_LOW>; +@@ -64,6 +76,14 @@ vcc_phy: vcc-phy-regulator { + regulator-always-on; + regulator-boot-on; + }; ++ ++ xin32k: xin32k { ++ compatible = "fixed-clock"; ++ clock-frequency = <32768>; ++ clock-output-names = "xin32k"; ++ #clock-cells = <0>; ++ }; ++ + }; + + &cpu0 { +@@ -109,7 +129,7 @@ rk805: pmic@18 { + interrupt-parent = <&gpio2>; + interrupts = <6 IRQ_TYPE_LEVEL_LOW>; + #clock-cells = <1>; +- clock-output-names = "xin32k", "rk805-clkout2"; ++ clock-output-names = "rk805-clkout1", "rk805-clkout2"; + gpio-controller; + #gpio-cells = <2>; + pinctrl-names = "default"; +diff --git a/arch/arm64/boot/dts/rockchip/rk3328-nanopi-r2s.dts b/arch/arm64/boot/dts/rockchip/rk3328-nanopi-r2s.dts +index 83a0bdbe00d6..1eecad724f04 100644 +--- a/arch/arm64/boot/dts/rockchip/rk3328-nanopi-r2s.dts ++++ b/arch/arm64/boot/dts/rockchip/rk3328-nanopi-r2s.dts +@@ -71,8 +71,8 @@ vcc_io_sdio: sdmmcio-regulator { + regulator-settling-time-us = <5000>; + regulator-type = "voltage"; + startup-delay-us = <2000>; +- states = <1800000 0x1>, +- <3300000 0x0>; ++ states = <1800000 0x1 ++ 3300000 0x0>; + vin-supply = <&vcc_io_33>; + }; + +diff --git a/arch/arm64/boot/dts/rockchip/rk3328-roc-cc.dts b/arch/arm64/boot/dts/rockchip/rk3328-roc-cc.dts +index daa9a0c601a9..b76282e704de 100644 +--- a/arch/arm64/boot/dts/rockchip/rk3328-roc-cc.dts ++++ b/arch/arm64/boot/dts/rockchip/rk3328-roc-cc.dts +@@ -45,8 +45,8 @@ vcc_sd: sdmmc-regulator { + vcc_sdio: sdmmcio-regulator { + compatible = "regulator-gpio"; + gpios = <&grf_gpio 0 GPIO_ACTIVE_HIGH>; +- states = <1800000 0x1>, +- <3300000 0x0>; ++ states = <1800000 0x1 ++ 3300000 0x0>; + regulator-name = "vcc_sdio"; + regulator-type = "voltage"; + regulator-min-microvolt = <1800000>; +diff --git a/arch/arm64/boot/dts/rockchip/rk3328-rock64-android-avb.dts b/arch/arm64/boot/dts/rockchip/rk3328-rock64-android-avb.dts +new file mode 100755 +index 000000000000..adc1dd7cd4da +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3328-rock64-android-avb.dts +@@ -0,0 +1,32 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ */ ++ ++/dts-v1/; ++#include "rk3328-rock64-android.dtsi" ++ ++/ { ++ model = "Pine64 Rock64 avb"; ++ compatible = "pine64,rock64-android-avb", "rockchip,rk3328"; ++}; ++ ++&firmware_android{ ++ compatible = "android,firmware"; ++ boot_devices = "ff520000.dwmmc"; ++ vbmeta { ++ compatible = "android,vbmeta"; ++ parts = "vbmeta,boot,system,vendor,dtbo"; ++ }; ++ fstab { ++ compatible = "android,fstab"; ++ vendor { ++ compatible = "android,vendor"; ++ dev = "/dev/block/by-name/vendor"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,avb"; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3328-rock64-android.dts b/arch/arm64/boot/dts/rockchip/rk3328-rock64-android.dts +new file mode 100755 +index 000000000000..66e83916c728 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3328-rock64-android.dts +@@ -0,0 +1,34 @@ ++/* ++ * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++ */ ++ ++/dts-v1/; ++#include "rk3328-rock64-android.dtsi" ++ ++/ { ++ model = "Pine64 Rock64"; ++ compatible = "pine64,rock64-android", "rockchip,rk3328"; ++}; ++ ++&firmware_android{ ++ compatible = "android,firmware"; ++ fstab { ++ compatible = "android,fstab"; ++ system { ++ compatible = "android,system"; ++ dev = "/dev/block/platform/ff520000.dwmmc/by-name/system"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,verify"; ++ }; ++ vendor { ++ compatible = "android,vendor"; ++ dev = "/dev/block/platform/ff520000.dwmmc/by-name/vendor"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,verify"; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3328-rock64-android.dtsi b/arch/arm64/boot/dts/rockchip/rk3328-rock64-android.dtsi +new file mode 100755 +index 000000000000..3dad4f4fafcb +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3328-rock64-android.dtsi +@@ -0,0 +1,612 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ */ ++ ++#include "rk3328.dtsi" ++#include "rk3328-android.dtsi" ++#include ++ ++/ { ++ gmac_clkin: external-gmac-clock { ++ compatible = "fixed-clock"; ++ clock-frequency = <125000000>; ++ clock-output-names = "gmac_clkin"; ++ #clock-cells = <0>; ++ }; ++ ++ vcc_sd: sdmmc-regulator { ++ compatible = "regulator-fixed"; ++ gpio = <&gpio0 30 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc0m1_gpio>; ++ regulator-name = "vcc_sd"; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ vin-supply = <&vcc_io>; ++ }; ++ ++ vcc_host_5v: vcc-host-5v-regulator { ++ compatible = "regulator-fixed"; ++ enable-active-high; ++ gpio = <&gpio0 0 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&usb30_host_drv>; ++ regulator-name = "vcc_host_5v"; ++ regulator-always-on; ++ vin-supply = <&vcc_sys>; ++ }; ++ ++ vcc_host1_5v: vcc_otg_5v: vcc-host1-5v-regulator { ++ compatible = "regulator-fixed"; ++ enable-active-high; ++ gpio = <&gpio0 27 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&usb20_host_drv>; ++ regulator-name = "vcc_host1_5v"; ++ regulator-always-on; ++ vin-supply = <&vcc_sys>; ++ }; ++ ++ vcc_sys: vcc-sys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ }; ++ ++ sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,name = "rockchip-rk3328"; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s1>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&codec>; ++ }; ++ }; ++ ++ hdmi-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,mclk-fs = <128>; ++ simple-audio-card,name = "rockchip-hdmi"; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s0>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&hdmi>; ++ }; ++ }; ++ ++ vcc_phy: vcc-phy-regulator { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_phy"; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ xin32k: xin32k { ++ compatible = "fixed-clock"; ++ clock-frequency = <32768>; ++ clock-output-names = "xin32k"; ++ #clock-cells = <0>; ++ }; ++}; ++ ++&avsd { ++ status = "okay"; ++}; ++ ++&codec { ++ #sound-dai-cells = <0>; ++ status = "okay"; ++}; ++ ++&cpu0 { ++ cpu-supply = <&vdd_arm>; ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&dmc { ++ center-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&emmc { ++ bus-width = <8>; ++ cap-mmc-highspeed; ++ mmc-hs200-1_8v; ++ no-sdio; ++ no-sd; ++ disable-wp; ++ non-removable; ++ num-slots = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&emmc_clk &emmc_cmd &emmc_bus8>; ++ status = "okay"; ++}; ++ ++&gmac2io { ++ phy-supply = <&vcc_phy>; ++ phy-mode = "rgmii"; ++ clock_in_out = "input"; ++ snps,reset-gpio = <&gpio1 RK_PC2 GPIO_ACTIVE_LOW>; ++ snps,reset-active-low; ++ snps,reset-delays-us = <0 10000 50000>; ++ assigned-clocks = <&cru SCLK_MAC2IO>, <&cru SCLK_MAC2IO_EXT>; ++ assigned-clock-parents = <&gmac_clkin>, <&gmac_clkin>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&rgmiim1_pins>; ++ tx_delay = <0x26>; ++ rx_delay = <0x11>; ++ status = "okay"; ++}; ++ ++&gmac2phy { ++ phy-supply = <&vcc_phy>; ++ clock_in_out = "output"; ++ assigned-clocks = <&cru SCLK_MAC2PHY_SRC>; ++ assigned-clock-rate = <50000000>; ++ assigned-clocks = <&cru SCLK_MAC2PHY>; ++ assigned-clock-parents = <&cru SCLK_MAC2PHY_SRC>; ++ status = "disabled"; ++}; ++ ++&gpu { ++ status = "okay"; ++ mali-supply = <&vdd_logic>; ++}; ++ ++&hdmi { ++ #sound-dai-cells = <0>; ++ status = "okay"; ++}; ++ ++&hdmiphy { ++ status = "okay"; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ ++ rk805: rk805@18 { ++ compatible = "rockchip,rk805"; ++ status = "okay"; ++ reg = <0x18>; ++ interrupt-parent = <&gpio2>; ++ interrupts = <6 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pmic_int_l>; ++ wakeup-source; ++ gpio-controller; ++ #gpio-cells = <2>; ++ #clock-cells = <1>; ++ clock-output-names = "rk805-clkout1", "rk805-clkout2"; ++ ++ vcc1-supply = <&vcc_sys>; ++ vcc2-supply = <&vcc_sys>; ++ vcc3-supply = <&vcc_sys>; ++ vcc4-supply = <&vcc_sys>; ++ vcc5-supply = <&vcc_io>; ++ vcc6-supply = <&vcc_io>; ++ ++ rtc { ++ status = "okay"; ++ }; ++ ++ pwrkey { ++ status = "disabled"; ++ }; ++ ++ gpio { ++ status = "okay"; ++ }; ++ ++ regulators { ++ vdd_logic: DCDC_REG1 { ++ regulator-name = "vdd_logic"; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1450000>; ++ regulator-initial-mode = <0x1>; ++ regulator-ramp-delay = <12500>; ++ regulator-boot-on; ++ regulator-always-on; ++ regulator-state-mem { ++ regulator-mode = <0x2>; ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vdd_arm: DCDC_REG2 { ++ regulator-name = "vdd_arm"; ++ regulator-init-microvolt = <1225000>; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1450000>; ++ regulator-initial-mode = <0x1>; ++ regulator-ramp-delay = <12500>; ++ regulator-boot-on; ++ regulator-always-on; ++ regulator-state-mem { ++ regulator-mode = <0x2>; ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <950000>; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-name = "vcc_ddr"; ++ regulator-initial-mode = <0x1>; ++ regulator-boot-on; ++ regulator-always-on; ++ regulator-state-mem { ++ regulator-mode = <0x2>; ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_io: DCDC_REG4 { ++ regulator-name = "vcc_io"; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-initial-mode = <0x1>; ++ regulator-boot-on; ++ regulator-always-on; ++ regulator-state-mem { ++ regulator-mode = <0x2>; ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vdd_18: LDO_REG1 { ++ regulator-name = "vdd_18"; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-boot-on; ++ regulator-always-on; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vcc_18emmc: LDO_REG2 { ++ regulator-name = "vcc_18emmc"; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-boot-on; ++ regulator-always-on; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd_11: LDO_REG3 { ++ regulator-name = "vdd_11"; ++ regulator-min-microvolt = <1100000>; ++ regulator-max-microvolt = <1100000>; ++ regulator-boot-on; ++ regulator-always-on; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1100000>; ++ }; ++ }; ++ }; ++ }; ++}; ++ ++&i2s0 { ++ #sound-dai-cells = <0>; ++ rockchip,bclk-fs = <128>; ++ status = "okay"; ++}; ++ ++&i2s1 { ++ #sound-dai-cells = <0>; ++ status = "okay"; ++}; ++ ++&iep { ++ status = "okay"; ++}; ++ ++&iep_mmu { ++ status = "okay"; ++}; ++ ++&io_domains { ++ status = "okay"; ++ ++ vccio1-supply = <&vcc_io>; ++ vccio2-supply = <&vcc_18emmc>; ++ vccio3-supply = <&vcc_io>; ++ vccio4-supply = <&vdd_18>; ++ vccio5-supply = <&vcc_io>; ++ vccio6-supply = <&vcc_io>; ++ pmuio-supply = <&vcc_io>; ++}; ++ ++&mpp_srv { ++ status = "okay"; ++}; ++ ++&pinctrl { ++ pmic { ++ pmic_int_l: pmic-int-l { ++ rockchip,pins = <2 RK_PA6 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ usb2 { ++ usb20_host_drv: usb20-host-drv { ++ rockchip,pins = <0 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ usb3 { ++ usb30_host_drv: usb30-host-drv { ++ rockchip,pins = <0 RK_PA0 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++&pwm3 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pwmir_pin>; ++ compatible = "rockchip,remotectl-pwm"; ++ remote_pwm_id = <3>; ++ handle_cpu_id = <1>; ++ remote_support_psci = <1>; ++ ++ ir_key1 { ++ rockchip,usercode = <0x4040>; ++ rockchip,key_table = ++ <0xf2 KEY_REPLY>, ++ <0xba KEY_BACK>, ++ <0xf4 KEY_UP>, ++ <0xf1 KEY_DOWN>, ++ <0xef KEY_LEFT>, ++ <0xee KEY_RIGHT>, ++ <0xbd KEY_HOME>, ++ <0xea KEY_VOLUMEUP>, ++ <0xe3 KEY_VOLUMEDOWN>, ++ <0xe2 KEY_SEARCH>, ++ <0xb2 KEY_POWER>, ++ <0xbc KEY_MUTE>, ++ <0xec KEY_MENU>, ++ <0xbf 0x190>, ++ <0xe0 0x191>, ++ <0xe1 0x192>, ++ <0xe9 183>, ++ <0xe6 248>, ++ <0xe8 185>, ++ <0xe7 186>, ++ <0xf0 388>, ++ <0xbe 0x175>; ++ }; ++ ++ ir_key2 { ++ rockchip,usercode = <0xff00>; ++ rockchip,key_table = ++ <0xf9 KEY_HOME>, ++ <0xbf KEY_BACK>, ++ <0xfb KEY_MENU>, ++ <0xaa KEY_REPLY>, ++ <0xb9 KEY_UP>, ++ <0xe9 KEY_DOWN>, ++ <0xb8 KEY_LEFT>, ++ <0xea KEY_RIGHT>, ++ <0xeb KEY_VOLUMEDOWN>, ++ <0xef KEY_VOLUMEUP>, ++ <0xf7 KEY_MUTE>, ++ <0xe7 KEY_POWER>, ++ <0xfc KEY_POWER>, ++ <0xa9 KEY_VOLUMEDOWN>, ++ <0xa8 KEY_PLAYPAUSE>, ++ <0xe0 KEY_VOLUMEDOWN>, ++ <0xa5 KEY_VOLUMEDOWN>, ++ <0xab 183>, ++ <0xb7 388>, ++ <0xe8 388>, ++ <0xf8 184>, ++ <0xaf 185>, ++ <0xed KEY_VOLUMEDOWN>, ++ <0xee 186>, ++ <0xb3 KEY_VOLUMEDOWN>, ++ <0xf1 KEY_VOLUMEDOWN>, ++ <0xf2 KEY_VOLUMEDOWN>, ++ <0xf3 KEY_SEARCH>, ++ <0xb4 KEY_VOLUMEDOWN>, ++ <0xa4 KEY_SETUP>, ++ <0xbe KEY_SEARCH>; ++ }; ++ ++ ir_key3 { ++ rockchip,usercode = <0x1dcc>; ++ rockchip,key_table = ++ <0xee KEY_REPLY>, ++ <0xf0 KEY_BACK>, ++ <0xf8 KEY_UP>, ++ <0xbb KEY_DOWN>, ++ <0xef KEY_LEFT>, ++ <0xed KEY_RIGHT>, ++ <0xfc KEY_HOME>, ++ <0xf1 KEY_VOLUMEUP>, ++ <0xfd KEY_VOLUMEDOWN>, ++ <0xb7 KEY_SEARCH>, ++ <0xff KEY_POWER>, ++ <0xf3 KEY_MUTE>, ++ <0xbf KEY_MENU>, ++ <0xf9 0x191>, ++ <0xf5 0x192>, ++ <0xb3 388>, ++ <0xbe KEY_1>, ++ <0xba KEY_2>, ++ <0xb2 KEY_3>, ++ <0xbd KEY_4>, ++ <0xf9 KEY_5>, ++ <0xb1 KEY_6>, ++ <0xfc KEY_7>, ++ <0xf8 KEY_8>, ++ <0xb0 KEY_9>, ++ <0xb6 KEY_0>, ++ <0xb5 KEY_BACKSPACE>; ++ }; ++}; ++ ++&rga { ++ status = "okay"; ++}; ++ ++&rkvdec { ++ status = "okay"; ++ vcodec-supply = <&vdd_logic>; ++}; ++ ++&rkvdec_mmu { ++ status = "okay"; ++}; ++ ++&rockchip_suspend { ++ status = "okay"; ++ rockchip,virtual-poweroff = <1>; ++}; ++ ++&sdmmc { ++ bus-width = <4>; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ disable-wp; ++ max-frequency = <150000000>; ++ num-slots = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc0_clk &sdmmc0_cmd &sdmmc0_dectn &sdmmc0_bus4>; ++ no-sdio; ++ no-mmc; ++ status = "okay"; ++ vmmc-supply = <&vcc_sd>; ++}; ++ ++&spi0 { ++ status = "okay"; ++ ++ flash@0 { ++ compatible = "gigadevice,gd25q128", "jedec,spi-nor"; ++ #address-cells = <1>; ++ #size-cells = <1>; ++ reg = <0>; ++ m25p,fast-read; ++ /* The max SCLK of the flash 104/80 MHZ */ ++ spi-max-frequency = <50000000>; ++ }; ++}; ++ ++&threshold { ++ temperature = <90000>; /* millicelsius */ ++}; ++ ++&target { ++ temperature = <105000>; /* millicelsius */ ++}; ++ ++&soc_crit { ++ temperature = <115000>; /* millicelsius */ ++}; ++ ++&tsadc { ++ rockchip,hw-tshut-temp = <120000>; ++ status = "okay"; ++}; ++ ++&u2phy { ++ status = "okay"; ++ ++}; ++ ++&u2phy_host { ++ phy-supply = <&vcc_host1_5v>; ++ status = "okay"; ++}; ++ ++&u2phy_otg { ++ phy-supply = <&vcc_otg_5v>; ++ status = "okay"; ++}; ++ ++&u3phy { ++ phy-supply = <&vcc_host_5v>; ++ status = "okay"; ++}; ++ ++&u3phy_utmi { ++ status = "okay"; ++}; ++ ++&u3phy_pipe { ++ status = "okay"; ++}; ++ ++&usb20_otg { ++ status = "okay"; ++}; ++ ++&usb_host0_ehci { ++ status = "okay"; ++}; ++ ++&usb_host0_ohci { ++ status = "okay"; ++}; ++ ++&usbdrd3 { ++ status = "okay"; ++}; ++ ++&usbdrd_dwc3 { ++ status = "okay"; ++}; ++ ++&vdpu { ++ status = "okay"; ++}; ++ ++&vpu_mmu { ++ status = "okay"; ++}; ++ ++&vepu { ++ status = "okay"; ++}; ++ ++&vepu_mmu { ++ status = "okay"; ++}; ++ ++&vepu22 { ++ status = "okay"; ++}; ++ ++&vepu22_mmu { ++ status = "okay"; ++}; ++ ++&vop { ++ status = "okay"; ++}; ++ ++&vop_mmu { ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3328-rock64.dts b/arch/arm64/boot/dts/rockchip/rk3328-rock64.dts +index 95ab6928cfd4..4c33c21eee8d 100644 +--- a/arch/arm64/boot/dts/rockchip/rk3328-rock64.dts ++++ b/arch/arm64/boot/dts/rockchip/rk3328-rock64.dts +@@ -21,6 +21,13 @@ gmac_clkin: external-gmac-clock { + #clock-cells = <0>; + }; + ++ xin32k: xin32k { ++ compatible = "fixed-clock"; ++ clock-frequency = <32768>; ++ clock-output-names = "xin32k"; ++ #clock-cells = <0>; ++ }; ++ + vcc_sd: sdmmc-regulator { + compatible = "regulator-fixed"; + gpio = <&gpio0 RK_PD6 GPIO_ACTIVE_LOW>; +@@ -176,7 +183,7 @@ rk805: pmic@18 { + interrupt-parent = <&gpio2>; + interrupts = <6 IRQ_TYPE_LEVEL_LOW>; + #clock-cells = <1>; +- clock-output-names = "xin32k", "rk805-clkout2"; ++ clock-output-names = "rk805-clkout1", "rk805-clkout2"; + gpio-controller; + #gpio-cells = <2>; + pinctrl-names = "default"; +@@ -384,11 +391,6 @@ &usb20_otg { + status = "okay"; + }; + +-&usbdrd3 { +- dr_mode = "host"; +- status = "okay"; +-}; +- + &usb_host0_ehci { + status = "okay"; + }; +diff --git a/arch/arm64/boot/dts/rockchip/rk3328.dtsi b/arch/arm64/boot/dts/rockchip/rk3328.dtsi +index e546c9d1d646..03f8b2fe42c3 100644 +--- a/arch/arm64/boot/dts/rockchip/rk3328.dtsi ++++ b/arch/arm64/boot/dts/rockchip/rk3328.dtsi +@@ -318,13 +318,13 @@ power: power-controller { + #address-cells = <1>; + #size-cells = <0>; + +- power-domain@RK3328_PD_HEVC { ++ pd_hevc@RK3328_PD_HEVC { + reg = ; + }; +- power-domain@RK3328_PD_VIDEO { ++ pd_video@RK3328_PD_VIDEO { + reg = ; + }; +- power-domain@RK3328_PD_VPU { ++ pd_vpu@RK3328_PD_VPU { + reg = ; + clocks = <&cru ACLK_VPU>, <&cru HCLK_VPU>; + }; +@@ -456,7 +456,7 @@ wdt: watchdog@ff1a0000 { + compatible = "snps,dw-wdt"; + reg = <0x0 0xff1a0000 0x0 0x100>; + interrupts = ; +- clocks = <&cru PCLK_WDT>; ++ clocks = <&cru PCLK_BUS_PRE>; + }; + + pwm0: pwm@ff1b0000 { +@@ -464,7 +464,7 @@ pwm0: pwm@ff1b0000 { + reg = <0x0 0xff1b0000 0x0 0x10>; + clocks = <&cru SCLK_PWM>, <&cru PCLK_PWM>; + clock-names = "pwm", "pclk"; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm0_pin>; + #pwm-cells = <3>; + status = "disabled"; +@@ -475,7 +475,7 @@ pwm1: pwm@ff1b0010 { + reg = <0x0 0xff1b0010 0x0 0x10>; + clocks = <&cru SCLK_PWM>, <&cru PCLK_PWM>; + clock-names = "pwm", "pclk"; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm1_pin>; + #pwm-cells = <3>; + status = "disabled"; +@@ -486,7 +486,7 @@ pwm2: pwm@ff1b0020 { + reg = <0x0 0xff1b0020 0x0 0x10>; + clocks = <&cru SCLK_PWM>, <&cru PCLK_PWM>; + clock-names = "pwm", "pclk"; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm2_pin>; + #pwm-cells = <3>; + status = "disabled"; +@@ -498,7 +498,7 @@ pwm3: pwm@ff1b0030 { + interrupts = ; + clocks = <&cru SCLK_PWM>, <&cru PCLK_PWM>; + clock-names = "pwm", "pclk"; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwmir_pin>; + #pwm-cells = <3>; + status = "disabled"; +@@ -552,10 +552,9 @@ tsadc: tsadc@ff250000 { + assigned-clock-rates = <50000>; + clocks = <&cru SCLK_TSADC>, <&cru PCLK_TSADC>; + clock-names = "tsadc", "apb_pclk"; +- pinctrl-names = "init", "default", "sleep"; ++ pinctrl-names = "gpio", "otpout"; + pinctrl-0 = <&otp_pin>; + pinctrl-1 = <&otp_out>; +- pinctrl-2 = <&otp_pin>; + resets = <&cru SRST_TSADC>; + reset-names = "tsadc-apb"; + rockchip,grf = <&grf>; +@@ -984,25 +983,6 @@ usb_host0_ohci: usb@ff5d0000 { + status = "disabled"; + }; + +- usbdrd3: usb@ff600000 { +- compatible = "rockchip,rk3328-dwc3", "snps,dwc3"; +- reg = <0x0 0xff600000 0x0 0x100000>; +- interrupts = ; +- clocks = <&cru SCLK_USB3OTG_REF>, <&cru SCLK_USB3OTG_SUSPEND>, +- <&cru ACLK_USB3OTG>; +- clock-names = "ref_clk", "suspend_clk", +- "bus_clk"; +- dr_mode = "otg"; +- phy_type = "utmi_wide"; +- snps,dis-del-phy-power-chg-quirk; +- snps,dis_enblslpm_quirk; +- snps,dis-tx-ipgap-linecheck-quirk; +- snps,dis-u2-freeclk-exists-quirk; +- snps,dis_u2_susphy_quirk; +- snps,dis_u3_susphy_quirk; +- status = "disabled"; +- }; +- + gic: interrupt-controller@ff811000 { + compatible = "arm,gic-400"; + #interrupt-cells = <3>; +diff --git a/arch/arm64/boot/dts/rockchip/rk3368-808-evb.dts b/arch/arm64/boot/dts/rockchip/rk3368-808-evb.dts +new file mode 100755 +index 000000000000..7e803a8ed30c +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3368-808-evb.dts +@@ -0,0 +1,189 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++#include "rk3368-808.dtsi" ++ ++/ { ++ model = "Rockchip rk3368 808 evb board"; ++ compatible = "rockchip,rk3368-808-evb", "rockchip,rk3368"; ++}; ++ ++&chosen { ++ bootargs = "earlycon=uart8250,mmio32,0xff690000 console=ttyFIQ0 androidboot.baseband=N/A androidboot.veritymode=enforcing androidboot.hardware=rk30board androidboot.console=ttyFIQ0 androidboot.selinux=permissive init=/init kpti=0"; ++}; ++ ++&fiq_debugger { ++ status = "okay"; ++}; ++ ++&cif { ++ status = "disabled"; ++}; ++ ++&cif_clkout { ++ /* cif_clkout */ ++ rockchip,pins = <1 RK_PB3 1 &pcfg_pull_none_4ma>; ++}; ++ ++&dmc { ++ vop-dclk-mode = <1>; ++ status = "okay"; ++}; ++ ++&isp_dvp_d2d9 { ++ rockchip,pins = ++ /* cif_data4 ... cif_data9 */ ++ <1 RK_PA2 1 &pcfg_pull_down>, ++ <1 RK_PA3 1 &pcfg_pull_down>, ++ <1 RK_PA4 1 &pcfg_pull_down>, ++ <1 RK_PA5 1 &pcfg_pull_down>, ++ <1 RK_PA6 1 &pcfg_pull_down>, ++ <1 RK_PA7 1 &pcfg_pull_down>, ++ /* cif_sync, cif_href */ ++ <1 RK_PB0 1 &pcfg_pull_down>, ++ <1 RK_PB1 1 &pcfg_pull_down>, ++ /* cif_clkin */ ++ <1 RK_PB2 1 &pcfg_pull_down>; ++}; ++ ++&isp_dvp_d10d11 { ++ rockchip,pins = ++ /* cif_data10, cif_data11 */ ++ <1 RK_PB6 1 &pcfg_pull_down>, ++ <1 RK_PB7 1 &pcfg_pull_down>; ++}; ++ ++&i2c3 { ++ status = "okay"; ++ ++ gc2145: gc2145@3c { ++ compatible = "galaxycore,gc2145"; ++ reg = <0x3c>; ++ clocks = <&cru SCLK_VIP_OUT>; ++ clock-names = "xvclk"; ++ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&isp_dvp_d2d9 &isp_dvp_d10d11 &cif_clkout>; ++ power-gpio = <&gpio0 RK_PD4 GPIO_ACTIVE_HIGH>; ++ pwdn-gpios = <&gpio1 RK_PB4 GPIO_ACTIVE_HIGH>; ++ rockchip,camera-module-index = <1>; ++ rockchip,camera-module-facing = "front"; ++ rockchip,camera-module-name = "CameraKing"; ++ rockchip,camera-module-lens-name = "Largan"; ++ port { ++ gc2145_out: endpoint { ++ remote-endpoint = <&isp_dvp_in>; ++ }; ++ }; ++ }; ++ ++ ov5695: ov5695@36 { ++ compatible = "ovti,ov5695"; ++ reg = <0x36>; ++ clocks = <&cru SCLK_VIP_OUT>; ++ clock-names = "xvclk"; ++ pwdn-gpios = <&gpio1 13 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_clkout>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "TongJu"; ++ rockchip,camera-module-lens-name = "CHT842-MD"; ++ port { ++ ov5695_out: endpoint { ++ remote-endpoint = <&mipi_in>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++ ++}; ++ ++&isp { ++ status = "disabled"; ++}; ++ ++&isp_mmu { ++ status = "okay"; ++}; ++ ++&mipi_dphy_rx0 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ov5695_out>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_rx_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp_mipi_in>; ++ }; ++ }; ++ }; ++}; ++ ++&pinctrl { ++ pcfg_pull_none_4ma: pcfg-pull-none-4ma { ++ bias-disable; ++ drive-strength = <4>; ++ }; ++}; ++ ++&rkisp1 { ++ status = "okay"; ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp_dvp_in: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&gc2145_out>; ++ }; ++ ++ isp_mipi_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dphy_rx_out>; ++ }; ++ ++ }; ++}; ++ ++/* ++ * In sleep mode, should be close vcca_33 and vcc_lan, ++ * but due to small defects in hardware settings and ++ * the system sleeps and wakes up, 4g module can not disconnect the network, ++ * so system sleep mode cannot be turned off vcca_33 and vcc_lan. ++ * This configuration will result in increased power consumption, ++ * please configure according to the actual needs of the project. ++ */ ++&vcca_33 { ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++}; ++ ++&vcc_lan { ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3368-808.dtsi b/arch/arm64/boot/dts/rockchip/rk3368-808.dtsi +new file mode 100755 +index 000000000000..fdea0c73932e +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3368-808.dtsi +@@ -0,0 +1,982 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++#include ++#include ++#include ++#include "rk3368.dtsi" ++#include "rk3368-android.dtsi" ++ ++/ { ++ rt5640-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,rt5640-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,widgets = ++ "Microphone", "Mic Jack", ++ "Headphone", "Headphone Jack"; ++ simple-audio-card,routing = ++ "Mic Jack", "MICBIAS1", ++ "IN1P", "Mic Jack", ++ "Headphone Jack", "HPOL", ++ "Headphone Jack", "HPOR"; ++ simple-audio-card,dai-link@0 { ++ format = "i2s"; ++ cpu { ++ sound-dai = <&i2s_8ch>; ++ }; ++ codec { ++ sound-dai = <&rt5640>; ++ }; ++ }; ++ ++ simple-audio-card,dai-link@1 { ++ format = "i2s"; ++ cpu { ++ sound-dai = <&i2s_8ch>; ++ }; ++ codec { ++ sound-dai = <&hdmi>; ++ }; ++ }; ++ }; ++ ++ rk_headset { ++ compatible = "rockchip_headset"; ++ headset_gpio = <&gpio2 RK_PC3 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hp_det>; ++ io-channels = <&saradc 2>; ++ }; ++ ++ ext_gmac: gmac-clk { ++ compatible = "fixed-clock"; ++ clock-frequency = <125000000>; ++ clock-output-names = "ext_gmac"; ++ #clock-cells = <0>; ++ }; ++ ++ vcc_phy: vcc-phy-regulator { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_phy"; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ vcc_camera: vcc-camera-regulator { ++ compatible = "regulator-fixed"; ++ gpio = <&gpio0 RK_PB2 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&camera_pwr>; ++ regulator-name = "vcc_camera"; ++ enable-active-high; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ vcc_lcd: vcc-lcd-regulator { ++ compatible = "regulator-fixed"; ++ gpio = <&gpio3 RK_PD5 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&lcd_pwr>; ++ regulator-name = "vcc_lcd"; ++ enable-active-high; ++ regulator-boot-on; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_otg_vbus: otg-vbus-regulator { ++ compatible = "regulator-fixed"; ++ gpio = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&otg_vbus_drv>; ++ regulator-name = "vcc_otg_vbus"; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ enable-active-high; ++ }; ++ ++ backlight: backlight { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm3 0 25000 0>; ++ brightness-levels = < ++ 0 1 2 3 4 5 6 7 ++ 8 9 10 11 12 13 14 15 ++ 16 17 18 19 20 21 22 23 ++ 24 25 26 27 28 29 30 31 ++ 32 33 34 35 36 37 38 39 ++ 40 41 42 43 44 45 46 47 ++ 48 49 50 51 52 53 54 55 ++ 56 57 58 59 60 61 62 63 ++ 64 65 66 67 68 69 70 71 ++ 72 73 74 75 76 77 78 79 ++ 80 81 82 83 84 85 86 87 ++ 88 89 90 91 92 93 94 95 ++ 96 97 98 99 100 101 102 103 ++ 104 105 106 107 108 109 110 111 ++ 112 113 114 115 116 117 118 119 ++ 120 121 122 123 124 125 126 127 ++ 128 129 130 131 132 133 134 135 ++ 136 137 138 139 140 141 142 143 ++ 144 145 146 147 148 149 150 151 ++ 152 153 154 155 156 157 158 159 ++ 160 161 162 163 164 165 166 167 ++ 168 169 170 171 172 173 174 175 ++ 176 177 178 179 180 181 182 183 ++ 184 185 186 187 188 189 190 191 ++ 192 193 194 195 196 197 198 199 ++ 200 201 202 203 204 205 206 207 ++ 208 209 210 211 212 213 214 215 ++ 216 217 218 219 220 221 222 223 ++ 224 225 226 227 228 229 230 231 ++ 232 233 234 235 236 237 238 239 ++ 240 241 242 243 244 245 246 247 ++ 248 249 250 251 252 253 254 255>; ++ default-brightness-level = <200>; ++ }; ++ ++ panel { ++ compatible = "samsung,lsl070nl01", "simple-panel"; ++ backlight = <&backlight>; ++ enable-gpios = <&gpio3 RK_PD5 GPIO_ACTIVE_HIGH>; ++ enable-delay-ms = <120>; ++ prepare-delay-ms = <2>; ++ unprepare-delay-ms = <20>; ++ disable-delay-ms = <50>; ++ width-mm = <68>; ++ height-mm = <121>; ++ rockchip,data-mapping = "vesa"; ++ rockchip,data-width = <24>; ++ rockchip,output = "lvds"; ++ status = "disabled"; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <160000000>; ++ hactive = <1200>; ++ vactive = <1920>; ++ hback-porch = <60>; ++ hfront-porch = <80>; ++ vback-porch = <25>; ++ vfront-porch = <35>; ++ hsync-len = <1>; ++ vsync-len = <1>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ port { ++ panel_in: endpoint { ++ remote-endpoint = <&edp_out>; ++ }; ++ }; ++ }; ++ ++ charge-animation { ++ compatible = "rockchip,uboot-charge"; ++ rockchip,uboot-charge-on = <1>; ++ rockchip,android-charge-on = <0>; ++ rockchip,uboot-low-power-voltage = <3500>; ++ rockchip,screen-on-voltage = <3600>; ++ status = "okay"; ++ }; ++ ++ gpio_keys: gpio-keys { ++ compatible = "gpio-keys"; ++ autorepeat; ++ ++ power { ++ debounce-interval = <100>; ++ gpios = <&gpio0 2 GPIO_ACTIVE_LOW>; ++ label = "GPIO Key Power"; ++ linux,code = ; ++ wakeup-source; ++ }; ++ }; ++ ++ adc_keys: adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 1>; ++ io-channel-names = "buttons"; ++ keyup-threshold-microvolt = <1024000>; ++ poll-interval = <100>; ++ ++ vol-up-key { ++ label = "volume up"; ++ linux,code = ; ++ press-threshold-microvolt = <1000>; ++ }; ++ ++ vol-down-key { ++ label = "volume down"; ++ linux,code = ; ++ press-threshold-microvolt = <170000>; ++ }; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ clocks = <&rk808 1>; ++ clock-names = "ext_clock"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio3 RK_PA4 GPIO_ACTIVE_LOW>; /* GPIO3_A4 */ ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ rockchip,grf = <&grf>; ++ wifi_chip_type = "ap6255"; ++ sdio_vref = <1800>; //1800mv or 3300mv ++ WIFI,host_wake_irq = <&gpio3 6 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&rk808 1>; ++ clock-names = "ext_clock"; ++ uart_rts_gpios = <&gpio2 27 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default","rts_gpio"; ++ pinctrl-0 = <&uart0_rts>; ++ pinctrl-1 = <&uart0_rts_gpio>; ++ ++ /* BT,power_gpio = <&gpio3 3 GPIO_ACTIVE_HIGH>; */ ++ BT,reset_gpio = <&gpio3 5 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio3 2 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio3 7 GPIO_ACTIVE_HIGH>; ++ ++ status = "okay"; ++ }; ++ ++ rk_modem: rk-modem { ++ compatible="4g-modem-platdata"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <<e_vbat <e_power_en <e_reset>; ++ 4G,vbat-gpio = <&gpio0 RK_PA4 GPIO_ACTIVE_HIGH>; ++ 4G,power-gpio = <&gpio0 RK_PC0 GPIO_ACTIVE_LOW>; ++ 4G,reset-gpio = <&gpio0 RK_PB7 GPIO_ACTIVE_LOW>; ++ status = "okay"; ++ }; ++ ++ vcc_sys: vcc-sys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ }; ++ ++ xin32k: xin32k { ++ compatible = "fixed-clock"; ++ clock-frequency = <32768>; ++ clock-output-names = "xin32k"; ++ #clock-cells = <0>; ++ }; ++}; ++ ++&cpu_l0 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu_l1 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu_l2 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu_l3 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu_b0 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu_b1 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu_b2 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu_b3 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&gpu { ++ logic-supply = <&vdd_log>; ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&dmc { ++ status = "okay"; ++ center-supply = <&vdd_log>; ++ devfreq-events = <&dfi>; ++ upthreshold = <60>; ++ downdifferential = <20>; ++ system-status-freq = < ++ /*system status freq(KHz)*/ ++ SYS_STATUS_NORMAL 600000 ++ SYS_STATUS_REBOOT 600000 ++ SYS_STATUS_SUSPEND 240000 ++ SYS_STATUS_VIDEO_1080P 396000 ++ SYS_STATUS_VIDEO_4K 600000 ++ SYS_STATUS_PERFORMANCE 600000 ++ SYS_STATUS_BOOST 396000 ++ SYS_STATUS_DUALVIEW 600000 ++ SYS_STATUS_ISP 528000 ++ >; ++ vop-bw-dmc-freq = < ++ /* min_bw(MB/s) max_bw(MB/s) freq(KHz) */ ++ 0 582 240000 ++ 583 99999 396000 ++ >; ++ auto-min-freq = <240000>; ++ auto-freq-en = <0>; ++}; ++ ++&rockchip_suspend { ++ status = "okay"; ++ rockchip,sleep-mode-config = < ++ (0 ++ | RKPM_SLP_ARMOFF ++ | RKPM_SLP_PMU_PLLS_PWRDN ++ | RKPM_SLP_PMU_PMUALIVE_32K ++ | RKPM_SLP_SFT_PLLS_DEEP ++ | RKPM_SLP_PMU_DIS_OSC ++ | RKPM_SLP_SFT_PD_NBSCUS ++ ) ++ >; ++ rockchip,wakeup-config = < ++ (0 ++ | RKPM_GPIO_WKUP_EN ++ | RKPM_USB_WKUP_EN ++ | RKPM_CLUSTER_L_WKUP_EN ++ ) ++ >; ++}; ++ ++&emmc { ++ bus-width = <8>; ++ cap-mmc-highspeed; ++ mmc-hs200-1_8v; ++ no-sdio; ++ no-sd; ++ disable-wp; ++ non-removable; ++ num-slots = <1>; ++ status = "okay"; ++}; ++ ++&nandc0 { ++ status = "okay"; ++}; ++ ++&sdmmc { ++ clock-frequency = <37500000>; ++ clock-freq-min-max = <400000 37500000>; ++ no-sdio; ++ no-mmc; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ card-detect-delay = <200>; ++ disable-wp; ++ num-slots = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_cd &sdmmc_bus4>; ++ status = "disabled"; ++}; ++ ++&sdio0 { ++ clock-frequency = <100000000>; ++ clock-freq-min-max = <200000 100000000>; ++ no-sd; ++ no-mmc; ++ bus-width = <4>; ++ disable-wp; ++ cap-sd-highspeed; ++ cap-sdio-irq; ++ keep-power-in-suspend; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ non-removable; ++ num-slots = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ ++ rk808: pmic@1b { ++ status = "okay"; ++ compatible = "rockchip,rk808"; ++ reg = <0x1b>; ++ interrupt-parent = <&gpio0>; ++ interrupts = <1 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pmic_int>, <&pmic_sleep>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ vcc1-supply = <&vcc_sys>; ++ vcc2-supply = <&vcc_sys>; ++ vcc3-supply = <&vcc_sys>; ++ vcc4-supply = <&vcc_sys>; ++ vcc6-supply = <&vcc_sys>; ++ vcc7-supply = <&vcc_sys>; ++ vcc8-supply = <&vcc_io>; ++ vcc9-supply = <&vcc_sys>; ++ vcc10-supply = <&vcc_sys>; ++ vcc11-supply = <&vcc_sys>; ++ vcc12-supply = <&vcc_io>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ #clock-cells = <1>; ++ ++ regulators { ++ vdd_cpu: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <700000>; ++ regulator-max-microvolt = <1500000>; ++ regulator-name = "vdd_cpu"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdd_log: DCDC_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <700000>; ++ regulator-max-microvolt = <1500000>; ++ regulator-name = "vdd_log"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc_ddr"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_io: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcc_io"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc18_flash: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc18_flash"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vcca_33: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcca_33"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdd_10: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ regulator-name = "vdd_10"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcca_18: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcca_18"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vccio_sd: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vccio_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vdd10_lcd: LDO_REG6 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ regulator-name = "vdd10_lcd"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc_18: LDO_REG7 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc_18"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vcc18_lcd: LDO_REG8 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc18_lcd"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vcc_sd: SWITCH_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_lan: SWITCH_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc_lan"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ }; ++ }; ++ ++}; ++ ++&i2c1 { ++ status = "okay"; ++ ++ rt5640: rt5640@1c { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++ compatible = "realtek,rt5640"; ++ reg = <0x1c>; ++ clocks = <&cru SCLK_I2S_8CH_OUT>; ++ clock-names = "mclk"; ++ realtek,in1-differential; ++ /* spk-con-gpio = <&gpio3 9 GPIO_ACTIVE_HIGH>; */ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s_8ch_mclk>; ++ }; ++ ++ mpu6500_acc: mpu_acc@68 { ++ status = "okay"; ++ compatible = "mpu6500_acc"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&mpu6500_irq_gpio>; ++ reg = <0x68>; ++ irq-gpio = <&gpio2 17 IRQ_TYPE_LEVEL_LOW>; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ type = ; ++ power-off-in-suspend = <1>; ++ layout = <5>; ++ ++ }; ++ ++ mpu6500_gyro: mpu_gyro@68 { ++ status = "okay"; ++ compatible = "mpu6500_gyro"; ++ reg = <0x68>; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ type = ; ++ power-off-in-suspend = <1>; ++ layout = <5>; ++ }; ++ ++ ak8963_compass: ak8963_compass@d { ++ status = "okay"; ++ compatible = "ak8963"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&ak8963_irq_gpio>; ++ reg = <0x0d>; ++ type = ; ++ irq-gpio = <&gpio2 18 IRQ_TYPE_EDGE_RISING>; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ layout = <7>; ++ }; ++}; ++ ++&i2c2 { ++ status = "okay"; ++ ++ gslx680@40 { ++ compatible = "gslX6801"; ++ reg = <0x40>; ++ screen_max_x = <1920>; ++ screen_max_y = <1200>; ++ power-supply = <&vcc_lcd>; ++ touch-gpio = <&gpio0 12 IRQ_TYPE_LEVEL_LOW>; ++ reset-gpio = <&gpio0 11 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++}; ++ ++&i2c3 { ++ status = "okay"; ++}; ++ ++&i2s_8ch { ++ status = "okay"; ++ rockchip,i2s-broken-burst-len; ++ rockchip,playback-channels = <8>; ++ rockchip,capture-channels = <2>; ++ #sound-dai-cells = <0>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s_2ch_bus>; ++}; ++ ++&io_domains { ++ status = "okay"; ++ ++ audio-supply = <&vcca_18>; ++ dvp-supply = <&vcc_18>; ++ flash0-supply = <&vcc18_flash>; ++ gpio30-supply = <&vcc_io>; ++ gpio1830-supply = <&vcc_io>; ++ sdcard-supply = <&vccio_sd>; ++ wifi-supply = <&vcc_io>; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ ++ pmu-supply = <&vcc_io>; ++ vop-supply = <&vcca_33>; ++}; ++ ++&pwm0 { ++ status = "okay"; ++}; ++ ++&pwm3 { ++ status = "okay"; ++}; ++ ++&uart0 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart0_xfer &uart0_cts>; ++ status = "okay"; ++}; ++ ++&uart2 { ++ status = "disabled"; ++}; ++ ++&saradc { ++ status = "okay"; ++}; ++ ++&u2phy { ++ status = "okay"; ++ ++ u2phy_otg: otg-port { ++ status = "okay"; ++ vbus-supply = <&vcc_otg_vbus>; ++ }; ++ ++ u2phy_host: host-port { ++ status = "okay"; ++ }; ++}; ++ ++&usb_host0_ehci { ++ status = "okay"; ++}; ++ ++&usb_host0_ohci { ++ status = "okay"; ++}; ++ ++&edp { ++ status = "disabled"; ++ force-hpd; ++ ++ ports { ++ port@1 { ++ reg = <1>; ++ ++ edp_out: endpoint { ++ remote-endpoint = <&panel_in>; ++ }; ++ }; ++ }; ++}; ++ ++&dsi { ++ status = "okay"; ++ ++ panel@0 { ++ compatible = "sitronix,st7703", "simple-panel-dsi"; ++ reg = <0>; ++ backlight = <&backlight>; ++ power-supply = <&vcc_lcd>; ++ prepare-delay-ms = <2>; ++ reset-delay-ms = <1>; ++ init-delay-ms = <20>; ++ enable-delay-ms = <120>; ++ disable-delay-ms = <50>; ++ unprepare-delay-ms = <20>; ++ ++ width-mm = <68>; ++ height-mm = <121>; ++ ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | ++ MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ ++ display-timings { ++ native-mode = <&st7703_timing>; ++ ++ st7703_timing: timing0 { ++ clock-frequency = <160000000>; ++ hactive = <1200>; ++ vactive = <1920>; ++ hback-porch = <60>; ++ hfront-porch = <80>; ++ vback-porch = <25>; ++ vfront-porch = <35>; ++ hsync-len = <1>; ++ vsync-len = <1>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ panel_in_dsi: endpoint { ++ remote-endpoint = <&dsi_out_panel>; ++ }; ++ }; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@1 { ++ reg = <1>; ++ dsi_out_panel: endpoint { ++ remote-endpoint = <&panel_in_dsi>; ++ }; ++ }; ++ }; ++}; ++ ++&video_phy { ++ status = "okay"; ++}; ++ ++&route_dsi { ++ status = "okay"; ++}; ++ ++&tsadc { ++ tsadc-supply = <&vdd_cpu>; ++ status = "okay"; ++}; ++ ++&gmac { ++ phy-supply = <&vcc_phy>; ++ phy-mode = "rgmii"; ++ clock_in_out = "input"; ++ snps,reset-gpio = <&gpio3 11 GPIO_ACTIVE_LOW>; ++ snps,reset-active-low; ++ snps,reset-delays-us = <0 10000 50000>; ++ assigned-clocks = <&cru SCLK_MAC>; ++ assigned-clock-parents = <&ext_gmac>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&rgmii_pins>; ++ tx_delay = <0x28>; ++ rx_delay = <0x11>; ++ status = "okay"; ++}; ++ ++&hdmi { ++ #sound-dai-cells = <0>; ++ status = "okay"; ++}; ++ ++&route_hdmi { ++ status = "okay"; ++}; ++ ++&pinctrl { ++ camera { ++ camera_pwr: camera-pwr { ++ rockchip,pins = <0 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ lcd { ++ lcd_pwr: lcd-pwr { ++ rockchip,pins = <3 RK_PD5 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ headphone { ++ hp_det: hp-det { ++ rockchip,pins = <2 RK_PC3 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ i2s { ++ i2s_2ch_bus: i2s-2ch-bus { ++ rockchip,pins = <2 RK_PB4 1 &pcfg_pull_none>, ++ <2 RK_PB5 1 &pcfg_pull_none>, ++ <2 RK_PB6 1 &pcfg_pull_none>, ++ <2 RK_PB7 1 &pcfg_pull_none>, ++ <2 RK_PC0 1 &pcfg_pull_none>; ++ }; ++ }; ++ ++ pmic { ++ pmic_sleep: pmic-sleep { ++ rockchip,pins = <0 RK_PA0 2 &pcfg_pull_none>; ++ }; ++ ++ pmic_int: pmic-int { ++ rockchip,pins = <0 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ mpu6500 { ++ mpu6500_irq_gpio: mpu6500-irq-gpio { ++ rockchip,pins = <2 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ ak8963 { ++ ak8963_irq_gpio: ak8963_irq_gpio { ++ rockchip,pins = <2 RK_PC2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ dc_det { ++ dc_irq_gpio: dc-irq-gpio { ++ rockchip,pins = <0 RK_PC1 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ usb { ++ otg_vbus_drv: otg-vbus-drv { ++ rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart0_rts_gpio: uart0-rts-gpio { ++ rockchip,pins = <2 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ rk-modem { ++ lte_vbat: lte-vbat { ++ rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ lte_power_en: lte-power-en { ++ rockchip,pins = <0 RK_PC0 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ lte_reset: lte-reset { ++ rockchip,pins = <0 RK_PB7 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ +diff --git a/arch/arm64/boot/dts/rockchip/rk3368-android.dtsi b/arch/arm64/boot/dts/rockchip/rk3368-android.dtsi +new file mode 100755 +index 000000000000..a3e0bbaaf7c5 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3368-android.dtsi +@@ -0,0 +1,357 @@ ++/* ++ * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This file is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This file is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++/ { ++ chosen: chosen { ++ bootargs = "earlycon=uart8250,mmio32,0xff690000 firmware_class.path=/system/vendor/firmware"; ++ }; ++ ++ fiq_debugger: fiq-debugger { ++ compatible = "rockchip,fiq-debugger"; ++ rockchip,serial-id = <2>; ++ rockchip,wake-irq = <0>; ++ rockchip,irq-mode-enable = <0>; /* If enable uart uses irq instead of fiq */ ++ rockchip,baudrate = <115200>; /* Only 115200 and 1500000 */ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart2_xfer>; ++ interrupts = ; /* signal irq */ ++ }; ++ ++ reserved-memory { ++ #address-cells = <2>; ++ #size-cells = <2>; ++ ranges; ++ ++ drm_logo: drm-logo@00000000 { ++ compatible = "rockchip,drm-logo"; ++ reg = <0x0 0x0 0x0 0x0>; ++ }; ++ ++ ramoops: ramoops@110000 { ++ compatible = "ramoops"; ++ reg = <0x0 0x110000 0x0 0xf0000>; ++ record-size = <0x20000>; ++ console-size = <0x80000>; ++ ftrace-size = <0x00000>; ++ pmsg-size = <0x50000>; ++ }; ++ ++ /* global autoconfigured region for contiguous allocations */ ++ linux,cma { ++ compatible = "shared-dma-pool"; ++ reusable; ++ size = <0x0 0x2000000>; ++ linux,cma-default; ++ }; ++ }; ++ ++ ion { ++ compatible = "rockchip,ion"; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ cma-heap { ++ reg = <0x00000000 0x2800000>; ++ }; ++ ++ system-heap { ++ }; ++ }; ++ ++ firmware { ++ firmware_android: android {}; ++ }; ++ ++ rga@ff920000 { ++ compatible = "rockchip,rga2"; ++ dev_mode = <1>; ++ reg = <0x0 0xff920000 0x0 0x1000>; ++ interrupts = ; ++ clocks = <&cru ACLK_RGA>, <&cru HCLK_RGA>, <&cru SCLK_RGA>; ++ clock-names = "aclk_rga", "hclk_rga", "clk_rga"; ++ status = "okay"; ++ }; ++}; ++ ++&cluster1_opp { ++ rockchip,avs = <1>; ++}; ++ ++&display_subsystem { ++ status = "okay"; ++ ++ logo-memory-region = <&drm_logo>; ++ route { ++ route_dsi: route-dsi { ++ status = "disabled"; ++ logo,uboot = "logo.bmp"; ++ logo,kernel = "logo_kernel.bmp"; ++ logo,mode = "center"; ++ charge_logo,mode = "center"; ++ connect = <&vop_out_dsi>; ++ }; ++ ++ route_edp: route-edp { ++ status = "disabled"; ++ logo,uboot = "logo.bmp"; ++ logo,kernel = "logo_kernel.bmp"; ++ logo,mode = "center"; ++ charge_logo,mode = "center"; ++ connect = <&vop_out_edp>; ++ }; ++ ++ route_hdmi: route-hdmi { ++ status = "disabled"; ++ logo,uboot = "logo.bmp"; ++ logo,kernel = "logo_kernel.bmp"; ++ logo,mode = "center"; ++ charge_logo,mode = "center"; ++ connect = <&vop_out_hdmi>; ++ }; ++ ++ route_lvds: route-lvds { ++ status = "disabled"; ++ logo,uboot = "logo.bmp"; ++ logo,kernel = "logo_kernel.bmp"; ++ logo,mode = "center"; ++ charge_logo,mode = "center"; ++ connect = <&vop_out_lvds>; ++ }; ++ ++ route_rgb: route-rgb { ++ status = "disabled"; ++ logo,uboot = "logo.bmp"; ++ logo,kernel = "logo_kernel.bmp"; ++ logo,mode = "center"; ++ charge_logo,mode = "center"; ++ connect = <&vop_out_rgb>; ++ }; ++ ++ }; ++}; ++ ++&dsi { ++ panel@0 { ++ reg = <0>; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ ++ panel_in_dsi: endpoint { ++ remote-endpoint = <&dsi_out_panel>; ++ }; ++ }; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@1 { ++ reg = <1>; ++ ++ dsi_out_panel: endpoint { ++ remote-endpoint = <&panel_in_dsi>; ++ }; ++ }; ++ }; ++}; ++ ++&hevc { ++ status = "okay"; ++}; ++ ++&hevc_mmu { ++ status = "okay"; ++}; ++ ++&iep { ++ status = "okay"; ++}; ++ ++&iep_mmu { ++ status = "okay"; ++}; ++ ++&mailbox { ++ status = "okay"; ++}; ++ ++&mailbox_scpi { ++ status = "okay"; ++}; ++ ++&mpp_srv { ++ status = "okay"; ++}; ++ ++&vdpu { ++ status = "okay"; ++}; ++ ++&vepu { ++ status = "okay"; ++}; ++ ++&vpu_mmu { ++ status = "okay"; ++}; ++ ++&vop { ++ support-multi-area; ++ status = "okay"; ++}; ++ ++&vop_mmu { ++ status = "okay"; ++}; ++ ++&isp { ++ status = "okay"; ++}; ++ ++&isp_mmu { ++ status = "okay"; ++}; ++ ++&cif { ++ status = "okay"; ++}; ++ ++&rng { ++ status = "okay"; ++}; ++ ++&vip_mmu { ++ status = "okay"; ++}; ++ ++&video_phy { ++ status = "okay"; ++}; ++ ++&usb_otg { ++ status = "okay"; ++}; ++ ++&pinctrl { ++ isp { ++ cif_clkout: cif-clkout { ++ rockchip,pins = <1 RK_PB3 1 &pcfg_pull_none>;//cif_clkout ++ }; ++ ++ isp_dvp_d2d9: isp-dvp-d2d9 { ++ rockchip,pins = ++ <1 RK_PA0 1 &pcfg_pull_none>,//cif_data2 ++ <1 RK_PA1 1 &pcfg_pull_none>,//cif_data3 ++ <1 RK_PA2 1 &pcfg_pull_none>,//cif_data4 ++ <1 RK_PA3 1 &pcfg_pull_none>,//cif_data5 ++ <1 RK_PA4 1 &pcfg_pull_none>,//cif_data6 ++ <1 RK_PA5 1 &pcfg_pull_none>,//cif_data7 ++ <1 RK_PA6 1 &pcfg_pull_none>,//cif_data8 ++ <1 RK_PA7 1 &pcfg_pull_none>,//cif_data9 ++ <1 RK_PB0 1 &pcfg_pull_none>,//cif_sync ++ <1 RK_PB1 1 &pcfg_pull_none>,//cif_href ++ <1 RK_PB2 1 &pcfg_pull_none>,//cif_clkin ++ <1 RK_PB3 1 &pcfg_pull_none>;//cif_clkout ++ }; ++ ++ isp_dvp_d0d1: isp-dvp-d0d1 { ++ rockchip,pins = ++ <1 RK_PB4 1 &pcfg_pull_none>,//cif_data0 ++ <1 RK_PB5 1 &pcfg_pull_none>;//cif_data1 ++ }; ++ ++ isp_dvp_d10d11:isp_d10d11 { ++ rockchip,pins = ++ <1 RK_PB6 1 &pcfg_pull_none>,//cif_data10 ++ <1 RK_PB7 1 &pcfg_pull_none>;//cif_data11 ++ }; ++ ++ isp_dvp_d0d7: isp-dvp-d0d7 { ++ rockchip,pins = ++ <1 RK_PB4 1 &pcfg_pull_none>,//cif_data0 ++ <1 RK_PB5 1 &pcfg_pull_none>,//cif_data1 ++ <1 RK_PA0 1 &pcfg_pull_none>,//cif_data2 ++ <1 RK_PA1 1 &pcfg_pull_none>,//cif_data3 ++ <1 RK_PA2 1 &pcfg_pull_none>,//cif_data4 ++ <1 RK_PA3 1 &pcfg_pull_none>,//cif_data5 ++ <1 RK_PA4 1 &pcfg_pull_none>,//cif_data6 ++ <1 RK_PA5 1 &pcfg_pull_none>;//cif_data7 ++ }; ++ ++ isp_dvp_d4d11: isp-dvp-d4d11 { ++ rockchip,pins = ++ <1 RK_PA2 1 &pcfg_pull_none>,//cif_data4 ++ <1 RK_PA3 1 &pcfg_pull_none>,//cif_data5 ++ <1 RK_PA4 1 &pcfg_pull_none>,//cif_data6 ++ <1 RK_PA5 1 &pcfg_pull_none>,//cif_data7 ++ <1 RK_PA6 1 &pcfg_pull_none>,//cif_data8 ++ <1 RK_PA7 1 &pcfg_pull_none>,//cif_data9 ++ <1 RK_PB6 1 &pcfg_pull_none>,//cif_data10 ++ <1 RK_PC1 1 &pcfg_pull_none>;//cif_data11 ++ }; ++ ++ isp_shutter: isp-shutter { ++ rockchip,pins = ++ <3 RK_PC3 2 &pcfg_pull_none>, //SHUTTEREN ++ <3 RK_PC6 2 &pcfg_pull_none>;//SHUTTERTRIG ++ }; ++ ++ isp_flash_trigger: isp-flash-trigger { ++ rockchip,pins = <3 RK_PC4 2 &pcfg_pull_none>; //ISP_FLASHTRIGOU ++ }; ++ ++ isp_prelight: isp-prelight { ++ rockchip,pins = <3 RK_PC5 2 &pcfg_pull_none>;//ISP_PRELIGHTTRIG ++ }; ++ ++ isp_flash_trigger_as_gpio: isp_flash_trigger_as_gpio { ++ rockchip,pins = <3 RK_PC4 RK_FUNC_GPIO &pcfg_pull_none>;//ISP_FLASHTRIGOU ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3368-cif-sensor.dtsi b/arch/arm64/boot/dts/rockchip/rk3368-cif-sensor.dtsi +new file mode 100755 +index 000000000000..7d2aac22c020 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3368-cif-sensor.dtsi +@@ -0,0 +1,35 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++#include "../../../../../drivers/soc/rockchip/rk_camera_sensor_info.h" ++ ++/{ ++ cif_sensor: cif_sensor { ++ compatible = "rockchip,sensor"; ++ status = "okay"; ++ ++ tp2825 { ++ status = "okay"; ++ is_front = <0>; ++ powerdown-gpios = <&gpio1 12 GPIO_ACTIVE_HIGH>; ++ irq-gpios = <&gpio1 13 IRQ_TYPE_EDGE_FALLING>; ++ pwdn_active = ; ++ mir = <0>; ++ flash_attach = <1>; ++ flash_active = <1>; ++ resolution = ; ++ powerup_sequence = ; ++ orientation = <0>; ++ i2c_add = ; ++ i2c_chl = <3>; ++ cif_chl = <0>; ++ ad_chl = <0>; // 0 ~ 4; ++ mclk_rate = <24>; ++ rockchip,camera-module-defrect0 = <960 480 0 4 960 472>; ++ rockchip,camera-module-interface0 = "cvbs_ntsc"; ++ rockchip,camera-module-defrect1 = <960 576 0 4 960 568>; ++ rockchip,camera-module-interface1 = "cvbs_pal"; ++ rockchip,camera-module-defrect2 = <1280 720 8 20 1280 720>; ++ rockchip,camera-module-interface2 = "bt601_8_pp"; ++ rockchip,camera-module-channel = <4 0>; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3368-dram-default-timing.dtsi b/arch/arm64/boot/dts/rockchip/rk3368-dram-default-timing.dtsi +new file mode 100755 +index 000000000000..10bfbebda989 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3368-dram-default-timing.dtsi +@@ -0,0 +1,66 @@ ++/* ++ * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This library is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This library is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++#include ++#include ++ ++/ { ++ ddr_timing: ddr_timing { ++ compatible = "rockchip,ddr-timing"; ++ dram_spd_bin = ; ++ sr_idle = <1>; ++ pd_idle = <0x20>; ++ dram_dll_disb_freq = <300>; ++ phy_dll_disb_freq = <400>; ++ dram_odt_disb_freq = <333>; ++ phy_odt_disb_freq = <333>; ++ ddr3_drv = ; ++ ddr3_odt = ; ++ lpddr3_drv = ; ++ lpddr3_odt = ; ++ lpddr2_drv = ; /* lpddr2 not supported odt */ ++ phy_clk_drv = ; ++ phy_cmd_drv = ; ++ phy_dqs_drv = ; ++ phy_odt = ; ++ ddr_2t = ; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3368-p9-avb.dts b/arch/arm64/boot/dts/rockchip/rk3368-p9-avb.dts +new file mode 100755 +index 000000000000..6a971781032a +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3368-p9-avb.dts +@@ -0,0 +1,35 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++#include "rk3368-p9.dtsi" ++ ++/ { ++ model = "Rockchip rk3368 p9 avb board"; ++ compatible = "rockchip,p9-avb", "rockchip,rk3368"; ++}; ++ ++&firmware_android { ++ compatible = "android,firmware"; ++ boot_devices = "ff0f0000.dwmmc,ff400000.nandc"; ++ vbmeta { ++ compatible = "android,vbmeta"; ++ parts = "vbmeta,dtbo"; ++ }; ++ fstab { ++ compatible = "android,fstab"; ++ vendor { ++ compatible = "android,vendor"; ++ dev = "/dev/block/by-name/vendor"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,avb"; ++ }; ++ }; ++}; ++ ++&chosen { ++ bootargs = "earlycon=uart8250,mmio32,0xff690000 console=ttyFIQ0 androidboot.baseband=N/A androidboot.veritymode=enforcing androidboot.hardware=rk30board androidboot.console=ttyFIQ0 androidboot.selinux=permissive init=/init kpti=0"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3368-p9.dts b/arch/arm64/boot/dts/rockchip/rk3368-p9.dts +new file mode 100755 +index 000000000000..29658be08696 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3368-p9.dts +@@ -0,0 +1,34 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++#include "rk3368-p9.dtsi" ++ ++/ { ++ model = "Rockchip rk3368 p9 board"; ++ compatible = "rockchip,p9", "rockchip,rk3368"; ++}; ++ ++&firmware_android { ++ compatible = "android,firmware"; ++ fstab { ++ compatible = "android,fstab"; ++ system { ++ compatible = "android,system"; ++ dev = "/dev/block/by-name/system"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,verify"; ++ }; ++ vendor { ++ compatible = "android,vendor"; ++ dev = "/dev/block/by-name/vendor"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,verify"; ++ }; ++ }; ++}; ++ +diff --git a/arch/arm64/boot/dts/rockchip/rk3368-p9.dtsi b/arch/arm64/boot/dts/rockchip/rk3368-p9.dtsi +new file mode 100755 +index 000000000000..7eeb9db4f13b +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3368-p9.dtsi +@@ -0,0 +1,841 @@ ++/* ++ * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This file is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This file is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++/dts-v1/; ++#include ++#include ++#include "rk3368.dtsi" ++#include "rk3368-android.dtsi" ++ ++/ { ++ es8316-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,rk-es8316-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,widgets = ++ "Microphone", "Mic Jack", ++ "Headphone", "Headphone Jack"; ++ simple-audio-card,routing = ++ "Mic Jack", "MICBIAS1", ++ "IN1P", "Mic Jack", ++ "Headphone Jack", "HPOL", ++ "Headphone Jack", "HPOR"; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s_8ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&es8316>; ++ }; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ clocks = <&rk818 1>; ++ clock-names = "ext_clock"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio3 4 GPIO_ACTIVE_LOW>; /* GPIO3_A4 */ ++ }; ++ ++ backlight: backlight { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm0 0 25000 0>; ++ brightness-levels = < ++ 0 1 2 3 4 5 6 7 ++ 8 9 10 11 12 13 14 15 ++ 16 17 18 19 20 21 22 23 ++ 24 25 26 27 28 29 30 31 ++ 32 33 34 35 36 37 38 39 ++ 40 41 42 43 44 45 46 47 ++ 48 49 50 51 52 53 54 55 ++ 56 57 58 59 60 61 62 63 ++ 64 65 66 67 68 69 70 71 ++ 72 73 74 75 76 77 78 79 ++ 80 81 82 83 84 85 86 87 ++ 88 89 90 91 92 93 94 95 ++ 96 97 98 99 100 101 102 103 ++ 104 105 106 107 108 109 110 111 ++ 112 113 114 115 116 117 118 119 ++ 120 121 122 123 124 125 126 127 ++ 128 129 130 131 132 133 134 135 ++ 136 137 138 139 140 141 142 143 ++ 144 145 146 147 148 149 150 151 ++ 152 153 154 155 156 157 158 159 ++ 160 161 162 163 164 165 166 167 ++ 168 169 170 171 172 173 174 175 ++ 176 177 178 179 180 181 182 183 ++ 184 185 186 187 188 189 190 191 ++ 192 193 194 195 196 197 198 199 ++ 200 201 202 203 204 205 206 207 ++ 208 209 210 211 212 213 214 215 ++ 216 217 218 219 220 221 222 223 ++ 224 225 226 227 228 229 230 231 ++ 232 233 234 235 236 237 238 239 ++ 240 241 242 243 244 245 246 247 ++ 248 249 250 251 252 253 254 255>; ++ default-brightness-level = <200>; ++ enable-gpios = <&gpio0 20 GPIO_ACTIVE_HIGH>; ++ }; ++ ++ charge-animation { ++ compatible = "rockchip,uboot-charge"; ++ rockchip,uboot-charge-on = <1>; ++ rockchip,android-charge-on = <0>; ++ rockchip,uboot-low-power-voltage = <3500>; ++ rockchip,screen-on-voltage = <3600>; ++ status = "okay"; ++ }; ++ ++ rk_key: rockchip-key { ++ compatible = "rockchip,key"; ++ status = "okay"; ++ ++ io-channels = <&saradc 1>; ++ ++ vol-up-key { ++ linux,code = <115>; ++ label = "volume up"; ++ rockchip,adc_value = <1>; ++ }; ++ ++ vol-down-key { ++ linux,code = <114>; ++ label = "volume down"; ++ rockchip,adc_value = <170>; ++ }; ++ ++ power-key { ++ gpios = <&gpio0 2 GPIO_ACTIVE_LOW>; ++ linux,code = <116>; ++ label = "power"; ++ gpio-key,wakeup; ++ }; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ rockchip,grf = <&grf>; ++ /* wifi_chip_type - wifi chip define ++ * ap6210, ap6330, ap6335 ++ * rtl8188eu, rtl8723bs, rtl8723bu ++ * esp8089 ++ */ ++ wifi_chip_type = "ap6210"; ++ sdio_vref = <1800>; //1800mv or 3300mv ++ WIFI,host_wake_irq = <&gpio3 6 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&rk818 1>; ++ clock-names = "ext_clock"; ++ uart_rts_gpios = <&gpio2 27 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default","rts_gpio"; ++ pinctrl-0 = <&uart0_rts>; ++ pinctrl-1 = <&uart0_rts_gpio>; ++ ++ //BT,power_gpio = <&gpio3 3 GPIO_ACTIVE_HIGH>; ++ BT,reset_gpio = <&gpio3 5 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio3 2 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio3 7 GPIO_ACTIVE_HIGH>; ++ ++ status = "okay"; ++ }; ++ ++ vcc_sys: vcc-sys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3800000>; ++ regulator-max-microvolt = <3800000>; ++ }; ++ ++ vcc_host: vcc-host { ++ compatible = "regulator-fixed"; ++ enable-active-high; ++ gpio = <&gpio0 4 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&host_vbus_drv>; ++ regulator-name = "vcc_host"; ++ regulator-always-on; ++ }; ++ ++ xin32k: xin32k { ++ compatible = "fixed-clock"; ++ clock-frequency = <32768>; ++ clock-output-names = "xin32k"; ++ #clock-cells = <0>; ++ }; ++}; ++ ++&cpu_l0 { ++ cpu-supply = <&syr827>; ++}; ++ ++&cpu_l1 { ++ cpu-supply = <&syr827>; ++}; ++ ++&cpu_l2 { ++ cpu-supply = <&syr827>; ++}; ++ ++&cpu_l3 { ++ cpu-supply = <&syr827>; ++}; ++ ++&cpu_b0 { ++ cpu-supply = <&syr827>; ++}; ++ ++&cpu_b1 { ++ cpu-supply = <&syr827>; ++}; ++ ++&cpu_b2 { ++ cpu-supply = <&syr827>; ++}; ++ ++&cpu_b3 { ++ cpu-supply = <&syr827>; ++}; ++ ++&gpu { ++ logic-supply = <&vdd_logic>; ++}; ++ ++&rockchip_suspend { ++ status = "okay"; ++ rockchip,sleep-mode-config = < ++ (0 ++ | RKPM_SLP_ARMOFF ++ | RKPM_SLP_PMU_PLLS_PWRDN ++ | RKPM_SLP_PMU_PMUALIVE_32K ++ | RKPM_SLP_SFT_PLLS_DEEP ++ | RKPM_SLP_PMU_DIS_OSC ++ | RKPM_SLP_SFT_PD_NBSCUS ++ ) ++ >; ++ rockchip,wakeup-config = < ++ (0 ++ | RKPM_GPIO_WKUP_EN ++ | RKPM_USB_WKUP_EN ++ | RKPM_CLUSTER_L_WKUP_EN ++ ) ++ >; ++}; ++ ++&emmc { ++ bus-width = <8>; ++ cap-mmc-highspeed; ++ mmc-hs200-1_8v; ++ no-sdio; ++ no-sd; ++ disable-wp; ++ non-removable; ++ num-slots = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&emmc_clk &emmc_cmd &emmc_bus8>; ++ status = "okay"; ++}; ++ ++&nandc0 { ++ status = "disabled"; ++}; ++ ++&sdmmc { ++ clock-frequency = <37500000>; ++ clock-freq-min-max = <400000 37500000>; ++ no-sdio; ++ no-mmc; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ card-detect-delay = <200>; ++ disable-wp; ++ num-slots = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_cd &sdmmc_bus4>; ++ status = "disabled"; ++}; ++ ++&sdio0 { ++ clock-frequency = <50000000>; ++ clock-freq-min-max = <200000 50000000>; ++ no-sd; ++ no-mmc; ++ bus-width = <4>; ++ disable-wp; ++ cap-sd-highspeed; ++ cap-sdio-irq; ++ keep-power-in-suspend; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ non-removable; ++ num-slots = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ ++ syr827: syr827@40 { ++ compatible = "silergy,syr827"; ++ status = "okay"; ++ reg = <0x40>; ++ regulator-compatible = "fan53555-reg"; ++ regulator-name = "vdd_arm"; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1500000>; ++ regulator-ramp-delay = <1000>; ++ fcs,suspend-voltage-selector = <1>; ++ pinctrl-0 = <&vsel_gpio>; ++ vsel-gpios = <&gpio0 0 GPIO_ACTIVE_HIGH>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-state = <3>; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <900000>; ++ }; ++ }; ++ ++ rk818: pmic@1c { ++ compatible = "rockchip,rk818"; ++ status = "okay"; ++ reg = <0x1c>; ++ clock-output-names = "rk818-clkout1", "wifibt_32kin"; ++ interrupt-parent = <&gpio0>; ++ interrupts = <1 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pmic_int_l>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ ++ vcc1-supply = <&vcc_sys>; ++ vcc2-supply = <&vcc_sys>; ++ vcc3-supply = <&vcc_sys>; ++ vcc4-supply = <&vcc_sys>; ++ vcc6-supply = <&vcc_sys>; ++ vcc7-supply = <&vcc_sys>; ++ vcc8-supply = <&vcc_sys>; ++ vcc9-supply = <&vcc_io>; ++ ++ regulators { ++ vdd_logic: DCDC_REG1 { ++ regulator-name = "vdd_logic"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <750000>; ++ regulator-max-microvolt = <1450000>; ++ regulator-ramp-delay = <6001>; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vdd_gpu: DCDC_REG2 { ++ regulator-name = "vdd_gpu"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <800000>; ++ regulator-max-microvolt = <1250000>; ++ regulator-ramp-delay = <6001>; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc_ddr"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_io: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcc_io"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcca_codec: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcca_codec"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc_tp: LDO_REG2 { ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-name = "vcc_tp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdd_10: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ regulator-name = "vdd_10"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc18_lcd: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc18_lcd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vccio_pmu: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vccio_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd10_lcd: LDO_REG6 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ regulator-name = "vdd10_lcd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc_18: LDO_REG7 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc_18"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vccio_wl: LDO_REG8 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vccio_wl"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vccio_sd: LDO_REG9 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vccio_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc_sd: SWITCH_REG { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ boost_otg: DCDC_BOOST { ++ regulator-name = "boost_otg"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <5000000>; ++ }; ++ }; ++ ++ otg_switch: OTG_SWITCH { ++ regulator-name = "otg_switch"; ++ }; ++ ++ hdmi_switch: HDMI_SWITCH { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "hdmi_switch"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ }; ++ ++ battery { ++ compatible = "rk818-battery"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&dc_irq_gpio>; ++ ocv_table = < ++ 3400 3650 3693 3707 3731 3749 3760 ++ 3770 3782 3796 3812 3829 3852 3882 ++ 3915 3951 3981 4047 4086 4132 4182>; ++ design_capacity = <8650>; ++ design_qmax = <8800>; ++ bat_res = <85>; ++ max_input_current = <2000>; ++ max_chrg_current = <1800>; ++ max_chrg_voltage = <4200>; ++ sleep_enter_current = <600>; ++ sleep_exit_current = <600>; ++ power_off_thresd = <3400>; ++ zero_algorithm_vol = <3850>; ++ fb_temperature = <115>; ++ sample_res = <20>; ++ max_soc_offset = <60>; ++ energy_mode = <0>; ++ monitor_sec = <5>; ++ virtual_power = <0>; ++ power_dc2otg = <1>; ++ support_usb_adp = <1>; ++ support_dc_adp = <1>; ++ dc_det_gpio = <&gpio0 17 GPIO_ACTIVE_LOW>; ++ }; ++ }; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ ++ es8316: es8316@10 { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++ compatible = "everest,es8316"; ++ reg = <0x10>; ++ clocks = <&cru SCLK_I2S_8CH_OUT>; ++ clock-names = "mclk"; ++ spk-con-gpio = <&gpio0 27 GPIO_ACTIVE_HIGH>; ++ hp-det-gpio = <&gpio0 23 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s_8ch_mclk>; ++ }; ++}; ++ ++&i2c2 { ++ status = "okay"; ++ ++ gt9xx: gt9xx@14 { ++ compatible = "goodix,gt9xx"; ++ reg = <0x14>; ++ touch-gpio = <&gpio0 12 IRQ_TYPE_LEVEL_LOW>; ++ reset-gpio = <&gpio0 11 GPIO_ACTIVE_HIGH>; ++ max-x = <1920>; ++ max-y = <1200>; ++ tp-size = <89>; ++ status = "okay"; ++ tp-supply = <&vcc_tp>; ++ }; ++}; ++ ++&i2c3 { ++ status = "okay"; ++}; ++ ++&i2c4 { ++ status = "okay"; ++ ++ mpu6500_acc: mpu_acc@68 { ++ compatible = "mpu6500_acc"; ++ reg = <0x68>; ++ irq-gpio = <&gpio3 14 IRQ_TYPE_LEVEL_LOW>; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ type = ; ++ layout = <7>; ++ }; ++ ++ mpu6500_gyro: mpu_gyro@68 { ++ compatible = "mpu6500_gyro"; ++ reg = <0x68>; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ type = ; ++ layout = <7>; ++ }; ++ ++ mpu6500@68 { ++ status = "disabled"; ++ compatible = "invensense,mpu6500"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&mpu6500_irq_gpio>; ++ reg = <0x68>; ++ irq-gpio = <&gpio3 14 IRQ_TYPE_EDGE_RISING>; ++ mpu-int_config = <0x10>; ++ mpu-level_shifter = <0>; ++ mpu-orientation = <1 0 0 0 1 0 0 0 1>; ++ orientation-x= <1>; ++ orientation-y= <0>; ++ orientation-z= <1>; ++ support-hw-poweroff = <1>; ++ mpu-debug = <1>; ++ }; ++}; ++ ++&i2s_8ch { ++ status = "okay"; ++ rockchip,i2s-broken-burst-len; ++ rockchip,playback-channels = <8>; ++ rockchip,capture-channels = <2>; ++ #sound-dai-cells = <0>; ++}; ++ ++&io_domains { ++ status = "okay"; ++ ++ dvp-supply = <&vcc_18>; ++ audio-supply = <&vcc_io>; ++ gpio30-supply = <&vcc_io>; ++ gpio1830-supply = <&vcc_io>; ++ sdcard-supply = <&vccio_sd>; ++ wifi-supply = <&vccio_wl>; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ ++ pmu-supply = <&vccio_pmu>; ++ vop-supply = <&vccio_pmu>; ++}; ++ ++&pwm0 { ++ status = "okay"; ++}; ++ ++&uart0 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart0_xfer &uart0_cts>; ++ status = "okay"; ++}; ++ ++&saradc { ++ status = "okay"; ++}; ++ ++&u2phy { ++ status = "okay"; ++ ++ u2phy_host: host-port { ++ phy-supply = <&vcc_host>; ++ status = "okay"; ++ }; ++}; ++ ++&usb_host0_ehci { ++ status = "okay"; ++}; ++ ++&usb_host0_ohci { ++ status = "okay"; ++}; ++ ++&dsi { ++ status = "okay"; ++ ++ panel@0 { ++ compatible = "simple-panel-dsi"; ++ reg = <0>; ++ backlight = <&backlight>; ++ enable-gpios = <&gpio0 22 GPIO_ACTIVE_HIGH>; ++ prepare-delay-ms = <120>; ++ enable-delay-ms = <200>; ++ ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | ++ MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <145000000>; ++ hactive = <1920>; ++ vactive = <1200>; ++ hback-porch = <16>; ++ hfront-porch = <24>; ++ vback-porch = <10>; ++ vfront-porch = <16>; ++ hsync-len = <10>; ++ vsync-len = <3>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ panel_in_dsi: endpoint { ++ remote-endpoint = <&dsi_out_panel>; ++ }; ++ }; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@1 { ++ reg = <1>; ++ dsi_out_panel: endpoint { ++ remote-endpoint = <&panel_in_dsi>; ++ }; ++ }; ++ }; ++}; ++ ++&route_dsi { ++ status = "okay"; ++}; ++ ++&tsadc { ++ tsadc-supply = <&syr827>; ++ status = "okay"; ++}; ++ ++&pinctrl { ++ pmic { ++ pmic_int_l: pmic-int-l { ++ rockchip,pins = <0 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ vsel_gpio: vsel-gpio { ++ rockchip,pins = <0 RK_PA0 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ mpu6500 { ++ mpu6500_irq_gpio: mpu6500-irq-gpio { ++ rockchip,pins = <3 RK_PB6 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ dc_det { ++ dc_irq_gpio: dc-irq-gpio { ++ rockchip,pins = <0 RK_PC1 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ usb2 { ++ host_vbus_drv: host-vbus-drv { ++ rockchip,pins = <0 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart0_rts_gpio: uart0-rts-gpio { ++ rockchip,pins = <2 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ +diff --git a/arch/arm64/boot/dts/rockchip/rk3368-px5-evb-android.dts b/arch/arm64/boot/dts/rockchip/rk3368-px5-evb-android.dts +new file mode 100755 +index 000000000000..1d50d3fa549f +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3368-px5-evb-android.dts +@@ -0,0 +1,993 @@ ++/* ++ * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This file is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This file is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++/dts-v1/; ++#include "rk3368.dtsi" ++#include "rk3368-android.dtsi" ++#include "rk3368-cif-sensor.dtsi" ++#include ++#include ++ ++/ { ++ model = "Rockchip PX5 EVB V11"; ++ compatible = "rockchip,px5-evb", "rockchip,px5", "rockchip,rk3368"; ++ ++ chosen: chosen { ++ bootargs = "earlycon=uart8250,mmio32,0xff1c0000 firmware_class.path=/system/vendor/firmware"; ++ }; ++ ++ fiq_debugger: fiq-debugger { ++ compatible = "rockchip,fiq-debugger"; ++ rockchip,serial-id = <4>; ++ rockchip,wake-irq = <0>; ++ /* If enable uart uses irq instead of fiq */ ++ rockchip,irq-mode-enable = <0>; ++ /* Only 115200 and 1500000 */ ++ rockchip,baudrate = <115200>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart4_xfer>; ++ interrupts = ; ++ }; ++ ++ firmware { ++ android { ++ compatible = "android,firmware"; ++ fstab { ++ compatible = "android,fstab"; ++ system { ++ compatible = "android,system"; ++ dev = "/dev/block/platform/ff0f0000.dwmmc/by-name/system"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait"; ++ }; ++ vendor { ++ compatible = "android,vendor"; ++ dev = "/dev/block/platform/ff0f0000.dwmmc/by-name/vendor"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait"; ++ }; ++ }; ++ }; ++ }; ++ ++ xin32k: xin32k { ++ status = "okay"; ++ compatible = "pwm-clock"; ++ #clock-cells = <0>; ++ clock-frequency = <32768>; ++ clock-output-names = "xin32k"; ++ pwms = <&pwm1 0 30518 0>; /* 1 / 30518 ns = 32.7675 KHz */ ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ clocks = <&rk808 1>; ++ clock-names = "ext_clock"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio3 RK_PA5 GPIO_ACTIVE_LOW>; /* GPIO3_A5 */ ++ }; ++ ++ es8396-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,es8396-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,widgets = ++ "Microphone", "Microphone Jack", ++ "Line", "Microphone Headset", ++ "Headphone", "Headphone Jack"; ++ simple-audio-card,routing = ++ "MIC", "Microphone Jack", ++ "DMIC", "Microphone Headset", ++ "Headphone Jack", "LOUTP", ++ "Headphone Jack", "ROUTN"; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s_8ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&es8396>; ++ }; ++ }; ++ ++ backlight: backlight { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm0 0 25000 0>; ++ brightness-levels = < ++ 0 1 2 3 4 5 6 7 ++ 8 9 10 11 12 13 14 15 ++ 16 17 18 19 20 21 22 23 ++ 24 25 26 27 28 29 30 31 ++ 32 33 34 35 36 37 38 39 ++ 40 41 42 43 44 45 46 47 ++ 48 49 50 51 52 53 54 55 ++ 56 57 58 59 60 61 62 63 ++ 64 65 66 67 68 69 70 71 ++ 72 73 74 75 76 77 78 79 ++ 80 81 82 83 84 85 86 87 ++ 88 89 90 91 92 93 94 95 ++ 96 97 98 99 100 101 102 103 ++ 104 105 106 107 108 109 110 111 ++ 112 113 114 115 116 117 118 119 ++ 120 121 122 123 124 125 126 127 ++ 128 129 130 131 132 133 134 135 ++ 136 137 138 139 140 141 142 143 ++ 144 145 146 147 148 149 150 151 ++ 152 153 154 155 156 157 158 159 ++ 160 161 162 163 164 165 166 167 ++ 168 169 170 171 172 173 174 175 ++ 176 177 178 179 180 181 182 183 ++ 184 185 186 187 188 189 190 191 ++ 192 193 194 195 196 197 198 199 ++ 200 201 202 203 204 205 206 207 ++ 208 209 210 211 212 213 214 215 ++ 216 217 218 219 220 221 222 223 ++ 224 225 226 227 228 229 230 231 ++ 232 233 234 235 236 237 238 239 ++ 240 241 242 243 244 245 246 247 ++ 248 249 250 251 252 253 254 255>; ++ default-brightness-level = <200>; ++ enable-gpios = <&gpio0 RK_PB6 GPIO_ACTIVE_HIGH>; ++ }; ++ ++ rk_key: rockchip-key { ++ compatible = "rockchip,key"; ++ status = "okay"; ++ ++ io-channels = <&saradc 1>; ++ ++ vol-up-key { ++ linux,code = <115>; ++ label = "volume up"; ++ rockchip,adc_value = <1>; ++ }; ++ ++ vol-down-key { ++ linux,code = <114>; ++ label = "volume down"; ++ rockchip,adc_value = <170>; ++ }; ++ ++ power-key { ++ gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_LOW>; ++ linux,code = <116>; ++ label = "power"; ++ gpio-key,wakeup; ++ }; ++ ++ menu-key { ++ linux,code = <59>; ++ label = "menu"; ++ rockchip,adc_value = <355>; ++ }; ++ ++ home-key { ++ linux,code = <102>; ++ label = "home"; ++ rockchip,adc_value = <746>; ++ }; ++ ++ back-key { ++ linux,code = <158>; ++ label = "back"; ++ rockchip,adc_value = <560>; ++ }; ++ ++ camera-key { ++ linux,code = <212>; ++ label = "camera"; ++ rockchip,adc_value = <450>; ++ }; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ ++ wifi_chip_type = "rtl8723ds"; ++ sdio_vref = <1800>; /*1800mv or 3300mv*/ ++ WIFI,host_wake_irq = <&gpio3 RK_PA6 GPIO_ACTIVE_HIGH>; ++ WIFI,vbat_gpio = <&gpio3 RK_PA4 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ ++ keep_bt_power_on; ++ BT,power_gpio = <&gpio3 4 GPIO_ACTIVE_HIGH>; /* GPIO3_A4 */ ++ BT,reset_gpio = <&gpio3 2 GPIO_ACTIVE_HIGH>; /* GPIO3_A2 */ ++ BT,wake_gpio = <&gpio3 7 GPIO_ACTIVE_HIGH>; /* GPIO3_A7 */ ++ BT,wake_host_irq = <&gpio3 3 GPIO_ACTIVE_HIGH>; /* GPIO3_A3 */ ++ ++ status = "okay"; ++ }; ++ ++ gpio_det: gpio-det { ++ compatible = "gpio-detection"; ++ status = "okay"; ++ ++ pinctrl-0 = <&gpio3_b1 &gpio3_b2>; ++ pinctrl-names = "default"; ++ ++ car-reverse { ++ car-reverse-gpios = <&gpio3 10 GPIO_ACTIVE_LOW>; ++ linux,debounce-ms = <5>; ++ label = "car-reverse"; ++ gpio,wakeup; ++ }; ++ ++ car-acc { ++ car-acc-gpios = <&gpio3 9 GPIO_ACTIVE_LOW>; ++ linux,debounce-ms = <5>; ++ label = "car-acc"; ++ gpio,wakeup; ++ }; ++ }; ++ ++ vcc_sys: vcc-sys-regulator { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3800000>; ++ regulator-max-microvolt = <3800000>; ++ }; ++ ++ vcc_host: vcc-host { ++ compatible = "regulator-fixed"; ++ enable-active-low; ++ gpio = <&gpio0 RK_PA3 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&host_vbus_drv>; ++ regulator-name = "vcc_host"; ++ regulator-always-on; ++ }; ++ ++ vcc18_lcd_n: vcc18-lcd-n { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc18_lcd_n"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ gpio = <&gpio2 RK_PD3 GPIO_ACTIVE_HIGH>; ++ enable-active-high; ++ startup-delay-us = <70000>; ++ vin-supply = <&vcc_18>; ++ ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ panel { ++ compatible = "samsung,lsl070nl01", "simple-panel"; ++ power-supply = <&vcc33_lcd>; ++ backlight = <&backlight>; ++ prepare-delay-ms = <120>; ++ unprepare-delay-ms = <120>; ++ bus-format = ; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <48000000>; ++ hactive = <1024>; ++ vactive = <600>; ++ hback-porch = <90>; ++ hfront-porch = <90>; ++ vback-porch = <10>; ++ vfront-porch = <10>; ++ hsync-len = <90>; ++ vsync-len = <10>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ port { ++ panel_in_lvds: endpoint { ++ remote-endpoint = <&lvds_out_panel>; ++ }; ++ }; ++ }; ++}; ++ ++&firmware_android { ++ compatible = "android,firmware"; ++ fstab { ++ compatible = "android,fstab"; ++ system { ++ compatible = "android,system"; ++ dev = "/dev/block/by-name/system"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,verify"; ++ }; ++ vendor { ++ compatible = "android,vendor"; ++ dev = "/dev/block/by-name/vendor"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,verify"; ++ }; ++ }; ++}; ++ ++&cif_sensor { ++ status = "okay"; ++}; ++ ++&cluster0_opp { ++ rockchip,threshold-freq = <408000>; ++ rockchip,freq-limit; ++}; ++ ++&cluster1_opp { ++ rockchip,threshold-freq = <1416000>; ++}; ++ ++&cpu_l0 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu_l1 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu_l2 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu_l3 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu_b0 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu_b1 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu_b2 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu_b3 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&gpu { ++ logic-supply = <&vdd_log>; ++}; ++ ++&isp { ++ pinctrl-names = ++ "isp_mipi_fl_prefl", "isp_flash_as_gpio", ++ "isp_flash_as_trigger_out"; ++ pinctrl-0 = <&isp_prelight>; ++ pinctrl-1 = <&isp_flash_trigger_as_gpio>; ++ pinctrl-2 = <&isp_flash_trigger>; ++}; ++ ++&rockchip_suspend { ++ status = "okay"; ++ rockchip,sleep-mode-config = < ++ (0 ++ | RKPM_SLP_ARMOFF ++ | RKPM_SLP_PMU_PLLS_PWRDN ++ | RKPM_SLP_PMU_PMUALIVE_32K ++ | RKPM_SLP_SFT_PLLS_DEEP ++ | RKPM_SLP_PMU_DIS_OSC ++ | RKPM_SLP_SFT_PD_NBSCUS ++ ) ++ >; ++ rockchip,wakeup-config = < ++ (0 ++ | RKPM_GPIO_WKUP_EN ++ | RKPM_USB_WKUP_EN ++ | RKPM_CLUSTER_L_WKUP_EN ++ ) ++ >; ++}; ++ ++&emmc { ++ status = "okay"; ++ bus-width = <8>; ++ clock-frequency = <150000000>; ++ keep-power-in-suspend; ++ cap-mmc-highspeed; ++ mmc-hs200-1_8v; ++ no-sdio; ++ no-sd; ++ disable-wp; ++ non-removable; ++ num-slots = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&emmc_clk &emmc_cmd &emmc_bus8>; ++ vmmc-supply = <&vcc_io>; ++ vqmmc-supply = <&vcc18_flash>; ++}; ++ ++&sdmmc { ++ status = "okay"; ++ clock-frequency = <37500000>; ++ clock-freq-min-max = <400000 37500000>; ++ no-sdio; ++ no-mmc; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ card-detect-delay = <200>; ++ disable-wp; ++ num-slots = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_cd &sdmmc_bus4>; ++ vmmc-supply = <&vcc_sd>; ++ vqmmc-supply = <&vccio_sd>; ++}; ++ ++&sdio0 { ++ status = "okay"; ++ clock-frequency = <50000000>; ++ clock-freq-min-max = <200000 50000000>; ++ no-sd; ++ no-mmc; ++ bus-width = <4>; ++ disable-wp; ++ cap-sd-highspeed; ++ cap-sdio-irq; ++ keep-power-in-suspend; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ non-removable; ++ num-slots = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>; ++ sd-uhs-sdr104; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ ++ rk808: pmic@1b { ++ compatible = "rockchip,rk808"; ++ reg = <0x1b>; ++ interrupt-parent = <&gpio0>; ++ interrupts = <5 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pmic_int>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ ++ vcc1-supply = <&vcc_sys>; ++ vcc2-supply = <&vcc_sys>; ++ vcc3-supply = <&vcc_sys>; ++ vcc4-supply = <&vcc_sys>; ++ vcc6-supply = <&vcc_sys>; ++ vcc7-supply = <&vcc_sys>; ++ vcc8-supply = <&vcc_io>; ++ vcc9-supply = <&vcc_sys>; ++ vcc10-supply = <&vcc_sys>; ++ vcc11-supply = <&vcc_sys>; ++ vcc12-supply = <&vcc_io>; ++ ++ regulators { ++ vdd_cpu: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <700000>; ++ regulator-max-microvolt = <1500000>; ++ regulator-name = "vdd_cpu"; ++ regulator-ramp-delay = <6000>; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vdd_log: DCDC_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <700000>; ++ regulator-max-microvolt = <1500000>; ++ regulator-name = "vdd_log"; ++ regulator-ramp-delay = <6000>; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc_ddr"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_io: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcc_io"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc18_flash: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc18_flash"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vcca_33: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcca_33"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vdd_10: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ regulator-name = "vdd_10"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ avdd_33: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "avdd_33"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vccio_sd: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vccio_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vdd10_lcd: LDO_REG6 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ regulator-name = "vdd10_lcd"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc_18: LDO_REG7 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc_18"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vcc18_lcd: LDO_REG8 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc18_lcd"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vcc_sd: SWITCH_REG1 { ++ regulator-boot-on; ++ regulator-name = "vcc_sd"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc33_lcd: SWITCH_REG2 { ++ regulator-boot-on; ++ regulator-name = "vcc33_lcd"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ }; ++ }; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ ++ bma2x2: bma250@18 { ++ compatible = "bma2xx_acc"; ++ status = "okay"; ++ reg = <0x18>; ++ type = ; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&gpio2_c1>; ++ irq-gpio = <&gpio2 17 IRQ_TYPE_LEVEL_LOW>; ++ irq_enable = <1>; ++ poll_delay_ms = <200>; ++ layout = <6>; ++ reprobe_en = <1>;/* this sensor need to be probe again */ ++ }; ++ ++ ls_stk3410: light@48 { ++ compatible = "ls_stk3410"; ++ status = "okay"; ++ reg = <0x48>; ++ type = ; ++ irq_enable = <0>; ++ als_threshold_high = <100>; ++ als_threshold_low = <10>; ++ als_ctrl_gain = <2>; /* 0:x1 1:x4 2:x16 3:x64 */ ++ poll_delay_ms = <100>; ++ }; ++ ++ ps_stk3410: proximity@48 { ++ compatible = "ps_stk3410"; ++ status = "okay"; ++ reg = <0x48>; ++ type = ; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&gpio2_c3>; ++ irq-gpio = <&gpio2 19 IRQ_TYPE_LEVEL_LOW>; ++ irq_enable = <1>; ++ ps_threshold_high = <0x200>; ++ ps_threshold_low = <0x100>; ++ ps_ctrl_gain = <3>; /* 0:x1 1:x4 2:x16 3:x64 */ ++ ps_led_current = <3>; /* 0:12.5mA 1:25mA 2:50mA 3:100mA */ ++ poll_delay_ms = <100>; ++ }; ++}; ++ ++&i2c2 { ++ status = "okay"; ++ clock-frequency = <200000>; ++ ++ gsl1680: touchscreen@40 { ++ compatible = "silead,gsl1680"; ++ reg = <0x40>; ++ interrupt-parent = <&gpio3>; ++ interrupts = <28 IRQ_TYPE_EDGE_FALLING>; ++ power-gpios = <&gpio3 RK_PB7 GPIO_ACTIVE_HIGH>; ++ touchscreen-size-x = <800>; ++ touchscreen-size-y = <1280>; ++ silead,max-fingers = <5>; ++ }; ++ ++ gt9xx: gt9xx@14 { ++ compatible = "goodix,gt9xx"; ++ reg = <0x14>; ++ touch-gpio = <&gpio3 RK_PD4 IRQ_TYPE_LEVEL_HIGH>; ++ reset-gpio = <&gpio3 RK_PB7 GPIO_ACTIVE_LOW>; ++ max-x = <1024>; ++ max-y = <600>; ++ tp-size = <910>; ++ tp-supply = <&vcc_io>; ++ status = "okay"; ++ }; ++}; ++ ++&i2c3 { ++ status = "okay"; ++ clock-frequency = <200000>; ++ ++ fm1288: fm1288@60{ ++ compatible = "fm1288"; ++ reg = <0x60>; ++ pwd-gpios = <&gpio0 RK_PD1 GPIO_ACTIVE_HIGH>; ++ bypass-gpios = <&gpio0 RK_PD4 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ es8396: es8396@10 { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++ compatible = "es8396"; ++ reg = <0x10>; ++ clocks = <&cru SCLK_I2S_8CH_OUT>; ++ clock-names = "mclk"; ++ spk-con-gpio = <&gpio2 RK_PC7 GPIO_ACTIVE_HIGH>; ++ lineout-con-gpio = <&gpio0 RK_PD7 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s_8ch_mclk>; ++ }; ++}; ++ ++&i2c4 { ++ status = "okay"; ++}; ++ ++&i2s_8ch { ++ status = "okay"; ++ rockchip,i2s-broken-burst-len; ++ rockchip,playback-channels = <8>; ++ rockchip,capture-channels = <2>; ++ #sound-dai-cells = <0>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s_8ch_bus>; ++}; ++ ++&io_domains { ++ status = "okay"; ++ dvp-supply = <&vcc_18>; ++ audio-supply = <&vcc_io>; ++ gpio30-supply = <&vcc_io>; ++ gpio1830-supply = <&vcc_io>; ++ sdcard-supply = <&vccio_sd>; ++ wifi-supply = <&vcc_io>; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ pmu-supply = <&vcc_io>; ++ vop-supply = <&vcca_33>; ++}; ++ ++&pwm0 { ++ status = "okay"; ++}; ++ ++&pwm1 { ++ status = "okay"; ++}; ++ ++&u2phy { ++ status = "okay"; ++ ++ u2phy_host: host-port { ++ phy-supply = <&vcc_host>; ++ status = "okay"; ++ }; ++}; ++ ++&usb_host0_ehci { ++ status = "okay"; ++}; ++ ++&usb_host0_ohci { ++ status = "okay"; ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&dmc { ++ vop-dclk-mode = <1>; ++ center-supply = <&vdd_log>; ++ system-status-freq = < ++ /*system status freq(KHz)*/ ++ SYS_STATUS_NORMAL 600000 ++ SYS_STATUS_REBOOT 600000 ++ SYS_STATUS_SUSPEND 192000 ++ SYS_STATUS_VIDEO_1080P 600000 ++ SYS_STATUS_VIDEO_4K 600000 ++ SYS_STATUS_PERFORMANCE 600000 ++ SYS_STATUS_BOOST 600000 ++ SYS_STATUS_DUALVIEW 600000 ++ SYS_STATUS_ISP 600000 ++ >; ++ status = "okay"; ++}; ++ ++&lvds { ++ status = "okay"; ++ ++ ports { ++ port@1 { ++ reg = <1>; ++ ++ lvds_out_panel: endpoint { ++ remote-endpoint = <&panel_in_lvds>; ++ }; ++ }; ++ }; ++}; ++ ++&route_lvds { ++ status = "okay"; ++}; ++ ++&mailbox { ++ status = "okay"; ++}; ++ ++&mailbox_scpi { ++ status = "okay"; ++}; ++ ++&saradc { ++ status = "okay"; ++}; ++ ++&thermal_zones { ++ status = "okay"; ++}; ++ ++&tsadc { ++ tsadc-supply = <&vdd_cpu>; ++ status = "okay"; ++}; ++ ++&uart1 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart1_xfer &uart1_cts>; ++ status = "okay"; ++}; ++ ++&uart2 { ++ status = "disabled"; ++}; ++ ++&uart4 { ++ status = "okay"; ++}; ++ ++&pinctrl { ++ pmic { ++ pmic_int: pmic-int { ++ rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ dc_det { ++ dc_irq_gpio: dc-irq-gpio { ++ rockchip,pins = <0 RK_PC1 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <3 RK_PA5 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ usb2 { ++ host_vbus_drv: host-vbus-drv { ++ rockchip,pins = <0 RK_PA3 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart0_rts_gpio: uart0-rts-gpio { ++ rockchip,pins = <2 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ gpio0_gpio { ++ gpio0_c7: gpio0-c7 { ++ rockchip,pins = <0 RK_PC7 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ gpio0_a3: gpio0-a3 { ++ rockchip,pins = <0 RK_PA3 3 &pcfg_pull_none>; ++ }; ++ gpio0_c2: gpio0-c2 { ++ rockchip,pins = <0 RK_PC2 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ gpio0_c3: gpio0-c3 { ++ rockchip,pins = <0 RK_PC4 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ gpio0_c1: gpio0-c1 { ++ rockchip,pins = <0 RK_PC1 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ gpio2_c1: gpio2-c1 { ++ rockchip,pins = <2 RK_PC1 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ gpio2_c3: gpio2-c3 { ++ rockchip,pins = <2 RK_PC3 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ gpio3_b1: gpio3-b1 { ++ rockchip,pins = <3 RK_PB1 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ gpio3_b2: gpio3-b2 { ++ rockchip,pins = <3 RK_PB2 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ i2s { ++ i2s_8ch_bus: i2s-8ch-bus { ++ rockchip,pins = <2 RK_PB4 1 &pcfg_pull_none>, ++ <2 RK_PB5 1 &pcfg_pull_none>, ++ <2 RK_PB6 1 &pcfg_pull_none>, ++ <2 RK_PB7 1 &pcfg_pull_none>, ++ <2 RK_PC0 1 &pcfg_pull_none>, ++ <2 RK_PC2 1 &pcfg_pull_none>; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3368-r88-dcdc.dts b/arch/arm64/boot/dts/rockchip/rk3368-r88-dcdc.dts +new file mode 100755 +index 000000000000..4a755f1207c1 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3368-r88-dcdc.dts +@@ -0,0 +1,676 @@ ++/* ++ * Copyright (c) 2015 Heiko Stuebner ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This file is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This file is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++/dts-v1/; ++#include "rk3368.dtsi" ++#include "rk3368-android.dtsi" ++#include ++ ++/ { ++ model = "Rockchip R88"; ++ compatible = "rockchip,r88", "rockchip,rk3368"; ++ ++ hdmi_sound: hdmi-sound { ++ status = "okay"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,name = "rockchip,hdmi"; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s_8ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&hdmi>; ++ }; ++ }; ++ ++ keys: gpio-keys { ++ compatible = "gpio-keys"; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pwr_key>; ++ ++ button@0 { ++ gpio-key,wakeup = <1>; ++ gpios = <&gpio0 2 GPIO_ACTIVE_LOW>; ++ label = "GPIO Power"; ++ linux,code = <116>; ++ }; ++ }; ++ ++ leds: gpio-leds { ++ compatible = "gpio-leds"; ++ ++ work { ++ gpios = <&gpio3 29 GPIO_ACTIVE_HIGH>; ++ label = "r88:green:led"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&led_ctl>; ++ }; ++ }; ++ ++ ir: ir-receiver { ++ compatible = "gpio-ir-receiver"; ++ gpios = <&gpio3 30 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&ir_int>; ++ }; ++ ++ vcc_sys: vcc-sys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3800000>; ++ regulator-max-microvolt = <3800000>; ++ }; ++ ++ vcc_host: vcc-host { ++ compatible = "regulator-fixed"; ++ enable-active-high; ++ gpio = <&gpio0 4 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&host_vbus_drv>; ++ regulator-name = "vcc_host"; ++ regulator-always-on; ++ }; ++ ++ vcc_phy: vcc-phy-regulator { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_phy"; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ regulators { ++ compatible = "simple-bus"; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ vccio_1v8_reg: regulator@0 { ++ compatible = "regulator-fixed"; ++ regulator-name = "vccio_1v8"; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-always-on; ++ }; ++ ++ vccio_3v3_reg: regulator@1 { ++ compatible = "regulator-fixed"; ++ regulator-name = "vccio_3v3"; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-always-on; ++ }; ++ }; ++ ++ vdd_gpu: vdd-arm-regulator { ++ compatible = "pwm-regulator"; ++ rockchip,pwm_id = <1>; ++ rockchip,pwm_voltage = <1100000>; ++ pwms = <&pwm1 0 25000 1>; ++ regulator-name = "vdd_gpu"; ++ regulator-min-microvolt = <950000>; ++ regulator-max-microvolt = <1400000>; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ clkin_gmac: external-gmac-clock { ++ compatible = "fixed-clock"; ++ clock-frequency = <125000000>; ++ clock-output-names = "clkin_gmac"; ++ #clock-cells = <0>; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ rockchip,grf = <&grf>; ++ wifi_chip_type = "rtl8189es"; ++ sdio_vref = <1800>; ++ WIFI,host_wake_irq = <&gpio3 6 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ //clocks = <&rk808 1>; ++ //clock-names = "ext_clock"; ++ /* wifi-bt-power-toggle; */ ++ uart_rts_gpios = <&gpio2 27 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart0_rts>; ++ pinctrl-1 = <&uart0_gpios>; ++ /* BT,power_gpio = <&gpio3 19 GPIO_ACTIVE_HIGH>; */ ++ BT,reset_gpio = <&gpio3 5 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio3 2 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio3 7 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++}; ++ ++&firmware_android { ++ compatible = "android,firmware"; ++ fstab { ++ compatible = "android,fstab"; ++ system { ++ compatible = "android,system"; ++ dev = "/dev/block/by-name/system"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,verify"; ++ }; ++ vendor { ++ compatible = "android,vendor"; ++ dev = "/dev/block/by-name/vendor"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,verify"; ++ }; ++ }; ++}; ++ ++&i2s_8ch { ++ status = "okay"; ++ rockchip,i2s-broken-burst-len; ++ rockchip,playback-channels = <8>; ++ rockchip,capture-channels = <2>; ++ #sound-dai-cells = <0>; ++}; ++ ++&emmc { ++ status = "okay"; ++ bus-width = <8>; ++ cap-mmc-highspeed; ++ mmc-hs200-1_8v; ++ no-sdio; ++ no-sd; ++ disable-wp; ++ non-removable; ++ num-slots = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&emmc_clk &emmc_cmd &emmc_bus8>; ++}; ++ ++&sdmmc { ++ status = "disabled"; ++ clock-frequency = <37500000>; ++ clock-freq-min-max = <400000 37500000>; ++ no-sdio; ++ no-mmc; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ card-detect-delay = <200>; ++ disable-wp; ++ num-slots = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_cd &sdmmc_bus4>; ++}; ++ ++&gmac { ++ phy-supply = <&vcc_phy>; ++ phy-mode = "rmii"; ++ clock_in_out = "output"; ++ snps,reset-gpio = <&gpio3 12 0>; ++ snps,reset-active-low; ++ snps,reset-delays-us = <0 50000 50000>; ++ //assigned-clocks = <&cru SCLK_RMII_SRC>; ++ //assigned-clock-parents = <&clkin_gmac>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&rmii_pins>; ++ tx_delay = <0x30>; ++ rx_delay = <0x10>; ++ status = "ok"; ++}; ++ ++&pwm1 { ++ status = "okay"; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm1_pin_pull_down>; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ clock-frequency = <100000>; ++ ++ vdd_cpu: syr827@40 { ++ compatible = "silergy,syr827"; ++ reg = <0x40>; ++ status = "okay"; ++ vin-supply = <&vcc_sys>; ++ regulator-compatible = "fan53555-reg"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&vsel1_gpio>; ++ vsel-gpios = <&gpio0 0 GPIO_ACTIVE_HIGH>; ++ regulator-name = "vdd_cpu"; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1500000>; ++ regulator-ramp-delay = <1000>; ++ fcs,suspend-voltage-selector = <1>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-state = <3>; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++/* xz3215: xz3215@40 { ++ compatible = "xz3216"; ++ reg = <0x40>; ++ status = "disabled"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&vsel1_gpio>; ++ vsel-gpios = <&gpio0 0 GPIO_ACTIVE_HIGH>; ++ regulators { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ vdd_cpu: regulator@0 { ++ reg = <0>; ++ regulator-compatible = "xz_dcdc1"; ++ regulator-name = "vdd_cpu"; ++ regulator-min-microvolt = <603000>; ++ regulator-max-microvolt = <1400000>; ++ regulator-ramp-delay = <1000>; ++ regulator-always-on; ++ regulator-boot-on; ++ fcs,suspend-voltage-selector = <1>; ++ //regulator-initial-mode = <0x1>; ++ regulator-initial-state = <3>; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ }; ++ }; ++*/ ++ ++ hym8563: hym8563@51 { ++ compatible = "haoyu,hym8563"; ++ reg = <0x51>; ++ #clock-cells = <0>; ++ clock-frequency = <32768>; ++ clock-output-names = "xin32k"; ++ /* rtc_int is not connected */ ++ }; ++}; ++ ++&io_domains { ++ status = "ok"; ++ ++ dvp-supply = <&vccio_1v8_reg>; /* DVPIO_VDD */ ++ /*flash0-supply = <&vcc18_flash>;*/ /* FLASH0_VDD (emmc) */ ++ sdcard-supply = <&vccio_3v3_reg>; /* SDMMC0_VDD (sdmmc) */ ++ ++ audio-supply = <&vccio_3v3_reg>; /* APIO3_VDD */ ++ gpio30-supply = <&vccio_3v3_reg>; /* APIO1_VDD */ ++ gpio1830-supply = <&vccio_3v3_reg>; /* APIO4_VDD (gpujtag) */ ++ wifi-supply = <&vccio_3v3_reg>; /* APIO2_VDD (sdio0) */ ++}; ++ ++&sdio0 { ++ bus-width = <4>; ++ cap-sd-highspeed; ++ cap-sdio-irq; ++ keep-power-in-suspend; ++ max-frequency = <100000000>; ++ non-removable; ++ num-slots = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>; ++ sd-uhs-sdr104; ++ no-sd; ++ no-mmc; ++ status = "okay"; ++}; ++ ++&pinctrl { ++ pcfg_pull_none_drv_8ma: pcfg-pull-none-drv-8ma { ++ bias-disable; ++ drive-strength = <8>; ++ }; ++ ++ pcfg_pull_up_drv_8ma: pcfg-pull-up-drv-8ma { ++ bias-pull-up; ++ drive-strength = <8>; ++ }; ++ ++ pmic { ++ pmic_int: pmic-int { ++ rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ vsel1_gpio:vsel1_gpio{ ++ rockchip,pins = <0 RK_PA0 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ emmc { ++ emmc_bus8: emmc-bus8 { ++ rockchip,pins = <1 RK_PC2 2 &pcfg_pull_up_drv_8ma>, ++ <1 RK_PC3 2 &pcfg_pull_up_drv_8ma>, ++ <1 RK_PC4 2 &pcfg_pull_up_drv_8ma>, ++ <1 RK_PC5 2 &pcfg_pull_up_drv_8ma>, ++ <1 RK_PC6 2 &pcfg_pull_up_drv_8ma>, ++ <1 RK_PC7 2 &pcfg_pull_up_drv_8ma>, ++ <1 RK_PD0 2 &pcfg_pull_up_drv_8ma>, ++ <1 RK_PD1 2 &pcfg_pull_up_drv_8ma>; ++ }; ++ ++ emmc-clk { ++ rockchip,pins = <2 RK_PA4 2 &pcfg_pull_none_drv_8ma>; ++ }; ++ ++ emmc-cmd { ++ rockchip,pins = <1 RK_PD2 2 &pcfg_pull_up_drv_8ma>; ++ }; ++ ++ emmc_reset: emmc-reset { ++ rockchip,pins = <2 RK_PA3 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ ir { ++ ir_int: ir-int { ++ rockchip,pins = <3 RK_PD6 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ keys { ++ pwr_key: pwr-key { ++ rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ leds { ++ stby_pwren: stby-pwren { ++ rockchip,pins = <0 RK_PB4 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ led_ctl: led-ctl { ++ rockchip,pins = <3 RK_PD5 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart0_gpios: uart0-gpios { ++ rockchip,pins = <2 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ usb { ++ host_vbus_drv: host-vbus-drv { ++ rockchip,pins = <0 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ ++ pmu-supply = <&vccio_3v3_reg>; ++ vop-supply = <&vccio_3v3_reg>; ++}; ++ ++&saradc { ++ vref-supply = <&vccio_1v8_reg>; ++ status = "okay"; ++}; ++ ++&uart2 { ++ status = "okay"; ++}; ++ ++&usb_host0_ehci { ++ status = "okay"; ++}; ++ ++&usb_host0_ohci { ++ status = "okay"; ++}; ++ ++&usb_otg { ++ dr_mode = "device"; ++ status = "okay"; ++}; ++ ++&u2phy { ++ status = "okay"; ++}; ++ ++&u2phy_host { ++ status = "okay"; ++}; ++ ++&wdt { ++ status = "okay"; ++}; ++ ++&route_hdmi { ++ status = "okay"; ++}; ++ ++&hdmi { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++}; ++ ++&mailbox { ++ status = "okay"; ++}; ++ ++&mailbox_scpi { ++ status = "okay"; ++}; ++ ++&tsadc { ++ tsadc-supply = <&vdd_cpu>; ++ status = "okay"; ++}; ++ ++&cpu_l0 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu_l1 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu_l2 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu_l3 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu_b0 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu_b1 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu_b2 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu_b3 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&gpu { ++ logic-supply = <&vdd_gpu>; ++}; ++ ++&rockchip_suspend { ++ status = "okay"; ++ rockchip,sleep-mode-config = < ++ (0 ++ | RKPM_SLP_ARMOFF ++ | RKPM_SLP_PMU_PLLS_PWRDN ++ | RKPM_SLP_PMU_PMUALIVE_32K ++ | RKPM_SLP_SFT_PLLS_DEEP ++ | RKPM_SLP_PMU_DIS_OSC ++ | RKPM_SLP_SFT_PD_NBSCUS ++ ) ++ >; ++ rockchip,wakeup-config = < ++ (0 ++ | RKPM_GPIO_WKUP_EN ++ | RKPM_USB_WKUP_EN ++ | RKPM_CLUSTER_L_WKUP_EN ++ ) ++ >; ++}; ++ ++&pwm3 { ++ status = "okay"; ++ ++ interrupts = ; ++ compatible = "rockchip,remotectl-pwm"; ++ remote_pwm_id = <3>; ++ handle_cpu_id = <1>; ++ ++ ir_key1 { ++ rockchip,usercode = <0x4040>; ++ rockchip,key_table = ++ <0xf2 KEY_REPLY>, ++ <0xba KEY_BACK>, ++ <0xf4 KEY_UP>, ++ <0xf1 KEY_DOWN>, ++ <0xef KEY_LEFT>, ++ <0xee KEY_RIGHT>, ++ <0xbd KEY_HOME>, ++ <0xea KEY_VOLUMEUP>, ++ <0xe3 KEY_VOLUMEDOWN>, ++ <0xe2 KEY_SEARCH>, ++ <0xb2 KEY_POWER>, ++ <0xbc KEY_MUTE>, ++ <0xec KEY_MENU>, ++ <0xbf 0x190>, ++ <0xe0 0x191>, ++ <0xe1 0x192>, ++ <0xe9 183>, ++ <0xe6 248>, ++ <0xe8 185>, ++ <0xe7 186>, ++ <0xf0 388>, ++ <0xbe 0x175>; ++ }; ++ ++ ir_key2 { ++ rockchip,usercode = <0xff00>; ++ rockchip,key_table = ++ <0xf9 KEY_HOME>, ++ <0xbf KEY_BACK>, ++ <0xfb KEY_MENU>, ++ <0xaa KEY_REPLY>, ++ <0xb9 KEY_UP>, ++ <0xe9 KEY_DOWN>, ++ <0xb8 KEY_LEFT>, ++ <0xea KEY_RIGHT>, ++ <0xeb KEY_VOLUMEDOWN>, ++ <0xef KEY_VOLUMEUP>, ++ <0xf7 KEY_MUTE>, ++ <0xe7 KEY_POWER>, ++ <0xfc KEY_POWER>, ++ <0xa9 KEY_VOLUMEDOWN>, ++ <0xa8 KEY_VOLUMEDOWN>, ++ <0xe0 KEY_VOLUMEDOWN>, ++ <0xa5 KEY_VOLUMEDOWN>, ++ <0xab 183>, ++ <0xb7 388>, ++ <0xe8 388>, ++ <0xf8 184>, ++ <0xaf 185>, ++ <0xed KEY_VOLUMEDOWN>, ++ <0xee 186>, ++ <0xb3 KEY_VOLUMEDOWN>, ++ <0xb3 KEY_VOLUMEDOWN>, ++ <0xf1 KEY_VOLUMEDOWN>, ++ <0xf2 KEY_VOLUMEDOWN>, ++ <0xf3 KEY_SEARCH>, ++ <0xb4 KEY_VOLUMEDOWN>, ++ <0xbe KEY_SEARCH>; ++ }; ++ ++ ir_key3 { ++ rockchip,usercode = <0x1dcc>; ++ rockchip,key_table = ++ <0xee KEY_REPLY>, ++ <0xf0 KEY_BACK>, ++ <0xf8 KEY_UP>, ++ <0xbb KEY_DOWN>, ++ <0xef KEY_LEFT>, ++ <0xed KEY_RIGHT>, ++ <0xfc KEY_HOME>, ++ <0xf1 KEY_VOLUMEUP>, ++ <0xfd KEY_VOLUMEDOWN>, ++ <0xb7 KEY_SEARCH>, ++ <0xff KEY_POWER>, ++ <0xf3 KEY_MUTE>, ++ <0xbf KEY_MENU>, ++ <0xf9 0x191>, ++ <0xf5 0x192>, ++ <0xb3 388>, ++ <0xbe KEY_1>, ++ <0xba KEY_2>, ++ <0xb2 KEY_3>, ++ <0xbd KEY_4>, ++ <0xf9 KEY_5>, ++ <0xb1 KEY_6>, ++ <0xfc KEY_7>, ++ <0xf8 KEY_8>, ++ <0xb0 KEY_9>, ++ <0xb6 KEY_0>, ++ <0xb5 KEY_BACKSPACE>; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3368-sheep-lvds.dts b/arch/arm64/boot/dts/rockchip/rk3368-sheep-lvds.dts +new file mode 100755 +index 000000000000..b7b11f895cab +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3368-sheep-lvds.dts +@@ -0,0 +1,662 @@ ++/* ++ * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This file is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This file is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++/dts-v1/; ++#include ++#include "rk3368.dtsi" ++#include "rk3368-android.dtsi" ++ ++/ { ++ model = "Rockchip Sheep board"; ++ compatible = "rockchip,sheep", "rockchip,rk3368"; ++ ++ sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,rt5640-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,widgets = ++ "Microphone", "Mic Jack", ++ "Headphone", "Headphone Jack"; ++ simple-audio-card,routing = ++ "Mic Jack", "MICBIAS1", ++ "IN1P", "Mic Jack", ++ "Headphone Jack", "HPOL", ++ "Headphone Jack", "HPOR"; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s_8ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&rt5640>; ++ }; ++ }; ++ ++ backlight: backlight { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm0 0 25000 PWM_POLARITY_INVERTED>; ++ brightness-levels = < ++ 135 135 136 136 137 137 138 138 ++ 139 139 140 140 141 141 142 142 ++ 143 143 143 144 144 145 145 146 ++ 146 147 147 148 148 149 149 150 ++ 150 151 151 151 152 152 153 153 ++ 154 154 155 155 156 156 157 157 ++ 158 158 159 159 159 160 160 161 ++ 161 162 162 163 163 164 164 165 ++ 165 166 166 167 167 167 168 168 ++ 169 169 170 170 171 171 172 172 ++ 173 173 174 174 175 175 175 176 ++ 176 177 177 178 178 179 179 180 ++ 180 181 181 182 182 183 183 183 ++ 184 184 185 185 186 186 187 187 ++ 188 188 189 189 190 190 191 191 ++ 191 192 192 193 193 194 194 195 ++ 195 196 196 197 197 198 198 199 ++ 199 199 200 200 201 201 202 202 ++ 203 203 204 204 205 205 206 206 ++ 207 207 207 208 208 209 209 210 ++ 210 211 211 212 212 213 213 214 ++ 214 215 215 215 216 216 217 217 ++ 218 218 219 219 220 220 221 221 ++ 222 222 223 223 223 224 224 225 ++ 225 226 226 227 227 228 228 229 ++ 229 230 230 231 231 231 232 232 ++ 233 233 234 234 235 235 236 236 ++ 237 237 238 238 239 239 239 240 ++ 240 241 241 242 242 243 243 244 ++ 244 245 245 246 246 247 247 247 ++ 248 248 249 249 250 250 251 251 ++ 252 252 253 253 254 254 255 255>; ++ default-brightness-level = <200>; ++ enable-gpios = <&gpio0 20 GPIO_ACTIVE_HIGH>; ++ }; ++ ++ rk_key: rockchip-key { ++ compatible = "rockchip,key"; ++ status = "okay"; ++ ++ io-channels = <&saradc 1>; ++ ++ vol-up-key { ++ linux,code = <115>; ++ label = "volume up"; ++ rockchip,adc_value = <1>; ++ }; ++ ++ vol-down-key { ++ linux,code = <114>; ++ label = "volume down"; ++ rockchip,adc_value = <170>; ++ }; ++ ++ power-key { ++ gpios = <&gpio0 2 GPIO_ACTIVE_LOW>; ++ linux,code = <116>; ++ label = "power"; ++ gpio-key,wakeup; ++ }; ++ ++ menu-key { ++ linux,code = <59>; ++ label = "menu"; ++ rockchip,adc_value = <355>; ++ }; ++ ++ home-key { ++ linux,code = <102>; ++ label = "home"; ++ rockchip,adc_value = <746>; ++ }; ++ ++ back-key { ++ linux,code = <158>; ++ label = "back"; ++ rockchip,adc_value = <560>; ++ }; ++ ++ camera-key { ++ linux,code = <212>; ++ label = "camera"; ++ rockchip,adc_value = <450>; ++ }; ++ }; ++ ++ vcc_sys: vcc-sys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3800000>; ++ regulator-max-microvolt = <3800000>; ++ }; ++ ++ vcc_host: vcc-host { ++ compatible = "regulator-fixed"; ++ enable-active-high; ++ gpio = <&gpio0 4 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&host_vbus_drv>; ++ regulator-name = "vcc_host"; ++ regulator-always-on; ++ }; ++ ++ xin32k: xin32k { ++ compatible = "fixed-clock"; ++ clock-frequency = <32768>; ++ clock-output-names = "xin32k"; ++ #clock-cells = <0>; ++ }; ++ ++ panel { ++ compatible = "simple-panel"; ++ backlight = <&backlight>; ++ enable-gpios = <&gpio0 22 GPIO_ACTIVE_HIGH>; ++ bus-format = ; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <54000000>; ++ hactive = <1024>; ++ vactive = <600>; ++ hback-porch = <134>; ++ hfront-porch = <134>; ++ vback-porch = <10>; ++ vfront-porch = <10>; ++ hsync-len = <134>; ++ vsync-len = <10>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ port { ++ panel_in_lvds: endpoint { ++ remote-endpoint = <&lvds_out_panel>; ++ }; ++ }; ++ }; ++}; ++ ++&firmware_android { ++ compatible = "android,firmware"; ++ fstab { ++ compatible = "android,fstab"; ++ system { ++ compatible = "android,system"; ++ dev = "/dev/block/by-name/system"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,verify"; ++ }; ++ vendor { ++ compatible = "android,vendor"; ++ dev = "/dev/block/by-name/vendor"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,verify"; ++ }; ++ }; ++}; ++ ++&emmc { ++ status = "okay"; ++ bus-width = <8>; ++ cap-mmc-highspeed; ++ mmc-hs200-1_8v; ++ no-sdio; ++ no-sd; ++ disable-wp; ++ non-removable; ++ num-slots = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&emmc_clk &emmc_cmd &emmc_bus8>; ++}; ++ ++&sdmmc { ++ status = "okay"; ++ clock-frequency = <37500000>; ++ clock-freq-min-max = <400000 37500000>; ++ no-sdio; ++ no-mmc; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ card-detect-delay = <200>; ++ disable-wp; ++ num-slots = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_cd &sdmmc_bus4>; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ ++ syr827: syr827@40 { ++ compatible = "silergy,syr827"; ++ reg = <0x40>; ++ status = "okay"; ++ ++ regulator-compatible = "fan53555-reg"; ++ regulator-name = "vdd_arm"; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1500000>; ++ regulator-ramp-delay = <1000>; ++ fcs,suspend-voltage-selector = <1>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-state = <3>; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <900000>; ++ }; ++ }; ++ ++ rk818: pmic@1c { ++ compatible = "rockchip,rk818"; ++ reg = <0x1c>; ++ status = "okay"; ++ ++ clock-output-names = "rk818-clkout1", "wifibt_32kin"; ++ interrupt-parent = <&gpio0>; ++ interrupts = <1 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pmic_int_l>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ ++ vcc1-supply = <&vcc_sys>; ++ vcc2-supply = <&vcc_sys>; ++ vcc3-supply = <&vcc_sys>; ++ vcc4-supply = <&vcc_sys>; ++ vcc6-supply = <&vcc_sys>; ++ vcc7-supply = <&vcc_sys>; ++ vcc8-supply = <&vcc_sys>; ++ vcc9-supply = <&vcc_io>; ++ ++ regulators { ++ vdd_logic: DCDC_REG1 { ++ regulator-name = "vdd_logic"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <750000>; ++ regulator-max-microvolt = <1450000>; ++ regulator-ramp-delay = <6001>; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vdd_gpu: DCDC_REG2 { ++ regulator-name = "vdd_gpu"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <800000>; ++ regulator-max-microvolt = <1250000>; ++ regulator-ramp-delay = <6001>; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc_ddr"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_io: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcc_io"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcca_codec: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcca_codec"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc_tp: LDO_REG2 { ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-name = "vcc_tp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdd_10: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ regulator-name = "vdd_10"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc18_lcd: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc18_lcd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vccio_pmu: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vccio_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd10_lcd: LDO_REG6 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ regulator-name = "vdd10_lcd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc_18: LDO_REG7 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc_18"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vccio_wl: LDO_REG8 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vccio_wl"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vccio_sd: LDO_REG9 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vccio_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc_sd: SWITCH_REG { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ boost_otg: DCDC_BOOST { ++ regulator-name = "boost_otg"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <5000000>; ++ }; ++ }; ++ }; ++ }; ++}; ++ ++&cpu_l0 { ++ cpu-supply = <&syr827>; ++}; ++ ++&cpu_l1 { ++ cpu-supply = <&syr827>; ++}; ++ ++&cpu_l2 { ++ cpu-supply = <&syr827>; ++}; ++ ++&cpu_l3 { ++ cpu-supply = <&syr827>; ++}; ++ ++&cpu_b0 { ++ cpu-supply = <&syr827>; ++}; ++ ++&cpu_b1 { ++ cpu-supply = <&syr827>; ++}; ++ ++&cpu_b2 { ++ cpu-supply = <&syr827>; ++}; ++ ++&cpu_b3 { ++ cpu-supply = <&syr827>; ++}; ++ ++&gpu { ++ logic-supply = <&vdd_logic>; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ ++ rt5640: rt5640@1c { ++ compatible = "realtek,rt5640"; ++ reg = <0x1c>; ++ #sound-dai-cells = <0>; ++ clocks = <&cru SCLK_I2S_8CH_OUT>; ++ clock-names = "mclk"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s_8ch_mclk>; ++ realtek,in1-differential; ++ status = "okay"; ++ }; ++}; ++ ++&i2c2 { ++ status = "okay"; ++ ++ gt9xx: gt9xx@14 { ++ compatible = "goodix,gt9xx"; ++ reg = <0x14>; ++ touch-gpio = <&gpio0 12 IRQ_TYPE_LEVEL_LOW>; ++ reset-gpio = <&gpio0 11 GPIO_ACTIVE_HIGH>; ++ max-x = <1200>; ++ max-y = <1900>; ++ tp-size = <911>; ++ tp-supply = <&vcc_tp>; ++ status = "okay"; ++ }; ++}; ++ ++&i2s_8ch { ++ status = "okay"; ++ rockchip,i2s-broken-burst-len; ++ rockchip,playback-channels = <8>; ++ rockchip,capture-channels = <2>; ++ #sound-dai-cells = <0>; ++}; ++ ++&io_domains { ++ status = "okay"; ++ dvp-supply = <&vcc_18>; ++ audio-supply = <&vcc_io>; ++ gpio30-supply = <&vcc_io>; ++ gpio1830-supply = <&vcc_io>; ++ sdcard-supply = <&vccio_sd>; ++ wifi-supply = <&vccio_wl>; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ pmu-supply = <&vcc_io>; ++ vop-supply = <&vcc_io>; ++}; ++ ++&pwm0 { ++ status = "okay"; ++}; ++ ++&u2phy { ++ status = "okay"; ++ ++ u2phy_host: host-port { ++ phy-supply = <&vcc_host>; ++ status = "okay"; ++ }; ++}; ++ ++&usb_host0_ehci { ++ status = "okay"; ++}; ++ ++&usb_host0_ohci { ++ status = "okay"; ++}; ++ ++&mailbox { ++ status = "okay"; ++}; ++ ++&mailbox_scpi { ++ status = "okay"; ++}; ++ ++&lvds { ++ status = "okay"; ++ ++ ports { ++ port@1 { ++ reg = <1>; ++ ++ lvds_out_panel: endpoint { ++ remote-endpoint = <&panel_in_lvds>; ++ }; ++ }; ++ }; ++ ++}; ++ ++&route_lvds { ++ status = "okay"; ++}; ++ ++&saradc { ++ status = "okay"; ++}; ++ ++&tsadc { ++ tsadc-supply = <&syr827>; ++ status = "okay"; ++}; ++ ++&hdmi { ++ status = "okay"; ++}; ++ ++&pinctrl { ++ pmic { ++ pmic_int_l: pmic-int-l { ++ rockchip,pins = <0 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ usb2 { ++ host_vbus_drv: host-vbus-drv { ++ rockchip,pins = <0 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3368-sheep.dts b/arch/arm64/boot/dts/rockchip/rk3368-sheep.dts +new file mode 100755 +index 000000000000..b9b6b24ff37b +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3368-sheep.dts +@@ -0,0 +1,746 @@ ++/* ++ * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This file is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This file is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++/dts-v1/; ++#include ++#include "rk3368.dtsi" ++#include "rk3368-android.dtsi" ++ ++/ { ++ model = "Rockchip Sheep board"; ++ compatible = "rockchip,sheep", "rockchip,rk3368"; ++ ++ sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,rt5640-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,widgets = ++ "Microphone", "Mic Jack", ++ "Headphone", "Headphone Jack"; ++ simple-audio-card,routing = ++ "Mic Jack", "MICBIAS1", ++ "IN1P", "Mic Jack", ++ "Headphone Jack", "HPOL", ++ "Headphone Jack", "HPOR"; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s_8ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&rt5640>; ++ }; ++ }; ++ ++ backlight: backlight { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm0 0 25000 PWM_POLARITY_INVERTED>; ++ brightness-levels = < ++ 135 135 136 136 137 137 138 138 ++ 139 139 140 140 141 141 142 142 ++ 143 143 143 144 144 145 145 146 ++ 146 147 147 148 148 149 149 150 ++ 150 151 151 151 152 152 153 153 ++ 154 154 155 155 156 156 157 157 ++ 158 158 159 159 159 160 160 161 ++ 161 162 162 163 163 164 164 165 ++ 165 166 166 167 167 167 168 168 ++ 169 169 170 170 171 171 172 172 ++ 173 173 174 174 175 175 175 176 ++ 176 177 177 178 178 179 179 180 ++ 180 181 181 182 182 183 183 183 ++ 184 184 185 185 186 186 187 187 ++ 188 188 189 189 190 190 191 191 ++ 191 192 192 193 193 194 194 195 ++ 195 196 196 197 197 198 198 199 ++ 199 199 200 200 201 201 202 202 ++ 203 203 204 204 205 205 206 206 ++ 207 207 207 208 208 209 209 210 ++ 210 211 211 212 212 213 213 214 ++ 214 215 215 215 216 216 217 217 ++ 218 218 219 219 220 220 221 221 ++ 222 222 223 223 223 224 224 225 ++ 225 226 226 227 227 228 228 229 ++ 229 230 230 231 231 231 232 232 ++ 233 233 234 234 235 235 236 236 ++ 237 237 238 238 239 239 239 240 ++ 240 241 241 242 242 243 243 244 ++ 244 245 245 246 246 247 247 247 ++ 248 248 249 249 250 250 251 251 ++ 252 252 253 253 254 254 255 255>; ++ default-brightness-level = <200>; ++ enable-gpios = <&gpio0 20 GPIO_ACTIVE_HIGH>; ++ }; ++ ++ rk_key: rockchip-key { ++ compatible = "rockchip,key"; ++ status = "okay"; ++ ++ io-channels = <&saradc 1>; ++ ++ vol-up-key { ++ linux,code = <115>; ++ label = "volume up"; ++ rockchip,adc_value = <1>; ++ }; ++ ++ vol-down-key { ++ linux,code = <114>; ++ label = "volume down"; ++ rockchip,adc_value = <170>; ++ }; ++ ++ power-key { ++ gpios = <&gpio0 2 GPIO_ACTIVE_LOW>; ++ linux,code = <116>; ++ label = "power"; ++ gpio-key,wakeup; ++ }; ++ ++ menu-key { ++ linux,code = <59>; ++ label = "menu"; ++ rockchip,adc_value = <355>; ++ }; ++ ++ home-key { ++ linux,code = <102>; ++ label = "home"; ++ rockchip,adc_value = <746>; ++ }; ++ ++ back-key { ++ linux,code = <158>; ++ label = "back"; ++ rockchip,adc_value = <560>; ++ }; ++ ++ camera-key { ++ linux,code = <212>; ++ label = "camera"; ++ rockchip,adc_value = <450>; ++ }; ++ }; ++ ++ vcc_sys: vcc-sys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3800000>; ++ regulator-max-microvolt = <3800000>; ++ }; ++ ++ vcc_host: vcc-host { ++ compatible = "regulator-fixed"; ++ enable-active-high; ++ gpio = <&gpio0 4 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&host_vbus_drv>; ++ regulator-name = "vcc_host"; ++ regulator-always-on; ++ }; ++ ++ vcc_otg_vbus: otg-vbus-regulator { ++ compatible = "regulator-fixed"; ++ gpio = <&gpio0 RK_PD1 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&otg_vbus_drv>; ++ regulator-name = "vcc_otg_vbus"; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ enable-active-high; ++ }; ++ ++ xin32k: xin32k { ++ compatible = "fixed-clock"; ++ clock-frequency = <32768>; ++ clock-output-names = "xin32k"; ++ #clock-cells = <0>; ++ }; ++}; ++ ++&firmware_android { ++ compatible = "android,firmware"; ++ fstab { ++ compatible = "android,fstab"; ++ system { ++ compatible = "android,system"; ++ dev = "/dev/block/by-name/system"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,verify"; ++ }; ++ vendor { ++ compatible = "android,vendor"; ++ dev = "/dev/block/by-name/vendor"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,verify"; ++ }; ++ }; ++}; ++ ++&emmc { ++ status = "okay"; ++ bus-width = <8>; ++ cap-mmc-highspeed; ++ mmc-hs200-1_8v; ++ no-sdio; ++ no-sd; ++ disable-wp; ++ non-removable; ++ num-slots = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&emmc_clk &emmc_cmd &emmc_bus8>; ++}; ++ ++&sdmmc { ++ status = "okay"; ++ clock-frequency = <37500000>; ++ clock-freq-min-max = <400000 37500000>; ++ no-sdio; ++ no-mmc; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ card-detect-delay = <200>; ++ disable-wp; ++ num-slots = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_cd &sdmmc_bus4>; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ ++ syr827: syr827@40 { ++ compatible = "silergy,syr827"; ++ reg = <0x40>; ++ status = "okay"; ++ ++ regulator-compatible = "fan53555-reg"; ++ regulator-name = "vdd_arm"; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1500000>; ++ regulator-ramp-delay = <1000>; ++ fcs,suspend-voltage-selector = <1>; ++ pinctrl-0 = <&vsel_gpio>; ++ vsel-gpios = <&gpio0 0 GPIO_ACTIVE_HIGH>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-state = <3>; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <900000>; ++ }; ++ }; ++ ++ rk818: pmic@1c { ++ compatible = "rockchip,rk818"; ++ reg = <0x1c>; ++ status = "okay"; ++ ++ clock-output-names = "rk818-clkout1", "wifibt_32kin"; ++ interrupt-parent = <&gpio0>; ++ interrupts = <1 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pmic_int_l>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ ++ vcc1-supply = <&vcc_sys>; ++ vcc2-supply = <&vcc_sys>; ++ vcc3-supply = <&vcc_sys>; ++ vcc4-supply = <&vcc_sys>; ++ vcc6-supply = <&vcc_sys>; ++ vcc7-supply = <&vcc_sys>; ++ vcc8-supply = <&vcc_sys>; ++ vcc9-supply = <&vcc_io>; ++ ++ regulators { ++ vdd_logic: DCDC_REG1 { ++ regulator-name = "vdd_logic"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <750000>; ++ regulator-max-microvolt = <1450000>; ++ regulator-ramp-delay = <6001>; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vdd_gpu: DCDC_REG2 { ++ regulator-name = "vdd_gpu"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <800000>; ++ regulator-max-microvolt = <1250000>; ++ regulator-ramp-delay = <6001>; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc_ddr"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_io: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcc_io"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcca_codec: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcca_codec"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc_tp: LDO_REG2 { ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-name = "vcc_tp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdd_10: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ regulator-name = "vdd_10"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc18_lcd: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc18_lcd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vccio_pmu: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vccio_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd10_lcd: LDO_REG6 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ regulator-name = "vdd10_lcd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc_18: LDO_REG7 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc_18"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vccio_wl: LDO_REG8 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vccio_wl"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vccio_sd: LDO_REG9 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vccio_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc_sd: SWITCH_REG { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ boost_otg: DCDC_BOOST { ++ regulator-name = "boost_otg"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <5000000>; ++ }; ++ }; ++ }; ++ ++ battery { ++ compatible = "rk818-battery"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&dc_irq_gpio>; ++ ocv_table = < ++ 3400 3650 3693 3707 3731 3749 3760 ++ 3770 3782 3796 3812 3829 3852 3882 ++ 3915 3951 3981 4047 4086 4132 4182>; ++ design_capacity = <8650>; ++ design_qmax = <8800>; ++ bat_res = <85>; ++ max_input_current = <2000>; ++ max_chrg_current = <1800>; ++ max_chrg_voltage = <4200>; ++ sleep_enter_current = <600>; ++ sleep_exit_current = <600>; ++ power_off_thresd = <3400>; ++ zero_algorithm_vol = <3850>; ++ fb_temperature = <115>; ++ sample_res = <10>; ++ max_soc_offset = <60>; ++ energy_mode = <0>; ++ monitor_sec = <5>; ++ virtual_power = <0>; ++ power_dc2otg = <1>; ++ support_usb_adp = <1>; ++ support_dc_adp = <1>; ++ dc_det_gpio = <&gpio0 17 GPIO_ACTIVE_LOW>; ++ }; ++ }; ++}; ++ ++&cpu_l0 { ++ cpu-supply = <&syr827>; ++}; ++ ++&cpu_l1 { ++ cpu-supply = <&syr827>; ++}; ++ ++&cpu_l2 { ++ cpu-supply = <&syr827>; ++}; ++ ++&cpu_l3 { ++ cpu-supply = <&syr827>; ++}; ++ ++&cpu_b0 { ++ cpu-supply = <&syr827>; ++}; ++ ++&cpu_b1 { ++ cpu-supply = <&syr827>; ++}; ++ ++&cpu_b2 { ++ cpu-supply = <&syr827>; ++}; ++ ++&cpu_b3 { ++ cpu-supply = <&syr827>; ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&dmc { ++ center-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&gpu { ++ logic-supply = <&vdd_logic>; ++}; ++ ++&rockchip_suspend { ++ status = "okay"; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ ++ rt5640: rt5640@1c { ++ compatible = "realtek,rt5640"; ++ reg = <0x1c>; ++ #sound-dai-cells = <0>; ++ clocks = <&cru SCLK_I2S_8CH_OUT>; ++ clock-names = "mclk"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s_8ch_mclk>; ++ realtek,in1-differential; ++ status = "okay"; ++ }; ++}; ++ ++&i2c2 { ++ status = "okay"; ++ ++ gt9xx: gt9xx@14 { ++ compatible = "goodix,gt9xx"; ++ reg = <0x14>; ++ touch-gpio = <&gpio0 12 IRQ_TYPE_LEVEL_LOW>; ++ reset-gpio = <&gpio0 11 GPIO_ACTIVE_HIGH>; ++ max-x = <1200>; ++ max-y = <1900>; ++ tp-size = <911>; ++ tp-supply = <&vcc_tp>; ++ status = "okay"; ++ }; ++}; ++ ++&i2s_8ch { ++ status = "okay"; ++ rockchip,i2s-broken-burst-len; ++ rockchip,playback-channels = <8>; ++ rockchip,capture-channels = <2>; ++ #sound-dai-cells = <0>; ++}; ++ ++&io_domains { ++ status = "okay"; ++ dvp-supply = <&vcc_18>; ++ audio-supply = <&vcc_io>; ++ gpio30-supply = <&vcc_io>; ++ gpio1830-supply = <&vcc_io>; ++ sdcard-supply = <&vccio_sd>; ++ wifi-supply = <&vccio_wl>; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ ++ pmu-supply = <&vcc_io>; ++ vop-supply = <&vcc_io>; ++}; ++ ++&pwm0 { ++ status = "okay"; ++}; ++ ++&u2phy { ++ status = "okay"; ++ ++ u2phy_host: host-port { ++ phy-supply = <&vcc_host>; ++ status = "okay"; ++ }; ++ ++ u2phy_otg: otg-port { ++ vbus-supply = <&vcc_otg_vbus>; ++ status = "okay"; ++ }; ++}; ++ ++&usb_host0_ehci { ++ status = "okay"; ++}; ++ ++&usb_host0_ohci { ++ status = "okay"; ++}; ++ ++&mailbox { ++ status = "okay"; ++}; ++ ++&mailbox_scpi { ++ status = "okay"; ++}; ++ ++&dsi { ++ status = "okay"; ++ ++ panel@0 { ++ compatible = "simple-panel-dsi"; ++ reg = <0>; ++ backlight = <&backlight>; ++ enable-gpios = <&gpio0 22 GPIO_ACTIVE_HIGH>; ++ prepare-delay-ms = <120>; ++ ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | ++ MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <150000000>; ++ hactive = <1200>; ++ vactive = <1920>; ++ hback-porch = <80>; ++ hfront-porch = <81>; ++ vback-porch = <21>; ++ vfront-porch = <21>; ++ hsync-len = <10>; ++ vsync-len = <3>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ panel_in_dsi: endpoint { ++ remote-endpoint = <&dsi_out_panel>; ++ }; ++ }; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@1 { ++ reg = <1>; ++ dsi_out_panel: endpoint { ++ remote-endpoint = <&panel_in_dsi>; ++ }; ++ }; ++ }; ++}; ++ ++&route_dsi { ++ status = "okay"; ++}; ++ ++&saradc { ++ status = "okay"; ++}; ++ ++&tsadc { ++ tsadc-supply = <&syr827>; ++ status = "okay"; ++}; ++ ++&pinctrl { ++ pmic { ++ pmic_int_l: pmic-int-l { ++ rockchip,pins = <0 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ vsel_gpio: vsel-gpio { ++ rockchip,pins = <0 RK_PA0 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ dc_det { ++ dc_irq_gpio: dc-irq-gpio { ++ rockchip,pins = <0 RK_PC1 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ usb2 { ++ host_vbus_drv: host-vbus-drv { ++ rockchip,pins = <0 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ otg_vbus_drv: otg-bus-drv { ++ rockchip,pins = <0 RK_PD1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3368-sziauto-rk618.dts b/arch/arm64/boot/dts/rockchip/rk3368-sziauto-rk618.dts +new file mode 100755 +index 000000000000..eff7c2fed990 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3368-sziauto-rk618.dts +@@ -0,0 +1,808 @@ ++/* ++ * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This file is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This file is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++/dts-v1/; ++#include "rk3368.dtsi" ++#include "rk3368-android.dtsi" ++#include ++#include ++ ++/ { ++ model = "Rockchip rk3368 Sziauto board"; ++ compatible = "rockchip,sziauto", "rockchip,rk3368"; ++ ++ panel { ++ compatible = "simple-panel"; ++ backlight = <&backlight>; ++ enable-gpios = <&gpio3 13 GPIO_ACTIVE_LOW>; ++ prepare-delay-ms = <20>; ++ enable-delay-ms = <20>; ++ disable-delay-ms = <20>; ++ unprepare-delay-ms = <20>; ++ bus-format = ; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <136000000>; ++ hactive = <1920>; ++ vactive = <1080>; ++ hback-porch = <60>; ++ hfront-porch = <60>; ++ hsync-len = <40>; ++ vback-porch = <4>; ++ vfront-porch = <4>; ++ vsync-len = <3>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ port { ++ panel_in_lvds: endpoint { ++ remote-endpoint = <&lvds_out_panel>; ++ }; ++ }; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ clocks = <&rk818 1>; ++ clock-names = "ext_clock"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio3 4 GPIO_ACTIVE_LOW>; ++ }; ++ ++ backlight: backlight { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm0 0 50000 0>; ++ brightness-levels = < ++ 32 32 34 34 36 36 38 38 40 40 ++ 42 42 44 44 46 46 48 48 50 50 ++ 52 52 54 54 56 56 58 58 60 60 ++ 62 62 64 64 66 66 68 68 70 70 ++ 72 72 74 74 76 76 78 78 80 80 ++ 82 82 84 84 86 86 88 88 90 90 ++ 92 92 94 94 96 96 98 98 100 100 ++ 102 102 104 104 106 106 108 108 110 110 ++ 112 112 114 114 116 116 118 118 120 120 ++ 122 122 124 124 126 126 128 128 130 130 ++ 132 132 134 134 136 136 138 138 140 140 ++ 142 142 144 144 146 146 148 148 150 150 ++ 152 152 154 154 156 156 158 158 160 160 ++ 162 162 164 164 166 166 168 168 170 170 ++ 172 172 174 174 176 176 178 178 180 180 ++ 182 182 184 184 186 186 188 188 190 190 ++ 192 192 194 194 196 196 198 198 200 200 ++ 202 202 204 204 206 206 208 208 210 210 ++ 212 212 214 214 216 216 218 218 220 220 ++ 222 222 224 224 225 225 226 226 227 227 ++ 228 228 229 229 230 230 231 231 232 232 ++ 233 233 234 234 235 235 236 236 237 237 ++ 238 238 239 239 240 240 241 241 242 242 ++ 243 243 244 244 245 245 246 246 247 247 ++ 248 248 249 249 250 250 251 251 252 252 ++ 253 253 254 254 255 255>; ++ default-brightness-level = <120>; ++ enable-gpios = <&gpio3 28 GPIO_ACTIVE_HIGH>; ++ }; ++ ++ rockchip-key { ++ compatible = "rockchip,key"; ++ io-channels = <&saradc 1>; ++ status = "okay"; ++ ++ vol-up-key { ++ linux,code = <115>; ++ label = "volume up"; ++ rockchip,adc_value = <1>; ++ }; ++ ++ vol-down-key { ++ linux,code = <114>; ++ label = "volume down"; ++ rockchip,adc_value = <170>; ++ }; ++ ++ power-key { ++ gpios = <&gpio0 2 GPIO_ACTIVE_LOW>; ++ linux,code = <116>; ++ label = "power"; ++ gpio-key,wakeup; ++ }; ++ ++ menu-key { ++ linux,code = <59>; ++ label = "menu"; ++ rockchip,adc_value = <355>; ++ }; ++ ++ home-key { ++ linux,code = <102>; ++ label = "home"; ++ rockchip,adc_value = <746>; ++ }; ++ ++ back-key { ++ linux,code = <158>; ++ label = "back"; ++ rockchip,adc_value = <560>; ++ }; ++ ++ camera-key { ++ linux,code = <212>; ++ label = "camera"; ++ rockchip,adc_value = <450>; ++ }; ++ }; ++ ++ vcc_sys: vcc-sys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3800000>; ++ regulator-max-microvolt = <3800000>; ++ }; ++ ++ vcc_host: vcc-host { ++ compatible = "regulator-fixed"; ++ enable-active-high; ++ gpio = <&gpio0 4 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&host_vbus_drv>; ++ regulator-name = "vcc_host"; ++ regulator-always-on; ++ }; ++ ++ xin32k: xin32k { ++ compatible = "fixed-clock"; ++ clock-frequency = <32768>; ++ clock-output-names = "xin32k"; ++ #clock-cells = <0>; ++ }; ++}; ++ ++&firmware_android { ++ compatible = "android,firmware"; ++ fstab { ++ compatible = "android,fstab"; ++ system { ++ compatible = "android,system"; ++ dev = "/dev/block/by-name/system"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,verify"; ++ }; ++ vendor { ++ compatible = "android,vendor"; ++ dev = "/dev/block/by-name/vendor"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,verify"; ++ }; ++ }; ++}; ++ ++&i2c5 { ++ status = "okay"; ++ ++ rk618@50 { ++ compatible = "rockchip,rk618"; ++ reg = <0x50>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s_8ch_mclk>; ++ clocks = <&cru SCLK_I2S_8CH_OUT>; ++ clock-names = "clkin"; ++ assigned-clocks = <&cru SCLK_I2S_8CH_OUT>; ++ assigned-clock-rates = <11289600>; ++ reset-gpios = <&gpio3 14 GPIO_ACTIVE_LOW>; ++ status = "okay"; ++ ++ clock: cru { ++ compatible = "rockchip,rk618-cru"; ++ clocks = <&cru SCLK_I2S_8CH_OUT>, <&cru DCLK_VOP>; ++ clock-names = "clkin", "lcdc0_dclkp"; ++ assigned-clocks = <&clock SCALER_PLLIN_CLK>, <&clock VIF_PLLIN_CLK>, ++ <&clock SCALER_CLK>, <&clock VIF0_PRE_CLK>, ++ <&clock CODEC_CLK>, <&clock DITHER_CLK>; ++ assigned-clock-parents = <&cru SCLK_I2S_8CH_OUT>, <&clock LCDC0_CLK>, ++ <&clock SCALER_PLL_CLK>, <&clock VIF_PLL_CLK>, ++ <&cru SCLK_I2S_8CH_OUT>, <&clock VIF0_CLK>; ++ #clock-cells = <1>; ++ status = "okay"; ++ }; ++ ++ hdmi { ++ compatible = "rockchip,rk618-hdmi"; ++ clocks = <&clock HDMI_CLK>; ++ clock-names = "hdmi"; ++ assigned-clocks = <&clock HDMI_CLK>; ++ assigned-clock-parents = <&clock VIF0_CLK>; ++ interrupt-parent = <&gpio3>; ++ interrupts = <23 IRQ_TYPE_LEVEL_HIGH>; ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ ++ hdmi_in_vif: endpoint { ++ remote-endpoint = <&vif_out_hdmi>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ ++ hdmi_out_scaler: endpoint { ++ remote-endpoint = <&scaler_in_hdmi>; ++ }; ++ }; ++ }; ++ }; ++ ++ lvds { ++ compatible = "rockchip,rk618-lvds"; ++ clocks = <&clock LVDS_CLK>; ++ clock-names = "lvds"; ++ dual-channel; ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ ++ lvds_in_scaler: endpoint { ++ remote-endpoint = <&scaler_out_lvds>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ ++ lvds_out_panel: endpoint { ++ remote-endpoint = <&panel_in_lvds>; ++ }; ++ }; ++ }; ++ }; ++ ++ scaler { ++ compatible = "rockchip,rk618-scaler"; ++ clocks = <&clock SCALER_CLK>, <&clock VIF0_CLK>, ++ <&clock DITHER_CLK>; ++ clock-names = "scaler", "vif", "dither"; ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ ++ scaler_in_hdmi: endpoint { ++ remote-endpoint = <&hdmi_out_scaler>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ ++ scaler_out_lvds: endpoint { ++ remote-endpoint = <&lvds_in_scaler>; ++ }; ++ }; ++ }; ++ }; ++ ++ vif { ++ compatible = "rockchip,rk618-vif"; ++ clocks = <&clock VIF0_CLK>, <&clock VIF0_PRE_CLK>; ++ clock-names = "vif", "vif_pre"; ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ ++ vif_in_rgb: endpoint { ++ remote-endpoint = <&rgb_out_vif>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ ++ vif_out_hdmi: endpoint { ++ remote-endpoint = <&hdmi_in_vif>; ++ }; ++ }; ++ }; ++ }; ++ }; ++}; ++ ++&rgb { ++ status = "okay"; ++ ++ ports { ++ port@1 { ++ reg = <1>; ++ ++ rgb_out_vif: endpoint { ++ remote-endpoint = <&vif_in_rgb>; ++ }; ++ }; ++ }; ++}; ++ ++&route_rgb { ++ status = "okay"; ++}; ++ ++&emmc { ++ status = "okay"; ++ bus-width = <8>; ++ cap-mmc-highspeed; ++ mmc-hs200-1_8v; ++ no-sdio; ++ no-sd; ++ disable-wp; ++ non-removable; ++ num-slots = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&emmc_clk &emmc_cmd &emmc_bus8>; ++}; ++ ++&sdmmc { ++ status = "disabled"; ++ clock-frequency = <37500000>; ++ clock-freq-min-max = <400000 37500000>; ++ no-sdio; ++ no-mmc; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ card-detect-delay = <200>; ++ disable-wp; ++ num-slots = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_cd &sdmmc_bus4>; ++}; ++ ++&sdio0 { ++ clock-frequency = <50000000>; ++ clock-freq-min-max = <200000 50000000>; ++ no-sd; ++ no-mmc; ++ bus-width = <4>; ++ disable-wp; ++ cap-sd-highspeed; ++ cap-sdio-irq; ++ keep-power-in-suspend; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ non-removable; ++ num-slots = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ ++ syr827: syr827@40 { ++ compatible = "silergy,syr827"; ++ reg = <0x40>; ++ fcs,suspend-voltage-selector = <1>; ++ ++ regulator-compatible = "fan53555-reg"; ++ regulator-name = "vdd_arm"; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1500000>; ++ regulator-ramp-delay = <1000>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-state = <3>; ++ ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <900000>; ++ }; ++ }; ++ ++ rk818: pmic@1c { ++ compatible = "rockchip,rk818"; ++ reg = <0x1c>; ++ clock-output-names = "rk818-clkout1", "wifibt_32kin"; ++ interrupt-parent = <&gpio0>; ++ interrupts = <1 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pmic_int_l>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ ++ vcc1-supply = <&vcc_sys>; ++ vcc2-supply = <&vcc_sys>; ++ vcc3-supply = <&vcc_sys>; ++ vcc4-supply = <&vcc_sys>; ++ vcc6-supply = <&vcc_sys>; ++ vcc7-supply = <&vcc_sys>; ++ vcc8-supply = <&vcc_sys>; ++ vcc9-supply = <&vcc_io>; ++ ++ regulators { ++ vdd_logic: DCDC_REG1 { ++ regulator-name = "vdd_logic"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <750000>; ++ regulator-max-microvolt = <1450000>; ++ regulator-ramp-delay = <6001>; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vdd_gpu: DCDC_REG2 { ++ regulator-name = "vdd_gpu"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <800000>; ++ regulator-max-microvolt = <1250000>; ++ regulator-ramp-delay = <6001>; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc_ddr"; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_io: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcc_io"; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcca_codec: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcca_codec"; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc_tp: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-name = "vcc_tp"; ++ ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdd_10: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ regulator-name = "vdd_10"; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc18_lcd: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc18_lcd"; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vccio_pmu: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vccio_pmu"; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd10_lcd: LDO_REG6 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ regulator-name = "vdd10_lcd"; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc_18: LDO_REG7 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc_18"; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vccio_wl: LDO_REG8 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vccio_wl"; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vccio_sd: LDO_REG9 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vccio_sd"; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc_sd: SWITCH_REG { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc_sd"; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ boost_otg: DCDC_BOOST { ++ regulator-name = "boost_otg"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <5000000>; ++ }; ++ }; ++ }; ++ }; ++}; ++ ++&cpu_l0 { ++ cpu-supply = <&syr827>; ++}; ++ ++&cpu_l1 { ++ cpu-supply = <&syr827>; ++}; ++ ++&cpu_l2 { ++ cpu-supply = <&syr827>; ++}; ++ ++&cpu_l3 { ++ cpu-supply = <&syr827>; ++}; ++ ++&cpu_b0 { ++ cpu-supply = <&syr827>; ++}; ++ ++&cpu_b1 { ++ cpu-supply = <&syr827>; ++}; ++ ++&cpu_b2 { ++ cpu-supply = <&syr827>; ++}; ++ ++&cpu_b3 { ++ cpu-supply = <&syr827>; ++}; ++ ++&gpu { ++ logic-supply = <&vdd_logic>; ++}; ++ ++&io_domains { ++ dvp-supply = <&vcc_18>; ++ audio-supply = <&vcc_io>; ++ gpio30-supply = <&vcc_io>; ++ gpio1830-supply = <&vcc_io>; ++ sdcard-supply = <&vccio_sd>; ++ wifi-supply = <&vccio_wl>; ++ status = "okay"; ++}; ++ ++&pmu_io_domains { ++ pmu-supply = <&vcc_io>; ++ vop-supply = <&vcc_io>; ++ status = "okay"; ++}; ++ ++&pwm0 { ++ status = "okay"; ++}; ++ ++&pwm1 { ++ status = "okay"; ++}; ++ ++&uart0 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart0_xfer &uart0_cts>; ++ status = "okay"; ++}; ++ ++&u2phy { ++ status = "okay"; ++ ++ u2phy_host: host-port { ++ phy-supply = <&vcc_host>; ++ status = "okay"; ++ }; ++}; ++ ++&usb_host0_ehci { ++ status = "okay"; ++}; ++ ++&usb_host0_ohci { ++ status = "okay"; ++}; ++ ++&mailbox { ++ status = "okay"; ++}; ++ ++&mailbox_scpi { ++ status = "okay"; ++}; ++ ++&saradc { ++ status = "okay"; ++}; ++ ++&tsadc { ++ tsadc-supply = <&syr827>; ++ status = "okay"; ++}; ++ ++&pinctrl { ++ pmic { ++ pmic_int_l: pmic-int-l { ++ rockchip,pins = <0 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ usb2 { ++ host_vbus_drv: host-vbus-drv { ++ rockchip,pins = <0 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3368-tablet.dts b/arch/arm64/boot/dts/rockchip/rk3368-tablet.dts +new file mode 100755 +index 000000000000..d7dc6d67e242 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3368-tablet.dts +@@ -0,0 +1,1070 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++#include ++#include ++#include ++#include "rk3368.dtsi" ++#include "rk3368-android.dtsi" ++/ { ++ model = "Rockchip rk3368 tablet board"; ++ compatible = "rockchip,tablet", "rockchip,rk3368"; ++ ++ adc_keys: adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 1>; ++ io-channel-names = "buttons"; ++ keyup-threshold-microvolt = <1024000>; ++ poll-interval = <100>; ++ ++ vol-up-key { ++ label = "volume up"; ++ linux,code = ; ++ press-threshold-microvolt = <1000>; ++ }; ++ ++ vol-down-key { ++ label = "volume down"; ++ linux,code = ; ++ press-threshold-microvolt = <170000>; ++ }; ++ }; ++ ++ vcc_camera: vcc-camera-regulator { ++ compatible = "regulator-fixed"; ++ gpio = <&gpio0 RK_PD0 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&camera_pwr>; ++ regulator-name = "vcc_camera"; ++ enable-active-high; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ backlight: backlight { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm0 0 25000 1>; ++ brightness-levels = < ++ 0 1 2 3 4 5 6 7 ++ 8 9 10 11 12 13 14 15 ++ 16 16 16 16 16 16 16 16 ++ 17 17 17 17 17 17 17 17 ++ 18 18 18 18 18 18 18 18 ++ 19 19 19 19 19 19 19 19 ++ 20 20 20 20 20 20 20 20 ++ 21 21 21 21 21 21 21 21 ++ 22 22 22 22 22 22 22 22 ++ 23 23 23 23 23 23 23 23 ++ 24 24 24 24 24 24 24 24 ++ 25 25 25 25 25 25 25 25 ++ 26 26 26 26 26 26 26 26 ++ 27 27 27 27 27 27 27 27 ++ 28 28 28 28 28 28 28 28 ++ 27 27 27 27 27 27 27 27 ++ 30 30 30 30 30 30 30 30 ++ 31 31 31 31 31 31 31 31 ++ 32 32 32 32 32 32 32 32 ++ 33 33 33 33 33 33 33 33 ++ 34 34 34 34 34 34 34 34 ++ 35 35 35 35 35 35 35 35 ++ 36 36 36 36 36 36 36 36 ++ 37 37 37 37 37 37 37 37 ++ 38 38 38 38 38 38 38 38 ++ 38 38 38 38 38 38 38 38 ++ 38 38 38 38 38 38 38 38 ++ 38 38 38 38 38 38 38 38 ++ 38 38 38 38 38 38 38 38 ++ 38 38 38 38 38 38 38 38 ++ 38 38 38 38 38 38 38 38 ++ 38 38 38 38 38 38 38 38>; ++ default-brightness-level = <200>; ++ enable-gpios = <&gpio0 20 GPIO_ACTIVE_HIGH>; ++ }; ++ ++ charge-animation { ++ compatible = "rockchip,uboot-charge"; ++ rockchip,uboot-charge-on = <1>; ++ rockchip,android-charge-on = <0>; ++ rockchip,uboot-low-power-voltage = <3500>; ++ rockchip,screen-on-voltage = <3600>; ++ status = "okay"; ++ }; ++ ++ chosen: chosen { ++ bootargs = "earlycon=uart8250,mmio32,0xff690000 console=ttyFIQ0 androidboot.baseband=N/A androidboot.veritymode=enforcing androidboot.hardware=rk30board androidboot.console=ttyFIQ0 init=/init kpti=0"; ++ }; ++ ++ es8316-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,es8316-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,widgets = ++ "Microphone", "Mic Jack", ++ "Headphone", "Headphone Jack"; ++ simple-audio-card,routing = ++ "Mic Jack", "MICBIAS1", ++ "IN1P", "Mic Jack", ++ "Headphone Jack", "HPOL", ++ "Headphone Jack", "HPOR"; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s_8ch>; ++ system-clock-frequency = <11289600>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&es8316>; ++ system-clock-frequency = <11289600>; ++ }; ++ }; ++ ++ rk_headset: rk-headset { ++ compatible = "rockchip_headset"; ++ headset_gpio = <&gpio0 RK_PC7 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hp_det>; ++ io-channels = <&saradc 2>; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ clocks = <&rk818 1>; ++ clock-names = "ext_clock"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio3 4 GPIO_ACTIVE_LOW>; /* GPIO3_A4 */ ++ }; ++ ++ gpio_keys: gpio-keys { ++ compatible = "gpio-keys"; ++ autorepeat; ++ ++ power { ++ debounce-interval = <100>; ++ gpios = <&gpio0 2 GPIO_ACTIVE_LOW>; ++ label = "GPIO Key Power"; ++ linux,code = ; ++ wakeup-source; ++ }; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ /* wifi_chip_type - wifi chip define ++ * ap6210, ap6330, ap6335 ++ * rtl8188eu, rtl8723bs, rtl8723bu ++ * esp8089 ++ */ ++ wifi_chip_type = "ap6255"; ++ WIFI,host_wake_irq = <&gpio3 6 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&rk818 1>; ++ clock-names = "ext_clock"; ++ uart_rts_gpios = <&gpio2 27 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default","rts_gpio"; ++ pinctrl-0 = <&uart0_rts>; ++ pinctrl-1 = <&uart0_rts_gpio>; ++ ++ //BT,power_gpio = <&gpio3 3 GPIO_ACTIVE_HIGH>; ++ BT,reset_gpio = <&gpio3 5 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio3 2 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio3 7 GPIO_ACTIVE_HIGH>; ++ ++ status = "okay"; ++ }; ++ ++ vcc_sys: vcc-sys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3800000>; ++ regulator-max-microvolt = <3800000>; ++ }; ++ ++ vcc_host: vcc-host { ++ compatible = "regulator-fixed"; ++ enable-active-high; ++ gpio = <&gpio0 4 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&host_vbus_drv>; ++ regulator-name = "vcc_host"; ++ regulator-always-on; ++ }; ++ ++ xin32k: xin32k { ++ compatible = "fixed-clock"; ++ clock-frequency = <32768>; ++ clock-output-names = "xin32k"; ++ #clock-cells = <0>; ++ }; ++ ++}; ++ ++&cif_clkout { ++ /* cif_clkout */ ++ rockchip,pins = <1 RK_PB3 1 &pcfg_pull_none_4ma>; ++}; ++ ++&cpu_l0 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu_l1 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu_l2 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu_l3 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu_b0 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu_b1 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu_b2 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu_b3 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&gpu { ++ logic-supply = <&vdd_logic>; ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&dmc { ++ status = "okay"; ++ center-supply = <&vdd_logic>; ++ devfreq-events = <&dfi>; ++ upthreshold = <60>; ++ downdifferential = <20>; ++ system-status-freq = < ++ /*system status freq(KHz)*/ ++ SYS_STATUS_NORMAL 600000 ++ SYS_STATUS_REBOOT 600000 ++ SYS_STATUS_SUSPEND 240000 ++ SYS_STATUS_VIDEO_1080P 396000 ++ SYS_STATUS_VIDEO_4K 600000 ++ SYS_STATUS_PERFORMANCE 600000 ++ SYS_STATUS_BOOST 396000 ++ SYS_STATUS_DUALVIEW 600000 ++ SYS_STATUS_ISP 528000 ++ >; ++ vop-bw-dmc-freq = < ++ /* min_bw(MB/s) max_bw(MB/s) freq(KHz) */ ++ 0 582 240000 ++ 583 99999 396000 ++ >; ++ auto-min-freq = <240000>; ++ auto-freq-en = <1>; ++}; ++ ++&dsi { ++ status = "okay"; ++ ++ panel@0 { ++ compatible = "simple-panel-dsi"; ++ reg = <0>; ++ backlight = <&backlight>; ++ enable-gpios = <&gpio0 RK_PC6 GPIO_ACTIVE_HIGH>; ++ back-gpios = <&gpio2 RK_PC3 GPIO_ACTIVE_HIGH>; ++ reset-gpios = <&gpio3 RK_PC4 GPIO_ACTIVE_LOW>; ++ prepare-delay-ms = <8>; ++ enable-delay-ms = <3>; ++ reset-delay-ms = <50>; ++ init-delay-ms = <20>; ++ ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | ++ MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ width-mm = <153>; ++ height-mm = <85>; ++ panel-init-sequence = [ ++ 05 1e 01 01 ++ 15 00 02 80 47 ++ 15 00 02 81 40 ++ 15 00 02 82 04 ++ 15 00 02 83 77 ++ 15 00 02 84 0f ++ 15 00 02 85 70 ++ 15 78 02 86 70 ++ ]; ++ panel-exit-sequence = [ ++ 05 00 01 28 ++ 05 00 01 10 ++ ]; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <49500000>; ++ hactive = <1024>; ++ vactive = <600>; ++ hback-porch = <120>; ++ hfront-porch = <80>; ++ vback-porch = <14>; ++ vfront-porch = <14>; ++ hsync-len = <40>; ++ vsync-len = <4>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ panel_in_dsi: endpoint { ++ remote-endpoint = <&dsi_out_panel>; ++ }; ++ }; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@1 { ++ reg = <1>; ++ dsi_out_panel: endpoint { ++ remote-endpoint = <&panel_in_dsi>; ++ }; ++ }; ++ }; ++}; ++ ++&emmc { ++ bus-width = <8>; ++ cap-mmc-highspeed; ++ mmc-hs200-1_8v; ++ no-sdio; ++ no-sd; ++ disable-wp; ++ non-removable; ++ num-slots = <1>; ++ status = "okay"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ ++ vdd_cpu: tcs4525@10 { ++ compatible = "tcs,tcs452x"; ++ reg = <0x10>; ++ regulator-compatible = "fan53555-reg"; ++ pinctrl-0 = <&vsel_gpio>; ++ vsel-gpios = <&gpio0 0 GPIO_ACTIVE_HIGH>; ++ regulator-name = "vdd_cpu"; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1390000>; ++ regulator-ramp-delay = <2300>; ++ fcs,suspend-voltage-selector = <1>; ++ regulator-boot-on; ++ regulator-always-on; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ rk818: pmic@1c { ++ compatible = "rockchip,rk818"; ++ status = "okay"; ++ reg = <0x1c>; ++ clock-output-names = "rk818-clkout1", "wifibt_32kin"; ++ interrupt-parent = <&gpio0>; ++ interrupts = <1 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pmic_int_l>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ extcon = <&u2phy>; ++ #clock-cells = <1>; ++ ++ vcc1-supply = <&vcc_sys>; ++ vcc2-supply = <&vcc_sys>; ++ vcc3-supply = <&vcc_sys>; ++ vcc4-supply = <&vcc_sys>; ++ vcc6-supply = <&vcc_sys>; ++ vcc7-supply = <&vcc_sys>; ++ vcc8-supply = <&vcc_sys>; ++ vcc9-supply = <&vcc_io>; ++ ++ regulators { ++ vdd_logic: DCDC_REG1 { ++ regulator-name = "vdd_logic"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <750000>; ++ regulator-max-microvolt = <1450000>; ++ regulator-ramp-delay = <6001>; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vdd_gpu: DCDC_REG2 { ++ regulator-name = "vdd_gpu"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <800000>; ++ regulator-max-microvolt = <1250000>; ++ regulator-ramp-delay = <6001>; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc_ddr"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_io: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcc_io"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcca_codec: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcca_codec"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc_tp: LDO_REG2 { ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-name = "vcc_tp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdd_10: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ regulator-name = "vdd_10"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc18_lcd: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc18_lcd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vccio_pmu: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vccio_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd10_lcd: LDO_REG6 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ regulator-name = "vdd10_lcd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc_18: LDO_REG7 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc_18"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vccio_wl: LDO_REG8 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vccio_wl"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vccio_sd: LDO_REG9 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vccio_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc_sd: SWITCH_REG { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ boost_otg: DCDC_BOOST { ++ regulator-name = "boost_otg"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <5000000>; ++ }; ++ }; ++ ++ otg_switch: OTG_SWITCH { ++ regulator-name = "otg_switch"; ++ }; ++ }; ++ ++ battery { ++ compatible = "rk818-battery"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&dc_irq_gpio>; ++ ocv_table = < ++ 3400 3652 3680 3707 3730 3747 3764 ++ 3772 3781 3792 3807 3828 3861 3899 ++ 3929 3958 3987 4038 4079 4127 4186>; ++ design_capacity = <7536>; ++ design_qmax = <8290>; ++ bat_res = <100>; ++ max_input_current = <1750>; ++ max_chrg_current = <2000>; ++ max_chrg_voltage = <4200>; ++ sleep_enter_current = <600>; ++ sleep_exit_current = <600>; ++ power_off_thresd = <3400>; ++ zero_algorithm_vol = <3850>; ++ fb_temperature = <115>; ++ sample_res = <20>; ++ max_soc_offset = <60>; ++ energy_mode = <0>; ++ monitor_sec = <5>; ++ virtual_power = <0>; ++ power_dc2otg = <0>; ++ support_usb_adp = <1>; ++ support_dc_adp = <1>; ++ dc_det_gpio = <&gpio0 17 GPIO_ACTIVE_LOW>; ++ }; ++ }; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ ++ es8316: es8316@10 { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++ compatible = "everest,es8316"; ++ reg = <0x10>; ++ clocks = <&cru SCLK_I2S_8CH_OUT>; ++ clock-names = "mclk"; ++ spk-con-gpio = <&gpio0 27 GPIO_ACTIVE_HIGH>; ++ hp-det-gpio = <&gpio0 23 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s_8ch_mclk>; ++ extcon = <&rk_headset>; ++ }; ++}; ++ ++&i2c2 { ++ status = "okay"; ++ ++ ts@5a { ++ compatible = "cst2xxse"; ++ reg = <0x5a>; ++ irq-gpio = <&gpio0 RK_PB4 IRQ_TYPE_LEVEL_LOW>; ++ //touch-gpio = <&gpio1 GPIO_B0 IRQ_TYPE_LEVEL_LOW>; /* TP_INT == GPIO1_B0 */ ++ //reset-gpio = <&gpio0 GPIO_D1 GPIO_ACTIVE_LOW>; /* TP_RST == GPIO0_D1 */ ++ //power-gpio = <&gpio0 GPIO_C5 GPIO_ACTIVE_LOW>; ++ //max-x = <800>; ++ //max-y = <480>; ++ status = "okay"; ++ }; ++ ++}; ++ ++&i2c3 { ++ status = "okay"; ++ ++ gc0312: gc0312@21 { ++ status = "okay"; ++ compatible = "galaxycore,gc0312"; ++ reg = <0x21>; ++ clocks = <&cru SCLK_VIP_OUT>; ++ clock-names = "xvclk"; ++ pwdn-gpios = <&gpio1 RK_PB5 GPIO_ACTIVE_HIGH>; ++ ++ rockchip,camera-module-index = <1>; ++ rockchip,camera-module-facing = "front"; ++ rockchip,camera-module-name = "CameraKing"; ++ rockchip,camera-module-lens-name = "Largan"; ++ port { ++ gc0312_out: endpoint { ++ remote-endpoint = <&dvp_in_fcam>; ++ }; ++ }; ++ }; ++ ++ gc2145: gc2145@3c { ++ status = "okay"; ++ compatible = "galaxycore,gc2145"; ++ reg = <0x3c>; ++ clocks = <&cru SCLK_VIP_OUT>; ++ clock-names = "xvclk"; ++ pwdn-gpios = <&gpio1 RK_PB4 GPIO_ACTIVE_HIGH>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "CameraKing"; ++ rockchip,camera-module-lens-name = "Largan"; ++ port { ++ gc2145_out: endpoint { ++ remote-endpoint = <&dvp_in_bcam>; ++ }; ++ }; ++ }; ++ ++ ov8858: ov8858@36 { ++ status = "disabled"; ++ compatible = "ovti,ov8858"; ++ reg = <0x36>; ++ clocks = <&cru SCLK_VIP_OUT>; ++ clock-names = "xvclk"; ++ ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "CameraKing"; ++ rockchip,camera-module-lens-name = "Largan-9569A2"; ++ power-gpio = <&gpio0 RK_PD4 GPIO_ACTIVE_HIGH>; ++ pwdn-gpios = <&gpio1 RK_PB5 GPIO_ACTIVE_HIGH>; ++ ++ port { ++ ov8858_out: endpoint { ++ remote-endpoint = <&mipi_in>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++}; ++ ++&i2c4 { ++ status = "okay"; ++ ++ mpu6500@68 { ++ status = "disabled"; ++ compatible = "invensense,mpu6500"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&mpu6500_irq_gpio>; ++ reg = <0x68>; ++ irq-gpio = <&gpio3 14 IRQ_TYPE_EDGE_RISING>; ++ mpu-int_config = <0x10>; ++ mpu-level_shifter = <0>; ++ mpu-orientation = <1 0 0 0 1 0 0 0 1>; ++ orientation-x= <1>; ++ orientation-y= <0>; ++ orientation-z= <1>; ++ support-hw-poweroff = <1>; ++ mpu-debug = <1>; ++ }; ++ ++ sensor@4c { ++ status = "okay"; ++ compatible = "gs_mc3230"; ++ reg = <0x4c>; ++ type = ; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ layout = <9>; ++ reprobe_en = <1>; ++ irq-gpio = <&gpio3 RK_PB6 IRQ_TYPE_LEVEL_LOW>; ++ }; ++ ++ sensor@18 { ++ status = "okay"; ++ compatible = "gs_sc7a30"; ++ reg = <0x18>; ++ type = ; ++ irq-gpio = <&gpio3 RK_PB6 IRQ_TYPE_LEVEL_LOW>; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ layout = <6>; ++ reprobe_en = <1>; ++ }; ++ ++ sensor@10 { ++ status = "okay"; ++ compatible = "light_cm3218"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cm3218_irq_gpio>; ++ reg = <0x10>; ++ type = ; ++ irq-gpio = <&gpio3 15 IRQ_TYPE_EDGE_FALLING>; ++ irq_enable = <1>; ++ poll_delay_ms = <30>; ++ }; ++}; ++ ++&i2s_8ch { ++ status = "okay"; ++ rockchip,i2s-broken-burst-len; ++ rockchip,playback-channels = <8>; ++ rockchip,capture-channels = <2>; ++ #sound-dai-cells = <0>; ++}; ++ ++&io_domains { ++ status = "okay"; ++ ++ dvp-supply = <&vcc_18>; ++ audio-supply = <&vcc_io>; ++ gpio30-supply = <&vcc_io>; ++ gpio1830-supply = <&vcc_io>; ++ sdcard-supply = <&vccio_sd>; ++ wifi-supply = <&vccio_wl>; ++}; ++ ++&isp_dvp_d2d9 { ++ rockchip,pins = ++ /* cif_data4 ... cif_data9 */ ++ <1 RK_PA2 1 &pcfg_pull_down>, ++ <1 RK_PA3 1 &pcfg_pull_down>, ++ <1 RK_PA4 1 &pcfg_pull_down>, ++ <1 RK_PA5 1 &pcfg_pull_down>, ++ <1 RK_PA6 1 &pcfg_pull_down>, ++ <1 RK_PA7 1 &pcfg_pull_down>, ++ /* cif_sync, cif_href */ ++ <1 RK_PB0 1 &pcfg_pull_down>, ++ <1 RK_PB1 1 &pcfg_pull_down>, ++ /* cif_clkin */ ++ <1 RK_PB2 1 &pcfg_pull_down>; ++}; ++ ++&isp_dvp_d10d11 { ++ rockchip,pins = ++ /* cif_data10, cif_data11 */ ++ <1 RK_PB6 1 &pcfg_pull_down>, ++ <1 RK_PB7 1 &pcfg_pull_down>; ++}; ++ ++&isp_mmu { ++ status = "okay"; ++}; ++ ++&mipi_dphy_rx0 { ++ status = "disabled"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ov8858_out>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_rx_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp_mipi_in>; ++ }; ++ }; ++ }; ++}; ++ ++&nandc0 { ++ status = "okay"; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ ++ pmu-supply = <&vccio_pmu>; ++ vop-supply = <&vccio_pmu>; ++}; ++ ++&pwm0 { ++ status = "okay"; ++}; ++ ++&pinctrl { ++ camera { ++ camera_pwr: camera-pwr { ++ rockchip,pins = <0 RK_PD0 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ headphone { ++ hp_det: hp-det { ++ rockchip,pins = <0 RK_PC7 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ pmic { ++ pmic_int_l: pmic-int-l { ++ rockchip,pins = <0 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ vsel_gpio: vsel-gpio { ++ rockchip,pins = <0 RK_PA0 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ mpu6500 { ++ mpu6500_irq_gpio: mpu6500-irq-gpio { ++ rockchip,pins = <3 RK_PB6 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ cm3218 { ++ cm3218_irq_gpio: cm3218-irq-gpio { ++ rockchip,pins = <3 RK_PB7 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ dc_det { ++ dc_irq_gpio: dc-irq-gpio { ++ rockchip,pins = <0 RK_PC1 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ pcfg_pull_none_4ma: pcfg-pull-none-4ma { ++ bias-disable; ++ drive-strength = <4>; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ usb2 { ++ host_vbus_drv: host-vbus-drv { ++ rockchip,pins = <0 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart0_rts_gpio: uart0-rts-gpio { ++ rockchip,pins = <2 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++&rkisp1 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&isp_dvp_d2d9 &isp_dvp_d10d11 &cif_clkout>; ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dvp_in_fcam: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&gc0312_out>; ++ }; ++ ++ dvp_in_bcam: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&gc2145_out>; ++ }; ++ ++ isp_mipi_in: endpoint@2 { ++ reg = <2>; ++ remote-endpoint = <&dphy_rx_out>; ++ }; ++ }; ++}; ++ ++&route_dsi { ++ status = "okay"; ++}; ++ ++&rockchip_suspend { ++ status = "okay"; ++ rockchip,sleep-mode-config = < ++ (0 ++ | RKPM_SLP_ARMOFF ++ | RKPM_SLP_PMU_PLLS_PWRDN ++ | RKPM_SLP_PMU_PMUALIVE_32K ++ | RKPM_SLP_SFT_PLLS_DEEP ++ | RKPM_SLP_PMU_DIS_OSC ++ | RKPM_SLP_SFT_PD_NBSCUS ++ ) ++ >; ++ rockchip,wakeup-config = < ++ (0 ++ | RKPM_GPIO_WKUP_EN ++ | RKPM_USB_WKUP_EN ++ | RKPM_CLUSTER_L_WKUP_EN ++ ) ++ >; ++}; ++ ++&saradc { ++ status = "okay"; ++}; ++ ++&sdmmc { ++ clock-frequency = <37500000>; ++ clock-freq-min-max = <400000 37500000>; ++ no-sdio; ++ no-mmc; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ card-detect-delay = <200>; ++ disable-wp; ++ num-slots = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_cd &sdmmc_bus4>; ++ status = "disabled"; ++}; ++ ++&sdio0 { ++ max-frequency = <100000000>; ++ no-sd; ++ no-mmc; ++ bus-width = <4>; ++ disable-wp; ++ cap-sd-highspeed; ++ cap-sdio-irq; ++ keep-power-in-suspend; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ non-removable; ++ num-slots = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>; ++ rockchip,default-sample-phase = <90>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&tsadc { ++ tsadc-supply = <&vdd_cpu>; ++ status = "okay"; ++}; ++ ++&uart0 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart0_xfer &uart0_cts>; ++ status = "okay"; ++}; ++ ++&u2phy { ++ status = "okay"; ++ ++ u2phy_otg: otg-port { ++ status = "okay"; ++ }; ++ ++ u2phy_host: host-port { ++ phy-supply = <&vcc_host>; ++ status = "okay"; ++ }; ++}; ++ ++&usb_host0_ehci { ++ status = "okay"; ++}; ++ ++&usb_host0_ohci { ++ status = "okay"; ++}; ++ +diff --git a/arch/arm64/boot/dts/rockchip/rk3368-xikp-avb.dts b/arch/arm64/boot/dts/rockchip/rk3368-xikp-avb.dts +new file mode 100755 +index 000000000000..b455e32c3e29 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3368-xikp-avb.dts +@@ -0,0 +1,139 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++#include "rk3368-xikp.dtsi" ++ ++/ { ++ model = "Rockchip rk3368 xkp avb board"; ++ compatible = "rockchip,xkp-avb", "rockchip,rk3368"; ++}; ++ ++&firmware_android { ++ compatible = "android,firmware"; ++ boot_devices = "ff0f0000.dwmmc,ff400000.nandc"; ++ vbmeta { ++ compatible = "android,vbmeta"; ++ parts = "vbmeta,dtbo"; ++ }; ++ fstab { ++ compatible = "android,fstab"; ++ vendor { ++ compatible = "android,vendor"; ++ dev = "/dev/block/by-name/vendor"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,avb"; ++ }; ++ }; ++}; ++ ++&chosen { ++ bootargs = "earlycon=uart8250,mmio32,0xff690000 console=ttyFIQ0 androidboot.baseband=N/A androidboot.veritymode=enforcing androidboot.hardware=rk30board androidboot.console=ttyFIQ0 androidboot.selinux=permissive init=/init kpti=0"; ++}; ++ ++&i2c3 { ++ status = "okay"; ++ ++ gc2145: gc2145@3c { ++ compatible = "galaxycore,gc2145"; ++ reg = <0x3c>; ++ clocks = <&cru SCLK_VIP_OUT>; ++ clock-names = "xvclk"; ++ ++ power-gpio = <&gpio0 RK_PD4 GPIO_ACTIVE_HIGH>; ++ pwdn-gpios = <&gpio1 RK_PB4 GPIO_ACTIVE_HIGH>; ++ rockchip,camera-module-index = <1>; ++ rockchip,camera-module-facing = "front"; ++ rockchip,camera-module-name = "CameraKing"; ++ rockchip,camera-module-lens-name = "Largan"; ++ port { ++ gc2145_out: endpoint { ++ remote-endpoint = <&isp_dvp_in>; ++ }; ++ }; ++ }; ++ ++ ov8858: ov8858@36 { ++ compatible = "ovti,ov8858"; ++ reg = <0x36>; ++ clocks = <&cru SCLK_VIP_OUT>; ++ clock-names = "xvclk"; ++ ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "CameraKing"; ++ rockchip,camera-module-lens-name = "Largan-9569A2"; ++ power-gpio = <&gpio0 RK_PD4 GPIO_ACTIVE_HIGH>; ++ pwdn-gpios = <&gpio1 RK_PB5 GPIO_ACTIVE_HIGH>; ++ port { ++ ov8858_out: endpoint { ++ remote-endpoint = <&mipi_in>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++ ++}; ++ ++&isp { ++ status = "disabled"; ++}; ++ ++&isp_mmu { ++ status = "okay"; ++}; ++ ++&mipi_dphy_rx0 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ov8858_out>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_rx_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp_mipi_in>; ++ }; ++ }; ++ }; ++}; ++ ++&rkisp1 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&isp_dvp_d2d9 &isp_dvp_d10d11 &cif_clkout>; ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp_dvp_in: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&gc2145_out>; ++ }; ++ ++ isp_mipi_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dphy_rx_out>; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3368-xikp.dts b/arch/arm64/boot/dts/rockchip/rk3368-xikp.dts +new file mode 100755 +index 000000000000..bc320ffdc3b7 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3368-xikp.dts +@@ -0,0 +1,33 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++#include "rk3368-xikp.dtsi" ++ ++/ { ++ model = "Rockchip rk3368 xkp board"; ++ compatible = "rockchip,xkp", "rockchip,rk3368"; ++}; ++ ++&firmware_android { ++ compatible = "android,firmware"; ++ fstab { ++ compatible = "android,fstab"; ++ system { ++ compatible = "android,system"; ++ dev = "/dev/block/by-name/system"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,verify"; ++ }; ++ vendor { ++ compatible = "android,vendor"; ++ dev = "/dev/block/by-name/vendor"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,verify"; ++ }; ++ }; ++}; +\ No newline at end of file +diff --git a/arch/arm64/boot/dts/rockchip/rk3368-xikp.dtsi b/arch/arm64/boot/dts/rockchip/rk3368-xikp.dtsi +new file mode 100755 +index 000000000000..aa73096aff9d +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3368-xikp.dtsi +@@ -0,0 +1,893 @@ ++/* ++ * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This file is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This file is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++/dts-v1/; ++#include ++#include ++#include ++#include "rk3368.dtsi" ++#include "rk3368-android.dtsi" ++ ++/ { ++ es8316-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,rk-es8316-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,widgets = ++ "Microphone", "Mic Jack", ++ "Headphone", "Headphone Jack"; ++ simple-audio-card,routing = ++ "Mic Jack", "MICBIAS1", ++ "IN1P", "Mic Jack", ++ "Headphone Jack", "HPOL", ++ "Headphone Jack", "HPOR"; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s_8ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&es8316>; ++ }; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ clocks = <&rk818 1>; ++ clock-names = "ext_clock"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio3 4 GPIO_ACTIVE_LOW>; /* GPIO3_A4 */ ++ }; ++ ++ backlight: backlight { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm0 0 25000 0>; ++ brightness-levels = < ++ 0 1 2 3 4 5 6 7 ++ 8 9 10 11 12 13 14 15 ++ 16 17 18 19 20 21 22 23 ++ 24 25 26 27 28 29 30 31 ++ 32 33 34 35 36 37 38 39 ++ 40 41 42 43 44 45 46 47 ++ 48 49 50 51 52 53 54 55 ++ 56 57 58 59 60 61 62 63 ++ 64 65 66 67 68 69 70 71 ++ 72 73 74 75 76 77 78 79 ++ 80 81 82 83 84 85 86 87 ++ 88 89 90 91 92 93 94 95 ++ 96 97 98 99 100 101 102 103 ++ 104 105 106 107 108 109 110 111 ++ 112 113 114 115 116 117 118 119 ++ 120 121 122 123 124 125 126 127 ++ 128 129 130 131 132 133 134 135 ++ 136 137 138 139 140 141 142 143 ++ 144 145 146 147 148 149 150 151 ++ 152 153 154 155 156 157 158 159 ++ 160 161 162 163 164 165 166 167 ++ 168 169 170 171 172 173 174 175 ++ 176 177 178 179 180 181 182 183 ++ 184 185 186 187 188 189 190 191 ++ 192 193 194 195 196 197 198 199 ++ 200 201 202 203 204 205 206 207 ++ 208 209 210 211 212 213 214 215 ++ 216 217 218 219 220 221 222 223 ++ 224 225 226 227 228 229 230 231 ++ 232 233 234 235 236 237 238 239 ++ 240 241 242 243 244 245 246 247 ++ 248 249 250 251 252 253 254 255>; ++ default-brightness-level = <200>; ++ }; ++ ++ charge-animation { ++ compatible = "rockchip,uboot-charge"; ++ rockchip,uboot-charge-on = <1>; ++ rockchip,android-charge-on = <0>; ++ rockchip,uboot-low-power-voltage = <3500>; ++ rockchip,screen-on-voltage = <3600>; ++ status = "okay"; ++ }; ++ ++ gpio_keys: gpio-keys { ++ compatible = "gpio-keys"; ++ autorepeat; ++ ++ power { ++ debounce-interval = <100>; ++ gpios = <&gpio0 2 GPIO_ACTIVE_LOW>; ++ label = "GPIO Key Power"; ++ linux,code = ; ++ wakeup-source; ++ }; ++ }; ++ ++ adc_keys: adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 1>; ++ io-channel-names = "buttons"; ++ keyup-threshold-microvolt = <1024000>; ++ poll-interval = <100>; ++ ++ vol-up-key { ++ label = "volume up"; ++ linux,code = ; ++ press-threshold-microvolt = <1000>; ++ }; ++ ++ vol-down-key { ++ label = "volume down"; ++ linux,code = ; ++ press-threshold-microvolt = <170000>; ++ }; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ rockchip,grf = <&grf>; ++ /* wifi_chip_type - wifi chip define ++ * ap6210, ap6330, ap6335 ++ * rtl8188eu, rtl8723bs, rtl8723bu ++ * esp8089 ++ */ ++ wifi_chip_type = "ap6255"; ++ sdio_vref = <1800>; //1800mv or 3300mv ++ WIFI,host_wake_irq = <&gpio3 6 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&rk818 1>; ++ clock-names = "ext_clock"; ++ uart_rts_gpios = <&gpio2 27 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default","rts_gpio"; ++ pinctrl-0 = <&uart0_rts>; ++ pinctrl-1 = <&uart0_rts_gpio>; ++ ++ //BT,power_gpio = <&gpio3 3 GPIO_ACTIVE_HIGH>; ++ BT,reset_gpio = <&gpio3 5 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio3 2 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio3 7 GPIO_ACTIVE_HIGH>; ++ ++ status = "okay"; ++ }; ++ ++ vcc_sys: vcc-sys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3800000>; ++ regulator-max-microvolt = <3800000>; ++ }; ++ ++ vcc_host: vcc-host { ++ compatible = "regulator-fixed"; ++ enable-active-high; ++ gpio = <&gpio0 4 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&host_vbus_drv>; ++ regulator-name = "vcc_host"; ++ regulator-always-on; ++ }; ++ ++ xin32k: xin32k { ++ compatible = "fixed-clock"; ++ clock-frequency = <32768>; ++ clock-output-names = "xin32k"; ++ #clock-cells = <0>; ++ }; ++}; ++ ++&cpu_l0 { ++ cpu-supply = <&syr827>; ++}; ++ ++&cpu_l1 { ++ cpu-supply = <&syr827>; ++}; ++ ++&cpu_l2 { ++ cpu-supply = <&syr827>; ++}; ++ ++&cpu_l3 { ++ cpu-supply = <&syr827>; ++}; ++ ++&cpu_b0 { ++ cpu-supply = <&syr827>; ++}; ++ ++&cpu_b1 { ++ cpu-supply = <&syr827>; ++}; ++ ++&cpu_b2 { ++ cpu-supply = <&syr827>; ++}; ++ ++&cpu_b3 { ++ cpu-supply = <&syr827>; ++}; ++ ++&gpu { ++ logic-supply = <&vdd_logic>; ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&dmc { ++ status = "okay"; ++ center-supply = <&vdd_logic>; ++ devfreq-events = <&dfi>; ++ upthreshold = <60>; ++ downdifferential = <20>; ++ system-status-freq = < ++ /*system status freq(KHz)*/ ++ SYS_STATUS_NORMAL 600000 ++ SYS_STATUS_REBOOT 600000 ++ SYS_STATUS_SUSPEND 240000 ++ SYS_STATUS_VIDEO_1080P 396000 ++ SYS_STATUS_VIDEO_4K 600000 ++ SYS_STATUS_PERFORMANCE 600000 ++ SYS_STATUS_BOOST 396000 ++ SYS_STATUS_DUALVIEW 600000 ++ SYS_STATUS_ISP 528000 ++ >; ++ vop-bw-dmc-freq = < ++ /* min_bw(MB/s) max_bw(MB/s) freq(KHz) */ ++ 0 582 240000 ++ 583 99999 396000 ++ >; ++ auto-min-freq = <240000>; ++ auto-freq-en = <1>; ++}; ++ ++&rockchip_suspend { ++ status = "okay"; ++ rockchip,sleep-mode-config = < ++ (0 ++ | RKPM_SLP_ARMOFF ++ | RKPM_SLP_PMU_PLLS_PWRDN ++ | RKPM_SLP_PMU_PMUALIVE_32K ++ | RKPM_SLP_SFT_PLLS_DEEP ++ | RKPM_SLP_PMU_DIS_OSC ++ | RKPM_SLP_SFT_PD_NBSCUS ++ ) ++ >; ++ rockchip,wakeup-config = < ++ (0 ++ | RKPM_GPIO_WKUP_EN ++ | RKPM_USB_WKUP_EN ++ | RKPM_CLUSTER_L_WKUP_EN ++ ) ++ >; ++}; ++ ++&emmc { ++ bus-width = <8>; ++ cap-mmc-highspeed; ++ mmc-hs200-1_8v; ++ no-sdio; ++ no-sd; ++ disable-wp; ++ non-removable; ++ num-slots = <1>; ++ status = "okay"; ++}; ++ ++&nandc0 { ++ status = "okay"; ++}; ++ ++&sdmmc { ++ clock-frequency = <37500000>; ++ clock-freq-min-max = <400000 37500000>; ++ no-sdio; ++ no-mmc; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ card-detect-delay = <200>; ++ disable-wp; ++ num-slots = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_cd &sdmmc_bus4>; ++ status = "disabled"; ++}; ++ ++&sdio0 { ++ clock-frequency = <100000000>; ++ clock-freq-min-max = <200000 100000000>; ++ no-sd; ++ no-mmc; ++ bus-width = <4>; ++ disable-wp; ++ cap-sd-highspeed; ++ cap-sdio-irq; ++ keep-power-in-suspend; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ non-removable; ++ num-slots = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ ++ syr827: syr827@40 { ++ compatible = "silergy,syr827"; ++ status = "okay"; ++ reg = <0x40>; ++ ++ regulator-compatible = "fan53555-reg"; ++ regulator-name = "vdd_arm"; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1500000>; ++ regulator-ramp-delay = <1000>; ++ fcs,suspend-voltage-selector = <1>; ++ pinctrl-0 = <&vsel_gpio>; ++ vsel-gpios = <&gpio0 0 GPIO_ACTIVE_HIGH>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-state = <3>; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <900000>; ++ }; ++ }; ++ ++ rk818: pmic@1c { ++ compatible = "rockchip,rk818"; ++ status = "okay"; ++ reg = <0x1c>; ++ clock-output-names = "rk818-clkout1", "wifibt_32kin"; ++ interrupt-parent = <&gpio0>; ++ interrupts = <1 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pmic_int_l>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ extcon = <&u2phy>; ++ #clock-cells = <1>; ++ ++ vcc1-supply = <&vcc_sys>; ++ vcc2-supply = <&vcc_sys>; ++ vcc3-supply = <&vcc_sys>; ++ vcc4-supply = <&vcc_sys>; ++ vcc6-supply = <&vcc_sys>; ++ vcc7-supply = <&vcc_sys>; ++ vcc8-supply = <&vcc_sys>; ++ vcc9-supply = <&vcc_io>; ++ ++ regulators { ++ vdd_logic: DCDC_REG1 { ++ regulator-name = "vdd_logic"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <750000>; ++ regulator-max-microvolt = <1450000>; ++ regulator-ramp-delay = <6001>; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vdd_gpu: DCDC_REG2 { ++ regulator-name = "vdd_gpu"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <800000>; ++ regulator-max-microvolt = <1250000>; ++ regulator-ramp-delay = <6001>; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc_ddr"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_io: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcc_io"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcca_codec: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcca_codec"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc_tp: LDO_REG2 { ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-name = "vcc_tp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdd_10: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ regulator-name = "vdd_10"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc18_lcd: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc18_lcd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vccio_pmu: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vccio_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd10_lcd: LDO_REG6 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ regulator-name = "vdd10_lcd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc_18: LDO_REG7 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc_18"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vccio_wl: LDO_REG8 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vccio_wl"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vccio_sd: LDO_REG9 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vccio_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc_sd: SWITCH_REG { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ boost_otg: DCDC_BOOST { ++ regulator-name = "boost_otg"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <5000000>; ++ }; ++ }; ++ ++ otg_switch: OTG_SWITCH { ++ regulator-name = "otg_switch"; ++ }; ++ }; ++ ++ battery { ++ compatible = "rk818-battery"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&dc_irq_gpio>; ++ ocv_table = < ++ 3400 3652 3680 3707 3730 3747 3764 ++ 3772 3781 3792 3807 3828 3861 3899 ++ 3929 3958 3987 4038 4079 4127 4186>; ++ design_capacity = <7536>; ++ design_qmax = <8290>; ++ bat_res = <100>; ++ max_input_current = <1750>; ++ max_chrg_current = <2000>; ++ max_chrg_voltage = <4200>; ++ sleep_enter_current = <600>; ++ sleep_exit_current = <600>; ++ power_off_thresd = <3400>; ++ zero_algorithm_vol = <3850>; ++ fb_temperature = <115>; ++ sample_res = <20>; ++ max_soc_offset = <60>; ++ energy_mode = <0>; ++ monitor_sec = <5>; ++ virtual_power = <0>; ++ power_dc2otg = <0>; ++ support_usb_adp = <1>; ++ support_dc_adp = <1>; ++ dc_det_gpio = <&gpio0 17 GPIO_ACTIVE_LOW>; ++ }; ++ }; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ ++ es8316: es8316@10 { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++ compatible = "everest,es8316"; ++ reg = <0x10>; ++ clocks = <&cru SCLK_I2S_8CH_OUT>; ++ clock-names = "mclk"; ++ spk-con-gpio = <&gpio3 9 GPIO_ACTIVE_HIGH>; ++ hp-det-gpio = <&gpio0 23 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s_8ch_mclk>; ++ }; ++}; ++ ++&i2c2 { ++ status = "okay"; ++ ++ gt9xx: gt9xx@14 { ++ compatible = "goodix,gt9xx"; ++ reg = <0x14>; ++ touch-gpio = <&gpio0 12 IRQ_TYPE_LEVEL_HIGH>; ++ reset-gpio = <&gpio0 11 GPIO_ACTIVE_HIGH>; ++ max-x = <1920>; ++ max-y = <1200>; ++ tp-size = <89>; ++ configfile-num = <1>; ++ status = "okay"; ++ tp-supply = <&vcc_tp>; ++ }; ++}; ++ ++&i2c3 { ++ status = "okay"; ++}; ++ ++&i2c4 { ++ status = "okay"; ++ ++ mpu6500@68 { ++ status = "disabled"; ++ compatible = "invensense,mpu6500"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&mpu6500_irq_gpio>; ++ reg = <0x68>; ++ irq-gpio = <&gpio3 14 IRQ_TYPE_EDGE_RISING>; ++ mpu-int_config = <0x10>; ++ mpu-level_shifter = <0>; ++ mpu-orientation = <1 0 0 0 1 0 0 0 1>; ++ orientation-x= <1>; ++ orientation-y= <0>; ++ orientation-z= <1>; ++ support-hw-poweroff = <1>; ++ mpu-debug = <1>; ++ }; ++ ++ sensor@4c { ++ status = "okay"; ++ compatible = "gs_mc3230"; ++ reg = <0x4c>; ++ type = ; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ layout = <4>; ++ reprobe_en = <1>; ++ }; ++ ++ sensor@19 { ++ status = "okay"; ++ compatible = "gs_lis3dh"; ++ reg = <0x19>; ++ type = ; ++ irq-gpio = <&gpio3 14 IRQ_TYPE_LEVEL_LOW>; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ layout = <6>; ++ reprobe_en = <1>; ++ }; ++ ++ sensor@10 { ++ status = "okay"; ++ compatible = "light_cm3218"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cm3218_irq_gpio>; ++ reg = <0x10>; ++ type = ; ++ irq-gpio = <&gpio3 15 IRQ_TYPE_EDGE_FALLING>; ++ irq_enable = <1>; ++ poll_delay_ms = <30>; ++ }; ++}; ++ ++&i2s_8ch { ++ status = "okay"; ++ rockchip,i2s-broken-burst-len; ++ rockchip,playback-channels = <8>; ++ rockchip,capture-channels = <2>; ++ #sound-dai-cells = <0>; ++}; ++ ++&io_domains { ++ status = "okay"; ++ ++ dvp-supply = <&vcc_18>; ++ audio-supply = <&vcc_io>; ++ gpio30-supply = <&vcc_io>; ++ gpio1830-supply = <&vcc_io>; ++ sdcard-supply = <&vccio_sd>; ++ wifi-supply = <&vccio_wl>; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ ++ pmu-supply = <&vccio_pmu>; ++ vop-supply = <&vccio_pmu>; ++}; ++ ++&pwm0 { ++ status = "okay"; ++}; ++ ++&uart0 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart0_xfer &uart0_cts>; ++ status = "okay"; ++}; ++ ++&saradc { ++ status = "okay"; ++}; ++ ++&u2phy { ++ status = "okay"; ++ ++ u2phy_host: host-port { ++ phy-supply = <&vcc_host>; ++ status = "okay"; ++ }; ++}; ++ ++&usb_host0_ehci { ++ status = "okay"; ++}; ++ ++&usb_host0_ohci { ++ status = "okay"; ++}; ++ ++&dsi { ++ status = "okay"; ++ ++ panel@0 { ++ compatible = "simple-panel-dsi"; ++ reg = <0>; ++ backlight = <&backlight>; ++ enable-gpios = <&gpio0 22 GPIO_ACTIVE_HIGH>; ++ prepare-delay-ms = <120>; ++ enable-delay-ms = <200>; ++ ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | ++ MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <159000000>; ++ hactive = <1200>; ++ vactive = <1920>; ++ hback-porch = <60>; ++ hfront-porch = <80>; ++ vback-porch = <25>; ++ vfront-porch = <35>; ++ hsync-len = <1>; ++ vsync-len = <1>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ panel_in_dsi: endpoint { ++ remote-endpoint = <&dsi_out_panel>; ++ }; ++ }; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@1 { ++ reg = <1>; ++ dsi_out_panel: endpoint { ++ remote-endpoint = <&panel_in_dsi>; ++ }; ++ }; ++ }; ++}; ++ ++&route_dsi { ++ status = "okay"; ++}; ++ ++&tsadc { ++ tsadc-supply = <&syr827>; ++ status = "okay"; ++}; ++ ++&pinctrl { ++ pmic { ++ pmic_int_l: pmic-int-l { ++ rockchip,pins = <0 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ vsel_gpio: vsel-gpio { ++ rockchip,pins = <0 RK_PA0 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ mpu6500 { ++ mpu6500_irq_gpio: mpu6500-irq-gpio { ++ rockchip,pins = <3 RK_PB6 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ cm3218 { ++ cm3218_irq_gpio: cm3218-irq-gpio { ++ rockchip,pins = <3 RK_PB7 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ dc_det { ++ dc_irq_gpio: dc-irq-gpio { ++ rockchip,pins = <0 RK_PC1 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ usb2 { ++ host_vbus_drv: host-vbus-drv { ++ rockchip,pins = <0 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart0_rts_gpio: uart0-rts-gpio { ++ rockchip,pins = <2 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ +diff --git a/arch/arm64/boot/dts/rockchip/rk3368.dtsi b/arch/arm64/boot/dts/rockchip/rk3368.dtsi +index 3746f23dc3df..e962c7962789 100644 +--- a/arch/arm64/boot/dts/rockchip/rk3368.dtsi ++++ b/arch/arm64/boot/dts/rockchip/rk3368.dtsi +@@ -562,7 +562,7 @@ pwm0: pwm@ff680000 { + compatible = "rockchip,rk3368-pwm", "rockchip,rk3288-pwm"; + reg = <0x0 0xff680000 0x0 0x10>; + #pwm-cells = <3>; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm0_pin>; + clocks = <&cru PCLK_PWM1>; + clock-names = "pwm"; +@@ -573,7 +573,7 @@ pwm1: pwm@ff680010 { + compatible = "rockchip,rk3368-pwm", "rockchip,rk3288-pwm"; + reg = <0x0 0xff680010 0x0 0x10>; + #pwm-cells = <3>; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm1_pin>; + clocks = <&cru PCLK_PWM1>; + clock-names = "pwm"; +@@ -593,7 +593,7 @@ pwm3: pwm@ff680030 { + compatible = "rockchip,rk3368-pwm", "rockchip,rk3288-pwm"; + reg = <0x0 0xff680030 0x0 0x10>; + #pwm-cells = <3>; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm3_pin>; + clocks = <&cru PCLK_PWM1>; + clock-names = "pwm"; +@@ -1009,17 +1009,33 @@ pwm0 { + pwm0_pin: pwm0-pin { + rockchip,pins = <3 RK_PB0 2 &pcfg_pull_none>; + }; ++ ++ pwm0_pin_pull_down: pwm0-pin-pull-down { ++ rockchip,pins = <3 RK_PB0 2 &pcfg_pull_down>; ++ }; ++ ++ vop_pwm_pin: vop-pwm { ++ rockchip,pins = <3 RK_PB0 3 &pcfg_pull_none>; ++ }; + }; + + pwm1 { + pwm1_pin: pwm1-pin { + rockchip,pins = <0 RK_PB0 2 &pcfg_pull_none>; + }; ++ ++ pwm1_pin_pull_down: pwm1-pin-pull-down { ++ rockchip,pins = <0 RK_PB0 2 &pcfg_pull_down>; ++ }; + }; + + pwm3 { + pwm3_pin: pwm3-pin { +- rockchip,pins = <3 RK_PD5 3 &pcfg_pull_none>; ++ rockchip,pins = <3 RK_PD6 3 &pcfg_pull_none>; ++ }; ++ ++ pwm3_pin_pull_down: pwm3-pin-pull-down { ++ rockchip,pins = <3 RK_PD6 3 &pcfg_pull_down>; + }; + }; + +diff --git a/arch/arm64/boot/dts/rockchip/rk3368a-817-tablet-bnd.dts b/arch/arm64/boot/dts/rockchip/rk3368a-817-tablet-bnd.dts +new file mode 100755 +index 000000000000..0a489bf8fc84 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3368a-817-tablet-bnd.dts +@@ -0,0 +1,1074 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ */ ++ ++/dts-v1/; ++#include ++#include ++#include ++#include "rk3368.dtsi" ++#include "rk3368-android.dtsi" ++/ { ++ model = "Rockchip rk3368a tablet rk817 board"; ++ compatible = "rockchip,tablet", "rockchip,rk3368a", "rockchip,rk3368"; ++ ++ adc_keys: adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 1>; ++ io-channel-names = "buttons"; ++ keyup-threshold-microvolt = <1024000>; ++ poll-interval = <100>; ++ ++ vol-up-key { ++ label = "volume up"; ++ linux,code = ; ++ press-threshold-microvolt = <1000>; ++ }; ++ ++ vol-down-key { ++ label = "volume down"; ++ linux,code = ; ++ press-threshold-microvolt = <170000>; ++ }; ++ }; ++ ++ backlight: backlight { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm0 0 25000 1>; ++ brightness-levels = < ++ 10 10 10 11 12 13 14 15 ++ 16 16 16 16 16 16 16 16 ++ 17 17 17 17 17 17 17 17 ++ 18 18 18 18 18 18 18 18 ++ 19 19 19 19 19 19 19 19 ++ 20 20 20 20 20 20 20 20 ++ 21 21 21 21 21 21 21 21 ++ 22 22 22 22 22 22 22 22 ++ 23 23 23 23 23 23 23 23 ++ 24 24 24 24 24 24 24 24 ++ 25 25 25 25 25 25 25 25 ++ 26 26 26 26 26 26 26 26 ++ 27 27 27 27 27 27 27 27 ++ 28 28 28 28 28 28 28 28 ++ 27 27 27 27 27 27 27 27 ++ 30 30 30 30 30 30 30 30 ++ 31 31 31 31 31 31 31 31 ++ 32 32 32 32 32 32 32 32 ++ 33 33 33 33 33 33 33 33 ++ 34 34 34 34 34 34 34 34 ++ 35 35 35 35 35 35 35 35 ++ 36 36 36 36 36 36 36 36 ++ 37 37 37 37 37 37 37 37 ++ 38 38 38 38 38 38 38 38 ++ 38 38 38 38 38 38 38 38 ++ 38 38 38 38 38 38 38 38 ++ 38 38 38 38 38 38 38 38 ++ 38 38 38 38 38 38 38 38 ++ 38 38 38 38 38 38 38 38 ++ 38 38 38 38 38 38 38 38 ++ 38 38 38 38 38 38 38 38>; ++ default-brightness-level = <200>; ++ enable-gpios = <&gpio0 20 GPIO_ACTIVE_HIGH>; ++ }; ++ ++ charge-animation { ++ compatible = "rockchip,uboot-charge"; ++ rockchip,uboot-charge-on = <1>; ++ rockchip,android-charge-on = <0>; ++ rockchip,uboot-low-power-voltage = <3500>; ++ rockchip,screen-on-voltage = <3600>; ++ status = "okay"; ++ }; ++ ++ chosen: chosen { ++ bootargs = "earlycon=uart8250,mmio32,0xff690000 console=ttyFIQ0 androidboot.baseband=N/A androidboot.veritymode=enforcing androidboot.hardware=rk30board androidboot.console=ttyFIQ0 init=/init kpti=0"; ++ }; ++ ++ rk817-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip-rk817-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,widgets = ++ "Microphone", "Mic Jack", ++ "Headphone", "Headphone Jack"; ++ simple-audio-card,routing = ++ "Mic Jack", "MICBIAS1", ++ "IN1P", "Mic Jack", ++ "Headphone Jack", "HPOL", ++ "Headphone Jack", "HPOR"; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s_8ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&rk817_codec>; ++ }; ++ }; ++ ++ rk_headset: rk-headset { ++ compatible = "rockchip_headset"; ++ headset_gpio = <&gpio0 RK_PC7 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hp_det>; ++ io-channels = <&saradc 2>; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio3 4 GPIO_ACTIVE_LOW>; /* GPIO3_A4 */ ++ }; ++ ++ vccsys: vccsys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc3v8_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3800000>; ++ regulator-max-microvolt = <3800000>; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ /* wifi_chip_type - wifi chip define ++ * ap6210, ap6330, ap6335 ++ * rtl8188eu, rtl8723bs, rtl8723bu ++ * esp8089 ++ */ ++ wifi_chip_type = "ap6255"; ++ WIFI,vbat_gpio = <&gpio0 2 GPIO_ACTIVE_HIGH>; ++ WIFI,host_wake_irq = <&gpio3 6 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ uart_rts_gpios = <&gpio2 27 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default","rts_gpio"; ++ pinctrl-0 = <&uart0_rts>; ++ pinctrl-1 = <&uart0_rts_gpio>; ++ ++ //BT,power_gpio = <&gpio3 3 GPIO_ACTIVE_HIGH>; ++ BT,reset_gpio = <&gpio3 5 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio3 2 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio3 7 GPIO_ACTIVE_HIGH>; ++ ++ status = "okay"; ++ }; ++ ++ vcc_sys: vcc-sys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3800000>; ++ regulator-max-microvolt = <3800000>; ++ }; ++ ++ vcc_host: vcc-host { ++ compatible = "regulator-fixed"; ++ enable-active-high; ++ gpio = <&gpio0 4 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&host_vbus_drv>; ++ regulator-name = "vcc_host"; ++ regulator-always-on; ++ }; ++ ++ xin32k: xin32k { ++ compatible = "fixed-clock"; ++ clock-frequency = <32768>; ++ clock-output-names = "xin32k"; ++ #clock-cells = <0>; ++ }; ++ ++}; ++ ++&cif_clkout { ++ /* cif_clkout */ ++ rockchip,pins = <1 RK_PB3 1 &pcfg_pull_none_4ma>; ++}; ++ ++&cpu_l0 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu_l1 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu_l2 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu_l3 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu_b0 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu_b1 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu_b2 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu_b3 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&gpu { ++ logic-supply = <&vdd_logic>; ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&dmc { ++ status = "okay"; ++ center-supply = <&vdd_logic>; ++ devfreq-events = <&dfi>; ++ upthreshold = <60>; ++ downdifferential = <20>; ++ system-status-freq = < ++ /*system status freq(KHz)*/ ++ SYS_STATUS_NORMAL 600000 ++ SYS_STATUS_REBOOT 600000 ++ SYS_STATUS_SUSPEND 240000 ++ SYS_STATUS_VIDEO_1080P 396000 ++ SYS_STATUS_VIDEO_4K 600000 ++ SYS_STATUS_PERFORMANCE 600000 ++ SYS_STATUS_BOOST 396000 ++ SYS_STATUS_DUALVIEW 600000 ++ SYS_STATUS_ISP 528000 ++ >; ++ vop-bw-dmc-freq = < ++ /* min_bw(MB/s) max_bw(MB/s) freq(KHz) */ ++ 0 582 240000 ++ 583 99999 396000 ++ >; ++ vop-dclk-mode = <1>; ++ auto-min-freq = <240000>; ++ auto-freq-en = <0>; ++}; ++ ++&dsi { ++ status = "okay"; ++ ++ panel@0 { ++ compatible = "simple-panel-dsi"; ++ reg = <0>; ++ backlight = <&backlight>; ++ enable-gpios = <&gpio0 RK_PC6 GPIO_ACTIVE_HIGH>; ++ /* back-gpios = <&gpio3 RK_PD4 GPIO_ACTIVE_HIGH>; */ ++ reset-gpios = <&gpio3 RK_PB1 GPIO_ACTIVE_LOW>; ++ prepare-delay-ms = <8>; ++ enable-delay-ms = <3>; ++ reset-delay-ms = <50>; ++ init-delay-ms = <20>; ++ ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | ++ MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ width-mm = <153>; ++ height-mm = <85>; ++ panel-init-sequence = [ ++ 05 1e 01 01 ++ 15 00 02 80 47 ++ 15 00 02 81 40 ++ 15 00 02 82 04 ++ 15 00 02 83 77 ++ 15 00 02 84 0f ++ 15 00 02 85 70 ++ 15 78 02 86 70 ++ ]; ++ panel-exit-sequence = [ ++ 05 00 01 28 ++ 05 00 01 10 ++ ]; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <49500000>; ++ hactive = <1024>; ++ vactive = <600>; ++ hback-porch = <120>; ++ hfront-porch = <80>; ++ vback-porch = <14>; ++ vfront-porch = <14>; ++ hsync-len = <40>; ++ vsync-len = <4>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ }; ++}; ++ ++&emmc { ++ bus-width = <8>; ++ cap-mmc-highspeed; ++ mmc-hs200-1_8v; ++ no-sdio; ++ no-sd; ++ disable-wp; ++ non-removable; ++ num-slots = <1>; ++ status = "okay"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ ++ vdd_cpu: tcs4525@10 { ++ compatible = "tcs,tcs452x"; ++ reg = <0x10>; ++ regulator-compatible = "fan53555-reg"; ++ pinctrl-0 = <&vsel_gpio>; ++ vsel-gpios = <&gpio0 0 GPIO_ACTIVE_HIGH>; ++ regulator-name = "vdd_cpu"; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1390000>; ++ regulator-ramp-delay = <2300>; ++ fcs,suspend-voltage-selector = <1>; ++ regulator-boot-on; ++ regulator-always-on; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ rk817: pmic@20 { ++ compatible = "rockchip,rk817"; ++ reg = <0x20>; ++ interrupt-parent = <&gpio0>; ++ interrupts = <1 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default", "pmic-sleep", ++ "pmic-power-off", "pmic-reset"; ++ pinctrl-0 = <&pmic_int>; ++ pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; ++ pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; ++ pinctrl-3 = <&soc_slppin_slp>, <&rk817_slppin_slp>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ //fb-inner-reg-idxs = <2>; ++ /* 1: rst regs (default in codes), 0: rst the pmic */ ++ pmic-reset-func = <1>; ++ ++ vcc1-supply = <&vccsys>; ++ vcc2-supply = <&vccsys>; ++ vcc3-supply = <&vccsys>; ++ vcc4-supply = <&vccsys>; ++ vcc5-supply = <&vccsys>; ++ vcc6-supply = <&vccsys>; ++ vcc7-supply = <&vcc_io>; ++ vcc8-supply = <&vccsys>; ++ vcc9-supply = <&dcdc_boost>; ++ ++ pwrkey { ++ status = "okay"; ++ }; ++ ++ pinctrl_rk8xx: pinctrl_rk8xx { ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ rk817_ts_gpio1: rk817_ts_gpio1 { ++ pins = "gpio_ts"; ++ function = "pin_fun1"; ++ /* output-low; */ ++ /* input-enable; */ ++ }; ++ ++ rk817_gt_gpio2: rk817_gt_gpio2 { ++ pins = "gpio_gt"; ++ function = "pin_fun1"; ++ }; ++ ++ rk817_pin_ts: rk817_pin_ts { ++ pins = "gpio_ts"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_pin_gt: rk817_pin_gt { ++ pins = "gpio_gt"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_slppin_null: rk817_slppin_null { ++ pins = "gpio_slp"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_slppin_slp: rk817_slppin_slp { ++ pins = "gpio_slp"; ++ function = "pin_fun1"; ++ }; ++ ++ rk817_slppin_pwrdn: rk817_slppin_pwrdn { ++ pins = "gpio_slp"; ++ function = "pin_fun2"; ++ }; ++ ++ rk817_slppin_rst: rk817_slppin_rst { ++ pins = "gpio_slp"; ++ function = "pin_fun3"; ++ }; ++ }; ++ ++ regulators { ++ vdd_logic: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <950000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_logic"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <950000>; ++ }; ++ }; ++ ++ vcc_3v3: DCDC_REG2 { ++ regulator-name = "vcc_3v3"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_ddr"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_io: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-name = "vcc_io"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ ++ vcc_1v0: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ regulator-name = "vcc_1v0"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc1v8_soc: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcc1v8_soc"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd1v0_soc: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ ++ regulator-name = "vcc1v0_soc"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc3v3_pmu: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vcc3v3_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ ++ }; ++ }; ++ ++ vccio_sd: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vccio_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc_sd: LDO_REG6 { ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vcc_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ ++ }; ++ }; ++ ++ vcc2v8_dvp: LDO_REG7 { ++ regulator-boot-on; ++ regulator-min-microvolt = <2800000>; ++ regulator-max-microvolt = <2800000>; ++ ++ regulator-name = "vcc2v8_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <2800000>; ++ }; ++ }; ++ ++ vcc1v8_dvp: LDO_REG8 { ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcc1v8_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd1v5_dvp: LDO_REG9 { ++ regulator-boot-on; ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ ++ regulator-name = "vdd1v5_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <1500000>; ++ }; ++ }; ++ ++ dcdc_boost: BOOST { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ regulator-name = "boost"; ++ }; ++ ++ otg_switch: OTG_SWITCH { ++ regulator-name = "otg_switch"; ++ }; ++ }; ++ ++ battery { ++ compatible = "rk817,battery"; ++ ocv_table = <3500 3548 3592 3636 3687 3740 3780 ++ 3806 3827 3846 3864 3889 3929 3964 ++ 3993 4015 4030 4041 4056 4076 4148>; ++ design_capacity = <4000>; ++ design_qmax = <4200>; ++ bat_res = <100>; ++ sleep_enter_current = <150>; ++ sleep_exit_current = <180>; ++ sleep_filter_current = <100>; ++ power_off_thresd = <3500>; ++ zero_algorithm_vol = <3850>; ++ max_soc_offset = <60>; ++ monitor_sec = <5>; ++ sample_res = <10>; ++ virtual_power = <0>; ++ }; ++ ++ charger { ++ compatible = "rk817,charger"; ++ min_input_voltage = <4500>; ++ max_input_current = <1500>; ++ max_chrg_current = <2000>; ++ max_chrg_voltage = <4200>; ++ chrg_term_mode = <0>; ++ chrg_finish_cur = <300>; ++ virtual_power = <0>; ++ dc_det_adc = <0>; ++ extcon = <&u2phy>; ++ }; ++ ++ rk817_codec: codec { ++ #sound-dai-cells = <0>; ++ compatible = "rockchip,rk817-codec"; ++ clocks = <&cru SCLK_I2S_8CH_OUT>; ++ clock-names = "mclk"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s_8ch_mclk>; ++ hp-volume = <20>; ++ spk-volume = <3>; ++ mic-in-differential; ++ status = "okay"; ++ }; ++ }; ++}; ++ ++&i2c2 { ++ status = "okay"; ++ ++ ts@5a { ++ compatible = "cst2xxse"; ++ reg = <0x5a>; ++ irq-gpio = <&gpio0 RK_PB4 IRQ_TYPE_LEVEL_LOW>; ++ //touch-gpio = <&gpio1 GPIO_B0 IRQ_TYPE_LEVEL_LOW>; /* TP_INT == GPIO1_B0 */ ++ //reset-gpio = <&gpio0 GPIO_D1 GPIO_ACTIVE_LOW>; /* TP_RST == GPIO0_D1 */ ++ //power-gpio = <&gpio0 GPIO_C5 GPIO_ACTIVE_LOW>; ++ //max-x = <800>; ++ //max-y = <480>; ++ status = "okay"; ++ }; ++}; ++ ++&i2c3 { ++ status = "okay"; ++ ++ gc0312: gc0312@21 { ++ status = "okay"; ++ compatible = "galaxycore,gc0312"; ++ reg = <0x21>; ++ clocks = <&cru SCLK_VIP_OUT>; ++ clock-names = "xvclk"; ++ ++ avdd-supply = <&vcc2v8_dvp>; ++ dovdd-supply = <&vcc1v8_dvp>; ++ dvdd-supply = <&vdd1v5_dvp>; ++ ++ pwdn-gpios = <&gpio1 RK_PB5 GPIO_ACTIVE_HIGH>; ++ ++ rockchip,camera-module-index = <1>; ++ rockchip,camera-module-facing = "front"; ++ rockchip,camera-module-name = "CameraKing"; ++ rockchip,camera-module-lens-name = "Largan"; ++ port { ++ gc0312_out: endpoint { ++ remote-endpoint = <&dvp_in_fcam>; ++ }; ++ }; ++ }; ++ ++ gc2145: gc2145@3c { ++ status = "okay"; ++ compatible = "galaxycore,gc2145"; ++ reg = <0x3c>; ++ clocks = <&cru SCLK_VIP_OUT>; ++ clock-names = "xvclk"; ++ ++ avdd-supply = <&vcc2v8_dvp>; ++ dovdd-supply = <&vcc1v8_dvp>; ++ dvdd-supply = <&vdd1v5_dvp>; ++ ++ pwdn-gpios = <&gpio1 RK_PB4 GPIO_ACTIVE_HIGH>; ++ reset-gpios = <&gpio0 RK_PB6 GPIO_ACTIVE_HIGH>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "CameraKing"; ++ rockchip,camera-module-lens-name = "Largan"; ++ port { ++ gc2145_out: endpoint { ++ remote-endpoint = <&dvp_in_bcam>; ++ }; ++ }; ++ }; ++ ++ ov8858: ov8858@36 { ++ status = "disabled"; ++ compatible = "ovti,ov8858"; ++ reg = <0x36>; ++ clocks = <&cru SCLK_VIP_OUT>; ++ clock-names = "xvclk"; ++ ++ avdd-supply = <&vcc2v8_dvp>; ++ dovdd-supply = <&vcc1v8_dvp>; ++ dvdd-supply = <&vdd1v5_dvp>; ++ ++ pwdn-gpios = <&gpio1 RK_PB4 GPIO_ACTIVE_HIGH>; ++ reset-gpios = <&gpio0 RK_PB6 GPIO_ACTIVE_HIGH>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "CameraKing"; ++ rockchip,camera-module-lens-name = "Largan-9569A2"; ++ port { ++ ov8858_out: endpoint { ++ remote-endpoint = <&mipi_in>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++}; ++ ++&i2c4 { ++ status = "okay"; ++ ++ sc7a30: sc7a30@18 { ++ status = "okay"; ++ compatible = "gs_sc7a30"; ++ reg = <0x18>; ++ type = ; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sc7a30_irq_gpio>; ++ irq-gpio = <&gpio3 RK_PB6 IRQ_TYPE_LEVEL_LOW>; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ layout = <6>; ++ reprobe_en = <1>; ++ }; ++ ++}; ++ ++&i2s_8ch { ++ status = "okay"; ++ rockchip,i2s-broken-burst-len; ++ rockchip,playback-channels = <8>; ++ rockchip,capture-channels = <2>; ++ #sound-dai-cells = <0>; ++}; ++ ++&io_domains { ++ status = "okay"; ++ ++ dvp-supply = <&vcc1v8_dvp>; ++ audio-supply = <&vcc_io>; ++ gpio30-supply = <&vcc_io>; ++ gpio1830-supply = <&vcc_io>; ++ sdcard-supply = <&vccio_sd>; ++ wifi-supply = <&vcc_3v3>; ++}; ++ ++&isp_dvp_d2d9 { ++ rockchip,pins = ++ /* cif_data4 ... cif_data9 */ ++ <1 RK_PA2 1 &pcfg_pull_down>, ++ <1 RK_PA3 1 &pcfg_pull_down>, ++ <1 RK_PA4 1 &pcfg_pull_down>, ++ <1 RK_PA5 1 &pcfg_pull_down>, ++ <1 RK_PA6 1 &pcfg_pull_down>, ++ <1 RK_PA7 1 &pcfg_pull_down>, ++ /* cif_sync, cif_href */ ++ <1 RK_PB0 1 &pcfg_pull_down>, ++ <1 RK_PB1 1 &pcfg_pull_down>, ++ /* cif_clkin */ ++ <1 RK_PB2 1 &pcfg_pull_down>; ++}; ++ ++&isp_dvp_d10d11 { ++ rockchip,pins = ++ /* cif_data10, cif_data11 */ ++ <1 RK_PB6 1 &pcfg_pull_down>, ++ <1 RK_PB7 1 &pcfg_pull_down>; ++}; ++ ++&isp_mmu { ++ status = "okay"; ++}; ++ ++&mipi_dphy_rx0 { ++ status = "disabled"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ov8858_out>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_rx_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp_mipi_in>; ++ }; ++ }; ++ }; ++}; ++ ++&nandc0 { ++ status = "okay"; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ ++ pmu-supply = <&vcc3v3_pmu>; ++ vop-supply = <&vcc3v3_pmu>; ++}; ++ ++&pwm0 { ++ status = "okay"; ++}; ++ ++&pinctrl { ++ ++ headphone { ++ hp_det: hp-det { ++ rockchip,pins = <0 RK_PC7 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ pmic { ++ pmic_int: pmic_int { ++ rockchip,pins = ++ <0 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ soc_slppin_gpio: soc_slppin_gpio { ++ rockchip,pins = ++ <0 RK_PA0 RK_FUNC_GPIO &pcfg_output_low>; ++ }; ++ ++ soc_slppin_slp: soc_slppin_slp { ++ rockchip,pins = ++ <0 RK_PA0 1 &pcfg_pull_none>; ++ }; ++ ++ soc_slppin_rst: soc_slppin_rst { ++ rockchip,pins = ++ <0 RK_PA0 2 &pcfg_pull_none>; ++ }; ++ ++ vsel_gpio: vsel-gpio { ++ rockchip,pins = <0 RK_PA0 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ sc7a30 { ++ sc7a30_irq_gpio: sc7a30_irq_gpio { ++ rockchip,pins = <3 RK_PB6 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ dc_det { ++ dc_irq_gpio: dc-irq-gpio { ++ rockchip,pins = <0 RK_PC1 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ pcfg_pull_none_4ma: pcfg-pull-none-4ma { ++ bias-disable; ++ drive-strength = <4>; ++ }; ++ ++ pcfg_pull_none_smt: pcfg-pull-none-smt { ++ bias-disable; ++ input-schmitt-enable; ++ }; ++ ++ pcfg_output_high: pcfg-output-high { ++ output-high; ++ }; ++ ++ pcfg_output_low: pcfg-output-low { ++ output-low; ++ }; ++ ++ pcfg_input_high: pcfg-input-high { ++ bias-pull-up; ++ input-enable; ++ }; ++ ++ pcfg_input: pcfg-input { ++ input-enable; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ usb2 { ++ host_vbus_drv: host-vbus-drv { ++ rockchip,pins = <0 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart0_rts_gpio: uart0-rts-gpio { ++ rockchip,pins = <2 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++&rkisp1 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&isp_dvp_d2d9 &isp_dvp_d10d11 &cif_clkout>; ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dvp_in_fcam: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&gc0312_out>; ++ }; ++ ++ dvp_in_bcam: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&gc2145_out>; ++ }; ++ ++ isp_mipi_in: endpoint@2 { ++ reg = <2>; ++ remote-endpoint = <&dphy_rx_out>; ++ }; ++ }; ++}; ++ ++&route_dsi { ++ status = "okay"; ++}; ++ ++&rockchip_suspend { ++ status = "okay"; ++ rockchip,sleep-mode-config = < ++ (0 ++ | RKPM_SLP_ARMOFF ++ | RKPM_SLP_PMU_PLLS_PWRDN ++ | RKPM_SLP_PMU_PMUALIVE_32K ++ | RKPM_SLP_SFT_PLLS_DEEP ++ | RKPM_SLP_PMU_DIS_OSC ++ | RKPM_SLP_SFT_PD_NBSCUS ++ ) ++ >; ++ rockchip,wakeup-config = < ++ (0 ++ | RKPM_GPIO_WKUP_EN ++ | RKPM_USB_WKUP_EN ++ | RKPM_CLUSTER_L_WKUP_EN ++ ) ++ >; ++}; ++ ++&saradc { ++ status = "okay"; ++}; ++ ++&sdmmc { ++ clock-frequency = <37500000>; ++ clock-freq-min-max = <400000 37500000>; ++ no-sdio; ++ no-mmc; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ card-detect-delay = <200>; ++ disable-wp; ++ num-slots = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_cd &sdmmc_bus4>; ++ vmmc-supply = <&vcc_sd>; ++ status = "disabled"; ++}; ++ ++&sdio0 { ++ max-frequency = <50000000>; ++ no-sd; ++ no-mmc; ++ bus-width = <4>; ++ disable-wp; ++ cap-sd-highspeed; ++ cap-sdio-irq; ++ keep-power-in-suspend; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ non-removable; ++ num-slots = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>; ++ rockchip,default-sample-phase = <90>; ++ status = "okay"; ++}; ++ ++&tsadc { ++ tsadc-supply = <&vdd_cpu>; ++ status = "okay"; ++}; ++ ++&uart0 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart0_xfer &uart0_cts>; ++ status = "okay"; ++}; ++ ++&u2phy { ++ status = "okay"; ++ ++ u2phy_otg: otg-port { ++ status = "okay"; ++ }; ++ ++ u2phy_host: host-port { ++ phy-supply = <&vcc_host>; ++ status = "okay"; ++ }; ++}; ++ ++&usb_host0_ehci { ++ status = "okay"; ++}; ++ ++&usb_host0_ohci { ++ status = "okay"; ++}; ++ +diff --git a/arch/arm64/boot/dts/rockchip/rk3368a-817-tablet.dts b/arch/arm64/boot/dts/rockchip/rk3368a-817-tablet.dts +new file mode 100755 +index 000000000000..31f25e44504d +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3368a-817-tablet.dts +@@ -0,0 +1,1333 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ */ ++ ++/dts-v1/; ++#include ++#include ++#include ++#include "rk3368.dtsi" ++#include "rk3368-android.dtsi" ++/ { ++ model = "Rockchip rk3368a tablet rk817 board"; ++ compatible = "rockchip,tablet", "rockchip,rk3368a", "rockchip,rk3368"; ++ ++ adc_keys: adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 1>; ++ io-channel-names = "buttons"; ++ keyup-threshold-microvolt = <1024000>; ++ poll-interval = <100>; ++ ++ vol-up-key { ++ label = "volume up"; ++ linux,code = ; ++ press-threshold-microvolt = <1000>; ++ }; ++ ++ vol-down-key { ++ label = "volume down"; ++ linux,code = ; ++ press-threshold-microvolt = <170000>; ++ }; ++ }; ++ ++ backlight: backlight { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm0 0 25000 1>; ++ brightness-levels = < ++ 30 30 30 31 31 31 32 32 ++ 32 33 33 33 34 34 34 35 ++ 35 35 36 36 36 37 37 37 ++ 38 38 38 39 39 39 40 40 ++ 40 41 41 41 42 42 42 43 ++ 43 43 44 44 44 45 46 47 ++ 48 49 50 51 52 53 54 55 ++ 56 57 58 59 60 61 62 63 ++ 64 65 66 67 68 69 70 71 ++ 72 73 74 75 76 77 78 79 ++ 80 81 82 83 84 85 86 87 ++ 88 89 90 91 92 93 94 95 ++ 96 97 98 99 100 101 102 103 ++ 104 105 106 107 108 109 110 111 ++ 112 113 114 115 116 117 118 119 ++ 120 121 122 123 124 125 126 127 ++ 128 129 130 131 132 133 134 135 ++ 136 137 138 139 140 141 142 143 ++ 144 145 146 147 148 149 150 151 ++ 152 153 154 155 156 157 158 159 ++ 160 161 162 163 164 165 166 167 ++ 168 169 170 171 172 173 174 175 ++ 176 177 178 179 180 181 182 183 ++ 184 185 186 187 188 189 190 191 ++ 192 193 194 195 196 197 198 199 ++ 200 201 202 203 204 205 206 207 ++ 208 209 210 211 212 213 214 215 ++ 216 217 218 219 220 221 222 223 ++ 224 225 226 227 228 229 230 231 ++ 232 232 232 233 233 233 234 234 ++ 234 235 235 235 236 236 236 237 ++ 237 238 238 239 239 240 240 240>; ++ default-brightness-level = <200>; ++ enable-gpios = <&gpio3 28 GPIO_ACTIVE_HIGH>; ++ }; ++ ++ charge-animation { ++ compatible = "rockchip,uboot-charge"; ++ rockchip,uboot-charge-on = <1>; ++ rockchip,android-charge-on = <0>; ++ rockchip,uboot-low-power-voltage = <3400>; ++ rockchip,screen-on-voltage = <3400>; ++ status = "okay"; ++ }; ++ ++ chosen: chosen { ++ bootargs = "earlycon=uart8250,mmio32,0xff690000 console=ttyFIQ0 androidboot.baseband=N/A androidboot.veritymode=enforcing androidboot.hardware=rk30board androidboot.console=ttyFIQ0 init=/init kpti=0"; ++ }; ++ ++ rk817-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip-rk817-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,widgets = ++ "Microphone", "Mic Jack", ++ "Headphone", "Headphone Jack"; ++ simple-audio-card,routing = ++ "Mic Jack", "MICBIAS1", ++ "IN1P", "Mic Jack", ++ "Headphone Jack", "HPOL", ++ "Headphone Jack", "HPOR"; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s_8ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&rk817_codec>; ++ }; ++ }; ++ ++ rk_headset: rk-headset { ++ compatible = "rockchip_headset"; ++ headset_gpio = <&gpio0 RK_PC7 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hp_det>; ++ io-channels = <&saradc 2>; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio3 4 GPIO_ACTIVE_LOW>; /* GPIO3_A4 */ ++ }; ++ ++ vccsys: vccsys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc3v8_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3800000>; ++ regulator-max-microvolt = <3800000>; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ /* wifi_chip_type - wifi chip define ++ * ap6210, ap6330, ap6335 ++ * rtl8188eu, rtl8723bs, rtl8723bu ++ * esp8089 ++ */ ++ wifi_chip_type = "rtl8723bs"; ++ WIFI,vbat_gpio = <&gpio0 2 GPIO_ACTIVE_HIGH>; ++ WIFI,host_wake_irq = <&gpio3 6 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ uart_rts_gpios = <&gpio2 27 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default","rts_gpio"; ++ pinctrl-0 = <&uart0_rts>; ++ pinctrl-1 = <&uart0_rts_gpio>; ++ ++ //BT,power_gpio = <&gpio3 3 GPIO_ACTIVE_HIGH>; ++ BT,reset_gpio = <&gpio3 13 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio3 2 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio3 7 GPIO_ACTIVE_HIGH>; ++ ++ status = "okay"; ++ }; ++ ++ vcc_sys: vcc-sys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3800000>; ++ regulator-max-microvolt = <3800000>; ++ }; ++ ++ vcc_host: vcc-host { ++ compatible = "regulator-fixed"; ++ enable-active-high; ++ gpio = <&gpio0 4 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&host_vbus_drv>; ++ regulator-name = "vcc_host"; ++ regulator-always-on; ++ }; ++ ++ xin32k: xin32k { ++ compatible = "fixed-clock"; ++ clock-frequency = <32768>; ++ clock-output-names = "xin32k"; ++ #clock-cells = <0>; ++ }; ++ ++}; ++ ++&cif_clkout { ++ /* cif_clkout */ ++ rockchip,pins = <1 RK_PB3 1 &pcfg_pull_none_4ma>; ++}; ++ ++&cpu_l0 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu_l1 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu_l2 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu_l3 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu_b0 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu_b1 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu_b2 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu_b3 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&gpu { ++ logic-supply = <&vdd_logic>; ++}; ++ ++&dsi { ++ status = "okay"; ++ //rockchip,lane-rate = <500>; ++ ++ panel:panel@0 { ++ compatible = "simple-panel-dsi"; ++ reg = <0>; ++ backlight = <&backlight>; ++ enable-gpios = <&gpio0 RK_PC6 GPIO_ACTIVE_HIGH>; ++ prepare-delay-ms = <20>; ++ reset-delay-ms = <20>; ++ init-delay-ms = <20>; ++ enable-delay-ms = <120>; ++ disable-delay-ms = <20>; ++ unprepare-delay-ms = <20>; ++ ++ width-mm = <135>; ++ height-mm = <216>; ++ ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | ++ MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ ++ panel-init-sequence = [ ++ 39 00 04 FF 98 81 03 ++ 15 00 02 01 00 ++ 15 00 02 02 00 ++ 15 00 02 03 53 ++ 15 00 02 04 00 ++ 15 00 02 05 00 ++ 15 00 02 06 08 ++ 15 00 02 07 00 ++ 15 00 02 08 00 ++ 15 00 02 09 00 ++ 15 00 02 0A 00 ++ 15 00 02 0B 00 ++ 15 00 02 0C 00 ++ 15 00 02 0D 00 ++ 15 00 02 0E 00 ++ 15 00 02 0F 26 ++ 15 00 02 10 26 ++ 15 00 02 11 00 ++ 15 00 02 12 00 ++ 15 00 02 13 00 ++ 15 00 02 14 00 ++ 15 00 02 15 00 ++ 15 00 02 16 00 ++ 15 00 02 17 00 ++ 15 00 02 18 00 ++ 15 00 02 19 00 ++ 15 00 02 1A 00 ++ 15 00 02 1B 00 ++ 15 00 02 1C 00 ++ 15 00 02 1D 00 ++ 15 00 02 1E 40 ++ 15 00 02 1F C0 ++ 15 00 02 20 06 ++ 15 00 02 21 01 ++ 15 00 02 22 07 ++ 15 00 02 23 00 ++ 15 00 02 24 8A ++ 15 00 02 25 8A ++ 15 00 02 26 00 ++ 15 00 02 27 00 ++ 15 00 02 28 33 ++ 15 00 02 29 33 ++ 15 00 02 2A 00 ++ 15 00 02 2B 00 ++ 15 00 02 2C 08 ++ 15 00 02 2D 08 ++ 15 00 02 2E 0B ++ 15 00 02 2F 0B ++ 15 00 02 30 00 ++ 15 00 02 31 00 ++ 15 00 02 32 42 ++ 15 00 02 33 00 ++ 15 00 02 34 00 ++ 15 00 02 35 0A ++ 15 00 02 36 00 ++ 15 00 02 37 08 ++ 15 00 02 38 3C ++ 15 00 02 39 00 ++ 15 00 02 3A 00 ++ 15 00 02 3B 00 ++ 15 00 02 3C 00 ++ 15 00 02 3D 00 ++ 15 00 02 3E 00 ++ 15 00 02 3F 00 ++ 15 00 02 40 00 ++ 15 00 02 41 00 ++ 15 00 02 42 00 ++ 15 00 02 43 08 ++ 15 00 02 44 00 ++ 15 00 02 50 01 ++ 15 00 02 51 23 ++ 15 00 02 52 45 ++ 15 00 02 53 67 ++ 15 00 02 54 89 ++ 15 00 02 55 AB ++ 15 00 02 56 01 ++ 15 00 02 57 23 ++ 15 00 02 58 45 ++ 15 00 02 59 67 ++ 15 00 02 5A 89 ++ 15 00 02 5B AB ++ 15 00 02 5C CD ++ 15 00 02 5D EF ++ 15 00 02 5E 00 ++ 15 00 02 5F 01 ++ 15 00 02 60 01 ++ 15 00 02 61 06 ++ 15 00 02 62 06 ++ 15 00 02 63 06 ++ 15 00 02 64 06 ++ 15 00 02 65 00 ++ 15 00 02 66 00 ++ 15 00 02 67 17 ++ 15 00 02 68 02 ++ 15 00 02 69 16 ++ 15 00 02 6A 16 ++ 15 00 02 6B 02 ++ 15 00 02 6C 0D ++ 15 00 02 6D 0D ++ 15 00 02 6E 0C ++ 15 00 02 6F 0C ++ 15 00 02 70 0F ++ 15 00 02 71 0F ++ 15 00 02 72 0E ++ 15 00 02 73 0E ++ 15 00 02 74 02 ++ 15 00 02 75 01 ++ 15 00 02 76 01 ++ 15 00 02 77 06 ++ 15 00 02 78 06 ++ 15 00 02 79 06 ++ 15 00 02 7A 06 ++ 15 00 02 7B 00 ++ 15 00 02 7C 00 ++ 15 00 02 7D 17 ++ 15 00 02 7E 02 ++ 15 00 02 7F 16 ++ 15 00 02 80 16 ++ 15 00 02 81 02 ++ 15 00 02 82 0D ++ 15 00 02 83 0D ++ 15 00 02 84 0C ++ 15 00 02 85 0C ++ 15 00 02 86 0F ++ 15 00 02 87 0F ++ 15 00 02 88 0E ++ 15 00 02 89 0E ++ 15 00 02 8A 02 ++ 39 00 04 FF 98 81 04 ++ 15 00 02 6E 2B ++ 15 00 02 6F 35 ++ 15 00 02 3A A4 ++ 15 00 02 8D 1A ++ 15 00 02 87 BA ++ 15 00 02 B2 D1 ++ 15 00 02 88 0B ++ 15 00 02 38 01 ++ 15 00 02 39 00 ++ 15 00 02 B5 07 ++ 15 00 02 31 75 ++ 15 00 02 3B 98 ++ 39 00 04 FF 98 81 01 ++ 15 00 02 22 0A ++ 15 00 02 31 00 ++ 15 00 02 53 40 ++ 15 00 02 55 40 ++ 15 00 02 50 95 ++ 15 00 02 51 90 ++ 15 00 02 60 22 ++ 15 00 02 62 20 ++ 15 00 02 A0 00 ++ 15 00 02 A1 1B ++ 15 00 02 A2 2A ++ 15 00 02 A3 14 ++ 15 00 02 A4 17 ++ 15 00 02 A5 2B ++ 15 00 02 A6 1F ++ 15 00 02 A7 20 ++ 15 00 02 A8 93 ++ 15 00 02 A9 1E ++ 15 00 02 AA 2A ++ 15 00 02 AB 7E ++ 15 00 02 AC 1B ++ 15 00 02 AD 19 ++ 15 00 02 AE 4C ++ 15 00 02 AF 22 ++ 15 00 02 B0 28 ++ 15 00 02 B1 4B ++ 15 00 02 B2 59 ++ 15 00 02 B3 23 ++ 15 00 02 C0 00 ++ 15 00 02 C1 1B ++ 15 00 02 C2 2A ++ 15 00 02 C3 14 ++ 15 00 02 C4 17 ++ 15 00 02 C5 2B ++ 15 00 02 C6 1F ++ 15 00 02 C7 20 ++ 15 00 02 C8 93 ++ 15 00 02 C9 1E ++ 15 00 02 CA 2A ++ 15 00 02 CB 7E ++ 15 00 02 CC 1B ++ 15 00 02 CD 19 ++ 15 00 02 CE 4C ++ 15 00 02 CF 22 ++ 15 00 02 D0 28 ++ 15 00 02 D1 4B ++ 15 00 02 D2 59 ++ 15 00 02 D3 23 ++ //39 00 04 FF 98 81 04 ++ //05 00 02 2D 80 ++ //05 00 02 2F 31 ++ 39 00 04 FF 98 81 00 ++ 05 78 01 11 ++ 05 14 01 29 ++ 15 00 02 35 00 ++ ]; ++ ++ panel-exit-sequence = [ ++ 05 00 01 28 ++ 05 00 01 10 ++ ]; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <78000000>; ++ hactive = <800>; ++ vactive = <1280>; ++ hfront-porch = <60>;//70 //16 ++ hsync-len = <30>; //20 //5 ++ hback-porch = <60>; //59 ++ vfront-porch = <20>; //16 //8 ++ vsync-len = <8>; //5 ++ vback-porch = <16>; //22 //3 ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ }; ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&dmc { ++ status = "okay"; ++ center-supply = <&vdd_logic>; ++ devfreq-events = <&dfi>; ++ upthreshold = <60>; ++ downdifferential = <20>; ++ system-status-freq = < ++ /*system status freq(KHz)*/ ++ SYS_STATUS_NORMAL 600000 ++ SYS_STATUS_REBOOT 600000 ++ SYS_STATUS_SUSPEND 240000 ++ SYS_STATUS_VIDEO_1080P 396000 ++ SYS_STATUS_VIDEO_4K 600000 ++ SYS_STATUS_PERFORMANCE 600000 ++ SYS_STATUS_BOOST 396000 ++ SYS_STATUS_DUALVIEW 600000 ++ SYS_STATUS_ISP 528000 ++ >; ++ vop-bw-dmc-freq = < ++ /* min_bw(MB/s) max_bw(MB/s) freq(KHz) */ ++ 0 582 240000 ++ 583 99999 396000 ++ >; ++ vop-dclk-mode = <1>; ++ auto-min-freq = <240000>; ++ auto-freq-en = <0>; ++}; ++ ++&emmc { ++ bus-width = <8>; ++ cap-mmc-highspeed; ++ mmc-hs200-1_8v; ++ no-sdio; ++ no-sd; ++ disable-wp; ++ non-removable; ++ num-slots = <1>; ++ status = "okay"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ ++ vdd_cpu: syr827@40 { ++ compatible = "silergy,syr827"; ++ status = "okay"; ++ reg = <0x40>; ++ ++ regulator-compatible = "fan53555-reg"; ++ regulator-name = "vdd_cpu"; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1500000>; ++ regulator-ramp-delay = <1000>; ++ fcs,suspend-voltage-selector = <1>; ++ pinctrl-0 = <&vsel_gpio>; ++ vsel-gpios = <&gpio0 0 GPIO_ACTIVE_HIGH>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-state = <3>; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ rk817: pmic@20 { ++ compatible = "rockchip,rk817"; ++ reg = <0x20>; ++ interrupt-parent = <&gpio0>; ++ interrupts = <1 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default", "pmic-sleep", ++ "pmic-power-off", "pmic-reset"; ++ pinctrl-0 = <&pmic_int>; ++ pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; ++ pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; ++ pinctrl-3 = <&soc_slppin_slp>, <&rk817_slppin_slp>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ //fb-inner-reg-idxs = <2>; ++ /* 1: rst regs (default in codes), 0: rst the pmic */ ++ pmic-reset-func = <1>; ++ ++ vcc1-supply = <&vccsys>; ++ vcc2-supply = <&vccsys>; ++ vcc3-supply = <&vccsys>; ++ vcc4-supply = <&vccsys>; ++ vcc5-supply = <&vccsys>; ++ vcc6-supply = <&vccsys>; ++ vcc7-supply = <&vcc_io>; ++ vcc8-supply = <&vccsys>; ++ vcc9-supply = <&dcdc_boost>; ++ ++ pwrkey { ++ status = "okay"; ++ }; ++ ++ pinctrl_rk8xx: pinctrl_rk8xx { ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ rk817_ts_gpio1: rk817_ts_gpio1 { ++ pins = "gpio_ts"; ++ function = "pin_fun1"; ++ /* output-low; */ ++ /* input-enable; */ ++ }; ++ ++ rk817_gt_gpio2: rk817_gt_gpio2 { ++ pins = "gpio_gt"; ++ function = "pin_fun1"; ++ }; ++ ++ rk817_pin_ts: rk817_pin_ts { ++ pins = "gpio_ts"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_pin_gt: rk817_pin_gt { ++ pins = "gpio_gt"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_slppin_null: rk817_slppin_null { ++ pins = "gpio_slp"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_slppin_slp: rk817_slppin_slp { ++ pins = "gpio_slp"; ++ function = "pin_fun1"; ++ }; ++ ++ rk817_slppin_pwrdn: rk817_slppin_pwrdn { ++ pins = "gpio_slp"; ++ function = "pin_fun2"; ++ }; ++ ++ rk817_slppin_rst: rk817_slppin_rst { ++ pins = "gpio_slp"; ++ function = "pin_fun3"; ++ }; ++ }; ++ ++ regulators { ++ vdd_logic: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <950000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_logic"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <950000>; ++ }; ++ }; ++ ++ vcc_3v3: DCDC_REG2 { ++ regulator-name = "vcc_3v3"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_ddr"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_io: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcc_io"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc_1v0: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ regulator-name = "vcc_1v0"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc1v8_soc: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcc1v8_soc"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd1v0_soc: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1000000>; ++ regulator-max-microvolt = <1000000>; ++ ++ regulator-name = "vcc1v0_soc"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1000000>; ++ }; ++ }; ++ ++ vcc3v3_pmu: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vcc3v3_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ ++ }; ++ }; ++ ++ vccio_sd: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vccio_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc_sd: LDO_REG6 { ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vcc_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ ++ }; ++ }; ++ ++ vcc2v8_dvp: LDO_REG7 { ++ regulator-boot-on; ++ regulator-min-microvolt = <2800000>; ++ regulator-max-microvolt = <2800000>; ++ ++ regulator-name = "vcc2v8_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <2800000>; ++ }; ++ }; ++ ++ vcc1v8_dvp: LDO_REG8 { ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcc1v8_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdd1v5_dvp: LDO_REG9 { ++ regulator-boot-on; ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ ++ regulator-name = "vdd1v5_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <1500000>; ++ }; ++ }; ++ ++ dcdc_boost: BOOST { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ regulator-name = "boost"; ++ }; ++ ++ otg_switch: OTG_SWITCH { ++ regulator-name = "otg_switch"; ++ }; ++ }; ++ ++ battery { ++ compatible = "rk817,battery"; ++ ocv_table = <3500 3548 3592 3636 3687 3740 3780 ++ 3806 3827 3846 3864 3889 3929 3964 ++ 3993 4015 4030 4041 4056 4076 4148>; ++ design_capacity = <5000>; ++ design_qmax = <5500>; ++ bat_res = <100>; ++ sleep_enter_current = <150>; ++ sleep_exit_current = <180>; ++ sleep_filter_current = <100>; ++ power_off_thresd = <3400>; ++ zero_algorithm_vol = <3850>; ++ max_soc_offset = <60>; ++ monitor_sec = <5>; ++ sample_res = <10>; ++ virtual_power = <0>; ++ }; ++ ++ charger { ++ compatible = "rk817,charger"; ++ min_input_voltage = <4500>; ++ max_input_current = <1500>; ++ max_chrg_current = <2000>; ++ max_chrg_voltage = <4200>; ++ chrg_term_mode = <0>; ++ chrg_finish_cur = <300>; ++ virtual_power = <0>; ++ dc_det_adc = <0>; ++ extcon = <&u2phy>; ++ }; ++ ++ rk817_codec: codec { ++ #sound-dai-cells = <0>; ++ compatible = "rockchip,rk817-codec"; ++ clocks = <&cru SCLK_I2S_8CH_OUT>; ++ clock-names = "mclk"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s_8ch_mclk>; ++ hp-volume = <20>; ++ spk-volume = <3>; ++ mic-in-differential; ++ out-l2spk-r2hp; ++ spk-ctl-gpios = <&gpio0 RK_PD3 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ }; ++}; ++ ++&i2c2 { ++ status = "okay"; ++ ++ ts@40 { ++ status = "okay"; ++ compatible = "GSL,GSL_THZY"; ++ reg = <0x40>; ++ irq_gpio_number = <&gpio0 RK_PB4 IRQ_TYPE_LEVEL_HIGH>; ++ rst_gpio_number = <&gpio0 RK_PB3 GPIO_ACTIVE_HIGH>; ++ }; ++}; ++ ++&i2c3 { ++ status = "okay"; ++ ++ gc032a: gc032a@21 { ++ status = "okay"; ++ compatible = "galaxycore,gc032a"; ++ reg = <0x21>; ++ clocks = <&cru SCLK_VIP_OUT>; ++ clock-names = "xvclk"; ++ ++ avdd-supply = <&vcc2v8_dvp>; ++ dovdd-supply = <&vcc1v8_dvp>; ++ dvdd-supply = <&vdd1v5_dvp>; ++ ++ pwdn-gpios = <&gpio1 RK_PB4 GPIO_ACTIVE_HIGH>; ++ rockchip,camera-module-index = <1>; ++ rockchip,camera-module-facing = "front"; ++ rockchip,camera-module-name = "CameraKing"; ++ rockchip,camera-module-lens-name = "Largan"; ++ port { ++ gc0312_out: endpoint { ++ remote-endpoint = <&dvp_in_fcam>; ++ }; ++ }; ++ }; ++ ++ gc2145: gc2145@3c { ++ status = "okay"; ++ compatible = "galaxycore,gc2145"; ++ reg = <0x3c>; ++ clocks = <&cru SCLK_VIP_OUT>; ++ clock-names = "xvclk"; ++ ++ avdd-supply = <&vcc2v8_dvp>; ++ dovdd-supply = <&vcc1v8_dvp>; ++ dvdd-supply = <&vdd1v5_dvp>; ++ ++ pwdn-gpios = <&gpio1 RK_PB5 GPIO_ACTIVE_HIGH>; ++ reset-gpios = <&gpio3 RK_PC4 GPIO_ACTIVE_LOW>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "CameraKing"; ++ rockchip,camera-module-lens-name = "Largan"; ++ port { ++ gc2145_mipi_out: endpoint { ++ remote-endpoint = <&mipi_in_bcam>; ++ data-lanes = <1>; ++ }; ++ }; ++ }; ++ ++ ov8858: ov8858@36 { ++ status = "disabled"; ++ compatible = "ovti,ov8858"; ++ reg = <0x36>; ++ clocks = <&cru SCLK_VIP_OUT>; ++ clock-names = "xvclk"; ++ ++ avdd-supply = <&vcc2v8_dvp>; ++ dovdd-supply = <&vcc1v8_dvp>; ++ dvdd-supply = <&vdd1v5_dvp>; ++ ++ pwdn-gpios = <&gpio1 RK_PB4 GPIO_ACTIVE_HIGH>; ++ reset-gpios = <&gpio0 RK_PB6 GPIO_ACTIVE_HIGH>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "CameraKing"; ++ rockchip,camera-module-lens-name = "Largan-9569A2"; ++ port { ++ ov8858_out: endpoint { ++ remote-endpoint = <&mipi_in>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++}; ++ ++&i2c4 { ++ status = "okay"; ++ ++ sensor@26 { ++ compatible = "gs_da223"; ++ reg = <0x26>; ++ type = ; ++ irq_enable = <0>; ++ poll_delay_ms = <10>; ++ layout = <3>; ++ }; ++ ++ sensor@19 { ++ compatible = "gs_sc7a20"; ++ reg = <0x19>; ++ type = ; ++ irq_enable = <0>; ++ poll_delay_ms = <10>; ++ layout = <1>; ++ }; ++ ++ mpu6500@68 { ++ status = "disabled"; ++ compatible = "invensense,mpu6500"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&mpu6500_irq_gpio>; ++ reg = <0x68>; ++ irq-gpio = <&gpio3 14 IRQ_TYPE_EDGE_RISING>; ++ mpu-int_config = <0x10>; ++ mpu-level_shifter = <0>; ++ mpu-orientation = <1 0 0 0 1 0 0 0 1>; ++ orientation-x= <1>; ++ orientation-y= <0>; ++ orientation-z= <1>; ++ support-hw-poweroff = <1>; ++ mpu-debug = <1>; ++ }; ++ ++ sensor@4c { ++ status = "disabled"; ++ compatible = "gs_mc3230"; ++ reg = <0x4c>; ++ type = ; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ layout = <9>; ++ reprobe_en = <1>; ++ irq-gpio = <&gpio3 RK_PB6 IRQ_TYPE_LEVEL_LOW>; ++ }; ++ ++ sensor@18 { ++ status = "gs_mc3230"; ++ compatible = "gs_sc7a30"; ++ reg = <0x18>; ++ type = ; ++ irq-gpio = <&gpio3 RK_PB6 IRQ_TYPE_LEVEL_LOW>; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ layout = <6>; ++ reprobe_en = <1>; ++ }; ++ ++ sensor@10 { ++ status = "gs_mc3230"; ++ compatible = "light_cm3218"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cm3218_irq_gpio>; ++ reg = <0x10>; ++ type = ; ++ irq-gpio = <&gpio3 15 IRQ_TYPE_EDGE_FALLING>; ++ irq_enable = <1>; ++ poll_delay_ms = <30>; ++ }; ++}; ++ ++&i2s_8ch { ++ status = "okay"; ++ rockchip,i2s-broken-burst-len; ++ rockchip,playback-channels = <8>; ++ rockchip,capture-channels = <2>; ++ #sound-dai-cells = <0>; ++}; ++ ++&io_domains { ++ status = "okay"; ++ ++ dvp-supply = <&vcc1v8_dvp>; ++ audio-supply = <&vcc_io>; ++ gpio30-supply = <&vcc_io>; ++ gpio1830-supply = <&vcc_io>; ++ sdcard-supply = <&vccio_sd>; ++ wifi-supply = <&vcc_3v3>; ++}; ++ ++&isp_dvp_d2d9 { ++ rockchip,pins = ++ /* cif_data4 ... cif_data9 */ ++ <1 RK_PA2 1 &pcfg_pull_down>, ++ <1 RK_PA3 1 &pcfg_pull_down>, ++ <1 RK_PA4 1 &pcfg_pull_down>, ++ <1 RK_PA5 1 &pcfg_pull_down>, ++ <1 RK_PA6 1 &pcfg_pull_down>, ++ <1 RK_PA7 1 &pcfg_pull_down>, ++ /* cif_sync, cif_href */ ++ <1 RK_PB0 1 &pcfg_pull_down>, ++ <1 RK_PB1 1 &pcfg_pull_down>, ++ /* cif_clkin */ ++ <1 RK_PB2 1 &pcfg_pull_down>; ++}; ++ ++&isp_dvp_d10d11 { ++ rockchip,pins = ++ /* cif_data10, cif_data11 */ ++ <1 RK_PB6 1 &pcfg_pull_down>, ++ <1 RK_PB7 1 &pcfg_pull_down>; ++}; ++ ++&isp_mmu { ++ status = "okay"; ++}; ++ ++&mipi_dphy_rx0 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ mipi_in_bcam: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&gc2145_mipi_out>; ++ data-lanes = <1>; ++ }; ++ ++ mipi_in: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ov8858_out>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_rx_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp_mipi_in>; ++ }; ++ }; ++ }; ++}; ++ ++&nandc0 { ++ status = "okay"; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ ++ pmu-supply = <&vcc3v3_pmu>; ++ vop-supply = <&vcc3v3_pmu>; ++}; ++ ++&pwm0 { ++ status = "okay"; ++}; ++ ++&pinctrl { ++ ++ headphone { ++ hp_det: hp-det { ++ rockchip,pins = <0 RK_PC7 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ pmic { ++ pmic_int: pmic_int { ++ rockchip,pins = ++ <0 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ soc_slppin_gpio: soc_slppin_gpio { ++ rockchip,pins = ++ <0 RK_PA0 RK_FUNC_GPIO &pcfg_output_low>; ++ }; ++ ++ soc_slppin_slp: soc_slppin_slp { ++ rockchip,pins = ++ <0 RK_PA0 1 &pcfg_pull_none>; ++ }; ++ ++ soc_slppin_rst: soc_slppin_rst { ++ rockchip,pins = ++ <0 RK_PA0 2 &pcfg_pull_none>; ++ }; ++ ++ vsel_gpio: vsel-gpio { ++ rockchip,pins = <0 RK_PA0 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ mpu6500 { ++ mpu6500_irq_gpio: mpu6500-irq-gpio { ++ rockchip,pins = <3 RK_PB6 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ cm3218 { ++ cm3218_irq_gpio: cm3218-irq-gpio { ++ rockchip,pins = <3 RK_PB7 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ dc_det { ++ dc_irq_gpio: dc-irq-gpio { ++ rockchip,pins = <0 RK_PC1 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ pcfg_pull_none_4ma: pcfg-pull-none-4ma { ++ bias-disable; ++ drive-strength = <4>; ++ }; ++ ++ pcfg_pull_none_smt: pcfg-pull-none-smt { ++ bias-disable; ++ input-schmitt-enable; ++ }; ++ ++ pcfg_output_high: pcfg-output-high { ++ output-high; ++ }; ++ ++ pcfg_output_low: pcfg-output-low { ++ output-low; ++ }; ++ ++ pcfg_input_high: pcfg-input-high { ++ bias-pull-up; ++ input-enable; ++ }; ++ ++ pcfg_input: pcfg-input { ++ input-enable; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ usb2 { ++ host_vbus_drv: host-vbus-drv { ++ rockchip,pins = <0 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart0_rts_gpio: uart0-rts-gpio { ++ rockchip,pins = <2 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++&rkisp1 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&isp_dvp_d2d9 &isp_dvp_d10d11 &cif_clkout>; ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dvp_in_fcam: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&gc0312_out>; ++ }; ++ ++ isp_mipi_in: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&dphy_rx_out>; ++ }; ++ }; ++}; ++ ++&route_dsi { ++ status = "okay"; ++}; ++ ++&rockchip_suspend { ++ status = "okay"; ++ Rockchip,sleep-debug-en = <1>; ++ rockchip,sleep-mode-config = < ++ (0 ++ | RKPM_SLP_ARMOFF ++ | RKPM_SLP_PMU_PLLS_PWRDN ++ | RKPM_SLP_PMU_PMUALIVE_32K ++ | RKPM_SLP_SFT_PLLS_DEEP ++ | RKPM_SLP_PMU_DIS_OSC ++ | RKPM_SLP_SFT_PD_NBSCUS ++ ) ++ >; ++ rockchip,wakeup-config = < ++ (0 ++ | RKPM_GPIO_WKUP_EN ++ | RKPM_USB_WKUP_EN ++ | RKPM_CLUSTER_L_WKUP_EN ++ ) ++ >; ++}; ++ ++&saradc { ++ status = "okay"; ++}; ++ ++&sdmmc { ++ clock-frequency = <37500000>; ++ clock-freq-min-max = <400000 37500000>; ++ no-sdio; ++ no-mmc; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ card-detect-delay = <200>; ++ disable-wp; ++ num-slots = <1>; ++ vmmc-supply = <&vcc_sd>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_cd &sdmmc_bus4>; ++ status = "disabled"; ++}; ++ ++&sdio0 { ++ max-frequency = <50000000>; ++ no-sd; ++ no-mmc; ++ bus-width = <4>; ++ disable-wp; ++ cap-sd-highspeed; ++ cap-sdio-irq; ++ keep-power-in-suspend; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ non-removable; ++ num-slots = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>; ++ rockchip,default-sample-phase = <90>; ++ status = "okay"; ++}; ++ ++&tsadc { ++ tsadc-supply = <&vdd_cpu>; ++ status = "okay"; ++}; ++ ++&uart0 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart0_xfer &uart0_cts>; ++ status = "okay"; ++}; ++ ++&u2phy { ++ status = "okay"; ++ ++ u2phy_otg: otg-port { ++ status = "okay"; ++ }; ++ ++ u2phy_host: host-port { ++ phy-supply = <&vcc_host>; ++ status = "okay"; ++ }; ++}; ++ ++&usb_host0_ehci { ++ status = "okay"; ++}; ++ ++&usb_host0_ohci { ++ status = "okay"; ++}; ++ +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-android.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-android.dtsi +new file mode 100755 +index 000000000000..ffaa92b7a89e +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-android.dtsi +@@ -0,0 +1,339 @@ ++/* ++ * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This file is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This file is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++#include ++#include ++#include "rk3399-vop-clk-set.dtsi" ++#include ++ ++/ { ++ compatible = "rockchip,android", "rockchip,rk3399"; ++ ++ chosen: chosen { ++ bootargs = "earlycon=uart8250,mmio32,0xff1a0000 coherent_pool=1m"; ++ }; ++ ++ cpuinfo { ++ compatible = "rockchip,cpuinfo"; ++ nvmem-cells = <&cpu_id>; ++ nvmem-cell-names = "id"; ++ }; ++ ++ fiq_debugger: fiq-debugger { ++ compatible = "rockchip,fiq-debugger"; ++ rockchip,serial-id = <2>; ++ rockchip,wake-irq = <0>; ++ rockchip,irq-mode-enable = <1>; /* If enable uart uses irq instead of fiq */ ++ rockchip,baudrate = <1500000>; /* Only 115200 and 1500000 */ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart2c_xfer>; ++ interrupts = ; ++ }; ++ ++ reserved-memory { ++ #address-cells = <2>; ++ #size-cells = <2>; ++ ranges; ++ ++ drm_logo: drm-logo@00000000 { ++ compatible = "rockchip,drm-logo"; ++ reg = <0x0 0x0 0x0 0x0>; ++ }; ++ ++ ramoops: ramoops@110000 { ++ compatible = "ramoops"; ++ reg = <0x0 0x110000 0x0 0xf0000>; ++ record-size = <0x20000>; ++ console-size = <0x80000>; ++ ftrace-size = <0x00000>; ++ pmsg-size = <0x50000>; ++ }; ++ ++ secure_memory: secure-memory@20000000 { ++ compatible = "rockchip,secure-memory"; ++ reg = <0x0 0x20000000 0x0 0x10000000>; ++ status = "disabled"; ++ }; ++ ++ stb_devinfo: stb-devinfo@00000000 { ++ compatible = "rockchip,stb-devinfo"; ++ reg = <0x0 0x0 0x0 0x0>; ++ }; ++ }; ++ ++ gpio_keys: gpio-keys { ++ compatible = "gpio-keys"; ++ autorepeat; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pwrbtn>; ++ ++ power { ++ debounce-interval = <100>; ++ gpios = <&gpio0 5 GPIO_ACTIVE_LOW>; ++ label = "GPIO Key Power"; ++ linux,code = ; ++ wakeup-source; ++ }; ++ }; ++ ++ rga: rga@ff680000 { ++ compatible = "rockchip,rga2"; ++ dev_mode = <1>; ++ reg = <0x0 0xff680000 0x0 0x1000>; ++ interrupts = ; ++ clocks = <&cru ACLK_RGA>, <&cru HCLK_RGA>, <&cru SCLK_RGA_CORE>; ++ clock-names = "aclk_rga", "hclk_rga", "clk_rga"; ++ power-domains = <&power RK3399_PD_RGA>; ++ status = "okay"; ++ }; ++ ++ hdmi_dp_sound: hdmi-dp-sound { ++ status = "disabled"; ++ compatible = "rockchip,rk3399-hdmi-dp"; ++ rockchip,cpu = <&i2s2>; ++ rockchip,codec = <&hdmi>, <&cdn_dp>; ++ }; ++ ++ firmware { ++ firmware_android: android {}; ++ }; ++}; ++ ++&uart2 { ++ status = "disabled"; ++}; ++ ++&vopb { ++ support-multi-area; ++ status = "okay"; ++}; ++ ++&vopb_mmu { ++ status = "okay"; ++}; ++ ++&vopl { ++ support-multi-area; ++ status = "okay"; ++}; ++ ++&vopl_mmu { ++ status = "okay"; ++}; ++ ++&hdmi { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ #sound-dai-cells = <0>; ++ ddc-i2c-scl-high-time-ns = <9625>; ++ ddc-i2c-scl-low-time-ns = <10000>; ++ status = "okay"; ++}; ++ ++&display_subsystem { ++ status = "okay"; ++ ++ ports = <&vopb_out>, <&vopl_out>; ++ logo-memory-region = <&drm_logo>; ++ secure-memory-region = <&secure_memory>; ++ route { ++ route_hdmi: route-hdmi { ++ status = "disabled"; ++ logo,uboot = "logo.bmp"; ++ logo,kernel = "logo_kernel.bmp"; ++ logo,mode = "center"; ++ charge_logo,mode = "center"; ++ connect = <&vopb_out_hdmi>; ++ }; ++ ++ route_dsi: route-dsi { ++ status = "disabled"; ++ logo,uboot = "logo.bmp"; ++ logo,kernel = "logo_kernel.bmp"; ++ logo,mode = "center"; ++ charge_logo,mode = "center"; ++ connect = <&vopb_out_dsi>; ++ }; ++ ++ route_dsi1: route-dsi1 { ++ status = "disabled"; ++ logo,uboot = "logo.bmp"; ++ logo,kernel = "logo_kernel.bmp"; ++ logo,mode = "center"; ++ charge_logo,mode = "center"; ++ connect = <&vopl_out_dsi1>; ++ }; ++ ++ route_edp: route-edp { ++ status = "disabled"; ++ logo,uboot = "logo.bmp"; ++ logo,kernel = "logo_kernel.bmp"; ++ logo,mode = "center"; ++ charge_logo,mode = "center"; ++ connect = <&vopb_out_edp>; ++ }; ++ }; ++}; ++ ++&dsi { ++ panel@0 { ++ reg = <0>; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ ++ panel_in_dsi: endpoint { ++ remote-endpoint = <&dsi_out_panel>; ++ }; ++ }; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@1 { ++ reg = <1>; ++ ++ dsi_out_panel: endpoint { ++ remote-endpoint = <&panel_in_dsi>; ++ }; ++ }; ++ }; ++}; ++ ++&i2s2 { ++ #sound-dai-cells = <0>; ++ rockchip,bclk-fs = <128>; ++}; ++ ++&rkvdec { ++ status = "okay"; ++}; ++ ++&rkvdec_mmu { ++ status = "okay"; ++}; ++ ++&usbdrd_dwc3_0 { ++ dr_mode = "otg"; ++}; ++ ++&iep { ++ status = "okay"; ++}; ++ ++&iep_mmu { ++ status = "okay"; ++}; ++ ++&mpp_srv { ++ status = "okay"; ++}; ++ ++&vdpu { ++ status = "okay"; ++}; ++ ++&vepu { ++ status = "okay"; ++}; ++ ++&vpu_mmu { ++ status = "okay"; ++}; ++ ++&pvtm { ++ status = "okay"; ++}; ++ ++&rng { ++ status = "okay"; ++}; ++ ++&pinctrl { ++ isp { ++ cif_clkout: cif-clkout { ++ rockchip,pins = ++ /*cif_clkout*/ ++ <2 RK_PB3 3 &pcfg_pull_none>; ++ }; ++ ++ isp_dvp_d0d7: isp-dvp-d0d7 { ++ rockchip,pins = ++ /*cif_data0*/ ++ <2 RK_PA0 3 &pcfg_pull_none>, ++ /*cif_data1*/ ++ <2 RK_PA1 3 &pcfg_pull_none>, ++ /*cif_data2*/ ++ <2 RK_PA2 3 &pcfg_pull_none>, ++ /*cif_data3*/ ++ <2 RK_PA3 3 &pcfg_pull_none>, ++ /*cif_data4*/ ++ <2 RK_PA4 3 &pcfg_pull_none>, ++ /*cif_data5*/ ++ <2 RK_PA5 3 &pcfg_pull_none>, ++ /*cif_data6*/ ++ <2 RK_PA6 3 &pcfg_pull_none>, ++ /*cif_data7*/ ++ <2 RK_PA7 3 &pcfg_pull_none>, ++ /*cif_sync*/ ++ <2 RK_PB0 3 &pcfg_pull_none>, ++ /*cif_href*/ ++ <2 RK_PB1 3 &pcfg_pull_none>, ++ /*cif_clkin*/ ++ <2 RK_PB2 3 &pcfg_pull_none>; ++ }; ++ }; ++ ++ buttons { ++ pwrbtn: pwrbtn { ++ rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++}; ++ +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-box-rev1.dts b/arch/arm64/boot/dts/rockchip/rk3399-box-rev1.dts +new file mode 100755 +index 000000000000..c4572da2ec87 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-box-rev1.dts +@@ -0,0 +1,134 @@ ++/* ++ * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This file is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This file is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++/dts-v1/; ++#include "rk3399-box.dtsi" ++ ++/ { ++ model = "Rockchip RK3399 Board rev1 (BOX)"; ++ compatible = "rockchip-box-rev1","rockchip,rk3399-box"; ++}; ++ ++&pinctrl { ++ sdio0 { ++ sdio0_bus1: sdio0-bus1 { ++ rockchip,pins = ++ <2 RK_PC4 1 &pcfg_pull_up_20ma>; ++ }; ++ ++ sdio0_bus4: sdio0-bus4 { ++ rockchip,pins = ++ <2 RK_PC4 1 &pcfg_pull_up_20ma>, ++ <2 RK_PC5 1 &pcfg_pull_up_20ma>, ++ <2 RK_PC6 1 &pcfg_pull_up_20ma>, ++ <2 RK_PC7 1 &pcfg_pull_up_20ma>; ++ }; ++ ++ sdio0_cmd: sdio0-cmd { ++ rockchip,pins = ++ <2 RK_PD0 1 &pcfg_pull_up_20ma>; ++ }; ++ ++ sdio0_clk: sdio0-clk { ++ rockchip,pins = ++ <2 RK_PD1 1 &pcfg_pull_none_20ma>; ++ }; ++ }; ++ ++ sdmmc { ++ sdmmc_bus1: sdmmc-bus1 { ++ rockchip,pins = ++ <4 RK_PB0 1 &pcfg_pull_up_8ma>; ++ }; ++ ++ sdmmc_bus4: sdmmc-bus4 { ++ rockchip,pins = ++ <4 RK_PB0 1 &pcfg_pull_up_8ma>, ++ <4 RK_PB1 1 &pcfg_pull_up_8ma>, ++ <4 RK_PB2 1 &pcfg_pull_up_8ma>, ++ <4 RK_PB3 1 &pcfg_pull_up_8ma>; ++ }; ++ ++ sdmmc_clk: sdmmc-clk { ++ rockchip,pins = ++ <4 RK_PB4 1 &pcfg_pull_none_18ma>; ++ }; ++ ++ sdmmc_cmd: sdmmc-cmd { ++ rockchip,pins = ++ <4 RK_PB5 1 &pcfg_pull_up_8ma>; ++ }; ++ }; ++ ++ fusb30x { ++ fusb0_int: fusb0-int { ++ rockchip,pins = ++ <1 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++}; ++ ++&i2c4 { ++ status = "okay"; ++ fusb0: fusb30x@22 { ++ compatible = "fairchild,fusb302"; ++ reg = <0x22>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&fusb0_int>; ++ vbus-5v-gpios = <&gpio1 3 GPIO_ACTIVE_HIGH>; ++ int-n-gpios = <&gpio1 2 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++}; ++ ++&cdn_dp { ++ status = "okay"; ++ extcon = <&fusb0>; ++}; ++ ++&hdmi_in_vopl { ++ status = "disabled"; ++}; ++ ++&dp_in_vopb { ++ status = "disabled"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-box-rev2.dts b/arch/arm64/boot/dts/rockchip/rk3399-box-rev2.dts +new file mode 100755 +index 000000000000..1c9b884736b1 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-box-rev2.dts +@@ -0,0 +1,158 @@ ++/* ++ * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This file is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This file is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++/dts-v1/; ++#include "rk3399-box.dtsi" ++ ++/ { ++ model = "Rockchip RK3399 Board rev2 (BOX)"; ++ compatible = "rockchip-box-rev2","rockchip,rk3399-box"; ++}; ++ ++&pinctrl { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cpt_gpio>; ++ ++ sdio0 { ++ sdio0_bus1: sdio0-bus1 { ++ rockchip,pins = ++ <2 RK_PC4 1 &pcfg_pull_up_20ma>; ++ }; ++ ++ sdio0_bus4: sdio0-bus4 { ++ rockchip,pins = ++ <2 RK_PC4 1 &pcfg_pull_up_20ma>, ++ <2 RK_PC5 1 &pcfg_pull_up_20ma>, ++ <2 RK_PC6 1 &pcfg_pull_up_20ma>, ++ <2 RK_PC7 1 &pcfg_pull_up_20ma>; ++ }; ++ ++ sdio0_cmd: sdio0-cmd { ++ rockchip,pins = ++ <2 RK_PD0 1 &pcfg_pull_up_20ma>; ++ }; ++ ++ sdio0_clk: sdio0-clk { ++ rockchip,pins = ++ <2 RK_PD1 1 &pcfg_pull_none_20ma>; ++ }; ++ }; ++ ++ sdmmc { ++ sdmmc_bus1: sdmmc-bus1 { ++ rockchip,pins = ++ <4 RK_PB0 1 &pcfg_pull_up_8ma>; ++ }; ++ ++ sdmmc_bus4: sdmmc-bus4 { ++ rockchip,pins = ++ <4 RK_PB0 1 &pcfg_pull_up_8ma>, ++ <4 RK_PB1 1 &pcfg_pull_up_8ma>, ++ <4 RK_PB2 1 &pcfg_pull_up_8ma>, ++ <4 RK_PB3 1 &pcfg_pull_up_8ma>; ++ }; ++ ++ sdmmc_clk: sdmmc-clk { ++ rockchip,pins = ++ <4 RK_PB4 1 &pcfg_pull_none_18ma>; ++ }; ++ ++ sdmmc_cmd: sdmmc-cmd { ++ rockchip,pins = ++ <4 RK_PB5 1 &pcfg_pull_up_8ma>; ++ }; ++ }; ++ ++ fusb30x { ++ fusb0_int: fusb0-int { ++ rockchip,pins = ++ <1 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ compat { ++ cpt_gpio: cpt-gpio { ++ rockchip,pins = ++ <1 RK_PC2 RK_FUNC_GPIO &pcfg_output_low>; ++ }; ++ }; ++}; ++ ++&i2c4 { ++ status = "okay"; ++ fusb0: fusb30x@22 { ++ compatible = "fairchild,fusb302"; ++ reg = <0x22>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&fusb0_int>; ++ vbus-5v-gpios = <&gpio1 3 GPIO_ACTIVE_HIGH>; ++ int-n-gpios = <&gpio1 2 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++}; ++ ++&cdn_dp { ++ status = "okay"; ++ extcon = <&fusb0>; ++}; ++ ++&hdmi_in_vopl { ++ status = "disabled"; ++}; ++ ++&dp_in_vopb { ++ status = "disabled"; ++}; ++ ++&route_hdmi { ++ status = "okay"; ++}; ++ ++&hdmi { ++ status = "okay"; ++ rockchip,phy-table = ++ <74250000 0x8009 0x0004 0x0272>, ++ <165000000 0x802b 0x0004 0x0209>, ++ <297000000 0x8039 0x0005 0x028d>, ++ <594000000 0x8039 0x0000 0x019d>, ++ <000000000 0x0000 0x0000 0x0000>; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-box.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-box.dtsi +new file mode 100755 +index 000000000000..704e0df4e122 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-box.dtsi +@@ -0,0 +1,891 @@ ++/* ++ * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This file is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This file is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++#include ++#include ++#include "rk3399.dtsi" ++#include "rk3399-android.dtsi" ++#include "rk3399-opp.dtsi" ++ ++/ { ++ compatible = "rockchip,rk3399-box","rockchip,rk3399"; ++ ++ vcc1v8_s0: vcc1v8-s0 { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc1v8_s0"; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-always-on; ++ }; ++ ++ vcc_sys: vcc-sys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_sys"; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ regulator-always-on; ++ }; ++ ++ vcc_phy: vcc-phy-regulator { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_phy"; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ vcc3v3_sys: vcc3v3-sys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc3v3_sys"; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-always-on; ++ vin-supply = <&vcc_sys>; ++ }; ++ ++ vcc5v0_host: vcc5v0-host-regulator { ++ compatible = "regulator-fixed"; ++ enable-active-high; ++ gpio = <&gpio4 25 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&host_vbus_drv>; ++ regulator-name = "vcc5v0_host"; ++ regulator-always-on; ++ }; ++ ++ vdd_log: vdd-log { ++ compatible = "pwm-regulator"; ++ pwms = <&pwm2 0 25000 1>; ++ regulator-name = "vdd_log"; ++ regulator-min-microvolt = <800000>; ++ regulator-max-microvolt = <1400000>; ++ regulator-always-on; ++ regulator-boot-on; ++ ++ /* for rockchip boot on */ ++ rockchip,pwm_id= <2>; ++ rockchip,pwm_voltage = <900000>; ++ ++ vin-supply = <&vcc_sys>; ++ }; ++ ++ xin32k: xin32k { ++ compatible = "fixed-clock"; ++ clock-frequency = <32768>; ++ clock-output-names = "xin32k"; ++ #clock-cells = <0>; ++ }; ++ ++ clkin_gmac: external-gmac-clock { ++ compatible = "fixed-clock"; ++ clock-frequency = <125000000>; ++ clock-output-names = "clkin_gmac"; ++ #clock-cells = <0>; ++ }; ++ ++ spdif-sound { ++ status = "okay"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,name = "ROCKCHIP,SPDIF"; ++ simple-audio-card,cpu { ++ sound-dai = <&spdif>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&spdif_out>; ++ }; ++ }; ++ ++ spdif_out: spdif-out { ++ status = "okay"; ++ compatible = "linux,spdif-dit"; ++ #sound-dai-cells = <0>; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ clocks = <&rk808 1>; ++ clock-names = "ext_clock"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio0 10 GPIO_ACTIVE_LOW>; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ rockchip,grf = <&grf>; ++ wifi_chip_type = "ap6354"; ++ sdio_vref = <1800>; ++ WIFI,host_wake_irq = <&gpio0 3 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&rk808 1>; ++ clock-names = "ext_clock"; ++ /* wifi-bt-power-toggle; */ ++ uart_rts_gpios = <&gpio2 19 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart0_rts>; ++ pinctrl-1 = <&uart0_gpios>; ++ /* BT,power_gpio = <&gpio3 19 GPIO_ACTIVE_HIGH>; */ ++ BT,reset_gpio = <&gpio0 9 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio2 26 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio0 4 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ test-power { ++ status = "okay"; ++ }; ++}; ++ ++&hdmi_dp_sound { ++ status = "okay"; ++}; ++ ++&hdmi { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hdmi_i2c_xfer>, <&hdmi_cec>; ++}; ++ ++&sdmmc { ++ clock-frequency = <100000000>; ++ clock-freq-min-max = <100000 100000000>; ++ no-sdio; ++ no-mmc; ++ bus-width = <4>; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ disable-wp; ++ num-slots = <1>; ++ //sd-uhs-sdr104; ++ vqmmc-supply = <&vcc_sd>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_cd &sdmmc_bus4>; ++ card-detect-delay = <800>; ++ status = "okay"; ++}; ++ ++&sdio0 { ++ clock-frequency = <100000000>; ++ clock-freq-min-max = <200000 100000000>; ++ no-sd; ++ no-mmc; ++ bus-width = <4>; ++ disable-wp; ++ cap-sd-highspeed; ++ cap-sdio-irq; ++ keep-power-in-suspend; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ non-removable; ++ num-slots = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&emmc_phy { ++ status = "okay"; ++}; ++ ++&sdhci { ++ bus-width = <8>; ++ mmc-hs400-1_8v; ++ no-sdio; ++ no-sd; ++ non-removable; ++ mmc-hs400-enhanced-strobe; ++ status = "okay"; ++}; ++ ++&i2s0 { ++ status = "okay"; ++ rockchip,i2s-broken-burst-len; ++ rockchip,playback-channels = <8>; ++ rockchip,capture-channels = <8>; ++ #sound-dai-cells = <0>; ++}; ++ ++&i2s2 { ++ #sound-dai-cells = <0>; ++}; ++ ++&spdif { ++ pinctrl-0 = <&spdif_bus_1>; ++ status = "okay"; ++ #sound-dai-cells = <0>; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <168>; ++ i2c-scl-falling-time-ns = <4>; ++ clock-frequency = <400000>; ++ ++ vdd_cpu_b: syr827@40 { ++ compatible = "silergy,syr827"; ++ reg = <0x40>; ++ regulator-compatible = "fan53555-reg"; ++ pinctrl-0 = <&vsel1_gpio>; ++ vsel-gpios = <&gpio1 17 GPIO_ACTIVE_HIGH>; ++ regulator-name = "vdd_cpu_b"; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1500000>; ++ regulator-ramp-delay = <1000>; ++ fcs,suspend-voltage-selector = <1>; ++ regulator-always-on; ++ regulator-boot-on; ++ vin-supply = <&vcc_sys>; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdd_gpu: syr828@41 { ++ compatible = "silergy,syr828"; ++ reg = <0x41>; ++ regulator-compatible = "fan53555-reg"; ++ pinctrl-0 = <&vsel2_gpio>; ++ vsel-gpios = <&gpio1 14 GPIO_ACTIVE_HIGH>; ++ regulator-name = "vdd_gpu"; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1500000>; ++ regulator-ramp-delay = <1000>; ++ fcs,suspend-voltage-selector = <1>; ++ regulator-always-on; ++ regulator-boot-on; ++ vin-supply = <&vcc_sys>; ++ regulator-initial-mode = <1>; /* 1:force PWM 2:auto */ ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ rk808: pmic@1b { ++ compatible = "rockchip,rk808"; ++ reg = <0x1b>; ++ interrupt-parent = <&gpio1>; ++ interrupts = <21 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pmic_int_l>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ ++ vcc1-supply = <&vcc_sys>; ++ vcc2-supply = <&vcc_sys>; ++ vcc3-supply = <&vcc_sys>; ++ vcc4-supply = <&vcc_sys>; ++ vcc6-supply = <&vcc_sys>; ++ vcc7-supply = <&vcc_sys>; ++ vcc8-supply = <&vcc3v3_sys>; ++ vcc9-supply = <&vcc_sys>; ++ vcc10-supply = <&vcc_sys>; ++ vcc11-supply = <&vcc_sys>; ++ vcc12-supply = <&vcc3v3_sys>; ++ vddio-supply = <&vcc_1v8>; ++ ++ regulators { ++ vdd_center: DCDC_REG1 { ++ regulator-name = "vdd_center"; ++ regulator-min-microvolt = <750000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdd_cpu_l: DCDC_REG2 { ++ regulator-name = "vdd_cpu_l"; ++ regulator-min-microvolt = <750000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-name = "vcc_ddr"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_1v8: DCDC_REG4 { ++ regulator-name = "vcc_1v8"; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vcc1v8_dvp: LDO_REG1 { ++ regulator-name = "vcc1v8_dvp"; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vcca1v8_hdmi: LDO_REG2 { ++ regulator-name = "vcca1v8_hdmi"; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vcca_1v8: LDO_REG3 { ++ regulator-name = "vcca_1v8"; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vcc_sd: LDO_REG4 { ++ regulator-name = "vcc_sd"; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ ++ vcc3v0_sd: LDO_REG5 { ++ regulator-name = "vcc3v0_sd"; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ ++ vcc_1v5: LDO_REG6 { ++ regulator-name = "vcc_1v5"; ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1500000>; ++ }; ++ }; ++ ++ vcca0v9_hdmi: LDO_REG7 { ++ regulator-name = "vcca0v9_hdmi"; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <900000>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <900000>; ++ }; ++ }; ++ ++ vcc_3v0: LDO_REG8 { ++ regulator-name = "vcc_3v0"; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ ++ vcc3v3_s3: SWITCH_REG1 { ++ regulator-name = "vcc3v3_s3"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc3v3_s0: SWITCH_REG2 { ++ regulator-name = "vcc3v3_s0"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ }; ++ }; ++}; ++ ++&cpu_l0 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_l1 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_l2 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_l3 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_b0 { ++ cpu-supply = <&vdd_cpu_b>; ++}; ++ ++&cpu_b1 { ++ cpu-supply = <&vdd_cpu_b>; ++}; ++ ++&firmware_android { ++ compatible = "android,firmware"; ++ fstab { ++ compatible = "android,fstab"; ++ system { ++ compatible = "android,system"; ++ dev = "/dev/block/by-name/system"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,verify"; ++ }; ++ vendor { ++ compatible = "android,vendor"; ++ dev = "/dev/block/by-name/vendor"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,verify"; ++ }; ++ }; ++}; ++ ++&gpu { ++ status = "okay"; ++ mali-supply = <&vdd_gpu>; ++}; ++ ++&threshold { ++ temperature = <85000>; ++}; ++ ++&target { ++ temperature = <100000>; ++}; ++ ++&soc_crit { ++ temperature = <115000>; ++}; ++ ++&tcphy0 { ++ extcon = <&fusb0>; ++ status = "okay"; ++}; ++ ++&tcphy1 { ++ status = "okay"; ++}; ++ ++&tsadc { ++ /* tshut mode 0:CRU 1:GPIO */ ++ rockchip,hw-tshut-mode = <1>; ++ /* tshut polarity 0:LOW 1:HIGH */ ++ rockchip,hw-tshut-polarity = <1>; ++ rockchip,hw-tshut-temp = <120000>; ++ status = "okay"; ++}; ++ ++&u2phy0 { ++ status = "okay"; ++ extcon = <&fusb0>; ++ ++ u2phy0_otg: otg-port { ++ status = "okay"; ++ }; ++ ++ u2phy0_host: host-port { ++ phy-supply = <&vcc5v0_host>; ++ status = "okay"; ++ }; ++}; ++ ++&u2phy1 { ++ status = "okay"; ++ ++ u2phy1_otg: otg-port { ++ status = "okay"; ++ }; ++ ++ u2phy1_host: host-port { ++ phy-supply = <&vcc5v0_host>; ++ status = "okay"; ++ }; ++}; ++ ++&uart0 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart0_xfer &uart0_cts>; ++ status = "okay"; ++}; ++ ++&uart2 { ++ status = "okay"; ++}; ++ ++&usb_host0_ehci { ++ status = "okay"; ++}; ++ ++&usb_host0_ohci { ++ status = "okay"; ++}; ++ ++&usb_host1_ehci { ++ status = "okay"; ++}; ++ ++&usb_host1_ohci { ++ status = "okay"; ++}; ++ ++&usbdrd3_0 { ++ status = "okay"; ++}; ++ ++&usbdrd_dwc3_0 { ++ dr_mode = "otg"; ++ status = "okay"; ++ extcon = <&fusb0>; ++}; ++ ++&usbdrd3_1 { ++ status = "okay"; ++}; ++ ++&usbdrd_dwc3_1 { ++ dr_mode = "host"; ++ status = "okay"; ++}; ++ ++&pwm2 { ++ status = "okay"; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm2_pin_pull_down>; ++}; ++ ++&pwm3 { ++ status = "okay"; ++ ++ interrupts = ; ++ compatible = "rockchip,remotectl-pwm"; ++ remote_pwm_id = <3>; ++ handle_cpu_id = <1>; ++ remote_support_psci = <1>; ++ ++ ir_key1 { ++ rockchip,usercode = <0x4040>; ++ rockchip,key_table = ++ <0xf2 KEY_REPLY>, ++ <0xba KEY_BACK>, ++ <0xf4 KEY_UP>, ++ <0xf1 KEY_DOWN>, ++ <0xef KEY_LEFT>, ++ <0xee KEY_RIGHT>, ++ <0xbd KEY_HOME>, ++ <0xea KEY_VOLUMEUP>, ++ <0xe3 KEY_VOLUMEDOWN>, ++ <0xe2 KEY_SEARCH>, ++ <0xb2 KEY_POWER>, ++ <0xbc KEY_MUTE>, ++ <0xec KEY_MENU>, ++ <0xbf 0x190>, ++ <0xe0 0x191>, ++ <0xe1 0x192>, ++ <0xe9 183>, ++ <0xe6 248>, ++ <0xe8 185>, ++ <0xe7 186>, ++ <0xf0 388>, ++ <0xbe 0x175>; ++ }; ++ ++ ir_key2 { ++ rockchip,usercode = <0xff00>; ++ rockchip,key_table = ++ <0xf9 KEY_HOME>, ++ <0xbf KEY_BACK>, ++ <0xfb KEY_MENU>, ++ <0xaa KEY_REPLY>, ++ <0xb9 KEY_UP>, ++ <0xe9 KEY_DOWN>, ++ <0xb8 KEY_LEFT>, ++ <0xea KEY_RIGHT>, ++ <0xeb KEY_VOLUMEDOWN>, ++ <0xef KEY_VOLUMEUP>, ++ <0xf7 KEY_MUTE>, ++ <0xe7 KEY_POWER>, ++ <0xfc KEY_POWER>, ++ <0xa9 KEY_VOLUMEDOWN>, ++ <0xa8 KEY_VOLUMEDOWN>, ++ <0xe0 KEY_VOLUMEDOWN>, ++ <0xa5 KEY_VOLUMEDOWN>, ++ <0xab 183>, ++ <0xb7 388>, ++ <0xe8 388>, ++ <0xf8 184>, ++ <0xaf 185>, ++ <0xed KEY_VOLUMEDOWN>, ++ <0xee 186>, ++ <0xb3 KEY_VOLUMEDOWN>, ++ <0xf1 KEY_VOLUMEDOWN>, ++ <0xf2 KEY_VOLUMEDOWN>, ++ <0xf3 KEY_SEARCH>, ++ <0xb4 KEY_VOLUMEDOWN>, ++ <0xbe KEY_SEARCH>; ++ }; ++ ++ ir_key3 { ++ rockchip,usercode = <0x1dcc>; ++ rockchip,key_table = ++ <0xee KEY_REPLY>, ++ <0xf0 KEY_BACK>, ++ <0xf8 KEY_UP>, ++ <0xbb KEY_DOWN>, ++ <0xef KEY_LEFT>, ++ <0xed KEY_RIGHT>, ++ <0xfc KEY_HOME>, ++ <0xf1 KEY_VOLUMEUP>, ++ <0xfd KEY_VOLUMEDOWN>, ++ <0xb7 KEY_SEARCH>, ++ <0xff KEY_POWER>, ++ <0xf3 KEY_MUTE>, ++ <0xbf KEY_MENU>, ++ <0xf9 0x191>, ++ <0xf5 0x192>, ++ <0xb3 388>, ++ <0xbe KEY_1>, ++ <0xba KEY_2>, ++ <0xb2 KEY_3>, ++ <0xbd KEY_4>, ++ <0xf9 KEY_5>, ++ <0xb1 KEY_6>, ++ <0xfc KEY_7>, ++ <0xf8 KEY_8>, ++ <0xb0 KEY_9>, ++ <0xb6 KEY_0>, ++ <0xb5 KEY_BACKSPACE>; ++ }; ++}; ++ ++&gmac { ++ phy-supply = <&vcc_phy>; ++ phy-mode = "rgmii"; ++ clock_in_out = "input"; ++ snps,reset-gpio = <&gpio3 15 GPIO_ACTIVE_LOW>; ++ snps,reset-active-low; ++ snps,reset-delays-us = <0 10000 50000>; ++ assigned-clocks = <&cru SCLK_RMII_SRC>; ++ assigned-clock-parents = <&clkin_gmac>; ++ pinctrl-names = "default", "sleep"; ++ pinctrl-0 = <&rgmii_pins>; ++ pinctrl-1 = <&rgmii_sleep_pins>; ++ tx_delay = <0x28>; ++ rx_delay = <0x11>; ++ status = "okay"; ++}; ++ ++&saradc { ++ status = "okay"; ++}; ++ ++&i2s2 { ++ status = "okay"; ++}; ++ ++&io_domains { ++ status = "okay"; ++ ++ bt656-supply = <&vcc1v8_s0>; /* bt656_gpio2ab_ms */ ++ audio-supply = <&vcc1v8_s0>; /* audio_gpio3d4a_ms */ ++ sdmmc-supply = <&vcc_sd>; /* sdmmc_gpio4b_ms */ ++ gpio1830-supply = <&vcc_3v0>; /* gpio1833_gpio4cd_ms */ ++}; ++ ++&pinctrl { ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = ++ <0 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart0_gpios: uart0-gpios { ++ rockchip,pins = ++ <2 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ usb2 { ++ host_vbus_drv: host-vbus-drv { ++ rockchip,pins = ++ <4 RK_PD1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ pmic { ++ pmic_int_l: pmic-int-l { ++ rockchip,pins = ++ <1 RK_PC5 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ vsel1_gpio: vsel1-gpio { ++ rockchip,pins = ++ <1 RK_PC1 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ ++ vsel2_gpio: vsel2-gpio { ++ rockchip,pins = ++ <1 RK_PB6 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ gmac { ++ rgmii_sleep_pins: rgmii-sleep-pins { ++ rockchip,pins = ++ <3 RK_PB7 RK_FUNC_GPIO &pcfg_output_low>; ++ }; ++ }; ++}; ++ ++&pvtm { ++ status = "okay"; ++}; ++ ++&pmu_pvtm { ++ status = "okay"; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ pmu1830-supply = <&vcc_1v8>; ++}; ++ ++&rockchip_suspend { ++ status = "okay"; ++ rockchip,sleep-debug-en = <0>; ++ rockchip,sleep-mode-config = < ++ (0 ++ | RKPM_SLP_ARMPD ++ | RKPM_SLP_PERILPPD ++ | RKPM_SLP_DDR_RET ++ | RKPM_SLP_PLLPD ++ | RKPM_SLP_CENTER_PD ++ | RKPM_SLP_AP_PWROFF ++ ) ++ >; ++ rockchip,wakeup-config = < ++ (0 ++ | RKPM_GPIO_WKUP_EN ++ | RKPM_PWM_WKUP_EN ++ ) ++ >; ++ rockchip,pwm-regulator-config = < ++ (0 ++ | PWM2_REGULATOR_EN ++ ) ++ >; ++ rockchip,power-ctrl = ++ <&gpio1 17 GPIO_ACTIVE_HIGH>, ++ <&gpio1 14 GPIO_ACTIVE_HIGH>; ++}; ++ ++&vopb { ++ assigned-clocks = <&cru DCLK_VOP0_DIV>; ++ assigned-clock-parents = <&cru PLL_VPLL>; ++}; ++ ++&vopl { ++ assigned-clocks = <&cru DCLK_VOP1_DIV>; ++ assigned-clock-parents = <&cru PLL_CPLL>; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-dram-default-timing.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-dram-default-timing.dtsi +new file mode 100755 +index 000000000000..981777b82329 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-dram-default-timing.dtsi +@@ -0,0 +1,80 @@ ++/* ++ * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This library is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This library is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++#include ++ ++/ { ++ ddr_timing: ddr_timing { ++ compatible = "rockchip,ddr-timing"; ++ ddr3_speed_bin = <21>; ++ pd_idle = <0>; ++ sr_idle = <0>; ++ sr_mc_gate_idle = <0>; ++ srpd_lite_idle = <0>; ++ standby_idle = <0>; ++ auto_lp_dis_freq = <666>; ++ ddr3_dll_dis_freq = <300>; ++ phy_dll_dis_freq = <260>; ++ ++ ddr3_odt_dis_freq = <666>; ++ ddr3_drv = ; ++ ddr3_odt = ; ++ phy_ddr3_ca_drv = ; ++ phy_ddr3_dq_drv = ; ++ phy_ddr3_odt = ; ++ ++ lpddr3_odt_dis_freq = <666>; ++ lpddr3_drv = ; ++ lpddr3_odt = ; ++ phy_lpddr3_ca_drv = ; ++ phy_lpddr3_dq_drv = ; ++ phy_lpddr3_odt = ; ++ ++ lpddr4_odt_dis_freq = <800>; ++ lpddr4_drv = ; ++ lpddr4_dq_odt = ; ++ lpddr4_ca_odt = ; ++ phy_lpddr4_ca_drv = ; ++ phy_lpddr4_ck_cs_drv = ; ++ phy_lpddr4_dq_drv = ; ++ phy_lpddr4_odt = ; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-early-opp.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-early-opp.dtsi +new file mode 100755 +index 000000000000..2a675ec7c342 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-early-opp.dtsi +@@ -0,0 +1,127 @@ ++/* ++ * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This library is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This library is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++/* ++ * NOTE: this file exists for the sake of early (pre-ES2) silicon. ES2 silicon ++ * will have different power characteristics. ++ */ ++ ++/ { ++ /delete-node/ opp-table0; ++ /delete-node/ opp-table1; ++ /delete-node/ opp-table2; ++ ++ cluster0_opp: opp-table0 { ++ compatible = "operating-points-v2"; ++ opp-shared; ++ ++ opp-408000000 { ++ opp-hz = /bits/ 64 <408000000>; ++ opp-microvolt = <900000 900000 1200000>; ++ clock-latency-ns = <40000>; ++ }; ++ opp-600000000 { ++ opp-hz = /bits/ 64 <600000000>; ++ opp-microvolt = <900000 900000 1200000>; ++ }; ++ opp-816000000 { ++ opp-hz = /bits/ 64 <816000000>; ++ opp-microvolt = <900000 900000 1200000>; ++ }; ++ opp-1008000000 { ++ opp-hz = /bits/ 64 <1008000000>; ++ opp-microvolt = <900000 900000 1200000>; ++ }; ++ opp-1200000000 { ++ opp-hz = /bits/ 64 <1200000000>; ++ opp-microvolt = <950000 950000 1200000>; ++ }; ++ opp-1416000000 { ++ opp-hz = /bits/ 64 <1416000000>; ++ opp-microvolt = <1050000 1050000 1200000>; ++ }; ++ }; ++ ++ cluster1_opp: opp-table1 { ++ compatible = "operating-points-v2"; ++ opp-shared; ++ ++ opp-408000000 { ++ opp-hz = /bits/ 64 <408000000>; ++ opp-microvolt = <900000 900000 1200000>; ++ clock-latency-ns = <40000>; ++ }; ++ opp-600000000 { ++ opp-hz = /bits/ 64 <600000000>; ++ opp-microvolt = <900000 900000 1200000>; ++ }; ++ opp-816000000 { ++ opp-hz = /bits/ 64 <816000000>; ++ opp-microvolt = <900000 900000 1200000>; ++ }; ++ opp-1008000000 { ++ opp-hz = /bits/ 64 <1008000000>; ++ opp-microvolt = <950000 950000 1200000>; ++ }; ++ opp-1200000000 { ++ opp-hz = /bits/ 64 <1200000000>; ++ opp-microvolt = <1000000 1000000 1200000>; ++ }; ++ }; ++ ++ gpu_opp_table: opp-table2 { ++ compatible = "operating-points-v2"; ++ ++ opp-200000000 { ++ opp-hz = /bits/ 64 <200000000>; ++ opp-microvolt = <900000>; ++ }; ++ opp-297000000 { ++ opp-hz = /bits/ 64 <297000000>; ++ opp-microvolt = <900000>; ++ }; ++ opp-400000000 { ++ opp-hz = /bits/ 64 <400000000>; ++ opp-microvolt = <900000>; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-evb-cros.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-evb-cros.dtsi +new file mode 100755 +index 000000000000..4cac4981b267 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-evb-cros.dtsi +@@ -0,0 +1,181 @@ ++/* ++ * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This file is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This file is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++/ { ++ model = "Rockchip RK3399 Evaluation Board (Chrome OS)"; ++ compatible = "google,rk3399evb", "rockchip,rk3399-evb", "rockchip,rk3399"; ++ ++ edp_panel: edp-panel { ++ compatible = "lg,lp097qx1-spa1", "panel-simple"; ++ backlight = <&backlight>; ++ power-supply = <&vcc3v3_s0>; ++ enable-gpios = <&gpio1 13 GPIO_ACTIVE_HIGH>; ++ ++ ports { ++ panel_in_edp: endpoint { ++ remote-endpoint = <&edp_out_panel>; ++ }; ++ }; ++ }; ++ ++ hdmi_codec: hdmi-codec { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,name = "HDMI-CODEC"; ++ ++ simple-audio-card,cpu { ++ sound-dai = <&i2s2>; ++ }; ++ ++ simple-audio-card,codec { ++ sound-dai = <&hdmi>; ++ }; ++ }; ++ ++ sound { ++ compatible = "rockchip,cdndp-sound"; ++ rockchip,cpu = <&i2s2>; ++ rockchip,codec = <&cdn_dp>; ++ }; ++}; ++ ++&cdn_dp { ++ status = "okay"; ++ extcon = <&fusb0>, <&fusb1>; ++ ++ ports { ++ /* Don't use vopl for dp, save it for edp */ ++ dp_in: port { ++ /delete-node/ endpoint@1; ++ }; ++ }; ++}; ++ ++&dsi { ++ status = "okay"; ++ ++ panel@0 { ++ compatible ="boe,tv080wum-nl0"; ++ reg = <0>; ++ backlight = <&backlight>; ++ enable-gpios = <&gpio1 13 GPIO_ACTIVE_HIGH>; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ panel_in_dsi: endpoint { ++ remote-endpoint = <&dsi_out_panel>; ++ }; ++ }; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@1 { ++ reg = <1>; ++ dsi_out_panel: endpoint { ++ remote-endpoint = <&panel_in_dsi>; ++ }; ++ }; ++ }; ++}; ++ ++&edp { ++ status = "disabled"; ++ ++ ports { ++ edp_out: port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ edp_out_panel: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&panel_in_edp>; ++ }; ++ }; ++ }; ++}; ++ ++&hdmi { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ #sound-dai-cells = <0>; ++ status = "disabled"; ++}; ++ ++&i2s2 { ++ #sound-dai-cells = <0>; ++ status = "okay"; ++}; ++ ++&vopb { ++ status = "okay"; ++}; ++ ++&vopb_mmu { ++ status = "okay"; ++}; ++ ++&vopl { ++ status = "okay"; ++ ++ /* Don't use vopl for dp, save it for edp */ ++ vopl_out: port { ++ /delete-node/ endpoint@3; ++ }; ++}; ++ ++&vopl_mmu { ++ status = "okay"; ++}; ++ ++&display_subsystem { ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-evb-ind-lpddr4-android-avb.dts b/arch/arm64/boot/dts/rockchip/rk3399-evb-ind-lpddr4-android-avb.dts +new file mode 100755 +index 000000000000..343a9fc389e1 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-evb-ind-lpddr4-android-avb.dts +@@ -0,0 +1,389 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++ ++#include "rk3399-evb-ind.dtsi" ++#include "rk3399-android.dtsi" ++ ++/ { ++ model = "Rockchip RK3399 EVB IND LPDDR4 Board edp (Android)"; ++ compatible = "rockchip,android", "rockchip,rk3399-evb-ind-lpddr4-android", "rockchip,rk3399"; ++ chosen: chosen { ++ bootargs = "earlycon=uart8250,mmio32,0xff1a0000 console=ttyFIQ0 androidboot.baseband=N/A androidboot.veritymode=enforcing androidboot.hardware=rk30board androidboot.console=ttyFIQ0 init=/init initrd=0x62000001,0x00800000 coherent_pool=1m"; ++ }; ++ ++ iram: sram@ff8d0000 { ++ compatible = "mmio-sram"; ++ reg = <0x0 0xff8d0000 0x0 0x20000>; ++ }; ++ ++ vcc_lcd: vcc-lcd { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_lcd"; ++ startup-delay-us = <20000>; ++ enable-active-high; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-boot-on; ++ vin-supply = <&vcc5v0_sys>; ++ }; ++ ++ panel: panel { ++ compatible = "simple-panel"; ++ backlight = <&backlight>; ++ power-supply = <&vcc_lcd>; ++ reset-gpios = <&gpio1 23 GPIO_ACTIVE_LOW>; ++ prepare-delay-ms = <20>; ++ reset-delay-ms = <20>; ++ enable-delay-ms = <20>; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <200000000>; ++ hactive = <1536>; ++ vactive = <2048>; ++ hfront-porch = <12>; ++ hsync-len = <16>; ++ hback-porch = <48>; ++ vfront-porch = <8>; ++ vsync-len = <4>; ++ vback-porch = <8>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ ports { ++ panel_in: endpoint { ++ remote-endpoint = <&edp_out>; ++ }; ++ }; ++ }; ++ ++ test-power { ++ status = "okay"; ++ }; ++}; ++ ++&backlight { ++ enable-gpios = <&gpio1 13 GPIO_ACTIVE_HIGH>; ++}; ++ ++&dmac_bus { ++ iram = <&iram>; ++ rockchip,force-iram; ++}; ++ ++&dp_sound { ++ status = "disabled"; ++}; ++ ++&edp { ++ status = "okay"; ++ force-hpd; ++ ++ ports { ++ port@1 { ++ reg = <1>; ++ ++ edp_out: endpoint { ++ remote-endpoint = <&panel_in>; ++ }; ++ }; ++ }; ++}; ++ ++&edp_in_vopl { ++ status = "disabled"; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ ++ sgm3784: sgm3784@30 { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ compatible = "sgmicro,gsm3784"; ++ reg = <0x30>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ //enable-gpio = <&gpio2 RK_PB4 GPIO_ACTIVE_HIGH>; ++ //strobe-gpio = <&gpio1 RK_PA3 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ sgm3784_led0: led@0 { ++ reg = <0x0>; ++ led-max-microamp = <299200>; ++ flash-max-microamp = <1122000>; ++ flash-max-timeout-us = <1600000>; ++ }; ++ ++ sgm3784_led1: led@1 { ++ reg = <0x1>; ++ led-max-microamp = <299200>; ++ flash-max-microamp = <1122000>; ++ flash-max-timeout-us = <1600000>; ++ }; ++ }; ++ ++ vm149c: vm149c@0c { ++ compatible = "silicon touch,vm149c"; ++ status = "okay"; ++ reg = <0x0c>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ }; ++ ++ gc2145: gc2145@3c{ ++ status = "okay"; ++ compatible = "galaxycore,gc2145"; ++ reg = <0x3c>; ++ pinctrl-names = "rockchip,camera_default"; ++ pinctrl-0 = <&cif_clkout>; ++ ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ ++ /* avdd-supply = <>; */ ++ /* dvdd-supply = <>; */ ++ /* dovdd-supply = <>; */ ++ power-gpios = <&gpio4 26 GPIO_ACTIVE_HIGH>; ++ pwdn-gpios = <&gpio2 12 GPIO_ACTIVE_HIGH>; ++ rockchip,camera-module-index = <1>; ++ rockchip,camera-module-facing = "front"; ++ rockchip,camera-module-name = "CameraKing"; ++ rockchip,camera-module-lens-name = "Largan"; ++ port { ++ gc2145_out: endpoint { ++ remote-endpoint = <&dvp_in_fcam>; ++ }; ++ }; ++ }; ++ ++ ov13850: ov13850@10 { ++ compatible = "ovti,ov13850"; ++ status = "okay"; ++ reg = <0x10>; ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ /* avdd-supply = <>; */ ++ /* dvdd-supply = <>; */ ++ /* dovdd-supply = <>; */ ++ /* reset-gpios = <>; */ ++ reset-gpios = <&gpio1 3 GPIO_ACTIVE_HIGH>; ++ pwdn-gpios = <&gpio1 24 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "rockchip,camera_default"; ++ pinctrl-0 = <&cif_clkout>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "CMK-CT0116"; ++ rockchip,camera-module-lens-name = "Largan-50013A1"; ++ lens-focus = <&vm149c>; ++ flash-leds = <&sgm3784_led0 &sgm3784_led1>; ++ port { ++ ucam_out0: endpoint { ++ remote-endpoint = <&mipi_in_ucam0>; ++ //remote-endpoint = <&mipi_in_ucam1>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++ ++ ov4689: ov4689@36 { ++ compatible = "ovti,ov4689"; ++ status = "disabled"; ++ reg = <0x36>; ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ /* avdd-supply = <>; */ ++ /* dvdd-supply = <>; */ ++ /* dovdd-supply = <>; */ ++ /* reset-gpios = <>; */ ++ reset-gpios = <&gpio2 10 GPIO_ACTIVE_HIGH>; ++ pwdn-gpios = <&gpio4 29 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "rockchip,camera_default"; ++ pinctrl-0 = <&cif_clkout>; ++ rockchip,camera-module-index = <1>; ++ rockchip,camera-module-facing = "front"; ++ rockchip,camera-module-name = "JSD3425-C1"; ++ rockchip,camera-module-lens-name = "JSD3425-C1"; ++ port { ++ ucam_out1: endpoint { ++ //remote-endpoint = <&mipi_in_ucam0>; ++ remote-endpoint = <&mipi_in_ucam1>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++}; ++ ++&i2s2 { ++ #sound-dai-cells = <0>; ++ status = "okay"; ++}; ++ ++&isp0_mmu { ++ status = "okay"; ++}; ++ ++&isp1_mmu { ++ status = "okay"; ++}; ++ ++&mipi_dphy_rx0 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam0: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ucam_out0>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_rx0_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp0_mipi_in>; ++ }; ++ }; ++ }; ++}; ++ ++&mipi_dphy_tx1rx1 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam1: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ucam_out1>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_tx1rx1_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp1_mipi_in>; ++ }; ++ }; ++ }; ++}; ++ ++&hdmi_sound { ++ status = "okay"; ++}; ++ ++&route_edp { ++ status = "okay"; ++}; ++ ++&route_hdmi { ++ status = "okay"; ++ connect = <&vopl_out_hdmi>; ++}; ++ ++&i2s1 { ++ #sound-dai-cells = <0>; ++ status = "okay"; ++}; ++ ++&rk809_sound { ++ status = "okay"; ++}; ++ ++&hdmi_in_vopb { ++ status = "disabled"; ++}; ++ ++&rkisp1_0 { ++ status = "okay"; ++ ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp0_mipi_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dphy_rx0_out>; ++ }; ++ }; ++}; ++ ++&rkisp1_1 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_clkout &isp_dvp_d0d7>; ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp1_mipi_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dphy_tx1rx1_out>; ++ }; ++ dvp_in_fcam: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&gc2145_out>; ++ }; ++ }; ++}; ++ ++/* ++ * if enable dp_sound, should disable spdif_sound and spdif_out ++ */ ++&spdif_out { ++ status = "disabled"; ++}; ++ ++&spdif_sound { ++ status = "disabled"; ++}; ++ ++&i2s0 { ++ #sound-dai-cells = <0>; ++ status = "disabled"; ++}; ++ ++&tc358749x_sound { ++ status = "disabled"; ++}; ++ ++&pinctrl { ++ lcd-panel { ++ lcd_panel_reset: lcd-panel-reset { ++ rockchip,pins = <1 RK_PC7 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++}; ++ +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-evb-ind-lpddr4-android.dts b/arch/arm64/boot/dts/rockchip/rk3399-evb-ind-lpddr4-android.dts +new file mode 100755 +index 000000000000..72aa97affa99 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-evb-ind-lpddr4-android.dts +@@ -0,0 +1,157 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++ ++#include "rk3399-evb-ind.dtsi" ++#include "rk3399-android.dtsi" ++ ++/ { ++ model = "Rockchip RK3399 EVB IND LPDDR4 Board edp (Android)"; ++ compatible = "rockchip,android", "rockchip,rk3399-evb-ind-lpddr4-android", "rockchip,rk3399"; ++ ++ iram: sram@ff8d0000 { ++ compatible = "mmio-sram"; ++ reg = <0x0 0xff8d0000 0x0 0x20000>; ++ }; ++ ++ vcc_lcd: vcc-lcd { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_lcd"; ++ startup-delay-us = <20000>; ++ enable-active-high; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-boot-on; ++ vin-supply = <&vcc5v0_sys>; ++ }; ++ ++ panel: panel { ++ compatible = "simple-panel"; ++ backlight = <&backlight>; ++ power-supply = <&vcc_lcd>; ++ enable-gpios = <&gpio1 13 GPIO_ACTIVE_HIGH>; ++ reset-gpios = <&gpio1 23 GPIO_ACTIVE_LOW>; ++ prepare-delay-ms = <20>; ++ enable-delay-ms = <20>; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <200000000>; ++ hactive = <1536>; ++ vactive = <2048>; ++ hfront-porch = <12>; ++ hsync-len = <16>; ++ hback-porch = <48>; ++ vfront-porch = <8>; ++ vsync-len = <4>; ++ vback-porch = <8>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ ports { ++ panel_in: endpoint { ++ remote-endpoint = <&edp_out>; ++ }; ++ }; ++ }; ++ ++ test-power { ++ status = "okay"; ++ }; ++}; ++ ++&backlight { ++ enable-gpios = <&gpio1 18 GPIO_ACTIVE_HIGH>; ++}; ++ ++&dmac_bus { ++ iram = <&iram>; ++ rockchip,force-iram; ++}; ++ ++&dp_sound { ++ status = "disabled"; ++}; ++ ++&edp { ++ status = "okay"; ++ force-hpd; ++ ++ ports { ++ port@1 { ++ reg = <1>; ++ ++ edp_out: endpoint { ++ remote-endpoint = <&panel_in>; ++ }; ++ }; ++ }; ++}; ++ ++&edp_in_vopl { ++ status = "disabled"; ++}; ++ ++&i2s2 { ++ #sound-dai-cells = <0>; ++ status = "okay"; ++}; ++ ++&hdmi_sound { ++ status = "okay"; ++}; ++ ++&route_edp { ++ status = "okay"; ++}; ++ ++&i2s1 { ++ #sound-dai-cells = <0>; ++ status = "okay"; ++}; ++ ++&rk809_sound { ++ status = "okay"; ++}; ++ ++&hdmi_in_vopb { ++ status = "disabled"; ++}; ++ ++/* ++ * if enable dp_sound, should disable spdif_sound and spdif_out ++ */ ++&spdif_out { ++ status = "disabled"; ++}; ++ ++&spdif_sound { ++ status = "disabled"; ++}; ++ ++&i2s0 { ++ #sound-dai-cells = <0>; ++ status = "disabled"; ++}; ++ ++&tc358749x_sound { ++ status = "disabled"; ++}; ++ ++&pinctrl { ++ lcd-panel { ++ lcd_panel_reset: lcd-panel-reset { ++ rockchip,pins = <1 RK_PC7 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++}; ++ +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-evb-ind-lpddr4-linux.dts b/arch/arm64/boot/dts/rockchip/rk3399-evb-ind-lpddr4-linux.dts +new file mode 100755 +index 000000000000..5b6b7131fdd1 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-evb-ind-lpddr4-linux.dts +@@ -0,0 +1,321 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2019-2020 Fuzhou Rockchip Electronics Co., Ltd. ++ */ ++ ++/dts-v1/; ++ ++#include "rk3399-evb-ind.dtsi" ++#include "rk3399-linux.dtsi" ++ ++/ { ++ model = "Rockchip RK3399 EVB IND LPDDR4 Board edp (Linux)"; ++ compatible = "rockchip,linux", "rockchip,rk3399-evb-ind-lpddr4-linux", "rockchip,rk3399"; ++ ++ iram: sram@ff8d0000 { ++ compatible = "mmio-sram"; ++ reg = <0x0 0xff8d0000 0x0 0x20000>; ++ }; ++ ++ vcc_lcd: vcc-lcd { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_lcd"; ++ startup-delay-us = <20000>; ++ enable-active-high; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-boot-on; ++ vin-supply = <&vcc5v0_sys>; ++ }; ++ ++ panel: panel { ++ compatible = "simple-panel"; ++ backlight = <&backlight>; ++ power-supply = <&vcc_lcd>; ++ reset-gpios = <&gpio1 23 GPIO_ACTIVE_LOW>; ++ prepare-delay-ms = <20>; ++ enable-delay-ms = <20>; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <200000000>; ++ hactive = <1536>; ++ vactive = <2048>; ++ hfront-porch = <12>; ++ hsync-len = <16>; ++ hback-porch = <48>; ++ vfront-porch = <8>; ++ vsync-len = <4>; ++ vback-porch = <8>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ ports { ++ panel_in: endpoint { ++ remote-endpoint = <&edp_out>; ++ }; ++ }; ++ }; ++ ++ test-power { ++ status = "okay"; ++ }; ++}; ++ ++&backlight { ++ enable-gpios = <&gpio1 13 GPIO_ACTIVE_HIGH>; ++}; ++ ++&display_subsystem { ++ status = "okay"; ++}; ++ ++&dmac_bus { ++ iram = <&iram>; ++ rockchip,force-iram; ++}; ++ ++&dp_sound { ++ status = "disabled"; ++}; ++ ++&edp { ++ status = "okay"; ++ force-hpd; ++ ++ ports { ++ port@1 { ++ reg = <1>; ++ ++ edp_out: endpoint { ++ remote-endpoint = <&panel_in>; ++ }; ++ }; ++ }; ++}; ++ ++&edp_in_vopb { ++ status = "disabled"; ++}; ++ ++&hdmi { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hdmi_i2c_xfer>, <&hdmi_cec>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ #sound-dai-cells = <0>; ++ status = "okay"; ++}; ++ ++&hdmi_in_vopl { ++ status = "disabled"; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ ++ vm149c: vm149c@0c { ++ compatible = "silicon touch,vm149c"; ++ status = "okay"; ++ reg = <0x0c>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ }; ++ ++ ov13850: ov13850@10 { ++ compatible = "ovti,ov13850"; ++ status = "okay"; ++ reg = <0x10>; ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ ++ /* conflict with csi-ctl-gpios */ ++ reset-gpios = <&gpio1 RK_PA3 GPIO_ACTIVE_HIGH>; ++ pwdn-gpios = <&gpio1 RK_PD0 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "rockchip,camera_default"; ++ pinctrl-0 = <&cif_clkout>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "CMK-CT0116"; ++ rockchip,camera-module-lens-name = "Largan-50013A1"; ++ ++ lens-focus = <&vm149c>; ++ ++ port { ++ ucam_out0: endpoint { ++ remote-endpoint = <&mipi_in_ucam0>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++}; ++ ++&i2s0 { ++ #sound-dai-cells = <0>; ++ status = "okay"; ++}; ++ ++&i2s1 { ++ #sound-dai-cells = <0>; ++ status = "disabled"; ++}; ++ ++&i2s2 { ++ #sound-dai-cells = <0>; ++ status = "okay"; ++}; ++ ++&hdmi_sound { ++ status = "okay"; ++}; ++ ++&mipi_dphy_rx0 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam0: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ucam_out0>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_rx0_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp0_mipi_in>; ++ }; ++ }; ++ }; ++}; ++ ++&mipi_dphy_tx1rx1 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam1: endpoint@1 { ++ reg = <1>; ++ /* Unlinked camera */ ++ //remote-endpoint = <&ucam_out1>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_tx1rx1_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp1_mipi_in>; ++ }; ++ }; ++ }; ++}; ++ ++&rk809_sound { ++ status = "okay"; ++}; ++ ++&rkisp1_0 { ++ status = "okay"; ++ ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp0_mipi_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dphy_rx0_out>; ++ }; ++ }; ++}; ++ ++&rkisp1_1 { ++ status = "okay"; ++ ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp1_mipi_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dphy_tx1rx1_out>; ++ }; ++ }; ++}; ++ ++&route_edp { ++ status = "okay"; ++}; ++ ++/* ++ * if enable dp_sound, should disable spdif_sound and spdif_out ++ */ ++&spdif_out { ++ status = "disabled"; ++}; ++ ++&spdif_sound { ++ status = "disabled"; ++}; ++ ++&tc358749x_sound { ++ status = "disabled"; ++}; ++ ++&vopb { ++ status = "okay"; ++}; ++ ++&vopb_mmu { ++ status = "okay"; ++}; ++ ++&vopl { ++ status = "okay"; ++}; ++ ++&vopl_mmu { ++ status = "okay"; ++}; ++ ++&pinctrl { ++ lcd-panel { ++ lcd_panel_reset: lcd-panel-reset { ++ rockchip,pins = <1 RK_PC7 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++}; ++ ++/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ ++/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ ++/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ ++ +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-evb-ind-lpddr4-v13-android-avb.dts b/arch/arm64/boot/dts/rockchip/rk3399-evb-ind-lpddr4-v13-android-avb.dts +new file mode 100755 +index 000000000000..8439d91dd125 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-evb-ind-lpddr4-v13-android-avb.dts +@@ -0,0 +1,425 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++ ++#include "rk3399-evb-ind.dtsi" ++#include "rk3399-android.dtsi" ++ ++/ { ++ model = "Rockchip RK3399 EVB IND LPDDR4 Board edp (Android)"; ++ compatible = "rockchip,android", "rockchip,rk3399-evb-ind-v13-lpddr4-android", "rockchip,rk3399"; ++ chosen: chosen { ++ bootargs = "earlycon=uart8250,mmio32,0xff1a0000 console=ttyFIQ0 androidboot.baseband=N/A androidboot.veritymode=enforcing androidboot.hardware=rk30board androidboot.console=ttyFIQ0 init=/init initrd=0x62000001,0x00800000 coherent_pool=1m"; ++ }; ++ ++ iram: sram@ff8d0000 { ++ compatible = "mmio-sram"; ++ reg = <0x0 0xff8d0000 0x0 0x20000>; ++ }; ++ ++ hub_reset: hub_reset { ++ compatible = "regulator-fixed"; ++ enable-active-high; ++ gpio = <&gpio4 18 GPIO_ACTIVE_HIGH>; ++ regulator-name = "hub_reset"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ }; ++ ++ vcc_lcd: vcc-lcd { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_lcd"; ++ startup-delay-us = <20000>; ++ enable-active-high; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-boot-on; ++ vin-supply = <&vcc5v0_sys>; ++ }; ++ ++ panel: panel { ++ compatible = "simple-panel"; ++ backlight = <&backlight>; ++ power-supply = <&vcc_lcd>; ++ reset-gpios = <&gpio1 23 GPIO_ACTIVE_LOW>; ++ prepare-delay-ms = <20>; ++ reset-delay-ms = <20>; ++ enable-delay-ms = <20>; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <200000000>; ++ hactive = <1536>; ++ vactive = <2048>; ++ hfront-porch = <12>; ++ hsync-len = <16>; ++ hback-porch = <48>; ++ vfront-porch = <8>; ++ vsync-len = <4>; ++ vback-porch = <8>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ ports { ++ panel_in: endpoint { ++ remote-endpoint = <&edp_out>; ++ }; ++ }; ++ }; ++ ++ test-power { ++ status = "okay"; ++ }; ++}; ++ ++&backlight { ++ enable-gpios = <&gpio1 13 GPIO_ACTIVE_HIGH>; ++}; ++ ++&dmac_bus { ++ iram = <&iram>; ++ rockchip,force-iram; ++}; ++ ++&dp_sound { ++ status = "disabled"; ++}; ++ ++&edp { ++ status = "okay"; ++ force-hpd; ++ ++ ports { ++ port@1 { ++ reg = <1>; ++ ++ edp_out: endpoint { ++ remote-endpoint = <&panel_in>; ++ }; ++ }; ++ }; ++}; ++ ++&edp_in_vopl { ++ status = "disabled"; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ ++ sgm3784: sgm3784@30 { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ compatible = "sgmicro,gsm3784"; ++ reg = <0x30>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ //enable-gpio = <&gpio2 RK_PB4 GPIO_ACTIVE_HIGH>; ++ //strobe-gpio = <&gpio1 RK_PA3 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ sgm3784_led0: led@0 { ++ reg = <0x0>; ++ led-max-microamp = <299200>; ++ flash-max-microamp = <1122000>; ++ flash-max-timeout-us = <1600000>; ++ }; ++ ++ sgm3784_led1: led@1 { ++ reg = <0x1>; ++ led-max-microamp = <299200>; ++ flash-max-microamp = <1122000>; ++ flash-max-timeout-us = <1600000>; ++ }; ++ }; ++ ++ vm149c: vm149c@0c { ++ compatible = "silicon touch,vm149c"; ++ status = "okay"; ++ reg = <0x0c>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ }; ++ ++ gc2145: gc2145@3c{ ++ status = "okay"; ++ compatible = "galaxycore,gc2145"; ++ reg = <0x3c>; ++ pinctrl-names = "rockchip,camera_default"; ++ pinctrl-0 = <&cif_clkout>; ++ ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ ++ /* avdd-supply = <>; */ ++ /* dvdd-supply = <>; */ ++ /* dovdd-supply = <>; */ ++ power-gpios = <&gpio4 26 GPIO_ACTIVE_HIGH>; ++ pwdn-gpios = <&gpio2 12 GPIO_ACTIVE_HIGH>; ++ rockchip,camera-module-index = <1>; ++ rockchip,camera-module-facing = "front"; ++ rockchip,camera-module-name = "CameraKing"; ++ rockchip,camera-module-lens-name = "Largan"; ++ port { ++ gc2145_out: endpoint { ++ remote-endpoint = <&dvp_in_fcam>; ++ }; ++ }; ++ }; ++ ++ ov13850: ov13850@10 { ++ compatible = "ovti,ov13850"; ++ status = "okay"; ++ reg = <0x10>; ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ /* avdd-supply = <>; */ ++ /* dvdd-supply = <>; */ ++ /* dovdd-supply = <>; */ ++ /* reset-gpios = <>; */ ++ reset-gpios = <&gpio1 3 GPIO_ACTIVE_HIGH>; ++ pwdn-gpios = <&gpio1 24 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "rockchip,camera_default"; ++ pinctrl-0 = <&cif_clkout>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "CMK-CT0116"; ++ rockchip,camera-module-lens-name = "Largan-50013A1"; ++ lens-focus = <&vm149c>; ++ flash-leds = <&sgm3784_led0 &sgm3784_led1>; ++ port { ++ ucam_out0: endpoint { ++ remote-endpoint = <&mipi_in_ucam0>; ++ //remote-endpoint = <&mipi_in_ucam1>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++ ++ ov4689: ov4689@36 { ++ compatible = "ovti,ov4689"; ++ status = "disabled"; ++ reg = <0x36>; ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ /* avdd-supply = <>; */ ++ /* dvdd-supply = <>; */ ++ /* dovdd-supply = <>; */ ++ /* reset-gpios = <>; */ ++ reset-gpios = <&gpio2 10 GPIO_ACTIVE_HIGH>; ++ pwdn-gpios = <&gpio4 29 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "rockchip,camera_default"; ++ pinctrl-0 = <&cif_clkout>; ++ rockchip,camera-module-index = <1>; ++ rockchip,camera-module-facing = "front"; ++ rockchip,camera-module-name = "JSD3425-C1"; ++ rockchip,camera-module-lens-name = "JSD3425-C1"; ++ port { ++ ucam_out1: endpoint { ++ //remote-endpoint = <&mipi_in_ucam0>; ++ remote-endpoint = <&mipi_in_ucam1>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++}; ++ ++&i2s2 { ++ #sound-dai-cells = <0>; ++ status = "okay"; ++}; ++ ++&isp0_mmu { ++ status = "okay"; ++}; ++ ++&isp1_mmu { ++ status = "okay"; ++}; ++ ++&mipi_dphy_rx0 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam0: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ucam_out0>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_rx0_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp0_mipi_in>; ++ }; ++ }; ++ }; ++}; ++ ++&mipi_dphy_tx1rx1 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam1: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ucam_out1>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_tx1rx1_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp1_mipi_in>; ++ }; ++ }; ++ }; ++}; ++ ++&hdmi_sound { ++ status = "okay"; ++}; ++ ++&route_edp { ++ status = "okay"; ++}; ++ ++&route_hdmi { ++ status = "okay"; ++ connect = <&vopl_out_hdmi>; ++}; ++ ++&i2s1 { ++ #sound-dai-cells = <0>; ++ status = "okay"; ++}; ++ ++&rk809_sound { ++ status = "okay"; ++}; ++ ++&hdmi_in_vopb { ++ status = "disabled"; ++}; ++ ++&rkisp1_0 { ++ status = "okay"; ++ ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp0_mipi_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dphy_rx0_out>; ++ }; ++ }; ++}; ++ ++&rkisp1_1 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_clkout &isp_dvp_d0d7>; ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp1_mipi_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dphy_tx1rx1_out>; ++ }; ++ dvp_in_fcam: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&gc2145_out>; ++ }; ++ }; ++}; ++ ++ ++&vcca_0v9 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <900000>; ++ regulator-name = "vcca_0v9"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <900000>; ++ }; ++}; ++ ++&vcc0v9_soc { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <900000>; ++ ++ regulator-name = "vcc0v9_soc"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++}; ++ ++/* ++ * if enable dp_sound, should disable spdif_sound and spdif_out ++ */ ++&spdif_out { ++ status = "disabled"; ++}; ++ ++&spdif_sound { ++ status = "disabled"; ++}; ++ ++&i2s0 { ++ #sound-dai-cells = <0>; ++ status = "disabled"; ++}; ++ ++&tc358749x_sound { ++ status = "disabled"; ++}; ++ ++&pinctrl { ++ lcd-panel { ++ lcd_panel_reset: lcd-panel-reset { ++ rockchip,pins = <1 RK_PC7 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++}; ++ +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-evb-ind.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-evb-ind.dtsi +new file mode 100755 +index 000000000000..dc821a2d45c4 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-evb-ind.dtsi +@@ -0,0 +1,1430 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++#include "dt-bindings/pwm/pwm.h" ++#include "rk3399.dtsi" ++#include "rk3399-opp.dtsi" ++#include ++#include "rk3399-vop-clk-set.dtsi" ++#include ++ ++/ { ++ compatible = "rockchip,rk3399-evb-ind", "rockchip,rk3399"; ++ ++ adc_keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 1>; ++ io-channel-names = "buttons"; ++ keyup-threshold-microvolt = <1800000>; ++ poll-interval = <100>; ++ ++ vol-up-key { ++ label = "volume up"; ++ linux,code = ; ++ press-threshold-microvolt = <1000>; ++ }; ++ ++ vol-down-key { ++ label = "volume down"; ++ linux,code = ; ++ press-threshold-microvolt = <170000>; ++ }; ++ }; ++ ++ backlight: backlight { ++ status = "okay"; ++ compatible = "pwm-backlight"; ++ pwms = <&pwm2 0 25000 0>; ++ brightness-levels = < ++ 0 1 2 3 4 5 6 7 ++ 8 9 10 11 12 13 14 15 ++ 16 17 18 19 20 21 22 23 ++ 24 25 26 27 28 29 30 31 ++ 32 33 34 35 36 37 38 39 ++ 40 41 42 43 44 45 46 47 ++ 48 49 50 51 52 53 54 55 ++ 56 57 58 59 60 61 62 63 ++ 64 65 66 67 68 69 70 71 ++ 72 73 74 75 76 77 78 79 ++ 80 81 82 83 84 85 86 87 ++ 88 89 90 91 92 93 94 95 ++ 96 97 98 99 100 101 102 103 ++ 104 105 106 107 108 109 110 111 ++ 112 113 114 115 116 117 118 119 ++ 120 121 122 123 124 125 126 127 ++ 128 129 130 131 132 133 134 135 ++ 136 137 138 139 140 141 142 143 ++ 144 145 146 147 148 149 150 151 ++ 152 153 154 155 156 157 158 159 ++ 160 161 162 163 164 165 166 167 ++ 168 169 170 171 172 173 174 175 ++ 176 177 178 179 180 181 182 183 ++ 184 185 186 187 188 189 190 191 ++ 192 193 194 195 196 197 198 199 ++ 200 201 202 203 204 205 206 207 ++ 208 209 210 211 212 213 214 215 ++ 216 217 218 219 220 221 222 223 ++ 224 225 226 227 228 229 230 231 ++ 232 233 234 235 236 237 238 239 ++ 240 241 242 243 244 245 246 247 ++ 248 249 250 251 252 253 254 255>; ++ default-brightness-level = <200>; ++ }; ++ ++ clkin_gmac: external-gmac-clock { ++ compatible = "fixed-clock"; ++ clock-frequency = <125000000>; ++ clock-output-names = "clkin_gmac"; ++ #clock-cells = <0>; ++ }; ++ ++ dw_hdmi_audio: dw-hdmi-audio { ++ status = "disabled"; ++ compatible = "rockchip,dw-hdmi-audio"; ++ #sound-dai-cells = <0>; ++ }; ++ ++ rk809_sound: rk809-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,rk809-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,widgets = ++ "Microphone", "Mic Jack", ++ "Headphone", "Headphone Jack"; ++ simple-audio-card,routing = ++ "Mic Jack", "MICBIAS1", ++ "IN1P", "Mic Jack", ++ "Headphone Jack", "HPOL", ++ "Headphone Jack", "HPOR"; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s1>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&rk809_codec>; ++ }; ++ }; ++ ++ dp_sound: dp-sound { ++ status = "disabled"; ++ compatible = "rockchip,cdndp-sound"; ++ rockchip,cpu = <&spdif>; ++ rockchip,codec = <&cdn_dp 1>; ++ }; ++ ++ hdmi_sound: hdmi-sound { ++ status = "disabled"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,name = "rockchip,hdmi"; ++ ++ simple-audio-card,cpu { ++ sound-dai = <&i2s2>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&hdmi>; ++ }; ++ }; ++ ++ rk_headset: rk-headset { ++ compatible = "rockchip_headset"; ++ headset_gpio = <&gpio0 11 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hp_det>; ++ io-channels = <&saradc 2>; ++ }; ++ ++ spdif_sound: spdif-sound { ++ status = "okay"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,name = "ROCKCHIP,SPDIF"; ++ simple-audio-card,cpu { ++ sound-dai = <&spdif>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&spdif_out>; ++ }; ++ }; ++ ++ spdif_out: spdif-out { ++ status = "okay"; ++ compatible = "linux,spdif-dit"; ++ #sound-dai-cells = <0>; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ clocks = <&rk809 1>; ++ clock-names = "ext_clock"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio0 10 GPIO_ACTIVE_LOW>; /* GPIO0_B2 */ ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ rockchip,grf = <&grf>; ++ wifi_chip_type = "ap6354"; ++ sdio_vref = <1800>; ++ WIFI,host_wake_irq = <&gpio0 3 GPIO_ACTIVE_HIGH>; /* GPIO0_a3 */ ++ status = "okay"; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&rk809 1>; ++ clock-names = "ext_clock"; ++ //wifi-bt-power-toggle; ++ uart_rts_gpios = <&gpio2 19 GPIO_ACTIVE_LOW>; /* GPIO2_C3 */ ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart0_rts>, <&bt_reset_gpio>, <&bt_wake_gpio>, <&bt_irq_gpio>; ++ pinctrl-1 = <&uart0_gpios>; ++ //BT,power_gpio = <&gpio3 19 GPIO_ACTIVE_HIGH>; /* GPIOx_xx */ ++ BT,reset_gpio = <&gpio0 9 GPIO_ACTIVE_HIGH>; /* GPIO0_B1 */ ++ BT,wake_gpio = <&gpio0 4 GPIO_ACTIVE_HIGH>; /* GPIO0_A4 */ ++ BT,wake_host_irq = <&gpio2 27 GPIO_ACTIVE_HIGH>; /* GPIO2_D3 */ ++ status = "okay"; ++ }; ++ ++ rk_modem: rk-modem { ++ compatible="4g-modem-platdata"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <<e_vbat <e_power_en <e_reset>; ++ 4G,vbat-gpio = <&gpio4 RK_PD0 GPIO_ACTIVE_HIGH>; ++ 4G,power-gpio = <&gpio4 RK_PC6 GPIO_ACTIVE_LOW>; ++ 4G,reset-gpio = <&gpio4 RK_PD4 GPIO_ACTIVE_LOW>; ++ status = "okay"; ++ }; ++ ++ tc358749x_sound:tc358749x-sound { ++ status = "okay"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rk,hdmiin-tc358749x-codec"; ++ simple-audio-card,bitclock-master = <&sound0_master>; ++ simple-audio-card,frame-master = <&sound0_master>; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s0>; ++ }; ++ sound0_master: simple-audio-card,codec { ++ sound-dai = <&tc358749x>; ++ }; ++ }; ++ ++ vcc5v0_host: vcc5v0-host-regulator { ++ compatible = "regulator-fixed"; ++ enable-active-high; ++ gpio = <&gpio0 8 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&host_vbus_drv>; ++ regulator-name = "vcc5v0_host"; ++ regulator-always-on; ++ }; ++ ++ vcc5v0_usbnet: vcc5v0-usbnet { ++ compatible = "regulator-fixed"; ++ enable-active-high; ++ /*disabled r8152 usb net default*/ ++ //regulator-always-on; ++ //regulator-boot-on; ++ gpio = <&gpio4 18 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&usbnet_pwr_drv>; ++ regulator-name = "vcc5v0_usbnet"; ++ startup-delay-us = <20000>; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ status = "okay"; ++ }; ++ ++ vcc5v0_sys: vcc5v0-sys { ++ compatible = "regulator-fixed"; ++ enable-active-high; ++ gpio = <&gpio4 26 GPIO_ACTIVE_HIGH>; ++ regulator-name = "vcc5v0_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ }; ++ ++ vcc_phy: vcc-phy-regulator { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_phy"; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ vdd_log: vdd-log { ++ compatible = "regulator-fixed"; ++ regulator-name = "vdd_log"; ++ regulator-min-microvolt = <950000>; ++ regulator-max-microvolt = <950000>; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++}; ++ ++&cpu_l0 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_l1 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_l2 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_l3 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_b0 { ++ cpu-supply = <&vdd_cpu_b>; ++}; ++ ++&cpu_b1 { ++ cpu-supply = <&vdd_cpu_b>; ++}; ++ ++&cdn_dp { ++ status = "okay"; ++ extcon = <&fusb0>; ++ phys = <&tcphy0_dp>; ++}; ++ ++&dp_in_vopb { ++ status = "disabled"; ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&dmc { ++ status = "okay"; ++ center-supply = <&vdd_center>; ++ upthreshold = <40>; ++ downdifferential = <20>; ++ system-status-freq = < ++ /*system status freq(KHz)*/ ++ SYS_STATUS_NORMAL 856000 ++ SYS_STATUS_REBOOT 856000 ++ SYS_STATUS_SUSPEND 328000 ++ SYS_STATUS_VIDEO_1080P 666000 ++ SYS_STATUS_VIDEO_4K 856000 ++ SYS_STATUS_VIDEO_4K_10B 856000 ++ SYS_STATUS_PERFORMANCE 856000 ++ SYS_STATUS_BOOST 856000 ++ SYS_STATUS_DUALVIEW 856000 ++ SYS_STATUS_ISP 856000 ++ >; ++ vop-bw-dmc-freq = < ++ /* min_bw(MB/s) max_bw(MB/s) freq(KHz) */ ++ 0 762 416000 ++ 763 3012 666000 ++ 3013 99999 856000 ++ >; ++ ++ vop-pn-msch-readlatency = < ++ 0 0x20 ++ 4 0x20 ++ >; ++ ++ auto-min-freq = <328000>; ++ auto-freq-en = <0>; ++}; ++ ++&dmc_opp_table { ++ compatible = "operating-points-v2"; ++ ++ opp-200000000 { ++ opp-hz = /bits/ 64 <200000000>; ++ opp-microvolt = <900000>; ++ status = "disabled"; ++ }; ++ opp-300000000 { ++ opp-hz = /bits/ 64 <300000000>; ++ opp-microvolt = <900000>; ++ status = "disabled"; ++ }; ++ opp-328000000 { ++ opp-hz = /bits/ 64 <328000000>; ++ opp-microvolt = <900000>; ++ }; ++ opp-400000000 { ++ opp-hz = /bits/ 64 <400000000>; ++ opp-microvolt = <900000>; ++ status = "disabled"; ++ }; ++ opp-416000000 { ++ opp-hz = /bits/ 64 <416000000>; ++ opp-microvolt = <900000>; ++ }; ++ opp-528000000 { ++ opp-hz = /bits/ 64 <528000000>; ++ opp-microvolt = <900000>; ++ status = "disabled"; ++ }; ++ opp-600000000 { ++ opp-hz = /bits/ 64 <600000000>; ++ opp-microvolt = <900000>; ++ status = "disabled"; ++ }; ++ opp-666000000 { ++ opp-hz = /bits/ 64 <666000000>; ++ opp-microvolt = <900000>; ++ }; ++ opp-800000000 { ++ opp-hz = /bits/ 64 <800000000>; ++ opp-microvolt = <900000>; ++ status = "disabled"; ++ }; ++ opp-856000000 { ++ opp-hz = /bits/ 64 <856000000>; ++ opp-microvolt = <900000>; ++ }; ++ opp-928000000 { ++ opp-hz = /bits/ 64 <928000000>; ++ opp-microvolt = <900000>; ++ status = "disabled"; ++ }; ++}; ++ ++&emmc_phy { ++ status = "okay"; ++}; ++ ++&gmac { ++ phy-supply = <&vcc_phy>; ++ phy-mode = "rgmii"; ++ clock_in_out = "input"; ++ snps,reset-gpio = <&gpio3 15 GPIO_ACTIVE_LOW>; ++ snps,reset-active-low; ++ snps,reset-delays-us = <0 10000 150000>; ++ assigned-clocks = <&cru SCLK_RMII_SRC>; ++ assigned-clock-parents = <&clkin_gmac>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&rgmii_pins>; ++ tx_delay = <0x22>; ++ rx_delay = <0x08>; ++ status = "okay"; ++}; ++ ++&gpu { ++ status = "okay"; ++ mali-supply = <&vdd_gpu>; ++}; ++ ++&hdmi { ++ status = "okay"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <168>; ++ i2c-scl-falling-time-ns = <4>; ++ clock-frequency = <400000>; ++ ++ vdd_cpu_b: tcs452x@1c { ++ compatible = "tcs,tcs452x"; ++ reg = <0x1c>; ++ vin-supply = <&vcc5v0_sys>; ++ regulator-compatible = "fan53555-reg"; ++ pinctrl-0 = <&vsel1_gpio>; ++ vsel-gpios = <&gpio1 RK_PC1 GPIO_ACTIVE_HIGH>; ++ regulator-name = "vdd_cpu_b"; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1500000>; ++ regulator-ramp-delay = <1000>; ++ fcs,suspend-voltage-selector = <1>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-state = <3>; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdd_gpu: tcs452x@10 { ++ compatible = "tcs,tcs452x"; ++ reg = <0x10>; ++ vin-supply = <&vcc5v0_sys>; ++ regulator-compatible = "fan53555-reg"; ++ pinctrl-0 = <&vsel2_gpio>; ++ vsel-gpios = <&gpio1 RK_PB6 GPIO_ACTIVE_HIGH>; ++ regulator-name = "vdd_gpu"; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1500000>; ++ regulator-ramp-delay = <1000>; ++ fcs,suspend-voltage-selector = <1>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-state = <3>; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ rk809: pmic@20 { ++ compatible = "rockchip,rk809"; ++ reg = <0x20>; ++ interrupt-parent = <&gpio1>; ++ interrupts = <21 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default", "pmic-sleep", ++ "pmic-power-off", "pmic-reset"; ++ pinctrl-0 = <&pmic_int_l>; ++ pinctrl-1 = <&soc_slppin_slp>, <&rk809_slppin_slp>; ++ pinctrl-2 = <&soc_slppin_gpio>, <&rk809_slppin_pwrdn>; ++ pinctrl-3 = <&soc_slppin_gpio>,<&rk809_slppin_null>; ++ rockchip,system-power-controller; ++ #clock-cells = <1>; ++ pmic-reset-func = <0>; ++ wakeup-source; ++ clock-output-names = "xin32k", "rk808-clkout2"; ++ ++ vcc1-supply = <&vcc5v0_sys>; ++ vcc2-supply = <&vcc5v0_sys>; ++ vcc3-supply = <&vcc5v0_sys>; ++ vcc4-supply = <&vcc5v0_sys>; ++ vcc5-supply = <&vcc_buck5>; ++ vcc6-supply = <&vcc_buck5>; ++ vcc7-supply = <&vcc5v0_sys>; ++ vcc8-supply = <&vcc3v3_sys>; ++ vcc9-supply = <&vcc5v0_sys>; ++ ++ pwrkey { ++ status = "okay"; ++ }; ++ ++ rtc { ++ status = "okay"; ++ }; ++ ++ pinctrl_rk8xx: pinctrl_rk8xx { ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ rk809_slppin_null: rk809_slppin_null { ++ pins = "gpio_slp"; ++ function = "pin_fun0"; ++ }; ++ ++ rk809_slppin_slp: rk809_slppin_slp { ++ pins = "gpio_slp"; ++ function = "pin_fun1"; ++ }; ++ ++ rk809_slppin_pwrdn: rk809_slppin_pwrdn { ++ pins = "gpio_slp"; ++ function = "pin_fun2"; ++ }; ++ ++ rk809_slppin_rst: rk809_slppin_rst { ++ pins = "gpio_slp"; ++ function = "pin_fun3"; ++ }; ++ }; ++ ++ regulators { ++ vdd_center: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <750000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_center"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdd_cpu_l: DCDC_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <750000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_cpu_l"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc_ddr"; ++ regulator-initial-mode = <0x2>; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc3v3_sys: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc3v3_sys"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc_buck5: DCDC_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <2200000>; ++ regulator-max-microvolt = <2200000>; ++ regulator-name = "vcc_buck5"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <2200000>; ++ }; ++ }; ++ ++ vcca_0v9: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <900000>; ++ regulator-name = "vcca_0v9"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_1v8: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcc_1v8"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vcc0v9_soc: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <900000>; ++ ++ regulator-name = "vcc0v9_soc"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <900000>; ++ }; ++ }; ++ ++ vcca_1v8: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcca_1v8"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdd1v5_dvp: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ ++ regulator-name = "vdd1v5_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_1v5: LDO_REG6 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ ++ regulator-name = "vcc_1v5"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_3v0: LDO_REG7 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ ++ regulator-name = "vcc_3v0"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vccio_sd: LDO_REG8 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vccio_sd"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_sd: LDO_REG9 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vcc_sd"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc5v0_usb: SWITCH_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc5v0_usb"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vccio_3v3: SWITCH_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vccio_3v3"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ }; ++ /* ++ battery { ++ compatible = "rk817,battery"; ++ ocv_table = <7000 7250 7370 7384 7436 7470 7496 ++ 7520 7548 7576 7604 7632 7668 7706 ++ 7754 7816 7892 7950 8036 8142 8212>; ++ design_capacity = <2500>; ++ design_qmax = <2750>; ++ bat_res = <100>; ++ sleep_enter_current = <300>; ++ sleep_exit_current = <300>; ++ sleep_filter_current = <100>; ++ power_off_thresd = <7000>; ++ zero_algorithm_vol = <7700>; ++ max_soc_offset = <60>; ++ monitor_sec = <5>; ++ sample_res = <10>; ++ virtual_power = <0>; ++ bat_res_up = <140>; ++ bat_res_down = <20>; ++ }; ++ */ ++ rk809_codec: codec { ++ #sound-dai-cells = <0>; ++ compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; ++ clocks = <&cru SCLK_I2S_8CH_OUT>; ++ clock-names = "mclk"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s_8ch_mclk>; ++ assigned-clocks = <&cru SCLK_I2SOUT_SRC>; ++ assigned-clock-parents = <&cru SCLK_I2S1_8CH>; ++ hp-volume = <20>; ++ spk-volume = <3>; ++ status = "okay"; ++ }; ++ }; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <300>; ++ i2c-scl-falling-time-ns = <15>; ++ clock-frequency = <400000>; ++ ++ gsl3673: gsl3673@40 { ++ compatible = "GSL,GSL3673"; ++ reg = <0x40>; ++ screen_max_x = <1536>; ++ screen_max_y = <2048>; ++ irq_gpio_number = <&gpio1 20 IRQ_TYPE_LEVEL_LOW>; ++ rst_gpio_number = <&gpio1 1 GPIO_ACTIVE_HIGH>; ++ }; ++ ++ tc358749x: tc358749x@0f { ++ #sound-dai-cells = <0>; ++ compatible = "toshiba,tc358749x"; ++ reg = <0x0f>; ++ power-gpios = <&gpio0 13 GPIO_ACTIVE_HIGH>; ++ stanby-gpios = <&gpio3 25 GPIO_ACTIVE_HIGH>; ++ reset-gpios = <&gpio0 2 GPIO_ACTIVE_HIGH>; ++ int-gpios = <&gpio4 4 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hdmiin_gpios>; ++ status = "okay"; ++ }; ++}; ++ ++&i2c4 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <475>; ++ i2c-scl-falling-time-ns = <26>; ++ ++ mpu6500@68 { ++ status = "disabled"; ++ compatible = "invensense,mpu6500"; ++ reg = <0x68>; ++ irq-gpio = <&gpio1 22 IRQ_TYPE_EDGE_RISING>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&mpu6500_irq_gpio>; ++ mpu-int_config = <0x10>; ++ mpu-level_shifter = <0>; ++ mpu-orientation = <0 1 0 1 0 0 0 0 1>; ++ orientation-x= <0>; ++ orientation-y= <0>; ++ orientation-z= <0>; ++ mpu-debug = <1>; ++ }; ++ ++ mpu6500_acc: mpu_acc@68 { ++ compatible = "mpu6500_acc"; ++ reg = <0x68>; ++ irq-gpio = <&gpio1 22 IRQ_TYPE_EDGE_RISING>; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ type = ; ++ layout = <8>; ++ }; ++ ++ mpu6500_gyro: mpu_gyro@68 { ++ compatible = "mpu6500_gyro"; ++ reg = <0x68>; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ type = ; ++ layout = <8>; ++ }; ++ ++ fusb0: fusb30x@22 { ++ compatible = "fairchild,fusb302"; ++ reg = <0x22>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&fusb0_int>; ++ int-n-gpios = <&gpio1 2 GPIO_ACTIVE_HIGH>; ++ vbus-5v-gpios = <&gpio1 18 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ sensor@0d { ++ status = "okay"; ++ compatible = "ak8963"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&ak8963_irq_gpio>; ++ reg = <0x0d>; ++ type = ; ++ irq-gpio = <&gpio1 0 IRQ_TYPE_EDGE_RISING>; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ layout = <6>; ++ }; ++ ++ bq25700: bq25700@6b {//6a ++ compatible = "ti,bq25703"; ++ reg = <0x6b>; ++ extcon = <&fusb0>; ++ ++ interrupt-parent = <&gpio0>; ++ interrupts = <5 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&charger_ok>; ++ ti,charge-current = <2500000>; ++ ti,max-input-voltage = <20000000>; ++ ti,max-input-current = <6000000>; ++ ti,max-charge-voltage = <8750000>; ++ ti,input-current = <500000>; ++ ti,input-current-sdp = <500000>; ++ ti,input-current-dcp = <2000000>; ++ ti,input-current-cdp = <2000000>; ++ ti,minimum-sys-voltage = <7400000>; ++ ti,otg-voltage = <5000000>; ++ ti,otg-current = <500000>; ++ pd-charge-only = <0>; ++ status = "disabled"; ++ }; ++}; ++ ++&i2c6 { ++ cw2015@62 { ++ status = "disabled"; ++ compatible = "cw201x"; ++ reg = <0x62>; ++ bat_config_info = <0x15 0x42 0x60 0x59 0x52 0x58 0x4D 0x48 ++ 0x48 0x44 0x44 0x46 0x49 0x48 0x32 0x24 ++ 0x20 0x17 0x13 0x0F 0x19 0x3E 0x51 0x45 ++ 0x08 0x76 0x0B 0x85 0x0E 0x1C 0x2E 0x3E ++ 0x4D 0x52 0x52 0x57 0x3D 0x1B 0x6A 0x2D ++ 0x25 0x43 0x52 0x87 0x8F 0x91 0x94 0x52 ++ 0x82 0x8C 0x92 0x96 0xFF 0x7B 0xBB 0xCB ++ 0x2F 0x7D 0x72 0xA5 0xB5 0xC1 0x46 0xAE>; ++ monitor_sec = <5>; ++ virtual_power = <0>; ++ }; ++}; ++ ++&i2s0 { ++ status = "disabled"; ++ rockchip,i2s-broken-burst-len; ++ rockchip,playback-channels = <8>; ++ rockchip,capture-channels = <8>; ++ #sound-dai-cells = <0>; ++}; ++ ++&i2s1 { ++ #sound-dai-cells = <0>; ++ status = "disabled"; ++}; ++ ++&i2s2 { ++ #sound-dai-cells = <0>; ++ dmas = <&dmac_bus 4>; ++ dma-names = "tx"; ++ status = "disabled"; ++}; ++ ++&io_domains { ++ status = "okay"; ++ ++ bt656-supply = <&vcc_3v0>; /* bt656_gpio2ab_ms */ ++ audio-supply = <&vcca_1v8>; /* audio_gpio3d4a_ms */ ++ sdmmc-supply = <&vccio_sd>; /* sdmmc_gpio4b_ms */ ++ gpio1830-supply = <&vcc_3v0>; /* gpio1833_gpio4cd_ms */ ++}; ++ ++&pmu_io_domains { ++ pmu1830-supply = <&vcc_1v8>; ++ status = "okay"; ++}; ++ ++&isp0_mmu { ++ status = "okay"; ++}; ++ ++&isp1_mmu { ++ status = "okay"; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ pmu1830-supply = <&vcc_1v8>; ++}; ++ ++&pcie_phy { ++ status = "okay"; ++}; ++ ++&pcie0 { ++ ep-gpios = <&gpio2 RK_PA4 GPIO_ACTIVE_HIGH>; ++ num-lanes = <4>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pcie_clkreqn_cpm>; ++ status = "okay"; ++}; ++ ++&pwm0 { ++ status = "disabled"; ++}; ++ ++&pwm2 { ++ status = "okay"; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm2_pin_pull_down>; ++}; ++ ++&pwm3 { ++ status = "okay"; ++ ++ interrupts = ; ++ compatible = "rockchip,remotectl-pwm"; ++ remote_pwm_id = <3>; ++ handle_cpu_id = <1>; ++ remote_support_psci = <4>; ++ pinctrl-names = "default"; ++ ++ ir_key1 { ++ rockchip,usercode = <0x4040>; ++ rockchip,key_table = ++ <0xf2 KEY_REPLY>, ++ <0xba KEY_BACK>, ++ <0xf4 KEY_UP>, ++ <0xf1 KEY_DOWN>, ++ <0xef KEY_LEFT>, ++ <0xee KEY_RIGHT>, ++ <0xbd KEY_HOME>, ++ <0xea KEY_VOLUMEUP>, ++ <0xe3 KEY_VOLUMEDOWN>, ++ <0xe2 KEY_SEARCH>, ++ <0xb2 KEY_POWER>, ++ <0xbc KEY_MUTE>, ++ <0xec KEY_MENU>, ++ <0xbf 0x190>, ++ <0xe0 0x191>, ++ <0xe1 0x192>, ++ <0xe9 183>, ++ <0xe6 248>, ++ <0xe8 185>, ++ <0xe7 186>, ++ <0xf0 388>, ++ <0xbe 0x175>; ++ }; ++ ++ ir_key2 { ++ rockchip,usercode = <0xff00>; ++ rockchip,key_table = ++ <0xf9 KEY_HOME>, ++ <0xbf KEY_BACK>, ++ <0xfb KEY_MENU>, ++ <0xaa KEY_REPLY>, ++ <0xb9 KEY_UP>, ++ <0xe9 KEY_DOWN>, ++ <0xb8 KEY_LEFT>, ++ <0xea KEY_RIGHT>, ++ <0xeb KEY_VOLUMEDOWN>, ++ <0xef KEY_VOLUMEUP>, ++ <0xf7 KEY_MUTE>, ++ <0xe7 KEY_POWER>, ++ <0xfc KEY_POWER>, ++ <0xa9 KEY_VOLUMEDOWN>, ++ <0xa8 KEY_VOLUMEDOWN>, ++ <0xe0 KEY_VOLUMEDOWN>, ++ <0xa5 KEY_VOLUMEDOWN>, ++ <0xab 183>, ++ <0xb7 388>, ++ <0xe8 388>, ++ <0xf8 184>, ++ <0xaf 185>, ++ <0xed KEY_VOLUMEDOWN>, ++ <0xee 186>, ++ <0xb3 KEY_VOLUMEDOWN>, ++ <0xf1 KEY_VOLUMEDOWN>, ++ <0xf2 KEY_VOLUMEDOWN>, ++ <0xf3 KEY_SEARCH>, ++ <0xb4 KEY_VOLUMEDOWN>, ++ <0xbe KEY_SEARCH>; ++ }; ++ ++ ir_key3 { ++ rockchip,usercode = <0x1dcc>; ++ rockchip,key_table = ++ <0xee KEY_REPLY>, ++ <0xf0 KEY_BACK>, ++ <0xf8 KEY_UP>, ++ <0xbb KEY_DOWN>, ++ <0xef KEY_LEFT>, ++ <0xed KEY_RIGHT>, ++ <0xfc KEY_HOME>, ++ <0xf1 KEY_VOLUMEUP>, ++ <0xfd KEY_VOLUMEDOWN>, ++ <0xb7 KEY_SEARCH>, ++ <0xff KEY_POWER>, ++ <0xf3 KEY_MUTE>, ++ <0xbf KEY_MENU>, ++ <0xf9 0x191>, ++ <0xf5 0x192>, ++ <0xb3 388>, ++ <0xbe KEY_1>, ++ <0xba KEY_2>, ++ <0xb2 KEY_3>, ++ <0xbd KEY_4>, ++ <0xf9 KEY_5>, ++ <0xb1 KEY_6>, ++ <0xfc KEY_7>, ++ <0xf8 KEY_8>, ++ <0xb0 KEY_9>, ++ <0xb6 KEY_0>, ++ <0xb5 KEY_BACKSPACE>; ++ }; ++}; ++ ++&rockchip_suspend { ++ status = "okay"; ++ rockchip,sleep-debug-en = <1>; ++ rockchip,sleep-mode-config = < ++ (0 ++ | RKPM_SLP_ARMPD ++ | RKPM_SLP_PERILPPD ++ | RKPM_SLP_DDR_RET ++ | RKPM_SLP_PLLPD ++ | RKPM_SLP_CENTER_PD ++ | RKPM_SLP_AP_PWROFF ++ ) ++ >; ++ rockchip,wakeup-config = < ++ (0 ++ | RKPM_GPIO_WKUP_EN ++ | RKPM_PWM_WKUP_EN ++ ) ++ >; ++ rockchip,pwm-regulator-config = < ++ (0 ++ | PWM2_REGULATOR_EN ++ ) ++ >; ++ rockchip,power-ctrl = ++ <&gpio1 17 GPIO_ACTIVE_HIGH>, ++ <&gpio1 14 GPIO_ACTIVE_HIGH>; ++}; ++ ++&saradc { ++ status = "okay"; ++}; ++ ++&spdif { ++ status = "okay"; ++ pinctrl-0 = <&spdif_bus>; ++ i2c-scl-rising-time-ns = <450>; ++ i2c-scl-falling-time-ns = <15>; ++ #sound-dai-cells = <0>; ++}; ++ ++&sdio0 { ++ clock-frequency = <150000000>; ++ clock-freq-min-max = <200000 150000000>; ++ no-sd; ++ no-mmc; ++ bus-width = <4>; ++ disable-wp; ++ cap-sd-highspeed; ++ cap-sdio-irq; ++ keep-power-in-suspend; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ non-removable; ++ num-slots = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&sdmmc { ++ clock-frequency = <150000000>; ++ clock-freq-min-max = <100000 150000000>; ++ no-sdio; ++ no-mmc; ++ bus-width = <4>; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ disable-wp; ++ num-slots = <1>; ++ sd-uhs-sdr104; ++ vmmc-supply = <&vcc_sd>; ++ vqmmc-supply = <&vccio_sd>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_cd &sdmmc_bus4>; ++ status = "okay"; ++}; ++ ++&sdhci { ++ bus-width = <8>; ++ mmc-hs400-1_8v; ++ no-sdio; ++ no-sd; ++ non-removable; ++ keep-power-in-suspend; ++ mmc-hs400-enhanced-strobe; ++ status = "okay"; ++}; ++ ++&saradc { ++ status = "okay"; ++ vref-supply = <&vcc_1v8>; ++}; ++ ++&tcphy0 { ++ extcon = <&fusb0>; ++ status = "okay"; ++}; ++ ++&tcphy1 { ++ status = "okay"; ++}; ++ ++&tsadc { ++ /* tshut mode 0:CRU 1:GPIO */ ++ rockchip,hw-tshut-mode = <1>; ++ /* tshut polarity 0:LOW 1:HIGH */ ++ rockchip,hw-tshut-polarity = <1>; ++ status = "okay"; ++}; ++ ++&uart0 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart0_xfer &uart0_cts>; ++ status = "okay"; ++}; ++ ++&uart2 { ++ status = "disabled"; ++}; ++ ++&u2phy0 { ++ status = "okay"; ++ extcon = <&fusb0>; ++ ++ u2phy0_otg: otg-port { ++ status = "okay"; ++ }; ++ ++ u2phy0_host: host-port { ++ phy-supply = <&vcc5v0_usb>; ++ status = "okay"; ++ }; ++}; ++ ++&u2phy1 { ++ status = "okay"; ++ ++ u2phy1_otg: otg-port { ++ status = "okay"; ++ }; ++ ++ u2phy1_host: host-port { ++ status = "okay"; ++ }; ++}; ++ ++&usbdrd3_0 { ++ status = "okay"; ++}; ++ ++&usbdrd3_1 { ++ status = "okay"; ++}; ++ ++&usbdrd_dwc3_0 { ++ status = "okay"; ++ extcon = <&fusb0>; ++}; ++ ++&usbdrd_dwc3_1 { ++ status = "okay"; ++ dr_mode = "host"; ++}; ++ ++&usb_host0_ehci { ++ status = "okay"; ++}; ++ ++&usb_host0_ohci { ++ status = "okay"; ++}; ++ ++&usb_host1_ehci { ++ status = "okay"; ++}; ++ ++&usb_host1_ohci { ++ status = "okay"; ++}; ++ ++&vopb { ++ assigned-clocks = <&cru DCLK_VOP0_DIV>; ++ assigned-clock-parents = <&cru PLL_CPLL>; ++}; ++ ++&vopl { ++ assigned-clocks = <&cru DCLK_VOP1_DIV>; ++ assigned-clock-parents = <&cru PLL_VPLL>; ++}; ++ ++&pinctrl { ++ ++ ak8963 { ++ ak8963_irq_gpio: ak8963-irq-gpio { ++ rockchip,pins = <1 RK_PA0 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ charger { ++ charger_ok: charge-ok { ++ rockchip,pins = ++ <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ i2s0 { ++ i2s0_8ch_bus: i2s0-8ch-bus { ++ rockchip,pins = ++ <3 RK_PD0 1 &pcfg_pull_none>, ++ <3 RK_PD2 1 &pcfg_pull_none>, ++ <3 RK_PD3 1 &pcfg_pull_none>, ++ <3 RK_PD4 1 &pcfg_pull_none>, ++ <3 RK_PD5 1 &pcfg_pull_none>, ++ <3 RK_PD6 1 &pcfg_pull_none>, ++ <3 RK_PD7 1 &pcfg_pull_none>; ++ }; ++ ++ i2s_8ch_mclk: i2s-8ch-mclk { ++ rockchip,pins = <4 RK_PA0 1 &pcfg_pull_none>; ++ }; ++ }; ++ ++ i2s1 { ++ i2s1_2ch_bus: i2s1-2ch-bus { ++ rockchip,pins = ++ <4 RK_PA3 1 &pcfg_pull_none>, ++ <4 RK_PA5 1 &pcfg_pull_none>, ++ <4 RK_PA6 1 &pcfg_pull_none>, ++ <4 RK_PA7 1 &pcfg_pull_none>; ++ }; ++ }; ++ ++ headphone { ++ hp_det: hp-det { ++ rockchip,pins = ++ <0 RK_PB3 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ hdmiin { ++ hdmiin_gpios: hdmiin_gpios { ++ rockchip,pins = ++ <0 RK_PB5 RK_FUNC_GPIO &pcfg_pull_none>, ++ <3 RK_PD1 RK_FUNC_GPIO &pcfg_pull_none>, ++ <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>, ++ <4 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ fusb30x { ++ fusb0_int: fusb0-int { ++ rockchip,pins = <1 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>, ++ <1 RK_PC2 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ mpu6500 { ++ mpu6500_irq_gpio: mpu6500-irq-gpio { ++ rockchip,pins = <1 RK_PC6 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ pmic { ++ pmic_int_l: pmic-int-l { ++ rockchip,pins = ++ <1 RK_PC5 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ vsel1_gpio: vsel1-gpio { ++ rockchip,pins = ++ <1 RK_PC1 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ ++ vsel2_gpio: vsel2-gpio { ++ rockchip,pins = ++ <1 RK_PB6 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ ++ soc_slppin_gpio: soc-slppin-gpio { ++ rockchip,pins = ++ <1 RK_PA5 RK_FUNC_GPIO &pcfg_output_low>; ++ }; ++ ++ soc_slppin_slp: soc-slppin-slp { ++ rockchip,pins = ++ <1 RK_PA5 1 &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = ++ <0 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart0_gpios: uart0-gpios { ++ rockchip,pins = ++ <2 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ bt_reset_gpio: bt-reset-gpio { ++ rockchip,pins = ++ <0 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ bt_wake_gpio: bt-wake-gpio { ++ rockchip,pins = ++ <0 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ bt_irq_gpio: bt-irq-gpio { ++ rockchip,pins = ++ <2 RK_PD3 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ rk-modem { ++ lte_vbat: lte-vbat { ++ rockchip,pins = <4 RK_PD0 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ lte_power_en: lte-power-en { ++ rockchip,pins = <4 RK_PC6 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ lte_reset: lte-reset { ++ rockchip,pins = <4 RK_PD4 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ usb2 { ++ host_vbus_drv: host-vbus-drv { ++ rockchip,pins = ++ <0 RK_PB0 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ vcc_sd { ++ vcc_sd_h: vcc-sd-h { ++ rockchip,pins = ++ <0 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ usbnet { ++ usbnet_pwr_drv: usbnet-pwr-drv { ++ rockchip,pins = ++ <4 RK_PC2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-evb-rev1-android.dts b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev1-android.dts +new file mode 100755 +index 000000000000..bb7ddac77f23 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev1-android.dts +@@ -0,0 +1,145 @@ ++/* ++ * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This file is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This file is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++/dts-v1/; ++ ++#include "rk3399-evb-rev1.dtsi" ++#include "rk3399-android.dtsi" ++ ++/ { ++ model = "Rockchip RK3399 Evaluation Board v1 (Android)"; ++ compatible = "rockchip,android", "rockchip,rk3399-evb-rev1", "rockchip,rk3399"; ++}; ++ ++&vdd_log { ++ rockchip,pwm_id= <2>; ++ rockchip,pwm_voltage = <900000>; ++}; ++ ++&vdd_center { ++ rockchip,pwm_id= <3>; ++ rockchip,pwm_voltage = <900000>; ++}; ++ ++&dsi { ++ status = "okay"; ++ ++ panel@0 { ++ compatible ="simple-panel-dsi"; ++ reg = <0>; ++ backlight = <&backlight>; ++ enable-gpios = <&gpio1 13 GPIO_ACTIVE_HIGH>; ++ ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <160000000>; ++ hactive = <1200>; ++ vactive = <1920>; ++ hback-porch = <21>; ++ hfront-porch = <120>; ++ vback-porch = <18>; ++ vfront-porch = <21>; ++ hsync-len = <20>; ++ vsync-len = <3>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ panel_in_dsi: endpoint { ++ remote-endpoint = <&dsi_out_panel>; ++ }; ++ }; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@1 { ++ reg = <1>; ++ dsi_out_panel: endpoint { ++ remote-endpoint = <&panel_in_dsi>; ++ }; ++ }; ++ }; ++}; ++ ++&route_dsi { ++ status = "okay"; ++}; ++ ++&firmware_android { ++ compatible = "android,firmware"; ++ fstab { ++ compatible = "android,fstab"; ++ system { ++ compatible = "android,system"; ++ dev = "/dev/block/by-name/system"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,verify"; ++ }; ++ vendor { ++ compatible = "android,vendor"; ++ dev = "/dev/block/by-name/vendor"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,verify"; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-evb-rev1-cros.dts b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev1-cros.dts +new file mode 100755 +index 000000000000..bf27556295ec +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev1-cros.dts +@@ -0,0 +1,51 @@ ++/* ++ * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This file is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This file is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++/dts-v1/; ++ ++#include "rk3399-evb-rev1.dtsi" ++#include "rk3399-evb-cros.dtsi" ++ ++/ { ++ model = "Rockchip RK3399 Evaluation Board v1 (Chrome OS)"; ++ compatible = "google,rk3399evb-rev1", "rockchip,rk3399-evb-rev1", "rockchip,rk3399"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-evb-rev1.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev1.dtsi +new file mode 100755 +index 000000000000..f35d6ee56072 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev1.dtsi +@@ -0,0 +1,352 @@ ++/* ++ * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This file is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This file is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++#include "rk3399-evb.dtsi" ++#include "rk3399-early-opp.dtsi" ++ ++/ { ++ compatible = "rockchip,rk3399-evb-rev1", "rockchip,rk3399"; ++ ++ vdd_log: vdd-log { ++ compatible = "pwm-regulator"; ++ rockchip,pwm_id = <2>; ++ rockchip,pwm_voltage = <900000>; ++ pwms = <&pwm2 0 25000 1>; ++ regulator-name = "vdd_log"; ++ regulator-min-microvolt = <800000>; ++ regulator-max-microvolt = <1400000>; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ vdd_center: vdd-center { ++ compatible = "pwm-regulator"; ++ rockchip,pwm_id = <3>; ++ rockchip,pwm_voltage = <900000>; ++ pwms = <&pwm3 0 25000 1>; ++ regulator-name = "vdd_center"; ++ regulator-min-microvolt = <800000>; ++ regulator-max-microvolt = <1400000>; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ xin32k: xin32k { ++ compatible = "fixed-clock"; ++ clock-frequency = <32768>; ++ clock-output-names = "xin32k"; ++ #clock-cells = <0>; ++ }; ++}; ++ ++&cpu_l0 { ++ dynamic-power-coefficient = <121>; ++}; ++ ++&cpu_b0 { ++ dynamic-power-coefficient = <1068>; ++}; ++ ++&soc_thermal { ++ sustainable-power = <1600>; /* milliwatts */ ++ ++ cooling-maps { ++ map0 { ++ trip = <&target>; ++ cooling-device = ++ <&cpu_l0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; ++ contribution = <10240>; ++ }; ++ map1 { ++ trip = <&target>; ++ cooling-device = ++ <&cpu_b0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; ++ contribution = <1024>; ++ }; ++ map2 { ++ trip = <&target>; ++ cooling-device = ++ <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; ++ contribution = <10240>; ++ }; ++ }; ++}; ++ ++&gpu_power_model { ++ dynamic-power = <1780>; ++}; ++ ++&i2c0 { ++ fusb0: fusb30x@22 { ++ compatible = "fairchild,fusb302"; ++ reg = <0x22>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&fusb0_int>; ++ vbus-5v-gpios = <&gpio1 3 GPIO_ACTIVE_HIGH>; ++ int-n-gpios = <&gpio1 2 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ mp8865: mp8865@68 { ++ compatible = "mps,mp8865"; ++ reg = <0x68>; ++ regulators { ++ vdd_gpu: mp8865_dcdc1 { ++ regulator-name = "vdd_gpu"; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1500000>; ++ regulator-ramp-delay = <8000>; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ }; ++ }; ++ ++ rk808: pmic@1b { ++ compatible = "rockchip,rk808"; ++ reg = <0x1b>; ++ interrupt-parent = <&gpio1>; ++ interrupts = <21 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pmic_int_l &pmic_dvs2>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ ++ vcc1-supply = <&vcc3v3_sys>; ++ vcc2-supply = <&vcc3v3_sys>; ++ vcc3-supply = <&vcc3v3_sys>; ++ vcc4-supply = <&vcc3v3_sys>; ++ vcc6-supply = <&vcc3v3_sys>; ++ vcc7-supply = <&vcc3v3_sys>; ++ vcc8-supply = <&vcc3v3_sys>; ++ vcc9-supply = <&vcc3v3_sys>; ++ vcc10-supply = <&vcc3v3_sys>; ++ vcc11-supply = <&vcc3v3_sys>; ++ vcc12-supply = <&vcc3v3_sys>; ++ vddio-supply = <&vcc1v8_pmu>; ++ ++ regulators { ++ vdd_cpu_b: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <750000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-name = "vdd_cpu_b"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdd_cpu_l: DCDC_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <750000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-name = "vdd_cpu_l"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc_ddr"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_1v8: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc_1v8"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vcc1v8_dvp: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc1v8_dvp"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vcc3v0_tp: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-name = "vcc3v0_tp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc1v8_pmu: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc1v8_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vcc_sd: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-name = "vcc_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ ++ vcca3v0_codec: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-name = "vcca3v0_codec"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ ++ vcc_1v5: LDO_REG6 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ regulator-name = "vcc_1v5"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1500000>; ++ }; ++ }; ++ ++ vcca1v8_codec: LDO_REG7 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcca1v8_codec"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vcc_3v0: LDO_REG8 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-name = "vcc_3v0"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ ++ vcc3v3_s3: SWITCH_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc3v3_s3"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc3v3_s0: SWITCH_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc3v3_s0"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ }; ++ }; ++}; ++ ++&i2c4 { ++ fusb1: fusb30x@22 { ++ compatible = "fairchild,fusb302"; ++ reg = <0x22>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&fusb1_int>; ++ vbus-5v-gpios = <&gpio1 4 GPIO_ACTIVE_HIGH>; ++ int-n-gpios = <&gpio1 24 GPIO_ACTIVE_HIGH>; ++ }; ++}; ++ ++&pwm2 { ++ status = "okay"; ++}; ++ ++&pwm3 { ++ status = "okay"; ++}; ++ ++&u2phy0_otg { ++ rockchip,utmi-avalid; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-evb-rev2-android.dts b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev2-android.dts +new file mode 100755 +index 000000000000..b15fb8a9d088 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev2-android.dts +@@ -0,0 +1,156 @@ ++/* ++ * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This file is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This file is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++/dts-v1/; ++ ++#include "rk3399-evb-rev2.dtsi" ++#include "rk3399-android.dtsi" ++ ++/ { ++ model = "Rockchip RK3399 Evaluation Board v2 (Android)"; ++ compatible = "rockchip,android", "rockchip,rk3399-evb-rev2", "rockchip,rk3399"; ++}; ++ ++&vdd_center { ++ rockchip,pwm_id= <3>; ++ rockchip,pwm_voltage = <900000>; ++}; ++ ++&i2s2 { ++ status = "okay"; ++}; ++ ++&spdif { ++ status = "okay"; ++}; ++ ++&spdif_out { ++ status = "okay"; ++}; ++ ++&spdif_sound { ++ status = "okay"; ++}; ++ ++&dsi { ++ status = "okay"; ++ ++ panel@0 { ++ compatible ="simple-panel-dsi"; ++ reg = <0>; ++ backlight = <&backlight>; ++ enable-gpios = <&gpio1 13 GPIO_ACTIVE_HIGH>; ++ ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <160000000>; ++ hactive = <1200>; ++ vactive = <1920>; ++ hback-porch = <21>; ++ hfront-porch = <120>; ++ vback-porch = <18>; ++ vfront-porch = <21>; ++ hsync-len = <20>; ++ vsync-len = <3>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ panel_in_dsi: endpoint { ++ remote-endpoint = <&dsi_out_panel>; ++ }; ++ }; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@1 { ++ reg = <1>; ++ dsi_out_panel: endpoint { ++ remote-endpoint = <&panel_in_dsi>; ++ }; ++ }; ++ }; ++}; ++ ++&route_dsi { ++ status = "okay"; ++}; ++ ++&firmware_android { ++ compatible = "android,firmware"; ++ fstab { ++ compatible = "android,fstab"; ++ system { ++ compatible = "android,system"; ++ dev = "/dev/block/by-name/system"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,verify"; ++ }; ++ vendor { ++ compatible = "android,vendor"; ++ dev = "/dev/block/by-name/vendor"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,verify"; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-evb-rev2-cros.dts b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev2-cros.dts +new file mode 100755 +index 000000000000..2daf9f1235f3 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev2-cros.dts +@@ -0,0 +1,51 @@ ++/* ++ * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This file is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This file is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++/dts-v1/; ++ ++#include "rk3399-evb-rev2.dtsi" ++#include "rk3399-evb-cros.dtsi" ++ ++/ { ++ model = "Rockchip RK3399 Evaluation Board v2 (Chrome OS)"; ++ compatible = "google,rk3399evb-rev2", "rockchip,rk3399-evb-rev2", "rockchip,rk3399"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-evb-rev2.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev2.dtsi +new file mode 100755 +index 000000000000..4b23c1fcadd8 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev2.dtsi +@@ -0,0 +1,366 @@ ++/* ++ * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This file is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This file is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++#include "rk3399-evb.dtsi" ++#include "rk3399-early-opp.dtsi" ++ ++/ { ++ compatible = "rockchip,rk3399-evb-rev2", "rockchip,rk3399"; ++ ++ vcc5v0_sys: vcc5v0-sys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc5v0_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ }; ++ ++ vdd_center: vdd-center { ++ compatible = "pwm-regulator"; ++ rockchip,pwm_id = <3>; ++ rockchip,pwm_voltage = <900000>; ++ pwms = <&pwm3 0 25000 1>; ++ regulator-name = "vdd_center"; ++ regulator-min-microvolt = <800000>; ++ regulator-max-microvolt = <1400000>; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ xin32k: xin32k { ++ compatible = "fixed-clock"; ++ clock-frequency = <32768>; ++ clock-output-names = "xin32k"; ++ #clock-cells = <0>; ++ }; ++}; ++ ++&cpu_l0 { ++ dynamic-power-coefficient = <121>; ++}; ++ ++&cpu_b0 { ++ dynamic-power-coefficient = <1068>; ++}; ++ ++&soc_thermal { ++ sustainable-power = <1600>; /* milliwatts */ ++ ++ cooling-maps { ++ map0 { ++ trip = <&target>; ++ cooling-device = ++ <&cpu_l0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; ++ contribution = <10240>; ++ }; ++ map1 { ++ trip = <&target>; ++ cooling-device = ++ <&cpu_b0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; ++ contribution = <1024>; ++ }; ++ map2 { ++ trip = <&target>; ++ cooling-device = ++ <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; ++ contribution = <10240>; ++ }; ++ }; ++}; ++ ++&gpu_power_model { ++ dynamic-power = <1780>; ++}; ++ ++&i2c0 { ++ fusb1: fusb30x@22 { ++ compatible = "fairchild,fusb302"; ++ reg = <0x22>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&fusb1_int>; ++ vbus-5v-gpios = <&gpio1 4 GPIO_ACTIVE_HIGH>; ++ int-n-gpios = <&gpio1 24 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ vdd_cpu_b: syr827@40 { ++ compatible = "silergy,syr827"; ++ reg = <0x40>; ++ vin-supply = <&vcc5v0_sys>; ++ regulator-compatible = "fan53555-reg"; ++ regulator-name = "vdd_cpu_b"; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1500000>; ++ regulator-ramp-delay = <1000>; ++ fcs,suspend-voltage-selector = <1>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-state = <3>; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ lp8752: lp8752@60 { ++ compatible = "ti,lp8752"; ++ reg = <0x60>; ++ vin0-supply = <&vcc5v0_sys>; ++ regulators { ++ vdd_gpu: lp8752_buck0 { ++ regulator-name = "vdd_gpu"; ++ regulator-min-microvolt = <735000>; ++ regulator-max-microvolt = <1400000>; ++ regulator-ramp-delay = <6000>; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ }; ++ }; ++ ++ rk808: pmic@1b { ++ compatible = "rockchip,rk808"; ++ reg = <0x1b>; ++ interrupt-parent = <&gpio1>; ++ interrupts = <21 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pmic_int_l &pmic_dvs2>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ ++ vcc1-supply = <&vcc3v3_sys>; ++ vcc2-supply = <&vcc3v3_sys>; ++ vcc3-supply = <&vcc3v3_sys>; ++ vcc4-supply = <&vcc3v3_sys>; ++ vcc6-supply = <&vcc3v3_sys>; ++ vcc7-supply = <&vcc3v3_sys>; ++ vcc8-supply = <&vcc3v3_sys>; ++ vcc9-supply = <&vcc3v3_sys>; ++ vcc10-supply = <&vcc3v3_sys>; ++ vcc11-supply = <&vcc3v3_sys>; ++ vcc12-supply = <&vcc3v3_sys>; ++ vddio-supply = <&vcc1v8_pmu>; ++ ++ regulators { ++ vdd_log: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <750000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-name = "vdd_log"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <900000>; ++ }; ++ }; ++ ++ vdd_cpu_l: DCDC_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <750000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-name = "vdd_cpu_l"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc_ddr"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_1v8: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc_1v8"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vcc1v8_dvp: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc1v8_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc3v0_tp: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-name = "vcc3v0_tp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc1v8_pmu: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc1v8_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vcc_sd: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-name = "vcc_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ ++ vcca3v0_codec: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-name = "vcca3v0_codec"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_1v5: LDO_REG6 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ regulator-name = "vcc_1v5"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1500000>; ++ }; ++ }; ++ ++ vcca1v8_codec: LDO_REG7 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcca1v8_codec"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_3v0: LDO_REG8 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-name = "vcc_3v0"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ ++ vcc3v3_s3: SWITCH_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc3v3_s3"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc3v3_s0: SWITCH_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc3v3_s0"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ }; ++ }; ++}; ++ ++&pwm3 { ++ status = "okay"; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm3a_pin_pull_down>; ++}; ++ ++&u2phy0_otg { ++ rockchip,utmi-avalid; ++}; ++ ++&i2c6 { ++ status = "okay"; ++ fusb0: fusb30x@22 { ++ compatible = "fairchild,fusb302"; ++ reg = <0x22>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&fusb0_int>; ++ vbus-5v-gpios = <&gpio1 3 GPIO_ACTIVE_HIGH>; ++ int-n-gpios = <&gpio1 2 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-evb-rev3-android-edp.dts b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev3-android-edp.dts +new file mode 100755 +index 000000000000..7f730f5633dd +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev3-android-edp.dts +@@ -0,0 +1,126 @@ ++/* ++ * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This file is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This file is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++/dts-v1/; ++ ++#include "rk3399-evb-rev3.dtsi" ++#include "rk3399-android.dtsi" ++ ++/ { ++ model = "Rockchip RK3399 Evaluation Board v3 edp (Android)"; ++ compatible = "rockchip,android", "rockchip,rk3399-evb-rev3", "rockchip,rk3399"; ++ ++ edp_panel: edp-panel { ++ compatible = "lg,lp079qx1-sp0v", "panel-simple"; ++ backlight = <&backlight>; ++ power-supply = <&vcc3v3_s0>; ++ enable-gpios = <&gpio1 13 GPIO_ACTIVE_HIGH>; ++ ++ ports { ++ panel_in_edp: endpoint { ++ remote-endpoint = <&edp_out_panel>; ++ }; ++ }; ++ }; ++}; ++ ++&i2s2 { ++ status = "okay"; ++}; ++ ++>9xx { ++ status = "disabled"; ++}; ++ ++&i2c4 { ++ gsl3673: gsl3673@40 { ++ compatible = "GSL,GSL3673"; ++ reg = <0x40>; ++ screen_max_x = <1536>; ++ screen_max_y = <2048>; ++ irq_gpio_number = <&gpio1 20 IRQ_TYPE_LEVEL_LOW>; ++ rst_gpio_number = <&gpio4 22 GPIO_ACTIVE_HIGH>; ++ }; ++}; ++ ++&edp { ++ force-hpd; ++ status = "okay"; ++ ++ ports { ++ edp_out: port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ edp_out_panel: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&panel_in_edp>; ++ }; ++ }; ++ }; ++}; ++ ++&route_edp { ++ status = "okay"; ++}; ++ ++&firmware_android { ++ compatible = "android,firmware"; ++ fstab { ++ compatible = "android,fstab"; ++ system { ++ compatible = "android,system"; ++ dev = "/dev/block/by-name/system"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,verify"; ++ }; ++ vendor { ++ compatible = "android,vendor"; ++ dev = "/dev/block/by-name/vendor"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,verify"; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-evb-rev3-android-lp4.dts b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev3-android-lp4.dts +new file mode 100755 +index 000000000000..85f4356a0030 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev3-android-lp4.dts +@@ -0,0 +1,233 @@ ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++ */ ++ ++/dts-v1/; ++ ++#include "rk3399-evb-rev3.dtsi" ++#include "rk3399-android.dtsi" ++ ++/ { ++ model = "Rockchip RK3399 Evaluation Board v3 (Android) LPDDR4"; ++ compatible = "rockchip,android", "rockchip,rk3399-evb-rev3-android-lp4", "rockchip,rk3399"; ++ ++ /* first 64k(0xff8c0000~0xff8d0000) for ddr and suspend */ ++ iram: sram@ff8d0000 { ++ compatible = "mmio-sram"; ++ reg = <0x0 0xff8d0000 0x0 0x20000>; /* 128k */ ++ }; ++}; ++ ++&dmac_bus { ++ iram = <&iram>; ++ rockchip,force-iram; ++}; ++ ++&dsi { ++ status = "okay"; ++ ++ panel@0 { ++ compatible ="simple-panel-dsi"; ++ reg = <0>; ++ backlight = <&backlight>; ++ enable-gpios = <&gpio1 13 GPIO_ACTIVE_HIGH>; ++ ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <160000000>; ++ hactive = <1200>; ++ vactive = <1920>; ++ hback-porch = <21>; ++ hfront-porch = <120>; ++ vback-porch = <18>; ++ vfront-porch = <21>; ++ hsync-len = <20>; ++ vsync-len = <3>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ dsp_lut: dsp-lut { ++ gamma-lut = < ++ 0x00000000 0x00010101 0x00020202 0x00030303 0x00040404 0x00050505 0x00060606 0x00070707 ++ 0x00080808 0x00090909 0x000a0a0a 0x000b0b0b 0x000c0c0c 0x000d0d0d 0x000e0e0e 0x000f0f0f ++ 0x00101010 0x00111111 0x00121212 0x00131313 0x00141414 0x00151515 0x00161616 0x00171717 ++ 0x00181818 0x00191919 0x001a1a1a 0x001b1b1b 0x001c1c1c 0x001d1d1d 0x001e1e1e 0x001f1f1f ++ 0x00202020 0x00212121 0x00222222 0x00232323 0x00242424 0x00252525 0x00262626 0x00272727 ++ 0x00282828 0x00292929 0x002a2a2a 0x002b2b2b 0x002c2c2c 0x002d2d2d 0x002e2e2e 0x002f2f2f ++ 0x00303030 0x00313131 0x00323232 0x00333333 0x00343434 0x00353535 0x00363636 0x00373737 ++ 0x00383838 0x00393939 0x003a3a3a 0x003b3b3b 0x003c3c3c 0x003d3d3d 0x003e3e3e 0x003f3f3f ++ 0x00404040 0x00414141 0x00424242 0x00434343 0x00444444 0x00454545 0x00464646 0x00474747 ++ 0x00484848 0x00494949 0x004a4a4a 0x004b4b4b 0x004c4c4c 0x004d4d4d 0x004e4e4e 0x004f4f4f ++ 0x00505050 0x00515151 0x00525252 0x00535353 0x00545454 0x00555555 0x00565656 0x00575757 ++ 0x00585858 0x00595959 0x005a5a5a 0x005b5b5b 0x005c5c5c 0x005d5d5d 0x005e5e5e 0x005f5f5f ++ 0x00606060 0x00616161 0x00626262 0x00636363 0x00646464 0x00656565 0x00666666 0x00676767 ++ 0x00686868 0x00696969 0x006a6a6a 0x006b6b6b 0x006c6c6c 0x006d6d6d 0x006e6e6e 0x006f6f6f ++ 0x00707070 0x00717171 0x00727272 0x00737373 0x00747474 0x00757575 0x00767676 0x00777777 ++ 0x00787878 0x00797979 0x007a7a7a 0x007b7b7b 0x007c7c7c 0x007d7d7d 0x007e7e7e 0x007f7f7f ++ 0x00808080 0x00818181 0x00828282 0x00838383 0x00848484 0x00858585 0x00868686 0x00878787 ++ 0x00888888 0x00898989 0x008a8a8a 0x008b8b8b 0x008c8c8c 0x008d8d8d 0x008e8e8e 0x008f8f8f ++ 0x00909090 0x00919191 0x00929292 0x00939393 0x00949494 0x00959595 0x00969696 0x00979797 ++ 0x00989898 0x00999999 0x009a9a9a 0x009b9b9b 0x009c9c9c 0x009d9d9d 0x009e9e9e 0x009f9f9f ++ 0x00a0a0a0 0x00a1a1a1 0x00a2a2a2 0x00a3a3a3 0x00a4a4a4 0x00a5a5a5 0x00a6a6a6 0x00a7a7a7 ++ 0x00a8a8a8 0x00a9a9a9 0x00aaaaaa 0x00ababab 0x00acacac 0x00adadad 0x00aeaeae 0x00afafaf ++ 0x00b0b0b0 0x00b1b1b1 0x00b2b2b2 0x00b3b3b3 0x00b4b4b4 0x00b5b5b5 0x00b6b6b6 0x00b7b7b7 ++ 0x00b8b8b8 0x00b9b9b9 0x00bababa 0x00bbbbbb 0x00bcbcbc 0x00bdbdbd 0x00bebebe 0x00bfbfbf ++ 0x00c0c0c0 0x00c1c1c1 0x00c2c2c2 0x00c3c3c3 0x00c4c4c4 0x00c5c5c5 0x00c6c6c6 0x00c7c7c7 ++ 0x00c8c8c8 0x00c9c9c9 0x00cacaca 0x00cbcbcb 0x00cccccc 0x00cdcdcd 0x00cecece 0x00cfcfcf ++ 0x00d0d0d0 0x00d1d1d1 0x00d2d2d2 0x00d3d3d3 0x00d4d4d4 0x00d5d5d5 0x00d6d6d6 0x00d7d7d7 ++ 0x00d8d8d8 0x00d9d9d9 0x00dadada 0x00dbdbdb 0x00dcdcdc 0x00dddddd 0x00dedede 0x00dfdfdf ++ 0x00e0e0e0 0x00e1e1e1 0x00e2e2e2 0x00e3e3e3 0x00e4e4e4 0x00e5e5e5 0x00e6e6e6 0x00e7e7e7 ++ 0x00e8e8e8 0x00e9e9e9 0x00eaeaea 0x00ebebeb 0x00ececec 0x00ededed 0x00eeeeee 0x00efefef ++ 0x00f0f0f0 0x00f1f1f1 0x00f2f2f2 0x00f3f3f3 0x00f4f4f4 0x00f5f5f5 0x00f6f6f6 0x00f7f7f7 ++ 0x00f8f8f8 0x00f9f9f9 0x00fafafa 0x00fbfbfb 0x00fcfcfc 0x00fdfdfd 0x00fefefe 0x00ffffff>; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ panel_in_dsi: endpoint { ++ remote-endpoint = <&dsi_out_panel>; ++ }; ++ }; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@1 { ++ reg = <1>; ++ dsi_out_panel: endpoint { ++ remote-endpoint = <&panel_in_dsi>; ++ }; ++ }; ++ }; ++}; ++ ++&cdn_dp { ++ extcon = <&fusb0>, <&fusb1>; ++ status = "okay"; ++}; ++ ++&route_dsi { ++ status = "okay"; ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&dmc { ++ status = "okay"; ++ center-supply = <&vdd_center>; ++ system-status-freq = < ++ /*system status freq(KHz)*/ ++ SYS_STATUS_NORMAL 856000 ++ SYS_STATUS_REBOOT 416000 ++ SYS_STATUS_SUSPEND 416000 ++ SYS_STATUS_VIDEO_1080P 416000 ++ SYS_STATUS_VIDEO_4K 856000 ++ SYS_STATUS_VIDEO_4K_10B 856000 ++ SYS_STATUS_PERFORMANCE 856000 ++ SYS_STATUS_BOOST 856000 ++ SYS_STATUS_DUALVIEW 856000 ++ SYS_STATUS_ISP 856000 ++ >; ++ vop-bw-dmc-freq = < ++ /* min_bw(MB/s) max_bw(MB/s) freq(KHz) */ ++ 0 577 416000 ++ 578 99999 856000 ++ >; ++ auto-min-freq = <416000>; ++ auto-freq-en = <1>; ++}; ++ ++&dmc_opp_table { ++ compatible = "operating-points-v2"; ++ ++ opp-200000000 { ++ opp-hz = /bits/ 64 <200000000>; ++ opp-microvolt = <900000>; ++ status = "disabled"; ++ }; ++ opp-300000000 { ++ opp-hz = /bits/ 64 <300000000>; ++ opp-microvolt = <900000>; ++ status = "disabled"; ++ }; ++ opp-400000000 { ++ opp-hz = /bits/ 64 <400000000>; ++ opp-microvolt = <900000>; ++ status = "disabled"; ++ }; ++ opp-416000000 { ++ opp-hz = /bits/ 64 <416000000>; ++ opp-microvolt = <900000>; ++ }; ++ opp-528000000 { ++ opp-hz = /bits/ 64 <528000000>; ++ opp-microvolt = <900000>; ++ status = "disabled"; ++ }; ++ opp-600000000 { ++ opp-hz = /bits/ 64 <600000000>; ++ opp-microvolt = <900000>; ++ status = "disabled"; ++ }; ++ opp-800000000 { ++ opp-hz = /bits/ 64 <800000000>; ++ opp-microvolt = <900000>; ++ status = "disabled"; ++ }; ++ opp-856000000 { ++ opp-hz = /bits/ 64 <856000000>; ++ opp-microvolt = <900000>; ++ }; ++ opp-928000000 { ++ opp-hz = /bits/ 64 <928000000>; ++ opp-microvolt = <900000>; ++ status = "disabled"; ++ }; ++ opp-1056000000 { ++ opp-hz = /bits/ 64 <1056000000>; ++ opp-microvolt = <900000>; ++ status = "disabled"; ++ }; ++}; ++ ++&firmware_android { ++ compatible = "android,firmware"; ++ fstab { ++ compatible = "android,fstab"; ++ system { ++ compatible = "android,system"; ++ dev = "/dev/block/by-name/system"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,verify"; ++ }; ++ vendor { ++ compatible = "android,vendor"; ++ dev = "/dev/block/by-name/vendor"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,verify"; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-evb-rev3-android-mipi-edp.dts b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev3-android-mipi-edp.dts +new file mode 100755 +index 000000000000..fffd92f00ff4 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev3-android-mipi-edp.dts +@@ -0,0 +1,300 @@ ++/* ++ * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This file is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This file is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++/dts-v1/; ++ ++#include "rk3399-evb-rev3.dtsi" ++#include "rk3399-android.dtsi" ++ ++/ { ++ model = "Rockchip RK3399 Evaluation Board v3 (Android)"; ++ compatible = "rockchip,android", "rockchip,rk3399-evb-rev3", "rockchip,rk3399"; ++ ++ edp_panel: edp-panel { ++ compatible = "lg,lp079qx1-sp0v", "panel-simple"; ++ backlight = <&backlight1>; ++ power-supply = <&vcc3v3_s0>; ++ ++ ports { ++ panel_in_edp: endpoint { ++ remote-endpoint = <&edp_out_panel>; ++ }; ++ }; ++ ++ disp_timings: display-timings { ++ native-mode = <&F402>; ++ ++ F402: timing0 { ++ clock-frequency = <200000000>; ++ hactive = <1536>; ++ vactive = <2048>; ++ hfront-porch = <12>; ++ hsync-len = <16>; ++ hback-porch = <48>; ++ vfront-porch = <8>; ++ vsync-len = <4>; ++ vback-porch = <8>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ }; ++ ++ backlight1: backlight1 { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm1 0 25000 0>; ++ brightness-levels = < ++ 0 1 2 3 4 5 6 7 ++ 8 9 10 11 12 13 14 15 ++ 16 17 18 19 20 21 22 23 ++ 24 25 26 27 28 29 30 31 ++ 32 33 34 35 36 37 38 39 ++ 40 41 42 43 44 45 46 47 ++ 48 49 50 51 52 53 54 55 ++ 56 57 58 59 60 61 62 63 ++ 64 65 66 67 68 69 70 71 ++ 72 73 74 75 76 77 78 79 ++ 80 81 82 83 84 85 86 87 ++ 88 89 90 91 92 93 94 95 ++ 96 97 98 99 100 101 102 103 ++ 104 105 106 107 108 109 110 111 ++ 112 113 114 115 116 117 118 119 ++ 120 121 122 123 124 125 126 127 ++ 128 129 130 131 132 133 134 135 ++ 136 137 138 139 140 141 142 143 ++ 144 145 146 147 148 149 150 151 ++ 152 153 154 155 156 157 158 159 ++ 160 161 162 163 164 165 166 167 ++ 168 169 170 171 172 173 174 175 ++ 176 177 178 179 180 181 182 183 ++ 184 185 186 187 188 189 190 191 ++ 192 193 194 195 196 197 198 199 ++ 200 201 202 203 204 205 206 207 ++ 208 209 210 211 212 213 214 215 ++ 216 217 218 219 220 221 222 223 ++ 224 225 226 227 228 229 230 231 ++ 232 233 234 235 236 237 238 239 ++ 240 241 242 243 244 245 246 247 ++ 248 249 250 251 252 253 254 255>; ++ default-brightness-level = <200>; ++ }; ++}; ++ ++&pwm1 { ++ status = "okay"; ++}; ++ ++&backlight1 { ++ status = "okay"; ++}; ++ ++&dsi_in_vopl { ++ status = "disabled"; ++}; ++ ++&dsi_in_vopb { ++ status = "okay"; ++}; ++ ++&edp_in_vopl { ++ status = "okay"; ++}; ++ ++&edp_in_vopb { ++ status = "disabled"; ++}; ++ ++&dsi { ++ status = "okay"; ++ ++ panel@0 { ++ compatible ="simple-panel-dsi"; ++ reg = <0>; ++ backlight = <&backlight>; ++ enable-gpios = <&gpio1 13 GPIO_ACTIVE_HIGH>; ++ ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <160000000>; ++ hactive = <1200>; ++ vactive = <1920>; ++ hback-porch = <21>; ++ hfront-porch = <120>; ++ vback-porch = <18>; ++ vfront-porch = <21>; ++ hsync-len = <20>; ++ vsync-len = <3>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ dsp_lut: dsp-lut { ++ gamma-lut = < ++ 0x00000000 0x00010101 0x00020202 0x00030303 0x00040404 0x00050505 0x00060606 0x00070707 ++ 0x00080808 0x00090909 0x000a0a0a 0x000b0b0b 0x000c0c0c 0x000d0d0d 0x000e0e0e 0x000f0f0f ++ 0x00101010 0x00111111 0x00121212 0x00131313 0x00141414 0x00151515 0x00161616 0x00171717 ++ 0x00181818 0x00191919 0x001a1a1a 0x001b1b1b 0x001c1c1c 0x001d1d1d 0x001e1e1e 0x001f1f1f ++ 0x00202020 0x00212121 0x00222222 0x00232323 0x00242424 0x00252525 0x00262626 0x00272727 ++ 0x00282828 0x00292929 0x002a2a2a 0x002b2b2b 0x002c2c2c 0x002d2d2d 0x002e2e2e 0x002f2f2f ++ 0x00303030 0x00313131 0x00323232 0x00333333 0x00343434 0x00353535 0x00363636 0x00373737 ++ 0x00383838 0x00393939 0x003a3a3a 0x003b3b3b 0x003c3c3c 0x003d3d3d 0x003e3e3e 0x003f3f3f ++ 0x00404040 0x00414141 0x00424242 0x00434343 0x00444444 0x00454545 0x00464646 0x00474747 ++ 0x00484848 0x00494949 0x004a4a4a 0x004b4b4b 0x004c4c4c 0x004d4d4d 0x004e4e4e 0x004f4f4f ++ 0x00505050 0x00515151 0x00525252 0x00535353 0x00545454 0x00555555 0x00565656 0x00575757 ++ 0x00585858 0x00595959 0x005a5a5a 0x005b5b5b 0x005c5c5c 0x005d5d5d 0x005e5e5e 0x005f5f5f ++ 0x00606060 0x00616161 0x00626262 0x00636363 0x00646464 0x00656565 0x00666666 0x00676767 ++ 0x00686868 0x00696969 0x006a6a6a 0x006b6b6b 0x006c6c6c 0x006d6d6d 0x006e6e6e 0x006f6f6f ++ 0x00707070 0x00717171 0x00727272 0x00737373 0x00747474 0x00757575 0x00767676 0x00777777 ++ 0x00787878 0x00797979 0x007a7a7a 0x007b7b7b 0x007c7c7c 0x007d7d7d 0x007e7e7e 0x007f7f7f ++ 0x00808080 0x00818181 0x00828282 0x00838383 0x00848484 0x00858585 0x00868686 0x00878787 ++ 0x00888888 0x00898989 0x008a8a8a 0x008b8b8b 0x008c8c8c 0x008d8d8d 0x008e8e8e 0x008f8f8f ++ 0x00909090 0x00919191 0x00929292 0x00939393 0x00949494 0x00959595 0x00969696 0x00979797 ++ 0x00989898 0x00999999 0x009a9a9a 0x009b9b9b 0x009c9c9c 0x009d9d9d 0x009e9e9e 0x009f9f9f ++ 0x00a0a0a0 0x00a1a1a1 0x00a2a2a2 0x00a3a3a3 0x00a4a4a4 0x00a5a5a5 0x00a6a6a6 0x00a7a7a7 ++ 0x00a8a8a8 0x00a9a9a9 0x00aaaaaa 0x00ababab 0x00acacac 0x00adadad 0x00aeaeae 0x00afafaf ++ 0x00b0b0b0 0x00b1b1b1 0x00b2b2b2 0x00b3b3b3 0x00b4b4b4 0x00b5b5b5 0x00b6b6b6 0x00b7b7b7 ++ 0x00b8b8b8 0x00b9b9b9 0x00bababa 0x00bbbbbb 0x00bcbcbc 0x00bdbdbd 0x00bebebe 0x00bfbfbf ++ 0x00c0c0c0 0x00c1c1c1 0x00c2c2c2 0x00c3c3c3 0x00c4c4c4 0x00c5c5c5 0x00c6c6c6 0x00c7c7c7 ++ 0x00c8c8c8 0x00c9c9c9 0x00cacaca 0x00cbcbcb 0x00cccccc 0x00cdcdcd 0x00cecece 0x00cfcfcf ++ 0x00d0d0d0 0x00d1d1d1 0x00d2d2d2 0x00d3d3d3 0x00d4d4d4 0x00d5d5d5 0x00d6d6d6 0x00d7d7d7 ++ 0x00d8d8d8 0x00d9d9d9 0x00dadada 0x00dbdbdb 0x00dcdcdc 0x00dddddd 0x00dedede 0x00dfdfdf ++ 0x00e0e0e0 0x00e1e1e1 0x00e2e2e2 0x00e3e3e3 0x00e4e4e4 0x00e5e5e5 0x00e6e6e6 0x00e7e7e7 ++ 0x00e8e8e8 0x00e9e9e9 0x00eaeaea 0x00ebebeb 0x00ececec 0x00ededed 0x00eeeeee 0x00efefef ++ 0x00f0f0f0 0x00f1f1f1 0x00f2f2f2 0x00f3f3f3 0x00f4f4f4 0x00f5f5f5 0x00f6f6f6 0x00f7f7f7 ++ 0x00f8f8f8 0x00f9f9f9 0x00fafafa 0x00fbfbfb 0x00fcfcfc 0x00fdfdfd 0x00fefefe 0x00ffffff>; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ panel_in_dsi: endpoint { ++ remote-endpoint = <&dsi_out_panel>; ++ }; ++ }; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@1 { ++ reg = <1>; ++ dsi_out_panel: endpoint { ++ remote-endpoint = <&panel_in_dsi>; ++ }; ++ }; ++ }; ++}; ++ ++&cdn_dp { ++ extcon = <&fusb0>, <&fusb1>; ++ status = "okay"; ++}; ++ ++&route_dsi { ++ status = "okay"; ++ connect = <&vopb_out_dsi>; ++}; ++ ++&edp { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&edp_hpd>; ++ ++ ports { ++ edp_out: port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ edp_out_panel: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&panel_in_edp>; ++ }; ++ }; ++ }; ++}; ++ ++&route_edp { ++ status = "okay"; ++ connect = <&vopl_out_edp>; ++}; ++ ++&firmware_android { ++ compatible = "android,firmware"; ++ fstab { ++ compatible = "android,fstab"; ++ system { ++ compatible = "android,system"; ++ dev = "/dev/block/by-name/system"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,verify"; ++ }; ++ vendor { ++ compatible = "android,vendor"; ++ dev = "/dev/block/by-name/vendor"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,verify"; ++ }; ++ }; ++}; ++ ++>9xx { ++ status = "disabled"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-evb-rev3-android.dts b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev3-android.dts +new file mode 100755 +index 000000000000..9ba1b0381fed +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev3-android.dts +@@ -0,0 +1,176 @@ ++/* ++ * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This file is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This file is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++/dts-v1/; ++ ++#include "rk3399-evb-rev3.dtsi" ++#include "rk3399-android.dtsi" ++ ++/ { ++ model = "Rockchip RK3399 Evaluation Board v3 (Android)"; ++ compatible = "rockchip,android", "rockchip,rk3399-evb-rev3", "rockchip,rk3399"; ++}; ++ ++&dsi { ++ status = "okay"; ++ ++ panel@0 { ++ compatible ="simple-panel-dsi"; ++ reg = <0>; ++ backlight = <&backlight>; ++ enable-gpios = <&gpio1 13 GPIO_ACTIVE_HIGH>; ++ ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <160000000>; ++ hactive = <1200>; ++ vactive = <1920>; ++ hback-porch = <21>; ++ hfront-porch = <120>; ++ vback-porch = <18>; ++ vfront-porch = <21>; ++ hsync-len = <20>; ++ vsync-len = <3>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ dsp_lut: dsp-lut { ++ gamma-lut = < ++ 0x00000000 0x00010101 0x00020202 0x00030303 0x00040404 0x00050505 0x00060606 0x00070707 ++ 0x00080808 0x00090909 0x000a0a0a 0x000b0b0b 0x000c0c0c 0x000d0d0d 0x000e0e0e 0x000f0f0f ++ 0x00101010 0x00111111 0x00121212 0x00131313 0x00141414 0x00151515 0x00161616 0x00171717 ++ 0x00181818 0x00191919 0x001a1a1a 0x001b1b1b 0x001c1c1c 0x001d1d1d 0x001e1e1e 0x001f1f1f ++ 0x00202020 0x00212121 0x00222222 0x00232323 0x00242424 0x00252525 0x00262626 0x00272727 ++ 0x00282828 0x00292929 0x002a2a2a 0x002b2b2b 0x002c2c2c 0x002d2d2d 0x002e2e2e 0x002f2f2f ++ 0x00303030 0x00313131 0x00323232 0x00333333 0x00343434 0x00353535 0x00363636 0x00373737 ++ 0x00383838 0x00393939 0x003a3a3a 0x003b3b3b 0x003c3c3c 0x003d3d3d 0x003e3e3e 0x003f3f3f ++ 0x00404040 0x00414141 0x00424242 0x00434343 0x00444444 0x00454545 0x00464646 0x00474747 ++ 0x00484848 0x00494949 0x004a4a4a 0x004b4b4b 0x004c4c4c 0x004d4d4d 0x004e4e4e 0x004f4f4f ++ 0x00505050 0x00515151 0x00525252 0x00535353 0x00545454 0x00555555 0x00565656 0x00575757 ++ 0x00585858 0x00595959 0x005a5a5a 0x005b5b5b 0x005c5c5c 0x005d5d5d 0x005e5e5e 0x005f5f5f ++ 0x00606060 0x00616161 0x00626262 0x00636363 0x00646464 0x00656565 0x00666666 0x00676767 ++ 0x00686868 0x00696969 0x006a6a6a 0x006b6b6b 0x006c6c6c 0x006d6d6d 0x006e6e6e 0x006f6f6f ++ 0x00707070 0x00717171 0x00727272 0x00737373 0x00747474 0x00757575 0x00767676 0x00777777 ++ 0x00787878 0x00797979 0x007a7a7a 0x007b7b7b 0x007c7c7c 0x007d7d7d 0x007e7e7e 0x007f7f7f ++ 0x00808080 0x00818181 0x00828282 0x00838383 0x00848484 0x00858585 0x00868686 0x00878787 ++ 0x00888888 0x00898989 0x008a8a8a 0x008b8b8b 0x008c8c8c 0x008d8d8d 0x008e8e8e 0x008f8f8f ++ 0x00909090 0x00919191 0x00929292 0x00939393 0x00949494 0x00959595 0x00969696 0x00979797 ++ 0x00989898 0x00999999 0x009a9a9a 0x009b9b9b 0x009c9c9c 0x009d9d9d 0x009e9e9e 0x009f9f9f ++ 0x00a0a0a0 0x00a1a1a1 0x00a2a2a2 0x00a3a3a3 0x00a4a4a4 0x00a5a5a5 0x00a6a6a6 0x00a7a7a7 ++ 0x00a8a8a8 0x00a9a9a9 0x00aaaaaa 0x00ababab 0x00acacac 0x00adadad 0x00aeaeae 0x00afafaf ++ 0x00b0b0b0 0x00b1b1b1 0x00b2b2b2 0x00b3b3b3 0x00b4b4b4 0x00b5b5b5 0x00b6b6b6 0x00b7b7b7 ++ 0x00b8b8b8 0x00b9b9b9 0x00bababa 0x00bbbbbb 0x00bcbcbc 0x00bdbdbd 0x00bebebe 0x00bfbfbf ++ 0x00c0c0c0 0x00c1c1c1 0x00c2c2c2 0x00c3c3c3 0x00c4c4c4 0x00c5c5c5 0x00c6c6c6 0x00c7c7c7 ++ 0x00c8c8c8 0x00c9c9c9 0x00cacaca 0x00cbcbcb 0x00cccccc 0x00cdcdcd 0x00cecece 0x00cfcfcf ++ 0x00d0d0d0 0x00d1d1d1 0x00d2d2d2 0x00d3d3d3 0x00d4d4d4 0x00d5d5d5 0x00d6d6d6 0x00d7d7d7 ++ 0x00d8d8d8 0x00d9d9d9 0x00dadada 0x00dbdbdb 0x00dcdcdc 0x00dddddd 0x00dedede 0x00dfdfdf ++ 0x00e0e0e0 0x00e1e1e1 0x00e2e2e2 0x00e3e3e3 0x00e4e4e4 0x00e5e5e5 0x00e6e6e6 0x00e7e7e7 ++ 0x00e8e8e8 0x00e9e9e9 0x00eaeaea 0x00ebebeb 0x00ececec 0x00ededed 0x00eeeeee 0x00efefef ++ 0x00f0f0f0 0x00f1f1f1 0x00f2f2f2 0x00f3f3f3 0x00f4f4f4 0x00f5f5f5 0x00f6f6f6 0x00f7f7f7 ++ 0x00f8f8f8 0x00f9f9f9 0x00fafafa 0x00fbfbfb 0x00fcfcfc 0x00fdfdfd 0x00fefefe 0x00ffffff>; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ panel_in_dsi: endpoint { ++ remote-endpoint = <&dsi_out_panel>; ++ }; ++ }; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@1 { ++ reg = <1>; ++ dsi_out_panel: endpoint { ++ remote-endpoint = <&panel_in_dsi>; ++ }; ++ }; ++ }; ++}; ++ ++&cdn_dp { ++ extcon = <&fusb0>, <&fusb1>; ++ status = "okay"; ++}; ++ ++&route_dsi { ++ status = "okay"; ++}; ++ ++&firmware_android { ++ compatible = "android,firmware"; ++ fstab { ++ compatible = "android,fstab"; ++ system { ++ compatible = "android,system"; ++ dev = "/dev/block/by-name/system"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,verify"; ++ }; ++ vendor { ++ compatible = "android,vendor"; ++ dev = "/dev/block/by-name/vendor"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,verify"; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-evb-rev3-cros.dts b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev3-cros.dts +new file mode 100755 +index 000000000000..e1bb5f129680 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev3-cros.dts +@@ -0,0 +1,51 @@ ++/* ++ * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This file is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This file is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++/dts-v1/; ++ ++#include "rk3399-evb-rev3.dtsi" ++#include "rk3399-evb-cros.dtsi" ++ ++/ { ++ model = "Rockchip RK3399 Evaluation Board v3 (Chrome OS)"; ++ compatible = "google,rk3399evb-rev3", "rockchip,rk3399-evb-rev3", "rockchip,rk3399"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-evb-rev3.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev3.dtsi +new file mode 100755 +index 000000000000..e34379ded3a5 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev3.dtsi +@@ -0,0 +1,353 @@ ++/* ++ * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This file is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This file is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++#include "rk3399-evb.dtsi" ++#include ++ ++/ { ++ compatible = "rockchip,rk3399-evb-rev3", "rockchip,rk3399"; ++ ++ vcc5v0_sys: vcc5v0-sys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc5v0_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ }; ++ ++ vdd_center: vdd-center { ++ compatible = "pwm-regulator"; ++ rockchip,pwm_id = <2>; ++ rockchip,pwm_voltage = <900000>; ++ pwms = <&pwm2 0 25000 1>; ++ regulator-name = "vdd_center"; ++ regulator-min-microvolt = <800000>; ++ regulator-max-microvolt = <1400000>; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ xin32k: xin32k { ++ compatible = "fixed-clock"; ++ clock-frequency = <32768>; ++ clock-output-names = "xin32k"; ++ #clock-cells = <0>; ++ }; ++ ++ rockchip_suspend: rockchip-suspend { ++ compatible = "rockchip,pm-rk3399"; ++ status = "okay"; ++ rockchip,sleep-debug-en = <1>; ++ rockchip,wakeup-config = < ++ (0 ++ | RKPM_GPIO_WKUP_EN ++ ) ++ >; ++ rockchip,pwm-regulator-config = < ++ (0 ++ | PWM2_REGULATOR_EN ++ ) ++ >; ++ rockchip,power-ctrl = ++ <&gpio1 17 GPIO_ACTIVE_HIGH>; ++ }; ++}; ++ ++&i2c0 { ++ fusb1: fusb30x@22 { ++ compatible = "fairchild,fusb302"; ++ reg = <0x22>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&fusb1_int>; ++ vbus-5v-gpios = <&gpio1 4 GPIO_ACTIVE_HIGH>; ++ int-n-gpios = <&gpio1 24 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ vdd_cpu_b: syr827@40 { ++ compatible = "silergy,syr827"; ++ reg = <0x40>; ++ vin-supply = <&vcc5v0_sys>; ++ regulator-compatible = "fan53555-reg"; ++ pinctrl-0 = <&vsel1_gpio>; ++ vsel-gpios = <&gpio1 17 GPIO_ACTIVE_HIGH>; ++ regulator-name = "vdd_cpu_b"; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1500000>; ++ regulator-ramp-delay = <1000>; ++ fcs,suspend-voltage-selector = <1>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-state = <3>; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdd_gpu: syr828@41 { ++ compatible = "silergy,syr828"; ++ reg = <0x41>; ++ vin-supply = <&vcc5v0_sys>; ++ regulator-compatible = "fan53555-reg"; ++ pinctrl-0 = <&vsel2_gpio>; ++ vsel-gpios = <&gpio1 14 GPIO_ACTIVE_HIGH>; ++ regulator-name = "vdd_gpu"; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1500000>; ++ regulator-ramp-delay = <1000>; ++ fcs,suspend-voltage-selector = <1>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-state = <3>; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ rk808: pmic@1b { ++ compatible = "rockchip,rk808"; ++ reg = <0x1b>; ++ interrupt-parent = <&gpio1>; ++ interrupts = <21 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pmic_int_l &pmic_dvs2>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ ++ vcc1-supply = <&vcc3v3_sys>; ++ vcc2-supply = <&vcc3v3_sys>; ++ vcc3-supply = <&vcc3v3_sys>; ++ vcc4-supply = <&vcc3v3_sys>; ++ vcc6-supply = <&vcc3v3_sys>; ++ vcc7-supply = <&vcc3v3_sys>; ++ vcc8-supply = <&vcc3v3_sys>; ++ vcc9-supply = <&vcc3v3_sys>; ++ vcc10-supply = <&vcc3v3_sys>; ++ vcc11-supply = <&vcc3v3_sys>; ++ vcc12-supply = <&vcc3v3_sys>; ++ vddio-supply = <&vcc1v8_pmu>; ++ ++ regulators { ++ vdd_log: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <750000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-name = "vdd_log"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <900000>; ++ }; ++ }; ++ ++ vdd_cpu_l: DCDC_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <750000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-name = "vdd_cpu_l"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc_ddr"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_1v8: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc_1v8"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vcc1v8_dvp: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc1v8_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc3v0_tp: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-name = "vcc3v0_tp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc1v8_pmu: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc1v8_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vcc_sd: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-name = "vcc_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ ++ vcca3v0_codec: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-name = "vcca3v0_codec"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_1v5: LDO_REG6 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ regulator-name = "vcc_1v5"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1500000>; ++ }; ++ }; ++ ++ vcca1v8_codec: LDO_REG7 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcca1v8_codec"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_3v0: LDO_REG8 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-name = "vcc_3v0"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ ++ vcc3v3_s3: SWITCH_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc3v3_s3"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc3v3_s0: SWITCH_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc3v3_s0"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ }; ++ }; ++}; ++ ++&es8316 { ++ reg = <0x11>; ++}; ++ ++&i2c6 { ++ status = "okay"; ++ fusb0: fusb30x@22 { ++ compatible = "fairchild,fusb302"; ++ reg = <0x22>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&fusb0_int>; ++ vbus-5v-gpios = <&gpio1 3 GPIO_ACTIVE_HIGH>; ++ int-n-gpios = <&gpio1 2 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++}; ++ ++&pwm2 { ++ status = "okay"; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm2_pin_pull_down>; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-evb.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-evb.dtsi +new file mode 100755 +index 000000000000..aa8ea436ae9b +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-evb.dtsi +@@ -0,0 +1,646 @@ ++/* ++ * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This file is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This file is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++#include ++#include ++#include "rk3399.dtsi" ++#include "rk3399-opp.dtsi" ++ ++/ { ++ compatible = "rockchip,rk3399-evb", "rockchip,rk3399"; ++ ++ adc_keys: adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 1>; ++ io-channel-names = "buttons"; ++ keyup-threshold-microvolt = <1800000>; ++ poll-interval = <100>; ++ ++ vol-up-key { ++ label = "volume up"; ++ linux,code = ; ++ press-threshold-microvolt = <1750>; ++ }; ++ ++ vol-down-key { ++ label = "volume down"; ++ linux,code = ; ++ press-threshold-microvolt = <297500>; ++ }; ++ ++ menu-key { ++ label = "menu"; ++ linux,code = ; ++ press-threshold-microvolt = <1305500>; ++ }; ++ ++ home-key { ++ label = "home"; ++ linux,code = ; ++ press-threshold-microvolt = <621250>; ++ }; ++ ++ back-key { ++ label = "back"; ++ linux,code = ; ++ press-threshold-microvolt = <980000>; ++ }; ++ ++ camera-key { ++ label = "camera"; ++ linux,code = ; ++ press-threshold-microvolt = <787500>; ++ }; ++ }; ++ ++ vcc3v3_sys: vcc3v3-sys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc3v3_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ }; ++ ++ vcc5v0_host: vcc5v0-host-regulator { ++ compatible = "regulator-fixed"; ++ enable-active-high; ++ gpio = <&gpio4 25 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&host_vbus_drv>; ++ regulator-name = "vcc5v0_host"; ++ regulator-always-on; ++ }; ++ ++ backlight: backlight { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm0 0 25000 0>; ++ brightness-levels = < ++ 0 1 2 3 4 5 6 7 ++ 8 9 10 11 12 13 14 15 ++ 16 17 18 19 20 21 22 23 ++ 24 25 26 27 28 29 30 31 ++ 32 33 34 35 36 37 38 39 ++ 40 41 42 43 44 45 46 47 ++ 48 49 50 51 52 53 54 55 ++ 56 57 58 59 60 61 62 63 ++ 64 65 66 67 68 69 70 71 ++ 72 73 74 75 76 77 78 79 ++ 80 81 82 83 84 85 86 87 ++ 88 89 90 91 92 93 94 95 ++ 96 97 98 99 100 101 102 103 ++ 104 105 106 107 108 109 110 111 ++ 112 113 114 115 116 117 118 119 ++ 120 121 122 123 124 125 126 127 ++ 128 129 130 131 132 133 134 135 ++ 136 137 138 139 140 141 142 143 ++ 144 145 146 147 148 149 150 151 ++ 152 153 154 155 156 157 158 159 ++ 160 161 162 163 164 165 166 167 ++ 168 169 170 171 172 173 174 175 ++ 176 177 178 179 180 181 182 183 ++ 184 185 186 187 188 189 190 191 ++ 192 193 194 195 196 197 198 199 ++ 200 201 202 203 204 205 206 207 ++ 208 209 210 211 212 213 214 215 ++ 216 217 218 219 220 221 222 223 ++ 224 225 226 227 228 229 230 231 ++ 232 233 234 235 236 237 238 239 ++ 240 241 242 243 244 245 246 247 ++ 248 249 250 251 252 253 254 255>; ++ default-brightness-level = <200>; ++ }; ++ ++ clkin_gmac: external-gmac-clock { ++ compatible = "fixed-clock"; ++ clock-frequency = <125000000>; ++ clock-output-names = "clkin_gmac"; ++ #clock-cells = <0>; ++ }; ++ ++ vcc_phy: vcc-phy-regulator { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_phy"; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ es8316-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,es8316-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,widgets = ++ "Microphone", "Mic Jack", ++ "Headphone", "Headphone Jack"; ++ simple-audio-card,routing = ++ "Mic Jack", "MICBIAS1", ++ "IN1P", "Mic Jack", ++ "Headphone Jack", "HPOL", ++ "Headphone Jack", "HPOR"; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s0>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&es8316>; ++ }; ++ }; ++ ++ hdmi_sound: hdmi-sound { ++ status = "disabled"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,name = "rockchip,hdmi"; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s2>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&dw_hdmi_audio>; ++ }; ++ }; ++ ++ dw_hdmi_audio: dw-hdmi-audio { ++ status = "disabled"; ++ compatible = "rockchip,dw-hdmi-audio"; ++ #sound-dai-cells = <0>; ++ }; ++ ++ spdif_sound: spdif-sound { ++ status = "disabled"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,name = "ROCKCHIP,SPDIF"; ++ simple-audio-card,cpu { ++ sound-dai = <&spdif>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&spdif_out>; ++ }; ++ }; ++ ++ spdif_out: spdif-out { ++ status = "disabled"; ++ compatible = "linux,spdif-dit"; ++ #sound-dai-cells = <0>; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ clocks = <&rk808 1>; ++ clock-names = "ext_clock"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio0 10 GPIO_ACTIVE_LOW>; /* GPIO0_B2 */ ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ rockchip,grf = <&grf>; ++ wifi_chip_type = "ap6354"; ++ sdio_vref = <1800>; ++ WIFI,host_wake_irq = <&gpio0 3 GPIO_ACTIVE_HIGH>; /* GPIO0_a3 */ ++ status = "okay"; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&rk808 1>; ++ clock-names = "ext_clock"; ++ //wifi-bt-power-toggle; ++ uart_rts_gpios = <&gpio2 19 GPIO_ACTIVE_LOW>; /* GPIO2_C3 */ ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart0_rts>; ++ pinctrl-1 = <&uart0_gpios>; ++ //BT,power_gpio = <&gpio3 19 GPIO_ACTIVE_HIGH>; /* GPIOx_xx */ ++ BT,reset_gpio = <&gpio0 9 GPIO_ACTIVE_HIGH>; /* GPIO0_B1 */ ++ BT,wake_gpio = <&gpio2 26 GPIO_ACTIVE_HIGH>; /* GPIO2_D2 */ ++ BT,wake_host_irq = <&gpio0 4 GPIO_ACTIVE_HIGH>; /* GPIO0_A4 */ ++ status = "okay"; ++ }; ++ ++ test-power { ++ status = "okay"; ++ }; ++}; ++ ++&cpu_l0 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_l1 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_l2 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_l3 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_b0 { ++ cpu-supply = <&vdd_cpu_b>; ++}; ++ ++&cpu_b1 { ++ cpu-supply = <&vdd_cpu_b>; ++}; ++ ++&gpu { ++ status = "okay"; ++ mali-supply = <&vdd_gpu>; ++}; ++ ++&sdmmc { ++ clock-frequency = <150000000>; ++ clock-freq-min-max = <400000 150000000>; ++ no-sdio; ++ no-mmc; ++ bus-width = <4>; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ disable-wp; ++ num-slots = <1>; ++ //sd-uhs-sdr104; ++ vqmmc-supply = <&vcc_sd>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_cd &sdmmc_bus4>; ++ status = "okay"; ++}; ++ ++&sdio0 { ++ clock-frequency = <150000000>; ++ clock-freq-min-max = <200000 150000000>; ++ no-sd; ++ no-mmc; ++ bus-width = <4>; ++ disable-wp; ++ cap-sd-highspeed; ++ cap-sdio-irq; ++ keep-power-in-suspend; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ non-removable; ++ num-slots = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&emmc_phy { ++ status = "okay"; ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&dmc { ++ status = "okay"; ++ center-supply = <&vdd_center>; ++ upthreshold = <40>; ++ downdifferential = <20>; ++ system-status-freq = < ++ /*system status freq(KHz)*/ ++ SYS_STATUS_NORMAL 800000 ++ SYS_STATUS_REBOOT 528000 ++ SYS_STATUS_SUSPEND 200000 ++ SYS_STATUS_VIDEO_1080P 200000 ++ SYS_STATUS_VIDEO_4K 600000 ++ SYS_STATUS_VIDEO_4K_10B 800000 ++ SYS_STATUS_PERFORMANCE 800000 ++ SYS_STATUS_BOOST 400000 ++ SYS_STATUS_DUALVIEW 600000 ++ SYS_STATUS_ISP 600000 ++ >; ++ vop-bw-dmc-freq = < ++ /* min_bw(MB/s) max_bw(MB/s) freq(KHz) */ ++ 0 577 200000 ++ 578 1701 300000 ++ 1702 99999 400000 ++ >; ++ auto-min-freq = <200000>; ++}; ++ ++&sdhci { ++ bus-width = <8>; ++ mmc-hs400-1_8v; ++ no-sdio; ++ no-sd; ++ non-removable; ++ keep-power-in-suspend; ++ mmc-hs400-enhanced-strobe; ++ status = "okay"; ++}; ++ ++&i2s0 { ++ status = "okay"; ++ rockchip,i2s-broken-burst-len; ++ rockchip,playback-channels = <8>; ++ rockchip,capture-channels = <8>; ++ #sound-dai-cells = <0>; ++}; ++ ++&i2s2 { ++ #sound-dai-cells = <0>; ++}; ++ ++&spdif { ++ #sound-dai-cells = <0>; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <450>; ++ i2c-scl-falling-time-ns = <15>; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <300>; ++ i2c-scl-falling-time-ns = <15>; ++ ++ es8316: es8316@10 { ++ #sound-dai-cells = <0>; ++ compatible = "everest,es8316"; ++ reg = <0x10>; ++ clocks = <&cru SCLK_I2S_8CH_OUT>; ++ clock-names = "mclk"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s_8ch_mclk>; ++ spk-con-gpio = <&gpio0 11 GPIO_ACTIVE_HIGH>; ++ hp-det-gpio = <&gpio4 28 GPIO_ACTIVE_LOW>; ++ }; ++}; ++ ++&i2c4 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <600>; ++ i2c-scl-falling-time-ns = <20>; ++ ++ gt9xx: gt9xx@14 { ++ compatible = "goodix,gt9xx"; ++ reg = <0x14>; ++ touch-gpio = <&gpio1 20 IRQ_TYPE_LEVEL_LOW>; ++ reset-gpio = <&gpio4 22 GPIO_ACTIVE_HIGH>; ++ max-x = <1200>; ++ max-y = <1900>; ++ tp-size = <911>; ++ tp-supply = <&vcc3v0_tp>; ++ }; ++ ++ gsl3673: gsl3673@40 { ++ compatible = "GSL,GSL3673"; ++ reg = <0x40>; ++ screen_max_x = <1536>; ++ screen_max_y = <2048>; ++ irq_gpio_number = <&gpio1 20 IRQ_TYPE_LEVEL_LOW>; ++ rst_gpio_number = <&gpio4 22 GPIO_ACTIVE_HIGH>; ++ }; ++}; ++ ++&io_domains { ++ status = "okay"; ++ ++ bt656-supply = <&vcc1v8_dvp>; ++ audio-supply = <&vcca1v8_codec>; ++ sdmmc-supply = <&vcc_sd>; ++ gpio1830-supply = <&vcc_3v0>; ++}; ++ ++&pcie_phy { ++ status = "disabled"; ++}; ++ ++&pcie0 { ++ ep-gpios = <&gpio3 13 GPIO_ACTIVE_HIGH>; ++ num-lanes = <4>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pcie_clkreqn_cpm>; ++ status = "disabled"; ++}; ++ ++&tcphy0 { ++ extcon = <&fusb0>; ++ status = "okay"; ++}; ++ ++&tcphy1 { ++ extcon = <&fusb1>; ++ status = "okay"; ++}; ++ ++&tsadc { ++ rockchip,hw-tshut-mode = <1>; /* tshut mode 0:CRU 1:GPIO */ ++ rockchip,hw-tshut-polarity = <1>; /* tshut polarity 0:LOW 1:HIGH */ ++ status = "okay"; ++}; ++ ++&u2phy0 { ++ status = "okay"; ++ extcon = <&fusb0>; ++ ++ u2phy0_otg: otg-port { ++ status = "okay"; ++ }; ++ ++ u2phy0_host: host-port { ++ phy-supply = <&vcc5v0_host>; ++ status = "okay"; ++ }; ++}; ++ ++&u2phy1 { ++ status = "okay"; ++ extcon = <&fusb1>; ++ ++ u2phy1_otg: otg-port { ++ status = "okay"; ++ }; ++ ++ u2phy1_host: host-port { ++ phy-supply = <&vcc5v0_host>; ++ status = "okay"; ++ }; ++}; ++ ++&uart0 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart0_xfer &uart0_cts>; ++ status = "okay"; ++}; ++ ++&uart2 { ++ status = "okay"; ++}; ++ ++&usb_host0_ehci { ++ status = "okay"; ++}; ++ ++&usb_host0_ohci { ++ status = "okay"; ++}; ++ ++&usb_host1_ehci { ++ status = "okay"; ++}; ++ ++&usb_host1_ohci { ++ status = "okay"; ++}; ++ ++&usbdrd3_0 { ++ status = "okay"; ++}; ++ ++&usbdrd_dwc3_0 { ++ status = "okay"; ++ extcon = <&fusb0>; ++}; ++ ++&usbdrd3_1 { ++ status = "okay"; ++}; ++ ++&usbdrd_dwc3_1 { ++ status = "okay"; ++ extcon = <&fusb1>; ++}; ++ ++&pwm0 { ++ status = "okay"; ++}; ++ ++&gmac { ++ phy-supply = <&vcc_phy>; ++ phy-mode = "rgmii"; ++ clock_in_out = "input"; ++ snps,reset-gpio = <&gpio3 15 GPIO_ACTIVE_LOW>; ++ snps,reset-active-low; ++ snps,reset-delays-us = <0 10000 50000>; ++ assigned-clocks = <&cru SCLK_RMII_SRC>; ++ assigned-clock-parents = <&clkin_gmac>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&rgmii_pins>; ++ tx_delay = <0x28>; ++ rx_delay = <0x11>; ++ status = "okay"; ++}; ++ ++&saradc { ++ status = "okay"; ++ vref-supply = <&vcc_1v8>; ++}; ++ ++&pinctrl { ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <0 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart0_gpios: uart0-gpios { ++ rockchip,pins = <2 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ pmic { ++ pmic_int_l: pmic-int-l { ++ rockchip,pins = ++ <1 RK_PC5 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ pmic_dvs2: pmic-dvs2 { ++ rockchip,pins = ++ <1 RK_PC2 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ vsel1_gpio: vsel1-gpio { ++ rockchip,pins = ++ <1 RK_PC1 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ vsel2_gpio: vsel2-gpio { ++ rockchip,pins = ++ <1 RK_PB6 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ usb2 { ++ host_vbus_drv: host-vbus-drv { ++ rockchip,pins = ++ <4 RK_PD1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ fusb30x { ++ fusb0_int: fusb0-int { ++ rockchip,pins = <1 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ fusb1_int: fusb1-int { ++ rockchip,pins = <1 RK_PD0 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++}; ++ ++&pvtm { ++ status = "okay"; ++}; ++ ++&pmu_pvtm { ++ status = "okay"; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ pmu1830-supply = <&vcc1v8_pmu>; ++}; ++ +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-excavator-sapphire.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-excavator-sapphire.dtsi +new file mode 100755 +index 000000000000..adbda6ba4e0d +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-excavator-sapphire.dtsi +@@ -0,0 +1,324 @@ ++/* ++ * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This file is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This file is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++#include "rk3399-sapphire.dtsi" ++#include ++/ { ++ compatible = "rockchip,rk3399-sapphire-excavator", "rockchip,rk3399"; ++ ++ rt5651_sound: rt5651-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "realtek,rt5651-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,widgets = ++ "Microphone", "Mic Jack", ++ "Headphone", "Headphone Jack"; ++ simple-audio-card,routing = ++ "Mic Jack", "MICBIAS1", ++ "IN1P", "Mic Jack", ++ "Headphone Jack", "HPOL", ++ "Headphone Jack", "HPOR"; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s0>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&rt5651>; ++ }; ++ }; ++ ++ spdif-sound { ++ status = "okay"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,name = "ROCKCHIP,SPDIF"; ++ simple-audio-card,cpu { ++ sound-dai = <&spdif>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&spdif_out>; ++ }; ++ }; ++ ++ spdif_out: spdif-out { ++ status = "okay"; ++ compatible = "linux,spdif-dit"; ++ #sound-dai-cells = <0>; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ clocks = <&rk808 1>; ++ clock-names = "ext_clock"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio0 10 GPIO_ACTIVE_LOW>; /* GPIO0_B2 */ ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ rockchip,grf = <&grf>; ++ wifi_chip_type = "ap6354"; ++ sdio_vref = <1800>; ++ WIFI,host_wake_irq = <&gpio0 3 GPIO_ACTIVE_HIGH>; /* GPIO0_a3 */ ++ status = "okay"; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&rk808 1>; ++ clock-names = "ext_clock"; ++ //wifi-bt-power-toggle; ++ uart_rts_gpios = <&gpio2 19 GPIO_ACTIVE_LOW>; /* GPIO2_C3 */ ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart0_rts>, <&bt_reset_gpio>, <&bt_wake_gpio>, <&bt_irq_gpio>; ++ pinctrl-1 = <&uart0_gpios>; ++ //BT,power_gpio = <&gpio3 19 GPIO_ACTIVE_HIGH>; /* GPIOx_xx */ ++ BT,reset_gpio = <&gpio0 9 GPIO_ACTIVE_HIGH>; /* GPIO0_B1 */ ++ BT,wake_gpio = <&gpio2 26 GPIO_ACTIVE_HIGH>; /* GPIO2_D2 */ ++ BT,wake_host_irq = <&gpio0 4 GPIO_ACTIVE_HIGH>; /* GPIO0_A4 */ ++ status = "okay"; ++ }; ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&dmc { ++ status = "okay"; ++ center-supply = <&vdd_center>; ++ upthreshold = <40>; ++ downdifferential = <20>; ++ system-status-freq = < ++ /*system status freq(KHz)*/ ++ SYS_STATUS_NORMAL 800000 ++ SYS_STATUS_REBOOT 528000 ++ SYS_STATUS_SUSPEND 200000 ++ SYS_STATUS_VIDEO_1080P 200000 ++ SYS_STATUS_VIDEO_4K 600000 ++ SYS_STATUS_VIDEO_4K_10B 800000 ++ SYS_STATUS_PERFORMANCE 800000 ++ SYS_STATUS_BOOST 600000 ++ SYS_STATUS_DUALVIEW 600000 ++ SYS_STATUS_ISP 600000 ++ >; ++ vop-bw-dmc-freq = < ++ /* min_bw(MB/s) max_bw(MB/s) freq(KHz) */ ++ 0 762 200000 ++ 763 1893 400000 ++ 1894 3012 528000 ++ 3013 99999 800000 ++ >; ++ auto-freq-en = <1>; ++ auto-min-freq = <200000>; ++}; ++ ++&spdif { ++ status = "okay"; ++ pinctrl-0 = <&spdif_bus>; ++ i2c-scl-rising-time-ns = <450>; ++ i2c-scl-falling-time-ns = <15>; ++ #sound-dai-cells = <0>; ++}; ++ ++&i2s0 { ++ status = "okay"; ++ rockchip,playback-channels = <8>; ++ rockchip,capture-channels = <8>; ++ #sound-dai-cells = <0>; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <300>; ++ i2c-scl-falling-time-ns = <15>; ++ ++ rt5651: rt5651@1a { ++ #sound-dai-cells = <0>; ++ compatible = "rockchip,rt5651"; ++ reg = <0x1a>; ++ clocks = <&cru SCLK_I2S_8CH_OUT>; ++ clock-names = "mclk"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s_8ch_mclk>; ++ spk-con-gpio = <&gpio0 11 GPIO_ACTIVE_HIGH>; ++ hp-det-gpio = <&gpio4 28 GPIO_ACTIVE_LOW>; ++ }; ++}; ++ ++&i2c4 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <600>; ++ i2c-scl-falling-time-ns = <20>; ++ ++ mpu6500@68 { ++ status = "disabled"; ++ compatible = "invensense,mpu6500"; ++ reg = <0x68>; ++ irq-gpio = <&gpio1 22 IRQ_TYPE_EDGE_RISING>; ++ mpu-int_config = <0x10>; ++ mpu-level_shifter = <0>; ++ mpu-orientation = <0 1 0 1 0 0 0 0 1>; ++ orientation-x= <1>; ++ orientation-y= <0>; ++ orientation-z= <0>; ++ mpu-debug = <1>; ++ }; ++ ++ mpu6500_acc: mpu_acc@68 { ++ compatible = "mpu6500_acc"; ++ reg = <0x68>; ++ irq-gpio = <&gpio1 22 IRQ_TYPE_EDGE_RISING>; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ type = ; ++ layout = <2>; ++ }; ++ ++ mpu6500_gyro: mpu_gyro@68 { ++ compatible = "mpu6500_gyro"; ++ reg = <0x68>; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ type = ; ++ layout = <2>; ++ }; ++}; ++ ++&pcie0 { ++ ep-gpios = <&gpio2 4 GPIO_ACTIVE_HIGH>; ++ num-lanes = <4>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pcie_clkreqn_cpm>; ++}; ++ ++&uart0 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart0_xfer &uart0_cts>; ++ status = "okay"; ++}; ++ ++&saradc { ++ status = "okay"; ++}; ++ ++&sdio0 { ++ max-frequency = <100000000>; ++ no-sd; ++ no-mmc; ++ bus-width = <4>; ++ disable-wp; ++ cap-sd-highspeed; ++ cap-sdio-irq; ++ keep-power-in-suspend; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ non-removable; ++ num-slots = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&rockchip_suspend { ++ status = "okay"; ++ rockchip,sleep-debug-en = <1>; ++ rockchip,sleep-mode-config = < ++ (0 ++ | RKPM_SLP_ARMPD ++ | RKPM_SLP_PERILPPD ++ | RKPM_SLP_DDR_RET ++ | RKPM_SLP_PLLPD ++ | RKPM_SLP_CENTER_PD ++ | RKPM_SLP_AP_PWROFF ++ ) ++ >; ++ rockchip,wakeup-config = < ++ (0 ++ | RKPM_GPIO_WKUP_EN ++ | RKPM_PWM_WKUP_EN ++ ) ++ >; ++ rockchip,pwm-regulator-config = < ++ (0 ++ | PWM2_REGULATOR_EN ++ ) ++ >; ++ rockchip,power-ctrl = ++ <&gpio1 17 GPIO_ACTIVE_HIGH>, ++ <&gpio1 14 GPIO_ACTIVE_HIGH>; ++}; ++ ++&pinctrl { ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = ++ <0 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart0_gpios: uart0-gpios { ++ rockchip,pins = ++ <2 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ bt_reset_gpio: bt-reset-gpio { ++ rockchip,pins = ++ <0 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ bt_wake_gpio: bt-wake-gpio { ++ rockchip,pins = ++ <2 RK_PD2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ bt_irq_gpio: bt-irq-gpio { ++ rockchip,pins = ++ <0 RK_PA4 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-firefly-android.dts b/arch/arm64/boot/dts/rockchip/rk3399-firefly-android.dts +new file mode 100755 +index 000000000000..c372ade536f2 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-firefly-android.dts +@@ -0,0 +1,1102 @@ ++/* ++ * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This file is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This file is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++/dts-v1/; ++ ++#include "dt-bindings/pwm/pwm.h" ++#include "rk3399.dtsi" ++#include "rk3399-opp.dtsi" ++#include ++#include ++#include "rk3399-vop-clk-set.dtsi" ++ ++/ { ++ model = "Rockchip RK3399 Firefly Board (Android)"; ++ compatible = "rockchip,rk3399-firefly-android", "rockchip,rk3399"; ++ ++ chosen: chosen { ++ bootargs = "earlycon=uart8250,mmio32,0xff1a0000 coherent_pool=1m"; ++ }; ++ ++ cpuinfo { ++ compatible = "rockchip,cpuinfo"; ++ nvmem-cells = <&cpu_id>; ++ nvmem-cell-names = "id"; ++ }; ++ ++ backlight: backlight { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm0 0 25000 0>; ++ brightness-levels = < ++ 0 1 2 3 4 5 6 7 ++ 8 9 10 11 12 13 14 15 ++ 16 17 18 19 20 21 22 23 ++ 24 25 26 27 28 29 30 31 ++ 32 33 34 35 36 37 38 39 ++ 40 41 42 43 44 45 46 47 ++ 48 49 50 51 52 53 54 55 ++ 56 57 58 59 60 61 62 63 ++ 64 65 66 67 68 69 70 71 ++ 72 73 74 75 76 77 78 79 ++ 80 81 82 83 84 85 86 87 ++ 88 89 90 91 92 93 94 95 ++ 96 97 98 99 100 101 102 103 ++ 104 105 106 107 108 109 110 111 ++ 112 113 114 115 116 117 118 119 ++ 120 121 122 123 124 125 126 127 ++ 128 129 130 131 132 133 134 135 ++ 136 137 138 139 140 141 142 143 ++ 144 145 146 147 148 149 150 151 ++ 152 153 154 155 156 157 158 159 ++ 160 161 162 163 164 165 166 167 ++ 168 169 170 171 172 173 174 175 ++ 176 177 178 179 180 181 182 183 ++ 184 185 186 187 188 189 190 191 ++ 192 193 194 195 196 197 198 199 ++ 200 201 202 203 204 205 206 207 ++ 208 209 210 211 212 213 214 215 ++ 216 217 218 219 220 221 222 223 ++ 224 225 226 227 228 229 230 231 ++ 232 233 234 235 236 237 238 239 ++ 240 241 242 243 244 245 246 247 ++ 248 249 250 251 252 253 254 255>; ++ default-brightness-level = <200>; ++ }; ++ ++ clkin_gmac: external-gmac-clock { ++ compatible = "fixed-clock"; ++ clock-frequency = <125000000>; ++ clock-output-names = "clkin_gmac"; ++ #clock-cells = <0>; ++ }; ++ ++ dw_hdmi_audio: dw-hdmi-audio { ++ status = "okay"; ++ compatible = "rockchip,dw-hdmi-audio"; ++ #sound-dai-cells = <0>; ++ }; ++ ++ edp_panel: edp-panel { ++ compatible = "sharp,lcd-f402", "panel-simple"; ++ status = "okay"; ++ ++ backlight = <&backlight>; ++ power-supply = <&vcc_lcd>; ++ enable-gpios = <&gpio4 29 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&lcd_panel_reset>; ++ ++ ports { ++ panel_in_edp: endpoint { ++ remote-endpoint = <&edp_out_panel>; ++ }; ++ }; ++ }; ++ ++ fiq_debugger: fiq-debugger { ++ compatible = "rockchip,fiq-debugger"; ++ rockchip,serial-id = <2>; ++ rockchip,wake-irq = <0>; ++ /* If enable uart uses irq instead of fiq */ ++ rockchip,irq-mode-enable = <0>; ++ /* Only 115200 and 1500000 */ ++ rockchip,baudrate = <1500000>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart2c_xfer>; ++ interrupts = ; ++ }; ++ ++ reserved-memory { ++ #address-cells = <2>; ++ #size-cells = <2>; ++ ranges; ++ ++ drm_logo: drm-logo@00000000 { ++ compatible = "rockchip,drm-logo"; ++ reg = <0x0 0x0 0x0 0x0>; ++ }; ++ ++ stb_devinfo: stb-devinfo@00000000 { ++ compatible = "rockchip,stb-devinfo"; ++ reg = <0x0 0x0 0x0 0x0>; ++ }; ++ ++ ramoops: ramoops@110000 { ++ compatible = "ramoops"; ++ reg = <0x0 0x110000 0x0 0xf0000>; ++ record-size = <0x20000>; ++ console-size = <0x80000>; ++ ftrace-size = <0x00000>; ++ pmsg-size = <0x50000>; ++ }; ++ }; ++ ++ rockchip-key { ++ compatible = "rockchip,key"; ++ status = "okay"; ++ ++ io-channels = <&saradc 1>; ++ power-key { ++ gpios = <&gpio0 5 GPIO_ACTIVE_LOW>; ++ linux,code = <116>; ++ label = "power"; ++ gpio-key,wakeup; ++ }; ++ }; ++ ++ rt5640-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,rt5640-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,widgets = ++ "Microphone", "Mic Jack", ++ "Headphone", "Headphone Jack"; ++ simple-audio-card,routing = ++ "Mic Jack", "MICBIAS1", ++ "IN1P", "Mic Jack", ++ "Headphone Jack", "HPOL", ++ "Headphone Jack", "HPOR"; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s1>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&rt5640>; ++ }; ++ }; ++ ++ hdmi_sound: hdmi-sound { ++ status = "disabled"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,name = "rockchip,hdmi"; ++ ++ simple-audio-card,cpu { ++ sound-dai = <&i2s2>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&dw_hdmi_audio>; ++ }; ++ }; ++ ++ hdmi_codec: hdmi-codec { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,name = "HDMI-CODEC"; ++ ++ simple-audio-card,cpu { ++ sound-dai = <&i2s2>; ++ }; ++ ++ simple-audio-card,codec { ++ sound-dai = <&hdmi>; ++ }; ++ }; ++ ++ rga: rga@ff680000 { ++ compatible = "rockchip,rga2"; ++ dev_mode = <1>; ++ reg = <0x0 0xff680000 0x0 0x1000>; ++ interrupts = ; ++ clocks = <&cru ACLK_RGA>, <&cru HCLK_RGA>, <&cru SCLK_RGA_CORE>; ++ clock-names = "aclk_rga", "hclk_rga", "clk_rga"; ++ power-domains = <&power RK3399_PD_RGA>; ++ dma-coherent; ++ status = "okay"; ++ }; ++ ++ spdif-sound { ++ compatible = "simple-audio-card"; ++ status = "okay"; ++ ++ simple-audio-card,name = "ROCKCHIP,SPDIF"; ++ simple-audio-card,cpu { ++ sound-dai = <&spdif>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&spdif_out>; ++ }; ++ }; ++ ++ spdif_out: spdif-out { ++ compatible = "linux,spdif-dit"; ++ status = "okay"; ++ ++ #sound-dai-cells = <0>; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ clocks = <&rk808 1>; ++ clock-names = "ext_clock"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio0 10 GPIO_ACTIVE_LOW>; /* GPIO0_B2 */ ++ }; ++ ++ vcc3v3_pcie: vcc3v3-pcie-regulator { ++ compatible = "regulator-fixed"; ++ enable-active-high; ++ regulator-always-on; ++ regulator-boot-on; ++ gpio = <&gpio1 17 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pcie_drv>; ++ regulator-name = "vcc3v3_pcie"; ++ }; ++ ++ vcc3v3_sys: vcc3v3-sys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc3v3_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ }; ++ ++ vcc5v0_host: vcc5v0-host-regulator { ++ compatible = "regulator-fixed"; ++ enable-active-high; ++ gpio = <&gpio1 0 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&host_vbus_drv>; ++ regulator-name = "vcc5v0_host"; ++ regulator-always-on; ++ }; ++ ++ vcc5v0_sys: vcc5v0-sys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc5v0_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ }; ++ ++ vcc_phy: vcc-phy-regulator { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_phy"; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ vdd_log: vdd-log { ++ compatible = "pwm-regulator"; ++ pwms = <&pwm2 0 25000 1>; ++ regulator-name = "vdd_log"; ++ regulator-min-microvolt = <800000>; ++ /* ++ * the firefly hardware using 3.0 v as APIO2_VDD ++ * voltage, but the pwm divider resistance is designed ++ * based on hardware which the APIO2_VDD is 1.8v, so we ++ * need to change the regulator-max-microvolt from 1.4v ++ * to 1.0v, so the pwm can output 0.9v voltage. ++ */ ++ regulator-max-microvolt = <1000000>; ++ regulator-always-on; ++ regulator-boot-on; ++ ++ /* for rockchip boot on */ ++ rockchip,pwm_id= <2>; ++ rockchip,pwm_voltage = <900000>; ++ }; ++ ++ vccadc_ref: vccadc-ref { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc1v8_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ }; ++ ++ vcc_lcd: vcc-lcd-regulator { ++ compatible = "regulator-fixed"; ++ regulator-always-on; ++ regulator-boot-on; ++ enable-active-high; ++ gpio = <&gpio1 1 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&lcd_en>; ++ regulator-name = "vcc_lcd"; ++ }; ++ ++ xin32k: xin32k { ++ compatible = "fixed-clock"; ++ clock-frequency = <32768>; ++ clock-output-names = "xin32k"; ++ #clock-cells = <0>; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ rockchip,grf = <&grf>; ++ wifi_chip_type = "ap6354"; ++ sdio_vref = <1800>; ++ WIFI,host_wake_irq = <&gpio0 3 GPIO_ACTIVE_HIGH>; /* GPIO0_a3 */ ++ status = "okay"; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ //wifi-bt-power-toggle; ++ uart_rts_gpios = <&gpio2 19 GPIO_ACTIVE_LOW>; /* GPIO2_C3 */ ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart0_rts>; ++ pinctrl-1 = <&uart0_gpios>; ++ //BT,power_gpio = <&gpio3 19 GPIO_ACTIVE_HIGH>; /* GPIOx_xx */ ++ BT,reset_gpio = <&gpio0 9 GPIO_ACTIVE_HIGH>; /* GPIO0_B1 */ ++ BT,wake_gpio = <&gpio2 26 GPIO_ACTIVE_HIGH>; /* GPIO2_D2 */ ++ BT,wake_host_irq = <&gpio0 4 GPIO_ACTIVE_HIGH>; /* GPIO0_A4 */ ++ status = "okay"; ++ }; ++}; ++ ++&cpu_l0 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_l1 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_l2 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_l3 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_b0 { ++ cpu-supply = <&vdd_cpu_b>; ++}; ++ ++&cpu_b1 { ++ cpu-supply = <&vdd_cpu_b>; ++}; ++ ++&display_subsystem { ++ status = "okay"; ++ ++ ports = <&vopb_out>, <&vopl_out>; ++ logo-memory-region = <&drm_logo>; ++ ++ route { ++ route_hdmi: route-hdmi { ++ status = "okay"; ++ logo,uboot = "logo.bmp"; ++ logo,kernel = "logo_kernel.bmp"; ++ logo,mode = "fullscreen"; ++ charge_logo,mode = "center"; ++ connect = <&vopl_out_hdmi>; ++ }; ++ ++ route_edp: route-edp { ++ status = "okay"; ++ logo,uboot = "logo.bmp"; ++ logo,kernel = "logo_kernel.bmp"; ++ logo,mode = "fullscreen"; ++ charge_logo,mode = "center"; ++ connect = <&vopb_out_edp>; ++ }; ++ }; ++}; ++ ++&edp { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&edp_hpd>; ++ ++ ports { ++ edp_out: port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ edp_out_panel: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&panel_in_edp>; ++ }; ++ }; ++ }; ++}; ++ ++&emmc_phy { ++ status = "okay"; ++}; ++ ++&gmac { ++ phy-supply = <&vcc_phy>; ++ phy-mode = "rgmii"; ++ clock_in_out = "input"; ++ snps,reset-gpio = <&gpio3 15 GPIO_ACTIVE_LOW>; ++ snps,reset-active-low; ++ snps,reset-delays-us = <0 10000 50000>; ++ assigned-clocks = <&cru SCLK_RMII_SRC>; ++ assigned-clock-parents = <&clkin_gmac>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&rgmii_pins>; ++ tx_delay = <0x28>; ++ rx_delay = <0x11>; ++ status = "okay"; ++}; ++ ++&gpu { ++ status = "okay"; ++ mali-supply = <&vdd_gpu>; ++}; ++ ++&hdmi { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ #sound-dai-cells = <0>; ++ status = "okay"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <168>; ++ i2c-scl-falling-time-ns = <4>; ++ clock-frequency = <400000>; ++ ++ vdd_cpu_b: syr827@40 { ++ compatible = "silergy,syr827"; ++ reg = <0x40>; ++ vin-supply = <&vcc5v0_sys>; ++ regulator-compatible = "fan53555-reg"; ++ regulator-name = "vdd_cpu_b"; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1500000>; ++ regulator-ramp-delay = <1000>; ++ vsel-gpios = <&gpio1 18 GPIO_ACTIVE_HIGH>; ++ fcs,suspend-voltage-selector = <1>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-state = <3>; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdd_gpu: syr828@41 { ++ compatible = "silergy,syr828"; ++ reg = <0x41>; ++ vin-supply = <&vcc5v0_sys>; ++ regulator-compatible = "fan53555-reg"; ++ regulator-name = "vdd_gpu"; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1500000>; ++ regulator-ramp-delay = <1000>; ++ fcs,suspend-voltage-selector = <1>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-state = <3>; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ rk808: pmic@1b { ++ compatible = "rockchip,rk808"; ++ reg = <0x1b>; ++ interrupt-parent = <&gpio1>; ++ interrupts = <21 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pmic_int_l &pmic_dvs2>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ ++ vcc1-supply = <&vcc3v3_sys>; ++ vcc2-supply = <&vcc3v3_sys>; ++ vcc3-supply = <&vcc3v3_sys>; ++ vcc4-supply = <&vcc3v3_sys>; ++ vcc6-supply = <&vcc3v3_sys>; ++ vcc7-supply = <&vcc3v3_sys>; ++ vcc8-supply = <&vcc3v3_sys>; ++ vcc9-supply = <&vcc3v3_sys>; ++ vcc10-supply = <&vcc3v3_sys>; ++ vcc11-supply = <&vcc3v3_sys>; ++ vcc12-supply = <&vcc3v3_sys>; ++ vddio-supply = <&vcc1v8_pmu>; ++ ++ regulators { ++ vdd_center: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <750000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-name = "vdd_center"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdd_cpu_l: DCDC_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <750000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-name = "vdd_cpu_l"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc_ddr"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_1v8: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc_1v8"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vcc1v8_dvp: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc1v8_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc3v0_tp: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-name = "vcc3v0_tp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc1v8_pmu: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc1v8_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vcc_sd: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-name = "vcc_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ ++ vcca3v0_codec: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-name = "vcca3v0_codec"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_1v5: LDO_REG6 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ regulator-name = "vcc_1v5"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1500000>; ++ }; ++ }; ++ ++ vcca1v8_codec: LDO_REG7 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcca1v8_codec"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_3v0: LDO_REG8 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-name = "vcc_3v0"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ ++ vcc3v3_s3: SWITCH_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc3v3_s3"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc3v3_s0: SWITCH_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc3v3_s0"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ }; ++ }; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <300>; ++ i2c-scl-falling-time-ns = <15>; ++ ++ rt5640: rt5640@1c { ++ #sound-dai-cells = <0>; ++ compatible = "realtek,rt5640"; ++ reg = <0x1c>; ++ clocks = <&cru SCLK_I2S_8CH_OUT>; ++ clock-names = "mclk"; ++ realtek,in1-differential; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&rt5640_hpcon &i2s_8ch_mclk>; ++ hp-con-gpio = <&gpio4 21 GPIO_ACTIVE_HIGH>; ++ //hp-det-gpio = <&gpio4 28 GPIO_ACTIVE_LOW>; ++ io-channels = <&saradc 4>; ++ hp-det-adc-value = <500>; ++ status = "okay"; ++ }; ++}; ++ ++&i2c3 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <450>; ++ i2c-scl-falling-time-ns = <15>; ++}; ++ ++&i2c4 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <600>; ++ i2c-scl-falling-time-ns = <20>; ++ ++ fusb0: fusb30x@22 { ++ compatible = "fairchild,fusb302"; ++ reg = <0x22>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&fusb0_int>; ++ int-n-gpios = <&gpio1 2 GPIO_ACTIVE_HIGH>; ++ vbus-5v-gpios = <&gpio2 0 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ gsl3680: gsl3680@41 { ++ compatible = "gslX680-pad"; ++ reg = <0x41>; ++ screen_max_x = <1536>; ++ screen_max_y = <2048>; ++ touch-gpio = <&gpio1 20 IRQ_TYPE_LEVEL_LOW>; ++ reset-gpio = <&gpio0 12 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ mpu6050: mpu@68 { ++ compatible = "invensense,mpu6050"; ++ reg = <0x68>; ++ mpu-int_config = <0x10>; ++ mpu-level_shifter = <0>; ++ mpu-orientation = <0 1 0 1 0 0 0 0 1>; ++ orientation-x= <1>; ++ orientation-y= <1>; ++ orientation-z= <1>; ++ irq-gpio = <&gpio1 4 IRQ_TYPE_LEVEL_LOW>; ++ mpu-debug = <1>; ++ status = "okay"; ++ }; ++}; ++ ++&i2s0 { ++ status = "okay"; ++ rockchip,i2s-broken-burst-len; ++ rockchip,playback-channels = <8>; ++ rockchip,capture-channels = <8>; ++ #sound-dai-cells = <0>; ++}; ++ ++&i2s1 { ++ status = "okay"; ++ rockchip,i2s-broken-burst-len; ++ rockchip,playback-channels = <2>; ++ rockchip,capture-channels = <2>; ++ #sound-dai-cells = <0>; ++}; ++ ++&i2s2 { ++ #sound-dai-cells = <0>; ++ status = "okay"; ++}; ++ ++&io_domains { ++ status = "okay"; ++ ++ bt656-supply = <&vcc1v8_dvp>; /* bt656_gpio2ab_ms */ ++ audio-supply = <&vcca1v8_codec>; /* audio_gpio3d4a_ms */ ++ sdmmc-supply = <&vcc_sd>; /* sdmmc_gpio4b_ms */ ++ gpio1830-supply = <&vcc_3v0>; /* gpio1833_gpio4cd_ms */ ++}; ++ ++&pcie_phy { ++ status = "okay"; ++}; ++ ++&pcie0 { ++ ep-gpios = <&gpio4 25 GPIO_ACTIVE_HIGH>; ++ num-lanes = <4>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pcie_clkreqn_cpm>; ++ status = "okay"; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ pmu1830-supply = <&vcc_3v0>; ++}; ++ ++&pinctrl { ++ ++ lcd-panel { ++ lcd_panel_reset: lcd-panel-reset { ++ rockchip,pins = <4 RK_PD5 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ lcd_en: lcd-en { ++ rockchip,pins = <1 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ pcie { ++ pcie_drv: pcie-drv { ++ rockchip,pins = ++ <1 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ pcie_3g_drv: pcie-3g-drv { ++ rockchip,pins = ++ <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ pmic { ++ vsel1_gpio: vsel1-gpio { ++ rockchip,pins = ++ <1 RK_PC2 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ ++ vsel2_gpio: vsel2-gpio { ++ rockchip,pins = ++ <1 RK_PB6 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = ++ <0 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart0_gpios: uart0-gpios { ++ rockchip,pins = ++ <2 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ rt5640 { ++ rt5640_hpcon: rt5640-hpcon { ++ rockchip,pins = <4 RK_PC5 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ pmic { ++ pmic_int_l: pmic-int-l { ++ rockchip,pins = ++ <1 RK_PC5 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ pmic_dvs2: pmic-dvs2 { ++ rockchip,pins = ++ <1 RK_PC2 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ usb2 { ++ host_vbus_drv: host-vbus-drv { ++ rockchip,pins = ++ <4 RK_PD1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ fusb30x { ++ fusb0_int: fusb0-int { ++ rockchip,pins = <1 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++}; ++ ++&pwm0 { ++ status = "okay"; ++}; ++ ++&pwm2 { ++ status = "okay"; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm2_pin_pull_down>; ++}; ++ ++&rockchip_suspend { ++ rockchip,power-ctrl = ++ <&gpio1 18 GPIO_ACTIVE_LOW>, ++ <&gpio1 14 GPIO_ACTIVE_HIGH>; ++}; ++ ++&saradc { ++ status = "okay"; ++ vref-supply = <&vccadc_ref>; ++}; ++ ++&sdhci { ++ bus-width = <8>; ++ keep-power-in-suspend; ++ mmc-hs400-1_8v; ++ mmc-hs400-enhanced-strobe; ++ non-removable; ++ status = "okay"; ++ no-sdio; ++ no-sd; ++}; ++ ++&sdmmc { ++ max-frequency = <150000000>; ++ no-sdio; ++ no-mmc; ++ bus-width = <4>; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ disable-wp; ++ num-slots = <1>; ++ vqmmc-supply = <&vcc_sd>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_cd &sdmmc_bus4>; ++ status = "okay"; ++}; ++ ++&sdio0 { ++ max-frequency = <50000000>; ++ no-sd; ++ no-mmc; ++ bus-width = <4>; ++ disable-wp; ++ cap-sd-highspeed; ++ keep-power-in-suspend; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ non-removable; ++ num-slots = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&spdif { ++ status = "okay"; ++ pinctrl-0 = <&spdif_bus_1>; ++ i2c-scl-rising-time-ns = <450>; ++ i2c-scl-falling-time-ns = <15>; ++ #sound-dai-cells = <0>; ++}; ++ ++&tcphy0 { ++ extcon = <&fusb0>; ++ status = "okay"; ++}; ++ ++&tcphy1 { ++ status = "okay"; ++}; ++ ++&tsadc { ++ /* tshut mode 0:CRU 1:GPIO */ ++ rockchip,hw-tshut-mode = <1>; ++ /* tshut polarity 0:LOW 1:HIGH */ ++ rockchip,hw-tshut-polarity = <1>; ++ status = "okay"; ++}; ++ ++&u2phy0 { ++ status = "okay"; ++ extcon = <&fusb0>; ++ ++ u2phy0_otg: otg-port { ++ status = "okay"; ++ }; ++ ++ u2phy0_host: host-port { ++ phy-supply = <&vcc5v0_host>; ++ status = "okay"; ++ }; ++}; ++ ++&u2phy1 { ++ status = "okay"; ++ ++ u2phy1_otg: otg-port { ++ status = "okay"; ++ }; ++ ++ u2phy1_host: host-port { ++ phy-supply = <&vcc5v0_host>; ++ status = "okay"; ++ }; ++}; ++ ++&uart0 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart0_xfer &uart0_cts>; ++ status = "okay"; ++}; ++ ++&uart2 { ++ status = "okay"; ++}; ++ ++&usbdrd3_0 { ++ status = "okay"; ++}; ++ ++&usbdrd3_1 { ++ status = "okay"; ++}; ++ ++&usbdrd_dwc3_0 { ++ status = "okay"; ++ extcon = <&fusb0>; ++}; ++ ++&usbdrd_dwc3_1 { ++ status = "okay"; ++ dr_mode = "host"; ++}; ++ ++&usb_host0_ehci { ++ status = "okay"; ++}; ++ ++&usb_host0_ohci { ++ status = "okay"; ++}; ++ ++&usb_host1_ehci { ++ status = "okay"; ++}; ++ ++&usb_host1_ohci { ++ status = "okay"; ++}; ++ ++&vopb { ++ status = "okay"; ++}; ++ ++&vopb_mmu { ++ status = "okay"; ++}; ++ ++&vopl { ++ status = "okay"; ++}; ++ ++&vopl_mmu { ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-firefly-linux.dts b/arch/arm64/boot/dts/rockchip/rk3399-firefly-linux.dts +new file mode 100755 +index 000000000000..5a023389a033 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-firefly-linux.dts +@@ -0,0 +1,1074 @@ ++/* ++ * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This file is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This file is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++/dts-v1/; ++ ++#include "dt-bindings/pwm/pwm.h" ++#include "rk3399.dtsi" ++#include "rk3399-opp.dtsi" ++#include "rk3399-linux.dtsi" ++#include ++ ++/ { ++ model = "Rockchip RK3399 Firefly Board (Linux Opensource)"; ++ compatible = "rockchip,rk3399-firefly-linux", "rockchip,rk3399"; ++ ++ backlight: backlight { ++ status = "disabled"; ++ compatible = "pwm-backlight"; ++ pwms = <&pwm0 0 25000 0>; ++ brightness-levels = < ++ 0 1 2 3 4 5 6 7 ++ 8 9 10 11 12 13 14 15 ++ 16 17 18 19 20 21 22 23 ++ 24 25 26 27 28 29 30 31 ++ 32 33 34 35 36 37 38 39 ++ 40 41 42 43 44 45 46 47 ++ 48 49 50 51 52 53 54 55 ++ 56 57 58 59 60 61 62 63 ++ 64 65 66 67 68 69 70 71 ++ 72 73 74 75 76 77 78 79 ++ 80 81 82 83 84 85 86 87 ++ 88 89 90 91 92 93 94 95 ++ 96 97 98 99 100 101 102 103 ++ 104 105 106 107 108 109 110 111 ++ 112 113 114 115 116 117 118 119 ++ 120 121 122 123 124 125 126 127 ++ 128 129 130 131 132 133 134 135 ++ 136 137 138 139 140 141 142 143 ++ 144 145 146 147 148 149 150 151 ++ 152 153 154 155 156 157 158 159 ++ 160 161 162 163 164 165 166 167 ++ 168 169 170 171 172 173 174 175 ++ 176 177 178 179 180 181 182 183 ++ 184 185 186 187 188 189 190 191 ++ 192 193 194 195 196 197 198 199 ++ 200 201 202 203 204 205 206 207 ++ 208 209 210 211 212 213 214 215 ++ 216 217 218 219 220 221 222 223 ++ 224 225 226 227 228 229 230 231 ++ 232 233 234 235 236 237 238 239 ++ 240 241 242 243 244 245 246 247 ++ 248 249 250 251 252 253 254 255>; ++ default-brightness-level = <200>; ++ }; ++ ++ clkin_gmac: external-gmac-clock { ++ compatible = "fixed-clock"; ++ clock-frequency = <125000000>; ++ clock-output-names = "clkin_gmac"; ++ #clock-cells = <0>; ++ }; ++ ++ dw_hdmi_audio: dw-hdmi-audio { ++ status = "disabled"; ++ compatible = "rockchip,dw-hdmi-audio"; ++ #sound-dai-cells = <0>; ++ }; ++ ++ edp_panel: edp-panel { ++ status = "disabled"; ++ compatible = "sharp,lcd-f402", "panel-simple"; ++ backlight = <&backlight>; ++ power-supply = <&vcc_lcd>; ++ enable-gpios = <&gpio4 29 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&lcd_panel_reset>; ++ ++ ports { ++ panel_in_edp: endpoint { ++ remote-endpoint = <&edp_out_panel>; ++ }; ++ }; ++ }; ++ ++ gpio-keys { ++ compatible = "gpio-keys"; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ autorepeat; ++ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pwrbtn>; ++ ++ button@0 { ++ gpios = <&gpio0 5 GPIO_ACTIVE_LOW>; ++ linux,code = ; ++ label = "GPIO Key Power"; ++ linux,input-type = <1>; ++ gpio-key,wakeup = <1>; ++ debounce-interval = <100>; ++ }; ++ }; ++ ++ rt5640-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,rt5640-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,widgets = ++ "Microphone", "Mic Jack", ++ "Headphone", "Headphone Jack"; ++ simple-audio-card,routing = ++ "Mic Jack", "MICBIAS1", ++ "IN1P", "Mic Jack", ++ "Headphone Jack", "HPOL", ++ "Headphone Jack", "HPOR"; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s1>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&rt5640>; ++ }; ++ }; ++ ++ hdmi_sound: hdmi-sound { ++ status = "disabled"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,name = "rockchip,hdmi"; ++ ++ simple-audio-card,cpu { ++ sound-dai = <&i2s2>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&dw_hdmi_audio>; ++ }; ++ }; ++ ++ hdmi_codec: hdmi-codec { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,name = "HDMI-CODEC"; ++ ++ simple-audio-card,cpu { ++ sound-dai = <&i2s2>; ++ }; ++ ++ simple-audio-card,codec { ++ sound-dai = <&hdmi>; ++ }; ++ }; ++ ++ spdif-sound { ++ status = "okay"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,name = "ROCKCHIP,SPDIF"; ++ simple-audio-card,cpu { ++ sound-dai = <&spdif>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&spdif_out>; ++ }; ++ }; ++ ++ spdif_out: spdif-out { ++ status = "okay"; ++ compatible = "linux,spdif-dit"; ++ #sound-dai-cells = <0>; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ clocks = <&rk808 1>; ++ clock-names = "ext_clock"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio0 10 GPIO_ACTIVE_LOW>; /* GPIO0_B2 */ ++ }; ++ ++ vcc3v3_pcie: vcc3v3-pcie-regulator { ++ compatible = "regulator-fixed"; ++ enable-active-high; ++ regulator-always-on; ++ regulator-boot-on; ++ gpio = <&gpio1 17 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pcie_drv>; ++ regulator-name = "vcc3v3_pcie"; ++ }; ++ ++ vcc3v3_sys: vcc3v3-sys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc3v3_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ }; ++ ++ vcc5v0_host: vcc5v0-host-regulator { ++ compatible = "regulator-fixed"; ++ enable-active-high; ++ gpio = <&gpio1 0 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&host_vbus_drv>; ++ regulator-name = "vcc5v0_host"; ++ regulator-always-on; ++ }; ++ ++ vcc5v0_sys: vcc5v0-sys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc5v0_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ }; ++ ++ vcc_phy: vcc-phy-regulator { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_phy"; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ vdd_log: vdd-log { ++ compatible = "pwm-regulator"; ++ pwms = <&pwm2 0 25000 1>; ++ regulator-name = "vdd_log"; ++ regulator-min-microvolt = <800000>; ++ /* ++ * the firefly hardware using 3.0 v as APIO2_VDD ++ * voltage, but the pwm divider resistance is designed ++ * based on hardware which the APIO2_VDD is 1.8v, so we ++ * need to change the regulator-max-microvolt from 1.4v ++ * to 1.0v, so the pwm can output 0.9v voltage. ++ */ ++ regulator-max-microvolt = <1000000>; ++ regulator-always-on; ++ regulator-boot-on; ++ ++ /* for rockchip boot on */ ++ rockchip,pwm_id= <2>; ++ rockchip,pwm_voltage = <900000>; ++ }; ++ ++ vccadc_ref: vccadc-ref { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc1v8_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ }; ++ ++ vcc_lcd: vcc-lcd-regulator { ++ compatible = "regulator-fixed"; ++ regulator-always-on; ++ regulator-boot-on; ++ enable-active-high; ++ gpio = <&gpio1 1 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&lcd_en>; ++ regulator-name = "vcc_lcd"; ++ }; ++ ++ xin32k: xin32k { ++ compatible = "fixed-clock"; ++ clock-frequency = <32768>; ++ clock-output-names = "xin32k"; ++ #clock-cells = <0>; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ rockchip,grf = <&grf>; ++ wifi_chip_type = "ap6354"; ++ sdio_vref = <1800>; ++ WIFI,host_wake_irq = <&gpio0 3 GPIO_ACTIVE_HIGH>; /* GPIO0_a3 */ ++ status = "okay"; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ //wifi-bt-power-toggle; ++ uart_rts_gpios = <&gpio2 19 GPIO_ACTIVE_LOW>; /* GPIO2_C3 */ ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart0_rts>; ++ pinctrl-1 = <&uart0_gpios>; ++ //BT,power_gpio = <&gpio3 19 GPIO_ACTIVE_HIGH>; /* GPIOx_xx */ ++ BT,reset_gpio = <&gpio0 9 GPIO_ACTIVE_HIGH>; /* GPIO0_B1 */ ++ BT,wake_gpio = <&gpio2 26 GPIO_ACTIVE_HIGH>; /* GPIO2_D2 */ ++ BT,wake_host_irq = <&gpio0 4 GPIO_ACTIVE_HIGH>; /* GPIO0_A4 */ ++ status = "okay"; ++ }; ++}; ++ ++&cpu_l0 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_l1 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_l2 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_l3 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_b0 { ++ cpu-supply = <&vdd_cpu_b>; ++}; ++ ++&cpu_b1 { ++ cpu-supply = <&vdd_cpu_b>; ++}; ++ ++&display_subsystem { ++ status = "okay"; ++}; ++ ++&edp { ++ status = "disabled"; ++ ++ ports { ++ edp_out: port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ edp_out_panel: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&panel_in_edp>; ++ }; ++ }; ++ }; ++}; ++ ++&emmc_phy { ++ status = "okay"; ++}; ++ ++&gmac { ++ phy-supply = <&vcc_phy>; ++ phy-mode = "rgmii"; ++ clock_in_out = "input"; ++ snps,reset-gpio = <&gpio3 15 GPIO_ACTIVE_LOW>; ++ snps,reset-active-low; ++ snps,reset-delays-us = <0 10000 50000>; ++ assigned-clocks = <&cru SCLK_RMII_SRC>; ++ assigned-clock-parents = <&clkin_gmac>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&rgmii_pins>; ++ tx_delay = <0x28>; ++ rx_delay = <0x11>; ++ status = "okay"; ++}; ++ ++&gpu { ++ status = "okay"; ++ mali-supply = <&vdd_gpu>; ++}; ++ ++&hdmi { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ #sound-dai-cells = <0>; ++ status = "okay"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <168>; ++ i2c-scl-falling-time-ns = <4>; ++ clock-frequency = <400000>; ++ ++ vdd_cpu_b: syr827@40 { ++ compatible = "silergy,syr827"; ++ reg = <0x40>; ++ vin-supply = <&vcc5v0_sys>; ++ regulator-compatible = "fan53555-reg"; ++ regulator-name = "vdd_cpu_b"; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1500000>; ++ regulator-ramp-delay = <1000>; ++ vsel-gpios = <&gpio1 18 GPIO_ACTIVE_HIGH>; ++ fcs,suspend-voltage-selector = <1>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-state = <3>; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdd_gpu: syr828@41 { ++ compatible = "silergy,syr828"; ++ reg = <0x41>; ++ vin-supply = <&vcc5v0_sys>; ++ regulator-compatible = "fan53555-reg"; ++ regulator-name = "vdd_gpu"; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1500000>; ++ regulator-ramp-delay = <1000>; ++ fcs,suspend-voltage-selector = <1>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-state = <3>; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ rk808: pmic@1b { ++ compatible = "rockchip,rk808"; ++ reg = <0x1b>; ++ interrupt-parent = <&gpio1>; ++ interrupts = <21 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pmic_int_l &pmic_dvs2>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ ++ vcc1-supply = <&vcc3v3_sys>; ++ vcc2-supply = <&vcc3v3_sys>; ++ vcc3-supply = <&vcc3v3_sys>; ++ vcc4-supply = <&vcc3v3_sys>; ++ vcc6-supply = <&vcc3v3_sys>; ++ vcc7-supply = <&vcc3v3_sys>; ++ vcc8-supply = <&vcc3v3_sys>; ++ vcc9-supply = <&vcc3v3_sys>; ++ vcc10-supply = <&vcc3v3_sys>; ++ vcc11-supply = <&vcc3v3_sys>; ++ vcc12-supply = <&vcc3v3_sys>; ++ vddio-supply = <&vcc1v8_pmu>; ++ ++ regulators { ++ vdd_center: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <750000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-name = "vdd_center"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdd_cpu_l: DCDC_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <750000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-name = "vdd_cpu_l"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc_ddr"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_1v8: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc_1v8"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vcc1v8_dvp: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc1v8_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc3v0_tp: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-name = "vcc3v0_tp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc1v8_pmu: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc1v8_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vcc_sd: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-name = "vcc_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ ++ vcca3v0_codec: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-name = "vcca3v0_codec"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_1v5: LDO_REG6 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ regulator-name = "vcc_1v5"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1500000>; ++ }; ++ }; ++ ++ vcca1v8_codec: LDO_REG7 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcca1v8_codec"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_3v0: LDO_REG8 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-name = "vcc_3v0"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ ++ vcc3v3_s3: SWITCH_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc3v3_s3"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc3v3_s0: SWITCH_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc3v3_s0"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ }; ++ }; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <300>; ++ i2c-scl-falling-time-ns = <15>; ++ ++ rt5640: rt5640@1c { ++ #sound-dai-cells = <0>; ++ compatible = "realtek,rt5640"; ++ reg = <0x1c>; ++ clocks = <&cru SCLK_I2S_8CH_OUT>; ++ clock-names = "mclk"; ++ realtek,in1-differential; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&rt5640_hpcon &i2s_8ch_mclk>; ++ hp-con-gpio = <&gpio4 21 GPIO_ACTIVE_HIGH>; ++ //hp-det-gpio = <&gpio4 28 GPIO_ACTIVE_LOW>; ++ io-channels = <&saradc 4>; ++ hp-det-adc-value = <500>; ++ }; ++ ++ camera0: ov13850@10 { ++ status = "okay"; ++ compatible = "omnivision,ov13850-v4l2-i2c-subdev"; ++ reg = < 0x10 >; ++ device_type = "v4l2-i2c-subdev"; ++ ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "clk_cif_out"; ++ ++ pinctrl-names = "rockchip,camera_default", "rockchip,camera_sleep"; ++ pinctrl-0 = <&cam0_default_pins>; ++ pinctrl-1 = <&cam0_sleep_pins>; ++ ++ rockchip,pd-gpio = <&gpio2 12 GPIO_ACTIVE_LOW>; ++ rockchip,pwr-gpio = <&gpio1 23 GPIO_ACTIVE_HIGH>; ++ rockchip,pwr-2nd-gpio = <&gpio1 22 GPIO_ACTIVE_HIGH>; ++ rockchip,rst-gpio = <&gpio0 8 GPIO_ACTIVE_LOW>; ++ ++ rockchip,camera-module-mclk-name = "clk_cif_out"; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "cmk-cb0695-fv1"; ++ rockchip,camera-module-len-name = "lg9569a2"; ++ rockchip,camera-module-fov-h = "66.0"; ++ rockchip,camera-module-fov-v = "50.1"; ++ rockchip,camera-module-orientation = <0>; ++ rockchip,camera-module-iq-flip = <0>; ++ rockchip,camera-module-iq-mirror = <0>; ++ rockchip,camera-module-flip = <1>; ++ rockchip,camera-module-mirror = <0>; ++ ++ rockchip,camera-module-defrect0 = <2112 1568 0 0 2112 1568>; ++ rockchip,camera-module-defrect1 = <4224 3136 0 0 4224 3136>; ++ rockchip,camera-module-defrect3 = <3264 2448 0 0 3264 2448>; ++ rockchip,camera-module-flash-support = <0>; ++ rockchip,camera-module-mipi-dphy-index = <0>; ++ }; ++}; ++ ++&i2c3 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <450>; ++ i2c-scl-falling-time-ns = <15>; ++}; ++ ++&i2c4 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <600>; ++ i2c-scl-falling-time-ns = <20>; ++ ++ fusb0: fusb30x@22 { ++ compatible = "fairchild,fusb302"; ++ reg = <0x22>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&fusb0_int>; ++ int-n-gpios = <&gpio1 2 GPIO_ACTIVE_HIGH>; ++ vbus-5v-gpios = <&gpio2 0 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ gsl3680: gsl3680@41 { ++ status = "disabled"; ++ compatible = "gslX680-pad"; ++ reg = <0x41>; ++ screen_max_x = <1536>; ++ screen_max_y = <2048>; ++ touch-gpio = <&gpio1 20 IRQ_TYPE_LEVEL_LOW>; ++ reset-gpio = <&gpio0 12 GPIO_ACTIVE_HIGH>; ++ }; ++ ++ mpu6050: mpu@68 { ++ status = "disabled"; ++ compatible = "invensense,mpu6050"; ++ reg = <0x68>; ++ mpu-int_config = <0x10>; ++ mpu-level_shifter = <0>; ++ mpu-orientation = <0 1 0 1 0 0 0 0 1>; ++ orientation-x= <1>; ++ orientation-y= <1>; ++ orientation-z= <1>; ++ irq-gpio = <&gpio1 4 IRQ_TYPE_LEVEL_LOW>; ++ mpu-debug = <1>; ++ }; ++}; ++ ++&i2s0 { ++ status = "okay"; ++ rockchip,i2s-broken-burst-len; ++ rockchip,playback-channels = <8>; ++ rockchip,capture-channels = <8>; ++ #sound-dai-cells = <0>; ++}; ++ ++&i2s1 { ++ status = "okay"; ++ rockchip,i2s-broken-burst-len; ++ rockchip,playback-channels = <2>; ++ rockchip,capture-channels = <2>; ++ #sound-dai-cells = <0>; ++}; ++ ++&i2s2 { ++ #sound-dai-cells = <0>; ++ status = "okay"; ++}; ++ ++&io_domains { ++ status = "okay"; ++ ++ bt656-supply = <&vcc1v8_dvp>; /* bt656_gpio2ab_ms */ ++ audio-supply = <&vcca1v8_codec>; /* audio_gpio3d4a_ms */ ++ sdmmc-supply = <&vcc_sd>; /* sdmmc_gpio4b_ms */ ++ gpio1830-supply = <&vcc_3v0>; /* gpio1833_gpio4cd_ms */ ++}; ++ ++&pcie_phy { ++ status = "okay"; ++}; ++ ++&pcie0 { ++ ep-gpios = <&gpio4 25 GPIO_ACTIVE_HIGH>; ++ num-lanes = <4>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pcie_clkreqn_cpm>; ++ status = "okay"; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ pmu1830-supply = <&vcc_3v0>; ++}; ++ ++&pinctrl { ++ buttons { ++ pwrbtn: pwrbtn { ++ rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ lcd-panel { ++ lcd_panel_reset: lcd-panel-reset { ++ rockchip,pins = <4 RK_PD5 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ lcd_en: lcd-en { ++ rockchip,pins = <1 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ pcie { ++ pcie_drv: pcie-drv { ++ rockchip,pins = ++ <1 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ pcie_3g_drv: pcie-3g-drv { ++ rockchip,pins = ++ <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ pmic { ++ vsel1_gpio: vsel1-gpio { ++ rockchip,pins = ++ <1 RK_PC2 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ ++ vsel2_gpio: vsel2-gpio { ++ rockchip,pins = ++ <1 RK_PB6 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = ++ <0 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart0_gpios: uart0-gpios { ++ rockchip,pins = ++ <2 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ rt5640 { ++ rt5640_hpcon: rt5640-hpcon { ++ rockchip,pins = <4 RK_PC5 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ pmic { ++ pmic_int_l: pmic-int-l { ++ rockchip,pins = ++ <1 RK_PC5 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ pmic_dvs2: pmic-dvs2 { ++ rockchip,pins = ++ <1 RK_PC2 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ usb2 { ++ host_vbus_drv: host-vbus-drv { ++ rockchip,pins = ++ <4 RK_PD1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ fusb30x { ++ fusb0_int: fusb0-int { ++ rockchip,pins = <1 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++}; ++ ++&pwm0 { ++ status = "okay"; ++}; ++ ++&pwm2 { ++ status = "okay"; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm2_pin_pull_down>; ++}; ++ ++&rockchip_suspend { ++ rockchip,power-ctrl = ++ <&gpio1 18 GPIO_ACTIVE_LOW>, ++ <&gpio1 14 GPIO_ACTIVE_HIGH>; ++}; ++ ++&route_edp { ++ status = "disabled"; ++}; ++ ++&saradc { ++ status = "okay"; ++ vref-supply = <&vccadc_ref>; ++}; ++ ++&sdhci { ++ bus-width = <8>; ++ keep-power-in-suspend; ++ mmc-hs400-1_8v; ++ mmc-hs400-enhanced-strobe; ++ non-removable; ++ status = "okay"; ++ no-sdio; ++ no-sd; ++}; ++ ++&sdmmc { ++ max-frequency = <150000000>; ++ no-sdio; ++ no-mmc; ++ bus-width = <4>; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ disable-wp; ++ num-slots = <1>; ++ vqmmc-supply = <&vcc_sd>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_cd &sdmmc_bus4>; ++ status = "okay"; ++}; ++ ++&sdio0 { ++ max-frequency = <50000000>; ++ no-sd; ++ no-mmc; ++ bus-width = <4>; ++ disable-wp; ++ cap-sd-highspeed; ++ keep-power-in-suspend; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ non-removable; ++ num-slots = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&spdif { ++ status = "okay"; ++ pinctrl-0 = <&spdif_bus_1>; ++ i2c-scl-rising-time-ns = <450>; ++ i2c-scl-falling-time-ns = <15>; ++ #sound-dai-cells = <0>; ++}; ++ ++&tcphy0 { ++ extcon = <&fusb0>; ++ status = "okay"; ++}; ++ ++&tcphy1 { ++ status = "okay"; ++}; ++ ++&tsadc { ++ /* tshut mode 0:CRU 1:GPIO */ ++ rockchip,hw-tshut-mode = <1>; ++ /* tshut polarity 0:LOW 1:HIGH */ ++ rockchip,hw-tshut-polarity = <1>; ++ status = "okay"; ++}; ++ ++&u2phy0 { ++ status = "okay"; ++ extcon = <&fusb0>; ++ ++ u2phy0_otg: otg-port { ++ status = "okay"; ++ }; ++ ++ u2phy0_host: host-port { ++ phy-supply = <&vcc5v0_host>; ++ status = "okay"; ++ }; ++}; ++ ++&u2phy1 { ++ status = "okay"; ++ ++ u2phy1_otg: otg-port { ++ status = "okay"; ++ }; ++ ++ u2phy1_host: host-port { ++ phy-supply = <&vcc5v0_host>; ++ status = "okay"; ++ }; ++}; ++ ++&uart0 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart0_xfer &uart0_cts>; ++ status = "okay"; ++}; ++ ++&uart2 { ++ status = "okay"; ++}; ++ ++&usbdrd3_0 { ++ status = "okay"; ++}; ++ ++&usbdrd3_1 { ++ status = "okay"; ++}; ++ ++&usbdrd_dwc3_0 { ++ status = "okay"; ++ extcon = <&fusb0>; ++}; ++ ++&usbdrd_dwc3_1 { ++ status = "okay"; ++ dr_mode = "host"; ++}; ++ ++&usb_host0_ehci { ++ status = "okay"; ++}; ++ ++&usb_host0_ohci { ++ status = "okay"; ++}; ++ ++&usb_host1_ehci { ++ status = "okay"; ++}; ++ ++&usb_host1_ohci { ++ status = "okay"; ++}; ++ ++&cif_isp0 { ++ rockchip,camera-modules-attached = <&camera0>; ++ status = "okay"; ++}; ++ ++&isp0_mmu { ++ status = "okay"; ++}; ++ ++&vopb { ++ status = "okay"; ++}; ++ ++&vopb_mmu { ++ status = "okay"; ++}; ++ ++&vopl { ++ status = "okay"; ++}; ++ ++&vopl_mmu { ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-fpga.dts b/arch/arm64/boot/dts/rockchip/rk3399-fpga.dts +new file mode 100755 +index 000000000000..1192dfa42940 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-fpga.dts +@@ -0,0 +1,64 @@ ++/* ++ * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This library is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This library is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++/dts-v1/; ++#include "rk3399.dtsi" ++ ++/ { ++ model = "Rockchip RK3399 FPGA Board"; ++ compatible = "rockchip,fpga", "rockchip,rk3399"; ++ ++ chosen { ++ bootargs = "init=/init console=uart,mmio32,0xff1a0000"; ++ }; ++ ++ memory@00000000 { ++ device_type = "memory"; ++ reg = <0x0 0x00000000 0x0 0x20000000>; ++ }; ++}; ++ ++&uart2 { ++ status = "okay"; ++ clocks = <&xin24m>, <&xin24m>; ++}; ++ +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-gru-gru.dts b/arch/arm64/boot/dts/rockchip/rk3399-gru-gru.dts +new file mode 100755 +index 000000000000..e8d771e2b90e +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-gru-gru.dts +@@ -0,0 +1,165 @@ ++/* ++ * Google Gru-Gru Rev 0+ board device tree source ++ * ++ * Copyright 2016 Google, Inc ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This file is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This file is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++/dts-v1/; ++#include "rk3399-gru.dtsi" ++ ++/ { ++ model = "Google Gru"; ++ compatible = "google,gru-rev15", "google,gru-rev14", ++ "google,gru-rev13", "google,gru-rev12", ++ "google,gru-rev11", "google,gru-rev10", ++ "google,gru-rev9", "google,gru-rev8", ++ "google,gru-rev7", "google,gru-rev6", ++ "google,gru-rev5", "google,gru-rev4", ++ "google,gru-rev3", "google,gru-rev2", ++ "google,gru-rev1", "google,gru-rev0", ++ "google,gru", "rockchip,rk3399"; ++ ++ // TODO: Model: ++ // - pp1200_mipi_cam ++ // - pp1800_mipi_cam ++ // - pp2800_mipi_cam ++ ++ /* pp1800 children */ ++ ++ pp1800_fp: pp1800-fp { ++ compatible = "regulator-fixed"; ++ regulator-name = "pp1800_fp"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&fp_en>; ++ ++ enable-active-high; ++ gpio = <&gpio1 1 GPIO_ACTIVE_HIGH>; ++ ++ regulator-always-on; // For bringup??? ++ regulator-boot-on; // For bringup??? ++ ++ vin-supply = <&pp1800>; ++ }; ++ ++ /* pp3300 children */ ++ ++ pp3300_fp: pp3300-fp { ++ compatible = "regulator-fixed"; ++ regulator-name = "pp3300_fp"; ++ /* NOTE: fp_en pinctrl in pp1800_fp */ ++ ++ enable-active-high; ++ gpio = <&gpio1 1 GPIO_ACTIVE_HIGH>; ++ ++ regulator-always-on; // For bringup??? ++ regulator-boot-on; // For bringup??? ++ ++ vin-supply = <&pp3300>; ++ }; ++}; ++ ++ap_i2c_dvs: &i2c0 { ++ status = "okay"; ++ ++ // TODO: bus speed ++ // ...with no speed, it should just use 100kHz ++ // TODO: rise / fall times? ++ ++ /* LP8556 */ ++ backlight@2c { ++ compatible = "ti,lp8556"; ++ reg = <0x2c>; ++ ++ // TODO: Where do we specify AP_BL_EN??? ++ ++ bl-name = "lcd-bl"; ++ dev-ctrl = /bits/ 8 <0x85>; // TODO: It depends on the device. ++ init-brt = /bits/ 8 <0x10>; // TODO: What should it be? ++ ++ power-supply = <&pp3300_disp>; ++ }; ++}; ++ ++ap_i2c_cam: &i2c2 { ++ status = "okay"; ++ ++ // TODO: bus speed ++ // ...with no speed, it should just use 100kHz ++ // TODO: rise / fall times? ++ ++ // TODO: I belive this is for the MIPI camera. ++}; ++ ++ap_i2c_nfc: &i2c7 { ++ status = "okay"; ++ ++ // TODO: bus speed ++ // ...with no speed, it should just use 100kHz ++ // TODO: rise / fall times? ++ ++ // TODO: Add the proper NFC reference... ++}; ++ ++&spi4 { ++ status = "okay"; ++ ++ // TODO: more properly. Hacky spidev for now??? ++ fingerprint@0 { ++ compatible = "spidev"; ++ spi-max-frequency = <10000000>; ++ reg = <0>; ++ }; ++}; ++ ++/* PINCTRL: always below everything else */ ++ ++&pinctrl { ++ discrete-regulators { ++ fp_en: fp-en { ++ rockchip,pins = <1 RK_PA1 RK_FUNC_GPIO ++ &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ ++/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ ++/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-gru-kevin-r0.dts b/arch/arm64/boot/dts/rockchip/rk3399-gru-kevin-r0.dts +new file mode 100755 +index 000000000000..7a1c36e981d5 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-gru-kevin-r0.dts +@@ -0,0 +1,118 @@ ++/* ++ * Google Gru-Kevin Rev 0 board device tree source ++ * ++ * Copyright 2016 Google, Inc ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This file is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This file is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++/dts-v1/; ++#include "rk3399-gru.dtsi" ++ ++/ { ++ model = "Google Kevin Rev 0"; ++ compatible = "google,kevin-rev0", ++ "google,kevin", "google,gru", "rockchip,rk3399"; ++}; ++ ++&ap_i2c_tp { ++ trackpad@4a { ++ compatible = "atmel,maxtouch"; ++ reg = <0x4a>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&trackpad_int_l>; ++ interrupt-parent = <&gpio1>; ++ interrupts = <4 IRQ_TYPE_LEVEL_LOW>; ++ linux,gpio-keymap = ; ++ }; ++}; ++ ++/* GPIO overrides for -r0; in same order as parent */ ++ ++&pp3000 { ++ gpio = <&gpio1 12 GPIO_ACTIVE_HIGH>; ++}; ++ ++&pp1800_audio { ++ gpio = <&gpio0 1 GPIO_ACTIVE_HIGH>; ++}; ++ ++&pp1800_pcie { ++ gpio = <&gpio0 8 GPIO_ACTIVE_HIGH>; ++}; ++ ++&sdmmc { ++ cd-gpios = <&gpio4 26 GPIO_ACTIVE_LOW>; ++}; ++ ++&cros_ec { ++ interrupts = <2 IRQ_TYPE_LEVEL_LOW>; ++}; ++ ++/* Pinctrl overrides for -r0; in same order as parent */ ++ ++&ec_ap_int_l { ++ rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>; ++}; ++ ++&pp1500_en { ++ rockchip,pins = <1 RK_PB3 RK_FUNC_GPIO &pcfg_pull_none>; ++}; ++ ++&pp1800_audio_en { ++ rockchip,pins = <0 RK_PA1 RK_FUNC_GPIO ++ &pcfg_pull_none>; ++}; ++ ++&pp3000_en { ++ rockchip,pins = <1 RK_PB4 RK_FUNC_GPIO ++ &pcfg_pull_none>; ++}; ++ ++&wlan_module_pd_l { ++ rockchip,pins = <0 RK_PB0 RK_FUNC_GPIO ++ &pcfg_pull_none>; ++}; ++ ++&sdmmc_cd_gpio { ++ rockchip,pins = <4 RK_PD2 RK_FUNC_GPIO &pcfg_pull_none>; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-gru-kevin-r1.dts b/arch/arm64/boot/dts/rockchip/rk3399-gru-kevin-r1.dts +new file mode 100755 +index 000000000000..44b04e1606fc +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-gru-kevin-r1.dts +@@ -0,0 +1,85 @@ ++/* ++ * Google Gru-Kevin Rev 1+ board device tree source ++ * ++ * Copyright 2016 Google, Inc ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This file is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This file is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++/dts-v1/; ++#include "rk3399-gru.dtsi" ++ ++/ { ++ model = "Google Kevin"; ++ compatible = "google,kevin-rev15", "google,kevin-rev14", ++ "google,kevin-rev13", "google,kevin-rev12", ++ "google,kevin-rev11", "google,kevin-rev10", ++ "google,kevin-rev9", "google,kevin-rev8", ++ "google,kevin-rev7", "google,kevin-rev6", ++ "google,kevin-rev5", "google,kevin-rev4", ++ "google,kevin-rev3", "google,kevin-rev2", ++ "google,kevin-rev1", "google,kevin", ++ "google,gru", "rockchip,rk3399"; ++}; ++ ++&ap_i2c_tp { ++ trackpad@4a { ++ compatible = "atmel,maxtouch"; ++ reg = <0x4a>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&trackpad_int_l>; ++ interrupt-parent = <&gpio1>; ++ interrupts = <4 IRQ_TYPE_LEVEL_LOW>; ++ linux,gpio-keymap = ; ++ }; ++}; ++ ++/* PINCTRL: always below everything else */ ++ ++&pinctrl { ++ pen-eject { ++ pen_eject_l: pen-eject-l { ++ rockchip,pins = <0 RK_PB5 RK_FUNC_GPIO ++ &pcfg_pull_up>; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-gru-scarlet.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-gru-scarlet.dtsi +index e9ecffc409c0..60cd1c18cd4e 100644 +--- a/arch/arm64/boot/dts/rockchip/rk3399-gru-scarlet.dtsi ++++ b/arch/arm64/boot/dts/rockchip/rk3399-gru-scarlet.dtsi +@@ -245,7 +245,7 @@ &ppvar_gpu_pwm { + }; + + &ppvar_sd_card_io { +- states = <1800000 0x0>, <3300000 0x1>; ++ states = <1800000 0x0 3300000 0x1>; + regulator-max-microvolt = <3300000>; + }; + +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-gru.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-gru.dtsi +index 765b24a2bcbf..32dcaf210085 100644 +--- a/arch/arm64/boot/dts/rockchip/rk3399-gru.dtsi ++++ b/arch/arm64/boot/dts/rockchip/rk3399-gru.dtsi +@@ -247,8 +247,8 @@ ppvar_sd_card_io: ppvar-sd-card-io { + enable-active-high; + enable-gpio = <&gpio2 2 GPIO_ACTIVE_HIGH>; + gpios = <&gpio2 28 GPIO_ACTIVE_HIGH>; +- states = <1800000 0x1>, +- <3000000 0x0>; ++ states = <1800000 0x1 ++ 3000000 0x0>; + + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3000000>; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-linux.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-linux.dtsi +new file mode 100755 +index 000000000000..0549701d615e +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-linux.dtsi +@@ -0,0 +1,306 @@ ++/* ++ * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This file is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This file is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++#include ++#include "rk3399-vop-clk-set.dtsi" ++ ++/ { ++ compatible = "rockchip,linux", "rockchip,rk3399"; ++ ++ chosen { ++ bootargs = "earlycon=uart8250,mmio32,0xff1a0000 console=ttyFIQ0 rw root=PARTUUID=614e0000-0000 rootfstype=ext4 rootwait coherent_pool=1m"; ++ }; ++ ++ reserved-memory { ++ #address-cells = <2>; ++ #size-cells = <2>; ++ ranges; ++ ++ drm_logo: drm-logo@00000000 { ++ compatible = "rockchip,drm-logo"; ++ reg = <0x0 0x0 0x0 0x0>; ++ }; ++ ++ ramoops_mem: region@110000 { ++ reg = <0x0 0x110000 0x0 0xf0000>; ++ reg-names = "ramoops_mem"; ++ }; ++ }; ++ ++ ramoops: ramoops { ++ compatible = "ramoops"; ++ record-size = <0x0 0x40000>; ++ console-size = <0x0 0x80000>; ++ ftrace-size = <0x0 0x00000>; ++ pmsg-size = <0x0 0x00000>; ++ memory-region = <&ramoops_mem>; ++ }; ++ ++ fiq_debugger: fiq-debugger { ++ compatible = "rockchip,fiq-debugger"; ++ rockchip,serial-id = <2>; ++ rockchip,signal-irq = <182>; ++ rockchip,wake-irq = <0>; ++ rockchip,irq-mode-enable = <1>; /* If enable uart uses irq instead of fiq */ ++ rockchip,baudrate = <1500000>; /* Only 115200 and 1500000 */ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart2c_xfer>; ++ }; ++ ++ cif_isp0: cif_isp@ff910000 { ++ compatible = "rockchip,rk3399-cif-isp"; ++ rockchip,grf = <&grf>; ++ reg = <0x0 0xff910000 0x0 0x4000>, <0x0 0xff968000 0x0 0x8000>; ++ reg-names = "register", "dsihost-register"; ++ clocks = ++ <&cru ACLK_ISP0_NOC>, <&cru ACLK_ISP0_WRAPPER>, ++ <&cru HCLK_ISP0_NOC>, <&cru HCLK_ISP0_WRAPPER>, ++ <&cru SCLK_ISP0>, <&cru SCLK_DPHY_RX0_CFG>, ++ <&cru SCLK_CIF_OUT>, <&cru SCLK_CIF_OUT>, ++ <&cru SCLK_MIPIDPHY_REF>; ++ clock-names = ++ "aclk_isp0_noc", "aclk_isp0_wrapper", ++ "hclk_isp0_noc", "hclk_isp0_wrapper", ++ "clk_isp0", "pclk_dphyrx", ++ "clk_cif_out", "clk_cif_pll", ++ "pclk_dphy_ref"; ++ interrupts = ; ++ interrupt-names = "cif_isp10_irq"; ++ power-domains = <&power RK3399_PD_ISP0>; ++ rockchip,isp,iommu-enable = <1>; ++ iommus = <&isp0_mmu>; ++ status = "disabled"; ++ }; ++ ++ cif_isp1: cif_isp@ff920000 { ++ compatible = "rockchip,rk3399-cif-isp"; ++ rockchip,grf = <&grf>; ++ reg = <0x0 0xff920000 0x0 0x4000>, <0x0 0xff968000 0x0 0x8000>; ++ reg-names = "register", "dsihost-register"; ++ clocks = ++ <&cru ACLK_ISP1_NOC>, <&cru ACLK_ISP1_WRAPPER>, ++ <&cru HCLK_ISP1_NOC>, <&cru HCLK_ISP1_WRAPPER>, ++ <&cru SCLK_ISP1>, <&cru PCLK_ISP1_WRAPPER>, ++ <&cru SCLK_DPHY_TX1RX1_CFG>, ++ <&cru PCLK_MIPI_DSI1>, <&cru SCLK_MIPIDPHY_CFG>, ++ <&cru SCLK_CIF_OUT>, <&cru SCLK_CIF_OUT>, ++ <&cru SCLK_MIPIDPHY_REF>; ++ clock-names = ++ "aclk_isp1_noc", "aclk_isp1_wrapper", ++ "hclk_isp1_noc", "hclk_isp1_wrapper", ++ "clk_isp1", "pclkin_isp1", ++ "pclk_dphytxrx", ++ "pclk_mipi_dsi","mipi_dphy_cfg", ++ "clk_cif_out", "clk_cif_pll", ++ "pclk_dphy_ref"; ++ interrupts = ; ++ interrupt-names = "cif_isp10_irq"; ++ power-domains = <&power RK3399_PD_ISP1>; ++ rockchip,isp,iommu-enable = <1>; ++ iommus = <&isp1_mmu>; ++ status = "disabled"; ++ }; ++ ++ rga: rga@ff680000 { ++ compatible = "rockchip,rga2"; ++ dev_mode = <1>; ++ reg = <0x0 0xff680000 0x0 0x1000>; ++ interrupts = ; ++ clocks = <&cru ACLK_RGA>, <&cru HCLK_RGA>, <&cru SCLK_RGA_CORE>; ++ clock-names = "aclk_rga", "hclk_rga", "clk_rga"; ++ power-domains = <&power RK3399_PD_RGA>; ++ status = "okay"; ++ }; ++}; ++ ++&display_subsystem { ++ status = "disabled"; ++ ++ ports = <&vopb_out>, <&vopl_out>; ++ logo-memory-region = <&drm_logo>; ++ ++ route { ++ route_hdmi: route-hdmi { ++ status = "disabled"; ++ logo,uboot = "logo.bmp"; ++ logo,kernel = "logo_kernel.bmp"; ++ logo,mode = "center"; ++ charge_logo,mode = "center"; ++ connect = <&vopb_out_hdmi>; ++ }; ++ ++ route_dsi: route-dsi { ++ status = "disabled"; ++ logo,uboot = "logo.bmp"; ++ logo,kernel = "logo_kernel.bmp"; ++ logo,mode = "center"; ++ charge_logo,mode = "center"; ++ connect = <&vopl_out_dsi>; ++ }; ++ ++ route_edp: route-edp { ++ status = "disabled"; ++ logo,uboot = "logo.bmp"; ++ logo,kernel = "logo_kernel.bmp"; ++ logo,mode = "center"; ++ charge_logo,mode = "center"; ++ connect = <&vopl_out_edp>; ++ }; ++ }; ++}; ++ ++&edp { ++ /delete-property/pinctrl-names; ++ /delete-property/pinctrl-0; ++}; ++ ++&iep { ++ status = "okay"; ++}; ++ ++&iep_mmu { ++ status = "okay"; ++}; ++ ++&mpp_srv { ++ status = "okay"; ++}; ++ ++&pvtm { ++ status = "okay"; ++}; ++ ++&rkvdec { ++ status = "okay"; ++ /* 0 means ion, 1 means drm */ ++ //allocator = <0>; ++}; ++ ++&rkvdec_mmu { ++ status = "okay"; ++}; ++ ++&vdpu { ++ status = "okay"; ++}; ++ ++&vepu { ++ status = "okay"; ++}; ++ ++&vpu_mmu { ++ status = "okay"; ++}; ++ ++&uart2 { ++ status = "disabled"; ++}; ++ ++&pinctrl { ++ isp { ++ cif_clkout: cif-clkout { ++ rockchip,pins = ++ /* cif_clkout */ ++ <2 RK_PB3 3 &pcfg_pull_none>; ++ }; ++ ++ isp_dvp_d0d7: isp-dvp-d0d7 { ++ rockchip,pins = ++ <4 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>, ++ /* cif_clkout */ ++ <2 RK_PB3 3 &pcfg_pull_none>, ++ /* cif_data0 */ ++ <2 RK_PA0 3 &pcfg_pull_none>, ++ /* cif_data1 */ ++ <2 RK_PA1 3 &pcfg_pull_none>, ++ /* cif_data2 */ ++ <2 RK_PA2 3 &pcfg_pull_none>, ++ /* cif_data3 */ ++ <2 RK_PA3 3 &pcfg_pull_none>, ++ /* cif_data4 */ ++ <2 RK_PA4 3 &pcfg_pull_none>, ++ /* cif_data5 */ ++ <2 RK_PA5 3 &pcfg_pull_none>, ++ /* cif_data6 */ ++ <2 RK_PA6 3 &pcfg_pull_none>, ++ /* cif_data7 */ ++ <2 RK_PA7 3 &pcfg_pull_none>, ++ /* cif_sync */ ++ <2 RK_PB0 3 &pcfg_pull_none>, ++ /* cif_href */ ++ <2 RK_PB1 3 &pcfg_pull_none>, ++ /* cif_clkin */ ++ <2 RK_PB2 3 &pcfg_pull_none>; ++ }; ++ ++ isp_shutter: isp-shutter { ++ rockchip,pins = ++ /* SHUTTEREN */ ++ <1 RK_PA1 1 &pcfg_pull_none>, ++ /* SHUTTERTRIG */ ++ <1 RK_PA0 1 &pcfg_pull_none>; ++ }; ++ ++ isp_flash_trigger: isp-flash-trigger { ++ /* ISP_FLASHTRIGOU */ ++ rockchip,pins = <1 RK_PA3 1 &pcfg_pull_none>; ++ }; ++ ++ isp_flash_trigger_as_gpio: isp-flash-trigger-as-gpio { ++ /* ISP_FLASHTRIGOU */ ++ rockchip,pins = <0 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ cam_pins { ++ cam0_default_pins: cam0-default-pins { ++ rockchip,pins = ++ <4 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>, ++ <2 RK_PB3 3 &pcfg_pull_none>; ++ }; ++ cam0_sleep_pins: cam0-sleep-pins { ++ rockchip,pins = ++ <4 RK_PD3 3 &pcfg_pull_none>, ++ <2 RK_PB3 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-mid-818-android.dts b/arch/arm64/boot/dts/rockchip/rk3399-mid-818-android.dts +new file mode 100755 +index 000000000000..8610539ef284 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-mid-818-android.dts +@@ -0,0 +1,1121 @@ ++/* ++ * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This file is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This file is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++/dts-v1/; ++#include ++#include ++#include ++#include ++#include "rk3399.dtsi" ++#include "rk3399-android.dtsi" ++#include "rk3399-opp.dtsi" ++#include "rk3399-vop-clk-set.dtsi" ++ ++/ { ++ compatible = "rockchip,rk3399-mid", "rockchip,rk3399"; ++ chosen: chosen { ++ bootargs = "earlycon=uart8250,mmio32,0xff1a0000 console=ttyFIQ0 androidboot.baseband=N/A androidboot.veritymode=enforcing androidboot.hardware=rk30board androidboot.console=ttyFIQ0 init=/init initrd=0x62000001,0x00800000 coherent_pool=1m"; ++ }; ++ ++ adc_keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 1>; ++ io-channel-names = "buttons"; ++ keyup-threshold-microvolt = <1800000>; ++ poll-interval = <100>; ++ ++ vol-up-key { ++ label = "volume up"; ++ linux,code = ; ++ press-threshold-microvolt = <1000>; ++ }; ++ ++ vol-down-key { ++ label = "volume down"; ++ linux,code = ; ++ press-threshold-microvolt = <170000>; ++ }; ++ }; ++ ++ rk_headset: rk-headset { ++ compatible = "rockchip_headset"; ++ headset_gpio = <&gpio4 RK_PD4 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hp_det>; ++ io-channels = <&saradc 2>; ++ }; ++ ++ charge-animation { ++ compatible = "rockchip,uboot-charge"; ++ rockchip,uboot-charge-on = <1>; ++ rockchip,android-charge-on = <0>; ++ rockchip,uboot-low-power-voltage = <6700>; ++ rockchip,screen-on-voltage = <6800>; ++ status = "okay"; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ clocks = <&rk818 1>; ++ clock-names = "ext_clock"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio0 10 GPIO_ACTIVE_LOW>; /* GPIO0_B2 */ ++ }; ++ ++ hall_sensor: hall-mh248 { ++ compatible = "hall-mh248"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&mh248_irq_gpio>; ++ irq-gpio = <&gpio1 2 IRQ_TYPE_EDGE_BOTH>; ++ hall-active = <1>; ++ status = "okay"; ++ }; ++ ++ vcc_sys: vcc-sys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3900000>; ++ regulator-max-microvolt = <3900000>; ++ }; ++ ++ vcc3v3_sys: vcc3v3-sys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc3v3_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ }; ++ ++ vcc5v0_host: vcc5v0-host-regulator { ++ compatible = "regulator-fixed"; ++ enable-active-high; ++ gpio = <&gpio4 25 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&host_vbus_drv>; ++ regulator-name = "vcc5v0_host"; ++ regulator-always-on; ++ }; ++ ++ vdd_log: vdd-log { ++ compatible = "pwm-regulator"; ++ pwms = <&pwm2 0 25000 1>; ++ rockchip,pwm_id= <2>; ++ rockchip,pwm_voltage = <900000>; ++ regulator-name = "vdd_log"; ++ regulator-min-microvolt = <750000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ xin32k: xin32k { ++ compatible = "fixed-clock"; ++ clock-frequency = <32768>; ++ clock-output-names = "xin32k"; ++ #clock-cells = <0>; ++ }; ++ ++ edp_panel: edp-panel { ++ compatible = "lg,lp079qx1-sp0v", "panel-simple"; ++ bus-format = ; ++ bpc = <6>; ++ backlight = <&backlight>; ++ power-supply = <&vcc3v3_s0>; ++ enable-gpios = <&gpio3 8 GPIO_ACTIVE_HIGH>; ++ ports { ++ panel_in_edp: endpoint { ++ remote-endpoint = <&edp_out_panel>; ++ }; ++ }; ++ }; ++ ++ backlight: backlight { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm0 0 25000 0>; ++ brightness-levels = < ++ 255 200 199 198 197 197 196 195 194 193 193 192 ++ 191 190 189 189 188 187 186 185 185 184 183 182 ++ 181 181 180 179 178 177 177 176 175 174 173 173 ++ 172 171 170 169 169 168 167 166 165 165 164 163 ++ 162 161 161 160 159 158 157 157 156 155 154 153 ++ 153 152 151 150 149 149 148 147 146 145 145 144 ++ 143 142 141 141 140 139 138 137 137 136 135 134 ++ 133 133 132 131 130 129 129 128 127 126 125 125 ++ 124 123 122 121 121 120 119 118 117 117 116 115 ++ 114 113 113 112 111 110 109 109 108 107 106 105 ++ 105 104 103 102 101 101 100 99 98 97 97 96 ++ 95 94 93 93 92 91 90 89 89 88 87 86 ++ 85 85 84 83 82 81 81 80 79 78 77 77 ++ 76 75 74 73 73 72 71 70 69 69 68 67 ++ 66 65 65 64 63 62 61 61 60 59 58 57 ++ 57 56 55 54 53 53 52 51 50 49 49 48 ++ 47 46 45 45 44 43 42 41 41 40 39 38 ++ 37 37 36 35 34 33 33 32 31 30 29 29 ++ 28 27 26 25 25 24 23 22 21 21 20 19 ++ 18 17 17 16 15 14 13 13 12 11 10 9 ++ 9 8 7 6 5 5 4 3 2 1 1 0 ++ 0 0 0 0>; ++ default-brightness-level = <200>; ++ enable-gpios = <&gpio1 13 GPIO_ACTIVE_HIGH>; ++ }; ++ ++ vcc_phy: vcc-phy-regulator { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_phy"; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ es8316-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,es8316-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,widgets = ++ "Microphone", "Mic Jack", ++ "Headphone", "Headphone Jack"; ++ simple-audio-card,routing = ++ "Mic Jack", "MICBIAS1", ++ "IN1P", "Mic Jack", ++ "Headphone Jack", "HPOL", ++ "Headphone Jack", "HPOR"; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s0>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&es8316>; ++ }; ++ }; ++ ++ spdif-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,name = "rockchip,spdif"; ++ simple-audio-card,cpu { ++ sound-dai = <&spdif>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&spdif_out>; ++ }; ++ }; ++ ++ spdif_out: spdif-out { ++ compatible = "linux,spdif-dit"; ++ #sound-dai-cells = <0>; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ clocks = <&rk818 1>; ++ clock-names = "ext_clock"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio0 10 GPIO_ACTIVE_LOW>; /* GPIO0_B2 */ ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ rockchip,grf = <&grf>; ++ wifi_chip_type = "ap6354"; ++ sdio_vref = <1800>; ++ WIFI,host_wake_irq = <&gpio0 3 GPIO_ACTIVE_HIGH>; /* GPIO0_a3 */ ++ status = "okay"; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&rk818 1>; ++ clock-names = "ext_clock"; ++ //wifi-bt-power-toggle; ++ uart_rts_gpios = <&gpio2 19 GPIO_ACTIVE_LOW>; /* GPIO2_C3 */ ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart0_rts>, <&bt_reset_gpio>, <&bt_wake_gpio>, <&bt_irq_gpio>; ++ pinctrl-1 = <&uart0_gpios>; ++ //BT,power_gpio = <&gpio3 19 GPIO_ACTIVE_HIGH>; /* GPIOx_xx */ ++ BT,reset_gpio = <&gpio0 9 GPIO_ACTIVE_HIGH>; /* GPIO0_B1 */ ++ BT,wake_gpio = <&gpio2 26 GPIO_ACTIVE_HIGH>; /* GPIO2_D2 */ ++ BT,wake_host_irq = <&gpio0 4 GPIO_ACTIVE_HIGH>; /* GPIO0_A4 */ ++ status = "okay"; ++ }; ++ ++ vibrator { ++ compatible = "rk-vibrator-gpio"; ++ vibrator-gpio = <&gpio4 30 GPIO_ACTIVE_LOW>; ++ status = "okay"; ++ }; ++ ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&dmc { ++ status = "okay"; ++ center-supply = <&vdd_center>; ++ upthreshold = <40>; ++ downdifferential = <20>; ++ system-status-freq = < ++ /*system status freq(KHz)*/ ++ SYS_STATUS_NORMAL 800000 ++ SYS_STATUS_REBOOT 528000 ++ SYS_STATUS_SUSPEND 200000 ++ SYS_STATUS_VIDEO_1080P 200000 ++ SYS_STATUS_VIDEO_4K 600000 ++ SYS_STATUS_VIDEO_4K_10B 800000 ++ SYS_STATUS_PERFORMANCE 800000 ++ SYS_STATUS_BOOST 600000 ++ SYS_STATUS_DUALVIEW 600000 ++ SYS_STATUS_ISP 600000 ++ >; ++ vop-bw-dmc-freq = < ++ /* min_bw(MB/s) max_bw(MB/s) freq(KHz) */ ++ 0 762 200000 ++ 763 1893 400000 ++ 1894 3012 528000 ++ 3013 99999 800000 ++ >; ++ ++ auto-min-freq = <200000>; ++}; ++ ++&sdmmc { ++ clock-frequency = <50000000>; ++ clock-freq-min-max = <400000 150000000>; ++ no-sdio; ++ no-mmc; ++ bus-width = <4>; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ disable-wp; ++ num-slots = <1>; ++ //sd-uhs-sdr104; ++ vqmmc-supply = <&vcc_sd>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_cd &sdmmc_bus4>; ++ status = "okay"; ++}; ++ ++&sdio0 { ++ clock-frequency = <150000000>; ++ clock-freq-min-max = <200000 150000000>; ++ no-sd; ++ no-mmc; ++ bus-width = <4>; ++ disable-wp; ++ cap-sd-highspeed; ++ cap-sdio-irq; ++ keep-power-in-suspend; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ non-removable; ++ num-slots = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&emmc_phy { ++ status = "okay"; ++}; ++ ++&sdhci { ++ bus-width = <8>; ++ mmc-hs400-1_8v; ++ no-sdio; ++ no-sd; ++ non-removable; ++ keep-power-in-suspend; ++ mmc-hs400-enhanced-strobe; ++ status = "okay"; ++}; ++ ++&i2s0 { ++ status = "okay"; ++ rockchip,i2s-broken-burst-len; ++ rockchip,playback-channels = <8>; ++ rockchip,capture-channels = <8>; ++ #sound-dai-cells = <0>; ++}; ++ ++&i2s2 { ++ #sound-dai-cells = <0>; ++}; ++ ++&spdif { ++ status = "disabled"; ++ #sound-dai-cells = <0>; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <180>; ++ i2c-scl-falling-time-ns = <30>; ++ clock-frequency = <400000>; ++ ++ vdd_cpu_b: syr837@40 { ++ compatible = "silergy,syr827"; ++ reg = <0x40>; ++ vin-supply = <&vcc_sys>; ++ regulator-compatible = "fan53555-reg"; ++ pinctrl-0 = <&vsel1_gpio>; ++ vsel-gpios = <&gpio1 17 GPIO_ACTIVE_HIGH>; ++ regulator-name = "vdd_cpu_b"; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1500000>; ++ regulator-ramp-delay = <1000>; ++ fcs,suspend-voltage-selector = <1>; ++ regulator-always-on; ++ regulator-initial-state = <3>; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdd_gpu: syr828@41 { ++ compatible = "silergy,syr828"; ++ status = "okay"; ++ reg = <0x41>; ++ vin-supply = <&vcc_sys>; ++ regulator-compatible = "fan53555-reg"; ++ pinctrl-0 = <&vsel2_gpio>; ++ vsel-gpios = <&gpio1 14 GPIO_ACTIVE_HIGH>; ++ regulator-name = "vdd_gpu"; ++ regulator-min-microvolt = <735000>; ++ regulator-max-microvolt = <1400000>; ++ regulator-ramp-delay = <1000>; ++ fcs,suspend-voltage-selector = <1>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ rk818: pmic@1c { ++ compatible = "rockchip,rk818"; ++ status = "okay"; ++ reg = <0x1c>; ++ clock-output-names = "rk818-clkout1", "wifibt_32kin"; ++ interrupt-parent = <&gpio1>; ++ interrupts = <21 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pmic_int_l>; ++ rockchip,system-power-controller; ++ rk818,support_dc_chg = <1>;/*1: dc chg; 0:usb chg*/ ++ wakeup-source; ++ extcon = <&fusb0>; ++ #clock-cells = <1>; ++ ++ vcc1-supply = <&vcc_sys>; ++ vcc2-supply = <&vcc_sys>; ++ vcc3-supply = <&vcc_sys>; ++ vcc4-supply = <&vcc_sys>; ++ vcc6-supply = <&vcc_sys>; ++ vcc7-supply = <&vcc3v3_sys>; ++ vcc8-supply = <&vcc_sys>; ++ vcc9-supply = <&vcc3v3_sys>; ++ ++ regulators { ++ vdd_cpu_l: DCDC_REG1 { ++ regulator-name = "vdd_cpu_l"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <750000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdd_center: DCDC_REG2 { ++ regulator-name = "vdd_center"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <800000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-name = "vcc_ddr"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_1v8: DCDC_REG4 { ++ regulator-name = "vcc_1v8"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vcca3v0_codec: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-name = "vcca3v0_codec"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc3v0_tp: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-name = "vcc3v0_tp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcca1v8_codec: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcca1v8_codec"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_power_on: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcc_power_on"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc_3v0: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-name = "vcc_3v0"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ ++ vcc_1v5: LDO_REG6 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ regulator-name = "vcc_1v5"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1500000>; ++ }; ++ }; ++ ++ vcc1v8_dvp: LDO_REG7 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc1v8_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc3v3_s3: LDO_REG8 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcc3v3_s3"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_sd: LDO_REG9 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-name = "vcc_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ ++ vcc3v3_s0: SWITCH_REG { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc3v3_s0"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ boost_otg: DCDC_BOOST { ++ regulator-name = "boost_otg"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <5000000>; ++ }; ++ }; ++ ++ otg_switch: OTG_SWITCH { ++ regulator-name = "otg_switch"; ++ }; ++ }; ++ ++ battery { ++ compatible = "rk818-battery"; ++ ocv_table = <3400 3675 3689 3716 3740 3756 3768 3780 ++ 3793 3807 3827 3853 3896 3937 3974 4007 4066 ++ 4110 4161 4217 4308>; ++ design_capacity = <7916>; ++ design_qmax = <8708>; ++ bat_res = <65>; ++ max_input_current = <3000>; ++ max_chrg_current = <3000>; ++ max_chrg_voltage = <4350>; ++ sleep_enter_current = <300>; ++ sleep_exit_current = <300>; ++ power_off_thresd = <3400>; ++ zero_algorithm_vol = <3950>; ++ fb_temperature = <105>; ++ sample_res = <20>; ++ max_soc_offset = <60>; ++ energy_mode = <0>; ++ monitor_sec = <5>; ++ virtual_power = <0>; ++ power_dc2otg = <0>; ++ }; ++ }; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <140>; ++ i2c-scl-falling-time-ns = <30>; ++ ++ es8316: es8316@10 { ++ #sound-dai-cells = <0>; ++ compatible = "everest,es8316"; ++ reg = <0x11>; ++ clocks = <&cru SCLK_I2S_8CH_OUT>; ++ clock-names = "mclk"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s_8ch_mclk>; ++ spk-con-gpio = <&gpio0 11 GPIO_ACTIVE_HIGH>; ++ }; ++}; ++ ++&i2c4 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <345>; ++ i2c-scl-falling-time-ns = <11>; ++ clock-frequency = <400000>; ++ ++ lsm330_accel@1e { ++ status = "okay"; ++ compatible = "lsm330_acc"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&lsm330a_irq_gpio>; ++ reg = <0x1e>; ++ irq-gpio = <&gpio2 27 IRQ_TYPE_EDGE_RISING>; ++ type = ; ++ irq_enable = <1>; ++ poll_delay_ms = <30>; ++ power-off-in-suspend = <1>; ++ layout = <4>; ++ }; ++ ++ lsm330_gyro@6a { ++ status = "okay"; ++ compatible = "lsm330_gyro"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&lsm330g_irq_gpio>; ++ reg = <0x6a>; ++ irq-gpio = <&gpio1 20 IRQ_TYPE_EDGE_RISING>; ++ type = ; ++ irq_enable = <0>; ++ power-off-in-suspend = <1>; ++ poll_delay_ms = <30>; ++ }; ++ ++ mpu6500@68 { ++ status = "disabled"; ++ compatible = "invensense,mpu6500"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&mpu6500_irq_gpio>; ++ reg = <0x68>; ++ irq-gpio = <&gpio2 27 IRQ_TYPE_EDGE_RISING>; ++ mpu-int_config = <0x10>; ++ mpu-level_shifter = <0>; ++ mpu-orientation = <1 0 0 0 1 0 0 0 1>; ++ orientation-x= <1>; ++ orientation-y= <1>; ++ orientation-z= <0>; ++ support-hw-poweroff = <1>; ++ mpu-debug = <1>; ++ }; ++ ++ sensor@0d { ++ status = "okay"; ++ compatible = "ak8963"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&ak8963_irq_gpio>; ++ reg = <0x0d>; ++ type = ; ++ irq-gpio = <&gpio2 28 IRQ_TYPE_EDGE_RISING>; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ layout = <3>; ++ }; ++ ++ sensor@10 { ++ status = "okay"; ++ compatible = "capella,light_cm3218"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cm3218_irq_gpio>; ++ reg = <0x10>; ++ type = ; ++ irq-gpio = <&gpio4 24 IRQ_TYPE_EDGE_FALLING>; ++ irq_enable = <1>; ++ poll_delay_ms = <30>; ++ }; ++ ++ fusb0: fusb30x@22 { ++ compatible = "fairchild,fusb302"; ++ reg = <0x22>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&fusb0_int>; ++ int-n-gpios = <&gpio1 1 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++}; ++ ++&i2c5 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <150>; ++ i2c-scl-falling-time-ns = <30>; ++ clock-frequency = <400000>; ++ ++ gt9xx: gt9xx@14 { ++ compatible = "goodix,gt9xx"; ++ reg = <0x14>; ++ touch-gpio = <&gpio3 12 IRQ_TYPE_LEVEL_LOW>; ++ reset-gpio = <&gpio3 13 GPIO_ACTIVE_HIGH>; ++ max-x = <1536>; ++ max-y = <2048>; ++ tp-size = <970>; ++ tp-supply = <&vcc3v0_tp>; ++ }; ++}; ++ ++&io_domains { ++ status = "okay"; ++ ++ bt656-supply = <&vcc1v8_dvp>; ++ audio-supply = <&vcca1v8_codec>; ++ sdmmc-supply = <&vcc_sd>; ++ gpio1830-supply = <&vcc_3v0>; ++}; ++ ++&isp0_mmu { ++ status = "okay"; ++}; ++ ++&isp1_mmu { ++ status = "okay"; ++}; ++ ++&cpu_l0 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_l1 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_l2 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_l3 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_b0 { ++ cpu-supply = <&vdd_cpu_b>; ++}; ++ ++&cpu_b1 { ++ cpu-supply = <&vdd_cpu_b>; ++}; ++ ++&gpu { ++ status = "okay"; ++ mali-supply = <&vdd_gpu>; ++}; ++ ++&spi1 { ++ status = "disabled"; ++ max-freq = <50000000>; ++ mpu6500@0 { ++ status = "disabled"; ++ compatible = "inv-spi,mpu6500"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&mpu6500_irq_gpio>; ++ irq-gpio = <&gpio2 27 IRQ_TYPE_EDGE_RISING>; ++ reg = <0>; ++ spi-max-frequency = <1000000>; ++ spi-cpha; ++ spi-cpol; ++ mpu-int_config = <0x00>; ++ mpu-level_shifter = <0>; ++ mpu-orientation = <1 0 0 0 1 0 0 0 1>; ++ orientation-x= <1>; ++ orientation-y= <0>; ++ orientation-z= <1>; ++ support-hw-poweroff = <1>; ++ mpu-debug = <1>; ++ }; ++}; ++ ++&tcphy0 { ++ extcon = <&fusb0>; ++ status = "okay"; ++}; ++ ++&tsadc { ++ rockchip,hw-tshut-mode = <1>; /* tshut mode 0:CRU 1:GPIO */ ++ rockchip,hw-tshut-polarity = <1>; /* tshut polarity 0:LOW 1:HIGH */ ++ status = "okay"; ++}; ++ ++&u2phy0 { ++ status = "okay"; ++ extcon = <&fusb0>; ++ ++ u2phy0_otg: otg-port { ++ status = "okay"; ++ }; ++ ++ u2phy0_host: host-port { ++ phy-supply = <&vcc5v0_host>; ++ status = "okay"; ++ }; ++}; ++ ++&uart0 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart0_xfer &uart0_cts>; ++ status = "okay"; ++}; ++ ++&uart2 { ++ status = "okay"; ++}; ++ ++&usb_host0_ehci { ++ status = "okay"; ++}; ++ ++&usb_host0_ohci { ++ status = "okay"; ++}; ++ ++&usbdrd3_0 { ++ status = "okay"; ++}; ++ ++&usbdrd_dwc3_0 { ++ status = "okay"; ++ extcon = <&fusb0>; ++}; ++ ++&pwm0 { ++ status = "okay"; ++}; ++ ++&pwm2 { ++ status = "okay"; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm2_pin_pull_down>; ++}; ++ ++&saradc { ++ status = "okay"; ++}; ++ ++&rockchip_suspend { ++ status = "okay"; ++ rockchip,sleep-debug-en = <1>; ++ rockchip,sleep-mode-config = < ++ (0 ++ | RKPM_SLP_ARMPD ++ | RKPM_SLP_PERILPPD ++ | RKPM_SLP_DDR_RET ++ | RKPM_SLP_PLLPD ++ | RKPM_SLP_CENTER_PD ++ | RKPM_SLP_OSC_DIS ++ | RKPM_SLP_AP_PWROFF ++ ) ++ >; ++ rockchip,wakeup-config = < ++ (0 ++ | RKPM_GPIO_WKUP_EN ++ ) ++ >; ++ rockchip,pwm-regulator-config = < ++ (0 ++ | PWM2_REGULATOR_EN ++ ) ++ >; ++ rockchip,power-ctrl = ++ <&gpio1 17 GPIO_ACTIVE_HIGH>, ++ <&gpio1 14 GPIO_ACTIVE_HIGH>; ++}; ++ ++&pinctrl { ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <0 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart0_gpios: uart0-gpios { ++ rockchip,pins = <2 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ bt_reset_gpio: bt-reset-gpio { ++ rockchip,pins = <0 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ bt_wake_gpio: bt-wake-gpio { ++ rockchip,pins = <2 RK_PD2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ bt_irq_gpio: bt-irq-gpio { ++ rockchip,pins = <0 RK_PA4 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ pmic { ++ pmic_int_l: pmic-int-l { ++ rockchip,pins = ++ <1 RK_PC5 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ pmic_dvs2: pmic-dvs2 { ++ rockchip,pins = ++ <1 RK_PC2 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ vsel1_gpio: vsel1-gpio { ++ rockchip,pins = ++ <1 RK_PC1 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ vsel2_gpio: vsel2-gpio { ++ rockchip,pins = ++ <1 RK_PB6 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ hallsensor { ++ mh248_irq_gpio: mh248-irq-gpio { ++ rockchip,pins = <1 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ headphone { ++ hp_det: hp-det { ++ rockchip,pins = <4 RK_PD4 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ lsm330_a { ++ lsm330a_irq_gpio: lsm330a-irq-gpio { ++ rockchip,pins = <2 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ lsm330_g { ++ lsm330g_irq_gpio: lsm330g-irq-gpio { ++ rockchip,pins = <1 RK_PC4 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ mpu6500 { ++ mpu6500_irq_gpio: mpu6500-irq-gpio { ++ rockchip,pins = <2 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ ak8963 { ++ ak8963_irq_gpio: ak8963-irq-gpio { ++ rockchip,pins = <2 RK_PD4 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ cm3218 { ++ cm3218_irq_gpio: cm3218-irq-gpio { ++ rockchip,pins = <4 RK_PD0 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ usb2 { ++ host_vbus_drv: host-vbus-drv { ++ rockchip,pins = ++ <4 RK_PD1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ fusb30x { ++ fusb0_int: fusb0-int { ++ rockchip,pins = ++ <1 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++}; ++ ++&edp { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&edp_hpd>; ++ ++ ports { ++ edp_out: port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ edp_out_panel: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&panel_in_edp>; ++ }; ++ }; ++ }; ++}; ++ ++&edp_in_vopl { ++ status = "disabled"; ++}; ++ ++&hdmi { ++ status = "disabled"; ++}; ++ ++&hdmi_in_vopb { ++ status = "disabled"; ++}; ++ ++&cdn_dp { ++ status = "okay"; ++ extcon = <&fusb0>; ++ phys = <&tcphy0_dp>; ++}; ++ ++&dp_in_vopb { ++ status = "disabled"; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ pmu1830-supply = <&vcc_1v8>; ++}; ++ ++&route_edp { ++ status = "okay"; ++ logo,mode = "center"; ++}; ++ ++&vopb { ++ assigned-clocks = <&cru DCLK_VOP0_DIV>; ++ assigned-clock-parents = <&cru PLL_CPLL>; ++}; ++ ++&vopl { ++ assigned-clocks = <&cru DCLK_VOP1_DIV>; ++ assigned-clock-parents = <&cru PLL_VPLL>; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-opp.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-opp.dtsi +index d6f1095abb04..f9f2cc8abec7 100644 +--- a/arch/arm64/boot/dts/rockchip/rk3399-opp.dtsi ++++ b/arch/arm64/boot/dts/rockchip/rk3399-opp.dtsi +@@ -3,35 +3,99 @@ + * Copyright (c) 2016-2017 Fuzhou Rockchip Electronics Co., Ltd + */ + ++#include "rk3399-sched-energy.dtsi" ++ + / { + cluster0_opp: opp-table0 { + compatible = "operating-points-v2"; + opp-shared; + +- opp00 { ++ rockchip,temp-hysteresis = <5000>; ++ rockchip,low-temp = <0>; ++ rockchip,low-temp-min-volt = <900000>; ++ ++ nvmem-cells = <&cpul_leakage>, <&specification_serial_number>, ++ <&customer_demand>; ++ nvmem-cell-names = "cpu_leakage", ++ "specification_serial_number", ++ "customer_demand"; ++ clocks = <&cru PLL_APLLL>; ++ rockchip,avs-scale = <20>; ++ rockchip,bin-scaling-sel = < ++ 0 30 ++ 1 34 ++ >; ++ ++ rockchip,pvtm-voltage-sel = < ++ 0 143500 0 ++ 143501 148500 1 ++ 148501 152000 2 ++ 152001 999999 3 ++ >; ++ rockchip,pvtm-freq = <408000>; ++ rockchip,pvtm-volt = <1000000>; ++ rockchip,pvtm-ch = <0 0>; ++ rockchip,pvtm-sample-time = <1000>; ++ rockchip,pvtm-number = <10>; ++ rockchip,pvtm-error = <1000>; ++ rockchip,pvtm-ref-temp = <41>; ++ rockchip,pvtm-temp-prop = <115 66>; ++ rockchip,pvtm-thermal-zone = "soc-thermal"; ++ ++ opp-408000000 { + opp-hz = /bits/ 64 <408000000>; +- opp-microvolt = <800000>; ++ opp-microvolt = <825000 825000 1250000>; ++ opp-microvolt-L0 = <825000 825000 1250000>; ++ opp-microvolt-L1 = <825000 825000 1250000>; ++ opp-microvolt-L2 = <825000 825000 1250000>; ++ opp-microvolt-L3 = <825000 825000 1250000>; + clock-latency-ns = <40000>; + }; +- opp01 { ++ opp-600000000 { + opp-hz = /bits/ 64 <600000000>; +- opp-microvolt = <800000>; ++ opp-microvolt = <825000 825000 1250000>; ++ opp-microvolt-L0 = <825000 825000 1250000>; ++ opp-microvolt-L1 = <825000 825000 1250000>; ++ opp-microvolt-L2 = <825000 825000 1250000>; ++ opp-microvolt-L3 = <825000 825000 1250000>; ++ clock-latency-ns = <40000>; + }; +- opp02 { ++ opp-816000000 { + opp-hz = /bits/ 64 <816000000>; +- opp-microvolt = <850000>; ++ opp-microvolt = <850000 850000 1250000>; ++ opp-microvolt-L0 = <850000 850000 1250000>; ++ opp-microvolt-L1 = <825000 825000 1250000>; ++ opp-microvolt-L2 = <825000 825000 1250000>; ++ opp-microvolt-L3 = <825000 825000 1250000>; ++ clock-latency-ns = <40000>; ++ opp-suspend; + }; +- opp03 { ++ opp-1008000000 { + opp-hz = /bits/ 64 <1008000000>; +- opp-microvolt = <925000>; ++ opp-microvolt = <925000 925000 1250000>; ++ opp-microvolt-L0 = <925000 925000 1250000>; ++ opp-microvolt-L1 = <900000 900000 1250000>; ++ opp-microvolt-L2 = <875000 875000 1250000>; ++ opp-microvolt-L3 = <850000 850000 1250000>; ++ clock-latency-ns = <40000>; + }; +- opp04 { ++ opp-1200000000 { + opp-hz = /bits/ 64 <1200000000>; +- opp-microvolt = <1000000>; ++ opp-microvolt = <1000000 1000000 1250000>; ++ opp-microvolt-L0 = <1000000 1000000 1250000>; ++ opp-microvolt-L1 = <975000 975000 1250000>; ++ opp-microvolt-L2 = <950000 950000 1250000>; ++ opp-microvolt-L3 = <925000 925000 1250000>; ++ clock-latency-ns = <40000>; + }; +- opp05 { ++ opp-1416000000 { + opp-hz = /bits/ 64 <1416000000>; +- opp-microvolt = <1125000>; ++ opp-microvolt = <1125000 1125000 1250000>; ++ opp-microvolt-L0 = <1125000 1125000 1250000>; ++ opp-microvolt-L1 = <1100000 1100000 1250000>; ++ opp-microvolt-L2 = <1075000 1075000 1200000>; ++ opp-microvolt-L3 = <1050000 1050000 1250000>; ++ clock-latency-ns = <40000>; + }; + }; + +@@ -39,93 +103,253 @@ cluster1_opp: opp-table1 { + compatible = "operating-points-v2"; + opp-shared; + +- opp00 { ++ rockchip,temp-hysteresis = <5000>; ++ rockchip,low-temp = <0>; ++ rockchip,low-temp-min-volt = <900000>; ++ ++ nvmem-cells = <&cpub_leakage>, <&specification_serial_number>, ++ <&customer_demand>; ++ nvmem-cell-names = "cpu_leakage", ++ "specification_serial_number", ++ "customer_demand"; ++ clocks = <&cru PLL_APLLB>; ++ rockchip,avs-scale = <8>; ++ rockchip,bin-scaling-sel = < ++ 0 8 ++ 1 17 ++ >; ++ ++ rockchip,pvtm-voltage-sel = < ++ 0 149000 0 ++ 149001 155000 1 ++ 155001 159000 2 ++ 159001 161000 3 ++ 161001 999999 4 ++ >; ++ rockchip,pvtm-freq = <408000>; ++ rockchip,pvtm-volt = <1000000>; ++ rockchip,pvtm-ch = <1 0>; ++ rockchip,pvtm-sample-time = <1000>; ++ rockchip,pvtm-number = <10>; ++ rockchip,pvtm-error = <1000>; ++ rockchip,pvtm-ref-temp = <41>; ++ rockchip,pvtm-temp-prop = <71 35>; ++ rockchip,pvtm-thermal-zone = "soc-thermal"; ++ ++ opp-408000000 { + opp-hz = /bits/ 64 <408000000>; +- opp-microvolt = <800000>; ++ opp-microvolt = <825000 825000 1250000>; ++ opp-microvolt-L0 = <825000 825000 1250000>; ++ opp-microvolt-L1 = <825000 825000 1250000>; ++ opp-microvolt-L2 = <825000 825000 1250000>; ++ opp-microvolt-L3 = <825000 825000 1250000>; ++ opp-microvolt-L4 = <825000 825000 1250000>; + clock-latency-ns = <40000>; + }; +- opp01 { ++ opp-600000000 { + opp-hz = /bits/ 64 <600000000>; +- opp-microvolt = <800000>; ++ opp-microvolt = <825000 825000 1250000>; ++ opp-microvolt-L0 = <825000 825000 1250000>; ++ opp-microvolt-L1 = <825000 825000 1250000>; ++ opp-microvolt-L2 = <825000 825000 1250000>; ++ opp-microvolt-L3 = <825000 825000 1250000>; ++ opp-microvolt-L4 = <825000 825000 1250000>; ++ clock-latency-ns = <40000>; + }; +- opp02 { ++ opp-816000000 { + opp-hz = /bits/ 64 <816000000>; +- opp-microvolt = <825000>; ++ opp-microvolt = <825000 825000 1250000>; ++ opp-microvolt-L0 = <825000 825000 1250000>; ++ opp-microvolt-L1 = <825000 825000 1250000>; ++ opp-microvolt-L2 = <825000 825000 1250000>; ++ opp-microvolt-L3 = <825000 825000 1250000>; ++ opp-microvolt-L4 = <825000 825000 1250000>; ++ clock-latency-ns = <40000>; ++ opp-suspend; + }; +- opp03 { ++ opp-1008000000 { + opp-hz = /bits/ 64 <1008000000>; +- opp-microvolt = <875000>; ++ opp-microvolt = <875000 875000 1250000>; ++ opp-microvolt-L0 = <875000 875000 1250000>; ++ opp-microvolt-L1 = <850000 850000 1250000>; ++ opp-microvolt-L2 = <850000 850000 1250000>; ++ opp-microvolt-L3 = <850000 850000 1250000>; ++ opp-microvolt-L4 = <850000 850000 1250000>; ++ clock-latency-ns = <40000>; + }; +- opp04 { ++ opp-1200000000 { + opp-hz = /bits/ 64 <1200000000>; +- opp-microvolt = <950000>; ++ opp-microvolt = <950000 950000 1250000>; ++ opp-microvolt-L0 = <950000 950000 1250000>; ++ opp-microvolt-L1 = <925000 925000 1250000>; ++ opp-microvolt-L2 = <900000 900000 1250000>; ++ opp-microvolt-L3 = <875000 875000 1250000>; ++ opp-microvolt-L4 = <875000 875000 1250000>; ++ clock-latency-ns = <40000>; + }; +- opp05 { ++ opp-1416000000 { + opp-hz = /bits/ 64 <1416000000>; +- opp-microvolt = <1025000>; ++ opp-microvolt = <1025000 1025000 1250000>; ++ opp-microvolt-L0 = <1025000 1025000 1250000>; ++ opp-microvolt-L1 = <1000000 1000000 1250000>; ++ opp-microvolt-L2 = <1000000 1000000 1250000>; ++ opp-microvolt-L3 = <975000 975000 1250000>; ++ opp-microvolt-L4 = <975000 975000 1250000>; ++ clock-latency-ns = <40000>; + }; +- opp06 { ++ opp-1608000000 { + opp-hz = /bits/ 64 <1608000000>; +- opp-microvolt = <1100000>; ++ opp-microvolt = <1100000 1100000 1250000>; ++ opp-microvolt-L0 = <1100000 1100000 1250000>; ++ opp-microvolt-L1 = <1075000 1075000 1250000>; ++ opp-microvolt-L2 = <1050000 1050000 1250000>; ++ opp-microvolt-L3 = <1025000 1025000 1250000>; ++ opp-microvolt-L4 = <1025000 1025000 1250000>; ++ clock-latency-ns = <40000>; + }; +- opp07 { ++ opp-1800000000 { + opp-hz = /bits/ 64 <1800000000>; +- opp-microvolt = <1200000>; ++ opp-microvolt = <1200000 1200000 1250000>; ++ opp-microvolt-L0 = <1200000 1200000 1250000>; ++ opp-microvolt-L1 = <1175000 1175000 1250000>; ++ opp-microvolt-L2 = <1150000 1150000 1250000>; ++ opp-microvolt-L3 = <1125000 1125000 1250000>; ++ opp-microvolt-L4 = <1100000 1100000 1250000>; ++ clock-latency-ns = <40000>; + }; + }; + + gpu_opp_table: opp-table2 { + compatible = "operating-points-v2"; + +- opp00 { ++ rockchip,thermal-zone = "soc-thermal"; ++ rockchip,temp-hysteresis = <5000>; ++ rockchip,low-temp = <0>; ++ rockchip,low-temp-min-volt = <900000>; ++ ++ nvmem-cells = <&gpu_leakage>; ++ nvmem-cell-names = "gpu_leakage"; ++ ++ rockchip,pvtm-voltage-sel = < ++ 0 121000 0 ++ 121001 125500 1 ++ 125501 128500 2 ++ 128501 999999 3 ++ >; ++ rockchip,pvtm-freq = <200000>; ++ rockchip,pvtm-volt = <900000>; ++ rockchip,pvtm-ch = <3 0>; ++ rockchip,pvtm-sample-time = <1000>; ++ rockchip,pvtm-number = <10>; ++ rockchip,pvtm-error = <1000>; ++ rockchip,pvtm-ref-temp = <41>; ++ rockchip,pvtm-temp-prop = <46 12>; ++ rockchip,pvtm-thermal-zone = "gpu-thermal"; ++ ++ opp-200000000 { + opp-hz = /bits/ 64 <200000000>; +- opp-microvolt = <800000>; ++ opp-microvolt = <825000>; ++ opp-microvolt-L0 = <825000>; ++ opp-microvolt-L1 = <825000>; ++ opp-microvolt-L2 = <825000>; ++ opp-microvolt-L3 = <825000>; + }; +- opp01 { +- opp-hz = /bits/ 64 <297000000>; +- opp-microvolt = <800000>; ++ opp-300000000 { ++ opp-hz = /bits/ 64 <300000000>; ++ opp-microvolt = <825000>; ++ opp-microvolt-L0 = <825000>; ++ opp-microvolt-L1 = <825000>; ++ opp-microvolt-L2 = <825000>; ++ opp-microvolt-L3 = <825000>; + }; +- opp02 { ++ opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + opp-microvolt = <825000>; ++ opp-microvolt-L0 = <825000>; ++ opp-microvolt-L1 = <825000>; ++ opp-microvolt-L2 = <825000>; ++ opp-microvolt-L3 = <825000>; + }; +- opp03 { +- opp-hz = /bits/ 64 <500000000>; +- opp-microvolt = <875000>; +- }; +- opp04 { ++ opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + opp-microvolt = <925000>; ++ opp-microvolt-L0 = <925000>; ++ opp-microvolt-L1 = <925000>; ++ opp-microvolt-L2 = <900000>; ++ opp-microvolt-L3 = <900000>; + }; +- opp05 { ++ opp-800000000 { + opp-hz = /bits/ 64 <800000000>; + opp-microvolt = <1100000>; ++ opp-microvolt-L0 = <1100000>; ++ opp-microvolt-L1 = <1075000>; ++ opp-microvolt-L2 = <1050000>; ++ opp-microvolt-L3 = <1025000>; ++ }; ++ }; ++ ++ dmc_opp_table: opp-table3 { ++ compatible = "operating-points-v2"; ++ ++ opp-200000000 { ++ opp-hz = /bits/ 64 <200000000>; ++ opp-microvolt = <900000>; ++ }; ++ opp-300000000 { ++ opp-hz = /bits/ 64 <300000000>; ++ opp-microvolt = <900000>; ++ }; ++ opp-400000000 { ++ opp-hz = /bits/ 64 <400000000>; ++ opp-microvolt = <900000>; ++ }; ++ opp-528000000 { ++ opp-hz = /bits/ 64 <528000000>; ++ opp-microvolt = <900000>; ++ }; ++ opp-600000000 { ++ opp-hz = /bits/ 64 <600000000>; ++ opp-microvolt = <900000>; ++ }; ++ opp-800000000 { ++ opp-hz = /bits/ 64 <800000000>; ++ opp-microvolt = <900000>; + }; + }; + }; + + &cpu_l0 { + operating-points-v2 = <&cluster0_opp>; ++ sched-energy-costs = <&RK3399_CPU_COST_0 &RK3399_CLUSTER_COST_0>; + }; + + &cpu_l1 { + operating-points-v2 = <&cluster0_opp>; ++ sched-energy-costs = <&RK3399_CPU_COST_0 &RK3399_CLUSTER_COST_0>; + }; + + &cpu_l2 { + operating-points-v2 = <&cluster0_opp>; ++ sched-energy-costs = <&RK3399_CPU_COST_0 &RK3399_CLUSTER_COST_0>; + }; + + &cpu_l3 { + operating-points-v2 = <&cluster0_opp>; ++ sched-energy-costs = <&RK3399_CPU_COST_0 &RK3399_CLUSTER_COST_0>; + }; + + &cpu_b0 { + operating-points-v2 = <&cluster1_opp>; ++ sched-energy-costs = <&RK3399_CPU_COST_1 &RK3399_CLUSTER_COST_1>; + }; + + &cpu_b1 { + operating-points-v2 = <&cluster1_opp>; ++ sched-energy-costs = <&RK3399_CPU_COST_1 &RK3399_CLUSTER_COST_1>; ++}; ++ ++&dmc { ++ operating-points-v2 = <&dmc_opp_table>; + }; + + &gpu { +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-roc-pc.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-roc-pc.dtsi +index 35b7ab3bf10c..20309076dbac 100644 +--- a/arch/arm64/boot/dts/rockchip/rk3399-roc-pc.dtsi ++++ b/arch/arm64/boot/dts/rockchip/rk3399-roc-pc.dtsi +@@ -384,7 +384,6 @@ regulator-state-mem { + + vcc_sdio: LDO_REG4 { + regulator-name = "vcc_sdio"; +- regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3000000>; +@@ -489,8 +488,6 @@ vdd_gpu: regulator@41 { + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1500000>; + regulator-ramp-delay = <1000>; +- regulator-always-on; +- regulator-boot-on; + vin-supply = <&vcc3v3_sys>; + + regulator-state-mem { +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-rock960-ab.dts b/arch/arm64/boot/dts/rockchip/rk3399-rock960-ab.dts +new file mode 100755 +index 000000000000..86e45dc0074d +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-rock960-ab.dts +@@ -0,0 +1,1088 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++ ++#include ++#include ++#include "rk3399.dtsi" ++#include "rk3399-linux.dtsi" ++#include "rk3399-opp.dtsi" ++ ++/ { ++ ++ model = "ROCK960 - 96boards based on Rockchip RK3399"; ++ compatible = "rockchip,rock960","rockchip,rk3399"; ++ ++ fiq_debugger: fiq-debugger { ++ compatible = "rockchip,fiq-debugger"; ++ rockchip,serial-id = <2>; ++ rockchip,signal-irq = <182>; ++ rockchip,wake-irq = <0>; ++ rockchip,irq-mode-enable = <1>; ++ rockchip,baudrate = <1500000>; /* Only 115200 and 1500000 */ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart2c_xfer>; ++ }; ++ ++ vcc1v8_s0: vcc1v8-s0 { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc1v8_s0"; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-always-on; ++ }; ++ ++ vcc_sys: vcc-sys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_sys"; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ regulator-always-on; ++ }; ++ ++ vcc_phy: vcc-phy-regulator { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_phy"; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ vcc3v3_sys: vcc3v3-sys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc3v3_sys"; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-always-on; ++ vin-supply = <&vcc_sys>; ++ }; ++ ++ vcc3v3_pcie: vcc3v3-pcie-regulator { ++ compatible = "regulator-fixed"; ++ gpio = <&gpio3 11 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pcie_drv>; ++ regulator-boot-on; ++ regulator-always-on; ++ regulator-name = "vcc3v3_pcie"; ++ vin-supply = <&vcc3v3_sys>; ++ }; ++ ++ vcc5v0_host: vcc5v0-host-regulator { ++ compatible = "regulator-fixed"; ++ enable-active-high; ++ gpio = <&gpio4 25 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&host_vbus_drv>; ++ regulator-name = "vcc5v0_host"; ++ regulator-always-on; ++ }; ++ ++ vdd_log: vdd-log { ++ compatible = "pwm-regulator"; ++ pwms = <&pwm2 0 25000 1>; ++ regulator-name = "vdd_log"; ++ regulator-min-microvolt = <800000>; ++ regulator-max-microvolt = <1400000>; ++ regulator-always-on; ++ regulator-boot-on; ++ ++ /* for rockchip boot on */ ++ rockchip,pwm_id= <2>; ++ rockchip,pwm_voltage = <900000>; ++ ++ vin-supply = <&vcc_sys>; ++ }; ++ ++ clkin_gmac: external-gmac-clock { ++ compatible = "fixed-clock"; ++ clock-frequency = <125000000>; ++ clock-output-names = "clkin_gmac"; ++ #clock-cells = <0>; ++ }; ++ ++ hdmi_codec: hdmi-codec { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,name = "HDMI-CODEC"; ++ ++ simple-audio-card,cpu { ++ sound-dai = <&i2s2>; ++ }; ++ ++ simple-audio-card,codec { ++ sound-dai = <&hdmi>; ++ }; ++ }; ++ ++ spdif-sound { ++ status = "okay"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,name = "ROCKCHIP,SPDIF"; ++ simple-audio-card,cpu { ++ sound-dai = <&spdif>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&spdif_out>; ++ }; ++ }; ++ ++ spdif_out: spdif-out { ++ status = "okay"; ++ compatible = "linux,spdif-dit"; ++ #sound-dai-cells = <0>; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ clocks = <&rk808 1>; ++ clock-names = "ext_clock"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio0 10 GPIO_ACTIVE_LOW>; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ rockchip,grf = <&grf>; ++ wifi_chip_type = "ap6354"; ++ sdio_vref = <1800>; ++ WIFI,host_wake_irq = <&gpio0 3 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&rk808 1>; ++ clock-names = "ext_clock"; ++ /* wifi-bt-power-toggle; */ ++ uart_rts_gpios = <&gpio2 19 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart0_rts>; ++ pinctrl-1 = <&uart0_gpios>; ++ /* BT,power_gpio = <&gpio3 19 GPIO_ACTIVE_HIGH>; */ ++ BT,reset_gpio = <&gpio0 9 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio2 27 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio0 4 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ test-power { ++ status = "okay"; ++ }; ++}; ++ ++&hdmi { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ #sound-dai-cells = <0>; ++ status = "okay"; ++}; ++ ++&sdmmc { ++ clock-frequency = <100000000>; ++ clock-freq-min-max = <100000 100000000>; ++ no-sdio; ++ no-mmc; ++ bus-width = <4>; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ disable-wp; ++ num-slots = <1>; ++ //sd-uhs-sdr104; ++ vqmmc-supply = <&vcc_sd>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_cd &sdmmc_bus4>; ++ card-detect-delay = <800>; ++ status = "okay"; ++}; ++ ++&sdio0 { ++ clock-frequency = <100000000>; ++ clock-freq-min-max = <200000 100000000>; ++ no-sd; ++ no-mmc; ++ bus-width = <4>; ++ disable-wp; ++ cap-sd-highspeed; ++ cap-sdio-irq; ++ keep-power-in-suspend; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ non-removable; ++ num-slots = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&emmc_phy { ++ status = "okay"; ++}; ++ ++&sdhci { ++ bus-width = <8>; ++ mmc-hs400-1_8v; ++ no-sdio; ++ no-sd; ++ non-removable; ++ mmc-hs400-enhanced-strobe; ++ status = "okay"; ++}; ++ ++&i2s0 { ++ status = "okay"; ++ rockchip,i2s-broken-burst-len; ++ rockchip,playback-channels = <8>; ++ rockchip,capture-channels = <8>; ++ #sound-dai-cells = <0>; ++}; ++ ++&i2s2 { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++}; ++ ++&spdif { ++ pinctrl-0 = <&spdif_bus_1>; ++ status = "okay"; ++ #sound-dai-cells = <0>; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <168>; ++ i2c-scl-falling-time-ns = <4>; ++ clock-frequency = <400000>; ++ ++ vdd_cpu_b: syr827@40 { ++ compatible = "silergy,syr827"; ++ reg = <0x40>; ++ regulator-compatible = "fan53555-reg"; ++ pinctrl-0 = <&vsel1_gpio>; ++ vsel-gpios = <&gpio1 17 GPIO_ACTIVE_HIGH>; ++ regulator-name = "vdd_cpu_b"; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1500000>; ++ regulator-ramp-delay = <1000>; ++ fcs,suspend-voltage-selector = <1>; ++ regulator-always-on; ++ regulator-boot-on; ++ vin-supply = <&vcc_sys>; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdd_gpu: syr828@41 { ++ compatible = "silergy,syr828"; ++ reg = <0x41>; ++ regulator-compatible = "fan53555-reg"; ++ pinctrl-0 = <&vsel2_gpio>; ++ vsel-gpios = <&gpio1 14 GPIO_ACTIVE_HIGH>; ++ regulator-name = "vdd_gpu"; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1500000>; ++ regulator-ramp-delay = <1000>; ++ fcs,suspend-voltage-selector = <1>; ++ regulator-always-on; ++ regulator-boot-on; ++ vin-supply = <&vcc_sys>; ++ regulator-initial-mode = <1>; /* 1:force PWM 2:auto */ ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ rk808: pmic@1b { ++ compatible = "rockchip,rk808"; ++ reg = <0x1b>; ++ interrupt-parent = <&gpio1>; ++ interrupts = <21 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pmic_int_l>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "xin32k", "rk808-clkout2"; ++ ++ vcc1-supply = <&vcc_sys>; ++ vcc2-supply = <&vcc_sys>; ++ vcc3-supply = <&vcc_sys>; ++ vcc4-supply = <&vcc_sys>; ++ vcc6-supply = <&vcc_sys>; ++ vcc7-supply = <&vcc_sys>; ++ vcc8-supply = <&vcc3v3_sys>; ++ vcc9-supply = <&vcc_sys>; ++ vcc10-supply = <&vcc_sys>; ++ vcc11-supply = <&vcc_sys>; ++ vcc12-supply = <&vcc3v3_sys>; ++ vddio-supply = <&vcc_1v8>; ++ ++ regulators { ++ vdd_center: DCDC_REG1 { ++ regulator-name = "vdd_center"; ++ regulator-min-microvolt = <750000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdd_cpu_l: DCDC_REG2 { ++ regulator-name = "vdd_cpu_l"; ++ regulator-min-microvolt = <750000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-name = "vcc_ddr"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_1v8: DCDC_REG4 { ++ regulator-name = "vcc_1v8"; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vcc1v8_dvp: LDO_REG1 { ++ regulator-name = "vcc1v8_dvp"; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vcca1v8_hdmi: LDO_REG2 { ++ regulator-name = "vcca1v8_hdmi"; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vcca_1v8: LDO_REG3 { ++ regulator-name = "vcca_1v8"; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vcc_sd: LDO_REG4 { ++ regulator-name = "vcc_sd"; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ ++ vcc3v0_sd: LDO_REG5 { ++ regulator-name = "vcc3v0_sd"; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ ++ vcc_1v5: LDO_REG6 { ++ regulator-name = "vcc_1v5"; ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1500000>; ++ }; ++ }; ++ ++ vcca0v9_hdmi: LDO_REG7 { ++ regulator-name = "vcca0v9_hdmi"; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <900000>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <900000>; ++ }; ++ }; ++ ++ vcc_3v0: LDO_REG8 { ++ regulator-name = "vcc_3v0"; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ ++ vcc3v3_s3: SWITCH_REG1 { ++ regulator-name = "vcc3v3_s3"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc3v3_s0: SWITCH_REG2 { ++ regulator-name = "vcc3v3_s0"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ }; ++ }; ++}; ++ ++&i2c1 { ++ status = "okay"; ++}; ++ ++&i2c6 { ++ status = "okay"; ++}; ++ ++&i2c4 { ++ status = "okay"; ++ fusb0: fusb30x@22 { ++ compatible = "fairchild,fusb302"; ++ reg = <0x22>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&fusb0_int>; ++ vbus-5v-gpios = <&gpio1 3 GPIO_ACTIVE_HIGH>; ++ int-n-gpios = <&gpio1 2 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++}; ++ ++&i2c2 { ++ status = "okay"; ++ camera0: camera-module@10 { ++ status = "disabled"; ++ compatible = "omnivision,ov13850-v4l2-i2c-subdev"; ++ reg = < 0x10 >; ++ device_type = "v4l2-i2c-subdev"; ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "clk_cif_out"; ++ pinctrl-names = "rockchip,camera_default", ++ "rockchip,camera_sleep"; ++ pinctrl-0 = <&cam0_default_pins>; ++ pinctrl-1 = <&cam0_sleep_pins>; ++ //rockchip,pd-gpio = <&gpio4 4 GPIO_ACTIVE_LOW>; ++ rockchip,pwr-gpio = <&gpio4 4 GPIO_ACTIVE_HIGH>; ++ rockchip,rst-gpio = <&gpio3 29 GPIO_ACTIVE_LOW>; ++ rockchip,camera-module-mclk-name = "clk_cif_out"; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "cmk-cb0695-fv1"; ++ rockchip,camera-module-len-name = "lg9569a2"; ++ rockchip,camera-module-fov-h = "66.0"; ++ rockchip,camera-module-fov-v = "50.1"; ++ rockchip,camera-module-orientation = <0>; ++ rockchip,camera-module-iq-flip = <0>; ++ rockchip,camera-module-iq-mirror = <0>; ++ rockchip,camera-module-flip = <1>; ++ rockchip,camera-module-mirror = <0>; ++ ++ rockchip,camera-module-defrect0 = <2112 1568 0 0 2112 1568>; ++ rockchip,camera-module-defrect1 = <4224 3136 0 0 4224 3136>; ++ rockchip,camera-module-defrect3 = <3264 2448 0 0 3264 2448>; ++ rockchip,camera-module-flash-support = <1>; ++ rockchip,camera-module-mipi-dphy-index = <0>; ++ }; ++ ++ camera1: camera-module@36 { ++ status = "disabled"; ++ compatible = "omnivision,ov4690-v4l2-i2c-subdev"; ++ reg = <0x36>; ++ device_type = "v4l2-i2c-subdev"; ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "clk_cif_out"; ++ pinctrl-names = "rockchip,camera_default", ++ "rockchip,camera_sleep"; ++ pinctrl-0 = <&cam0_default_pins>; ++ pinctrl-1 = <&cam0_sleep_pins>; ++ rockchip,pd-gpio = <&gpio3 4 GPIO_ACTIVE_LOW>; ++ //rockchip,pwr-gpio = <&gpio3 13 0>; ++ rockchip,rst-gpio = <&gpio2 10 GPIO_ACTIVE_LOW>; ++ rockchip,camera-module-mclk-name = "clk_cif_out"; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "LA6111PA"; ++ rockchip,camera-module-len-name = "YM6011P"; ++ rockchip,camera-module-fov-h = "116"; ++ rockchip,camera-module-fov-v = "61"; ++ rockchip,camera-module-orientation = <0>; ++ rockchip,camera-module-iq-flip = <0>; ++ rockchip,camera-module-iq-mirror = <0>; ++ rockchip,camera-module-flip = <0>; ++ rockchip,camera-module-mirror = <1>; ++ ++ rockchip,camera-module-defrect0 = <2688 1520 0 0 2688 1520>; ++ rockchip,camera-module-flash-support = <0>; ++ rockchip,camera-module-mipi-dphy-index = <0>; ++ }; ++ ++}; ++ ++&cpu_l0 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_l1 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_l2 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_l3 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_b0 { ++ cpu-supply = <&vdd_cpu_b>; ++}; ++ ++&cpu_b1 { ++ cpu-supply = <&vdd_cpu_b>; ++}; ++ ++&gpu { ++ status = "okay"; ++ mali-supply = <&vdd_gpu>; ++}; ++ ++&threshold { ++ temperature = <85000>; ++}; ++ ++&target { ++ temperature = <100000>; ++}; ++ ++&soc_crit { ++ temperature = <105000>; ++}; ++ ++&tcphy0 { ++ extcon = <&fusb0>; ++ status = "okay"; ++}; ++ ++&tcphy1 { ++ status = "okay"; ++}; ++ ++&tsadc { ++ /* tshut mode 0:CRU 1:GPIO */ ++ rockchip,hw-tshut-mode = <1>; ++ /* tshut polarity 0:LOW 1:HIGH */ ++ rockchip,hw-tshut-polarity = <1>; ++ rockchip,hw-tshut-temp = <110000>; ++ status = "okay"; ++}; ++ ++&u2phy0 { ++ status = "okay"; ++ extcon = <&fusb0>; ++ ++ u2phy0_otg: otg-port { ++ status = "okay"; ++ }; ++ ++ u2phy0_host: host-port { ++ phy-supply = <&vcc5v0_host>; ++ status = "okay"; ++ }; ++}; ++ ++&u2phy1 { ++ status = "okay"; ++ ++ u2phy1_otg: otg-port { ++ status = "okay"; ++ }; ++ ++ u2phy1_host: host-port { ++ phy-supply = <&vcc5v0_host>; ++ status = "okay"; ++ }; ++}; ++ ++&uart0 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart0_xfer &uart0_cts>; ++ dmas = <&dmac_peri 0>, <&dmac_peri 1>; ++ dma-names = "tx", "rx"; ++ status = "okay"; ++}; ++ ++&uart3 { ++ compatible = "rockchip,rk3399-uart", "snps,dw-apb-uart"; ++ reg = <0x0 0xff1b0000 0x0 0x100>; ++ clocks = <&cru SCLK_UART3>, <&cru PCLK_UART3>; ++ clock-names = "baudclk", "apb_pclk"; ++ interrupts = ; ++ dmas = <&dmac_peri 6>, <&dmac_peri 7>; ++ dma-names = "tx", "rx"; ++ reg-shift = <2>; ++ reg-io-width = <4>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart3_xfer &uart3_cts &uart3_rts>; ++ status = "okay"; ++}; ++ ++&uart4 { ++ status = "okay"; ++ dmas = <&dmac_peri 8>, <&dmac_peri 9>; ++ dma-names = "tx", "rx"; ++}; ++ ++&usb_host0_ehci { ++ status = "okay"; ++}; ++ ++&usb_host0_ohci { ++ status = "okay"; ++}; ++ ++&usb_host1_ehci { ++ status = "okay"; ++}; ++ ++&usb_host1_ohci { ++ status = "okay"; ++}; ++ ++&usbdrd3_0 { ++ status = "okay"; ++}; ++ ++&usbdrd_dwc3_0 { ++ dr_mode = "otg"; ++ status = "okay"; ++ extcon = <&fusb0>; ++}; ++ ++&usbdrd3_1 { ++ status = "okay"; ++}; ++ ++&usbdrd_dwc3_1 { ++ dr_mode = "host"; ++ status = "okay"; ++}; ++ ++&pwm2 { ++ status = "okay"; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm2_pin_pull_down>; ++}; ++ ++&pwm3 { ++ status = "okay"; ++ ++ interrupts = ; ++ compatible = "rockchip,remotectl-pwm"; ++ remote_pwm_id = <3>; ++ handle_cpu_id = <1>; ++ remote_support_psci = <1>; ++ ++ ir_key1 { ++ rockchip,usercode = <0x4040>; ++ rockchip,key_table = ++ <0xf2 KEY_REPLY>, ++ <0xba KEY_BACK>, ++ <0xf4 KEY_UP>, ++ <0xf1 KEY_DOWN>, ++ <0xef KEY_LEFT>, ++ <0xee KEY_RIGHT>, ++ <0xbd KEY_HOME>, ++ <0xea KEY_VOLUMEUP>, ++ <0xe3 KEY_VOLUMEDOWN>, ++ <0xe2 KEY_SEARCH>, ++ <0xb2 KEY_POWER>, ++ <0xbc KEY_MUTE>, ++ <0xec KEY_MENU>, ++ <0xbf 0x190>, ++ <0xe0 0x191>, ++ <0xe1 0x192>, ++ <0xe9 183>, ++ <0xe6 248>, ++ <0xe8 185>, ++ <0xe7 186>, ++ <0xf0 388>, ++ <0xbe 0x175>; ++ }; ++ ++ ir_key2 { ++ rockchip,usercode = <0xff00>; ++ rockchip,key_table = ++ <0xf9 KEY_HOME>, ++ <0xbf KEY_BACK>, ++ <0xfb KEY_MENU>, ++ <0xaa KEY_REPLY>, ++ <0xb9 KEY_UP>, ++ <0xe9 KEY_DOWN>, ++ <0xb8 KEY_LEFT>, ++ <0xea KEY_RIGHT>, ++ <0xeb KEY_VOLUMEDOWN>, ++ <0xef KEY_VOLUMEUP>, ++ <0xf7 KEY_MUTE>, ++ <0xe7 KEY_POWER>, ++ <0xfc KEY_POWER>, ++ <0xa9 KEY_VOLUMEDOWN>, ++ <0xa8 KEY_VOLUMEDOWN>, ++ <0xe0 KEY_VOLUMEDOWN>, ++ <0xa5 KEY_VOLUMEDOWN>, ++ <0xab 183>, ++ <0xb7 388>, ++ <0xe8 388>, ++ <0xf8 184>, ++ <0xaf 185>, ++ <0xed KEY_VOLUMEDOWN>, ++ <0xee 186>, ++ <0xb3 KEY_VOLUMEDOWN>, ++ <0xf1 KEY_VOLUMEDOWN>, ++ <0xf2 KEY_VOLUMEDOWN>, ++ <0xf3 KEY_SEARCH>, ++ <0xb4 KEY_VOLUMEDOWN>, ++ <0xbe KEY_SEARCH>; ++ }; ++ ++ ir_key3 { ++ rockchip,usercode = <0x1dcc>; ++ rockchip,key_table = ++ <0xee KEY_REPLY>, ++ <0xf0 KEY_BACK>, ++ <0xf8 KEY_UP>, ++ <0xbb KEY_DOWN>, ++ <0xef KEY_LEFT>, ++ <0xed KEY_RIGHT>, ++ <0xfc KEY_HOME>, ++ <0xf1 KEY_VOLUMEUP>, ++ <0xfd KEY_VOLUMEDOWN>, ++ <0xb7 KEY_SEARCH>, ++ <0xff KEY_POWER>, ++ <0xf3 KEY_MUTE>, ++ <0xbf KEY_MENU>, ++ <0xf9 0x191>, ++ <0xf5 0x192>, ++ <0xb3 388>, ++ <0xbe KEY_1>, ++ <0xba KEY_2>, ++ <0xb2 KEY_3>, ++ <0xbd KEY_4>, ++ <0xf9 KEY_5>, ++ <0xb1 KEY_6>, ++ <0xfc KEY_7>, ++ <0xf8 KEY_8>, ++ <0xb0 KEY_9>, ++ <0xb6 KEY_0>, ++ <0xb5 KEY_BACKSPACE>; ++ }; ++}; ++ ++&gmac { ++ phy-supply = <&vcc_phy>; ++ phy-mode = "rgmii"; ++ clock_in_out = "input"; ++ snps,reset-gpio = <&gpio3 15 GPIO_ACTIVE_LOW>; ++ snps,reset-active-low; ++ snps,reset-delays-us = <0 10000 50000>; ++ assigned-clocks = <&cru SCLK_RMII_SRC>; ++ assigned-clock-parents = <&clkin_gmac>; ++ pinctrl-names = "default", "sleep"; ++ pinctrl-0 = <&rgmii_pins>; ++ pinctrl-1 = <&rgmii_sleep_pins>; ++ tx_delay = <0x28>; ++ rx_delay = <0x11>; ++ status = "disabled"; ++}; ++ ++&saradc { ++ status = "okay"; ++}; ++ ++&io_domains { ++ status = "okay"; ++ ++ bt656-supply = <&vcc1v8_s0>; /* bt656_gpio2ab_ms */ ++ audio-supply = <&vcc1v8_s0>; /* audio_gpio3d4a_ms */ ++ sdmmc-supply = <&vcc_sd>; /* sdmmc_gpio4b_ms */ ++ gpio1830-supply = <&vcc_3v0>; /* gpio1833_gpio4cd_ms */ ++}; ++ ++&pcie_phy { ++ status = "okay"; ++}; ++ ++&pcie0 { ++ ep-gpios = <&gpio3 9 GPIO_ACTIVE_HIGH>; ++ num-lanes = <4>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pcie_clkreqn_cpm>; ++ status = "okay"; ++}; ++ ++&pinctrl { ++ ++ sdio0 { ++ sdio0_bus1: sdio0-bus1 { ++ rockchip,pins = ++ <2 RK_PC4 1 &pcfg_pull_up_20ma>; ++ }; ++ ++ sdio0_bus4: sdio0-bus4 { ++ rockchip,pins = ++ <2 RK_PC4 1 &pcfg_pull_up_20ma>, ++ <2 RK_PC5 1 &pcfg_pull_up_20ma>, ++ <2 RK_PC6 1 &pcfg_pull_up_20ma>, ++ <2 RK_PC7 1 &pcfg_pull_up_20ma>; ++ }; ++ ++ sdio0_cmd: sdio0-cmd { ++ rockchip,pins = ++ <2 RK_PD0 1 &pcfg_pull_up_20ma>; ++ }; ++ ++ sdio0_clk: sdio0-clk { ++ rockchip,pins = ++ <2 RK_PD1 1 &pcfg_pull_none_20ma>; ++ }; ++ }; ++ ++ sdmmc { ++ sdmmc_bus1: sdmmc-bus1 { ++ rockchip,pins = ++ <4 RK_PB0 1 &pcfg_pull_up_8ma>; ++ }; ++ ++ sdmmc_bus4: sdmmc-bus4 { ++ rockchip,pins = ++ <4 RK_PB0 1 &pcfg_pull_up_8ma>, ++ <4 RK_PB1 1 &pcfg_pull_up_8ma>, ++ <4 RK_PB2 1 &pcfg_pull_up_8ma>, ++ <4 RK_PB3 1 &pcfg_pull_up_8ma>; ++ }; ++ ++ sdmmc_clk: sdmmc-clk { ++ rockchip,pins = ++ <4 RK_PB4 1 &pcfg_pull_none_18ma>; ++ }; ++ ++ sdmmc_cmd: sdmmc-cmd { ++ rockchip,pins = ++ <4 RK_PB5 1 &pcfg_pull_up_8ma>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = ++ <0 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart0_gpios: uart0-gpios { ++ rockchip,pins = ++ <2 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ usb2 { ++ host_vbus_drv: host-vbus-drv { ++ rockchip,pins = ++ <4 RK_PD1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ pcie { ++ pcie_drv: pcie-drv { ++ rockchip,pins = ++ <3 RK_PB3 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ pmic { ++ pmic_int_l: pmic-int-l { ++ rockchip,pins = ++ <1 RK_PC5 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ vsel1_gpio: vsel1-gpio { ++ rockchip,pins = ++ <1 RK_PC1 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ ++ vsel2_gpio: vsel2-gpio { ++ rockchip,pins = ++ <1 RK_PB6 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ gmac { ++ rgmii_sleep_pins: rgmii-sleep-pins { ++ rockchip,pins = ++ <3 RK_PB7 RK_FUNC_GPIO &pcfg_output_low>; ++ }; ++ }; ++ ++ fusb30x { ++ fusb0_int: fusb0-int { ++ rockchip,pins = ++ <1 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++}; ++ ++&pvtm { ++ status = "okay"; ++}; ++ ++&pmu_pvtm { ++ status = "okay"; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ pmu1830-supply = <&vcc_1v8>; ++}; ++ ++&rockchip_suspend { ++ status = "okay"; ++ rockchip,sleep-debug-en = <0>; ++ rockchip,sleep-mode-config = < ++ (0 ++ | RKPM_SLP_ARMPD ++ | RKPM_SLP_PERILPPD ++ | RKPM_SLP_DDR_RET ++ | RKPM_SLP_PLLPD ++ | RKPM_SLP_CENTER_PD ++ | RKPM_SLP_AP_PWROFF ++ ) ++ >; ++ rockchip,wakeup-config = < ++ (0 ++ | RKPM_GPIO_WKUP_EN ++ | RKPM_PWM_WKUP_EN ++ ) ++ >; ++ rockchip,pwm-regulator-config = < ++ (0 ++ | PWM2_REGULATOR_EN ++ ) ++ >; ++ rockchip,power-ctrl = ++ <&gpio1 17 GPIO_ACTIVE_HIGH>, ++ <&gpio1 14 GPIO_ACTIVE_HIGH>; ++}; ++ ++&vopb { ++ status = "okay"; ++}; ++ ++&vopb_mmu { ++ status = "okay"; ++}; ++ ++&vopl { ++ status = "okay"; ++}; ++ ++&vopl_mmu { ++ status = "okay"; ++}; ++ ++&cif_isp0 { ++ rockchip,camera-modules-attached = <&camera0>; ++ status = "okay"; ++}; ++ ++&isp0_mmu { ++ status = "okay"; ++}; ++ ++&cif_isp1 { ++ rockchip,camera-modules-attached = <&camera1>; ++ status = "disabled"; ++}; ++ ++&isp1_mmu { ++ status = "okay"; ++}; ++ ++&vpu { ++ status = "okay"; ++ /* 0 means ion, 1 means drm */ ++ //allocator = <0>; ++}; ++ ++&rkvdec { ++ status = "okay"; ++ /* 0 means ion, 1 means drm */ ++ //allocator = <0>; ++}; ++ ++&display_subsystem { ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-rock960.dts b/arch/arm64/boot/dts/rockchip/rk3399-rock960.dts +index c88295782e7b..1a23e8f3cdf6 100644 +--- a/arch/arm64/boot/dts/rockchip/rk3399-rock960.dts ++++ b/arch/arm64/boot/dts/rockchip/rk3399-rock960.dts +@@ -63,6 +63,20 @@ bt_active_led: led-6 { + + }; + ++&cpu_alert0 { ++ temperature = <65000>; ++}; ++ ++&cpu_thermal { ++ sustainable-power = <1550>; ++ ++ cooling-maps { ++ map0 { ++ trip = <&cpu_alert1>; ++ }; ++ }; ++}; ++ + &pcie0 { + ep-gpios = <&gpio2 RK_PA2 GPIO_ACTIVE_HIGH>; + }; +@@ -125,45 +139,6 @@ &spi4 { + status = "okay"; + }; + +-&thermal_zones { +- cpu_thermal: cpu { +- polling-delay-passive = <100>; +- polling-delay = <1000>; +- thermal-sensors = <&tsadc 0>; +- sustainable-power = <1550>; +- +- trips { +- cpu_alert0: cpu_alert0 { +- temperature = <65000>; +- hysteresis = <2000>; +- type = "passive"; +- }; +- +- cpu_alert1: cpu_alert1 { +- temperature = <75000>; +- hysteresis = <2000>; +- type = "passive"; +- }; +- +- cpu_crit: cpu_crit { +- temperature = <95000>; +- hysteresis = <2000>; +- type = "critical"; +- }; +- }; +- +- cooling-maps { +- map0 { +- +- trip = <&cpu_alert1>; +- cooling-device = +- <&cpu_b0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, +- <&cpu_b1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; +- }; +- }; +- }; +-}; +- + &usbdrd_dwc3_0 { + dr_mode = "otg"; + }; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-box.dts b/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-box.dts +new file mode 100755 +index 000000000000..3106512b87cd +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-box.dts +@@ -0,0 +1,181 @@ ++/* ++ * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This file is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This file is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++/dts-v1/; ++ ++#include ++#include "rk3399-excavator-sapphire.dtsi" ++#include "rk3399-android.dtsi" ++ ++/ { ++ compatible = "rockchip,rk3399-excavator-box", "rockchip,rk3399"; ++ ++ test-power { ++ status = "okay"; ++ }; ++}; ++ ++&firmware_android { ++ compatible = "android,firmware"; ++ fstab { ++ compatible = "android,fstab"; ++ system { ++ compatible = "android,system"; ++ dev = "/dev/block/by-name/system"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,verify"; ++ }; ++ vendor { ++ compatible = "android,vendor"; ++ dev = "/dev/block/by-name/vendor"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,verify"; ++ }; ++ }; ++}; ++ ++&hdmi_dp_sound { ++ status = "okay"; ++}; ++ ++&pwm3 { ++ status = "okay"; ++ interrupts = ; ++ compatible = "rockchip,remotectl-pwm"; ++ remote_pwm_id = <3>; ++ handle_cpu_id = <1>; ++ ++ ir_key1 { ++ rockchip,usercode = <0x4040>; ++ rockchip,key_table = ++ <0xf2 KEY_REPLY>, ++ <0xba KEY_BACK>, ++ <0xf4 KEY_UP>, ++ <0xf1 KEY_DOWN>, ++ <0xef KEY_LEFT>, ++ <0xee KEY_RIGHT>, ++ <0xbd KEY_HOME>, ++ <0xea KEY_VOLUMEUP>, ++ <0xe3 KEY_VOLUMEDOWN>, ++ <0xe2 KEY_SEARCH>, ++ <0xb2 KEY_POWER>, ++ <0xbc KEY_MUTE>, ++ <0xec KEY_MENU>, ++ <0xbf 0x190>, ++ <0xe0 0x191>, ++ <0xe1 0x192>, ++ <0xe9 183>, ++ <0xe6 248>, ++ <0xe8 185>, ++ <0xe7 186>, ++ <0xf0 388>, ++ <0xbe 0x175>; ++ }; ++ ++ ir_key2 { ++ rockchip,usercode = <0xff00>; ++ rockchip,key_table = ++ <0xf9 KEY_HOME>, ++ <0xbf KEY_BACK>, ++ <0xfb KEY_MENU>, ++ <0xaa KEY_REPLY>, ++ <0xb9 KEY_UP>, ++ <0xe9 KEY_DOWN>, ++ <0xb8 KEY_LEFT>, ++ <0xea KEY_RIGHT>, ++ <0xeb KEY_VOLUMEDOWN>, ++ <0xef KEY_VOLUMEUP>, ++ <0xf7 KEY_MUTE>, ++ <0xe7 KEY_POWER>, ++ <0xfc KEY_POWER>, ++ <0xa9 KEY_VOLUMEDOWN>, ++ <0xa8 KEY_VOLUMEDOWN>, ++ <0xe0 KEY_VOLUMEDOWN>, ++ <0xa5 KEY_VOLUMEDOWN>, ++ <0xab 183>, ++ <0xb7 388>, ++ <0xf8 184>, ++ <0xaf 185>, ++ <0xed KEY_VOLUMEDOWN>, ++ <0xee 186>, ++ <0xb3 KEY_VOLUMEDOWN>, ++ <0xf1 KEY_VOLUMEDOWN>, ++ <0xf2 KEY_VOLUMEDOWN>, ++ <0xf3 KEY_SEARCH>, ++ <0xb4 KEY_VOLUMEDOWN>, ++ <0xbe KEY_SEARCH>; ++ }; ++ ++ ir_key3 { ++ rockchip,usercode = <0x1dcc>; ++ rockchip,key_table = ++ <0xee KEY_REPLY>, ++ <0xf0 KEY_BACK>, ++ <0xf8 KEY_UP>, ++ <0xbb KEY_DOWN>, ++ <0xef KEY_LEFT>, ++ <0xed KEY_RIGHT>, ++ <0xfc KEY_HOME>, ++ <0xf1 KEY_VOLUMEUP>, ++ <0xfd KEY_VOLUMEDOWN>, ++ <0xb7 KEY_SEARCH>, ++ <0xff KEY_POWER>, ++ <0xf3 KEY_MUTE>, ++ <0xbf KEY_MENU>, ++ <0xf9 0x191>, ++ <0xf5 0x192>, ++ <0xb3 388>, ++ <0xbe KEY_1>, ++ <0xba KEY_2>, ++ <0xb2 KEY_3>, ++ <0xbd KEY_4>, ++ <0xf9 KEY_5>, ++ <0xb1 KEY_6>, ++ <0xfc KEY_7>, ++ <0xf8 KEY_8>, ++ <0xb0 KEY_9>, ++ <0xb6 KEY_0>, ++ <0xb5 KEY_BACKSPACE>; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-edp-avb.dts b/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-edp-avb.dts +new file mode 100755 +index 000000000000..3e9ef7b0773b +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-edp-avb.dts +@@ -0,0 +1,128 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++#include "rk3399-sapphire-excavator-edp.dtsi" ++ ++/ { ++ model = "Rockchip RK3399 Excavator Board edp avb (Android)"; ++ compatible = "rockchip,android", "rockchip,rk3399-excavator-edp-avb", "rockchip,rk3399"; ++ chosen: chosen { ++ bootargs = "earlycon=uart8250,mmio32,0xff1a0000 console=ttyFIQ0 androidboot.baseband=N/A androidboot.veritymode=enforcing androidboot.hardware=rk30board androidboot.console=ttyFIQ0 init=/init initrd=0x62000001,0x00800000 coherent_pool=1m"; ++ }; ++ ++ ext_cam_clk: external-camera-clock { ++ compatible = "fixed-clock"; ++ clock-frequency = <27000000>; ++ clock-output-names = "CLK_CAMERA_27MHZ"; ++ #clock-cells = <0>; ++ }; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ ++ /delete-node/ tc358749x@0f; ++ ++ tc35874x: tc35874x@0f { ++ status = "disabled"; ++ reg = <0x0f>; ++ compatible = "toshiba,tc358749"; ++ clocks = <&ext_cam_clk>; ++ clock-names = "refclk"; ++ reset-gpios = <&gpio2 7 GPIO_ACTIVE_LOW>; ++ /* interrupt-parent = <&gpio2>; */ ++ /* interrupts = <12 IRQ_TYPE_LEVEL_HIGH>; */ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&tc35874x_gpios>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "TC358749XBG"; ++ rockchip,camera-module-lens-name = "NC"; ++ ++ port { ++ hdmiin_out0: endpoint { ++ remote-endpoint = <&hdmi_to_mipi_in>; ++ data-lanes = <1 2 3 4>; ++ clock-noncontinuous; ++ link-frequencies = ++ /bits/ 64 <297000000>; ++ }; ++ }; ++ }; ++}; ++ ++&mipi_dphy_rx0 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam0: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ucam_out0>; ++ data-lanes = <1 2>; ++ }; ++ ++ hdmi_to_mipi_in: endpoint@2 { ++ reg = <2>; ++ remote-endpoint = <&hdmiin_out0>; ++ data-lanes = <1 2 3 4>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_rx0_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp0_mipi_in>; ++ }; ++ }; ++ }; ++}; ++ ++&mipi_dphy_tx1rx1 { ++ status = "disabled"; ++}; ++ ++&pinctrl { ++ hdmiin { ++ tc35874x_gpios: tc35874x_gpios { ++ rockchip,pins = ++ /* PWREN_3.3 */ ++ <2 RK_PA5 RK_FUNC_GPIO &pcfg_output_high>, ++ /* PWREN_1.2 */ ++ <2 RK_PA6 RK_FUNC_GPIO &pcfg_output_high>, ++ /* HDMIIN_RST */ ++ <2 RK_PA7 RK_FUNC_GPIO &pcfg_pull_none>, ++ /* HDMIIN_STBY */ ++ <2 RK_PB0 RK_FUNC_GPIO &pcfg_output_high>, ++ /* MIPI_RST */ ++ <2 RK_PB1 RK_FUNC_GPIO &pcfg_output_high>, ++ /* CSI_CTL */ ++ <2 RK_PB2 RK_FUNC_GPIO &pcfg_output_low>, ++ /* HDMIIN_INT */ ++ <2 RK_PB4 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++&rkisp1_0 { ++ status = "okay"; ++}; ++ ++&rkisp1_1 { ++ status = "disabled"; ++}; ++ +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-edp.dts b/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-edp.dts +new file mode 100755 +index 000000000000..c2f8673198d5 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-edp.dts +@@ -0,0 +1,82 @@ ++/* ++ * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This file is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This file is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++/dts-v1/; ++#include "rk3399-sapphire-excavator-edp.dtsi" ++ ++/ { ++ model = "Rockchip RK3399 Excavator Board edp (Android)"; ++ compatible = "rockchip,android", "rockchip,rk3399-excavator-edp", "rockchip,rk3399"; ++}; ++ ++&firmware_android { ++ compatible = "android,firmware"; ++ fstab { ++ compatible = "android,fstab"; ++ system { ++ compatible = "android,system"; ++ dev = "/dev/block/by-name/system"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,verify"; ++ }; ++ vendor { ++ compatible = "android,vendor"; ++ dev = "/dev/block/by-name/vendor"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,verify"; ++ }; ++ }; ++}; ++ ++&dp_sound { ++ status = "disabled"; ++}; ++ ++&hdmi_dp_sound { ++ status = "disabled"; ++}; ++ ++&hdmi_sound { ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-edp.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-edp.dtsi +new file mode 100755 +index 000000000000..8156e5f7c795 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-edp.dtsi +@@ -0,0 +1,488 @@ ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This file is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This file is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++/dts-v1/; ++ ++#include "rk3399-excavator-sapphire.dtsi" ++#include "rk3399-android.dtsi" ++#include "rk3399-vop-clk-set.dtsi" ++ ++/ { ++ vcc_lcd: vcc-lcd { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_lcd"; ++ gpio = <&gpio4 30 GPIO_ACTIVE_HIGH>; ++ startup-delay-us = <20000>; ++ enable-active-high; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-boot-on; ++ vin-supply = <&vcc5v0_sys>; ++ }; ++ ++ panel: panel { ++ compatible = "simple-panel"; ++ backlight = <&backlight>; ++ power-supply = <&vcc_lcd>; ++ enable-gpios = <&gpio1 13 GPIO_ACTIVE_HIGH>; ++ prepare-delay-ms = <20>; ++ enable-delay-ms = <20>; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <200000000>; ++ hactive = <1536>; ++ vactive = <2048>; ++ hfront-porch = <12>; ++ hsync-len = <16>; ++ hback-porch = <48>; ++ vfront-porch = <8>; ++ vsync-len = <4>; ++ vback-porch = <8>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ ports { ++ panel_in: endpoint { ++ remote-endpoint = <&edp_out>; ++ }; ++ }; ++ }; ++ ++ test-power { ++ status = "okay"; ++ }; ++ ++ hdmiin_sound: hdmiin-sound { ++ compatible = "rockchip,rockchip-rt5651-sound"; ++ rockchip,cpu = <&i2s0>; ++ rockchip,codec = <&rt5651 &rt5651>; ++ status = "okay"; ++ }; ++}; ++ ++&backlight { ++ status = "okay"; ++ enable-gpios = <&gpio4 29 GPIO_ACTIVE_HIGH>; ++}; ++ ++&edp { ++ status = "okay"; ++ force-hpd; ++ ++ ports { ++ port@1 { ++ reg = <1>; ++ ++ edp_out: endpoint { ++ remote-endpoint = <&panel_in>; ++ }; ++ }; ++ }; ++}; ++ ++&edp_in_vopl { ++ status = "disabled"; ++}; ++ ++&hdmi_in_vopb { ++ status = "disabled"; ++}; ++ ++&rt5651 { ++ status = "okay"; ++}; ++ ++&cdn_dp { ++ status = "okay"; ++ extcon = <&fusb0>; ++ phys = <&tcphy0_dp>; ++}; ++ ++&hdmi_dp_sound { ++ status = "okay"; ++}; ++ ++&hdmiin_sound { ++ status = "disabled"; ++}; ++ ++&dp_in_vopb { ++ status = "disabled"; ++}; ++ ++&i2s2 { ++ status = "okay"; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ ++ gsl3673: gsl3673@40 { ++ compatible = "GSL,GSL3673"; ++ reg = <0x40>; ++ screen_max_x = <1536>; ++ screen_max_y = <2048>; ++ irq_gpio_number = <&gpio1 20 IRQ_TYPE_LEVEL_LOW>; ++ rst_gpio_number = <&gpio4 22 GPIO_ACTIVE_HIGH>; ++ }; ++ ++ sgm3784: sgm3784@30 { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ compatible = "sgmicro,gsm3784"; ++ reg = <0x30>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ enable-gpio = <&gpio2 RK_PB4 GPIO_ACTIVE_HIGH>; ++ strobe-gpio = <&gpio1 RK_PA3 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ sgm3784_led0: led@0 { ++ reg = <0x0>; ++ led-max-microamp = <299200>; ++ flash-max-microamp = <1122000>; ++ flash-max-timeout-us = <1600000>; ++ }; ++ ++ sgm3784_led1: led@1 { ++ reg = <0x1>; ++ led-max-microamp = <299200>; ++ flash-max-microamp = <1122000>; ++ flash-max-timeout-us = <1600000>; ++ }; ++ }; ++ ++ tc358749x: tc358749x@0f { ++ compatible = "toshiba,tc358749x"; ++ reg = <0x0f>; ++ power-gpios = <&gpio2 6 GPIO_ACTIVE_HIGH>; ++ power18-gpios = <&gpio2 9 GPIO_ACTIVE_HIGH>; ++ power33-gpios = <&gpio2 5 GPIO_ACTIVE_HIGH>; ++ csi-ctl-gpios = <&gpio2 10 GPIO_ACTIVE_HIGH>; ++ stanby-gpios = <&gpio2 8 GPIO_ACTIVE_HIGH>; ++ reset-gpios = <&gpio2 7 GPIO_ACTIVE_HIGH>; ++ int-gpios = <&gpio2 12 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hdmiin_gpios>; ++ status = "disabled"; ++ }; ++ ++ vm149c: vm149c@0c { ++ compatible = "silicon touch,vm149c"; ++ status = "okay"; ++ reg = <0x0c>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ }; ++ ++ gc2145: gc2145@3c{ ++ status = "okay"; ++ compatible = "galaxycore,gc2145"; ++ reg = <0x3c>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_clkout>; ++ ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ ++ /* avdd-supply = <>; */ ++ /* dvdd-supply = <>; */ ++ /* dovdd-supply = <>; */ ++ pwdn-gpios = <&gpio2 28 GPIO_ACTIVE_HIGH>; //ok ++ rockchip,camera-module-index = <1>; ++ rockchip,camera-module-facing = "front"; ++ rockchip,camera-module-name = "CameraKing"; ++ rockchip,camera-module-lens-name = "Largan"; ++ port { ++ gc2145_out: endpoint { ++ remote-endpoint = <&dvp_in_fcam>; ++ }; ++ }; ++ }; ++ ++ vm149c: vm149c@0c { ++ compatible = "silicon touch,vm149c"; ++ status = "okay"; ++ reg = <0x0c>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ }; ++ ++ ov13850: ov13850@10 { ++ compatible = "ovti,ov13850"; ++ status = "okay"; ++ reg = <0x10>; ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ /* avdd-supply = <>; */ ++ /* dvdd-supply = <>; */ ++ /* dovdd-supply = <>; */ ++ /* reset-gpios = <>; */ ++ reset-gpios = <&gpio2 10 GPIO_ACTIVE_HIGH>; // conflict with csi-ctl-gpios ++ pwdn-gpios = <&gpio1 4 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "rockchip,camera_default"; ++ pinctrl-0 = <&cif_clkout>; ++ ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "CMK-CT0116"; ++ rockchip,camera-module-lens-name = "Largan-50013A1"; ++ lens-focus = <&vm149c>; ++ flash-leds = <&sgm3784_led0 &sgm3784_led1>; ++ ++ port { ++ ucam_out0: endpoint { ++ remote-endpoint = <&mipi_in_ucam0>; ++ //remote-endpoint = <&mipi_in_ucam1>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++ ++ ov4689: ov4689@36 { ++ compatible = "ovti,ov4689"; ++ status = "disabled"; ++ reg = <0x36>; ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ /* avdd-supply = <>; */ ++ /* dvdd-supply = <>; */ ++ /* dovdd-supply = <>; */ ++ /* reset-gpios = <>; */ ++ pwdn-gpios = <&gpio4 29 GPIO_ACTIVE_HIGH>; // conflict with backlight ++ pinctrl-names = "rockchip,camera_default"; ++ pinctrl-0 = <&cif_clkout>; ++ rockchip,camera-module-index = <1>; ++ rockchip,camera-module-facing = "front"; ++ rockchip,camera-module-name = "JSD3425-C1"; ++ rockchip,camera-module-lens-name = "JSD3425-C1"; ++ port { ++ ucam_out1: endpoint { ++ //remote-endpoint = <&mipi_in_ucam0>; ++ remote-endpoint = <&mipi_in_ucam1>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++}; ++ ++&i2c6 { ++ cw2015@62 { ++ status = "disabled"; ++ compatible = "cw201x"; ++ reg = <0x62>; ++ bat_config_info = <0x15 0x42 0x60 0x59 0x52 0x58 0x4D 0x48 ++ 0x48 0x44 0x44 0x46 0x49 0x48 0x32 0x24 ++ 0x20 0x17 0x13 0x0F 0x19 0x3E 0x51 0x45 ++ 0x08 0x76 0x0B 0x85 0x0E 0x1C 0x2E 0x3E ++ 0x4D 0x52 0x52 0x57 0x3D 0x1B 0x6A 0x2D ++ 0x25 0x43 0x52 0x87 0x8F 0x91 0x94 0x52 ++ 0x82 0x8C 0x92 0x96 0xFF 0x7B 0xBB 0xCB ++ 0x2F 0x7D 0x72 0xA5 0xB5 0xC1 0x46 0xAE>; ++ monitor_sec = <5>; ++ virtual_power = <0>; ++ }; ++}; ++ ++&isp0_mmu { ++ status = "okay"; ++}; ++ ++&isp1_mmu { ++ status = "okay"; ++}; ++ ++&mipi_dphy_rx0 { ++ status = "disabled"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam0: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ucam_out0>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_rx0_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp0_mipi_in>; ++ }; ++ }; ++ }; ++}; ++ ++&mipi_dphy_tx1rx1 { ++ status = "disabled"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam1: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ucam_out1>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_tx1rx1_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp1_mipi_in>; ++ }; ++ }; ++ }; ++}; ++ ++&vopb { ++ status = "okay"; ++ assigned-clocks = <&cru DCLK_VOP0_DIV>; ++ assigned-clock-parents = <&cru PLL_CPLL>; ++}; ++ ++&vopl { ++ status = "okay"; ++ assigned-clocks = <&cru DCLK_VOP1_DIV>; ++ assigned-clock-parents = <&cru PLL_VPLL>; ++}; ++ ++&pcie_phy { ++ status = "okay"; ++}; ++ ++&pcie0 { ++ status = "okay"; ++}; ++ ++&rkisp1_0 { ++ status = "disabled"; ++ ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp0_mipi_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dphy_rx0_out>; ++ }; ++ }; ++}; ++ ++&rkisp1_1 { ++ status = "disabled"; ++ ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp1_mipi_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dphy_tx1rx1_out>; ++ }; ++ dvp_in_fcam: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&gc2145_out>; ++ }; ++ }; ++}; ++ ++&route_edp { ++ status = "okay"; ++}; ++ ++&route_hdmi { ++ status = "okay"; ++ connect = <&vopl_out_hdmi>; ++}; ++ ++&rt5651_sound { ++ status = "okay"; ++}; ++ ++&pinctrl { ++ lcd-panel { ++ lcd_panel_reset: lcd-panel-reset { ++ rockchip,pins = <4 RK_PD6 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ hdmiin { ++ hdmiin_gpios: hdmiin_gpios { ++ rockchip,pins = ++ <2 RK_PA5 RK_FUNC_GPIO &pcfg_pull_none>, ++ <2 RK_PA6 RK_FUNC_GPIO &pcfg_pull_none>, ++ <2 RK_PA7 RK_FUNC_GPIO &pcfg_pull_none>, ++ <2 RK_PB0 RK_FUNC_GPIO &pcfg_pull_none>, ++ <2 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>, ++ <2 RK_PB4 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-linux-for-rk1808-cascade.dts b/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-linux-for-rk1808-cascade.dts +new file mode 100755 +index 000000000000..e1e482938481 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-linux-for-rk1808-cascade.dts +@@ -0,0 +1,487 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd. ++ */ ++ ++/dts-v1/; ++ ++#include "rk3399-excavator-sapphire.dtsi" ++#include "rk3399-linux.dtsi" ++#include ++ ++/ { ++ model = "Rockchip RK3399 Excavator Board (Linux Opensource)"; ++ compatible = "rockchip,rk3399-excavator-linux", "rockchip,rk3399"; ++ ++ fiq_debugger: fiq-debugger { ++ compatible = "rockchip,fiq-debugger"; ++ rockchip,serial-id = <2>; ++ rockchip,signal-irq = <182>; ++ rockchip,wake-irq = <0>; ++ rockchip,irq-mode-enable = <1>; /* If enable uart uses irq instead of fiq */ ++ rockchip,baudrate = <1500000>; /* Only 115200 and 1500000 */ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart2c_xfer>; ++ }; ++ ++ edp_panel: edp-panel { ++ compatible = "lg,lp079qx1-sp0v", "panel-simple"; ++ backlight = <&backlight>; ++ power-supply = <&vcc3v3_s0>; ++ enable-gpios = <&gpio4 30 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&lcd_panel_reset>; ++ ++ ports { ++ panel_in_edp: endpoint { ++ remote-endpoint = <&edp_out_panel>; ++ }; ++ }; ++ }; ++ ++ hdmi_sound: hdmi-sound { ++ status = "okay"; ++ }; ++ ++ gpio-keys { ++ compatible = "gpio-keys"; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ autorepeat; ++ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pwrbtn>; ++ ++ button@0 { ++ gpios = <&gpio0 5 GPIO_ACTIVE_LOW>; ++ linux,code = ; ++ label = "GPIO Key Power"; ++ linux,input-type = <1>; ++ gpio-key,wakeup = <1>; ++ debounce-interval = <100>; ++ }; ++ }; ++ ++ vccadc_ref: vccadc-ref { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc1v8_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ }; ++ ++ ext_cam_clk: external-camera-clock { ++ compatible = "fixed-clock"; ++ clock-frequency = <27000000>; ++ clock-output-names = "CLK_CAMERA_27MHZ"; ++ #clock-cells = <0>; ++ }; ++ ++ adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 1>; ++ io-channel-names = "buttons"; ++ poll-interval = <100>; ++ keyup-threshold-microvolt = <1800000>; ++ ++ button-up { ++ label = "Volume Up"; ++ linux,code = ; ++ press-threshold-microvolt = <100000>; ++ }; ++ ++ button-down { ++ label = "Volume Down"; ++ linux,code = ; ++ press-threshold-microvolt = <300000>; ++ }; ++ ++ back { ++ label = "Back"; ++ linux,code = ; ++ press-threshold-microvolt = <985000>; ++ }; ++ ++ menu { ++ label = "Menu"; ++ linux,code = ; ++ press-threshold-microvolt = <1314000>; ++ }; ++ }; ++}; ++ ++&rkisp1_0 { ++ status = "okay"; ++ ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp0_mipi_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dphy_rx0_out>; ++ }; ++ }; ++}; ++ ++&mipi_dphy_rx0 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam0: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ucam_out0>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_rx0_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp0_mipi_in>; ++ }; ++ }; ++ }; ++}; ++ ++&isp0_mmu { ++ status = "okay"; ++}; ++ ++&rkisp1_1 { ++ status = "okay"; ++ ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp1_mipi_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dphy_tx1rx1_out>; ++ }; ++ }; ++}; ++ ++&mipi_dphy_tx1rx1 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam1: endpoint@1 { ++ reg = <1>; ++ /* Unlinked camera */ ++ //remote-endpoint = <&ucam_out1>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_tx1rx1_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp1_mipi_in>; ++ }; ++ }; ++ }; ++}; ++ ++&isp1_mmu { ++ status = "okay"; ++}; ++ ++&saradc { ++ vref-supply = <&vccadc_ref>; ++}; ++ ++&backlight { ++ status = "okay"; ++ enable-gpios = <&gpio1 13 GPIO_ACTIVE_HIGH>; ++}; ++ ++&cdn_dp { ++ status = "disabled"; ++}; ++ ++&display_subsystem { ++ status = "okay"; ++}; ++ ++&dsi_in_vopl { ++ status = "okay"; ++}; ++ ++&dsi_in_vopb { ++ status = "disabled"; ++}; ++ ++&dsi { ++ status = "okay"; ++ ++ panel@0 { ++ compatible ="simple-panel-dsi"; ++ reg = <0>; ++ ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ ++ display-timings { ++ native-mode = <&timing0_720p_30hz>; ++ ++ timing0_720p_30hz: timing0-720p-30hz { ++ clock-frequency = <96000000>; ++ hactive = <1280>; ++ vactive = <720>; ++ hback-porch = <200>; ++ hfront-porch = <1000>; ++ vback-porch = <100>; ++ vfront-porch = <200>; ++ hsync-len = <200>; ++ vsync-len = <200>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ ++ timing1_1080p_30hz: timing0-1080p-30hz { ++ clock-frequency = <76000000>; ++ hactive = <1920>; ++ vactive = <1080>; ++ hback-porch = <100>; ++ hfront-porch = <200>; ++ vback-porch = <10>; ++ vfront-porch = <10>; ++ hsync-len = <20>; ++ vsync-len = <20>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ ++ timing2_1080p_87hz: timing1-1080p-87hz { ++ clock-frequency = <220000000>; ++ hactive = <1920>; ++ vactive = <1080>; ++ hback-porch = <200>; ++ hfront-porch = <120>; ++ vback-porch = <20>; ++ vfront-porch = <2>; ++ hsync-len = <20>; ++ vsync-len = <10>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ panel_in_dsi: endpoint { ++ remote-endpoint = <&dsi_out_panel>; ++ }; ++ }; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@1 { ++ reg = <1>; ++ dsi_out_panel: endpoint { ++ remote-endpoint = <&panel_in_dsi>; ++ }; ++ }; ++ }; ++}; ++ ++&edp_in_vopl { ++ status = "disabled"; ++}; ++ ++&edp_in_vopb { ++ status = "okay"; ++}; ++ ++&route_edp { ++ status = "okay"; ++}; ++ ++&edp { ++ force-hpd; ++ status = "okay"; ++ ++ ports { ++ edp_out: port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ edp_out_panel: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&panel_in_edp>; ++ }; ++ }; ++ }; ++}; ++ ++&hdmi { ++ /* remove the hdmi_cec, reused by edp_hpd */ ++ pinctrl-0 = <&hdmi_i2c_xfer>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ #sound-dai-cells = <0>; ++ status = "okay"; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ ++ gsl3673: gsl3673@40 { ++ compatible = "GSL,GSL3673"; ++ reg = <0x40>; ++ screen_max_x = <1536>; ++ screen_max_y = <2048>; ++ irq_gpio_number = <&gpio1 20 IRQ_TYPE_LEVEL_LOW>; ++ rst_gpio_number = <&gpio4 22 GPIO_ACTIVE_HIGH>; ++ }; ++ ++ tc358749x: tc358749x@f { ++ compatible = "toshiba,tc358749"; ++ reg = <0xf>; ++ clocks = <&ext_cam_clk>; ++ clock-names = "refclk"; ++ reset-gpios = <&gpio2 7 GPIO_ACTIVE_LOW>; ++ interrupt-parent = <&gpio2>; ++ interrupts = <12 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hdmiin_gpios>; ++ status = "disabled"; ++ port { ++ hdmiin_out0: endpoint { ++ /* Unlinked mipi dphy rx0 */ ++ //remote-endpoint = <&mipi_in_ucam0>; ++ data-lanes = <1 2 3 4>; ++ clock-noncontinuous; ++ link-frequencies = ++ /bits/ 64 <297000000>; ++ }; ++ }; ++ }; ++ ++ vm149c: vm149c@0c { ++ compatible = "silicon touch,vm149c"; ++ status = "okay"; ++ reg = <0x0c>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ }; ++ ++ ov13850: ov13850@10 { ++ compatible = "ovti,ov13850"; ++ status = "okay"; ++ reg = <0x10>; ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ ++ /* conflict with csi-ctl-gpios */ ++ reset-gpios = <&gpio2 10 GPIO_ACTIVE_HIGH>; ++ pwdn-gpios = <&gpio1 4 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "rockchip,camera_default"; ++ pinctrl-0 = <&cif_clkout>; ++ lens-focus = <&vm149c>; ++ ++ port { ++ ucam_out0: endpoint { ++ remote-endpoint = <&mipi_in_ucam0>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++}; ++ ++&i2c4 { ++ status = "okay"; ++}; ++ ++&pcie_phy { ++ status = "okay"; ++}; ++ ++&pcie0 { ++ status = "okay"; ++}; ++ ++&pinctrl { ++ buttons { ++ pwrbtn: pwrbtn { ++ rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ lcd-panel { ++ lcd_panel_reset: lcd-panel-reset { ++ rockchip,pins = <4 RK_PD6 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ hdmiin { ++ hdmiin_gpios: hdmiin-gpios { ++ rockchip,pins = ++ <2 RK_PA5 RK_FUNC_GPIO &pcfg_output_high>, ++ <2 RK_PA6 RK_FUNC_GPIO &pcfg_output_high>, ++ <2 RK_PA7 RK_FUNC_GPIO &pcfg_pull_none>, ++ <2 RK_PB0 RK_FUNC_GPIO &pcfg_output_high>, ++ <2 RK_PB1 RK_FUNC_GPIO &pcfg_output_high>, ++ <2 RK_PB2 RK_FUNC_GPIO &pcfg_output_low>, ++ <2 RK_PB4 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++&vopb { ++ status = "okay"; ++}; ++ ++&vopb_mmu { ++ status = "okay"; ++}; ++ ++&vopl { ++ status = "okay"; ++}; ++ ++&vopl_mmu { ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-linux.dts b/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-linux.dts +new file mode 100755 +index 000000000000..9f370a7bff1c +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-linux.dts +@@ -0,0 +1,452 @@ ++/* ++ * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This file is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This file is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++/dts-v1/; ++ ++#include "rk3399-excavator-sapphire.dtsi" ++#include "rk3399-linux.dtsi" ++#include ++ ++/ { ++ model = "Rockchip RK3399 Excavator Board (Linux Opensource)"; ++ compatible = "rockchip,rk3399-excavator-linux", "rockchip,rk3399"; ++ ++ vcc_lcd: vcc-lcd { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_lcd"; ++ gpio = <&gpio4 30 GPIO_ACTIVE_HIGH>; ++ startup-delay-us = <20000>; ++ enable-active-high; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-boot-on; ++ vin-supply = <&vcc5v0_sys>; ++ }; ++ ++ panel: panel { ++ compatible = "simple-panel"; ++ backlight = <&backlight>; ++ power-supply = <&vcc_lcd>; ++ enable-gpios = <&gpio1 13 GPIO_ACTIVE_HIGH>; ++ prepare-delay-ms = <20>; ++ enable-delay-ms = <20>; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <200000000>; ++ hactive = <1536>; ++ vactive = <2048>; ++ hfront-porch = <12>; ++ hsync-len = <16>; ++ hback-porch = <48>; ++ vfront-porch = <8>; ++ vsync-len = <4>; ++ vback-porch = <8>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ ports { ++ panel_in: endpoint { ++ remote-endpoint = <&edp_out>; ++ }; ++ }; ++ }; ++ ++ hdmi_sound: hdmi-sound { ++ status = "okay"; ++ }; ++ ++ gpio-keys { ++ compatible = "gpio-keys"; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ autorepeat; ++ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pwrbtn>; ++ ++ button@0 { ++ gpios = <&gpio0 5 GPIO_ACTIVE_LOW>; ++ linux,code = ; ++ label = "GPIO Key Power"; ++ linux,input-type = <1>; ++ gpio-key,wakeup = <1>; ++ debounce-interval = <100>; ++ }; ++ }; ++ ++ vccadc_ref: vccadc-ref { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc1v8_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ }; ++ ++ ext_cam_clk: external-camera-clock { ++ compatible = "fixed-clock"; ++ clock-frequency = <27000000>; ++ clock-output-names = "CLK_CAMERA_27MHZ"; ++ #clock-cells = <0>; ++ }; ++ ++ adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 1>; ++ io-channel-names = "buttons"; ++ poll-interval = <100>; ++ keyup-threshold-microvolt = <1800000>; ++ ++ button-up { ++ label = "Volume Up"; ++ linux,code = ; ++ press-threshold-microvolt = <100000>; ++ }; ++ ++ button-down { ++ label = "Volume Down"; ++ linux,code = ; ++ press-threshold-microvolt = <300000>; ++ }; ++ ++ back { ++ label = "Back"; ++ linux,code = ; ++ press-threshold-microvolt = <985000>; ++ }; ++ ++ menu { ++ label = "Menu"; ++ linux,code = ; ++ press-threshold-microvolt = <1314000>; ++ }; ++ }; ++}; ++ ++&rkisp1_0 { ++ status = "okay"; ++ ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp0_mipi_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dphy_rx0_out>; ++ }; ++ }; ++}; ++ ++&mipi_dphy_rx0 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam0: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ucam_out0>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_rx0_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp0_mipi_in>; ++ }; ++ }; ++ }; ++}; ++ ++&isp0_mmu { ++ status = "okay"; ++}; ++ ++&rkisp1_1 { ++ status = "okay"; ++ ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp1_mipi_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dphy_tx1rx1_out>; ++ }; ++ }; ++}; ++ ++&mipi_dphy_tx1rx1 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam1: endpoint@1 { ++ reg = <1>; ++ /* Unlinked camera */ ++ //remote-endpoint = <&ucam_out1>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_tx1rx1_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp1_mipi_in>; ++ }; ++ }; ++ }; ++}; ++ ++&isp1_mmu { ++ status = "okay"; ++}; ++ ++&saradc { ++ vref-supply = <&vccadc_ref>; ++}; ++ ++&backlight { ++ status = "okay"; ++ enable-gpios = <&gpio4 29 GPIO_ACTIVE_HIGH>; ++}; ++ ++&cdn_dp { ++ status = "okay"; ++ extcon = <&fusb0>; ++ phys = <&tcphy0_dp>; ++}; ++ ++&display_subsystem { ++ status = "okay"; ++}; ++ ++&route_edp { ++ status = "okay"; ++}; ++ ++&edp { ++ status = "okay"; ++ force-hpd; ++ ++ ports { ++ port@1 { ++ reg = <1>; ++ ++ edp_out: endpoint { ++ remote-endpoint = <&panel_in>; ++ }; ++ }; ++ }; ++}; ++ ++&edp_in_vopb { ++ status = "disabled"; ++}; ++ ++&hdmi { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hdmi_i2c_xfer>, <&hdmi_cec>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ #sound-dai-cells = <0>; ++ status = "okay"; ++}; ++ ++&hdmi_in_vopl { ++ status = "disabled"; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ ++ gsl3673: gsl3673@40 { ++ compatible = "GSL,GSL3673"; ++ reg = <0x40>; ++ screen_max_x = <1536>; ++ screen_max_y = <2048>; ++ irq_gpio_number = <&gpio1 20 IRQ_TYPE_LEVEL_LOW>; ++ rst_gpio_number = <&gpio4 22 GPIO_ACTIVE_HIGH>; ++ }; ++ ++ tc358749x: tc358749x@f { ++ compatible = "toshiba,tc358749"; ++ reg = <0xf>; ++ clocks = <&ext_cam_clk>; ++ clock-names = "refclk"; ++ reset-gpios = <&gpio2 7 GPIO_ACTIVE_LOW>; ++ interrupt-parent = <&gpio2>; ++ interrupts = <12 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hdmiin_gpios>; ++ status = "disabled"; ++ port { ++ hdmiin_out0: endpoint { ++ /* Unlinked mipi dphy rx0 */ ++ //remote-endpoint = <&mipi_in_ucam0>; ++ data-lanes = <1 2 3 4>; ++ clock-noncontinuous; ++ link-frequencies = ++ /bits/ 64 <297000000>; ++ }; ++ }; ++ }; ++ ++ vm149c: vm149c@0c { ++ compatible = "silicon touch,vm149c"; ++ status = "okay"; ++ reg = <0x0c>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ }; ++ ++ ov13850: ov13850@10 { ++ compatible = "ovti,ov13850"; ++ status = "okay"; ++ reg = <0x10>; ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ ++ /* conflict with csi-ctl-gpios */ ++ reset-gpios = <&gpio2 10 GPIO_ACTIVE_HIGH>; ++ pwdn-gpios = <&gpio1 4 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "rockchip,camera_default"; ++ pinctrl-0 = <&cif_clkout>; ++ ++ lens-focus = <&vm149c>; ++ ++ port { ++ ucam_out0: endpoint { ++ remote-endpoint = <&mipi_in_ucam0>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++}; ++ ++&i2c4 { ++ status = "okay"; ++}; ++ ++&pcie_phy { ++ status = "okay"; ++}; ++ ++&pcie0 { ++ status = "okay"; ++}; ++ ++&vopb { ++ status = "okay"; ++ assigned-clocks = <&cru DCLK_VOP0_DIV>; ++ assigned-clock-parents = <&cru PLL_CPLL>; ++}; ++ ++&vopb_mmu { ++ status = "okay"; ++}; ++ ++&vopl { ++ status = "okay"; ++ assigned-clocks = <&cru DCLK_VOP1_DIV>; ++ assigned-clock-parents = <&cru PLL_VPLL>; ++}; ++ ++&vopl_mmu { ++ status = "okay"; ++}; ++ ++&pinctrl { ++ buttons { ++ pwrbtn: pwrbtn { ++ rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ lcd-panel { ++ lcd_panel_reset: lcd-panel-reset { ++ rockchip,pins = <4 RK_PD6 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ hdmiin { ++ hdmiin_gpios: hdmiin-gpios { ++ rockchip,pins = ++ <2 RK_PA5 RK_FUNC_GPIO &pcfg_output_high>, ++ <2 RK_PA6 RK_FUNC_GPIO &pcfg_output_high>, ++ <2 RK_PA7 RK_FUNC_GPIO &pcfg_pull_none>, ++ <2 RK_PB0 RK_FUNC_GPIO &pcfg_output_high>, ++ <2 RK_PB1 RK_FUNC_GPIO &pcfg_output_high>, ++ <2 RK_PB2 RK_FUNC_GPIO &pcfg_output_low>, ++ <2 RK_PB4 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-lp4-linux.dts b/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-lp4-linux.dts +new file mode 100755 +index 000000000000..da471a636103 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-lp4-linux.dts +@@ -0,0 +1,497 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++// Copyright (c) 2020 Fuzhou Rockchip Electronics Co., Ltd. ++ ++/dts-v1/; ++ ++#include "rk3399-excavator-sapphire.dtsi" ++#include "rk3399-linux.dtsi" ++#include ++ ++/ { ++ model = "Rockchip RK3399 Excavator Board (Linux Opensource)"; ++ compatible = "rockchip,rk3399-excavator-linux", "rockchip,rk3399"; ++ ++ vcc_lcd: vcc-lcd { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_lcd"; ++ gpio = <&gpio4 30 GPIO_ACTIVE_HIGH>; ++ startup-delay-us = <20000>; ++ enable-active-high; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-boot-on; ++ vin-supply = <&vcc5v0_sys>; ++ }; ++ ++ panel: panel { ++ compatible = "simple-panel"; ++ backlight = <&backlight>; ++ power-supply = <&vcc_lcd>; ++ enable-gpios = <&gpio1 13 GPIO_ACTIVE_HIGH>; ++ prepare-delay-ms = <20>; ++ enable-delay-ms = <20>; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <200000000>; ++ hactive = <1536>; ++ vactive = <2048>; ++ hfront-porch = <12>; ++ hsync-len = <16>; ++ hback-porch = <48>; ++ vfront-porch = <8>; ++ vsync-len = <4>; ++ vback-porch = <8>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ ports { ++ panel_in: endpoint { ++ remote-endpoint = <&edp_out>; ++ }; ++ }; ++ }; ++ ++ hdmi_sound: hdmi-sound { ++ status = "okay"; ++ }; ++ ++ gpio-keys { ++ compatible = "gpio-keys"; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ autorepeat; ++ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pwrbtn>; ++ ++ button@0 { ++ gpios = <&gpio0 5 GPIO_ACTIVE_LOW>; ++ linux,code = ; ++ label = "GPIO Key Power"; ++ linux,input-type = <1>; ++ gpio-key,wakeup = <1>; ++ debounce-interval = <100>; ++ }; ++ }; ++ ++ vccadc_ref: vccadc-ref { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc1v8_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ }; ++ ++ ext_cam_clk: external-camera-clock { ++ compatible = "fixed-clock"; ++ clock-frequency = <27000000>; ++ clock-output-names = "CLK_CAMERA_27MHZ"; ++ #clock-cells = <0>; ++ }; ++ ++ adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 1>; ++ io-channel-names = "buttons"; ++ poll-interval = <100>; ++ keyup-threshold-microvolt = <1800000>; ++ ++ button-up { ++ label = "Volume Up"; ++ linux,code = ; ++ press-threshold-microvolt = <100000>; ++ }; ++ ++ button-down { ++ label = "Volume Down"; ++ linux,code = ; ++ press-threshold-microvolt = <300000>; ++ }; ++ ++ back { ++ label = "Back"; ++ linux,code = ; ++ press-threshold-microvolt = <985000>; ++ }; ++ ++ menu { ++ label = "Menu"; ++ linux,code = ; ++ press-threshold-microvolt = <1314000>; ++ }; ++ }; ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&dmc { ++ status = "okay"; ++ center-supply = <&vdd_center>; ++ upthreshold = <40>; ++ downdifferential = <20>; ++ system-status-freq = < ++ /*system status freq(KHz)*/ ++ SYS_STATUS_NORMAL 856000 ++ SYS_STATUS_REBOOT 856000 ++ SYS_STATUS_SUSPEND 328000 ++ SYS_STATUS_VIDEO_1080P 666000 ++ SYS_STATUS_VIDEO_4K 856000 ++ SYS_STATUS_VIDEO_4K_10B 856000 ++ SYS_STATUS_PERFORMANCE 856000 ++ SYS_STATUS_BOOST 856000 ++ SYS_STATUS_DUALVIEW 856000 ++ SYS_STATUS_ISP 856000 ++ >; ++ vop-bw-dmc-freq = < ++ /* min_bw(MB/s) max_bw(MB/s) freq(KHz) */ ++ 0 762 416000 ++ 763 3012 666000 ++ 3013 99999 856000 ++ >; ++ ++ vop-pn-msch-readlatency = < ++ 0 0x20 ++ 4 0x20 ++ >; ++ ++ auto-min-freq = <328000>; ++ auto-freq-en = <0>; ++}; ++ ++&dmc_opp_table { ++ compatible = "operating-points-v2"; ++ ++ opp-200000000 { ++ opp-hz = /bits/ 64 <200000000>; ++ opp-microvolt = <900000>; ++ status = "disabled"; ++ }; ++ opp-300000000 { ++ opp-hz = /bits/ 64 <300000000>; ++ opp-microvolt = <900000>; ++ status = "disabled"; ++ }; ++ opp-328000000 { ++ opp-hz = /bits/ 64 <328000000>; ++ opp-microvolt = <900000>; ++ }; ++ opp-400000000 { ++ opp-hz = /bits/ 64 <400000000>; ++ opp-microvolt = <900000>; ++ status = "disabled"; ++ }; ++ opp-416000000 { ++ opp-hz = /bits/ 64 <416000000>; ++ opp-microvolt = <900000>; ++ }; ++ opp-528000000 { ++ opp-hz = /bits/ 64 <528000000>; ++ opp-microvolt = <900000>; ++ status = "disabled"; ++ }; ++ opp-600000000 { ++ opp-hz = /bits/ 64 <600000000>; ++ opp-microvolt = <900000>; ++ status = "disabled"; ++ }; ++ opp-666000000 { ++ opp-hz = /bits/ 64 <666000000>; ++ opp-microvolt = <900000>; ++ }; ++ opp-800000000 { ++ opp-hz = /bits/ 64 <800000000>; ++ opp-microvolt = <900000>; ++ status = "disabled"; ++ }; ++ opp-856000000 { ++ opp-hz = /bits/ 64 <856000000>; ++ opp-microvolt = <900000>; ++ }; ++ opp-928000000 { ++ opp-hz = /bits/ 64 <928000000>; ++ opp-microvolt = <900000>; ++ status = "disabled"; ++ }; ++}; ++ ++&rkisp1_0 { ++ status = "okay"; ++ ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp0_mipi_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dphy_rx0_out>; ++ }; ++ }; ++}; ++ ++&mipi_dphy_rx0 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam0: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ucam_out0>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_rx0_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp0_mipi_in>; ++ }; ++ }; ++ }; ++}; ++ ++&isp0_mmu { ++ status = "okay"; ++}; ++ ++&rkisp1_1 { ++ status = "okay"; ++ ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp1_mipi_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dphy_tx1rx1_out>; ++ }; ++ }; ++}; ++ ++&mipi_dphy_tx1rx1 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam1: endpoint@1 { ++ reg = <1>; ++ /* Unlinked camera */ ++ //remote-endpoint = <&ucam_out1>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_tx1rx1_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp1_mipi_in>; ++ }; ++ }; ++ }; ++}; ++ ++&isp1_mmu { ++ status = "okay"; ++}; ++ ++&saradc { ++ vref-supply = <&vccadc_ref>; ++}; ++ ++&backlight { ++ status = "okay"; ++ enable-gpios = <&gpio4 29 GPIO_ACTIVE_HIGH>; ++}; ++ ++&cdn_dp { ++ status = "okay"; ++ extcon = <&fusb0>; ++ phys = <&tcphy0_dp>; ++}; ++ ++&display_subsystem { ++ status = "okay"; ++}; ++ ++&route_edp { ++ status = "okay"; ++}; ++ ++&edp { ++ status = "okay"; ++ force-hpd; ++ ++ ports { ++ port@1 { ++ reg = <1>; ++ ++ edp_out: endpoint { ++ remote-endpoint = <&panel_in>; ++ }; ++ }; ++ }; ++}; ++ ++&edp_in_vopb { ++ status = "disabled"; ++}; ++ ++&hdmi { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hdmi_i2c_xfer>, <&hdmi_cec>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ #sound-dai-cells = <0>; ++ status = "okay"; ++}; ++ ++&hdmi_in_vopl { ++ status = "disabled"; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ ++ gsl3673: gsl3673@40 { ++ compatible = "GSL,GSL3673"; ++ reg = <0x40>; ++ screen_max_x = <1536>; ++ screen_max_y = <2048>; ++ irq_gpio_number = <&gpio1 20 IRQ_TYPE_LEVEL_LOW>; ++ rst_gpio_number = <&gpio4 22 GPIO_ACTIVE_HIGH>; ++ }; ++ ++ tc358749x: tc358749x@f { ++ compatible = "toshiba,tc358749"; ++ reg = <0xf>; ++ clocks = <&ext_cam_clk>; ++ clock-names = "refclk"; ++ reset-gpios = <&gpio2 7 GPIO_ACTIVE_LOW>; ++ interrupt-parent = <&gpio2>; ++ interrupts = <12 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hdmiin_gpios>; ++ status = "disabled"; ++ port { ++ hdmiin_out0: endpoint { ++ /* Unlinked mipi dphy rx0 */ ++ //remote-endpoint = <&mipi_in_ucam0>; ++ data-lanes = <1 2 3 4>; ++ clock-noncontinuous; ++ link-frequencies = ++ /bits/ 64 <297000000>; ++ }; ++ }; ++ }; ++ ++ ov13850: ov13850@10 { ++ compatible = "ovti,ov13850"; ++ status = "okay"; ++ reg = <0x10>; ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ ++ /* conflict with csi-ctl-gpios */ ++ reset-gpios = <&gpio2 10 GPIO_ACTIVE_HIGH>; ++ pwdn-gpios = <&gpio1 4 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "rockchip,camera_default"; ++ pinctrl-0 = <&cif_clkout>; ++ ++ port { ++ ucam_out0: endpoint { ++ remote-endpoint = <&mipi_in_ucam0>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++}; ++ ++&i2c4 { ++ status = "okay"; ++}; ++ ++&pcie_phy { ++ status = "okay"; ++}; ++ ++&pcie0 { ++ status = "okay"; ++}; ++ ++&vopb { ++ status = "okay"; ++ assigned-clocks = <&cru DCLK_VOP0_DIV>; ++ assigned-clock-parents = <&cru PLL_CPLL>; ++}; ++ ++&vopb_mmu { ++ status = "okay"; ++}; ++ ++&vopl { ++ status = "okay"; ++ assigned-clocks = <&cru DCLK_VOP1_DIV>; ++ assigned-clock-parents = <&cru PLL_VPLL>; ++}; ++ ++&vopl_mmu { ++ status = "okay"; ++}; ++ ++&pinctrl { ++ buttons { ++ pwrbtn: pwrbtn { ++ rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ lcd-panel { ++ lcd_panel_reset: lcd-panel-reset { ++ rockchip,pins = <4 RK_PD6 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ hdmiin { ++ hdmiin_gpios: hdmiin-gpios { ++ rockchip,pins = ++ <2 RK_PA5 RK_FUNC_GPIO &pcfg_output_high>, ++ <2 RK_PA6 RK_FUNC_GPIO &pcfg_output_high>, ++ <2 RK_PA7 RK_FUNC_GPIO &pcfg_pull_none>, ++ <2 RK_PB0 RK_FUNC_GPIO &pcfg_output_high>, ++ <2 RK_PB1 RK_FUNC_GPIO &pcfg_output_high>, ++ <2 RK_PB2 RK_FUNC_GPIO &pcfg_output_low>, ++ <2 RK_PB4 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-sapphire.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-sapphire.dtsi +index 701a567d7638..3d9e27750139 100644 +--- a/arch/arm64/boot/dts/rockchip/rk3399-sapphire.dtsi ++++ b/arch/arm64/boot/dts/rockchip/rk3399-sapphire.dtsi +@@ -515,6 +515,8 @@ &pwm0 { + + &pwm2 { + status = "okay"; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm2_pin_pull_down>; + }; + + &saradc { +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-sched-energy.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-sched-energy.dtsi +new file mode 100755 +index 000000000000..373a776b9207 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-sched-energy.dtsi +@@ -0,0 +1,121 @@ ++/* ++ * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This library is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This library is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++/ { ++ energy-costs { ++ RK3399_CPU_COST_0: rk3399-core-cost0 { ++ busy-cost-data = < ++ 108 46 /* 408M */ ++ 159 67 /* 600M */ ++ 216 90 /* 816M */ ++ 267 120 /* 1008M */ ++ 318 153 /* 1200M */ ++ 375 198 /* 1416M */ ++ 401 222 /* 1512M */ ++ >; ++ idle-cost-data = < ++ 6 ++ 6 ++ 0 ++ 0 ++ >; ++ }; ++ ++ RK3399_CPU_COST_1: rk3399-core-cost1 { ++ busy-cost-data = < ++ 210 129 /* 408MHz */ ++ 308 184 /* 600MHz */ ++ 419 246 /* 816MHz */ ++ 518 335 /* 1008MHz */ ++ 617 428 /* 1200MHz */ ++ 728 573 /* 1416MHz */ ++ 827 724 /* 1608MHz */ ++ 925 900 /* 1800MHz */ ++ 1024 1108 /* 1992MHz */ ++ >; ++ idle-cost-data = < ++ 15 ++ 15 ++ 0 ++ 0 ++ >; ++ }; ++ ++ RK3399_CLUSTER_COST_0: rk3399-cluster-cost0 { ++ busy-cost-data = < ++ 108 46 /* 408M */ ++ 159 67 /* 600M */ ++ 216 90 /* 816M */ ++ 267 120 /* 1008M */ ++ 318 153 /* 1200M */ ++ 375 198 /* 1416M */ ++ 401 222 /* 1512M */ ++ >; ++ idle-cost-data = < ++ 56 ++ 56 ++ 56 ++ 56 ++ >; ++ }; ++ ++ RK3399_CLUSTER_COST_1: rk3399-cluster-cost1 { ++ busy-cost-data = < ++ 210 129 /* 408MHz */ ++ 308 184 /* 600MHz */ ++ 419 246 /* 816MHz */ ++ 518 335 /* 1008MHz */ ++ 617 428 /* 1200MHz */ ++ 728 573 /* 1416MHz */ ++ 827 724 /* 1608MHz */ ++ 925 900 /* 1800MHz */ ++ 1024 1108 /* 1992MHz */ ++ >; ++ idle-cost-data = < ++ 65 ++ 65 ++ 65 ++ 65 ++ >; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-tve1030g-avb.dts b/arch/arm64/boot/dts/rockchip/rk3399-tve1030g-avb.dts +new file mode 100755 +index 000000000000..eeca9cf54373 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-tve1030g-avb.dts +@@ -0,0 +1,170 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++#include "rk3399-tve1030g.dtsi" ++ ++/ { ++ compatible = "rockchip,rk3399-tve1030g-avb", "rockchip,rk3399"; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ ++ vm149c: vm149c@0c { ++ compatible = "silicon touch,vm149c"; ++ status = "okay"; ++ reg = <0x0c>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ }; ++ ++ ov13850: ov13850@10 { ++ compatible = "ovti,ov13850"; ++ status = "disabled"; ++ reg = <0x10>; ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ /* avdd-supply = <>; */ ++ /* dvdd-supply = <>; */ ++ /* dovdd-supply = <>; */ ++ /* reset-gpios = <>; */ ++ reset-gpios = <&gpio2 10 GPIO_ACTIVE_HIGH>; ++ pwdn-gpios = <&gpio1 4 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "rockchip,camera_default"; ++ pinctrl-0 = <&cif_clkout>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "CMK-CT0116"; ++ rockchip,camera-module-lens-name = "Largan-50013A1"; ++ lens-focus = <&vm149c>; ++ port { ++ ucam_out0: endpoint { ++ remote-endpoint = <&mipi_in_ucam0>; ++ //remote-endpoint = <&mipi_in_ucam1>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++ ++ gc2355: gc2355@3c { ++ status = "okay"; ++ compatible = "galaxycore,gc2355"; ++ reg = <0x3c>; ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ /* avdd-supply = <>; */ ++ /* dvdd-supply = <>; */ ++ /* dovdd-supply = <>; */ ++ /* reset-gpios = <>; */ ++ pwdn-gpios = <&gpio2 12 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "rockchip,camera_default"; ++ pinctrl-0 = <&cif_clkout>; ++ rockchip,camera-module-index = <1>; ++ rockchip,camera-module-facing = "front"; ++ rockchip,camera-module-name = "CMK-CW2392"; ++ rockchip,camera-module-lens-name = "M206A-201"; ++ port { ++ ucam_out1: endpoint { ++ //remote-endpoint = <&mipi_in_ucam0>; ++ remote-endpoint = <&mipi_in_ucam1>; ++ data-lanes = <1>; ++ }; ++ }; ++ }; ++}; ++ ++&mipi_dphy_rx0 { ++ status = "disabled"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam0: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ucam_out0>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_rx0_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp0_mipi_in>; ++ }; ++ }; ++ }; ++}; ++ ++&mipi_dphy_tx1rx1 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam1: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ucam_out1>; ++ data-lanes = <1>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_tx1rx1_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp1_mipi_in>; ++ }; ++ }; ++ }; ++}; ++ ++&rkisp1_0 { ++ status = "disabled"; ++ ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp0_mipi_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dphy_rx0_out>; ++ }; ++ }; ++}; ++ ++&rkisp1_1 { ++ status = "okay"; ++ ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp1_mipi_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dphy_tx1rx1_out>; ++ }; ++ }; ++}; ++ +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-tve1030g.dts b/arch/arm64/boot/dts/rockchip/rk3399-tve1030g.dts +new file mode 100755 +index 000000000000..28f81ee8f7f5 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-tve1030g.dts +@@ -0,0 +1,42 @@ ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++ */ ++ ++/dts-v1/; ++#include "rk3399-tve1030g.dtsi" ++ ++/ { ++ compatible = "rockchip,rk3399-tve1030g", "rockchip,rk3399"; ++ ++}; ++ ++&firmware_android { ++ compatible = "android,firmware"; ++ fstab { ++ compatible = "android,fstab"; ++ system { ++ compatible = "android,system"; ++ dev = "/dev/block/by-name/system"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,verify"; ++ }; ++ vendor { ++ compatible = "android,vendor"; ++ dev = "/dev/block/by-name/vendor"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,verify"; ++ }; ++ }; ++}; ++ ++&isp0 { ++ status = "okay"; ++}; ++ ++&isp1 { ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-tve1030g.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-tve1030g.dtsi +new file mode 100755 +index 000000000000..fb700431e356 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-tve1030g.dtsi +@@ -0,0 +1,1039 @@ ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++ */ ++ ++/dts-v1/; ++#include ++#include ++#include "rk3399.dtsi" ++#include "rk3399-android.dtsi" ++#include "rk3399-opp.dtsi" ++#include "rk3399-vop-clk-set.dtsi" ++#include ++ ++/ { ++ adc_keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 1>; ++ io-channel-names = "buttons"; ++ keyup-threshold-microvolt = <1800000>; ++ poll-interval = <100>; ++ ++ vol-up-key { ++ label = "volume up"; ++ linux,code = ; ++ press-threshold-microvolt = <1000>; ++ }; ++ ++ vol-down-key { ++ label = "volume down"; ++ linux,code = ; ++ press-threshold-microvolt = <170000>; ++ }; ++ }; ++ ++ backlight: backlight { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm0 0 25000 0>; ++ brightness-levels = < ++ 0 20 20 21 21 22 22 23 ++ 23 24 24 25 25 26 26 27 ++ 27 28 28 29 29 30 30 31 ++ 31 32 32 33 33 34 34 35 ++ 35 36 36 37 37 38 38 39 ++ 40 41 42 43 44 45 46 47 ++ 48 49 50 51 52 53 54 55 ++ 56 57 58 59 60 61 62 63 ++ 64 65 66 67 68 69 70 71 ++ 72 73 74 75 76 77 78 79 ++ 80 81 82 83 84 85 86 87 ++ 88 89 90 91 92 93 94 95 ++ 96 97 98 99 100 101 102 103 ++ 104 105 106 107 108 109 110 111 ++ 112 113 114 115 116 117 118 119 ++ 120 121 122 123 124 125 126 127 ++ 128 129 130 131 132 133 134 135 ++ 136 137 138 139 140 141 142 143 ++ 144 145 146 147 148 149 150 151 ++ 152 153 154 155 156 157 158 159 ++ 160 161 162 163 164 165 166 167 ++ 168 169 170 171 172 173 174 175 ++ 176 177 178 179 180 181 182 183 ++ 184 185 186 187 188 189 190 191 ++ 192 193 194 195 196 197 198 199 ++ 200 201 202 203 204 205 206 207 ++ 208 209 210 211 212 213 214 215 ++ 216 217 218 219 220 221 222 223 ++ 224 225 226 227 228 229 230 231 ++ 232 233 234 235 236 237 238 239 ++ 240 241 242 243 244 245 246 247 ++ 248 249 250 251 252 253 254 255 ++ >; ++ default-brightness-level = <200>; ++ }; ++ ++ es8316-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,es8316-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,widgets = ++ "Microphone", "Mic Jack", ++ "Headphone", "Headphone Jack"; ++ simple-audio-card,routing = ++ "Mic Jack", "MICBIAS1", ++ "IN1P", "Mic Jack", ++ "Headphone Jack", "HPOL", ++ "Headphone Jack", "HPOR"; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s0>; ++ system-clock-frequency = <11289600>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&es8316>; ++ system-clock-frequency = <11289600>; ++ }; ++ }; ++ ++ rk_headset: rk-headset { ++ compatible = "rockchip_headset"; ++ headset_gpio = <&gpio4 RK_PD4 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hp_det>; ++ io-channels = <&saradc 2>; ++ }; ++ ++ charge-animation { ++ compatible = "rockchip,uboot-charge"; ++ rockchip,uboot-charge-on = <1>; ++ rockchip,android-charge-on = <0>; ++ rockchip,uboot-low-power-voltage = <6700>; ++ rockchip,screen-on-voltage = <6800>; ++ status = "okay"; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ clocks = <&rk808 1>; ++ clock-names = "ext_clock"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio0 RK_PB2 GPIO_ACTIVE_LOW>; ++ }; ++ ++ vcc_sys: vcc-sys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3900000>; ++ regulator-max-microvolt = <3900000>; ++ }; ++ ++ vcc3v3_sys: vcc3v3-sys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc3v3_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ }; ++ ++ vcc5v0_host: vcc5v0-host-regulator { ++ compatible = "regulator-fixed"; ++ gpio = <&gpio4 RK_PD1 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&host_vbus_drv>; ++ regulator-name = "vcc5v0_host"; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ regulator-always-on; ++ enable-active-high; ++ }; ++ ++ vdd_log: vdd-log { ++ compatible = "pwm-regulator"; ++ pwms = <&pwm2 0 25000 1>; ++ rockchip,pwm_id= <2>; ++ rockchip,pwm_voltage = <900000>; ++ regulator-name = "vdd_log"; ++ regulator-min-microvolt = <750000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ xin32k: xin32k { ++ compatible = "fixed-clock"; ++ clock-frequency = <32768>; ++ clock-output-names = "xin32k"; ++ #clock-cells = <0>; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ rockchip,grf = <&grf>; ++ wifi_chip_type = "ap6255"; ++ sdio_vref = <1800>; ++ WIFI,host_wake_irq = <&gpio0 RK_PA3 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&rk808 1>; ++ clock-names = "ext_clock"; ++ uart_rts_gpios = <&gpio2 RK_PC3 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart0_rts>, <&bt_reset_gpio>, <&bt_wake_gpio>, <&bt_irq_gpio>; ++ pinctrl-1 = <&uart0_gpios>; ++ BT,reset_gpio = <&gpio0 RK_PB1 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio2 RK_PD2 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio0 RK_PA4 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++}; ++ ++&cdn_dp { ++ status = "okay"; ++ extcon = <&fusb0>; ++ phys = <&tcphy0_dp>; ++}; ++ ++&cpu_l0 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_l1 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_l2 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_l3 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_b0 { ++ cpu-supply = <&vdd_cpu_b>; ++}; ++ ++&cpu_b1 { ++ cpu-supply = <&vdd_cpu_b>; ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&dmc { ++ status = "okay"; ++ center-supply = <&vdd_center>; ++ upthreshold = <20>; ++ downdifferential = <10>; ++ system-status-freq = < ++ /*system status freq(KHz)*/ ++ SYS_STATUS_NORMAL 856000 ++ SYS_STATUS_REBOOT 856000 ++ SYS_STATUS_SUSPEND 416000 ++ SYS_STATUS_VIDEO_1080P 416000 ++ SYS_STATUS_VIDEO_4K 666000 ++ SYS_STATUS_VIDEO_4K_10B 856000 ++ SYS_STATUS_PERFORMANCE 856000 ++ SYS_STATUS_BOOST 856000 ++ SYS_STATUS_DUALVIEW 856000 ++ SYS_STATUS_ISP 856000 ++ >; ++ vop-bw-dmc-freq = < ++ /* min_bw(MB/s) max_bw(MB/s) freq(KHz) */ ++ 0 762 328000 ++ 763 3012 666000 ++ 3013 99999 856000 ++ >; ++ ++ auto-min-freq = <328000>; ++ auto-freq-en = <1>; ++}; ++ ++&dmc_opp_table { ++ compatible = "operating-points-v2"; ++ ++ opp-200000000 { ++ opp-hz = /bits/ 64 <200000000>; ++ opp-microvolt = <900000>; ++ status = "disabled"; ++ }; ++ opp-300000000 { ++ opp-hz = /bits/ 64 <300000000>; ++ opp-microvolt = <900000>; ++ status = "disabled"; ++ }; ++ opp-328000000 { ++ opp-hz = /bits/ 64 <328000000>; ++ opp-microvolt = <900000>; ++ }; ++ opp-400000000 { ++ opp-hz = /bits/ 64 <400000000>; ++ opp-microvolt = <900000>; ++ status = "disabled"; ++ }; ++ opp-416000000 { ++ opp-hz = /bits/ 64 <416000000>; ++ opp-microvolt = <900000>; ++ }; ++ opp-528000000 { ++ opp-hz = /bits/ 64 <528000000>; ++ opp-microvolt = <900000>; ++ status = "disabled"; ++ }; ++ opp-600000000 { ++ opp-hz = /bits/ 64 <600000000>; ++ opp-microvolt = <900000>; ++ status = "disabled"; ++ }; ++ opp-666000000 { ++ opp-hz = /bits/ 64 <666000000>; ++ opp-microvolt = <900000>; ++ }; ++ opp-800000000 { ++ opp-hz = /bits/ 64 <800000000>; ++ opp-microvolt = <900000>; ++ status = "disabled"; ++ }; ++ opp-856000000 { ++ opp-hz = /bits/ 64 <856000000>; ++ opp-microvolt = <900000>; ++ }; ++ opp-928000000 { ++ opp-hz = /bits/ 64 <928000000>; ++ opp-microvolt = <900000>; ++ status = "disabled"; ++ }; ++}; ++&dp_in_vopb { ++ status = "disabled"; ++}; ++ ++&dsi { ++ status = "okay"; ++ rockchip,lane-rate = <1000>; ++ dsi_panel: panel@0 { ++ status = "okay"; ++ compatible = "simple-panel-dsi"; ++ reg = <0>; ++ backlight = <&backlight>; ++ reset-gpios = <&gpio3 RK_PA4 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&lcd_rst_gpio>; ++ reset-delay-ms = <60>; ++ enable-delay-ms = <60>; ++ prepare-delay-ms = <60>; ++ unprepare-delay-ms = <60>; ++ disable-delay-ms = <60>; ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | ++ MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ panel-init-sequence = [ ++ 15 05 02 8F A5 ++ 15 14 02 01 00 ++ 15 05 02 8F A5 ++ 15 00 02 83 AA ++ 15 00 02 84 11 ++ 15 00 02 A9 4B ++ 15 00 02 83 00 ++ 15 00 02 84 00 ++ 15 00 02 8F 00 ++ ]; ++ ++ disp_timings: display-timings { ++ native-mode = <&timing0>; ++ timing0: timing0 { ++ clock-frequency = <150000000>; ++ hactive = <1200>; ++ hfront-porch = <80>; ++ hback-porch = <60>; ++ hsync-len = <1>; ++ vactive = <1920>; ++ vfront-porch = <35>; ++ vback-porch = <25>; ++ vsync-len = <1>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ panel_in_dsi: endpoint { ++ remote-endpoint = <&dsi_out_panel>; ++ }; ++ }; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@1 { ++ reg = <1>; ++ dsi_out_panel: endpoint { ++ remote-endpoint = <&panel_in_dsi>; ++ }; ++ }; ++ }; ++}; ++ ++&dsi_in_vopl { ++ status = "disabled"; ++}; ++ ++&emmc_phy { ++ status = "okay"; ++}; ++ ++&gpu { ++ status = "okay"; ++ mali-supply = <&vdd_gpu>; ++}; ++ ++&hdmi { ++ status = "okay"; ++}; ++ ++&hdmi_dp_sound { ++ status = "okay"; ++}; ++ ++&hdmi_in_vopb { ++ status = "disabled"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <180>; ++ i2c-scl-falling-time-ns = <30>; ++ clock-frequency = <400000>; ++ ++ vdd_cpu_b: syr837@40 { ++ compatible = "silergy,syr827"; ++ reg = <0x40>; ++ vin-supply = <&vcc_sys>; ++ regulator-compatible = "fan53555-reg"; ++ pinctrl-0 = <&vsel1_gpio>; ++ vsel-gpios = <&gpio1 RK_PC1 GPIO_ACTIVE_HIGH>; ++ regulator-name = "vdd_cpu_b"; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1500000>; ++ regulator-ramp-delay = <1000>; ++ fcs,suspend-voltage-selector = <1>; ++ regulator-always-on; ++ regulator-initial-state = <3>; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdd_gpu: syr828@41 { ++ compatible = "silergy,syr828"; ++ status = "okay"; ++ reg = <0x41>; ++ vin-supply = <&vcc_sys>; ++ regulator-compatible = "fan53555-reg"; ++ pinctrl-0 = <&vsel2_gpio>; ++ vsel-gpios = <&gpio1 RK_PB6 GPIO_ACTIVE_HIGH>; ++ regulator-name = "vdd_gpu"; ++ regulator-min-microvolt = <735000>; ++ regulator-max-microvolt = <1400000>; ++ regulator-ramp-delay = <1000>; ++ fcs,suspend-voltage-selector = <1>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ rk808: pmic@1b { ++ compatible = "rockchip,rk808"; ++ reg = <0x1b>; ++ interrupt-parent = <&gpio1>; ++ interrupts = ; ++ pinctrl-0 = <&pmic_int_l>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ vcc1-supply = <&vcc3v3_sys>; ++ vcc2-supply = <&vcc3v3_sys>; ++ vcc3-supply = <&vcc3v3_sys>; ++ vcc4-supply = <&vcc3v3_sys>; ++ vcc6-supply = <&vcc3v3_sys>; ++ vcc7-supply = <&vcc3v3_sys>; ++ vcc8-supply = <&vcc3v3_sys>; ++ vcc9-supply = <&vcc3v3_sys>; ++ vcc10-supply = <&vcc3v3_sys>; ++ vcc11-supply = <&vcc3v3_sys>; ++ vcc12-supply = <&vcc3v3_sys>; ++ vddio-supply = <&vcc1v8_pmu>; ++ ++ regulators { ++ ++ vdd_center: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <750000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-name = "vdd_center"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ vdd_cpu_l: DCDC_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <750000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-name = "vdd_cpu_l"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc_ddr"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ vcc_1v8: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc_1v8"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ vcc1v8_dvp: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc1v8_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ vcc3v0_tp: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-name = "vcc3v0_tp"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ vcc1v8_pmu: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc1v8_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ vcc_sd: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-name = "vcc_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ vcca3v0_codec: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-name = "vcca3v0_codec"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ vcc_1v5: LDO_REG6 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ regulator-name = "vcc_1v5"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1500000>; ++ }; ++ }; ++ vcca1v8_codec: LDO_REG7 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcca1v8_codec"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ vcc_3v0: LDO_REG8 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-name = "vcc_3v0"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ vcc3v3_s3: SWITCH_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc3v3_s3"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ vcc3v3_s0: SWITCH_REG2 { ++ regulator-boot-on; ++ regulator-always-on; ++ regulator-name = "vcc3v3_s0"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ }; ++ }; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <140>; ++ i2c-scl-falling-time-ns = <30>; ++ ++ es8316: es8316@11 { ++ #sound-dai-cells = <0>; ++ compatible = "everest,es8316"; ++ reg = <0x11>; ++ clocks = <&cru SCLK_I2S_8CH_OUT>; ++ clock-names = "mclk"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s_8ch_mclk>; ++ spk-con-gpio = <&gpio0 RK_PB3 GPIO_ACTIVE_HIGH>; ++ extcon = <&rk_headset>; ++ }; ++}; ++ ++&i2c4 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <345>; ++ i2c-scl-falling-time-ns = <11>; ++ clock-frequency = <100000>; ++ ++ bq25700: bq25700@6b { ++ compatible = "ti,bq25703"; ++ reg = <0x6b>; ++ extcon = <&fusb0>; ++ interrupt-parent = <&gpio1>; ++ interrupts = ; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&charger_ok>; ++ ti,charge-current = <1500000>; ++ ti,max-charge-voltage = <8704000>; ++ ti,max-input-voltage = <20000000>; ++ ti,max-input-current = <6000000>; ++ ti,input-current-sdp = <500000>; ++ ti,input-current-dcp = <2000000>; ++ ti,input-current-cdp = <2000000>; ++ ti,input-current-dc = <2000000>; ++ ti,minimum-sys-voltage = <6700000>; ++ ti,otg-voltage = <5000000>; ++ ti,otg-current = <500000>; ++ ti,input-current = <500000>; ++ pd-charge-only = <0>; ++ typec0-enable-gpios = <&gpio1 RK_PA3 GPIO_ACTIVE_LOW>; ++ status = "okay"; ++ }; ++ ++ cw2015: cw2015@62 { ++ status = "okay"; ++ compatible = "cw201x"; ++ reg = <0x62>; ++ bat_config_info = <0x15 0xA8 0x5D 0x5D 0x59 0x55 0x57 0x50 ++ 0x4B 0x4F 0x55 0x53 0x43 0x37 0x2F 0x28 ++ 0x21 0x18 0x15 0x17 0x27 0x43 0x57 0x4F ++ 0x13 0x5E 0x0A 0xE1 0x19 0x31 0x3C 0x46 ++ 0x4C 0x52 0x50 0x54 0x44 0x1E 0x7E 0x4C ++ 0x1C 0x4A 0x52 0x87 0x8F 0x91 0x94 0x52 ++ 0x82 0x8C 0x92 0x96 0x00 0xAD 0xFB 0xCB ++ 0x2F 0x7D 0x72 0xA5 0xB5 0xC1 0x1C 0x09>; ++ monitor_sec = <2>; ++ virtual_power = <0>; ++ divider_res1 = <200>; ++ divider_res2 = <200>; ++ }; ++ ++ fusb0: fusb30x@22 { ++ compatible = "fairchild,fusb302"; ++ reg = <0x22>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&fusb0_int>; ++ vbus-5v-gpios = <&gpio1 RK_PD0 GPIO_ACTIVE_HIGH>; ++ int-n-gpios = <&gpio1 RK_PA2 GPIO_ACTIVE_HIGH>; ++ discharge-gpios = <&gpio0 RK_PB4 GPIO_ACTIVE_HIGH>; ++ charge-dev = <&bq25700>; ++ support-uboot-charge = <1>; ++ port-num = <0>; ++ status = "okay"; ++ }; ++ ++ kxtj: kxtj2@0e { ++ status = "okay"; ++ compatible = "gs_kxtj9"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&kxtj2_irq_gpio>; ++ reg = <0x0e>; ++ irq-gpio = <&gpio1 RK_PC6 IRQ_TYPE_EDGE_RISING>; ++ type = ; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ power-off-in-suspend = <1>; ++ layout = <5>; ++ }; ++}; ++ ++&i2c5 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <150>; ++ i2c-scl-falling-time-ns = <30>; ++ clock-frequency = <100000>; ++ ++ gslx680: gslx680@40 { ++ compatible = "gslX680_tve"; ++ reg = <0x40>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&tp_irq_gpio>; ++ touch-gpio = <&gpio3 RK_PB0 IRQ_TYPE_EDGE_RISING>; ++ reset-gpio = <&gpio4 RK_PC6 GPIO_ACTIVE_LOW>; ++ max-x = <1200>; ++ max-y = <1920>; ++ tp-size = <80>; ++ tp-supply = <&vcc3v0_tp>; ++ status = "okay"; ++ }; ++}; ++ ++&i2s0 { ++ status = "okay"; ++ rockchip,i2s-broken-burst-len; ++ rockchip,playback-channels = <8>; ++ rockchip,capture-channels = <8>; ++ #sound-dai-cells = <0>; ++}; ++ ++&i2s2 { ++ #sound-dai-cells = <0>; ++ status = "okay"; ++}; ++ ++&io_domains { ++ status = "okay"; ++ bt656-supply = <&vcc1v8_dvp>; ++ audio-supply = <&vcca1v8_codec>; ++ sdmmc-supply = <&vcc_sd>; ++ gpio1830-supply = <&vcc_3v0>; ++}; ++ ++&isp0_mmu { ++ status = "okay"; ++}; ++ ++&isp1_mmu { ++ status = "okay"; ++}; ++ ++&pinctrl { ++ ++ charger { ++ charger_ok: charge-ok { ++ rockchip,pins = ++ <1 RK_PC7 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ fusb30x { ++ fusb0_int: fusb0-int { ++ rockchip,pins = ++ <1 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>, ++ <0 RK_PB4 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ headphone { ++ hp_det: hp-det { ++ rockchip,pins = <4 RK_PD4 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ kxtj2 { ++ kxtj2_irq_gpio: kxtj2-irq-gpio { ++ rockchip,pins = <1 RK_PC6 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ lcd_rst { ++ lcd_rst_gpio: lcd-rst-gpio { ++ rockchip,pins = <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ pmic { ++ pmic_int_l: pmic-int-l { ++ rockchip,pins = ++ <1 RK_PC5 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ vsel1_gpio: vsel1-gpio { ++ rockchip,pins = ++ <1 RK_PC1 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ vsel2_gpio: vsel2-gpio { ++ rockchip,pins = ++ <1 RK_PB6 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <0 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ tp_irq { ++ tp_irq_gpio: tp-irq-gpio { ++ rockchip,pins = <3 RK_PB0 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ usb2 { ++ host_vbus_drv: host-vbus-drv { ++ rockchip,pins = ++ <4 RK_PD1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart0_gpios: uart0-gpios { ++ rockchip,pins = <2 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ bt_reset_gpio: bt-reset-gpio { ++ rockchip,pins = <0 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ bt_wake_gpio: bt-wake-gpio { ++ rockchip,pins = <2 RK_PD2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ bt_irq_gpio: bt-irq-gpio { ++ rockchip,pins = <0 RK_PA4 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ pmu1830-supply = <&vcc_1v8>; ++}; ++ ++&pwm0 { ++ status = "okay"; ++}; ++ ++&pwm2 { ++ status = "okay"; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm2_pin_pull_down>; ++}; ++ ++&rockchip_suspend { ++ status = "okay"; ++ rockchip,sleep-debug-en = <1>; ++ rockchip,sleep-mode-config = < ++ (0 ++ | RKPM_SLP_ARMPD ++ | RKPM_SLP_PERILPPD ++ | RKPM_SLP_DDR_RET ++ | RKPM_SLP_PLLPD ++ | RKPM_SLP_CENTER_PD ++ | RKPM_SLP_OSC_DIS ++ | RKPM_SLP_AP_PWROFF ++ ) ++ >; ++ rockchip,wakeup-config = < ++ (0 ++ | RKPM_GPIO_WKUP_EN ++ ) ++ >; ++ rockchip,pwm-regulator-config = < ++ (0 ++ | PWM2_REGULATOR_EN ++ ) ++ >; ++ rockchip,power-ctrl = ++ <&gpio1 RK_PC1 GPIO_ACTIVE_HIGH>, ++ <&gpio1 RK_PB6 GPIO_ACTIVE_HIGH>; ++}; ++ ++&route_dsi { ++ status = "okay"; ++ logo,mode = "center"; ++}; ++ ++&saradc { ++ status = "okay"; ++}; ++ ++&sdhci { ++ bus-width = <8>; ++ mmc-hs400-1_8v; ++ no-sdio; ++ no-sd; ++ non-removable; ++ keep-power-in-suspend; ++ mmc-hs400-enhanced-strobe; ++ status = "okay"; ++}; ++ ++&sdio0 { ++ clock-frequency = <100000000>; ++ clock-freq-min-max = <200000 100000000>; ++ no-sd; ++ no-mmc; ++ bus-width = <4>; ++ disable-wp; ++ cap-sd-highspeed; ++ cap-sdio-irq; ++ keep-power-in-suspend; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ non-removable; ++ num-slots = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&sdmmc { ++ clock-frequency = <50000000>; ++ clock-freq-min-max = <400000 150000000>; ++ no-sdio; ++ no-mmc; ++ bus-width = <4>; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ disable-wp; ++ num-slots = <1>; ++ //sd-uhs-sdr104; ++ vqmmc-supply = <&vcc_sd>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_cd &sdmmc_bus4>; ++ status = "okay"; ++}; ++ ++&tcphy0 { ++ extcon = <&fusb0>; ++ status = "okay"; ++}; ++ ++&tsadc { ++ rockchip,hw-tshut-mode = <1>; /* tshut mode 0:CRU 1:GPIO */ ++ rockchip,hw-tshut-polarity = <1>; /* tshut polarity 0:LOW 1:HIGH */ ++ status = "okay"; ++}; ++ ++&u2phy0 { ++ status = "okay"; ++ extcon = <&fusb0>; ++ ++ u2phy0_otg: otg-port { ++ status = "okay"; ++ }; ++ ++ u2phy0_host: host-port { ++ phy-supply = <&vcc5v0_host>; ++ status = "okay"; ++ }; ++}; ++ ++&uart0 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart0_xfer &uart0_cts>; ++ status = "okay"; ++}; ++ ++&uart2 { ++ status = "disabled"; ++}; ++ ++&usb_host0_ehci { ++ status = "okay"; ++}; ++ ++&usb_host0_ohci { ++ status = "okay"; ++}; ++ ++&usbdrd3_0 { ++ status = "okay"; ++}; ++ ++&usbdrd_dwc3_0 { ++ status = "okay"; ++ extcon = <&fusb0>; ++}; ++ ++&vopb { ++ assigned-clocks = <&cru DCLK_VOP0_DIV>; ++ assigned-clock-parents = <&cru PLL_CPLL>; ++}; ++ ++&vopl { ++ assigned-clocks = <&cru DCLK_VOP1_DIV>; ++ assigned-clock-parents = <&cru PLL_VPLL>; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-tve1205g.dts b/arch/arm64/boot/dts/rockchip/rk3399-tve1205g.dts +new file mode 100755 +index 000000000000..ac9d28db5b76 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-tve1205g.dts +@@ -0,0 +1,1179 @@ ++/* ++ * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This file is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This file is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++/dts-v1/; ++#include ++#include "rk3399.dtsi" ++#include "rk3399-android.dtsi" ++#include "rk3399-opp.dtsi" ++#include ++#include ++ ++/ { ++ compatible = "rockchip,rk3399-mid", "rockchip,rk3399"; ++ ++ edp_panel: edp-panel { ++ compatible = "auo,b125han03"; ++ backlight = <&backlight>; ++ power-supply = <&vcc3v3_s0>; ++ enable-gpios = <&gpio3 RK_PA0 GPIO_ACTIVE_HIGH>; ++ bus-format = ; ++ bpc = <6>; ++ prepare-delay-ms = <50>; ++ ports { ++ panel_in_edp: endpoint { ++ remote-endpoint = <&edp_out_panel>; ++ }; ++ }; ++ }; ++ ++ usb_cam_gpio: usb-cam-gpio { ++ compatible = "usb-cam-gpio"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&usb_cam_on_gpio>; ++ hd-cam-gpios = <&gpio3 RK_PA1 GPIO_ACTIVE_HIGH>; ++ ir-cam-gpios = <&gpio3 RK_PA2 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ vcc_sys: vcc-sys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3900000>; ++ regulator-max-microvolt = <3900000>; ++ }; ++ ++ vcc3v3_sys: vcc3v3-sys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc3v3_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ }; ++ ++ vcc5v0_host: vcc5v0-host-regulator { ++ compatible = "regulator-fixed"; ++ enable-active-high; ++ gpio = <&gpio4 25 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&host_vbus_drv>; ++ regulator-name = "vcc5v0_host"; ++ regulator-always-on; ++ }; ++ ++ vdd_log: vdd-log { ++ compatible = "pwm-regulator"; ++ pwms = <&pwm2 0 25000 1>; ++ rockchip,pwm_id= <2>; ++ rockchip,pwm_voltage = <900000>; ++ regulator-name = "vdd_log"; ++ regulator-min-microvolt = <750000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ xin32k: xin32k { ++ compatible = "fixed-clock"; ++ clock-frequency = <32768>; ++ clock-output-names = "xin32k"; ++ #clock-cells = <0>; ++ }; ++ ++ backlight: backlight { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm0 0 25000 0>; ++ brightness-levels = < ++ 0 1 51 52 52 53 53 54 ++ 54 55 55 56 56 57 57 58 ++ 58 59 59 60 61 61 62 63 ++ 63 64 65 65 66 67 67 68 ++ 69 69 70 71 71 72 73 73 ++ 74 75 75 76 77 77 78 79 ++ 79 80 80 81 81 82 83 83 ++ 84 85 86 86 87 88 89 89 ++ 90 91 92 92 93 94 95 95 ++ 96 97 98 98 99 100 101 101 ++ 102 103 104 104 105 106 107 107 ++ 108 109 110 110 111 112 113 113 ++ 114 115 116 116 117 118 119 119 ++ 120 121 122 122 123 124 125 125 ++ 126 127 128 128 129 130 131 131 ++ 132 133 134 134 135 136 137 137 ++ 138 139 140 140 141 142 143 143 ++ 144 145 146 146 147 148 149 149 ++ 150 151 152 152 153 154 155 155 ++ 156 157 158 158 159 160 161 161 ++ 162 163 164 164 165 166 167 167 ++ 168 169 170 170 171 172 173 173 ++ 174 175 176 176 177 178 179 179 ++ 180 181 182 182 183 184 185 185 ++ 186 187 188 188 189 190 191 191 ++ 216 217 218 218 219 220 221 221 ++ 222 223 224 224 225 226 227 227 ++ 228 229 230 230 231 232 233 233 ++ 234 235 236 236 237 238 239 239 ++ 240 241 242 242 243 244 245 245 ++ 246 247 248 248 249 250 251 251 ++ 252 253 254 254 255 255 255 255>; ++ default-brightness-level = <200>; ++ enable-gpios = <&gpio3 RK_PA4 GPIO_ACTIVE_HIGH>; ++ }; ++ ++ vcc_phy: vcc-phy-regulator { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_phy"; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ cx2072x-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,cx2072x-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,widgets = ++ "Microphone", "Microphone Jack", ++ "Line", "Microphone Headset", ++ "Headphone", "Headphone Jack", ++ "Speaker", "Speaker External"; ++ simple-audio-card,routing = ++ "PORTC", "Microphone Jack", ++ "PortD Mic Bias", "Microphone Headset", ++ "Headphone Jack", "PORTA", ++ "Speaker External", "PORTG"; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s0>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&cx2072x>; ++ }; ++ }; ++ ++ sound { ++ compatible = "rockchip,cdndp-sound"; ++ rockchip,cpu = <&i2s2>; ++ rockchip,codec = <&cdn_dp>; ++ status = "okay"; ++ }; ++ ++ bt-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "dsp_b"; ++ simple-audio-card,bitclock-inversion = <1>; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,name = "rockchip,bt"; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s1>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&bt_sco>; ++ }; ++ }; ++ ++ bt_sco: bt-sco { ++ compatible = "delta,dfbmcs320"; ++ #sound-dai-cells = <0>; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ clocks = <&rk808 1>; ++ clock-names = "ext_clock"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio0 10 GPIO_ACTIVE_LOW>; /* GPIO0_B2 */ ++ }; ++ ++ leds: gpio-leds { ++ compatible = "gpio-leds"; ++ pinctrl-names = "default"; ++ pinctrl-0 =<&leds_gpio>; ++ ++ led@1 { ++ gpios = <&gpio1 RK_PC6 GPIO_ACTIVE_HIGH>; ++ label = "battery_led_amber"; ++ retain-state-suspended; ++ }; ++ ++ led@2 { ++ gpios = <&gpio1 RK_PD0 GPIO_ACTIVE_HIGH>; ++ label = "battery_led_white"; ++ retain-state-suspended; ++ }; ++ ++ led@3 { ++ gpios = <&gpio3 RK_PA5 GPIO_ACTIVE_HIGH>; ++ label = "call_answer_led"; ++ }; ++ ++ led@4 { ++ gpios = <&gpio3 RK_PB1 GPIO_ACTIVE_HIGH>; ++ label = "call_decline_led"; ++ }; ++ ++ led@5 { ++ gpios = <&gpio2 RK_PD3 GPIO_ACTIVE_HIGH>; ++ label = "rec_mute_led"; ++ }; ++ ++ led@6 { ++ gpios = <&gpio2 RK_PD4 GPIO_ACTIVE_HIGH>; ++ label = "play_mute_led"; ++ }; ++ ++ led@7 { ++ gpios = <&gpio4 RK_PD2 GPIO_ACTIVE_HIGH>; ++ label = "wl_led"; ++ }; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ rockchip,grf = <&grf>; ++ wifi_chip_type = "ap6354"; ++ sdio_vref = <1800>; ++ WIFI,host_wake_irq = <&gpio0 3 GPIO_ACTIVE_HIGH>; /* GPIO0_a3 */ ++ status = "okay"; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&rk808 1>; ++ clock-names = "ext_clock"; ++ //wifi-bt-power-toggle; ++ uart_rts_gpios = <&gpio2 19 GPIO_ACTIVE_LOW>; /* GPIO2_C3 */ ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart0_rts>; ++ pinctrl-1 = <&uart0_gpios>; ++ //BT,power_gpio = <&gpio3 19 GPIO_ACTIVE_HIGH>; /* GPIOx_xx */ ++ BT,reset_gpio = <&gpio0 9 GPIO_ACTIVE_HIGH>; /* GPIO0_B1 */ ++ BT,wake_gpio = <&gpio2 26 GPIO_ACTIVE_HIGH>; /* GPIO2_D2 */ ++ BT,wake_host_irq = <&gpio0 4 GPIO_ACTIVE_HIGH>; /* GPIO0_A4 */ ++ status = "okay"; ++ }; ++ ++ uboot-charge { ++ compatible = "rockchip,uboot-charge"; ++ rockchip,uboot-charge-on = <1>; ++ rockchip,android-charge-on = <0>; ++ rockchip,uboot-exit-charge-level = <2>; ++ rockchip,uboot-low-power-level = <1>; ++ rockchip,uboot-charge-brightness = <0>; ++ max-input-voltage = <20000>; ++ max-input-current = <6000>; ++ }; ++ ++ vibrator { ++ compatible = "rk-vibrator-gpio"; ++ vibrator-gpio = <&gpio4 30 GPIO_ACTIVE_LOW>; ++ status = "okay"; ++ }; ++ ++ rk_headset: rk-headset { ++ compatible = "rockchip_headset"; ++ headset_gpio = <&gpio4 28 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hp_det>; ++ io-channels = <&saradc 2>; ++ }; ++ ++ hall_sensor: hall-mh248 { ++ compatible = "hall-mh248"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&mh248_irq_gpio>; ++ irq-gpio = <&gpio0 RK_PA1 IRQ_TYPE_EDGE_BOTH>; ++ hall-active = <1>; ++ status = "okay"; ++ }; ++}; ++ ++&rk_key { ++ compatible = "rockchip,key"; ++ status = "okay"; ++ ++ io-channels = <&saradc 1>; ++ ++ vol-up-key { ++ linux,code = <114>; ++ label = "volume up"; ++ rockchip,adc_value = <1>; ++ }; ++ ++ vol-down-key { ++ linux,code = <115>; ++ label = "volume down"; ++ rockchip,adc_value = <170>; ++ }; ++ ++ power-key { ++ gpios = <&gpio0 5 GPIO_ACTIVE_LOW>; ++ linux,code = <116>; ++ label = "power"; ++ gpio-key,wakeup; ++ }; ++ ++ menu-key { ++ linux,code = <59>; ++ label = "menu"; ++ rockchip,adc_value = <746>; ++ }; ++ ++ home-key { ++ linux,code = <102>; ++ label = "home"; ++ rockchip,adc_value = <355>; ++ }; ++ ++ back-key { ++ linux,code = <158>; ++ label = "back"; ++ rockchip,adc_value = <560>; ++ }; ++ ++ camera-key { ++ linux,code = <212>; ++ label = "camera"; ++ rockchip,adc_value = <450>; ++ }; ++}; ++ ++&cpu_l0 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_l1 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_l2 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_l3 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_b0 { ++ cpu-supply = <&vdd_cpu_b>; ++}; ++ ++&cpu_b1 { ++ cpu-supply = <&vdd_cpu_b>; ++}; ++ ++&edp { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&edp_hpd>; ++ ++ ports { ++ edp_out: port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ edp_out_panel: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&panel_in_edp>; ++ }; ++ }; ++ }; ++}; ++ ++&edp_in_vopl { ++ status = "disabled"; ++}; ++ ++&emmc_phy { ++ status = "okay"; ++}; ++ ++&firmware_android { ++ compatible = "android,firmware"; ++ fstab { ++ compatible = "android,fstab"; ++ system { ++ compatible = "android,system"; ++ dev = "/dev/block/by-name/system"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,verify"; ++ }; ++ vendor { ++ compatible = "android,vendor"; ++ dev = "/dev/block/by-name/vendor"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,verify"; ++ }; ++ }; ++}; ++ ++&gpu { ++ status = "okay"; ++ mali-supply = <&vdd_gpu>; ++}; ++ ++&hdmi { ++ status = "disabled"; ++}; ++ ++&cdn_dp { ++ status = "okay"; ++ extcon = <&fusb0>, <&fusb1>; ++}; ++ ++&dp_in_vopb { ++ status = "disabled"; ++}; ++ ++&i2s0 { ++ status = "okay"; ++ rockchip,i2s-broken-burst-len; ++ rockchip,playback-channels = <8>; ++ rockchip,capture-channels = <8>; ++ rockchip,bclk-fs = <32>; ++ #sound-dai-cells = <0>; ++}; ++ ++&i2s1 { ++ status = "okay"; ++ rockchip,i2s-broken-burst-len; ++ rockchip,playback-channels = <2>; ++ rockchip,capture-channels = <2>; ++ #sound-dai-cells = <0>; ++}; ++ ++&i2s2 { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <180>; ++ i2c-scl-falling-time-ns = <30>; ++ clock-frequency = <400000>; ++ ++ vdd_cpu_b: syr837@40 { ++ compatible = "silergy,syr827"; ++ reg = <0x40>; ++ vin-supply = <&vcc_sys>; ++ regulator-compatible = "fan53555-reg"; ++ pinctrl-0 = <&vsel1_gpio>; ++ vsel-gpios = <&gpio1 17 GPIO_ACTIVE_HIGH>; ++ regulator-name = "vdd_cpu_b"; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1500000>; ++ regulator-ramp-delay = <1000>; ++ fcs,suspend-voltage-selector = <1>; ++ regulator-always-on; ++ regulator-initial-state = <3>; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdd_gpu: syr828@41 { ++ compatible = "silergy,syr828"; ++ status = "okay"; ++ reg = <0x41>; ++ vin-supply = <&vcc_sys>; ++ regulator-compatible = "fan53555-reg"; ++ pinctrl-0 = <&vsel2_gpio>; ++ vsel-gpios = <&gpio1 14 GPIO_ACTIVE_HIGH>; ++ regulator-name = "vdd_gpu"; ++ regulator-min-microvolt = <735000>; ++ regulator-max-microvolt = <1400000>; ++ regulator-ramp-delay = <1000>; ++ fcs,suspend-voltage-selector = <1>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ rk808: pmic@1b { ++ compatible = "rockchip,rk808"; ++ reg = <0x1b>; ++ interrupt-parent = <&gpio1>; ++ interrupts = <21 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-0 = <&pmic_int_l>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ ++ vcc1-supply = <&vcc3v3_sys>; ++ vcc2-supply = <&vcc3v3_sys>; ++ vcc3-supply = <&vcc3v3_sys>; ++ vcc4-supply = <&vcc3v3_sys>; ++ vcc6-supply = <&vcc3v3_sys>; ++ vcc7-supply = <&vcc3v3_sys>; ++ vcc8-supply = <&vcc3v3_sys>; ++ vcc9-supply = <&vcc3v3_sys>; ++ vcc10-supply = <&vcc3v3_sys>; ++ vcc11-supply = <&vcc3v3_sys>; ++ vcc12-supply = <&vcc3v3_sys>; ++ vddio-supply = <&vcc1v8_pmu>; ++ ++ regulators { ++ vdd_center: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <750000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-name = "vdd_center"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdd_cpu_l: DCDC_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <750000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-name = "vdd_cpu_l"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc_ddr"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_1v8: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc_1v8"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vcc1v8_dvp: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc1v8_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc3v0_tp: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-name = "vcc3v0_tp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc1v8_pmu: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc1v8_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vcc_sd: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-name = "vcc_sd"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ ++ vcca3v0_codec: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-name = "vcca3v0_codec"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_1v5: LDO_REG6 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ regulator-name = "vcc_1v5"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1500000>; ++ }; ++ }; ++ ++ vcca1v8_codec: LDO_REG7 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcca1v8_codec"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_3v0: LDO_REG8 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ regulator-name = "vcc_3v0"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ ++ vcc3v3_s3: SWITCH_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc3v3_s3"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc3v3_s0: SWITCH_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc3v3_s0"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ }; ++ }; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <140>; ++ i2c-scl-falling-time-ns = <30>; ++ ++ cx2072x:cx2072x@33 { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++ compatible = "cnxt,cx20723"; ++ reg = <0x33>; ++ clocks = <&cru SCLK_I2S_8CH_OUT>; ++ clock-names = "mclk"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s_8ch_mclk>; ++ spk-con-gpio = <&gpio0 11 GPIO_ACTIVE_HIGH>; ++ }; ++}; ++ ++&i2c3 { ++ status="okay"; ++ ++ hidkey@68 { ++ clock-frequency = <100000>; ++ compatible = "hid-over-i2c"; ++ interrupt-parent = <&gpio0>; ++ interrupts = ; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hidkey_irq_gpio>; ++ reg = <0x68>; ++ hid-descr-addr = <0x0001>; ++ hid-support-wakeup; ++ }; ++ ++ ec_battery@76 { ++ compatible = "rockchip,ec-battery"; ++ reg = <0x76>; ++ virtual_power = <0>; ++ monitor_sec = <5>; ++ ec-notify-gpios = <&gpio1 RK_PC4 GPIO_ACTIVE_HIGH>; ++ }; ++}; ++ ++&i2c2 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <345>; ++ i2c-scl-falling-time-ns = <11>; ++ clock-frequency = <400000>; ++ ++ touchpad: touchpad@2c { ++ compatible = "hid-over-i2c"; ++ interrupt-parent = <&gpio1>; ++ interrupts = ; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&touchpad_irq_gpio>; ++ reg = <0x2c>; ++ hid-descr-addr = <0x002c>; ++ }; ++}; ++ ++&i2c4 { ++ status = "okay"; ++ clock-frequency = <100000>; ++ bq25700: bq25700@09 {//6a ++ compatible = "ti,bq25700"; ++ reg = <0x09>; ++ extcon = <&fusb0>, <&fusb1>; ++ ++ interrupt-parent = <&gpio1>; ++ interrupts = <23 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&charger_ok>; ++ ti,charge-current = <2500000>; ++ ti,max-input-voltage = <20000000>; ++ ti,max-input-current = <6000000>; ++ ti,max-charge-voltage = <8750000>; ++ ti,input-current = <500000>; ++ ti,input-current-sdp = <500000>; ++ ti,input-current-dcp = <2000000>; ++ ti,input-current-cdp = <2000000>; ++ ti,minimum-sys-voltage = <7400000>; ++ ti,otg-voltage = <5000000>; ++ ti,otg-current = <500000>; ++ pd-charge-only = <1>; ++ typec0-enable-gpios = <&gpio1 RK_PA3 GPIO_ACTIVE_HIGH>; ++ typec1-enable-gpios = <&gpio1 RK_PA4 GPIO_ACTIVE_HIGH>; ++ }; ++}; ++ ++&i2c6 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <345>; ++ i2c-scl-falling-time-ns = <11>; ++ clock-frequency = <400000>; ++ ++ fusb1: fusb30x@22 { ++ compatible = "fairchild,fusb302"; ++ reg = <0x22>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&fusb1_int>; ++ vbus-5v-gpios = <&gpio4 RK_PD6 GPIO_ACTIVE_HIGH>; ++ int-n-gpios = <&gpio1 RK_PB5 GPIO_ACTIVE_HIGH>; ++ discharge-gpios = <&gpio0 RK_PB5 GPIO_ACTIVE_HIGH>; ++ charge-dev = <&bq25700>; ++ support-uboot-charge = <1>; ++ port-num = <1>; ++ status = "okay"; ++ }; ++}; ++ ++&i2c7 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <345>; ++ i2c-scl-falling-time-ns = <11>; ++ clock-frequency = <400000>; ++ ++ fusb0: fusb30x@22 { ++ compatible = "fairchild,fusb302"; ++ reg = <0x22>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&fusb0_int>; ++ vbus-5v-gpios = <&gpio4 RK_PD5 GPIO_ACTIVE_HIGH>; ++ int-n-gpios = <&gpio1 RK_PA2 GPIO_ACTIVE_HIGH>; ++ discharge-gpios = <&gpio0 RK_PB4 GPIO_ACTIVE_HIGH>; ++ charge-dev = <&bq25700>; ++ support-uboot-charge = <1>; ++ port-num = <0>; ++ status = "okay"; ++ }; ++}; ++ ++&io_domains { ++ status = "okay"; ++ bt656-supply = <&vcc_3v0>; ++ audio-supply = <&vcca1v8_codec>; ++ sdmmc-supply = <&vcc_sd>; ++ gpio1830-supply = <&vcc_3v0>; ++}; ++ ++&dsi { ++ status = "disabled"; ++}; ++ ++&pcie_phy { ++ status = "okay"; ++}; ++ ++&pcie0 { ++ ep-gpios = <&gpio3 7 GPIO_ACTIVE_HIGH>; ++ num-lanes = <4>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pcie_clkreqn_cpm>; ++ status = "okay"; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ pmu1830-supply = <&vcc_1v8>; ++}; ++ ++&pwm0 { ++ status = "okay"; ++}; ++ ++&pwm2 { ++ status = "okay"; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm2_pin_pull_down>; ++}; ++ ++&route_edp { ++ status = "okay"; ++ logo,mode = "center"; ++}; ++ ++&saradc { ++ status = "okay"; ++}; ++ ++&sdmmc { ++ clock-frequency = <50000000>; ++ clock-freq-min-max = <400000 150000000>; ++ no-sdio; ++ no-mmc; ++ bus-width = <4>; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ disable-wp; ++ num-slots = <1>; ++ //sd-uhs-sdr104; ++ vqmmc-supply = <&vcc_sd>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_cd &sdmmc_bus4>; ++ status = "okay"; ++}; ++ ++&sdio0 { ++ clock-frequency = <150000000>; ++ clock-freq-min-max = <200000 150000000>; ++ no-sd; ++ no-mmc; ++ bus-width = <4>; ++ disable-wp; ++ cap-sd-highspeed; ++ cap-sdio-irq; ++ keep-power-in-suspend; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ non-removable; ++ num-slots = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&sdhci { ++ bus-width = <8>; ++ mmc-hs400-1_8v; ++ no-sdio; ++ no-sd; ++ non-removable; ++ keep-power-in-suspend; ++ mmc-hs400-enhanced-strobe; ++ status = "okay"; ++}; ++ ++&spdif { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++}; ++ ++&spi1 { ++ status = "disabled"; ++}; ++ ++&tcphy0 { ++ extcon = <&fusb0>; ++ status = "okay"; ++}; ++ ++&tcphy1 { ++ extcon = <&fusb1>; ++ status = "okay"; ++}; ++ ++&tsadc { ++ rockchip,hw-tshut-mode = <1>; /* tshut mode 0:CRU 1:GPIO */ ++ rockchip,hw-tshut-polarity = <1>; /* tshut polarity 0:LOW 1:HIGH */ ++ status = "okay"; ++}; ++ ++&u2phy0 { ++ status = "okay"; ++ extcon = <&fusb0>; ++ ++ u2phy0_otg: otg-port { ++ status = "okay"; ++ }; ++ ++ u2phy0_host: host-port { ++ phy-supply = <&vcc5v0_host>; ++ status = "okay"; ++ }; ++}; ++ ++&u2phy1 { ++ status = "okay"; ++ extcon = <&fusb1>; ++ ++ u2phy1_otg: otg-port { ++ status = "okay"; ++ }; ++ ++ u2phy1_host: host-port { ++ phy-supply = <&vcc5v0_host>; ++ status = "okay"; ++ }; ++}; ++ ++&uart0 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart0_xfer &uart0_cts>; ++ status = "okay"; ++}; ++ ++&uart2 { ++ status = "okay"; ++}; ++ ++&usb_host0_ehci { ++ status = "okay"; ++}; ++ ++&usb_host0_ohci { ++ status = "okay"; ++}; ++ ++&usbdrd3_0 { ++ status = "okay"; ++}; ++ ++&usbdrd_dwc3_0 { ++ status = "okay"; ++ extcon = <&fusb0>; ++}; ++ ++&usb_host1_ehci { ++ status = "okay"; ++}; ++ ++&usb_host1_ohci { ++ status = "okay"; ++}; ++ ++&usbdrd3_1 { ++ status = "okay"; ++}; ++ ++&usbdrd_dwc3_1 { ++ status = "okay"; ++ extcon = <&fusb1>; ++}; ++ ++&rockchip_suspend { ++ status = "okay"; ++ rockchip,sleep-debug-en = <1>; ++ rockchip,sleep-mode-config = < ++ (0 ++ | RKPM_SLP_ARMPD ++ | RKPM_SLP_PERILPPD ++ | RKPM_SLP_DDR_RET ++ | RKPM_SLP_PLLPD ++ | RKPM_SLP_CENTER_PD ++ | RKPM_SLP_AP_PWROFF ++ ) ++ >; ++ rockchip,wakeup-config = < ++ (0 ++ | RKPM_GPIO_WKUP_EN ++ | RKPM_PWM_WKUP_EN ++ ) ++ >; ++ rockchip,pwm-regulator-config = < ++ (0 ++ | PWM2_REGULATOR_EN ++ ) ++ >; ++ rockchip,power-ctrl = ++ <&gpio1 17 GPIO_ACTIVE_HIGH>, ++ <&gpio1 14 GPIO_ACTIVE_HIGH>; ++}; ++ ++&pinctrl { ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <0 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart0_gpios: uart0-gpios { ++ rockchip,pins = <2 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ pmic { ++ pmic_int_l: pmic-int-l { ++ rockchip,pins = ++ <1 RK_PC5 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ vsel1_gpio: vsel1-gpio { ++ rockchip,pins = ++ <1 RK_PC1 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ vsel2_gpio: vsel2-gpio { ++ rockchip,pins = ++ <1 RK_PB6 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ headphone { ++ hp_det: hp-det { ++ rockchip,pins = <4 RK_PD4 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ hallsensor { ++ mh248_irq_gpio: mh248-irq-gpio { ++ rockchip,pins = <0 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ hidkey { ++ hidkey_irq_gpio: hidkey-irq-gpio { ++ rockchip,pins = <0 RK_PB0 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ touchpad { ++ touchpad_irq_gpio: touchpad-irq-gpio { ++ rockchip,pins = <1 RK_PC2 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ charger { ++ charger_ok: charge-ok { ++ rockchip,pins = ++ <1 RK_PC7 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ gpio-leds { ++ leds_gpio: leds-gpio { ++ rockchip,pins = ++ <1 RK_PC6 RK_FUNC_GPIO &pcfg_pull_up>, ++ <1 RK_PD0 RK_FUNC_GPIO &pcfg_pull_up>, ++ <3 RK_PA5 RK_FUNC_GPIO &pcfg_pull_up>, ++ <3 RK_PB1 RK_FUNC_GPIO &pcfg_pull_up>, ++ <2 RK_PD3 RK_FUNC_GPIO &pcfg_pull_up>, ++ <2 RK_PD4 RK_FUNC_GPIO &pcfg_pull_up>, ++ <4 RK_PD2 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ usb2 { ++ host_vbus_drv: host-vbus-drv { ++ rockchip,pins = ++ <4 RK_PD1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ usb_camera { ++ usb_cam_on_gpio: usb-cam-on-gpio { ++ rockchip,pins = ++ <3 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>, ++ <3 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ fusb30x { ++ fusb0_int: fusb0-int { ++ rockchip,pins = ++ <1 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>, ++ <0 RK_PB4 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ fusb1_int: fusb1-int { ++ rockchip,pins = ++ <1 RK_PB5 RK_FUNC_GPIO &pcfg_pull_up>, ++ <0 RK_PB5 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-videostrong-linux.dts b/arch/arm64/boot/dts/rockchip/rk3399-videostrong-linux.dts +new file mode 100755 +index 000000000000..10ba2048483d +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-videostrong-linux.dts +@@ -0,0 +1,293 @@ ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++ */ ++ ++/dts-v1/; ++ ++#include "rk3399-sapphire.dtsi" ++#include "rk3399-linux.dtsi" ++#include ++ ++/ { ++ model = "Rockchip RK3399 Videostrong Board (Linux Opensource)"; ++ compatible = "rockchip,rk3399-videostrong-linux", "rockchip,rk3399"; ++ ++ fiq_debugger: fiq-debugger { ++ compatible = "rockchip,fiq-debugger"; ++ rockchip,serial-id = <2>; ++ rockchip,signal-irq = <182>; ++ rockchip,wake-irq = <0>; ++ rockchip,irq-mode-enable = <1>; /* If enable uart uses irq instead of fiq */ ++ rockchip,baudrate = <1500000>; /* Only 115200 and 1500000 */ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart2c_xfer>; ++ }; ++ ++ gpio-keys { ++ compatible = "gpio-keys"; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ autorepeat; ++ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pwrbtn>; ++ ++ button@0 { ++ gpios = <&gpio0 5 GPIO_ACTIVE_LOW>; ++ linux,code = ; ++ label = "GPIO Key Power"; ++ linux,input-type = <1>; ++ gpio-key,wakeup = <1>; ++ debounce-interval = <100>; ++ }; ++ }; ++ ++ hdmi_sound: hdmi-sound { ++ status = "okay"; ++ }; ++ ++ rt5640-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,rt5640-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,widgets = ++ "Microphone", "Mic Jack", ++ "Headphone", "Headphone Jack"; ++ simple-audio-card,routing = ++ "Mic Jack", "MICBIAS1", ++ "IN1P", "Mic Jack", ++ "Headphone Jack", "HPOL", ++ "Headphone Jack", "HPOR"; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s1>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&rt5640>; ++ }; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ clocks = <&rk808 1>; ++ clock-names = "ext_clock"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio0 10 GPIO_ACTIVE_LOW>; /* GPIO0_B2 */ ++ }; ++ ++ spdif-sound { ++ status = "okay"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,name = "ROCKCHIP,SPDIF"; ++ simple-audio-card,cpu { ++ sound-dai = <&spdif>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&spdif_out>; ++ }; ++ }; ++ ++ spdif_out: spdif-out { ++ status = "okay"; ++ compatible = "linux,spdif-dit"; ++ #sound-dai-cells = <0>; ++ }; ++ ++ vccadc_ref: vccadc-ref { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc1v8_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&rk808 1>; ++ clock-names = "ext_clock"; ++ uart_rts_gpios = <&gpio2 19 GPIO_ACTIVE_LOW>; /* GPIO2_C3 */ ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart0_rts>; ++ pinctrl-1 = <&uart0_gpios>; ++ BT,reset_gpio = <&gpio0 9 GPIO_ACTIVE_HIGH>; /* GPIO0_B1 */ ++ BT,wake_gpio = <&gpio2 26 GPIO_ACTIVE_HIGH>; /* GPIO2_D2 */ ++ BT,wake_host_irq = <&gpio0 4 GPIO_ACTIVE_HIGH>; /* GPIO0_A4 */ ++ status = "okay"; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ rockchip,grf = <&grf>; ++ wifi_chip_type = "ap6354"; ++ sdio_vref = <1800>; ++ WIFI,host_wake_irq = <&gpio0 3 GPIO_ACTIVE_HIGH>; /* GPIO0_a3 */ ++ status = "okay"; ++ }; ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&display_subsystem { ++ status = "okay"; ++}; ++ ++&dmc { ++ status = "okay"; ++ center-supply = <&vdd_center>; ++ upthreshold = <40>; ++ downdifferential = <20>; ++ system-status-freq = < ++ /* system status freq(KHz) */ ++ SYS_STATUS_NORMAL 800000 ++ SYS_STATUS_REBOOT 528000 ++ SYS_STATUS_SUSPEND 200000 ++ SYS_STATUS_VIDEO_1080P 200000 ++ SYS_STATUS_VIDEO_4K 600000 ++ SYS_STATUS_VIDEO_4K_10B 800000 ++ SYS_STATUS_PERFORMANCE 800000 ++ SYS_STATUS_BOOST 400000 ++ SYS_STATUS_DUALVIEW 600000 ++ SYS_STATUS_ISP 600000 ++ >; ++ vop-bw-dmc-freq = < ++ /* min_bw(MB/s) max_bw(MB/s) freq(KHz) */ ++ 0 577 200000 ++ 578 1701 300000 ++ 1702 99999 400000 ++ >; ++ auto-min-freq = <200000>; ++}; ++ ++&hdmi { ++ /* remove the hdmi_cec, reused by edp_hpd */ ++ pinctrl-0 = <&hdmi_i2c_xfer>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ #sound-dai-cells = <0>; ++ status = "okay"; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <300>; ++ i2c-scl-falling-time-ns = <15>; ++ ++ rt5640: rt5640@1c { ++ #sound-dai-cells = <0>; ++ compatible = "realtek,rt5640"; ++ reg = <0x1c>; ++ clocks = <&cru SCLK_I2S_8CH_OUT>; ++ clock-names = "mclk"; ++ realtek,in1-differential; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s_8ch_mclk>; ++ }; ++}; ++ ++&i2s1 { ++ status = "okay"; ++ rockchip,i2s-broken-burst-len; ++ rockchip,playback-channels = <2>; ++ rockchip,capture-channels = <2>; ++ #sound-dai-cells = <0>; ++}; ++ ++&rkvdec { ++ status = "okay"; ++ /* 0 means ion, 1 means drm */ ++ //allocator = <0>; ++}; ++ ++&rockchip_suspend { ++ status = "okay"; ++ rockchip,sleep-debug-en = <1>; ++ rockchip,sleep-mode-config = < ++ (0 ++ | RKPM_SLP_ARMPD ++ | RKPM_SLP_PERILPPD ++ | RKPM_SLP_DDR_RET ++ | RKPM_SLP_PLLPD ++ | RKPM_SLP_CENTER_PD ++ | RKPM_SLP_AP_PWROFF ++ ) ++ >; ++ rockchip,wakeup-config = < ++ (0 ++ | RKPM_GPIO_WKUP_EN ++ | RKPM_PWM_WKUP_EN ++ ) ++ >; ++ rockchip,pwm-regulator-config = < ++ (0 ++ | PWM2_REGULATOR_EN ++ ) ++ >; ++ rockchip,power-ctrl = ++ <&gpio1 17 GPIO_ACTIVE_HIGH>, ++ <&gpio1 14 GPIO_ACTIVE_HIGH>; ++}; ++ ++&spdif { ++ status = "okay"; ++ pinctrl-0 = <&spdif_bus>; ++ i2c-scl-rising-time-ns = <450>; ++ i2c-scl-falling-time-ns = <15>; ++ #sound-dai-cells = <0>; ++}; ++ ++&uart0 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart0_xfer &uart0_cts>; ++ status = "okay"; ++}; ++ ++&vopb { ++ status = "okay"; ++}; ++ ++&vopb_mmu { ++ status = "okay"; ++}; ++ ++&vopl { ++ status = "okay"; ++}; ++ ++&vopl_mmu { ++ status = "okay"; ++}; ++ ++&pinctrl { ++ buttons { ++ pwrbtn: pwrbtn { ++ rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = ++ <0 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart0_gpios: uart0-gpios { ++ rockchip,pins = ++ <2 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399-vop-clk-set.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-vop-clk-set.dtsi +new file mode 100755 +index 000000000000..5ed8dac6cf7d +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399-vop-clk-set.dtsi +@@ -0,0 +1,181 @@ ++/* ++ * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This file is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This file is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++/* ++ * This define is for support double show any dclk frequency. ++ * dclk_vop will have a exclusive pll as parent. ++ * set dclk_vop will change the pll rate as well. ++ */ ++ ++#ifdef RK3399_TWO_PLL_FOR_VOP ++ ++&sdhci { ++ assigned-clocks = <&cru SCLK_EMMC>; ++ assigned-clock-parents = <&cru PLL_GPLL>; ++ assigned-clock-rates = <200000000>; ++}; ++ ++&uart0 { ++ assigned-clocks = <&cru SCLK_UART0_SRC>; ++ assigned-clock-parents = <&cru PLL_GPLL>; ++}; ++ ++&uart1 { ++ assigned-clocks = <&cru SCLK_UART_SRC>; ++ assigned-clock-parents = <&cru PLL_GPLL>; ++}; ++ ++&uart2 { ++ assigned-clocks = <&cru SCLK_UART_SRC>; ++ assigned-clock-parents = <&cru PLL_GPLL>; ++}; ++ ++&uart3 { ++ assigned-clocks = <&cru SCLK_UART_SRC>; ++ assigned-clock-parents = <&cru PLL_GPLL>; ++}; ++ ++&uart4 { ++ assigned-clocks = <&pmucru SCLK_UART4_SRC>; ++ assigned-clock-parents = <&pmucru PLL_PPLL>; ++}; ++ ++&spdif { ++ assigned-clocks = <&cru SCLK_SPDIF_DIV>; ++ assigned-clock-parents = <&cru PLL_GPLL>; ++}; ++ ++&i2s0{ ++ assigned-clocks = <&cru SCLK_I2S0_DIV>; ++ assigned-clock-parents = <&cru PLL_GPLL>; ++}; ++ ++&i2s1 { ++ assigned-clocks = <&cru SCLK_I2S1_DIV>; ++ assigned-clock-parents = <&cru PLL_GPLL>; ++}; ++ ++&i2s2 { ++ assigned-clocks = <&cru SCLK_I2S2_DIV>; ++ assigned-clock-parents = <&cru PLL_GPLL>; ++}; ++ ++&cru { ++ assigned-clocks = ++ <&cru ACLK_PERIHP>, <&cru ACLK_PERILP0>, ++ <&cru HCLK_PERILP1>, <&cru SCLK_SDMMC>, ++ <&cru ACLK_EMMC>, <&cru ACLK_CENTER>, ++ <&cru HCLK_SD>, <&cru SCLK_VDU_CA>, ++ <&cru SCLK_VDU_CORE>, <&cru ACLK_USB3>, ++ <&cru FCLK_CM0S>, <&cru ACLK_CCI>, ++ <&cru PCLK_ALIVE>, <&cru ACLK_GMAC>, ++ <&cru SCLK_CS>, <&cru SCLK_CCI_TRACE>, ++ <&cru ARMCLKL>, <&cru ARMCLKB>, ++ <&cru PLL_NPLL>, <&cru ACLK_GPU>, ++ <&cru PLL_GPLL>, <&cru ACLK_PERIHP>, ++ <&cru HCLK_PERIHP>, <&cru PCLK_PERIHP>, ++ <&cru ACLK_PERILP0>, <&cru HCLK_PERILP0>, ++ <&cru PCLK_PERILP0>, <&cru HCLK_PERILP1>, ++ <&cru PCLK_PERILP1>, <&cru SCLK_I2C1>, ++ <&cru SCLK_I2C2>, <&cru SCLK_I2C3>, ++ <&cru SCLK_I2C5>, <&cru SCLK_I2C6>, ++ <&cru SCLK_I2C7>, <&cru SCLK_SPI0>, ++ <&cru SCLK_SPI1>, <&cru SCLK_SPI2>, ++ <&cru SCLK_SPI4>, <&cru SCLK_SPI5>, ++ <&cru ACLK_GIC>, <&cru ACLK_ISP0>, ++ <&cru ACLK_ISP1>, <&cru SCLK_VOP0_PWM>, ++ <&cru SCLK_VOP1_PWM>, <&cru PCLK_EDP>, ++ <&cru ACLK_HDCP>, <&cru ACLK_VIO>, ++ <&cru HCLK_SD>, <&cru SCLK_CRYPTO0>, ++ <&cru SCLK_CRYPTO1>, <&cru SCLK_EMMC>, ++ <&cru ACLK_EMMC>, <&cru ACLK_CENTER>, ++ <&cru ACLK_IEP>, <&cru ACLK_RGA>, ++ <&cru SCLK_RGA_CORE>, <&cru ACLK_VDU>, ++ <&cru ACLK_VCODEC>, <&cru PCLK_DDR>, ++ <&cru ACLK_GMAC>, <&cru SCLK_VDU_CA>, ++ <&cru SCLK_VDU_CORE>, <&cru ACLK_USB3>, ++ <&cru FCLK_CM0S>, <&cru ACLK_CCI>, ++ <&cru PCLK_ALIVE>, <&cru SCLK_CS>, ++ <&cru SCLK_CCI_TRACE>, <&cru ACLK_VOP0>, ++ <&cru HCLK_VOP0>, <&cru ACLK_VOP1>, ++ <&cru HCLK_VOP1>; ++ assigned-clock-rates = ++ <75000000>, <50000000>, ++ <50000000>, <50000000>, ++ <50000000>, <100000000>, ++ <50000000>, <150000000>, ++ <150000000>, <150000000>, ++ <50000000>, <150000000>, ++ <50000000>, <100000000>, ++ <75000000>, <75000000>, ++ <816000000>, <816000000>, ++ <600000000>, <200000000>, ++ <800000000>, <150000000>, ++ <75000000>, <37500000>, ++ <300000000>, <100000000>, ++ <50000000>, <100000000>, ++ <50000000>, <100000000>, ++ <100000000>, <100000000>, ++ <100000000>, <100000000>, ++ <100000000>, <50000000>, ++ <50000000>, <50000000>, ++ <50000000>, <50000000>, ++ <200000000>, <400000000>, ++ <400000000>, <100000000>, ++ <100000000>, <100000000>, ++ <400000000>, <400000000>, ++ <200000000>, <100000000>, ++ <200000000>, <200000000>, ++ <100000000>, <400000000>, ++ <400000000>, <400000000>, ++ <400000000>, <300000000>, ++ <400000000>, <200000000>, ++ <400000000>, <300000000>, ++ <300000000>, <300000000>, ++ <300000000>, <600000000>,/* aclk_cci */ ++ <100000000>, <150000000>, ++ <150000000>, <400000000>, ++ <100000000>, <400000000>, ++ <100000000>; ++}; ++#endif ++ +diff --git a/arch/arm64/boot/dts/rockchip/rk3399.dtsi b/arch/arm64/boot/dts/rockchip/rk3399.dtsi +index 4b6065dbba55..d1bf1b6a0d10 100644 +--- a/arch/arm64/boot/dts/rockchip/rk3399.dtsi ++++ b/arch/arm64/boot/dts/rockchip/rk3399.dtsi +@@ -9,8 +9,13 @@ + #include + #include + #include ++#include ++#include ++#include + #include + ++#include "rk3399-dram-default-timing.dtsi" ++ + / { + compatible = "rockchip,rk3399"; + +@@ -164,7 +169,7 @@ CLUSTER_SLEEP: cluster-sleep { + }; + }; + +- display-subsystem { ++ display_subsystem: display-subsystem { + compatible = "rockchip,display-subsystem"; + ports = <&vopl_out>, <&vopb_out>; + }; +@@ -200,6 +205,20 @@ xin24m: xin24m { + #clock-cells = <0>; + }; + ++ dummy_cpll: dummy_cpll { ++ compatible = "fixed-clock"; ++ clock-frequency = <0>; ++ clock-output-names = "dummy_cpll"; ++ #clock-cells = <0>; ++ }; ++ ++ dummy_vpll: dummy_vpll { ++ compatible = "fixed-clock"; ++ clock-frequency = <0>; ++ clock-output-names = "dummy_vpll"; ++ #clock-cells = <0>; ++ }; ++ + amba: bus { + compatible = "simple-bus"; + #address-cells = <2>; +@@ -346,6 +365,7 @@ sdhci: sdhci@fe330000 { + phy-names = "phy_arasan"; + power-domains = <&power RK3399_PD_EMMC>; + disable-cqe-dcmd; ++ disable-cqe; + status = "disabled"; + }; + +@@ -764,76 +784,58 @@ spi5: spi@ff200000 { + }; + + thermal_zones: thermal-zones { +- cpu_thermal: cpu { +- polling-delay-passive = <100>; ++ soc_thermal: cpu_thermal: cpu-thermal { ++ polling-delay-passive = <20>; + polling-delay = <1000>; ++ sustainable-power = <1000>; /* milliwatts */ + + thermal-sensors = <&tsadc 0>; + + trips { +- cpu_alert0: cpu_alert0 { ++ threshold: cpu_alert0: cpu_alert0 { + temperature = <70000>; + hysteresis = <2000>; + type = "passive"; + }; +- cpu_alert1: cpu_alert1 { +- temperature = <75000>; ++ target: cpu_alert1: cpu_alert1 { ++ temperature = <85000>; + hysteresis = <2000>; + type = "passive"; + }; +- cpu_crit: cpu_crit { +- temperature = <95000>; +- hysteresis = <2000>; ++ soc_crit: cpu_crit: cpu_crit { ++ temperature = <115000>; /* millicelsius */ ++ hysteresis = <2000>; /* millicelsius */ + type = "critical"; + }; + }; + + cooling-maps { + map0 { +- trip = <&cpu_alert0>; ++ trip = <&target>; + cooling-device = +- <&cpu_b0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, +- <&cpu_b1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; ++ <&cpu_l0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; ++ contribution = <4096>; + }; + map1 { +- trip = <&cpu_alert1>; ++ trip = <&target>; ++ cooling-device = ++ <&cpu_b0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; ++ contribution = <1024>; ++ }; ++ map2 { ++ trip = <&target>; + cooling-device = +- <&cpu_l0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, +- <&cpu_l1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, +- <&cpu_l2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, +- <&cpu_l3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, +- <&cpu_b0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, +- <&cpu_b1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; ++ <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; ++ contribution = <4096>; + }; + }; + }; + +- gpu_thermal: gpu { ++ gpu_thermal: gpu-thermal { + polling-delay-passive = <100>; + polling-delay = <1000>; + + thermal-sensors = <&tsadc 1>; +- +- trips { +- gpu_alert0: gpu_alert0 { +- temperature = <75000>; +- hysteresis = <2000>; +- type = "passive"; +- }; +- gpu_crit: gpu_crit { +- temperature = <95000>; +- hysteresis = <2000>; +- type = "critical"; +- }; +- }; +- +- cooling-maps { +- map0 { +- trip = <&gpu_alert0>; +- cooling-device = +- <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; +- }; +- }; + }; + }; + +@@ -849,10 +851,9 @@ tsadc: tsadc@ff260000 { + reset-names = "tsadc-apb"; + rockchip,grf = <&grf>; + rockchip,hw-tshut-temp = <95000>; +- pinctrl-names = "init", "default", "sleep"; ++ pinctrl-names = "gpio", "otpout"; + pinctrl-0 = <&otp_pin>; + pinctrl-1 = <&otp_out>; +- pinctrl-2 = <&otp_pin>; + #thermal-sensor-cells = <1>; + status = "disabled"; + }; +@@ -1000,26 +1001,26 @@ power: power-controller { + #size-cells = <0>; + + /* These power domains are grouped by VD_CENTER */ +- power-domain@RK3399_PD_IEP { ++ pd_iep@RK3399_PD_IEP { + reg = ; + clocks = <&cru ACLK_IEP>, + <&cru HCLK_IEP>; + pm_qos = <&qos_iep>; + }; +- power-domain@RK3399_PD_RGA { ++ pd_rga@RK3399_PD_RGA { + reg = ; + clocks = <&cru ACLK_RGA>, + <&cru HCLK_RGA>; + pm_qos = <&qos_rga_r>, + <&qos_rga_w>; + }; +- power-domain@RK3399_PD_VCODEC { ++ pd_vcodec@RK3399_PD_VCODEC { + reg = ; + clocks = <&cru ACLK_VCODEC>, + <&cru HCLK_VCODEC>; + pm_qos = <&qos_video_m0>; + }; +- power-domain@RK3399_PD_VDU { ++ pd_vdu@RK3399_PD_VDU { + reg = ; + clocks = <&cru ACLK_VDU>, + <&cru HCLK_VDU>; +@@ -1028,94 +1029,94 @@ power-domain@RK3399_PD_VDU { + }; + + /* These power domains are grouped by VD_GPU */ +- power-domain@RK3399_PD_GPU { ++ pd_gpu@RK3399_PD_GPU { + reg = ; + clocks = <&cru ACLK_GPU>; + pm_qos = <&qos_gpu>; + }; + + /* These power domains are grouped by VD_LOGIC */ +- power-domain@RK3399_PD_EDP { ++ pd_edp@RK3399_PD_EDP { + reg = ; + clocks = <&cru PCLK_EDP_CTRL>; + }; +- power-domain@RK3399_PD_EMMC { ++ pd_emmc@RK3399_PD_EMMC { + reg = ; + clocks = <&cru ACLK_EMMC>; + pm_qos = <&qos_emmc>; + }; +- power-domain@RK3399_PD_GMAC { ++ pd_gmac@RK3399_PD_GMAC { + reg = ; + clocks = <&cru ACLK_GMAC>, + <&cru PCLK_GMAC>; + pm_qos = <&qos_gmac>; + }; +- power-domain@RK3399_PD_SD { ++ pd_sd@RK3399_PD_SD { + reg = ; + clocks = <&cru HCLK_SDMMC>, + <&cru SCLK_SDMMC>; + pm_qos = <&qos_sd>; + }; +- power-domain@RK3399_PD_SDIOAUDIO { ++ pd_sdioaudio@RK3399_PD_SDIOAUDIO { + reg = ; + clocks = <&cru HCLK_SDIO>; + pm_qos = <&qos_sdioaudio>; + }; +- power-domain@RK3399_PD_TCPD0 { ++ pd_tcpc0@RK3399_PD_TCPD0 { + reg = ; + clocks = <&cru SCLK_UPHY0_TCPDCORE>, + <&cru SCLK_UPHY0_TCPDPHY_REF>; + }; +- power-domain@RK3399_PD_TCPD1 { ++ pd_tcpc1@RK3399_PD_TCPD1 { + reg = ; + clocks = <&cru SCLK_UPHY1_TCPDCORE>, + <&cru SCLK_UPHY1_TCPDPHY_REF>; + }; +- power-domain@RK3399_PD_USB3 { ++ pd_usb3@RK3399_PD_USB3 { + reg = ; + clocks = <&cru ACLK_USB3>; + pm_qos = <&qos_usb_otg0>, + <&qos_usb_otg1>; + }; +- power-domain@RK3399_PD_VIO { ++ pd_vio@RK3399_PD_VIO { + reg = ; + #address-cells = <1>; + #size-cells = <0>; + +- power-domain@RK3399_PD_HDCP { ++ pd_hdcp@RK3399_PD_HDCP { + reg = ; + clocks = <&cru ACLK_HDCP>, + <&cru HCLK_HDCP>, + <&cru PCLK_HDCP>; + pm_qos = <&qos_hdcp>; + }; +- power-domain@RK3399_PD_ISP0 { ++ pd_isp0@RK3399_PD_ISP0 { + reg = ; + clocks = <&cru ACLK_ISP0>, + <&cru HCLK_ISP0>; + pm_qos = <&qos_isp0_m0>, + <&qos_isp0_m1>; + }; +- power-domain@RK3399_PD_ISP1 { ++ pd_isp1@RK3399_PD_ISP1 { + reg = ; + clocks = <&cru ACLK_ISP1>, + <&cru HCLK_ISP1>; + pm_qos = <&qos_isp1_m0>, + <&qos_isp1_m1>; + }; +- power-domain@RK3399_PD_VO { ++ pd_vo@RK3399_PD_VO { + reg = ; + #address-cells = <1>; + #size-cells = <0>; + +- power-domain@RK3399_PD_VOPB { ++ pd_vopb@RK3399_PD_VOPB { + reg = ; + clocks = <&cru ACLK_VOP0>, + <&cru HCLK_VOP0>; + pm_qos = <&qos_vop_big_r>, + <&qos_vop_big_w>; + }; +- power-domain@RK3399_PD_VOPL { ++ pd_vopl@RK3399_PD_VOPL { + reg = ; + clocks = <&cru ACLK_VOP1>, + <&cru HCLK_VOP1>; +@@ -1134,6 +1135,33 @@ pmu_io_domains: io-domains { + compatible = "rockchip,rk3399-pmu-io-voltage-domain"; + status = "disabled"; + }; ++ ++ reboot_mode: reboot-mode { ++ compatible = "syscon-reboot-mode"; ++ offset = <0x300>; ++ mode-charge = ; ++ mode-fastboot = ; ++ mode-loader = ; ++ mode-normal = ; ++ mode-panic = ; ++ mode-recovery = ; ++ mode-ums = ; ++ }; ++ ++ pmu_pvtm: pmu-pvtm { ++ compatible = "rockchip,rk3399-pmu-pvtm"; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ status = "disabled"; ++ ++ pvtm@4 { ++ reg = <4>; ++ clocks = <&pmucru SCLK_PVTM_PMU>; ++ clock-names = "clk"; ++ resets = <&pmucru SRST_PVTM>; ++ reset-names = "rst"; ++ }; ++ }; + }; + + spi3: spi@ff350000 { +@@ -1211,7 +1239,7 @@ pwm0: pwm@ff420000 { + compatible = "rockchip,rk3399-pwm", "rockchip,rk3288-pwm"; + reg = <0x0 0xff420000 0x0 0x10>; + #pwm-cells = <3>; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm0_pin>; + clocks = <&pmucru PCLK_RKPWM_PMU>; + clock-names = "pwm"; +@@ -1222,7 +1250,7 @@ pwm1: pwm@ff420010 { + compatible = "rockchip,rk3399-pwm", "rockchip,rk3288-pwm"; + reg = <0x0 0xff420010 0x0 0x10>; + #pwm-cells = <3>; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm1_pin>; + clocks = <&pmucru PCLK_RKPWM_PMU>; + clock-names = "pwm"; +@@ -1233,7 +1261,7 @@ pwm2: pwm@ff420020 { + compatible = "rockchip,rk3399-pwm", "rockchip,rk3288-pwm"; + reg = <0x0 0xff420020 0x0 0x10>; + #pwm-cells = <3>; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm2_pin>; + clocks = <&pmucru PCLK_RKPWM_PMU>; + clock-names = "pwm"; +@@ -1244,13 +1272,32 @@ pwm3: pwm@ff420030 { + compatible = "rockchip,rk3399-pwm", "rockchip,rk3288-pwm"; + reg = <0x0 0xff420030 0x0 0x10>; + #pwm-cells = <3>; +- pinctrl-names = "default"; ++ pinctrl-names = "active"; + pinctrl-0 = <&pwm3a_pin>; + clocks = <&pmucru PCLK_RKPWM_PMU>; + clock-names = "pwm"; + status = "disabled"; + }; + ++ dfi: dfi@ff630000 { ++ reg = <0x00 0xff630000 0x00 0x4000>; ++ compatible = "rockchip,rk3399-dfi"; ++ rockchip,pmu = <&pmugrf>; ++ clocks = <&cru PCLK_DDR_MON>; ++ clock-names = "pclk_ddr_mon"; ++ status = "disabled"; ++ }; ++ ++ dmc: dmc { ++ compatible = "rockchip,rk3399-dmc"; ++ devfreq-events = <&dfi>; ++ interrupts = ; ++ clocks = <&cru SCLK_DDRC>; ++ clock-names = "dmc_clk"; ++ ddr_timing = <&ddr_timing>; ++ status = "disabled"; ++ }; ++ + vpu: video-codec@ff650000 { + compatible = "rockchip,rk3399-vpu"; + reg = <0x0 0xff650000 0x0 0x800>; +@@ -1296,6 +1343,20 @@ vdec_mmu: iommu@ff660480 { + #iommu-cells = <0>; + }; + ++ iep: iep@ff670000 { ++ compatible = "rockchip,iep"; ++ iommu_enabled = <1>; ++ iommus = <&iep_mmu>; ++ reg = <0x0 0xff670000 0x0 0x800>; ++ interrupts = ; ++ clocks = <&cru ACLK_IEP>, <&cru HCLK_IEP>; ++ clock-names = "aclk_iep", "hclk_iep"; ++ power-domains = <&power RK3399_PD_IEP>; ++ allocator = <1>; ++ version = <2>; ++ status = "disabled"; ++ }; ++ + iep_mmu: iommu@ff670800 { + compatible = "rockchip,iommu"; + reg = <0x0 0xff670800 0x0 0x40>; +@@ -1327,6 +1388,10 @@ efuse0: efuse@ff690000 { + clock-names = "pclk_efuse"; + + /* Data cells */ ++ specification_serial_number: specification-serial-number@6 { ++ reg = <0x06 0x1>; ++ bits = <0 5>; ++ }; + cpu_id: cpu-id@7 { + reg = <0x07 0x10>; + }; +@@ -1348,6 +1413,10 @@ logic_leakage: logic-leakage@1b { + wafer_info: wafer-info@1c { + reg = <0x1c 0x1>; + }; ++ customer_demand: customer-demand@22 { ++ reg = <0x22 0x1>; ++ bits = <4 4>; ++ }; + }; + + pmucru: pmu-clock-controller@ff750000 { +@@ -1485,6 +1554,42 @@ pcie_phy: pcie-phy { + reset-names = "phy"; + status = "disabled"; + }; ++ ++ pvtm: pvtm { ++ compatible = "rockchip,rk3399-pvtm"; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ status = "disabled"; ++ ++ pvtm@0 { ++ reg = <0>; ++ clocks = <&cru SCLK_PVTM_CORE_L>; ++ clock-names = "clk"; ++ resets = <&cru SRST_PVTM_CORE_L>; ++ reset-names = "rst"; ++ }; ++ pvtm@1 { ++ reg = <1>; ++ clocks = <&cru SCLK_PVTM_CORE_B>; ++ clock-names = "clk"; ++ resets = <&cru SRST_PVTM_CORE_B>; ++ reset-names = "rst"; ++ }; ++ pvtm@2 { ++ reg = <2>; ++ clocks = <&cru SCLK_PVTM_DDR>; ++ clock-names = "clk"; ++ resets = <&cru SRST_PVTM_DDR>; ++ reset-names = "rst"; ++ }; ++ pvtm@3 { ++ reg = <3>; ++ clocks = <&cru SCLK_PVTM_GPU>; ++ clock-names = "clk"; ++ resets = <&cru SRST_PVTM_GPU>; ++ reset-names = "rst"; ++ }; ++ }; + }; + + tcphy0: phy@ff7c0000 { +@@ -1611,6 +1716,16 @@ i2s2: i2s@ff8a0000 { + status = "disabled"; + }; + ++ rng: rng@ff8b8000 { ++ compatible = "rockchip,cryptov1-rng"; ++ reg = <0x0 0xff8b8000 0x0 0x1000>; ++ clocks = <&cru SCLK_CRYPTO1>, <&cru HCLK_S_CRYPTO1>; ++ clock-names = "clk_crypto", "hclk_crypto"; ++ assigned-clocks = <&cru SCLK_CRYPTO1>, <&cru HCLK_S_CRYPTO1>; ++ assigned-clock-rates = <150000000>, <100000000>; ++ status = "disabled"; ++ }; ++ + vopl: vop@ff8f0000 { + compatible = "rockchip,rk3399-vop-lit"; + reg = <0x0 0xff8f0000 0x0 0x3efc>; +@@ -1656,6 +1771,17 @@ vopl_out_dp: endpoint@4 { + }; + }; + ++ vop1_pwm: voppwm@ff8f01a0 { ++ compatible = "rockchip,vop-pwm"; ++ reg = <0x0 0xff8f01a0 0x0 0x10>; ++ #pwm-cells = <3>; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&vop1_pwm_pin>; ++ clocks = <&cru SCLK_VOP1_PWM>; ++ clock-names = "pwm"; ++ status = "disabled"; ++ }; ++ + vopl_mmu: iommu@ff8f3f00 { + compatible = "rockchip,iommu"; + reg = <0x0 0xff8f3f00 0x0 0x100>; +@@ -1713,6 +1839,17 @@ vopb_out_dp: endpoint@4 { + }; + }; + ++ vop0_pwm: voppwm@ff9001a0 { ++ compatible = "rockchip,vop-pwm"; ++ reg = <0x0 0xff9001a0 0x0 0x10>; ++ #pwm-cells = <3>; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&vop0_pwm_pin>; ++ clocks = <&cru SCLK_VOP0_PWM>; ++ clock-names = "pwm"; ++ status = "disabled"; ++ }; ++ + vopb_mmu: iommu@ff903f00 { + compatible = "rockchip,iommu"; + reg = <0x0 0xff903f00 0x0 0x100>; +@@ -1915,7 +2052,95 @@ gpu: gpu@ff9a0000 { + clocks = <&cru ACLK_GPU>; + #cooling-cells = <2>; + power-domains = <&power RK3399_PD_GPU>; ++ power-off-delay-ms = <200>; ++ upthreshold = <40>; ++ downdifferential = <10>; + status = "disabled"; ++ ++ gpu_power_model: power_model { ++ compatible = "arm,mali-simple-power-model"; ++ static-coefficient = <411000>; ++ dynamic-coefficient = <733>; ++ ts = <32000 4700 (-80) 2>; ++ thermal-zone = "gpu-thermal"; ++ }; ++ }; ++ ++ nocp_cci_msch0: nocp-cci-msch0@ffa86000 { ++ compatible = "rockchip,rk3399-nocp"; ++ reg = <0x0 0xffa86000 0x0 0x400>; ++ }; ++ ++ nocp_gpu_msch0: nocp-gpu-msch0@ffa86400 { ++ compatible = "rockchip,rk3399-nocp"; ++ reg = <0x0 0xffa86400 0x0 0x400>; ++ }; ++ ++ nocp_hp_msch0: nocp-hp-msch0@ffa86800 { ++ compatible = "rockchip,rk3399-nocp"; ++ reg = <0x0 0xffa86800 0x0 0x400>; ++ }; ++ ++ nocp_lp_msch0: nocp-lp-msch0@ffa86c00 { ++ compatible = "rockchip,rk3399-nocp"; ++ reg = <0x0 0xffa86c00 0x0 0x400>; ++ }; ++ ++ nocp_video_msch0: nocp-video-msch0@ffa87000 { ++ compatible = "rockchip,rk3399-nocp"; ++ reg = <0x0 0xffa87000 0x0 0x400>; ++ }; ++ ++ nocp_vio0_msch0: nocp-vio0-msch0@ffa87400 { ++ compatible = "rockchip,rk3399-nocp"; ++ reg = <0x0 0xffa87400 0x0 0x400>; ++ }; ++ ++ nocp_vio1_msch0: nocp-vio1-msch0@ffa87800 { ++ compatible = "rockchip,rk3399-nocp"; ++ reg = <0x0 0xffa87800 0x0 0x400>; ++ }; ++ ++ nocp_cci_msch1: nocp-cci-msch1@ffa8e000 { ++ compatible = "rockchip,rk3399-nocp"; ++ reg = <0x0 0xffa8e000 0x0 0x400>; ++ }; ++ ++ nocp_gpu_msch1: nocp-gpu-msch1@ffa8e400 { ++ compatible = "rockchip,rk3399-nocp"; ++ reg = <0x0 0xffa8e400 0x0 0x400>; ++ }; ++ ++ nocp_hp_msch1: nocp-hp-msch1@ffa8e800 { ++ compatible = "rockchip,rk3399-nocp"; ++ reg = <0x0 0xffa8e800 0x0 0x400>; ++ }; ++ ++ nocp_lp_msch1: nocp-lp-msch1@ffa8ec00 { ++ compatible = "rockchip,rk3399-nocp"; ++ reg = <0x0 0xffa8ec00 0x0 0x400>; ++ }; ++ ++ nocp_video_msch1: nocp-video-msch1@ffa8f000 { ++ compatible = "rockchip,rk3399-nocp"; ++ reg = <0x0 0xffa8f000 0x0 0x400>; ++ }; ++ ++ nocp_vio0_msch1: nocp-vio0-msch1@ffa8f400 { ++ compatible = "rockchip,rk3399-nocp"; ++ reg = <0x0 0xffa8f400 0x0 0x400>; ++ }; ++ ++ nocp_vio1_msch1: nocp-vio1-msch1@ffa8f800 { ++ compatible = "rockchip,rk3399-nocp"; ++ reg = <0x0 0xffa8f800 0x0 0x400>; ++ }; ++ ++ rockchip_system_monitor: rockchip-system-monitor { ++ compatible = "rockchip,system-monitor"; ++ ++ rockchip,thermal-zone = "soc-thermal"; ++ rockchip,polling-delay = <200>; /* milliseconds */ + }; + + pinctrl: pinctrl { +@@ -2179,6 +2404,13 @@ i2c3_xfer: i2c3-xfer { + <4 RK_PC1 1 &pcfg_pull_none>, + <4 RK_PC0 1 &pcfg_pull_none>; + }; ++ ++ i2c3_gpio: i2c3_gpio { ++ rockchip,pins = ++ <4 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>, ++ <4 RK_PC0 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ + }; + + i2c4 { +@@ -2342,7 +2574,7 @@ sdmmc_wp: sdmmc-wp { + }; + }; + +- suspend { ++ sleep { + ap_pwroff: ap-pwroff { + rockchip,pins = <1 RK_PA5 1 &pcfg_pull_none>; + }; +@@ -2644,6 +2876,11 @@ pwm3a_pin: pwm3a-pin { + rockchip,pins = + <0 RK_PA6 1 &pcfg_pull_none>; + }; ++ ++ pwm3a_pin_pull_down: pwm3a-pin-pull-down { ++ rockchip,pins = ++ <0 RK_PA6 1 &pcfg_pull_down>; ++ }; + }; + + pwm3b { +@@ -2651,6 +2888,11 @@ pwm3b_pin: pwm3b-pin { + rockchip,pins = + <1 RK_PB6 1 &pcfg_pull_none>; + }; ++ ++ pwm3b_pin_pull_down: pwm3b-pin-pull-down { ++ rockchip,pins = ++ <1 RK_PB6 1 &pcfg_pull_down>; ++ }; + }; + + hdmi { +@@ -2679,4 +2921,27 @@ pcie_clkreqnb_cpm: pci-clkreqnb-cpm { + }; + + }; ++ ++ rockchip_suspend: rockchip-suspend { ++ compatible = "rockchip,pm-rk3399"; ++ status = "disabled"; ++ rockchip,sleep-debug-en = <0>; ++ rockchip,virtual-poweroff = <0>; ++ rockchip,sleep-mode-config = < ++ (0 ++ | RKPM_SLP_ARMPD ++ | RKPM_SLP_PERILPPD ++ | RKPM_SLP_DDR_RET ++ | RKPM_SLP_PLLPD ++ | RKPM_SLP_OSC_DIS ++ | RKPM_SLP_CENTER_PD ++ | RKPM_SLP_AP_PWROFF ++ ) ++ >; ++ rockchip,wakeup-config = < ++ (0 ++ | RKPM_GPIO_WKUP_EN ++ ) ++ >; ++ }; + }; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399k-opp.dtsi b/arch/arm64/boot/dts/rockchip/rk3399k-opp.dtsi +new file mode 100755 +index 000000000000..59f200e0b9cb +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399k-opp.dtsi +@@ -0,0 +1,24 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++&cluster0_opp { ++ rockchip,high-temp = <70000>; ++ rockchip,high-temp-max-volt = <1125000>; ++ opp-1512000000 { ++ opp-hz = /bits/ 64 <1512000000>; ++ opp-microvolt = <1150000 1150000 1250000>; ++ clock-latency-ns = <40000>; ++ }; ++}; ++ ++&cluster1_opp { ++ rockchip,high-temp = <70000>; ++ rockchip,high-temp-max-volt = <1200000>; ++ opp-2016000000 { ++ opp-hz = /bits/ 64 <2016000000>; ++ opp-microvolt = <1250000 1250000 1250000>; ++ clock-latency-ns = <40000>; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399pro-evb-lp4-v11-linux.dts b/arch/arm64/boot/dts/rockchip/rk3399pro-evb-lp4-v11-linux.dts +new file mode 100755 +index 000000000000..f975f47b76d6 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399pro-evb-lp4-v11-linux.dts +@@ -0,0 +1,1293 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++// Copyright (c) 2020 Fuzhou Rockchip Electronics Co., Ltd ++ ++/dts-v1/; ++#include ++#include ++#include ++#include ++#include ++#include ++#include "rk3399pro.dtsi" ++#include "rk3399-linux.dtsi" ++#include "rk3399-opp.dtsi" ++#include "rk3399-vop-clk-set.dtsi" ++ ++/ { ++ compatible = "rockchip,rk3399pro-evb-v11-linux", "rockchip,rk3399pro"; ++ ++ adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 2>; ++ io-channel-names = "buttons"; ++ poll-interval = <100>; ++ keyup-threshold-microvolt = <1800000>; ++ ++ esc-key { ++ linux,code = ; ++ label = "esc"; ++ press-threshold-microvolt = <1310000>; ++ }; ++ ++ menu-key { ++ linux,code = ; ++ label = "menu"; ++ press-threshold-microvolt = <987000>; ++ }; ++ ++ home-key { ++ linux,code = ; ++ label = "home"; ++ press-threshold-microvolt = <624000>; ++ }; ++ ++ vol-down-key { ++ linux,code = ; ++ label = "volume down"; ++ press-threshold-microvolt = <300000>; ++ }; ++ ++ vol-up-key { ++ linux,code = ; ++ label = "volume up"; ++ press-threshold-microvolt = <17000>; ++ }; ++ }; ++ ++ backlight: backlight { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm0 0 25000 0>; ++ brightness-levels = < ++ 0 20 20 21 21 22 22 23 ++ 23 24 24 25 25 26 26 27 ++ 27 28 28 29 29 30 30 31 ++ 31 32 32 33 33 34 34 35 ++ 35 36 36 37 37 38 38 39 ++ 40 41 42 43 44 45 46 47 ++ 48 49 50 51 52 53 54 55 ++ 56 57 58 59 60 61 62 63 ++ 64 65 66 67 68 69 70 71 ++ 72 73 74 75 76 77 78 79 ++ 80 81 82 83 84 85 86 87 ++ 88 89 90 91 92 93 94 95 ++ 96 97 98 99 100 101 102 103 ++ 104 105 106 107 108 109 110 111 ++ 112 113 114 115 116 117 118 119 ++ 120 121 122 123 124 125 126 127 ++ 128 129 130 131 132 133 134 135 ++ 136 137 138 139 140 141 142 143 ++ 144 145 146 147 148 149 150 151 ++ 152 153 154 155 156 157 158 159 ++ 160 161 162 163 164 165 166 167 ++ 168 169 170 171 172 173 174 175 ++ 176 177 178 179 180 181 182 183 ++ 184 185 186 187 188 189 190 191 ++ 192 193 194 195 196 197 198 199 ++ 200 201 202 203 204 205 206 207 ++ 208 209 210 211 212 213 214 215 ++ 216 217 218 219 220 221 222 223 ++ 224 225 226 227 228 229 230 231 ++ 232 233 234 235 236 237 238 239 ++ 240 241 242 243 244 245 246 247 ++ 248 249 250 251 252 253 254 255 ++ >; ++ default-brightness-level = <200>; ++ enable-gpios = <&gpio1 RK_PB5 GPIO_ACTIVE_HIGH>; ++ }; ++ ++ clkin_gmac: external-gmac-clock { ++ compatible = "fixed-clock"; ++ clock-frequency = <125000000>; ++ clock-output-names = "clkin_gmac"; ++ #clock-cells = <0>; ++ }; ++ ++ fiq_debugger: fiq-debugger { ++ compatible = "rockchip,fiq-debugger"; ++ rockchip,serial-id = <2>; ++ rockchip,wake-irq = <0>; ++ rockchip,irq-mode-enable = <0>; /* If enable uart uses irq instead of fiq */ ++ rockchip,baudrate = <1500000>; /* Only 115200 and 1500000 */ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart2c_xfer>; ++ interrupts = ; ++ }; ++ ++ hdmi_sound: hdmi-sound { ++ status = "okay"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,name = "rockchip,hdmi"; ++ ++ simple-audio-card,cpu { ++ sound-dai = <&i2s2>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&hdmi>; ++ }; ++ }; ++ ++ panel: panel { ++ compatible = "simple-panel"; ++ backlight = <&backlight>; ++ enable-gpios = <&gpio4 RK_PD6 GPIO_ACTIVE_HIGH>; ++ prepare-delay-ms = <20>; ++ enable-delay-ms = <20>; ++ reset-delay-ms = <20>; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <200000000>; ++ hactive = <1536>; ++ vactive = <2048>; ++ hfront-porch = <12>; ++ hsync-len = <16>; ++ hback-porch = <48>; ++ vfront-porch = <8>; ++ vsync-len = <4>; ++ vback-porch = <8>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ ports { ++ panel_in: endpoint { ++ remote-endpoint = <&edp_out>; ++ }; ++ }; ++ }; ++ ++ rk809_sound: rk809-sound { ++ compatible = "rockchip,multicodecs-card"; ++ rockchip,card-name = "rockchip,rk809-codec"; ++ rockchip,codec-hp-det; ++ rockchip,mclk-fs = <256>; ++ rockchip,cpu = <&i2s1>; ++ rockchip,codec = <&rk809_codec>; ++ }; ++ ++ rk_headset: rk-headset { ++ compatible = "rockchip_headset"; ++ headset_gpio = <&gpio0 RK_PB5 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hp_det>; ++ io-channels = <&saradc 3>; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ clocks = <&rk809 1>; ++ clock-names = "ext_clock"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio2 RK_PD3 GPIO_ACTIVE_LOW>; ++ }; ++ ++ usbacm_video_control: usbacm-video-control { ++ compatible = "rockchip,usbacm-video-control"; ++ status = "disabled"; ++ }; ++ ++ vcc_phy: vcc-phy-regulator { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_phy"; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ vcc5v0_sys: vccsys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc5v0_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ rockchip,grf = <&grf>; ++ wifi_chip_type = "ap6398s"; ++ sdio_vref = <1800>; ++ WIFI,host_wake_irq = <&gpio0 RK_PA3 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&rk809 1>; ++ clock-names = "ext_clock"; ++ uart_rts_gpios = <&gpio2 RK_PC3 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart0_rts>, <&bt_irq_gpio>; ++ pinctrl-1 = <&uart0_gpios>; ++ BT,reset_gpio = <&gpio2 RK_PD4 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio2 RK_PD2 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++}; ++ ++&cdn_dp { ++ status = "okay"; ++ extcon = <&fusb0>; ++ phys = <&tcphy0_dp>; ++}; ++ ++&cpu_l0 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_l1 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_l2 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_l3 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_b0 { ++ cpu-supply = <&vdd_cpu_b>; ++}; ++ ++&cpu_b1 { ++ cpu-supply = <&vdd_cpu_b>; ++}; ++ ++&display_subsystem { ++ status = "okay"; ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&dmc { ++ status = "okay"; ++ center-supply = <&vdd_center>; ++ upthreshold = <40>; ++ downdifferential = <20>; ++ system-status-freq = < ++ /*system status freq(KHz)*/ ++ SYS_STATUS_NORMAL 856000 ++ SYS_STATUS_REBOOT 856000 ++ SYS_STATUS_SUSPEND 328000 ++ SYS_STATUS_VIDEO_1080P 666000 ++ SYS_STATUS_VIDEO_4K 856000 ++ SYS_STATUS_VIDEO_4K_10B 856000 ++ SYS_STATUS_PERFORMANCE 856000 ++ SYS_STATUS_BOOST 856000 ++ SYS_STATUS_DUALVIEW 856000 ++ SYS_STATUS_ISP 856000 ++ >; ++ vop-bw-dmc-freq = < ++ /* min_bw(MB/s) max_bw(MB/s) freq(KHz) */ ++ 0 762 416000 ++ 763 3012 666000 ++ 3013 99999 856000 ++ >; ++ ++ vop-pn-msch-readlatency = < ++ 0 0x20 ++ 4 0x20 ++ >; ++ ++ auto-min-freq = <328000>; ++ auto-freq-en = <0>; ++}; ++ ++&dmc_opp_table { ++ compatible = "operating-points-v2"; ++ ++ opp-200000000 { ++ opp-hz = /bits/ 64 <200000000>; ++ opp-microvolt = <900000>; ++ status = "disabled"; ++ }; ++ opp-300000000 { ++ opp-hz = /bits/ 64 <300000000>; ++ opp-microvolt = <900000>; ++ status = "disabled"; ++ }; ++ opp-328000000 { ++ opp-hz = /bits/ 64 <328000000>; ++ opp-microvolt = <900000>; ++ }; ++ opp-400000000 { ++ opp-hz = /bits/ 64 <400000000>; ++ opp-microvolt = <900000>; ++ status = "disabled"; ++ }; ++ opp-416000000 { ++ opp-hz = /bits/ 64 <416000000>; ++ opp-microvolt = <900000>; ++ }; ++ opp-528000000 { ++ opp-hz = /bits/ 64 <528000000>; ++ opp-microvolt = <900000>; ++ status = "disabled"; ++ }; ++ opp-600000000 { ++ opp-hz = /bits/ 64 <600000000>; ++ opp-microvolt = <900000>; ++ status = "disabled"; ++ }; ++ opp-666000000 { ++ opp-hz = /bits/ 64 <666000000>; ++ opp-microvolt = <900000>; ++ }; ++ opp-800000000 { ++ opp-hz = /bits/ 64 <800000000>; ++ opp-microvolt = <900000>; ++ status = "disabled"; ++ }; ++ opp-856000000 { ++ opp-hz = /bits/ 64 <856000000>; ++ opp-microvolt = <900000>; ++ }; ++ opp-928000000 { ++ opp-hz = /bits/ 64 <928000000>; ++ opp-microvolt = <900000>; ++ status = "disabled"; ++ }; ++}; ++ ++&dp_in_vopb { ++ status = "disabled"; ++}; ++ ++&edp { ++ status = "okay"; ++ force-hpd; ++ ++ ports { ++ port@1 { ++ reg = <1>; ++ ++ edp_out: endpoint { ++ remote-endpoint = <&panel_in>; ++ }; ++ }; ++ }; ++}; ++ ++&edp_in_vopb { ++ status = "disabled"; ++}; ++ ++&emmc_phy { ++ status = "okay"; ++}; ++ ++&fiq_debugger { ++ pinctrl-0 = <&uart2a_xfer>; ++}; ++ ++&gmac { ++ phy-supply = <&vcc_phy>; ++ phy-mode = "rgmii"; ++ clock_in_out = "input"; ++ snps,reset-gpio = <&gpio3 RK_PB7 GPIO_ACTIVE_LOW>; ++ snps,reset-active-low; ++ snps,reset-delays-us = <0 10000 50000>; ++ assigned-clocks = <&cru SCLK_RMII_SRC>; ++ assigned-clock-parents = <&clkin_gmac>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&rgmii_pins>; ++ tx_delay = <0x28>; ++ rx_delay = <0x11>; ++ status = "okay"; ++}; ++ ++&gpu { ++ status = "okay"; ++ mali-supply = <&vdd_gpu>; ++}; ++ ++&hdmi { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++ rockchip,phy-table = ++ <74250000 0x8009 0x0004 0x0272>, ++ <165000000 0x802b 0x0004 0x0209>, ++ <297000000 0x8039 0x0005 0x028d>, ++ <594000000 0x8039 0x0000 0x00f6>, ++ <000000000 0x0000 0x0000 0x0000>; ++}; ++ ++&hdmi_in_vopl { ++ status = "disabled"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <180>; ++ i2c-scl-falling-time-ns = <30>; ++ clock-frequency = <400000>; ++ ++ rk809: pmic@20 { ++ compatible = "rockchip,rk809"; ++ reg = <0x20>; ++ interrupt-parent = <&gpio1>; ++ interrupts = ; ++ pinctrl-names = "default", "pmic-sleep", ++ "pmic-power-off", "pmic-reset"; ++ pinctrl-0 = <&pmic_int_l>; ++ pinctrl-1 = <&soc_slppin_slp>, <&rk809_slppin_slp>; ++ pinctrl-2 = <&soc_slppin_gpio>, <&rk809_slppin_pwrdn>; ++ pinctrl-3 = <&soc_slppin_gpio>, <&rk809_slppin_null>; ++ rockchip,system-power-controller; ++ pmic-reset-func = <0>; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ ++ vcc1-supply = <&vcc5v0_sys>; ++ vcc2-supply = <&vcc5v0_sys>; ++ vcc3-supply = <&vcc5v0_sys>; ++ vcc4-supply = <&vcc5v0_sys>; ++ vcc5-supply = <&vcc_buck5>; ++ vcc6-supply = <&vcc_buck5>; ++ vcc7-supply = <&vcc3v3_sys>; ++ vcc8-supply = <&vcc3v3_sys>; ++ vcc9-supply = <&vcc5v0_sys>; ++ ++ pwrkey { ++ status = "okay"; ++ }; ++ ++ rtc { ++ status = "okay"; ++ }; ++ ++ pinctrl_rk8xx: pinctrl_rk8xx { ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ rk809_slppin_null: rk809_slppin_null { ++ pins = "gpio_slp"; ++ function = "pin_fun0"; ++ }; ++ ++ rk809_slppin_slp: rk809_slppin_slp { ++ pins = "gpio_slp"; ++ function = "pin_fun1"; ++ }; ++ ++ rk809_slppin_pwrdn: rk809_slppin_pwrdn { ++ pins = "gpio_slp"; ++ function = "pin_fun2"; ++ }; ++ ++ rk809_slppin_rst: rk809_slppin_rst { ++ pins = "gpio_slp"; ++ function = "pin_fun3"; ++ }; ++ }; ++ ++ regulators { ++ vdd_center: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <750000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_center"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <900000>; ++ }; ++ }; ++ ++ vdd_cpu_l: DCDC_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <750000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_cpu_l"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc_ddr"; ++ regulator-initial-mode = <0x2>; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc3v3_sys: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc3v3_sys"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc_buck5: DCDC_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <2200000>; ++ regulator-max-microvolt = <2200000>; ++ regulator-name = "vcc_buck5"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <2200000>; ++ }; ++ }; ++ ++ vcca_0v9: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <900000>; ++ regulator-name = "vcca_0v9"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_1v8: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcc_1v8"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vcc0v9_soc: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <900000>; ++ ++ regulator-name = "vcc0v9_soc"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <900000>; ++ }; ++ }; ++ ++ vcca_1v8: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcca_1v8"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdd1v5_dvp: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ ++ regulator-name = "vdd1v5_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_1v5: LDO_REG6 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ ++ regulator-name = "vcc_1v5"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_3v0: LDO_REG7 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ ++ regulator-name = "vcc_3v0"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vccio_sd: LDO_REG8 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vccio_sd"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_sd: LDO_REG9 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vcc_sd"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc5v0_usb: SWITCH_REG1 { ++ regulator-name = "vcc5v0_usb"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vccio_3v3: SWITCH_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vccio_3v3"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ }; ++ ++ rk809_codec: codec { ++ #sound-dai-cells = <0>; ++ compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; ++ clocks = <&cru SCLK_I2S_8CH_OUT>; ++ clock-names = "mclk"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s_8ch_mclk>; ++ hp-volume = <20>; ++ spk-volume = <3>; ++ status = "okay"; ++ }; ++ }; ++ ++ vdd_cpu_b: tcs452x@1c { ++ compatible = "tcs,tcs452x"; ++ reg = <0x1c>; ++ vin-supply = <&vcc5v0_sys>; ++ regulator-compatible = "fan53555-reg"; ++ pinctrl-0 = <&vsel1_gpio>; ++ vsel-gpios = <&gpio1 RK_PC1 GPIO_ACTIVE_HIGH>; ++ regulator-name = "vdd_cpu_b"; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1500000>; ++ regulator-ramp-delay = <2300>; ++ fcs,suspend-voltage-selector = <1>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-state = <3>; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdd_gpu: tcs452x@10 { ++ compatible = "tcs,tcs452x"; ++ reg = <0x10>; ++ vin-supply = <&vcc5v0_sys>; ++ regulator-compatible = "fan53555-reg"; ++ pinctrl-0 = <&vsel2_gpio>; ++ vsel-gpios = <&gpio1 RK_PB6 GPIO_ACTIVE_HIGH>; ++ regulator-name = "vdd_gpu"; ++ regulator-min-microvolt = <735000>; ++ regulator-max-microvolt = <1400000>; ++ regulator-ramp-delay = <1000>; ++ fcs,suspend-voltage-selector = <1>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-state = <3>; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ bq25700: bq25700@6b { ++ compatible = "ti,bq25703"; ++ reg = <0x6b>; ++ extcon = <&fusb0>; ++ interrupt-parent = <&gpio1>; ++ interrupts = ; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&charger_ok_int>; ++ ti,charge-current = <1500000>; ++ ti,max-charge-voltage = <8704000>; ++ ti,max-input-voltage = <20000000>; ++ ti,max-input-current = <6000000>; ++ ti,input-current-sdp = <500000>; ++ ti,input-current-dcp = <2000000>; ++ ti,input-current-cdp = <2000000>; ++ ti,input-current-dc = <2000000>; ++ ti,minimum-sys-voltage = <6700000>; ++ ti,otg-voltage = <5000000>; ++ ti,otg-current = <500000>; ++ ti,input-current = <500000>; ++ pd-charge-only = <0>; ++ status = "disabled"; ++ }; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <140>; ++ i2c-scl-falling-time-ns = <30>; ++ ++ mpu6500@68 { ++ status = "okay"; ++ compatible = "invensense,mpu6500"; ++ reg = <0x68>; ++ irq-gpio = <&gpio3 RK_PD2 IRQ_TYPE_EDGE_RISING>; ++ mpu-int_config = <0x10>; ++ mpu-level_shifter = <0>; ++ mpu-orientation = <0 1 0 1 0 0 0 0 1>; ++ orientation-x= <1>; ++ orientation-y= <0>; ++ orientation-z= <0>; ++ mpu-debug = <1>; ++ }; ++ ++ sensor@d { ++ status = "okay"; ++ compatible = "ak8963"; ++ reg = <0x0d>; ++ type = ; ++ irq-gpio = <&gpio3 RK_PD7 IRQ_TYPE_EDGE_RISING>; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ layout = <3>; ++ }; ++ ++ ov13850: ov13850@10 { ++ compatible = "ovti,ov13850"; ++ status = "okay"; ++ reg = <0x10>; ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ ++ /* conflict with csi-ctl-gpios */ ++ reset-gpios = <&gpio4 26 GPIO_ACTIVE_HIGH>; ++ pwdn-gpios = <&gpio4 25 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "rockchip,camera_default"; ++ pinctrl-0 = <&cif_clkout>; ++ ++ port { ++ ucam_out0: endpoint { ++ remote-endpoint = <&mipi_in_ucam0>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++ ++ imx327: imx327@1a { ++ compatible = "sony,imx327"; ++ status = "okay"; ++ reg = <0x1a>; ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ /* conflict with csi-ctl-gpios */ ++ reset-gpios = <&gpio4 26 GPIO_ACTIVE_HIGH>; ++ pwdn-gpios = <&gpio4 25 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_clkout>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "TongJu"; ++ rockchip,camera-module-lens-name = "CHT842-MD"; ++ port { ++ ucam_out2: endpoint { ++ remote-endpoint = <&mipi_in_ucam2>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++ ++}; ++ ++&i2c4 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <345>; ++ i2c-scl-falling-time-ns = <11>; ++ ++ gsl3673: gsl3673@40 { ++ compatible = "GSL,GSL3673"; ++ reg = <0x40>; ++ screen_max_x = <1536>; ++ screen_max_y = <2048>; ++ irq_gpio_number = <&gpio4 RK_PC3 IRQ_TYPE_LEVEL_LOW>; ++ rst_gpio_number = <&gpio4 RK_PC6 GPIO_ACTIVE_HIGH>; ++ }; ++}; ++ ++&i2c8 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <345>; ++ i2c-scl-falling-time-ns = <11>; ++ clock-frequency = <100000>; ++ ++ fusb0: fusb30x@22 { ++ compatible = "fairchild,fusb302"; ++ reg = <0x22>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&fusb0_int>; ++ int-n-gpios = <&gpio1 RK_PA2 GPIO_ACTIVE_HIGH>; ++ vbus-5v-gpios = <&gpio0 RK_PA1 GPIO_ACTIVE_LOW>; ++ status = "okay"; ++ }; ++ ++}; ++ ++&i2s1 { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++}; ++ ++&i2s2 { ++ #sound-dai-cells = <0>; ++ status = "okay"; ++}; ++ ++&io_domains { ++ status = "okay"; ++ bt656-supply = <&vcca_1v8>; ++ audio-supply = <&vcca_1v8>; ++ sdmmc-supply = <&vccio_sd>; ++ gpio1830-supply = <&vcc_3v0>; ++}; ++ ++&isp0_mmu { ++ status = "okay"; ++}; ++ ++&isp1_mmu { ++ status = "okay"; ++}; ++ ++&mipi_dphy_tx1rx1 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam1: endpoint@1 { ++ reg = <1>; ++ /* Unlinked camera */ ++ //remote-endpoint = <&ucam_out1>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_tx1rx1_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp1_mipi_in>; ++ }; ++ }; ++ }; ++}; ++ ++&mipi_dphy_rx0 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam0: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ucam_out0>; ++ data-lanes = <1 2>; ++ }; ++ mipi_in_ucam2: endpoint@2 { ++ reg = <2>; ++ remote-endpoint = <&ucam_out2>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_rx0_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp0_mipi_in>; ++ }; ++ }; ++ }; ++}; ++ ++&pcie_phy { ++ status = "okay"; ++}; ++ ++&pcie0 { ++ status = "okay"; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ pmu1830-supply = <&vcc_1v8>; ++}; ++ ++&pwm0 { ++ status = "okay"; ++}; ++ ++&pwm2 { ++ status = "okay"; ++}; ++ ++&rkisp1_0 { ++ status = "okay"; ++ assigned-clocks = <&cru PLL_NPLL>, <&cru SCLK_CIF_OUT_SRC>, <&cru SCLK_CIF_OUT>; ++ assigned-clock-rates = <594000000>, <594000000>, <37125000>; ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp0_mipi_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dphy_rx0_out>; ++ }; ++ }; ++}; ++ ++&rkisp1_1 { ++ status = "okay"; ++ ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp1_mipi_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dphy_tx1rx1_out>; ++ }; ++ }; ++}; ++ ++&rockchip_suspend { ++ status = "okay"; ++ rockchip,sleep-debug-en = <1>; ++ rockchip,sleep-mode-config = < ++ (0 ++ | RKPM_SLP_ARMPD ++ | RKPM_SLP_PERILPPD ++ | RKPM_SLP_DDR_RET ++ | RKPM_SLP_PLLPD ++ | RKPM_SLP_CENTER_PD ++ | RKPM_SLP_OSC_DIS ++ | RKPM_SLP_AP_PWROFF ++ ) ++ >; ++ rockchip,wakeup-config = ; ++ rockchip,pwm-regulator-config = ; ++ rockchip,power-ctrl = ++ <&gpio1 RK_PC1 GPIO_ACTIVE_HIGH>, ++ <&gpio1 RK_PB6 GPIO_ACTIVE_HIGH>; ++}; ++ ++&route_edp { ++ status = "okay"; ++}; ++ ++&saradc { ++ status = "okay"; ++ vref-supply = <&vcc_1v8>; ++}; ++ ++&sdmmc { ++ sd-uhs-sdr12; ++ sd-uhs-sdr25; ++ sd-uhs-sdr50; ++ sd-uhs-sdr104; ++}; ++ ++&spi1 { ++ status = "okay"; ++ max-freq = <48000000>; /* spi internal clk, don't modify */ ++ spi_dev@0 { ++ compatible = "rockchip,spidev"; ++ reg = <0>; ++ spi-max-frequency = <12000000>; ++ spi-lsb-first; ++ }; ++}; ++ ++&tcphy0 { ++ extcon = <&fusb0>; ++ status = "okay"; ++}; ++ ++&tcphy1 { ++ status = "okay"; ++}; ++ ++&tsadc { ++ rockchip,hw-tshut-mode = <1>; /* tshut mode 0:CRU 1:GPIO */ ++ rockchip,hw-tshut-polarity = <1>; /* tshut polarity 0:LOW 1:HIGH */ ++ status = "okay"; ++}; ++ ++&u2phy0 { ++ status = "okay"; ++ extcon = <&fusb0>; ++ ++ u2phy0_otg: otg-port { ++ status = "okay"; ++ }; ++ ++ u2phy0_host: host-port { ++ phy-supply = <&vcc5v0_usb>; ++ status = "okay"; ++ }; ++}; ++ ++&u2phy1 { ++ status = "okay"; ++ ++ u2phy1_otg: otg-port { ++ status = "okay"; ++ }; ++ ++ u2phy1_host: host-port { ++ phy-supply = <&vcc5v0_usb>; ++ status = "okay"; ++ }; ++}; ++ ++&uart0 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart0_xfer &uart0_cts>; ++ status = "okay"; ++}; ++ ++&usb_host0_ehci { ++ status = "okay"; ++}; ++ ++&usb_host1_ehci { ++ status = "okay"; ++}; ++&usb_host0_ohci { ++ status = "okay"; ++}; ++ ++&usb_host1_ohci { ++ status = "okay"; ++}; ++ ++&usbdrd3_0 { ++ status = "okay"; ++}; ++ ++&usbdrd3_1 { ++ status = "okay"; ++}; ++ ++&usbdrd_dwc3_0 { ++ status = "okay"; ++ extcon = <&fusb0>; ++}; ++ ++&usbdrd_dwc3_1 { ++ status = "okay"; ++}; ++ ++&vopb { ++ status = "okay"; ++}; ++ ++&vopb_mmu { ++ status = "okay"; ++}; ++ ++&vopl { ++ status = "okay"; ++}; ++ ++&vopl_mmu { ++ status = "okay"; ++}; ++ ++&pinctrl { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&npu_ref_clk>; ++ ++ bq2570 { ++ charger_ok_int: charger-ok-int { ++ rockchip,pins = ++ <1 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ fusb30x { ++ fusb0_int: fusb0-int { ++ rockchip,pins = ++ <1 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ headphone { ++ hp_det: hp-det { ++ rockchip,pins = ++ <0 RK_PB5 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ lcd_rst { ++ lcd_rst_gpio: lcd-rst-gpio { ++ rockchip,pins = ++ <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ npu_clk { ++ npu_ref_clk: npu-ref-clk { ++ rockchip,pins = ++ <0 RK_PA2 1 &pcfg_pull_none>; ++ }; ++ }; ++ ++ pmic { ++ pmic_int_l: pmic-int-l { ++ rockchip,pins = ++ <1 RK_PC2 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ vsel1_gpio: vsel1-gpio { ++ rockchip,pins = ++ <1 RK_PC1 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ vsel2_gpio: vsel2-gpio { ++ rockchip,pins = ++ <1 RK_PB6 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ ++ soc_slppin_gpio: soc-slppin-gpio { ++ rockchip,pins = ++ <1 RK_PA5 RK_FUNC_GPIO &pcfg_output_low>; ++ }; ++ ++ soc_slppin_slp: soc-slppin-slp { ++ rockchip,pins = ++ <1 RK_PA5 1 &pcfg_pull_down>; ++ }; ++ ++ soc_slppin_rst: soc-slppin-rst { ++ rockchip,pins = ++ <1 RK_PA5 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = ++ <2 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdmmc { ++ sdmmc_bus1: sdmmc-bus1 { ++ rockchip,pins = ++ <4 RK_PB0 1 &pcfg_pull_up_10ma>; ++ }; ++ ++ sdmmc_bus4: sdmmc-bus4 { ++ rockchip,pins = ++ <4 RK_PB0 1 &pcfg_pull_up_10ma>, ++ <4 RK_PB1 1 &pcfg_pull_up_10ma>, ++ <4 RK_PB2 1 &pcfg_pull_up_10ma>, ++ <4 RK_PB3 1 &pcfg_pull_up_10ma>; ++ }; ++ ++ sdmmc_clk: sdmmc-clk { ++ rockchip,pins = ++ <4 RK_PB4 1 &pcfg_pull_none_10ma>; ++ }; ++ ++ sdmmc_cmd: sdmmc-cmd { ++ rockchip,pins = ++ <4 RK_PB5 1 &pcfg_pull_up_10ma>; ++ }; ++ }; ++ ++ tp_irq { ++ tp_irq_gpio: tp-irq-gpio { ++ rockchip,pins = ++ <3 RK_PB0 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ bt_irq_gpio: bt-irq-gpio { ++ rockchip,pins = ++ <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ ++ uart0_gpios: uart0-gpios { ++ rockchip,pins = ++ <2 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ ++/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ ++/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ +diff --git a/arch/arm64/boot/dts/rockchip/rk3399pro-evb-v10-linux.dts b/arch/arm64/boot/dts/rockchip/rk3399pro-evb-v10-linux.dts +new file mode 100755 +index 000000000000..08ac2a477852 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399pro-evb-v10-linux.dts +@@ -0,0 +1,1203 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++// Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ ++/dts-v1/; ++#include ++#include ++#include ++#include ++#include ++#include ++#include "rk3399pro.dtsi" ++#include "rk3399-linux.dtsi" ++#include "rk3399-opp.dtsi" ++#include "rk3399-vop-clk-set.dtsi" ++ ++/ { ++ compatible = "rockchip,rk3399pro-evb-v10-linux", "rockchip,rk3399pro"; ++ ++ adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 2>; ++ io-channel-names = "buttons"; ++ poll-interval = <100>; ++ keyup-threshold-microvolt = <1800000>; ++ ++ esc-key { ++ linux,code = ; ++ label = "esc"; ++ press-threshold-microvolt = <1310000>; ++ }; ++ ++ menu-key { ++ linux,code = ; ++ label = "menu"; ++ press-threshold-microvolt = <987000>; ++ }; ++ ++ home-key { ++ linux,code = ; ++ label = "home"; ++ press-threshold-microvolt = <624000>; ++ }; ++ ++ vol-down-key { ++ linux,code = ; ++ label = "volume down"; ++ press-threshold-microvolt = <300000>; ++ }; ++ ++ vol-up-key { ++ linux,code = ; ++ label = "volume up"; ++ press-threshold-microvolt = <17000>; ++ }; ++ }; ++ ++ backlight: backlight { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm0 0 25000 0>; ++ brightness-levels = < ++ 0 20 20 21 21 22 22 23 ++ 23 24 24 25 25 26 26 27 ++ 27 28 28 29 29 30 30 31 ++ 31 32 32 33 33 34 34 35 ++ 35 36 36 37 37 38 38 39 ++ 40 41 42 43 44 45 46 47 ++ 48 49 50 51 52 53 54 55 ++ 56 57 58 59 60 61 62 63 ++ 64 65 66 67 68 69 70 71 ++ 72 73 74 75 76 77 78 79 ++ 80 81 82 83 84 85 86 87 ++ 88 89 90 91 92 93 94 95 ++ 96 97 98 99 100 101 102 103 ++ 104 105 106 107 108 109 110 111 ++ 112 113 114 115 116 117 118 119 ++ 120 121 122 123 124 125 126 127 ++ 128 129 130 131 132 133 134 135 ++ 136 137 138 139 140 141 142 143 ++ 144 145 146 147 148 149 150 151 ++ 152 153 154 155 156 157 158 159 ++ 160 161 162 163 164 165 166 167 ++ 168 169 170 171 172 173 174 175 ++ 176 177 178 179 180 181 182 183 ++ 184 185 186 187 188 189 190 191 ++ 192 193 194 195 196 197 198 199 ++ 200 201 202 203 204 205 206 207 ++ 208 209 210 211 212 213 214 215 ++ 216 217 218 219 220 221 222 223 ++ 224 225 226 227 228 229 230 231 ++ 232 233 234 235 236 237 238 239 ++ 240 241 242 243 244 245 246 247 ++ 248 249 250 251 252 253 254 255 ++ >; ++ default-brightness-level = <200>; ++ enable-gpios = <&gpio1 RK_PB5 GPIO_ACTIVE_HIGH>; ++ }; ++ ++ clkin_gmac: external-gmac-clock { ++ compatible = "fixed-clock"; ++ clock-frequency = <125000000>; ++ clock-output-names = "clkin_gmac"; ++ #clock-cells = <0>; ++ }; ++ ++ fiq_debugger: fiq-debugger { ++ compatible = "rockchip,fiq-debugger"; ++ rockchip,serial-id = <2>; ++ rockchip,wake-irq = <0>; ++ rockchip,irq-mode-enable = <0>; /* If enable uart uses irq instead of fiq */ ++ rockchip,baudrate = <1500000>; /* Only 115200 and 1500000 */ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart2c_xfer>; ++ interrupts = ; ++ }; ++ ++ hdmi_sound: hdmi-sound { ++ status = "okay"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,name = "rockchip,hdmi"; ++ ++ simple-audio-card,cpu { ++ sound-dai = <&i2s2>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&hdmi>; ++ }; ++ }; ++ ++ panel: panel { ++ compatible = "simple-panel"; ++ backlight = <&backlight>; ++ enable-gpios = <&gpio4 RK_PD6 GPIO_ACTIVE_HIGH>; ++ prepare-delay-ms = <20>; ++ enable-delay-ms = <20>; ++ reset-delay-ms = <20>; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <200000000>; ++ hactive = <1536>; ++ vactive = <2048>; ++ hfront-porch = <12>; ++ hsync-len = <16>; ++ hback-porch = <48>; ++ vfront-porch = <8>; ++ vsync-len = <4>; ++ vback-porch = <8>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ ports { ++ panel_in: endpoint { ++ remote-endpoint = <&edp_out>; ++ }; ++ }; ++ }; ++ ++ rk809_sound: rk809-sound { ++ compatible = "rockchip,multicodecs-card"; ++ rockchip,card-name = "rockchip,rk809-codec"; ++ rockchip,codec-hp-det; ++ rockchip,mclk-fs = <256>; ++ rockchip,cpu = <&i2s1>; ++ rockchip,codec = <&rk809_codec>; ++ }; ++ ++ rk_headset: rk-headset { ++ compatible = "rockchip_headset"; ++ headset_gpio = <&gpio0 RK_PB5 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hp_det>; ++ io-channels = <&saradc 3>; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ clocks = <&rk809 1>; ++ clock-names = "ext_clock"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio2 RK_PD3 GPIO_ACTIVE_LOW>; ++ }; ++ ++ vcc_phy: vcc-phy-regulator { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_phy"; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ vcc5v0_sys: vccsys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc5v0_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ rockchip,grf = <&grf>; ++ wifi_chip_type = "ap6398s"; ++ sdio_vref = <1800>; ++ WIFI,host_wake_irq = <&gpio0 RK_PA3 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&rk809 1>; ++ clock-names = "ext_clock"; ++ uart_rts_gpios = <&gpio2 RK_PC3 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart0_rts>, <&bt_irq_gpio>; ++ pinctrl-1 = <&uart0_gpios>; ++ BT,reset_gpio = <&gpio2 RK_PD4 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio2 RK_PD2 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++}; ++ ++&cdn_dp { ++ status = "okay"; ++ extcon = <&fusb0>; ++ phys = <&tcphy0_dp>; ++}; ++ ++&cpu_l0 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_l1 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_l2 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_l3 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_b0 { ++ cpu-supply = <&vdd_cpu_b>; ++}; ++ ++&cpu_b1 { ++ cpu-supply = <&vdd_cpu_b>; ++}; ++ ++&display_subsystem { ++ status = "okay"; ++}; ++ ++&dp_in_vopb { ++ status = "disabled"; ++}; ++ ++&edp { ++ status = "okay"; ++ force-hpd; ++ ++ ports { ++ port@1 { ++ reg = <1>; ++ ++ edp_out: endpoint { ++ remote-endpoint = <&panel_in>; ++ }; ++ }; ++ }; ++}; ++ ++&edp_in_vopb { ++ status = "disabled"; ++}; ++ ++&emmc_phy { ++ status = "okay"; ++}; ++ ++&fiq_debugger { ++ pinctrl-0 = <&uart2a_xfer>; ++}; ++ ++&gmac { ++ phy-supply = <&vcc_phy>; ++ phy-mode = "rgmii"; ++ clock_in_out = "input"; ++ snps,reset-gpio = <&gpio3 RK_PB7 GPIO_ACTIVE_LOW>; ++ snps,reset-active-low; ++ snps,reset-delays-us = <0 10000 50000>; ++ assigned-clocks = <&cru SCLK_RMII_SRC>; ++ assigned-clock-parents = <&clkin_gmac>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&rgmii_pins>; ++ tx_delay = <0x28>; ++ rx_delay = <0x11>; ++ status = "okay"; ++}; ++ ++&gpu { ++ status = "okay"; ++ mali-supply = <&vdd_gpu>; ++}; ++ ++&hdmi { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++ rockchip,phy-table = ++ <74250000 0x8009 0x0004 0x0272>, ++ <165000000 0x802b 0x0004 0x0209>, ++ <297000000 0x8039 0x0005 0x028d>, ++ <594000000 0x8039 0x0000 0x00f6>, ++ <000000000 0x0000 0x0000 0x0000>; ++}; ++ ++&hdmi_in_vopl { ++ status = "disabled"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <180>; ++ i2c-scl-falling-time-ns = <30>; ++ clock-frequency = <400000>; ++ ++ rk809: pmic@20 { ++ compatible = "rockchip,rk809"; ++ reg = <0x20>; ++ interrupt-parent = <&gpio1>; ++ interrupts = ; ++ pinctrl-names = "default", "pmic-sleep", ++ "pmic-power-off", "pmic-reset"; ++ pinctrl-0 = <&pmic_int_l>; ++ pinctrl-1 = <&soc_slppin_slp>, <&rk809_slppin_slp>; ++ pinctrl-2 = <&soc_slppin_gpio>, <&rk809_slppin_pwrdn>; ++ pinctrl-3 = <&soc_slppin_gpio>, <&rk809_slppin_rst>; ++ rockchip,system-power-controller; ++ pmic-reset-func = <0>; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ ++ vcc1-supply = <&vcc5v0_sys>; ++ vcc2-supply = <&vcc5v0_sys>; ++ vcc3-supply = <&vcc5v0_sys>; ++ vcc4-supply = <&vcc5v0_sys>; ++ vcc5-supply = <&vcc_buck5>; ++ vcc6-supply = <&vcc_buck5>; ++ vcc7-supply = <&vcc3v3_sys>; ++ vcc8-supply = <&vcc3v3_sys>; ++ vcc9-supply = <&vcc5v0_sys>; ++ ++ pwrkey { ++ status = "okay"; ++ }; ++ ++ rtc { ++ status = "okay"; ++ }; ++ ++ pinctrl_rk8xx: pinctrl_rk8xx { ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ rk809_slppin_null: rk809_slppin_null { ++ pins = "gpio_slp"; ++ function = "pin_fun0"; ++ }; ++ ++ rk809_slppin_slp: rk809_slppin_slp { ++ pins = "gpio_slp"; ++ function = "pin_fun1"; ++ }; ++ ++ rk809_slppin_pwrdn: rk809_slppin_pwrdn { ++ pins = "gpio_slp"; ++ function = "pin_fun2"; ++ }; ++ ++ rk809_slppin_rst: rk809_slppin_rst { ++ pins = "gpio_slp"; ++ function = "pin_fun3"; ++ }; ++ }; ++ ++ regulators { ++ vdd_log: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <750000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_log"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <900000>; ++ }; ++ }; ++ ++ vdd_cpu_l: DCDC_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <750000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_cpu_l"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc_ddr"; ++ regulator-initial-mode = <0x2>; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc3v3_sys: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc3v3_sys"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc_buck5: DCDC_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <2200000>; ++ regulator-max-microvolt = <2200000>; ++ regulator-name = "vcc_buck5"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <2200000>; ++ }; ++ }; ++ ++ vcca_0v9: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <900000>; ++ regulator-name = "vcca_0v9"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_1v8: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcc_1v8"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vcc0v9_soc: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <900000>; ++ ++ regulator-name = "vcc0v9_soc"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <900000>; ++ }; ++ }; ++ ++ vcca_1v8: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcca_1v8"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdd1v5_dvp: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ ++ regulator-name = "vdd1v5_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_1v5: LDO_REG6 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ ++ regulator-name = "vcc_1v5"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_3v0: LDO_REG7 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ ++ regulator-name = "vcc_3v0"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vccio_sd: LDO_REG8 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vccio_sd"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_sd: LDO_REG9 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vcc_sd"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc5v0_usb: SWITCH_REG1 { ++ regulator-name = "vcc5v0_usb"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vccio_3v3: SWITCH_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vccio_3v3"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ }; ++ ++ rk809_codec: codec { ++ #sound-dai-cells = <0>; ++ compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; ++ clocks = <&cru SCLK_I2S_8CH_OUT>; ++ clock-names = "mclk"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s_8ch_mclk>; ++ hp-volume = <20>; ++ spk-volume = <3>; ++ status = "okay"; ++ }; ++ }; ++ ++ vdd_cpu_b: syr837@40 { ++ compatible = "silergy,syr827"; ++ reg = <0x40>; ++ vin-supply = <&vcc5v0_sys>; ++ regulator-compatible = "fan53555-reg"; ++ pinctrl-0 = <&vsel1_gpio>; ++ vsel-gpios = <&gpio1 RK_PC1 GPIO_ACTIVE_HIGH>; ++ regulator-name = "vdd_cpu_b"; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1500000>; ++ regulator-ramp-delay = <1000>; ++ fcs,suspend-voltage-selector = <1>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-state = <3>; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdd_gpu: syr828@41 { ++ compatible = "silergy,syr828"; ++ status = "okay"; ++ reg = <0x41>; ++ vin-supply = <&vcc5v0_sys>; ++ regulator-compatible = "fan53555-reg"; ++ pinctrl-0 = <&vsel2_gpio>; ++ vsel-gpios = <&gpio1 RK_PB6 GPIO_ACTIVE_HIGH>; ++ regulator-name = "vdd_gpu"; ++ regulator-min-microvolt = <735000>; ++ regulator-max-microvolt = <1400000>; ++ regulator-ramp-delay = <1000>; ++ fcs,suspend-voltage-selector = <1>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ bq25700: bq25700@6b { ++ compatible = "ti,bq25703"; ++ reg = <0x6b>; ++ extcon = <&fusb0>; ++ interrupt-parent = <&gpio1>; ++ interrupts = ; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&charger_ok_int>; ++ ti,charge-current = <1500000>; ++ ti,max-charge-voltage = <8704000>; ++ ti,max-input-voltage = <20000000>; ++ ti,max-input-current = <6000000>; ++ ti,input-current-sdp = <500000>; ++ ti,input-current-dcp = <2000000>; ++ ti,input-current-cdp = <2000000>; ++ ti,input-current-dc = <2000000>; ++ ti,minimum-sys-voltage = <6700000>; ++ ti,otg-voltage = <5000000>; ++ ti,otg-current = <500000>; ++ ti,input-current = <500000>; ++ pd-charge-only = <0>; ++ status = "disabled"; ++ }; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <140>; ++ i2c-scl-falling-time-ns = <30>; ++ ++ mpu6500@68 { ++ status = "okay"; ++ compatible = "invensense,mpu6500"; ++ reg = <0x68>; ++ irq-gpio = <&gpio3 RK_PD2 IRQ_TYPE_EDGE_RISING>; ++ mpu-int_config = <0x10>; ++ mpu-level_shifter = <0>; ++ mpu-orientation = <0 1 0 1 0 0 0 0 1>; ++ orientation-x= <1>; ++ orientation-y= <0>; ++ orientation-z= <0>; ++ mpu-debug = <1>; ++ }; ++ ++ sensor@d { ++ status = "okay"; ++ compatible = "ak8963"; ++ reg = <0x0d>; ++ type = ; ++ irq-gpio = <&gpio3 RK_PD7 IRQ_TYPE_EDGE_RISING>; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ layout = <3>; ++ }; ++ ++ vm149c: vm149c@0c { ++ compatible = "silicon touch,vm149c"; ++ status = "okay"; ++ reg = <0x0c>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ }; ++ ++ ov13850: ov13850@10 { ++ compatible = "ovti,ov13850"; ++ status = "disabled"; ++ reg = <0x10>; ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ ++ /* conflict with csi-ctl-gpios */ ++ reset-gpios = <&gpio4 26 GPIO_ACTIVE_HIGH>; ++ pwdn-gpios = <&gpio4 25 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "rockchip,camera_default"; ++ pinctrl-0 = <&cif_clkout>; ++ lens-focus = <&vm149c>; ++ ++ port { ++ ucam_out0: endpoint { ++ remote-endpoint = <&mipi_in_ucam0>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++ ++ imx327: imx327@1a { ++ compatible = "sony,imx327"; ++ status = "okay"; ++ reg = <0x1a>; ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ /* conflict with csi-ctl-gpios */ ++ reset-gpios = <&gpio4 26 GPIO_ACTIVE_HIGH>; ++ pwdn-gpios = <&gpio4 25 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_clkout>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "TongJu"; ++ rockchip,camera-module-lens-name = "CHT842-MD"; ++ port { ++ ucam_out2: endpoint { ++ remote-endpoint = <&mipi_in_ucam2>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++}; ++ ++&i2c4 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <345>; ++ i2c-scl-falling-time-ns = <11>; ++ ++ gsl3673: gsl3673@40 { ++ compatible = "GSL,GSL3673"; ++ reg = <0x40>; ++ screen_max_x = <1536>; ++ screen_max_y = <2048>; ++ irq_gpio_number = <&gpio4 RK_PC3 IRQ_TYPE_LEVEL_LOW>; ++ rst_gpio_number = <&gpio4 RK_PC6 GPIO_ACTIVE_HIGH>; ++ }; ++}; ++ ++&i2c8 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <345>; ++ i2c-scl-falling-time-ns = <11>; ++ clock-frequency = <100000>; ++ ++ fusb0: fusb30x@22 { ++ compatible = "fairchild,fusb302"; ++ reg = <0x22>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&fusb0_int>; ++ int-n-gpios = <&gpio1 RK_PA2 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++}; ++ ++&i2s1 { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++}; ++ ++&i2s2 { ++ #sound-dai-cells = <0>; ++ status = "okay"; ++}; ++ ++&io_domains { ++ status = "okay"; ++ bt656-supply = <&vcca_1v8>; ++ audio-supply = <&vcca_1v8>; ++ sdmmc-supply = <&vccio_sd>; ++ gpio1830-supply = <&vcc_3v0>; ++}; ++ ++&isp0_mmu { ++ status = "okay"; ++}; ++ ++&isp1_mmu { ++ status = "okay"; ++}; ++ ++&mipi_dphy_tx1rx1 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam1: endpoint@1 { ++ reg = <1>; ++ /* Unlinked camera */ ++ //remote-endpoint = <&ucam_out1>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_tx1rx1_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp1_mipi_in>; ++ }; ++ }; ++ }; ++}; ++ ++&mipi_dphy_rx0 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam0: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ucam_out0>; ++ data-lanes = <1 2>; ++ }; ++ ++ mipi_in_ucam2: endpoint@2 { ++ reg = <2>; ++ remote-endpoint = <&ucam_out2>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_rx0_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp0_mipi_in>; ++ }; ++ }; ++ }; ++}; ++ ++&pcie_phy { ++ status = "okay"; ++}; ++ ++&pcie0 { ++ status = "okay"; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ pmu1830-supply = <&vcc_1v8>; ++}; ++ ++&pwm0 { ++ status = "okay"; ++}; ++ ++&pwm2 { ++ status = "okay"; ++}; ++ ++&rkisp1_0 { ++ status = "okay"; ++ assigned-clocks = <&cru PLL_NPLL>, <&cru SCLK_CIF_OUT_SRC>, <&cru SCLK_CIF_OUT>; ++ assigned-clock-rates = <594000000>, <594000000>, <37125000>; ++ ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp0_mipi_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dphy_rx0_out>; ++ }; ++ }; ++}; ++ ++&rkisp1_1 { ++ status = "okay"; ++ ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp1_mipi_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dphy_tx1rx1_out>; ++ }; ++ }; ++}; ++ ++&rockchip_suspend { ++ status = "okay"; ++ rockchip,sleep-debug-en = <1>; ++ rockchip,sleep-mode-config = < ++ (0 ++ | RKPM_SLP_ARMPD ++ | RKPM_SLP_PERILPPD ++ | RKPM_SLP_DDR_RET ++ | RKPM_SLP_PLLPD ++ | RKPM_SLP_CENTER_PD ++ | RKPM_SLP_OSC_DIS ++ | RKPM_SLP_AP_PWROFF ++ ) ++ >; ++ rockchip,wakeup-config = ; ++ rockchip,pwm-regulator-config = ; ++ rockchip,power-ctrl = ++ <&gpio1 RK_PC1 GPIO_ACTIVE_HIGH>, ++ <&gpio1 RK_PB6 GPIO_ACTIVE_HIGH>; ++}; ++ ++&route_edp { ++ status = "okay"; ++}; ++ ++&saradc { ++ status = "okay"; ++ vref-supply = <&vcc_1v8>; ++}; ++ ++&sdmmc { ++ sd-uhs-sdr12; ++ sd-uhs-sdr25; ++ sd-uhs-sdr50; ++ sd-uhs-sdr104; ++}; ++ ++&spi1 { ++ status = "okay"; ++ max-freq = <48000000>; /* spi internal clk, don't modify */ ++ spi_dev@0 { ++ compatible = "rockchip,spidev"; ++ reg = <0>; ++ spi-max-frequency = <12000000>; ++ spi-lsb-first; ++ }; ++}; ++ ++&tcphy0 { ++ extcon = <&fusb0>; ++ status = "okay"; ++}; ++ ++&tcphy1 { ++ status = "okay"; ++}; ++ ++&tsadc { ++ rockchip,hw-tshut-mode = <1>; /* tshut mode 0:CRU 1:GPIO */ ++ rockchip,hw-tshut-polarity = <1>; /* tshut polarity 0:LOW 1:HIGH */ ++ status = "okay"; ++}; ++ ++&u2phy0 { ++ status = "okay"; ++ extcon = <&fusb0>; ++ ++ u2phy0_otg: otg-port { ++ status = "okay"; ++ }; ++ ++ u2phy0_host: host-port { ++ phy-supply = <&vcc5v0_usb>; ++ status = "okay"; ++ }; ++}; ++ ++&u2phy1 { ++ status = "okay"; ++ ++ u2phy1_otg: otg-port { ++ status = "okay"; ++ }; ++ ++ u2phy1_host: host-port { ++ phy-supply = <&vcc5v0_usb>; ++ status = "okay"; ++ }; ++}; ++ ++&uart0 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart0_xfer &uart0_cts>; ++ status = "okay"; ++}; ++ ++&usb_host0_ehci { ++ status = "okay"; ++}; ++ ++&usb_host1_ehci { ++ status = "okay"; ++}; ++&usb_host0_ohci { ++ status = "okay"; ++}; ++ ++&usb_host1_ohci { ++ status = "okay"; ++}; ++ ++&usbdrd3_0 { ++ status = "okay"; ++}; ++ ++&usbdrd3_1 { ++ status = "okay"; ++}; ++ ++&usbdrd_dwc3_0 { ++ status = "okay"; ++ extcon = <&fusb0>; ++}; ++ ++&usbdrd_dwc3_1 { ++ status = "okay"; ++}; ++ ++&vopb { ++ status = "okay"; ++}; ++ ++&vopb_mmu { ++ status = "okay"; ++}; ++ ++&vopl { ++ status = "okay"; ++}; ++ ++&vopl_mmu { ++ status = "okay"; ++}; ++ ++&pinctrl { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&npu_ref_clk>; ++ ++ bq2570 { ++ charger_ok_int: charger-ok-int { ++ rockchip,pins = ++ <1 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ fusb30x { ++ fusb0_int: fusb0-int { ++ rockchip,pins = ++ <1 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ headphone { ++ hp_det: hp-det { ++ rockchip,pins = ++ <0 RK_PB5 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ lcd_rst { ++ lcd_rst_gpio: lcd-rst-gpio { ++ rockchip,pins = ++ <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ npu_clk { ++ npu_ref_clk: npu-ref-clk { ++ rockchip,pins = ++ <0 RK_PA2 1 &pcfg_pull_none>; ++ }; ++ }; ++ ++ pmic { ++ pmic_int_l: pmic-int-l { ++ rockchip,pins = ++ <1 RK_PC2 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ vsel1_gpio: vsel1-gpio { ++ rockchip,pins = ++ <1 RK_PC1 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ vsel2_gpio: vsel2-gpio { ++ rockchip,pins = ++ <1 RK_PB6 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ ++ soc_slppin_gpio: soc-slppin-gpio { ++ rockchip,pins = ++ <1 RK_PA5 RK_FUNC_GPIO &pcfg_output_low>; ++ }; ++ ++ soc_slppin_slp: soc-slppin-slp { ++ rockchip,pins = ++ <1 RK_PA5 1 &pcfg_pull_none>; ++ }; ++ ++ soc_slppin_rst: soc-slppin-rst { ++ rockchip,pins = ++ <1 RK_PA5 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = ++ <2 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdmmc { ++ sdmmc_bus1: sdmmc-bus1 { ++ rockchip,pins = ++ <4 RK_PB0 1 &pcfg_pull_up_10ma>; ++ }; ++ ++ sdmmc_bus4: sdmmc-bus4 { ++ rockchip,pins = ++ <4 RK_PB0 1 &pcfg_pull_up_10ma>, ++ <4 RK_PB1 1 &pcfg_pull_up_10ma>, ++ <4 RK_PB2 1 &pcfg_pull_up_10ma>, ++ <4 RK_PB3 1 &pcfg_pull_up_10ma>; ++ }; ++ ++ sdmmc_clk: sdmmc-clk { ++ rockchip,pins = ++ <4 RK_PB4 1 &pcfg_pull_none_10ma>; ++ }; ++ ++ sdmmc_cmd: sdmmc-cmd { ++ rockchip,pins = ++ <4 RK_PB5 1 &pcfg_pull_up_10ma>; ++ }; ++ }; ++ ++ tp_irq { ++ tp_irq_gpio: tp-irq-gpio { ++ rockchip,pins = ++ <3 RK_PB0 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ bt_irq_gpio: bt-irq-gpio { ++ rockchip,pins = ++ <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ ++ uart0_gpios: uart0-gpios { ++ rockchip,pins = ++ <2 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ ++/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ ++/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ +diff --git a/arch/arm64/boot/dts/rockchip/rk3399pro-evb-v10.dts b/arch/arm64/boot/dts/rockchip/rk3399pro-evb-v10.dts +new file mode 100755 +index 000000000000..398f962114cf +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399pro-evb-v10.dts +@@ -0,0 +1,1061 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++// Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ ++/dts-v1/; ++#include ++#include ++#include ++#include ++#include ++#include ++#include "rk3399pro.dtsi" ++#include "rk3399-android.dtsi" ++#include "rk3399-opp.dtsi" ++#include "rk3399-vop-clk-set.dtsi" ++ ++/ { ++ compatible = "rockchip,rk3399pro-evb-v10", "rockchip,rk3399pro"; ++ ++ adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 2>; ++ io-channel-names = "buttons"; ++ poll-interval = <100>; ++ keyup-threshold-microvolt = <1800000>; ++ ++ esc-key { ++ linux,code = ; ++ label = "esc"; ++ press-threshold-microvolt = <1310000>; ++ }; ++ ++ menu-key { ++ linux,code = ; ++ label = "menu"; ++ press-threshold-microvolt = <987000>; ++ }; ++ ++ home-key { ++ linux,code = ; ++ label = "home"; ++ press-threshold-microvolt = <624000>; ++ }; ++ ++ vol-down-key { ++ linux,code = ; ++ label = "volume down"; ++ press-threshold-microvolt = <300000>; ++ }; ++ ++ vol-up-key { ++ linux,code = ; ++ label = "volume up"; ++ press-threshold-microvolt = <17000>; ++ }; ++ }; ++ ++ backlight: backlight { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm0 0 25000 0>; ++ brightness-levels = < ++ 0 20 20 21 21 22 22 23 ++ 23 24 24 25 25 26 26 27 ++ 27 28 28 29 29 30 30 31 ++ 31 32 32 33 33 34 34 35 ++ 35 36 36 37 37 38 38 39 ++ 40 41 42 43 44 45 46 47 ++ 48 49 50 51 52 53 54 55 ++ 56 57 58 59 60 61 62 63 ++ 64 65 66 67 68 69 70 71 ++ 72 73 74 75 76 77 78 79 ++ 80 81 82 83 84 85 86 87 ++ 88 89 90 91 92 93 94 95 ++ 96 97 98 99 100 101 102 103 ++ 104 105 106 107 108 109 110 111 ++ 112 113 114 115 116 117 118 119 ++ 120 121 122 123 124 125 126 127 ++ 128 129 130 131 132 133 134 135 ++ 136 137 138 139 140 141 142 143 ++ 144 145 146 147 148 149 150 151 ++ 152 153 154 155 156 157 158 159 ++ 160 161 162 163 164 165 166 167 ++ 168 169 170 171 172 173 174 175 ++ 176 177 178 179 180 181 182 183 ++ 184 185 186 187 188 189 190 191 ++ 192 193 194 195 196 197 198 199 ++ 200 201 202 203 204 205 206 207 ++ 208 209 210 211 212 213 214 215 ++ 216 217 218 219 220 221 222 223 ++ 224 225 226 227 228 229 230 231 ++ 232 233 234 235 236 237 238 239 ++ 240 241 242 243 244 245 246 247 ++ 248 249 250 251 252 253 254 255 ++ >; ++ default-brightness-level = <200>; ++ enable-gpios = <&gpio1 RK_PB5 GPIO_ACTIVE_HIGH>; ++ }; ++ ++ clkin_gmac: external-gmac-clock { ++ compatible = "fixed-clock"; ++ clock-frequency = <125000000>; ++ clock-output-names = "clkin_gmac"; ++ #clock-cells = <0>; ++ }; ++ ++ hdmi_sound: hdmi-sound { ++ status = "okay"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,name = "rockchip,hdmi"; ++ ++ simple-audio-card,cpu { ++ sound-dai = <&i2s2>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&hdmi>; ++ }; ++ }; ++ ++ panel: panel { ++ compatible = "simple-panel"; ++ backlight = <&backlight>; ++ enable-gpios = <&gpio4 RK_PD6 GPIO_ACTIVE_HIGH>; ++ prepare-delay-ms = <20>; ++ enable-delay-ms = <20>; ++ reset-delay-ms = <20>; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <200000000>; ++ hactive = <1536>; ++ vactive = <2048>; ++ hfront-porch = <12>; ++ hsync-len = <16>; ++ hback-porch = <48>; ++ vfront-porch = <8>; ++ vsync-len = <4>; ++ vback-porch = <8>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ ports { ++ panel_in: endpoint { ++ remote-endpoint = <&edp_out>; ++ }; ++ }; ++ }; ++ ++ rk809-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,rk809-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,widgets = ++ "Microphone", "Mic Jack", ++ "Headphone", "Headphone Jack"; ++ simple-audio-card,routing = ++ "Mic Jack", "MICBIAS1", ++ "IN1P", "Mic Jack", ++ "Headphone Jack", "HPOL", ++ "Headphone Jack", "HPOR"; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s1>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&rk809_codec>; ++ }; ++ }; ++ ++ rk_headset: rk-headset { ++ compatible = "rockchip_headset"; ++ headset_gpio = <&gpio0 RK_PB5 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hp_det>; ++ io-channels = <&saradc 3>; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ clocks = <&rk809 1>; ++ clock-names = "ext_clock"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio2 RK_PD3 GPIO_ACTIVE_LOW>; ++ }; ++ ++ vcc_phy: vcc-phy-regulator { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_phy"; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ vcc5v0_sys: vccsys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc5v0_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ rockchip,grf = <&grf>; ++ wifi_chip_type = "ap6398s"; ++ sdio_vref = <1800>; ++ WIFI,host_wake_irq = <&gpio0 RK_PA3 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&rk809 1>; ++ clock-names = "ext_clock"; ++ uart_rts_gpios = <&gpio2 RK_PC3 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart0_rts>, <&bt_irq_gpio>; ++ pinctrl-1 = <&uart0_gpios>; ++ BT,reset_gpio = <&gpio2 RK_PD4 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio2 RK_PD2 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++}; ++ ++&cdn_dp { ++ status = "okay"; ++ extcon = <&fusb0>; ++ phys = <&tcphy0_dp>; ++}; ++ ++&cpu_l0 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_l1 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_l2 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_l3 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_b0 { ++ cpu-supply = <&vdd_cpu_b>; ++}; ++ ++&cpu_b1 { ++ cpu-supply = <&vdd_cpu_b>; ++}; ++ ++&dp_in_vopb { ++ status = "disabled"; ++}; ++ ++&edp { ++ status = "okay"; ++ force-hpd; ++ ++ ports { ++ port@1 { ++ reg = <1>; ++ ++ edp_out: endpoint { ++ remote-endpoint = <&panel_in>; ++ }; ++ }; ++ }; ++}; ++ ++&edp_in_vopl { ++ status = "disabled"; ++}; ++ ++&emmc_phy { ++ status = "okay"; ++}; ++ ++&fiq_debugger { ++ pinctrl-0 = <&uart2a_xfer>; ++}; ++ ++&firmware_android { ++ compatible = "android,firmware"; ++ fstab { ++ compatible = "android,fstab"; ++ system { ++ compatible = "android,system"; ++ dev = "/dev/block/by-name/system"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,verify"; ++ }; ++ vendor { ++ compatible = "android,vendor"; ++ dev = "/dev/block/by-name/vendor"; ++ type = "ext4"; ++ mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; ++ fsmgr_flags = "wait,verify"; ++ }; ++ }; ++}; ++ ++&gmac { ++ phy-supply = <&vcc_phy>; ++ phy-mode = "rgmii"; ++ clock_in_out = "input"; ++ snps,reset-gpio = <&gpio3 RK_PB7 GPIO_ACTIVE_LOW>; ++ snps,reset-active-low; ++ snps,reset-delays-us = <0 10000 50000>; ++ assigned-clocks = <&cru SCLK_RMII_SRC>; ++ assigned-clock-parents = <&clkin_gmac>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&rgmii_pins>; ++ tx_delay = <0x28>; ++ rx_delay = <0x11>; ++ status = "okay"; ++}; ++ ++&gpu { ++ status = "okay"; ++ mali-supply = <&vdd_gpu>; ++}; ++ ++&hdmi { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++ rockchip,phy-table = ++ <74250000 0x8009 0x0004 0x0272>, ++ <165000000 0x802b 0x0004 0x0209>, ++ <297000000 0x8039 0x0005 0x028d>, ++ <594000000 0x8039 0x0000 0x00f6>, ++ <000000000 0x0000 0x0000 0x0000>; ++}; ++ ++&hdmi_dp_sound { ++ status = "okay"; ++}; ++ ++&hdmi_in_vopb { ++ status = "disabled"; ++}; ++ ++&i2s2 { ++ #sound-dai-cells = <0>; ++ status = "okay"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <180>; ++ i2c-scl-falling-time-ns = <30>; ++ clock-frequency = <400000>; ++ ++ rk809: pmic@20 { ++ compatible = "rockchip,rk809"; ++ reg = <0x20>; ++ interrupt-parent = <&gpio1>; ++ interrupts = ; ++ pinctrl-names = "default", "pmic-sleep", ++ "pmic-power-off", "pmic-reset"; ++ pinctrl-0 = <&pmic_int_l>; ++ pinctrl-1 = <&soc_slppin_slp>, <&rk809_slppin_slp>; ++ pinctrl-2 = <&soc_slppin_gpio>, <&rk809_slppin_pwrdn>; ++ pinctrl-3 = <&soc_slppin_gpio>,<&rk809_slppin_rst>; ++ rockchip,system-power-controller; ++ pmic-reset-func = <0>; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ ++ vcc1-supply = <&vcc5v0_sys>; ++ vcc2-supply = <&vcc5v0_sys>; ++ vcc3-supply = <&vcc5v0_sys>; ++ vcc4-supply = <&vcc5v0_sys>; ++ vcc5-supply = <&vcc_buck5>; ++ vcc6-supply = <&vcc_buck5>; ++ vcc7-supply = <&vcc3v3_sys>; ++ vcc8-supply = <&vcc3v3_sys>; ++ vcc9-supply = <&vcc5v0_sys>; ++ ++ pwrkey { ++ status = "okay"; ++ }; ++ ++ rtc { ++ status = "okay"; ++ }; ++ ++ pinctrl_rk8xx: pinctrl_rk8xx { ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ rk809_slppin_null: rk809_slppin_null { ++ pins = "gpio_slp"; ++ function = "pin_fun0"; ++ }; ++ ++ rk809_slppin_slp: rk809_slppin_slp { ++ pins = "gpio_slp"; ++ function = "pin_fun1"; ++ }; ++ ++ rk809_slppin_pwrdn: rk809_slppin_pwrdn { ++ pins = "gpio_slp"; ++ function = "pin_fun2"; ++ }; ++ ++ rk809_slppin_rst: rk809_slppin_rst { ++ pins = "gpio_slp"; ++ function = "pin_fun3"; ++ }; ++ }; ++ ++ regulators { ++ vdd_log: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <750000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_log"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <900000>; ++ }; ++ }; ++ ++ vdd_cpu_l: DCDC_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <750000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_cpu_l"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc_ddr"; ++ regulator-initial-mode = <0x2>; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc3v3_sys: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc3v3_sys"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc_buck5: DCDC_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <2200000>; ++ regulator-max-microvolt = <2200000>; ++ regulator-name = "vcc_buck5"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <2200000>; ++ }; ++ }; ++ ++ vcca_0v9: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <900000>; ++ regulator-name = "vcca_0v9"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_1v8: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcc_1v8"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vcc0v9_soc: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <900000>; ++ ++ regulator-name = "vcc0v9_soc"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <900000>; ++ }; ++ }; ++ ++ vcca_1v8: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcca_1v8"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdd1v5_dvp: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ ++ regulator-name = "vdd1v5_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_1v5: LDO_REG6 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ ++ regulator-name = "vcc_1v5"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_3v0: LDO_REG7 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ ++ regulator-name = "vcc_3v0"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vccio_sd: LDO_REG8 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vccio_sd"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_sd: LDO_REG9 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vcc_sd"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc5v0_usb: SWITCH_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc5v0_usb"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vccio_3v3: SWITCH_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vccio_3v3"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ }; ++ ++ rk809_codec: codec { ++ #sound-dai-cells = <0>; ++ compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; ++ clocks = <&cru SCLK_I2S_8CH_OUT>; ++ clock-names = "mclk"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s_8ch_mclk>; ++ hp-volume = <20>; ++ spk-volume = <3>; ++ status = "okay"; ++ }; ++ }; ++ ++ vdd_cpu_b: syr837@40 { ++ compatible = "silergy,syr827"; ++ reg = <0x40>; ++ vin-supply = <&vcc5v0_sys>; ++ regulator-compatible = "fan53555-reg"; ++ pinctrl-0 = <&vsel1_gpio>; ++ vsel-gpios = <&gpio1 RK_PC1 GPIO_ACTIVE_HIGH>; ++ regulator-name = "vdd_cpu_b"; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1500000>; ++ regulator-ramp-delay = <1000>; ++ fcs,suspend-voltage-selector = <1>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-state = <3>; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdd_gpu: syr828@41 { ++ compatible = "silergy,syr828"; ++ status = "okay"; ++ reg = <0x41>; ++ vin-supply = <&vcc5v0_sys>; ++ regulator-compatible = "fan53555-reg"; ++ pinctrl-0 = <&vsel2_gpio>; ++ vsel-gpios = <&gpio1 RK_PB6 GPIO_ACTIVE_HIGH>; ++ regulator-name = "vdd_gpu"; ++ regulator-min-microvolt = <735000>; ++ regulator-max-microvolt = <1400000>; ++ regulator-ramp-delay = <1000>; ++ fcs,suspend-voltage-selector = <1>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ bq25700: bq25700@6b { ++ compatible = "ti,bq25703"; ++ reg = <0x6b>; ++ extcon = <&fusb0>; ++ interrupt-parent = <&gpio1>; ++ interrupts = ; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&charger_ok_int>; ++ ti,charge-current = <1500000>; ++ ti,max-charge-voltage = <8704000>; ++ ti,max-input-voltage = <20000000>; ++ ti,max-input-current = <6000000>; ++ ti,input-current-sdp = <500000>; ++ ti,input-current-dcp = <2000000>; ++ ti,input-current-cdp = <2000000>; ++ ti,input-current-dc = <2000000>; ++ ti,minimum-sys-voltage = <6700000>; ++ ti,otg-voltage = <5000000>; ++ ti,otg-current = <500000>; ++ ti,input-current = <500000>; ++ pd-charge-only = <0>; ++ status = "disabled"; ++ }; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <140>; ++ i2c-scl-falling-time-ns = <30>; ++ ++ mpu6500@68 { ++ status = "okay"; ++ compatible = "invensense,mpu6500"; ++ reg = <0x68>; ++ irq-gpio = <&gpio3 RK_PD2 IRQ_TYPE_EDGE_RISING>; ++ mpu-int_config = <0x10>; ++ mpu-level_shifter = <0>; ++ mpu-orientation = <0 1 0 1 0 0 0 0 1>; ++ orientation-x= <0>; ++ orientation-y= <0>; ++ orientation-z= <1>; ++ mpu-debug = <1>; ++ }; ++ ++ sensor@d { ++ status = "okay"; ++ compatible = "ak8963"; ++ reg = <0x0d>; ++ type = ; ++ irq-gpio = <&gpio3 RK_PD7 IRQ_TYPE_EDGE_RISING>; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ layout = <3>; ++ }; ++}; ++ ++&i2c4 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <345>; ++ i2c-scl-falling-time-ns = <11>; ++ ++ gsl3673: gsl3673@40 { ++ compatible = "GSL,GSL3673"; ++ reg = <0x40>; ++ screen_max_x = <1536>; ++ screen_max_y = <2048>; ++ irq_gpio_number = <&gpio4 RK_PC3 IRQ_TYPE_LEVEL_LOW>; ++ rst_gpio_number = <&gpio4 RK_PC6 GPIO_ACTIVE_HIGH>; ++ }; ++}; ++ ++&i2c8 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <345>; ++ i2c-scl-falling-time-ns = <11>; ++ clock-frequency = <100000>; ++ ++ fusb0: fusb30x@22 { ++ compatible = "fairchild,fusb302"; ++ reg = <0x22>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&fusb0_int>; ++ int-n-gpios = <&gpio1 RK_PA2 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++}; ++ ++&i2s1 { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++}; ++ ++&io_domains { ++ status = "okay"; ++ bt656-supply = <&vcca_1v8>; ++ audio-supply = <&vcca_1v8>; ++ sdmmc-supply = <&vccio_sd>; ++ gpio1830-supply = <&vcc_3v0>; ++}; ++ ++&isp0_mmu { ++ status = "okay"; ++}; ++ ++&isp1_mmu { ++ status = "okay"; ++}; ++ ++&pcie_phy { ++ status = "disabled"; ++}; ++ ++&pcie0 { ++ status = "disabled"; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ pmu1830-supply = <&vcc_1v8>; ++}; ++ ++&pwm0 { ++ status = "okay"; ++}; ++ ++&pwm2 { ++ status = "okay"; ++}; ++ ++&rk_key { ++ status = "disabled"; ++}; ++ ++&rockchip_suspend { ++ status = "okay"; ++ rockchip,sleep-debug-en = <1>; ++ rockchip,sleep-mode-config = < ++ (0 ++ | RKPM_SLP_ARMPD ++ | RKPM_SLP_PERILPPD ++ | RKPM_SLP_DDR_RET ++ | RKPM_SLP_PLLPD ++ | RKPM_SLP_CENTER_PD ++ | RKPM_SLP_OSC_DIS ++ | RKPM_SLP_AP_PWROFF ++ ) ++ >; ++ rockchip,wakeup-config = ; ++ rockchip,pwm-regulator-config = ; ++ rockchip,power-ctrl = ++ <&gpio1 RK_PC1 GPIO_ACTIVE_HIGH>, ++ <&gpio1 RK_PB6 GPIO_ACTIVE_HIGH>; ++}; ++ ++&route_edp { ++ status = "okay"; ++}; ++ ++&saradc { ++ status = "okay"; ++ vref-supply = <&vcc_1v8>; ++}; ++ ++&sdmmc { ++ sd-uhs-sdr12; ++ sd-uhs-sdr25; ++ sd-uhs-sdr50; ++ sd-uhs-sdr104; ++}; ++ ++&spi1 { ++ status = "okay"; ++ max-freq = <48000000>; /* spi internal clk, don't modify */ ++ spi_dev@0 { ++ compatible = "rockchip,spidev"; ++ reg = <0>; ++ spi-max-frequency = <12000000>; ++ spi-lsb-first; ++ }; ++}; ++ ++&tcphy0 { ++ extcon = <&fusb0>; ++ status = "okay"; ++}; ++ ++&tcphy1 { ++ status = "okay"; ++}; ++ ++&tsadc { ++ rockchip,hw-tshut-mode = <1>; /* tshut mode 0:CRU 1:GPIO */ ++ rockchip,hw-tshut-polarity = <1>; /* tshut polarity 0:LOW 1:HIGH */ ++ status = "okay"; ++}; ++ ++&u2phy0 { ++ status = "okay"; ++ extcon = <&fusb0>; ++ ++ u2phy0_otg: otg-port { ++ status = "okay"; ++ }; ++ ++ u2phy0_host: host-port { ++ phy-supply = <&vcc5v0_usb>; ++ status = "okay"; ++ }; ++}; ++ ++&u2phy1 { ++ status = "okay"; ++ ++ u2phy1_otg: otg-port { ++ status = "okay"; ++ }; ++ ++ u2phy1_host: host-port { ++ phy-supply = <&vcc5v0_usb>; ++ status = "okay"; ++ }; ++}; ++ ++&uart0 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart0_xfer &uart0_cts>; ++ status = "okay"; ++}; ++ ++&usb_host0_ehci { ++ status = "okay"; ++}; ++ ++&usb_host1_ehci { ++ status = "okay"; ++}; ++&usb_host0_ohci { ++ status = "okay"; ++}; ++ ++&usb_host1_ohci { ++ status = "okay"; ++}; ++ ++&usbdrd3_0 { ++ status = "okay"; ++}; ++ ++&usbdrd3_1 { ++ status = "okay"; ++}; ++ ++&usbdrd_dwc3_0 { ++ status = "okay"; ++ extcon = <&fusb0>; ++}; ++ ++&usbdrd_dwc3_1 { ++ status = "okay"; ++}; ++ ++&vopb { ++ assigned-clocks = <&cru DCLK_VOP0_DIV>; ++ assigned-clock-parents = <&cru PLL_CPLL>; ++}; ++ ++&vopl { ++ assigned-clocks = <&cru DCLK_VOP1_DIV>; ++ assigned-clock-parents = <&cru PLL_VPLL>; ++}; ++ ++&pinctrl { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&npu_ref_clk>; ++ ++ bq2570 { ++ charger_ok_int: charger-ok-int { ++ rockchip,pins = ++ <1 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ fusb30x { ++ fusb0_int: fusb0-int { ++ rockchip,pins = ++ <1 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ headphone { ++ hp_det: hp-det { ++ rockchip,pins = ++ <0 RK_PB5 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ lcd_rst { ++ lcd_rst_gpio: lcd-rst-gpio { ++ rockchip,pins = ++ <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ npu_clk { ++ npu_ref_clk: npu-ref-clk { ++ rockchip,pins = ++ <0 RK_PA2 1 &pcfg_pull_none>; ++ }; ++ }; ++ ++ pmic { ++ pmic_int_l: pmic-int-l { ++ rockchip,pins = ++ <1 RK_PC2 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ vsel1_gpio: vsel1-gpio { ++ rockchip,pins = ++ <1 RK_PC1 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ vsel2_gpio: vsel2-gpio { ++ rockchip,pins = ++ <1 RK_PB6 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ ++ soc_slppin_gpio: soc-slppin-gpio { ++ rockchip,pins = ++ <1 RK_PA5 RK_FUNC_GPIO &pcfg_output_low>; ++ }; ++ ++ soc_slppin_slp: soc-slppin-slp { ++ rockchip,pins = ++ <1 RK_PA5 1 &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = ++ <2 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdmmc { ++ sdmmc_bus1: sdmmc-bus1 { ++ rockchip,pins = ++ <4 RK_PB0 1 &pcfg_pull_up_10ma>; ++ }; ++ ++ sdmmc_bus4: sdmmc-bus4 { ++ rockchip,pins = ++ <4 RK_PB0 1 &pcfg_pull_up_10ma>, ++ <4 RK_PB1 1 &pcfg_pull_up_10ma>, ++ <4 RK_PB2 1 &pcfg_pull_up_10ma>, ++ <4 RK_PB3 1 &pcfg_pull_up_10ma>; ++ }; ++ ++ sdmmc_clk: sdmmc-clk { ++ rockchip,pins = ++ <4 RK_PB4 1 &pcfg_pull_none_10ma>; ++ }; ++ ++ sdmmc_cmd: sdmmc-cmd { ++ rockchip,pins = ++ <4 RK_PB5 1 &pcfg_pull_up_10ma>; ++ }; ++ }; ++ ++ tp_irq { ++ tp_irq_gpio: tp-irq-gpio { ++ rockchip,pins = ++ <3 RK_PB0 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart0_gpios: uart0-gpios { ++ rockchip,pins = ++ <2 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ bt_irq_gpio: bt-irq-gpio { ++ rockchip,pins = ++ <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399pro-evb-v11-linux.dts b/arch/arm64/boot/dts/rockchip/rk3399pro-evb-v11-linux.dts +new file mode 100755 +index 000000000000..c3faaa1ba4d3 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399pro-evb-v11-linux.dts +@@ -0,0 +1,1223 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++// Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ ++/dts-v1/; ++#include ++#include ++#include ++#include ++#include ++#include ++#include "rk3399pro.dtsi" ++#include "rk3399-linux.dtsi" ++#include "rk3399-opp.dtsi" ++#include "rk3399-vop-clk-set.dtsi" ++ ++/ { ++ compatible = "rockchip,rk3399pro-evb-v11-linux", "rockchip,rk3399pro"; ++ ++ adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 2>; ++ io-channel-names = "buttons"; ++ poll-interval = <100>; ++ keyup-threshold-microvolt = <1800000>; ++ ++ esc-key { ++ linux,code = ; ++ label = "esc"; ++ press-threshold-microvolt = <1310000>; ++ }; ++ ++ menu-key { ++ linux,code = ; ++ label = "menu"; ++ press-threshold-microvolt = <987000>; ++ }; ++ ++ home-key { ++ linux,code = ; ++ label = "home"; ++ press-threshold-microvolt = <624000>; ++ }; ++ ++ vol-down-key { ++ linux,code = ; ++ label = "volume down"; ++ press-threshold-microvolt = <300000>; ++ }; ++ ++ vol-up-key { ++ linux,code = ; ++ label = "volume up"; ++ press-threshold-microvolt = <17000>; ++ }; ++ }; ++ ++ backlight: backlight { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm0 0 25000 0>; ++ brightness-levels = < ++ 0 20 20 21 21 22 22 23 ++ 23 24 24 25 25 26 26 27 ++ 27 28 28 29 29 30 30 31 ++ 31 32 32 33 33 34 34 35 ++ 35 36 36 37 37 38 38 39 ++ 40 41 42 43 44 45 46 47 ++ 48 49 50 51 52 53 54 55 ++ 56 57 58 59 60 61 62 63 ++ 64 65 66 67 68 69 70 71 ++ 72 73 74 75 76 77 78 79 ++ 80 81 82 83 84 85 86 87 ++ 88 89 90 91 92 93 94 95 ++ 96 97 98 99 100 101 102 103 ++ 104 105 106 107 108 109 110 111 ++ 112 113 114 115 116 117 118 119 ++ 120 121 122 123 124 125 126 127 ++ 128 129 130 131 132 133 134 135 ++ 136 137 138 139 140 141 142 143 ++ 144 145 146 147 148 149 150 151 ++ 152 153 154 155 156 157 158 159 ++ 160 161 162 163 164 165 166 167 ++ 168 169 170 171 172 173 174 175 ++ 176 177 178 179 180 181 182 183 ++ 184 185 186 187 188 189 190 191 ++ 192 193 194 195 196 197 198 199 ++ 200 201 202 203 204 205 206 207 ++ 208 209 210 211 212 213 214 215 ++ 216 217 218 219 220 221 222 223 ++ 224 225 226 227 228 229 230 231 ++ 232 233 234 235 236 237 238 239 ++ 240 241 242 243 244 245 246 247 ++ 248 249 250 251 252 253 254 255 ++ >; ++ default-brightness-level = <200>; ++ enable-gpios = <&gpio1 RK_PB5 GPIO_ACTIVE_HIGH>; ++ }; ++ ++ clkin_gmac: external-gmac-clock { ++ compatible = "fixed-clock"; ++ clock-frequency = <125000000>; ++ clock-output-names = "clkin_gmac"; ++ #clock-cells = <0>; ++ }; ++ ++ fiq_debugger: fiq-debugger { ++ compatible = "rockchip,fiq-debugger"; ++ rockchip,serial-id = <2>; ++ rockchip,wake-irq = <0>; ++ rockchip,irq-mode-enable = <0>; /* If enable uart uses irq instead of fiq */ ++ rockchip,baudrate = <1500000>; /* Only 115200 and 1500000 */ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart2c_xfer>; ++ interrupts = ; ++ }; ++ ++ hdmi_sound: hdmi-sound { ++ status = "okay"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,name = "rockchip,hdmi"; ++ ++ simple-audio-card,cpu { ++ sound-dai = <&i2s2>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&hdmi>; ++ }; ++ }; ++ ++ panel: panel { ++ compatible = "simple-panel"; ++ backlight = <&backlight>; ++ enable-gpios = <&gpio4 RK_PD6 GPIO_ACTIVE_HIGH>; ++ prepare-delay-ms = <20>; ++ enable-delay-ms = <20>; ++ reset-delay-ms = <20>; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <200000000>; ++ hactive = <1536>; ++ vactive = <2048>; ++ hfront-porch = <12>; ++ hsync-len = <16>; ++ hback-porch = <48>; ++ vfront-porch = <8>; ++ vsync-len = <4>; ++ vback-porch = <8>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ ports { ++ panel_in: endpoint { ++ remote-endpoint = <&edp_out>; ++ }; ++ }; ++ }; ++ ++ rk809_sound: rk809-sound { ++ compatible = "rockchip,multicodecs-card"; ++ rockchip,card-name = "rockchip,rk809-codec"; ++ rockchip,codec-hp-det; ++ rockchip,mclk-fs = <256>; ++ rockchip,cpu = <&i2s1>; ++ rockchip,codec = <&rk809_codec>; ++ }; ++ ++ rk_headset: rk-headset { ++ compatible = "rockchip_headset"; ++ headset_gpio = <&gpio0 RK_PB5 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hp_det>; ++ io-channels = <&saradc 3>; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ clocks = <&rk809 1>; ++ clock-names = "ext_clock"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio2 RK_PD3 GPIO_ACTIVE_LOW>; ++ }; ++ ++ usbacm_video_control: usbacm-video-control { ++ compatible = "rockchip,usbacm-video-control"; ++ status = "disabled"; ++ }; ++ ++ vcc_phy: vcc-phy-regulator { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_phy"; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ vcc5v0_sys: vccsys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc5v0_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ rockchip,grf = <&grf>; ++ wifi_chip_type = "ap6398s"; ++ sdio_vref = <1800>; ++ WIFI,host_wake_irq = <&gpio0 RK_PA3 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&rk809 1>; ++ clock-names = "ext_clock"; ++ uart_rts_gpios = <&gpio2 RK_PC3 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart0_rts>, <&bt_irq_gpio>; ++ pinctrl-1 = <&uart0_gpios>; ++ BT,reset_gpio = <&gpio2 RK_PD4 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio2 RK_PD2 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++}; ++ ++&cdn_dp { ++ status = "okay"; ++ extcon = <&fusb0>; ++ phys = <&tcphy0_dp>; ++}; ++ ++&cpu_l0 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_l1 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_l2 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_l3 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_b0 { ++ cpu-supply = <&vdd_cpu_b>; ++}; ++ ++&cpu_b1 { ++ cpu-supply = <&vdd_cpu_b>; ++}; ++ ++&display_subsystem { ++ status = "okay"; ++}; ++ ++&dmc { ++ status = "okay"; ++ center-supply = <&vdd_center>; ++}; ++ ++&dp_in_vopb { ++ status = "disabled"; ++}; ++ ++&edp { ++ status = "okay"; ++ force-hpd; ++ ++ ports { ++ port@1 { ++ reg = <1>; ++ ++ edp_out: endpoint { ++ remote-endpoint = <&panel_in>; ++ }; ++ }; ++ }; ++}; ++ ++&edp_in_vopb { ++ status = "disabled"; ++}; ++ ++&emmc_phy { ++ status = "okay"; ++}; ++ ++&fiq_debugger { ++ pinctrl-0 = <&uart2a_xfer>; ++}; ++ ++&gmac { ++ phy-supply = <&vcc_phy>; ++ phy-mode = "rgmii"; ++ clock_in_out = "input"; ++ snps,reset-gpio = <&gpio3 RK_PB7 GPIO_ACTIVE_LOW>; ++ snps,reset-active-low; ++ snps,reset-delays-us = <0 10000 50000>; ++ assigned-clocks = <&cru SCLK_RMII_SRC>; ++ assigned-clock-parents = <&clkin_gmac>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&rgmii_pins>; ++ tx_delay = <0x28>; ++ rx_delay = <0x11>; ++ status = "okay"; ++}; ++ ++&gpu { ++ status = "okay"; ++ mali-supply = <&vdd_gpu>; ++}; ++ ++&hdmi { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++ rockchip,phy-table = ++ <74250000 0x8009 0x0004 0x0272>, ++ <165000000 0x802b 0x0004 0x0209>, ++ <297000000 0x8039 0x0005 0x028d>, ++ <594000000 0x8039 0x0000 0x00f6>, ++ <000000000 0x0000 0x0000 0x0000>; ++}; ++ ++&hdmi_in_vopl { ++ status = "disabled"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <180>; ++ i2c-scl-falling-time-ns = <30>; ++ clock-frequency = <400000>; ++ ++ rk809: pmic@20 { ++ compatible = "rockchip,rk809"; ++ reg = <0x20>; ++ interrupt-parent = <&gpio1>; ++ interrupts = ; ++ pinctrl-names = "default", "pmic-sleep", ++ "pmic-power-off", "pmic-reset"; ++ pinctrl-0 = <&pmic_int_l>; ++ pinctrl-1 = <&soc_slppin_slp>, <&rk809_slppin_slp>; ++ pinctrl-2 = <&soc_slppin_gpio>, <&rk809_slppin_pwrdn>; ++ pinctrl-3 = <&soc_slppin_gpio>, <&rk809_slppin_null>; ++ rockchip,system-power-controller; ++ pmic-reset-func = <0>; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ ++ vcc1-supply = <&vcc5v0_sys>; ++ vcc2-supply = <&vcc5v0_sys>; ++ vcc3-supply = <&vcc5v0_sys>; ++ vcc4-supply = <&vcc5v0_sys>; ++ vcc5-supply = <&vcc_buck5>; ++ vcc6-supply = <&vcc_buck5>; ++ vcc7-supply = <&vcc3v3_sys>; ++ vcc8-supply = <&vcc3v3_sys>; ++ vcc9-supply = <&vcc5v0_sys>; ++ ++ pwrkey { ++ status = "okay"; ++ }; ++ ++ rtc { ++ status = "okay"; ++ }; ++ ++ pinctrl_rk8xx: pinctrl_rk8xx { ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ rk809_slppin_null: rk809_slppin_null { ++ pins = "gpio_slp"; ++ function = "pin_fun0"; ++ }; ++ ++ rk809_slppin_slp: rk809_slppin_slp { ++ pins = "gpio_slp"; ++ function = "pin_fun1"; ++ }; ++ ++ rk809_slppin_pwrdn: rk809_slppin_pwrdn { ++ pins = "gpio_slp"; ++ function = "pin_fun2"; ++ }; ++ ++ rk809_slppin_rst: rk809_slppin_rst { ++ pins = "gpio_slp"; ++ function = "pin_fun3"; ++ }; ++ }; ++ ++ regulators { ++ vdd_center: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <750000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_center"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <900000>; ++ }; ++ }; ++ ++ vdd_cpu_l: DCDC_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <750000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_cpu_l"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc_ddr"; ++ regulator-initial-mode = <0x2>; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc3v3_sys: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc3v3_sys"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc_buck5: DCDC_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <2200000>; ++ regulator-max-microvolt = <2200000>; ++ regulator-name = "vcc_buck5"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <2200000>; ++ }; ++ }; ++ ++ vcca_0v9: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <900000>; ++ regulator-name = "vcca_0v9"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_1v8: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcc_1v8"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vcc0v9_soc: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <900000>; ++ ++ regulator-name = "vcc0v9_soc"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <900000>; ++ }; ++ }; ++ ++ vcca_1v8: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcca_1v8"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdd1v5_dvp: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ ++ regulator-name = "vdd1v5_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_1v5: LDO_REG6 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ ++ regulator-name = "vcc_1v5"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_3v0: LDO_REG7 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ ++ regulator-name = "vcc_3v0"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vccio_sd: LDO_REG8 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vccio_sd"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_sd: LDO_REG9 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vcc_sd"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc5v0_usb: SWITCH_REG1 { ++ regulator-name = "vcc5v0_usb"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vccio_3v3: SWITCH_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vccio_3v3"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ }; ++ ++ rk809_codec: codec { ++ #sound-dai-cells = <0>; ++ compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; ++ clocks = <&cru SCLK_I2S_8CH_OUT>; ++ clock-names = "mclk"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s_8ch_mclk>; ++ hp-volume = <20>; ++ spk-volume = <3>; ++ status = "okay"; ++ }; ++ }; ++ ++ vdd_cpu_b: tcs452x@1c { ++ compatible = "tcs,tcs452x"; ++ reg = <0x1c>; ++ vin-supply = <&vcc5v0_sys>; ++ regulator-compatible = "fan53555-reg"; ++ pinctrl-0 = <&vsel1_gpio>; ++ vsel-gpios = <&gpio1 RK_PC1 GPIO_ACTIVE_HIGH>; ++ regulator-name = "vdd_cpu_b"; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1500000>; ++ regulator-ramp-delay = <2300>; ++ fcs,suspend-voltage-selector = <1>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-state = <3>; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdd_gpu: tcs452x@10 { ++ compatible = "tcs,tcs452x"; ++ reg = <0x10>; ++ vin-supply = <&vcc5v0_sys>; ++ regulator-compatible = "fan53555-reg"; ++ pinctrl-0 = <&vsel2_gpio>; ++ vsel-gpios = <&gpio1 RK_PB6 GPIO_ACTIVE_HIGH>; ++ regulator-name = "vdd_gpu"; ++ regulator-min-microvolt = <735000>; ++ regulator-max-microvolt = <1400000>; ++ regulator-ramp-delay = <1000>; ++ fcs,suspend-voltage-selector = <1>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-state = <3>; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ bq25700: bq25700@6b { ++ compatible = "ti,bq25703"; ++ reg = <0x6b>; ++ extcon = <&fusb0>; ++ interrupt-parent = <&gpio1>; ++ interrupts = ; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&charger_ok_int>; ++ ti,charge-current = <1500000>; ++ ti,max-charge-voltage = <8704000>; ++ ti,max-input-voltage = <20000000>; ++ ti,max-input-current = <6000000>; ++ ti,input-current-sdp = <500000>; ++ ti,input-current-dcp = <2000000>; ++ ti,input-current-cdp = <2000000>; ++ ti,input-current-dc = <2000000>; ++ ti,minimum-sys-voltage = <6700000>; ++ ti,otg-voltage = <5000000>; ++ ti,otg-current = <500000>; ++ ti,input-current = <500000>; ++ pd-charge-only = <0>; ++ status = "disabled"; ++ }; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <140>; ++ i2c-scl-falling-time-ns = <30>; ++ ++ mpu6500@68 { ++ status = "okay"; ++ compatible = "invensense,mpu6500"; ++ reg = <0x68>; ++ irq-gpio = <&gpio3 RK_PD2 IRQ_TYPE_EDGE_RISING>; ++ mpu-int_config = <0x10>; ++ mpu-level_shifter = <0>; ++ mpu-orientation = <0 1 0 1 0 0 0 0 1>; ++ orientation-x= <1>; ++ orientation-y= <0>; ++ orientation-z= <0>; ++ mpu-debug = <1>; ++ }; ++ ++ sensor@d { ++ status = "okay"; ++ compatible = "ak8963"; ++ reg = <0x0d>; ++ type = ; ++ irq-gpio = <&gpio3 RK_PD7 IRQ_TYPE_EDGE_RISING>; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ layout = <3>; ++ }; ++ ++ vm149c: vm149c@0c { ++ compatible = "silicon touch,vm149c"; ++ status = "okay"; ++ reg = <0x0c>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ }; ++ ++ ov13850: ov13850@10 { ++ compatible = "ovti,ov13850"; ++ status = "okay"; ++ reg = <0x10>; ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ ++ /* conflict with csi-ctl-gpios */ ++ reset-gpios = <&gpio4 26 GPIO_ACTIVE_HIGH>; ++ pwdn-gpios = <&gpio4 25 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "rockchip,camera_default"; ++ pinctrl-0 = <&cif_clkout>; ++ ++ lens-focus = <&vm149c>; ++ ++ port { ++ ucam_out0: endpoint { ++ remote-endpoint = <&mipi_in_ucam0>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++ ++ imx327: imx327@1a { ++ compatible = "sony,imx327"; ++ status = "okay"; ++ reg = <0x1a>; ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ /* conflict with csi-ctl-gpios */ ++ reset-gpios = <&gpio4 26 GPIO_ACTIVE_HIGH>; ++ pwdn-gpios = <&gpio4 25 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_clkout>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "TongJu"; ++ rockchip,camera-module-lens-name = "CHT842-MD"; ++ port { ++ ucam_out2: endpoint { ++ remote-endpoint = <&mipi_in_ucam2>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++ ++}; ++ ++&i2c4 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <345>; ++ i2c-scl-falling-time-ns = <11>; ++ ++ gsl3673: gsl3673@40 { ++ compatible = "GSL,GSL3673"; ++ reg = <0x40>; ++ screen_max_x = <1536>; ++ screen_max_y = <2048>; ++ irq_gpio_number = <&gpio4 RK_PC3 IRQ_TYPE_LEVEL_LOW>; ++ rst_gpio_number = <&gpio4 RK_PC6 GPIO_ACTIVE_HIGH>; ++ }; ++}; ++ ++&i2c8 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <345>; ++ i2c-scl-falling-time-ns = <11>; ++ clock-frequency = <100000>; ++ ++ fusb0: fusb30x@22 { ++ compatible = "fairchild,fusb302"; ++ reg = <0x22>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&fusb0_int>; ++ int-n-gpios = <&gpio1 RK_PA2 GPIO_ACTIVE_HIGH>; ++ vbus-5v-gpios = <&gpio0 RK_PA1 GPIO_ACTIVE_LOW>; ++ status = "okay"; ++ }; ++ ++}; ++ ++&i2s1 { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++}; ++ ++&i2s2 { ++ #sound-dai-cells = <0>; ++ status = "okay"; ++}; ++ ++&io_domains { ++ status = "okay"; ++ bt656-supply = <&vcca_1v8>; ++ audio-supply = <&vcca_1v8>; ++ sdmmc-supply = <&vccio_sd>; ++ gpio1830-supply = <&vcc_3v0>; ++}; ++ ++&isp0_mmu { ++ status = "okay"; ++}; ++ ++&isp1_mmu { ++ status = "okay"; ++}; ++ ++&mipi_dphy_tx1rx1 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam1: endpoint@1 { ++ reg = <1>; ++ /* Unlinked camera */ ++ //remote-endpoint = <&ucam_out1>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_tx1rx1_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp1_mipi_in>; ++ }; ++ }; ++ }; ++}; ++ ++&mipi_dphy_rx0 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam0: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ucam_out0>; ++ data-lanes = <1 2>; ++ }; ++ ++ mipi_in_ucam2: endpoint@2 { ++ reg = <2>; ++ remote-endpoint = <&ucam_out2>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_rx0_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp0_mipi_in>; ++ }; ++ }; ++ }; ++}; ++ ++&pcie_phy { ++ status = "okay"; ++}; ++ ++&pcie0 { ++ status = "okay"; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ pmu1830-supply = <&vcc_1v8>; ++}; ++ ++&pwm0 { ++ status = "okay"; ++}; ++ ++&pwm2 { ++ status = "okay"; ++}; ++ ++&rkisp1_0 { ++ status = "okay"; ++ assigned-clocks = <&cru PLL_NPLL>, <&cru SCLK_CIF_OUT_SRC>, <&cru SCLK_CIF_OUT>; ++ assigned-clock-rates = <594000000>, <594000000>, <37125000>; ++ ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp0_mipi_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dphy_rx0_out>; ++ }; ++ }; ++}; ++ ++&rkisp1_1 { ++ status = "okay"; ++ ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp1_mipi_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dphy_tx1rx1_out>; ++ }; ++ }; ++}; ++ ++&rockchip_suspend { ++ status = "okay"; ++ rockchip,sleep-debug-en = <1>; ++ rockchip,sleep-mode-config = < ++ (0 ++ | RKPM_SLP_ARMPD ++ | RKPM_SLP_PERILPPD ++ | RKPM_SLP_DDR_RET ++ | RKPM_SLP_PLLPD ++ | RKPM_SLP_CENTER_PD ++ | RKPM_SLP_OSC_DIS ++ | RKPM_SLP_AP_PWROFF ++ ) ++ >; ++ rockchip,wakeup-config = ; ++ rockchip,pwm-regulator-config = ; ++ rockchip,power-ctrl = ++ <&gpio1 RK_PC1 GPIO_ACTIVE_HIGH>, ++ <&gpio1 RK_PB6 GPIO_ACTIVE_HIGH>; ++}; ++ ++&route_edp { ++ status = "okay"; ++}; ++ ++&saradc { ++ status = "okay"; ++ vref-supply = <&vcc_1v8>; ++}; ++ ++&sdmmc { ++ sd-uhs-sdr12; ++ sd-uhs-sdr25; ++ sd-uhs-sdr50; ++ sd-uhs-sdr104; ++}; ++ ++&spi1 { ++ status = "okay"; ++ max-freq = <48000000>; /* spi internal clk, don't modify */ ++ spi_dev@0 { ++ compatible = "rockchip,spidev"; ++ reg = <0>; ++ spi-max-frequency = <12000000>; ++ spi-lsb-first; ++ }; ++}; ++ ++&tcphy0 { ++ extcon = <&fusb0>; ++ status = "okay"; ++}; ++ ++&tcphy1 { ++ status = "okay"; ++}; ++ ++&tsadc { ++ rockchip,hw-tshut-mode = <1>; /* tshut mode 0:CRU 1:GPIO */ ++ rockchip,hw-tshut-polarity = <1>; /* tshut polarity 0:LOW 1:HIGH */ ++ status = "okay"; ++}; ++ ++&u2phy0 { ++ status = "okay"; ++ extcon = <&fusb0>; ++ ++ u2phy0_otg: otg-port { ++ status = "okay"; ++ }; ++ ++ u2phy0_host: host-port { ++ phy-supply = <&vcc5v0_usb>; ++ status = "okay"; ++ }; ++}; ++ ++&u2phy1 { ++ status = "okay"; ++ ++ u2phy1_otg: otg-port { ++ status = "okay"; ++ }; ++ ++ u2phy1_host: host-port { ++ phy-supply = <&vcc5v0_usb>; ++ status = "okay"; ++ }; ++}; ++ ++&uart0 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart0_xfer &uart0_cts>; ++ status = "okay"; ++}; ++ ++&usb_host0_ehci { ++ status = "okay"; ++}; ++ ++&usb_host1_ehci { ++ status = "okay"; ++}; ++&usb_host0_ohci { ++ status = "okay"; ++}; ++ ++&usb_host1_ohci { ++ status = "okay"; ++}; ++ ++&usbdrd3_0 { ++ status = "okay"; ++}; ++ ++&usbdrd3_1 { ++ status = "okay"; ++}; ++ ++&usbdrd_dwc3_0 { ++ status = "okay"; ++ extcon = <&fusb0>; ++}; ++ ++&usbdrd_dwc3_1 { ++ status = "okay"; ++}; ++ ++&vopb { ++ status = "okay"; ++}; ++ ++&vopb_mmu { ++ status = "okay"; ++}; ++ ++&vopl { ++ status = "okay"; ++}; ++ ++&vopl_mmu { ++ status = "okay"; ++}; ++ ++&pinctrl { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&npu_ref_clk>; ++ ++ bq2570 { ++ charger_ok_int: charger-ok-int { ++ rockchip,pins = ++ <1 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ camera { ++ cam_pwren_high: cam-pwren-high { ++ rockchip,pins = ++ <4 RK_PC5 RK_FUNC_GPIO &pcfg_output_high>; ++ }; ++ }; ++ ++ fusb30x { ++ fusb0_int: fusb0-int { ++ rockchip,pins = ++ <1 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ headphone { ++ hp_det: hp-det { ++ rockchip,pins = ++ <0 RK_PB5 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ lcd_rst { ++ lcd_rst_gpio: lcd-rst-gpio { ++ rockchip,pins = ++ <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ npu_clk { ++ npu_ref_clk: npu-ref-clk { ++ rockchip,pins = ++ <0 RK_PA2 1 &pcfg_pull_none>; ++ }; ++ }; ++ ++ pmic { ++ pmic_int_l: pmic-int-l { ++ rockchip,pins = ++ <1 RK_PC2 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ vsel1_gpio: vsel1-gpio { ++ rockchip,pins = ++ <1 RK_PC1 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ vsel2_gpio: vsel2-gpio { ++ rockchip,pins = ++ <1 RK_PB6 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ ++ soc_slppin_gpio: soc-slppin-gpio { ++ rockchip,pins = ++ <1 RK_PA5 RK_FUNC_GPIO &pcfg_output_low>; ++ }; ++ ++ soc_slppin_slp: soc-slppin-slp { ++ rockchip,pins = ++ <1 RK_PA5 1 &pcfg_pull_down>; ++ }; ++ ++ soc_slppin_rst: soc-slppin-rst { ++ rockchip,pins = ++ <1 RK_PA5 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = ++ <2 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdmmc { ++ sdmmc_bus1: sdmmc-bus1 { ++ rockchip,pins = ++ <4 RK_PB0 1 &pcfg_pull_up_10ma>; ++ }; ++ ++ sdmmc_bus4: sdmmc-bus4 { ++ rockchip,pins = ++ <4 RK_PB0 1 &pcfg_pull_up_10ma>, ++ <4 RK_PB1 1 &pcfg_pull_up_10ma>, ++ <4 RK_PB2 1 &pcfg_pull_up_10ma>, ++ <4 RK_PB3 1 &pcfg_pull_up_10ma>; ++ }; ++ ++ sdmmc_clk: sdmmc-clk { ++ rockchip,pins = ++ <4 RK_PB4 1 &pcfg_pull_none_10ma>; ++ }; ++ ++ sdmmc_cmd: sdmmc-cmd { ++ rockchip,pins = ++ <4 RK_PB5 1 &pcfg_pull_up_10ma>; ++ }; ++ }; ++ ++ tp_irq { ++ tp_irq_gpio: tp-irq-gpio { ++ rockchip,pins = ++ <3 RK_PB0 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ bt_irq_gpio: bt-irq-gpio { ++ rockchip,pins = ++ <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ ++ uart0_gpios: uart0-gpios { ++ rockchip,pins = ++ <2 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ ++/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ ++/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ +diff --git a/arch/arm64/boot/dts/rockchip/rk3399pro-evb-v11.dts b/arch/arm64/boot/dts/rockchip/rk3399pro-evb-v11.dts +new file mode 100755 +index 000000000000..7d118a783353 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399pro-evb-v11.dts +@@ -0,0 +1,1045 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++// Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ ++/dts-v1/; ++#include ++#include ++#include ++#include ++#include ++#include ++#include "rk3399pro.dtsi" ++#include "rk3399-android.dtsi" ++#include "rk3399-opp.dtsi" ++#include "rk3399-vop-clk-set.dtsi" ++ ++/ { ++ model = "Rockchip RK3399pro evb v11 board"; ++ compatible = "rockchip,rk3399pro-evb-v11", "rockchip,rk3399pro"; ++ ++ adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 2>; ++ io-channel-names = "buttons"; ++ poll-interval = <100>; ++ keyup-threshold-microvolt = <1800000>; ++ ++ esc-key { ++ linux,code = ; ++ label = "esc"; ++ press-threshold-microvolt = <1310000>; ++ }; ++ ++ menu-key { ++ linux,code = ; ++ label = "menu"; ++ press-threshold-microvolt = <987000>; ++ }; ++ ++ home-key { ++ linux,code = ; ++ label = "home"; ++ press-threshold-microvolt = <624000>; ++ }; ++ ++ vol-down-key { ++ linux,code = ; ++ label = "volume down"; ++ press-threshold-microvolt = <300000>; ++ }; ++ ++ vol-up-key { ++ linux,code = ; ++ label = "volume up"; ++ press-threshold-microvolt = <17000>; ++ }; ++ }; ++ ++ backlight: backlight { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm0 0 25000 0>; ++ brightness-levels = < ++ 0 20 20 21 21 22 22 23 ++ 23 24 24 25 25 26 26 27 ++ 27 28 28 29 29 30 30 31 ++ 31 32 32 33 33 34 34 35 ++ 35 36 36 37 37 38 38 39 ++ 40 41 42 43 44 45 46 47 ++ 48 49 50 51 52 53 54 55 ++ 56 57 58 59 60 61 62 63 ++ 64 65 66 67 68 69 70 71 ++ 72 73 74 75 76 77 78 79 ++ 80 81 82 83 84 85 86 87 ++ 88 89 90 91 92 93 94 95 ++ 96 97 98 99 100 101 102 103 ++ 104 105 106 107 108 109 110 111 ++ 112 113 114 115 116 117 118 119 ++ 120 121 122 123 124 125 126 127 ++ 128 129 130 131 132 133 134 135 ++ 136 137 138 139 140 141 142 143 ++ 144 145 146 147 148 149 150 151 ++ 152 153 154 155 156 157 158 159 ++ 160 161 162 163 164 165 166 167 ++ 168 169 170 171 172 173 174 175 ++ 176 177 178 179 180 181 182 183 ++ 184 185 186 187 188 189 190 191 ++ 192 193 194 195 196 197 198 199 ++ 200 201 202 203 204 205 206 207 ++ 208 209 210 211 212 213 214 215 ++ 216 217 218 219 220 221 222 223 ++ 224 225 226 227 228 229 230 231 ++ 232 233 234 235 236 237 238 239 ++ 240 241 242 243 244 245 246 247 ++ 248 249 250 251 252 253 254 255 ++ >; ++ default-brightness-level = <200>; ++ enable-gpios = <&gpio1 RK_PB5 GPIO_ACTIVE_HIGH>; ++ }; ++ ++ clkin_gmac: external-gmac-clock { ++ compatible = "fixed-clock"; ++ clock-frequency = <125000000>; ++ clock-output-names = "clkin_gmac"; ++ #clock-cells = <0>; ++ }; ++ ++ hdmi_sound: hdmi-sound { ++ status = "okay"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,name = "rockchip,hdmi"; ++ ++ simple-audio-card,cpu { ++ sound-dai = <&i2s2>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&hdmi>; ++ }; ++ }; ++ ++ panel: panel { ++ compatible = "simple-panel"; ++ backlight = <&backlight>; ++ enable-gpios = <&gpio4 RK_PD6 GPIO_ACTIVE_HIGH>; ++ prepare-delay-ms = <20>; ++ enable-delay-ms = <20>; ++ reset-delay-ms = <20>; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <200000000>; ++ hactive = <1536>; ++ vactive = <2048>; ++ hfront-porch = <12>; ++ hsync-len = <16>; ++ hback-porch = <48>; ++ vfront-porch = <8>; ++ vsync-len = <4>; ++ vback-porch = <8>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ ports { ++ panel_in: endpoint { ++ remote-endpoint = <&edp_out>; ++ }; ++ }; ++ }; ++ ++ rk809-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,rk809-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,widgets = ++ "Microphone", "Mic Jack", ++ "Headphone", "Headphone Jack"; ++ simple-audio-card,routing = ++ "Mic Jack", "MICBIAS1", ++ "IN1P", "Mic Jack", ++ "Headphone Jack", "HPOL", ++ "Headphone Jack", "HPOR"; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s1>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&rk809_codec>; ++ }; ++ }; ++ ++ rk_headset: rk-headset { ++ compatible = "rockchip_headset"; ++ headset_gpio = <&gpio0 RK_PB5 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hp_det>; ++ io-channels = <&saradc 3>; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ clocks = <&rk809 1>; ++ clock-names = "ext_clock"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio2 RK_PD3 GPIO_ACTIVE_LOW>; ++ }; ++ ++ vcc_phy: vcc-phy-regulator { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_phy"; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ vcc5v0_sys: vccsys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc5v0_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ rockchip,grf = <&grf>; ++ wifi_chip_type = "ap6398s"; ++ sdio_vref = <1800>; ++ WIFI,host_wake_irq = <&gpio0 RK_PA3 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&rk809 1>; ++ clock-names = "ext_clock"; ++ uart_rts_gpios = <&gpio2 RK_PC3 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart0_rts>, <&bt_irq_gpio>; ++ pinctrl-1 = <&uart0_gpios>; ++ BT,reset_gpio = <&gpio2 RK_PD4 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio2 RK_PD2 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++}; ++ ++&cdn_dp { ++ status = "okay"; ++ extcon = <&fusb0>; ++ phys = <&tcphy0_dp>; ++}; ++ ++&cpu_l0 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_l1 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_l2 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_l3 { ++ cpu-supply = <&vdd_cpu_l>; ++}; ++ ++&cpu_b0 { ++ cpu-supply = <&vdd_cpu_b>; ++}; ++ ++&cpu_b1 { ++ cpu-supply = <&vdd_cpu_b>; ++}; ++ ++&dmc { ++ status = "okay"; ++ center-supply = <&vdd_center>; ++}; ++ ++&dp_in_vopb { ++ status = "disabled"; ++}; ++ ++&edp { ++ status = "okay"; ++ force-hpd; ++ ++ ports { ++ port@1 { ++ reg = <1>; ++ ++ edp_out: endpoint { ++ remote-endpoint = <&panel_in>; ++ }; ++ }; ++ }; ++}; ++ ++&edp_in_vopl { ++ status = "disabled"; ++}; ++ ++&emmc_phy { ++ status = "okay"; ++}; ++ ++&fiq_debugger { ++ pinctrl-0 = <&uart2a_xfer>; ++}; ++ ++&gmac { ++ phy-supply = <&vcc_phy>; ++ phy-mode = "rgmii"; ++ clock_in_out = "input"; ++ snps,reset-gpio = <&gpio3 RK_PB7 GPIO_ACTIVE_LOW>; ++ snps,reset-active-low; ++ snps,reset-delays-us = <0 10000 50000>; ++ assigned-clocks = <&cru SCLK_RMII_SRC>; ++ assigned-clock-parents = <&clkin_gmac>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&rgmii_pins>; ++ tx_delay = <0x28>; ++ rx_delay = <0x11>; ++ status = "okay"; ++}; ++ ++&gpu { ++ status = "okay"; ++ mali-supply = <&vdd_gpu>; ++}; ++ ++&hdmi { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++ rockchip,phy-table = ++ <74250000 0x8009 0x0004 0x0272>, ++ <165000000 0x802b 0x0004 0x0209>, ++ <297000000 0x8039 0x0005 0x028d>, ++ <594000000 0x8039 0x0000 0x00f6>, ++ <000000000 0x0000 0x0000 0x0000>; ++}; ++ ++&hdmi_dp_sound { ++ status = "okay"; ++}; ++ ++&hdmi_in_vopb { ++ status = "disabled"; ++}; ++ ++&i2s2 { ++ #sound-dai-cells = <0>; ++ status = "okay"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <180>; ++ i2c-scl-falling-time-ns = <30>; ++ clock-frequency = <400000>; ++ ++ rk809: pmic@20 { ++ compatible = "rockchip,rk809"; ++ reg = <0x20>; ++ interrupt-parent = <&gpio1>; ++ interrupts = ; ++ pinctrl-names = "default", "pmic-sleep", ++ "pmic-power-off", "pmic-reset"; ++ pinctrl-0 = <&pmic_int_l>; ++ pinctrl-1 = <&soc_slppin_slp>, <&rk809_slppin_slp>; ++ pinctrl-2 = <&soc_slppin_gpio>, <&rk809_slppin_pwrdn>; ++ pinctrl-3 = <&soc_slppin_gpio>,<&rk809_slppin_null>; ++ rockchip,system-power-controller; ++ pmic-reset-func = <0>; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ ++ vcc1-supply = <&vcc5v0_sys>; ++ vcc2-supply = <&vcc5v0_sys>; ++ vcc3-supply = <&vcc5v0_sys>; ++ vcc4-supply = <&vcc5v0_sys>; ++ vcc5-supply = <&vcc_buck5>; ++ vcc6-supply = <&vcc_buck5>; ++ vcc7-supply = <&vcc3v3_sys>; ++ vcc8-supply = <&vcc3v3_sys>; ++ vcc9-supply = <&vcc5v0_sys>; ++ ++ pwrkey { ++ status = "okay"; ++ }; ++ ++ rtc { ++ status = "okay"; ++ }; ++ ++ pinctrl_rk8xx: pinctrl_rk8xx { ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ rk809_slppin_null: rk809_slppin_null { ++ pins = "gpio_slp"; ++ function = "pin_fun0"; ++ }; ++ ++ rk809_slppin_slp: rk809_slppin_slp { ++ pins = "gpio_slp"; ++ function = "pin_fun1"; ++ }; ++ ++ rk809_slppin_pwrdn: rk809_slppin_pwrdn { ++ pins = "gpio_slp"; ++ function = "pin_fun2"; ++ }; ++ ++ rk809_slppin_rst: rk809_slppin_rst { ++ pins = "gpio_slp"; ++ function = "pin_fun3"; ++ }; ++ }; ++ ++ regulators { ++ vdd_center: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <750000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_center"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <900000>; ++ }; ++ }; ++ ++ vdd_cpu_l: DCDC_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <750000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_cpu_l"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc_ddr"; ++ regulator-initial-mode = <0x2>; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc3v3_sys: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc3v3_sys"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc_buck5: DCDC_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <2200000>; ++ regulator-max-microvolt = <2200000>; ++ regulator-name = "vcc_buck5"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <2200000>; ++ }; ++ }; ++ ++ vcca_0v9: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <900000>; ++ regulator-name = "vcca_0v9"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_1v8: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcc_1v8"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vcc0v9_soc: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <900000>; ++ ++ regulator-name = "vcc0v9_soc"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <900000>; ++ }; ++ }; ++ ++ vcca_1v8: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ ++ regulator-name = "vcca_1v8"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdd1v5_dvp: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ ++ regulator-name = "vdd1v5_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_1v5: LDO_REG6 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1500000>; ++ regulator-max-microvolt = <1500000>; ++ ++ regulator-name = "vcc_1v5"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_3v0: LDO_REG7 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3000000>; ++ regulator-max-microvolt = <3000000>; ++ ++ regulator-name = "vcc_3v0"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vccio_sd: LDO_REG8 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vccio_sd"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_sd: LDO_REG9 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ ++ regulator-name = "vcc_sd"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc5v0_usb: SWITCH_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc5v0_usb"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vccio_3v3: SWITCH_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vccio_3v3"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ }; ++ ++ rk809_codec: codec { ++ #sound-dai-cells = <0>; ++ compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; ++ clocks = <&cru SCLK_I2S_8CH_OUT>; ++ clock-names = "mclk"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s_8ch_mclk>; ++ hp-volume = <20>; ++ spk-volume = <3>; ++ status = "okay"; ++ }; ++ }; ++ ++ vdd_cpu_b: tcs452x@1c { ++ compatible = "tcs,tcs452x"; ++ reg = <0x1c>; ++ vin-supply = <&vcc5v0_sys>; ++ regulator-compatible = "fan53555-reg"; ++ pinctrl-0 = <&vsel1_gpio>; ++ vsel-gpios = <&gpio1 RK_PC1 GPIO_ACTIVE_HIGH>; ++ regulator-name = "vdd_cpu_b"; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1500000>; ++ regulator-ramp-delay = <2300>; ++ fcs,suspend-voltage-selector = <1>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-state = <3>; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdd_gpu: tcs452x@10 { ++ compatible = "tcs,tcs452x"; ++ reg = <0x10>; ++ vin-supply = <&vcc5v0_sys>; ++ regulator-compatible = "fan53555-reg"; ++ pinctrl-0 = <&vsel2_gpio>; ++ vsel-gpios = <&gpio1 RK_PB6 GPIO_ACTIVE_HIGH>; ++ regulator-name = "vdd_gpu"; ++ regulator-min-microvolt = <735000>; ++ regulator-max-microvolt = <1400000>; ++ regulator-ramp-delay = <2300>; ++ fcs,suspend-voltage-selector = <1>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-state = <3>; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ bq25700: bq25700@6b { ++ compatible = "ti,bq25703"; ++ reg = <0x6b>; ++ extcon = <&fusb0>; ++ interrupt-parent = <&gpio1>; ++ interrupts = ; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&charger_ok_int>; ++ ti,charge-current = <1500000>; ++ ti,max-charge-voltage = <8704000>; ++ ti,max-input-voltage = <20000000>; ++ ti,max-input-current = <6000000>; ++ ti,input-current-sdp = <500000>; ++ ti,input-current-dcp = <2000000>; ++ ti,input-current-cdp = <2000000>; ++ ti,input-current-dc = <2000000>; ++ ti,minimum-sys-voltage = <6700000>; ++ ti,otg-voltage = <5000000>; ++ ti,otg-current = <500000>; ++ ti,input-current = <500000>; ++ pd-charge-only = <0>; ++ status = "disabled"; ++ }; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <140>; ++ i2c-scl-falling-time-ns = <30>; ++ ++ mpu6500@68 { ++ status = "okay"; ++ compatible = "invensense,mpu6500"; ++ reg = <0x68>; ++ irq-gpio = <&gpio3 RK_PD2 IRQ_TYPE_EDGE_RISING>; ++ mpu-int_config = <0x10>; ++ mpu-level_shifter = <0>; ++ mpu-orientation = <0 1 0 1 0 0 0 0 1>; ++ orientation-x= <0>; ++ orientation-y= <0>; ++ orientation-z= <1>; ++ mpu-debug = <1>; ++ }; ++ ++ sensor@d { ++ status = "okay"; ++ compatible = "ak8963"; ++ reg = <0x0d>; ++ type = ; ++ irq-gpio = <&gpio3 RK_PD7 IRQ_TYPE_EDGE_RISING>; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ layout = <3>; ++ }; ++}; ++ ++&i2c4 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <345>; ++ i2c-scl-falling-time-ns = <11>; ++ ++ gsl3673: gsl3673@40 { ++ compatible = "GSL,GSL3673"; ++ reg = <0x40>; ++ screen_max_x = <1536>; ++ screen_max_y = <2048>; ++ irq_gpio_number = <&gpio4 RK_PC3 IRQ_TYPE_LEVEL_LOW>; ++ rst_gpio_number = <&gpio4 RK_PC6 GPIO_ACTIVE_HIGH>; ++ }; ++}; ++ ++&i2c8 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <345>; ++ i2c-scl-falling-time-ns = <11>; ++ clock-frequency = <100000>; ++ ++ fusb0: fusb30x@22 { ++ compatible = "fairchild,fusb302"; ++ reg = <0x22>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&fusb0_int>; ++ int-n-gpios = <&gpio1 RK_PA2 GPIO_ACTIVE_HIGH>; ++ vbus-5v-gpios = <&gpio0 RK_PA1 GPIO_ACTIVE_LOW>; ++ status = "okay"; ++ }; ++ ++}; ++ ++&i2s1 { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++}; ++ ++&io_domains { ++ status = "okay"; ++ bt656-supply = <&vcca_1v8>; ++ audio-supply = <&vcca_1v8>; ++ sdmmc-supply = <&vccio_sd>; ++ gpio1830-supply = <&vcc_3v0>; ++}; ++ ++&isp0_mmu { ++ status = "okay"; ++}; ++ ++&isp1_mmu { ++ status = "okay"; ++}; ++ ++&pcie_phy { ++ status = "disabled"; ++}; ++ ++&pcie0 { ++ status = "disabled"; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ pmu1830-supply = <&vcc_1v8>; ++}; ++ ++&pwm0 { ++ status = "okay"; ++}; ++ ++&pwm2 { ++ status = "okay"; ++}; ++ ++&rockchip_suspend { ++ status = "okay"; ++ rockchip,sleep-debug-en = <1>; ++ rockchip,sleep-mode-config = < ++ (0 ++ | RKPM_SLP_ARMPD ++ | RKPM_SLP_PERILPPD ++ | RKPM_SLP_DDR_RET ++ | RKPM_SLP_PLLPD ++ | RKPM_SLP_CENTER_PD ++ | RKPM_SLP_OSC_DIS ++ | RKPM_SLP_AP_PWROFF ++ ) ++ >; ++ rockchip,wakeup-config = ; ++ rockchip,pwm-regulator-config = ; ++ rockchip,power-ctrl = ++ <&gpio1 RK_PC1 GPIO_ACTIVE_HIGH>, ++ <&gpio1 RK_PB6 GPIO_ACTIVE_HIGH>; ++}; ++ ++&route_edp { ++ status = "okay"; ++}; ++ ++&saradc { ++ status = "okay"; ++ vref-supply = <&vcc_1v8>; ++}; ++ ++&sdmmc { ++ sd-uhs-sdr12; ++ sd-uhs-sdr25; ++ sd-uhs-sdr50; ++ sd-uhs-sdr104; ++}; ++ ++&spi1 { ++ status = "okay"; ++ max-freq = <48000000>; /* spi internal clk, don't modify */ ++ spi_dev@0 { ++ compatible = "rockchip,spidev"; ++ reg = <0>; ++ spi-max-frequency = <12000000>; ++ spi-lsb-first; ++ }; ++}; ++ ++&tcphy0 { ++ extcon = <&fusb0>; ++ status = "okay"; ++}; ++ ++&tcphy1 { ++ status = "okay"; ++}; ++ ++&tsadc { ++ rockchip,hw-tshut-mode = <1>; /* tshut mode 0:CRU 1:GPIO */ ++ rockchip,hw-tshut-polarity = <1>; /* tshut polarity 0:LOW 1:HIGH */ ++ status = "okay"; ++}; ++ ++&u2phy0 { ++ status = "okay"; ++ extcon = <&fusb0>; ++ ++ u2phy0_otg: otg-port { ++ status = "okay"; ++ }; ++ ++ u2phy0_host: host-port { ++ phy-supply = <&vcc5v0_usb>; ++ status = "okay"; ++ }; ++}; ++ ++&u2phy1 { ++ status = "okay"; ++ ++ u2phy1_otg: otg-port { ++ status = "okay"; ++ }; ++ ++ u2phy1_host: host-port { ++ phy-supply = <&vcc5v0_usb>; ++ status = "okay"; ++ }; ++}; ++ ++&uart0 { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart0_xfer &uart0_cts>; ++ status = "okay"; ++}; ++ ++&usb_host0_ehci { ++ status = "okay"; ++}; ++ ++&usb_host1_ehci { ++ status = "okay"; ++}; ++ ++&usb_host0_ohci { ++ status = "okay"; ++}; ++ ++&usb_host1_ohci { ++ status = "okay"; ++}; ++ ++&usbdrd3_0 { ++ status = "okay"; ++}; ++ ++&usbdrd3_1 { ++ status = "okay"; ++}; ++ ++&usbdrd_dwc3_0 { ++ status = "okay"; ++ extcon = <&fusb0>; ++}; ++ ++&usbdrd_dwc3_1 { ++ status = "okay"; ++}; ++ ++&vopb { ++ assigned-clocks = <&cru DCLK_VOP0_DIV>; ++ assigned-clock-parents = <&cru PLL_CPLL>; ++}; ++ ++&vopl { ++ assigned-clocks = <&cru DCLK_VOP1_DIV>; ++ assigned-clock-parents = <&cru PLL_VPLL>; ++}; ++ ++&pinctrl { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&npu_ref_clk>; ++ ++ bq2570 { ++ charger_ok_int: charger-ok-int { ++ rockchip,pins = ++ <1 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ fusb30x { ++ fusb0_int: fusb0-int { ++ rockchip,pins = ++ <1 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ headphone { ++ hp_det: hp-det { ++ rockchip,pins = ++ <0 RK_PB5 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ lcd_rst { ++ lcd_rst_gpio: lcd-rst-gpio { ++ rockchip,pins = ++ <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ npu_clk { ++ npu_ref_clk: npu-ref-clk { ++ rockchip,pins = ++ <0 RK_PA2 1 &pcfg_pull_none>; ++ }; ++ }; ++ ++ pmic { ++ pmic_int_l: pmic-int-l { ++ rockchip,pins = ++ <1 RK_PC2 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ vsel1_gpio: vsel1-gpio { ++ rockchip,pins = ++ <1 RK_PC1 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ vsel2_gpio: vsel2-gpio { ++ rockchip,pins = ++ <1 RK_PB6 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ ++ soc_slppin_gpio: soc-slppin-gpio { ++ rockchip,pins = ++ <1 RK_PA5 RK_FUNC_GPIO &pcfg_output_low>; ++ }; ++ ++ soc_slppin_slp: soc-slppin-slp { ++ rockchip,pins = ++ <1 RK_PA5 1 &pcfg_pull_down>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = ++ <2 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdmmc { ++ sdmmc_bus1: sdmmc-bus1 { ++ rockchip,pins = ++ <4 RK_PB0 1 &pcfg_pull_up_10ma>; ++ }; ++ ++ sdmmc_bus4: sdmmc-bus4 { ++ rockchip,pins = ++ <4 RK_PB0 1 &pcfg_pull_up_10ma>, ++ <4 RK_PB1 1 &pcfg_pull_up_10ma>, ++ <4 RK_PB2 1 &pcfg_pull_up_10ma>, ++ <4 RK_PB3 1 &pcfg_pull_up_10ma>; ++ }; ++ ++ sdmmc_clk: sdmmc-clk { ++ rockchip,pins = ++ <4 RK_PB4 1 &pcfg_pull_none_10ma>; ++ }; ++ ++ sdmmc_cmd: sdmmc-cmd { ++ rockchip,pins = ++ <4 RK_PB5 1 &pcfg_pull_up_10ma>; ++ }; ++ }; ++ ++ tp_irq { ++ tp_irq_gpio: tp-irq-gpio { ++ rockchip,pins = ++ <3 RK_PB0 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart0_gpios: uart0-gpios { ++ rockchip,pins = ++ <2 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ bt_irq_gpio: bt-irq-gpio { ++ rockchip,pins = ++ <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399pro-evb-v14-linux.dts b/arch/arm64/boot/dts/rockchip/rk3399pro-evb-v14-linux.dts +new file mode 100755 +index 000000000000..4f8546eb80bf +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399pro-evb-v14-linux.dts +@@ -0,0 +1,247 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ */ ++ ++/dts-v1/; ++#include "rk3399pro-evb-v11-linux.dts" ++ ++/ { ++ model = "Rockchip RK3399pro evb v14 board for linux"; ++ compatible = "rockchip,rk3399pro-evb-v14-linux", "rockchip,rk3399pro"; ++ ++ reserved-memory { ++ #address-cells = <2>; ++ #size-cells = <2>; ++ ++ dma_trans: dma_trans@3c000000 { ++ //no-map; ++ reg = <0x0 0x3c000000 0x0 0x04000000>; ++ }; ++ }; ++}; ++ ++/delete-node/ &imx327; ++/delete-node/ &ov13850; ++/delete-node/ &vm149c; ++&i2c1 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <345>; ++ i2c-scl-falling-time-ns = <11>; ++ pinctrl-0 = <&i2c1_xfer>, <&cam_pwren_high>; ++ ++ jaguar1: jaguar1@30 { ++ compatible = "jaguar1-v4l2"; ++ status = "okay"; ++ reg = <0x30>; ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ /* ++ * pd-gpios = <&gpio4 21 GPIO_ACTIVE_HIGH>; // conflict with csi-ctl-gpios ++ * rst-gpios = <&gpio0 12 GPIO_ACTIVE_HIGH>; ++ */ ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "jaguar1"; ++ rockchip,camera-module-lens-name = "jaguar1"; ++ port { ++ cam_out: endpoint { ++ remote-endpoint = <&usbacm_video_control_in>; ++ data-lanes = <1 2 3 4>; ++ }; ++ }; ++ }; ++}; ++ ++&i2c4 { ++ status = "okay"; ++ ++ i2c-scl-rising-time-ns = <345>; ++ i2c-scl-falling-time-ns = <11>; ++ ++ vm149c: vm149c@0c { ++ compatible = "silicon touch,vm149c"; ++ status = "okay"; ++ reg = <0x0c>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ }; ++ ++ ov13850: ov13850@10 { ++ compatible = "ovti,ov13850"; ++ status = "okay"; ++ reg = <0x10>; ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ ++ /* conflict with csi-ctl-gpios */ ++ reset-gpios = <&gpio4 RK_PD2 GPIO_ACTIVE_HIGH>; ++ pwdn-gpios = <&gpio4 RK_PD1 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "rockchip,camera_default"; ++ pinctrl-0 = <&cif_clkout>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "CMK-CT0116"; ++ rockchip,camera-module-lens-name = "Largan-50013A1"; ++ lens-focus = <&vm149c>; ++ ++ port { ++ ucam_out1: endpoint { ++ remote-endpoint = <&mipi_in_ucam1>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++ ++ imx327: imx327@1a { ++ compatible = "sony,imx327"; ++ status = "okay"; ++ reg = <0x1a>; ++ clocks = <&cru SCLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ /* conflict with csi-ctl-gpios */ ++ reset-gpios = <&gpio4 26 GPIO_ACTIVE_HIGH>; ++ pwdn-gpios = <&gpio4 25 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_clkout>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "TongJu"; ++ rockchip,camera-module-lens-name = "CHT842-MD"; ++ port { ++ ucam_out2: endpoint { ++ remote-endpoint = <&mipi_in_ucam2>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++}; ++ ++&mipi_dphy_rx0 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam0: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&usbacm_video_control_out>; ++ data-lanes = <1 2 3 4>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_rx0_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp0_mipi_in>; ++ }; ++ }; ++ }; ++}; ++ ++&mipi_dphy_tx1rx1 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam1: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ucam_out1>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy_tx1rx1_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp1_mipi_in>; ++ }; ++ }; ++ }; ++}; ++ ++&pcie0 { ++ /delete-property/ ep-gpios; ++ num-lanes = <4>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pcie_clkreqn_cpm>; ++ max-link-speed = <1>; ++ memory-region = <&dma_trans>; ++ busno = <0>; ++ rockchip,dma_trx_enabled = <1>; ++ rockchip,deferred = <1>; ++ status = "okay"; ++}; ++ ++&rkisp1_0 { ++ status = "okay"; ++ ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp0_mipi_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dphy_rx0_out>; ++ }; ++ }; ++}; ++ ++&usbacm_video_control { ++ status = "okay"; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "usbacm_video_control"; ++ rockchip,camera-module-lens-name = "usbacm_video_control"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ usbacm_video_control_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&cam_out>; ++ data-lanes = <1 2 3 4>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ usbacm_video_control_out: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&mipi_in_ucam0>; ++ data-lanes = <1 2 3 4>; ++ }; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399pro-npu-evb-v10.dts b/arch/arm64/boot/dts/rockchip/rk3399pro-npu-evb-v10.dts +new file mode 100755 +index 000000000000..96a3fb56cfe5 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399pro-npu-evb-v10.dts +@@ -0,0 +1,140 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++/dts-v1/; ++#include ++#include ++#include "rk3399pro-npu.dtsi" ++ ++/ { ++ model = "Rockchip RK3399pro-npu EVB V10 Board"; ++ compatible = "rockchip,rk3399pro-npu-evb-v10", "rockchip,rk3399pro-npu"; ++ ++ chosen { ++ bootargs = "earlycon=uart8250,mmio32,0xff550000 console=ttyFIQ0 init=/init kpti=0"; ++ }; ++ ++ keys: gpio-keys { ++ compatible = "gpio-keys"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pwr_key>; ++ ++ power { ++ gpios = <&gpio0 RK_PB0 GPIO_ACTIVE_HIGH>; ++ label = "GPIO Power"; ++ linux,code = <116>; ++ wakeup-source; ++ }; ++ }; ++ ++ fiq-debugger { ++ compatible = "rockchip,fiq-debugger"; ++ rockchip,serial-id = <2>; ++ rockchip,wake-irq = <0>; ++ /* If enable uart uses irq instead of fiq */ ++ rockchip,irq-mode-enable = <0>; ++ rockchip,baudrate = <1500000>; /* Only 115200 and 1500000 */ ++ interrupts = ; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart2m0_xfer>; ++ status = "okay"; ++ }; ++ ++ vdd_cpu: vdd-cpu { ++ compatible = "regulator-fixed"; ++ regulator-name = "vdd_cpu"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <850000>; ++ regulator-max-microvolt = <850000>; ++ }; ++ ++ vcc5v0_sys: vcc5v0-sys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc5v0_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ }; ++}; ++ ++&cpu0 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu1 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ ++ vdd_npu: tcs452x@1c { ++ compatible = "tcs,tcs452x"; ++ reg = <0x1c>; ++ vin-supply = <&vcc5v0_sys>; ++ regulator-compatible = "fan53555-reg"; ++ pinctrl-0 = <&vsel_gpio>; ++ vsel-gpios = <&gpio0 RK_PC6 GPIO_ACTIVE_HIGH>; ++ regulator-name = "vdd_npu"; ++ regulator-min-microvolt = <750000>; ++ regulator-max-microvolt = <800000>; ++ regulator-ramp-delay = <2300>; ++ fcs,suspend-voltage-selector = <1>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-state = <3>; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++}; ++ ++&npu { ++ npu-supply = <&vdd_npu>; ++ status = "okay"; ++}; ++ ++&combphy { ++ status = "okay"; ++}; ++ ++&u2phy { ++ status = "okay"; ++}; ++ ++&u2phy_otg { ++ status = "okay"; ++}; ++ ++&usbdrd3 { ++ status = "okay"; ++}; ++ ++&usbdrd_dwc3 { ++ status = "okay"; ++}; ++ ++&tsadc { ++ rockchip,hw-tshut-mode = <1>; /* tshut mode 0:CRU 1:GPIO */ ++ rockchip,hw-tshut-polarity = <1>; /* tshut polarity 0:LOW 1:HIGH */ ++ pinctrl-names = "init", "default"; ++ pinctrl-0 = <&tsadc_otp_gpio>; ++ pinctrl-1 = <&tsadc_otp_out>; ++ status = "okay"; ++}; ++ ++&pinctrl { ++ vsel_gpio: vsel-gpio { ++ rockchip,pins = ++ <0 RK_PC6 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ ++ pwr_key: pwr-key { ++ rockchip,pins = ++ <0 RK_PB0 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3399pro-npu.dtsi b/arch/arm64/boot/dts/rockchip/rk3399pro-npu.dtsi +new file mode 100755 +index 000000000000..3f176b3bc94f +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3399pro-npu.dtsi +@@ -0,0 +1,826 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++// Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd. ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++/ { ++ compatible = "rockchip,rk3399pro-npu"; ++ ++ interrupt-parent = <&gic>; ++ #address-cells = <2>; ++ #size-cells = <2>; ++ ++ aliases { ++ i2c0 = &i2c0; ++ i2c1 = &i2c1; ++ serial2 = &uart2; ++ }; ++ ++ cpus { ++ #address-cells = <2>; ++ #size-cells = <0>; ++ ++ cpu0: cpu@0 { ++ device_type = "cpu"; ++ compatible = "arm,cortex-a35", "arm,armv8"; ++ reg = <0x0 0x0>; ++ enable-method = "psci"; ++ clocks = <&cru ARMCLK>; ++ operating-points-v2 = <&cpu0_opp_table>; ++ dynamic-power-coefficient = <74>; ++ #cooling-cells = <2>; ++ power-model { ++ compatible = "simple-power-model"; ++ ref-leakage = <31>; ++ static-coefficient = <100000>; ++ ts = <597400 241050 (-2450) 70>; ++ thermal-zone = "soc-thermal"; ++ }; ++ }; ++ ++ cpu1: cpu@1 { ++ device_type = "cpu"; ++ compatible = "arm,cortex-a35", "arm,armv8"; ++ reg = <0x0 0x1>; ++ enable-method = "psci"; ++ clocks = <&cru ARMCLK>; ++ operating-points-v2 = <&cpu0_opp_table>; ++ dynamic-power-coefficient = <74>; ++ }; ++ }; ++ ++ cpu0_opp_table: cpu0-opp-table { ++ compatible = "operating-points-v2"; ++ opp-shared; ++ ++ opp-408000000 { ++ opp-hz = /bits/ 64 <408000000>; ++ opp-microvolt = <750000 750000 950000>; ++ clock-latency-ns = <40000>; ++ opp-suspend; ++ }; ++ opp-600000000 { ++ opp-hz = /bits/ 64 <600000000>; ++ opp-microvolt = <750000 750000 950000>; ++ clock-latency-ns = <40000>; ++ }; ++ opp-816000000 { ++ opp-hz = /bits/ 64 <816000000>; ++ opp-microvolt = <750000 750000 950000>; ++ clock-latency-ns = <40000>; ++ }; ++ opp-1008000000 { ++ opp-hz = /bits/ 64 <1008000000>; ++ opp-microvolt = <750000 750000 950000>; ++ clock-latency-ns = <40000>; ++ }; ++ opp-1200000000 { ++ opp-hz = /bits/ 64 <1200000000>; ++ opp-microvolt = <750000 750000 950000>; ++ clock-latency-ns = <40000>; ++ }; ++ }; ++ ++ arm-pmu { ++ compatible = "arm,cortex-a53-pmu"; ++ interrupts = , ++ ; ++ interrupt-affinity = <&cpu0>, <&cpu1>; ++ }; ++ ++ psci { ++ compatible = "arm,psci-1.0"; ++ method = "smc"; ++ }; ++ ++ timer { ++ compatible = "arm,armv8-timer"; ++ interrupts = , ++ , ++ , ++ ; ++ arm,no-tick-in-suspend; ++ }; ++ ++ xin24m: xin24m { ++ compatible = "fixed-clock"; ++ clock-frequency = <24000000>; ++ clock-output-names = "xin24m"; ++ #clock-cells = <0>; ++ }; ++ ++ xin32k: xin32k { ++ compatible = "fixed-clock"; ++ clock-frequency = <32768>; ++ clock-output-names = "xin32k"; ++ #clock-cells = <0>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&clkin_32k>; ++ }; ++ ++ usbdrd3: usb { ++ compatible = "rockchip,rk1808-dwc3", "rockchip,rk3399-dwc3"; ++ clocks = <&cru SCLK_USB3_OTG0_REF>, <&cru ACLK_USB3OTG>, ++ <&cru SCLK_USB3_OTG0_SUSPEND>; ++ clock-names = "ref_clk", "bus_clk", ++ "suspend_clk"; ++ assigned-clocks = <&cru SCLK_USB3_OTG0_SUSPEND>; ++ assigned-clock-rates = <24000000>; ++ power-domains = <&power RK1808_PD_PCIE>; ++ resets = <&cru SRST_USB3_OTG_A>; ++ reset-names = "usb3-otg"; ++ #address-cells = <2>; ++ #size-cells = <2>; ++ ranges; ++ status = "disabled"; ++ ++ usbdrd_dwc3: dwc3@fd000000 { ++ compatible = "snps,dwc3"; ++ reg = <0x0 0xfd000000 0x0 0x200000>; ++ interrupts = ; ++ dr_mode = "peripheral"; ++ phys = <&u2phy_otg>, <&combphy PHY_TYPE_USB3>; ++ phy-names = "usb2-phy", "usb3-phy"; ++ phy_type = "utmi_wide"; ++ snps,dis_enblslpm_quirk; ++ snps,dis-u1u2-quirk; ++ snps,dis-u2-freeclk-exists-quirk; ++ snps,dis_u2_susphy_quirk; ++ snps,dis_u3_susphy_quirk; ++ snps,dis-del-phy-power-chg-quirk; ++ snps,tx-ipgap-linecheck-dis-quirk; ++ snps,xhci-trb-ent-quirk; ++ status = "disabled"; ++ }; ++ }; ++ ++ grf: syscon@fe000000 { ++ compatible = "rockchip,rk1808-grf", "syscon", "simple-mfd"; ++ reg = <0x0 0xfe000000 0x0 0x1000>; ++ #address-cells = <1>; ++ #size-cells = <1>; ++ ++ npu_pvtm: npu-pvtm { ++ compatible = "rockchip,rk1808-npu-pvtm"; ++ clocks = <&cru SCLK_PVTM_NPU>; ++ clock-names = "npu"; ++ status = "okay"; ++ }; ++ }; ++ ++ usb2phy_grf: syscon@fe010000 { ++ compatible = "rockchip,rk1808-usb2phy-grf", "syscon", ++ "simple-mfd"; ++ reg = <0x0 0xfe010000 0x0 0x8000>; ++ #address-cells = <1>; ++ #size-cells = <1>; ++ ++ u2phy: usb2-phy@100 { ++ compatible = "rockchip,rk1808-usb2phy"; ++ reg = <0x100 0x10>; ++ clocks = <&cru SCLK_USBPHY_REF>; ++ clock-names = "phyclk"; ++ #clock-cells = <0>; ++ assigned-clocks = <&cru USB480M>; ++ assigned-clock-parents = <&u2phy>; ++ clock-output-names = "usb480m_phy"; ++ status = "disabled"; ++ ++ u2phy_host: host-port { ++ #phy-cells = <0>; ++ interrupts = ; ++ interrupt-names = "linestate"; ++ status = "disabled"; ++ }; ++ ++ u2phy_otg: otg-port { ++ #phy-cells = <0>; ++ interrupts = , ++ , ++ ; ++ interrupt-names = "otg-bvalid", "otg-id", ++ "linestate"; ++ status = "disabled"; ++ }; ++ }; ++ }; ++ ++ combphy_grf: syscon@fe018000 { ++ compatible = "rockchip,usb3phy-grf", "syscon"; ++ reg = <0x0 0xfe018000 0x0 0x8000>; ++ }; ++ ++ pmugrf: syscon@fe020000 { ++ compatible = "rockchip,rk1808-pmugrf", "syscon", "simple-mfd"; ++ reg = <0x0 0xfe020000 0x0 0x1000>; ++ #address-cells = <1>; ++ #size-cells = <1>; ++ ++ pmu_pvtm: pmu-pvtm { ++ compatible = "rockchip,rk1808-pmu-pvtm"; ++ clocks = <&cru SCLK_PVTM_PMU>; ++ clock-names = "pmu"; ++ status = "okay"; ++ }; ++ }; ++ ++ usb_pcie_grf: syscon@fe040000 { ++ compatible = "rockchip,usb-pcie-grf", "syscon"; ++ reg = <0x0 0xfe040000 0x0 0x1000>; ++ }; ++ ++ coregrf: syscon@fe050000 { ++ compatible = "rockchip,rk1808-coregrf", "syscon", "simple-mfd"; ++ reg = <0x0 0xfe050000 0x0 0x1000>; ++ #address-cells = <1>; ++ #size-cells = <1>; ++ ++ pvtm: pvtm { ++ compatible = "rockchip,rk1808-pvtm"; ++ clocks = <&cru SCLK_PVTM_CORE>; ++ clock-names = "core"; ++ status = "okay"; ++ }; ++ }; ++ ++ qos_npu: qos@fe850000 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe850000 0x0 0x20>; ++ }; ++ ++ qos_pcie: qos@fe880000 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe880000 0x0 0x20>; ++ status = "disabled"; ++ }; ++ ++ qos_usb2: qos@fe890000 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe890000 0x0 0x20>; ++ status = "disabled"; ++ }; ++ ++ qos_usb3: qos@fe890080 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe890080 0x0 0x20>; ++ status = "disabled"; ++ }; ++ ++ qos_isp: qos@fe8a0000 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe8a0000 0x0 0x20>; ++ }; ++ ++ qos_rga_rd: qos@fe8a0080 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe8a0080 0x0 0x20>; ++ }; ++ ++ qos_rga_wr: qos@fe8a0100 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe8a0100 0x0 0x20>; ++ }; ++ ++ qos_cif: qos@fe8a0180 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe8a0180 0x0 0x20>; ++ }; ++ ++ qos_vop_raw: qos@fe8b0000 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe8b0000 0x0 0x20>; ++ }; ++ ++ qos_vop_lite: qos@fe8b0080 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe8b0080 0x0 0x20>; ++ }; ++ ++ qos_vpu: qos@fe8c0000 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe8c0000 0x0 0x20>; ++ }; ++ ++ gic: interrupt-controller@ff100000 { ++ compatible = "arm,gic-v3"; ++ #interrupt-cells = <3>; ++ #address-cells = <2>; ++ #size-cells = <2>; ++ ranges; ++ interrupt-controller; ++ ++ reg = <0x0 0xff100000 0 0x10000>, /* GICD */ ++ <0x0 0xff140000 0 0xc0000>, /* GICR */ ++ <0x0 0xff300000 0 0x10000>, /* GICC */ ++ <0x0 0xff310000 0 0x10000>, /* GICH */ ++ <0x0 0xff320000 0 0x10000>; /* GICV */ ++ interrupts = ; ++ its: interrupt-controller@ff120000 { ++ compatible = "arm,gic-v3-its"; ++ msi-controller; ++ reg = <0x0 0xff120000 0x0 0x20000>; ++ }; ++ }; ++ ++ cru: clock-controller@ff350000 { ++ compatible = "rockchip,rk1808-cru"; ++ reg = <0x0 0xff350000 0x0 0x5000>; ++ rockchip,grf = <&grf>; ++ #clock-cells = <1>; ++ #reset-cells = <1>; ++ ++ assigned-clocks = ++ <&cru PLL_GPLL>, <&cru PLL_CPLL>, ++ <&cru PLL_PPLL>, <&cru ARMCLK>, ++ <&cru MSCLK_PERI>, <&cru LSCLK_PERI>, ++ <&cru HSCLK_BUS_PRE>, <&cru MSCLK_BUS_PRE>, ++ <&cru LSCLK_BUS_PRE>; ++ assigned-clock-rates = ++ <1188000000>, <1000000000>, ++ <100000000>, <1200000000>, ++ <200000000>, <100000000>, ++ <300000000>, <200000000>, ++ <100000000>; ++ }; ++ ++ combphy: phy@ff380000 { ++ compatible = "rockchip,rk1808-combphy"; ++ reg = <0x0 0xff380000 0x0 0x10000>; ++ #phy-cells = <1>; ++ clocks = <&cru SCLK_PCIEPHY_REF>; ++ clock-names = "refclk"; ++ assigned-clocks = <&cru SCLK_PCIEPHY_REF>; ++ assigned-clock-rates = <25000000>; ++ resets = <&cru SRST_USB3_OTG_A>, <&cru SRST_PCIEPHY_POR>, ++ <&cru SRST_PCIEPHY_P>, <&cru SRST_PCIEPHY_PIPE>; ++ reset-names = "otg-rst", "combphy-por", ++ "combphy-apb", "combphy-pipe"; ++ rockchip,combphygrf = <&combphy_grf>; ++ rockchip,usbpciegrf = <&usb_pcie_grf>; ++ status = "disabled"; ++ }; ++ ++ thermal_zones: thermal-zones { ++ soc_thermal: soc-thermal { ++ polling-delay-passive = <20>; /* milliseconds */ ++ polling-delay = <1000>; /* milliseconds */ ++ sustainable-power = <977>; /* milliwatts */ ++ ++ thermal-sensors = <&tsadc 0>; ++ ++ trips { ++ threshold: trip-point-0 { ++ /* millicelsius */ ++ temperature = <75000>; ++ /* millicelsius */ ++ hysteresis = <2000>; ++ type = "passive"; ++ }; ++ target: trip-point-1 { ++ /* millicelsius */ ++ temperature = <85000>; ++ /* millicelsius */ ++ hysteresis = <2000>; ++ type = "passive"; ++ }; ++ soc_crit: soc-crit { ++ /* millicelsius */ ++ temperature = <115000>; ++ /* millicelsius */ ++ hysteresis = <2000>; ++ type = "critical"; ++ }; ++ }; ++ ++ cooling-maps { ++ map0 { ++ trip = <&target>; ++ cooling-device = ++ <&cpu0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; ++ contribution = <4096>; ++ }; ++ map1 { ++ trip = <&target>; ++ cooling-device = ++ <&npu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; ++ contribution = <1024>; ++ }; ++ }; ++ }; ++ }; ++ ++ tsadc: tsadc@ff3a0000 { ++ compatible = "rockchip,rk1808-tsadc"; ++ reg = <0x0 0xff3a0000 0x0 0x100>; ++ interrupts = ; ++ rockchip,grf = <&grf>; ++ clocks = <&cru SCLK_TSADC>, <&cru PCLK_TSADC>; ++ clock-names = "tsadc", "apb_pclk"; ++ assigned-clocks = <&cru SCLK_TSADC>; ++ assigned-clock-rates = <650000>; ++ resets = <&cru SRST_TSADC>; ++ reset-names = "tsadc-apb"; ++ #thermal-sensor-cells = <1>; ++ rockchip,hw-tshut-temp = <120000>; ++ status = "disabled"; ++ }; ++ ++ pmu: power-management@ff3e0000 { ++ compatible = "rockchip,rk1808-pmu", "syscon", "simple-mfd"; ++ reg = <0x0 0xff3e0000 0x0 0x1000>; ++ ++ power: power-controller { ++ compatible = "rockchip,rk1808-power-controller"; ++ #power-domain-cells = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ status = "okay"; ++ ++ /* These power domains are grouped by VD_NPU */ ++ pd_npu@RK1808_VD_NPU { ++ reg = ; ++ clocks = <&cru SCLK_NPU>, ++ <&cru ACLK_NPU>, ++ <&cru HCLK_NPU>; ++ pm_qos = <&qos_npu>; ++ }; ++ ++ /* These power domains are grouped by VD_LOGIC */ ++ pd_pcie@RK1808_PD_PCIE { ++ reg = ; ++ clocks = <&cru HSCLK_PCIE>, ++ <&cru LSCLK_PCIE>, ++ <&cru ACLK_PCIE>, ++ <&cru ACLK_PCIE_MST>, ++ <&cru ACLK_PCIE_SLV>, ++ <&cru PCLK_PCIE>, ++ <&cru SCLK_PCIE_AUX>, ++ <&cru SCLK_PCIE_AUX>, ++ <&cru ACLK_USB3OTG>, ++ <&cru HCLK_HOST>, ++ <&cru HCLK_HOST_ARB>, ++ <&cru SCLK_USB3_OTG0_REF>, ++ <&cru SCLK_USB3_OTG0_SUSPEND>; ++ pm_qos = <&qos_pcie>, ++ <&qos_usb2>, ++ <&qos_usb3>; ++ }; ++ pd_vpu@RK1808_PD_VPU { ++ reg = ; ++ clocks = <&cru ACLK_VPU>, ++ <&cru HCLK_VPU>; ++ pm_qos = <&qos_vpu>; ++ }; ++ pd_vio@RK1808_PD_VIO { ++ reg = ; ++ clocks = <&cru HSCLK_VIO>, ++ <&cru LSCLK_VIO>, ++ <&cru ACLK_VOPRAW>, ++ <&cru HCLK_VOPRAW>, ++ <&cru ACLK_VOPLITE>, ++ <&cru HCLK_VOPLITE>, ++ <&cru PCLK_DSI_TX>, ++ <&cru PCLK_CSI_TX>, ++ <&cru ACLK_RGA>, ++ <&cru HCLK_RGA>, ++ <&cru ACLK_ISP>, ++ <&cru HCLK_ISP>, ++ <&cru ACLK_CIF>, ++ <&cru HCLK_CIF>, ++ <&cru PCLK_CSI2HOST>, ++ <&cru DCLK_VOPRAW>, ++ <&cru DCLK_VOPLITE>; ++ pm_qos = <&qos_rga_rd>, <&qos_rga_wr>, ++ <&qos_isp>, <&qos_cif>, ++ <&qos_vop_raw>, <&qos_vop_lite>; ++ }; ++ }; ++ }; ++ ++ i2c0: i2c@ff410000 { ++ compatible = "rockchip,rk3399-i2c"; ++ reg = <0x0 0xff410000 0x0 0x1000>; ++ clocks = <&cru SCLK_PMU_I2C0>, <&cru PCLK_I2C0_PMU>; ++ clock-names = "i2c", "pclk"; ++ interrupts = ; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2c0_xfer>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ status = "disabled"; ++ }; ++ ++ dmac: dmac@ff4e0000 { ++ compatible = "arm,pl330", "arm,primecell"; ++ reg = <0x0 0xff4e0000 0x0 0x4000>; ++ interrupts = ; ++ clocks = <&cru ACLK_DMAC>; ++ clock-names = "apb_pclk"; ++ #dma-cells = <1>; ++ arm,pl330-periph-burst; ++ }; ++ ++ i2c1: i2c@ff500000 { ++ compatible = "rockchip,rk3399-i2c"; ++ reg = <0x0 0xff500000 0x0 0x1000>; ++ clocks = <&cru SCLK_I2C1>, <&cru PCLK_I2C1>; ++ clock-names = "i2c", "pclk"; ++ interrupts = ; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2c1_xfer>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ status = "disabled"; ++ }; ++ ++ uart2: serial@ff550000 { ++ compatible = "rockchip,rk1808-uart", "snps,dw-apb-uart"; ++ reg = <0x0 0xff550000 0x0 0x100>; ++ interrupts = ; ++ clocks = <&cru SCLK_UART2>, <&cru PCLK_UART2>; ++ clock-names = "baudclk", "apb_pclk"; ++ reg-shift = <2>; ++ reg-io-width = <4>; ++ dmas = <&dmac 4>, <&dmac 5>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart2m0_xfer>; ++ status = "disabled"; ++ }; ++ ++ rktimer: rktimer@ff700000 { ++ compatible = "rockchip,rk3288-timer"; ++ reg = <0x0 0xff700000 0x0 0x1000>; ++ interrupts = ; ++ clocks = <&cru PCLK_TIMER>, <&cru SCLK_TIMER0>; ++ clock-names = "pclk", "timer"; ++ }; ++ ++ npu: npu@ffbc0000 { ++ compatible = "rockchip,npu"; ++ reg = <0x0 0xffbc0000 0x0 0x1000>; ++ clocks = <&cru SCLK_NPU>, <&cru ACLK_NPU>, <&cru HCLK_NPU>; ++ clock-names = "sclk_npu", "aclk_npu", "hclk_npu"; ++ assigned-clocks = <&cru SCLK_NPU>; ++ assigned-clock-rates = <800000000>; ++ interrupts = ; ++ power-domains = <&power RK1808_VD_NPU>; ++ operating-points-v2 = <&npu_opp_table>; ++ #cooling-cells = <2>; ++ status = "disabled"; ++ ++ npu_power_model: power-model { ++ compatible = "simple-power-model"; ++ ref-leakage = <31>; ++ static-coefficient = <100000>; ++ dynamic-coefficient = <3080>; ++ ts = <88610 303120 (-5000) 100>; ++ thermal-zone = "soc-thermal"; ++ }; ++ }; ++ ++ npu_opp_table: npu-opp-table { ++ compatible = "operating-points-v2"; ++ ++ rockchip,max-volt = <880000>; ++ rockchip,evb-irdrop = <37500>; ++ ++ rockchip,pvtm-voltage-sel = < ++ 0 69000 0 ++ 69001 74000 1 ++ 74001 99999 2 ++ >; ++ rockchip,pvtm-freq = <200000>; ++ rockchip,pvtm-volt = <800000>; ++ rockchip,pvtm-ch = <0 0>; ++ rockchip,pvtm-sample-time = <1000>; ++ rockchip,pvtm-number = <10>; ++ rockchip,pvtm-error = <1000>; ++ rockchip,pvtm-ref-temp = <25>; ++ rockchip,pvtm-temp-prop = <(-20) (-26)>; ++ rockchip,thermal-zone = "soc-thermal"; ++ ++ opp-200000000 { ++ opp-hz = /bits/ 64 <200000000>; ++ opp-microvolt = <750000 750000 880000>; ++ }; ++ opp-297000000 { ++ opp-hz = /bits/ 64 <297000000>; ++ opp-microvolt = <750000 750000 880000>; ++ }; ++ opp-400000000 { ++ opp-hz = /bits/ 64 <400000000>; ++ opp-microvolt = <750000 750000 880000>; ++ }; ++ opp-594000000 { ++ opp-hz = /bits/ 64 <594000000>; ++ opp-microvolt = <750000 750000 880000>; ++ }; ++ opp-792000000 { ++ opp-hz = /bits/ 64 <792000000>; ++ opp-microvolt = <850000 850000 880000>; ++ opp-microvolt-L0 = <850000 850000 880000>; ++ opp-microvolt-L1 = <825000 825000 880000>; ++ opp-microvolt-L2 = <800000 800000 880000>; ++ }; ++ }; ++ ++ pinctrl: pinctrl { ++ compatible = "rockchip,rk1808-pinctrl"; ++ rockchip,grf = <&grf>; ++ rockchip,pmu = <&pmugrf>; ++ #address-cells = <2>; ++ #size-cells = <2>; ++ ranges; ++ ++ gpio0: gpio0@ff4c0000 { ++ compatible = "rockchip,gpio-bank"; ++ reg = <0x0 0xff4c0000 0x0 0x100>; ++ interrupts = ; ++ clocks = <&cru PCLK_GPIO0_PMU>, <&cru DBCLK_PMU_GPIO0>; ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ interrupt-controller; ++ #interrupt-cells = <2>; ++ }; ++ ++ gpio1: gpio1@ff690000 { ++ compatible = "rockchip,gpio-bank"; ++ reg = <0x0 0xff690000 0x0 0x100>; ++ interrupts = ; ++ clocks = <&cru PCLK_GPIO1>, <&cru DBCLK_GPIO1>; ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ interrupt-controller; ++ #interrupt-cells = <2>; ++ }; ++ ++ gpio2: gpio2@ff6a0000 { ++ compatible = "rockchip,gpio-bank"; ++ reg = <0x0 0xff6a0000 0x0 0x100>; ++ interrupts = ; ++ clocks = <&cru PCLK_GPIO2>, <&cru DBCLK_GPIO2>; ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ interrupt-controller; ++ #interrupt-cells = <2>; ++ }; ++ ++ gpio3: gpio3@ff6b0000 { ++ compatible = "rockchip,gpio-bank"; ++ reg = <0x0 0xff6b0000 0x0 0x100>; ++ interrupts = ; ++ clocks = <&cru PCLK_GPIO3>, <&cru DBCLK_GPIO3>; ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ interrupt-controller; ++ #interrupt-cells = <2>; ++ }; ++ ++ gpio4: gpio4@ff6c0000 { ++ compatible = "rockchip,gpio-bank"; ++ reg = <0x0 0xff6c0000 0x0 0x100>; ++ interrupts = ; ++ clocks = <&cru PCLK_GPIO4>, <&cru DBCLK_GPIO4>; ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ interrupt-controller; ++ #interrupt-cells = <2>; ++ }; ++ ++ pcfg_pull_down: pcfg-pull-down { ++ bias-pull-down; ++ }; ++ ++ pcfg_pull_none: pcfg-pull-none { ++ bias-disable; ++ }; ++ ++ pcfg_pull_up_2ma: pcfg-pull-up-2ma { ++ bias-pull-up; ++ drive-strength = <2>; ++ }; ++ ++ pcfg_pull_none_smt: pcfg-pull-none-smt { ++ bias-disable; ++ input-schmitt-enable; ++ }; ++ ++ pcfg_pull_none_2ma_smt: pcfg-pull-none-2ma-smt { ++ bias-disable; ++ drive-strength = <2>; ++ input-schmitt-enable; ++ }; ++ ++ pcfg_output_high: pcfg-output-high { ++ output-high; ++ }; ++ ++ pcfg_input_smt: pcfg-input-smt { ++ input-enable; ++ input-schmitt-enable; ++ }; ++ ++ i2c0 { ++ i2c0_xfer: i2c0-xfer { ++ rockchip,pins = ++ /* i2c0_sda */ ++ <0 RK_PB1 1 &pcfg_pull_none_2ma_smt>, ++ /* i2c0_scl */ ++ <0 RK_PB0 1 &pcfg_pull_none_2ma_smt>; ++ }; ++ }; ++ ++ i2c1 { ++ i2c1_xfer: i2c1-xfer { ++ rockchip,pins = ++ /* i2c1_sda */ ++ <0 RK_PC1 1 &pcfg_pull_none_2ma_smt>, ++ /* i2c1_scl */ ++ <0 RK_PC0 1 &pcfg_pull_none_2ma_smt>; ++ }; ++ }; ++ ++ pciusb { ++ pciusb_pins: pciusb-pins { ++ rockchip,pins = ++ /* pciusb_debug0 */ ++ <4 RK_PB4 3 &pcfg_pull_none>, ++ /* pciusb_debug1 */ ++ <4 RK_PB5 3 &pcfg_pull_none>, ++ /* pciusb_debug2 */ ++ <4 RK_PB6 3 &pcfg_pull_none>, ++ /* pciusb_debug3 */ ++ <4 RK_PB7 3 &pcfg_pull_none>, ++ /* pciusb_debug4 */ ++ <4 RK_PC0 3 &pcfg_pull_none>, ++ /* pciusb_debug5 */ ++ <4 RK_PC1 3 &pcfg_pull_none>, ++ /* pciusb_debug6 */ ++ <4 RK_PC2 3 &pcfg_pull_none>, ++ /* pciusb_debug7 */ ++ <4 RK_PC3 3 &pcfg_pull_none>; ++ }; ++ }; ++ ++ uart2 { ++ uart2m0_xfer: uart2m0-xfer { ++ rockchip,pins = ++ /* uart2_rxm0 */ ++ <4 RK_PA3 2 &pcfg_pull_up_2ma>, ++ /* uart2_txm0 */ ++ <4 RK_PA2 2 &pcfg_pull_up_2ma>; ++ }; ++ ++ uart2m1_xfer: uart2m1-xfer { ++ rockchip,pins = ++ /* uart2_rxm1 */ ++ <2 RK_PD1 2 &pcfg_pull_up_2ma>, ++ /* uart2_txm1 */ ++ <2 RK_PD0 2 &pcfg_pull_up_2ma>; ++ }; ++ ++ uart2m2_xfer: uart2m2-xfer { ++ rockchip,pins = ++ /* uart2_rxm2 */ ++ <3 RK_PA4 2 &pcfg_pull_up_2ma>, ++ /* uart2_txm2 */ ++ <3 RK_PA3 2 &pcfg_pull_up_2ma>; ++ }; ++ }; ++ ++ tsadc { ++ tsadc_otp_gpio: tsadc-otp-gpio { ++ rockchip,pins = ++ <0 RK_PA6 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ tsadc_otp_out: tsadc-otp-out { ++ rockchip,pins = ++ <0 RK_PA6 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ xin32k { ++ clkin_32k: clkin-32k { ++ rockchip,pins = ++ <0 RK_PC2 1 &pcfg_input_smt>; ++ }; ++ ++ clkout_32k: clkout-32k { ++ rockchip,pins = ++ <0 RK_PC2 1 &pcfg_output_high>; ++ }; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3566-box-demo-v10.dts b/arch/arm64/boot/dts/rockchip/rk3566-box-demo-v10.dts +new file mode 100755 +index 000000000000..1bd285bfc695 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3566-box-demo-v10.dts +@@ -0,0 +1,15 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2021 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++/dts-v1/; ++ ++#include "rk3566-box-demo-v10.dtsi" ++#include "rk3568-android.dtsi" ++ ++/ { ++ model = "Rockchip RK3566 BOX DEMO V10 ANDROID Board"; ++ compatible = "rockchip,rk3566-box-demo-v10", "rockchip,rk3566"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3566-box-demo-v10.dtsi b/arch/arm64/boot/dts/rockchip/rk3566-box-demo-v10.dtsi +new file mode 100755 +index 000000000000..41ac0af11188 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3566-box-demo-v10.dtsi +@@ -0,0 +1,528 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2021 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++/dts-v1/; ++ ++#include "rk3566-box.dtsi" ++ ++/ { ++ model = "Rockchip RK3566 BOX DEMO V10 Board"; ++ compatible = "rockchip,rk3568-box-demo-v10", "rockchip,rk3566"; ++ ++ gpio-leds { ++ compatible = "gpio-leds"; ++ ++ ir-led { ++ gpios = <&gpio4 RK_PC5 GPIO_ACTIVE_HIGH>; ++ default-state = "off"; ++ }; ++ ++ work-led { ++ gpios = <&gpio0 RK_PC3 GPIO_ACTIVE_HIGH>; ++ linux,default-trigger = "timer"; ++ }; ++ }; ++ ++ vcc2v5_sys: vcc2v5-ddr { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc2v5-sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <2500000>; ++ regulator-max-microvolt = <2500000>; ++ vin-supply = <&vcc3v3_sys>; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ clocks = <&pmucru CLK_RTC_32K>; ++ clock-names = "ext_clock"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h &wifi_32k>; ++ reset-gpios = <&gpio2 RK_PB1 GPIO_ACTIVE_LOW>; ++ }; ++ ++ vcc3v3_sd: vcc3v3-sd-regulator { ++ compatible = "regulator-gpio"; ++ regulator-name = "vcc3v3_sd"; ++ regulator-min-microvolt = <100000>; ++ regulator-max-microvolt = <3300000>; ++ gpios = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>; ++ gpios-states = <0x1>; ++ states = <100000 0x1 ++ 3300000 0x0>; ++ }; ++ ++ vccio_sd: vccio-sd-regulator { ++ compatible = "regulator-gpio"; ++ regulator-name = "vccio_sd"; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ gpios = <&gpio0 RK_PB6 GPIO_ACTIVE_HIGH>; ++ gpios-states = <0x1>; ++ states = <1800000 0x0 ++ 3300000 0x1>; ++ }; ++ ++ vcc5v0_host: vcc5v0-host-regulator { ++ compatible = "regulator-fixed"; ++ enable-active-high; ++ gpio = <&gpio0 RK_PA6 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&vcc5v0_host_en>; ++ regulator-name = "vcc5v0_host"; ++ regulator-always-on; ++ }; ++ ++ vcc5v0_otg: vcc5v0-otg-regulator { ++ compatible = "regulator-fixed"; ++ enable-active-high; ++ gpio = <&gpio0 RK_PC6 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&vcc5v0_otg_en>; ++ regulator-name = "vcc5v0_otg"; ++ }; ++ ++ vcc_camera: vcc-camera-regulator { ++ compatible = "regulator-fixed"; ++ gpio = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&camera_pwr>; ++ regulator-name = "vcc_camera"; ++ enable-active-high; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ wireless_wlan: wireless-wlan { ++ compatible = "wlan-platdata"; ++ rockchip,grf = <&grf>; ++ wifi_chip_type = "ap6398s"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_host_wake_irq>; ++ WIFI,host_wake_irq = <&gpio2 RK_PB2 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ wireless_bluetooth: wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&pmucru CLK_RTC_32K>; ++ clock-names = "ext_clock"; ++ //wifi-bt-power-toggle; ++ uart_rts_gpios = <&gpio2 RK_PB5 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart1m0_rtsn>; ++ pinctrl-1 = <&uart1_gpios>; ++ BT,reset_gpio = <&gpio2 RK_PB7 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio2 RK_PC1 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio2 RK_PC0 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++}; ++ ++&combphy1_usq { ++ assigned-clocks = <&pmucru CLK_PCIEPHY1_REF>; ++ assigned-clock-rates = <100000000>; ++ status = "okay"; ++}; ++ ++&combphy2_psq { ++ status = "okay"; ++}; ++ ++&csi2_dphy_hw { ++ status = "okay"; ++}; ++ ++&csi2_dphy0 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam0: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&gc4c33_out>; ++ data-lanes = <1 2>; ++ }; ++ ++ mipi_in_ucam1: endpoint@2 { ++ reg = <2>; ++ remote-endpoint = <&gc8034_out>; ++ data-lanes = <1 2 3 4>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ csidphy_out: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&isp0_in>; ++ }; ++ }; ++ }; ++}; ++ ++&gmac1 { ++ phy-mode = "rgmii"; ++ clock_in_out = "input"; ++ ++ snps,reset-gpio = <&gpio4 RK_PC2 GPIO_ACTIVE_LOW>; ++ snps,reset-active-low; ++ /* Reset time is 20ms, 100ms for rtl8211f */ ++ snps,reset-delays-us = <0 20000 100000>; ++ ++ assigned-clocks = <&cru SCLK_GMAC1_RX_TX>, <&cru SCLK_GMAC1>; ++ assigned-clock-parents = <&cru SCLK_GMAC1_RGMII_SPEED>, <&gmac1_clkin>; ++ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&gmac1m1_miim ++ &gmac1m1_tx_bus2 ++ &gmac1m1_rx_bus2 ++ &gmac1m1_rgmii_clk ++ &gmac1m1_rgmii_bus ++ &gmac1m1_clkinout>; ++ ++ tx_delay = <0x4f>; ++ rx_delay = <0x2d>; ++ ++ phy-handle = <&rgmii_phy1>; ++ status = "okay"; ++}; ++ ++&i2c2{ ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2c2m1_xfer>; ++ ++ gc8034: gc8034@37 { ++ status = "okay"; ++ compatible = "galaxycore,gc8034"; ++ reg = <0x37>; ++ clocks = <&cru CLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ power-domains = <&power RK3568_PD_VI>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_clk>; ++ /*pwren-gpios = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>;*/ ++ reset-gpios = <&gpio3 RK_PD5 GPIO_ACTIVE_LOW>; ++ pwdn-gpios = <&gpio4 RK_PB2 GPIO_ACTIVE_LOW>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "RK-CMK-8M-2-v1"; ++ rockchip,camera-module-lens-name = "CK8401"; ++ port { ++ gc8034_out: endpoint { ++ remote-endpoint = <&mipi_in_ucam1>; ++ data-lanes = <1 2 3 4>; ++ }; ++ }; ++ }; ++ ++ gc4c33: gc4c33@29 { ++ status = "okay"; ++ compatible = "galaxycore,gc4c33"; ++ reg = <0x29>; ++ clocks = <&cru CLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ power-domains = <&power RK3568_PD_VI>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_clk>; ++ /*pwren-gpios = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>;*/ ++ reset-gpios = <&gpio3 RK_PD5 GPIO_ACTIVE_HIGH>; ++ pwdn-gpios = <&gpio4 RK_PB2 GPIO_ACTIVE_HIGH>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "PCORW0009A"; ++ rockchip,camera-module-lens-name = "40IRC-4M"; ++ port { ++ gc4c33_out: endpoint { ++ remote-endpoint = <&mipi_in_ucam0>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++}; ++ ++&mdio1 { ++ rgmii_phy1: phy@0 { ++ compatible = "ethernet-phy-ieee802.3-c22"; ++ reg = <0x0>; ++ }; ++}; ++ ++&pwm15 { ++ compatible = "rockchip,remotectl-pwm"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pwm15m1_pins>; ++ remote_pwm_id = <3>; ++ handle_cpu_id = <1>; ++ remote_support_psci = <0>; ++ status = "okay"; ++ ++ ir_key1 { ++ rockchip,usercode = <0x4040>; ++ rockchip,key_table = ++ <0xf2 KEY_REPLY>, ++ <0xba KEY_BACK>, ++ <0xf4 KEY_UP>, ++ <0xf1 KEY_DOWN>, ++ <0xef KEY_LEFT>, ++ <0xee KEY_RIGHT>, ++ <0xbd KEY_HOME>, ++ <0xea KEY_VOLUMEUP>, ++ <0xe3 KEY_VOLUMEDOWN>, ++ <0xe2 KEY_SEARCH>, ++ <0xb2 KEY_POWER>, ++ <0xbc KEY_MUTE>, ++ <0xec KEY_MENU>, ++ <0xbf 0x190>, ++ <0xe0 0x191>, ++ <0xe1 0x192>, ++ <0xe9 183>, ++ <0xe6 248>, ++ <0xe8 185>, ++ <0xe7 186>, ++ <0xf0 388>, ++ <0xbe 0x175>; ++ }; ++ ++ ir_key2 { ++ rockchip,usercode = <0xff00>; ++ rockchip,key_table = ++ <0xf9 KEY_HOME>, ++ <0xbf KEY_BACK>, ++ <0xfb KEY_MENU>, ++ <0xaa KEY_REPLY>, ++ <0xb9 KEY_UP>, ++ <0xe9 KEY_DOWN>, ++ <0xb8 KEY_LEFT>, ++ <0xea KEY_RIGHT>, ++ <0xeb KEY_VOLUMEDOWN>, ++ <0xef KEY_VOLUMEUP>, ++ <0xf7 KEY_MUTE>, ++ <0xe7 KEY_POWER>, ++ <0xfc KEY_POWER>, ++ <0xa9 KEY_VOLUMEDOWN>, ++ <0xa8 KEY_PLAYPAUSE>, ++ <0xe0 KEY_VOLUMEDOWN>, ++ <0xa5 KEY_VOLUMEDOWN>, ++ <0xab 183>, ++ <0xb7 388>, ++ <0xe8 388>, ++ <0xf8 184>, ++ <0xaf 185>, ++ <0xed KEY_VOLUMEDOWN>, ++ <0xee 186>, ++ <0xb3 KEY_VOLUMEDOWN>, ++ <0xf1 KEY_VOLUMEDOWN>, ++ <0xf2 KEY_VOLUMEDOWN>, ++ <0xf3 KEY_SEARCH>, ++ <0xb4 KEY_VOLUMEDOWN>, ++ <0xa4 KEY_SETUP>, ++ <0xbe KEY_SEARCH>; ++ }; ++ ++ ir_key3 { ++ rockchip,usercode = <0x1dcc>; ++ rockchip,key_table = ++ <0xee KEY_REPLY>, ++ <0xf0 KEY_BACK>, ++ <0xf8 KEY_UP>, ++ <0xbb KEY_DOWN>, ++ <0xef KEY_LEFT>, ++ <0xed KEY_RIGHT>, ++ <0xfc KEY_HOME>, ++ <0xf1 KEY_VOLUMEUP>, ++ <0xfd KEY_VOLUMEDOWN>, ++ <0xb7 KEY_SEARCH>, ++ <0xff KEY_POWER>, ++ <0xf3 KEY_MUTE>, ++ <0xbf KEY_MENU>, ++ <0xf9 0x191>, ++ <0xf5 0x192>, ++ <0xb3 388>, ++ <0xbe KEY_1>, ++ <0xba KEY_2>, ++ <0xb2 KEY_3>, ++ <0xbd KEY_4>, ++ <0xf9 KEY_5>, ++ <0xb1 KEY_6>, ++ <0xfc KEY_7>, ++ <0xf8 KEY_8>, ++ <0xb0 KEY_9>, ++ <0xb6 KEY_0>, ++ <0xb5 KEY_BACKSPACE>; ++ }; ++}; ++ ++/* Need to be modified according to the actual hardware */ ++&pmu_io_domains { ++ status = "okay"; ++ pmuio2-supply = <&vcc_3v3>; ++ vccio1-supply = <&vcc_3v3>; ++ vccio3-supply = <&vcc_3v3>; ++ vccio4-supply = <&vcc_3v3>; ++ vccio5-supply = <&vcc_3v3>; ++ vccio6-supply = <&vcc_3v3>; ++ vccio7-supply = <&vcc_3v3>; ++}; ++ ++&rkisp { ++ status = "okay"; ++}; ++ ++&rkisp_mmu { ++ status = "okay"; ++}; ++ ++&rkisp_vir0 { ++ status = "okay"; ++ ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp0_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&csidphy_out>; ++ }; ++ }; ++}; ++ ++&sata2 { ++ status = "okay"; ++}; ++ ++&sdmmc0 { ++ max-frequency = <50000000>; ++ no-sdio; ++ no-mmc; ++ bus-width = <4>; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ disable-wp; ++ //sd-uhs-sdr104; ++ vmmc-supply = <&vcc3v3_sd>; ++ vqmmc-supply = <&vccio_sd>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc0_bus4 &sdmmc0_clk &sdmmc0_cmd &sdmmc0_det>; ++ status = "okay"; ++}; ++ ++&sdmmc1 { ++ max-frequency = <150000000>; ++ no-sd; ++ no-mmc; ++ bus-width = <4>; ++ disable-wp; ++ cap-sd-highspeed; ++ cap-sdio-irq; ++ keep-power-in-suspend; ++ non-removable; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc1_bus4 &sdmmc1_cmd &sdmmc1_clk>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&uart1 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart1m0_xfer &uart1m0_ctsn>; ++}; ++ ++&u2phy0_host { ++ phy-supply = <&vcc5v0_host>; ++ status = "okay"; ++}; ++ ++&u2phy0_otg { ++ vbus-supply = <&vcc5v0_otg>; ++ status = "okay"; ++}; ++ ++&u2phy1_host { ++ phy-supply = <&vcc5v0_host>; ++ status = "okay"; ++}; ++ ++&u2phy1_otg { ++ phy-supply = <&vcc5v0_host>; ++ status = "okay"; ++}; ++ ++&usb2phy1 { ++ status = "okay"; ++}; ++ ++&usb_host0_ehci { ++ status = "okay"; ++}; ++ ++&usb_host0_ohci { ++ status = "okay"; ++}; ++ ++&usb_host1_ehci { ++ status = "okay"; ++}; ++ ++&usb_host1_ohci { ++ status = "okay"; ++}; ++ ++&pinctrl { ++ cam { ++ camera_pwr: camera-pwr { ++ rockchip,pins = ++ /* camera power en */ ++ <0 RK_PC5 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <2 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ wifi_32k: wifi-32k { ++ rockchip,pins = <2 RK_PC6 1 &pcfg_pull_none>; ++ }; ++ }; ++ ++ usb { ++ vcc5v0_host_en: vcc5v0-host-en { ++ rockchip,pins = <0 RK_PA6 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ vcc5v0_otg_en: vcc5v0-otg-en { ++ rockchip,pins = <0 RK_PC6 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ }; ++ ++ wireless-wlan { ++ wifi_host_wake_irq: wifi-host-wake-irq { ++ rockchip,pins = <2 RK_PB2 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart1_gpios: uart1-gpios { ++ rockchip,pins = <2 RK_PB5 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3566-box.dtsi b/arch/arm64/boot/dts/rockchip/rk3566-box.dtsi +new file mode 100755 +index 000000000000..f0feae6730e3 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3566-box.dtsi +@@ -0,0 +1,436 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2021 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++#include "rk3566.dtsi" ++#include ++#include ++#include ++#include ++#include ++ ++/ { ++ adc_keys: adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 0>; ++ io-channel-names = "buttons"; ++ keyup-threshold-microvolt = <1800000>; ++ poll-interval = <100>; ++ ++ vol-up-key { ++ label = "volume up"; ++ linux,code = ; ++ press-threshold-microvolt = <1750>; ++ }; ++ ++ vol-down-key { ++ label = "volume down"; ++ linux,code = ; ++ press-threshold-microvolt = <297500>; ++ }; ++ }; ++ ++ dc_12v: dc-12v { ++ compatible = "regulator-fixed"; ++ regulator-name = "dc_12v"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <12000000>; ++ regulator-max-microvolt = <12000000>; ++ }; ++ ++ hdmi_sound: hdmi-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,name = "hdmi-sound"; ++ status = "okay"; ++ ++ simple-audio-card,cpu { ++ sound-dai = <&i2s0_8ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&hdmi>; ++ }; ++ }; ++ ++ reserved-memory { ++ #address-cells = <2>; ++ #size-cells = <2>; ++ ranges; ++ ++ rknpu_reserved: rknpu { ++ compatible = "shared-dma-pool"; ++ inactive; ++ reusable; ++ size = <0x0 0x20000000>; ++ alignment = <0x0 0x1000>; ++ status = "disabled"; ++ }; ++ }; ++ ++ spdif-sound { ++ status = "okay"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,name = "ROCKCHIP,SPDIF"; ++ simple-audio-card,cpu { ++ sound-dai = <&spdif_8ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&spdif_out>; ++ }; ++ }; ++ ++ spdif_out: spdif-out { ++ status = "okay"; ++ compatible = "linux,spdif-dit"; ++ #sound-dai-cells = <0>; ++ }; ++ ++ vcc3v3_sys: vcc3v3-sys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc3v3_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ vin-supply = <&dc_12v>; ++ }; ++ ++ vcc5v0_sys: vcc5v0-sys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc5v0_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ vin-supply = <&dc_12v>; ++ }; ++ ++ vcc_1v8: vcc_1v8 { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_1v8"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ vin-supply = <&vcc5v0_sys>; ++ }; ++ ++ vcc_3v3: vcc_3v3{ ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_3v3"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ vin-supply = <&vcc5v0_sys>; ++ }; ++ ++ vdd_fixed: vdd-fixed { ++ compatible = "regulator-fixed"; ++ regulator-name = "vdd_fixed"; ++ regulator-min-microvolt = <950000>; ++ regulator-max-microvolt = <950000>; ++ regulator-always-on; ++ regulator-boot-on; ++ vin-supply = <&vcc5v0_sys>; ++ }; ++ ++ vdd_cpu: vdd-cpu { ++ compatible = "pwm-regulator"; ++ pwms = <&pwm0 0 5000 1>; ++ regulator-name = "vdd_cpu"; ++ regulator-min-microvolt = <800000>; ++ regulator-max-microvolt = <1200000>; ++ regulator-init-microvolt = <950000>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-settling-time-up-us = <250>; ++ pwm-supply = <&vcc5v0_sys>; ++ status = "okay"; ++ }; ++ ++ vdd_logic: vdd-logic { ++ compatible = "pwm-regulator"; ++ pwms = <&pwm1 0 5000 1>; ++ regulator-name = "vdd_logic"; ++ regulator-min-microvolt = <800000>; ++ regulator-max-microvolt = <1100000>; ++ regulator-init-microvolt = <950000>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-settling-time-up-us = <250>; ++ pwm-supply = <&vcc5v0_sys>; ++ status = "okay"; ++ }; ++}; ++ ++&bus_npu { ++ bus-supply = <&vdd_logic>; ++ pvtm-supply = <&vdd_cpu>; ++ status = "okay"; ++}; ++ ++&cpu0 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&dmc { ++ auto-freq-en = <0>; ++ center-supply = <&vdd_fixed>; ++ status = "okay"; ++}; ++ ++&gpu { ++ mali-supply = <&vdd_fixed>; ++ status = "okay"; ++}; ++ ++&gpu_opp_table { ++ /delete-node/ opp-800000000; ++}; ++ ++&hdmi { ++ status = "okay"; ++ rockchip,phy-table = ++ <92812500 0x8009 0x0000 0x0270>, ++ <165000000 0x800b 0x0000 0x026d>, ++ <185625000 0x800b 0x0000 0x01ed>, ++ <297000000 0x800b 0x0000 0x01ad>, ++ <594000000 0x8029 0x0000 0x0088>, ++ <000000000 0x0000 0x0000 0x0000>; ++}; ++ ++&hdmi_in_vp0 { ++ status = "okay"; ++}; ++ ++&hdmi_in_vp1 { ++ status = "disabled"; ++}; ++ ++&i2s0_8ch { ++ status = "okay"; ++}; ++ ++&i2s1_8ch { ++ status = "okay"; ++ rockchip,clk-trcm = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1m0_sclktx ++ &i2s1m0_lrcktx ++ &i2s1m0_sdi0 ++ &i2s1m0_sdo0>; ++}; ++ ++&iep { ++ status = "okay"; ++}; ++ ++&iep_mmu { ++ status = "okay"; ++}; ++ ++&jpegd { ++ status = "okay"; ++}; ++ ++&jpegd_mmu { ++ status = "okay"; ++}; ++ ++&video_phy0 { ++ status = "okay"; ++}; ++ ++&video_phy1 { ++ status = "okay"; ++}; ++ ++&mpp_srv { ++ status = "okay"; ++}; ++ ++&pwm0 { ++ status = "okay"; ++ pinctrl-names = "active"; ++}; ++ ++&pwm1 { ++ status = "okay"; ++ pinctrl-names = "active"; ++}; ++ ++&rk_rga { ++ status = "okay"; ++}; ++ ++&rknpu { ++ memory-region = <&rknpu_reserved>; ++ rknpu-supply = <&vdd_fixed>; ++ status = "okay"; ++}; ++ ++&rknpu_mmu { ++ status = "disabled"; ++}; ++ ++&rkvdec { ++ status = "okay"; ++}; ++ ++&rkvdec_mmu { ++ status = "okay"; ++}; ++ ++&rkvenc { ++ status = "okay"; ++}; ++ ++&rkvenc_mmu { ++ status = "okay"; ++}; ++ ++&rockchip_suspend { ++ status = "okay"; ++ rockchip,sleep-mode-config = < ++ (0 ++ | RKPM_SLP_CENTER_OFF ++ | RKPM_SLP_HW_PLLS_OFF ++ | RKPM_SLP_PMUALIVE_32K ++ | RKPM_SLP_PMIC_LP ++ | RKPM_SLP_32K_PVTM ++ ) ++ >; ++ rockchip,wakeup-config = < ++ (0 ++ | RKPM_PWM0_WKUP_EN ++ | RKPM_CPU0_WKUP_EN ++ ) ++ >; ++}; ++ ++&route_hdmi { ++ status = "okay"; ++ connect = <&vp0_out_hdmi>; ++}; ++ ++&saradc { ++ status = "okay"; ++ vref-supply = <&vcc_1v8>; ++}; ++ ++&sdhci { ++ bus-width = <8>; ++ no-sdio; ++ no-sd; ++ non-removable; ++ status = "okay"; ++}; ++ ++&sfc { ++ status = "okay"; ++}; ++ ++&spdif_8ch { ++ status = "okay"; ++}; ++ ++&tsadc { ++ status = "okay"; ++}; ++ ++&u2phy0_host { ++ status = "okay"; ++}; ++ ++&u2phy0_otg { ++ status = "okay"; ++}; ++ ++&usb2phy0 { ++ status = "okay"; ++}; ++ ++&u2phy1_host { ++ status = "disabled"; ++}; ++ ++&u2phy1_otg { ++ status = "disabled"; ++}; ++ ++&usb2phy1 { ++ status = "disabled"; ++}; ++ ++&usb_host0_ehci { ++ status = "disabled"; ++}; ++ ++&usb_host0_ohci { ++ status = "disabled"; ++}; ++ ++&usb_host1_ehci { ++ status = "disabled"; ++}; ++ ++&usb_host1_ohci { ++ status = "disabled"; ++}; ++ ++&usbdrd_dwc3 { ++ dr_mode = "otg"; ++ phys = <&u2phy0_otg>; ++ maximum-speed = "high-speed"; ++ extcon = <&usb2phy0>; ++ status = "okay"; ++}; ++ ++&usbdrd30 { ++ status = "okay"; ++}; ++ ++&usbhost_dwc3 { ++ status = "okay"; ++}; ++ ++&usbhost30 { ++ status = "okay"; ++}; ++ ++&vdpu { ++ status = "okay"; ++}; ++ ++&vdpu_mmu { ++ status = "okay"; ++}; ++ ++&vepu { ++ status = "okay"; ++}; ++ ++&vepu_mmu { ++ status = "okay"; ++}; ++ ++&vop { ++ status = "okay"; ++ assigned-clocks = <&cru DCLK_VOP1>; ++ assigned-clock-parents = <&cru PLL_VPLL>; ++}; ++ ++&vop_mmu { ++ status = "okay"; ++}; ++ ++ +diff --git a/arch/arm64/boot/dts/rockchip/rk3566-eink.dtsi b/arch/arm64/boot/dts/rockchip/rk3566-eink.dtsi +new file mode 100755 +index 000000000000..09791eb16abf +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3566-eink.dtsi +@@ -0,0 +1,100 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ */ ++ ++/ { ++ reserved-memory { ++ #address-cells = <2>; ++ #size-cells = <2>; ++ ranges; ++ ++ waveform_reserved: waveform@10800000 { ++ reg = <0x0 0x10800000 0x0 0x100000>; ++ }; ++ ++ display_reserved: framebuffer@10900000 { ++ reg = <0x0 0x10900000 0x0 0x2000000>; ++ }; ++ }; ++ ++ ebc_dev: ebc-dev { ++ compatible = "rockchip,ebc-dev"; ++ ebc_tcon = <&ebc>; ++ eink_tcon = <&eink>; ++ memory-region = <&display_reserved>; ++ waveform-region = <&waveform_reserved>; ++ status = "okay"; ++ }; ++}; ++ ++&cpu0_opp_table { ++ opp-216000000 { ++ opp-hz = /bits/ 64 <216000000>; ++ opp-microvolt = <825000 825000 1150000>; ++ clock-latency-ns = <40000>; ++ }; ++ opp-312000000 { ++ opp-hz = /bits/ 64 <312000000>; ++ opp-microvolt = <825000 825000 1150000>; ++ clock-latency-ns = <40000>; ++ }; ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&dmc { ++ center-supply = <&vdd_logic>; ++ auto-freq-en = <0>; ++ status = "okay"; ++}; ++ ++&dmc_opp_table { ++ opp-324000000 { ++ opp-hz = /bits/ 64 <324000000>; ++ opp-microvolt = <875000>; ++ }; ++ opp-528000000 { ++ opp-hz = /bits/ 64 <528000000>; ++ opp-microvolt = <875000>; ++ }; ++}; ++ ++&ebc { ++ status = "okay"; ++}; ++ ++&eink { ++ status = "okay"; ++}; ++ ++&gpu_opp_table { ++ opp-100000000 { ++ opp-hz = /bits/ 64 <100000000>; ++ opp-microvolt = <825000>; ++ }; ++ opp-150000000 { ++ opp-hz = /bits/ 64 <150000000>; ++ opp-microvolt = <825000>; ++ }; ++}; ++ ++&rockchip_suspend { ++ status = "okay"; ++ ++ rockchip,sleep-debug-en = <0>; ++ rockchip,sleep-mode-config = < ++ (0 ++ | RKPM_SLP_ARMOFF_LOGOFF ++ | RKPM_SLP_CENTER_OFF ++ | RKPM_SLP_HW_PLLS_OFF ++ | RKPM_SLP_PMUALIVE_32K ++ | RKPM_SLP_OSC_DIS ++ | RKPM_SLP_PMIC_LP ++ | RKPM_SLP_32K_PVTM ++ ) ++ >; ++}; ++ +diff --git a/arch/arm64/boot/dts/rockchip/rk3566-evb-mipitest-v10.dts b/arch/arm64/boot/dts/rockchip/rk3566-evb-mipitest-v10.dts +new file mode 100755 +index 000000000000..3f215884a070 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3566-evb-mipitest-v10.dts +@@ -0,0 +1,7 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ */ ++ ++#include "rk3566-evb-mipitest-v10.dtsi" ++#include "rk3568-android.dtsi" +diff --git a/arch/arm64/boot/dts/rockchip/rk3566-evb-mipitest-v10.dtsi b/arch/arm64/boot/dts/rockchip/rk3566-evb-mipitest-v10.dtsi +new file mode 100755 +index 000000000000..227bc4e39d99 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3566-evb-mipitest-v10.dtsi +@@ -0,0 +1,507 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ */ ++ ++/dts-v1/; ++ ++#include ++#include ++#include "rk3566.dtsi" ++#include "rk3566-evb.dtsi" ++ ++/ { ++ model = "Rockchip RK3566 EVB MIPITEST V10 Board"; ++ compatible = "rockchip,rk3566-evb-mipitest-v10", "rockchip,rk3566"; ++ ++ pcie20_3v3: gpio-regulator { ++ compatible = "regulator-gpio"; ++ regulator-name = "pcie20_3v3"; ++ regulator-min-microvolt = <0100000>; ++ regulator-max-microvolt = <3300000>; ++ gpios = <&gpio0 RK_PB7 GPIO_ACTIVE_HIGH>; ++ gpios-states = <0x1>; ++ states = <0100000 0x0 ++ 3300000 0x1>; ++ }; ++ ++ rk_headset: rk-headset { ++ compatible = "rockchip_headset"; ++ headset_gpio = <&gpio3 RK_PA1 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hp_det>; ++ }; ++ ++ vcc3v3_vga: vcc3v3-vga { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc3v3_vga"; ++ regulator-always-on; ++ regulator-boot-on; ++ gpio = <&gpio2 RK_PB1 GPIO_ACTIVE_HIGH>; ++ enable-active-high; ++ vin-supply = <&vcc3v3_sys>; ++ }; ++ ++ vcc_camera: vcc-camera-regulator { ++ compatible = "regulator-fixed"; ++ gpio = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&camera_pwr>; ++ regulator-name = "vcc_camera"; ++ enable-active-high; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++}; ++ ++&audiopwmout_diff { ++ status = "disabled"; ++}; ++ ++&combphy1_usq { ++ status = "okay"; ++}; ++ ++&combphy2_psq { ++ status = "okay"; ++}; ++ ++&csi2_dphy_hw { ++ status = "okay"; ++}; ++ ++&csi2_dphy1 { ++ status = "okay"; ++ ++ /* ++ * dphy1 only used for split mode, ++ * can be used concurrently with dphy2 ++ * full mode and split mode are mutually exclusive ++ */ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy1_in: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ov5695_out>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy1_out: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&isp0_in>; ++ }; ++ }; ++ }; ++}; ++ ++&csi2_dphy2 { ++ status = "okay"; ++ ++ /* ++ * dphy2 only used for split mode, ++ * can be used concurrently with dphy1 ++ * full mode and split mode are mutually exclusive ++ */ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy2_in: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ov02k10_out>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy2_out: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&mipi_csi2_input>; ++ }; ++ }; ++ }; ++}; ++ ++&dig_acodec { ++ status = "disabled"; ++ rockchip,pwm-output-mode; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&audiopwm_loutp ++ &audiopwm_loutn ++ &audiopwm_routp ++ &audiopwm_routn ++ >; ++}; ++ ++/* ++ * video_phy0 needs to be enabled ++ * when dsi0 is enabled ++ */ ++&dsi0 { ++ status = "okay"; ++}; ++ ++&dsi0_in_vp0 { ++ status = "disabled"; ++}; ++ ++&dsi0_in_vp1 { ++ status = "okay"; ++}; ++ ++&dsi0_panel { ++ power-supply = <&vcc3v3_lcd0_n>; ++ reset-gpios = <&gpio4 RK_PC5 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&lcd0_rst_gpio>; ++}; ++ ++/* ++ * video_phy1 needs to be enabled ++ * when dsi1 is enabled ++ */ ++&dsi1 { ++ status = "disabled"; ++}; ++ ++&dsi1_in_vp0 { ++ status = "disabled"; ++}; ++ ++&dsi1_in_vp1 { ++ status = "disabled"; ++}; ++ ++&dsi1_panel { ++ power-supply = <&vcc3v3_lcd1_n>; ++ reset-gpios = <&gpio4 RK_PC6 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&lcd1_rst_gpio>; ++}; ++ ++&edp { ++ hpd-gpios = <&gpio4 RK_PC4 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++}; ++ ++&edp_phy { ++ status = "okay"; ++}; ++ ++&edp_in_vp0 { ++ status = "okay"; ++}; ++ ++&edp_in_vp1 { ++ status = "disabled"; ++}; ++ ++/* ++ * power-supply should switche to vcc3v3_lcd1_n ++ * when mipi panel is connected to dsi1. ++ */ ++>1x { ++ status = "disabled"; ++ power-supply = <&vcc3v3_lcd0_n>; ++}; ++ ++&hdmi { ++ status = "disabled"; ++}; ++ ++&i2c2 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2c2m1_xfer>; ++ ++ /* split mode: lane0/1 */ ++ ov5695: ov5695@36 { ++ status = "okay"; ++ compatible = "ovti,ov5695"; ++ reg = <0x36>; ++ clocks = <&cru CLK_CAM0_OUT>; ++ clock-names = "xvclk"; ++ power-domains = <&power RK3568_PD_VI>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cam_clkout0>; ++ reset-gpios = <&gpio4 RK_PC0 GPIO_ACTIVE_HIGH>; ++ pwdn-gpios = <&gpio2 RK_PC5 GPIO_ACTIVE_HIGH>; ++ /*power-gpios = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>;*/ ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "TongJu"; ++ rockchip,camera-module-lens-name = "CHT842-MD"; ++ port { ++ ov5695_out: endpoint { ++ remote-endpoint = <&dphy1_in>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++ ++ ov02k10: ov02k10@36 { ++ status = "okay"; ++ compatible = "ovti,ov02k10"; ++ reg = <0x36>; ++ clocks = <&cru CLK_CAM1_OUT>; ++ clock-names = "xvclk"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cam_clkout1>; ++ reset-gpios = <&gpio3 RK_PB0 GPIO_ACTIVE_HIGH>; ++ pwdn-gpios = <&gpio2 RK_PC6 GPIO_ACTIVE_HIGH>; ++ power-gpios = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "TongJu"; ++ rockchip,camera-module-lens-name = "CHT842-MD"; ++ port { ++ ov02k10_out: endpoint { ++ remote-endpoint = <&dphy2_in>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++}; ++ ++&i2s3_2ch { ++ status = "disabled"; ++}; ++ ++&mipi_csi2 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_csi2_input: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&dphy2_out>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_csi2_output: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&cif_mipi_in>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++}; ++ ++&video_phy0 { ++ status = "okay"; ++}; ++ ++&video_phy1 { ++ status = "disabled"; ++}; ++ ++&pcie2x1 { ++ reset-gpios = <&gpio1 RK_PB2 GPIO_ACTIVE_HIGH>; ++ vpcie3v3-supply = <&pcie20_3v3>; ++ status = "disabled"; ++}; ++ ++&pdm { ++ status = "disabled"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pdmm1_clk1 ++ &pdmm1_sdi1 ++ &pdmm1_sdi2 ++ &pdmm1_sdi3>; ++}; ++ ++&pdmics { ++ status = "disabled"; ++}; ++ ++&pdm_mic_array { ++ status = "disabled"; ++}; ++ ++&rkcif { ++ status = "okay"; ++}; ++ ++&rkcif_mipi_lvds { ++ status = "okay"; ++ ++ port { ++ cif_mipi_in: endpoint { ++ remote-endpoint = <&mipi_csi2_output>; ++ data-lanes = <1 2>; ++ }; ++ }; ++}; ++ ++&rkcif_mmu { ++ status = "okay"; ++}; ++ ++&rkisp { ++ status = "okay"; ++}; ++ ++&rkisp_mmu { ++ status = "okay"; ++}; ++ ++&rkisp_vir0 { ++ status = "okay"; ++ ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp0_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dphy1_out>; ++ }; ++ }; ++}; ++ ++&route_dsi0 { ++ status = "okay"; ++ connect = <&vp1_out_dsi0>; ++}; ++ ++&sdmmc2 { ++ max-frequency = <150000000>; ++ no-sd; ++ no-mmc; ++ bus-width = <4>; ++ disable-wp; ++ cap-sd-highspeed; ++ cap-sdio-irq; ++ keep-power-in-suspend; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ non-removable; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc2m0_bus4 &sdmmc2m0_cmd &sdmmc2m0_clk>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&uart1 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart1m1_xfer &uart1m1_ctsn>; ++}; ++ ++&u2phy1_host { ++ status = "disabled"; ++}; ++ ++&u2phy1_otg { ++ status = "disabled"; ++}; ++ ++&usb2phy1 { ++ status = "disabled"; ++}; ++ ++&usb_host1_ohci { ++ status = "disabled"; ++}; ++ ++&vcc3v3_lcd0_n { ++ gpio = <&gpio0 RK_PC0 GPIO_ACTIVE_HIGH>; ++ enable-active-high; ++}; ++ ++&vcc3v3_lcd1_n { ++ gpio = <&gpio0 RK_PC2 GPIO_ACTIVE_HIGH>; ++ enable-active-high; ++}; ++ ++&wireless_bluetooth { ++ uart_rts_gpios = <&gpio4 RK_PB6 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart1m1_rtsn>; ++ pinctrl-1 = <&uart1_gpios>; ++ BT,reset_gpio = <&gpio4 RK_PA5 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio4 RK_PA6 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio4 RK_PB1 GPIO_ACTIVE_HIGH>; ++ status = "disabled"; ++}; ++ ++&wireless_wlan { ++ status = "disabled"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_host_wake_irq>; ++ WIFI,host_wake_irq = <&gpio3 RK_PD4 GPIO_ACTIVE_HIGH>; ++}; ++ ++&pinctrl { ++ cam { ++ camera_pwr: camera-pwr { ++ rockchip,pins = ++ /* camera power en */ ++ <0 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ headphone { ++ hp_det: hp-det { ++ rockchip,pins = <3 RK_PA1 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ lcd0 { ++ lcd0_rst_gpio: lcd0-rst-gpio { ++ rockchip,pins = <4 RK_PC5 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ lcd1 { ++ lcd1_rst_gpio: lcd1-rst-gpio { ++ rockchip,pins = <4 RK_PC6 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ wireless-wlan { ++ wifi_host_wake_irq: wifi-host-wake-irq { ++ rockchip,pins = <3 RK_PD4 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart1_gpios: uart1-gpios { ++ rockchip,pins = <4 RK_PB6 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3566-evb.dtsi b/arch/arm64/boot/dts/rockchip/rk3566-evb.dtsi +new file mode 100755 +index 000000000000..d1aa123cd347 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3566-evb.dtsi +@@ -0,0 +1,7 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++#include "rk3568-evb.dtsi" +diff --git a/arch/arm64/boot/dts/rockchip/rk3566-evb1-ddr4-v10-linux.dts b/arch/arm64/boot/dts/rockchip/rk3566-evb1-ddr4-v10-linux.dts +new file mode 100755 +index 000000000000..a22cc8cec6af +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3566-evb1-ddr4-v10-linux.dts +@@ -0,0 +1,13 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++#include "rk3566-evb1-ddr4-v10.dtsi" ++#include "rk3568-linux.dtsi" ++ ++/ { ++ model = "Rockchip RK3566 EVB1 DDR4 V10 Linux Board"; ++ compatible = "rockchip,rk3566-evb1-ddr4-v10-linux", "rockchip,rk3566"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3566-evb1-ddr4-v10-lvds.dts b/arch/arm64/boot/dts/rockchip/rk3566-evb1-ddr4-v10-lvds.dts +new file mode 100755 +index 000000000000..98a384041b8b +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3566-evb1-ddr4-v10-lvds.dts +@@ -0,0 +1,99 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ */ ++ ++#include ++#include "rk3566-evb1-ddr4-v10.dtsi" ++#include "rk3568-android.dtsi" ++ ++/ { ++ panel { ++ compatible = "simple-panel"; ++ backlight = <&backlight>; ++ power-supply = <&vcc3v3_lcd1_n>; ++ enable-delay-ms = <20>; ++ prepare-delay-ms = <20>; ++ unprepare-delay-ms = <20>; ++ disable-delay-ms = <20>; ++ bus-format = ; ++ width-mm = <217>; ++ height-mm = <136>; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <68000000>; ++ hactive = <800>; ++ vactive = <1280>; ++ hback-porch = <30>; ++ hfront-porch = <30>; ++ vback-porch = <4>; ++ vfront-porch = <2>; ++ hsync-len = <4>; ++ vsync-len = <2>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ dual-lvds-even-pixels; ++ panel_in_lvds: endpoint { ++ remote-endpoint = <&lvds_out_panel>; ++ }; ++ }; ++ }; ++ }; ++}; ++ ++&dsi0 { ++ status = "disabled"; ++}; ++ ++&dsi0_in_vp0 { ++ status = "disabled"; ++}; ++ ++&dsi0_in_vp1 { ++ status = "disabled"; ++}; ++ ++&video_phy0 { ++ status = "okay"; ++}; ++ ++&lvds { ++ status = "okay"; ++ ++ ports { ++ port@1 { ++ reg = <1>; ++ ++ lvds_out_panel: endpoint { ++ remote-endpoint = <&panel_in_lvds>; ++ }; ++ }; ++ }; ++}; ++ ++&lvds_in_vp1 { ++ status = "okay"; ++}; ++ ++&lvds_in_vp2 { ++ status = "disabled"; ++}; ++ ++&route_lvds { ++ status = "okay"; ++ connect = <&vp1_out_lvds>; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3566-evb1-ddr4-v10.dts b/arch/arm64/boot/dts/rockchip/rk3566-evb1-ddr4-v10.dts +new file mode 100755 +index 000000000000..f602ed98d7ce +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3566-evb1-ddr4-v10.dts +@@ -0,0 +1,7 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ */ ++ ++#include "rk3566-evb1-ddr4-v10.dtsi" ++#include "rk3568-android.dtsi" +diff --git a/arch/arm64/boot/dts/rockchip/rk3566-evb1-ddr4-v10.dtsi b/arch/arm64/boot/dts/rockchip/rk3566-evb1-ddr4-v10.dtsi +new file mode 100755 +index 000000000000..385933be9f45 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3566-evb1-ddr4-v10.dtsi +@@ -0,0 +1,489 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ */ ++ ++/dts-v1/; ++ ++#include ++#include ++#include "rk3566.dtsi" ++#include "rk3566-evb.dtsi" ++ ++/ { ++ model = "Rockchip RK3566 EVB1 DDR4 V10 Board"; ++ compatible = "rockchip,rk3566-evb1-ddr4-v10", "rockchip,rk3566"; ++ ++ pcie20_3v3: gpio-regulator { ++ compatible = "regulator-gpio"; ++ regulator-name = "pcie20_3v3"; ++ regulator-min-microvolt = <100000>; ++ regulator-max-microvolt = <3300000>; ++ gpios = <&gpio0 RK_PB7 GPIO_ACTIVE_HIGH>; ++ gpios-states = <0x1>; ++ states = <100000 0x0 ++ 3300000 0x1>; ++ }; ++ ++ rk_headset: rk-headset { ++ compatible = "rockchip_headset"; ++ headset_gpio = <&gpio3 RK_PA1 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hp_det>; ++ }; ++ ++ vcc3v3_vga: vcc3v3-vga { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc3v3_vga"; ++ regulator-always-on; ++ regulator-boot-on; ++ gpio = <&gpio2 RK_PB1 GPIO_ACTIVE_HIGH>; ++ enable-active-high; ++ vin-supply = <&vcc3v3_sys>; ++ }; ++ ++ vcc_camera: vcc-camera-regulator { ++ compatible = "regulator-fixed"; ++ gpio = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&camera_pwr>; ++ regulator-name = "vcc_camera"; ++ enable-active-high; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++}; ++ ++&audiopwmout_diff { ++ status = "disabled"; ++}; ++ ++&combphy1_usq { ++ status = "okay"; ++}; ++ ++&combphy2_psq { ++ status = "okay"; ++}; ++ ++&csi2_dphy_hw { ++ status = "okay"; ++}; ++ ++&csi2_dphy1 { ++ status = "okay"; ++ ++ /* ++ * dphy1 only used for split mode, ++ * can be used concurrently with dphy2 ++ * full mode and split mode are mutually exclusive ++ */ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy1_in: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ov5695_out>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy1_out: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&isp0_in>; ++ }; ++ }; ++ }; ++}; ++ ++&csi2_dphy2 { ++ status = "okay"; ++ ++ /* ++ * dphy2 only used for split mode, ++ * can be used concurrently with dphy1 ++ * full mode and split mode are mutually exclusive ++ */ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy2_in: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ov02k10_out>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy2_out: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&mipi_csi2_input>; ++ }; ++ }; ++ }; ++}; ++ ++&dig_acodec { ++ status = "disabled"; ++ rockchip,pwm-output-mode; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&audiopwm_loutp ++ &audiopwm_loutn ++ &audiopwm_routp ++ &audiopwm_routn ++ >; ++}; ++ ++/* ++ * video_phy0 needs to be enabled ++ * when dsi0 is enabled ++ */ ++&dsi0 { ++ status = "okay"; ++}; ++ ++&dsi0_in_vp0 { ++ status = "disabled"; ++}; ++ ++&dsi0_in_vp1 { ++ status = "okay"; ++}; ++ ++&dsi0_panel { ++ power-supply = <&vcc3v3_lcd0_n>; ++ reset-gpios = <&gpio4 RK_PC5 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&lcd0_rst_gpio>; ++}; ++ ++/* ++ * video_phy1 needs to be enabled ++ * when dsi1 is enabled ++ */ ++&dsi1 { ++ status = "disabled"; ++}; ++ ++&dsi1_in_vp0 { ++ status = "disabled"; ++}; ++ ++&dsi1_in_vp1 { ++ status = "disabled"; ++}; ++ ++&dsi1_panel { ++ power-supply = <&vcc3v3_lcd1_n>; ++ reset-gpios = <&gpio4 RK_PC6 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&lcd1_rst_gpio>; ++}; ++ ++&edp { ++ hpd-gpios = <&gpio4 RK_PC4 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++}; ++ ++&edp_phy { ++ status = "okay"; ++}; ++ ++&edp_in_vp0 { ++ status = "okay"; ++}; ++ ++&edp_in_vp1 { ++ status = "disabled"; ++}; ++ ++/* ++ * power-supply should switche to vcc3v3_lcd1_n ++ * when mipi panel is connected to dsi1. ++ */ ++>1x { ++ power-supply = <&vcc3v3_lcd0_n>; ++}; ++ ++&i2c2 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2c2m1_xfer>; ++ ++ /* split mode: lane0/1 */ ++ ov5695: ov5695@36 { ++ status = "okay"; ++ compatible = "ovti,ov5695"; ++ reg = <0x36>; ++ clocks = <&cru CLK_CAM0_OUT>; ++ clock-names = "xvclk"; ++ power-domains = <&power RK3568_PD_VI>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cam_clkout0>; ++ reset-gpios = <&gpio4 RK_PC0 GPIO_ACTIVE_HIGH>; ++ pwdn-gpios = <&gpio2 RK_PC5 GPIO_ACTIVE_HIGH>; ++ /*power-gpios = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>;*/ ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "TongJu"; ++ rockchip,camera-module-lens-name = "CHT842-MD"; ++ port { ++ ov5695_out: endpoint { ++ remote-endpoint = <&dphy1_in>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++ ++ ov02k10: ov02k10@36 { ++ status = "okay"; ++ compatible = "ovti,ov02k10"; ++ reg = <0x36>; ++ clocks = <&cru CLK_CAM1_OUT>; ++ clock-names = "xvclk"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cam_clkout1>; ++ reset-gpios = <&gpio3 RK_PB0 GPIO_ACTIVE_HIGH>; ++ pwdn-gpios = <&gpio2 RK_PC6 GPIO_ACTIVE_HIGH>; ++ power-gpios = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "TongJu"; ++ rockchip,camera-module-lens-name = "CHT842-MD"; ++ port { ++ ov02k10_out: endpoint { ++ remote-endpoint = <&dphy2_in>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++}; ++ ++&i2s3_2ch { ++ status = "disabled"; ++}; ++ ++&mipi_csi2 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_csi2_input: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&dphy2_out>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_csi2_output: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&cif_mipi_in>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++}; ++ ++&video_phy0 { ++ status = "okay"; ++}; ++ ++&video_phy1 { ++ status = "disabled"; ++}; ++ ++&pcie2x1 { ++ reset-gpios = <&gpio1 RK_PB2 GPIO_ACTIVE_HIGH>; ++ vpcie3v3-supply = <&pcie20_3v3>; ++ status = "okay"; ++}; ++ ++&pdm { ++ status = "disabled"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pdmm1_clk1 ++ &pdmm1_sdi1 ++ &pdmm1_sdi2 ++ &pdmm1_sdi3>; ++}; ++ ++&pdmics { ++ status = "disabled"; ++}; ++ ++&pdm_mic_array { ++ status = "disabled"; ++}; ++ ++&rkcif { ++ status = "okay"; ++}; ++ ++&rkcif_mipi_lvds { ++ status = "okay"; ++ ++ port { ++ cif_mipi_in: endpoint { ++ remote-endpoint = <&mipi_csi2_output>; ++ data-lanes = <1 2>; ++ }; ++ }; ++}; ++ ++&rkcif_mmu { ++ status = "okay"; ++}; ++ ++&rkisp { ++ status = "okay"; ++}; ++ ++&rkisp_mmu { ++ status = "okay"; ++}; ++ ++&rkisp_vir0 { ++ status = "okay"; ++ ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp0_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dphy1_out>; ++ }; ++ }; ++}; ++ ++&route_dsi0 { ++ status = "okay"; ++ connect = <&vp1_out_dsi0>; ++}; ++ ++&sdmmc2 { ++ max-frequency = <150000000>; ++ no-sd; ++ no-mmc; ++ bus-width = <4>; ++ disable-wp; ++ cap-sd-highspeed; ++ cap-sdio-irq; ++ keep-power-in-suspend; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ non-removable; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc2m0_bus4 &sdmmc2m0_cmd &sdmmc2m0_clk>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&uart1 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart1m1_xfer &uart1m1_ctsn>; ++}; ++ ++&vcc3v3_lcd0_n { ++ gpio = <&gpio0 RK_PC0 GPIO_ACTIVE_HIGH>; ++ enable-active-high; ++}; ++ ++&vcc3v3_lcd1_n { ++ gpio = <&gpio0 RK_PC2 GPIO_ACTIVE_HIGH>; ++ enable-active-high; ++}; ++ ++&wireless_bluetooth { ++ uart_rts_gpios = <&gpio4 RK_PB6 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart1m1_rtsn>; ++ pinctrl-1 = <&uart1_gpios>; ++ BT,reset_gpio = <&gpio4 RK_PA5 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio4 RK_PA6 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio4 RK_PB1 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++}; ++ ++&wireless_wlan { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_host_wake_irq>; ++ WIFI,host_wake_irq = <&gpio3 RK_PD4 GPIO_ACTIVE_HIGH>; ++}; ++ ++&work_led { ++ gpios = <&gpio2 RK_PB2 GPIO_ACTIVE_HIGH>; ++}; ++ ++&pinctrl { ++ cam { ++ camera_pwr: camera-pwr { ++ rockchip,pins = ++ /* camera power en */ ++ <0 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ headphone { ++ hp_det: hp-det { ++ rockchip,pins = <3 RK_PA1 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ lcd0 { ++ lcd0_rst_gpio: lcd0-rst-gpio { ++ rockchip,pins = <4 RK_PC5 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ lcd1 { ++ lcd1_rst_gpio: lcd1-rst-gpio { ++ rockchip,pins = <4 RK_PC6 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ wireless-wlan { ++ wifi_host_wake_irq: wifi-host-wake-irq { ++ rockchip,pins = <3 RK_PD4 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart1_gpios: uart1-gpios { ++ rockchip,pins = <4 RK_PB6 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3566-evb2-lp4x-v10-eink.dts b/arch/arm64/boot/dts/rockchip/rk3566-evb2-lp4x-v10-eink.dts +new file mode 100755 +index 000000000000..83546eef6529 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3566-evb2-lp4x-v10-eink.dts +@@ -0,0 +1,345 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++#include "rk3566-evb2-lp4x-v10.dtsi" ++#include "rk3568-android.dtsi" ++#include "rk3566-eink.dtsi" ++ ++/ { ++ model = "Rockchip RK3566 EVB2 LP4X V10 Eink Board"; ++ compatible = "rockchip,rk3566-evb2-lp4x-v10-eink", "rockchip,rk3566"; ++}; ++ ++&backlight { ++ status = "disabled"; ++}; ++ ++&backlight1 { ++ status = "disabled"; ++}; ++ ++&dsi0 { ++ status = "disabled"; ++}; ++ ++&dsi0_in_vp0 { ++ status = "disabled"; ++}; ++ ++&dsi0_panel { ++ status = "disabled"; ++}; ++ ++&dsi1_panel { ++ status = "disabled"; ++}; ++ ++&ebc { ++ /* clock rate 1000M/n, (n=1~32) */ ++ assigned-clocks = <&cru CPLL_333M>, <&cru DCLK_EBC>; ++ //assigned-clock-rates = <340000000>, <340000000>; ++ assigned-clock-rates = <250000000>, <250000000>; ++ //assigned-clock-rates = <100000000>, <100000000>; ++ status = "okay"; ++}; ++ ++&ebc_dev { ++ pmic = <&tps65185>; ++ status = "okay"; ++#if 0 ++ /* ED097TC2U1 */ ++ panel,width = <1200>; ++ panel,height = <825>; ++ panel,vir_width = <1200>; ++ panel,vir_height = <825>; ++ panel,sdck = <25000000>; ++ panel,lsl = <4>; ++ panel,lbl = <4>; ++ panel,ldl = <300>; ++ panel,lel = <36>; ++ panel,gdck-sta = <18>; ++ panel,lgonl = <265>; ++ panel,fsl = <2>; ++ panel,fbl = <4>; ++ panel,fdl = <825>; ++ panel,fel = <24>; ++ panel,mirror = <0>; ++ panel,panel_16bit = <0>; ++ panel,panel_color = <0>; ++ panel,width-mm = <203>; ++ panel,height-mm = <140>; ++#endif ++#if 1 ++ /* ES103TC1 */ ++ panel,width = <1872>; ++ panel,height = <1404>; ++ panel,vir_width = <1872>; ++ panel,vir_height = <1404>; ++ panel,sdck = <33300000>; ++ panel,lsl = <18>; ++ panel,lbl = <17>; ++ panel,ldl = <234>; ++ panel,lel = <7>; ++ panel,gdck-sta = <34>; ++ panel,lgonl = <192>; ++ panel,fsl = <1>; ++ panel,fbl = <4>; ++ panel,fdl = <1404>; ++ panel,fel = <12>; ++ panel,mirror = <0>; ++ panel,panel_16bit = <1>; ++ panel,panel_color = <0>; ++ panel,width-mm = <157>; ++ panel,height-mm = <210>; ++#endif ++#if 0 ++ /* ES133TC1 */ ++ panel,width = <2200>; ++ panel,height = <1650>; ++ panel,vir_width = <2208>; ++ panel,vir_height = <1650>; ++ panel,sdck = <37500000>; ++ panel,lsl = <4>; ++ panel,lbl = <8>; ++ panel,ldl = <275>; ++ panel,lel = <14>; ++ panel,gdck-sta = <34>; ++ panel,lgonl = <217>; ++ panel,fsl = <1>; ++ panel,fbl = <4>; ++ panel,fdl = <1650>; ++ panel,fel = <6>; ++ panel,mirror = <0>; ++ panel,panel_16bit = <1>; ++ panel,panel_color = <0>; ++ panel,width-mm = <157>; ++ panel,height-mm = <210>; ++#endif ++#if 0 ++ panel,width = <2232>; ++ panel,height = <1680>; ++ panel,vir_width = <2240>; ++ panel,vir_height = <1680>; ++ panel,sdck = <33300000>; ++ panel,lsl = <4>; ++ panel,lbl = <8>; ++ panel,ldl = <279>; ++ panel,lel = <14>; ++ panel,gdck-sta = <34>; ++ panel,lgonl = <217>; ++ panel,fsl = <1>; ++ panel,fbl = <4>; ++ panel,fdl = <1680>; ++ panel,fel = <6>; ++ panel,mirror = <0>; ++ panel,panel_16bit = <1>; ++ panel,panel_color = <0>; ++ panel,width-mm = <157>; ++ panel,height-mm = <210>; ++#endif ++}; ++ ++&gmac1 { ++ status = "disabled"; ++}; ++ ++>1x { ++ status = "disabled"; ++}; ++ ++&hdmi { ++ status = "disabled"; ++}; ++ ++&hdmi_in_vp0 { ++ status = "disabled"; ++}; ++ ++&hdmi_sound{ ++ status = "disabled"; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ ++ tsc@24 { ++ status = "okay"; ++ compatible = "cy,cyttsp5_i2c_adapter"; ++ reg = <0x24>; ++ cy,adapter_id = "cyttsp5_i2c_adapter"; ++ //cytp-supply = <&vcc_sd>; ++ cy,core { ++ cy,name = "cyttsp5_core"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&tsc_gpio>; ++ cy,irq_gpio = <&gpio0 RK_PB5 GPIO_ACTIVE_HIGH>; ++ cy,rst_gpio = <&gpio0 RK_PB6 GPIO_ACTIVE_HIGH>; ++ cy,hid_desc_register = <1>; ++ /* CY_CORE_FLAG_RESTORE_PARAMETERS */ ++ cy,flags = <6>; ++ /* CY_CORE_EWG_NONE */ ++ cy,easy_wakeup_gesture = <0>; ++ cy,btn_keys = <172 /* KEY_HOMEPAGE */ ++ /* previously was KEY_HOME, new Android versions use KEY_HOMEPAGE */ ++ 139 /* KEY_MENU */ ++ 158 /* KEY_BACK */ ++ 217 /* KEY_SEARCH */ ++ 114 /* KEY_VOLUMEDOWN */ ++ 115 /* KEY_VOLUMEUP */ ++ 212 /* KEY_CAMERA */ ++ 116>; /* KEY_POWER */ ++ cy,btn_keys-tag = <0>; ++ cy,mt { ++ cy,name = "cyttsp5_mt"; ++ cy,inp_dev_name = "cyttsp5_mt"; ++ cy,flags = <0>; ++ cy,abs = ++ /* ABS_MT_POSITION_X, CY_ABS_MIN_X, CY_ABS_MAX_X, 0, 0 */ ++ <0x35 0 1872 0 0 ++ /* ABS_MT_POSITION_Y, CY_ABS_MIN_Y, CY_ABS_MAX_Y, 0, 0 */ ++ 0x36 0 1404 0 0 ++ /* ABS_MT_PRESSURE, CY_ABS_MIN_P, CY_ABS_MAX_P, 0, 0 */ ++ 0x3a 0 255 0 0 ++ /* CY_IGNORE_VALUE, CY_ABS_MIN_W, CY_ABS_MAX_W, 0, 0 */ ++ 0xffff 0 255 0 0 ++ /* ABS_MT_TRACKING_ID, CY_ABS_MIN_T, CY_ABS_MAX_T, 0, 0 */ ++ 0x39 0 15 0 0 ++ /* ABS_MT_TOUCH_MAJOR, 0, 255, 0, 0 */ ++ 0x30 0 255 0 0 ++ /* ABS_MT_TOUCH_MINOR, 0, 255, 0, 0 */ ++ 0x31 0 255 0 0 ++ /* ABS_MT_ORIENTATION, -127, 127, 0, 0 */ ++ 0x34 0xffffff81 127 0 0 ++ /* ABS_MT_TOOL_TYPE, 0, MT_TOOL_MAX, 0, 0 */ ++ 0x37 0 1 0 0 ++ /* ABS_DISTANCE, 0, 255, 0, 0 */ ++ 0x19 0 255 0 0>; ++ ++ cy,vkeys_x = <1872>; ++ cy,vkeys_y = <1404>; ++ ++ cy,revert_x = <0>; ++ cy,revert_y = <0>; ++ cy,xy_exchange = <0>; ++ ++ cy,virtual_keys = /* KeyCode CenterX CenterY Width Height */ ++ /* KEY_BACK */ ++ <158 1360 90 160 180 ++ /* KEY_MENU */ ++ 139 1360 270 160 180 ++ /* KEY_HOMEPAGE */ ++ 172 1360 450 160 180 ++ /* KEY SEARCH */ ++ 217 1360 630 160 180>; ++ }; ++ ++ cy,btn { ++ cy,name = "cyttsp5_btn"; ++ cy,inp_dev_name = "cyttsp5_btn"; ++ }; ++ ++ cy,proximity { ++ cy,name = "cyttsp5_proximity"; ++ cy,inp_dev_name = "cyttsp5_proximity"; ++ cy,abs = ++ /* ABS_DISTANCE, CY_PROXIMITY_MIN_VAL, CY_PROXIMITY_MAX_VAL, 0, 0 */ ++ <0x19 0 1 0 0>; ++ }; ++ }; ++ }; ++ ++ tps65185: tps65185@68 { ++ status = "okay"; ++ compatible = "ti,tps65185"; ++ reg = <0x68>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&tps65185_gpio>; ++ int-gpios = <&gpio3 RK_PA7 GPIO_ACTIVE_HIGH>; ++ wakeup-gpios = <&gpio3 RK_PC3 GPIO_ACTIVE_HIGH>; ++ vcomctl-gpios = <&gpio4 RK_PB2 GPIO_ACTIVE_HIGH>; ++ powerup-gpios = <&gpio4 RK_PC5 GPIO_ACTIVE_HIGH>; ++ }; ++}; ++ ++&i2c2 { ++ status = "disabled"; ++}; ++ ++&i2c3 { ++ status = "okay"; ++ ++ wacom: wacom@9 { ++ compatible = "wacom,w9013"; ++ reg = <0x09>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wacom_gpio>; ++ gpio_detect = <&gpio4 RK_PC4 GPIO_ACTIVE_HIGH>; ++ gpio_intr = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; ++ gpio_rst = <&gpio4 RK_PB1 GPIO_ACTIVE_HIGH>; ++ revert_x = <0>; ++ revert_y = <0>; ++ xy_exchange = <0>; ++ }; ++}; ++ ++&video_phy0 { ++ status = "disabled"; ++}; ++ ++&mxc6655xa { ++ status = "disabled"; ++}; ++ ++&pinctrl { ++ tps_pmic { ++ tps65185_gpio: tps65185-gpio { ++ rockchip,pins = ++ <4 RK_PB2 RK_FUNC_GPIO &pcfg_pull_up>, ++ <4 RK_PC5 RK_FUNC_GPIO &pcfg_pull_up>, ++ <3 RK_PA7 RK_FUNC_GPIO &pcfg_pull_up>, ++ <3 RK_PC3 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ tsc { ++ tsc_gpio: tsc-gpio { ++ rockchip,pins = ++ <0 RK_PB5 RK_FUNC_GPIO &pcfg_pull_up>, ++ <0 RK_PB6 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ wacom { ++ wacom_gpio: wacom-gpio { ++ rockchip,pins = ++ <4 RK_PC4 RK_FUNC_GPIO &pcfg_pull_up>, ++ <0 RK_PC1 RK_FUNC_GPIO &pcfg_pull_up>, ++ <4 RK_PB1 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ pmuio1-supply = <&vcc3v3_pmu>; ++ pmuio2-supply = <&vcc3v3_pmu>; ++ vccio1-supply = <&vccio_acodec>; ++ vccio3-supply = <&vccio_sd>; ++ vccio4-supply = <&vcc_1v8>; ++ vccio5-supply = <&vcc_3v3>; ++ vccio6-supply = <&vcc_3v3>; ++ vccio7-supply = <&vcc_3v3>; ++}; ++ ++&vcc_camera { ++ status = "disabled"; ++}; ++ ++&wireless_bluetooth { ++ status = "disabled"; ++}; ++ ++&wireless_wlan { ++ status = "disabled"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3566-evb2-lp4x-v10-i2s-mic-array.dts b/arch/arm64/boot/dts/rockchip/rk3566-evb2-lp4x-v10-i2s-mic-array.dts +new file mode 100755 +index 000000000000..5ba1318bcee1 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3566-evb2-lp4x-v10-i2s-mic-array.dts +@@ -0,0 +1,102 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++#include "rk3566-evb2-lp4x-v10.dtsi" ++#include "rk3568-android.dtsi" ++ ++/ { ++ model = "Rockchip RK3566 EVB2 LP4X V10 Board I2S Mic Array"; ++ compatible = "rockchip,rk3566-evb2-lp4x-v10", "rockchip,rk3566"; ++ ++ rk809_sound_micarray: rk809-sound-micarray { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,rk809-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ ++ simple-audio-card,dai-link@0 { ++ format = "i2s"; ++ cpu { ++ sound-dai = <&i2s1_8ch>; ++ }; ++ codec { ++ sound-dai = <&rk809_codec 0>; ++ }; ++ }; ++ simple-audio-card,dai-link@1 { ++ format = "i2s"; ++ cpu { ++ sound-dai = <&i2s1_8ch>; ++ }; ++ codec { ++ sound-dai = <&es7243e>; ++ }; ++ }; ++ }; ++}; ++ ++&i2c3 { ++ status = "okay"; ++ ++ es7243e: es7243e@10 { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++ compatible = "ES7243E_MicArray_0"; ++ reg = <0x10>; ++ }; ++ ++ es7243e_11: es7243e@11 { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++ compatible = "ES7243E_MicArray_1"; ++ reg = <0x11>; ++ }; ++ ++ es7243e_12: es7243e@12 { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++ compatible = "ES7243E_MicArray_2"; ++ reg = <0x12>; ++ }; ++}; ++ ++&i2s1_8ch { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++ rockchip,clk-trcm = <0>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1m0_sclktx ++ &i2s1m0_lrcktx ++ &i2s1m0_sclkrx ++ &i2s1m0_lrckrx ++ &i2s1m0_sdo0 ++ &i2s1m0_sdi0 ++ &i2s1m0_sdi1 ++ &i2s1m0_sdi2 ++ &i2s1m0_sdi3>; ++}; ++ ++&rk809_codec { ++ #sound-dai-cells = <1>; ++ compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; ++ clocks = <&cru I2S1_MCLKOUT>; ++ clock-names = "mclk"; ++ assigned-clocks = <&cru I2S1_MCLKOUT>, <&cru I2S1_MCLK_TX_IOE>; ++ assigned-clock-rates = <12288000>; ++ assigned-clock-parents = <&cru I2S1_MCLKOUT_TX>, <&cru I2S1_MCLKOUT_TX>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1m0_mclk>; ++ pdmdata-out-enable; ++ adc-for-loopback; ++ hp-volume = <20>; ++ spk-volume = <3>; ++ mic-in-differential; ++ status = "okay"; ++}; ++ ++&rk809_sound { ++ status = "disabled"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3566-evb2-lp4x-v10-linux.dts b/arch/arm64/boot/dts/rockchip/rk3566-evb2-lp4x-v10-linux.dts +new file mode 100755 +index 000000000000..957a99b168bb +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3566-evb2-lp4x-v10-linux.dts +@@ -0,0 +1,8 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++#include "rk3566-evb2-lp4x-v10.dtsi" ++#include "rk3568-linux.dtsi" +diff --git a/arch/arm64/boot/dts/rockchip/rk3566-evb2-lp4x-v10-pdm-mic-array.dts b/arch/arm64/boot/dts/rockchip/rk3566-evb2-lp4x-v10-pdm-mic-array.dts +new file mode 100755 +index 000000000000..55e9679194cb +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3566-evb2-lp4x-v10-pdm-mic-array.dts +@@ -0,0 +1,111 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++#include "rk3566-evb2-lp4x-v10.dtsi" ++#include "rk3568-android.dtsi" ++ ++/ { ++ model = "Rockchip RK3566 EVB2 LP4X V10 Board PDM Mic Array"; ++ compatible = "rockchip,rk3566-evb2-lp4x-v10", "rockchip,rk3566"; ++ ++ rk809_sound_micarray: rk809-sound-micarray { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,rk809-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ ++ simple-audio-card,dai-link@0 { ++ format = "i2s"; ++ cpu { ++ sound-dai = <&i2s1_8ch>; ++ }; ++ codec { ++ sound-dai = <&rk809_codec 0>; ++ }; ++ }; ++ simple-audio-card,dai-link@1 { ++ format = "pdm"; ++ cpu { ++ sound-dai = <&pdm>; ++ }; ++ codec { ++ sound-dai = <&rk809_codec 1>; ++ }; ++ }; ++ simple-audio-card,dai-link@2 { ++ format = "pdm"; ++ cpu { ++ sound-dai = <&pdm>; ++ }; ++ codec { ++ sound-dai = <&es7202>; ++ }; ++ }; ++ }; ++}; ++ ++&i2c3 { ++ status = "okay"; ++ ++ es7202: es7202@30 { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++ compatible = "ES7202_PDM_ADC_1"; ++ reg = <0x30>; ++ }; ++ ++ es7202_31: es7202@31 { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++ compatible = "ES7202_PDM_ADC_2"; ++ reg = <0x31>; ++ }; ++}; ++ ++&i2s1_8ch { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++ rockchip,clk-trcm = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1m0_sclktx ++ &i2s1m0_lrcktx ++ &i2s1m0_sdo0>; ++}; ++ ++&pdm { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pdmm0_clk ++ &pdmm0_clk1 ++ &pdmm0_sdi0 ++ &pdmm0_sdi1 ++ &pdmm0_sdi2 ++ &pdmm0_sdi3>; ++}; ++ ++&rk809_codec { ++ #sound-dai-cells = <1>; ++ compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; ++ clocks = <&cru I2S1_MCLKOUT>; ++ clock-names = "mclk"; ++ assigned-clocks = <&cru I2S1_MCLKOUT>, <&cru I2S1_MCLK_TX_IOE>; ++ assigned-clock-rates = <12288000>; ++ assigned-clock-parents = <&cru I2S1_MCLKOUT_TX>, <&cru I2S1_MCLKOUT_TX>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1m0_mclk>; ++ pdmdata-out-enable; ++ adc-for-loopback; ++ hp-volume = <20>; ++ spk-volume = <3>; ++ mic-in-differential; ++ status = "okay"; ++}; ++ ++&rk809_sound { ++ status = "disabled"; ++}; ++ +diff --git a/arch/arm64/boot/dts/rockchip/rk3566-evb2-lp4x-v10.dts b/arch/arm64/boot/dts/rockchip/rk3566-evb2-lp4x-v10.dts +new file mode 100755 +index 000000000000..3b36bdba12b6 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3566-evb2-lp4x-v10.dts +@@ -0,0 +1,8 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++#include "rk3566-evb2-lp4x-v10.dtsi" ++#include "rk3568-android.dtsi" +diff --git a/arch/arm64/boot/dts/rockchip/rk3566-evb2-lp4x-v10.dtsi b/arch/arm64/boot/dts/rockchip/rk3566-evb2-lp4x-v10.dtsi +new file mode 100755 +index 000000000000..4a6bec70f33e +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3566-evb2-lp4x-v10.dtsi +@@ -0,0 +1,599 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++/dts-v1/; ++ ++#include ++#include ++#include ++#include "rk3566.dtsi" ++#include "rk3566-evb.dtsi" ++ ++/ { ++ model = "Rockchip RK3566 EVB2 LP4X V10 Board"; ++ compatible = "rockchip,rk3566-evb2-lp4x-v10", "rockchip,rk3566"; ++ ++ pcie20_3v3: gpio-regulator { ++ compatible = "regulator-gpio"; ++ regulator-name = "pcie20_3v3"; ++ regulator-min-microvolt = <100000>; ++ regulator-max-microvolt = <3300000>; ++ gpios = <&gpio0 RK_PC2 GPIO_ACTIVE_HIGH>; ++ gpios-states = <0x1>; ++ states = <100000 0x0 ++ 3300000 0x1>; ++ }; ++ ++ rk_headset: rk-headset { ++ compatible = "rockchip_headset"; ++ headset_gpio = <&gpio3 RK_PA1 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hp_det>; ++ }; ++ ++ vcc_camera: vcc-camera-regulator { ++ compatible = "regulator-fixed"; ++ gpio = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&camera_pwr>; ++ regulator-name = "vcc_camera"; ++ enable-active-high; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++}; ++ ++&combphy1_usq { ++ status = "okay"; ++}; ++ ++&combphy2_psq { ++ status = "okay"; ++}; ++ ++&csi2_dphy_hw { ++ status = "okay"; ++}; ++ ++&csi2_dphy0 { ++ status = "disabled"; ++ /* ++ * dphy0 only used for full mode, ++ * full mode and split mode are mutually exclusive ++ */ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy0_in: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&gc8034_out>; ++ data-lanes = <1 2 3 4>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy0_out: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&isp0_in>; ++ }; ++ }; ++ }; ++}; ++ ++&csi2_dphy1 { ++ status = "okay"; ++ ++ /* ++ * dphy1 only used for split mode, ++ * can be used concurrently with dphy2 ++ * full mode and split mode are mutually exclusive ++ */ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy1_in: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ov5695_out>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy1_out: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&isp0_in>; ++ }; ++ }; ++ }; ++}; ++ ++&csi2_dphy2 { ++ status = "okay"; ++ ++ /* ++ * dphy2 only used for split mode, ++ * can be used concurrently with dphy1 ++ * full mode and split mode are mutually exclusive ++ */ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy2_in: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&gc5025_out>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dphy2_out: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&mipi_csi2_input>; ++ }; ++ }; ++ }; ++}; ++ ++/* ++ * video_phy0 needs to be enabled ++ * when dsi0 is enabled ++ */ ++&dsi0 { ++ status = "okay"; ++}; ++ ++&dsi0_in_vp0 { ++ status = "disabled"; ++}; ++ ++&dsi0_in_vp1 { ++ status = "okay"; ++}; ++ ++&dsi0_panel { ++ power-supply = <&vcc3v3_lcd0_n>; ++ reset-gpios = <&gpio3 RK_PA3 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&lcd0_rst_gpio>; ++}; ++ ++/* ++ * video_phy1 needs to be enabled ++ * when dsi1 is enabled ++ */ ++&dsi1 { ++ status = "disabled"; ++}; ++ ++&dsi1_in_vp0 { ++ status = "disabled"; ++}; ++ ++&dsi1_in_vp1 { ++ status = "disabled"; ++}; ++ ++&dsi1_panel { ++ power-supply = <&vcc3v3_lcd1_n>; ++ reset-gpios = <&gpio3 RK_PA4 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&lcd1_rst_gpio>; ++}; ++ ++&gmac1 { ++ phy-mode = "rgmii"; ++ clock_in_out = "output"; ++ ++ snps,reset-gpio = <&gpio3 RK_PC2 GPIO_ACTIVE_LOW>; ++ snps,reset-active-low; ++ /* Reset time is 20ms, 100ms for rtl8211f */ ++ snps,reset-delays-us = <0 20000 100000>; ++ ++ assigned-clocks = <&cru SCLK_GMAC1_RX_TX>, <&cru SCLK_GMAC1>; ++ assigned-clock-parents = <&cru SCLK_GMAC1_RGMII_SPEED>, <&cru CLK_MAC1_2TOP>; ++ assigned-clock-rates = <0>, <125000000>; ++ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&gmac1m1_miim ++ &gmac1m1_tx_bus2 ++ &gmac1m1_rx_bus2 ++ &gmac1m1_rgmii_clk ++ &gmac1m1_rgmii_bus>; ++ ++ tx_delay = <0x4f>; ++ rx_delay = <0x25>; ++ ++ phy-handle = <&rgmii_phy0>; ++ status = "okay"; ++}; ++ ++&i2c2 { ++ status = "okay"; ++ pinctrl-0 = <&i2c2m1_xfer>; ++ ++ /* split mode: lane0/1 */ ++ ov5695: ov5695@36 { ++ status = "okay"; ++ compatible = "ovti,ov5695"; ++ reg = <0x36>; ++ clocks = <&cru CLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ power-domains = <&power RK3568_PD_VI>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_clk>; ++ reset-gpios = <&gpio3 RK_PA6 GPIO_ACTIVE_HIGH>; ++ pwdn-gpios = <&gpio4 RK_PB2 GPIO_ACTIVE_HIGH>; ++ /*power-gpios = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>;*/ ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "TongJu"; ++ rockchip,camera-module-lens-name = "CHT842-MD"; ++ port { ++ ov5695_out: endpoint { ++ remote-endpoint = <&dphy1_in>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++ ++ /* split mode: lane:2/3 */ ++ gc5025: gc5025@37 { ++ status = "okay"; ++ compatible = "galaxycore,gc5025"; ++ reg = <0x37>; ++ clocks = <&pmucru CLK_WIFI>; ++ clock-names = "xvclk"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&refclk_pins>; ++ reset-gpios = <&gpio3 RK_PA5 GPIO_ACTIVE_LOW>; ++ pwdn-gpios = <&gpio3 RK_PB0 GPIO_ACTIVE_LOW>; ++ power-domains = <&power RK3568_PD_VI>; ++ /*power-gpios = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>;*/ ++ rockchip,camera-module-index = <1>; ++ rockchip,camera-module-facing = "front"; ++ rockchip,camera-module-name = "TongJu"; ++ rockchip,camera-module-lens-name = "CHT842-MD"; ++ port { ++ gc5025_out: endpoint { ++ remote-endpoint = <&dphy2_in>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++ ++ /* full mode: lane0-3 */ ++ gc8034: gc8034@37 { ++ compatible = "galaxycore,gc8034"; ++ status = "okay"; ++ reg = <0x37>; ++ clocks = <&cru CLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ power-domains = <&power RK3568_PD_VI>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_clk>; ++ reset-gpios = <&gpio3 RK_PA6 GPIO_ACTIVE_LOW>; ++ pwdn-gpios = <&gpio4 RK_PB2 GPIO_ACTIVE_LOW>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "RK-CMK-8M-2-v1"; ++ rockchip,camera-module-lens-name = "CK8401"; ++ port { ++ gc8034_out: endpoint { ++ remote-endpoint = <&dphy0_in>; ++ data-lanes = <1 2 3 4>; ++ }; ++ }; ++ }; ++}; ++ ++&i2c4 { ++ /* i2c4 sda conflict with camera pwdn */ ++ status = "disabled"; ++ ++ /* ++ * gc2145 needs to be disabled, ++ * when gmac1 is enabled; ++ * pinctrl conflicts; ++ */ ++ gc2145: gc2145@3c { ++ status = "disabled"; ++ compatible = "galaxycore,gc2145"; ++ reg = <0x3c>; ++ clocks = <&cru CLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ power-domains = <&power RK3568_PD_VI>; ++ pinctrl-names = "default"; ++ /* conflict with gmac1m1_rgmii_pins & cif_clk*/ ++ pinctrl-0 = <&cif_clk &cif_dvp_clk &cif_dvp_bus16>; ++ ++ /*avdd-supply = <&vcc2v8_dvp>;*/ ++ /*dovdd-supply = <&vcc1v8_dvp>;*/ ++ /*dvdd-supply = <&vcc1v8_dvp>;*/ ++ ++ /*reset-gpios = <&gpio3 RK_PB5 GPIO_ACTIVE_LOW>;*/ ++ pwdn-gpios = <&gpio4 RK_PA6 GPIO_ACTIVE_HIGH>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "CameraKing"; ++ rockchip,camera-module-lens-name = "Largan"; ++ port { ++ gc2145_out: endpoint { ++ remote-endpoint = <&dvp_in_bcam>; ++ }; ++ }; ++ }; ++}; ++ ++&mdio1 { ++ rgmii_phy0: phy@0 { ++ compatible = "ethernet-phy-ieee802.3-c22"; ++ reg = <0x0>; ++ }; ++}; ++ ++ ++ ++/* ++ * power-supply should switche to vcc3v3_lcd1_n ++ * when mipi panel is connected to dsi1. ++ */ ++>1x { ++ power-supply = <&vcc3v3_lcd0_n>; ++}; ++ ++&mipi_csi2 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_csi2_input: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&dphy2_out>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_csi2_output: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&cif_mipi_in>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++}; ++ ++&video_phy0 { ++ status = "okay"; ++}; ++ ++&video_phy1 { ++ status = "disabled"; ++}; ++ ++&pcie2x1 { ++ reset-gpios = <&gpio1 RK_PB2 GPIO_ACTIVE_HIGH>; ++ vpcie3v3-supply = <&pcie20_3v3>; ++ status = "okay"; ++}; ++ ++&pinctrl { ++ cam { ++ camera_pwr: camera-pwr { ++ rockchip,pins = ++ /* camera power en */ ++ <0 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ headphone { ++ hp_det: hp-det { ++ rockchip,pins = <3 RK_PA1 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <2 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ wireless-wlan { ++ wifi_host_wake_irq: wifi-host-wake-irq { ++ rockchip,pins = <2 RK_PB2 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart1_gpios: uart1-gpios { ++ rockchip,pins = <2 RK_PB5 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ lcd0 { ++ lcd0_rst_gpio: lcd0-rst-gpio { ++ rockchip,pins = <3 RK_PA3 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ lcd1 { ++ lcd1_rst_gpio: lcd1-rst-gpio { ++ rockchip,pins = <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++&rkcif { ++ status = "okay"; ++}; ++ ++&rkcif_dvp { ++ status = "disabled"; ++ ++ port { ++ /* Parallel bus endpoint */ ++ dvp_in_bcam: endpoint { ++ remote-endpoint = <&gc2145_out>; ++ bus-width = <8>; ++ vsync-active = <0>; ++ hsync-active = <1>; ++ }; ++ }; ++}; ++ ++&rkcif_mipi_lvds { ++ status = "okay"; ++ ++ port { ++ cif_mipi_in: endpoint { ++ remote-endpoint = <&mipi_csi2_output>; ++ data-lanes = <1 2>; ++ }; ++ }; ++}; ++ ++&rkcif_mmu { ++ status = "okay"; ++}; ++ ++&rkisp { ++ status = "okay"; ++}; ++ ++&rkisp_mmu { ++ status = "okay"; ++}; ++ ++&rkisp_vir0 { ++ status = "okay"; ++ ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp0_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dphy1_out>; ++ }; ++ }; ++}; ++ ++&route_dsi0 { ++ status = "okay"; ++ connect = <&vp1_out_dsi0>; ++}; ++ ++&sdmmc2 { ++ status = "disabled"; ++}; ++ ++&sdmmc1 { ++ max-frequency = <150000000>; ++ no-sd; ++ no-mmc; ++ bus-width = <4>; ++ disable-wp; ++ cap-sd-highspeed; ++ cap-sdio-irq; ++ keep-power-in-suspend; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ non-removable; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc1_bus4 &sdmmc1_cmd &sdmmc1_clk>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&sdio_pwrseq { ++ reset-gpios = <&gpio2 RK_PB1 GPIO_ACTIVE_LOW>; ++}; ++ ++&spdif_8ch { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&spdifm1_tx>; ++}; ++ ++&uart1 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart1m0_xfer &uart1m0_ctsn>; ++}; ++ ++&vcc3v3_lcd0_n { ++ gpio = <&gpio0 RK_PC7 GPIO_ACTIVE_HIGH>; ++ enable-active-high; ++}; ++ ++&vcc3v3_lcd1_n { ++ gpio = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; ++ enable-active-high; ++}; ++&wireless_wlan { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_host_wake_irq>; ++ WIFI,host_wake_irq = <&gpio2 RK_PB2 GPIO_ACTIVE_HIGH>; ++}; ++ ++&work_led { ++ gpios = <&gpio0 RK_PB7 GPIO_ACTIVE_HIGH>; ++}; ++ ++&wireless_bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&rk809 1>; ++ clock-names = "ext_clock"; ++ //wifi-bt-power-toggle; ++ uart_rts_gpios = <&gpio2 RK_PB5 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart1m0_rtsn>; ++ pinctrl-1 = <&uart1_gpios>; ++ BT,reset_gpio = <&gpio2 RK_PB7 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio2 RK_PC1 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio2 RK_PC0 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3566-evb3-ddr3-v10-linux.dts b/arch/arm64/boot/dts/rockchip/rk3566-evb3-ddr3-v10-linux.dts +new file mode 100755 +index 000000000000..e292b0dd1846 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3566-evb3-ddr3-v10-linux.dts +@@ -0,0 +1,8 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++#include "rk3566-evb3-ddr3-v10.dtsi" ++#include "rk3568-linux.dtsi" +diff --git a/arch/arm64/boot/dts/rockchip/rk3566-evb3-ddr3-v10.dts b/arch/arm64/boot/dts/rockchip/rk3566-evb3-ddr3-v10.dts +new file mode 100755 +index 000000000000..09f5260fb375 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3566-evb3-ddr3-v10.dts +@@ -0,0 +1,8 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++#include "rk3566-evb3-ddr3-v10.dtsi" ++#include "rk3568-android.dtsi" +diff --git a/arch/arm64/boot/dts/rockchip/rk3566-evb3-ddr3-v10.dtsi b/arch/arm64/boot/dts/rockchip/rk3566-evb3-ddr3-v10.dtsi +new file mode 100755 +index 000000000000..f936cc186cfb +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3566-evb3-ddr3-v10.dtsi +@@ -0,0 +1,499 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++/dts-v1/; ++ ++#include ++#include ++#include "rk3566.dtsi" ++#include "rk3566-evb.dtsi" ++ ++/ { ++ model = "Rockchip RK3566 EVB3 DDR3 V10 Board"; ++ compatible = "rockchip,rk3566-evb3-DDR3-v10", "rockchip,rk3566"; ++ ++ rk_headset: rk-headset { ++ compatible = "rockchip_headset"; ++ headset_gpio = <&gpio4 RK_PB3 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hp_det>; ++ }; ++ ++ vcc3v3_vga: vcc3v3-vga { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc3v3_vga"; ++ regulator-always-on; ++ regulator-boot-on; ++ gpio = <&gpio4 RK_PB2 GPIO_ACTIVE_HIGH>; ++ enable-active-high; ++ vin-supply = <&vcc3v3_sys>; ++ }; ++ ++ vcc_camera: vcc-camera-regulator { ++ compatible = "regulator-fixed"; ++ gpio = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&camera_pwr>; ++ regulator-name = "vcc_camera"; ++ enable-active-high; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++}; ++ ++&combphy1_usq { ++ status = "okay"; ++}; ++ ++&csi2_dphy_hw { ++ status = "okay"; ++}; ++ ++&csi2_dphy0 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam0: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ov5695_out>; ++ data-lanes = <1 2>; ++ }; ++ mipi_in_ucam1: endpoint@2 { ++ reg = <2>; ++ remote-endpoint = <&gc8034_out>; ++ data-lanes = <1 2 3 4>; ++ }; ++ }; ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ csidphy_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp0_in>; ++ }; ++ }; ++ }; ++}; ++ ++/* ++ * video_phy0 needs to be enabled ++ * when dsi0 is enabled ++ */ ++&dsi0 { ++ status = "okay"; ++}; ++ ++&dsi0_in_vp0 { ++ status = "disabled"; ++}; ++ ++&dsi0_in_vp1 { ++ status = "okay"; ++}; ++ ++&dsi0_panel { ++ power-supply = <&vcc3v3_lcd0_n>; ++ reset-gpios = <&gpio1 RK_PA5 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&lcd0_rst_gpio>; ++}; ++ ++/* ++ * video_phy1 needs to be enabled ++ * when dsi1 is enabled ++ */ ++&dsi1 { ++ status = "disabled"; ++}; ++ ++&dsi1_in_vp0 { ++ status = "disabled"; ++}; ++ ++&dsi1_in_vp1 { ++ status = "disabled"; ++}; ++ ++&dsi1_panel { ++ power-supply = <&vcc3v3_lcd1_n>; ++ reset-gpios = <&gpio4 RK_PC6 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&lcd1_rst_gpio>; ++}; ++ ++&edp { ++ hpd-gpios = <&gpio1 RK_PA7 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++}; ++ ++&edp_phy { ++ status = "okay"; ++}; ++ ++&edp_in_vp0 { ++ status = "okay"; ++}; ++ ++&edp_in_vp1 { ++ status = "disabled"; ++}; ++ ++&gmac1 { ++ phy-mode = "rgmii"; ++ clock_in_out = "output"; ++ ++ snps,reset-gpio = <&gpio3 RK_PA1 GPIO_ACTIVE_LOW>; ++ snps,reset-active-low; ++ /* Reset time is 20ms, 100ms for rtl8211f */ ++ snps,reset-delays-us = <0 20000 100000>; ++ ++ assigned-clocks = <&cru SCLK_GMAC1_RX_TX>, <&cru SCLK_GMAC1>; ++ assigned-clock-parents = <&cru SCLK_GMAC1_RGMII_SPEED>, <&cru CLK_MAC1_2TOP>; ++ assigned-clock-rates = <0>, <125000000>; ++ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&gmac1m0_miim ++ &gmac1m0_tx_bus2_level3 ++ &gmac1m0_rx_bus2 ++ &gmac1m0_rgmii_clk_level2 ++ &gmac1m0_rgmii_bus_level3>; ++ ++ tx_delay = <0x41>; ++ rx_delay = <0x2e>; ++ ++ phy-handle = <&rgmii_phy1>; ++ status = "okay"; ++}; ++ ++>1x { ++ power-supply = <&vcc3v3_lcd0_n>; ++}; ++ ++&i2c2 { ++ status = "okay"; ++ pinctrl-0 = <&i2c2m1_xfer>; ++ ++ gc2145: gc2145@3c { ++ status = "okay"; ++ compatible = "galaxycore,gc2145"; ++ reg = <0x3c>; ++ clocks = <&cru CLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ power-domains = <&power RK3568_PD_VI>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_clk &cif_dvp_clk &cif_dvp_bus16>; ++ ++ /*avdd-supply = <&vcc2v8_dvp>;*/ ++ /*dovdd-supply = <&vcc1v8_dvp>;*/ ++ /*dvdd-supply = <&vcc1v8_dvp>;*/ ++ ++ power-gpios = <&gpio3 RK_PD4 GPIO_ACTIVE_HIGH>; ++ /*reset-gpios = <&gpio3 RK_PB5 GPIO_ACTIVE_LOW>;*/ ++ pwdn-gpios = <&gpio3 RK_PD2 GPIO_ACTIVE_HIGH>; ++ rockchip,camera-module-index = <1>; ++ rockchip,camera-module-facing = "front"; ++ rockchip,camera-module-name = "CameraKing"; ++ rockchip,camera-module-lens-name = "Largan"; ++ port { ++ gc2145_out: endpoint { ++ remote-endpoint = <&dvp_in_bcam>; ++ }; ++ }; ++ }; ++ ++ ov5695: ov5695@36 { ++ status = "okay"; ++ compatible = "ovti,ov5695"; ++ reg = <0x36>; ++ clocks = <&cru CLK_CAM0_OUT>; ++ clock-names = "xvclk"; ++ power-domains = <&power RK3568_PD_VI>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cam_clkout0>; ++ reset-gpios = <&gpio3 RK_PD0 GPIO_ACTIVE_HIGH>; ++ pwdn-gpios = <&gpio3 RK_PC6 GPIO_ACTIVE_HIGH>; ++ /*power-gpios = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>;*/ ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "TongJu"; ++ rockchip,camera-module-lens-name = "CHT842-MD"; ++ port { ++ ov5695_out: endpoint { ++ remote-endpoint = <&mipi_in_ucam0>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++ ++ gc8034: gc8034@37 { ++ compatible = "galaxycore,gc8034"; ++ status = "okay"; ++ reg = <0x37>; ++ clocks = <&cru CLK_CAM0_OUT>; ++ clock-names = "xvclk"; ++ power-domains = <&power RK3568_PD_VI>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cam_clkout0>; ++ reset-gpios = <&gpio3 RK_PD0 GPIO_ACTIVE_LOW>; ++ pwdn-gpios = <&gpio3 RK_PC6 GPIO_ACTIVE_LOW>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "RK-CMK-8M-2-v1"; ++ rockchip,camera-module-lens-name = "CK8401"; ++ port { ++ gc8034_out: endpoint { ++ remote-endpoint = <&mipi_in_ucam1>; ++ data-lanes = <1 2 3 4>; ++ }; ++ }; ++ }; ++ ++}; ++ ++&i2s1_8ch { ++ status = "disabled"; ++}; ++ ++&i2s3_2ch { ++ status = "okay"; ++ pinctrl-names = "default"; ++ rockchip,clk-trcm = <1>; ++ pinctrl-0 = <&i2s3m1_sclk ++ &i2s3m1_lrck ++ &i2s3m1_sdi ++ &i2s3m1_sdo>; ++}; ++ ++&mdio1 { ++ rgmii_phy1: phy@0 { ++ compatible = "ethernet-phy-ieee802.3-c22"; ++ reg = <0x0>; ++ }; ++}; ++ ++&video_phy0 { ++ status = "okay"; ++}; ++ ++&video_phy1 { ++ status = "disabled"; ++}; ++ ++&pdm { ++ status = "disabled"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pdmm1_clk1 ++ &pdmm1_sdi1 ++ &pdmm1_sdi2 ++ &pdmm1_sdi3>; ++}; ++ ++&pdmics { ++ status = "disabled"; ++}; ++ ++&pdm_mic_array { ++ status = "disabled"; ++}; ++ ++&pinctrl { ++ cam { ++ camera_pwr: camera-pwr { ++ rockchip,pins = ++ /* camera power en */ ++ <0 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ headphone { ++ hp_det: hp-det { ++ rockchip,pins = <4 RK_PB3 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ lcd0 { ++ lcd0_rst_gpio: lcd-rst-gpio { ++ rockchip,pins = <1 RK_PA5 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ lcd1 { ++ lcd1_rst_gpio: lcd1-rst-gpio { ++ rockchip,pins = <4 RK_PC6 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <2 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ wireless-wlan { ++ wifi_host_wake_irq: wifi-host-wake-irq { ++ rockchip,pins = <2 RK_PB2 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart1_gpios: uart1-gpios { ++ rockchip,pins = <2 RK_PB5 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++&rkcif { ++ status = "okay"; ++}; ++ ++&rkcif_dvp { ++ status = "okay"; ++ ++ port { ++ /* Parallel bus endpoint */ ++ dvp_in_bcam: endpoint { ++ remote-endpoint = <&gc2145_out>; ++ bus-width = <8>; ++ vsync-active = <0>; ++ hsync-active = <1>; ++ }; ++ }; ++}; ++ ++&rkisp { ++ status = "okay"; ++}; ++ ++&rkisp_mmu { ++ status = "okay"; ++}; ++ ++&rkisp_vir0 { ++ status = "okay"; ++ ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp0_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&csidphy_out>; ++ }; ++ }; ++}; ++ ++&rk809_codec { ++ compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; ++ clocks = <&cru I2S3_MCLKOUT>; ++ clock-names = "mclk"; ++ assigned-clocks = <&cru I2S3_MCLKOUT>, <&cru I2S3_MCLK_IOE>; ++ assigned-clock-rates = <12288000>; ++ assigned-clock-parents = <&cru I2S3_MCLKOUT_TX>, <&cru I2S3_MCLKOUT>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s3m1_mclk>; ++ hp-volume = <20>; ++ spk-volume = <3>; ++ mic-in-differential; ++ status = "okay"; ++}; ++ ++&rk809_sound { ++ status = "okay"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,rk809-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ ++ simple-audio-card,cpu { ++ sound-dai = <&i2s3_2ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&rk809_codec>; ++ }; ++}; ++ ++&route_dsi0 { ++ status = "okay"; ++ connect = <&vp1_out_dsi0>; ++}; ++ ++&sata1 { ++ status = "okay"; ++}; ++ ++&sdio_pwrseq { ++ reset-gpios = <&gpio2 RK_PB1 GPIO_ACTIVE_LOW>; ++}; ++ ++&sdmmc1 { ++ max-frequency = <150000000>; ++ no-sd; ++ no-mmc; ++ bus-width = <4>; ++ disable-wp; ++ cap-sd-highspeed; ++ cap-sdio-irq; ++ keep-power-in-suspend; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ non-removable; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc1_bus4 &sdmmc1_cmd &sdmmc1_clk>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&sdmmc2 { ++ status = "disabled"; ++}; ++ ++&spdif_8ch { ++ status = "disabled"; ++}; ++ ++&uart1 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart1m0_xfer &uart1m0_ctsn>; ++}; ++ ++&vcc3v3_lcd0_n { ++ gpio = <&gpio0 RK_PC0 GPIO_ACTIVE_HIGH>; ++ enable-active-high; ++}; ++ ++&wireless_bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&rk809 1>; ++ clock-names = "ext_clock"; ++ //wifi-bt-power-toggle; ++ uart_rts_gpios = <&gpio2 RK_PB5 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart1m0_rtsn>; ++ pinctrl-1 = <&uart1_gpios>; ++ BT,reset_gpio = <&gpio2 RK_PB7 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio2 RK_PC1 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio2 RK_PC0 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++}; ++ ++&wireless_wlan { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_host_wake_irq>; ++ WIFI,host_wake_irq = <&gpio2 RK_PB2 GPIO_ACTIVE_HIGH>; ++}; ++ ++&work_led { ++ gpios = <&gpio0 RK_PD3 GPIO_ACTIVE_HIGH>; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3566-evb5-lp4x-v10.dts b/arch/arm64/boot/dts/rockchip/rk3566-evb5-lp4x-v10.dts +new file mode 100755 +index 000000000000..600fc3c39586 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3566-evb5-lp4x-v10.dts +@@ -0,0 +1,7 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ */ ++ ++#include "rk3566-evb5-lp4x-v10.dtsi" ++#include "rk3568-android.dtsi" +diff --git a/arch/arm64/boot/dts/rockchip/rk3566-evb5-lp4x-v10.dtsi b/arch/arm64/boot/dts/rockchip/rk3566-evb5-lp4x-v10.dtsi +new file mode 100755 +index 000000000000..83586659530d +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3566-evb5-lp4x-v10.dtsi +@@ -0,0 +1,317 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ */ ++ ++/dts-v1/; ++ ++#include ++#include ++#include "rk3566.dtsi" ++#include "rk3566-evb.dtsi" ++ ++/ { ++ model = "Rockchip RK3566 EVB5 LP4X V10 Board"; ++ compatible = "rockchip,rk3566-evb5-lp4x-v10", "rockchip,rk3566"; ++ ++ pcie20_3v3: gpio-regulator { ++ compatible = "regulator-gpio"; ++ regulator-name = "pcie20_3v3"; ++ regulator-min-microvolt = <100000>; ++ regulator-max-microvolt = <3300000>; ++ gpios = <&gpio0 RK_PB7 GPIO_ACTIVE_HIGH>; ++ gpios-states = <0x1>; ++ states = <100000 0x0 ++ 3300000 0x1>; ++ }; ++ ++ rk_headset: rk-headset { ++ compatible = "rockchip_headset"; ++ headset_gpio = <&gpio3 RK_PA1 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hp_det>; ++ }; ++ ++ vcc3v3_vga: vcc3v3-vga { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc3v3_vga"; ++ regulator-always-on; ++ regulator-boot-on; ++ gpio = <&gpio2 RK_PB1 GPIO_ACTIVE_HIGH>; ++ enable-active-high; ++ vin-supply = <&vcc3v3_sys>; ++ }; ++}; ++ ++&audiopwmout_diff { ++ status = "disabled"; ++}; ++ ++&combphy1_usq { ++ status = "okay"; ++}; ++ ++&combphy2_psq { ++ status = "disabled"; ++}; ++ ++&dig_acodec { ++ status = "disabled"; ++ rockchip,pwm-output-mode; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&audiopwm_loutp ++ &audiopwm_loutn ++ &audiopwm_routp ++ &audiopwm_routn ++ >; ++}; ++ ++/* ++ * video_phy0 needs to be enabled ++ * when dsi0 is enabled ++ */ ++&dsi0 { ++ status = "okay"; ++ connect = <&vp1_out_dsi0>; ++}; ++ ++&dsi0_in_vp0 { ++ status = "disabled"; ++}; ++ ++&dsi0_in_vp1 { ++ status = "okay"; ++}; ++ ++&dsi0_panel { ++ power-supply = <&vcc3v3_lcd0_n>; ++ reset-gpios = <&gpio4 RK_PC5 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&lcd0_rst_gpio>; ++}; ++ ++/* ++ * video_phy1 needs to be enabled ++ * when dsi1 is enabled ++ */ ++&dsi1 { ++ status = "disabled"; ++}; ++ ++&dsi1_in_vp0 { ++ status = "disabled"; ++}; ++ ++&dsi1_in_vp1 { ++ status = "disabled"; ++}; ++ ++&dsi1_panel { ++ power-supply = <&vcc3v3_lcd1_n>; ++ reset-gpios = <&gpio4 RK_PC6 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&lcd1_rst_gpio>; ++}; ++ ++&edp { ++ hpd-gpios = <&gpio4 RK_PC4 GPIO_ACTIVE_HIGH>; ++ status = "disabled"; ++}; ++ ++&edp_phy { ++ status = "disabled"; ++}; ++ ++&edp_in_vp0 { ++ status = "disabled"; ++}; ++ ++&edp_in_vp1 { ++ status = "disabled"; ++}; ++ ++&gmac1 { ++ phy-mode = "rgmii"; ++ clock_in_out = "output"; ++ ++ snps,reset-gpio = <&gpio4 RK_PC2 GPIO_ACTIVE_LOW>; ++ snps,reset-active-low; ++ /* Reset time is 20ms, 100ms for rtl8211f */ ++ snps,reset-delays-us = <0 20000 100000>; ++ ++ assigned-clocks = <&cru SCLK_GMAC1_RX_TX>, <&cru SCLK_GMAC1>; ++ assigned-clock-parents = <&cru SCLK_GMAC1_RGMII_SPEED>, <&cru CLK_MAC1_2TOP>; ++ assigned-clock-rates = <0>, <125000000>; ++ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&gmac1m0_miim ++ &gmac1m0_tx_bus2_level3 ++ &gmac1m0_rx_bus2 ++ &gmac1m0_rgmii_clk_level2 ++ &gmac1m0_rgmii_bus_level3>; ++ ++ tx_delay = <0x41>; ++ rx_delay = <0x2e>; ++ ++ phy-handle = <&rgmii_phy1>; ++ status = "disabled"; ++}; ++ ++/* ++ * power-supply should switche to vcc3v3_lcd1_n ++ * when mipi panel is connected to dsi1. ++ */ ++>1x { ++ power-supply = <&vcc3v3_lcd0_n>; ++}; ++ ++&i2c5 { ++ status = "disabled"; ++}; ++ ++&i2s3_2ch { ++ status = "disabled"; ++}; ++ ++&mdio1 { ++ rgmii_phy1: phy@0 { ++ compatible = "ethernet-phy-ieee802.3-c22"; ++ reg = <0x0>; ++ }; ++}; ++ ++&video_phy0 { ++ status = "okay"; ++}; ++ ++&video_phy1 { ++ status = "disabled"; ++}; ++ ++&pcie2x1 { ++ reset-gpios = <&gpio1 RK_PB2 GPIO_ACTIVE_HIGH>; ++ vpcie3v3-supply = <&pcie20_3v3>; ++ status = "disabled"; ++}; ++ ++&pdm { ++ status = "disabled"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pdmm1_clk1 ++ &pdmm1_sdi1 ++ &pdmm1_sdi2 ++ &pdmm1_sdi3>; ++}; ++ ++&pdmics { ++ status = "disabled"; ++}; ++ ++&pdm_mic_array { ++ status = "disabled"; ++}; ++ ++&route_dsi0 { ++ status = "okay"; ++}; ++ ++&sdmmc2 { ++ status = "disabled"; ++}; ++ ++&uart1 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart1m1_xfer &uart1m1_ctsn>; ++}; ++ ++&uart3 { ++ status = "disabled"; ++}; ++ ++&uart4 { ++ status = "disabled"; ++}; ++ ++&uart7 { ++ status = "disabled"; ++}; ++ ++&usb_host0_ehci { ++ status = "okay"; ++}; ++ ++&usb_host0_ohci { ++ status = "disabled"; ++}; ++ ++&usb_host1_ehci { ++ status = "okay"; ++}; ++ ++&usb_host1_ohci { ++ status = "disabled"; ++}; ++ ++&vcc3v3_lcd0_n { ++ gpio = <&gpio0 RK_PC0 GPIO_ACTIVE_HIGH>; ++ enable-active-high; ++}; ++ ++&vcc3v3_lcd1_n { ++ gpio = <&gpio0 RK_PC2 GPIO_ACTIVE_HIGH>; ++ enable-active-high; ++}; ++ ++&wireless_bluetooth { ++ uart_rts_gpios = <&gpio4 RK_PB6 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart1m1_rtsn>; ++ pinctrl-1 = <&uart1_gpios>; ++ BT,reset_gpio = <&gpio4 RK_PA5 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio4 RK_PA6 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio4 RK_PB1 GPIO_ACTIVE_HIGH>; ++ status = "disabled"; ++}; ++ ++&wireless_wlan { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_host_wake_irq>; ++ status = "disabled"; ++}; ++ ++&work_led { ++ status = "disabled"; ++}; ++ ++&pinctrl { ++ headphone { ++ hp_det: hp-det { ++ rockchip,pins = <3 RK_PA1 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ lcd0 { ++ lcd0_rst_gpio: lcd0-rst-gpio { ++ rockchip,pins = <4 RK_PC5 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ lcd1 { ++ lcd1_rst_gpio: lcd1-rst-gpio { ++ rockchip,pins = <4 RK_PC6 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ wireless-wlan { ++ wifi_host_wake_irq: wifi-host-wake-irq { ++ rockchip,pins = <3 RK_PD4 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart1_gpios: uart1-gpios { ++ rockchip,pins = <4 RK_PB6 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3566-rk817-eink-w103.dts b/arch/arm64/boot/dts/rockchip/rk3566-rk817-eink-w103.dts +new file mode 100755 +index 000000000000..3f559fca27f8 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3566-rk817-eink-w103.dts +@@ -0,0 +1,1115 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++/dts-v1/; ++ ++#include ++#include ++#include ++#include ++#include ++#include "rk3566.dtsi" ++#include "rk3568-android.dtsi" ++#include "rk3566-eink.dtsi" ++ ++/ { ++ model = "Rockchip RK3566 RK817 EINK LP4X Board"; ++ compatible = "rockchip,rk3566-rk817-eink", "rockchip,rk3566"; ++ ++ charge-animation { ++ compatible = "rockchip,uboot-charge"; ++ rockchip,uboot-charge-on = <1>; ++ rockchip,android-charge-on = <0>; ++ rockchip,uboot-low-power-voltage = <3350>; ++ rockchip,screen-on-voltage = <3400>; ++ rockchip,auto-wakeup-interval = <60>; ++ status = "okay"; ++ }; ++ ++ adc_keys: adc-keys { ++ status = "okay"; ++ compatible = "adc-keys"; ++ io-channels = <&saradc 0>; ++ io-channel-names = "buttons"; ++ keyup-threshold-microvolt = <1800000>; ++ poll-interval = <100>; ++ ++ vol-up-key { ++ label = "volume up"; ++ linux,code = ; ++ press-threshold-microvolt = <9000>; ++ }; ++ ++ vol-down-key { ++ label = "volume down"; ++ linux,code = ; ++ press-threshold-microvolt = <235000>; ++ }; ++ }; ++ ++ backlight: backlight { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm0 0 25000 0>; ++ brightness-levels = < ++ 0 20 20 21 21 22 22 23 ++ 23 24 24 25 25 26 26 27 ++ 27 28 28 29 29 30 30 31 ++ 31 32 32 33 33 34 34 35 ++ 35 36 36 37 37 38 38 39 ++ 40 41 42 43 44 45 46 47 ++ 48 49 50 51 52 53 54 55 ++ 56 57 58 59 60 61 62 63 ++ 64 65 66 67 68 69 70 71 ++ 72 73 74 75 76 77 78 79 ++ 80 81 82 83 84 85 86 87 ++ 88 89 90 91 92 93 94 95 ++ 96 97 98 99 100 101 102 103 ++ 104 105 106 107 108 109 110 111 ++ 112 113 114 115 116 117 118 119 ++ 120 121 122 123 124 125 126 127 ++ 128 129 130 131 132 133 134 135 ++ 136 137 138 139 140 141 142 143 ++ 144 145 146 147 148 149 150 151 ++ 152 153 154 155 156 157 158 159 ++ 160 161 162 163 164 165 166 167 ++ 168 169 170 171 172 173 174 175 ++ 176 177 178 179 180 181 182 183 ++ 184 185 186 187 188 189 190 191 ++ 192 193 194 195 196 197 198 199 ++ 200 201 202 203 204 205 206 207 ++ 208 209 210 211 212 213 214 215 ++ 216 217 218 219 220 221 222 223 ++ 224 225 226 227 228 229 230 231 ++ 232 233 234 235 236 237 238 239 ++ 240 241 242 243 244 245 246 247 ++ 248 249 250 251 252 253 254 255 ++ >; ++ default-brightness-level = <200>; ++ }; ++ ++ gpio_keys: gpio-keys { ++ status = "disabled"; ++ compatible = "gpio-keys"; ++ autorepeat; ++ ++ BACK { ++ label = "GPIO Key Home"; ++ debounce-interval = <10>; ++ interrupt-parent = <&gpio0>; ++ interrupts = <13 IRQ_TYPE_LEVEL_LOW>; ++ linux,input-type = ; ++ linux,code = ; ++ }; ++ }; ++ ++ hdmi_sound: hdmi-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,mclk-fs = <128>; ++ simple-audio-card,name = "rockchip,hdmi"; ++ status = "disabled"; ++ ++ simple-audio-card,cpu { ++ sound-dai = <&i2s0_8ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&hdmi>; ++ }; ++ }; ++ ++ leds: gpio-leds { ++ compatible = "gpio-leds"; ++ ++ led@2 { ++ gpios = <&gpio3 RK_PC5 GPIO_ACTIVE_HIGH>; ++ linux,default-trigger = "battery-charging"; ++ label = "battery_charging"; ++ retain-state-suspended; ++ }; ++ }; ++ ++ hall_sensor: hall-mh248 { ++ compatible = "hall-mh248"; ++ irq-gpio = <&gpio0 RK_PC7 IRQ_TYPE_EDGE_BOTH>; ++ hall-active = <1>; ++ status = "okay"; ++ }; ++ ++ vccsys: vccsys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc3v8_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3800000>; ++ regulator-max-microvolt = <3800000>; ++ }; ++ ++ vcc_tp: vcc-tp-regulator { ++ compatible = "regulator-fixed"; ++ enable-active-high; ++ gpio = <&gpio0 RK_PD6 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&vcc_tp_en>; ++ regulator-name = "vcc_tp"; ++ regulator-boot-on; ++ startup-delay-us = <10000>; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ }; ++ ++ rk817-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,rk817-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ ++ simple-audio-card,cpu { ++ sound-dai = <&i2s1_8ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&rk817_codec>; ++ }; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ clocks = <&rk817 1>; ++ clock-names = "ext_clock"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ post-power-on-delay-ms = <200>; ++ reset-gpios = <&gpio0 RK_PC6 GPIO_ACTIVE_LOW>; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ rockchip,grf = <&grf>; ++ wifi_chip_type = "ap6255"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_vbat &wifi_host_wake_irq>; ++ WIFI,vbat_gpio = <&gpio0 RK_PA0 GPIO_ACTIVE_HIGH>; ++ WIFI,host_wake_irq = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&rk817 1>; ++ clock-names = "ext_clock"; ++ wifi-bt-power-toggle; ++ uart_rts_gpios = <&gpio2 RK_PB5 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart1m0_rtsn>, ++ <&bt_reset_gpio>, ++ <&bt_wake_gpio>, ++ <&bt_irq_gpio>; ++ pinctrl-1 = <&uart1_gpios>; ++ BT,reset_gpio = <&gpio0 RK_PC4 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio0 RK_PC2 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio0 RK_PC3 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++}; ++ ++&ebc { ++ /* clock rate 1000M/n, (n=1~32) */ ++ assigned-clocks = <&cru CPLL_333M>, <&cru DCLK_EBC>; ++ //assigned-clock-rates = <340000000>, <340000000>; ++ assigned-clock-rates = <250000000>, <250000000>; ++ status = "okay"; ++}; ++ ++&ebc_dev { ++ pmic = <&tps65185>; ++ status = "okay"; ++#if 0 ++ /* ED097TC2U1 */ ++ panel,width = <1200>; ++ panel,height = <825>; ++ panel,vir_width = <1200>; ++ panel,vir_height = <825>; ++ panel,sdck = <25000000>; ++ panel,lsl = <4>; ++ panel,lbl = <4>; ++ panel,ldl = <300>; ++ panel,lel = <36>; ++ panel,gdck-sta = <18>; ++ panel,lgonl = <265>; ++ panel,fsl = <2>; ++ panel,fbl = <4>; ++ panel,fdl = <825>; ++ panel,fel = <24>; ++ panel,mirror = <0>; ++ panel,panel_16bit = <0>; ++ panel,panel_color = <0>; ++ panel,width-mm = <203>; ++ panel,height-mm = <140>; ++#endif ++#if 1 ++ /* ES103TC1 */ ++ panel,width = <1872>; ++ panel,height = <1404>; ++ panel,vir_width = <1872>; ++ panel,vir_height = <1404>; ++ panel,sdck = <33300000>; ++ panel,lsl = <18>; ++ panel,lbl = <17>; ++ panel,ldl = <234>; ++ panel,lel = <7>; ++ panel,gdck-sta = <34>; ++ panel,lgonl = <192>; ++ panel,fsl = <1>; ++ panel,fbl = <4>; ++ panel,fdl = <1404>; ++ panel,fel = <12>; ++ panel,mirror = <0>; ++ panel,panel_16bit = <1>; ++ panel,panel_color = <0>; ++ panel,width-mm = <157>; ++ panel,height-mm = <210>; ++#endif ++#if 0 ++ /* ES133TC1 */ ++ panel,width = <2200>; ++ panel,height = <1650>; ++ panel,vir_width = <2208>; ++ panel,vir_height = <1650>; ++ panel,sdck = <37500000>; ++ panel,lsl = <4>; ++ panel,lbl = <8>; ++ panel,ldl = <275>; ++ panel,lel = <14>; ++ panel,gdck-sta = <34>; ++ panel,lgonl = <217>; ++ panel,fsl = <1>; ++ panel,fbl = <4>; ++ panel,fdl = <1650>; ++ panel,fel = <6>; ++ panel,mirror = <0>; ++ panel,panel_16bit = <1>; ++ panel,panel_color = <0>; ++ panel,width-mm = <157>; ++ panel,height-mm = <210>; ++#endif ++}; ++ ++&cpu0 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&csi2_dphy_hw { ++ status = "okay"; ++}; ++ ++&csi2_dphy0 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam0: endpoint@0 { ++ reg = <0>; ++ /*remote-endpoint = <&ov5648_out>;*/ ++ data-lanes = <1 2>; ++ }; ++ }; ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ csidphy0_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp0_in>; ++ }; ++ }; ++ }; ++}; ++ ++&gpu { ++ mali-supply = <&vdd_gpu>; ++ status = "okay"; ++}; ++ ++&hdmi { ++ status = "disabled"; ++}; ++ ++&hdmi_in_vp0 { ++ status = "disabled"; ++}; ++ ++&hdmi_in_vp1 { ++ status = "disabled"; ++}; ++ ++&hdmi_sound { ++ status = "disabled"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ ++ vdd_cpu: tcs4525@1c { ++ compatible = "tcs,tcs452x"; ++ reg = <0x1c>; ++ vin-supply = <&vccsys>; ++ regulator-compatible = "fan53555-reg"; ++ regulator-name = "vdd_cpu"; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1390000>; ++ regulator-init-microvolt = <900000>; ++ regulator-ramp-delay = <2300>; ++ fcs,suspend-voltage-selector = <0>; ++ regulator-initial-mode = <0x2>; ++ regulator-boot-on; ++ regulator-always-on; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1100000>; ++ regulator-changeable-in-suspend; ++ }; ++ }; ++ ++ rk817: pmic@20 { ++ compatible = "rockchip,rk817"; ++ reg = <0x20>; ++ interrupt-parent = <&gpio0>; ++ interrupts = <3 IRQ_TYPE_LEVEL_LOW>; ++ ++ pinctrl-names = "default"; ++// pinctrl-names = "default", "pmic-sleep", ++// "pmic-power-off", "pmic-reset"; ++ pinctrl-0 = <&pmic_int>; ++// pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; ++// pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; ++// pinctrl-3 = <&soc_slppin_gpio>, <&rk817_slppin_rst>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ //fb-inner-reg-idxs = <2>; ++ /* 1: rst regs (default in codes), 0: rst the pmic */ ++ pmic-reset-func = <0>; ++ /* not save the PMIC_POWER_EN register in uboot */ ++ not-save-power-en = <1>; ++ vcc1-supply = <&vccsys>; ++ vcc2-supply = <&vccsys>; ++ vcc3-supply = <&vccsys>; ++ vcc4-supply = <&vccsys>; ++ vcc5-supply = <&vccsys>; ++ vcc6-supply = <&vccsys>; ++ vcc7-supply = <&vccsys>; ++ vcc8-supply = <&vccsys>; ++ vcc9-supply = <&dcdc_boost>; ++ ++ pwrkey { ++ status = "okay"; ++ }; ++ ++ pinctrl_rk8xx: pinctrl_rk8xx { ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ rk817_slppin_null: rk817_slppin_null { ++ pins = "gpio_slp"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_slppin_slp: rk817_slppin_slp { ++ pins = "gpio_slp"; ++ function = "pin_fun1"; ++ }; ++ ++ rk817_slppin_pwrdn: rk817_slppin_pwrdn { ++ pins = "gpio_slp"; ++ function = "pin_fun2"; ++ }; ++ ++ rk817_slppin_rst: rk817_slppin_rst { ++ pins = "gpio_slp"; ++ function = "pin_fun3"; ++ }; ++ }; ++ ++ regulators { ++ vdd_logic: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <500000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-init-microvolt = <900000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_logic"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <900000>; ++ regulator-changeable-in-suspend; ++ }; ++ }; ++ ++ vdd_gpu: DCDC_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <500000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-init-microvolt = <900000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_gpu"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-changeable-in-suspend; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_ddr"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_3v3: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_3v3"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-changeable-in-suspend; ++ }; ++ }; ++ ++ vcca1v8_pmu: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcca1v8_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ regulator-changeable-in-suspend; ++ }; ++ }; ++ ++ vdda_0v9: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <900000>; ++ regulator-name = "vdda_0v9"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-changeable-in-suspend; ++ }; ++ }; ++ ++ vdda0v9_pmu: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <900000>; ++ regulator-name = "vdda0v9_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <900000>; ++ regulator-changeable-in-suspend; ++ }; ++ }; ++ ++ vccio_acodec: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vccio_acodec"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-changeable-in-suspend; ++ }; ++ }; ++ ++ vccio_sd: LDO_REG5 { ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vccio_sd"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-changeable-in-suspend; ++ }; ++ }; ++ ++ vcc3v3_pmu: LDO_REG6 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcc3v3_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ regulator-changeable-in-suspend; ++ }; ++ }; ++ ++ vcc_1v8: LDO_REG7 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc_1v8"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-changeable-in-suspend; ++ }; ++ }; ++ ++ vcc1v8_dvp: LDO_REG8 { ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc1v8_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-changeable-in-suspend; ++ }; ++ }; ++ ++ sleep_sta_ctl: LDO_REG9 { ++ regulator-name = "sleep_sta_ctl"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ regulator-changeable-in-suspend; ++ }; ++ }; ++ ++ dcdc_boost: BOOST { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <4700000>; ++ regulator-max-microvolt = <5400000>; ++ regulator-name = "boost"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-changeable-in-suspend; ++ }; ++ }; ++ ++ otg_switch: OTG_SWITCH { ++ regulator-name = "otg_switch"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-changeable-in-suspend; ++ }; ++ }; ++ }; ++ ++ battery { ++ compatible = "rk817,battery"; ++ ocv_table = <3400 3513 3578 3687 3734 3752 3763 ++ 3766 3771 3784 3804 3836 3885 3925 ++ 3962 4005 4063 4114 4169 4227 4303>; ++ design_capacity = <4150>; ++ design_qmax = <4565>; ++ bat_res = <100>; ++ sleep_enter_current = <150>; ++ sleep_exit_current = <180>; ++ sleep_filter_current = <100>; ++ power_off_thresd = <3450>; ++ zero_algorithm_vol = <3850>; ++ max_soc_offset = <60>; ++ monitor_sec = <5>; ++ sample_res = <10>; ++ virtual_power = <0>; ++ low_power_sleep = <1>; ++ }; ++ ++ charger { ++ compatible = "rk817,charger"; ++ min_input_voltage = <4500>; ++ max_input_current = <1500>; ++ max_chrg_current = <2000>; ++ max_chrg_voltage = <4300>; ++ chrg_term_mode = <0>; ++ chrg_finish_cur = <300>; ++ virtual_power = <0>; ++ dc_det_adc = <0>; ++ extcon = <&usb2phy0>; ++ gate_function_disable = <1>; ++ }; ++ ++ rk817_codec: codec { ++ #sound-dai-cells = <0>; ++ compatible = "rockchip,rk817-codec"; ++ clocks = <&cru I2S1_MCLKOUT>; ++ clock-names = "mclk"; ++ assigned-clocks = <&cru I2S1_MCLKOUT>, <&cru I2S1_MCLK_TX_IOE>; ++ assigned-clock-rates = <12288000>; ++ assigned-clock-parents = <&cru I2S1_MCLKOUT_TX>, <&cru I2S1_MCLKOUT_TX>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1m0_mclk>; ++ hp-volume = <20>; ++ spk-volume = <3>; ++ out-l2spk-r2hp; ++ spk-ctl-gpios = <&gpio1 RK_PB1 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ }; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ ++ wacom: wacom@9 { ++ compatible = "wacom,w9013"; ++ reg = <0x09>; ++ pwr-supply = <&vcc_tp>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wacom_gpio>; ++ gpio_detect = <&gpio0 RK_PC0 GPIO_ACTIVE_HIGH>; ++ gpio_intr = <&gpio0 RK_PB6 GPIO_ACTIVE_HIGH>; ++ gpio_rst = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; ++ revert_x = <0>; ++ revert_y = <0>; ++ xy_exchange = <0>; ++ }; ++}; ++ ++&i2c3 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2c3m1_xfer>; ++ ++ tps65185: tps65185@68 { ++ status = "okay"; ++ compatible = "ti,tps65185"; ++ reg = <0x68>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&tps65185_gpio>; ++ wakeup-gpios = <&gpio3 RK_PA5 GPIO_ACTIVE_HIGH>; ++ vcomctl-gpios = <&gpio4 RK_PB2 GPIO_ACTIVE_HIGH>; ++ int-gpios = <&gpio3 RK_PA6 GPIO_ACTIVE_HIGH>; ++ powerup-gpios = <&gpio3 RK_PB0 GPIO_ACTIVE_HIGH>; ++ poweren-gpios = <&gpio3 RK_PB1 GPIO_ACTIVE_HIGH>; ++ }; ++}; ++ ++&i2c4 { ++ //camera ++}; ++ ++&i2c5 { ++ status = "okay"; ++ ++ sensor@4c { ++ status = "okay"; ++ compatible = "gs_mma7660"; ++ reg = <0x4c>; ++ type = ; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ layout = <6>; ++ reprobe_en = <1>; ++ }; ++ ++ tsc@24 { ++ status = "okay"; ++ compatible = "cy,cyttsp5_i2c_adapter"; ++ reg = <0x24>; ++ cy,adapter_id = "cyttsp5_i2c_adapter"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&tsc_gpio>; ++ cytp-supply = <&vcc_tp>; ++ cy,core { ++ cy,name = "cyttsp5_core"; ++ cy,irq_gpio = <&gpio0 RK_PA6 GPIO_ACTIVE_HIGH>; ++ cy,rst_gpio = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>; ++ cy,hid_desc_register = <1>; ++ /* CY_CORE_FLAG_RESTORE_PARAMETERS */ ++ cy,flags = <6>; ++ /* CY_CORE_EWG_NONE */ ++ cy,easy_wakeup_gesture = <0>; ++ cy,btn_keys = <172 /* KEY_HOMEPAGE */ ++ /* previously was KEY_HOME, new Android versions use KEY_HOMEPAGE */ ++ 139 /* KEY_MENU */ ++ 158 /* KEY_BACK */ ++ 217 /* KEY_SEARCH */ ++ 114 /* KEY_VOLUMEDOWN */ ++ 115 /* KEY_VOLUMEUP */ ++ 212 /* KEY_CAMERA */ ++ 116>; /* KEY_POWER */ ++ cy,btn_keys-tag = <0>; ++ cy,mt { ++ cy,name = "cyttsp5_mt"; ++ cy,inp_dev_name = "cyttsp5_mt"; ++ cy,flags = <0xA8>; ++ cy,abs = ++ /* ABS_MT_POSITION_X, CY_ABS_MIN_X, CY_ABS_MAX_X, 0, 0 */ ++ <0x35 0 1404 0 0 ++ /* ABS_MT_POSITION_Y, CY_ABS_MIN_Y, CY_ABS_MAX_Y, 0, 0 */ ++ 0x36 0 1872 0 0 ++ /* ABS_MT_PRESSURE, CY_ABS_MIN_P, CY_ABS_MAX_P, 0, 0 */ ++ 0x3a 0 255 0 0 ++ /* CY_IGNORE_VALUE, CY_ABS_MIN_W, CY_ABS_MAX_W, 0, 0 */ ++ 0xffff 0 255 0 0 ++ /* ABS_MT_TRACKING_ID, CY_ABS_MIN_T, CY_ABS_MAX_T, 0, 0 */ ++ 0x39 0 15 0 0 ++ /* ABS_MT_TOUCH_MAJOR, 0, 255, 0, 0 */ ++ 0x30 0 255 0 0 ++ /* ABS_MT_TOUCH_MINOR, 0, 255, 0, 0 */ ++ 0x31 0 255 0 0 ++ /* ABS_MT_ORIENTATION, -127, 127, 0, 0 */ ++ 0x34 0xffffff81 127 0 0 ++ /* ABS_MT_TOOL_TYPE, 0, MT_TOOL_MAX, 0, 0 */ ++ 0x37 0 1 0 0 ++ /* ABS_DISTANCE, 0, 255, 0, 0 */ ++ 0x19 0 255 0 0>; ++ ++ cy,vkeys_x = <1404>; ++ cy,vkeys_y = <1872>; ++ cy,revert_x = <0>; ++ cy,revert_y = <0>; ++ cy,xy_exchange = <0>; ++ ++ cy,virtual_keys = ++ /* KeyCode CenterX CenterY Width Height */ ++ /* KEY_BACK */ ++ <158 1360 90 160 180 ++ /* KEY_MENU */ ++ 139 1360 270 160 180 ++ /* KEY_HOMEPAGE */ ++ 172 1360 450 160 180 ++ /* KEY SEARCH */ ++ 217 1360 630 160 180>; ++ }; ++ ++ cy,btn { ++ cy,name = "cyttsp5_btn"; ++ cy,inp_dev_name = "cyttsp5_btn"; ++ }; ++ ++ cy,proximity { ++ cy,name = "cyttsp5_proximity"; ++ cy,inp_dev_name = "cyttsp5_proximity"; ++ cy,abs = ++ /* ABS_DISTANCE, CY_PROXIMITY_MIN_VAL, CY_PROXIMITY_MAX_VAL, 0, 0 */ ++ <0x19 0 1 0 0>; ++ }; ++ }; ++ }; ++}; ++ ++&i2s0_8ch { ++ status = "disabled"; ++}; ++ ++&i2s1_8ch { ++ status = "okay"; ++ rockchip,clk-trcm = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1m0_sclktx ++ &i2s1m0_lrcktx ++ &i2s1m0_sdi0 ++ &i2s1m0_sdo0>; ++}; ++ ++&jpegd { ++ status = "okay"; ++}; ++ ++&jpegd_mmu { ++ status = "okay"; ++}; ++ ++&video_phy0 { ++ status = "disabled"; ++}; ++ ++&mpp_srv { ++ status = "okay"; ++}; ++ ++&nandc0 { ++ status = "disabled"; ++}; ++ ++&pinctrl { ++ wacom { ++ wacom_gpio: wacom-gpio { ++ rockchip,pins = ++ <0 RK_PC0 RK_FUNC_GPIO &pcfg_pull_up>, ++ <0 RK_PC1 RK_FUNC_GPIO &pcfg_pull_up>, ++ <0 RK_PB6 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ tsc { ++ tsc_gpio: tsc-gpio { ++ rockchip,pins = ++ <0 RK_PB5 RK_FUNC_GPIO &pcfg_pull_down>, //touch q gpio ++ <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_up>, ++ <0 RK_PA6 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ tps_pmic { ++ tps65185_gpio: tps65185-gpio { ++ rockchip,pins = ++ <4 RK_PB2 RK_FUNC_GPIO &pcfg_pull_up>, ++ <3 RK_PA5 RK_FUNC_GPIO &pcfg_pull_up>, ++ <3 RK_PA6 RK_FUNC_GPIO &pcfg_pull_up>, ++ <3 RK_PB0 RK_FUNC_GPIO &pcfg_pull_up>, ++ <3 RK_PB1 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ pmic { ++ pmic_int: pmic_int { ++ rockchip,pins = ++ <0 RK_PA3 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ soc_slppin_gpio: soc_slppin_gpio { ++ rockchip,pins = ++ <0 RK_PA2 RK_FUNC_GPIO &pcfg_output_low>; ++ }; ++ ++ soc_slppin_slp: soc_slppin_slp { ++ rockchip,pins = ++ <0 RK_PA2 1 &pcfg_pull_none>; ++ }; ++ ++ soc_slppin_rst: soc_slppin_rst { ++ rockchip,pins = ++ <0 RK_PA2 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <0 RK_PC6 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ wireless-wlan { ++ wifi_vbat: wifi-vbat { ++ rockchip,pins = <0 RK_PA0 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ wifi_host_wake_irq: wifi-host-wake-irq { ++ rockchip,pins = <0 RK_PC5 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart1_gpios: uart1-gpios { ++ rockchip,pins = <2 RK_PB5 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ bt_reset_gpio: bt-reset-gpio { ++ rockchip,pins = <0 RK_PC4 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ bt_wake_gpio: bt-wake-gpio { ++ rockchip,pins = <0 RK_PC2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ bt_irq_gpio: bt-irq-gpio { ++ rockchip,pins = <0 RK_PC3 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ vcc-tp { ++ vcc_tp_en: vcc-tp-en { ++ rockchip,pins = <0 RK_PD6 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ pmuio1-supply = <&vcc3v3_pmu>; ++ pmuio2-supply = <&vcc3v3_pmu>; ++ vccio1-supply = <&vccio_acodec>; ++ vccio3-supply = <&vcc_1v8>; ++ vccio4-supply = <&vcca1v8_pmu>; ++ vccio5-supply = <&vcc_3v3>; ++ vccio6-supply = <&vcc_3v3>; ++ vccio7-supply = <&vcc_1v8>; ++}; ++ ++&pwm0 { ++ status = "okay"; ++}; ++ ++&rk_rga { ++ status = "okay"; ++}; ++ ++&rkisp { ++ status = "okay"; ++}; ++ ++&rkisp_mmu { ++ status = "okay"; ++}; ++ ++&rkisp_vir0 { ++ status = "okay"; ++ ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp0_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&csidphy0_out>; ++ }; ++ }; ++}; ++ ++&rkvdec { ++ status = "okay"; ++}; ++ ++&rkvdec_mmu { ++ status = "okay"; ++}; ++ ++&rkvenc { ++ status = "okay"; ++}; ++ ++&rkvenc_mmu { ++ status = "okay"; ++}; ++ ++&rockchip_suspend { ++ status = "okay"; ++ ++ rockchip,regulator-off-in-mem-lite = ++ <&vdd_cpu>, <&vdd_logic>, <&vdd_gpu>, <&vcc_3v3>, <&vdda_0v9>, <&vcc_1v8>, ++ <&vccio_acodec>, <&vccio_sd>, <&vcc1v8_dvp>, <&dcdc_boost>, <&otg_switch>, ++ <&sleep_sta_ctl>; ++ rockchip,regulator-on-in-mem-lite = ++ <&vcc_ddr>, <&vdda0v9_pmu>, <&vcca1v8_pmu>, <&vcc3v3_pmu>; ++ ++ rockchip,regulator-off-in-mem = ++ <&vdd_cpu>, <&vdd_logic>, <&vdd_gpu>, <&vcc_3v3>, <&vdda_0v9>, <&vcc_1v8>, ++ <&vccio_acodec>, <&vccio_sd>, <&vcc1v8_dvp>, <&dcdc_boost>, <&otg_switch>, ++ <&sleep_sta_ctl>; ++ rockchip,regulator-on-in-mem = ++ <&vcc_ddr>, <&vdda0v9_pmu>, <&vcca1v8_pmu>, <&vcc3v3_pmu>; ++ ++ rockchip,regulator-off-in-mem-ultra = ++ <&vdd_logic>, <&vdd_gpu>, <&vcc_ddr>, <&vcc_3v3>, <&vdda_0v9>, <&vcc_1v8>, ++ <&vdda0v9_pmu>, <&vcca1v8_pmu>, <&vcc3v3_pmu>, <&vccio_acodec>, <&vccio_sd>, ++ <&vcc1v8_dvp>, <&dcdc_boost>, <&otg_switch>; ++ rockchip,regulator-on-in-mem-ultra = <&vdd_cpu>, <&sleep_sta_ctl>; ++}; ++ ++&saradc { ++ status = "okay"; ++ vref-supply = <&vcc_1v8>; ++}; ++ ++&sdhci { ++ bus-width = <8>; ++ no-sdio; ++ no-sd; ++ non-removable; ++ keep-power-in-suspend; ++ max-frequency = <200000000>; ++ status = "okay"; ++}; ++ ++&sdmmc1 { ++ max-frequency = <150000000>; ++ no-sd; ++ no-mmc; ++ bus-width = <4>; ++ disable-wp; ++ cap-sd-highspeed; ++ cap-sdio-irq; ++ keep-power-in-suspend; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ non-removable; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc1_bus4 &sdmmc1_cmd &sdmmc1_clk>; ++ sd-uhs-sdr104; ++ rockchip,default-sample-phase = <90>; ++ status = "okay"; ++}; ++ ++&tsadc { ++ status = "okay"; ++}; ++ ++&uart1 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart1m0_xfer &uart1m0_ctsn>; ++}; ++ ++&u2phy0_otg { ++ status = "okay"; ++}; ++ ++&usb2phy0 { ++ status = "okay"; ++}; ++ ++&usbdrd_dwc3 { ++ status = "okay"; ++}; ++ ++&usbdrd30 { ++ status = "okay"; ++}; ++ ++&vdpu { ++ status = "okay"; ++}; ++ ++&vdpu_mmu { ++ status = "okay"; ++}; ++ ++&vepu { ++ status = "okay"; ++}; ++ ++&vepu_mmu { ++ status = "okay"; ++}; ++ ++&vop { ++ status = "okay"; ++}; ++ ++&vop_mmu { ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3566-rk817-eink-w6.dts b/arch/arm64/boot/dts/rockchip/rk3566-rk817-eink-w6.dts +new file mode 100755 +index 000000000000..2e6d3160aa16 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3566-rk817-eink-w6.dts +@@ -0,0 +1,968 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2021 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++/dts-v1/; ++ ++#include ++#include ++#include ++#include ++#include ++#include "rk3566.dtsi" ++#include "rk3568-android.dtsi" ++#include "rk3566-eink.dtsi" ++ ++/ { ++ model = "Rockchip RK3566 RK817 EINK W6 LP4X Board"; ++ compatible = "rockchip,rk3566-rk817-eink-W6", "rockchip,rk3566"; ++ ++ charge-animation { ++ compatible = "rockchip,uboot-charge"; ++ rockchip,uboot-charge-on = <1>; ++ rockchip,android-charge-on = <0>; ++ rockchip,uboot-low-power-voltage = <3350>; ++ rockchip,screen-on-voltage = <3400>; ++ rockchip,auto-wakeup-interval = <60>; ++ status = "okay"; ++ }; ++ ++ adc_keys: adc-keys { ++ status = "disabled"; ++ compatible = "adc-keys"; ++ io-channels = <&saradc 0>; ++ io-channel-names = "buttons"; ++ keyup-threshold-microvolt = <1800000>; ++ poll-interval = <100>; ++ ++ vol-up-key { ++ label = "volume up"; ++ linux,code = ; ++ press-threshold-microvolt = <1750>; ++ }; ++ }; ++ ++ hdmi_sound: hdmi-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,mclk-fs = <128>; ++ simple-audio-card,name = "rockchip,hdmi"; ++ status = "disabled"; ++ ++ simple-audio-card,cpu { ++ sound-dai = <&i2s0_8ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&hdmi>; ++ }; ++ }; ++ ++ leds: gpio-leds { ++ compatible = "gpio-leds"; ++ pinctrl-names = "default"; ++ pinctrl-0 =<&leds_gpio>; ++ ++ led@1 { ++ gpios = <&gpio3 RK_PC4 GPIO_ACTIVE_HIGH>; ++ linux,default-trigger = "battery-full"; ++ label = "battery_full"; ++ retain-state-suspended; ++ }; ++ ++ led@2 { ++ gpios = <&gpio3 RK_PC5 GPIO_ACTIVE_HIGH>; ++ linux,default-trigger = "battery-charging"; ++ label = "battery_charging"; ++ retain-state-suspended; ++ }; ++ }; ++ ++ vccsys: vccsys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc3v8_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3800000>; ++ regulator-max-microvolt = <3800000>; ++ }; ++ ++ vcc_camera: vcc-camera-regulator { ++ compatible = "regulator-fixed"; ++ gpio = <&gpio4 RK_PC3 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&camera_rst>; ++ regulator-name = "vcc_camera"; ++ enable-active-high; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ vcc_tp: vcc-tp-regulator { ++ compatible = "regulator-fixed"; ++ enable-active-high; ++ gpio = <&gpio0 RK_PC0 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&vcc_tp_en>; ++ regulator-name = "vcc_tp"; ++ }; ++ ++ dummy_codec: dummy-codec { ++ compatible = "rockchip,dummy-codec"; ++ #sound-dai-cells = <0>; ++ }; ++ ++ mic_sound: mic-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,name = "rockchip,rk-mic-sound"; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s1_8ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&dummy_codec>; ++ }; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ clocks = <&rk817 1>; ++ clock-names = "ext_clock"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ post-power-on-delay-ms = <200>; ++ reset-gpios = <&gpio0 RK_PC6 GPIO_ACTIVE_LOW>; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ rockchip,grf = <&grf>; ++ wifi_chip_type = "ap6255"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_vbat &wifi_host_wake_irq>; ++ WIFI,vbat_gpio = <&gpio0 RK_PA0 GPIO_ACTIVE_HIGH>; ++ WIFI,host_wake_irq = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&rk817 1>; ++ clock-names = "ext_clock"; ++ wifi-bt-power-toggle; ++ uart_rts_gpios = <&gpio2 RK_PB5 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart1m0_rtsn>; ++ pinctrl-1 = <&uart1_gpios>; ++ BT,reset_gpio = <&gpio0 RK_PC4 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio0 RK_PC2 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio0 RK_PC3 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++}; ++ ++&ebc { ++ /* clock rate 1000M/n, (n=1~32) */ ++ assigned-clocks = <&cru CPLL_333M>, <&cru DCLK_EBC>; ++ assigned-clock-rates = <85000000>, <85000000>; ++ status = "okay"; ++}; ++ ++&ebc_dev { ++ pmic = <&tps65185>; ++ status = "okay"; ++ ++ /* ED060XCD */ ++ panel,width = <1024>; ++ panel,height = <758>; ++ panel,vir_width = <1024>; ++ panel,vir_height = <758>; ++ panel,sdck = <20000000>; ++ panel,lsl = <6>; ++ panel,lbl = <6>; ++ panel,ldl = <256>; ++ panel,lel = <38>; ++ panel,gdck-sta = <4>; ++ panel,lgonl = <262>; ++ panel,fsl = <2>; ++ panel,fbl = <4>; ++ panel,fdl = <758>; ++ panel,fel = <5>; ++ panel,mirror = <0>; ++ panel,panel_16bit = <0>; ++ panel,panel_color = <0>; ++ panel,width-mm = <90>; ++ panel,height-mm = <122>; ++}; ++ ++&cpu0 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&csi2_dphy_hw { ++ status = "okay"; ++}; ++ ++&csi2_dphy0 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam0: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&ov5648_out>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ csidphy0_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp0_in>; ++ }; ++ }; ++ }; ++}; ++ ++&gpu { ++ mali-supply = <&vdd_gpu>; ++ status = "okay"; ++}; ++ ++&hdmi { ++ status = "disabled"; ++}; ++ ++&hdmi_in_vp0 { ++ status = "disabled"; ++}; ++ ++&hdmi_in_vp1 { ++ status = "disabled"; ++}; ++ ++&hdmi_sound { ++ status = "disabled"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ ++ vdd_cpu: tcs4525@1c { ++ compatible = "tcs,tcs452x"; ++ reg = <0x1c>; ++ vin-supply = <&vccsys>; ++ regulator-compatible = "fan53555-reg"; ++ regulator-name = "vdd_cpu"; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1390000>; ++ regulator-init-microvolt = <900000>; ++ regulator-ramp-delay = <2300>; ++ fcs,suspend-voltage-selector = <0>; ++ regulator-boot-on; ++ regulator-always-on; ++ regulator-initial-mode = <0x2>; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1100000>; ++ regulator-changeable-in-suspend; ++ }; ++ }; ++ ++ rk817: pmic@20 { ++ compatible = "rockchip,rk817"; ++ reg = <0x20>; ++ interrupt-parent = <&gpio0>; ++ interrupts = <3 IRQ_TYPE_LEVEL_LOW>; ++ pinctrl-names = "default"; ++// pinctrl-names = "default", "pmic-sleep", ++// "pmic-power-off", "pmic-reset"; ++ pinctrl-0 = <&pmic_int>; ++// pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; ++// pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; ++// pinctrl-3 = <&soc_slppin_gpio>, <&rk817_slppin_rst>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ //fb-inner-reg-idxs = <2>; ++ /* 1: rst regs (default in codes), 0: rst the pmic */ ++ pmic-reset-func = <0>; ++ /* not save the PMIC_POWER_EN register in uboot */ ++ not-save-power-en = <1>; ++ vcc1-supply = <&vccsys>; ++ vcc2-supply = <&vccsys>; ++ vcc3-supply = <&vccsys>; ++ vcc4-supply = <&vccsys>; ++ vcc5-supply = <&vccsys>; ++ vcc6-supply = <&vccsys>; ++ vcc7-supply = <&vccsys>; ++ vcc8-supply = <&vccsys>; ++ vcc9-supply = <&dcdc_boost>; ++ ++ pwrkey { ++ status = "okay"; ++ }; ++ ++ pinctrl_rk8xx: pinctrl_rk8xx { ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ rk817_slppin_null: rk817_slppin_null { ++ pins = "gpio_slp"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_slppin_slp: rk817_slppin_slp { ++ pins = "gpio_slp"; ++ function = "pin_fun1"; ++ }; ++ ++ rk817_slppin_pwrdn: rk817_slppin_pwrdn { ++ pins = "gpio_slp"; ++ function = "pin_fun2"; ++ }; ++ ++ rk817_slppin_rst: rk817_slppin_rst { ++ pins = "gpio_slp"; ++ function = "pin_fun3"; ++ }; ++ }; ++ ++ regulators { ++ vdd_logic: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <500000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-init-microvolt = <900000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_logic"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <900000>; ++ regulator-changeable-in-suspend; ++ }; ++ }; ++ ++ vdd_gpu: DCDC_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <500000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-init-microvolt = <900000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_gpu"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-changeable-in-suspend; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_ddr"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_3v3: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_3v3"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-changeable-in-suspend; ++ }; ++ }; ++ ++ vcca1v8_pmu: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcca1v8_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ regulator-changeable-in-suspend; ++ }; ++ }; ++ ++ vdda_0v9: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <900000>; ++ regulator-name = "vdda_0v9"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-changeable-in-suspend; ++ }; ++ }; ++ ++ vdda0v9_pmu: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <900000>; ++ regulator-name = "vdda0v9_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <900000>; ++ regulator-changeable-in-suspend; ++ }; ++ }; ++ ++ vccio_acodec: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vccio_acodec"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-changeable-in-suspend; ++ }; ++ }; ++ ++ vccio_sd: LDO_REG5 { ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vccio_sd"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-changeable-in-suspend; ++ }; ++ }; ++ ++ vcc3v3_pmu: LDO_REG6 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcc3v3_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ regulator-changeable-in-suspend; ++ }; ++ }; ++ ++ vcc_1v8: LDO_REG7 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc_1v8"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-changeable-in-suspend; ++ }; ++ }; ++ ++ vcc1v8_dvp: LDO_REG8 { ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc1v8_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-changeable-in-suspend; ++ }; ++ }; ++ ++ sleep_sta_ctl: LDO_REG9 { ++ regulator-name = "sleep_sta_ctl"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ regulator-changeable-in-suspend; ++ }; ++ }; ++ ++ dcdc_boost: BOOST { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <4700000>; ++ regulator-max-microvolt = <5400000>; ++ regulator-name = "boost"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-changeable-in-suspend; ++ }; ++ }; ++ ++ otg_switch: OTG_SWITCH { ++ regulator-name = "otg_switch"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-changeable-in-suspend; ++ }; ++ }; ++ }; ++ ++ battery { ++ compatible = "rk817,battery"; ++ ocv_table = <3400 3513 3578 3687 3734 3752 3763 ++ 3766 3771 3784 3804 3836 3885 3925 ++ 3962 4005 4063 4114 4169 4227 4303>; ++ design_capacity = <2250>; ++ design_qmax = <2750>; ++ bat_res = <100>; ++ sleep_enter_current = <150>; ++ sleep_exit_current = <180>; ++ sleep_filter_current = <100>; ++ power_off_thresd = <3450>; ++ zero_algorithm_vol = <3850>; ++ max_soc_offset = <60>; ++ monitor_sec = <5>; ++ sample_res = <10>; ++ virtual_power = <0>; ++ low_power_sleep = <1>; ++ }; ++ ++ charger { ++ compatible = "rk817,charger"; ++ min_input_voltage = <4500>; ++ max_input_current = <1500>; ++ max_chrg_current = <2000>; ++ max_chrg_voltage = <4300>; ++ chrg_term_mode = <0>; ++ chrg_finish_cur = <300>; ++ virtual_power = <0>; ++ dc_det_adc = <0>; ++ extcon = <&usb2phy0>; ++ gate_function_disable = <1>; ++ }; ++ }; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ ++ ov5648: ov5648@36 { ++ status = "okay"; ++ compatible = "ovti,ov5648"; ++ reg = <0x36>; ++ clocks = <&cru CLK_CAM0_OUT>; ++ clock-names = "xvclk"; ++ /* avdd-supply = <&vcc2v8_dvp>; */ ++ dovdd-supply = <&vcc1v8_dvp>; ++ /* dvdd-supply = <&vcc1v8_dvp>; */ ++ ++ power-domains = <&power RK3568_PD_VI>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cam_clkout0>; ++ //reset pin control by hardware,used this pin switch to mipi input ++ //1->2LANE(LANE 0&1) FRONT camera, 0->4LANE REAR camera ++ //reset-gpios = <&gpio4 RK_PC3 GPIO_ACTIVE_LOW>; ++ pwdn-gpios = <&gpio4 RK_PC2 GPIO_ACTIVE_HIGH>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "HS5885-BNSM1018-V01"; ++ rockchip,camera-module-lens-name = "default"; ++ port { ++ ov5648_out: endpoint { ++ remote-endpoint = <&mipi_in_ucam0>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++}; ++ ++&i2c3 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2c3m1_xfer>; ++ ++ tps65185: tps65185@68 { ++ compatible = "ti,tps65185"; ++ reg = <0x68>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&tps65185_gpio>; ++ int-gpios = <&gpio3 RK_PA6 GPIO_ACTIVE_HIGH>; ++ wakeup-gpios = <&gpio3 RK_PA5 GPIO_ACTIVE_HIGH>; ++ vcomctl-gpios = <&gpio4 RK_PB2 GPIO_ACTIVE_HIGH>; ++ powerup-gpios = <&gpio3 RK_PB0 GPIO_ACTIVE_HIGH>; ++ poweren-gpios = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; ++ }; ++ ++}; ++ ++&i2c5 { ++ status = "okay"; ++ ++ sensor@4c { ++ status = "okay"; ++ compatible = "gs_mma7660"; ++ reg = <0x4c>; ++ type = ; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ layout = <6>; ++ reprobe_en = <1>; ++ }; ++ ++ gt9xx: gt9xx@14 { ++ compatible = "goodix,gt9xx"; ++ reg = <0x14>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&tp_gpio>; ++ touch-gpio = <&gpio0 RK_PA6 GPIO_ACTIVE_HIGH>; ++ reset-gpio = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>; ++ touchscreen-size-x = <1024>; ++ touchscreen-size-y = <758>; ++ max-x = <1024>; ++ max-y = <758>; ++ tp-size = <9111>; ++ tp-supply = <&vcc_tp>; ++ wakeup-source; ++ touchscreen-key-map = <158>; //KEY_HOMEPAGE=172,KEY_BACK=158,KEY_MENU=139 ++ goodix,driver-send-cfg = <0>; ++ goodix,cfg-group0 =[ ++ 42 00 03 00 04 0A 45 03 22 1F 28 0F 64 3C 03 0F 00 00 00 00 11 00 ++ 08 00 00 00 00 8B 29 0E 71 6F B2 04 00 00 00 39 02 10 00 21 00 00 ++ 00 03 64 32 00 00 00 3C 78 94 D5 02 07 00 00 04 C8 40 00 B1 4A 00 ++ 9E 55 00 8E 61 00 7F 70 00 7F 70 00 00 00 F0 90 3C FF FF 07 00 00 ++ 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ++ 00 00 1C 1A 18 16 14 12 10 0E 0C 0A 08 06 04 02 FF FF FF FF FF FF ++ FF FF FF FF FF FF FF FF FF FF 00 02 04 06 08 0A 0C 0F 10 12 13 16 ++ 18 1C 1D 1E 1F 20 21 22 FF FF FF FF FF FF FF FF FF FF FF FF FF FF ++ FF FF FF FF FF FF FF FF F6 01 ++ ]; ++ }; ++ ++ ft5436: focaltech@38 { ++ status = "okay"; ++ compatible = "focaltech,ft5436"; ++ reg = <0x38>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&tp_gpio>; ++ vdd-supply = <&vcc_tp>; ++ focaltech,reset-gpio = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>; ++ focaltech,irq-gpio = <&gpio0 RK_PA6 IRQ_TYPE_EDGE_FALLING>; ++ focaltech,max-touch-number = <5>; ++ focaltech,display-coords = <0 0 1024 758>; ++ focaltech,have-key = <1>; ++ focaltech,key-number = <1>; ++ ++ focaltech,key-x-coords = <300>; ++ focaltech,key-y-coords = <1200>; ++ wakeup-source; ++ }; ++}; ++ ++&i2s0_8ch { ++ status = "disabled"; ++}; ++ ++&i2s1_8ch { ++ status = "okay"; ++ rockchip,clk-trcm = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1m0_sclktx ++ &i2s1m0_lrcktx ++ &i2s1m0_sdi0 ++ &i2s1m0_sdo0>; ++}; ++ ++&jpegd { ++ status = "okay"; ++}; ++ ++&jpegd_mmu { ++ status = "okay"; ++}; ++ ++&video_phy0 { ++ status = "disabled"; ++}; ++ ++&mpp_srv { ++ status = "okay"; ++}; ++ ++&nandc0 { ++ status = "disabled"; ++}; ++ ++&pinctrl { ++ cam { ++ cam_clkout0: cam-clkout0 { ++ rockchip,pins = ++ /* cam_clkout0 */ ++ <4 RK_PA7 1 &pcfg_pull_none>; ++ }; ++ ++ camera_rst: camera-rst { ++ rockchip,pins = ++ <4 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ leds { ++ leds_gpio: leds-gpio { ++ rockchip,pins = ++ <3 RK_PC4 RK_FUNC_GPIO &pcfg_pull_none>, ++ <3 RK_PC5 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ tps_pmic { ++ tps65185_gpio: tps65185-gpio { ++ rockchip,pins = ++ <4 RK_PB2 RK_FUNC_GPIO &pcfg_pull_up>, ++ <3 RK_PA5 RK_FUNC_GPIO &pcfg_pull_up>, ++ <3 RK_PA6 RK_FUNC_GPIO &pcfg_pull_up>, ++ <3 RK_PB0 RK_FUNC_GPIO &pcfg_pull_up>, ++ <0 RK_PC1 RK_FUNC_GPIO &pcfg_output_high>; ++ }; ++ }; ++ ++ tp { ++ tp_gpio: tp-gpio { ++ rockchip,pins = ++ <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_up>, ++ <0 RK_PA6 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ pmic { ++ pmic_int: pmic_int { ++ rockchip,pins = ++ <0 RK_PA3 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ soc_slppin_gpio: soc_slppin_gpio { ++ rockchip,pins = ++ <0 RK_PA2 RK_FUNC_GPIO &pcfg_output_low>; ++ }; ++ ++ soc_slppin_slp: soc_slppin_slp { ++ rockchip,pins = ++ <0 RK_PA2 1 &pcfg_pull_none>; ++ }; ++ ++ soc_slppin_rst: soc_slppin_rst { ++ rockchip,pins = ++ <0 RK_PA2 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <0 RK_PC6 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ vcc-tp { ++ vcc_tp_en: vcc-tp-en { ++ rockchip,pins = <0 RK_PC0 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ wireless-wlan { ++ wifi_vbat: wifi-vbat { ++ rockchip,pins = <0 RK_PA0 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ wifi_host_wake_irq: wifi-host-wake-irq { ++ rockchip,pins = <0 RK_PC5 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart1_gpios: uart1-gpios { ++ rockchip,pins = <2 RK_PB5 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ pmuio1-supply = <&vcc3v3_pmu>; ++ pmuio2-supply = <&vcca1v8_pmu>; ++ vccio1-supply = <&vccio_acodec>; ++ vccio3-supply = <&vcc_3v3>; ++ vccio4-supply = <&vcca1v8_pmu>; ++ vccio5-supply = <&vcc_3v3>; ++ vccio6-supply = <&vcc_3v3>; ++ vccio7-supply = <&vcc1v8_dvp>; ++}; ++ ++&pwm4 { ++ status = "disabled"; ++}; ++ ++&rk_rga { ++ status = "okay"; ++}; ++ ++&rkisp { ++ status = "okay"; ++}; ++ ++&rkisp_mmu { ++ status = "okay"; ++}; ++ ++&rkisp_vir0 { ++ status = "okay"; ++ ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp0_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&csidphy0_out>; ++ }; ++ }; ++}; ++ ++&rkvdec { ++ status = "okay"; ++}; ++ ++&rkvdec_mmu { ++ status = "okay"; ++}; ++ ++&rkvenc { ++ status = "okay"; ++}; ++ ++&rkvenc_mmu { ++ status = "okay"; ++}; ++ ++&rockchip_suspend { ++ status = "okay"; ++ ++ rockchip,regulator-off-in-mem-lite = ++ <&vdd_cpu>, <&vdd_logic>, <&vdd_gpu>, <&vcc_3v3>, <&vdda_0v9>, <&vcc_1v8>, ++ <&vccio_acodec>, <&vccio_sd>, <&vcc1v8_dvp>, <&dcdc_boost>, <&otg_switch>, ++ <&sleep_sta_ctl>; ++ rockchip,regulator-on-in-mem-lite = ++ <&vcc_ddr>, <&vdda0v9_pmu>, <&vcca1v8_pmu>, <&vcc3v3_pmu>; ++ ++ rockchip,regulator-off-in-mem = ++ <&vdd_cpu>, <&vdd_logic>, <&vdd_gpu>, <&vcc_3v3>, <&vdda_0v9>, <&vcc_1v8>, ++ <&vccio_acodec>, <&vccio_sd>, <&vcc1v8_dvp>, <&dcdc_boost>, <&otg_switch>, ++ <&sleep_sta_ctl>; ++ rockchip,regulator-on-in-mem = ++ <&vcc_ddr>, <&vdda0v9_pmu>, <&vcca1v8_pmu>, <&vcc3v3_pmu>; ++ ++ rockchip,regulator-off-in-mem-ultra = ++ <&vdd_logic>, <&vdd_gpu>, <&vcc_ddr>, <&vcc_3v3>, <&vdda_0v9>, <&vcc_1v8>, ++ <&vdda0v9_pmu>, <&vcca1v8_pmu>, <&vcc3v3_pmu>, <&vccio_acodec>, <&vccio_sd>, ++ <&vcc1v8_dvp>, <&dcdc_boost>, <&otg_switch>; ++ rockchip,regulator-on-in-mem-ultra = <&vdd_cpu>, <&sleep_sta_ctl>; ++}; ++ ++&saradc { ++ status = "disabled"; ++ vref-supply = <&vcc_1v8>; ++}; ++ ++&sdhci { ++ bus-width = <8>; ++ no-sdio; ++ no-sd; ++ non-removable; ++ keep-power-in-suspend; ++ max-frequency = <200000000>; ++ status = "okay"; ++}; ++ ++&sdmmc1 { ++ max-frequency = <150000000>; ++ no-sd; ++ no-mmc; ++ bus-width = <4>; ++ disable-wp; ++ cap-sd-highspeed; ++ cap-sdio-irq; ++ keep-power-in-suspend; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ non-removable; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc1_bus4 &sdmmc1_cmd &sdmmc1_clk>; ++ sd-uhs-sdr104; ++ rockchip,default-sample-phase = <90>; ++ status = "okay"; ++}; ++ ++&tsadc { ++ status = "okay"; ++}; ++ ++&uart1 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart1m0_xfer &uart1m0_ctsn>; ++}; ++ ++&u2phy0_otg { ++ status = "okay"; ++}; ++ ++&usb2phy0 { ++ status = "okay"; ++}; ++ ++&usbdrd_dwc3 { ++ status = "okay"; ++}; ++ ++&usbdrd30 { ++ status = "okay"; ++}; ++ ++&vdpu { ++ status = "okay"; ++}; ++ ++&vdpu_mmu { ++ status = "okay"; ++}; ++ ++&vepu { ++ status = "okay"; ++}; ++ ++&vepu_mmu { ++ status = "okay"; ++}; ++ ++&vop { ++ status = "okay"; ++}; ++ ++&vop_mmu { ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3566-rk817-eink.dts b/arch/arm64/boot/dts/rockchip/rk3566-rk817-eink.dts +new file mode 100755 +index 000000000000..50412b44bb71 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3566-rk817-eink.dts +@@ -0,0 +1,957 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++/dts-v1/; ++ ++#include ++#include ++#include ++#include ++#include ++#include "rk3566.dtsi" ++#include "rk3568-android.dtsi" ++#include "rk3566-eink.dtsi" ++ ++/ { ++ model = "Rockchip RK3566 RK817 EINK LP4X Board"; ++ compatible = "rockchip,rk3566-rk817-eink", "rockchip,rk3566"; ++ ++ charge-animation { ++ compatible = "rockchip,uboot-charge"; ++ rockchip,uboot-charge-on = <1>; ++ rockchip,android-charge-on = <0>; ++ rockchip,uboot-low-power-voltage = <3350>; ++ rockchip,screen-on-voltage = <3400>; ++ rockchip,auto-wakeup-interval = <60>; ++ status = "okay"; ++ }; ++ ++ adc_keys: adc-keys { ++ status = "disabled"; ++ compatible = "adc-keys"; ++ io-channels = <&saradc 0>; ++ io-channel-names = "buttons"; ++ keyup-threshold-microvolt = <1800000>; ++ poll-interval = <100>; ++ ++ vol-up-key { ++ label = "volume up"; ++ linux,code = ; ++ press-threshold-microvolt = <1750>; ++ }; ++ ++ vol-down-key { ++ label = "volume down"; ++ linux,code = ; ++ press-threshold-microvolt = <297500>; ++ }; ++ }; ++ ++ hdmi_sound: hdmi-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,mclk-fs = <128>; ++ simple-audio-card,name = "rockchip,hdmi"; ++ status = "disabled"; ++ ++ simple-audio-card,cpu { ++ sound-dai = <&i2s0_8ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&hdmi>; ++ }; ++ }; ++ ++ vccsys: vccsys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc3v8_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3800000>; ++ regulator-max-microvolt = <3800000>; ++ }; ++ ++ rk817-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,rk817-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ ++ simple-audio-card,cpu { ++ sound-dai = <&i2s1_8ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&rk817_codec>; ++ }; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ status = "disabled"; ++ compatible = "mmc-pwrseq-simple"; ++ clocks = <&rk817 1>; ++ clock-names = "ext_clock"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ post-power-on-delay-ms = <200>; ++ reset-gpios = <&gpio2 RK_PB1 GPIO_ACTIVE_LOW>; ++ }; ++ ++ vcc_sd: vcc-sd { ++ compatible = "regulator-gpio"; ++ enable-active-low; ++ enable-gpio = <&gpio0 RK_PA5 GPIO_ACTIVE_LOW>; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&vcc_sd_h>; ++ regulator-name = "vcc_sd"; ++ states = <3300000 0x0 ++ 3300000 0x1>; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ rockchip,grf = <&grf>; ++ wifi_chip_type = "ap6255"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_host_wake_irq>; ++ WIFI,host_wake_irq = <&gpio2 RK_PB2 GPIO_ACTIVE_HIGH>; ++ status = "disabled"; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&rk817 1>; ++ clock-names = "ext_clock"; ++ //wifi-bt-power-toggle; ++ uart_rts_gpios = <&gpio2 RK_PB5 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart1m0_rtsn>; ++ pinctrl-1 = <&uart1_gpios>; ++ BT,reset_gpio = <&gpio2 RK_PB7 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio2 RK_PC1 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio2 RK_PC0 GPIO_ACTIVE_HIGH>; ++ status = "disabled"; ++ }; ++}; ++ ++&ebc { ++ /* clock rate 1000M/n, (n=1~32) */ ++ assigned-clocks = <&cru CPLL_333M>, <&cru DCLK_EBC>; ++ assigned-clock-rates = <250000000>, <250000000>; ++ status = "okay"; ++}; ++ ++&ebc_dev { ++ pmic = <&tps65185>; ++ status = "okay"; ++#if 0 ++ /* ED097TC2U1 */ ++ panel,width = <1200>; ++ panel,height = <825>; ++ panel,vir_width = <1200>; ++ panel,vir_height = <825>; ++ panel,sdck = <25000000>; ++ panel,lsl = <4>; ++ panel,lbl = <4>; ++ panel,ldl = <300>; ++ panel,lel = <36>; ++ panel,gdck-sta = <18>; ++ panel,lgonl = <265>; ++ panel,fsl = <2>; ++ panel,fbl = <4>; ++ panel,fdl = <825>; ++ panel,fel = <24>; ++ panel,mirror = <0>; ++ panel,panel_16bit = <0>; ++ panel,panel_color = <0>; ++ panel,width-mm = <203>; ++ panel,height-mm = <140>; ++#else ++ /* ES103TC1 */ ++ panel,width = <1872>; ++ panel,height = <1404>; ++ panel,vir_width = <1872>; ++ panel,vir_height = <1404>; ++ panel,sdck = <33300000>; ++ panel,lsl = <18>; ++ panel,lbl = <17>; ++ panel,ldl = <234>; ++ panel,lel = <7>; ++ panel,gdck-sta = <34>; ++ panel,lgonl = <192>; ++ panel,fsl = <1>; ++ panel,fbl = <4>; ++ panel,fdl = <1404>; ++ panel,fel = <12>; ++ panel,mirror = <0>; ++ panel,panel_16bit = <1>; ++ panel,panel_color = <0>; ++ panel,width-mm = <157>; ++ panel,height-mm = <210>; ++#endif ++}; ++ ++&cpu0 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&gpu { ++ mali-supply = <&vdd_gpu>; ++ status = "okay"; ++}; ++ ++&hdmi { ++ status = "disabled"; ++}; ++ ++&hdmi_in_vp0 { ++ status = "disabled"; ++}; ++ ++&hdmi_in_vp1 { ++ status = "disabled"; ++}; ++ ++&hdmi_sound { ++ status = "disabled"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ ++ vdd_cpu: tcs4525@1c { ++ compatible = "tcs,tcs452x"; ++ reg = <0x1c>; ++ vin-supply = <&vccsys>; ++ regulator-compatible = "fan53555-reg"; ++ regulator-name = "vdd_cpu"; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1390000>; ++ regulator-init-microvolt = <900000>; ++ regulator-ramp-delay = <2300>; ++ fcs,suspend-voltage-selector = <1>; ++ regulator-boot-on; ++ regulator-always-on; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ rk817: pmic@20 { ++ compatible = "rockchip,rk817"; ++ reg = <0x20>; ++ interrupt-parent = <&gpio0>; ++ interrupts = <3 IRQ_TYPE_LEVEL_LOW>; ++ ++ pinctrl-names = "default", "pmic-sleep", ++ "pmic-power-off", "pmic-reset"; ++ pinctrl-0 = <&pmic_int>; ++ pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; ++ pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; ++ pinctrl-3 = <&soc_slppin_gpio>, <&rk817_slppin_rst>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ //fb-inner-reg-idxs = <2>; ++ /* 1: rst regs (default in codes), 0: rst the pmic */ ++ pmic-reset-func = <0>; ++ /* not save the PMIC_POWER_EN register in uboot */ ++ not-save-power-en = <1>; ++ vcc1-supply = <&vccsys>; ++ vcc2-supply = <&vccsys>; ++ vcc3-supply = <&vccsys>; ++ vcc4-supply = <&vccsys>; ++ vcc5-supply = <&vccsys>; ++ vcc6-supply = <&vccsys>; ++ vcc7-supply = <&vccsys>; ++ vcc8-supply = <&vccsys>; ++ vcc9-supply = <&dcdc_boost>; ++ ++ pwrkey { ++ status = "okay"; ++ }; ++ ++ pinctrl_rk8xx: pinctrl_rk8xx { ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ rk817_slppin_null: rk817_slppin_null { ++ pins = "gpio_slp"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_slppin_slp: rk817_slppin_slp { ++ pins = "gpio_slp"; ++ function = "pin_fun1"; ++ }; ++ ++ rk817_slppin_pwrdn: rk817_slppin_pwrdn { ++ pins = "gpio_slp"; ++ function = "pin_fun2"; ++ }; ++ ++ rk817_slppin_rst: rk817_slppin_rst { ++ pins = "gpio_slp"; ++ function = "pin_fun3"; ++ }; ++ }; ++ ++ regulators { ++ vdd_logic: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <500000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-init-microvolt = <900000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_logic"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <900000>; ++ }; ++ }; ++ ++ vdd_gpu: DCDC_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <500000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-init-microvolt = <900000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_gpu"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_ddr"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_3v3: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_3v3"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcca1v8_pmu: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcca1v8_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdda_0v9: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <900000>; ++ regulator-name = "vdda_0v9"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdda0v9_pmu: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <900000>; ++ regulator-name = "vdda0v9_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <900000>; ++ }; ++ }; ++ ++ vccio_acodec: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vccio_acodec"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vccio_sd: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vccio_sd"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc3v3_pmu: LDO_REG6 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcc3v3_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc_1v8: LDO_REG7 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc_1v8"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc1v8_dvp: LDO_REG8 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc1v8_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc2v8_dvp: LDO_REG9 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <2800000>; ++ regulator-max-microvolt = <2800000>; ++ regulator-name = "vcc2v8_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ dcdc_boost: BOOST { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <4700000>; ++ regulator-max-microvolt = <5400000>; ++ regulator-name = "boost"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ otg_switch: OTG_SWITCH { ++ regulator-name = "otg_switch"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ }; ++ ++ battery { ++ compatible = "rk817,battery"; ++ ocv_table = <3400 3513 3578 3687 3734 3752 3763 ++ 3766 3771 3784 3804 3836 3885 3925 ++ 3962 4005 4063 4114 4169 4227 4303>; ++ design_capacity = <5000>; ++ design_qmax = <5500>; ++ bat_res = <100>; ++ sleep_enter_current = <150>; ++ sleep_exit_current = <180>; ++ sleep_filter_current = <100>; ++ power_off_thresd = <3450>; ++ zero_algorithm_vol = <3850>; ++ max_soc_offset = <60>; ++ monitor_sec = <5>; ++ sample_res = <10>; ++ virtual_power = <0>; ++ }; ++ ++ charger { ++ compatible = "rk817,charger"; ++ min_input_voltage = <4500>; ++ max_input_current = <1500>; ++ max_chrg_current = <2000>; ++ max_chrg_voltage = <4300>; ++ chrg_term_mode = <0>; ++ chrg_finish_cur = <300>; ++ virtual_power = <0>; ++ dc_det_adc = <0>; ++ extcon = <&usb2phy0>; ++ gate_function_disable = <1>; ++ }; ++ ++ rk817_codec: codec { ++ #sound-dai-cells = <0>; ++ compatible = "rockchip,rk817-codec"; ++ clocks = <&cru I2S1_MCLKOUT>; ++ clock-names = "mclk"; ++ assigned-clocks = <&cru I2S1_MCLKOUT>, <&cru I2S1_MCLK_TX_IOE>; ++ assigned-clock-rates = <12288000>; ++ assigned-clock-parents = <&cru I2S1_MCLKOUT_TX>, <&cru I2S1_MCLKOUT_TX>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1m0_mclk>; ++ hp-volume = <20>; ++ spk-volume = <3>; ++ out-l2spk-r2hp; ++ spk-ctl-gpios = <&gpio0 RK_PA6 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ }; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ tps65185: tps65185@68 { ++ status = "okay"; ++ compatible = "ti,tps65185"; ++ reg = <0x68>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&tps65185_gpio>; ++ int-gpios = <&gpio3 RK_PA6 GPIO_ACTIVE_HIGH>; ++ wakeup-gpios = <&gpio3 RK_PA5 GPIO_ACTIVE_HIGH>; ++ vcomctl-gpios = <&gpio4 RK_PB2 GPIO_ACTIVE_HIGH>; ++ powerup-gpios = <&gpio3 RK_PB0 GPIO_ACTIVE_HIGH>; ++ }; ++}; ++ ++&i2c2 { ++ status = "okay"; ++ wacom: wacom@9 { ++ compatible = "wacom,w9013"; ++ reg = <0x09>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wacom_gpio>; ++ gpio_detect = <&gpio3 RK_PA3 GPIO_ACTIVE_HIGH>; ++ gpio_intr = <&gpio3 RK_PA1 GPIO_ACTIVE_HIGH>; ++ gpio_rst = <&gpio3 RK_PA2 GPIO_ACTIVE_HIGH>; ++ revert_x = <0>; ++ revert_y = <0>; ++ xy_exchange = <0>; ++ }; ++}; ++ ++&i2c3 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2c3m1_xfer>; ++ ++ ts@40 { ++ compatible = "gslX680-pad"; ++ reg = <0x40>; ++ touch-gpio = <&gpio3 RK_PB0 IRQ_TYPE_LEVEL_HIGH>; ++ reset-gpio = <&gpio3 RK_PB1 IRQ_TYPE_LEVEL_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&tp_gpio>; ++ screen_max_x = <1200>; ++ screen_max_y = <1920>; ++ revert_x = <0>; ++ revert_y = <1>; ++ revert_xy = <0>; ++ chip_id = <1>; ++ status = "disabled"; ++ }; ++ ++ tsc@24 { ++ status = "okay"; ++ compatible = "cy,cyttsp5_i2c_adapter"; ++ reg = <0x24>; ++ cy,adapter_id = "cyttsp5_i2c_adapter"; ++ //cytp-supply = <&vcc_sd>; ++ cy,core { ++ cy,name = "cyttsp5_core"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&tsc_gpio>; ++ cy,irq_gpio = <&gpio3 RK_PB1 GPIO_ACTIVE_HIGH>; ++ cy,rst_gpio = <&gpio3 RK_PB2 GPIO_ACTIVE_HIGH>; ++ cy,1v8_gpio = <&gpio3 RK_PC4 GPIO_ACTIVE_HIGH>; ++ cy,2v8_gpio = <&gpio3 RK_PC5 GPIO_ACTIVE_HIGH>; ++ cy,hid_desc_register = <1>; ++ /* CY_CORE_FLAG_RESTORE_PARAMETERS */ ++ cy,flags = <6>; ++ /* CY_CORE_EWG_NONE */ ++ cy,easy_wakeup_gesture = <0>; ++ cy,btn_keys = <172 /* KEY_HOMEPAGE */ ++ /* previously was KEY_HOME, new Android versions use KEY_HOMEPAGE */ ++ 139 /* KEY_MENU */ ++ 158 /* KEY_BACK */ ++ 217 /* KEY_SEARCH */ ++ 114 /* KEY_VOLUMEDOWN */ ++ 115 /* KEY_VOLUMEUP */ ++ 212 /* KEY_CAMERA */ ++ 116>; /* KEY_POWER */ ++ cy,btn_keys-tag = <0>; ++ cy,mt { ++ cy,name = "cyttsp5_mt"; ++ cy,inp_dev_name = "cyttsp5_mt"; ++ cy,flags = <0>; ++ cy,abs = ++ /* ABS_MT_POSITION_X, CY_ABS_MIN_X, CY_ABS_MAX_X, 0, 0 */ ++ <0x35 0 1872 0 0 ++ /* ABS_MT_POSITION_Y, CY_ABS_MIN_Y, CY_ABS_MAX_Y, 0, 0 */ ++ 0x36 0 1404 0 0 ++ /* ABS_MT_PRESSURE, CY_ABS_MIN_P, CY_ABS_MAX_P, 0, 0 */ ++ 0x3a 0 255 0 0 ++ /* CY_IGNORE_VALUE, CY_ABS_MIN_W, CY_ABS_MAX_W, 0, 0 */ ++ 0xffff 0 255 0 0 ++ /* ABS_MT_TRACKING_ID, CY_ABS_MIN_T, CY_ABS_MAX_T, 0, 0 */ ++ 0x39 0 15 0 0 ++ /* ABS_MT_TOUCH_MAJOR, 0, 255, 0, 0 */ ++ 0x30 0 255 0 0 ++ /* ABS_MT_TOUCH_MINOR, 0, 255, 0, 0 */ ++ 0x31 0 255 0 0 ++ /* ABS_MT_ORIENTATION, -127, 127, 0, 0 */ ++ 0x34 0xffffff81 127 0 0 ++ /* ABS_MT_TOOL_TYPE, 0, MT_TOOL_MAX, 0, 0 */ ++ 0x37 0 1 0 0 ++ /* ABS_DISTANCE, 0, 255, 0, 0 */ ++ 0x19 0 255 0 0>; ++ ++ cy,vkeys_x = <1872>; ++ cy,vkeys_y = <1404>; ++ cy,revert_x = <0>; ++ cy,revert_y = <1>; ++ cy,xy_exchange = <0>; ++ ++ cy,virtual_keys = ++ /* KeyCode CenterX CenterY Width Height */ ++ /* KEY_BACK */ ++ <158 1360 90 160 180 ++ /* KEY_MENU */ ++ 139 1360 270 160 180 ++ /* KEY_HOMEPAGE */ ++ 172 1360 450 160 180 ++ /* KEY SEARCH */ ++ 217 1360 630 160 180>; ++ }; ++ ++ cy,btn { ++ cy,name = "cyttsp5_btn"; ++ cy,inp_dev_name = "cyttsp5_btn"; ++ }; ++ ++ cy,proximity { ++ cy,name = "cyttsp5_proximity"; ++ cy,inp_dev_name = "cyttsp5_proximity"; ++ cy,abs = ++ /* ABS_DISTANCE, CY_PROXIMITY_MIN_VAL, CY_PROXIMITY_MAX_VAL, 0, 0 */ ++ <0x19 0 1 0 0>; ++ }; ++ }; ++ }; ++}; ++ ++&i2c5 { ++ status = "disabled"; ++ ++ kxtj: kxtj3@e { ++ status = "disabled"; ++ compatible = "gs_kxtj9"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&kxtj3_irq_gpio>; ++ reg = <0x0e>; ++ irq-gpio = <&gpio4 RK_PC6 IRQ_TYPE_EDGE_RISING>; ++ type = ; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ power-off-in-suspend = <1>; ++ layout = <5>; ++ }; ++}; ++ ++&i2s0_8ch { ++ status = "disabled"; ++}; ++ ++&i2s1_8ch { ++ status = "okay"; ++ rockchip,clk-trcm = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1m0_sclktx ++ &i2s1m0_lrcktx ++ &i2s1m0_sdi0 ++ &i2s1m0_sdo0>; ++}; ++ ++&jpegd { ++ status = "okay"; ++}; ++ ++&jpegd_mmu { ++ status = "okay"; ++}; ++ ++&video_phy0 { ++ status = "disabled"; ++}; ++ ++&mpp_srv { ++ status = "okay"; ++}; ++ ++&nandc0 { ++ status = "disabled"; ++}; ++ ++&pinctrl { ++ wacom { ++ wacom_gpio: wacom-gpio { ++ rockchip,pins = ++ <3 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>, ++ <3 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>, ++ <3 RK_PA3 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ tsc { ++ tsc_gpio: tsc-gpio { ++ rockchip,pins = ++ <3 RK_PB1 RK_FUNC_GPIO &pcfg_pull_up>, ++ <3 RK_PB2 RK_FUNC_GPIO &pcfg_pull_up>, ++ <3 RK_PC4 RK_FUNC_GPIO &pcfg_pull_up>, ++ <3 RK_PC5 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ tps_pmic { ++ tps65185_gpio: tps65185-gpio { ++ rockchip,pins = ++ <4 RK_PB2 RK_FUNC_GPIO &pcfg_pull_up>, ++ <3 RK_PA5 RK_FUNC_GPIO &pcfg_pull_up>, ++ <3 RK_PA6 RK_FUNC_GPIO &pcfg_pull_up>, ++ <3 RK_PB0 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ tp { ++ tp_gpio: tp-gpio { ++ rockchip,pins = ++ <3 RK_PB0 RK_FUNC_GPIO &pcfg_pull_up>, ++ <3 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ headphone { ++ hp_det: hp-det { ++ rockchip,pins = <4 RK_PC4 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ sensor { ++ kxtj3_irq_gpio: kxtj3-irq-gpio { ++ rockchip,pins = <4 RK_PC6 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ pmic { ++ pmic_int: pmic_int { ++ rockchip,pins = ++ <0 RK_PA3 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ soc_slppin_gpio: soc_slppin_gpio { ++ rockchip,pins = ++ <0 RK_PA2 RK_FUNC_GPIO &pcfg_output_low>; ++ }; ++ ++ soc_slppin_slp: soc_slppin_slp { ++ rockchip,pins = ++ <0 RK_PA2 1 &pcfg_pull_none>; ++ }; ++ ++ soc_slppin_rst: soc_slppin_rst { ++ rockchip,pins = ++ <0 RK_PA2 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <2 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ vcc_sd { ++ vcc_sd_h: vcc-sd-h { ++ rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ wireless-wlan { ++ wifi_host_wake_irq: wifi-host-wake-irq { ++ rockchip,pins = <2 RK_PB2 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart1_gpios: uart1-gpios { ++ rockchip,pins = <2 RK_PB5 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ pmuio1-supply = <&vcc3v3_pmu>; ++ pmuio2-supply = <&vcc3v3_pmu>; ++ vccio1-supply = <&vccio_acodec>; ++ vccio3-supply = <&vccio_sd>; ++ vccio4-supply = <&vcc_3v3>; ++ vccio5-supply = <&vcc_3v3>; ++ vccio6-supply = <&vcc_3v3>; ++ vccio7-supply = <&vcc_3v3>; ++}; ++ ++&pwm4 { ++ status = "disabled"; ++}; ++ ++&rk_rga { ++ status = "okay"; ++}; ++ ++&rkvdec { ++ status = "okay"; ++}; ++ ++&rkvdec_mmu { ++ status = "okay"; ++}; ++ ++&rkvenc { ++ status = "okay"; ++}; ++ ++&rkvenc_mmu { ++ status = "okay"; ++}; ++ ++&saradc { ++ status = "disabled"; ++ vref-supply = <&vcc_1v8>; ++}; ++ ++&sdhci { ++ bus-width = <8>; ++ no-sdio; ++ no-sd; ++ non-removable; ++ keep-power-in-suspend; ++ max-frequency = <200000000>; ++ status = "okay"; ++}; ++ ++&sdmmc0 { ++ max-frequency = <50000000>; ++ no-sdio; ++ no-mmc; ++ bus-width = <4>; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ disable-wp; ++ sd-uhs-sdr104; ++ vmmc-supply = <&vcc_sd>; ++ vqmmc-supply = <&vccio_sd>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc0_bus4 &sdmmc0_clk &sdmmc0_cmd &sdmmc0_det>; ++ status = "okay"; ++}; ++ ++&sdmmc1 { ++ max-frequency = <150000000>; ++ no-sd; ++ no-mmc; ++ bus-width = <4>; ++ disable-wp; ++ cap-sd-highspeed; ++ cap-sdio-irq; ++ keep-power-in-suspend; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ non-removable; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc1_bus4 &sdmmc1_cmd &sdmmc1_clk>; ++ sd-uhs-sdr104; ++ status = "disabled"; ++}; ++ ++&tsadc { ++ status = "okay"; ++}; ++ ++&uart1 { ++ status = "disabled"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart1m0_xfer &uart1m0_ctsn>; ++}; ++ ++&u2phy0_otg { ++ status = "okay"; ++}; ++ ++&usb2phy0 { ++ status = "okay"; ++}; ++ ++&usbdrd_dwc3 { ++ status = "okay"; ++}; ++ ++&usbdrd30 { ++ status = "okay"; ++}; ++ ++&vdpu { ++ status = "okay"; ++}; ++ ++&vdpu_mmu { ++ status = "okay"; ++}; ++ ++&vepu { ++ status = "okay"; ++}; ++ ++&vepu_mmu { ++ status = "okay"; ++}; ++ ++&vop { ++ status = "okay"; ++}; ++ ++&vop_mmu { ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3566-rk817-tablet-k108.dts b/arch/arm64/boot/dts/rockchip/rk3566-rk817-tablet-k108.dts +new file mode 100755 +index 000000000000..f6f4534aad50 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3566-rk817-tablet-k108.dts +@@ -0,0 +1,1307 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++/dts-v1/; ++ ++#include ++#include ++#include ++#include ++#include ++#include "rk3566.dtsi" ++#include "rk3568-android.dtsi" ++ ++/ { ++ model = "Rockchip RK3566 RK817 TABLET K108 LP4X Board"; ++ compatible = "rockchip,rk3566-rk817-tablet-k108", "rockchip,rk3566"; ++ ++ adc_keys: adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 0>; ++ io-channel-names = "buttons"; ++ keyup-threshold-microvolt = <1800000>; ++ poll-interval = <100>; ++ ++ vol-up-key { ++ label = "volume up"; ++ linux,code = ; ++ press-threshold-microvolt = <1750>; ++ }; ++ ++ vol-down-key { ++ label = "volume down"; ++ linux,code = ; ++ press-threshold-microvolt = <297500>; ++ }; ++ }; ++ ++ backlight: backlight { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm4 0 25000 0>; ++ brightness-levels = < ++ 0 20 20 21 21 22 22 23 ++ 23 24 24 25 25 26 26 27 ++ 27 28 28 29 29 30 30 31 ++ 31 32 32 33 33 34 34 35 ++ 35 36 36 37 37 38 38 39 ++ 40 41 42 43 44 45 46 47 ++ 48 49 50 51 52 53 54 55 ++ 56 57 58 59 60 61 62 63 ++ 64 65 66 67 68 69 70 71 ++ 72 73 74 75 76 77 78 79 ++ 80 81 82 83 84 85 86 87 ++ 88 89 90 91 92 93 94 95 ++ 96 97 98 99 100 101 102 103 ++ 104 105 106 107 108 109 110 111 ++ 112 113 114 115 116 117 118 119 ++ 120 121 122 123 124 125 126 127 ++ 128 129 130 131 132 133 134 135 ++ 136 137 138 139 140 141 142 143 ++ 144 145 146 147 148 149 150 151 ++ 152 153 154 155 156 157 158 159 ++ 160 161 162 163 164 165 166 167 ++ 168 169 170 171 172 173 174 175 ++ 176 177 178 179 180 181 182 183 ++ 184 185 186 187 188 189 190 191 ++ 192 193 194 195 196 197 198 199 ++ 200 201 202 203 204 205 206 207 ++ 208 209 210 211 212 213 214 215 ++ 216 217 218 219 220 221 222 223 ++ 224 225 226 227 228 229 230 231 ++ 232 233 234 235 236 237 238 239 ++ 240 241 242 243 244 245 246 247 ++ 248 249 250 251 252 253 254 255 ++ >; ++ default-brightness-level = <200>; ++ }; ++ ++ charge-animation { ++ compatible = "rockchip,uboot-charge"; ++ rockchip,uboot-charge-on = <1>; ++ rockchip,android-charge-on = <0>; ++ rockchip,uboot-low-power-voltage = <3350>; ++ rockchip,screen-on-voltage = <3400>; ++ status = "okay"; ++ }; ++ ++ flash_rgb13h: flash-rgb13h { ++ status = "okay"; ++ compatible = "led,rgb13h"; ++ label = "gpio-flash"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&flash_led_gpios>; ++ led-max-microamp = <20000>; ++ flash-max-microamp = <20000>; ++ flash-max-timeout-us = <1000000>; ++ enable-gpio = <&gpio4 RK_PB7 GPIO_ACTIVE_HIGH>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ }; ++ ++ hall_sensor: hall-mh248 { ++ compatible = "hall-mh248"; ++ irq-gpio = <&gpio3 RK_PA6 IRQ_TYPE_EDGE_BOTH>; ++ hall-active = <1>; ++ status = "okay"; ++ }; ++ ++ hdmi_sound: hdmi-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,mclk-fs = <128>; ++ simple-audio-card,name = "rockchip,hdmi"; ++ status = "okay"; ++ ++ simple-audio-card,cpu { ++ sound-dai = <&i2s0_8ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&hdmi>; ++ }; ++ }; ++ ++ vccsys: vccsys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc3v8_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3800000>; ++ regulator-max-microvolt = <3800000>; ++ }; ++ ++ vcc_camera: vcc-camera-regulator { ++ compatible = "regulator-fixed"; ++ gpio = <&gpio4 RK_PB1 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&camera_rst>; ++ regulator-name = "vcc_camera"; ++ enable-active-high; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ rk817-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,rk817-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ ++ simple-audio-card,cpu { ++ sound-dai = <&i2s1_8ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&rk817_codec>; ++ }; ++ }; ++ ++ rk_headset: rk-headset { ++ compatible = "rockchip_headset"; ++ headset_gpio = <&gpio4 RK_PC4 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hp_det>; ++ io-channels = <&saradc 2>; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ clocks = <&rk817 1>; ++ clock-names = "ext_clock"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ post-power-on-delay-ms = <200>; ++ reset-gpios = <&gpio2 RK_PB1 GPIO_ACTIVE_LOW>; ++ }; ++ ++ vcc_sd: vcc-sd { ++ compatible = "regulator-gpio"; ++ enable-active-low; ++ enable-gpio = <&gpio0 RK_PA5 GPIO_ACTIVE_LOW>; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&vcc_sd_h>; ++ regulator-name = "vcc_sd"; ++ states = <3300000 0x0 ++ 3300000 0x1>; ++ }; ++ ++ vcc5v0_host: vcc5v0-host-regulator { ++ compatible = "regulator-fixed"; ++ enable-active-high; ++ gpio = <&gpio3 RK_PA3 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&vcc5v0_host_en>; ++ regulator-name = "vcc5v0_host"; ++ regulator-always-on; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ rockchip,grf = <&grf>; ++ wifi_chip_type = "rtl8723cs"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_host_wake_irq>; ++ WIFI,host_wake_irq = <&gpio2 RK_PB2 GPIO_ACTIVE_HIGH>; ++ WIFI,vbat_gpio = <&gpio0 RK_PC1 GPIO_ACTIVE_LOW>; ++ status = "okay"; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&rk817 1>; ++ clock-names = "ext_clock"; ++ //wifi-bt-power-toggle; ++ uart_rts_gpios = <&gpio2 RK_PB5 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart1m0_rtsn>; ++ pinctrl-1 = <&uart1_gpios>; ++ BT,reset_gpio = <&gpio2 RK_PB7 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio2 RK_PC1 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio2 RK_PC0 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++}; ++ ++&combphy1_usq { ++ rockchip,dis-u3otg1-port; ++ status = "okay"; ++}; ++ ++&cpu0 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&csi2_dphy_hw { ++ status = "okay"; ++}; ++ ++&csi2_dphy0 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam0: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&gc2385_out>; ++ data-lanes = <1>; ++ }; ++ mipi_in_ucam1: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ov8858_out>; ++ data-lanes = <1 2 3 4>; ++ }; ++ }; ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ csidphy0_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp0_in>; ++ }; ++ }; ++ }; ++}; ++ ++&dsi0 { ++ status = "okay"; ++ rockchip,lane-rate = <1000>; ++ panel@0 { ++ compatible = "aoly,sl008pa21y1285-b00", "simple-panel-dsi"; ++ reg = <0>; ++ ++ backlight = <&backlight>; ++ power-supply=<&vcc_3v3>; ++ enable-gpios = <&gpio0 RK_PC7 GPIO_ACTIVE_HIGH>; ++ stbyb-gpios = <&gpio0 RK_PC0 GPIO_ACTIVE_HIGH>; ++ reset-gpios = <&gpio0 RK_PC2 GPIO_ACTIVE_LOW>; ++ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&lcd_enable_gpio>, <&lcd_rst_gpio>, <&lcd_stanby_gpio>; ++ ++ prepare-delay-ms = <120>; ++ reset-delay-ms = <120>; ++ init-delay-ms = <120>; ++ stbyb-delay-ms = <120>; ++ enable-delay-ms = <120>; ++ disable-delay-ms = <120>; ++ unprepare-delay-ms = <120>; ++ ++ width-mm = <229>; ++ height-mm = <143>; ++ ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | ++ MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ ++ panel-init-sequence = [ ++ 29 00 04 FF 98 81 03 ++ 23 00 02 01 00 ++ 23 00 02 02 00 ++ 23 00 02 03 73 ++ 23 00 02 04 00 ++ 23 00 02 05 00 ++ 23 00 02 06 08 ++ 23 00 02 07 00 ++ 23 00 02 08 00 ++ 23 00 02 09 00 ++ 23 00 02 0A 01 ++ 23 00 02 0B 01 ++ 23 00 02 0C 00 ++ 23 00 02 0D 01 ++ 23 00 02 0E 01 ++ 23 00 02 0F 00 ++ 23 00 02 10 00 ++ 23 00 02 11 00 ++ 23 00 02 12 00 ++ 23 00 02 13 1F ++ 23 00 02 14 1F ++ 23 00 02 15 00 ++ 23 00 02 16 00 ++ 23 00 02 17 00 ++ 23 00 02 18 00 ++ 23 00 02 19 00 ++ 23 00 02 1A 00 ++ 23 00 02 1B 00 ++ 23 00 02 1C 00 ++ 23 00 02 1D 00 ++ 23 00 02 1E 40 ++ 23 00 02 1F C0 ++ 23 00 02 20 06 ++ 23 00 02 21 01 ++ 23 00 02 22 06 ++ 23 00 02 23 01 ++ 23 00 02 24 88 ++ 23 00 02 25 88 ++ 23 00 02 26 00 ++ 23 00 02 27 00 ++ 23 00 02 28 3B ++ 23 00 02 29 03 ++ 23 00 02 2A 00 ++ 23 00 02 2B 00 ++ 23 00 02 2C 00 ++ 23 00 02 2D 00 ++ 23 00 02 2E 00 ++ 23 00 02 2F 00 ++ 23 00 02 30 00 ++ 23 00 02 31 00 ++ 23 00 02 32 00 ++ 23 00 02 33 00 ++ 23 00 02 34 00 ++ 23 00 02 35 00 ++ 23 00 02 36 00 ++ 23 00 02 37 00 ++ 23 00 02 38 00 ++ 23 00 02 39 00 ++ 23 00 02 3A 00 ++ 23 00 02 3B 00 ++ 23 00 02 3C 00 ++ 23 00 02 3D 00 ++ 23 00 02 3E 00 ++ 23 00 02 3F 00 ++ 23 00 02 40 00 ++ 23 00 02 41 00 ++ 23 00 02 42 00 ++ 23 00 02 43 00 ++ 23 00 02 44 00 ++ 23 00 02 50 01 ++ 23 00 02 51 23 ++ 23 00 02 52 45 ++ 23 00 02 53 67 ++ 23 00 02 54 89 ++ 23 00 02 55 AB ++ 23 00 02 56 01 ++ 23 00 02 57 23 ++ 23 00 02 58 45 ++ 23 00 02 59 67 ++ 23 00 02 5A 89 ++ 23 00 02 5B AB ++ 23 00 02 5C CD ++ 23 00 02 5D EF ++ 23 00 02 5E 00 ++ 23 00 02 5F 01 ++ 23 00 02 60 01 ++ 23 00 02 61 06 ++ 23 00 02 62 06 ++ 23 00 02 63 07 ++ 23 00 02 64 07 ++ 23 00 02 65 00 ++ 23 00 02 66 00 ++ 23 00 02 67 02 ++ 23 00 02 68 02 ++ 23 00 02 69 05 ++ 23 00 02 6A 05 ++ 23 00 02 6B 02 ++ 23 00 02 6C 0D ++ 23 00 02 6D 0D ++ 23 00 02 6E 0C ++ 23 00 02 6F 0C ++ 23 00 02 70 0F ++ 23 00 02 71 0F ++ 23 00 02 72 0E ++ 23 00 02 73 0E ++ 23 00 02 74 02 ++ 23 00 02 75 01 ++ 23 00 02 76 01 ++ 23 00 02 77 06 ++ 23 00 02 78 06 ++ 23 00 02 79 07 ++ 23 00 02 7A 07 ++ 23 00 02 7B 00 ++ 23 00 02 7C 00 ++ 23 00 02 7D 02 ++ 23 00 02 7E 02 ++ 23 00 02 7F 05 ++ 23 00 02 80 05 ++ 23 00 02 81 02 ++ 23 00 02 82 0D ++ 23 00 02 83 0D ++ 23 00 02 84 0C ++ 23 00 02 85 0C ++ 23 00 02 86 0F ++ 23 00 02 87 0F ++ 23 00 02 88 0E ++ 23 00 02 89 0E ++ 23 00 02 8A 02 ++ 29 00 04 FF 98 81 04 ++ 23 00 02 6C 15 ++ 23 00 02 6E 2A ++ 23 00 02 6F 33 ++ 23 00 02 8D 1B ++ 23 00 02 87 BA ++ 23 00 02 3A 24 ++ 23 00 02 26 76 ++ 23 00 02 B2 D1 ++ 29 00 04 FF 98 81 01 ++ 23 00 02 22 0A ++ 23 00 02 31 00 ++ 23 00 02 43 66 ++ 23 00 02 53 40 ++ 23 00 02 50 87 ++ 23 00 02 51 82 ++ 23 00 02 60 15 ++ 23 00 02 61 01 ++ 23 00 02 62 0C ++ 23 00 02 63 00 ++ 29 00 04 FF 98 81 00 ++ 23 00 02 35 00 ++ ++ 05 78 01 11 ++ 05 dc 01 29 ++ //05 00 01 35 ++ ]; ++ ++ panel-exit-sequence = [ ++ 05 dc 01 28 ++ 05 78 01 10 ++ ]; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <160000000>; ++ hactive = <1200>; ++ vactive = <1920>; ++ ++ hsync-len = <1>;//19 ++ hback-porch = <60>;//40 ++ hfront-porch = <80>;//123 ++ ++ vsync-len = <1>; ++ vback-porch = <25>; ++ vfront-porch = <35>; ++ ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <1>; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ panel_in_dsi: endpoint { ++ remote-endpoint = <&dsi_out_panel>; ++ }; ++ }; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@1 { ++ reg = <1>; ++ dsi_out_panel: endpoint { ++ remote-endpoint = <&panel_in_dsi>; ++ }; ++ }; ++ }; ++ ++}; ++ ++&dsi0_in_vp0 { ++ status = "okay"; ++}; ++ ++&dsi0_in_vp1 { ++ status = "disabled"; ++}; ++ ++&gpu { ++ mali-supply = <&vdd_gpu>; ++ status = "okay"; ++}; ++ ++&hdmi { ++ status = "okay"; ++}; ++ ++&hdmi_in_vp0 { ++ status = "okay"; ++}; ++ ++&hdmi_in_vp1 { ++ status = "disabled"; ++}; ++ ++&hdmi_sound { ++ status = "okay"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ ++ vdd_cpu: tcs4525@1c { ++ compatible = "tcs,tcs452x"; ++ reg = <0x1c>; ++ vin-supply = <&vccsys>; ++ regulator-compatible = "fan53555-reg"; ++ regulator-name = "vdd_cpu"; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1390000>; ++ regulator-init-microvolt = <900000>; ++ regulator-ramp-delay = <2300>; ++ fcs,suspend-voltage-selector = <1>; ++ regulator-boot-on; ++ regulator-always-on; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ rk817: pmic@20 { ++ compatible = "rockchip,rk817"; ++ reg = <0x20>; ++ interrupt-parent = <&gpio0>; ++ interrupts = <3 IRQ_TYPE_LEVEL_LOW>; ++ ++ pinctrl-names = "default", "pmic-sleep", ++ "pmic-power-off", "pmic-reset"; ++ pinctrl-0 = <&pmic_int>; ++ pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; ++ pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; ++ pinctrl-3 = <&soc_slppin_gpio>, <&rk817_slppin_rst>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ //fb-inner-reg-idxs = <2>; ++ /* 1: rst regs (default in codes), 0: rst the pmic */ ++ pmic-reset-func = <0>; ++ ++ vcc1-supply = <&vccsys>; ++ vcc2-supply = <&vccsys>; ++ vcc3-supply = <&vccsys>; ++ vcc4-supply = <&vccsys>; ++ vcc5-supply = <&vccsys>; ++ vcc6-supply = <&vccsys>; ++ vcc7-supply = <&vccsys>; ++ vcc8-supply = <&vccsys>; ++ vcc9-supply = <&dcdc_boost>; ++ ++ pwrkey { ++ status = "okay"; ++ }; ++ ++ pinctrl_rk8xx: pinctrl_rk8xx { ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ rk817_slppin_null: rk817_slppin_null { ++ pins = "gpio_slp"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_slppin_slp: rk817_slppin_slp { ++ pins = "gpio_slp"; ++ function = "pin_fun1"; ++ }; ++ ++ rk817_slppin_pwrdn: rk817_slppin_pwrdn { ++ pins = "gpio_slp"; ++ function = "pin_fun2"; ++ }; ++ ++ rk817_slppin_rst: rk817_slppin_rst { ++ pins = "gpio_slp"; ++ function = "pin_fun3"; ++ }; ++ }; ++ ++ regulators { ++ vdd_logic: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <500000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-init-microvolt = <900000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_logic"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <900000>; ++ }; ++ }; ++ ++ vdd_gpu: DCDC_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <500000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-init-microvolt = <900000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_gpu"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_ddr"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_3v3: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_3v3"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcca1v8_pmu: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcca1v8_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdda_0v9: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <900000>; ++ regulator-name = "vdda_0v9"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdda0v9_pmu: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <900000>; ++ regulator-name = "vdda0v9_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <900000>; ++ }; ++ }; ++ ++ vccio_acodec: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vccio_acodec"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vccio_sd: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vccio_sd"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc3v3_pmu: LDO_REG6 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcc3v3_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ ++ vcc_1v8: LDO_REG7 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc_1v8"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc1v8_dvp: LDO_REG8 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc1v8_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc2v8_dvp: LDO_REG9 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <2800000>; ++ regulator-max-microvolt = <2800000>; ++ regulator-name = "vcc2v8_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ dcdc_boost: BOOST { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <4700000>; ++ regulator-max-microvolt = <5400000>; ++ regulator-name = "boost"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ otg_switch: OTG_SWITCH { ++ regulator-name = "otg_switch"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ }; ++ ++ battery { ++ compatible = "rk817,battery"; ++ ocv_table = <3400 3513 3578 3687 3734 3752 3763 ++ 3766 3771 3784 3804 3836 3885 3925 ++ 3962 4005 4063 4114 4169 4227 4303>; ++ design_capacity = <5000>; ++ design_qmax = <5500>; ++ bat_res = <100>; ++ sleep_enter_current = <150>; ++ sleep_exit_current = <180>; ++ sleep_filter_current = <100>; ++ power_off_thresd = <3450>; ++ zero_algorithm_vol = <3850>; ++ max_soc_offset = <60>; ++ monitor_sec = <5>; ++ sample_res = <10>; ++ virtual_power = <0>; ++ }; ++ ++ charger { ++ compatible = "rk817,charger"; ++ min_input_voltage = <4500>; ++ max_input_current = <1500>; ++ max_chrg_current = <2000>; ++ max_chrg_voltage = <4300>; ++ chrg_term_mode = <0>; ++ chrg_finish_cur = <300>; ++ virtual_power = <0>; ++ dc_det_adc = <0>; ++ extcon = <&usb2phy0>; ++ }; ++ ++ rk817_codec: codec { ++ #sound-dai-cells = <0>; ++ compatible = "rockchip,rk817-codec"; ++ clocks = <&cru I2S1_MCLKOUT>; ++ clock-names = "mclk"; ++ assigned-clocks = <&cru I2S1_MCLKOUT>, <&cru I2S1_MCLK_TX_IOE>; ++ assigned-clock-rates = <12288000>; ++ assigned-clock-parents = <&cru I2S1_MCLKOUT_TX>, <&cru I2S1_MCLKOUT_TX>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1m0_mclk>; ++ hp-volume = <20>; ++ spk-volume = <3>; ++ out-l2spk-r2hp; ++ spk-ctl-gpios = <&gpio0 RK_PA6 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ }; ++}; ++ ++&i2c2 { ++ status = "okay"; ++ pinctrl-0 = <&i2c2m1_xfer>; ++ ++ dw9714: dw9714@c { ++ compatible = "dongwoon,dw9714"; ++ status = "okay"; ++ reg = <0x0c>; ++ rockchip,camera-module-index = <0>; ++ rockchip,vcm-start-current = <10>; ++ rockchip,vcm-rated-current = <85>; ++ rockchip,vcm-step-mode = <5>; ++ rockchip,camera-module-facing = "back"; ++ }; ++ ++ gc2385: gc2385@37 { ++ compatible = "galaxycore,gc2385"; ++ status = "okay"; ++ reg = <0x37>; ++ clocks = <&cru CLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ power-domains = <&power RK3568_PD_VI>; ++ pinctrl-names = "rockchip,camera_default"; ++ pinctrl-0 = <&cif_clk>; ++ ++ //reset pin control by hardware,used this pin switch to mipi input ++ //1->2LANE(LANE 0&1) FRONT camera, 0->4LANE REAR camera ++ reset-gpios = <&gpio4 17 GPIO_ACTIVE_LOW>; ++ pwdn-gpios = <&gpio4 10 GPIO_ACTIVE_HIGH>; ++ rockchip,camera-module-index = <1>; ++ rockchip,camera-module-facing = "front"; ++ rockchip,camera-module-name = "HS5885-BNSM1018-V01"; ++ rockchip,camera-module-lens-name = "default"; ++ port { ++ gc2385_out: endpoint { ++ remote-endpoint = <&mipi_in_ucam0>; ++ data-lanes = <1>; ++ }; ++ }; ++ }; ++ ++ ov8858: ov8858@36 { ++ status = "okay"; ++ compatible = "ovti,ov8858"; ++ reg = <0x36>; ++ clocks = <&cru CLK_CAM0_OUT>; ++ clock-names = "xvclk"; ++ power-domains = <&power RK3568_PD_VI>; ++ pinctrl-names = "rockchip,camera_default", "rockchip,camera_sleep"; ++ pinctrl-0 = <&cam_clkout0>; ++ pinctrl-1 = <&cam_sleep>; ++ //reset pin control by hardware,used this pin switch to mipi input ++ //1->2LANE(LANE 0&1) FRONT camera, 0->4LANE REAR camera ++ reset-gpios = <&gpio4 17 GPIO_ACTIVE_LOW>; ++ pwdn-gpios = <&gpio4 11 GPIO_ACTIVE_HIGH>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "HS5885-BNSM1018-V01"; ++ rockchip,camera-module-lens-name = "default"; ++ flash-leds = <&flash_rgb13h>; ++ lens-focus = <&dw9714>; ++ port { ++ ov8858_out: endpoint { ++ remote-endpoint = <&mipi_in_ucam1>; ++ data-lanes = <1 2 3 4>; ++ }; ++ }; ++ }; ++}; ++ ++&i2c3 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2c3m1_xfer>; ++ clock-frequency = <400000>; ++ i2c-scl-rising-time-ns = <138>; ++ i2c-scl-falling-time-ns = <4>; ++ ++ gt9xx: gt9xx@14 { ++ compatible = "goodix,gt9xx"; ++ reg = <0x14>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <>9xx_gpio>; ++ touch-gpio = <&gpio3 RK_PB0 IRQ_TYPE_LEVEL_HIGH>; ++ reset-gpio = <&gpio3 RK_PB1 GPIO_ACTIVE_HIGH>; ++ max-x = <1200>; ++ max-y = <1920>; ++ tp-size = <9110>; ++ tp-supply = <&vcc_3v3>; ++ }; ++}; ++ ++&i2c5 { ++ status = "okay"; ++ clock-frequency = <400000>; ++ i2c-scl-rising-time-ns = <144>; ++ i2c-scl-falling-time-ns = <4>; ++ ++ sensor@4c { ++ compatible = "gs_mc3230"; ++ reg = <0x4c>; ++ type = ; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ layout = <9>; ++ reprobe_en = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sensor_gpio>; ++ irq-gpio = <&gpio3 RK_PA2 IRQ_TYPE_LEVEL_LOW>; ++ }; ++}; ++ ++&i2s0_8ch { ++ status = "okay"; ++}; ++ ++&i2s1_8ch { ++ status = "okay"; ++ rockchip,clk-trcm = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1m0_sclktx ++ &i2s1m0_lrcktx ++ &i2s1m0_sdi0 ++ &i2s1m0_sdo0>; ++}; ++ ++&jpegd { ++ status = "okay"; ++}; ++ ++&jpegd_mmu { ++ status = "okay"; ++}; ++ ++&video_phy0 { ++ status = "okay"; ++}; ++ ++&mpp_srv { ++ status = "okay"; ++}; ++ ++&nandc0 { ++ status = "okay"; ++}; ++ ++&pinctrl { ++ cam { ++ cam_clkout0: cam-clkout0 { ++ rockchip,pins = ++ /* cam_clkout0 */ ++ <4 RK_PA7 1 &pcfg_pull_none>; ++ }; ++ ++ cam_sleep: cam-sleep { ++ rockchip,pins = ++ /* cam_sleep */ ++ <4 RK_PA7 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ camera_rst: camera-rst { ++ rockchip,pins = ++ /* front camera reset */ ++ <4 RK_PB0 RK_FUNC_GPIO &pcfg_pull_none>, ++ /* back camra reset */ ++ <4 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ flash_led_gpios: flash-led { ++ rockchip,pins = ++ /* flash led enable */ ++ <4 RK_PB7 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ gt9xx { ++ gt9xx_gpio: gt9xx-gpio { ++ rockchip,pins = <3 RK_PB0 RK_FUNC_GPIO &pcfg_pull_up>, ++ <3 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ headphone { ++ hp_det: hp-det { ++ rockchip,pins = <4 RK_PC4 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ lcd { ++ lcd_rst_gpio: lcd-rst-gpio { ++ rockchip,pins = <0 RK_PC2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ lcd_enable_gpio: lcd-enable-gpio { ++ rockchip,pins = <0 RK_PC7 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ lcd_stanby_gpio: lcd-stanby-gpio { ++ rockchip,pins = <0 RK_PC0 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ pmic { ++ pmic_int: pmic_int { ++ rockchip,pins = ++ <0 RK_PA3 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ soc_slppin_gpio: soc_slppin_gpio { ++ rockchip,pins = ++ <0 RK_PA2 RK_FUNC_GPIO &pcfg_output_low>; ++ }; ++ ++ soc_slppin_slp: soc_slppin_slp { ++ rockchip,pins = ++ <0 RK_PA2 1 &pcfg_pull_none>; ++ }; ++ ++ soc_slppin_rst: soc_slppin_rst { ++ rockchip,pins = ++ <0 RK_PA2 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ sensor { ++ sensor_gpio: sensor-gpio { ++ rockchip,pins = <3 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <2 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ vcc_sd { ++ vcc_sd_h: vcc-sd-h { ++ rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdmmc0 { ++ sdmmc0_det_gpio: sdmmc0-det-gpio { ++ rockchip,pins = <0 RK_PA4 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ usb { ++ vcc5v0_host_en: vcc5v0-host-en { ++ rockchip,pins = <3 RK_PA3 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ wireless-wlan { ++ wifi_host_wake_irq: wifi-host-wake-irq { ++ rockchip,pins = <2 RK_PB2 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart1_gpios: uart1-gpios { ++ rockchip,pins = <2 RK_PB5 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ pmuio1-supply = <&vcc3v3_pmu>; ++ pmuio2-supply = <&vcc3v3_pmu>; ++ vccio1-supply = <&vccio_acodec>; ++ vccio3-supply = <&vccio_sd>; ++ vccio4-supply = <&vcc3v3_pmu>; ++ vccio5-supply = <&vcc_1v8>; ++ vccio6-supply = <&vcc1v8_dvp>; ++ vccio7-supply = <&vcc_3v3>; ++}; ++ ++&pwm4 { ++ status = "okay"; ++}; ++ ++&rk_rga { ++ status = "okay"; ++}; ++ ++&rkisp { ++ status = "okay"; ++}; ++ ++&rkisp_mmu { ++ status = "okay"; ++}; ++ ++&rkisp_vir0 { ++ status = "okay"; ++ ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp0_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&csidphy0_out>; ++ }; ++ }; ++}; ++ ++&rkvdec { ++ status = "okay"; ++}; ++ ++&rkvdec_mmu { ++ status = "okay"; ++}; ++ ++&rkvenc { ++ status = "okay"; ++}; ++ ++&rkvenc_mmu { ++ status = "okay"; ++}; ++ ++&route_dsi0 { ++ status = "okay"; ++}; ++ ++&route_hdmi { ++ status = "okay"; ++ connect = <&vp0_out_hdmi>; ++}; ++ ++&saradc { ++ status = "okay"; ++ vref-supply = <&vcc_1v8>; ++}; ++ ++&sdhci { ++ bus-width = <8>; ++ no-sdio; ++ no-sd; ++ non-removable; ++ max-frequency = <200000000>; ++ status = "okay"; ++}; ++ ++&sdmmc0 { ++ max-frequency = <150000000>; ++ no-sdio; ++ no-mmc; ++ bus-width = <4>; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ disable-wp; ++ sd-uhs-sdr104; ++ vmmc-supply = <&vcc_sd>; ++ vqmmc-supply = <&vccio_sd>; ++ cd-gpios = <&gpio0 RK_PA4 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc0_bus4 &sdmmc0_clk &sdmmc0_cmd &sdmmc0_det_gpio>; ++ status = "okay"; ++}; ++ ++&sdmmc1 { ++ max-frequency = <150000000>; ++ no-sd; ++ no-mmc; ++ bus-width = <4>; ++ disable-wp; ++ cap-sd-highspeed; ++ cap-sdio-irq; ++ keep-power-in-suspend; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ non-removable; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc1_bus4 &sdmmc1_cmd &sdmmc1_clk>; ++ sd-uhs-sdr104; ++ rockchip,default-sample-phase = <90>; ++ status = "okay"; ++}; ++ ++&tsadc { ++ status = "okay"; ++}; ++ ++&uart1 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart1m0_xfer &uart1m0_ctsn>; ++}; ++ ++&u2phy0_host { ++ phy-supply = <&vcc5v0_host>; ++ status = "okay"; ++}; ++ ++&u2phy0_otg { ++ status = "okay"; ++}; ++ ++&usb2phy0 { ++ status = "okay"; ++}; ++ ++&usbdrd_dwc3 { ++ status = "okay"; ++}; ++ ++&usbdrd30 { ++ status = "okay"; ++}; ++ ++&usbhost30 { ++ status = "okay"; ++}; ++ ++&usbhost_dwc3 { ++ phys = <&u2phy0_host>; ++ phy-names = "usb2-phy"; ++ maximum-speed = "high-speed"; ++ status = "okay"; ++}; ++ ++&vdpu { ++ status = "okay"; ++}; ++ ++&vdpu_mmu { ++ status = "okay"; ++}; ++ ++&vepu { ++ status = "okay"; ++}; ++ ++&vepu_mmu { ++ status = "okay"; ++}; ++ ++&vop { ++ status = "okay"; ++}; ++ ++&vop_mmu { ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3566-rk817-tablet-rkg11.dts b/arch/arm64/boot/dts/rockchip/rk3566-rk817-tablet-rkg11.dts +new file mode 100755 +index 000000000000..8102da0270fa +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3566-rk817-tablet-rkg11.dts +@@ -0,0 +1,1180 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++/dts-v1/; ++ ++#include ++#include ++#include ++#include ++#include ++#include "rk3566.dtsi" ++#include "rk3568-android.dtsi" ++ ++/ { ++ model = "Rockchip RK3566 RK817 TABLET RKG11 LP4 Board"; ++ compatible = "rockchip,rk3566-rk817-tablet-rkg11", "rockchip,rk3566"; ++ ++ adc_keys: adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 0>; ++ io-channel-names = "buttons"; ++ keyup-threshold-microvolt = <1800000>; ++ poll-interval = <100>; ++ ++ vol-up-key { ++ label = "volume up"; ++ linux,code = ; ++ press-threshold-microvolt = <1750>; ++ }; ++ ++ vol-down-key { ++ label = "volume down"; ++ linux,code = ; ++ press-threshold-microvolt = <297500>; ++ }; ++ }; ++ ++ backlight: backlight { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm4 0 25000 0>; ++ brightness-levels = < ++ 0 20 20 21 21 22 22 23 ++ 23 24 24 25 25 26 26 27 ++ 27 28 28 29 29 30 30 31 ++ 31 32 32 33 33 34 34 35 ++ 35 36 36 37 37 38 38 39 ++ 40 41 42 43 44 45 46 47 ++ 48 49 50 51 52 53 54 55 ++ 56 57 58 59 60 61 62 63 ++ 64 65 66 67 68 69 70 71 ++ 72 73 74 75 76 77 78 79 ++ 80 81 82 83 84 85 86 87 ++ 88 89 90 91 92 93 94 95 ++ 96 97 98 99 100 101 102 103 ++ 104 105 106 107 108 109 110 111 ++ 112 113 114 115 116 117 118 119 ++ 120 121 122 123 124 125 126 127 ++ 128 129 130 131 132 133 134 135 ++ 136 137 138 139 140 141 142 143 ++ 144 145 146 147 148 149 150 151 ++ 152 153 154 155 156 157 158 159 ++ 160 161 162 163 164 165 166 167 ++ 168 169 170 171 172 173 174 175 ++ 176 177 178 179 180 181 182 183 ++ 184 185 186 187 188 189 190 191 ++ 192 193 194 195 196 197 198 199 ++ 200 201 202 203 204 205 206 207 ++ 208 209 210 211 212 213 214 215 ++ 216 217 218 219 220 221 222 223 ++ 224 225 226 227 228 229 230 231 ++ 232 233 234 235 236 237 238 239 ++ 240 241 242 243 244 245 246 247 ++ 248 249 250 251 252 253 254 255 ++ >; ++ default-brightness-level = <200>; ++ }; ++ ++ charge-animation { ++ compatible = "rockchip,uboot-charge"; ++ rockchip,uboot-charge-on = <1>; ++ rockchip,android-charge-on = <0>; ++ rockchip,uboot-low-power-voltage = <3350>; ++ rockchip,screen-on-voltage = <3400>; ++ status = "okay"; ++ }; ++ ++ es7210_sound: es7210-sound { ++ status = "okay"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,name = "rockchip,es7210"; ++ ++ simple-audio-card,cpu { ++ sound-dai = <&i2s1_8ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&es7210>; ++ }; ++ }; ++ ++ vccsys: vccsys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc3v8_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3800000>; ++ regulator-max-microvolt = <3800000>; ++ }; ++ ++ vcc3v3_lcd0_n: vcc3v3-lcd0-n { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc3v3_lcd0_n"; ++ gpio = <&gpio0 RK_PC7 GPIO_ACTIVE_HIGH>; ++ enable-active-high; ++ regulator-boot-on; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ rk817-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,rk817-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ ++ simple-audio-card,cpu { ++ sound-dai = <&i2s3_2ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&rk817_codec>; ++ }; ++ }; ++ ++ rk_headset: rk-headset { ++ compatible = "rockchip_headset"; ++ headset_gpio = <&gpio0 RK_PC4 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hp_det>; ++ io-channels = <&saradc 2>; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ clocks = <&rk817 1>; ++ clock-names = "ext_clock"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ post-power-on-delay-ms = <200>; ++ reset-gpios = <&gpio2 RK_PC6 GPIO_ACTIVE_LOW>, ++ <&gpio2 RK_PB1 GPIO_ACTIVE_LOW>; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ rockchip,grf = <&grf>; ++ wifi_chip_type = "rtl8821cs"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_host_wake_irq>; ++ WIFI,vbat_gpio = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; ++ WIFI,host_wake_irq = <&gpio2 RK_PB2 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&rk817 1>; ++ clock-names = "ext_clock"; ++ //wifi-bt-power-toggle; ++ uart_rts_gpios = <&gpio2 RK_PB5 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart1m0_rtsn>; ++ pinctrl-1 = <&uart1_gpios>; ++ BT,reset_gpio = <&gpio2 RK_PB7 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio2 RK_PC1 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio2 RK_PC0 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ hall_sensor: hall-mh248 { ++ compatible = "hall-mh248"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&mh248_irq_gpio>; ++ irq-gpio = <&gpio0 RK_PC6 IRQ_TYPE_EDGE_BOTH>; ++ hall-active = <1>; ++ status = "okay"; ++ }; ++ ++ vibrator { ++ compatible = "rk-vibrator-gpio"; ++ vibrator-gpio = <&gpio0 RK_PC5 GPIO_ACTIVE_LOW>; ++ status = "okay"; ++ }; ++ ++ reserved-memory { ++ #address-cells = <2>; ++ #size-cells = <2>; ++ ranges; ++ ++ rknpu_reserved: rknpu { ++ compatible = "shared-dma-pool"; ++ inactive; ++ reusable; ++ size = <0x0 0x20000000>; ++ alignment = <0x0 0x1000>; ++ }; ++ }; ++}; ++ ++&cpu0 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&csi2_dphy_hw { ++ status = "okay"; ++}; ++ ++&csi2_dphy0 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam0: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&gc5035_out>; ++ data-lanes = <1 2>; ++ }; ++ mipi_in_ucam1: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ov8858_out>; ++ data-lanes = <1 2 3 4>; ++ }; ++ }; ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ csidphy0_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp0_in>; ++ }; ++ }; ++ }; ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&dmc { ++ center-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&dsi0 { ++ status = "okay"; ++ rockchip,dual-channel = <&dsi1>; ++ panel@0 { ++ status = "okay"; ++ compatible = "simple-panel-dsi"; ++ reg = <0>; ++ backlight = <&backlight>; ++ power-supply = <&vcc3v3_lcd0_n>; ++ //vsp-supply = <&outp>; ++ //vsn-supply = <&outn>; ++ //enable-gpios = <&gpio0 RK_PC7 GPIO_ACTIVE_HIGH>; ++ reset-gpios = <&gpio0 RK_PC2 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&lcd_enable_gpio>, <&lcd_rst_gpio>, <&lcd_panel_vsp>, <&lcd_panel_vsn>; ++ ++ prepare-delay-ms = <60>; ++ unprepare-delay-ms = <60>; ++ enable-delay-ms = <60>; ++ disable-delay-ms = <60>; ++ init-delay-ms = <60>; ++ reset-delay-ms = <60>; ++ ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | ++ MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; ++ dsi,format = ; ++ dsi,lanes = <8>; ++ panel-init-sequence = [ ++ 05 20 01 11 ++ 05 96 01 29 ++ ]; ++ ++ panel-exit-sequence = [ ++ 05 05 01 28 ++ 05 78 01 10 ++ ]; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ timing0: timing0 { ++ clock-frequency = <255000000>; ++ hactive = <1600>; ++ vactive = <2176>; ++ hsync-len = <14>; //20, 50 ++ hback-porch = <25>; //50, 56 ++ hfront-porch = <25>;//50, 30 ++ vsync-len = <8>; ++ vback-porch = <73>; ++ vfront-porch = <250>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <1>; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ panel_in_dsi: endpoint { ++ remote-endpoint = <&dsi_out_panel>; ++ }; ++ }; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@1 { ++ reg = <1>; ++ dsi_out_panel: endpoint { ++ remote-endpoint = <&panel_in_dsi>; ++ }; ++ }; ++ }; ++ ++}; ++ ++&dsi1 { ++ status = "okay"; ++}; ++ ++&dsi0_in_vp0 { ++ status = "okay"; ++}; ++ ++&dsi0_in_vp1 { ++ status = "disabled"; ++}; ++ ++&gpu { ++ mali-supply = <&vdd_gpu>; ++ status = "okay"; ++}; ++ ++ ++&i2c0 { ++ status = "okay"; ++ ++ vdd_cpu: tcs4525@1c { ++ compatible = "tcs,tcs452x"; ++ reg = <0x1c>; ++ vin-supply = <&vccsys>; ++ regulator-compatible = "fan53555-reg"; ++ regulator-name = "vdd_cpu"; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1390000>; ++ regulator-init-microvolt = <900000>; ++ regulator-ramp-delay = <2300>; ++ fcs,suspend-voltage-selector = <1>; ++ regulator-boot-on; ++ regulator-always-on; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ rk817: pmic@20 { ++ compatible = "rockchip,rk817"; ++ reg = <0x20>; ++ interrupt-parent = <&gpio0>; ++ interrupts = <3 IRQ_TYPE_LEVEL_LOW>; ++ ++ pinctrl-names = "default", "pmic-sleep", ++ "pmic-power-off", "pmic-reset"; ++ pinctrl-0 = <&pmic_int>; ++ pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; ++ pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; ++ pinctrl-3 = <&soc_slppin_gpio>, <&rk817_slppin_rst>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ //fb-inner-reg-idxs = <2>; ++ /* 1: rst regs (default in codes), 0: rst the pmic */ ++ pmic-reset-func = <0>; ++ ++ vcc1-supply = <&vccsys>; ++ vcc2-supply = <&vccsys>; ++ vcc3-supply = <&vccsys>; ++ vcc4-supply = <&vccsys>; ++ vcc5-supply = <&vccsys>; ++ vcc6-supply = <&vccsys>; ++ vcc7-supply = <&vccsys>; ++ vcc8-supply = <&vccsys>; ++ vcc9-supply = <&dcdc_boost>; ++ ++ pwrkey { ++ status = "okay"; ++ }; ++ ++ pinctrl_rk8xx: pinctrl_rk8xx { ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ rk817_slppin_null: rk817_slppin_null { ++ pins = "gpio_slp"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_slppin_slp: rk817_slppin_slp { ++ pins = "gpio_slp"; ++ function = "pin_fun1"; ++ }; ++ ++ rk817_slppin_pwrdn: rk817_slppin_pwrdn { ++ pins = "gpio_slp"; ++ function = "pin_fun2"; ++ }; ++ ++ rk817_slppin_rst: rk817_slppin_rst { ++ pins = "gpio_slp"; ++ function = "pin_fun3"; ++ }; ++ }; ++ ++ regulators { ++ vdd_logic: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <500000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-init-microvolt = <900000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_logic"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <900000>; ++ }; ++ }; ++ ++ vdd_gpu: DCDC_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <500000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-init-microvolt = <900000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_gpu"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_ddr"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_3v3: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_3v3"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcca1v8_pmu: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcca1v8_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdda_0v9: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <900000>; ++ regulator-name = "vdda_0v9"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdda0v9_pmu: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <900000>; ++ regulator-name = "vdda0v9_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <900000>; ++ }; ++ }; ++ ++ vccio_acodec: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vccio_acodec"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vccio_sd: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vccio_sd"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc3v3_pmu: LDO_REG6 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcc3v3_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcc_1v8: LDO_REG7 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc_1v8"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc1v8_dvp: LDO_REG8 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc1v8_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc2v8_dvp: LDO_REG9 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <2800000>; ++ regulator-max-microvolt = <2800000>; ++ regulator-name = "vcc2v8_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ dcdc_boost: BOOST { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <4700000>; ++ regulator-max-microvolt = <5400000>; ++ regulator-name = "boost"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ otg_switch: OTG_SWITCH { ++ regulator-name = "otg_switch"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ }; ++ ++ battery { ++ compatible = "rk817,battery"; ++ ocv_table = <3510 3679 3691 3714 3738 3759 3776 ++ 3795 3811 3834 3852 3881 3942 3976 ++ 4012 4075 4114 4177 4232 4277 4351>; ++ design_capacity = <7916>; ++ design_qmax = <8708>; ++ bat_res = <110>; ++ sleep_enter_current = <150>; ++ sleep_exit_current = <180>; ++ sleep_filter_current = <100>; ++ power_off_thresd = <3450>; ++ zero_algorithm_vol = <3850>; ++ max_soc_offset = <60>; ++ monitor_sec = <5>; ++ sample_res = <10>; ++ virtual_power = <0>; ++ }; ++ ++ charger { ++ compatible = "rk817,charger"; ++ min_input_voltage = <4500>; ++ max_input_current = <1500>; ++ max_chrg_current = <2000>; ++ max_chrg_voltage = <4300>; ++ chrg_term_mode = <0>; ++ chrg_finish_cur = <300>; ++ virtual_power = <0>; ++ dc_det_adc = <0>; ++ extcon = <&usb2phy0>; ++ }; ++ ++ rk817_codec: codec { ++ #sound-dai-cells = <0>; ++ compatible = "rockchip,rk817-codec"; ++ clocks = <&cru I2S3_MCLKOUT>; ++ clock-names = "mclk"; ++ assigned-clocks = <&cru I2S3_MCLKOUT>, <&cru I2S3_MCLK_IOE>; ++ assigned-clock-rates = <12288000>; ++ assigned-clock-parents = <&cru I2S3_MCLKOUT_TX>, <&cru I2S3_MCLKOUT>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s3m1_mclk>; ++ hp-volume = <20>; ++ spk-volume = <3>; ++ mic-in-differential; ++ use-ext-amplifier; ++ //out-l2spk-r2hp; ++ spk-ctl-gpios = <&gpio0 RK_PA6 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ }; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ ++ dio5632@3e { ++ compatible = "DIO5632"; ++ reg = <0x3e>; ++ status = "disabled"; ++ ++ outp: outp@3e { ++ regulator-name = "LCD_VSP"; ++ vin-supply = <&vccsys>; ++ enable-gpios = <&gpio0 RK_PB7 GPIO_ACTIVE_HIGH>; ++ }; ++ ++ outn: outn@3e { ++ regulator-name = "LCD_VSN"; ++ vin-supply = <&vccsys>; ++ enable-gpios = <&gpio0 RK_PC0 GPIO_ACTIVE_HIGH>; ++ }; ++ }; ++ ++ es7210: es7210@43 { ++ #sound-dai-cells = <0>; ++ compatible = "ES7210_MicArray_0"; ++ reg = <0x43>; ++ clocks = <&cru I2S1_MCLKOUT_RX>; ++ clock-names = "mclk"; ++ assigned-clocks = <&cru I2S1_MCLKOUT_RX>; ++ assigned-clock-parents = <&cru CLK_I2S1_8CH_RX>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1m0_mclk>; ++ }; ++}; ++ ++&i2c2 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2c2m1_xfer>; ++ ++ gc5035: gc5035@37 { ++ compatible = "galaxycore,gc5035"; ++ status = "okay"; ++ reg = <0x37>; ++ clocks = <&cru CLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ power-domains = <&power RK3568_PD_VI>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_clk>; ++ ++ //reset pin control by hardware,used this pin switch to mipi input ++ //0->FRONT camera, 1->REAR camera ++ reset-gpios = <&gpio4 17 GPIO_ACTIVE_LOW>; ++ pwdn-gpios = <&gpio4 10 GPIO_ACTIVE_HIGH>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "front"; ++ rockchip,camera-module-name = "XHG-RKX11F-V5"; ++ rockchip,camera-module-lens-name = "HR232H65"; ++ port { ++ gc5035_out: endpoint { ++ remote-endpoint = <&mipi_in_ucam0>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++ ++ ov8858: ov8858@36 { ++ status = "okay"; ++ compatible = "ovti,ov8858"; ++ reg = <0x36>; ++ clocks = <&cru CLK_CAM0_OUT>; ++ clock-names = "xvclk"; ++ power-domains = <&power RK3568_PD_VI>; ++ pinctrl-names = "rockchip,camera_default", "rockchip,camera_sleep"; ++ pinctrl-0 = <&cam_clkout0>; ++ pinctrl-1 = <&cam_sleep>; ++ //reset pin control by hardware,used this pin switch to mipi input ++ //0->FRONT camera, 1->REAR camera ++ reset-gpios = <&gpio4 17 GPIO_ACTIVE_HIGH>; ++ pwdn-gpios = <&gpio4 11 GPIO_ACTIVE_HIGH>; ++ rockchip,camera-module-index = <1>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "XHG-RKX11B-V10"; ++ rockchip,camera-module-lens-name = "default"; ++ port { ++ ov8858_out: endpoint { ++ remote-endpoint = <&mipi_in_ucam1>; ++ data-lanes = <1 2 3 4>; ++ }; ++ }; ++ }; ++}; ++ ++&i2c3 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2c3m1_xfer>; ++ ++ focaltech: focaltech@38 { ++ status = "okay"; ++ compatible = "focaltech,fts"; ++ reg = <0x38>; ++ power-supply = <&vcc3v3_lcd0_n>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&tp_gpio>; ++ focaltech,irq-gpio = <&gpio3 RK_PB0 IRQ_TYPE_LEVEL_LOW>; ++ focaltech,reset-gpio = <&gpio3 RK_PB1 GPIO_ACTIVE_HIGH>; ++ focaltech,have-key = <0>; ++ focaltech,key-number = <3>; ++ focaltech,keys = <256 1068 64 64 128 1068 64 64 192 1068 64 64>; ++ focaltech,key-x-coord = <1600>; ++ focaltech,key-y-coord = <2176>; ++ focaltech,max-touch-number = <5>; ++ }; ++}; ++ ++&i2c5 { ++ status = "okay"; ++ ++ sensor@18 { ++ compatible = "gs_sc7a20"; ++ reg = <0x18>; ++ type = ; ++ irq_enable = <0>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sensor_gpio>; ++ irq-gpio = <&gpio3 RK_PA2 IRQ_TYPE_LEVEL_LOW>; ++ poll_delay_ms = <10>; ++ layout = <7>; ++ status = "disabled"; ++ }; ++ ++ ls_em3071x@24 { ++ compatible = "ls_em3071x"; ++ reg = <0x24>; ++ type = ; ++ irq_enable = <0>; ++ poll_delay_ms = <100>; ++ status = "okay"; ++ }; ++ ++ ps_em3071x@24 { ++ compatible = "ps_em3071x"; ++ reg = <0x24>; ++ type = ; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&em3071x_irq_gpio>; ++ irq-gpio = <&gpio3 RK_PA6 IRQ_TYPE_LEVEL_LOW>; ++ irq_enable = <1>; ++ ps_threshold_high = <25>; ++ ps_threshold_low = <15>; ++ poll_delay_ms = <100>; ++ status = "okay"; ++ }; ++ ++ icm20607_acc@68 { ++ compatible = "icm2060x_acc"; ++ reg = <0x68>; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ type = ; ++ layout = <1>; ++ status = "okay"; ++ }; ++ ++ icm20607_gyro@68 { ++ compatible = "icm2060x_gyro"; ++ reg = <0x68>; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ type = ; ++ layout = <1>; ++ status = "okay"; ++ }; ++ ++ ak09918_compass: ak09918_compass@c { ++ compatible = "ak09918"; ++ reg = <0x0c>; ++ type = ; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ layout = <1>; ++ status = "okay"; ++ }; ++}; ++ ++&i2s1_8ch { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++ rockchip,clk-trcm = <2>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1m0_sclkrx ++ &i2s1m0_lrckrx ++ &i2s1m0_sdi0 ++ &i2s1m0_sdi1 ++ &i2s1m0_sdi2 ++ &i2s1m0_sdi3>; ++}; ++ ++&i2s3_2ch { ++ status = "okay"; ++ rockchip,clk-trcm = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s3m1_sclk ++ &i2s3m1_lrck ++ &i2s3m1_sdi ++ &i2s3m1_sdo>; ++}; ++ ++&jpegd { ++ status = "okay"; ++}; ++ ++&jpegd_mmu { ++ status = "okay"; ++}; ++ ++&video_phy0 { ++ status = "okay"; ++}; ++ ++&video_phy1 { ++ status = "okay"; ++}; ++ ++&mpp_srv { ++ status = "okay"; ++}; ++ ++&nandc0 { ++ status = "okay"; ++}; ++ ++&pinctrl { ++ cam { ++ cam_clkout0: cam-clkout0 { ++ rockchip,pins = ++ /* cam_clkout0 */ ++ <4 RK_PA7 1 &pcfg_pull_none>; ++ }; ++ ++ cam_sleep: cam-sleep { ++ rockchip,pins = ++ /* cam_sleep */ ++ <4 RK_PA7 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ camera_rst: camera-rst { ++ rockchip,pins = ++ /* front camera reset */ ++ <4 RK_PB0 RK_FUNC_GPIO &pcfg_pull_none>, ++ /* back camra reset */ ++ <4 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ flash_led_gpios: flash-led { ++ rockchip,pins = ++ /* flash led enable */ ++ <4 RK_PA6 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ tp { ++ tp_gpio: tp-gpio { ++ rockchip,pins = <3 RK_PB0 RK_FUNC_GPIO &pcfg_pull_up>, ++ <3 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ headphone { ++ hp_det: hp-det { ++ rockchip,pins = <0 RK_PC4 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ lcd { ++ lcd_rst_gpio: lcd-rst-gpio { ++ rockchip,pins = <0 RK_PC2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ lcd_enable_gpio: lcd-enable-gpio { ++ rockchip,pins = <0 RK_PC7 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ lcd_panel_vsp: lcd-panel-vsp { ++ rockchip,pins = <0 RK_PB7 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ lcd_panel_vsn: lcd-panel-vsn { ++ rockchip,pins = <0 RK_PC0 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ pmic { ++ pmic_int: pmic_int { ++ rockchip,pins = ++ <0 RK_PA3 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ soc_slppin_gpio: soc_slppin_gpio { ++ rockchip,pins = ++ <0 RK_PA2 RK_FUNC_GPIO &pcfg_output_low>; ++ }; ++ ++ soc_slppin_slp: soc_slppin_slp { ++ rockchip,pins = ++ <0 RK_PA2 1 &pcfg_pull_none>; ++ }; ++ ++ soc_slppin_rst: soc_slppin_rst { ++ rockchip,pins = ++ <0 RK_PA2 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ sensor { ++ sensor_gpio:sensor-gpio { ++ rockchip,pins = <3 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ em3071x_irq_gpio: em3071x-irq-gpio { ++ rockchip,pins = <3 RK_PA6 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ mh248_irq_gpio: mh248-irq-gpio { ++ rockchip,pins = <0 RK_PC6 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <2 RK_PC6 RK_FUNC_GPIO &pcfg_pull_none>, ++ <2 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ wireless-wlan { ++ wifi_host_wake_irq: wifi-host-wake-irq { ++ rockchip,pins = <2 RK_PB2 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart1_gpios: uart1-gpios { ++ rockchip,pins = <2 RK_PB5 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ pmuio1-supply = <&vcc3v3_pmu>; ++ pmuio2-supply = <&vcc3v3_pmu>; ++ vccio1-supply = <&vccio_acodec>; ++ vccio3-supply = <&vccio_sd>; ++ vccio4-supply = <&vcca1v8_pmu>; ++ vccio5-supply = <&vcc_1v8>; ++ vccio6-supply = <&vcc1v8_dvp>; ++ vccio7-supply = <&vccio_acodec>; ++}; ++ ++&pwm4 { ++ status = "okay"; ++}; ++ ++&rkisp { ++ status = "okay"; ++}; ++ ++&rkisp_mmu { ++ status = "okay"; ++}; ++ ++&rkisp_vir0 { ++ status = "okay"; ++ ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp0_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&csidphy0_out>; ++ }; ++ }; ++}; ++ ++&rknpu { ++ memory-region = <&rknpu_reserved>; ++ rknpu-supply = <&vdd_gpu>; ++ status = "okay"; ++}; ++ ++&rknpu_mmu { ++ status = "disabled"; ++}; ++ ++&rk_rga { ++ status = "okay"; ++}; ++ ++&rkvdec { ++ status = "okay"; ++}; ++ ++&rkvdec_mmu { ++ status = "okay"; ++}; ++ ++&rkvenc { ++ status = "okay"; ++}; ++ ++&rkvenc_mmu { ++ status = "okay"; ++}; ++ ++&route_dsi0 { ++ status = "okay"; ++}; ++ ++&saradc { ++ status = "okay"; ++ vref-supply = <&vcc_1v8>; ++}; ++ ++&sdhci { ++ bus-width = <8>; ++ no-sdio; ++ no-sd; ++ non-removable; ++ max-frequency = <200000000>; ++ status = "okay"; ++}; ++ ++&sdmmc1 { ++ max-frequency = <150000000>; ++ no-sd; ++ no-mmc; ++ bus-width = <4>; ++ disable-wp; ++ cap-sd-highspeed; ++ cap-sdio-irq; ++ keep-power-in-suspend; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ non-removable; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc1_bus4 &sdmmc1_cmd &sdmmc1_clk>; ++ sd-uhs-sdr104; ++ rockchip,default-sample-phase = <90>; ++ status = "okay"; ++}; ++ ++&tsadc { ++ status = "okay"; ++}; ++ ++&uart1 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart1m0_xfer &uart1m0_ctsn>; ++}; ++ ++&u2phy0_otg { ++ status = "okay"; ++}; ++ ++&usb2phy0 { ++ status = "okay"; ++}; ++ ++&usbdrd_dwc3 { ++ status = "okay"; ++}; ++ ++&usbdrd30 { ++ status = "okay"; ++}; ++ ++&vdpu { ++ status = "okay"; ++}; ++ ++&vdpu_mmu { ++ status = "okay"; ++}; ++ ++&vepu { ++ status = "okay"; ++}; ++ ++&vepu_mmu { ++ status = "okay"; ++}; ++ ++&vop { ++ status = "okay"; ++}; ++ ++&vop_mmu { ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3566-rk817-tablet-v10.dts b/arch/arm64/boot/dts/rockchip/rk3566-rk817-tablet-v10.dts +new file mode 100755 +index 000000000000..1d4a0484473a +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3566-rk817-tablet-v10.dts +@@ -0,0 +1,1208 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++/dts-v1/; ++ ++#include ++#include ++#include ++#include ++#include ++#include "rk3566.dtsi" ++#include "rk3568-android.dtsi" ++ ++/ { ++ model = "Rockchip RK3566 RK817 TABLET LP4X Board"; ++ compatible = "rockchip,rk3566-rk817-tablet", "rockchip,rk3566"; ++ ++ adc_keys: adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 0>; ++ io-channel-names = "buttons"; ++ keyup-threshold-microvolt = <1800000>; ++ poll-interval = <100>; ++ ++ vol-up-key { ++ label = "volume up"; ++ linux,code = ; ++ press-threshold-microvolt = <1750>; ++ }; ++ ++ vol-down-key { ++ label = "volume down"; ++ linux,code = ; ++ press-threshold-microvolt = <297500>; ++ }; ++ }; ++ ++ backlight: backlight { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm4 0 25000 0>; ++ brightness-levels = < ++ 0 20 20 21 21 22 22 23 ++ 23 24 24 25 25 26 26 27 ++ 27 28 28 29 29 30 30 31 ++ 31 32 32 33 33 34 34 35 ++ 35 36 36 37 37 38 38 39 ++ 40 41 42 43 44 45 46 47 ++ 48 49 50 50 51 52 53 54 ++ 55 55 56 57 58 59 60 61 ++ 62 63 64 64 65 65 66 67 ++ 68 69 70 71 71 72 73 74 ++ 75 76 77 78 79 79 80 81 ++ 82 83 84 85 86 86 87 88 ++ 89 90 91 92 93 94 94 95 ++ 96 97 98 99 100 101 101 102 ++ 103 104 105 106 107 107 108 109 ++ 110 111 112 113 114 115 115 116 ++ 117 118 119 120 121 122 123 123 ++ 124 125 126 127 128 129 130 130 ++ 131 132 133 134 135 136 136 137 ++ 138 139 140 141 142 143 143 144 ++ 145 146 147 147 148 149 150 151 ++ 152 153 154 155 156 156 157 158 ++ 159 157 158 159 160 161 162 162 ++ 163 164 165 166 167 168 169 169 ++ 170 171 172 173 174 175 175 176 ++ 177 178 179 180 181 182 182 183 ++ 184 185 186 187 188 189 190 190 ++ 191 192 193 194 195 196 197 197 ++ 198 199 200 201 202 203 204 204 ++ 205 206 207 208 209 209 210 211 ++ 212 213 213 214 214 215 215 216 ++ 216 217 217 218 218 219 219 220 ++ >; ++ default-brightness-level = <200>; ++ }; ++ ++ charge-animation { ++ compatible = "rockchip,uboot-charge"; ++ rockchip,uboot-charge-on = <1>; ++ rockchip,android-charge-on = <0>; ++ rockchip,uboot-low-power-voltage = <3350>; ++ rockchip,screen-on-voltage = <3400>; ++ status = "okay"; ++ }; ++ ++ flash_rgb13h: flash-rgb13h { ++ status = "okay"; ++ compatible = "led,rgb13h"; ++ label = "gpio-flash"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&flash_led_gpios>; ++ led-max-microamp = <20000>; ++ flash-max-microamp = <20000>; ++ flash-max-timeout-us = <1000000>; ++ enable-gpio = <&gpio4 6 GPIO_ACTIVE_HIGH>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ }; ++ ++ hdmi_sound: hdmi-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,mclk-fs = <128>; ++ simple-audio-card,name = "rockchip,hdmi"; ++ status = "okay"; ++ ++ simple-audio-card,cpu { ++ sound-dai = <&i2s0_8ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&hdmi>; ++ }; ++ }; ++ ++ vccsys: vccsys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc3v8_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3800000>; ++ regulator-max-microvolt = <3800000>; ++ }; ++ ++ vcc_camera: vcc-camera-regulator { ++ compatible = "regulator-fixed"; ++ gpio = <&gpio4 RK_PB1 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&camera_rst>; ++ regulator-name = "vcc_camera"; ++ enable-active-high; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ rk817-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,rk817-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ ++ simple-audio-card,cpu { ++ sound-dai = <&i2s1_8ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&rk817_codec>; ++ }; ++ }; ++ ++ rk_headset: rk-headset { ++ compatible = "rockchip_headset"; ++ headset_gpio = <&gpio4 RK_PC4 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hp_det>; ++ io-channels = <&saradc 2>; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ clocks = <&rk817 1>; ++ clock-names = "ext_clock"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ post-power-on-delay-ms = <200>; ++ reset-gpios = <&gpio2 RK_PB1 GPIO_ACTIVE_LOW>; ++ }; ++ ++ vcc_sd: vcc-sd { ++ compatible = "regulator-gpio"; ++ enable-active-low; ++ enable-gpio = <&gpio0 RK_PA5 GPIO_ACTIVE_LOW>; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&vcc_sd_h>; ++ regulator-name = "vcc_sd"; ++ states = <3300000 0x0 ++ 3300000 0x1>; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ rockchip,grf = <&grf>; ++ wifi_chip_type = "ap6255"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_host_wake_irq>; ++ WIFI,host_wake_irq = <&gpio2 RK_PB2 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&rk817 1>; ++ clock-names = "ext_clock"; ++ //wifi-bt-power-toggle; ++ uart_rts_gpios = <&gpio2 RK_PB5 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart1m0_rtsn>; ++ pinctrl-1 = <&uart1_gpios>; ++ BT,reset_gpio = <&gpio2 RK_PB7 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio2 RK_PC1 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio2 RK_PC0 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++}; ++ ++&cpu0 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&csi2_dphy_hw { ++ status = "okay"; ++}; ++ ++&csi2_dphy0 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam0: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&gc2385_out>; ++ data-lanes = <1>; ++ }; ++ mipi_in_ucam1: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ov8858_out>; ++ data-lanes = <1 2 3 4>; ++ }; ++ }; ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ csidphy0_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp0_in>; ++ }; ++ }; ++ }; ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&dmc { ++ center-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&dsi0 { ++ status = "okay"; ++ rockchip,lane-rate = <1000>; ++ panel@0 { ++ compatible = "aoly,sl008pa21y1285-b00", "simple-panel-dsi"; ++ reg = <0>; ++ ++ backlight = <&backlight>; ++ //power-supply=<&vcc_3v3>; ++ enable-gpios = <&gpio0 RK_PC7 GPIO_ACTIVE_HIGH>; ++ stbyb-gpios = <&gpio0 RK_PC0 GPIO_ACTIVE_HIGH>; ++ reset-gpios = <&gpio0 RK_PC2 GPIO_ACTIVE_LOW>; ++ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&lcd_enable_gpio>, <&lcd_rst_gpio>, <&lcd_stanby_gpio>; ++ ++ prepare-delay-ms = <120>; ++ reset-delay-ms = <120>; ++ init-delay-ms = <120>; ++ stbyb-delay-ms = <120>; ++ enable-delay-ms = <120>; ++ disable-delay-ms = <120>; ++ unprepare-delay-ms = <120>; ++ ++ width-mm = <229>; ++ height-mm = <143>; ++ ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | ++ MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ ++ panel-init-sequence = [ ++ 23 00 02 B0 01 ++ 23 00 02 C3 0F ++ 23 00 02 C4 00 ++ 23 00 02 C5 00 ++ 23 00 02 C6 00 ++ 23 00 02 C7 00 ++ 23 00 02 C8 0D ++ 23 00 02 C9 12 ++ 23 00 02 CA 11 ++ 23 00 02 CD 1D ++ 23 00 02 CE 1B ++ 23 00 02 CF 0B ++ 23 00 02 D0 09 ++ 23 00 02 D1 07 ++ 23 00 02 D2 05 ++ 23 00 02 D3 01 ++ 23 00 02 D7 10 ++ 23 00 02 D8 00 ++ 23 00 02 D9 00 ++ 23 00 02 DA 00 ++ 23 00 02 DB 00 ++ 23 00 02 DC 0E ++ 23 00 02 DD 12 ++ 23 00 02 DE 11 ++ 23 00 02 E1 1E ++ 23 00 02 E2 1C ++ 23 00 02 E3 0C ++ 23 00 02 E4 0A ++ 23 00 02 E5 08 ++ 23 00 02 E6 06 ++ 23 00 02 E7 02 ++ 23 00 02 B0 03 ++ 23 00 02 BE 03 ++ 23 00 02 CC 44 ++ 23 00 02 C8 07 ++ 23 00 02 C9 05 ++ 23 00 02 CA 42 ++ 23 00 02 CD 3E ++ 23 00 02 CF 60 ++ 23 00 02 D2 04 ++ 23 00 02 D3 04 ++ 23 00 02 D4 01 ++ 23 00 02 D5 00 ++ 23 00 02 D6 03 ++ 23 00 02 D7 04 ++ 23 00 02 D9 01 ++ 23 00 02 DB 01 ++ 23 00 02 E4 F0 ++ 23 00 02 E5 0A ++ 23 00 02 B0 00 ++ 23 00 02 BA 8F// NEW ADD ++ 23 00 02 BD 63 ++ 23 00 02 C2 08 ++ 23 00 02 C4 10 ++ 23 00 02 B0 02 ++ 23 00 02 C0 00 ++ 23 00 02 C1 0A ++ 23 00 02 C2 20 ++ 23 00 02 C3 24 ++ 23 00 02 C4 23 ++ 23 00 02 C5 29 ++ 23 00 02 C6 23 ++ 23 00 02 C7 1C ++ 23 00 02 C8 19 ++ 23 00 02 C9 17 ++ 23 00 02 CA 17 ++ 23 00 02 CB 18 ++ 23 00 02 CC 1A ++ 23 00 02 CD 1E ++ 23 00 02 CE 20 ++ 23 00 02 CF 23 ++ 23 00 02 D0 07 ++ 23 00 02 D1 00 ++ 23 00 02 D2 00 ++ 23 00 02 D3 0A ++ 23 00 02 D4 13 ++ 23 00 02 D5 1C ++ 23 00 02 D6 1A ++ 23 00 02 D7 13 ++ 23 00 02 D8 17 ++ 23 00 02 D9 1C ++ 23 00 02 DA 19 ++ 23 00 02 DB 17 ++ 23 00 02 DC 17 ++ 23 00 02 DD 18 ++ 23 00 02 DE 1A ++ 23 00 02 DF 1E ++ 23 00 02 E0 20 ++ 23 00 02 E1 23 ++ 23 00 02 E2 07 ++ ++ 05 78 01 11 ++ 05 32 01 29 ++ ]; ++ ++ panel-exit-sequence = [ ++ 05 dc 01 28 ++ 05 78 01 10 ++ ]; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <160000000>; ++ hactive = <1200>; ++ vactive = <1920>; ++ ++ hsync-len = <1>;//19 ++ hback-porch = <60>;//40 ++ hfront-porch = <80>;//123 ++ ++ vsync-len = <1>; ++ vback-porch = <25>; ++ vfront-porch = <35>; ++ ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <1>; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ panel_in_dsi: endpoint { ++ remote-endpoint = <&dsi_out_panel>; ++ }; ++ }; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@1 { ++ reg = <1>; ++ dsi_out_panel: endpoint { ++ remote-endpoint = <&panel_in_dsi>; ++ }; ++ }; ++ }; ++ ++}; ++ ++&dsi0_in_vp0 { ++ status = "okay"; ++}; ++ ++&dsi0_in_vp1 { ++ status = "disabled"; ++}; ++ ++&gpu { ++ mali-supply = <&vdd_gpu>; ++ status = "okay"; ++}; ++ ++&hdmi { ++ status = "okay"; ++}; ++ ++&hdmi_in_vp0 { ++ status = "okay"; ++}; ++ ++&hdmi_in_vp1 { ++ status = "disabled"; ++}; ++ ++&hdmi_sound { ++ status = "okay"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ ++ vdd_cpu: tcs4525@1c { ++ compatible = "tcs,tcs452x"; ++ reg = <0x1c>; ++ vin-supply = <&vccsys>; ++ regulator-compatible = "fan53555-reg"; ++ regulator-name = "vdd_cpu"; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1390000>; ++ regulator-init-microvolt = <900000>; ++ regulator-ramp-delay = <2300>; ++ fcs,suspend-voltage-selector = <1>; ++ regulator-boot-on; ++ regulator-always-on; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ rk817: pmic@20 { ++ compatible = "rockchip,rk817"; ++ reg = <0x20>; ++ interrupt-parent = <&gpio0>; ++ interrupts = <3 IRQ_TYPE_LEVEL_LOW>; ++ ++ pinctrl-names = "default", "pmic-sleep", ++ "pmic-power-off", "pmic-reset"; ++ pinctrl-0 = <&pmic_int>; ++ pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; ++ pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; ++ pinctrl-3 = <&soc_slppin_gpio>, <&rk817_slppin_rst>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ //fb-inner-reg-idxs = <2>; ++ /* 1: rst regs (default in codes), 0: rst the pmic */ ++ pmic-reset-func = <0>; ++ /* not save the PMIC_POWER_EN register in uboot */ ++ not-save-power-en = <1>; ++ ++ vcc1-supply = <&vccsys>; ++ vcc2-supply = <&vccsys>; ++ vcc3-supply = <&vccsys>; ++ vcc4-supply = <&vccsys>; ++ vcc5-supply = <&vccsys>; ++ vcc6-supply = <&vccsys>; ++ vcc7-supply = <&vccsys>; ++ vcc8-supply = <&vccsys>; ++ vcc9-supply = <&dcdc_boost>; ++ ++ pwrkey { ++ status = "okay"; ++ }; ++ ++ pinctrl_rk8xx: pinctrl_rk8xx { ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ rk817_slppin_null: rk817_slppin_null { ++ pins = "gpio_slp"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_slppin_slp: rk817_slppin_slp { ++ pins = "gpio_slp"; ++ function = "pin_fun1"; ++ }; ++ ++ rk817_slppin_pwrdn: rk817_slppin_pwrdn { ++ pins = "gpio_slp"; ++ function = "pin_fun2"; ++ }; ++ ++ rk817_slppin_rst: rk817_slppin_rst { ++ pins = "gpio_slp"; ++ function = "pin_fun3"; ++ }; ++ }; ++ ++ regulators { ++ vdd_logic: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <500000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-init-microvolt = <900000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_logic"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <900000>; ++ }; ++ }; ++ ++ vdd_gpu: DCDC_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <500000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-init-microvolt = <900000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_gpu"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_ddr"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_3v3: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_3v3"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcca1v8_pmu: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcca1v8_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdda_0v9: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <900000>; ++ regulator-name = "vdda_0v9"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdda0v9_pmu: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <900000>; ++ regulator-name = "vdda0v9_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <900000>; ++ }; ++ }; ++ ++ vccio_acodec: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vccio_acodec"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vccio_sd: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vccio_sd"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc3v3_pmu: LDO_REG6 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcc3v3_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ ++ vcc_1v8: LDO_REG7 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc_1v8"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc1v8_dvp: LDO_REG8 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc1v8_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc2v8_dvp: LDO_REG9 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <2800000>; ++ regulator-max-microvolt = <2800000>; ++ regulator-name = "vcc2v8_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ dcdc_boost: BOOST { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <4700000>; ++ regulator-max-microvolt = <5400000>; ++ regulator-name = "boost"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ otg_switch: OTG_SWITCH { ++ regulator-name = "otg_switch"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ }; ++ ++ battery { ++ compatible = "rk817,battery"; ++ ocv_table = <3400 3513 3578 3687 3734 3752 3763 ++ 3766 3771 3784 3804 3836 3885 3925 ++ 3962 4005 4063 4114 4169 4227 4303>; ++ design_capacity = <5000>; ++ design_qmax = <5500>; ++ bat_res = <100>; ++ sleep_enter_current = <150>; ++ sleep_exit_current = <180>; ++ sleep_filter_current = <100>; ++ power_off_thresd = <3350>; ++ zero_algorithm_vol = <3850>; ++ max_soc_offset = <60>; ++ monitor_sec = <5>; ++ sample_res = <10>; ++ virtual_power = <0>; ++ }; ++ ++ charger { ++ compatible = "rk817,charger"; ++ min_input_voltage = <4500>; ++ max_input_current = <1500>; ++ max_chrg_current = <2000>; ++ max_chrg_voltage = <4300>; ++ chrg_term_mode = <0>; ++ chrg_finish_cur = <300>; ++ virtual_power = <0>; ++ dc_det_adc = <0>; ++ extcon = <&usb2phy0>; ++ gate_function_disable = <1>; ++ }; ++ ++ rk817_codec: codec { ++ #sound-dai-cells = <0>; ++ compatible = "rockchip,rk817-codec"; ++ clocks = <&cru I2S1_MCLKOUT>; ++ clock-names = "mclk"; ++ assigned-clocks = <&cru I2S1_MCLKOUT>, <&cru I2S1_MCLK_TX_IOE>; ++ assigned-clock-rates = <12288000>; ++ assigned-clock-parents = <&cru I2S1_MCLKOUT_TX>, <&cru I2S1_MCLKOUT_TX>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1m0_mclk>; ++ hp-volume = <20>; ++ spk-volume = <3>; ++ out-l2spk-r2hp; ++ spk-ctl-gpios = <&gpio0 RK_PA6 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ }; ++}; ++ ++&i2c2 { ++ status = "okay"; ++ pinctrl-0 = <&i2c2m1_xfer>; ++ ++ dw9714: dw9714@c { ++ compatible = "dongwoon,dw9714"; ++ status = "okay"; ++ reg = <0x0c>; ++ rockchip,camera-module-index = <0>; ++ rockchip,vcm-start-current = <10>; ++ rockchip,vcm-rated-current = <85>; ++ rockchip,vcm-step-mode = <5>; ++ rockchip,camera-module-facing = "back"; ++ }; ++ ++ gc2385: gc2385@37 { ++ compatible = "galaxycore,gc2385"; ++ status = "okay"; ++ reg = <0x37>; ++ clocks = <&cru CLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ power-domains = <&power RK3568_PD_VI>; ++ pinctrl-names = "rockchip,camera_default"; ++ pinctrl-0 = <&cif_clk>; ++ ++ //reset pin control by hardware,used this pin switch to mipi input ++ //1->2LANE(LANE 0&1) FRONT camera, 0->4LANE REAR camera ++ reset-gpios = <&gpio4 17 GPIO_ACTIVE_LOW>; ++ pwdn-gpios = <&gpio4 10 GPIO_ACTIVE_HIGH>; ++ rockchip,camera-module-index = <1>; ++ rockchip,camera-module-facing = "front"; ++ rockchip,camera-module-name = "HS5885-BNSM1018-V01"; ++ rockchip,camera-module-lens-name = "default"; ++ port { ++ gc2385_out: endpoint { ++ remote-endpoint = <&mipi_in_ucam0>; ++ data-lanes = <1>; ++ }; ++ }; ++ }; ++ ++ ov8858: ov8858@36 { ++ status = "okay"; ++ compatible = "ovti,ov8858"; ++ reg = <0x36>; ++ clocks = <&cru CLK_CAM0_OUT>; ++ clock-names = "xvclk"; ++ power-domains = <&power RK3568_PD_VI>; ++ pinctrl-names = "rockchip,camera_default", "rockchip,camera_sleep"; ++ pinctrl-0 = <&cam_clkout0>; ++ pinctrl-1 = <&cam_sleep>; ++ //reset pin control by hardware,used this pin switch to mipi input ++ //1->2LANE(LANE 0&1) FRONT camera, 0->4LANE REAR camera ++ reset-gpios = <&gpio4 17 GPIO_ACTIVE_LOW>; ++ pwdn-gpios = <&gpio4 11 GPIO_ACTIVE_HIGH>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "HS5885-BNSM1018-V01"; ++ rockchip,camera-module-lens-name = "default"; ++ flash-leds = <&flash_rgb13h>; ++ lens-focus = <&dw9714>; ++ port { ++ ov8858_out: endpoint { ++ remote-endpoint = <&mipi_in_ucam1>; ++ data-lanes = <1 2 3 4>; ++ }; ++ }; ++ }; ++}; ++ ++&i2c3 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2c3m1_xfer>; ++ clock-frequency = <400000>; ++ i2c-scl-rising-time-ns = <138>; ++ i2c-scl-falling-time-ns = <4>; ++ ++ ts@40 { ++ compatible = "gslX680-pad"; ++ reg = <0x40>; ++ touch-gpio = <&gpio3 RK_PB0 IRQ_TYPE_LEVEL_HIGH>; ++ reset-gpio = <&gpio3 RK_PB1 IRQ_TYPE_LEVEL_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&tp_gpio>; ++ screen_max_x = <1200>; ++ screen_max_y = <1920>; ++ revert_x = <0>; ++ revert_y = <1>; ++ revert_xy = <0>; ++ chip_id = <1>; ++ status = "okay"; ++ }; ++}; ++ ++&i2c5 { ++ status = "okay"; ++ clock-frequency = <400000>; ++ i2c-scl-rising-time-ns = <144>; ++ i2c-scl-falling-time-ns = <4>; ++ ++ sensor@18 { ++ compatible = "gs_sc7a20"; ++ reg = <0x18>; ++ type = ; ++ irq_enable = <0>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sensor_gpio>; ++ irq-gpio = <&gpio3 RK_PA2 IRQ_TYPE_EDGE_RISING>; ++ poll_delay_ms = <10>; ++ layout = <1>; ++ }; ++}; ++ ++&i2s0_8ch { ++ status = "okay"; ++}; ++ ++&i2s1_8ch { ++ status = "okay"; ++ rockchip,clk-trcm = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1m0_sclktx ++ &i2s1m0_lrcktx ++ &i2s1m0_sdi0 ++ &i2s1m0_sdo0>; ++}; ++ ++&jpegd { ++ status = "okay"; ++}; ++ ++&jpegd_mmu { ++ status = "okay"; ++}; ++ ++&video_phy0 { ++ status = "okay"; ++}; ++ ++&mpp_srv { ++ status = "okay"; ++}; ++ ++&nandc0 { ++ status = "okay"; ++}; ++ ++&pinctrl { ++ cam { ++ cam_clkout0: cam-clkout0 { ++ rockchip,pins = ++ /* cam_clkout0 */ ++ <4 RK_PA7 1 &pcfg_pull_none>; ++ }; ++ ++ cam_sleep: cam-sleep { ++ rockchip,pins = ++ /* cam_sleep */ ++ <4 RK_PA7 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ camera_rst: camera-rst { ++ rockchip,pins = ++ /* front camera reset */ ++ <4 RK_PB0 RK_FUNC_GPIO &pcfg_pull_none>, ++ /* back camra reset */ ++ <4 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ flash_led_gpios: flash-led { ++ rockchip,pins = ++ /* flash led enable */ ++ <4 RK_PA6 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ tp { ++ tp_gpio: tp-gpio { ++ rockchip,pins = <3 RK_PB0 RK_FUNC_GPIO &pcfg_pull_up>, ++ <3 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ headphone { ++ hp_det: hp-det { ++ rockchip,pins = <4 RK_PC4 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ lcd { ++ lcd_rst_gpio: lcd-rst-gpio { ++ rockchip,pins = <0 RK_PC2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ lcd_enable_gpio: lcd-enable-gpio { ++ rockchip,pins = <0 RK_PC7 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ lcd_stanby_gpio: lcd-stanby-gpio { ++ rockchip,pins = <0 RK_PC0 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ pmic { ++ pmic_int: pmic_int { ++ rockchip,pins = ++ <0 RK_PA3 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ soc_slppin_gpio: soc_slppin_gpio { ++ rockchip,pins = ++ <0 RK_PA2 RK_FUNC_GPIO &pcfg_output_low>; ++ }; ++ ++ soc_slppin_slp: soc_slppin_slp { ++ rockchip,pins = ++ <0 RK_PA2 1 &pcfg_pull_none>; ++ }; ++ ++ soc_slppin_rst: soc_slppin_rst { ++ rockchip,pins = ++ <0 RK_PA2 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ sensor { ++ sensor_gpio: sensor-gpio { ++ rockchip,pins = <3 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <2 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ vcc_sd { ++ vcc_sd_h: vcc-sd-h { ++ rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ wireless-wlan { ++ wifi_host_wake_irq: wifi-host-wake-irq { ++ rockchip,pins = <2 RK_PB2 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart1_gpios: uart1-gpios { ++ rockchip,pins = <2 RK_PB5 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ pmuio1-supply = <&vcc3v3_pmu>; ++ pmuio2-supply = <&vcc3v3_pmu>; ++ vccio1-supply = <&vccio_acodec>; ++ vccio3-supply = <&vccio_sd>; ++ vccio4-supply = <&vcca1v8_pmu>; ++ vccio5-supply = <&vcc_1v8>; ++ vccio6-supply = <&vcc1v8_dvp>; ++ vccio7-supply = <&vcc_3v3>; ++}; ++ ++&pwm4 { ++ status = "okay"; ++}; ++ ++&rk_rga { ++ status = "okay"; ++}; ++ ++&rkisp { ++ status = "okay"; ++}; ++ ++&rkisp_mmu { ++ status = "okay"; ++}; ++ ++&rkisp_vir0 { ++ status = "okay"; ++ ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp0_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&csidphy0_out>; ++ }; ++ }; ++}; ++ ++&rkvdec { ++ status = "okay"; ++}; ++ ++&rkvdec_mmu { ++ status = "okay"; ++}; ++ ++&rkvenc { ++ status = "okay"; ++}; ++ ++&rkvenc_mmu { ++ status = "okay"; ++}; ++ ++&route_dsi0 { ++ status = "okay"; ++}; ++ ++&route_hdmi { ++ status = "okay"; ++ connect = <&vp0_out_hdmi>; ++}; ++ ++&saradc { ++ status = "okay"; ++ vref-supply = <&vcc_1v8>; ++}; ++ ++&sdhci { ++ bus-width = <8>; ++ no-sdio; ++ no-sd; ++ non-removable; ++ max-frequency = <200000000>; ++ status = "okay"; ++}; ++ ++&sdmmc0 { ++ max-frequency = <150000000>; ++ no-sdio; ++ no-mmc; ++ bus-width = <4>; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ disable-wp; ++ sd-uhs-sdr104; ++ vmmc-supply = <&vcc_sd>; ++ vqmmc-supply = <&vccio_sd>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc0_bus4 &sdmmc0_clk &sdmmc0_cmd &sdmmc0_det>; ++ status = "okay"; ++}; ++ ++&sdmmc1 { ++ max-frequency = <150000000>; ++ no-sd; ++ no-mmc; ++ bus-width = <4>; ++ disable-wp; ++ cap-sd-highspeed; ++ cap-sdio-irq; ++ keep-power-in-suspend; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ non-removable; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc1_bus4 &sdmmc1_cmd &sdmmc1_clk>; ++ sd-uhs-sdr104; ++ rockchip,default-sample-phase = <90>; ++ status = "okay"; ++}; ++ ++&tsadc { ++ status = "okay"; ++}; ++ ++&uart1 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart1m0_xfer &uart1m0_ctsn>; ++}; ++ ++&u2phy0_otg { ++ status = "okay"; ++}; ++ ++&usb2phy0 { ++ status = "okay"; ++}; ++ ++&usbdrd_dwc3 { ++ status = "okay"; ++}; ++ ++&usbdrd30 { ++ status = "okay"; ++}; ++ ++&vdpu { ++ status = "okay"; ++}; ++ ++&vdpu_mmu { ++ status = "okay"; ++}; ++ ++&vepu { ++ status = "okay"; ++}; ++ ++&vepu_mmu { ++ status = "okay"; ++}; ++ ++&vop { ++ status = "okay"; ++}; ++ ++&vop_mmu { ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3566-rk817-tablet.dts b/arch/arm64/boot/dts/rockchip/rk3566-rk817-tablet.dts +new file mode 100755 +index 000000000000..40022eb6c646 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3566-rk817-tablet.dts +@@ -0,0 +1,1213 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++/dts-v1/; ++ ++#include ++#include ++#include ++#include ++#include ++#include "rk3566.dtsi" ++#include "rk3568-android.dtsi" ++ ++/ { ++ model = "Rockchip RK3566 RK817 TABLET LP4X Board"; ++ compatible = "rockchip,rk3566-rk817-tablet", "rockchip,rk3566"; ++ ++ adc_keys: adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 0>; ++ io-channel-names = "buttons"; ++ keyup-threshold-microvolt = <1800000>; ++ poll-interval = <100>; ++ ++ vol-up-key { ++ label = "volume up"; ++ linux,code = ; ++ press-threshold-microvolt = <1750>; ++ }; ++ ++ vol-down-key { ++ label = "volume down"; ++ linux,code = ; ++ press-threshold-microvolt = <297500>; ++ }; ++ }; ++ ++ backlight: backlight { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm4 0 25000 0>; ++ brightness-levels = < ++ 0 20 20 21 21 22 22 23 ++ 23 24 24 25 25 26 26 27 ++ 27 28 28 29 29 30 30 31 ++ 31 32 32 33 33 34 34 35 ++ 35 36 36 37 37 38 38 39 ++ 40 41 42 43 44 45 46 47 ++ 48 49 50 50 51 52 53 54 ++ 55 55 56 57 58 59 60 61 ++ 62 63 64 64 65 65 66 67 ++ 68 69 70 71 71 72 73 74 ++ 75 76 77 78 79 79 80 81 ++ 82 83 84 85 86 86 87 88 ++ 89 90 91 92 93 94 94 95 ++ 96 97 98 99 100 101 101 102 ++ 103 104 105 106 107 107 108 109 ++ 110 111 112 113 114 115 115 116 ++ 117 118 119 120 121 122 123 123 ++ 124 125 126 127 128 129 130 130 ++ 131 132 133 134 135 136 136 137 ++ 138 139 140 141 142 143 143 144 ++ 145 146 147 147 148 149 150 151 ++ 152 153 154 155 156 156 157 158 ++ 159 157 158 159 160 161 162 162 ++ 163 164 165 166 167 168 169 169 ++ 170 171 172 173 174 175 175 176 ++ 177 178 179 180 181 182 182 183 ++ 184 185 186 187 188 189 190 190 ++ 191 192 193 194 195 196 197 197 ++ 198 199 200 201 202 203 204 204 ++ 205 206 207 208 209 209 210 211 ++ 212 213 213 214 214 215 215 216 ++ 216 217 217 218 218 219 219 220 ++ >; ++ default-brightness-level = <200>; ++ }; ++ ++ charge-animation { ++ compatible = "rockchip,uboot-charge"; ++ rockchip,uboot-charge-on = <1>; ++ rockchip,android-charge-on = <0>; ++ rockchip,uboot-low-power-voltage = <3350>; ++ rockchip,screen-on-voltage = <3400>; ++ status = "okay"; ++ }; ++ ++ flash_rgb13h: flash-rgb13h { ++ status = "okay"; ++ compatible = "led,rgb13h"; ++ label = "gpio-flash"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&flash_led_gpios>; ++ led-max-microamp = <20000>; ++ flash-max-microamp = <20000>; ++ flash-max-timeout-us = <1000000>; ++ enable-gpio = <&gpio4 6 GPIO_ACTIVE_HIGH>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ }; ++ ++ hdmi_sound: hdmi-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,mclk-fs = <128>; ++ simple-audio-card,name = "rockchip,hdmi"; ++ status = "okay"; ++ ++ simple-audio-card,cpu { ++ sound-dai = <&i2s0_8ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&hdmi>; ++ }; ++ }; ++ ++ vccsys: vccsys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc3v8_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3800000>; ++ regulator-max-microvolt = <3800000>; ++ }; ++ ++ vcc_camera: vcc-camera-regulator { ++ compatible = "regulator-fixed"; ++ gpio = <&gpio4 RK_PB1 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&camera_rst>; ++ regulator-name = "vcc_camera"; ++ enable-active-high; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ rk817-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,rk817-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ ++ simple-audio-card,cpu { ++ sound-dai = <&i2s1_8ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&rk817_codec>; ++ }; ++ }; ++ ++ rk_headset: rk-headset { ++ compatible = "rockchip_headset"; ++ headset_gpio = <&gpio4 RK_PC4 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hp_det>; ++ io-channels = <&saradc 2>; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ clocks = <&rk817 1>; ++ clock-names = "ext_clock"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ post-power-on-delay-ms = <200>; ++ reset-gpios = <&gpio0 RK_PC0 GPIO_ACTIVE_LOW>; ++ }; ++ ++ vcc_sd: vcc-sd { ++ compatible = "regulator-gpio"; ++ enable-active-low; ++ enable-gpio = <&gpio0 RK_PA5 GPIO_ACTIVE_LOW>; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&vcc_sd_h>; ++ regulator-name = "vcc_sd"; ++ states = <3300000 0x0 ++ 3300000 0x1>; ++ }; ++ ++ wireless-wlan { ++ compatible = "wlan-platdata"; ++ rockchip,grf = <&grf>; ++ wifi_chip_type = "ap6255"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_host_wake_irq>; ++ WIFI,host_wake_irq = <&gpio0 RK_PC4 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&rk817 1>; ++ clock-names = "ext_clock"; ++ //wifi-bt-power-toggle; ++ uart_rts_gpios = <&gpio2 RK_PB5 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart1m0_rtsn>; ++ pinctrl-1 = <&uart1_gpios>; ++ BT,reset_gpio = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio0 RK_PB4 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio0 RK_PB3 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++}; ++ ++&bus_npu { ++ bus-supply = <&vdd_logic>; ++ pvtm-supply = <&vdd_cpu>; ++ status = "okay"; ++}; ++ ++&cpu0 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&csi2_dphy_hw { ++ status = "okay"; ++}; ++ ++&csi2_dphy0 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam0: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&gc2385_out>; ++ data-lanes = <1>; ++ }; ++ mipi_in_ucam1: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ov8858_out>; ++ data-lanes = <1 2 3 4>; ++ }; ++ }; ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ csidphy0_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp0_in>; ++ }; ++ }; ++ }; ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&dmc { ++ center-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&dsi0 { ++ status = "okay"; ++ rockchip,lane-rate = <1000>; ++ panel@0 { ++ compatible = "aoly,sl008pa21y1285-b00", "simple-panel-dsi"; ++ reg = <0>; ++ ++ backlight = <&backlight>; ++ //power-supply=<&vcc_3v3>; ++ enable-gpios = <&gpio0 RK_PC7 GPIO_ACTIVE_HIGH>; ++ reset-gpios = <&gpio0 RK_PC2 GPIO_ACTIVE_LOW>; ++ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&lcd_enable_gpio>, <&lcd_rst_gpio>; ++ ++ prepare-delay-ms = <120>; ++ reset-delay-ms = <120>; ++ init-delay-ms = <120>; ++ stbyb-delay-ms = <120>; ++ enable-delay-ms = <120>; ++ disable-delay-ms = <120>; ++ unprepare-delay-ms = <120>; ++ ++ width-mm = <229>; ++ height-mm = <143>; ++ ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | ++ MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ ++ panel-init-sequence = [ ++ 23 00 02 B0 01 ++ 23 00 02 C3 0F ++ 23 00 02 C4 00 ++ 23 00 02 C5 00 ++ 23 00 02 C6 00 ++ 23 00 02 C7 00 ++ 23 00 02 C8 0D ++ 23 00 02 C9 12 ++ 23 00 02 CA 11 ++ 23 00 02 CD 1D ++ 23 00 02 CE 1B ++ 23 00 02 CF 0B ++ 23 00 02 D0 09 ++ 23 00 02 D1 07 ++ 23 00 02 D2 05 ++ 23 00 02 D3 01 ++ 23 00 02 D7 10 ++ 23 00 02 D8 00 ++ 23 00 02 D9 00 ++ 23 00 02 DA 00 ++ 23 00 02 DB 00 ++ 23 00 02 DC 0E ++ 23 00 02 DD 12 ++ 23 00 02 DE 11 ++ 23 00 02 E1 1E ++ 23 00 02 E2 1C ++ 23 00 02 E3 0C ++ 23 00 02 E4 0A ++ 23 00 02 E5 08 ++ 23 00 02 E6 06 ++ 23 00 02 E7 02 ++ 23 00 02 B0 03 ++ 23 00 02 BE 03 ++ 23 00 02 CC 44 ++ 23 00 02 C8 07 ++ 23 00 02 C9 05 ++ 23 00 02 CA 42 ++ 23 00 02 CD 3E ++ 23 00 02 CF 60 ++ 23 00 02 D2 04 ++ 23 00 02 D3 04 ++ 23 00 02 D4 01 ++ 23 00 02 D5 00 ++ 23 00 02 D6 03 ++ 23 00 02 D7 04 ++ 23 00 02 D9 01 ++ 23 00 02 DB 01 ++ 23 00 02 E4 F0 ++ 23 00 02 E5 0A ++ 23 00 02 B0 00 ++ 23 00 02 BA 8F// NEW ADD ++ 23 00 02 BD 63 ++ 23 00 02 C2 08 ++ 23 00 02 C4 10 ++ 23 00 02 B0 02 ++ 23 00 02 C0 00 ++ 23 00 02 C1 0A ++ 23 00 02 C2 20 ++ 23 00 02 C3 24 ++ 23 00 02 C4 23 ++ 23 00 02 C5 29 ++ 23 00 02 C6 23 ++ 23 00 02 C7 1C ++ 23 00 02 C8 19 ++ 23 00 02 C9 17 ++ 23 00 02 CA 17 ++ 23 00 02 CB 18 ++ 23 00 02 CC 1A ++ 23 00 02 CD 1E ++ 23 00 02 CE 20 ++ 23 00 02 CF 23 ++ 23 00 02 D0 07 ++ 23 00 02 D1 00 ++ 23 00 02 D2 00 ++ 23 00 02 D3 0A ++ 23 00 02 D4 13 ++ 23 00 02 D5 1C ++ 23 00 02 D6 1A ++ 23 00 02 D7 13 ++ 23 00 02 D8 17 ++ 23 00 02 D9 1C ++ 23 00 02 DA 19 ++ 23 00 02 DB 17 ++ 23 00 02 DC 17 ++ 23 00 02 DD 18 ++ 23 00 02 DE 1A ++ 23 00 02 DF 1E ++ 23 00 02 E0 20 ++ 23 00 02 E1 23 ++ 23 00 02 E2 07 ++ ++ 05 78 01 11 ++ 05 32 01 29 ++ ]; ++ ++ panel-exit-sequence = [ ++ 05 dc 01 28 ++ 05 78 01 10 ++ ]; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <160000000>; ++ hactive = <1200>; ++ vactive = <1920>; ++ ++ hsync-len = <1>;//19 ++ hback-porch = <60>;//40 ++ hfront-porch = <80>;//123 ++ ++ vsync-len = <1>; ++ vback-porch = <25>; ++ vfront-porch = <35>; ++ ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <1>; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ panel_in_dsi: endpoint { ++ remote-endpoint = <&dsi_out_panel>; ++ }; ++ }; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@1 { ++ reg = <1>; ++ dsi_out_panel: endpoint { ++ remote-endpoint = <&panel_in_dsi>; ++ }; ++ }; ++ }; ++ ++}; ++ ++&dsi0_in_vp0 { ++ status = "disabled"; ++}; ++ ++&dsi0_in_vp1 { ++ status = "okay"; ++}; ++ ++&gpu { ++ mali-supply = <&vdd_gpu>; ++ status = "okay"; ++}; ++ ++&hdmi { ++ status = "okay"; ++}; ++ ++&hdmi_in_vp0 { ++ status = "okay"; ++}; ++ ++&hdmi_in_vp1 { ++ status = "disabled"; ++}; ++ ++&hdmi_sound { ++ status = "okay"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ ++ vdd_cpu: tcs4525@1c { ++ compatible = "tcs,tcs452x"; ++ reg = <0x1c>; ++ vin-supply = <&vccsys>; ++ regulator-compatible = "fan53555-reg"; ++ regulator-name = "vdd_cpu"; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1390000>; ++ regulator-init-microvolt = <900000>; ++ regulator-ramp-delay = <2300>; ++ fcs,suspend-voltage-selector = <1>; ++ regulator-boot-on; ++ regulator-always-on; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ rk817: pmic@20 { ++ compatible = "rockchip,rk817"; ++ reg = <0x20>; ++ interrupt-parent = <&gpio0>; ++ interrupts = <3 IRQ_TYPE_LEVEL_LOW>; ++ ++ pinctrl-names = "default", "pmic-sleep", ++ "pmic-power-off", "pmic-reset"; ++ pinctrl-0 = <&pmic_int>; ++ pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; ++ pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; ++ pinctrl-3 = <&soc_slppin_gpio>, <&rk817_slppin_rst>; ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ //fb-inner-reg-idxs = <2>; ++ /* 1: rst regs (default in codes), 0: rst the pmic */ ++ pmic-reset-func = <0>; ++ /* not save the PMIC_POWER_EN register in uboot */ ++ not-save-power-en = <1>; ++ ++ vcc1-supply = <&vccsys>; ++ vcc2-supply = <&vccsys>; ++ vcc3-supply = <&vccsys>; ++ vcc4-supply = <&vccsys>; ++ vcc5-supply = <&vccsys>; ++ vcc6-supply = <&vccsys>; ++ vcc7-supply = <&vccsys>; ++ vcc8-supply = <&vccsys>; ++ vcc9-supply = <&dcdc_boost>; ++ ++ pwrkey { ++ status = "okay"; ++ }; ++ ++ pinctrl_rk8xx: pinctrl_rk8xx { ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ rk817_slppin_null: rk817_slppin_null { ++ pins = "gpio_slp"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_slppin_slp: rk817_slppin_slp { ++ pins = "gpio_slp"; ++ function = "pin_fun1"; ++ }; ++ ++ rk817_slppin_pwrdn: rk817_slppin_pwrdn { ++ pins = "gpio_slp"; ++ function = "pin_fun2"; ++ }; ++ ++ rk817_slppin_rst: rk817_slppin_rst { ++ pins = "gpio_slp"; ++ function = "pin_fun3"; ++ }; ++ }; ++ ++ regulators { ++ vdd_logic: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <500000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-init-microvolt = <900000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_logic"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ regulator-suspend-microvolt = <900000>; ++ }; ++ }; ++ ++ vdd_gpu: DCDC_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <500000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-init-microvolt = <900000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_gpu"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_ddr"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vcc_3v3: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_3v3"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcca1v8_pmu: LDO_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcca1v8_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vdda_0v9: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <900000>; ++ regulator-name = "vdda_0v9"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdda0v9_pmu: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <900000>; ++ regulator-name = "vdda0v9_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <900000>; ++ }; ++ }; ++ ++ vccio_acodec: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vccio_acodec"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vccio_sd: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vccio_sd"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc3v3_pmu: LDO_REG6 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcc3v3_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3000000>; ++ }; ++ }; ++ ++ vcc_1v8: LDO_REG7 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc_1v8"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc1v8_dvp: LDO_REG8 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc1v8_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc2v8_dvp: LDO_REG9 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <2800000>; ++ regulator-max-microvolt = <2800000>; ++ regulator-name = "vcc2v8_dvp"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ dcdc_boost: BOOST { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ regulator-name = "boost"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ otg_switch: OTG_SWITCH { ++ regulator-name = "otg_switch"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ }; ++ ++ battery { ++ compatible = "rk817,battery"; ++ ocv_table = <3400 3513 3578 3687 3734 3752 3763 ++ 3766 3771 3784 3804 3836 3885 3925 ++ 3962 4005 4063 4114 4169 4227 4303>; ++ design_capacity = <5000>; ++ design_qmax = <5500>; ++ bat_res = <100>; ++ sleep_enter_current = <150>; ++ sleep_exit_current = <180>; ++ sleep_filter_current = <100>; ++ power_off_thresd = <3350>; ++ zero_algorithm_vol = <3850>; ++ max_soc_offset = <60>; ++ monitor_sec = <5>; ++ sample_res = <10>; ++ virtual_power = <0>; ++ }; ++ ++ charger { ++ compatible = "rk817,charger"; ++ min_input_voltage = <4500>; ++ max_input_current = <1500>; ++ max_chrg_current = <2000>; ++ max_chrg_voltage = <4300>; ++ chrg_term_mode = <0>; ++ chrg_finish_cur = <300>; ++ virtual_power = <0>; ++ dc_det_adc = <0>; ++ extcon = <&usb2phy0>; ++ }; ++ ++ rk817_codec: codec { ++ #sound-dai-cells = <0>; ++ compatible = "rockchip,rk817-codec"; ++ clocks = <&cru I2S1_MCLKOUT>; ++ clock-names = "mclk"; ++ assigned-clocks = <&cru I2S1_MCLKOUT>, <&cru I2S1_MCLK_TX_IOE>; ++ assigned-clock-rates = <12288000>; ++ assigned-clock-parents = <&cru I2S1_MCLKOUT_TX>, <&cru I2S1_MCLKOUT_TX>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1m0_mclk>; ++ hp-volume = <20>; ++ spk-volume = <3>; ++ out-l2spk-r2hp; ++ spk-ctl-gpios = <&gpio0 RK_PA6 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ }; ++}; ++ ++&i2c2 { ++ status = "okay"; ++ pinctrl-0 = <&i2c2m1_xfer>; ++ ++ dw9714: dw9714@c { ++ compatible = "dongwoon,dw9714"; ++ status = "okay"; ++ reg = <0x0c>; ++ rockchip,camera-module-index = <0>; ++ rockchip,vcm-start-current = <10>; ++ rockchip,vcm-rated-current = <85>; ++ rockchip,vcm-step-mode = <5>; ++ rockchip,camera-module-facing = "back"; ++ }; ++ ++ gc2385: gc2385@37 { ++ compatible = "galaxycore,gc2385"; ++ status = "okay"; ++ reg = <0x37>; ++ clocks = <&cru CLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ power-domains = <&power RK3568_PD_VI>; ++ pinctrl-names = "rockchip,camera_default"; ++ pinctrl-0 = <&cif_clk>; ++ ++ //reset pin control by hardware,used this pin switch to mipi input ++ //1->2LANE(LANE 0&1) FRONT camera, 0->4LANE REAR camera ++ reset-gpios = <&gpio4 17 GPIO_ACTIVE_LOW>; ++ pwdn-gpios = <&gpio4 10 GPIO_ACTIVE_HIGH>; ++ rockchip,camera-module-index = <1>; ++ rockchip,camera-module-facing = "front"; ++ rockchip,camera-module-name = "HS5885-BNSM1018-V01"; ++ rockchip,camera-module-lens-name = "default"; ++ port { ++ gc2385_out: endpoint { ++ remote-endpoint = <&mipi_in_ucam0>; ++ data-lanes = <1>; ++ }; ++ }; ++ }; ++ ++ ov8858: ov8858@36 { ++ status = "okay"; ++ compatible = "ovti,ov8858"; ++ reg = <0x36>; ++ clocks = <&cru CLK_CAM0_OUT>; ++ clock-names = "xvclk"; ++ power-domains = <&power RK3568_PD_VI>; ++ pinctrl-names = "rockchip,camera_default", "rockchip,camera_sleep"; ++ pinctrl-0 = <&cam_clkout0>; ++ pinctrl-1 = <&cam_sleep>; ++ //reset pin control by hardware,used this pin switch to mipi input ++ //1->2LANE(LANE 0&1) FRONT camera, 0->4LANE REAR camera ++ reset-gpios = <&gpio4 17 GPIO_ACTIVE_LOW>; ++ pwdn-gpios = <&gpio4 11 GPIO_ACTIVE_HIGH>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "HS5885-BNSM1018-V01"; ++ rockchip,camera-module-lens-name = "default"; ++ flash-leds = <&flash_rgb13h>; ++ lens-focus = <&dw9714>; ++ port { ++ ov8858_out: endpoint { ++ remote-endpoint = <&mipi_in_ucam1>; ++ data-lanes = <1 2 3 4>; ++ }; ++ }; ++ }; ++}; ++ ++&i2c3 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2c3m1_xfer>; ++ clock-frequency = <400000>; ++ i2c-scl-rising-time-ns = <138>; ++ i2c-scl-falling-time-ns = <4>; ++ ++ ts@40 { ++ compatible = "gslX680-pad"; ++ reg = <0x40>; ++ touch-gpio = <&gpio3 RK_PB0 IRQ_TYPE_LEVEL_HIGH>; ++ reset-gpio = <&gpio3 RK_PB1 IRQ_TYPE_LEVEL_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&tp_gpio>; ++ screen_max_x = <1200>; ++ screen_max_y = <1920>; ++ revert_x = <0>; ++ revert_y = <1>; ++ revert_xy = <0>; ++ chip_id = <1>; ++ status = "okay"; ++ }; ++}; ++ ++&i2c5 { ++ status = "okay"; ++ clock-frequency = <400000>; ++ i2c-scl-rising-time-ns = <144>; ++ i2c-scl-falling-time-ns = <4>; ++ ++ sensor@18 { ++ compatible = "gs_sc7a20"; ++ reg = <0x18>; ++ type = ; ++ irq_enable = <0>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sensor_gpio>; ++ irq-gpio = <&gpio3 RK_PA2 IRQ_TYPE_EDGE_RISING>; ++ poll_delay_ms = <10>; ++ layout = <1>; ++ }; ++}; ++ ++&i2s0_8ch { ++ status = "okay"; ++}; ++ ++&i2s1_8ch { ++ status = "okay"; ++ rockchip,clk-trcm = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1m0_sclktx ++ &i2s1m0_lrcktx ++ &i2s1m0_sdi0 ++ &i2s1m0_sdo0>; ++}; ++ ++&jpegd { ++ status = "okay"; ++}; ++ ++&jpegd_mmu { ++ status = "okay"; ++}; ++ ++&video_phy0 { ++ status = "okay"; ++}; ++ ++&mpp_srv { ++ status = "okay"; ++}; ++ ++&nandc0 { ++ status = "okay"; ++}; ++ ++&pinctrl { ++ cam { ++ cam_clkout0: cam-clkout0 { ++ rockchip,pins = ++ /* cam_clkout0 */ ++ <4 RK_PA7 1 &pcfg_pull_none>; ++ }; ++ ++ cam_sleep: cam-sleep { ++ rockchip,pins = ++ /* cam_sleep */ ++ <4 RK_PA7 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ camera_rst: camera-rst { ++ rockchip,pins = ++ /* front camera reset */ ++ <4 RK_PB0 RK_FUNC_GPIO &pcfg_pull_none>, ++ /* back camra reset */ ++ <4 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ flash_led_gpios: flash-led { ++ rockchip,pins = ++ /* flash led enable */ ++ <4 RK_PA6 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ tp { ++ tp_gpio: tp-gpio { ++ rockchip,pins = <3 RK_PB0 RK_FUNC_GPIO &pcfg_pull_up>, ++ <3 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ headphone { ++ hp_det: hp-det { ++ rockchip,pins = <4 RK_PC4 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ lcd { ++ lcd_rst_gpio: lcd-rst-gpio { ++ rockchip,pins = <0 RK_PC2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ lcd_enable_gpio: lcd-enable-gpio { ++ rockchip,pins = <0 RK_PC7 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ }; ++ ++ pmic { ++ pmic_int: pmic_int { ++ rockchip,pins = ++ <0 RK_PA3 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ soc_slppin_gpio: soc_slppin_gpio { ++ rockchip,pins = ++ <0 RK_PA2 RK_FUNC_GPIO &pcfg_output_low>; ++ }; ++ ++ soc_slppin_slp: soc_slppin_slp { ++ rockchip,pins = ++ <0 RK_PA2 1 &pcfg_pull_none>; ++ }; ++ ++ soc_slppin_rst: soc_slppin_rst { ++ rockchip,pins = ++ <0 RK_PA2 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ sensor { ++ sensor_gpio: sensor-gpio { ++ rockchip,pins = <3 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <0 RK_PC0 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ vcc_sd { ++ vcc_sd_h: vcc-sd-h { ++ rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ wireless-wlan { ++ wifi_host_wake_irq: wifi-host-wake-irq { ++ rockchip,pins = <0 RK_PC4 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart1_gpios: uart1-gpios { ++ rockchip,pins = <2 RK_PB5 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++&pmu_io_domains { ++ status = "okay"; ++ pmuio1-supply = <&vcc3v3_pmu>; ++ pmuio2-supply = <&vcc_3v3>; ++ vccio1-supply = <&vccio_acodec>; ++ vccio3-supply = <&vccio_sd>; ++ vccio4-supply = <&vcc_3v3>; ++ vccio5-supply = <&vcc_3v3>; ++ vccio6-supply = <&vcc_3v3>; ++ vccio7-supply = <&vcc_3v3>; ++}; ++ ++&pwm4 { ++ status = "okay"; ++}; ++ ++&rk_rga { ++ status = "okay"; ++}; ++ ++&rkisp { ++ status = "okay"; ++}; ++ ++&rkisp_mmu { ++ status = "okay"; ++}; ++ ++&rkisp_vir0 { ++ status = "okay"; ++ ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp0_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&csidphy0_out>; ++ }; ++ }; ++}; ++ ++&rkvdec { ++ status = "okay"; ++}; ++ ++&rkvdec_mmu { ++ status = "okay"; ++}; ++ ++&rkvenc { ++ status = "okay"; ++}; ++ ++&rkvenc_mmu { ++ status = "okay"; ++}; ++ ++&route_dsi0 { ++ status = "okay"; ++ connect = <&vp1_out_dsi0>; ++}; ++ ++&route_hdmi { ++ status = "okay"; ++ connect = <&vp0_out_hdmi>; ++}; ++ ++&saradc { ++ status = "okay"; ++ vref-supply = <&vcc_1v8>; ++}; ++ ++&sdhci { ++ bus-width = <8>; ++ no-sdio; ++ no-sd; ++ non-removable; ++ max-frequency = <200000000>; ++ status = "okay"; ++}; ++ ++&sdmmc0 { ++ max-frequency = <150000000>; ++ no-sdio; ++ no-mmc; ++ bus-width = <4>; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ disable-wp; ++ sd-uhs-sdr104; ++ vmmc-supply = <&vcc_sd>; ++ vqmmc-supply = <&vccio_sd>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc0_bus4 &sdmmc0_clk &sdmmc0_cmd &sdmmc0_det>; ++ status = "okay"; ++}; ++ ++&sdmmc1 { ++ max-frequency = <150000000>; ++ no-sd; ++ no-mmc; ++ bus-width = <4>; ++ disable-wp; ++ cap-sd-highspeed; ++ cap-sdio-irq; ++ keep-power-in-suspend; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ non-removable; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc1_bus4 &sdmmc1_cmd &sdmmc1_clk>; ++ sd-uhs-sdr104; ++ rockchip,default-sample-phase = <90>; ++ status = "okay"; ++}; ++ ++&tsadc { ++ status = "okay"; ++}; ++ ++&uart1 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart1m0_xfer &uart1m0_ctsn>; ++}; ++ ++&u2phy0_otg { ++ status = "okay"; ++}; ++ ++&usb2phy0 { ++ status = "okay"; ++}; ++ ++&usbdrd_dwc3 { ++ status = "okay"; ++}; ++ ++&usbdrd30 { ++ status = "okay"; ++}; ++ ++&vdpu { ++ status = "okay"; ++}; ++ ++&vdpu_mmu { ++ status = "okay"; ++}; ++ ++&vepu { ++ status = "okay"; ++}; ++ ++&vepu_mmu { ++ status = "okay"; ++}; ++ ++&vop { ++ status = "okay"; ++ assigned-clocks = <&cru DCLK_VOP0>, <&cru DCLK_VOP1>; ++ assigned-clock-parents = <&pmucru PLL_HPLL>, <&cru PLL_VPLL>; ++}; ++ ++&vop_mmu { ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3566.dtsi b/arch/arm64/boot/dts/rockchip/rk3566.dtsi +new file mode 100755 +index 000000000000..01b6499b678d +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3566.dtsi +@@ -0,0 +1,53 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ */ ++ ++#include "rk3568.dtsi" ++ ++/ { ++ aliases { ++ /delete-property/ ethernet0; ++ }; ++}; ++ ++&cpu0_opp_table { ++ /delete-node/ opp-1992000000; ++}; ++ ++&power { ++ pd_pipe@RK3568_PD_PIPE { ++ reg = ; ++ clocks = <&cru PCLK_PIPE>; ++ pm_qos = <&qos_pcie2x1>, ++ <&qos_sata1>, ++ <&qos_sata2>, ++ <&qos_usb3_0>, ++ <&qos_usb3_1>; ++ }; ++}; ++ ++&rkisp { ++ rockchip,iq-feature = /bits/ 64 <0x3FBF7FE67FF>; ++}; ++ ++&usbdrd_dwc3 { ++ phys = <&u2phy0_otg>; ++ phy-names = "usb2-phy"; ++ extcon = <&usb2phy0>; ++ maximum-speed = "high-speed"; ++ snps,dis_u2_susphy_quirk; ++}; ++ ++/delete-node/ &combphy0_us; ++/delete-node/ &gmac0_clkin; ++/delete-node/ &gmac0_xpcsclk; ++/delete-node/ &gmac0; ++/delete-node/ &pcie30_phy_grf; ++/delete-node/ &pcie30phy; ++/delete-node/ &pcie3x1; ++/delete-node/ &pcie3x2; ++/delete-node/ &qos_pcie3x1; ++/delete-node/ &qos_pcie3x2; ++/delete-node/ &qos_sata0; ++/delete-node/ &sata0; +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-android.dtsi b/arch/arm64/boot/dts/rockchip/rk3568-android.dtsi +new file mode 100755 +index 000000000000..01cd37c04e1a +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-android.dtsi +@@ -0,0 +1,74 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++/ { ++ chosen: chosen { ++ bootargs = "earlycon=uart8250,mmio32,0xfe660000 console=ttyFIQ0"; ++ }; ++ ++ aliases { ++ mmc0 = &sdmmc0; ++ mmc1 = &sdmmc1; ++ mmc2 = &sdhci; ++ mmc3 = &sdmmc2; ++ }; ++ ++ fiq-debugger { ++ compatible = "rockchip,fiq-debugger"; ++ rockchip,serial-id = <2>; ++ rockchip,wake-irq = <0>; ++ /* If enable uart uses irq instead of fiq */ ++ rockchip,irq-mode-enable = <1>; ++ rockchip,baudrate = <1500000>; /* Only 115200 and 1500000 */ ++ interrupts = ; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart2m0_xfer>; ++ status = "okay"; ++ }; ++ ++ debug: debug@fd904000 { ++ compatible = "rockchip,debug"; ++ reg = <0x0 0xfd904000 0x0 0x1000>, ++ <0x0 0xfd905000 0x0 0x1000>, ++ <0x0 0xfd906000 0x0 0x1000>, ++ <0x0 0xfd907000 0x0 0x1000>; ++ }; ++ ++ cspmu: cspmu@fd90c000 { ++ compatible = "rockchip,cspmu"; ++ reg = <0x0 0xfd90c000 0x0 0x1000>, ++ <0x0 0xfd90d000 0x0 0x1000>, ++ <0x0 0xfd90e000 0x0 0x1000>, ++ <0x0 0xfd90f000 0x0 0x1000>; ++ }; ++}; ++ ++&reserved_memory { ++ linux,cma { ++ compatible = "shared-dma-pool"; ++ inactive; ++ reusable; ++ reg = <0x0 0x10000000 0x0 0x00800000>; ++ linux,cma-default; ++ }; ++ ++ ramoops: ramoops@110000 { ++ compatible = "ramoops"; ++ reg = <0x0 0x110000 0x0 0xf0000>; ++ record-size = <0x20000>; ++ console-size = <0x80000>; ++ ftrace-size = <0x00000>; ++ pmsg-size = <0x50000>; ++ }; ++}; ++ ++&rng { ++ status = "okay"; ++}; ++ ++&vop { ++ support-multi-area; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-dram-default-timing.dtsi b/arch/arm64/boot/dts/rockchip/rk3568-dram-default-timing.dtsi +new file mode 100755 +index 000000000000..43f978809c02 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-dram-default-timing.dtsi +@@ -0,0 +1,81 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++#include ++#include ++ ++/ { ++ ddr_timing: ddr_timing { ++ compatible = "rockchip,ddr-timing"; ++ ddr2_speed_bin = ; ++ ddr3_speed_bin = ; ++ ddr4_speed_bin = ; ++ pd_idle = <13>; ++ sr_idle = <93>; ++ sr_mc_gate_idle = <0>; ++ srpd_lite_idle = <0>; ++ standby_idle = <0>; ++ ++ auto_pd_dis_freq = <1066>; ++ auto_sr_dis_freq = <800>; ++ ddr2_dll_dis_freq = <300>; ++ ddr3_dll_dis_freq = <300>; ++ ddr4_dll_dis_freq = <625>; ++ phy_dll_dis_freq = <400>; ++ ++ ddr2_odt_dis_freq = <100>; ++ phy_ddr2_odt_dis_freq = <100>; ++ ddr2_drv = ; ++ ddr2_odt = ; ++ phy_ddr2_ca_drv = ; ++ phy_ddr2_ck_drv = ; ++ phy_ddr2_dq_drv = ; ++ phy_ddr2_odt = ; ++ ++ ddr3_odt_dis_freq = <333>; ++ phy_ddr3_odt_dis_freq = <333>; ++ ddr3_drv = ; ++ ddr3_odt = ; ++ phy_ddr3_ca_drv = ; ++ phy_ddr3_ck_drv = ; ++ phy_ddr3_dq_drv = ; ++ phy_ddr3_odt = ; ++ ++ phy_lpddr2_odt_dis_freq = <333>; ++ lpddr2_drv = ; ++ phy_lpddr2_ca_drv = ; ++ phy_lpddr2_ck_drv = ; ++ phy_lpddr2_dq_drv = ; ++ phy_lpddr2_odt = ; ++ ++ lpddr3_odt_dis_freq = <333>; ++ phy_lpddr3_odt_dis_freq = <333>; ++ lpddr3_drv = ; ++ lpddr3_odt = ; ++ phy_lpddr3_ca_drv = ; ++ phy_lpddr3_ck_drv = ; ++ phy_lpddr3_dq_drv = ; ++ phy_lpddr3_odt = ; ++ ++ lpddr4_odt_dis_freq = <333>; ++ phy_lpddr4_odt_dis_freq = <333>; ++ lpddr4_drv = ; ++ lpddr4_dq_odt = ; ++ lpddr4_ca_odt = ; ++ phy_lpddr4_ca_drv = ; ++ phy_lpddr4_ck_cs_drv = ; ++ phy_lpddr4_dq_drv = ; ++ phy_lpddr4_odt = ; ++ ++ ddr4_odt_dis_freq = <625>; ++ phy_ddr4_odt_dis_freq = <625>; ++ ddr4_drv = ; ++ ddr4_odt = ; ++ phy_ddr4_ca_drv = ; ++ phy_ddr4_ck_drv = ; ++ phy_ddr4_dq_drv = ; ++ phy_ddr4_odt = ; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-evb.dtsi b/arch/arm64/boot/dts/rockchip/rk3568-evb.dtsi +new file mode 100755 +index 000000000000..2ca4a18314d2 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-evb.dtsi +@@ -0,0 +1,1807 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++/ { ++ ++ adc_keys: adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 0>; ++ io-channel-names = "buttons"; ++ keyup-threshold-microvolt = <1800000>; ++ poll-interval = <100>; ++ ++ vol-up-key { ++ label = "volume up"; ++ linux,code = ; ++ press-threshold-microvolt = <1750>; ++ }; ++ ++ vol-down-key { ++ label = "volume down"; ++ linux,code = ; ++ press-threshold-microvolt = <297500>; ++ }; ++ ++ menu-key { ++ label = "menu"; ++ linux,code = ; ++ press-threshold-microvolt = <980000>; ++ }; ++ ++ back-key { ++ label = "back"; ++ linux,code = ; ++ press-threshold-microvolt = <1305500>; ++ }; ++ }; ++ ++ audiopwmout_diff: audiopwmout-diff { ++ status = "disabled"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,audiopwmout-diff"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,bitclock-master = <&master>; ++ simple-audio-card,frame-master = <&master>; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s3_2ch>; ++ }; ++ master: simple-audio-card,codec { ++ sound-dai = <&dig_acodec>; ++ }; ++ }; ++ ++ backlight: backlight { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm4 0 25000 0>; ++ brightness-levels = < ++ 0 20 20 21 21 22 22 23 ++ 23 24 24 25 25 26 26 27 ++ 27 28 28 29 29 30 30 31 ++ 31 32 32 33 33 34 34 35 ++ 35 36 36 37 37 38 38 39 ++ 40 41 42 43 44 45 46 47 ++ 48 49 50 51 52 53 54 55 ++ 56 57 58 59 60 61 62 63 ++ 64 65 66 67 68 69 70 71 ++ 72 73 74 75 76 77 78 79 ++ 80 81 82 83 84 85 86 87 ++ 88 89 90 91 92 93 94 95 ++ 96 97 98 99 100 101 102 103 ++ 104 105 106 107 108 109 110 111 ++ 112 113 114 115 116 117 118 119 ++ 120 121 122 123 124 125 126 127 ++ 128 129 130 131 132 133 134 135 ++ 136 137 138 139 140 141 142 143 ++ 144 145 146 147 148 149 150 151 ++ 152 153 154 155 156 157 158 159 ++ 160 161 162 163 164 165 166 167 ++ 168 169 170 171 172 173 174 175 ++ 176 177 178 179 180 181 182 183 ++ 184 185 186 187 188 189 190 191 ++ 192 193 194 195 196 197 198 199 ++ 200 201 202 203 204 205 206 207 ++ 208 209 210 211 212 213 214 215 ++ 216 217 218 219 220 221 222 223 ++ 224 225 226 227 228 229 230 231 ++ 232 233 234 235 236 237 238 239 ++ 240 241 242 243 244 245 246 247 ++ 248 249 250 251 252 253 254 255 ++ >; ++ default-brightness-level = <200>; ++ }; ++ ++ backlight1: backlight1 { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm5 0 25000 0>; ++ brightness-levels = < ++ 0 20 20 21 21 22 22 23 ++ 23 24 24 25 25 26 26 27 ++ 27 28 28 29 29 30 30 31 ++ 31 32 32 33 33 34 34 35 ++ 35 36 36 37 37 38 38 39 ++ 40 41 42 43 44 45 46 47 ++ 48 49 50 51 52 53 54 55 ++ 56 57 58 59 60 61 62 63 ++ 64 65 66 67 68 69 70 71 ++ 72 73 74 75 76 77 78 79 ++ 80 81 82 83 84 85 86 87 ++ 88 89 90 91 92 93 94 95 ++ 96 97 98 99 100 101 102 103 ++ 104 105 106 107 108 109 110 111 ++ 112 113 114 115 116 117 118 119 ++ 120 121 122 123 124 125 126 127 ++ 128 129 130 131 132 133 134 135 ++ 136 137 138 139 140 141 142 143 ++ 144 145 146 147 148 149 150 151 ++ 152 153 154 155 156 157 158 159 ++ 160 161 162 163 164 165 166 167 ++ 168 169 170 171 172 173 174 175 ++ 176 177 178 179 180 181 182 183 ++ 184 185 186 187 188 189 190 191 ++ 192 193 194 195 196 197 198 199 ++ 200 201 202 203 204 205 206 207 ++ 208 209 210 211 212 213 214 215 ++ 216 217 218 219 220 221 222 223 ++ 224 225 226 227 228 229 230 231 ++ 232 233 234 235 236 237 238 239 ++ 240 241 242 243 244 245 246 247 ++ 248 249 250 251 252 253 254 255 ++ >; ++ default-brightness-level = <200>; ++ }; ++ ++ dc_12v: dc-12v { ++ compatible = "regulator-fixed"; ++ regulator-name = "dc_12v"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <12000000>; ++ regulator-max-microvolt = <12000000>; ++ }; ++ ++ hdmi_sound: hdmi-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,mclk-fs = <128>; ++ simple-audio-card,name = "rockchip,hdmi"; ++ status = "disabled"; ++ ++ simple-audio-card,cpu { ++ sound-dai = <&i2s0_8ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&hdmi>; ++ }; ++ }; ++ ++ leds: leds { ++ compatible = "gpio-leds"; ++ work_led: work { ++ gpios = <&gpio0 RK_PC0 GPIO_ACTIVE_HIGH>; ++ linux,default-trigger = "heartbeat"; ++ }; ++ }; ++ ++ pdmics: dummy-codec { ++ status = "disabled"; ++ compatible = "rockchip,dummy-codec"; ++ #sound-dai-cells = <0>; ++ }; ++ ++ pdm_mic_array: pdm-mic-array { ++ status = "disabled"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,name = "rockchip,pdm-mic-array"; ++ simple-audio-card,cpu { ++ sound-dai = <&pdm>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&pdmics>; ++ }; ++ }; ++ ++ rk809_sound: rk809-sound { ++ status = "okay"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,rk809-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ ++ simple-audio-card,cpu { ++ sound-dai = <&i2s1_8ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&rk809_codec>; ++ }; ++ }; ++ ++ spdif-sound { ++ status = "okay"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,name = "ROCKCHIP,SPDIF"; ++ simple-audio-card,cpu { ++ sound-dai = <&spdif_8ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&spdif_out>; ++ }; ++ }; ++ ++ spdif_out: spdif-out { ++ status = "okay"; ++ compatible = "linux,spdif-dit"; ++ #sound-dai-cells = <0>; ++ }; ++ ++ vad_sound: vad-sound { ++ status = "disabled"; ++ compatible = "rockchip,multicodecs-card"; ++ rockchip,card-name = "rockchip,rk3568-vad"; ++ rockchip,cpu = <&i2s1_8ch>; ++ rockchip,codec = <&rk809_codec>, <&vad>; ++ }; ++ ++ vcc3v3_sys: vcc3v3-sys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc3v3_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ vin-supply = <&dc_12v>; ++ }; ++ ++ vcc5v0_sys: vcc5v0-sys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc5v0_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ vin-supply = <&dc_12v>; ++ }; ++ ++ vcc5v0_host: vcc5v0-host-regulator { ++ compatible = "regulator-fixed"; ++ enable-active-high; ++ gpio = <&gpio0 RK_PA6 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&vcc5v0_host_en>; ++ regulator-name = "vcc5v0_host"; ++ regulator-always-on; ++ }; ++ ++ vcc5v0_otg: vcc5v0-otg-regulator { ++ compatible = "regulator-fixed"; ++ enable-active-high; ++ gpio = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&vcc5v0_otg_en>; ++ regulator-name = "vcc5v0_otg"; ++ }; ++ ++ vcc3v3_lcd0_n: vcc3v3-lcd0-n { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc3v3_lcd0_n"; ++ regulator-boot-on; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc3v3_lcd1_n: vcc3v3-lcd1-n { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc3v3_lcd1_n"; ++ regulator-boot-on; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ clocks = <&rk809 1>; ++ clock-names = "ext_clock"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ post-power-on-delay-ms = <200>; ++ reset-gpios = <&gpio3 RK_PD5 GPIO_ACTIVE_LOW>; ++ }; ++ ++ wireless_wlan: wireless-wlan { ++ compatible = "wlan-platdata"; ++ rockchip,grf = <&grf>; ++ wifi_chip_type = "ap6398s"; ++ status = "okay"; ++ }; ++ ++ wireless_bluetooth: wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&rk809 1>; ++ clock-names = "ext_clock"; ++ //wifi-bt-power-toggle; ++ uart_rts_gpios = <&gpio2 RK_PB1 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart8m0_rtsn>; ++ pinctrl-1 = <&uart8_gpios>; ++ BT,reset_gpio = <&gpio3 RK_PA0 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio3 RK_PA1 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio3 RK_PA2 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ test-power { ++ status = "okay"; ++ }; ++}; ++ ++&bus_npu { ++ bus-supply = <&vdd_logic>; ++ pvtm-supply = <&vdd_cpu>; ++ status = "okay"; ++}; ++ ++&can0 { ++ assigned-clocks = <&cru CLK_CAN0>; ++ assigned-clock-rates = <150000000>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&can0m1_pins>; ++ status = "disabled"; ++}; ++ ++&can1 { ++ assigned-clocks = <&cru CLK_CAN1>; ++ assigned-clock-rates = <150000000>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&can1m1_pins>; ++ status = "disabled"; ++}; ++ ++&can2 { ++ assigned-clocks = <&cru CLK_CAN2>; ++ assigned-clock-rates = <150000000>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&can2m1_pins>; ++ status = "disabled"; ++}; ++ ++&cpu0 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&dmc { ++ center-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&dsi0 { ++ status = "disabled"; ++ //rockchip,lane-rate = <1000>; ++ dsi0_panel: panel@0 { ++ status = "okay"; ++ compatible = "simple-panel-dsi"; ++ reg = <0>; ++ backlight = <&backlight>; ++ reset-delay-ms = <60>; ++ enable-delay-ms = <60>; ++ prepare-delay-ms = <60>; ++ unprepare-delay-ms = <60>; ++ disable-delay-ms = <60>; ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | ++ MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ panel-init-sequence = [ ++ 23 00 02 FE 21 ++ 23 00 02 04 00 ++ 23 00 02 00 64 ++ 23 00 02 2A 00 ++ 23 00 02 26 64 ++ 23 00 02 54 00 ++ 23 00 02 50 64 ++ 23 00 02 7B 00 ++ 23 00 02 77 64 ++ 23 00 02 A2 00 ++ 23 00 02 9D 64 ++ 23 00 02 C9 00 ++ 23 00 02 C5 64 ++ 23 00 02 01 71 ++ 23 00 02 27 71 ++ 23 00 02 51 71 ++ 23 00 02 78 71 ++ 23 00 02 9E 71 ++ 23 00 02 C6 71 ++ 23 00 02 02 89 ++ 23 00 02 28 89 ++ 23 00 02 52 89 ++ 23 00 02 79 89 ++ 23 00 02 9F 89 ++ 23 00 02 C7 89 ++ 23 00 02 03 9E ++ 23 00 02 29 9E ++ 23 00 02 53 9E ++ 23 00 02 7A 9E ++ 23 00 02 A0 9E ++ 23 00 02 C8 9E ++ 23 00 02 09 00 ++ 23 00 02 05 B0 ++ 23 00 02 31 00 ++ 23 00 02 2B B0 ++ 23 00 02 5A 00 ++ 23 00 02 55 B0 ++ 23 00 02 80 00 ++ 23 00 02 7C B0 ++ 23 00 02 A7 00 ++ 23 00 02 A3 B0 ++ 23 00 02 CE 00 ++ 23 00 02 CA B0 ++ 23 00 02 06 C0 ++ 23 00 02 2D C0 ++ 23 00 02 56 C0 ++ 23 00 02 7D C0 ++ 23 00 02 A4 C0 ++ 23 00 02 CB C0 ++ 23 00 02 07 CF ++ 23 00 02 2F CF ++ 23 00 02 58 CF ++ 23 00 02 7E CF ++ 23 00 02 A5 CF ++ 23 00 02 CC CF ++ 23 00 02 08 DD ++ 23 00 02 30 DD ++ 23 00 02 59 DD ++ 23 00 02 7F DD ++ 23 00 02 A6 DD ++ 23 00 02 CD DD ++ 23 00 02 0E 15 ++ 23 00 02 0A E9 ++ 23 00 02 36 15 ++ 23 00 02 32 E9 ++ 23 00 02 5F 15 ++ 23 00 02 5B E9 ++ 23 00 02 85 15 ++ 23 00 02 81 E9 ++ 23 00 02 AD 15 ++ 23 00 02 A9 E9 ++ 23 00 02 D3 15 ++ 23 00 02 CF E9 ++ 23 00 02 0B 14 ++ 23 00 02 33 14 ++ 23 00 02 5C 14 ++ 23 00 02 82 14 ++ 23 00 02 AA 14 ++ 23 00 02 D0 14 ++ 23 00 02 0C 36 ++ 23 00 02 34 36 ++ 23 00 02 5D 36 ++ 23 00 02 83 36 ++ 23 00 02 AB 36 ++ 23 00 02 D1 36 ++ 23 00 02 0D 6B ++ 23 00 02 35 6B ++ 23 00 02 5E 6B ++ 23 00 02 84 6B ++ 23 00 02 AC 6B ++ 23 00 02 D2 6B ++ 23 00 02 13 5A ++ 23 00 02 0F 94 ++ 23 00 02 3B 5A ++ 23 00 02 37 94 ++ 23 00 02 64 5A ++ 23 00 02 60 94 ++ 23 00 02 8A 5A ++ 23 00 02 86 94 ++ 23 00 02 B2 5A ++ 23 00 02 AE 94 ++ 23 00 02 D8 5A ++ 23 00 02 D4 94 ++ 23 00 02 10 D1 ++ 23 00 02 38 D1 ++ 23 00 02 61 D1 ++ 23 00 02 87 D1 ++ 23 00 02 AF D1 ++ 23 00 02 D5 D1 ++ 23 00 02 11 04 ++ 23 00 02 39 04 ++ 23 00 02 62 04 ++ 23 00 02 88 04 ++ 23 00 02 B0 04 ++ 23 00 02 D6 04 ++ 23 00 02 12 05 ++ 23 00 02 3A 05 ++ 23 00 02 63 05 ++ 23 00 02 89 05 ++ 23 00 02 B1 05 ++ 23 00 02 D7 05 ++ 23 00 02 18 AA ++ 23 00 02 14 36 ++ 23 00 02 42 AA ++ 23 00 02 3D 36 ++ 23 00 02 69 AA ++ 23 00 02 65 36 ++ 23 00 02 8F AA ++ 23 00 02 8B 36 ++ 23 00 02 B7 AA ++ 23 00 02 B3 36 ++ 23 00 02 DD AA ++ 23 00 02 D9 36 ++ 23 00 02 15 74 ++ 23 00 02 3F 74 ++ 23 00 02 66 74 ++ 23 00 02 8C 74 ++ 23 00 02 B4 74 ++ 23 00 02 DA 74 ++ 23 00 02 16 9F ++ 23 00 02 40 9F ++ 23 00 02 67 9F ++ 23 00 02 8D 9F ++ 23 00 02 B5 9F ++ 23 00 02 DB 9F ++ 23 00 02 17 DC ++ 23 00 02 41 DC ++ 23 00 02 68 DC ++ 23 00 02 8E DC ++ 23 00 02 B6 DC ++ 23 00 02 DC DC ++ 23 00 02 1D FF ++ 23 00 02 19 03 ++ 23 00 02 47 FF ++ 23 00 02 43 03 ++ 23 00 02 6E FF ++ 23 00 02 6A 03 ++ 23 00 02 94 FF ++ 23 00 02 90 03 ++ 23 00 02 BC FF ++ 23 00 02 B8 03 ++ 23 00 02 E2 FF ++ 23 00 02 DE 03 ++ 23 00 02 1A 35 ++ 23 00 02 44 35 ++ 23 00 02 6B 35 ++ 23 00 02 91 35 ++ 23 00 02 B9 35 ++ 23 00 02 DF 35 ++ 23 00 02 1B 45 ++ 23 00 02 45 45 ++ 23 00 02 6C 45 ++ 23 00 02 92 45 ++ 23 00 02 BA 45 ++ 23 00 02 E0 45 ++ 23 00 02 1C 55 ++ 23 00 02 46 55 ++ 23 00 02 6D 55 ++ 23 00 02 93 55 ++ 23 00 02 BB 55 ++ 23 00 02 E1 55 ++ 23 00 02 22 FF ++ 23 00 02 1E 68 ++ 23 00 02 4C FF ++ 23 00 02 48 68 ++ 23 00 02 73 FF ++ 23 00 02 6F 68 ++ 23 00 02 99 FF ++ 23 00 02 95 68 ++ 23 00 02 C1 FF ++ 23 00 02 BD 68 ++ 23 00 02 E7 FF ++ 23 00 02 E3 68 ++ 23 00 02 1F 7E ++ 23 00 02 49 7E ++ 23 00 02 70 7E ++ 23 00 02 96 7E ++ 23 00 02 BE 7E ++ 23 00 02 E4 7E ++ 23 00 02 20 97 ++ 23 00 02 4A 97 ++ 23 00 02 71 97 ++ 23 00 02 97 97 ++ 23 00 02 BF 97 ++ 23 00 02 E5 97 ++ 23 00 02 21 B5 ++ 23 00 02 4B B5 ++ 23 00 02 72 B5 ++ 23 00 02 98 B5 ++ 23 00 02 C0 B5 ++ 23 00 02 E6 B5 ++ 23 00 02 25 F0 ++ 23 00 02 23 E8 ++ 23 00 02 4F F0 ++ 23 00 02 4D E8 ++ 23 00 02 76 F0 ++ 23 00 02 74 E8 ++ 23 00 02 9C F0 ++ 23 00 02 9A E8 ++ 23 00 02 C4 F0 ++ 23 00 02 C2 E8 ++ 23 00 02 EA F0 ++ 23 00 02 E8 E8 ++ 23 00 02 24 FF ++ 23 00 02 4E FF ++ 23 00 02 75 FF ++ 23 00 02 9B FF ++ 23 00 02 C3 FF ++ 23 00 02 E9 FF ++ 23 00 02 FE 3D ++ 23 00 02 00 04 ++ 23 00 02 FE 23 ++ 23 00 02 08 82 ++ 23 00 02 0A 00 ++ 23 00 02 0B 00 ++ 23 00 02 0C 01 ++ 23 00 02 16 00 ++ 23 00 02 18 02 ++ 23 00 02 1B 04 ++ 23 00 02 19 04 ++ 23 00 02 1C 81 ++ 23 00 02 1F 00 ++ 23 00 02 20 03 ++ 23 00 02 23 04 ++ 23 00 02 21 01 ++ 23 00 02 54 63 ++ 23 00 02 55 54 ++ 23 00 02 6E 45 ++ 23 00 02 6D 36 ++ 23 00 02 FE 3D ++ 23 00 02 55 78 ++ 23 00 02 FE 20 ++ 23 00 02 26 30 ++ 23 00 02 FE 3D ++ 23 00 02 20 71 ++ 23 00 02 50 8F ++ 23 00 02 51 8F ++ 23 00 02 FE 00 ++ 23 00 02 35 00 ++ 05 78 01 11 ++ 05 1E 01 29 ++ ]; ++ ++ panel-exit-sequence = [ ++ 05 00 01 28 ++ 05 00 01 10 ++ ]; ++ ++ disp_timings0: display-timings { ++ native-mode = <&dsi0_timing0>; ++ dsi0_timing0: timing0 { ++ clock-frequency = <132000000>; ++ hactive = <1080>; ++ vactive = <1920>; ++ hfront-porch = <15>; ++ hsync-len = <2>; ++ hback-porch = <30>; ++ vfront-porch = <15>; ++ vsync-len = <2>; ++ vback-porch = <15>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <1>; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ panel_in_dsi: endpoint { ++ remote-endpoint = <&dsi_out_panel>; ++ }; ++ }; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@1 { ++ reg = <1>; ++ dsi_out_panel: endpoint { ++ remote-endpoint = <&panel_in_dsi>; ++ }; ++ }; ++ }; ++ ++}; ++ ++&dsi1 { ++ status = "disabled"; ++ //rockchip,lane-rate = <1000>; ++ dsi1_panel: panel@0 { ++ status = "okay"; ++ compatible = "simple-panel-dsi"; ++ reg = <0>; ++ backlight = <&backlight1>; ++ reset-delay-ms = <60>; ++ enable-delay-ms = <60>; ++ prepare-delay-ms = <60>; ++ unprepare-delay-ms = <60>; ++ disable-delay-ms = <60>; ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | ++ MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ panel-init-sequence = [ ++ 23 00 02 FE 21 ++ 23 00 02 04 00 ++ 23 00 02 00 64 ++ 23 00 02 2A 00 ++ 23 00 02 26 64 ++ 23 00 02 54 00 ++ 23 00 02 50 64 ++ 23 00 02 7B 00 ++ 23 00 02 77 64 ++ 23 00 02 A2 00 ++ 23 00 02 9D 64 ++ 23 00 02 C9 00 ++ 23 00 02 C5 64 ++ 23 00 02 01 71 ++ 23 00 02 27 71 ++ 23 00 02 51 71 ++ 23 00 02 78 71 ++ 23 00 02 9E 71 ++ 23 00 02 C6 71 ++ 23 00 02 02 89 ++ 23 00 02 28 89 ++ 23 00 02 52 89 ++ 23 00 02 79 89 ++ 23 00 02 9F 89 ++ 23 00 02 C7 89 ++ 23 00 02 03 9E ++ 23 00 02 29 9E ++ 23 00 02 53 9E ++ 23 00 02 7A 9E ++ 23 00 02 A0 9E ++ 23 00 02 C8 9E ++ 23 00 02 09 00 ++ 23 00 02 05 B0 ++ 23 00 02 31 00 ++ 23 00 02 2B B0 ++ 23 00 02 5A 00 ++ 23 00 02 55 B0 ++ 23 00 02 80 00 ++ 23 00 02 7C B0 ++ 23 00 02 A7 00 ++ 23 00 02 A3 B0 ++ 23 00 02 CE 00 ++ 23 00 02 CA B0 ++ 23 00 02 06 C0 ++ 23 00 02 2D C0 ++ 23 00 02 56 C0 ++ 23 00 02 7D C0 ++ 23 00 02 A4 C0 ++ 23 00 02 CB C0 ++ 23 00 02 07 CF ++ 23 00 02 2F CF ++ 23 00 02 58 CF ++ 23 00 02 7E CF ++ 23 00 02 A5 CF ++ 23 00 02 CC CF ++ 23 00 02 08 DD ++ 23 00 02 30 DD ++ 23 00 02 59 DD ++ 23 00 02 7F DD ++ 23 00 02 A6 DD ++ 23 00 02 CD DD ++ 23 00 02 0E 15 ++ 23 00 02 0A E9 ++ 23 00 02 36 15 ++ 23 00 02 32 E9 ++ 23 00 02 5F 15 ++ 23 00 02 5B E9 ++ 23 00 02 85 15 ++ 23 00 02 81 E9 ++ 23 00 02 AD 15 ++ 23 00 02 A9 E9 ++ 23 00 02 D3 15 ++ 23 00 02 CF E9 ++ 23 00 02 0B 14 ++ 23 00 02 33 14 ++ 23 00 02 5C 14 ++ 23 00 02 82 14 ++ 23 00 02 AA 14 ++ 23 00 02 D0 14 ++ 23 00 02 0C 36 ++ 23 00 02 34 36 ++ 23 00 02 5D 36 ++ 23 00 02 83 36 ++ 23 00 02 AB 36 ++ 23 00 02 D1 36 ++ 23 00 02 0D 6B ++ 23 00 02 35 6B ++ 23 00 02 5E 6B ++ 23 00 02 84 6B ++ 23 00 02 AC 6B ++ 23 00 02 D2 6B ++ 23 00 02 13 5A ++ 23 00 02 0F 94 ++ 23 00 02 3B 5A ++ 23 00 02 37 94 ++ 23 00 02 64 5A ++ 23 00 02 60 94 ++ 23 00 02 8A 5A ++ 23 00 02 86 94 ++ 23 00 02 B2 5A ++ 23 00 02 AE 94 ++ 23 00 02 D8 5A ++ 23 00 02 D4 94 ++ 23 00 02 10 D1 ++ 23 00 02 38 D1 ++ 23 00 02 61 D1 ++ 23 00 02 87 D1 ++ 23 00 02 AF D1 ++ 23 00 02 D5 D1 ++ 23 00 02 11 04 ++ 23 00 02 39 04 ++ 23 00 02 62 04 ++ 23 00 02 88 04 ++ 23 00 02 B0 04 ++ 23 00 02 D6 04 ++ 23 00 02 12 05 ++ 23 00 02 3A 05 ++ 23 00 02 63 05 ++ 23 00 02 89 05 ++ 23 00 02 B1 05 ++ 23 00 02 D7 05 ++ 23 00 02 18 AA ++ 23 00 02 14 36 ++ 23 00 02 42 AA ++ 23 00 02 3D 36 ++ 23 00 02 69 AA ++ 23 00 02 65 36 ++ 23 00 02 8F AA ++ 23 00 02 8B 36 ++ 23 00 02 B7 AA ++ 23 00 02 B3 36 ++ 23 00 02 DD AA ++ 23 00 02 D9 36 ++ 23 00 02 15 74 ++ 23 00 02 3F 74 ++ 23 00 02 66 74 ++ 23 00 02 8C 74 ++ 23 00 02 B4 74 ++ 23 00 02 DA 74 ++ 23 00 02 16 9F ++ 23 00 02 40 9F ++ 23 00 02 67 9F ++ 23 00 02 8D 9F ++ 23 00 02 B5 9F ++ 23 00 02 DB 9F ++ 23 00 02 17 DC ++ 23 00 02 41 DC ++ 23 00 02 68 DC ++ 23 00 02 8E DC ++ 23 00 02 B6 DC ++ 23 00 02 DC DC ++ 23 00 02 1D FF ++ 23 00 02 19 03 ++ 23 00 02 47 FF ++ 23 00 02 43 03 ++ 23 00 02 6E FF ++ 23 00 02 6A 03 ++ 23 00 02 94 FF ++ 23 00 02 90 03 ++ 23 00 02 BC FF ++ 23 00 02 B8 03 ++ 23 00 02 E2 FF ++ 23 00 02 DE 03 ++ 23 00 02 1A 35 ++ 23 00 02 44 35 ++ 23 00 02 6B 35 ++ 23 00 02 91 35 ++ 23 00 02 B9 35 ++ 23 00 02 DF 35 ++ 23 00 02 1B 45 ++ 23 00 02 45 45 ++ 23 00 02 6C 45 ++ 23 00 02 92 45 ++ 23 00 02 BA 45 ++ 23 00 02 E0 45 ++ 23 00 02 1C 55 ++ 23 00 02 46 55 ++ 23 00 02 6D 55 ++ 23 00 02 93 55 ++ 23 00 02 BB 55 ++ 23 00 02 E1 55 ++ 23 00 02 22 FF ++ 23 00 02 1E 68 ++ 23 00 02 4C FF ++ 23 00 02 48 68 ++ 23 00 02 73 FF ++ 23 00 02 6F 68 ++ 23 00 02 99 FF ++ 23 00 02 95 68 ++ 23 00 02 C1 FF ++ 23 00 02 BD 68 ++ 23 00 02 E7 FF ++ 23 00 02 E3 68 ++ 23 00 02 1F 7E ++ 23 00 02 49 7E ++ 23 00 02 70 7E ++ 23 00 02 96 7E ++ 23 00 02 BE 7E ++ 23 00 02 E4 7E ++ 23 00 02 20 97 ++ 23 00 02 4A 97 ++ 23 00 02 71 97 ++ 23 00 02 97 97 ++ 23 00 02 BF 97 ++ 23 00 02 E5 97 ++ 23 00 02 21 B5 ++ 23 00 02 4B B5 ++ 23 00 02 72 B5 ++ 23 00 02 98 B5 ++ 23 00 02 C0 B5 ++ 23 00 02 E6 B5 ++ 23 00 02 25 F0 ++ 23 00 02 23 E8 ++ 23 00 02 4F F0 ++ 23 00 02 4D E8 ++ 23 00 02 76 F0 ++ 23 00 02 74 E8 ++ 23 00 02 9C F0 ++ 23 00 02 9A E8 ++ 23 00 02 C4 F0 ++ 23 00 02 C2 E8 ++ 23 00 02 EA F0 ++ 23 00 02 E8 E8 ++ 23 00 02 24 FF ++ 23 00 02 4E FF ++ 23 00 02 75 FF ++ 23 00 02 9B FF ++ 23 00 02 C3 FF ++ 23 00 02 E9 FF ++ 23 00 02 FE 3D ++ 23 00 02 00 04 ++ 23 00 02 FE 23 ++ 23 00 02 08 82 ++ 23 00 02 0A 00 ++ 23 00 02 0B 00 ++ 23 00 02 0C 01 ++ 23 00 02 16 00 ++ 23 00 02 18 02 ++ 23 00 02 1B 04 ++ 23 00 02 19 04 ++ 23 00 02 1C 81 ++ 23 00 02 1F 00 ++ 23 00 02 20 03 ++ 23 00 02 23 04 ++ 23 00 02 21 01 ++ 23 00 02 54 63 ++ 23 00 02 55 54 ++ 23 00 02 6E 45 ++ 23 00 02 6D 36 ++ 23 00 02 FE 3D ++ 23 00 02 55 78 ++ 23 00 02 FE 20 ++ 23 00 02 26 30 ++ 23 00 02 FE 3D ++ 23 00 02 20 71 ++ 23 00 02 50 8F ++ 23 00 02 51 8F ++ 23 00 02 FE 00 ++ 23 00 02 35 00 ++ 05 78 01 11 ++ 05 1E 01 29 ++ ]; ++ ++ panel-exit-sequence = [ ++ 05 00 01 28 ++ 05 00 01 10 ++ ]; ++ ++ disp_timings1: display-timings { ++ native-mode = <&dsi1_timing0>; ++ dsi1_timing0: timing0 { ++ clock-frequency = <132000000>; ++ hactive = <1080>; ++ vactive = <1920>; ++ hfront-porch = <15>; ++ hsync-len = <2>; ++ hback-porch = <30>; ++ vfront-porch = <15>; ++ vsync-len = <2>; ++ vback-porch = <15>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <1>; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ panel_in_dsi1: endpoint { ++ remote-endpoint = <&dsi1_out_panel>; ++ }; ++ }; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@1 { ++ reg = <1>; ++ dsi1_out_panel: endpoint { ++ remote-endpoint = <&panel_in_dsi1>; ++ }; ++ }; ++ }; ++ ++}; ++ ++&gpu { ++ mali-supply = <&vdd_gpu>; ++ status = "okay"; ++}; ++ ++&hdmi { ++ status = "okay"; ++ rockchip,phy-table = ++ <92812500 0x8009 0x0000 0x0270>, ++ <165000000 0x800b 0x0000 0x026d>, ++ <185625000 0x800b 0x0000 0x01ed>, ++ <297000000 0x800b 0x0000 0x01ad>, ++ <594000000 0x8029 0x0000 0x0088>, ++ <000000000 0x0000 0x0000 0x0000>; ++}; ++ ++&hdmi_in_vp0 { ++ status = "okay"; ++}; ++ ++&hdmi_in_vp1 { ++ status = "disabled"; ++}; ++ ++&hdmi_sound { ++ status = "okay"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ ++ vdd_cpu: tcs4525@1c { ++ compatible = "tcs,tcs452x"; ++ reg = <0x1c>; ++ vin-supply = <&vcc5v0_sys>; ++ regulator-compatible = "fan53555-reg"; ++ regulator-name = "vdd_cpu"; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1390000>; ++ regulator-init-microvolt = <900000>; ++ regulator-ramp-delay = <2300>; ++ fcs,suspend-voltage-selector = <1>; ++ regulator-boot-on; ++ regulator-always-on; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ rk809: pmic@20 { ++ compatible = "rockchip,rk809"; ++ reg = <0x20>; ++ interrupt-parent = <&gpio0>; ++ interrupts = <3 IRQ_TYPE_LEVEL_LOW>; ++ ++ pinctrl-names = "default", "pmic-sleep", ++ "pmic-power-off", "pmic-reset"; ++ pinctrl-0 = <&pmic_int>; ++ pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; ++ pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; ++ pinctrl-3 = <&soc_slppin_gpio>, <&rk817_slppin_rst>; ++ ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ //fb-inner-reg-idxs = <2>; ++ /* 1: rst regs (default in codes), 0: rst the pmic */ ++ pmic-reset-func = <0>; ++ /* not save the PMIC_POWER_EN register in uboot */ ++ not-save-power-en = <1>; ++ ++ vcc1-supply = <&vcc3v3_sys>; ++ vcc2-supply = <&vcc3v3_sys>; ++ vcc3-supply = <&vcc3v3_sys>; ++ vcc4-supply = <&vcc3v3_sys>; ++ vcc5-supply = <&vcc3v3_sys>; ++ vcc6-supply = <&vcc3v3_sys>; ++ vcc7-supply = <&vcc3v3_sys>; ++ vcc8-supply = <&vcc3v3_sys>; ++ vcc9-supply = <&vcc3v3_sys>; ++ ++ pwrkey { ++ status = "okay"; ++ }; ++ ++ pinctrl_rk8xx: pinctrl_rk8xx { ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ rk817_slppin_null: rk817_slppin_null { ++ pins = "gpio_slp"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_slppin_slp: rk817_slppin_slp { ++ pins = "gpio_slp"; ++ function = "pin_fun1"; ++ }; ++ ++ rk817_slppin_pwrdn: rk817_slppin_pwrdn { ++ pins = "gpio_slp"; ++ function = "pin_fun2"; ++ }; ++ ++ rk817_slppin_rst: rk817_slppin_rst { ++ pins = "gpio_slp"; ++ function = "pin_fun3"; ++ }; ++ }; ++ ++ regulators { ++ vdd_logic: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <500000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-init-microvolt = <900000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_logic"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdd_gpu: DCDC_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <500000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-init-microvolt = <900000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_gpu"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_ddr"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vdd_npu: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <500000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-init-microvolt = <900000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_npu"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdda0v9_image: LDO_REG1 { ++ regulator-boot-on; ++ regulator-always-on; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <900000>; ++ regulator-name = "vdda0v9_image"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdda_0v9: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <900000>; ++ regulator-name = "vdda_0v9"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdda0v9_pmu: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <900000>; ++ regulator-name = "vdda0v9_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <900000>; ++ }; ++ }; ++ ++ vccio_acodec: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vccio_acodec"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vccio_sd: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vccio_sd"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc3v3_pmu: LDO_REG6 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcc3v3_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcca_1v8: LDO_REG7 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcca_1v8"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcca1v8_pmu: LDO_REG8 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcca1v8_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vcca1v8_image: LDO_REG9 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcca1v8_image"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_1v8: DCDC_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc_1v8"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_3v3: SWITCH_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc_3v3"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc3v3_sd: SWITCH_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc3v3_sd"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ }; ++ ++ rk809_codec: codec { ++ #sound-dai-cells = <0>; ++ compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; ++ clocks = <&cru I2S1_MCLKOUT>; ++ clock-names = "mclk"; ++ assigned-clocks = <&cru I2S1_MCLKOUT>, <&cru I2S1_MCLK_TX_IOE>; ++ assigned-clock-rates = <12288000>; ++ assigned-clock-parents = <&cru I2S1_MCLKOUT_TX>, <&cru I2S1_MCLKOUT_TX>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1m0_mclk>; ++ hp-volume = <20>; ++ spk-volume = <3>; ++ mic-in-differential; ++ status = "okay"; ++ }; ++ }; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ ++ gt1x: gt1x@14 { ++ compatible = "goodix,gt1x"; ++ reg = <0x14>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&touch_gpio>; ++ goodix,rst-gpio = <&gpio0 RK_PB6 GPIO_ACTIVE_HIGH>; ++ goodix,irq-gpio = <&gpio0 RK_PB5 IRQ_TYPE_LEVEL_LOW>; ++ }; ++}; ++ ++&i2c5 { ++ status = "okay"; ++ ++ mxc6655xa: mxc6655xa@15 { ++ status = "okay"; ++ compatible = "gs_mxc6655xa"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&mxc6655xa_irq_gpio>; ++ reg = <0x15>; ++ irq-gpio = <&gpio3 RK_PC1 IRQ_TYPE_LEVEL_LOW>; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ type = ; ++ power-off-in-suspend = <1>; ++ layout = <1>; ++ }; ++}; ++ ++&i2s0_8ch { ++ status = "okay"; ++}; ++ ++&i2s1_8ch { ++ status = "okay"; ++ rockchip,clk-trcm = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1m0_sclktx ++ &i2s1m0_lrcktx ++ &i2s1m0_sdi0 ++ &i2s1m0_sdo0>; ++}; ++ ++&iep { ++ status = "okay"; ++}; ++ ++&iep_mmu { ++ status = "okay"; ++}; ++ ++&jpegd { ++ status = "okay"; ++}; ++ ++&jpegd_mmu { ++ status = "okay"; ++}; ++ ++&mpp_srv { ++ status = "okay"; ++}; ++ ++&nandc0 { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ status = "okay"; ++ ++ nand@0 { ++ reg = <0>; ++ nand-bus-width = <8>; ++ nand-ecc-mode = "hw"; ++ nand-ecc-strength = <16>; ++ nand-ecc-step-size = <1024>; ++ }; ++}; ++ ++&pinctrl { ++ ++ mxc6655xa { ++ mxc6655xa_irq_gpio: mxc6655xa_irq_gpio { ++ rockchip,pins = <3 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ pmic { ++ pmic_int: pmic_int { ++ rockchip,pins = ++ <0 RK_PA3 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ soc_slppin_gpio: soc_slppin_gpio { ++ rockchip,pins = ++ <0 RK_PA2 RK_FUNC_GPIO &pcfg_output_low>; ++ }; ++ ++ soc_slppin_slp: soc_slppin_slp { ++ rockchip,pins = ++ <0 RK_PA2 1 &pcfg_pull_none>; ++ }; ++ ++ soc_slppin_rst: soc_slppin_rst { ++ rockchip,pins = ++ <0 RK_PA2 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ touch { ++ touch_gpio: touch-gpio { ++ rockchip,pins = ++ <0 RK_PB5 RK_FUNC_GPIO &pcfg_pull_up>, ++ <0 RK_PB6 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <3 RK_PD5 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ usb { ++ vcc5v0_host_en: vcc5v0-host-en { ++ rockchip,pins = <0 RK_PA6 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ vcc5v0_otg_en: vcc5v0-otg-en { ++ rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart8_gpios: uart8-gpios { ++ rockchip,pins = <2 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++ /* ++ * There are 10 independent IO domains in RK3566/RK3568, including PMUIO[0:2] and VCCIO[1:7]. ++ * 1/ PMUIO0 and PMUIO1 are fixed-level power domains which cannot be configured; ++ * 2/ PMUIO2 and VCCIO1,VCCIO[3:7] domains require that their hardware power supply voltages ++ * must be consistent with the software configuration correspondingly ++ * a/ When the hardware IO level is connected to 1.8V, the software voltage configuration ++ * should also be configured to 1.8V accordingly; ++ * b/ When the hardware IO level is connected to 3.3V, the software voltage configuration ++ * should also be configured to 3.3V accordingly; ++ * 3/ VCCIO2 voltage control selection (0xFDC20140) ++ * BIT[0]: 0x0: from GPIO_0A7 (default) ++ * BIT[0]: 0x1: from GRF ++ * Default is determined by Pin FLASH_VOL_SEL/GPIO0_A7: ++ * L:VCCIO2 must supply 3.3V ++ * H:VCCIO2 must supply 1.8V ++ */ ++&pmu_io_domains { ++ status = "okay"; ++ pmuio2-supply = <&vcc3v3_pmu>; ++ vccio1-supply = <&vccio_acodec>; ++ vccio3-supply = <&vccio_sd>; ++ vccio4-supply = <&vcc_3v3>; ++ vccio5-supply = <&vcc_3v3>; ++ vccio6-supply = <&vcc_3v3>; ++ vccio7-supply = <&vcc_3v3>; ++}; ++ ++&pwm4 { ++ status = "okay"; ++}; ++ ++&pwm5 { ++ status = "okay"; ++}; ++ ++&pwm7 { ++ status = "okay"; ++ ++ compatible = "rockchip,remotectl-pwm"; ++ remote_pwm_id = <3>; ++ handle_cpu_id = <1>; ++ remote_support_psci = <0>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pwm7_pins>; ++ ++ ir_key1 { ++ rockchip,usercode = <0x4040>; ++ rockchip,key_table = ++ <0xf2 KEY_REPLY>, ++ <0xba KEY_BACK>, ++ <0xf4 KEY_UP>, ++ <0xf1 KEY_DOWN>, ++ <0xef KEY_LEFT>, ++ <0xee KEY_RIGHT>, ++ <0xbd KEY_HOME>, ++ <0xea KEY_VOLUMEUP>, ++ <0xe3 KEY_VOLUMEDOWN>, ++ <0xe2 KEY_SEARCH>, ++ <0xb2 KEY_POWER>, ++ <0xbc KEY_MUTE>, ++ <0xec KEY_MENU>, ++ <0xbf 0x190>, ++ <0xe0 0x191>, ++ <0xe1 0x192>, ++ <0xe9 183>, ++ <0xe6 248>, ++ <0xe8 185>, ++ <0xe7 186>, ++ <0xf0 388>, ++ <0xbe 0x175>; ++ }; ++ ++ ir_key2 { ++ rockchip,usercode = <0xff00>; ++ rockchip,key_table = ++ <0xf9 KEY_HOME>, ++ <0xbf KEY_BACK>, ++ <0xfb KEY_MENU>, ++ <0xaa KEY_REPLY>, ++ <0xb9 KEY_UP>, ++ <0xe9 KEY_DOWN>, ++ <0xb8 KEY_LEFT>, ++ <0xea KEY_RIGHT>, ++ <0xeb KEY_VOLUMEDOWN>, ++ <0xef KEY_VOLUMEUP>, ++ <0xf7 KEY_MUTE>, ++ <0xe7 KEY_POWER>, ++ <0xfc KEY_POWER>, ++ <0xa9 KEY_VOLUMEDOWN>, ++ <0xa8 KEY_VOLUMEDOWN>, ++ <0xe0 KEY_VOLUMEDOWN>, ++ <0xa5 KEY_VOLUMEDOWN>, ++ <0xab 183>, ++ <0xb7 388>, ++ <0xe8 388>, ++ <0xf8 184>, ++ <0xaf 185>, ++ <0xed KEY_VOLUMEDOWN>, ++ <0xee 186>, ++ <0xb3 KEY_VOLUMEDOWN>, ++ <0xf1 KEY_VOLUMEDOWN>, ++ <0xf2 KEY_VOLUMEDOWN>, ++ <0xf3 KEY_SEARCH>, ++ <0xb4 KEY_VOLUMEDOWN>, ++ <0xbe KEY_SEARCH>; ++ }; ++ ++ ir_key3 { ++ rockchip,usercode = <0x1dcc>; ++ rockchip,key_table = ++ <0xee KEY_REPLY>, ++ <0xf0 KEY_BACK>, ++ <0xf8 KEY_UP>, ++ <0xbb KEY_DOWN>, ++ <0xef KEY_LEFT>, ++ <0xed KEY_RIGHT>, ++ <0xfc KEY_HOME>, ++ <0xf1 KEY_VOLUMEUP>, ++ <0xfd KEY_VOLUMEDOWN>, ++ <0xb7 KEY_SEARCH>, ++ <0xff KEY_POWER>, ++ <0xf3 KEY_MUTE>, ++ <0xbf KEY_MENU>, ++ <0xf9 0x191>, ++ <0xf5 0x192>, ++ <0xb3 388>, ++ <0xbe KEY_1>, ++ <0xba KEY_2>, ++ <0xb2 KEY_3>, ++ <0xbd KEY_4>, ++ <0xf9 KEY_5>, ++ <0xb1 KEY_6>, ++ <0xfc KEY_7>, ++ <0xf8 KEY_8>, ++ <0xb0 KEY_9>, ++ <0xb6 KEY_0>, ++ <0xb5 KEY_BACKSPACE>; ++ }; ++}; ++ ++&rk_rga { ++ status = "okay"; ++}; ++ ++&rkvdec { ++ status = "okay"; ++}; ++ ++&rkvdec_mmu { ++ status = "okay"; ++}; ++ ++&rkvenc { ++ venc-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&rkvenc_mmu { ++ status = "okay"; ++}; ++ ++&rknpu { ++ rknpu-supply = <&vdd_npu>; ++ status = "okay"; ++}; ++ ++&rknpu_mmu { ++ status = "okay"; ++}; ++ ++&route_hdmi { ++ status = "okay"; ++ connect = <&vp0_out_hdmi>; ++}; ++ ++&saradc { ++ status = "okay"; ++ vref-supply = <&vcca_1v8>; ++}; ++ ++&sdhci { ++ bus-width = <8>; ++ no-sdio; ++ no-sd; ++ non-removable; ++ max-frequency = <200000000>; ++ status = "okay"; ++}; ++ ++&sdmmc0 { ++ max-frequency = <150000000>; ++ no-sdio; ++ no-mmc; ++ bus-width = <4>; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ disable-wp; ++ sd-uhs-sdr104; ++ vmmc-supply = <&vcc3v3_sd>; ++ vqmmc-supply = <&vccio_sd>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc0_bus4 &sdmmc0_clk &sdmmc0_cmd &sdmmc0_det>; ++ status = "okay"; ++}; ++ ++&sfc { ++ status = "okay"; ++}; ++ ++&spdif_8ch { ++ status = "okay"; ++}; ++ ++&tsadc { ++ status = "okay"; ++}; ++ ++&u2phy0_host { ++ phy-supply = <&vcc5v0_host>; ++ status = "okay"; ++}; ++ ++&u2phy0_otg { ++ vbus-supply = <&vcc5v0_otg>; ++ status = "okay"; ++}; ++ ++&u2phy1_host { ++ phy-supply = <&vcc5v0_host>; ++ status = "okay"; ++}; ++ ++&u2phy1_otg { ++ phy-supply = <&vcc5v0_host>; ++ status = "okay"; ++}; ++ ++&usb2phy0 { ++ status = "okay"; ++}; ++ ++&usb2phy1 { ++ status = "okay"; ++}; ++ ++&usb_host0_ehci { ++ status = "okay"; ++}; ++ ++&usb_host0_ohci { ++ status = "okay"; ++}; ++ ++&usb_host1_ehci { ++ status = "okay"; ++}; ++ ++&usb_host1_ohci { ++ status = "okay"; ++}; ++ ++&usbdrd_dwc3 { ++ dr_mode = "otg"; ++ extcon = <&usb2phy0>; ++ status = "okay"; ++}; ++ ++&usbdrd30 { ++ status = "okay"; ++}; ++ ++&usbhost_dwc3 { ++ status = "okay"; ++}; ++ ++&usbhost30 { ++ status = "okay"; ++}; ++ ++&vad { ++ rockchip,audio-src = <&i2s1_8ch>; ++ rockchip,buffer-time-ms = <128>; ++ rockchip,det-channel = <0>; ++ rockchip,mode = <0>; ++}; ++ ++&vdpu { ++ status = "okay"; ++}; ++ ++&vdpu_mmu { ++ status = "okay"; ++}; ++ ++&vepu { ++ status = "okay"; ++}; ++ ++&vepu_mmu { ++ status = "okay"; ++}; ++ ++&vop { ++ status = "okay"; ++ assigned-clocks = <&cru DCLK_VOP0>, <&cru DCLK_VOP1>; ++ assigned-clock-parents = <&pmucru PLL_HPLL>, <&cru PLL_VPLL>; ++}; ++ ++&vop_mmu { ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-evb1-ddr4-v10-linux-spi-nor.dts b/arch/arm64/boot/dts/rockchip/rk3568-evb1-ddr4-v10-linux-spi-nor.dts +new file mode 100755 +index 000000000000..259b483af0f4 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-evb1-ddr4-v10-linux-spi-nor.dts +@@ -0,0 +1,18 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2021 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++#include "rk3568-evb1-ddr4-v10.dtsi" ++#include "rk3568-linux.dtsi" ++ ++/ { ++ model = "Rockchip RK3568 EVB1 DDR4 V10 Linux SPI NOR Board"; ++ compatible = "rockchip,rk3568-evb1-ddr4-v10-linux-spi-nor", "rockchip,rk3568"; ++ ++ chosen { ++ bootargs = "earlycon=uart8250,mmio32,0xfe660000 console=ttyFIQ0 root=/dev/mtdblock3 rootfstype=squashfs rootwait"; ++ }; ++ ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-evb1-ddr4-v10-linux.dts b/arch/arm64/boot/dts/rockchip/rk3568-evb1-ddr4-v10-linux.dts +new file mode 100755 +index 000000000000..55c255998473 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-evb1-ddr4-v10-linux.dts +@@ -0,0 +1,8 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++#include "rk3568-evb1-ddr4-v10.dtsi" ++#include "rk3568-linux.dtsi" +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-evb1-ddr4-v10.dts b/arch/arm64/boot/dts/rockchip/rk3568-evb1-ddr4-v10.dts +new file mode 100755 +index 000000000000..00c28f39269b +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-evb1-ddr4-v10.dts +@@ -0,0 +1,8 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++#include "rk3568-evb1-ddr4-v10.dtsi" ++#include "rk3568-android.dtsi" +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-evb1-ddr4-v10.dtsi b/arch/arm64/boot/dts/rockchip/rk3568-evb1-ddr4-v10.dtsi +new file mode 100755 +index 000000000000..64881d20183f +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-evb1-ddr4-v10.dtsi +@@ -0,0 +1,490 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++/dts-v1/; ++ ++#include ++#include ++#include "rk3568.dtsi" ++#include "rk3568-evb.dtsi" ++ ++/ { ++ model = "Rockchip RK3568 EVB1 DDR4 V10 Board"; ++ compatible = "rockchip,rk3568-evb1-ddr4-v10", "rockchip,rk3568"; ++ ++ rk_headset: rk-headset { ++ compatible = "rockchip_headset"; ++ headset_gpio = <&gpio3 RK_PC2 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hp_det>; ++ }; ++ ++ vcc2v5_sys: vcc2v5-ddr { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc2v5-sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <2500000>; ++ regulator-max-microvolt = <2500000>; ++ vin-supply = <&vcc3v3_sys>; ++ }; ++ ++ vcc3v3_vga: vcc3v3-vga { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc3v3_vga"; ++ regulator-always-on; ++ regulator-boot-on; ++ gpio = <&gpio0 RK_PD5 GPIO_ACTIVE_HIGH>; ++ enable-active-high; ++ vin-supply = <&vcc3v3_sys>; ++ }; ++ ++ pcie30_avdd0v9: pcie30-avdd0v9 { ++ compatible = "regulator-fixed"; ++ regulator-name = "pcie30_avdd0v9"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <900000>; ++ vin-supply = <&vcc3v3_sys>; ++ }; ++ ++ pcie30_avdd1v8: pcie30-avdd1v8 { ++ compatible = "regulator-fixed"; ++ regulator-name = "pcie30_avdd1v8"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ vin-supply = <&vcc3v3_sys>; ++ }; ++ ++ pcie30_3v3: gpio-regulator { ++ compatible = "regulator-gpio"; ++ regulator-name = "pcie30_3v3"; ++ regulator-min-microvolt = <100000>; ++ regulator-max-microvolt = <3300000>; ++ gpios = <&gpio0 RK_PD4 GPIO_ACTIVE_HIGH>; ++ gpios-states = <0x1>; ++ states = <100000 0x0 ++ 3300000 0x1>; ++ }; ++ ++ vcc3v3_bu: vcc3v3-bu { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc3v3_bu"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ vin-supply = <&vcc5v0_sys>; ++ }; ++ ++ vcc_camera: vcc-camera-regulator { ++ compatible = "regulator-fixed"; ++ gpio = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&camera_pwr>; ++ regulator-name = "vcc_camera"; ++ enable-active-high; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++}; ++ ++&combphy0_us { ++ status = "okay"; ++}; ++ ++&combphy1_usq { ++ status = "okay"; ++}; ++ ++&combphy2_psq { ++ status = "okay"; ++}; ++ ++&csi2_dphy_hw { ++ status = "okay"; ++}; ++ ++&csi2_dphy0 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam0: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ucam_out0>; ++ data-lanes = <1 2 3 4>; ++ }; ++ mipi_in_ucam1: endpoint@2 { ++ reg = <2>; ++ remote-endpoint = <&gc8034_out>; ++ data-lanes = <1 2 3 4>; ++ }; ++ mipi_in_ucam2: endpoint@3 { ++ reg = <3>; ++ remote-endpoint = <&ov5695_out>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ csidphy_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp0_in>; ++ }; ++ }; ++ }; ++}; ++ ++/* ++ * video_phy0 needs to be enabled ++ * when dsi0 is enabled ++ */ ++&dsi0 { ++ status = "okay"; ++}; ++ ++&dsi0_in_vp0 { ++ status = "disabled"; ++}; ++ ++&dsi0_in_vp1 { ++ status = "okay"; ++}; ++ ++&dsi0_panel { ++ power-supply = <&vcc3v3_lcd0_n>; ++}; ++ ++/* ++ * video_phy1 needs to be enabled ++ * when dsi1 is enabled ++ */ ++&dsi1 { ++ status = "disabled"; ++}; ++ ++&dsi1_in_vp0 { ++ status = "disabled"; ++}; ++ ++&dsi1_in_vp1 { ++ status = "disabled"; ++}; ++ ++&dsi1_panel { ++ power-supply = <&vcc3v3_lcd1_n>; ++}; ++ ++&edp { ++ hpd-gpios = <&gpio0 RK_PC2 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++}; ++ ++&edp_phy { ++ status = "okay"; ++}; ++ ++&edp_in_vp0 { ++ status = "okay"; ++}; ++ ++&edp_in_vp1 { ++ status = "disabled"; ++}; ++ ++&gmac0 { ++ phy-mode = "rgmii"; ++ clock_in_out = "output"; ++ ++ snps,reset-gpio = <&gpio2 RK_PD3 GPIO_ACTIVE_LOW>; ++ snps,reset-active-low; ++ /* Reset time is 20ms, 100ms for rtl8211f */ ++ snps,reset-delays-us = <0 20000 100000>; ++ ++ assigned-clocks = <&cru SCLK_GMAC0_RX_TX>, <&cru SCLK_GMAC0>; ++ assigned-clock-parents = <&cru SCLK_GMAC0_RGMII_SPEED>, <&cru CLK_MAC0_2TOP>; ++ assigned-clock-rates = <0>, <125000000>; ++ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&gmac0_miim ++ &gmac0_tx_bus2 ++ &gmac0_rx_bus2 ++ &gmac0_rgmii_clk ++ &gmac0_rgmii_bus>; ++ ++ tx_delay = <0x3c>; ++ rx_delay = <0x2f>; ++ ++ phy-handle = <&rgmii_phy0>; ++ status = "okay"; ++}; ++ ++&gmac1 { ++ phy-mode = "rgmii"; ++ clock_in_out = "output"; ++ ++ snps,reset-gpio = <&gpio2 RK_PD1 GPIO_ACTIVE_LOW>; ++ snps,reset-active-low; ++ /* Reset time is 20ms, 100ms for rtl8211f */ ++ snps,reset-delays-us = <0 20000 100000>; ++ ++ assigned-clocks = <&cru SCLK_GMAC1_RX_TX>, <&cru SCLK_GMAC1>; ++ assigned-clock-parents = <&cru SCLK_GMAC1_RGMII_SPEED>, <&cru CLK_MAC1_2TOP>; ++ assigned-clock-rates = <0>, <125000000>; ++ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&gmac1m1_miim ++ &gmac1m1_tx_bus2 ++ &gmac1m1_rx_bus2 ++ &gmac1m1_rgmii_clk ++ &gmac1m1_rgmii_bus>; ++ ++ tx_delay = <0x4f>; ++ rx_delay = <0x26>; ++ ++ phy-handle = <&rgmii_phy1>; ++ status = "okay"; ++}; ++ ++/* ++ * power-supply should switche to vcc3v3_lcd1_n ++ * when mipi panel is connected to dsi1. ++ */ ++>1x { ++ power-supply = <&vcc3v3_lcd0_n>; ++}; ++ ++&i2c4 { ++ status = "okay"; ++ gc8034: gc8034@37 { ++ compatible = "galaxycore,gc8034"; ++ status = "okay"; ++ reg = <0x37>; ++ clocks = <&cru CLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_clk>; ++ reset-gpios = <&gpio3 RK_PB6 GPIO_ACTIVE_LOW>; ++ pwdn-gpios = <&gpio4 RK_PB4 GPIO_ACTIVE_LOW>; ++ rockchip,grf = <&grf>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "RK-CMK-8M-2-v1"; ++ rockchip,camera-module-lens-name = "CK8401"; ++ port { ++ gc8034_out: endpoint { ++ remote-endpoint = <&mipi_in_ucam1>; ++ data-lanes = <1 2 3 4>; ++ }; ++ }; ++ }; ++ os04a10: os04a10@36 { ++ compatible = "ovti,os04a10"; ++ reg = <0x36>; ++ clocks = <&cru CLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ power-domains = <&power RK3568_PD_VI>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_clk>; ++ reset-gpios = <&gpio3 RK_PB6 GPIO_ACTIVE_LOW>; ++ pwdn-gpios = <&gpio4 RK_PB4 GPIO_ACTIVE_HIGH>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "CMK-OT1607-FV1"; ++ rockchip,camera-module-lens-name = "M12-40IRC-4MP-F16"; ++ port { ++ ucam_out0: endpoint { ++ remote-endpoint = <&mipi_in_ucam0>; ++ data-lanes = <1 2 3 4>; ++ }; ++ }; ++ }; ++ ov5695: ov5695@36 { ++ status = "okay"; ++ compatible = "ovti,ov5695"; ++ reg = <0x36>; ++ clocks = <&cru CLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ power-domains = <&power RK3568_PD_VI>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_clk>; ++ reset-gpios = <&gpio3 RK_PB6 GPIO_ACTIVE_HIGH>; ++ pwdn-gpios = <&gpio4 RK_PB4 GPIO_ACTIVE_HIGH>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "TongJu"; ++ rockchip,camera-module-lens-name = "CHT842-MD"; ++ port { ++ ov5695_out: endpoint { ++ remote-endpoint = <&mipi_in_ucam2>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++}; ++ ++&mdio0 { ++ rgmii_phy0: phy@0 { ++ compatible = "ethernet-phy-ieee802.3-c22"; ++ reg = <0x0>; ++ }; ++}; ++ ++&mdio1 { ++ rgmii_phy1: phy@0 { ++ compatible = "ethernet-phy-ieee802.3-c22"; ++ reg = <0x0>; ++ }; ++}; ++ ++&video_phy0 { ++ status = "okay"; ++}; ++ ++&video_phy1 { ++ status = "disabled"; ++}; ++ ++&pcie30phy { ++ status = "okay"; ++}; ++ ++&pcie3x2 { ++ reset-gpios = <&gpio2 RK_PD6 GPIO_ACTIVE_HIGH>; ++ vpcie3v3-supply = <&pcie30_3v3>; ++ status = "okay"; ++}; ++ ++&pinctrl { ++ cam { ++ camera_pwr: camera-pwr { ++ rockchip,pins = ++ /* camera power en */ ++ <0 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ headphone { ++ hp_det: hp-det { ++ rockchip,pins = <3 RK_PC2 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ wireless-wlan { ++ wifi_host_wake_irq: wifi-host-wake-irq { ++ rockchip,pins = <3 RK_PD4 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart8_gpios: uart8-gpios { ++ rockchip,pins = <2 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++&rkisp { ++ status = "okay"; ++}; ++ ++&rkisp_mmu { ++ status = "okay"; ++}; ++ ++&rkisp_vir0 { ++ status = "okay"; ++ ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp0_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&csidphy_out>; ++ }; ++ }; ++}; ++ ++&route_dsi0 { ++ status = "okay"; ++ connect = <&vp1_out_dsi0>; ++}; ++ ++&sata2 { ++ status = "okay"; ++}; ++ ++&sdmmc2 { ++ max-frequency = <150000000>; ++ no-sd; ++ no-mmc; ++ bus-width = <4>; ++ disable-wp; ++ cap-sd-highspeed; ++ cap-sdio-irq; ++ keep-power-in-suspend; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ non-removable; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc2m0_bus4 &sdmmc2m0_cmd &sdmmc2m0_clk>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&spdif_8ch { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&spdifm1_tx>; ++}; ++ ++&uart8 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart8m0_xfer &uart8m0_ctsn>; ++}; ++ ++&vcc3v3_lcd0_n { ++ gpio = <&gpio0 RK_PC7 GPIO_ACTIVE_HIGH>; ++ enable-active-high; ++}; ++ ++&vcc3v3_lcd1_n { ++ gpio = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; ++ enable-active-high; ++}; ++ ++&wireless_wlan { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_host_wake_irq>; ++ WIFI,host_wake_irq = <&gpio3 RK_PD4 GPIO_ACTIVE_HIGH>; ++}; ++ ++&wireless_bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&rk809 1>; ++ clock-names = "ext_clock"; ++ //wifi-bt-power-toggle; ++ uart_rts_gpios = <&gpio2 RK_PB1 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart8m0_rtsn>; ++ pinctrl-1 = <&uart8_gpios>; ++ BT,reset_gpio = <&gpio3 RK_PA0 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio3 RK_PA2 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio3 RK_PA1 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-evb2-lp4x-v10-bt1120-to-hdmi.dts b/arch/arm64/boot/dts/rockchip/rk3568-evb2-lp4x-v10-bt1120-to-hdmi.dts +new file mode 100755 +index 000000000000..7a5a7feb52a7 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-evb2-lp4x-v10-bt1120-to-hdmi.dts +@@ -0,0 +1,79 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++#include "rk3568-evb2-lp4x-v10.dtsi" ++#include "rk3568-android.dtsi" ++ ++&dsi0 { ++ status = "disabled"; ++}; ++ ++&hdmi { ++ status = "disabled"; ++}; ++ ++&i2c3 { ++ clock-frequency = <400000>; ++ status = "okay"; ++ ++ sii9022: sii9022@39 { ++ compatible = "sil,sii9022"; ++ reg = <0x39>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sii902x_hdmi_int>; ++ interrupt-parent = <&gpio3>; ++ interrupts = ; ++ reset-gpio = <&gpio3 RK_PC4 GPIO_ACTIVE_LOW>; ++ enable-gpio = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; ++ bus-format = <1>; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ ++ sii9022_in_rgb: endpoint { ++ remote-endpoint = <&rgb_out_sii9022>; ++ }; ++ }; ++ }; ++ }; ++}; ++ ++&video_phy0 { ++ status = "disabled"; ++}; ++ ++&rgb { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&bt1120_pins>; ++ ++ ports { ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ rgb_out_sii9022: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&sii9022_in_rgb>; ++ }; ++ }; ++ }; ++}; ++ ++&rgb_in_vp2 { ++ status = "okay"; ++}; ++ ++&vcc3v3_lcd1_n { ++ status = "disabled"; ++ gpio = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; ++ enable-active-high; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-evb2-lp4x-v10.dts b/arch/arm64/boot/dts/rockchip/rk3568-evb2-lp4x-v10.dts +new file mode 100755 +index 000000000000..23f5f036c4b2 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-evb2-lp4x-v10.dts +@@ -0,0 +1,8 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++#include "rk3568-evb2-lp4x-v10.dtsi" ++#include "rk3568-android.dtsi" +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-evb2-lp4x-v10.dtsi b/arch/arm64/boot/dts/rockchip/rk3568-evb2-lp4x-v10.dtsi +new file mode 100755 +index 000000000000..c2d0b697a9bd +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-evb2-lp4x-v10.dtsi +@@ -0,0 +1,493 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++/dts-v1/; ++ ++#include ++#include ++#include "rk3568.dtsi" ++#include "rk3568-evb.dtsi" ++ ++/ { ++ model = "Rockchip RK3568 EVB2 LP4X V10 Board"; ++ compatible = "rockchip,rk3568-evb2-lp4x-v10", "rockchip,rk3568"; ++ ++ rk_headset: rk-headset { ++ compatible = "rockchip_headset"; ++ headset_gpio = <&gpio4 RK_PC1 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hp_det>; ++ }; ++ ++ vcc2v5_sys: vcc2v5-ddr { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc2v5-sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <2500000>; ++ regulator-max-microvolt = <2500000>; ++ vin-supply = <&vcc3v3_sys>; ++ }; ++ ++ pcie20_3v3: gpio-regulator { ++ compatible = "regulator-gpio"; ++ regulator-name = "pcie20_3v3"; ++ regulator-min-microvolt = <100000>; ++ regulator-max-microvolt = <3300000>; ++ gpios = <&gpio0 RK_PD4 GPIO_ACTIVE_HIGH>; ++ gpios-states = <0x1>; ++ states = <100000 0x0 ++ 3300000 0x1>; ++ }; ++ ++ pcie30_avdd0v9: pcie30-avdd0v9 { ++ compatible = "regulator-fixed"; ++ regulator-name = "pcie30_avdd0v9"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <900000>; ++ vin-supply = <&vcc3v3_sys>; ++ }; ++ ++ pcie30_avdd1v8: pcie30-avdd1v8 { ++ compatible = "regulator-fixed"; ++ regulator-name = "pcie30_avdd1v8"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ vin-supply = <&vcc3v3_sys>; ++ }; ++ ++ pcie30_3v3: gpio-regulator { ++ compatible = "regulator-gpio"; ++ regulator-name = "pcie30_3v3"; ++ regulator-min-microvolt = <100000>; ++ regulator-max-microvolt = <3300000>; ++ gpios = <&gpio0 RK_PD4 GPIO_ACTIVE_HIGH>; ++ gpios-states = <0x1>; ++ states = <100000 0x0 ++ 3300000 0x1>; ++ }; ++ ++ qsgmii_3v3: gpio-regulator { ++ compatible = "regulator-gpio"; ++ regulator-name = "qsgmii_3v3"; ++ regulator-min-microvolt = <0100000>; ++ regulator-max-microvolt = <3300000>; ++ gpios = <&gpio0 RK_PB7 GPIO_ACTIVE_HIGH>; ++ gpios-states = <0x1>; ++ states = <0100000 0x0 ++ 3300000 0x1>; ++ }; ++ ++ vcc3v3_bu: vcc3v3-bu { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc3v3_bu"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ vin-supply = <&vcc5v0_sys>; ++ }; ++}; ++ ++&combphy0_us { ++ status = "okay"; ++}; ++ ++&combphy1_usq { ++ status = "okay"; ++}; ++ ++&combphy2_psq { ++ status = "okay"; ++}; ++ ++/* ++ * video_phy0 needs to be enabled ++ * when dsi0 is enabled ++ */ ++&dsi0 { ++ status = "okay"; ++}; ++ ++&dsi0_in_vp0 { ++ status = "disabled"; ++}; ++ ++&dsi0_in_vp1 { ++ status = "okay"; ++}; ++ ++&dsi0_panel { ++ power-supply = <&vcc3v3_lcd0_n>; ++}; ++ ++/* ++ * video_phy1 needs to be enabled ++ * when dsi1 is enabled ++ */ ++&dsi1 { ++ status = "disabled"; ++}; ++ ++&dsi1_in_vp0 { ++ status = "disabled"; ++}; ++ ++&dsi1_in_vp1 { ++ status = "disabled"; ++}; ++ ++&dsi1_panel { ++ power-supply = <&vcc3v3_lcd1_n>; ++}; ++ ++&gmac0 { ++ phy-supply = <&qsgmii_3v3>; ++ phy-mode = "qsgmii"; ++ rockchip,xpcs = <&xpcs>; ++ ++ assigned-clocks = <&cru SCLK_GMAC0_RX_TX>; ++ assigned-clock-parents = <&gmac0_xpcsclk>; ++ ++ power-domains = <&power RK3568_PD_PIPE>; ++ phys = <&combphy1_usq PHY_TYPE_QSGMII>; ++ phy-handle = <&qsgmii_phy0>; ++ ++ status = "okay"; ++}; ++ ++&gmac1 { ++ phy-supply = <&qsgmii_3v3>; ++ phy-mode = "qsgmii"; ++ ++ snps,reset-gpio = <&gpio3 RK_PD5 GPIO_ACTIVE_LOW>; ++ snps,reset-active-low; ++ /* Reset time is 20ms, 100ms for rtl8211f */ ++ snps,reset-delays-us = <0 20000 100000>; ++ ++ assigned-clocks = <&cru SCLK_GMAC1_RX_TX>; ++ assigned-clock-parents = <&gmac1_xpcsclk>; ++ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&gmac1m1_miim>; ++ ++ power-domains = <&power RK3568_PD_PIPE>; ++ phy-handle = <&qsgmii_phy1>; ++ ++ status = "okay"; ++}; ++ ++/* ++ * power-supply should switche to vcc3v3_lcd1_n ++ * when mipi panel is connected to dsi1. ++ */ ++>1x { ++ power-supply = <&vcc3v3_lcd0_n>; ++}; ++ ++&i2c4 { ++ status = "okay"; ++ ++ gs_mxc6655xa: gs_mxc6655xa@15 { ++ status = "okay"; ++ compatible = "gs_mxc6655xa"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&mxc6655xa_irq_gpio>; ++ reg = <0x15>; ++ irq-gpio = <&gpio3 RK_PD7 IRQ_TYPE_LEVEL_LOW>; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ type = ; ++ power-off-in-suspend = <1>; ++ layout = <1>; ++ }; ++}; ++ ++&i2c5 { ++ status = "disabled"; ++}; ++ ++&mdio1 { ++ qsgmii_phy0: phy@0 { ++ compatible = "ethernet-phy-id001c.c942", "ethernet-phy-ieee802.3-c22"; ++ reg = <0x0>; ++ }; ++ qsgmii_phy1: phy@1 { ++ compatible = "ethernet-phy-id001c.c942", "ethernet-phy-ieee802.3-c22"; ++ reg = <0x1>; ++ }; ++ qsgmii_phy2: phy@2 { ++ compatible = "ethernet-phy-id001c.c942", "ethernet-phy-ieee802.3-c22"; ++ reg = <0x2>; ++ }; ++ qsgmii_phy3: phy@3 { ++ compatible = "ethernet-phy-id001c.c942", "ethernet-phy-ieee802.3-c22"; ++ reg = <0x3>; ++ }; ++}; ++ ++&video_phy0 { ++ status = "okay"; ++}; ++ ++&video_phy1 { ++ status = "disabled"; ++}; ++ ++&pcie2x1 { ++ reset-gpios = <&gpio1 RK_PB2 GPIO_ACTIVE_HIGH>; ++ vpcie3v3-supply = <&pcie20_3v3>; ++ status = "okay"; ++}; ++ ++&pcie30phy { ++ status = "okay"; ++}; ++ ++&pcie3x2 { ++ reset-gpios = <&gpio0 RK_PC6 GPIO_ACTIVE_HIGH>; ++ vpcie3v3-supply = <&pcie30_3v3>; ++ status = "okay"; ++}; ++ ++&pinctrl { ++ headphone { ++ hp_det: hp-det { ++ rockchip,pins = <4 RK_PC1 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ mxc6655xa { ++ mxc6655xa_irq_gpio: mxc6655xa_irq_gpio { ++ rockchip,pins = <3 RK_PD7 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ sii902x { ++ sii902x_hdmi_int: sii902x-hdmi-int { ++ rockchip,pins = <3 RK_PA0 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <2 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ wifi_32k: wifi-32k { ++ rockchip,pins = <2 RK_PC6 1 &pcfg_pull_none>; ++ }; ++ }; ++ ++ wireless-wlan { ++ wifi_host_wake_irq: wifi-host-wake-irq { ++ rockchip,pins = <2 RK_PB2 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart1_gpios: uart1-gpios { ++ rockchip,pins = <2 RK_PB5 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++&pmu_io_domains { ++ vccio6-supply = <&vcc_3v3>; ++}; ++ ++&pwm3 { ++ status = "okay"; ++ ++ compatible = "rockchip,remotectl-pwm"; ++ remote_pwm_id = <3>; ++ handle_cpu_id = <1>; ++ remote_support_psci = <0>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pwm3_pins>; ++ ++ ir_key1 { ++ rockchip,usercode = <0x4040>; ++ rockchip,key_table = ++ <0xf2 KEY_REPLY>, ++ <0xba KEY_BACK>, ++ <0xf4 KEY_UP>, ++ <0xf1 KEY_DOWN>, ++ <0xef KEY_LEFT>, ++ <0xee KEY_RIGHT>, ++ <0xbd KEY_HOME>, ++ <0xea KEY_VOLUMEUP>, ++ <0xe3 KEY_VOLUMEDOWN>, ++ <0xe2 KEY_SEARCH>, ++ <0xb2 KEY_POWER>, ++ <0xbc KEY_MUTE>, ++ <0xec KEY_MENU>, ++ <0xbf 0x190>, ++ <0xe0 0x191>, ++ <0xe1 0x192>, ++ <0xe9 183>, ++ <0xe6 248>, ++ <0xe8 185>, ++ <0xe7 186>, ++ <0xf0 388>, ++ <0xbe 0x175>; ++ }; ++ ++ ir_key2 { ++ rockchip,usercode = <0xff00>; ++ rockchip,key_table = ++ <0xf9 KEY_HOME>, ++ <0xbf KEY_BACK>, ++ <0xfb KEY_MENU>, ++ <0xaa KEY_REPLY>, ++ <0xb9 KEY_UP>, ++ <0xe9 KEY_DOWN>, ++ <0xb8 KEY_LEFT>, ++ <0xea KEY_RIGHT>, ++ <0xeb KEY_VOLUMEDOWN>, ++ <0xef KEY_VOLUMEUP>, ++ <0xf7 KEY_MUTE>, ++ <0xe7 KEY_POWER>, ++ <0xfc KEY_POWER>, ++ <0xa9 KEY_VOLUMEDOWN>, ++ <0xa8 KEY_VOLUMEDOWN>, ++ <0xe0 KEY_VOLUMEDOWN>, ++ <0xa5 KEY_VOLUMEDOWN>, ++ <0xab 183>, ++ <0xb7 388>, ++ <0xe8 388>, ++ <0xf8 184>, ++ <0xaf 185>, ++ <0xed KEY_VOLUMEDOWN>, ++ <0xee 186>, ++ <0xb3 KEY_VOLUMEDOWN>, ++ <0xf1 KEY_VOLUMEDOWN>, ++ <0xf2 KEY_VOLUMEDOWN>, ++ <0xf3 KEY_SEARCH>, ++ <0xb4 KEY_VOLUMEDOWN>, ++ <0xbe KEY_SEARCH>; ++ }; ++ ++ ir_key3 { ++ rockchip,usercode = <0x1dcc>; ++ rockchip,key_table = ++ <0xee KEY_REPLY>, ++ <0xf0 KEY_BACK>, ++ <0xf8 KEY_UP>, ++ <0xbb KEY_DOWN>, ++ <0xef KEY_LEFT>, ++ <0xed KEY_RIGHT>, ++ <0xfc KEY_HOME>, ++ <0xf1 KEY_VOLUMEUP>, ++ <0xfd KEY_VOLUMEDOWN>, ++ <0xb7 KEY_SEARCH>, ++ <0xff KEY_POWER>, ++ <0xf3 KEY_MUTE>, ++ <0xbf KEY_MENU>, ++ <0xf9 0x191>, ++ <0xf5 0x192>, ++ <0xb3 388>, ++ <0xbe KEY_1>, ++ <0xba KEY_2>, ++ <0xb2 KEY_3>, ++ <0xbd KEY_4>, ++ <0xf9 KEY_5>, ++ <0xb1 KEY_6>, ++ <0xfc KEY_7>, ++ <0xf8 KEY_8>, ++ <0xb0 KEY_9>, ++ <0xb6 KEY_0>, ++ <0xb5 KEY_BACKSPACE>; ++ }; ++}; ++ ++&pwm7 { ++ status = "disabled"; ++}; ++ ++&route_dsi0 { ++ status = "okay"; ++ connect = <&vp1_out_dsi0>; ++}; ++ ++&sdio_pwrseq { ++ clocks = <&pmucru CLK_RTC_32K>; ++ pinctrl-0 = <&wifi_enable_h &wifi_32k>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio2 RK_PB1 GPIO_ACTIVE_LOW>; ++}; ++ ++&sdmmc1 { ++ max-frequency = <150000000>; ++ no-sd; ++ no-mmc; ++ bus-width = <4>; ++ disable-wp; ++ cap-sd-highspeed; ++ cap-sdio-irq; ++ keep-power-in-suspend; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ non-removable; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc1_bus4 &sdmmc1_cmd &sdmmc1_clk>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&sdmmc2 { ++ status = "disabled"; ++}; ++ ++&uart1 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart1m0_xfer &uart1m0_ctsn>; ++}; ++ ++&vcc3v3_lcd0_n { ++ gpio = <&gpio0 RK_PC7 GPIO_ACTIVE_HIGH>; ++ enable-active-high; ++}; ++ ++&vcc3v3_lcd1_n { ++ gpio = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; ++ enable-active-high; ++}; ++ ++&wireless_wlan { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_host_wake_irq>; ++ WIFI,host_wake_irq = <&gpio2 RK_PB2 GPIO_ACTIVE_HIGH>; ++}; ++ ++&wireless_bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&pmucru CLK_RTC_32K>; ++ clock-names = "ext_clock"; ++ //wifi-bt-power-toggle; ++ uart_rts_gpios = <&gpio2 RK_PB5 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart1m0_rtsn>; ++ pinctrl-1 = <&uart1_gpios>; ++ BT,reset_gpio = <&gpio2 RK_PB7 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio2 RK_PC1 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio2 RK_PC0 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++}; ++ ++&xpcs { ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-evb4-lp3-v10.dts b/arch/arm64/boot/dts/rockchip/rk3568-evb4-lp3-v10.dts +new file mode 100755 +index 000000000000..5884da41f121 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-evb4-lp3-v10.dts +@@ -0,0 +1,12 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++#include "rk3568-evb1-ddr4-v10.dtsi" ++#include "rk3568-android.dtsi" ++/{ ++ model = "Rockchip RK3568 EVB4 LP3 V10 Board"; ++ compatible = "rockchip,rk3568-evb4-lp3-v10", "rockchip,rk3568"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-evb5-ddr4-v10.dts b/arch/arm64/boot/dts/rockchip/rk3568-evb5-ddr4-v10.dts +new file mode 100755 +index 000000000000..e9eb333079a2 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-evb5-ddr4-v10.dts +@@ -0,0 +1,8 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++#include "rk3568-evb5-ddr4-v10.dtsi" ++#include "rk3568-android.dtsi" +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-evb5-ddr4-v10.dtsi b/arch/arm64/boot/dts/rockchip/rk3568-evb5-ddr4-v10.dtsi +new file mode 100755 +index 000000000000..e8a00fc85483 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-evb5-ddr4-v10.dtsi +@@ -0,0 +1,539 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++/dts-v1/; ++ ++#include ++#include ++#include "rk3568.dtsi" ++#include "rk3568-evb.dtsi" ++ ++/ { ++ model = "Rockchip RK3568 EVB5 DDR4 V10 Board"; ++ compatible = "rockchip,rk3568-evb5-ddr4-v10", "rockchip,rk3568"; ++ ++ rk_headset: rk-headset { ++ compatible = "rockchip_headset"; ++ headset_gpio = <&gpio3 RK_PC2 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hp_det>; ++ }; ++ ++ vcc2v5_sys: vcc2v5-ddr { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc2v5-sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <2500000>; ++ regulator-max-microvolt = <2500000>; ++ vin-supply = <&vcc3v3_sys>; ++ }; ++ ++ pcie30_avdd0v9: pcie30-avdd0v9 { ++ compatible = "regulator-fixed"; ++ regulator-name = "pcie30_avdd0v9"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <900000>; ++ vin-supply = <&vcc3v3_sys>; ++ }; ++ ++ pcie30_avdd1v8: pcie30-avdd1v8 { ++ compatible = "regulator-fixed"; ++ regulator-name = "pcie30_avdd1v8"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ vin-supply = <&vcc3v3_sys>; ++ }; ++ ++ pcie30_3v3: gpio-regulator { ++ compatible = "regulator-gpio"; ++ regulator-name = "pcie30_3v3"; ++ regulator-min-microvolt = <100000>; ++ regulator-max-microvolt = <3300000>; ++ gpios = <&gpio0 RK_PD4 GPIO_ACTIVE_HIGH>; ++ gpios-states = <0x1>; ++ states = <100000 0x0 ++ 3300000 0x1>; ++ }; ++ ++ vcc3v3_bu: vcc3v3-bu { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc3v3_bu"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ vin-supply = <&vcc5v0_sys>; ++ }; ++ ++ vcc_camera: vcc-camera-regulator { ++ compatible = "regulator-fixed"; ++ gpio = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&camera_pwr>; ++ regulator-name = "vcc_camera"; ++ enable-active-high; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++}; ++ ++&combphy0_us { ++ status = "okay"; ++}; ++ ++&combphy1_usq { ++ rockchip,sgmii-mac-sel = <0>; ++ status = "okay"; ++}; ++ ++&combphy2_psq { ++ status = "okay"; ++}; ++ ++&csi2_dphy_hw { ++ status = "okay"; ++}; ++ ++&csi2_dphy0 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam0: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ucam_out0>; ++ data-lanes = <1 2 3 4>; ++ }; ++ mipi_in_ucam1: endpoint@2 { ++ reg = <2>; ++ remote-endpoint = <&gc8034_out>; ++ data-lanes = <1 2 3 4>; ++ }; ++ mipi_in_ucam2: endpoint@3 { ++ reg = <3>; ++ remote-endpoint = <&ov5695_out>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ csidphy_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp0_in>; ++ }; ++ }; ++ }; ++}; ++ ++/* ++ * video_phy0 needs to be enabled ++ * when dsi0 is enabled ++ */ ++&dsi0 { ++ status = "okay"; ++}; ++ ++&dsi0_in_vp0 { ++ status = "disabled"; ++}; ++ ++&dsi0_in_vp1 { ++ status = "okay"; ++}; ++ ++&dsi0_panel { ++ power-supply = <&vcc3v3_lcd0_n>; ++}; ++ ++/* ++ * video_phy1 needs to be enabled ++ * when dsi1 is enabled ++ */ ++&dsi1 { ++ status = "disabled"; ++}; ++ ++&dsi1_in_vp0 { ++ status = "disabled"; ++}; ++ ++&dsi1_in_vp1 { ++ status = "disabled"; ++}; ++ ++&dsi1_panel { ++ power-supply = <&vcc3v3_lcd1_n>; ++}; ++ ++&edp { ++ hpd-gpios = <&gpio0 RK_PC2 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++}; ++ ++&edp_phy { ++ status = "okay"; ++}; ++ ++&edp_in_vp0 { ++ status = "okay"; ++}; ++ ++&edp_in_vp1 { ++ status = "disabled"; ++}; ++ ++&gmac0 { ++ phy-mode = "sgmii"; ++ ++ rockchip,pipegrf = <&pipegrf>; ++ rockchip,xpcs = <&xpcs>; ++ ++ snps,reset-gpio = <&gpio2 RK_PC2 GPIO_ACTIVE_LOW>; ++ snps,reset-active-low; ++ /* Reset time is 20ms, 100ms for rtl8211f */ ++ snps,reset-delays-us = <0 20000 100000>; ++ ++ assigned-clocks = <&cru SCLK_GMAC0_RX_TX>; ++ assigned-clock-parents = <&gmac0_xpcsclk>; ++ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&gmac0_miim>; ++ ++ power-domains = <&power RK3568_PD_PIPE>; ++ phys = <&combphy1_usq PHY_TYPE_SGMII>; ++ phy-handle = <&sgmii_phy>; ++ ++ status = "okay"; ++}; ++ ++&gmac1 { ++ phy-mode = "rgmii"; ++ clock_in_out = "output"; ++ ++ snps,reset-gpio = <&gpio3 RK_PD5 GPIO_ACTIVE_LOW>; ++ snps,reset-active-low; ++ /* Reset time is 20ms, 100ms for rtl8211f */ ++ snps,reset-delays-us = <0 20000 100000>; ++ ++ assigned-clocks = <&cru SCLK_GMAC1_RX_TX>, <&cru SCLK_GMAC1>; ++ assigned-clock-parents = <&cru SCLK_GMAC1_RGMII_SPEED>, <&cru CLK_MAC1_2TOP>; ++ assigned-clock-rates = <0>, <125000000>; ++ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&gmac1m1_miim ++ &gmac1m1_tx_bus2_level3 ++ &gmac1m1_rx_bus2 ++ &gmac1m1_rgmii_clk_level2 ++ &gmac1m1_rgmii_bus_level3>; ++ ++ tx_delay = <0x46>; ++ rx_delay = <0x2f>; ++ ++ phy-handle = <&rgmii_phy1>; ++ status = "okay"; ++}; ++ ++/* ++ * power-supply should switche to vcc3v3_lcd1_n ++ * when mipi panel is connected to dsi1. ++ */ ++>1x { ++ power-supply = <&vcc3v3_lcd0_n>; ++}; ++ ++&i2c3 { ++ clock-frequency = <400000>; ++ status = "okay"; ++ ++ sii9022: sii9022@39 { ++ compatible = "sil,sii9022"; ++ reg = <0x39>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sii902x_hdmi_int>; ++ interrupt-parent = <&gpio4>; ++ interrupts = ; ++ reset-gpio = <&gpio3 RK_PC4 GPIO_ACTIVE_LOW>; ++ enable-gpio = <&gpio0 RK_PD5 GPIO_ACTIVE_HIGH>; ++ bus-format = <0>; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ ++ sii9022_in_rgb: endpoint { ++ remote-endpoint = <&rgb_out_sii9022>; ++ }; ++ }; ++ }; ++ }; ++}; ++ ++&i2c4 { ++ status = "okay"; ++ gc8034: gc8034@37 { ++ compatible = "galaxycore,gc8034"; ++ status = "okay"; ++ reg = <0x37>; ++ clocks = <&cru CLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_clk>; ++ reset-gpios = <&gpio3 RK_PB6 GPIO_ACTIVE_LOW>; ++ pwdn-gpios = <&gpio4 RK_PB4 GPIO_ACTIVE_LOW>; ++ rockchip,grf = <&grf>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "RK-CMK-8M-2-v1"; ++ rockchip,camera-module-lens-name = "CK8401"; ++ port { ++ gc8034_out: endpoint { ++ remote-endpoint = <&mipi_in_ucam1>; ++ data-lanes = <1 2 3 4>; ++ }; ++ }; ++ }; ++ os04a10: os04a10@36 { ++ compatible = "ovti,os04a10"; ++ reg = <0x36>; ++ clocks = <&cru CLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ power-domains = <&power RK3568_PD_VI>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_clk>; ++ reset-gpios = <&gpio3 RK_PB6 GPIO_ACTIVE_LOW>; ++ pwdn-gpios = <&gpio4 RK_PB4 GPIO_ACTIVE_HIGH>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "CMK-OT1607-FV1"; ++ rockchip,camera-module-lens-name = "M12-40IRC-4MP-F16"; ++ port { ++ ucam_out0: endpoint { ++ remote-endpoint = <&mipi_in_ucam0>; ++ data-lanes = <1 2 3 4>; ++ }; ++ }; ++ }; ++ ov5695: ov5695@36 { ++ status = "okay"; ++ compatible = "ovti,ov5695"; ++ reg = <0x36>; ++ clocks = <&cru CLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ power-domains = <&power RK3568_PD_VI>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_clk>; ++ reset-gpios = <&gpio3 RK_PB6 GPIO_ACTIVE_HIGH>; ++ pwdn-gpios = <&gpio4 RK_PB4 GPIO_ACTIVE_HIGH>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "TongJu"; ++ rockchip,camera-module-lens-name = "CHT842-MD"; ++ port { ++ ov5695_out: endpoint { ++ remote-endpoint = <&mipi_in_ucam2>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++}; ++ ++&i2c5 { ++ status = "disabled"; ++}; ++ ++&mdio0 { ++ sgmii_phy: phy@1 { ++ compatible = "ethernet-phy-ieee802.3-c22"; ++ reg = <0x1>; ++ }; ++}; ++ ++&mdio1 { ++ rgmii_phy1: phy@0 { ++ compatible = "ethernet-phy-ieee802.3-c22"; ++ reg = <0x0>; ++ }; ++}; ++ ++&video_phy0 { ++ status = "okay"; ++}; ++ ++&video_phy1 { ++ status = "disabled"; ++}; ++ ++&pcie30phy { ++ status = "okay"; ++}; ++ ++&pcie3x2 { ++ reset-gpios = <&gpio2 RK_PD6 GPIO_ACTIVE_HIGH>; ++ vpcie3v3-supply = <&pcie30_3v3>; ++ status = "okay"; ++}; ++ ++&pinctrl { ++ cam { ++ camera_pwr: camera-pwr { ++ rockchip,pins = ++ /* camera power en */ ++ <0 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ headphone { ++ hp_det: hp-det { ++ rockchip,pins = <3 RK_PC2 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ sii902x { ++ sii902x_hdmi_int: sii902x-hdmi-int { ++ rockchip,pins = <4 RK_PD2 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ wireless-wlan { ++ wifi_host_wake_irq: wifi-host-wake-irq { ++ rockchip,pins = <3 RK_PD4 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart8_gpios: uart8-gpios { ++ rockchip,pins = <2 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++&rgb { ++ status = "okay"; ++ ++ ports { ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ rgb_out_sii9022: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&sii9022_in_rgb>; ++ }; ++ }; ++ }; ++}; ++ ++&rgb_in_vp2 { ++ status = "okay"; ++}; ++ ++&rkisp { ++ status = "okay"; ++}; ++ ++&rkisp_mmu { ++ status = "okay"; ++}; ++ ++&rkisp_vir0 { ++ status = "okay"; ++ ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp0_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&csidphy_out>; ++ }; ++ }; ++}; ++ ++&route_dsi0 { ++ status = "okay"; ++ connect = <&vp1_out_dsi0>; ++}; ++ ++&sata2 { ++ status = "okay"; ++}; ++ ++&sdmmc2 { ++ max-frequency = <150000000>; ++ no-sd; ++ no-mmc; ++ bus-width = <4>; ++ disable-wp; ++ cap-sd-highspeed; ++ cap-sdio-irq; ++ keep-power-in-suspend; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ non-removable; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc2m0_bus4 &sdmmc2m0_cmd &sdmmc2m0_clk>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&spdif_8ch { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&spdifm1_tx>; ++}; ++ ++&uart8 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart8m0_xfer &uart8m0_ctsn>; ++}; ++ ++&vcc3v3_lcd0_n { ++ gpio = <&gpio0 RK_PC7 GPIO_ACTIVE_HIGH>; ++ enable-active-high; ++}; ++ ++&vcc3v3_lcd1_n { ++ gpio = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; ++ enable-active-high; ++}; ++ ++&wireless_wlan { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_host_wake_irq>; ++ WIFI,host_wake_irq = <&gpio3 RK_PD4 GPIO_ACTIVE_HIGH>; ++}; ++ ++&wireless_bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&rk809 1>; ++ clock-names = "ext_clock"; ++ //wifi-bt-power-toggle; ++ uart_rts_gpios = <&gpio2 RK_PB1 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart8m0_rtsn>; ++ pinctrl-1 = <&uart8_gpios>; ++ BT,reset_gpio = <&gpio3 RK_PA0 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio3 RK_PA2 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio3 RK_PA1 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-evb6-ddr3-v10-linux.dts b/arch/arm64/boot/dts/rockchip/rk3568-evb6-ddr3-v10-linux.dts +new file mode 100755 +index 000000000000..a11975fc7d53 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-evb6-ddr3-v10-linux.dts +@@ -0,0 +1,8 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++#include "rk3568-evb6-ddr3-v10.dtsi" ++#include "rk3568-linux.dtsi" +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-evb6-ddr3-v10-rk628-bt1120-to-hdmi.dts b/arch/arm64/boot/dts/rockchip/rk3568-evb6-ddr3-v10-rk628-bt1120-to-hdmi.dts +new file mode 100755 +index 000000000000..0c5ea5eb32ba +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-evb6-ddr3-v10-rk628-bt1120-to-hdmi.dts +@@ -0,0 +1,127 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ */ ++ ++#include "rk3568-evb6-ddr3-v10.dtsi" ++#include "rk3568-android.dtsi" ++ ++&dsi0 { ++ status = "disabled"; ++}; ++ ++&i2c3 { ++ clock-frequency = <400000>; ++ status = "okay"; ++ ++ rk628: rk628@50 { ++ reg = <0x50>; ++ interrupt-parent = <&gpio0>; ++ interrupts = ; ++ enable-gpios = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; ++ reset-gpios = <&gpio0 RK_PB7 GPIO_ACTIVE_LOW>; ++ status = "okay"; ++ }; ++}; ++ ++&video_phy0 { ++ status = "disabled"; ++}; ++ ++#include ++ ++&rk628_hdmi { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ ++ hdmi_in_post_process: endpoint { ++ remote-endpoint = <&post_process_out_hdmi>; ++ }; ++ }; ++ }; ++}; ++ ++&rk628_post_process { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&vop_pins>; ++ status = "okay"; ++ ++ mode-sync-pol = <0>; ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ ++ post_process_in_bt1120: endpoint { ++ remote-endpoint = <&bt1120_out_post_process>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ ++ post_process_out_hdmi: endpoint { ++ remote-endpoint = <&hdmi_in_post_process>; ++ }; ++ }; ++ }; ++}; ++ ++&rk628_bt1120_rx { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ ++ bt1120_in_rgb: endpoint { ++ remote-endpoint = <&rgb_out_bt1120>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ ++ bt1120_out_post_process: endpoint { ++ remote-endpoint = <&post_process_in_bt1120>; ++ }; ++ }; ++ }; ++}; ++ ++&rgb { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&bt1120_pins>; ++ ++ ports { ++ port@1 { ++ reg = <1>; ++ ++ rgb_out_bt1120: endpoint { ++ remote-endpoint = <&bt1120_in_rgb>; ++ }; ++ }; ++ }; ++}; ++ ++&rgb_in_vp2 { ++ status = "okay"; ++}; ++ ++&vcc3v3_lcd1_n { ++ status = "disabled"; ++ gpio = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; ++ enable-active-high; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-evb6-ddr3-v10-rk628-rgb2hdmi.dts b/arch/arm64/boot/dts/rockchip/rk3568-evb6-ddr3-v10-rk628-rgb2hdmi.dts +new file mode 100755 +index 000000000000..597fa0b0b089 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-evb6-ddr3-v10-rk628-rgb2hdmi.dts +@@ -0,0 +1,96 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ */ ++ ++#include "rk3568-evb6-ddr3-v10.dtsi" ++#include "rk3568-android.dtsi" ++ ++&dsi0 { ++ status = "disabled"; ++}; ++ ++&i2c3 { ++ clock-frequency = <400000>; ++ status = "okay"; ++ ++ rk628: rk628@50 { ++ reg = <0x50>; ++ interrupt-parent = <&gpio0>; ++ interrupts = ; ++ enable-gpios = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; ++ reset-gpios = <&gpio0 RK_PB7 GPIO_ACTIVE_LOW>; ++ status = "okay"; ++ }; ++}; ++ ++#include ++ ++&rk628_hdmi { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ ++ hdmi_in_post_process: endpoint { ++ remote-endpoint = <&post_process_out_hdmi>; ++ }; ++ }; ++ }; ++}; ++ ++&rk628_post_process { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&vop_pins>; ++ status = "okay"; ++ ++ mode-sync-pol = <0>; ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ ++ post_process_in_rgb: endpoint { ++ remote-endpoint = <&rgb_out_post_process>; ++ }; ++ }; ++ ++ port@1 { ++ reg = <1>; ++ ++ post_process_out_hdmi: endpoint { ++ remote-endpoint = <&hdmi_in_post_process>; ++ }; ++ }; ++ }; ++}; ++ ++&rgb { ++ status = "okay"; ++ ++ ports { ++ port@1 { ++ reg = <1>; ++ ++ rgb_out_post_process: endpoint { ++ remote-endpoint = <&post_process_in_rgb>; ++ }; ++ }; ++ }; ++}; ++ ++&rgb_in_vp2 { ++ status = "okay"; ++}; ++ ++&vcc3v3_lcd1_n { ++ status = "disabled"; ++ gpio = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; ++ enable-active-high; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-evb6-ddr3-v10-rk630-bt656-to-cvbs.dts b/arch/arm64/boot/dts/rockchip/rk3568-evb6-ddr3-v10-rk630-bt656-to-cvbs.dts +new file mode 100755 +index 000000000000..24c4debec4f3 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-evb6-ddr3-v10-rk630-bt656-to-cvbs.dts +@@ -0,0 +1,70 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ */ ++ ++#include ++#include "rk3568-evb6-ddr3-v10.dtsi" ++#include "rk3568-android.dtsi" ++ ++&dsi0 { ++ status = "disabled"; ++}; ++ ++&hdmi { ++ status = "disabled"; ++}; ++ ++&i2c3 { ++ status = "okay"; ++ clock-frequency = <100000>; ++ ++ rk630: rk630@50 { ++ compatible = "rockchip,rk630"; ++ reg = <0x50>; ++ reset-gpios = <&gpio2 RK_PC7 GPIO_ACTIVE_LOW>; ++ status = "okay"; ++ ++ rk630_tve: rk630-tve { ++ compatible = "rockchip,rk630-tve"; ++ status = "okay"; ++ ++ ports { ++ port { ++ rk630_tve_in_rgb: endpoint { ++ remote-endpoint = <&rgb_out_rk630_tve>; ++ }; ++ }; ++ }; ++ }; ++ }; ++}; ++ ++&rgb { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&bt656m0_pins>; /* bt656m0_pins or bt656m1_pins */ ++ status = "okay"; ++ ++ ports { ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ rgb_out_rk630_tve: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&rk630_tve_in_rgb>; ++ }; ++ }; ++ }; ++}; ++ ++&rgb_in_vp2 { ++ status = "okay"; ++}; ++ ++&vcc3v3_lcd1_n { ++ status = "disabled"; ++ gpio = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; ++ enable-active-high; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-evb6-ddr3-v10.dts b/arch/arm64/boot/dts/rockchip/rk3568-evb6-ddr3-v10.dts +new file mode 100755 +index 000000000000..6e34fa48abc4 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-evb6-ddr3-v10.dts +@@ -0,0 +1,8 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++#include "rk3568-evb6-ddr3-v10.dtsi" ++#include "rk3568-android.dtsi" +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-evb6-ddr3-v10.dtsi b/arch/arm64/boot/dts/rockchip/rk3568-evb6-ddr3-v10.dtsi +new file mode 100755 +index 000000000000..2f3b5d74e1cf +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-evb6-ddr3-v10.dtsi +@@ -0,0 +1,490 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++/dts-v1/; ++ ++#include "rk3568.dtsi" ++#include "rk3568-evb.dtsi" ++ ++/ { ++ model = "Rockchip RK3568 EVB6 DDR3 V10 Board"; ++ compatible = "rockchip,rk3568-evb6-ddr3-v10", "rockchip,rk3568"; ++ ++ pcie20_3v3: gpio-regulator { ++ compatible = "regulator-gpio"; ++ regulator-name = "pcie20_3v3"; ++ regulator-min-microvolt = <100000>; ++ regulator-max-microvolt = <3300000>; ++ gpios = <&gpio0 RK_PD4 GPIO_ACTIVE_HIGH>; ++ gpios-states = <0x1>; ++ states = <100000 0x0 ++ 3300000 0x1>; ++ }; ++ ++ pcie30_avdd0v9: pcie30-avdd0v9 { ++ compatible = "regulator-fixed"; ++ regulator-name = "pcie30_avdd0v9"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <900000>; ++ vin-supply = <&vcc3v3_sys>; ++ }; ++ ++ pcie30_avdd1v8: pcie30-avdd1v8 { ++ compatible = "regulator-fixed"; ++ regulator-name = "pcie30_avdd1v8"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ vin-supply = <&vcc3v3_sys>; ++ }; ++ ++ pcie30_3v3: gpio-regulator { ++ compatible = "regulator-gpio"; ++ regulator-name = "pcie30_3v3"; ++ regulator-min-microvolt = <0100000>; ++ regulator-max-microvolt = <3300000>; ++ gpios = <&gpio0 RK_PD4 GPIO_ACTIVE_HIGH>; ++ gpios-states = <0x1>; ++ states = <0100000 0x0 ++ 3300000 0x1>; ++ }; ++ ++ rk_headset: rk-headset { ++ compatible = "rockchip_headset"; ++ headset_gpio = <&gpio3 RK_PC2 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hp_det>; ++ }; ++ ++ vcc_camera: vcc-camera-regulator { ++ compatible = "regulator-fixed"; ++ gpio = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&camera_pwr>; ++ regulator-name = "vcc_camera"; ++ enable-active-high; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++}; ++ ++&combphy0_us { ++ status = "okay"; ++}; ++ ++&combphy1_usq { ++ rockchip,dis-u3otg1-port; ++ status = "okay"; ++}; ++ ++&combphy2_psq { ++ status = "okay"; ++}; ++ ++/* ++ * video_phy0 needs to be enabled ++ * when dsi0 is enabled ++ */ ++&dsi0 { ++ status = "okay"; ++}; ++ ++&dsi0_in_vp0 { ++ status = "disabled"; ++}; ++ ++&dsi0_in_vp1 { ++ status = "okay"; ++}; ++ ++&dsi0_panel { ++ power-supply = <&vcc3v3_lcd0_n>; ++}; ++ ++/* ++ * video_phy1 needs to be enabled ++ * when dsi1 is enabled ++ */ ++&dsi1 { ++ status = "disabled"; ++}; ++ ++&dsi1_in_vp0 { ++ status = "disabled"; ++}; ++ ++&dsi1_in_vp1 { ++ status = "disabled"; ++}; ++ ++&dsi1_panel { ++ power-supply = <&vcc3v3_lcd1_n>; ++}; ++ ++/* ++ * power-supply should switche to vcc3v3_lcd1_n ++ * when mipi panel is connected to dsi1. ++ */ ++>1x { ++ power-supply = <&vcc3v3_lcd0_n>; ++}; ++ ++&i2c2 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2c2m1_xfer>; ++ ++ mxc6655xa: mxc6655xa@15 { ++ status = "okay"; ++ compatible = "gs_mxc6655xa"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&mxc6655xa_irq_gpio>; ++ reg = <0x15>; ++ irq-gpio = <&gpio3 RK_PC1 IRQ_TYPE_LEVEL_LOW>; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ type = ; ++ power-off-in-suspend = <1>; ++ layout = <4>; ++ }; ++}; ++ ++&i2c4 { ++ status = "okay"; ++ os04a10: os04a10@36 { ++ compatible = "ovti,os04a10"; ++ reg = <0x36>; ++ clocks = <&cru CLK_CAM0_OUT>; ++ clock-names = "xvclk"; ++ power-domains = <&power RK3568_PD_VI>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cam_clkout0>; ++ reset-gpios = <&gpio2 RK_PD5 GPIO_ACTIVE_LOW>; ++ pwdn-gpios = <&gpio4 RK_PA6 GPIO_ACTIVE_HIGH>; ++ /* power-gpios = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; */ ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "CMK-OT1607-FV1"; ++ /* rockchip,camera-module-lens-name = "M12-4IR-4MP-F16"; */ ++ rockchip,camera-module-lens-name = "M12-40IRC-4MP-F16"; ++ port { ++ ucam_out0: endpoint { ++ remote-endpoint = <&mipi_in_ucam0>; ++ data-lanes = <1 2 3 4>; ++ }; ++ }; ++ }; ++ gc8034: gc8034@37 { ++ compatible = "galaxycore,gc8034"; ++ reg = <0x37>; ++ clocks = <&cru CLK_CAM0_OUT>; ++ clock-names = "xvclk"; ++ power-domains = <&power RK3568_PD_VI>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cam_clkout0>; ++ reset-gpios = <&gpio2 RK_PD5 GPIO_ACTIVE_LOW>; ++ pwdn-gpios = <&gpio4 RK_PA6 GPIO_ACTIVE_LOW>; ++ rockchip,grf = <&grf>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "RK-CMK-8M-2-v1"; ++ rockchip,camera-module-lens-name = "CK8401"; ++ port { ++ gc8034_out: endpoint { ++ remote-endpoint = <&mipi_in_ucam1>; ++ data-lanes = <1 2 3 4>; ++ }; ++ }; ++ }; ++ ov5695: ov5695@36 { ++ status = "okay"; ++ compatible = "ovti,ov5695"; ++ reg = <0x36>; ++ clocks = <&cru CLK_CAM0_OUT>; ++ clock-names = "xvclk"; ++ power-domains = <&power RK3568_PD_VI>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cam_clkout0>; ++ reset-gpios = <&gpio2 RK_PD5 GPIO_ACTIVE_LOW>; ++ pwdn-gpios = <&gpio4 RK_PA6 GPIO_ACTIVE_HIGH>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "TongJu"; ++ rockchip,camera-module-lens-name = "CHT842-MD"; ++ port { ++ ov5695_out: endpoint { ++ remote-endpoint = <&mipi_in_ucam2>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++}; ++ ++&i2c5 { ++ status = "disabled"; ++ ++ /delete-node/ mxc6655xa@15; ++}; ++ ++&csi2_dphy_hw { ++ status = "okay"; ++}; ++ ++&csi2_dphy0 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam0: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ucam_out0>; ++ data-lanes = <1 2 3 4>; ++ }; ++ mipi_in_ucam1: endpoint@2 { ++ reg = <2>; ++ remote-endpoint = <&gc8034_out>; ++ data-lanes = <1 2 3 4>; ++ }; ++ mipi_in_ucam2: endpoint@3 { ++ reg = <3>; ++ remote-endpoint = <&ov5695_out>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ csidphy_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp0_in>; ++ }; ++ }; ++ }; ++}; ++ ++&rkisp { ++ status = "okay"; ++}; ++ ++&rkisp_mmu { ++ status = "okay"; ++}; ++ ++&rkisp_vir0 { ++ status = "okay"; ++ ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp0_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&csidphy_out>; ++ }; ++ }; ++}; ++ ++&video_phy0 { ++ status = "okay"; ++}; ++ ++&video_phy1 { ++ status = "disabled"; ++}; ++ ++&pcie30phy { ++ status = "okay"; ++}; ++ ++&pcie2x1 { ++ reset-gpios = <&gpio1 RK_PB2 GPIO_ACTIVE_HIGH>; ++ vpcie3v3-supply = <&pcie20_3v3>; ++ status = "okay"; ++}; ++ ++&pcie3x1 { ++ rockchip,bifurcation; ++ reset-gpios = <&gpio3 RK_PA1 GPIO_ACTIVE_HIGH>; ++ vpcie3v3-supply = <&pcie30_3v3>; ++ status = "okay"; ++}; ++ ++&pcie3x2 { ++ rockchip,bifurcation; ++ reset-gpios = <&gpio2 RK_PD6 GPIO_ACTIVE_HIGH>; ++ vpcie3v3-supply = <&pcie30_3v3>; ++ status = "okay"; ++}; ++ ++&pinctrl { ++ cam { ++ camera_pwr: camera-pwr { ++ rockchip,pins = ++ /* camera power en */ ++ <0 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ headphone { ++ hp_det: hp-det { ++ rockchip,pins = <3 RK_PC2 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <2 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ wifi_32k: wifi-32k { ++ rockchip,pins = <2 RK_PC6 1 &pcfg_pull_none>; ++ }; ++ }; ++ ++ wireless-wlan { ++ wifi_host_wake_irq: wifi-host-wake-irq { ++ rockchip,pins = <2 RK_PB2 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart1_gpios: uart1-gpios { ++ rockchip,pins = <2 RK_PB5 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++&route_dsi0 { ++ status = "okay"; ++ connect = <&vp1_out_dsi0>; ++}; ++ ++&sdmmc1 { ++ max-frequency = <150000000>; ++ no-sd; ++ no-mmc; ++ bus-width = <4>; ++ disable-wp; ++ cap-sd-highspeed; ++ cap-sdio-irq; ++ keep-power-in-suspend; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ non-removable; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc1_bus4 &sdmmc1_cmd &sdmmc1_clk>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&sdmmc2 { ++ status = "disabled"; ++}; ++ ++&sdio_pwrseq { ++ clocks = <&pmucru CLK_RTC_32K>; ++ pinctrl-0 = <&wifi_enable_h &wifi_32k>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio2 RK_PB1 GPIO_ACTIVE_LOW>; ++}; ++ ++&spdif_8ch { ++ status = "disabled"; ++}; ++ ++&uart1 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart1m0_xfer &uart1m0_ctsn>; ++}; ++ ++&usbhost_dwc3 { ++ phys = <&u2phy0_host>; ++ phy-names = "usb2-phy"; ++ maximum-speed = "high-speed"; ++ status = "okay"; ++}; ++ ++&vcc3v3_lcd0_n { ++ gpio = <&gpio0 RK_PC7 GPIO_ACTIVE_HIGH>; ++ enable-active-high; ++}; ++ ++&vcc3v3_lcd1_n { ++ gpio = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; ++ enable-active-high; ++}; ++ ++&wireless_wlan { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_host_wake_irq>; ++ WIFI,host_wake_irq = <&gpio2 RK_PB2 GPIO_ACTIVE_HIGH>; ++}; ++ ++&wireless_bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&pmucru CLK_RTC_32K>; ++ clock-names = "ext_clock"; ++ //wifi-bt-power-toggle; ++ uart_rts_gpios = <&gpio2 RK_PB5 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart1m0_rtsn>; ++ pinctrl-1 = <&uart1_gpios>; ++ BT,reset_gpio = <&gpio2 RK_PB7 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio2 RK_PC1 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio2 RK_PC0 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++}; ++ ++&gmac1 { ++ phy-mode = "rgmii"; ++ clock_in_out = "output"; ++ ++ snps,reset-gpio = <&gpio2 RK_PD1 GPIO_ACTIVE_LOW>; ++ snps,reset-active-low; ++ /* Reset time is 20ms, 100ms for rtl8211f */ ++ snps,reset-delays-us = <0 20000 100000>; ++ ++ assigned-clocks = <&cru SCLK_GMAC1_RX_TX>, <&cru SCLK_GMAC1>; ++ assigned-clock-parents = <&cru SCLK_GMAC1_RGMII_SPEED>, <&cru CLK_MAC1_2TOP>; ++ assigned-clock-rates = <0>, <125000000>; ++ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&gmac1m0_miim ++ &gmac1m0_tx_bus2_level3 ++ &gmac1m0_rx_bus2 ++ &gmac1m0_rgmii_clk_level2 ++ &gmac1m0_rgmii_bus_level3>; ++ ++ tx_delay = <0x46>; ++ rx_delay = <0x2f>; ++ ++ phy-handle = <&rgmii_phy1>; ++ status = "okay"; ++}; ++ ++&mdio1 { ++ rgmii_phy1: phy@0 { ++ compatible = "ethernet-phy-ieee802.3-c22"; ++ reg = <0x0>; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-evb7-ddr4-v10.dts b/arch/arm64/boot/dts/rockchip/rk3568-evb7-ddr4-v10.dts +new file mode 100755 +index 000000000000..ecb96832c292 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-evb7-ddr4-v10.dts +@@ -0,0 +1,12 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++#include "rk3568-evb1-ddr4-v10.dtsi" ++#include "rk3568-android.dtsi" ++/{ ++ model = "Rockchip RK3568 EVB7 DDR4 V10 Board"; ++ compatible = "rockchip,rk3568-evb7-ddr4-v10", "rockchip,rk3568"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-iotest-ddr3-v10-linux.dts b/arch/arm64/boot/dts/rockchip/rk3568-iotest-ddr3-v10-linux.dts +new file mode 100755 +index 000000000000..d832a40d37e9 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-iotest-ddr3-v10-linux.dts +@@ -0,0 +1,69 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++/dts-v1/; ++ ++#include "rk3568.dtsi" ++#include ++ ++/ { ++ model = "Rockchip RK3568 IOTEST DDR3 V10 Board"; ++ compatible = "rockchip,rk3568-iotest-ddr3-v10", "rockchip,rk3568"; ++ ++ chosen: chosen { ++ bootargs = "earlycon=uart8250,mmio32,0xfe660000 console=ttyFIQ0 root=PARTUUID=614e0000-0000 rw rootwait"; ++ }; ++ ++ fiq-debugger { ++ compatible = "rockchip,fiq-debugger"; ++ rockchip,serial-id = <2>; ++ rockchip,wake-irq = <0>; ++ /* If enable uart uses irq instead of fiq */ ++ rockchip,irq-mode-enable = <1>; ++ rockchip,baudrate = <1500000>; /* Only 115200 and 1500000 */ ++ interrupts = ; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart2m0_xfer>; ++ status = "okay"; ++ }; ++ ++ test-power { ++ status = "okay"; ++ }; ++}; ++ ++&sdhci { ++ bus-width = <8>; ++ no-sdio; ++ no-sd; ++ non-removable; ++ max-frequency = <200000000>; ++ status = "okay"; ++}; ++ ++&u2phy0_otg { ++ status = "okay"; ++}; ++ ++&usb2phy0 { ++ status = "okay"; ++}; ++ ++&usbdrd_dwc3 { ++ dr_mode = "otg"; ++ phys = <&u2phy0_otg>; ++ phy-names = "usb2-phy"; ++ extcon = <&usb2phy0>; ++ maximum-speed = "high-speed"; ++ snps,dis_u2_susphy_quirk; ++ status = "okay"; ++}; ++ ++&usbdrd30 { ++ status = "okay"; ++}; ++ ++/delete-node/ &display_subsystem; +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-iotest-ddr3-v10.dts b/arch/arm64/boot/dts/rockchip/rk3568-iotest-ddr3-v10.dts +new file mode 100755 +index 000000000000..489967f1318f +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-iotest-ddr3-v10.dts +@@ -0,0 +1,47 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++/dts-v1/; ++ ++#include "rk3568.dtsi" ++#include "rk3568-evb.dtsi" ++#include "rk3568-android.dtsi" ++ ++/ { ++ model = "Rockchip RK3568 IOTEST DDR3 V10 Board"; ++ compatible = "rockchip,rk3568-iotest-ddr3-v10", "rockchip,rk3568"; ++}; ++ ++&usb_host0_ehci { ++ status = "disabled"; ++}; ++ ++&usb_host0_ohci { ++ status = "disabled"; ++}; ++ ++&usb_host1_ehci { ++ status = "disabled"; ++}; ++ ++&usb_host1_ohci { ++ status = "disabled"; ++}; ++ ++&usbdrd_dwc3 { ++ phys = <&u2phy0_otg>; ++ phy-names = "usb2-phy"; ++ maximum-speed = "high-speed"; ++ snps,dis_u2_susphy_quirk; ++ status = "okay"; ++}; ++ ++&usbhost_dwc3 { ++ phys = <&u2phy1_otg>; ++ phy-names = "usb2-phy"; ++ maximum-speed = "high-speed"; ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-linux.dtsi b/arch/arm64/boot/dts/rockchip/rk3568-linux.dtsi +new file mode 100755 +index 000000000000..d869144aa806 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-linux.dtsi +@@ -0,0 +1,59 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++/ { ++ chosen: chosen { ++ bootargs = "earlycon=uart8250,mmio32,0xfe660000 console=ttyFIQ0 initcall_debug=y loglevel=15 root=PARTUUID=614e0000-0000 rw rootwait"; ++ }; ++ ++ fiq-debugger { ++ compatible = "rockchip,fiq-debugger"; ++ rockchip,serial-id = <2>; ++ rockchip,wake-irq = <0>; ++ /* If enable uart uses irq instead of fiq */ ++ rockchip,irq-mode-enable = <1>; ++ rockchip,baudrate = <1500000>; /* Only 115200 and 1500000 */ ++ interrupts = ; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart2m0_xfer>; ++ status = "okay"; ++ }; ++ ++ debug: debug@fd904000 { ++ compatible = "rockchip,debug"; ++ reg = <0x0 0xfd904000 0x0 0x1000>, ++ <0x0 0xfd905000 0x0 0x1000>, ++ <0x0 0xfd906000 0x0 0x1000>, ++ <0x0 0xfd907000 0x0 0x1000>; ++ }; ++ ++ cspmu: cspmu@fd90c000 { ++ compatible = "rockchip,cspmu"; ++ reg = <0x0 0xfd90c000 0x0 0x1000>, ++ <0x0 0xfd90d000 0x0 0x1000>, ++ <0x0 0xfd90e000 0x0 0x1000>, ++ <0x0 0xfd90f000 0x0 0x1000>; ++ }; ++}; ++ ++&reserved_memory { ++ ramoops: ramoops@110000 { ++ compatible = "ramoops"; ++ reg = <0x0 0x110000 0x0 0xf0000>; ++ record-size = <0x20000>; ++ console-size = <0x80000>; ++ ftrace-size = <0x00000>; ++ pmsg-size = <0x50000>; ++ }; ++}; ++ ++&rng { ++ status = "okay"; ++}; ++ ++&vop { ++ disable-afbc-win; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v10-linux-spi-nand.dts b/arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v10-linux-spi-nand.dts +new file mode 100755 +index 000000000000..50bd024528df +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v10-linux-spi-nand.dts +@@ -0,0 +1,31 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2021 Rockchip Electronics Co., Ltd. ++ */ ++ ++/dts-v1/; ++ ++#include "rk3568-nvr-demo-v10.dtsi" ++#include "rk3568-nvr-linux.dtsi" ++ ++/ { ++ model = "Rockchip RK3568 NVR DEMO DDR4 V10 Linux SPI NAND Board"; ++ compatible = "rockchip,rk3568-nvr-demo-ddr4-v10-linux-spi-nand", "rockchip,rk3568"; ++ ++ chosen { ++ bootargs = "earlycon=uart8250,mmio32,0xfe660000 console=ttyFIQ0 ubi.mtd=3 root=ubi0:rootfs rootfstype=ubifs"; ++ }; ++ ++}; ++ ++&pcie30phy { ++ status = "disabled"; ++}; ++ ++&pcie3x1 { ++ status = "disabled"; ++}; ++ ++&pcie3x2 { ++ status = "disabled"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v10-linux.dts b/arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v10-linux.dts +new file mode 100755 +index 000000000000..3317db6ee671 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v10-linux.dts +@@ -0,0 +1,27 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++/dts-v1/; ++ ++#include "rk3568-nvr-demo-v10.dtsi" ++#include "rk3568-nvr-linux.dtsi" ++ ++/ { ++ model = "Rockchip RK3568 NVR DEMO DDR4 V10 Linux Board"; ++ compatible = "rockchip,rk3568-nvr-demo-ddr4-v10-linux", "rockchip,rk3568"; ++}; ++ ++&pcie30phy { ++ status = "disabled"; ++}; ++ ++&pcie3x1 { ++ status = "disabled"; ++}; ++ ++&pcie3x2 { ++ status = "disabled"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v10.dts b/arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v10.dts +new file mode 100755 +index 000000000000..053d20259e93 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v10.dts +@@ -0,0 +1,27 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++/dts-v1/; ++ ++#include "rk3568-nvr-demo-v10.dtsi" ++#include "rk3568-android.dtsi" ++ ++/ { ++ model = "Rockchip RK3568 NVR DEMO DDR4 V10 ANDROID Board"; ++ compatible = "rockchip,rk3568-nvr-demo-ddr4-v10", "rockchip,rk3568"; ++}; ++ ++&pcie30phy { ++ status = "disabled"; ++}; ++ ++&pcie3x1 { ++ status = "disabled"; ++}; ++ ++&pcie3x2 { ++ status = "disabled"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v10.dtsi b/arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v10.dtsi +new file mode 100755 +index 000000000000..40a2096bc58b +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v10.dtsi +@@ -0,0 +1,442 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++/dts-v1/; ++ ++#include "rk3568-nvr.dtsi" ++#include ++ ++/ { ++ model = "Rockchip RK3568 NVR DEMO V10 Board"; ++ compatible = "rockchip,rk3568-nvr-demo-v10", "rockchip,rk3568"; ++ ++ gpio-leds { ++ compatible = "gpio-leds"; ++ ++ hdd-led { ++ gpios = <&gpio4 RK_PC5 GPIO_ACTIVE_HIGH>; ++ default-state = "off"; ++ }; ++ net-led { ++ gpios = <&gpio4 RK_PC6 GPIO_ACTIVE_HIGH>; ++ default-state = "off"; ++ }; ++ work-led { ++ gpios = <&gpio0 RK_PC4 GPIO_ACTIVE_HIGH>; ++ linux,default-trigger = "timer"; ++ }; ++ }; ++ ++ i2s1_sound: i2s1-sound { ++ status = "okay"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,name = "rockchip,i2s1-sound"; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s1_8ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&es8311>; ++ }; ++ }; ++ ++ vcc2v5_sys: vcc2v5-ddr { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc2v5-sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <2500000>; ++ regulator-max-microvolt = <2500000>; ++ vin-supply = <&vcc3v3_sys>; ++ }; ++ ++ pcie30_3v3: gpio-regulator { ++ compatible = "regulator-gpio"; ++ regulator-name = "pcie30_3v3"; ++ regulator-min-microvolt = <100000>; ++ regulator-max-microvolt = <3300000>; ++ gpios = <&gpio0 RK_PD4 GPIO_ACTIVE_HIGH>; ++ gpios-states = <0x1>; ++ states = <100000 0x0 ++ 3300000 0x1>; ++ }; ++ ++ pcie30_avdd0v9: pcie30-avdd0v9 { ++ compatible = "regulator-fixed"; ++ regulator-name = "pcie30_avdd0v9"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <900000>; ++ vin-supply = <&vcc3v3_sys>; ++ }; ++ ++ pcie30_avdd1v8: pcie30-avdd1v8 { ++ compatible = "regulator-fixed"; ++ regulator-name = "pcie30_avdd1v8"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ vin-supply = <&vcc3v3_sys>; ++ }; ++ ++ vcc3v3_bu: vcc3v3-bu { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc3v3_bu"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ vin-supply = <&vcc5v0_sys>; ++ }; ++}; ++ ++&combphy1_usq { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sata_pm_reset>; ++ rockchip,dis-u3otg1-port; ++ status = "okay"; ++}; ++ ++&combphy2_psq{ ++ status = "okay"; ++}; ++ ++&gmac0 { ++ phy-mode = "rgmii"; ++ clock_in_out = "output"; ++ ++ snps,reset-gpio = <&gpio1 RK_PB0 GPIO_ACTIVE_LOW>; ++ snps,reset-active-low; ++ /* Reset time is 20ms, 100ms for rtl8211f */ ++ snps,reset-delays-us = <0 20000 100000>; ++ ++ assigned-clocks = <&cru SCLK_GMAC0_RX_TX>, <&cru SCLK_GMAC0>; ++ assigned-clock-parents = <&cru SCLK_GMAC0_RGMII_SPEED>, <&cru CLK_MAC0_2TOP>; ++ assigned-clock-rates = <0>, <125000000>; ++ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&gmac0_miim ++ &gmac0_tx_bus2 ++ &gmac0_rx_bus2 ++ &gmac0_rgmii_clk ++ &gmac0_rgmii_bus>; ++ ++ tx_delay = <0x43>; ++ rx_delay = <0x33>; ++ ++ phy-handle = <&rgmii_phy0>; ++ status = "okay"; ++}; ++ ++&gmac1 { ++ phy-mode = "rgmii"; ++ clock_in_out = "output"; ++ ++ snps,reset-gpio = <&gpio1 RK_PB1 GPIO_ACTIVE_LOW>; ++ snps,reset-active-low; ++ /* Reset time is 20ms, 100ms for rtl8211f */ ++ snps,reset-delays-us = <0 20000 100000>; ++ ++ assigned-clocks = <&cru SCLK_GMAC1_RX_TX>, <&cru SCLK_GMAC1>; ++ assigned-clock-parents = <&cru SCLK_GMAC1_RGMII_SPEED>, <&cru CLK_MAC1_2TOP>; ++ assigned-clock-rates = <0>, <125000000>; ++ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&gmac1m1_miim ++ &gmac1m1_tx_bus2 ++ &gmac1m1_rx_bus2 ++ &gmac1m1_rgmii_clk ++ &gmac1m1_rgmii_bus>; ++ ++ tx_delay = <0x4f>; ++ rx_delay = <0x2d>; ++ ++ phy-handle = <&rgmii_phy1>; ++ status = "okay"; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ hym8563: hym8563@51 { ++ compatible = "haoyu,hym8563"; ++ reg = <0x51>; ++ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&rtc_int>; ++ ++ interrupt-parent = <&gpio0>; ++ interrupts = ; ++ }; ++}; ++ ++&i2c3 { ++ status = "okay"; ++ clock-frequency = <400000>; ++ ++ es8311: es8311@18 { ++ compatible = "everest,es8311"; ++ reg = <0x18>; ++ clocks = <&cru I2S1_MCLKOUT>; ++ clock-names = "mclk"; ++ adc-pga-gain = <6>; /* 18dB */ ++ adc-volume = <0xbf>; /* 0dB */ ++ dac-volume = <0xbf>; /* 0dB */ ++ aec-mode = "dac left, adc right"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1m0_mclk>; ++ assigned-clocks = <&cru I2S1_MCLKOUT>, <&cru I2S1_MCLK_TX_IOE>; ++ assigned-clock-rates = <12288000>; ++ assigned-clock-parents = <&cru I2S1_MCLKOUT_TX>, <&cru I2S1_MCLKOUT_TX>; ++ spk-ctl-gpios = <&gpio2 RK_PA5 GPIO_ACTIVE_HIGH>; ++ #sound-dai-cells = <0>; ++ }; ++ ++ rk618@50 { ++ compatible = "rockchip,rk618"; ++ reg = <0x50>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s3m1_mclk &rk618_int>; ++ clocks = <&cru I2S3_MCLKOUT>; ++ clock-names = "clkin"; ++ assigned-clocks =<&cru I2S3_MCLKOUT>, <&cru I2S3_MCLK_IOE>; ++ assigned-clock-parents = <&cru I2S3_MCLKOUT_TX>, <&cru I2S3_MCLKOUT>; ++ assigned-clock-rates = <11289600>; ++ reset-gpios = <&gpio1 RK_PB2 GPIO_ACTIVE_LOW>; ++ status = "okay"; ++ ++ clock: cru { ++ compatible = "rockchip,rk618-cru"; ++ clocks = <&cru I2S3_MCLKOUT>, <&cru DCLK_VOP2>; ++ clock-names = "clkin", "lcdc0_dclkp"; ++ assigned-clocks = <&clock SCALER_PLLIN_CLK>, ++ <&clock VIF_PLLIN_CLK>, ++ <&clock SCALER_CLK>, ++ <&clock VIF0_PRE_CLK>, ++ <&clock CODEC_CLK>, ++ <&clock DITHER_CLK>; ++ assigned-clock-parents = <&cru I2S3_MCLKOUT_TX>, ++ <&clock LCDC0_CLK>, ++ <&clock SCALER_PLL_CLK>, ++ <&clock VIF_PLL_CLK>, ++ <&cru I2S3_MCLKOUT>, ++ <&clock VIF0_CLK>; ++ #clock-cells = <1>; ++ status = "okay"; ++ }; ++ ++ hdmi { ++ compatible = "rockchip,rk618-hdmi"; ++ clocks = <&clock HDMI_CLK>; ++ clock-names = "hdmi"; ++ assigned-clocks = <&clock HDMI_CLK>; ++ assigned-clock-parents = <&clock VIF0_CLK>; ++ interrupt-parent = <&gpio0>; ++ interrupts = <8 IRQ_TYPE_LEVEL_HIGH>; ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ ++ hdmi_in_rgb: endpoint { ++ remote-endpoint = <&rgb_out_hdmi>; ++ }; ++ }; ++ }; ++ }; ++ }; ++}; ++ ++&mdio0 { ++ rgmii_phy0: phy@0 { ++ compatible = "ethernet-phy-ieee802.3-c22"; ++ reg = <0x0>; ++ }; ++}; ++ ++&mdio1 { ++ rgmii_phy1: phy@0 { ++ compatible = "ethernet-phy-ieee802.3-c22"; ++ reg = <0x0>; ++ }; ++}; ++ ++&pcie30phy { ++ status = "okay"; ++}; ++ ++&pcie3x1 { ++ reset-gpios = <&gpio0 RK_PC3 GPIO_ACTIVE_HIGH>; ++ vpcie3v3-supply = <&pcie30_3v3>; ++ status = "okay"; ++}; ++ ++&pcie3x2 { ++ reset-gpios = <&gpio0 RK_PC6 GPIO_ACTIVE_HIGH>; ++ vpcie3v3-supply = <&pcie30_3v3>; ++ status = "okay"; ++}; ++ ++&pwm15 { ++ compatible = "rockchip,remotectl-pwm"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pwm15m1_pins>; ++ remote_pwm_id = <3>; ++ handle_cpu_id = <1>; ++ remote_support_psci = <0>; ++ status = "okay"; ++ ++ ir_key1 { ++ rockchip,usercode = <0x4040>; ++ rockchip,key_table = ++ <0xf2 KEY_REPLY>, ++ <0xba KEY_BACK>, ++ <0xf4 KEY_UP>, ++ <0xf1 KEY_DOWN>, ++ <0xef KEY_LEFT>, ++ <0xee KEY_RIGHT>, ++ <0xbd KEY_HOME>, ++ <0xea KEY_VOLUMEUP>, ++ <0xe3 KEY_VOLUMEDOWN>, ++ <0xe2 KEY_SEARCH>, ++ <0xb2 KEY_POWER>, ++ <0xbc KEY_MUTE>, ++ <0xec KEY_MENU>, ++ <0xbf 0x190>, ++ <0xe0 0x191>, ++ <0xe1 0x192>, ++ <0xe9 183>, ++ <0xe6 248>, ++ <0xe8 185>, ++ <0xe7 186>, ++ <0xf0 388>, ++ <0xbe 0x175>; ++ }; ++ ++ ir_key2 { ++ rockchip,usercode = <0xff00>; ++ rockchip,key_table = ++ <0xf9 KEY_HOME>, ++ <0xbf KEY_BACK>, ++ <0xfb KEY_MENU>, ++ <0xaa KEY_REPLY>, ++ <0xb9 KEY_UP>, ++ <0xe9 KEY_DOWN>, ++ <0xb8 KEY_LEFT>, ++ <0xea KEY_RIGHT>, ++ <0xeb KEY_VOLUMEDOWN>, ++ <0xef KEY_VOLUMEUP>, ++ <0xf7 KEY_MUTE>, ++ <0xe7 KEY_POWER>, ++ <0xfc KEY_POWER>, ++ <0xa9 KEY_VOLUMEDOWN>, ++ <0xa8 KEY_PLAYPAUSE>, ++ <0xe0 KEY_VOLUMEDOWN>, ++ <0xa5 KEY_VOLUMEDOWN>, ++ <0xab 183>, ++ <0xb7 388>, ++ <0xe8 388>, ++ <0xf8 184>, ++ <0xaf 185>, ++ <0xed KEY_VOLUMEDOWN>, ++ <0xee 186>, ++ <0xb3 KEY_VOLUMEDOWN>, ++ <0xf1 KEY_VOLUMEDOWN>, ++ <0xf2 KEY_VOLUMEDOWN>, ++ <0xf3 KEY_SEARCH>, ++ <0xb4 KEY_VOLUMEDOWN>, ++ <0xa4 KEY_SETUP>, ++ <0xbe KEY_SEARCH>; ++ }; ++ ++ ir_key3 { ++ rockchip,usercode = <0x1dcc>; ++ rockchip,key_table = ++ <0xee KEY_REPLY>, ++ <0xf0 KEY_BACK>, ++ <0xf8 KEY_UP>, ++ <0xbb KEY_DOWN>, ++ <0xef KEY_LEFT>, ++ <0xed KEY_RIGHT>, ++ <0xfc KEY_HOME>, ++ <0xf1 KEY_VOLUMEUP>, ++ <0xfd KEY_VOLUMEDOWN>, ++ <0xb7 KEY_SEARCH>, ++ <0xff KEY_POWER>, ++ <0xf3 KEY_MUTE>, ++ <0xbf KEY_MENU>, ++ <0xf9 0x191>, ++ <0xf5 0x192>, ++ <0xb3 388>, ++ <0xbe KEY_1>, ++ <0xba KEY_2>, ++ <0xb2 KEY_3>, ++ <0xbd KEY_4>, ++ <0xf9 KEY_5>, ++ <0xb1 KEY_6>, ++ <0xfc KEY_7>, ++ <0xf8 KEY_8>, ++ <0xb0 KEY_9>, ++ <0xb6 KEY_0>, ++ <0xb5 KEY_BACKSPACE>; ++ }; ++}; ++ ++&rgb { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&lcdc_ctl>; ++ ports { ++ port@1 { ++ reg = <1>; ++ ++ rgb_out_hdmi: endpoint { ++ remote-endpoint = <&hdmi_in_rgb>; ++ }; ++ }; ++ }; ++}; ++ ++&rgb_in_vp2 { ++ status = "okay"; ++}; ++ ++&sata1 { ++ status = "okay"; ++}; ++ ++&sata2 { ++ status = "okay"; ++}; ++ ++&pinctrl { ++ rk618 { ++ rk618_reset: rk618-reeset { ++ rockchip,pins = <1 RK_PB2 RK_FUNC_GPIO &pcfg_output_high>; ++ }; ++ rk618_int: rk618-int { ++ rockchip,pins = <0 RK_PB0 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ rtc { ++ rtc_int: rtc-int { ++ rockchip,pins = <0 RK_PD3 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ sata { ++ sata_pm_reset: sata-pm-reset { ++ rockchip,pins = <4 RK_PD2 RK_FUNC_GPIO &pcfg_output_high>; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v12-linux-spi-nand.dts b/arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v12-linux-spi-nand.dts +new file mode 100755 +index 000000000000..de8c1a1e734f +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v12-linux-spi-nand.dts +@@ -0,0 +1,20 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2021 Rockchip Electronics Co., Ltd. ++ */ ++ ++/dts-v1/; ++ ++#include "rk3568-nvr-demo-v12.dtsi" ++#include "rk3568-nvr-linux.dtsi" ++ ++/ { ++ model = "Rockchip RK3568 NVR DEMO DDR4 V12 Linux SPI NAND Board"; ++ compatible = "rockchip,rk3568-nvr-demo-ddr4-v12-linux-spi-nand", "rockchip,rk3568"; ++ ++ chosen { ++ bootargs = "earlycon=uart8250,mmio32,0xfe660000 console=ttyFIQ0 ubi.mtd=3 root=ubi0:rootfs rootfstype=ubifs"; ++ }; ++ ++}; ++ +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v12-linux.dts b/arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v12-linux.dts +new file mode 100755 +index 000000000000..b605c3d57d26 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v12-linux.dts +@@ -0,0 +1,16 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2021 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++/dts-v1/; ++ ++#include "rk3568-nvr-demo-v12.dtsi" ++#include "rk3568-nvr-linux.dtsi" ++ ++/ { ++ model = "Rockchip RK3568 NVR DEMO DDR4 V12 Linux Board"; ++ compatible = "rockchip,rk3568-nvr-demo-ddr4-v12-linux", "rockchip,rk3568"; ++}; ++ +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v12.dtsi b/arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v12.dtsi +new file mode 100755 +index 000000000000..6bb50368a73b +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v12.dtsi +@@ -0,0 +1,39 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2021 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++/dts-v1/; ++ ++#include "rk3568-nvr-demo-v10.dtsi" ++ ++/ { ++ model = "Rockchip RK3568 NVR DEMO V12 Board"; ++ compatible = "rockchip,rk3568-nvr-demo-v12", "rockchip,rk3568"; ++}; ++ ++&gmac0 { ++ tx_delay = <0x35>; ++ rx_delay = <0x2d>; ++}; ++ ++&gmac1 { ++ tx_delay = <0x43>; ++ rx_delay = <0x27>; ++}; ++ ++&gpu { ++ mali-supply = <&vdd_npu>; ++ status = "okay"; ++}; ++ ++&vdd_logic { ++ regulator-min-microvolt = <810000>; ++ regulator-max-microvolt = <1000000>; ++}; ++ ++&vdd_npu { ++ regulator-min-microvolt = <810000>; ++ regulator-max-microvolt = <1100000>; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-nvr-linux.dtsi b/arch/arm64/boot/dts/rockchip/rk3568-nvr-linux.dtsi +new file mode 100755 +index 000000000000..f9908b61cb41 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-nvr-linux.dtsi +@@ -0,0 +1,55 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2021 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++/ { ++ chosen: chosen { ++ bootargs = "earlycon=uart8250,mmio32,0xfe660000 console=ttyFIQ0 root=PARTUUID=614e0000-0000 rw rootwait"; ++ }; ++ ++ fiq-debugger { ++ compatible = "rockchip,fiq-debugger"; ++ rockchip,serial-id = <2>; ++ rockchip,wake-irq = <0>; ++ /* If enable uart uses irq instead of fiq */ ++ rockchip,irq-mode-enable = <1>; ++ rockchip,baudrate = <1500000>; /* Only 115200 and 1500000 */ ++ interrupts = ; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart2m0_xfer>; ++ status = "okay"; ++ }; ++ ++ debug: debug@fd904000 { ++ compatible = "rockchip,debug"; ++ reg = <0x0 0xfd904000 0x0 0x1000>, ++ <0x0 0xfd905000 0x0 0x1000>, ++ <0x0 0xfd906000 0x0 0x1000>, ++ <0x0 0xfd907000 0x0 0x1000>; ++ }; ++ ++ cspmu: cspmu@fd90c000 { ++ compatible = "rockchip,cspmu"; ++ reg = <0x0 0xfd90c000 0x0 0x1000>, ++ <0x0 0xfd90d000 0x0 0x1000>, ++ <0x0 0xfd90e000 0x0 0x1000>, ++ <0x0 0xfd90f000 0x0 0x1000>; ++ }; ++}; ++ ++&reserved_memory { ++ ramoops: ramoops@110000 { ++ compatible = "ramoops"; ++ reg = <0x0 0x110000 0x0 0xf0000>; ++ record-size = <0x20000>; ++ console-size = <0x80000>; ++ ftrace-size = <0x00000>; ++ pmsg-size = <0x50000>; ++ }; ++}; ++ ++&rng { ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-nvr.dtsi b/arch/arm64/boot/dts/rockchip/rk3568-nvr.dtsi +new file mode 100755 +index 000000000000..e1ec294b2c82 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-nvr.dtsi +@@ -0,0 +1,510 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++#include "rk3568.dtsi" ++#include ++#include ++#include ++#include ++#include ++ ++/ { ++ adc_keys: adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 0>; ++ io-channel-names = "buttons"; ++ keyup-threshold-microvolt = <1800000>; ++ poll-interval = <100>; ++ ++ vol-up-key { ++ label = "volume up"; ++ linux,code = ; ++ press-threshold-microvolt = <1750>; ++ }; ++ ++ vol-down-key { ++ label = "volume down"; ++ linux,code = ; ++ press-threshold-microvolt = <297500>; ++ }; ++ }; ++ ++ dc_12v: dc-12v { ++ compatible = "regulator-fixed"; ++ regulator-name = "dc_12v"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <12000000>; ++ regulator-max-microvolt = <12000000>; ++ }; ++ ++ hdmi_sound: hdmi-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,mclk-fs = <128>; ++ simple-audio-card,name = "hdmi-sound"; ++ status = "okay"; ++ ++ simple-audio-card,cpu { ++ sound-dai = <&i2s0_8ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&hdmi>; ++ }; ++ }; ++ ++ reserved-memory { ++ #address-cells = <2>; ++ #size-cells = <2>; ++ ranges; ++ ++ rknpu_reserved: rknpu { ++ compatible = "shared-dma-pool"; ++ inactive; ++ reusable; ++ size = <0x0 0x20000000>; ++ alignment = <0x0 0x1000>; ++ }; ++ }; ++ ++ spdif-sound { ++ status = "okay"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,name = "ROCKCHIP,SPDIF"; ++ simple-audio-card,cpu { ++ sound-dai = <&spdif_8ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&spdif_out>; ++ }; ++ }; ++ ++ spdif_out: spdif-out { ++ status = "okay"; ++ compatible = "linux,spdif-dit"; ++ #sound-dai-cells = <0>; ++ }; ++ ++ vcc3v3_sys: vcc3v3-sys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc3v3_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ vin-supply = <&dc_12v>; ++ }; ++ ++ vcc5v0_sys: vcc5v0-sys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc5v0_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ vin-supply = <&dc_12v>; ++ }; ++ ++ vcc5v0_host: vcc5v0-host-regulator { ++ compatible = "regulator-fixed"; ++ enable-active-high; ++ gpio = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&vcc5v0_host_en>; ++ regulator-name = "vcc5v0_host"; ++ regulator-always-on; ++ }; ++ ++ vcc_1v8: vcc_1v8 { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_1v8"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ vin-supply = <&vcc5v0_sys>; ++ }; ++ ++ vcc_3v3: vcc_3v3{ ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc_3v3"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ vin-supply = <&vcc5v0_sys>; ++ }; ++ ++ vdd_fixed: vdd-fixed { ++ compatible = "regulator-fixed"; ++ regulator-name = "vdd_fixed"; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <900000>; ++ regulator-always-on; ++ regulator-boot-on; ++ vin-supply = <&vcc5v0_sys>; ++ }; ++ ++ vdd_logic: vdd-logic { ++ compatible = "pwm-regulator"; ++ pwms = <&pwm1 0 5000 1>; ++ regulator-name = "vdd_logic"; ++ regulator-min-microvolt = <800000>; ++ regulator-max-microvolt = <1200000>; ++ regulator-init-microvolt = <900000>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-settling-time-up-us = <250>; ++ pwm-supply = <&vcc5v0_sys>; ++ status = "okay"; ++ }; ++ ++ vdd_npu: vdd-npu { ++ compatible = "pwm-regulator"; ++ pwms = <&pwm2 0 5000 1>; ++ regulator-name = "vdd_npu"; ++ regulator-min-microvolt = <800000>; ++ regulator-max-microvolt = <1200000>; ++ regulator-init-microvolt = <950000>; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-settling-time-up-us = <250>; ++ pwm-supply = <&vcc5v0_sys>; ++ status = "okay"; ++ }; ++}; ++ ++&bus_npu { ++ bus-supply = <&vdd_logic>; ++ pvtm-supply = <&vdd_cpu>; ++ status = "okay"; ++}; ++ ++&combphy0_us { ++ status = "okay"; ++}; ++ ++&cpu0 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&cpu0_opp_table { ++ /delete-node/ opp-1992000000; ++}; ++ ++&CPU_SLEEP { ++ status = "disabled"; ++}; ++ ++&dsi0_in_vp0 { ++ status = "disabled"; ++}; ++ ++&dsi0_in_vp1 { ++ status = "okay"; ++}; ++ ++&dsi1_in_vp0 { ++ status = "disabled"; ++}; ++ ++&dsi1_in_vp1 { ++ status = "okay"; ++}; ++ ++&edp { ++ hpd-gpios = <&gpio4 RK_PC4 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++}; ++ ++&edp_in_vp0 { ++ status = "disabled"; ++}; ++ ++&edp_in_vp1 { ++ status = "okay"; ++}; ++ ++&edp_phy { ++ status = "okay"; ++}; ++ ++&gpu { ++ mali-supply = <&vdd_fixed>; ++ status = "okay"; ++}; ++ ++&gpu_opp_table { ++ /delete-node/ opp-800000000; ++}; ++ ++&hdmi { ++ status = "okay"; ++}; ++ ++&hdmi_in_vp0 { ++ status = "okay"; ++}; ++ ++&hdmi_in_vp1 { ++ status = "disabled"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ ++ vdd_cpu: tcs4525@1c { ++ compatible = "tcs,tcs452x"; ++ reg = <0x1c>; ++ vin-supply = <&vcc5v0_sys>; ++ regulator-compatible = "fan53555-reg"; ++ regulator-name = "vdd_cpu"; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1390000>; ++ regulator-init-microvolt = <900000>; ++ regulator-ramp-delay = <2300>; ++ fcs,suspend-voltage-selector = <1>; ++ regulator-boot-on; ++ regulator-always-on; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++}; ++ ++&i2s0_8ch { ++ status = "okay"; ++}; ++ ++&i2s1_8ch { ++ status = "okay"; ++ rockchip,clk-trcm = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1m0_sclktx ++ &i2s1m0_lrcktx ++ &i2s1m0_sdi0 ++ &i2s1m0_sdo0>; ++}; ++ ++&iep { ++ status = "okay"; ++}; ++ ++&iep_mmu { ++ status = "okay"; ++}; ++ ++&jpegd { ++ status = "okay"; ++}; ++ ++&jpegd_mmu { ++ status = "okay"; ++}; ++ ++&video_phy0 { ++ status = "okay"; ++}; ++ ++&video_phy1 { ++ status = "okay"; ++}; ++ ++&mpp_srv { ++ status = "okay"; ++}; ++ ++/* Need to be modified according to the actual hardware */ ++&pmu_io_domains { ++ status = "okay"; ++ pmuio2-supply = <&vcc_3v3>; ++ vccio1-supply = <&vcc_3v3>; ++ vccio3-supply = <&vcc_3v3>; ++ vccio4-supply = <&vcc_3v3>; ++ vccio5-supply = <&vcc_3v3>; ++ vccio6-supply = <&vcc_3v3>; ++ vccio7-supply = <&vcc_3v3>; ++}; ++ ++&pwm1 { ++ status = "okay"; ++ pinctrl-names = "active"; ++}; ++ ++&pwm2 { ++ status = "okay"; ++ pinctrl-names = "active"; ++}; ++ ++&rk_rga { ++ status = "okay"; ++}; ++ ++&rknpu { ++ memory-region = <&rknpu_reserved>; ++ rknpu-supply = <&vdd_npu>; ++ status = "okay"; ++}; ++ ++&rknpu_mmu { ++ status = "disabled"; ++}; ++ ++&rkvdec { ++ rockchip,disable-auto-freq; ++ assigned-clock-rates = <396000000>, <396000000>, <396000000>, <600000000>; ++ status = "okay"; ++}; ++ ++&rkvdec_mmu { ++ status = "okay"; ++}; ++ ++&rkvdec_sram { ++ reg = <0x0 0x10000>; ++}; ++ ++&rkvenc { ++ status = "okay"; ++}; ++ ++&rkvenc_mmu { ++ status = "okay"; ++}; ++ ++&route_hdmi { ++ status = "okay"; ++ connect = <&vp0_out_hdmi>; ++}; ++ ++&saradc { ++ status = "okay"; ++ vref-supply = <&vcc_1v8>; ++}; ++ ++&sdhci { ++ bus-width = <8>; ++ no-sdio; ++ no-sd; ++ non-removable; ++ status = "okay"; ++}; ++ ++&sfc { ++ status = "okay"; ++}; ++ ++&spdif_8ch { ++ status = "okay"; ++}; ++ ++&sram { ++ reg = <0x0 0xfdcc0000 0x0 0x10000>; ++ ranges = <0x0 0x0 0xfdcc0000 0x10000>; ++}; ++ ++&tsadc { ++ status = "okay"; ++}; ++ ++&u2phy0_host { ++ phy-supply = <&vcc5v0_host>; ++ status = "okay"; ++}; ++ ++&u2phy0_otg { ++ phy-supply = <&vcc5v0_host>; ++ status = "okay"; ++}; ++ ++&u2phy1_host { ++ phy-supply = <&vcc5v0_host>; ++ status = "okay"; ++}; ++ ++&u2phy1_otg { ++ phy-supply = <&vcc5v0_host>; ++ status = "okay"; ++}; ++ ++&usb2phy0 { ++ status = "okay"; ++}; ++ ++&usb2phy1 { ++ status = "okay"; ++}; ++ ++&usb_host0_ehci { ++ status = "okay"; ++}; ++ ++&usb_host0_ohci { ++ status = "okay"; ++}; ++ ++&usb_host1_ehci { ++ status = "okay"; ++}; ++ ++&usb_host1_ohci { ++ status = "okay"; ++}; ++ ++&usbdrd_dwc3 { ++ dr_mode = "otg"; ++ extcon = <&usb2phy0>; ++ status = "okay"; ++}; ++ ++&usbdrd30 { ++ status = "okay"; ++}; ++ ++&usbhost_dwc3 { ++ phys = <&u2phy0_host>; ++ phy-names = "usb2-phy"; ++ maximum-speed = "high-speed"; ++ status = "okay"; ++}; ++ ++&usbhost30 { ++ status = "okay"; ++}; ++ ++&vdpu { ++ status = "okay"; ++}; ++ ++&vdpu_mmu { ++ status = "okay"; ++}; ++ ++&vepu { ++ status = "okay"; ++}; ++ ++&vepu_mmu { ++ status = "okay"; ++}; ++ ++&vop { ++ status = "okay"; ++ assigned-clocks = <&cru DCLK_VOP1>; ++ assigned-clock-parents = <&cru PLL_VPLL>; ++}; ++ ++&vop_mmu { ++ status = "okay"; ++}; ++ ++&pinctrl { ++ usb { ++ vcc5v0_host_en: vcc5v0-host-en { ++ rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-pinctrl.dtsi b/arch/arm64/boot/dts/rockchip/rk3568-pinctrl.dtsi +new file mode 100755 +index 000000000000..30d8cedfbdc7 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-pinctrl.dtsi +@@ -0,0 +1,3119 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ */ ++ ++#include ++#include "rockchip-pinconf.dtsi" ++ ++/* ++ * This file is auto generated by pin2dts tool, please keep these code ++ * by adding changes at end of this file. ++ */ ++&pinctrl { ++ acodec { ++ /omit-if-no-ref/ ++ acodec_pins: acodec-pins { ++ rockchip,pins = ++ /* acodec_adc_sync */ ++ <1 RK_PB1 5 &pcfg_pull_none>, ++ /* acodec_adcclk */ ++ <1 RK_PA1 5 &pcfg_pull_none>, ++ /* acodec_adcdata */ ++ <1 RK_PA0 5 &pcfg_pull_none>, ++ /* acodec_dac_datal */ ++ <1 RK_PA7 5 &pcfg_pull_none>, ++ /* acodec_dac_datar */ ++ <1 RK_PB0 5 &pcfg_pull_none>, ++ /* acodec_dacclk */ ++ <1 RK_PA3 5 &pcfg_pull_none>, ++ /* acodec_dacsync */ ++ <1 RK_PA5 5 &pcfg_pull_none>; ++ }; ++ }; ++ ++ audiopwm { ++ /omit-if-no-ref/ ++ audiopwm_lout: audiopwm-lout { ++ rockchip,pins = ++ /* audiopwm_lout */ ++ <1 RK_PA0 4 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ audiopwm_loutn: audiopwm-loutn { ++ rockchip,pins = ++ /* audiopwm_loutn */ ++ <1 RK_PA1 6 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ audiopwm_loutp: audiopwm-loutp { ++ rockchip,pins = ++ /* audiopwm_loutp */ ++ <1 RK_PA0 6 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ audiopwm_rout: audiopwm-rout { ++ rockchip,pins = ++ /* audiopwm_rout */ ++ <1 RK_PA1 4 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ audiopwm_routn: audiopwm-routn { ++ rockchip,pins = ++ /* audiopwm_routn */ ++ <1 RK_PA7 4 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ audiopwm_routp: audiopwm-routp { ++ rockchip,pins = ++ /* audiopwm_routp */ ++ <1 RK_PA6 4 &pcfg_pull_none>; ++ }; ++ }; ++ ++ bt656 { ++ /omit-if-no-ref/ ++ bt656m0_pins: bt656m0-pins { ++ rockchip,pins = ++ /* bt656_clkm0 */ ++ <3 RK_PA0 2 &pcfg_pull_none>, ++ /* bt656_d0m0 */ ++ <2 RK_PD0 2 &pcfg_pull_none>, ++ /* bt656_d1m0 */ ++ <2 RK_PD1 2 &pcfg_pull_none>, ++ /* bt656_d2m0 */ ++ <2 RK_PD2 2 &pcfg_pull_none>, ++ /* bt656_d3m0 */ ++ <2 RK_PD3 2 &pcfg_pull_none>, ++ /* bt656_d4m0 */ ++ <2 RK_PD4 2 &pcfg_pull_none>, ++ /* bt656_d5m0 */ ++ <2 RK_PD5 2 &pcfg_pull_none>, ++ /* bt656_d6m0 */ ++ <2 RK_PD6 2 &pcfg_pull_none>, ++ /* bt656_d7m0 */ ++ <2 RK_PD7 2 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ bt656m1_pins: bt656m1-pins { ++ rockchip,pins = ++ /* bt656_clkm1 */ ++ <4 RK_PB4 5 &pcfg_pull_none>, ++ /* bt656_d0m1 */ ++ <3 RK_PC6 5 &pcfg_pull_none>, ++ /* bt656_d1m1 */ ++ <3 RK_PC7 5 &pcfg_pull_none>, ++ /* bt656_d2m1 */ ++ <3 RK_PD0 5 &pcfg_pull_none>, ++ /* bt656_d3m1 */ ++ <3 RK_PD1 5 &pcfg_pull_none>, ++ /* bt656_d4m1 */ ++ <3 RK_PD2 5 &pcfg_pull_none>, ++ /* bt656_d5m1 */ ++ <3 RK_PD3 5 &pcfg_pull_none>, ++ /* bt656_d6m1 */ ++ <3 RK_PD4 5 &pcfg_pull_none>, ++ /* bt656_d7m1 */ ++ <3 RK_PD5 5 &pcfg_pull_none>; ++ }; ++ }; ++ ++ bt1120 { ++ /omit-if-no-ref/ ++ bt1120_pins: bt1120-pins { ++ rockchip,pins = ++ /* bt1120_clk */ ++ <3 RK_PA6 2 &pcfg_pull_none>, ++ /* bt1120_d0 */ ++ <3 RK_PA1 2 &pcfg_pull_none>, ++ /* bt1120_d1 */ ++ <3 RK_PA2 2 &pcfg_pull_none>, ++ /* bt1120_d2 */ ++ <3 RK_PA3 2 &pcfg_pull_none>, ++ /* bt1120_d3 */ ++ <3 RK_PA4 2 &pcfg_pull_none>, ++ /* bt1120_d4 */ ++ <3 RK_PA5 2 &pcfg_pull_none>, ++ /* bt1120_d5 */ ++ <3 RK_PA7 2 &pcfg_pull_none>, ++ /* bt1120_d6 */ ++ <3 RK_PB0 2 &pcfg_pull_none>, ++ /* bt1120_d7 */ ++ <3 RK_PB1 2 &pcfg_pull_none>, ++ /* bt1120_d8 */ ++ <3 RK_PB2 2 &pcfg_pull_none>, ++ /* bt1120_d9 */ ++ <3 RK_PB3 2 &pcfg_pull_none>, ++ /* bt1120_d10 */ ++ <3 RK_PB4 2 &pcfg_pull_none>, ++ /* bt1120_d11 */ ++ <3 RK_PB5 2 &pcfg_pull_none>, ++ /* bt1120_d12 */ ++ <3 RK_PB6 2 &pcfg_pull_none>, ++ /* bt1120_d13 */ ++ <3 RK_PC1 2 &pcfg_pull_none>, ++ /* bt1120_d14 */ ++ <3 RK_PC2 2 &pcfg_pull_none>, ++ /* bt1120_d15 */ ++ <3 RK_PC3 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ cam { ++ /omit-if-no-ref/ ++ cam_clkout0: cam-clkout0 { ++ rockchip,pins = ++ /* cam_clkout0 */ ++ <4 RK_PA7 1 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ cam_clkout1: cam-clkout1 { ++ rockchip,pins = ++ /* cam_clkout1 */ ++ <4 RK_PB0 1 &pcfg_pull_none>; ++ }; ++ }; ++ ++ can0 { ++ /omit-if-no-ref/ ++ can0m0_pins: can0m0-pins { ++ rockchip,pins = ++ /* can0_rxm0 */ ++ <0 RK_PB4 2 &pcfg_pull_none>, ++ /* can0_txm0 */ ++ <0 RK_PB3 2 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ can0m1_pins: can0m1-pins { ++ rockchip,pins = ++ /* can0_rxm1 */ ++ <2 RK_PA2 4 &pcfg_pull_none>, ++ /* can0_txm1 */ ++ <2 RK_PA1 4 &pcfg_pull_none>; ++ }; ++ }; ++ ++ can1 { ++ /omit-if-no-ref/ ++ can1m0_pins: can1m0-pins { ++ rockchip,pins = ++ /* can1_rxm0 */ ++ <1 RK_PA0 3 &pcfg_pull_none>, ++ /* can1_txm0 */ ++ <1 RK_PA1 3 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ can1m1_pins: can1m1-pins { ++ rockchip,pins = ++ /* can1_rxm1 */ ++ <4 RK_PC2 3 &pcfg_pull_none>, ++ /* can1_txm1 */ ++ <4 RK_PC3 3 &pcfg_pull_none>; ++ }; ++ }; ++ ++ can2 { ++ /omit-if-no-ref/ ++ can2m0_pins: can2m0-pins { ++ rockchip,pins = ++ /* can2_rxm0 */ ++ <4 RK_PB4 3 &pcfg_pull_none>, ++ /* can2_txm0 */ ++ <4 RK_PB5 3 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ can2m1_pins: can2m1-pins { ++ rockchip,pins = ++ /* can2_rxm1 */ ++ <2 RK_PB1 4 &pcfg_pull_none>, ++ /* can2_txm1 */ ++ <2 RK_PB2 4 &pcfg_pull_none>; ++ }; ++ }; ++ ++ cif { ++ /omit-if-no-ref/ ++ cif_clk: cif-clk { ++ rockchip,pins = ++ /* cif_clkout */ ++ <4 RK_PC0 1 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ cif_dvp_clk: cif-dvp-clk { ++ rockchip,pins = ++ /* cif_clkin */ ++ <4 RK_PC1 1 &pcfg_pull_none>, ++ /* cif_href */ ++ <4 RK_PB6 1 &pcfg_pull_none>, ++ /* cif_vsync */ ++ <4 RK_PB7 1 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ cif_dvp_bus16: cif-dvp-bus16 { ++ rockchip,pins = ++ /* cif_d8 */ ++ <3 RK_PD6 1 &pcfg_pull_none>, ++ /* cif_d9 */ ++ <3 RK_PD7 1 &pcfg_pull_none>, ++ /* cif_d10 */ ++ <4 RK_PA0 1 &pcfg_pull_none>, ++ /* cif_d11 */ ++ <4 RK_PA1 1 &pcfg_pull_none>, ++ /* cif_d12 */ ++ <4 RK_PA2 1 &pcfg_pull_none>, ++ /* cif_d13 */ ++ <4 RK_PA3 1 &pcfg_pull_none>, ++ /* cif_d14 */ ++ <4 RK_PA4 1 &pcfg_pull_none>, ++ /* cif_d15 */ ++ <4 RK_PA5 1 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ cif_dvp_bus8: cif-dvp-bus8 { ++ rockchip,pins = ++ /* cif_d0 */ ++ <3 RK_PC6 1 &pcfg_pull_none>, ++ /* cif_d1 */ ++ <3 RK_PC7 1 &pcfg_pull_none>, ++ /* cif_d2 */ ++ <3 RK_PD0 1 &pcfg_pull_none>, ++ /* cif_d3 */ ++ <3 RK_PD1 1 &pcfg_pull_none>, ++ /* cif_d4 */ ++ <3 RK_PD2 1 &pcfg_pull_none>, ++ /* cif_d5 */ ++ <3 RK_PD3 1 &pcfg_pull_none>, ++ /* cif_d6 */ ++ <3 RK_PD4 1 &pcfg_pull_none>, ++ /* cif_d7 */ ++ <3 RK_PD5 1 &pcfg_pull_none>; ++ }; ++ }; ++ ++ clk32k { ++ /omit-if-no-ref/ ++ clk32k_in: clk32k-in { ++ rockchip,pins = ++ /* clk32k_in */ ++ <0 RK_PB0 1 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ clk32k_out0: clk32k-out0 { ++ rockchip,pins = ++ /* clk32k_out0 */ ++ <0 RK_PB0 2 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ clk32k_out1: clk32k-out1 { ++ rockchip,pins = ++ /* clk32k_out1 */ ++ <2 RK_PC6 1 &pcfg_pull_none>; ++ }; ++ }; ++ ++ cpu { ++ /omit-if-no-ref/ ++ cpu_pins: cpu-pins { ++ rockchip,pins = ++ /* cpu_avs */ ++ <0 RK_PB7 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ ebc { ++ /omit-if-no-ref/ ++ ebc_extern: ebc-extern { ++ rockchip,pins = ++ /* ebc_sdce1 */ ++ <4 RK_PA7 2 &pcfg_pull_none>, ++ /* ebc_sdce2 */ ++ <4 RK_PB0 2 &pcfg_pull_none>, ++ /* ebc_sdce3 */ ++ <4 RK_PB1 2 &pcfg_pull_none>, ++ /* ebc_sdshr */ ++ <4 RK_PB5 2 &pcfg_pull_none>, ++ /* ebc_vcom */ ++ <4 RK_PB2 2 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ ebc_pins: ebc-pins { ++ rockchip,pins = ++ /* ebc_gdclk */ ++ <4 RK_PC0 2 &pcfg_pull_none>, ++ /* ebc_gdoe */ ++ <4 RK_PB3 2 &pcfg_pull_none>, ++ /* ebc_gdsp */ ++ <4 RK_PB4 2 &pcfg_pull_none>, ++ /* ebc_sdce0 */ ++ <4 RK_PA6 2 &pcfg_pull_none>, ++ /* ebc_sdclk */ ++ <4 RK_PC1 2 &pcfg_pull_none>, ++ /* ebc_sddo0 */ ++ <3 RK_PC6 2 &pcfg_pull_none>, ++ /* ebc_sddo1 */ ++ <3 RK_PC7 2 &pcfg_pull_none>, ++ /* ebc_sddo2 */ ++ <3 RK_PD0 2 &pcfg_pull_none>, ++ /* ebc_sddo3 */ ++ <3 RK_PD1 2 &pcfg_pull_none>, ++ /* ebc_sddo4 */ ++ <3 RK_PD2 2 &pcfg_pull_none>, ++ /* ebc_sddo5 */ ++ <3 RK_PD3 2 &pcfg_pull_none>, ++ /* ebc_sddo6 */ ++ <3 RK_PD4 2 &pcfg_pull_none>, ++ /* ebc_sddo7 */ ++ <3 RK_PD5 2 &pcfg_pull_none>, ++ /* ebc_sddo8 */ ++ <3 RK_PD6 2 &pcfg_pull_none>, ++ /* ebc_sddo9 */ ++ <3 RK_PD7 2 &pcfg_pull_none>, ++ /* ebc_sddo10 */ ++ <4 RK_PA0 2 &pcfg_pull_none>, ++ /* ebc_sddo11 */ ++ <4 RK_PA1 2 &pcfg_pull_none>, ++ /* ebc_sddo12 */ ++ <4 RK_PA2 2 &pcfg_pull_none>, ++ /* ebc_sddo13 */ ++ <4 RK_PA3 2 &pcfg_pull_none>, ++ /* ebc_sddo14 */ ++ <4 RK_PA4 2 &pcfg_pull_none>, ++ /* ebc_sddo15 */ ++ <4 RK_PA5 2 &pcfg_pull_none>, ++ /* ebc_sdle */ ++ <4 RK_PB6 2 &pcfg_pull_none>, ++ /* ebc_sdoe */ ++ <4 RK_PB7 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ edpdp { ++ /omit-if-no-ref/ ++ edpdpm0_pins: edpdpm0-pins { ++ rockchip,pins = ++ /* edpdp_hpdinm0 */ ++ <4 RK_PC4 1 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ edpdpm1_pins: edpdpm1-pins { ++ rockchip,pins = ++ /* edpdp_hpdinm1 */ ++ <0 RK_PC2 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ emmc { ++ /omit-if-no-ref/ ++ emmc_rstnout: emmc-rstnout { ++ rockchip,pins = ++ /* emmc_rstn */ ++ <1 RK_PC7 1 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ emmc_bus8: emmc-bus8 { ++ rockchip,pins = ++ /* emmc_d0 */ ++ <1 RK_PB4 1 &pcfg_pull_up_drv_level_2>, ++ /* emmc_d1 */ ++ <1 RK_PB5 1 &pcfg_pull_up_drv_level_2>, ++ /* emmc_d2 */ ++ <1 RK_PB6 1 &pcfg_pull_up_drv_level_2>, ++ /* emmc_d3 */ ++ <1 RK_PB7 1 &pcfg_pull_up_drv_level_2>, ++ /* emmc_d4 */ ++ <1 RK_PC0 1 &pcfg_pull_up_drv_level_2>, ++ /* emmc_d5 */ ++ <1 RK_PC1 1 &pcfg_pull_up_drv_level_2>, ++ /* emmc_d6 */ ++ <1 RK_PC2 1 &pcfg_pull_up_drv_level_2>, ++ /* emmc_d7 */ ++ <1 RK_PC3 1 &pcfg_pull_up_drv_level_2>; ++ }; ++ ++ /omit-if-no-ref/ ++ emmc_clk: emmc-clk { ++ rockchip,pins = ++ /* emmc_clkout */ ++ <1 RK_PC5 1 &pcfg_pull_up_drv_level_2>; ++ }; ++ ++ /omit-if-no-ref/ ++ emmc_cmd: emmc-cmd { ++ rockchip,pins = ++ /* emmc_cmd */ ++ <1 RK_PC4 1 &pcfg_pull_up_drv_level_2>; ++ }; ++ ++ /omit-if-no-ref/ ++ emmc_datastrobe: emmc-datastrobe { ++ rockchip,pins = ++ /* emmc_datastrobe */ ++ <1 RK_PC6 1 &pcfg_pull_none>; ++ }; ++ }; ++ ++ eth0 { ++ /omit-if-no-ref/ ++ eth0_pins: eth0-pins { ++ rockchip,pins = ++ /* eth0_refclko25m */ ++ <2 RK_PC1 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ eth1 { ++ /omit-if-no-ref/ ++ eth1m0_pins: eth1m0-pins { ++ rockchip,pins = ++ /* eth1_refclko25mm0 */ ++ <3 RK_PB0 3 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ eth1m1_pins: eth1m1-pins { ++ rockchip,pins = ++ /* eth1_refclko25mm1 */ ++ <4 RK_PB3 3 &pcfg_pull_none>; ++ }; ++ }; ++ ++ flash { ++ /omit-if-no-ref/ ++ flash_pins: flash-pins { ++ rockchip,pins = ++ /* flash_ale */ ++ <1 RK_PD0 2 &pcfg_pull_none>, ++ /* flash_cle */ ++ <1 RK_PC6 3 &pcfg_pull_none>, ++ /* flash_cs0n */ ++ <1 RK_PD3 2 &pcfg_pull_none>, ++ /* flash_cs1n */ ++ <1 RK_PD4 2 &pcfg_pull_none>, ++ /* flash_d0 */ ++ <1 RK_PB4 2 &pcfg_pull_none>, ++ /* flash_d1 */ ++ <1 RK_PB5 2 &pcfg_pull_none>, ++ /* flash_d2 */ ++ <1 RK_PB6 2 &pcfg_pull_none>, ++ /* flash_d3 */ ++ <1 RK_PB7 2 &pcfg_pull_none>, ++ /* flash_d4 */ ++ <1 RK_PC0 2 &pcfg_pull_none>, ++ /* flash_d5 */ ++ <1 RK_PC1 2 &pcfg_pull_none>, ++ /* flash_d6 */ ++ <1 RK_PC2 2 &pcfg_pull_none>, ++ /* flash_d7 */ ++ <1 RK_PC3 2 &pcfg_pull_none>, ++ /* flash_dqs */ ++ <1 RK_PC5 2 &pcfg_pull_none>, ++ /* flash_rdn */ ++ <1 RK_PD2 2 &pcfg_pull_none>, ++ /* flash_rdy */ ++ <1 RK_PD1 2 &pcfg_pull_none>, ++ /* flash_volsel */ ++ <0 RK_PA7 1 &pcfg_pull_none>, ++ /* flash_wpn */ ++ <1 RK_PC7 3 &pcfg_pull_none>, ++ /* flash_wrn */ ++ <1 RK_PC4 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ fspi { ++ /omit-if-no-ref/ ++ fspi_pins: fspi-pins { ++ rockchip,pins = ++ /* fspi_clk */ ++ <1 RK_PD0 1 &pcfg_pull_none>, ++ /* fspi_cs0n */ ++ <1 RK_PD3 1 &pcfg_pull_none>, ++ /* fspi_d0 */ ++ <1 RK_PD1 1 &pcfg_pull_none>, ++ /* fspi_d1 */ ++ <1 RK_PD2 1 &pcfg_pull_none>, ++ /* fspi_d2 */ ++ <1 RK_PC7 2 &pcfg_pull_none>, ++ /* fspi_d3 */ ++ <1 RK_PD4 1 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ fspi_cs1: fspi-cs1 { ++ rockchip,pins = ++ /* fspi_cs1n */ ++ <1 RK_PC6 2 &pcfg_pull_up>; ++ }; ++ }; ++ ++ gmac0 { ++ /omit-if-no-ref/ ++ gmac0_miim: gmac0-miim { ++ rockchip,pins = ++ /* gmac0_mdc */ ++ <2 RK_PC3 2 &pcfg_pull_none>, ++ /* gmac0_mdio */ ++ <2 RK_PC4 2 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ gmac0_clkinout: gmac0-clkinout { ++ rockchip,pins = ++ /* gmac0_mclkinout */ ++ <2 RK_PC2 2 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ gmac0_rx_er: gmac0-rx-er { ++ rockchip,pins = ++ /* gmac0_rxer */ ++ <2 RK_PC5 2 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ gmac0_rx_bus2: gmac0-rx-bus2 { ++ rockchip,pins = ++ /* gmac0_rxd0 */ ++ <2 RK_PB6 1 &pcfg_pull_none>, ++ /* gmac0_rxd1 */ ++ <2 RK_PB7 2 &pcfg_pull_none>, ++ /* gmac0_rxdvcrs */ ++ <2 RK_PC0 2 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ gmac0_tx_bus2: gmac0-tx-bus2 { ++ rockchip,pins = ++ /* gmac0_txd0 */ ++ <2 RK_PB3 1 &pcfg_pull_none_drv_level_2>, ++ /* gmac0_txd1 */ ++ <2 RK_PB4 1 &pcfg_pull_none_drv_level_2>, ++ /* gmac0_txen */ ++ <2 RK_PB5 1 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ gmac0_rgmii_clk: gmac0-rgmii-clk { ++ rockchip,pins = ++ /* gmac0_rxclk */ ++ <2 RK_PA5 2 &pcfg_pull_none>, ++ /* gmac0_txclk */ ++ <2 RK_PB0 2 &pcfg_pull_none_drv_level_1>; ++ }; ++ ++ /omit-if-no-ref/ ++ gmac0_rgmii_bus: gmac0-rgmii-bus { ++ rockchip,pins = ++ /* gmac0_rxd2 */ ++ <2 RK_PA3 2 &pcfg_pull_none>, ++ /* gmac0_rxd3 */ ++ <2 RK_PA4 2 &pcfg_pull_none>, ++ /* gmac0_txd2 */ ++ <2 RK_PA6 2 &pcfg_pull_none_drv_level_2>, ++ /* gmac0_txd3 */ ++ <2 RK_PA7 2 &pcfg_pull_none_drv_level_2>; ++ }; ++ }; ++ ++ gmac1 { ++ /omit-if-no-ref/ ++ gmac1m0_miim: gmac1m0-miim { ++ rockchip,pins = ++ /* gmac1_mdcm0 */ ++ <3 RK_PC4 3 &pcfg_pull_none>, ++ /* gmac1_mdiom0 */ ++ <3 RK_PC5 3 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ gmac1m0_clkinout: gmac1m0-clkinout { ++ rockchip,pins = ++ /* gmac1_mclkinoutm0 */ ++ <3 RK_PC0 3 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ gmac1m0_rx_er: gmac1m0-rx-er { ++ rockchip,pins = ++ /* gmac1_rxerm0 */ ++ <3 RK_PB4 3 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ gmac1m0_rx_bus2: gmac1m0-rx-bus2 { ++ rockchip,pins = ++ /* gmac1_rxd0m0 */ ++ <3 RK_PB1 3 &pcfg_pull_none>, ++ /* gmac1_rxd1m0 */ ++ <3 RK_PB2 3 &pcfg_pull_none>, ++ /* gmac1_rxdvcrsm0 */ ++ <3 RK_PB3 3 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ gmac1m0_tx_bus2: gmac1m0-tx-bus2 { ++ rockchip,pins = ++ /* gmac1_txd0m0 */ ++ <3 RK_PB5 3 &pcfg_pull_none_drv_level_2>, ++ /* gmac1_txd1m0 */ ++ <3 RK_PB6 3 &pcfg_pull_none_drv_level_2>, ++ /* gmac1_txenm0 */ ++ <3 RK_PB7 3 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ gmac1m0_rgmii_clk: gmac1m0-rgmii-clk { ++ rockchip,pins = ++ /* gmac1_rxclkm0 */ ++ <3 RK_PA7 3 &pcfg_pull_none>, ++ /* gmac1_txclkm0 */ ++ <3 RK_PA6 3 &pcfg_pull_none_drv_level_1>; ++ }; ++ ++ /omit-if-no-ref/ ++ gmac1m0_rgmii_bus: gmac1m0-rgmii-bus { ++ rockchip,pins = ++ /* gmac1_rxd2m0 */ ++ <3 RK_PA4 3 &pcfg_pull_none>, ++ /* gmac1_rxd3m0 */ ++ <3 RK_PA5 3 &pcfg_pull_none>, ++ /* gmac1_txd2m0 */ ++ <3 RK_PA2 3 &pcfg_pull_none_drv_level_2>, ++ /* gmac1_txd3m0 */ ++ <3 RK_PA3 3 &pcfg_pull_none_drv_level_2>; ++ }; ++ ++ /omit-if-no-ref/ ++ gmac1m1_miim: gmac1m1-miim { ++ rockchip,pins = ++ /* gmac1_mdcm1 */ ++ <4 RK_PB6 3 &pcfg_pull_none>, ++ /* gmac1_mdiom1 */ ++ <4 RK_PB7 3 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ gmac1m1_clkinout: gmac1m1-clkinout { ++ rockchip,pins = ++ /* gmac1_mclkinoutm1 */ ++ <4 RK_PC1 3 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ gmac1m1_rx_er: gmac1m1-rx-er { ++ rockchip,pins = ++ /* gmac1_rxerm1 */ ++ <4 RK_PB2 3 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ gmac1m1_rx_bus2: gmac1m1-rx-bus2 { ++ rockchip,pins = ++ /* gmac1_rxd0m1 */ ++ <4 RK_PA7 3 &pcfg_pull_none>, ++ /* gmac1_rxd1m1 */ ++ <4 RK_PB0 3 &pcfg_pull_none>, ++ /* gmac1_rxdvcrsm1 */ ++ <4 RK_PB1 3 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ gmac1m1_tx_bus2: gmac1m1-tx-bus2 { ++ rockchip,pins = ++ /* gmac1_txd0m1 */ ++ <4 RK_PA4 3 &pcfg_pull_none_drv_level_2>, ++ /* gmac1_txd1m1 */ ++ <4 RK_PA5 3 &pcfg_pull_none_drv_level_2>, ++ /* gmac1_txenm1 */ ++ <4 RK_PA6 3 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ gmac1m1_rgmii_clk: gmac1m1-rgmii-clk { ++ rockchip,pins = ++ /* gmac1_rxclkm1 */ ++ <4 RK_PA3 3 &pcfg_pull_none>, ++ /* gmac1_txclkm1 */ ++ <4 RK_PA0 3 &pcfg_pull_none_drv_level_1>; ++ }; ++ ++ /omit-if-no-ref/ ++ gmac1m1_rgmii_bus: gmac1m1-rgmii-bus { ++ rockchip,pins = ++ /* gmac1_rxd2m1 */ ++ <4 RK_PA1 3 &pcfg_pull_none>, ++ /* gmac1_rxd3m1 */ ++ <4 RK_PA2 3 &pcfg_pull_none>, ++ /* gmac1_txd2m1 */ ++ <3 RK_PD6 3 &pcfg_pull_none_drv_level_2>, ++ /* gmac1_txd3m1 */ ++ <3 RK_PD7 3 &pcfg_pull_none_drv_level_2>; ++ }; ++ }; ++ ++ gpu { ++ /omit-if-no-ref/ ++ gpu_pins: gpu-pins { ++ rockchip,pins = ++ /* gpu_avs */ ++ <0 RK_PC0 2 &pcfg_pull_none>, ++ /* gpu_pwren */ ++ <0 RK_PA6 4 &pcfg_pull_none>; ++ }; ++ }; ++ ++ hdmitx { ++ /omit-if-no-ref/ ++ hdmitxm0_cec: hdmitxm0-cec { ++ rockchip,pins = ++ /* hdmitxm0_cec */ ++ <4 RK_PD1 1 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ hdmitxm1_cec: hdmitxm1-cec { ++ rockchip,pins = ++ /* hdmitxm1_cec */ ++ <0 RK_PC7 1 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ hdmitx_scl: hdmitx-scl { ++ rockchip,pins = ++ /* hdmitx_scl */ ++ <4 RK_PC7 1 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ hdmitx_sda: hdmitx-sda { ++ rockchip,pins = ++ /* hdmitx_sda */ ++ <4 RK_PD0 1 &pcfg_pull_none>; ++ }; ++ }; ++ ++ i2c0 { ++ /omit-if-no-ref/ ++ i2c0_xfer: i2c0-xfer { ++ rockchip,pins = ++ /* i2c0_scl */ ++ <0 RK_PB1 1 &pcfg_pull_none_smt>, ++ /* i2c0_sda */ ++ <0 RK_PB2 1 &pcfg_pull_none_smt>; ++ }; ++ }; ++ ++ i2c1 { ++ /omit-if-no-ref/ ++ i2c1_xfer: i2c1-xfer { ++ rockchip,pins = ++ /* i2c1_scl */ ++ <0 RK_PB3 1 &pcfg_pull_none_smt>, ++ /* i2c1_sda */ ++ <0 RK_PB4 1 &pcfg_pull_none_smt>; ++ }; ++ }; ++ ++ i2c2 { ++ /omit-if-no-ref/ ++ i2c2m0_xfer: i2c2m0-xfer { ++ rockchip,pins = ++ /* i2c2_sclm0 */ ++ <0 RK_PB5 1 &pcfg_pull_none_smt>, ++ /* i2c2_sdam0 */ ++ <0 RK_PB6 1 &pcfg_pull_none_smt>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2c2m1_xfer: i2c2m1-xfer { ++ rockchip,pins = ++ /* i2c2_sclm1 */ ++ <4 RK_PB5 1 &pcfg_pull_none_smt>, ++ /* i2c2_sdam1 */ ++ <4 RK_PB4 1 &pcfg_pull_none_smt>; ++ }; ++ }; ++ ++ i2c3 { ++ /omit-if-no-ref/ ++ i2c3m0_xfer: i2c3m0-xfer { ++ rockchip,pins = ++ /* i2c3_sclm0 */ ++ <1 RK_PA1 1 &pcfg_pull_none_smt>, ++ /* i2c3_sdam0 */ ++ <1 RK_PA0 1 &pcfg_pull_none_smt>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2c3m1_xfer: i2c3m1-xfer { ++ rockchip,pins = ++ /* i2c3_sclm1 */ ++ <3 RK_PB5 4 &pcfg_pull_none_smt>, ++ /* i2c3_sdam1 */ ++ <3 RK_PB6 4 &pcfg_pull_none_smt>; ++ }; ++ }; ++ ++ i2c4 { ++ /omit-if-no-ref/ ++ i2c4m0_xfer: i2c4m0-xfer { ++ rockchip,pins = ++ /* i2c4_sclm0 */ ++ <4 RK_PB3 1 &pcfg_pull_none_smt>, ++ /* i2c4_sdam0 */ ++ <4 RK_PB2 1 &pcfg_pull_none_smt>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2c4m1_xfer: i2c4m1-xfer { ++ rockchip,pins = ++ /* i2c4_sclm1 */ ++ <2 RK_PB2 2 &pcfg_pull_none_smt>, ++ /* i2c4_sdam1 */ ++ <2 RK_PB1 2 &pcfg_pull_none_smt>; ++ }; ++ }; ++ ++ i2c5 { ++ /omit-if-no-ref/ ++ i2c5m0_xfer: i2c5m0-xfer { ++ rockchip,pins = ++ /* i2c5_sclm0 */ ++ <3 RK_PB3 4 &pcfg_pull_none_smt>, ++ /* i2c5_sdam0 */ ++ <3 RK_PB4 4 &pcfg_pull_none_smt>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2c5m1_xfer: i2c5m1-xfer { ++ rockchip,pins = ++ /* i2c5_sclm1 */ ++ <4 RK_PC7 2 &pcfg_pull_none_smt>, ++ /* i2c5_sdam1 */ ++ <4 RK_PD0 2 &pcfg_pull_none_smt>; ++ }; ++ }; ++ ++ i2s1 { ++ /omit-if-no-ref/ ++ i2s1m0_lrckrx: i2s1m0-lrckrx { ++ rockchip,pins = ++ /* i2s1m0_lrckrx */ ++ <1 RK_PA6 1 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s1m0_lrcktx: i2s1m0-lrcktx { ++ rockchip,pins = ++ /* i2s1m0_lrcktx */ ++ <1 RK_PA5 1 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s1m0_mclk: i2s1m0-mclk { ++ rockchip,pins = ++ /* i2s1m0_mclk */ ++ <1 RK_PA2 1 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s1m0_sclkrx: i2s1m0-sclkrx { ++ rockchip,pins = ++ /* i2s1m0_sclkrx */ ++ <1 RK_PA4 1 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s1m0_sclktx: i2s1m0-sclktx { ++ rockchip,pins = ++ /* i2s1m0_sclktx */ ++ <1 RK_PA3 1 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s1m0_sdi0: i2s1m0-sdi0 { ++ rockchip,pins = ++ /* i2s1m0_sdi0 */ ++ <1 RK_PB3 1 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s1m0_sdi1: i2s1m0-sdi1 { ++ rockchip,pins = ++ /* i2s1m0_sdi1 */ ++ <1 RK_PB2 2 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s1m0_sdi2: i2s1m0-sdi2 { ++ rockchip,pins = ++ /* i2s1m0_sdi2 */ ++ <1 RK_PB1 2 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s1m0_sdi3: i2s1m0-sdi3 { ++ rockchip,pins = ++ /* i2s1m0_sdi3 */ ++ <1 RK_PB0 2 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s1m0_sdo0: i2s1m0-sdo0 { ++ rockchip,pins = ++ /* i2s1m0_sdo0 */ ++ <1 RK_PA7 1 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s1m0_sdo1: i2s1m0-sdo1 { ++ rockchip,pins = ++ /* i2s1m0_sdo1 */ ++ <1 RK_PB0 1 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s1m0_sdo2: i2s1m0-sdo2 { ++ rockchip,pins = ++ /* i2s1m0_sdo2 */ ++ <1 RK_PB1 1 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s1m0_sdo3: i2s1m0-sdo3 { ++ rockchip,pins = ++ /* i2s1m0_sdo3 */ ++ <1 RK_PB2 1 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s1m1_lrckrx: i2s1m1-lrckrx { ++ rockchip,pins = ++ /* i2s1m1_lrckrx */ ++ <4 RK_PA7 5 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s1m1_lrcktx: i2s1m1-lrcktx { ++ rockchip,pins = ++ /* i2s1m1_lrcktx */ ++ <3 RK_PD0 4 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s1m1_mclk: i2s1m1-mclk { ++ rockchip,pins = ++ /* i2s1m1_mclk */ ++ <3 RK_PC6 4 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s1m1_sclkrx: i2s1m1-sclkrx { ++ rockchip,pins = ++ /* i2s1m1_sclkrx */ ++ <4 RK_PA6 5 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s1m1_sclktx: i2s1m1-sclktx { ++ rockchip,pins = ++ /* i2s1m1_sclktx */ ++ <3 RK_PC7 4 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s1m1_sdi0: i2s1m1-sdi0 { ++ rockchip,pins = ++ /* i2s1m1_sdi0 */ ++ <3 RK_PD2 4 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s1m1_sdi1: i2s1m1-sdi1 { ++ rockchip,pins = ++ /* i2s1m1_sdi1 */ ++ <3 RK_PD3 4 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s1m1_sdi2: i2s1m1-sdi2 { ++ rockchip,pins = ++ /* i2s1m1_sdi2 */ ++ <3 RK_PD4 4 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s1m1_sdi3: i2s1m1-sdi3 { ++ rockchip,pins = ++ /* i2s1m1_sdi3 */ ++ <3 RK_PD5 4 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s1m1_sdo0: i2s1m1-sdo0 { ++ rockchip,pins = ++ /* i2s1m1_sdo0 */ ++ <3 RK_PD1 4 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s1m1_sdo1: i2s1m1-sdo1 { ++ rockchip,pins = ++ /* i2s1m1_sdo1 */ ++ <4 RK_PB0 5 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s1m1_sdo2: i2s1m1-sdo2 { ++ rockchip,pins = ++ /* i2s1m1_sdo2 */ ++ <4 RK_PB1 4 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s1m1_sdo3: i2s1m1-sdo3 { ++ rockchip,pins = ++ /* i2s1m1_sdo3 */ ++ <4 RK_PB5 4 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s1m2_lrckrx: i2s1m2-lrckrx { ++ rockchip,pins = ++ /* i2s1m2_lrckrx */ ++ <3 RK_PC5 5 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s1m2_lrcktx: i2s1m2-lrcktx { ++ rockchip,pins = ++ /* i2s1m2_lrcktx */ ++ <2 RK_PD2 5 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s1m2_mclk: i2s1m2-mclk { ++ rockchip,pins = ++ /* i2s1m2_mclk */ ++ <2 RK_PD0 5 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s1m2_sclkrx: i2s1m2-sclkrx { ++ rockchip,pins = ++ /* i2s1m2_sclkrx */ ++ <3 RK_PC3 5 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s1m2_sclktx: i2s1m2-sclktx { ++ rockchip,pins = ++ /* i2s1m2_sclktx */ ++ <2 RK_PD1 5 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s1m2_sdi0: i2s1m2-sdi0 { ++ rockchip,pins = ++ /* i2s1m2_sdi0 */ ++ <2 RK_PD3 5 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s1m2_sdi1: i2s1m2-sdi1 { ++ rockchip,pins = ++ /* i2s1m2_sdi1 */ ++ <2 RK_PD4 5 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s1m2_sdi2: i2s1m2-sdi2 { ++ rockchip,pins = ++ /* i2s1m2_sdi2 */ ++ <2 RK_PD5 5 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s1m2_sdi3: i2s1m2-sdi3 { ++ rockchip,pins = ++ /* i2s1m2_sdi3 */ ++ <2 RK_PD6 5 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s1m2_sdo0: i2s1m2-sdo0 { ++ rockchip,pins = ++ /* i2s1m2_sdo0 */ ++ <2 RK_PD7 5 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s1m2_sdo1: i2s1m2-sdo1 { ++ rockchip,pins = ++ /* i2s1m2_sdo1 */ ++ <3 RK_PA0 5 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s1m2_sdo2: i2s1m2-sdo2 { ++ rockchip,pins = ++ /* i2s1m2_sdo2 */ ++ <3 RK_PC1 5 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s1m2_sdo3: i2s1m2-sdo3 { ++ rockchip,pins = ++ /* i2s1m2_sdo3 */ ++ <3 RK_PC2 5 &pcfg_pull_none>; ++ }; ++ }; ++ ++ i2s2 { ++ /omit-if-no-ref/ ++ i2s2m0_lrckrx: i2s2m0-lrckrx { ++ rockchip,pins = ++ /* i2s2m0_lrckrx */ ++ <2 RK_PC0 1 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s2m0_lrcktx: i2s2m0-lrcktx { ++ rockchip,pins = ++ /* i2s2m0_lrcktx */ ++ <2 RK_PC3 1 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s2m0_mclk: i2s2m0-mclk { ++ rockchip,pins = ++ /* i2s2m0_mclk */ ++ <2 RK_PC1 1 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s2m0_sclkrx: i2s2m0-sclkrx { ++ rockchip,pins = ++ /* i2s2m0_sclkrx */ ++ <2 RK_PB7 1 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s2m0_sclktx: i2s2m0-sclktx { ++ rockchip,pins = ++ /* i2s2m0_sclktx */ ++ <2 RK_PC2 1 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s2m0_sdi: i2s2m0-sdi { ++ rockchip,pins = ++ /* i2s2m0_sdi */ ++ <2 RK_PC5 1 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s2m0_sdo: i2s2m0-sdo { ++ rockchip,pins = ++ /* i2s2m0_sdo */ ++ <2 RK_PC4 1 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s2m1_lrckrx: i2s2m1-lrckrx { ++ rockchip,pins = ++ /* i2s2m1_lrckrx */ ++ <4 RK_PA5 5 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s2m1_lrcktx: i2s2m1-lrcktx { ++ rockchip,pins = ++ /* i2s2m1_lrcktx */ ++ <4 RK_PA4 5 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s2m1_mclk: i2s2m1-mclk { ++ rockchip,pins = ++ /* i2s2m1_mclk */ ++ <4 RK_PB6 5 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s2m1_sclkrx: i2s2m1-sclkrx { ++ rockchip,pins = ++ /* i2s2m1_sclkrx */ ++ <4 RK_PC1 5 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s2m1_sclktx: i2s2m1-sclktx { ++ rockchip,pins = ++ /* i2s2m1_sclktx */ ++ <4 RK_PB7 4 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s2m1_sdi: i2s2m1-sdi { ++ rockchip,pins = ++ /* i2s2m1_sdi */ ++ <4 RK_PB2 5 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s2m1_sdo: i2s2m1-sdo { ++ rockchip,pins = ++ /* i2s2m1_sdo */ ++ <4 RK_PB3 5 &pcfg_pull_none>; ++ }; ++ }; ++ ++ i2s3 { ++ /omit-if-no-ref/ ++ i2s3m0_lrck: i2s3m0-lrck { ++ rockchip,pins = ++ /* i2s3m0_lrck */ ++ <3 RK_PA4 4 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s3m0_mclk: i2s3m0-mclk { ++ rockchip,pins = ++ /* i2s3m0_mclk */ ++ <3 RK_PA2 4 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s3m0_sclk: i2s3m0-sclk { ++ rockchip,pins = ++ /* i2s3m0_sclk */ ++ <3 RK_PA3 4 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s3m0_sdi: i2s3m0-sdi { ++ rockchip,pins = ++ /* i2s3m0_sdi */ ++ <3 RK_PA6 4 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s3m0_sdo: i2s3m0-sdo { ++ rockchip,pins = ++ /* i2s3m0_sdo */ ++ <3 RK_PA5 4 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s3m1_lrck: i2s3m1-lrck { ++ rockchip,pins = ++ /* i2s3m1_lrck */ ++ <4 RK_PC4 5 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s3m1_mclk: i2s3m1-mclk { ++ rockchip,pins = ++ /* i2s3m1_mclk */ ++ <4 RK_PC2 5 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s3m1_sclk: i2s3m1-sclk { ++ rockchip,pins = ++ /* i2s3m1_sclk */ ++ <4 RK_PC3 5 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s3m1_sdi: i2s3m1-sdi { ++ rockchip,pins = ++ /* i2s3m1_sdi */ ++ <4 RK_PC6 5 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ i2s3m1_sdo: i2s3m1-sdo { ++ rockchip,pins = ++ /* i2s3m1_sdo */ ++ <4 RK_PC5 5 &pcfg_pull_none>; ++ }; ++ }; ++ ++ isp { ++ /omit-if-no-ref/ ++ isp_pins: isp-pins { ++ rockchip,pins = ++ /* isp_flashtrigin */ ++ <4 RK_PB4 4 &pcfg_pull_none>, ++ /* isp_flashtrigout */ ++ <4 RK_PA6 1 &pcfg_pull_none>, ++ /* isp_prelighttrig */ ++ <4 RK_PB1 1 &pcfg_pull_none>; ++ }; ++ }; ++ ++ jtag { ++ /omit-if-no-ref/ ++ jtag_pins: jtag-pins { ++ rockchip,pins = ++ /* jtag_tck */ ++ <1 RK_PD7 2 &pcfg_pull_none>, ++ /* jtag_tms */ ++ <2 RK_PA0 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ lcdc { ++ /omit-if-no-ref/ ++ lcdc_ctl: lcdc-ctl { ++ rockchip,pins = ++ /* lcdc_clk */ ++ <3 RK_PA0 1 &pcfg_pull_none>, ++ /* lcdc_d0 */ ++ <2 RK_PD0 1 &pcfg_pull_none>, ++ /* lcdc_d1 */ ++ <2 RK_PD1 1 &pcfg_pull_none>, ++ /* lcdc_d2 */ ++ <2 RK_PD2 1 &pcfg_pull_none>, ++ /* lcdc_d3 */ ++ <2 RK_PD3 1 &pcfg_pull_none>, ++ /* lcdc_d4 */ ++ <2 RK_PD4 1 &pcfg_pull_none>, ++ /* lcdc_d5 */ ++ <2 RK_PD5 1 &pcfg_pull_none>, ++ /* lcdc_d6 */ ++ <2 RK_PD6 1 &pcfg_pull_none>, ++ /* lcdc_d7 */ ++ <2 RK_PD7 1 &pcfg_pull_none>, ++ /* lcdc_d8 */ ++ <3 RK_PA1 1 &pcfg_pull_none>, ++ /* lcdc_d9 */ ++ <3 RK_PA2 1 &pcfg_pull_none>, ++ /* lcdc_d10 */ ++ <3 RK_PA3 1 &pcfg_pull_none>, ++ /* lcdc_d11 */ ++ <3 RK_PA4 1 &pcfg_pull_none>, ++ /* lcdc_d12 */ ++ <3 RK_PA5 1 &pcfg_pull_none>, ++ /* lcdc_d13 */ ++ <3 RK_PA6 1 &pcfg_pull_none>, ++ /* lcdc_d14 */ ++ <3 RK_PA7 1 &pcfg_pull_none>, ++ /* lcdc_d15 */ ++ <3 RK_PB0 1 &pcfg_pull_none>, ++ /* lcdc_d16 */ ++ <3 RK_PB1 1 &pcfg_pull_none>, ++ /* lcdc_d17 */ ++ <3 RK_PB2 1 &pcfg_pull_none>, ++ /* lcdc_d18 */ ++ <3 RK_PB3 1 &pcfg_pull_none>, ++ /* lcdc_d19 */ ++ <3 RK_PB4 1 &pcfg_pull_none>, ++ /* lcdc_d20 */ ++ <3 RK_PB5 1 &pcfg_pull_none>, ++ /* lcdc_d21 */ ++ <3 RK_PB6 1 &pcfg_pull_none>, ++ /* lcdc_d22 */ ++ <3 RK_PB7 1 &pcfg_pull_none>, ++ /* lcdc_d23 */ ++ <3 RK_PC0 1 &pcfg_pull_none>, ++ /* lcdc_den */ ++ <3 RK_PC3 1 &pcfg_pull_none>, ++ /* lcdc_hsync */ ++ <3 RK_PC1 1 &pcfg_pull_none>, ++ /* lcdc_vsync */ ++ <3 RK_PC2 1 &pcfg_pull_none>; ++ }; ++ }; ++ ++ mcu { ++ /omit-if-no-ref/ ++ mcu_pins: mcu-pins { ++ rockchip,pins = ++ /* mcu_jtagtck */ ++ <0 RK_PB4 4 &pcfg_pull_none>, ++ /* mcu_jtagtdi */ ++ <0 RK_PC1 4 &pcfg_pull_none>, ++ /* mcu_jtagtdo */ ++ <0 RK_PB3 4 &pcfg_pull_none>, ++ /* mcu_jtagtms */ ++ <0 RK_PC2 4 &pcfg_pull_none>, ++ /* mcu_jtagtrstn */ ++ <0 RK_PC3 4 &pcfg_pull_none>; ++ }; ++ }; ++ ++ npu { ++ /omit-if-no-ref/ ++ npu_pins: npu-pins { ++ rockchip,pins = ++ /* npu_avs */ ++ <0 RK_PC1 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ pcie20 { ++ /omit-if-no-ref/ ++ pcie20m0_pins: pcie20m0-pins { ++ rockchip,pins = ++ /* pcie20_clkreqnm0 */ ++ <0 RK_PA5 3 &pcfg_pull_none>, ++ /* pcie20_perstnm0 */ ++ <0 RK_PB6 3 &pcfg_pull_none>, ++ /* pcie20_wakenm0 */ ++ <0 RK_PB5 3 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcie20m1_pins: pcie20m1-pins { ++ rockchip,pins = ++ /* pcie20_clkreqnm1 */ ++ <2 RK_PD0 4 &pcfg_pull_none>, ++ /* pcie20_perstnm1 */ ++ <3 RK_PC1 4 &pcfg_pull_none>, ++ /* pcie20_wakenm1 */ ++ <2 RK_PD1 4 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcie20m2_pins: pcie20m2-pins { ++ rockchip,pins = ++ /* pcie20_clkreqnm2 */ ++ <1 RK_PB0 4 &pcfg_pull_none>, ++ /* pcie20_perstnm2 */ ++ <1 RK_PB2 4 &pcfg_pull_none>, ++ /* pcie20_wakenm2 */ ++ <1 RK_PB1 4 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcie20_buttonrstn: pcie20-buttonrstn { ++ rockchip,pins = ++ /* pcie20_buttonrstn */ ++ <0 RK_PB4 3 &pcfg_pull_none>; ++ }; ++ }; ++ ++ pcie30x1 { ++ /omit-if-no-ref/ ++ pcie30x1m0_pins: pcie30x1m0-pins { ++ rockchip,pins = ++ /* pcie30x1_clkreqnm0 */ ++ <0 RK_PA4 3 &pcfg_pull_none>, ++ /* pcie30x1_perstnm0 */ ++ <0 RK_PC3 3 &pcfg_pull_none>, ++ /* pcie30x1_wakenm0 */ ++ <0 RK_PC2 3 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcie30x1m1_pins: pcie30x1m1-pins { ++ rockchip,pins = ++ /* pcie30x1_clkreqnm1 */ ++ <2 RK_PD2 4 &pcfg_pull_none>, ++ /* pcie30x1_perstnm1 */ ++ <3 RK_PA1 4 &pcfg_pull_none>, ++ /* pcie30x1_wakenm1 */ ++ <2 RK_PD3 4 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcie30x1m2_pins: pcie30x1m2-pins { ++ rockchip,pins = ++ /* pcie30x1_clkreqnm2 */ ++ <1 RK_PA5 4 &pcfg_pull_none>, ++ /* pcie30x1_perstnm2 */ ++ <1 RK_PA2 4 &pcfg_pull_none>, ++ /* pcie30x1_wakenm2 */ ++ <1 RK_PA3 4 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcie30x1_buttonrstn: pcie30x1-buttonrstn { ++ rockchip,pins = ++ /* pcie30x1_buttonrstn */ ++ <0 RK_PB3 3 &pcfg_pull_none>; ++ }; ++ }; ++ ++ pcie30x2 { ++ /omit-if-no-ref/ ++ pcie30x2m0_pins: pcie30x2m0-pins { ++ rockchip,pins = ++ /* pcie30x2_clkreqnm0 */ ++ <0 RK_PA6 2 &pcfg_pull_none>, ++ /* pcie30x2_perstnm0 */ ++ <0 RK_PC6 3 &pcfg_pull_none>, ++ /* pcie30x2_wakenm0 */ ++ <0 RK_PC5 3 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcie30x2m1_pins: pcie30x2m1-pins { ++ rockchip,pins = ++ /* pcie30x2_clkreqnm1 */ ++ <2 RK_PD4 4 &pcfg_pull_none>, ++ /* pcie30x2_perstnm1 */ ++ <2 RK_PD6 4 &pcfg_pull_none>, ++ /* pcie30x2_wakenm1 */ ++ <2 RK_PD5 4 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcie30x2m2_pins: pcie30x2m2-pins { ++ rockchip,pins = ++ /* pcie30x2_clkreqnm2 */ ++ <4 RK_PC2 4 &pcfg_pull_none>, ++ /* pcie30x2_perstnm2 */ ++ <4 RK_PC4 4 &pcfg_pull_none>, ++ /* pcie30x2_wakenm2 */ ++ <4 RK_PC3 4 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcie30x2_buttonrstn: pcie30x2-buttonrstn { ++ rockchip,pins = ++ /* pcie30x2_buttonrstn */ ++ <0 RK_PB0 3 &pcfg_pull_none>; ++ }; ++ }; ++ ++ pdm { ++ /omit-if-no-ref/ ++ pdmm0_clk: pdmm0-clk { ++ rockchip,pins = ++ /* pdm_clk0m0 */ ++ <1 RK_PA6 3 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ pdmm0_clk1: pdmm0-clk1 { ++ rockchip,pins = ++ /* pdmm0_clk1 */ ++ <1 RK_PA4 3 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ pdmm0_sdi0: pdmm0-sdi0 { ++ rockchip,pins = ++ /* pdmm0_sdi0 */ ++ <1 RK_PB3 2 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ pdmm0_sdi1: pdmm0-sdi1 { ++ rockchip,pins = ++ /* pdmm0_sdi1 */ ++ <1 RK_PB2 3 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ pdmm0_sdi2: pdmm0-sdi2 { ++ rockchip,pins = ++ /* pdmm0_sdi2 */ ++ <1 RK_PB1 3 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ pdmm0_sdi3: pdmm0-sdi3 { ++ rockchip,pins = ++ /* pdmm0_sdi3 */ ++ <1 RK_PB0 3 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ pdmm1_clk: pdmm1-clk { ++ rockchip,pins = ++ /* pdm_clk0m1 */ ++ <3 RK_PD6 5 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ pdmm1_clk1: pdmm1-clk1 { ++ rockchip,pins = ++ /* pdmm1_clk1 */ ++ <4 RK_PA0 4 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ pdmm1_sdi0: pdmm1-sdi0 { ++ rockchip,pins = ++ /* pdmm1_sdi0 */ ++ <3 RK_PD7 5 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ pdmm1_sdi1: pdmm1-sdi1 { ++ rockchip,pins = ++ /* pdmm1_sdi1 */ ++ <4 RK_PA1 4 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ pdmm1_sdi2: pdmm1-sdi2 { ++ rockchip,pins = ++ /* pdmm1_sdi2 */ ++ <4 RK_PA2 5 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ pdmm1_sdi3: pdmm1-sdi3 { ++ rockchip,pins = ++ /* pdmm1_sdi3 */ ++ <4 RK_PA3 5 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ pdmm2_clk1: pdmm2-clk1 { ++ rockchip,pins = ++ /* pdmm2_clk1 */ ++ <3 RK_PC4 5 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ pdmm2_sdi0: pdmm2-sdi0 { ++ rockchip,pins = ++ /* pdmm2_sdi0 */ ++ <3 RK_PB3 5 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ pdmm2_sdi1: pdmm2-sdi1 { ++ rockchip,pins = ++ /* pdmm2_sdi1 */ ++ <3 RK_PB4 5 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ pdmm2_sdi2: pdmm2-sdi2 { ++ rockchip,pins = ++ /* pdmm2_sdi2 */ ++ <3 RK_PB7 5 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ pdmm2_sdi3: pdmm2-sdi3 { ++ rockchip,pins = ++ /* pdmm2_sdi3 */ ++ <3 RK_PC0 5 &pcfg_pull_none>; ++ }; ++ }; ++ ++ pmic { ++ /omit-if-no-ref/ ++ pmic_pins: pmic-pins { ++ rockchip,pins = ++ /* pmic_sleep */ ++ <0 RK_PA2 1 &pcfg_pull_none>; ++ }; ++ }; ++ ++ pmu { ++ /omit-if-no-ref/ ++ pmu_pins: pmu-pins { ++ rockchip,pins = ++ /* pmu_debug0 */ ++ <0 RK_PA5 4 &pcfg_pull_none>, ++ /* pmu_debug1 */ ++ <0 RK_PA6 3 &pcfg_pull_none>, ++ /* pmu_debug2 */ ++ <0 RK_PC4 4 &pcfg_pull_none>, ++ /* pmu_debug3 */ ++ <0 RK_PC5 4 &pcfg_pull_none>, ++ /* pmu_debug4 */ ++ <0 RK_PC6 4 &pcfg_pull_none>, ++ /* pmu_debug5 */ ++ <0 RK_PC7 4 &pcfg_pull_none>; ++ }; ++ }; ++ ++ pwm0 { ++ /omit-if-no-ref/ ++ pwm0m0_pins: pwm0m0-pins { ++ rockchip,pins = ++ /* pwm0_m0 */ ++ <0 RK_PB7 1 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ pwm0m1_pins: pwm0m1-pins { ++ rockchip,pins = ++ /* pwm0_m1 */ ++ <0 RK_PC7 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ pwm1 { ++ /omit-if-no-ref/ ++ pwm1m0_pins: pwm1m0-pins { ++ rockchip,pins = ++ /* pwm1_m0 */ ++ <0 RK_PC0 1 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ pwm1m1_pins: pwm1m1-pins { ++ rockchip,pins = ++ /* pwm1_m1 */ ++ <0 RK_PB5 4 &pcfg_pull_none>; ++ }; ++ }; ++ ++ pwm2 { ++ /omit-if-no-ref/ ++ pwm2m0_pins: pwm2m0-pins { ++ rockchip,pins = ++ /* pwm2_m0 */ ++ <0 RK_PC1 1 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ pwm2m1_pins: pwm2m1-pins { ++ rockchip,pins = ++ /* pwm2_m1 */ ++ <0 RK_PB6 4 &pcfg_pull_none>; ++ }; ++ }; ++ ++ pwm3 { ++ /omit-if-no-ref/ ++ pwm3_pins: pwm3-pins { ++ rockchip,pins = ++ /* pwm3_ir */ ++ <0 RK_PC2 1 &pcfg_pull_none>; ++ }; ++ }; ++ ++ pwm4 { ++ /omit-if-no-ref/ ++ pwm4_pins: pwm4-pins { ++ rockchip,pins = ++ /* pwm4 */ ++ <0 RK_PC3 1 &pcfg_pull_none>; ++ }; ++ }; ++ ++ pwm5 { ++ /omit-if-no-ref/ ++ pwm5_pins: pwm5-pins { ++ rockchip,pins = ++ /* pwm5 */ ++ <0 RK_PC4 1 &pcfg_pull_none>; ++ }; ++ }; ++ ++ pwm6 { ++ /omit-if-no-ref/ ++ pwm6_pins: pwm6-pins { ++ rockchip,pins = ++ /* pwm6 */ ++ <0 RK_PC5 1 &pcfg_pull_none>; ++ }; ++ }; ++ ++ pwm7 { ++ /omit-if-no-ref/ ++ pwm7_pins: pwm7-pins { ++ rockchip,pins = ++ /* pwm7_ir */ ++ <0 RK_PC6 1 &pcfg_pull_none>; ++ }; ++ }; ++ ++ pwm8 { ++ /omit-if-no-ref/ ++ pwm8m0_pins: pwm8m0-pins { ++ rockchip,pins = ++ /* pwm8_m0 */ ++ <3 RK_PB1 5 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ pwm8m1_pins: pwm8m1-pins { ++ rockchip,pins = ++ /* pwm8_m1 */ ++ <1 RK_PD5 4 &pcfg_pull_none>; ++ }; ++ }; ++ ++ pwm9 { ++ /omit-if-no-ref/ ++ pwm9m0_pins: pwm9m0-pins { ++ rockchip,pins = ++ /* pwm9_m0 */ ++ <3 RK_PB2 5 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ pwm9m1_pins: pwm9m1-pins { ++ rockchip,pins = ++ /* pwm9_m1 */ ++ <1 RK_PD6 4 &pcfg_pull_none>; ++ }; ++ }; ++ ++ pwm10 { ++ /omit-if-no-ref/ ++ pwm10m0_pins: pwm10m0-pins { ++ rockchip,pins = ++ /* pwm10_m0 */ ++ <3 RK_PB5 5 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ pwm10m1_pins: pwm10m1-pins { ++ rockchip,pins = ++ /* pwm10_m1 */ ++ <2 RK_PA1 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ pwm11 { ++ /omit-if-no-ref/ ++ pwm11m0_pins: pwm11m0-pins { ++ rockchip,pins = ++ /* pwm11_irm0 */ ++ <3 RK_PB6 5 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ pwm11m1_pins: pwm11m1-pins { ++ rockchip,pins = ++ /* pwm11_irm1 */ ++ <4 RK_PC0 3 &pcfg_pull_none>; ++ }; ++ }; ++ ++ pwm12 { ++ /omit-if-no-ref/ ++ pwm12m0_pins: pwm12m0-pins { ++ rockchip,pins = ++ /* pwm12_m0 */ ++ <3 RK_PB7 2 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ pwm12m1_pins: pwm12m1-pins { ++ rockchip,pins = ++ /* pwm12_m1 */ ++ <4 RK_PC5 1 &pcfg_pull_none>; ++ }; ++ }; ++ ++ pwm13 { ++ /omit-if-no-ref/ ++ pwm13m0_pins: pwm13m0-pins { ++ rockchip,pins = ++ /* pwm13_m0 */ ++ <3 RK_PC0 2 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ pwm13m1_pins: pwm13m1-pins { ++ rockchip,pins = ++ /* pwm13_m1 */ ++ <4 RK_PC6 1 &pcfg_pull_none>; ++ }; ++ }; ++ ++ pwm14 { ++ /omit-if-no-ref/ ++ pwm14m0_pins: pwm14m0-pins { ++ rockchip,pins = ++ /* pwm14_m0 */ ++ <3 RK_PC4 1 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ pwm14m1_pins: pwm14m1-pins { ++ rockchip,pins = ++ /* pwm14_m1 */ ++ <4 RK_PC2 1 &pcfg_pull_none>; ++ }; ++ }; ++ ++ pwm15 { ++ /omit-if-no-ref/ ++ pwm15m0_pins: pwm15m0-pins { ++ rockchip,pins = ++ /* pwm15_irm0 */ ++ <3 RK_PC5 1 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ pwm15m1_pins: pwm15m1-pins { ++ rockchip,pins = ++ /* pwm15_irm1 */ ++ <4 RK_PC3 1 &pcfg_pull_none>; ++ }; ++ }; ++ ++ refclk { ++ /omit-if-no-ref/ ++ refclk_pins: refclk-pins { ++ rockchip,pins = ++ /* refclk_ou */ ++ <0 RK_PA0 1 &pcfg_pull_none>; ++ }; ++ }; ++ ++ sata { ++ /omit-if-no-ref/ ++ sata_pins: sata-pins { ++ rockchip,pins = ++ /* sata_cpdet */ ++ <0 RK_PA4 2 &pcfg_pull_none>, ++ /* sata_cppod */ ++ <0 RK_PA6 1 &pcfg_pull_none>, ++ /* sata_mpswitch */ ++ <0 RK_PA5 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ sata0 { ++ /omit-if-no-ref/ ++ sata0_pins: sata0-pins { ++ rockchip,pins = ++ /* sata0_actled */ ++ <4 RK_PC6 3 &pcfg_pull_none>; ++ }; ++ }; ++ ++ sata1 { ++ /omit-if-no-ref/ ++ sata1_pins: sata1-pins { ++ rockchip,pins = ++ /* sata1_actled */ ++ <4 RK_PC5 3 &pcfg_pull_none>; ++ }; ++ }; ++ ++ sata2 { ++ /omit-if-no-ref/ ++ sata2_pins: sata2-pins { ++ rockchip,pins = ++ /* sata2_actled */ ++ <4 RK_PC4 3 &pcfg_pull_none>; ++ }; ++ }; ++ ++ scr { ++ /omit-if-no-ref/ ++ scr_pins: scr-pins { ++ rockchip,pins = ++ /* scr_clk */ ++ <1 RK_PA2 3 &pcfg_pull_none>, ++ /* scr_det */ ++ <1 RK_PA7 3 &pcfg_pull_up>, ++ /* scr_io */ ++ <1 RK_PA3 3 &pcfg_pull_up>, ++ /* scr_rst */ ++ <1 RK_PA5 3 &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdmmc0 { ++ /omit-if-no-ref/ ++ sdmmc0_bus4: sdmmc0-bus4 { ++ rockchip,pins = ++ /* sdmmc0_d0 */ ++ <1 RK_PD5 1 &pcfg_pull_up_drv_level_2>, ++ /* sdmmc0_d1 */ ++ <1 RK_PD6 1 &pcfg_pull_up_drv_level_2>, ++ /* sdmmc0_d2 */ ++ <1 RK_PD7 1 &pcfg_pull_up_drv_level_2>, ++ /* sdmmc0_d3 */ ++ <2 RK_PA0 1 &pcfg_pull_up_drv_level_2>; ++ }; ++ ++ /omit-if-no-ref/ ++ sdmmc0_clk: sdmmc0-clk { ++ rockchip,pins = ++ /* sdmmc0_clk */ ++ <2 RK_PA2 1 &pcfg_pull_up_drv_level_2>; ++ }; ++ ++ /omit-if-no-ref/ ++ sdmmc0_cmd: sdmmc0-cmd { ++ rockchip,pins = ++ /* sdmmc0_cmd */ ++ <2 RK_PA1 1 &pcfg_pull_up_drv_level_2>; ++ }; ++ ++ /omit-if-no-ref/ ++ sdmmc0_det: sdmmc0-det { ++ rockchip,pins = ++ /* sdmmc0_det */ ++ <0 RK_PA4 1 &pcfg_pull_up>; ++ }; ++ ++ /omit-if-no-ref/ ++ sdmmc0_pwren: sdmmc0-pwren { ++ rockchip,pins = ++ /* sdmmc0_pwren */ ++ <0 RK_PA5 1 &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdmmc1 { ++ /omit-if-no-ref/ ++ sdmmc1_bus4: sdmmc1-bus4 { ++ rockchip,pins = ++ /* sdmmc1_d0 */ ++ <2 RK_PA3 1 &pcfg_pull_up_drv_level_2>, ++ /* sdmmc1_d1 */ ++ <2 RK_PA4 1 &pcfg_pull_up_drv_level_2>, ++ /* sdmmc1_d2 */ ++ <2 RK_PA5 1 &pcfg_pull_up_drv_level_2>, ++ /* sdmmc1_d3 */ ++ <2 RK_PA6 1 &pcfg_pull_up_drv_level_2>; ++ }; ++ ++ /omit-if-no-ref/ ++ sdmmc1_clk: sdmmc1-clk { ++ rockchip,pins = ++ /* sdmmc1_clk */ ++ <2 RK_PB0 1 &pcfg_pull_up_drv_level_2>; ++ }; ++ ++ /omit-if-no-ref/ ++ sdmmc1_cmd: sdmmc1-cmd { ++ rockchip,pins = ++ /* sdmmc1_cmd */ ++ <2 RK_PA7 1 &pcfg_pull_up_drv_level_2>; ++ }; ++ ++ /omit-if-no-ref/ ++ sdmmc1_det: sdmmc1-det { ++ rockchip,pins = ++ /* sdmmc1_det */ ++ <2 RK_PB2 1 &pcfg_pull_up>; ++ }; ++ ++ /omit-if-no-ref/ ++ sdmmc1_pwren: sdmmc1-pwren { ++ rockchip,pins = ++ /* sdmmc1_pwren */ ++ <2 RK_PB1 1 &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdmmc2 { ++ /omit-if-no-ref/ ++ sdmmc2m0_bus4: sdmmc2m0-bus4 { ++ rockchip,pins = ++ /* sdmmc2_d0m0 */ ++ <3 RK_PC6 3 &pcfg_pull_up_drv_level_2>, ++ /* sdmmc2_d1m0 */ ++ <3 RK_PC7 3 &pcfg_pull_up_drv_level_2>, ++ /* sdmmc2_d2m0 */ ++ <3 RK_PD0 3 &pcfg_pull_up_drv_level_2>, ++ /* sdmmc2_d3m0 */ ++ <3 RK_PD1 3 &pcfg_pull_up_drv_level_2>; ++ }; ++ ++ /omit-if-no-ref/ ++ sdmmc2m0_clk: sdmmc2m0-clk { ++ rockchip,pins = ++ /* sdmmc2_clkm0 */ ++ <3 RK_PD3 3 &pcfg_pull_up_drv_level_2>; ++ }; ++ ++ /omit-if-no-ref/ ++ sdmmc2m0_cmd: sdmmc2m0-cmd { ++ rockchip,pins = ++ /* sdmmc2_cmdm0 */ ++ <3 RK_PD2 3 &pcfg_pull_up_drv_level_2>; ++ }; ++ ++ /omit-if-no-ref/ ++ sdmmc2m0_det: sdmmc2m0-det { ++ rockchip,pins = ++ /* sdmmc2_detm0 */ ++ <3 RK_PD4 3 &pcfg_pull_up>; ++ }; ++ ++ /omit-if-no-ref/ ++ sdmmc2m0_pwren: sdmmc2m0-pwren { ++ rockchip,pins = ++ /* sdmmc2m0_pwren */ ++ <3 RK_PD5 3 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ sdmmc2m1_bus4: sdmmc2m1-bus4 { ++ rockchip,pins = ++ /* sdmmc2_d0m1 */ ++ <3 RK_PA1 5 &pcfg_pull_up_drv_level_2>, ++ /* sdmmc2_d1m1 */ ++ <3 RK_PA2 5 &pcfg_pull_up_drv_level_2>, ++ /* sdmmc2_d2m1 */ ++ <3 RK_PA3 5 &pcfg_pull_up_drv_level_2>, ++ /* sdmmc2_d3m1 */ ++ <3 RK_PA4 5 &pcfg_pull_up_drv_level_2>; ++ }; ++ ++ /omit-if-no-ref/ ++ sdmmc2m1_clk: sdmmc2m1-clk { ++ rockchip,pins = ++ /* sdmmc2_clkm1 */ ++ <3 RK_PA6 5 &pcfg_pull_up_drv_level_2>; ++ }; ++ ++ /omit-if-no-ref/ ++ sdmmc2m1_cmd: sdmmc2m1-cmd { ++ rockchip,pins = ++ /* sdmmc2_cmdm1 */ ++ <3 RK_PA5 5 &pcfg_pull_up_drv_level_2>; ++ }; ++ ++ /omit-if-no-ref/ ++ sdmmc2m1_det: sdmmc2m1-det { ++ rockchip,pins = ++ /* sdmmc2_detm1 */ ++ <3 RK_PA7 4 &pcfg_pull_up>; ++ }; ++ ++ /omit-if-no-ref/ ++ sdmmc2m1_pwren: sdmmc2m1-pwren { ++ rockchip,pins = ++ /* sdmmc2m1_pwren */ ++ <3 RK_PB0 4 &pcfg_pull_none>; ++ }; ++ }; ++ ++ spdif { ++ /omit-if-no-ref/ ++ spdifm0_tx: spdifm0-tx { ++ rockchip,pins = ++ /* spdifm0_tx */ ++ <1 RK_PA4 4 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ spdifm1_tx: spdifm1-tx { ++ rockchip,pins = ++ /* spdifm1_tx */ ++ <3 RK_PC5 2 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ spdifm2_tx: spdifm2-tx { ++ rockchip,pins = ++ /* spdifm2_tx */ ++ <4 RK_PC4 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ spi0 { ++ /omit-if-no-ref/ ++ spi0m0_pins: spi0m0-pins { ++ rockchip,pins = ++ /* spi0_clkm0 */ ++ <0 RK_PB5 2 &pcfg_pull_none>, ++ /* spi0_misom0 */ ++ <0 RK_PC5 2 &pcfg_pull_none>, ++ /* spi0_mosim0 */ ++ <0 RK_PB6 2 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ spi0m0_cs0: spi0m0-cs0 { ++ rockchip,pins = ++ /* spi0_cs0m0 */ ++ <0 RK_PC6 2 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ spi0m0_cs1: spi0m0-cs1 { ++ rockchip,pins = ++ /* spi0_cs1m0 */ ++ <0 RK_PC4 2 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ spi0m1_pins: spi0m1-pins { ++ rockchip,pins = ++ /* spi0_clkm1 */ ++ <2 RK_PD3 3 &pcfg_pull_none>, ++ /* spi0_misom1 */ ++ <2 RK_PD0 3 &pcfg_pull_none>, ++ /* spi0_mosim1 */ ++ <2 RK_PD1 3 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ spi0m1_cs0: spi0m1-cs0 { ++ rockchip,pins = ++ /* spi0_cs0m1 */ ++ <2 RK_PD2 3 &pcfg_pull_none>; ++ }; ++ }; ++ ++ spi1 { ++ /omit-if-no-ref/ ++ spi1m0_pins: spi1m0-pins { ++ rockchip,pins = ++ /* spi1_clkm0 */ ++ <2 RK_PB5 3 &pcfg_pull_none>, ++ /* spi1_misom0 */ ++ <2 RK_PB6 3 &pcfg_pull_none>, ++ /* spi1_mosim0 */ ++ <2 RK_PB7 4 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ spi1m0_cs0: spi1m0-cs0 { ++ rockchip,pins = ++ /* spi1_cs0m0 */ ++ <2 RK_PC0 4 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ spi1m0_cs1: spi1m0-cs1 { ++ rockchip,pins = ++ /* spi1_cs1m0 */ ++ <2 RK_PC6 3 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ spi1m1_pins: spi1m1-pins { ++ rockchip,pins = ++ /* spi1_clkm1 */ ++ <3 RK_PC3 3 &pcfg_pull_none>, ++ /* spi1_misom1 */ ++ <3 RK_PC2 3 &pcfg_pull_none>, ++ /* spi1_mosim1 */ ++ <3 RK_PC1 3 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ spi1m1_cs0: spi1m1-cs0 { ++ rockchip,pins = ++ /* spi1_cs0m1 */ ++ <3 RK_PA1 3 &pcfg_pull_none>; ++ }; ++ }; ++ ++ spi2 { ++ /omit-if-no-ref/ ++ spi2m0_pins: spi2m0-pins { ++ rockchip,pins = ++ /* spi2_clkm0 */ ++ <2 RK_PC1 4 &pcfg_pull_none>, ++ /* spi2_misom0 */ ++ <2 RK_PC2 4 &pcfg_pull_none>, ++ /* spi2_mosim0 */ ++ <2 RK_PC3 4 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ spi2m0_cs0: spi2m0-cs0 { ++ rockchip,pins = ++ /* spi2_cs0m0 */ ++ <2 RK_PC4 4 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ spi2m0_cs1: spi2m0-cs1 { ++ rockchip,pins = ++ /* spi2_cs1m0 */ ++ <2 RK_PC5 4 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ spi2m1_pins: spi2m1-pins { ++ rockchip,pins = ++ /* spi2_clkm1 */ ++ <3 RK_PA0 3 &pcfg_pull_none>, ++ /* spi2_misom1 */ ++ <2 RK_PD7 3 &pcfg_pull_none>, ++ /* spi2_mosim1 */ ++ <2 RK_PD6 3 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ spi2m1_cs0: spi2m1-cs0 { ++ rockchip,pins = ++ /* spi2_cs0m1 */ ++ <2 RK_PD5 3 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ spi2m1_cs1: spi2m1-cs1 { ++ rockchip,pins = ++ /* spi2_cs1m1 */ ++ <2 RK_PD4 3 &pcfg_pull_none>; ++ }; ++ }; ++ ++ spi3 { ++ /omit-if-no-ref/ ++ spi3m0_pins: spi3m0-pins { ++ rockchip,pins = ++ /* spi3_clkm0 */ ++ <4 RK_PB3 4 &pcfg_pull_none>, ++ /* spi3_misom0 */ ++ <4 RK_PB0 4 &pcfg_pull_none>, ++ /* spi3_mosim0 */ ++ <4 RK_PB2 4 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ spi3m0_cs0: spi3m0-cs0 { ++ rockchip,pins = ++ /* spi3_cs0m0 */ ++ <4 RK_PA6 4 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ spi3m0_cs1: spi3m0-cs1 { ++ rockchip,pins = ++ /* spi3_cs1m0 */ ++ <4 RK_PA7 4 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ spi3m1_pins: spi3m1-pins { ++ rockchip,pins = ++ /* spi3_clkm1 */ ++ <4 RK_PC2 2 &pcfg_pull_none>, ++ /* spi3_misom1 */ ++ <4 RK_PC5 2 &pcfg_pull_none>, ++ /* spi3_mosim1 */ ++ <4 RK_PC3 2 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ spi3m1_cs0: spi3m1-cs0 { ++ rockchip,pins = ++ /* spi3_cs0m1 */ ++ <4 RK_PC6 2 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ spi3m1_cs1: spi3m1-cs1 { ++ rockchip,pins = ++ /* spi3_cs1m1 */ ++ <4 RK_PD1 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ tsadc { ++ /omit-if-no-ref/ ++ tsadcm0_shut: tsadcm0-shut { ++ rockchip,pins = ++ /* tsadcm0_shut */ ++ <0 RK_PA1 1 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ tsadcm1_shut: tsadcm1-shut { ++ rockchip,pins = ++ /* tsadcm1_shut */ ++ <0 RK_PA2 2 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ tsadc_shutorg: tsadc-shutorg { ++ rockchip,pins = ++ /* tsadc_shutorg */ ++ <0 RK_PA1 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ uart0 { ++ /omit-if-no-ref/ ++ uart0_xfer: uart0-xfer { ++ rockchip,pins = ++ /* uart0_rx */ ++ <0 RK_PC0 3 &pcfg_pull_up>, ++ /* uart0_tx */ ++ <0 RK_PC1 3 &pcfg_pull_up>; ++ }; ++ ++ /omit-if-no-ref/ ++ uart0_ctsn: uart0-ctsn { ++ rockchip,pins = ++ /* uart0_ctsn */ ++ <0 RK_PC7 3 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ uart0_rtsn: uart0-rtsn { ++ rockchip,pins = ++ /* uart0_rtsn */ ++ <0 RK_PC4 3 &pcfg_pull_none>; ++ }; ++ }; ++ ++ uart1 { ++ /omit-if-no-ref/ ++ uart1m0_xfer: uart1m0-xfer { ++ rockchip,pins = ++ /* uart1_rxm0 */ ++ <2 RK_PB3 2 &pcfg_pull_up>, ++ /* uart1_txm0 */ ++ <2 RK_PB4 2 &pcfg_pull_up>; ++ }; ++ ++ /omit-if-no-ref/ ++ uart1m0_ctsn: uart1m0-ctsn { ++ rockchip,pins = ++ /* uart1m0_ctsn */ ++ <2 RK_PB6 2 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ uart1m0_rtsn: uart1m0-rtsn { ++ rockchip,pins = ++ /* uart1m0_rtsn */ ++ <2 RK_PB5 2 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ uart1m1_xfer: uart1m1-xfer { ++ rockchip,pins = ++ /* uart1_rxm1 */ ++ <3 RK_PD7 4 &pcfg_pull_up>, ++ /* uart1_txm1 */ ++ <3 RK_PD6 4 &pcfg_pull_up>; ++ }; ++ ++ /omit-if-no-ref/ ++ uart1m1_ctsn: uart1m1-ctsn { ++ rockchip,pins = ++ /* uart1m1_ctsn */ ++ <4 RK_PC1 4 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ uart1m1_rtsn: uart1m1-rtsn { ++ rockchip,pins = ++ /* uart1m1_rtsn */ ++ <4 RK_PB6 4 &pcfg_pull_none>; ++ }; ++ }; ++ ++ uart2 { ++ /omit-if-no-ref/ ++ uart2m0_xfer: uart2m0-xfer { ++ rockchip,pins = ++ /* uart2_rxm0 */ ++ <0 RK_PD0 1 &pcfg_pull_up>, ++ /* uart2_txm0 */ ++ <0 RK_PD1 1 &pcfg_pull_up>; ++ }; ++ ++ /omit-if-no-ref/ ++ uart2m1_xfer: uart2m1-xfer { ++ rockchip,pins = ++ /* uart2_rxm1 */ ++ <1 RK_PD6 2 &pcfg_pull_up>, ++ /* uart2_txm1 */ ++ <1 RK_PD5 2 &pcfg_pull_up>; ++ }; ++ }; ++ ++ uart3 { ++ /omit-if-no-ref/ ++ uart3m0_xfer: uart3m0-xfer { ++ rockchip,pins = ++ /* uart3_rxm0 */ ++ <1 RK_PA0 2 &pcfg_pull_up>, ++ /* uart3_txm0 */ ++ <1 RK_PA1 2 &pcfg_pull_up>; ++ }; ++ ++ /omit-if-no-ref/ ++ uart3m0_ctsn: uart3m0-ctsn { ++ rockchip,pins = ++ /* uart3m0_ctsn */ ++ <1 RK_PA3 2 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ uart3m0_rtsn: uart3m0-rtsn { ++ rockchip,pins = ++ /* uart3m0_rtsn */ ++ <1 RK_PA2 2 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ uart3m1_xfer: uart3m1-xfer { ++ rockchip,pins = ++ /* uart3_rxm1 */ ++ <3 RK_PC0 4 &pcfg_pull_up>, ++ /* uart3_txm1 */ ++ <3 RK_PB7 4 &pcfg_pull_up>; ++ }; ++ }; ++ ++ uart4 { ++ /omit-if-no-ref/ ++ uart4m0_xfer: uart4m0-xfer { ++ rockchip,pins = ++ /* uart4_rxm0 */ ++ <1 RK_PA4 2 &pcfg_pull_up>, ++ /* uart4_txm0 */ ++ <1 RK_PA6 2 &pcfg_pull_up>; ++ }; ++ ++ /omit-if-no-ref/ ++ uart4m0_ctsn: uart4m0-ctsn { ++ rockchip,pins = ++ /* uart4m0_ctsn */ ++ <1 RK_PA7 2 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ uart4m0_rtsn: uart4m0-rtsn { ++ rockchip,pins = ++ /* uart4m0_rtsn */ ++ <1 RK_PA5 2 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ uart4m1_xfer: uart4m1-xfer { ++ rockchip,pins = ++ /* uart4_rxm1 */ ++ <3 RK_PB1 4 &pcfg_pull_up>, ++ /* uart4_txm1 */ ++ <3 RK_PB2 4 &pcfg_pull_up>; ++ }; ++ }; ++ ++ uart5 { ++ /omit-if-no-ref/ ++ uart5m0_xfer: uart5m0-xfer { ++ rockchip,pins = ++ /* uart5_rxm0 */ ++ <2 RK_PA1 3 &pcfg_pull_up>, ++ /* uart5_txm0 */ ++ <2 RK_PA2 3 &pcfg_pull_up>; ++ }; ++ ++ /omit-if-no-ref/ ++ uart5m0_ctsn: uart5m0-ctsn { ++ rockchip,pins = ++ /* uart5m0_ctsn */ ++ <1 RK_PD7 3 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ uart5m0_rtsn: uart5m0-rtsn { ++ rockchip,pins = ++ /* uart5m0_rtsn */ ++ <2 RK_PA0 3 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ uart5m1_xfer: uart5m1-xfer { ++ rockchip,pins = ++ /* uart5_rxm1 */ ++ <3 RK_PC3 4 &pcfg_pull_up>, ++ /* uart5_txm1 */ ++ <3 RK_PC2 4 &pcfg_pull_up>; ++ }; ++ }; ++ ++ uart6 { ++ /omit-if-no-ref/ ++ uart6m0_xfer: uart6m0-xfer { ++ rockchip,pins = ++ /* uart6_rxm0 */ ++ <2 RK_PA3 3 &pcfg_pull_up>, ++ /* uart6_txm0 */ ++ <2 RK_PA4 3 &pcfg_pull_up>; ++ }; ++ ++ /omit-if-no-ref/ ++ uart6m0_ctsn: uart6m0-ctsn { ++ rockchip,pins = ++ /* uart6m0_ctsn */ ++ <2 RK_PC0 3 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ uart6m0_rtsn: uart6m0-rtsn { ++ rockchip,pins = ++ /* uart6m0_rtsn */ ++ <2 RK_PB7 3 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ uart6m1_xfer: uart6m1-xfer { ++ rockchip,pins = ++ /* uart6_rxm1 */ ++ <1 RK_PD6 3 &pcfg_pull_up>, ++ /* uart6_txm1 */ ++ <1 RK_PD5 3 &pcfg_pull_up>; ++ }; ++ }; ++ ++ uart7 { ++ /omit-if-no-ref/ ++ uart7m0_xfer: uart7m0-xfer { ++ rockchip,pins = ++ /* uart7_rxm0 */ ++ <2 RK_PA5 3 &pcfg_pull_up>, ++ /* uart7_txm0 */ ++ <2 RK_PA6 3 &pcfg_pull_up>; ++ }; ++ ++ /omit-if-no-ref/ ++ uart7m0_ctsn: uart7m0-ctsn { ++ rockchip,pins = ++ /* uart7m0_ctsn */ ++ <2 RK_PC2 3 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ uart7m0_rtsn: uart7m0-rtsn { ++ rockchip,pins = ++ /* uart7m0_rtsn */ ++ <2 RK_PC1 3 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ uart7m1_xfer: uart7m1-xfer { ++ rockchip,pins = ++ /* uart7_rxm1 */ ++ <3 RK_PC5 4 &pcfg_pull_up>, ++ /* uart7_txm1 */ ++ <3 RK_PC4 4 &pcfg_pull_up>; ++ }; ++ ++ /omit-if-no-ref/ ++ uart7m2_xfer: uart7m2-xfer { ++ rockchip,pins = ++ /* uart7_rxm2 */ ++ <4 RK_PA3 4 &pcfg_pull_up>, ++ /* uart7_txm2 */ ++ <4 RK_PA2 4 &pcfg_pull_up>; ++ }; ++ }; ++ ++ uart8 { ++ /omit-if-no-ref/ ++ uart8m0_xfer: uart8m0-xfer { ++ rockchip,pins = ++ /* uart8_rxm0 */ ++ <2 RK_PC6 2 &pcfg_pull_up>, ++ /* uart8_txm0 */ ++ <2 RK_PC5 3 &pcfg_pull_up>; ++ }; ++ ++ /omit-if-no-ref/ ++ uart8m0_ctsn: uart8m0-ctsn { ++ rockchip,pins = ++ /* uart8m0_ctsn */ ++ <2 RK_PB2 3 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ uart8m0_rtsn: uart8m0-rtsn { ++ rockchip,pins = ++ /* uart8m0_rtsn */ ++ <2 RK_PB1 3 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ uart8m1_xfer: uart8m1-xfer { ++ rockchip,pins = ++ /* uart8_rxm1 */ ++ <3 RK_PA0 4 &pcfg_pull_up>, ++ /* uart8_txm1 */ ++ <2 RK_PD7 4 &pcfg_pull_up>; ++ }; ++ }; ++ ++ uart9 { ++ /omit-if-no-ref/ ++ uart9m0_xfer: uart9m0-xfer { ++ rockchip,pins = ++ /* uart9_rxm0 */ ++ <2 RK_PA7 3 &pcfg_pull_up>, ++ /* uart9_txm0 */ ++ <2 RK_PB0 3 &pcfg_pull_up>; ++ }; ++ ++ /omit-if-no-ref/ ++ uart9m0_ctsn: uart9m0-ctsn { ++ rockchip,pins = ++ /* uart9m0_ctsn */ ++ <2 RK_PC4 3 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ uart9m0_rtsn: uart9m0-rtsn { ++ rockchip,pins = ++ /* uart9m0_rtsn */ ++ <2 RK_PC3 3 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ uart9m1_xfer: uart9m1-xfer { ++ rockchip,pins = ++ /* uart9_rxm1 */ ++ <4 RK_PC6 4 &pcfg_pull_up>, ++ /* uart9_txm1 */ ++ <4 RK_PC5 4 &pcfg_pull_up>; ++ }; ++ ++ /omit-if-no-ref/ ++ uart9m2_xfer: uart9m2-xfer { ++ rockchip,pins = ++ /* uart9_rxm2 */ ++ <4 RK_PA5 4 &pcfg_pull_up>, ++ /* uart9_txm2 */ ++ <4 RK_PA4 4 &pcfg_pull_up>; ++ }; ++ }; ++ ++ vop { ++ /omit-if-no-ref/ ++ vopm0_pins: vopm0-pins { ++ rockchip,pins = ++ /* vop_pwmm0 */ ++ <0 RK_PC3 2 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ vopm1_pins: vopm1-pins { ++ rockchip,pins = ++ /* vop_pwmm1 */ ++ <3 RK_PC4 2 &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++/* ++ * This part is edited handly. ++ */ ++&pinctrl { ++ spi0-hs { ++ /omit-if-no-ref/ ++ spi0m0_pins_hs: spi0m0-pins { ++ rockchip,pins = ++ /* spi0_clkm0 */ ++ <0 RK_PB5 2 &pcfg_pull_up_drv_level_1>, ++ /* spi0_misom0 */ ++ <0 RK_PC5 2 &pcfg_pull_up_drv_level_1>, ++ /* spi0_mosim0 */ ++ <0 RK_PB6 2 &pcfg_pull_up_drv_level_1>; ++ }; ++ ++ /omit-if-no-ref/ ++ spi0m0_cs0_hs: spi0m0-cs0 { ++ rockchip,pins = ++ /* spi0_cs0m0 */ ++ <0 RK_PC6 2 &pcfg_pull_up_drv_level_1>; ++ }; ++ ++ /omit-if-no-ref/ ++ spi0m0_cs1_hs: spi0m0-cs1 { ++ rockchip,pins = ++ /* spi0_cs1m0 */ ++ <0 RK_PC4 2 &pcfg_pull_up_drv_level_1>; ++ }; ++ ++ /omit-if-no-ref/ ++ spi0m1_pins_hs: spi0m1-pins { ++ rockchip,pins = ++ /* spi0_clkm1 */ ++ <2 RK_PD3 3 &pcfg_pull_up_drv_level_1>, ++ /* spi0_misom1 */ ++ <2 RK_PD0 3 &pcfg_pull_up_drv_level_1>, ++ /* spi0_mosim1 */ ++ <2 RK_PD1 3 &pcfg_pull_up_drv_level_1>; ++ }; ++ ++ /omit-if-no-ref/ ++ spi0m1_cs0_hs: spi0m1-cs0 { ++ rockchip,pins = ++ /* spi0_cs0m1 */ ++ <2 RK_PD2 3 &pcfg_pull_up_drv_level_1>; ++ }; ++ }; ++ ++ spi1-hs { ++ /omit-if-no-ref/ ++ spi1m0_pins_hs: spi1m0-pins { ++ rockchip,pins = ++ /* spi1_clkm0 */ ++ <2 RK_PB5 3 &pcfg_pull_up_drv_level_1>, ++ /* spi1_misom0 */ ++ <2 RK_PB6 3 &pcfg_pull_up_drv_level_1>, ++ /* spi1_mosim0 */ ++ <2 RK_PB7 4 &pcfg_pull_up_drv_level_1>; ++ }; ++ ++ /omit-if-no-ref/ ++ spi1m0_cs0_hs: spi1m0-cs0 { ++ rockchip,pins = ++ /* spi1_cs0m0 */ ++ <2 RK_PC0 4 &pcfg_pull_up_drv_level_1>; ++ }; ++ ++ /omit-if-no-ref/ ++ spi1m0_cs1_hs: spi1m0-cs1 { ++ rockchip,pins = ++ /* spi1_cs1m0 */ ++ <2 RK_PC6 3 &pcfg_pull_up_drv_level_1>; ++ }; ++ ++ /omit-if-no-ref/ ++ spi1m1_pins_hs: spi1m1-pins { ++ rockchip,pins = ++ /* spi1_clkm1 */ ++ <3 RK_PC3 3 &pcfg_pull_up_drv_level_1>, ++ /* spi1_misom1 */ ++ <3 RK_PC2 3 &pcfg_pull_up_drv_level_1>, ++ /* spi1_mosim1 */ ++ <3 RK_PC1 3 &pcfg_pull_up_drv_level_1>; ++ }; ++ ++ /omit-if-no-ref/ ++ spi1m1_cs0_hs: spi1m1-cs0 { ++ rockchip,pins = ++ /* spi1_cs0m1 */ ++ <3 RK_PA1 3 &pcfg_pull_up_drv_level_1>; ++ }; ++ }; ++ ++ spi2-hs { ++ /omit-if-no-ref/ ++ spi2m0_pins_hs: spi2m0-pins { ++ rockchip,pins = ++ /* spi2_clkm0 */ ++ <2 RK_PC1 4 &pcfg_pull_up_drv_level_1>, ++ /* spi2_misom0 */ ++ <2 RK_PC2 4 &pcfg_pull_up_drv_level_1>, ++ /* spi2_mosim0 */ ++ <2 RK_PC3 4 &pcfg_pull_up_drv_level_1>; ++ }; ++ ++ /omit-if-no-ref/ ++ spi2m0_cs0_hs: spi2m0-cs0 { ++ rockchip,pins = ++ /* spi2_cs0m0 */ ++ <2 RK_PC4 4 &pcfg_pull_up_drv_level_1>; ++ }; ++ ++ /omit-if-no-ref/ ++ spi2m0_cs1_hs: spi2m0-cs1 { ++ rockchip,pins = ++ /* spi2_cs1m0 */ ++ <2 RK_PC5 4 &pcfg_pull_up_drv_level_1>; ++ }; ++ ++ /omit-if-no-ref/ ++ spi2m1_pins_hs: spi2m1-pins { ++ rockchip,pins = ++ /* spi2_clkm1 */ ++ <3 RK_PA0 3 &pcfg_pull_up_drv_level_1>, ++ /* spi2_misom1 */ ++ <2 RK_PD7 3 &pcfg_pull_up_drv_level_1>, ++ /* spi2_mosim1 */ ++ <2 RK_PD6 3 &pcfg_pull_up_drv_level_1>; ++ }; ++ ++ /omit-if-no-ref/ ++ spi2m1_cs0_hs: spi2m1-cs0 { ++ rockchip,pins = ++ /* spi2_cs0m1 */ ++ <2 RK_PD5 3 &pcfg_pull_up_drv_level_1>; ++ }; ++ ++ /omit-if-no-ref/ ++ spi2m1_cs1_hs: spi2m1-cs1 { ++ rockchip,pins = ++ /* spi2_cs1m1 */ ++ <2 RK_PD4 3 &pcfg_pull_up_drv_level_1>; ++ }; ++ }; ++ ++ spi3-hs { ++ /omit-if-no-ref/ ++ spi3m0_pins_hs: spi3m0-pins { ++ rockchip,pins = ++ /* spi3_clkm0 */ ++ <4 RK_PB3 4 &pcfg_pull_up_drv_level_1>, ++ /* spi3_misom0 */ ++ <4 RK_PB0 4 &pcfg_pull_up_drv_level_1>, ++ /* spi3_mosim0 */ ++ <4 RK_PB2 4 &pcfg_pull_up_drv_level_1>; ++ }; ++ ++ /omit-if-no-ref/ ++ spi3m0_cs0_hs: spi3m0-cs0 { ++ rockchip,pins = ++ /* spi3_cs0m0 */ ++ <4 RK_PA6 4 &pcfg_pull_up_drv_level_1>; ++ }; ++ ++ /omit-if-no-ref/ ++ spi3m0_cs1_hs: spi3m0-cs1 { ++ rockchip,pins = ++ /* spi3_cs1m0 */ ++ <4 RK_PA7 4 &pcfg_pull_up_drv_level_1>; ++ }; ++ ++ /omit-if-no-ref/ ++ spi3m1_pins_hs: spi3m1-pins { ++ rockchip,pins = ++ /* spi3_clkm1 */ ++ <4 RK_PC2 2 &pcfg_pull_up_drv_level_1>, ++ /* spi3_misom1 */ ++ <4 RK_PC5 2 &pcfg_pull_up_drv_level_1>, ++ /* spi3_mosim1 */ ++ <4 RK_PC3 2 &pcfg_pull_up_drv_level_1>; ++ }; ++ ++ /omit-if-no-ref/ ++ spi3m1_cs0_hs: spi3m1-cs0 { ++ rockchip,pins = ++ /* spi3_cs0m1 */ ++ <4 RK_PC6 2 &pcfg_pull_up_drv_level_1>; ++ }; ++ ++ /omit-if-no-ref/ ++ spi3m1_cs1_hs: spi3m1-cs1 { ++ rockchip,pins = ++ /* spi3_cs1m1 */ ++ <4 RK_PD1 2 &pcfg_pull_up_drv_level_1>; ++ }; ++ }; ++ ++ gmac-txd-level3 { ++ /omit-if-no-ref/ ++ gmac0_tx_bus2_level3: gmac0-tx-bus2-level3 { ++ rockchip,pins = ++ /* gmac0_txd0 */ ++ <2 RK_PB3 1 &pcfg_pull_none_drv_level_3>, ++ /* gmac0_txd1 */ ++ <2 RK_PB4 1 &pcfg_pull_none_drv_level_3>, ++ /* gmac0_txen */ ++ <2 RK_PB5 1 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ gmac0_rgmii_bus_level3: gmac0-rgmii-bus-level3 { ++ rockchip,pins = ++ /* gmac0_rxd2 */ ++ <2 RK_PA3 2 &pcfg_pull_none>, ++ /* gmac0_rxd3 */ ++ <2 RK_PA4 2 &pcfg_pull_none>, ++ /* gmac0_txd2 */ ++ <2 RK_PA6 2 &pcfg_pull_none_drv_level_3>, ++ /* gmac0_txd3 */ ++ <2 RK_PA7 2 &pcfg_pull_none_drv_level_3>; ++ }; ++ ++ /omit-if-no-ref/ ++ gmac1m0_tx_bus2_level3: gmac1m0-tx-bus2-level3 { ++ rockchip,pins = ++ /* gmac1_txd0m0 */ ++ <3 RK_PB5 3 &pcfg_pull_none_drv_level_3>, ++ /* gmac1_txd1m0 */ ++ <3 RK_PB6 3 &pcfg_pull_none_drv_level_3>, ++ /* gmac1_txenm0 */ ++ <3 RK_PB7 3 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ gmac1m0_rgmii_bus_level3: gmac1m0-rgmii-bus-level3 { ++ rockchip,pins = ++ /* gmac1_rxd2m0 */ ++ <3 RK_PA4 3 &pcfg_pull_none>, ++ /* gmac1_rxd3m0 */ ++ <3 RK_PA5 3 &pcfg_pull_none>, ++ /* gmac1_txd2m0 */ ++ <3 RK_PA2 3 &pcfg_pull_none_drv_level_3>, ++ /* gmac1_txd3m0 */ ++ <3 RK_PA3 3 &pcfg_pull_none_drv_level_3>; ++ }; ++ ++ /omit-if-no-ref/ ++ gmac1m1_tx_bus2_level3: gmac1m1-tx-bus2-level3 { ++ rockchip,pins = ++ /* gmac1_txd0m1 */ ++ <4 RK_PA4 3 &pcfg_pull_none_drv_level_3>, ++ /* gmac1_txd1m1 */ ++ <4 RK_PA5 3 &pcfg_pull_none_drv_level_3>, ++ /* gmac1_txenm1 */ ++ <4 RK_PA6 3 &pcfg_pull_none>; ++ }; ++ ++ /omit-if-no-ref/ ++ gmac1m1_rgmii_bus_level3: gmac1m1-rgmii-bus-level3 { ++ rockchip,pins = ++ /* gmac1_rxd2m1 */ ++ <4 RK_PA1 3 &pcfg_pull_none>, ++ /* gmac1_rxd3m1 */ ++ <4 RK_PA2 3 &pcfg_pull_none>, ++ /* gmac1_txd2m1 */ ++ <3 RK_PD6 3 &pcfg_pull_none_drv_level_3>, ++ /* gmac1_txd3m1 */ ++ <3 RK_PD7 3 &pcfg_pull_none_drv_level_3>; ++ }; ++ }; ++ ++ gmac-txc-level2 { ++ /omit-if-no-ref/ ++ gmac0_rgmii_clk_level2: gmac0-rgmii-clk-level2 { ++ rockchip,pins = ++ /* gmac0_rxclk */ ++ <2 RK_PA5 2 &pcfg_pull_none>, ++ /* gmac0_txclk */ ++ <2 RK_PB0 2 &pcfg_pull_none_drv_level_2>; ++ }; ++ ++ /omit-if-no-ref/ ++ gmac1m0_rgmii_clk_level2: gmac1m0-rgmii-clk-level2 { ++ rockchip,pins = ++ /* gmac1_rxclkm0 */ ++ <3 RK_PA7 3 &pcfg_pull_none>, ++ /* gmac1_txclkm0 */ ++ <3 RK_PA6 3 &pcfg_pull_none_drv_level_2>; ++ }; ++ ++ /omit-if-no-ref/ ++ gmac1m1_rgmii_clk_level2: gmac1m1-rgmii-clk-level2 { ++ rockchip,pins = ++ /* gmac1_rxclkm1 */ ++ <4 RK_PA3 3 &pcfg_pull_none>, ++ /* gmac1_txclkm1 */ ++ <4 RK_PA0 3 &pcfg_pull_none_drv_level_2>; ++ }; ++ }; ++ ++ gpio-func { ++ /omit-if-no-ref/ ++ tsadc_gpio_func: tsadc-gpio-func { ++ rockchip,pins = ++ <0 RK_PA1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-toybrick-base.dts b/arch/arm64/boot/dts/rockchip/rk3568-toybrick-base.dts +new file mode 100755 +index 000000000000..0ef442123d05 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-toybrick-base.dts +@@ -0,0 +1,14 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++/dts-v1/; ++#include "rk3568.dtsi" ++#include "rk3568-linux.dtsi" ++#include "rk3568-toybrick-x0.dtsi" ++//#include "rk3568-toybrick-isp-camera-board.dtsi" ++/ { ++ compatible = "rockchip,rk3568-toybrick-dev-linux-x0","rockchip,rk3568"; ++}; ++ +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-toybrick-edp.dtsi b/arch/arm64/boot/dts/rockchip/rk3568-toybrick-edp.dtsi +new file mode 100755 +index 000000000000..459fc3b6d6a0 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-toybrick-edp.dtsi +@@ -0,0 +1,141 @@ ++/* ++ * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This file is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This file is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++/ { ++ compatible = "rockchip,rk3568-toybrick-edp", "rockchip,rk3568"; ++ ++ edp_panel: edp_panel { ++ compatible = "simple-panel"; ++ backlight = <&backlight>; ++ prepare-delay-ms = <20>; ++ enable-delay-ms = <20>; ++ reset-delay-ms = <20>; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <200000000>; ++ hactive = <1536>; ++ vactive = <2048>; ++ hfront-porch = <12>; ++ hsync-len = <16>; ++ hback-porch = <48>; ++ vfront-porch = <8>; ++ vsync-len = <4>; ++ vback-porch = <8>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ ports { ++ panel_in: endpoint { ++ remote-endpoint = <&edp_out>; ++ }; ++ }; ++ }; ++ ++}; ++ ++ ++&backlight { ++ status = "okay"; ++}; ++ ++&edp { ++ status = "okay"; ++ force-hpd; ++ ++ ports { ++ port@1 { ++ reg = <1>; ++ ++ edp_out: endpoint { ++ remote-endpoint = <&panel_in>; ++ }; ++ }; ++ }; ++ ++}; ++ ++&route_edp { ++ status = "okay"; ++}; ++ ++&edp_phy { ++ status = "okay"; ++}; ++ ++&edp_in_vp0 { ++ status = "disabled"; ++}; ++ ++&edp_in_vp1 { ++ status = "okay"; ++}; ++ ++&edp_panel { ++ power-supply = <&vcc3v3_lcd0_n>; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <600>; ++ i2c-scl-falling-time-ns = <20>; ++ gsl3673: gsl3673@40 { ++ status = "okay"; ++ compatible = "GSL,GSL3673"; ++ reg = <0x40>; ++ screen_max_x = <1536>; ++ screen_max_y = <2048>; ++ irq_gpio_number = <&gpio0 RK_PB5 IRQ_TYPE_LEVEL_LOW>; ++ rst_gpio_number = <&gpio0 RK_PB6 GPIO_ACTIVE_HIGH>; ++ }; ++}; ++ ++&vcc3v3_lcd0_n { ++ gpio = <&gpio0 RK_PC7 GPIO_ACTIVE_HIGH>; ++ enable-active-high; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-toybrick-mipi-tx0-beiqicloud.dtsi b/arch/arm64/boot/dts/rockchip/rk3568-toybrick-mipi-tx0-beiqicloud.dtsi +new file mode 100755 +index 000000000000..25889a206778 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-toybrick-mipi-tx0-beiqicloud.dtsi +@@ -0,0 +1,395 @@ ++/* ++ * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This file is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This file is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++/ { ++ compatible = "rockchip,rk3568-toybrick-mipi-tx0", "rockchip,rk3568"; ++}; ++ ++/* ++ * mipi_dphy0 needs to be enabled ++ * when dsi0 is enabled ++ */ ++&dsi0 { ++ status = "okay"; ++ ++ dsi0_panel: panel@0 { ++ compatible = "simple-panel-dsi"; ++ reg = <0>; ++ backlight = <&backlight>; ++ // power-supply = <&lcd_pwr>; ++ prepare-delay-ms = <2>; ++ reset-delay-ms = <100>; ++ init-delay-ms = <20>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&mipi_power_en>; ++ enable-gpios = <&gpio3 RK_PB5 GPIO_ACTIVE_HIGH>; ++ enable-delay-ms = <120>; ++ disable-delay-ms = <50>; ++ unprepare-delay-ms = <20>; ++ width-mm = <68>; ++ height-mm = <121>; ++ ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | ++ MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ pnel-type = <5>; ++ ++ panel-init-sequence = [ ++ 05 64 01 11 ++ 39 00 04 FF 98 81 03 ++ 15 00 02 01 00 ++ 15 00 02 02 00 ++ 15 00 02 03 72 ++ 15 00 02 04 00 ++ 15 00 02 05 00 ++ 15 00 02 06 09 ++ 15 00 02 07 00 ++ 15 00 02 08 00 ++ 15 00 02 09 01 ++ 15 00 02 0A 00 ++ 15 00 02 0B 00 ++ 15 00 02 0C 01 ++ 15 00 02 0D 00 ++ 15 00 02 0E 00 ++ 15 00 02 0F 00 ++ 15 00 02 10 00 ++ 15 00 02 11 00 ++ 15 00 02 12 00 ++ 15 00 02 13 00 ++ 15 00 02 14 00 ++ 15 00 02 15 00 ++ 15 00 02 16 00 ++ 15 00 02 17 00 ++ 15 00 02 18 00 ++ 15 00 02 19 00 ++ 15 00 02 1A 00 ++ 15 00 02 1B 00 ++ 15 00 02 1C 00 ++ 15 00 02 1D 00 ++ 15 00 02 1E 40 ++ 15 00 02 1F 80 ++ 15 00 02 20 05 ++ 15 00 02 21 02 ++ 15 00 02 22 00 ++ 15 00 02 23 00 ++ 15 00 02 24 00 ++ 15 00 02 25 00 ++ 15 00 02 26 00 ++ 15 00 02 27 00 ++ 15 00 02 28 33 ++ 15 00 02 29 02 ++ 15 00 02 2A 00 ++ 15 00 02 2B 00 ++ 15 00 02 2C 00 ++ 15 00 02 2D 00 ++ 15 00 02 2E 00 ++ 15 00 02 2F 00 ++ 15 00 02 30 00 ++ 15 00 02 31 00 ++ 15 00 02 32 00 ++ 15 00 02 33 00 ++ 15 00 02 34 04 ++ 15 00 02 35 00 ++ 15 00 02 36 00 ++ 15 00 02 37 00 ++ 15 00 02 38 3C ++ 15 00 02 39 00 ++ 15 00 02 3A 40 ++ 15 00 02 3B 40 ++ 15 00 02 3C 00 ++ 15 00 02 3D 00 ++ 15 00 02 3E 00 ++ 15 00 02 3F 00 ++ 15 00 02 40 00 ++ 15 00 02 41 00 ++ 15 00 02 42 00 ++ 15 00 02 43 00 ++ 15 00 02 44 00 ++ 15 00 02 50 01 ++ 15 00 02 51 23 ++ 15 00 02 52 45 ++ 15 00 02 53 67 ++ 15 00 02 54 89 ++ 15 00 02 55 AB ++ 15 00 02 56 01 ++ 15 00 02 57 23 ++ 15 00 02 58 45 ++ 15 00 02 59 67 ++ 15 00 02 5A 89 ++ 15 00 02 5B AB ++ 15 00 02 5C CD ++ 15 00 02 5D EF ++ 15 00 02 5E 11 ++ 15 00 02 5F 01 ++ 15 00 02 60 00 ++ 15 00 02 61 15 ++ 15 00 02 62 14 ++ 15 00 02 63 0E ++ 15 00 02 64 0F ++ 15 00 02 65 0C ++ 15 00 02 66 0D ++ 15 00 02 67 06 ++ 15 00 02 68 02 ++ 15 00 02 69 02 ++ 15 00 02 6A 02 ++ 15 00 02 6B 02 ++ 15 00 02 6C 02 ++ 15 00 02 6D 02 ++ 15 00 02 6E 07 ++ 15 00 02 6F 02 ++ 15 00 02 70 02 ++ 15 00 02 71 02 ++ 15 00 02 72 02 ++ 15 00 02 73 02 ++ 15 00 02 74 02 ++ 15 00 02 75 01 ++ 15 00 02 76 00 ++ 15 00 02 77 14 ++ 15 00 02 78 15 ++ 15 00 02 79 0E ++ 15 00 02 7A 0F ++ 15 00 02 7B 0C ++ 15 00 02 7C 0D ++ 15 00 02 7D 06 ++ 15 00 02 7E 02 ++ 15 00 02 7F 02 ++ 15 00 02 80 02 ++ 15 00 02 81 02 ++ 15 00 02 82 02 ++ 15 00 02 83 02 ++ 15 00 02 84 07 ++ 15 00 02 85 02 ++ 15 00 02 86 02 ++ 15 00 02 87 02 ++ 15 00 02 88 02 ++ 15 00 02 89 02 ++ 15 00 02 8A 02 ++ 39 00 04 FF 98 81 04 ++ 15 00 02 6C 15 ++ 15 00 02 6E 2A ++ 15 00 02 6F 33 ++ 15 00 02 3A 94 ++ 15 00 02 8D 1A ++ 15 00 02 87 BA ++ 15 00 02 26 76 ++ 15 00 02 B2 D1 ++ 15 00 02 B5 06 ++ 39 00 04 FF 98 81 01 ++ 15 00 02 22 0A ++ 15 00 02 31 00 ++ 15 00 02 40 13 ++ 15 00 02 53 84 ++ 15 00 02 55 8F ++ 15 00 02 50 AE ++ 15 00 02 51 AE ++ 15 00 02 60 28 ++ 15 00 02 A0 0F ++ 15 00 02 A1 1B ++ 15 00 02 A2 28 ++ 15 00 02 A3 12 ++ 15 00 02 A4 15 ++ 15 00 02 A5 28 ++ 15 00 02 A6 1B ++ 15 00 02 A7 1E ++ 15 00 02 A8 79 ++ 15 00 02 A9 1B ++ 15 00 02 AA 27 ++ 15 00 02 AB 69 ++ 15 00 02 AC 19 ++ 15 00 02 AD 18 ++ 15 00 02 AE 4C ++ 15 00 02 AF 21 ++ 15 00 02 B0 28 ++ 15 00 02 B1 52 ++ 15 00 02 B2 65 ++ 15 00 02 C0 04 ++ 15 00 02 C1 1B ++ 15 00 02 C2 27 ++ 15 00 02 C3 13 ++ 15 00 02 C4 15 ++ 15 00 02 C5 28 ++ 15 00 02 C6 1C ++ 15 00 02 C7 1E ++ 15 00 02 C8 79 ++ 15 00 02 C9 1A ++ 15 00 02 CA 27 ++ 15 00 02 CB 69 ++ 15 00 02 CC 1A ++ 15 00 02 CD 18 ++ 15 00 02 CE 4C ++ 15 00 02 CF 21 ++ 15 00 02 D0 27 ++ 15 00 02 D1 52 ++ 15 00 02 D2 65 ++ 15 00 02 D3 3F ++ 39 00 04 FF 98 81 00 ++ ++ 05 32 01 29 ++ 15 00 02 35 00 ++ ]; ++ ++ panel-exit-sequence = [ ++ 05 32 01 28 ++ 05 C8 01 10 ++ ]; ++ ++ disp_timings:display-timings { ++ native-mode = <&dsi0_timing0>; ++ ++ dsi0_timing0: timing0 { ++ clock-frequency = <75000000>; ++ hactive = <720>; ++ vactive = <1280>; ++ hfront-porch = <40>; ++ hsync-len = <10>; ++ hback-porch = <40>; ++ vfront-porch = <10>; ++ vsync-len = <36>; ++ vback-porch = <15>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ panel_in_dsi: endpoint { ++ remote-endpoint = <&dsi_out_panel>; ++ }; ++ }; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@1 { ++ reg = <1>; ++ dsi_out_panel: endpoint { ++ remote-endpoint = <&panel_in_dsi>; ++ }; ++ }; ++ }; ++ ++}; ++ ++&i2c1 { ++ status = "okay"; ++ ++ gt1x: gt1x@14 { ++ compatible = "goodix,gt1x"; ++ reg = <0x5d>; ++ pinctrl-names = "default"; ++ power-supply = <&vcc3v3_lcd0_n>; ++ pinctrl-0 = <&touch_gpio>; ++ // goodix,enable-gpio = <&gpio3 RK_PB5 GPIO_ACTIVE_HIGH>; ++ goodix,rst-gpio = <&gpio0 RK_PB6 GPIO_ACTIVE_HIGH>; ++ goodix,irq-gpio = <&gpio0 RK_PB5 IRQ_TYPE_LEVEL_LOW>; ++ }; ++ ++ gt9xx: gt9xx@5d { ++ compatible = "goodix,gt9xx"; ++ status = "okay"; ++ reg = <0x5d>; ++ reset-gpio = <&gpio0 RK_PB6 GPIO_ACTIVE_HIGH>; ++ touch-gpio = <&gpio0 RK_PB5 IRQ_TYPE_LEVEL_LOW>; ++ max-x = <7200>; ++ max-y = <1280>; ++ tp-size = <911>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&touch_gpio>; ++ power-supply = <&vcc3v3_lcd0_n>; ++ }; ++}; ++ ++&dsi1 { ++ status = "disabled"; ++}; ++ ++&edp { ++ status = "disabled"; ++}; ++ ++&dsi0_in_vp0 { ++ status = "disabled"; ++}; ++ ++&dsi0_in_vp1 { ++ status = "okay"; ++}; ++ ++&dsi0_panel { ++ power-supply = <&vcc3v3_lcd0_n>; ++}; ++ ++&backlight { ++ status = "okay"; ++}; ++ ++&video_phy0 { ++ status = "okay"; ++}; ++ ++&vcc3v3_lcd0_n { ++ gpio = <&gpio0 RK_PC7 GPIO_ACTIVE_HIGH>; ++ enable-active-high; ++}; ++ ++&route_dsi0 { ++ status = "okay"; ++}; ++ ++&pinctrl { ++ mipi_pwren { ++ mipi_power_en: mipi_power_en { ++ rockchip,pins = <3 RK_PB5 RK_FUNC_GPIO &pcfg_output_high>; ++ }; ++ }; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-toybrick-mipi-tx0.dtsi b/arch/arm64/boot/dts/rockchip/rk3568-toybrick-mipi-tx0.dtsi +new file mode 100755 +index 000000000000..bd32a4246df3 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-toybrick-mipi-tx0.dtsi +@@ -0,0 +1,91 @@ ++/* ++ * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This file is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This file is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++/ { ++ compatible = "rockchip,rk3568-toybrick-mipi-tx0", "rockchip,rk3568"; ++}; ++ ++/* ++ * mipi_dphy0 needs to be enabled ++ * when dsi0 is enabled ++ */ ++&dsi0 { ++ status = "okay"; ++}; ++ ++&dsi1 { ++ status = "disabled"; ++}; ++ ++&edp { ++ status = "disabled"; ++}; ++ ++&dsi0_in_vp0 { ++ status = "disabled"; ++}; ++ ++&dsi0_in_vp1 { ++ status = "okay"; ++}; ++ ++&dsi0_panel { ++ power-supply = <&vcc3v3_lcd0_n>; ++}; ++ ++&backlight { ++ status = "okay"; ++}; ++ ++>1x { ++ status = "okay"; ++ power-supply = <&vcc3v3_lcd0_n>; ++}; ++ ++&video_phy0 { ++ status = "okay"; ++}; ++ ++&vcc3v3_lcd0_n { ++ gpio = <&gpio0 RK_PC7 GPIO_ACTIVE_HIGH>; ++ enable-active-high; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-toybrick-mipi-tx1.dtsi b/arch/arm64/boot/dts/rockchip/rk3568-toybrick-mipi-tx1.dtsi +new file mode 100755 +index 000000000000..70fc8ffc3e41 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-toybrick-mipi-tx1.dtsi +@@ -0,0 +1,97 @@ ++/* ++ * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This file is dual-licensed: you can use it either under the terms ++ * of the GPL or the X11 license, at your option. Note that this dual ++ * licensing only applies to this file, and not this project as a ++ * whole. ++ * ++ * a) This file is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License as ++ * published by the Free Software Foundation; either version 2 of the ++ * License, or (at your option) any later version. ++ * ++ * This file is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Or, alternatively, ++ * ++ * b) Permission is hereby granted, free of charge, to any person ++ * obtaining a copy of this software and associated documentation ++ * files (the "Software"), to deal in the Software without ++ * restriction, including without limitation the rights to use, ++ * copy, modify, merge, publish, distribute, sublicense, and/or ++ * sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following ++ * conditions: ++ * ++ * The above copyright notice and this permission notice shall be ++ * included in all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, ++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES ++ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT ++ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++/ { ++ compatible = "rockchip,rk3568-toybrick-mipi-tx1", "rockchip,rk3568"; ++}; ++ ++&dsi0 { ++ status = "disabled"; ++}; ++ ++/* ++ * mipi_dphy1 needs to be enabled ++ * when dsi1 is enabled ++ */ ++&dsi1 { ++ status = "okay"; ++}; ++ ++&edp { ++ status = "disabled"; ++}; ++ ++&dsi1_in_vp0 { ++ status = "disabled"; ++ ++}; ++ ++&dsi1_in_vp1 { ++ status = "okay"; ++}; ++ ++&dsi1_panel { ++ power-supply = <&vcc3v3_lcd1_n>; ++}; ++ ++&backlight1 { ++ status = "okay"; ++}; ++ ++&video_phy1 { ++ status = "okay"; ++}; ++ ++>1x { ++ status = "okay"; ++ power-supply = <&vcc3v3_lcd0_n>; ++}; ++ ++&vcc3v3_lcd0_n { ++ gpio = <&gpio0 RK_PC7 GPIO_ACTIVE_HIGH>; ++ enable-active-high; ++}; ++ ++&vcc3v3_lcd1_n { ++ gpio = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; ++ enable-active-high; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-toybrick-x0-android.dts b/arch/arm64/boot/dts/rockchip/rk3568-toybrick-x0-android.dts +new file mode 100755 +index 000000000000..ccfd369d0cd3 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-toybrick-x0-android.dts +@@ -0,0 +1,13 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++/dts-v1/; ++#include "rk3568.dtsi" ++#include "rk3568-android.dtsi" ++#include "rk3568-toybrick-x0.dtsi" ++/ { ++ compatible = "rockchip,rk3568-toybrick-dev-android-x0","rockchip,rk3568"; ++}; ++ +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-toybrick-x0-linux-factory.dts b/arch/arm64/boot/dts/rockchip/rk3568-toybrick-x0-linux-factory.dts +new file mode 100755 +index 000000000000..3107e3a9937a +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-toybrick-x0-linux-factory.dts +@@ -0,0 +1,155 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++/dts-v1/; ++#include "rk3568.dtsi" ++#include "rk3568-linux.dtsi" ++#include "rk3568-toybrick-x0.dtsi" ++/delete-node/ &board_id; ++/ { ++ compatible = "rockchip,rk3568-toybrick-core-linux-factory-edp-mipi1-x0", "rockchip,rk3568"; ++ ++ edp_panel: edp_panel { ++ compatible = "simple-panel"; ++ backlight = <&backlight>; ++ prepare-delay-ms = <20>; ++ enable-delay-ms = <20>; ++ reset-delay-ms = <20>; ++ ++ display-timings { ++ native-mode = <&timing0>; ++ ++ timing0: timing0 { ++ clock-frequency = <200000000>; ++ hactive = <1536>; ++ vactive = <2048>; ++ hfront-porch = <12>; ++ hsync-len = <16>; ++ hback-porch = <48>; ++ vfront-porch = <8>; ++ vsync-len = <4>; ++ vback-porch = <8>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <0>; ++ }; ++ }; ++ ++ ports { ++ panel_in: endpoint { ++ remote-endpoint = <&edp_out>; ++ }; ++ }; ++ }; ++}; ++ ++&backlight { ++ status = "okay"; ++}; ++ ++&edp { ++ status = "okay"; ++ force-hpd; ++ ++ ports { ++ port@1 { ++ reg = <1>; ++ ++ edp_out: endpoint { ++ remote-endpoint = <&panel_in>; ++ }; ++ }; ++ }; ++ ++}; ++ ++&edp_phy { ++ status = "okay"; ++}; ++ ++&edp_in_vp0 { ++ status = "disabled"; ++}; ++ ++&edp_in_vp1 { ++ status = "okay"; ++}; ++ ++&edp_panel { ++ power-supply = <&vcc3v3_lcd0_n>; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ i2c-scl-rising-time-ns = <600>; ++ i2c-scl-falling-time-ns = <20>; ++ gsl3673: gsl3673@40 { ++ status = "okay"; ++ compatible = "GSL,GSL3673"; ++ reg = <0x40>; ++ screen_max_x = <1536>; ++ screen_max_y = <2048>; ++ irq_gpio_number = <&gpio0 RK_PB5 IRQ_TYPE_LEVEL_LOW>; ++ rst_gpio_number = <&gpio0 RK_PB6 GPIO_ACTIVE_HIGH>; ++ }; ++}; ++ ++&vcc3v3_lcd0_n { ++ gpio = <&gpio0 RK_PC7 GPIO_ACTIVE_HIGH>; ++ enable-active-high; ++}; ++ ++&hdmi { ++ status = "disabled"; ++}; ++ ++&dsi0 { ++ status = "disabled"; ++}; ++ ++/* ++ * mipi_dphy1 needs to be enabled ++ * when dsi1 is enabled ++ */ ++&dsi1 { ++ status = "okay"; ++}; ++ ++&dsi1_in_vp0 { ++ status = "okay"; ++}; ++ ++&dsi1_in_vp1 { ++ status = "disabled"; ++}; ++ ++&dsi1_panel { ++ power-supply = <&vcc3v3_lcd1_n>; ++}; ++ ++&backlight1 { ++ status = "okay"; ++}; ++ ++&video_phy1 { ++ status = "okay"; ++}; ++ ++>1x { ++ status = "okay"; ++ power-supply = <&vcc3v3_lcd0_n>; ++}; ++ ++&vcc3v3_lcd0_n { ++ gpio = <&gpio0 RK_PC7 GPIO_ACTIVE_HIGH>; ++ enable-active-high; ++}; ++ ++&vcc3v3_lcd1_n { ++ gpio = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; ++ enable-active-high; ++}; ++ +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-toybrick-x0-linux.dts b/arch/arm64/boot/dts/rockchip/rk3568-toybrick-x0-linux.dts +new file mode 100755 +index 000000000000..0fb7b0a65fe5 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-toybrick-x0-linux.dts +@@ -0,0 +1,14 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++/dts-v1/; ++#include "rk3568.dtsi" ++#include "rk3568-linux.dtsi" ++#include "rk3568-toybrick-x0.dtsi" ++#include "rk3568-toybrick-mipi-tx0-beiqicloud.dtsi" ++/delete-node/ &board_id; ++/ { ++ compatible = "rockchip,rk3568-toybrick-dev-linux-x0","rockchip,rk3568"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-toybrick-x0.dtsi b/arch/arm64/boot/dts/rockchip/rk3568-toybrick-x0.dtsi +new file mode 100755 +index 000000000000..e34a074e7888 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-toybrick-x0.dtsi +@@ -0,0 +1,856 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++#include ++#include ++#include "rk3568-toybrick.dtsi" ++/delete-node/ &adc_keys; ++ ++/ { ++ compatible = "rockchip,rk3568-toybrick", "rockchip,rk3568"; ++ ++ rk_headset: rk-headset { ++ compatible = "rockchip_headset"; ++ headset_gpio = <&gpio3 RK_PC3 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hp_det>; ++ io-channels = <&saradc 1>; ++ }; ++ ++ gpio_leds: gpio-leds { ++ compatible = "gpio-leds"; ++ led@1 { ++ gpios = <&gpio4 RK_PC2 GPIO_ACTIVE_HIGH>; ++ label = "blue"; // Blue LED ++ retain-state-suspended; ++ }; ++ ++ led@2 { ++ gpios = <&gpio4 RK_PC3 GPIO_ACTIVE_HIGH>; ++ label = "red"; // Red LED ++ retain-state-suspended; ++ }; ++ ++ led@3 { ++ gpios = <&gpio4 RK_PC5 GPIO_ACTIVE_HIGH>; ++ label = "green"; // Green LED ++ retain-state-suspended; ++ }; ++ }; ++ ++ adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 0>; ++ io-channel-names = "buttons"; ++ poll-interval = <100>; ++ keyup-threshold-microvolt = <1800000>; ++ ++ menu-key { ++ linux,code = ; ++ label = "menu"; ++ press-threshold-microvolt = <1250000>; ++ }; ++ ++ mute-key { ++ linux,code = ; ++ label = "mute"; ++ press-threshold-microvolt = <850000>; ++ }; ++ ++ vol-down-key { ++ linux,code = ; ++ label = "volume down"; ++ press-threshold-microvolt = <400000>; ++ }; ++ ++ vol-up-key { ++ linux,code = ; ++ label = "volume up"; ++ press-threshold-microvolt = <20000>; ++ }; ++ }; ++ ++ rt5672-sound { ++ compatible = "rockchip-rt5670"; ++ status = "disabled"; ++ dais { ++ dai0 { ++ audio-codec = <&rt5670>; ++ audio-controller = <&i2s1_8ch>; ++ format = "i2s"; ++ }; ++ dai1 { ++ audio-codec = <&rt5670>; ++ audio-controller = <&i2s1_8ch>; ++ format = "i2s"; ++ }; ++ dai2 { ++ audio-codec = <&es7210>; ++ audio-controller = <&i2s1_8ch>; ++ format = "i2s"; ++ }; ++ }; ++ }; ++ ++ vcc2v5_sys: vcc2v5-ddr { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc2v5-sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <2500000>; ++ regulator-max-microvolt = <2500000>; ++ vin-supply = <&vcc3v3_sys>; ++ }; ++ ++ pcie20_3v3: gpio-regulator { ++ compatible = "regulator-gpio"; ++ regulator-name = "pcie20_3v3"; ++ regulator-min-microvolt = <100000>; ++ regulator-max-microvolt = <3300000>; ++ gpios = <&gpio0 RK_PD4 GPIO_ACTIVE_HIGH>; ++ gpios-states = <0x1>; ++ states = <100000 0x0 ++ 3300000 0x1>; ++ }; ++ ++ pcie30_avdd0v9: pcie30-avdd0v9 { ++ compatible = "regulator-fixed"; ++ regulator-name = "pcie30_avdd0v9"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <900000>; ++ vin-supply = <&vcc3v3_sys>; ++ }; ++ ++ pcie30_avdd1v8: pcie30-avdd1v8 { ++ compatible = "regulator-fixed"; ++ regulator-name = "pcie30_avdd1v8"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ vin-supply = <&vcc3v3_sys>; ++ }; ++ ++ pcie30_3v3: gpio-regulator { ++ compatible = "regulator-gpio"; ++ regulator-name = "pcie30_3v3"; ++ regulator-min-microvolt = <100000>; ++ regulator-max-microvolt = <3300000>; ++ gpios = <&gpio0 RK_PD4 GPIO_ACTIVE_HIGH>; ++ gpios-states = <0x1>; ++ states = <100000 0x0 ++ 3300000 0x1>; ++ }; ++ ++ vcc3v3_bu: vcc3v3-bu { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc3v3_bu"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ vin-supply = <&vcc5v0_sys>; ++ }; ++ ++ rk809_sound_micarray: rk809-sound-micarray { ++ status = "okay"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,rk809-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ ++ simple-audio-card,dai-link@0 { ++ format = "i2s"; ++ cpu { ++ sound-dai = <&i2s1_8ch>; ++ }; ++ codec { ++ sound-dai = <&rk809_codec 0>; ++ }; ++ }; ++ simple-audio-card,dai-link@1 { ++ format = "i2s"; ++ cpu { ++ sound-dai = <&i2s1_8ch>; ++ }; ++ codec { ++ sound-dai = <&es7210>; ++ }; ++ }; ++ }; ++ ++ vcc_camera: vcc-camera-regulator { ++ compatible = "regulator-fixed"; ++ gpio = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&camera_pwr>; ++ regulator-name = "vcc_camera"; ++ enable-active-high; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++}; ++ ++ ++ ++&i2s1_8ch { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++ rockchip,clk-trcm = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1m0_sclktx ++ &i2s1m0_sclkrx ++ &i2s1m0_lrcktx ++ &i2s1m0_sclkrx ++ &i2s1m0_lrckrx ++ &i2s1m0_sdo0 ++ &i2s1m0_sdi0 ++ &i2s1m0_sdi1 ++ &i2s1m0_sdi2 ++ &i2s1m0_sdi3>; ++}; ++ ++ ++#if 0 ++ ++&rk809_codec { ++ #sound-dai-cells = <1>; ++ compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; ++ clocks = <&cru I2S1_MCLKOUT>; ++ clock-names = "mclk"; ++ assigned-clocks = <&cru I2S1_MCLKOUT>, <&cru I2S1_MCLK_TX_IOE>; ++ assigned-clock-rates = <12288000>; ++ assigned-clock-parents = <&cru I2S1_MCLKOUT_TX>, <&cru I2S1_MCLKOUT_TX>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1m0_mclk>; ++ pdmdata-out-enable; ++ adc-for-loopback; ++ hp-volume = <20>; ++ spk-volume = <3>; ++ mic-in-differential; ++ status = "okay"; ++}; ++#endif ++&rk809_sound { ++ status = "disabled"; ++}; ++ ++&combphy0_us { ++ status = "okay"; ++}; ++ ++&combphy1_usq { ++ status = "okay"; ++}; ++ ++&combphy2_psq { ++ status = "okay"; ++}; ++ ++/* ++ * mipi_dphy0 needs to be enabled ++ * when dsi0 is enabled ++ */ ++&dsi0 { ++ status = "okay"; ++}; ++ ++&dsi0_in_vp0 { ++ status = "disabled"; ++}; ++ ++&dsi0_in_vp1 { ++ status = "okay"; ++}; ++ ++&dsi0_panel { ++ power-supply = <&vcc3v3_lcd0_n>; ++}; ++ ++/* ++ * mipi_dphy1 needs to be enabled ++ * when dsi1 is enabled ++ */ ++&dsi1 { ++ status = "disabled"; ++}; ++ ++&dsi1_in_vp0 { ++ status = "disabled"; ++}; ++ ++&dsi1_in_vp1 { ++ status = "disabled"; ++}; ++ ++&dsi1_panel { ++ power-supply = <&vcc3v3_lcd1_n>; ++}; ++ ++&gmac0 { ++ phy-mode = "rgmii"; ++ clock_in_out = "output"; ++ ++ snps,reset-gpio = <&gpio2 RK_PD3 GPIO_ACTIVE_LOW>; ++ snps,reset-active-low; ++ /* Reset time is 20ms, 100ms for rtl8211f */ ++ snps,reset-delays-us = <0 20000 100000>; ++ ++ assigned-clocks = <&cru SCLK_GMAC0_RX_TX>, <&cru SCLK_GMAC0>; ++ assigned-clock-parents = <&cru SCLK_GMAC0_RGMII_SPEED>, <&cru CLK_MAC0_2TOP>; ++ assigned-clock-rates = <0>, <125000000>; ++ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&gmac0_miim ++ &gmac0_tx_bus2 ++ &gmac0_rx_bus2 ++ &gmac0_rgmii_clk ++ &gmac0_rgmii_bus>; ++ ++ tx_delay = <0x37>; ++ rx_delay = <0x2e>; ++ ++ phy-handle = <&rgmii_phy0>; ++ status = "okay"; ++}; ++ ++&gmac1 { ++ phy-mode = "rgmii"; ++ clock_in_out = "output"; ++ ++ snps,reset-gpio = <&gpio2 RK_PD1 GPIO_ACTIVE_LOW>; ++ snps,reset-active-low; ++ /* Reset time is 20ms, 100ms for rtl8211f */ ++ snps,reset-delays-us = <0 20000 100000>; ++ ++ assigned-clocks = <&cru SCLK_GMAC1_RX_TX>, <&cru SCLK_GMAC1>; ++ assigned-clock-parents = <&cru SCLK_GMAC1_RGMII_SPEED>, <&cru CLK_MAC1_2TOP>; ++ assigned-clock-rates = <0>, <125000000>; ++ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&gmac1m1_miim ++ &gmac1m1_tx_bus2 ++ &gmac1m1_rx_bus2 ++ &gmac1m1_rgmii_clk ++ &gmac1m1_rgmii_bus>; ++ ++ tx_delay = <0x47>; ++ rx_delay = <0x28>; ++ ++ phy-handle = <&rgmii_phy1>; ++ status = "okay"; ++}; ++ ++/* ++ * power-supply should switche to vcc3v3_lcd1_n ++ * when mipi panel is connected to dsi1. ++ */ ++>1x { ++ power-supply = <&vcc3v3_lcd0_n>; ++}; ++ ++&i2c3 { ++ status = "okay"; ++ rt5670: rt5670@1c { ++ status = "okay"; ++ #sound-dai-cell = <0>; ++ compatible = "realtek,rt5670"; ++ reg = <0x1c>; ++ }; ++ ++ es7210: es7210@40 { ++ #sound-dai-cells = <0>; ++ compatible = "MicArray_0"; ++ reg = <0x40>; ++ clocks = <&cru I2S1_MCLKOUT_RX>;//csqerr ++ clock-names = "mclk"; ++ }; ++ ++ es7210_1: es7210@42 { ++ compatible = "MicArray_1"; ++ reg = <0x42>; ++ }; ++}; ++ ++&i2c5 { ++ status = "okay"; ++ ++ gs_mxc6655xa: gs_mxc6655xa@15 { ++ status = "okay"; ++ compatible = "gs_mxc6655xa"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&mxc6655xa_irq_gpio>; ++ reg = <0x15>; ++ irq-gpio = <&gpio3 RK_PC1 IRQ_TYPE_LEVEL_LOW>; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ type = ; ++ power-off-in-suspend = <1>; ++ layout = <1>; ++ }; ++ ++ mxc6655xa: mxc6655xa@15 { ++ status = "disabled"; ++ compatible = "gs_mxc6655xa"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&mxc6655xa_irq_gpio>; ++ reg = <0x15>; ++ irq-gpio = <&gpio3 RK_PC1 IRQ_TYPE_LEVEL_LOW>; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ type = ; ++ power-off-in-suspend = <1>; ++ layout = <1>; ++ }; ++ ++ hym8563: hym8563@51 { ++ compatible = "haoyu,hym8563"; ++ reg = <0x51>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&rtc_int>; ++ ++ interrupt-parent = <&gpio0>; ++ interrupts = ; ++ }; ++}; ++ ++&mdio0 { ++ rgmii_phy0: phy@0 { ++ compatible = "ethernet-phy-ieee802.3-c22"; ++ reg = <0x0>; ++ }; ++}; ++ ++&mdio1 { ++ rgmii_phy1: phy@0 { ++ compatible = "ethernet-phy-ieee802.3-c22"; ++ reg = <0x0>; ++ }; ++}; ++ ++&i2c4 { ++ status = "okay"; ++ ++ gc8034: gc8034@37 { ++ compatible = "galaxycore,gc8034"; ++ reg = <0x37>; ++ clocks = <&cru CLK_CIF_OUT>;//CLK_CAM0_OUT>; ++ clock-names = "xvclk"; ++ power-domains = <&power RK3568_PD_VI>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_clk>; ++ reset-gpios = <&gpio0 RK_PD6 GPIO_ACTIVE_LOW>; ++ pwdn-gpios = <&gpio4 RK_PB4 GPIO_ACTIVE_LOW>; ++ rockchip,grf = <&grf>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "RK-CMK-8M-2-v1"; ++ rockchip,camera-module-lens-name = "CK8401"; ++ port { ++ gc8034_out: endpoint { ++ remote-endpoint = <&mipi_in_ucam1>; ++ data-lanes = <1 2 3 4>; ++ }; ++ }; ++ }; ++ ++ ov9750_1: ov9750_1@36 { ++ compatible = "ovti,ov9750"; ++ reg = <0x36>; ++ clocks = <&cru CLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_clk>; ++ power-domains = <&power RK3568_PD_VI>; ++ reset-gpios = <&gpio0 RK_PD6 GPIO_ACTIVE_HIGH>; ++ pwdn-gpios = <&gpio4 RK_PB4 GPIO_ACTIVE_HIGH>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "CMK-OT0854-FV1"; ++ rockchip,camera-module-lens-name = "CHT-842B-MD"; ++ port { ++ ucam_out0: endpoint { ++ remote-endpoint = <&mipi_in_ucam0>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++#if 0 ++ os04a10: os04a10@36 { ++ compatible = "ovti,os04a10"; ++ reg = <0x36>; ++ clocks = <&cru CLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ power-domains = <&power RK3568_PD_VI>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_clk>; ++ reset-gpios = <&gpio0 RK_PD6 GPIO_ACTIVE_LOW>; ++ pwdn-gpios = <&gpio4 RK_PB4 GPIO_ACTIVE_LOW>; ++ /* power-gpios = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; */ ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "CMK-OT1607-FV1"; ++ /* rockchip,camera-module-lens-name = "M12-4IR-4MP-F16"; */ ++ rockchip,camera-module-lens-name = "M12-40IRC-4MP-F16"; ++ port { ++ ucam_out0: endpoint { ++ remote-endpoint = <&mipi_in_ucam0>; ++ data-lanes = <1 2 3 4>; ++ }; ++ }; ++ }; ++#endif ++ ov5695: ov5695@36 { ++ status = "okay"; ++ compatible = "ovti,ov5695"; ++ reg = <0x36>; ++ clocks = <&cru CLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ power-domains = <&power RK3568_PD_VI>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_clk>; ++ reset-gpios = <&gpio0 RK_PD6 GPIO_ACTIVE_LOW>; ++ pwdn-gpios = <&gpio4 RK_PB4 GPIO_ACTIVE_LOW>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "TongJu"; ++ rockchip,camera-module-lens-name = "CHT842-MD"; ++ port { ++ ov5695_out: endpoint { ++ remote-endpoint = <&mipi_in_ucam2>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++}; ++ ++&csi2_dphy_hw { ++ status = "okay"; ++}; ++ ++&csi2_dphy0 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam0: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ucam_out0>; ++ data-lanes = <1 2>; ++ }; ++ mipi_in_ucam1: endpoint@2 { ++ reg = <2>; ++ remote-endpoint = <&gc8034_out>; ++ data-lanes = <1 2 3 4>; ++ }; ++ mipi_in_ucam2: endpoint@3 { ++ reg = <3>; ++ remote-endpoint = <&ov5695_out>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ csidphy_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp0_in>; ++ }; ++ }; ++ }; ++}; ++ ++&rkisp { ++ status = "okay"; ++}; ++ ++&rkisp_mmu { ++ status = "okay"; ++}; ++ ++&rkisp_vir0 { ++ status = "okay"; ++ ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp0_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&csidphy_out>; ++ }; ++ }; ++}; ++ ++&video_phy0 { ++ status = "okay"; ++}; ++ ++&video_phy1 { ++ status = "disabled"; ++}; ++ ++/* ++&pcie2x1 { ++ reset-gpios = <&gpio1 RK_PB2 GPIO_ACTIVE_HIGH>; ++ vpcie3v3-supply = <&pcie20_3v3>; ++ status = "okay"; ++}; ++*/ ++ ++&pcie30phy { ++ status = "okay"; ++}; ++ ++&pcie3x2 { ++ reset-gpios = <&gpio2 RK_PD6 GPIO_ACTIVE_HIGH>; ++ vpcie3v3-supply = <&pcie30_3v3>; ++ status = "okay"; ++}; ++ ++&pinctrl { ++ headphone { ++ hp_det: hp-det { ++ rockchip,pins = <3 RK_PC3 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ leds_gpio: leds-gpio { ++ rockchip,pins = ++ <4 RK_PC2 RK_FUNC_GPIO &pcfg_pull_up>, ++ <4 RK_PC3 RK_FUNC_GPIO &pcfg_pull_up>, ++ <4 RK_PC5 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ mxc6655xa { ++ mxc6655xa_irq_gpio: mxc6655xa_irq_gpio { ++ rockchip,pins = <3 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++/* ++ sii902x { ++ sii902x_hdmi_int: sii902x-hdmi-int { ++ rockchip,pins = <3 RK_PA0 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++*/ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <3 RK_PD5 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++/* ++ wifi_32k: wifi-32k {//csqerr ++ rockchip,pins = <2 RK_PC6 1 &pcfg_pull_none>; ++ }; ++*/ ++ ++ }; ++ ++ wireless-wlan { ++ wifi_host_wake_irq: wifi-host-wake-irq { ++ rockchip,pins = <3 RK_PD4 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart8_gpios: uart8-gpios { ++ rockchip,pins = <2 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ rtc { ++ rtc_int: rtc-int { ++ rockchip,pins = <0 RK_PD3 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ i2s1 { ++ /omit-if-no-ref/ ++ i2s1m0_lrckrx: i2s1m0-lrckrx { ++ rockchip,pins = ++ /* i2s1m0_lrckrx */ ++ <1 RK_PA6 1 &pcfg_pull_up_drv_level_4>; ++ }; ++ /omit-if-no-ref/ ++ i2s1m0_lrcktx: i2s1m0-lrcktx { ++ rockchip,pins = ++ /* i2s1m0_lrcktx */ ++ <1 RK_PA5 1 &pcfg_pull_up_drv_level_4>; ++ }; ++ /omit-if-no-ref/ ++ i2s1m0_mclk: i2s1m0-mclk { ++ rockchip,pins = ++ /* i2s1m0_mclk */ ++ <1 RK_PA2 1 &pcfg_pull_up_drv_level_4>; ++ }; ++ /omit-if-no-ref/ ++ i2s1m0_sclkrx: i2s1m0-sclkrx { ++ rockchip,pins = ++ /* i2s1m0_sclkrx */ ++ <1 RK_PA4 1 &pcfg_pull_up_drv_level_4>; ++ }; ++ /omit-if-no-ref/ ++ i2s1m0_sclktx: i2s1m0-sclktx { ++ rockchip,pins = ++ /* i2s1m0_sclktx */ ++ <1 RK_PA3 1 &pcfg_pull_up_drv_level_4>; ++ }; ++ /omit-if-no-ref/ ++ i2s1m0_sdi0: i2s1m0-sdi0 { ++ rockchip,pins = ++ /* i2s1m0_sdi0 */ ++ <1 RK_PB3 1 &pcfg_pull_up_drv_level_4>; ++ }; ++ /omit-if-no-ref/ ++ i2s1m0_sdi1: i2s1m0-sdi1 { ++ rockchip,pins = ++ /* i2s1m0_sdi1 */ ++ <1 RK_PB2 2 &pcfg_pull_up_drv_level_4>; ++ }; ++ /omit-if-no-ref/ ++ i2s1m0_sdi2: i2s1m0-sdi2 { ++ rockchip,pins = ++ /* i2s1m0_sdi2 */ ++ <1 RK_PB1 2 &pcfg_pull_up_drv_level_4>; ++ }; ++ /omit-if-no-ref/ ++ i2s1m0_sdi3: i2s1m0-sdi3 { ++ rockchip,pins = ++ /* i2s1m0_sdi3 */ ++ <1 RK_PB0 2 &pcfg_pull_up_drv_level_4>; ++ }; ++ /omit-if-no-ref/ ++ i2s1m0_sdo0: i2s1m0-sdo0 { ++ rockchip,pins = ++ /* i2s1m0_sdo0 */ ++ <1 RK_PA7 1 &pcfg_pull_up_drv_level_4>; ++ }; ++ }; ++ ++ cam { ++ camera_pwr: camera-pwr { ++ rockchip,pins = ++ /* camera power en */ ++ <0 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++&pwm7 { ++ status = "okay"; ++}; ++ ++&route_dsi0 { ++ status = "okay"; ++ connect = <&vp1_out_dsi0>; ++}; ++ ++&sdio_pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ clocks = <&rk809 1>; ++ clock-names = "ext_clock"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio3 RK_PD5 GPIO_ACTIVE_LOW>; ++ post-power-on-delay-ms = <20>; ++ status = "okay"; ++}; ++ ++&sdmmc1 { ++ status = "disabled"; ++}; ++ ++&sdmmc2 { ++ max-frequency = <150000000>; ++ supports-sdio; ++ bus-width = <4>; ++ disable-wp; ++ cap-sd-highspeed; ++ cap-sdio-irq; ++ keep-power-in-suspend; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ non-removable; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc2m0_bus4 &sdmmc2m0_cmd &sdmmc2m0_clk>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&uart1 { ++ status = "disabled"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart1m0_xfer &uart1m0_ctsn>; ++}; ++ ++&uart8 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart8m0_xfer &uart8m0_ctsn>; ++}; ++ ++&vcc3v3_lcd0_n { ++ gpio = <&gpio0 RK_PC7 GPIO_ACTIVE_HIGH>; ++ enable-active-high; ++}; ++ ++&vcc3v3_lcd1_n { ++ gpio = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; ++ enable-active-high; ++}; ++ ++&wireless_wlan { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_host_wake_irq>; ++ WIFI,host_wake_irq = <&gpio3 RK_PD4 GPIO_ACTIVE_HIGH>; ++}; ++ ++&wireless_bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&rk809 1>; ++ clock-names = "ext_clock"; ++ //wifi-bt-power-toggle; ++ uart_rts_gpios = <&gpio2 RK_PB1 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart8m0_rtsn>; ++ pinctrl-1 = <&uart8_gpios>; ++ BT,reset_gpio = <&gpio3 RK_PA0 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio3 RK_PA2 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio3 RK_PA1 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++}; ++ ++&sata2 { ++ status = "okay"; ++}; ++ ++&edp { ++ status = "disabled"; ++}; ++ ++ ++&dsi0 { ++ status = "disabled"; ++}; ++ ++&dsi1 { ++ status = "disabled"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-toybrick-x10-linux.dts b/arch/arm64/boot/dts/rockchip/rk3568-toybrick-x10-linux.dts +new file mode 100755 +index 000000000000..8dd494ce3a1f +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-toybrick-x10-linux.dts +@@ -0,0 +1,13 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++/dts-v1/; ++#include "rk3568.dtsi" ++#include "rk3568-linux.dtsi" ++#include "rk3568-toybrick-x10.dtsi" ++/delete-node/ &board_id; ++/ { ++ compatible = "rockchip,rk3568-toybrick-dev-linux-x0","rockchip,rk3568"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-toybrick-x10.dtsi b/arch/arm64/boot/dts/rockchip/rk3568-toybrick-x10.dtsi +new file mode 100755 +index 000000000000..7c1919d7d258 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-toybrick-x10.dtsi +@@ -0,0 +1,816 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++#include ++#include ++#include "rk3568-toybrick.dtsi" ++/delete-node/ &adc_keys; ++ ++/ { ++ compatible = "rockchip,rk3568-toybrick", "rockchip,rk3568"; ++ ++ rk_headset: rk-headset { ++ compatible = "rockchip_headset"; ++ headset_gpio = <&gpio3 RK_PC3 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hp_det>; ++ io-channels = <&saradc 1>; ++ }; ++ ++ gpio_leds: gpio-leds { ++ compatible = "gpio-leds"; ++ led@1 { ++ gpios = <&gpio4 RK_PC2 GPIO_ACTIVE_HIGH>; ++ label = "blue"; // Blue LED ++ retain-state-suspended; ++ }; ++ ++ led@2 { ++ gpios = <&gpio4 RK_PC3 GPIO_ACTIVE_HIGH>; ++ label = "red"; // Red LED ++ retain-state-suspended; ++ }; ++ ++ led@3 { ++ gpios = <&gpio4 RK_PC5 GPIO_ACTIVE_HIGH>; ++ label = "green"; // Green LED ++ retain-state-suspended; ++ }; ++ }; ++ ++ adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 0>; ++ io-channel-names = "buttons"; ++ poll-interval = <100>; ++ keyup-threshold-microvolt = <1800000>; ++ ++ menu-key { ++ linux,code = ; ++ label = "menu"; ++ press-threshold-microvolt = <1250000>; ++ }; ++ ++ mute-key { ++ linux,code = ; ++ label = "mute"; ++ press-threshold-microvolt = <850000>; ++ }; ++ ++ vol-down-key { ++ linux,code = ; ++ label = "volume down"; ++ press-threshold-microvolt = <400000>; ++ }; ++ ++ vol-up-key { ++ linux,code = ; ++ label = "volume up"; ++ press-threshold-microvolt = <20000>; ++ }; ++ }; ++ ++ rt5672-sound { ++ compatible = "rockchip-rt5670"; ++ status = "disabled"; ++ dais { ++ dai0 { ++ audio-codec = <&rt5670>; ++ audio-controller = <&i2s1_8ch>; ++ format = "i2s"; ++ }; ++ dai1 { ++ audio-codec = <&rt5670>; ++ audio-controller = <&i2s1_8ch>; ++ format = "i2s"; ++ }; ++ dai2 { ++ audio-codec = <&es7210>; ++ audio-controller = <&i2s1_8ch>; ++ format = "i2s"; ++ }; ++ }; ++ }; ++ ++ vcc2v5_sys: vcc2v5-ddr { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc2v5-sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <2500000>; ++ regulator-max-microvolt = <2500000>; ++ vin-supply = <&vcc3v3_sys>; ++ }; ++ ++ pcie20_3v3: gpio-regulator { ++ compatible = "regulator-gpio"; ++ regulator-name = "pcie20_3v3"; ++ regulator-min-microvolt = <100000>; ++ regulator-max-microvolt = <3300000>; ++ gpios = <&gpio0 RK_PD4 GPIO_ACTIVE_HIGH>; ++ gpios-states = <0x1>; ++ states = <100000 0x0 ++ 3300000 0x1>; ++ }; ++ ++ pcie30_avdd0v9: pcie30-avdd0v9 { ++ compatible = "regulator-fixed"; ++ regulator-name = "pcie30_avdd0v9"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <900000>; ++ vin-supply = <&vcc3v3_sys>; ++ }; ++ ++ pcie30_avdd1v8: pcie30-avdd1v8 { ++ compatible = "regulator-fixed"; ++ regulator-name = "pcie30_avdd1v8"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ vin-supply = <&vcc3v3_sys>; ++ }; ++ ++ pcie30_3v3: gpio-regulator { ++ compatible = "regulator-gpio"; ++ regulator-name = "pcie30_3v3"; ++ regulator-min-microvolt = <100000>; ++ regulator-max-microvolt = <3300000>; ++ gpios = <&gpio0 RK_PD4 GPIO_ACTIVE_HIGH>; ++ gpios-states = <0x1>; ++ states = <100000 0x0 ++ 3300000 0x1>; ++ }; ++ ++ vcc3v3_bu: vcc3v3-bu { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc3v3_bu"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ vin-supply = <&vcc5v0_sys>; ++ }; ++ ++ rk809_sound_micarray: rk809-sound-micarray { ++ status = "okay"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,rk809-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ ++ simple-audio-card,dai-link@0 { ++ format = "i2s"; ++ cpu { ++ sound-dai = <&i2s1_8ch>; ++ }; ++ codec { ++ sound-dai = <&rk809_codec 0>; ++ }; ++ }; ++ simple-audio-card,dai-link@1 { ++ format = "i2s"; ++ cpu { ++ sound-dai = <&i2s1_8ch>; ++ }; ++ codec { ++ sound-dai = <&es7210>; ++ }; ++ }; ++ }; ++ ++ vcc_camera: vcc-camera-regulator { ++ compatible = "regulator-fixed"; ++ gpio = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&camera_pwr>; ++ regulator-name = "vcc_camera"; ++ enable-active-high; ++ regulator-always-on; ++ regulator-boot-on; ++ }; ++ ++ vcc_4g: vcc-4g-regulator { ++ compatible = "regulator-fixed"; ++ gpio = <&gpio0 RK_PC0 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&vcc_4g_drv>; ++ regulator-name = "vcc_4g"; ++ enable-active-low; ++ regulator-always-on; ++ regulator-boot-on; ++ /*regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>;*/ ++ }; ++}; ++ ++ ++ ++&i2s1_8ch { ++ status = "okay"; ++ #sound-dai-cells = <0>; ++ rockchip,clk-trcm = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1m0_sclktx ++ &i2s1m0_sclkrx ++ &i2s1m0_lrcktx ++ &i2s1m0_sclkrx ++ &i2s1m0_lrckrx ++ &i2s1m0_sdo0 ++ &i2s1m0_sdi0 ++ &i2s1m0_sdi1 ++ &i2s1m0_sdi2 ++ &i2s1m0_sdi3>; ++}; ++ ++ ++#if 0 ++ ++&rk809_codec { ++ #sound-dai-cells = <1>; ++ compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; ++ clocks = <&cru I2S1_MCLKOUT>; ++ clock-names = "mclk"; ++ assigned-clocks = <&cru I2S1_MCLKOUT>, <&cru I2S1_MCLK_TX_IOE>; ++ assigned-clock-rates = <12288000>; ++ assigned-clock-parents = <&cru I2S1_MCLKOUT_TX>, <&cru I2S1_MCLKOUT_TX>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1m0_mclk>; ++ pdmdata-out-enable; ++ adc-for-loopback; ++ hp-volume = <20>; ++ spk-volume = <3>; ++ mic-in-differential; ++ status = "okay"; ++}; ++#endif ++&rk809_sound { ++ status = "disabled"; ++}; ++ ++&combphy0_us { ++ status = "okay"; ++}; ++ ++&combphy1_usq { ++ status = "okay"; ++}; ++ ++&combphy2_psq { ++ status = "okay"; ++}; ++ ++&gmac0 { ++ phy-mode = "rgmii"; ++ clock_in_out = "output"; ++ ++ snps,reset-gpio = <&gpio2 RK_PD3 GPIO_ACTIVE_LOW>; ++ snps,reset-active-low; ++ /* Reset time is 20ms, 100ms for rtl8211f */ ++ snps,reset-delays-us = <0 20000 100000>; ++ ++ assigned-clocks = <&cru SCLK_GMAC0_RX_TX>, <&cru SCLK_GMAC0>; ++ assigned-clock-parents = <&cru SCLK_GMAC0_RGMII_SPEED>, <&cru CLK_MAC0_2TOP>; ++ assigned-clock-rates = <0>, <125000000>; ++ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&gmac0_miim ++ &gmac0_tx_bus2 ++ &gmac0_rx_bus2 ++ &gmac0_rgmii_clk ++ &gmac0_rgmii_bus>; ++ ++ tx_delay = <0x37>; ++ rx_delay = <0x2e>; ++ ++ phy-handle = <&rgmii_phy0>; ++ status = "okay"; ++}; ++ ++&gmac1 { ++ phy-mode = "rgmii"; ++ clock_in_out = "output"; ++ ++ snps,reset-gpio = <&gpio2 RK_PD1 GPIO_ACTIVE_LOW>; ++ snps,reset-active-low; ++ /* Reset time is 20ms, 100ms for rtl8211f */ ++ snps,reset-delays-us = <0 20000 100000>; ++ ++ assigned-clocks = <&cru SCLK_GMAC1_RX_TX>, <&cru SCLK_GMAC1>; ++ assigned-clock-parents = <&cru SCLK_GMAC1_RGMII_SPEED>, <&cru CLK_MAC1_2TOP>; ++ assigned-clock-rates = <0>, <125000000>; ++ ++ pinctrl-names = "default"; ++ pinctrl-0 = <&gmac1m1_miim ++ &gmac1m1_tx_bus2 ++ &gmac1m1_rx_bus2 ++ &gmac1m1_rgmii_clk ++ &gmac1m1_rgmii_bus>; ++ ++ tx_delay = <0x47>; ++ rx_delay = <0x28>; ++ ++ phy-handle = <&rgmii_phy1>; ++ status = "okay"; ++}; ++ ++&i2c3 { ++ status = "okay"; ++ rt5670: rt5670@1c { ++ status = "okay"; ++ #sound-dai-cell = <0>; ++ compatible = "realtek,rt5670"; ++ reg = <0x1c>; ++ }; ++ ++ es7210: es7210@40 { ++ #sound-dai-cells = <0>; ++ compatible = "MicArray_0"; ++ reg = <0x40>; ++ clocks = <&cru I2S1_MCLKOUT_RX>;//csqerr ++ clock-names = "mclk"; ++ }; ++ ++ es7210_1: es7210@42 { ++ compatible = "MicArray_1"; ++ reg = <0x42>; ++ }; ++}; ++ ++&i2c5 { ++ status = "okay"; ++ ++ gs_mxc6655xa: gs_mxc6655xa@15 { ++ status = "okay"; ++ compatible = "gs_mxc6655xa"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&mxc6655xa_irq_gpio>; ++ reg = <0x15>; ++ irq-gpio = <&gpio3 RK_PC1 IRQ_TYPE_LEVEL_LOW>; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ type = ; ++ power-off-in-suspend = <1>; ++ layout = <1>; ++ }; ++ ++ mxc6655xa: mxc6655xa@15 { ++ status = "disabled"; ++ compatible = "gs_mxc6655xa"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&mxc6655xa_irq_gpio>; ++ reg = <0x15>; ++ irq-gpio = <&gpio3 RK_PC1 IRQ_TYPE_LEVEL_LOW>; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ type = ; ++ power-off-in-suspend = <1>; ++ layout = <1>; ++ }; ++ ++ hym8563: hym8563@51 { ++ compatible = "haoyu,hym8563"; ++ reg = <0x51>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&rtc_int>; ++ ++ interrupt-parent = <&gpio0>; ++ interrupts = ; ++ }; ++}; ++ ++&mdio0 { ++ rgmii_phy0: phy@0 { ++ compatible = "ethernet-phy-ieee802.3-c22"; ++ reg = <0x0>; ++ }; ++}; ++ ++&mdio1 { ++ rgmii_phy1: phy@0 { ++ compatible = "ethernet-phy-ieee802.3-c22"; ++ reg = <0x0>; ++ }; ++}; ++ ++&i2c4 { ++ status = "okay"; ++ ++ gc8034: gc8034@37 { ++ compatible = "galaxycore,gc8034"; ++ reg = <0x37>; ++ clocks = <&cru CLK_CIF_OUT>;//CLK_CAM0_OUT>; ++ clock-names = "xvclk"; ++ power-domains = <&power RK3568_PD_VI>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_clk>; ++ reset-gpios = <&gpio0 RK_PD6 GPIO_ACTIVE_LOW>; ++ pwdn-gpios = <&gpio4 RK_PB4 GPIO_ACTIVE_LOW>; ++ rockchip,grf = <&grf>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "RK-CMK-8M-2-v1"; ++ rockchip,camera-module-lens-name = "CK8401"; ++ port { ++ gc8034_out: endpoint { ++ remote-endpoint = <&mipi_in_ucam1>; ++ data-lanes = <1 2 3 4>; ++ }; ++ }; ++ }; ++ ++ ov9750_1: ov9750_1@36 { ++ compatible = "ovti,ov9750"; ++ reg = <0x36>; ++ clocks = <&cru CLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_clk>; ++ power-domains = <&power RK3568_PD_VI>; ++ reset-gpios = <&gpio0 RK_PD6 GPIO_ACTIVE_HIGH>; ++ pwdn-gpios = <&gpio4 RK_PB4 GPIO_ACTIVE_HIGH>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "CMK-OT0854-FV1"; ++ rockchip,camera-module-lens-name = "CHT-842B-MD"; ++ port { ++ ucam_out0: endpoint { ++ remote-endpoint = <&mipi_in_ucam0>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++#if 0 ++ os04a10: os04a10@36 { ++ compatible = "ovti,os04a10"; ++ reg = <0x36>; ++ clocks = <&cru CLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ power-domains = <&power RK3568_PD_VI>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_clk>; ++ reset-gpios = <&gpio0 RK_PD6 GPIO_ACTIVE_LOW>; ++ pwdn-gpios = <&gpio4 RK_PB4 GPIO_ACTIVE_LOW>; ++ /* power-gpios = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; */ ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "CMK-OT1607-FV1"; ++ /* rockchip,camera-module-lens-name = "M12-4IR-4MP-F16"; */ ++ rockchip,camera-module-lens-name = "M12-40IRC-4MP-F16"; ++ port { ++ ucam_out0: endpoint { ++ remote-endpoint = <&mipi_in_ucam0>; ++ data-lanes = <1 2 3 4>; ++ }; ++ }; ++ }; ++#endif ++ ov5695: ov5695@36 { ++ status = "okay"; ++ compatible = "ovti,ov5695"; ++ reg = <0x36>; ++ clocks = <&cru CLK_CIF_OUT>; ++ clock-names = "xvclk"; ++ power-domains = <&power RK3568_PD_VI>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&cif_clk>; ++ reset-gpios = <&gpio0 RK_PD6 GPIO_ACTIVE_LOW>; ++ pwdn-gpios = <&gpio4 RK_PB4 GPIO_ACTIVE_LOW>; ++ rockchip,camera-module-index = <0>; ++ rockchip,camera-module-facing = "back"; ++ rockchip,camera-module-name = "TongJu"; ++ rockchip,camera-module-lens-name = "CHT842-MD"; ++ port { ++ ov5695_out: endpoint { ++ remote-endpoint = <&mipi_in_ucam2>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ }; ++}; ++ ++&csi2_dphy_hw { ++ status = "okay"; ++}; ++ ++&csi2_dphy0 { ++ status = "okay"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ mipi_in_ucam0: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&ucam_out0>; ++ data-lanes = <1 2>; ++ }; ++ mipi_in_ucam1: endpoint@2 { ++ reg = <2>; ++ remote-endpoint = <&gc8034_out>; ++ data-lanes = <1 2 3 4>; ++ }; ++ mipi_in_ucam2: endpoint@3 { ++ reg = <3>; ++ remote-endpoint = <&ov5695_out>; ++ data-lanes = <1 2>; ++ }; ++ }; ++ port@1 { ++ reg = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ csidphy_out: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&isp0_in>; ++ }; ++ }; ++ }; ++}; ++ ++&rkisp { ++ status = "okay"; ++}; ++ ++&rkisp_mmu { ++ status = "okay"; ++}; ++ ++&rkisp_vir0 { ++ status = "okay"; ++ ++ port { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ isp0_in: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&csidphy_out>; ++ }; ++ }; ++}; ++ ++&video_phy0 { ++ status = "okay"; ++}; ++ ++&video_phy1 { ++ status = "disabled"; ++}; ++ ++/* ++&pcie2x1 { ++ reset-gpios = <&gpio1 RK_PB2 GPIO_ACTIVE_HIGH>; ++ vpcie3v3-supply = <&pcie20_3v3>; ++ status = "okay"; ++}; ++*/ ++ ++&pcie30phy { ++ status = "okay"; ++}; ++ ++&pcie3x2 { ++ reset-gpios = <&gpio2 RK_PD6 GPIO_ACTIVE_HIGH>; ++ vpcie3v3-supply = <&pcie30_3v3>; ++ status = "okay"; ++}; ++ ++&pinctrl { ++ headphone { ++ hp_det: hp-det { ++ rockchip,pins = <3 RK_PC3 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ leds_gpio: leds-gpio { ++ rockchip,pins = ++ <4 RK_PC2 RK_FUNC_GPIO &pcfg_pull_up>, ++ <4 RK_PC3 RK_FUNC_GPIO &pcfg_pull_up>, ++ <4 RK_PC5 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ mxc6655xa { ++ mxc6655xa_irq_gpio: mxc6655xa_irq_gpio { ++ rockchip,pins = <3 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++/* ++ sii902x { ++ sii902x_hdmi_int: sii902x-hdmi-int { ++ rockchip,pins = <3 RK_PA0 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++*/ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <3 RK_PD5 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++/* ++ wifi_32k: wifi-32k {//csqerr ++ rockchip,pins = <2 RK_PC6 1 &pcfg_pull_none>; ++ }; ++*/ ++ ++ }; ++ ++ wireless-wlan { ++ wifi_host_wake_irq: wifi-host-wake-irq { ++ rockchip,pins = <3 RK_PD4 RK_FUNC_GPIO &pcfg_pull_down>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart8_gpios: uart8-gpios { ++ rockchip,pins = <2 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ rtc { ++ rtc_int: rtc-int { ++ rockchip,pins = <0 RK_PD3 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ }; ++ ++ i2s1 { ++ /omit-if-no-ref/ ++ i2s1m0_lrckrx: i2s1m0-lrckrx { ++ rockchip,pins = ++ /* i2s1m0_lrckrx */ ++ <1 RK_PA6 1 &pcfg_pull_up_drv_level_4>; ++ }; ++ /omit-if-no-ref/ ++ i2s1m0_lrcktx: i2s1m0-lrcktx { ++ rockchip,pins = ++ /* i2s1m0_lrcktx */ ++ <1 RK_PA5 1 &pcfg_pull_up_drv_level_4>; ++ }; ++ /omit-if-no-ref/ ++ i2s1m0_mclk: i2s1m0-mclk { ++ rockchip,pins = ++ /* i2s1m0_mclk */ ++ <1 RK_PA2 1 &pcfg_pull_up_drv_level_4>; ++ }; ++ /omit-if-no-ref/ ++ i2s1m0_sclkrx: i2s1m0-sclkrx { ++ rockchip,pins = ++ /* i2s1m0_sclkrx */ ++ <1 RK_PA4 1 &pcfg_pull_up_drv_level_4>; ++ }; ++ /omit-if-no-ref/ ++ i2s1m0_sclktx: i2s1m0-sclktx { ++ rockchip,pins = ++ /* i2s1m0_sclktx */ ++ <1 RK_PA3 1 &pcfg_pull_up_drv_level_4>; ++ }; ++ /omit-if-no-ref/ ++ i2s1m0_sdi0: i2s1m0-sdi0 { ++ rockchip,pins = ++ /* i2s1m0_sdi0 */ ++ <1 RK_PB3 1 &pcfg_pull_up_drv_level_4>; ++ }; ++ /omit-if-no-ref/ ++ i2s1m0_sdi1: i2s1m0-sdi1 { ++ rockchip,pins = ++ /* i2s1m0_sdi1 */ ++ <1 RK_PB2 2 &pcfg_pull_up_drv_level_4>; ++ }; ++ /omit-if-no-ref/ ++ i2s1m0_sdi2: i2s1m0-sdi2 { ++ rockchip,pins = ++ /* i2s1m0_sdi2 */ ++ <1 RK_PB1 2 &pcfg_pull_up_drv_level_4>; ++ }; ++ /omit-if-no-ref/ ++ i2s1m0_sdi3: i2s1m0-sdi3 { ++ rockchip,pins = ++ /* i2s1m0_sdi3 */ ++ <1 RK_PB0 2 &pcfg_pull_up_drv_level_4>; ++ }; ++ /omit-if-no-ref/ ++ i2s1m0_sdo0: i2s1m0-sdo0 { ++ rockchip,pins = ++ /* i2s1m0_sdo0 */ ++ <1 RK_PA7 1 &pcfg_pull_up_drv_level_4>; ++ }; ++ }; ++ ++ cam { ++ camera_pwr: camera-pwr { ++ rockchip,pins = ++ /* camera power en */ ++ <0 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ 4g { ++ vcc_4g_drv: vcc-4g-drv { ++ rockchip,pins = ++ <0 RK_PC0 RK_FUNC_GPIO &pcfg_output_low>, ++ <3 RK_PC5 RK_FUNC_GPIO &pcfg_output_low>, ++ <3 RK_PB2 RK_FUNC_GPIO &pcfg_output_high>; ++ }; ++ }; ++}; ++ ++&pwm7 { ++ status = "okay"; ++}; ++ ++&sdio_pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ clocks = <&rk809 1>; ++ clock-names = "ext_clock"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ reset-gpios = <&gpio3 RK_PD5 GPIO_ACTIVE_LOW>; ++ post-power-on-delay-ms = <20>; ++ status = "okay"; ++}; ++ ++&sdmmc1 { ++ status = "disabled"; ++}; ++ ++&sdmmc2 { ++ max-frequency = <150000000>; ++ supports-sdio; ++ bus-width = <4>; ++ disable-wp; ++ cap-sd-highspeed; ++ cap-sdio-irq; ++ keep-power-in-suspend; ++ mmc-pwrseq = <&sdio_pwrseq>; ++ non-removable; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc2m0_bus4 &sdmmc2m0_cmd &sdmmc2m0_clk>; ++ sd-uhs-sdr104; ++ status = "okay"; ++}; ++ ++&uart1 { ++ status = "disabled"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart1m0_xfer &uart1m0_ctsn>; ++}; ++ ++&uart8 { ++ status = "okay"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart8m0_xfer &uart8m0_ctsn>; ++}; ++ ++&vcc3v3_lcd0_n { ++ gpio = <&gpio0 RK_PC7 GPIO_ACTIVE_HIGH>; ++ enable-active-high; ++}; ++ ++&vcc3v3_lcd1_n { ++ gpio = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; ++ enable-active-high; ++}; ++ ++&wireless_wlan { ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_host_wake_irq>; ++ WIFI,host_wake_irq = <&gpio3 RK_PD4 GPIO_ACTIVE_HIGH>; ++}; ++ ++&wireless_bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&rk809 1>; ++ clock-names = "ext_clock"; ++ //wifi-bt-power-toggle; ++ uart_rts_gpios = <&gpio2 RK_PB1 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart8m0_rtsn>; ++ pinctrl-1 = <&uart8_gpios>; ++ BT,reset_gpio = <&gpio3 RK_PA0 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio3 RK_PA2 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio3 RK_PA1 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++}; ++ ++&sata2 { ++ status = "okay"; ++}; ++ ++&leds { ++ status = "disabled"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3568-toybrick.dtsi b/arch/arm64/boot/dts/rockchip/rk3568-toybrick.dtsi +new file mode 100755 +index 000000000000..8be2d36f38c0 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568-toybrick.dtsi +@@ -0,0 +1,1891 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ * ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++/ { ++ /* ++ * extliux conf: extlinux.conf.${FLAG}.${BOARD_ID} ++ * dtb file: toybrick.dtb.${FLAG}.${BOARD_ID} ++ */ ++ board_id: board-id { ++ compatible = "board-id"; ++ io-channels = <&saradc 4>; ++ /* ++ * ID: adc-value/adc-io ++ * ------------------------- ++ * 0: adc-io is low level ++ * 1: 0 ~ 100 ++ * 2: 100 ~ 199 ++ * 3: 200 ~ 299 ++ * 4: 300 ~ 399 ++ * 5: 400 ~ 499 ++ * 6: 500 ~ 599 ++ * 7: 600 ~ 699 ++ * 8: 700 ~ 799 ++ * 9: 800 ~ 899 ++ * 10: 900 ~ 1024 ++ */ ++ adc-io = <29>; // GPIO0_D5 ++ thresholds = <100 200 300 400 500 600 700 800 900>; ++ }; ++ ++ adc_keys: adc-keys { ++ compatible = "adc-keys"; ++ io-channels = <&saradc 0>; ++ io-channel-names = "buttons"; ++ keyup-threshold-microvolt = <1800000>; ++ poll-interval = <100>; ++ ++ vol-up-key { ++ label = "volume up"; ++ linux,code = ; ++ press-threshold-microvolt = <1750>; ++ }; ++ ++ vol-down-key { ++ label = "volume down"; ++ linux,code = ; ++ press-threshold-microvolt = <297500>; ++ }; ++ ++ menu-key { ++ label = "menu"; ++ linux,code = ; ++ press-threshold-microvolt = <980000>; ++ }; ++ ++ back-key { ++ label = "back"; ++ linux,code = ; ++ press-threshold-microvolt = <1305500>; ++ }; ++ }; ++ ++ audiopwmout_diff: audiopwmout-diff { ++ status = "disabled"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,audiopwmout-diff"; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,bitclock-master = <&master>; ++ simple-audio-card,frame-master = <&master>; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s3_2ch>; ++ }; ++ master: simple-audio-card,codec { ++ sound-dai = <&dig_acodec>; ++ }; ++ }; ++ ++ backlight: backlight { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm4 0 25000 0>; ++ brightness-levels = < ++ 0 20 20 21 21 22 22 23 ++ 23 24 24 25 25 26 26 27 ++ 27 28 28 29 29 30 30 31 ++ 31 32 32 33 33 34 34 35 ++ 35 36 36 37 37 38 38 39 ++ 40 41 42 43 44 45 46 47 ++ 48 49 50 51 52 53 54 55 ++ 56 57 58 59 60 61 62 63 ++ 64 65 66 67 68 69 70 71 ++ 72 73 74 75 76 77 78 79 ++ 80 81 82 83 84 85 86 87 ++ 88 89 90 91 92 93 94 95 ++ 96 97 98 99 100 101 102 103 ++ 104 105 106 107 108 109 110 111 ++ 112 113 114 115 116 117 118 119 ++ 120 121 122 123 124 125 126 127 ++ 128 129 130 131 132 133 134 135 ++ 136 137 138 139 140 141 142 143 ++ 144 145 146 147 148 149 150 151 ++ 152 153 154 155 156 157 158 159 ++ 160 161 162 163 164 165 166 167 ++ 168 169 170 171 172 173 174 175 ++ 176 177 178 179 180 181 182 183 ++ 184 185 186 187 188 189 190 191 ++ 192 193 194 195 196 197 198 199 ++ 200 201 202 203 204 205 206 207 ++ 208 209 210 211 212 213 214 215 ++ 216 217 218 219 220 221 222 223 ++ 224 225 226 227 228 229 230 231 ++ 232 233 234 235 236 237 238 239 ++ 240 241 242 243 244 245 246 247 ++ 248 249 250 251 252 253 254 255 ++ >; ++ default-brightness-level = <200>; ++ }; ++ ++ backlight1: backlight1 { ++ compatible = "pwm-backlight"; ++ pwms = <&pwm5 0 25000 0>; ++ brightness-levels = < ++ 0 20 20 21 21 22 22 23 ++ 23 24 24 25 25 26 26 27 ++ 27 28 28 29 29 30 30 31 ++ 31 32 32 33 33 34 34 35 ++ 35 36 36 37 37 38 38 39 ++ 40 41 42 43 44 45 46 47 ++ 48 49 50 51 52 53 54 55 ++ 56 57 58 59 60 61 62 63 ++ 64 65 66 67 68 69 70 71 ++ 72 73 74 75 76 77 78 79 ++ 80 81 82 83 84 85 86 87 ++ 88 89 90 91 92 93 94 95 ++ 96 97 98 99 100 101 102 103 ++ 104 105 106 107 108 109 110 111 ++ 112 113 114 115 116 117 118 119 ++ 120 121 122 123 124 125 126 127 ++ 128 129 130 131 132 133 134 135 ++ 136 137 138 139 140 141 142 143 ++ 144 145 146 147 148 149 150 151 ++ 152 153 154 155 156 157 158 159 ++ 160 161 162 163 164 165 166 167 ++ 168 169 170 171 172 173 174 175 ++ 176 177 178 179 180 181 182 183 ++ 184 185 186 187 188 189 190 191 ++ 192 193 194 195 196 197 198 199 ++ 200 201 202 203 204 205 206 207 ++ 208 209 210 211 212 213 214 215 ++ 216 217 218 219 220 221 222 223 ++ 224 225 226 227 228 229 230 231 ++ 232 233 234 235 236 237 238 239 ++ 240 241 242 243 244 245 246 247 ++ 248 249 250 251 252 253 254 255 ++ >; ++ default-brightness-level = <200>; ++ }; ++ ++ dc_12v: dc-12v { ++ compatible = "regulator-fixed"; ++ regulator-name = "dc_12v"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <12000000>; ++ regulator-max-microvolt = <12000000>; ++ }; ++ ++ hdmi_sound: hdmi-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,mclk-fs = <128>; ++ simple-audio-card,name = "rockchip,hdmi"; ++ status = "disabled"; ++ ++ simple-audio-card,cpu { ++ sound-dai = <&i2s0_8ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&hdmi>; ++ }; ++ }; ++ ++ leds: leds { ++ compatible = "gpio-leds"; ++ work_led: work { ++ gpios = <&gpio0 RK_PC0 GPIO_ACTIVE_HIGH>; ++ linux,default-trigger = "heartbeat"; ++ }; ++ }; ++ ++ pdmics: dummy-codec { ++ status = "disabled"; ++ compatible = "rockchip,dummy-codec"; ++ #sound-dai-cells = <0>; ++ }; ++ ++ pdm_mic_array: pdm-mic-array { ++ status = "disabled"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,name = "rockchip,pdm-mic-array"; ++ simple-audio-card,cpu { ++ sound-dai = <&pdm>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&pdmics>; ++ }; ++ }; ++ ++ rk809_sound: rk809-sound { ++ status = "okay"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "i2s"; ++ simple-audio-card,name = "rockchip,rk809-codec"; ++ simple-audio-card,mclk-fs = <256>; ++ ++ simple-audio-card,cpu { ++ sound-dai = <&i2s1_8ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&rk809_codec>; ++ }; ++ }; ++ ++ spdif-sound { ++ status = "okay"; ++ compatible = "simple-audio-card"; ++ simple-audio-card,name = "ROCKCHIP,SPDIF"; ++ simple-audio-card,cpu { ++ sound-dai = <&spdif_8ch>; ++ }; ++ simple-audio-card,codec { ++ sound-dai = <&spdif_out>; ++ }; ++ }; ++ ++ spdif_out: spdif-out { ++ status = "okay"; ++ compatible = "linux,spdif-dit"; ++ #sound-dai-cells = <0>; ++ }; ++ ++ vad_sound: vad-sound { ++ status = "disabled"; ++ compatible = "rockchip,multicodecs-card"; ++ rockchip,card-name = "rockchip,rk3568-vad"; ++ rockchip,cpu = <&i2s1_8ch>; ++ rockchip,codec = <&rk809_codec>, <&vad>; ++ }; ++ ++ bt-sound { ++ compatible = "simple-audio-card"; ++ simple-audio-card,format = "dsp_b"; ++ simple-audio-card,bitclock-inversion = <1>; ++ simple-audio-card,mclk-fs = <256>; ++ simple-audio-card,name = "rockchip,bt"; ++ #simple-audio-card,bitclock-master = <&sound2_master>; ++ #simple-audio-card,frame-master = <&sound2_master>; ++ simple-audio-card,cpu { ++ sound-dai = <&i2s3_2ch>; ++ }; ++ sound2_master:simple-audio-card,codec { ++ #sound-dai-cells = <0>; ++ sound-dai = <&bt_sco>; ++ }; ++ }; ++ ++ bt_sco: bt-sco { ++ compatible = "delta,dfbmcs320"; ++ #sound-dai-cells = <0>; ++ status = "okay"; ++ }; ++ ++ vcc3v3_sys: vcc3v3-sys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc3v3_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ vin-supply = <&dc_12v>; ++ }; ++ ++ vcc5v0_sys: vcc5v0-sys { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc5v0_sys"; ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <5000000>; ++ regulator-max-microvolt = <5000000>; ++ vin-supply = <&dc_12v>; ++ }; ++ ++ vcc5v0_host: vcc5v0-host-regulator { ++ compatible = "regulator-fixed"; ++ enable-active-high; ++ gpio = <&gpio0 RK_PA6 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&vcc5v0_host_en>; ++ regulator-name = "vcc5v0_host"; ++ regulator-always-on; ++ }; ++ ++ vcc5v0_otg: vcc5v0-otg-regulator { ++ compatible = "regulator-fixed"; ++ enable-active-high; ++ gpio = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&vcc5v0_otg_en>; ++ regulator-name = "vcc5v0_otg"; ++ }; ++ ++ vcc3v3_lcd0_n: vcc3v3-lcd0-n { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc3v3_lcd0_n"; ++ regulator-boot-on; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc3v3_lcd1_n: vcc3v3-lcd1-n { ++ compatible = "regulator-fixed"; ++ regulator-name = "vcc3v3_lcd1_n"; ++ regulator-boot-on; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ sdio_pwrseq: sdio-pwrseq { ++ compatible = "mmc-pwrseq-simple"; ++ clocks = <&rk809 1>; ++ clock-names = "ext_clock"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&wifi_enable_h>; ++ ++ /* ++ * On the module itself this is one of these (depending ++ * on the actual card populated): ++ * - SDIO_RESET_L_WL_REG_ON ++ * - PDN (power down when low) ++ */ ++ post-power-on-delay-ms = <200>; ++ reset-gpios = <&gpio3 RK_PD5 GPIO_ACTIVE_LOW>; ++ }; ++ ++ wireless_wlan: wireless-wlan { ++ compatible = "wlan-platdata"; ++ rockchip,grf = <&grf>; ++ wifi_chip_type = "ap6398s"; ++ status = "okay"; ++ }; ++ ++ wireless_bluetooth: wireless-bluetooth { ++ compatible = "bluetooth-platdata"; ++ clocks = <&rk809 1>; ++ clock-names = "ext_clock"; ++ //wifi-bt-power-toggle; ++ uart_rts_gpios = <&gpio2 RK_PB1 GPIO_ACTIVE_LOW>; ++ pinctrl-names = "default", "rts_gpio"; ++ pinctrl-0 = <&uart8m0_rtsn>; ++ pinctrl-1 = <&uart8_gpios>; ++ BT,reset_gpio = <&gpio3 RK_PA0 GPIO_ACTIVE_HIGH>; ++ BT,wake_gpio = <&gpio3 RK_PA1 GPIO_ACTIVE_HIGH>; ++ BT,wake_host_irq = <&gpio3 RK_PA2 GPIO_ACTIVE_HIGH>; ++ status = "okay"; ++ }; ++ ++ test-power { ++ status = "okay"; ++ }; ++}; ++ ++&i2s3_2ch { ++ status = "okay"; ++}; ++ ++&bus_npu { ++ bus-supply = <&vdd_logic>; ++ pvtm-supply = <&vdd_cpu>; ++ status = "okay"; ++}; ++&can0 { ++ assigned-clocks = <&cru CLK_CAN0>; ++ assigned-clock-rates = <150000000>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&can0m1_pins>; ++ status = "disabled"; ++}; ++ ++&can1 { ++ assigned-clocks = <&cru CLK_CAN1>; ++ assigned-clock-rates = <150000000>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&can1m1_pins>; ++ status = "disabled"; ++}; ++ ++&can2 { ++ assigned-clocks = <&cru CLK_CAN2>; ++ assigned-clock-rates = <150000000>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&can2m1_pins>; ++ status = "disabled"; ++}; ++ ++&cpu0 { ++ cpu-supply = <&vdd_cpu>; ++}; ++ ++&dfi { ++ status = "okay"; ++}; ++ ++&dmc { ++ center-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&dsi0 { ++ status = "disabled"; ++ //rockchip,lane-rate = <1000>; ++ dsi0_panel: panel@0 { ++ status = "okay"; ++ compatible = "simple-panel-dsi"; ++ reg = <0>; ++ backlight = <&backlight>; ++ reset-delay-ms = <60>; ++ enable-delay-ms = <60>; ++ prepare-delay-ms = <60>; ++ unprepare-delay-ms = <60>; ++ disable-delay-ms = <60>; ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | ++ MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ panel-init-sequence = [ ++ 23 00 02 FE 21 ++ 23 00 02 04 00 ++ 23 00 02 00 64 ++ 23 00 02 2A 00 ++ 23 00 02 26 64 ++ 23 00 02 54 00 ++ 23 00 02 50 64 ++ 23 00 02 7B 00 ++ 23 00 02 77 64 ++ 23 00 02 A2 00 ++ 23 00 02 9D 64 ++ 23 00 02 C9 00 ++ 23 00 02 C5 64 ++ 23 00 02 01 71 ++ 23 00 02 27 71 ++ 23 00 02 51 71 ++ 23 00 02 78 71 ++ 23 00 02 9E 71 ++ 23 00 02 C6 71 ++ 23 00 02 02 89 ++ 23 00 02 28 89 ++ 23 00 02 52 89 ++ 23 00 02 79 89 ++ 23 00 02 9F 89 ++ 23 00 02 C7 89 ++ 23 00 02 03 9E ++ 23 00 02 29 9E ++ 23 00 02 53 9E ++ 23 00 02 7A 9E ++ 23 00 02 A0 9E ++ 23 00 02 C8 9E ++ 23 00 02 09 00 ++ 23 00 02 05 B0 ++ 23 00 02 31 00 ++ 23 00 02 2B B0 ++ 23 00 02 5A 00 ++ 23 00 02 55 B0 ++ 23 00 02 80 00 ++ 23 00 02 7C B0 ++ 23 00 02 A7 00 ++ 23 00 02 A3 B0 ++ 23 00 02 CE 00 ++ 23 00 02 CA B0 ++ 23 00 02 06 C0 ++ 23 00 02 2D C0 ++ 23 00 02 56 C0 ++ 23 00 02 7D C0 ++ 23 00 02 A4 C0 ++ 23 00 02 CB C0 ++ 23 00 02 07 CF ++ 23 00 02 2F CF ++ 23 00 02 58 CF ++ 23 00 02 7E CF ++ 23 00 02 A5 CF ++ 23 00 02 CC CF ++ 23 00 02 08 DD ++ 23 00 02 30 DD ++ 23 00 02 59 DD ++ 23 00 02 7F DD ++ 23 00 02 A6 DD ++ 23 00 02 CD DD ++ 23 00 02 0E 15 ++ 23 00 02 0A E9 ++ 23 00 02 36 15 ++ 23 00 02 32 E9 ++ 23 00 02 5F 15 ++ 23 00 02 5B E9 ++ 23 00 02 85 15 ++ 23 00 02 81 E9 ++ 23 00 02 AD 15 ++ 23 00 02 A9 E9 ++ 23 00 02 D3 15 ++ 23 00 02 CF E9 ++ 23 00 02 0B 14 ++ 23 00 02 33 14 ++ 23 00 02 5C 14 ++ 23 00 02 82 14 ++ 23 00 02 AA 14 ++ 23 00 02 D0 14 ++ 23 00 02 0C 36 ++ 23 00 02 34 36 ++ 23 00 02 5D 36 ++ 23 00 02 83 36 ++ 23 00 02 AB 36 ++ 23 00 02 D1 36 ++ 23 00 02 0D 6B ++ 23 00 02 35 6B ++ 23 00 02 5E 6B ++ 23 00 02 84 6B ++ 23 00 02 AC 6B ++ 23 00 02 D2 6B ++ 23 00 02 13 5A ++ 23 00 02 0F 94 ++ 23 00 02 3B 5A ++ 23 00 02 37 94 ++ 23 00 02 64 5A ++ 23 00 02 60 94 ++ 23 00 02 8A 5A ++ 23 00 02 86 94 ++ 23 00 02 B2 5A ++ 23 00 02 AE 94 ++ 23 00 02 D8 5A ++ 23 00 02 D4 94 ++ 23 00 02 10 D1 ++ 23 00 02 38 D1 ++ 23 00 02 61 D1 ++ 23 00 02 87 D1 ++ 23 00 02 AF D1 ++ 23 00 02 D5 D1 ++ 23 00 02 11 04 ++ 23 00 02 39 04 ++ 23 00 02 62 04 ++ 23 00 02 88 04 ++ 23 00 02 B0 04 ++ 23 00 02 D6 04 ++ 23 00 02 12 05 ++ 23 00 02 3A 05 ++ 23 00 02 63 05 ++ 23 00 02 89 05 ++ 23 00 02 B1 05 ++ 23 00 02 D7 05 ++ 23 00 02 18 AA ++ 23 00 02 14 36 ++ 23 00 02 42 AA ++ 23 00 02 3D 36 ++ 23 00 02 69 AA ++ 23 00 02 65 36 ++ 23 00 02 8F AA ++ 23 00 02 8B 36 ++ 23 00 02 B7 AA ++ 23 00 02 B3 36 ++ 23 00 02 DD AA ++ 23 00 02 D9 36 ++ 23 00 02 15 74 ++ 23 00 02 3F 74 ++ 23 00 02 66 74 ++ 23 00 02 8C 74 ++ 23 00 02 B4 74 ++ 23 00 02 DA 74 ++ 23 00 02 16 9F ++ 23 00 02 40 9F ++ 23 00 02 67 9F ++ 23 00 02 8D 9F ++ 23 00 02 B5 9F ++ 23 00 02 DB 9F ++ 23 00 02 17 DC ++ 23 00 02 41 DC ++ 23 00 02 68 DC ++ 23 00 02 8E DC ++ 23 00 02 B6 DC ++ 23 00 02 DC DC ++ 23 00 02 1D FF ++ 23 00 02 19 03 ++ 23 00 02 47 FF ++ 23 00 02 43 03 ++ 23 00 02 6E FF ++ 23 00 02 6A 03 ++ 23 00 02 94 FF ++ 23 00 02 90 03 ++ 23 00 02 BC FF ++ 23 00 02 B8 03 ++ 23 00 02 E2 FF ++ 23 00 02 DE 03 ++ 23 00 02 1A 35 ++ 23 00 02 44 35 ++ 23 00 02 6B 35 ++ 23 00 02 91 35 ++ 23 00 02 B9 35 ++ 23 00 02 DF 35 ++ 23 00 02 1B 45 ++ 23 00 02 45 45 ++ 23 00 02 6C 45 ++ 23 00 02 92 45 ++ 23 00 02 BA 45 ++ 23 00 02 E0 45 ++ 23 00 02 1C 55 ++ 23 00 02 46 55 ++ 23 00 02 6D 55 ++ 23 00 02 93 55 ++ 23 00 02 BB 55 ++ 23 00 02 E1 55 ++ 23 00 02 22 FF ++ 23 00 02 1E 68 ++ 23 00 02 4C FF ++ 23 00 02 48 68 ++ 23 00 02 73 FF ++ 23 00 02 6F 68 ++ 23 00 02 99 FF ++ 23 00 02 95 68 ++ 23 00 02 C1 FF ++ 23 00 02 BD 68 ++ 23 00 02 E7 FF ++ 23 00 02 E3 68 ++ 23 00 02 1F 7E ++ 23 00 02 49 7E ++ 23 00 02 70 7E ++ 23 00 02 96 7E ++ 23 00 02 BE 7E ++ 23 00 02 E4 7E ++ 23 00 02 20 97 ++ 23 00 02 4A 97 ++ 23 00 02 71 97 ++ 23 00 02 97 97 ++ 23 00 02 BF 97 ++ 23 00 02 E5 97 ++ 23 00 02 21 B5 ++ 23 00 02 4B B5 ++ 23 00 02 72 B5 ++ 23 00 02 98 B5 ++ 23 00 02 C0 B5 ++ 23 00 02 E6 B5 ++ 23 00 02 25 F0 ++ 23 00 02 23 E8 ++ 23 00 02 4F F0 ++ 23 00 02 4D E8 ++ 23 00 02 76 F0 ++ 23 00 02 74 E8 ++ 23 00 02 9C F0 ++ 23 00 02 9A E8 ++ 23 00 02 C4 F0 ++ 23 00 02 C2 E8 ++ 23 00 02 EA F0 ++ 23 00 02 E8 E8 ++ 23 00 02 24 FF ++ 23 00 02 4E FF ++ 23 00 02 75 FF ++ 23 00 02 9B FF ++ 23 00 02 C3 FF ++ 23 00 02 E9 FF ++ 23 00 02 FE 3D ++ 23 00 02 00 04 ++ 23 00 02 FE 23 ++ 23 00 02 08 82 ++ 23 00 02 0A 00 ++ 23 00 02 0B 00 ++ 23 00 02 0C 01 ++ 23 00 02 16 00 ++ 23 00 02 18 02 ++ 23 00 02 1B 04 ++ 23 00 02 19 04 ++ 23 00 02 1C 81 ++ 23 00 02 1F 00 ++ 23 00 02 20 03 ++ 23 00 02 23 04 ++ 23 00 02 21 01 ++ 23 00 02 54 63 ++ 23 00 02 55 54 ++ 23 00 02 6E 45 ++ 23 00 02 6D 36 ++ 23 00 02 FE 3D ++ 23 00 02 55 78 ++ 23 00 02 FE 20 ++ 23 00 02 26 30 ++ 23 00 02 FE 3D ++ 23 00 02 20 71 ++ 23 00 02 50 8F ++ 23 00 02 51 8F ++ 23 00 02 FE 00 ++ 23 00 02 35 00 ++ 05 78 01 11 ++ 05 1E 01 29 ++ ]; ++ ++ panel-exit-sequence = [ ++ 05 00 01 28 ++ 05 00 01 10 ++ ]; ++ ++ disp_timings0: display-timings { ++ native-mode = <&dsi0_timing0>; ++ dsi0_timing0: timing0 { ++ clock-frequency = <132000000>; ++ hactive = <1080>; ++ vactive = <1920>; ++ hfront-porch = <15>; ++ hsync-len = <2>; ++ hback-porch = <30>; ++ vfront-porch = <15>; ++ vsync-len = <2>; ++ vback-porch = <15>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <1>; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ panel_in_dsi: endpoint { ++ remote-endpoint = <&dsi_out_panel>; ++ }; ++ }; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@1 { ++ reg = <1>; ++ dsi_out_panel: endpoint { ++ remote-endpoint = <&panel_in_dsi>; ++ }; ++ }; ++ }; ++ ++}; ++ ++&dsi1 { ++ status = "disabled"; ++ //rockchip,lane-rate = <1000>; ++ dsi1_panel: panel@0 { ++ status = "okay"; ++ compatible = "simple-panel-dsi"; ++ reg = <0>; ++ backlight = <&backlight1>; ++ reset-delay-ms = <60>; ++ enable-delay-ms = <60>; ++ prepare-delay-ms = <60>; ++ unprepare-delay-ms = <60>; ++ disable-delay-ms = <60>; ++ dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | ++ MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; ++ dsi,format = ; ++ dsi,lanes = <4>; ++ panel-init-sequence = [ ++ 23 00 02 FE 21 ++ 23 00 02 04 00 ++ 23 00 02 00 64 ++ 23 00 02 2A 00 ++ 23 00 02 26 64 ++ 23 00 02 54 00 ++ 23 00 02 50 64 ++ 23 00 02 7B 00 ++ 23 00 02 77 64 ++ 23 00 02 A2 00 ++ 23 00 02 9D 64 ++ 23 00 02 C9 00 ++ 23 00 02 C5 64 ++ 23 00 02 01 71 ++ 23 00 02 27 71 ++ 23 00 02 51 71 ++ 23 00 02 78 71 ++ 23 00 02 9E 71 ++ 23 00 02 C6 71 ++ 23 00 02 02 89 ++ 23 00 02 28 89 ++ 23 00 02 52 89 ++ 23 00 02 79 89 ++ 23 00 02 9F 89 ++ 23 00 02 C7 89 ++ 23 00 02 03 9E ++ 23 00 02 29 9E ++ 23 00 02 53 9E ++ 23 00 02 7A 9E ++ 23 00 02 A0 9E ++ 23 00 02 C8 9E ++ 23 00 02 09 00 ++ 23 00 02 05 B0 ++ 23 00 02 31 00 ++ 23 00 02 2B B0 ++ 23 00 02 5A 00 ++ 23 00 02 55 B0 ++ 23 00 02 80 00 ++ 23 00 02 7C B0 ++ 23 00 02 A7 00 ++ 23 00 02 A3 B0 ++ 23 00 02 CE 00 ++ 23 00 02 CA B0 ++ 23 00 02 06 C0 ++ 23 00 02 2D C0 ++ 23 00 02 56 C0 ++ 23 00 02 7D C0 ++ 23 00 02 A4 C0 ++ 23 00 02 CB C0 ++ 23 00 02 07 CF ++ 23 00 02 2F CF ++ 23 00 02 58 CF ++ 23 00 02 7E CF ++ 23 00 02 A5 CF ++ 23 00 02 CC CF ++ 23 00 02 08 DD ++ 23 00 02 30 DD ++ 23 00 02 59 DD ++ 23 00 02 7F DD ++ 23 00 02 A6 DD ++ 23 00 02 CD DD ++ 23 00 02 0E 15 ++ 23 00 02 0A E9 ++ 23 00 02 36 15 ++ 23 00 02 32 E9 ++ 23 00 02 5F 15 ++ 23 00 02 5B E9 ++ 23 00 02 85 15 ++ 23 00 02 81 E9 ++ 23 00 02 AD 15 ++ 23 00 02 A9 E9 ++ 23 00 02 D3 15 ++ 23 00 02 CF E9 ++ 23 00 02 0B 14 ++ 23 00 02 33 14 ++ 23 00 02 5C 14 ++ 23 00 02 82 14 ++ 23 00 02 AA 14 ++ 23 00 02 D0 14 ++ 23 00 02 0C 36 ++ 23 00 02 34 36 ++ 23 00 02 5D 36 ++ 23 00 02 83 36 ++ 23 00 02 AB 36 ++ 23 00 02 D1 36 ++ 23 00 02 0D 6B ++ 23 00 02 35 6B ++ 23 00 02 5E 6B ++ 23 00 02 84 6B ++ 23 00 02 AC 6B ++ 23 00 02 D2 6B ++ 23 00 02 13 5A ++ 23 00 02 0F 94 ++ 23 00 02 3B 5A ++ 23 00 02 37 94 ++ 23 00 02 64 5A ++ 23 00 02 60 94 ++ 23 00 02 8A 5A ++ 23 00 02 86 94 ++ 23 00 02 B2 5A ++ 23 00 02 AE 94 ++ 23 00 02 D8 5A ++ 23 00 02 D4 94 ++ 23 00 02 10 D1 ++ 23 00 02 38 D1 ++ 23 00 02 61 D1 ++ 23 00 02 87 D1 ++ 23 00 02 AF D1 ++ 23 00 02 D5 D1 ++ 23 00 02 11 04 ++ 23 00 02 39 04 ++ 23 00 02 62 04 ++ 23 00 02 88 04 ++ 23 00 02 B0 04 ++ 23 00 02 D6 04 ++ 23 00 02 12 05 ++ 23 00 02 3A 05 ++ 23 00 02 63 05 ++ 23 00 02 89 05 ++ 23 00 02 B1 05 ++ 23 00 02 D7 05 ++ 23 00 02 18 AA ++ 23 00 02 14 36 ++ 23 00 02 42 AA ++ 23 00 02 3D 36 ++ 23 00 02 69 AA ++ 23 00 02 65 36 ++ 23 00 02 8F AA ++ 23 00 02 8B 36 ++ 23 00 02 B7 AA ++ 23 00 02 B3 36 ++ 23 00 02 DD AA ++ 23 00 02 D9 36 ++ 23 00 02 15 74 ++ 23 00 02 3F 74 ++ 23 00 02 66 74 ++ 23 00 02 8C 74 ++ 23 00 02 B4 74 ++ 23 00 02 DA 74 ++ 23 00 02 16 9F ++ 23 00 02 40 9F ++ 23 00 02 67 9F ++ 23 00 02 8D 9F ++ 23 00 02 B5 9F ++ 23 00 02 DB 9F ++ 23 00 02 17 DC ++ 23 00 02 41 DC ++ 23 00 02 68 DC ++ 23 00 02 8E DC ++ 23 00 02 B6 DC ++ 23 00 02 DC DC ++ 23 00 02 1D FF ++ 23 00 02 19 03 ++ 23 00 02 47 FF ++ 23 00 02 43 03 ++ 23 00 02 6E FF ++ 23 00 02 6A 03 ++ 23 00 02 94 FF ++ 23 00 02 90 03 ++ 23 00 02 BC FF ++ 23 00 02 B8 03 ++ 23 00 02 E2 FF ++ 23 00 02 DE 03 ++ 23 00 02 1A 35 ++ 23 00 02 44 35 ++ 23 00 02 6B 35 ++ 23 00 02 91 35 ++ 23 00 02 B9 35 ++ 23 00 02 DF 35 ++ 23 00 02 1B 45 ++ 23 00 02 45 45 ++ 23 00 02 6C 45 ++ 23 00 02 92 45 ++ 23 00 02 BA 45 ++ 23 00 02 E0 45 ++ 23 00 02 1C 55 ++ 23 00 02 46 55 ++ 23 00 02 6D 55 ++ 23 00 02 93 55 ++ 23 00 02 BB 55 ++ 23 00 02 E1 55 ++ 23 00 02 22 FF ++ 23 00 02 1E 68 ++ 23 00 02 4C FF ++ 23 00 02 48 68 ++ 23 00 02 73 FF ++ 23 00 02 6F 68 ++ 23 00 02 99 FF ++ 23 00 02 95 68 ++ 23 00 02 C1 FF ++ 23 00 02 BD 68 ++ 23 00 02 E7 FF ++ 23 00 02 E3 68 ++ 23 00 02 1F 7E ++ 23 00 02 49 7E ++ 23 00 02 70 7E ++ 23 00 02 96 7E ++ 23 00 02 BE 7E ++ 23 00 02 E4 7E ++ 23 00 02 20 97 ++ 23 00 02 4A 97 ++ 23 00 02 71 97 ++ 23 00 02 97 97 ++ 23 00 02 BF 97 ++ 23 00 02 E5 97 ++ 23 00 02 21 B5 ++ 23 00 02 4B B5 ++ 23 00 02 72 B5 ++ 23 00 02 98 B5 ++ 23 00 02 C0 B5 ++ 23 00 02 E6 B5 ++ 23 00 02 25 F0 ++ 23 00 02 23 E8 ++ 23 00 02 4F F0 ++ 23 00 02 4D E8 ++ 23 00 02 76 F0 ++ 23 00 02 74 E8 ++ 23 00 02 9C F0 ++ 23 00 02 9A E8 ++ 23 00 02 C4 F0 ++ 23 00 02 C2 E8 ++ 23 00 02 EA F0 ++ 23 00 02 E8 E8 ++ 23 00 02 24 FF ++ 23 00 02 4E FF ++ 23 00 02 75 FF ++ 23 00 02 9B FF ++ 23 00 02 C3 FF ++ 23 00 02 E9 FF ++ 23 00 02 FE 3D ++ 23 00 02 00 04 ++ 23 00 02 FE 23 ++ 23 00 02 08 82 ++ 23 00 02 0A 00 ++ 23 00 02 0B 00 ++ 23 00 02 0C 01 ++ 23 00 02 16 00 ++ 23 00 02 18 02 ++ 23 00 02 1B 04 ++ 23 00 02 19 04 ++ 23 00 02 1C 81 ++ 23 00 02 1F 00 ++ 23 00 02 20 03 ++ 23 00 02 23 04 ++ 23 00 02 21 01 ++ 23 00 02 54 63 ++ 23 00 02 55 54 ++ 23 00 02 6E 45 ++ 23 00 02 6D 36 ++ 23 00 02 FE 3D ++ 23 00 02 55 78 ++ 23 00 02 FE 20 ++ 23 00 02 26 30 ++ 23 00 02 FE 3D ++ 23 00 02 20 71 ++ 23 00 02 50 8F ++ 23 00 02 51 8F ++ 23 00 02 FE 00 ++ 23 00 02 35 00 ++ 05 78 01 11 ++ 05 1E 01 29 ++ ]; ++ ++ panel-exit-sequence = [ ++ 05 00 01 28 ++ 05 00 01 10 ++ ]; ++ ++ disp_timings1: display-timings { ++ native-mode = <&dsi1_timing0>; ++ dsi1_timing0: timing0 { ++ clock-frequency = <132000000>; ++ hactive = <1080>; ++ vactive = <1920>; ++ hfront-porch = <15>; ++ hsync-len = <2>; ++ hback-porch = <30>; ++ vfront-porch = <15>; ++ vsync-len = <2>; ++ vback-porch = <15>; ++ hsync-active = <0>; ++ vsync-active = <0>; ++ de-active = <0>; ++ pixelclk-active = <1>; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ panel_in_dsi1: endpoint { ++ remote-endpoint = <&dsi1_out_panel>; ++ }; ++ }; ++ }; ++ }; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@1 { ++ reg = <1>; ++ dsi1_out_panel: endpoint { ++ remote-endpoint = <&panel_in_dsi1>; ++ }; ++ }; ++ }; ++ ++}; ++ ++&gpu { ++ mali-supply = <&vdd_gpu>; ++ status = "okay"; ++}; ++ ++&hdmi { ++ status = "okay"; ++ rockchip,phy-table = ++ <92812500 0x8009 0x0000 0x0270>, ++ <165000000 0x800b 0x0000 0x026d>, ++ <185625000 0x800b 0x0000 0x01ed>, ++ <297000000 0x800b 0x0000 0x01ad>, ++ <594000000 0x8029 0x0000 0x0088>, ++ <000000000 0x0000 0x0000 0x0000>; ++}; ++ ++&hdmi_in_vp0 { ++ status = "okay"; ++}; ++ ++&hdmi_in_vp1 { ++ status = "disabled"; ++}; ++ ++&hdmi_sound { ++ status = "okay"; ++}; ++ ++&i2c0 { ++ status = "okay"; ++ ++ vdd_cpu: tcs4525@1c { ++ compatible = "tcs,tcs452x"; ++ reg = <0x1c>; ++ vin-supply = <&vcc5v0_sys>; ++ regulator-compatible = "fan53555-reg"; ++ regulator-name = "vdd_cpu"; ++ regulator-min-microvolt = <712500>; ++ regulator-max-microvolt = <1390000>; ++ regulator-init-microvolt = <900000>; ++ regulator-ramp-delay = <2300>; ++ fcs,suspend-voltage-selector = <1>; ++ regulator-boot-on; ++ regulator-always-on; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ rk809: pmic@20 { ++ compatible = "rockchip,rk809"; ++ reg = <0x20>; ++ interrupt-parent = <&gpio0>; ++ interrupts = <3 IRQ_TYPE_LEVEL_LOW>; ++ ++ pinctrl-names = "default", "pmic-sleep", ++ "pmic-power-off", "pmic-reset"; ++ pinctrl-0 = <&pmic_int>; ++ pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; ++ pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; ++ pinctrl-3 = <&soc_slppin_gpio>, <&rk817_slppin_rst>; ++ ++ rockchip,system-power-controller; ++ wakeup-source; ++ #clock-cells = <1>; ++ clock-output-names = "rk808-clkout1", "rk808-clkout2"; ++ //fb-inner-reg-idxs = <2>; ++ /* 1: rst regs (default in codes), 0: rst the pmic */ ++ pmic-reset-func = <0>; ++ /* not save the PMIC_POWER_EN register in uboot */ ++ not-save-power-en = <1>; ++ ++ vcc1-supply = <&vcc3v3_sys>; ++ vcc2-supply = <&vcc3v3_sys>; ++ vcc3-supply = <&vcc3v3_sys>; ++ vcc4-supply = <&vcc3v3_sys>; ++ vcc5-supply = <&vcc3v3_sys>; ++ vcc6-supply = <&vcc3v3_sys>; ++ vcc7-supply = <&vcc3v3_sys>; ++ vcc8-supply = <&vcc3v3_sys>; ++ vcc9-supply = <&vcc3v3_sys>; ++ ++ pwrkey { ++ status = "okay"; ++ }; ++ ++ pinctrl_rk8xx: pinctrl_rk8xx { ++ gpio-controller; ++ #gpio-cells = <2>; ++ ++ rk817_slppin_null: rk817_slppin_null { ++ pins = "gpio_slp"; ++ function = "pin_fun0"; ++ }; ++ ++ rk817_slppin_slp: rk817_slppin_slp { ++ pins = "gpio_slp"; ++ function = "pin_fun1"; ++ }; ++ ++ rk817_slppin_pwrdn: rk817_slppin_pwrdn { ++ pins = "gpio_slp"; ++ function = "pin_fun2"; ++ }; ++ ++ rk817_slppin_rst: rk817_slppin_rst { ++ pins = "gpio_slp"; ++ function = "pin_fun3"; ++ }; ++ }; ++ ++ regulators { ++ vdd_logic: DCDC_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <500000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-init-microvolt = <900000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_logic"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdd_gpu: DCDC_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <500000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-init-microvolt = <900000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_gpu"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_ddr: DCDC_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vcc_ddr"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ }; ++ }; ++ ++ vdd_npu: DCDC_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <500000>; ++ regulator-max-microvolt = <1350000>; ++ regulator-init-microvolt = <900000>; ++ regulator-ramp-delay = <6001>; ++ regulator-initial-mode = <0x2>; ++ regulator-name = "vdd_npu"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdda0v9_image: LDO_REG1 { ++ regulator-boot-on; ++ regulator-always-on; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <900000>; ++ regulator-name = "vdda0v9_image"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdda_0v9: LDO_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <900000>; ++ regulator-name = "vdda_0v9"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vdda0v9_pmu: LDO_REG3 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <900000>; ++ regulator-max-microvolt = <900000>; ++ regulator-name = "vdda0v9_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <900000>; ++ }; ++ }; ++ ++ vccio_acodec: LDO_REG4 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vccio_acodec"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vccio_sd: LDO_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vccio_sd"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc3v3_pmu: LDO_REG6 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <3300000>; ++ regulator-max-microvolt = <3300000>; ++ regulator-name = "vcc3v3_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <3300000>; ++ }; ++ }; ++ ++ vcca_1v8: LDO_REG7 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcca_1v8"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcca1v8_pmu: LDO_REG8 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcca1v8_pmu"; ++ regulator-state-mem { ++ regulator-on-in-suspend; ++ regulator-suspend-microvolt = <1800000>; ++ }; ++ }; ++ ++ vcca1v8_image: LDO_REG9 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcca1v8_image"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_1v8: DCDC_REG5 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-min-microvolt = <1800000>; ++ regulator-max-microvolt = <1800000>; ++ regulator-name = "vcc_1v8"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc_3v3: SWITCH_REG1 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc_3v3"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ ++ vcc3v3_sd: SWITCH_REG2 { ++ regulator-always-on; ++ regulator-boot-on; ++ regulator-name = "vcc3v3_sd"; ++ regulator-state-mem { ++ regulator-off-in-suspend; ++ }; ++ }; ++ }; ++ ++ rk809_codec: codec { ++ #sound-dai-cells = <0>; ++ compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; ++ clocks = <&cru I2S1_MCLKOUT>; ++ clock-names = "mclk"; ++ assigned-clocks = <&cru I2S1_MCLKOUT>, <&cru I2S1_MCLK_TX_IOE>; ++ assigned-clock-rates = <12288000>; ++ assigned-clock-parents = <&cru I2S1_MCLKOUT_TX>, <&cru I2S1_MCLKOUT_TX>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1m0_mclk>; ++ hp-volume = <20>; ++ spk-volume = <3>; ++ mic-in-differential; ++ status = "okay"; ++ }; ++ }; ++}; ++ ++&i2c1 { ++ status = "okay"; ++ ++ gt1x: gt1x@14 { ++ compatible = "goodix,gt1x"; ++ status = "disabled"; ++ reg = <0x14>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&touch_gpio>; ++ goodix,rst-gpio = <&gpio0 RK_PB6 GPIO_ACTIVE_HIGH>; ++ goodix,irq-gpio = <&gpio0 RK_PB5 IRQ_TYPE_LEVEL_LOW>; ++ }; ++ ++ gt9xx: gt9xx@5d { ++ compatible = "goodix,gt9xx"; ++ status = "okay"; ++ reg = <0x5d>; ++ reset-gpio = <&gpio0 RK_PB6 GPIO_ACTIVE_HIGH>; ++ touch-gpio = <&gpio0 RK_PB5 IRQ_TYPE_LEVEL_LOW>; ++ max-x = <7200>; ++ max-y = <1280>; ++ tp-size = <911>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&touch_gpio>; ++ power-supply = <&vcc3v3_lcd0_n>; ++ }; ++}; ++ ++&i2c5 { ++ status = "okay"; ++ ++ mxc6655xa: mxc6655xa@15 { ++ status = "okay"; ++ compatible = "gs_mxc6655xa"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&mxc6655xa_irq_gpio>; ++ reg = <0x15>; ++ irq-gpio = <&gpio3 RK_PC1 IRQ_TYPE_LEVEL_LOW>; ++ irq_enable = <0>; ++ poll_delay_ms = <30>; ++ type = ; ++ power-off-in-suspend = <1>; ++ layout = <1>; ++ }; ++}; ++ ++&i2s0_8ch { ++ status = "okay"; ++}; ++ ++&i2s1_8ch { ++ status = "okay"; ++ rockchip,clk-trcm = <1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1m0_sclktx ++ &i2s1m0_lrcktx ++ &i2s1m0_sdi0 ++ &i2s1m0_sdo0>; ++}; ++ ++&iep { ++ status = "okay"; ++}; ++ ++&iep_mmu { ++ status = "okay"; ++}; ++ ++&jpegd { ++ status = "okay"; ++}; ++ ++&jpegd_mmu { ++ status = "okay"; ++}; ++ ++&mpp_srv { ++ status = "okay"; ++}; ++ ++&nandc0 { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ status = "okay"; ++ ++ nand@0 { ++ reg = <0>; ++ nand-bus-width = <8>; ++ nand-ecc-mode = "hw"; ++ nand-ecc-strength = <16>; ++ nand-ecc-step-size = <1024>; ++ }; ++}; ++ ++&pinctrl { ++ ++ mxc6655xa { ++ mxc6655xa_irq_gpio: mxc6655xa_irq_gpio { ++ rockchip,pins = <3 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ pmic { ++ pmic_int: pmic_int { ++ rockchip,pins = ++ <0 RK_PA3 RK_FUNC_GPIO &pcfg_pull_up>; ++ }; ++ ++ soc_slppin_gpio: soc_slppin_gpio { ++ rockchip,pins = ++ <0 RK_PA2 RK_FUNC_GPIO &pcfg_output_low_pull_down>; ++ }; ++ ++ soc_slppin_slp: soc_slppin_slp { ++ rockchip,pins = ++ <0 RK_PA2 1 &pcfg_pull_up>; ++ }; ++ ++ soc_slppin_rst: soc_slppin_rst { ++ rockchip,pins = ++ <0 RK_PA2 2 &pcfg_pull_none>; ++ }; ++ }; ++ ++ touch { ++ touch_gpio: touch-gpio { ++ rockchip,pins = ++ <0 RK_PB5 RK_FUNC_GPIO &pcfg_pull_up>, ++ <0 RK_PB6 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ sdio-pwrseq { ++ wifi_enable_h: wifi-enable-h { ++ rockchip,pins = <3 RK_PD5 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ usb { ++ vcc5v0_host_en: vcc5v0-host-en { ++ rockchip,pins = <0 RK_PA6 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ ++ vcc5v0_otg_en: vcc5v0-otg-en { ++ rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++ ++ wireless-bluetooth { ++ uart8_gpios: uart8-gpios { ++ rockchip,pins = <2 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; ++ }; ++ }; ++}; ++ ++/** ++ * Model: TB-RK3568X ++ * ----------------------------------------------------------- ++ * There are 10 independent IO domains in RK3566/RK3568, including PMUIO[0:2] and VCCIO[1:7]. ++ * 1/ PMUIO0 and PMUIO1 are fixed-level power domains which cannot be configured; ++ * 2/ PMUIO2 and VCCIO1,VCCIO[3:7] domains require that their hardware power supply voltages ++ * must be consistent with the software configuration correspondingly ++ * a/ When the hardware IO level is connected to 1.8V, the software voltage configuration ++ * should also be configured to 1.8V accordingly; ++ * b/ When the hardware IO level is connected to 3.3V, the software voltage configuration ++ * should also be configured to 3.3V accordingly; ++ * 3/ VCCIO2 voltage control selection (0xFDC20140) ++ * BIT[0]: 0x0: from GPIO_0A7 (default) ++ * BIT[0]: 0x1: from GRF ++ * Default is determined by Pin FLASH_VOL_SEL/GPIO0_A7: ++ * L:VCCIO2 must supply 3.3V ++ * H:VCCIO2 must supply 1.8V ++ * | supply | domain | net | source | voltage | ++ * ----------------------------------------------------------- ++ * | pmuio1-supply | PMUIO1 | vcc3v3_pmu | LDO6 | 3.3V | ++ * | pmuio2-supply | PMUIO2 | vcc3v3_pmu | LDO6 | 3.3V | ++ * | vccio1-supply | VCCIO1 | vccio_acodec | LDO4 | 1.8V | ++ * | vccio2-supply | VCCIO2 | vccio_flash | vcc_1v8 | 1.8V | ++ * | vccio3-supply | VCCIO3 | vccio_sd | LDO5 | 3.3V | ++ * | vccio4-supply | VCCIO4 | vcc_1v8 | DCDC5 | 1.8V | ++ * | vccio5-supply | VCCIO5 | vcc_3v3 | SWITCH1 | 3.3V | ++ * | vccio6-supply | VCCIO6 | vcc_1v8 | DCDC5 | 1.8V | ++ * | vccio7-supply | VCCIO7 | vcc_3v3 | SWITCH1 | 3.3V | ++ * ----------------------------------------------------------- ++ */ ++&pmu_io_domains { ++ status = "okay"; ++ pmuio1-supply = <&vcc3v3_pmu>; ++ pmuio2-supply = <&vcc3v3_pmu>; ++ vccio1-supply = <&vccio_acodec>; ++ // vccio2-supply = <&vccio_flash>; ++ vccio3-supply = <&vccio_sd>; ++ vccio4-supply = <&vcc_1v8>; ++ vccio5-supply = <&vcc_3v3>; ++ vccio6-supply = <&vcc_1v8>; ++ vccio7-supply = <&vcc_3v3>; ++}; ++ ++&pwm0 { ++ status = "okay"; ++}; ++ ++&pwm4 { ++ status = "okay"; ++}; ++ ++&pwm5 { ++ status = "okay"; ++}; ++ ++&pwm7 { ++ status = "okay"; ++ ++ compatible = "rockchip,remotectl-pwm"; ++ remote_pwm_id = <3>; ++ handle_cpu_id = <1>; ++ remote_support_psci = <0>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pwm7_pins>; ++ ++ ir_key1 { ++ rockchip,usercode = <0x4040>; ++ rockchip,key_table = ++ <0xf2 KEY_REPLY>, ++ <0xba KEY_BACK>, ++ <0xf4 KEY_UP>, ++ <0xf1 KEY_DOWN>, ++ <0xef KEY_LEFT>, ++ <0xee KEY_RIGHT>, ++ <0xbd KEY_HOME>, ++ <0xea KEY_VOLUMEUP>, ++ <0xe3 KEY_VOLUMEDOWN>, ++ <0xe2 KEY_SEARCH>, ++ <0xb2 KEY_POWER>, ++ <0xbc KEY_MUTE>, ++ <0xec KEY_MENU>, ++ <0xbf 0x190>, ++ <0xe0 0x191>, ++ <0xe1 0x192>, ++ <0xe9 183>, ++ <0xe6 248>, ++ <0xe8 185>, ++ <0xe7 186>, ++ <0xf0 388>, ++ <0xbe 0x175>; ++ }; ++ ++ ir_key2 { ++ rockchip,usercode = <0xff00>; ++ rockchip,key_table = ++ <0xf9 KEY_HOME>, ++ <0xbf KEY_BACK>, ++ <0xfb KEY_MENU>, ++ <0xaa KEY_REPLY>, ++ <0xb9 KEY_UP>, ++ <0xe9 KEY_DOWN>, ++ <0xb8 KEY_LEFT>, ++ <0xea KEY_RIGHT>, ++ <0xeb KEY_VOLUMEDOWN>, ++ <0xef KEY_VOLUMEUP>, ++ <0xf7 KEY_MUTE>, ++ <0xe7 KEY_POWER>, ++ <0xfc KEY_POWER>, ++ <0xa9 KEY_VOLUMEDOWN>, ++ <0xa8 KEY_VOLUMEDOWN>, ++ <0xe0 KEY_VOLUMEDOWN>, ++ <0xa5 KEY_VOLUMEDOWN>, ++ <0xab 183>, ++ <0xb7 388>, ++ <0xe8 388>, ++ <0xf8 184>, ++ <0xaf 185>, ++ <0xed KEY_VOLUMEDOWN>, ++ <0xee 186>, ++ <0xb3 KEY_VOLUMEDOWN>, ++ <0xf1 KEY_VOLUMEDOWN>, ++ <0xf2 KEY_VOLUMEDOWN>, ++ <0xf3 KEY_SEARCH>, ++ <0xb4 KEY_VOLUMEDOWN>, ++ <0xbe KEY_SEARCH>; ++ }; ++ ++ ir_key3 { ++ rockchip,usercode = <0x1dcc>; ++ rockchip,key_table = ++ <0xee KEY_REPLY>, ++ <0xf0 KEY_BACK>, ++ <0xf8 KEY_UP>, ++ <0xbb KEY_DOWN>, ++ <0xef KEY_LEFT>, ++ <0xed KEY_RIGHT>, ++ <0xfc KEY_HOME>, ++ <0xf1 KEY_VOLUMEUP>, ++ <0xfd KEY_VOLUMEDOWN>, ++ <0xb7 KEY_SEARCH>, ++ <0xff KEY_POWER>, ++ <0xf3 KEY_MUTE>, ++ <0xbf KEY_MENU>, ++ <0xf9 0x191>, ++ <0xf5 0x192>, ++ <0xb3 388>, ++ <0xbe KEY_1>, ++ <0xba KEY_2>, ++ <0xb2 KEY_3>, ++ <0xbd KEY_4>, ++ <0xf9 KEY_5>, ++ <0xb1 KEY_6>, ++ <0xfc KEY_7>, ++ <0xf8 KEY_8>, ++ <0xb0 KEY_9>, ++ <0xb6 KEY_0>, ++ <0xb5 KEY_BACKSPACE>; ++ }; ++}; ++ ++&rk_rga { ++ status = "okay"; ++}; ++ ++&rkvdec { ++ status = "okay"; ++}; ++ ++&rkvdec_mmu { ++ status = "okay"; ++}; ++ ++&rkvenc { ++ venc-supply = <&vdd_logic>; ++ status = "okay"; ++}; ++ ++&rkvenc_mmu { ++ status = "okay"; ++}; ++ ++&rknpu { ++ rknpu-supply = <&vdd_npu>; ++ status = "okay"; ++}; ++ ++&rknpu_mmu { ++ status = "okay"; ++}; ++ ++&route_hdmi { ++ status = "okay"; ++ connect = <&vp0_out_hdmi>; ++}; ++ ++&saradc { ++ status = "okay"; ++ vref-supply = <&vcca_1v8>; ++}; ++ ++&sdhci { ++ bus-width = <8>; ++ supports-emmc; ++ non-removable; ++ max-frequency = <200000000>; ++ status = "okay"; ++}; ++ ++&sdmmc0 { ++ max-frequency = <150000000>; ++ supports-sd; ++ bus-width = <4>; ++ cap-mmc-highspeed; ++ cap-sd-highspeed; ++ disable-wp; ++ sd-uhs-sdr104; ++ vmmc-supply = <&vcc3v3_sd>; ++ vqmmc-supply = <&vccio_sd>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&sdmmc0_bus4 &sdmmc0_clk &sdmmc0_cmd &sdmmc0_det>; ++ status = "okay"; ++}; ++ ++&sfc { ++ status = "okay"; ++}; ++ ++&spdif_8ch { ++ status = "okay"; ++}; ++ ++&tsadc { ++ status = "okay"; ++}; ++ ++&u2phy0_host { ++ phy-supply = <&vcc5v0_host>; ++ status = "okay"; ++}; ++ ++&u2phy0_otg { ++ vbus-supply = <&vcc5v0_otg>; ++ status = "okay"; ++}; ++ ++&u2phy1_host { ++ phy-supply = <&vcc5v0_host>; ++ status = "okay"; ++}; ++ ++&u2phy1_otg { ++ phy-supply = <&vcc5v0_host>; ++ status = "okay"; ++}; ++ ++&usb2phy0 { ++ status = "okay"; ++}; ++ ++&usb2phy1 { ++ status = "okay"; ++}; ++ ++&usb_host0_ehci { ++ status = "okay"; ++}; ++ ++&usb_host0_ohci { ++ status = "okay"; ++}; ++ ++&usb_host1_ehci { ++ status = "okay"; ++}; ++ ++&usb_host1_ohci { ++ status = "okay"; ++}; ++ ++&usbdrd_dwc3 { ++ dr_mode = "otg"; ++ extcon = <&usb2phy0>; ++ status = "okay"; ++}; ++ ++&usbdrd30 { ++ status = "okay"; ++}; ++ ++&usbhost_dwc3 { ++ status = "okay"; ++}; ++ ++&usbhost30 { ++ status = "okay"; ++}; ++ ++&vad { ++ rockchip,audio-src = <&i2s1_8ch>; ++ rockchip,buffer-time-ms = <128>; ++ rockchip,det-channel = <0>; ++ rockchip,mode = <0>; ++}; ++ ++&vdpu { ++ status = "okay"; ++}; ++ ++&vdpu_mmu { ++ status = "okay"; ++}; ++ ++&vepu { ++ status = "okay"; ++}; ++ ++&vepu_mmu { ++ status = "okay"; ++}; ++ ++&vop { ++ status = "okay"; ++ assigned-clocks = <&cru DCLK_VOP0>, <&cru DCLK_VOP1>; ++ assigned-clock-parents = <&pmucru PLL_HPLL>, <&cru PLL_VPLL>; ++}; ++ ++&vop_mmu { ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/rockchip/rk3568.dtsi b/arch/arm64/boot/dts/rockchip/rk3568.dtsi +new file mode 100755 +index 000000000000..372617fee467 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rk3568.dtsi +@@ -0,0 +1,3449 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co., Ltd. ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include "rk3568-dram-default-timing.dtsi" ++ ++/ { ++ compatible = "rockchip,rk3568"; ++ ++ interrupt-parent = <&gic>; ++ #address-cells = <2>; ++ #size-cells = <2>; ++ ++ aliases { ++ csi2dphy0 = &csi2_dphy0; ++ csi2dphy1 = &csi2_dphy1; ++ csi2dphy2 = &csi2_dphy2; ++ dsi0 = &dsi0; ++ dsi1 = &dsi1; ++ ethernet0 = &gmac0; ++ ethernet1 = &gmac1; ++ gpio0 = &gpio0; ++ gpio1 = &gpio1; ++ gpio2 = &gpio2; ++ gpio3 = &gpio3; ++ gpio4 = &gpio4; ++ i2c0 = &i2c0; ++ i2c1 = &i2c1; ++ i2c2 = &i2c2; ++ i2c3 = &i2c3; ++ i2c4 = &i2c4; ++ i2c5 = &i2c5; ++ mmc0 = &sdhci; ++ mmc1 = &sdmmc0; ++ mmc2 = &sdmmc1; ++ mmc3 = &sdmmc2; ++ serial0 = &uart0; ++ serial1 = &uart1; ++ serial2 = &uart2; ++ serial3 = &uart3; ++ serial4 = &uart4; ++ serial5 = &uart5; ++ serial6 = &uart6; ++ serial7 = &uart7; ++ serial8 = &uart8; ++ serial9 = &uart9; ++ spi0 = &spi0; ++ spi1 = &spi1; ++ spi2 = &spi2; ++ spi3 = &spi3; ++ }; ++ ++ cpus { ++ #address-cells = <2>; ++ #size-cells = <0>; ++ ++ cpu0: cpu@0 { ++ device_type = "cpu"; ++ compatible = "arm,cortex-a55"; ++ reg = <0x0 0x0>; ++ enable-method = "psci"; ++ clocks = <&scmi_clk 0>; ++ operating-points-v2 = <&cpu0_opp_table>; ++ cpu-idle-states = <&CPU_SLEEP>; ++ #cooling-cells = <2>; ++ dynamic-power-coefficient = <187>; ++ }; ++ ++ cpu1: cpu@100 { ++ device_type = "cpu"; ++ compatible = "arm,cortex-a55"; ++ reg = <0x0 0x100>; ++ enable-method = "psci"; ++ clocks = <&scmi_clk 0>; ++ operating-points-v2 = <&cpu0_opp_table>; ++ cpu-idle-states = <&CPU_SLEEP>; ++ }; ++ ++ cpu2: cpu@200 { ++ device_type = "cpu"; ++ compatible = "arm,cortex-a55"; ++ reg = <0x0 0x200>; ++ enable-method = "psci"; ++ clocks = <&scmi_clk 0>; ++ operating-points-v2 = <&cpu0_opp_table>; ++ cpu-idle-states = <&CPU_SLEEP>; ++ }; ++ ++ cpu3: cpu@300 { ++ device_type = "cpu"; ++ compatible = "arm,cortex-a55"; ++ reg = <0x0 0x300>; ++ enable-method = "psci"; ++ clocks = <&scmi_clk 0>; ++ operating-points-v2 = <&cpu0_opp_table>; ++ cpu-idle-states = <&CPU_SLEEP>; ++ }; ++ ++ idle-states { ++ entry-method = "psci"; ++ CPU_SLEEP: cpu-sleep { ++ compatible = "arm,idle-state"; ++ local-timer-stop; ++ arm,psci-suspend-param = <0x0010000>; ++ entry-latency-us = <100>; ++ exit-latency-us = <120>; ++ min-residency-us = <1000>; ++ }; ++ }; ++ }; ++ ++ cpu0_opp_table: cpu0-opp-table { ++ compatible = "operating-points-v2"; ++ opp-shared; ++ ++ mbist-vmin = <825000 900000 950000>; ++ nvmem-cells = <&cpu_leakage>, <&core_pvtm>, <&mbist_vmin>; ++ nvmem-cell-names = "leakage", "pvtm", "mbist-vmin"; ++ rockchip,pvtm-voltage-sel = < ++ 0 82000 0 ++ 82001 93000 1 ++ 93001 100000 2 ++ >; ++ rockchip,pvtm-freq = <408000>; ++ rockchip,pvtm-volt = <900000>; ++ rockchip,pvtm-ch = <0 5>; ++ rockchip,pvtm-sample-time = <1000>; ++ rockchip,pvtm-number = <10>; ++ rockchip,pvtm-error = <1000>; ++ rockchip,pvtm-ref-temp = <40>; ++ rockchip,pvtm-temp-prop = <26 26>; ++ rockchip,thermal-zone = "soc-thermal"; ++ rockchip,temp-hysteresis = <5000>; ++ rockchip,low-temp = <0>; ++ rockchip,low-temp-adjust-volt = < ++ /* MHz MHz uV */ ++ 0 1608 75000 ++ >; ++ ++ opp-408000000 { ++ opp-hz = /bits/ 64 <408000000>; ++ opp-microvolt = <825000 825000 1150000>; ++ clock-latency-ns = <40000>; ++ }; ++ opp-600000000 { ++ opp-hz = /bits/ 64 <600000000>; ++ opp-microvolt = <825000 825000 1150000>; ++ clock-latency-ns = <40000>; ++ }; ++ opp-816000000 { ++ opp-hz = /bits/ 64 <816000000>; ++ opp-microvolt = <825000 825000 1150000>; ++ clock-latency-ns = <40000>; ++ opp-suspend; ++ }; ++ opp-1104000000 { ++ opp-hz = /bits/ 64 <1104000000>; ++ opp-microvolt = <825000 825000 1150000>; ++ clock-latency-ns = <40000>; ++ }; ++ opp-1416000000 { ++ opp-hz = /bits/ 64 <1416000000>; ++ opp-microvolt = <925000 925000 1150000>; ++ clock-latency-ns = <40000>; ++ }; ++ opp-1608000000 { ++ opp-hz = /bits/ 64 <1608000000>; ++ opp-microvolt = <1000000 1000000 1150000>; ++ clock-latency-ns = <40000>; ++ }; ++ opp-1800000000 { ++ opp-hz = /bits/ 64 <1800000000>; ++ opp-microvolt = <1050000 1050000 1150000>; ++ clock-latency-ns = <40000>; ++ }; ++ opp-1992000000 { ++ opp-hz = /bits/ 64 <1992000000>; ++ opp-microvolt = <1150000 1150000 1150000>; ++ clock-latency-ns = <40000>; ++ }; ++ }; ++ ++ arm-pmu { ++ compatible = "arm,cortex-a55-pmu", "arm,armv8-pmuv3"; ++ interrupts = , ++ , ++ , ++ ; ++ interrupt-affinity = <&cpu0>, <&cpu1>, <&cpu2>, <&cpu3>; ++ }; ++ ++ cpuinfo { ++ compatible = "rockchip,cpuinfo"; ++ nvmem-cells = <&otp_id>, <&otp_cpu_version>, <&cpu_code>; ++ nvmem-cell-names = "id", "cpu-version", "cpu-code"; ++ }; ++ ++ display_subsystem: display-subsystem { ++ compatible = "rockchip,display-subsystem"; ++ memory-region = <&drm_logo>, <&drm_cubic_lut>; ++ memory-region-names = "drm-logo", "drm-cubic-lut"; ++ ports = <&vop_out>; ++ devfreq = <&dmc>; ++ ++ route { ++ route_dsi0: route-dsi0 { ++ status = "disabled"; ++ logo,uboot = "logo.bmp"; ++ logo,kernel = "logo_kernel.bmp"; ++ logo,mode = "center"; ++ charge_logo,mode = "center"; ++ connect = <&vp0_out_dsi0>; ++ }; ++ route_dsi1: route-dsi1 { ++ status = "disabled"; ++ logo,uboot = "logo.bmp"; ++ logo,kernel = "logo_kernel.bmp"; ++ logo,mode = "center"; ++ charge_logo,mode = "center"; ++ connect = <&vp0_out_dsi1>; ++ }; ++ route_edp: route-edp { ++ status = "disabled"; ++ logo,uboot = "logo.bmp"; ++ logo,kernel = "logo_kernel.bmp"; ++ logo,mode = "center"; ++ charge_logo,mode = "center"; ++ connect = <&vp0_out_edp>; ++ }; ++ route_hdmi: route-hdmi { ++ status = "disabled"; ++ logo,uboot = "logo.bmp"; ++ logo,kernel = "logo_kernel.bmp"; ++ logo,mode = "center"; ++ charge_logo,mode = "center"; ++ connect = <&vp1_out_hdmi>; ++ }; ++ route_lvds: route-lvds { ++ status = "disabled"; ++ logo,uboot = "logo.bmp"; ++ logo,kernel = "logo_kernel.bmp"; ++ logo,mode = "center"; ++ charge_logo,mode = "center"; ++ connect = <&vp1_out_lvds>; ++ }; ++ route_rgb: route-rgb { ++ status = "disabled"; ++ logo,uboot = "logo.bmp"; ++ logo,kernel = "logo_kernel.bmp"; ++ logo,mode = "center"; ++ charge_logo,mode = "center"; ++ connect = <&vp2_out_rgb>; ++ }; ++ }; ++ }; ++ ++ firmware { ++ optee: optee { ++ compatible = "linaro,optee-tz"; ++ method = "smc"; ++ }; ++ ++ scmi: scmi { ++ compatible = "arm,scmi-smc"; ++ shmem = <&scmi_shmem>; ++ arm,smc-id = <0x82000010>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ scmi_clk: protocol@14 { ++ reg = <0x14>; ++ #clock-cells = <1>; ++ ++ rockchip,clk-init = <1416000000>; ++ }; ++ }; ++ ++ sdei: sdei { ++ compatible = "arm,sdei-1.0"; ++ method = "smc"; ++ }; ++ }; ++ ++ mpp_srv: mpp-srv { ++ compatible = "rockchip,mpp-service"; ++ rockchip,taskqueue-count = <6>; ++ rockchip,resetgroup-count = <6>; ++ status = "disabled"; ++ }; ++ ++ psci { ++ compatible = "arm,psci-1.0"; ++ method = "smc"; ++ }; ++ ++ reserved_memory: reserved-memory { ++ #address-cells = <2>; ++ #size-cells = <2>; ++ ranges; ++ ++ drm_logo: drm-logo@00000000 { ++ compatible = "rockchip,drm-logo"; ++ reg = <0x0 0x0 0x0 0x0>; ++ }; ++ ++ drm_cubic_lut: drm-cubic-lut@00000000 { ++ compatible = "rockchip,drm-cubic-lut"; ++ reg = <0x0 0x0 0x0 0x0>; ++ }; ++ }; ++ ++ rockchip_suspend: rockchip-suspend { ++ compatible = "rockchip,pm-rk3568"; ++ status = "disabled"; ++ rockchip,sleep-debug-en = <1>; ++ rockchip,sleep-mode-config = < ++ (0 ++ | RKPM_SLP_ARMOFF_LOGOFF ++ | RKPM_SLP_CENTER_OFF ++ | RKPM_SLP_HW_PLLS_OFF ++ | RKPM_SLP_PMUALIVE_32K ++ | RKPM_SLP_OSC_DIS ++ | RKPM_SLP_PMIC_LP ++ | RKPM_SLP_32K_PVTM ++ ) ++ >; ++ rockchip,wakeup-config = < ++ (0 ++ | RKPM_GPIO_WKUP_EN ++ ) ++ >; ++ }; ++ ++ rockchip_system_monitor: rockchip-system-monitor { ++ compatible = "rockchip,system-monitor"; ++ ++ rockchip,thermal-zone = "soc-thermal"; ++ }; ++ ++ thermal_zones: thermal-zones { ++ soc_thermal: soc-thermal { ++ polling-delay-passive = <20>; /* milliseconds */ ++ polling-delay = <1000>; /* milliseconds */ ++ sustainable-power = <905>; /* milliwatts */ ++ ++ thermal-sensors = <&tsadc 0>; ++ trips { ++ threshold: trip-point-0 { ++ temperature = <75000>; ++ hysteresis = <2000>; ++ type = "passive"; ++ }; ++ target: trip-point-1 { ++ temperature = <85000>; ++ hysteresis = <2000>; ++ type = "passive"; ++ }; ++ soc_crit: soc-crit { ++ /* millicelsius */ ++ temperature = <115000>; ++ /* millicelsius */ ++ hysteresis = <2000>; ++ type = "critical"; ++ }; ++ }; ++ cooling-maps { ++ map0 { ++ trip = <&target>; ++ cooling-device = <&cpu0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; ++ contribution = <1024>; ++ }; ++ map1 { ++ trip = <&target>; ++ cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; ++ contribution = <1024>; ++ }; ++ }; ++ }; ++ ++ gpu_thermal: gpu-thermal { ++ polling-delay-passive = <20>; /* milliseconds */ ++ polling-delay = <1000>; /* milliseconds */ ++ ++ thermal-sensors = <&tsadc 1>; ++ }; ++ }; ++ ++ timer { ++ compatible = "arm,armv8-timer"; ++ interrupts = , ++ , ++ , ++ ; ++ arm,no-tick-in-suspend; ++ }; ++ ++ gmac0_clkin: external-gmac0-clock { ++ compatible = "fixed-clock"; ++ clock-frequency = <125000000>; ++ clock-output-names = "gmac0_clkin"; ++ #clock-cells = <0>; ++ }; ++ ++ gmac1_clkin: external-gmac1-clock { ++ compatible = "fixed-clock"; ++ clock-frequency = <125000000>; ++ clock-output-names = "gmac1_clkin"; ++ #clock-cells = <0>; ++ }; ++ ++ gmac0_xpcsclk: xpcs-gmac0-clock { ++ compatible = "fixed-clock"; ++ clock-frequency = <125000000>; ++ clock-output-names = "clk_gmac0_xpcs_mii"; ++ #clock-cells = <0>; ++ }; ++ ++ gmac1_xpcsclk: xpcs-gmac1-clock { ++ compatible = "fixed-clock"; ++ clock-frequency = <125000000>; ++ clock-output-names = "clk_gmac1_xpcs_mii"; ++ #clock-cells = <0>; ++ }; ++ ++ i2s1_mclkin_rx: i2s1-mclkin-rx { ++ compatible = "fixed-clock"; ++ #clock-cells = <0>; ++ clock-frequency = <12288000>; ++ clock-output-names = "i2s1_mclkin_rx"; ++ }; ++ ++ i2s1_mclkin_tx: i2s1-mclkin-tx { ++ compatible = "fixed-clock"; ++ #clock-cells = <0>; ++ clock-frequency = <12288000>; ++ clock-output-names = "i2s1_mclkin_tx"; ++ }; ++ ++ i2s2_mclkin: i2s2-mclkin { ++ compatible = "fixed-clock"; ++ #clock-cells = <0>; ++ clock-frequency = <12288000>; ++ clock-output-names = "i2s2_mclkin"; ++ }; ++ ++ i2s3_mclkin: i2s3-mclkin { ++ compatible = "fixed-clock"; ++ #clock-cells = <0>; ++ clock-frequency = <12288000>; ++ clock-output-names = "i2s3_mclkin"; ++ }; ++ ++ mpll: mpll { ++ compatible = "fixed-clock"; ++ #clock-cells = <0>; ++ clock-frequency = <800000000>; ++ clock-output-names = "mpll"; ++ }; ++ ++ xin24m: xin24m { ++ compatible = "fixed-clock"; ++ #clock-cells = <0>; ++ clock-frequency = <24000000>; ++ clock-output-names = "xin24m"; ++ }; ++ ++ xin32k: xin32k { ++ compatible = "fixed-clock"; ++ clock-frequency = <32768>; ++ clock-output-names = "xin32k"; ++ #clock-cells = <0>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&clk32k_out0>; ++ }; ++ ++ scmi_shmem: scmi-shmem@10f000 { ++ compatible = "arm,scmi-shmem"; ++ reg = <0x0 0x0010f000 0x0 0x100>; ++ }; ++ ++ sata0: sata@fc000000 { ++ compatible = "snps,dwc-ahci"; ++ reg = <0 0xfc000000 0 0x1000>; ++ clocks = <&cru ACLK_SATA0>, <&cru CLK_SATA0_PMALIVE>, ++ <&cru CLK_SATA0_RXOOB>; ++ clock-names = "sata", "pmalive", "rxoob"; ++ interrupts = ; ++ interrupt-names = "hostc"; ++ phys = <&combphy0_us PHY_TYPE_SATA>; ++ phy-names = "sata-phy"; ++ ports-implemented = <0x1>; ++ power-domains = <&power RK3568_PD_PIPE>; ++ status = "disabled"; ++ }; ++ ++ sata1: sata@fc400000 { ++ compatible = "snps,dwc-ahci"; ++ reg = <0 0xfc400000 0 0x1000>; ++ clocks = <&cru ACLK_SATA1>, <&cru CLK_SATA1_PMALIVE>, ++ <&cru CLK_SATA1_RXOOB>; ++ clock-names = "sata", "pmalive", "rxoob"; ++ interrupts = ; ++ interrupt-names = "hostc"; ++ phys = <&combphy1_usq PHY_TYPE_SATA>; ++ phy-names = "sata-phy"; ++ ports-implemented = <0x1>; ++ power-domains = <&power RK3568_PD_PIPE>; ++ status = "disabled"; ++ }; ++ ++ sata2: sata@fc800000 { ++ compatible = "snps,dwc-ahci"; ++ reg = <0 0xfc800000 0 0x1000>; ++ clocks = <&cru ACLK_SATA2>, <&cru CLK_SATA2_PMALIVE>, ++ <&cru CLK_SATA2_RXOOB>; ++ clock-names = "sata", "pmalive", "rxoob"; ++ interrupts = ; ++ interrupt-names = "hostc"; ++ phys = <&combphy2_psq PHY_TYPE_SATA>; ++ phy-names = "sata-phy"; ++ ports-implemented = <0x1>; ++ power-domains = <&power RK3568_PD_PIPE>; ++ status = "disabled"; ++ }; ++ ++ usbdrd30: usbdrd { ++ compatible = "rockchip,rk3568-dwc3", "rockchip,rk3399-dwc3"; ++ clocks = <&cru CLK_USB3OTG0_REF>, <&cru CLK_USB3OTG0_SUSPEND>, ++ <&cru ACLK_USB3OTG0>, <&cru PCLK_PIPE>; ++ clock-names = "ref_clk", "suspend_clk", ++ "bus_clk", "pipe_clk"; ++ #address-cells = <2>; ++ #size-cells = <2>; ++ ranges; ++ status = "disabled"; ++ ++ usbdrd_dwc3: dwc3@fcc00000 { ++ compatible = "snps,dwc3"; ++ reg = <0x0 0xfcc00000 0x0 0x400000>; ++ interrupts = ; ++ dr_mode = "otg"; ++ phys = <&u2phy0_otg>, <&combphy0_us PHY_TYPE_USB3>; ++ phy-names = "usb2-phy", "usb3-phy"; ++ phy_type = "utmi_wide"; ++ power-domains = <&power RK3568_PD_PIPE>; ++ resets = <&cru SRST_USB3OTG0>; ++ reset-names = "usb3-otg"; ++ snps,dis_enblslpm_quirk; ++ snps,dis-u2-freeclk-exists-quirk; ++ snps,dis-del-phy-power-chg-quirk; ++ snps,dis-tx-ipgap-linecheck-quirk; ++ snps,xhci-trb-ent-quirk; ++ status = "disabled"; ++ }; ++ }; ++ ++ usbhost30: usbhost { ++ compatible = "rockchip,rk3568-dwc3", "rockchip,rk3399-dwc3"; ++ clocks = <&cru CLK_USB3OTG1_REF>, <&cru CLK_USB3OTG1_SUSPEND>, ++ <&cru ACLK_USB3OTG1>, <&cru PCLK_PIPE>; ++ clock-names = "ref_clk", "suspend_clk", ++ "bus_clk", "pipe_clk"; ++ #address-cells = <2>; ++ #size-cells = <2>; ++ ranges; ++ status = "disabled"; ++ ++ usbhost_dwc3: dwc3@fd000000 { ++ compatible = "snps,dwc3"; ++ reg = <0x0 0xfd000000 0x0 0x400000>; ++ interrupts = ; ++ dr_mode = "host"; ++ phys = <&u2phy0_host>, <&combphy1_usq PHY_TYPE_USB3>; ++ phy-names = "usb2-phy", "usb3-phy"; ++ phy_type = "utmi_wide"; ++ power-domains = <&power RK3568_PD_PIPE>; ++ resets = <&cru SRST_USB3OTG1>; ++ reset-names = "usb3-host"; ++ snps,dis_enblslpm_quirk; ++ snps,dis-u2-freeclk-exists-quirk; ++ snps,dis-del-phy-power-chg-quirk; ++ snps,dis-tx-ipgap-linecheck-quirk; ++ snps,xhci-trb-ent-quirk; ++ status = "disabled"; ++ }; ++ }; ++ ++ gic: interrupt-controller@fd400000 { ++ compatible = "arm,gic-v3"; ++ #interrupt-cells = <3>; ++ #address-cells = <2>; ++ #size-cells = <2>; ++ ranges; ++ interrupt-controller; ++ ++ reg = <0x0 0xfd400000 0 0x10000>, /* GICD */ ++ <0x0 0xfd460000 0 0xc0000>; /* GICR */ ++ interrupts = ; ++ its: interrupt-controller@fd440000 { ++ compatible = "arm,gic-v3-its"; ++ msi-controller; ++ #msi-cells = <1>; ++ reg = <0x0 0xfd440000 0x0 0x20000>; ++ }; ++ }; ++ ++ usb_host0_ehci: usb@fd800000 { ++ compatible = "generic-ehci"; ++ reg = <0x0 0xfd800000 0x0 0x40000>; ++ interrupts = ; ++ clocks = <&cru HCLK_USB2HOST0>, <&cru HCLK_USB2HOST0_ARB>, ++ <&cru PCLK_USB>, <&usb2phy1>; ++ clock-names = "usbhost", "arbiter", "pclk", "utmi"; ++ phys = <&u2phy1_otg>; ++ phy-names = "usb2-phy"; ++ status = "disabled"; ++ }; ++ ++ usb_host0_ohci: usb@fd840000 { ++ compatible = "generic-ohci"; ++ reg = <0x0 0xfd840000 0x0 0x40000>; ++ interrupts = ; ++ clocks = <&cru HCLK_USB2HOST0>, <&cru HCLK_USB2HOST0_ARB>, ++ <&cru PCLK_USB>, <&usb2phy1>; ++ clock-names = "usbhost", "arbiter", "pclk", "utmi"; ++ phys = <&u2phy1_otg>; ++ phy-names = "usb2-phy"; ++ status = "disabled"; ++ }; ++ ++ usb_host1_ehci: usb@fd880000 { ++ compatible = "generic-ehci"; ++ reg = <0x0 0xfd880000 0x0 0x40000>; ++ interrupts = ; ++ clocks = <&cru HCLK_USB2HOST1>, <&cru HCLK_USB2HOST1_ARB>, ++ <&cru PCLK_USB>, <&usb2phy1>; ++ clock-names = "usbhost", "arbiter", "pclk", "utmi"; ++ phys = <&u2phy1_host>; ++ phy-names = "usb2-phy"; ++ status = "disabled"; ++ }; ++ ++ usb_host1_ohci: usb@fd8c0000 { ++ compatible = "generic-ohci"; ++ reg = <0x0 0xfd8c0000 0x0 0x40000>; ++ interrupts = ; ++ clocks = <&cru HCLK_USB2HOST1>, <&cru HCLK_USB2HOST1_ARB>, ++ <&cru PCLK_USB>, <&usb2phy1>; ++ clock-names = "usbhost", "arbiter", "pclk", "utmi"; ++ phys = <&u2phy1_host>; ++ phy-names = "usb2-phy"; ++ status = "disabled"; ++ }; ++ ++ xpcs: syscon@fda00000 { ++ compatible = "rockchip,rk3568-xpcs", "syscon"; ++ reg = <0x0 0xfda00000 0x0 0x200000>; ++ status = "disabled"; ++ }; ++ ++ pmugrf: syscon@fdc20000 { ++ compatible = "rockchip,rk3568-pmugrf", "syscon", "simple-mfd"; ++ reg = <0x0 0xfdc20000 0x0 0x10000>; ++ ++ pmu_io_domains: io-domains { ++ compatible = "rockchip,rk3568-pmu-io-voltage-domain"; ++ status = "disabled"; ++ }; ++ ++ reboot_mode: reboot-mode { ++ compatible = "syscon-reboot-mode"; ++ offset = <0x200>; ++ mode-bootloader = ; ++ mode-charge = ; ++ mode-fastboot = ; ++ mode-loader = ; ++ mode-normal = ; ++ mode-recovery = ; ++ mode-ums = ; ++ mode-panic = ; ++ mode-watchdog = ; ++ }; ++ }; ++ ++ pipegrf: syscon@fdc50000 { ++ compatible = "rockchip,rk3568-pipegrf", "syscon"; ++ reg = <0x0 0xfdc50000 0x0 0x1000>; ++ }; ++ ++ grf: syscon@fdc60000 { ++ compatible = "rockchip,rk3568-grf", "syscon", "simple-mfd"; ++ reg = <0x0 0xfdc60000 0x0 0x10000>; ++ ++ io_domains: io-domains { ++ compatible = "rockchip,rk3568-io-voltage-domain"; ++ status = "disabled"; ++ }; ++ ++ lvds: lvds { ++ compatible = "rockchip,rk3568-lvds"; ++ phys = <&video_phy0>; ++ phy-names = "phy"; ++ status = "disabled"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ lvds_in_vp1: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&vp1_out_lvds>; ++ status = "disabled"; ++ }; ++ ++ lvds_in_vp2: endpoint@2 { ++ reg = <2>; ++ remote-endpoint = <&vp2_out_lvds>; ++ status = "disabled"; ++ }; ++ }; ++ }; ++ }; ++ ++ rgb: rgb { ++ compatible = "rockchip,rk3568-rgb"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&lcdc_ctl>; ++ status = "disabled"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ rgb_in_vp2: endpoint@2 { ++ reg = <2>; ++ remote-endpoint = <&vp2_out_rgb>; ++ status = "disabled"; ++ }; ++ }; ++ }; ++ }; ++ ++ }; ++ ++ pipe_phy_grf0: syscon@fdc70000 { ++ compatible = "rockchip,pipe-phy-grf", "syscon"; ++ reg = <0x0 0xfdc70000 0x0 0x1000>; ++ }; ++ ++ pipe_phy_grf1: syscon@fdc80000 { ++ compatible = "rockchip,pipe-phy-grf", "syscon"; ++ reg = <0x0 0xfdc80000 0x0 0x1000>; ++ }; ++ ++ pipe_phy_grf2: syscon@fdc90000 { ++ compatible = "rockchip,pipe-phy-grf", "syscon"; ++ reg = <0x0 0xfdc90000 0x0 0x1000>; ++ }; ++ ++ usb2phy0_grf: syscon@fdca0000 { ++ compatible = "rockchip,rk3568-usb2phy-grf", "syscon"; ++ reg = <0x0 0xfdca0000 0x0 0x8000>; ++ }; ++ ++ usb2phy1_grf: syscon@fdca8000 { ++ compatible = "rockchip,rk3568-usb2phy-grf", "syscon"; ++ reg = <0x0 0xfdca8000 0x0 0x8000>; ++ }; ++ ++ edp_phy: edp-phy@fdcb0000 { ++ compatible = "rockchip,rk3568-edp-phy"; ++ reg = <0x0 0xfdcb0000 0x0 0x8000>; ++ clocks = <&pmucru XIN_OSC0_EDPPHY_G>, <&cru PCLK_EDPPHY_GRF>; ++ clock-names = "refclk", "pclk"; ++ resets = <&cru SRST_P_EDPPHY_GRF>; ++ reset-names = "apb"; ++ #phy-cells = <0>; ++ status = "disabled"; ++ }; ++ ++ pcie30_phy_grf: syscon@fdcb8000 { ++ compatible = "rockchip,pcie30-phy-grf", "syscon"; ++ reg = <0x0 0xfdcb8000 0x0 0x10000>; ++ }; ++ ++ sram: sram@fdcc0000 { ++ compatible = "mmio-sram"; ++ reg = <0x0 0xfdcc0000 0x0 0xb000>; ++ ++ #address-cells = <1>; ++ #size-cells = <1>; ++ ranges = <0x0 0x0 0xfdcc0000 0xb000>; ++ ++ /* start address and size should be 4k algin */ ++ rkvdec_sram: rkvdec-sram@0 { ++ reg = <0x0 0xb000>; ++ }; ++ }; ++ ++ pmucru: clock-controller@fdd00000 { ++ compatible = "rockchip,rk3568-pmucru"; ++ reg = <0x0 0xfdd00000 0x0 0x1000>; ++ rockchip,grf = <&grf>; ++ rockchip,pmugrf = <&pmugrf>; ++ #clock-cells = <1>; ++ #reset-cells = <1>; ++ ++ assigned-clocks = <&pmucru SCLK_32K_IOE>; ++ assigned-clock-parents = <&pmucru CLK_RTC_32K>; ++ }; ++ ++ cru: clock-controller@fdd20000 { ++ compatible = "rockchip,rk3568-cru"; ++ reg = <0x0 0xfdd20000 0x0 0x1000>; ++ rockchip,grf = <&grf>; ++ #clock-cells = <1>; ++ #reset-cells = <1>; ++ ++ assigned-clocks = ++ <&pmucru CLK_RTC_32K>, <&cru ACLK_RKVDEC_PRE>, ++ <&cru CLK_RKVDEC_CORE>, <&pmucru PLL_PPLL>, ++ <&pmucru PCLK_PMU>, <&cru PLL_CPLL>, ++ <&cru CPLL_500M>, <&cru CPLL_333M>, ++ <&cru CPLL_250M>, <&cru CPLL_125M>, ++ <&cru CPLL_100M>, <&cru CPLL_62P5M>, ++ <&cru CPLL_50M>, <&cru CPLL_25M>, ++ <&cru PLL_GPLL>, ++ <&cru ACLK_BUS>, <&cru PCLK_BUS>, ++ <&cru ACLK_TOP_HIGH>, <&cru ACLK_TOP_LOW>, ++ <&cru HCLK_TOP>, <&cru PCLK_TOP>, ++ <&cru ACLK_PERIMID>, <&cru HCLK_PERIMID>, ++ <&cru PLL_NPLL>, <&cru ACLK_PIPE>, ++ <&cru PCLK_PIPE>, <&cru CLK_I2S0_8CH_TX_SRC>, ++ <&cru CLK_I2S0_8CH_RX_SRC>, <&cru CLK_I2S1_8CH_TX_SRC>, ++ <&cru CLK_I2S1_8CH_RX_SRC>, <&cru CLK_I2S2_2CH_SRC>, ++ <&cru CLK_I2S2_2CH_SRC>, <&cru CLK_I2S3_2CH_RX_SRC>, ++ <&cru CLK_I2S3_2CH_TX_SRC>, <&cru MCLK_SPDIF_8CH_SRC>, ++ <&cru ACLK_VOP>; ++ assigned-clock-rates = ++ <32768>, <300000000>, ++ <300000000>, <200000000>, ++ <100000000>, <1000000000>, ++ <500000000>, <333000000>, ++ <250000000>, <125000000>, ++ <100000000>, <62500000>, ++ <50000000>, <25000000>, ++ <1188000000>, ++ <150000000>, <100000000>, ++ <500000000>, <400000000>, ++ <150000000>, <100000000>, ++ <300000000>, <150000000>, ++ <1200000000>, <400000000>, ++ <100000000>, <1188000000>, ++ <1188000000>, <1188000000>, ++ <1188000000>, <1188000000>, ++ <1188000000>, <1188000000>, ++ <1188000000>, <1188000000>, ++ <500000000>; ++ assigned-clock-parents = ++ <&pmucru CLK_RTC32K_FRAC>, <&cru PLL_GPLL>, ++ <&cru PLL_GPLL>; ++ }; ++ ++ i2c0: i2c@fdd40000 { ++ compatible = "rockchip,rk3399-i2c"; ++ reg = <0x0 0xfdd40000 0x0 0x1000>; ++ clocks = <&pmucru CLK_I2C0>, <&pmucru PCLK_I2C0>; ++ clock-names = "i2c", "pclk"; ++ interrupts = ; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2c0_xfer>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ status = "disabled"; ++ }; ++ ++ uart0: serial@fdd50000 { ++ compatible = "rockchip,rk3568-uart", "snps,dw-apb-uart"; ++ reg = <0x0 0xfdd50000 0x0 0x100>; ++ interrupts = ; ++ clocks = <&pmucru SCLK_UART0>, <&pmucru PCLK_UART0>; ++ clock-names = "baudclk", "apb_pclk"; ++ reg-shift = <2>; ++ reg-io-width = <4>; ++ dmas = <&dmac0 0>, <&dmac0 1>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart0_xfer>; ++ status = "disabled"; ++ }; ++ ++ pwm0: pwm@fdd70000 { ++ compatible = "rockchip,rk3568-pwm", "rockchip,rk3328-pwm"; ++ reg = <0x0 0xfdd70000 0x0 0x10>; ++ #pwm-cells = <3>; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm0m0_pins>; ++ clocks = <&pmucru CLK_PWM0>, <&pmucru PCLK_PWM0>; ++ clock-names = "pwm", "pclk"; ++ status = "disabled"; ++ }; ++ ++ pwm1: pwm@fdd70010 { ++ compatible = "rockchip,rk3568-pwm", "rockchip,rk3328-pwm"; ++ reg = <0x0 0xfdd70010 0x0 0x10>; ++ #pwm-cells = <3>; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm1m0_pins>; ++ clocks = <&pmucru CLK_PWM0>, <&pmucru PCLK_PWM0>; ++ clock-names = "pwm", "pclk"; ++ status = "disabled"; ++ }; ++ ++ pwm2: pwm@fdd70020 { ++ compatible = "rockchip,rk3568-pwm", "rockchip,rk3328-pwm"; ++ reg = <0x0 0xfdd70020 0x0 0x10>; ++ #pwm-cells = <3>; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm2m0_pins>; ++ clocks = <&pmucru CLK_PWM0>, <&pmucru PCLK_PWM0>; ++ clock-names = "pwm", "pclk"; ++ status = "disabled"; ++ }; ++ ++ pwm3: pwm@fdd70030 { ++ compatible = "rockchip,rk3568-pwm", "rockchip,rk3328-pwm"; ++ reg = <0x0 0xfdd70030 0x0 0x10>; ++ interrupts = , ++ ; ++ #pwm-cells = <3>; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm3_pins>; ++ clocks = <&pmucru CLK_PWM0>, <&pmucru PCLK_PWM0>; ++ clock-names = "pwm", "pclk"; ++ status = "disabled"; ++ }; ++ ++ pmu: power-management@fdd90000 { ++ compatible = "rockchip,rk3568-pmu", "syscon", "simple-mfd"; ++ reg = <0x0 0xfdd90000 0x0 0x1000>; ++ ++ power: power-controller { ++ compatible = "rockchip,rk3568-power-controller"; ++ #power-domain-cells = <1>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ status = "okay"; ++ ++ /* These power domains are grouped by VD_NPU */ ++ /*pd_npu@RK3568_PD_NPU { ++ reg = ; ++ clocks = <&cru ACLK_NPU_PRE>, ++ <&cru HCLK_NPU_PRE>, ++ <&cru PCLK_NPU_PRE>; ++ pm_qos = <&qos_npu>; ++ };*/ ++ /* These power domains are grouped by VD_GPU */ ++ pd_gpu@RK3568_PD_GPU { ++ reg = ; ++ clocks = <&cru ACLK_GPU_PRE>, ++ <&cru PCLK_GPU_PRE>; ++ pm_qos = <&qos_gpu>; ++ }; ++ /* These power domains are grouped by VD_LOGIC */ ++ pd_vi@RK3568_PD_VI { ++ reg = ; ++ clocks = <&cru HCLK_VI>, ++ <&cru PCLK_VI>; ++ pm_qos = <&qos_isp>, ++ <&qos_vicap0>, ++ <&qos_vicap1>; ++ }; ++ pd_vo@RK3568_PD_VO { ++ reg = ; ++ clocks = <&cru HCLK_VO>, ++ <&cru PCLK_VO>, ++ <&cru ACLK_VOP_PRE>; ++ pm_qos = <&qos_hdcp>, ++ <&qos_vop_m0>, ++ <&qos_vop_m1>; ++ }; ++ pd_rga@RK3568_PD_RGA { ++ reg = ; ++ clocks = <&cru HCLK_RGA_PRE>, ++ <&cru PCLK_RGA_PRE>; ++ pm_qos = <&qos_ebc>, ++ <&qos_iep>, ++ <&qos_jpeg_dec>, ++ <&qos_jpeg_enc>, ++ <&qos_rga_rd>, ++ <&qos_rga_wr>; ++ }; ++ pd_vpu@RK3568_PD_VPU { ++ reg = ; ++ clocks = <&cru HCLK_VPU_PRE>; ++ pm_qos = <&qos_vpu>; ++ }; ++ pd_rkvdec@RK3568_PD_RKVDEC { ++ clocks = <&cru HCLK_RKVDEC_PRE>; ++ reg = ; ++ pm_qos = <&qos_rkvdec>; ++ }; ++ pd_rkvenc@RK3568_PD_RKVENC { ++ reg = ; ++ clocks = <&cru HCLK_RKVENC_PRE>; ++ pm_qos = <&qos_rkvenc_rd_m0>, ++ <&qos_rkvenc_rd_m1>, ++ <&qos_rkvenc_wr_m0>; ++ }; ++ pd_pipe@RK3568_PD_PIPE { ++ reg = ; ++ clocks = <&cru PCLK_PIPE>; ++ pm_qos = <&qos_pcie2x1>, ++ <&qos_pcie3x1>, ++ <&qos_pcie3x2>, ++ <&qos_sata0>, ++ <&qos_sata1>, ++ <&qos_sata2>, ++ <&qos_usb3_0>, ++ <&qos_usb3_1>; ++ }; ++ }; ++ }; ++ ++ pvtm@fde00000 { ++ compatible = "rockchip,rk3568-core-pvtm"; ++ reg = <0x0 0xfde00000 0x0 0x100>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ pvtm@0 { ++ reg = <0>; ++ clocks = <&cru CLK_CORE_PVTM>, <&cru PCLK_CORE_PVTM>; ++ clock-names = "clk", "pclk"; ++ resets = <&cru SRST_CORE_PVTM>, <&cru SRST_P_CORE_PVTM>; ++ reset-names = "rts", "rst-p"; ++ thermal-zone = "soc-thermal"; ++ }; ++ }; ++ ++ rknpu: npu@fde40000 { ++ compatible = "rockchip,rk3568-rknpu", "rockchip,rknpu"; ++ reg = <0x0 0xfde40000 0x0 0x10000>; ++ interrupts = ; ++ clocks = <&scmi_clk 2>, <&cru CLK_NPU>, <&cru ACLK_NPU>, <&cru HCLK_NPU>; ++ clock-names = "scmi_clk", "clk", "aclk", "hclk"; ++ assigned-clocks = <&cru CLK_NPU>; ++ assigned-clock-rates = <600000000>; ++ resets = <&cru SRST_A_NPU>, <&cru SRST_H_NPU>; ++ reset-names = "srst_a", "srst_h"; ++ power-domains = <&power RK3568_PD_NPU>; ++ operating-points-v2 = <&npu_opp_table>; ++ iommus = <&rknpu_mmu>; ++ status = "disabled"; ++ }; ++ ++ npu_opp_table: npu-opp-table { ++ compatible = "operating-points-v2"; ++ ++ mbist-vmin = <825000 900000 950000>; ++ nvmem-cells = <&npu_leakage>, <&core_pvtm>, <&mbist_vmin>; ++ nvmem-cell-names = "leakage", "pvtm", "mbist-vmin"; ++ rockchip,temp-hysteresis = <5000>; ++ rockchip,low-temp = <0>; ++ rockchip,low-temp-adjust-volt = < ++ /* MHz MHz uV */ ++ 0 700 50000 ++ >; ++ ++ opp-200000000 { ++ opp-hz = /bits/ 64 <200000000>; ++ opp-microvolt = <825000 825000 1000000>; ++ }; ++ opp-300000000 { ++ opp-hz = /bits/ 64 <297000000>; ++ opp-microvolt = <825000 825000 1000000>; ++ }; ++ opp-400000000 { ++ opp-hz = /bits/ 64 <400000000>; ++ opp-microvolt = <825000 825000 1000000>; ++ }; ++ opp-600000000 { ++ opp-hz = /bits/ 64 <600000000>; ++ opp-microvolt = <825000 825000 1000000>; ++ }; ++ opp-700000000 { ++ opp-hz = /bits/ 64 <700000000>; ++ opp-microvolt = <850000 850000 1000000>; ++ }; ++ opp-800000000 { ++ opp-hz = /bits/ 64 <800000000>; ++ opp-microvolt = <875000 875000 1000000>; ++ }; ++ opp-900000000 { ++ opp-hz = /bits/ 64 <900000000>; ++ opp-microvolt = <925000 925000 1000000>; ++ }; ++ opp-1000000000 { ++ opp-hz = /bits/ 64 <1000000000>; ++ opp-microvolt = <1000000 1000000 1000000>; ++ status = "disabled"; ++ }; ++ }; ++ ++ bus_npu: bus-npu { ++ compatible = "rockchip,rk3568-bus"; ++ rockchip,busfreq-policy = "clkfreq"; ++ clocks = <&scmi_clk 2>; ++ clock-names = "bus"; ++ operating-points-v2 = <&bus_npu_opp_table>; ++ status = "disabled"; ++ }; ++ ++ bus_npu_opp_table: bus-npu-opp-table { ++ compatible = "operating-points-v2"; ++ opp-shared; ++ ++ nvmem-cells = <&core_pvtm>; ++ nvmem-cell-names = "pvtm"; ++ rockchip,pvtm-voltage-sel = < ++ 0 82000 0 ++ 82001 93000 1 ++ 93001 100000 2 ++ >; ++ rockchip,pvtm-ch = <0 5>; ++ ++ opp-1000000000 { ++ opp-hz = /bits/ 64 <1000000000>; ++ opp-microvolt = <950000>; ++ opp-microvolt-L0 = <950000>; ++ opp-microvolt-L1 = <925000>; ++ opp-microvolt-L2 = <0>; ++ }; ++ opp-900000000 { ++ opp-hz = /bits/ 64 <900000000>; ++ opp-microvolt = <0>; ++ }; ++ }; ++ ++ rknpu_mmu: iommu@fde4b000 { ++ compatible = "rockchip,iommu-v2"; ++ reg = <0x0 0xfde4b000 0x0 0x40>; ++ interrupts = ; ++ interrupt-names = "rknpu_mmu"; ++ clocks = <&cru ACLK_NPU>, <&cru HCLK_NPU>; ++ clock-names = "aclk", "iface"; ++ power-domains = <&power RK3568_PD_NPU>; ++ #iommu-cells = <0>; ++ status = "disabled"; ++ }; ++ ++ gpu: gpu@fde60000 { ++ compatible = "arm,mali-bifrost"; ++ reg = <0x0 0xfde60000 0x0 0x4000>; ++ ++ interrupts = , ++ , ++ ; ++ interrupt-names = "GPU", "MMU", "JOB"; ++ ++ upthreshold = <40>; ++ downdifferential = <10>; ++ ++ clocks = <&scmi_clk 1>, <&cru CLK_GPU>; ++ clock-names = "clk_mali", "clk_gpu"; ++ power-domains = <&power RK3568_PD_GPU>; ++ #cooling-cells = <2>; ++ operating-points-v2 = <&gpu_opp_table>; ++ ++ status = "disabled"; ++ gpu_power_model: power-model { ++ compatible = "simple-power-model"; ++ leakage-range= <5 15>; ++ ls = <(-24002) 22823 0>; ++ static-coefficient = <100000>; ++ dynamic-coefficient = <953>; ++ ts = <(-108890) 63610 (-1355) 20>; ++ thermal-zone = "gpu-thermal"; ++ }; ++ }; ++ ++ gpu_opp_table: opp-table2 { ++ compatible = "operating-points-v2"; ++ ++ mbist-vmin = <825000 900000 950000>; ++ nvmem-cells = <&gpu_leakage>, <&core_pvtm>, <&mbist_vmin>; ++ nvmem-cell-names = "leakage", "pvtm", "mbist-vmin"; ++ ++ opp-200000000 { ++ opp-hz = /bits/ 64 <200000000>; ++ opp-microvolt = <825000>; ++ }; ++ opp-300000000 { ++ opp-hz = /bits/ 64 <300000000>; ++ opp-microvolt = <825000>; ++ }; ++ opp-400000000 { ++ opp-hz = /bits/ 64 <400000000>; ++ opp-microvolt = <825000>; ++ }; ++ opp-600000000 { ++ opp-hz = /bits/ 64 <600000000>; ++ opp-microvolt = <825000>; ++ }; ++ opp-700000000 { ++ opp-hz = /bits/ 64 <700000000>; ++ opp-microvolt = <900000>; ++ }; ++ opp-800000000 { ++ opp-hz = /bits/ 64 <800000000>; ++ opp-microvolt = <950000>; ++ }; ++ }; ++ ++ pvtm@fde80000 { ++ compatible = "rockchip,rk3568-gpu-pvtm"; ++ reg = <0x0 0xfde80000 0x0 0x100>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ pvtm@1 { ++ reg = <1>; ++ clocks = <&cru CLK_GPU_PVTM>, <&cru PCLK_GPU_PVTM>; ++ clock-names = "clk", "pclk"; ++ resets = <&cru SRST_GPU_PVTM>, <&cru SRST_P_GPU_PVTM>; ++ reset-names = "rts", "rst-p"; ++ thermal-zone = "gpu-thermal"; ++ }; ++ }; ++ ++ pvtm@fde90000 { ++ compatible = "rockchip,rk3568-npu-pvtm"; ++ reg = <0x0 0xfde90000 0x0 0x100>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ pvtm@2 { ++ reg = <2>; ++ clocks = <&cru CLK_NPU_PVTM>, <&cru PCLK_NPU_PVTM>, ++ <&cru HCLK_NPU_PRE>; ++ clock-names = "clk", "pclk", "hclk"; ++ resets = <&cru SRST_NPU_PVTM>, <&cru SRST_P_NPU_PVTM>; ++ reset-names = "rts", "rst-p"; ++ thermal-zone = "soc-thermal"; ++ }; ++ }; ++ ++ vdpu: vdpu@fdea0400 { ++ compatible = "rockchip,vpu-decoder-v2"; ++ reg = <0x0 0xfdea0400 0x0 0x400>; ++ interrupts = ; ++ interrupt-names = "irq_dec"; ++ clocks = <&cru ACLK_VPU>, <&cru HCLK_VPU>; ++ clock-names = "aclk_vcodec", "hclk_vcodec"; ++ resets = <&cru SRST_A_VPU>, <&cru SRST_H_VPU>; ++ reset-names = "video_a", "video_h"; ++ iommus = <&vdpu_mmu>; ++ power-domains = <&power RK3568_PD_VPU>; ++ rockchip,srv = <&mpp_srv>; ++ rockchip,taskqueue-node = <0>; ++ rockchip,resetgroup-node = <0>; ++ status = "disabled"; ++ }; ++ ++ vdpu_mmu: iommu@fdea0800 { ++ compatible = "rockchip,iommu-v2"; ++ reg = <0x0 0xfdea0800 0x0 0x40>; ++ interrupts = ; ++ interrupt-names = "vdpu_mmu"; ++ clock-names = "aclk", "iface"; ++ clocks = <&cru ACLK_VPU>, <&cru HCLK_VPU>; ++ power-domains = <&power RK3568_PD_VPU>; ++ #iommu-cells = <0>; ++ status = "disabled"; ++ }; ++ ++ rk_rga: rk_rga@fdeb0000 { ++ compatible = "rockchip,rga2"; ++ reg = <0x0 0xfdeb0000 0x0 0x1000>; ++ interrupts = ; ++ clocks = <&cru ACLK_RGA>, <&cru HCLK_RGA>, <&cru CLK_RGA_CORE>; ++ clock-names = "aclk_rga", "hclk_rga", "clk_rga"; ++ power-domains = <&power RK3568_PD_RGA>; ++ status = "disabled"; ++ }; ++ ++ ebc: ebc@fdec0000 { ++ compatible = "rockchip,rk3568-ebc-tcon"; ++ reg = <0x0 0xfdec0000 0x0 0x5000>; ++ interrupts = ; ++ clocks = <&cru HCLK_EBC>, <&cru DCLK_EBC>; ++ clock-names = "hclk", "dclk"; ++ power-domains = <&power RK3568_PD_RGA>; ++ rockchip,grf = <&grf>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&ebc_pins>; ++ status = "disabled"; ++ }; ++ ++ jpegd: jpegd@fded0000 { ++ compatible = "rockchip,rkv-jpeg-decoder-v1"; ++ reg = <0x0 0xfded0000 0x0 0x400>; ++ interrupts = ; ++ clocks = <&cru ACLK_JDEC>, <&cru HCLK_JDEC>; ++ clock-names = "aclk_vcodec", "hclk_vcodec"; ++ rockchip,disable-auto-freq; ++ resets = <&cru SRST_A_JDEC>, <&cru SRST_H_JDEC>; ++ reset-names = "video_a", "video_h"; ++ iommus = <&jpegd_mmu>; ++ rockchip,srv = <&mpp_srv>; ++ rockchip,taskqueue-node = <1>; ++ rockchip,resetgroup-node = <1>; ++ power-domains = <&power RK3568_PD_RGA>; ++ status = "disabled"; ++ }; ++ ++ jpegd_mmu: iommu@fded0480 { ++ compatible = "rockchip,iommu-v2"; ++ reg = <0x0 0xfded0480 0x0 0x40>; ++ interrupts = ; ++ interrupt-names = "jpegd_mmu"; ++ clock-names = "aclk", "iface"; ++ clocks = <&cru ACLK_JDEC>, <&cru HCLK_JDEC>; ++ power-domains = <&power RK3568_PD_RGA>; ++ #iommu-cells = <0>; ++ status = "disabled"; ++ }; ++ ++ vepu: vepu@fdee0000 { ++ compatible = "rockchip,vpu-encoder-v2"; ++ reg = <0x0 0xfdee0000 0x0 0x400>; ++ interrupts = ; ++ clocks = <&cru ACLK_JENC>, <&cru HCLK_JENC>; ++ clock-names = "aclk_vcodec", "hclk_vcodec"; ++ rockchip,disable-auto-freq; ++ resets = <&cru SRST_A_JENC>, <&cru SRST_H_JENC>; ++ reset-names = "video_a", "video_h"; ++ iommus = <&vepu_mmu>; ++ rockchip,srv = <&mpp_srv>; ++ rockchip,taskqueue-node = <2>; ++ rockchip,resetgroup-node = <2>; ++ power-domains = <&power RK3568_PD_RGA>; ++ status = "disabled"; ++ }; ++ ++ vepu_mmu: iommu@fdee0800 { ++ compatible = "rockchip,iommu-v2"; ++ reg = <0x0 0xfdee0800 0x0 0x40>; ++ interrupts = ; ++ interrupt-names = "vepu_mmu"; ++ clock-names = "aclk", "iface"; ++ clocks = <&cru ACLK_JENC>, <&cru HCLK_JENC>; ++ power-domains = <&power RK3568_PD_RGA>; ++ #iommu-cells = <0>; ++ status = "disabled"; ++ }; ++ ++ iep: iep@fdef0000 { ++ compatible = "rockchip,iep-v2"; ++ reg = <0x0 0xfdef0000 0x0 0x500>; ++ interrupts = ; ++ clocks = <&cru ACLK_IEP>, <&cru HCLK_IEP>, <&cru CLK_IEP_CORE>; ++ clock-names = "aclk", "hclk", "sclk"; ++ resets = <&cru SRST_A_IEP>, <&cru SRST_H_IEP>, ++ <&cru SRST_IEP_CORE>; ++ reset-names = "rst_a", "rst_h", "rst_s"; ++ power-domains = <&power RK3568_PD_RGA>; ++ rockchip,srv = <&mpp_srv>; ++ rockchip,taskqueue-node = <5>; ++ rockchip,resetgroup-node = <5>; ++ iommus = <&iep_mmu>; ++ status = "disabled"; ++ }; ++ ++ iep_mmu: iommu@fdef0800 { ++ compatible = "rockchip,iommu-v2"; ++ reg = <0x0 0xfdef0800 0x0 0x100>; ++ interrupts = ; ++ interrupt-names = "iep_mmu"; ++ clocks = <&cru ACLK_IEP>, <&cru HCLK_IEP>; ++ clock-names = "aclk", "iface"; ++ #iommu-cells = <0>; ++ power-domains = <&power RK3568_PD_RGA>; ++ //rockchip,disable-device-link-resume; ++ status = "disabled"; ++ }; ++ ++ eink: eink@fdf00000 { ++ compatible = "rockchip,rk3568-eink-tcon"; ++ reg = <0x0 0xfdf00000 0x0 0x74>; ++ interrupts = ; ++ clocks = <&cru PCLK_EINK>, <&cru HCLK_EINK>; ++ clock-names = "pclk", "hclk"; ++ status = "disabled"; ++ }; ++ ++ rkvenc: rkvenc@fdf40000 { ++ compatible = "rockchip,rkv-encoder-v1"; ++ reg = <0x0 0xfdf40000 0x0 0x400>; ++ interrupts = ; ++ interrupt-names = "irq_enc"; ++ clocks = <&cru ACLK_RKVENC>, <&cru HCLK_RKVENC>, ++ <&cru CLK_RKVENC_CORE>; ++ clock-names = "aclk_vcodec", "hclk_vcodec", "clk_core"; ++ rockchip,normal-rates = <297000000>, <0>, <297000000>; ++ resets = <&cru SRST_A_RKVENC>, <&cru SRST_H_RKVENC>, ++ <&cru SRST_RKVENC_CORE>; ++ reset-names = "video_a", "video_h", "video_core"; ++ assigned-clocks = <&cru ACLK_RKVENC>, <&cru CLK_RKVENC_CORE>; ++ assigned-clock-rates = <297000000>, <297000000>; ++ iommus = <&rkvenc_mmu>; ++ node-name = "rkvenc"; ++ rockchip,srv = <&mpp_srv>; ++ rockchip,taskqueue-node = <3>; ++ rockchip,resetgroup-node = <3>; ++ power-domains = <&power RK3568_PD_RKVENC>; ++ operating-points-v2 = <&rkvenc_opp_table>; ++ status = "disabled"; ++ }; ++ ++ rkvenc_opp_table: rkvenc-opp-table { ++ compatible = "operating-points-v2"; ++ ++ nvmem-cells = <&core_pvtm>; ++ nvmem-cell-names = "pvtm"; ++ rockchip,pvtm-voltage-sel = < ++ 0 82000 0 ++ 82001 93000 1 ++ 93001 100000 2 ++ >; ++ rockchip,pvtm-ch = <0 5>; ++ ++ opp-297000000 { ++ opp-hz = /bits/ 64 <297000000>; ++ opp-microvolt = <0>; ++ }; ++ opp-400000000 { ++ opp-hz = /bits/ 64 <400000000>; ++ opp-microvolt = <950000>; ++ opp-microvolt-L0 = <950000>; ++ opp-microvolt-L1 = <925000>; ++ opp-microvolt-L2 = <0>; ++ }; ++ }; ++ ++ rkvenc_mmu: iommu@fdf40f00 { ++ compatible = "rockchip,iommu-v2"; ++ reg = <0x0 0xfdf40f00 0x0 0x40>, <0x0 0xfdf40f40 0x0 0x40>; ++ interrupts = , ++ ; ++ interrupt-names = "rkvenc_mmu0", "rkvenc_mmu1"; ++ clocks = <&cru ACLK_RKVENC>, <&cru HCLK_RKVENC>; ++ clock-names = "aclk", "iface"; ++ rockchip,disable-mmu-reset; ++ rockchip,enable-cmd-retry; ++ #iommu-cells = <0>; ++ power-domains = <&power RK3568_PD_RKVENC>; ++ status = "disabled"; ++ }; ++ ++ rkvdec: rkvdec@fdf80200 { ++ compatible = "rockchip,rkv-decoder-rk3568", "rockchip,rkv-decoder-v2"; ++ reg = <0x0 0xfdf80200 0x0 0x400>; ++ interrupts = ; ++ interrupt-names = "irq_dec"; ++ clocks = <&cru ACLK_RKVDEC>, <&cru HCLK_RKVDEC>, ++ <&cru CLK_RKVDEC_CA>, <&cru CLK_RKVDEC_CORE>, ++ <&cru CLK_RKVDEC_HEVC_CA>; ++ clock-names = "aclk_vcodec", "hclk_vcodec","clk_cabac", ++ "clk_core", "clk_hevc_cabac"; ++ rockchip,normal-rates = <297000000>, <0>, <297000000>, ++ <297000000>, <600000000>; ++ rockchip,advanced-rates = <396000000>, <0>, <396000000>, ++ <396000000>, <600000000>; ++ rockchip,default-max-load = <2088960>; ++ resets = <&cru SRST_A_RKVDEC>, <&cru SRST_H_RKVDEC>, ++ <&cru SRST_RKVDEC_CA>, <&cru SRST_RKVDEC_CORE>, ++ <&cru SRST_RKVDEC_HEVC_CA>; ++ assigned-clocks = <&cru ACLK_RKVDEC>, <&cru CLK_RKVDEC_CA>, ++ <&cru CLK_RKVDEC_CORE>, <&cru CLK_RKVDEC_HEVC_CA>; ++ assigned-clock-rates = <297000000>, <297000000>, <297000000>, <297000000>; ++ reset-names = "video_a", "video_h", "video_cabac", ++ "video_core", "video_hevc_cabac"; ++ power-domains = <&power RK3568_PD_RKVDEC>; ++ iommus = <&rkvdec_mmu>; ++ rockchip,srv = <&mpp_srv>; ++ rockchip,taskqueue-node = <4>; ++ rockchip,resetgroup-node = <4>; ++ rockchip,sram = <&rkvdec_sram>; ++ /* rcb_iova: start and size */ ++ rockchip,rcb-iova = <0x10000000 65536>; ++ rockchip,rcb-min-width = <512>; ++ status = "disabled"; ++ }; ++ ++ rkvdec_mmu: iommu@fdf80800 { ++ compatible = "rockchip,iommu-v2"; ++ reg = <0x0 0xfdf80800 0x0 0x40>, <0x0 0xfdf80840 0x0 0x40>; ++ interrupts = ; ++ interrupt-names = "rkvdec_mmu"; ++ clocks = <&cru ACLK_RKVDEC>, <&cru HCLK_RKVDEC>; ++ clock-names = "aclk", "iface"; ++ power-domains = <&power RK3568_PD_RKVDEC>; ++ #iommu-cells = <0>; ++ status = "disabled"; ++ }; ++ ++ mipi_csi2: mipi-csi2@fdfb0000 { ++ compatible = "rockchip,rk3568-mipi-csi2"; ++ reg = <0x0 0xfdfb0000 0x0 0x10000>; ++ reg-names = "csihost_regs"; ++ interrupts = , ++ ; ++ interrupt-names = "csi-intr1", "csi-intr2"; ++ clocks = <&cru PCLK_CSI2HOST1>; ++ clock-names = "pclk_csi2host"; ++ resets = <&cru SRST_P_CSI2HOST1>; ++ reset-names = "srst_csihost_p"; ++ status = "disabled"; ++ }; ++ ++ rkcif: rkcif@fdfe0000 { ++ compatible = "rockchip,rk3568-cif"; ++ reg = <0x0 0xfdfe0000 0x0 0x8000>; ++ reg-names = "cif_regs"; ++ interrupts = ; ++ interrupt-names = "cif-intr"; ++ ++ clocks = <&cru ACLK_VICAP>, <&cru HCLK_VICAP>, ++ <&cru DCLK_VICAP>, <&cru ICLK_VICAP_G>; ++ clock-names = "aclk_cif", "hclk_cif", ++ "dclk_cif", "iclk_cif_g"; ++ resets = <&cru SRST_A_VICAP>, <&cru SRST_H_VICAP>, ++ <&cru SRST_D_VICAP>, <&cru SRST_P_VICAP>, ++ <&cru SRST_I_VICAP>; ++ reset-names = "rst_cif_a", "rst_cif_h", ++ "rst_cif_d", "rst_cif_p", ++ "rst_cif_i"; ++ assigned-clocks = <&cru DCLK_VICAP>; ++ assigned-clock-rates = <300000000>; ++ power-domains = <&power RK3568_PD_VI>; ++ rockchip,grf = <&grf>; ++ iommus = <&rkcif_mmu>; ++ status = "disabled"; ++ }; ++ ++ rkcif_mmu: iommu@fdfe0800 { ++ compatible = "rockchip,iommu-v2"; ++ reg = <0x0 0xfdfe0800 0x0 0x100>; ++ interrupts = ; ++ interrupt-names = "cif_mmu"; ++ clocks = <&cru ACLK_VICAP>, <&cru HCLK_VICAP>; ++ clock-names = "aclk", "iface"; ++ power-domains = <&power RK3568_PD_VI>; ++ rockchip,disable-mmu-reset; ++ #iommu-cells = <0>; ++ status = "disabled"; ++ }; ++ ++ rkcif_dvp: rkcif_dvp { ++ compatible = "rockchip,rkcif-dvp"; ++ rockchip,hw = <&rkcif>; ++ status = "disabled"; ++ }; ++ ++ rkcif_dvp_sditf: rkcif_dvp_sditf { ++ compatible = "rockchip,rkcif-sditf"; ++ rockchip,cif = <&rkcif_dvp>; ++ status = "disabled"; ++ }; ++ ++ rkcif_mipi_lvds: rkcif_mipi_lvds { ++ compatible = "rockchip,rkcif-mipi-lvds"; ++ rockchip,hw = <&rkcif>; ++ status = "disabled"; ++ }; ++ ++ rkcif_mipi_lvds_sditf: rkcif_mipi_lvds_sditf { ++ compatible = "rockchip,rkcif-sditf"; ++ rockchip,cif = <&rkcif_mipi_lvds>; ++ status = "disabled"; ++ }; ++ ++ rkisp: rkisp@fdff0000 { ++ compatible = "rockchip,rk3568-rkisp"; ++ reg = <0x0 0xfdff0000 0x0 0x10000>; ++ interrupts = , ++ , ++ ; ++ interrupt-names = "mipi_irq", "mi_irq", "isp_irq"; ++ clocks = <&cru ACLK_ISP>, <&cru HCLK_ISP>, <&cru CLK_ISP>; ++ clock-names = "aclk_isp", "hclk_isp", "clk_isp"; ++ resets = <&cru SRST_ISP>, <&cru SRST_H_ISP>; ++ reset-names = "isp", "isp-h"; ++ rockchip,grf = <&grf>; ++ power-domains = <&power RK3568_PD_VI>; ++ iommus = <&rkisp_mmu>; ++ rockchip,iq-feature = /bits/ 64 <0x3FBFFFE67FF>; ++ status = "disabled"; ++ }; ++ ++ rkisp_mmu: iommu@fdff1a00 { ++ compatible = "rockchip,iommu-v2"; ++ reg = <0x0 0xfdff1a00 0x0 0x100>; ++ interrupts = ; ++ interrupt-names = "isp_mmu"; ++ clocks = <&cru ACLK_ISP>, <&cru HCLK_ISP>; ++ clock-names = "aclk", "iface"; ++ power-domains = <&power RK3568_PD_VI>; ++ #iommu-cells = <0>; ++ rockchip,disable-mmu-reset; ++ status = "disabled"; ++ }; ++ ++ rkisp_vir0: rkisp-vir0 { ++ compatible = "rockchip,rkisp-vir"; ++ rockchip,hw = <&rkisp>; ++ status = "disabled"; ++ }; ++ ++ rkisp_vir1: rkisp-vir1 { ++ compatible = "rockchip,rkisp-vir"; ++ rockchip,hw = <&rkisp>; ++ status = "disabled"; ++ }; ++ ++ gmac1: ethernet@fe010000 { ++ compatible = "rockchip,rk3568-gmac", "snps,dwmac-4.20a"; ++ reg = <0x0 0xfe010000 0x0 0x10000>; ++ interrupts = , ++ ; ++ interrupt-names = "macirq", "eth_wake_irq"; ++ rockchip,grf = <&grf>; ++ clocks = <&cru SCLK_GMAC1>, <&cru SCLK_GMAC1_RX_TX>, ++ <&cru SCLK_GMAC1_RX_TX>, <&cru CLK_MAC1_REFOUT>, ++ <&cru ACLK_GMAC1>, <&cru PCLK_GMAC1>, ++ <&cru SCLK_GMAC1_RX_TX>, <&cru CLK_GMAC1_PTP_REF>, ++ <&cru PCLK_XPCS>; ++ clock-names = "stmmaceth", "mac_clk_rx", ++ "mac_clk_tx", "clk_mac_refout", ++ "aclk_mac", "pclk_mac", ++ "clk_mac_speed", "ptp_ref", ++ "pclk_xpcs"; ++ resets = <&cru SRST_A_GMAC1>; ++ reset-names = "stmmaceth"; ++ ++ snps,mixed-burst; ++ snps,tso; ++ ++ snps,axi-config = <&gmac1_stmmac_axi_setup>; ++ snps,mtl-rx-config = <&gmac1_mtl_rx_setup>; ++ snps,mtl-tx-config = <&gmac1_mtl_tx_setup>; ++ status = "disabled"; ++ ++ mdio1: mdio { ++ compatible = "snps,dwmac-mdio"; ++ #address-cells = <0x1>; ++ #size-cells = <0x0>; ++ }; ++ ++ gmac1_stmmac_axi_setup: stmmac-axi-config { ++ snps,wr_osr_lmt = <4>; ++ snps,rd_osr_lmt = <8>; ++ snps,blen = <0 0 0 0 16 8 4>; ++ }; ++ ++ gmac1_mtl_rx_setup: rx-queues-config { ++ snps,rx-queues-to-use = <1>; ++ queue0 {}; ++ }; ++ ++ gmac1_mtl_tx_setup: tx-queues-config { ++ snps,tx-queues-to-use = <1>; ++ queue0 {}; ++ }; ++ }; ++ ++ vop: vop@fe040000 { ++ compatible = "rockchip,rk3568-vop"; ++ reg = <0x0 0xfe040000 0x0 0x3000>, <0x0 0xfe044000 0x0 0x1000>; ++ reg-names = "regs", "gamma_lut"; ++ rockchip,grf = <&grf>; ++ interrupts = ; ++ clocks = <&cru ACLK_VOP>, <&cru HCLK_VOP>, <&cru DCLK_VOP0>, <&cru DCLK_VOP1>, <&cru DCLK_VOP2>; ++ clock-names = "aclk_vop", "hclk_vop", "dclk_vp0", "dclk_vp1", "dclk_vp2"; ++ iommus = <&vop_mmu>; ++ power-domains = <&power RK3568_PD_VO>; ++ status = "disabled"; ++ ++ vop_out: ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ vp0: port@0 { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ reg = <0>; ++ ++ vp0_out_dsi0: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dsi0_in_vp0>; ++ }; ++ ++ vp0_out_dsi1: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&dsi1_in_vp0>; ++ }; ++ ++ vp0_out_edp: endpoint@2 { ++ reg = <2>; ++ remote-endpoint = <&edp_in_vp0>; ++ }; ++ ++ vp0_out_hdmi: endpoint@3 { ++ reg = <3>; ++ remote-endpoint = <&hdmi_in_vp0>; ++ }; ++ }; ++ ++ vp1: port@1 { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ reg = <1>; ++ ++ vp1_out_dsi0: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&dsi0_in_vp1>; ++ }; ++ ++ vp1_out_dsi1: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&dsi1_in_vp1>; ++ }; ++ ++ vp1_out_edp: endpoint@2 { ++ reg = <2>; ++ remote-endpoint = <&edp_in_vp1>; ++ }; ++ ++ vp1_out_hdmi: endpoint@3 { ++ reg = <3>; ++ remote-endpoint = <&hdmi_in_vp1>; ++ }; ++ ++ vp1_out_lvds: endpoint@4 { ++ reg = <4>; ++ remote-endpoint = <&lvds_in_vp1>; ++ }; ++ }; ++ ++ vp2: port@2 { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ reg = <2>; ++ ++ vp2_out_lvds: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&lvds_in_vp2>; ++ }; ++ ++ vp2_out_rgb: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&rgb_in_vp2>; ++ }; ++ }; ++ }; ++ }; ++ ++ vop_mmu: iommu@fe043e00 { ++ compatible = "rockchip,iommu-v2"; ++ reg = <0x0 0xfe043e00 0x0 0x100>, <0x0 0xfe043f00 0x0 0x100>; ++ interrupts = ; ++ interrupt-names = "vop_mmu"; ++ clocks = <&cru ACLK_VOP>, <&cru HCLK_VOP>; ++ clock-names = "aclk", "iface"; ++ #iommu-cells = <0>; ++ rockchip,disable-device-link-resume; ++ status = "disabled"; ++ }; ++ ++ dsi0: dsi@fe060000 { ++ compatible = "rockchip,rk3568-mipi-dsi"; ++ reg = <0x0 0xfe060000 0x0 0x10000>; ++ interrupts = ; ++ clocks = <&cru PCLK_DSITX_0>, <&cru HCLK_VO>; ++ clock-names = "pclk", "hclk"; ++ resets = <&cru SRST_P_DSITX_0>; ++ reset-names = "apb"; ++ phys = <&video_phy0>; ++ phy-names = "dphy"; ++ power-domains = <&power RK3568_PD_VO>; ++ rockchip,grf = <&grf>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ status = "disabled"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dsi0_in: port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dsi0_in_vp0: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&vp0_out_dsi0>; ++ status = "disabled"; ++ }; ++ ++ dsi0_in_vp1: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&vp1_out_dsi0>; ++ status = "disabled"; ++ }; ++ }; ++ }; ++ }; ++ ++ dsi1: dsi@fe070000 { ++ compatible = "rockchip,rk3568-mipi-dsi"; ++ reg = <0x0 0xfe070000 0x0 0x10000>; ++ interrupts = ; ++ clocks = <&cru PCLK_DSITX_1>, <&cru HCLK_VO>; ++ clock-names = "pclk", "hclk"; ++ resets = <&cru SRST_P_DSITX_1>; ++ reset-names = "apb"; ++ phys = <&video_phy1>; ++ phy-names = "dphy"; ++ power-domains = <&power RK3568_PD_VO>; ++ rockchip,grf = <&grf>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ status = "disabled"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dsi1_in: port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ dsi1_in_vp0: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&vp0_out_dsi1>; ++ status = "disabled"; ++ }; ++ ++ dsi1_in_vp1: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&vp1_out_dsi1>; ++ status = "disabled"; ++ }; ++ }; ++ }; ++ }; ++ ++ hdmi: hdmi@fe0a0000 { ++ compatible = "rockchip,rk3568-dw-hdmi"; ++ reg = <0x0 0xfe0a0000 0x0 0x20000>; ++ interrupts = ; ++ clocks = <&cru PCLK_HDMI_HOST>, ++ <&cru CLK_HDMI_SFR>, ++ <&cru CLK_HDMI_CEC>, ++ <&pmucru PLL_HPLL>, ++ <&cru HCLK_VOP>; ++ clock-names = "iahb", "isfr", "cec", "ref", "hclk"; ++ power-domains = <&power RK3568_PD_VO>; ++ reg-io-width = <4>; ++ rockchip,grf = <&grf>; ++ #sound-dai-cells = <0>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&hdmitx_scl &hdmitx_sda &hdmitxm0_cec>; ++ status = "disabled"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ hdmi_in: port { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ hdmi_in_vp0: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&vp0_out_hdmi>; ++ status = "disabled"; ++ }; ++ hdmi_in_vp1: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&vp1_out_hdmi>; ++ status = "disabled"; ++ }; ++ }; ++ }; ++ }; ++ ++ edp: edp@fe0c0000 { ++ compatible = "rockchip,rk3568-edp"; ++ reg = <0x0 0xfe0c0000 0x0 0x10000>; ++ interrupts = ; ++ clocks = <&pmucru XIN_OSC0_EDPPHY_G>, <&cru PCLK_EDP_CTRL>, ++ <&cru CLK_EDP_200M>, <&cru HCLK_VO>; ++ clock-names = "dp", "pclk", "spdif", "hclk"; ++ resets = <&cru SRST_EDP_24M>, <&cru SRST_P_EDP_CTRL>; ++ reset-names = "dp", "apb"; ++ phys = <&edp_phy>; ++ phy-names = "dp"; ++ power-domains = <&power RK3568_PD_VO>; ++ status = "disabled"; ++ ++ ports { ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ edp_in: port@0 { ++ reg = <0>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ ++ edp_in_vp0: endpoint@0 { ++ reg = <0>; ++ remote-endpoint = <&vp0_out_edp>; ++ status = "disabled"; ++ }; ++ ++ edp_in_vp1: endpoint@1 { ++ reg = <1>; ++ remote-endpoint = <&vp1_out_edp>; ++ status = "disabled"; ++ }; ++ }; ++ }; ++ }; ++ ++ qos_gpu: qos@fe128000 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe128000 0x0 0x20>; ++ }; ++ ++ qos_rkvenc_rd_m0: qos@fe138080 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe138080 0x0 0x20>; ++ }; ++ ++ qos_rkvenc_rd_m1: qos@fe138100 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe138100 0x0 0x20>; ++ }; ++ ++ qos_rkvenc_wr_m0: qos@fe138180 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe138180 0x0 0x20>; ++ }; ++ ++ qos_isp: qos@fe148000 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe148000 0x0 0x20>; ++ }; ++ ++ qos_vicap0: qos@fe148080 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe148080 0x0 0x20>; ++ }; ++ ++ qos_vicap1: qos@fe148100 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe148100 0x0 0x20>; ++ }; ++ ++ qos_vpu: qos@fe150000 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe150000 0x0 0x20>; ++ }; ++ ++ qos_ebc: qos@fe158000 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe158000 0x0 0x20>; ++ }; ++ ++ qos_iep: qos@fe158100 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe158100 0x0 0x20>; ++ }; ++ ++ qos_jpeg_dec: qos@fe158180 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe158180 0x0 0x20>; ++ }; ++ ++ qos_jpeg_enc: qos@fe158200 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe158200 0x0 0x20>; ++ }; ++ ++ qos_rga_rd: qos@fe158280 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe158280 0x0 0x20>; ++ }; ++ ++ qos_rga_wr: qos@fe158300 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe158300 0x0 0x20>; ++ }; ++ ++ qos_npu: qos@fe180000 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe180000 0x0 0x20>; ++ }; ++ ++ qos_pcie2x1: qos@fe190000 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe190000 0x0 0x20>; ++ }; ++ ++ qos_pcie3x1: qos@fe190080 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe190080 0x0 0x20>; ++ }; ++ ++ qos_pcie3x2: qos@fe190100 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe190100 0x0 0x20>; ++ }; ++ ++ qos_sata0: qos@fe190200 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe190200 0x0 0x20>; ++ }; ++ ++ qos_sata1: qos@fe190280 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe190280 0x0 0x20>; ++ }; ++ ++ qos_sata2: qos@fe190300 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe190300 0x0 0x20>; ++ }; ++ ++ qos_usb3_0: qos@fe190380 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe190380 0x0 0x20>; ++ }; ++ ++ qos_usb3_1: qos@fe190400 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe190400 0x0 0x20>; ++ }; ++ ++ qos_rkvdec: qos@fe198000 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe198000 0x0 0x20>; ++ }; ++ ++ qos_hdcp: qos@fe1a8000 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe1a8000 0x0 0x20>; ++ }; ++ ++ qos_vop_m0: qos@fe1a8080 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe1a8080 0x0 0x20>; ++ }; ++ ++ qos_vop_m1: qos@fe1a8100 { ++ compatible = "syscon"; ++ reg = <0x0 0xfe1a8100 0x0 0x20>; ++ }; ++ ++ sdmmc2: dwmmc@fe000000 { ++ compatible = "rockchip,rk3568-dw-mshc", ++ "rockchip,rk3288-dw-mshc"; ++ reg = <0x0 0xfe000000 0x0 0x4000>; ++ interrupts = ; ++ max-frequency = <150000000>; ++ clocks = <&cru HCLK_SDMMC2>, <&cru CLK_SDMMC2>, ++ <&cru SCLK_SDMMC2_DRV>, <&cru SCLK_SDMMC2_SAMPLE>; ++ clock-names = "biu", "ciu", "ciu-drive", "ciu-sample"; ++ fifo-depth = <0x100>; ++ resets = <&cru SRST_SDMMC2>; ++ reset-names = "reset"; ++ status = "disabled"; ++ }; ++ ++ dfi: dfi@fe230000 { ++ reg = <0x00 0xfe230000 0x00 0x400>; ++ compatible = "rockchip,rk3568-dfi"; ++ rockchip,pmugrf = <&pmugrf>; ++ status = "disabled"; ++ }; ++ ++ dmc: dmc { ++ compatible = "rockchip,rk3568-dmc"; ++ interrupts = ; ++ interrupt-names = "complete"; ++ devfreq-events = <&dfi>; ++ clocks = <&scmi_clk 3>; ++ clock-names = "dmc_clk"; ++ operating-points-v2 = <&dmc_opp_table>; ++ ddr_timing = <&ddr_timing>; ++ vop-bw-dmc-freq = < ++ /* min_bw(MB/s) max_bw(MB/s) freq(KHz) */ ++ 0 505 324000 ++ 506 99999 528000 ++ >; ++ upthreshold = <40>; ++ downdifferential = <20>; ++ system-status-level = < ++ /*system status freq level*/ ++ SYS_STATUS_NORMAL DMC_FREQ_LEVEL_MID_HIGH ++ SYS_STATUS_REBOOT DMC_FREQ_LEVEL_HIGH ++ SYS_STATUS_SUSPEND DMC_FREQ_LEVEL_LOW ++ SYS_STATUS_VIDEO_4K DMC_FREQ_LEVEL_MID_HIGH ++ SYS_STATUS_VIDEO_4K_10B DMC_FREQ_LEVEL_MID_HIGH ++ SYS_STATUS_BOOST DMC_FREQ_LEVEL_HIGH ++ SYS_STATUS_ISP DMC_FREQ_LEVEL_HIGH ++ SYS_STATUS_PERFORMANCE DMC_FREQ_LEVEL_HIGH ++ SYS_STATUS_DUALVIEW DMC_FREQ_LEVEL_HIGH ++ >; ++ auto-min-freq = <324000>; ++ auto-freq-en = <1>; ++ #cooling-cells = <2>; ++ status = "disabled"; ++ }; ++ ++ dmc_opp_table: dmc-opp-table { ++ compatible = "operating-points-v2"; ++ ++ mbist-vmin = <825000 900000 950000>; ++ nvmem-cells = <&log_leakage>, <&core_pvtm>, <&mbist_vmin>; ++ nvmem-cell-names = "leakage", "pvtm", "mbist-vmin"; ++ rockchip,temp-hysteresis = <5000>; ++ rockchip,low-temp = <0>; ++ rockchip,low-temp-adjust-volt = < ++ /* MHz MHz uV */ ++ 0 1560 25000 ++ >; ++ rockchip,leakage-voltage-sel = < ++ 1 80 0 ++ 81 254 1 ++ >; ++ ++ opp-1560000000 { ++ opp-hz = /bits/ 64 <1560000000>; ++ opp-microvolt = <900000>; ++ opp-microvolt-L0 = <900000>; ++ opp-microvolt-L1 = <850000>; ++ }; ++ }; ++ ++ pcie2x1: pcie@fe260000 { ++ compatible = "rockchip,rk3568-pcie", "snps,dw-pcie"; ++ #address-cells = <3>; ++ #size-cells = <2>; ++ bus-range = <0x0 0xf>; ++ clocks = <&cru ACLK_PCIE20_MST>, <&cru ACLK_PCIE20_SLV>, ++ <&cru ACLK_PCIE20_DBI>, <&cru PCLK_PCIE20>, ++ <&cru CLK_PCIE20_AUX_NDFT>; ++ clock-names = "aclk_mst", "aclk_slv", ++ "aclk_dbi", "pclk", "aux"; ++ device_type = "pci"; ++ interrupts = , ++ , ++ , ++ , ++ ; ++ interrupt-names = "sys", "pmc", "msg", "legacy", "err"; ++ #interrupt-cells = <1>; ++ interrupt-map-mask = <0 0 0 7>; ++ interrupt-map = <0 0 0 1 &pcie2x1_intc 0>, ++ <0 0 0 2 &pcie2x1_intc 1>, ++ <0 0 0 3 &pcie2x1_intc 2>, ++ <0 0 0 4 &pcie2x1_intc 3>; ++ linux,pci-domain = <0>; ++ num-ib-windows = <6>; ++ num-ob-windows = <2>; ++ max-link-speed = <2>; ++ msi-map = <0x0 &its 0x0 0x1000>; ++ num-lanes = <1>; ++ phys = <&combphy2_psq PHY_TYPE_PCIE>; ++ phy-names = "pcie-phy"; ++ power-domains = <&power RK3568_PD_PIPE>; ++ ranges = <0x00000800 0x0 0x00000000 0x3 0x00000000 0x0 0x800000 ++ 0x81000000 0x0 0x00800000 0x3 0x00800000 0x0 0x100000 ++ 0x83000000 0x0 0x00900000 0x3 0x00900000 0x0 0x3f700000>; ++ reg = <0x3 0xc0000000 0x0 0x400000>, ++ <0x0 0xfe260000 0x0 0x10000>; ++ reg-names = "pcie-dbi", "pcie-apb"; ++ resets = <&cru SRST_PCIE20_POWERUP>; ++ reset-names = "pipe"; ++ status = "disabled"; ++ ++ pcie2x1_intc: legacy-interrupt-controller { ++ interrupt-controller; ++ #address-cells = <0>; ++ #interrupt-cells = <1>; ++ interrupt-parent = <&gic>; ++ interrupts = ; ++ }; ++ }; ++ ++ pcie3x1: pcie@fe270000 { ++ compatible = "rockchip,rk3568-pcie", "snps,dw-pcie"; ++ #address-cells = <3>; ++ #size-cells = <2>; ++ bus-range = <0x10 0x1f>; ++ clocks = <&cru ACLK_PCIE30X1_MST>, <&cru ACLK_PCIE30X1_SLV>, ++ <&cru ACLK_PCIE30X1_DBI>, <&cru PCLK_PCIE30X1>, ++ <&cru CLK_PCIE30X1_AUX_NDFT>; ++ clock-names = "aclk_mst", "aclk_slv", ++ "aclk_dbi", "pclk", "aux"; ++ device_type = "pci"; ++ interrupts = , ++ , ++ , ++ , ++ ; ++ interrupt-names = "sys", "pmc", "msg", "legacy", "err"; ++ #interrupt-cells = <1>; ++ interrupt-map-mask = <0 0 0 7>; ++ interrupt-map = <0 0 0 1 &pcie3x1_intc 0>, ++ <0 0 0 2 &pcie3x1_intc 1>, ++ <0 0 0 3 &pcie3x1_intc 2>, ++ <0 0 0 4 &pcie3x1_intc 3>; ++ linux,pci-domain = <1>; ++ num-ib-windows = <6>; ++ num-ob-windows = <2>; ++ max-link-speed = <3>; ++ msi-map = <0x1000 &its 0x1000 0x1000>; ++ num-lanes = <1>; ++ phys = <&pcie30phy>; ++ phy-names = "pcie-phy"; ++ power-domains = <&power RK3568_PD_PIPE>; ++ ranges = <0x00000800 0x0 0x40000000 0x3 0x40000000 0x0 0x800000 ++ 0x81000000 0x0 0x40800000 0x3 0x40800000 0x0 0x100000 ++ 0x83000000 0x0 0x40900000 0x3 0x40900000 0x0 0x3f700000>; ++ reg = <0x3 0xc0400000 0x0 0x400000>, ++ <0x0 0xfe270000 0x0 0x10000>; ++ reg-names = "pcie-dbi", "pcie-apb"; ++ resets = <&cru SRST_PCIE30X1_POWERUP>; ++ reset-names = "pipe"; ++ /* rockchip,bifurcation; lane1 when using 1+1 */ ++ status = "disabled"; ++ ++ pcie3x1_intc: legacy-interrupt-controller { ++ interrupt-controller; ++ #address-cells = <0>; ++ #interrupt-cells = <1>; ++ interrupt-parent = <&gic>; ++ interrupts = ; ++ }; ++ }; ++ ++ pcie3x2: pcie@fe280000 { ++ compatible = "rockchip,rk3568-pcie", "snps,dw-pcie"; ++ #address-cells = <3>; ++ #size-cells = <2>; ++ bus-range = <0x20 0x2f>; ++ clocks = <&cru ACLK_PCIE30X2_MST>, <&cru ACLK_PCIE30X2_SLV>, ++ <&cru ACLK_PCIE30X2_DBI>, <&cru PCLK_PCIE30X2>, ++ <&cru CLK_PCIE30X2_AUX_NDFT>; ++ clock-names = "aclk_mst", "aclk_slv", ++ "aclk_dbi", "pclk", "aux"; ++ device_type = "pci"; ++ interrupts = , ++ , ++ , ++ , ++ ; ++ interrupt-names = "sys", "pmc", "msg", "legacy", "err"; ++ #interrupt-cells = <1>; ++ interrupt-map-mask = <0 0 0 7>; ++ interrupt-map = <0 0 0 1 &pcie3x2_intc 0>, ++ <0 0 0 2 &pcie3x2_intc 1>, ++ <0 0 0 3 &pcie3x2_intc 2>, ++ <0 0 0 4 &pcie3x2_intc 3>; ++ linux,pci-domain = <2>; ++ num-ib-windows = <6>; ++ num-ob-windows = <2>; ++ max-link-speed = <3>; ++ msi-map = <0x2000 &its 0x2000 0x1000>; ++ num-lanes = <2>; ++ phys = <&pcie30phy>; ++ phy-names = "pcie-phy"; ++ power-domains = <&power RK3568_PD_PIPE>; ++ ranges = <0x00000800 0x0 0x80000000 0x3 0x80000000 0x0 0x800000 ++ 0x81000000 0x0 0x80800000 0x3 0x80800000 0x0 0x100000 ++ 0x83000000 0x0 0x80900000 0x3 0x80900000 0x0 0x3f700000>; ++ reg = <0x3 0xc0800000 0x0 0x400000>, ++ <0x0 0xfe280000 0x0 0x10000>; ++ reg-names = "pcie-dbi", "pcie-apb"; ++ resets = <&cru SRST_PCIE30X2_POWERUP>; ++ reset-names = "pipe"; ++ /* rockchip,bifurcation; lane0 when using 1+1 */ ++ status = "disabled"; ++ ++ pcie3x2_intc: legacy-interrupt-controller { ++ interrupt-controller; ++ #address-cells = <0>; ++ #interrupt-cells = <1>; ++ interrupt-parent = <&gic>; ++ interrupts = ; ++ }; ++ }; ++ ++ gmac0: ethernet@fe2a0000 { ++ compatible = "rockchip,rk3568-gmac", "snps,dwmac-4.20a"; ++ reg = <0x0 0xfe2a0000 0x0 0x10000>; ++ interrupts = , ++ ; ++ interrupt-names = "macirq", "eth_wake_irq"; ++ rockchip,grf = <&grf>; ++ clocks = <&cru SCLK_GMAC0>, <&cru SCLK_GMAC0_RX_TX>, ++ <&cru SCLK_GMAC0_RX_TX>, <&cru CLK_MAC0_REFOUT>, ++ <&cru ACLK_GMAC0>, <&cru PCLK_GMAC0>, ++ <&cru SCLK_GMAC0_RX_TX>, <&cru CLK_GMAC0_PTP_REF>, ++ <&cru PCLK_XPCS>; ++ clock-names = "stmmaceth", "mac_clk_rx", ++ "mac_clk_tx", "clk_mac_refout", ++ "aclk_mac", "pclk_mac", ++ "clk_mac_speed", "ptp_ref", ++ "pclk_xpcs"; ++ resets = <&cru SRST_A_GMAC0>; ++ reset-names = "stmmaceth"; ++ ++ snps,mixed-burst; ++ snps,tso; ++ ++ snps,axi-config = <&gmac0_stmmac_axi_setup>; ++ snps,mtl-rx-config = <&gmac0_mtl_rx_setup>; ++ snps,mtl-tx-config = <&gmac0_mtl_tx_setup>; ++ status = "disabled"; ++ ++ mdio0: mdio { ++ compatible = "snps,dwmac-mdio"; ++ #address-cells = <0x1>; ++ #size-cells = <0x0>; ++ }; ++ ++ gmac0_stmmac_axi_setup: stmmac-axi-config { ++ snps,wr_osr_lmt = <4>; ++ snps,rd_osr_lmt = <8>; ++ snps,blen = <0 0 0 0 16 8 4>; ++ }; ++ ++ gmac0_mtl_rx_setup: rx-queues-config { ++ snps,rx-queues-to-use = <1>; ++ queue0 {}; ++ }; ++ ++ gmac0_mtl_tx_setup: tx-queues-config { ++ snps,tx-queues-to-use = <1>; ++ queue0 {}; ++ }; ++ }; ++ ++ sdmmc0: dwmmc@fe2b0000 { ++ compatible = "rockchip,rk3568-dw-mshc", ++ "rockchip,rk3288-dw-mshc"; ++ reg = <0x0 0xfe2b0000 0x0 0x4000>; ++ interrupts = ; ++ max-frequency = <150000000>; ++ clocks = <&cru HCLK_SDMMC0>, <&cru CLK_SDMMC0>, ++ <&cru SCLK_SDMMC0_DRV>, <&cru SCLK_SDMMC0_SAMPLE>; ++ clock-names = "biu", "ciu", "ciu-drive", "ciu-sample"; ++ fifo-depth = <0x100>; ++ resets = <&cru SRST_SDMMC0>; ++ reset-names = "reset"; ++ status = "disabled"; ++ }; ++ ++ sdmmc1: dwmmc@fe2c0000 { ++ compatible = "rockchip,rk3568-dw-mshc", ++ "rockchip,rk3288-dw-mshc"; ++ reg = <0x0 0xfe2c0000 0x0 0x4000>; ++ interrupts = ; ++ max-frequency = <150000000>; ++ clocks = <&cru HCLK_SDMMC1>, <&cru CLK_SDMMC1>, ++ <&cru SCLK_SDMMC1_DRV>, <&cru SCLK_SDMMC1_SAMPLE>; ++ clock-names = "biu", "ciu", "ciu-drive", "ciu-sample"; ++ fifo-depth = <0x100>; ++ resets = <&cru SRST_SDMMC1>; ++ reset-names = "reset"; ++ status = "disabled"; ++ }; ++ ++ sfc: sfc@fe300000 { ++ compatible = "rockchip,sfc"; ++ reg = <0x0 0xfe300000 0x0 0x4000>; ++ interrupts = ; ++ clocks = <&cru SCLK_SFC>, <&cru HCLK_SFC>; ++ clock-names = "clk_sfc", "hclk_sfc"; ++ assigned-clocks = <&cru SCLK_SFC>; ++ assigned-clock-rates = <100000000>; ++ status = "disabled"; ++ }; ++ ++ sdhci: sdhci@fe310000 { ++ compatible = "rockchip,dwcmshc-sdhci", "snps,dwcmshc-sdhci"; ++ reg = <0x0 0xfe310000 0x0 0x10000>; ++ interrupts = ; ++ assigned-clocks = <&cru BCLK_EMMC>, <&cru TCLK_EMMC>; ++ assigned-clock-rates = <200000000>, <24000000>; ++ clocks = <&cru CCLK_EMMC>, <&cru HCLK_EMMC>, ++ <&cru ACLK_EMMC>, <&cru BCLK_EMMC>, ++ <&cru TCLK_EMMC>; ++ clock-names = "core", "bus", "axi", "block", "timer"; ++ status = "disabled"; ++ }; ++ ++ nandc0: nandc@fe330000 { ++ compatible = "rockchip,rk-nandc-v9"; ++ reg = <0x0 0xfe330000 0x0 0x4000>; ++ interrupts = ; ++ nandc_id = <0>; ++ clocks = <&cru NCLK_NANDC>, <&cru HCLK_NANDC>; ++ clock-names = "clk_nandc", "hclk_nandc"; ++ status = "disabled"; ++ }; ++ ++ crypto: crypto@fe380000 { ++ compatible = "rockchip,rk3568-crypto"; ++ reg = <0x0 0xfe380000 0x0 0x4000>; ++ interrupts = ; ++ clocks = <&cru ACLK_CRYPTO_NS>, <&cru HCLK_CRYPTO_NS>, ++ <&cru CLK_CRYPTO_NS_CORE>, <&cru CLK_CRYPTO_NS_PKA>; ++ clock-names = "aclk", "hclk", "sclk", "apb_pclk"; ++ assigned-clocks = <&cru CLK_CRYPTO_NS_CORE>; ++ assigned-clock-rates = <200000000>; ++ resets = <&cru SRST_CRYPTO_NS_CORE>; ++ reset-names = "crypto-rst"; ++ status = "disabled"; ++ }; ++ ++ rng: rng@fe388000 { ++ compatible = "rockchip,cryptov2-rng"; ++ reg = <0x0 0xfe388000 0x0 0x2000>; ++ clocks = <&cru CLK_TRNG_NS>, <&cru HCLK_TRNG_NS>; ++ clock-names = "clk_trng", "hclk_trng"; ++ resets = <&cru SRST_TRNG_NS>; ++ reset-names = "reset"; ++ status = "disabled"; ++ }; ++ ++ otp: otp@fe38c000 { ++ compatible = "rockchip,rk3568-otp"; ++ reg = <0x0 0xfe38c000 0x0 0x4000>; ++ #address-cells = <1>; ++ #size-cells = <1>; ++ clocks = <&cru CLK_OTPC_NS_USR>, <&cru CLK_OTPC_NS_SBPI>, ++ <&cru PCLK_OTPC_NS>, <&cru PCLK_OTPPHY>; ++ clock-names = "usr", "sbpi", "apb", "phy"; ++ resets = <&cru SRST_OTPPHY>; ++ reset-names = "otp_phy"; ++ ++ /* Data cells */ ++ cpu_code: cpu-code@2 { ++ reg = <0x02 0x2>; ++ }; ++ otp_cpu_version: cpu-version@8 { ++ reg = <0x08 0x1>; ++ bits = <3 3>; ++ }; ++ mbist_vmin: mbist-vmin@9 { ++ reg = <0x09 0x1>; ++ bits = <0 4>; ++ }; ++ otp_id: id@a { ++ reg = <0x0a 0x10>; ++ }; ++ cpu_leakage: cpu-leakage@1a { ++ reg = <0x1a 0x1>; ++ }; ++ log_leakage: log-leakage@1b { ++ reg = <0x1b 0x1>; ++ }; ++ npu_leakage: npu-leakage@1c { ++ reg = <0x1c 0x1>; ++ }; ++ gpu_leakage: gpu-leakage@1d { ++ reg = <0x1d 0x1>; ++ }; ++ core_pvtm:core-pvtm@2a { ++ reg = <0x2a 0x2>; ++ }; ++ }; ++ ++ i2s0_8ch: i2s@fe400000 { ++ compatible = "rockchip,rk3568-i2s-tdm"; ++ reg = <0x0 0xfe400000 0x0 0x1000>; ++ interrupts = ; ++ clocks = <&cru MCLK_I2S0_8CH_TX>, <&cru MCLK_I2S0_8CH_RX>, <&cru HCLK_I2S0_8CH>; ++ clock-names = "mclk_tx", "mclk_rx", "hclk"; ++ dmas = <&dmac1 0>; ++ dma-names = "tx"; ++ resets = <&cru SRST_M_I2S0_8CH_TX>, <&cru SRST_M_I2S0_8CH_RX>; ++ reset-names = "tx-m", "rx-m"; ++ rockchip,cru = <&cru>; ++ rockchip,grf = <&grf>; ++ rockchip,playback-only; ++ #sound-dai-cells = <0>; ++ status = "disabled"; ++ }; ++ ++ i2s1_8ch: i2s@fe410000 { ++ compatible = "rockchip,rk3568-i2s-tdm"; ++ reg = <0x0 0xfe410000 0x0 0x1000>; ++ interrupts = ; ++ clocks = <&cru MCLK_I2S1_8CH_TX>, <&cru MCLK_I2S1_8CH_RX>, <&cru HCLK_I2S1_8CH>; ++ clock-names = "mclk_tx", "mclk_rx", "hclk"; ++ dmas = <&dmac1 2>, <&dmac1 3>; ++ dma-names = "tx", "rx"; ++ resets = <&cru SRST_M_I2S1_8CH_TX>, <&cru SRST_M_I2S1_8CH_RX>; ++ reset-names = "tx-m", "rx-m"; ++ rockchip,cru = <&cru>; ++ rockchip,grf = <&grf>; ++ #sound-dai-cells = <0>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s1m0_sclktx ++ &i2s1m0_sclkrx ++ &i2s1m0_lrcktx ++ &i2s1m0_lrckrx ++ &i2s1m0_sdi0 ++ &i2s1m0_sdi1 ++ &i2s1m0_sdi2 ++ &i2s1m0_sdi3 ++ &i2s1m0_sdo0 ++ &i2s1m0_sdo1 ++ &i2s1m0_sdo2 ++ &i2s1m0_sdo3>; ++ status = "disabled"; ++ }; ++ ++ i2s2_2ch: i2s@fe420000 { ++ compatible = "rockchip,rk3568-i2s-tdm"; ++ reg = <0x0 0xfe420000 0x0 0x1000>; ++ interrupts = ; ++ clocks = <&cru MCLK_I2S2_2CH>, <&cru MCLK_I2S2_2CH>, <&cru HCLK_I2S2_2CH>; ++ clock-names = "mclk_tx", "mclk_rx", "hclk"; ++ dmas = <&dmac1 4>, <&dmac1 5>; ++ dma-names = "tx", "rx"; ++ rockchip,cru = <&cru>; ++ rockchip,grf = <&grf>; ++ rockchip,clk-trcm = <1>; ++ #sound-dai-cells = <0>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s2m0_sclktx ++ &i2s2m0_lrcktx ++ &i2s2m0_sdi ++ &i2s2m0_sdo>; ++ status = "disabled"; ++ }; ++ ++ i2s3_2ch: i2s@fe430000 { ++ compatible = "rockchip,rk3568-i2s-tdm"; ++ reg = <0x0 0xfe430000 0x0 0x1000>; ++ interrupts = ; ++ clocks = <&cru MCLK_I2S3_2CH_TX>, <&cru MCLK_I2S3_2CH_RX>, <&cru HCLK_I2S3_2CH>; ++ clock-names = "mclk_tx", "mclk_rx", "hclk"; ++ dmas = <&dmac1 6>, <&dmac1 7>; ++ dma-names = "tx", "rx"; ++ resets = <&cru SRST_M_I2S3_2CH_TX>, <&cru SRST_M_I2S3_2CH_RX>; ++ reset-names = "tx-m", "rx-m"; ++ rockchip,cru = <&cru>; ++ rockchip,grf = <&grf>; ++ rockchip,clk-trcm = <1>; ++ #sound-dai-cells = <0>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2s3m0_sclk ++ &i2s3m0_lrck ++ &i2s3m0_sdi ++ &i2s3m0_sdo>; ++ status = "disabled"; ++ }; ++ ++ pdm: pdm@fe440000 { ++ compatible = "rockchip,rk3568-pdm", "rockchip,pdm"; ++ reg = <0x0 0xfe440000 0x0 0x1000>; ++ clocks = <&cru MCLK_PDM>, <&cru HCLK_PDM>; ++ clock-names = "pdm_clk", "pdm_hclk"; ++ dmas = <&dmac1 9>; ++ dma-names = "rx"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&pdmm0_clk ++ &pdmm0_clk1 ++ &pdmm0_sdi0 ++ &pdmm0_sdi1 ++ &pdmm0_sdi2 ++ &pdmm0_sdi3>; ++ #sound-dai-cells = <0>; ++ status = "disabled"; ++ }; ++ ++ vad: vad@fe450000 { ++ compatible = "rockchip,rk3568-vad"; ++ reg = <0x0 0xfe450000 0x0 0x10000>; ++ reg-names = "vad"; ++ clocks = <&cru HCLK_VAD>; ++ clock-names = "hclk"; ++ interrupts = ; ++ rockchip,audio-src = <0>; ++ rockchip,det-channel = <0>; ++ rockchip,mode = <0>; ++ #sound-dai-cells = <0>; ++ status = "disabled"; ++ }; ++ ++ spdif_8ch: spdif@fe460000 { ++ compatible = "rockchip,rk3568-spdif"; ++ reg = <0x0 0xfe460000 0x0 0x1000>; ++ interrupts = ; ++ dmas = <&dmac1 1>; ++ dma-names = "tx"; ++ clock-names = "mclk", "hclk"; ++ clocks = <&cru MCLK_SPDIF_8CH>, <&cru HCLK_SPDIF_8CH>; ++ #sound-dai-cells = <0>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&spdifm0_tx>; ++ status = "disabled"; ++ }; ++ ++ audpwm: audpwm@fe470000 { ++ compatible = "rockchip,rk3568-audio-pwm", "rockchip,audio-pwm-v1"; ++ reg = <0x0 0xfe470000 0x0 0x1000>; ++ clocks = <&cru SCLK_AUDPWM>, <&cru HCLK_AUDPWM>; ++ clock-names = "clk", "hclk"; ++ dmas = <&dmac1 8>; ++ dma-names = "tx"; ++ #sound-dai-cells = <0>; ++ rockchip,sample-width-bits = <11>; ++ rockchip,interpolat-points = <1>; ++ status = "disabled"; ++ }; ++ ++ dig_acodec: codec-digital@fe478000 { ++ compatible = "rockchip,rk3568-codec-digital", "rockchip,codec-digital-v1"; ++ reg = <0x0 0xfe478000 0x0 0x1000>; ++ clocks = <&cru CLK_ACDCDIG_ADC>, <&cru CLK_ACDCDIG_DAC>, ++ <&cru CLK_ACDCDIG_I2C>, <&cru HCLK_ACDCDIG>; ++ clock-names = "adc", "dac", "i2c", "pclk"; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&acodec_pins>; ++ resets = <&cru SRST_ACDCDIG>; ++ reset-names = "reset" ; ++ rockchip,grf = <&grf>; ++ #sound-dai-cells = <0>; ++ status = "disabled"; ++ }; ++ ++ dmac0: dmac@fe530000 { ++ compatible = "arm,pl330", "arm,primecell"; ++ reg = <0x0 0xfe530000 0x0 0x4000>; ++ interrupts = , ++ ; ++ clocks = <&cru ACLK_BUS>; ++ clock-names = "apb_pclk"; ++ #dma-cells = <1>; ++ arm,pl330-periph-burst; ++ }; ++ ++ dmac1: dmac@fe550000 { ++ compatible = "arm,pl330", "arm,primecell"; ++ reg = <0x0 0xfe550000 0x0 0x4000>; ++ interrupts = , ++ ; ++ clocks = <&cru ACLK_BUS>; ++ clock-names = "apb_pclk"; ++ #dma-cells = <1>; ++ arm,pl330-periph-burst; ++ }; ++ ++ scr: rkscr@fe560000 { ++ compatible = "rockchip-scr"; ++ reg = <0x0 0xfe560000 0x0 0x10000>; ++ interrupts = ; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&scr_pins>; ++ clocks = <&cru PCLK_SCR>; ++ clock-names = "g_pclk_sim_card"; ++ status = "disabled"; ++ }; ++ ++ can0: can@fe570000 { ++ compatible = "rockchip,canfd-1.0"; ++ reg = <0x0 0xfe570000 0x0 0x1000>; ++ interrupts = ; ++ clocks = <&cru CLK_CAN0>, <&cru PCLK_CAN0>; ++ clock-names = "baudclk", "apb_pclk"; ++ resets = <&cru SRST_CAN0>, <&cru SRST_P_CAN0>; ++ reset-names = "can", "can-apb"; ++ tx-fifo-depth = <1>; ++ rx-fifo-depth = <6>; ++ status = "disabled"; ++ }; ++ ++ can1: can@fe580000 { ++ compatible = "rockchip,canfd-1.0"; ++ reg = <0x0 0xfe580000 0x0 0x1000>; ++ interrupts = ; ++ clocks = <&cru CLK_CAN1>, <&cru PCLK_CAN1>; ++ clock-names = "baudclk", "apb_pclk"; ++ resets = <&cru SRST_CAN1>, <&cru SRST_P_CAN1>; ++ reset-names = "can", "can-apb"; ++ tx-fifo-depth = <1>; ++ rx-fifo-depth = <6>; ++ status = "disabled"; ++ }; ++ ++ can2: can@fe590000 { ++ compatible = "rockchip,canfd-1.0"; ++ reg = <0x0 0xfe590000 0x0 0x1000>; ++ interrupts = ; ++ clocks = <&cru CLK_CAN2>, <&cru PCLK_CAN2>; ++ clock-names = "baudclk", "apb_pclk"; ++ resets = <&cru SRST_CAN2>, <&cru SRST_P_CAN2>; ++ reset-names = "can", "can-apb"; ++ tx-fifo-depth = <1>; ++ rx-fifo-depth = <6>; ++ status = "disabled"; ++ }; ++ ++ i2c1: i2c@fe5a0000 { ++ compatible = "rockchip,rk3399-i2c"; ++ reg = <0x0 0xfe5a0000 0x0 0x1000>; ++ clocks = <&cru CLK_I2C1>, <&cru PCLK_I2C1>; ++ clock-names = "i2c", "pclk"; ++ interrupts = ; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2c1_xfer>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ status = "disabled"; ++ }; ++ ++ i2c2: i2c@fe5b0000 { ++ compatible = "rockchip,rk3399-i2c"; ++ reg = <0x0 0xfe5b0000 0x0 0x1000>; ++ clocks = <&cru CLK_I2C2>, <&cru PCLK_I2C2>; ++ clock-names = "i2c", "pclk"; ++ interrupts = ; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2c2m0_xfer>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ status = "disabled"; ++ }; ++ ++ i2c3: i2c@fe5c0000 { ++ compatible = "rockchip,rk3399-i2c"; ++ reg = <0x0 0xfe5c0000 0x0 0x1000>; ++ clocks = <&cru CLK_I2C3>, <&cru PCLK_I2C3>; ++ clock-names = "i2c", "pclk"; ++ interrupts = ; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2c3m0_xfer>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ status = "disabled"; ++ }; ++ ++ i2c4: i2c@fe5d0000 { ++ compatible = "rockchip,rk3399-i2c"; ++ reg = <0x0 0xfe5d0000 0x0 0x1000>; ++ clocks = <&cru CLK_I2C4>, <&cru PCLK_I2C4>; ++ clock-names = "i2c", "pclk"; ++ interrupts = ; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2c4m0_xfer>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ status = "disabled"; ++ }; ++ ++ i2c5: i2c@fe5e0000 { ++ compatible = "rockchip,rk3399-i2c"; ++ reg = <0x0 0xfe5e0000 0x0 0x1000>; ++ clocks = <&cru CLK_I2C5>, <&cru PCLK_I2C5>; ++ clock-names = "i2c", "pclk"; ++ interrupts = ; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&i2c5m0_xfer>; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ status = "disabled"; ++ }; ++ ++ rktimer: timer@fe5f0000 { ++ compatible = "rockchip,rk3568-timer", "rockchip,rk3288-timer"; ++ reg = <0x0 0xfe5f0000 0x0 0x1000>; ++ interrupts = ; ++ clocks = <&cru PCLK_TIMER>, <&cru CLK_TIMER0>; ++ clock-names = "pclk", "timer"; ++ }; ++ ++ wdt: watchdog@fe600000 { ++ compatible = "snps,dw-wdt"; ++ reg = <0x0 0xfe600000 0x0 0x100>; ++ clocks = <&cru TCLK_WDT_NS>, <&cru PCLK_WDT_NS>; ++ clock-names = "tclk", "pclk"; ++ interrupts = ; ++ status = "okay"; ++ }; ++ ++ spi0: spi@fe610000 { ++ compatible = "rockchip,rk3066-spi"; ++ reg = <0x0 0xfe610000 0x0 0x1000>; ++ interrupts = ; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ clocks = <&cru CLK_SPI0>, <&cru PCLK_SPI0>; ++ clock-names = "spiclk", "apb_pclk"; ++ dmas = <&dmac0 20>, <&dmac0 21>; ++ dma-names = "tx", "rx"; ++ pinctrl-names = "default", "high_speed"; ++ pinctrl-0 = <&spi0m0_cs0 &spi0m0_cs1 &spi0m0_pins>; ++ pinctrl-1 = <&spi0m0_cs0 &spi0m0_cs1 &spi0m0_pins_hs>; ++ status = "disabled"; ++ }; ++ ++ spi1: spi@fe620000 { ++ compatible = "rockchip,rk3066-spi"; ++ reg = <0x0 0xfe620000 0x0 0x1000>; ++ interrupts = ; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ clocks = <&cru CLK_SPI1>, <&cru PCLK_SPI1>; ++ clock-names = "spiclk", "apb_pclk"; ++ dmas = <&dmac0 22>, <&dmac0 23>; ++ dma-names = "tx", "rx"; ++ pinctrl-names = "default", "high_speed"; ++ pinctrl-0 = <&spi1m0_cs0 &spi1m0_cs1 &spi1m0_pins>; ++ pinctrl-1 = <&spi1m0_cs0 &spi1m0_cs1 &spi1m0_pins_hs>; ++ status = "disabled"; ++ }; ++ ++ spi2: spi@fe630000 { ++ compatible = "rockchip,rk3066-spi"; ++ reg = <0x0 0xfe630000 0x0 0x1000>; ++ interrupts = ; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ clocks = <&cru CLK_SPI2>, <&cru PCLK_SPI2>; ++ clock-names = "spiclk", "apb_pclk"; ++ dmas = <&dmac0 24>, <&dmac0 25>; ++ dma-names = "tx", "rx"; ++ pinctrl-names = "default", "high_speed"; ++ pinctrl-0 = <&spi2m0_cs0 &spi2m0_cs1 &spi2m0_pins>; ++ pinctrl-1 = <&spi2m0_cs0 &spi2m0_cs1 &spi2m0_pins_hs>; ++ status = "disabled"; ++ }; ++ ++ spi3: spi@fe640000 { ++ compatible = "rockchip,rk3066-spi"; ++ reg = <0x0 0xfe640000 0x0 0x1000>; ++ interrupts = ; ++ #address-cells = <1>; ++ #size-cells = <0>; ++ clocks = <&cru CLK_SPI3>, <&cru PCLK_SPI3>; ++ clock-names = "spiclk", "apb_pclk"; ++ dmas = <&dmac0 26>, <&dmac0 27>; ++ dma-names = "tx", "rx"; ++ pinctrl-names = "default", "high_speed"; ++ pinctrl-0 = <&spi3m0_cs0 &spi3m0_cs1 &spi3m0_pins>; ++ pinctrl-1 = <&spi3m0_cs0 &spi3m0_cs1 &spi3m0_pins_hs>; ++ status = "disabled"; ++ }; ++ ++ uart1: serial@fe650000 { ++ compatible = "rockchip,rk3568-uart", "snps,dw-apb-uart"; ++ reg = <0x0 0xfe650000 0x0 0x100>; ++ interrupts = ; ++ clocks = <&cru SCLK_UART1>, <&cru PCLK_UART1>; ++ clock-names = "baudclk", "apb_pclk"; ++ reg-shift = <2>; ++ reg-io-width = <4>; ++ dmas = <&dmac0 2>, <&dmac0 3>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart1m0_xfer>; ++ status = "disabled"; ++ }; ++ ++ uart2: serial@fe660000 { ++ compatible = "rockchip,rk3568-uart", "snps,dw-apb-uart"; ++ reg = <0x0 0xfe660000 0x0 0x100>; ++ interrupts = ; ++ clocks = <&cru SCLK_UART2>, <&cru PCLK_UART2>; ++ clock-names = "baudclk", "apb_pclk"; ++ reg-shift = <2>; ++ reg-io-width = <4>; ++ dmas = <&dmac0 4>, <&dmac0 5>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart2m0_xfer>; ++ status = "disabled"; ++ }; ++ ++ uart3: serial@fe670000 { ++ compatible = "rockchip,rk3568-uart", "snps,dw-apb-uart"; ++ reg = <0x0 0xfe670000 0x0 0x100>; ++ interrupts = ; ++ clocks = <&cru SCLK_UART3>, <&cru PCLK_UART3>; ++ clock-names = "baudclk", "apb_pclk"; ++ reg-shift = <2>; ++ reg-io-width = <4>; ++ dmas = <&dmac0 6>, <&dmac0 7>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart3m0_xfer>; ++ status = "disabled"; ++ }; ++ ++ uart4: serial@fe680000 { ++ compatible = "rockchip,rk3568-uart", "snps,dw-apb-uart"; ++ reg = <0x0 0xfe680000 0x0 0x100>; ++ interrupts = ; ++ clocks = <&cru SCLK_UART4>, <&cru PCLK_UART4>; ++ clock-names = "baudclk", "apb_pclk"; ++ reg-shift = <2>; ++ reg-io-width = <4>; ++ dmas = <&dmac0 8>, <&dmac0 9>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart4m0_xfer>; ++ status = "disabled"; ++ }; ++ ++ uart5: serial@fe690000 { ++ compatible = "rockchip,rk3568-uart", "snps,dw-apb-uart"; ++ reg = <0x0 0xfe690000 0x0 0x100>; ++ interrupts = ; ++ clocks = <&cru SCLK_UART5>, <&cru PCLK_UART5>; ++ clock-names = "baudclk", "apb_pclk"; ++ reg-shift = <2>; ++ reg-io-width = <4>; ++ dmas = <&dmac0 10>, <&dmac0 11>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart5m0_xfer>; ++ status = "disabled"; ++ }; ++ ++ uart6: serial@fe6a0000 { ++ compatible = "rockchip,rk3568-uart", "snps,dw-apb-uart"; ++ reg = <0x0 0xfe6a0000 0x0 0x100>; ++ interrupts = ; ++ clocks = <&cru SCLK_UART6>, <&cru PCLK_UART6>; ++ clock-names = "baudclk", "apb_pclk"; ++ reg-shift = <2>; ++ reg-io-width = <4>; ++ dmas = <&dmac0 12>, <&dmac0 13>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart6m0_xfer>; ++ status = "disabled"; ++ }; ++ ++ uart7: serial@fe6b0000 { ++ compatible = "rockchip,rk3568-uart", "snps,dw-apb-uart"; ++ reg = <0x0 0xfe6b0000 0x0 0x100>; ++ interrupts = ; ++ clocks = <&cru SCLK_UART7>, <&cru PCLK_UART7>; ++ clock-names = "baudclk", "apb_pclk"; ++ reg-shift = <2>; ++ reg-io-width = <4>; ++ dmas = <&dmac0 14>, <&dmac0 15>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart7m0_xfer>; ++ status = "disabled"; ++ }; ++ ++ uart8: serial@fe6c0000 { ++ compatible = "rockchip,rk3568-uart", "snps,dw-apb-uart"; ++ reg = <0x0 0xfe6c0000 0x0 0x100>; ++ interrupts = ; ++ clocks = <&cru SCLK_UART8>, <&cru PCLK_UART8>; ++ clock-names = "baudclk", "apb_pclk"; ++ reg-shift = <2>; ++ reg-io-width = <4>; ++ dmas = <&dmac0 16>, <&dmac0 17>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart8m0_xfer>; ++ status = "disabled"; ++ }; ++ ++ uart9: serial@fe6d0000 { ++ compatible = "rockchip,rk3568-uart", "snps,dw-apb-uart"; ++ reg = <0x0 0xfe6d0000 0x0 0x100>; ++ interrupts = ; ++ clocks = <&cru SCLK_UART9>, <&cru PCLK_UART9>; ++ clock-names = "baudclk", "apb_pclk"; ++ reg-shift = <2>; ++ reg-io-width = <4>; ++ dmas = <&dmac0 18>, <&dmac0 19>; ++ pinctrl-names = "default"; ++ pinctrl-0 = <&uart9m0_xfer>; ++ status = "disabled"; ++ }; ++ ++ pwm4: pwm@fe6e0000 { ++ compatible = "rockchip,rk3568-pwm", "rockchip,rk3328-pwm"; ++ reg = <0x0 0xfe6e0000 0x0 0x10>; ++ #pwm-cells = <3>; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm4_pins>; ++ clocks = <&cru CLK_PWM1>, <&cru PCLK_PWM1>; ++ clock-names = "pwm", "pclk"; ++ status = "disabled"; ++ }; ++ ++ pwm5: pwm@fe6e0010 { ++ compatible = "rockchip,rk3568-pwm", "rockchip,rk3328-pwm"; ++ reg = <0x0 0xfe6e0010 0x0 0x10>; ++ #pwm-cells = <3>; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm5_pins>; ++ clocks = <&cru CLK_PWM1>, <&cru PCLK_PWM1>; ++ clock-names = "pwm", "pclk"; ++ status = "disabled"; ++ }; ++ ++ pwm6: pwm@fe6e0020 { ++ compatible = "rockchip,rk3568-pwm", "rockchip,rk3328-pwm"; ++ reg = <0x0 0xfe6e0020 0x0 0x10>; ++ #pwm-cells = <3>; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm6_pins>; ++ clocks = <&cru CLK_PWM1>, <&cru PCLK_PWM1>; ++ clock-names = "pwm", "pclk"; ++ status = "disabled"; ++ }; ++ ++ pwm7: pwm@fe6e0030 { ++ compatible = "rockchip,rk3568-pwm", "rockchip,rk3328-pwm"; ++ reg = <0x0 0xfe6e0030 0x0 0x10>; ++ interrupts = , ++ ; ++ #pwm-cells = <3>; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm7_pins>; ++ clocks = <&cru CLK_PWM1>, <&cru PCLK_PWM1>; ++ clock-names = "pwm", "pclk"; ++ status = "disabled"; ++ }; ++ ++ pwm8: pwm@fe6f0000 { ++ compatible = "rockchip,rk3568-pwm", "rockchip,rk3328-pwm"; ++ reg = <0x0 0xfe6f0000 0x0 0x10>; ++ #pwm-cells = <3>; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm8m0_pins>; ++ clocks = <&cru CLK_PWM2>, <&cru PCLK_PWM2>; ++ clock-names = "pwm", "pclk"; ++ status = "disabled"; ++ }; ++ ++ pwm9: pwm@fe6f0010 { ++ compatible = "rockchip,rk3568-pwm", "rockchip,rk3328-pwm"; ++ reg = <0x0 0xfe6f0010 0x0 0x10>; ++ #pwm-cells = <3>; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm9m0_pins>; ++ clocks = <&cru CLK_PWM2>, <&cru PCLK_PWM2>; ++ clock-names = "pwm", "pclk"; ++ status = "disabled"; ++ }; ++ ++ pwm10: pwm@fe6f0020 { ++ compatible = "rockchip,rk3568-pwm", "rockchip,rk3328-pwm"; ++ reg = <0x0 0xfe6f0020 0x0 0x10>; ++ #pwm-cells = <3>; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm10m0_pins>; ++ clocks = <&cru CLK_PWM2>, <&cru PCLK_PWM2>; ++ clock-names = "pwm", "pclk"; ++ status = "disabled"; ++ }; ++ ++ pwm11: pwm@fe6f0030 { ++ compatible = "rockchip,rk3568-pwm", "rockchip,rk3328-pwm"; ++ reg = <0x0 0xfe6f0030 0x0 0x10>; ++ interrupts = , ++ ; ++ #pwm-cells = <3>; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm11m0_pins>; ++ clocks = <&cru CLK_PWM2>, <&cru PCLK_PWM2>; ++ clock-names = "pwm", "pclk"; ++ status = "disabled"; ++ }; ++ ++ pwm12: pwm@fe700000 { ++ compatible = "rockchip,rk3568-pwm", "rockchip,rk3328-pwm"; ++ reg = <0x0 0xfe700000 0x0 0x10>; ++ #pwm-cells = <3>; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm12m0_pins>; ++ clocks = <&cru CLK_PWM3>, <&cru PCLK_PWM3>; ++ clock-names = "pwm", "pclk"; ++ status = "disabled"; ++ }; ++ ++ pwm13: pwm@fe700010 { ++ compatible = "rockchip,rk3568-pwm", "rockchip,rk3328-pwm"; ++ reg = <0x0 0xfe700010 0x0 0x10>; ++ #pwm-cells = <3>; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm13m0_pins>; ++ clocks = <&cru CLK_PWM3>, <&cru PCLK_PWM3>; ++ clock-names = "pwm", "pclk"; ++ status = "disabled"; ++ }; ++ ++ pwm14: pwm@fe700020 { ++ compatible = "rockchip,rk3568-pwm", "rockchip,rk3328-pwm"; ++ reg = <0x0 0xfe700020 0x0 0x10>; ++ #pwm-cells = <3>; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm14m0_pins>; ++ clocks = <&cru CLK_PWM3>, <&cru PCLK_PWM3>; ++ clock-names = "pwm", "pclk"; ++ status = "disabled"; ++ }; ++ ++ pwm15: pwm@fe700030 { ++ compatible = "rockchip,rk3568-pwm", "rockchip,rk3328-pwm"; ++ reg = <0x0 0xfe700030 0x0 0x10>; ++ interrupts = , ++ ; ++ #pwm-cells = <3>; ++ pinctrl-names = "active"; ++ pinctrl-0 = <&pwm15m0_pins>; ++ clocks = <&cru CLK_PWM3>, <&cru PCLK_PWM3>; ++ clock-names = "pwm", "pclk"; ++ status = "disabled"; ++ }; ++ ++ tsadc: tsadc@fe710000 { ++ compatible = "rockchip,rk3568-tsadc"; ++ reg = <0x0 0xfe710000 0x0 0x100>; ++ interrupts = ; ++ rockchip,grf = <&grf>; ++ clocks = <&cru CLK_TSADC>, <&cru PCLK_TSADC>; ++ clock-names = "tsadc", "apb_pclk"; ++ assigned-clocks = <&cru CLK_TSADC_TSEN>, <&cru CLK_TSADC>; ++ assigned-clock-rates = <17000000>, <700000>; ++ resets = <&cru SRST_TSADC>, <&cru SRST_P_TSADC>, ++ <&cru SRST_TSADCPHY>; ++ reset-names = "tsadc", "tsadc-apb", "tsadc-phy"; ++ #thermal-sensor-cells = <1>; ++ rockchip,hw-tshut-temp = <120000>; ++ rockchip,hw-tshut-mode = <0>; /* tshut mode 0:CRU 1:GPIO */ ++ rockchip,hw-tshut-polarity = <0>; /* tshut polarity 0:LOW 1:HIGH */ ++ pinctrl-names = "gpio", "otpout"; ++ pinctrl-0 = <&tsadc_gpio_func>; ++ pinctrl-1 = <&tsadc_shutorg>; ++ status = "disabled"; ++ }; ++ ++ saradc: saradc@fe720000 { ++ compatible = "rockchip,rk3568-saradc", "rockchip,rk3399-saradc"; ++ reg = <0x0 0xfe720000 0x0 0x100>; ++ interrupts = ; ++ #io-channel-cells = <1>; ++ clocks = <&cru CLK_SARADC>, <&cru PCLK_SARADC>; ++ clock-names = "saradc", "apb_pclk"; ++ resets = <&cru SRST_P_SARADC>; ++ reset-names = "saradc-apb"; ++ status = "disabled"; ++ }; ++ ++ mailbox: mailbox@fe780000 { ++ compatible = "rockchip,rk3568-mailbox", ++ "rockchip,rk3368-mailbox"; ++ reg = <0x0 0xfe780000 0x0 0x1000>; ++ interrupts = , ++ , ++ , ++ ; ++ clocks = <&cru PCLK_MAILBOX>; ++ clock-names = "pclk_mailbox"; ++ #mbox-cells = <1>; ++ status = "disabled"; ++ }; ++ ++ combphy0_us: phy@fe820000 { ++ compatible = "rockchip,rk3568-naneng-combphy"; ++ reg = <0x0 0xfe820000 0x0 0x100>; ++ #phy-cells = <1>; ++ clocks = <&pmucru CLK_PCIEPHY0_REF>, <&cru PCLK_PIPEPHY0>, ++ <&cru PCLK_PIPE>; ++ clock-names = "refclk", "apbclk", "pipe_clk"; ++ assigned-clocks = <&pmucru CLK_PCIEPHY0_REF>; ++ assigned-clock-rates = <100000000>; ++ resets = <&cru SRST_P_PIPEPHY0>, <&cru SRST_PIPEPHY0>; ++ reset-names = "combphy-apb", "combphy"; ++ rockchip,pipe-grf = <&pipegrf>; ++ rockchip,pipe-phy-grf = <&pipe_phy_grf0>; ++ status = "disabled"; ++ }; ++ ++ combphy1_usq: phy@fe830000 { ++ compatible = "rockchip,rk3568-naneng-combphy"; ++ reg = <0x0 0xfe830000 0x0 0x100>; ++ #phy-cells = <1>; ++ clocks = <&pmucru CLK_PCIEPHY1_REF>, <&cru PCLK_PIPEPHY1>, ++ <&cru PCLK_PIPE>; ++ clock-names = "refclk", "apbclk", "pipe_clk"; ++ assigned-clocks = <&pmucru CLK_PCIEPHY1_REF>; ++ assigned-clock-rates = <100000000>; ++ resets = <&cru SRST_P_PIPEPHY1>, <&cru SRST_PIPEPHY1>; ++ reset-names = "combphy-apb", "combphy"; ++ rockchip,pipe-grf = <&pipegrf>; ++ rockchip,pipe-phy-grf = <&pipe_phy_grf1>; ++ status = "disabled"; ++ }; ++ ++ combphy2_psq: phy@fe840000 { ++ compatible = "rockchip,rk3568-naneng-combphy"; ++ reg = <0x0 0xfe840000 0x0 0x100>; ++ #phy-cells = <1>; ++ clocks = <&pmucru CLK_PCIEPHY2_REF>, <&cru PCLK_PIPEPHY2>, ++ <&cru PCLK_PIPE>; ++ clock-names = "refclk", "apbclk", "pipe_clk"; ++ assigned-clocks = <&pmucru CLK_PCIEPHY2_REF>; ++ assigned-clock-rates = <100000000>; ++ resets = <&cru SRST_P_PIPEPHY2>, <&cru SRST_PIPEPHY2>; ++ reset-names = "combphy-apb", "combphy"; ++ rockchip,pipe-grf = <&pipegrf>; ++ rockchip,pipe-phy-grf = <&pipe_phy_grf2>; ++ status = "disabled"; ++ }; ++ ++ video_phy0: phy@fe850000 { ++ compatible = "rockchip,rk3568-dsi-dphy", "rockchip,rk3568-video-phy"; ++ reg = <0x0 0xfe850000 0x0 0x10000>, ++ <0x0 0xfe060000 0x0 0x10000>; ++ reg-names = "phy", "host"; ++ clocks = <&pmucru CLK_MIPIDSIPHY0_REF>, ++ <&cru PCLK_MIPIDSIPHY0>, <&cru PCLK_DSITX_0>; ++ clock-names = "ref", "pclk", "pclk_host"; ++ #clock-cells = <0>; ++ resets = <&cru SRST_P_MIPIDSIPHY0>; ++ reset-names = "apb"; ++ power-domains = <&power RK3568_PD_VO>; ++ #phy-cells = <0>; ++ status = "disabled"; ++ }; ++ ++ video_phy1: phy@fe860000 { ++ compatible = "rockchip,rk3568-dsi-dphy", "rockchip,rk3568-video-phy"; ++ reg = <0x0 0xfe860000 0x0 0x10000>, ++ <0x0 0xfe070000 0x0 0x10000>; ++ reg-names = "phy", "host"; ++ clocks = <&pmucru CLK_MIPIDSIPHY1_REF>, ++ <&cru PCLK_MIPIDSIPHY1>, <&cru PCLK_DSITX_1>; ++ clock-names = "ref", "pclk", "pclk_host"; ++ #clock-cells = <0>; ++ resets = <&cru SRST_P_MIPIDSIPHY1>; ++ reset-names = "apb"; ++ power-domains = <&power RK3568_PD_VO>; ++ #phy-cells = <0>; ++ status = "disabled"; ++ }; ++ ++ csi2_dphy_hw: csi2-dphy-hw@fe870000 { ++ compatible = "rockchip,rk3568-csi2-dphy-hw"; ++ reg = <0x0 0xfe870000 0x0 0x1000>; ++ clocks = <&cru PCLK_MIPICSIPHY>; ++ clock-names = "pclk"; ++ rockchip,grf = <&grf>; ++ status = "disabled"; ++ }; ++ ++ /* ++ * csi2_dphy0: used for csi2 dphy full mode, ++ is mutually exclusive with ++ csi2_dphy1 and csi2_dphy2 ++ * csi2_dphy1: used for csi2 dphy split mode, ++ physical lanes use lane0 and lane1, ++ can be used with csi2_dphy2 parallel ++ * csi2_dphy2: used for csi2 dphy split mode, ++ physical lanes use lane2 and lane3, ++ can be used with csi2_dphy1 parallel ++ */ ++ csi2_dphy0: csi2-dphy0 { ++ compatible = "rockchip,rk3568-csi2-dphy"; ++ rockchip,hw = <&csi2_dphy_hw>; ++ status = "disabled"; ++ }; ++ ++ csi2_dphy1: csi2-dphy1 { ++ compatible = "rockchip,rk3568-csi2-dphy"; ++ rockchip,hw = <&csi2_dphy_hw>; ++ status = "disabled"; ++ }; ++ ++ csi2_dphy2: csi2-dphy2 { ++ compatible = "rockchip,rk3568-csi2-dphy"; ++ rockchip,hw = <&csi2_dphy_hw>; ++ status = "disabled"; ++ }; ++ ++ usb2phy0: usb2-phy@fe8a0000 { ++ compatible = "rockchip,rk3568-usb2phy"; ++ reg = <0x0 0xfe8a0000 0x0 0x10000>; ++ interrupts = ; ++ clocks = <&pmucru CLK_USBPHY0_REF>; ++ clock-names = "phyclk"; ++ #clock-cells = <0>; ++ assigned-clocks = <&cru USB480M>; ++ assigned-clock-parents = <&usb2phy0>; ++ clock-output-names = "usb480m_phy"; ++ rockchip,usbgrf = <&usb2phy0_grf>; ++ status = "disabled"; ++ ++ u2phy0_host: host-port { ++ #phy-cells = <0>; ++ status = "disabled"; ++ }; ++ ++ u2phy0_otg: otg-port { ++ #phy-cells = <0>; ++ status = "disabled"; ++ }; ++ }; ++ ++ usb2phy1: usb2-phy@fe8b0000 { ++ compatible = "rockchip,rk3568-usb2phy"; ++ reg = <0x0 0xfe8b0000 0x0 0x10000>; ++ interrupts = ; ++ clocks = <&pmucru CLK_USBPHY1_REF>; ++ clock-names = "phyclk"; ++ #clock-cells = <0>; ++ rockchip,usbgrf = <&usb2phy1_grf>; ++ status = "disabled"; ++ ++ u2phy1_host: host-port { ++ #phy-cells = <0>; ++ status = "disabled"; ++ }; ++ ++ u2phy1_otg: otg-port { ++ #phy-cells = <0>; ++ status = "disabled"; ++ }; ++ }; ++ ++ pcie30phy: phy@fe8c0000 { ++ compatible = "rockchip,rk3568-pcie3-phy"; ++ reg = <0x0 0xfe8c0000 0x0 0x20000>; ++ #phy-cells = <0>; ++ clocks = <&pmucru CLK_PCIE30PHY_REF_M>, <&pmucru CLK_PCIE30PHY_REF_N>, ++ <&cru PCLK_PCIE30PHY>; ++ clock-names = "refclk_m", "refclk_n", "pclk"; ++ resets = <&cru SRST_PCIE30PHY>; ++ reset-names = "phy"; ++ rockchip,phy-grf = <&pcie30_phy_grf>; ++ status = "disabled"; ++ }; ++ ++ pinctrl: pinctrl { ++ compatible = "rockchip,rk3568-pinctrl"; ++ rockchip,grf = <&grf>; ++ rockchip,pmu = <&pmugrf>; ++ #address-cells = <2>; ++ #size-cells = <2>; ++ ranges; ++ ++ gpio0: gpio0@fdd60000 { ++ compatible = "rockchip,gpio-bank"; ++ reg = <0x0 0xfdd60000 0x0 0x100>; ++ interrupts = ; ++ clocks = <&pmucru PCLK_GPIO0>, <&pmucru DBCLK_GPIO0>; ++ ++ gpio-controller; ++ #gpio-cells = <2>; ++ interrupt-controller; ++ #interrupt-cells = <2>; ++ }; ++ ++ gpio1: gpio1@fe740000 { ++ compatible = "rockchip,gpio-bank"; ++ reg = <0x0 0xfe740000 0x0 0x100>; ++ interrupts = ; ++ clocks = <&cru PCLK_GPIO1>, <&cru DBCLK_GPIO1>; ++ ++ gpio-controller; ++ #gpio-cells = <2>; ++ interrupt-controller; ++ #interrupt-cells = <2>; ++ }; ++ ++ gpio2: gpio2@fe750000 { ++ compatible = "rockchip,gpio-bank"; ++ reg = <0x0 0xfe750000 0x0 0x100>; ++ interrupts = ; ++ clocks = <&cru PCLK_GPIO2>, <&cru DBCLK_GPIO2>; ++ ++ gpio-controller; ++ #gpio-cells = <2>; ++ interrupt-controller; ++ #interrupt-cells = <2>; ++ }; ++ ++ gpio3: gpio3@fe760000 { ++ compatible = "rockchip,gpio-bank"; ++ reg = <0x0 0xfe760000 0x0 0x100>; ++ interrupts = ; ++ clocks = <&cru PCLK_GPIO3>, <&cru DBCLK_GPIO3>; ++ ++ gpio-controller; ++ #gpio-cells = <2>; ++ interrupt-controller; ++ #interrupt-cells = <2>; ++ }; ++ ++ gpio4: gpio4@fe770000 { ++ compatible = "rockchip,gpio-bank"; ++ reg = <0x0 0xfe770000 0x0 0x100>; ++ interrupts = ; ++ clocks = <&cru PCLK_GPIO4>, <&cru DBCLK_GPIO4>; ++ ++ gpio-controller; ++ #gpio-cells = <2>; ++ interrupt-controller; ++ #interrupt-cells = <2>; ++ }; ++ }; ++}; ++ ++#include "rk3568-pinctrl.dtsi" +diff --git a/arch/arm64/boot/dts/rockchip/rockchip-pinconf.dtsi b/arch/arm64/boot/dts/rockchip/rockchip-pinconf.dtsi +new file mode 100755 +index 000000000000..fc0145333257 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/rockchip-pinconf.dtsi +@@ -0,0 +1,382 @@ ++// SPDX-License-Identifier: (GPL-2.0+ OR MIT) ++/* ++ * Copyright (c) 2020~2021 Rockchip Electronics Co., Ltd. ++ */ ++ ++&pinctrl { ++ ++ /omit-if-no-ref/ ++ pcfg_pull_up: pcfg-pull-up { ++ bias-pull-up; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_down: pcfg-pull-down { ++ bias-pull-down; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_none: pcfg-pull-none { ++ bias-disable; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_none_drv_level_0: pcfg-pull-none-drv-level-0 { ++ bias-disable; ++ drive-strength = <0>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_none_drv_level_1: pcfg-pull-none-drv-level-1 { ++ bias-disable; ++ drive-strength = <1>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_none_drv_level_2: pcfg-pull-none-drv-level-2 { ++ bias-disable; ++ drive-strength = <2>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_none_drv_level_3: pcfg-pull-none-drv-level-3 { ++ bias-disable; ++ drive-strength = <3>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_none_drv_level_4: pcfg-pull-none-drv-level-4 { ++ bias-disable; ++ drive-strength = <4>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_none_drv_level_5: pcfg-pull-none-drv-level-5 { ++ bias-disable; ++ drive-strength = <5>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_none_drv_level_6: pcfg-pull-none-drv-level-6 { ++ bias-disable; ++ drive-strength = <6>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_none_drv_level_7: pcfg-pull-none-drv-level-7 { ++ bias-disable; ++ drive-strength = <7>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_none_drv_level_8: pcfg-pull-none-drv-level-8 { ++ bias-disable; ++ drive-strength = <8>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_none_drv_level_9: pcfg-pull-none-drv-level-9 { ++ bias-disable; ++ drive-strength = <9>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_none_drv_level_10: pcfg-pull-none-drv-level-10 { ++ bias-disable; ++ drive-strength = <10>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_none_drv_level_11: pcfg-pull-none-drv-level-11 { ++ bias-disable; ++ drive-strength = <11>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_none_drv_level_12: pcfg-pull-none-drv-level-12 { ++ bias-disable; ++ drive-strength = <12>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_none_drv_level_13: pcfg-pull-none-drv-level-13 { ++ bias-disable; ++ drive-strength = <13>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_none_drv_level_14: pcfg-pull-none-drv-level-14 { ++ bias-disable; ++ drive-strength = <14>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_none_drv_level_15: pcfg-pull-none-drv-level-15 { ++ bias-disable; ++ drive-strength = <15>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_up_drv_level_0: pcfg-pull-up-drv-level-0 { ++ bias-pull-up; ++ drive-strength = <0>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_up_drv_level_1: pcfg-pull-up-drv-level-1 { ++ bias-pull-up; ++ drive-strength = <1>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_up_drv_level_2: pcfg-pull-up-drv-level-2 { ++ bias-pull-up; ++ drive-strength = <2>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_up_drv_level_3: pcfg-pull-up-drv-level-3 { ++ bias-pull-up; ++ drive-strength = <3>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_up_drv_level_4: pcfg-pull-up-drv-level-4 { ++ bias-pull-up; ++ drive-strength = <4>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_up_drv_level_5: pcfg-pull-up-drv-level-5 { ++ bias-pull-up; ++ drive-strength = <5>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_up_drv_level_6: pcfg-pull-up-drv-level-6 { ++ bias-pull-up; ++ drive-strength = <6>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_up_drv_level_7: pcfg-pull-up-drv-level-7 { ++ bias-pull-up; ++ drive-strength = <7>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_up_drv_level_8: pcfg-pull-up-drv-level-8 { ++ bias-pull-up; ++ drive-strength = <8>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_up_drv_level_9: pcfg-pull-up-drv-level-9 { ++ bias-pull-up; ++ drive-strength = <9>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_up_drv_level_10: pcfg-pull-up-drv-level-10 { ++ bias-pull-up; ++ drive-strength = <10>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_up_drv_level_11: pcfg-pull-up-drv-level-11 { ++ bias-pull-up; ++ drive-strength = <11>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_up_drv_level_12: pcfg-pull-up-drv-level-12 { ++ bias-pull-up; ++ drive-strength = <12>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_up_drv_level_13: pcfg-pull-up-drv-level-13 { ++ bias-pull-up; ++ drive-strength = <13>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_up_drv_level_14: pcfg-pull-up-drv-level-14 { ++ bias-pull-up; ++ drive-strength = <14>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_up_drv_level_15: pcfg-pull-up-drv-level-15 { ++ bias-pull-up; ++ drive-strength = <15>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_down_drv_level_0: pcfg-pull-down-drv-level-0 { ++ bias-pull-down; ++ drive-strength = <0>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_down_drv_level_1: pcfg-pull-down-drv-level-1 { ++ bias-pull-down; ++ drive-strength = <1>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_down_drv_level_2: pcfg-pull-down-drv-level-2 { ++ bias-pull-down; ++ drive-strength = <2>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_down_drv_level_3: pcfg-pull-down-drv-level-3 { ++ bias-pull-down; ++ drive-strength = <3>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_down_drv_level_4: pcfg-pull-down-drv-level-4 { ++ bias-pull-down; ++ drive-strength = <4>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_down_drv_level_5: pcfg-pull-down-drv-level-5 { ++ bias-pull-down; ++ drive-strength = <5>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_down_drv_level_6: pcfg-pull-down-drv-level-6 { ++ bias-pull-down; ++ drive-strength = <6>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_down_drv_level_7: pcfg-pull-down-drv-level-7 { ++ bias-pull-down; ++ drive-strength = <7>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_down_drv_level_8: pcfg-pull-down-drv-level-8 { ++ bias-pull-down; ++ drive-strength = <8>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_down_drv_level_9: pcfg-pull-down-drv-level-9 { ++ bias-pull-down; ++ drive-strength = <9>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_down_drv_level_10: pcfg-pull-down-drv-level-10 { ++ bias-pull-down; ++ drive-strength = <10>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_down_drv_level_11: pcfg-pull-down-drv-level-11 { ++ bias-pull-down; ++ drive-strength = <11>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_down_drv_level_12: pcfg-pull-down-drv-level-12 { ++ bias-pull-down; ++ drive-strength = <12>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_down_drv_level_13: pcfg-pull-down-drv-level-13 { ++ bias-pull-down; ++ drive-strength = <13>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_down_drv_level_14: pcfg-pull-down-drv-level-14 { ++ bias-pull-down; ++ drive-strength = <14>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_down_drv_level_15: pcfg-pull-down-drv-level-15 { ++ bias-pull-down; ++ drive-strength = <15>; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_up_smt: pcfg-pull-up-smt { ++ bias-pull-up; ++ input-schmitt-enable; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_down_smt: pcfg-pull-down-smt { ++ bias-pull-down; ++ input-schmitt-enable; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_none_smt: pcfg-pull-none-smt { ++ bias-disable; ++ input-schmitt-enable; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_pull_none_drv_level_0_smt: pcfg-pull-none-drv-level-0-smt { ++ bias-disable; ++ drive-strength = <0>; ++ input-schmitt-enable; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_output_high: pcfg-output-high { ++ output-high; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_output_high_pull_up: pcfg-output-high-pull-up { ++ output-high; ++ bias-pull-up; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_output_high_pull_down: pcfg-output-high-pull-down { ++ output-high; ++ bias-pull-down; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_output_high_pull_none: pcfg-output-high-pull-none { ++ output-high; ++ bias-disable; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_output_low: pcfg-output-low { ++ output-low; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_output_low_pull_up: pcfg-output-low-pull-up { ++ output-low; ++ bias-pull-up; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_output_low_pull_down: pcfg-output-low-pull-down { ++ output-low; ++ bias-pull-down; ++ }; ++ ++ /omit-if-no-ref/ ++ pcfg_output_low_pull_none: pcfg-output-low-pull-none { ++ output-low; ++ bias-disable; ++ }; ++}; ++ +diff --git a/arch/arm64/include/asm/system_info.h b/arch/arm64/include/asm/system_info.h +new file mode 100755 +index 000000000000..a82fe791b2f6 +--- /dev/null ++++ b/arch/arm64/include/asm/system_info.h +@@ -0,0 +1,14 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#ifndef __ASM_ARM_SYSTEM_INFO_H ++#define __ASM_ARM_SYSTEM_INFO_H ++ ++#ifndef __ASSEMBLY__ ++ ++/* information about the system we're running on */ ++extern unsigned int system_rev; ++extern unsigned int system_serial_low; ++extern unsigned int system_serial_high; ++ ++#endif /* !__ASSEMBLY__ */ ++ ++#endif /* __ASM_ARM_SYSTEM_INFO_H */ +diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c +index 77605aec25fe..32be174c84d7 100644 +--- a/arch/arm64/kernel/cpuinfo.c ++++ b/arch/arm64/kernel/cpuinfo.c +@@ -25,6 +25,12 @@ + #include + #include + ++unsigned int system_serial_low; ++EXPORT_SYMBOL(system_serial_low); ++ ++unsigned int system_serial_high; ++EXPORT_SYMBOL(system_serial_high); ++ + /* + * In case the boot CPU is hotpluggable, we record its initial state and + * current state separately. Certain system registers may contain different +diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c +index 4999caff3281..b47d477f4ce7 100644 +--- a/arch/arm64/kernel/process.c ++++ b/arch/arm64/kernel/process.c +@@ -191,6 +191,8 @@ void machine_restart(char *cmd) + local_irq_disable(); + smp_send_stop(); + ++ do_kernel_pre_restart(cmd); ++ + /* + * UpdateCapsule() depends on the system being reset via + * ResetSystem(). +diff --git a/drivers/Kconfig b/drivers/Kconfig +index dcecc9f6e33f..8c97ea01aabd 100644 +--- a/drivers/Kconfig ++++ b/drivers/Kconfig +@@ -235,4 +235,8 @@ source "drivers/interconnect/Kconfig" + source "drivers/counter/Kconfig" + + source "drivers/most/Kconfig" ++ ++source "drivers/rkflash/Kconfig" ++ ++source "drivers/rk_nand/Kconfig" + endmenu +diff --git a/drivers/Makefile b/drivers/Makefile +index 576228037718..18cc78f5385c 100644 +--- a/drivers/Makefile ++++ b/drivers/Makefile +@@ -189,3 +189,5 @@ obj-$(CONFIG_GNSS) += gnss/ + obj-$(CONFIG_INTERCONNECT) += interconnect/ + obj-$(CONFIG_COUNTER) += counter/ + obj-$(CONFIG_MOST) += most/ ++obj-$(CONFIG_RK_FLASH) += rkflash/ ++obj-$(CONFIG_RK_NAND) += rk_nand/ +diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c +index f7f1d9dbdc80..59c452fff835 100644 +--- a/drivers/block/nbd.c ++++ b/drivers/block/nbd.c +@@ -2313,12 +2313,6 @@ static int nbd_genl_status(struct sk_buff *skb, struct genl_info *info) + } + + dev_list = nla_nest_start_noflag(reply, NBD_ATTR_DEVICE_LIST); +- if (!dev_list) { +- nlmsg_free(reply); +- ret = -EMSGSIZE; +- goto out; +- } +- + if (index == -1) { + ret = idr_for_each(&nbd_index_idr, &status_cb, reply); + if (ret) { +diff --git a/drivers/clk/Kconfig b/drivers/clk/Kconfig +index c715d4681a0b..42bb63d80971 100644 +--- a/drivers/clk/Kconfig ++++ b/drivers/clk/Kconfig +@@ -38,6 +38,13 @@ menuconfig COMMON_CLK + + if COMMON_CLK + ++config COMMON_CLK_PROCFS ++ bool "Common Clock PROCFS interface" ++ depends on COMMON_CLK && PROC_FS && ARCH_ROCKCHIP ++ default n ++ help ++ Turns on the PROCFS interface for clock. ++ + config COMMON_CLK_WM831X + tristate "Clock driver for WM831x/2x PMICs" + depends on MFD_WM831X +diff --git a/drivers/clk/clk.c b/drivers/clk/clk.c +index 61c78714c095..5c825b523c6b 100644 +--- a/drivers/clk/clk.c ++++ b/drivers/clk/clk.c +@@ -1296,7 +1296,7 @@ static int __init clk_disable_unused(void) + + return 0; + } +-late_initcall_sync(clk_disable_unused); ++//late_initcall_sync(clk_disable_unused); + + static int clk_core_determine_round_nolock(struct clk_core *core, + struct clk_rate_request *req) +diff --git a/drivers/clk/rockchip/Kconfig b/drivers/clk/rockchip/Kconfig +index 47cd6c5de837..02c5df791017 100644 +--- a/drivers/clk/rockchip/Kconfig ++++ b/drivers/clk/rockchip/Kconfig +@@ -2,7 +2,7 @@ + # common clock support for ROCKCHIP SoC family. + + config COMMON_CLK_ROCKCHIP +- bool "Rockchip clock controller common support" ++ tristate "Rockchip clock controller common support" + depends on ARCH_ROCKCHIP + default ARCH_ROCKCHIP + help +@@ -10,69 +10,105 @@ config COMMON_CLK_ROCKCHIP + + if COMMON_CLK_ROCKCHIP + config CLK_PX30 +- bool "Rockchip PX30 clock controller support" ++ tristate "Rockchip PX30 clock controller support" ++ depends on ARM64 || COMPILE_TEST + default y + help + Build the driver for PX30 Clock Driver. + + config CLK_RV110X +- bool "Rockchip RV110x clock controller support" ++ tristate "Rockchip RV110x clock controller support" ++ depends on ARM || COMPILE_TEST + default y + help + Build the driver for RV110x Clock Driver. + ++config CLK_RV1126 ++ tristate "Rockchip RV1126 clock controller support" ++ depends on ARM || COMPILE_TEST ++ default y ++ help ++ Build the driver for RV1126 Clock Driver. ++ ++config CLK_RK1808 ++ tristate "Rockchip RK1808 clock controller support" ++ depends on ARM64 || COMPILE_TEST ++ default y ++ help ++ Build the driver for RK1808 Clock Driver. ++ + config CLK_RK3036 +- bool "Rockchip RK3036 clock controller support" ++ tristate "Rockchip RK3036 clock controller support" ++ depends on ARM || COMPILE_TEST + default y + help + Build the driver for RK3036 Clock Driver. + + config CLK_RK312X +- bool "Rockchip RK312x clock controller support" ++ tristate "Rockchip RK312x clock controller support" ++ depends on ARM || COMPILE_TEST + default y + help + Build the driver for RK312x Clock Driver. + + config CLK_RK3188 +- bool "Rockchip RK3188 clock controller support" ++ tristate "Rockchip RK3188 clock controller support" ++ depends on ARM || COMPILE_TEST + default y + help + Build the driver for RK3188 Clock Driver. + + config CLK_RK322X +- bool "Rockchip RK322x clock controller support" ++ tristate "Rockchip RK322x clock controller support" ++ depends on ARM || COMPILE_TEST + default y + help + Build the driver for RK322x Clock Driver. + + config CLK_RK3288 +- bool "Rockchip RK3288 clock controller support" +- depends on ARM ++ tristate "Rockchip RK3288 clock controller support" ++ depends on ARM || COMPILE_TEST + default y + help + Build the driver for RK3288 Clock Driver. + + config CLK_RK3308 +- bool "Rockchip RK3308 clock controller support" ++ tristate "Rockchip RK3308 clock controller support" ++ depends on ARM64 || COMPILE_TEST + default y + help + Build the driver for RK3308 Clock Driver. + + config CLK_RK3328 +- bool "Rockchip RK3328 clock controller support" ++ tristate "Rockchip RK3328 clock controller support" ++ depends on ARM64 || COMPILE_TEST + default y + help + Build the driver for RK3328 Clock Driver. + + config CLK_RK3368 +- bool "Rockchip RK3368 clock controller support" ++ tristate "Rockchip RK3368 clock controller support" ++ depends on ARM64 || COMPILE_TEST + default y + help + Build the driver for RK3368 Clock Driver. + + config CLK_RK3399 + tristate "Rockchip RK3399 clock controller support" ++ depends on ARM64 || COMPILE_TEST + default y + help + Build the driver for RK3399 Clock Driver. ++ ++config CLK_RK3568 ++ tristate "Rockchip RK3568 clock controller support" ++ depends on ARM64 || COMPILE_TEST ++ default y ++ help ++ Build the driver for RK3568 Clock Driver. ++ ++config ROCKCHIP_CLK_COMPENSATION ++ bool "Rockchip Clk Compensation" ++ help ++ Say y here to enable clk compensation(+/- 1000 ppm). + endif +diff --git a/drivers/clk/rockchip/Makefile b/drivers/clk/rockchip/Makefile +index a99e4d9bbae1..a4c718bf1126 100644 +--- a/drivers/clk/rockchip/Makefile ++++ b/drivers/clk/rockchip/Makefile +@@ -13,10 +13,14 @@ clk-rockchip-y += clk-inverter.o + clk-rockchip-y += clk-mmc-phase.o + clk-rockchip-y += clk-muxgrf.o + clk-rockchip-y += clk-ddr.o ++clk-rockchip-y += clk-dclk-divider.o ++clk-rockchip-y += clk-pvtm.o + clk-rockchip-$(CONFIG_RESET_CONTROLLER) += softrst.o + + obj-$(CONFIG_CLK_PX30) += clk-px30.o + obj-$(CONFIG_CLK_RV110X) += clk-rv1108.o ++obj-$(CONFIG_CLK_RV1126) += clk-rv1126.o ++obj-$(CONFIG_CLK_RK1808) += clk-rk1808.o + obj-$(CONFIG_CLK_RK3036) += clk-rk3036.o + obj-$(CONFIG_CLK_RK312X) += clk-rk3128.o + obj-$(CONFIG_CLK_RK3188) += clk-rk3188.o +@@ -26,3 +30,4 @@ obj-$(CONFIG_CLK_RK3308) += clk-rk3308.o + obj-$(CONFIG_CLK_RK3328) += clk-rk3328.o + obj-$(CONFIG_CLK_RK3368) += clk-rk3368.o + obj-$(CONFIG_CLK_RK3399) += clk-rk3399.o ++obj-$(CONFIG_CLK_RK3568) += clk-rk3568.o +diff --git a/drivers/clk/rockchip/clk-cpu.c b/drivers/clk/rockchip/clk-cpu.c +index 0dc478a19451..55416812bed2 100644 +--- a/drivers/clk/rockchip/clk-cpu.c ++++ b/drivers/clk/rockchip/clk-cpu.c +@@ -51,6 +51,7 @@ + */ + struct rockchip_cpuclk { + struct clk_hw hw; ++ struct clk_hw *pll_hw; + + struct clk_mux cpu_mux; + const struct clk_ops *cpu_mux_ops; +@@ -88,10 +89,10 @@ static unsigned long rockchip_cpuclk_recalc_rate(struct clk_hw *hw, + { + struct rockchip_cpuclk *cpuclk = to_rockchip_cpuclk_hw(hw); + const struct rockchip_cpuclk_reg_data *reg_data = cpuclk->reg_data; +- u32 clksel0 = readl_relaxed(cpuclk->reg_base + reg_data->core_reg); ++ u32 clksel0 = readl_relaxed(cpuclk->reg_base + reg_data->core_reg[0]); + +- clksel0 >>= reg_data->div_core_shift; +- clksel0 &= reg_data->div_core_mask; ++ clksel0 >>= reg_data->div_core_shift[0]; ++ clksel0 &= reg_data->div_core_mask[0]; + return parent_rate / (clksel0 + 1); + } + +@@ -124,6 +125,7 @@ static int rockchip_cpuclk_pre_rate_change(struct rockchip_cpuclk *cpuclk, + const struct rockchip_cpuclk_rate_table *rate; + unsigned long alt_prate, alt_div; + unsigned long flags; ++ int i = 0; + + /* check validity of the new rate */ + rate = rockchip_get_cpuclk_settings(cpuclk, ndata->new_rate); +@@ -133,6 +135,8 @@ static int rockchip_cpuclk_pre_rate_change(struct rockchip_cpuclk *cpuclk, + return -EINVAL; + } + ++ rockchip_boost_enable_recovery_sw_low(cpuclk->pll_hw); ++ + alt_prate = clk_get_rate(cpuclk->alt_parent); + + spin_lock_irqsave(cpuclk->lock, flags); +@@ -146,10 +150,10 @@ static int rockchip_cpuclk_pre_rate_change(struct rockchip_cpuclk *cpuclk, + if (alt_prate > ndata->old_rate) { + /* calculate dividers */ + alt_div = DIV_ROUND_UP(alt_prate, ndata->old_rate) - 1; +- if (alt_div > reg_data->div_core_mask) { ++ if (alt_div > reg_data->div_core_mask[0]) { + pr_warn("%s: limiting alt-divider %lu to %d\n", +- __func__, alt_div, reg_data->div_core_mask); +- alt_div = reg_data->div_core_mask; ++ __func__, alt_div, reg_data->div_core_mask[0]); ++ alt_div = reg_data->div_core_mask[0]; + } + + /* +@@ -162,20 +166,21 @@ static int rockchip_cpuclk_pre_rate_change(struct rockchip_cpuclk *cpuclk, + pr_debug("%s: setting div %lu as alt-rate %lu > old-rate %lu\n", + __func__, alt_div, alt_prate, ndata->old_rate); + +- writel(HIWORD_UPDATE(alt_div, reg_data->div_core_mask, +- reg_data->div_core_shift) | +- HIWORD_UPDATE(reg_data->mux_core_alt, +- reg_data->mux_core_mask, +- reg_data->mux_core_shift), +- cpuclk->reg_base + reg_data->core_reg); +- } else { +- /* select alternate parent */ +- writel(HIWORD_UPDATE(reg_data->mux_core_alt, +- reg_data->mux_core_mask, +- reg_data->mux_core_shift), +- cpuclk->reg_base + reg_data->core_reg); ++ for (i = 0; i < reg_data->num_cores; i++) { ++ writel(HIWORD_UPDATE(alt_div, reg_data->div_core_mask[i], ++ reg_data->div_core_shift[i]), ++ cpuclk->reg_base + reg_data->core_reg[i]); ++ } + } + ++ rockchip_boost_add_core_div(cpuclk->pll_hw, alt_prate); ++ ++ /* select alternate parent */ ++ writel(HIWORD_UPDATE(reg_data->mux_core_alt, ++ reg_data->mux_core_mask, ++ reg_data->mux_core_shift), ++ cpuclk->reg_base + reg_data->core_reg[0]); ++ + spin_unlock_irqrestore(cpuclk->lock, flags); + return 0; + } +@@ -186,6 +191,7 @@ static int rockchip_cpuclk_post_rate_change(struct rockchip_cpuclk *cpuclk, + const struct rockchip_cpuclk_reg_data *reg_data = cpuclk->reg_data; + const struct rockchip_cpuclk_rate_table *rate; + unsigned long flags; ++ int i = 0; + + rate = rockchip_get_cpuclk_settings(cpuclk, ndata->new_rate); + if (!rate) { +@@ -206,16 +212,23 @@ static int rockchip_cpuclk_post_rate_change(struct rockchip_cpuclk *cpuclk, + * primary parent by the extra dividers that were needed for the alt. + */ + +- writel(HIWORD_UPDATE(0, reg_data->div_core_mask, +- reg_data->div_core_shift) | +- HIWORD_UPDATE(reg_data->mux_core_main, +- reg_data->mux_core_mask, +- reg_data->mux_core_shift), +- cpuclk->reg_base + reg_data->core_reg); ++ writel(HIWORD_UPDATE(reg_data->mux_core_main, ++ reg_data->mux_core_mask, ++ reg_data->mux_core_shift), ++ cpuclk->reg_base + reg_data->core_reg[0]); ++ ++ /* remove dividers */ ++ for (i = 0; i < reg_data->num_cores; i++) { ++ writel(HIWORD_UPDATE(0, reg_data->div_core_mask[i], ++ reg_data->div_core_shift[i]), ++ cpuclk->reg_base + reg_data->core_reg[i]); ++ } + + if (ndata->old_rate > ndata->new_rate) + rockchip_cpuclk_set_dividers(cpuclk, rate); + ++ rockchip_boost_disable_recovery_sw(cpuclk->pll_hw); ++ + spin_unlock_irqrestore(cpuclk->lock, flags); + return 0; + } +@@ -244,14 +257,16 @@ static int rockchip_cpuclk_notifier_cb(struct notifier_block *nb, + } + + struct clk *rockchip_clk_register_cpuclk(const char *name, +- const char *const *parent_names, u8 num_parents, ++ u8 num_parents, ++ struct clk *parent, struct clk *alt_parent, + const struct rockchip_cpuclk_reg_data *reg_data, + const struct rockchip_cpuclk_rate_table *rates, + int nrates, void __iomem *reg_base, spinlock_t *lock) + { + struct rockchip_cpuclk *cpuclk; + struct clk_init_data init; +- struct clk *clk, *cclk; ++ struct clk *clk, *cclk, *pll_clk; ++ const char *parent_name; + int ret; + + if (num_parents < 2) { +@@ -259,12 +274,18 @@ struct clk *rockchip_clk_register_cpuclk(const char *name, + return ERR_PTR(-EINVAL); + } + ++ if (IS_ERR(parent) || IS_ERR(alt_parent)) { ++ pr_err("%s: invalid parent clock(s)\n", __func__); ++ return ERR_PTR(-EINVAL); ++ } ++ + cpuclk = kzalloc(sizeof(*cpuclk), GFP_KERNEL); + if (!cpuclk) + return ERR_PTR(-ENOMEM); + ++ parent_name = clk_hw_get_name(__clk_get_hw(parent)); + init.name = name; +- init.parent_names = &parent_names[reg_data->mux_core_main]; ++ init.parent_names = &parent_name; + init.num_parents = 1; + init.ops = &rockchip_cpuclk_ops; + +@@ -281,8 +302,19 @@ struct clk *rockchip_clk_register_cpuclk(const char *name, + cpuclk->reg_data = reg_data; + cpuclk->clk_nb.notifier_call = rockchip_cpuclk_notifier_cb; + cpuclk->hw.init = &init; ++ if (reg_data->pll_name) { ++ pll_clk = clk_get_parent(parent); ++ if (!pll_clk) { ++ pr_err("%s: could not lookup pll clock: (%s)\n", ++ __func__, reg_data->pll_name); ++ ret = -EINVAL; ++ goto free_cpuclk; ++ } ++ cpuclk->pll_hw = __clk_get_hw(pll_clk); ++ rockchip_boost_init(cpuclk->pll_hw); ++ } + +- cpuclk->alt_parent = __clk_lookup(parent_names[reg_data->mux_core_alt]); ++ cpuclk->alt_parent = alt_parent; + if (!cpuclk->alt_parent) { + pr_err("%s: could not lookup alternate parent: (%d)\n", + __func__, reg_data->mux_core_alt); +@@ -297,11 +329,11 @@ struct clk *rockchip_clk_register_cpuclk(const char *name, + goto free_cpuclk; + } + +- clk = __clk_lookup(parent_names[reg_data->mux_core_main]); ++ clk = parent; + if (!clk) { + pr_err("%s: could not lookup parent clock: (%d) %s\n", + __func__, reg_data->mux_core_main, +- parent_names[reg_data->mux_core_main]); ++ parent_name); + ret = -EINVAL; + goto free_alt_parent; + } +diff --git a/drivers/clk/rockchip/clk-dclk-divider.c b/drivers/clk/rockchip/clk-dclk-divider.c +new file mode 100755 +index 000000000000..77c35b42207e +--- /dev/null ++++ b/drivers/clk/rockchip/clk-dclk-divider.c +@@ -0,0 +1,168 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include "clk.h" ++ ++#define div_mask(width) ((1 << (width)) - 1) ++ ++static unsigned long clk_dclk_recalc_rate(struct clk_hw *hw, ++ unsigned long parent_rate) ++{ ++ struct clk_divider *divider = to_clk_divider(hw); ++ unsigned int val; ++ ++ val = readl(divider->reg) >> divider->shift; ++ val &= div_mask(divider->width); ++ ++ return DIV_ROUND_UP_ULL(((u64)parent_rate), val + 1); ++} ++ ++static long clk_dclk_round_rate(struct clk_hw *hw, unsigned long rate, ++ unsigned long *prate) ++{ ++ struct clk_divider *divider = to_clk_divider(hw); ++ int div, maxdiv = div_mask(divider->width) + 1; ++ ++ div = DIV_ROUND_UP_ULL(divider->max_prate, rate); ++ if (div % 2) ++ div = __rounddown_pow_of_two(div); ++ div = div > maxdiv ? maxdiv : div; ++ *prate = div * rate; ++ return rate; ++} ++ ++static int clk_dclk_set_rate(struct clk_hw *hw, unsigned long rate, ++ unsigned long parent_rate) ++{ ++ struct clk_divider *divider = to_clk_divider(hw); ++ unsigned int value; ++ unsigned long flags = 0; ++ u32 val; ++ ++ value = divider_get_val(rate, parent_rate, divider->table, ++ divider->width, divider->flags); ++ ++ if (divider->lock) ++ spin_lock_irqsave(divider->lock, flags); ++ else ++ __acquire(divider->lock); ++ ++ if (divider->flags & CLK_DIVIDER_HIWORD_MASK) { ++ val = div_mask(divider->width) << (divider->shift + 16); ++ } else { ++ val = readl(divider->reg); ++ val &= ~(div_mask(divider->width) << divider->shift); ++ } ++ val |= value << divider->shift; ++ writel(val, divider->reg); ++ ++ if (divider->lock) ++ spin_unlock_irqrestore(divider->lock, flags); ++ else ++ __release(divider->lock); ++ ++ return 0; ++} ++ ++const struct clk_ops clk_dclk_divider_ops = { ++ .recalc_rate = clk_dclk_recalc_rate, ++ .round_rate = clk_dclk_round_rate, ++ .set_rate = clk_dclk_set_rate, ++}; ++EXPORT_SYMBOL_GPL(clk_dclk_divider_ops); ++ ++/** ++ * Register a clock branch. ++ * Most clock branches have a form like ++ * ++ * src1 --|--\ ++ * |M |--[GATE]-[DIV]- ++ * src2 --|--/ ++ * ++ * sometimes without one of those components. ++ */ ++struct clk *rockchip_clk_register_dclk_branch(const char *name, ++ const char *const *parent_names, ++ u8 num_parents, ++ void __iomem *base, ++ int muxdiv_offset, u8 mux_shift, ++ u8 mux_width, u8 mux_flags, ++ int div_offset, u8 div_shift, ++ u8 div_width, u8 div_flags, ++ struct clk_div_table *div_table, ++ int gate_offset, ++ u8 gate_shift, u8 gate_flags, ++ unsigned long flags, ++ unsigned long max_prate, ++ spinlock_t *lock) ++{ ++ struct clk *clk; ++ struct clk_mux *mux = NULL; ++ struct clk_gate *gate = NULL; ++ struct clk_divider *div = NULL; ++ const struct clk_ops *mux_ops = NULL, *div_ops = NULL, ++ *gate_ops = NULL; ++ ++ if (num_parents > 1) { ++ mux = kzalloc(sizeof(*mux), GFP_KERNEL); ++ if (!mux) ++ return ERR_PTR(-ENOMEM); ++ ++ mux->reg = base + muxdiv_offset; ++ mux->shift = mux_shift; ++ mux->mask = BIT(mux_width) - 1; ++ mux->flags = mux_flags; ++ mux->lock = lock; ++ mux_ops = (mux_flags & CLK_MUX_READ_ONLY) ? &clk_mux_ro_ops ++ : &clk_mux_ops; ++ } ++ ++ if (gate_offset >= 0) { ++ gate = kzalloc(sizeof(*gate), GFP_KERNEL); ++ if (!gate) ++ goto err_gate; ++ ++ gate->flags = gate_flags; ++ gate->reg = base + gate_offset; ++ gate->bit_idx = gate_shift; ++ gate->lock = lock; ++ gate_ops = &clk_gate_ops; ++ } ++ ++ if (div_width > 0) { ++ div = kzalloc(sizeof(*div), GFP_KERNEL); ++ if (!div) ++ goto err_div; ++ ++ div->flags = div_flags; ++ if (div_offset) ++ div->reg = base + div_offset; ++ else ++ div->reg = base + muxdiv_offset; ++ div->shift = div_shift; ++ div->width = div_width; ++ div->lock = lock; ++ div->max_prate = max_prate; ++ div_ops = &clk_dclk_divider_ops; ++ } ++ ++ clk = clk_register_composite(NULL, name, parent_names, num_parents, ++ mux ? &mux->hw : NULL, mux_ops, ++ div ? &div->hw : NULL, div_ops, ++ gate ? &gate->hw : NULL, gate_ops, ++ flags); ++ ++ return clk; ++err_div: ++ kfree(gate); ++err_gate: ++ kfree(mux); ++ return ERR_PTR(-ENOMEM); ++} +diff --git a/drivers/clk/rockchip/clk-ddr.c b/drivers/clk/rockchip/clk-ddr.c +index 86718c54e56b..3c8bcbee2048 100644 +--- a/drivers/clk/rockchip/clk-ddr.c ++++ b/drivers/clk/rockchip/clk-ddr.c +@@ -8,10 +8,20 @@ + #include + #include + #include ++#include ++#include + #include + #include ++#include ++#include ++#ifdef CONFIG_ARM ++#include ++#endif ++ + #include "clk.h" + ++#define MHZ (1000000) ++ + struct rockchip_ddrclk { + struct clk_hw hw; + void __iomem *reg_base; +@@ -21,25 +31,47 @@ struct rockchip_ddrclk { + int div_shift; + int div_width; + int ddr_flag; +- spinlock_t *lock; + }; + + #define to_rockchip_ddrclk_hw(hw) container_of(hw, struct rockchip_ddrclk, hw) + ++struct share_params_ddrclk { ++ u32 hz; ++ u32 lcdc_type; ++}; ++ ++struct rockchip_ddrclk_data { ++ void __iomem *params; ++ int (*dmcfreq_wait_complete)(void); ++}; ++ ++static struct rockchip_ddrclk_data ddr_data = {NULL, NULL}; ++ ++void rockchip_set_ddrclk_params(void __iomem *params) ++{ ++ ddr_data.params = params; ++} ++EXPORT_SYMBOL(rockchip_set_ddrclk_params); ++ ++void rockchip_set_ddrclk_dmcfreq_wait_complete(int (*func)(void)) ++{ ++ ddr_data.dmcfreq_wait_complete = func; ++} ++EXPORT_SYMBOL(rockchip_set_ddrclk_dmcfreq_wait_complete); ++ + static int rockchip_ddrclk_sip_set_rate(struct clk_hw *hw, unsigned long drate, + unsigned long prate) + { +- struct rockchip_ddrclk *ddrclk = to_rockchip_ddrclk_hw(hw); +- unsigned long flags; + struct arm_smccc_res res; + +- spin_lock_irqsave(ddrclk->lock, flags); + arm_smccc_smc(ROCKCHIP_SIP_DRAM_FREQ, drate, 0, + ROCKCHIP_SIP_CONFIG_DRAM_SET_RATE, + 0, 0, 0, 0, &res); +- spin_unlock_irqrestore(ddrclk->lock, flags); + +- return res.a0; ++ if (res.a0) ++ return 0; ++ else ++ return -EPERM; + } + + static unsigned long +@@ -87,18 +119,134 @@ static const struct clk_ops rockchip_ddrclk_sip_ops = { + .get_parent = rockchip_ddrclk_get_parent, + }; + ++static u32 ddr_clk_cached; ++ ++static int rockchip_ddrclk_scpi_set_rate(struct clk_hw *hw, unsigned long drate, ++ unsigned long prate) ++{ ++ u32 ret; ++ u32 lcdc_type = 0; ++ struct share_params_ddrclk *p; ++ ++ p = (struct share_params_ddrclk *)ddr_data.params; ++ if (p) ++ lcdc_type = p->lcdc_type; ++ ++ ret = scpi_ddr_set_clk_rate(drate / MHZ, lcdc_type); ++ if (ret) { ++ ddr_clk_cached = ret; ++ ret = 0; ++ } else { ++ ddr_clk_cached = 0; ++ ret = -1; ++ } ++ ++ return ret; ++} ++ ++static unsigned long rockchip_ddrclk_scpi_recalc_rate(struct clk_hw *hw, ++ unsigned long parent_rate) ++{ ++ if (ddr_clk_cached) ++ return (MHZ * ddr_clk_cached); ++ else ++ return (MHZ * scpi_ddr_get_clk_rate()); ++} ++ ++static long rockchip_ddrclk_scpi_round_rate(struct clk_hw *hw, ++ unsigned long rate, ++ unsigned long *prate) ++{ ++ rate = rate / MHZ; ++ rate = (rate / 12) * 12; ++ ++ return (rate * MHZ); ++} ++ ++static const struct clk_ops rockchip_ddrclk_scpi_ops = { ++ .recalc_rate = rockchip_ddrclk_scpi_recalc_rate, ++ .set_rate = rockchip_ddrclk_scpi_set_rate, ++ .round_rate = rockchip_ddrclk_scpi_round_rate, ++ .get_parent = rockchip_ddrclk_get_parent, ++}; ++ ++static int rockchip_ddrclk_sip_set_rate_v2(struct clk_hw *hw, ++ unsigned long drate, ++ unsigned long prate) ++{ ++ struct share_params_ddrclk *p; ++ struct arm_smccc_res res; ++ ++ p = (struct share_params_ddrclk *)ddr_data.params; ++ if (p) ++ p->hz = drate; ++ ++ res = sip_smc_dram(SHARE_PAGE_TYPE_DDR, 0, ++ ROCKCHIP_SIP_CONFIG_DRAM_SET_RATE); ++ ++ if ((int)res.a1 == SIP_RET_SET_RATE_TIMEOUT) { ++ if (ddr_data.dmcfreq_wait_complete) ++ ddr_data.dmcfreq_wait_complete(); ++ } ++ ++ return res.a0; ++} ++ ++static unsigned long rockchip_ddrclk_sip_recalc_rate_v2 ++ (struct clk_hw *hw, unsigned long parent_rate) ++{ ++ struct arm_smccc_res res; ++ ++ res = sip_smc_dram(SHARE_PAGE_TYPE_DDR, 0, ++ ROCKCHIP_SIP_CONFIG_DRAM_GET_RATE); ++ if (!res.a0) ++ return res.a1; ++ else ++ return 0; ++} ++ ++static long rockchip_ddrclk_sip_round_rate_v2(struct clk_hw *hw, ++ unsigned long rate, ++ unsigned long *prate) ++{ ++ struct share_params_ddrclk *p; ++ struct arm_smccc_res res; ++ ++ p = (struct share_params_ddrclk *)ddr_data.params; ++ if (p) ++ p->hz = rate; ++ ++ res = sip_smc_dram(SHARE_PAGE_TYPE_DDR, 0, ++ ROCKCHIP_SIP_CONFIG_DRAM_ROUND_RATE); ++ if (!res.a0) ++ return res.a1; ++ else ++ return 0; ++} ++ ++static const struct clk_ops rockchip_ddrclk_sip_ops_v2 = { ++ .recalc_rate = rockchip_ddrclk_sip_recalc_rate_v2, ++ .set_rate = rockchip_ddrclk_sip_set_rate_v2, ++ .round_rate = rockchip_ddrclk_sip_round_rate_v2, ++ .get_parent = rockchip_ddrclk_get_parent, ++}; ++ + struct clk *rockchip_clk_register_ddrclk(const char *name, int flags, + const char *const *parent_names, + u8 num_parents, int mux_offset, + int mux_shift, int mux_width, + int div_shift, int div_width, +- int ddr_flag, void __iomem *reg_base, +- spinlock_t *lock) ++ int ddr_flag, void __iomem *reg_base) + { + struct rockchip_ddrclk *ddrclk; + struct clk_init_data init; + struct clk *clk; + ++#ifdef CONFIG_ARM ++ if (!psci_smp_available()) ++ return NULL; ++#endif ++ + ddrclk = kzalloc(sizeof(*ddrclk), GFP_KERNEL); + if (!ddrclk) + return ERR_PTR(-ENOMEM); +@@ -114,6 +262,12 @@ struct clk *rockchip_clk_register_ddrclk(const char *name, int flags, + case ROCKCHIP_DDRCLK_SIP: + init.ops = &rockchip_ddrclk_sip_ops; + break; ++ case ROCKCHIP_DDRCLK_SCPI: ++ init.ops = &rockchip_ddrclk_scpi_ops; ++ break; ++ case ROCKCHIP_DDRCLK_SIP_V2: ++ init.ops = &rockchip_ddrclk_sip_ops_v2; ++ break; + default: + pr_err("%s: unsupported ddrclk type %d\n", __func__, ddr_flag); + kfree(ddrclk); +@@ -121,7 +275,6 @@ struct clk *rockchip_clk_register_ddrclk(const char *name, int flags, + } + + ddrclk->reg_base = reg_base; +- ddrclk->lock = lock; + ddrclk->hw.init = &init; + ddrclk->mux_offset = mux_offset; + ddrclk->mux_shift = mux_shift; +diff --git a/drivers/clk/rockchip/clk-half-divider.c b/drivers/clk/rockchip/clk-half-divider.c +index ccd5c270c213..b978af08d84f 100644 +--- a/drivers/clk/rockchip/clk-half-divider.c ++++ b/drivers/clk/rockchip/clk-half-divider.c +@@ -14,9 +14,9 @@ static bool _is_best_half_div(unsigned long rate, unsigned long now, + unsigned long best, unsigned long flags) + { + if (flags & CLK_DIVIDER_ROUND_CLOSEST) +- return abs(rate - now) < abs(rate - best); ++ return abs(rate - now) <= abs(rate - best); + +- return now <= rate && now > best; ++ return now <= rate && now >= best; + } + + static unsigned long clk_half_divider_recalc_rate(struct clk_hw *hw, +@@ -38,7 +38,7 @@ static int clk_half_divider_bestdiv(struct clk_hw *hw, unsigned long rate, + { + unsigned int i, bestdiv = 0; + unsigned long parent_rate, best = 0, now, maxdiv; +- unsigned long parent_rate_saved = *best_parent_rate; ++ bool is_bestdiv = false; + + if (!rate) + rate = 1; +@@ -51,7 +51,7 @@ static int clk_half_divider_bestdiv(struct clk_hw *hw, unsigned long rate, + if (bestdiv < 3) + bestdiv = 0; + else +- bestdiv = (bestdiv - 3) / 2; ++ bestdiv = DIV_ROUND_UP(bestdiv - 3, 2); + bestdiv = bestdiv > maxdiv ? maxdiv : bestdiv; + return bestdiv; + } +@@ -63,28 +63,20 @@ static int clk_half_divider_bestdiv(struct clk_hw *hw, unsigned long rate, + maxdiv = min(ULONG_MAX / rate, maxdiv); + + for (i = 0; i <= maxdiv; i++) { +- if (((u64)rate * (i * 2 + 3)) == ((u64)parent_rate_saved * 2)) { +- /* +- * It's the most ideal case if the requested rate can be +- * divided from parent clock without needing to change +- * parent rate, so return the divider immediately. +- */ +- *best_parent_rate = parent_rate_saved; +- return i; +- } + parent_rate = clk_hw_round_rate(clk_hw_get_parent(hw), + ((u64)rate * (i * 2 + 3)) / 2); + now = DIV_ROUND_UP_ULL(((u64)parent_rate * 2), + (i * 2 + 3)); + + if (_is_best_half_div(rate, now, best, flags)) { ++ is_bestdiv = true; + bestdiv = i; + best = now; + *best_parent_rate = parent_rate; + } + } + +- if (!bestdiv) { ++ if (!is_bestdiv) { + bestdiv = div_mask(width); + *best_parent_rate = clk_hw_round_rate(clk_hw_get_parent(hw), 1); + } +@@ -114,7 +106,7 @@ static int clk_half_divider_set_rate(struct clk_hw *hw, unsigned long rate, + u32 val; + + value = DIV_ROUND_UP_ULL(((u64)parent_rate * 2), rate); +- value = (value - 3) / 2; ++ value = DIV_ROUND_UP(value - 3, 2); + value = min_t(unsigned int, value, div_mask(divider->width)); + + if (divider->lock) +@@ -160,10 +152,10 @@ struct clk *rockchip_clk_register_halfdiv(const char *name, + u8 num_parents, void __iomem *base, + int muxdiv_offset, u8 mux_shift, + u8 mux_width, u8 mux_flags, +- u8 div_shift, u8 div_width, +- u8 div_flags, int gate_offset, +- u8 gate_shift, u8 gate_flags, +- unsigned long flags, ++ int div_offset, u8 div_shift, ++ u8 div_width, u8 div_flags, ++ int gate_offset, u8 gate_shift, ++ u8 gate_flags, unsigned long flags, + spinlock_t *lock) + { + struct clk_hw *hw = ERR_PTR(-ENOMEM); +@@ -205,7 +197,10 @@ struct clk *rockchip_clk_register_halfdiv(const char *name, + goto err_div; + + div->flags = div_flags; +- div->reg = base + muxdiv_offset; ++ if (div_offset) ++ div->reg = base + div_offset; ++ else ++ div->reg = base + muxdiv_offset; + div->shift = div_shift; + div->width = div_width; + div->lock = lock; +diff --git a/drivers/clk/rockchip/clk-pll.c b/drivers/clk/rockchip/clk-pll.c +index bbbf9ce42867..f4946461609d 100644 +--- a/drivers/clk/rockchip/clk-pll.c ++++ b/drivers/clk/rockchip/clk-pll.c +@@ -15,6 +15,9 @@ + #include + #include + #include ++#include ++#include ++#include + #include "clk.h" + + #define PLL_MODE_MASK 0x3 +@@ -38,15 +41,291 @@ struct rockchip_clk_pll { + u8 flags; + const struct rockchip_pll_rate_table *rate_table; + unsigned int rate_count; ++ int sel; ++ unsigned long scaling; + spinlock_t *lock; + + struct rockchip_clk_provider *ctx; ++ ++ bool boost_enabled; ++ u32 boost_backup_pll_usage; ++ unsigned long boost_backup_pll_rate; ++ unsigned long boost_low_rate; ++ unsigned long boost_high_rate; ++ struct regmap *boost; ++#ifdef CONFIG_DEBUG_FS ++ struct hlist_node debug_node; ++#endif + }; + + #define to_rockchip_clk_pll(_hw) container_of(_hw, struct rockchip_clk_pll, hw) + #define to_rockchip_clk_pll_nb(nb) \ + container_of(nb, struct rockchip_clk_pll, clk_nb) + ++static void rockchip_boost_disable_low(struct rockchip_clk_pll *pll); ++ ++#define MHZ (1000UL * 1000UL) ++#define KHZ (1000UL) ++ ++/* CLK_PLL_TYPE_RK3066_AUTO type ops */ ++#define PLL_FREF_MIN (269 * KHZ) ++#define PLL_FREF_MAX (2200 * MHZ) ++ ++#define PLL_FVCO_MIN (440 * MHZ) ++#define PLL_FVCO_MAX (2200 * MHZ) ++ ++#define PLL_FOUT_MIN (27500 * KHZ) ++#define PLL_FOUT_MAX (2200 * MHZ) ++ ++#define PLL_NF_MAX (4096) ++#define PLL_NR_MAX (64) ++#define PLL_NO_MAX (16) ++ ++/* CLK_PLL_TYPE_RK3036/3366/3399_AUTO type ops */ ++#define MIN_FOUTVCO_FREQ (800 * MHZ) ++#define MAX_FOUTVCO_FREQ (2000 * MHZ) ++ ++static struct rockchip_pll_rate_table auto_table; ++#ifdef CONFIG_DEBUG_FS ++static HLIST_HEAD(clk_boost_list); ++static DEFINE_MUTEX(clk_boost_lock); ++#endif ++ ++int rockchip_pll_clk_adaptive_scaling(struct clk *clk, int sel) ++{ ++ struct clk *parent = clk_get_parent(clk); ++ struct rockchip_clk_pll *pll; ++ ++ if (IS_ERR_OR_NULL(parent)) ++ return -EINVAL; ++ ++ pll = to_rockchip_clk_pll(__clk_get_hw(parent)); ++ if (!pll) ++ return -EINVAL; ++ ++ pll->sel = sel; ++ ++ return 0; ++} ++EXPORT_SYMBOL(rockchip_pll_clk_adaptive_scaling); ++ ++int rockchip_pll_clk_rate_to_scale(struct clk *clk, unsigned long rate) ++{ ++ const struct rockchip_pll_rate_table *rate_table; ++ struct clk *parent = clk_get_parent(clk); ++ struct rockchip_clk_pll *pll; ++ unsigned int i; ++ ++ if (IS_ERR_OR_NULL(parent)) ++ return -EINVAL; ++ ++ pll = to_rockchip_clk_pll(__clk_get_hw(parent)); ++ if (!pll) ++ return -EINVAL; ++ ++ rate_table = pll->rate_table; ++ for (i = 0; i < pll->rate_count; i++) { ++ if (rate >= rate_table[i].rate) ++ return i; ++ } ++ ++ return -EINVAL; ++} ++EXPORT_SYMBOL(rockchip_pll_clk_rate_to_scale); ++ ++int rockchip_pll_clk_scale_to_rate(struct clk *clk, unsigned int scale) ++{ ++ const struct rockchip_pll_rate_table *rate_table; ++ struct clk *parent = clk_get_parent(clk); ++ struct rockchip_clk_pll *pll; ++ unsigned int i; ++ ++ if (IS_ERR_OR_NULL(parent)) ++ return -EINVAL; ++ ++ pll = to_rockchip_clk_pll(__clk_get_hw(parent)); ++ if (!pll) ++ return -EINVAL; ++ ++ rate_table = pll->rate_table; ++ for (i = 0; i < pll->rate_count; i++) { ++ if (i == scale) ++ return rate_table[i].rate; ++ } ++ ++ return -EINVAL; ++} ++EXPORT_SYMBOL(rockchip_pll_clk_scale_to_rate); ++ ++static struct rockchip_pll_rate_table *rk_pll_rate_table_get(void) ++{ ++ return &auto_table; ++} ++ ++static int rockchip_pll_clk_set_postdiv(unsigned long fout_hz, ++ u32 *postdiv1, ++ u32 *postdiv2, ++ u32 *foutvco) ++{ ++ unsigned long freq; ++ ++ if (fout_hz < MIN_FOUTVCO_FREQ) { ++ for (*postdiv1 = 1; *postdiv1 <= 7; (*postdiv1)++) { ++ for (*postdiv2 = 1; *postdiv2 <= 7; (*postdiv2)++) { ++ freq = fout_hz * (*postdiv1) * (*postdiv2); ++ if (freq >= MIN_FOUTVCO_FREQ && ++ freq <= MAX_FOUTVCO_FREQ) { ++ *foutvco = freq; ++ return 0; ++ } ++ } ++ } ++ pr_err("CANNOT FIND postdiv1/2 to make fout in range from 800M to 2000M,fout = %lu\n", ++ fout_hz); ++ } else { ++ *postdiv1 = 1; ++ *postdiv2 = 1; ++ } ++ return 0; ++} ++ ++static struct rockchip_pll_rate_table * ++rockchip_pll_clk_set_by_auto(struct rockchip_clk_pll *pll, ++ unsigned long fin_hz, ++ unsigned long fout_hz) ++{ ++ struct rockchip_pll_rate_table *rate_table = rk_pll_rate_table_get(); ++ /* FIXME set postdiv1/2 always 1*/ ++ u32 foutvco = fout_hz; ++ u64 fin_64, frac_64; ++ u32 f_frac, postdiv1, postdiv2; ++ unsigned long clk_gcd = 0; ++ ++ if (fin_hz == 0 || fout_hz == 0 || fout_hz == fin_hz) ++ return NULL; ++ ++ rockchip_pll_clk_set_postdiv(fout_hz, &postdiv1, &postdiv2, &foutvco); ++ rate_table->postdiv1 = postdiv1; ++ rate_table->postdiv2 = postdiv2; ++ rate_table->dsmpd = 1; ++ ++ if (fin_hz / MHZ * MHZ == fin_hz && fout_hz / MHZ * MHZ == fout_hz) { ++ fin_hz /= MHZ; ++ foutvco /= MHZ; ++ clk_gcd = gcd(fin_hz, foutvco); ++ rate_table->refdiv = fin_hz / clk_gcd; ++ rate_table->fbdiv = foutvco / clk_gcd; ++ ++ rate_table->frac = 0; ++ ++ pr_debug("fin = %lu, fout = %lu, clk_gcd = %lu, refdiv = %u, fbdiv = %u, postdiv1 = %u, postdiv2 = %u, frac = %u\n", ++ fin_hz, fout_hz, clk_gcd, rate_table->refdiv, ++ rate_table->fbdiv, rate_table->postdiv1, ++ rate_table->postdiv2, rate_table->frac); ++ } else { ++ pr_debug("frac div running, fin_hz = %lu, fout_hz = %lu, fin_INT_mhz = %lu, fout_INT_mhz = %lu\n", ++ fin_hz, fout_hz, ++ fin_hz / MHZ * MHZ, ++ fout_hz / MHZ * MHZ); ++ pr_debug("frac get postdiv1 = %u, postdiv2 = %u, foutvco = %u\n", ++ rate_table->postdiv1, rate_table->postdiv2, foutvco); ++ clk_gcd = gcd(fin_hz / MHZ, foutvco / MHZ); ++ rate_table->refdiv = fin_hz / MHZ / clk_gcd; ++ rate_table->fbdiv = foutvco / MHZ / clk_gcd; ++ pr_debug("frac get refdiv = %u, fbdiv = %u\n", ++ rate_table->refdiv, rate_table->fbdiv); ++ ++ rate_table->frac = 0; ++ ++ f_frac = (foutvco % MHZ); ++ fin_64 = fin_hz; ++ do_div(fin_64, (u64)rate_table->refdiv); ++ frac_64 = (u64)f_frac << 24; ++ do_div(frac_64, fin_64); ++ rate_table->frac = (u32)frac_64; ++ if (rate_table->frac > 0) ++ rate_table->dsmpd = 0; ++ pr_debug("frac = %x\n", rate_table->frac); ++ } ++ return rate_table; ++} ++ ++static struct rockchip_pll_rate_table * ++rockchip_rk3066_pll_clk_set_by_auto(struct rockchip_clk_pll *pll, ++ unsigned long fin_hz, ++ unsigned long fout_hz) ++{ ++ struct rockchip_pll_rate_table *rate_table = rk_pll_rate_table_get(); ++ u32 nr, nf, no, nonr; ++ u32 nr_out, nf_out, no_out; ++ u32 n; ++ u32 numerator, denominator; ++ u64 fref, fvco, fout; ++ unsigned long clk_gcd = 0; ++ ++ nr_out = PLL_NR_MAX + 1; ++ no_out = 0; ++ nf_out = 0; ++ ++ if (fin_hz == 0 || fout_hz == 0 || fout_hz == fin_hz) ++ return NULL; ++ ++ clk_gcd = gcd(fin_hz, fout_hz); ++ ++ numerator = fout_hz / clk_gcd; ++ denominator = fin_hz / clk_gcd; ++ ++ for (n = 1;; n++) { ++ nf = numerator * n; ++ nonr = denominator * n; ++ if (nf > PLL_NF_MAX || nonr > (PLL_NO_MAX * PLL_NR_MAX)) ++ break; ++ ++ for (no = 1; no <= PLL_NO_MAX; no++) { ++ if (!(no == 1 || !(no % 2))) ++ continue; ++ ++ if (nonr % no) ++ continue; ++ nr = nonr / no; ++ ++ if (nr > PLL_NR_MAX) ++ continue; ++ ++ fref = fin_hz / nr; ++ if (fref < PLL_FREF_MIN || fref > PLL_FREF_MAX) ++ continue; ++ ++ fvco = fref * nf; ++ if (fvco < PLL_FVCO_MIN || fvco > PLL_FVCO_MAX) ++ continue; ++ ++ fout = fvco / no; ++ if (fout < PLL_FOUT_MIN || fout > PLL_FOUT_MAX) ++ continue; ++ ++ /* select the best from all available PLL settings */ ++ if ((no > no_out) || ++ ((no == no_out) && (nr < nr_out))) { ++ nr_out = nr; ++ nf_out = nf; ++ no_out = no; ++ } ++ } ++ } ++ ++ /* output the best PLL setting */ ++ if ((nr_out <= PLL_NR_MAX) && (no_out > 0)) { ++ rate_table->nr = nr_out; ++ rate_table->nf = nf_out; ++ rate_table->no = no_out; ++ } else { ++ return NULL; ++ } ++ ++ return rate_table; ++} ++ + static const struct rockchip_pll_rate_table *rockchip_get_pll_settings( + struct rockchip_clk_pll *pll, unsigned long rate) + { +@@ -54,28 +333,27 @@ static const struct rockchip_pll_rate_table *rockchip_get_pll_settings( + int i; + + for (i = 0; i < pll->rate_count; i++) { +- if (rate == rate_table[i].rate) ++ if (rate == rate_table[i].rate) { ++ if (i < pll->sel) { ++ pll->scaling = rate; ++ return &rate_table[pll->sel]; ++ } ++ pll->scaling = 0; + return &rate_table[i]; ++ } + } ++ pll->scaling = 0; + +- return NULL; ++ if (pll->type == pll_rk3066) ++ return rockchip_rk3066_pll_clk_set_by_auto(pll, 24 * MHZ, rate); ++ else ++ return rockchip_pll_clk_set_by_auto(pll, 24 * MHZ, rate); + } + + static long rockchip_pll_round_rate(struct clk_hw *hw, + unsigned long drate, unsigned long *prate) + { +- struct rockchip_clk_pll *pll = to_rockchip_clk_pll(hw); +- const struct rockchip_pll_rate_table *rate_table = pll->rate_table; +- int i; +- +- /* Assumming rate_table is in descending order */ +- for (i = 0; i < pll->rate_count; i++) { +- if (drate >= rate_table[i].rate) +- return rate_table[i].rate; +- } +- +- /* return minimum supported value */ +- return rate_table[i - 1].rate; ++ return drate; + } + + /* +@@ -136,6 +414,30 @@ static int rockchip_rk3036_pll_wait_lock(struct rockchip_clk_pll *pll) + return ret; + } + ++static unsigned long ++rockchip_rk3036_pll_con_to_rate(struct rockchip_clk_pll *pll, ++ u32 con0, u32 con1) ++{ ++ unsigned int fbdiv, postdiv1, refdiv, postdiv2; ++ u64 rate64 = 24000000; ++ ++ fbdiv = ((con0 >> RK3036_PLLCON0_FBDIV_SHIFT) & ++ RK3036_PLLCON0_FBDIV_MASK); ++ postdiv1 = ((con0 >> RK3036_PLLCON0_POSTDIV1_SHIFT) & ++ RK3036_PLLCON0_POSTDIV1_MASK); ++ refdiv = ((con1 >> RK3036_PLLCON1_REFDIV_SHIFT) & ++ RK3036_PLLCON1_REFDIV_MASK); ++ postdiv2 = ((con1 >> RK3036_PLLCON1_POSTDIV2_SHIFT) & ++ RK3036_PLLCON1_POSTDIV2_MASK); ++ ++ rate64 *= fbdiv; ++ do_div(rate64, refdiv); ++ do_div(rate64, postdiv1); ++ do_div(rate64, postdiv2); ++ ++ return (unsigned long)rate64; ++} ++ + static void rockchip_rk3036_pll_get_params(struct rockchip_clk_pll *pll, + struct rockchip_pll_rate_table *rate) + { +@@ -165,7 +467,10 @@ static unsigned long rockchip_rk3036_pll_recalc_rate(struct clk_hw *hw, + { + struct rockchip_clk_pll *pll = to_rockchip_clk_pll(hw); + struct rockchip_pll_rate_table cur; +- u64 rate64 = prate; ++ u64 rate64 = prate, frac_rate64 = prate; ++ ++ if (pll->sel && pll->scaling) ++ return pll->scaling; + + rockchip_rk3036_pll_get_params(pll, &cur); + +@@ -174,7 +479,7 @@ static unsigned long rockchip_rk3036_pll_recalc_rate(struct clk_hw *hw, + + if (cur.dsmpd == 0) { + /* fractional mode */ +- u64 frac_rate64 = prate * cur.frac; ++ frac_rate64 *= cur.frac; + + do_div(frac_rate64, cur.refdiv); + rate64 += frac_rate64 >> 24; +@@ -231,6 +536,8 @@ static int rockchip_rk3036_pll_set_params(struct rockchip_clk_pll *pll, + pllcon |= rate->frac << RK3036_PLLCON2_FRAC_SHIFT; + writel_relaxed(pllcon, pll->reg_base + RK3036_PLLCON(2)); + ++ rockchip_boost_disable_low(pll); ++ + /* wait for the pll to lock */ + ret = rockchip_rk3036_pll_wait_lock(pll); + if (ret) { +@@ -412,6 +719,9 @@ static unsigned long rockchip_rk3066_pll_recalc_rate(struct clk_hw *hw, + return prate; + } + ++ if (pll->sel && pll->scaling) ++ return pll->scaling; ++ + rockchip_rk3066_pll_get_params(pll, &cur); + + rate64 *= cur.nf; +@@ -485,9 +795,18 @@ static int rockchip_rk3066_pll_set_rate(struct clk_hw *hw, unsigned long drate, + { + struct rockchip_clk_pll *pll = to_rockchip_clk_pll(hw); + const struct rockchip_pll_rate_table *rate; ++ unsigned long old_rate = rockchip_rk3066_pll_recalc_rate(hw, prate); ++ struct regmap *grf = pll->ctx->grf; ++ int ret; + +- pr_debug("%s: changing %s to %lu with a parent rate of %lu\n", +- __func__, clk_hw_get_name(hw), drate, prate); ++ if (IS_ERR(grf)) { ++ pr_debug("%s: grf regmap not available, aborting rate change\n", ++ __func__); ++ return PTR_ERR(grf); ++ } ++ ++ pr_debug("%s: changing %s from %lu to %lu with a parent rate of %lu\n", ++ __func__, clk_hw_get_name(hw), old_rate, drate, prate); + + /* Get required rate settings from table */ + rate = rockchip_get_pll_settings(pll, drate); +@@ -497,7 +816,11 @@ static int rockchip_rk3066_pll_set_rate(struct clk_hw *hw, unsigned long drate, + return -EINVAL; + } + +- return rockchip_rk3066_pll_set_params(pll, rate); ++ ret = rockchip_rk3066_pll_set_params(pll, rate); ++ if (ret) ++ pll->scaling = 0; ++ ++ return ret; + } + + static int rockchip_rk3066_pll_enable(struct clk_hw *hw) +@@ -649,6 +972,9 @@ static unsigned long rockchip_rk3399_pll_recalc_rate(struct clk_hw *hw, + struct rockchip_pll_rate_table cur; + u64 rate64 = prate; + ++ if (pll->sel && pll->scaling) ++ return pll->scaling; ++ + rockchip_rk3399_pll_get_params(pll, &cur); + + rate64 *= cur.fbdiv; +@@ -692,6 +1018,11 @@ static int rockchip_rk3399_pll_set_params(struct rockchip_clk_pll *pll, + rate_change_remuxed = 1; + } + ++ /* set pll power down */ ++ writel(HIWORD_UPDATE(RK3399_PLLCON3_PWRDOWN, ++ RK3399_PLLCON3_PWRDOWN, 0), ++ pll->reg_base + RK3399_PLLCON(3)); ++ + /* update pll values */ + writel_relaxed(HIWORD_UPDATE(rate->fbdiv, RK3399_PLLCON0_FBDIV_MASK, + RK3399_PLLCON0_FBDIV_SHIFT), +@@ -715,6 +1046,11 @@ static int rockchip_rk3399_pll_set_params(struct rockchip_clk_pll *pll, + RK3399_PLLCON3_DSMPD_SHIFT), + pll->reg_base + RK3399_PLLCON(3)); + ++ /* set pll power up */ ++ writel(HIWORD_UPDATE(0, ++ RK3399_PLLCON3_PWRDOWN, 0), ++ pll->reg_base + RK3399_PLLCON(3)); ++ + /* wait for the pll to lock */ + ret = rockchip_rk3399_pll_wait_lock(pll); + if (ret) { +@@ -734,9 +1070,11 @@ static int rockchip_rk3399_pll_set_rate(struct clk_hw *hw, unsigned long drate, + { + struct rockchip_clk_pll *pll = to_rockchip_clk_pll(hw); + const struct rockchip_pll_rate_table *rate; ++ unsigned long old_rate = rockchip_rk3399_pll_recalc_rate(hw, prate); ++ int ret; + +- pr_debug("%s: changing %s to %lu with a parent rate of %lu\n", +- __func__, __clk_get_name(hw->clk), drate, prate); ++ pr_debug("%s: changing %s from %lu to %lu with a parent rate of %lu\n", ++ __func__, __clk_get_name(hw->clk), old_rate, drate, prate); + + /* Get required rate settings from table */ + rate = rockchip_get_pll_settings(pll, drate); +@@ -746,7 +1084,11 @@ static int rockchip_rk3399_pll_set_rate(struct clk_hw *hw, unsigned long drate, + return -EINVAL; + } + +- return rockchip_rk3399_pll_set_params(pll, rate); ++ ret = rockchip_rk3399_pll_set_params(pll, rate); ++ if (ret) ++ pll->scaling = 0; ++ ++ return ret; + } + + static int rockchip_rk3399_pll_enable(struct clk_hw *hw) +@@ -842,6 +1184,80 @@ static const struct clk_ops rockchip_rk3399_pll_clk_ops = { + .init = rockchip_rk3399_pll_init, + }; + ++#ifdef CONFIG_ROCKCHIP_CLK_COMPENSATION ++int rockchip_pll_clk_compensation(struct clk *clk, int ppm) ++{ ++ struct clk *parent = clk_get_parent(clk); ++ struct rockchip_clk_pll *pll; ++ static u32 frac, fbdiv; ++ bool negative; ++ u32 pllcon, pllcon0, pllcon2, fbdiv_mask, frac_mask, frac_shift; ++ u64 fracdiv, m, n; ++ ++ if ((ppm > 1000) || (ppm < -1000)) ++ return -EINVAL; ++ ++ if (IS_ERR_OR_NULL(parent)) ++ return -EINVAL; ++ ++ pll = to_rockchip_clk_pll(__clk_get_hw(parent)); ++ if (!pll) ++ return -EINVAL; ++ ++ switch (pll->type) { ++ case pll_rk3036: ++ case pll_rk3328: ++ pllcon0 = RK3036_PLLCON(0); ++ pllcon2 = RK3036_PLLCON(2); ++ fbdiv_mask = RK3036_PLLCON0_FBDIV_MASK; ++ frac_mask = RK3036_PLLCON2_FRAC_MASK; ++ frac_shift = RK3036_PLLCON2_FRAC_SHIFT; ++ break; ++ case pll_rk3066: ++ return -EINVAL; ++ case pll_rk3399: ++ pllcon0 = RK3399_PLLCON(0); ++ pllcon2 = RK3399_PLLCON(2); ++ fbdiv_mask = RK3399_PLLCON0_FBDIV_MASK; ++ frac_mask = RK3399_PLLCON2_FRAC_MASK; ++ frac_shift = RK3399_PLLCON2_FRAC_SHIFT; ++ break; ++ default: ++ return -EINVAL; ++ } ++ ++ negative = !!(ppm & BIT(31)); ++ ppm = negative ? ~ppm + 1 : ppm; ++ ++ if (!frac) { ++ frac = readl_relaxed(pll->reg_base + pllcon2) & frac_mask; ++ fbdiv = readl_relaxed(pll->reg_base + pllcon0) & fbdiv_mask; ++ } ++ ++ /* ++ * delta frac frac ppm ++ * -------------- = (fbdiv + ----------) * --------- ++ * 1 << 24 1 << 24 1000000 ++ * ++ */ ++ m = div64_u64((uint64_t)frac * ppm, 1000000); ++ n = div64_u64((uint64_t)ppm << 24, 1000000) * fbdiv; ++ ++ fracdiv = negative ? frac - (m + n) : frac + (m + n); ++ ++ if (!frac || fracdiv > frac_mask) ++ return -EINVAL; ++ ++ pllcon = readl_relaxed(pll->reg_base + pllcon2); ++ pllcon &= ~(frac_mask << frac_shift); ++ pllcon |= fracdiv << frac_shift; ++ writel_relaxed(pllcon, pll->reg_base + pllcon2); ++ ++ return 0; ++} ++EXPORT_SYMBOL(rockchip_pll_clk_compensation); ++#endif ++ + /* + * Common registering of pll clocks + */ +@@ -914,8 +1330,12 @@ struct clk *rockchip_clk_register_pll(struct rockchip_clk_provider *ctx, + /* now create the actual pll */ + init.name = pll_name; + ++#ifndef CONFIG_ROCKCHIP_LOW_PERFORMANCE + /* keep all plls untouched for now */ + init.flags = flags | CLK_IGNORE_UNUSED; ++#else ++ init.flags = flags; ++#endif + + init.parent_names = &parent_names[0]; + init.num_parents = 1; +@@ -940,7 +1360,7 @@ struct clk *rockchip_clk_register_pll(struct rockchip_clk_provider *ctx, + switch (pll_type) { + case pll_rk3036: + case pll_rk3328: +- if (!pll->rate_table) ++ if (!pll->rate_table || IS_ERR(ctx->grf)) + init.ops = &rockchip_rk3036_pll_clk_norate_ops; + else + init.ops = &rockchip_rk3036_pll_clk_ops; +@@ -987,3 +1407,316 @@ struct clk *rockchip_clk_register_pll(struct rockchip_clk_provider *ctx, + kfree(pll); + return mux_clk; + } ++ ++static unsigned long rockchip_pll_con_to_rate(struct rockchip_clk_pll *pll, ++ u32 con0, u32 con1) ++{ ++ switch (pll->type) { ++ case pll_rk3036: ++ case pll_rk3328: ++ return rockchip_rk3036_pll_con_to_rate(pll, con0, con1); ++ case pll_rk3066: ++ break; ++ case pll_rk3399: ++ break; ++ default: ++ pr_warn("%s: Unknown pll type\n", __func__); ++ } ++ ++ return 0; ++} ++ ++void rockchip_boost_init(struct clk_hw *hw) ++{ ++ struct rockchip_clk_pll *pll; ++ struct device_node *np; ++ u32 value, con0, con1; ++ ++ if (!hw) ++ return; ++ pll = to_rockchip_clk_pll(hw); ++ np = of_parse_phandle(pll->ctx->cru_node, "rockchip,boost", 0); ++ if (!np) { ++ pr_debug("%s: failed to get boost np\n", __func__); ++ return; ++ } ++ pll->boost = syscon_node_to_regmap(np); ++ if (IS_ERR(pll->boost)) { ++ pr_debug("%s: failed to get boost regmap\n", __func__); ++ return; ++ } ++ ++ if (!of_property_read_u32(np, "rockchip,boost-low-con0", &con0) && ++ !of_property_read_u32(np, "rockchip,boost-low-con1", &con1)) { ++ pr_debug("boost-low-con=0x%x 0x%x\n", con0, con1); ++ regmap_write(pll->boost, BOOST_PLL_L_CON(0), ++ HIWORD_UPDATE(con0, BOOST_PLL_CON_MASK, 0)); ++ regmap_write(pll->boost, BOOST_PLL_L_CON(1), ++ HIWORD_UPDATE(con1, BOOST_PLL_CON_MASK, 0)); ++ pll->boost_low_rate = rockchip_pll_con_to_rate(pll, con0, ++ con1); ++ pr_debug("boost-low-rate=%lu\n", pll->boost_low_rate); ++ } ++ if (!of_property_read_u32(np, "rockchip,boost-high-con0", &con0) && ++ !of_property_read_u32(np, "rockchip,boost-high-con1", &con1)) { ++ pr_debug("boost-high-con=0x%x 0x%x\n", con0, con1); ++ regmap_write(pll->boost, BOOST_PLL_H_CON(0), ++ HIWORD_UPDATE(con0, BOOST_PLL_CON_MASK, 0)); ++ regmap_write(pll->boost, BOOST_PLL_H_CON(1), ++ HIWORD_UPDATE(con1, BOOST_PLL_CON_MASK, 0)); ++ pll->boost_high_rate = rockchip_pll_con_to_rate(pll, con0, ++ con1); ++ pr_debug("boost-high-rate=%lu\n", pll->boost_high_rate); ++ } ++ if (!of_property_read_u32(np, "rockchip,boost-backup-pll", &value)) { ++ pr_debug("boost-backup-pll=0x%x\n", value); ++ regmap_write(pll->boost, BOOST_CLK_CON, ++ HIWORD_UPDATE(value, BOOST_BACKUP_PLL_MASK, ++ BOOST_BACKUP_PLL_SHIFT)); ++ } ++ if (!of_property_read_u32(np, "rockchip,boost-backup-pll-usage", ++ &pll->boost_backup_pll_usage)) { ++ pr_debug("boost-backup-pll-usage=0x%x\n", ++ pll->boost_backup_pll_usage); ++ regmap_write(pll->boost, BOOST_CLK_CON, ++ HIWORD_UPDATE(pll->boost_backup_pll_usage, ++ BOOST_BACKUP_PLL_USAGE_MASK, ++ BOOST_BACKUP_PLL_USAGE_SHIFT)); ++ } ++ if (!of_property_read_u32(np, "rockchip,boost-switch-threshold", ++ &value)) { ++ pr_debug("boost-switch-threshold=0x%x\n", value); ++ regmap_write(pll->boost, BOOST_SWITCH_THRESHOLD, value); ++ } ++ if (!of_property_read_u32(np, "rockchip,boost-statis-threshold", ++ &value)) { ++ pr_debug("boost-statis-threshold=0x%x\n", value); ++ regmap_write(pll->boost, BOOST_STATIS_THRESHOLD, value); ++ } ++ if (!of_property_read_u32(np, "rockchip,boost-statis-enable", ++ &value)) { ++ pr_debug("boost-statis-enable=0x%x\n", value); ++ regmap_write(pll->boost, BOOST_BOOST_CON, ++ HIWORD_UPDATE(value, BOOST_STATIS_ENABLE_MASK, ++ BOOST_STATIS_ENABLE_SHIFT)); ++ } ++ if (!of_property_read_u32(np, "rockchip,boost-enable", &value)) { ++ pr_debug("boost-enable=0x%x\n", value); ++ regmap_write(pll->boost, BOOST_BOOST_CON, ++ HIWORD_UPDATE(value, BOOST_ENABLE_MASK, ++ BOOST_ENABLE_SHIFT)); ++ if (value) ++ pll->boost_enabled = true; ++ } ++#ifdef CONFIG_DEBUG_FS ++ if (pll->boost_enabled) { ++ mutex_lock(&clk_boost_lock); ++ hlist_add_head(&pll->debug_node, &clk_boost_list); ++ mutex_unlock(&clk_boost_lock); ++ } ++#endif ++} ++ ++void rockchip_boost_enable_recovery_sw_low(struct clk_hw *hw) ++{ ++ struct rockchip_clk_pll *pll; ++ unsigned int val; ++ ++ if (!hw) ++ return; ++ pll = to_rockchip_clk_pll(hw); ++ if (!pll->boost_enabled) ++ return; ++ ++ regmap_write(pll->boost, BOOST_BOOST_CON, ++ HIWORD_UPDATE(1, BOOST_RECOVERY_MASK, ++ BOOST_RECOVERY_SHIFT)); ++ do { ++ regmap_read(pll->boost, BOOST_FSM_STATUS, &val); ++ } while (!(val & BOOST_BUSY_STATE)); ++ ++ regmap_write(pll->boost, BOOST_BOOST_CON, ++ HIWORD_UPDATE(1, BOOST_SW_CTRL_MASK, ++ BOOST_SW_CTRL_SHIFT) | ++ HIWORD_UPDATE(1, BOOST_LOW_FREQ_EN_MASK, ++ BOOST_LOW_FREQ_EN_SHIFT)); ++} ++ ++static void rockchip_boost_disable_low(struct rockchip_clk_pll *pll) ++{ ++ if (!pll->boost_enabled) ++ return; ++ ++ regmap_write(pll->boost, BOOST_BOOST_CON, ++ HIWORD_UPDATE(0, BOOST_LOW_FREQ_EN_MASK, ++ BOOST_LOW_FREQ_EN_SHIFT)); ++} ++ ++void rockchip_boost_disable_recovery_sw(struct clk_hw *hw) ++{ ++ struct rockchip_clk_pll *pll; ++ ++ if (!hw) ++ return; ++ pll = to_rockchip_clk_pll(hw); ++ if (!pll->boost_enabled) ++ return; ++ ++ regmap_write(pll->boost, BOOST_BOOST_CON, ++ HIWORD_UPDATE(0, BOOST_RECOVERY_MASK, ++ BOOST_RECOVERY_SHIFT)); ++ regmap_write(pll->boost, BOOST_BOOST_CON, ++ HIWORD_UPDATE(0, BOOST_SW_CTRL_MASK, ++ BOOST_SW_CTRL_SHIFT)); ++} ++ ++void rockchip_boost_add_core_div(struct clk_hw *hw, unsigned long prate) ++{ ++ struct rockchip_clk_pll *pll; ++ unsigned int div; ++ ++ if (!hw) ++ return; ++ pll = to_rockchip_clk_pll(hw); ++ if (!pll->boost_enabled || pll->boost_backup_pll_rate == prate) ++ return; ++ ++ /* todo */ ++ if (pll->boost_backup_pll_usage == BOOST_BACKUP_PLL_USAGE_TARGET) ++ return; ++ /* ++ * cpu clock rate should be less than or equal to ++ * low rate when change pll rate in boost module ++ */ ++ if (pll->boost_low_rate && prate > pll->boost_low_rate) { ++ div = DIV_ROUND_UP(prate, pll->boost_low_rate) - 1; ++ regmap_write(pll->boost, BOOST_CLK_CON, ++ HIWORD_UPDATE(div, BOOST_CORE_DIV_MASK, ++ BOOST_CORE_DIV_SHIFT)); ++ pll->boost_backup_pll_rate = prate; ++ } ++} ++ ++#ifdef CONFIG_DEBUG_FS ++#include ++ ++#ifndef MODULE ++static int boost_summary_show(struct seq_file *s, void *data) ++{ ++ struct rockchip_clk_pll *pll = (struct rockchip_clk_pll *)s->private; ++ u32 boost_count = 0; ++ u32 freq_cnt0 = 0, freq_cnt1 = 0; ++ u64 freq_cnt = 0, high_freq_time = 0; ++ u32 short_count = 0, short_threshold = 0; ++ u32 interval_time = 0; ++ ++ seq_puts(s, " device boost_count high_freq_count high_freq_time short_count short_threshold interval_count\n"); ++ seq_puts(s, "------------------------------------------------------------------------------------------------------\n"); ++ seq_printf(s, " %s\n", clk_hw_get_name(&pll->hw)); ++ ++ regmap_read(pll->boost, BOOST_SWITCH_CNT, &boost_count); ++ ++ regmap_read(pll->boost, BOOST_HIGH_PERF_CNT0, &freq_cnt0); ++ regmap_read(pll->boost, BOOST_HIGH_PERF_CNT1, &freq_cnt1); ++ freq_cnt = ((u64)freq_cnt1 << 32) + (u64)freq_cnt0; ++ high_freq_time = freq_cnt; ++ do_div(high_freq_time, 24); ++ ++ regmap_read(pll->boost, BOOST_SHORT_SWITCH_CNT, &short_count); ++ regmap_read(pll->boost, BOOST_STATIS_THRESHOLD, &short_threshold); ++ regmap_read(pll->boost, BOOST_SWITCH_THRESHOLD, &interval_time); ++ ++ seq_printf(s, "%22u %17llu %15llu %12u %16u %15u\n", ++ boost_count, freq_cnt, high_freq_time, short_count, ++ short_threshold, interval_time); ++ ++ return 0; ++} ++ ++static int boost_summary_open(struct inode *inode, struct file *file) ++{ ++ return single_open(file, boost_summary_show, inode->i_private); ++} ++ ++static const struct file_operations boost_summary_fops = { ++ .open = boost_summary_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = single_release, ++}; ++ ++static int boost_config_show(struct seq_file *s, void *data) ++{ ++ struct rockchip_clk_pll *pll = (struct rockchip_clk_pll *)s->private; ++ ++ seq_printf(s, "boost_enabled: %d\n", pll->boost_enabled); ++ seq_printf(s, "boost_low_rate: %lu\n", pll->boost_low_rate); ++ seq_printf(s, "boost_high_rate: %lu\n", pll->boost_high_rate); ++ ++ return 0; ++} ++ ++static int boost_config_open(struct inode *inode, struct file *file) ++{ ++ return single_open(file, boost_config_show, inode->i_private); ++} ++ ++static const struct file_operations boost_config_fops = { ++ .open = boost_config_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = single_release, ++}; ++ ++static int boost_debug_create_one(struct rockchip_clk_pll *pll, ++ struct dentry *rootdir) ++{ ++ struct dentry *pdentry, *d; ++ ++ pdentry = debugfs_lookup(clk_hw_get_name(&pll->hw), rootdir); ++ if (!pdentry) { ++ pr_err("%s: failed to lookup %s dentry\n", __func__, ++ clk_hw_get_name(&pll->hw)); ++ return -ENOMEM; ++ } ++ ++ d = debugfs_create_file("boost_summary", 0444, pdentry, ++ pll, &boost_summary_fops); ++ if (!d) { ++ pr_err("%s: failed to create boost_summary file\n", __func__); ++ return -ENOMEM; ++ } ++ ++ d = debugfs_create_file("boost_config", 0444, pdentry, ++ pll, &boost_config_fops); ++ if (!d) { ++ pr_err("%s: failed to create boost config file\n", __func__); ++ return -ENOMEM; ++ } ++ ++ return 0; ++} ++ ++static int __init boost_debug_init(void) ++{ ++ struct rockchip_clk_pll *pll; ++ struct dentry *rootdir; ++ ++ rootdir = debugfs_lookup("clk", NULL); ++ if (!rootdir) { ++ pr_err("%s: failed to lookup clk dentry\n", __func__); ++ return -ENOMEM; ++ } ++ ++ mutex_lock(&clk_boost_lock); ++ ++ hlist_for_each_entry(pll, &clk_boost_list, debug_node) ++ boost_debug_create_one(pll, rootdir); ++ ++ mutex_unlock(&clk_boost_lock); ++ ++ return 0; ++} ++late_initcall(boost_debug_init); ++#endif /* MODULE */ ++#endif /* CONFIG_DEBUG_FS */ +diff --git a/drivers/clk/rockchip/clk-pvtm.c b/drivers/clk/rockchip/clk-pvtm.c +new file mode 100755 +index 000000000000..ad02b6a571d8 +--- /dev/null ++++ b/drivers/clk/rockchip/clk-pvtm.c +@@ -0,0 +1,310 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#define CLK_SEL_EXTERNAL_32K 0 ++#define CLK_SEL_INTERNAL_PVTM 1 ++ ++#define wr_msk_bit(v, off, msk) ((v) << (off) | (msk << (16 + (off)))) ++ ++struct rockchip_clock_pvtm; ++ ++struct rockchip_clock_pvtm_info { ++ u32 con; ++ u32 sta; ++ u32 sel_con; ++ u32 sel_shift; ++ u32 sel_value; ++ u32 sel_mask; ++ u32 div_shift; ++ u32 div_mask; ++ ++ u32 (*get_value)(struct rockchip_clock_pvtm *pvtm, ++ unsigned int time_us); ++ int (*init_freq)(struct rockchip_clock_pvtm *pvtm); ++ int (*sel_enable)(struct rockchip_clock_pvtm *pvtm); ++}; ++ ++struct rockchip_clock_pvtm { ++ const struct rockchip_clock_pvtm_info *info; ++ struct regmap *grf; ++ struct clk *pvtm_clk; ++ struct clk *clk; ++ unsigned long rate; ++}; ++ ++static unsigned long xin32k_pvtm_recalc_rate(struct clk_hw *hw, ++ unsigned long parent_rate) ++{ ++ return 32768; ++} ++ ++static const struct clk_ops xin32k_pvtm = { ++ .recalc_rate = xin32k_pvtm_recalc_rate, ++}; ++ ++static void rockchip_clock_pvtm_delay(unsigned int delay) ++{ ++ unsigned int ms = delay / 1000; ++ unsigned int us = delay % 1000; ++ ++ if (ms > 0) { ++ if (ms < 20) ++ us += ms * 1000; ++ else ++ msleep(ms); ++ } ++ ++ if (us >= 10) ++ usleep_range(us, us + 100); ++ else ++ udelay(us); ++} ++ ++static int rockchip_clock_sel_internal_pvtm(struct rockchip_clock_pvtm *pvtm) ++{ ++ int ret = 0; ++ ++ ret = regmap_write(pvtm->grf, pvtm->info->sel_con, ++ wr_msk_bit(pvtm->info->sel_value, ++ pvtm->info->sel_shift, ++ pvtm->info->sel_mask)); ++ if (ret != 0) ++ pr_err("%s: fail to write register\n", __func__); ++ ++ return ret; ++} ++ ++/* get pmu pvtm value */ ++static u32 rockchip_clock_pvtm_get_value(struct rockchip_clock_pvtm *pvtm, ++ u32 time_us) ++{ ++ const struct rockchip_clock_pvtm_info *info = pvtm->info; ++ u32 val = 0, sta = 0; ++ u32 clk_cnt, check_cnt; ++ ++ /* 24m clk ,24cnt=1us */ ++ clk_cnt = time_us * 24; ++ ++ regmap_write(pvtm->grf, info->con + 0x4, clk_cnt); ++ regmap_write(pvtm->grf, info->con, wr_msk_bit(3, 0, 0x3)); ++ ++ rockchip_clock_pvtm_delay(time_us); ++ ++ check_cnt = 100; ++ while (check_cnt--) { ++ regmap_read(pvtm->grf, info->sta, &sta); ++ if (sta & 0x1) ++ break; ++ udelay(4); ++ } ++ ++ if (check_cnt) { ++ regmap_read(pvtm->grf, info->sta + 0x4, &val); ++ } else { ++ pr_err("%s: wait pvtm_done timeout!\n", __func__); ++ val = 0; ++ } ++ ++ regmap_write(pvtm->grf, info->con, wr_msk_bit(0, 0, 0x3)); ++ ++ return val; ++} ++ ++static int rockchip_clock_pvtm_init_freq(struct rockchip_clock_pvtm *pvtm) ++{ ++ u32 pvtm_cnt = 0; ++ u32 div, time_us; ++ int ret = 0; ++ ++ time_us = 1000; ++ pvtm_cnt = pvtm->info->get_value(pvtm, time_us); ++ pr_debug("get pvtm_cnt = %d\n", pvtm_cnt); ++ ++ /* set pvtm_div to get rate */ ++ div = DIV_ROUND_UP(1000 * pvtm_cnt, pvtm->rate); ++ if (div > pvtm->info->div_mask) { ++ pr_err("pvtm_div out of bounary! set max instead\n"); ++ div = pvtm->info->div_mask; ++ } ++ ++ pr_debug("set div %d, rate %luKHZ\n", div, pvtm->rate); ++ ret = regmap_write(pvtm->grf, pvtm->info->con, ++ wr_msk_bit(div, pvtm->info->div_shift, ++ pvtm->info->div_mask)); ++ if (ret != 0) ++ goto out; ++ ++ /* pmu pvtm oscilator enable */ ++ ret = regmap_write(pvtm->grf, pvtm->info->con, ++ wr_msk_bit(1, 1, 0x1)); ++ if (ret != 0) ++ goto out; ++ ++ ret = pvtm->info->sel_enable(pvtm); ++out: ++ if (ret != 0) ++ pr_err("%s: fail to write register\n", __func__); ++ ++ return ret; ++} ++ ++static int clock_pvtm_regitstor(struct device *dev, ++ struct rockchip_clock_pvtm *pvtm) ++{ ++ struct clk_init_data init = {}; ++ struct clk_hw *clk_hw; ++ ++ /* Init the xin32k_pvtm */ ++ pvtm->info->init_freq(pvtm); ++ ++ init.parent_names = NULL; ++ init.num_parents = 0; ++ init.name = "xin32k_pvtm"; ++ init.ops = &xin32k_pvtm; ++ ++ clk_hw = devm_kzalloc(dev, sizeof(*clk_hw), GFP_KERNEL); ++ if (!clk_hw) ++ return -ENOMEM; ++ clk_hw->init = &init; ++ ++ /* optional override of the clockname */ ++ of_property_read_string_index(dev->of_node, "clock-output-names", ++ 0, &init.name); ++ pvtm->clk = devm_clk_register(dev, clk_hw); ++ if (IS_ERR(pvtm->clk)) ++ return PTR_ERR(pvtm->clk); ++ ++ return of_clk_add_provider(dev->of_node, of_clk_src_simple_get, ++ pvtm->clk); ++} ++ ++static const struct rockchip_clock_pvtm_info rk3368_pvtm_data = { ++ .con = 0x180, ++ .sta = 0x190, ++ .sel_con = 0x100, ++ .sel_shift = 6, ++ .sel_value = CLK_SEL_INTERNAL_PVTM, ++ .sel_mask = 0x1, ++ .div_shift = 2, ++ .div_mask = 0x3f, ++ ++ .sel_enable = rockchip_clock_sel_internal_pvtm, ++ .get_value = rockchip_clock_pvtm_get_value, ++ .init_freq = rockchip_clock_pvtm_init_freq, ++}; ++ ++static const struct of_device_id rockchip_clock_pvtm_match[] = { ++ { ++ .compatible = "rockchip,rk3368-pvtm-clock", ++ .data = (void *)&rk3368_pvtm_data, ++ }, ++ {} ++}; ++MODULE_DEVICE_TABLE(of, rockchip_clock_pvtm_match); ++ ++static int rockchip_clock_pvtm_probe(struct platform_device *pdev) ++{ ++ struct device *dev = &pdev->dev; ++ struct device_node *np = pdev->dev.of_node; ++ const struct of_device_id *match; ++ struct rockchip_clock_pvtm *pvtm; ++ int error; ++ u32 rate; ++ ++ pvtm = devm_kzalloc(dev, sizeof(*pvtm), GFP_KERNEL); ++ if (!pvtm) ++ return -ENOMEM; ++ ++ match = of_match_node(rockchip_clock_pvtm_match, np); ++ if (!match) ++ return -ENXIO; ++ ++ pvtm->info = (const struct rockchip_clock_pvtm_info *)match->data; ++ if (!pvtm->info) ++ return -EINVAL; ++ ++ if (!dev->parent || !dev->parent->of_node) ++ return -EINVAL; ++ ++ pvtm->grf = syscon_node_to_regmap(dev->parent->of_node); ++ if (IS_ERR(pvtm->grf)) ++ return PTR_ERR(pvtm->grf); ++ ++ if (!of_property_read_u32(np, "pvtm-rate", &rate)) ++ pvtm->rate = rate; ++ else ++ pvtm->rate = 32768; ++ ++ pvtm->pvtm_clk = devm_clk_get(&pdev->dev, "pvtm_pmu_clk"); ++ if (IS_ERR(pvtm->pvtm_clk)) { ++ error = PTR_ERR(pvtm->pvtm_clk); ++ if (error != -EPROBE_DEFER) ++ dev_err(&pdev->dev, ++ "failed to get pvtm core clock: %d\n", ++ error); ++ goto out_probe; ++ } ++ ++ error = clk_prepare_enable(pvtm->pvtm_clk); ++ if (error) { ++ dev_err(&pdev->dev, "failed to enable the clock: %d\n", ++ error); ++ goto out_probe; ++ } ++ ++ platform_set_drvdata(pdev, pvtm); ++ ++ error = clock_pvtm_regitstor(&pdev->dev, pvtm); ++ if (error) { ++ dev_err(&pdev->dev, "failed to registor clock: %d\n", ++ error); ++ goto out_clk_put; ++ } ++ ++ return error; ++ ++out_clk_put: ++ clk_disable_unprepare(pvtm->pvtm_clk); ++out_probe: ++ return error; ++} ++ ++static int rockchip_clock_pvtm_remove(struct platform_device *pdev) ++{ ++ struct rockchip_clock_pvtm *pvtm = platform_get_drvdata(pdev); ++ struct device_node *np = pdev->dev.of_node; ++ ++ of_clk_del_provider(np); ++ clk_disable_unprepare(pvtm->pvtm_clk); ++ ++ return 0; ++} ++ ++static struct platform_driver rockchip_clock_pvtm_driver = { ++ .driver = { ++ .name = "rockchip-clcok-pvtm", ++ .of_match_table = rockchip_clock_pvtm_match, ++ }, ++ .probe = rockchip_clock_pvtm_probe, ++ .remove = rockchip_clock_pvtm_remove, ++}; ++ ++module_platform_driver(rockchip_clock_pvtm_driver); ++ ++MODULE_DESCRIPTION("Rockchip Clock Pvtm Driver"); ++MODULE_LICENSE("GPL v2"); +diff --git a/drivers/clk/rockchip/clk-px30.c b/drivers/clk/rockchip/clk-px30.c +index 6fb9c98b7d24..89ecbd4ed0c9 100644 +--- a/drivers/clk/rockchip/clk-px30.c ++++ b/drivers/clk/rockchip/clk-px30.c +@@ -6,13 +6,16 @@ + + #include + #include ++#include + #include + #include ++#include + #include + #include + #include "clk.h" + + #define PX30_GRF_SOC_STATUS0 0x480 ++#define PX30_FRAC_MAX_PRATE 600000000 + + enum px30_plls { + apll, dpll, cpll, npll, apll_b_h, apll_b_l, +@@ -124,29 +127,31 @@ static struct rockchip_cpuclk_rate_table px30_cpuclk_rates[] __initdata = { + }; + + static const struct rockchip_cpuclk_reg_data px30_cpuclk_data = { +- .core_reg = PX30_CLKSEL_CON(0), +- .div_core_shift = 0, +- .div_core_mask = 0xf, ++ .core_reg[0] = PX30_CLKSEL_CON(0), ++ .div_core_shift[0] = 0, ++ .div_core_mask[0] = 0xf, ++ .num_cores = 1, + .mux_core_alt = 1, + .mux_core_main = 0, + .mux_core_shift = 7, + .mux_core_mask = 0x1, ++ .pll_name = "pll_apll", + }; + + PNAME(mux_pll_p) = { "xin24m"}; + PNAME(mux_usb480m_p) = { "xin24m", "usb480m_phy", "clk_rtc32k_pmu" }; +-PNAME(mux_armclk_p) = { "apll_core", "gpll_core" }; + PNAME(mux_ddrphy_p) = { "dpll_ddr", "gpll_ddr" }; + PNAME(mux_ddrstdby_p) = { "clk_ddrphy1x", "clk_stdby_2wrap" }; +-PNAME(mux_4plls_p) = { "gpll", "dummy_cpll", "usb480m", "npll" }; ++PNAME(mux_gpll_dmycpll_usb480m_npll_p) = { "gpll", "dummy_cpll", "usb480m", "npll" }; ++PNAME(mux_gpll_dmycpll_usb480m_dmynpll_p) = { "gpll", "dummy_cpll", "usb480m", "dummy_npll" }; + PNAME(mux_cpll_npll_p) = { "cpll", "npll" }; + PNAME(mux_npll_cpll_p) = { "npll", "cpll" }; + PNAME(mux_gpll_cpll_p) = { "gpll", "dummy_cpll" }; +-PNAME(mux_gpll_npll_p) = { "gpll", "npll" }; ++PNAME(mux_gpll_npll_p) = { "gpll", "dummy_npll" }; + PNAME(mux_gpll_xin24m_p) = { "gpll", "xin24m"}; +-PNAME(mux_gpll_cpll_npll_p) = { "gpll", "dummy_cpll", "npll" }; +-PNAME(mux_gpll_cpll_npll_xin24m_p) = { "gpll", "dummy_cpll", "npll", "xin24m" }; +-PNAME(mux_gpll_xin24m_npll_p) = { "gpll", "xin24m", "npll"}; ++PNAME(mux_gpll_cpll_npll_p) = { "gpll", "dummy_cpll", "dummy_npll" }; ++PNAME(mux_gpll_cpll_npll_xin24m_p) = { "gpll", "dummy_cpll", "dummy_npll", "xin24m" }; ++PNAME(mux_gpll_xin24m_npll_p) = { "gpll", "xin24m", "dummy_npll"}; + PNAME(mux_pdm_p) = { "clk_pdm_src", "clk_pdm_frac" }; + PNAME(mux_i2s0_tx_p) = { "clk_i2s0_tx_src", "clk_i2s0_tx_frac", "mclk_i2s0_tx_in", "xin12m"}; + PNAME(mux_i2s0_rx_p) = { "clk_i2s0_rx_src", "clk_i2s0_rx_frac", "mclk_i2s0_rx_in", "xin12m"}; +@@ -158,13 +163,13 @@ PNAME(mux_i2s1_out_p) = { "clk_i2s1", "xin12m"}; + PNAME(mux_i2s2_out_p) = { "clk_i2s2", "xin12m"}; + PNAME(mux_i2s0_tx_rx_p) = { "clk_i2s0_tx_mux", "clk_i2s0_rx_mux"}; + PNAME(mux_i2s0_rx_tx_p) = { "clk_i2s0_rx_mux", "clk_i2s0_tx_mux"}; +-PNAME(mux_uart_src_p) = { "gpll", "xin24m", "usb480m", "npll" }; ++PNAME(mux_uart_src_p) = { "gpll", "xin24m", "usb480m", "dummy_npll" }; + PNAME(mux_uart1_p) = { "clk_uart1_src", "clk_uart1_np5", "clk_uart1_frac" }; + PNAME(mux_uart2_p) = { "clk_uart2_src", "clk_uart2_np5", "clk_uart2_frac" }; + PNAME(mux_uart3_p) = { "clk_uart3_src", "clk_uart3_np5", "clk_uart3_frac" }; + PNAME(mux_uart4_p) = { "clk_uart4_src", "clk_uart4_np5", "clk_uart4_frac" }; + PNAME(mux_uart5_p) = { "clk_uart5_src", "clk_uart5_np5", "clk_uart5_frac" }; +-PNAME(mux_cif_out_p) = { "xin24m", "dummy_cpll", "npll", "usb480m" }; ++PNAME(mux_cif_out_p) = { "xin24m", "dummy_cpll", "dummy_npll", "usb480m" }; + PNAME(mux_dclk_vopb_p) = { "dclk_vopb_src", "dclk_vopb_frac", "xin24m" }; + PNAME(mux_dclk_vopl_p) = { "dclk_vopl_src", "dclk_vopl_frac", "xin24m" }; + PNAME(mux_nandc_p) = { "clk_nandc_div", "clk_nandc_div50" }; +@@ -191,7 +196,7 @@ static struct rockchip_pll_clock px30_pll_clks[] __initdata = { + 0, PX30_PLL_CON(16), + PX30_MODE_CON, 2, 2, 0, px30_pll_rates), + [npll] = PLL(pll_rk3328, PLL_NPLL, "npll", mux_pll_p, +- 0, PX30_PLL_CON(24), ++ CLK_IS_CRITICAL, PX30_PLL_CON(24), + PX30_MODE_CON, 6, 4, 0, px30_pll_rates), + }; + +@@ -209,11 +214,11 @@ static struct rockchip_clk_branch px30_pdm_fracmux __initdata = + PX30_CLKSEL_CON(26), 15, 1, MFLAGS); + + static struct rockchip_clk_branch px30_i2s0_tx_fracmux __initdata = +- MUX(0, "clk_i2s0_tx_mux", mux_i2s0_tx_p, CLK_SET_RATE_PARENT, ++ MUX(SCLK_I2S0_TX_MUX, "clk_i2s0_tx_mux", mux_i2s0_tx_p, CLK_SET_RATE_PARENT, + PX30_CLKSEL_CON(28), 10, 2, MFLAGS); + + static struct rockchip_clk_branch px30_i2s0_rx_fracmux __initdata = +- MUX(0, "clk_i2s0_rx_mux", mux_i2s0_rx_p, CLK_SET_RATE_PARENT, ++ MUX(SCLK_I2S0_RX_MUX, "clk_i2s0_rx_mux", mux_i2s0_rx_p, CLK_SET_RATE_PARENT, + PX30_CLKSEL_CON(58), 10, 2, MFLAGS); + + static struct rockchip_clk_branch px30_i2s1_fracmux __initdata = +@@ -301,22 +306,12 @@ static struct rockchip_clk_branch px30_clk_branches[] __initdata = { + PX30_CLKGATE_CON(17), 4, GFLAGS), + + /* PD_GPU */ +- COMPOSITE_NODIV(0, "clk_gpu_src", mux_4plls_p, 0, +- PX30_CLKSEL_CON(1), 6, 2, MFLAGS, +- PX30_CLKGATE_CON(0), 8, GFLAGS), +- COMPOSITE_NOMUX(0, "clk_gpu_div", "clk_gpu_src", 0, +- PX30_CLKSEL_CON(1), 0, 4, DFLAGS, +- PX30_CLKGATE_CON(0), 12, GFLAGS), +- COMPOSITE_NOMUX_HALFDIV(0, "clk_gpu_np5", "clk_gpu_src", 0, +- PX30_CLKSEL_CON(1), 8, 4, DFLAGS, +- PX30_CLKGATE_CON(0), 9, GFLAGS), +- COMPOSITE_NODIV(SCLK_GPU, "clk_gpu", mux_gpu_p, CLK_SET_RATE_PARENT, +- PX30_CLKSEL_CON(1), 15, 1, MFLAGS, ++ GATE(SCLK_GPU, "clk_gpu", "clk_gpu_src", 0, + PX30_CLKGATE_CON(0), 10, GFLAGS), + COMPOSITE_NOMUX(0, "aclk_gpu", "clk_gpu", CLK_IGNORE_UNUSED, + PX30_CLKSEL_CON(1), 13, 2, DFLAGS, + PX30_CLKGATE_CON(17), 10, GFLAGS), +- GATE(0, "aclk_gpu_niu", "aclk_gpu", CLK_IGNORE_UNUSED, ++ GATE(0, "aclk_gpu_niu", "aclk_gpu", CLK_IS_CRITICAL, + PX30_CLKGATE_CON(0), 11, GFLAGS), + GATE(0, "aclk_gpu_prf", "aclk_gpu", CLK_IGNORE_UNUSED, + PX30_CLKGATE_CON(17), 8, GFLAGS), +@@ -424,16 +419,16 @@ static struct rockchip_clk_branch px30_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "dclk_vopb_frac", "dclk_vopb_src", CLK_SET_RATE_PARENT, + PX30_CLKSEL_CON(6), 0, + PX30_CLKGATE_CON(2), 3, GFLAGS, +- &px30_dclk_vopb_fracmux), ++ &px30_dclk_vopb_fracmux, 0), + GATE(DCLK_VOPB, "dclk_vopb", "dclk_vopb_mux", CLK_SET_RATE_PARENT, + PX30_CLKGATE_CON(2), 4, GFLAGS), +- COMPOSITE(0, "dclk_vopl_src", mux_npll_cpll_p, 0, ++ COMPOSITE(0, "dclk_vopl_src", mux_npll_cpll_p, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, + PX30_CLKSEL_CON(8), 11, 1, MFLAGS, 0, 8, DFLAGS, + PX30_CLKGATE_CON(2), 6, GFLAGS), + COMPOSITE_FRACMUX(0, "dclk_vopl_frac", "dclk_vopl_src", CLK_SET_RATE_PARENT, + PX30_CLKSEL_CON(9), 0, + PX30_CLKGATE_CON(2), 7, GFLAGS, +- &px30_dclk_vopl_fracmux), ++ &px30_dclk_vopl_fracmux, 0), + GATE(DCLK_VOPL, "dclk_vopl", "dclk_vopl_mux", CLK_SET_RATE_PARENT, + PX30_CLKGATE_CON(2), 8, GFLAGS), + +@@ -452,13 +447,13 @@ static struct rockchip_clk_branch px30_clk_branches[] __initdata = { + * Clock-Architecture Diagram 7 + */ + +- COMPOSITE_NODIV(ACLK_PERI_SRC, "aclk_peri_src", mux_gpll_cpll_p, 0, ++ COMPOSITE_NODIV(ACLK_PERI_SRC, "aclk_peri_src", mux_gpll_cpll_p, CLK_IS_CRITICAL, + PX30_CLKSEL_CON(14), 15, 1, MFLAGS, + PX30_CLKGATE_CON(5), 7, GFLAGS), +- COMPOSITE_NOMUX(ACLK_PERI_PRE, "aclk_peri_pre", "aclk_peri_src", CLK_IGNORE_UNUSED, ++ COMPOSITE_NOMUX(ACLK_PERI_PRE, "aclk_peri_pre", "aclk_peri_src", CLK_IS_CRITICAL, + PX30_CLKSEL_CON(14), 0, 5, DFLAGS, + PX30_CLKGATE_CON(5), 8, GFLAGS), +- DIV(HCLK_PERI_PRE, "hclk_peri_pre", "aclk_peri_src", CLK_IGNORE_UNUSED, ++ DIV(HCLK_PERI_PRE, "hclk_peri_pre", "aclk_peri_src", CLK_IS_CRITICAL, + PX30_CLKSEL_CON(14), 8, 5, DFLAGS), + + /* PD_MMC_NAND */ +@@ -535,7 +530,7 @@ static struct rockchip_clk_branch px30_clk_branches[] __initdata = { + PX30_CLKGATE_CON(6), 15, GFLAGS), + + /* PD_USB */ +- GATE(HCLK_USB, "hclk_usb", "hclk_peri_pre", 0, ++ GATE(HCLK_USB, "hclk_usb", "hclk_peri_pre", CLK_IS_CRITICAL, + PX30_CLKGATE_CON(7), 2, GFLAGS), + GATE(SCLK_OTG_ADP, "clk_otg_adp", "clk_rtc32k_pmu", 0, + PX30_CLKGATE_CON(7), 3, GFLAGS), +@@ -570,19 +565,19 @@ static struct rockchip_clk_branch px30_clk_branches[] __initdata = { + */ + + /* PD_BUS */ +- COMPOSITE_NODIV(ACLK_BUS_SRC, "aclk_bus_src", mux_gpll_cpll_p, CLK_IGNORE_UNUSED, ++ COMPOSITE_NODIV(ACLK_BUS_SRC, "aclk_bus_src", mux_gpll_cpll_p, CLK_IS_CRITICAL, + PX30_CLKSEL_CON(23), 15, 1, MFLAGS, + PX30_CLKGATE_CON(8), 6, GFLAGS), +- COMPOSITE_NOMUX(HCLK_BUS_PRE, "hclk_bus_pre", "aclk_bus_src", CLK_IGNORE_UNUSED, ++ COMPOSITE_NOMUX(HCLK_BUS_PRE, "hclk_bus_pre", "aclk_bus_src", CLK_IS_CRITICAL, + PX30_CLKSEL_CON(24), 0, 5, DFLAGS, + PX30_CLKGATE_CON(8), 8, GFLAGS), +- COMPOSITE_NOMUX(ACLK_BUS_PRE, "aclk_bus_pre", "aclk_bus_src", CLK_IGNORE_UNUSED, ++ COMPOSITE_NOMUX(ACLK_BUS_PRE, "aclk_bus_pre", "aclk_bus_src", CLK_IS_CRITICAL, + PX30_CLKSEL_CON(23), 8, 5, DFLAGS, + PX30_CLKGATE_CON(8), 7, GFLAGS), +- COMPOSITE_NOMUX(PCLK_BUS_PRE, "pclk_bus_pre", "aclk_bus_pre", CLK_IGNORE_UNUSED, ++ COMPOSITE_NOMUX(PCLK_BUS_PRE, "pclk_bus_pre", "aclk_bus_pre", CLK_IS_CRITICAL, + PX30_CLKSEL_CON(24), 8, 2, DFLAGS, + PX30_CLKGATE_CON(8), 9, GFLAGS), +- GATE(0, "pclk_top_pre", "pclk_bus_pre", CLK_IGNORE_UNUSED, ++ GATE(0, "pclk_top_pre", "pclk_bus_pre", CLK_IS_CRITICAL, + PX30_CLKGATE_CON(8), 10, GFLAGS), + + COMPOSITE(0, "clk_pdm_src", mux_gpll_xin24m_npll_p, 0, +@@ -591,7 +586,7 @@ static struct rockchip_clk_branch px30_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "clk_pdm_frac", "clk_pdm_src", CLK_SET_RATE_PARENT, + PX30_CLKSEL_CON(27), 0, + PX30_CLKGATE_CON(9), 10, GFLAGS, +- &px30_pdm_fracmux), ++ &px30_pdm_fracmux, PX30_FRAC_MAX_PRATE), + GATE(SCLK_PDM, "clk_pdm", "clk_pdm_mux", CLK_SET_RATE_PARENT, + PX30_CLKGATE_CON(9), 11, GFLAGS), + +@@ -601,11 +596,11 @@ static struct rockchip_clk_branch px30_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "clk_i2s0_tx_frac", "clk_i2s0_tx_src", CLK_SET_RATE_PARENT, + PX30_CLKSEL_CON(29), 0, + PX30_CLKGATE_CON(9), 13, GFLAGS, +- &px30_i2s0_tx_fracmux), ++ &px30_i2s0_tx_fracmux, PX30_FRAC_MAX_PRATE), + COMPOSITE_NODIV(SCLK_I2S0_TX, "clk_i2s0_tx", mux_i2s0_tx_rx_p, CLK_SET_RATE_PARENT, + PX30_CLKSEL_CON(28), 12, 1, MFLAGS, + PX30_CLKGATE_CON(9), 14, GFLAGS), +- COMPOSITE_NODIV(0, "clk_i2s0_tx_out_pre", mux_i2s0_tx_out_p, 0, ++ COMPOSITE_NODIV(0, "clk_i2s0_tx_out_pre", mux_i2s0_tx_out_p, CLK_SET_RATE_PARENT, + PX30_CLKSEL_CON(28), 14, 2, MFLAGS, + PX30_CLKGATE_CON(9), 15, GFLAGS), + GATE(SCLK_I2S0_TX_OUT, "clk_i2s0_tx_out", "clk_i2s0_tx_out_pre", CLK_SET_RATE_PARENT, +@@ -617,7 +612,7 @@ static struct rockchip_clk_branch px30_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "clk_i2s0_rx_frac", "clk_i2s0_rx_src", CLK_SET_RATE_PARENT, + PX30_CLKSEL_CON(59), 0, + PX30_CLKGATE_CON(17), 1, GFLAGS, +- &px30_i2s0_rx_fracmux), ++ &px30_i2s0_rx_fracmux, PX30_FRAC_MAX_PRATE), + COMPOSITE_NODIV(SCLK_I2S0_RX, "clk_i2s0_rx", mux_i2s0_rx_tx_p, CLK_SET_RATE_PARENT, + PX30_CLKSEL_CON(58), 12, 1, MFLAGS, + PX30_CLKGATE_CON(17), 2, GFLAGS), +@@ -633,10 +628,10 @@ static struct rockchip_clk_branch px30_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "clk_i2s1_frac", "clk_i2s1_src", CLK_SET_RATE_PARENT, + PX30_CLKSEL_CON(31), 0, + PX30_CLKGATE_CON(10), 1, GFLAGS, +- &px30_i2s1_fracmux), ++ &px30_i2s1_fracmux, PX30_FRAC_MAX_PRATE), + GATE(SCLK_I2S1, "clk_i2s1", "clk_i2s1_mux", CLK_SET_RATE_PARENT, + PX30_CLKGATE_CON(10), 2, GFLAGS), +- COMPOSITE_NODIV(0, "clk_i2s1_out_pre", mux_i2s1_out_p, 0, ++ COMPOSITE_NODIV(0, "clk_i2s1_out_pre", mux_i2s1_out_p, CLK_SET_RATE_PARENT, + PX30_CLKSEL_CON(30), 15, 1, MFLAGS, + PX30_CLKGATE_CON(10), 3, GFLAGS), + GATE(SCLK_I2S1_OUT, "clk_i2s1_out", "clk_i2s1_out_pre", CLK_SET_RATE_PARENT, +@@ -648,7 +643,7 @@ static struct rockchip_clk_branch px30_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "clk_i2s2_frac", "clk_i2s2_src", CLK_SET_RATE_PARENT, + PX30_CLKSEL_CON(33), 0, + PX30_CLKGATE_CON(10), 5, GFLAGS, +- &px30_i2s2_fracmux), ++ &px30_i2s2_fracmux, PX30_FRAC_MAX_PRATE), + GATE(SCLK_I2S2, "clk_i2s2", "clk_i2s2_mux", CLK_SET_RATE_PARENT, + PX30_CLKGATE_CON(10), 6, GFLAGS), + COMPOSITE_NODIV(0, "clk_i2s2_out_pre", mux_i2s2_out_p, 0, +@@ -666,7 +661,7 @@ static struct rockchip_clk_branch px30_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "clk_uart1_frac", "clk_uart1_src", CLK_SET_RATE_PARENT, + PX30_CLKSEL_CON(36), 0, + PX30_CLKGATE_CON(10), 14, GFLAGS, +- &px30_uart1_fracmux), ++ &px30_uart1_fracmux, PX30_FRAC_MAX_PRATE), + GATE(SCLK_UART1, "clk_uart1", "clk_uart1_mux", CLK_SET_RATE_PARENT, + PX30_CLKGATE_CON(10), 15, GFLAGS), + +@@ -679,8 +674,8 @@ static struct rockchip_clk_branch px30_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "clk_uart2_frac", "clk_uart2_src", CLK_SET_RATE_PARENT, + PX30_CLKSEL_CON(39), 0, + PX30_CLKGATE_CON(11), 2, GFLAGS, +- &px30_uart2_fracmux), +- GATE(SCLK_UART2, "clk_uart2", "clk_uart2_mux", CLK_SET_RATE_PARENT, ++ &px30_uart2_fracmux, PX30_FRAC_MAX_PRATE), ++ GATE(SCLK_UART2, "clk_uart2", "clk_uart2_mux", CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, + PX30_CLKGATE_CON(11), 3, GFLAGS), + + COMPOSITE(0, "clk_uart3_src", mux_uart_src_p, 0, +@@ -692,7 +687,7 @@ static struct rockchip_clk_branch px30_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "clk_uart3_frac", "clk_uart3_src", CLK_SET_RATE_PARENT, + PX30_CLKSEL_CON(42), 0, + PX30_CLKGATE_CON(11), 6, GFLAGS, +- &px30_uart3_fracmux), ++ &px30_uart3_fracmux, PX30_FRAC_MAX_PRATE), + GATE(SCLK_UART3, "clk_uart3", "clk_uart3_mux", CLK_SET_RATE_PARENT, + PX30_CLKGATE_CON(11), 7, GFLAGS), + +@@ -705,7 +700,7 @@ static struct rockchip_clk_branch px30_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "clk_uart4_frac", "clk_uart4_src", CLK_SET_RATE_PARENT, + PX30_CLKSEL_CON(45), 0, + PX30_CLKGATE_CON(11), 10, GFLAGS, +- &px30_uart4_fracmux), ++ &px30_uart4_fracmux, PX30_FRAC_MAX_PRATE), + GATE(SCLK_UART4, "clk_uart4", "clk_uart4_mux", CLK_SET_RATE_PARENT, + PX30_CLKGATE_CON(11), 11, GFLAGS), + +@@ -718,7 +713,7 @@ static struct rockchip_clk_branch px30_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "clk_uart5_frac", "clk_uart5_src", CLK_SET_RATE_PARENT, + PX30_CLKSEL_CON(48), 0, + PX30_CLKGATE_CON(11), 14, GFLAGS, +- &px30_uart5_fracmux), ++ &px30_uart5_fracmux, PX30_FRAC_MAX_PRATE), + GATE(SCLK_UART5, "clk_uart5", "clk_uart5_mux", CLK_SET_RATE_PARENT, + PX30_CLKGATE_CON(11), 15, GFLAGS), + +@@ -799,29 +794,29 @@ static struct rockchip_clk_branch px30_clk_branches[] __initdata = { + GATE(0, "pclk_ddrphy", "pclk_top_pre", CLK_IGNORE_UNUSED, PX30_CLKGATE_CON(16), 3, GFLAGS), + GATE(PCLK_MIPIDSIPHY, "pclk_mipidsiphy", "pclk_top_pre", 0, PX30_CLKGATE_CON(16), 4, GFLAGS), + GATE(PCLK_MIPICSIPHY, "pclk_mipicsiphy", "pclk_top_pre", 0, PX30_CLKGATE_CON(16), 5, GFLAGS), +- GATE(PCLK_USB_GRF, "pclk_usb_grf", "pclk_top_pre", 0, PX30_CLKGATE_CON(16), 6, GFLAGS), ++ GATE(PCLK_USB_GRF, "pclk_usb_grf", "pclk_top_pre", CLK_IS_CRITICAL, PX30_CLKGATE_CON(16), 6, GFLAGS), + GATE(0, "pclk_cpu_hoost", "pclk_top_pre", CLK_IGNORE_UNUSED, PX30_CLKGATE_CON(16), 7, GFLAGS), + + /* PD_VI */ +- GATE(0, "aclk_vi_niu", "aclk_vi_pre", 0, PX30_CLKGATE_CON(4), 15, GFLAGS), ++ GATE(0, "aclk_vi_niu", "aclk_vi_pre", CLK_IS_CRITICAL, PX30_CLKGATE_CON(4), 15, GFLAGS), + GATE(ACLK_CIF, "aclk_cif", "aclk_vi_pre", 0, PX30_CLKGATE_CON(5), 1, GFLAGS), + GATE(ACLK_ISP, "aclk_isp", "aclk_vi_pre", 0, PX30_CLKGATE_CON(5), 3, GFLAGS), +- GATE(0, "hclk_vi_niu", "hclk_vi_pre", 0, PX30_CLKGATE_CON(5), 0, GFLAGS), ++ GATE(0, "hclk_vi_niu", "hclk_vi_pre", CLK_IS_CRITICAL, PX30_CLKGATE_CON(5), 0, GFLAGS), + GATE(HCLK_CIF, "hclk_cif", "hclk_vi_pre", 0, PX30_CLKGATE_CON(5), 2, GFLAGS), + GATE(HCLK_ISP, "hclk_isp", "hclk_vi_pre", 0, PX30_CLKGATE_CON(5), 4, GFLAGS), + + /* PD_VO */ +- GATE(0, "aclk_vo_niu", "aclk_vo_pre", 0, PX30_CLKGATE_CON(3), 0, GFLAGS), ++ GATE(0, "aclk_vo_niu", "aclk_vo_pre", CLK_IS_CRITICAL, PX30_CLKGATE_CON(3), 0, GFLAGS), + GATE(ACLK_VOPB, "aclk_vopb", "aclk_vo_pre", 0, PX30_CLKGATE_CON(3), 3, GFLAGS), + GATE(ACLK_RGA, "aclk_rga", "aclk_vo_pre", 0, PX30_CLKGATE_CON(3), 7, GFLAGS), + GATE(ACLK_VOPL, "aclk_vopl", "aclk_vo_pre", 0, PX30_CLKGATE_CON(3), 5, GFLAGS), + +- GATE(0, "hclk_vo_niu", "hclk_vo_pre", 0, PX30_CLKGATE_CON(3), 1, GFLAGS), ++ GATE(0, "hclk_vo_niu", "hclk_vo_pre", CLK_IS_CRITICAL, PX30_CLKGATE_CON(3), 1, GFLAGS), + GATE(HCLK_VOPB, "hclk_vopb", "hclk_vo_pre", 0, PX30_CLKGATE_CON(3), 4, GFLAGS), + GATE(HCLK_RGA, "hclk_rga", "hclk_vo_pre", 0, PX30_CLKGATE_CON(3), 8, GFLAGS), + GATE(HCLK_VOPL, "hclk_vopl", "hclk_vo_pre", 0, PX30_CLKGATE_CON(3), 6, GFLAGS), + +- GATE(0, "pclk_vo_niu", "pclk_vo_pre", 0, PX30_CLKGATE_CON(3), 2, GFLAGS), ++ GATE(0, "pclk_vo_niu", "pclk_vo_pre", CLK_IS_CRITICAL, PX30_CLKGATE_CON(3), 2, GFLAGS), + GATE(PCLK_MIPI_DSI, "pclk_mipi_dsi", "pclk_vo_pre", 0, PX30_CLKGATE_CON(3), 9, GFLAGS), + + /* PD_BUS */ +@@ -843,7 +838,7 @@ static struct rockchip_clk_branch px30_clk_branches[] __initdata = { + GATE(0, "pclk_bus_niu", "pclk_bus_pre", CLK_IGNORE_UNUSED, PX30_CLKGATE_CON(13), 10, GFLAGS), + GATE(PCLK_DCF, "pclk_dcf", "pclk_bus_pre", 0, PX30_CLKGATE_CON(14), 0, GFLAGS), + GATE(PCLK_UART1, "pclk_uart1", "pclk_bus_pre", 0, PX30_CLKGATE_CON(14), 5, GFLAGS), +- GATE(PCLK_UART2, "pclk_uart2", "pclk_bus_pre", 0, PX30_CLKGATE_CON(14), 6, GFLAGS), ++ GATE(PCLK_UART2, "pclk_uart2", "pclk_bus_pre", CLK_IS_CRITICAL, PX30_CLKGATE_CON(14), 6, GFLAGS), + GATE(PCLK_UART3, "pclk_uart3", "pclk_bus_pre", 0, PX30_CLKGATE_CON(14), 7, GFLAGS), + GATE(PCLK_UART4, "pclk_uart4", "pclk_bus_pre", 0, PX30_CLKGATE_CON(14), 8, GFLAGS), + GATE(PCLK_UART5, "pclk_uart5", "pclk_bus_pre", 0, PX30_CLKGATE_CON(14), 9, GFLAGS), +@@ -884,7 +879,7 @@ static struct rockchip_clk_branch px30_clk_branches[] __initdata = { + GATE(HCLK_SDMMC, "hclk_sdmmc", "hclk_sdmmc_pre", 0, PX30_CLKGATE_CON(7), 1, GFLAGS), + + /* PD_PERI */ +- GATE(0, "aclk_peri_niu", "aclk_peri_pre", CLK_IGNORE_UNUSED, PX30_CLKGATE_CON(5), 9, GFLAGS), ++ GATE(0, "aclk_peri_niu", "aclk_peri_pre", CLK_IS_CRITICAL, PX30_CLKGATE_CON(5), 9, GFLAGS), + + /* PD_MMC_NAND */ + GATE(HCLK_NANDC, "hclk_nandc", "hclk_mmc_nand", 0, PX30_CLKGATE_CON(5), 15, GFLAGS), +@@ -894,7 +889,7 @@ static struct rockchip_clk_branch px30_clk_branches[] __initdata = { + GATE(HCLK_SFC, "hclk_sfc", "hclk_mmc_nand", 0, PX30_CLKGATE_CON(6), 11, GFLAGS), + + /* PD_USB */ +- GATE(0, "hclk_usb_niu", "hclk_usb", CLK_IGNORE_UNUSED, PX30_CLKGATE_CON(7), 4, GFLAGS), ++ GATE(0, "hclk_usb_niu", "hclk_usb", CLK_IS_CRITICAL, PX30_CLKGATE_CON(7), 4, GFLAGS), + GATE(HCLK_OTG, "hclk_otg", "hclk_usb", 0, PX30_CLKGATE_CON(7), 5, GFLAGS), + GATE(HCLK_HOST, "hclk_host", "hclk_usb", 0, PX30_CLKGATE_CON(7), 6, GFLAGS), + GATE(HCLK_HOST_ARB, "hclk_host_arb", "hclk_usb", CLK_IGNORE_UNUSED, PX30_CLKGATE_CON(7), 8, GFLAGS), +@@ -910,6 +905,18 @@ static struct rockchip_clk_branch px30_clk_branches[] __initdata = { + PX30_CLKGATE_CON(8), 3, GFLAGS), + }; + ++static struct rockchip_clk_branch px30_gpu_src_clk[] __initdata = { ++ COMPOSITE(0, "clk_gpu_src", mux_gpll_dmycpll_usb480m_dmynpll_p, 0, ++ PX30_CLKSEL_CON(1), 6, 2, MFLAGS, 0, 4, DFLAGS, ++ PX30_CLKGATE_CON(0), 8, GFLAGS), ++}; ++ ++static struct rockchip_clk_branch rk3326_gpu_src_clk[] __initdata = { ++ COMPOSITE(0, "clk_gpu_src", mux_gpll_dmycpll_usb480m_npll_p, 0, ++ PX30_CLKSEL_CON(1), 6, 2, MFLAGS, 0, 4, DFLAGS, ++ PX30_CLKGATE_CON(0), 8, GFLAGS), ++}; ++ + static struct rockchip_clk_branch px30_clk_pmu_branches[] __initdata = { + /* + * Clock-Architecture Diagram 2 +@@ -918,7 +925,7 @@ static struct rockchip_clk_branch px30_clk_pmu_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "clk_rtc32k_frac", "xin24m", CLK_IGNORE_UNUSED, + PX30_PMU_CLKSEL_CON(1), 0, + PX30_PMU_CLKGATE_CON(0), 13, GFLAGS, +- &px30_rtc32k_pmu_fracmux), ++ &px30_rtc32k_pmu_fracmux, 0), + + COMPOSITE_NOMUX(XIN24M_DIV, "xin24m_div", "xin24m", CLK_IGNORE_UNUSED, + PX30_PMU_CLKSEL_CON(0), 8, 5, DFLAGS, +@@ -940,14 +947,14 @@ static struct rockchip_clk_branch px30_clk_pmu_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "clk_uart0_frac", "clk_uart0_pmu_src", CLK_SET_RATE_PARENT, + PX30_PMU_CLKSEL_CON(5), 0, + PX30_PMU_CLKGATE_CON(1), 2, GFLAGS, +- &px30_uart0_pmu_fracmux), ++ &px30_uart0_pmu_fracmux, PX30_FRAC_MAX_PRATE), + GATE(SCLK_UART0_PMU, "clk_uart0_pmu", "clk_uart0_pmu_mux", CLK_SET_RATE_PARENT, + PX30_PMU_CLKGATE_CON(1), 3, GFLAGS), + + GATE(SCLK_PVTM_PMU, "clk_pvtm_pmu", "xin24m", 0, + PX30_PMU_CLKGATE_CON(1), 4, GFLAGS), + +- COMPOSITE_NOMUX(PCLK_PMU_PRE, "pclk_pmu_pre", "gpll", 0, ++ COMPOSITE_NOMUX(PCLK_PMU_PRE, "pclk_pmu_pre", "gpll", CLK_IS_CRITICAL, + PX30_PMU_CLKSEL_CON(0), 0, 5, DFLAGS, + PX30_PMU_CLKGATE_CON(0), 0, GFLAGS), + +@@ -976,28 +983,7 @@ static struct rockchip_clk_branch px30_clk_pmu_branches[] __initdata = { + GATE(0, "pclk_cru_pmu", "pclk_pmu_pre", CLK_IGNORE_UNUSED, PX30_PMU_CLKGATE_CON(0), 8, GFLAGS), + }; + +-static const char *const px30_cru_critical_clocks[] __initconst = { +- "aclk_bus_pre", +- "pclk_bus_pre", +- "hclk_bus_pre", +- "aclk_peri_pre", +- "hclk_peri_pre", +- "aclk_gpu_niu", +- "pclk_top_pre", +- "pclk_pmu_pre", +- "hclk_usb_niu", +- "pclk_vo_niu", +- "aclk_vo_niu", +- "hclk_vo_niu", +- "aclk_vi_niu", +- "hclk_vi_niu", +- "pll_npll", +- "usb480m", +- "clk_uart2", +- "pclk_uart2", +- "pclk_usb_grf", +-}; +- ++static struct rockchip_clk_provider *cru_ctx; + static void __init px30_clk_init(struct device_node *np) + { + struct rockchip_clk_provider *ctx; +@@ -1021,14 +1007,12 @@ static void __init px30_clk_init(struct device_node *np) + PX30_GRF_SOC_STATUS0); + rockchip_clk_register_branches(ctx, px30_clk_branches, + ARRAY_SIZE(px30_clk_branches)); +- +- rockchip_clk_register_armclk(ctx, ARMCLK, "armclk", +- mux_armclk_p, ARRAY_SIZE(mux_armclk_p), +- &px30_cpuclk_data, px30_cpuclk_rates, +- ARRAY_SIZE(px30_cpuclk_rates)); +- +- rockchip_clk_protect_critical(px30_cru_critical_clocks, +- ARRAY_SIZE(px30_cru_critical_clocks)); ++ if (of_machine_is_compatible("rockchip,px30")) ++ rockchip_clk_register_branches(ctx, px30_gpu_src_clk, ++ ARRAY_SIZE(px30_gpu_src_clk)); ++ else ++ rockchip_clk_register_branches(ctx, rk3326_gpu_src_clk, ++ ARRAY_SIZE(rk3326_gpu_src_clk)); + + rockchip_register_softrst(np, 12, reg_base + PX30_SOFTRST_CON(0), + ROCKCHIP_SOFTRST_HIWORD_MASK); +@@ -1036,6 +1020,8 @@ static void __init px30_clk_init(struct device_node *np) + rockchip_register_restart_notifier(ctx, PX30_GLB_SRST_FST, NULL); + + rockchip_clk_of_add_provider(np, ctx); ++ ++ cru_ctx = ctx; + } + CLK_OF_DECLARE(px30_cru, "rockchip,px30-cru", px30_clk_init); + +@@ -1043,6 +1029,7 @@ static void __init px30_pmu_clk_init(struct device_node *np) + { + struct rockchip_clk_provider *ctx; + void __iomem *reg_base; ++ struct clk **pmucru_clks, **cru_clks; + + reg_base = of_iomap(np, 0); + if (!reg_base) { +@@ -1055,13 +1042,72 @@ static void __init px30_pmu_clk_init(struct device_node *np) + pr_err("%s: rockchip pmu clk init failed\n", __func__); + return; + } ++ pmucru_clks = ctx->clk_data.clks; ++ cru_clks = cru_ctx->clk_data.clks; + + rockchip_clk_register_plls(ctx, px30_pmu_pll_clks, + ARRAY_SIZE(px30_pmu_pll_clks), PX30_GRF_SOC_STATUS0); + ++ rockchip_clk_register_armclk(cru_ctx, ARMCLK, "armclk", ++ 2, cru_clks[PLL_APLL], pmucru_clks[PLL_GPLL], ++ &px30_cpuclk_data, px30_cpuclk_rates, ++ ARRAY_SIZE(px30_cpuclk_rates)); ++ + rockchip_clk_register_branches(ctx, px30_clk_pmu_branches, + ARRAY_SIZE(px30_clk_pmu_branches)); + + rockchip_clk_of_add_provider(np, ctx); + } + CLK_OF_DECLARE(px30_cru_pmu, "rockchip,px30-pmucru", px30_pmu_clk_init); ++ ++struct clk_px30_inits { ++ void (*inits)(struct device_node *np); ++}; ++ ++static const struct clk_px30_inits clk_px30_init = { ++ .inits = px30_clk_init, ++}; ++ ++static const struct clk_px30_inits clk_px30_pmu_init = { ++ .inits = px30_pmu_clk_init, ++}; ++ ++static const struct of_device_id clk_px30_match_table[] = { ++ { ++ .compatible = "rockchip,px30-cru", ++ .data = &clk_px30_init, ++ }, { ++ .compatible = "rockchip,px30-pmucru", ++ .data = &clk_px30_pmu_init, ++ }, ++ { } ++}; ++MODULE_DEVICE_TABLE(of, clk_px30_match_table); ++ ++static int __init clk_px30_probe(struct platform_device *pdev) ++{ ++ struct device_node *np = pdev->dev.of_node; ++ const struct of_device_id *match; ++ const struct clk_px30_inits *init_data; ++ ++ match = of_match_device(clk_px30_match_table, &pdev->dev); ++ if (!match || !match->data) ++ return -EINVAL; ++ ++ init_data = match->data; ++ if (init_data->inits) ++ init_data->inits(np); ++ ++ return 0; ++} ++ ++static struct platform_driver clk_px30_driver = { ++ .driver = { ++ .name = "clk-px30", ++ .of_match_table = clk_px30_match_table, ++ }, ++}; ++builtin_platform_driver_probe(clk_px30_driver, clk_px30_probe); ++ ++MODULE_DESCRIPTION("Rockchip PX30 Clock Driver"); ++MODULE_LICENSE("GPL"); +diff --git a/drivers/clk/rockchip/clk-rk1808.c b/drivers/clk/rockchip/clk-rk1808.c +new file mode 100755 +index 000000000000..cb9483623f13 +--- /dev/null ++++ b/drivers/clk/rockchip/clk-rk1808.c +@@ -0,0 +1,1249 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd ++ * Author: Elaine Zhang ++ */ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include "clk.h" ++ ++#define RK1808_GRF_SOC_STATUS0 0x480 ++#define RK1808_PMUGRF_SOC_CON0 0x100 ++#define RK1808_UART_FRAC_MAX_PRATE 800000000 ++#define RK1808_PDM_FRAC_MAX_PRATE 300000000 ++#define RK1808_I2S_FRAC_MAX_PRATE 600000000 ++#define RK1808_VOP_RAW_FRAC_MAX_PRATE 300000000 ++#define RK1808_VOP_LITE_FRAC_MAX_PRATE 400000000 ++ ++enum rk1808_plls { ++ apll, dpll, cpll, gpll, npll, ppll, ++}; ++ ++static struct rockchip_pll_rate_table rk1808_pll_rates[] = { ++ /* _mhz, _refdiv, _fbdiv, _postdiv1, _postdiv2, _dsmpd, _frac */ ++ RK3036_PLL_RATE(1608000000, 1, 67, 1, 1, 1, 0), ++ RK3036_PLL_RATE(1584000000, 1, 66, 1, 1, 1, 0), ++ RK3036_PLL_RATE(1560000000, 1, 65, 1, 1, 1, 0), ++ RK3036_PLL_RATE(1536000000, 1, 64, 1, 1, 1, 0), ++ RK3036_PLL_RATE(1512000000, 1, 63, 1, 1, 1, 0), ++ RK3036_PLL_RATE(1488000000, 1, 62, 1, 1, 1, 0), ++ RK3036_PLL_RATE(1464000000, 1, 61, 1, 1, 1, 0), ++ RK3036_PLL_RATE(1440000000, 1, 60, 1, 1, 1, 0), ++ RK3036_PLL_RATE(1416000000, 1, 59, 1, 1, 1, 0), ++ RK3036_PLL_RATE(1392000000, 1, 58, 1, 1, 1, 0), ++ RK3036_PLL_RATE(1368000000, 1, 57, 1, 1, 1, 0), ++ RK3036_PLL_RATE(1344000000, 1, 56, 1, 1, 1, 0), ++ RK3036_PLL_RATE(1320000000, 1, 55, 1, 1, 1, 0), ++ RK3036_PLL_RATE(1296000000, 1, 54, 1, 1, 1, 0), ++ RK3036_PLL_RATE(1272000000, 1, 53, 1, 1, 1, 0), ++ RK3036_PLL_RATE(1248000000, 1, 52, 1, 1, 1, 0), ++ RK3036_PLL_RATE(1200000000, 1, 50, 1, 1, 1, 0), ++ RK3036_PLL_RATE(1188000000, 1, 99, 2, 1, 1, 0), ++ RK3036_PLL_RATE(1104000000, 1, 46, 1, 1, 1, 0), ++ RK3036_PLL_RATE(1100000000, 2, 275, 3, 1, 1, 0), ++ RK3036_PLL_RATE(1008000000, 1, 84, 2, 1, 1, 0), ++ RK3036_PLL_RATE(1000000000, 1, 125, 3, 1, 1, 0), ++ RK3036_PLL_RATE(984000000, 1, 82, 2, 1, 1, 0), ++ RK3036_PLL_RATE(960000000, 1, 80, 2, 1, 1, 0), ++ RK3036_PLL_RATE(936000000, 1, 78, 2, 1, 1, 0), ++ RK3036_PLL_RATE(912000000, 1, 76, 2, 1, 1, 0), ++ RK3036_PLL_RATE(900000000, 1, 75, 2, 1, 1, 0), ++ RK3036_PLL_RATE(888000000, 1, 74, 2, 1, 1, 0), ++ RK3036_PLL_RATE(864000000, 1, 72, 2, 1, 1, 0), ++ RK3036_PLL_RATE(840000000, 1, 70, 2, 1, 1, 0), ++ RK3036_PLL_RATE(816000000, 1, 68, 2, 1, 1, 0), ++ RK3036_PLL_RATE(800000000, 1, 100, 3, 1, 1, 0), ++ RK3036_PLL_RATE(700000000, 1, 175, 2, 1, 1, 0), ++ RK3036_PLL_RATE(696000000, 1, 58, 2, 1, 1, 0), ++ RK3036_PLL_RATE(624000000, 1, 52, 2, 1, 1, 0), ++ RK3036_PLL_RATE(600000000, 1, 75, 3, 1, 1, 0), ++ RK3036_PLL_RATE(594000000, 1, 99, 4, 1, 1, 0), ++ RK3036_PLL_RATE(504000000, 1, 63, 3, 1, 1, 0), ++ RK3036_PLL_RATE(500000000, 1, 125, 6, 1, 1, 0), ++ RK3036_PLL_RATE(416000000, 1, 52, 3, 1, 1, 0), ++ RK3036_PLL_RATE(408000000, 1, 68, 2, 2, 1, 0), ++ RK3036_PLL_RATE(312000000, 1, 52, 2, 2, 1, 0), ++ RK3036_PLL_RATE(216000000, 1, 72, 4, 2, 1, 0), ++ RK3036_PLL_RATE(200000000, 1, 200, 6, 4, 1, 0), ++ RK3036_PLL_RATE(100000000, 1, 150, 6, 6, 1, 0), ++ RK3036_PLL_RATE(96000000, 1, 64, 4, 4, 1, 0), ++ { /* sentinel */ }, ++}; ++ ++#define RK1808_DIV_ACLKM_MASK 0x7 ++#define RK1808_DIV_ACLKM_SHIFT 12 ++#define RK1808_DIV_PCLK_DBG_MASK 0xf ++#define RK1808_DIV_PCLK_DBG_SHIFT 8 ++ ++#define RK1808_CLKSEL0(_aclk_core, _pclk_dbg) \ ++{ \ ++ .reg = RK1808_CLKSEL_CON(0), \ ++ .val = HIWORD_UPDATE(_aclk_core, RK1808_DIV_ACLKM_MASK, \ ++ RK1808_DIV_ACLKM_SHIFT) | \ ++ HIWORD_UPDATE(_pclk_dbg, RK1808_DIV_PCLK_DBG_MASK, \ ++ RK1808_DIV_PCLK_DBG_SHIFT), \ ++} ++ ++#define RK1808_CPUCLK_RATE(_prate, _aclk_core, _pclk_dbg) \ ++{ \ ++ .prate = _prate, \ ++ .divs = { \ ++ RK1808_CLKSEL0(_aclk_core, _pclk_dbg), \ ++ }, \ ++} ++ ++static struct rockchip_cpuclk_rate_table rk1808_cpuclk_rates[] __initdata = { ++ RK1808_CPUCLK_RATE(1608000000, 1, 7), ++ RK1808_CPUCLK_RATE(1512000000, 1, 7), ++ RK1808_CPUCLK_RATE(1488000000, 1, 5), ++ RK1808_CPUCLK_RATE(1416000000, 1, 5), ++ RK1808_CPUCLK_RATE(1392000000, 1, 5), ++ RK1808_CPUCLK_RATE(1296000000, 1, 5), ++ RK1808_CPUCLK_RATE(1200000000, 1, 5), ++ RK1808_CPUCLK_RATE(1104000000, 1, 5), ++ RK1808_CPUCLK_RATE(1008000000, 1, 5), ++ RK1808_CPUCLK_RATE(912000000, 1, 5), ++ RK1808_CPUCLK_RATE(816000000, 1, 3), ++ RK1808_CPUCLK_RATE(696000000, 1, 3), ++ RK1808_CPUCLK_RATE(600000000, 1, 3), ++ RK1808_CPUCLK_RATE(408000000, 1, 1), ++ RK1808_CPUCLK_RATE(312000000, 1, 1), ++ RK1808_CPUCLK_RATE(216000000, 1, 1), ++ RK1808_CPUCLK_RATE(96000000, 1, 1), ++}; ++ ++static const struct rockchip_cpuclk_reg_data rk1808_cpuclk_data = { ++ .core_reg[0] = RK1808_CLKSEL_CON(0), ++ .div_core_shift[0] = 0, ++ .div_core_mask[0] = 0xf, ++ .num_cores = 1, ++ .mux_core_alt = 2, ++ .mux_core_main = 0, ++ .mux_core_shift = 6, ++ .mux_core_mask = 0x3, ++}; ++ ++PNAME(mux_pll_p) = { "xin24m", "xin32k"}; ++PNAME(mux_usb480m_p) = { "xin24m", "usb480m_phy", "xin32k" }; ++PNAME(mux_gpll_cpll_p) = { "gpll", "cpll" }; ++PNAME(mux_gpll_cpll_apll_p) = { "gpll", "cpll", "apll" }; ++PNAME(mux_npu_p) = { "clk_npu_div", "clk_npu_np5" }; ++PNAME(mux_ddr_p) = { "dpll_ddr", "gpll_ddr" }; ++PNAME(mux_cpll_gpll_npll_p) = { "cpll", "gpll", "npll" }; ++PNAME(mux_gpll_cpll_npll_p) = { "gpll", "cpll", "npll" }; ++PNAME(mux_dclk_vopraw_p) = { "dclk_vopraw_src", "dclk_vopraw_frac", "xin24m" }; ++PNAME(mux_dclk_voplite_p) = { "dclk_voplite_src", "dclk_voplite_frac", "xin24m" }; ++PNAME(mux_24m_npll_gpll_usb480m_p) = { "xin24m", "npll", "gpll", "usb480m" }; ++PNAME(mux_usb3_otg0_suspend_p) = { "xin32k", "xin24m" }; ++PNAME(mux_pcie_aux_p) = { "xin24m", "clk_pcie_src" }; ++PNAME(mux_gpll_cpll_npll_24m_p) = { "gpll", "cpll", "npll", "xin24m" }; ++PNAME(mux_sdio_p) = { "clk_sdio_div", "clk_sdio_div50" }; ++PNAME(mux_sdmmc_p) = { "clk_sdmmc_div", "clk_sdmmc_div50" }; ++PNAME(mux_emmc_p) = { "clk_emmc_div", "clk_emmc_div50" }; ++PNAME(mux_cpll_npll_ppll_p) = { "cpll", "npll", "ppll" }; ++PNAME(mux_gmac_p) = { "clk_gmac_src", "gmac_clkin" }; ++PNAME(mux_gmac_rgmii_speed_p) = { "clk_gmac_tx_src", "clk_gmac_tx_src", "clk_gmac_tx_div50", "clk_gmac_tx_div5" }; ++PNAME(mux_gmac_rmii_speed_p) = { "clk_gmac_rx_div20", "clk_gmac_rx_div2" }; ++PNAME(mux_gmac_rx_tx_p) = { "clk_gmac_rgmii_speed", "clk_gmac_rmii_speed" }; ++PNAME(mux_gpll_usb480m_cpll_npll_p) = { "gpll", "usb480m", "cpll", "npll" }; ++PNAME(mux_uart1_p) = { "clk_uart1_src", "clk_uart1_np5", "clk_uart1_frac", "xin24m" }; ++PNAME(mux_uart2_p) = { "clk_uart2_src", "clk_uart2_np5", "clk_uart2_frac", "xin24m" }; ++PNAME(mux_uart3_p) = { "clk_uart3_src", "clk_uart3_np5", "clk_uart3_frac", "xin24m" }; ++PNAME(mux_uart4_p) = { "clk_uart4_src", "clk_uart4_np5", "clk_uart4_frac", "xin24m" }; ++PNAME(mux_uart5_p) = { "clk_uart5_src", "clk_uart5_np5", "clk_uart5_frac", "xin24m" }; ++PNAME(mux_uart6_p) = { "clk_uart6_src", "clk_uart6_np5", "clk_uart6_frac", "xin24m" }; ++PNAME(mux_uart7_p) = { "clk_uart7_src", "clk_uart7_np5", "clk_uart7_frac", "xin24m" }; ++PNAME(mux_gpll_xin24m_p) = { "gpll", "xin24m" }; ++PNAME(mux_gpll_cpll_xin24m_p) = { "gpll", "cpll", "xin24m" }; ++PNAME(mux_gpll_xin24m_cpll_npll_p) = { "gpll", "xin24m", "cpll", "npll" }; ++PNAME(mux_pdm_p) = { "clk_pdm_src", "clk_pdm_frac" }; ++PNAME(mux_i2s0_8ch_tx_p) = { "clk_i2s0_8ch_tx_src", "clk_i2s0_8ch_tx_frac", "mclk_i2s0_8ch_in", "xin12m" }; ++PNAME(mux_i2s0_8ch_tx_rx_p) = { "clk_i2s0_8ch_tx_mux", "clk_i2s0_8ch_rx_mux"}; ++PNAME(mux_i2s0_8ch_tx_out_p) = { "clk_i2s0_8ch_tx", "xin12m", "clk_i2s0_8ch_rx" }; ++PNAME(mux_i2s0_8ch_rx_p) = { "clk_i2s0_8ch_rx_src", "clk_i2s0_8ch_rx_frac", "mclk_i2s0_8ch_in", "xin12m" }; ++PNAME(mux_i2s0_8ch_rx_tx_p) = { "clk_i2s0_8ch_rx_mux", "clk_i2s0_8ch_tx_mux"}; ++PNAME(mux_i2s0_8ch_rx_out_p) = { "clk_i2s0_8ch_rx", "xin12m", "clk_i2s0_8ch_tx" }; ++PNAME(mux_i2s1_2ch_p) = { "clk_i2s1_2ch_src", "clk_i2s1_2ch_frac", "mclk_i2s1_2ch_in", "xin12m" }; ++PNAME(mux_i2s1_2ch_out_p) = { "clk_i2s1_2ch", "xin12m" }; ++PNAME(mux_rtc32k_pmu_p) = { "xin32k", "pmu_pvtm_32k", "clk_rtc32k_frac" }; ++PNAME(mux_wifi_pmu_p) = { "xin24m", "clk_wifi_pmu_src" }; ++PNAME(mux_gpll_usb480m_cpll_ppll_p) = { "gpll", "usb480m", "cpll", "ppll" }; ++PNAME(mux_uart0_pmu_p) = { "clk_uart0_pmu_src", "clk_uart0_np5", "clk_uart0_frac", "xin24m" }; ++PNAME(mux_usbphy_ref_p) = { "xin24m", "clk_ref24m_pmu" }; ++PNAME(mux_mipidsiphy_ref_p) = { "xin24m", "clk_ref24m_pmu" }; ++PNAME(mux_pciephy_ref_p) = { "xin24m", "clk_pciephy_src" }; ++PNAME(mux_ppll_xin24m_p) = { "ppll", "xin24m" }; ++PNAME(mux_xin24m_32k_p) = { "xin24m", "xin32k" }; ++PNAME(mux_clk_32k_ioe_p) = { "clk_rtc32k_pmu", "xin32k" }; ++ ++static struct rockchip_pll_clock rk1808_pll_clks[] __initdata = { ++ [apll] = PLL(pll_rk3036, PLL_APLL, "apll", mux_pll_p, ++ 0, RK1808_PLL_CON(0), ++ RK1808_MODE_CON, 0, 0, 0, rk1808_pll_rates), ++ [dpll] = PLL(pll_rk3036, PLL_DPLL, "dpll", mux_pll_p, ++ 0, RK1808_PLL_CON(8), ++ RK1808_MODE_CON, 2, 1, 0, NULL), ++ [cpll] = PLL(pll_rk3036, PLL_CPLL, "cpll", mux_pll_p, ++ 0, RK1808_PLL_CON(16), ++ RK1808_MODE_CON, 4, 2, 0, rk1808_pll_rates), ++ [gpll] = PLL(pll_rk3036, PLL_GPLL, "gpll", mux_pll_p, ++ 0, RK1808_PLL_CON(24), ++ RK1808_MODE_CON, 6, 3, 0, rk1808_pll_rates), ++ [npll] = PLL(pll_rk3036, PLL_NPLL, "npll", mux_pll_p, ++ 0, RK1808_PLL_CON(32), ++ RK1808_MODE_CON, 8, 5, 0, rk1808_pll_rates), ++ [ppll] = PLL(pll_rk3036, PLL_PPLL, "ppll", mux_pll_p, ++ 0, RK1808_PMU_PLL_CON(0), ++ RK1808_PMU_MODE_CON, 0, 4, 0, rk1808_pll_rates), ++}; ++ ++#define MFLAGS CLK_MUX_HIWORD_MASK ++#define DFLAGS CLK_DIVIDER_HIWORD_MASK ++#define GFLAGS (CLK_GATE_HIWORD_MASK | CLK_GATE_SET_TO_DISABLE) ++ ++static struct rockchip_clk_branch rk1808_uart1_fracmux __initdata = ++ MUX(0, "clk_uart1_mux", mux_uart1_p, CLK_SET_RATE_PARENT, ++ RK1808_CLKSEL_CON(39), 14, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rk1808_uart2_fracmux __initdata = ++ MUX(0, "clk_uart2_mux", mux_uart2_p, CLK_SET_RATE_PARENT, ++ RK1808_CLKSEL_CON(42), 14, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rk1808_uart3_fracmux __initdata = ++ MUX(0, "clk_uart3_mux", mux_uart3_p, CLK_SET_RATE_PARENT, ++ RK1808_CLKSEL_CON(45), 14, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rk1808_uart4_fracmux __initdata = ++ MUX(0, "clk_uart4_mux", mux_uart4_p, CLK_SET_RATE_PARENT, ++ RK1808_CLKSEL_CON(48), 14, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rk1808_uart5_fracmux __initdata = ++ MUX(0, "clk_uart5_mux", mux_uart5_p, CLK_SET_RATE_PARENT, ++ RK1808_CLKSEL_CON(51), 14, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rk1808_uart6_fracmux __initdata = ++ MUX(0, "clk_uart6_mux", mux_uart6_p, CLK_SET_RATE_PARENT, ++ RK1808_CLKSEL_CON(54), 14, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rk1808_uart7_fracmux __initdata = ++ MUX(0, "clk_uart7_mux", mux_uart7_p, CLK_SET_RATE_PARENT, ++ RK1808_CLKSEL_CON(57), 14, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rk1808_dclk_vopraw_fracmux __initdata = ++ MUX(0, "dclk_vopraw_mux", mux_dclk_vopraw_p, CLK_SET_RATE_PARENT, ++ RK1808_CLKSEL_CON(5), 14, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rk1808_dclk_voplite_fracmux __initdata = ++ MUX(0, "dclk_voplite_mux", mux_dclk_voplite_p, CLK_SET_RATE_PARENT, ++ RK1808_CLKSEL_CON(7), 14, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rk1808_pdm_fracmux __initdata = ++ MUX(0, "clk_pdm_mux", mux_pdm_p, CLK_SET_RATE_PARENT, ++ RK1808_CLKSEL_CON(30), 15, 1, MFLAGS); ++ ++static struct rockchip_clk_branch rk1808_i2s0_8ch_tx_fracmux __initdata = ++ MUX(SCLK_I2S0_8CH_TX_MUX, "clk_i2s0_8ch_tx_mux", mux_i2s0_8ch_tx_p, CLK_SET_RATE_PARENT, ++ RK1808_CLKSEL_CON(32), 10, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rk1808_i2s0_8ch_rx_fracmux __initdata = ++ MUX(SCLK_I2S0_8CH_RX_MUX, "clk_i2s0_8ch_rx_mux", mux_i2s0_8ch_rx_p, CLK_SET_RATE_PARENT, ++ RK1808_CLKSEL_CON(34), 10, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rk1808_i2s1_2ch_fracmux __initdata = ++ MUX(0, "clk_i2s1_2ch_mux", mux_i2s1_2ch_p, CLK_SET_RATE_PARENT, ++ RK1808_CLKSEL_CON(36), 10, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rk1808_rtc32k_pmu_fracmux __initdata = ++ MUX(SCLK_RTC32K_PMU, "clk_rtc32k_pmu", mux_rtc32k_pmu_p, CLK_SET_RATE_PARENT, ++ RK1808_PMU_CLKSEL_CON(0), 14, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rk1808_uart0_pmu_fracmux __initdata = ++ MUX(0, "clk_uart0_pmu_mux", mux_uart0_pmu_p, CLK_SET_RATE_PARENT, ++ RK1808_PMU_CLKSEL_CON(4), 14, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rk1808_clk_branches[] __initdata = { ++ /* ++ * Clock-Architecture Diagram 1 ++ */ ++ ++ MUX(USB480M, "usb480m", mux_usb480m_p, CLK_SET_RATE_PARENT, ++ RK1808_MODE_CON, 10, 2, MFLAGS), ++ FACTOR(0, "xin12m", "xin24m", 0, 1, 2), ++ ++ /* ++ * Clock-Architecture Diagram 2 ++ */ ++ ++ GATE(0, "apll_core", "apll", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(0), 0, GFLAGS), ++ GATE(0, "cpll_core", "cpll", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(0), 0, GFLAGS), ++ GATE(0, "gpll_core", "gpll", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(0), 0, GFLAGS), ++ COMPOSITE_NOMUX(0, "pclk_core_dbg", "armclk", CLK_IGNORE_UNUSED, ++ RK1808_CLKSEL_CON(0), 8, 4, DFLAGS | CLK_DIVIDER_READ_ONLY, ++ RK1808_CLKGATE_CON(0), 3, GFLAGS), ++ COMPOSITE_NOMUX(0, "aclk_core", "armclk", CLK_IGNORE_UNUSED, ++ RK1808_CLKSEL_CON(0), 12, 3, DFLAGS | CLK_DIVIDER_READ_ONLY, ++ RK1808_CLKGATE_CON(0), 2, GFLAGS), ++ ++ GATE(0, "clk_jtag", "jtag_clkin", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(0), 4, GFLAGS), ++ ++ GATE(SCLK_PVTM_CORE, "clk_pvtm_core", "xin24m", 0, ++ RK1808_CLKGATE_CON(0), 5, GFLAGS), ++ ++ COMPOSITE_NOMUX(MSCLK_CORE_NIU, "msclk_core_niu", "gpll", CLK_IS_CRITICAL, ++ RK1808_CLKSEL_CON(18), 0, 5, DFLAGS, ++ RK1808_CLKGATE_CON(0), 1, GFLAGS), ++ ++ /* ++ * Clock-Architecture Diagram 3 ++ */ ++ ++ COMPOSITE(ACLK_GIC_PRE, "aclk_gic_pre", mux_gpll_cpll_p, CLK_IS_CRITICAL, ++ RK1808_CLKSEL_CON(15), 11, 1, MFLAGS, 12, 4, DFLAGS, ++ RK1808_CLKGATE_CON(1), 0, GFLAGS), ++ GATE(0, "aclk_gic_niu", "aclk_gic_pre", CLK_IS_CRITICAL, ++ RK1808_CLKGATE_CON(1), 1, GFLAGS), ++ GATE(ACLK_GIC, "aclk_gic", "aclk_gic_pre", CLK_IS_CRITICAL, ++ RK1808_CLKGATE_CON(1), 2, GFLAGS), ++ GATE(0, "aclk_core2gic", "aclk_gic_pre", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(1), 3, GFLAGS), ++ GATE(0, "aclk_gic2core", "aclk_gic_pre", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(1), 4, GFLAGS), ++ GATE(0, "aclk_spinlock", "aclk_gic_pre", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(1), 4, GFLAGS), ++ ++ COMPOSITE(0, "aclk_vpu_pre", mux_gpll_cpll_p, 0, ++ RK1808_CLKSEL_CON(16), 7, 1, MFLAGS, 0, 5, DFLAGS, ++ RK1808_CLKGATE_CON(8), 8, GFLAGS), ++ COMPOSITE_NOMUX(0, "hclk_vpu_pre", "aclk_vpu_pre", 0, ++ RK1808_CLKSEL_CON(16), 8, 4, DFLAGS, ++ RK1808_CLKGATE_CON(8), 9, GFLAGS), ++ GATE(ACLK_VPU, "aclk_vpu", "aclk_vpu_pre", 0, ++ RK1808_CLKGATE_CON(8), 12, GFLAGS), ++ GATE(0, "aclk_vpu_niu", "aclk_vpu_pre", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(8), 10, GFLAGS), ++ GATE(HCLK_VPU, "hclk_vpu", "hclk_vpu_pre", 0, ++ RK1808_CLKGATE_CON(8), 13, GFLAGS), ++ GATE(0, "hclk_vpu_niu", "hclk_vpu_pre", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(8), 11, GFLAGS), ++ ++ /* ++ * Clock-Architecture Diagram 4 ++ */ ++ COMPOSITE_NOGATE(0, "clk_npu_div", mux_gpll_cpll_p, CLK_OPS_PARENT_ENABLE, ++ RK1808_CLKSEL_CON(1), 8, 2, MFLAGS, 0, 4, DFLAGS), ++ COMPOSITE_NOGATE_HALFDIV(0, "clk_npu_np5", mux_gpll_cpll_p, CLK_OPS_PARENT_ENABLE, ++ RK1808_CLKSEL_CON(1), 10, 2, MFLAGS, 4, 4, DFLAGS), ++ MUX(0, "clk_npu_pre", mux_npu_p, CLK_SET_RATE_PARENT, ++ RK1808_CLKSEL_CON(1), 15, 1, MFLAGS), ++ FACTOR(0, "clk_npu_scan", "clk_npu_pre", 0, 1, 2), ++ GATE(SCLK_NPU, "clk_npu", "clk_npu_pre", 0, ++ RK1808_CLKGATE_CON(1), 10, GFLAGS), ++ ++ COMPOSITE(0, "aclk_npu_pre", mux_gpll_cpll_p, 0, ++ RK1808_CLKSEL_CON(2), 14, 1, MFLAGS, 0, 4, DFLAGS, ++ RK1808_CLKGATE_CON(1), 8, GFLAGS), ++ COMPOSITE(0, "hclk_npu_pre", mux_gpll_cpll_p, 0, ++ RK1808_CLKSEL_CON(2), 15, 1, MFLAGS, 8, 4, DFLAGS, ++ RK1808_CLKGATE_CON(1), 9, GFLAGS), ++ GATE(ACLK_NPU, "aclk_npu", "aclk_npu_pre", 0, ++ RK1808_CLKGATE_CON(1), 11, GFLAGS), ++ GATE(0, "aclk_npu_niu", "aclk_npu_pre", CLK_IS_CRITICAL, ++ RK1808_CLKGATE_CON(1), 13, GFLAGS), ++ COMPOSITE_NOMUX(0, "aclk_npu2mem", "aclk_npu_pre", CLK_IGNORE_UNUSED, ++ RK1808_CLKSEL_CON(2), 4, 4, DFLAGS, ++ RK1808_CLKGATE_CON(1), 15, GFLAGS), ++ GATE(HCLK_NPU, "hclk_npu", "hclk_npu_pre", 0, ++ RK1808_CLKGATE_CON(1), 12, GFLAGS), ++ GATE(0, "hclk_npu_niu", "hclk_npu_pre", CLK_IS_CRITICAL, ++ RK1808_CLKGATE_CON(1), 14, GFLAGS), ++ ++ GATE(SCLK_PVTM_NPU, "clk_pvtm_npu", "xin24m", 0, ++ RK1808_CLKGATE_CON(0), 15, GFLAGS), ++ ++ COMPOSITE(ACLK_IMEM_PRE, "aclk_imem_pre", mux_gpll_cpll_p, CLK_IS_CRITICAL, ++ RK1808_CLKSEL_CON(17), 7, 1, MFLAGS, 0, 5, DFLAGS, ++ RK1808_CLKGATE_CON(7), 0, GFLAGS), ++ GATE(ACLK_IMEM0, "aclk_imem0", "aclk_imem_pre", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(7), 6, GFLAGS), ++ GATE(0, "aclk_imem0_niu", "aclk_imem_pre", CLK_IS_CRITICAL, ++ RK1808_CLKGATE_CON(7), 10, GFLAGS), ++ GATE(ACLK_IMEM1, "aclk_imem1", "aclk_imem_pre", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(7), 7, GFLAGS), ++ GATE(0, "aclk_imem1_niu", "aclk_imem_pre", CLK_IS_CRITICAL, ++ RK1808_CLKGATE_CON(7), 11, GFLAGS), ++ GATE(ACLK_IMEM2, "aclk_imem2", "aclk_imem_pre", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(7), 8, GFLAGS), ++ GATE(0, "aclk_imem2_niu", "aclk_imem_pre", CLK_IS_CRITICAL, ++ RK1808_CLKGATE_CON(7), 12, GFLAGS), ++ GATE(ACLK_IMEM3, "aclk_imem3", "aclk_imem_pre", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(7), 9, GFLAGS), ++ GATE(0, "aclk_imem3_niu", "aclk_imem_pre", CLK_IS_CRITICAL, ++ RK1808_CLKGATE_CON(7), 13, GFLAGS), ++ ++ COMPOSITE(HSCLK_IMEM, "hsclk_imem", mux_gpll_cpll_p, CLK_IS_CRITICAL, ++ RK1808_CLKSEL_CON(17), 15, 1, MFLAGS, 8, 5, DFLAGS, ++ RK1808_CLKGATE_CON(7), 5, GFLAGS), ++ ++ /* ++ * Clock-Architecture Diagram 5 ++ */ ++ GATE(0, "clk_ddr_mon_timer", "xin24m", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(2), 0, GFLAGS), ++ ++ GATE(0, "clk_ddr_mon", "clk_ddrphy1x_out", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(2), 11, GFLAGS), ++ GATE(0, "aclk_split", "clk_ddrphy1x_out", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(2), 15, GFLAGS), ++ GATE(0, "clk_ddr_msch", "clk_ddrphy1x_out", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(2), 8, GFLAGS), ++ GATE(0, "clk_ddrdfi_ctl", "clk_ddrphy1x_out", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(2), 3, GFLAGS), ++ GATE(0, "clk_stdby", "clk_ddrphy1x_out", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(2), 13, GFLAGS), ++ GATE(0, "aclk_ddrc", "clk_ddrphy1x_out", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(2), 5, GFLAGS), ++ GATE(0, "clk_core_ddrc", "clk_ddrphy1x_out", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(2), 6, GFLAGS), ++ ++ GATE(0, "dpll_ddr", "dpll", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(8), 5, GFLAGS), ++ GATE(0, "gpll_ddr", "gpll", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(8), 6, GFLAGS), ++ ++ COMPOSITE_NOGATE(SCLK_DDRCLK, "sclk_ddrc", mux_ddr_p, CLK_IGNORE_UNUSED, ++ RK1808_CLKSEL_CON(3), 7, 1, MFLAGS, 0, 5, DFLAGS), ++ FACTOR(0, "clk_ddrphy1x_out", "sclk_ddrc", CLK_IGNORE_UNUSED, 1, 1), ++ ++ COMPOSITE_NOMUX(PCLK_DDR, "pclk_ddr", "gpll", CLK_IS_CRITICAL, ++ RK1808_CLKSEL_CON(3), 8, 5, DFLAGS, ++ RK1808_CLKGATE_CON(2), 1, GFLAGS), ++ GATE(PCLK_DDRMON, "pclk_ddrmon", "pclk_ddr", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(2), 10, GFLAGS), ++ GATE(PCLK_DDRC, "pclk_ddrc", "pclk_ddr", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(2), 7, GFLAGS), ++ GATE(PCLK_MSCH, "pclk_msch", "pclk_ddr", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(2), 9, GFLAGS), ++ GATE(PCLK_STDBY, "pclk_stdby", "pclk_ddr", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(2), 12, GFLAGS), ++ GATE(0, "pclk_ddr_grf", "pclk_ddr", CLK_IS_CRITICAL, ++ RK1808_CLKGATE_CON(2), 14, GFLAGS), ++ GATE(0, "pclk_ddrdfi_ctl", "pclk_ddr", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(2), 2, GFLAGS), ++ ++ /* ++ * Clock-Architecture Diagram 6 ++ */ ++ ++ COMPOSITE(HSCLK_VIO, "hsclk_vio", mux_gpll_cpll_p, 0, ++ RK1808_CLKSEL_CON(4), 7, 1, MFLAGS, 0, 5, DFLAGS, ++ RK1808_CLKGATE_CON(3), 0, GFLAGS), ++ COMPOSITE_NOMUX(LSCLK_VIO, "lsclk_vio", "hsclk_vio", 0, ++ RK1808_CLKSEL_CON(4), 8, 4, DFLAGS, ++ RK1808_CLKGATE_CON(3), 12, GFLAGS), ++ GATE(0, "hsclk_vio_niu", "hsclk_vio", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(4), 0, GFLAGS), ++ GATE(0, "lsclk_vio_niu", "lsclk_vio", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(4), 1, GFLAGS), ++ GATE(ACLK_VOPRAW, "aclk_vopraw", "hsclk_vio", 0, ++ RK1808_CLKGATE_CON(4), 2, GFLAGS), ++ GATE(HCLK_VOPRAW, "hclk_vopraw", "lsclk_vio", 0, ++ RK1808_CLKGATE_CON(4), 3, GFLAGS), ++ GATE(ACLK_VOPLITE, "aclk_voplite", "hsclk_vio", 0, ++ RK1808_CLKGATE_CON(4), 4, GFLAGS), ++ GATE(HCLK_VOPLITE, "hclk_voplite", "lsclk_vio", 0, ++ RK1808_CLKGATE_CON(4), 5, GFLAGS), ++ GATE(PCLK_DSI_TX, "pclk_dsi_tx", "lsclk_vio", 0, ++ RK1808_CLKGATE_CON(4), 6, GFLAGS), ++ GATE(PCLK_CSI_TX, "pclk_csi_tx", "lsclk_vio", 0, ++ RK1808_CLKGATE_CON(4), 7, GFLAGS), ++ GATE(ACLK_RGA, "aclk_rga", "hsclk_vio", 0, ++ RK1808_CLKGATE_CON(4), 8, GFLAGS), ++ GATE(HCLK_RGA, "hclk_rga", "lsclk_vio", 0, ++ RK1808_CLKGATE_CON(4), 9, GFLAGS), ++ GATE(ACLK_ISP, "aclk_isp", "hsclk_vio", 0, ++ RK1808_CLKGATE_CON(4), 13, GFLAGS), ++ GATE(HCLK_ISP, "hclk_isp", "lsclk_vio", 0, ++ RK1808_CLKGATE_CON(4), 14, GFLAGS), ++ GATE(ACLK_CIF, "aclk_cif", "hsclk_vio", 0, ++ RK1808_CLKGATE_CON(4), 10, GFLAGS), ++ GATE(HCLK_CIF, "hclk_cif", "lsclk_vio", 0, ++ RK1808_CLKGATE_CON(4), 11, GFLAGS), ++ GATE(PCLK_CSI2HOST, "pclk_csi2host", "lsclk_vio", 0, ++ RK1808_CLKGATE_CON(4), 12, GFLAGS), ++ ++ COMPOSITE(0, "dclk_vopraw_src", mux_cpll_gpll_npll_p, 0, ++ RK1808_CLKSEL_CON(5), 10, 2, MFLAGS, 0, 8, DFLAGS, ++ RK1808_CLKGATE_CON(3), 1, GFLAGS), ++ COMPOSITE_FRACMUX(0, "dclk_vopraw_frac", "dclk_vopraw_src", CLK_SET_RATE_PARENT, ++ RK1808_CLKSEL_CON(6), 0, ++ RK1808_CLKGATE_CON(3), 2, GFLAGS, ++ &rk1808_dclk_vopraw_fracmux, RK1808_VOP_RAW_FRAC_MAX_PRATE), ++ GATE(DCLK_VOPRAW, "dclk_vopraw", "dclk_vopraw_mux", 0, ++ RK1808_CLKGATE_CON(3), 3, GFLAGS), ++ ++ COMPOSITE(0, "dclk_voplite_src", mux_cpll_gpll_npll_p, 0, ++ RK1808_CLKSEL_CON(7), 10, 2, MFLAGS, 0, 8, DFLAGS, ++ RK1808_CLKGATE_CON(3), 4, GFLAGS), ++ COMPOSITE_FRACMUX(0, "dclk_voplite_frac", "dclk_voplite_src", CLK_SET_RATE_PARENT, ++ RK1808_CLKSEL_CON(8), 0, ++ RK1808_CLKGATE_CON(3), 5, GFLAGS, ++ &rk1808_dclk_voplite_fracmux, RK1808_VOP_LITE_FRAC_MAX_PRATE), ++ GATE(DCLK_VOPLITE, "dclk_voplite", "dclk_voplite_mux", 0, ++ RK1808_CLKGATE_CON(3), 6, GFLAGS), ++ ++ COMPOSITE_NOMUX(SCLK_TXESC, "clk_txesc", "gpll", 0, ++ RK1808_CLKSEL_CON(9), 0, 12, DFLAGS, ++ RK1808_CLKGATE_CON(3), 7, GFLAGS), ++ ++ COMPOSITE(SCLK_RGA, "clk_rga", mux_gpll_cpll_npll_p, 0, ++ RK1808_CLKSEL_CON(10), 6, 2, MFLAGS, 0, 5, DFLAGS, ++ RK1808_CLKGATE_CON(3), 8, GFLAGS), ++ ++ COMPOSITE(SCLK_ISP, "clk_isp", mux_gpll_cpll_npll_p, 0, ++ RK1808_CLKSEL_CON(10), 14, 2, MFLAGS, 8, 5, DFLAGS, ++ RK1808_CLKGATE_CON(3), 10, GFLAGS), ++ ++ COMPOSITE(DCLK_CIF, "dclk_cif", mux_cpll_gpll_npll_p, 0, ++ RK1808_CLKSEL_CON(11), 14, 2, MFLAGS, 8, 5, DFLAGS, ++ RK1808_CLKGATE_CON(3), 11, GFLAGS), ++ ++ COMPOSITE(SCLK_CIF_OUT, "clk_cif_out", mux_24m_npll_gpll_usb480m_p, 0, ++ RK1808_CLKSEL_CON(11), 6, 2, MFLAGS, 0, 6, DFLAGS, ++ RK1808_CLKGATE_CON(3), 9, GFLAGS), ++ ++ /* ++ * Clock-Architecture Diagram 7 ++ */ ++ ++ /* PD_PCIE */ ++ COMPOSITE_NODIV(0, "clk_pcie_src", mux_gpll_cpll_p, 0, ++ RK1808_CLKSEL_CON(12), 15, 1, MFLAGS, ++ RK1808_CLKGATE_CON(5), 0, GFLAGS), ++ DIV(HSCLK_PCIE, "hsclk_pcie", "clk_pcie_src", 0, ++ RK1808_CLKSEL_CON(12), 0, 5, DFLAGS), ++ DIV(LSCLK_PCIE, "lsclk_pcie", "clk_pcie_src", 0, ++ RK1808_CLKSEL_CON(12), 8, 5, DFLAGS), ++ GATE(0, "hsclk_pcie_niu", "hsclk_pcie", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(6), 0, GFLAGS), ++ GATE(0, "lsclk_pcie_niu", "lsclk_pcie", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(6), 1, GFLAGS), ++ GATE(0, "pclk_pcie_grf", "lsclk_pcie", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(6), 5, GFLAGS), ++ GATE(ACLK_USB3OTG, "aclk_usb3otg", "hsclk_pcie", 0, ++ RK1808_CLKGATE_CON(6), 6, GFLAGS), ++ GATE(HCLK_HOST, "hclk_host", "lsclk_pcie", 0, ++ RK1808_CLKGATE_CON(6), 7, GFLAGS), ++ GATE(HCLK_HOST_ARB, "hclk_host_arb", "lsclk_pcie", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(6), 8, GFLAGS), ++ ++ COMPOSITE(ACLK_PCIE, "aclk_pcie", mux_gpll_cpll_p, 0, ++ RK1808_CLKSEL_CON(15), 8, 1, MFLAGS, 0, 4, DFLAGS, ++ RK1808_CLKGATE_CON(5), 5, GFLAGS), ++ DIV(0, "pclk_pcie_pre", "aclk_pcie", 0, ++ RK1808_CLKSEL_CON(15), 4, 4, DFLAGS), ++ GATE(0, "aclk_pcie_niu", "aclk_pcie", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(6), 10, GFLAGS), ++ GATE(ACLK_PCIE_MST, "aclk_pcie_mst", "aclk_pcie", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(6), 2, GFLAGS), ++ GATE(ACLK_PCIE_SLV, "aclk_pcie_slv", "aclk_pcie", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(6), 3, GFLAGS), ++ GATE(0, "pclk_pcie_niu", "pclk_pcie_pre", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(6), 11, GFLAGS), ++ GATE(0, "pclk_pcie_dbi", "pclk_pcie_pre", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(6), 4, GFLAGS), ++ GATE(PCLK_PCIE, "pclk_pcie", "pclk_pcie_pre", 0, ++ RK1808_CLKGATE_CON(6), 9, GFLAGS), ++ ++ COMPOSITE(0, "clk_pcie_aux_src", mux_cpll_gpll_npll_p, 0, ++ RK1808_CLKSEL_CON(14), 8, 2, MFLAGS, 0, 7, DFLAGS, ++ RK1808_CLKGATE_CON(5), 3, GFLAGS), ++ COMPOSITE_NODIV(SCLK_PCIE_AUX, "clk_pcie_aux", mux_pcie_aux_p, CLK_SET_RATE_PARENT, ++ RK1808_CLKSEL_CON(14), 12, 1, MFLAGS, ++ RK1808_CLKGATE_CON(5), 4, GFLAGS), ++ ++ GATE(SCLK_USB3_OTG0_REF, "clk_usb3_otg0_ref", "xin24m", 0, ++ RK1808_CLKGATE_CON(5), 1, GFLAGS), ++ ++ COMPOSITE(SCLK_USB3_OTG0_SUSPEND, "clk_usb3_otg0_suspend", mux_usb3_otg0_suspend_p, 0, ++ RK1808_CLKSEL_CON(13), 12, 1, MFLAGS, 0, 10, DFLAGS, ++ RK1808_CLKGATE_CON(5), 2, GFLAGS), ++ ++ /* ++ * Clock-Architecture Diagram 8 ++ */ ++ ++ /* PD_PHP */ ++ ++ COMPOSITE_NODIV(0, "clk_peri_src", mux_gpll_cpll_p, CLK_IS_CRITICAL, ++ RK1808_CLKSEL_CON(19), 15, 1, MFLAGS, ++ RK1808_CLKGATE_CON(8), 0, GFLAGS), ++ COMPOSITE_NOMUX(MSCLK_PERI, "msclk_peri", "clk_peri_src", CLK_IS_CRITICAL, ++ RK1808_CLKSEL_CON(19), 0, 5, DFLAGS, ++ RK1808_CLKGATE_CON(8), 1, GFLAGS), ++ COMPOSITE_NOMUX(LSCLK_PERI, "lsclk_peri", "clk_peri_src", CLK_IS_CRITICAL, ++ RK1808_CLKSEL_CON(19), 8, 5, DFLAGS, ++ RK1808_CLKGATE_CON(8), 2, GFLAGS), ++ GATE(0, "msclk_peri_niu", "msclk_peri", CLK_IS_CRITICAL, ++ RK1808_CLKGATE_CON(8), 3, GFLAGS), ++ GATE(0, "lsclk_peri_niu", "lsclk_peri", CLK_IS_CRITICAL, ++ RK1808_CLKGATE_CON(8), 4, GFLAGS), ++ ++ /* PD_MMC */ ++ ++ GATE(0, "hclk_mmc_sfc", "msclk_peri", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(9), 0, GFLAGS), ++ GATE(0, "hclk_mmc_sfc_niu", "hclk_mmc_sfc", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(9), 11, GFLAGS), ++ GATE(HCLK_EMMC, "hclk_emmc", "hclk_mmc_sfc", 0, ++ RK1808_CLKGATE_CON(9), 12, GFLAGS), ++ GATE(HCLK_SFC, "hclk_sfc", "hclk_mmc_sfc", 0, ++ RK1808_CLKGATE_CON(9), 13, GFLAGS), ++ ++ COMPOSITE(SCLK_SDIO_DIV, "clk_sdio_div", mux_gpll_cpll_npll_24m_p, CLK_IGNORE_UNUSED, ++ RK1808_CLKSEL_CON(22), 14, 2, MFLAGS, 0, 8, DFLAGS, ++ RK1808_CLKGATE_CON(9), 1, GFLAGS), ++ COMPOSITE_DIV_OFFSET(SCLK_SDIO_DIV50, "clk_sdio_div50", ++ mux_gpll_cpll_npll_24m_p, CLK_IGNORE_UNUSED, ++ RK1808_CLKSEL_CON(22), 14, 2, MFLAGS, ++ RK1808_CLKSEL_CON(23), 0, 8, DFLAGS, ++ RK1808_CLKGATE_CON(9), 2, GFLAGS), ++ COMPOSITE_NODIV(SCLK_SDIO, "clk_sdio", mux_sdio_p, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, ++ RK1808_CLKSEL_CON(23), 15, 1, MFLAGS, ++ RK1808_CLKGATE_CON(9), 3, GFLAGS), ++ ++ MMC(SCLK_SDIO_DRV, "sdio_drv", "clk_sdio", RK1808_SDIO_CON0, 1), ++ MMC(SCLK_SDIO_SAMPLE, "sdio_sample", "clk_sdio", RK1808_SDIO_CON1, 1), ++ ++ COMPOSITE(SCLK_EMMC_DIV, "clk_emmc_div", ++ mux_gpll_cpll_npll_24m_p, CLK_IGNORE_UNUSED, ++ RK1808_CLKSEL_CON(24), 14, 2, MFLAGS, 0, 8, DFLAGS, ++ RK1808_CLKGATE_CON(9), 4, GFLAGS), ++ COMPOSITE_DIV_OFFSET(SCLK_EMMC_DIV50, "clk_emmc_div50", mux_gpll_cpll_npll_24m_p, CLK_IGNORE_UNUSED, ++ RK1808_CLKSEL_CON(24), 14, 2, MFLAGS, ++ RK1808_CLKSEL_CON(25), 0, 8, DFLAGS, ++ RK1808_CLKGATE_CON(9), 5, GFLAGS), ++ COMPOSITE_NODIV(SCLK_EMMC, "clk_emmc", mux_emmc_p, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, ++ RK1808_CLKSEL_CON(25), 15, 1, MFLAGS, ++ RK1808_CLKGATE_CON(9), 6, GFLAGS), ++ MMC(SCLK_EMMC_DRV, "emmc_drv", "clk_emmc", RK1808_EMMC_CON0, 1), ++ MMC(SCLK_EMMC_SAMPLE, "emmc_sample", "clk_emmc", RK1808_EMMC_CON1, 1), ++ ++ COMPOSITE(SCLK_SDMMC_DIV, "clk_sdmmc_div", mux_gpll_cpll_npll_24m_p, CLK_IGNORE_UNUSED, ++ RK1808_CLKSEL_CON(20), 14, 2, MFLAGS, 0, 8, DFLAGS, ++ RK1808_CLKGATE_CON(9), 7, GFLAGS), ++ COMPOSITE_DIV_OFFSET(SCLK_SDMMC_DIV50, "clk_sdmmc_div50", ++ mux_gpll_cpll_npll_24m_p, CLK_IGNORE_UNUSED, ++ RK1808_CLKSEL_CON(20), 14, 2, MFLAGS, ++ RK1808_CLKSEL_CON(21), 0, 8, DFLAGS, ++ RK1808_CLKGATE_CON(9), 8, GFLAGS), ++ COMPOSITE_NODIV(SCLK_SDMMC, "clk_sdmmc", mux_sdmmc_p, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, ++ RK1808_CLKSEL_CON(21), 15, 1, MFLAGS, ++ RK1808_CLKGATE_CON(9), 9, GFLAGS), ++ MMC(SCLK_SDMMC_DRV, "sdmmc_drv", "clk_sdmmc", RK1808_SDMMC_CON0, 1), ++ MMC(SCLK_SDMMC_SAMPLE, "sdmmc_sample", "clk_sdmmc", RK1808_SDMMC_CON1, 1), ++ ++ COMPOSITE(SCLK_SFC, "clk_sfc", mux_gpll_cpll_p, 0, ++ RK1808_CLKSEL_CON(26), 7, 1, MFLAGS, 0, 7, DFLAGS, ++ RK1808_CLKGATE_CON(9), 10, GFLAGS), ++ ++ /* PD_MAC */ ++ ++ GATE(0, "pclk_sd_gmac", "lsclk_peri", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(10), 2, GFLAGS), ++ GATE(0, "aclk_sd_gmac", "msclk_peri", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(10), 0, GFLAGS), ++ GATE(0, "hclk_sd_gmac", "msclk_peri", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(10), 1, GFLAGS), ++ GATE(0, "pclk_gmac_niu", "pclk_sd_gmac", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(10), 10, GFLAGS), ++ GATE(PCLK_GMAC, "pclk_gmac", "pclk_sd_gmac", 0, ++ RK1808_CLKGATE_CON(10), 12, GFLAGS), ++ GATE(0, "aclk_gmac_niu", "aclk_sd_gmac", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(10), 8, GFLAGS), ++ GATE(ACLK_GMAC, "aclk_gmac", "aclk_sd_gmac", 0, ++ RK1808_CLKGATE_CON(10), 11, GFLAGS), ++ GATE(0, "hclk_gmac_niu", "hclk_sd_gmac", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(10), 9, GFLAGS), ++ GATE(HCLK_SDIO, "hclk_sdio", "hclk_sd_gmac", 0, ++ RK1808_CLKGATE_CON(10), 13, GFLAGS), ++ GATE(HCLK_SDMMC, "hclk_sdmmc", "hclk_sd_gmac", 0, ++ RK1808_CLKGATE_CON(10), 14, GFLAGS), ++ ++ COMPOSITE(SCLK_GMAC_OUT, "clk_gmac_out", mux_cpll_npll_ppll_p, 0, ++ RK1808_CLKSEL_CON(18), 14, 2, MFLAGS, 8, 5, DFLAGS, ++ RK1808_CLKGATE_CON(10), 15, GFLAGS), ++ ++ COMPOSITE(SCLK_GMAC_SRC, "clk_gmac_src", mux_cpll_npll_ppll_p, 0, ++ RK1808_CLKSEL_CON(26), 14, 2, MFLAGS, 8, 5, DFLAGS, ++ RK1808_CLKGATE_CON(10), 3, GFLAGS), ++ MUX(SCLK_GMAC, "clk_gmac", mux_gmac_p, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, ++ RK1808_CLKSEL_CON(27), 0, 1, MFLAGS), ++ GATE(SCLK_GMAC_REF, "clk_gmac_ref", "clk_gmac", 0, ++ RK1808_CLKGATE_CON(10), 4, GFLAGS), ++ GATE(0, "clk_gmac_tx_src", "clk_gmac", 0, ++ RK1808_CLKGATE_CON(10), 7, GFLAGS), ++ GATE(0, "clk_gmac_rx_src", "clk_gmac", 0, ++ RK1808_CLKGATE_CON(10), 6, GFLAGS), ++ GATE(SCLK_GMAC_REFOUT, "clk_gmac_refout", "clk_gmac", 0, ++ RK1808_CLKGATE_CON(10), 5, GFLAGS), ++ FACTOR(0, "clk_gmac_tx_div5", "clk_gmac_tx_src", 0, 1, 5), ++ FACTOR(0, "clk_gmac_tx_div50", "clk_gmac_tx_src", 0, 1, 50), ++ FACTOR(0, "clk_gmac_rx_div2", "clk_gmac_rx_src", 0, 1, 2), ++ FACTOR(0, "clk_gmac_rx_div20", "clk_gmac_rx_src", 0, 1, 20), ++ MUX(SCLK_GMAC_RGMII_SPEED, "clk_gmac_rgmii_speed", mux_gmac_rgmii_speed_p, CLK_SET_RATE_PARENT, ++ RK1808_CLKSEL_CON(27), 2, 2, MFLAGS), ++ MUX(SCLK_GMAC_RMII_SPEED, "clk_gmac_rmii_speed", mux_gmac_rmii_speed_p, CLK_SET_RATE_PARENT, ++ RK1808_CLKSEL_CON(27), 1, 1, MFLAGS), ++ MUX(SCLK_GMAC_RX_TX, "clk_gmac_rx_tx", mux_gmac_rx_tx_p, CLK_SET_RATE_PARENT, ++ RK1808_CLKSEL_CON(27), 4, 1, MFLAGS), ++ ++ /* ++ * Clock-Architecture Diagram 9 ++ */ ++ ++ /* PD_BUS */ ++ ++ COMPOSITE_NODIV(0, "clk_bus_src", mux_gpll_cpll_p, CLK_IS_CRITICAL, ++ RK1808_CLKSEL_CON(27), 15, 1, MFLAGS, ++ RK1808_CLKGATE_CON(11), 0, GFLAGS), ++ COMPOSITE_NOMUX(HSCLK_BUS_PRE, "hsclk_bus_pre", "clk_bus_src", CLK_IS_CRITICAL, ++ RK1808_CLKSEL_CON(27), 8, 5, DFLAGS, ++ RK1808_CLKGATE_CON(11), 1, GFLAGS), ++ COMPOSITE_NOMUX(MSCLK_BUS_PRE, "msclk_bus_pre", "clk_bus_src", CLK_IS_CRITICAL, ++ RK1808_CLKSEL_CON(28), 0, 5, DFLAGS, ++ RK1808_CLKGATE_CON(11), 2, GFLAGS), ++ COMPOSITE_NOMUX(LSCLK_BUS_PRE, "lsclk_bus_pre", "clk_bus_src", CLK_IS_CRITICAL, ++ RK1808_CLKSEL_CON(28), 8, 5, DFLAGS, ++ RK1808_CLKGATE_CON(11), 3, GFLAGS), ++ GATE(0, "hsclk_bus_niu", "hsclk_bus_pre", CLK_IS_CRITICAL, ++ RK1808_CLKGATE_CON(15), 0, GFLAGS), ++ GATE(0, "msclk_bus_niu", "msclk_bus_pre", CLK_IS_CRITICAL, ++ RK1808_CLKGATE_CON(15), 1, GFLAGS), ++ GATE(0, "msclk_sub", "msclk_bus_pre", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(15), 2, GFLAGS), ++ GATE(ACLK_DMAC, "aclk_dmac", "msclk_bus_pre", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(14), 15, GFLAGS), ++ GATE(HCLK_ROM, "hclk_rom", "msclk_bus_pre", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(15), 4, GFLAGS), ++ GATE(ACLK_CRYPTO, "aclk_crypto", "msclk_bus_pre", 0, ++ RK1808_CLKGATE_CON(15), 5, GFLAGS), ++ GATE(HCLK_CRYPTO, "hclk_crypto", "msclk_bus_pre", 0, ++ RK1808_CLKGATE_CON(15), 6, GFLAGS), ++ GATE(ACLK_DCF, "aclk_dcf", "msclk_bus_pre", 0, ++ RK1808_CLKGATE_CON(15), 7, GFLAGS), ++ GATE(0, "lsclk_bus_niu", "lsclk_bus_pre", CLK_IS_CRITICAL, ++ RK1808_CLKGATE_CON(15), 3, GFLAGS), ++ GATE(PCLK_DCF, "pclk_dcf", "lsclk_bus_pre", 0, ++ RK1808_CLKGATE_CON(15), 8, GFLAGS), ++ GATE(PCLK_UART1, "pclk_uart1", "lsclk_bus_pre", 0, ++ RK1808_CLKGATE_CON(15), 9, GFLAGS), ++ GATE(PCLK_UART2, "pclk_uart2", "lsclk_bus_pre", 0, ++ RK1808_CLKGATE_CON(15), 10, GFLAGS), ++ GATE(PCLK_UART3, "pclk_uart3", "lsclk_bus_pre", 0, ++ RK1808_CLKGATE_CON(15), 11, GFLAGS), ++ GATE(PCLK_UART4, "pclk_uart4", "lsclk_bus_pre", 0, ++ RK1808_CLKGATE_CON(15), 12, GFLAGS), ++ GATE(PCLK_UART5, "pclk_uart5", "lsclk_bus_pre", 0, ++ RK1808_CLKGATE_CON(15), 13, GFLAGS), ++ GATE(PCLK_UART6, "pclk_uart6", "lsclk_bus_pre", 0, ++ RK1808_CLKGATE_CON(15), 14, GFLAGS), ++ GATE(PCLK_UART7, "pclk_uart7", "lsclk_bus_pre", 0, ++ RK1808_CLKGATE_CON(15), 15, GFLAGS), ++ GATE(PCLK_I2C1, "pclk_i2c1", "lsclk_bus_pre", 0, ++ RK1808_CLKGATE_CON(16), 0, GFLAGS), ++ GATE(PCLK_I2C2, "pclk_i2c2", "lsclk_bus_pre", 0, ++ RK1808_CLKGATE_CON(16), 1, GFLAGS), ++ GATE(PCLK_I2C3, "pclk_i2c3", "lsclk_bus_pre", 0, ++ RK1808_CLKGATE_CON(16), 2, GFLAGS), ++ GATE(PCLK_I2C4, "pclk_i2c4", "lsclk_bus_pre", 0, ++ RK1808_CLKGATE_CON(17), 4, GFLAGS), ++ GATE(PCLK_I2C5, "pclk_i2c5", "lsclk_bus_pre", 0, ++ RK1808_CLKGATE_CON(17), 5, GFLAGS), ++ GATE(PCLK_SPI0, "pclk_spi0", "lsclk_bus_pre", 0, ++ RK1808_CLKGATE_CON(16), 3, GFLAGS), ++ GATE(PCLK_SPI1, "pclk_spi1", "lsclk_bus_pre", 0, ++ RK1808_CLKGATE_CON(16), 4, GFLAGS), ++ GATE(PCLK_SPI2, "pclk_spi2", "lsclk_bus_pre", 0, ++ RK1808_CLKGATE_CON(16), 5, GFLAGS), ++ GATE(PCLK_TSADC, "pclk_tsadc", "lsclk_bus_pre", 0, ++ RK1808_CLKGATE_CON(16), 9, GFLAGS), ++ GATE(PCLK_SARADC, "pclk_saradc", "lsclk_bus_pre", 0, ++ RK1808_CLKGATE_CON(16), 10, GFLAGS), ++ GATE(PCLK_EFUSE, "pclk_efuse", "lsclk_bus_pre", 0, ++ RK1808_CLKGATE_CON(16), 11, GFLAGS), ++ GATE(PCLK_GPIO1, "pclk_gpio1", "lsclk_bus_pre", 0, ++ RK1808_CLKGATE_CON(16), 12, GFLAGS), ++ GATE(PCLK_GPIO2, "pclk_gpio2", "lsclk_bus_pre", 0, ++ RK1808_CLKGATE_CON(16), 13, GFLAGS), ++ GATE(PCLK_GPIO3, "pclk_gpio3", "lsclk_bus_pre", 0, ++ RK1808_CLKGATE_CON(16), 14, GFLAGS), ++ GATE(PCLK_GPIO4, "pclk_gpio4", "lsclk_bus_pre", 0, ++ RK1808_CLKGATE_CON(16), 15, GFLAGS), ++ GATE(PCLK_PWM0, "pclk_pwm0", "lsclk_bus_pre", 0, ++ RK1808_CLKGATE_CON(16), 6, GFLAGS), ++ GATE(PCLK_PWM1, "pclk_pwm1", "lsclk_bus_pre", 0, ++ RK1808_CLKGATE_CON(16), 7, GFLAGS), ++ GATE(PCLK_PWM2, "pclk_pwm2", "lsclk_bus_pre", 0, ++ RK1808_CLKGATE_CON(16), 8, GFLAGS), ++ GATE(PCLK_TIMER, "pclk_timer", "lsclk_bus_pre", 0, ++ RK1808_CLKGATE_CON(17), 0, GFLAGS), ++ GATE(PCLK_WDT, "pclk_wdt", "lsclk_bus_pre", 0, ++ RK1808_CLKGATE_CON(17), 1, GFLAGS), ++ GATE(0, "pclk_grf", "lsclk_bus_pre", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(17), 2, GFLAGS), ++ GATE(0, "pclk_sgrf", "lsclk_bus_pre", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(17), 3, GFLAGS), ++ GATE(0, "hclk_audio_pre", "msclk_bus_pre", 0, ++ RK1808_CLKGATE_CON(17), 8, GFLAGS), ++ GATE(0, "pclk_top_pre", "lsclk_bus_pre", CLK_IS_CRITICAL, ++ RK1808_CLKGATE_CON(11), 4, GFLAGS), ++ ++ COMPOSITE(SCLK_CRYPTO, "clk_crypto", mux_gpll_cpll_p, 0, ++ RK1808_CLKSEL_CON(29), 7, 1, MFLAGS, 0, 5, DFLAGS, ++ RK1808_CLKGATE_CON(11), 5, GFLAGS), ++ COMPOSITE(SCLK_CRYPTO_APK, "clk_crypto_apk", mux_gpll_cpll_p, 0, ++ RK1808_CLKSEL_CON(29), 15, 1, MFLAGS, 8, 5, DFLAGS, ++ RK1808_CLKGATE_CON(11), 6, GFLAGS), ++ ++ COMPOSITE(0, "clk_uart1_src", mux_gpll_usb480m_cpll_npll_p, 0, ++ RK1808_CLKSEL_CON(38), 14, 2, MFLAGS, 0, 7, DFLAGS, ++ RK1808_CLKGATE_CON(11), 8, GFLAGS), ++ COMPOSITE_NOMUX_HALFDIV(0, "clk_uart1_np5", "clk_uart1_src", 0, ++ RK1808_CLKSEL_CON(39), 0, 7, DFLAGS, ++ RK1808_CLKGATE_CON(11), 9, GFLAGS), ++ COMPOSITE_FRACMUX(0, "clk_uart1_frac", "clk_uart1_src", CLK_SET_RATE_PARENT, ++ RK1808_CLKSEL_CON(40), 0, ++ RK1808_CLKGATE_CON(11), 10, GFLAGS, ++ &rk1808_uart1_fracmux, RK1808_UART_FRAC_MAX_PRATE), ++ GATE(SCLK_UART1, "clk_uart1", "clk_uart1_mux", 0, ++ RK1808_CLKGATE_CON(11), 11, GFLAGS), ++ ++ COMPOSITE(0, "clk_uart2_src", mux_gpll_usb480m_cpll_npll_p, 0, ++ RK1808_CLKSEL_CON(41), 14, 2, MFLAGS, 0, 7, DFLAGS, ++ RK1808_CLKGATE_CON(11), 12, GFLAGS), ++ COMPOSITE_NOMUX_HALFDIV(0, "clk_uart2_np5", "clk_uart2_src", 0, ++ RK1808_CLKSEL_CON(42), 0, 7, DFLAGS, ++ RK1808_CLKGATE_CON(11), 13, GFLAGS), ++ COMPOSITE_FRACMUX(0, "clk_uart2_frac", "clk_uart2_src", CLK_SET_RATE_PARENT, ++ RK1808_CLKSEL_CON(43), 0, ++ RK1808_CLKGATE_CON(11), 14, GFLAGS, ++ &rk1808_uart2_fracmux, RK1808_UART_FRAC_MAX_PRATE), ++ GATE(SCLK_UART2, "clk_uart2", "clk_uart2_mux", 0, ++ RK1808_CLKGATE_CON(11), 15, GFLAGS), ++ ++ COMPOSITE(0, "clk_uart3_src", mux_gpll_usb480m_cpll_npll_p, 0, ++ RK1808_CLKSEL_CON(44), 14, 2, MFLAGS, 0, 7, DFLAGS, ++ RK1808_CLKGATE_CON(12), 0, GFLAGS), ++ COMPOSITE_NOMUX_HALFDIV(0, "clk_uart3_np5", "clk_uart3_src", 0, ++ RK1808_CLKSEL_CON(45), 0, 7, DFLAGS, ++ RK1808_CLKGATE_CON(12), 1, GFLAGS), ++ COMPOSITE_FRACMUX(0, "clk_uart3_frac", "clk_uart3_src", CLK_SET_RATE_PARENT, ++ RK1808_CLKSEL_CON(46), 0, ++ RK1808_CLKGATE_CON(12), 2, GFLAGS, ++ &rk1808_uart3_fracmux, RK1808_UART_FRAC_MAX_PRATE), ++ GATE(SCLK_UART3, "clk_uart3", "clk_uart3_mux", 0, ++ RK1808_CLKGATE_CON(12), 3, GFLAGS), ++ ++ COMPOSITE(0, "clk_uart4_src", mux_gpll_usb480m_cpll_npll_p, 0, ++ RK1808_CLKSEL_CON(47), 14, 2, MFLAGS, 0, 7, DFLAGS, ++ RK1808_CLKGATE_CON(12), 4, GFLAGS), ++ COMPOSITE_NOMUX_HALFDIV(0, "clk_uart4_np5", "clk_uart4_src", 0, ++ RK1808_CLKSEL_CON(48), 0, 7, DFLAGS, ++ RK1808_CLKGATE_CON(12), 5, GFLAGS), ++ COMPOSITE_FRACMUX(0, "clk_uart4_frac", "clk_uart4_src", CLK_SET_RATE_PARENT, ++ RK1808_CLKSEL_CON(49), 0, ++ RK1808_CLKGATE_CON(12), 6, GFLAGS, ++ &rk1808_uart4_fracmux, RK1808_UART_FRAC_MAX_PRATE), ++ GATE(SCLK_UART4, "clk_uart4", "clk_uart4_mux", 0, ++ RK1808_CLKGATE_CON(12), 7, GFLAGS), ++ ++ COMPOSITE(0, "clk_uart5_src", mux_gpll_usb480m_cpll_npll_p, 0, ++ RK1808_CLKSEL_CON(50), 14, 2, MFLAGS, 0, 7, DFLAGS, ++ RK1808_CLKGATE_CON(12), 8, GFLAGS), ++ COMPOSITE_NOMUX_HALFDIV(0, "clk_uart5_np5", "clk_uart5_src", 0, ++ RK1808_CLKSEL_CON(51), 0, 7, DFLAGS, ++ RK1808_CLKGATE_CON(12), 9, GFLAGS), ++ COMPOSITE_FRACMUX(0, "clk_uart5_frac", "clk_uart5_src", CLK_SET_RATE_PARENT, ++ RK1808_CLKSEL_CON(52), 0, ++ RK1808_CLKGATE_CON(12), 10, GFLAGS, ++ &rk1808_uart5_fracmux, RK1808_UART_FRAC_MAX_PRATE), ++ GATE(SCLK_UART5, "clk_uart5", "clk_uart5_mux", 0, ++ RK1808_CLKGATE_CON(12), 11, GFLAGS), ++ ++ COMPOSITE(0, "clk_uart6_src", mux_gpll_usb480m_cpll_npll_p, 0, ++ RK1808_CLKSEL_CON(53), 14, 2, MFLAGS, 0, 7, DFLAGS, ++ RK1808_CLKGATE_CON(12), 12, GFLAGS), ++ COMPOSITE_NOMUX_HALFDIV(0, "clk_uart6_np5", "clk_uart6_src", 0, ++ RK1808_CLKSEL_CON(54), 0, 7, DFLAGS, ++ RK1808_CLKGATE_CON(12), 13, GFLAGS), ++ COMPOSITE_FRACMUX(0, "clk_uart6_frac", "clk_uart6_src", CLK_SET_RATE_PARENT, ++ RK1808_CLKSEL_CON(55), 0, ++ RK1808_CLKGATE_CON(12), 14, GFLAGS, ++ &rk1808_uart6_fracmux, RK1808_UART_FRAC_MAX_PRATE), ++ GATE(SCLK_UART6, "clk_uart6", "clk_uart6_mux", 0, ++ RK1808_CLKGATE_CON(12), 15, GFLAGS), ++ ++ COMPOSITE(0, "clk_uart7_src", mux_gpll_usb480m_cpll_npll_p, 0, ++ RK1808_CLKSEL_CON(56), 14, 2, MFLAGS, 0, 7, DFLAGS, ++ RK1808_CLKGATE_CON(13), 0, GFLAGS), ++ COMPOSITE_NOMUX_HALFDIV(0, "clk_uart7_np5", "clk_uart7_src", 0, ++ RK1808_CLKSEL_CON(57), 0, 7, DFLAGS, ++ RK1808_CLKGATE_CON(13), 1, GFLAGS), ++ COMPOSITE_FRACMUX(0, "clk_uart7_frac", "clk_uart7_src", CLK_SET_RATE_PARENT, ++ RK1808_CLKSEL_CON(58), 0, ++ RK1808_CLKGATE_CON(13), 2, GFLAGS, ++ &rk1808_uart7_fracmux, RK1808_UART_FRAC_MAX_PRATE), ++ GATE(SCLK_UART7, "clk_uart7", "clk_uart7_mux", 0, ++ RK1808_CLKGATE_CON(13), 3, GFLAGS), ++ ++ COMPOSITE(SCLK_I2C1, "clk_i2c1", mux_gpll_xin24m_p, 0, ++ RK1808_CLKSEL_CON(59), 7, 1, MFLAGS, 0, 7, DFLAGS, ++ RK1808_CLKGATE_CON(13), 4, GFLAGS), ++ COMPOSITE(SCLK_I2C2, "clk_i2c2", mux_gpll_xin24m_p, 0, ++ RK1808_CLKSEL_CON(59), 15, 1, MFLAGS, 8, 7, DFLAGS, ++ RK1808_CLKGATE_CON(13), 5, GFLAGS), ++ COMPOSITE(SCLK_I2C3, "clk_i2c3", mux_gpll_xin24m_p, 0, ++ RK1808_CLKSEL_CON(60), 7, 1, MFLAGS, 0, 7, DFLAGS, ++ RK1808_CLKGATE_CON(13), 6, GFLAGS), ++ COMPOSITE(SCLK_I2C4, "clk_i2c4", mux_gpll_xin24m_p, 0, ++ RK1808_CLKSEL_CON(71), 7, 1, MFLAGS, 0, 7, DFLAGS, ++ RK1808_CLKGATE_CON(14), 6, GFLAGS), ++ COMPOSITE(SCLK_I2C5, "clk_i2c5", mux_gpll_xin24m_p, 0, ++ RK1808_CLKSEL_CON(71), 15, 1, MFLAGS, 8, 7, DFLAGS, ++ RK1808_CLKGATE_CON(14), 7, GFLAGS), ++ ++ COMPOSITE(SCLK_SPI0, "clk_spi0", mux_gpll_xin24m_p, 0, ++ RK1808_CLKSEL_CON(60), 15, 1, MFLAGS, 8, 7, DFLAGS, ++ RK1808_CLKGATE_CON(13), 7, GFLAGS), ++ COMPOSITE(SCLK_SPI1, "clk_spi1", mux_gpll_xin24m_p, 0, ++ RK1808_CLKSEL_CON(61), 7, 1, MFLAGS, 0, 7, DFLAGS, ++ RK1808_CLKGATE_CON(13), 8, GFLAGS), ++ COMPOSITE(SCLK_SPI2, "clk_spi2", mux_gpll_xin24m_p, 0, ++ RK1808_CLKSEL_CON(61), 15, 1, MFLAGS, 8, 7, DFLAGS, ++ RK1808_CLKGATE_CON(13), 9, GFLAGS), ++ ++ COMPOSITE_NOMUX(SCLK_TSADC, "clk_tsadc", "xin24m", 0, ++ RK1808_CLKSEL_CON(62), 0, 11, DFLAGS, ++ RK1808_CLKGATE_CON(13), 13, GFLAGS), ++ COMPOSITE_NOMUX(SCLK_SARADC, "clk_saradc", "xin24m", 0, ++ RK1808_CLKSEL_CON(63), 0, 11, DFLAGS, ++ RK1808_CLKGATE_CON(13), 14, GFLAGS), ++ ++ COMPOSITE(SCLK_EFUSE_S, "clk_efuse_s", mux_gpll_cpll_xin24m_p, 0, ++ RK1808_CLKSEL_CON(64), 6, 2, MFLAGS, 0, 6, DFLAGS, ++ RK1808_CLKGATE_CON(14), 0, GFLAGS), ++ COMPOSITE(SCLK_EFUSE_NS, "clk_efuse_ns", mux_gpll_cpll_xin24m_p, 0, ++ RK1808_CLKSEL_CON(64), 14, 2, MFLAGS, 8, 6, DFLAGS, ++ RK1808_CLKGATE_CON(14), 1, GFLAGS), ++ ++ COMPOSITE(DBCLK_GPIO1, "dbclk_gpio1", mux_xin24m_32k_p, 0, ++ RK1808_CLKSEL_CON(65), 15, 1, MFLAGS, 0, 11, DFLAGS, ++ RK1808_CLKGATE_CON(14), 2, GFLAGS), ++ COMPOSITE(DBCLK_GPIO2, "dbclk_gpio2", mux_xin24m_32k_p, 0, ++ RK1808_CLKSEL_CON(66), 15, 1, MFLAGS, 0, 11, DFLAGS, ++ RK1808_CLKGATE_CON(14), 3, GFLAGS), ++ COMPOSITE(DBCLK_GPIO3, "dbclk_gpio3", mux_xin24m_32k_p, 0, ++ RK1808_CLKSEL_CON(67), 15, 1, MFLAGS, 0, 11, DFLAGS, ++ RK1808_CLKGATE_CON(14), 4, GFLAGS), ++ COMPOSITE(DBCLK_GPIO4, "dbclk_gpio4", mux_xin24m_32k_p, 0, ++ RK1808_CLKSEL_CON(68), 15, 1, MFLAGS, 0, 11, DFLAGS, ++ RK1808_CLKGATE_CON(14), 5, GFLAGS), ++ ++ COMPOSITE(SCLK_PWM0, "clk_pwm0", mux_gpll_xin24m_p, 0, ++ RK1808_CLKSEL_CON(69), 7, 1, MFLAGS, 0, 7, DFLAGS, ++ RK1808_CLKGATE_CON(13), 10, GFLAGS), ++ COMPOSITE(SCLK_PWM1, "clk_pwm1", mux_gpll_xin24m_p, 0, ++ RK1808_CLKSEL_CON(69), 15, 1, MFLAGS, 8, 7, DFLAGS, ++ RK1808_CLKGATE_CON(13), 11, GFLAGS), ++ COMPOSITE(SCLK_PWM2, "clk_pwm2", mux_gpll_xin24m_p, 0, ++ RK1808_CLKSEL_CON(70), 7, 1, MFLAGS, 0, 7, DFLAGS, ++ RK1808_CLKGATE_CON(13), 12, GFLAGS), ++ ++ GATE(SCLK_TIMER0, "sclk_timer0", "xin24m", 0, ++ RK1808_CLKGATE_CON(14), 8, GFLAGS), ++ GATE(SCLK_TIMER1, "sclk_timer1", "xin24m", 0, ++ RK1808_CLKGATE_CON(14), 9, GFLAGS), ++ GATE(SCLK_TIMER2, "sclk_timer2", "xin24m", 0, ++ RK1808_CLKGATE_CON(14), 10, GFLAGS), ++ GATE(SCLK_TIMER3, "sclk_timer3", "xin24m", 0, ++ RK1808_CLKGATE_CON(14), 11, GFLAGS), ++ GATE(SCLK_TIMER4, "sclk_timer4", "xin24m", 0, ++ RK1808_CLKGATE_CON(14), 12, GFLAGS), ++ GATE(SCLK_TIMER5, "sclk_timer5", "xin24m", 0, ++ RK1808_CLKGATE_CON(14), 13, GFLAGS), ++ ++ /* ++ * Clock-Architecture Diagram 10 ++ */ ++ ++ /* PD_AUDIO */ ++ ++ GATE(0, "hclk_audio_niu", "hclk_audio_pre", CLK_IGNORE_UNUSED, ++ RK1808_CLKGATE_CON(18), 11, GFLAGS), ++ GATE(HCLK_VAD, "hclk_vad", "hclk_audio_pre", 0, ++ RK1808_CLKGATE_CON(18), 12, GFLAGS), ++ GATE(HCLK_PDM, "hclk_pdm", "hclk_audio_pre", 0, ++ RK1808_CLKGATE_CON(18), 13, GFLAGS), ++ GATE(HCLK_I2S0_8CH, "hclk_i2s0_8ch", "hclk_audio_pre", 0, ++ RK1808_CLKGATE_CON(18), 14, GFLAGS), ++ GATE(HCLK_I2S1_2CH, "hclk_i2s1_2ch", "hclk_audio_pre", 0, ++ RK1808_CLKGATE_CON(18), 15, GFLAGS), ++ ++ COMPOSITE(0, "clk_pdm_src", mux_gpll_xin24m_cpll_npll_p, 0, ++ RK1808_CLKSEL_CON(30), 8, 2, MFLAGS, 0, 7, DFLAGS, ++ RK1808_CLKGATE_CON(17), 9, GFLAGS), ++ COMPOSITE_FRACMUX(0, "clk_pdm_frac", "clk_pdm_src", CLK_SET_RATE_PARENT, ++ RK1808_CLKSEL_CON(31), 0, ++ RK1808_CLKGATE_CON(17), 10, GFLAGS, ++ &rk1808_pdm_fracmux, RK1808_PDM_FRAC_MAX_PRATE), ++ GATE(SCLK_PDM, "clk_pdm", "clk_pdm_mux", 0, ++ RK1808_CLKGATE_CON(17), 11, GFLAGS), ++ ++ COMPOSITE(SCLK_I2S0_8CH_TX_SRC, "clk_i2s0_8ch_tx_src", mux_gpll_cpll_npll_p, 0, ++ RK1808_CLKSEL_CON(32), 8, 2, MFLAGS, 0, 7, DFLAGS, ++ RK1808_CLKGATE_CON(17), 12, GFLAGS), ++ COMPOSITE_FRACMUX(0, "clk_i2s0_8ch_tx_frac", "clk_i2s0_8ch_tx_src", CLK_SET_RATE_PARENT, ++ RK1808_CLKSEL_CON(33), 0, ++ RK1808_CLKGATE_CON(17), 13, GFLAGS, ++ &rk1808_i2s0_8ch_tx_fracmux, RK1808_I2S_FRAC_MAX_PRATE), ++ COMPOSITE_NODIV(SCLK_I2S0_8CH_TX, "clk_i2s0_8ch_tx", mux_i2s0_8ch_tx_rx_p, CLK_SET_RATE_PARENT, ++ RK1808_CLKSEL_CON(32), 12, 1, MFLAGS, ++ RK1808_CLKGATE_CON(17), 14, GFLAGS), ++ COMPOSITE_NODIV(SCLK_I2S0_8CH_TX_OUT, "clk_i2s0_8ch_tx_out", mux_i2s0_8ch_tx_out_p, CLK_SET_RATE_PARENT, ++ RK1808_CLKSEL_CON(32), 14, 2, MFLAGS, ++ RK1808_CLKGATE_CON(17), 15, GFLAGS), ++ ++ COMPOSITE(SCLK_I2S0_8CH_RX_SRC, "clk_i2s0_8ch_rx_src", mux_gpll_cpll_npll_p, 0, ++ RK1808_CLKSEL_CON(34), 8, 2, MFLAGS, 0, 7, DFLAGS, ++ RK1808_CLKGATE_CON(18), 0, GFLAGS), ++ COMPOSITE_FRACMUX(0, "clk_i2s0_8ch_rx_frac", "clk_i2s0_8ch_rx_src", CLK_SET_RATE_PARENT, ++ RK1808_CLKSEL_CON(35), 0, ++ RK1808_CLKGATE_CON(18), 1, GFLAGS, ++ &rk1808_i2s0_8ch_rx_fracmux, RK1808_I2S_FRAC_MAX_PRATE), ++ COMPOSITE_NODIV(SCLK_I2S0_8CH_RX, "clk_i2s0_8ch_rx", mux_i2s0_8ch_rx_tx_p, CLK_SET_RATE_PARENT, ++ RK1808_CLKSEL_CON(34), 12, 1, MFLAGS, ++ RK1808_CLKGATE_CON(18), 2, GFLAGS), ++ COMPOSITE_NODIV(SCLK_I2S0_8CH_RX_OUT, "clk_i2s0_8ch_rx_out", mux_i2s0_8ch_rx_out_p, CLK_SET_RATE_PARENT, ++ RK1808_CLKSEL_CON(34), 14, 2, MFLAGS, ++ RK1808_CLKGATE_CON(18), 3, GFLAGS), ++ ++ COMPOSITE(SCLK_I2S1_2CH_SRC, "clk_i2s1_2ch_src", mux_gpll_cpll_npll_p, 0, ++ RK1808_CLKSEL_CON(36), 8, 2, MFLAGS, 0, 7, DFLAGS, ++ RK1808_CLKGATE_CON(18), 4, GFLAGS), ++ COMPOSITE_FRACMUX(0, "clk_i2s1_2ch_frac", "clk_i2s1_2ch_src", CLK_SET_RATE_PARENT, ++ RK1808_CLKSEL_CON(37), 0, ++ RK1808_CLKGATE_CON(18), 5, GFLAGS, ++ &rk1808_i2s1_2ch_fracmux, RK1808_I2S_FRAC_MAX_PRATE), ++ GATE(SCLK_I2S1_2CH, "clk_i2s1_2ch", "clk_i2s1_2ch_mux", 0, ++ RK1808_CLKGATE_CON(18), 6, GFLAGS), ++ COMPOSITE_NODIV(SCLK_I2S1_2CH_OUT, "clk_i2s1_2ch_out", mux_i2s1_2ch_out_p, CLK_SET_RATE_PARENT, ++ RK1808_CLKSEL_CON(36), 15, 1, MFLAGS, ++ RK1808_CLKGATE_CON(18), 7, GFLAGS), ++ ++ /* ++ * Clock-Architecture Diagram 10 ++ */ ++ ++ /* PD_BUS */ ++ ++ GATE(0, "pclk_top_niu", "pclk_top_pre", CLK_IGNORE_UNUSED, RK1808_CLKGATE_CON(19), 0, GFLAGS), ++ GATE(0, "pclk_top_cru", "pclk_top_pre", CLK_IGNORE_UNUSED, RK1808_CLKGATE_CON(19), 1, GFLAGS), ++ GATE(0, "pclk_ddrphy", "pclk_top_pre", CLK_IGNORE_UNUSED, RK1808_CLKGATE_CON(19), 2, GFLAGS), ++ GATE(PCLK_MIPIDSIPHY, "pclk_mipidsiphy", "pclk_top_pre", 0, RK1808_CLKGATE_CON(19), 3, GFLAGS), ++ GATE(PCLK_MIPICSIPHY, "pclk_mipicsiphy", "pclk_top_pre", 0, RK1808_CLKGATE_CON(19), 4, GFLAGS), ++ ++ GATE(PCLK_USB3PHY_PIPE, "pclk_usb3phy_pipe", "pclk_top_pre", CLK_IGNORE_UNUSED, RK1808_CLKGATE_CON(19), 6, GFLAGS), ++ GATE(0, "pclk_usb3_grf", "pclk_top_pre", CLK_IGNORE_UNUSED, RK1808_CLKGATE_CON(19), 7, GFLAGS), ++ GATE(0, "pclk_usb_grf", "pclk_top_pre", CLK_IGNORE_UNUSED, RK1808_CLKGATE_CON(19), 8, GFLAGS), ++ ++ /* ++ * Clock-Architecture Diagram 11 ++ */ ++ ++ /* PD_PMU */ ++ ++ COMPOSITE_FRACMUX(SCLK_RTC32K_FRAC, "clk_rtc32k_frac", "xin24m", CLK_IGNORE_UNUSED, ++ RK1808_PMU_CLKSEL_CON(1), 0, ++ RK1808_PMU_CLKGATE_CON(0), 13, GFLAGS, ++ &rk1808_rtc32k_pmu_fracmux, 0), ++ ++ COMPOSITE_NOMUX(XIN24M_DIV, "xin24m_div", "xin24m", CLK_IGNORE_UNUSED, ++ RK1808_PMU_CLKSEL_CON(0), 8, 5, DFLAGS, ++ RK1808_PMU_CLKGATE_CON(0), 12, GFLAGS), ++ ++ COMPOSITE_NOMUX(0, "clk_wifi_pmu_src", "ppll", 0, ++ RK1808_PMU_CLKSEL_CON(2), 8, 6, DFLAGS, ++ RK1808_PMU_CLKGATE_CON(0), 14, GFLAGS), ++ COMPOSITE_NODIV(SCLK_WIFI_PMU, "clk_wifi_pmu", mux_wifi_pmu_p, CLK_SET_RATE_PARENT, ++ RK1808_PMU_CLKSEL_CON(2), 15, 1, MFLAGS, ++ RK1808_PMU_CLKGATE_CON(0), 15, GFLAGS), ++ ++ COMPOSITE(0, "clk_uart0_pmu_src", mux_gpll_usb480m_cpll_ppll_p, 0, ++ RK1808_PMU_CLKSEL_CON(3), 14, 2, MFLAGS, 0, 7, DFLAGS, ++ RK1808_PMU_CLKGATE_CON(1), 0, GFLAGS), ++ COMPOSITE_NOMUX_HALFDIV(0, "clk_uart0_np5", "clk_uart0_pmu_src", 0, ++ RK1808_PMU_CLKSEL_CON(4), 0, 7, DFLAGS, ++ RK1808_PMU_CLKGATE_CON(1), 1, GFLAGS), ++ COMPOSITE_FRACMUX(0, "clk_uart0_frac", "clk_uart0_pmu_src", CLK_SET_RATE_PARENT, ++ RK1808_PMU_CLKSEL_CON(5), 0, ++ RK1808_PMU_CLKGATE_CON(1), 2, GFLAGS, ++ &rk1808_uart0_pmu_fracmux, RK1808_UART_FRAC_MAX_PRATE), ++ GATE(SCLK_UART0_PMU, "clk_uart0_pmu", "clk_uart0_pmu_mux", CLK_SET_RATE_PARENT, ++ RK1808_PMU_CLKGATE_CON(1), 3, GFLAGS), ++ ++ GATE(SCLK_PVTM_PMU, "clk_pvtm_pmu", "xin24m", 0, ++ RK1808_PMU_CLKGATE_CON(1), 4, GFLAGS), ++ ++ COMPOSITE(SCLK_PMU_I2C0, "clk_pmu_i2c0", mux_ppll_xin24m_p, 0, ++ RK1808_PMU_CLKSEL_CON(7), 15, 1, MFLAGS, 8, 7, DFLAGS, ++ RK1808_PMU_CLKGATE_CON(1), 5, GFLAGS), ++ ++ COMPOSITE(DBCLK_PMU_GPIO0, "dbclk_gpio0", mux_xin24m_32k_p, 0, ++ RK1808_PMU_CLKSEL_CON(6), 15, 1, MFLAGS, 0, 11, DFLAGS, ++ RK1808_PMU_CLKGATE_CON(1), 6, GFLAGS), ++ ++ COMPOSITE_NOMUX(SCLK_REF24M_PMU, "clk_ref24m_pmu", "ppll", 0, ++ RK1808_PMU_CLKSEL_CON(2), 0, 6, DFLAGS, ++ RK1808_PMU_CLKGATE_CON(1), 8, GFLAGS), ++ COMPOSITE_NODIV(SCLK_USBPHY_REF, "clk_usbphy_ref", mux_usbphy_ref_p, CLK_SET_RATE_PARENT, ++ RK1808_PMU_CLKSEL_CON(2), 6, 1, MFLAGS, ++ RK1808_PMU_CLKGATE_CON(1), 9, GFLAGS), ++ COMPOSITE_NODIV(SCLK_MIPIDSIPHY_REF, "clk_mipidsiphy_ref", mux_mipidsiphy_ref_p, CLK_SET_RATE_PARENT, ++ RK1808_PMU_CLKSEL_CON(2), 7, 1, MFLAGS, ++ RK1808_PMU_CLKGATE_CON(1), 10, GFLAGS), ++ ++ FACTOR(0, "clk_ppll_ph0", "ppll", 0, 1, 2), ++ COMPOSITE_NOMUX(0, "clk_pciephy_src", "clk_ppll_ph0", 0, ++ RK1808_PMU_CLKSEL_CON(7), 0, 2, DFLAGS, ++ RK1808_PMU_CLKGATE_CON(1), 11, GFLAGS), ++ COMPOSITE_NODIV(SCLK_PCIEPHY_REF, "clk_pciephy_ref", mux_pciephy_ref_p, CLK_SET_RATE_PARENT, ++ RK1808_PMU_CLKSEL_CON(7), 4, 1, MFLAGS, ++ RK1808_PMU_CLKGATE_CON(1), 12, GFLAGS), ++ ++ COMPOSITE_NOMUX(PCLK_PMU_PRE, "pclk_pmu_pre", "ppll", CLK_IS_CRITICAL, ++ RK1808_PMU_CLKSEL_CON(0), 0, 5, DFLAGS, ++ RK1808_PMU_CLKGATE_CON(0), 0, GFLAGS), ++ ++ GATE(0, "pclk_pmu_niu", "pclk_pmu_pre", CLK_IS_CRITICAL, RK1808_PMU_CLKGATE_CON(0), 1, GFLAGS), ++ GATE(0, "pclk_pmu_sgrf", "pclk_pmu_pre", CLK_IGNORE_UNUSED, RK1808_PMU_CLKGATE_CON(0), 2, GFLAGS), ++ GATE(0, "pclk_pmu_grf", "pclk_pmu_pre", CLK_IGNORE_UNUSED, RK1808_PMU_CLKGATE_CON(0), 3, GFLAGS), ++ GATE(0, "pclk_pmu", "pclk_pmu_pre", CLK_IGNORE_UNUSED, RK1808_PMU_CLKGATE_CON(0), 4, GFLAGS), ++ GATE(0, "pclk_pmu_mem", "pclk_pmu_pre", CLK_IGNORE_UNUSED, RK1808_PMU_CLKGATE_CON(0), 5, GFLAGS), ++ GATE(PCLK_GPIO0_PMU, "pclk_gpio0_pmu", "pclk_pmu_pre", 0, RK1808_PMU_CLKGATE_CON(0), 6, GFLAGS), ++ GATE(PCLK_UART0_PMU, "pclk_uart0_pmu", "pclk_pmu_pre", 0, RK1808_PMU_CLKGATE_CON(0), 7, GFLAGS), ++ GATE(0, "pclk_cru_pmu", "pclk_pmu_pre", CLK_IGNORE_UNUSED, RK1808_PMU_CLKGATE_CON(0), 8, GFLAGS), ++ GATE(PCLK_I2C0_PMU, "pclk_i2c0_pmu", "pclk_pmu_pre", 0, RK1808_PMU_CLKGATE_CON(0), 9, GFLAGS), ++ ++ MUXPMUGRF(SCLK_32K_IOE, "clk_32k_ioe", mux_clk_32k_ioe_p, 0, ++ RK1808_PMUGRF_SOC_CON0, 0, 1, MFLAGS) ++}; ++ ++static void __iomem *rk1808_cru_base; ++ ++void rk1808_dump_cru(void) ++{ ++ if (rk1808_cru_base) { ++ pr_warn("CRU:\n"); ++ print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET, ++ 32, 4, rk1808_cru_base, ++ 0x500, false); ++ print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET, ++ 32, 4, rk1808_cru_base + 0x4000, ++ 0x100, false); ++ } ++} ++EXPORT_SYMBOL_GPL(rk1808_dump_cru); ++ ++static int rk1808_clk_panic(struct notifier_block *this, ++ unsigned long ev, void *ptr) ++{ ++ rk1808_dump_cru(); ++ return NOTIFY_DONE; ++} ++ ++static struct notifier_block rk1808_clk_panic_block = { ++ .notifier_call = rk1808_clk_panic, ++}; ++ ++static void __init rk1808_clk_init(struct device_node *np) ++{ ++ struct rockchip_clk_provider *ctx; ++ void __iomem *reg_base; ++ struct clk **clks; ++ ++ reg_base = of_iomap(np, 0); ++ if (!reg_base) { ++ pr_err("%s: could not map cru region\n", __func__); ++ return; ++ } ++ ++ rk1808_cru_base = reg_base; ++ ++ ctx = rockchip_clk_init(np, reg_base, CLK_NR_CLKS); ++ if (IS_ERR(ctx)) { ++ pr_err("%s: rockchip clk init failed\n", __func__); ++ iounmap(reg_base); ++ return; ++ } ++ clks = ctx->clk_data.clks; ++ ++ rockchip_clk_register_plls(ctx, rk1808_pll_clks, ++ ARRAY_SIZE(rk1808_pll_clks), ++ RK1808_GRF_SOC_STATUS0); ++ rockchip_clk_register_branches(ctx, rk1808_clk_branches, ++ ARRAY_SIZE(rk1808_clk_branches)); ++ ++ rockchip_clk_register_armclk(ctx, ARMCLK, "armclk", ++ 3, clks[PLL_APLL], clks[PLL_GPLL], ++ &rk1808_cpuclk_data, rk1808_cpuclk_rates, ++ ARRAY_SIZE(rk1808_cpuclk_rates)); ++ ++ rockchip_register_softrst(np, 16, reg_base + RK1808_SOFTRST_CON(0), ++ ROCKCHIP_SOFTRST_HIWORD_MASK); ++ ++ rockchip_register_restart_notifier(ctx, RK1808_GLB_SRST_FST, NULL); ++ ++ rockchip_clk_of_add_provider(np, ctx); ++ ++ atomic_notifier_chain_register(&panic_notifier_list, ++ &rk1808_clk_panic_block); ++} ++ ++CLK_OF_DECLARE(rk1808_cru, "rockchip,rk1808-cru", rk1808_clk_init); ++ ++static int __init clk_rk1808_probe(struct platform_device *pdev) ++{ ++ struct device_node *np = pdev->dev.of_node; ++ ++ rk1808_clk_init(np); ++ ++ return 0; ++} ++ ++static const struct of_device_id clk_rk1808_match_table[] = { ++ { ++ .compatible = "rockchip,rk1808-cru", ++ }, ++ { } ++}; ++MODULE_DEVICE_TABLE(of, clk_rk1808_match_table); ++ ++static struct platform_driver clk_rk1808_driver = { ++ .driver = { ++ .name = "clk-rk1808", ++ .of_match_table = clk_rk1808_match_table, ++ }, ++}; ++builtin_platform_driver_probe(clk_rk1808_driver, clk_rk1808_probe); ++ ++MODULE_DESCRIPTION("Rockchip RK1808 Clock Driver"); ++MODULE_LICENSE("GPL"); +diff --git a/drivers/clk/rockchip/clk-rk3036.c b/drivers/clk/rockchip/clk-rk3036.c +index 6a46f85ad837..02770ff67e14 100644 +--- a/drivers/clk/rockchip/clk-rk3036.c ++++ b/drivers/clk/rockchip/clk-rk3036.c +@@ -9,13 +9,18 @@ + + #include + #include ++#include + #include + #include ++#include + #include + #include + #include "clk.h" + + #define RK3036_GRF_SOC_STATUS0 0x14c ++#define RK3036_UART_FRAC_MAX_PRATE 600000000 ++#define RK3036_I2S_FRAC_MAX_PRATE 600000000 ++#define RK3036_SPDIF_FRAC_MAX_PRATE 600000000 + + enum rk3036_plls { + apll, dpll, gpll, +@@ -96,15 +101,19 @@ static struct rockchip_pll_rate_table rk3036_pll_rates[] = { + } + + static struct rockchip_cpuclk_rate_table rk3036_cpuclk_rates[] __initdata = { ++ RK3036_CPUCLK_RATE(1200000000, 4), ++ RK3036_CPUCLK_RATE(1008000000, 4), + RK3036_CPUCLK_RATE(816000000, 4), + RK3036_CPUCLK_RATE(600000000, 4), ++ RK3036_CPUCLK_RATE(408000000, 4), + RK3036_CPUCLK_RATE(312000000, 4), + }; + + static const struct rockchip_cpuclk_reg_data rk3036_cpuclk_data = { +- .core_reg = RK2928_CLKSEL_CON(0), +- .div_core_shift = 0, +- .div_core_mask = 0x1f, ++ .core_reg[0] = RK2928_CLKSEL_CON(0), ++ .div_core_shift[0] = 0, ++ .div_core_mask[0] = 0x1f, ++ .num_cores = 1, + .mux_core_alt = 1, + .mux_core_main = 0, + .mux_core_shift = 7, +@@ -113,13 +122,13 @@ static const struct rockchip_cpuclk_reg_data rk3036_cpuclk_data = { + + PNAME(mux_pll_p) = { "xin24m", "xin24m" }; + +-PNAME(mux_armclk_p) = { "apll", "gpll_armclk" }; + PNAME(mux_busclk_p) = { "apll", "dpll_cpu", "gpll_cpu" }; + PNAME(mux_ddrphy_p) = { "dpll_ddr", "gpll_ddr" }; + PNAME(mux_pll_src_3plls_p) = { "apll", "dpll", "gpll" }; + PNAME(mux_timer_p) = { "xin24m", "pclk_peri_src" }; + + PNAME(mux_pll_src_apll_dpll_gpll_usb480m_p) = { "apll", "dpll", "gpll", "usb480m" }; ++PNAME(mux_pll_src_dmyapll_dpll_gpll_xin24_p) = { "dummy_apll", "dpll", "gpll", "xin24m" }; + + PNAME(mux_mmc_src_p) = { "apll", "dpll", "gpll", "xin24m" }; + PNAME(mux_i2s_pre_p) = { "i2s_src", "i2s_frac", "ext_i2s", "xin12m" }; +@@ -157,7 +166,7 @@ static struct rockchip_clk_branch rk3036_uart2_fracmux __initdata = + RK2928_CLKSEL_CON(15), 8, 2, MFLAGS); + + static struct rockchip_clk_branch rk3036_i2s_fracmux __initdata = +- MUX(0, "i2s_pre", mux_i2s_pre_p, CLK_SET_RATE_PARENT, ++ MUX(SCLK_I2S_PRE, "i2s_pre", mux_i2s_pre_p, CLK_SET_RATE_PARENT, + RK2928_CLKSEL_CON(3), 8, 2, MFLAGS); + + static struct rockchip_clk_branch rk3036_spdif_fracmux __initdata = +@@ -193,32 +202,32 @@ static struct rockchip_clk_branch rk3036_clk_branches[] __initdata = { + RK2928_CLKSEL_CON(1), 4, 3, DFLAGS | CLK_DIVIDER_READ_ONLY, + RK2928_CLKGATE_CON(0), 7, GFLAGS), + +- GATE(0, "dpll_cpu", "dpll", 0, RK2928_CLKGATE_CON(10), 8, GFLAGS), +- GATE(0, "gpll_cpu", "gpll", 0, RK2928_CLKGATE_CON(0), 1, GFLAGS), +- COMPOSITE_NOGATE(0, "aclk_cpu_src", mux_busclk_p, 0, ++ GATE(0, "dpll_cpu", "dpll", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(10), 8, GFLAGS), ++ GATE(0, "gpll_cpu", "gpll", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(0), 1, GFLAGS), ++ COMPOSITE_NOGATE(0, "aclk_cpu_src", mux_busclk_p, CLK_IS_CRITICAL, + RK2928_CLKSEL_CON(0), 14, 2, MFLAGS, 8, 5, DFLAGS), +- GATE(ACLK_CPU, "aclk_cpu", "aclk_cpu_src", CLK_IGNORE_UNUSED, ++ GATE(ACLK_CPU, "aclk_cpu", "aclk_cpu_src", CLK_IS_CRITICAL, + RK2928_CLKGATE_CON(0), 3, GFLAGS), +- COMPOSITE_NOMUX(PCLK_CPU, "pclk_cpu", "aclk_cpu_src", CLK_IGNORE_UNUSED, ++ COMPOSITE_NOMUX(PCLK_CPU, "pclk_cpu", "aclk_cpu_src", CLK_IS_CRITICAL, + RK2928_CLKSEL_CON(1), 12, 3, DFLAGS | CLK_DIVIDER_READ_ONLY, + RK2928_CLKGATE_CON(0), 5, GFLAGS), +- COMPOSITE_NOMUX(HCLK_CPU, "hclk_cpu", "aclk_cpu_src", CLK_IGNORE_UNUSED, ++ COMPOSITE_NOMUX(HCLK_CPU, "hclk_cpu", "aclk_cpu_src", CLK_IS_CRITICAL, + RK2928_CLKSEL_CON(1), 8, 2, DFLAGS | CLK_DIVIDER_READ_ONLY, + RK2928_CLKGATE_CON(0), 4, GFLAGS), + +- COMPOSITE(0, "aclk_peri_src", mux_pll_src_3plls_p, 0, ++ COMPOSITE(0, "aclk_peri_src", mux_pll_src_3plls_p, CLK_IS_CRITICAL, + RK2928_CLKSEL_CON(10), 14, 2, MFLAGS, 0, 5, DFLAGS, + RK2928_CLKGATE_CON(2), 0, GFLAGS), + +- GATE(ACLK_PERI, "aclk_peri", "aclk_peri_src", 0, ++ GATE(ACLK_PERI, "aclk_peri", "aclk_peri_src", CLK_IS_CRITICAL, + RK2928_CLKGATE_CON(2), 1, GFLAGS), +- DIV(0, "pclk_peri_src", "aclk_peri_src", CLK_IGNORE_UNUSED, ++ DIV(0, "pclk_peri_src", "aclk_peri_src", CLK_IS_CRITICAL, + RK2928_CLKSEL_CON(10), 12, 2, DFLAGS | CLK_DIVIDER_POWER_OF_TWO), +- GATE(PCLK_PERI, "pclk_peri", "pclk_peri_src", 0, ++ GATE(PCLK_PERI, "pclk_peri", "pclk_peri_src", CLK_IS_CRITICAL, + RK2928_CLKGATE_CON(2), 3, GFLAGS), +- DIV(0, "hclk_peri_src", "aclk_peri_src", CLK_IGNORE_UNUSED, ++ DIV(0, "hclk_peri_src", "aclk_peri_src", CLK_IS_CRITICAL, + RK2928_CLKSEL_CON(10), 8, 2, DFLAGS | CLK_DIVIDER_POWER_OF_TWO), +- GATE(HCLK_PERI, "hclk_peri", "hclk_peri_src", 0, ++ GATE(HCLK_PERI, "hclk_peri", "hclk_peri_src", CLK_IS_CRITICAL, + RK2928_CLKGATE_CON(2), 2, GFLAGS), + + COMPOSITE_NODIV(SCLK_TIMER0, "sclk_timer0", mux_timer_p, CLK_IGNORE_UNUSED, +@@ -248,15 +257,15 @@ static struct rockchip_clk_branch rk3036_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "uart0_frac", "uart0_src", CLK_SET_RATE_PARENT, + RK2928_CLKSEL_CON(17), 0, + RK2928_CLKGATE_CON(1), 9, GFLAGS, +- &rk3036_uart0_fracmux), ++ &rk3036_uart0_fracmux, RK3036_UART_FRAC_MAX_PRATE), + COMPOSITE_FRACMUX(0, "uart1_frac", "uart1_src", CLK_SET_RATE_PARENT, + RK2928_CLKSEL_CON(18), 0, + RK2928_CLKGATE_CON(1), 11, GFLAGS, +- &rk3036_uart1_fracmux), ++ &rk3036_uart1_fracmux, RK3036_UART_FRAC_MAX_PRATE), + COMPOSITE_FRACMUX(0, "uart2_frac", "uart2_src", CLK_SET_RATE_PARENT, + RK2928_CLKSEL_CON(19), 0, + RK2928_CLKGATE_CON(1), 13, GFLAGS, +- &rk3036_uart2_fracmux), ++ &rk3036_uart2_fracmux, RK3036_UART_FRAC_MAX_PRATE), + + COMPOSITE(0, "aclk_vcodec", mux_pll_src_3plls_p, 0, + RK2928_CLKSEL_CON(32), 14, 2, MFLAGS, 8, 5, DFLAGS, +@@ -264,7 +273,7 @@ static struct rockchip_clk_branch rk3036_clk_branches[] __initdata = { + FACTOR_GATE(HCLK_VCODEC, "hclk_vcodec", "aclk_vcodec", 0, 1, 4, + RK2928_CLKGATE_CON(3), 12, GFLAGS), + +- COMPOSITE(0, "aclk_hvec", mux_pll_src_3plls_p, 0, ++ COMPOSITE(ACLK_HEVC, "aclk_hevc", mux_pll_src_3plls_p, 0, + RK2928_CLKSEL_CON(20), 0, 2, MFLAGS, 2, 5, DFLAGS, + RK2928_CLKGATE_CON(10), 6, GFLAGS), + +@@ -306,10 +315,10 @@ static struct rockchip_clk_branch rk3036_clk_branches[] __initdata = { + COMPOSITE(0, "i2s_src", mux_pll_src_3plls_p, 0, + RK2928_CLKSEL_CON(3), 14, 2, MFLAGS, 0, 7, DFLAGS, + RK2928_CLKGATE_CON(0), 9, GFLAGS), +- COMPOSITE_FRACMUX(0, "i2s_frac", "i2s_src", CLK_SET_RATE_PARENT, ++ COMPOSITE_FRACMUX(SCLK_I2S_FRAC, "i2s_frac", "i2s_src", CLK_SET_RATE_PARENT, + RK2928_CLKSEL_CON(7), 0, + RK2928_CLKGATE_CON(0), 10, GFLAGS, +- &rk3036_i2s_fracmux), ++ &rk3036_i2s_fracmux, RK3036_I2S_FRAC_MAX_PRATE), + COMPOSITE_NODIV(SCLK_I2S_OUT, "i2s_clkout", mux_i2s_clkout_p, 0, + RK2928_CLKSEL_CON(3), 12, 1, MFLAGS, + RK2928_CLKGATE_CON(0), 13, GFLAGS), +@@ -322,7 +331,7 @@ static struct rockchip_clk_branch rk3036_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "spdif_frac", "spdif_src", 0, + RK2928_CLKSEL_CON(9), 0, + RK2928_CLKGATE_CON(2), 12, GFLAGS, +- &rk3036_spdif_fracmux), ++ &rk3036_spdif_fracmux, RK3036_SPDIF_FRAC_MAX_PRATE), + + GATE(SCLK_OTGPHY0, "sclk_otgphy0", "xin12m", CLK_IGNORE_UNUSED, + RK2928_CLKGATE_CON(1), 5, GFLAGS), +@@ -339,7 +348,7 @@ static struct rockchip_clk_branch rk3036_clk_branches[] __initdata = { + RK2928_CLKSEL_CON(16), 8, 2, MFLAGS, 10, 5, DFLAGS, + RK2928_CLKGATE_CON(10), 4, GFLAGS), + +- COMPOSITE(SCLK_SFC, "sclk_sfc", mux_pll_src_apll_dpll_gpll_usb480m_p, 0, ++ COMPOSITE(SCLK_SFC, "sclk_sfc", mux_pll_src_dmyapll_dpll_gpll_xin24_p, 0, + RK2928_CLKSEL_CON(16), 0, 2, MFLAGS, 2, 5, DFLAGS, + RK2928_CLKGATE_CON(10), 5, GFLAGS), + +@@ -369,7 +378,7 @@ static struct rockchip_clk_branch rk3036_clk_branches[] __initdata = { + + /* pclk_cpu gates */ + GATE(PCLK_GRF, "pclk_grf", "pclk_cpu", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(5), 4, GFLAGS), +- GATE(PCLK_DDRUPCTL, "pclk_ddrupctl", "pclk_cpu", 0, RK2928_CLKGATE_CON(5), 7, GFLAGS), ++ GATE(PCLK_DDRUPCTL, "pclk_ddrupctl", "pclk_cpu", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(5), 7, GFLAGS), + GATE(PCLK_ACODEC, "pclk_acodec", "pclk_cpu", 0, RK2928_CLKGATE_CON(5), 14, GFLAGS), + GATE(PCLK_HDMI, "pclk_hdmi", "pclk_cpu", 0, RK2928_CLKGATE_CON(3), 8, GFLAGS), + +@@ -402,7 +411,7 @@ static struct rockchip_clk_branch rk3036_clk_branches[] __initdata = { + GATE(HCLK_OTG0, "hclk_otg0", "hclk_peri", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(5), 13, GFLAGS), + GATE(HCLK_OTG1, "hclk_otg1", "hclk_peri", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(7), 3, GFLAGS), + GATE(HCLK_I2S, "hclk_i2s", "hclk_peri", 0, RK2928_CLKGATE_CON(7), 2, GFLAGS), +- GATE(0, "hclk_sfc", "hclk_peri", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(3), 14, GFLAGS), ++ GATE(HCLK_SFC, "hclk_sfc", "hclk_peri", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(3), 14, GFLAGS), + GATE(HCLK_MAC, "hclk_mac", "hclk_peri", 0, RK2928_CLKGATE_CON(3), 5, GFLAGS), + + /* pclk_peri gates */ +@@ -423,19 +432,24 @@ static struct rockchip_clk_branch rk3036_clk_branches[] __initdata = { + GATE(PCLK_GPIO2, "pclk_gpio2", "pclk_peri", 0, RK2928_CLKGATE_CON(8), 11, GFLAGS), + }; + +-static const char *const rk3036_critical_clocks[] __initconst = { +- "aclk_cpu", +- "aclk_peri", +- "hclk_peri", +- "pclk_peri", +- "pclk_ddrupctl", +-}; ++static void __iomem *rk3036_cru_base; ++ ++static void rk3036_dump_cru(void) ++{ ++ if (rk3036_cru_base) { ++ pr_warn("CRU:\n"); ++ print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET, ++ 32, 4, rk3036_cru_base, ++ 0x1f8, false); ++ } ++} + + static void __init rk3036_clk_init(struct device_node *np) + { + struct rockchip_clk_provider *ctx; + void __iomem *reg_base; + struct clk *clk; ++ struct clk **clks; + + reg_base = of_iomap(np, 0); + if (!reg_base) { +@@ -456,6 +470,7 @@ static void __init rk3036_clk_init(struct device_node *np) + iounmap(reg_base); + return; + } ++ clks = ctx->clk_data.clks; + + clk = clk_register_fixed_factor(NULL, "usb480m", "xin24m", 0, 20, 1); + if (IS_ERR(clk)) +@@ -467,11 +482,9 @@ static void __init rk3036_clk_init(struct device_node *np) + RK3036_GRF_SOC_STATUS0); + rockchip_clk_register_branches(ctx, rk3036_clk_branches, + ARRAY_SIZE(rk3036_clk_branches)); +- rockchip_clk_protect_critical(rk3036_critical_clocks, +- ARRAY_SIZE(rk3036_critical_clocks)); + + rockchip_clk_register_armclk(ctx, ARMCLK, "armclk", +- mux_armclk_p, ARRAY_SIZE(mux_armclk_p), ++ 2, clks[PLL_APLL], clks[PLL_GPLL], + &rk3036_cpuclk_data, rk3036_cpuclk_rates, + ARRAY_SIZE(rk3036_cpuclk_rates)); + +@@ -481,5 +494,38 @@ static void __init rk3036_clk_init(struct device_node *np) + rockchip_register_restart_notifier(ctx, RK2928_GLB_SRST_FST, NULL); + + rockchip_clk_of_add_provider(np, ctx); ++ ++ if (!rk_dump_cru) { ++ rk3036_cru_base = reg_base; ++ rk_dump_cru = rk3036_dump_cru; ++ } + } + CLK_OF_DECLARE(rk3036_cru, "rockchip,rk3036-cru", rk3036_clk_init); ++ ++static int __init clk_rk3036_probe(struct platform_device *pdev) ++{ ++ struct device_node *np = pdev->dev.of_node; ++ ++ rk3036_clk_init(np); ++ ++ return 0; ++} ++ ++static const struct of_device_id clk_rk3036_match_table[] = { ++ { ++ .compatible = "rockchip,rk3036-cru", ++ }, ++ { } ++}; ++MODULE_DEVICE_TABLE(of, clk_rk3036_match_table); ++ ++static struct platform_driver clk_rk3036_driver = { ++ .driver = { ++ .name = "clk-rk3036", ++ .of_match_table = clk_rk3036_match_table, ++ }, ++}; ++builtin_platform_driver_probe(clk_rk3036_driver, clk_rk3036_probe); ++ ++MODULE_DESCRIPTION("Rockchip RK3036 Clock Driver"); ++MODULE_LICENSE("GPL"); +diff --git a/drivers/clk/rockchip/clk-rk3128.c b/drivers/clk/rockchip/clk-rk3128.c +index 4b1122e98e16..6f6f44ac7257 100644 +--- a/drivers/clk/rockchip/clk-rk3128.c ++++ b/drivers/clk/rockchip/clk-rk3128.c +@@ -6,13 +6,19 @@ + + #include + #include ++#include + #include + #include ++#include ++#include + #include + #include + #include "clk.h" + + #define RK3128_GRF_SOC_STATUS0 0x14c ++#define RK3128_UART_FRAC_MAX_PRATE 600000000 ++#define RK3128_I2S_FRAC_MAX_PRATE 600000000 ++#define RK3128_SPDIF_FRAC_MAX_PRATE 600000000 + + enum rk3128_plls { + apll, dpll, cpll, gpll, +@@ -117,9 +123,10 @@ static struct rockchip_cpuclk_rate_table rk3128_cpuclk_rates[] __initdata = { + }; + + static const struct rockchip_cpuclk_reg_data rk3128_cpuclk_data = { +- .core_reg = RK2928_CLKSEL_CON(0), +- .div_core_shift = 0, +- .div_core_mask = 0x1f, ++ .core_reg[0] = RK2928_CLKSEL_CON(0), ++ .div_core_shift[0] = 0, ++ .div_core_mask[0] = 0x1f, ++ .num_cores = 1, + .mux_core_alt = 1, + .mux_core_main = 0, + .mux_core_shift = 7, +@@ -129,7 +136,6 @@ static const struct rockchip_cpuclk_reg_data rk3128_cpuclk_data = { + PNAME(mux_pll_p) = { "clk_24m", "xin24m" }; + + PNAME(mux_ddrphy_p) = { "dpll_ddr", "gpll_div2_ddr" }; +-PNAME(mux_armclk_p) = { "apll_core", "gpll_div2_core" }; + PNAME(mux_usb480m_p) = { "usb480m_phy", "xin24m" }; + PNAME(mux_aclk_cpu_src_p) = { "cpll", "gpll", "gpll_div2", "gpll_div3" }; + +@@ -231,15 +237,15 @@ static struct rockchip_clk_branch common_clk_branches[] __initdata = { + RK2928_MISC_CON, 15, 1, MFLAGS), + + /* PD_CPU */ +- COMPOSITE(0, "aclk_cpu_src", mux_aclk_cpu_src_p, 0, ++ COMPOSITE(0, "aclk_cpu_src", mux_aclk_cpu_src_p, CLK_IS_CRITICAL, + RK2928_CLKSEL_CON(0), 13, 2, MFLAGS, 8, 5, DFLAGS, + RK2928_CLKGATE_CON(0), 1, GFLAGS), +- GATE(ACLK_CPU, "aclk_cpu", "aclk_cpu_src", 0, ++ GATE(ACLK_CPU, "aclk_cpu", "aclk_cpu_src", CLK_IS_CRITICAL, + RK2928_CLKGATE_CON(0), 3, GFLAGS), +- COMPOSITE_NOMUX(HCLK_CPU, "hclk_cpu", "aclk_cpu_src", 0, ++ COMPOSITE_NOMUX(HCLK_CPU, "hclk_cpu", "aclk_cpu_src", CLK_IS_CRITICAL, + RK2928_CLKSEL_CON(1), 8, 2, DFLAGS, + RK2928_CLKGATE_CON(0), 4, GFLAGS), +- COMPOSITE_NOMUX(PCLK_CPU, "pclk_cpu", "aclk_cpu_src", 0, ++ COMPOSITE_NOMUX(PCLK_CPU, "pclk_cpu", "aclk_cpu_src", CLK_IS_CRITICAL, + RK2928_CLKSEL_CON(1), 12, 2, DFLAGS, + RK2928_CLKGATE_CON(0), 5, GFLAGS), + COMPOSITE_NOMUX(SCLK_CRYPTO, "clk_crypto", "aclk_cpu_src", 0, +@@ -263,34 +269,33 @@ static struct rockchip_clk_branch common_clk_branches[] __initdata = { + RK2928_CLKGATE_CON(3), 10, GFLAGS), + + /* PD_VIO */ +- COMPOSITE(ACLK_VIO0, "aclk_vio0", mux_pll_src_5plls_p, 0, ++ COMPOSITE(ACLK_VIO0, "aclk_vio0", mux_pll_src_5plls_p, CLK_IS_CRITICAL, + RK2928_CLKSEL_CON(31), 5, 3, MFLAGS, 0, 5, DFLAGS, + RK2928_CLKGATE_CON(3), 0, GFLAGS), + COMPOSITE(ACLK_VIO1, "aclk_vio1", mux_pll_src_5plls_p, 0, + RK2928_CLKSEL_CON(31), 13, 3, MFLAGS, 8, 5, DFLAGS, + RK2928_CLKGATE_CON(1), 4, GFLAGS), +- COMPOSITE(HCLK_VIO, "hclk_vio", mux_pll_src_4plls_p, 0, +- RK2928_CLKSEL_CON(30), 14, 2, MFLAGS, 8, 5, DFLAGS, ++ FACTOR_GATE(HCLK_VIO, "hclk_vio", "aclk_vio0", CLK_IS_CRITICAL, 1, 4, + RK2928_CLKGATE_CON(0), 11, GFLAGS), + + /* PD_PERI */ +- GATE(0, "gpll_peri", "gpll", CLK_IGNORE_UNUSED, ++ GATE(0, "gpll_peri", "gpll", CLK_IS_CRITICAL, + RK2928_CLKGATE_CON(2), 0, GFLAGS), +- GATE(0, "cpll_peri", "cpll", CLK_IGNORE_UNUSED, ++ GATE(0, "cpll_peri", "cpll", CLK_IS_CRITICAL, + RK2928_CLKGATE_CON(2), 0, GFLAGS), +- GATE(0, "gpll_div2_peri", "gpll_div2", CLK_IGNORE_UNUSED, ++ GATE(0, "gpll_div2_peri", "gpll_div2", CLK_IS_CRITICAL, + RK2928_CLKGATE_CON(2), 0, GFLAGS), +- GATE(0, "gpll_div3_peri", "gpll_div3", CLK_IGNORE_UNUSED, ++ GATE(0, "gpll_div3_peri", "gpll_div3", CLK_IS_CRITICAL, + RK2928_CLKGATE_CON(2), 0, GFLAGS), +- COMPOSITE_NOGATE(0, "aclk_peri_src", mux_aclk_peri_src_p, 0, ++ COMPOSITE_NOGATE(0, "aclk_peri_src", mux_aclk_peri_src_p, CLK_IS_CRITICAL, + RK2928_CLKSEL_CON(10), 14, 2, MFLAGS, 0, 5, DFLAGS), +- COMPOSITE_NOMUX(PCLK_PERI, "pclk_peri", "aclk_peri_src", 0, ++ COMPOSITE_NOMUX(PCLK_PERI, "pclk_peri", "aclk_peri_src", CLK_IS_CRITICAL, + RK2928_CLKSEL_CON(10), 12, 2, DFLAGS | CLK_DIVIDER_POWER_OF_TWO, + RK2928_CLKGATE_CON(2), 3, GFLAGS), +- COMPOSITE_NOMUX(HCLK_PERI, "hclk_peri", "aclk_peri_src", 0, ++ COMPOSITE_NOMUX(HCLK_PERI, "hclk_peri", "aclk_peri_src", CLK_IS_CRITICAL, + RK2928_CLKSEL_CON(10), 8, 2, DFLAGS | CLK_DIVIDER_POWER_OF_TWO, + RK2928_CLKGATE_CON(2), 2, GFLAGS), +- GATE(ACLK_PERI, "aclk_peri", "aclk_peri_src", 0, ++ GATE(ACLK_PERI, "aclk_peri", "aclk_peri_src", CLK_IS_CRITICAL, + RK2928_CLKGATE_CON(2), 1, GFLAGS), + + GATE(SCLK_TIMER0, "sclk_timer0", "xin24m", 0, +@@ -303,7 +308,7 @@ static struct rockchip_clk_branch common_clk_branches[] __initdata = { + RK2928_CLKGATE_CON(10), 6, GFLAGS), + GATE(SCLK_TIMER4, "sclk_timer4", "xin24m", 0, + RK2928_CLKGATE_CON(10), 7, GFLAGS), +- GATE(SCLK_TIMER5, "sclk_timer5", "xin24m", 0, ++ GATE(SCLK_TIMER5, "sclk_timer5", "xin24m", CLK_IS_CRITICAL, + RK2928_CLKGATE_CON(10), 8, GFLAGS), + + GATE(SCLK_PVTM_CORE, "clk_pvtm_core", "xin24m", 0, +@@ -312,7 +317,7 @@ static struct rockchip_clk_branch common_clk_branches[] __initdata = { + RK2928_CLKGATE_CON(10), 1, GFLAGS), + GATE(SCLK_PVTM_FUNC, "clk_pvtm_func", "xin24m", 0, + RK2928_CLKGATE_CON(10), 2, GFLAGS), +- GATE(SCLK_MIPI_24M, "clk_mipi_24m", "xin24m", CLK_IGNORE_UNUSED, ++ GATE(SCLK_MIPI_24M, "clk_mipi_24m", "xin24m", 0, + RK2928_CLKGATE_CON(2), 15, GFLAGS), + + COMPOSITE(SCLK_SDMMC, "sclk_sdmmc0", mux_mmc_src_p, 0, +@@ -359,7 +364,7 @@ static struct rockchip_clk_branch common_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "i2s0_frac", "i2s0_src", CLK_SET_RATE_PARENT, + RK2928_CLKSEL_CON(8), 0, + RK2928_CLKGATE_CON(4), 5, GFLAGS, +- &rk3128_i2s0_fracmux), ++ &rk3128_i2s0_fracmux, RK3128_I2S_FRAC_MAX_PRATE), + GATE(SCLK_I2S0, "sclk_i2s0", "i2s0_pre", CLK_SET_RATE_PARENT, + RK2928_CLKGATE_CON(4), 6, GFLAGS), + +@@ -369,7 +374,7 @@ static struct rockchip_clk_branch common_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "i2s1_frac", "i2s1_src", CLK_SET_RATE_PARENT, + RK2928_CLKSEL_CON(7), 0, + RK2928_CLKGATE_CON(0), 10, GFLAGS, +- &rk3128_i2s1_fracmux), ++ &rk3128_i2s1_fracmux, RK3128_I2S_FRAC_MAX_PRATE), + GATE(SCLK_I2S1, "sclk_i2s1", "i2s1_pre", CLK_SET_RATE_PARENT, + RK2928_CLKGATE_CON(0), 14, GFLAGS), + COMPOSITE_NODIV(SCLK_I2S_OUT, "i2s_out", mux_i2s_out_p, 0, +@@ -382,7 +387,7 @@ static struct rockchip_clk_branch common_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "spdif_frac", "sclk_spdif_src", CLK_SET_RATE_PARENT, + RK2928_CLKSEL_CON(20), 0, + RK2928_CLKGATE_CON(2), 12, GFLAGS, +- &rk3128_spdif_fracmux), ++ &rk3128_spdif_fracmux, RK3128_SPDIF_FRAC_MAX_PRATE), + + GATE(0, "jtag", "ext_jtag", CLK_IGNORE_UNUSED, + RK2928_CLKGATE_CON(1), 3, GFLAGS), +@@ -419,15 +424,15 @@ static struct rockchip_clk_branch common_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "uart0_frac", "uart0_src", CLK_SET_RATE_PARENT, + RK2928_CLKSEL_CON(17), 0, + RK2928_CLKGATE_CON(1), 9, GFLAGS, +- &rk3128_uart0_fracmux), ++ &rk3128_uart0_fracmux, RK3128_UART_FRAC_MAX_PRATE), + COMPOSITE_FRACMUX(0, "uart1_frac", "uart1_src", CLK_SET_RATE_PARENT, + RK2928_CLKSEL_CON(18), 0, + RK2928_CLKGATE_CON(1), 11, GFLAGS, +- &rk3128_uart1_fracmux), ++ &rk3128_uart1_fracmux, RK3128_UART_FRAC_MAX_PRATE), + COMPOSITE_FRACMUX(0, "uart2_frac", "uart2_src", CLK_SET_RATE_PARENT, + RK2928_CLKSEL_CON(19), 0, + RK2928_CLKGATE_CON(1), 13, GFLAGS, +- &rk3128_uart2_fracmux), ++ &rk3128_uart2_fracmux, RK3128_UART_FRAC_MAX_PRATE), + + COMPOSITE(SCLK_MAC_SRC, "sclk_gmac_src", mux_pll_src_3plls_p, 0, + RK2928_CLKSEL_CON(5), 6, 2, MFLAGS, 0, 5, DFLAGS, +@@ -451,7 +456,7 @@ static struct rockchip_clk_branch common_clk_branches[] __initdata = { + RK2928_CLKSEL_CON(2), 14, 2, MFLAGS, 8, 5, DFLAGS, + RK2928_CLKGATE_CON(10), 15, GFLAGS), + +- COMPOSITE_NOMUX(PCLK_PMU_PRE, "pclk_pmu_pre", "cpll", 0, ++ COMPOSITE_NOMUX(PCLK_PMU_PRE, "pclk_pmu_pre", "cpll", CLK_IS_CRITICAL, + RK2928_CLKSEL_CON(29), 8, 6, DFLAGS, + RK2928_CLKGATE_CON(1), 0, GFLAGS), + +@@ -473,7 +478,7 @@ static struct rockchip_clk_branch common_clk_branches[] __initdata = { + GATE(HCLK_RGA, "hclk_rga", "hclk_vio", 0, RK2928_CLKGATE_CON(6), 10, GFLAGS), + GATE(HCLK_LCDC0, "hclk_lcdc0", "hclk_vio", 0, RK2928_CLKGATE_CON(6), 1, GFLAGS), + GATE(HCLK_IEP, "hclk_iep", "hclk_vio", 0, RK2928_CLKGATE_CON(9), 7, GFLAGS), +- GATE(0, "hclk_vio_niu", "hclk_vio", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(6), 12, GFLAGS), ++ GATE(0, "hclk_vio_niu", "hclk_vio", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(6), 12, GFLAGS), + GATE(HCLK_CIF, "hclk_cif", "hclk_vio", 0, RK2928_CLKGATE_CON(6), 4, GFLAGS), + GATE(HCLK_EBC, "hclk_ebc", "hclk_vio", 0, RK2928_CLKGATE_CON(9), 9, GFLAGS), + +@@ -499,6 +504,7 @@ static struct rockchip_clk_branch common_clk_branches[] __initdata = { + GATE(0, "hclk_emmc_peri", "hclk_peri", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(3), 6, GFLAGS), + GATE(HCLK_NANDC, "hclk_nandc", "hclk_peri", 0, RK2928_CLKGATE_CON(5), 9, GFLAGS), + GATE(HCLK_USBHOST, "hclk_usbhost", "hclk_peri", 0, RK2928_CLKGATE_CON(10), 14, GFLAGS), ++ GATE(HCLK_SFC, "hclk_sfc", "hclk_peri", 0, RK2928_CLKGATE_CON(7), 1, GFLAGS), + + GATE(PCLK_SIM_CARD, "pclk_sim_card", "pclk_peri", 0, RK2928_CLKGATE_CON(9), 12, GFLAGS), + GATE(PCLK_GMAC, "pclk_gmac", "pclk_peri", 0, RK2928_CLKGATE_CON(10), 11, GFLAGS), +@@ -533,8 +539,8 @@ static struct rockchip_clk_branch common_clk_branches[] __initdata = { + GATE(0, "pclk_grf", "pclk_cpu", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(5), 4, GFLAGS), + GATE(0, "pclk_mipiphy", "pclk_cpu", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(5), 0, GFLAGS), + +- GATE(0, "pclk_pmu", "pclk_pmu_pre", 0, RK2928_CLKGATE_CON(9), 2, GFLAGS), +- GATE(0, "pclk_pmu_niu", "pclk_pmu_pre", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(9), 3, GFLAGS), ++ GATE(0, "pclk_pmu", "pclk_pmu_pre", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(9), 2, GFLAGS), ++ GATE(0, "pclk_pmu_niu", "pclk_pmu_pre", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(9), 3, GFLAGS), + + /* PD_MMC */ + MMC(SCLK_SDMMC_DRV, "sdmmc_drv", "sclk_sdmmc", RK3228_SDMMC_CON0, 1), +@@ -562,21 +568,30 @@ static struct rockchip_clk_branch rk3128_clk_branches[] __initdata = { + GATE(PCLK_HDMI, "pclk_hdmi", "pclk_cpu", 0, RK2928_CLKGATE_CON(3), 8, GFLAGS), + }; + +-static const char *const rk3128_critical_clocks[] __initconst = { +- "aclk_cpu", +- "hclk_cpu", +- "pclk_cpu", +- "aclk_peri", +- "hclk_peri", +- "pclk_peri", +- "pclk_pmu", +- "sclk_timer5", +-}; ++static void __iomem *rk312x_reg_base; ++ ++void rkclk_cpuclk_div_setting(int div) ++{ ++ if (cpu_is_rk312x()) ++ writel_relaxed((0x001f0000 | (div - 1)), ++ rk312x_reg_base + RK2928_CLKSEL_CON(0)); ++} ++ ++static void rk3128_dump_cru(void) ++{ ++ if (rk312x_reg_base) { ++ pr_warn("CRU:\n"); ++ print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET, ++ 32, 4, rk312x_reg_base, ++ 0x1f8, false); ++ } ++} + + static struct rockchip_clk_provider *__init rk3128_common_clk_init(struct device_node *np) + { + struct rockchip_clk_provider *ctx; + void __iomem *reg_base; ++ struct clk **clks; + + reg_base = of_iomap(np, 0); + if (!reg_base) { +@@ -584,12 +599,14 @@ static struct rockchip_clk_provider *__init rk3128_common_clk_init(struct device + return ERR_PTR(-ENOMEM); + } + ++ rk312x_reg_base = reg_base; + ctx = rockchip_clk_init(np, reg_base, CLK_NR_CLKS); + if (IS_ERR(ctx)) { + pr_err("%s: rockchip clk init failed\n", __func__); + iounmap(reg_base); + return ERR_PTR(-ENOMEM); + } ++ clks = ctx->clk_data.clks; + + rockchip_clk_register_plls(ctx, rk3128_pll_clks, + ARRAY_SIZE(rk3128_pll_clks), +@@ -598,7 +615,7 @@ static struct rockchip_clk_provider *__init rk3128_common_clk_init(struct device + ARRAY_SIZE(common_clk_branches)); + + rockchip_clk_register_armclk(ctx, ARMCLK, "armclk", +- mux_armclk_p, ARRAY_SIZE(mux_armclk_p), ++ 2, clks[PLL_APLL], clks[PLL_GPLL_DIV2], + &rk3128_cpuclk_data, rk3128_cpuclk_rates, + ARRAY_SIZE(rk3128_cpuclk_rates)); + +@@ -607,6 +624,9 @@ static struct rockchip_clk_provider *__init rk3128_common_clk_init(struct device + + rockchip_register_restart_notifier(ctx, RK2928_GLB_SRST_FST, NULL); + ++ if (!rk_dump_cru) ++ rk_dump_cru = rk3128_dump_cru; ++ + return ctx; + } + +@@ -620,8 +640,6 @@ static void __init rk3126_clk_init(struct device_node *np) + + rockchip_clk_register_branches(ctx, rk3126_clk_branches, + ARRAY_SIZE(rk3126_clk_branches)); +- rockchip_clk_protect_critical(rk3128_critical_clocks, +- ARRAY_SIZE(rk3128_critical_clocks)); + + rockchip_clk_of_add_provider(np, ctx); + } +@@ -638,10 +656,60 @@ static void __init rk3128_clk_init(struct device_node *np) + + rockchip_clk_register_branches(ctx, rk3128_clk_branches, + ARRAY_SIZE(rk3128_clk_branches)); +- rockchip_clk_protect_critical(rk3128_critical_clocks, +- ARRAY_SIZE(rk3128_critical_clocks)); + + rockchip_clk_of_add_provider(np, ctx); + } + + CLK_OF_DECLARE(rk3128_cru, "rockchip,rk3128-cru", rk3128_clk_init); ++ ++struct clk_rk3128_inits { ++ void (*inits)(struct device_node *np); ++}; ++ ++static const struct clk_rk3128_inits clk_rk3126_init = { ++ .inits = rk3126_clk_init, ++}; ++ ++static const struct clk_rk3128_inits clk_rk3128_init = { ++ .inits = rk3128_clk_init, ++}; ++ ++static const struct of_device_id clk_rk3128_match_table[] = { ++ { ++ .compatible = "rockchip,rk3126-cru", ++ .data = &clk_rk3126_init, ++ }, { ++ .compatible = "rockchip,rk3128-cru", ++ .data = &clk_rk3128_init, ++ }, ++ { } ++}; ++MODULE_DEVICE_TABLE(of, clk_rk3128_match_table); ++ ++static int __init clk_rk3128_probe(struct platform_device *pdev) ++{ ++ struct device_node *np = pdev->dev.of_node; ++ const struct of_device_id *match; ++ const struct clk_rk3128_inits *init_data; ++ ++ match = of_match_device(clk_rk3128_match_table, &pdev->dev); ++ if (!match || !match->data) ++ return -EINVAL; ++ ++ init_data = match->data; ++ if (init_data->inits) ++ init_data->inits(np); ++ ++ return 0; ++} ++ ++static struct platform_driver clk_rk3128_driver = { ++ .driver = { ++ .name = "clk-rk3128", ++ .of_match_table = clk_rk3128_match_table, ++ }, ++}; ++builtin_platform_driver_probe(clk_rk3128_driver, clk_rk3128_probe); ++ ++MODULE_DESCRIPTION("Rockchip RK3128 Clock Driver"); ++MODULE_LICENSE("GPL"); +diff --git a/drivers/clk/rockchip/clk-rk3188.c b/drivers/clk/rockchip/clk-rk3188.c +index 730020fcc7fe..052669bf8978 100644 +--- a/drivers/clk/rockchip/clk-rk3188.c ++++ b/drivers/clk/rockchip/clk-rk3188.c +@@ -5,15 +5,21 @@ + */ + + #include ++#include + #include + #include + #include + #include ++#include + #include + #include "clk.h" + + #define RK3066_GRF_SOC_STATUS 0x15c + #define RK3188_GRF_SOC_STATUS 0xac ++#define RK3188_UART_FRAC_MAX_PRATE 600000000 ++#define RK3188_I2S_FRAC_MAX_PRATE 600000000 ++#define RK3188_SPDIF_FRAC_MAX_PRATE 600000000 ++#define RK3188_HSADC_FRAC_MAX_PRATE 300000000 + + enum rk3188_plls { + apll, cpll, dpll, gpll, +@@ -145,9 +151,10 @@ static struct rockchip_cpuclk_rate_table rk3066_cpuclk_rates[] __initdata = { + }; + + static const struct rockchip_cpuclk_reg_data rk3066_cpuclk_data = { +- .core_reg = RK2928_CLKSEL_CON(0), +- .div_core_shift = 0, +- .div_core_mask = 0x1f, ++ .core_reg[0] = RK2928_CLKSEL_CON(0), ++ .div_core_shift[0] = 0, ++ .div_core_mask[0] = 0x1f, ++ .num_cores = 1, + .mux_core_alt = 1, + .mux_core_main = 0, + .mux_core_shift = 8, +@@ -184,9 +191,10 @@ static struct rockchip_cpuclk_rate_table rk3188_cpuclk_rates[] __initdata = { + }; + + static const struct rockchip_cpuclk_reg_data rk3188_cpuclk_data = { +- .core_reg = RK2928_CLKSEL_CON(0), +- .div_core_shift = 9, +- .div_core_mask = 0x1f, ++ .core_reg[0] = RK2928_CLKSEL_CON(0), ++ .div_core_shift[0] = 9, ++ .div_core_mask[0] = 0x1f, ++ .num_cores = 1, + .mux_core_alt = 1, + .mux_core_main = 0, + .mux_core_shift = 8, +@@ -194,7 +202,6 @@ static const struct rockchip_cpuclk_reg_data rk3188_cpuclk_data = { + }; + + PNAME(mux_pll_p) = { "xin24m", "xin32k" }; +-PNAME(mux_armclk_p) = { "apll", "gpll_armclk" }; + PNAME(mux_ddrphy_p) = { "dpll", "gpll_ddr" }; + PNAME(mux_pll_src_gpll_cpll_p) = { "gpll", "cpll" }; + PNAME(mux_pll_src_cpll_gpll_p) = { "cpll", "gpll" }; +@@ -299,14 +306,14 @@ static struct rockchip_clk_branch common_clk_branches[] __initdata = { + RK2928_CLKSEL_CON(26), 8, 1, MFLAGS, 0, 2, DFLAGS | CLK_DIVIDER_POWER_OF_TWO, + RK2928_CLKGATE_CON(0), 2, GFLAGS), + +- GATE(ACLK_CPU, "aclk_cpu", "aclk_cpu_pre", 0, ++ GATE(ACLK_CPU, "aclk_cpu", "aclk_cpu_pre", CLK_IS_CRITICAL, + RK2928_CLKGATE_CON(0), 3, GFLAGS), + + GATE(0, "atclk_cpu", "pclk_cpu_pre", 0, + RK2928_CLKGATE_CON(0), 6, GFLAGS), +- GATE(PCLK_CPU, "pclk_cpu", "pclk_cpu_pre", 0, ++ GATE(PCLK_CPU, "pclk_cpu", "pclk_cpu_pre", CLK_IS_CRITICAL, + RK2928_CLKGATE_CON(0), 5, GFLAGS), +- GATE(HCLK_CPU, "hclk_cpu", "hclk_cpu_pre", CLK_IGNORE_UNUSED, ++ GATE(HCLK_CPU, "hclk_cpu", "hclk_cpu_pre", CLK_IS_CRITICAL, + RK2928_CLKGATE_CON(0), 4, GFLAGS), + + COMPOSITE(0, "aclk_lcdc0_pre", mux_pll_src_cpll_gpll_p, CLK_IGNORE_UNUSED, +@@ -316,12 +323,12 @@ static struct rockchip_clk_branch common_clk_branches[] __initdata = { + RK2928_CLKSEL_CON(31), 15, 1, MFLAGS, 8, 5, DFLAGS, + RK2928_CLKGATE_CON(1), 4, GFLAGS), + +- GATE(ACLK_PERI, "aclk_peri", "aclk_peri_pre", 0, ++ GATE(ACLK_PERI, "aclk_peri", "aclk_peri_pre", CLK_IS_CRITICAL, + RK2928_CLKGATE_CON(2), 1, GFLAGS), +- COMPOSITE_NOMUX(HCLK_PERI, "hclk_peri", "aclk_peri_pre", 0, ++ COMPOSITE_NOMUX(HCLK_PERI, "hclk_peri", "aclk_peri_pre", CLK_IS_CRITICAL, + RK2928_CLKSEL_CON(10), 8, 2, DFLAGS | CLK_DIVIDER_POWER_OF_TWO, + RK2928_CLKGATE_CON(2), 2, GFLAGS), +- COMPOSITE_NOMUX(PCLK_PERI, "pclk_peri", "aclk_peri_pre", 0, ++ COMPOSITE_NOMUX(PCLK_PERI, "pclk_peri", "aclk_peri_pre", CLK_IS_CRITICAL, + RK2928_CLKSEL_CON(10), 12, 2, DFLAGS | CLK_DIVIDER_POWER_OF_TWO, + RK2928_CLKGATE_CON(2), 3, GFLAGS), + +@@ -354,7 +361,7 @@ static struct rockchip_clk_branch common_clk_branches[] __initdata = { + RK2928_CLKGATE_CON(2), 5, GFLAGS), + MUX(SCLK_MAC, "sclk_macref", mux_sclk_macref_p, CLK_SET_RATE_PARENT, + RK2928_CLKSEL_CON(21), 4, 1, MFLAGS), +- GATE(0, "sclk_mac_lbtest", "sclk_macref", 0, ++ GATE(0, "sclk_mac_lbtest", "sclk_macref", CLK_IS_CRITICAL, + RK2928_CLKGATE_CON(2), 12, GFLAGS), + + COMPOSITE(0, "hsadc_src", mux_pll_src_gpll_cpll_p, 0, +@@ -363,7 +370,7 @@ static struct rockchip_clk_branch common_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "hsadc_frac", "hsadc_src", 0, + RK2928_CLKSEL_CON(23), 0, + RK2928_CLKGATE_CON(2), 7, GFLAGS, +- &common_hsadc_out_fracmux), ++ &common_hsadc_out_fracmux, RK3188_HSADC_FRAC_MAX_PRATE), + INVERTER(SCLK_HSADC, "sclk_hsadc", "sclk_hsadc_out", + RK2928_CLKSEL_CON(22), 7, IFLAGS), + +@@ -377,7 +384,7 @@ static struct rockchip_clk_branch common_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "spdif_frac", "spdif_pre", CLK_SET_RATE_PARENT, + RK2928_CLKSEL_CON(9), 0, + RK2928_CLKGATE_CON(0), 14, GFLAGS, +- &common_spdif_fracmux), ++ &common_spdif_fracmux, RK3188_SPDIF_FRAC_MAX_PRATE), + + /* + * Clock-Architecture Diagram 4 +@@ -411,28 +418,28 @@ static struct rockchip_clk_branch common_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "uart0_frac", "uart0_pre", 0, + RK2928_CLKSEL_CON(17), 0, + RK2928_CLKGATE_CON(1), 9, GFLAGS, +- &common_uart0_fracmux), ++ &common_uart0_fracmux, RK3188_UART_FRAC_MAX_PRATE), + COMPOSITE_NOMUX(0, "uart1_pre", "uart_src", 0, + RK2928_CLKSEL_CON(14), 0, 7, DFLAGS, + RK2928_CLKGATE_CON(1), 10, GFLAGS), + COMPOSITE_FRACMUX(0, "uart1_frac", "uart1_pre", 0, + RK2928_CLKSEL_CON(18), 0, + RK2928_CLKGATE_CON(1), 11, GFLAGS, +- &common_uart1_fracmux), ++ &common_uart1_fracmux, RK3188_UART_FRAC_MAX_PRATE), + COMPOSITE_NOMUX(0, "uart2_pre", "uart_src", 0, + RK2928_CLKSEL_CON(15), 0, 7, DFLAGS, + RK2928_CLKGATE_CON(1), 12, GFLAGS), + COMPOSITE_FRACMUX(0, "uart2_frac", "uart2_pre", 0, + RK2928_CLKSEL_CON(19), 0, + RK2928_CLKGATE_CON(1), 13, GFLAGS, +- &common_uart2_fracmux), ++ &common_uart2_fracmux, RK3188_UART_FRAC_MAX_PRATE), + COMPOSITE_NOMUX(0, "uart3_pre", "uart_src", 0, + RK2928_CLKSEL_CON(16), 0, 7, DFLAGS, + RK2928_CLKGATE_CON(1), 14, GFLAGS), + COMPOSITE_FRACMUX(0, "uart3_frac", "uart3_pre", 0, + RK2928_CLKSEL_CON(20), 0, + RK2928_CLKGATE_CON(1), 15, GFLAGS, +- &common_uart3_fracmux), ++ &common_uart3_fracmux, RK3188_UART_FRAC_MAX_PRATE), + + GATE(SCLK_JTAG, "jtag", "ext_jtag", 0, RK2928_CLKGATE_CON(1), 3, GFLAGS), + +@@ -449,11 +456,11 @@ static struct rockchip_clk_branch common_clk_branches[] __initdata = { + + /* hclk_cpu gates */ + GATE(HCLK_ROM, "hclk_rom", "hclk_cpu", 0, RK2928_CLKGATE_CON(5), 6, GFLAGS), +- GATE(HCLK_I2S0, "hclk_i2s0", "hclk_cpu", 0, RK2928_CLKGATE_CON(7), 2, GFLAGS), ++ GATE(HCLK_I2S0_2CH, "hclk_i2s0_2ch", "hclk_cpu", 0, RK2928_CLKGATE_CON(7), 2, GFLAGS), + GATE(HCLK_SPDIF, "hclk_spdif", "hclk_cpu", 0, RK2928_CLKGATE_CON(7), 1, GFLAGS), +- GATE(0, "hclk_cpubus", "hclk_cpu", 0, RK2928_CLKGATE_CON(4), 8, GFLAGS), ++ GATE(0, "hclk_cpubus", "hclk_cpu", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(4), 8, GFLAGS), + /* hclk_ahb2apb is part of a clk branch */ +- GATE(0, "hclk_vio_bus", "hclk_cpu", 0, RK2928_CLKGATE_CON(6), 12, GFLAGS), ++ GATE(0, "hclk_vio_bus", "hclk_cpu", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(6), 12, GFLAGS), + GATE(HCLK_LCDC0, "hclk_lcdc0", "hclk_cpu", 0, RK2928_CLKGATE_CON(6), 1, GFLAGS), + GATE(HCLK_LCDC1, "hclk_lcdc1", "hclk_cpu", 0, RK2928_CLKGATE_CON(6), 2, GFLAGS), + GATE(HCLK_CIF0, "hclk_cif0", "hclk_cpu", 0, RK2928_CLKGATE_CON(6), 4, GFLAGS), +@@ -571,7 +578,7 @@ static struct rockchip_clk_branch rk3066a_clk_branches[] __initdata = { + GATE(CORE_L2C, "core_l2c", "aclk_cpu", CLK_IGNORE_UNUSED, + RK2928_CLKGATE_CON(9), 4, GFLAGS), + +- COMPOSITE(0, "aclk_peri_pre", mux_pll_src_gpll_cpll_p, 0, ++ COMPOSITE(0, "aclk_peri_pre", mux_pll_src_gpll_cpll_p, CLK_IS_CRITICAL, + RK2928_CLKSEL_CON(10), 15, 1, MFLAGS, 0, 5, DFLAGS, + RK2928_CLKGATE_CON(2), 0, GFLAGS), + +@@ -618,24 +625,24 @@ static struct rockchip_clk_branch rk3066a_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "i2s0_frac", "i2s0_pre", 0, + RK2928_CLKSEL_CON(6), 0, + RK2928_CLKGATE_CON(0), 8, GFLAGS, +- &rk3066a_i2s0_fracmux), ++ &rk3066a_i2s0_fracmux, RK3188_I2S_FRAC_MAX_PRATE), + COMPOSITE_NOMUX(0, "i2s1_pre", "i2s_src", 0, + RK2928_CLKSEL_CON(3), 0, 7, DFLAGS, + RK2928_CLKGATE_CON(0), 9, GFLAGS), + COMPOSITE_FRACMUX(0, "i2s1_frac", "i2s1_pre", 0, + RK2928_CLKSEL_CON(7), 0, + RK2928_CLKGATE_CON(0), 10, GFLAGS, +- &rk3066a_i2s1_fracmux), ++ &rk3066a_i2s1_fracmux, RK3188_I2S_FRAC_MAX_PRATE), + COMPOSITE_NOMUX(0, "i2s2_pre", "i2s_src", 0, + RK2928_CLKSEL_CON(4), 0, 7, DFLAGS, + RK2928_CLKGATE_CON(0), 11, GFLAGS), + COMPOSITE_FRACMUX(0, "i2s2_frac", "i2s2_pre", 0, + RK2928_CLKSEL_CON(8), 0, + RK2928_CLKGATE_CON(0), 12, GFLAGS, +- &rk3066a_i2s2_fracmux), ++ &rk3066a_i2s2_fracmux, RK3188_I2S_FRAC_MAX_PRATE), + +- GATE(HCLK_I2S1, "hclk_i2s1", "hclk_cpu", 0, RK2928_CLKGATE_CON(7), 3, GFLAGS), +- GATE(HCLK_I2S2, "hclk_i2s2", "hclk_cpu", 0, RK2928_CLKGATE_CON(7), 4, GFLAGS), ++ GATE(HCLK_I2S1_2CH, "hclk_i2s1_2ch", "hclk_cpu", 0, RK2928_CLKGATE_CON(7), 3, GFLAGS), ++ GATE(HCLK_I2S_8CH, "hclk_i2s_8ch", "hclk_cpu", 0, RK2928_CLKGATE_CON(7), 4, GFLAGS), + GATE(HCLK_CIF1, "hclk_cif1", "hclk_cpu", 0, RK2928_CLKGATE_CON(6), 6, GFLAGS), + GATE(HCLK_HDMI, "hclk_hdmi", "hclk_cpu", 0, RK2928_CLKGATE_CON(4), 14, GFLAGS), + +@@ -676,7 +683,7 @@ static struct rockchip_clk_branch rk3188_clk_branches[] __initdata = { + div_rk3188_aclk_core_t, RK2928_CLKGATE_CON(0), 7, GFLAGS), + + /* do not source aclk_cpu_pre from the apll, to keep complexity down */ +- COMPOSITE_NOGATE(0, "aclk_cpu_pre", mux_aclk_cpu_p, CLK_SET_RATE_NO_REPARENT, ++ COMPOSITE_NOGATE(ACLK_CPU_PRE, "aclk_cpu_pre", mux_aclk_cpu_p, CLK_SET_RATE_NO_REPARENT, + RK2928_CLKSEL_CON(0), 5, 1, MFLAGS, 0, 5, DFLAGS), + DIV(0, "pclk_cpu_pre", "aclk_cpu_pre", 0, + RK2928_CLKSEL_CON(1), 12, 2, DFLAGS | CLK_DIVIDER_POWER_OF_TWO), +@@ -689,7 +696,7 @@ static struct rockchip_clk_branch rk3188_clk_branches[] __initdata = { + GATE(CORE_L2C, "core_l2c", "armclk", CLK_IGNORE_UNUSED, + RK2928_CLKGATE_CON(9), 4, GFLAGS), + +- COMPOSITE(0, "aclk_peri_pre", mux_pll_src_cpll_gpll_p, 0, ++ COMPOSITE(0, "aclk_peri_pre", mux_pll_src_cpll_gpll_p, CLK_IS_CRITICAL, + RK2928_CLKSEL_CON(10), 15, 1, MFLAGS, 0, 5, DFLAGS, + RK2928_CLKGATE_CON(2), 0, GFLAGS), + +@@ -726,7 +733,7 @@ static struct rockchip_clk_branch rk3188_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "i2s0_frac", "i2s0_pre", CLK_SET_RATE_PARENT, + RK2928_CLKSEL_CON(7), 0, + RK2928_CLKGATE_CON(0), 10, GFLAGS, +- &rk3188_i2s0_fracmux), ++ &rk3188_i2s0_fracmux, RK3188_I2S_FRAC_MAX_PRATE), + + GATE(0, "hclk_imem0", "hclk_cpu", 0, RK2928_CLKGATE_CON(4), 14, GFLAGS), + GATE(0, "hclk_imem1", "hclk_cpu", 0, RK2928_CLKGATE_CON(4), 15, GFLAGS), +@@ -743,17 +750,6 @@ static struct rockchip_clk_branch rk3188_clk_branches[] __initdata = { + GATE(ACLK_GPS, "aclk_gps", "aclk_peri", 0, RK2928_CLKGATE_CON(8), 13, GFLAGS), + }; + +-static const char *const rk3188_critical_clocks[] __initconst = { +- "aclk_cpu", +- "aclk_peri", +- "hclk_peri", +- "pclk_cpu", +- "pclk_peri", +- "hclk_cpubus", +- "hclk_vio_bus", +- "sclk_mac_lbtest", +-}; +- + static struct rockchip_clk_provider *__init rk3188_common_clk_init(struct device_node *np) + { + struct rockchip_clk_provider *ctx; +@@ -786,10 +782,12 @@ static struct rockchip_clk_provider *__init rk3188_common_clk_init(struct device + static void __init rk3066a_clk_init(struct device_node *np) + { + struct rockchip_clk_provider *ctx; ++ struct clk **clks; + + ctx = rk3188_common_clk_init(np); + if (IS_ERR(ctx)) + return; ++ clks = ctx->clk_data.clks; + + rockchip_clk_register_plls(ctx, rk3066_pll_clks, + ARRAY_SIZE(rk3066_pll_clks), +@@ -797,11 +795,9 @@ static void __init rk3066a_clk_init(struct device_node *np) + rockchip_clk_register_branches(ctx, rk3066a_clk_branches, + ARRAY_SIZE(rk3066a_clk_branches)); + rockchip_clk_register_armclk(ctx, ARMCLK, "armclk", +- mux_armclk_p, ARRAY_SIZE(mux_armclk_p), ++ 2, clks[PLL_APLL], clks[PLL_GPLL], + &rk3066_cpuclk_data, rk3066_cpuclk_rates, + ARRAY_SIZE(rk3066_cpuclk_rates)); +- rockchip_clk_protect_critical(rk3188_critical_clocks, +- ARRAY_SIZE(rk3188_critical_clocks)); + rockchip_clk_of_add_provider(np, ctx); + } + CLK_OF_DECLARE(rk3066a_cru, "rockchip,rk3066a-cru", rk3066a_clk_init); +@@ -809,13 +805,14 @@ CLK_OF_DECLARE(rk3066a_cru, "rockchip,rk3066a-cru", rk3066a_clk_init); + static void __init rk3188a_clk_init(struct device_node *np) + { + struct rockchip_clk_provider *ctx; +- struct clk *clk1, *clk2; ++ struct clk **clks; + unsigned long rate; + int ret; + + ctx = rk3188_common_clk_init(np); + if (IS_ERR(ctx)) + return; ++ clks = ctx->clk_data.clks; + + rockchip_clk_register_plls(ctx, rk3188_pll_clks, + ARRAY_SIZE(rk3188_pll_clks), +@@ -823,29 +820,25 @@ static void __init rk3188a_clk_init(struct device_node *np) + rockchip_clk_register_branches(ctx, rk3188_clk_branches, + ARRAY_SIZE(rk3188_clk_branches)); + rockchip_clk_register_armclk(ctx, ARMCLK, "armclk", +- mux_armclk_p, ARRAY_SIZE(mux_armclk_p), ++ 2, clks[PLL_APLL], clks[PLL_GPLL], + &rk3188_cpuclk_data, rk3188_cpuclk_rates, + ARRAY_SIZE(rk3188_cpuclk_rates)); + + /* reparent aclk_cpu_pre from apll */ +- clk1 = __clk_lookup("aclk_cpu_pre"); +- clk2 = __clk_lookup("gpll"); +- if (clk1 && clk2) { +- rate = clk_get_rate(clk1); ++ if (clks[ACLK_CPU_PRE] && clks[PLL_GPLL]) { ++ rate = clk_get_rate(clks[ACLK_CPU_PRE]); + +- ret = clk_set_parent(clk1, clk2); ++ ret = clk_set_parent(clks[ACLK_CPU_PRE], clks[PLL_GPLL]); + if (ret < 0) + pr_warn("%s: could not reparent aclk_cpu_pre to gpll\n", + __func__); + +- clk_set_rate(clk1, rate); ++ clk_set_rate(clks[ACLK_CPU_PRE], rate); + } else { + pr_warn("%s: missing clocks to reparent aclk_cpu_pre to gpll\n", + __func__); + } + +- rockchip_clk_protect_critical(rk3188_critical_clocks, +- ARRAY_SIZE(rk3188_critical_clocks)); + rockchip_clk_of_add_provider(np, ctx); + } + CLK_OF_DECLARE(rk3188a_cru, "rockchip,rk3188a-cru", rk3188a_clk_init); +@@ -871,3 +864,62 @@ static void __init rk3188_clk_init(struct device_node *np) + rk3188a_clk_init(np); + } + CLK_OF_DECLARE(rk3188_cru, "rockchip,rk3188-cru", rk3188_clk_init); ++ ++struct clk_rk3188_inits { ++ void (*inits)(struct device_node *np); ++}; ++ ++static const struct clk_rk3188_inits clk_rk3066a_init = { ++ .inits = rk3066a_clk_init, ++}; ++ ++static const struct clk_rk3188_inits clk_rk3188a_init = { ++ .inits = rk3188a_clk_init, ++}; ++ ++static const struct clk_rk3188_inits clk_rk3188_init = { ++ .inits = rk3188_clk_init, ++}; ++ ++static const struct of_device_id clk_rk3188_match_table[] = { ++ { ++ .compatible = "rockchip,rk3066a-cru", ++ .data = &clk_rk3066a_init, ++ }, { ++ .compatible = "rockchip,rk3188a-cru", ++ .data = &clk_rk3188a_init, ++ }, { ++ .compatible = "rockchip,rk3188-cru", ++ .data = &rk3188_clk_init, ++ }, ++ { } ++}; ++MODULE_DEVICE_TABLE(of, clk_rk3188_match_table); ++ ++static int __init clk_rk3188_probe(struct platform_device *pdev) ++{ ++ struct device_node *np = pdev->dev.of_node; ++ const struct of_device_id *match; ++ const struct clk_rk3188_inits *init_data; ++ ++ match = of_match_device(clk_rk3188_match_table, &pdev->dev); ++ if (!match || !match->data) ++ return -EINVAL; ++ ++ init_data = match->data; ++ if (init_data->inits) ++ init_data->inits(np); ++ ++ return 0; ++} ++ ++static struct platform_driver clk_rk3188_driver = { ++ .driver = { ++ .name = "clk-rk3188", ++ .of_match_table = clk_rk3188_match_table, ++ }, ++}; ++builtin_platform_driver_probe(clk_rk3188_driver, clk_rk3188_probe); ++ ++MODULE_DESCRIPTION("Rockchip RK3188 Clock Driver"); ++MODULE_LICENSE("GPL"); +diff --git a/drivers/clk/rockchip/clk-rk3228.c b/drivers/clk/rockchip/clk-rk3228.c +index 47d6482dda9d..01ff90c8a59f 100644 +--- a/drivers/clk/rockchip/clk-rk3228.c ++++ b/drivers/clk/rockchip/clk-rk3228.c +@@ -7,14 +7,20 @@ + + #include + #include ++#include + #include + #include ++#include + #include + #include + #include "clk.h" + + #define RK3228_GRF_SOC_STATUS0 0x480 + ++#define RK3228_UART_FRAC_MAX_PRATE 600000000 ++#define RK3228_SPDIF_FRAC_MAX_PRATE 600000000 ++#define RK3228_I2S_FRAC_MAX_PRATE 600000000 ++ + enum rk3228_plls { + apll, dpll, cpll, gpll, + }; +@@ -78,22 +84,22 @@ static struct rockchip_pll_rate_table rk3228_pll_rates[] = { + #define RK3228_DIV_PCLK_MASK 0x7 + #define RK3228_DIV_PCLK_SHIFT 12 + +-#define RK3228_CLKSEL1(_core_aclk_div, _core_peri_div) \ +- { \ +- .reg = RK2928_CLKSEL_CON(1), \ +- .val = HIWORD_UPDATE(_core_peri_div, RK3228_DIV_PERI_MASK, \ +- RK3228_DIV_PERI_SHIFT) | \ +- HIWORD_UPDATE(_core_aclk_div, RK3228_DIV_ACLK_MASK, \ +- RK3228_DIV_ACLK_SHIFT), \ ++#define RK3228_CLKSEL1(_core_aclk_div, _core_peri_div) \ ++{ \ ++ .reg = RK2928_CLKSEL_CON(1), \ ++ .val = HIWORD_UPDATE(_core_peri_div, RK3228_DIV_PERI_MASK, \ ++ RK3228_DIV_PERI_SHIFT) | \ ++ HIWORD_UPDATE(_core_aclk_div, RK3228_DIV_ACLK_MASK, \ ++ RK3228_DIV_ACLK_SHIFT), \ + } + +-#define RK3228_CPUCLK_RATE(_prate, _core_aclk_div, _core_peri_div) \ +- { \ +- .prate = _prate, \ +- .divs = { \ +- RK3228_CLKSEL1(_core_aclk_div, _core_peri_div), \ +- }, \ +- } ++#define RK3228_CPUCLK_RATE(_prate, _core_aclk_div, _core_peri_div) \ ++{ \ ++ .prate = _prate, \ ++ .divs = { \ ++ RK3228_CLKSEL1(_core_aclk_div, _core_peri_div), \ ++ }, \ ++} + + static struct rockchip_cpuclk_rate_table rk3228_cpuclk_rates[] __initdata = { + RK3228_CPUCLK_RATE(1800000000, 1, 7), +@@ -119,9 +125,10 @@ static struct rockchip_cpuclk_rate_table rk3228_cpuclk_rates[] __initdata = { + }; + + static const struct rockchip_cpuclk_reg_data rk3228_cpuclk_data = { +- .core_reg = RK2928_CLKSEL_CON(0), +- .div_core_shift = 0, +- .div_core_mask = 0x1f, ++ .core_reg[0] = RK2928_CLKSEL_CON(0), ++ .div_core_shift[0] = 0, ++ .div_core_mask[0] = 0x1f, ++ .num_cores = 1, + .mux_core_alt = 1, + .mux_core_main = 0, + .mux_core_shift = 6, +@@ -131,7 +138,6 @@ static const struct rockchip_cpuclk_reg_data rk3228_cpuclk_data = { + PNAME(mux_pll_p) = { "clk_24m", "xin24m" }; + + PNAME(mux_ddrphy_p) = { "dpll_ddr", "gpll_ddr", "apll_ddr" }; +-PNAME(mux_armclk_p) = { "apll_core", "gpll_core", "dpll_core" }; + PNAME(mux_usb480m_phy_p) = { "usb480m_phy0", "usb480m_phy1" }; + PNAME(mux_usb480m_p) = { "usb480m_phy", "xin24m" }; + PNAME(mux_hdmiphy_p) = { "hdmiphy_phy", "xin24m" }; +@@ -231,12 +237,12 @@ static struct rockchip_clk_branch rk3228_clk_branches[] __initdata = { + RK2928_CLKGATE_CON(7), 0, GFLAGS), + + /* PD_CORE */ +- GATE(0, "dpll_core", "dpll", CLK_IGNORE_UNUSED, +- RK2928_CLKGATE_CON(0), 6, GFLAGS), + GATE(0, "apll_core", "apll", CLK_IGNORE_UNUSED, + RK2928_CLKGATE_CON(0), 6, GFLAGS), + GATE(0, "gpll_core", "gpll", CLK_IGNORE_UNUSED, + RK2928_CLKGATE_CON(0), 6, GFLAGS), ++ GATE(0, "dpll_core", "dpll", CLK_IGNORE_UNUSED, ++ RK2928_CLKGATE_CON(0), 6, GFLAGS), + COMPOSITE_NOMUX(0, "pclk_dbg", "armclk", CLK_IGNORE_UNUSED, + RK2928_CLKSEL_CON(1), 0, 4, DFLAGS | CLK_DIVIDER_READ_ONLY, + RK2928_CLKGATE_CON(4), 1, GFLAGS), +@@ -253,27 +259,27 @@ static struct rockchip_clk_branch rk3228_clk_branches[] __initdata = { + RK2928_MISC_CON, 15, 1, MFLAGS), + + /* PD_BUS */ +- GATE(0, "hdmiphy_aclk_cpu", "hdmiphy", CLK_IGNORE_UNUSED, ++ GATE(0, "hdmiphy_aclk_cpu", "hdmiphy", CLK_IS_CRITICAL, + RK2928_CLKGATE_CON(0), 1, GFLAGS), +- GATE(0, "gpll_aclk_cpu", "gpll", CLK_IGNORE_UNUSED, ++ GATE(0, "gpll_aclk_cpu", "gpll", CLK_IS_CRITICAL, + RK2928_CLKGATE_CON(0), 1, GFLAGS), +- GATE(0, "cpll_aclk_cpu", "cpll", CLK_IGNORE_UNUSED, ++ GATE(0, "cpll_aclk_cpu", "cpll", CLK_IS_CRITICAL, + RK2928_CLKGATE_CON(0), 1, GFLAGS), +- COMPOSITE_NOGATE(0, "aclk_cpu_src", mux_aclk_cpu_src_p, 0, ++ COMPOSITE_NOGATE(0, "aclk_cpu_src", mux_aclk_cpu_src_p, CLK_IS_CRITICAL, + RK2928_CLKSEL_CON(0), 13, 2, MFLAGS, 8, 5, DFLAGS), +- GATE(ACLK_CPU, "aclk_cpu", "aclk_cpu_src", 0, ++ GATE(ACLK_CPU, "aclk_cpu", "aclk_cpu_src", CLK_IS_CRITICAL, + RK2928_CLKGATE_CON(6), 0, GFLAGS), +- COMPOSITE_NOMUX(HCLK_CPU, "hclk_cpu", "aclk_cpu_src", 0, ++ COMPOSITE_NOMUX(HCLK_CPU, "hclk_cpu", "aclk_cpu_src", CLK_IS_CRITICAL, + RK2928_CLKSEL_CON(1), 8, 2, DFLAGS, + RK2928_CLKGATE_CON(6), 1, GFLAGS), +- COMPOSITE_NOMUX(0, "pclk_bus_src", "aclk_cpu_src", 0, ++ COMPOSITE_NOMUX(0, "pclk_bus_src", "aclk_cpu_src", CLK_IS_CRITICAL, + RK2928_CLKSEL_CON(1), 12, 3, DFLAGS, + RK2928_CLKGATE_CON(6), 2, GFLAGS), +- GATE(PCLK_CPU, "pclk_cpu", "pclk_bus_src", 0, ++ GATE(PCLK_CPU, "pclk_cpu", "pclk_bus_src", CLK_IS_CRITICAL, + RK2928_CLKGATE_CON(6), 3, GFLAGS), +- GATE(0, "pclk_phy_pre", "pclk_bus_src", 0, ++ GATE(0, "pclk_phy_pre", "pclk_bus_src", CLK_IS_CRITICAL, + RK2928_CLKGATE_CON(6), 4, GFLAGS), +- GATE(0, "pclk_ddr_pre", "pclk_bus_src", 0, ++ GATE(0, "pclk_ddr_pre", "pclk_bus_src", CLK_IS_CRITICAL, + RK2928_CLKGATE_CON(6), 13, GFLAGS), + + /* PD_VIDEO */ +@@ -308,9 +314,9 @@ static struct rockchip_clk_branch rk3228_clk_branches[] __initdata = { + RK2928_CLKSEL_CON(31), 13, 2, MFLAGS, 8, 5, DFLAGS, + RK2928_CLKGATE_CON(1), 4, GFLAGS), + +- MUX(0, "sclk_rga_src", mux_pll_src_4plls_p, 0, ++ MUX(0, "sclk_rga_src", mux_pll_src_4plls_p, CLK_IS_CRITICAL, + RK2928_CLKSEL_CON(33), 13, 2, MFLAGS), +- COMPOSITE_NOMUX(ACLK_RGA_PRE, "aclk_rga_pre", "sclk_rga_src", 0, ++ COMPOSITE_NOMUX(ACLK_RGA_PRE, "aclk_rga_pre", "sclk_rga_src", CLK_IS_CRITICAL, + RK2928_CLKSEL_CON(33), 8, 5, DFLAGS, + RK2928_CLKGATE_CON(1), 2, GFLAGS), + COMPOSITE(SCLK_RGA, "sclk_rga", mux_sclk_rga_p, 0, +@@ -333,21 +339,21 @@ static struct rockchip_clk_branch rk3228_clk_branches[] __initdata = { + RK2928_CLKGATE_CON(3), 8, GFLAGS), + + /* PD_PERI */ +- GATE(0, "cpll_peri", "cpll", CLK_IGNORE_UNUSED, ++ GATE(0, "cpll_peri", "cpll", CLK_IS_CRITICAL, + RK2928_CLKGATE_CON(2), 0, GFLAGS), +- GATE(0, "gpll_peri", "gpll", CLK_IGNORE_UNUSED, ++ GATE(0, "gpll_peri", "gpll", CLK_IS_CRITICAL, + RK2928_CLKGATE_CON(2), 0, GFLAGS), +- GATE(0, "hdmiphy_peri", "hdmiphy", CLK_IGNORE_UNUSED, ++ GATE(0, "hdmiphy_peri", "hdmiphy", CLK_IS_CRITICAL, + RK2928_CLKGATE_CON(2), 0, GFLAGS), +- COMPOSITE_NOGATE(0, "aclk_peri_src", mux_aclk_peri_src_p, 0, ++ COMPOSITE_NOGATE(0, "aclk_peri_src", mux_aclk_peri_src_p, CLK_IS_CRITICAL, + RK2928_CLKSEL_CON(10), 10, 2, MFLAGS, 0, 5, DFLAGS), +- COMPOSITE_NOMUX(PCLK_PERI, "pclk_peri", "aclk_peri_src", 0, ++ COMPOSITE_NOMUX(PCLK_PERI, "pclk_peri", "aclk_peri_src", CLK_IS_CRITICAL, + RK2928_CLKSEL_CON(10), 12, 3, DFLAGS, + RK2928_CLKGATE_CON(5), 2, GFLAGS), +- COMPOSITE_NOMUX(HCLK_PERI, "hclk_peri", "aclk_peri_src", 0, ++ COMPOSITE_NOMUX(HCLK_PERI, "hclk_peri", "aclk_peri_src", CLK_IS_CRITICAL, + RK2928_CLKSEL_CON(10), 8, 2, DFLAGS, + RK2928_CLKGATE_CON(5), 1, GFLAGS), +- GATE(ACLK_PERI, "aclk_peri", "aclk_peri_src", 0, ++ GATE(ACLK_PERI, "aclk_peri", "aclk_peri_src", CLK_IS_CRITICAL, + RK2928_CLKGATE_CON(5), 0, GFLAGS), + + GATE(SCLK_TIMER0, "sclk_timer0", "xin24m", 0, +@@ -419,7 +425,7 @@ static struct rockchip_clk_branch rk3228_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "i2s0_frac", "i2s0_src", CLK_SET_RATE_PARENT, + RK2928_CLKSEL_CON(8), 0, + RK2928_CLKGATE_CON(0), 4, GFLAGS, +- &rk3228_i2s0_fracmux), ++ &rk3228_i2s0_fracmux, RK3228_I2S_FRAC_MAX_PRATE), + GATE(SCLK_I2S0, "sclk_i2s0", "i2s0_pre", CLK_SET_RATE_PARENT, + RK2928_CLKGATE_CON(0), 5, GFLAGS), + +@@ -429,7 +435,7 @@ static struct rockchip_clk_branch rk3228_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "i2s1_frac", "i2s1_src", CLK_SET_RATE_PARENT, + RK2928_CLKSEL_CON(7), 0, + RK2928_CLKGATE_CON(0), 11, GFLAGS, +- &rk3228_i2s1_fracmux), ++ &rk3228_i2s1_fracmux, RK3228_I2S_FRAC_MAX_PRATE), + GATE(SCLK_I2S1, "sclk_i2s1", "i2s1_pre", CLK_SET_RATE_PARENT, + RK2928_CLKGATE_CON(0), 14, GFLAGS), + COMPOSITE_NODIV(SCLK_I2S_OUT, "i2s_out", mux_i2s_out_p, 0, +@@ -442,7 +448,7 @@ static struct rockchip_clk_branch rk3228_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "i2s2_frac", "i2s2_src", CLK_SET_RATE_PARENT, + RK2928_CLKSEL_CON(30), 0, + RK2928_CLKGATE_CON(0), 8, GFLAGS, +- &rk3228_i2s2_fracmux), ++ &rk3228_i2s2_fracmux, RK3228_I2S_FRAC_MAX_PRATE), + GATE(SCLK_I2S2, "sclk_i2s2", "i2s2_pre", CLK_SET_RATE_PARENT, + RK2928_CLKGATE_CON(0), 9, GFLAGS), + +@@ -452,7 +458,7 @@ static struct rockchip_clk_branch rk3228_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "spdif_frac", "sclk_spdif_src", CLK_SET_RATE_PARENT, + RK2928_CLKSEL_CON(20), 0, + RK2928_CLKGATE_CON(2), 12, GFLAGS, +- &rk3228_spdif_fracmux), ++ &rk3228_spdif_fracmux, RK3228_SPDIF_FRAC_MAX_PRATE), + + GATE(0, "jtag", "ext_jtag", CLK_IGNORE_UNUSED, + RK2928_CLKGATE_CON(1), 3, GFLAGS), +@@ -487,15 +493,15 @@ static struct rockchip_clk_branch rk3228_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "uart0_frac", "uart0_src", CLK_SET_RATE_PARENT, + RK2928_CLKSEL_CON(17), 0, + RK2928_CLKGATE_CON(1), 9, GFLAGS, +- &rk3228_uart0_fracmux), ++ &rk3228_uart0_fracmux, RK3228_UART_FRAC_MAX_PRATE), + COMPOSITE_FRACMUX(0, "uart1_frac", "uart1_src", CLK_SET_RATE_PARENT, + RK2928_CLKSEL_CON(18), 0, + RK2928_CLKGATE_CON(1), 11, GFLAGS, +- &rk3228_uart1_fracmux), ++ &rk3228_uart1_fracmux, RK3228_UART_FRAC_MAX_PRATE), + COMPOSITE_FRACMUX(0, "uart2_frac", "uart2_src", CLK_SET_RATE_PARENT, + RK2928_CLKSEL_CON(19), 0, + RK2928_CLKGATE_CON(1), 13, GFLAGS, +- &rk3228_uart2_fracmux), ++ &rk3228_uart2_fracmux, RK3228_UART_FRAC_MAX_PRATE), + + COMPOSITE(SCLK_NANDC, "sclk_nandc", mux_pll_src_2plls_p, 0, + RK2928_CLKSEL_CON(2), 14, 1, MFLAGS, 8, 5, DFLAGS, +@@ -529,22 +535,22 @@ static struct rockchip_clk_branch rk3228_clk_branches[] __initdata = { + + /* PD_VOP */ + GATE(ACLK_RGA, "aclk_rga", "aclk_rga_pre", 0, RK2928_CLKGATE_CON(13), 0, GFLAGS), +- GATE(0, "aclk_rga_noc", "aclk_rga_pre", 0, RK2928_CLKGATE_CON(13), 11, GFLAGS), ++ GATE(0, "aclk_rga_noc", "aclk_rga_pre", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(13), 11, GFLAGS), + GATE(ACLK_IEP, "aclk_iep", "aclk_iep_pre", 0, RK2928_CLKGATE_CON(13), 2, GFLAGS), +- GATE(0, "aclk_iep_noc", "aclk_iep_pre", 0, RK2928_CLKGATE_CON(13), 9, GFLAGS), ++ GATE(0, "aclk_iep_noc", "aclk_iep_pre", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(13), 9, GFLAGS), + + GATE(ACLK_VOP, "aclk_vop", "aclk_vop_pre", 0, RK2928_CLKGATE_CON(13), 5, GFLAGS), +- GATE(0, "aclk_vop_noc", "aclk_vop_pre", 0, RK2928_CLKGATE_CON(13), 12, GFLAGS), ++ GATE(0, "aclk_vop_noc", "aclk_vop_pre", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(13), 12, GFLAGS), + + GATE(ACLK_HDCP, "aclk_hdcp", "aclk_hdcp_pre", 0, RK2928_CLKGATE_CON(14), 10, GFLAGS), +- GATE(0, "aclk_hdcp_noc", "aclk_hdcp_pre", 0, RK2928_CLKGATE_CON(13), 10, GFLAGS), ++ GATE(0, "aclk_hdcp_noc", "aclk_hdcp_pre", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(13), 10, GFLAGS), + + GATE(HCLK_RGA, "hclk_rga", "hclk_vio_pre", 0, RK2928_CLKGATE_CON(13), 1, GFLAGS), + GATE(HCLK_IEP, "hclk_iep", "hclk_vio_pre", 0, RK2928_CLKGATE_CON(13), 3, GFLAGS), + GATE(HCLK_VOP, "hclk_vop", "hclk_vio_pre", 0, RK2928_CLKGATE_CON(13), 6, GFLAGS), +- GATE(0, "hclk_vio_ahb_arbi", "hclk_vio_pre", 0, RK2928_CLKGATE_CON(13), 7, GFLAGS), +- GATE(0, "hclk_vio_noc", "hclk_vio_pre", 0, RK2928_CLKGATE_CON(13), 8, GFLAGS), +- GATE(0, "hclk_vop_noc", "hclk_vio_pre", 0, RK2928_CLKGATE_CON(13), 13, GFLAGS), ++ GATE(0, "hclk_vio_ahb_arbi", "hclk_vio_pre", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(13), 7, GFLAGS), ++ GATE(0, "hclk_vio_noc", "hclk_vio_pre", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(13), 8, GFLAGS), ++ GATE(0, "hclk_vop_noc", "hclk_vio_pre", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(13), 13, GFLAGS), + GATE(HCLK_VIO_H2P, "hclk_vio_h2p", "hclk_vio_pre", 0, RK2928_CLKGATE_CON(14), 7, GFLAGS), + GATE(HCLK_HDCP_MMU, "hclk_hdcp_mmu", "hclk_vio_pre", 0, RK2928_CLKGATE_CON(14), 12, GFLAGS), + GATE(PCLK_HDMI_CTRL, "pclk_hdmi_ctrl", "hclk_vio_pre", 0, RK2928_CLKGATE_CON(14), 6, GFLAGS), +@@ -560,29 +566,29 @@ static struct rockchip_clk_branch rk3228_clk_branches[] __initdata = { + GATE(HCLK_EMMC, "hclk_emmc", "hclk_peri", 0, RK2928_CLKGATE_CON(11), 2, GFLAGS), + GATE(HCLK_NANDC, "hclk_nandc", "hclk_peri", 0, RK2928_CLKGATE_CON(11), 3, GFLAGS), + GATE(HCLK_HOST0, "hclk_host0", "hclk_peri", 0, RK2928_CLKGATE_CON(11), 6, GFLAGS), +- GATE(0, "hclk_host0_arb", "hclk_peri", 0, RK2928_CLKGATE_CON(11), 7, GFLAGS), ++ GATE(0, "hclk_host0_arb", "hclk_peri", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(11), 7, GFLAGS), + GATE(HCLK_HOST1, "hclk_host1", "hclk_peri", 0, RK2928_CLKGATE_CON(11), 8, GFLAGS), +- GATE(0, "hclk_host1_arb", "hclk_peri", 0, RK2928_CLKGATE_CON(11), 9, GFLAGS), ++ GATE(0, "hclk_host1_arb", "hclk_peri", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(11), 9, GFLAGS), + GATE(HCLK_HOST2, "hclk_host2", "hclk_peri", 0, RK2928_CLKGATE_CON(11), 10, GFLAGS), + GATE(HCLK_OTG, "hclk_otg", "hclk_peri", 0, RK2928_CLKGATE_CON(11), 12, GFLAGS), +- GATE(0, "hclk_otg_pmu", "hclk_peri", 0, RK2928_CLKGATE_CON(11), 13, GFLAGS), +- GATE(0, "hclk_host2_arb", "hclk_peri", 0, RK2928_CLKGATE_CON(11), 14, GFLAGS), ++ GATE(0, "hclk_otg_pmu", "hclk_peri", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(11), 13, GFLAGS), ++ GATE(0, "hclk_host2_arb", "hclk_peri", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(11), 14, GFLAGS), + GATE(0, "hclk_peri_noc", "hclk_peri", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(12), 1, GFLAGS), + + GATE(PCLK_GMAC, "pclk_gmac", "pclk_peri", 0, RK2928_CLKGATE_CON(11), 5, GFLAGS), +- GATE(0, "pclk_peri_noc", "pclk_peri", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(12), 2, GFLAGS), ++ GATE(0, "pclk_peri_noc", "pclk_peri", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(12), 2, GFLAGS), + + /* PD_GPU */ + GATE(ACLK_GPU, "aclk_gpu", "aclk_gpu_pre", 0, RK2928_CLKGATE_CON(7), 14, GFLAGS), +- GATE(0, "aclk_gpu_noc", "aclk_gpu_pre", 0, RK2928_CLKGATE_CON(7), 15, GFLAGS), ++ GATE(0, "aclk_gpu_noc", "aclk_gpu_pre", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(7), 15, GFLAGS), + + /* PD_BUS */ +- GATE(0, "sclk_initmem_mbist", "aclk_cpu", 0, RK2928_CLKGATE_CON(8), 1, GFLAGS), +- GATE(0, "aclk_initmem", "aclk_cpu", 0, RK2928_CLKGATE_CON(8), 0, GFLAGS), ++ GATE(0, "sclk_initmem_mbist", "aclk_cpu", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(8), 1, GFLAGS), ++ GATE(0, "aclk_initmem", "aclk_cpu", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(8), 0, GFLAGS), + GATE(ACLK_DMAC, "aclk_dmac_bus", "aclk_cpu", 0, RK2928_CLKGATE_CON(8), 2, GFLAGS), + GATE(0, "aclk_bus_noc", "aclk_cpu", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(10), 1, GFLAGS), + +- GATE(0, "hclk_rom", "hclk_cpu", 0, RK2928_CLKGATE_CON(8), 3, GFLAGS), ++ GATE(0, "hclk_rom", "hclk_cpu", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(8), 3, GFLAGS), + GATE(HCLK_I2S0_8CH, "hclk_i2s0_8ch", "hclk_cpu", 0, RK2928_CLKGATE_CON(8), 7, GFLAGS), + GATE(HCLK_I2S1_8CH, "hclk_i2s1_8ch", "hclk_cpu", 0, RK2928_CLKGATE_CON(8), 8, GFLAGS), + GATE(HCLK_I2S2_2CH, "hclk_i2s2_2ch", "hclk_cpu", 0, RK2928_CLKGATE_CON(8), 9, GFLAGS), +@@ -591,9 +597,9 @@ static struct rockchip_clk_branch rk3228_clk_branches[] __initdata = { + GATE(HCLK_M_CRYPTO, "hclk_crypto_mst", "hclk_cpu", 0, RK2928_CLKGATE_CON(8), 11, GFLAGS), + GATE(HCLK_S_CRYPTO, "hclk_crypto_slv", "hclk_cpu", 0, RK2928_CLKGATE_CON(8), 12, GFLAGS), + +- GATE(0, "pclk_ddrupctl", "pclk_ddr_pre", 0, RK2928_CLKGATE_CON(8), 4, GFLAGS), +- GATE(0, "pclk_ddrmon", "pclk_ddr_pre", 0, RK2928_CLKGATE_CON(8), 6, GFLAGS), +- GATE(0, "pclk_msch_noc", "pclk_ddr_pre", 0, RK2928_CLKGATE_CON(10), 2, GFLAGS), ++ GATE(0, "pclk_ddrupctl", "pclk_ddr_pre", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(8), 4, GFLAGS), ++ GATE(0, "pclk_ddrmon", "pclk_ddr_pre", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(8), 6, GFLAGS), ++ GATE(0, "pclk_msch_noc", "pclk_ddr_pre", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(10), 2, GFLAGS), + + GATE(PCLK_EFUSE_1024, "pclk_efuse_1024", "pclk_cpu", 0, RK2928_CLKGATE_CON(8), 13, GFLAGS), + GATE(PCLK_EFUSE_256, "pclk_efuse_256", "pclk_cpu", 0, RK2928_CLKGATE_CON(8), 14, GFLAGS), +@@ -602,7 +608,7 @@ static struct rockchip_clk_branch rk3228_clk_branches[] __initdata = { + GATE(PCLK_I2C2, "pclk_i2c2", "pclk_cpu", 0, RK2928_CLKGATE_CON(9), 1, GFLAGS), + GATE(PCLK_I2C3, "pclk_i2c3", "pclk_cpu", 0, RK2928_CLKGATE_CON(9), 2, GFLAGS), + GATE(PCLK_TIMER, "pclk_timer0", "pclk_cpu", 0, RK2928_CLKGATE_CON(9), 4, GFLAGS), +- GATE(0, "pclk_stimer", "pclk_cpu", 0, RK2928_CLKGATE_CON(9), 5, GFLAGS), ++ GATE(0, "pclk_stimer", "pclk_cpu", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(9), 5, GFLAGS), + GATE(PCLK_SPI0, "pclk_spi0", "pclk_cpu", 0, RK2928_CLKGATE_CON(9), 6, GFLAGS), + GATE(PCLK_PWM, "pclk_rk_pwm", "pclk_cpu", 0, RK2928_CLKGATE_CON(9), 7, GFLAGS), + GATE(PCLK_GPIO0, "pclk_gpio0", "pclk_cpu", 0, RK2928_CLKGATE_CON(9), 8, GFLAGS), +@@ -616,73 +622,51 @@ static struct rockchip_clk_branch rk3228_clk_branches[] __initdata = { + GATE(PCLK_GRF, "pclk_grf", "pclk_cpu", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(10), 0, GFLAGS), + GATE(0, "pclk_cru", "pclk_cpu", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(10), 1, GFLAGS), + GATE(0, "pclk_sgrf", "pclk_cpu", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(10), 2, GFLAGS), +- GATE(0, "pclk_sim", "pclk_cpu", 0, RK2928_CLKGATE_CON(10), 3, GFLAGS), ++ GATE(0, "pclk_sim", "pclk_cpu", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(10), 3, GFLAGS), + +- GATE(0, "pclk_ddrphy", "pclk_phy_pre", 0, RK2928_CLKGATE_CON(10), 3, GFLAGS), +- GATE(0, "pclk_acodecphy", "pclk_phy_pre", 0, RK2928_CLKGATE_CON(10), 5, GFLAGS), ++ GATE(0, "pclk_ddrphy", "pclk_phy_pre", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(10), 3, GFLAGS), ++ GATE(0, "pclk_acodecphy", "pclk_phy_pre", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(10), 5, GFLAGS), + GATE(PCLK_HDMI_PHY, "pclk_hdmiphy", "pclk_phy_pre", 0, RK2928_CLKGATE_CON(10), 7, GFLAGS), +- GATE(0, "pclk_vdacphy", "pclk_phy_pre", 0, RK2928_CLKGATE_CON(10), 8, GFLAGS), +- GATE(0, "pclk_phy_noc", "pclk_phy_pre", 0, RK2928_CLKGATE_CON(10), 9, GFLAGS), ++ GATE(0, "pclk_vdacphy", "pclk_phy_pre", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(10), 8, GFLAGS), ++ GATE(0, "pclk_phy_noc", "pclk_phy_pre", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(10), 9, GFLAGS), + + GATE(ACLK_VPU, "aclk_vpu", "aclk_vpu_pre", 0, RK2928_CLKGATE_CON(15), 0, GFLAGS), +- GATE(0, "aclk_vpu_noc", "aclk_vpu_pre", 0, RK2928_CLKGATE_CON(15), 4, GFLAGS), ++ GATE(0, "aclk_vpu_noc", "aclk_vpu_pre", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(15), 4, GFLAGS), + GATE(ACLK_RKVDEC, "aclk_rkvdec", "aclk_rkvdec_pre", 0, RK2928_CLKGATE_CON(15), 2, GFLAGS), +- GATE(0, "aclk_rkvdec_noc", "aclk_rkvdec_pre", 0, RK2928_CLKGATE_CON(15), 6, GFLAGS), ++ GATE(0, "aclk_rkvdec_noc", "aclk_rkvdec_pre", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(15), 6, GFLAGS), + GATE(HCLK_VPU, "hclk_vpu", "hclk_vpu_pre", 0, RK2928_CLKGATE_CON(15), 1, GFLAGS), +- GATE(0, "hclk_vpu_noc", "hclk_vpu_pre", 0, RK2928_CLKGATE_CON(15), 5, GFLAGS), ++ GATE(0, "hclk_vpu_noc", "hclk_vpu_pre", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(15), 5, GFLAGS), + GATE(HCLK_RKVDEC, "hclk_rkvdec", "hclk_rkvdec_pre", 0, RK2928_CLKGATE_CON(15), 3, GFLAGS), +- GATE(0, "hclk_rkvdec_noc", "hclk_rkvdec_pre", 0, RK2928_CLKGATE_CON(15), 7, GFLAGS), ++ GATE(0, "hclk_rkvdec_noc", "hclk_rkvdec_pre", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(15), 7, GFLAGS), + + /* PD_MMC */ + MMC(SCLK_SDMMC_DRV, "sdmmc_drv", "sclk_sdmmc", RK3228_SDMMC_CON0, 1), +- MMC(SCLK_SDMMC_SAMPLE, "sdmmc_sample", "sclk_sdmmc", RK3228_SDMMC_CON1, 0), ++ MMC(SCLK_SDMMC_SAMPLE, "sdmmc_sample", "sclk_sdmmc", RK3228_SDMMC_CON1, 1), + + MMC(SCLK_SDIO_DRV, "sdio_drv", "sclk_sdio", RK3228_SDIO_CON0, 1), +- MMC(SCLK_SDIO_SAMPLE, "sdio_sample", "sclk_sdio", RK3228_SDIO_CON1, 0), ++ MMC(SCLK_SDIO_SAMPLE, "sdio_sample", "sclk_sdio", RK3228_SDIO_CON1, 1), + + MMC(SCLK_EMMC_DRV, "emmc_drv", "sclk_emmc", RK3228_EMMC_CON0, 1), +- MMC(SCLK_EMMC_SAMPLE, "emmc_sample", "sclk_emmc", RK3228_EMMC_CON1, 0), ++ MMC(SCLK_EMMC_SAMPLE, "emmc_sample", "sclk_emmc", RK3228_EMMC_CON1, 1), + }; + +-static const char *const rk3228_critical_clocks[] __initconst = { +- "aclk_cpu", +- "pclk_cpu", +- "hclk_cpu", +- "aclk_peri", +- "hclk_peri", +- "pclk_peri", +- "aclk_rga_noc", +- "aclk_iep_noc", +- "aclk_vop_noc", +- "aclk_hdcp_noc", +- "hclk_vio_ahb_arbi", +- "hclk_vio_noc", +- "hclk_vop_noc", +- "hclk_host0_arb", +- "hclk_host1_arb", +- "hclk_host2_arb", +- "hclk_otg_pmu", +- "aclk_gpu_noc", +- "sclk_initmem_mbist", +- "aclk_initmem", +- "hclk_rom", +- "pclk_ddrupctl", +- "pclk_ddrmon", +- "pclk_msch_noc", +- "pclk_stimer", +- "pclk_ddrphy", +- "pclk_acodecphy", +- "pclk_phy_noc", +- "aclk_vpu_noc", +- "aclk_rkvdec_noc", +- "hclk_vpu_noc", +- "hclk_rkvdec_noc", +-}; ++static void __iomem *rk3228_cru_base; ++ ++static void rk3228_dump_cru(void) ++{ ++ if (rk3228_cru_base) { ++ pr_warn("CRU:\n"); ++ print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET, ++ 32, 4, rk3228_cru_base, ++ 0x1f8, false); ++ } ++} + + static void __init rk3228_clk_init(struct device_node *np) + { + struct rockchip_clk_provider *ctx; + void __iomem *reg_base; ++ struct clk **clks; + + reg_base = of_iomap(np, 0); + if (!reg_base) { +@@ -696,17 +680,16 @@ static void __init rk3228_clk_init(struct device_node *np) + iounmap(reg_base); + return; + } ++ clks = ctx->clk_data.clks; + + rockchip_clk_register_plls(ctx, rk3228_pll_clks, + ARRAY_SIZE(rk3228_pll_clks), + RK3228_GRF_SOC_STATUS0); + rockchip_clk_register_branches(ctx, rk3228_clk_branches, + ARRAY_SIZE(rk3228_clk_branches)); +- rockchip_clk_protect_critical(rk3228_critical_clocks, +- ARRAY_SIZE(rk3228_critical_clocks)); + + rockchip_clk_register_armclk(ctx, ARMCLK, "armclk", +- mux_armclk_p, ARRAY_SIZE(mux_armclk_p), ++ 3, clks[PLL_APLL], clks[PLL_GPLL], + &rk3228_cpuclk_data, rk3228_cpuclk_rates, + ARRAY_SIZE(rk3228_cpuclk_rates)); + +@@ -716,5 +699,38 @@ static void __init rk3228_clk_init(struct device_node *np) + rockchip_register_restart_notifier(ctx, RK3228_GLB_SRST_FST, NULL); + + rockchip_clk_of_add_provider(np, ctx); ++ ++ if (!rk_dump_cru) { ++ rk3228_cru_base = reg_base; ++ rk_dump_cru = rk3228_dump_cru; ++ } + } + CLK_OF_DECLARE(rk3228_cru, "rockchip,rk3228-cru", rk3228_clk_init); ++ ++static int __init clk_rk3228_probe(struct platform_device *pdev) ++{ ++ struct device_node *np = pdev->dev.of_node; ++ ++ rk3228_clk_init(np); ++ ++ return 0; ++} ++ ++static const struct of_device_id clk_rk3228_match_table[] = { ++ { ++ .compatible = "rockchip,rk3228-cru", ++ }, ++ { } ++}; ++MODULE_DEVICE_TABLE(of, clk_rk3228_match_table); ++ ++static struct platform_driver clk_rk3228_driver = { ++ .driver = { ++ .name = "clk-rk3228", ++ .of_match_table = clk_rk3228_match_table, ++ }, ++}; ++builtin_platform_driver_probe(clk_rk3228_driver, clk_rk3228_probe); ++ ++MODULE_DESCRIPTION("Rockchip RK3228 Clock Driver"); ++MODULE_LICENSE("GPL"); +diff --git a/drivers/clk/rockchip/clk-rk3288.c b/drivers/clk/rockchip/clk-rk3288.c +index 93c794695c46..3e054ddf1931 100644 +--- a/drivers/clk/rockchip/clk-rk3288.c ++++ b/drivers/clk/rockchip/clk-rk3288.c +@@ -6,14 +6,20 @@ + + #include + #include ++#include + #include + #include ++#include + #include + #include + #include "clk.h" ++#include + + #define RK3288_GRF_SOC_CON(x) (0x244 + x * 4) + #define RK3288_GRF_SOC_STATUS1 0x284 ++#define RK3288_UART_FRAC_MAX_PRATE 600000000 ++#define RK3288_I2S_FRAC_MAX_PRATE 600000000 ++#define RK3288_SPDIF_FRAC_MAX_PRATE 600000000 + + enum rk3288_variant { + RK3288_CRU, +@@ -66,32 +72,32 @@ static struct rockchip_pll_rate_table rk3288_pll_rates[] = { + RK3066_PLL_RATE(1248000000, 1, 52, 1), + RK3066_PLL_RATE(1224000000, 1, 51, 1), + RK3066_PLL_RATE(1200000000, 1, 50, 1), +- RK3066_PLL_RATE(1188000000, 2, 99, 1), ++ RK3066_PLL_RATE(1188000000, 1, 99, 2), + RK3066_PLL_RATE(1176000000, 1, 49, 1), + RK3066_PLL_RATE(1128000000, 1, 47, 1), + RK3066_PLL_RATE(1104000000, 1, 46, 1), + RK3066_PLL_RATE(1008000000, 1, 84, 2), + RK3066_PLL_RATE( 912000000, 1, 76, 2), +- RK3066_PLL_RATE( 891000000, 8, 594, 2), ++ RK3066_PLL_RATE( 891000000, 2, 297, 4), + RK3066_PLL_RATE( 888000000, 1, 74, 2), + RK3066_PLL_RATE( 816000000, 1, 68, 2), +- RK3066_PLL_RATE( 798000000, 2, 133, 2), ++ RK3066_PLL_RATE( 798000000, 1, 133, 4), + RK3066_PLL_RATE( 792000000, 1, 66, 2), + RK3066_PLL_RATE( 768000000, 1, 64, 2), +- RK3066_PLL_RATE( 742500000, 8, 495, 2), ++ RK3066_PLL_RATE( 742500000, 4, 495, 4), + RK3066_PLL_RATE( 696000000, 1, 58, 2), + RK3066_PLL_RATE_NB(621000000, 1, 207, 8, 1), + RK3066_PLL_RATE( 600000000, 1, 50, 2), +- RK3066_PLL_RATE_NB(594000000, 1, 198, 8, 1), ++ RK3066_PLL_RATE_NB(594000000, 1, 99, 4, 1), + RK3066_PLL_RATE( 552000000, 1, 46, 2), + RK3066_PLL_RATE( 504000000, 1, 84, 4), +- RK3066_PLL_RATE( 500000000, 3, 125, 2), ++ RK3066_PLL_RATE( 500000000, 1, 125, 6), + RK3066_PLL_RATE( 456000000, 1, 76, 4), + RK3066_PLL_RATE( 428000000, 1, 107, 6), + RK3066_PLL_RATE( 408000000, 1, 68, 4), +- RK3066_PLL_RATE( 400000000, 3, 100, 2), ++ RK3066_PLL_RATE( 400000000, 1, 100, 6), + RK3066_PLL_RATE_NB( 394000000, 1, 197, 12, 1), +- RK3066_PLL_RATE( 384000000, 2, 128, 4), ++ RK3066_PLL_RATE( 384000000, 1, 64, 4), + RK3066_PLL_RATE( 360000000, 1, 60, 4), + RK3066_PLL_RATE_NB( 356000000, 1, 178, 12, 1), + RK3066_PLL_RATE_NB( 324000000, 1, 189, 14, 1), +@@ -100,6 +106,7 @@ static struct rockchip_pll_rate_table rk3288_pll_rates[] = { + RK3066_PLL_RATE_NB( 303000000, 1, 202, 16, 1), + RK3066_PLL_RATE( 300000000, 1, 75, 6), + RK3066_PLL_RATE_NB( 297750000, 2, 397, 16, 1), ++ RK3066_PLL_RATE( 297000000, 1, 99, 8), + RK3066_PLL_RATE_NB( 293250000, 2, 391, 16, 1), + RK3066_PLL_RATE_NB( 292500000, 1, 195, 16, 1), + RK3066_PLL_RATE( 273600000, 1, 114, 10), +@@ -117,6 +124,7 @@ static struct rockchip_pll_rate_table rk3288_pll_rates[] = { + RK3066_PLL_RATE( 195428571, 1, 114, 14), + RK3066_PLL_RATE( 160000000, 1, 80, 12), + RK3066_PLL_RATE( 157500000, 1, 105, 16), ++ RK3066_PLL_RATE( 148500000, 1, 99, 16), + RK3066_PLL_RATE( 126000000, 1, 84, 16), + { /* sentinel */ }, + }; +@@ -179,9 +187,10 @@ static struct rockchip_cpuclk_rate_table rk3288_cpuclk_rates[] __initdata = { + }; + + static const struct rockchip_cpuclk_reg_data rk3288_cpuclk_data = { +- .core_reg = RK3288_CLKSEL_CON(0), +- .div_core_shift = 8, +- .div_core_mask = 0x1f, ++ .core_reg[0] = RK3288_CLKSEL_CON(0), ++ .div_core_shift[0] = 8, ++ .div_core_mask[0] = 0x1f, ++ .num_cores = 1, + .mux_core_alt = 1, + .mux_core_main = 0, + .mux_core_shift = 15, +@@ -189,7 +198,6 @@ static const struct rockchip_cpuclk_reg_data rk3288_cpuclk_data = { + }; + + PNAME(mux_pll_p) = { "xin24m", "xin32k" }; +-PNAME(mux_armclk_p) = { "apll_core", "gpll_core" }; + PNAME(mux_ddrphy_p) = { "dpll_ddr", "gpll_ddr" }; + PNAME(mux_aclk_cpu_src_p) = { "cpll_aclk_cpu", "gpll_aclk_cpu" }; + +@@ -330,20 +338,20 @@ static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = { + RK3288_CLKSEL_CON(26), 2, 1, MFLAGS, 0, 2, + DFLAGS | CLK_DIVIDER_POWER_OF_TWO), + +- GATE(0, "gpll_aclk_cpu", "gpll", CLK_IGNORE_UNUSED, ++ GATE(0, "gpll_aclk_cpu", "gpll", CLK_IS_CRITICAL, + RK3288_CLKGATE_CON(0), 10, GFLAGS), +- GATE(0, "cpll_aclk_cpu", "cpll", CLK_IGNORE_UNUSED, ++ GATE(0, "cpll_aclk_cpu", "cpll", CLK_IS_CRITICAL, + RK3288_CLKGATE_CON(0), 11, GFLAGS), +- COMPOSITE_NOGATE(0, "aclk_cpu_src", mux_aclk_cpu_src_p, CLK_IGNORE_UNUSED, ++ COMPOSITE_NOGATE(0, "aclk_cpu_src", mux_aclk_cpu_src_p, CLK_IS_CRITICAL, + RK3288_CLKSEL_CON(1), 15, 1, MFLAGS, 3, 5, DFLAGS), + DIV(0, "aclk_cpu_pre", "aclk_cpu_src", CLK_SET_RATE_PARENT, + RK3288_CLKSEL_CON(1), 0, 3, DFLAGS), +- GATE(ACLK_CPU, "aclk_cpu", "aclk_cpu_pre", CLK_IGNORE_UNUSED, ++ GATE(ACLK_CPU, "aclk_cpu", "aclk_cpu_pre", CLK_IS_CRITICAL, + RK3288_CLKGATE_CON(0), 3, GFLAGS), +- COMPOSITE_NOMUX(PCLK_CPU, "pclk_cpu", "aclk_cpu_pre", CLK_IGNORE_UNUSED, ++ COMPOSITE_NOMUX(PCLK_CPU, "pclk_cpu", "aclk_cpu_pre", CLK_IS_CRITICAL, + RK3288_CLKSEL_CON(1), 12, 3, DFLAGS, + RK3288_CLKGATE_CON(0), 5, GFLAGS), +- COMPOSITE_NOMUX_DIVTBL(HCLK_CPU, "hclk_cpu", "aclk_cpu_pre", CLK_IGNORE_UNUSED, ++ COMPOSITE_NOMUX_DIVTBL(HCLK_CPU, "hclk_cpu", "aclk_cpu_pre", CLK_IS_CRITICAL, + RK3288_CLKSEL_CON(1), 8, 2, DFLAGS, div_hclk_cpu_t, + RK3288_CLKGATE_CON(0), 4, GFLAGS), + GATE(0, "c2c_host", "aclk_cpu_src", 0, +@@ -362,7 +370,7 @@ static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "i2s_frac", "i2s_src", CLK_SET_RATE_PARENT, + RK3288_CLKSEL_CON(8), 0, + RK3288_CLKGATE_CON(4), 2, GFLAGS, +- &rk3288_i2s_fracmux), ++ &rk3288_i2s_fracmux, RK3288_I2S_FRAC_MAX_PRATE), + COMPOSITE_NODIV(SCLK_I2S0_OUT, "i2s0_clkout", mux_i2s_clkout_p, 0, + RK3288_CLKSEL_CON(4), 12, 1, MFLAGS, + RK3288_CLKGATE_CON(4), 0, GFLAGS), +@@ -377,7 +385,7 @@ static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "spdif_frac", "spdif_src", CLK_SET_RATE_PARENT, + RK3288_CLKSEL_CON(9), 0, + RK3288_CLKGATE_CON(4), 5, GFLAGS, +- &rk3288_spdif_fracmux), ++ &rk3288_spdif_fracmux, RK3288_SPDIF_FRAC_MAX_PRATE), + GATE(SCLK_SPDIF, "sclk_spdif", "spdif_mux", CLK_SET_RATE_PARENT, + RK3288_CLKGATE_CON(4), 6, GFLAGS), + COMPOSITE_NOMUX(0, "spdif_8ch_pre", "spdif_src", CLK_SET_RATE_PARENT, +@@ -386,7 +394,7 @@ static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "spdif_8ch_frac", "spdif_8ch_pre", CLK_SET_RATE_PARENT, + RK3288_CLKSEL_CON(41), 0, + RK3288_CLKGATE_CON(4), 8, GFLAGS, +- &rk3288_spdif_8ch_fracmux), ++ &rk3288_spdif_8ch_fracmux, RK3288_SPDIF_FRAC_MAX_PRATE), + GATE(SCLK_SPDIF8CH, "sclk_spdif_8ch", "spdif_8ch_mux", CLK_SET_RATE_PARENT, + RK3288_CLKGATE_CON(4), 9, GFLAGS), + +@@ -486,9 +494,9 @@ static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = { + COMPOSITE_NOGATE(SCLK_VIP_OUT, "sclk_vip_out", mux_vip_out_p, 0, + RK3288_CLKSEL_CON(26), 15, 1, MFLAGS, 9, 5, DFLAGS), + +- DIV(0, "pclk_pd_alive", "gpll", 0, ++ DIV(PCLK_PD_ALIVE, "pclk_pd_alive", "gpll", CLK_IS_CRITICAL, + RK3288_CLKSEL_CON(33), 8, 5, DFLAGS), +- COMPOSITE_NOMUX(0, "pclk_pd_pmu", "gpll", CLK_IGNORE_UNUSED, ++ COMPOSITE_NOMUX(PCLK_PD_PMU, "pclk_pd_pmu", "gpll", CLK_IS_CRITICAL, + RK3288_CLKSEL_CON(33), 0, 5, DFLAGS, + RK3288_CLKGATE_CON(5), 8, GFLAGS), + +@@ -496,16 +504,16 @@ static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = { + RK3288_CLKSEL_CON(34), 6, 2, MFLAGS, 0, 5, DFLAGS, + RK3288_CLKGATE_CON(5), 7, GFLAGS), + +- COMPOSITE(0, "aclk_peri_src", mux_pll_src_cpll_gpll_p, CLK_IGNORE_UNUSED, ++ COMPOSITE(0, "aclk_peri_src", mux_pll_src_cpll_gpll_p, CLK_IS_CRITICAL, + RK3288_CLKSEL_CON(10), 15, 1, MFLAGS, 0, 5, DFLAGS, + RK3288_CLKGATE_CON(2), 0, GFLAGS), +- COMPOSITE_NOMUX(PCLK_PERI, "pclk_peri", "aclk_peri_src", 0, ++ COMPOSITE_NOMUX(PCLK_PERI, "pclk_peri", "aclk_peri_src", CLK_IS_CRITICAL, + RK3288_CLKSEL_CON(10), 12, 2, DFLAGS | CLK_DIVIDER_POWER_OF_TWO, + RK3288_CLKGATE_CON(2), 3, GFLAGS), +- COMPOSITE_NOMUX(HCLK_PERI, "hclk_peri", "aclk_peri_src", CLK_IGNORE_UNUSED, ++ COMPOSITE_NOMUX(HCLK_PERI, "hclk_peri", "aclk_peri_src", CLK_IS_CRITICAL, + RK3288_CLKSEL_CON(10), 8, 2, DFLAGS | CLK_DIVIDER_POWER_OF_TWO, + RK3288_CLKGATE_CON(2), 2, GFLAGS), +- GATE(ACLK_PERI, "aclk_peri", "aclk_peri_src", CLK_IGNORE_UNUSED, ++ GATE(ACLK_PERI, "aclk_peri", "aclk_peri_src", CLK_IS_CRITICAL, + RK3288_CLKGATE_CON(2), 1, GFLAGS), + + /* +@@ -587,7 +595,7 @@ static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "uart0_frac", "uart0_src", CLK_SET_RATE_PARENT, + RK3288_CLKSEL_CON(17), 0, + RK3288_CLKGATE_CON(1), 9, GFLAGS, +- &rk3288_uart0_fracmux), ++ &rk3288_uart0_fracmux, RK3288_UART_FRAC_MAX_PRATE), + MUX(0, "uart_src", mux_pll_src_cpll_gpll_p, 0, + RK3288_CLKSEL_CON(13), 15, 1, MFLAGS), + COMPOSITE_NOMUX(0, "uart1_src", "uart_src", 0, +@@ -596,28 +604,28 @@ static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "uart1_frac", "uart1_src", CLK_SET_RATE_PARENT, + RK3288_CLKSEL_CON(18), 0, + RK3288_CLKGATE_CON(1), 11, GFLAGS, +- &rk3288_uart1_fracmux), ++ &rk3288_uart1_fracmux, RK3288_UART_FRAC_MAX_PRATE), + COMPOSITE_NOMUX(0, "uart2_src", "uart_src", 0, + RK3288_CLKSEL_CON(15), 0, 7, DFLAGS, + RK3288_CLKGATE_CON(1), 12, GFLAGS), + COMPOSITE_FRACMUX(0, "uart2_frac", "uart2_src", CLK_SET_RATE_PARENT, + RK3288_CLKSEL_CON(19), 0, + RK3288_CLKGATE_CON(1), 13, GFLAGS, +- &rk3288_uart2_fracmux), ++ &rk3288_uart2_fracmux, RK3288_UART_FRAC_MAX_PRATE), + COMPOSITE_NOMUX(0, "uart3_src", "uart_src", 0, + RK3288_CLKSEL_CON(16), 0, 7, DFLAGS, + RK3288_CLKGATE_CON(1), 14, GFLAGS), + COMPOSITE_FRACMUX(0, "uart3_frac", "uart3_src", CLK_SET_RATE_PARENT, + RK3288_CLKSEL_CON(20), 0, + RK3288_CLKGATE_CON(1), 15, GFLAGS, +- &rk3288_uart3_fracmux), ++ &rk3288_uart3_fracmux, RK3288_UART_FRAC_MAX_PRATE), + COMPOSITE_NOMUX(0, "uart4_src", "uart_src", 0, + RK3288_CLKSEL_CON(3), 0, 7, DFLAGS, + RK3288_CLKGATE_CON(2), 12, GFLAGS), + COMPOSITE_FRACMUX(0, "uart4_frac", "uart4_src", CLK_SET_RATE_PARENT, + RK3288_CLKSEL_CON(7), 0, + RK3288_CLKGATE_CON(2), 13, GFLAGS, +- &rk3288_uart4_fracmux), ++ &rk3288_uart4_fracmux, RK3288_UART_FRAC_MAX_PRATE), + + COMPOSITE(0, "mac_pll_src", mux_pll_src_npll_cpll_gpll_p, 0, + RK3288_CLKSEL_CON(21), 0, 2, MFLAGS, 8, 5, DFLAGS, +@@ -665,7 +673,7 @@ static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = { + GATE(0, "sclk_intmem0", "aclk_cpu", CLK_IGNORE_UNUSED, RK3288_CLKGATE_CON(10), 5, GFLAGS), + GATE(0, "sclk_intmem1", "aclk_cpu", CLK_IGNORE_UNUSED, RK3288_CLKGATE_CON(10), 6, GFLAGS), + GATE(0, "sclk_intmem2", "aclk_cpu", CLK_IGNORE_UNUSED, RK3288_CLKGATE_CON(10), 7, GFLAGS), +- GATE(ACLK_DMAC1, "aclk_dmac1", "aclk_cpu", 0, RK3288_CLKGATE_CON(10), 12, GFLAGS), ++ GATE(ACLK_DMAC1, "aclk_dmac1", "aclk_cpu", CLK_IS_CRITICAL, RK3288_CLKGATE_CON(10), 12, GFLAGS), + GATE(0, "aclk_strc_sys", "aclk_cpu", CLK_IGNORE_UNUSED, RK3288_CLKGATE_CON(10), 13, GFLAGS), + GATE(0, "aclk_intmem", "aclk_cpu", CLK_IGNORE_UNUSED, RK3288_CLKGATE_CON(10), 4, GFLAGS), + GATE(ACLK_CRYPTO, "aclk_crypto", "aclk_cpu", 0, RK3288_CLKGATE_CON(11), 6, GFLAGS), +@@ -691,7 +699,7 @@ static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = { + GATE(PCLK_TZPC, "pclk_tzpc", "pclk_cpu", 0, RK3288_CLKGATE_CON(11), 3, GFLAGS), + GATE(PCLK_UART2, "pclk_uart2", "pclk_cpu", 0, RK3288_CLKGATE_CON(11), 9, GFLAGS), + GATE(PCLK_EFUSE256, "pclk_efuse_256", "pclk_cpu", 0, RK3288_CLKGATE_CON(11), 10, GFLAGS), +- GATE(PCLK_RKPWM, "pclk_rkpwm", "pclk_cpu", 0, RK3288_CLKGATE_CON(11), 11, GFLAGS), ++ GATE(PCLK_RKPWM, "pclk_rkpwm", "pclk_cpu", CLK_IS_CRITICAL, RK3288_CLKGATE_CON(11), 11, GFLAGS), + + /* ddrctrl [DDR Controller PHY clock] gates */ + GATE(0, "nclk_ddrupctl0", "ddrphy", CLK_IGNORE_UNUSED, RK3288_CLKGATE_CON(11), 4, GFLAGS), +@@ -727,7 +735,7 @@ static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = { + GATE(HCLK_SDIO1, "hclk_sdio1", "hclk_peri", 0, RK3288_CLKGATE_CON(8), 5, GFLAGS), + GATE(HCLK_EMMC, "hclk_emmc", "hclk_peri", 0, RK3288_CLKGATE_CON(8), 6, GFLAGS), + GATE(HCLK_HSADC, "hclk_hsadc", "hclk_peri", 0, RK3288_CLKGATE_CON(8), 7, GFLAGS), +- GATE(0, "pmu_hclk_otg0", "hclk_peri", 0, RK3288_CLKGATE_CON(7), 5, GFLAGS), ++ GATE(0, "pmu_hclk_otg0", "hclk_peri", CLK_IS_CRITICAL, RK3288_CLKGATE_CON(7), 5, GFLAGS), + + /* pclk_peri gates */ + GATE(0, "pclk_peri_matrix", "pclk_peri", CLK_IGNORE_UNUSED, RK3288_CLKGATE_CON(6), 1, GFLAGS), +@@ -767,7 +775,7 @@ static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = { + GATE(PCLK_GPIO5, "pclk_gpio5", "pclk_pd_alive", 0, RK3288_CLKGATE_CON(14), 5, GFLAGS), + GATE(PCLK_GPIO6, "pclk_gpio6", "pclk_pd_alive", 0, RK3288_CLKGATE_CON(14), 6, GFLAGS), + GATE(PCLK_GRF, "pclk_grf", "pclk_pd_alive", CLK_IGNORE_UNUSED, RK3288_CLKGATE_CON(14), 11, GFLAGS), +- GATE(0, "pclk_alive_niu", "pclk_pd_alive", 0, RK3288_CLKGATE_CON(14), 12, GFLAGS), ++ GATE(0, "pclk_alive_niu", "pclk_pd_alive", CLK_IS_CRITICAL, RK3288_CLKGATE_CON(14), 12, GFLAGS), + + /* Watchdog pclk is controlled by RK3288_SGRF_SOC_CON0[1]. */ + SGRF_GATE(PCLK_WDT, "pclk_wdt", "pclk_pd_alive"), +@@ -775,7 +783,7 @@ static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = { + /* pclk_pd_pmu gates */ + GATE(PCLK_PMU, "pclk_pmu", "pclk_pd_pmu", CLK_IGNORE_UNUSED, RK3288_CLKGATE_CON(17), 0, GFLAGS), + GATE(0, "pclk_intmem1", "pclk_pd_pmu", CLK_IGNORE_UNUSED, RK3288_CLKGATE_CON(17), 1, GFLAGS), +- GATE(0, "pclk_pmu_niu", "pclk_pd_pmu", 0, RK3288_CLKGATE_CON(17), 2, GFLAGS), ++ GATE(0, "pclk_pmu_niu", "pclk_pd_pmu", CLK_IS_CRITICAL, RK3288_CLKGATE_CON(17), 2, GFLAGS), + GATE(PCLK_SGRF, "pclk_sgrf", "pclk_pd_pmu", CLK_IGNORE_UNUSED, RK3288_CLKGATE_CON(17), 3, GFLAGS), + GATE(PCLK_GPIO0, "pclk_gpio0", "pclk_pd_pmu", 0, RK3288_CLKGATE_CON(17), 4, GFLAGS), + +@@ -784,7 +792,7 @@ static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = { + GATE(HCLK_VOP0, "hclk_vop0", "hclk_vio", 0, RK3288_CLKGATE_CON(15), 6, GFLAGS), + GATE(HCLK_VOP1, "hclk_vop1", "hclk_vio", 0, RK3288_CLKGATE_CON(15), 8, GFLAGS), + GATE(HCLK_VIO_AHB_ARBI, "hclk_vio_ahb_arbi", "hclk_vio", CLK_IGNORE_UNUSED, RK3288_CLKGATE_CON(15), 9, GFLAGS), +- GATE(HCLK_VIO_NIU, "hclk_vio_niu", "hclk_vio", 0, RK3288_CLKGATE_CON(15), 10, GFLAGS), ++ GATE(HCLK_VIO_NIU, "hclk_vio_niu", "hclk_vio", CLK_IS_CRITICAL, RK3288_CLKGATE_CON(15), 10, GFLAGS), + GATE(HCLK_VIP, "hclk_vip", "hclk_vio", 0, RK3288_CLKGATE_CON(15), 15, GFLAGS), + GATE(HCLK_IEP, "hclk_iep", "hclk_vio", 0, RK3288_CLKGATE_CON(15), 3, GFLAGS), + GATE(HCLK_ISP, "hclk_isp", "hclk_vio", 0, RK3288_CLKGATE_CON(16), 1, GFLAGS), +@@ -800,17 +808,17 @@ static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = { + /* aclk_vio0 gates */ + GATE(ACLK_VOP0, "aclk_vop0", "aclk_vio0", 0, RK3288_CLKGATE_CON(15), 5, GFLAGS), + GATE(ACLK_IEP, "aclk_iep", "aclk_vio0", 0, RK3288_CLKGATE_CON(15), 2, GFLAGS), +- GATE(ACLK_VIO0_NIU, "aclk_vio0_niu", "aclk_vio0", 0, RK3288_CLKGATE_CON(15), 11, GFLAGS), ++ GATE(ACLK_VIO0_NIU, "aclk_vio0_niu", "aclk_vio0", CLK_IS_CRITICAL, RK3288_CLKGATE_CON(15), 11, GFLAGS), + GATE(ACLK_VIP, "aclk_vip", "aclk_vio0", 0, RK3288_CLKGATE_CON(15), 14, GFLAGS), + + /* aclk_vio1 gates */ + GATE(ACLK_VOP1, "aclk_vop1", "aclk_vio1", 0, RK3288_CLKGATE_CON(15), 7, GFLAGS), + GATE(ACLK_ISP, "aclk_isp", "aclk_vio1", 0, RK3288_CLKGATE_CON(16), 2, GFLAGS), +- GATE(ACLK_VIO1_NIU, "aclk_vio1_niu", "aclk_vio1", 0, RK3288_CLKGATE_CON(15), 12, GFLAGS), ++ GATE(ACLK_VIO1_NIU, "aclk_vio1_niu", "aclk_vio1", CLK_IS_CRITICAL, RK3288_CLKGATE_CON(15), 12, GFLAGS), + + /* aclk_rga_pre gates */ + GATE(ACLK_RGA, "aclk_rga", "aclk_rga_pre", 0, RK3288_CLKGATE_CON(15), 0, GFLAGS), +- GATE(ACLK_RGA_NIU, "aclk_rga_niu", "aclk_rga_pre", 0, RK3288_CLKGATE_CON(15), 13, GFLAGS), ++ GATE(ACLK_RGA_NIU, "aclk_rga_niu", "aclk_rga_pre", CLK_IS_CRITICAL, RK3288_CLKGATE_CON(15), 13, GFLAGS), + + /* + * Other ungrouped clocks. +@@ -832,23 +840,6 @@ static struct rockchip_clk_branch rk3288_hclkvio_branch[] __initdata = { + RK3288_CLKSEL_CON(28), 8, 5, DFLAGS), + }; + +-static const char *const rk3288_critical_clocks[] __initconst = { +- "aclk_cpu", +- "aclk_peri", +- "aclk_peri_niu", +- "aclk_vio0_niu", +- "aclk_vio1_niu", +- "aclk_rga_niu", +- "hclk_peri", +- "hclk_vio_niu", +- "pclk_alive_niu", +- "pclk_pd_pmu", +- "pclk_pmu_niu", +- "pmu_hclk_otg0", +- /* pwm-regulators on some boards, so handoff-critical later */ +- "pclk_rkpwm", +-}; +- + static void __iomem *rk3288_cru_base; + + /* +@@ -927,10 +918,21 @@ static struct syscore_ops rk3288_clk_syscore_ops = { + .resume = rk3288_clk_resume, + }; + ++static void rk3288_dump_cru(void) ++{ ++ if (rk3288_cru_base) { ++ pr_warn("CRU:\n"); ++ print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET, ++ 32, 4, rk3288_cru_base, ++ 0x21c, false); ++ } ++} ++ + static void __init rk3288_common_init(struct device_node *np, + enum rk3288_variant soc) + { + struct rockchip_clk_provider *ctx; ++ struct clk **clks; + + rk3288_cru_base = of_iomap(np, 0); + if (!rk3288_cru_base) { +@@ -944,6 +946,7 @@ static void __init rk3288_common_init(struct device_node *np, + iounmap(rk3288_cru_base); + return; + } ++ clks = ctx->clk_data.clks; + + rockchip_clk_register_plls(ctx, rk3288_pll_clks, + ARRAY_SIZE(rk3288_pll_clks), +@@ -958,11 +961,8 @@ static void __init rk3288_common_init(struct device_node *np, + rockchip_clk_register_branches(ctx, rk3288_hclkvio_branch, + ARRAY_SIZE(rk3288_hclkvio_branch)); + +- rockchip_clk_protect_critical(rk3288_critical_clocks, +- ARRAY_SIZE(rk3288_critical_clocks)); +- + rockchip_clk_register_armclk(ctx, ARMCLK, "armclk", +- mux_armclk_p, ARRAY_SIZE(mux_armclk_p), ++ 2, clks[PLL_APLL], clks[PLL_GPLL], + &rk3288_cpuclk_data, rk3288_cpuclk_rates, + ARRAY_SIZE(rk3288_cpuclk_rates)); + +@@ -972,9 +972,14 @@ static void __init rk3288_common_init(struct device_node *np, + + rockchip_register_restart_notifier(ctx, RK3288_GLB_SRST_FST, + rk3288_clk_shutdown); +- register_syscore_ops(&rk3288_clk_syscore_ops); ++ ++ if (!psci_smp_available()) ++ register_syscore_ops(&rk3288_clk_syscore_ops); + + rockchip_clk_of_add_provider(np, ctx); ++ ++ if (!rk_dump_cru) ++ rk_dump_cru = rk3288_dump_cru; + } + + static void __init rk3288_clk_init(struct device_node *np) +@@ -988,3 +993,55 @@ static void __init rk3288w_clk_init(struct device_node *np) + rk3288_common_init(np, RK3288W_CRU); + } + CLK_OF_DECLARE(rk3288w_cru, "rockchip,rk3288w-cru", rk3288w_clk_init); ++ ++struct clk_rk3288_inits { ++ void (*inits)(struct device_node *np); ++}; ++ ++static const struct clk_rk3288_inits clk_rk3288_init = { ++ .inits = rk3288_clk_init, ++}; ++ ++static const struct clk_rk3288_inits clk_rk3288w_init = { ++ .inits = rk3288w_clk_init, ++}; ++ ++static const struct of_device_id clk_rk3288_match_table[] = { ++ { ++ .compatible = "rockchip,rk3288-cru", ++ .data = &clk_rk3288_init, ++ }, { ++ .compatible = "rockchip,rk3288w-cru", ++ .data = &clk_rk3288w_init, ++ }, ++ { } ++}; ++MODULE_DEVICE_TABLE(of, clk_rk3288_match_table); ++ ++static int __init clk_rk3288_probe(struct platform_device *pdev) ++{ ++ struct device_node *np = pdev->dev.of_node; ++ const struct of_device_id *match; ++ const struct clk_rk3288_inits *init_data; ++ ++ match = of_match_device(clk_rk3288_match_table, &pdev->dev); ++ if (!match || !match->data) ++ return -EINVAL; ++ ++ init_data = match->data; ++ if (init_data->inits) ++ init_data->inits(np); ++ ++ return 0; ++} ++ ++static struct platform_driver clk_rk3288_driver = { ++ .driver = { ++ .name = "clk-rk3288", ++ .of_match_table = clk_rk3288_match_table, ++ }, ++}; ++builtin_platform_driver_probe(clk_rk3288_driver, clk_rk3288_probe); ++ ++MODULE_DESCRIPTION("Rockchip RK3288 Clock Driver"); ++MODULE_LICENSE("GPL"); +diff --git a/drivers/clk/rockchip/clk-rk3308.c b/drivers/clk/rockchip/clk-rk3308.c +index 5bf15f2a44b7..539d37ae89c9 100644 +--- a/drivers/clk/rockchip/clk-rk3308.c ++++ b/drivers/clk/rockchip/clk-rk3308.c +@@ -8,11 +8,20 @@ + #include + #include + #include ++#include ++#include ++#include + #include + #include + #include "clk.h" + + #define RK3308_GRF_SOC_STATUS0 0x380 ++#define RK3308_VOP_FRAC_MAX_PRATE 270000000 ++#define RK3308B_VOP_FRAC_MAX_PRATE 800000000 ++#define RK3308_UART_FRAC_MAX_PRATE 800000000 ++#define RK3308_PDM_FRAC_MAX_PRATE 800000000 ++#define RK3308_SPDIF_FRAC_MAX_PRATE 800000000 ++#define RK3308_I2S_FRAC_MAX_PRATE 800000000 + + enum rk3308_plls { + apll, dpll, vpll0, vpll1, +@@ -109,9 +118,10 @@ static struct rockchip_cpuclk_rate_table rk3308_cpuclk_rates[] __initdata = { + }; + + static const struct rockchip_cpuclk_reg_data rk3308_cpuclk_data = { +- .core_reg = RK3308_CLKSEL_CON(0), +- .div_core_shift = 0, +- .div_core_mask = 0xf, ++ .core_reg[0] = RK3308_CLKSEL_CON(0), ++ .div_core_shift[0] = 0, ++ .div_core_mask[0] = 0xf, ++ .num_cores = 1, + .mux_core_alt = 1, + .mux_core_main = 0, + .mux_core_shift = 6, +@@ -120,7 +130,6 @@ static const struct rockchip_cpuclk_reg_data rk3308_cpuclk_data = { + + PNAME(mux_pll_p) = { "xin24m" }; + PNAME(mux_usb480m_p) = { "xin24m", "usb480m_phy", "clk_rtc32k" }; +-PNAME(mux_armclk_p) = { "apll_core", "vpll0_core", "vpll1_core" }; + PNAME(mux_dpll_vpll0_p) = { "dpll", "vpll0" }; + PNAME(mux_dpll_vpll0_xin24m_p) = { "dpll", "vpll0", "xin24m" }; + PNAME(mux_dpll_vpll0_vpll1_p) = { "dpll", "vpll0", "vpll1" }; +@@ -174,6 +183,8 @@ PNAME(mux_spdif_tx_src_p) = { "clk_spdif_tx_div", "clk_spdif_tx_div50" }; + PNAME(mux_spdif_tx_p) = { "clk_spdif_tx_src", "clk_spdif_tx_frac", "mclk_i2s0_2ch_in" }; + PNAME(mux_spdif_rx_src_p) = { "clk_spdif_rx_div", "clk_spdif_rx_div50" }; + PNAME(mux_spdif_rx_p) = { "clk_spdif_rx_src", "clk_spdif_rx_frac" }; ++PNAME(mux_uart_src_p) = { "xin24m", "usb480m", "dpll", "vpll0", "vpll1" }; ++static u32 uart_src_mux_idx[] = { 4, 3, 0, 1, 2 }; + + static struct rockchip_pll_clock rk3308_pll_clks[] __initdata = { + [apll] = PLL(pll_rk3328, PLL_APLL, "apll", mux_pll_p, +@@ -311,68 +322,68 @@ static struct rockchip_clk_branch rk3308_clk_branches[] __initdata = { + * Clock-Architecture Diagram 3 + */ + +- COMPOSITE_NODIV(ACLK_BUS_SRC, "clk_bus_src", mux_dpll_vpll0_vpll1_p, CLK_IGNORE_UNUSED, ++ COMPOSITE_NODIV(ACLK_BUS_SRC, "clk_bus_src", mux_dpll_vpll0_vpll1_p, CLK_IS_CRITICAL, + RK3308_CLKSEL_CON(5), 6, 2, MFLAGS, + RK3308_CLKGATE_CON(1), 0, GFLAGS), +- COMPOSITE_NOMUX(PCLK_BUS, "pclk_bus", "clk_bus_src", CLK_IGNORE_UNUSED, ++ COMPOSITE_NOMUX(PCLK_BUS, "pclk_bus", "clk_bus_src", CLK_IS_CRITICAL, + RK3308_CLKSEL_CON(6), 8, 5, DFLAGS, + RK3308_CLKGATE_CON(1), 3, GFLAGS), + GATE(PCLK_DDR, "pclk_ddr", "pclk_bus", CLK_IGNORE_UNUSED, + RK3308_CLKGATE_CON(4), 15, GFLAGS), +- COMPOSITE_NOMUX(HCLK_BUS, "hclk_bus", "clk_bus_src", CLK_IGNORE_UNUSED, ++ COMPOSITE_NOMUX(HCLK_BUS, "hclk_bus", "clk_bus_src", CLK_IS_CRITICAL, + RK3308_CLKSEL_CON(6), 0, 5, DFLAGS, + RK3308_CLKGATE_CON(1), 2, GFLAGS), +- COMPOSITE_NOMUX(ACLK_BUS, "aclk_bus", "clk_bus_src", CLK_IGNORE_UNUSED, ++ COMPOSITE_NOMUX(ACLK_BUS, "aclk_bus", "clk_bus_src", CLK_IS_CRITICAL, + RK3308_CLKSEL_CON(5), 0, 5, DFLAGS, + RK3308_CLKGATE_CON(1), 1, GFLAGS), + +- COMPOSITE(0, "clk_uart0_src", mux_dpll_vpll0_vpll1_usb480m_xin24m_p, 0, +- RK3308_CLKSEL_CON(10), 13, 3, MFLAGS, 0, 5, DFLAGS, ++ COMPOSITE_MUXTBL(0, "clk_uart0_src", mux_uart_src_p, 0, ++ RK3308_CLKSEL_CON(10), 13, 3, MFLAGS, uart_src_mux_idx, 0, 5, DFLAGS, + RK3308_CLKGATE_CON(1), 9, GFLAGS), + COMPOSITE_FRACMUX(0, "clk_uart0_frac", "clk_uart0_src", CLK_SET_RATE_PARENT, + RK3308_CLKSEL_CON(12), 0, + RK3308_CLKGATE_CON(1), 11, GFLAGS, +- &rk3308_uart0_fracmux), ++ &rk3308_uart0_fracmux, RK3308_UART_FRAC_MAX_PRATE), + GATE(SCLK_UART0, "clk_uart0", "clk_uart0_mux", 0, + RK3308_CLKGATE_CON(1), 12, GFLAGS), + +- COMPOSITE(0, "clk_uart1_src", mux_dpll_vpll0_vpll1_usb480m_xin24m_p, 0, +- RK3308_CLKSEL_CON(13), 13, 3, MFLAGS, 0, 5, DFLAGS, ++ COMPOSITE_MUXTBL(0, "clk_uart1_src", mux_uart_src_p, 0, ++ RK3308_CLKSEL_CON(13), 13, 3, MFLAGS, uart_src_mux_idx, 0, 5, DFLAGS, + RK3308_CLKGATE_CON(1), 13, GFLAGS), + COMPOSITE_FRACMUX(0, "clk_uart1_frac", "clk_uart1_src", CLK_SET_RATE_PARENT, + RK3308_CLKSEL_CON(15), 0, + RK3308_CLKGATE_CON(1), 15, GFLAGS, +- &rk3308_uart1_fracmux), ++ &rk3308_uart1_fracmux, RK3308_UART_FRAC_MAX_PRATE), + GATE(SCLK_UART1, "clk_uart1", "clk_uart1_mux", 0, + RK3308_CLKGATE_CON(2), 0, GFLAGS), + +- COMPOSITE(0, "clk_uart2_src", mux_dpll_vpll0_vpll1_usb480m_xin24m_p, 0, +- RK3308_CLKSEL_CON(16), 13, 3, MFLAGS, 0, 5, DFLAGS, ++ COMPOSITE_MUXTBL(0, "clk_uart2_src", mux_uart_src_p, 0, ++ RK3308_CLKSEL_CON(16), 13, 3, MFLAGS, uart_src_mux_idx, 0, 5, DFLAGS, + RK3308_CLKGATE_CON(2), 1, GFLAGS), + COMPOSITE_FRACMUX(0, "clk_uart2_frac", "clk_uart2_src", CLK_SET_RATE_PARENT, + RK3308_CLKSEL_CON(18), 0, + RK3308_CLKGATE_CON(2), 3, GFLAGS, +- &rk3308_uart2_fracmux), ++ &rk3308_uart2_fracmux, RK3308_UART_FRAC_MAX_PRATE), + GATE(SCLK_UART2, "clk_uart2", "clk_uart2_mux", CLK_SET_RATE_PARENT, + RK3308_CLKGATE_CON(2), 4, GFLAGS), + +- COMPOSITE(0, "clk_uart3_src", mux_dpll_vpll0_vpll1_usb480m_xin24m_p, 0, +- RK3308_CLKSEL_CON(19), 13, 3, MFLAGS, 0, 5, DFLAGS, ++ COMPOSITE_MUXTBL(0, "clk_uart3_src", mux_uart_src_p, 0, ++ RK3308_CLKSEL_CON(19), 13, 3, MFLAGS, uart_src_mux_idx, 0, 5, DFLAGS, + RK3308_CLKGATE_CON(2), 5, GFLAGS), + COMPOSITE_FRACMUX(0, "clk_uart3_frac", "clk_uart3_src", CLK_SET_RATE_PARENT, + RK3308_CLKSEL_CON(21), 0, + RK3308_CLKGATE_CON(2), 7, GFLAGS, +- &rk3308_uart3_fracmux), ++ &rk3308_uart3_fracmux, RK3308_UART_FRAC_MAX_PRATE), + GATE(SCLK_UART3, "clk_uart3", "clk_uart3_mux", 0, + RK3308_CLKGATE_CON(2), 8, GFLAGS), + +- COMPOSITE(0, "clk_uart4_src", mux_dpll_vpll0_vpll1_usb480m_xin24m_p, 0, +- RK3308_CLKSEL_CON(22), 13, 3, MFLAGS, 0, 5, DFLAGS, ++ COMPOSITE_MUXTBL(0, "clk_uart4_src", mux_uart_src_p, 0, ++ RK3308_CLKSEL_CON(22), 13, 3, MFLAGS, uart_src_mux_idx, 0, 5, DFLAGS, + RK3308_CLKGATE_CON(2), 9, GFLAGS), + COMPOSITE_FRACMUX(0, "clk_uart4_frac", "clk_uart4_src", CLK_SET_RATE_PARENT, + RK3308_CLKSEL_CON(24), 0, + RK3308_CLKGATE_CON(2), 11, GFLAGS, +- &rk3308_uart4_fracmux), ++ &rk3308_uart4_fracmux, RK3308_UART_FRAC_MAX_PRATE), + GATE(SCLK_UART4, "clk_uart4", "clk_uart4_mux", 0, + RK3308_CLKGATE_CON(2), 12, GFLAGS), + +@@ -449,10 +460,6 @@ static struct rockchip_clk_branch rk3308_clk_branches[] __initdata = { + COMPOSITE(0, "dclk_vop_src", mux_dpll_vpll0_vpll1_p, 0, + RK3308_CLKSEL_CON(8), 10, 2, MFLAGS, 0, 8, DFLAGS, + RK3308_CLKGATE_CON(1), 6, GFLAGS), +- COMPOSITE_FRACMUX(0, "dclk_vop_frac", "dclk_vop_src", CLK_SET_RATE_PARENT, +- RK3308_CLKSEL_CON(9), 0, +- RK3308_CLKGATE_CON(1), 7, GFLAGS, +- &rk3308_dclk_vop_fracmux), + GATE(DCLK_VOP, "dclk_vop", "dclk_vop_mux", 0, + RK3308_CLKGATE_CON(1), 8, GFLAGS), + +@@ -460,16 +467,16 @@ static struct rockchip_clk_branch rk3308_clk_branches[] __initdata = { + * Clock-Architecture Diagram 4 + */ + +- COMPOSITE_NODIV(ACLK_PERI_SRC, "clk_peri_src", mux_dpll_vpll0_vpll1_p, CLK_IGNORE_UNUSED, ++ COMPOSITE_NODIV(ACLK_PERI_SRC, "clk_peri_src", mux_dpll_vpll0_vpll1_p, CLK_IS_CRITICAL, + RK3308_CLKSEL_CON(36), 6, 2, MFLAGS, + RK3308_CLKGATE_CON(8), 0, GFLAGS), +- COMPOSITE_NOMUX(ACLK_PERI, "aclk_peri", "clk_peri_src", CLK_IGNORE_UNUSED, ++ COMPOSITE_NOMUX(ACLK_PERI, "aclk_peri", "clk_peri_src", CLK_IS_CRITICAL, + RK3308_CLKSEL_CON(36), 0, 5, DFLAGS, + RK3308_CLKGATE_CON(8), 1, GFLAGS), +- COMPOSITE_NOMUX(HCLK_PERI, "hclk_peri", "clk_peri_src", CLK_IGNORE_UNUSED, ++ COMPOSITE_NOMUX(HCLK_PERI, "hclk_peri", "clk_peri_src", CLK_IS_CRITICAL, + RK3308_CLKSEL_CON(37), 0, 5, DFLAGS, + RK3308_CLKGATE_CON(8), 2, GFLAGS), +- COMPOSITE_NOMUX(PCLK_PERI, "pclk_peri", "clk_peri_src", CLK_IGNORE_UNUSED, ++ COMPOSITE_NOMUX(PCLK_PERI, "pclk_peri", "clk_peri_src", CLK_IS_CRITICAL, + RK3308_CLKSEL_CON(37), 8, 5, DFLAGS, + RK3308_CLKGATE_CON(8), 3, GFLAGS), + +@@ -560,10 +567,10 @@ static struct rockchip_clk_branch rk3308_clk_branches[] __initdata = { + GATE(0, "clk_ddr_msch_peribus", "clk_ddrphy1x_out", CLK_IGNORE_UNUSED, + RK3308_CLKGATE_CON(4), 13, GFLAGS), + +- COMPOSITE(SCLK_DDRCLK, "clk_ddrphy4x_src", mux_dpll_vpll0_vpll1_p, CLK_IGNORE_UNUSED, ++ COMPOSITE(SCLK_DDRCLK, "clk_ddrphy4x_src", mux_dpll_vpll0_vpll1_p, CLK_IS_CRITICAL, + RK3308_CLKSEL_CON(1), 6, 2, MFLAGS, 0, 3, DFLAGS, + RK3308_CLKGATE_CON(0), 10, GFLAGS), +- GATE(0, "clk_ddrphy4x", "clk_ddrphy4x_src", CLK_IGNORE_UNUSED, ++ GATE(0, "clk_ddrphy4x", "clk_ddrphy4x_src", CLK_IS_CRITICAL, + RK3308_CLKGATE_CON(0), 11, GFLAGS), + FACTOR_GATE(0, "clk_ddr_stdby_div4", "clk_ddrphy4x", CLK_IGNORE_UNUSED, 1, 4, + RK3308_CLKGATE_CON(0), 13, GFLAGS), +@@ -583,7 +590,7 @@ static struct rockchip_clk_branch rk3308_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "clk_rtc32k_frac", "xin24m", CLK_IGNORE_UNUSED, + RK3308_CLKSEL_CON(3), 0, + RK3308_CLKGATE_CON(4), 3, GFLAGS, +- &rk3308_rtc32k_fracmux), ++ &rk3308_rtc32k_fracmux, 0), + MUX(0, "clk_rtc32k_div_src", mux_vpll0_vpll1_p, 0, + RK3308_CLKSEL_CON(2), 10, 1, MFLAGS), + COMPOSITE_NOMUX(0, "clk_rtc32k_div", "clk_rtc32k_div_src", CLK_IGNORE_UNUSED | CLK_SET_RATE_PARENT, +@@ -617,13 +624,13 @@ static struct rockchip_clk_branch rk3308_clk_branches[] __initdata = { + * Clock-Architecture Diagram 7 + */ + +- COMPOSITE_NODIV(0, "clk_audio_src", mux_vpll0_vpll1_xin24m_p, 0, ++ COMPOSITE_NODIV(0, "clk_audio_src", mux_vpll0_vpll1_xin24m_p, CLK_IS_CRITICAL, + RK3308_CLKSEL_CON(45), 6, 2, MFLAGS, + RK3308_CLKGATE_CON(10), 0, GFLAGS), +- COMPOSITE_NOMUX(HCLK_AUDIO, "hclk_audio", "clk_audio_src", 0, ++ COMPOSITE_NOMUX(HCLK_AUDIO, "hclk_audio", "clk_audio_src", CLK_IS_CRITICAL, + RK3308_CLKSEL_CON(45), 0, 5, DFLAGS, + RK3308_CLKGATE_CON(10), 1, GFLAGS), +- COMPOSITE_NOMUX(PCLK_AUDIO, "pclk_audio", "clk_audio_src", 0, ++ COMPOSITE_NOMUX(PCLK_AUDIO, "pclk_audio", "clk_audio_src", CLK_IS_CRITICAL, + RK3308_CLKSEL_CON(45), 8, 5, DFLAGS, + RK3308_CLKGATE_CON(10), 2, GFLAGS), + +@@ -633,7 +640,7 @@ static struct rockchip_clk_branch rk3308_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "clk_pdm_frac", "clk_pdm_src", CLK_SET_RATE_PARENT, + RK3308_CLKSEL_CON(47), 0, + RK3308_CLKGATE_CON(10), 4, GFLAGS, +- &rk3308_pdm_fracmux), ++ &rk3308_pdm_fracmux, RK3308_PDM_FRAC_MAX_PRATE), + GATE(SCLK_PDM, "clk_pdm", "clk_pdm_mux", 0, + RK3308_CLKGATE_CON(10), 5, GFLAGS), + +@@ -643,7 +650,7 @@ static struct rockchip_clk_branch rk3308_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "clk_i2s0_8ch_tx_frac", "clk_i2s0_8ch_tx_src", CLK_SET_RATE_PARENT, + RK3308_CLKSEL_CON(53), 0, + RK3308_CLKGATE_CON(10), 13, GFLAGS, +- &rk3308_i2s0_8ch_tx_fracmux), ++ &rk3308_i2s0_8ch_tx_fracmux, RK3308_I2S_FRAC_MAX_PRATE), + COMPOSITE_NODIV(SCLK_I2S0_8CH_TX, "clk_i2s0_8ch_tx", mux_i2s0_8ch_tx_rx_p, CLK_SET_RATE_PARENT, + RK3308_CLKSEL_CON(52), 12, 1, MFLAGS, + RK3308_CLKGATE_CON(10), 14, GFLAGS), +@@ -657,7 +664,7 @@ static struct rockchip_clk_branch rk3308_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "clk_i2s0_8ch_rx_frac", "clk_i2s0_8ch_rx_src", CLK_SET_RATE_PARENT, + RK3308_CLKSEL_CON(55), 0, + RK3308_CLKGATE_CON(11), 1, GFLAGS, +- &rk3308_i2s0_8ch_rx_fracmux), ++ &rk3308_i2s0_8ch_rx_fracmux, RK3308_I2S_FRAC_MAX_PRATE), + COMPOSITE_NODIV(SCLK_I2S0_8CH_RX, "clk_i2s0_8ch_rx", mux_i2s0_8ch_rx_tx_p, CLK_SET_RATE_PARENT, + RK3308_CLKSEL_CON(54), 12, 1, MFLAGS, + RK3308_CLKGATE_CON(11), 2, GFLAGS), +@@ -670,7 +677,7 @@ static struct rockchip_clk_branch rk3308_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "clk_i2s1_8ch_tx_frac", "clk_i2s1_8ch_tx_src", CLK_SET_RATE_PARENT, + RK3308_CLKSEL_CON(57), 0, + RK3308_CLKGATE_CON(11), 5, GFLAGS, +- &rk3308_i2s1_8ch_tx_fracmux), ++ &rk3308_i2s1_8ch_tx_fracmux, RK3308_I2S_FRAC_MAX_PRATE), + COMPOSITE_NODIV(SCLK_I2S1_8CH_TX, "clk_i2s1_8ch_tx", mux_i2s1_8ch_tx_rx_p, CLK_SET_RATE_PARENT, + RK3308_CLKSEL_CON(56), 12, 1, MFLAGS, + RK3308_CLKGATE_CON(11), 6, GFLAGS), +@@ -684,7 +691,7 @@ static struct rockchip_clk_branch rk3308_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "clk_i2s1_8ch_rx_frac", "clk_i2s1_8ch_rx_src", CLK_SET_RATE_PARENT, + RK3308_CLKSEL_CON(59), 0, + RK3308_CLKGATE_CON(11), 9, GFLAGS, +- &rk3308_i2s1_8ch_rx_fracmux), ++ &rk3308_i2s1_8ch_rx_fracmux, RK3308_I2S_FRAC_MAX_PRATE), + COMPOSITE_NODIV(SCLK_I2S1_8CH_RX, "clk_i2s1_8ch_rx", mux_i2s1_8ch_rx_tx_p, CLK_SET_RATE_PARENT, + RK3308_CLKSEL_CON(58), 12, 1, MFLAGS, + RK3308_CLKGATE_CON(11), 10, GFLAGS), +@@ -697,7 +704,7 @@ static struct rockchip_clk_branch rk3308_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "clk_i2s2_8ch_tx_frac", "clk_i2s2_8ch_tx_src", CLK_SET_RATE_PARENT, + RK3308_CLKSEL_CON(61), 0, + RK3308_CLKGATE_CON(11), 13, GFLAGS, +- &rk3308_i2s2_8ch_tx_fracmux), ++ &rk3308_i2s2_8ch_tx_fracmux, RK3308_I2S_FRAC_MAX_PRATE), + COMPOSITE_NODIV(SCLK_I2S2_8CH_TX, "clk_i2s2_8ch_tx", mux_i2s2_8ch_tx_rx_p, CLK_SET_RATE_PARENT, + RK3308_CLKSEL_CON(60), 12, 1, MFLAGS, + RK3308_CLKGATE_CON(11), 14, GFLAGS), +@@ -711,7 +718,7 @@ static struct rockchip_clk_branch rk3308_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "clk_i2s2_8ch_rx_frac", "clk_i2s2_8ch_rx_src", CLK_SET_RATE_PARENT, + RK3308_CLKSEL_CON(63), 0, + RK3308_CLKGATE_CON(12), 1, GFLAGS, +- &rk3308_i2s2_8ch_rx_fracmux), ++ &rk3308_i2s2_8ch_rx_fracmux, RK3308_I2S_FRAC_MAX_PRATE), + COMPOSITE_NODIV(SCLK_I2S2_8CH_RX, "clk_i2s2_8ch_rx", mux_i2s2_8ch_rx_tx_p, CLK_SET_RATE_PARENT, + RK3308_CLKSEL_CON(62), 12, 1, MFLAGS, + RK3308_CLKGATE_CON(12), 2, GFLAGS), +@@ -724,7 +731,7 @@ static struct rockchip_clk_branch rk3308_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "clk_i2s3_8ch_tx_frac", "clk_i2s3_8ch_tx_src", CLK_SET_RATE_PARENT, + RK3308_CLKSEL_CON(65), 0, + RK3308_CLKGATE_CON(12), 5, GFLAGS, +- &rk3308_i2s3_8ch_tx_fracmux), ++ &rk3308_i2s3_8ch_tx_fracmux, RK3308_I2S_FRAC_MAX_PRATE), + COMPOSITE_NODIV(SCLK_I2S3_8CH_TX, "clk_i2s3_8ch_tx", mux_i2s3_8ch_tx_rx_p, CLK_SET_RATE_PARENT, + RK3308_CLKSEL_CON(64), 12, 1, MFLAGS, + RK3308_CLKGATE_CON(12), 6, GFLAGS), +@@ -738,7 +745,7 @@ static struct rockchip_clk_branch rk3308_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "clk_i2s3_8ch_rx_frac", "clk_i2s3_8ch_rx_src", CLK_SET_RATE_PARENT, + RK3308_CLKSEL_CON(67), 0, + RK3308_CLKGATE_CON(12), 9, GFLAGS, +- &rk3308_i2s3_8ch_rx_fracmux), ++ &rk3308_i2s3_8ch_rx_fracmux, RK3308_I2S_FRAC_MAX_PRATE), + COMPOSITE_NODIV(SCLK_I2S3_8CH_RX, "clk_i2s3_8ch_rx", mux_i2s3_8ch_rx_tx_p, CLK_SET_RATE_PARENT, + RK3308_CLKSEL_CON(66), 12, 1, MFLAGS, + RK3308_CLKGATE_CON(12), 10, GFLAGS), +@@ -751,7 +758,7 @@ static struct rockchip_clk_branch rk3308_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "clk_i2s0_2ch_frac", "clk_i2s0_2ch_src", CLK_SET_RATE_PARENT, + RK3308_CLKSEL_CON(69), 0, + RK3308_CLKGATE_CON(12), 13, GFLAGS, +- &rk3308_i2s0_2ch_fracmux), ++ &rk3308_i2s0_2ch_fracmux, RK3308_I2S_FRAC_MAX_PRATE), + GATE(SCLK_I2S0_2CH, "clk_i2s0_2ch", "clk_i2s0_2ch_mux", 0, + RK3308_CLKGATE_CON(12), 14, GFLAGS), + COMPOSITE_NODIV(SCLK_I2S0_2CH_OUT, "clk_i2s0_2ch_out", mux_i2s0_2ch_out_p, CLK_SET_RATE_PARENT, +@@ -764,7 +771,7 @@ static struct rockchip_clk_branch rk3308_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "clk_i2s1_2ch_frac", "clk_i2s1_2ch_src", CLK_SET_RATE_PARENT, + RK3308_CLKSEL_CON(71), 0, + RK3308_CLKGATE_CON(13), 1, GFLAGS, +- &rk3308_i2s1_2ch_fracmux), ++ &rk3308_i2s1_2ch_fracmux, RK3308_I2S_FRAC_MAX_PRATE), + GATE(SCLK_I2S1_2CH, "clk_i2s1_2ch", "clk_i2s1_2ch_mux", 0, + RK3308_CLKGATE_CON(13), 2, GFLAGS), + COMPOSITE_NODIV(SCLK_I2S1_2CH_OUT, "clk_i2s1_2ch_out", mux_i2s1_2ch_out_p, CLK_SET_RATE_PARENT, +@@ -782,7 +789,7 @@ static struct rockchip_clk_branch rk3308_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "clk_spdif_tx_frac", "clk_spdif_tx_src", CLK_SET_RATE_PARENT, + RK3308_CLKSEL_CON(49), 0, + RK3308_CLKGATE_CON(10), 7, GFLAGS, +- &rk3308_spdif_tx_fracmux), ++ &rk3308_spdif_tx_fracmux, RK3308_SPDIF_FRAC_MAX_PRATE), + GATE(SCLK_SPDIF_TX, "clk_spdif_tx", "clk_spdif_tx_mux", 0, + RK3308_CLKGATE_CON(10), 8, GFLAGS), + +@@ -797,7 +804,7 @@ static struct rockchip_clk_branch rk3308_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "clk_spdif_rx_frac", "clk_spdif_rx_src", CLK_SET_RATE_PARENT, + RK3308_CLKSEL_CON(51), 0, + RK3308_CLKGATE_CON(10), 10, GFLAGS, +- &rk3308_spdif_rx_fracmux), ++ &rk3308_spdif_rx_fracmux, RK3308_SPDIF_FRAC_MAX_PRATE), + GATE(SCLK_SPDIF_RX, "clk_spdif_rx", "clk_spdif_rx_mux", 0, + RK3308_CLKGATE_CON(10), 11, GFLAGS), + +@@ -900,22 +907,37 @@ static struct rockchip_clk_branch rk3308_clk_branches[] __initdata = { + GATE(PCLK_OWIRE, "pclk_owire", "pclk_bus", CLK_IGNORE_UNUSED, RK3308_CLKGATE_CON(7), 15, GFLAGS), + }; + +-static const char *const rk3308_critical_clocks[] __initconst = { +- "aclk_bus", +- "hclk_bus", +- "pclk_bus", +- "aclk_peri", +- "hclk_peri", +- "pclk_peri", +- "hclk_audio", +- "pclk_audio", +- "sclk_ddrc", ++static struct rockchip_clk_branch rk3308_dclk_vop_frac[] __initdata = { ++ COMPOSITE_FRACMUX(0, "dclk_vop_frac", "dclk_vop_src", CLK_SET_RATE_PARENT, ++ RK3308_CLKSEL_CON(9), 0, ++ RK3308_CLKGATE_CON(1), 7, GFLAGS, ++ &rk3308_dclk_vop_fracmux, RK3308_VOP_FRAC_MAX_PRATE), ++}; ++ ++static struct rockchip_clk_branch rk3308b_dclk_vop_frac[] __initdata = { ++ COMPOSITE_FRACMUX(0, "dclk_vop_frac", "dclk_vop_src", CLK_SET_RATE_PARENT, ++ RK3308_CLKSEL_CON(9), 0, ++ RK3308_CLKGATE_CON(1), 7, GFLAGS, ++ &rk3308_dclk_vop_fracmux, RK3308B_VOP_FRAC_MAX_PRATE), + }; + ++static void __iomem *rk3308_cru_base; ++ ++void rk3308_dump_cru(void) ++{ ++ if (rk3308_cru_base) { ++ pr_warn("CRU:\n"); ++ print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET, ++ 32, 4, rk3308_cru_base, ++ 0x500, false); ++ } ++} ++ + static void __init rk3308_clk_init(struct device_node *np) + { + struct rockchip_clk_provider *ctx; + void __iomem *reg_base; ++ struct clk **clks; + + reg_base = of_iomap(np, 0); + if (!reg_base) { +@@ -929,17 +951,22 @@ static void __init rk3308_clk_init(struct device_node *np) + iounmap(reg_base); + return; + } ++ clks = ctx->clk_data.clks; + + rockchip_clk_register_plls(ctx, rk3308_pll_clks, + ARRAY_SIZE(rk3308_pll_clks), + RK3308_GRF_SOC_STATUS0); + rockchip_clk_register_branches(ctx, rk3308_clk_branches, + ARRAY_SIZE(rk3308_clk_branches)); +- rockchip_clk_protect_critical(rk3308_critical_clocks, +- ARRAY_SIZE(rk3308_critical_clocks)); ++ if (soc_is_rk3308b()) ++ rockchip_clk_register_branches(ctx, rk3308b_dclk_vop_frac, ++ ARRAY_SIZE(rk3308b_dclk_vop_frac)); ++ else ++ rockchip_clk_register_branches(ctx, rk3308_dclk_vop_frac, ++ ARRAY_SIZE(rk3308_dclk_vop_frac)); + + rockchip_clk_register_armclk(ctx, ARMCLK, "armclk", +- mux_armclk_p, ARRAY_SIZE(mux_armclk_p), ++ 3, clks[PLL_APLL], clks[PLL_VPLL0], + &rk3308_cpuclk_data, rk3308_cpuclk_rates, + ARRAY_SIZE(rk3308_cpuclk_rates)); + +@@ -949,6 +976,39 @@ static void __init rk3308_clk_init(struct device_node *np) + rockchip_register_restart_notifier(ctx, RK3308_GLB_SRST_FST, NULL); + + rockchip_clk_of_add_provider(np, ctx); ++ ++ if (!rk_dump_cru) { ++ rk3308_cru_base = reg_base; ++ rk_dump_cru = rk3308_dump_cru; ++ } + } + + CLK_OF_DECLARE(rk3308_cru, "rockchip,rk3308-cru", rk3308_clk_init); ++ ++static int __init clk_rk3308_probe(struct platform_device *pdev) ++{ ++ struct device_node *np = pdev->dev.of_node; ++ ++ rk3308_clk_init(np); ++ ++ return 0; ++} ++ ++static const struct of_device_id clk_rk3308_match_table[] = { ++ { ++ .compatible = "rockchip,rk3308-cru", ++ }, ++ { } ++}; ++MODULE_DEVICE_TABLE(of, clk_rk3308_match_table); ++ ++static struct platform_driver clk_rk3308_driver = { ++ .driver = { ++ .name = "clk-rk3308", ++ .of_match_table = clk_rk3308_match_table, ++ }, ++}; ++builtin_platform_driver_probe(clk_rk3308_driver, clk_rk3308_probe); ++ ++MODULE_DESCRIPTION("Rockchip RK3308 Clock Driver"); ++MODULE_LICENSE("GPL"); +diff --git a/drivers/clk/rockchip/clk-rk3328.c b/drivers/clk/rockchip/clk-rk3328.c +index 2429b7c2a8b3..b8064dd74bd6 100644 +--- a/drivers/clk/rockchip/clk-rk3328.c ++++ b/drivers/clk/rockchip/clk-rk3328.c +@@ -6,8 +6,10 @@ + + #include + #include ++#include + #include + #include ++#include + #include + #include + #include "clk.h" +@@ -16,6 +18,9 @@ + #define RK3328_GRF_SOC_STATUS0 0x480 + #define RK3328_GRF_MAC_CON1 0x904 + #define RK3328_GRF_MAC_CON2 0x908 ++#define RK3328_I2S_FRAC_MAX_PRATE 600000000 ++#define RK3328_UART_FRAC_MAX_PRATE 600000000 ++#define RK3328_SPDIF_FRAC_MAX_PRATE 600000000 + + enum rk3328_plls { + apll, dpll, cpll, gpll, npll, +@@ -130,9 +135,10 @@ static struct rockchip_cpuclk_rate_table rk3328_cpuclk_rates[] __initdata = { + }; + + static const struct rockchip_cpuclk_reg_data rk3328_cpuclk_data = { +- .core_reg = RK3328_CLKSEL_CON(0), +- .div_core_shift = 0, +- .div_core_mask = 0x1f, ++ .core_reg[0] = RK3328_CLKSEL_CON(0), ++ .div_core_shift[0] = 0, ++ .div_core_mask[0] = 0x1f, ++ .num_cores = 1, + .mux_core_alt = 1, + .mux_core_main = 3, + .mux_core_shift = 6, +@@ -290,18 +296,18 @@ static struct rockchip_clk_branch rk3328_clk_branches[] __initdata = { + RK3328_CLKGATE_CON(0), 1, GFLAGS), + GATE(0, "npll_core", "npll", CLK_IGNORE_UNUSED, + RK3328_CLKGATE_CON(0), 12, GFLAGS), +- COMPOSITE_NOMUX(0, "pclk_dbg", "armclk", CLK_IGNORE_UNUSED, ++ COMPOSITE_NOMUX(0, "pclk_dbg", "armclk", CLK_IS_CRITICAL, + RK3328_CLKSEL_CON(1), 0, 4, DFLAGS | CLK_DIVIDER_READ_ONLY, + RK3328_CLKGATE_CON(7), 0, GFLAGS), +- COMPOSITE_NOMUX(0, "aclk_core", "armclk", CLK_IGNORE_UNUSED, ++ COMPOSITE_NOMUX(0, "aclk_core", "armclk", CLK_IS_CRITICAL, + RK3328_CLKSEL_CON(1), 4, 3, DFLAGS | CLK_DIVIDER_READ_ONLY, + RK3328_CLKGATE_CON(7), 1, GFLAGS), +- GATE(0, "aclk_core_niu", "aclk_core", 0, ++ GATE(0, "aclk_core_niu", "aclk_core", CLK_IS_CRITICAL, + RK3328_CLKGATE_CON(13), 0, GFLAGS), +- GATE(0, "aclk_gic400", "aclk_core", CLK_IGNORE_UNUSED, ++ GATE(0, "aclk_gic400", "aclk_core", CLK_IS_CRITICAL, + RK3328_CLKGATE_CON(13), 1, GFLAGS), + +- GATE(0, "clk_jtag", "jtag_clkin", CLK_IGNORE_UNUSED, ++ GATE(0, "clk_jtag", "jtag_clkin", CLK_IS_CRITICAL, + RK3328_CLKGATE_CON(7), 2, GFLAGS), + + /* PD_GPU */ +@@ -310,34 +316,34 @@ static struct rockchip_clk_branch rk3328_clk_branches[] __initdata = { + RK3328_CLKGATE_CON(6), 6, GFLAGS), + GATE(ACLK_GPU, "aclk_gpu", "aclk_gpu_pre", CLK_SET_RATE_PARENT, + RK3328_CLKGATE_CON(14), 0, GFLAGS), +- GATE(0, "aclk_gpu_niu", "aclk_gpu_pre", 0, ++ GATE(0, "aclk_gpu_niu", "aclk_gpu_pre", CLK_IS_CRITICAL, + RK3328_CLKGATE_CON(14), 1, GFLAGS), + + /* PD_DDR */ +- COMPOSITE(0, "clk_ddr", mux_ddrphy_p, CLK_IGNORE_UNUSED, ++ COMPOSITE(0, "clk_ddr", mux_ddrphy_p, CLK_IS_CRITICAL, + RK3328_CLKSEL_CON(3), 8, 2, MFLAGS, 0, 3, DFLAGS | CLK_DIVIDER_POWER_OF_TWO, + RK3328_CLKGATE_CON(0), 4, GFLAGS), +- GATE(0, "clk_ddrmsch", "clk_ddr", CLK_IGNORE_UNUSED, ++ GATE(0, "clk_ddrmsch", "clk_ddr", CLK_IS_CRITICAL, + RK3328_CLKGATE_CON(18), 6, GFLAGS), +- GATE(0, "clk_ddrupctl", "clk_ddr", CLK_IGNORE_UNUSED, ++ GATE(0, "clk_ddrupctl", "clk_ddr", CLK_IS_CRITICAL, + RK3328_CLKGATE_CON(18), 5, GFLAGS), + GATE(0, "aclk_ddrupctl", "clk_ddr", CLK_IGNORE_UNUSED, + RK3328_CLKGATE_CON(18), 4, GFLAGS), + GATE(0, "clk_ddrmon", "xin24m", CLK_IGNORE_UNUSED, + RK3328_CLKGATE_CON(0), 6, GFLAGS), + +- COMPOSITE(PCLK_DDR, "pclk_ddr", mux_2plls_hdmiphy_p, 0, ++ COMPOSITE(PCLK_DDR, "pclk_ddr", mux_2plls_hdmiphy_p, CLK_IS_CRITICAL, + RK3328_CLKSEL_CON(4), 13, 2, MFLAGS, 8, 3, DFLAGS, + RK3328_CLKGATE_CON(7), 4, GFLAGS), +- GATE(0, "pclk_ddrupctl", "pclk_ddr", CLK_IGNORE_UNUSED, ++ GATE(0, "pclk_ddrupctl", "pclk_ddr", CLK_IS_CRITICAL, + RK3328_CLKGATE_CON(18), 1, GFLAGS), +- GATE(0, "pclk_ddr_msch", "pclk_ddr", CLK_IGNORE_UNUSED, ++ GATE(0, "pclk_ddr_msch", "pclk_ddr", CLK_IS_CRITICAL, + RK3328_CLKGATE_CON(18), 2, GFLAGS), +- GATE(0, "pclk_ddr_mon", "pclk_ddr", CLK_IGNORE_UNUSED, ++ GATE(0, "pclk_ddr_mon", "pclk_ddr", CLK_IS_CRITICAL, + RK3328_CLKGATE_CON(18), 3, GFLAGS), + GATE(0, "pclk_ddrstdby", "pclk_ddr", CLK_IGNORE_UNUSED, + RK3328_CLKGATE_CON(18), 7, GFLAGS), +- GATE(0, "pclk_ddr_grf", "pclk_ddr", CLK_IGNORE_UNUSED, ++ GATE(0, "pclk_ddr_grf", "pclk_ddr", CLK_IS_CRITICAL, + RK3328_CLKGATE_CON(18), 9, GFLAGS), + + /* +@@ -345,18 +351,18 @@ static struct rockchip_clk_branch rk3328_clk_branches[] __initdata = { + */ + + /* PD_BUS */ +- COMPOSITE(ACLK_BUS_PRE, "aclk_bus_pre", mux_2plls_hdmiphy_p, 0, ++ COMPOSITE(ACLK_BUS_PRE, "aclk_bus_pre", mux_2plls_hdmiphy_p, CLK_IS_CRITICAL, + RK3328_CLKSEL_CON(0), 13, 2, MFLAGS, 8, 5, DFLAGS, + RK3328_CLKGATE_CON(8), 0, GFLAGS), +- COMPOSITE_NOMUX(HCLK_BUS_PRE, "hclk_bus_pre", "aclk_bus_pre", 0, ++ COMPOSITE_NOMUX(HCLK_BUS_PRE, "hclk_bus_pre", "aclk_bus_pre", CLK_IS_CRITICAL, + RK3328_CLKSEL_CON(1), 8, 2, DFLAGS, + RK3328_CLKGATE_CON(8), 1, GFLAGS), +- COMPOSITE_NOMUX(PCLK_BUS_PRE, "pclk_bus_pre", "aclk_bus_pre", 0, ++ COMPOSITE_NOMUX(PCLK_BUS_PRE, "pclk_bus_pre", "aclk_bus_pre", CLK_IS_CRITICAL, + RK3328_CLKSEL_CON(1), 12, 3, DFLAGS, + RK3328_CLKGATE_CON(8), 2, GFLAGS), +- GATE(0, "pclk_bus", "pclk_bus_pre", 0, ++ GATE(0, "pclk_bus", "pclk_bus_pre", CLK_IS_CRITICAL, + RK3328_CLKGATE_CON(8), 3, GFLAGS), +- GATE(0, "pclk_phy_pre", "pclk_bus_pre", 0, ++ GATE(0, "pclk_phy_pre", "pclk_bus_pre", CLK_IS_CRITICAL, + RK3328_CLKGATE_CON(8), 4, GFLAGS), + + COMPOSITE(SCLK_TSP, "clk_tsp", mux_2plls_p, 0, +@@ -372,7 +378,7 @@ static struct rockchip_clk_branch rk3328_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "clk_i2s0_frac", "clk_i2s0_div", CLK_SET_RATE_PARENT, + RK3328_CLKSEL_CON(7), 0, + RK3328_CLKGATE_CON(1), 2, GFLAGS, +- &rk3328_i2s0_fracmux), ++ &rk3328_i2s0_fracmux, RK3328_I2S_FRAC_MAX_PRATE), + GATE(SCLK_I2S0, "clk_i2s0", "i2s0_pre", CLK_SET_RATE_PARENT, + RK3328_CLKGATE_CON(1), 3, GFLAGS), + +@@ -382,7 +388,7 @@ static struct rockchip_clk_branch rk3328_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "clk_i2s1_frac", "clk_i2s1_div", CLK_SET_RATE_PARENT, + RK3328_CLKSEL_CON(9), 0, + RK3328_CLKGATE_CON(1), 5, GFLAGS, +- &rk3328_i2s1_fracmux), ++ &rk3328_i2s1_fracmux, RK3328_I2S_FRAC_MAX_PRATE), + GATE(SCLK_I2S1, "clk_i2s1", "i2s1_pre", CLK_SET_RATE_PARENT, + RK3328_CLKGATE_CON(1), 6, GFLAGS), + COMPOSITE_NODIV(SCLK_I2S1_OUT, "i2s1_out", mux_i2s1out_p, 0, +@@ -395,7 +401,7 @@ static struct rockchip_clk_branch rk3328_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "clk_i2s2_frac", "clk_i2s2_div", CLK_SET_RATE_PARENT, + RK3328_CLKSEL_CON(11), 0, + RK3328_CLKGATE_CON(1), 9, GFLAGS, +- &rk3328_i2s2_fracmux), ++ &rk3328_i2s2_fracmux, RK3328_I2S_FRAC_MAX_PRATE), + GATE(SCLK_I2S2, "clk_i2s2", "i2s2_pre", CLK_SET_RATE_PARENT, + RK3328_CLKGATE_CON(1), 10, GFLAGS), + COMPOSITE_NODIV(SCLK_I2S2_OUT, "i2s2_out", mux_i2s2out_p, 0, +@@ -408,7 +414,7 @@ static struct rockchip_clk_branch rk3328_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "clk_spdif_frac", "clk_spdif_div", CLK_SET_RATE_PARENT, + RK3328_CLKSEL_CON(13), 0, + RK3328_CLKGATE_CON(1), 13, GFLAGS, +- &rk3328_spdif_fracmux), ++ &rk3328_spdif_fracmux, RK3328_SPDIF_FRAC_MAX_PRATE), + + /* PD_UART */ + COMPOSITE(0, "clk_uart0_div", mux_2plls_u480m_p, 0, +@@ -423,15 +429,15 @@ static struct rockchip_clk_branch rk3328_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "clk_uart0_frac", "clk_uart0_div", CLK_SET_RATE_PARENT, + RK3328_CLKSEL_CON(15), 0, + RK3328_CLKGATE_CON(1), 15, GFLAGS, +- &rk3328_uart0_fracmux), ++ &rk3328_uart0_fracmux, RK3328_UART_FRAC_MAX_PRATE), + COMPOSITE_FRACMUX(0, "clk_uart1_frac", "clk_uart1_div", CLK_SET_RATE_PARENT, + RK3328_CLKSEL_CON(17), 0, + RK3328_CLKGATE_CON(2), 1, GFLAGS, +- &rk3328_uart1_fracmux), ++ &rk3328_uart1_fracmux, RK3328_UART_FRAC_MAX_PRATE), + COMPOSITE_FRACMUX(0, "clk_uart2_frac", "clk_uart2_div", CLK_SET_RATE_PARENT, + RK3328_CLKSEL_CON(19), 0, + RK3328_CLKGATE_CON(2), 3, GFLAGS, +- &rk3328_uart2_fracmux), ++ &rk3328_uart2_fracmux, RK3328_UART_FRAC_MAX_PRATE), + + /* + * Clock-Architecture Diagram 4 +@@ -505,9 +511,9 @@ static struct rockchip_clk_branch rk3328_clk_branches[] __initdata = { + RK3328_CLKGATE_CON(24), 0, GFLAGS), + GATE(HCLK_RKVDEC, "hclk_rkvdec", "hclk_rkvdec_pre", CLK_SET_RATE_PARENT, + RK3328_CLKGATE_CON(24), 1, GFLAGS), +- GATE(0, "aclk_rkvdec_niu", "aclk_rkvdec_pre", 0, ++ GATE(0, "aclk_rkvdec_niu", "aclk_rkvdec_pre", CLK_IS_CRITICAL, + RK3328_CLKGATE_CON(24), 2, GFLAGS), +- GATE(0, "hclk_rkvdec_niu", "hclk_rkvdec_pre", 0, ++ GATE(0, "hclk_rkvdec_niu", "hclk_rkvdec_pre", CLK_IS_CRITICAL, + RK3328_CLKGATE_CON(24), 3, GFLAGS), + + COMPOSITE(SCLK_VDEC_CABAC, "sclk_vdec_cabac", mux_4plls_p, 0, +@@ -527,35 +533,36 @@ static struct rockchip_clk_branch rk3328_clk_branches[] __initdata = { + RK3328_CLKGATE_CON(23), 0, GFLAGS), + GATE(HCLK_VPU, "hclk_vpu", "hclk_vpu_pre", CLK_SET_RATE_PARENT, + RK3328_CLKGATE_CON(23), 1, GFLAGS), +- GATE(0, "aclk_vpu_niu", "aclk_vpu_pre", 0, ++ GATE(0, "aclk_vpu_niu", "aclk_vpu_pre", CLK_IS_CRITICAL, + RK3328_CLKGATE_CON(23), 2, GFLAGS), +- GATE(0, "hclk_vpu_niu", "hclk_vpu_pre", 0, ++ GATE(0, "hclk_vpu_niu", "hclk_vpu_pre", CLK_IS_CRITICAL, + RK3328_CLKGATE_CON(23), 3, GFLAGS), + + COMPOSITE(ACLK_RKVENC, "aclk_rkvenc", mux_4plls_p, 0, + RK3328_CLKSEL_CON(51), 6, 2, MFLAGS, 0, 5, DFLAGS, + RK3328_CLKGATE_CON(6), 3, GFLAGS), +- FACTOR_GATE(HCLK_RKVENC, "hclk_rkvenc", "aclk_rkvenc", 0, 1, 4, ++ ++ COMPOSITE(SCLK_VENC_CORE, "sclk_venc_core", mux_4plls_p, 0, ++ RK3328_CLKSEL_CON(51), 14, 2, MFLAGS, 8, 5, DFLAGS, ++ RK3328_CLKGATE_CON(6), 4, GFLAGS), ++ FACTOR_GATE(0, "hclk_venc", "sclk_venc_core", 0, 1, 4, + RK3328_CLKGATE_CON(11), 4, GFLAGS), +- GATE(0, "aclk_rkvenc_niu", "aclk_rkvenc", 0, ++ ++ GATE(0, "aclk_rkvenc_niu", "sclk_venc_core", CLK_IS_CRITICAL, + RK3328_CLKGATE_CON(25), 0, GFLAGS), +- GATE(0, "hclk_rkvenc_niu", "hclk_rkvenc", 0, ++ GATE(0, "hclk_rkvenc_niu", "hclk_venc", CLK_IS_CRITICAL, + RK3328_CLKGATE_CON(25), 1, GFLAGS), +- GATE(ACLK_H265, "aclk_h265", "aclk_rkvenc", 0, ++ GATE(ACLK_H265, "aclk_h265", "sclk_venc_core", 0, + RK3328_CLKGATE_CON(25), 2, GFLAGS), +- GATE(PCLK_H265, "pclk_h265", "hclk_rkvenc", 0, ++ GATE(PCLK_H265, "pclk_h265", "hclk_venc", 0, + RK3328_CLKGATE_CON(25), 3, GFLAGS), +- GATE(ACLK_H264, "aclk_h264", "aclk_rkvenc", 0, ++ GATE(ACLK_H264, "aclk_h264", "sclk_venc_core", 0, + RK3328_CLKGATE_CON(25), 4, GFLAGS), +- GATE(HCLK_H264, "hclk_h264", "hclk_rkvenc", 0, ++ GATE(HCLK_H264, "hclk_h264", "hclk_venc", 0, + RK3328_CLKGATE_CON(25), 5, GFLAGS), +- GATE(ACLK_AXISRAM, "aclk_axisram", "aclk_rkvenc", CLK_IGNORE_UNUSED, ++ GATE(ACLK_AXISRAM, "aclk_axisram", "sclk_venc_core", CLK_IGNORE_UNUSED, + RK3328_CLKGATE_CON(25), 6, GFLAGS), + +- COMPOSITE(SCLK_VENC_CORE, "sclk_venc_core", mux_4plls_p, 0, +- RK3328_CLKSEL_CON(51), 14, 2, MFLAGS, 8, 5, DFLAGS, +- RK3328_CLKGATE_CON(6), 4, GFLAGS), +- + COMPOSITE(SCLK_VENC_DSP, "sclk_venc_dsp", mux_4plls_p, 0, + RK3328_CLKSEL_CON(52), 14, 2, MFLAGS, 8, 5, DFLAGS, + RK3328_CLKGATE_CON(6), 7, GFLAGS), +@@ -602,21 +609,21 @@ static struct rockchip_clk_branch rk3328_clk_branches[] __initdata = { + */ + + /* PD_PERI */ +- GATE(0, "gpll_peri", "gpll", CLK_IGNORE_UNUSED, ++ GATE(0, "gpll_peri", "gpll", CLK_IS_CRITICAL, + RK3328_CLKGATE_CON(4), 0, GFLAGS), +- GATE(0, "cpll_peri", "cpll", CLK_IGNORE_UNUSED, ++ GATE(0, "cpll_peri", "cpll", CLK_IS_CRITICAL, + RK3328_CLKGATE_CON(4), 1, GFLAGS), +- GATE(0, "hdmiphy_peri", "hdmiphy", CLK_IGNORE_UNUSED, ++ GATE(0, "hdmiphy_peri", "hdmiphy", CLK_IS_CRITICAL, + RK3328_CLKGATE_CON(4), 2, GFLAGS), +- COMPOSITE_NOGATE(ACLK_PERI_PRE, "aclk_peri_pre", mux_aclk_peri_pre_p, 0, ++ COMPOSITE_NOGATE(ACLK_PERI_PRE, "aclk_peri_pre", mux_aclk_peri_pre_p, CLK_IS_CRITICAL, + RK3328_CLKSEL_CON(28), 6, 2, MFLAGS, 0, 5, DFLAGS), +- COMPOSITE_NOMUX(PCLK_PERI, "pclk_peri", "aclk_peri_pre", CLK_IGNORE_UNUSED, ++ COMPOSITE_NOMUX(PCLK_PERI, "pclk_peri", "aclk_peri_pre", CLK_IS_CRITICAL, + RK3328_CLKSEL_CON(29), 0, 2, DFLAGS, + RK3328_CLKGATE_CON(10), 2, GFLAGS), +- COMPOSITE_NOMUX(HCLK_PERI, "hclk_peri", "aclk_peri_pre", CLK_IGNORE_UNUSED, ++ COMPOSITE_NOMUX(HCLK_PERI, "hclk_peri", "aclk_peri_pre", CLK_IS_CRITICAL, + RK3328_CLKSEL_CON(29), 4, 3, DFLAGS, + RK3328_CLKGATE_CON(10), 1, GFLAGS), +- GATE(ACLK_PERI, "aclk_peri", "aclk_peri_pre", CLK_IGNORE_UNUSED | CLK_SET_RATE_PARENT, ++ GATE(ACLK_PERI, "aclk_peri", "aclk_peri_pre", CLK_IS_CRITICAL | CLK_SET_RATE_PARENT, + RK3328_CLKGATE_CON(10), 0, GFLAGS), + + COMPOSITE(SCLK_SDMMC, "clk_sdmmc", mux_2plls_24m_u480m_p, 0, +@@ -701,30 +708,30 @@ static struct rockchip_clk_branch rk3328_clk_branches[] __initdata = { + + /* PD_VOP */ + GATE(ACLK_RGA, "aclk_rga", "aclk_rga_pre", 0, RK3328_CLKGATE_CON(21), 10, GFLAGS), +- GATE(0, "aclk_rga_niu", "aclk_rga_pre", 0, RK3328_CLKGATE_CON(22), 3, GFLAGS), ++ GATE(0, "aclk_rga_niu", "aclk_rga_pre", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(22), 3, GFLAGS), + GATE(ACLK_VOP, "aclk_vop", "aclk_vop_pre", 0, RK3328_CLKGATE_CON(21), 2, GFLAGS), +- GATE(0, "aclk_vop_niu", "aclk_vop_pre", 0, RK3328_CLKGATE_CON(21), 4, GFLAGS), ++ GATE(0, "aclk_vop_niu", "aclk_vop_pre", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(21), 4, GFLAGS), + + GATE(ACLK_IEP, "aclk_iep", "aclk_vio_pre", 0, RK3328_CLKGATE_CON(21), 6, GFLAGS), + GATE(ACLK_CIF, "aclk_cif", "aclk_vio_pre", 0, RK3328_CLKGATE_CON(21), 8, GFLAGS), + GATE(ACLK_HDCP, "aclk_hdcp", "aclk_vio_pre", 0, RK3328_CLKGATE_CON(21), 15, GFLAGS), +- GATE(0, "aclk_vio_niu", "aclk_vio_pre", 0, RK3328_CLKGATE_CON(22), 2, GFLAGS), ++ GATE(0, "aclk_vio_niu", "aclk_vio_pre", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(22), 2, GFLAGS), + + GATE(HCLK_VOP, "hclk_vop", "hclk_vio_pre", 0, RK3328_CLKGATE_CON(21), 3, GFLAGS), +- GATE(0, "hclk_vop_niu", "hclk_vio_pre", 0, RK3328_CLKGATE_CON(21), 5, GFLAGS), ++ GATE(0, "hclk_vop_niu", "hclk_vio_pre", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(21), 5, GFLAGS), + GATE(HCLK_IEP, "hclk_iep", "hclk_vio_pre", 0, RK3328_CLKGATE_CON(21), 7, GFLAGS), + GATE(HCLK_CIF, "hclk_cif", "hclk_vio_pre", 0, RK3328_CLKGATE_CON(21), 9, GFLAGS), + GATE(HCLK_RGA, "hclk_rga", "hclk_vio_pre", 0, RK3328_CLKGATE_CON(21), 11, GFLAGS), +- GATE(0, "hclk_ahb1tom", "hclk_vio_pre", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(21), 12, GFLAGS), +- GATE(0, "pclk_vio_h2p", "hclk_vio_pre", 0, RK3328_CLKGATE_CON(21), 13, GFLAGS), +- GATE(0, "hclk_vio_h2p", "hclk_vio_pre", 0, RK3328_CLKGATE_CON(21), 14, GFLAGS), ++ GATE(0, "hclk_ahb1tom", "hclk_vio_pre", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(21), 12, GFLAGS), ++ GATE(0, "pclk_vio_h2p", "hclk_vio_pre", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(21), 13, GFLAGS), ++ GATE(0, "hclk_vio_h2p", "hclk_vio_pre", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(21), 14, GFLAGS), + GATE(HCLK_HDCP, "hclk_hdcp", "hclk_vio_pre", 0, RK3328_CLKGATE_CON(22), 0, GFLAGS), +- GATE(0, "hclk_vio_niu", "hclk_vio_pre", 0, RK3328_CLKGATE_CON(22), 1, GFLAGS), ++ GATE(0, "hclk_vio_niu", "hclk_vio_pre", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(22), 1, GFLAGS), + GATE(PCLK_HDMI, "pclk_hdmi", "hclk_vio_pre", 0, RK3328_CLKGATE_CON(22), 4, GFLAGS), + GATE(PCLK_HDCP, "pclk_hdcp", "hclk_vio_pre", 0, RK3328_CLKGATE_CON(22), 5, GFLAGS), + + /* PD_PERI */ +- GATE(0, "aclk_peri_noc", "aclk_peri", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(19), 11, GFLAGS), ++ GATE(0, "aclk_peri_noc", "aclk_peri", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(19), 11, GFLAGS), + GATE(ACLK_USB3OTG, "aclk_usb3otg", "aclk_peri", 0, RK3328_CLKGATE_CON(19), 14, GFLAGS), + + GATE(HCLK_SDMMC, "hclk_sdmmc", "hclk_peri", 0, RK3328_CLKGATE_CON(19), 0, GFLAGS), +@@ -734,26 +741,26 @@ static struct rockchip_clk_branch rk3328_clk_branches[] __initdata = { + GATE(HCLK_HOST0, "hclk_host0", "hclk_peri", 0, RK3328_CLKGATE_CON(19), 6, GFLAGS), + GATE(HCLK_HOST0_ARB, "hclk_host0_arb", "hclk_peri", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(19), 7, GFLAGS), + GATE(HCLK_OTG, "hclk_otg", "hclk_peri", 0, RK3328_CLKGATE_CON(19), 8, GFLAGS), +- GATE(HCLK_OTG_PMU, "hclk_otg_pmu", "hclk_peri", 0, RK3328_CLKGATE_CON(19), 9, GFLAGS), +- GATE(0, "hclk_peri_niu", "hclk_peri", 0, RK3328_CLKGATE_CON(19), 12, GFLAGS), +- GATE(0, "pclk_peri_niu", "hclk_peri", 0, RK3328_CLKGATE_CON(19), 13, GFLAGS), ++ GATE(HCLK_OTG_PMU, "hclk_otg_pmu", "hclk_peri", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(19), 9, GFLAGS), ++ GATE(0, "hclk_peri_niu", "hclk_peri", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(19), 12, GFLAGS), ++ GATE(0, "pclk_peri_niu", "hclk_peri", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(19), 13, GFLAGS), + + /* PD_GMAC */ + GATE(ACLK_MAC2PHY, "aclk_mac2phy", "aclk_gmac", 0, RK3328_CLKGATE_CON(26), 0, GFLAGS), + GATE(ACLK_MAC2IO, "aclk_mac2io", "aclk_gmac", 0, RK3328_CLKGATE_CON(26), 2, GFLAGS), +- GATE(0, "aclk_gmac_niu", "aclk_gmac", 0, RK3328_CLKGATE_CON(26), 4, GFLAGS), ++ GATE(0, "aclk_gmac_niu", "aclk_gmac", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(26), 4, GFLAGS), + GATE(PCLK_MAC2PHY, "pclk_mac2phy", "pclk_gmac", 0, RK3328_CLKGATE_CON(26), 1, GFLAGS), + GATE(PCLK_MAC2IO, "pclk_mac2io", "pclk_gmac", 0, RK3328_CLKGATE_CON(26), 3, GFLAGS), +- GATE(0, "pclk_gmac_niu", "pclk_gmac", 0, RK3328_CLKGATE_CON(26), 5, GFLAGS), ++ GATE(0, "pclk_gmac_niu", "pclk_gmac", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(26), 5, GFLAGS), + + /* PD_BUS */ +- GATE(0, "aclk_bus_niu", "aclk_bus_pre", 0, RK3328_CLKGATE_CON(15), 12, GFLAGS), ++ GATE(0, "aclk_bus_niu", "aclk_bus_pre", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(15), 12, GFLAGS), + GATE(ACLK_DCF, "aclk_dcf", "aclk_bus_pre", 0, RK3328_CLKGATE_CON(15), 11, GFLAGS), + GATE(ACLK_TSP, "aclk_tsp", "aclk_bus_pre", 0, RK3328_CLKGATE_CON(17), 12, GFLAGS), +- GATE(0, "aclk_intmem", "aclk_bus_pre", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(15), 0, GFLAGS), ++ GATE(0, "aclk_intmem", "aclk_bus_pre", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(15), 0, GFLAGS), + GATE(ACLK_DMAC, "aclk_dmac_bus", "aclk_bus_pre", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(15), 1, GFLAGS), + +- GATE(0, "hclk_rom", "hclk_bus_pre", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(15), 2, GFLAGS), ++ GATE(0, "hclk_rom", "hclk_bus_pre", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(15), 2, GFLAGS), + GATE(HCLK_I2S0_8CH, "hclk_i2s0_8ch", "hclk_bus_pre", 0, RK3328_CLKGATE_CON(15), 3, GFLAGS), + GATE(HCLK_I2S1_8CH, "hclk_i2s1_8ch", "hclk_bus_pre", 0, RK3328_CLKGATE_CON(15), 4, GFLAGS), + GATE(HCLK_I2S2_2CH, "hclk_i2s2_2ch", "hclk_bus_pre", 0, RK3328_CLKGATE_CON(15), 5, GFLAGS), +@@ -761,17 +768,17 @@ static struct rockchip_clk_branch rk3328_clk_branches[] __initdata = { + GATE(HCLK_TSP, "hclk_tsp", "hclk_bus_pre", 0, RK3328_CLKGATE_CON(17), 11, GFLAGS), + GATE(HCLK_CRYPTO_MST, "hclk_crypto_mst", "hclk_bus_pre", 0, RK3328_CLKGATE_CON(15), 7, GFLAGS), + GATE(HCLK_CRYPTO_SLV, "hclk_crypto_slv", "hclk_bus_pre", 0, RK3328_CLKGATE_CON(15), 8, GFLAGS), +- GATE(0, "hclk_bus_niu", "hclk_bus_pre", 0, RK3328_CLKGATE_CON(15), 13, GFLAGS), ++ GATE(0, "hclk_bus_niu", "hclk_bus_pre", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(15), 13, GFLAGS), + GATE(HCLK_PDM, "hclk_pdm", "hclk_bus_pre", 0, RK3328_CLKGATE_CON(28), 0, GFLAGS), + +- GATE(0, "pclk_bus_niu", "pclk_bus", 0, RK3328_CLKGATE_CON(15), 14, GFLAGS), ++ GATE(0, "pclk_bus_niu", "pclk_bus", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(15), 14, GFLAGS), + GATE(0, "pclk_efuse", "pclk_bus", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(15), 9, GFLAGS), + GATE(0, "pclk_otp", "pclk_bus", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(28), 4, GFLAGS), + GATE(PCLK_I2C0, "pclk_i2c0", "pclk_bus", 0, RK3328_CLKGATE_CON(15), 10, GFLAGS), + GATE(PCLK_I2C1, "pclk_i2c1", "pclk_bus", 0, RK3328_CLKGATE_CON(16), 0, GFLAGS), + GATE(PCLK_I2C2, "pclk_i2c2", "pclk_bus", 0, RK3328_CLKGATE_CON(16), 1, GFLAGS), + GATE(PCLK_I2C3, "pclk_i2c3", "pclk_bus", 0, RK3328_CLKGATE_CON(16), 2, GFLAGS), +- GATE(PCLK_TIMER, "pclk_timer0", "pclk_bus", 0, RK3328_CLKGATE_CON(16), 3, GFLAGS), ++ GATE(PCLK_TIMER, "pclk_timer0", "pclk_bus", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(16), 3, GFLAGS), + GATE(0, "pclk_stimer", "pclk_bus", 0, RK3328_CLKGATE_CON(16), 4, GFLAGS), + GATE(PCLK_SPI, "pclk_spi", "pclk_bus", 0, RK3328_CLKGATE_CON(16), 5, GFLAGS), + GATE(PCLK_PWM, "pclk_rk_pwm", "pclk_bus", 0, RK3328_CLKGATE_CON(16), 6, GFLAGS), +@@ -784,12 +791,12 @@ static struct rockchip_clk_branch rk3328_clk_branches[] __initdata = { + GATE(PCLK_UART2, "pclk_uart2", "pclk_bus", 0, RK3328_CLKGATE_CON(16), 13, GFLAGS), + GATE(PCLK_TSADC, "pclk_tsadc", "pclk_bus", 0, RK3328_CLKGATE_CON(16), 14, GFLAGS), + GATE(PCLK_DCF, "pclk_dcf", "pclk_bus", 0, RK3328_CLKGATE_CON(16), 15, GFLAGS), +- GATE(PCLK_GRF, "pclk_grf", "pclk_bus", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(17), 0, GFLAGS), +- GATE(0, "pclk_cru", "pclk_bus", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(17), 4, GFLAGS), +- GATE(0, "pclk_sgrf", "pclk_bus", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(17), 6, GFLAGS), ++ GATE(PCLK_GRF, "pclk_grf", "pclk_bus", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(17), 0, GFLAGS), ++ GATE(0, "pclk_cru", "pclk_bus", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(17), 4, GFLAGS), ++ GATE(0, "pclk_sgrf", "pclk_bus", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(17), 6, GFLAGS), + GATE(0, "pclk_sim", "pclk_bus", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(17), 10, GFLAGS), + GATE(PCLK_SARADC, "pclk_saradc", "pclk_bus", 0, RK3328_CLKGATE_CON(17), 15, GFLAGS), +- GATE(0, "pclk_pmu", "pclk_bus", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(28), 3, GFLAGS), ++ GATE(0, "pclk_pmu", "pclk_bus", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(28), 3, GFLAGS), + + /* Watchdog pclk is controlled from the secure GRF */ + SGRF_GATE(PCLK_WDT, "pclk_wdt", "pclk_bus"), +@@ -798,11 +805,11 @@ static struct rockchip_clk_branch rk3328_clk_branches[] __initdata = { + GATE(PCLK_USB3PHY_PIPE, "pclk_usb3phy_pipe", "pclk_phy_pre", 0, RK3328_CLKGATE_CON(28), 2, GFLAGS), + GATE(PCLK_USB3_GRF, "pclk_usb3_grf", "pclk_phy_pre", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(17), 2, GFLAGS), + GATE(PCLK_USB2_GRF, "pclk_usb2_grf", "pclk_phy_pre", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(17), 14, GFLAGS), +- GATE(0, "pclk_ddrphy", "pclk_phy_pre", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(17), 13, GFLAGS), ++ GATE(0, "pclk_ddrphy", "pclk_phy_pre", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(17), 13, GFLAGS), + GATE(PCLK_ACODECPHY, "pclk_acodecphy", "pclk_phy_pre", 0, RK3328_CLKGATE_CON(17), 5, GFLAGS), + GATE(PCLK_HDMIPHY, "pclk_hdmiphy", "pclk_phy_pre", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(17), 7, GFLAGS), + GATE(0, "pclk_vdacphy", "pclk_phy_pre", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(17), 8, GFLAGS), +- GATE(0, "pclk_phy_niu", "pclk_phy_pre", 0, RK3328_CLKGATE_CON(15), 15, GFLAGS), ++ GATE(0, "pclk_phy_niu", "pclk_phy_pre", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(15), 15, GFLAGS), + + /* PD_MMC */ + MMC(SCLK_SDMMC_DRV, "sdmmc_drv", "clk_sdmmc", +@@ -826,61 +833,11 @@ static struct rockchip_clk_branch rk3328_clk_branches[] __initdata = { + RK3328_SDMMC_EXT_CON1, 1), + }; + +-static const char *const rk3328_critical_clocks[] __initconst = { +- "aclk_bus", +- "aclk_bus_niu", +- "pclk_bus", +- "pclk_bus_niu", +- "hclk_bus", +- "hclk_bus_niu", +- "aclk_peri", +- "hclk_peri", +- "hclk_peri_niu", +- "pclk_peri", +- "pclk_peri_niu", +- "pclk_dbg", +- "aclk_core_niu", +- "aclk_gic400", +- "aclk_intmem", +- "hclk_rom", +- "pclk_grf", +- "pclk_cru", +- "pclk_sgrf", +- "pclk_timer0", +- "clk_timer0", +- "pclk_ddr_msch", +- "pclk_ddr_mon", +- "pclk_ddr_grf", +- "clk_ddrupctl", +- "clk_ddrmsch", +- "hclk_ahb1tom", +- "clk_jtag", +- "pclk_ddrphy", +- "pclk_pmu", +- "hclk_otg_pmu", +- "aclk_rga_niu", +- "pclk_vio_h2p", +- "hclk_vio_h2p", +- "aclk_vio_niu", +- "hclk_vio_niu", +- "aclk_vop_niu", +- "hclk_vop_niu", +- "aclk_gpu_niu", +- "aclk_rkvdec_niu", +- "hclk_rkvdec_niu", +- "aclk_vpu_niu", +- "hclk_vpu_niu", +- "aclk_rkvenc_niu", +- "hclk_rkvenc_niu", +- "aclk_gmac_niu", +- "pclk_gmac_niu", +- "pclk_phy_niu", +-}; +- + static void __init rk3328_clk_init(struct device_node *np) + { + struct rockchip_clk_provider *ctx; + void __iomem *reg_base; ++ struct clk **clks; + + reg_base = of_iomap(np, 0); + if (!reg_base) { +@@ -894,17 +851,16 @@ static void __init rk3328_clk_init(struct device_node *np) + iounmap(reg_base); + return; + } ++ clks = ctx->clk_data.clks; + + rockchip_clk_register_plls(ctx, rk3328_pll_clks, + ARRAY_SIZE(rk3328_pll_clks), + RK3328_GRF_SOC_STATUS0); + rockchip_clk_register_branches(ctx, rk3328_clk_branches, + ARRAY_SIZE(rk3328_clk_branches)); +- rockchip_clk_protect_critical(rk3328_critical_clocks, +- ARRAY_SIZE(rk3328_critical_clocks)); + + rockchip_clk_register_armclk(ctx, ARMCLK, "armclk", +- mux_armclk_p, ARRAY_SIZE(mux_armclk_p), ++ 4, clks[PLL_APLL], clks[PLL_GPLL], + &rk3328_cpuclk_data, rk3328_cpuclk_rates, + ARRAY_SIZE(rk3328_cpuclk_rates)); + +@@ -916,3 +872,31 @@ static void __init rk3328_clk_init(struct device_node *np) + rockchip_clk_of_add_provider(np, ctx); + } + CLK_OF_DECLARE(rk3328_cru, "rockchip,rk3328-cru", rk3328_clk_init); ++ ++static int __init clk_rk3328_probe(struct platform_device *pdev) ++{ ++ struct device_node *np = pdev->dev.of_node; ++ ++ rk3328_clk_init(np); ++ ++ return 0; ++} ++ ++static const struct of_device_id clk_rk3328_match_table[] = { ++ { ++ .compatible = "rockchip,rk3328-cru", ++ }, ++ { } ++}; ++MODULE_DEVICE_TABLE(of, clk_rk3328_match_table); ++ ++static struct platform_driver clk_rk3328_driver = { ++ .driver = { ++ .name = "clk-rk3328", ++ .of_match_table = clk_rk3328_match_table, ++ }, ++}; ++builtin_platform_driver_probe(clk_rk3328_driver, clk_rk3328_probe); ++ ++MODULE_DESCRIPTION("Rockchip RK3328 Clock Driver"); ++MODULE_LICENSE("GPL"); +diff --git a/drivers/clk/rockchip/clk-rk3368.c b/drivers/clk/rockchip/clk-rk3368.c +index 55443349439b..4b4513a26740 100644 +--- a/drivers/clk/rockchip/clk-rk3368.c ++++ b/drivers/clk/rockchip/clk-rk3368.c +@@ -5,13 +5,19 @@ + + #include + #include ++#include + #include + #include ++#include + #include + #include + #include "clk.h" + + #define RK3368_GRF_SOC_STATUS0 0x480 ++#define RK3368_I2S_FRAC_MAX_PRATE 600000000 ++#define RK3368_UART_FRAC_MAX_PRATE 600000000 ++#define RK3368_SPDIF_FRAC_MAX_PRATE 600000000 ++#define RK3368_DCLK_PARENT_MAX_PRATE 600000000 + + enum rk3368_plls { + apllb, aplll, dpll, cpll, gpll, npll, +@@ -87,23 +93,40 @@ static struct rockchip_pll_rate_table rk3368_pll_rates[] = { + { /* sentinel */ }, + }; + ++static struct rockchip_pll_rate_table rk3368_npll_rates[] = { ++ RK3066_PLL_RATE_NB(594000000, 1, 99, 4, 32), ++ RK3066_PLL_RATE_NB(585000000, 6, 585, 4, 32), ++ RK3066_PLL_RATE_NB(432000000, 3, 216, 4, 32), ++ RK3066_PLL_RATE_NB(426000000, 3, 213, 4, 32), ++ RK3066_PLL_RATE_NB(400000000, 1, 100, 6, 32), ++ RK3066_PLL_RATE_NB(342000000, 3, 171, 4, 32), ++ RK3066_PLL_RATE_NB(297000000, 2, 198, 8, 16), ++ RK3066_PLL_RATE_NB(270000000, 1, 135, 12, 32), ++ RK3066_PLL_RATE_NB(260000000, 1, 130, 12, 32), ++ RK3066_PLL_RATE_NB(148500000, 1, 99, 16, 32), ++ RK3066_PLL_RATE_NB(146250000, 6, 585, 16, 32), ++ RK3066_PLL_RATE_NB(108000000, 1, 54, 12, 32), ++ RK3066_PLL_RATE_NB(106500000, 4, 213, 12, 32), ++ RK3066_PLL_RATE_NB(85500000, 4, 171, 12, 32), ++ RK3066_PLL_RATE_NB(74250000, 4, 198, 16, 32), ++}; ++ + PNAME(mux_pll_p) = { "xin24m", "xin32k" }; +-PNAME(mux_armclkb_p) = { "apllb_core", "gpllb_core" }; +-PNAME(mux_armclkl_p) = { "aplll_core", "gplll_core" }; + PNAME(mux_ddrphy_p) = { "dpll_ddr", "gpll_ddr" }; + PNAME(mux_cs_src_p) = { "apllb_cs", "aplll_cs", "gpll_cs"}; + PNAME(mux_aclk_bus_src_p) = { "cpll_aclk_bus", "gpll_aclk_bus" }; + + PNAME(mux_pll_src_cpll_gpll_p) = { "cpll", "gpll" }; +-PNAME(mux_pll_src_cpll_gpll_npll_p) = { "cpll", "gpll", "npll" }; +-PNAME(mux_pll_src_npll_cpll_gpll_p) = { "npll", "cpll", "gpll" }; ++PNAME(mux_pll_src_cpll_gpll_npll_p) = { "cpll", "gpll", "dummy_npll" }; ++PNAME(mux_pll_src_dmycpll_dmygpll_npll_p) = { "dummy_cpll", "dummy_gpll", "npll" }; ++PNAME(mux_pll_src_npll_cpll_gpll_p) = { "dummy_npll", "cpll", "gpll" }; + PNAME(mux_pll_src_cpll_gpll_usb_p) = { "cpll", "gpll", "usbphy_480m" }; + PNAME(mux_pll_src_cpll_gpll_usb_usb_p) = { "cpll", "gpll", "usbphy_480m", + "usbphy_480m" }; + PNAME(mux_pll_src_cpll_gpll_usb_npll_p) = { "cpll", "gpll", "usbphy_480m", +- "npll" }; +-PNAME(mux_pll_src_cpll_gpll_npll_npll_p) = { "cpll", "gpll", "npll", "npll" }; +-PNAME(mux_pll_src_cpll_gpll_npll_usb_p) = { "cpll", "gpll", "npll", ++ "dummy_npll" }; ++PNAME(mux_pll_src_cpll_gpll_npll_npll_p) = { "cpll", "gpll", "dummy_npll", "dummy_npll" }; ++PNAME(mux_pll_src_cpll_gpll_npll_usb_p) = { "cpll", "gpll", "dummy_npll", + "usbphy_480m" }; + + PNAME(mux_i2s_8ch_pre_p) = { "i2s_8ch_src", "i2s_8ch_frac", +@@ -138,7 +161,7 @@ static struct rockchip_pll_clock rk3368_pll_clks[] __initdata = { + [gpll] = PLL(pll_rk3066, PLL_GPLL, "gpll", mux_pll_p, 0, RK3368_PLL_CON(16), + RK3368_PLL_CON(19), 8, 4, ROCKCHIP_PLL_SYNC_RATE, rk3368_pll_rates), + [npll] = PLL(pll_rk3066, PLL_NPLL, "npll", mux_pll_p, 0, RK3368_PLL_CON(20), +- RK3368_PLL_CON(23), 8, 5, ROCKCHIP_PLL_SYNC_RATE, rk3368_pll_rates), ++ RK3368_PLL_CON(23), 8, 5, 0, rk3368_npll_rates), + }; + + static struct clk_div_table div_ddrphy_t[] = { +@@ -154,9 +177,10 @@ static struct clk_div_table div_ddrphy_t[] = { + #define IFLAGS ROCKCHIP_INVERTER_HIWORD_MASK + + static const struct rockchip_cpuclk_reg_data rk3368_cpuclkb_data = { +- .core_reg = RK3368_CLKSEL_CON(0), +- .div_core_shift = 0, +- .div_core_mask = 0x1f, ++ .core_reg[0] = RK3368_CLKSEL_CON(0), ++ .div_core_shift[0] = 0, ++ .div_core_mask[0] = 0x1f, ++ .num_cores = 1, + .mux_core_alt = 1, + .mux_core_main = 0, + .mux_core_shift = 7, +@@ -164,11 +188,12 @@ static const struct rockchip_cpuclk_reg_data rk3368_cpuclkb_data = { + }; + + static const struct rockchip_cpuclk_reg_data rk3368_cpuclkl_data = { +- .core_reg = RK3368_CLKSEL_CON(2), +- .div_core_shift = 0, ++ .core_reg[0] = RK3368_CLKSEL_CON(2), ++ .div_core_shift[0] = 0, + .mux_core_alt = 1, ++ .num_cores = 1, + .mux_core_main = 0, +- .div_core_mask = 0x1f, ++ .div_core_mask[0] = 0x1f, + .mux_core_shift = 7, + .mux_core_mask = 0x1, + }; +@@ -315,8 +340,8 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { + RK3368_CLKSEL_CON(4), 8, 5, DFLAGS, + RK3368_CLKGATE_CON(0), 13, GFLAGS), + +- COMPOSITE(0, "aclk_cci_pre", mux_pll_src_cpll_gpll_usb_npll_p, CLK_IGNORE_UNUSED, +- RK3368_CLKSEL_CON(5), 6, 2, MFLAGS, 0, 7, DFLAGS, ++ COMPOSITE(ACLK_CCI_PRE, "aclk_cci_pre", mux_pll_src_cpll_gpll_usb_npll_p, CLK_IGNORE_UNUSED, ++ RK3368_CLKSEL_CON(5), 6, 2, MFLAGS, 0, 5, DFLAGS, + RK3368_CLKGATE_CON(0), 12, GFLAGS), + GATE(SCLK_PVTM_CORE, "sclk_pvtm_core", "xin24m", 0, RK3368_CLKGATE_CON(7), 10, GFLAGS), + +@@ -332,19 +357,19 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { + GATE(0, "sclk_ddr4x", "ddrphy_src", CLK_IGNORE_UNUSED, + RK3368_CLKGATE_CON(6), 15, GFLAGS), + +- GATE(0, "gpll_aclk_bus", "gpll", CLK_IGNORE_UNUSED, ++ GATE(0, "gpll_aclk_bus", "gpll", CLK_IS_CRITICAL, + RK3368_CLKGATE_CON(1), 10, GFLAGS), +- GATE(0, "cpll_aclk_bus", "cpll", CLK_IGNORE_UNUSED, ++ GATE(0, "cpll_aclk_bus", "cpll", CLK_IS_CRITICAL, + RK3368_CLKGATE_CON(1), 11, GFLAGS), +- COMPOSITE_NOGATE(0, "aclk_bus_src", mux_aclk_bus_src_p, CLK_IGNORE_UNUSED, ++ COMPOSITE_NOGATE(0, "aclk_bus_src", mux_aclk_bus_src_p, CLK_IS_CRITICAL, + RK3368_CLKSEL_CON(8), 7, 1, MFLAGS, 0, 5, DFLAGS), + +- GATE(ACLK_BUS, "aclk_bus", "aclk_bus_src", CLK_IGNORE_UNUSED, ++ GATE(ACLK_BUS, "aclk_bus", "aclk_bus_src", CLK_IS_CRITICAL, + RK3368_CLKGATE_CON(1), 0, GFLAGS), +- COMPOSITE_NOMUX(PCLK_BUS, "pclk_bus", "aclk_bus_src", CLK_IGNORE_UNUSED, ++ COMPOSITE_NOMUX(PCLK_BUS, "pclk_bus", "aclk_bus_src", CLK_IS_CRITICAL, + RK3368_CLKSEL_CON(8), 12, 3, DFLAGS, + RK3368_CLKGATE_CON(1), 2, GFLAGS), +- COMPOSITE_NOMUX(HCLK_BUS, "hclk_bus", "aclk_bus_src", CLK_IGNORE_UNUSED, ++ COMPOSITE_NOMUX(HCLK_BUS, "hclk_bus", "aclk_bus_src", CLK_IS_CRITICAL, + RK3368_CLKSEL_CON(8), 8, 2, DFLAGS, + RK3368_CLKGATE_CON(1), 1, GFLAGS), + COMPOSITE_NOMUX(0, "sclk_crypto", "aclk_bus_src", 0, +@@ -358,7 +383,7 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { + * stclk_mcu is listed as child of fclk_mcu_src in diagram 5, + * but stclk_mcu has an additional own divider in diagram 2 + */ +- COMPOSITE_NOMUX(0, "stclk_mcu", "fclk_mcu_src", 0, ++ COMPOSITE_NOMUX(0, "stclk_mcu", "fclk_mcu_src", CLK_IGNORE_UNUSED, + RK3368_CLKSEL_CON(12), 8, 3, DFLAGS, + RK3368_CLKGATE_CON(13), 13, GFLAGS), + +@@ -368,7 +393,7 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "i2s_8ch_frac", "i2s_8ch_src", CLK_SET_RATE_PARENT, + RK3368_CLKSEL_CON(28), 0, + RK3368_CLKGATE_CON(6), 2, GFLAGS, +- &rk3368_i2s_8ch_fracmux), ++ &rk3368_i2s_8ch_fracmux, RK3368_I2S_FRAC_MAX_PRATE), + COMPOSITE_NODIV(SCLK_I2S_8CH_OUT, "i2s_8ch_clkout", mux_i2s_8ch_clkout_p, 0, + RK3368_CLKSEL_CON(27), 15, 1, MFLAGS, + RK3368_CLKGATE_CON(6), 0, GFLAGS), +@@ -380,7 +405,7 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "spdif_8ch_frac", "spdif_8ch_src", CLK_SET_RATE_PARENT, + RK3368_CLKSEL_CON(32), 0, + RK3368_CLKGATE_CON(6), 5, GFLAGS, +- &rk3368_spdif_8ch_fracmux), ++ &rk3368_spdif_8ch_fracmux, RK3368_SPDIF_FRAC_MAX_PRATE), + GATE(SCLK_SPDIF_8CH, "sclk_spdif_8ch", "spdif_8ch_pre", CLK_SET_RATE_PARENT, + RK3368_CLKGATE_CON(6), 6, GFLAGS), + COMPOSITE(0, "i2s_2ch_src", mux_pll_src_cpll_gpll_p, 0, +@@ -389,7 +414,7 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "i2s_2ch_frac", "i2s_2ch_src", CLK_SET_RATE_PARENT, + RK3368_CLKSEL_CON(54), 0, + RK3368_CLKGATE_CON(5), 14, GFLAGS, +- &rk3368_i2s_2ch_fracmux), ++ &rk3368_i2s_2ch_fracmux, RK3368_I2S_FRAC_MAX_PRATE), + GATE(SCLK_I2S_2CH, "sclk_i2s_2ch", "i2s_2ch_pre", CLK_SET_RATE_PARENT, + RK3368_CLKGATE_CON(5), 15, GFLAGS), + +@@ -445,9 +470,9 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { + RK3368_CLKSEL_CON(18), 6, 2, MFLAGS, 0, 5, DFLAGS, + RK3368_CLKGATE_CON(4), 4, GFLAGS), + +- COMPOSITE(DCLK_VOP, "dclk_vop", mux_pll_src_cpll_gpll_npll_p, 0, ++ COMPOSITE_DCLK(DCLK_VOP, "dclk_vop", mux_pll_src_dmycpll_dmygpll_npll_p, CLK_SET_RATE_PARENT, + RK3368_CLKSEL_CON(20), 8, 2, MFLAGS, 0, 8, DFLAGS, +- RK3368_CLKGATE_CON(4), 1, GFLAGS), ++ RK3368_CLKGATE_CON(4), 1, GFLAGS, RK3368_DCLK_PARENT_MAX_PRATE), + + GATE(SCLK_VOP0_PWM, "sclk_vop0_pwm", "xin24m", 0, + RK3368_CLKGATE_CON(4), 2, GFLAGS), +@@ -488,12 +513,12 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { + RK3368_CLKSEL_CON(55), 6, 2, MFLAGS, 0, 6, DFLAGS, + RK3368_CLKGATE_CON(5), 5, GFLAGS), + +- DIV(0, "pclk_pd_alive", "gpll", 0, ++ DIV(0, "pclk_pd_alive", "gpll", CLK_IS_CRITICAL, + RK3368_CLKSEL_CON(10), 8, 5, DFLAGS), + + /* sclk_timer has a gate in the sgrf */ + +- COMPOSITE_NOMUX(0, "pclk_pd_pmu", "gpll", CLK_IGNORE_UNUSED, ++ COMPOSITE_NOMUX(0, "pclk_pd_pmu", "gpll", CLK_IS_CRITICAL, + RK3368_CLKSEL_CON(10), 0, 5, DFLAGS, + RK3368_CLKGATE_CON(7), 9, GFLAGS), + GATE(SCLK_PVTM_PMU, "sclk_pvtm_pmu", "xin24m", 0, +@@ -512,16 +537,16 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { + GATE(SCLK_PVTM_GPU, "sclk_pvtm_gpu", "xin24m", 0, + RK3368_CLKGATE_CON(7), 11, GFLAGS), + +- COMPOSITE(0, "aclk_peri_src", mux_pll_src_cpll_gpll_p, CLK_IGNORE_UNUSED, ++ COMPOSITE(0, "aclk_peri_src", mux_pll_src_cpll_gpll_p, CLK_IS_CRITICAL, + RK3368_CLKSEL_CON(9), 7, 1, MFLAGS, 0, 5, DFLAGS, + RK3368_CLKGATE_CON(3), 0, GFLAGS), +- COMPOSITE_NOMUX(PCLK_PERI, "pclk_peri", "aclk_peri_src", 0, ++ COMPOSITE_NOMUX(PCLK_PERI, "pclk_peri", "aclk_peri_src", CLK_IS_CRITICAL, + RK3368_CLKSEL_CON(9), 12, 2, DFLAGS | CLK_DIVIDER_POWER_OF_TWO, + RK3368_CLKGATE_CON(3), 3, GFLAGS), +- COMPOSITE_NOMUX(HCLK_PERI, "hclk_peri", "aclk_peri_src", CLK_IGNORE_UNUSED, ++ COMPOSITE_NOMUX(HCLK_PERI, "hclk_peri", "aclk_peri_src", CLK_IS_CRITICAL, + RK3368_CLKSEL_CON(9), 8, 2, DFLAGS | CLK_DIVIDER_POWER_OF_TWO, + RK3368_CLKGATE_CON(3), 2, GFLAGS), +- GATE(ACLK_PERI, "aclk_peri", "aclk_peri_src", CLK_IGNORE_UNUSED, ++ GATE(ACLK_PERI, "aclk_peri", "aclk_peri_src", CLK_IS_CRITICAL, + RK3368_CLKGATE_CON(3), 1, GFLAGS), + + GATE(0, "sclk_mipidsi_24m", "xin24m", 0, RK3368_CLKGATE_CON(4), 14, GFLAGS), +@@ -590,7 +615,7 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "uart0_frac", "uart0_src", CLK_SET_RATE_PARENT, + RK3368_CLKSEL_CON(34), 0, + RK3368_CLKGATE_CON(2), 1, GFLAGS, +- &rk3368_uart0_fracmux), ++ &rk3368_uart0_fracmux, RK3368_UART_FRAC_MAX_PRATE), + + COMPOSITE_NOMUX(0, "uart1_src", "uart_src", 0, + RK3368_CLKSEL_CON(35), 0, 7, DFLAGS, +@@ -598,7 +623,7 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "uart1_frac", "uart1_src", CLK_SET_RATE_PARENT, + RK3368_CLKSEL_CON(36), 0, + RK3368_CLKGATE_CON(2), 3, GFLAGS, +- &rk3368_uart1_fracmux), ++ &rk3368_uart1_fracmux, RK3368_UART_FRAC_MAX_PRATE), + + COMPOSITE_NOMUX(0, "uart3_src", "uart_src", 0, + RK3368_CLKSEL_CON(39), 0, 7, DFLAGS, +@@ -606,7 +631,7 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "uart3_frac", "uart3_src", CLK_SET_RATE_PARENT, + RK3368_CLKSEL_CON(40), 0, + RK3368_CLKGATE_CON(2), 7, GFLAGS, +- &rk3368_uart3_fracmux), ++ &rk3368_uart3_fracmux, RK3368_UART_FRAC_MAX_PRATE), + + COMPOSITE_NOMUX(0, "uart4_src", "uart_src", 0, + RK3368_CLKSEL_CON(41), 0, 7, DFLAGS, +@@ -614,7 +639,7 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "uart4_frac", "uart4_src", CLK_SET_RATE_PARENT, + RK3368_CLKSEL_CON(42), 0, + RK3368_CLKGATE_CON(2), 9, GFLAGS, +- &rk3368_uart4_fracmux), ++ &rk3368_uart4_fracmux, RK3368_UART_FRAC_MAX_PRATE), + + COMPOSITE(0, "mac_pll_src", mux_pll_src_npll_cpll_gpll_p, 0, + RK3368_CLKSEL_CON(43), 6, 2, MFLAGS, 0, 5, DFLAGS, +@@ -668,7 +693,7 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { + + /* aclk_bus gates */ + GATE(0, "aclk_strc_sys", "aclk_bus", CLK_IGNORE_UNUSED, RK3368_CLKGATE_CON(12), 12, GFLAGS), +- GATE(ACLK_DMAC_BUS, "aclk_dmac_bus", "aclk_bus", 0, RK3368_CLKGATE_CON(12), 11, GFLAGS), ++ GATE(ACLK_DMAC_BUS, "aclk_dmac_bus", "aclk_bus", CLK_IS_CRITICAL, RK3368_CLKGATE_CON(12), 11, GFLAGS), + GATE(0, "sclk_intmem1", "aclk_bus", CLK_IGNORE_UNUSED, RK3368_CLKGATE_CON(12), 6, GFLAGS), + GATE(0, "sclk_intmem0", "aclk_bus", CLK_IGNORE_UNUSED, RK3368_CLKGATE_CON(12), 5, GFLAGS), + GATE(0, "aclk_intmem", "aclk_bus", CLK_IGNORE_UNUSED, RK3368_CLKGATE_CON(12), 4, GFLAGS), +@@ -680,9 +705,9 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { + /* clk_hsadc_tsp is part of diagram2 */ + + /* fclk_mcu_src gates */ +- GATE(0, "hclk_noc_mcu", "fclk_mcu_src", 0, RK3368_CLKGATE_CON(13), 14, GFLAGS), +- GATE(0, "fclk_mcu", "fclk_mcu_src", 0, RK3368_CLKGATE_CON(13), 12, GFLAGS), +- GATE(0, "hclk_mcu", "fclk_mcu_src", 0, RK3368_CLKGATE_CON(13), 11, GFLAGS), ++ GATE(0, "hclk_noc_mcu", "fclk_mcu_src", CLK_IGNORE_UNUSED, RK3368_CLKGATE_CON(13), 14, GFLAGS), ++ GATE(0, "fclk_mcu", "fclk_mcu_src", CLK_IGNORE_UNUSED, RK3368_CLKGATE_CON(13), 12, GFLAGS), ++ GATE(0, "hclk_mcu", "fclk_mcu_src", CLK_IGNORE_UNUSED, RK3368_CLKGATE_CON(13), 11, GFLAGS), + + /* hclk_cpu gates */ + GATE(HCLK_SPDIF, "hclk_spdif", "hclk_bus", 0, RK3368_CLKGATE_CON(12), 10, GFLAGS), +@@ -694,14 +719,14 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { + GATE(MCLK_CRYPTO, "mclk_crypto", "hclk_bus", 0, RK3368_CLKGATE_CON(13), 3, GFLAGS), + + /* pclk_cpu gates */ +- GATE(PCLK_DDRPHY, "pclk_ddrphy", "pclk_bus", 0, RK3368_CLKGATE_CON(12), 14, GFLAGS), +- GATE(PCLK_DDRUPCTL, "pclk_ddrupctl", "pclk_bus", 0, RK3368_CLKGATE_CON(12), 13, GFLAGS), ++ GATE(PCLK_DDRPHY, "pclk_ddrphy", "pclk_bus", CLK_IS_CRITICAL, RK3368_CLKGATE_CON(12), 14, GFLAGS), ++ GATE(PCLK_DDRUPCTL, "pclk_ddrupctl", "pclk_bus", CLK_IS_CRITICAL, RK3368_CLKGATE_CON(12), 13, GFLAGS), + GATE(PCLK_I2C1, "pclk_i2c1", "pclk_bus", 0, RK3368_CLKGATE_CON(12), 3, GFLAGS), + GATE(PCLK_I2C0, "pclk_i2c0", "pclk_bus", 0, RK3368_CLKGATE_CON(12), 2, GFLAGS), + GATE(PCLK_MAILBOX, "pclk_mailbox", "pclk_bus", 0, RK3368_CLKGATE_CON(12), 1, GFLAGS), + GATE(PCLK_PWM0, "pclk_pwm0", "pclk_bus", CLK_IGNORE_UNUSED, RK3368_CLKGATE_CON(12), 0, GFLAGS), + GATE(PCLK_SIM, "pclk_sim", "pclk_bus", 0, RK3368_CLKGATE_CON(13), 8, GFLAGS), +- GATE(PCLK_PWM1, "pclk_pwm1", "pclk_bus", 0, RK3368_CLKGATE_CON(13), 6, GFLAGS), ++ GATE(PCLK_PWM1, "pclk_pwm1", "pclk_bus", CLK_IS_CRITICAL, RK3368_CLKGATE_CON(13), 6, GFLAGS), + GATE(PCLK_UART2, "pclk_uart2", "pclk_bus", 0, RK3368_CLKGATE_CON(13), 5, GFLAGS), + GATE(PCLK_EFUSE256, "pclk_efuse_256", "pclk_bus", 0, RK3368_CLKGATE_CON(13), 1, GFLAGS), + GATE(0, "pclk_efuse_1024", "pclk_bus", 0, RK3368_CLKGATE_CON(13), 0, GFLAGS), +@@ -778,7 +803,7 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { + GATE(HCLK_HSIC, "hclk_hsic", "hclk_peri", 0, RK3368_CLKGATE_CON(20), 5, GFLAGS), + GATE(HCLK_HOST1, "hclk_host1", "hclk_peri", CLK_IGNORE_UNUSED, RK3368_CLKGATE_CON(20), 4, GFLAGS), + GATE(HCLK_HOST0, "hclk_host0", "hclk_peri", 0, RK3368_CLKGATE_CON(20), 3, GFLAGS), +- GATE(0, "pmu_hclk_otg0", "hclk_peri", 0, RK3368_CLKGATE_CON(20), 2, GFLAGS), ++ GATE(0, "pmu_hclk_otg0", "hclk_peri", CLK_IS_CRITICAL, RK3368_CLKGATE_CON(20), 2, GFLAGS), + GATE(HCLK_OTG0, "hclk_otg0", "hclk_peri", CLK_IGNORE_UNUSED, RK3368_CLKGATE_CON(20), 1, GFLAGS), + GATE(HCLK_HSADC, "hclk_hsadc", "hclk_peri", 0, RK3368_CLKGATE_CON(21), 3, GFLAGS), + GATE(HCLK_EMMC, "hclk_emmc", "hclk_peri", 0, RK3368_CLKGATE_CON(21), 2, GFLAGS), +@@ -805,8 +830,8 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { + /* pclk_pd_alive gates */ + GATE(PCLK_TIMER1, "pclk_timer1", "pclk_pd_alive", 0, RK3368_CLKGATE_CON(22), 13, GFLAGS), + GATE(PCLK_TIMER0, "pclk_timer0", "pclk_pd_alive", 0, RK3368_CLKGATE_CON(22), 12, GFLAGS), +- GATE(0, "pclk_alive_niu", "pclk_pd_alive", CLK_IGNORE_UNUSED, RK3368_CLKGATE_CON(22), 9, GFLAGS), +- GATE(PCLK_GRF, "pclk_grf", "pclk_pd_alive", CLK_IGNORE_UNUSED, RK3368_CLKGATE_CON(22), 8, GFLAGS), ++ GATE(0, "pclk_alive_niu", "pclk_pd_alive", CLK_IS_CRITICAL, RK3368_CLKGATE_CON(22), 9, GFLAGS), ++ GATE(PCLK_GRF, "pclk_grf", "pclk_pd_alive", CLK_IS_CRITICAL, RK3368_CLKGATE_CON(22), 8, GFLAGS), + GATE(PCLK_GPIO3, "pclk_gpio3", "pclk_pd_alive", 0, RK3368_CLKGATE_CON(22), 3, GFLAGS), + GATE(PCLK_GPIO2, "pclk_gpio2", "pclk_pd_alive", 0, RK3368_CLKGATE_CON(22), 2, GFLAGS), + GATE(PCLK_GPIO1, "pclk_gpio1", "pclk_pd_alive", 0, RK3368_CLKGATE_CON(22), 1, GFLAGS), +@@ -844,27 +869,23 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { + GATE(SCLK_TIMER00, "sclk_timer00", "xin24m", CLK_IGNORE_UNUSED, RK3368_CLKGATE_CON(24), 0, GFLAGS), + }; + +-static const char *const rk3368_critical_clocks[] __initconst = { +- "aclk_bus", +- "aclk_peri", +- /* +- * pwm1 supplies vdd_logic on a lot of boards, is currently unhandled +- * but needs to stay enabled there (including its parents) at all times. +- */ +- "pclk_pwm1", +- "pclk_pd_pmu", +- "pclk_pd_alive", +- "pclk_peri", +- "hclk_peri", +- "pclk_ddrphy", +- "pclk_ddrupctl", +- "pmu_hclk_otg0", +-}; ++static void __iomem *rk3368_cru_base; ++ ++static void rk3368_dump_cru(void) ++{ ++ if (rk3368_cru_base) { ++ pr_warn("CRU:\n"); ++ print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET, ++ 32, 4, rk3368_cru_base, ++ 0x41c, false); ++ } ++} + + static void __init rk3368_clk_init(struct device_node *np) + { + struct rockchip_clk_provider *ctx; + void __iomem *reg_base; ++ struct clk **clks; + + reg_base = of_iomap(np, 0); + if (!reg_base) { +@@ -878,22 +899,21 @@ static void __init rk3368_clk_init(struct device_node *np) + iounmap(reg_base); + return; + } ++ clks = ctx->clk_data.clks; + + rockchip_clk_register_plls(ctx, rk3368_pll_clks, + ARRAY_SIZE(rk3368_pll_clks), + RK3368_GRF_SOC_STATUS0); + rockchip_clk_register_branches(ctx, rk3368_clk_branches, + ARRAY_SIZE(rk3368_clk_branches)); +- rockchip_clk_protect_critical(rk3368_critical_clocks, +- ARRAY_SIZE(rk3368_critical_clocks)); + + rockchip_clk_register_armclk(ctx, ARMCLKB, "armclkb", +- mux_armclkb_p, ARRAY_SIZE(mux_armclkb_p), ++ 2, clks[PLL_APLLB], clks[PLL_GPLL], + &rk3368_cpuclkb_data, rk3368_cpuclkb_rates, + ARRAY_SIZE(rk3368_cpuclkb_rates)); + + rockchip_clk_register_armclk(ctx, ARMCLKL, "armclkl", +- mux_armclkl_p, ARRAY_SIZE(mux_armclkl_p), ++ 2, clks[PLL_APLLL], clks[PLL_GPLL], + &rk3368_cpuclkl_data, rk3368_cpuclkl_rates, + ARRAY_SIZE(rk3368_cpuclkl_rates)); + +@@ -903,5 +923,38 @@ static void __init rk3368_clk_init(struct device_node *np) + rockchip_register_restart_notifier(ctx, RK3368_GLB_SRST_FST, NULL); + + rockchip_clk_of_add_provider(np, ctx); ++ ++ if (!rk_dump_cru) { ++ rk3368_cru_base = reg_base; ++ rk_dump_cru = rk3368_dump_cru; ++ } + } + CLK_OF_DECLARE(rk3368_cru, "rockchip,rk3368-cru", rk3368_clk_init); ++ ++static int __init clk_rk3368_probe(struct platform_device *pdev) ++{ ++ struct device_node *np = pdev->dev.of_node; ++ ++ rk3368_clk_init(np); ++ ++ return 0; ++} ++ ++static const struct of_device_id clk_rk3368_match_table[] = { ++ { ++ .compatible = "rockchip,rk3368-cru", ++ }, ++ { } ++}; ++MODULE_DEVICE_TABLE(of, clk_rk3368_match_table); ++ ++static struct platform_driver clk_rk3368_driver = { ++ .driver = { ++ .name = "clk-rk3368", ++ .of_match_table = clk_rk3368_match_table, ++ }, ++}; ++builtin_platform_driver_probe(clk_rk3368_driver, clk_rk3368_probe); ++ ++MODULE_DESCRIPTION("Rockchip RK3368 Clock Driver"); ++MODULE_LICENSE("GPL"); +diff --git a/drivers/clk/rockchip/clk-rk3399.c b/drivers/clk/rockchip/clk-rk3399.c +index 7df2f1e00347..d5071884b3d5 100644 +--- a/drivers/clk/rockchip/clk-rk3399.c ++++ b/drivers/clk/rockchip/clk-rk3399.c +@@ -15,6 +15,12 @@ + #include + #include "clk.h" + ++#define RK3399_I2S_FRAC_MAX_PRATE 800000000 ++#define RK3399_UART_FRAC_MAX_PRATE 800000000 ++#define RK3399_SPDIF_FRAC_MAX_PRATE 600000000 ++#define RK3399_VOP_FRAC_MAX_PRATE 600000000 ++#define RK3399_WIFI_FRAC_MAX_PRATE 600000000 ++ + enum rk3399_plls { + lpll, bpll, dpll, cpll, gpll, npll, vpll, + }; +@@ -105,25 +111,95 @@ static struct rockchip_pll_rate_table rk3399_pll_rates[] = { + { /* sentinel */ }, + }; + ++static struct rockchip_pll_rate_table rk3399_vpll_rates[] = { ++ /* _mhz, _refdiv, _fbdiv, _postdiv1, _postdiv2, _dsmpd, _frac */ ++ RK3036_PLL_RATE( 594000000, 1, 123, 5, 1, 0, 12582912), /* vco = 2970000000 */ ++ RK3036_PLL_RATE( 593406593, 1, 123, 5, 1, 0, 10508804), /* vco = 2967032965 */ ++ RK3036_PLL_RATE( 297000000, 1, 123, 5, 2, 0, 12582912), /* vco = 2970000000 */ ++ RK3036_PLL_RATE( 296703297, 1, 123, 5, 2, 0, 10508807), /* vco = 2967032970 */ ++ RK3036_PLL_RATE( 148500000, 1, 129, 7, 3, 0, 15728640), /* vco = 3118500000 */ ++ RK3036_PLL_RATE( 148351648, 1, 123, 5, 4, 0, 10508800), /* vco = 2967032960 */ ++ RK3036_PLL_RATE( 106500000, 1, 124, 7, 4, 0, 4194304), /* vco = 2982000000 */ ++ RK3036_PLL_RATE( 74250000, 1, 129, 7, 6, 0, 15728640), /* vco = 3118500000 */ ++ RK3036_PLL_RATE( 74175824, 1, 129, 7, 6, 0, 13550823), /* vco = 3115384608 */ ++ RK3036_PLL_RATE( 65000000, 1, 113, 7, 6, 0, 12582912), /* vco = 2730000000 */ ++ RK3036_PLL_RATE( 59340659, 1, 121, 7, 7, 0, 2581098), /* vco = 2907692291 */ ++ RK3036_PLL_RATE( 54000000, 1, 110, 7, 7, 0, 4194304), /* vco = 2646000000 */ ++ RK3036_PLL_RATE( 27000000, 1, 55, 7, 7, 0, 2097152), /* vco = 1323000000 */ ++ RK3036_PLL_RATE( 26973027, 1, 55, 7, 7, 0, 1173232), /* vco = 1321678323 */ ++ { /* sentinel */ }, ++}; ++ + /* CRU parents */ + PNAME(mux_pll_p) = { "xin24m", "xin32k" }; + +-PNAME(mux_armclkl_p) = { "clk_core_l_lpll_src", +- "clk_core_l_bpll_src", +- "clk_core_l_dpll_src", +- "clk_core_l_gpll_src" }; +-PNAME(mux_armclkb_p) = { "clk_core_b_lpll_src", +- "clk_core_b_bpll_src", +- "clk_core_b_dpll_src", +- "clk_core_b_gpll_src" }; + PNAME(mux_ddrclk_p) = { "clk_ddrc_lpll_src", + "clk_ddrc_bpll_src", + "clk_ddrc_dpll_src", + "clk_ddrc_gpll_src" }; ++ ++PNAME(mux_pll_src_vpll_cpll_gpll_p) = { "vpll", "cpll", "gpll" }; ++PNAME(mux_pll_src_dmyvpll_cpll_gpll_p) = { "dummy_vpll", "cpll", "gpll" }; ++ ++#ifdef RK3399_TWO_PLL_FOR_VOP ++PNAME(mux_aclk_cci_p) = { "dummy_cpll", ++ "gpll_aclk_cci_src", ++ "npll_aclk_cci_src", ++ "dummy_vpll" }; ++PNAME(mux_cci_trace_p) = { "dummy_cpll", ++ "gpll_cci_trace" }; ++PNAME(mux_cs_p) = { "dummy_cpll", "gpll_cs", ++ "npll_cs"}; ++PNAME(mux_aclk_perihp_p) = { "dummy_cpll", ++ "gpll_aclk_perihp_src" }; ++ ++PNAME(mux_pll_src_cpll_gpll_p) = { "dummy_cpll", "gpll" }; ++PNAME(mux_pll_src_cpll_gpll_npll_p) = { "dummy_cpll", "gpll", "npll" }; ++PNAME(mux_pll_src_cpll_gpll_ppll_p) = { "dummy_cpll", "gpll", "ppll" }; ++PNAME(mux_pll_src_cpll_gpll_upll_p) = { "dummy_cpll", "gpll", "upll" }; ++PNAME(mux_pll_src_npll_cpll_gpll_p) = { "npll", "dummy_cpll", "gpll" }; ++PNAME(mux_pll_src_cpll_gpll_npll_ppll_p) = { "dummy_cpll", "gpll", "npll", ++ "ppll" }; ++PNAME(mux_pll_src_cpll_gpll_npll_24m_p) = { "dummy_cpll", "gpll", "npll", ++ "xin24m" }; ++PNAME(mux_pll_src_cpll_gpll_npll_usbphy480m_p) = { "dummy_cpll", "gpll", "npll", ++ "clk_usbphy_480m" }; ++PNAME(mux_pll_src_ppll_cpll_gpll_npll_p) = { "ppll", "dummy_cpll", "gpll", ++ "npll", "upll" }; ++PNAME(mux_pll_src_cpll_gpll_npll_upll_24m_p) = { "dummy_cpll", "gpll", "npll", ++ "upll", "xin24m" }; ++PNAME(mux_pll_src_cpll_gpll_npll_ppll_upll_24m_p) = { "dummy_cpll", "gpll", "npll", ++ "ppll", "upll", "xin24m" }; ++/* ++ * We hope to be able to HDMI/DP can obtain better signal quality, ++ * therefore, we move VOP pwm and aclk clocks to other PLLs, let ++ * HDMI/DP phyclock can monopolize VPLL. ++ */ ++PNAME(mux_pll_src_dmyvpll_cpll_gpll_npll_p) = { "dummy_vpll", "dummy_cpll", "gpll", ++ "npll" }; ++PNAME(mux_pll_src_dmyvpll_cpll_gpll_gpll_p) = { "dummy_vpll", "dummy_cpll", "gpll", ++ "gpll" }; ++PNAME(mux_pll_src_24m_32k_cpll_gpll_p) = { "xin24m", "xin32k", ++ "dummy_cpll", "gpll" }; ++ ++PNAME(mux_aclk_emmc_p) = { "dummy_cpll", ++ "gpll_aclk_emmc_src" }; ++ ++PNAME(mux_aclk_perilp0_p) = { "dummy_cpll", ++ "gpll_aclk_perilp0_src" }; ++ ++PNAME(mux_fclk_cm0s_p) = { "dummy_cpll", ++ "gpll_fclk_cm0s_src" }; ++ ++PNAME(mux_hclk_perilp1_p) = { "dummy_cpll", ++ "gpll_hclk_perilp1_src" }; ++PNAME(mux_aclk_gmac_p) = { "dummy_cpll", ++ "gpll_aclk_gmac_src" }; ++#else + PNAME(mux_aclk_cci_p) = { "cpll_aclk_cci_src", + "gpll_aclk_cci_src", + "npll_aclk_cci_src", +- "vpll_aclk_cci_src" }; ++ "dummy_vpll" }; + PNAME(mux_cci_trace_p) = { "cpll_cci_trace", + "gpll_cci_trace" }; + PNAME(mux_cs_p) = { "cpll_cs", "gpll_cs", +@@ -148,26 +224,17 @@ PNAME(mux_pll_src_cpll_gpll_npll_upll_24m_p) = { "cpll", "gpll", "npll", + "upll", "xin24m" }; + PNAME(mux_pll_src_cpll_gpll_npll_ppll_upll_24m_p) = { "cpll", "gpll", "npll", + "ppll", "upll", "xin24m" }; +- +-PNAME(mux_pll_src_vpll_cpll_gpll_p) = { "vpll", "cpll", "gpll" }; +-PNAME(mux_pll_src_vpll_cpll_gpll_npll_p) = { "vpll", "cpll", "gpll", ++/* ++ * We hope to be able to HDMI/DP can obtain better signal quality, ++ * therefore, we move VOP pwm and aclk clocks to other PLLs, let ++ * HDMI/DP phyclock can monopolize VPLL. ++ */ ++PNAME(mux_pll_src_dmyvpll_cpll_gpll_npll_p) = { "dummy_vpll", "cpll", "gpll", + "npll" }; +-PNAME(mux_pll_src_vpll_cpll_gpll_24m_p) = { "vpll", "cpll", "gpll", +- "xin24m" }; +- +-PNAME(mux_dclk_vop0_p) = { "dclk_vop0_div", +- "dclk_vop0_frac" }; +-PNAME(mux_dclk_vop1_p) = { "dclk_vop1_div", +- "dclk_vop1_frac" }; +- +-PNAME(mux_clk_cif_p) = { "clk_cifout_src", "xin24m" }; +- +-PNAME(mux_pll_src_24m_usbphy480m_p) = { "xin24m", "clk_usbphy_480m" }; +-PNAME(mux_pll_src_24m_pciephy_p) = { "xin24m", "clk_pciephy_ref100m" }; ++PNAME(mux_pll_src_dmyvpll_cpll_gpll_gpll_p) = { "dummy_vpll", "cpll", "gpll", ++ "gpll" }; + PNAME(mux_pll_src_24m_32k_cpll_gpll_p) = { "xin24m", "xin32k", + "cpll", "gpll" }; +-PNAME(mux_pciecore_cru_phy_p) = { "clk_pcie_core_cru", +- "clk_pcie_core_phy" }; + + PNAME(mux_aclk_emmc_p) = { "cpll_aclk_emmc_src", + "gpll_aclk_emmc_src" }; +@@ -180,14 +247,26 @@ PNAME(mux_fclk_cm0s_p) = { "cpll_fclk_cm0s_src", + + PNAME(mux_hclk_perilp1_p) = { "cpll_hclk_perilp1_src", + "gpll_hclk_perilp1_src" }; ++PNAME(mux_aclk_gmac_p) = { "cpll_aclk_gmac_src", ++ "gpll_aclk_gmac_src" }; ++#endif ++ ++PNAME(mux_dclk_vop0_p) = { "dclk_vop0_div", ++ "dummy_dclk_vop0_frac" }; ++PNAME(mux_dclk_vop1_p) = { "dclk_vop1_div", ++ "dummy_dclk_vop1_frac" }; ++ ++PNAME(mux_clk_cif_p) = { "clk_cifout_src", "xin24m" }; + ++PNAME(mux_pll_src_24m_usbphy480m_p) = { "xin24m", "clk_usbphy_480m" }; ++PNAME(mux_pll_src_24m_pciephy_p) = { "xin24m", "clk_pciephy_ref100m" }; ++PNAME(mux_pciecore_cru_phy_p) = { "clk_pcie_core_cru", ++ "clk_pcie_core_phy" }; + PNAME(mux_clk_testout1_p) = { "clk_testout1_pll_src", "xin24m" }; + PNAME(mux_clk_testout2_p) = { "clk_testout2_pll_src", "xin24m" }; + + PNAME(mux_usbphy_480m_p) = { "clk_usbphy0_480m_src", + "clk_usbphy1_480m_src" }; +-PNAME(mux_aclk_gmac_p) = { "cpll_aclk_gmac_src", +- "gpll_aclk_gmac_src" }; + PNAME(mux_rmii_p) = { "clk_gmac", "clkin_gmac" }; + PNAME(mux_spdif_p) = { "clk_spdif_div", "clk_spdif_frac", + "clkin_i2s", "xin12m" }; +@@ -201,20 +280,22 @@ PNAME(mux_i2sch_p) = { "clk_i2s0", "clk_i2s1", + "clk_i2s2" }; + PNAME(mux_i2sout_p) = { "clk_i2sout_src", "xin12m" }; + +-PNAME(mux_uart0_p) = { "clk_uart0_div", "clk_uart0_frac", "xin24m" }; +-PNAME(mux_uart1_p) = { "clk_uart1_div", "clk_uart1_frac", "xin24m" }; +-PNAME(mux_uart2_p) = { "clk_uart2_div", "clk_uart2_frac", "xin24m" }; +-PNAME(mux_uart3_p) = { "clk_uart3_div", "clk_uart3_frac", "xin24m" }; ++PNAME(mux_uart0_p) = { "xin24m", "clk_uart0_div", "clk_uart0_frac" }; ++PNAME(mux_uart1_p) = { "xin24m", "clk_uart1_div", "clk_uart1_frac" }; ++PNAME(mux_uart2_p) = { "xin24m", "clk_uart2_div", "clk_uart2_frac" }; ++PNAME(mux_uart3_p) = { "xin24m", "clk_uart3_div", "clk_uart3_frac" }; + + /* PMU CRU parents */ + PNAME(mux_ppll_24m_p) = { "ppll", "xin24m" }; + PNAME(mux_24m_ppll_p) = { "xin24m", "ppll" }; + PNAME(mux_fclk_cm0s_pmu_ppll_p) = { "fclk_cm0s_pmu_ppll_src", "xin24m" }; + PNAME(mux_wifi_pmu_p) = { "clk_wifi_div", "clk_wifi_frac" }; +-PNAME(mux_uart4_pmu_p) = { "clk_uart4_div", "clk_uart4_frac", +- "xin24m" }; ++PNAME(mux_uart4_pmu_p) = { "xin24m", "clk_uart4_div", ++ "clk_uart4_frac" }; + PNAME(mux_clk_testout2_2io_p) = { "clk_testout2", "clk_32k_suspend_pmu" }; + ++static u32 uart_mux_idx[] = { 2, 0, 1 }; ++ + static struct rockchip_pll_clock rk3399_pll_clks[] __initdata = { + [lpll] = PLL(pll_rk3399, PLL_APLLL, "lpll", mux_pll_p, 0, RK3399_PLL_CON(0), + RK3399_PLL_CON(3), 8, 31, 0, rk3399_pll_rates), +@@ -222,18 +303,23 @@ static struct rockchip_pll_clock rk3399_pll_clks[] __initdata = { + RK3399_PLL_CON(11), 8, 31, 0, rk3399_pll_rates), + [dpll] = PLL(pll_rk3399, PLL_DPLL, "dpll", mux_pll_p, 0, RK3399_PLL_CON(16), + RK3399_PLL_CON(19), 8, 31, 0, NULL), ++#ifdef RK3399_TWO_PLL_FOR_VOP ++ [cpll] = PLL(pll_rk3399, PLL_CPLL, "cpll", mux_pll_p, 0, RK3399_PLL_CON(24), ++ RK3399_PLL_CON(27), 8, 31, 0, rk3399_pll_rates), ++#else + [cpll] = PLL(pll_rk3399, PLL_CPLL, "cpll", mux_pll_p, 0, RK3399_PLL_CON(24), + RK3399_PLL_CON(27), 8, 31, ROCKCHIP_PLL_SYNC_RATE, rk3399_pll_rates), ++#endif + [gpll] = PLL(pll_rk3399, PLL_GPLL, "gpll", mux_pll_p, 0, RK3399_PLL_CON(32), +- RK3399_PLL_CON(35), 8, 31, ROCKCHIP_PLL_SYNC_RATE, rk3399_pll_rates), ++ RK3399_PLL_CON(35), 8, 31, 0, rk3399_pll_rates), + [npll] = PLL(pll_rk3399, PLL_NPLL, "npll", mux_pll_p, 0, RK3399_PLL_CON(40), + RK3399_PLL_CON(43), 8, 31, ROCKCHIP_PLL_SYNC_RATE, rk3399_pll_rates), + [vpll] = PLL(pll_rk3399, PLL_VPLL, "vpll", mux_pll_p, 0, RK3399_PLL_CON(48), +- RK3399_PLL_CON(51), 8, 31, ROCKCHIP_PLL_SYNC_RATE, rk3399_pll_rates), ++ RK3399_PLL_CON(51), 8, 31, 0, rk3399_vpll_rates), + }; + + static struct rockchip_pll_clock rk3399_pmu_pll_clks[] __initdata = { +- [ppll] = PLL(pll_rk3399, PLL_PPLL, "ppll", mux_pll_p, 0, RK3399_PMU_PLL_CON(0), ++ [ppll] = PLL(pll_rk3399, PLL_PPLL, "ppll", mux_pll_p, CLK_IS_CRITICAL, RK3399_PMU_PLL_CON(0), + RK3399_PMU_PLL_CON(3), 8, 31, ROCKCHIP_PLL_SYNC_RATE, rk3399_pll_rates), + }; + +@@ -259,24 +345,24 @@ static struct rockchip_clk_branch rk3399_i2s2_fracmux __initdata = + RK3399_CLKSEL_CON(30), 8, 2, MFLAGS); + + static struct rockchip_clk_branch rk3399_uart0_fracmux __initdata = +- MUX(SCLK_UART0, "clk_uart0", mux_uart0_p, CLK_SET_RATE_PARENT, +- RK3399_CLKSEL_CON(33), 8, 2, MFLAGS); ++ MUXTBL(SCLK_UART0, "clk_uart0", mux_uart0_p, CLK_SET_RATE_PARENT, ++ RK3399_CLKSEL_CON(33), 8, 2, MFLAGS, uart_mux_idx); + + static struct rockchip_clk_branch rk3399_uart1_fracmux __initdata = +- MUX(SCLK_UART1, "clk_uart1", mux_uart1_p, CLK_SET_RATE_PARENT, +- RK3399_CLKSEL_CON(34), 8, 2, MFLAGS); ++ MUXTBL(SCLK_UART1, "clk_uart1", mux_uart1_p, CLK_SET_RATE_PARENT, ++ RK3399_CLKSEL_CON(34), 8, 2, MFLAGS, uart_mux_idx); + + static struct rockchip_clk_branch rk3399_uart2_fracmux __initdata = +- MUX(SCLK_UART2, "clk_uart2", mux_uart2_p, CLK_SET_RATE_PARENT, +- RK3399_CLKSEL_CON(35), 8, 2, MFLAGS); ++ MUXTBL(SCLK_UART2, "clk_uart2", mux_uart2_p, CLK_SET_RATE_PARENT, ++ RK3399_CLKSEL_CON(35), 8, 2, MFLAGS, uart_mux_idx); + + static struct rockchip_clk_branch rk3399_uart3_fracmux __initdata = +- MUX(SCLK_UART3, "clk_uart3", mux_uart3_p, CLK_SET_RATE_PARENT, +- RK3399_CLKSEL_CON(36), 8, 2, MFLAGS); ++ MUXTBL(SCLK_UART3, "clk_uart3", mux_uart3_p, CLK_SET_RATE_PARENT, ++ RK3399_CLKSEL_CON(36), 8, 2, MFLAGS, uart_mux_idx); + + static struct rockchip_clk_branch rk3399_uart4_pmu_fracmux __initdata = +- MUX(SCLK_UART4_PMU, "clk_uart4_pmu", mux_uart4_pmu_p, CLK_SET_RATE_PARENT, +- RK3399_PMU_CLKSEL_CON(5), 8, 2, MFLAGS); ++ MUXTBL(SCLK_UART4_PMU, "clk_uart4_pmu", mux_uart4_pmu_p, CLK_SET_RATE_PARENT, ++ RK3399_PMU_CLKSEL_CON(5), 8, 2, MFLAGS, uart_mux_idx); + + static struct rockchip_clk_branch rk3399_dclk_vop0_fracmux __initdata = + MUX(DCLK_VOP0, "dclk_vop0", mux_dclk_vop0_p, CLK_SET_RATE_PARENT, +@@ -291,9 +377,10 @@ static struct rockchip_clk_branch rk3399_pmuclk_wifi_fracmux __initdata = + RK3399_PMU_CLKSEL_CON(1), 14, 1, MFLAGS); + + static const struct rockchip_cpuclk_reg_data rk3399_cpuclkl_data = { +- .core_reg = RK3399_CLKSEL_CON(0), +- .div_core_shift = 0, +- .div_core_mask = 0x1f, ++ .core_reg[0] = RK3399_CLKSEL_CON(0), ++ .div_core_shift[0] = 0, ++ .div_core_mask[0] = 0x1f, ++ .num_cores = 1, + .mux_core_alt = 3, + .mux_core_main = 0, + .mux_core_shift = 6, +@@ -301,9 +388,10 @@ static const struct rockchip_cpuclk_reg_data rk3399_cpuclkl_data = { + }; + + static const struct rockchip_cpuclk_reg_data rk3399_cpuclkb_data = { +- .core_reg = RK3399_CLKSEL_CON(2), +- .div_core_shift = 0, +- .div_core_mask = 0x1f, ++ .core_reg[0] = RK3399_CLKSEL_CON(2), ++ .div_core_shift[0] = 0, ++ .div_core_mask[0] = 0x1f, ++ .num_cores = 1, + .mux_core_alt = 3, + .mux_core_main = 1, + .mux_core_shift = 6, +@@ -406,9 +494,9 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { + GATE(SCLK_USB2PHY1_REF, "clk_usb2phy1_ref", "xin24m", CLK_IGNORE_UNUSED, + RK3399_CLKGATE_CON(6), 6, GFLAGS), + +- GATE(0, "clk_usbphy0_480m_src", "clk_usbphy0_480m", 0, ++ GATE(SCLK_USBPHY0_480M_SRC, "clk_usbphy0_480m_src", "clk_usbphy0_480m", CLK_IGNORE_UNUSED, + RK3399_CLKGATE_CON(13), 12, GFLAGS), +- GATE(0, "clk_usbphy1_480m_src", "clk_usbphy1_480m", 0, ++ GATE(SCLK_USBPHY1_480M_SRC, "clk_usbphy1_480m_src", "clk_usbphy1_480m", CLK_IGNORE_UNUSED, + RK3399_CLKGATE_CON(13), 12, GFLAGS), + MUX(0, "clk_usbphy_480m", mux_usbphy_480m_p, 0, + RK3399_CLKSEL_CON(14), 6, 1, MFLAGS), +@@ -423,7 +511,7 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { + COMPOSITE(ACLK_USB3, "aclk_usb3", mux_pll_src_cpll_gpll_npll_p, 0, + RK3399_CLKSEL_CON(39), 6, 2, MFLAGS, 0, 5, DFLAGS, + RK3399_CLKGATE_CON(12), 0, GFLAGS), +- GATE(ACLK_USB3_NOC, "aclk_usb3_noc", "aclk_usb3", CLK_IGNORE_UNUSED, ++ GATE(ACLK_USB3_NOC, "aclk_usb3_noc", "aclk_usb3", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(30), 0, GFLAGS), + GATE(ACLK_USB3OTG0, "aclk_usb3otg0", "aclk_usb3", 0, + RK3399_CLKGATE_CON(30), 1, GFLAGS), +@@ -549,7 +637,7 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { + + GATE(ACLK_GMAC, "aclk_gmac", "aclk_gmac_pre", 0, + RK3399_CLKGATE_CON(32), 0, GFLAGS), +- GATE(ACLK_GMAC_NOC, "aclk_gmac_noc", "aclk_gmac_pre", CLK_IGNORE_UNUSED, ++ GATE(ACLK_GMAC_NOC, "aclk_gmac_noc", "aclk_gmac_pre", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(32), 1, GFLAGS), + GATE(ACLK_PERF_GMAC, "aclk_perf_gmac", "aclk_gmac_pre", 0, + RK3399_CLKGATE_CON(32), 4, GFLAGS), +@@ -559,7 +647,7 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { + RK3399_CLKGATE_CON(6), 11, GFLAGS), + GATE(PCLK_GMAC, "pclk_gmac", "pclk_gmac_pre", 0, + RK3399_CLKGATE_CON(32), 2, GFLAGS), +- GATE(PCLK_GMAC_NOC, "pclk_gmac_noc", "pclk_gmac_pre", CLK_IGNORE_UNUSED, ++ GATE(PCLK_GMAC_NOC, "pclk_gmac_noc", "pclk_gmac_pre", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(32), 3, GFLAGS), + + COMPOSITE(SCLK_MAC, "clk_gmac", mux_pll_src_cpll_gpll_npll_p, 0, +@@ -578,13 +666,13 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { + RK3399_CLKGATE_CON(5), 9, GFLAGS), + + /* spdif */ +- COMPOSITE(0, "clk_spdif_div", mux_pll_src_cpll_gpll_p, 0, ++ COMPOSITE(SCLK_SPDIF_DIV, "clk_spdif_div", mux_pll_src_cpll_gpll_p, 0, + RK3399_CLKSEL_CON(32), 7, 1, MFLAGS, 0, 7, DFLAGS, + RK3399_CLKGATE_CON(8), 13, GFLAGS), +- COMPOSITE_FRACMUX(0, "clk_spdif_frac", "clk_spdif_div", 0, ++ COMPOSITE_FRACMUX(0, "clk_spdif_frac", "clk_spdif_div", CLK_SET_RATE_PARENT, + RK3399_CLKSEL_CON(99), 0, + RK3399_CLKGATE_CON(8), 14, GFLAGS, +- &rk3399_spdif_fracmux), ++ &rk3399_spdif_fracmux, RK3399_SPDIF_FRAC_MAX_PRATE), + GATE(SCLK_SPDIF_8CH, "clk_spdif", "clk_spdif_mux", CLK_SET_RATE_PARENT, + RK3399_CLKGATE_CON(8), 15, GFLAGS), + +@@ -592,84 +680,84 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { + RK3399_CLKSEL_CON(32), 15, 1, MFLAGS, 8, 5, DFLAGS, + RK3399_CLKGATE_CON(10), 6, GFLAGS), + /* i2s */ +- COMPOSITE(0, "clk_i2s0_div", mux_pll_src_cpll_gpll_p, 0, ++ COMPOSITE(SCLK_I2S0_DIV, "clk_i2s0_div", mux_pll_src_cpll_gpll_p, 0, + RK3399_CLKSEL_CON(28), 7, 1, MFLAGS, 0, 7, DFLAGS, + RK3399_CLKGATE_CON(8), 3, GFLAGS), +- COMPOSITE_FRACMUX(0, "clk_i2s0_frac", "clk_i2s0_div", 0, ++ COMPOSITE_FRACMUX(0, "clk_i2s0_frac", "clk_i2s0_div", CLK_SET_RATE_PARENT, + RK3399_CLKSEL_CON(96), 0, + RK3399_CLKGATE_CON(8), 4, GFLAGS, +- &rk3399_i2s0_fracmux), ++ &rk3399_i2s0_fracmux, RK3399_I2S_FRAC_MAX_PRATE), + GATE(SCLK_I2S0_8CH, "clk_i2s0", "clk_i2s0_mux", CLK_SET_RATE_PARENT, + RK3399_CLKGATE_CON(8), 5, GFLAGS), + +- COMPOSITE(0, "clk_i2s1_div", mux_pll_src_cpll_gpll_p, 0, ++ COMPOSITE(SCLK_I2S1_DIV, "clk_i2s1_div", mux_pll_src_cpll_gpll_p, 0, + RK3399_CLKSEL_CON(29), 7, 1, MFLAGS, 0, 7, DFLAGS, + RK3399_CLKGATE_CON(8), 6, GFLAGS), +- COMPOSITE_FRACMUX(0, "clk_i2s1_frac", "clk_i2s1_div", 0, ++ COMPOSITE_FRACMUX(0, "clk_i2s1_frac", "clk_i2s1_div", CLK_SET_RATE_PARENT, + RK3399_CLKSEL_CON(97), 0, + RK3399_CLKGATE_CON(8), 7, GFLAGS, +- &rk3399_i2s1_fracmux), ++ &rk3399_i2s1_fracmux, RK3399_I2S_FRAC_MAX_PRATE), + GATE(SCLK_I2S1_8CH, "clk_i2s1", "clk_i2s1_mux", CLK_SET_RATE_PARENT, + RK3399_CLKGATE_CON(8), 8, GFLAGS), + +- COMPOSITE(0, "clk_i2s2_div", mux_pll_src_cpll_gpll_p, 0, ++ COMPOSITE(SCLK_I2S2_DIV, "clk_i2s2_div", mux_pll_src_cpll_gpll_p, 0, + RK3399_CLKSEL_CON(30), 7, 1, MFLAGS, 0, 7, DFLAGS, + RK3399_CLKGATE_CON(8), 9, GFLAGS), +- COMPOSITE_FRACMUX(0, "clk_i2s2_frac", "clk_i2s2_div", 0, ++ COMPOSITE_FRACMUX(0, "clk_i2s2_frac", "clk_i2s2_div", CLK_SET_RATE_PARENT, + RK3399_CLKSEL_CON(98), 0, + RK3399_CLKGATE_CON(8), 10, GFLAGS, +- &rk3399_i2s2_fracmux), ++ &rk3399_i2s2_fracmux, RK3399_I2S_FRAC_MAX_PRATE), + GATE(SCLK_I2S2_8CH, "clk_i2s2", "clk_i2s2_mux", CLK_SET_RATE_PARENT, + RK3399_CLKGATE_CON(8), 11, GFLAGS), + +- MUX(0, "clk_i2sout_src", mux_i2sch_p, CLK_SET_RATE_PARENT, ++ MUX(SCLK_I2SOUT_SRC, "clk_i2sout_src", mux_i2sch_p, CLK_SET_RATE_PARENT, + RK3399_CLKSEL_CON(31), 0, 2, MFLAGS), + COMPOSITE_NODIV(SCLK_I2S_8CH_OUT, "clk_i2sout", mux_i2sout_p, CLK_SET_RATE_PARENT, + RK3399_CLKSEL_CON(31), 2, 1, MFLAGS, + RK3399_CLKGATE_CON(8), 12, GFLAGS), + + /* uart */ +- MUX(0, "clk_uart0_src", mux_pll_src_cpll_gpll_upll_p, 0, ++ MUX(SCLK_UART0_SRC, "clk_uart0_src", mux_pll_src_cpll_gpll_upll_p, 0, + RK3399_CLKSEL_CON(33), 12, 2, MFLAGS), + COMPOSITE_NOMUX(0, "clk_uart0_div", "clk_uart0_src", 0, + RK3399_CLKSEL_CON(33), 0, 7, DFLAGS, + RK3399_CLKGATE_CON(9), 0, GFLAGS), +- COMPOSITE_FRACMUX(0, "clk_uart0_frac", "clk_uart0_div", 0, ++ COMPOSITE_FRACMUX(0, "clk_uart0_frac", "clk_uart0_div", CLK_SET_RATE_PARENT, + RK3399_CLKSEL_CON(100), 0, + RK3399_CLKGATE_CON(9), 1, GFLAGS, +- &rk3399_uart0_fracmux), ++ &rk3399_uart0_fracmux, RK3399_UART_FRAC_MAX_PRATE), + +- MUX(0, "clk_uart_src", mux_pll_src_cpll_gpll_p, 0, ++ MUX(SCLK_UART_SRC, "clk_uart_src", mux_pll_src_cpll_gpll_p, 0, + RK3399_CLKSEL_CON(33), 15, 1, MFLAGS), + COMPOSITE_NOMUX(0, "clk_uart1_div", "clk_uart_src", 0, + RK3399_CLKSEL_CON(34), 0, 7, DFLAGS, + RK3399_CLKGATE_CON(9), 2, GFLAGS), +- COMPOSITE_FRACMUX(0, "clk_uart1_frac", "clk_uart1_div", 0, ++ COMPOSITE_FRACMUX(0, "clk_uart1_frac", "clk_uart1_div", CLK_SET_RATE_PARENT, + RK3399_CLKSEL_CON(101), 0, + RK3399_CLKGATE_CON(9), 3, GFLAGS, +- &rk3399_uart1_fracmux), ++ &rk3399_uart1_fracmux, RK3399_UART_FRAC_MAX_PRATE), + + COMPOSITE_NOMUX(0, "clk_uart2_div", "clk_uart_src", 0, + RK3399_CLKSEL_CON(35), 0, 7, DFLAGS, + RK3399_CLKGATE_CON(9), 4, GFLAGS), +- COMPOSITE_FRACMUX(0, "clk_uart2_frac", "clk_uart2_div", 0, ++ COMPOSITE_FRACMUX(0, "clk_uart2_frac", "clk_uart2_div", CLK_SET_RATE_PARENT, + RK3399_CLKSEL_CON(102), 0, + RK3399_CLKGATE_CON(9), 5, GFLAGS, +- &rk3399_uart2_fracmux), ++ &rk3399_uart2_fracmux, RK3399_UART_FRAC_MAX_PRATE), + + COMPOSITE_NOMUX(0, "clk_uart3_div", "clk_uart_src", 0, + RK3399_CLKSEL_CON(36), 0, 7, DFLAGS, + RK3399_CLKGATE_CON(9), 6, GFLAGS), +- COMPOSITE_FRACMUX(0, "clk_uart3_frac", "clk_uart3_div", 0, ++ COMPOSITE_FRACMUX(0, "clk_uart3_frac", "clk_uart3_div", CLK_SET_RATE_PARENT, + RK3399_CLKSEL_CON(103), 0, + RK3399_CLKGATE_CON(9), 7, GFLAGS, +- &rk3399_uart3_fracmux), ++ &rk3399_uart3_fracmux, RK3399_UART_FRAC_MAX_PRATE), + +- COMPOSITE(PCLK_DDR, "pclk_ddr", mux_pll_src_cpll_gpll_p, CLK_IGNORE_UNUSED, ++ COMPOSITE(PCLK_DDR, "pclk_ddr", mux_pll_src_cpll_gpll_p, CLK_IS_CRITICAL, + RK3399_CLKSEL_CON(6), 15, 1, MFLAGS, 8, 5, DFLAGS, + RK3399_CLKGATE_CON(3), 4, GFLAGS), + +- GATE(PCLK_CENTER_MAIN_NOC, "pclk_center_main_noc", "pclk_ddr", CLK_IGNORE_UNUSED, ++ GATE(PCLK_CENTER_MAIN_NOC, "pclk_center_main_noc", "pclk_ddr", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(18), 10, GFLAGS), + GATE(PCLK_DDR_MON, "pclk_ddr_mon", "pclk_ddr", 0, + RK3399_CLKGATE_CON(18), 12, GFLAGS), +@@ -686,30 +774,30 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { + RK3399_CLKGATE_CON(3), 6, GFLAGS), + + /* cci */ +- GATE(0, "cpll_aclk_cci_src", "cpll", CLK_IGNORE_UNUSED, ++ GATE(0, "cpll_aclk_cci_src", "cpll", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(2), 0, GFLAGS), +- GATE(0, "gpll_aclk_cci_src", "gpll", CLK_IGNORE_UNUSED, ++ GATE(0, "gpll_aclk_cci_src", "gpll", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(2), 1, GFLAGS), +- GATE(0, "npll_aclk_cci_src", "npll", CLK_IGNORE_UNUSED, ++ GATE(0, "npll_aclk_cci_src", "npll", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(2), 2, GFLAGS), +- GATE(0, "vpll_aclk_cci_src", "vpll", CLK_IGNORE_UNUSED, ++ GATE(0, "vpll_aclk_cci_src", "vpll", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(2), 3, GFLAGS), + +- COMPOSITE(0, "aclk_cci_pre", mux_aclk_cci_p, CLK_IGNORE_UNUSED, ++ COMPOSITE(0, "aclk_cci_pre", mux_aclk_cci_p, CLK_IS_CRITICAL, + RK3399_CLKSEL_CON(5), 6, 2, MFLAGS, 0, 5, DFLAGS, + RK3399_CLKGATE_CON(2), 4, GFLAGS), + +- GATE(ACLK_ADB400M_PD_CORE_L, "aclk_adb400m_pd_core_l", "aclk_cci_pre", CLK_IGNORE_UNUSED, ++ GATE(ACLK_ADB400M_PD_CORE_L, "aclk_adb400m_pd_core_l", "aclk_cci_pre", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(15), 0, GFLAGS), +- GATE(ACLK_ADB400M_PD_CORE_B, "aclk_adb400m_pd_core_b", "aclk_cci_pre", CLK_IGNORE_UNUSED, ++ GATE(ACLK_ADB400M_PD_CORE_B, "aclk_adb400m_pd_core_b", "aclk_cci_pre", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(15), 1, GFLAGS), +- GATE(ACLK_CCI, "aclk_cci", "aclk_cci_pre", CLK_IGNORE_UNUSED, ++ GATE(ACLK_CCI, "aclk_cci", "aclk_cci_pre", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(15), 2, GFLAGS), +- GATE(ACLK_CCI_NOC0, "aclk_cci_noc0", "aclk_cci_pre", CLK_IGNORE_UNUSED, ++ GATE(ACLK_CCI_NOC0, "aclk_cci_noc0", "aclk_cci_pre", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(15), 3, GFLAGS), +- GATE(ACLK_CCI_NOC1, "aclk_cci_noc1", "aclk_cci_pre", CLK_IGNORE_UNUSED, ++ GATE(ACLK_CCI_NOC1, "aclk_cci_noc1", "aclk_cci_pre", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(15), 4, GFLAGS), +- GATE(ACLK_CCI_GRF, "aclk_cci_grf", "aclk_cci_pre", CLK_IGNORE_UNUSED, ++ GATE(ACLK_CCI_GRF, "aclk_cci_grf", "aclk_cci_pre", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(15), 7, GFLAGS), + + GATE(0, "cpll_cci_trace", "cpll", CLK_IGNORE_UNUSED, +@@ -717,20 +805,20 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { + GATE(0, "gpll_cci_trace", "gpll", CLK_IGNORE_UNUSED, + RK3399_CLKGATE_CON(2), 6, GFLAGS), + COMPOSITE(SCLK_CCI_TRACE, "clk_cci_trace", mux_cci_trace_p, CLK_IGNORE_UNUSED, +- RK3399_CLKSEL_CON(5), 15, 2, MFLAGS, 8, 5, DFLAGS, ++ RK3399_CLKSEL_CON(5), 15, 1, MFLAGS, 8, 5, DFLAGS, + RK3399_CLKGATE_CON(2), 7, GFLAGS), + +- GATE(0, "cpll_cs", "cpll", CLK_IGNORE_UNUSED, ++ GATE(0, "cpll_cs", "cpll", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(2), 8, GFLAGS), +- GATE(0, "gpll_cs", "gpll", CLK_IGNORE_UNUSED, ++ GATE(0, "gpll_cs", "gpll", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(2), 9, GFLAGS), +- GATE(0, "npll_cs", "npll", CLK_IGNORE_UNUSED, ++ GATE(0, "npll_cs", "npll", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(2), 10, GFLAGS), +- COMPOSITE_NOGATE(0, "clk_cs", mux_cs_p, CLK_IGNORE_UNUSED, ++ COMPOSITE_NOGATE(SCLK_CS, "clk_cs", mux_cs_p, CLK_IS_CRITICAL, + RK3399_CLKSEL_CON(4), 6, 2, MFLAGS, 0, 5, DFLAGS), + GATE(0, "clk_dbg_cxcs", "clk_cs", CLK_IGNORE_UNUSED, + RK3399_CLKGATE_CON(15), 5, GFLAGS), +- GATE(0, "clk_dbg_noc", "clk_cs", CLK_IGNORE_UNUSED, ++ GATE(0, "clk_dbg_noc", "clk_cs", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(15), 6, GFLAGS), + + /* vcodec */ +@@ -742,12 +830,12 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { + RK3399_CLKGATE_CON(4), 1, GFLAGS), + GATE(HCLK_VCODEC, "hclk_vcodec", "hclk_vcodec_pre", 0, + RK3399_CLKGATE_CON(17), 2, GFLAGS), +- GATE(0, "hclk_vcodec_noc", "hclk_vcodec_pre", CLK_IGNORE_UNUSED, ++ GATE(0, "hclk_vcodec_noc", "hclk_vcodec_pre", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(17), 3, GFLAGS), + + GATE(ACLK_VCODEC, "aclk_vcodec", "aclk_vcodec_pre", 0, + RK3399_CLKGATE_CON(17), 0, GFLAGS), +- GATE(0, "aclk_vcodec_noc", "aclk_vcodec_pre", CLK_IGNORE_UNUSED, ++ GATE(0, "aclk_vcodec_noc", "aclk_vcodec_pre", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(17), 1, GFLAGS), + + /* vdu */ +@@ -766,12 +854,12 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { + RK3399_CLKGATE_CON(4), 3, GFLAGS), + GATE(HCLK_VDU, "hclk_vdu", "hclk_vdu_pre", 0, + RK3399_CLKGATE_CON(17), 10, GFLAGS), +- GATE(HCLK_VDU_NOC, "hclk_vdu_noc", "hclk_vdu_pre", CLK_IGNORE_UNUSED, ++ GATE(HCLK_VDU_NOC, "hclk_vdu_noc", "hclk_vdu_pre", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(17), 11, GFLAGS), + + GATE(ACLK_VDU, "aclk_vdu", "aclk_vdu_pre", 0, + RK3399_CLKGATE_CON(17), 8, GFLAGS), +- GATE(ACLK_VDU_NOC, "aclk_vdu_noc", "aclk_vdu_pre", CLK_IGNORE_UNUSED, ++ GATE(ACLK_VDU_NOC, "aclk_vdu_noc", "aclk_vdu_pre", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(17), 9, GFLAGS), + + /* iep */ +@@ -783,12 +871,12 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { + RK3399_CLKGATE_CON(4), 7, GFLAGS), + GATE(HCLK_IEP, "hclk_iep", "hclk_iep_pre", 0, + RK3399_CLKGATE_CON(16), 2, GFLAGS), +- GATE(HCLK_IEP_NOC, "hclk_iep_noc", "hclk_iep_pre", CLK_IGNORE_UNUSED, ++ GATE(HCLK_IEP_NOC, "hclk_iep_noc", "hclk_iep_pre", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(16), 3, GFLAGS), + + GATE(ACLK_IEP, "aclk_iep", "aclk_iep_pre", 0, + RK3399_CLKGATE_CON(16), 0, GFLAGS), +- GATE(ACLK_IEP_NOC, "aclk_iep_noc", "aclk_iep_pre", CLK_IGNORE_UNUSED, ++ GATE(ACLK_IEP_NOC, "aclk_iep_noc", "aclk_iep_pre", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(16), 1, GFLAGS), + + /* rga */ +@@ -804,21 +892,21 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { + RK3399_CLKGATE_CON(4), 9, GFLAGS), + GATE(HCLK_RGA, "hclk_rga", "hclk_rga_pre", 0, + RK3399_CLKGATE_CON(16), 10, GFLAGS), +- GATE(HCLK_RGA_NOC, "hclk_rga_noc", "hclk_rga_pre", CLK_IGNORE_UNUSED, ++ GATE(HCLK_RGA_NOC, "hclk_rga_noc", "hclk_rga_pre", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(16), 11, GFLAGS), + + GATE(ACLK_RGA, "aclk_rga", "aclk_rga_pre", 0, + RK3399_CLKGATE_CON(16), 8, GFLAGS), +- GATE(ACLK_RGA_NOC, "aclk_rga_noc", "aclk_rga_pre", CLK_IGNORE_UNUSED, ++ GATE(ACLK_RGA_NOC, "aclk_rga_noc", "aclk_rga_pre", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(16), 9, GFLAGS), + + /* center */ +- COMPOSITE(0, "aclk_center", mux_pll_src_cpll_gpll_npll_p, CLK_IGNORE_UNUSED, ++ COMPOSITE(ACLK_CENTER, "aclk_center", mux_pll_src_cpll_gpll_npll_p, CLK_IS_CRITICAL, + RK3399_CLKSEL_CON(12), 14, 2, MFLAGS, 8, 5, DFLAGS, + RK3399_CLKGATE_CON(3), 7, GFLAGS), +- GATE(ACLK_CENTER_MAIN_NOC, "aclk_center_main_noc", "aclk_center", CLK_IGNORE_UNUSED, ++ GATE(ACLK_CENTER_MAIN_NOC, "aclk_center_main_noc", "aclk_center", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(19), 0, GFLAGS), +- GATE(ACLK_CENTER_PERI_NOC, "aclk_center_peri_noc", "aclk_center", CLK_IGNORE_UNUSED, ++ GATE(ACLK_CENTER_PERI_NOC, "aclk_center_peri_noc", "aclk_center", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(19), 1, GFLAGS), + + /* gpu */ +@@ -835,25 +923,25 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { + RK3399_CLKGATE_CON(13), 1, GFLAGS), + + /* perihp */ +- GATE(0, "cpll_aclk_perihp_src", "cpll", CLK_IGNORE_UNUSED, ++ GATE(0, "cpll_aclk_perihp_src", "cpll", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(5), 1, GFLAGS), +- GATE(0, "gpll_aclk_perihp_src", "gpll", CLK_IGNORE_UNUSED, ++ GATE(0, "gpll_aclk_perihp_src", "gpll", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(5), 0, GFLAGS), +- COMPOSITE(ACLK_PERIHP, "aclk_perihp", mux_aclk_perihp_p, CLK_IGNORE_UNUSED, ++ COMPOSITE(ACLK_PERIHP, "aclk_perihp", mux_aclk_perihp_p, CLK_IS_CRITICAL, + RK3399_CLKSEL_CON(14), 7, 1, MFLAGS, 0, 5, DFLAGS, + RK3399_CLKGATE_CON(5), 2, GFLAGS), +- COMPOSITE_NOMUX(HCLK_PERIHP, "hclk_perihp", "aclk_perihp", CLK_IGNORE_UNUSED, ++ COMPOSITE_NOMUX(HCLK_PERIHP, "hclk_perihp", "aclk_perihp", CLK_IS_CRITICAL, + RK3399_CLKSEL_CON(14), 8, 2, DFLAGS, + RK3399_CLKGATE_CON(5), 3, GFLAGS), +- COMPOSITE_NOMUX(PCLK_PERIHP, "pclk_perihp", "aclk_perihp", CLK_IGNORE_UNUSED, +- RK3399_CLKSEL_CON(14), 12, 2, DFLAGS, ++ COMPOSITE_NOMUX(PCLK_PERIHP, "pclk_perihp", "aclk_perihp", CLK_IS_CRITICAL, ++ RK3399_CLKSEL_CON(14), 12, 3, DFLAGS, + RK3399_CLKGATE_CON(5), 4, GFLAGS), + + GATE(ACLK_PERF_PCIE, "aclk_perf_pcie", "aclk_perihp", 0, + RK3399_CLKGATE_CON(20), 2, GFLAGS), + GATE(ACLK_PCIE, "aclk_pcie", "aclk_perihp", 0, + RK3399_CLKGATE_CON(20), 10, GFLAGS), +- GATE(0, "aclk_perihp_noc", "aclk_perihp", CLK_IGNORE_UNUSED, ++ GATE(0, "aclk_perihp_noc", "aclk_perihp", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(20), 12, GFLAGS), + + GATE(HCLK_HOST0, "hclk_host0", "hclk_perihp", 0, +@@ -866,16 +954,16 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { + RK3399_CLKGATE_CON(20), 8, GFLAGS), + GATE(HCLK_HSIC, "hclk_hsic", "hclk_perihp", 0, + RK3399_CLKGATE_CON(20), 9, GFLAGS), +- GATE(0, "hclk_perihp_noc", "hclk_perihp", CLK_IGNORE_UNUSED, ++ GATE(0, "hclk_perihp_noc", "hclk_perihp", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(20), 13, GFLAGS), + GATE(0, "hclk_ahb1tom", "hclk_perihp", CLK_IGNORE_UNUSED, + RK3399_CLKGATE_CON(20), 15, GFLAGS), + +- GATE(PCLK_PERIHP_GRF, "pclk_perihp_grf", "pclk_perihp", CLK_IGNORE_UNUSED, ++ GATE(PCLK_PERIHP_GRF, "pclk_perihp_grf", "pclk_perihp", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(20), 4, GFLAGS), + GATE(PCLK_PCIE, "pclk_pcie", "pclk_perihp", 0, + RK3399_CLKGATE_CON(20), 11, GFLAGS), +- GATE(0, "pclk_perihp_noc", "pclk_perihp", CLK_IGNORE_UNUSED, ++ GATE(0, "pclk_perihp_noc", "pclk_perihp", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(20), 14, GFLAGS), + GATE(PCLK_HSICPHY, "pclk_hsicphy", "pclk_perihp", 0, + RK3399_CLKGATE_CON(31), 8, GFLAGS), +@@ -886,7 +974,7 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { + RK3399_CLKGATE_CON(12), 13, GFLAGS), + GATE(HCLK_SDMMC, "hclk_sdmmc", "hclk_sd", 0, + RK3399_CLKGATE_CON(33), 8, GFLAGS), +- GATE(0, "hclk_sdmmc_noc", "hclk_sd", CLK_IGNORE_UNUSED, ++ GATE(0, "hclk_sdmmc_noc", "hclk_sd", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(33), 9, GFLAGS), + + COMPOSITE(SCLK_SDIO, "clk_sdio", mux_pll_src_cpll_gpll_npll_ppll_upll_24m_p, 0, +@@ -933,23 +1021,23 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { + RK3399_CLKSEL_CON(21), 7, 1, MFLAGS, 0, 5, DFLAGS), + GATE(ACLK_EMMC_CORE, "aclk_emmccore", "aclk_emmc", CLK_IGNORE_UNUSED, + RK3399_CLKGATE_CON(32), 8, GFLAGS), +- GATE(ACLK_EMMC_NOC, "aclk_emmc_noc", "aclk_emmc", CLK_IGNORE_UNUSED, ++ GATE(ACLK_EMMC_NOC, "aclk_emmc_noc", "aclk_emmc", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(32), 9, GFLAGS), + GATE(ACLK_EMMC_GRF, "aclk_emmcgrf", "aclk_emmc", CLK_IGNORE_UNUSED, + RK3399_CLKGATE_CON(32), 10, GFLAGS), + + /* perilp0 */ +- GATE(0, "cpll_aclk_perilp0_src", "cpll", CLK_IGNORE_UNUSED, ++ GATE(0, "cpll_aclk_perilp0_src", "cpll", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(7), 1, GFLAGS), +- GATE(0, "gpll_aclk_perilp0_src", "gpll", CLK_IGNORE_UNUSED, ++ GATE(0, "gpll_aclk_perilp0_src", "gpll", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(7), 0, GFLAGS), +- COMPOSITE(ACLK_PERILP0, "aclk_perilp0", mux_aclk_perilp0_p, CLK_IGNORE_UNUSED, ++ COMPOSITE(ACLK_PERILP0, "aclk_perilp0", mux_aclk_perilp0_p, CLK_IS_CRITICAL, + RK3399_CLKSEL_CON(23), 7, 1, MFLAGS, 0, 5, DFLAGS, + RK3399_CLKGATE_CON(7), 2, GFLAGS), +- COMPOSITE_NOMUX(HCLK_PERILP0, "hclk_perilp0", "aclk_perilp0", CLK_IGNORE_UNUSED, ++ COMPOSITE_NOMUX(HCLK_PERILP0, "hclk_perilp0", "aclk_perilp0", CLK_IS_CRITICAL, + RK3399_CLKSEL_CON(23), 8, 2, DFLAGS, + RK3399_CLKGATE_CON(7), 3, GFLAGS), +- COMPOSITE_NOMUX(PCLK_PERILP0, "pclk_perilp0", "aclk_perilp0", 0, ++ COMPOSITE_NOMUX(PCLK_PERILP0, "pclk_perilp0", "aclk_perilp0", CLK_IS_CRITICAL, + RK3399_CLKSEL_CON(23), 12, 3, DFLAGS, + RK3399_CLKGATE_CON(7), 4, GFLAGS), + +@@ -964,8 +1052,8 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { + GATE(SCLK_INTMEM5, "clk_intmem5", "aclk_perilp0", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(23), 7, GFLAGS), + GATE(ACLK_DCF, "aclk_dcf", "aclk_perilp0", 0, RK3399_CLKGATE_CON(23), 8, GFLAGS), + GATE(ACLK_DMAC0_PERILP, "aclk_dmac0_perilp", "aclk_perilp0", 0, RK3399_CLKGATE_CON(25), 5, GFLAGS), +- GATE(ACLK_DMAC1_PERILP, "aclk_dmac1_perilp", "aclk_perilp0", 0, RK3399_CLKGATE_CON(25), 6, GFLAGS), +- GATE(ACLK_PERILP0_NOC, "aclk_perilp0_noc", "aclk_perilp0", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(25), 7, GFLAGS), ++ GATE(ACLK_DMAC1_PERILP, "aclk_dmac1_perilp", "aclk_perilp0", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(25), 6, GFLAGS), ++ GATE(ACLK_PERILP0_NOC, "aclk_perilp0_noc", "aclk_perilp0", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(25), 7, GFLAGS), + + /* hclk_perilp0 gates */ + GATE(HCLK_ROM, "hclk_rom", "hclk_perilp0", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(24), 4, GFLAGS), +@@ -973,7 +1061,7 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { + GATE(HCLK_S_CRYPTO0, "hclk_s_crypto0", "hclk_perilp0", 0, RK3399_CLKGATE_CON(24), 6, GFLAGS), + GATE(HCLK_M_CRYPTO1, "hclk_m_crypto1", "hclk_perilp0", 0, RK3399_CLKGATE_CON(24), 14, GFLAGS), + GATE(HCLK_S_CRYPTO1, "hclk_s_crypto1", "hclk_perilp0", 0, RK3399_CLKGATE_CON(24), 15, GFLAGS), +- GATE(HCLK_PERILP0_NOC, "hclk_perilp0_noc", "hclk_perilp0", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(25), 8, GFLAGS), ++ GATE(HCLK_PERILP0_NOC, "hclk_perilp0_noc", "hclk_perilp0", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(25), 8, GFLAGS), + + /* pclk_perilp0 gates */ + GATE(PCLK_DCF, "pclk_dcf", "pclk_perilp0", 0, RK3399_CLKGATE_CON(23), 9, GFLAGS), +@@ -1001,29 +1089,29 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { + GATE(HCLK_M0_PERILP, "hclk_m0_perilp", "fclk_cm0s", 0, RK3399_CLKGATE_CON(24), 9, GFLAGS), + GATE(DCLK_M0_PERILP, "dclk_m0_perilp", "fclk_cm0s", 0, RK3399_CLKGATE_CON(24), 10, GFLAGS), + GATE(SCLK_M0_PERILP_DEC, "clk_m0_perilp_dec", "fclk_cm0s", 0, RK3399_CLKGATE_CON(24), 11, GFLAGS), +- GATE(HCLK_M0_PERILP_NOC, "hclk_m0_perilp_noc", "fclk_cm0s", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(25), 11, GFLAGS), ++ GATE(HCLK_M0_PERILP_NOC, "hclk_m0_perilp_noc", "fclk_cm0s", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(25), 11, GFLAGS), + + /* perilp1 */ +- GATE(0, "cpll_hclk_perilp1_src", "cpll", CLK_IGNORE_UNUSED, ++ GATE(0, "cpll_hclk_perilp1_src", "cpll", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(8), 1, GFLAGS), +- GATE(0, "gpll_hclk_perilp1_src", "gpll", CLK_IGNORE_UNUSED, ++ GATE(0, "gpll_hclk_perilp1_src", "gpll", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(8), 0, GFLAGS), +- COMPOSITE_NOGATE(HCLK_PERILP1, "hclk_perilp1", mux_hclk_perilp1_p, CLK_IGNORE_UNUSED, ++ COMPOSITE_NOGATE(HCLK_PERILP1, "hclk_perilp1", mux_hclk_perilp1_p, CLK_IS_CRITICAL, + RK3399_CLKSEL_CON(25), 7, 1, MFLAGS, 0, 5, DFLAGS), +- COMPOSITE_NOMUX(PCLK_PERILP1, "pclk_perilp1", "hclk_perilp1", CLK_IGNORE_UNUSED, ++ COMPOSITE_NOMUX(PCLK_PERILP1, "pclk_perilp1", "hclk_perilp1", CLK_IS_CRITICAL, + RK3399_CLKSEL_CON(25), 8, 3, DFLAGS, + RK3399_CLKGATE_CON(8), 2, GFLAGS), + + /* hclk_perilp1 gates */ +- GATE(0, "hclk_perilp1_noc", "hclk_perilp1", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(25), 9, GFLAGS), +- GATE(0, "hclk_sdio_noc", "hclk_perilp1", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(25), 12, GFLAGS), ++ GATE(0, "hclk_perilp1_noc", "hclk_perilp1", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(25), 9, GFLAGS), ++ GATE(0, "hclk_sdio_noc", "hclk_perilp1", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(25), 12, GFLAGS), + GATE(HCLK_I2S0_8CH, "hclk_i2s0", "hclk_perilp1", 0, RK3399_CLKGATE_CON(34), 0, GFLAGS), + GATE(HCLK_I2S1_8CH, "hclk_i2s1", "hclk_perilp1", 0, RK3399_CLKGATE_CON(34), 1, GFLAGS), + GATE(HCLK_I2S2_8CH, "hclk_i2s2", "hclk_perilp1", 0, RK3399_CLKGATE_CON(34), 2, GFLAGS), + GATE(HCLK_SPDIF, "hclk_spdif", "hclk_perilp1", 0, RK3399_CLKGATE_CON(34), 3, GFLAGS), + GATE(HCLK_SDIO, "hclk_sdio", "hclk_perilp1", 0, RK3399_CLKGATE_CON(34), 4, GFLAGS), + GATE(PCLK_SPI5, "pclk_spi5", "hclk_perilp1", 0, RK3399_CLKGATE_CON(34), 5, GFLAGS), +- GATE(0, "hclk_sdioaudio_noc", "hclk_perilp1", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(34), 6, GFLAGS), ++ GATE(0, "hclk_sdioaudio_noc", "hclk_perilp1", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(34), 6, GFLAGS), + + /* pclk_perilp1 gates */ + GATE(PCLK_UART0, "pclk_uart0", "pclk_perilp1", 0, RK3399_CLKGATE_CON(22), 0, GFLAGS), +@@ -1046,7 +1134,7 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { + GATE(PCLK_SPI2, "pclk_spi2", "pclk_perilp1", 0, RK3399_CLKGATE_CON(23), 12, GFLAGS), + GATE(PCLK_SPI4, "pclk_spi4", "pclk_perilp1", 0, RK3399_CLKGATE_CON(23), 13, GFLAGS), + GATE(PCLK_PERIHP_GRF, "pclk_perilp_sgrf", "pclk_perilp1", 0, RK3399_CLKGATE_CON(24), 13, GFLAGS), +- GATE(0, "pclk_perilp1_noc", "pclk_perilp1", 0, RK3399_CLKGATE_CON(25), 10, GFLAGS), ++ GATE(0, "pclk_perilp1_noc", "pclk_perilp1", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(25), 10, GFLAGS), + + /* saradc */ + COMPOSITE_NOMUX(SCLK_SARADC, "clk_saradc", "xin24m", 0, +@@ -1075,24 +1163,23 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { + COMPOSITE(ACLK_VIO, "aclk_vio", mux_pll_src_cpll_gpll_ppll_p, CLK_IGNORE_UNUSED, + RK3399_CLKSEL_CON(42), 6, 2, MFLAGS, 0, 5, DFLAGS, + RK3399_CLKGATE_CON(11), 0, GFLAGS), +- COMPOSITE_NOMUX(PCLK_VIO, "pclk_vio", "aclk_vio", 0, ++ COMPOSITE_NOMUX(PCLK_VIO, "pclk_vio", "aclk_vio", CLK_IS_CRITICAL, + RK3399_CLKSEL_CON(43), 0, 5, DFLAGS, + RK3399_CLKGATE_CON(11), 1, GFLAGS), + +- GATE(ACLK_VIO_NOC, "aclk_vio_noc", "aclk_vio", CLK_IGNORE_UNUSED, ++ GATE(ACLK_VIO_NOC, "aclk_vio_noc", "aclk_vio", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(29), 0, GFLAGS), + + GATE(PCLK_MIPI_DSI0, "pclk_mipi_dsi0", "pclk_vio", 0, + RK3399_CLKGATE_CON(29), 1, GFLAGS), + GATE(PCLK_MIPI_DSI1, "pclk_mipi_dsi1", "pclk_vio", 0, + RK3399_CLKGATE_CON(29), 2, GFLAGS), +- GATE(PCLK_VIO_GRF, "pclk_vio_grf", "pclk_vio", CLK_IGNORE_UNUSED, ++ GATE(PCLK_VIO_GRF, "pclk_vio_grf", "pclk_vio", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(29), 12, GFLAGS), + + /* hdcp */ +- COMPOSITE(ACLK_HDCP, "aclk_hdcp", mux_pll_src_cpll_gpll_ppll_p, 0, +- RK3399_CLKSEL_CON(42), 14, 2, MFLAGS, 8, 5, DFLAGS, +- RK3399_CLKGATE_CON(11), 12, GFLAGS), ++ COMPOSITE_NOGATE(ACLK_HDCP, "aclk_hdcp", mux_pll_src_cpll_gpll_ppll_p, 0, ++ RK3399_CLKSEL_CON(42), 14, 2, MFLAGS, 8, 5, DFLAGS), + COMPOSITE_NOMUX(HCLK_HDCP, "hclk_hdcp", "aclk_hdcp", 0, + RK3399_CLKSEL_CON(43), 5, 5, DFLAGS, + RK3399_CLKGATE_CON(11), 3, GFLAGS), +@@ -1100,17 +1187,17 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { + RK3399_CLKSEL_CON(43), 10, 5, DFLAGS, + RK3399_CLKGATE_CON(11), 10, GFLAGS), + +- GATE(ACLK_HDCP_NOC, "aclk_hdcp_noc", "aclk_hdcp", CLK_IGNORE_UNUSED, ++ GATE(ACLK_HDCP_NOC, "aclk_hdcp_noc", "aclk_hdcp", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(29), 4, GFLAGS), + GATE(ACLK_HDCP22, "aclk_hdcp22", "aclk_hdcp", 0, + RK3399_CLKGATE_CON(29), 10, GFLAGS), + +- GATE(HCLK_HDCP_NOC, "hclk_hdcp_noc", "hclk_hdcp", CLK_IGNORE_UNUSED, ++ GATE(HCLK_HDCP_NOC, "hclk_hdcp_noc", "hclk_hdcp", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(29), 5, GFLAGS), + GATE(HCLK_HDCP22, "hclk_hdcp22", "hclk_hdcp", 0, + RK3399_CLKGATE_CON(29), 9, GFLAGS), + +- GATE(PCLK_HDCP_NOC, "pclk_hdcp_noc", "pclk_hdcp", CLK_IGNORE_UNUSED, ++ GATE(PCLK_HDCP_NOC, "pclk_hdcp_noc", "pclk_hdcp", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(29), 3, GFLAGS), + GATE(PCLK_HDMI_CTRL, "pclk_hdmi_ctrl", "pclk_hdcp", 0, + RK3399_CLKGATE_CON(29), 6, GFLAGS), +@@ -1129,7 +1216,7 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { + COMPOSITE(PCLK_EDP, "pclk_edp", mux_pll_src_cpll_gpll_p, 0, + RK3399_CLKSEL_CON(44), 15, 1, MFLAGS, 8, 6, DFLAGS, + RK3399_CLKGATE_CON(11), 11, GFLAGS), +- GATE(PCLK_EDP_NOC, "pclk_edp_noc", "pclk_edp", CLK_IGNORE_UNUSED, ++ GATE(PCLK_EDP_NOC, "pclk_edp_noc", "pclk_edp", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(32), 12, GFLAGS), + GATE(PCLK_EDP_CTRL, "pclk_edp_ctrl", "pclk_edp", 0, + RK3399_CLKGATE_CON(32), 13, GFLAGS), +@@ -1143,7 +1230,7 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { + RK3399_CLKGATE_CON(11), 7, GFLAGS), + + /* vop0 */ +- COMPOSITE(ACLK_VOP0_PRE, "aclk_vop0_pre", mux_pll_src_vpll_cpll_gpll_npll_p, 0, ++ COMPOSITE(ACLK_VOP0_PRE, "aclk_vop0_pre", mux_pll_src_dmyvpll_cpll_gpll_npll_p, 0, + RK3399_CLKSEL_CON(47), 6, 2, MFLAGS, 0, 5, DFLAGS, + RK3399_CLKGATE_CON(10), 8, GFLAGS), + COMPOSITE_NOMUX(0, "hclk_vop0_pre", "aclk_vop0_pre", 0, +@@ -1152,28 +1239,35 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { + + GATE(ACLK_VOP0, "aclk_vop0", "aclk_vop0_pre", 0, + RK3399_CLKGATE_CON(28), 3, GFLAGS), +- GATE(ACLK_VOP0_NOC, "aclk_vop0_noc", "aclk_vop0_pre", CLK_IGNORE_UNUSED, ++ GATE(ACLK_VOP0_NOC, "aclk_vop0_noc", "aclk_vop0_pre", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(28), 1, GFLAGS), + + GATE(HCLK_VOP0, "hclk_vop0", "hclk_vop0_pre", 0, + RK3399_CLKGATE_CON(28), 2, GFLAGS), +- GATE(HCLK_VOP0_NOC, "hclk_vop0_noc", "hclk_vop0_pre", CLK_IGNORE_UNUSED, ++ GATE(HCLK_VOP0_NOC, "hclk_vop0_noc", "hclk_vop0_pre", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(28), 0, GFLAGS), + +- COMPOSITE(DCLK_VOP0_DIV, "dclk_vop0_div", mux_pll_src_vpll_cpll_gpll_p, 0, ++#ifdef RK3399_TWO_PLL_FOR_VOP ++ COMPOSITE(DCLK_VOP0_DIV, "dclk_vop0_div", mux_pll_src_vpll_cpll_gpll_p, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, ++ RK3399_CLKSEL_CON(49), 8, 2, MFLAGS, 0, 8, DFLAGS, ++ RK3399_CLKGATE_CON(10), 12, GFLAGS), ++#else ++ COMPOSITE(DCLK_VOP0_DIV, "dclk_vop0_div", mux_pll_src_vpll_cpll_gpll_p, CLK_SET_RATE_PARENT, + RK3399_CLKSEL_CON(49), 8, 2, MFLAGS, 0, 8, DFLAGS, + RK3399_CLKGATE_CON(10), 12, GFLAGS), ++#endif + +- COMPOSITE_FRACMUX_NOGATE(DCLK_VOP0_FRAC, "dclk_vop0_frac", "dclk_vop0_div", 0, ++ /* The VOP0 is main screen, it is able to re-set parent rate. */ ++ COMPOSITE_FRACMUX_NOGATE(0, "dclk_vop0_frac", "dclk_vop0_div", CLK_SET_RATE_PARENT, + RK3399_CLKSEL_CON(106), 0, +- &rk3399_dclk_vop0_fracmux), ++ &rk3399_dclk_vop0_fracmux, RK3399_VOP_FRAC_MAX_PRATE), + +- COMPOSITE(SCLK_VOP0_PWM, "clk_vop0_pwm", mux_pll_src_vpll_cpll_gpll_24m_p, 0, ++ COMPOSITE(SCLK_VOP0_PWM, "clk_vop0_pwm", mux_pll_src_dmyvpll_cpll_gpll_gpll_p, 0, + RK3399_CLKSEL_CON(51), 6, 2, MFLAGS, 0, 5, DFLAGS, + RK3399_CLKGATE_CON(10), 14, GFLAGS), + + /* vop1 */ +- COMPOSITE(ACLK_VOP1_PRE, "aclk_vop1_pre", mux_pll_src_vpll_cpll_gpll_npll_p, 0, ++ COMPOSITE(ACLK_VOP1_PRE, "aclk_vop1_pre", mux_pll_src_dmyvpll_cpll_gpll_npll_p, 0, + RK3399_CLKSEL_CON(48), 6, 2, MFLAGS, 0, 5, DFLAGS, + RK3399_CLKGATE_CON(10), 10, GFLAGS), + COMPOSITE_NOMUX(0, "hclk_vop1_pre", "aclk_vop1_pre", 0, +@@ -1182,23 +1276,30 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { + + GATE(ACLK_VOP1, "aclk_vop1", "aclk_vop1_pre", 0, + RK3399_CLKGATE_CON(28), 7, GFLAGS), +- GATE(ACLK_VOP1_NOC, "aclk_vop1_noc", "aclk_vop1_pre", CLK_IGNORE_UNUSED, ++ GATE(ACLK_VOP1_NOC, "aclk_vop1_noc", "aclk_vop1_pre", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(28), 5, GFLAGS), + + GATE(HCLK_VOP1, "hclk_vop1", "hclk_vop1_pre", 0, + RK3399_CLKGATE_CON(28), 6, GFLAGS), +- GATE(HCLK_VOP1_NOC, "hclk_vop1_noc", "hclk_vop1_pre", CLK_IGNORE_UNUSED, ++ GATE(HCLK_VOP1_NOC, "hclk_vop1_noc", "hclk_vop1_pre", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(28), 4, GFLAGS), + +- COMPOSITE(DCLK_VOP1_DIV, "dclk_vop1_div", mux_pll_src_vpll_cpll_gpll_p, 0, ++ /* The VOP1 is sub screen, it is note able to re-set parent rate. */ ++#ifdef RK3399_TWO_PLL_FOR_VOP ++ COMPOSITE(DCLK_VOP1_DIV, "dclk_vop1_div", mux_pll_src_vpll_cpll_gpll_p, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, + RK3399_CLKSEL_CON(50), 8, 2, MFLAGS, 0, 8, DFLAGS, + RK3399_CLKGATE_CON(10), 13, GFLAGS), ++#else ++ COMPOSITE(DCLK_VOP1_DIV, "dclk_vop1_div", mux_pll_src_dmyvpll_cpll_gpll_p, 0, ++ RK3399_CLKSEL_CON(50), 8, 2, MFLAGS, 0, 8, DFLAGS, ++ RK3399_CLKGATE_CON(10), 13, GFLAGS), ++#endif + + COMPOSITE_FRACMUX_NOGATE(DCLK_VOP1_FRAC, "dclk_vop1_frac", "dclk_vop1_div", 0, + RK3399_CLKSEL_CON(107), 0, +- &rk3399_dclk_vop1_fracmux), ++ &rk3399_dclk_vop1_fracmux, RK3399_VOP_FRAC_MAX_PRATE), + +- COMPOSITE(SCLK_VOP1_PWM, "clk_vop1_pwm", mux_pll_src_vpll_cpll_gpll_24m_p, CLK_IGNORE_UNUSED, ++ COMPOSITE(SCLK_VOP1_PWM, "clk_vop1_pwm", mux_pll_src_dmyvpll_cpll_gpll_gpll_p, 0, + RK3399_CLKSEL_CON(52), 6, 2, MFLAGS, 0, 5, DFLAGS, + RK3399_CLKGATE_CON(10), 15, GFLAGS), + +@@ -1210,14 +1311,12 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { + RK3399_CLKSEL_CON(53), 8, 5, DFLAGS, + RK3399_CLKGATE_CON(12), 9, GFLAGS), + +- GATE(ACLK_ISP0_NOC, "aclk_isp0_noc", "aclk_isp0", CLK_IGNORE_UNUSED, ++ GATE(ACLK_ISP0_NOC, "aclk_isp0_noc", "aclk_isp0", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(27), 1, GFLAGS), + GATE(ACLK_ISP0_WRAPPER, "aclk_isp0_wrapper", "aclk_isp0", 0, + RK3399_CLKGATE_CON(27), 5, GFLAGS), +- GATE(HCLK_ISP1_WRAPPER, "hclk_isp1_wrapper", "aclk_isp0", 0, +- RK3399_CLKGATE_CON(27), 7, GFLAGS), + +- GATE(HCLK_ISP0_NOC, "hclk_isp0_noc", "hclk_isp0", CLK_IGNORE_UNUSED, ++ GATE(HCLK_ISP0_NOC, "hclk_isp0_noc", "hclk_isp0", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(27), 0, GFLAGS), + GATE(HCLK_ISP0_WRAPPER, "hclk_isp0_wrapper", "hclk_isp0", 0, + RK3399_CLKGATE_CON(27), 4, GFLAGS), +@@ -1233,13 +1332,15 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { + RK3399_CLKSEL_CON(54), 8, 5, DFLAGS, + RK3399_CLKGATE_CON(12), 11, GFLAGS), + +- GATE(ACLK_ISP1_NOC, "aclk_isp1_noc", "aclk_isp1", CLK_IGNORE_UNUSED, ++ GATE(ACLK_ISP1_NOC, "aclk_isp1_noc", "aclk_isp1", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(27), 3, GFLAGS), ++ GATE(ACLK_ISP1_WRAPPER, "aclk_isp1_wrapper", "aclk_isp1", 0, ++ RK3399_CLKGATE_CON(27), 8, GFLAGS), + +- GATE(HCLK_ISP1_NOC, "hclk_isp1_noc", "hclk_isp1", CLK_IGNORE_UNUSED, ++ GATE(HCLK_ISP1_NOC, "hclk_isp1_noc", "hclk_isp1", CLK_IS_CRITICAL, + RK3399_CLKGATE_CON(27), 2, GFLAGS), +- GATE(ACLK_ISP1_WRAPPER, "aclk_isp1_wrapper", "hclk_isp1", 0, +- RK3399_CLKGATE_CON(27), 8, GFLAGS), ++ GATE(HCLK_ISP1_WRAPPER, "hclk_isp1_wrapper", "hclk_isp1", 0, ++ RK3399_CLKGATE_CON(27), 7, GFLAGS), + + COMPOSITE(SCLK_ISP1, "clk_isp1", mux_pll_src_cpll_gpll_npll_p, 0, + RK3399_CLKSEL_CON(55), 14, 2, MFLAGS, 8, 5, DFLAGS, +@@ -1257,7 +1358,7 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { + RK3399_CLKGATE_CON(27), 6, GFLAGS), + + /* cif */ +- COMPOSITE_NODIV(0, "clk_cifout_src", mux_pll_src_cpll_gpll_npll_p, 0, ++ COMPOSITE_NODIV(SCLK_CIF_OUT_SRC, "clk_cifout_src", mux_pll_src_cpll_gpll_npll_p, 0, + RK3399_CLKSEL_CON(56), 6, 2, MFLAGS, + RK3399_CLKGATE_CON(10), 7, GFLAGS), + +@@ -1265,12 +1366,12 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { + RK3399_CLKSEL_CON(56), 5, 1, MFLAGS, 0, 5, DFLAGS), + + /* gic */ +- COMPOSITE(ACLK_GIC_PRE, "aclk_gic_pre", mux_pll_src_cpll_gpll_p, CLK_IGNORE_UNUSED, ++ COMPOSITE(ACLK_GIC_PRE, "aclk_gic_pre", mux_pll_src_cpll_gpll_p, CLK_IS_CRITICAL, + RK3399_CLKSEL_CON(56), 15, 1, MFLAGS, 8, 5, DFLAGS, + RK3399_CLKGATE_CON(12), 12, GFLAGS), + +- GATE(ACLK_GIC, "aclk_gic", "aclk_gic_pre", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(33), 0, GFLAGS), +- GATE(ACLK_GIC_NOC, "aclk_gic_noc", "aclk_gic_pre", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(33), 1, GFLAGS), ++ GATE(ACLK_GIC, "aclk_gic", "aclk_gic_pre", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(33), 0, GFLAGS), ++ GATE(ACLK_GIC_NOC, "aclk_gic_noc", "aclk_gic_pre", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(33), 1, GFLAGS), + GATE(ACLK_GIC_ADB400_CORE_L_2_GIC, "aclk_gic_adb400_core_l_2_gic", "aclk_gic_pre", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(33), 2, GFLAGS), + GATE(ACLK_GIC_ADB400_CORE_B_2_GIC, "aclk_gic_adb400_core_b_2_gic", "aclk_gic_pre", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(33), 3, GFLAGS), + GATE(ACLK_GIC_ADB400_GIC_2_CORE_L, "aclk_gic_adb400_gic_2_core_l", "aclk_gic_pre", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(33), 4, GFLAGS), +@@ -1301,19 +1402,19 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { + SGRF_GATE(PCLK_WDT, "pclk_wdt", "pclk_alive"), + + GATE(SCLK_MIPIDPHY_REF, "clk_mipidphy_ref", "xin24m", 0, RK3399_CLKGATE_CON(11), 14, GFLAGS), +- GATE(SCLK_DPHY_PLL, "clk_dphy_pll", "clk_mipidphy_ref", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(21), 0, GFLAGS), ++ GATE(SCLK_DPHY_PLL, "clk_dphy_pll", "clk_mipidphy_ref", 0, RK3399_CLKGATE_CON(21), 0, GFLAGS), + + GATE(SCLK_MIPIDPHY_CFG, "clk_mipidphy_cfg", "xin24m", 0, RK3399_CLKGATE_CON(11), 15, GFLAGS), +- GATE(SCLK_DPHY_TX0_CFG, "clk_dphy_tx0_cfg", "clk_mipidphy_cfg", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(21), 1, GFLAGS), +- GATE(SCLK_DPHY_TX1RX1_CFG, "clk_dphy_tx1rx1_cfg", "clk_mipidphy_cfg", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(21), 2, GFLAGS), +- GATE(SCLK_DPHY_RX0_CFG, "clk_dphy_rx0_cfg", "clk_mipidphy_cfg", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(21), 3, GFLAGS), ++ GATE(SCLK_DPHY_TX0_CFG, "clk_dphy_tx0_cfg", "clk_mipidphy_cfg", 0, RK3399_CLKGATE_CON(21), 1, GFLAGS), ++ GATE(SCLK_DPHY_TX1RX1_CFG, "clk_dphy_tx1rx1_cfg", "clk_mipidphy_cfg", 0, RK3399_CLKGATE_CON(21), 2, GFLAGS), ++ GATE(SCLK_DPHY_RX0_CFG, "clk_dphy_rx0_cfg", "clk_mipidphy_cfg", 0, RK3399_CLKGATE_CON(21), 3, GFLAGS), + + /* testout */ + MUX(0, "clk_test_pre", mux_pll_src_cpll_gpll_p, CLK_SET_RATE_PARENT, + RK3399_CLKSEL_CON(58), 7, 1, MFLAGS), + COMPOSITE_FRAC(0, "clk_test_frac", "clk_test_pre", 0, + RK3399_CLKSEL_CON(105), 0, +- RK3399_CLKGATE_CON(13), 9, GFLAGS), ++ RK3399_CLKGATE_CON(13), 9, GFLAGS, 0), + + DIV(0, "clk_test_24m", "xin24m", 0, + RK3399_CLKSEL_CON(57), 6, 10, DFLAGS), +@@ -1385,13 +1486,13 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { + RK3399_CLKGATE_CON(13), 11, GFLAGS), + + /* ddrc */ +- GATE(0, "clk_ddrc_lpll_src", "lpll", 0, RK3399_CLKGATE_CON(3), ++ GATE(0, "clk_ddrc_lpll_src", "lpll", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(3), + 0, GFLAGS), +- GATE(0, "clk_ddrc_bpll_src", "bpll", 0, RK3399_CLKGATE_CON(3), ++ GATE(0, "clk_ddrc_bpll_src", "bpll", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(3), + 1, GFLAGS), +- GATE(0, "clk_ddrc_dpll_src", "dpll", 0, RK3399_CLKGATE_CON(3), ++ GATE(0, "clk_ddrc_dpll_src", "dpll", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(3), + 2, GFLAGS), +- GATE(0, "clk_ddrc_gpll_src", "gpll", 0, RK3399_CLKGATE_CON(3), ++ GATE(0, "clk_ddrc_gpll_src", "gpll", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(3), + 3, GFLAGS), + COMPOSITE_DDRCLK(SCLK_DDRC, "sclk_ddrc", mux_ddrclk_p, 0, + RK3399_CLKSEL_CON(6), 4, 2, 0, 0, ROCKCHIP_DDRCLK_SIP), +@@ -1402,10 +1503,10 @@ static struct rockchip_clk_branch rk3399_clk_pmu_branches[] __initdata = { + * PMU CRU Clock-Architecture + */ + +- GATE(0, "fclk_cm0s_pmu_ppll_src", "ppll", 0, ++ GATE(0, "fclk_cm0s_pmu_ppll_src", "ppll", CLK_IS_CRITICAL, + RK3399_PMU_CLKGATE_CON(0), 1, GFLAGS), + +- COMPOSITE_NOGATE(FCLK_CM0S_SRC_PMU, "fclk_cm0s_src_pmu", mux_fclk_cm0s_pmu_ppll_p, 0, ++ COMPOSITE_NOGATE(FCLK_CM0S_SRC_PMU, "fclk_cm0s_src_pmu", mux_fclk_cm0s_pmu_ppll_p, CLK_IS_CRITICAL, + RK3399_PMU_CLKSEL_CON(0), 15, 1, MFLAGS, 8, 5, DFLAGS), + + COMPOSITE(SCLK_SPI3_PMU, "clk_spi3_pmu", mux_24m_ppll_p, 0, +@@ -1416,9 +1517,9 @@ static struct rockchip_clk_branch rk3399_clk_pmu_branches[] __initdata = { + RK3399_PMU_CLKSEL_CON(1), 13, 1, MFLAGS, 8, 5, DFLAGS, + RK3399_PMU_CLKGATE_CON(0), 8, GFLAGS), + +- COMPOSITE_FRACMUX_NOGATE(0, "clk_wifi_frac", "clk_wifi_div", 0, ++ COMPOSITE_FRACMUX_NOGATE(0, "clk_wifi_frac", "clk_wifi_div", CLK_SET_RATE_PARENT, + RK3399_PMU_CLKSEL_CON(7), 0, +- &rk3399_pmuclk_wifi_fracmux), ++ &rk3399_pmuclk_wifi_fracmux, RK3399_WIFI_FRAC_MAX_PRATE), + + MUX(0, "clk_timer_src_pmu", mux_pll_p, CLK_IGNORE_UNUSED, + RK3399_PMU_CLKSEL_CON(1), 15, 1, MFLAGS), +@@ -1440,23 +1541,26 @@ static struct rockchip_clk_branch rk3399_clk_pmu_branches[] __initdata = { + MUX(0, "clk_testout_2io", mux_clk_testout2_2io_p, CLK_IGNORE_UNUSED, + RK3399_PMU_CLKSEL_CON(4), 15, 1, MFLAGS), + +- COMPOSITE(0, "clk_uart4_div", mux_24m_ppll_p, 0, +- RK3399_PMU_CLKSEL_CON(5), 10, 1, MFLAGS, 0, 7, DFLAGS, ++ MUX(SCLK_UART4_SRC, "clk_uart4_src", mux_24m_ppll_p, CLK_SET_RATE_NO_REPARENT, ++ RK3399_PMU_CLKSEL_CON(5), 10, 1, MFLAGS), ++ ++ COMPOSITE_NOMUX(0, "clk_uart4_div", "clk_uart4_src", CLK_SET_RATE_PARENT, ++ RK3399_PMU_CLKSEL_CON(5), 0, 7, DFLAGS, + RK3399_PMU_CLKGATE_CON(0), 5, GFLAGS), + +- COMPOSITE_FRACMUX(0, "clk_uart4_frac", "clk_uart4_div", 0, ++ COMPOSITE_FRACMUX(0, "clk_uart4_frac", "clk_uart4_div", CLK_SET_RATE_PARENT, + RK3399_PMU_CLKSEL_CON(6), 0, + RK3399_PMU_CLKGATE_CON(0), 6, GFLAGS, +- &rk3399_uart4_pmu_fracmux), ++ &rk3399_uart4_pmu_fracmux, RK3399_UART_FRAC_MAX_PRATE), + +- DIV(PCLK_SRC_PMU, "pclk_pmu_src", "ppll", CLK_IGNORE_UNUSED, ++ DIV(PCLK_SRC_PMU, "pclk_pmu_src", "ppll", CLK_IS_CRITICAL, + RK3399_PMU_CLKSEL_CON(0), 0, 5, DFLAGS), + + /* pmu clock gates */ + GATE(SCLK_TIMER12_PMU, "clk_timer0_pmu", "clk_timer_src_pmu", 0, RK3399_PMU_CLKGATE_CON(0), 3, GFLAGS), + GATE(SCLK_TIMER13_PMU, "clk_timer1_pmu", "clk_timer_src_pmu", 0, RK3399_PMU_CLKGATE_CON(0), 4, GFLAGS), + +- GATE(SCLK_PVTM_PMU, "clk_pvtm_pmu", "xin24m", CLK_IGNORE_UNUSED, RK3399_PMU_CLKGATE_CON(0), 7, GFLAGS), ++ GATE(SCLK_PVTM_PMU, "clk_pvtm_pmu", "xin24m", 0, RK3399_PMU_CLKGATE_CON(0), 7, GFLAGS), + + GATE(PCLK_PMU, "pclk_pmu", "pclk_pmu_src", CLK_IGNORE_UNUSED, RK3399_PMU_CLKGATE_CON(1), 0, GFLAGS), + GATE(PCLK_PMUGRF_PMU, "pclk_pmugrf_pmu", "pclk_pmu_src", CLK_IGNORE_UNUSED, RK3399_PMU_CLKGATE_CON(1), 1, GFLAGS), +@@ -1464,69 +1568,60 @@ static struct rockchip_clk_branch rk3399_clk_pmu_branches[] __initdata = { + GATE(PCLK_GPIO0_PMU, "pclk_gpio0_pmu", "pclk_pmu_src", 0, RK3399_PMU_CLKGATE_CON(1), 3, GFLAGS), + GATE(PCLK_GPIO1_PMU, "pclk_gpio1_pmu", "pclk_pmu_src", 0, RK3399_PMU_CLKGATE_CON(1), 4, GFLAGS), + GATE(PCLK_SGRF_PMU, "pclk_sgrf_pmu", "pclk_pmu_src", CLK_IGNORE_UNUSED, RK3399_PMU_CLKGATE_CON(1), 5, GFLAGS), +- GATE(PCLK_NOC_PMU, "pclk_noc_pmu", "pclk_pmu_src", CLK_IGNORE_UNUSED, RK3399_PMU_CLKGATE_CON(1), 6, GFLAGS), ++ GATE(PCLK_NOC_PMU, "pclk_noc_pmu", "pclk_pmu_src", CLK_IS_CRITICAL, RK3399_PMU_CLKGATE_CON(1), 6, GFLAGS), + GATE(PCLK_I2C0_PMU, "pclk_i2c0_pmu", "pclk_pmu_src", 0, RK3399_PMU_CLKGATE_CON(1), 7, GFLAGS), + GATE(PCLK_I2C4_PMU, "pclk_i2c4_pmu", "pclk_pmu_src", 0, RK3399_PMU_CLKGATE_CON(1), 8, GFLAGS), + GATE(PCLK_I2C8_PMU, "pclk_i2c8_pmu", "pclk_pmu_src", 0, RK3399_PMU_CLKGATE_CON(1), 9, GFLAGS), +- GATE(PCLK_RKPWM_PMU, "pclk_rkpwm_pmu", "pclk_pmu_src", 0, RK3399_PMU_CLKGATE_CON(1), 10, GFLAGS), ++ GATE(PCLK_RKPWM_PMU, "pclk_rkpwm_pmu", "pclk_pmu_src", CLK_IS_CRITICAL, RK3399_PMU_CLKGATE_CON(1), 10, GFLAGS), + GATE(PCLK_SPI3_PMU, "pclk_spi3_pmu", "pclk_pmu_src", 0, RK3399_PMU_CLKGATE_CON(1), 11, GFLAGS), + GATE(PCLK_TIMER_PMU, "pclk_timer_pmu", "pclk_pmu_src", 0, RK3399_PMU_CLKGATE_CON(1), 12, GFLAGS), + GATE(PCLK_MAILBOX_PMU, "pclk_mailbox_pmu", "pclk_pmu_src", 0, RK3399_PMU_CLKGATE_CON(1), 13, GFLAGS), + GATE(PCLK_UART4_PMU, "pclk_uart4_pmu", "pclk_pmu_src", 0, RK3399_PMU_CLKGATE_CON(1), 14, GFLAGS), + GATE(PCLK_WDT_M0_PMU, "pclk_wdt_m0_pmu", "pclk_pmu_src", 0, RK3399_PMU_CLKGATE_CON(1), 15, GFLAGS), + +- GATE(FCLK_CM0S_PMU, "fclk_cm0s_pmu", "fclk_cm0s_src_pmu", CLK_IGNORE_UNUSED, RK3399_PMU_CLKGATE_CON(2), 0, GFLAGS), +- GATE(SCLK_CM0S_PMU, "sclk_cm0s_pmu", "fclk_cm0s_src_pmu", CLK_IGNORE_UNUSED, RK3399_PMU_CLKGATE_CON(2), 1, GFLAGS), +- GATE(HCLK_CM0S_PMU, "hclk_cm0s_pmu", "fclk_cm0s_src_pmu", CLK_IGNORE_UNUSED, RK3399_PMU_CLKGATE_CON(2), 2, GFLAGS), +- GATE(DCLK_CM0S_PMU, "dclk_cm0s_pmu", "fclk_cm0s_src_pmu", CLK_IGNORE_UNUSED, RK3399_PMU_CLKGATE_CON(2), 3, GFLAGS), +- GATE(HCLK_NOC_PMU, "hclk_noc_pmu", "fclk_cm0s_src_pmu", CLK_IGNORE_UNUSED, RK3399_PMU_CLKGATE_CON(2), 5, GFLAGS), ++ GATE(FCLK_CM0S_PMU, "fclk_cm0s_pmu", "fclk_cm0s_src_pmu", 0, RK3399_PMU_CLKGATE_CON(2), 0, GFLAGS), ++ GATE(SCLK_CM0S_PMU, "sclk_cm0s_pmu", "fclk_cm0s_src_pmu", 0, RK3399_PMU_CLKGATE_CON(2), 1, GFLAGS), ++ GATE(HCLK_CM0S_PMU, "hclk_cm0s_pmu", "fclk_cm0s_src_pmu", 0, RK3399_PMU_CLKGATE_CON(2), 2, GFLAGS), ++ GATE(DCLK_CM0S_PMU, "dclk_cm0s_pmu", "fclk_cm0s_src_pmu", 0, RK3399_PMU_CLKGATE_CON(2), 3, GFLAGS), ++ GATE(HCLK_NOC_PMU, "hclk_noc_pmu", "fclk_cm0s_src_pmu", CLK_IS_CRITICAL, RK3399_PMU_CLKGATE_CON(2), 5, GFLAGS), + }; + +-static const char *const rk3399_cru_critical_clocks[] __initconst = { +- "aclk_cci_pre", +- "aclk_gic", +- "aclk_gic_noc", +- "aclk_hdcp_noc", +- "hclk_hdcp_noc", +- "pclk_hdcp_noc", +- "pclk_perilp0", +- "pclk_perilp0", +- "hclk_perilp0", +- "hclk_perilp0_noc", +- "pclk_perilp1", +- "pclk_perilp1_noc", +- "pclk_perihp", +- "pclk_perihp_noc", +- "hclk_perihp", +- "aclk_perihp", +- "aclk_perihp_noc", +- "aclk_perilp0", +- "aclk_perilp0_noc", +- "hclk_perilp1", +- "hclk_perilp1_noc", +- "aclk_dmac0_perilp", +- "aclk_emmc_noc", +- "gpll_hclk_perilp1_src", +- "gpll_aclk_perilp0_src", +- "gpll_aclk_perihp_src", +- "aclk_vio_noc", ++static void __iomem *rk3399_cru_base; ++static void __iomem *rk3399_pmucru_base; + +- /* ddrc */ +- "sclk_ddrc" +-}; ++void rk3399_dump_cru(void) ++{ ++ if (rk3399_cru_base) { ++ pr_warn("CRU:\n"); ++ print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET, ++ 32, 4, rk3399_cru_base, ++ 0x594, false); ++ } ++ if (rk3399_pmucru_base) { ++ pr_warn("PMU CRU:\n"); ++ print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET, ++ 32, 4, rk3399_pmucru_base, ++ 0x134, false); ++ } ++} ++EXPORT_SYMBOL_GPL(rk3399_dump_cru); ++ ++static int rk3399_clk_panic(struct notifier_block *this, ++ unsigned long ev, void *ptr) ++{ ++ rk3399_dump_cru(); ++ return NOTIFY_DONE; ++} + +-static const char *const rk3399_pmucru_critical_clocks[] __initconst = { +- "ppll", +- "pclk_pmu_src", +- "fclk_cm0s_src_pmu", +- "clk_timer_src_pmu", +- "pclk_rkpwm_pmu", ++static struct notifier_block rk3399_clk_panic_block = { ++ .notifier_call = rk3399_clk_panic, + }; + + static void __init rk3399_clk_init(struct device_node *np) + { + struct rockchip_clk_provider *ctx; + void __iomem *reg_base; ++ struct clk **clks; + + reg_base = of_iomap(np, 0); + if (!reg_base) { +@@ -1534,12 +1629,15 @@ static void __init rk3399_clk_init(struct device_node *np) + return; + } + ++ rk3399_cru_base = reg_base; ++ + ctx = rockchip_clk_init(np, reg_base, CLK_NR_CLKS); + if (IS_ERR(ctx)) { + pr_err("%s: rockchip clk init failed\n", __func__); + iounmap(reg_base); + return; + } ++ clks = ctx->clk_data.clks; + + rockchip_clk_register_plls(ctx, rk3399_pll_clks, + ARRAY_SIZE(rk3399_pll_clks), -1); +@@ -1547,16 +1645,13 @@ static void __init rk3399_clk_init(struct device_node *np) + rockchip_clk_register_branches(ctx, rk3399_clk_branches, + ARRAY_SIZE(rk3399_clk_branches)); + +- rockchip_clk_protect_critical(rk3399_cru_critical_clocks, +- ARRAY_SIZE(rk3399_cru_critical_clocks)); +- + rockchip_clk_register_armclk(ctx, ARMCLKL, "armclkl", +- mux_armclkl_p, ARRAY_SIZE(mux_armclkl_p), ++ 4, clks[PLL_APLLL], clks[PLL_GPLL], + &rk3399_cpuclkl_data, rk3399_cpuclkl_rates, + ARRAY_SIZE(rk3399_cpuclkl_rates)); + + rockchip_clk_register_armclk(ctx, ARMCLKB, "armclkb", +- mux_armclkb_p, ARRAY_SIZE(mux_armclkb_p), ++ 4, clks[PLL_APLLB], clks[PLL_GPLL], + &rk3399_cpuclkb_data, rk3399_cpuclkb_rates, + ARRAY_SIZE(rk3399_cpuclkb_rates)); + +@@ -1580,6 +1675,8 @@ static void __init rk3399_pmu_clk_init(struct device_node *np) + return; + } + ++ rk3399_pmucru_base = reg_base; ++ + ctx = rockchip_clk_init(np, reg_base, CLKPMU_NR_CLKS); + if (IS_ERR(ctx)) { + pr_err("%s: rockchip pmu clk init failed\n", __func__); +@@ -1593,13 +1690,13 @@ static void __init rk3399_pmu_clk_init(struct device_node *np) + rockchip_clk_register_branches(ctx, rk3399_clk_pmu_branches, + ARRAY_SIZE(rk3399_clk_pmu_branches)); + +- rockchip_clk_protect_critical(rk3399_pmucru_critical_clocks, +- ARRAY_SIZE(rk3399_pmucru_critical_clocks)); +- + rockchip_register_softrst(np, 2, reg_base + RK3399_PMU_SOFTRST_CON(0), + ROCKCHIP_SOFTRST_HIWORD_MASK); + + rockchip_clk_of_add_provider(np, ctx); ++ ++ atomic_notifier_chain_register(&panic_notifier_list, ++ &rk3399_clk_panic_block); + } + CLK_OF_DECLARE(rk3399_cru_pmu, "rockchip,rk3399-pmucru", rk3399_pmu_clk_init); + +diff --git a/drivers/clk/rockchip/clk-rk3568.c b/drivers/clk/rockchip/clk-rk3568.c +new file mode 100755 +index 000000000000..44ce9cc542ce +--- /dev/null ++++ b/drivers/clk/rockchip/clk-rk3568.c +@@ -0,0 +1,1757 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co. Ltd. ++ * Author: Elaine Zhang ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include "clk.h" ++ ++#define RK3568_GRF_SOC_CON1 0x504 ++#define RK3568_GRF_SOC_CON2 0x508 ++#define RK3568_GRF_SOC_STATUS0 0x580 ++#define RK3568_PMU_GRF_SOC_CON0 0x100 ++ ++#define RK3568_FRAC_MAX_PRATE 1000000000 ++#define RK3568_SPDIF_FRAC_MAX_PRATE 600000000 ++#define RK3568_UART_FRAC_MAX_PRATE 600000000 ++#define RK3568_DCLK_PARENT_MAX_PRATE 600000000 ++ ++enum rk3568_pmu_plls { ++ ppll, hpll, ++}; ++ ++enum rk3568_plls { ++ apll, dpll, gpll, cpll, npll, vpll, ++}; ++ ++static struct rockchip_pll_rate_table rk3568_pll_rates[] = { ++ /* _mhz, _refdiv, _fbdiv, _postdiv1, _postdiv2, _dsmpd, _frac */ ++ RK3036_PLL_RATE(2208000000, 1, 92, 1, 1, 1, 0), ++ RK3036_PLL_RATE(2184000000, 1, 91, 1, 1, 1, 0), ++ RK3036_PLL_RATE(2160000000, 1, 90, 1, 1, 1, 0), ++ RK3036_PLL_RATE(2088000000, 1, 87, 1, 1, 1, 0), ++ RK3036_PLL_RATE(2064000000, 1, 86, 1, 1, 1, 0), ++ RK3036_PLL_RATE(2040000000, 1, 85, 1, 1, 1, 0), ++ RK3036_PLL_RATE(2016000000, 1, 84, 1, 1, 1, 0), ++ RK3036_PLL_RATE(1992000000, 1, 83, 1, 1, 1, 0), ++ RK3036_PLL_RATE(1920000000, 1, 80, 1, 1, 1, 0), ++ RK3036_PLL_RATE(1896000000, 1, 79, 1, 1, 1, 0), ++ RK3036_PLL_RATE(1800000000, 1, 75, 1, 1, 1, 0), ++ RK3036_PLL_RATE(1704000000, 1, 71, 1, 1, 1, 0), ++ RK3036_PLL_RATE(1608000000, 1, 67, 1, 1, 1, 0), ++ RK3036_PLL_RATE(1600000000, 3, 200, 1, 1, 1, 0), ++ RK3036_PLL_RATE(1584000000, 1, 132, 2, 1, 1, 0), ++ RK3036_PLL_RATE(1560000000, 1, 130, 2, 1, 1, 0), ++ RK3036_PLL_RATE(1536000000, 1, 128, 2, 1, 1, 0), ++ RK3036_PLL_RATE(1512000000, 1, 126, 2, 1, 1, 0), ++ RK3036_PLL_RATE(1488000000, 1, 124, 2, 1, 1, 0), ++ RK3036_PLL_RATE(1464000000, 1, 122, 2, 1, 1, 0), ++ RK3036_PLL_RATE(1440000000, 1, 120, 2, 1, 1, 0), ++ RK3036_PLL_RATE(1416000000, 1, 118, 2, 1, 1, 0), ++ RK3036_PLL_RATE(1400000000, 3, 350, 2, 1, 1, 0), ++ RK3036_PLL_RATE(1392000000, 1, 116, 2, 1, 1, 0), ++ RK3036_PLL_RATE(1368000000, 1, 114, 2, 1, 1, 0), ++ RK3036_PLL_RATE(1344000000, 1, 112, 2, 1, 1, 0), ++ RK3036_PLL_RATE(1320000000, 1, 110, 2, 1, 1, 0), ++ RK3036_PLL_RATE(1296000000, 1, 108, 2, 1, 1, 0), ++ RK3036_PLL_RATE(1272000000, 1, 106, 2, 1, 1, 0), ++ RK3036_PLL_RATE(1248000000, 1, 104, 2, 1, 1, 0), ++ RK3036_PLL_RATE(1200000000, 1, 100, 2, 1, 1, 0), ++ RK3036_PLL_RATE(1188000000, 1, 99, 2, 1, 1, 0), ++ RK3036_PLL_RATE(1104000000, 1, 92, 2, 1, 1, 0), ++ RK3036_PLL_RATE(1100000000, 3, 275, 2, 1, 1, 0), ++ RK3036_PLL_RATE(1008000000, 1, 84, 2, 1, 1, 0), ++ RK3036_PLL_RATE(1000000000, 3, 250, 2, 1, 1, 0), ++ RK3036_PLL_RATE(912000000, 1, 76, 2, 1, 1, 0), ++ RK3036_PLL_RATE(816000000, 1, 68, 2, 1, 1, 0), ++ RK3036_PLL_RATE(800000000, 3, 200, 2, 1, 1, 0), ++ RK3036_PLL_RATE(700000000, 3, 350, 4, 1, 1, 0), ++ RK3036_PLL_RATE(696000000, 1, 116, 4, 1, 1, 0), ++ RK3036_PLL_RATE(600000000, 1, 100, 4, 1, 1, 0), ++ RK3036_PLL_RATE(594000000, 1, 99, 4, 1, 1, 0), ++ RK3036_PLL_RATE(500000000, 1, 125, 6, 1, 1, 0), ++ RK3036_PLL_RATE(408000000, 1, 68, 2, 2, 1, 0), ++ RK3036_PLL_RATE(312000000, 1, 78, 6, 1, 1, 0), ++ RK3036_PLL_RATE(216000000, 1, 72, 4, 2, 1, 0), ++ RK3036_PLL_RATE(200000000, 1, 100, 3, 4, 1, 0), ++ RK3036_PLL_RATE(148500000, 1, 99, 4, 4, 1, 0), ++ RK3036_PLL_RATE(100000000, 1, 150, 6, 6, 1, 0), ++ RK3036_PLL_RATE(96000000, 1, 96, 6, 4, 1, 0), ++ RK3036_PLL_RATE(74250000, 2, 99, 4, 4, 1, 0), ++ { /* sentinel */ }, ++}; ++ ++#define RK3568_DIV_ATCLK_CORE_MASK 0x1f ++#define RK3568_DIV_ATCLK_CORE_SHIFT 0 ++#define RK3568_DIV_GICCLK_CORE_MASK 0x1f ++#define RK3568_DIV_GICCLK_CORE_SHIFT 8 ++#define RK3568_DIV_PCLK_CORE_MASK 0x1f ++#define RK3568_DIV_PCLK_CORE_SHIFT 0 ++#define RK3568_DIV_PERIPHCLK_CORE_MASK 0x1f ++#define RK3568_DIV_PERIPHCLK_CORE_SHIFT 8 ++#define RK3568_DIV_ACLK_CORE_MASK 0x1f ++#define RK3568_DIV_ACLK_CORE_SHIFT 8 ++ ++#define RK3568_DIV_SCLK_CORE_MASK 0xf ++#define RK3568_DIV_SCLK_CORE_SHIFT 0 ++#define RK3568_MUX_SCLK_CORE_MASK 0x3 ++#define RK3568_MUX_SCLK_CORE_SHIFT 8 ++#define RK3568_MUX_SCLK_CORE_NPLL_MASK 0x1 ++#define RK3568_MUX_SCLK_CORE_NPLL_SHIFT 15 ++#define RK3568_MUX_CLK_CORE_APLL_MASK 0x1 ++#define RK3568_MUX_CLK_CORE_APLL_SHIFT 7 ++#define RK3568_MUX_CLK_PVTPLL_MASK 0x1 ++#define RK3568_MUX_CLK_PVTPLL_SHIFT 15 ++ ++#define RK3568_CLKSEL1(_sclk_core) \ ++{ \ ++ .reg = RK3568_CLKSEL_CON(2), \ ++ .val = HIWORD_UPDATE(_sclk_core, RK3568_MUX_SCLK_CORE_NPLL_MASK, \ ++ RK3568_MUX_SCLK_CORE_NPLL_SHIFT) | \ ++ HIWORD_UPDATE(_sclk_core, RK3568_MUX_SCLK_CORE_MASK, \ ++ RK3568_MUX_SCLK_CORE_SHIFT) | \ ++ HIWORD_UPDATE(1, RK3568_DIV_SCLK_CORE_MASK, \ ++ RK3568_DIV_SCLK_CORE_SHIFT), \ ++} ++ ++#define RK3568_CLKSEL2(_aclk_core) \ ++{ \ ++ .reg = RK3568_CLKSEL_CON(5), \ ++ .val = HIWORD_UPDATE(_aclk_core, RK3568_DIV_ACLK_CORE_MASK, \ ++ RK3568_DIV_ACLK_CORE_SHIFT), \ ++} ++ ++#define RK3568_CLKSEL3(_atclk_core, _gic_core) \ ++{ \ ++ .reg = RK3568_CLKSEL_CON(3), \ ++ .val = HIWORD_UPDATE(_atclk_core, RK3568_DIV_ATCLK_CORE_MASK, \ ++ RK3568_DIV_ATCLK_CORE_SHIFT) | \ ++ HIWORD_UPDATE(_gic_core, RK3568_DIV_GICCLK_CORE_MASK, \ ++ RK3568_DIV_GICCLK_CORE_SHIFT), \ ++} ++ ++#define RK3568_CLKSEL4(_pclk_core, _periph_core) \ ++{ \ ++ .reg = RK3568_CLKSEL_CON(4), \ ++ .val = HIWORD_UPDATE(_pclk_core, RK3568_DIV_PCLK_CORE_MASK, \ ++ RK3568_DIV_PCLK_CORE_SHIFT) | \ ++ HIWORD_UPDATE(_periph_core, RK3568_DIV_PERIPHCLK_CORE_MASK, \ ++ RK3568_DIV_PERIPHCLK_CORE_SHIFT), \ ++} ++ ++#define RK3568_CPUCLK_RATE(_prate, _sclk, _acore, _atcore, _gicclk, _pclk, _periph) \ ++{ \ ++ .prate = _prate##U, \ ++ .divs = { \ ++ RK3568_CLKSEL1(_sclk), \ ++ RK3568_CLKSEL2(_acore), \ ++ RK3568_CLKSEL3(_atcore, _gicclk), \ ++ RK3568_CLKSEL4(_pclk, _periph), \ ++ }, \ ++} ++ ++static struct rockchip_cpuclk_rate_table rk3568_cpuclk_rates[] __initdata = { ++ RK3568_CPUCLK_RATE(1800000000, 0, 1, 7, 7, 7, 7), ++ RK3568_CPUCLK_RATE(1704000000, 0, 1, 7, 7, 7, 7), ++ RK3568_CPUCLK_RATE(1608000000, 0, 1, 5, 5, 5, 5), ++ RK3568_CPUCLK_RATE(1584000000, 0, 1, 5, 5, 5, 5), ++ RK3568_CPUCLK_RATE(1560000000, 0, 1, 5, 5, 5, 5), ++ RK3568_CPUCLK_RATE(1536000000, 0, 1, 5, 5, 5, 5), ++ RK3568_CPUCLK_RATE(1512000000, 0, 1, 5, 5, 5, 5), ++ RK3568_CPUCLK_RATE(1488000000, 0, 1, 5, 5, 5, 5), ++ RK3568_CPUCLK_RATE(1464000000, 0, 1, 5, 5, 5, 5), ++ RK3568_CPUCLK_RATE(1440000000, 0, 1, 5, 5, 5, 5), ++ RK3568_CPUCLK_RATE(1416000000, 0, 1, 5, 5, 5, 5), ++ RK3568_CPUCLK_RATE(1392000000, 0, 1, 5, 5, 5, 5), ++ RK3568_CPUCLK_RATE(1368000000, 0, 1, 5, 5, 5, 5), ++ RK3568_CPUCLK_RATE(1344000000, 0, 1, 5, 5, 5, 5), ++ RK3568_CPUCLK_RATE(1320000000, 0, 1, 5, 5, 5, 5), ++ RK3568_CPUCLK_RATE(1296000000, 0, 1, 5, 5, 5, 5), ++ RK3568_CPUCLK_RATE(1272000000, 0, 1, 5, 5, 5, 5), ++ RK3568_CPUCLK_RATE(1248000000, 0, 1, 5, 5, 5, 5), ++ RK3568_CPUCLK_RATE(1224000000, 0, 1, 5, 5, 5, 5), ++ RK3568_CPUCLK_RATE(1200000000, 0, 1, 3, 3, 3, 3), ++ RK3568_CPUCLK_RATE(1104000000, 0, 1, 3, 3, 3, 3), ++ RK3568_CPUCLK_RATE(1008000000, 0, 1, 3, 3, 3, 3), ++ RK3568_CPUCLK_RATE(912000000, 0, 1, 3, 3, 3, 3), ++ RK3568_CPUCLK_RATE(816000000, 0, 1, 3, 3, 3, 3), ++ RK3568_CPUCLK_RATE(696000000, 0, 1, 3, 3, 3, 3), ++ RK3568_CPUCLK_RATE(600000000, 0, 1, 3, 3, 3, 3), ++ RK3568_CPUCLK_RATE(408000000, 0, 1, 3, 3, 3, 3), ++ RK3568_CPUCLK_RATE(312000000, 0, 1, 3, 3, 3, 3), ++ RK3568_CPUCLK_RATE(216000000, 0, 1, 3, 3, 3, 3), ++ RK3568_CPUCLK_RATE(96000000, 0, 1, 3, 3, 3, 3), ++}; ++ ++static const struct rockchip_cpuclk_reg_data rk3568_cpuclk_data = { ++ .core_reg[0] = RK3568_CLKSEL_CON(0), ++ .div_core_shift[0] = 0, ++ .div_core_mask[0] = 0x1f, ++ .core_reg[1] = RK3568_CLKSEL_CON(0), ++ .div_core_shift[1] = 8, ++ .div_core_mask[1] = 0x1f, ++ .core_reg[2] = RK3568_CLKSEL_CON(1), ++ .div_core_shift[2] = 0, ++ .div_core_mask[2] = 0x1f, ++ .core_reg[3] = RK3568_CLKSEL_CON(1), ++ .div_core_shift[3] = 8, ++ .div_core_mask[3] = 0x1f, ++ .num_cores = 4, ++ .mux_core_alt = 1, ++ .mux_core_main = 0, ++ .mux_core_shift = 6, ++ .mux_core_mask = 0x1, ++}; ++ ++PNAME(mux_pll_p) = { "xin24m" }; ++PNAME(mux_usb480m_p) = { "xin24m", "usb480m_phy", "clk_rtc_32k" }; ++PNAME(clk_i2s0_8ch_tx_p) = { "clk_i2s0_8ch_tx_src", "clk_i2s0_8ch_tx_frac", "i2s0_mclkin", "xin_osc0_half" }; ++PNAME(clk_i2s0_8ch_rx_p) = { "clk_i2s0_8ch_rx_src", "clk_i2s0_8ch_rx_frac", "i2s0_mclkin", "xin_osc0_half" }; ++PNAME(clk_i2s1_8ch_tx_p) = { "clk_i2s1_8ch_tx_src", "clk_i2s1_8ch_tx_frac", "i2s1_mclkin", "xin_osc0_half" }; ++PNAME(clk_i2s1_8ch_rx_p) = { "clk_i2s1_8ch_rx_src", "clk_i2s1_8ch_rx_frac", "i2s1_mclkin", "xin_osc0_half" }; ++PNAME(clk_i2s2_2ch_p) = { "clk_i2s2_2ch_src", "clk_i2s2_2ch_frac", "i2s2_mclkin", "xin_osc0_half "}; ++PNAME(clk_i2s3_2ch_tx_p) = { "clk_i2s3_2ch_tx_src", "clk_i2s3_2ch_tx_frac", "i2s3_mclkin", "xin_osc0_half" }; ++PNAME(clk_i2s3_2ch_rx_p) = { "clk_i2s3_2ch_rx_src", "clk_i2s3_2ch_rx_frac", "i2s3_mclkin", "xin_osc0_half" }; ++PNAME(mclk_spdif_8ch_p) = { "mclk_spdif_8ch_src", "mclk_spdif_8ch_frac" }; ++PNAME(sclk_audpwm_p) = { "sclk_audpwm_src", "sclk_audpwm_frac" }; ++PNAME(sclk_uart1_p) = { "clk_uart1_src", "clk_uart1_frac", "xin24m" }; ++PNAME(sclk_uart2_p) = { "clk_uart2_src", "clk_uart2_frac", "xin24m" }; ++PNAME(sclk_uart3_p) = { "clk_uart3_src", "clk_uart3_frac", "xin24m" }; ++PNAME(sclk_uart4_p) = { "clk_uart4_src", "clk_uart4_frac", "xin24m" }; ++PNAME(sclk_uart5_p) = { "clk_uart5_src", "clk_uart5_frac", "xin24m" }; ++PNAME(sclk_uart6_p) = { "clk_uart6_src", "clk_uart6_frac", "xin24m" }; ++PNAME(sclk_uart7_p) = { "clk_uart7_src", "clk_uart7_frac", "xin24m" }; ++PNAME(sclk_uart8_p) = { "clk_uart8_src", "clk_uart8_frac", "xin24m" }; ++PNAME(sclk_uart9_p) = { "clk_uart9_src", "clk_uart9_frac", "xin24m" }; ++PNAME(sclk_uart0_p) = { "sclk_uart0_div", "sclk_uart0_frac", "xin24m" }; ++PNAME(clk_rtc32k_pmu_p) = { "clk_32k_pvtm", "xin32k", "clk_rtc32k_frac" }; ++PNAME(mpll_gpll_cpll_npll_p) = { "mpll", "gpll", "cpll", "npll" }; ++PNAME(gpll_cpll_npll_p) = { "gpll", "cpll", "npll" }; ++PNAME(npll_gpll_p) = { "npll", "gpll" }; ++PNAME(cpll_gpll_p) = { "cpll", "gpll" }; ++PNAME(gpll_cpll_p) = { "gpll", "cpll" }; ++PNAME(gpll_cpll_npll_vpll_p) = { "gpll", "cpll", "npll", "vpll" }; ++PNAME(apll_gpll_npll_p) = { "apll", "gpll", "npll" }; ++PNAME(sclk_core_pre_p) = { "sclk_core_src", "npll" }; ++PNAME(gpll150_gpll100_gpll75_xin24m_p) = { "gpll_150m", "gpll_100m", "gpll_75m", "xin24m" }; ++PNAME(clk_gpu_pre_mux_p) = { "clk_gpu_src", "gpu_pvtpll_out" }; ++PNAME(clk_npu_pre_ndft_p) = { "clk_npu_src", "clk_npu_np5"}; ++PNAME(clk_npu_p) = { "clk_npu_pre_ndft", "npu_pvtpll_out" }; ++PNAME(dpll_gpll_cpll_p) = { "dpll", "gpll", "cpll" }; ++PNAME(clk_ddr1x_p) = { "clk_ddrphy1x_src", "dpll" }; ++PNAME(gpll200_gpll150_gpll100_xin24m_p) = { "gpll_200m", "gpll_150m", "gpll_100m", "xin24m" }; ++PNAME(gpll100_gpll75_gpll50_p) = { "gpll_100m", "gpll_75m", "cpll_50m" }; ++PNAME(i2s0_mclkout_tx_p) = { "mclk_i2s0_8ch_tx", "xin_osc0_half" }; ++PNAME(i2s0_mclkout_rx_p) = { "mclk_i2s0_8ch_rx", "xin_osc0_half" }; ++PNAME(i2s1_mclkout_tx_p) = { "mclk_i2s1_8ch_tx", "xin_osc0_half" }; ++PNAME(i2s1_mclkout_rx_p) = { "mclk_i2s1_8ch_rx", "xin_osc0_half" }; ++PNAME(i2s2_mclkout_p) = { "mclk_i2s2_2ch", "xin_osc0_half" }; ++PNAME(i2s3_mclkout_tx_p) = { "mclk_i2s3_2ch_tx", "xin_osc0_half" }; ++PNAME(i2s3_mclkout_rx_p) = { "mclk_i2s3_2ch_rx", "xin_osc0_half" }; ++PNAME(mclk_pdm_p) = { "gpll_300m", "cpll_250m", "gpll_200m", "gpll_100m" }; ++PNAME(clk_i2c_p) = { "gpll_200m", "gpll_100m", "xin24m", "cpll_100m" }; ++PNAME(gpll200_gpll150_gpll100_p) = { "gpll_200m", "gpll_150m", "gpll_100m" }; ++PNAME(gpll300_gpll200_gpll100_p) = { "gpll_300m", "gpll_200m", "gpll_100m" }; ++PNAME(clk_nandc_p) = { "gpll_200m", "gpll_150m", "cpll_100m", "xin24m" }; ++PNAME(sclk_sfc_p) = { "xin24m", "cpll_50m", "gpll_75m", "gpll_100m", "cpll_125m", "gpll_150m" }; ++PNAME(gpll200_gpll150_cpll125_p) = { "gpll_200m", "gpll_150m", "cpll_125m" }; ++PNAME(cclk_emmc_p) = { "xin24m", "gpll_200m", "gpll_150m", "cpll_100m", "cpll_50m", "clk_osc0_div_375k" }; ++PNAME(aclk_pipe_p) = { "gpll_400m", "gpll_300m", "gpll_200m", "xin24m" }; ++PNAME(gpll200_cpll125_p) = { "gpll_200m", "cpll_125m" }; ++PNAME(gpll300_gpll200_gpll100_xin24m_p) = { "gpll_300m", "gpll_200m", "gpll_100m", "xin24m" }; ++PNAME(clk_sdmmc_p) = { "xin24m", "gpll_400m", "gpll_300m", "cpll_100m", "cpll_50m", "clk_osc0_div_750k" }; ++PNAME(cpll125_cpll50_cpll25_xin24m_p) = { "cpll_125m", "cpll_50m", "cpll_25m", "xin24m" }; ++PNAME(clk_gmac_ptp_p) = { "cpll_62p5", "gpll_100m", "cpll_50m", "xin24m" }; ++PNAME(cpll333_gpll300_gpll200_p) = { "cpll_333m", "gpll_300m", "gpll_200m" }; ++PNAME(cpll_gpll_hpll_p) = { "cpll", "gpll", "hpll" }; ++PNAME(gpll_usb480m_xin24m_p) = { "gpll", "usb480m", "xin24m", "xin24m" }; ++PNAME(gpll300_cpll250_gpll100_xin24m_p) = { "gpll_300m", "cpll_250m", "gpll_100m", "xin24m" }; ++PNAME(cpll_gpll_hpll_vpll_p) = { "cpll", "gpll", "hpll", "vpll" }; ++PNAME(hpll_vpll_gpll_cpll_p) = { "hpll", "vpll", "gpll", "cpll" }; ++PNAME(gpll400_cpll333_gpll200_p) = { "gpll_400m", "cpll_333m", "gpll_200m" }; ++PNAME(gpll100_gpll75_cpll50_xin24m_p) = { "gpll_100m", "gpll_75m", "cpll_50m", "xin24m" }; ++PNAME(xin24m_gpll100_cpll100_p) = { "xin24m", "gpll_100m", "cpll_100m" }; ++PNAME(gpll_cpll_usb480m_p) = { "gpll", "cpll", "usb480m" }; ++PNAME(gpll100_xin24m_cpll100_p) = { "gpll_100m", "xin24m", "cpll_100m" }; ++PNAME(gpll200_xin24m_cpll100_p) = { "gpll_200m", "xin24m", "cpll_100m" }; ++PNAME(xin24m_32k_p) = { "xin24m", "clk_rtc_32k" }; ++PNAME(cpll500_gpll400_gpll300_xin24m_p) = { "cpll_500m", "gpll_400m", "gpll_300m", "xin24m" }; ++PNAME(gpll400_gpll300_gpll200_xin24m_p) = { "gpll_400m", "gpll_300m", "gpll_200m", "xin24m" }; ++PNAME(xin24m_cpll100_p) = { "xin24m", "cpll_100m" }; ++PNAME(ppll_usb480m_cpll_gpll_p) = { "ppll", "usb480m", "cpll", "gpll"}; ++PNAME(clk_usbphy0_ref_p) = { "clk_ref24m", "xin_osc0_usbphy0_g" }; ++PNAME(clk_usbphy1_ref_p) = { "clk_ref24m", "xin_osc0_usbphy1_g" }; ++PNAME(clk_mipidsiphy0_ref_p) = { "clk_ref24m", "xin_osc0_mipidsiphy0_g" }; ++PNAME(clk_mipidsiphy1_ref_p) = { "clk_ref24m", "xin_osc0_mipidsiphy1_g" }; ++PNAME(clk_wifi_p) = { "clk_wifi_osc0", "clk_wifi_div" }; ++PNAME(clk_pciephy0_ref_p) = { "clk_pciephy0_osc0", "clk_pciephy0_div" }; ++PNAME(clk_pciephy1_ref_p) = { "clk_pciephy1_osc0", "clk_pciephy1_div" }; ++PNAME(clk_pciephy2_ref_p) = { "clk_pciephy2_osc0", "clk_pciephy2_div" }; ++PNAME(mux_gmac0_p) = { "clk_mac0_2top", "gmac0_clkin" }; ++PNAME(mux_gmac0_rgmii_speed_p) = { "clk_gmac0", "clk_gmac0", "clk_gmac0_tx_div50", "clk_gmac0_tx_div5" }; ++PNAME(mux_gmac0_rmii_speed_p) = { "clk_gmac0_rx_div20", "clk_gmac0_rx_div2" }; ++PNAME(mux_gmac0_rx_tx_p) = { "clk_gmac0_rgmii_speed", "clk_gmac0_rmii_speed", "clk_gmac0_xpcs_mii" }; ++PNAME(mux_gmac1_p) = { "clk_mac1_2top", "gmac1_clkin" }; ++PNAME(mux_gmac1_rgmii_speed_p) = { "clk_gmac1", "clk_gmac1", "clk_gmac1_tx_div50", "clk_gmac1_tx_div5" }; ++PNAME(mux_gmac1_rmii_speed_p) = { "clk_gmac1_rx_div20", "clk_gmac1_rx_div2" }; ++PNAME(mux_gmac1_rx_tx_p) = { "clk_gmac1_rgmii_speed", "clk_gmac1_rmii_speed", "clk_gmac1_xpcs_mii" }; ++PNAME(clk_hdmi_ref_p) = { "hpll", "hpll_ph0" }; ++PNAME(clk_pdpmu_p) = { "ppll", "gpll" }; ++PNAME(clk_mac_2top_p) = { "cpll_125m", "cpll_50m", "cpll_25m", "ppll" }; ++PNAME(clk_pwm0_p) = { "xin24m", "clk_pdpmu" }; ++PNAME(aclk_rkvdec_pre_p) = { "gpll", "cpll" }; ++PNAME(clk_rkvdec_core_p) = { "gpll", "cpll", "dummy_npll", "dummy_vpll" }; ++PNAME(clk_32k_ioe_p) = { "clk_rtc_32k", "xin32k" }; ++PNAME(i2s1_mclkout_p) = { "i2s1_mclkout_rx", "i2s1_mclkout_tx" }; ++PNAME(i2s3_mclkout_p) = { "i2s3_mclkout_rx", "i2s3_mclkout_tx" }; ++PNAME(i2s1_mclk_rx_ioe_p) = { "i2s1_mclkin_rx", "i2s1_mclkout_rx" }; ++PNAME(i2s1_mclk_tx_ioe_p) = { "i2s1_mclkin_tx", "i2s1_mclkout_tx" }; ++PNAME(i2s2_mclk_ioe_p) = { "i2s2_mclkin", "i2s2_mclkout" }; ++PNAME(i2s3_mclk_ioe_p) = { "i2s3_mclkin", "i2s3_mclkout" }; ++ ++static struct rockchip_pll_clock rk3568_pmu_pll_clks[] __initdata = { ++ [ppll] = PLL(pll_rk3328, PLL_PPLL, "ppll", mux_pll_p, ++ 0, RK3568_PMU_PLL_CON(0), ++ RK3568_PMU_MODE_CON0, 0, 4, 0, rk3568_pll_rates), ++ [hpll] = PLL(pll_rk3328, PLL_HPLL, "hpll", mux_pll_p, ++ 0, RK3568_PMU_PLL_CON(16), ++ RK3568_PMU_MODE_CON0, 2, 7, 0, rk3568_pll_rates), ++}; ++ ++static struct rockchip_pll_clock rk3568_pll_clks[] __initdata = { ++ [apll] = PLL(pll_rk3328, PLL_APLL, "apll", mux_pll_p, ++ 0, RK3568_PLL_CON(0), ++ RK3568_MODE_CON0, 0, 0, 0, rk3568_pll_rates), ++ [dpll] = PLL(pll_rk3328, PLL_DPLL, "dpll", mux_pll_p, ++ 0, RK3568_PLL_CON(8), ++ RK3568_MODE_CON0, 2, 1, 0, NULL), ++ [cpll] = PLL(pll_rk3328, PLL_CPLL, "cpll", mux_pll_p, ++ 0, RK3568_PLL_CON(24), ++ RK3568_MODE_CON0, 4, 2, 0, rk3568_pll_rates), ++ [gpll] = PLL(pll_rk3328, PLL_GPLL, "gpll", mux_pll_p, ++ 0, RK3568_PLL_CON(16), ++ RK3568_MODE_CON0, 6, 3, 0, rk3568_pll_rates), ++ [npll] = PLL(pll_rk3328, PLL_NPLL, "npll", mux_pll_p, ++ CLK_IS_CRITICAL, RK3568_PLL_CON(32), ++ RK3568_MODE_CON0, 10, 5, 0, rk3568_pll_rates), ++ [vpll] = PLL(pll_rk3328, PLL_VPLL, "vpll", mux_pll_p, ++ 0, RK3568_PLL_CON(40), ++ RK3568_MODE_CON0, 12, 6, 0, rk3568_pll_rates), ++}; ++ ++#define MFLAGS CLK_MUX_HIWORD_MASK ++#define DFLAGS CLK_DIVIDER_HIWORD_MASK ++#define GFLAGS (CLK_GATE_HIWORD_MASK | CLK_GATE_SET_TO_DISABLE) ++ ++static struct rockchip_clk_branch rk3568_i2s0_8ch_tx_fracmux __initdata = ++ MUX(CLK_I2S0_8CH_TX, "clk_i2s0_8ch_tx", clk_i2s0_8ch_tx_p, CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(11), 10, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rk3568_i2s0_8ch_rx_fracmux __initdata = ++ MUX(CLK_I2S0_8CH_RX, "clk_i2s0_8ch_rx", clk_i2s0_8ch_rx_p, CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(13), 10, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rk3568_i2s1_8ch_tx_fracmux __initdata = ++ MUX(CLK_I2S1_8CH_TX, "clk_i2s1_8ch_tx", clk_i2s1_8ch_tx_p, CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(15), 10, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rk3568_i2s1_8ch_rx_fracmux __initdata = ++ MUX(CLK_I2S1_8CH_RX, "clk_i2s1_8ch_rx", clk_i2s1_8ch_rx_p, CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(17), 10, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rk3568_i2s2_2ch_fracmux __initdata = ++ MUX(CLK_I2S2_2CH, "clk_i2s2_2ch", clk_i2s2_2ch_p, CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(19), 10, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rk3568_i2s3_2ch_tx_fracmux __initdata = ++ MUX(CLK_I2S3_2CH_TX, "clk_i2s3_2ch_tx", clk_i2s3_2ch_tx_p, CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(21), 10, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rk3568_i2s3_2ch_rx_fracmux __initdata = ++ MUX(CLK_I2S3_2CH_RX, "clk_i2s3_2ch_rx", clk_i2s3_2ch_rx_p, CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(83), 10, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rk3568_spdif_8ch_fracmux __initdata = ++ MUX(MCLK_SPDIF_8CH, "mclk_spdif_8ch", mclk_spdif_8ch_p, CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(23), 15, 1, MFLAGS); ++ ++static struct rockchip_clk_branch rk3568_audpwm_fracmux __initdata = ++ MUX(SCLK_AUDPWM, "sclk_audpwm", sclk_audpwm_p, CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(25), 15, 1, MFLAGS); ++ ++static struct rockchip_clk_branch rk3568_uart1_fracmux __initdata = ++ MUX(0, "sclk_uart1_mux", sclk_uart1_p, CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(52), 12, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rk3568_uart2_fracmux __initdata = ++ MUX(0, "sclk_uart2_mux", sclk_uart2_p, CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(54), 12, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rk3568_uart3_fracmux __initdata = ++ MUX(0, "sclk_uart3_mux", sclk_uart3_p, CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(56), 12, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rk3568_uart4_fracmux __initdata = ++ MUX(0, "sclk_uart4_mux", sclk_uart4_p, CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(58), 12, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rk3568_uart5_fracmux __initdata = ++ MUX(0, "sclk_uart5_mux", sclk_uart5_p, CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(60), 12, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rk3568_uart6_fracmux __initdata = ++ MUX(0, "sclk_uart6_mux", sclk_uart6_p, CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(62), 12, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rk3568_uart7_fracmux __initdata = ++ MUX(0, "sclk_uart7_mux", sclk_uart7_p, CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(64), 12, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rk3568_uart8_fracmux __initdata = ++ MUX(0, "sclk_uart8_mux", sclk_uart8_p, CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(66), 12, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rk3568_uart9_fracmux __initdata = ++ MUX(0, "sclk_uart9_mux", sclk_uart9_p, CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(68), 12, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rk3568_uart0_fracmux __initdata = ++ MUX(0, "sclk_uart0_mux", sclk_uart0_p, CLK_SET_RATE_PARENT, ++ RK3568_PMU_CLKSEL_CON(4), 10, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rk3568_rtc32k_pmu_fracmux __initdata = ++ MUX(CLK_RTC_32K, "clk_rtc_32k", clk_rtc32k_pmu_p, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, ++ RK3568_PMU_CLKSEL_CON(0), 6, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rk3568_clk_npu_np5 __initdata = ++ COMPOSITE_HALFDIV(CLK_NPU_NP5, "clk_npu_np5", npll_gpll_p, 0, ++ RK3568_CLKSEL_CON(7), 7, 1, MFLAGS, 4, 2, DFLAGS, ++ RK3568_CLKGATE_CON(3), 1, GFLAGS); ++ ++static struct rockchip_clk_branch rk3568_clk_branches[] __initdata = { ++ /* ++ * Clock-Architecture Diagram 1 ++ */ ++ /* SRC_CLK */ ++ COMPOSITE_NOMUX(0, "gpll_400m", "gpll", CLK_IGNORE_UNUSED, ++ RK3568_CLKSEL_CON(75), 0, 5, DFLAGS, ++ RK3568_CLKGATE_CON(35), 0, GFLAGS), ++ COMPOSITE_NOMUX(0, "gpll_300m", "gpll", CLK_IGNORE_UNUSED, ++ RK3568_CLKSEL_CON(75), 8, 5, DFLAGS, ++ RK3568_CLKGATE_CON(35), 1, GFLAGS), ++ COMPOSITE_NOMUX(0, "gpll_200m", "gpll", CLK_IGNORE_UNUSED, ++ RK3568_CLKSEL_CON(76), 0, 5, DFLAGS, ++ RK3568_CLKGATE_CON(35), 2, GFLAGS), ++ COMPOSITE_NOMUX(0, "gpll_150m", "gpll", CLK_IGNORE_UNUSED, ++ RK3568_CLKSEL_CON(76), 8, 5, DFLAGS, ++ RK3568_CLKGATE_CON(35), 3, GFLAGS), ++ COMPOSITE_NOMUX(0, "gpll_100m", "gpll", CLK_IGNORE_UNUSED, ++ RK3568_CLKSEL_CON(77), 0, 5, DFLAGS, ++ RK3568_CLKGATE_CON(35), 4, GFLAGS), ++ COMPOSITE_NOMUX(0, "gpll_75m", "gpll", CLK_IGNORE_UNUSED, ++ RK3568_CLKSEL_CON(77), 8, 5, DFLAGS, ++ RK3568_CLKGATE_CON(35), 5, GFLAGS), ++ COMPOSITE_NOMUX(0, "gpll_20m", "gpll", CLK_IGNORE_UNUSED, ++ RK3568_CLKSEL_CON(78), 0, 6, DFLAGS, ++ RK3568_CLKGATE_CON(35), 6, GFLAGS), ++ COMPOSITE_NOMUX(CPLL_500M, "cpll_500m", "cpll", CLK_IGNORE_UNUSED, ++ RK3568_CLKSEL_CON(78), 8, 5, DFLAGS, ++ RK3568_CLKGATE_CON(35), 7, GFLAGS), ++ COMPOSITE_NOMUX(CPLL_333M, "cpll_333m", "cpll", CLK_IGNORE_UNUSED, ++ RK3568_CLKSEL_CON(79), 0, 5, DFLAGS, ++ RK3568_CLKGATE_CON(35), 8, GFLAGS), ++ COMPOSITE_NOMUX(CPLL_250M, "cpll_250m", "cpll", CLK_IGNORE_UNUSED, ++ RK3568_CLKSEL_CON(79), 8, 5, DFLAGS, ++ RK3568_CLKGATE_CON(35), 9, GFLAGS), ++ COMPOSITE_NOMUX(CPLL_125M, "cpll_125m", "cpll", CLK_IGNORE_UNUSED, ++ RK3568_CLKSEL_CON(80), 0, 5, DFLAGS, ++ RK3568_CLKGATE_CON(35), 10, GFLAGS), ++ COMPOSITE_NOMUX(CPLL_100M, "cpll_100m", "cpll", CLK_IGNORE_UNUSED, ++ RK3568_CLKSEL_CON(82), 0, 5, DFLAGS, ++ RK3568_CLKGATE_CON(35), 11, GFLAGS), ++ COMPOSITE_NOMUX(CPLL_62P5M, "cpll_62p5", "cpll", CLK_IGNORE_UNUSED, ++ RK3568_CLKSEL_CON(80), 8, 5, DFLAGS, ++ RK3568_CLKGATE_CON(35), 12, GFLAGS), ++ COMPOSITE_NOMUX(CPLL_50M, "cpll_50m", "cpll", CLK_IGNORE_UNUSED, ++ RK3568_CLKSEL_CON(81), 0, 5, DFLAGS, ++ RK3568_CLKGATE_CON(35), 13, GFLAGS), ++ COMPOSITE_NOMUX(CPLL_25M, "cpll_25m", "cpll", CLK_IGNORE_UNUSED, ++ RK3568_CLKSEL_CON(81), 8, 6, DFLAGS, ++ RK3568_CLKGATE_CON(35), 14, GFLAGS), ++ COMPOSITE_NOMUX(0, "clk_osc0_div_750k", "xin24m", CLK_IGNORE_UNUSED, ++ RK3568_CLKSEL_CON(82), 8, 6, DFLAGS, ++ RK3568_CLKGATE_CON(35), 15, GFLAGS), ++ FACTOR(0, "clk_osc0_div_375k", "clk_osc0_div_750k", 0, 1, 2), ++ FACTOR(0, "xin_osc0_half", "xin24m", 0, 1, 2), ++ MUX(USB480M, "usb480m", mux_usb480m_p, CLK_SET_RATE_PARENT, ++ RK3568_MODE_CON0, 14, 2, MFLAGS), ++ ++ /* PD_CORE */ ++ COMPOSITE(0, "sclk_core_src", apll_gpll_npll_p, CLK_IS_CRITICAL, ++ RK3568_CLKSEL_CON(2), 8, 2, MFLAGS, 0, 4, DFLAGS | CLK_DIVIDER_READ_ONLY, ++ RK3568_CLKGATE_CON(0), 5, GFLAGS), ++ COMPOSITE_NODIV(0, "sclk_core", sclk_core_pre_p, CLK_IS_CRITICAL, ++ RK3568_CLKSEL_CON(2), 15, 1, MFLAGS, ++ RK3568_CLKGATE_CON(0), 7, GFLAGS), ++ ++ COMPOSITE_NOMUX(0, "atclk_core", "armclk", CLK_IS_CRITICAL, ++ RK3568_CLKSEL_CON(3), 0, 5, DFLAGS | CLK_DIVIDER_READ_ONLY, ++ RK3568_CLKGATE_CON(0), 8, GFLAGS), ++ COMPOSITE_NOMUX(0, "gicclk_core", "armclk", CLK_IS_CRITICAL, ++ RK3568_CLKSEL_CON(3), 8, 5, DFLAGS | CLK_DIVIDER_READ_ONLY, ++ RK3568_CLKGATE_CON(0), 9, GFLAGS), ++ COMPOSITE_NOMUX(0, "pclk_core_pre", "armclk", CLK_IS_CRITICAL, ++ RK3568_CLKSEL_CON(4), 0, 5, DFLAGS | CLK_DIVIDER_READ_ONLY, ++ RK3568_CLKGATE_CON(0), 10, GFLAGS), ++ COMPOSITE_NOMUX(0, "periphclk_core_pre", "armclk", CLK_IS_CRITICAL, ++ RK3568_CLKSEL_CON(4), 8, 5, DFLAGS | CLK_DIVIDER_READ_ONLY, ++ RK3568_CLKGATE_CON(0), 11, GFLAGS), ++ COMPOSITE_NOMUX(0, "tsclk_core", "periphclk_core_pre", CLK_IS_CRITICAL, ++ RK3568_CLKSEL_CON(5), 0, 4, DFLAGS | CLK_DIVIDER_READ_ONLY, ++ RK3568_CLKGATE_CON(0), 14, GFLAGS), ++ COMPOSITE_NOMUX(0, "cntclk_core", "periphclk_core_pre", CLK_IS_CRITICAL, ++ RK3568_CLKSEL_CON(5), 4, 4, DFLAGS | CLK_DIVIDER_READ_ONLY, ++ RK3568_CLKGATE_CON(0), 15, GFLAGS), ++ COMPOSITE_NOMUX(0, "aclk_core", "sclk_core", CLK_IS_CRITICAL, ++ RK3568_CLKSEL_CON(5), 8, 5, DFLAGS | CLK_DIVIDER_READ_ONLY, ++ RK3568_CLKGATE_CON(1), 0, GFLAGS), ++ ++ COMPOSITE_NODIV(ACLK_CORE_NIU2BUS, "aclk_core_niu2bus", gpll150_gpll100_gpll75_xin24m_p, CLK_IS_CRITICAL, ++ RK3568_CLKSEL_CON(5), 14, 2, MFLAGS, ++ RK3568_CLKGATE_CON(1), 2, GFLAGS), ++ ++ GATE(CLK_CORE_PVTM, "clk_core_pvtm", "xin24m", 0, ++ RK3568_CLKGATE_CON(1), 10, GFLAGS), ++ GATE(CLK_CORE_PVTM_CORE, "clk_core_pvtm_core", "armclk", 0, ++ RK3568_CLKGATE_CON(1), 11, GFLAGS), ++ GATE(CLK_CORE_PVTPLL, "clk_core_pvtpll", "armclk", CLK_IGNORE_UNUSED, ++ RK3568_CLKGATE_CON(1), 12, GFLAGS), ++ GATE(PCLK_CORE_PVTM, "pclk_core_pvtm", "pclk_core_pre", 0, ++ RK3568_CLKGATE_CON(1), 9, GFLAGS), ++ ++ /* PD_GPU */ ++ COMPOSITE(CLK_GPU_SRC, "clk_gpu_src", mpll_gpll_cpll_npll_p, 0, ++ RK3568_CLKSEL_CON(6), 6, 2, MFLAGS | CLK_MUX_READ_ONLY, 0, 4, DFLAGS | CLK_DIVIDER_READ_ONLY, ++ RK3568_CLKGATE_CON(2), 0, GFLAGS), ++ MUX(CLK_GPU_PRE_MUX, "clk_gpu_pre_mux", clk_gpu_pre_mux_p, CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(6), 11, 1, MFLAGS | CLK_MUX_READ_ONLY), ++ DIV(ACLK_GPU_PRE, "aclk_gpu_pre", "clk_gpu_pre_mux", 0, ++ RK3568_CLKSEL_CON(6), 8, 2, DFLAGS), ++ DIV(PCLK_GPU_PRE, "pclk_gpu_pre", "clk_gpu_pre_mux", 0, ++ RK3568_CLKSEL_CON(6), 12, 4, DFLAGS), ++ GATE(CLK_GPU, "clk_gpu", "clk_gpu_pre_mux", 0, ++ RK3568_CLKGATE_CON(2), 3, GFLAGS), ++ ++ GATE(PCLK_GPU_PVTM, "pclk_gpu_pvtm", "pclk_gpu_pre", 0, ++ RK3568_CLKGATE_CON(2), 6, GFLAGS), ++ GATE(CLK_GPU_PVTM, "clk_gpu_pvtm", "xin24m", 0, ++ RK3568_CLKGATE_CON(2), 7, GFLAGS), ++ GATE(CLK_GPU_PVTM_CORE, "clk_gpu_pvtm_core", "clk_gpu_src", 0, ++ RK3568_CLKGATE_CON(2), 8, GFLAGS), ++ GATE(CLK_GPU_PVTPLL, "clk_gpu_pvtpll", "clk_gpu_src", CLK_IGNORE_UNUSED, ++ RK3568_CLKGATE_CON(2), 9, GFLAGS), ++ ++ /* PD_NPU */ ++ COMPOSITE_BROTHER(CLK_NPU_SRC, "clk_npu_src", npll_gpll_p, 0, ++ RK3568_CLKSEL_CON(7), 6, 1, MFLAGS, 0, 4, DFLAGS, ++ RK3568_CLKGATE_CON(3), 0, GFLAGS, ++ &rk3568_clk_npu_np5), ++ MUX(CLK_NPU_PRE_NDFT, "clk_npu_pre_ndft", clk_npu_pre_ndft_p, CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE, ++ RK3568_CLKSEL_CON(7), 8, 1, MFLAGS), ++ MUX(CLK_NPU, "clk_npu", clk_npu_p, CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(7), 15, 1, MFLAGS), ++ COMPOSITE_NOMUX(HCLK_NPU_PRE, "hclk_npu_pre", "clk_npu", 0, ++ RK3568_CLKSEL_CON(8), 0, 4, DFLAGS, ++ RK3568_CLKGATE_CON(3), 2, GFLAGS), ++ COMPOSITE_NOMUX(PCLK_NPU_PRE, "pclk_npu_pre", "clk_npu", 0, ++ RK3568_CLKSEL_CON(8), 4, 4, DFLAGS, ++ RK3568_CLKGATE_CON(3), 3, GFLAGS), ++ GATE(ACLK_NPU_PRE, "aclk_npu_pre", "clk_npu", 0, ++ RK3568_CLKGATE_CON(3), 4, GFLAGS), ++ GATE(ACLK_NPU, "aclk_npu", "aclk_npu_pre", 0, ++ RK3568_CLKGATE_CON(3), 7, GFLAGS), ++ GATE(HCLK_NPU, "hclk_npu", "hclk_npu_pre", 0, ++ RK3568_CLKGATE_CON(3), 8, GFLAGS), ++ ++ GATE(PCLK_NPU_PVTM, "pclk_npu_pvtm", "pclk_npu_pre", 0, ++ RK3568_CLKGATE_CON(3), 9, GFLAGS), ++ GATE(CLK_NPU_PVTM, "clk_npu_pvtm", "xin24m", 0, ++ RK3568_CLKGATE_CON(3), 10, GFLAGS), ++ GATE(CLK_NPU_PVTM_CORE, "clk_npu_pvtm_core", "clk_npu_pre_ndft", 0, ++ RK3568_CLKGATE_CON(3), 11, GFLAGS), ++ GATE(CLK_NPU_PVTPLL, "clk_npu_pvtpll", "clk_npu_pre_ndft", CLK_IGNORE_UNUSED, ++ RK3568_CLKGATE_CON(3), 12, GFLAGS), ++ ++ /* PD_DDR */ ++ COMPOSITE(CLK_DDRPHY1X_SRC, "clk_ddrphy1x_src", dpll_gpll_cpll_p, CLK_IGNORE_UNUSED, ++ RK3568_CLKSEL_CON(9), 6, 2, MFLAGS, 0, 5, DFLAGS, ++ RK3568_CLKGATE_CON(4), 0, GFLAGS), ++ MUXGRF(CLK_DDR1X, "clk_ddr1x", clk_ddr1x_p, CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(9), 15, 1, MFLAGS), ++ ++ COMPOSITE_NOMUX(CLK_MSCH, "clk_msch", "clk_ddr1x", CLK_IGNORE_UNUSED, ++ RK3568_CLKSEL_CON(10), 0, 2, DFLAGS, ++ RK3568_CLKGATE_CON(4), 2, GFLAGS), ++ GATE(CLK24_DDRMON, "clk24_ddrmon", "xin24m", CLK_IGNORE_UNUSED, ++ RK3568_CLKGATE_CON(4), 15, GFLAGS), ++ ++ /* PD_GIC_AUDIO */ ++ COMPOSITE_NODIV(ACLK_GIC_AUDIO, "aclk_gic_audio", gpll200_gpll150_gpll100_xin24m_p, CLK_IGNORE_UNUSED, ++ RK3568_CLKSEL_CON(10), 8, 2, MFLAGS, ++ RK3568_CLKGATE_CON(5), 0, GFLAGS), ++ COMPOSITE_NODIV(HCLK_GIC_AUDIO, "hclk_gic_audio", gpll150_gpll100_gpll75_xin24m_p, CLK_IGNORE_UNUSED, ++ RK3568_CLKSEL_CON(10), 10, 2, MFLAGS, ++ RK3568_CLKGATE_CON(5), 1, GFLAGS), ++ GATE(HCLK_SDMMC_BUFFER, "hclk_sdmmc_buffer", "hclk_gic_audio", 0, ++ RK3568_CLKGATE_CON(5), 8, GFLAGS), ++ COMPOSITE_NODIV(DCLK_SDMMC_BUFFER, "dclk_sdmmc_buffer", gpll100_gpll75_gpll50_p, 0, ++ RK3568_CLKSEL_CON(10), 12, 2, MFLAGS, ++ RK3568_CLKGATE_CON(5), 9, GFLAGS), ++ GATE(ACLK_GIC600, "aclk_gic600", "aclk_gic_audio", CLK_IGNORE_UNUSED, ++ RK3568_CLKGATE_CON(5), 4, GFLAGS), ++ GATE(ACLK_SPINLOCK, "aclk_spinlock", "aclk_gic_audio", CLK_IGNORE_UNUSED, ++ RK3568_CLKGATE_CON(5), 7, GFLAGS), ++ GATE(HCLK_I2S0_8CH, "hclk_i2s0_8ch", "hclk_gic_audio", 0, ++ RK3568_CLKGATE_CON(5), 10, GFLAGS), ++ GATE(HCLK_I2S1_8CH, "hclk_i2s1_8ch", "hclk_gic_audio", 0, ++ RK3568_CLKGATE_CON(5), 11, GFLAGS), ++ GATE(HCLK_I2S2_2CH, "hclk_i2s2_2ch", "hclk_gic_audio", 0, ++ RK3568_CLKGATE_CON(5), 12, GFLAGS), ++ GATE(HCLK_I2S3_2CH, "hclk_i2s3_2ch", "hclk_gic_audio", 0, ++ RK3568_CLKGATE_CON(5), 13, GFLAGS), ++ ++ COMPOSITE(CLK_I2S0_8CH_TX_SRC, "clk_i2s0_8ch_tx_src", gpll_cpll_npll_p, 0, ++ RK3568_CLKSEL_CON(11), 8, 2, MFLAGS, 0, 7, DFLAGS, ++ RK3568_CLKGATE_CON(6), 0, GFLAGS), ++ COMPOSITE_FRACMUX(CLK_I2S0_8CH_TX_FRAC, "clk_i2s0_8ch_tx_frac", "clk_i2s0_8ch_tx_src", CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(12), 0, ++ RK3568_CLKGATE_CON(6), 1, GFLAGS, ++ &rk3568_i2s0_8ch_tx_fracmux, RK3568_FRAC_MAX_PRATE), ++ GATE(MCLK_I2S0_8CH_TX, "mclk_i2s0_8ch_tx", "clk_i2s0_8ch_tx", 0, ++ RK3568_CLKGATE_CON(6), 2, GFLAGS), ++ COMPOSITE_NODIV(I2S0_MCLKOUT_TX, "i2s0_mclkout_tx", i2s0_mclkout_tx_p, CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(11), 15, 1, MFLAGS, ++ RK3568_CLKGATE_CON(6), 3, GFLAGS), ++ ++ COMPOSITE(CLK_I2S0_8CH_RX_SRC, "clk_i2s0_8ch_rx_src", gpll_cpll_npll_p, 0, ++ RK3568_CLKSEL_CON(13), 8, 2, MFLAGS, 0, 7, DFLAGS, ++ RK3568_CLKGATE_CON(6), 4, GFLAGS), ++ COMPOSITE_FRACMUX(CLK_I2S0_8CH_RX_FRAC, "clk_i2s0_8ch_rx_frac", "clk_i2s0_8ch_rx_src", CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(14), 0, ++ RK3568_CLKGATE_CON(6), 5, GFLAGS, ++ &rk3568_i2s0_8ch_rx_fracmux, RK3568_FRAC_MAX_PRATE), ++ GATE(MCLK_I2S0_8CH_RX, "mclk_i2s0_8ch_rx", "clk_i2s0_8ch_rx", 0, ++ RK3568_CLKGATE_CON(6), 6, GFLAGS), ++ COMPOSITE_NODIV(I2S0_MCLKOUT_RX, "i2s0_mclkout_rx", i2s0_mclkout_rx_p, CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(13), 15, 1, MFLAGS, ++ RK3568_CLKGATE_CON(6), 7, GFLAGS), ++ ++ COMPOSITE(CLK_I2S1_8CH_TX_SRC, "clk_i2s1_8ch_tx_src", gpll_cpll_npll_p, 0, ++ RK3568_CLKSEL_CON(15), 8, 2, MFLAGS, 0, 7, DFLAGS, ++ RK3568_CLKGATE_CON(6), 8, GFLAGS), ++ COMPOSITE_FRACMUX(CLK_I2S1_8CH_TX_FRAC, "clk_i2s1_8ch_tx_frac", "clk_i2s1_8ch_tx_src", CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(16), 0, ++ RK3568_CLKGATE_CON(6), 9, GFLAGS, ++ &rk3568_i2s1_8ch_tx_fracmux, RK3568_FRAC_MAX_PRATE), ++ GATE(MCLK_I2S1_8CH_TX, "mclk_i2s1_8ch_tx", "clk_i2s1_8ch_tx", 0, ++ RK3568_CLKGATE_CON(6), 10, GFLAGS), ++ COMPOSITE_NODIV(I2S1_MCLKOUT_TX, "i2s1_mclkout_tx", i2s1_mclkout_tx_p, CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(15), 15, 1, MFLAGS, ++ RK3568_CLKGATE_CON(6), 11, GFLAGS), ++ ++ COMPOSITE(CLK_I2S1_8CH_RX_SRC, "clk_i2s1_8ch_rx_src", gpll_cpll_npll_p, 0, ++ RK3568_CLKSEL_CON(17), 8, 2, MFLAGS, 0, 7, DFLAGS, ++ RK3568_CLKGATE_CON(6), 12, GFLAGS), ++ COMPOSITE_FRACMUX(CLK_I2S1_8CH_RX_FRAC, "clk_i2s1_8ch_rx_frac", "clk_i2s1_8ch_rx_src", CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(18), 0, ++ RK3568_CLKGATE_CON(6), 13, GFLAGS, ++ &rk3568_i2s1_8ch_rx_fracmux, RK3568_FRAC_MAX_PRATE), ++ GATE(MCLK_I2S1_8CH_RX, "mclk_i2s1_8ch_rx", "clk_i2s1_8ch_rx", 0, ++ RK3568_CLKGATE_CON(6), 14, GFLAGS), ++ COMPOSITE_NODIV(I2S1_MCLKOUT_RX, "i2s1_mclkout_rx", i2s1_mclkout_rx_p, CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(17), 15, 1, MFLAGS, ++ RK3568_CLKGATE_CON(6), 15, GFLAGS), ++ ++ COMPOSITE(CLK_I2S2_2CH_SRC, "clk_i2s2_2ch_src", gpll_cpll_npll_p, 0, ++ RK3568_CLKSEL_CON(19), 8, 2, MFLAGS, 0, 7, DFLAGS, ++ RK3568_CLKGATE_CON(7), 0, GFLAGS), ++ COMPOSITE_FRACMUX(CLK_I2S2_2CH_FRAC, "clk_i2s2_2ch_frac", "clk_i2s2_2ch_src", CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(20), 0, ++ RK3568_CLKGATE_CON(7), 1, GFLAGS, ++ &rk3568_i2s2_2ch_fracmux, RK3568_FRAC_MAX_PRATE), ++ GATE(MCLK_I2S2_2CH, "mclk_i2s2_2ch", "clk_i2s2_2ch", 0, ++ RK3568_CLKGATE_CON(7), 2, GFLAGS), ++ COMPOSITE_NODIV(I2S2_MCLKOUT, "i2s2_mclkout", i2s2_mclkout_p, CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(19), 15, 1, MFLAGS, ++ RK3568_CLKGATE_CON(7), 3, GFLAGS), ++ ++ COMPOSITE(CLK_I2S3_2CH_TX_SRC, "clk_i2s3_2ch_tx_src", gpll_cpll_npll_p, 0, ++ RK3568_CLKSEL_CON(21), 8, 2, MFLAGS, 0, 7, DFLAGS, ++ RK3568_CLKGATE_CON(7), 4, GFLAGS), ++ COMPOSITE_FRACMUX(CLK_I2S3_2CH_TX_FRAC, "clk_i2s3_2ch_tx_frac", "clk_i2s3_2ch_tx_src", CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(22), 0, ++ RK3568_CLKGATE_CON(7), 5, GFLAGS, ++ &rk3568_i2s3_2ch_tx_fracmux, RK3568_FRAC_MAX_PRATE), ++ GATE(MCLK_I2S3_2CH_TX, "mclk_i2s3_2ch_tx", "clk_i2s3_2ch_tx", 0, ++ RK3568_CLKGATE_CON(7), 6, GFLAGS), ++ COMPOSITE_NODIV(I2S3_MCLKOUT_TX, "i2s3_mclkout_tx", i2s3_mclkout_tx_p, CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(21), 15, 1, MFLAGS, ++ RK3568_CLKGATE_CON(7), 7, GFLAGS), ++ ++ COMPOSITE(CLK_I2S3_2CH_RX_SRC, "clk_i2s3_2ch_rx_src", gpll_cpll_npll_p, 0, ++ RK3568_CLKSEL_CON(83), 8, 2, MFLAGS, 0, 7, DFLAGS, ++ RK3568_CLKGATE_CON(7), 8, GFLAGS), ++ COMPOSITE_FRACMUX(CLK_I2S3_2CH_RX_FRAC, "clk_i2s3_2ch_rx_frac", "clk_i2s3_2ch_rx_src", CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(84), 0, ++ RK3568_CLKGATE_CON(7), 9, GFLAGS, ++ &rk3568_i2s3_2ch_rx_fracmux, RK3568_FRAC_MAX_PRATE), ++ GATE(MCLK_I2S3_2CH_RX, "mclk_i2s3_2ch_rx", "clk_i2s3_2ch_rx", 0, ++ RK3568_CLKGATE_CON(7), 10, GFLAGS), ++ COMPOSITE_NODIV(I2S3_MCLKOUT_RX, "i2s3_mclkout_rx", i2s3_mclkout_rx_p, CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(83), 15, 1, MFLAGS, ++ RK3568_CLKGATE_CON(7), 11, GFLAGS), ++ ++ MUXGRF(I2S1_MCLKOUT, "i2s1_mclkout", i2s1_mclkout_p, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, ++ RK3568_GRF_SOC_CON1, 5, 1, MFLAGS), ++ MUXGRF(I2S3_MCLKOUT, "i2s3_mclkout", i2s3_mclkout_p, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, ++ RK3568_GRF_SOC_CON2, 15, 1, MFLAGS), ++ MUXGRF(I2S1_MCLK_RX_IOE, "i2s1_mclk_rx_ioe", i2s1_mclk_rx_ioe_p, 0, ++ RK3568_GRF_SOC_CON2, 0, 1, MFLAGS), ++ MUXGRF(I2S1_MCLK_TX_IOE, "i2s1_mclk_tx_ioe", i2s1_mclk_tx_ioe_p, 0, ++ RK3568_GRF_SOC_CON2, 1, 1, MFLAGS), ++ MUXGRF(I2S2_MCLK_IOE, "i2s2_mclk_ioe", i2s2_mclk_ioe_p, 0, ++ RK3568_GRF_SOC_CON2, 2, 1, MFLAGS), ++ MUXGRF(I2S3_MCLK_IOE, "i2s3_mclk_ioe", i2s3_mclk_ioe_p, 0, ++ RK3568_GRF_SOC_CON2, 3, 1, MFLAGS), ++ ++ GATE(HCLK_PDM, "hclk_pdm", "hclk_gic_audio", 0, ++ RK3568_CLKGATE_CON(5), 14, GFLAGS), ++ COMPOSITE_NODIV(MCLK_PDM, "mclk_pdm", mclk_pdm_p, 0, ++ RK3568_CLKSEL_CON(23), 8, 2, MFLAGS, ++ RK3568_CLKGATE_CON(5), 15, GFLAGS), ++ GATE(HCLK_VAD, "hclk_vad", "hclk_gic_audio", 0, ++ RK3568_CLKGATE_CON(7), 12, GFLAGS), ++ GATE(HCLK_SPDIF_8CH, "hclk_spdif_8ch", "hclk_gic_audio", 0, ++ RK3568_CLKGATE_CON(7), 13, GFLAGS), ++ ++ COMPOSITE(MCLK_SPDIF_8CH_SRC, "mclk_spdif_8ch_src", cpll_gpll_p, 0, ++ RK3568_CLKSEL_CON(23), 14, 1, MFLAGS, 0, 7, DFLAGS, ++ RK3568_CLKGATE_CON(7), 14, GFLAGS), ++ COMPOSITE_FRACMUX(MCLK_SPDIF_8CH_FRAC, "mclk_spdif_8ch_frac", "mclk_spdif_8ch_src", CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(24), 0, ++ RK3568_CLKGATE_CON(7), 15, GFLAGS, ++ &rk3568_spdif_8ch_fracmux, RK3568_SPDIF_FRAC_MAX_PRATE), ++ ++ GATE(HCLK_AUDPWM, "hclk_audpwm", "hclk_gic_audio", 0, ++ RK3568_CLKGATE_CON(8), 0, GFLAGS), ++ COMPOSITE(SCLK_AUDPWM_SRC, "sclk_audpwm_src", gpll_cpll_p, 0, ++ RK3568_CLKSEL_CON(25), 14, 1, MFLAGS, 0, 6, DFLAGS, ++ RK3568_CLKGATE_CON(8), 1, GFLAGS), ++ COMPOSITE_FRACMUX(SCLK_AUDPWM_FRAC, "sclk_audpwm_frac", "sclk_audpwm_src", CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(26), 0, ++ RK3568_CLKGATE_CON(8), 2, GFLAGS, ++ &rk3568_audpwm_fracmux, RK3568_FRAC_MAX_PRATE), ++ ++ GATE(HCLK_ACDCDIG, "hclk_acdcdig", "hclk_gic_audio", 0, ++ RK3568_CLKGATE_CON(8), 3, GFLAGS), ++ COMPOSITE_NODIV(CLK_ACDCDIG_I2C, "clk_acdcdig_i2c", clk_i2c_p, 0, ++ RK3568_CLKSEL_CON(23), 10, 2, MFLAGS, ++ RK3568_CLKGATE_CON(8), 4, GFLAGS), ++ GATE(CLK_ACDCDIG_DAC, "clk_acdcdig_dac", "mclk_i2s3_2ch_tx", 0, ++ RK3568_CLKGATE_CON(8), 5, GFLAGS), ++ GATE(CLK_ACDCDIG_ADC, "clk_acdcdig_adc", "mclk_i2s3_2ch_rx", 0, ++ RK3568_CLKGATE_CON(8), 6, GFLAGS), ++ ++ /* PD_SECURE_FLASH */ ++ COMPOSITE_NODIV(ACLK_SECURE_FLASH, "aclk_secure_flash", gpll200_gpll150_gpll100_xin24m_p, CLK_IS_CRITICAL, ++ RK3568_CLKSEL_CON(27), 0, 2, MFLAGS, ++ RK3568_CLKGATE_CON(8), 7, GFLAGS), ++ COMPOSITE_NODIV(HCLK_SECURE_FLASH, "hclk_secure_flash", gpll150_gpll100_gpll75_xin24m_p, CLK_IS_CRITICAL, ++ RK3568_CLKSEL_CON(27), 2, 2, MFLAGS, ++ RK3568_CLKGATE_CON(8), 8, GFLAGS), ++ GATE(ACLK_CRYPTO_NS, "aclk_crypto_ns", "aclk_secure_flash", 0, ++ RK3568_CLKGATE_CON(8), 11, GFLAGS), ++ GATE(HCLK_CRYPTO_NS, "hclk_crypto_ns", "hclk_secure_flash", 0, ++ RK3568_CLKGATE_CON(8), 12, GFLAGS), ++ COMPOSITE_NODIV(CLK_CRYPTO_NS_CORE, "clk_crypto_ns_core", gpll200_gpll150_gpll100_p, 0, ++ RK3568_CLKSEL_CON(27), 4, 2, MFLAGS, ++ RK3568_CLKGATE_CON(8), 13, GFLAGS), ++ COMPOSITE_NODIV(CLK_CRYPTO_NS_PKA, "clk_crypto_ns_pka", gpll300_gpll200_gpll100_p, 0, ++ RK3568_CLKSEL_CON(27), 6, 2, MFLAGS, ++ RK3568_CLKGATE_CON(8), 14, GFLAGS), ++ GATE(CLK_CRYPTO_NS_RNG, "clk_crypto_ns_rng", "hclk_secure_flash", 0, ++ RK3568_CLKGATE_CON(8), 15, GFLAGS), ++ GATE(HCLK_TRNG_NS, "hclk_trng_ns", "hclk_secure_flash", CLK_IGNORE_UNUSED, ++ RK3568_CLKGATE_CON(9), 10, GFLAGS), ++ GATE(CLK_TRNG_NS, "clk_trng_ns", "hclk_secure_flash", CLK_IGNORE_UNUSED, ++ RK3568_CLKGATE_CON(9), 11, GFLAGS), ++ GATE(PCLK_OTPC_NS, "pclk_otpc_ns", "hclk_secure_flash", 0, ++ RK3568_CLKGATE_CON(26), 9, GFLAGS), ++ GATE(CLK_OTPC_NS_SBPI, "clk_otpc_ns_sbpi", "xin24m", 0, ++ RK3568_CLKGATE_CON(26), 10, GFLAGS), ++ GATE(CLK_OTPC_NS_USR, "clk_otpc_ns_usr", "xin_osc0_half", 0, ++ RK3568_CLKGATE_CON(26), 11, GFLAGS), ++ GATE(HCLK_NANDC, "hclk_nandc", "hclk_secure_flash", 0, ++ RK3568_CLKGATE_CON(9), 0, GFLAGS), ++ COMPOSITE_NODIV(NCLK_NANDC, "nclk_nandc", clk_nandc_p, 0, ++ RK3568_CLKSEL_CON(28), 0, 2, MFLAGS, ++ RK3568_CLKGATE_CON(9), 1, GFLAGS), ++ GATE(HCLK_SFC, "hclk_sfc", "hclk_secure_flash", 0, ++ RK3568_CLKGATE_CON(9), 2, GFLAGS), ++ GATE(HCLK_SFC_XIP, "hclk_sfc_xip", "hclk_secure_flash", 0, ++ RK3568_CLKGATE_CON(9), 3, GFLAGS), ++ COMPOSITE_NODIV(SCLK_SFC, "sclk_sfc", sclk_sfc_p, 0, ++ RK3568_CLKSEL_CON(28), 4, 3, MFLAGS, ++ RK3568_CLKGATE_CON(9), 4, GFLAGS), ++ GATE(ACLK_EMMC, "aclk_emmc", "aclk_secure_flash", 0, ++ RK3568_CLKGATE_CON(9), 5, GFLAGS), ++ GATE(HCLK_EMMC, "hclk_emmc", "hclk_secure_flash", 0, ++ RK3568_CLKGATE_CON(9), 6, GFLAGS), ++ COMPOSITE_NODIV(BCLK_EMMC, "bclk_emmc", gpll200_gpll150_cpll125_p, 0, ++ RK3568_CLKSEL_CON(28), 8, 2, MFLAGS, ++ RK3568_CLKGATE_CON(9), 7, GFLAGS), ++ COMPOSITE_NODIV(CCLK_EMMC, "cclk_emmc", cclk_emmc_p, 0, ++ RK3568_CLKSEL_CON(28), 12, 3, MFLAGS, ++ RK3568_CLKGATE_CON(9), 8, GFLAGS), ++ GATE(TCLK_EMMC, "tclk_emmc", "xin24m", 0, ++ RK3568_CLKGATE_CON(9), 9, GFLAGS), ++ MMC(SCLK_EMMC_DRV, "emmc_drv", "cclk_emmc", RK3568_EMMC_CON0, 1), ++ MMC(SCLK_EMMC_SAMPLE, "emmc_sample", "cclk_emmc", RK3568_EMMC_CON1, 1), ++ ++ /* PD_PIPE */ ++ COMPOSITE_NODIV(ACLK_PIPE, "aclk_pipe", aclk_pipe_p, 0, ++ RK3568_CLKSEL_CON(29), 0, 2, MFLAGS, ++ RK3568_CLKGATE_CON(10), 0, GFLAGS), ++ COMPOSITE_NOMUX(PCLK_PIPE, "pclk_pipe", "aclk_pipe", 0, ++ RK3568_CLKSEL_CON(29), 4, 4, DFLAGS, ++ RK3568_CLKGATE_CON(10), 1, GFLAGS), ++ GATE(ACLK_PCIE20_MST, "aclk_pcie20_mst", "aclk_pipe", 0, ++ RK3568_CLKGATE_CON(12), 0, GFLAGS), ++ GATE(ACLK_PCIE20_SLV, "aclk_pcie20_slv", "aclk_pipe", 0, ++ RK3568_CLKGATE_CON(12), 1, GFLAGS), ++ GATE(ACLK_PCIE20_DBI, "aclk_pcie20_dbi", "aclk_pipe", 0, ++ RK3568_CLKGATE_CON(12), 2, GFLAGS), ++ GATE(PCLK_PCIE20, "pclk_pcie20", "pclk_pipe", 0, ++ RK3568_CLKGATE_CON(12), 3, GFLAGS), ++ GATE(CLK_PCIE20_AUX_NDFT, "clk_pcie20_aux_ndft", "xin24m", 0, ++ RK3568_CLKGATE_CON(12), 4, GFLAGS), ++ GATE(ACLK_PCIE30X1_MST, "aclk_pcie30x1_mst", "aclk_pipe", 0, ++ RK3568_CLKGATE_CON(12), 8, GFLAGS), ++ GATE(ACLK_PCIE30X1_SLV, "aclk_pcie30x1_slv", "aclk_pipe", 0, ++ RK3568_CLKGATE_CON(12), 9, GFLAGS), ++ GATE(ACLK_PCIE30X1_DBI, "aclk_pcie30x1_dbi", "aclk_pipe", 0, ++ RK3568_CLKGATE_CON(12), 10, GFLAGS), ++ GATE(PCLK_PCIE30X1, "pclk_pcie30x1", "pclk_pipe", 0, ++ RK3568_CLKGATE_CON(12), 11, GFLAGS), ++ GATE(CLK_PCIE30X1_AUX_NDFT, "clk_pcie30x1_aux_ndft", "xin24m", 0, ++ RK3568_CLKGATE_CON(12), 12, GFLAGS), ++ GATE(ACLK_PCIE30X2_MST, "aclk_pcie30x2_mst", "aclk_pipe", 0, ++ RK3568_CLKGATE_CON(13), 0, GFLAGS), ++ GATE(ACLK_PCIE30X2_SLV, "aclk_pcie30x2_slv", "aclk_pipe", 0, ++ RK3568_CLKGATE_CON(13), 1, GFLAGS), ++ GATE(ACLK_PCIE30X2_DBI, "aclk_pcie30x2_dbi", "aclk_pipe", 0, ++ RK3568_CLKGATE_CON(13), 2, GFLAGS), ++ GATE(PCLK_PCIE30X2, "pclk_pcie30x2", "pclk_pipe", 0, ++ RK3568_CLKGATE_CON(13), 3, GFLAGS), ++ GATE(CLK_PCIE30X2_AUX_NDFT, "clk_pcie30x2_aux_ndft", "xin24m", 0, ++ RK3568_CLKGATE_CON(13), 4, GFLAGS), ++ GATE(ACLK_SATA0, "aclk_sata0", "aclk_pipe", 0, ++ RK3568_CLKGATE_CON(11), 0, GFLAGS), ++ GATE(CLK_SATA0_PMALIVE, "clk_sata0_pmalive", "gpll_20m", 0, ++ RK3568_CLKGATE_CON(11), 1, GFLAGS), ++ GATE(CLK_SATA0_RXOOB, "clk_sata0_rxoob", "cpll_50m", 0, ++ RK3568_CLKGATE_CON(11), 2, GFLAGS), ++ GATE(ACLK_SATA1, "aclk_sata1", "aclk_pipe", 0, ++ RK3568_CLKGATE_CON(11), 4, GFLAGS), ++ GATE(CLK_SATA1_PMALIVE, "clk_sata1_pmalive", "gpll_20m", 0, ++ RK3568_CLKGATE_CON(11), 5, GFLAGS), ++ GATE(CLK_SATA1_RXOOB, "clk_sata1_rxoob", "cpll_50m", 0, ++ RK3568_CLKGATE_CON(11), 6, GFLAGS), ++ GATE(ACLK_SATA2, "aclk_sata2", "aclk_pipe", 0, ++ RK3568_CLKGATE_CON(11), 8, GFLAGS), ++ GATE(CLK_SATA2_PMALIVE, "clk_sata2_pmalive", "gpll_20m", 0, ++ RK3568_CLKGATE_CON(11), 9, GFLAGS), ++ GATE(CLK_SATA2_RXOOB, "clk_sata2_rxoob", "cpll_50m", 0, ++ RK3568_CLKGATE_CON(11), 10, GFLAGS), ++ GATE(ACLK_USB3OTG0, "aclk_usb3otg0", "aclk_pipe", 0, ++ RK3568_CLKGATE_CON(10), 8, GFLAGS), ++ GATE(CLK_USB3OTG0_REF, "clk_usb3otg0_ref", "xin24m", 0, ++ RK3568_CLKGATE_CON(10), 9, GFLAGS), ++ COMPOSITE_NODIV(CLK_USB3OTG0_SUSPEND, "clk_usb3otg0_suspend", xin24m_32k_p, 0, ++ RK3568_CLKSEL_CON(29), 8, 1, MFLAGS, ++ RK3568_CLKGATE_CON(10), 10, GFLAGS), ++ GATE(ACLK_USB3OTG1, "aclk_usb3otg1", "aclk_pipe", 0, ++ RK3568_CLKGATE_CON(10), 12, GFLAGS), ++ GATE(CLK_USB3OTG1_REF, "clk_usb3otg1_ref", "xin24m", 0, ++ RK3568_CLKGATE_CON(10), 13, GFLAGS), ++ COMPOSITE_NODIV(CLK_USB3OTG1_SUSPEND, "clk_usb3otg1_suspend", xin24m_32k_p, 0, ++ RK3568_CLKSEL_CON(29), 9, 1, MFLAGS, ++ RK3568_CLKGATE_CON(10), 14, GFLAGS), ++ COMPOSITE_NODIV(CLK_XPCS_EEE, "clk_xpcs_eee", gpll200_cpll125_p, 0, ++ RK3568_CLKSEL_CON(29), 13, 1, MFLAGS, ++ RK3568_CLKGATE_CON(10), 4, GFLAGS), ++ GATE(PCLK_XPCS, "pclk_xpcs", "pclk_pipe", 0, ++ RK3568_CLKGATE_CON(13), 6, GFLAGS), ++ ++ /* PD_PHP */ ++ COMPOSITE_NODIV(ACLK_PHP, "aclk_php", gpll300_gpll200_gpll100_xin24m_p, 0, ++ RK3568_CLKSEL_CON(30), 0, 2, MFLAGS, ++ RK3568_CLKGATE_CON(14), 8, GFLAGS), ++ COMPOSITE_NODIV(HCLK_PHP, "hclk_php", gpll150_gpll100_gpll75_xin24m_p, CLK_IS_CRITICAL, ++ RK3568_CLKSEL_CON(30), 2, 2, MFLAGS, ++ RK3568_CLKGATE_CON(14), 9, GFLAGS), ++ COMPOSITE_NOMUX(PCLK_PHP, "pclk_php", "aclk_php", CLK_IS_CRITICAL, ++ RK3568_CLKSEL_CON(30), 4, 4, DFLAGS, ++ RK3568_CLKGATE_CON(14), 10, GFLAGS), ++ GATE(HCLK_SDMMC0, "hclk_sdmmc0", "hclk_php", 0, ++ RK3568_CLKGATE_CON(15), 0, GFLAGS), ++ COMPOSITE_NODIV(CLK_SDMMC0, "clk_sdmmc0", clk_sdmmc_p, 0, ++ RK3568_CLKSEL_CON(30), 8, 3, MFLAGS, ++ RK3568_CLKGATE_CON(15), 1, GFLAGS), ++ MMC(SCLK_SDMMC0_DRV, "sdmmc0_drv", "clk_sdmmc0", RK3568_SDMMC0_CON0, 1), ++ MMC(SCLK_SDMMC0_SAMPLE, "sdmmc0_sample", "clk_sdmmc0", RK3568_SDMMC0_CON1, 1), ++ ++ GATE(HCLK_SDMMC1, "hclk_sdmmc1", "hclk_php", 0, ++ RK3568_CLKGATE_CON(15), 2, GFLAGS), ++ COMPOSITE_NODIV(CLK_SDMMC1, "clk_sdmmc1", clk_sdmmc_p, 0, ++ RK3568_CLKSEL_CON(30), 12, 3, MFLAGS, ++ RK3568_CLKGATE_CON(15), 3, GFLAGS), ++ MMC(SCLK_SDMMC1_DRV, "sdmmc1_drv", "clk_sdmmc1", RK3568_SDMMC1_CON0, 1), ++ MMC(SCLK_SDMMC1_SAMPLE, "sdmmc1_sample", "clk_sdmmc1", RK3568_SDMMC1_CON1, 1), ++ ++ GATE(ACLK_GMAC0, "aclk_gmac0", "aclk_php", 0, ++ RK3568_CLKGATE_CON(15), 5, GFLAGS), ++ GATE(PCLK_GMAC0, "pclk_gmac0", "pclk_php", 0, ++ RK3568_CLKGATE_CON(15), 6, GFLAGS), ++ COMPOSITE_NODIV(CLK_MAC0_2TOP, "clk_mac0_2top", clk_mac_2top_p, 0, ++ RK3568_CLKSEL_CON(31), 8, 2, MFLAGS, ++ RK3568_CLKGATE_CON(15), 7, GFLAGS), ++ COMPOSITE_NODIV(CLK_MAC0_OUT, "clk_mac0_out", cpll125_cpll50_cpll25_xin24m_p, 0, ++ RK3568_CLKSEL_CON(31), 14, 2, MFLAGS, ++ RK3568_CLKGATE_CON(15), 8, GFLAGS), ++ GATE(CLK_MAC0_REFOUT, "clk_mac0_refout", "clk_mac0_2top", 0, ++ RK3568_CLKGATE_CON(15), 12, GFLAGS), ++ COMPOSITE_NODIV(CLK_GMAC0_PTP_REF, "clk_gmac0_ptp_ref", clk_gmac_ptp_p, 0, ++ RK3568_CLKSEL_CON(31), 12, 2, MFLAGS, ++ RK3568_CLKGATE_CON(15), 4, GFLAGS), ++ MUX(SCLK_GMAC0, "clk_gmac0", mux_gmac0_p, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, ++ RK3568_CLKSEL_CON(31), 2, 1, MFLAGS), ++ FACTOR(0, "clk_gmac0_tx_div5", "clk_gmac0", 0, 1, 5), ++ FACTOR(0, "clk_gmac0_tx_div50", "clk_gmac0", 0, 1, 50), ++ FACTOR(0, "clk_gmac0_rx_div2", "clk_gmac0", 0, 1, 2), ++ FACTOR(0, "clk_gmac0_rx_div20", "clk_gmac0", 0, 1, 20), ++ MUX(SCLK_GMAC0_RGMII_SPEED, "clk_gmac0_rgmii_speed", mux_gmac0_rgmii_speed_p, 0, ++ RK3568_CLKSEL_CON(31), 4, 2, MFLAGS), ++ MUX(SCLK_GMAC0_RMII_SPEED, "clk_gmac0_rmii_speed", mux_gmac0_rmii_speed_p, 0, ++ RK3568_CLKSEL_CON(31), 3, 1, MFLAGS), ++ MUX(SCLK_GMAC0_RX_TX, "clk_gmac0_rx_tx", mux_gmac0_rx_tx_p, CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(31), 0, 2, MFLAGS), ++ ++ /* PD_USB */ ++ COMPOSITE_NODIV(ACLK_USB, "aclk_usb", gpll300_gpll200_gpll100_xin24m_p, 0, ++ RK3568_CLKSEL_CON(32), 0, 2, MFLAGS, ++ RK3568_CLKGATE_CON(16), 0, GFLAGS), ++ COMPOSITE_NODIV(HCLK_USB, "hclk_usb", gpll150_gpll100_gpll75_xin24m_p, CLK_IS_CRITICAL, ++ RK3568_CLKSEL_CON(32), 2, 2, MFLAGS, ++ RK3568_CLKGATE_CON(16), 1, GFLAGS), ++ COMPOSITE_NOMUX(PCLK_USB, "pclk_usb", "aclk_usb", 0, ++ RK3568_CLKSEL_CON(32), 4, 4, DFLAGS, ++ RK3568_CLKGATE_CON(16), 2, GFLAGS), ++ GATE(HCLK_USB2HOST0, "hclk_usb2host0", "hclk_usb", 0, ++ RK3568_CLKGATE_CON(16), 12, GFLAGS), ++ GATE(HCLK_USB2HOST0_ARB, "hclk_usb2host0_arb", "hclk_usb", 0, ++ RK3568_CLKGATE_CON(16), 13, GFLAGS), ++ GATE(HCLK_USB2HOST1, "hclk_usb2host1", "hclk_usb", 0, ++ RK3568_CLKGATE_CON(16), 14, GFLAGS), ++ GATE(HCLK_USB2HOST1_ARB, "hclk_usb2host1_arb", "hclk_usb", 0, ++ RK3568_CLKGATE_CON(16), 15, GFLAGS), ++ GATE(HCLK_SDMMC2, "hclk_sdmmc2", "hclk_usb", 0, ++ RK3568_CLKGATE_CON(17), 0, GFLAGS), ++ COMPOSITE_NODIV(CLK_SDMMC2, "clk_sdmmc2", clk_sdmmc_p, 0, ++ RK3568_CLKSEL_CON(32), 8, 3, MFLAGS, ++ RK3568_CLKGATE_CON(17), 1, GFLAGS), ++ MMC(SCLK_SDMMC2_DRV, "sdmmc2_drv", "clk_sdmmc2", RK3568_SDMMC2_CON0, 1), ++ MMC(SCLK_SDMMC2_SAMPLE, "sdmmc2_sample", "clk_sdmmc2", RK3568_SDMMC2_CON1, 1), ++ ++ GATE(ACLK_GMAC1, "aclk_gmac1", "aclk_usb", 0, ++ RK3568_CLKGATE_CON(17), 3, GFLAGS), ++ GATE(PCLK_GMAC1, "pclk_gmac1", "pclk_usb", 0, ++ RK3568_CLKGATE_CON(17), 4, GFLAGS), ++ COMPOSITE_NODIV(CLK_MAC1_2TOP, "clk_mac1_2top", clk_mac_2top_p, 0, ++ RK3568_CLKSEL_CON(33), 8, 2, MFLAGS, ++ RK3568_CLKGATE_CON(17), 5, GFLAGS), ++ COMPOSITE_NODIV(CLK_MAC1_OUT, "clk_mac1_out", cpll125_cpll50_cpll25_xin24m_p, 0, ++ RK3568_CLKSEL_CON(33), 14, 2, MFLAGS, ++ RK3568_CLKGATE_CON(17), 6, GFLAGS), ++ GATE(CLK_MAC1_REFOUT, "clk_mac1_refout", "clk_mac1_2top", 0, ++ RK3568_CLKGATE_CON(17), 10, GFLAGS), ++ COMPOSITE_NODIV(CLK_GMAC1_PTP_REF, "clk_gmac1_ptp_ref", clk_gmac_ptp_p, 0, ++ RK3568_CLKSEL_CON(33), 12, 2, MFLAGS, ++ RK3568_CLKGATE_CON(17), 2, GFLAGS), ++ MUX(SCLK_GMAC1, "clk_gmac1", mux_gmac1_p, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, ++ RK3568_CLKSEL_CON(33), 2, 1, MFLAGS), ++ FACTOR(0, "clk_gmac1_tx_div5", "clk_gmac1", 0, 1, 5), ++ FACTOR(0, "clk_gmac1_tx_div50", "clk_gmac1", 0, 1, 50), ++ FACTOR(0, "clk_gmac1_rx_div2", "clk_gmac1", 0, 1, 2), ++ FACTOR(0, "clk_gmac1_rx_div20", "clk_gmac1", 0, 1, 20), ++ MUX(SCLK_GMAC1_RGMII_SPEED, "clk_gmac1_rgmii_speed", mux_gmac1_rgmii_speed_p, 0, ++ RK3568_CLKSEL_CON(33), 4, 2, MFLAGS), ++ MUX(SCLK_GMAC1_RMII_SPEED, "clk_gmac1_rmii_speed", mux_gmac1_rmii_speed_p, 0, ++ RK3568_CLKSEL_CON(33), 3, 1, MFLAGS), ++ MUX(SCLK_GMAC1_RX_TX, "clk_gmac1_rx_tx", mux_gmac1_rx_tx_p, CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(33), 0, 2, MFLAGS), ++ ++ /* PD_PERI */ ++ COMPOSITE_NODIV(ACLK_PERIMID, "aclk_perimid", gpll300_gpll200_gpll100_xin24m_p, CLK_IS_CRITICAL, ++ RK3568_CLKSEL_CON(10), 4, 2, MFLAGS, ++ RK3568_CLKGATE_CON(14), 0, GFLAGS), ++ COMPOSITE_NODIV(HCLK_PERIMID, "hclk_perimid", gpll150_gpll100_gpll75_xin24m_p, CLK_IS_CRITICAL, ++ RK3568_CLKSEL_CON(10), 6, 2, MFLAGS, ++ RK3568_CLKGATE_CON(14), 1, GFLAGS), ++ ++ /* PD_VI */ ++ COMPOSITE_NODIV(ACLK_VI, "aclk_vi", gpll400_gpll300_gpll200_xin24m_p, 0, ++ RK3568_CLKSEL_CON(34), 0, 2, MFLAGS, ++ RK3568_CLKGATE_CON(18), 0, GFLAGS), ++ COMPOSITE_NOMUX(HCLK_VI, "hclk_vi", "aclk_vi", 0, ++ RK3568_CLKSEL_CON(34), 4, 4, DFLAGS, ++ RK3568_CLKGATE_CON(18), 1, GFLAGS), ++ COMPOSITE_NOMUX(PCLK_VI, "pclk_vi", "aclk_vi", 0, ++ RK3568_CLKSEL_CON(34), 8, 4, DFLAGS, ++ RK3568_CLKGATE_CON(18), 2, GFLAGS), ++ GATE(ACLK_VICAP, "aclk_vicap", "aclk_vi", 0, ++ RK3568_CLKGATE_CON(18), 9, GFLAGS), ++ GATE(HCLK_VICAP, "hclk_vicap", "hclk_vi", 0, ++ RK3568_CLKGATE_CON(18), 10, GFLAGS), ++ COMPOSITE_NODIV(DCLK_VICAP, "dclk_vicap", cpll333_gpll300_gpll200_p, 0, ++ RK3568_CLKSEL_CON(34), 14, 2, MFLAGS, ++ RK3568_CLKGATE_CON(18), 11, GFLAGS), ++ GATE(ICLK_VICAP_G, "iclk_vicap_g", "iclk_vicap", 0, ++ RK3568_CLKGATE_CON(18), 13, GFLAGS), ++ GATE(ACLK_ISP, "aclk_isp", "aclk_vi", 0, ++ RK3568_CLKGATE_CON(19), 0, GFLAGS), ++ GATE(HCLK_ISP, "hclk_isp", "hclk_vi", 0, ++ RK3568_CLKGATE_CON(19), 1, GFLAGS), ++ COMPOSITE(CLK_ISP, "clk_isp", cpll_gpll_hpll_p, 0, ++ RK3568_CLKSEL_CON(35), 6, 2, MFLAGS, 0, 5, DFLAGS, ++ RK3568_CLKGATE_CON(19), 2, GFLAGS), ++ GATE(PCLK_CSI2HOST1, "pclk_csi2host1", "pclk_vi", 0, ++ RK3568_CLKGATE_CON(19), 4, GFLAGS), ++ COMPOSITE(CLK_CIF_OUT, "clk_cif_out", gpll_usb480m_xin24m_p, 0, ++ RK3568_CLKSEL_CON(35), 14, 2, MFLAGS, 8, 6, DFLAGS, ++ RK3568_CLKGATE_CON(19), 8, GFLAGS), ++ COMPOSITE(CLK_CAM0_OUT, "clk_cam0_out", gpll_usb480m_xin24m_p, 0, ++ RK3568_CLKSEL_CON(36), 6, 2, MFLAGS, 0, 6, DFLAGS, ++ RK3568_CLKGATE_CON(19), 9, GFLAGS), ++ COMPOSITE(CLK_CAM1_OUT, "clk_cam1_out", gpll_usb480m_xin24m_p, 0, ++ RK3568_CLKSEL_CON(36), 14, 2, MFLAGS, 8, 6, DFLAGS, ++ RK3568_CLKGATE_CON(19), 10, GFLAGS), ++ ++ /* PD_VO */ ++ COMPOSITE_NODIV(ACLK_VO, "aclk_vo", gpll300_cpll250_gpll100_xin24m_p, 0, ++ RK3568_CLKSEL_CON(37), 0, 2, MFLAGS, ++ RK3568_CLKGATE_CON(20), 0, GFLAGS), ++ COMPOSITE_NOMUX(HCLK_VO, "hclk_vo", "aclk_vo", 0, ++ RK3568_CLKSEL_CON(37), 8, 4, DFLAGS, ++ RK3568_CLKGATE_CON(20), 1, GFLAGS), ++ COMPOSITE_NOMUX(PCLK_VO, "pclk_vo", "aclk_vo", 0, ++ RK3568_CLKSEL_CON(37), 12, 4, DFLAGS, ++ RK3568_CLKGATE_CON(20), 2, GFLAGS), ++ COMPOSITE(ACLK_VOP_PRE, "aclk_vop_pre", cpll_gpll_hpll_vpll_p, 0, ++ RK3568_CLKSEL_CON(38), 6, 2, MFLAGS, 0, 5, DFLAGS, ++ RK3568_CLKGATE_CON(20), 6, GFLAGS), ++ GATE(ACLK_VOP, "aclk_vop", "aclk_vop_pre", 0, ++ RK3568_CLKGATE_CON(20), 8, GFLAGS), ++ GATE(HCLK_VOP, "hclk_vop", "hclk_vo", 0, ++ RK3568_CLKGATE_CON(20), 9, GFLAGS), ++ COMPOSITE(DCLK_VOP0, "dclk_vop0", hpll_vpll_gpll_cpll_p, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, ++ RK3568_CLKSEL_CON(39), 10, 2, MFLAGS, 0, 8, DFLAGS, ++ RK3568_CLKGATE_CON(20), 10, GFLAGS), ++ COMPOSITE_DCLK(DCLK_VOP1, "dclk_vop1", hpll_vpll_gpll_cpll_p, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, ++ RK3568_CLKSEL_CON(40), 10, 2, MFLAGS, 0, 8, DFLAGS, ++ RK3568_CLKGATE_CON(20), 11, GFLAGS, RK3568_DCLK_PARENT_MAX_PRATE), ++ COMPOSITE(DCLK_VOP2, "dclk_vop2", hpll_vpll_gpll_cpll_p, 0, ++ RK3568_CLKSEL_CON(41), 10, 2, MFLAGS, 0, 8, DFLAGS, ++ RK3568_CLKGATE_CON(20), 12, GFLAGS), ++ GATE(CLK_VOP_PWM, "clk_vop_pwm", "xin24m", 0, ++ RK3568_CLKGATE_CON(20), 13, GFLAGS), ++ GATE(ACLK_HDCP, "aclk_hdcp", "aclk_vo", 0, ++ RK3568_CLKGATE_CON(21), 0, GFLAGS), ++ GATE(HCLK_HDCP, "hclk_hdcp", "hclk_vo", 0, ++ RK3568_CLKGATE_CON(21), 1, GFLAGS), ++ GATE(PCLK_HDCP, "pclk_hdcp", "pclk_vo", 0, ++ RK3568_CLKGATE_CON(21), 2, GFLAGS), ++ GATE(PCLK_HDMI_HOST, "pclk_hdmi_host", "pclk_vo", 0, ++ RK3568_CLKGATE_CON(21), 3, GFLAGS), ++ GATE(CLK_HDMI_SFR, "clk_hdmi_sfr", "xin24m", 0, ++ RK3568_CLKGATE_CON(21), 4, GFLAGS), ++ GATE(CLK_HDMI_CEC, "clk_hdmi_cec", "clk_rtc_32k", 0, ++ RK3568_CLKGATE_CON(21), 5, GFLAGS), ++ GATE(PCLK_DSITX_0, "pclk_dsitx_0", "pclk_vo", 0, ++ RK3568_CLKGATE_CON(21), 6, GFLAGS), ++ GATE(PCLK_DSITX_1, "pclk_dsitx_1", "pclk_vo", 0, ++ RK3568_CLKGATE_CON(21), 7, GFLAGS), ++ GATE(PCLK_EDP_CTRL, "pclk_edp_ctrl", "pclk_vo", 0, ++ RK3568_CLKGATE_CON(21), 8, GFLAGS), ++ COMPOSITE_NODIV(CLK_EDP_200M, "clk_edp_200m", gpll200_gpll150_cpll125_p, 0, ++ RK3568_CLKSEL_CON(38), 8, 2, MFLAGS, ++ RK3568_CLKGATE_CON(21), 9, GFLAGS), ++ ++ /* PD_VPU */ ++ COMPOSITE(ACLK_VPU_PRE, "aclk_vpu_pre", gpll_cpll_p, 0, ++ RK3568_CLKSEL_CON(42), 7, 1, MFLAGS, 0, 5, DFLAGS, ++ RK3568_CLKGATE_CON(22), 0, GFLAGS), ++ COMPOSITE_NOMUX(HCLK_VPU_PRE, "hclk_vpu_pre", "aclk_vpu_pre", 0, ++ RK3568_CLKSEL_CON(42), 8, 4, DFLAGS, ++ RK3568_CLKGATE_CON(22), 1, GFLAGS), ++ GATE(ACLK_VPU, "aclk_vpu", "aclk_vpu_pre", 0, ++ RK3568_CLKGATE_CON(22), 4, GFLAGS), ++ GATE(HCLK_VPU, "hclk_vpu", "hclk_vpu_pre", 0, ++ RK3568_CLKGATE_CON(22), 5, GFLAGS), ++ ++ /* PD_RGA */ ++ COMPOSITE_NODIV(ACLK_RGA_PRE, "aclk_rga_pre", gpll300_cpll250_gpll100_xin24m_p, 0, ++ RK3568_CLKSEL_CON(43), 0, 2, MFLAGS, ++ RK3568_CLKGATE_CON(23), 0, GFLAGS), ++ COMPOSITE_NOMUX(HCLK_RGA_PRE, "hclk_rga_pre", "aclk_rga_pre", 0, ++ RK3568_CLKSEL_CON(43), 8, 4, DFLAGS, ++ RK3568_CLKGATE_CON(23), 1, GFLAGS), ++ COMPOSITE_NOMUX(PCLK_RGA_PRE, "pclk_rga_pre", "aclk_rga_pre", 0, ++ RK3568_CLKSEL_CON(43), 12, 4, DFLAGS, ++ RK3568_CLKGATE_CON(22), 12, GFLAGS), ++ GATE(ACLK_RGA, "aclk_rga", "aclk_rga_pre", 0, ++ RK3568_CLKGATE_CON(23), 4, GFLAGS), ++ GATE(HCLK_RGA, "hclk_rga", "hclk_rga_pre", 0, ++ RK3568_CLKGATE_CON(23), 5, GFLAGS), ++ COMPOSITE_NODIV(CLK_RGA_CORE, "clk_rga_core", gpll300_gpll200_gpll100_p, 0, ++ RK3568_CLKSEL_CON(43), 2, 2, MFLAGS, ++ RK3568_CLKGATE_CON(23), 6, GFLAGS), ++ GATE(ACLK_IEP, "aclk_iep", "aclk_rga_pre", 0, ++ RK3568_CLKGATE_CON(23), 7, GFLAGS), ++ GATE(HCLK_IEP, "hclk_iep", "hclk_rga_pre", 0, ++ RK3568_CLKGATE_CON(23), 8, GFLAGS), ++ COMPOSITE_NODIV(CLK_IEP_CORE, "clk_iep_core", gpll300_gpll200_gpll100_p, 0, ++ RK3568_CLKSEL_CON(43), 4, 2, MFLAGS, ++ RK3568_CLKGATE_CON(23), 9, GFLAGS), ++ GATE(HCLK_EBC, "hclk_ebc", "hclk_rga_pre", 0, RK3568_CLKGATE_CON(23), 10, GFLAGS), ++ COMPOSITE_NODIV(DCLK_EBC, "dclk_ebc", gpll400_cpll333_gpll200_p, 0, ++ RK3568_CLKSEL_CON(43), 6, 2, MFLAGS, ++ RK3568_CLKGATE_CON(23), 11, GFLAGS), ++ GATE(ACLK_JDEC, "aclk_jdec", "aclk_rga_pre", 0, ++ RK3568_CLKGATE_CON(23), 12, GFLAGS), ++ GATE(HCLK_JDEC, "hclk_jdec", "hclk_rga_pre", 0, ++ RK3568_CLKGATE_CON(23), 13, GFLAGS), ++ GATE(ACLK_JENC, "aclk_jenc", "aclk_rga_pre", 0, ++ RK3568_CLKGATE_CON(23), 14, GFLAGS), ++ GATE(HCLK_JENC, "hclk_jenc", "hclk_rga_pre", 0, ++ RK3568_CLKGATE_CON(23), 15, GFLAGS), ++ GATE(PCLK_EINK, "pclk_eink", "pclk_rga_pre", 0, ++ RK3568_CLKGATE_CON(22), 14, GFLAGS), ++ GATE(HCLK_EINK, "hclk_eink", "hclk_rga_pre", 0, ++ RK3568_CLKGATE_CON(22), 15, GFLAGS), ++ ++ /* PD_RKVENC */ ++ COMPOSITE(ACLK_RKVENC_PRE, "aclk_rkvenc_pre", gpll_cpll_npll_p, 0, ++ RK3568_CLKSEL_CON(44), 6, 2, MFLAGS, 0, 5, DFLAGS, ++ RK3568_CLKGATE_CON(24), 0, GFLAGS), ++ COMPOSITE_NOMUX(HCLK_RKVENC_PRE, "hclk_rkvenc_pre", "aclk_rkvenc_pre", 0, ++ RK3568_CLKSEL_CON(44), 8, 4, DFLAGS, ++ RK3568_CLKGATE_CON(24), 1, GFLAGS), ++ GATE(ACLK_RKVENC, "aclk_rkvenc", "aclk_rkvenc_pre", 0, ++ RK3568_CLKGATE_CON(24), 6, GFLAGS), ++ GATE(HCLK_RKVENC, "hclk_rkvenc", "hclk_rkvenc_pre", 0, ++ RK3568_CLKGATE_CON(24), 7, GFLAGS), ++ COMPOSITE(CLK_RKVENC_CORE, "clk_rkvenc_core", gpll_cpll_npll_vpll_p, 0, ++ RK3568_CLKSEL_CON(45), 14, 2, MFLAGS, 0, 5, DFLAGS, ++ RK3568_CLKGATE_CON(24), 8, GFLAGS), ++ COMPOSITE(ACLK_RKVDEC_PRE, "aclk_rkvdec_pre", aclk_rkvdec_pre_p, CLK_SET_RATE_NO_REPARENT, ++ RK3568_CLKSEL_CON(47), 7, 1, MFLAGS, 0, 5, DFLAGS, ++ RK3568_CLKGATE_CON(25), 0, GFLAGS), ++ COMPOSITE_NOMUX(HCLK_RKVDEC_PRE, "hclk_rkvdec_pre", "aclk_rkvdec_pre", 0, ++ RK3568_CLKSEL_CON(47), 8, 4, DFLAGS, ++ RK3568_CLKGATE_CON(25), 1, GFLAGS), ++ GATE(ACLK_RKVDEC, "aclk_rkvdec", "aclk_rkvdec_pre", 0, ++ RK3568_CLKGATE_CON(25), 4, GFLAGS), ++ GATE(HCLK_RKVDEC, "hclk_rkvdec", "hclk_rkvdec_pre", 0, ++ RK3568_CLKGATE_CON(25), 5, GFLAGS), ++ COMPOSITE(CLK_RKVDEC_CA, "clk_rkvdec_ca", gpll_cpll_npll_vpll_p, 0, ++ RK3568_CLKSEL_CON(48), 6, 2, MFLAGS, 0, 5, DFLAGS, ++ RK3568_CLKGATE_CON(25), 6, GFLAGS), ++ COMPOSITE(CLK_RKVDEC_CORE, "clk_rkvdec_core", clk_rkvdec_core_p, CLK_SET_RATE_NO_REPARENT, ++ RK3568_CLKSEL_CON(49), 14, 2, MFLAGS, 8, 5, DFLAGS, ++ RK3568_CLKGATE_CON(25), 7, GFLAGS), ++ COMPOSITE(CLK_RKVDEC_HEVC_CA, "clk_rkvdec_hevc_ca", gpll_cpll_npll_vpll_p, 0, ++ RK3568_CLKSEL_CON(49), 6, 2, MFLAGS, 0, 5, DFLAGS, ++ RK3568_CLKGATE_CON(25), 8, GFLAGS), ++ ++ /* PD_BUS */ ++ COMPOSITE_NODIV(ACLK_BUS, "aclk_bus", gpll200_gpll150_gpll100_xin24m_p, CLK_IS_CRITICAL, ++ RK3568_CLKSEL_CON(50), 0, 2, MFLAGS, ++ RK3568_CLKGATE_CON(26), 0, GFLAGS), ++ COMPOSITE_NODIV(PCLK_BUS, "pclk_bus", gpll100_gpll75_cpll50_xin24m_p, CLK_IS_CRITICAL, ++ RK3568_CLKSEL_CON(50), 4, 2, MFLAGS, ++ RK3568_CLKGATE_CON(26), 1, GFLAGS), ++ GATE(PCLK_TSADC, "pclk_tsadc", "pclk_bus", 0, ++ RK3568_CLKGATE_CON(26), 4, GFLAGS), ++ COMPOSITE(CLK_TSADC_TSEN, "clk_tsadc_tsen", xin24m_gpll100_cpll100_p, 0, ++ RK3568_CLKSEL_CON(51), 4, 2, MFLAGS, 0, 3, DFLAGS, ++ RK3568_CLKGATE_CON(26), 5, GFLAGS), ++ COMPOSITE_NOMUX(CLK_TSADC, "clk_tsadc", "clk_tsadc_tsen", 0, ++ RK3568_CLKSEL_CON(51), 8, 7, DFLAGS, ++ RK3568_CLKGATE_CON(26), 6, GFLAGS), ++ GATE(PCLK_SARADC, "pclk_saradc", "pclk_bus", 0, ++ RK3568_CLKGATE_CON(26), 7, GFLAGS), ++ GATE(CLK_SARADC, "clk_saradc", "xin24m", 0, ++ RK3568_CLKGATE_CON(26), 8, GFLAGS), ++ GATE(PCLK_SCR, "pclk_scr", "pclk_bus", CLK_IGNORE_UNUSED, ++ RK3568_CLKGATE_CON(26), 12, GFLAGS), ++ GATE(PCLK_WDT_NS, "pclk_wdt_ns", "pclk_bus", 0, ++ RK3568_CLKGATE_CON(26), 13, GFLAGS), ++ GATE(TCLK_WDT_NS, "tclk_wdt_ns", "xin24m", 0, ++ RK3568_CLKGATE_CON(26), 14, GFLAGS), ++ GATE(ACLK_MCU, "aclk_mcu", "aclk_bus", CLK_IGNORE_UNUSED, ++ RK3568_CLKGATE_CON(32), 13, GFLAGS), ++ GATE(PCLK_INTMUX, "pclk_intmux", "pclk_bus", CLK_IGNORE_UNUSED, ++ RK3568_CLKGATE_CON(32), 14, GFLAGS), ++ GATE(PCLK_MAILBOX, "pclk_mailbox", "pclk_bus", 0, ++ RK3568_CLKGATE_CON(32), 15, GFLAGS), ++ ++ GATE(PCLK_UART1, "pclk_uart1", "pclk_bus", 0, ++ RK3568_CLKGATE_CON(27), 12, GFLAGS), ++ COMPOSITE(CLK_UART1_SRC, "clk_uart1_src", gpll_cpll_usb480m_p, 0, ++ RK3568_CLKSEL_CON(52), 8, 2, MFLAGS, 0, 7, DFLAGS, ++ RK3568_CLKGATE_CON(27), 13, GFLAGS), ++ COMPOSITE_FRACMUX(CLK_UART1_FRAC, "clk_uart1_frac", "clk_uart1_src", CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(53), CLK_FRAC_DIVIDER_NO_LIMIT, ++ RK3568_CLKGATE_CON(27), 14, GFLAGS, ++ &rk3568_uart1_fracmux, RK3568_UART_FRAC_MAX_PRATE), ++ GATE(SCLK_UART1, "sclk_uart1", "sclk_uart1_mux", 0, ++ RK3568_CLKGATE_CON(27), 15, GFLAGS), ++ ++ GATE(PCLK_UART2, "pclk_uart2", "pclk_bus", 0, ++ RK3568_CLKGATE_CON(28), 0, GFLAGS), ++ COMPOSITE(CLK_UART2_SRC, "clk_uart2_src", gpll_cpll_usb480m_p, 0, ++ RK3568_CLKSEL_CON(54), 8, 2, MFLAGS, 0, 7, DFLAGS, ++ RK3568_CLKGATE_CON(28), 1, GFLAGS), ++ COMPOSITE_FRACMUX(CLK_UART2_FRAC, "clk_uart2_frac", "clk_uart2_src", CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(55), CLK_FRAC_DIVIDER_NO_LIMIT, ++ RK3568_CLKGATE_CON(28), 2, GFLAGS, ++ &rk3568_uart2_fracmux, RK3568_UART_FRAC_MAX_PRATE), ++ GATE(SCLK_UART2, "sclk_uart2", "sclk_uart2_mux", 0, ++ RK3568_CLKGATE_CON(28), 3, GFLAGS), ++ ++ GATE(PCLK_UART3, "pclk_uart3", "pclk_bus", 0, ++ RK3568_CLKGATE_CON(28), 4, GFLAGS), ++ COMPOSITE(CLK_UART3_SRC, "clk_uart3_src", gpll_cpll_usb480m_p, 0, ++ RK3568_CLKSEL_CON(56), 8, 2, MFLAGS, 0, 7, DFLAGS, ++ RK3568_CLKGATE_CON(28), 5, GFLAGS), ++ COMPOSITE_FRACMUX(CLK_UART3_FRAC, "clk_uart3_frac", "clk_uart3_src", CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(57), CLK_FRAC_DIVIDER_NO_LIMIT, ++ RK3568_CLKGATE_CON(28), 6, GFLAGS, ++ &rk3568_uart3_fracmux, RK3568_UART_FRAC_MAX_PRATE), ++ GATE(SCLK_UART3, "sclk_uart3", "sclk_uart3_mux", 0, ++ RK3568_CLKGATE_CON(28), 7, GFLAGS), ++ ++ GATE(PCLK_UART4, "pclk_uart4", "pclk_bus", 0, ++ RK3568_CLKGATE_CON(28), 8, GFLAGS), ++ COMPOSITE(CLK_UART4_SRC, "clk_uart4_src", gpll_cpll_usb480m_p, 0, ++ RK3568_CLKSEL_CON(58), 8, 2, MFLAGS, 0, 7, DFLAGS, ++ RK3568_CLKGATE_CON(28), 9, GFLAGS), ++ COMPOSITE_FRACMUX(CLK_UART4_FRAC, "clk_uart4_frac", "clk_uart4_src", CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(59), CLK_FRAC_DIVIDER_NO_LIMIT, ++ RK3568_CLKGATE_CON(28), 10, GFLAGS, ++ &rk3568_uart4_fracmux, RK3568_UART_FRAC_MAX_PRATE), ++ GATE(SCLK_UART4, "sclk_uart4", "sclk_uart4_mux", 0, ++ RK3568_CLKGATE_CON(28), 11, GFLAGS), ++ ++ GATE(PCLK_UART5, "pclk_uart5", "pclk_bus", 0, ++ RK3568_CLKGATE_CON(28), 12, GFLAGS), ++ COMPOSITE(CLK_UART5_SRC, "clk_uart5_src", gpll_cpll_usb480m_p, 0, ++ RK3568_CLKSEL_CON(60), 8, 2, MFLAGS, 0, 7, DFLAGS, ++ RK3568_CLKGATE_CON(28), 13, GFLAGS), ++ COMPOSITE_FRACMUX(CLK_UART5_FRAC, "clk_uart5_frac", "clk_uart5_src", CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(61), CLK_FRAC_DIVIDER_NO_LIMIT, ++ RK3568_CLKGATE_CON(28), 14, GFLAGS, ++ &rk3568_uart5_fracmux, RK3568_UART_FRAC_MAX_PRATE), ++ GATE(SCLK_UART5, "sclk_uart5", "sclk_uart5_mux", 0, ++ RK3568_CLKGATE_CON(28), 15, GFLAGS), ++ ++ GATE(PCLK_UART6, "pclk_uart6", "pclk_bus", 0, ++ RK3568_CLKGATE_CON(29), 0, GFLAGS), ++ COMPOSITE(CLK_UART6_SRC, "clk_uart6_src", gpll_cpll_usb480m_p, 0, ++ RK3568_CLKSEL_CON(62), 8, 2, MFLAGS, 0, 7, DFLAGS, ++ RK3568_CLKGATE_CON(29), 1, GFLAGS), ++ COMPOSITE_FRACMUX(CLK_UART6_FRAC, "clk_uart6_frac", "clk_uart6_src", CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(63), CLK_FRAC_DIVIDER_NO_LIMIT, ++ RK3568_CLKGATE_CON(29), 2, GFLAGS, ++ &rk3568_uart6_fracmux, RK3568_UART_FRAC_MAX_PRATE), ++ GATE(SCLK_UART6, "sclk_uart6", "sclk_uart6_mux", 0, ++ RK3568_CLKGATE_CON(29), 3, GFLAGS), ++ ++ GATE(PCLK_UART7, "pclk_uart7", "pclk_bus", 0, ++ RK3568_CLKGATE_CON(29), 4, GFLAGS), ++ COMPOSITE(CLK_UART7_SRC, "clk_uart7_src", gpll_cpll_usb480m_p, 0, ++ RK3568_CLKSEL_CON(64), 8, 2, MFLAGS, 0, 7, DFLAGS, ++ RK3568_CLKGATE_CON(29), 5, GFLAGS), ++ COMPOSITE_FRACMUX(CLK_UART7_FRAC, "clk_uart7_frac", "clk_uart7_src", CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(65), CLK_FRAC_DIVIDER_NO_LIMIT, ++ RK3568_CLKGATE_CON(29), 6, GFLAGS, ++ &rk3568_uart7_fracmux, RK3568_UART_FRAC_MAX_PRATE), ++ GATE(SCLK_UART7, "sclk_uart7", "sclk_uart7_mux", 0, ++ RK3568_CLKGATE_CON(29), 7, GFLAGS), ++ ++ GATE(PCLK_UART8, "pclk_uart8", "pclk_bus", 0, ++ RK3568_CLKGATE_CON(29), 8, GFLAGS), ++ COMPOSITE(CLK_UART8_SRC, "clk_uart8_src", gpll_cpll_usb480m_p, 0, ++ RK3568_CLKSEL_CON(66), 8, 2, MFLAGS, 0, 7, DFLAGS, ++ RK3568_CLKGATE_CON(29), 9, GFLAGS), ++ COMPOSITE_FRACMUX(CLK_UART8_FRAC, "clk_uart8_frac", "clk_uart8_src", CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(67), CLK_FRAC_DIVIDER_NO_LIMIT, ++ RK3568_CLKGATE_CON(29), 10, GFLAGS, ++ &rk3568_uart8_fracmux, RK3568_UART_FRAC_MAX_PRATE), ++ GATE(SCLK_UART8, "sclk_uart8", "sclk_uart8_mux", 0, ++ RK3568_CLKGATE_CON(29), 11, GFLAGS), ++ ++ GATE(PCLK_UART9, "pclk_uart9", "pclk_bus", 0, ++ RK3568_CLKGATE_CON(29), 12, GFLAGS), ++ COMPOSITE(CLK_UART9_SRC, "clk_uart9_src", gpll_cpll_usb480m_p, 0, ++ RK3568_CLKSEL_CON(68), 8, 2, MFLAGS, 0, 7, DFLAGS, ++ RK3568_CLKGATE_CON(29), 13, GFLAGS), ++ COMPOSITE_FRACMUX(CLK_UART9_FRAC, "clk_uart9_frac", "clk_uart9_src", CLK_SET_RATE_PARENT, ++ RK3568_CLKSEL_CON(69), CLK_FRAC_DIVIDER_NO_LIMIT, ++ RK3568_CLKGATE_CON(29), 14, GFLAGS, ++ &rk3568_uart9_fracmux, RK3568_UART_FRAC_MAX_PRATE), ++ GATE(SCLK_UART9, "sclk_uart9", "sclk_uart9_mux", 0, ++ RK3568_CLKGATE_CON(29), 15, GFLAGS), ++ ++ GATE(PCLK_CAN0, "pclk_can0", "pclk_bus", 0, ++ RK3568_CLKGATE_CON(27), 5, GFLAGS), ++ COMPOSITE(CLK_CAN0, "clk_can0", gpll_cpll_p, 0, ++ RK3568_CLKSEL_CON(70), 7, 1, MFLAGS, 0, 5, DFLAGS, ++ RK3568_CLKGATE_CON(27), 6, GFLAGS), ++ GATE(PCLK_CAN1, "pclk_can1", "pclk_bus", 0, ++ RK3568_CLKGATE_CON(27), 7, GFLAGS), ++ COMPOSITE(CLK_CAN1, "clk_can1", gpll_cpll_p, 0, ++ RK3568_CLKSEL_CON(70), 15, 1, MFLAGS, 8, 5, DFLAGS, ++ RK3568_CLKGATE_CON(27), 8, GFLAGS), ++ GATE(PCLK_CAN2, "pclk_can2", "pclk_bus", 0, ++ RK3568_CLKGATE_CON(27), 9, GFLAGS), ++ COMPOSITE(CLK_CAN2, "clk_can2", gpll_cpll_p, 0, ++ RK3568_CLKSEL_CON(71), 7, 1, MFLAGS, 0, 5, DFLAGS, ++ RK3568_CLKGATE_CON(27), 10, GFLAGS), ++ COMPOSITE_NODIV(CLK_I2C, "clk_i2c", clk_i2c_p, 0, ++ RK3568_CLKSEL_CON(71), 8, 2, MFLAGS, ++ RK3568_CLKGATE_CON(32), 10, GFLAGS), ++ GATE(PCLK_I2C1, "pclk_i2c1", "pclk_bus", 0, ++ RK3568_CLKGATE_CON(30), 0, GFLAGS), ++ GATE(CLK_I2C1, "clk_i2c1", "clk_i2c", 0, ++ RK3568_CLKGATE_CON(30), 1, GFLAGS), ++ GATE(PCLK_I2C2, "pclk_i2c2", "pclk_bus", 0, ++ RK3568_CLKGATE_CON(30), 2, GFLAGS), ++ GATE(CLK_I2C2, "clk_i2c2", "clk_i2c", 0, ++ RK3568_CLKGATE_CON(30), 3, GFLAGS), ++ GATE(PCLK_I2C3, "pclk_i2c3", "pclk_bus", 0, ++ RK3568_CLKGATE_CON(30), 4, GFLAGS), ++ GATE(CLK_I2C3, "clk_i2c3", "clk_i2c", 0, ++ RK3568_CLKGATE_CON(30), 5, GFLAGS), ++ GATE(PCLK_I2C4, "pclk_i2c4", "pclk_bus", 0, ++ RK3568_CLKGATE_CON(30), 6, GFLAGS), ++ GATE(CLK_I2C4, "clk_i2c4", "clk_i2c", 0, ++ RK3568_CLKGATE_CON(30), 7, GFLAGS), ++ GATE(PCLK_I2C5, "pclk_i2c5", "pclk_bus", 0, ++ RK3568_CLKGATE_CON(30), 8, GFLAGS), ++ GATE(CLK_I2C5, "clk_i2c5", "clk_i2c", 0, ++ RK3568_CLKGATE_CON(30), 9, GFLAGS), ++ GATE(PCLK_SPI0, "pclk_spi0", "pclk_bus", 0, ++ RK3568_CLKGATE_CON(30), 10, GFLAGS), ++ COMPOSITE_NODIV(CLK_SPI0, "clk_spi0", gpll200_xin24m_cpll100_p, 0, ++ RK3568_CLKSEL_CON(72), 0, 1, MFLAGS, ++ RK3568_CLKGATE_CON(30), 11, GFLAGS), ++ GATE(PCLK_SPI1, "pclk_spi1", "pclk_bus", 0, ++ RK3568_CLKGATE_CON(30), 12, GFLAGS), ++ COMPOSITE_NODIV(CLK_SPI1, "clk_spi1", gpll200_xin24m_cpll100_p, 0, ++ RK3568_CLKSEL_CON(72), 2, 1, MFLAGS, ++ RK3568_CLKGATE_CON(30), 13, GFLAGS), ++ GATE(PCLK_SPI2, "pclk_spi2", "pclk_bus", 0, ++ RK3568_CLKGATE_CON(30), 14, GFLAGS), ++ COMPOSITE_NODIV(CLK_SPI2, "clk_spi2", gpll200_xin24m_cpll100_p, 0, ++ RK3568_CLKSEL_CON(72), 4, 1, MFLAGS, ++ RK3568_CLKGATE_CON(30), 15, GFLAGS), ++ GATE(PCLK_SPI3, "pclk_spi3", "pclk_bus", 0, ++ RK3568_CLKGATE_CON(31), 0, GFLAGS), ++ COMPOSITE_NODIV(CLK_SPI3, "clk_spi3", gpll200_xin24m_cpll100_p, 0, ++ RK3568_CLKSEL_CON(72), 6, 1, MFLAGS, RK3568_CLKGATE_CON(31), 1, GFLAGS), ++ GATE(PCLK_PWM1, "pclk_pwm1", "pclk_bus", 0, RK3568_CLKGATE_CON(31), 10, GFLAGS), ++ COMPOSITE_NODIV(CLK_PWM1, "clk_pwm1", gpll100_xin24m_cpll100_p, 0, ++ RK3568_CLKSEL_CON(72), 8, 1, MFLAGS, ++ RK3568_CLKGATE_CON(31), 11, GFLAGS), ++ GATE(CLK_PWM1_CAPTURE, "clk_pwm1_capture", "xin24m", 0, ++ RK3568_CLKGATE_CON(31), 12, GFLAGS), ++ GATE(PCLK_PWM2, "pclk_pwm2", "pclk_bus", 0, ++ RK3568_CLKGATE_CON(31), 13, GFLAGS), ++ COMPOSITE_NODIV(CLK_PWM2, "clk_pwm2", gpll100_xin24m_cpll100_p, 0, ++ RK3568_CLKSEL_CON(72), 10, 1, MFLAGS, ++ RK3568_CLKGATE_CON(31), 14, GFLAGS), ++ GATE(CLK_PWM2_CAPTURE, "clk_pwm2_capture", "xin24m", 0, ++ RK3568_CLKGATE_CON(31), 15, GFLAGS), ++ GATE(PCLK_PWM3, "pclk_pwm3", "pclk_bus", 0, ++ RK3568_CLKGATE_CON(32), 0, GFLAGS), ++ COMPOSITE_NODIV(CLK_PWM3, "clk_pwm3", gpll100_xin24m_cpll100_p, 0, ++ RK3568_CLKSEL_CON(72), 12, 1, MFLAGS, ++ RK3568_CLKGATE_CON(32), 1, GFLAGS), ++ GATE(CLK_PWM3_CAPTURE, "clk_pwm3_capture", "xin24m", 0, ++ RK3568_CLKGATE_CON(32), 2, GFLAGS), ++ COMPOSITE_NODIV(DBCLK_GPIO, "dbclk_gpio", xin24m_32k_p, 0, ++ RK3568_CLKSEL_CON(72), 14, 1, MFLAGS, ++ RK3568_CLKGATE_CON(32), 11, GFLAGS), ++ GATE(PCLK_GPIO1, "pclk_gpio1", "pclk_bus", 0, ++ RK3568_CLKGATE_CON(31), 2, GFLAGS), ++ GATE(DBCLK_GPIO1, "dbclk_gpio1", "dbclk_gpio", 0, ++ RK3568_CLKGATE_CON(31), 3, GFLAGS), ++ GATE(PCLK_GPIO2, "pclk_gpio2", "pclk_bus", 0, ++ RK3568_CLKGATE_CON(31), 4, GFLAGS), ++ GATE(DBCLK_GPIO2, "dbclk_gpio2", "dbclk_gpio", 0, ++ RK3568_CLKGATE_CON(31), 5, GFLAGS), ++ GATE(PCLK_GPIO3, "pclk_gpio3", "pclk_bus", 0, ++ RK3568_CLKGATE_CON(31), 6, GFLAGS), ++ GATE(DBCLK_GPIO3, "dbclk_gpio3", "dbclk_gpio", 0, ++ RK3568_CLKGATE_CON(31), 7, GFLAGS), ++ GATE(PCLK_GPIO4, "pclk_gpio4", "pclk_bus", 0, ++ RK3568_CLKGATE_CON(31), 8, GFLAGS), ++ GATE(DBCLK_GPIO4, "dbclk_gpio4", "dbclk_gpio", 0, ++ RK3568_CLKGATE_CON(31), 9, GFLAGS), ++ GATE(PCLK_TIMER, "pclk_timer", "pclk_bus", 0, ++ RK3568_CLKGATE_CON(32), 3, GFLAGS), ++ GATE(CLK_TIMER0, "clk_timer0", "xin24m", 0, ++ RK3568_CLKGATE_CON(32), 4, GFLAGS), ++ GATE(CLK_TIMER1, "clk_timer1", "xin24m", 0, ++ RK3568_CLKGATE_CON(32), 5, GFLAGS), ++ GATE(CLK_TIMER2, "clk_timer2", "xin24m", 0, ++ RK3568_CLKGATE_CON(32), 6, GFLAGS), ++ GATE(CLK_TIMER3, "clk_timer3", "xin24m", 0, ++ RK3568_CLKGATE_CON(32), 7, GFLAGS), ++ GATE(CLK_TIMER4, "clk_timer4", "xin24m", 0, ++ RK3568_CLKGATE_CON(32), 8, GFLAGS), ++ GATE(CLK_TIMER5, "clk_timer5", "xin24m", 0, ++ RK3568_CLKGATE_CON(32), 9, GFLAGS), ++ ++ /* PD_TOP */ ++ COMPOSITE_NODIV(ACLK_TOP_HIGH, "aclk_top_high", cpll500_gpll400_gpll300_xin24m_p, CLK_IS_CRITICAL, ++ RK3568_CLKSEL_CON(73), 0, 2, MFLAGS, ++ RK3568_CLKGATE_CON(33), 0, GFLAGS), ++ COMPOSITE_NODIV(ACLK_TOP_LOW, "aclk_top_low", gpll400_gpll300_gpll200_xin24m_p, CLK_IS_CRITICAL, ++ RK3568_CLKSEL_CON(73), 4, 2, MFLAGS, ++ RK3568_CLKGATE_CON(33), 1, GFLAGS), ++ COMPOSITE_NODIV(HCLK_TOP, "hclk_top", gpll150_gpll100_gpll75_xin24m_p, CLK_IS_CRITICAL, ++ RK3568_CLKSEL_CON(73), 8, 2, MFLAGS, ++ RK3568_CLKGATE_CON(33), 2, GFLAGS), ++ COMPOSITE_NODIV(PCLK_TOP, "pclk_top", gpll100_gpll75_cpll50_xin24m_p, CLK_IS_CRITICAL, ++ RK3568_CLKSEL_CON(73), 12, 2, MFLAGS, ++ RK3568_CLKGATE_CON(33), 3, GFLAGS), ++ GATE(PCLK_PCIE30PHY, "pclk_pcie30phy", "pclk_top", 0, ++ RK3568_CLKGATE_CON(33), 8, GFLAGS), ++ COMPOSITE_NODIV(CLK_OPTC_ARB, "clk_optc_arb", xin24m_cpll100_p, CLK_IS_CRITICAL, ++ RK3568_CLKSEL_CON(73), 15, 1, MFLAGS, ++ RK3568_CLKGATE_CON(33), 9, GFLAGS), ++ GATE(PCLK_MIPICSIPHY, "pclk_mipicsiphy", "pclk_top", 0, ++ RK3568_CLKGATE_CON(33), 13, GFLAGS), ++ GATE(PCLK_MIPIDSIPHY0, "pclk_mipidsiphy0", "pclk_top", 0, ++ RK3568_CLKGATE_CON(33), 14, GFLAGS), ++ GATE(PCLK_MIPIDSIPHY1, "pclk_mipidsiphy1", "pclk_top", 0, ++ RK3568_CLKGATE_CON(33), 15, GFLAGS), ++ GATE(PCLK_PIPEPHY0, "pclk_pipephy0", "pclk_top", 0, ++ RK3568_CLKGATE_CON(34), 4, GFLAGS), ++ GATE(PCLK_PIPEPHY1, "pclk_pipephy1", "pclk_top", 0, ++ RK3568_CLKGATE_CON(34), 5, GFLAGS), ++ GATE(PCLK_PIPEPHY2, "pclk_pipephy2", "pclk_top", 0, ++ RK3568_CLKGATE_CON(34), 6, GFLAGS), ++ GATE(PCLK_CPU_BOOST, "pclk_cpu_boost", "pclk_top", 0, ++ RK3568_CLKGATE_CON(34), 11, GFLAGS), ++ GATE(CLK_CPU_BOOST, "clk_cpu_boost", "xin24m", 0, ++ RK3568_CLKGATE_CON(34), 12, GFLAGS), ++ GATE(PCLK_OTPPHY, "pclk_otpphy", "pclk_top", 0, ++ RK3568_CLKGATE_CON(34), 13, GFLAGS), ++ GATE(PCLK_EDPPHY_GRF, "pclk_edpphy_grf", "pclk_top", 0, ++ RK3568_CLKGATE_CON(34), 14, GFLAGS), ++}; ++ ++static struct rockchip_clk_branch rk3568_clk_pmu_branches[] __initdata = { ++ /* PD_PMU */ ++ FACTOR(0, "ppll_ph0", "ppll", 0, 1, 2), ++ FACTOR(0, "ppll_ph180", "ppll", 0, 1, 2), ++ FACTOR(0, "hpll_ph0", "hpll", 0, 1, 2), ++ ++ MUX(CLK_PDPMU, "clk_pdpmu", clk_pdpmu_p, 0, ++ RK3568_PMU_CLKSEL_CON(2), 15, 1, MFLAGS), ++ COMPOSITE_NOMUX(PCLK_PDPMU, "pclk_pdpmu", "clk_pdpmu", CLK_IS_CRITICAL, ++ RK3568_PMU_CLKSEL_CON(2), 0, 5, DFLAGS, ++ RK3568_PMU_CLKGATE_CON(0), 2, GFLAGS), ++ GATE(PCLK_PMU, "pclk_pmu", "pclk_pdpmu", CLK_IS_CRITICAL, ++ RK3568_PMU_CLKGATE_CON(0), 6, GFLAGS), ++ GATE(CLK_PMU, "clk_pmu", "xin24m", CLK_IS_CRITICAL, ++ RK3568_PMU_CLKGATE_CON(0), 7, GFLAGS), ++ GATE(PCLK_I2C0, "pclk_i2c0", "pclk_pdpmu", 0, ++ RK3568_PMU_CLKGATE_CON(1), 0, GFLAGS), ++ COMPOSITE_NOMUX(CLK_I2C0, "clk_i2c0", "clk_pdpmu", 0, ++ RK3568_PMU_CLKSEL_CON(3), 0, 7, DFLAGS, ++ RK3568_PMU_CLKGATE_CON(1), 1, GFLAGS), ++ GATE(PCLK_UART0, "pclk_uart0", "pclk_pdpmu", 0, ++ RK3568_PMU_CLKGATE_CON(1), 2, GFLAGS), ++ ++ COMPOSITE_FRACMUX(CLK_RTC32K_FRAC, "clk_rtc32k_frac", "xin24m", CLK_IGNORE_UNUSED, ++ RK3568_PMU_CLKSEL_CON(1), 0, ++ RK3568_PMU_CLKGATE_CON(0), 1, GFLAGS, ++ &rk3568_rtc32k_pmu_fracmux, 0), ++ ++ COMPOSITE_NOMUX(XIN_OSC0_DIV, "xin_osc0_div", "xin24m", CLK_IGNORE_UNUSED, ++ RK3568_PMU_CLKSEL_CON(0), 0, 5, DFLAGS, ++ RK3568_PMU_CLKGATE_CON(0), 0, GFLAGS), ++ ++ COMPOSITE(CLK_UART0_DIV, "sclk_uart0_div", ppll_usb480m_cpll_gpll_p, 0, ++ RK3568_PMU_CLKSEL_CON(4), 8, 2, MFLAGS, 0, 7, DFLAGS, ++ RK3568_PMU_CLKGATE_CON(1), 3, GFLAGS), ++ COMPOSITE_FRACMUX(CLK_UART0_FRAC, "sclk_uart0_frac", "sclk_uart0_div", CLK_SET_RATE_PARENT, ++ RK3568_PMU_CLKSEL_CON(5), CLK_FRAC_DIVIDER_NO_LIMIT, ++ RK3568_PMU_CLKGATE_CON(1), 4, GFLAGS, ++ &rk3568_uart0_fracmux, RK3568_UART_FRAC_MAX_PRATE), ++ GATE(SCLK_UART0, "sclk_uart0", "sclk_uart0_mux", 0, ++ RK3568_PMU_CLKGATE_CON(1), 5, GFLAGS), ++ ++ GATE(PCLK_GPIO0, "pclk_gpio0", "pclk_pdpmu", 0, ++ RK3568_PMU_CLKGATE_CON(1), 9, GFLAGS), ++ COMPOSITE_NODIV(DBCLK_GPIO0, "dbclk_gpio0", xin24m_32k_p, 0, ++ RK3568_PMU_CLKSEL_CON(6), 15, 1, MFLAGS, ++ RK3568_PMU_CLKGATE_CON(1), 10, GFLAGS), ++ GATE(PCLK_PWM0, "pclk_pwm0", "pclk_pdpmu", 0, ++ RK3568_PMU_CLKGATE_CON(1), 6, GFLAGS), ++ COMPOSITE(CLK_PWM0, "clk_pwm0", clk_pwm0_p, 0, ++ RK3568_PMU_CLKSEL_CON(6), 7, 1, MFLAGS, 0, 7, DFLAGS, ++ RK3568_PMU_CLKGATE_CON(1), 7, GFLAGS), ++ GATE(CLK_CAPTURE_PWM0_NDFT, "clk_capture_pwm0_ndft", "xin24m", 0, ++ RK3568_PMU_CLKGATE_CON(1), 8, GFLAGS), ++ GATE(PCLK_PMUPVTM, "pclk_pmupvtm", "pclk_pdpmu", 0, ++ RK3568_PMU_CLKGATE_CON(1), 11, GFLAGS), ++ GATE(CLK_PMUPVTM, "clk_pmupvtm", "xin24m", 0, ++ RK3568_PMU_CLKGATE_CON(1), 12, GFLAGS), ++ GATE(CLK_CORE_PMUPVTM, "clk_core_pmupvtm", "xin24m", 0, ++ RK3568_PMU_CLKGATE_CON(1), 13, GFLAGS), ++ COMPOSITE_NOMUX(CLK_REF24M, "clk_ref24m", "clk_pdpmu", 0, ++ RK3568_PMU_CLKSEL_CON(7), 0, 6, DFLAGS, ++ RK3568_PMU_CLKGATE_CON(2), 0, GFLAGS), ++ GATE(XIN_OSC0_USBPHY0_G, "xin_osc0_usbphy0_g", "xin24m", 0, ++ RK3568_PMU_CLKGATE_CON(2), 1, GFLAGS), ++ MUX(CLK_USBPHY0_REF, "clk_usbphy0_ref", clk_usbphy0_ref_p, 0, ++ RK3568_PMU_CLKSEL_CON(8), 0, 1, MFLAGS), ++ GATE(XIN_OSC0_USBPHY1_G, "xin_osc0_usbphy1_g", "xin24m", 0, ++ RK3568_PMU_CLKGATE_CON(2), 2, GFLAGS), ++ MUX(CLK_USBPHY1_REF, "clk_usbphy1_ref", clk_usbphy1_ref_p, 0, ++ RK3568_PMU_CLKSEL_CON(8), 1, 1, MFLAGS), ++ GATE(XIN_OSC0_MIPIDSIPHY0_G, "xin_osc0_mipidsiphy0_g", "xin24m", 0, ++ RK3568_PMU_CLKGATE_CON(2), 3, GFLAGS), ++ MUX(CLK_MIPIDSIPHY0_REF, "clk_mipidsiphy0_ref", clk_mipidsiphy0_ref_p, 0, ++ RK3568_PMU_CLKSEL_CON(8), 2, 1, MFLAGS), ++ GATE(XIN_OSC0_MIPIDSIPHY1_G, "xin_osc0_mipidsiphy1_g", "xin24m", 0, ++ RK3568_PMU_CLKGATE_CON(2), 4, GFLAGS), ++ MUX(CLK_MIPIDSIPHY1_REF, "clk_mipidsiphy1_ref", clk_mipidsiphy1_ref_p, 0, ++ RK3568_PMU_CLKSEL_CON(8), 3, 1, MFLAGS), ++ COMPOSITE_NOMUX(CLK_WIFI_DIV, "clk_wifi_div", "clk_pdpmu", 0, ++ RK3568_PMU_CLKSEL_CON(8), 8, 6, DFLAGS, ++ RK3568_PMU_CLKGATE_CON(2), 5, GFLAGS), ++ GATE(CLK_WIFI_OSC0, "clk_wifi_osc0", "xin24m", 0, ++ RK3568_PMU_CLKGATE_CON(2), 6, GFLAGS), ++ MUX(CLK_WIFI, "clk_wifi", clk_wifi_p, CLK_SET_RATE_PARENT, ++ RK3568_PMU_CLKSEL_CON(8), 15, 1, MFLAGS), ++ COMPOSITE_NOMUX(CLK_PCIEPHY0_DIV, "clk_pciephy0_div", "ppll_ph0", 0, ++ RK3568_PMU_CLKSEL_CON(9), 0, 3, DFLAGS, ++ RK3568_PMU_CLKGATE_CON(2), 7, GFLAGS), ++ GATE(CLK_PCIEPHY0_OSC0, "clk_pciephy0_osc0", "xin24m", 0, ++ RK3568_PMU_CLKGATE_CON(2), 8, GFLAGS), ++ MUX(CLK_PCIEPHY0_REF, "clk_pciephy0_ref", clk_pciephy0_ref_p, CLK_SET_RATE_PARENT, ++ RK3568_PMU_CLKSEL_CON(9), 3, 1, MFLAGS), ++ COMPOSITE_NOMUX(CLK_PCIEPHY1_DIV, "clk_pciephy1_div", "ppll_ph0", 0, ++ RK3568_PMU_CLKSEL_CON(9), 4, 3, DFLAGS, ++ RK3568_PMU_CLKGATE_CON(2), 9, GFLAGS), ++ GATE(CLK_PCIEPHY1_OSC0, "clk_pciephy1_osc0", "xin24m", 0, ++ RK3568_PMU_CLKGATE_CON(2), 10, GFLAGS), ++ MUX(CLK_PCIEPHY1_REF, "clk_pciephy1_ref", clk_pciephy1_ref_p, CLK_SET_RATE_PARENT, ++ RK3568_PMU_CLKSEL_CON(9), 7, 1, MFLAGS), ++ COMPOSITE_NOMUX(CLK_PCIEPHY2_DIV, "clk_pciephy2_div", "ppll_ph0", 0, ++ RK3568_PMU_CLKSEL_CON(9), 8, 3, DFLAGS, ++ RK3568_PMU_CLKGATE_CON(2), 11, GFLAGS), ++ GATE(CLK_PCIEPHY2_OSC0, "clk_pciephy2_osc0", "xin24m", 0, ++ RK3568_PMU_CLKGATE_CON(2), 12, GFLAGS), ++ MUX(CLK_PCIEPHY2_REF, "clk_pciephy2_ref", clk_pciephy2_ref_p, CLK_SET_RATE_PARENT, ++ RK3568_PMU_CLKSEL_CON(9), 11, 1, MFLAGS), ++ GATE(CLK_PCIE30PHY_REF_M, "clk_pcie30phy_ref_m", "ppll_ph0", 0, ++ RK3568_PMU_CLKGATE_CON(2), 13, GFLAGS), ++ GATE(CLK_PCIE30PHY_REF_N, "clk_pcie30phy_ref_n", "ppll_ph180", 0, ++ RK3568_PMU_CLKGATE_CON(2), 14, GFLAGS), ++ GATE(XIN_OSC0_EDPPHY_G, "xin_osc0_edpphy_g", "xin24m", 0, ++ RK3568_PMU_CLKGATE_CON(2), 15, GFLAGS), ++ MUX(CLK_HDMI_REF, "clk_hdmi_ref", clk_hdmi_ref_p, 0, ++ RK3568_PMU_CLKSEL_CON(8), 7, 1, MFLAGS), ++ ++ MUXPMUGRF(SCLK_32K_IOE, "clk_32k_ioe", clk_32k_ioe_p, 0, ++ RK3568_PMU_GRF_SOC_CON0, 0, 1, MFLAGS) ++}; ++ ++static void __iomem *rk3568_cru_base; ++static void __iomem *rk3568_pmucru_base; ++ ++static void rk3568_dump_cru(void) ++{ ++ if (rk3568_pmucru_base) { ++ pr_warn("PMU CRU:\n"); ++ print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET, ++ 32, 4, rk3568_pmucru_base, ++ 0x248, false); ++ } ++ if (rk3568_cru_base) { ++ pr_warn("CRU:\n"); ++ print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET, ++ 32, 4, rk3568_cru_base, ++ 0x588, false); ++ } ++} ++ ++static void __init rk3568_pmu_clk_init(struct device_node *np) ++{ ++ struct rockchip_clk_provider *ctx; ++ void __iomem *reg_base; ++ ++ reg_base = of_iomap(np, 0); ++ if (!reg_base) { ++ pr_err("%s: could not map cru pmu region\n", __func__); ++ return; ++ } ++ ++ rk3568_pmucru_base = reg_base; ++ ++ ctx = rockchip_clk_init(np, reg_base, CLKPMU_NR_CLKS); ++ if (IS_ERR(ctx)) { ++ pr_err("%s: rockchip pmu clk init failed\n", __func__); ++ return; ++ } ++ ++ rockchip_clk_register_plls(ctx, rk3568_pmu_pll_clks, ++ ARRAY_SIZE(rk3568_pmu_pll_clks), ++ RK3568_GRF_SOC_STATUS0); ++ ++ rockchip_clk_register_branches(ctx, rk3568_clk_pmu_branches, ++ ARRAY_SIZE(rk3568_clk_pmu_branches)); ++ ++ rockchip_register_softrst(np, 1, reg_base + RK3568_PMU_SOFTRST_CON(0), ++ ROCKCHIP_SOFTRST_HIWORD_MASK); ++ ++ rockchip_clk_of_add_provider(np, ctx); ++} ++ ++CLK_OF_DECLARE(rk3568_cru_pmu, "rockchip,rk3568-pmucru", rk3568_pmu_clk_init); ++ ++static void __init rk3568_clk_init(struct device_node *np) ++{ ++ struct rockchip_clk_provider *ctx; ++ void __iomem *reg_base; ++ struct clk **clks; ++ ++ reg_base = of_iomap(np, 0); ++ if (!reg_base) { ++ pr_err("%s: could not map cru region\n", __func__); ++ return; ++ } ++ ++ rk3568_cru_base = reg_base; ++ ++ ctx = rockchip_clk_init(np, reg_base, CLK_NR_CLKS); ++ if (IS_ERR(ctx)) { ++ pr_err("%s: rockchip clk init failed\n", __func__); ++ iounmap(reg_base); ++ return; ++ } ++ clks = ctx->clk_data.clks; ++ ++ rockchip_clk_register_plls(ctx, rk3568_pll_clks, ++ ARRAY_SIZE(rk3568_pll_clks), ++ RK3568_GRF_SOC_STATUS0); ++ ++ rockchip_clk_register_armclk(ctx, ARMCLK, "armclk", ++ 2, clks[PLL_APLL], clks[PLL_GPLL], ++ &rk3568_cpuclk_data, rk3568_cpuclk_rates, ++ ARRAY_SIZE(rk3568_cpuclk_rates)); ++ ++ rockchip_clk_register_branches(ctx, rk3568_clk_branches, ++ ARRAY_SIZE(rk3568_clk_branches)); ++ ++ rockchip_register_softrst(np, 30, reg_base + RK3568_SOFTRST_CON(0), ++ ROCKCHIP_SOFTRST_HIWORD_MASK); ++ ++ rockchip_register_restart_notifier(ctx, RK3568_GLB_SRST_FST, NULL); ++ ++ rockchip_clk_of_add_provider(np, ctx); ++ ++ if (!rk_dump_cru) ++ rk_dump_cru = rk3568_dump_cru; ++} ++ ++CLK_OF_DECLARE(rk3568_cru, "rockchip,rk3568-cru", rk3568_clk_init); ++ ++struct clk_rk3568_inits { ++ void (*inits)(struct device_node *np); ++}; ++ ++static const struct clk_rk3568_inits clk_rk3568_pmucru_init = { ++ .inits = rk3568_pmu_clk_init, ++}; ++ ++static const struct clk_rk3568_inits clk_3568_cru_init = { ++ .inits = rk3568_clk_init, ++}; ++ ++static const struct of_device_id clk_rk3568_match_table[] = { ++ { ++ .compatible = "rockchip,rk3568-cru", ++ .data = &clk_3568_cru_init, ++ }, { ++ .compatible = "rockchip,rk3568-pmucru", ++ .data = &clk_rk3568_pmucru_init, ++ }, ++ { } ++}; ++MODULE_DEVICE_TABLE(of, clk_rk3568_match_table); ++ ++static int __init clk_rk3568_probe(struct platform_device *pdev) ++{ ++ struct device_node *np = pdev->dev.of_node; ++ const struct of_device_id *match; ++ const struct clk_rk3568_inits *init_data; ++ ++ match = of_match_device(clk_rk3568_match_table, &pdev->dev); ++ if (!match || !match->data) ++ return -EINVAL; ++ ++ init_data = match->data; ++ if (init_data->inits) ++ init_data->inits(np); ++ ++ return 0; ++} ++ ++static struct platform_driver clk_rk3568_driver = { ++ .driver = { ++ .name = "clk-rk3568", ++ .of_match_table = clk_rk3568_match_table, ++ .suppress_bind_attrs = true, ++ }, ++}; ++builtin_platform_driver_probe(clk_rk3568_driver, clk_rk3568_probe); ++ ++MODULE_DESCRIPTION("Rockchip RK3568 Clock Driver"); ++MODULE_LICENSE("GPL"); ++MODULE_ALIAS("platform:clk-rk3568"); +diff --git a/drivers/clk/rockchip/clk-rv1108.c b/drivers/clk/rockchip/clk-rv1108.c +index 5947d3192866..53eaabb193a4 100644 +--- a/drivers/clk/rockchip/clk-rv1108.c ++++ b/drivers/clk/rockchip/clk-rv1108.c +@@ -7,13 +7,17 @@ + + #include + #include ++#include + #include + #include ++#include + #include + #include + #include "clk.h" + + #define RV1108_GRF_SOC_STATUS0 0x480 ++#define RV1108_I2S_FRAC_MAX_RATE 600000000 ++#define RV1108_UART_FRAC_MAX_RATE 600000000 + + enum rv1108_plls { + apll, dpll, gpll, +@@ -106,9 +110,10 @@ static struct rockchip_cpuclk_rate_table rv1108_cpuclk_rates[] __initdata = { + }; + + static const struct rockchip_cpuclk_reg_data rv1108_cpuclk_data = { +- .core_reg = RV1108_CLKSEL_CON(0), +- .div_core_shift = 0, +- .div_core_mask = 0x1f, ++ .core_reg[0] = RV1108_CLKSEL_CON(0), ++ .div_core_shift[0] = 0, ++ .div_core_mask[0] = 0x1f, ++ .num_cores = 1, + .mux_core_alt = 1, + .mux_core_main = 0, + .mux_core_shift = 8, +@@ -117,7 +122,6 @@ static const struct rockchip_cpuclk_reg_data rv1108_cpuclk_data = { + + PNAME(mux_pll_p) = { "xin24m", "xin24m"}; + PNAME(mux_ddrphy_p) = { "dpll_ddr", "gpll_ddr", "apll_ddr" }; +-PNAME(mux_armclk_p) = { "apll_core", "gpll_core", "dpll_core" }; + PNAME(mux_usb480m_pre_p) = { "usbphy", "xin24m" }; + PNAME(mux_hdmiphy_phy_p) = { "hdmiphy", "xin24m" }; + PNAME(mux_dclk_hdmiphy_pre_p) = { "dclk_hdmiphy_src_gpll", "dclk_hdmiphy_src_dpll" }; +@@ -209,7 +213,7 @@ static struct rockchip_clk_branch rv1108_clk_branches[] __initdata = { + COMPOSITE_NOMUX(ACLK_ENMCORE, "aclkenm_core", "armclk", CLK_IGNORE_UNUSED, + RV1108_CLKSEL_CON(1), 0, 3, DFLAGS | CLK_DIVIDER_READ_ONLY, + RV1108_CLKGATE_CON(0), 4, GFLAGS), +- GATE(ACLK_CORE, "aclk_core", "aclkenm_core", CLK_IGNORE_UNUSED, ++ GATE(ACLK_CORE, "aclk_core", "aclkenm_core", CLK_IS_CRITICAL, + RV1108_CLKGATE_CON(11), 0, GFLAGS), + GATE(0, "pclk_dbg", "pclken_dbg", CLK_IGNORE_UNUSED, + RV1108_CLKGATE_CON(11), 1, GFLAGS), +@@ -264,10 +268,10 @@ static struct rockchip_clk_branch rv1108_clk_branches[] __initdata = { + RV1108_CLKGATE_CON(19), 6, GFLAGS), + + /* PD_PMU_wrapper */ +- COMPOSITE_NOMUX(0, "pmu_24m_ena", "gpll", CLK_IGNORE_UNUSED, ++ COMPOSITE_NOMUX(0, "pmu_24m_ena", "gpll", CLK_IS_CRITICAL, + RV1108_CLKSEL_CON(38), 0, 5, DFLAGS, + RV1108_CLKGATE_CON(8), 12, GFLAGS), +- GATE(0, "pclk_pmu", "pmu_24m_ena", CLK_IGNORE_UNUSED, ++ GATE(0, "pclk_pmu", "pmu_24m_ena", CLK_IS_CRITICAL, + RV1108_CLKGATE_CON(10), 0, GFLAGS), + GATE(0, "pclk_intmem1", "pmu_24m_ena", CLK_IGNORE_UNUSED, + RV1108_CLKGATE_CON(10), 1, GFLAGS), +@@ -305,7 +309,7 @@ static struct rockchip_clk_branch rv1108_clk_branches[] __initdata = { + RV1108_CLKSEL_CON(41), 0, 5, DFLAGS, + RV1108_CLKGATE_CON(9), 12, GFLAGS), + +- GATE(0, "pclk_acodecphy", "pclk_top_pre", CLK_IGNORE_UNUSED, ++ GATE(0, "pclk_acodecphy", "pclk_top_pre", CLK_IS_CRITICAL, + RV1108_CLKGATE_CON(14), 6, GFLAGS), + GATE(0, "pclk_usbgrf", "pclk_top_pre", CLK_IGNORE_UNUSED, + RV1108_CLKGATE_CON(14), 14, GFLAGS), +@@ -503,7 +507,7 @@ static struct rockchip_clk_branch rv1108_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "i2s0_frac", "i2s0_src", CLK_SET_RATE_PARENT, + RV1108_CLKSEL_CON(8), 0, + RV1108_CLKGATE_CON(2), 1, GFLAGS, +- &rv1108_i2s0_fracmux), ++ &rv1108_i2s0_fracmux, RV1108_I2S_FRAC_MAX_RATE), + GATE(SCLK_I2S0, "sclk_i2s0", "i2s0_pre", CLK_SET_RATE_PARENT, + RV1108_CLKGATE_CON(2), 2, GFLAGS), + COMPOSITE_NODIV(0, "i2s_out", mux_i2s_out_p, 0, +@@ -516,7 +520,7 @@ static struct rockchip_clk_branch rv1108_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "i2s1_frac", "i2s1_src", CLK_SET_RATE_PARENT, + RK2928_CLKSEL_CON(9), 0, + RK2928_CLKGATE_CON(2), 5, GFLAGS, +- &rv1108_i2s1_fracmux), ++ &rv1108_i2s1_fracmux, RV1108_I2S_FRAC_MAX_RATE), + GATE(SCLK_I2S1, "sclk_i2s1", "i2s1_pre", CLK_SET_RATE_PARENT, + RV1108_CLKGATE_CON(2), 6, GFLAGS), + +@@ -526,28 +530,28 @@ static struct rockchip_clk_branch rv1108_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "i2s2_frac", "i2s2_src", CLK_SET_RATE_PARENT, + RV1108_CLKSEL_CON(10), 0, + RV1108_CLKGATE_CON(2), 9, GFLAGS, +- &rv1108_i2s2_fracmux), ++ &rv1108_i2s2_fracmux, RV1108_I2S_FRAC_MAX_RATE), + GATE(SCLK_I2S2, "sclk_i2s2", "i2s2_pre", CLK_SET_RATE_PARENT, + RV1108_CLKGATE_CON(2), 10, GFLAGS), + + /* PD_BUS */ +- GATE(0, "aclk_bus_src_gpll", "gpll", CLK_IGNORE_UNUSED, ++ GATE(0, "aclk_bus_src_gpll", "gpll", CLK_IS_CRITICAL, + RV1108_CLKGATE_CON(1), 0, GFLAGS), +- GATE(0, "aclk_bus_src_apll", "apll", CLK_IGNORE_UNUSED, ++ GATE(0, "aclk_bus_src_apll", "apll", CLK_IS_CRITICAL, + RV1108_CLKGATE_CON(1), 1, GFLAGS), +- GATE(0, "aclk_bus_src_dpll", "dpll", CLK_IGNORE_UNUSED, ++ GATE(0, "aclk_bus_src_dpll", "dpll", CLK_IS_CRITICAL, + RV1108_CLKGATE_CON(1), 2, GFLAGS), +- COMPOSITE_NOGATE(ACLK_PRE, "aclk_bus_pre", mux_aclk_bus_src_p, 0, ++ COMPOSITE_NOGATE(ACLK_PRE, "aclk_bus_pre", mux_aclk_bus_src_p, CLK_IS_CRITICAL, + RV1108_CLKSEL_CON(2), 8, 2, MFLAGS, 0, 5, DFLAGS), +- COMPOSITE_NOMUX(HCLK_BUS, "hclk_bus_pre", "aclk_bus_pre", 0, ++ COMPOSITE_NOMUX(HCLK_BUS, "hclk_bus_pre", "aclk_bus_pre", CLK_IS_CRITICAL, + RV1108_CLKSEL_CON(3), 0, 5, DFLAGS, + RV1108_CLKGATE_CON(1), 4, GFLAGS), +- COMPOSITE_NOMUX(0, "pclk_bus_pre", "aclk_bus_pre", 0, ++ COMPOSITE_NOMUX(0, "pclk_bus_pre", "aclk_bus_pre", CLK_IS_CRITICAL, + RV1108_CLKSEL_CON(3), 8, 5, DFLAGS, + RV1108_CLKGATE_CON(1), 5, GFLAGS), +- GATE(PCLK_BUS, "pclk_bus", "pclk_bus_pre", 0, ++ GATE(PCLK_BUS, "pclk_bus", "pclk_bus_pre", CLK_IS_CRITICAL, + RV1108_CLKGATE_CON(1), 6, GFLAGS), +- GATE(0, "pclk_top_pre", "pclk_bus_pre", CLK_IGNORE_UNUSED, ++ GATE(0, "pclk_top_pre", "pclk_bus_pre", CLK_IS_CRITICAL, + RV1108_CLKGATE_CON(1), 7, GFLAGS), + GATE(0, "pclk_ddr_pre", "pclk_bus_pre", CLK_IGNORE_UNUSED, + RV1108_CLKGATE_CON(1), 8, GFLAGS), +@@ -592,15 +596,15 @@ static struct rockchip_clk_branch rv1108_clk_branches[] __initdata = { + COMPOSITE_FRACMUX(0, "uart0_frac", "uart0_src", CLK_SET_RATE_PARENT, + RV1108_CLKSEL_CON(16), 0, + RV1108_CLKGATE_CON(3), 2, GFLAGS, +- &rv1108_uart0_fracmux), ++ &rv1108_uart0_fracmux, RV1108_UART_FRAC_MAX_RATE), + COMPOSITE_FRACMUX(0, "uart1_frac", "uart1_src", CLK_SET_RATE_PARENT, + RV1108_CLKSEL_CON(17), 0, + RV1108_CLKGATE_CON(3), 4, GFLAGS, +- &rv1108_uart1_fracmux), ++ &rv1108_uart1_fracmux, RV1108_UART_FRAC_MAX_RATE), + COMPOSITE_FRACMUX(0, "uart2_frac", "uart2_src", CLK_SET_RATE_PARENT, + RV1108_CLKSEL_CON(18), 0, + RV1108_CLKGATE_CON(3), 6, GFLAGS, +- &rv1108_uart2_fracmux), ++ &rv1108_uart2_fracmux, RV1108_UART_FRAC_MAX_RATE), + GATE(PCLK_UART0, "pclk_uart0", "pclk_bus_pre", 0, + RV1108_CLKGATE_CON(13), 10, GFLAGS), + GATE(PCLK_UART1, "pclk_uart1", "pclk_bus_pre", 0, +@@ -668,7 +672,7 @@ static struct rockchip_clk_branch rv1108_clk_branches[] __initdata = { + RV1108_CLKGATE_CON(0), 9, GFLAGS), + GATE(0, "gpll_ddr", "gpll", CLK_IGNORE_UNUSED, + RV1108_CLKGATE_CON(0), 10, GFLAGS), +- COMPOSITE_NOGATE(0, "clk_ddrphy_src", mux_ddrphy_p, CLK_IGNORE_UNUSED, ++ COMPOSITE_NOGATE(0, "clk_ddrphy_src", mux_ddrphy_p, CLK_IS_CRITICAL, + RV1108_CLKSEL_CON(4), 8, 2, MFLAGS, 0, 3, + DFLAGS | CLK_DIVIDER_POWER_OF_TWO), + FACTOR(0, "clk_ddr", "clk_ddrphy_src", 0, 1, 2), +@@ -676,9 +680,9 @@ static struct rockchip_clk_branch rv1108_clk_branches[] __initdata = { + RV1108_CLKGATE_CON(10), 9, GFLAGS), + GATE(0, "pclk_ddrupctl", "pclk_ddr_pre", CLK_IGNORE_UNUSED, + RV1108_CLKGATE_CON(12), 4, GFLAGS), +- GATE(0, "nclk_ddrupctl", "clk_ddr", CLK_IGNORE_UNUSED, ++ GATE(0, "nclk_ddrupctl", "clk_ddr", CLK_IS_CRITICAL, + RV1108_CLKGATE_CON(12), 5, GFLAGS), +- GATE(0, "pclk_ddrmon", "pclk_ddr_pre", CLK_IGNORE_UNUSED, ++ GATE(0, "pclk_ddrmon", "pclk_ddr_pre", CLK_IS_CRITICAL, + RV1108_CLKGATE_CON(12), 6, GFLAGS), + GATE(0, "timer_clk", "xin24m", CLK_IGNORE_UNUSED, + RV1108_CLKGATE_CON(0), 11, GFLAGS), +@@ -692,22 +696,22 @@ static struct rockchip_clk_branch rv1108_clk_branches[] __initdata = { + */ + + /* PD_PERI */ +- COMPOSITE_NOMUX(0, "pclk_periph_pre", "gpll", 0, ++ COMPOSITE_NOMUX(0, "pclk_periph_pre", "gpll", CLK_IS_CRITICAL, + RV1108_CLKSEL_CON(23), 10, 5, DFLAGS, + RV1108_CLKGATE_CON(4), 5, GFLAGS), +- GATE(PCLK_PERI, "pclk_periph", "pclk_periph_pre", CLK_IGNORE_UNUSED, ++ GATE(PCLK_PERI, "pclk_periph", "pclk_periph_pre", CLK_IS_CRITICAL, + RV1108_CLKGATE_CON(15), 13, GFLAGS), +- COMPOSITE_NOMUX(0, "hclk_periph_pre", "gpll", 0, ++ COMPOSITE_NOMUX(0, "hclk_periph_pre", "gpll", CLK_IS_CRITICAL, + RV1108_CLKSEL_CON(23), 5, 5, DFLAGS, + RV1108_CLKGATE_CON(4), 4, GFLAGS), +- GATE(HCLK_PERI, "hclk_periph", "hclk_periph_pre", CLK_IGNORE_UNUSED, ++ GATE(HCLK_PERI, "hclk_periph", "hclk_periph_pre", CLK_IS_CRITICAL, + RV1108_CLKGATE_CON(15), 12, GFLAGS), + +- GATE(0, "aclk_peri_src_dpll", "dpll", CLK_IGNORE_UNUSED, ++ GATE(0, "aclk_peri_src_dpll", "dpll", CLK_IS_CRITICAL, + RV1108_CLKGATE_CON(4), 1, GFLAGS), +- GATE(0, "aclk_peri_src_gpll", "gpll", CLK_IGNORE_UNUSED, ++ GATE(0, "aclk_peri_src_gpll", "gpll", CLK_IS_CRITICAL, + RV1108_CLKGATE_CON(4), 2, GFLAGS), +- COMPOSITE(ACLK_PERI, "aclk_periph", mux_aclk_peri_src_p, 0, ++ COMPOSITE(ACLK_PERI, "aclk_periph", mux_aclk_peri_src_p, CLK_IS_CRITICAL, + RV1108_CLKSEL_CON(23), 15, 1, MFLAGS, 0, 5, DFLAGS, + RV1108_CLKGATE_CON(15), 11, GFLAGS), + +@@ -767,24 +771,23 @@ static struct rockchip_clk_branch rv1108_clk_branches[] __initdata = { + MMC(SCLK_EMMC_SAMPLE, "emmc_sample", "sclk_emmc", RV1108_EMMC_CON1, 1), + }; + +-static const char *const rv1108_critical_clocks[] __initconst = { +- "aclk_core", +- "aclk_bus", +- "hclk_bus", +- "pclk_bus", +- "aclk_periph", +- "hclk_periph", +- "pclk_periph", +- "nclk_ddrupctl", +- "pclk_ddrmon", +- "pclk_acodecphy", +- "pclk_pmu", +-}; ++static void __iomem *rv1108_cru_base; ++ ++static void rv1108_dump_cru(void) ++{ ++ if (rv1108_cru_base) { ++ pr_warn("CRU:\n"); ++ print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET, ++ 32, 4, rv1108_cru_base, ++ 0x1f8, false); ++ } ++} + + static void __init rv1108_clk_init(struct device_node *np) + { + struct rockchip_clk_provider *ctx; + void __iomem *reg_base; ++ struct clk **clks; + + reg_base = of_iomap(np, 0); + if (!reg_base) { +@@ -798,17 +801,16 @@ static void __init rv1108_clk_init(struct device_node *np) + iounmap(reg_base); + return; + } ++ clks = ctx->clk_data.clks; + + rockchip_clk_register_plls(ctx, rv1108_pll_clks, + ARRAY_SIZE(rv1108_pll_clks), + RV1108_GRF_SOC_STATUS0); + rockchip_clk_register_branches(ctx, rv1108_clk_branches, + ARRAY_SIZE(rv1108_clk_branches)); +- rockchip_clk_protect_critical(rv1108_critical_clocks, +- ARRAY_SIZE(rv1108_critical_clocks)); + + rockchip_clk_register_armclk(ctx, ARMCLK, "armclk", +- mux_armclk_p, ARRAY_SIZE(mux_armclk_p), ++ 3, clks[PLL_APLL], clks[PLL_GPLL], + &rv1108_cpuclk_data, rv1108_cpuclk_rates, + ARRAY_SIZE(rv1108_cpuclk_rates)); + +@@ -818,5 +820,38 @@ static void __init rv1108_clk_init(struct device_node *np) + rockchip_register_restart_notifier(ctx, RV1108_GLB_SRST_FST, NULL); + + rockchip_clk_of_add_provider(np, ctx); ++ ++ if (!rk_dump_cru) { ++ rv1108_cru_base = reg_base; ++ rk_dump_cru = rv1108_dump_cru; ++ } + } + CLK_OF_DECLARE(rv1108_cru, "rockchip,rv1108-cru", rv1108_clk_init); ++ ++static int __init clk_rv1108_probe(struct platform_device *pdev) ++{ ++ struct device_node *np = pdev->dev.of_node; ++ ++ rv1108_clk_init(np); ++ ++ return 0; ++} ++ ++static const struct of_device_id clk_rv1108_match_table[] = { ++ { ++ .compatible = "rockchip,rv1108-cru", ++ }, ++ { } ++}; ++MODULE_DEVICE_TABLE(of, clk_rv1108_match_table); ++ ++static struct platform_driver clk_rv1108_driver = { ++ .driver = { ++ .name = "clk-rv1108", ++ .of_match_table = clk_rv1108_match_table, ++ }, ++}; ++builtin_platform_driver_probe(clk_rv1108_driver, clk_rv1108_probe); ++ ++MODULE_DESCRIPTION("Rockchip RV1108 Clock Driver"); ++MODULE_LICENSE("GPL"); +diff --git a/drivers/clk/rockchip/clk-rv1126.c b/drivers/clk/rockchip/clk-rv1126.c +new file mode 100755 +index 000000000000..09a376c51c0b +--- /dev/null ++++ b/drivers/clk/rockchip/clk-rv1126.c +@@ -0,0 +1,1586 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * Copyright (c) 2019 Rockchip Electronics Co. Ltd. ++ * Author: Finley Xiao ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include "clk.h" ++ ++#define RV1126_GMAC_CON 0x460 ++#define RV1126_GRF_IOFUNC_CON1 0x10264 ++#define RV1126_GRF_SOC_STATUS0 0x10 ++#define RV1126_PMUGRF_SOC_CON0 0x100 ++ ++#define RV1126_FRAC_MAX_PRATE 1200000000 ++#define RV1126_CSIOUT_FRAC_MAX_PRATE 300000000 ++ ++enum rv1126_pmu_plls { ++ gpll, ++}; ++ ++enum rv1126_plls { ++ apll, dpll, cpll, hpll, ++}; ++ ++static struct rockchip_pll_rate_table rv1126_pll_rates[] = { ++ /* _mhz, _refdiv, _fbdiv, _postdiv1, _postdiv2, _dsmpd, _frac */ ++ RK3036_PLL_RATE(1608000000, 1, 67, 1, 1, 1, 0), ++ RK3036_PLL_RATE(1600000000, 3, 200, 1, 1, 1, 0), ++ RK3036_PLL_RATE(1584000000, 1, 132, 2, 1, 1, 0), ++ RK3036_PLL_RATE(1560000000, 1, 130, 2, 1, 1, 0), ++ RK3036_PLL_RATE(1536000000, 1, 128, 2, 1, 1, 0), ++ RK3036_PLL_RATE(1512000000, 1, 126, 2, 1, 1, 0), ++ RK3036_PLL_RATE(1488000000, 1, 124, 2, 1, 1, 0), ++ RK3036_PLL_RATE(1464000000, 1, 122, 2, 1, 1, 0), ++ RK3036_PLL_RATE(1440000000, 1, 120, 2, 1, 1, 0), ++ RK3036_PLL_RATE(1416000000, 1, 118, 2, 1, 1, 0), ++ RK3036_PLL_RATE(1400000000, 3, 350, 2, 1, 1, 0), ++ RK3036_PLL_RATE(1392000000, 1, 116, 2, 1, 1, 0), ++ RK3036_PLL_RATE(1368000000, 1, 114, 2, 1, 1, 0), ++ RK3036_PLL_RATE(1344000000, 1, 112, 2, 1, 1, 0), ++ RK3036_PLL_RATE(1320000000, 1, 110, 2, 1, 1, 0), ++ RK3036_PLL_RATE(1296000000, 1, 108, 2, 1, 1, 0), ++ RK3036_PLL_RATE(1272000000, 1, 106, 2, 1, 1, 0), ++ RK3036_PLL_RATE(1248000000, 1, 104, 2, 1, 1, 0), ++ RK3036_PLL_RATE(1200000000, 1, 100, 2, 1, 1, 0), ++ RK3036_PLL_RATE(1188000000, 1, 99, 2, 1, 1, 0), ++ RK3036_PLL_RATE(1104000000, 1, 92, 2, 1, 1, 0), ++ RK3036_PLL_RATE(1100000000, 3, 275, 2, 1, 1, 0), ++ RK3036_PLL_RATE(1008000000, 1, 84, 2, 1, 1, 0), ++ RK3036_PLL_RATE(1000000000, 3, 250, 2, 1, 1, 0), ++ RK3036_PLL_RATE(984000000, 1, 82, 2, 1, 1, 0), ++ RK3036_PLL_RATE(960000000, 1, 80, 2, 1, 1, 0), ++ RK3036_PLL_RATE(936000000, 1, 78, 2, 1, 1, 0), ++ RK3036_PLL_RATE(912000000, 1, 76, 2, 1, 1, 0), ++ RK3036_PLL_RATE(900000000, 1, 75, 2, 1, 1, 0), ++ RK3036_PLL_RATE(888000000, 1, 74, 2, 1, 1, 0), ++ RK3036_PLL_RATE(864000000, 1, 72, 2, 1, 1, 0), ++ RK3036_PLL_RATE(840000000, 1, 70, 2, 1, 1, 0), ++ RK3036_PLL_RATE(816000000, 1, 68, 2, 1, 1, 0), ++ RK3036_PLL_RATE(800000000, 3, 200, 2, 1, 1, 0), ++ RK3036_PLL_RATE(700000000, 3, 350, 4, 1, 1, 0), ++ RK3036_PLL_RATE(696000000, 1, 116, 4, 1, 1, 0), ++ RK3036_PLL_RATE(624000000, 1, 104, 4, 1, 1, 0), ++#ifdef CONFIG_ROCKCHIP_LOW_PERFORMANCE ++ RK3036_PLL_RATE(600000000, 1, 50, 2, 1, 1, 0), ++#else ++ RK3036_PLL_RATE(600000000, 1, 100, 4, 1, 1, 0), ++#endif ++ RK3036_PLL_RATE(594000000, 1, 99, 4, 1, 1, 0), ++ RK3036_PLL_RATE(504000000, 1, 84, 4, 1, 1, 0), ++ RK3036_PLL_RATE(500000000, 1, 125, 6, 1, 1, 0), ++ RK3036_PLL_RATE(496742400, 1, 124, 6, 1, 0, 3113851), ++ RK3036_PLL_RATE(491520000, 1, 40, 2, 1, 0, 16106127), ++ RK3036_PLL_RATE(408000000, 1, 68, 2, 2, 1, 0), ++ RK3036_PLL_RATE(312000000, 1, 78, 6, 1, 1, 0), ++ RK3036_PLL_RATE(216000000, 1, 72, 4, 2, 1, 0), ++ RK3036_PLL_RATE(96000000, 1, 96, 6, 4, 1, 0), ++ { /* sentinel */ }, ++}; ++ ++#define RV1126_DIV_ACLK_CORE_MASK 0xf ++#define RV1126_DIV_ACLK_CORE_SHIFT 4 ++#define RV1126_DIV_PCLK_DBG_MASK 0x7 ++#define RV1126_DIV_PCLK_DBG_SHIFT 0 ++ ++#define RV1126_CLKSEL1(_aclk_core, _pclk_dbg) \ ++{ \ ++ .reg = RV1126_CLKSEL_CON(1), \ ++ .val = HIWORD_UPDATE(_aclk_core, RV1126_DIV_ACLK_CORE_MASK, \ ++ RV1126_DIV_ACLK_CORE_SHIFT) | \ ++ HIWORD_UPDATE(_pclk_dbg, RV1126_DIV_PCLK_DBG_MASK, \ ++ RV1126_DIV_PCLK_DBG_SHIFT), \ ++} ++ ++#define RV1126_CPUCLK_RATE(_prate, _aclk_core, _pclk_dbg) \ ++{ \ ++ .prate = _prate, \ ++ .divs = { \ ++ RV1126_CLKSEL1(_aclk_core, _pclk_dbg), \ ++ }, \ ++} ++ ++static struct rockchip_cpuclk_rate_table rv1126_cpuclk_rates[] __initdata = { ++ RV1126_CPUCLK_RATE(1608000000, 1, 7), ++ RV1126_CPUCLK_RATE(1584000000, 1, 7), ++ RV1126_CPUCLK_RATE(1560000000, 1, 7), ++ RV1126_CPUCLK_RATE(1536000000, 1, 7), ++ RV1126_CPUCLK_RATE(1512000000, 1, 7), ++ RV1126_CPUCLK_RATE(1488000000, 1, 5), ++ RV1126_CPUCLK_RATE(1464000000, 1, 5), ++ RV1126_CPUCLK_RATE(1440000000, 1, 5), ++ RV1126_CPUCLK_RATE(1416000000, 1, 5), ++ RV1126_CPUCLK_RATE(1392000000, 1, 5), ++ RV1126_CPUCLK_RATE(1368000000, 1, 5), ++ RV1126_CPUCLK_RATE(1344000000, 1, 5), ++ RV1126_CPUCLK_RATE(1320000000, 1, 5), ++ RV1126_CPUCLK_RATE(1296000000, 1, 5), ++ RV1126_CPUCLK_RATE(1272000000, 1, 5), ++ RV1126_CPUCLK_RATE(1248000000, 1, 5), ++ RV1126_CPUCLK_RATE(1224000000, 1, 5), ++ RV1126_CPUCLK_RATE(1200000000, 1, 5), ++ RV1126_CPUCLK_RATE(1104000000, 1, 5), ++ RV1126_CPUCLK_RATE(1008000000, 1, 5), ++ RV1126_CPUCLK_RATE(912000000, 1, 5), ++ RV1126_CPUCLK_RATE(816000000, 1, 3), ++ RV1126_CPUCLK_RATE(696000000, 1, 3), ++ RV1126_CPUCLK_RATE(600000000, 1, 3), ++ RV1126_CPUCLK_RATE(408000000, 1, 1), ++ RV1126_CPUCLK_RATE(312000000, 1, 1), ++ RV1126_CPUCLK_RATE(216000000, 1, 1), ++ RV1126_CPUCLK_RATE(96000000, 1, 1), ++}; ++ ++static const struct rockchip_cpuclk_reg_data rv1126_cpuclk_data = { ++ .core_reg[0] = RV1126_CLKSEL_CON(0), ++ .div_core_shift[0] = 0, ++ .div_core_mask[0] = 0x1f, ++ .num_cores = 1, ++ .mux_core_alt = 0, ++ .mux_core_main = 2, ++ .mux_core_shift = 6, ++ .mux_core_mask = 0x3, ++}; ++ ++PNAME(mux_pll_p) = { "xin24m" }; ++PNAME(mux_rtc32k_p) = { "clk_pmupvtm_divout", "xin32k", "clk_osc0_div32k" }; ++PNAME(mux_clk_32k_ioe_p) = { "xin32k", "clk_rtc32k" }; ++PNAME(mux_wifi_p) = { "clk_wifi_osc0", "clk_wifi_div" }; ++PNAME(mux_uart1_p) = { "sclk_uart1_div", "sclk_uart1_fracdiv", "xin24m" }; ++PNAME(mux_xin24m_gpll_p) = { "xin24m", "gpll" }; ++PNAME(mux_gpll_xin24m_p) = { "gpll", "xin24m" }; ++PNAME(mux_xin24m_32k_p) = { "xin24m", "clk_rtc32k" }; ++PNAME(mux_usbphy_otg_ref_p) = { "clk_ref12m", "xin_osc0_div2_usbphyref_otg" }; ++PNAME(mux_usbphy_host_ref_p) = { "clk_ref12m", "xin_osc0_div2_usbphyref_host" }; ++PNAME(mux_mipidsiphy_ref_p) = { "clk_ref24m", "xin_osc0_mipiphyref" }; ++PNAME(mux_usb480m_p) = { "xin24m", "usb480m_phy", "clk_rtc32k" }; ++PNAME(mux_hclk_pclk_pdbus_p) = { "gpll", "dummy_cpll" }; ++PNAME(mux_uart0_p) = { "sclk_uart0_div", "sclk_uart0_frac", "xin24m" }; ++PNAME(mux_uart2_p) = { "sclk_uart2_div", "sclk_uart2_frac", "xin24m" }; ++PNAME(mux_uart3_p) = { "sclk_uart3_div", "sclk_uart3_frac", "xin24m" }; ++PNAME(mux_uart4_p) = { "sclk_uart4_div", "sclk_uart4_frac", "xin24m" }; ++PNAME(mux_uart5_p) = { "sclk_uart5_div", "sclk_uart5_frac", "xin24m" }; ++PNAME(mux_i2s0_tx_p) = { "mclk_i2s0_tx_div", "mclk_i2s0_tx_fracdiv", "i2s0_mclkin", "xin12m" }; ++PNAME(mux_i2s0_rx_p) = { "mclk_i2s0_rx_div", "mclk_i2s0_rx_fracdiv", "i2s0_mclkin", "xin12m" }; ++PNAME(mux_i2s0_tx_out2io_p) = { "mclk_i2s0_tx", "xin12m" }; ++PNAME(mux_i2s0_rx_out2io_p) = { "mclk_i2s0_rx", "xin12m" }; ++PNAME(mux_i2s1_p) = { "mclk_i2s1_div", "mclk_i2s1_fracdiv", "i2s1_mclkin", "xin12m" }; ++PNAME(mux_i2s1_out2io_p) = { "mclk_i2s1", "xin12m" }; ++PNAME(mux_i2s2_p) = { "mclk_i2s2_div", "mclk_i2s2_fracdiv", "i2s2_mclkin", "xin12m" }; ++PNAME(mux_i2s2_out2io_p) = { "mclk_i2s2", "xin12m" }; ++PNAME(mux_audpwm_p) = { "sclk_audpwm_div", "sclk_audpwm_fracdiv", "xin24m" }; ++PNAME(mux_dclk_vop_p) = { "dclk_vop_div", "dclk_vop_fracdiv", "xin24m" }; ++PNAME(mux_aclk_pdvi_p) = { "aclk_pdvi_div", "aclk_pdvi_np5" }; ++PNAME(mux_clk_isp_p) = { "clk_isp_div", "clk_isp_np5" }; ++PNAME(mux_gpll_usb480m_p) = { "gpll", "usb480m" }; ++PNAME(mux_cif_out2io_p) = { "xin24m", "clk_cif_out2io_div", "clk_cif_out2io_fracdiv" }; ++PNAME(mux_mipicsi_out2io_p) = { "xin24m", "clk_mipicsi_out2io_div", "clk_mipicsi_out2io_fracdiv" }; ++PNAME(mux_aclk_pdispp_p) = { "aclk_pdispp_div", "aclk_pdispp_np5" }; ++PNAME(mux_clk_ispp_p) = { "clk_ispp_div", "clk_ispp_np5" }; ++PNAME(mux_usb480m_gpll_p) = { "usb480m", "gpll" }; ++PNAME(clk_gmac_src_m0_p) = { "clk_gmac_div", "clk_gmac_rgmii_m0" }; ++PNAME(clk_gmac_src_m1_p) = { "clk_gmac_div", "clk_gmac_rgmii_m1" }; ++PNAME(mux_clk_gmac_src_p) = { "clk_gmac_src_m0", "clk_gmac_src_m1" }; ++PNAME(mux_rgmii_clk_p) = { "clk_gmac_tx_div50", "clk_gmac_tx_div5", "clk_gmac_tx_src", "clk_gmac_tx_src"}; ++PNAME(mux_rmii_clk_p) = { "clk_gmac_rx_div20", "clk_gmac_rx_div2" }; ++PNAME(mux_gmac_tx_rx_p) = { "rgmii_mode_clk", "rmii_mode_clk" }; ++PNAME(mux_dpll_gpll_p) = { "dpll", "gpll" }; ++PNAME(mux_aclk_pdnpu_p) = { "aclk_pdnpu_div", "aclk_pdnpu_np5" }; ++PNAME(mux_clk_npu_p) = { "clk_npu_div", "clk_npu_np5" }; ++ ++ ++#ifndef CONFIG_ROCKCHIP_LOW_PERFORMANCE ++PNAME(mux_gpll_usb480m_cpll_xin24m_p) = { "gpll", "usb480m", "cpll", "xin24m" }; ++PNAME(mux_gpll_cpll_dpll_p) = { "gpll", "cpll", "dummy_dpll" }; ++PNAME(mux_gpll_cpll_p) = { "gpll", "cpll" }; ++PNAME(mux_gpll_cpll_usb480m_xin24m_p) = { "gpll", "cpll", "usb480m", "xin24m" }; ++PNAME(mux_cpll_gpll_p) = { "cpll", "gpll" }; ++PNAME(mux_gpll_cpll_xin24m_p) = { "gpll", "cpll", "xin24m" }; ++PNAME(mux_cpll_hpll_gpll_p) = { "cpll", "hpll", "gpll" }; ++PNAME(mux_cpll_gpll_hpll_p) = { "cpll", "gpll", "hpll" }; ++PNAME(mux_gpll_cpll_hpll_p) = { "gpll", "cpll", "hpll" }; ++PNAME(mux_gpll_cpll_apll_hpll_p) = { "gpll", "cpll", "dummy_apll", "hpll" }; ++#else ++PNAME(mux_gpll_usb480m_cpll_xin24m_p) = { "gpll", "usb480m", "dummy_cpll", "xin24m" }; ++PNAME(mux_gpll_cpll_dpll_p) = { "gpll", "dummy_cpll", "dummy_dpll" }; ++PNAME(mux_gpll_cpll_p) = { "gpll", "dummy_cpll" }; ++PNAME(mux_gpll_cpll_usb480m_xin24m_p) = { "gpll", "dummy_cpll", "usb480m", "xin24m" }; ++PNAME(mux_cpll_gpll_p) = { "dummy_cpll", "gpll" }; ++PNAME(mux_gpll_cpll_xin24m_p) = { "gpll", "dummy_cpll", "xin24m" }; ++PNAME(mux_cpll_hpll_gpll_p) = { "dummy_cpll", "dummy_hpll", "gpll" }; ++PNAME(mux_cpll_gpll_hpll_p) = { "dummy_cpll", "gpll", "dummy_hpll" }; ++PNAME(mux_gpll_cpll_hpll_p) = { "gpll", "dummy_cpll", "dummy_hpll" }; ++PNAME(mux_gpll_cpll_apll_hpll_p) = { "gpll", "dummy_cpll", "dummy_apll", "dummy_hpll" }; ++#endif ++ ++static u32 rgmii_mux_idx[] = { 2, 3, 0, 1 }; ++ ++static struct rockchip_pll_clock rv1126_pmu_pll_clks[] __initdata = { ++ [gpll] = PLL(pll_rk3328, PLL_GPLL, "gpll", mux_pll_p, ++ CLK_IS_CRITICAL, RV1126_PMU_PLL_CON(0), ++ RV1126_PMU_MODE, 0, 3, 0, rv1126_pll_rates), ++}; ++ ++static struct rockchip_pll_clock rv1126_pll_clks[] __initdata = { ++ [apll] = PLL(pll_rk3328, PLL_APLL, "apll", mux_pll_p, ++ CLK_IGNORE_UNUSED, RV1126_PLL_CON(0), ++ RV1126_MODE_CON, 0, 0, 0, rv1126_pll_rates), ++ [dpll] = PLL(pll_rk3328, PLL_DPLL, "dpll", mux_pll_p, ++ CLK_IGNORE_UNUSED, RV1126_PLL_CON(8), ++ RV1126_MODE_CON, 2, 1, 0, NULL), ++#ifndef CONFIG_ROCKCHIP_LOW_PERFORMANCE ++ [cpll] = PLL(pll_rk3328, PLL_CPLL, "cpll", mux_pll_p, ++ CLK_IS_CRITICAL, RV1126_PLL_CON(16), ++ RV1126_MODE_CON, 4, 2, 0, rv1126_pll_rates), ++ [hpll] = PLL(pll_rk3328, PLL_HPLL, "hpll", mux_pll_p, ++ CLK_IS_CRITICAL, RV1126_PLL_CON(24), ++ RV1126_MODE_CON, 6, 4, 0, rv1126_pll_rates), ++#else ++ [cpll] = PLL(pll_rk3328, PLL_CPLL, "cpll", mux_pll_p, ++ 0, RV1126_PLL_CON(16), ++ RV1126_MODE_CON, 4, 2, 0, rv1126_pll_rates), ++ [hpll] = PLL(pll_rk3328, PLL_HPLL, "hpll", mux_pll_p, ++ 0, RV1126_PLL_CON(24), ++ RV1126_MODE_CON, 6, 4, 0, rv1126_pll_rates), ++#endif ++}; ++ ++#define MFLAGS CLK_MUX_HIWORD_MASK ++#define DFLAGS CLK_DIVIDER_HIWORD_MASK ++#define GFLAGS (CLK_GATE_HIWORD_MASK | CLK_GATE_SET_TO_DISABLE) ++ ++static struct rockchip_clk_branch rv1126_rtc32k_fracmux __initdata = ++ MUX(CLK_RTC32K, "clk_rtc32k", mux_rtc32k_p, CLK_SET_RATE_PARENT, ++ RV1126_PMU_CLKSEL_CON(0), 7, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rv1126_uart1_fracmux __initdata = ++ MUX(SCLK_UART1_MUX, "sclk_uart1_mux", mux_uart1_p, CLK_SET_RATE_PARENT, ++ RV1126_PMU_CLKSEL_CON(4), 10, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rv1126_uart0_fracmux __initdata = ++ MUX(SCLK_UART0_MUX, "sclk_uart0_mux", mux_uart0_p, CLK_SET_RATE_PARENT, ++ RV1126_CLKSEL_CON(10), 10, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rv1126_uart2_fracmux __initdata = ++ MUX(SCLK_UART2_MUX, "sclk_uart2_mux", mux_uart2_p, CLK_SET_RATE_PARENT, ++ RV1126_CLKSEL_CON(12), 10, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rv1126_uart3_fracmux __initdata = ++ MUX(SCLK_UART3_MUX, "sclk_uart3_mux", mux_uart3_p, CLK_SET_RATE_PARENT, ++ RV1126_CLKSEL_CON(14), 10, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rv1126_uart4_fracmux __initdata = ++ MUX(SCLK_UART4_MUX, "sclk_uart4_mux", mux_uart4_p, CLK_SET_RATE_PARENT, ++ RV1126_CLKSEL_CON(16), 10, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rv1126_uart5_fracmux __initdata = ++ MUX(SCLK_UART5_MUX, "sclk_uart5_mux", mux_uart5_p, CLK_SET_RATE_PARENT, ++ RV1126_CLKSEL_CON(18), 10, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rv1126_i2s0_tx_fracmux __initdata = ++ MUX(MCLK_I2S0_TX_MUX, "mclk_i2s0_tx_mux", mux_i2s0_tx_p, CLK_SET_RATE_PARENT, ++ RV1126_CLKSEL_CON(30), 0, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rv1126_i2s0_rx_fracmux __initdata = ++ MUX(MCLK_I2S0_RX_MUX, "mclk_i2s0_rx_mux", mux_i2s0_rx_p, CLK_SET_RATE_PARENT, ++ RV1126_CLKSEL_CON(30), 2, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rv1126_i2s1_fracmux __initdata = ++ MUX(MCLK_I2S1_MUX, "mclk_i2s1_mux", mux_i2s1_p, CLK_SET_RATE_PARENT, ++ RV1126_CLKSEL_CON(31), 8, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rv1126_i2s2_fracmux __initdata = ++ MUX(MCLK_I2S2_MUX, "mclk_i2s2_mux", mux_i2s2_p, CLK_SET_RATE_PARENT, ++ RV1126_CLKSEL_CON(33), 8, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rv1126_audpwm_fracmux __initdata = ++ MUX(SCLK_AUDPWM_MUX, "mclk_audpwm_mux", mux_audpwm_p, CLK_SET_RATE_PARENT, ++ RV1126_CLKSEL_CON(36), 8, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rv1126_dclk_vop_fracmux __initdata = ++ MUX(DCLK_VOP_MUX, "dclk_vop_mux", mux_dclk_vop_p, CLK_SET_RATE_PARENT, ++ RV1126_CLKSEL_CON(47), 10, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rv1126_cif_out2io_fracmux __initdata = ++ MUX(CLK_CIF_OUT_MUX, "clk_cif_out2io_mux", mux_cif_out2io_p, CLK_SET_RATE_PARENT, ++ RV1126_CLKSEL_CON(50), 14, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rv1126_mipicsi_out2io_fracmux __initdata = ++ MUX(CLK_MIPICSI_OUT_MUX, "clk_mipicsi_out2io_mux", mux_mipicsi_out2io_p, CLK_SET_RATE_PARENT, ++ RV1126_CLKSEL_CON(73), 10, 2, MFLAGS); ++ ++static struct rockchip_clk_branch rv1126_aclk_pdvi_np5 __initdata = ++ COMPOSITE_HALFDIV_OFFSET(ACLK_PDVI_NP5, "aclk_pdvi_np5", mux_cpll_gpll_hpll_p, 0, ++ RV1126_CLKSEL_CON(49), 6, 2, MFLAGS, ++ RV1126_CLKSEL_CON(76), 0, 5, DFLAGS, ++ RV1126_CLKGATE_CON(16), 13, GFLAGS); ++ ++static struct rockchip_clk_branch rv1126_clk_isp_np5 __initdata = ++ COMPOSITE_HALFDIV_OFFSET(CLK_ISP_NP5, "clk_isp_np5", mux_gpll_cpll_hpll_p, 0, ++ RV1126_CLKSEL_CON(50), 6, 2, MFLAGS, ++ RV1126_CLKSEL_CON(76), 8, 5, DFLAGS, ++ RV1126_CLKGATE_CON(16), 14, GFLAGS); ++ ++static struct rockchip_clk_branch rv1126_aclk_pdispp_np5 __initdata = ++ COMPOSITE_HALFDIV_OFFSET(ACLK_PDISPP_NP5, "aclk_pdispp_np5", mux_cpll_gpll_hpll_p, 0, ++ RV1126_CLKSEL_CON(68), 6, 2, MFLAGS, ++ RV1126_CLKSEL_CON(77), 0, 5, DFLAGS, ++ RV1126_CLKGATE_CON(16), 8, GFLAGS); ++ ++static struct rockchip_clk_branch rv1126_clk_ispp_np5 __initdata = ++ COMPOSITE_HALFDIV_OFFSET(CLK_ISPP_NP5, "clk_ispp_np5", mux_cpll_gpll_hpll_p, 0, ++ RV1126_CLKSEL_CON(69), 6, 2, MFLAGS, ++ RV1126_CLKSEL_CON(77), 8, 5, DFLAGS, ++ RV1126_CLKGATE_CON(16), 7, GFLAGS); ++ ++static struct rockchip_clk_branch rv1126_aclk_pdnpu_npu5 __initdata = ++ COMPOSITE_HALFDIV(ACLK_PDNPU_NP5, "aclk_pdnpu_np5", mux_gpll_cpll_apll_hpll_p, 0, ++ RV1126_CLKSEL_CON(65), 8, 2, MFLAGS, 4, 4, DFLAGS, ++ RV1126_CLKGATE_CON(22), 1, GFLAGS); ++ ++static struct rockchip_clk_branch rv1126_clk_npu_np5 __initdata = ++ COMPOSITE_HALFDIV(CLK_NPU_NP5, "clk_npu_np5", mux_gpll_cpll_apll_hpll_p, 0, ++ RV1126_CLKSEL_CON(67), 8, 2, MFLAGS, 4, 4, DFLAGS, ++ RV1126_CLKGATE_CON(22), 10, GFLAGS); ++ ++static struct rockchip_clk_branch rv1126_clk_pmu_branches[] __initdata = { ++ /* ++ * Clock-Architecture Diagram 2 ++ */ ++ /* PD_PMU */ ++ COMPOSITE_NOMUX(PCLK_PDPMU, "pclk_pdpmu", "gpll", CLK_IS_CRITICAL, ++ RV1126_PMU_CLKSEL_CON(1), 0, 5, DFLAGS, ++ RV1126_PMU_CLKGATE_CON(0), 0, GFLAGS), ++ ++ COMPOSITE_FRACMUX(CLK_OSC0_DIV32K, "clk_osc0_div32k", "xin24m", CLK_IGNORE_UNUSED, ++ RV1126_PMU_CLKSEL_CON(13), 0, ++ RV1126_PMU_CLKGATE_CON(2), 9, GFLAGS, ++ &rv1126_rtc32k_fracmux, 0), ++ ++ MUXPMUGRF(CLK_32K_IOE, "clk_32k_ioe", mux_clk_32k_ioe_p, 0, ++ RV1126_PMUGRF_SOC_CON0, 0, 1, MFLAGS), ++ ++ COMPOSITE_NOMUX(CLK_WIFI_DIV, "clk_wifi_div", "gpll", 0, ++ RV1126_PMU_CLKSEL_CON(12), 0, 6, DFLAGS, ++ RV1126_PMU_CLKGATE_CON(2), 10, GFLAGS), ++ GATE(CLK_WIFI_OSC0, "clk_wifi_osc0", "xin24m", 0, ++ RV1126_PMU_CLKGATE_CON(2), 11, GFLAGS), ++ MUX(CLK_WIFI, "clk_wifi", mux_wifi_p, CLK_SET_RATE_PARENT, ++ RV1126_PMU_CLKSEL_CON(12), 8, 1, MFLAGS), ++ ++ GATE(PCLK_PMU, "pclk_pmu", "pclk_pdpmu", CLK_IGNORE_UNUSED, ++ RV1126_PMU_CLKGATE_CON(0), 1, GFLAGS), ++ ++ GATE(PCLK_UART1, "pclk_uart1", "pclk_pdpmu", 0, ++ RV1126_PMU_CLKGATE_CON(0), 11, GFLAGS), ++ COMPOSITE(SCLK_UART1_DIV, "sclk_uart1_div", mux_gpll_usb480m_cpll_xin24m_p, 0, ++ RV1126_PMU_CLKSEL_CON(4), 8, 2, MFLAGS, 0, 7, DFLAGS, ++ RV1126_PMU_CLKGATE_CON(0), 12, GFLAGS), ++ COMPOSITE_FRACMUX(SCLK_UART1_FRACDIV, "sclk_uart1_fracdiv", "sclk_uart1_div", CLK_SET_RATE_PARENT, ++ RV1126_PMU_CLKSEL_CON(5), 0, ++ RV1126_PMU_CLKGATE_CON(0), 13, GFLAGS, ++ &rv1126_uart1_fracmux, RV1126_FRAC_MAX_PRATE), ++ GATE(SCLK_UART1, "sclk_uart1", "sclk_uart1_mux", 0, ++ RV1126_PMU_CLKGATE_CON(0), 14, GFLAGS), ++ ++ GATE(PCLK_I2C0, "pclk_i2c0", "pclk_pdpmu", 0, ++ RV1126_PMU_CLKGATE_CON(0), 5, GFLAGS), ++ COMPOSITE_NOMUX(CLK_I2C0, "clk_i2c0", "gpll", 0, ++ RV1126_PMU_CLKSEL_CON(2), 0, 7, DFLAGS, ++ RV1126_PMU_CLKGATE_CON(0), 6, GFLAGS), ++ GATE(PCLK_I2C2, "pclk_i2c2", "pclk_pdpmu", 0, ++ RV1126_PMU_CLKGATE_CON(0), 9, GFLAGS), ++ COMPOSITE_NOMUX(CLK_I2C2, "clk_i2c2", "gpll", 0, ++ RV1126_PMU_CLKSEL_CON(3), 0, 7, DFLAGS, ++ RV1126_PMU_CLKGATE_CON(0), 10, GFLAGS), ++ ++ GATE(CLK_CAPTURE_PWM0, "clk_capture_pwm0", "xin24m", 0, ++ RV1126_PMU_CLKGATE_CON(1), 2, GFLAGS), ++ GATE(PCLK_PWM0, "pclk_pwm0", "pclk_pdpmu", 0, ++ RV1126_PMU_CLKGATE_CON(1), 0, GFLAGS), ++ COMPOSITE(CLK_PWM0, "clk_pwm0", mux_xin24m_gpll_p, 0, ++ RV1126_PMU_CLKSEL_CON(6), 7, 1, MFLAGS, 0, 7, DFLAGS, ++ RV1126_PMU_CLKGATE_CON(1), 1, GFLAGS), ++ GATE(CLK_CAPTURE_PWM1, "clk_capture_pwm1", "xin24m", 0, ++ RV1126_PMU_CLKGATE_CON(1), 5, GFLAGS), ++ GATE(PCLK_PWM1, "pclk_pwm1", "pclk_pdpmu", 0, ++ RV1126_PMU_CLKGATE_CON(1), 3, GFLAGS), ++ COMPOSITE(CLK_PWM1, "clk_pwm1", mux_xin24m_gpll_p, 0, ++ RV1126_PMU_CLKSEL_CON(6), 15, 1, MFLAGS, 8, 7, DFLAGS, ++ RV1126_PMU_CLKGATE_CON(1), 4, GFLAGS), ++ ++ GATE(PCLK_SPI0, "pclk_spi0", "pclk_pdpmu", 0, ++ RV1126_PMU_CLKGATE_CON(1), 11, GFLAGS), ++ COMPOSITE(CLK_SPI0, "clk_spi0", mux_gpll_xin24m_p, 0, ++ RV1126_PMU_CLKSEL_CON(9), 7, 1, MFLAGS, 0, 7, DFLAGS, ++ RV1126_PMU_CLKGATE_CON(1), 12, GFLAGS), ++ ++ GATE(PCLK_GPIO0, "pclk_gpio0", "pclk_pdpmu", 0, ++ RV1126_PMU_CLKGATE_CON(1), 9, GFLAGS), ++ COMPOSITE_NODIV(DBCLK_GPIO0, "dbclk_gpio0", mux_xin24m_32k_p, 0, ++ RV1126_PMU_CLKSEL_CON(8), 15, 1, MFLAGS, ++ RV1126_PMU_CLKGATE_CON(1), 10, GFLAGS), ++ ++ GATE(PCLK_PMUPVTM, "pclk_pmupvtm", "pclk_pdpmu", 0, ++ RV1126_PMU_CLKGATE_CON(2), 6, GFLAGS), ++ GATE(CLK_PMUPVTM, "clk_pmupvtm", "xin24m", 0, ++ RV1126_PMU_CLKGATE_CON(2), 5, GFLAGS), ++ GATE(CLK_CORE_PMUPVTM, "clk_core_pmupvtm", "xin24m", 0, ++ RV1126_PMU_CLKGATE_CON(2), 7, GFLAGS), ++ ++ COMPOSITE_NOMUX(CLK_REF12M, "clk_ref12m", "gpll", 0, ++ RV1126_PMU_CLKSEL_CON(7), 8, 7, DFLAGS, ++ RV1126_PMU_CLKGATE_CON(1), 15, GFLAGS), ++ GATE(0, "xin_osc0_usbphyref_otg", "xin24m", 0, ++ RV1126_PMU_CLKGATE_CON(1), 6, GFLAGS), ++ GATE(0, "xin_osc0_usbphyref_host", "xin24m", 0, ++ RV1126_PMU_CLKGATE_CON(1), 7, GFLAGS), ++ FACTOR(0, "xin_osc0_div2_usbphyref_otg", "xin_osc0_usbphyref_otg", 0, 1, 2), ++ FACTOR(0, "xin_osc0_div2_usbphyref_host", "xin_osc0_usbphyref_host", 0, 1, 2), ++ MUX(CLK_USBPHY_OTG_REF, "clk_usbphy_otg_ref", mux_usbphy_otg_ref_p, CLK_SET_RATE_PARENT, ++ RV1126_PMU_CLKSEL_CON(7), 6, 1, MFLAGS), ++ MUX(CLK_USBPHY_HOST_REF, "clk_usbphy_host_ref", mux_usbphy_host_ref_p, CLK_SET_RATE_PARENT, ++ RV1126_PMU_CLKSEL_CON(7), 7, 1, MFLAGS), ++ ++ COMPOSITE_NOMUX(CLK_REF24M, "clk_ref24m", "gpll", 0, ++ RV1126_PMU_CLKSEL_CON(7), 0, 6, DFLAGS, ++ RV1126_PMU_CLKGATE_CON(1), 14, GFLAGS), ++ GATE(0, "xin_osc0_mipiphyref", "xin24m", 0, ++ RV1126_PMU_CLKGATE_CON(1), 8, GFLAGS), ++ MUX(CLK_MIPIDSIPHY_REF, "clk_mipidsiphy_ref", mux_mipidsiphy_ref_p, CLK_SET_RATE_PARENT, ++ RV1126_PMU_CLKSEL_CON(7), 15, 1, MFLAGS), ++ ++#ifndef CONFIG_ROCKCHIP_LOW_PERFORMANCE ++ GATE(CLK_PMU, "clk_pmu", "xin24m", CLK_IGNORE_UNUSED, ++ RV1126_PMU_CLKGATE_CON(0), 15, GFLAGS), ++ ++ GATE(PCLK_PMUSGRF, "pclk_pmusgrf", "pclk_pdpmu", CLK_IGNORE_UNUSED, ++ RV1126_PMU_CLKGATE_CON(0), 4, GFLAGS), ++ GATE(PCLK_PMUGRF, "pclk_pmugrf", "pclk_pdpmu", CLK_IGNORE_UNUSED, ++ RV1126_PMU_CLKGATE_CON(1), 13, GFLAGS), ++ GATE(PCLK_PMUCRU, "pclk_pmucru", "pclk_pdpmu", CLK_IGNORE_UNUSED, ++ RV1126_PMU_CLKGATE_CON(2), 4, GFLAGS), ++ GATE(PCLK_CHIPVEROTP, "pclk_chipverotp", "pclk_pdpmu", CLK_IGNORE_UNUSED, ++ RV1126_PMU_CLKGATE_CON(2), 0, GFLAGS), ++ GATE(PCLK_PDPMU_NIU, "pclk_pdpmu_niu", "pclk_pdpmu", CLK_IGNORE_UNUSED, ++ RV1126_PMU_CLKGATE_CON(0), 2, GFLAGS), ++ ++ GATE(PCLK_SCRKEYGEN, "pclk_scrkeygen", "pclk_pdpmu", 0, ++ RV1126_PMU_CLKGATE_CON(0), 7, GFLAGS), ++#endif ++}; ++ ++static struct rockchip_clk_branch rv1126_clk_branches[] __initdata = { ++ /* ++ * Clock-Architecture Diagram 1 ++ */ ++ MUX(USB480M, "usb480m", mux_usb480m_p, CLK_SET_RATE_PARENT, ++ RV1126_MODE_CON, 10, 2, MFLAGS), ++ FACTOR(0, "xin12m", "xin24m", 0, 1, 2), ++ ++ /* ++ * Clock-Architecture Diagram 3 ++ */ ++ /* PD_CORE */ ++ COMPOSITE_NOMUX(0, "pclk_dbg", "armclk", CLK_IS_CRITICAL, ++ RV1126_CLKSEL_CON(1), 0, 3, DFLAGS | CLK_DIVIDER_READ_ONLY, ++ RV1126_CLKGATE_CON(0), 6, GFLAGS), ++ GATE(CLK_CORE_CPUPVTM, "clk_core_cpupvtm", "armclk", 0, ++ RV1126_CLKGATE_CON(0), 12, GFLAGS), ++ GATE(PCLK_CPUPVTM, "pclk_cpupvtm", "pclk_dbg", 0, ++ RV1126_CLKGATE_CON(0), 10, GFLAGS), ++ GATE(CLK_CPUPVTM, "clk_cpupvtm", "xin24m", 0, ++ RV1126_CLKGATE_CON(0), 11, GFLAGS), ++ COMPOSITE_NOMUX(HCLK_PDCORE_NIU, "hclk_pdcore_niu", "gpll", CLK_IGNORE_UNUSED, ++ RV1126_CLKSEL_CON(0), 8, 5, DFLAGS, ++ RV1126_CLKGATE_CON(0), 8, GFLAGS), ++ ++ /* ++ * Clock-Architecture Diagram 4 ++ */ ++ /* PD_BUS */ ++ COMPOSITE(0, "aclk_pdbus_pre", mux_gpll_cpll_dpll_p, CLK_IS_CRITICAL, ++ RV1126_CLKSEL_CON(2), 6, 2, MFLAGS, 0, 5, DFLAGS, ++ RV1126_CLKGATE_CON(2), 0, GFLAGS), ++ GATE(ACLK_PDBUS, "aclk_pdbus", "aclk_pdbus_pre", CLK_IS_CRITICAL, ++ RV1126_CLKGATE_CON(2), 11, GFLAGS), ++ COMPOSITE(0, "hclk_pdbus_pre", mux_hclk_pclk_pdbus_p, CLK_IS_CRITICAL, ++ RV1126_CLKSEL_CON(2), 15, 1, MFLAGS, 8, 5, DFLAGS, ++ RV1126_CLKGATE_CON(2), 1, GFLAGS), ++ GATE(HCLK_PDBUS, "hclk_pdbus", "hclk_pdbus_pre", CLK_IS_CRITICAL, ++ RV1126_CLKGATE_CON(2), 12, GFLAGS), ++ COMPOSITE(0, "pclk_pdbus_pre", mux_hclk_pclk_pdbus_p, CLK_IS_CRITICAL, ++ RV1126_CLKSEL_CON(3), 7, 1, MFLAGS, 0, 5, DFLAGS, ++ RV1126_CLKGATE_CON(2), 2, GFLAGS), ++ GATE(PCLK_PDBUS, "pclk_pdbus", "pclk_pdbus_pre", CLK_IS_CRITICAL, ++ RV1126_CLKGATE_CON(2), 13, GFLAGS), ++ /* aclk_dmac is controlled by sgrf_clkgat_con. */ ++ SGRF_GATE(ACLK_DMAC, "aclk_dmac", "hclk_pdbus"), ++ GATE(ACLK_DCF, "aclk_dcf", "hclk_pdbus", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(3), 6, GFLAGS), ++ GATE(PCLK_DCF, "pclk_dcf", "pclk_pdbus", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(3), 7, GFLAGS), ++ GATE(PCLK_WDT, "pclk_wdt", "pclk_pdbus", 0, ++ RV1126_CLKGATE_CON(6), 14, GFLAGS), ++ GATE(PCLK_MAILBOX, "pclk_mailbox", "pclk_pdbus", 0, ++ RV1126_CLKGATE_CON(7), 10, GFLAGS), ++ ++ COMPOSITE(CLK_SCR1, "clk_scr1", mux_gpll_cpll_p, 0, ++ RV1126_CLKSEL_CON(3), 15, 1, MFLAGS, 8, 5, DFLAGS, ++ RV1126_CLKGATE_CON(4), 7, GFLAGS), ++ GATE(0, "clk_scr1_niu", "clk_scr1", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(2), 14, GFLAGS), ++ GATE(CLK_SCR1_CORE, "clk_scr1_core", "clk_scr1", 0, ++ RV1126_CLKGATE_CON(4), 8, GFLAGS), ++ GATE(CLK_SCR1_RTC, "clk_scr1_rtc", "xin24m", 0, ++ RV1126_CLKGATE_CON(4), 9, GFLAGS), ++ GATE(CLK_SCR1_JTAG, "clk_scr1_jtag", "clk_scr1_jtag_io", 0, ++ RV1126_CLKGATE_CON(4), 10, GFLAGS), ++ ++ GATE(PCLK_UART0, "pclk_uart0", "pclk_pdbus", 0, ++ RV1126_CLKGATE_CON(5), 0, GFLAGS), ++ COMPOSITE(SCLK_UART0_DIV, "sclk_uart0_div", mux_gpll_cpll_usb480m_xin24m_p, 0, ++ RV1126_CLKSEL_CON(10), 8, 2, MFLAGS, 0, 7, DFLAGS, ++ RV1126_CLKGATE_CON(5), 1, GFLAGS), ++ COMPOSITE_FRACMUX(SCLK_UART0_FRAC, "sclk_uart0_frac", "sclk_uart0_div", CLK_SET_RATE_PARENT, ++ RV1126_CLKSEL_CON(11), 0, ++ RV1126_CLKGATE_CON(5), 2, GFLAGS, ++ &rv1126_uart0_fracmux, RV1126_FRAC_MAX_PRATE), ++ GATE(SCLK_UART0, "sclk_uart0", "sclk_uart0_mux", 0, ++ RV1126_CLKGATE_CON(5), 3, GFLAGS), ++ GATE(PCLK_UART2, "pclk_uart2", "pclk_pdbus", 0, ++ RV1126_CLKGATE_CON(5), 4, GFLAGS), ++ COMPOSITE(SCLK_UART2_DIV, "sclk_uart2_div", mux_gpll_cpll_usb480m_xin24m_p, 0, ++ RV1126_CLKSEL_CON(12), 8, 2, MFLAGS, 0, 7, DFLAGS, ++ RV1126_CLKGATE_CON(5), 5, GFLAGS), ++ COMPOSITE_FRACMUX(SCLK_UART2_FRAC, "sclk_uart2_frac", "sclk_uart2_div", CLK_SET_RATE_PARENT, ++ RV1126_CLKSEL_CON(13), 0, ++ RV1126_CLKGATE_CON(5), 6, GFLAGS, ++ &rv1126_uart2_fracmux, RV1126_FRAC_MAX_PRATE), ++ GATE(SCLK_UART2, "sclk_uart2", "sclk_uart2_mux", 0, ++ RV1126_CLKGATE_CON(5), 7, GFLAGS), ++ GATE(PCLK_UART3, "pclk_uart3", "pclk_pdbus", 0, ++ RV1126_CLKGATE_CON(5), 8, GFLAGS), ++ COMPOSITE(SCLK_UART3_DIV, "sclk_uart3_div", mux_gpll_cpll_usb480m_xin24m_p, 0, ++ RV1126_CLKSEL_CON(14), 8, 2, MFLAGS, 0, 7, DFLAGS, ++ RV1126_CLKGATE_CON(5), 9, GFLAGS), ++ COMPOSITE_FRACMUX(SCLK_UART3_FRAC, "sclk_uart3_frac", "sclk_uart3_div", CLK_SET_RATE_PARENT, ++ RV1126_CLKSEL_CON(15), 0, ++ RV1126_CLKGATE_CON(5), 10, GFLAGS, ++ &rv1126_uart3_fracmux, RV1126_FRAC_MAX_PRATE), ++ GATE(SCLK_UART3, "sclk_uart3", "sclk_uart3_mux", 0, ++ RV1126_CLKGATE_CON(5), 11, GFLAGS), ++ GATE(PCLK_UART4, "pclk_uart4", "pclk_pdbus", 0, ++ RV1126_CLKGATE_CON(5), 12, GFLAGS), ++ COMPOSITE(SCLK_UART4_DIV, "sclk_uart4_div", mux_gpll_cpll_usb480m_xin24m_p, 0, ++ RV1126_CLKSEL_CON(16), 8, 2, MFLAGS, 0, 7, ++ DFLAGS, RV1126_CLKGATE_CON(5), 13, GFLAGS), ++ COMPOSITE_FRACMUX(SCLK_UART4_FRAC, "sclk_uart4_frac", "sclk_uart4_div", CLK_SET_RATE_PARENT, ++ RV1126_CLKSEL_CON(17), 0, ++ RV1126_CLKGATE_CON(5), 14, GFLAGS, ++ &rv1126_uart4_fracmux, RV1126_FRAC_MAX_PRATE), ++ GATE(SCLK_UART4, "sclk_uart4", "sclk_uart4_mux", 0, ++ RV1126_CLKGATE_CON(5), 15, GFLAGS), ++ GATE(PCLK_UART5, "pclk_uart5", "pclk_pdbus", 0, ++ RV1126_CLKGATE_CON(6), 0, GFLAGS), ++ COMPOSITE(SCLK_UART5_DIV, "sclk_uart5_div", mux_gpll_cpll_usb480m_xin24m_p, 0, ++ RV1126_CLKSEL_CON(18), 8, 2, MFLAGS, 0, 7, ++ DFLAGS, RV1126_CLKGATE_CON(6), 1, GFLAGS), ++ COMPOSITE_FRACMUX(SCLK_UART5_FRAC, "sclk_uart5_frac", "sclk_uart5_div", CLK_SET_RATE_PARENT, ++ RV1126_CLKSEL_CON(19), 0, ++ RV1126_CLKGATE_CON(6), 2, GFLAGS, ++ &rv1126_uart5_fracmux, RV1126_FRAC_MAX_PRATE), ++ GATE(SCLK_UART5, "sclk_uart5", "sclk_uart5_mux", 0, ++ RV1126_CLKGATE_CON(6), 3, GFLAGS), ++ ++ GATE(PCLK_I2C1, "pclk_i2c1", "pclk_pdbus", 0, ++ RV1126_CLKGATE_CON(3), 10, GFLAGS), ++ COMPOSITE_NOMUX(CLK_I2C1, "clk_i2c1", "gpll", 0, ++ RV1126_CLKSEL_CON(5), 0, 7, DFLAGS, ++ RV1126_CLKGATE_CON(3), 11, GFLAGS), ++ GATE(PCLK_I2C3, "pclk_i2c3", "pclk_pdbus", 0, ++ RV1126_CLKGATE_CON(3), 12, GFLAGS), ++ COMPOSITE_NOMUX(CLK_I2C3, "clk_i2c3", "gpll", 0, ++ RV1126_CLKSEL_CON(5), 8, 7, DFLAGS, ++ RV1126_CLKGATE_CON(3), 13, GFLAGS), ++ GATE(PCLK_I2C4, "pclk_i2c4", "pclk_pdbus", 0, ++ RV1126_CLKGATE_CON(3), 14, GFLAGS), ++ COMPOSITE_NOMUX(CLK_I2C4, "clk_i2c4", "gpll", 0, ++ RV1126_CLKSEL_CON(6), 0, 7, DFLAGS, ++ RV1126_CLKGATE_CON(3), 15, GFLAGS), ++ GATE(PCLK_I2C5, "pclk_i2c5", "pclk_pdbus", 0, ++ RV1126_CLKGATE_CON(4), 0, GFLAGS), ++ COMPOSITE_NOMUX(CLK_I2C5, "clk_i2c5", "gpll", 0, ++ RV1126_CLKSEL_CON(6), 8, 7, DFLAGS, ++ RV1126_CLKGATE_CON(4), 1, GFLAGS), ++ ++ GATE(PCLK_SPI1, "pclk_spi1", "pclk_pdbus", 0, ++ RV1126_CLKGATE_CON(4), 2, GFLAGS), ++ COMPOSITE(CLK_SPI1, "clk_spi1", mux_gpll_xin24m_p, 0, ++ RV1126_CLKSEL_CON(8), 7, 1, MFLAGS, 0, 7, DFLAGS, ++ RV1126_CLKGATE_CON(4), 3, GFLAGS), ++ ++ GATE(CLK_CAPTURE_PWM2, "clk_capture_pwm2", "xin24m", 0, ++ RV1126_CLKGATE_CON(4), 6, GFLAGS), ++ GATE(PCLK_PWM2, "pclk_pwm2", "pclk_pdbus", 0, ++ RV1126_CLKGATE_CON(4), 4, GFLAGS), ++ COMPOSITE(CLK_PWM2, "clk_pwm2", mux_xin24m_gpll_p, 0, ++ RV1126_CLKSEL_CON(9), 15, 1, MFLAGS, 8, 7, DFLAGS, ++ RV1126_CLKGATE_CON(4), 5, GFLAGS), ++ ++ GATE(PCLK_GPIO1, "pclk_gpio1", "pclk_pdbus", 0, ++ RV1126_CLKGATE_CON(7), 0, GFLAGS), ++ COMPOSITE_NODIV(DBCLK_GPIO1, "dbclk_gpio1", mux_xin24m_32k_p, 0, ++ RV1126_CLKSEL_CON(21), 15, 1, MFLAGS, ++ RV1126_CLKGATE_CON(7), 1, GFLAGS), ++ GATE(PCLK_GPIO2, "pclk_gpio2", "pclk_pdbus", 0, ++ RV1126_CLKGATE_CON(7), 2, GFLAGS), ++ COMPOSITE_NODIV(DBCLK_GPIO2, "dbclk_gpio2", mux_xin24m_32k_p, 0, ++ RV1126_CLKSEL_CON(22), 15, 1, MFLAGS, ++ RV1126_CLKGATE_CON(7), 3, GFLAGS), ++ GATE(PCLK_GPIO3, "pclk_gpio3", "pclk_pdbus", 0, ++ RV1126_CLKGATE_CON(7), 4, GFLAGS), ++ COMPOSITE_NODIV(DBCLK_GPIO3, "dbclk_gpio3", mux_xin24m_32k_p, 0, ++ RV1126_CLKSEL_CON(23), 15, 1, MFLAGS, ++ RV1126_CLKGATE_CON(7), 5, GFLAGS), ++ GATE(PCLK_GPIO4, "pclk_gpio4", "pclk_pdbus", 0, ++ RV1126_CLKGATE_CON(7), 6, GFLAGS), ++ COMPOSITE_NODIV(DBCLK_GPIO4, "dbclk_gpio4", mux_xin24m_32k_p, 0, ++ RV1126_CLKSEL_CON(24), 15, 1, MFLAGS, ++ RV1126_CLKGATE_CON(7), 7, GFLAGS), ++ ++ GATE(PCLK_SARADC, "pclk_saradc", "pclk_pdbus", 0, ++ RV1126_CLKGATE_CON(6), 4, GFLAGS), ++ COMPOSITE_NOMUX(CLK_SARADC, "clk_saradc", "xin24m", 0, ++ RV1126_CLKSEL_CON(20), 0, 11, DFLAGS, ++ RV1126_CLKGATE_CON(6), 5, GFLAGS), ++ ++ GATE(PCLK_TIMER, "pclk_timer", "pclk_pdbus", 0, ++ RV1126_CLKGATE_CON(6), 7, GFLAGS), ++ GATE(CLK_TIMER0, "clk_timer0", "xin24m", 0, ++ RV1126_CLKGATE_CON(6), 8, GFLAGS), ++ GATE(CLK_TIMER1, "clk_timer1", "xin24m", 0, ++ RV1126_CLKGATE_CON(6), 9, GFLAGS), ++ GATE(CLK_TIMER2, "clk_timer2", "xin24m", 0, ++ RV1126_CLKGATE_CON(6), 10, GFLAGS), ++ GATE(CLK_TIMER3, "clk_timer3", "xin24m", 0, ++ RV1126_CLKGATE_CON(6), 11, GFLAGS), ++ GATE(CLK_TIMER4, "clk_timer4", "xin24m", 0, ++ RV1126_CLKGATE_CON(6), 12, GFLAGS), ++ GATE(CLK_TIMER5, "clk_timer5", "xin24m", 0, ++ RV1126_CLKGATE_CON(6), 13, GFLAGS), ++ ++ GATE(ACLK_SPINLOCK, "aclk_spinlock", "hclk_pdbus", 0, ++ RV1126_CLKGATE_CON(6), 6, GFLAGS), ++ ++ GATE(ACLK_DECOM, "aclk_decom", "aclk_pdbus", 0, ++ RV1126_CLKGATE_CON(7), 11, GFLAGS), ++ GATE(PCLK_DECOM, "pclk_decom", "pclk_pdbus", 0, ++ RV1126_CLKGATE_CON(7), 12, GFLAGS), ++ COMPOSITE(DCLK_DECOM, "dclk_decom", mux_gpll_cpll_p, 0, ++ RV1126_CLKSEL_CON(25), 15, 1, MFLAGS, 8, 7, DFLAGS, ++ RV1126_CLKGATE_CON(7), 13, GFLAGS), ++ ++ GATE(PCLK_CAN, "pclk_can", "pclk_pdbus", 0, ++ RV1126_CLKGATE_CON(7), 8, GFLAGS), ++ COMPOSITE(CLK_CAN, "clk_can", mux_gpll_xin24m_p, 0, ++ RV1126_CLKSEL_CON(25), 7, 1, MFLAGS, 0, 7, DFLAGS, ++ RV1126_CLKGATE_CON(7), 9, GFLAGS), ++ /* pclk_otp and clk_otp are controlled by sgrf_clkgat_con. */ ++ SGRF_GATE(CLK_OTP, "clk_otp", "xin24m"), ++ SGRF_GATE(PCLK_OTP, "pclk_otp", "pclk_pdbus"), ++ ++ GATE(PCLK_NPU_TSADC, "pclk_npu_tsadc", "pclk_pdbus", 0, ++ RV1126_CLKGATE_CON(24), 3, GFLAGS), ++ COMPOSITE_NOMUX(CLK_NPU_TSADC, "clk_npu_tsadc", "xin24m", 0, ++ RV1126_CLKSEL_CON(71), 0, 11, DFLAGS, ++ RV1126_CLKGATE_CON(24), 4, GFLAGS), ++ GATE(CLK_NPU_TSADCPHY, "clk_npu_tsadcphy", "clk_npu_tsadc", 0, ++ RV1126_CLKGATE_CON(24), 5, GFLAGS), ++ GATE(PCLK_CPU_TSADC, "pclk_cpu_tsadc", "pclk_pdbus", 0, ++ RV1126_CLKGATE_CON(24), 0, GFLAGS), ++ COMPOSITE_NOMUX(CLK_CPU_TSADC, "clk_cpu_tsadc", "xin24m", 0, ++ RV1126_CLKSEL_CON(70), 0, 11, DFLAGS, ++ RV1126_CLKGATE_CON(24), 1, GFLAGS), ++ GATE(CLK_CPU_TSADCPHY, "clk_cpu_tsadcphy", "clk_cpu_tsadc", 0, ++ RV1126_CLKGATE_CON(24), 2, GFLAGS), ++ ++ /* ++ * Clock-Architecture Diagram 5 ++ */ ++ /* PD_CRYPTO */ ++ COMPOSITE(ACLK_PDCRYPTO, "aclk_pdcrypto", mux_gpll_cpll_p, 0, ++ RV1126_CLKSEL_CON(4), 7, 1, MFLAGS, 0, 5, DFLAGS, ++ RV1126_CLKGATE_CON(4), 11, GFLAGS), ++ COMPOSITE_NOMUX(HCLK_PDCRYPTO, "hclk_pdcrypto", "aclk_pdcrypto", 0, ++ RV1126_CLKSEL_CON(4), 8, 5, DFLAGS, ++ RV1126_CLKGATE_CON(4), 12, GFLAGS), ++ GATE(ACLK_CRYPTO, "aclk_crypto", "aclk_pdcrypto", 0, ++ RV1126_CLKGATE_CON(3), 2, GFLAGS), ++ GATE(HCLK_CRYPTO, "hclk_crypto", "hclk_pdcrypto", 0, ++ RV1126_CLKGATE_CON(3), 3, GFLAGS), ++ COMPOSITE(CLK_CRYPTO_CORE, "aclk_crypto_core", mux_gpll_cpll_p, 0, ++ RV1126_CLKSEL_CON(7), 7, 1, MFLAGS, 0, 5, DFLAGS, ++ RV1126_CLKGATE_CON(3), 4, GFLAGS), ++ COMPOSITE(CLK_CRYPTO_PKA, "aclk_crypto_pka", mux_gpll_cpll_p, 0, ++ RV1126_CLKSEL_CON(7), 15, 1, MFLAGS, 8, 5, DFLAGS, ++ RV1126_CLKGATE_CON(3), 5, GFLAGS), ++ ++ /* ++ * Clock-Architecture Diagram 6 ++ */ ++ /* PD_AUDIO */ ++ COMPOSITE_NOMUX(HCLK_PDAUDIO, "hclk_pdaudio", "gpll", 0, ++ RV1126_CLKSEL_CON(26), 0, 5, DFLAGS, ++ RV1126_CLKGATE_CON(9), 0, GFLAGS), ++ ++ GATE(HCLK_I2S0, "hclk_i2s0", "hclk_pdaudio", 0, ++ RV1126_CLKGATE_CON(9), 4, GFLAGS), ++ COMPOSITE(MCLK_I2S0_TX_DIV, "mclk_i2s0_tx_div", mux_cpll_gpll_p, 0, ++ RV1126_CLKSEL_CON(27), 7, 1, MFLAGS, 0, 7, DFLAGS, ++ RV1126_CLKGATE_CON(9), 5, GFLAGS), ++ COMPOSITE_FRACMUX(MCLK_I2S0_TX_FRACDIV, "mclk_i2s0_tx_fracdiv", "mclk_i2s0_tx_div", CLK_SET_RATE_PARENT, ++ RV1126_CLKSEL_CON(28), 0, ++ RV1126_CLKGATE_CON(9), 6, GFLAGS, ++ &rv1126_i2s0_tx_fracmux, RV1126_FRAC_MAX_PRATE), ++ GATE(MCLK_I2S0_TX, "mclk_i2s0_tx", "mclk_i2s0_tx_mux", 0, ++ RV1126_CLKGATE_CON(9), 9, GFLAGS), ++ COMPOSITE(MCLK_I2S0_RX_DIV, "mclk_i2s0_rx_div", mux_cpll_gpll_p, 0, ++ RV1126_CLKSEL_CON(27), 15, 1, MFLAGS, 8, 7, DFLAGS, ++ RV1126_CLKGATE_CON(9), 7, GFLAGS), ++ COMPOSITE_FRACMUX(MCLK_I2S0_RX_FRACDIV, "mclk_i2s0_rx_fracdiv", "mclk_i2s0_rx_div", CLK_SET_RATE_PARENT, ++ RV1126_CLKSEL_CON(29), 0, ++ RV1126_CLKGATE_CON(9), 8, GFLAGS, ++ &rv1126_i2s0_rx_fracmux, RV1126_FRAC_MAX_PRATE), ++ GATE(MCLK_I2S0_RX, "mclk_i2s0_rx", "mclk_i2s0_rx_mux", 0, ++ RV1126_CLKGATE_CON(9), 10, GFLAGS), ++ COMPOSITE_NODIV(MCLK_I2S0_TX_OUT2IO, "mclk_i2s0_tx_out2io", mux_i2s0_tx_out2io_p, CLK_SET_RATE_PARENT, ++ RV1126_CLKSEL_CON(30), 6, 1, MFLAGS, ++ RV1126_CLKGATE_CON(9), 13, GFLAGS), ++ COMPOSITE_NODIV(MCLK_I2S0_RX_OUT2IO, "mclk_i2s0_rx_out2io", mux_i2s0_rx_out2io_p, CLK_SET_RATE_PARENT, ++ RV1126_CLKSEL_CON(30), 8, 1, MFLAGS, ++ RV1126_CLKGATE_CON(9), 14, GFLAGS), ++ ++ GATE(HCLK_I2S1, "hclk_i2s1", "hclk_pdaudio", 0, ++ RV1126_CLKGATE_CON(10), 0, GFLAGS), ++ COMPOSITE(MCLK_I2S1_DIV, "mclk_i2s1_div", mux_cpll_gpll_p, 0, ++ RV1126_CLKSEL_CON(31), 7, 1, MFLAGS, 0, 7, DFLAGS, ++ RV1126_CLKGATE_CON(10), 1, GFLAGS), ++ COMPOSITE_FRACMUX(MCLK_I2S1_FRACDIV, "mclk_i2s1_fracdiv", "mclk_i2s1_div", CLK_SET_RATE_PARENT, ++ RV1126_CLKSEL_CON(32), 0, ++ RV1126_CLKGATE_CON(10), 2, GFLAGS, ++ &rv1126_i2s1_fracmux, RV1126_FRAC_MAX_PRATE), ++ GATE(MCLK_I2S1, "mclk_i2s1", "mclk_i2s1_mux", 0, ++ RV1126_CLKGATE_CON(10), 3, GFLAGS), ++ COMPOSITE_NODIV(MCLK_I2S1_OUT2IO, "mclk_i2s1_out2io", mux_i2s1_out2io_p, CLK_SET_RATE_PARENT, ++ RV1126_CLKSEL_CON(31), 12, 1, MFLAGS, ++ RV1126_CLKGATE_CON(10), 4, GFLAGS), ++ GATE(HCLK_I2S2, "hclk_i2s2", "hclk_pdaudio", 0, ++ RV1126_CLKGATE_CON(10), 5, GFLAGS), ++ COMPOSITE(MCLK_I2S2_DIV, "mclk_i2s2_div", mux_cpll_gpll_p, 0, ++ RV1126_CLKSEL_CON(33), 7, 1, MFLAGS, 0, 7, DFLAGS, ++ RV1126_CLKGATE_CON(10), 6, GFLAGS), ++ COMPOSITE_FRACMUX(MCLK_I2S2_FRACDIV, "mclk_i2s2_fracdiv", "mclk_i2s2_div", CLK_SET_RATE_PARENT, ++ RV1126_CLKSEL_CON(34), 0, ++ RV1126_CLKGATE_CON(10), 7, GFLAGS, ++ &rv1126_i2s2_fracmux, RV1126_FRAC_MAX_PRATE), ++ GATE(MCLK_I2S2, "mclk_i2s2", "mclk_i2s2_mux", 0, ++ RV1126_CLKGATE_CON(10), 8, GFLAGS), ++ COMPOSITE_NODIV(MCLK_I2S2_OUT2IO, "mclk_i2s2_out2io", mux_i2s2_out2io_p, CLK_SET_RATE_PARENT, ++ RV1126_CLKSEL_CON(33), 10, 1, MFLAGS, ++ RV1126_CLKGATE_CON(10), 9, GFLAGS), ++ ++ GATE(HCLK_PDM, "hclk_pdm", "hclk_pdaudio", 0, ++ RV1126_CLKGATE_CON(10), 10, GFLAGS), ++ COMPOSITE(MCLK_PDM, "mclk_pdm", mux_gpll_cpll_xin24m_p, 0, ++ RV1126_CLKSEL_CON(35), 8, 2, MFLAGS, 0, 7, DFLAGS, ++ RV1126_CLKGATE_CON(10), 11, GFLAGS), ++ ++ GATE(HCLK_AUDPWM, "hclk_audpwm", "hclk_pdaudio", 0, ++ RV1126_CLKGATE_CON(10), 12, GFLAGS), ++ COMPOSITE(SCLK_ADUPWM_DIV, "sclk_audpwm_div", mux_gpll_cpll_p, 0, ++ RV1126_CLKSEL_CON(36), 7, 1, MFLAGS, 0, 7, DFLAGS, ++ RV1126_CLKGATE_CON(10), 13, GFLAGS), ++ COMPOSITE_FRACMUX(SCLK_AUDPWM_FRACDIV, "sclk_audpwm_fracdiv", "sclk_audpwm_div", CLK_SET_RATE_PARENT, ++ RV1126_CLKSEL_CON(37), 0, ++ RV1126_CLKGATE_CON(10), 14, GFLAGS, ++ &rv1126_audpwm_fracmux, RV1126_FRAC_MAX_PRATE), ++ GATE(SCLK_AUDPWM, "sclk_audpwm", "mclk_audpwm_mux", 0, ++ RV1126_CLKGATE_CON(10), 15, GFLAGS), ++ ++ GATE(PCLK_ACDCDIG, "pclk_acdcdig", "hclk_pdaudio", 0, ++ RV1126_CLKGATE_CON(11), 0, GFLAGS), ++ GATE(CLK_ACDCDIG_ADC, "clk_acdcdig_adc", "mclk_i2s0_rx", 0, ++ RV1126_CLKGATE_CON(11), 2, GFLAGS), ++ GATE(CLK_ACDCDIG_DAC, "clk_acdcdig_dac", "mclk_i2s0_tx", 0, ++ RV1126_CLKGATE_CON(11), 3, GFLAGS), ++ COMPOSITE(CLK_ACDCDIG_I2C, "clk_acdcdig_i2c", mux_gpll_xin24m_p, 0, ++ RV1126_CLKSEL_CON(72), 8, 1, MFLAGS, 0, 7, DFLAGS, ++ RV1126_CLKGATE_CON(11), 1, GFLAGS), ++ ++ /* ++ * Clock-Architecture Diagram 7 ++ */ ++ /* PD_VEPU */ ++ COMPOSITE(ACLK_PDVEPU, "aclk_pdvepu", mux_cpll_hpll_gpll_p, 0, ++ RV1126_CLKSEL_CON(40), 6, 2, MFLAGS, 0, 5, DFLAGS, ++ RV1126_CLKGATE_CON(12), 0, GFLAGS), ++ COMPOSITE_NOMUX(HCLK_PDVEPU, "hclk_pdvepu", "aclk_pdvepu", 0, ++ RV1126_CLKSEL_CON(41), 0, 5, DFLAGS, ++ RV1126_CLKGATE_CON(12), 2, GFLAGS), ++ GATE(ACLK_VENC, "aclk_venc", "aclk_pdvepu", 0, ++ RV1126_CLKGATE_CON(12), 5, GFLAGS), ++ GATE(HCLK_VENC, "hclk_venc", "hclk_pdvepu", 0, ++ RV1126_CLKGATE_CON(12), 6, GFLAGS), ++ COMPOSITE(CLK_VENC_CORE, "clk_venc_core", mux_cpll_gpll_hpll_p, 0, ++ RV1126_CLKSEL_CON(40), 14, 2, MFLAGS, 8, 5, DFLAGS, ++ RV1126_CLKGATE_CON(12), 1, GFLAGS), ++ ++ /* ++ * Clock-Architecture Diagram 8 ++ */ ++ /* PD_VDPU */ ++#if IS_ENABLED(CONFIG_ROCKCHIP_MPP_VDPU2) || IS_ENABLED(CONFIG_ROCKCHIP_MPP_RKVDEC) ++ COMPOSITE(ACLK_PDVDEC, "aclk_pdvdec", mux_cpll_hpll_gpll_p, CLK_IS_CRITICAL, ++ RV1126_CLKSEL_CON(42), 6, 2, MFLAGS, 0, 5, DFLAGS, ++ RV1126_CLKGATE_CON(13), 0, GFLAGS), ++ COMPOSITE_NOMUX(HCLK_PDVDEC, "hclk_pdvdec", "aclk_pdvdec", CLK_IS_CRITICAL, ++ RV1126_CLKSEL_CON(41), 8, 5, DFLAGS, ++ RV1126_CLKGATE_CON(13), 4, GFLAGS), ++ GATE(0, "aclk_pdvdec_niu", "aclk_pdvdec", CLK_IS_CRITICAL, ++ RV1126_CLKGATE_CON(13), 5, GFLAGS), ++ GATE(0, "hclk_pdvdec_niu", "hclk_pdvdec", CLK_IS_CRITICAL, ++ RV1126_CLKGATE_CON(13), 6, GFLAGS), ++ COMPOSITE(ACLK_PDJPEG, "aclk_pdjpeg", mux_cpll_hpll_gpll_p, CLK_IS_CRITICAL, ++ RV1126_CLKSEL_CON(44), 6, 2, MFLAGS, 0, 5, DFLAGS, ++ RV1126_CLKGATE_CON(13), 9, GFLAGS), ++ COMPOSITE_NOMUX(HCLK_PDJPEG, "hclk_pdjpeg", "aclk_pdjpeg", CLK_IS_CRITICAL, ++ RV1126_CLKSEL_CON(44), 8, 5, DFLAGS, ++ RV1126_CLKGATE_CON(13), 10, GFLAGS), ++ GATE(0, "aclk_pdjpeg_niu", "aclk_pdjpeg", CLK_IS_CRITICAL, ++ RV1126_CLKGATE_CON(13), 11, GFLAGS), ++ GATE(0, "hclk_pdjpeg_niu", "hclk_pdjpeg", CLK_IS_CRITICAL, ++ RV1126_CLKGATE_CON(13), 12, GFLAGS), ++#else ++ COMPOSITE(ACLK_PDVDEC, "aclk_pdvdec", mux_cpll_hpll_gpll_p, 0, ++ RV1126_CLKSEL_CON(42), 6, 2, MFLAGS, 0, 5, DFLAGS, ++ RV1126_CLKGATE_CON(13), 0, GFLAGS), ++ COMPOSITE_NOMUX(HCLK_PDVDEC, "hclk_pdvdec", "aclk_pdvdec", 0, ++ RV1126_CLKSEL_CON(41), 8, 5, DFLAGS, ++ RV1126_CLKGATE_CON(13), 4, GFLAGS), ++ GATE(0, "aclk_pdvdec_niu", "aclk_pdvdec", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(13), 5, GFLAGS), ++ GATE(0, "hclk_pdvdec_niu", "hclk_pdvdec", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(13), 6, GFLAGS), ++ COMPOSITE(ACLK_PDJPEG, "aclk_pdjpeg", mux_cpll_hpll_gpll_p, 0, ++ RV1126_CLKSEL_CON(44), 6, 2, MFLAGS, 0, 5, DFLAGS, ++ RV1126_CLKGATE_CON(13), 9, GFLAGS), ++ COMPOSITE_NOMUX(HCLK_PDJPEG, "hclk_pdjpeg", "aclk_pdjpeg", 0, ++ RV1126_CLKSEL_CON(44), 8, 5, DFLAGS, ++ RV1126_CLKGATE_CON(13), 10, GFLAGS), ++ GATE(0, "aclk_pdjpeg_niu", "aclk_pdjpeg", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(13), 11, GFLAGS), ++ GATE(0, "hclk_pdjpeg_niu", "hclk_pdjpeg", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(13), 12, GFLAGS), ++#endif ++ GATE(ACLK_VDEC, "aclk_vdec", "aclk_pdvdec", 0, ++ RV1126_CLKGATE_CON(13), 7, GFLAGS), ++ GATE(HCLK_VDEC, "hclk_vdec", "hclk_pdvdec", 0, ++ RV1126_CLKGATE_CON(13), 8, GFLAGS), ++ COMPOSITE(CLK_VDEC_CORE, "clk_vdec_core", mux_cpll_hpll_gpll_p, 0, ++ RV1126_CLKSEL_CON(42), 14, 2, MFLAGS, 8, 5, DFLAGS, ++ RV1126_CLKGATE_CON(13), 1, GFLAGS), ++ COMPOSITE(CLK_VDEC_CA, "clk_vdec_ca", mux_cpll_hpll_gpll_p, 0, ++ RV1126_CLKSEL_CON(43), 6, 2, MFLAGS, 0, 5, DFLAGS, ++ RV1126_CLKGATE_CON(13), 2, GFLAGS), ++ COMPOSITE(CLK_VDEC_HEVC_CA, "clk_vdec_hevc_ca", mux_cpll_hpll_gpll_p, 0, ++ RV1126_CLKSEL_CON(43), 14, 2, MFLAGS, 8, 5, DFLAGS, ++ RV1126_CLKGATE_CON(13), 3, GFLAGS), ++ GATE(ACLK_JPEG, "aclk_jpeg", "aclk_pdjpeg", 0, ++ RV1126_CLKGATE_CON(13), 13, GFLAGS), ++ GATE(HCLK_JPEG, "hclk_jpeg", "hclk_pdjpeg", 0, ++ RV1126_CLKGATE_CON(13), 14, GFLAGS), ++ ++ /* ++ * Clock-Architecture Diagram 9 ++ */ ++ /* PD_VO */ ++ COMPOSITE(ACLK_PDVO, "aclk_pdvo", mux_gpll_cpll_p, 0, ++ RV1126_CLKSEL_CON(45), 7, 1, MFLAGS, 0, 5, DFLAGS, ++ RV1126_CLKGATE_CON(14), 0, GFLAGS), ++ COMPOSITE_NOMUX(HCLK_PDVO, "hclk_pdvo", "aclk_pdvo", 0, ++ RV1126_CLKSEL_CON(45), 8, 5, DFLAGS, ++ RV1126_CLKGATE_CON(14), 1, GFLAGS), ++ COMPOSITE_NOMUX(PCLK_PDVO, "pclk_pdvo", "aclk_pdvo", 0, ++ RV1126_CLKSEL_CON(46), 8, 5, DFLAGS, ++ RV1126_CLKGATE_CON(14), 2, GFLAGS), ++ GATE(ACLK_RGA, "aclk_rga", "aclk_pdvo", 0, ++ RV1126_CLKGATE_CON(14), 6, GFLAGS), ++ GATE(HCLK_RGA, "hclk_rga", "hclk_pdvo", 0, ++ RV1126_CLKGATE_CON(14), 7, GFLAGS), ++ COMPOSITE(CLK_RGA_CORE, "clk_rga_core", mux_gpll_cpll_p, 0, ++ RV1126_CLKSEL_CON(46), 7, 1, MFLAGS, 0, 5, DFLAGS, ++ RV1126_CLKGATE_CON(14), 8, GFLAGS), ++ GATE(ACLK_VOP, "aclk_vop", "aclk_pdvo", 0, ++ RV1126_CLKGATE_CON(14), 9, GFLAGS), ++ GATE(HCLK_VOP, "hclk_vop", "hclk_pdvo", 0, ++ RV1126_CLKGATE_CON(14), 10, GFLAGS), ++ COMPOSITE(DCLK_VOP_DIV, "dclk_vop_div", mux_gpll_cpll_p, 0, ++ RV1126_CLKSEL_CON(47), 8, 1, MFLAGS, 0, 8, DFLAGS, ++ RV1126_CLKGATE_CON(14), 11, GFLAGS), ++ COMPOSITE_FRACMUX(DCLK_VOP_FRACDIV, "dclk_vop_fracdiv", "dclk_vop_div", CLK_SET_RATE_PARENT, ++ RV1126_CLKSEL_CON(48), 0, ++ RV1126_CLKGATE_CON(14), 12, GFLAGS, ++ &rv1126_dclk_vop_fracmux, RV1126_FRAC_MAX_PRATE), ++ GATE(DCLK_VOP, "dclk_vop", "dclk_vop_mux", 0, ++ RV1126_CLKGATE_CON(14), 13, GFLAGS), ++ GATE(PCLK_DSIHOST, "pclk_dsihost", "pclk_pdvo", 0, ++ RV1126_CLKGATE_CON(14), 14, GFLAGS), ++ GATE(ACLK_IEP, "aclk_iep", "aclk_pdvo", 0, ++ RV1126_CLKGATE_CON(12), 7, GFLAGS), ++ GATE(HCLK_IEP, "hclk_iep", "hclk_pdvo", 0, ++ RV1126_CLKGATE_CON(12), 8, GFLAGS), ++ COMPOSITE(CLK_IEP_CORE, "clk_iep_core", mux_gpll_cpll_p, 0, ++ RV1126_CLKSEL_CON(54), 7, 1, MFLAGS, 0, 5, DFLAGS, ++ RV1126_CLKGATE_CON(12), 9, GFLAGS), ++ ++ /* ++ * Clock-Architecture Diagram 10 ++ */ ++ /* PD_VI */ ++ COMPOSITE_BROTHER(ACLK_PDVI_DIV, "aclk_pdvi_div", mux_cpll_gpll_hpll_p, 0, ++ RV1126_CLKSEL_CON(49), 6, 2, MFLAGS, 0, 5, DFLAGS, ++ RV1126_CLKGATE_CON(15), 0, GFLAGS, ++ &rv1126_aclk_pdvi_np5), ++ MUX(ACLK_PDVI, "aclk_pdvi", mux_aclk_pdvi_p, CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE, ++ RV1126_CLKSEL_CON(76), 5, 1, MFLAGS), ++ COMPOSITE_NOMUX(HCLK_PDVI, "hclk_pdvi", "aclk_pdvi", 0, ++ RV1126_CLKSEL_CON(49), 8, 5, DFLAGS, ++ RV1126_CLKGATE_CON(15), 1, GFLAGS), ++ COMPOSITE_NOMUX(PCLK_PDVI, "pclk_pdvi", "aclk_pdvi", 0, ++ RV1126_CLKSEL_CON(50), 8, 5, DFLAGS, ++ RV1126_CLKGATE_CON(15), 2, GFLAGS), ++ GATE(ACLK_ISP, "aclk_isp", "aclk_pdvi", 0, ++ RV1126_CLKGATE_CON(15), 6, GFLAGS), ++ GATE(HCLK_ISP, "hclk_isp", "hclk_pdvi", 0, ++ RV1126_CLKGATE_CON(15), 7, GFLAGS), ++ COMPOSITE_BROTHER(CLK_ISP_DIV, "clk_isp_div", mux_gpll_cpll_hpll_p, 0, ++ RV1126_CLKSEL_CON(50), 6, 2, MFLAGS, 0, 5, DFLAGS, ++ RV1126_CLKGATE_CON(15), 8, GFLAGS, ++ &rv1126_clk_isp_np5), ++ MUX(CLK_ISP, "clk_isp", mux_clk_isp_p, CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE, ++ RV1126_CLKSEL_CON(76), 13, 1, MFLAGS), ++ GATE(ACLK_CIF, "aclk_cif", "aclk_pdvi", 0, ++ RV1126_CLKGATE_CON(15), 9, GFLAGS), ++ GATE(HCLK_CIF, "hclk_cif", "hclk_pdvi", 0, ++ RV1126_CLKGATE_CON(15), 10, GFLAGS), ++ COMPOSITE(DCLK_CIF, "dclk_cif", mux_gpll_cpll_hpll_p, 0, ++ RV1126_CLKSEL_CON(51), 6, 2, MFLAGS, 0, 5, DFLAGS, ++ RV1126_CLKGATE_CON(15), 11, GFLAGS), ++ COMPOSITE(CLK_CIF_OUT_DIV, "clk_cif_out2io_div", mux_gpll_usb480m_p, 0, ++ RV1126_CLKSEL_CON(51), 15, 1, MFLAGS, 8, 6, DFLAGS, ++ RV1126_CLKGATE_CON(15), 12, GFLAGS), ++ COMPOSITE_FRACMUX(CLK_CIF_OUT_FRACDIV, "clk_cif_out2io_fracdiv", "clk_cif_out2io_div", CLK_SET_RATE_PARENT, ++ RV1126_CLKSEL_CON(52), 0, ++ RV1126_CLKGATE_CON(15), 13, GFLAGS, ++ &rv1126_cif_out2io_fracmux, RV1126_FRAC_MAX_PRATE), ++ GATE(CLK_CIF_OUT, "clk_cif_out2io", "clk_cif_out2io_mux", 0, ++ RV1126_CLKGATE_CON(15), 14, GFLAGS), ++ COMPOSITE(CLK_MIPICSI_OUT_DIV, "clk_mipicsi_out2io_div", mux_gpll_usb480m_p, 0, ++ RV1126_CLKSEL_CON(73), 8, 1, MFLAGS, 0, 5, DFLAGS, ++ RV1126_CLKGATE_CON(23), 5, GFLAGS), ++ COMPOSITE_FRACMUX(CLK_MIPICSI_OUT_FRACDIV, "clk_mipicsi_out2io_fracdiv", "clk_mipicsi_out2io_div", CLK_SET_RATE_PARENT, ++ RV1126_CLKSEL_CON(74), 0, ++ RV1126_CLKGATE_CON(23), 6, GFLAGS, ++ &rv1126_mipicsi_out2io_fracmux, RV1126_CSIOUT_FRAC_MAX_PRATE), ++ GATE(CLK_MIPICSI_OUT, "clk_mipicsi_out2io", "clk_mipicsi_out2io_mux", 0, ++ RV1126_CLKGATE_CON(23), 7, GFLAGS), ++ GATE(PCLK_CSIHOST, "pclk_csihost", "pclk_pdvi", 0, ++ RV1126_CLKGATE_CON(15), 15, GFLAGS), ++ GATE(ACLK_CIFLITE, "aclk_ciflite", "aclk_pdvi", 0, ++ RV1126_CLKGATE_CON(16), 10, GFLAGS), ++ GATE(HCLK_CIFLITE, "hclk_ciflite", "hclk_pdvi", 0, ++ RV1126_CLKGATE_CON(16), 11, GFLAGS), ++ COMPOSITE(DCLK_CIFLITE, "dclk_ciflite", mux_gpll_cpll_hpll_p, 0, ++ RV1126_CLKSEL_CON(54), 14, 2, MFLAGS, 8, 5, DFLAGS, ++ RV1126_CLKGATE_CON(16), 12, GFLAGS), ++ ++ /* ++ * Clock-Architecture Diagram 11 ++ */ ++ /* PD_ISPP */ ++ COMPOSITE_BROTHER(ACLK_PDISPP_DIV, "aclk_pdispp_div", mux_cpll_gpll_hpll_p, 0, ++ RV1126_CLKSEL_CON(68), 6, 2, MFLAGS, 0, 5, DFLAGS, ++ RV1126_CLKGATE_CON(16), 0, GFLAGS, ++ &rv1126_aclk_pdispp_np5), ++ MUX(ACLK_PDISPP, "aclk_pdispp", mux_aclk_pdispp_p, CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE, ++ RV1126_CLKSEL_CON(77), 5, 1, MFLAGS), ++ COMPOSITE_NOMUX(HCLK_PDISPP, "hclk_pdispp", "aclk_pdispp", 0, ++ RV1126_CLKSEL_CON(69), 8, 5, DFLAGS, ++ RV1126_CLKGATE_CON(16), 1, GFLAGS), ++ GATE(ACLK_ISPP, "aclk_ispp", "aclk_pdispp", 0, ++ RV1126_CLKGATE_CON(16), 4, GFLAGS), ++ GATE(HCLK_ISPP, "hclk_ispp", "hclk_pdispp", 0, ++ RV1126_CLKGATE_CON(16), 5, GFLAGS), ++ COMPOSITE_BROTHER(CLK_ISPP_DIV, "clk_ispp_div", mux_cpll_gpll_hpll_p, 0, ++ RV1126_CLKSEL_CON(69), 6, 2, MFLAGS, 0, 5, DFLAGS, ++ RV1126_CLKGATE_CON(16), 6, GFLAGS, ++ &rv1126_clk_ispp_np5), ++ MUX(CLK_ISPP, "clk_ispp", mux_clk_ispp_p, CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE, ++ RV1126_CLKSEL_CON(77), 13, 1, MFLAGS), ++ ++ /* ++ * Clock-Architecture Diagram 12 ++ */ ++ /* PD_PHP */ ++ COMPOSITE(ACLK_PDPHP, "aclk_pdphp", mux_gpll_cpll_p, CLK_IS_CRITICAL, ++ RV1126_CLKSEL_CON(53), 7, 1, MFLAGS, 0, 5, DFLAGS, ++ RV1126_CLKGATE_CON(17), 0, GFLAGS), ++ COMPOSITE_NOMUX(HCLK_PDPHP, "hclk_pdphp", "gpll", CLK_IS_CRITICAL, ++ RV1126_CLKSEL_CON(53), 8, 5, DFLAGS, ++ RV1126_CLKGATE_CON(17), 1, GFLAGS), ++ /* PD_SDCARD */ ++ GATE(HCLK_PDSDMMC, "hclk_pdsdmmc", "hclk_pdphp", 0, ++ RV1126_CLKGATE_CON(17), 6, GFLAGS), ++ GATE(HCLK_SDMMC, "hclk_sdmmc", "hclk_pdsdmmc", 0, ++ RV1126_CLKGATE_CON(18), 4, GFLAGS), ++ COMPOSITE(CLK_SDMMC, "clk_sdmmc", mux_gpll_cpll_xin24m_p, 0, ++ RV1126_CLKSEL_CON(55), 14, 2, MFLAGS, 0, 8, ++ DFLAGS, RV1126_CLKGATE_CON(18), 5, GFLAGS), ++ MMC(SCLK_SDMMC_DRV, "sdmmc_drv", "clk_sdmmc", RV1126_SDMMC_CON0, 1), ++ MMC(SCLK_SDMMC_SAMPLE, "sdmmc_sample", "clk_sdmmc", RV1126_SDMMC_CON1, 1), ++ ++ /* PD_SDIO */ ++ GATE(HCLK_PDSDIO, "hclk_pdsdio", "hclk_pdphp", 0, ++ RV1126_CLKGATE_CON(17), 8, GFLAGS), ++ GATE(HCLK_SDIO, "hclk_sdio", "hclk_pdsdio", 0, ++ RV1126_CLKGATE_CON(18), 6, GFLAGS), ++ COMPOSITE(CLK_SDIO, "clk_sdio", mux_gpll_cpll_xin24m_p, 0, ++ RV1126_CLKSEL_CON(56), 14, 2, MFLAGS, 0, 8, DFLAGS, ++ RV1126_CLKGATE_CON(18), 7, GFLAGS), ++ MMC(SCLK_SDIO_DRV, "sdio_drv", "clk_sdio", RV1126_SDIO_CON0, 1), ++ MMC(SCLK_SDIO_SAMPLE, "sdio_sample", "clk_sdio", RV1126_SDIO_CON1, 1), ++ ++ /* PD_NVM */ ++ GATE(HCLK_PDNVM, "hclk_pdnvm", "hclk_pdphp", 0, ++ RV1126_CLKGATE_CON(18), 1, GFLAGS), ++ GATE(HCLK_EMMC, "hclk_emmc", "hclk_pdnvm", 0, ++ RV1126_CLKGATE_CON(18), 8, GFLAGS), ++ COMPOSITE(CLK_EMMC, "clk_emmc", mux_gpll_cpll_xin24m_p, 0, ++ RV1126_CLKSEL_CON(57), 14, 2, MFLAGS, 0, 8, DFLAGS, ++ RV1126_CLKGATE_CON(18), 9, GFLAGS), ++ GATE(HCLK_NANDC, "hclk_nandc", "hclk_pdnvm", 0, ++ RV1126_CLKGATE_CON(18), 13, GFLAGS), ++ COMPOSITE(CLK_NANDC, "clk_nandc", mux_gpll_cpll_p, 0, ++ RV1126_CLKSEL_CON(59), 15, 1, MFLAGS, 0, 8, DFLAGS, ++ RV1126_CLKGATE_CON(18), 14, GFLAGS), ++ GATE(HCLK_SFC, "hclk_sfc", "hclk_pdnvm", 0, ++ RV1126_CLKGATE_CON(18), 10, GFLAGS), ++ GATE(HCLK_SFCXIP, "hclk_sfcxip", "hclk_pdnvm", 0, ++ RV1126_CLKGATE_CON(18), 11, GFLAGS), ++ COMPOSITE(SCLK_SFC, "sclk_sfc", mux_cpll_gpll_p, 0, ++ RV1126_CLKSEL_CON(58), 15, 1, MFLAGS, 0, 8, DFLAGS, ++ RV1126_CLKGATE_CON(18), 12, GFLAGS), ++ MMC(SCLK_EMMC_DRV, "emmc_drv", "clk_emmc", RV1126_EMMC_CON0, 1), ++ MMC(SCLK_EMMC_SAMPLE, "emmc_sample", "clk_emmc", RV1126_EMMC_CON1, 1), ++ ++ /* PD_USB */ ++ GATE(ACLK_PDUSB, "aclk_pdusb", "aclk_pdphp", 0, ++ RV1126_CLKGATE_CON(19), 0, GFLAGS), ++ GATE(HCLK_PDUSB, "hclk_pdusb", "hclk_pdphp", 0, ++ RV1126_CLKGATE_CON(19), 1, GFLAGS), ++ GATE(HCLK_USBHOST, "hclk_usbhost", "hclk_pdusb", 0, ++ RV1126_CLKGATE_CON(19), 4, GFLAGS), ++ GATE(HCLK_USBHOST_ARB, "hclk_usbhost_arb", "hclk_pdusb", 0, ++ RV1126_CLKGATE_CON(19), 5, GFLAGS), ++#if IS_ENABLED(CONFIG_USB_EHCI_HCD_PLATFORM) || IS_ENABLED(CONFIG_USB_OHCI_HCD_PLATFORM) ++ COMPOSITE(CLK_USBHOST_UTMI_OHCI, "clk_usbhost_utmi_ohci", mux_usb480m_gpll_p, CLK_IS_CRITICAL, ++ RV1126_CLKSEL_CON(61), 7, 1, MFLAGS, 0, 5, DFLAGS, ++ RV1126_CLKGATE_CON(19), 6, GFLAGS), ++#else ++ COMPOSITE(CLK_USBHOST_UTMI_OHCI, "clk_usbhost_utmi_ohci", mux_usb480m_gpll_p, 0, ++ RV1126_CLKSEL_CON(61), 7, 1, MFLAGS, 0, 5, DFLAGS, ++ RV1126_CLKGATE_CON(19), 6, GFLAGS), ++#endif ++ GATE(ACLK_USBOTG, "aclk_usbotg", "aclk_pdusb", 0, ++ RV1126_CLKGATE_CON(19), 7, GFLAGS), ++ GATE(CLK_USBOTG_REF, "clk_usbotg_ref", "xin24m", 0, ++ RV1126_CLKGATE_CON(19), 8, GFLAGS), ++ /* PD_GMAC */ ++ GATE(ACLK_PDGMAC, "aclk_pdgmac", "aclk_pdphp", 0, ++ RV1126_CLKGATE_CON(20), 0, GFLAGS), ++ COMPOSITE_NOMUX(PCLK_PDGMAC, "pclk_pdgmac", "aclk_pdgmac", 0, ++ RV1126_CLKSEL_CON(63), 8, 5, DFLAGS, ++ RV1126_CLKGATE_CON(20), 1, GFLAGS), ++ GATE(ACLK_GMAC, "aclk_gmac", "aclk_pdgmac", 0, ++ RV1126_CLKGATE_CON(20), 4, GFLAGS), ++ GATE(PCLK_GMAC, "pclk_gmac", "pclk_pdgmac", 0, ++ RV1126_CLKGATE_CON(20), 5, GFLAGS), ++ ++ COMPOSITE(CLK_GMAC_DIV, "clk_gmac_div", mux_cpll_gpll_p, 0, ++ RV1126_CLKSEL_CON(63), 7, 1, MFLAGS, 0, 5, DFLAGS, ++ RV1126_CLKGATE_CON(20), 6, GFLAGS), ++ GATE(CLK_GMAC_RGMII_M0, "clk_gmac_rgmii_m0", "clk_gmac_rgmii_clkin_m0", 0, ++ RV1126_CLKGATE_CON(20), 12, GFLAGS), ++ MUX(CLK_GMAC_SRC_M0, "clk_gmac_src_m0", clk_gmac_src_m0_p, CLK_SET_RATE_PARENT, ++ RV1126_GMAC_CON, 0, 1, MFLAGS), ++ GATE(CLK_GMAC_RGMII_M1, "clk_gmac_rgmii_m1", "clk_gmac_rgmii_clkin_m1", 0, ++ RV1126_CLKGATE_CON(20), 13, GFLAGS), ++ MUX(CLK_GMAC_SRC_M1, "clk_gmac_src_m1", clk_gmac_src_m1_p, CLK_SET_RATE_PARENT, ++ RV1126_GMAC_CON, 5, 1, MFLAGS), ++ MUXGRF(CLK_GMAC_SRC, "clk_gmac_src", mux_clk_gmac_src_p, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, ++ RV1126_GRF_IOFUNC_CON1, 12, 1, MFLAGS), ++ ++ GATE(CLK_GMAC_REF, "clk_gmac_ref", "clk_gmac_src", 0, ++ RV1126_CLKGATE_CON(20), 7, GFLAGS), ++ ++ GATE(CLK_GMAC_TX_SRC, "clk_gmac_tx_src", "clk_gmac_src", 0, ++ RV1126_CLKGATE_CON(20), 9, GFLAGS), ++ FACTOR(CLK_GMAC_TX_DIV5, "clk_gmac_tx_div5", "clk_gmac_tx_src", 0, 1, 5), ++ FACTOR(CLK_GMAC_TX_DIV50, "clk_gmac_tx_div50", "clk_gmac_tx_src", 0, 1, 50), ++ MUXTBL(RGMII_MODE_CLK, "rgmii_mode_clk", mux_rgmii_clk_p, CLK_SET_RATE_PARENT, ++ RV1126_GMAC_CON, 2, 2, MFLAGS, rgmii_mux_idx), ++ GATE(CLK_GMAC_RX_SRC, "clk_gmac_rx_src", "clk_gmac_src", 0, ++ RV1126_CLKGATE_CON(20), 8, GFLAGS), ++ FACTOR(CLK_GMAC_RX_DIV2, "clk_gmac_rx_div2", "clk_gmac_rx_src", 0, 1, 2), ++ FACTOR(CLK_GMAC_RX_DIV20, "clk_gmac_rx_div20", "clk_gmac_rx_src", 0, 1, 20), ++ MUX(RMII_MODE_CLK, "rmii_mode_clk", mux_rmii_clk_p, CLK_SET_RATE_PARENT, ++ RV1126_GMAC_CON, 1, 1, MFLAGS), ++ MUX(CLK_GMAC_TX_RX, "clk_gmac_tx_rx", mux_gmac_tx_rx_p, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, ++ RV1126_GMAC_CON, 4, 1, MFLAGS), ++ ++ GATE(CLK_GMAC_PTPREF, "clk_gmac_ptpref", "xin24m", 0, ++ RV1126_CLKGATE_CON(20), 10, GFLAGS), ++ COMPOSITE(CLK_GMAC_ETHERNET_OUT, "clk_gmac_ethernet_out2io", mux_cpll_gpll_p, 0, ++ RV1126_CLKSEL_CON(61), 15, 1, MFLAGS, 8, 5, DFLAGS, ++ RV1126_CLKGATE_CON(20), 11, GFLAGS), ++ ++ ++ /* ++ * Clock-Architecture Diagram 14 ++ */ ++ /* PD_NPU */ ++ COMPOSITE_BROTHER(ACLK_PDNPU_DIV, "aclk_pdnpu_div", mux_gpll_cpll_apll_hpll_p, 0, ++ RV1126_CLKSEL_CON(65), 8, 2, MFLAGS, 0, 4, DFLAGS, ++ RV1126_CLKGATE_CON(22), 0, GFLAGS, ++ &rv1126_aclk_pdnpu_npu5), ++ MUX(ACLK_PDNPU, "aclk_pdnpu", mux_aclk_pdnpu_p, CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE, ++ RV1126_CLKSEL_CON(65), 12, 1, MFLAGS), ++ COMPOSITE_NOMUX(HCLK_PDNPU, "hclk_pdnpu", "gpll", 0, ++ RV1126_CLKSEL_CON(66), 8, 4, DFLAGS, ++ RV1126_CLKGATE_CON(22), 2, GFLAGS), ++ COMPOSITE_NOMUX(PCLK_PDNPU, "pclk_pdnpu", "hclk_pdnpu", 0, ++ RV1126_CLKSEL_CON(66), 0, 5, DFLAGS, ++ RV1126_CLKGATE_CON(22), 3, GFLAGS), ++ GATE(ACLK_NPU, "aclk_npu", "aclk_pdnpu", 0, ++ RV1126_CLKGATE_CON(22), 7, GFLAGS), ++ GATE(HCLK_NPU, "hclk_npu", "hclk_pdnpu", 0, ++ RV1126_CLKGATE_CON(22), 8, GFLAGS), ++ COMPOSITE_BROTHER(CLK_NPU_DIV, "clk_npu_div", mux_gpll_cpll_apll_hpll_p, 0, ++ RV1126_CLKSEL_CON(67), 8, 2, MFLAGS, 0, 4, DFLAGS, ++ RV1126_CLKGATE_CON(22), 9, GFLAGS, ++ &rv1126_clk_npu_np5), ++ MUX(CLK_CORE_NPU, "clk_core_npu", mux_clk_npu_p, CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE, ++ RV1126_CLKSEL_CON(67), 12, 1, MFLAGS), ++ GATE(CLK_CORE_NPUPVTM, "clk_core_npupvtm", "clk_core_npu", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(22), 14, GFLAGS), ++ GATE(CLK_NPUPVTM, "clk_npupvtm", "xin24m", 0, ++ RV1126_CLKGATE_CON(22), 13, GFLAGS), ++ GATE(PCLK_NPUPVTM, "pclk_npupvtm", "pclk_pdnpu", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(22), 12, GFLAGS), ++ ++ /* ++ * Clock-Architecture Diagram 15 ++ */ ++ GATE(PCLK_PDTOP, "pclk_pdtop", "pclk_pdbus", CLK_IS_CRITICAL, ++ RV1126_CLKGATE_CON(23), 8, GFLAGS), ++ GATE(PCLK_DSIPHY, "pclk_dsiphy", "pclk_pdtop", 0, ++ RV1126_CLKGATE_CON(23), 4, GFLAGS), ++ GATE(PCLK_CSIPHY0, "pclk_csiphy0", "pclk_pdtop", 0, ++ RV1126_CLKGATE_CON(23), 2, GFLAGS), ++ GATE(PCLK_CSIPHY1, "pclk_csiphy1", "pclk_pdtop", 0, ++ RV1126_CLKGATE_CON(23), 3, GFLAGS), ++ GATE(PCLK_USBPHY_HOST, "pclk_usbphy_host", "pclk_pdtop", 0, ++ RV1126_CLKGATE_CON(19), 13, GFLAGS), ++ GATE(PCLK_USBPHY_OTG, "pclk_usbphy_otg", "pclk_pdtop", 0, ++ RV1126_CLKGATE_CON(19), 12, GFLAGS), ++ ++#ifndef CONFIG_ROCKCHIP_LOW_PERFORMANCE ++ /* ++ * Clock-Architecture Diagram 3 ++ */ ++ /* PD_CORE */ ++ COMPOSITE_NOMUX(0, "aclk_core", "armclk", CLK_IS_CRITICAL, ++ RV1126_CLKSEL_CON(1), 4, 4, DFLAGS | CLK_DIVIDER_READ_ONLY, ++ RV1126_CLKGATE_CON(0), 2, GFLAGS), ++ GATE(0, "pclk_dbg_daplite", "pclk_dbg", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(0), 5, GFLAGS), ++ GATE(0, "clk_a7_jtag", "clk_jtag_ori", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(0), 9, GFLAGS), ++ GATE(0, "aclk_core_niu", "aclk_core", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(0), 3, GFLAGS), ++ GATE(0, "pclk_dbg_niu", "pclk_dbg", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(0), 4, GFLAGS), ++ /* ++ * Clock-Architecture Diagram 4 ++ */ ++ /* PD_BUS */ ++ GATE(0, "aclk_pdbus_hold_niu1", "aclk_pdbus", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(2), 10, GFLAGS), ++ GATE(0, "aclk_pdbus_niu1", "aclk_pdbus", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(2), 3, GFLAGS), ++ GATE(0, "hclk_pdbus_niu1", "hclk_pdbus", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(2), 4, GFLAGS), ++ GATE(0, "pclk_pdbus_niu1", "pclk_pdbus", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(2), 5, GFLAGS), ++ GATE(0, "aclk_pdbus_niu2", "aclk_pdbus", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(2), 6, GFLAGS), ++ GATE(0, "hclk_pdbus_niu2", "hclk_pdbus", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(2), 7, GFLAGS), ++ GATE(0, "aclk_pdbus_niu3", "aclk_pdbus", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(2), 8, GFLAGS), ++ GATE(0, "hclk_pdbus_niu3", "hclk_pdbus", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(2), 9, GFLAGS), ++ GATE(0, "pclk_grf", "pclk_pdbus", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(6), 15, GFLAGS), ++ GATE(0, "pclk_sgrf", "pclk_pdbus", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(8), 4, GFLAGS), ++ GATE(0, "aclk_sysram", "hclk_pdbus", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(3), 9, GFLAGS), ++ GATE(0, "pclk_intmux", "pclk_pdbus", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(7), 14, GFLAGS), ++ ++ /* ++ * Clock-Architecture Diagram 5 ++ */ ++ /* PD_CRYPTO */ ++ GATE(0, "aclk_pdcrypto_niu", "aclk_pdcrypto", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(4), 13, GFLAGS), ++ GATE(0, "hclk_pdcrypto_niu", "hclk_pdcrypto", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(4), 14, GFLAGS), ++ ++ /* ++ * Clock-Architecture Diagram 6 ++ */ ++ /* PD_AUDIO */ ++ GATE(0, "hclk_pdaudio_niu", "hclk_pdaudio", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(9), 2, GFLAGS), ++ GATE(0, "pclk_pdaudio_niu", "hclk_pdaudio", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(9), 3, GFLAGS), ++ ++ /* ++ * Clock-Architecture Diagram 7 ++ */ ++ /* PD_VEPU */ ++ GATE(0, "aclk_pdvepu_niu", "aclk_pdvepu", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(12), 3, GFLAGS), ++ GATE(0, "hclk_pdvepu_niu", "hclk_pdvepu", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(12), 4, GFLAGS), ++ ++ /* ++ * Clock-Architecture Diagram 9 ++ */ ++ /* PD_VO */ ++ GATE(0, "aclk_pdvo_niu", "aclk_pdvo", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(14), 3, GFLAGS), ++ GATE(0, "hclk_pdvo_niu", "hclk_pdvo", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(14), 4, GFLAGS), ++ GATE(0, "pclk_pdvo_niu", "pclk_pdvo", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(14), 5, GFLAGS), ++ ++ /* ++ * Clock-Architecture Diagram 10 ++ */ ++ /* PD_VI */ ++ GATE(0, "aclk_pdvi_niu", "aclk_pdvi", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(15), 3, GFLAGS), ++ GATE(0, "hclk_pdvi_niu", "hclk_pdvi", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(15), 4, GFLAGS), ++ GATE(0, "pclk_pdvi_niu", "pclk_pdvi", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(15), 5, GFLAGS), ++ /* ++ * Clock-Architecture Diagram 11 ++ */ ++ /* PD_ISPP */ ++ GATE(0, "aclk_pdispp_niu", "aclk_pdispp", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(16), 2, GFLAGS), ++ GATE(0, "hclk_pdispp_niu", "hclk_pdispp", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(16), 3, GFLAGS), ++ ++ /* ++ * Clock-Architecture Diagram 12 ++ */ ++ /* PD_PHP */ ++ GATE(0, "aclk_pdphpmid", "aclk_pdphp", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(17), 2, GFLAGS), ++ GATE(0, "hclk_pdphpmid", "hclk_pdphp", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(17), 3, GFLAGS), ++ GATE(0, "aclk_pdphpmid_niu", "aclk_pdphpmid", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(17), 4, GFLAGS), ++ GATE(0, "hclk_pdphpmid_niu", "hclk_pdphpmid", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(17), 5, GFLAGS), ++ ++ /* PD_SDCARD */ ++ GATE(0, "hclk_pdsdmmc_niu", "hclk_pdsdmmc", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(17), 7, GFLAGS), ++ ++ /* PD_SDIO */ ++ GATE(0, "hclk_pdsdio_niu", "hclk_pdsdio", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(17), 9, GFLAGS), ++ ++ /* PD_NVM */ ++ GATE(0, "hclk_pdnvm_niu", "hclk_pdnvm", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(18), 3, GFLAGS), ++ ++ /* PD_USB */ ++ GATE(0, "aclk_pdusb_niu", "aclk_pdusb", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(19), 2, GFLAGS), ++ GATE(0, "hclk_pdusb_niu", "hclk_pdusb", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(19), 3, GFLAGS), ++ ++ /* PD_GMAC */ ++ GATE(0, "aclk_pdgmac_niu", "aclk_pdgmac", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(20), 2, GFLAGS), ++ GATE(0, "pclk_pdgmac_niu", "pclk_pdgmac", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(20), 3, GFLAGS), ++ ++ /* ++ * Clock-Architecture Diagram 13 ++ */ ++ /* PD_DDR */ ++ COMPOSITE_NOMUX(0, "pclk_pdddr_pre", "gpll", CLK_IS_CRITICAL, ++ RV1126_CLKSEL_CON(64), 0, 5, DFLAGS, ++ RV1126_CLKGATE_CON(21), 0, GFLAGS), ++ GATE(PCLK_PDDDR, "pclk_pdddr", "pclk_pdddr_pre", CLK_IS_CRITICAL, ++ RV1126_CLKGATE_CON(21), 15, GFLAGS), ++ GATE(0, "pclk_ddr_msch", "pclk_pdddr", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(21), 6, GFLAGS), ++ COMPOSITE_NOGATE(SCLK_DDRCLK, "sclk_ddrc", mux_dpll_gpll_p, CLK_IS_CRITICAL, ++ RV1126_CLKSEL_CON(64), 15, 1, MFLAGS, 8, 5, DFLAGS), ++ COMPOSITE(CLK_DDRPHY, "clk_ddrphy", mux_dpll_gpll_p, CLK_IS_CRITICAL, ++ RV1126_CLKSEL_CON(64), 15, 1, MFLAGS, 8, 5, DFLAGS, ++ RV1126_CLKGATE_CON(21), 8, GFLAGS), ++ GATE(0, "clk1x_phy", "clk_ddrphy", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(23), 1, GFLAGS), ++ GATE(0, "clk_ddr_msch", "clk_ddrphy", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(21), 10, GFLAGS), ++ GATE(0, "pclk_ddr_dfictl", "pclk_pdddr", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(21), 2, GFLAGS), ++ GATE(0, "clk_ddr_dfictl", "clk_ddrphy", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(21), 13, GFLAGS), ++ GATE(0, "pclk_ddr_standby", "pclk_pdddr", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(21), 4, GFLAGS), ++ GATE(0, "clk_ddr_standby", "clk_ddrphy", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(21), 14, GFLAGS), ++ GATE(0, "aclk_ddr_split", "clk_ddrphy", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(21), 9, GFLAGS), ++ GATE(0, "pclk_ddr_grf", "pclk_pdddr", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(21), 5, GFLAGS), ++ GATE(PCLK_DDR_MON, "pclk_ddr_mon", "pclk_pdddr", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(21), 3, GFLAGS), ++ GATE(CLK_DDR_MON, "clk_ddr_mon", "clk_ddrphy", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(20), 15, GFLAGS), ++ GATE(TMCLK_DDR_MON, "tmclk_ddr_mon", "xin24m", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(21), 7, GFLAGS), ++ ++ /* ++ * Clock-Architecture Diagram 14 ++ */ ++ /* PD_NPU */ ++ GATE(0, "aclk_pdnpu_niu", "aclk_pdnpu", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(22), 4, GFLAGS), ++ GATE(0, "hclk_pdnpu_niu", "hclk_pdnpu", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(22), 5, GFLAGS), ++ GATE(0, "pclk_pdnpu_niu", "pclk_pdnpu", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(22), 6, GFLAGS), ++ ++ /* ++ * Clock-Architecture Diagram 15 ++ */ ++ GATE(0, "pclk_topniu", "pclk_pdtop", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(23), 9, GFLAGS), ++ GATE(PCLK_TOPCRU, "pclk_topcru", "pclk_pdtop", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(23), 10, GFLAGS), ++ GATE(PCLK_TOPGRF, "pclk_topgrf", "pclk_pdtop", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(23), 11, GFLAGS), ++ GATE(PCLK_CPUEMADET, "pclk_cpuemadet", "pclk_pdtop", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(23), 12, GFLAGS), ++ GATE(PCLK_DDRPHY, "pclk_ddrphy", "pclk_pdtop", CLK_IGNORE_UNUSED, ++ RV1126_CLKGATE_CON(23), 0, GFLAGS), ++#endif ++}; ++ ++static void __iomem *rv1126_cru_base; ++static void __iomem *rv1126_pmucru_base; ++ ++void rv1126_dump_cru(void) ++{ ++ if (rv1126_pmucru_base) { ++ pr_warn("PMU CRU:\n"); ++ print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET, ++ 32, 4, rv1126_pmucru_base, ++ 0x248, false); ++ } ++ if (rv1126_cru_base) { ++ pr_warn("CRU:\n"); ++ print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET, ++ 32, 4, rv1126_cru_base, ++ 0x588, false); ++ } ++} ++EXPORT_SYMBOL_GPL(rv1126_dump_cru); ++ ++static int rv1126_clk_panic(struct notifier_block *this, ++ unsigned long ev, void *ptr) ++{ ++ rv1126_dump_cru(); ++ return NOTIFY_DONE; ++} ++ ++static struct notifier_block rv1126_clk_panic_block = { ++ .notifier_call = rv1126_clk_panic, ++}; ++ ++static struct rockchip_clk_provider *pmucru_ctx; ++static void __init rv1126_pmu_clk_init(struct device_node *np) ++{ ++ struct rockchip_clk_provider *ctx; ++ void __iomem *reg_base; ++ ++ reg_base = of_iomap(np, 0); ++ if (!reg_base) { ++ pr_err("%s: could not map cru pmu region\n", __func__); ++ return; ++ } ++ ++ rv1126_pmucru_base = reg_base; ++ ++ ctx = rockchip_clk_init(np, reg_base, CLKPMU_NR_CLKS); ++ if (IS_ERR(ctx)) { ++ pr_err("%s: rockchip pmu clk init failed\n", __func__); ++ return; ++ } ++ ++ rockchip_clk_register_plls(ctx, rv1126_pmu_pll_clks, ++ ARRAY_SIZE(rv1126_pmu_pll_clks), ++ RV1126_GRF_SOC_STATUS0); ++ ++ rockchip_clk_register_branches(ctx, rv1126_clk_pmu_branches, ++ ARRAY_SIZE(rv1126_clk_pmu_branches)); ++ ++ rockchip_register_softrst(np, 2, reg_base + RV1126_PMU_SOFTRST_CON(0), ++ ROCKCHIP_SOFTRST_HIWORD_MASK); ++ ++ rockchip_clk_of_add_provider(np, ctx); ++ ++ pmucru_ctx = ctx; ++} ++ ++CLK_OF_DECLARE(rv1126_cru_pmu, "rockchip,rv1126-pmucru", rv1126_pmu_clk_init); ++ ++static void __init rv1126_clk_init(struct device_node *np) ++{ ++ struct rockchip_clk_provider *ctx; ++ void __iomem *reg_base; ++ struct clk **cru_clks, **pmucru_clks; ++ ++ reg_base = of_iomap(np, 0); ++ if (!reg_base) { ++ pr_err("%s: could not map cru region\n", __func__); ++ return; ++ } ++ ++ rv1126_cru_base = reg_base; ++ ++ ctx = rockchip_clk_init(np, reg_base, CLK_NR_CLKS); ++ if (IS_ERR(ctx)) { ++ pr_err("%s: rockchip clk init failed\n", __func__); ++ iounmap(reg_base); ++ return; ++ } ++ cru_clks = ctx->clk_data.clks; ++ pmucru_clks = pmucru_ctx->clk_data.clks; ++ ++ rockchip_clk_register_plls(ctx, rv1126_pll_clks, ++ ARRAY_SIZE(rv1126_pll_clks), ++ RV1126_GRF_SOC_STATUS0); ++ ++ rockchip_clk_register_armclk(ctx, ARMCLK, "armclk", ++ 3, cru_clks[PLL_APLL], pmucru_clks[PLL_GPLL], ++ &rv1126_cpuclk_data, rv1126_cpuclk_rates, ++ ARRAY_SIZE(rv1126_cpuclk_rates)); ++ ++ rockchip_clk_register_branches(ctx, rv1126_clk_branches, ++ ARRAY_SIZE(rv1126_clk_branches)); ++ ++ rockchip_register_softrst(np, 15, reg_base + RV1126_SOFTRST_CON(0), ++ ROCKCHIP_SOFTRST_HIWORD_MASK); ++ ++ rockchip_register_restart_notifier(ctx, RV1126_GLB_SRST_FST, NULL); ++ ++ rockchip_clk_of_add_provider(np, ctx); ++ ++ atomic_notifier_chain_register(&panic_notifier_list, ++ &rv1126_clk_panic_block); ++} ++ ++CLK_OF_DECLARE(rv1126_cru, "rockchip,rv1126-cru", rv1126_clk_init); ++ ++struct clk_rv1126_inits { ++ void (*inits)(struct device_node *np); ++}; ++ ++static const struct clk_rv1126_inits clk_rv1126_pmu_init = { ++ .inits = rv1126_pmu_clk_init, ++}; ++ ++static const struct clk_rv1126_inits clk_rv1126_init = { ++ .inits = rv1126_clk_init, ++}; ++ ++static const struct of_device_id clk_rv1126_match_table[] = { ++ { ++ .compatible = "rockchip,rv1126-cru", ++ .data = &clk_rv1126_init, ++ }, { ++ .compatible = "rockchip,rv1126-pmucru", ++ .data = &clk_rv1126_pmu_init, ++ }, ++ { } ++}; ++MODULE_DEVICE_TABLE(of, clk_rv1126_match_table); ++ ++static int __init clk_rv1126_probe(struct platform_device *pdev) ++{ ++ struct device_node *np = pdev->dev.of_node; ++ const struct of_device_id *match; ++ const struct clk_rv1126_inits *init_data; ++ ++ match = of_match_device(clk_rv1126_match_table, &pdev->dev); ++ if (!match || !match->data) ++ return -EINVAL; ++ ++ init_data = match->data; ++ if (init_data->inits) ++ init_data->inits(np); ++ ++ return 0; ++} ++ ++static struct platform_driver clk_rv1126_driver = { ++ .driver = { ++ .name = "clk-rv1126", ++ .of_match_table = clk_rv1126_match_table, ++ }, ++}; ++builtin_platform_driver_probe(clk_rv1126_driver, clk_rv1126_probe); ++ ++MODULE_DESCRIPTION("Rockchip RV1126 Clock Driver"); ++MODULE_LICENSE("GPL"); +diff --git a/drivers/clk/rockchip/clk.c b/drivers/clk/rockchip/clk.c +index b443169dd408..6c8e47067032 100644 +--- a/drivers/clk/rockchip/clk.c ++++ b/drivers/clk/rockchip/clk.c +@@ -38,6 +38,7 @@ static struct clk *rockchip_clk_register_branch(const char *name, + const char *const *parent_names, u8 num_parents, + void __iomem *base, + int muxdiv_offset, u8 mux_shift, u8 mux_width, u8 mux_flags, ++ u32 *mux_table, + int div_offset, u8 div_shift, u8 div_width, u8 div_flags, + struct clk_div_table *div_table, int gate_offset, + u8 gate_shift, u8 gate_flags, unsigned long flags, +@@ -60,6 +61,7 @@ static struct clk *rockchip_clk_register_branch(const char *name, + mux->shift = mux_shift; + mux->mask = BIT(mux_width) - 1; + mux->flags = mux_flags; ++ mux->table = mux_table; + mux->lock = lock; + mux_ops = (mux_flags & CLK_MUX_READ_ONLY) ? &clk_mux_ro_ops + : &clk_mux_ops; +@@ -182,12 +184,43 @@ static void rockchip_fractional_approximation(struct clk_hw *hw, + unsigned long p_rate, p_parent_rate; + struct clk_hw *p_parent; + unsigned long scale; ++ u32 div; + + p_rate = clk_hw_get_rate(clk_hw_get_parent(hw)); +- if ((rate * 20 > p_rate) && (p_rate % rate != 0)) { ++ if (((rate * 20 > p_rate) && (p_rate % rate != 0)) || ++ (fd->max_prate && fd->max_prate < p_rate)) { + p_parent = clk_hw_get_parent(clk_hw_get_parent(hw)); +- p_parent_rate = clk_hw_get_rate(p_parent); +- *parent_rate = p_parent_rate; ++ if (!p_parent) { ++ *parent_rate = p_rate; ++ } else { ++ p_parent_rate = clk_hw_get_rate(p_parent); ++ *parent_rate = p_parent_rate; ++ if (fd->max_prate && p_parent_rate > fd->max_prate) { ++ div = DIV_ROUND_UP(p_parent_rate, ++ fd->max_prate); ++ *parent_rate = p_parent_rate / div; ++ } ++ } ++ ++ if (*parent_rate < rate * 20) { ++ /* ++ * Fractional frequency divider to do ++ * integer frequency divider does not ++ * need 20 times the limit. ++ */ ++ if (!(*parent_rate % rate)) { ++ *m = 1; ++ *n = *parent_rate / rate; ++ return; ++ } else if (!(fd->flags & CLK_FRAC_DIVIDER_NO_LIMIT)) { ++ pr_warn("%s p_rate(%ld) is low than rate(%ld)*20, use integer or half-div\n", ++ clk_hw_get_name(hw), ++ *parent_rate, rate); ++ *m = 0; ++ *n = 1; ++ return; ++ } ++ } + } + + /* +@@ -210,7 +243,7 @@ static struct clk *rockchip_clk_register_frac_branch( + void __iomem *base, int muxdiv_offset, u8 div_flags, + int gate_offset, u8 gate_shift, u8 gate_flags, + unsigned long flags, struct rockchip_clk_branch *child, +- spinlock_t *lock) ++ unsigned long max_prate, spinlock_t *lock) + { + struct clk_hw *hw; + struct rockchip_clk_frac *frac; +@@ -251,6 +284,7 @@ static struct clk *rockchip_clk_register_frac_branch( + div->nmask = GENMASK(div->nwidth - 1, 0) << div->nshift; + div->lock = lock; + div->approximation = rockchip_fractional_approximation; ++ div->max_prate = max_prate; + div_ops = &clk_fractional_divider_ops; + + hw = clk_hw_register_composite(NULL, name, parent_names, num_parents, +@@ -278,6 +312,8 @@ static struct clk *rockchip_clk_register_frac_branch( + frac_mux->shift = child->mux_shift; + frac_mux->mask = BIT(child->mux_width) - 1; + frac_mux->flags = child->mux_flags; ++ if (child->mux_table) ++ frac_mux->table = child->mux_table; + frac_mux->lock = lock; + frac_mux->hw.init = &init; + +@@ -360,6 +396,61 @@ static struct clk *rockchip_clk_register_factor_branch(const char *name, + return hw->clk; + } + ++static struct clk *rockchip_clk_register_composite_brother_branch( ++ struct rockchip_clk_provider *ctx, const char *name, ++ const char *const *parent_names, u8 num_parents, ++ void __iomem *base, int muxdiv_offset, u8 mux_shift, ++ u8 mux_width, u8 mux_flags, u32 *mux_table, ++ int div_offset, u8 div_shift, u8 div_width, u8 div_flags, ++ struct clk_div_table *div_table, int gate_offset, ++ u8 gate_shift, u8 gate_flags, unsigned long flags, ++ struct rockchip_clk_branch *brother, spinlock_t *lock) ++{ ++ struct clk *clk, *brother_clk; ++ struct clk_composite *composite, *brother_composite; ++ struct clk_hw *hw, *brother_hw; ++ ++ if (brother && brother->branch_type != branch_half_divider) { ++ pr_err("%s: composite brother for %s can only be a halfdiv\n", ++ __func__, name); ++ return ERR_PTR(-EINVAL); ++ } ++ ++ clk = rockchip_clk_register_branch(name, parent_names, num_parents, ++ base, muxdiv_offset, mux_shift, ++ mux_width, mux_flags, mux_table, ++ div_offset, div_shift, div_width, ++ div_flags, div_table, ++ gate_offset, gate_shift, gate_flags, ++ flags, lock); ++ if (IS_ERR(clk)) ++ return clk; ++ ++ brother_clk = rockchip_clk_register_halfdiv(brother->name, ++ brother->parent_names, brother->num_parents, ++ base, brother->muxdiv_offset, ++ brother->mux_shift, brother->mux_width, ++ brother->mux_flags, brother->div_offset, ++ brother->div_shift, brother->div_width, ++ brother->div_flags, brother->gate_offset, ++ brother->gate_shift, brother->gate_flags, ++ flags, lock); ++ if (IS_ERR(brother_clk)) ++ return brother_clk; ++ rockchip_clk_add_lookup(ctx, brother_clk, brother->id); ++ ++ hw = __clk_get_hw(clk); ++ brother_hw = __clk_get_hw(brother_clk); ++ if (hw && brother_hw) { ++ composite = to_clk_composite(hw); ++ brother_composite = to_clk_composite(brother_hw); ++ composite->brother_hw = brother_hw; ++ brother_composite->brother_hw = hw; ++ } ++ ++ return clk; ++} ++ + struct rockchip_clk_provider *rockchip_clk_init(struct device_node *np, + void __iomem *base, + unsigned long nr_clks) +@@ -387,6 +478,8 @@ struct rockchip_clk_provider *rockchip_clk_init(struct device_node *np, + + ctx->grf = syscon_regmap_lookup_by_phandle(ctx->cru_node, + "rockchip,grf"); ++ ctx->pmugrf = syscon_regmap_lookup_by_phandle(ctx->cru_node, ++ "rockchip,pmugrf"); + + return ctx; + +@@ -452,11 +545,22 @@ void rockchip_clk_register_branches(struct rockchip_clk_provider *ctx, + /* catch simple muxes */ + switch (list->branch_type) { + case branch_mux: +- clk = clk_register_mux(NULL, list->name, +- list->parent_names, list->num_parents, +- flags, ctx->reg_base + list->muxdiv_offset, +- list->mux_shift, list->mux_width, +- list->mux_flags, &ctx->lock); ++ if (list->mux_table) ++ clk = clk_register_mux_table(NULL, list->name, ++ list->parent_names, list->num_parents, ++ flags, ++ ctx->reg_base + list->muxdiv_offset, ++ list->mux_shift, ++ BIT(list->mux_width) - 1, ++ list->mux_flags, list->mux_table, ++ &ctx->lock); ++ else ++ clk = clk_register_mux(NULL, list->name, ++ list->parent_names, list->num_parents, ++ flags, ++ ctx->reg_base + list->muxdiv_offset, ++ list->mux_shift, list->mux_width, ++ list->mux_flags, &ctx->lock); + break; + case branch_muxgrf: + clk = rockchip_clk_register_muxgrf(list->name, +@@ -465,6 +569,13 @@ void rockchip_clk_register_branches(struct rockchip_clk_provider *ctx, + list->mux_shift, list->mux_width, + list->mux_flags); + break; ++ case branch_muxpmugrf: ++ clk = rockchip_clk_register_muxgrf(list->name, ++ list->parent_names, list->num_parents, ++ flags, ctx->pmugrf, list->muxdiv_offset, ++ list->mux_shift, list->mux_width, ++ list->mux_flags); ++ break; + case branch_divider: + if (list->div_table) + clk = clk_register_divider_table(NULL, +@@ -488,17 +599,18 @@ void rockchip_clk_register_branches(struct rockchip_clk_provider *ctx, + list->div_flags, + list->gate_offset, list->gate_shift, + list->gate_flags, flags, list->child, +- &ctx->lock); ++ list->max_prate, &ctx->lock); + break; + case branch_half_divider: + clk = rockchip_clk_register_halfdiv(list->name, + list->parent_names, list->num_parents, + ctx->reg_base, list->muxdiv_offset, + list->mux_shift, list->mux_width, +- list->mux_flags, list->div_shift, +- list->div_width, list->div_flags, +- list->gate_offset, list->gate_shift, +- list->gate_flags, flags, &ctx->lock); ++ list->mux_flags, list->div_offset, ++ list->div_shift, list->div_width, ++ list->div_flags, list->gate_offset, ++ list->gate_shift, list->gate_flags, ++ flags, &ctx->lock); + break; + case branch_gate: + flags |= CLK_SET_RATE_PARENT; +@@ -514,11 +626,25 @@ void rockchip_clk_register_branches(struct rockchip_clk_provider *ctx, + ctx->reg_base, list->muxdiv_offset, + list->mux_shift, + list->mux_width, list->mux_flags, +- list->div_offset, list->div_shift, list->div_width, ++ list->mux_table, list->div_offset, ++ list->div_shift, list->div_width, + list->div_flags, list->div_table, + list->gate_offset, list->gate_shift, + list->gate_flags, flags, &ctx->lock); + break; ++ case branch_composite_brother: ++ clk = rockchip_clk_register_composite_brother_branch( ++ ctx, list->name, list->parent_names, ++ list->num_parents, ctx->reg_base, ++ list->muxdiv_offset, list->mux_shift, ++ list->mux_width, list->mux_flags, ++ list->mux_table, list->div_offset, ++ list->div_shift, list->div_width, ++ list->div_flags, list->div_table, ++ list->gate_offset, list->gate_shift, ++ list->gate_flags, flags, list->child, ++ &ctx->lock); ++ break; + case branch_mmc: + clk = rockchip_clk_register_mmc( + list->name, +@@ -549,7 +675,17 @@ void rockchip_clk_register_branches(struct rockchip_clk_provider *ctx, + list->muxdiv_offset, list->mux_shift, + list->mux_width, list->div_shift, + list->div_width, list->div_flags, +- ctx->reg_base, &ctx->lock); ++ ctx->reg_base); ++ break; ++ case branch_dclk_divider: ++ clk = rockchip_clk_register_dclk_branch(list->name, ++ list->parent_names, list->num_parents, ++ ctx->reg_base, list->muxdiv_offset, list->mux_shift, ++ list->mux_width, list->mux_flags, ++ list->div_offset, list->div_shift, list->div_width, ++ list->div_flags, list->div_table, ++ list->gate_offset, list->gate_shift, ++ list->gate_flags, flags, list->max_prate, &ctx->lock); + break; + } + +@@ -573,15 +709,17 @@ EXPORT_SYMBOL_GPL(rockchip_clk_register_branches); + + void rockchip_clk_register_armclk(struct rockchip_clk_provider *ctx, + unsigned int lookup_id, +- const char *name, const char *const *parent_names, ++ const char *name, + u8 num_parents, ++ struct clk *parent, struct clk *alt_parent, + const struct rockchip_cpuclk_reg_data *reg_data, + const struct rockchip_cpuclk_rate_table *rates, + int nrates) + { + struct clk *clk; + +- clk = rockchip_clk_register_cpuclk(name, parent_names, num_parents, ++ clk = rockchip_clk_register_cpuclk(name, num_parents, ++ parent, alt_parent, + reg_data, rates, nrates, + ctx->reg_base, &ctx->lock); + if (IS_ERR(clk)) { +@@ -594,20 +732,20 @@ void rockchip_clk_register_armclk(struct rockchip_clk_provider *ctx, + } + EXPORT_SYMBOL_GPL(rockchip_clk_register_armclk); + +-void rockchip_clk_protect_critical(const char *const clocks[], +- int nclocks) +-{ +- int i; +- +- /* Protect the clocks that needs to stay on */ +- for (i = 0; i < nclocks; i++) { +- struct clk *clk = __clk_lookup(clocks[i]); ++void (*rk_dump_cru)(void); ++EXPORT_SYMBOL(rk_dump_cru); + +- if (clk) +- clk_prepare_enable(clk); +- } ++static int rk_clk_panic(struct notifier_block *this, ++ unsigned long ev, void *ptr) ++{ ++ if (rk_dump_cru) ++ rk_dump_cru(); ++ return NOTIFY_DONE; + } +-EXPORT_SYMBOL_GPL(rockchip_clk_protect_critical); ++ ++static struct notifier_block rk_clk_panic_block = { ++ .notifier_call = rk_clk_panic, ++}; + + static void __iomem *rst_base; + static unsigned int reg_restart; +@@ -641,5 +779,7 @@ rockchip_register_restart_notifier(struct rockchip_clk_provider *ctx, + if (ret) + pr_err("%s: cannot register restart handler, %d\n", + __func__, ret); ++ atomic_notifier_chain_register(&panic_notifier_list, ++ &rk_clk_panic_block); + } + EXPORT_SYMBOL_GPL(rockchip_register_restart_notifier); +diff --git a/drivers/clk/rockchip/clk.h b/drivers/clk/rockchip/clk.h +index 2271a84124b0..509087750eeb 100644 +--- a/drivers/clk/rockchip/clk.h ++++ b/drivers/clk/rockchip/clk.h +@@ -37,12 +37,25 @@ struct clk; + #define BOOST_SWITCH_THRESHOLD 0x0024 + #define BOOST_FSM_STATUS 0x0028 + #define BOOST_PLL_L_CON(x) ((x) * 0x4 + 0x2c) ++#define BOOST_PLL_CON_MASK 0xffff ++#define BOOST_CORE_DIV_MASK 0x1f ++#define BOOST_CORE_DIV_SHIFT 0 ++#define BOOST_BACKUP_PLL_MASK 0x3 ++#define BOOST_BACKUP_PLL_SHIFT 8 ++#define BOOST_BACKUP_PLL_USAGE_MASK 0x1 ++#define BOOST_BACKUP_PLL_USAGE_SHIFT 12 ++#define BOOST_BACKUP_PLL_USAGE_BORROW 0 ++#define BOOST_BACKUP_PLL_USAGE_TARGET 1 ++#define BOOST_ENABLE_MASK 0x1 ++#define BOOST_ENABLE_SHIFT 0 + #define BOOST_RECOVERY_MASK 0x1 + #define BOOST_RECOVERY_SHIFT 1 + #define BOOST_SW_CTRL_MASK 0x1 + #define BOOST_SW_CTRL_SHIFT 2 + #define BOOST_LOW_FREQ_EN_MASK 0x1 + #define BOOST_LOW_FREQ_EN_SHIFT 3 ++#define BOOST_STATIS_ENABLE_MASK 0x1 ++#define BOOST_STATIS_ENABLE_SHIFT 4 + #define BOOST_BUSY_STATE BIT(8) + + #define PX30_PLL_CON(x) ((x) * 0x4) +@@ -79,6 +92,51 @@ struct clk; + #define RV1108_EMMC_CON0 0x1e8 + #define RV1108_EMMC_CON1 0x1ec + ++#define RV1126_PMU_MODE 0x0 ++#define RV1126_PMU_PLL_CON(x) ((x) * 0x4 + 0x10) ++#define RV1126_PMU_CLKSEL_CON(x) ((x) * 0x4 + 0x100) ++#define RV1126_PMU_CLKGATE_CON(x) ((x) * 0x4 + 0x180) ++#define RV1126_PMU_SOFTRST_CON(x) ((x) * 0x4 + 0x200) ++#define RV1126_PLL_CON(x) ((x) * 0x4) ++#define RV1126_MODE_CON 0x90 ++#define RV1126_CLKSEL_CON(x) ((x) * 0x4 + 0x100) ++#define RV1126_CLKGATE_CON(x) ((x) * 0x4 + 0x280) ++#define RV1126_SOFTRST_CON(x) ((x) * 0x4 + 0x300) ++#define RV1126_GLB_SRST_FST 0x408 ++#define RV1126_GLB_SRST_SND 0x40c ++#define RV1126_SDMMC_CON0 0x440 ++#define RV1126_SDMMC_CON1 0x444 ++#define RV1126_SDIO_CON0 0x448 ++#define RV1126_SDIO_CON1 0x44c ++#define RV1126_EMMC_CON0 0x450 ++#define RV1126_EMMC_CON1 0x454 ++ ++/* ++ * register positions shared by RK1808 RK2928, RK3036, ++ * RK3066, RK3188 and RK3228 ++ */ ++ ++#define RK1808_PLL_CON(x) ((x) * 0x4) ++#define RK1808_MODE_CON 0xa0 ++#define RK1808_MISC_CON 0xa4 ++#define RK1808_MISC1_CON 0xa8 ++#define RK1808_GLB_SRST_FST 0xb8 ++#define RK1808_GLB_SRST_SND 0xbc ++#define RK1808_CLKSEL_CON(x) ((x) * 0x4 + 0x100) ++#define RK1808_CLKGATE_CON(x) ((x) * 0x4 + 0x230) ++#define RK1808_SOFTRST_CON(x) ((x) * 0x4 + 0x300) ++#define RK1808_SDMMC_CON0 0x380 ++#define RK1808_SDMMC_CON1 0x384 ++#define RK1808_SDIO_CON0 0x388 ++#define RK1808_SDIO_CON1 0x38c ++#define RK1808_EMMC_CON0 0x390 ++#define RK1808_EMMC_CON1 0x394 ++ ++#define RK1808_PMU_PLL_CON(x) ((x) * 0x4 + 0x4000) ++#define RK1808_PMU_MODE_CON 0x4020 ++#define RK1808_PMU_CLKSEL_CON(x) ((x) * 0x4 + 0x4040) ++#define RK1808_PMU_CLKGATE_CON(x) ((x) * 0x4 + 0x4080) ++ + #define RK2928_PLL_CON(x) ((x) * 0x4) + #define RK2928_MODE_CON 0x40 + #define RK2928_CLKSEL_CON(x) ((x) * 0x4 + 0x44) +@@ -188,6 +246,34 @@ struct clk; + #define RK3399_PMU_CLKGATE_CON(x) ((x) * 0x4 + 0x100) + #define RK3399_PMU_SOFTRST_CON(x) ((x) * 0x4 + 0x110) + ++#define RK3568_PLL_CON(x) RK2928_PLL_CON(x) ++#define RK3568_MODE_CON0 0xc0 ++#define RK3568_MISC_CON0 0xc4 ++#define RK3568_MISC_CON1 0xc8 ++#define RK3568_MISC_CON2 0xcc ++#define RK3568_GLB_CNT_TH 0xd0 ++#define RK3568_GLB_SRST_FST 0xd4 ++#define RK3568_GLB_SRST_SND 0xd8 ++#define RK3568_GLB_RST_CON 0xdc ++#define RK3568_GLB_RST_ST 0xe0 ++#define RK3568_CLKSEL_CON(x) ((x) * 0x4 + 0x100) ++#define RK3568_CLKGATE_CON(x) ((x) * 0x4 + 0x300) ++#define RK3568_SOFTRST_CON(x) ((x) * 0x4 + 0x400) ++#define RK3568_SDMMC0_CON0 0x580 ++#define RK3568_SDMMC0_CON1 0x584 ++#define RK3568_SDMMC1_CON0 0x588 ++#define RK3568_SDMMC1_CON1 0x58c ++#define RK3568_SDMMC2_CON0 0x590 ++#define RK3568_SDMMC2_CON1 0x594 ++#define RK3568_EMMC_CON0 0x598 ++#define RK3568_EMMC_CON1 0x59c ++ ++#define RK3568_PMU_PLL_CON(x) RK2928_PLL_CON(x) ++#define RK3568_PMU_MODE_CON0 0x80 ++#define RK3568_PMU_CLKSEL_CON(x) ((x) * 0x4 + 0x100) ++#define RK3568_PMU_CLKGATE_CON(x) ((x) * 0x4 + 0x180) ++#define RK3568_PMU_SOFTRST_CON(x) ((x) * 0x4 + 0x200) ++ + enum rockchip_pll_type { + pll_rk3036, + pll_rk3066, +@@ -238,22 +324,30 @@ struct rockchip_clk_provider { + struct clk_onecell_data clk_data; + struct device_node *cru_node; + struct regmap *grf; ++ struct regmap *pmugrf; + spinlock_t lock; + }; + + struct rockchip_pll_rate_table { + unsigned long rate; +- unsigned int nr; +- unsigned int nf; +- unsigned int no; +- unsigned int nb; +- /* for RK3036/RK3399 */ +- unsigned int fbdiv; +- unsigned int postdiv1; +- unsigned int refdiv; +- unsigned int postdiv2; +- unsigned int dsmpd; +- unsigned int frac; ++ union { ++ struct { ++ /* for RK3066 */ ++ unsigned int nr; ++ unsigned int nf; ++ unsigned int no; ++ unsigned int nb; ++ }; ++ struct { ++ /* for RK3036/RK3399 */ ++ unsigned int fbdiv; ++ unsigned int postdiv1; ++ unsigned int refdiv; ++ unsigned int postdiv2; ++ unsigned int dsmpd; ++ unsigned int frac; ++ }; ++ }; + }; + + /** +@@ -317,12 +411,21 @@ struct clk *rockchip_clk_register_pll(struct rockchip_clk_provider *ctx, + struct rockchip_pll_rate_table *rate_table, + unsigned long flags, u8 clk_pll_flags); + ++void rockchip_boost_init(struct clk_hw *hw); ++ ++void rockchip_boost_enable_recovery_sw_low(struct clk_hw *hw); ++ ++void rockchip_boost_disable_recovery_sw(struct clk_hw *hw); ++ ++void rockchip_boost_add_core_div(struct clk_hw *hw, unsigned long prate); ++ + struct rockchip_cpuclk_clksel { + int reg; + u32 val; + }; + +-#define ROCKCHIP_CPUCLK_NUM_DIVIDERS 2 ++#define ROCKCHIP_CPUCLK_NUM_DIVIDERS 5 ++#define ROCKCHIP_CPUCLK_MAX_CORES 4 + struct rockchip_cpuclk_rate_table { + unsigned long prate; + struct rockchip_cpuclk_clksel divs[ROCKCHIP_CPUCLK_NUM_DIVIDERS]; +@@ -330,26 +433,29 @@ struct rockchip_cpuclk_rate_table { + + /** + * struct rockchip_cpuclk_reg_data - register offsets and masks of the cpuclock +- * @core_reg: register offset of the core settings register +- * @div_core_shift: core divider offset used to divide the pll value +- * @div_core_mask: core divider mask +- * @mux_core_alt: mux value to select alternate parent ++ * @core_reg[]: register offset of the cores setting register ++ * @div_core_shift[]: cores divider offset used to divide the pll value ++ * @div_core_mask[]: cores divider mask ++ * @num_cores: number of cpu cores + * @mux_core_main: mux value to select main parent of core + * @mux_core_shift: offset of the core multiplexer + * @mux_core_mask: core multiplexer mask + */ + struct rockchip_cpuclk_reg_data { +- int core_reg; +- u8 div_core_shift; +- u32 div_core_mask; +- u8 mux_core_alt; +- u8 mux_core_main; +- u8 mux_core_shift; +- u32 mux_core_mask; ++ int core_reg[ROCKCHIP_CPUCLK_MAX_CORES]; ++ u8 div_core_shift[ROCKCHIP_CPUCLK_MAX_CORES]; ++ u32 div_core_mask[ROCKCHIP_CPUCLK_MAX_CORES]; ++ int num_cores; ++ u8 mux_core_alt; ++ u8 mux_core_main; ++ u8 mux_core_shift; ++ u32 mux_core_mask; ++ const char *pll_name; + }; + + struct clk *rockchip_clk_register_cpuclk(const char *name, +- const char *const *parent_names, u8 num_parents, ++ u8 num_parents, ++ struct clk *parent, struct clk *alt_parent, + const struct rockchip_cpuclk_reg_data *reg_data, + const struct rockchip_cpuclk_rate_table *rates, + int nrates, void __iomem *reg_base, spinlock_t *lock); +@@ -361,16 +467,21 @@ struct clk *rockchip_clk_register_mmc(const char *name, + /* + * DDRCLK flags, including method of setting the rate + * ROCKCHIP_DDRCLK_SIP: use SIP call to bl31 to change ddrclk rate. ++ * ROCKCHIP_DDRCLK_SCPI: use SCPI APIs to let mcu change ddrclk rate. + */ + #define ROCKCHIP_DDRCLK_SIP BIT(0) ++#define ROCKCHIP_DDRCLK_SCPI 0x02 ++#define ROCKCHIP_DDRCLK_SIP_V2 0x03 ++ ++void rockchip_set_ddrclk_params(void __iomem *params); ++void rockchip_set_ddrclk_dmcfreq_wait_complete(int (*func)(void)); + + struct clk *rockchip_clk_register_ddrclk(const char *name, int flags, + const char *const *parent_names, + u8 num_parents, int mux_offset, + int mux_shift, int mux_width, + int div_shift, int div_width, +- int ddr_flags, void __iomem *reg_base, +- spinlock_t *lock); ++ int ddr_flags, void __iomem *reg_base); + + #define ROCKCHIP_INVERTER_HIWORD_MASK BIT(0) + +@@ -388,8 +499,10 @@ struct clk *rockchip_clk_register_muxgrf(const char *name, + + enum rockchip_clk_branch_type { + branch_composite, ++ branch_composite_brother, + branch_mux, + branch_muxgrf, ++ branch_muxpmugrf, + branch_divider, + branch_fraction_divider, + branch_gate, +@@ -398,6 +511,7 @@ enum rockchip_clk_branch_type { + branch_factor, + branch_ddrclk, + branch_half_divider, ++ branch_dclk_divider, + }; + + struct rockchip_clk_branch { +@@ -411,6 +525,7 @@ struct rockchip_clk_branch { + u8 mux_shift; + u8 mux_width; + u8 mux_flags; ++ u32 *mux_table; + int div_offset; + u8 div_shift; + u8 div_width; +@@ -420,6 +535,7 @@ struct rockchip_clk_branch { + u8 gate_shift; + u8 gate_flags; + struct rockchip_clk_branch *child; ++ unsigned long max_prate; + }; + + #define COMPOSITE(_id, cname, pnames, f, mo, ms, mw, mf, ds, dw,\ +@@ -443,6 +559,50 @@ struct rockchip_clk_branch { + .gate_flags = gf, \ + } + ++#define COMPOSITE_BROTHER(_id, cname, pnames, f, mo, ms, mw, mf,\ ++ ds, dw, df, go, gs, gf, bro) \ ++ { \ ++ .id = _id, \ ++ .branch_type = branch_composite_brother, \ ++ .name = cname, \ ++ .parent_names = pnames, \ ++ .num_parents = ARRAY_SIZE(pnames), \ ++ .flags = f, \ ++ .muxdiv_offset = mo, \ ++ .mux_shift = ms, \ ++ .mux_width = mw, \ ++ .mux_flags = mf, \ ++ .div_shift = ds, \ ++ .div_width = dw, \ ++ .div_flags = df, \ ++ .gate_offset = go, \ ++ .gate_shift = gs, \ ++ .gate_flags = gf, \ ++ .child = bro, \ ++ } ++ ++#define COMPOSITE_MUXTBL(_id, cname, pnames, f, mo, ms, mw, mf, \ ++ mt, ds, dw, df, go, gs, gf) \ ++ { \ ++ .id = _id, \ ++ .branch_type = branch_composite, \ ++ .name = cname, \ ++ .parent_names = pnames, \ ++ .num_parents = ARRAY_SIZE(pnames), \ ++ .flags = f, \ ++ .muxdiv_offset = mo, \ ++ .mux_shift = ms, \ ++ .mux_width = mw, \ ++ .mux_flags = mf, \ ++ .mux_table = mt, \ ++ .div_shift = ds, \ ++ .div_width = dw, \ ++ .div_flags = df, \ ++ .gate_offset = go, \ ++ .gate_shift = gs, \ ++ .gate_flags = gf, \ ++ } ++ + #define COMPOSITE_DIV_OFFSET(_id, cname, pnames, f, mo, ms, mw, \ + mf, do, ds, dw, df, go, gs, gf) \ + { \ +@@ -539,6 +699,26 @@ struct rockchip_clk_branch { + .gate_offset = -1, \ + } + ++#define COMPOSITE_BROTHER_NOGATE(_id, cname, pnames, f, mo, ms, \ ++ mw, mf, ds, dw, df, bro) \ ++ { \ ++ .id = _id, \ ++ .branch_type = branch_composite_brother, \ ++ .name = cname, \ ++ .parent_names = pnames, \ ++ .num_parents = ARRAY_SIZE(pnames), \ ++ .flags = f, \ ++ .muxdiv_offset = mo, \ ++ .mux_shift = ms, \ ++ .mux_width = mw, \ ++ .mux_flags = mf, \ ++ .div_shift = ds, \ ++ .div_width = dw, \ ++ .div_flags = df, \ ++ .gate_offset = -1, \ ++ .child = bro, \ ++ } ++ + #define COMPOSITE_NOGATE_DIVTBL(_id, cname, pnames, f, mo, ms, \ + mw, mf, ds, dw, df, dt) \ + { \ +@@ -559,7 +739,7 @@ struct rockchip_clk_branch { + .gate_offset = -1, \ + } + +-#define COMPOSITE_FRAC(_id, cname, pname, f, mo, df, go, gs, gf)\ ++#define COMPOSITE_FRAC(_id, cname, pname, f, mo, df, go, gs, gf, prate)\ + { \ + .id = _id, \ + .branch_type = branch_fraction_divider, \ +@@ -574,9 +754,10 @@ struct rockchip_clk_branch { + .gate_offset = go, \ + .gate_shift = gs, \ + .gate_flags = gf, \ ++ .max_prate = prate, \ + } + +-#define COMPOSITE_FRACMUX(_id, cname, pname, f, mo, df, go, gs, gf, ch) \ ++#define COMPOSITE_FRACMUX(_id, cname, pname, f, mo, df, go, gs, gf, ch, prate) \ + { \ + .id = _id, \ + .branch_type = branch_fraction_divider, \ +@@ -592,9 +773,10 @@ struct rockchip_clk_branch { + .gate_shift = gs, \ + .gate_flags = gf, \ + .child = ch, \ ++ .max_prate = prate, \ + } + +-#define COMPOSITE_FRACMUX_NOGATE(_id, cname, pname, f, mo, df, ch) \ ++#define COMPOSITE_FRACMUX_NOGATE(_id, cname, pname, f, mo, df, ch, prate) \ + { \ + .id = _id, \ + .branch_type = branch_fraction_divider, \ +@@ -608,6 +790,7 @@ struct rockchip_clk_branch { + .div_flags = df, \ + .gate_offset = -1, \ + .child = ch, \ ++ .max_prate = prate, \ + } + + #define COMPOSITE_DDRCLK(_id, cname, pnames, f, mo, ms, mw, \ +@@ -643,6 +826,22 @@ struct rockchip_clk_branch { + .gate_offset = -1, \ + } + ++#define MUXTBL(_id, cname, pnames, f, o, s, w, mf, mt) \ ++ { \ ++ .id = _id, \ ++ .branch_type = branch_mux, \ ++ .name = cname, \ ++ .parent_names = pnames, \ ++ .num_parents = ARRAY_SIZE(pnames), \ ++ .flags = f, \ ++ .muxdiv_offset = o, \ ++ .mux_shift = s, \ ++ .mux_width = w, \ ++ .mux_flags = mf, \ ++ .gate_offset = -1, \ ++ .mux_table = mt, \ ++ } ++ + #define MUXGRF(_id, cname, pnames, f, o, s, w, mf) \ + { \ + .id = _id, \ +@@ -658,6 +857,21 @@ struct rockchip_clk_branch { + .gate_offset = -1, \ + } + ++#define MUXPMUGRF(_id, cname, pnames, f, o, s, w, mf) \ ++ { \ ++ .id = _id, \ ++ .branch_type = branch_muxpmugrf, \ ++ .name = cname, \ ++ .parent_names = pnames, \ ++ .num_parents = ARRAY_SIZE(pnames), \ ++ .flags = f, \ ++ .muxdiv_offset = o, \ ++ .mux_shift = s, \ ++ .mux_width = w, \ ++ .mux_flags = mf, \ ++ .gate_offset = -1, \ ++ } ++ + #define DIV(_id, cname, pname, f, o, s, w, df) \ + { \ + .id = _id, \ +@@ -772,6 +986,28 @@ struct rockchip_clk_branch { + .gate_flags = gf, \ + } + ++#define COMPOSITE_HALFDIV_OFFSET(_id, cname, pnames, f, mo, ms, mw, mf, do,\ ++ ds, dw, df, go, gs, gf) \ ++ { \ ++ .id = _id, \ ++ .branch_type = branch_half_divider, \ ++ .name = cname, \ ++ .parent_names = pnames, \ ++ .num_parents = ARRAY_SIZE(pnames), \ ++ .flags = f, \ ++ .muxdiv_offset = mo, \ ++ .mux_shift = ms, \ ++ .mux_width = mw, \ ++ .mux_flags = mf, \ ++ .div_offset = do, \ ++ .div_shift = ds, \ ++ .div_width = dw, \ ++ .div_flags = df, \ ++ .gate_offset = go, \ ++ .gate_shift = gs, \ ++ .gate_flags = gf, \ ++ } ++ + #define COMPOSITE_NOGATE_HALFDIV(_id, cname, pnames, f, mo, ms, mw, mf, \ + ds, dw, df) \ + { \ +@@ -824,6 +1060,28 @@ struct rockchip_clk_branch { + .gate_offset = -1, \ + } + ++#define COMPOSITE_DCLK(_id, cname, pnames, f, mo, ms, mw, mf, ds, dw,\ ++ df, go, gs, gf, prate) \ ++ { \ ++ .id = _id, \ ++ .branch_type = branch_dclk_divider, \ ++ .name = cname, \ ++ .parent_names = pnames, \ ++ .num_parents = ARRAY_SIZE(pnames), \ ++ .flags = f, \ ++ .muxdiv_offset = mo, \ ++ .mux_shift = ms, \ ++ .mux_width = mw, \ ++ .mux_flags = mf, \ ++ .div_shift = ds, \ ++ .div_width = dw, \ ++ .div_flags = df, \ ++ .gate_offset = go, \ ++ .gate_shift = gs, \ ++ .gate_flags = gf, \ ++ .max_prate = prate, \ ++ } ++ + /* SGRF clocks are only accessible from secure mode, so not controllable */ + #define SGRF_GATE(_id, cname, pname) \ + FACTOR(_id, cname, pname, 0, 1, 1) +@@ -840,13 +1098,17 @@ void rockchip_clk_register_branches(struct rockchip_clk_provider *ctx, + void rockchip_clk_register_plls(struct rockchip_clk_provider *ctx, + struct rockchip_pll_clock *pll_list, + unsigned int nr_pll, int grf_lock_offset); +-void rockchip_clk_register_armclk(struct rockchip_clk_provider *ctx, +- unsigned int lookup_id, const char *name, +- const char *const *parent_names, u8 num_parents, +- const struct rockchip_cpuclk_reg_data *reg_data, +- const struct rockchip_cpuclk_rate_table *rates, +- int nrates); +-void rockchip_clk_protect_critical(const char *const clocks[], int nclocks); ++void __init rockchip_clk_register_armclk(struct rockchip_clk_provider *ctx, ++ unsigned int lookup_id, ++ const char *name, ++ u8 num_parents, ++ struct clk *parent, struct clk *alt_parent, ++ const struct rockchip_cpuclk_reg_data *reg_data, ++ const struct rockchip_cpuclk_rate_table *rates, ++ int nrates); ++int rockchip_pll_clk_rate_to_scale(struct clk *clk, unsigned long rate); ++int rockchip_pll_clk_scale_to_rate(struct clk *clk, unsigned int scale); ++int rockchip_pll_clk_adaptive_scaling(struct clk *clk, int sel); + void rockchip_register_restart_notifier(struct rockchip_clk_provider *ctx, + unsigned int reg, void (*cb)(void)); + +@@ -857,12 +1119,27 @@ struct clk *rockchip_clk_register_halfdiv(const char *name, + u8 num_parents, void __iomem *base, + int muxdiv_offset, u8 mux_shift, + u8 mux_width, u8 mux_flags, +- u8 div_shift, u8 div_width, +- u8 div_flags, int gate_offset, +- u8 gate_shift, u8 gate_flags, +- unsigned long flags, ++ int div_offset, u8 div_shift, ++ u8 div_width, u8 div_flags, ++ int gate_offset, u8 gate_shift, ++ u8 gate_flags, unsigned long flags, + spinlock_t *lock); + ++struct clk *rockchip_clk_register_dclk_branch(const char *name, ++ const char *const *parent_names, ++ u8 num_parents, ++ void __iomem *base, ++ int muxdiv_offset, u8 mux_shift, ++ u8 mux_width, u8 mux_flags, ++ int div_offset, u8 div_shift, ++ u8 div_width, u8 div_flags, ++ struct clk_div_table *div_table, ++ int gate_offset, ++ u8 gate_shift, u8 gate_flags, ++ unsigned long flags, ++ unsigned long max_prate, ++ spinlock_t *lock); ++ + #ifdef CONFIG_RESET_CONTROLLER + void rockchip_register_softrst(struct device_node *np, + unsigned int num_regs, +@@ -874,5 +1151,6 @@ static inline void rockchip_register_softrst(struct device_node *np, + { + } + #endif ++extern void (*rk_dump_cru)(void); + + #endif +diff --git a/drivers/clk/rockchip/regmap/Kconfig b/drivers/clk/rockchip/regmap/Kconfig +new file mode 100755 +index 000000000000..65f691bc4141 +--- /dev/null ++++ b/drivers/clk/rockchip/regmap/Kconfig +@@ -0,0 +1,16 @@ ++# SPDX-License-Identifier: GPL-2.0 ++ ++config COMMON_CLK_ROCKCHIP_REGMAP ++ tristate ++ ++config CLK_RK618 ++ tristate "Clock driver for Rockchip RK618" ++ depends on MFD_RK618 ++ default MFD_RK618 ++ select COMMON_CLK_ROCKCHIP_REGMAP ++ ++config CLK_RK628 ++ tristate "Clock driver for Rockchip RK628" ++ depends on MFD_RK628 ++ default MFD_RK628 ++ select COMMON_CLK_ROCKCHIP_REGMAP +diff --git a/drivers/clk/rockchip/regmap/Makefile b/drivers/clk/rockchip/regmap/Makefile +new file mode 100755 +index 000000000000..18d075d093d9 +--- /dev/null ++++ b/drivers/clk/rockchip/regmap/Makefile +@@ -0,0 +1,13 @@ ++# SPDX-License-Identifier: GPL-2.0 ++ ++obj-$(CONFIG_COMMON_CLK_ROCKCHIP_REGMAP) += clk-rockchip-regmap.o ++ ++clk-rockchip-regmap-objs := clk-regmap-mux.o \ ++ clk-regmap-divider.o \ ++ clk-regmap-gate.o \ ++ clk-regmap-fractional-divider.o \ ++ clk-regmap-composite.o \ ++ clk-regmap-pll.o ++ ++obj-$(CONFIG_CLK_RK618) += clk-rk618.o ++obj-$(CONFIG_CLK_RK628) += clk-rk628.o +diff --git a/drivers/clk/rockchip/regmap/clk-regmap-composite.c b/drivers/clk/rockchip/regmap/clk-regmap-composite.c +new file mode 100755 +index 000000000000..43d2b9a45aca +--- /dev/null ++++ b/drivers/clk/rockchip/regmap/clk-regmap-composite.c +@@ -0,0 +1,400 @@ ++/* ++ * Copyright (c) 2017 Rockchip Electronics Co. Ltd. ++ * ++ * Base on code in drivers/clk/clk-composite.c. ++ * See clk-composite.c for further copyright information. ++ * ++ * This program is free software; you can redistribute it and/or modify ++ * it under the terms of the GNU General Public License as published by ++ * the Free Software Foundation; either version 2 of the License, or ++ * (at your option) any later version. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ */ ++ ++#include "clk-regmap.h" ++ ++struct clk_regmap_composite { ++ struct device *dev; ++ struct clk_hw hw; ++ struct clk_ops ops; ++ ++ struct clk_hw *mux_hw; ++ struct clk_hw *rate_hw; ++ struct clk_hw *gate_hw; ++ ++ const struct clk_ops *mux_ops; ++ const struct clk_ops *rate_ops; ++ const struct clk_ops *gate_ops; ++}; ++ ++#define to_clk_regmap_composite(_hw) \ ++ container_of(_hw, struct clk_regmap_composite, hw) ++ ++static u8 clk_regmap_composite_get_parent(struct clk_hw *hw) ++{ ++ struct clk_regmap_composite *composite = to_clk_regmap_composite(hw); ++ const struct clk_ops *mux_ops = composite->mux_ops; ++ struct clk_hw *mux_hw = composite->mux_hw; ++ ++ __clk_hw_set_clk(mux_hw, hw); ++ ++ return mux_ops->get_parent(mux_hw); ++} ++ ++static int clk_regmap_composite_set_parent(struct clk_hw *hw, u8 index) ++{ ++ struct clk_regmap_composite *composite = to_clk_regmap_composite(hw); ++ const struct clk_ops *mux_ops = composite->mux_ops; ++ struct clk_hw *mux_hw = composite->mux_hw; ++ ++ __clk_hw_set_clk(mux_hw, hw); ++ ++ return mux_ops->set_parent(mux_hw, index); ++} ++ ++static unsigned long clk_regmap_composite_recalc_rate(struct clk_hw *hw, ++ unsigned long parent_rate) ++{ ++ struct clk_regmap_composite *composite = to_clk_regmap_composite(hw); ++ const struct clk_ops *rate_ops = composite->rate_ops; ++ struct clk_hw *rate_hw = composite->rate_hw; ++ ++ __clk_hw_set_clk(rate_hw, hw); ++ ++ return rate_ops->recalc_rate(rate_hw, parent_rate); ++} ++ ++static int clk_regmap_composite_determine_rate(struct clk_hw *hw, ++ struct clk_rate_request *req) ++{ ++ struct clk_regmap_composite *composite = to_clk_regmap_composite(hw); ++ const struct clk_ops *rate_ops = composite->rate_ops; ++ const struct clk_ops *mux_ops = composite->mux_ops; ++ struct clk_hw *rate_hw = composite->rate_hw; ++ struct clk_hw *mux_hw = composite->mux_hw; ++ struct clk_hw *parent; ++ unsigned long parent_rate; ++ long tmp_rate, best_rate = 0; ++ unsigned long rate_diff; ++ unsigned long best_rate_diff = ULONG_MAX; ++ long rate; ++ unsigned int i; ++ ++ if (rate_hw && rate_ops && rate_ops->determine_rate) { ++ __clk_hw_set_clk(rate_hw, hw); ++ return rate_ops->determine_rate(rate_hw, req); ++ } else if (rate_hw && rate_ops && rate_ops->round_rate && ++ mux_hw && mux_ops && mux_ops->set_parent) { ++ req->best_parent_hw = NULL; ++ ++ if (clk_hw_get_flags(hw) & CLK_SET_RATE_NO_REPARENT) { ++ parent = clk_hw_get_parent(mux_hw); ++ req->best_parent_hw = parent; ++ req->best_parent_rate = clk_hw_get_rate(parent); ++ ++ rate = rate_ops->round_rate(rate_hw, req->rate, ++ &req->best_parent_rate); ++ if (rate < 0) ++ return rate; ++ ++ req->rate = rate; ++ return 0; ++ } ++ ++ for (i = 0; i < clk_hw_get_num_parents(mux_hw); i++) { ++ parent = clk_hw_get_parent_by_index(mux_hw, i); ++ if (!parent) ++ continue; ++ ++ parent_rate = clk_hw_get_rate(parent); ++ ++ tmp_rate = rate_ops->round_rate(rate_hw, req->rate, ++ &parent_rate); ++ if (tmp_rate < 0) ++ continue; ++ ++ rate_diff = abs(req->rate - tmp_rate); ++ ++ if (!rate_diff || !req->best_parent_hw || ++ best_rate_diff > rate_diff) { ++ req->best_parent_hw = parent; ++ req->best_parent_rate = parent_rate; ++ best_rate_diff = rate_diff; ++ best_rate = tmp_rate; ++ } ++ ++ if (!rate_diff) ++ return 0; ++ } ++ ++ req->rate = best_rate; ++ return 0; ++ } else if (mux_hw && mux_ops && mux_ops->determine_rate) { ++ __clk_hw_set_clk(mux_hw, hw); ++ return mux_ops->determine_rate(mux_hw, req); ++ } else { ++ return -EINVAL; ++ } ++ ++ return 0; ++} ++ ++static long clk_regmap_composite_round_rate(struct clk_hw *hw, ++ unsigned long rate, ++ unsigned long *prate) ++{ ++ struct clk_regmap_composite *composite = to_clk_regmap_composite(hw); ++ const struct clk_ops *rate_ops = composite->rate_ops; ++ struct clk_hw *rate_hw = composite->rate_hw; ++ ++ __clk_hw_set_clk(rate_hw, hw); ++ ++ return rate_ops->round_rate(rate_hw, rate, prate); ++} ++ ++static int clk_regmap_composite_set_rate(struct clk_hw *hw, ++ unsigned long rate, ++ unsigned long parent_rate) ++{ ++ struct clk_regmap_composite *composite = to_clk_regmap_composite(hw); ++ const struct clk_ops *rate_ops = composite->rate_ops; ++ struct clk_hw *rate_hw = composite->rate_hw; ++ ++ __clk_hw_set_clk(rate_hw, hw); ++ ++ return rate_ops->set_rate(rate_hw, rate, parent_rate); ++} ++ ++static int clk_regmap_composite_is_prepared(struct clk_hw *hw) ++{ ++ struct clk_regmap_composite *composite = to_clk_regmap_composite(hw); ++ const struct clk_ops *gate_ops = composite->gate_ops; ++ struct clk_hw *gate_hw = composite->gate_hw; ++ ++ __clk_hw_set_clk(gate_hw, hw); ++ ++ return gate_ops->is_prepared(gate_hw); ++} ++ ++static int clk_regmap_composite_prepare(struct clk_hw *hw) ++{ ++ struct clk_regmap_composite *composite = to_clk_regmap_composite(hw); ++ const struct clk_ops *gate_ops = composite->gate_ops; ++ struct clk_hw *gate_hw = composite->gate_hw; ++ ++ __clk_hw_set_clk(gate_hw, hw); ++ ++ return gate_ops->prepare(gate_hw); ++} ++ ++static void clk_regmap_composite_unprepare(struct clk_hw *hw) ++{ ++ struct clk_regmap_composite *composite = to_clk_regmap_composite(hw); ++ const struct clk_ops *gate_ops = composite->gate_ops; ++ struct clk_hw *gate_hw = composite->gate_hw; ++ ++ __clk_hw_set_clk(gate_hw, hw); ++ ++ gate_ops->unprepare(gate_hw); ++} ++ ++struct clk * ++devm_clk_regmap_register_composite(struct device *dev, const char *name, ++ const char *const *parent_names, ++ u8 num_parents, struct regmap *regmap, ++ u32 mux_reg, u8 mux_shift, u8 mux_width, ++ u32 div_reg, u8 div_shift, u8 div_width, ++ u8 div_flags, ++ u32 gate_reg, u8 gate_shift, ++ unsigned long flags) ++{ ++ struct clk_regmap_gate *gate = NULL; ++ struct clk_regmap_mux *mux = NULL; ++ struct clk_regmap_divider *div = NULL; ++ struct clk_regmap_fractional_divider *fd = NULL; ++ const struct clk_ops *mux_ops = NULL, *div_ops = NULL, *gate_ops = NULL; ++ const struct clk_ops *fd_ops = NULL; ++ struct clk_hw *mux_hw = NULL, *div_hw = NULL, *gate_hw = NULL; ++ struct clk_hw *fd_hw = NULL; ++ struct clk *clk; ++ struct clk_init_data init = {}; ++ struct clk_regmap_composite *composite; ++ struct clk_ops *clk_composite_ops; ++ ++ if (num_parents > 1) { ++ mux = devm_kzalloc(dev, sizeof(*mux), GFP_KERNEL); ++ if (!mux) ++ return ERR_PTR(-ENOMEM); ++ ++ mux->dev = dev; ++ mux->regmap = regmap; ++ mux->reg = mux_reg; ++ mux->shift = mux_shift; ++ mux->mask = BIT(mux_width) - 1; ++ mux_ops = &clk_regmap_mux_ops; ++ mux_hw = &mux->hw; ++ } ++ ++ if (gate_reg > 0) { ++ gate = devm_kzalloc(dev, sizeof(*gate), GFP_KERNEL); ++ if (!gate) ++ return ERR_PTR(-ENOMEM); ++ ++ gate->dev = dev; ++ gate->regmap = regmap; ++ gate->reg = gate_reg; ++ gate->shift = gate_shift; ++ gate_ops = &clk_regmap_gate_ops; ++ gate_hw = &gate->hw; ++ } ++ ++ if (div_reg > 0) { ++ if (div_flags & CLK_DIVIDER_HIWORD_MASK) { ++ div = devm_kzalloc(dev, sizeof(*div), GFP_KERNEL); ++ if (!div) ++ return ERR_PTR(-ENOMEM); ++ ++ div->dev = dev; ++ div->regmap = regmap; ++ div->reg = div_reg; ++ div->shift = div_shift; ++ div->width = div_width; ++ div_ops = &clk_regmap_divider_ops; ++ div_hw = &div->hw; ++ } else { ++ fd = devm_kzalloc(dev, sizeof(*fd), GFP_KERNEL); ++ if (!fd) ++ return ERR_PTR(-ENOMEM); ++ ++ fd->dev = dev; ++ fd->regmap = regmap; ++ fd->reg = div_reg; ++ fd->mshift = 16; ++ fd->mwidth = 16; ++ fd->mmask = GENMASK(fd->mwidth - 1, 0) << fd->mshift; ++ fd->nshift = 0; ++ fd->nwidth = 16; ++ fd->nmask = GENMASK(fd->nwidth - 1, 0) << fd->nshift; ++ fd_ops = &clk_regmap_fractional_divider_ops; ++ fd_hw = &fd->hw; ++ } ++ } ++ ++ composite = devm_kzalloc(dev, sizeof(*composite), GFP_KERNEL); ++ if (!composite) ++ return ERR_PTR(-ENOMEM); ++ ++ init.name = name; ++ init.flags = flags; ++ init.parent_names = parent_names; ++ init.num_parents = num_parents; ++ ++ clk_composite_ops = &composite->ops; ++ ++ if (mux_hw && mux_ops) { ++ if (!mux_ops->get_parent) ++ return ERR_PTR(-EINVAL); ++ ++ composite->mux_hw = mux_hw; ++ composite->mux_ops = mux_ops; ++ clk_composite_ops->get_parent = ++ clk_regmap_composite_get_parent; ++ if (mux_ops->set_parent) ++ clk_composite_ops->set_parent = ++ clk_regmap_composite_set_parent; ++ if (mux_ops->determine_rate) ++ clk_composite_ops->determine_rate = ++ clk_regmap_composite_determine_rate; ++ } ++ ++ if (div_hw && div_ops) { ++ if (!div_ops->recalc_rate) ++ return ERR_PTR(-EINVAL); ++ ++ clk_composite_ops->recalc_rate = ++ clk_regmap_composite_recalc_rate; ++ ++ if (div_ops->determine_rate) ++ clk_composite_ops->determine_rate = ++ clk_regmap_composite_determine_rate; ++ else if (div_ops->round_rate) ++ clk_composite_ops->round_rate = ++ clk_regmap_composite_round_rate; ++ ++ /* .set_rate requires either .round_rate or .determine_rate */ ++ if (div_ops->set_rate) { ++ if (div_ops->determine_rate || div_ops->round_rate) ++ clk_composite_ops->set_rate = ++ clk_regmap_composite_set_rate; ++ else ++ WARN(1, "missing round_rate op\n"); ++ } ++ ++ composite->rate_hw = div_hw; ++ composite->rate_ops = div_ops; ++ } ++ ++ if (fd_hw && fd_ops) { ++ if (!fd_ops->recalc_rate) ++ return ERR_PTR(-EINVAL); ++ ++ clk_composite_ops->recalc_rate = ++ clk_regmap_composite_recalc_rate; ++ ++ if (fd_ops->determine_rate) ++ clk_composite_ops->determine_rate = ++ clk_regmap_composite_determine_rate; ++ else if (fd_ops->round_rate) ++ clk_composite_ops->round_rate = ++ clk_regmap_composite_round_rate; ++ ++ /* .set_rate requires either .round_rate or .determine_rate */ ++ if (fd_ops->set_rate) { ++ if (fd_ops->determine_rate || fd_ops->round_rate) ++ clk_composite_ops->set_rate = ++ clk_regmap_composite_set_rate; ++ else ++ WARN(1, "missing round_rate op\n"); ++ } ++ ++ composite->rate_hw = fd_hw; ++ composite->rate_ops = fd_ops; ++ } ++ ++ if (gate_hw && gate_ops) { ++ if (!gate_ops->is_prepared || !gate_ops->prepare || ++ !gate_ops->unprepare) ++ return ERR_PTR(-EINVAL); ++ ++ composite->gate_hw = gate_hw; ++ composite->gate_ops = gate_ops; ++ clk_composite_ops->is_prepared = ++ clk_regmap_composite_is_prepared; ++ clk_composite_ops->prepare = clk_regmap_composite_prepare; ++ clk_composite_ops->unprepare = clk_regmap_composite_unprepare; ++ } ++ ++ init.ops = clk_composite_ops; ++ composite->dev = dev; ++ composite->hw.init = &init; ++ ++ clk = devm_clk_register(dev, &composite->hw); ++ if (IS_ERR(clk)) ++ return clk; ++ ++ if (composite->mux_hw) ++ composite->mux_hw->clk = clk; ++ ++ if (composite->rate_hw) ++ composite->rate_hw->clk = clk; ++ ++ if (composite->gate_hw) ++ composite->gate_hw->clk = clk; ++ ++ return clk; ++} ++EXPORT_SYMBOL_GPL(devm_clk_regmap_register_composite); +diff --git a/drivers/clk/rockchip/regmap/clk-regmap-divider.c b/drivers/clk/rockchip/regmap/clk-regmap-divider.c +new file mode 100755 +index 000000000000..cb59a3d9acd2 +--- /dev/null ++++ b/drivers/clk/rockchip/regmap/clk-regmap-divider.c +@@ -0,0 +1,104 @@ ++/* ++ * Copyright (c) 2017 Rockchip Electronics Co. Ltd. ++ * ++ * Base on code in drivers/clk/clk-divider.c. ++ * See clk-divider.c for further copyright information. ++ * ++ * This program is free software; you can redistribute it and/or modify ++ * it under the terms of the GNU General Public License as published by ++ * the Free Software Foundation; either version 2 of the License, or ++ * (at your option) any later version. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ */ ++ ++#include "clk-regmap.h" ++ ++#define div_mask(width) ((1 << (width)) - 1) ++ ++#define to_clk_regmap_divider(_hw) \ ++ container_of(_hw, struct clk_regmap_divider, hw) ++ ++static unsigned long ++clk_regmap_divider_recalc_rate(struct clk_hw *hw, unsigned long parent_rate) ++{ ++ struct clk_regmap_divider *divider = to_clk_regmap_divider(hw); ++ unsigned int val, div; ++ ++ regmap_read(divider->regmap, divider->reg, &val); ++ ++ div = val >> divider->shift; ++ div &= div_mask(divider->width); ++ ++ return divider_recalc_rate(hw, parent_rate, div, NULL, ++ CLK_DIVIDER_ROUND_CLOSEST, divider->width); ++} ++ ++static long ++clk_regmap_divider_round_rate(struct clk_hw *hw, unsigned long rate, ++ unsigned long *prate) ++{ ++ struct clk_regmap_divider *divider = to_clk_regmap_divider(hw); ++ ++ return divider_round_rate(hw, rate, prate, NULL, divider->width, ++ CLK_DIVIDER_ROUND_CLOSEST); ++} ++ ++static int ++clk_regmap_divider_set_rate(struct clk_hw *hw, unsigned long rate, ++ unsigned long parent_rate) ++{ ++ struct clk_regmap_divider *divider = to_clk_regmap_divider(hw); ++ u32 val, div; ++ ++ div = divider_get_val(rate, parent_rate, NULL, divider->width, ++ CLK_DIVIDER_ROUND_CLOSEST); ++ ++ dev_dbg(divider->dev, "%s: parent_rate=%ld, div=%d, rate=%ld\n", ++ clk_hw_get_name(hw), parent_rate, div, rate); ++ ++ val = div_mask(divider->width) << (divider->shift + 16); ++ val |= div << divider->shift; ++ ++ return regmap_write(divider->regmap, divider->reg, val); ++} ++ ++const struct clk_ops clk_regmap_divider_ops = { ++ .recalc_rate = clk_regmap_divider_recalc_rate, ++ .round_rate = clk_regmap_divider_round_rate, ++ .set_rate = clk_regmap_divider_set_rate, ++}; ++EXPORT_SYMBOL_GPL(clk_regmap_divider_ops); ++ ++struct clk * ++devm_clk_regmap_register_divider(struct device *dev, const char *name, ++ const char *parent_name, struct regmap *regmap, ++ u32 reg, u8 shift, u8 width, ++ unsigned long flags) ++{ ++ struct clk_regmap_divider *divider; ++ struct clk_init_data init = {}; ++ ++ divider = devm_kzalloc(dev, sizeof(*divider), GFP_KERNEL); ++ if (!divider) ++ return ERR_PTR(-ENOMEM); ++ ++ init.name = name; ++ init.ops = &clk_regmap_divider_ops; ++ init.flags = flags; ++ init.parent_names = (parent_name ? &parent_name : NULL); ++ init.num_parents = (parent_name ? 1 : 0); ++ ++ divider->dev = dev; ++ divider->regmap = regmap; ++ divider->reg = reg; ++ divider->shift = shift; ++ divider->width = width; ++ divider->hw.init = &init; ++ ++ return devm_clk_register(dev, ÷r->hw); ++} ++EXPORT_SYMBOL_GPL(devm_clk_regmap_register_divider); +diff --git a/drivers/clk/rockchip/regmap/clk-regmap-fractional-divider.c b/drivers/clk/rockchip/regmap/clk-regmap-fractional-divider.c +new file mode 100755 +index 000000000000..3d5f1d2691e3 +--- /dev/null ++++ b/drivers/clk/rockchip/regmap/clk-regmap-fractional-divider.c +@@ -0,0 +1,157 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * Copyright (c) 2019 Rockchip Electronics Co. Ltd. ++ * ++ * Base on code in drivers/clk/clk-fractional-divider.c. ++ * See clk-fractional-divider.c for further copyright information. ++ */ ++ ++#include ++ ++#include "clk-regmap.h" ++ ++#define to_clk_regmap_fractional_divider(_hw) \ ++ container_of(_hw, struct clk_regmap_fractional_divider, hw) ++ ++static unsigned long ++clk_regmap_fractional_divider_recalc_rate(struct clk_hw *hw, ++ unsigned long parent_rate) ++{ ++ struct clk_regmap_fractional_divider *fd = ++ to_clk_regmap_fractional_divider(hw); ++ unsigned long m, n; ++ u32 val; ++ u64 ret; ++ ++ regmap_read(fd->regmap, fd->reg, &val); ++ ++ m = (val & fd->mmask) >> fd->mshift; ++ n = (val & fd->nmask) >> fd->nshift; ++ ++ if (!n || !m) ++ return parent_rate; ++ ++ ret = (u64)parent_rate * m; ++ do_div(ret, n); ++ ++ return ret; ++} ++ ++static void clk_regmap_fractional_divider_approximation(struct clk_hw *hw, ++ unsigned long rate, unsigned long *parent_rate, ++ unsigned long *m, unsigned long *n) ++{ ++ struct clk_regmap_fractional_divider *fd = ++ to_clk_regmap_fractional_divider(hw); ++ unsigned long p_rate, p_parent_rate; ++ struct clk_hw *p_parent; ++ unsigned long scale; ++ ++ p_rate = clk_hw_get_rate(clk_hw_get_parent(hw)); ++ if ((rate * 20 > p_rate) && (p_rate % rate != 0)) { ++ p_parent = clk_hw_get_parent(clk_hw_get_parent(hw)); ++ p_parent_rate = clk_hw_get_rate(p_parent); ++ *parent_rate = p_parent_rate; ++ } ++ ++ /* ++ * Get rate closer to *parent_rate to guarantee there is no overflow ++ * for m and n. In the result it will be the nearest rate left shifted ++ * by (scale - fd->nwidth) bits. ++ */ ++ scale = fls_long(*parent_rate / rate - 1); ++ if (scale > fd->nwidth) ++ rate <<= scale - fd->nwidth; ++ ++ rational_best_approximation(rate, *parent_rate, ++ GENMASK(fd->mwidth - 1, 0), ++ GENMASK(fd->nwidth - 1, 0), ++ m, n); ++} ++ ++static long ++clk_regmap_fractional_divider_round_rate(struct clk_hw *hw, unsigned long rate, ++ unsigned long *parent_rate) ++{ ++ unsigned long m, n; ++ u64 ret; ++ ++ if (!rate) ++ return *parent_rate; ++ ++ if (rate >= *parent_rate) ++ return *parent_rate; ++ ++ clk_regmap_fractional_divider_approximation(hw, rate, parent_rate, ++ &m, &n); ++ ++ ret = (u64)*parent_rate * m; ++ do_div(ret, n); ++ ++ return ret; ++} ++ ++static int ++clk_regmap_fractional_divider_set_rate(struct clk_hw *hw, unsigned long rate, ++ unsigned long parent_rate) ++{ ++ struct clk_regmap_fractional_divider *fd = ++ to_clk_regmap_fractional_divider(hw); ++ unsigned long m, n; ++ u32 val; ++ ++ rational_best_approximation(rate, parent_rate, ++ GENMASK(fd->mwidth - 1, 0), GENMASK(fd->nwidth - 1, 0), ++ &m, &n); ++ ++ dev_dbg(fd->dev, "%s: parent_rate=%ld, m=%ld, n=%ld, rate=%ld\n", ++ clk_hw_get_name(hw), parent_rate, m, n, rate); ++ ++ regmap_read(fd->regmap, fd->reg, &val); ++ val &= ~(fd->mmask | fd->nmask); ++ val |= (m << fd->mshift) | (n << fd->nshift); ++ ++ return regmap_write(fd->regmap, fd->reg, val); ++} ++ ++const struct clk_ops clk_regmap_fractional_divider_ops = { ++ .recalc_rate = clk_regmap_fractional_divider_recalc_rate, ++ .round_rate = clk_regmap_fractional_divider_round_rate, ++ .set_rate = clk_regmap_fractional_divider_set_rate, ++}; ++EXPORT_SYMBOL_GPL(clk_regmap_fractional_divider_ops); ++ ++struct clk * ++devm_clk_regmap_register_fractional_divider(struct device *dev, ++ const char *name, ++ const char *parent_name, ++ struct regmap *regmap, ++ u32 reg, unsigned long flags) ++{ ++ struct clk_regmap_fractional_divider *fd; ++ struct clk_init_data init; ++ ++ fd = devm_kzalloc(dev, sizeof(*fd), GFP_KERNEL); ++ if (!fd) ++ return ERR_PTR(-ENOMEM); ++ ++ init.name = name; ++ init.ops = &clk_regmap_fractional_divider_ops; ++ init.flags = flags; ++ init.parent_names = (parent_name ? &parent_name : NULL); ++ init.num_parents = (parent_name ? 1 : 0); ++ ++ fd->dev = dev; ++ fd->regmap = regmap; ++ fd->reg = reg; ++ fd->mshift = 16; ++ fd->mwidth = 16; ++ fd->mmask = GENMASK(fd->mwidth - 1, 0) << fd->mshift; ++ fd->nshift = 0; ++ fd->nwidth = 16; ++ fd->nmask = GENMASK(fd->nwidth - 1, 0) << fd->nshift; ++ fd->hw.init = &init; ++ ++ return devm_clk_register(dev, &fd->hw); ++} ++EXPORT_SYMBOL_GPL(devm_clk_regmap_register_fractional_divider); +diff --git a/drivers/clk/rockchip/regmap/clk-regmap-gate.c b/drivers/clk/rockchip/regmap/clk-regmap-gate.c +new file mode 100755 +index 000000000000..36549b912fa2 +--- /dev/null ++++ b/drivers/clk/rockchip/regmap/clk-regmap-gate.c +@@ -0,0 +1,82 @@ ++/* ++ * Copyright (c) 2017 Rockchip Electronics Co. Ltd. ++ * ++ * Base on code in drivers/clk/clk-gate.c. ++ * See clk-gate.c for further copyright information. ++ * ++ * This program is free software; you can redistribute it and/or modify ++ * it under the terms of the GNU General Public License as published by ++ * the Free Software Foundation; either version 2 of the License, or ++ * (at your option) any later version. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ */ ++ ++#include "clk-regmap.h" ++ ++#define to_clk_regmap_gate(_hw) container_of(_hw, struct clk_regmap_gate, hw) ++ ++static int clk_regmap_gate_prepare(struct clk_hw *hw) ++{ ++ struct clk_regmap_gate *gate = to_clk_regmap_gate(hw); ++ ++ return regmap_write(gate->regmap, gate->reg, ++ 0 | BIT(gate->shift + 16)); ++} ++ ++static void clk_regmap_gate_unprepare(struct clk_hw *hw) ++{ ++ struct clk_regmap_gate *gate = to_clk_regmap_gate(hw); ++ ++ regmap_write(gate->regmap, gate->reg, ++ BIT(gate->shift) | BIT(gate->shift + 16)); ++} ++ ++static int clk_regmap_gate_is_prepared(struct clk_hw *hw) ++{ ++ struct clk_regmap_gate *gate = to_clk_regmap_gate(hw); ++ u32 val; ++ ++ regmap_read(gate->regmap, gate->reg, &val); ++ ++ return !(val & BIT(gate->shift)); ++} ++ ++const struct clk_ops clk_regmap_gate_ops = { ++ .prepare = clk_regmap_gate_prepare, ++ .unprepare = clk_regmap_gate_unprepare, ++ .is_prepared = clk_regmap_gate_is_prepared, ++}; ++EXPORT_SYMBOL_GPL(clk_regmap_gate_ops); ++ ++struct clk * ++devm_clk_regmap_register_gate(struct device *dev, const char *name, ++ const char *parent_name, ++ struct regmap *regmap, u32 reg, u8 shift, ++ unsigned long flags) ++{ ++ struct clk_regmap_gate *gate; ++ struct clk_init_data init = {}; ++ ++ gate = devm_kzalloc(dev, sizeof(*gate), GFP_KERNEL); ++ if (!gate) ++ return ERR_PTR(-ENOMEM); ++ ++ init.name = name; ++ init.ops = &clk_regmap_gate_ops; ++ init.flags = flags; ++ init.parent_names = (parent_name ? &parent_name : NULL); ++ init.num_parents = (parent_name ? 1 : 0); ++ ++ gate->dev = dev; ++ gate->regmap = regmap; ++ gate->reg = reg; ++ gate->shift = shift; ++ gate->hw.init = &init; ++ ++ return devm_clk_register(dev, &gate->hw); ++} ++EXPORT_SYMBOL_GPL(devm_clk_regmap_register_gate); +diff --git a/drivers/clk/rockchip/regmap/clk-regmap-mux.c b/drivers/clk/rockchip/regmap/clk-regmap-mux.c +new file mode 100755 +index 000000000000..49d58b9c53ab +--- /dev/null ++++ b/drivers/clk/rockchip/regmap/clk-regmap-mux.c +@@ -0,0 +1,79 @@ ++/* ++ * Copyright (c) 2017 Rockchip Electronics Co. Ltd. ++ * ++ * Base on code in drivers/clk/clk-mux.c. ++ * See clk-mux.c for further copyright information. ++ * ++ * This program is free software; you can redistribute it and/or modify ++ * it under the terms of the GNU General Public License as published by ++ * the Free Software Foundation; either version 2 of the License, or ++ * (at your option) any later version. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ */ ++ ++#include "clk-regmap.h" ++ ++#define to_clk_regmap_mux(_hw) container_of(_hw, struct clk_regmap_mux, hw) ++ ++static u8 clk_regmap_mux_get_parent(struct clk_hw *hw) ++{ ++ struct clk_regmap_mux *mux = to_clk_regmap_mux(hw); ++ u8 index; ++ u32 val; ++ ++ regmap_read(mux->regmap, mux->reg, &val); ++ ++ index = val >> mux->shift; ++ index &= mux->mask; ++ ++ return index; ++} ++ ++static int clk_regmap_mux_set_parent(struct clk_hw *hw, u8 index) ++{ ++ struct clk_regmap_mux *mux = to_clk_regmap_mux(hw); ++ ++ return regmap_write(mux->regmap, mux->reg, (index << mux->shift) | ++ (mux->mask << (mux->shift + 16))); ++} ++ ++const struct clk_ops clk_regmap_mux_ops = { ++ .set_parent = clk_regmap_mux_set_parent, ++ .get_parent = clk_regmap_mux_get_parent, ++ .determine_rate = __clk_mux_determine_rate, ++}; ++EXPORT_SYMBOL_GPL(clk_regmap_mux_ops); ++ ++struct clk * ++devm_clk_regmap_register_mux(struct device *dev, const char *name, ++ const char * const *parent_names, u8 num_parents, ++ struct regmap *regmap, u32 reg, u8 shift, u8 width, ++ unsigned long flags) ++{ ++ struct clk_regmap_mux *mux; ++ struct clk_init_data init = {}; ++ ++ mux = devm_kzalloc(dev, sizeof(*mux), GFP_KERNEL); ++ if (!mux) ++ return ERR_PTR(-ENOMEM); ++ ++ init.name = name; ++ init.ops = &clk_regmap_mux_ops; ++ init.flags = flags; ++ init.parent_names = parent_names; ++ init.num_parents = num_parents; ++ ++ mux->dev = dev; ++ mux->regmap = regmap; ++ mux->reg = reg; ++ mux->shift = shift; ++ mux->mask = BIT(width) - 1; ++ mux->hw.init = &init; ++ ++ return devm_clk_register(dev, &mux->hw); ++} ++EXPORT_SYMBOL_GPL(devm_clk_regmap_register_mux); +diff --git a/drivers/clk/rockchip/regmap/clk-regmap-pll.c b/drivers/clk/rockchip/regmap/clk-regmap-pll.c +new file mode 100755 +index 000000000000..24ad7eda9d94 +--- /dev/null ++++ b/drivers/clk/rockchip/regmap/clk-regmap-pll.c +@@ -0,0 +1,363 @@ ++/* ++ * Copyright (c) 2017 Rockchip Electronics Co. Ltd. ++ * ++ * This program is free software; you can redistribute it and/or modify ++ * it under the terms of the GNU General Public License as published by ++ * the Free Software Foundation; either version 2 of the License, or ++ * (at your option) any later version. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ */ ++ ++#include "clk-regmap.h" ++ ++#define PLLCON_OFFSET(x) (x * 4) ++ ++#define PLL_BYPASS(x) HIWORD_UPDATE(x, 15, 15) ++#define PLL_BYPASS_MASK BIT(15) ++#define PLL_BYPASS_SHIFT 15 ++#define PLL_POSTDIV1(x) HIWORD_UPDATE(x, 14, 12) ++#define PLL_POSTDIV1_MASK GENMASK(14, 12) ++#define PLL_POSTDIV1_SHIFT 12 ++#define PLL_FBDIV(x) HIWORD_UPDATE(x, 11, 0) ++#define PLL_FBDIV_MASK GENMASK(11, 0) ++#define PLL_FBDIV_SHIFT 0 ++ ++#define PLL_POSTDIV2(x) HIWORD_UPDATE(x, 8, 6) ++#define PLL_POSTDIV2_MASK GENMASK(8, 6) ++#define PLL_POSTDIV2_SHIFT 6 ++#define PLL_REFDIV(x) HIWORD_UPDATE(x, 5, 0) ++#define PLL_REFDIV_MASK GENMASK(5, 0) ++#define PLL_REFDIV_SHIFT 0 ++ ++#define PLL_FOUT_4PHASE_CLK_POWER_DOWN BIT(27) ++#define PLL_FOUT_VCO_CLK_POWER_DOWN BIT(26) ++#define PLL_FOUT_POST_DIV_POWER_DOWN BIT(25) ++#define PLL_DAC_POWER_DOWN BIT(24) ++#define PLL_FRAC(x) UPDATE(x, 23, 0) ++#define PLL_FRAC_MASK GENMASK(23, 0) ++#define PLL_FRAC_SHIFT 0 ++ ++#define MIN_FREF_RATE 10000000UL ++#define MAX_FREF_RATE 800000000UL ++#define MIN_FREFDIV_RATE 1000000UL ++#define MAX_FREFDIV_RATE 40000000UL ++#define MIN_FVCO_RATE 400000000UL ++#define MAX_FVCO_RATE 1600000000UL ++#define MIN_FOUTPOSTDIV_RATE 8000000UL ++#define MAX_FOUTPOSTDIV_RATE 1600000000UL ++ ++struct clk_regmap_pll { ++ struct clk_hw hw; ++ struct device *dev; ++ struct regmap *regmap; ++ unsigned int reg; ++ u8 pd_shift; ++ u8 dsmpd_shift; ++ u8 lock_shift; ++}; ++ ++#define to_clk_regmap_pll(_hw) container_of(_hw, struct clk_regmap_pll, hw) ++ ++static unsigned long ++clk_regmap_pll_recalc_rate(struct clk_hw *hw, unsigned long prate) ++{ ++ struct clk_regmap_pll *pll = to_clk_regmap_pll(hw); ++ unsigned int postdiv1, fbdiv, dsmpd, postdiv2, refdiv, frac, bypass; ++ unsigned int con0, con1, con2; ++ u64 foutvco, foutpostdiv; ++ ++ regmap_read(pll->regmap, pll->reg + PLLCON_OFFSET(0), &con0); ++ regmap_read(pll->regmap, pll->reg + PLLCON_OFFSET(1), &con1); ++ regmap_read(pll->regmap, pll->reg + PLLCON_OFFSET(2), &con2); ++ ++ bypass = (con0 & PLL_BYPASS_MASK) >> PLL_BYPASS_SHIFT; ++ postdiv1 = (con0 & PLL_POSTDIV1_MASK) >> PLL_POSTDIV1_SHIFT; ++ fbdiv = (con0 & PLL_FBDIV_MASK) >> PLL_FBDIV_SHIFT; ++ dsmpd = (con1 & BIT(pll->dsmpd_shift)) >> pll->dsmpd_shift; ++ postdiv2 = (con1 & PLL_POSTDIV2_MASK) >> PLL_POSTDIV2_SHIFT; ++ refdiv = (con1 & PLL_REFDIV_MASK) >> PLL_REFDIV_SHIFT; ++ frac = (con2 & PLL_FRAC_MASK) >> PLL_FRAC_SHIFT; ++ ++ if (bypass) ++ return prate; ++ ++ foutvco = prate * fbdiv; ++ do_div(foutvco, refdiv); ++ ++ if (!dsmpd) { ++ u64 frac_rate = (u64)prate * frac; ++ ++ do_div(frac_rate, refdiv); ++ foutvco += frac_rate >> 24; ++ } ++ ++ foutpostdiv = foutvco; ++ do_div(foutpostdiv, postdiv1); ++ do_div(foutpostdiv, postdiv2); ++ ++ return foutpostdiv; ++} ++ ++static long clk_pll_round_rate(unsigned long fin, unsigned long fout, ++ u8 *refdiv, u16 *fbdiv, ++ u8 *postdiv1, u8 *postdiv2, ++ u32 *frac, u8 *dsmpd, u8 *bypass) ++{ ++ u8 min_refdiv, max_refdiv, postdiv; ++ u8 _dsmpd = 1, _postdiv1 = 0, _postdiv2 = 0, _refdiv = 0; ++ u16 _fbdiv = 0; ++ u32 _frac = 0; ++ u64 foutvco, foutpostdiv; ++ ++ /* ++ * FREF : 10MHz ~ 800MHz ++ * FREFDIV : 1MHz ~ 40MHz ++ * FOUTVCO : 400MHz ~ 1.6GHz ++ * FOUTPOSTDIV : 8MHz ~ 1.6GHz ++ */ ++ if (fin < MIN_FREF_RATE || fin > MAX_FREF_RATE) ++ return -EINVAL; ++ ++ if (fout < MIN_FOUTPOSTDIV_RATE || fout > MAX_FOUTPOSTDIV_RATE) ++ return -EINVAL; ++ ++ if (fin == fout) { ++ if (bypass) ++ *bypass = true; ++ return fin; ++ } ++ ++ min_refdiv = DIV_ROUND_UP(fin, MAX_FREFDIV_RATE); ++ max_refdiv = fin / MIN_FREFDIV_RATE; ++ if (max_refdiv > 64) ++ max_refdiv = 64; ++ ++ if (fout < MIN_FVCO_RATE) { ++ postdiv = DIV_ROUND_UP_ULL(MIN_FVCO_RATE, fout); ++ ++ for (_postdiv2 = 1; _postdiv2 < 8; _postdiv2++) { ++ if (postdiv % _postdiv2) ++ continue; ++ ++ _postdiv1 = postdiv / _postdiv2; ++ ++ if (_postdiv1 > 0 && _postdiv1 < 8) ++ break; ++ } ++ ++ if (_postdiv2 > 7) ++ return -EINVAL; ++ ++ fout *= _postdiv1 * _postdiv2; ++ } else { ++ _postdiv1 = 1; ++ _postdiv2 = 1; ++ } ++ ++ for (_refdiv = min_refdiv; _refdiv <= max_refdiv; _refdiv++) { ++ u64 tmp, frac_rate; ++ ++ if (fin % _refdiv) ++ continue; ++ ++ tmp = (u64)fout * _refdiv; ++ do_div(tmp, fin); ++ _fbdiv = tmp; ++ if (_fbdiv < 10 || _fbdiv > 1600) ++ continue; ++ ++ tmp = (u64)_fbdiv * fin; ++ do_div(tmp, _refdiv); ++ if (fout < MIN_FVCO_RATE || fout > MAX_FVCO_RATE) ++ continue; ++ ++ frac_rate = fout - tmp; ++ ++ if (frac_rate) { ++ tmp = (u64)frac_rate * _refdiv; ++ tmp <<= 24; ++ do_div(tmp, fin); ++ _frac = tmp; ++ _dsmpd = 0; ++ } ++ ++ break; ++ } ++ ++ /* ++ * If DSMPD = 1 (DSM is disabled, "integer mode") ++ * FOUTVCO = FREF / REFDIV * FBDIV ++ * FOUTPOSTDIV = FOUTVCO / POSTDIV1 / POSTDIV2 ++ * ++ * If DSMPD = 0 (DSM is enabled, "fractional mode") ++ * FOUTVCO = FREF / REFDIV * (FBDIV + FRAC / 2^24) ++ * FOUTPOSTDIV = FOUTVCO / POSTDIV1 / POSTDIV2 ++ */ ++ foutvco = fin * _fbdiv; ++ do_div(foutvco, _refdiv); ++ ++ if (!_dsmpd) { ++ u64 frac_rate = (u64)fin * _frac; ++ ++ do_div(frac_rate, _refdiv); ++ foutvco += frac_rate >> 24; ++ } ++ ++ foutpostdiv = foutvco; ++ do_div(foutpostdiv, _postdiv1); ++ do_div(foutpostdiv, _postdiv2); ++ ++ if (refdiv) ++ *refdiv = _refdiv; ++ if (fbdiv) ++ *fbdiv = _fbdiv; ++ if (postdiv1) ++ *postdiv1 = _postdiv1; ++ if (postdiv2) ++ *postdiv2 = _postdiv2; ++ if (frac) ++ *frac = _frac; ++ if (dsmpd) ++ *dsmpd = _dsmpd; ++ if (bypass) ++ *bypass = false; ++ ++ return (unsigned long)foutpostdiv; ++} ++ ++static long ++clk_regmap_pll_round_rate(struct clk_hw *hw, unsigned long drate, ++ unsigned long *prate) ++{ ++ struct clk_regmap_pll *pll = to_clk_regmap_pll(hw); ++ long rate; ++ ++ rate = clk_pll_round_rate(*prate, drate, NULL, NULL, NULL, NULL, NULL, ++ NULL, NULL); ++ ++ dev_dbg(pll->dev, "%s: prate=%ld, drate=%ld, rate=%ld\n", ++ clk_hw_get_name(hw), *prate, drate, rate); ++ ++ return rate; ++} ++ ++static int ++clk_regmap_pll_set_rate(struct clk_hw *hw, unsigned long drate, ++ unsigned long prate) ++{ ++ struct clk_regmap_pll *pll = to_clk_regmap_pll(hw); ++ u8 refdiv, postdiv1, postdiv2, dsmpd, bypass; ++ u16 fbdiv; ++ u32 frac; ++ long rate; ++ ++ rate = clk_pll_round_rate(prate, drate, &refdiv, &fbdiv, &postdiv1, ++ &postdiv2, &frac, &dsmpd, &bypass); ++ if (rate < 0) ++ return rate; ++ ++ dev_dbg(pll->dev, "%s: rate=%ld, bypass=%d\n", ++ clk_hw_get_name(hw), drate, bypass); ++ ++ if (bypass) { ++ regmap_write(pll->regmap, pll->reg + PLLCON_OFFSET(0), ++ PLL_BYPASS(1)); ++ } else { ++ regmap_write(pll->regmap, pll->reg + PLLCON_OFFSET(0), ++ PLL_BYPASS(0) | PLL_POSTDIV1(postdiv1) | ++ PLL_FBDIV(fbdiv)); ++ regmap_write(pll->regmap, pll->reg + PLLCON_OFFSET(1), ++ HIWORD_UPDATE(dsmpd, pll->dsmpd_shift, pll->dsmpd_shift) | ++ PLL_POSTDIV2(postdiv2) | PLL_REFDIV(refdiv)); ++ regmap_write(pll->regmap, pll->reg + PLLCON_OFFSET(2), ++ PLL_FRAC(frac)); ++ ++ dev_dbg(pll->dev, "refdiv=%d, fbdiv=%d, frac=%d\n", ++ refdiv, fbdiv, frac); ++ dev_dbg(pll->dev, "postdiv1=%d, postdiv2=%d\n", ++ postdiv1, postdiv2); ++ } ++ ++ return 0; ++} ++ ++static int clk_regmap_pll_prepare(struct clk_hw *hw) ++{ ++ struct clk_regmap_pll *pll = to_clk_regmap_pll(hw); ++ u32 v; ++ int ret; ++ ++ regmap_write(pll->regmap, pll->reg + PLLCON_OFFSET(1), ++ HIWORD_UPDATE(0, pll->pd_shift, pll->pd_shift)); ++ ++ ret = regmap_read_poll_timeout(pll->regmap, ++ pll->reg + PLLCON_OFFSET(1), ++ v, v & BIT(pll->lock_shift), 50, 50000); ++ if (ret) ++ dev_err(pll->dev, "%s is not lock\n", clk_hw_get_name(hw)); ++ ++ return 0; ++} ++ ++static void clk_regmap_pll_unprepare(struct clk_hw *hw) ++{ ++ struct clk_regmap_pll *pll = to_clk_regmap_pll(hw); ++ ++ regmap_write(pll->regmap, pll->reg + PLLCON_OFFSET(1), ++ HIWORD_UPDATE(1, pll->pd_shift, pll->pd_shift)); ++} ++ ++static int clk_regmap_pll_is_prepared(struct clk_hw *hw) ++{ ++ struct clk_regmap_pll *pll = to_clk_regmap_pll(hw); ++ unsigned int con1; ++ ++ regmap_read(pll->regmap, pll->reg + PLLCON_OFFSET(1), &con1); ++ ++ return !(con1 & BIT(pll->pd_shift)); ++} ++ ++static const struct clk_ops clk_regmap_pll_ops = { ++ .recalc_rate = clk_regmap_pll_recalc_rate, ++ .round_rate = clk_regmap_pll_round_rate, ++ .set_rate = clk_regmap_pll_set_rate, ++ .prepare = clk_regmap_pll_prepare, ++ .unprepare = clk_regmap_pll_unprepare, ++ .is_prepared = clk_regmap_pll_is_prepared, ++}; ++ ++struct clk * ++devm_clk_regmap_register_pll(struct device *dev, const char *name, ++ const char *parent_name, ++ struct regmap *regmap, u32 reg, u8 pd_shift, ++ u8 dsmpd_shift, u8 lock_shift, ++ unsigned long flags) ++{ ++ struct clk_regmap_pll *pll; ++ struct clk_init_data init = {}; ++ ++ pll = devm_kzalloc(dev, sizeof(*pll), GFP_KERNEL); ++ if (!pll) ++ return ERR_PTR(-ENOMEM); ++ ++ init.name = name; ++ init.ops = &clk_regmap_pll_ops; ++ init.flags = flags; ++ init.parent_names = (parent_name ? &parent_name : NULL); ++ init.num_parents = (parent_name ? 1 : 0); ++ ++ pll->dev = dev; ++ pll->regmap = regmap; ++ pll->reg = reg; ++ pll->pd_shift = pd_shift; ++ pll->dsmpd_shift = dsmpd_shift; ++ pll->lock_shift = lock_shift; ++ pll->hw.init = &init; ++ ++ return devm_clk_register(dev, &pll->hw); ++} ++EXPORT_SYMBOL_GPL(devm_clk_regmap_register_pll); +diff --git a/drivers/clk/rockchip/regmap/clk-regmap.h b/drivers/clk/rockchip/regmap/clk-regmap.h +new file mode 100755 +index 000000000000..255c0201a590 +--- /dev/null ++++ b/drivers/clk/rockchip/regmap/clk-regmap.h +@@ -0,0 +1,311 @@ ++/* ++ * Copyright (c) 2017 Rockchip Electronics Co. Ltd. ++ * ++ * This program is free software; you can redistribute it and/or modify ++ * it under the terms of the GNU General Public License as published by ++ * the Free Software Foundation; either version 2 of the License, or ++ * (at your option) any later version. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ */ ++ ++#ifndef __CLK_REGMAP_H__ ++#define __CLK_REGMAP_H__ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#define UPDATE(x, h, l) (((x) << (l)) & GENMASK((h), (l))) ++#define HIWORD_UPDATE(v, h, l) (((v) << (l)) | (GENMASK((h), (l)) << 16)) ++ ++struct clk_pll_data { ++ unsigned int id; ++ const char *name; ++ const char *parent_name; ++ u32 reg; ++ u8 pd_shift; ++ u8 dsmpd_shift; ++ u8 lock_shift; ++ unsigned long flags; ++}; ++ ++#define PLL(_id, _name, _parent_name, _reg, _pd_shift, _dsmpd_shift, \ ++ _lock_shift, _flags) \ ++{ \ ++ .id = _id, \ ++ .name = _name, \ ++ .parent_name = _parent_name, \ ++ .reg = _reg, \ ++ .pd_shift = _pd_shift, \ ++ .dsmpd_shift = _dsmpd_shift, \ ++ .lock_shift = _lock_shift, \ ++ .flags = _flags, \ ++} ++ ++#define RK618_PLL(_id, _name, _parent_name, _reg, _flags) \ ++ PLL(_id, _name, _parent_name, _reg, 10, 9, 15, _flags) ++ ++#define RK628_PLL(_id, _name, _parent_name, _reg, _flags) \ ++ PLL(_id, _name, _parent_name, _reg, 13, 12, 10, _flags) ++ ++struct clk_mux_data { ++ unsigned int id; ++ const char *name; ++ const char *const *parent_names; ++ u8 num_parents; ++ u32 reg; ++ u8 shift; ++ u8 width; ++ unsigned long flags; ++}; ++ ++#define MUX(_id, _name, _parent_names, _reg, _shift, _width, _flags) \ ++{ \ ++ .id = _id, \ ++ .name = _name, \ ++ .parent_names = _parent_names, \ ++ .num_parents = ARRAY_SIZE(_parent_names), \ ++ .reg = _reg, \ ++ .shift = _shift, \ ++ .width = _width, \ ++ .flags = _flags, \ ++} ++ ++struct clk_gate_data { ++ unsigned int id; ++ const char *name; ++ const char *parent_name; ++ u32 reg; ++ u8 shift; ++ unsigned long flags; ++}; ++ ++#define GATE(_id, _name, _parent_name, _reg, _shift, _flags) \ ++{ \ ++ .id = _id, \ ++ .name = _name, \ ++ .parent_name = _parent_name, \ ++ .reg = _reg, \ ++ .shift = _shift, \ ++ .flags = _flags, \ ++} ++ ++struct clk_divider_data { ++ unsigned int id; ++ const char *name; ++ const char *parent_name; ++ u32 reg; ++ u8 shift; ++ u8 width; ++ unsigned long flags; ++}; ++ ++#define DIV(_id, _name, _parent_name, _reg, _shift, _width, _flags) \ ++{ \ ++ .id = _id, \ ++ .name = _name, \ ++ .parent_name = _parent_name, \ ++ .reg = _reg, \ ++ .shift = _shift, \ ++ .width = _width, \ ++ .flags = _flags, \ ++} ++ ++struct clk_composite_data { ++ unsigned int id; ++ const char *name; ++ const char *const *parent_names; ++ u8 num_parents; ++ u32 mux_reg; ++ u8 mux_shift; ++ u8 mux_width; ++ u32 div_reg; ++ u8 div_shift; ++ u8 div_width; ++ u8 div_flags; ++ u32 gate_reg; ++ u8 gate_shift; ++ unsigned long flags; ++}; ++ ++#define COMPOSITE(_id, _name, _parent_names, \ ++ _mux_reg, _mux_shift, _mux_width, \ ++ _div_reg, _div_shift, _div_width, \ ++ _gate_reg, _gate_shift, _flags) \ ++{ \ ++ .id = _id, \ ++ .name = _name, \ ++ .parent_names = _parent_names, \ ++ .num_parents = ARRAY_SIZE(_parent_names), \ ++ .mux_reg = _mux_reg, \ ++ .mux_shift = _mux_shift, \ ++ .mux_width = _mux_width, \ ++ .div_reg = _div_reg, \ ++ .div_shift = _div_shift, \ ++ .div_width = _div_width, \ ++ .div_flags = CLK_DIVIDER_HIWORD_MASK, \ ++ .gate_reg = _gate_reg, \ ++ .gate_shift = _gate_shift, \ ++ .flags = _flags, \ ++} ++ ++#define COMPOSITE_NOMUX(_id, _name, _parent_name, \ ++ _div_reg, _div_shift, _div_width, \ ++ _gate_reg, _gate_shift, _flags) \ ++{ \ ++ .id = _id, \ ++ .name = _name, \ ++ .parent_names = (const char *[]){ _parent_name }, \ ++ .num_parents = 1, \ ++ .div_reg = _div_reg, \ ++ .div_shift = _div_shift, \ ++ .div_width = _div_width, \ ++ .div_flags = CLK_DIVIDER_HIWORD_MASK, \ ++ .gate_reg = _gate_reg, \ ++ .gate_shift = _gate_shift, \ ++ .flags = _flags, \ ++} ++ ++#define COMPOSITE_NODIV(_id, _name, _parent_names, \ ++ _mux_reg, _mux_shift, _mux_width, \ ++ _gate_reg, _gate_shift, _flags) \ ++ COMPOSITE(_id, _name, _parent_names, \ ++ _mux_reg, _mux_shift, _mux_width, \ ++ 0, 0, 0, \ ++ _gate_reg, _gate_shift, _flags) ++ ++#define COMPOSITE_FRAC(_id, _name, _parent_names, \ ++ _mux_reg, _mux_shift, _mux_width, \ ++ _div_reg, \ ++ _gate_reg, _gate_shift, _flags) \ ++{ \ ++ .id = _id, \ ++ .name = _name, \ ++ .parent_names = _parent_names, \ ++ .num_parents = ARRAY_SIZE(_parent_names), \ ++ .mux_reg = _mux_reg, \ ++ .mux_shift = _mux_shift, \ ++ .mux_width = _mux_width, \ ++ .div_reg = _div_reg, \ ++ .gate_reg = _gate_reg, \ ++ .gate_shift = _gate_shift, \ ++ .flags = _flags, \ ++} ++ ++#define COMPOSITE_FRAC_NOMUX(_id, _name, _parent_name, \ ++ _div_reg, \ ++ _gate_reg, _gate_shift, _flags) \ ++{ \ ++ .id = _id, \ ++ .name = _name, \ ++ .parent_names = (const char *[]){ _parent_name }, \ ++ .num_parents = 1, \ ++ .div_reg = _div_reg, \ ++ .gate_reg = _gate_reg, \ ++ .gate_shift = _gate_shift, \ ++ .flags = _flags, \ ++} ++ ++#define COMPOSITE_FRAC_NOGATE(_id, _name, _parent_names, \ ++ _mux_reg, _mux_shift, _mux_width, \ ++ _div_reg, \ ++ _flags) \ ++ COMPOSITE_FRAC(_id, _name, _parent_names, \ ++ _mux_reg, _mux_shift, _mux_width, \ ++ _div_reg, 0, 0, _flags) ++ ++struct clk_regmap_fractional_divider { ++ struct clk_hw hw; ++ struct device *dev; ++ struct regmap *regmap; ++ u32 reg; ++ u8 mshift; ++ u8 mwidth; ++ u32 mmask; ++ u8 nshift; ++ u8 nwidth; ++ u32 nmask; ++}; ++ ++struct clk_regmap_divider { ++ struct clk_hw hw; ++ struct device *dev; ++ struct regmap *regmap; ++ u32 reg; ++ u8 shift; ++ u8 width; ++}; ++ ++struct clk_regmap_gate { ++ struct clk_hw hw; ++ struct device *dev; ++ struct regmap *regmap; ++ u32 reg; ++ u8 shift; ++}; ++ ++struct clk_regmap_mux { ++ struct clk_hw hw; ++ struct device *dev; ++ struct regmap *regmap; ++ u32 reg; ++ u32 mask; ++ u8 shift; ++}; ++ ++extern const struct clk_ops clk_regmap_mux_ops; ++extern const struct clk_ops clk_regmap_divider_ops; ++extern const struct clk_ops clk_regmap_gate_ops; ++extern const struct clk_ops clk_regmap_fractional_divider_ops; ++ ++struct clk * ++devm_clk_regmap_register_pll(struct device *dev, const char *name, ++ const char *parent_name, ++ struct regmap *regmap, u32 reg, u8 pd_shift, ++ u8 dsmpd_shift, u8 lock_shift, ++ unsigned long flags); ++ ++struct clk * ++devm_clk_regmap_register_mux(struct device *dev, const char *name, ++ const char * const *parent_names, u8 num_parents, ++ struct regmap *regmap, u32 reg, u8 shift, u8 width, ++ unsigned long flags); ++ ++struct clk * ++devm_clk_regmap_register_divider(struct device *dev, const char *name, ++ const char *parent_name, struct regmap *regmap, ++ u32 reg, u8 shift, u8 width, ++ unsigned long flags); ++ ++struct clk * ++devm_clk_regmap_register_gate(struct device *dev, const char *name, ++ const char *parent_name, ++ struct regmap *regmap, u32 reg, u8 shift, ++ unsigned long flags); ++ ++struct clk * ++devm_clk_regmap_register_fractional_divider(struct device *dev, ++ const char *name, ++ const char *parent_name, ++ struct regmap *regmap, ++ u32 reg, unsigned long flags); ++ ++struct clk * ++devm_clk_regmap_register_composite(struct device *dev, const char *name, ++ const char *const *parent_names, ++ u8 num_parents, struct regmap *regmap, ++ u32 mux_reg, u8 mux_shift, u8 mux_width, ++ u32 div_reg, u8 div_shift, u8 div_width, ++ u8 div_flags, ++ u32 gate_reg, u8 gate_shift, ++ unsigned long flags); ++ ++#endif +diff --git a/drivers/clk/rockchip/regmap/clk-rk618.c b/drivers/clk/rockchip/regmap/clk-rk618.c +new file mode 100755 +index 000000000000..c780f502b354 +--- /dev/null ++++ b/drivers/clk/rockchip/regmap/clk-rk618.c +@@ -0,0 +1,408 @@ ++/* ++ * Copyright (c) 2017 Rockchip Electronics Co. Ltd. ++ * ++ * This program is free software; you can redistribute it and/or modify ++ * it under the terms of the GNU General Public License as published by ++ * the Free Software Foundation; either version 2 of the License, or ++ * (at your option) any later version. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++ ++#include "clk-regmap.h" ++ ++#define RK618_CRU_CLKSEL0 0x0058 ++#define RK618_CRU_CLKSEL1 0x005c ++#define RK618_CRU_CLKSEL2 0x0060 ++#define RK618_CRU_CLKSEL3 0x0064 ++#define RK618_CRU_PLL0_CON0 0x0068 ++#define RK618_CRU_PLL0_CON1 0x006c ++#define RK618_CRU_PLL0_CON2 0x0070 ++#define RK618_CRU_PLL1_CON0 0x0074 ++#define RK618_CRU_PLL1_CON1 0x0078 ++#define RK618_CRU_PLL1_CON2 0x007c ++ ++enum { ++ LCDC0_CLK = 1, ++ LCDC1_CLK, ++ VIF_PLLIN_CLK, ++ SCALER_PLLIN_CLK, ++ VIF_PLL_CLK, ++ SCALER_PLL_CLK, ++ VIF0_CLK, ++ VIF1_CLK, ++ SCALER_IN_CLK, ++ SCALER_CLK, ++ DITHER_CLK, ++ HDMI_CLK, ++ MIPI_CLK, ++ LVDS_CLK, ++ LVTTL_CLK, ++ RGB_CLK, ++ VIF0_PRE_CLK, ++ VIF1_PRE_CLK, ++ CODEC_CLK, ++ NR_CLKS, ++}; ++ ++struct rk618_cru { ++ struct device *dev; ++ struct rk618 *parent; ++ struct regmap *regmap; ++ ++ struct clk_onecell_data clk_data; ++}; ++ ++static char clkin_name[32] = "dummy"; ++static char lcdc0_dclkp_name[32] = "dummy"; ++static char lcdc1_dclkp_name[32] = "dummy"; ++ ++#define PNAME(x) static const char *const x[] ++ ++PNAME(mux_pll_in_p) = { "lcdc0_clk", "lcdc1_clk", clkin_name }; ++PNAME(mux_pll_src_p) = { "vif_pll_clk", "scaler_pll_clk", }; ++PNAME(mux_scaler_in_src_p) = { "vif0_clk", "vif1_clk" }; ++PNAME(mux_hdmi_src_p) = { "vif1_clk", "scaler_clk", "vif0_clk" }; ++PNAME(mux_dither_src_p) = { "vif0_clk", "scaler_clk" }; ++PNAME(mux_vif0_src_p) = { "vif0_pre_clk", lcdc0_dclkp_name }; ++PNAME(mux_vif1_src_p) = { "vif1_pre_clk", lcdc1_dclkp_name }; ++PNAME(mux_codec_src_p) = { "codec_pre_clk", clkin_name }; ++ ++/* Two PLL, one for dual datarate input logic, the other for scaler */ ++static const struct clk_pll_data rk618_clk_plls[] = { ++ RK618_PLL(VIF_PLL_CLK, "vif_pll_clk", "vif_pllin_clk", ++ RK618_CRU_PLL0_CON0, ++ 0), ++ RK618_PLL(SCALER_PLL_CLK, "scaler_pll_clk", "scaler_pllin_clk", ++ RK618_CRU_PLL1_CON0, ++ 0), ++}; ++ ++static const struct clk_mux_data rk618_clk_muxes[] = { ++ MUX(VIF_PLLIN_CLK, "vif_pllin_clk", mux_pll_in_p, ++ RK618_CRU_CLKSEL0, 6, 2, ++ 0), ++ MUX(SCALER_PLLIN_CLK, "scaler_pllin_clk", mux_pll_in_p, ++ RK618_CRU_CLKSEL0, 8, 2, ++ 0), ++ MUX(SCALER_IN_CLK, "scaler_in_clk", mux_scaler_in_src_p, ++ RK618_CRU_CLKSEL3, 15, 1, ++ 0), ++ MUX(DITHER_CLK, "dither_clk", mux_dither_src_p, ++ RK618_CRU_CLKSEL3, 14, 1, ++ 0), ++ MUX(VIF0_CLK, "vif0_clk", mux_vif0_src_p, ++ RK618_CRU_CLKSEL3, 1, 1, ++ CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT), ++ MUX(VIF1_CLK, "vif1_clk", mux_vif1_src_p, ++ RK618_CRU_CLKSEL3, 7, 1, ++ CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT), ++ MUX(CODEC_CLK, "codec_clk", mux_codec_src_p, ++ RK618_CRU_CLKSEL1, 1, 1, ++ CLK_SET_RATE_PARENT), ++}; ++ ++static const struct clk_divider_data rk618_clk_dividers[] = { ++ DIV(LCDC0_CLK, "lcdc0_clk", lcdc0_dclkp_name, ++ RK618_CRU_CLKSEL0, 0, 3, ++ 0), ++ DIV(LCDC1_CLK, "lcdc1_clk", lcdc1_dclkp_name, ++ RK618_CRU_CLKSEL0, 3, 3, ++ 0), ++}; ++ ++static const struct clk_gate_data rk618_clk_gates[] = { ++ GATE(MIPI_CLK, "mipi_clk", "dither_clk", ++ RK618_CRU_CLKSEL1, 10, ++ CLK_IGNORE_UNUSED), ++ GATE(LVDS_CLK, "lvds_clk", "dither_clk", ++ RK618_CRU_CLKSEL1, 9, ++ CLK_IGNORE_UNUSED), ++ GATE(LVTTL_CLK, "lvttl_clk", "dither_clk", ++ RK618_CRU_CLKSEL1, 12, ++ 0), ++ GATE(RGB_CLK, "rgb_clk", "dither_clk", ++ RK618_CRU_CLKSEL1, 11, ++ 0), ++}; ++ ++static const struct clk_composite_data rk618_clk_composites[] = { ++ COMPOSITE(SCALER_CLK, "scaler_clk", mux_pll_src_p, ++ RK618_CRU_CLKSEL1, 3, 1, ++ RK618_CRU_CLKSEL1, 5, 3, ++ RK618_CRU_CLKSEL1, 4, ++ CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT), ++ COMPOSITE_NODIV(HDMI_CLK, "hdmi_clk", mux_hdmi_src_p, ++ RK618_CRU_CLKSEL3, 12, 2, ++ RK618_CRU_CLKSEL1, 8, ++ 0), ++ COMPOSITE(VIF0_PRE_CLK, "vif0_pre_clk", mux_pll_src_p, ++ RK618_CRU_CLKSEL3, 0, 1, ++ RK618_CRU_CLKSEL3, 3, 3, ++ RK618_CRU_CLKSEL3, 2, ++ CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT), ++ COMPOSITE(VIF1_PRE_CLK, "vif1_pre_clk", mux_pll_src_p, ++ RK618_CRU_CLKSEL3, 6, 1, ++ RK618_CRU_CLKSEL3, 9, 3, ++ RK618_CRU_CLKSEL3, 8, ++ CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT), ++ COMPOSITE_FRAC_NOGATE(0, "codec_pre_clk", mux_pll_src_p, ++ RK618_CRU_CLKSEL1, 0, 1, ++ RK618_CRU_CLKSEL2, ++ 0), ++}; ++ ++static void rk618_clk_add_lookup(struct rk618_cru *cru, struct clk *clk, ++ unsigned int id) ++{ ++ if (cru->clk_data.clks && id) ++ cru->clk_data.clks[id] = clk; ++} ++ ++static void rk618_clk_register_muxes(struct rk618_cru *cru) ++{ ++ struct clk *clk; ++ unsigned int i; ++ ++ for (i = 0; i < ARRAY_SIZE(rk618_clk_muxes); i++) { ++ const struct clk_mux_data *data = &rk618_clk_muxes[i]; ++ ++ clk = devm_clk_regmap_register_mux(cru->dev, data->name, ++ data->parent_names, ++ data->num_parents, ++ cru->regmap, data->reg, ++ data->shift, data->width, ++ data->flags); ++ if (IS_ERR(clk)) { ++ dev_err(cru->dev, "failed to register clock %s\n", ++ data->name); ++ continue; ++ } ++ ++ rk618_clk_add_lookup(cru, clk, data->id); ++ } ++} ++ ++static void rk618_clk_register_dividers(struct rk618_cru *cru) ++{ ++ struct clk *clk; ++ unsigned int i; ++ ++ for (i = 0; i < ARRAY_SIZE(rk618_clk_dividers); i++) { ++ const struct clk_divider_data *data = &rk618_clk_dividers[i]; ++ ++ clk = devm_clk_regmap_register_divider(cru->dev, data->name, ++ data->parent_name, ++ cru->regmap, data->reg, ++ data->shift, data->width, ++ data->flags); ++ if (IS_ERR(clk)) { ++ dev_err(cru->dev, "failed to register clock %s\n", ++ data->name); ++ continue; ++ } ++ ++ rk618_clk_add_lookup(cru, clk, data->id); ++ } ++} ++ ++static void rk618_clk_register_gates(struct rk618_cru *cru) ++{ ++ struct clk *clk; ++ unsigned int i; ++ ++ for (i = 0; i < ARRAY_SIZE(rk618_clk_gates); i++) { ++ const struct clk_gate_data *data = &rk618_clk_gates[i]; ++ ++ clk = devm_clk_regmap_register_gate(cru->dev, data->name, ++ data->parent_name, ++ cru->regmap, ++ data->reg, data->shift, ++ data->flags); ++ if (IS_ERR(clk)) { ++ dev_err(cru->dev, "failed to register clock %s\n", ++ data->name); ++ continue; ++ } ++ ++ rk618_clk_add_lookup(cru, clk, data->id); ++ } ++} ++ ++static void rk618_clk_register_composites(struct rk618_cru *cru) ++{ ++ struct clk *clk; ++ unsigned int i; ++ ++ for (i = 0; i < ARRAY_SIZE(rk618_clk_composites); i++) { ++ const struct clk_composite_data *data = ++ &rk618_clk_composites[i]; ++ ++ clk = devm_clk_regmap_register_composite(cru->dev, data->name, ++ data->parent_names, ++ data->num_parents, ++ cru->regmap, ++ data->mux_reg, ++ data->mux_shift, ++ data->mux_width, ++ data->div_reg, ++ data->div_shift, ++ data->div_width, ++ data->div_flags, ++ data->gate_reg, ++ data->gate_shift, ++ data->flags); ++ if (IS_ERR(clk)) { ++ dev_err(cru->dev, "failed to register clock %s\n", ++ data->name); ++ continue; ++ } ++ ++ rk618_clk_add_lookup(cru, clk, data->id); ++ } ++} ++ ++static void rk618_clk_register_plls(struct rk618_cru *cru) ++{ ++ struct clk *clk; ++ unsigned int i; ++ ++ for (i = 0; i < ARRAY_SIZE(rk618_clk_plls); i++) { ++ const struct clk_pll_data *data = &rk618_clk_plls[i]; ++ ++ clk = devm_clk_regmap_register_pll(cru->dev, data->name, ++ data->parent_name, ++ cru->regmap, ++ data->reg, ++ data->pd_shift, ++ data->dsmpd_shift, ++ data->lock_shift, ++ data->flags); ++ if (IS_ERR(clk)) { ++ dev_err(cru->dev, "failed to register clock %s\n", ++ data->name); ++ continue; ++ } ++ ++ rk618_clk_add_lookup(cru, clk, data->id); ++ } ++} ++ ++static int rk618_cru_probe(struct platform_device *pdev) ++{ ++ struct rk618 *rk618 = dev_get_drvdata(pdev->dev.parent); ++ struct device *dev = &pdev->dev; ++ struct rk618_cru *cru; ++ struct clk **clk_table; ++ const char *parent_name; ++ struct clk *clk; ++ int ret, i; ++ ++ if (!of_device_is_available(dev->of_node)) ++ return -ENODEV; ++ ++ cru = devm_kzalloc(dev, sizeof(*cru), GFP_KERNEL); ++ if (!cru) ++ return -ENOMEM; ++ ++ clk_table = devm_kcalloc(dev, NR_CLKS, sizeof(struct clk *), ++ GFP_KERNEL); ++ if (!clk_table) ++ return -ENOMEM; ++ ++ for (i = 0; i < NR_CLKS; i++) ++ clk_table[i] = ERR_PTR(-ENOENT); ++ ++ cru->dev = dev; ++ cru->parent = rk618; ++ cru->regmap = rk618->regmap; ++ cru->clk_data.clks = clk_table; ++ cru->clk_data.clk_num = NR_CLKS; ++ platform_set_drvdata(pdev, cru); ++ ++ clk = devm_clk_get(dev, "clkin"); ++ if (IS_ERR(clk)) { ++ ret = PTR_ERR(clk); ++ dev_err(dev, "failed to get clkin: %d\n", ret); ++ return ret; ++ } ++ ++ strlcpy(clkin_name, __clk_get_name(clk), sizeof(clkin_name)); ++ ++ clk = devm_clk_get(dev, "lcdc0_dclkp"); ++ if (IS_ERR(clk)) { ++ if (PTR_ERR(clk) != -ENOENT) { ++ ret = PTR_ERR(clk); ++ dev_err(dev, "failed to get lcdc0_dclkp: %d\n", ret); ++ return ret; ++ } ++ ++ clk = NULL; ++ } ++ ++ parent_name = __clk_get_name(clk); ++ if (parent_name) ++ strlcpy(lcdc0_dclkp_name, parent_name, ++ sizeof(lcdc0_dclkp_name)); ++ ++ clk = devm_clk_get(dev, "lcdc1_dclkp"); ++ if (IS_ERR(clk)) { ++ if (PTR_ERR(clk) != -ENOENT) { ++ ret = PTR_ERR(clk); ++ dev_err(dev, "failed to get lcdc1_dclkp: %d\n", ret); ++ return ret; ++ } ++ ++ clk = NULL; ++ } ++ ++ parent_name = __clk_get_name(clk); ++ if (parent_name) ++ strlcpy(lcdc1_dclkp_name, parent_name, ++ sizeof(lcdc1_dclkp_name)); ++ ++ rk618_clk_register_plls(cru); ++ rk618_clk_register_muxes(cru); ++ rk618_clk_register_dividers(cru); ++ rk618_clk_register_gates(cru); ++ rk618_clk_register_composites(cru); ++ ++ return of_clk_add_provider(dev->of_node, of_clk_src_onecell_get, ++ &cru->clk_data); ++} ++ ++static int rk618_cru_remove(struct platform_device *pdev) ++{ ++ of_clk_del_provider(pdev->dev.of_node); ++ ++ return 0; ++} ++ ++static const struct of_device_id rk618_cru_of_match[] = { ++ { .compatible = "rockchip,rk618-cru", }, ++ {}, ++}; ++MODULE_DEVICE_TABLE(of, rk618_cru_of_match); ++ ++static struct platform_driver rk618_cru_driver = { ++ .driver = { ++ .name = "rk618-cru", ++ .of_match_table = of_match_ptr(rk618_cru_of_match), ++ }, ++ .probe = rk618_cru_probe, ++ .remove = rk618_cru_remove, ++}; ++module_platform_driver(rk618_cru_driver); ++ ++MODULE_AUTHOR("Wyon Bi "); ++MODULE_DESCRIPTION("Rockchip rk618 CRU driver"); ++MODULE_LICENSE("GPL v2"); +diff --git a/drivers/clk/rockchip/regmap/clk-rk628.c b/drivers/clk/rockchip/regmap/clk-rk628.c +new file mode 100755 +index 000000000000..4c3a9eac0e0c +--- /dev/null ++++ b/drivers/clk/rockchip/regmap/clk-rk628.c +@@ -0,0 +1,569 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co. Ltd. ++ * ++ * Author: Wyon Bi ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "clk-regmap.h" ++ ++#define REG(x) ((x) + 0xc0000) ++ ++#define CRU_CPLL_CON0 REG(0x0000) ++#define CRU_CPLL_CON1 REG(0x0004) ++#define CRU_CPLL_CON2 REG(0x0008) ++#define CRU_CPLL_CON3 REG(0x000c) ++#define CRU_CPLL_CON4 REG(0x0010) ++#define CRU_GPLL_CON0 REG(0x0020) ++#define CRU_GPLL_CON1 REG(0x0024) ++#define CRU_GPLL_CON2 REG(0x0028) ++#define CRU_GPLL_CON3 REG(0x002c) ++#define CRU_GPLL_CON4 REG(0x0030) ++#define CRU_MODE_CON REG(0x0060) ++#define CRU_CLKSEL_CON00 REG(0x0080) ++#define CRU_CLKSEL_CON01 REG(0x0084) ++#define CRU_CLKSEL_CON02 REG(0x0088) ++#define CRU_CLKSEL_CON03 REG(0x008c) ++#define CRU_CLKSEL_CON04 REG(0x0090) ++#define CRU_CLKSEL_CON05 REG(0x0094) ++#define CRU_CLKSEL_CON06 REG(0x0098) ++#define CRU_CLKSEL_CON07 REG(0x009c) ++#define CRU_CLKSEL_CON08 REG(0x00a0) ++#define CRU_CLKSEL_CON09 REG(0x00a4) ++#define CRU_CLKSEL_CON10 REG(0x00a8) ++#define CRU_CLKSEL_CON11 REG(0x00ac) ++#define CRU_CLKSEL_CON12 REG(0x00b0) ++#define CRU_CLKSEL_CON13 REG(0x00b4) ++#define CRU_CLKSEL_CON14 REG(0x00b8) ++#define CRU_CLKSEL_CON15 REG(0x00bc) ++#define CRU_CLKSEL_CON16 REG(0x00c0) ++#define CRU_CLKSEL_CON17 REG(0x00c4) ++#define CRU_CLKSEL_CON18 REG(0x00c8) ++#define CRU_CLKSEL_CON20 REG(0x00d0) ++#define CRU_CLKSEL_CON21 REG(0x00d4) ++#define CRU_GATE_CON00 REG(0x0180) ++#define CRU_GATE_CON01 REG(0x0184) ++#define CRU_GATE_CON02 REG(0x0188) ++#define CRU_GATE_CON03 REG(0x018c) ++#define CRU_GATE_CON04 REG(0x0190) ++#define CRU_GATE_CON05 REG(0x0194) ++#define CRU_SOFTRST_CON00 REG(0x0200) ++#define CRU_SOFTRST_CON01 REG(0x0204) ++#define CRU_SOFTRST_CON02 REG(0x0208) ++#define CRU_SOFTRST_CON04 REG(0x0210) ++#define CRU_MAX_REGISTER CRU_SOFTRST_CON04 ++ ++#define reset_to_cru(_rst) container_of(_rst, struct rk628_cru, rcdev) ++ ++struct rk628_cru { ++ struct device *dev; ++ struct rk628 *parent; ++ struct regmap *regmap; ++ struct reset_controller_dev rcdev; ++ struct clk_onecell_data clk_data; ++}; ++ ++#define CNAME(x) "rk628_" x ++ ++#define PNAME(x) static const char *const x[] ++ ++PNAME(mux_cpll_osc_p) = { "xin_osc0_func", CNAME("clk_cpll") }; ++PNAME(mux_gpll_osc_p) = { "xin_osc0_func", CNAME("clk_gpll") }; ++PNAME(mux_cpll_gpll_mux_p) = { CNAME("clk_cpll_mux"), CNAME("clk_gpll_mux") }; ++PNAME(mux_mclk_i2s_8ch_p) = { CNAME("clk_i2s_8ch_src"), CNAME("clk_i2s_8ch_frac"), "i2s_mclkin", "xin_osc0_half" }; ++PNAME(mux_i2s_mclkout_p) = { CNAME("mclk_i2s_8ch"), "xin_osc0_half" }; ++ ++static const struct clk_pll_data rk628_clk_plls[] = { ++ RK628_PLL(CGU_CLK_CPLL, CNAME("clk_cpll"), "xin_osc0_func", ++ CRU_CPLL_CON0, ++ 0), ++ RK628_PLL(CGU_CLK_GPLL, CNAME("clk_gpll"), "xin_osc0_func", ++ CRU_GPLL_CON0, ++ 0), ++}; ++ ++static const struct clk_mux_data rk628_clk_muxes[] = { ++ MUX(CGU_CLK_CPLL_MUX, CNAME("clk_cpll_mux"), mux_cpll_osc_p, ++ CRU_MODE_CON, 0, 1, ++ 0), ++ MUX(CGU_CLK_GPLL_MUX, CNAME("clk_gpll_mux"), mux_gpll_osc_p, ++ CRU_MODE_CON, 2, 1, ++ CLK_SET_RATE_NO_REPARENT | CLK_SET_RATE_PARENT), ++}; ++ ++static const struct clk_gate_data rk628_clk_gates[] = { ++ GATE(CGU_PCLK_GPIO0, CNAME("pclk_gpio0"), CNAME("pclk_logic"), ++ CRU_GATE_CON01, 0, ++ 0), ++ GATE(CGU_PCLK_GPIO1, CNAME("pclk_gpio1"), CNAME("pclk_logic"), ++ CRU_GATE_CON01, 1, ++ 0), ++ GATE(CGU_PCLK_GPIO2, CNAME("pclk_gpio2"), CNAME("pclk_logic"), ++ CRU_GATE_CON01, 2, ++ 0), ++ GATE(CGU_PCLK_GPIO3, CNAME("pclk_gpio3"), CNAME("pclk_logic"), ++ CRU_GATE_CON01, 3, ++ 0), ++ ++ GATE(CGU_PCLK_TXPHY_CON, CNAME("pclk_txphy_con"), CNAME("pclk_logic"), ++ CRU_GATE_CON02, 3, ++ CLK_IGNORE_UNUSED), ++ GATE(CGU_PCLK_EFUSE, CNAME("pclk_efuse"), CNAME("pclk_logic"), ++ CRU_GATE_CON00, 5, ++ 0), ++ GATE(0, CNAME("pclk_i2c2apb"), CNAME("pclk_logic"), ++ CRU_GATE_CON00, 3, ++ CLK_IGNORE_UNUSED), ++ GATE(0, CNAME("pclk_cru"), CNAME("pclk_logic"), ++ CRU_GATE_CON00, 1, ++ CLK_IGNORE_UNUSED), ++ GATE(0, CNAME("pclk_adapter"), CNAME("pclk_logic"), ++ CRU_GATE_CON00, 7, ++ CLK_IGNORE_UNUSED), ++ GATE(0, CNAME("pclk_regfile"), CNAME("pclk_logic"), ++ CRU_GATE_CON00, 2, ++ CLK_IGNORE_UNUSED), ++ GATE(CGU_PCLK_DSI0, CNAME("pclk_dsi0"), CNAME("pclk_logic"), ++ CRU_GATE_CON02, 6, ++ 0), ++ GATE(CGU_PCLK_DSI1, CNAME("pclk_dsi1"), CNAME("pclk_logic"), ++ CRU_GATE_CON02, 7, ++ 0), ++ GATE(CGU_PCLK_CSI, CNAME("pclk_csi"), CNAME("pclk_logic"), ++ CRU_GATE_CON02, 8, ++ 0), ++ GATE(CGU_PCLK_HDMITX, CNAME("pclk_hdmitx"), CNAME("pclk_logic"), ++ CRU_GATE_CON02, 4, ++ 0), ++ GATE(CGU_PCLK_RXPHY, CNAME("pclk_rxphy"), CNAME("pclk_logic"), ++ CRU_GATE_CON02, 0, ++ 0), ++ GATE(CGU_PCLK_HDMIRX, CNAME("pclk_hdmirx"), CNAME("pclk_logic"), ++ CRU_GATE_CON02, 2, ++ 0), ++ GATE(CGU_PCLK_GVIHOST, CNAME("pclk_gvihost"), CNAME("pclk_logic"), ++ CRU_GATE_CON02, 5, ++ 0), ++ GATE(CGU_CLK_CFG_DPHY0, CNAME("clk_cfg_dphy0"), "xin_osc0_func", ++ CRU_GATE_CON02, 13, ++ 0), ++ GATE(CGU_CLK_CFG_DPHY1, CNAME("clk_cfg_dphy1"), "xin_osc0_func", ++ CRU_GATE_CON02, 14, ++ 0), ++ GATE(CGU_CLK_TXESC, CNAME("clk_txesc"), "xin_osc0_func", ++ CRU_GATE_CON02, 12, ++ 0), ++}; ++ ++static const struct clk_composite_data rk628_clk_composites[] = { ++ COMPOSITE(CGU_CLK_IMODET, CNAME("clk_imodet"), mux_cpll_gpll_mux_p, ++ CRU_CLKSEL_CON05, 5, 1, ++ CRU_CLKSEL_CON05, 0, 5, ++ CRU_GATE_CON02, 11, ++ 0), ++ COMPOSITE(CGU_CLK_HDMIRX_AUD, CNAME("clk_hdmirx_aud"), mux_cpll_gpll_mux_p, ++ CRU_CLKSEL_CON05, 15, 1, ++ CRU_CLKSEL_CON05, 6, 8, ++ CRU_GATE_CON02, 10, ++ CLK_SET_RATE_NO_REPARENT | CLK_SET_RATE_PARENT), ++ COMPOSITE_FRAC_NOMUX(CGU_CLK_HDMIRX_CEC, CNAME("clk_hdmirx_cec"), "xin_osc0_func", ++ CRU_CLKSEL_CON12, ++ CRU_GATE_CON01, 15, ++ 0), ++ COMPOSITE_FRAC(CGU_CLK_RX_READ, CNAME("clk_rx_read"), mux_cpll_gpll_mux_p, ++ CRU_CLKSEL_CON02, 8, 1, ++ CRU_CLKSEL_CON14, ++ CRU_GATE_CON00, 11, ++ 0), ++ COMPOSITE_FRAC(CGU_SCLK_VOP, CNAME("sclk_vop"), mux_cpll_gpll_mux_p, ++ CRU_CLKSEL_CON02, 9, 1, ++ CRU_CLKSEL_CON13, ++ CRU_GATE_CON00, 13, ++ CLK_SET_RATE_NO_REPARENT), ++ COMPOSITE(CGU_PCLK_LOGIC, CNAME("pclk_logic"), mux_cpll_gpll_mux_p, ++ CRU_CLKSEL_CON00, 7, 1, ++ CRU_CLKSEL_CON00, 0, 5, ++ CRU_GATE_CON00, 0, ++ 0), ++ COMPOSITE_NOMUX(CGU_CLK_GPIO_DB0, CNAME("clk_gpio_db0"), "xin_osc0_func", ++ CRU_CLKSEL_CON08, 0, 10, ++ CRU_GATE_CON01, 4, ++ 0), ++ COMPOSITE_NOMUX(CGU_CLK_GPIO_DB1, CNAME("clk_gpio_db1"), "xin_osc0_func", ++ CRU_CLKSEL_CON09, 0, 10, ++ CRU_GATE_CON01, 5, ++ 0), ++ COMPOSITE_NOMUX(CGU_CLK_GPIO_DB2, CNAME("clk_gpio_db2"), "xin_osc0_func", ++ CRU_CLKSEL_CON10, 0, 10, ++ CRU_GATE_CON01, 6, ++ 0), ++ COMPOSITE_NOMUX(CGU_CLK_GPIO_DB3, CNAME("clk_gpio_db3"), "xin_osc0_func", ++ CRU_CLKSEL_CON11, 0, 10, ++ CRU_GATE_CON01, 7, ++ 0), ++ COMPOSITE(CGU_CLK_I2S_8CH_SRC, CNAME("clk_i2s_8ch_src"), mux_cpll_gpll_mux_p, ++ CRU_CLKSEL_CON03, 13, 1, ++ CRU_CLKSEL_CON03, 8, 5, ++ CRU_GATE_CON03, 9, ++ 0), ++ COMPOSITE_FRAC_NOMUX(CGU_CLK_I2S_8CH_FRAC, CNAME("clk_i2s_8ch_frac"), CNAME("clk_i2s_8ch_src"), ++ CRU_CLKSEL_CON04, ++ CRU_GATE_CON03, 10, ++ 0), ++ COMPOSITE_NODIV(CGU_MCLK_I2S_8CH, CNAME("mclk_i2s_8ch"), mux_mclk_i2s_8ch_p, ++ CRU_CLKSEL_CON03, 14, 2, ++ CRU_GATE_CON03, 11, ++ CLK_SET_RATE_PARENT), ++ COMPOSITE_NODIV(CGU_I2S_MCLKOUT, CNAME("i2s_mclkout"), mux_i2s_mclkout_p, ++ CRU_CLKSEL_CON03, 7, 1, ++ CRU_GATE_CON03, 12, ++ CLK_SET_RATE_PARENT), ++ COMPOSITE(CGU_BT1120DEC, CNAME("clk_bt1120dec"), mux_cpll_gpll_mux_p, ++ CRU_CLKSEL_CON02, 7, 1, ++ CRU_CLKSEL_CON02, 0, 5, ++ CRU_GATE_CON00, 12, ++ 0), ++}; ++ ++static void rk628_clk_add_lookup(struct rk628_cru *cru, struct clk *clk, ++ unsigned int id) ++{ ++ if (cru->clk_data.clks && id) ++ cru->clk_data.clks[id] = clk; ++} ++ ++static void rk628_clk_register_muxes(struct rk628_cru *cru) ++{ ++ struct clk *clk; ++ unsigned int i; ++ ++ for (i = 0; i < ARRAY_SIZE(rk628_clk_muxes); i++) { ++ const struct clk_mux_data *data = &rk628_clk_muxes[i]; ++ ++ clk = devm_clk_regmap_register_mux(cru->dev, data->name, ++ data->parent_names, ++ data->num_parents, ++ cru->regmap, data->reg, ++ data->shift, data->width, ++ data->flags); ++ if (IS_ERR(clk)) { ++ dev_err(cru->dev, "failed to register clock %s\n", ++ data->name); ++ continue; ++ } ++ ++ rk628_clk_add_lookup(cru, clk, data->id); ++ } ++} ++ ++static void rk628_clk_register_gates(struct rk628_cru *cru) ++{ ++ struct clk *clk; ++ unsigned int i; ++ ++ for (i = 0; i < ARRAY_SIZE(rk628_clk_gates); i++) { ++ const struct clk_gate_data *data = &rk628_clk_gates[i]; ++ ++ clk = devm_clk_regmap_register_gate(cru->dev, data->name, ++ data->parent_name, ++ cru->regmap, ++ data->reg, data->shift, ++ data->flags); ++ if (IS_ERR(clk)) { ++ dev_err(cru->dev, "failed to register clock %s\n", ++ data->name); ++ continue; ++ } ++ ++ rk628_clk_add_lookup(cru, clk, data->id); ++ } ++} ++ ++static void rk628_clk_register_composites(struct rk628_cru *cru) ++{ ++ struct clk *clk; ++ unsigned int i; ++ ++ for (i = 0; i < ARRAY_SIZE(rk628_clk_composites); i++) { ++ const struct clk_composite_data *data = ++ &rk628_clk_composites[i]; ++ ++ clk = devm_clk_regmap_register_composite(cru->dev, data->name, ++ data->parent_names, ++ data->num_parents, ++ cru->regmap, ++ data->mux_reg, ++ data->mux_shift, ++ data->mux_width, ++ data->div_reg, ++ data->div_shift, ++ data->div_width, ++ data->div_flags, ++ data->gate_reg, ++ data->gate_shift, ++ data->flags); ++ if (IS_ERR(clk)) { ++ dev_err(cru->dev, "failed to register clock %s\n", ++ data->name); ++ continue; ++ } ++ ++ rk628_clk_add_lookup(cru, clk, data->id); ++ } ++} ++ ++static void rk628_clk_register_plls(struct rk628_cru *cru) ++{ ++ struct clk *clk; ++ unsigned int i; ++ ++ for (i = 0; i < ARRAY_SIZE(rk628_clk_plls); i++) { ++ const struct clk_pll_data *data = &rk628_clk_plls[i]; ++ ++ clk = devm_clk_regmap_register_pll(cru->dev, data->name, ++ data->parent_name, ++ cru->regmap, ++ data->reg, ++ data->pd_shift, ++ data->dsmpd_shift, ++ data->lock_shift, ++ data->flags); ++ if (IS_ERR(clk)) { ++ dev_err(cru->dev, "failed to register clock %s\n", ++ data->name); ++ continue; ++ } ++ ++ rk628_clk_add_lookup(cru, clk, data->id); ++ } ++} ++ ++struct rk628_rgu_data { ++ unsigned int id; ++ unsigned int reg; ++ unsigned int bit; ++}; ++ ++#define RSTGEN(_id, _reg, _bit) \ ++ { \ ++ .id = (_id), \ ++ .reg = (_reg), \ ++ .bit = (_bit), \ ++ } ++ ++static const struct rk628_rgu_data rk628_rgu_data[] = { ++ RSTGEN(RGU_LOGIC, CRU_SOFTRST_CON00, 0), ++ RSTGEN(RGU_CRU, CRU_SOFTRST_CON00, 1), ++ RSTGEN(RGU_REGFILE, CRU_SOFTRST_CON00, 2), ++ RSTGEN(RGU_I2C2APB, CRU_SOFTRST_CON00, 3), ++ RSTGEN(RGU_EFUSE, CRU_SOFTRST_CON00, 5), ++ RSTGEN(RGU_ADAPTER, CRU_SOFTRST_CON00, 7), ++ RSTGEN(RGU_CLK_RX, CRU_SOFTRST_CON00, 11), ++ RSTGEN(RGU_BT1120DEC, CRU_SOFTRST_CON00, 12), ++ RSTGEN(RGU_VOP, CRU_SOFTRST_CON00, 13), ++ ++ RSTGEN(RGU_GPIO0, CRU_SOFTRST_CON01, 0), ++ RSTGEN(RGU_GPIO1, CRU_SOFTRST_CON01, 1), ++ RSTGEN(RGU_GPIO2, CRU_SOFTRST_CON01, 2), ++ RSTGEN(RGU_GPIO3, CRU_SOFTRST_CON01, 3), ++ RSTGEN(RGU_GPIO_DB0, CRU_SOFTRST_CON01, 4), ++ RSTGEN(RGU_GPIO_DB1, CRU_SOFTRST_CON01, 5), ++ RSTGEN(RGU_GPIO_DB2, CRU_SOFTRST_CON01, 6), ++ RSTGEN(RGU_GPIO_DB3, CRU_SOFTRST_CON01, 7), ++ ++ RSTGEN(RGU_RXPHY, CRU_SOFTRST_CON02, 0), ++ RSTGEN(RGU_HDMIRX, CRU_SOFTRST_CON02, 2), ++ RSTGEN(RGU_TXPHY_CON, CRU_SOFTRST_CON02, 3), ++ RSTGEN(RGU_HDMITX, CRU_SOFTRST_CON02, 4), ++ RSTGEN(RGU_GVIHOST, CRU_SOFTRST_CON02, 5), ++ RSTGEN(RGU_DSI0, CRU_SOFTRST_CON02, 6), ++ RSTGEN(RGU_DSI1, CRU_SOFTRST_CON02, 7), ++ RSTGEN(RGU_CSI, CRU_SOFTRST_CON02, 8), ++ RSTGEN(RGU_TXDATA, CRU_SOFTRST_CON02, 9), ++ RSTGEN(RGU_DECODER, CRU_SOFTRST_CON02, 10), ++ RSTGEN(RGU_ENCODER, CRU_SOFTRST_CON02, 11), ++ RSTGEN(RGU_HDMIRX_PON, CRU_SOFTRST_CON02, 12), ++ RSTGEN(RGU_TXBYTEHS, CRU_SOFTRST_CON02, 13), ++ RSTGEN(RGU_TXESC, CRU_SOFTRST_CON02, 14), ++}; ++ ++static int rk628_rgu_update(struct rk628_cru *cru, unsigned long id, int assert) ++{ ++ const struct rk628_rgu_data *data = &rk628_rgu_data[id]; ++ ++ return regmap_write(cru->regmap, data->reg, ++ BIT(data->bit + 16) | (assert << data->bit)); ++} ++ ++static int rk628_rgu_assert(struct reset_controller_dev *rcdev, ++ unsigned long id) ++{ ++ struct rk628_cru *cru = reset_to_cru(rcdev); ++ ++ return rk628_rgu_update(cru, id, 1); ++} ++ ++static int rk628_rgu_deassert(struct reset_controller_dev *rcdev, ++ unsigned long id) ++{ ++ struct rk628_cru *cru = reset_to_cru(rcdev); ++ ++ return rk628_rgu_update(cru, id, 0); ++} ++ ++static struct reset_control_ops rk628_rgu_ops = { ++ .assert = rk628_rgu_assert, ++ .deassert = rk628_rgu_deassert, ++}; ++ ++static int rk628_reset_controller_register(struct rk628_cru *cru) ++{ ++ struct device *dev = cru->dev; ++ ++ cru->rcdev.owner = THIS_MODULE; ++ cru->rcdev.nr_resets = ARRAY_SIZE(rk628_rgu_data); ++ cru->rcdev.of_node = dev->of_node; ++ cru->rcdev.ops = &rk628_rgu_ops; ++ ++ return reset_controller_register(&cru->rcdev); ++} ++ ++static const struct regmap_range rk628_cru_readable_ranges[] = { ++ regmap_reg_range(CRU_CPLL_CON0, CRU_CPLL_CON4), ++ regmap_reg_range(CRU_GPLL_CON0, CRU_GPLL_CON4), ++ regmap_reg_range(CRU_MODE_CON, CRU_MODE_CON), ++ regmap_reg_range(CRU_CLKSEL_CON00, CRU_CLKSEL_CON21), ++ regmap_reg_range(CRU_GATE_CON00, CRU_GATE_CON05), ++ regmap_reg_range(CRU_SOFTRST_CON00, CRU_SOFTRST_CON04), ++}; ++ ++static const struct regmap_access_table rk628_cru_readable_table = { ++ .yes_ranges = rk628_cru_readable_ranges, ++ .n_yes_ranges = ARRAY_SIZE(rk628_cru_readable_ranges), ++}; ++ ++static const struct regmap_config rk628_cru_regmap_config = { ++ .name = "cru", ++ .reg_bits = 32, ++ .val_bits = 32, ++ .reg_stride = 4, ++ .max_register = CRU_MAX_REGISTER, ++ .reg_format_endian = REGMAP_ENDIAN_LITTLE, ++ .val_format_endian = REGMAP_ENDIAN_LITTLE, ++ .rd_table = &rk628_cru_readable_table, ++}; ++ ++static int rk628_cru_probe(struct platform_device *pdev) ++{ ++ struct rk628 *rk628 = dev_get_drvdata(pdev->dev.parent); ++ struct device *dev = &pdev->dev; ++ struct rk628_cru *cru; ++ struct clk **clk_table; ++ unsigned int i; ++ int ret; ++ ++ cru = devm_kzalloc(dev, sizeof(*cru), GFP_KERNEL); ++ if (!cru) ++ return -ENOMEM; ++ ++ cru->regmap = devm_regmap_init_i2c(rk628->client, ++ &rk628_cru_regmap_config); ++ if (IS_ERR(cru->regmap)) { ++ ret = PTR_ERR(cru->regmap); ++ dev_err(dev, "failed to allocate register map: %d\n", ret); ++ return ret; ++ } ++ ++ /* clock switch and first set gpll almost 99MHz */ ++ regmap_write(cru->regmap, CRU_GPLL_CON0, 0xffff701d); ++ usleep_range(1000, 1100); ++ /* set clk_gpll_mux from gpll */ ++ regmap_write(cru->regmap, CRU_MODE_CON, 0xffff0004); ++ usleep_range(1000, 1100); ++ /* set pclk_logic from clk_gpll_mux and set pclk div 4 */ ++ regmap_write(cru->regmap, CRU_CLKSEL_CON00, 0xff0080); ++ regmap_write(cru->regmap, CRU_CLKSEL_CON00, 0xff0083); ++ /* set cpll almost 400MHz */ ++ regmap_write(cru->regmap, CRU_CPLL_CON0, 0xffff3063); ++ usleep_range(1000, 1100); ++ /* set clk_cpll_mux from clk_cpll */ ++ regmap_write(cru->regmap, CRU_MODE_CON, 0xffff0005); ++ /* set pclk use cpll, now div is 4 */ ++ regmap_write(cru->regmap, CRU_CLKSEL_CON00, 0xff0003); ++ /* set pclk use cpll, now div is 12 */ ++ regmap_write(cru->regmap, CRU_CLKSEL_CON00, 0xff000b); ++ /* gpll 983.04MHz */ ++ regmap_write(cru->regmap, CRU_GPLL_CON0, 0xffff1028); ++ usleep_range(1000, 1100); ++ /* set pclk use gpll, nuw div is 0xb */ ++ regmap_write(cru->regmap, CRU_CLKSEL_CON00, 0xff008b); ++ /* set cpll 1188MHz */ ++ regmap_write(cru->regmap, CRU_CPLL_CON0, 0xffff1063); ++ usleep_range(1000, 1100); ++ /* set pclk use cpll, and set pclk 99MHz */ ++ regmap_write(cru->regmap, CRU_CLKSEL_CON00, 0xff000b); ++ ++ clk_table = devm_kcalloc(dev, CGU_NR_CLKS, sizeof(struct clk *), ++ GFP_KERNEL); ++ if (!clk_table) ++ return -ENOMEM; ++ ++ for (i = 0; i < CGU_NR_CLKS; i++) ++ clk_table[i] = ERR_PTR(-ENOENT); ++ ++ cru->dev = dev; ++ cru->parent = rk628; ++ cru->clk_data.clks = clk_table; ++ cru->clk_data.clk_num = CGU_NR_CLKS; ++ platform_set_drvdata(pdev, cru); ++ ++ rk628_clk_register_plls(cru); ++ rk628_clk_register_muxes(cru); ++ rk628_clk_register_gates(cru); ++ rk628_clk_register_composites(cru); ++ rk628_reset_controller_register(cru); ++ ++ clk_prepare_enable(clk_table[CGU_PCLK_LOGIC]); ++ ++ return of_clk_add_provider(dev->of_node, of_clk_src_onecell_get, ++ &cru->clk_data); ++} ++ ++static int rk628_cru_remove(struct platform_device *pdev) ++{ ++ struct rk628_cru *cru = dev_get_drvdata(&pdev->dev); ++ ++ of_clk_del_provider(pdev->dev.of_node); ++ reset_controller_unregister(&cru->rcdev); ++ ++ return 0; ++} ++ ++static const struct of_device_id rk628_cru_of_match[] = { ++ { .compatible = "rockchip,rk628-cru", }, ++ {}, ++}; ++MODULE_DEVICE_TABLE(of, rk628_cru_of_match); ++ ++static struct platform_driver rk628_cru_driver = { ++ .driver = { ++ .name = "rk628-cru", ++ .of_match_table = of_match_ptr(rk628_cru_of_match), ++ }, ++ .probe = rk628_cru_probe, ++ .remove = rk628_cru_remove, ++}; ++module_platform_driver(rk628_cru_driver); ++ ++MODULE_AUTHOR("Wyon Bi "); ++MODULE_DESCRIPTION("Rockchip RK628 CRU driver"); ++MODULE_LICENSE("GPL v2"); +diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig +index 39f4d8866200..668711048df8 100644 +--- a/drivers/clocksource/Kconfig ++++ b/drivers/clocksource/Kconfig +@@ -84,7 +84,9 @@ config IXP4XX_TIMER + Enables support for the Intel XScale IXP4xx SoC timer. + + config ROCKCHIP_TIMER +- bool "Rockchip timer driver" if COMPILE_TEST ++ tristate "Rockchip timer driver" ++ default ARCH_ROCKCHIP ++ depends on ARCH_ROCKCHIP || COMPILE_TEST + depends on ARM || ARM64 + select TIMER_OF + select CLKSRC_MMIO +diff --git a/drivers/clocksource/timer-rockchip.c b/drivers/clocksource/timer-rockchip.c +index 1f95d0aca08f..2f4e970d7433 100644 +--- a/drivers/clocksource/timer-rockchip.c ++++ b/drivers/clocksource/timer-rockchip.c +@@ -8,11 +8,13 @@ + #include + #include + #include ++#include + #include + #include + #include + #include + #include ++#include + + #define TIMER_NAME "rk_timer" + +@@ -45,7 +47,9 @@ struct rk_clkevt { + }; + + static struct rk_clkevt *rk_clkevt; ++#ifndef MODULE + static struct rk_timer *rk_clksrc; ++#endif + + static inline struct rk_timer *rk_timer(struct clock_event_device *ce) + { +@@ -119,10 +123,12 @@ static irqreturn_t rk_timer_interrupt(int irq, void *dev_id) + return IRQ_HANDLED; + } + ++#ifndef MODULE + static u64 notrace rk_timer_sched_read(void) + { + return ~readl_relaxed(rk_clksrc->base + TIMER_CURRENT_VALUE0); + } ++#endif + + static int __init + rk_timer_probe(struct rk_timer *timer, struct device_node *np) +@@ -250,6 +256,7 @@ static int __init rk_clkevt_init(struct device_node *np) + return ret; + } + ++#ifndef MODULE + static int __init rk_clksrc_init(struct device_node *np) + { + int ret = -EINVAL; +@@ -287,14 +294,17 @@ static int __init rk_clksrc_init(struct device_node *np) + rk_clksrc = ERR_PTR(ret); + return ret; + } ++#endif + + static int __init rk_timer_init(struct device_node *np) + { + if (!rk_clkevt) + return rk_clkevt_init(np); + ++#ifndef MODULE + if (!rk_clksrc) + return rk_clksrc_init(np); ++#endif + + pr_err("Too many timer definitions for '%s'\n", TIMER_NAME); + return -EINVAL; +@@ -302,3 +312,26 @@ static int __init rk_timer_init(struct device_node *np) + + TIMER_OF_DECLARE(rk3288_timer, "rockchip,rk3288-timer", rk_timer_init); + TIMER_OF_DECLARE(rk3399_timer, "rockchip,rk3399-timer", rk_timer_init); ++ ++#ifdef MODULE ++static int __init rk_timer_driver_probe(struct platform_device *pdev) ++{ ++ return rk_timer_init(pdev->dev.of_node); ++} ++ ++static const struct of_device_id rk_timer_match_table[] = { ++ { .compatible = "rockchip,rk3288-timer" }, ++ { .compatible = "rockchip,rk3399-timer" }, ++ { /* sentinel */ }, ++}; ++ ++static struct platform_driver rk_timer_driver = { ++ .driver = { ++ .name = TIMER_NAME, ++ .of_match_table = rk_timer_match_table, ++ }, ++}; ++module_platform_driver_probe(rk_timer_driver, rk_timer_driver_probe); ++ ++MODULE_LICENSE("GPL"); ++#endif +diff --git a/drivers/cpufreq/Kconfig.arm b/drivers/cpufreq/Kconfig.arm +index 1f73fa75b1a0..0faef5fc6df3 100644 +--- a/drivers/cpufreq/Kconfig.arm ++++ b/drivers/cpufreq/Kconfig.arm +@@ -158,6 +158,16 @@ config ARM_RASPBERRYPI_CPUFREQ + + If in doubt, say N. + ++config ARM_ROCKCHIP_CPUFREQ ++ tristate "Rockchip CPUfreq driver" ++ depends on ARCH_ROCKCHIP && CPUFREQ_DT ++ select PM_OPP ++ help ++ This adds the CPUFreq driver support for Rockchip SoCs, ++ based on cpufreq-dt. ++ ++ If in doubt, say N. ++ + config ARM_S3C_CPUFREQ + bool + help +diff --git a/drivers/cpufreq/Makefile b/drivers/cpufreq/Makefile +index f1b7e3dd6e5d..2e0e827afd21 100644 +--- a/drivers/cpufreq/Makefile ++++ b/drivers/cpufreq/Makefile +@@ -64,6 +64,7 @@ obj-$(CONFIG_PXA3xx) += pxa3xx-cpufreq.o + obj-$(CONFIG_ARM_QCOM_CPUFREQ_HW) += qcom-cpufreq-hw.o + obj-$(CONFIG_ARM_QCOM_CPUFREQ_NVMEM) += qcom-cpufreq-nvmem.o + obj-$(CONFIG_ARM_RASPBERRYPI_CPUFREQ) += raspberrypi-cpufreq.o ++obj-$(CONFIG_ARM_ROCKCHIP_CPUFREQ) += rockchip-cpufreq.o + obj-$(CONFIG_ARM_S3C2410_CPUFREQ) += s3c2410-cpufreq.o + obj-$(CONFIG_ARM_S3C2412_CPUFREQ) += s3c2412-cpufreq.o + obj-$(CONFIG_ARM_S3C2416_CPUFREQ) += s3c2416-cpufreq.o +diff --git a/drivers/cpufreq/cpufreq-dt-platdev.c b/drivers/cpufreq/cpufreq-dt-platdev.c +index a3734014db47..0b91e3616a11 100644 +--- a/drivers/cpufreq/cpufreq-dt-platdev.c ++++ b/drivers/cpufreq/cpufreq-dt-platdev.c +@@ -66,21 +66,6 @@ static const struct of_device_id whitelist[] __initconst = { + { .compatible = "renesas,r8a7794", }, + { .compatible = "renesas,sh73a0", }, + +- { .compatible = "rockchip,rk2928", }, +- { .compatible = "rockchip,rk3036", }, +- { .compatible = "rockchip,rk3066a", }, +- { .compatible = "rockchip,rk3066b", }, +- { .compatible = "rockchip,rk3188", }, +- { .compatible = "rockchip,rk3228", }, +- { .compatible = "rockchip,rk3288", }, +- { .compatible = "rockchip,rk3328", }, +- { .compatible = "rockchip,rk3366", }, +- { .compatible = "rockchip,rk3368", }, +- { .compatible = "rockchip,rk3399", +- .data = &(struct cpufreq_dt_platform_data) +- { .have_governor_per_policy = true, }, +- }, +- + { .compatible = "st-ericsson,u8500", }, + { .compatible = "st-ericsson,u8540", }, + { .compatible = "st-ericsson,u9500", }, +@@ -137,6 +122,28 @@ static const struct of_device_id blacklist[] __initconst = { + { .compatible = "qcom,sc7180", }, + { .compatible = "qcom,sdm845", }, + { .compatible = "qcom,sm8150", }, ++ { .compatible = "rockchip,px30", }, ++ { .compatible = "rockchip,rk2928", }, ++ { .compatible = "rockchip,rk3036", }, ++ { .compatible = "rockchip,rk3066a", }, ++ { .compatible = "rockchip,rk3066b", }, ++ { .compatible = "rockchip,rk3126", }, ++ { .compatible = "rockchip,rk3128", }, ++ { .compatible = "rockchip,rk3188", }, ++ { .compatible = "rockchip,rk3228", }, ++ { .compatible = "rockchip,rk3229", }, ++ { .compatible = "rockchip,rk3288", }, ++ { .compatible = "rockchip,rk3288w", }, ++ { .compatible = "rockchip,rk3326", }, ++ { .compatible = "rockchip,rk3328", }, ++ { .compatible = "rockchip,rk3366", }, ++ { .compatible = "rockchip,rk3368", }, ++ { .compatible = "rockchip,rk3399", }, ++ { .compatible = "rockchip,rk3399pro", }, ++ { .compatible = "rockchip,rk3566", }, ++ { .compatible = "rockchip,rk3568", }, ++ { .compatible = "rockchip,rv1109", }, ++ { .compatible = "rockchip,rv1126", }, + + { .compatible = "st,stih407", }, + { .compatible = "st,stih410", }, +diff --git a/drivers/cpufreq/cpufreq-dt.c b/drivers/cpufreq/cpufreq-dt.c +index e363ae04aac6..5c8463fa079f 100644 +--- a/drivers/cpufreq/cpufreq-dt.c ++++ b/drivers/cpufreq/cpufreq-dt.c +@@ -23,6 +23,9 @@ + #include + + #include "cpufreq-dt.h" ++#ifdef CONFIG_ARCH_ROCKCHIP ++#include "rockchip-cpufreq.h" ++#endif + + struct private_data { + struct list_head node; +@@ -30,7 +33,7 @@ struct private_data { + cpumask_var_t cpus; + struct device *cpu_dev; + struct opp_table *opp_table; +- struct opp_table *reg_opp_table; ++ struct cpufreq_frequency_table *freq_table; + bool have_static_opps; + }; + +@@ -102,7 +105,6 @@ static const char *find_supply_name(struct device *dev) + + static int cpufreq_init(struct cpufreq_policy *policy) + { +- struct cpufreq_frequency_table *freq_table; + struct private_data *priv; + struct device *cpu_dev; + struct clk *cpu_clk; +@@ -114,9 +116,7 @@ static int cpufreq_init(struct cpufreq_policy *policy) + pr_err("failed to find data for cpu%d\n", policy->cpu); + return -ENODEV; + } +- + cpu_dev = priv->cpu_dev; +- cpumask_copy(policy->cpus, priv->cpus); + + cpu_clk = clk_get(cpu_dev, NULL); + if (IS_ERR(cpu_clk)) { +@@ -125,67 +125,32 @@ static int cpufreq_init(struct cpufreq_policy *policy) + return ret; + } + +- /* +- * Initialize OPP tables for all policy->cpus. They will be shared by +- * all CPUs which have marked their CPUs shared with OPP bindings. +- * +- * For platforms not using operating-points-v2 bindings, we do this +- * before updating policy->cpus. Otherwise, we will end up creating +- * duplicate OPPs for policy->cpus. +- * +- * OPPs might be populated at runtime, don't check for error here +- */ +- if (!dev_pm_opp_of_cpumask_add_table(policy->cpus)) +- priv->have_static_opps = true; +- +- /* +- * But we need OPP table to function so if it is not there let's +- * give platform code chance to provide it for us. +- */ +- ret = dev_pm_opp_get_opp_count(cpu_dev); +- if (ret <= 0) { +- dev_err(cpu_dev, "OPP table can't be empty\n"); +- ret = -ENODEV; +- goto out_free_opp; +- } +- +- ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &freq_table); +- if (ret) { +- dev_err(cpu_dev, "failed to init cpufreq table: %d\n", ret); +- goto out_free_opp; +- } ++ transition_latency = dev_pm_opp_get_max_transition_latency(cpu_dev); ++ if (!transition_latency) ++ transition_latency = CPUFREQ_ETERNAL; + ++ cpumask_copy(policy->cpus, priv->cpus); + policy->driver_data = priv; + policy->clk = cpu_clk; +- policy->freq_table = freq_table; +- ++ policy->freq_table = priv->freq_table; + policy->suspend_freq = dev_pm_opp_get_suspend_opp_freq(cpu_dev) / 1000; ++ policy->cpuinfo.transition_latency = transition_latency; ++ policy->dvfs_possible_from_any_cpu = true; + + /* Support turbo/boost mode */ + if (policy_has_boost_freq(policy)) { + /* This gets disabled by core on driver unregister */ + ret = cpufreq_enable_boost_support(); + if (ret) +- goto out_free_cpufreq_table; ++ goto out_clk_put; + cpufreq_dt_attr[1] = &cpufreq_freq_attr_scaling_boost_freqs; + } + +- transition_latency = dev_pm_opp_get_max_transition_latency(cpu_dev); +- if (!transition_latency) +- transition_latency = CPUFREQ_ETERNAL; +- +- policy->cpuinfo.transition_latency = transition_latency; +- policy->dvfs_possible_from_any_cpu = true; +- + dev_pm_opp_of_register_em(cpu_dev, policy->cpus); + + return 0; + +-out_free_cpufreq_table: +- dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table); +-out_free_opp: +- if (priv->have_static_opps) +- dev_pm_opp_of_cpumask_remove_table(policy->cpus); ++out_clk_put: + clk_put(cpu_clk); + + return ret; +@@ -208,11 +173,6 @@ static int cpufreq_offline(struct cpufreq_policy *policy) + + static int cpufreq_exit(struct cpufreq_policy *policy) + { +- struct private_data *priv = policy->driver_data; +- +- dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &policy->freq_table); +- if (priv->have_static_opps) +- dev_pm_opp_of_cpumask_remove_table(policy->related_cpus); + clk_put(policy->clk); + return 0; + } +@@ -236,6 +196,7 @@ static int dt_cpufreq_early_init(struct device *dev, int cpu) + { + struct private_data *priv; + struct device *cpu_dev; ++ bool fallback = false; + const char *reg_name; + int ret; + +@@ -254,68 +215,91 @@ static int dt_cpufreq_early_init(struct device *dev, int cpu) + if (!alloc_cpumask_var(&priv->cpus, GFP_KERNEL)) + return -ENOMEM; + ++ cpumask_set_cpu(cpu, priv->cpus); + priv->cpu_dev = cpu_dev; + +- /* Try to get OPP table early to ensure resources are available */ +- priv->opp_table = dev_pm_opp_get_opp_table(cpu_dev); +- if (IS_ERR(priv->opp_table)) { +- ret = PTR_ERR(priv->opp_table); +- if (ret != -EPROBE_DEFER) +- dev_err(cpu_dev, "failed to get OPP table: %d\n", ret); +- goto free_cpumask; +- } +- + /* + * OPP layer will be taking care of regulators now, but it needs to know + * the name of the regulator first. + */ + reg_name = find_supply_name(cpu_dev); + if (reg_name) { +- priv->reg_opp_table = dev_pm_opp_set_regulators(cpu_dev, +- ®_name, 1); +- if (IS_ERR(priv->reg_opp_table)) { +- ret = PTR_ERR(priv->reg_opp_table); ++ priv->opp_table = dev_pm_opp_set_regulators(cpu_dev, ®_name, ++ 1); ++ if (IS_ERR(priv->opp_table)) { ++ ret = PTR_ERR(priv->opp_table); + if (ret != -EPROBE_DEFER) + dev_err(cpu_dev, "failed to set regulators: %d\n", + ret); +- goto put_table; ++ goto free_cpumask; + } + } + +- /* Find OPP sharing information so we can fill pri->cpus here */ + /* Get OPP-sharing information from "operating-points-v2" bindings */ + ret = dev_pm_opp_of_get_sharing_cpus(cpu_dev, priv->cpus); + if (ret) { + if (ret != -ENOENT) +- goto put_reg; ++ goto out; + + /* + * operating-points-v2 not supported, fallback to all CPUs share + * OPP for backward compatibility if the platform hasn't set + * sharing CPUs. + */ +- if (dev_pm_opp_get_sharing_cpus(cpu_dev, priv->cpus)) { +- cpumask_setall(priv->cpus); +- +- /* +- * OPP tables are initialized only for cpu, do it for +- * others as well. +- */ +- ret = dev_pm_opp_set_sharing_cpus(cpu_dev, priv->cpus); +- if (ret) +- dev_err(cpu_dev, "%s: failed to mark OPPs as shared: %d\n", +- __func__, ret); +- } ++ if (dev_pm_opp_get_sharing_cpus(cpu_dev, priv->cpus)) ++ fallback = true; ++ } ++ ++ /* ++ * Initialize OPP tables for all priv->cpus. They will be shared by ++ * all CPUs which have marked their CPUs shared with OPP bindings. ++ * ++ * For platforms not using operating-points-v2 bindings, we do this ++ * before updating priv->cpus. Otherwise, we will end up creating ++ * duplicate OPPs for the CPUs. ++ * ++ * OPPs might be populated at runtime, don't check for error here. ++ */ ++ if (!dev_pm_opp_of_cpumask_add_table(priv->cpus)) ++ priv->have_static_opps = true; ++ ++ /* ++ * The OPP table must be initialized, statically or dynamically, by this ++ * point. ++ */ ++ ret = dev_pm_opp_get_opp_count(cpu_dev); ++ if (ret <= 0) { ++ dev_err(cpu_dev, "OPP table can't be empty\n"); ++ ret = -ENODEV; ++ goto out; ++ } ++ ++ if (fallback) { ++ cpumask_setall(priv->cpus); ++ ret = dev_pm_opp_set_sharing_cpus(cpu_dev, priv->cpus); ++ if (ret) ++ dev_err(cpu_dev, "%s: failed to mark OPPs as shared: %d\n", ++ __func__, ret); ++ } ++ ++#ifdef CONFIG_ARCH_ROCKCHIP ++ rockchip_cpufreq_adjust_power_scale(cpu_dev); ++#endif ++ ++ ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &priv->freq_table); ++ if (ret) { ++ dev_err(cpu_dev, "failed to init cpufreq table: %d\n", ret); ++ goto out; + } + + list_add(&priv->node, &priv_list); + return 0; + +-put_reg: +- if (priv->reg_opp_table) +- dev_pm_opp_put_regulators(priv->reg_opp_table); +-put_table: +- dev_pm_opp_put_opp_table(priv->opp_table); ++out: ++ if (priv->have_static_opps) ++ dev_pm_opp_of_cpumask_remove_table(priv->cpus); ++ if (priv->opp_table) ++ dev_pm_opp_put_regulators(priv->opp_table); + free_cpumask: + free_cpumask_var(priv->cpus); + return ret; +@@ -326,9 +310,11 @@ static void dt_cpufreq_release(void) + struct private_data *priv, *tmp; + + list_for_each_entry_safe(priv, tmp, &priv_list, node) { +- if (priv->reg_opp_table) +- dev_pm_opp_put_regulators(priv->reg_opp_table); +- dev_pm_opp_put_opp_table(priv->opp_table); ++ dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &priv->freq_table); ++ if (priv->have_static_opps) ++ dev_pm_opp_of_cpumask_remove_table(priv->cpus); ++ if (priv->opp_table) ++ dev_pm_opp_put_regulators(priv->opp_table); + free_cpumask_var(priv->cpus); + list_del(&priv->node); + } +diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c +index ebee0ad559fa..9a7538886556 100644 +--- a/drivers/cpufreq/cpufreq.c ++++ b/drivers/cpufreq/cpufreq.c +@@ -16,6 +16,7 @@ + + #include + #include ++#include + #include + #include + #include +@@ -387,6 +388,7 @@ static void cpufreq_notify_transition(struct cpufreq_policy *policy, + CPUFREQ_POSTCHANGE, freqs); + + cpufreq_stats_record_transition(policy, freqs->new); ++ cpufreq_times_record_transition(policy, freqs->new); + policy->cur = freqs->new; + } + } +@@ -688,8 +690,12 @@ static ssize_t show_##file_name \ + return sprintf(buf, "%u\n", policy->object); \ + } + ++static ssize_t show_cpuinfo_max_freq(struct cpufreq_policy *policy, char *buf) ++{ ++ unsigned int max_freq = policy->cpuinfo.max_freq; ++ return sprintf(buf, "%u\n", max_freq); ++} + show_one(cpuinfo_min_freq, cpuinfo.min_freq); +-show_one(cpuinfo_max_freq, cpuinfo.max_freq); + show_one(cpuinfo_transition_latency, cpuinfo.transition_latency); + show_one(scaling_min_freq, min); + show_one(scaling_max_freq, max); +@@ -1484,6 +1490,7 @@ static int cpufreq_online(unsigned int cpu) + goto out_destroy_policy; + + cpufreq_stats_create_table(policy); ++ cpufreq_times_create_policy(policy); + + write_lock_irqsave(&cpufreq_driver_lock, flags); + list_add(&policy->policy_list, &cpufreq_policy_list); +@@ -2518,7 +2525,6 @@ static int cpufreq_set_policy(struct cpufreq_policy *policy, + ret = cpufreq_start_governor(policy); + if (!ret) { + pr_debug("governor change\n"); +- sched_cpufreq_governor_change(policy, old_gov); + return 0; + } + cpufreq_exit_governor(policy); +@@ -2536,6 +2542,7 @@ static int cpufreq_set_policy(struct cpufreq_policy *policy, + + return ret; + } ++EXPORT_TRACEPOINT_SYMBOL_GPL(cpu_frequency_limits); + + /** + * cpufreq_update_policy - Re-evaluate an existing cpufreq policy. +diff --git a/drivers/cpufreq/cpufreq_userspace.c b/drivers/cpufreq/cpufreq_userspace.c +index 50a4d7846580..1f001d281718 100644 +--- a/drivers/cpufreq/cpufreq_userspace.c ++++ b/drivers/cpufreq/cpufreq_userspace.c +@@ -78,20 +78,18 @@ static int cpufreq_userspace_policy_start(struct cpufreq_policy *policy) + + mutex_lock(&userspace_mutex); + per_cpu(cpu_is_managed, policy->cpu) = 1; +- *setspeed = policy->cur; ++ if (!*setspeed) ++ *setspeed = policy->cur; + mutex_unlock(&userspace_mutex); + return 0; + } + + static void cpufreq_userspace_policy_stop(struct cpufreq_policy *policy) + { +- unsigned int *setspeed = policy->governor_data; +- + pr_debug("managing cpu %u stopped\n", policy->cpu); + + mutex_lock(&userspace_mutex); + per_cpu(cpu_is_managed, policy->cpu) = 0; +- *setspeed = 0; + mutex_unlock(&userspace_mutex); + } + +diff --git a/drivers/cpufreq/rockchip-cpufreq.c b/drivers/cpufreq/rockchip-cpufreq.c +new file mode 100755 +index 000000000000..d370fb4384dc +--- /dev/null ++++ b/drivers/cpufreq/rockchip-cpufreq.c +@@ -0,0 +1,509 @@ ++/* ++ * Rockchip CPUFreq Driver ++ * ++ * Copyright (C) 2017 Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This program is free software; you can redistribute it and/or modify ++ * it under the terms of the GNU General Public License version 2 as ++ * published by the Free Software Foundation. ++ * ++ * This program is distributed "as is" WITHOUT ANY WARRANTY of any ++ * kind, whether express or implied; without even the implied warranty ++ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "cpufreq-dt.h" ++#include "rockchip-cpufreq.h" ++ ++struct cluster_info { ++ struct opp_table *opp_table; ++ struct list_head list_head; ++ struct monitor_dev_info *mdev_info; ++ cpumask_t cpus; ++ int scale; ++ bool offline; ++ bool freq_limit; ++ bool is_check_init; ++}; ++static LIST_HEAD(cluster_info_list); ++ ++static int px30_get_soc_info(struct device *dev, struct device_node *np, ++ int *bin, int *process) ++{ ++ int ret = 0; ++ u8 value = 0; ++ ++ if (!bin) ++ return 0; ++ ++ if (of_property_match_string(np, "nvmem-cell-names", ++ "performance") >= 0) { ++ ret = rockchip_nvmem_cell_read_u8(np, "performance", &value); ++ if (ret) { ++ dev_err(dev, "Failed to get soc performance value\n"); ++ return ret; ++ } ++ *bin = value; ++ } ++ if (*bin >= 0) ++ dev_info(dev, "bin=%d\n", *bin); ++ ++ return ret; ++} ++ ++static int rk3288_get_soc_info(struct device *dev, struct device_node *np, ++ int *bin, int *process) ++{ ++ int ret = 0; ++ u8 value = 0; ++ char *name; ++ ++ if (!bin) ++ goto next; ++ if (of_property_match_string(np, "nvmem-cell-names", "special") >= 0) { ++ ret = rockchip_nvmem_cell_read_u8(np, "special", &value); ++ if (ret) { ++ dev_err(dev, "Failed to get soc special value\n"); ++ goto out; ++ } ++ if (value == 0xc) ++ *bin = 0; ++ else ++ *bin = 1; ++ } ++ ++ if (soc_is_rk3288w()) ++ name = "performance-w"; ++ else ++ name = "performance"; ++ ++ if (of_property_match_string(np, "nvmem-cell-names", name) >= 0) { ++ ret = rockchip_nvmem_cell_read_u8(np, name, &value); ++ if (ret) { ++ dev_err(dev, "Failed to get soc performance value\n"); ++ goto out; ++ } ++ if (value & 0x2) ++ *bin = 3; ++ else if (value & 0x01) ++ *bin = 2; ++ } ++ if (*bin >= 0) ++ dev_info(dev, "bin=%d\n", *bin); ++ ++next: ++ if (!process) ++ goto out; ++ if (of_property_match_string(np, "nvmem-cell-names", ++ "process") >= 0) { ++ ret = rockchip_nvmem_cell_read_u8(np, "process", &value); ++ if (ret) { ++ dev_err(dev, "Failed to get soc process version\n"); ++ goto out; ++ } ++ if (soc_is_rk3288() && (value == 0 || value == 1)) ++ *process = 0; ++ } ++ if (*process >= 0) ++ dev_info(dev, "process=%d\n", *process); ++ ++out: ++ return ret; ++} ++ ++static int rk3399_get_soc_info(struct device *dev, struct device_node *np, ++ int *bin, int *process) ++{ ++ int ret = 0; ++ u8 value = 0; ++ ++ if (!bin) ++ return 0; ++ ++ if (of_property_match_string(np, "nvmem-cell-names", ++ "specification_serial_number") >= 0) { ++ ret = rockchip_nvmem_cell_read_u8(np, ++ "specification_serial_number", ++ &value); ++ if (ret) { ++ dev_err(dev, ++ "Failed to get specification_serial_number\n"); ++ goto out; ++ } ++ ++ if (value == 0xb) { ++ *bin = 0; ++ } else if (value == 0x1) { ++ if (of_property_match_string(np, "nvmem-cell-names", ++ "customer_demand") >= 0) { ++ ret = rockchip_nvmem_cell_read_u8(np, ++ "customer_demand", ++ &value); ++ if (ret) { ++ dev_err(dev, "Failed to get customer_demand\n"); ++ goto out; ++ } ++ if (value == 0x0) ++ *bin = 0; ++ else ++ *bin = 1; ++ } ++ } else if (value == 0x10) { ++ *bin = 1; ++ } ++ } ++ ++out: ++ if (*bin >= 0) ++ dev_info(dev, "bin=%d\n", *bin); ++ ++ return ret; ++} ++ ++static int rv1126_get_soc_info(struct device *dev, struct device_node *np, ++ int *bin, int *process) ++{ ++ int ret = 0; ++ u8 value = 0; ++ ++ if (of_property_match_string(np, "nvmem-cell-names", "performance") >= 0) { ++ ret = rockchip_nvmem_cell_read_u8(np, "performance", &value); ++ if (ret) { ++ dev_err(dev, "Failed to get soc performance value\n"); ++ return ret; ++ } ++ if (value == 0x1) ++ *bin = 1; ++ else ++ *bin = 0; ++ } ++ if (*bin >= 0) ++ dev_info(dev, "bin=%d\n", *bin); ++ ++ return ret; ++} ++ ++static const struct of_device_id rockchip_cpufreq_of_match[] = { ++ { ++ .compatible = "rockchip,px30", ++ .data = (void *)&px30_get_soc_info, ++ }, ++ { ++ .compatible = "rockchip,rk3288", ++ .data = (void *)&rk3288_get_soc_info, ++ }, ++ { ++ .compatible = "rockchip,rk3288w", ++ .data = (void *)&rk3288_get_soc_info, ++ }, ++ { ++ .compatible = "rockchip,rk3326", ++ .data = (void *)&px30_get_soc_info, ++ }, ++ { ++ .compatible = "rockchip,rk3399", ++ .data = (void *)&rk3399_get_soc_info, ++ }, ++ { ++ .compatible = "rockchip,rv1109", ++ .data = (void *)&rv1126_get_soc_info, ++ }, ++ { ++ .compatible = "rockchip,rv1126", ++ .data = (void *)&rv1126_get_soc_info, ++ }, ++ {}, ++}; ++ ++static struct cluster_info *rockchip_cluster_info_lookup(int cpu) ++{ ++ struct cluster_info *cluster; ++ ++ list_for_each_entry(cluster, &cluster_info_list, list_head) { ++ if (cpumask_test_cpu(cpu, &cluster->cpus)) ++ return cluster; ++ } ++ ++ return NULL; ++} ++ ++static int rockchip_cpufreq_set_volt(struct device *dev, ++ struct regulator *reg, ++ struct dev_pm_opp_supply *supply, ++ char *reg_name) ++{ ++ int ret; ++ ++ dev_dbg(dev, "%s: %s voltages (mV): %lu %lu %lu\n", __func__, reg_name, ++ supply->u_volt_min, supply->u_volt, supply->u_volt_max); ++ ++ ret = regulator_set_voltage_triplet(reg, supply->u_volt_min, ++ supply->u_volt, supply->u_volt_max); ++ if (ret) ++ dev_err(dev, "%s: failed to set voltage (%lu %lu %lu mV): %d\n", ++ __func__, supply->u_volt_min, supply->u_volt, ++ supply->u_volt_max, ret); ++ ++ return ret; ++} ++ ++static int opp_helper(struct dev_pm_set_opp_data *data) ++{ ++ struct dev_pm_opp_supply *old_supply_vdd = &data->old_opp.supplies[0]; ++ struct dev_pm_opp_supply *new_supply_vdd = &data->new_opp.supplies[0]; ++ struct regulator *vdd_reg = data->regulators[0]; ++ struct device *dev = data->dev; ++ struct clk *clk = data->clk; ++ struct cluster_info *cluster; ++ unsigned long old_freq = data->old_opp.rate; ++ unsigned long new_freq = data->new_opp.rate; ++ int ret = 0; ++ ++ cluster = rockchip_cluster_info_lookup(dev->id); ++ if (!cluster) ++ return -ENOMEM; ++ ++ rockchip_monitor_volt_adjust_lock(cluster->mdev_info); ++ ++ /* Scaling up? Scale voltage before frequency */ ++ if (new_freq >= old_freq) { ++ ret = rockchip_cpufreq_set_volt(dev, vdd_reg, new_supply_vdd, ++ "vdd"); ++ if (ret) ++ goto restore_voltage; ++ } ++ ++ /* Change frequency */ ++ dev_dbg(dev, "%s: switching OPP: %lu Hz --> %lu Hz\n", __func__, ++ old_freq, new_freq); ++ ret = clk_set_rate(clk, new_freq); ++ if (ret) { ++ dev_err(dev, "%s: failed to set clk rate: %d\n", __func__, ret); ++ goto restore_voltage; ++ } ++ ++ /* Scaling down? Scale voltage after frequency */ ++ if (new_freq < old_freq) { ++ ret = rockchip_cpufreq_set_volt(dev, vdd_reg, new_supply_vdd, ++ "vdd"); ++ if (ret) ++ goto restore_freq; ++ } ++ ++ rockchip_monitor_volt_adjust_unlock(cluster->mdev_info); ++ ++ return 0; ++ ++restore_freq: ++ if (clk_set_rate(clk, old_freq)) ++ dev_err(dev, "%s: failed to restore old-freq (%lu Hz)\n", ++ __func__, old_freq); ++restore_voltage: ++ if (old_supply_vdd->u_volt) ++ rockchip_cpufreq_set_volt(dev, vdd_reg, old_supply_vdd, "vdd"); ++ ++ rockchip_monitor_volt_adjust_unlock(cluster->mdev_info); ++ ++ return ret; ++} ++ ++static int rockchip_cpufreq_cluster_init(int cpu, struct cluster_info *cluster) ++{ ++ struct opp_table *pname_table = NULL; ++ struct opp_table *opp_table; ++ struct device_node *np; ++ struct device *dev; ++ char *reg_name = NULL; ++ int bin = -EINVAL; ++ int process = -EINVAL; ++ int volt_sel = -EINVAL; ++ int ret = 0; ++ ++ dev = get_cpu_device(cpu); ++ if (!dev) ++ return -ENODEV; ++ ++ if (of_find_property(dev->of_node, "cpu-supply", NULL)) ++ reg_name = "cpu"; ++ else if (of_find_property(dev->of_node, "cpu0-supply", NULL)) ++ reg_name = "cpu0"; ++ else ++ return -ENOENT; ++ ++ np = of_parse_phandle(dev->of_node, "operating-points-v2", 0); ++ if (!np) { ++ dev_warn(dev, "OPP-v2 not supported\n"); ++ return -ENOENT; ++ } ++ ++ ret = dev_pm_opp_of_get_sharing_cpus(dev, &cluster->cpus); ++ if (ret) { ++ dev_err(dev, "Failed to get sharing cpus\n"); ++ goto np_err; ++ } ++ ++ rockchip_get_soc_info(dev, rockchip_cpufreq_of_match, &bin, &process); ++ rockchip_get_scale_volt_sel(dev, "cpu_leakage", reg_name, bin, process, ++ &cluster->scale, &volt_sel); ++ pname_table = rockchip_set_opp_prop_name(dev, process, volt_sel); ++ if (IS_ERR(pname_table)) { ++ ret = PTR_ERR(pname_table); ++ goto np_err; ++ } ++ ++ opp_table = dev_pm_opp_register_set_opp_helper(dev, opp_helper); ++ if (IS_ERR(opp_table)) { ++ ret = PTR_ERR(opp_table); ++ if (pname_table) ++ dev_pm_opp_put_prop_name(pname_table); ++ } ++ ++np_err: ++ of_node_put(np); ++ ++ return ret; ++} ++ ++int rockchip_cpufreq_adjust_power_scale(struct device *dev) ++{ ++ struct cluster_info *cluster; ++ ++ cluster = rockchip_cluster_info_lookup(dev->id); ++ if (!cluster) ++ return -EINVAL; ++ rockchip_adjust_power_scale(dev, cluster->scale); ++ ++ return 0; ++} ++EXPORT_SYMBOL_GPL(rockchip_cpufreq_adjust_power_scale); ++ ++static int rockchip_cpufreq_suspend(struct cpufreq_policy *policy) ++{ ++ int ret = 0; ++ ++ ret = cpufreq_generic_suspend(policy); ++ if (!ret) ++ rockchip_monitor_suspend_low_temp_adjust(policy->cpu); ++ ++ return ret; ++} ++ ++static int rockchip_cpufreq_notifier(struct notifier_block *nb, ++ unsigned long event, void *data) ++{ ++ struct device *dev; ++ struct cpufreq_policy *policy = data; ++ struct cluster_info *cluster; ++ struct monitor_dev_profile *mdevp = NULL; ++ struct monitor_dev_info *mdev_info = NULL; ++ ++ dev = get_cpu_device(policy->cpu); ++ if (!dev) ++ return NOTIFY_BAD; ++ ++ cluster = rockchip_cluster_info_lookup(policy->cpu); ++ if (!cluster) ++ return NOTIFY_BAD; ++ ++ if (event == CPUFREQ_CREATE_POLICY) { ++ mdevp = kzalloc(sizeof(*mdevp), GFP_KERNEL); ++ if (!mdevp) ++ return NOTIFY_BAD; ++ mdevp->type = MONITOR_TPYE_CPU; ++ mdevp->low_temp_adjust = rockchip_monitor_cpu_low_temp_adjust; ++ mdevp->high_temp_adjust = rockchip_monitor_cpu_high_temp_adjust; ++ mdevp->update_volt = rockchip_monitor_check_rate_volt; ++ mdevp->data = (void *)policy; ++ cpumask_copy(&mdevp->allowed_cpus, policy->cpus); ++ mdev_info = rockchip_system_monitor_register(dev, mdevp); ++ if (IS_ERR(mdev_info)) { ++ kfree(mdevp); ++ dev_err(dev, "failed to register system monitor\n"); ++ return NOTIFY_BAD; ++ } ++ mdev_info->devp = mdevp; ++ cluster->mdev_info = mdev_info; ++ } else if (event == CPUFREQ_REMOVE_POLICY) { ++ if (cluster->mdev_info) { ++ kfree(cluster->mdev_info->devp); ++ rockchip_system_monitor_unregister(cluster->mdev_info); ++ cluster->mdev_info = NULL; ++ } ++ } ++ ++ return NOTIFY_OK; ++} ++ ++static struct notifier_block rockchip_cpufreq_notifier_block = { ++ .notifier_call = rockchip_cpufreq_notifier, ++}; ++ ++static int __init rockchip_cpufreq_driver_init(void) ++{ ++ struct cluster_info *cluster, *pos; ++ struct cpufreq_dt_platform_data pdata = {0}; ++ int cpu, ret; ++ ++ for_each_possible_cpu(cpu) { ++ cluster = rockchip_cluster_info_lookup(cpu); ++ if (cluster) ++ continue; ++ ++ cluster = kzalloc(sizeof(*cluster), GFP_KERNEL); ++ if (!cluster) { ++ ret = -ENOMEM; ++ goto release_cluster_info; ++ } ++ ++ ret = rockchip_cpufreq_cluster_init(cpu, cluster); ++ if (ret) { ++ pr_err("Failed to initialize dvfs info cpu%d\n", cpu); ++ goto release_cluster_info; ++ } ++ list_add(&cluster->list_head, &cluster_info_list); ++ } ++ ++ pdata.have_governor_per_policy = true; ++ pdata.suspend = rockchip_cpufreq_suspend; ++ ++ ret = cpufreq_register_notifier(&rockchip_cpufreq_notifier_block, ++ CPUFREQ_POLICY_NOTIFIER); ++ if (ret) { ++ pr_err("failed to register cpufreq notifier\n"); ++ goto release_cluster_info; ++ } ++ ++ return PTR_ERR_OR_ZERO(platform_device_register_data(NULL, "cpufreq-dt", ++ -1, (void *)&pdata, ++ sizeof(struct cpufreq_dt_platform_data))); ++ ++release_cluster_info: ++ list_for_each_entry_safe(cluster, pos, &cluster_info_list, list_head) { ++ list_del(&cluster->list_head); ++ kfree(cluster); ++ } ++ return ret; ++} ++module_init(rockchip_cpufreq_driver_init); ++ ++MODULE_AUTHOR("Finley Xiao "); ++MODULE_DESCRIPTION("Rockchip cpufreq driver"); ++MODULE_LICENSE("GPL v2"); +diff --git a/drivers/cpufreq/rockchip-cpufreq.h b/drivers/cpufreq/rockchip-cpufreq.h +new file mode 100755 +index 000000000000..43015c079a1f +--- /dev/null ++++ b/drivers/cpufreq/rockchip-cpufreq.h +@@ -0,0 +1,17 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* ++ * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd ++ */ ++#ifndef __ROCKCHIP_CPUFREQ_H ++#define __ROCKCHIP_CPUFREQ_H ++ ++#if IS_ENABLED(CONFIG_ARM_ROCKCHIP_CPUFREQ) ++int rockchip_cpufreq_adjust_power_scale(struct device *dev); ++#else ++static inline int rockchip_cpufreq_adjust_power_scale(struct device *dev) ++{ ++ return -ENOTSUPP; ++} ++#endif /* CONFIG_ARM_ROCKCHIP_CPUFREQ */ ++ ++#endif +diff --git a/drivers/devfreq/Kconfig b/drivers/devfreq/Kconfig +index 37dc40d1fcfb..79217edf629a 100644 +--- a/drivers/devfreq/Kconfig ++++ b/drivers/devfreq/Kconfig +@@ -131,15 +131,21 @@ config ARM_TEGRA20_DEVFREQ + It reads Memory Controller counters and adjusts the operating + frequencies and voltages with OPP support. + +-config ARM_RK3399_DMC_DEVFREQ +- tristate "ARM RK3399 DMC DEVFREQ Driver" ++config ARM_ROCKCHIP_BUS_DEVFREQ ++ tristate "ARM ROCKCHIP BUS DEVFREQ Driver" ++ depends on ARCH_ROCKCHIP ++ help ++ This adds the DEVFREQ driver for the ROCKCHIP BUS. ++ ++config ARM_ROCKCHIP_DMC_DEVFREQ ++ tristate "ARM ROCKCHIP DMC DEVFREQ Driver" + depends on (ARCH_ROCKCHIP && HAVE_ARM_SMCCC) || \ + (COMPILE_TEST && HAVE_ARM_SMCCC) + select DEVFREQ_EVENT_ROCKCHIP_DFI + select DEVFREQ_GOV_SIMPLE_ONDEMAND + select PM_DEVFREQ_EVENT + help +- This adds the DEVFREQ driver for the RK3399 DMC(Dynamic Memory Controller). ++ This adds the DEVFREQ driver for the ROCKCHIP DMC(Dynamic Memory Controller). + It sets the frequency for the memory controller and reads the usage counts + from hardware. + +diff --git a/drivers/devfreq/Makefile b/drivers/devfreq/Makefile +index 3ca1ad0ecb97..1061fb5923f8 100644 +--- a/drivers/devfreq/Makefile ++++ b/drivers/devfreq/Makefile +@@ -11,7 +11,8 @@ obj-$(CONFIG_DEVFREQ_GOV_PASSIVE) += governor_passive.o + obj-$(CONFIG_ARM_EXYNOS_BUS_DEVFREQ) += exynos-bus.o + obj-$(CONFIG_ARM_IMX_BUS_DEVFREQ) += imx-bus.o + obj-$(CONFIG_ARM_IMX8M_DDRC_DEVFREQ) += imx8m-ddrc.o +-obj-$(CONFIG_ARM_RK3399_DMC_DEVFREQ) += rk3399_dmc.o ++obj-$(CONFIG_ARM_ROCKCHIP_BUS_DEVFREQ) += rockchip_bus.o ++obj-$(CONFIG_ARM_ROCKCHIP_DMC_DEVFREQ) += rockchip_dmc.o + obj-$(CONFIG_ARM_TEGRA_DEVFREQ) += tegra30-devfreq.o + obj-$(CONFIG_ARM_TEGRA20_DEVFREQ) += tegra20-devfreq.o + +diff --git a/drivers/devfreq/devfreq.c b/drivers/devfreq/devfreq.c +index 829128c0cc68..16b6d8f8767d 100644 +--- a/drivers/devfreq/devfreq.c ++++ b/drivers/devfreq/devfreq.c +@@ -1763,6 +1763,40 @@ static ssize_t timer_store(struct device *dev, struct device_attribute *attr, + } + static DEVICE_ATTR_RW(timer); + ++static ssize_t load_show(struct device *dev, struct device_attribute *attr, ++ char *buf) ++{ ++ int err; ++ struct devfreq *devfreq = to_devfreq(dev); ++ struct devfreq_dev_status stat = devfreq->last_status; ++ unsigned long freq; ++ ssize_t len; ++ ++ err = devfreq_update_stats(devfreq); ++ if (err) ++ return err; ++ ++ if (stat.total_time < stat.busy_time) { ++ err = devfreq_update_stats(devfreq); ++ if (err) ++ return err; ++ }; ++ ++ if (!stat.total_time) ++ return 0; ++ ++ len = sprintf(buf, "%lu", stat.busy_time * 100 / stat.total_time); ++ ++ if (devfreq->profile->get_cur_freq && ++ !devfreq->profile->get_cur_freq(devfreq->dev.parent, &freq)) ++ len += sprintf(buf + len, "@%luHz\n", freq); ++ else ++ len += sprintf(buf + len, "@%luHz\n", devfreq->previous_freq); ++ ++ return len; ++} ++static DEVICE_ATTR_RO(load); ++ + static struct attribute *devfreq_attrs[] = { + &dev_attr_name.attr, + &dev_attr_governor.attr, +@@ -1775,6 +1809,7 @@ static struct attribute *devfreq_attrs[] = { + &dev_attr_max_freq.attr, + &dev_attr_trans_stat.attr, + &dev_attr_timer.attr, ++ &dev_attr_load.attr, + NULL, + }; + ATTRIBUTE_GROUPS(devfreq); +diff --git a/drivers/devfreq/event/Kconfig b/drivers/devfreq/event/Kconfig +index 878825372f6f..4526c69c602e 100644 +--- a/drivers/devfreq/event/Kconfig ++++ b/drivers/devfreq/event/Kconfig +@@ -39,4 +39,11 @@ config DEVFREQ_EVENT_ROCKCHIP_DFI + This add the devfreq-event driver for Rockchip SoC. It provides DFI + (DDR Monitor Module) driver to count ddr load. + ++config DEVFREQ_EVENT_ROCKCHIP_NOCP ++ tristate "ROCKCHIP NoC (Network On Chip) Probe DEVFREQ event Driver" ++ depends on ARCH_ROCKCHIP ++ help ++ This add the devfreq-event driver for Rockchip SoC. It provides NoC ++ (Network on Chip) Probe counters to monitor traffic statistics. ++ + endif # PM_DEVFREQ_EVENT +diff --git a/drivers/devfreq/event/Makefile b/drivers/devfreq/event/Makefile +index 3c847e5d5a35..03d67f06c22e 100644 +--- a/drivers/devfreq/event/Makefile ++++ b/drivers/devfreq/event/Makefile +@@ -4,3 +4,4 @@ + obj-$(CONFIG_DEVFREQ_EVENT_EXYNOS_NOCP) += exynos-nocp.o + obj-$(CONFIG_DEVFREQ_EVENT_EXYNOS_PPMU) += exynos-ppmu.o + obj-$(CONFIG_DEVFREQ_EVENT_ROCKCHIP_DFI) += rockchip-dfi.o ++obj-$(CONFIG_DEVFREQ_EVENT_ROCKCHIP_NOCP) += rockchip-nocp.o +diff --git a/drivers/devfreq/event/rockchip-dfi.c b/drivers/devfreq/event/rockchip-dfi.c +index 9a88faaf8b27..9fd6a82f79d6 100644 +--- a/drivers/devfreq/event/rockchip-dfi.c ++++ b/drivers/devfreq/event/rockchip-dfi.c +@@ -20,23 +20,70 @@ + + #include + +-#define RK3399_DMC_NUM_CH 2 +- ++#define PX30_PMUGRF_OS_REG2 0x208 ++#define PX30_PMUGRF_OS_REG3 0x20c ++ ++#define RK3128_GRF_SOC_CON0 0x140 ++#define RK3128_GRF_OS_REG1 0x1cc ++#define RK3128_GRF_DFI_WRNUM 0x220 ++#define RK3128_GRF_DFI_RDNUM 0x224 ++#define RK3128_GRF_DFI_TIMERVAL 0x22c ++#define RK3128_DDR_MONITOR_EN ((1 << (16 + 6)) + (1 << 6)) ++#define RK3128_DDR_MONITOR_DISB ((1 << (16 + 6)) + (0 << 6)) ++ ++#define RK3288_PMU_SYS_REG2 0x9c ++#define RK3288_GRF_SOC_CON4 0x254 ++#define RK3288_GRF_SOC_STATUS(n) (0x280 + (n) * 4) ++#define RK3288_DFI_EN (0x30003 << 14) ++#define RK3288_DFI_DIS (0x30000 << 14) ++#define RK3288_LPDDR_SEL (0x10001 << 13) ++#define RK3288_DDR3_SEL (0x10000 << 13) ++ ++#define RK3328_GRF_OS_REG2 0x5d0 ++ ++#define RK3368_GRF_DDRC0_CON0 0x600 ++#define RK3368_GRF_SOC_STATUS5 0x494 ++#define RK3368_GRF_SOC_STATUS6 0x498 ++#define RK3368_GRF_SOC_STATUS8 0x4a0 ++#define RK3368_GRF_SOC_STATUS9 0x4a4 ++#define RK3368_GRF_SOC_STATUS10 0x4a8 ++#define RK3368_DFI_EN (0x30003 << 5) ++#define RK3368_DFI_DIS (0x30000 << 5) ++ ++#define MAX_DMC_NUM_CH 2 ++#define READ_DRAMTYPE_INFO(n) (((n) >> 13) & 0x7) ++#define READ_CH_INFO(n) (((n) >> 28) & 0x3) ++#define READ_DRAMTYPE_INFO_V3(n, m) ((((n) >> 13) & 0x7) | ((((m) >> 12) & 0x3) << 3)) ++#define READ_SYSREG_VERSION(m) (((m) >> 28) & 0xf) + /* DDRMON_CTRL */ +-#define DDRMON_CTRL 0x04 +-#define CLR_DDRMON_CTRL (0x1f0000 << 0) +-#define LPDDR4_EN (0x10001 << 4) +-#define HARDWARE_EN (0x10001 << 3) +-#define LPDDR3_EN (0x10001 << 2) +-#define SOFTWARE_EN (0x10001 << 1) +-#define SOFTWARE_DIS (0x10000 << 1) +-#define TIME_CNT_EN (0x10001 << 0) ++#define DDRMON_CTRL 0x04 ++#define CLR_DDRMON_CTRL (0x3f0000 << 0) ++#define DDR4_EN (0x10001 << 5) ++#define LPDDR4_EN (0x10001 << 4) ++#define HARDWARE_EN (0x10001 << 3) ++#define LPDDR2_3_EN (0x10001 << 2) ++#define SOFTWARE_EN (0x10001 << 1) ++#define SOFTWARE_DIS (0x10000 << 1) ++#define TIME_CNT_EN (0x10001 << 0) + + #define DDRMON_CH0_COUNT_NUM 0x28 + #define DDRMON_CH0_DFI_ACCESS_NUM 0x2c + #define DDRMON_CH1_COUNT_NUM 0x3c + #define DDRMON_CH1_DFI_ACCESS_NUM 0x40 + ++/* pmu grf */ ++#define PMUGRF_OS_REG2 0x308 ++ ++enum { ++ DDR4 = 0, ++ DDR3 = 3, ++ LPDDR2 = 5, ++ LPDDR3 = 6, ++ LPDDR4 = 7, ++ LPDDR4X = 8, ++ UNUSED = 0xFF ++}; ++ + struct dmc_usage { + u32 access; + u32 total; +@@ -50,33 +97,261 @@ struct dmc_usage { + struct rockchip_dfi { + struct devfreq_event_dev *edev; + struct devfreq_event_desc *desc; +- struct dmc_usage ch_usage[RK3399_DMC_NUM_CH]; ++ struct dmc_usage ch_usage[MAX_DMC_NUM_CH]; + struct device *dev; + void __iomem *regs; + struct regmap *regmap_pmu; ++ struct regmap *regmap_grf; ++ struct regmap *regmap_pmugrf; + struct clk *clk; ++ u32 dram_type; ++ /* ++ * available mask, 1: available, 0: not available ++ * each bit represent a channel ++ */ ++ u32 ch_msk; ++}; ++ ++static void rk3128_dfi_start_hardware_counter(struct devfreq_event_dev *edev) ++{ ++ struct rockchip_dfi *info = devfreq_event_get_drvdata(edev); ++ ++ regmap_write(info->regmap_grf, ++ RK3128_GRF_SOC_CON0, ++ RK3128_DDR_MONITOR_EN); ++} ++ ++static void rk3128_dfi_stop_hardware_counter(struct devfreq_event_dev *edev) ++{ ++ struct rockchip_dfi *info = devfreq_event_get_drvdata(edev); ++ ++ regmap_write(info->regmap_grf, ++ RK3128_GRF_SOC_CON0, ++ RK3128_DDR_MONITOR_DISB); ++} ++ ++static int rk3128_dfi_disable(struct devfreq_event_dev *edev) ++{ ++ rk3128_dfi_stop_hardware_counter(edev); ++ ++ return 0; ++} ++ ++static int rk3128_dfi_enable(struct devfreq_event_dev *edev) ++{ ++ rk3128_dfi_start_hardware_counter(edev); ++ ++ return 0; ++} ++ ++static int rk3128_dfi_set_event(struct devfreq_event_dev *edev) ++{ ++ return 0; ++} ++ ++static int rk3128_dfi_get_event(struct devfreq_event_dev *edev, ++ struct devfreq_event_data *edata) ++{ ++ struct rockchip_dfi *info = devfreq_event_get_drvdata(edev); ++ unsigned long flags; ++ u32 dfi_wr, dfi_rd, dfi_timer; ++ ++ local_irq_save(flags); ++ ++ rk3128_dfi_stop_hardware_counter(edev); ++ ++ regmap_read(info->regmap_grf, RK3128_GRF_DFI_WRNUM, &dfi_wr); ++ regmap_read(info->regmap_grf, RK3128_GRF_DFI_RDNUM, &dfi_rd); ++ regmap_read(info->regmap_grf, RK3128_GRF_DFI_TIMERVAL, &dfi_timer); ++ ++ edata->load_count = (dfi_wr + dfi_rd) * 4; ++ edata->total_count = dfi_timer; ++ ++ rk3128_dfi_start_hardware_counter(edev); ++ ++ local_irq_restore(flags); ++ ++ return 0; ++} ++ ++static const struct devfreq_event_ops rk3128_dfi_ops = { ++ .disable = rk3128_dfi_disable, ++ .enable = rk3128_dfi_enable, ++ .get_event = rk3128_dfi_get_event, ++ .set_event = rk3128_dfi_set_event, ++}; ++ ++static void rk3288_dfi_start_hardware_counter(struct devfreq_event_dev *edev) ++{ ++ struct rockchip_dfi *info = devfreq_event_get_drvdata(edev); ++ ++ regmap_write(info->regmap_grf, RK3288_GRF_SOC_CON4, RK3288_DFI_EN); ++} ++ ++static void rk3288_dfi_stop_hardware_counter(struct devfreq_event_dev *edev) ++{ ++ struct rockchip_dfi *info = devfreq_event_get_drvdata(edev); ++ ++ regmap_write(info->regmap_grf, RK3288_GRF_SOC_CON4, RK3288_DFI_DIS); ++} ++ ++static int rk3288_dfi_disable(struct devfreq_event_dev *edev) ++{ ++ rk3288_dfi_stop_hardware_counter(edev); ++ ++ return 0; ++} ++ ++static int rk3288_dfi_enable(struct devfreq_event_dev *edev) ++{ ++ rk3288_dfi_start_hardware_counter(edev); ++ ++ return 0; ++} ++ ++static int rk3288_dfi_set_event(struct devfreq_event_dev *edev) ++{ ++ return 0; ++} ++ ++static int rk3288_dfi_get_busier_ch(struct devfreq_event_dev *edev) ++{ ++ struct rockchip_dfi *info = devfreq_event_get_drvdata(edev); ++ u32 tmp, max = 0; ++ u32 i, busier_ch = 0; ++ u32 rd_count, wr_count, total_count; ++ ++ rk3288_dfi_stop_hardware_counter(edev); ++ ++ /* Find out which channel is busier */ ++ for (i = 0; i < MAX_DMC_NUM_CH; i++) { ++ if (!(info->ch_msk & BIT(i))) ++ continue; ++ regmap_read(info->regmap_grf, ++ RK3288_GRF_SOC_STATUS(11 + i * 4), &wr_count); ++ regmap_read(info->regmap_grf, ++ RK3288_GRF_SOC_STATUS(12 + i * 4), &rd_count); ++ regmap_read(info->regmap_grf, ++ RK3288_GRF_SOC_STATUS(14 + i * 4), &total_count); ++ info->ch_usage[i].access = (wr_count + rd_count) * 4; ++ info->ch_usage[i].total = total_count; ++ tmp = info->ch_usage[i].access; ++ if (tmp > max) { ++ busier_ch = i; ++ max = tmp; ++ } ++ } ++ rk3288_dfi_start_hardware_counter(edev); ++ ++ return busier_ch; ++} ++ ++static int rk3288_dfi_get_event(struct devfreq_event_dev *edev, ++ struct devfreq_event_data *edata) ++{ ++ struct rockchip_dfi *info = devfreq_event_get_drvdata(edev); ++ int busier_ch; ++ unsigned long flags; ++ ++ local_irq_save(flags); ++ busier_ch = rk3288_dfi_get_busier_ch(edev); ++ local_irq_restore(flags); ++ ++ edata->load_count = info->ch_usage[busier_ch].access; ++ edata->total_count = info->ch_usage[busier_ch].total; ++ ++ return 0; ++} ++ ++static const struct devfreq_event_ops rk3288_dfi_ops = { ++ .disable = rk3288_dfi_disable, ++ .enable = rk3288_dfi_enable, ++ .get_event = rk3288_dfi_get_event, ++ .set_event = rk3288_dfi_set_event, ++}; ++ ++static void rk3368_dfi_start_hardware_counter(struct devfreq_event_dev *edev) ++{ ++ struct rockchip_dfi *info = devfreq_event_get_drvdata(edev); ++ ++ regmap_write(info->regmap_grf, RK3368_GRF_DDRC0_CON0, RK3368_DFI_EN); ++} ++ ++static void rk3368_dfi_stop_hardware_counter(struct devfreq_event_dev *edev) ++{ ++ struct rockchip_dfi *info = devfreq_event_get_drvdata(edev); ++ ++ regmap_write(info->regmap_grf, RK3368_GRF_DDRC0_CON0, RK3368_DFI_DIS); ++} ++ ++static int rk3368_dfi_disable(struct devfreq_event_dev *edev) ++{ ++ rk3368_dfi_stop_hardware_counter(edev); ++ ++ return 0; ++} ++ ++static int rk3368_dfi_enable(struct devfreq_event_dev *edev) ++{ ++ rk3368_dfi_start_hardware_counter(edev); ++ ++ return 0; ++} ++ ++static int rk3368_dfi_set_event(struct devfreq_event_dev *edev) ++{ ++ return 0; ++} ++ ++static int rk3368_dfi_get_event(struct devfreq_event_dev *edev, ++ struct devfreq_event_data *edata) ++{ ++ struct rockchip_dfi *info = devfreq_event_get_drvdata(edev); ++ unsigned long flags; ++ u32 dfi0_wr, dfi0_rd, dfi1_wr, dfi1_rd, dfi_timer; ++ ++ local_irq_save(flags); ++ ++ rk3368_dfi_stop_hardware_counter(edev); ++ ++ regmap_read(info->regmap_grf, RK3368_GRF_SOC_STATUS5, &dfi0_wr); ++ regmap_read(info->regmap_grf, RK3368_GRF_SOC_STATUS6, &dfi0_rd); ++ regmap_read(info->regmap_grf, RK3368_GRF_SOC_STATUS9, &dfi1_wr); ++ regmap_read(info->regmap_grf, RK3368_GRF_SOC_STATUS10, &dfi1_rd); ++ regmap_read(info->regmap_grf, RK3368_GRF_SOC_STATUS8, &dfi_timer); ++ ++ edata->load_count = (dfi0_wr + dfi0_rd + dfi1_wr + dfi1_rd) * 2; ++ edata->total_count = dfi_timer; ++ ++ rk3368_dfi_start_hardware_counter(edev); ++ ++ local_irq_restore(flags); ++ ++ return 0; ++} ++ ++static const struct devfreq_event_ops rk3368_dfi_ops = { ++ .disable = rk3368_dfi_disable, ++ .enable = rk3368_dfi_enable, ++ .get_event = rk3368_dfi_get_event, ++ .set_event = rk3368_dfi_set_event, + }; + + static void rockchip_dfi_start_hardware_counter(struct devfreq_event_dev *edev) + { + struct rockchip_dfi *info = devfreq_event_get_drvdata(edev); + void __iomem *dfi_regs = info->regs; +- u32 val; +- u32 ddr_type; +- +- /* get ddr type */ +- regmap_read(info->regmap_pmu, RK3399_PMUGRF_OS_REG2, &val); +- ddr_type = (val >> RK3399_PMUGRF_DDRTYPE_SHIFT) & +- RK3399_PMUGRF_DDRTYPE_MASK; + + /* clear DDRMON_CTRL setting */ + writel_relaxed(CLR_DDRMON_CTRL, dfi_regs + DDRMON_CTRL); + + /* set ddr type to dfi */ +- if (ddr_type == RK3399_PMUGRF_DDRTYPE_LPDDR3) +- writel_relaxed(LPDDR3_EN, dfi_regs + DDRMON_CTRL); +- else if (ddr_type == RK3399_PMUGRF_DDRTYPE_LPDDR4) ++ if (info->dram_type == LPDDR3 || info->dram_type == LPDDR2) ++ writel_relaxed(LPDDR2_3_EN, dfi_regs + DDRMON_CTRL); ++ else if (info->dram_type == LPDDR4 || info->dram_type == LPDDR4X) + writel_relaxed(LPDDR4_EN, dfi_regs + DDRMON_CTRL); ++ else if (info->dram_type == DDR4) ++ writel_relaxed(DDR4_EN, dfi_regs + DDRMON_CTRL); + + /* enable count, use software mode */ + writel_relaxed(SOFTWARE_EN, dfi_regs + DDRMON_CTRL); +@@ -100,12 +375,22 @@ static int rockchip_dfi_get_busier_ch(struct devfreq_event_dev *edev) + rockchip_dfi_stop_hardware_counter(edev); + + /* Find out which channel is busier */ +- for (i = 0; i < RK3399_DMC_NUM_CH; i++) { +- info->ch_usage[i].access = readl_relaxed(dfi_regs + +- DDRMON_CH0_DFI_ACCESS_NUM + i * 20) * 4; ++ for (i = 0; i < MAX_DMC_NUM_CH; i++) { ++ if (!(info->ch_msk & BIT(i))) ++ continue; ++ + info->ch_usage[i].total = readl_relaxed(dfi_regs + + DDRMON_CH0_COUNT_NUM + i * 20); +- tmp = info->ch_usage[i].access; ++ ++ /* LPDDR4 and LPDDR4X BL = 16,other DDR type BL = 8 */ ++ tmp = readl_relaxed(dfi_regs + ++ DDRMON_CH0_DFI_ACCESS_NUM + i * 20); ++ if (info->dram_type == LPDDR4 || info->dram_type == LPDDR4X) ++ tmp *= 8; ++ else ++ tmp *= 4; ++ info->ch_usage[i].access = tmp; ++ + if (tmp > max) { + busier_ch = i; + max = tmp; +@@ -121,7 +406,8 @@ static int rockchip_dfi_disable(struct devfreq_event_dev *edev) + struct rockchip_dfi *info = devfreq_event_get_drvdata(edev); + + rockchip_dfi_stop_hardware_counter(edev); +- clk_disable_unprepare(info->clk); ++ if (info->clk) ++ clk_disable_unprepare(info->clk); + + return 0; + } +@@ -131,10 +417,13 @@ static int rockchip_dfi_enable(struct devfreq_event_dev *edev) + struct rockchip_dfi *info = devfreq_event_get_drvdata(edev); + int ret; + +- ret = clk_prepare_enable(info->clk); +- if (ret) { +- dev_err(&edev->dev, "failed to enable dfi clk: %d\n", ret); +- return ret; ++ if (info->clk) { ++ ret = clk_prepare_enable(info->clk); ++ if (ret) { ++ dev_err(&edev->dev, "failed to enable dfi clk: %d\n", ++ ret); ++ return ret; ++ } + } + + rockchip_dfi_start_hardware_counter(edev); +@@ -151,8 +440,11 @@ static int rockchip_dfi_get_event(struct devfreq_event_dev *edev, + { + struct rockchip_dfi *info = devfreq_event_get_drvdata(edev); + int busier_ch; ++ unsigned long flags; + ++ local_irq_save(flags); + busier_ch = rockchip_dfi_get_busier_ch(edev); ++ local_irq_restore(flags); + + edata->load_count = info->ch_usage[busier_ch].access; + edata->total_count = info->ch_usage[busier_ch].total; +@@ -167,22 +459,120 @@ static const struct devfreq_event_ops rockchip_dfi_ops = { + .set_event = rockchip_dfi_set_event, + }; + +-static const struct of_device_id rockchip_dfi_id_match[] = { +- { .compatible = "rockchip,rk3399-dfi" }, +- { }, +-}; +-MODULE_DEVICE_TABLE(of, rockchip_dfi_id_match); ++static __init int px30_dfi_init(struct platform_device *pdev, ++ struct rockchip_dfi *data, ++ struct devfreq_event_desc *desc) ++{ ++ struct device_node *np = pdev->dev.of_node, *node; ++ struct resource *res; ++ u32 val_2, val_3; + +-static int rockchip_dfi_probe(struct platform_device *pdev) ++ res = platform_get_resource(pdev, IORESOURCE_MEM, 0); ++ data->regs = devm_ioremap_resource(&pdev->dev, res); ++ if (IS_ERR(data->regs)) ++ return PTR_ERR(data->regs); ++ ++ node = of_parse_phandle(np, "rockchip,pmugrf", 0); ++ if (node) { ++ data->regmap_pmugrf = syscon_node_to_regmap(node); ++ if (IS_ERR(data->regmap_pmugrf)) ++ return PTR_ERR(data->regmap_pmugrf); ++ } ++ ++ regmap_read(data->regmap_pmugrf, PX30_PMUGRF_OS_REG2, &val_2); ++ regmap_read(data->regmap_pmugrf, PX30_PMUGRF_OS_REG3, &val_3); ++ if (READ_SYSREG_VERSION(val_3) >= 0x3) ++ data->dram_type = READ_DRAMTYPE_INFO_V3(val_2, val_3); ++ else ++ data->dram_type = READ_DRAMTYPE_INFO(val_2); ++ data->ch_msk = 1; ++ data->clk = NULL; ++ ++ desc->ops = &rockchip_dfi_ops; ++ ++ return 0; ++} ++ ++static __init int rk3128_dfi_init(struct platform_device *pdev, ++ struct rockchip_dfi *data, ++ struct devfreq_event_desc *desc) + { +- struct device *dev = &pdev->dev; +- struct rockchip_dfi *data; +- struct devfreq_event_desc *desc; + struct device_node *np = pdev->dev.of_node, *node; + +- data = devm_kzalloc(dev, sizeof(struct rockchip_dfi), GFP_KERNEL); +- if (!data) +- return -ENOMEM; ++ node = of_parse_phandle(np, "rockchip,grf", 0); ++ if (node) { ++ data->regmap_grf = syscon_node_to_regmap(node); ++ if (IS_ERR(data->regmap_grf)) ++ return PTR_ERR(data->regmap_grf); ++ } ++ ++ desc->ops = &rk3128_dfi_ops; ++ ++ return 0; ++} ++ ++static __init int rk3288_dfi_init(struct platform_device *pdev, ++ struct rockchip_dfi *data, ++ struct devfreq_event_desc *desc) ++{ ++ struct device_node *np = pdev->dev.of_node, *node; ++ u32 val; ++ ++ node = of_parse_phandle(np, "rockchip,pmu", 0); ++ if (node) { ++ data->regmap_pmu = syscon_node_to_regmap(node); ++ if (IS_ERR(data->regmap_pmu)) ++ return PTR_ERR(data->regmap_pmu); ++ } ++ ++ node = of_parse_phandle(np, "rockchip,grf", 0); ++ if (node) { ++ data->regmap_grf = syscon_node_to_regmap(node); ++ if (IS_ERR(data->regmap_grf)) ++ return PTR_ERR(data->regmap_grf); ++ } ++ ++ regmap_read(data->regmap_pmu, RK3288_PMU_SYS_REG2, &val); ++ data->dram_type = READ_DRAMTYPE_INFO(val); ++ data->ch_msk = READ_CH_INFO(val); ++ ++ if (data->dram_type == DDR3) ++ regmap_write(data->regmap_grf, RK3288_GRF_SOC_CON4, ++ RK3288_DDR3_SEL); ++ else ++ regmap_write(data->regmap_grf, RK3288_GRF_SOC_CON4, ++ RK3288_LPDDR_SEL); ++ ++ desc->ops = &rk3288_dfi_ops; ++ ++ return 0; ++} ++ ++static __init int rk3368_dfi_init(struct platform_device *pdev, ++ struct rockchip_dfi *data, ++ struct devfreq_event_desc *desc) ++{ ++ struct device *dev = &pdev->dev; ++ ++ if (!dev->parent || !dev->parent->of_node) ++ return -EINVAL; ++ ++ data->regmap_grf = syscon_node_to_regmap(dev->parent->of_node); ++ if (IS_ERR(data->regmap_grf)) ++ return PTR_ERR(data->regmap_grf); ++ ++ desc->ops = &rk3368_dfi_ops; ++ ++ return 0; ++} ++ ++static __init int rockchip_dfi_init(struct platform_device *pdev, ++ struct rockchip_dfi *data, ++ struct devfreq_event_desc *desc) ++{ ++ struct device *dev = &pdev->dev; ++ struct device_node *np = pdev->dev.of_node, *node; ++ u32 val; + + data->regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(data->regs)) +@@ -202,23 +592,100 @@ static int rockchip_dfi_probe(struct platform_device *pdev) + if (IS_ERR(data->regmap_pmu)) + return PTR_ERR(data->regmap_pmu); + } +- data->dev = dev; ++ ++ regmap_read(data->regmap_pmu, PMUGRF_OS_REG2, &val); ++ data->dram_type = READ_DRAMTYPE_INFO(val); ++ data->ch_msk = READ_CH_INFO(val); ++ ++ desc->ops = &rockchip_dfi_ops; ++ ++ return 0; ++} ++ ++static __init int rk3328_dfi_init(struct platform_device *pdev, ++ struct rockchip_dfi *data, ++ struct devfreq_event_desc *desc) ++{ ++ struct device_node *np = pdev->dev.of_node, *node; ++ struct resource *res; ++ u32 val; ++ ++ res = platform_get_resource(pdev, IORESOURCE_MEM, 0); ++ data->regs = devm_ioremap_resource(&pdev->dev, res); ++ if (IS_ERR(data->regs)) ++ return PTR_ERR(data->regs); ++ ++ node = of_parse_phandle(np, "rockchip,grf", 0); ++ if (node) { ++ data->regmap_grf = syscon_node_to_regmap(node); ++ if (IS_ERR(data->regmap_grf)) ++ return PTR_ERR(data->regmap_grf); ++ } ++ ++ regmap_read(data->regmap_grf, RK3328_GRF_OS_REG2, &val); ++ data->dram_type = READ_DRAMTYPE_INFO(val); ++ data->ch_msk = 1; ++ data->clk = NULL; ++ ++ desc->ops = &rockchip_dfi_ops; ++ ++ return 0; ++} ++ ++static const struct of_device_id rockchip_dfi_id_match[] = { ++ { .compatible = "rockchip,px30-dfi", .data = px30_dfi_init }, ++ { .compatible = "rockchip,rk1808-dfi", .data = px30_dfi_init }, ++ { .compatible = "rockchip,rk3128-dfi", .data = rk3128_dfi_init }, ++ { .compatible = "rockchip,rk3288-dfi", .data = rk3288_dfi_init }, ++ { .compatible = "rockchip,rk3328-dfi", .data = rk3328_dfi_init }, ++ { .compatible = "rockchip,rk3368-dfi", .data = rk3368_dfi_init }, ++ { .compatible = "rockchip,rk3399-dfi", .data = rockchip_dfi_init }, ++ { .compatible = "rockchip,rk3568-dfi", .data = px30_dfi_init }, ++ { .compatible = "rockchip,rv1126-dfi", .data = px30_dfi_init }, ++ { }, ++}; ++ ++static int rockchip_dfi_probe(struct platform_device *pdev) ++{ ++ struct device *dev = &pdev->dev; ++ struct rockchip_dfi *data; ++ struct devfreq_event_desc *desc; ++ struct device_node *np = pdev->dev.of_node; ++ const struct of_device_id *match; ++ int (*init)(struct platform_device *pdev, struct rockchip_dfi *data, ++ struct devfreq_event_desc *desc); ++ ++ data = devm_kzalloc(dev, sizeof(struct rockchip_dfi), GFP_KERNEL); ++ if (!data) ++ return -ENOMEM; + + desc = devm_kzalloc(dev, sizeof(*desc), GFP_KERNEL); + if (!desc) + return -ENOMEM; + +- desc->ops = &rockchip_dfi_ops; ++ match = of_match_node(rockchip_dfi_id_match, pdev->dev.of_node); ++ if (match) { ++ init = match->data; ++ if (init) { ++ if (init(pdev, data, desc)) ++ return -EINVAL; ++ } else { ++ return 0; ++ } ++ } else { ++ return 0; ++ } ++ + desc->driver_data = data; + desc->name = np->name; +- data->desc = desc; + +- data->edev = devm_devfreq_event_add_edev(&pdev->dev, desc); ++ data->edev = devm_devfreq_event_add_edev(dev, desc); + if (IS_ERR(data->edev)) { +- dev_err(&pdev->dev, +- "failed to add devfreq-event device\n"); ++ dev_err(dev, "failed to add devfreq-event device\n"); + return PTR_ERR(data->edev); + } ++ data->desc = desc; ++ data->dev = &pdev->dev; + + platform_set_drvdata(pdev, data); + +diff --git a/drivers/devfreq/event/rockchip-nocp.c b/drivers/devfreq/event/rockchip-nocp.c +new file mode 100755 +index 000000000000..957b84ee3290 +--- /dev/null ++++ b/drivers/devfreq/event/rockchip-nocp.c +@@ -0,0 +1,166 @@ ++/* ++ * Copyright (c) 2016, Fuzhou Rockchip Electronics Co., Ltd ++ * ++ * This program is free software; you can redistribute it and/or modify it ++ * under the terms and conditions of the GNU General Public License, ++ * version 2, as published by the Free Software Foundation. ++ * ++ * This program is distributed in the hope it will be useful, but WITHOUT ++ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or ++ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for ++ * more details. ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#define EVENT_BYTE 0x08 ++#define EVENT_CHAIN 0x10 ++ ++#define START_EN BIT(3) ++#define GLOBAL_EN BIT(0) ++#define START_GO BIT(0) ++ ++#define PROBE_MAINCTL 0x0008 ++#define PROBE_CFGCTL 0x000c ++#define PROBE_STATPERIOD 0x0024 ++#define PROBE_STATGO 0x0028 ++#define PROBE_COUNTERS_0_SRC 0x0138 ++#define PROBE_COUNTERS_0_VAL 0x013c ++#define PROBE_COUNTERS_1_SRC 0x014c ++#define PROBE_COUNTERS_1_VAL 0x0150 ++ ++struct rockchip_nocp { ++ void __iomem *reg_base; ++ struct device *dev; ++ struct devfreq_event_dev *edev; ++ struct devfreq_event_desc *desc; ++ ktime_t time; ++}; ++ ++static int rockchip_nocp_enable(struct devfreq_event_dev *edev) ++{ ++ struct rockchip_nocp *nocp = devfreq_event_get_drvdata(edev); ++ void __iomem *reg_base = nocp->reg_base; ++ ++ writel_relaxed(GLOBAL_EN, reg_base + PROBE_CFGCTL); ++ writel_relaxed(START_EN, reg_base + PROBE_MAINCTL); ++ writel_relaxed(0, reg_base + PROBE_STATPERIOD); ++ writel_relaxed(EVENT_BYTE, reg_base + PROBE_COUNTERS_0_SRC); ++ writel_relaxed(EVENT_CHAIN, reg_base + PROBE_COUNTERS_1_SRC); ++ writel_relaxed(START_GO, reg_base + PROBE_STATGO); ++ ++ nocp->time = ktime_get(); ++ ++ return 0; ++} ++ ++static int rockchip_nocp_disable(struct devfreq_event_dev *edev) ++{ ++ struct rockchip_nocp *nocp = devfreq_event_get_drvdata(edev); ++ void __iomem *reg_base = nocp->reg_base; ++ ++ writel_relaxed(0, reg_base + PROBE_STATGO); ++ writel_relaxed(0, reg_base + PROBE_MAINCTL); ++ writel_relaxed(0, reg_base + PROBE_CFGCTL); ++ writel_relaxed(0, reg_base + PROBE_COUNTERS_0_SRC); ++ writel_relaxed(0, reg_base + PROBE_COUNTERS_1_SRC); ++ ++ return 0; ++} ++ ++static int rockchip_nocp_get_event(struct devfreq_event_dev *edev, ++ struct devfreq_event_data *edata) ++{ ++ struct rockchip_nocp *nocp = devfreq_event_get_drvdata(edev); ++ void __iomem *reg_base = nocp->reg_base; ++ u32 counter = 0, counter0 = 0, counter1 = 0; ++ int time_ms = 0; ++ ++ time_ms = ktime_to_ms(ktime_sub(ktime_get(), nocp->time)); ++ ++ counter0 = readl_relaxed(reg_base + PROBE_COUNTERS_0_VAL); ++ counter1 = readl_relaxed(reg_base + PROBE_COUNTERS_1_VAL); ++ counter = (counter0 & 0xffff) | ((counter1 & 0xffff) << 16); ++ counter = counter / 1000000; ++ if (time_ms > 0) ++ edata->load_count = (counter * 1000) / time_ms; ++ ++ writel_relaxed(START_GO, reg_base + PROBE_STATGO); ++ nocp->time = ktime_get(); ++ ++ return 0; ++} ++ ++static int rockchip_nocp_set_event(struct devfreq_event_dev *edev) ++{ ++ return 0; ++} ++ ++static const struct devfreq_event_ops rockchip_nocp_ops = { ++ .disable = rockchip_nocp_disable, ++ .enable = rockchip_nocp_enable, ++ .get_event = rockchip_nocp_get_event, ++ .set_event = rockchip_nocp_set_event, ++}; ++ ++static const struct of_device_id rockchip_nocp_id_match[] = { ++ { .compatible = "rockchip,rk3288-nocp" }, ++ { .compatible = "rockchip,rk3368-nocp" }, ++ { .compatible = "rockchip,rk3399-nocp" }, ++ { }, ++}; ++ ++static int rockchip_nocp_probe(struct platform_device *pdev) ++{ ++ struct resource *res; ++ struct rockchip_nocp *nocp; ++ struct devfreq_event_desc *desc; ++ struct device_node *np = pdev->dev.of_node; ++ ++ nocp = devm_kzalloc(&pdev->dev, sizeof(*nocp), GFP_KERNEL); ++ if (!nocp) ++ return -ENOMEM; ++ ++ res = platform_get_resource(pdev, IORESOURCE_MEM, 0); ++ nocp->reg_base = devm_ioremap_resource(&pdev->dev, res); ++ if (IS_ERR(nocp->reg_base)) ++ return PTR_ERR(nocp->reg_base); ++ ++ desc = devm_kzalloc(&pdev->dev, sizeof(*desc), GFP_KERNEL); ++ if (!desc) ++ return -ENOMEM; ++ ++ desc->ops = &rockchip_nocp_ops; ++ desc->driver_data = nocp; ++ desc->name = np->name; ++ nocp->desc = desc; ++ nocp->dev = &pdev->dev; ++ nocp->edev = devm_devfreq_event_add_edev(&pdev->dev, desc); ++ if (IS_ERR(nocp->edev)) { ++ dev_err(&pdev->dev, "failed to add devfreq-event device\n"); ++ return PTR_ERR(nocp->edev); ++ } ++ ++ platform_set_drvdata(pdev, nocp); ++ ++ return 0; ++} ++ ++static struct platform_driver rockchip_nocp_driver = { ++ .probe = rockchip_nocp_probe, ++ .driver = { ++ .name = "rockchip-nocp", ++ .of_match_table = rockchip_nocp_id_match, ++ }, ++}; ++module_platform_driver(rockchip_nocp_driver); ++ ++MODULE_DESCRIPTION("Rockchip NoC (Network on Chip) Probe driver"); ++MODULE_AUTHOR("Finley Xiao "); ++MODULE_LICENSE("GPL v2"); +diff --git a/drivers/devfreq/rk3399_dmc.c b/drivers/devfreq/rk3399_dmc.c +deleted file mode 100644 +index 2e912166a993..000000000000 +--- a/drivers/devfreq/rk3399_dmc.c ++++ /dev/null +@@ -1,516 +0,0 @@ +-// SPDX-License-Identifier: GPL-2.0-only +-/* +- * Copyright (c) 2016, Fuzhou Rockchip Electronics Co., Ltd. +- * Author: Lin Huang +- */ +- +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +- +-#include +-#include +- +-struct dram_timing { +- unsigned int ddr3_speed_bin; +- unsigned int pd_idle; +- unsigned int sr_idle; +- unsigned int sr_mc_gate_idle; +- unsigned int srpd_lite_idle; +- unsigned int standby_idle; +- unsigned int auto_pd_dis_freq; +- unsigned int dram_dll_dis_freq; +- unsigned int phy_dll_dis_freq; +- unsigned int ddr3_odt_dis_freq; +- unsigned int ddr3_drv; +- unsigned int ddr3_odt; +- unsigned int phy_ddr3_ca_drv; +- unsigned int phy_ddr3_dq_drv; +- unsigned int phy_ddr3_odt; +- unsigned int lpddr3_odt_dis_freq; +- unsigned int lpddr3_drv; +- unsigned int lpddr3_odt; +- unsigned int phy_lpddr3_ca_drv; +- unsigned int phy_lpddr3_dq_drv; +- unsigned int phy_lpddr3_odt; +- unsigned int lpddr4_odt_dis_freq; +- unsigned int lpddr4_drv; +- unsigned int lpddr4_dq_odt; +- unsigned int lpddr4_ca_odt; +- unsigned int phy_lpddr4_ca_drv; +- unsigned int phy_lpddr4_ck_cs_drv; +- unsigned int phy_lpddr4_dq_drv; +- unsigned int phy_lpddr4_odt; +-}; +- +-struct rk3399_dmcfreq { +- struct device *dev; +- struct devfreq *devfreq; +- struct devfreq_simple_ondemand_data ondemand_data; +- struct clk *dmc_clk; +- struct devfreq_event_dev *edev; +- struct mutex lock; +- struct dram_timing timing; +- struct regulator *vdd_center; +- struct regmap *regmap_pmu; +- unsigned long rate, target_rate; +- unsigned long volt, target_volt; +- unsigned int odt_dis_freq; +- int odt_pd_arg0, odt_pd_arg1; +-}; +- +-static int rk3399_dmcfreq_target(struct device *dev, unsigned long *freq, +- u32 flags) +-{ +- struct rk3399_dmcfreq *dmcfreq = dev_get_drvdata(dev); +- struct dev_pm_opp *opp; +- unsigned long old_clk_rate = dmcfreq->rate; +- unsigned long target_volt, target_rate; +- struct arm_smccc_res res; +- bool odt_enable = false; +- int err; +- +- opp = devfreq_recommended_opp(dev, freq, flags); +- if (IS_ERR(opp)) +- return PTR_ERR(opp); +- +- target_rate = dev_pm_opp_get_freq(opp); +- target_volt = dev_pm_opp_get_voltage(opp); +- dev_pm_opp_put(opp); +- +- if (dmcfreq->rate == target_rate) +- return 0; +- +- mutex_lock(&dmcfreq->lock); +- +- if (dmcfreq->regmap_pmu) { +- if (target_rate >= dmcfreq->odt_dis_freq) +- odt_enable = true; +- +- /* +- * This makes a SMC call to the TF-A to set the DDR PD +- * (power-down) timings and to enable or disable the +- * ODT (on-die termination) resistors. +- */ +- arm_smccc_smc(ROCKCHIP_SIP_DRAM_FREQ, dmcfreq->odt_pd_arg0, +- dmcfreq->odt_pd_arg1, +- ROCKCHIP_SIP_CONFIG_DRAM_SET_ODT_PD, +- odt_enable, 0, 0, 0, &res); +- } +- +- /* +- * If frequency scaling from low to high, adjust voltage first. +- * If frequency scaling from high to low, adjust frequency first. +- */ +- if (old_clk_rate < target_rate) { +- err = regulator_set_voltage(dmcfreq->vdd_center, target_volt, +- target_volt); +- if (err) { +- dev_err(dev, "Cannot set voltage %lu uV\n", +- target_volt); +- goto out; +- } +- } +- +- err = clk_set_rate(dmcfreq->dmc_clk, target_rate); +- if (err) { +- dev_err(dev, "Cannot set frequency %lu (%d)\n", target_rate, +- err); +- regulator_set_voltage(dmcfreq->vdd_center, dmcfreq->volt, +- dmcfreq->volt); +- goto out; +- } +- +- /* +- * Check the dpll rate, +- * There only two result we will get, +- * 1. Ddr frequency scaling fail, we still get the old rate. +- * 2. Ddr frequency scaling sucessful, we get the rate we set. +- */ +- dmcfreq->rate = clk_get_rate(dmcfreq->dmc_clk); +- +- /* If get the incorrect rate, set voltage to old value. */ +- if (dmcfreq->rate != target_rate) { +- dev_err(dev, "Got wrong frequency, Request %lu, Current %lu\n", +- target_rate, dmcfreq->rate); +- regulator_set_voltage(dmcfreq->vdd_center, dmcfreq->volt, +- dmcfreq->volt); +- goto out; +- } else if (old_clk_rate > target_rate) +- err = regulator_set_voltage(dmcfreq->vdd_center, target_volt, +- target_volt); +- if (err) +- dev_err(dev, "Cannot set voltage %lu uV\n", target_volt); +- +- dmcfreq->rate = target_rate; +- dmcfreq->volt = target_volt; +- +-out: +- mutex_unlock(&dmcfreq->lock); +- return err; +-} +- +-static int rk3399_dmcfreq_get_dev_status(struct device *dev, +- struct devfreq_dev_status *stat) +-{ +- struct rk3399_dmcfreq *dmcfreq = dev_get_drvdata(dev); +- struct devfreq_event_data edata; +- int ret = 0; +- +- ret = devfreq_event_get_event(dmcfreq->edev, &edata); +- if (ret < 0) +- return ret; +- +- stat->current_frequency = dmcfreq->rate; +- stat->busy_time = edata.load_count; +- stat->total_time = edata.total_count; +- +- return ret; +-} +- +-static int rk3399_dmcfreq_get_cur_freq(struct device *dev, unsigned long *freq) +-{ +- struct rk3399_dmcfreq *dmcfreq = dev_get_drvdata(dev); +- +- *freq = dmcfreq->rate; +- +- return 0; +-} +- +-static struct devfreq_dev_profile rk3399_devfreq_dmc_profile = { +- .polling_ms = 200, +- .target = rk3399_dmcfreq_target, +- .get_dev_status = rk3399_dmcfreq_get_dev_status, +- .get_cur_freq = rk3399_dmcfreq_get_cur_freq, +-}; +- +-static __maybe_unused int rk3399_dmcfreq_suspend(struct device *dev) +-{ +- struct rk3399_dmcfreq *dmcfreq = dev_get_drvdata(dev); +- int ret = 0; +- +- ret = devfreq_event_disable_edev(dmcfreq->edev); +- if (ret < 0) { +- dev_err(dev, "failed to disable the devfreq-event devices\n"); +- return ret; +- } +- +- ret = devfreq_suspend_device(dmcfreq->devfreq); +- if (ret < 0) { +- dev_err(dev, "failed to suspend the devfreq devices\n"); +- return ret; +- } +- +- return 0; +-} +- +-static __maybe_unused int rk3399_dmcfreq_resume(struct device *dev) +-{ +- struct rk3399_dmcfreq *dmcfreq = dev_get_drvdata(dev); +- int ret = 0; +- +- ret = devfreq_event_enable_edev(dmcfreq->edev); +- if (ret < 0) { +- dev_err(dev, "failed to enable the devfreq-event devices\n"); +- return ret; +- } +- +- ret = devfreq_resume_device(dmcfreq->devfreq); +- if (ret < 0) { +- dev_err(dev, "failed to resume the devfreq devices\n"); +- return ret; +- } +- return ret; +-} +- +-static SIMPLE_DEV_PM_OPS(rk3399_dmcfreq_pm, rk3399_dmcfreq_suspend, +- rk3399_dmcfreq_resume); +- +-static int of_get_ddr_timings(struct dram_timing *timing, +- struct device_node *np) +-{ +- int ret = 0; +- +- ret = of_property_read_u32(np, "rockchip,ddr3_speed_bin", +- &timing->ddr3_speed_bin); +- ret |= of_property_read_u32(np, "rockchip,pd_idle", +- &timing->pd_idle); +- ret |= of_property_read_u32(np, "rockchip,sr_idle", +- &timing->sr_idle); +- ret |= of_property_read_u32(np, "rockchip,sr_mc_gate_idle", +- &timing->sr_mc_gate_idle); +- ret |= of_property_read_u32(np, "rockchip,srpd_lite_idle", +- &timing->srpd_lite_idle); +- ret |= of_property_read_u32(np, "rockchip,standby_idle", +- &timing->standby_idle); +- ret |= of_property_read_u32(np, "rockchip,auto_pd_dis_freq", +- &timing->auto_pd_dis_freq); +- ret |= of_property_read_u32(np, "rockchip,dram_dll_dis_freq", +- &timing->dram_dll_dis_freq); +- ret |= of_property_read_u32(np, "rockchip,phy_dll_dis_freq", +- &timing->phy_dll_dis_freq); +- ret |= of_property_read_u32(np, "rockchip,ddr3_odt_dis_freq", +- &timing->ddr3_odt_dis_freq); +- ret |= of_property_read_u32(np, "rockchip,ddr3_drv", +- &timing->ddr3_drv); +- ret |= of_property_read_u32(np, "rockchip,ddr3_odt", +- &timing->ddr3_odt); +- ret |= of_property_read_u32(np, "rockchip,phy_ddr3_ca_drv", +- &timing->phy_ddr3_ca_drv); +- ret |= of_property_read_u32(np, "rockchip,phy_ddr3_dq_drv", +- &timing->phy_ddr3_dq_drv); +- ret |= of_property_read_u32(np, "rockchip,phy_ddr3_odt", +- &timing->phy_ddr3_odt); +- ret |= of_property_read_u32(np, "rockchip,lpddr3_odt_dis_freq", +- &timing->lpddr3_odt_dis_freq); +- ret |= of_property_read_u32(np, "rockchip,lpddr3_drv", +- &timing->lpddr3_drv); +- ret |= of_property_read_u32(np, "rockchip,lpddr3_odt", +- &timing->lpddr3_odt); +- ret |= of_property_read_u32(np, "rockchip,phy_lpddr3_ca_drv", +- &timing->phy_lpddr3_ca_drv); +- ret |= of_property_read_u32(np, "rockchip,phy_lpddr3_dq_drv", +- &timing->phy_lpddr3_dq_drv); +- ret |= of_property_read_u32(np, "rockchip,phy_lpddr3_odt", +- &timing->phy_lpddr3_odt); +- ret |= of_property_read_u32(np, "rockchip,lpddr4_odt_dis_freq", +- &timing->lpddr4_odt_dis_freq); +- ret |= of_property_read_u32(np, "rockchip,lpddr4_drv", +- &timing->lpddr4_drv); +- ret |= of_property_read_u32(np, "rockchip,lpddr4_dq_odt", +- &timing->lpddr4_dq_odt); +- ret |= of_property_read_u32(np, "rockchip,lpddr4_ca_odt", +- &timing->lpddr4_ca_odt); +- ret |= of_property_read_u32(np, "rockchip,phy_lpddr4_ca_drv", +- &timing->phy_lpddr4_ca_drv); +- ret |= of_property_read_u32(np, "rockchip,phy_lpddr4_ck_cs_drv", +- &timing->phy_lpddr4_ck_cs_drv); +- ret |= of_property_read_u32(np, "rockchip,phy_lpddr4_dq_drv", +- &timing->phy_lpddr4_dq_drv); +- ret |= of_property_read_u32(np, "rockchip,phy_lpddr4_odt", +- &timing->phy_lpddr4_odt); +- +- return ret; +-} +- +-static int rk3399_dmcfreq_probe(struct platform_device *pdev) +-{ +- struct arm_smccc_res res; +- struct device *dev = &pdev->dev; +- struct device_node *np = pdev->dev.of_node, *node; +- struct rk3399_dmcfreq *data; +- int ret, index, size; +- uint32_t *timing; +- struct dev_pm_opp *opp; +- u32 ddr_type; +- u32 val; +- +- data = devm_kzalloc(dev, sizeof(struct rk3399_dmcfreq), GFP_KERNEL); +- if (!data) +- return -ENOMEM; +- +- mutex_init(&data->lock); +- +- data->vdd_center = devm_regulator_get(dev, "center"); +- if (IS_ERR(data->vdd_center)) { +- if (PTR_ERR(data->vdd_center) == -EPROBE_DEFER) +- return -EPROBE_DEFER; +- +- dev_err(dev, "Cannot get the regulator \"center\"\n"); +- return PTR_ERR(data->vdd_center); +- } +- +- data->dmc_clk = devm_clk_get(dev, "dmc_clk"); +- if (IS_ERR(data->dmc_clk)) { +- if (PTR_ERR(data->dmc_clk) == -EPROBE_DEFER) +- return -EPROBE_DEFER; +- +- dev_err(dev, "Cannot get the clk dmc_clk\n"); +- return PTR_ERR(data->dmc_clk); +- } +- +- data->edev = devfreq_event_get_edev_by_phandle(dev, "devfreq-events", 0); +- if (IS_ERR(data->edev)) +- return -EPROBE_DEFER; +- +- ret = devfreq_event_enable_edev(data->edev); +- if (ret < 0) { +- dev_err(dev, "failed to enable devfreq-event devices\n"); +- return ret; +- } +- +- /* +- * Get dram timing and pass it to arm trust firmware, +- * the dram driver in arm trust firmware will get these +- * timing and to do dram initial. +- */ +- if (!of_get_ddr_timings(&data->timing, np)) { +- timing = &data->timing.ddr3_speed_bin; +- size = sizeof(struct dram_timing) / 4; +- for (index = 0; index < size; index++) { +- arm_smccc_smc(ROCKCHIP_SIP_DRAM_FREQ, *timing++, index, +- ROCKCHIP_SIP_CONFIG_DRAM_SET_PARAM, +- 0, 0, 0, 0, &res); +- if (res.a0) { +- dev_err(dev, "Failed to set dram param: %ld\n", +- res.a0); +- ret = -EINVAL; +- goto err_edev; +- } +- } +- } +- +- node = of_parse_phandle(np, "rockchip,pmu", 0); +- if (!node) +- goto no_pmu; +- +- data->regmap_pmu = syscon_node_to_regmap(node); +- of_node_put(node); +- if (IS_ERR(data->regmap_pmu)) { +- ret = PTR_ERR(data->regmap_pmu); +- goto err_edev; +- } +- +- regmap_read(data->regmap_pmu, RK3399_PMUGRF_OS_REG2, &val); +- ddr_type = (val >> RK3399_PMUGRF_DDRTYPE_SHIFT) & +- RK3399_PMUGRF_DDRTYPE_MASK; +- +- switch (ddr_type) { +- case RK3399_PMUGRF_DDRTYPE_DDR3: +- data->odt_dis_freq = data->timing.ddr3_odt_dis_freq; +- break; +- case RK3399_PMUGRF_DDRTYPE_LPDDR3: +- data->odt_dis_freq = data->timing.lpddr3_odt_dis_freq; +- break; +- case RK3399_PMUGRF_DDRTYPE_LPDDR4: +- data->odt_dis_freq = data->timing.lpddr4_odt_dis_freq; +- break; +- default: +- ret = -EINVAL; +- goto err_edev; +- }; +- +-no_pmu: +- arm_smccc_smc(ROCKCHIP_SIP_DRAM_FREQ, 0, 0, +- ROCKCHIP_SIP_CONFIG_DRAM_INIT, +- 0, 0, 0, 0, &res); +- +- /* +- * In TF-A there is a platform SIP call to set the PD (power-down) +- * timings and to enable or disable the ODT (on-die termination). +- * This call needs three arguments as follows: +- * +- * arg0: +- * bit[0-7] : sr_idle +- * bit[8-15] : sr_mc_gate_idle +- * bit[16-31] : standby idle +- * arg1: +- * bit[0-11] : pd_idle +- * bit[16-27] : srpd_lite_idle +- * arg2: +- * bit[0] : odt enable +- */ +- data->odt_pd_arg0 = (data->timing.sr_idle & 0xff) | +- ((data->timing.sr_mc_gate_idle & 0xff) << 8) | +- ((data->timing.standby_idle & 0xffff) << 16); +- data->odt_pd_arg1 = (data->timing.pd_idle & 0xfff) | +- ((data->timing.srpd_lite_idle & 0xfff) << 16); +- +- /* +- * We add a devfreq driver to our parent since it has a device tree node +- * with operating points. +- */ +- if (dev_pm_opp_of_add_table(dev)) { +- dev_err(dev, "Invalid operating-points in device tree.\n"); +- ret = -EINVAL; +- goto err_edev; +- } +- +- of_property_read_u32(np, "upthreshold", +- &data->ondemand_data.upthreshold); +- of_property_read_u32(np, "downdifferential", +- &data->ondemand_data.downdifferential); +- +- data->rate = clk_get_rate(data->dmc_clk); +- +- opp = devfreq_recommended_opp(dev, &data->rate, 0); +- if (IS_ERR(opp)) { +- ret = PTR_ERR(opp); +- goto err_free_opp; +- } +- +- data->rate = dev_pm_opp_get_freq(opp); +- data->volt = dev_pm_opp_get_voltage(opp); +- dev_pm_opp_put(opp); +- +- rk3399_devfreq_dmc_profile.initial_freq = data->rate; +- +- data->devfreq = devm_devfreq_add_device(dev, +- &rk3399_devfreq_dmc_profile, +- DEVFREQ_GOV_SIMPLE_ONDEMAND, +- &data->ondemand_data); +- if (IS_ERR(data->devfreq)) { +- ret = PTR_ERR(data->devfreq); +- goto err_free_opp; +- } +- +- devm_devfreq_register_opp_notifier(dev, data->devfreq); +- +- data->dev = dev; +- platform_set_drvdata(pdev, data); +- +- return 0; +- +-err_free_opp: +- dev_pm_opp_of_remove_table(&pdev->dev); +-err_edev: +- devfreq_event_disable_edev(data->edev); +- +- return ret; +-} +- +-static int rk3399_dmcfreq_remove(struct platform_device *pdev) +-{ +- struct rk3399_dmcfreq *dmcfreq = dev_get_drvdata(&pdev->dev); +- +- /* +- * Before remove the opp table we need to unregister the opp notifier. +- */ +- devm_devfreq_unregister_opp_notifier(dmcfreq->dev, dmcfreq->devfreq); +- dev_pm_opp_of_remove_table(dmcfreq->dev); +- +- return 0; +-} +- +-static const struct of_device_id rk3399dmc_devfreq_of_match[] = { +- { .compatible = "rockchip,rk3399-dmc" }, +- { }, +-}; +-MODULE_DEVICE_TABLE(of, rk3399dmc_devfreq_of_match); +- +-static struct platform_driver rk3399_dmcfreq_driver = { +- .probe = rk3399_dmcfreq_probe, +- .remove = rk3399_dmcfreq_remove, +- .driver = { +- .name = "rk3399-dmc-freq", +- .pm = &rk3399_dmcfreq_pm, +- .of_match_table = rk3399dmc_devfreq_of_match, +- }, +-}; +-module_platform_driver(rk3399_dmcfreq_driver); +- +-MODULE_LICENSE("GPL v2"); +-MODULE_AUTHOR("Lin Huang "); +-MODULE_DESCRIPTION("RK3399 dmcfreq driver with devfreq framework"); +diff --git a/drivers/devfreq/rockchip_bus.c b/drivers/devfreq/rockchip_bus.c +new file mode 100755 +index 000000000000..7032b4815c5a +--- /dev/null ++++ b/drivers/devfreq/rockchip_bus.c +@@ -0,0 +1,500 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * Copyright (c) 2018, Fuzhou Rockchip Electronics Co., Ltd. ++ * Author: Tony Xie ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#define CLUSTER0 0 ++#define CLUSTER1 1 ++#define MAX_CLUSTERS 2 ++ ++#define to_rockchip_bus_clk_nb(nb) \ ++ container_of(nb, struct rockchip_bus, clk_nb) ++#define to_rockchip_bus_cpufreq_nb(nb) \ ++ container_of(nb, struct rockchip_bus, cpufreq_nb) ++ ++struct busfreq_table { ++ unsigned long freq; ++ unsigned long volt; ++}; ++ ++struct rockchip_bus { ++ struct device *dev; ++ struct regulator *regulator; ++ struct clk *clk; ++ struct notifier_block clk_nb; ++ struct notifier_block cpufreq_nb; ++ struct busfreq_table *freq_table; ++ ++ unsigned int max_state; ++ ++ unsigned long cur_volt; ++ unsigned long cur_rate; ++ ++ /* ++ * Busfreq-policy-cpufreq: ++ * If the cpu frequency of two clusters are both less than or equal to ++ * cpu_high_freq, change bus rate to low_rate, otherwise change it to ++ * high_rate. ++ */ ++ unsigned long high_rate; ++ unsigned long low_rate; ++ unsigned int cpu_high_freq; ++ unsigned int cpu_freq[MAX_CLUSTERS]; ++}; ++ ++static int rockchip_sip_bus_smc_config(u32 bus_id, u32 cfg, u32 enable_msk) ++{ ++ struct arm_smccc_res res; ++ ++ res = sip_smc_bus_config(bus_id, cfg, enable_msk); ++ ++ return res.a0; ++} ++ ++static int rockchip_bus_smc_config(struct rockchip_bus *bus) ++{ ++ struct device *dev = bus->dev; ++ struct device_node *np = dev->of_node; ++ struct device_node *child; ++ unsigned int enable_msk, bus_id, cfg; ++ int ret; ++ ++ for_each_available_child_of_node(np, child) { ++ ret = of_property_read_u32_index(child, "bus-id", 0, ++ &bus_id); ++ if (ret) ++ continue; ++ ++ ret = of_property_read_u32_index(child, "cfg-val", 0, ++ &cfg); ++ if (ret) { ++ dev_info(dev, "get cfg-val error\n"); ++ continue; ++ } ++ ++ if (!cfg) { ++ dev_info(dev, "cfg-val invalid\n"); ++ continue; ++ } ++ ++ ret = of_property_read_u32_index(child, "enable-msk", 0, ++ &enable_msk); ++ if (ret) { ++ dev_info(dev, "get enable_msk error\n"); ++ continue; ++ } ++ ++ ret = rockchip_sip_bus_smc_config(bus_id, cfg, ++ enable_msk); ++ if (ret) { ++ dev_info(dev, "bus smc config error: %x!\n", ret); ++ break; ++ } ++ } ++ ++ return 0; ++} ++ ++static int rockchip_bus_set_freq_table(struct rockchip_bus *bus) ++{ ++ struct device *dev = bus->dev; ++ struct dev_pm_opp *opp; ++ unsigned long freq; ++ int i, count; ++ ++ count = dev_pm_opp_get_opp_count(dev); ++ if (count <= 0) ++ return -EINVAL; ++ ++ bus->max_state = count; ++ bus->freq_table = devm_kcalloc(dev, ++ bus->max_state, ++ sizeof(*bus->freq_table), ++ GFP_KERNEL); ++ if (!bus->freq_table) { ++ bus->max_state = 0; ++ return -ENOMEM; ++ } ++ ++ for (i = 0, freq = 0; i < bus->max_state; i++, freq++) { ++ opp = dev_pm_opp_find_freq_ceil(dev, &freq); ++ if (IS_ERR(opp)) { ++ devm_kfree(dev, bus->freq_table); ++ bus->max_state = 0; ++ return PTR_ERR(opp); ++ } ++ bus->freq_table[i].volt = dev_pm_opp_get_voltage(opp); ++ bus->freq_table[i].freq = freq; ++ dev_pm_opp_put(opp); ++ } ++ ++ return 0; ++} ++ ++static int rockchip_bus_power_control_init(struct rockchip_bus *bus) ++{ ++ struct device *dev = bus->dev; ++ int ret = 0; ++ ++ bus->clk = devm_clk_get(dev, "bus"); ++ if (IS_ERR(bus->clk)) { ++ dev_err(dev, "failed to get bus clock\n"); ++ return PTR_ERR(bus->clk); ++ } ++ ++ bus->regulator = devm_regulator_get(dev, "bus"); ++ if (IS_ERR(bus->regulator)) { ++ dev_err(dev, "failed to get bus regulator\n"); ++ return PTR_ERR(bus->regulator); ++ } ++ ++ ret = rockchip_init_opp_table(dev, NULL, "leakage", "pvtm"); ++ if (ret < 0) { ++ dev_err(dev, "failed to get OPP table\n"); ++ return ret; ++ } ++ ++ ret = rockchip_bus_set_freq_table(bus); ++ if (ret < 0) { ++ dev_err(dev, "failed to set bus freq table\n"); ++ return ret; ++ } ++ ++ return 0; ++} ++ ++static int rockchip_bus_clkfreq_target(struct device *dev, unsigned long freq) ++{ ++ struct rockchip_bus *bus = dev_get_drvdata(dev); ++ unsigned long target_volt = bus->freq_table[bus->max_state - 1].volt; ++ int i; ++ ++ for (i = 0; i < bus->max_state; i++) { ++ if (freq <= bus->freq_table[i].freq) { ++ target_volt = bus->freq_table[i].volt; ++ break; ++ } ++ } ++ ++ if (bus->cur_volt != target_volt) { ++ dev_dbg(bus->dev, "target_volt: %lu\n", target_volt); ++ if (regulator_set_voltage(bus->regulator, target_volt, ++ INT_MAX)) { ++ dev_err(dev, "failed to set voltage %lu uV\n", ++ target_volt); ++ return -EINVAL; ++ } ++ bus->cur_volt = target_volt; ++ } ++ ++ return 0; ++} ++ ++static int rockchip_bus_clk_notifier(struct notifier_block *nb, ++ unsigned long event, void *data) ++{ ++ struct clk_notifier_data *ndata = data; ++ struct rockchip_bus *bus = to_rockchip_bus_clk_nb(nb); ++ int ret = 0; ++ ++ dev_dbg(bus->dev, "event %lu, old_rate %lu, new_rate: %lu\n", ++ event, ndata->old_rate, ndata->new_rate); ++ ++ switch (event) { ++ case PRE_RATE_CHANGE: ++ if (ndata->new_rate > ndata->old_rate) ++ ret = rockchip_bus_clkfreq_target(bus->dev, ++ ndata->new_rate); ++ break; ++ case POST_RATE_CHANGE: ++ if (ndata->new_rate < ndata->old_rate) ++ ret = rockchip_bus_clkfreq_target(bus->dev, ++ ndata->new_rate); ++ break; ++ case ABORT_RATE_CHANGE: ++ if (ndata->new_rate > ndata->old_rate) ++ ret = rockchip_bus_clkfreq_target(bus->dev, ++ ndata->old_rate); ++ break; ++ default: ++ break; ++ } ++ ++ return notifier_from_errno(ret); ++} ++ ++static int rockchip_bus_clkfreq(struct rockchip_bus *bus) ++{ ++ struct device *dev = bus->dev; ++ unsigned long init_rate; ++ int ret = 0; ++ ++ ret = rockchip_bus_power_control_init(bus); ++ if (ret) { ++ dev_err(dev, "failed to init power control\n"); ++ return ret; ++ } ++ ++ init_rate = clk_get_rate(bus->clk); ++ ret = rockchip_bus_clkfreq_target(dev, init_rate); ++ if (ret) ++ return ret; ++ ++ bus->clk_nb.notifier_call = rockchip_bus_clk_notifier; ++ ret = clk_notifier_register(bus->clk, &bus->clk_nb); ++ if (ret) { ++ dev_err(dev, "failed to register clock notifier\n"); ++ return ret; ++ } ++ ++ return 0; ++} ++ ++static int rockchip_bus_cpufreq_target(struct device *dev, unsigned long freq, ++ u32 flags) ++{ ++ struct rockchip_bus *bus = dev_get_drvdata(dev); ++ struct dev_pm_opp *opp; ++ unsigned long target_volt, target_rate = freq; ++ int ret = 0; ++ ++ if (!bus->regulator) { ++ dev_dbg(dev, "%luHz -> %luHz\n", bus->cur_rate, target_rate); ++ ret = clk_set_rate(bus->clk, target_rate); ++ if (ret) ++ dev_err(bus->dev, "failed to set bus rate %lu\n", ++ target_rate); ++ else ++ bus->cur_rate = target_rate; ++ return ret; ++ } ++ ++ opp = devfreq_recommended_opp(dev, &target_rate, flags); ++ if (IS_ERR(opp)) { ++ dev_err(dev, "failed to recommended opp %lu\n", target_rate); ++ return PTR_ERR(opp); ++ } ++ target_volt = dev_pm_opp_get_voltage(opp); ++ dev_pm_opp_put(opp); ++ ++ if (bus->cur_rate == target_rate) { ++ if (bus->cur_volt == target_volt) ++ return 0; ++ ret = regulator_set_voltage(bus->regulator, target_volt, ++ INT_MAX); ++ if (ret) { ++ dev_err(dev, "failed to set voltage %lu\n", ++ target_volt); ++ return ret; ++ } ++ bus->cur_volt = target_volt; ++ return 0; ++ } else if (!bus->cur_volt) { ++ bus->cur_volt = regulator_get_voltage(bus->regulator); ++ } ++ ++ if (bus->cur_rate < target_rate) { ++ ret = regulator_set_voltage(bus->regulator, target_volt, ++ INT_MAX); ++ if (ret) { ++ dev_err(dev, "failed to set voltage %lu\n", ++ target_volt); ++ return ret; ++ } ++ } ++ ++ ret = clk_set_rate(bus->clk, target_rate); ++ if (ret) { ++ dev_err(dev, "failed to set bus rate %lu\n", target_rate); ++ return ret; ++ } ++ ++ if (bus->cur_rate > target_rate) { ++ ret = regulator_set_voltage(bus->regulator, target_volt, ++ INT_MAX); ++ if (ret) { ++ dev_err(dev, "failed to set voltage %lu\n", ++ target_volt); ++ return ret; ++ } ++ } ++ ++ dev_dbg(dev, "%luHz %luuV -> %luHz %luuV\n", bus->cur_rate, ++ bus->cur_volt, target_rate, target_volt); ++ bus->cur_rate = target_rate; ++ bus->cur_volt = target_volt; ++ ++ return ret; ++} ++ ++static int rockchip_bus_cpufreq_notifier(struct notifier_block *nb, ++ unsigned long event, void *data) ++{ ++ struct rockchip_bus *bus = to_rockchip_bus_cpufreq_nb(nb); ++ struct cpufreq_freqs *freqs = data; ++ int id = topology_physical_package_id(freqs->policy->cpu); ++ ++ if (id < 0 || id >= MAX_CLUSTERS) ++ return NOTIFY_DONE; ++ ++ bus->cpu_freq[id] = freqs->new; ++ ++ if (!bus->cpu_freq[CLUSTER0] || !bus->cpu_freq[CLUSTER1]) ++ return NOTIFY_DONE; ++ ++ switch (event) { ++ case CPUFREQ_PRECHANGE: ++ if ((bus->cpu_freq[CLUSTER0] > bus->cpu_high_freq || ++ bus->cpu_freq[CLUSTER1] > bus->cpu_high_freq) && ++ bus->cur_rate != bus->high_rate) { ++ dev_dbg(bus->dev, "cpu%d freq=%d %d, up cci rate to %lu\n", ++ freqs->policy->cpu, ++ bus->cpu_freq[CLUSTER0], ++ bus->cpu_freq[CLUSTER1], ++ bus->high_rate); ++ rockchip_bus_cpufreq_target(bus->dev, bus->high_rate, ++ 0); ++ } ++ break; ++ case CPUFREQ_POSTCHANGE: ++ if (bus->cpu_freq[CLUSTER0] <= bus->cpu_high_freq && ++ bus->cpu_freq[CLUSTER1] <= bus->cpu_high_freq && ++ bus->cur_rate != bus->low_rate) { ++ dev_dbg(bus->dev, "cpu%d freq=%d %d, down cci rate to %lu\n", ++ freqs->policy->cpu, ++ bus->cpu_freq[CLUSTER0], ++ bus->cpu_freq[CLUSTER1], ++ bus->low_rate); ++ rockchip_bus_cpufreq_target(bus->dev, bus->low_rate, ++ 0); ++ } ++ break; ++ } ++ ++ return NOTIFY_OK; ++} ++ ++static int rockchip_bus_cpufreq(struct rockchip_bus *bus) ++{ ++ struct device *dev = bus->dev; ++ struct device_node *np = dev->of_node; ++ unsigned int freq; ++ int ret = 0; ++ ++ if (of_parse_phandle(dev->of_node, "operating-points-v2", 0)) { ++ ret = rockchip_bus_power_control_init(bus); ++ if (ret) { ++ dev_err(dev, "failed to init power control\n"); ++ return ret; ++ } ++ } else { ++ bus->clk = devm_clk_get(dev, "bus"); ++ if (IS_ERR(bus->clk)) { ++ dev_err(dev, "failed to get bus clock\n"); ++ return PTR_ERR(bus->clk); ++ } ++ bus->regulator = NULL; ++ } ++ ++ ret = of_property_read_u32(np, "cpu-high-freq", &bus->cpu_high_freq); ++ if (ret) { ++ dev_err(dev, "failed to get cpu-high-freq\n"); ++ return ret; ++ } ++ ret = of_property_read_u32(np, "cci-high-freq", &freq); ++ if (ret) { ++ dev_err(dev, "failed to get cci-high-freq\n"); ++ return ret; ++ } ++ bus->high_rate = freq * 1000; ++ ret = of_property_read_u32(np, "cci-low-freq", &freq); ++ if (ret) { ++ dev_err(dev, "failed to get cci-low-freq\n"); ++ return ret; ++ } ++ bus->low_rate = freq * 1000; ++ ++ bus->cpufreq_nb.notifier_call = rockchip_bus_cpufreq_notifier; ++ ret = cpufreq_register_notifier(&bus->cpufreq_nb, ++ CPUFREQ_TRANSITION_NOTIFIER); ++ if (ret) { ++ dev_err(dev, "failed to register cpufreq notifier\n"); ++ return ret; ++ } ++ ++ return 0; ++} ++ ++static const struct of_device_id rockchip_busfreq_of_match[] = { ++ { .compatible = "rockchip,px30-bus", }, ++ { .compatible = "rockchip,rk1808-bus", }, ++ { .compatible = "rockchip,rk3288-bus", }, ++ { .compatible = "rockchip,rk3368-bus", }, ++ { .compatible = "rockchip,rk3399-bus", }, ++ { .compatible = "rockchip,rk3568-bus", }, ++ { .compatible = "rockchip,rv1126-bus", }, ++ { }, ++}; ++ ++MODULE_DEVICE_TABLE(of, rockchip_busfreq_of_match); ++ ++static int rockchip_busfreq_probe(struct platform_device *pdev) ++{ ++ struct device *dev = &pdev->dev; ++ struct device_node *np = dev->of_node; ++ struct rockchip_bus *bus; ++ const char *policy_name; ++ int ret = 0; ++ ++ bus = devm_kzalloc(dev, sizeof(*bus), GFP_KERNEL); ++ if (!bus) ++ return -ENOMEM; ++ bus->dev = dev; ++ platform_set_drvdata(pdev, bus); ++ ++ ret = of_property_read_string(np, "rockchip,busfreq-policy", ++ &policy_name); ++ if (ret) { ++ dev_info(dev, "failed to get busfreq policy\n"); ++ return ret; ++ } ++ ++ if (!strcmp(policy_name, "smc")) ++ ret = rockchip_bus_smc_config(bus); ++ else if (!strcmp(policy_name, "clkfreq")) ++ ret = rockchip_bus_clkfreq(bus); ++ else if (!strcmp(policy_name, "cpufreq")) ++ ret = rockchip_bus_cpufreq(bus); ++ ++ return ret; ++} ++ ++static struct platform_driver rockchip_busfreq_driver = { ++ .probe = rockchip_busfreq_probe, ++ .driver = { ++ .name = "rockchip,bus", ++ .of_match_table = rockchip_busfreq_of_match, ++ }, ++}; ++ ++module_platform_driver(rockchip_busfreq_driver); ++ ++MODULE_LICENSE("GPL v2"); ++MODULE_AUTHOR("Tony Xie "); ++MODULE_DESCRIPTION("rockchip busfreq driver with devfreq framework"); +diff --git a/drivers/devfreq/rockchip_dmc.c b/drivers/devfreq/rockchip_dmc.c +new file mode 100755 +index 000000000000..4cb817617498 +--- /dev/null ++++ b/drivers/devfreq/rockchip_dmc.c +@@ -0,0 +1,3320 @@ ++/* ++ * Copyright (c) 2016, Fuzhou Rockchip Electronics Co., Ltd. ++ * Author: Lin Huang ++ * ++ * This program is free software; you can redistribute it and/or modify it ++ * under the terms and conditions of the GNU General Public License, ++ * version 2, as published by the Free Software Foundation. ++ * ++ * This program is distributed in the hope it will be useful, but WITHOUT ++ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or ++ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for ++ * more details. ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "governor.h" ++#include "rockchip_dmc_timing.h" ++#include "../clk/rockchip/clk.h" ++#include "../gpu/drm/rockchip/rockchip_drm_drv.h" ++ ++#define system_status_to_dmcfreq(nb) container_of(nb, struct rockchip_dmcfreq, \ ++ status_nb) ++#define reboot_to_dmcfreq(nb) container_of(nb, struct rockchip_dmcfreq, \ ++ reboot_nb) ++#define boost_to_dmcfreq(work) container_of(work, struct rockchip_dmcfreq, \ ++ boost_work) ++#define msch_rl_to_dmcfreq(work) container_of(to_delayed_work(work), \ ++ struct rockchip_dmcfreq, \ ++ msch_rl_work) ++#define input_hd_to_dmcfreq(hd) container_of(hd, struct rockchip_dmcfreq, \ ++ input_handler) ++ ++#define VIDEO_1080P_SIZE (1920 * 1080) ++#define FIQ_INIT_HANDLER (0x1) ++#define FIQ_CPU_TGT_BOOT (0x0) /* to booting cpu */ ++#define FIQ_NUM_FOR_DCF (143) /* NA irq map to fiq for dcf */ ++#define DTS_PAR_OFFSET (4096) ++#define MSCH_RL_DELAY_TIME 50 /* ms */ ++ ++#define FALLBACK_STATIC_TEMPERATURE 55000 ++ ++struct freq_map_table { ++ unsigned int min; ++ unsigned int max; ++ unsigned long freq; ++}; ++ ++struct rl_map_table { ++ unsigned int pn; /* panel number */ ++ unsigned int rl; /* readlatency */ ++}; ++ ++struct dmc_freq_table { ++ unsigned long freq; ++ unsigned long volt; ++}; ++ ++struct share_params { ++ u32 hz; ++ u32 lcdc_type; ++ u32 vop; ++ u32 vop_dclk_mode; ++ u32 sr_idle_en; ++ u32 addr_mcu_el3; ++ /* ++ * 1: need to wait flag1 ++ * 0: never wait flag1 ++ */ ++ u32 wait_flag1; ++ /* ++ * 1: need to wait flag1 ++ * 0: never wait flag1 ++ */ ++ u32 wait_flag0; ++ u32 complt_hwirq; ++ u32 update_drv_odt_cfg; ++ u32 update_deskew_cfg; ++ ++ u32 freq_count; ++ u32 freq_info_mhz[6]; ++ /* if need, add parameter after */ ++}; ++ ++static struct share_params *ddr_psci_param; ++ ++struct rockchip_dmcfreq { ++ struct device *dev; ++ struct devfreq *devfreq; ++ struct devfreq_simple_ondemand_data ondemand_data; ++ struct clk *dmc_clk; ++ struct devfreq_event_dev **edev; ++ struct mutex lock; /* serializes access to video_info_list */ ++ struct dram_timing *timing; ++ struct regulator *vdd_center; ++ struct notifier_block status_nb; ++ struct list_head video_info_list; ++ struct freq_map_table *vop_bw_tbl; ++ struct work_struct boost_work; ++ struct input_handler input_handler; ++ struct monitor_dev_info *mdev_info; ++ struct rl_map_table *vop_pn_rl_tbl; ++ struct delayed_work msch_rl_work; ++ struct share_params *set_rate_params; ++ ++ unsigned long *nocp_bw; ++ unsigned long rate, target_rate; ++ unsigned long volt, target_volt; ++ ++ unsigned long auto_min_rate; ++ unsigned long status_rate; ++ unsigned long normal_rate; ++ unsigned long video_1080p_rate; ++ unsigned long video_4k_rate; ++ unsigned long video_4k_10b_rate; ++ unsigned long performance_rate; ++ unsigned long hdmi_rate; ++ unsigned long idle_rate; ++ unsigned long suspend_rate; ++ unsigned long reboot_rate; ++ unsigned long boost_rate; ++ unsigned long fixed_rate; ++ unsigned long low_power_rate; ++ unsigned long vop_req_rate; ++ ++ unsigned long freq_count; ++ unsigned long freq_info_rate[6]; ++ unsigned long rate_low; ++ unsigned long rate_mid_low; ++ unsigned long rate_mid_high; ++ unsigned long rate_high; ++ ++ unsigned int min_cpu_freq; ++ unsigned int auto_freq_en; ++ unsigned int system_status_en; ++ unsigned int refresh; ++ unsigned int last_refresh; ++ unsigned int read_latency; ++ int edev_count; ++ int dfi_id; ++ ++ bool is_fixed; ++ bool is_msch_rl_work_started; ++ bool is_set_rate_direct; ++ ++ struct thermal_cooling_device *devfreq_cooling; ++ u32 static_coefficient; ++ s32 ts[4]; ++ struct thermal_zone_device *ddr_tz; ++ ++ unsigned int touchboostpulse_duration_val; ++ u64 touchboostpulse_endtime; ++ ++ int (*set_auto_self_refresh)(u32 en); ++ int (*set_msch_readlatency)(unsigned int rl); ++}; ++ ++static struct rockchip_dmcfreq *rk_dmcfreq; ++static struct pm_qos_request pm_qos; ++ ++static DECLARE_RWSEM(rockchip_dmcfreq_sem); ++ ++static inline unsigned long is_dualview(unsigned long status) ++{ ++ return (status & SYS_STATUS_LCDC0) && (status & SYS_STATUS_LCDC1); ++} ++ ++static inline unsigned long is_isp(unsigned long status) ++{ ++ return (status & SYS_STATUS_ISP) || ++ (status & SYS_STATUS_CIF0) || ++ (status & SYS_STATUS_CIF1); ++} ++ ++void rockchip_dmcfreq_lock(void) ++{ ++ down_read(&rockchip_dmcfreq_sem); ++} ++EXPORT_SYMBOL(rockchip_dmcfreq_lock); ++ ++void rockchip_dmcfreq_lock_nested(void) ++{ ++ down_read_nested(&rockchip_dmcfreq_sem, SINGLE_DEPTH_NESTING); ++} ++EXPORT_SYMBOL(rockchip_dmcfreq_lock_nested); ++ ++void rockchip_dmcfreq_unlock(void) ++{ ++ up_read(&rockchip_dmcfreq_sem); ++} ++EXPORT_SYMBOL(rockchip_dmcfreq_unlock); ++ ++/* ++ * function: packaging de-skew setting to px30_ddr_dts_config_timing, ++ * px30_ddr_dts_config_timing will pass to trust firmware, and ++ * used direct to set register. ++ * input: de_skew ++ * output: tim ++ */ ++static void px30_de_skew_set_2_reg(struct rk3328_ddr_de_skew_setting *de_skew, ++ struct px30_ddr_dts_config_timing *tim) ++{ ++ u32 n; ++ u32 offset; ++ u32 shift; ++ ++ memset_io(tim->ca_skew, 0, sizeof(tim->ca_skew)); ++ memset_io(tim->cs0_skew, 0, sizeof(tim->cs0_skew)); ++ memset_io(tim->cs1_skew, 0, sizeof(tim->cs1_skew)); ++ ++ /* CA de-skew */ ++ for (n = 0; n < ARRAY_SIZE(de_skew->ca_de_skew); n++) { ++ offset = n / 2; ++ shift = n % 2; ++ /* 0 => 4; 1 => 0 */ ++ shift = (shift == 0) ? 4 : 0; ++ tim->ca_skew[offset] &= ~(0xf << shift); ++ tim->ca_skew[offset] |= (de_skew->ca_de_skew[n] << shift); ++ } ++ ++ /* CS0 data de-skew */ ++ for (n = 0; n < ARRAY_SIZE(de_skew->cs0_de_skew); n++) { ++ offset = ((n / 21) * 11) + ((n % 21) / 2); ++ shift = ((n % 21) % 2); ++ if ((n % 21) == 20) ++ shift = 0; ++ else ++ /* 0 => 4; 1 => 0 */ ++ shift = (shift == 0) ? 4 : 0; ++ tim->cs0_skew[offset] &= ~(0xf << shift); ++ tim->cs0_skew[offset] |= (de_skew->cs0_de_skew[n] << shift); ++ } ++ ++ /* CS1 data de-skew */ ++ for (n = 0; n < ARRAY_SIZE(de_skew->cs1_de_skew); n++) { ++ offset = ((n / 21) * 11) + ((n % 21) / 2); ++ shift = ((n % 21) % 2); ++ if ((n % 21) == 20) ++ shift = 0; ++ else ++ /* 0 => 4; 1 => 0 */ ++ shift = (shift == 0) ? 4 : 0; ++ tim->cs1_skew[offset] &= ~(0xf << shift); ++ tim->cs1_skew[offset] |= (de_skew->cs1_de_skew[n] << shift); ++ } ++} ++ ++/* ++ * function: packaging de-skew setting to rk3328_ddr_dts_config_timing, ++ * rk3328_ddr_dts_config_timing will pass to trust firmware, and ++ * used direct to set register. ++ * input: de_skew ++ * output: tim ++ */ ++static void ++rk3328_de_skew_setting_2_register(struct rk3328_ddr_de_skew_setting *de_skew, ++ struct rk3328_ddr_dts_config_timing *tim) ++{ ++ u32 n; ++ u32 offset; ++ u32 shift; ++ ++ memset_io(tim->ca_skew, 0, sizeof(tim->ca_skew)); ++ memset_io(tim->cs0_skew, 0, sizeof(tim->cs0_skew)); ++ memset_io(tim->cs1_skew, 0, sizeof(tim->cs1_skew)); ++ ++ /* CA de-skew */ ++ for (n = 0; n < ARRAY_SIZE(de_skew->ca_de_skew); n++) { ++ offset = n / 2; ++ shift = n % 2; ++ /* 0 => 4; 1 => 0 */ ++ shift = (shift == 0) ? 4 : 0; ++ tim->ca_skew[offset] &= ~(0xf << shift); ++ tim->ca_skew[offset] |= (de_skew->ca_de_skew[n] << shift); ++ } ++ ++ /* CS0 data de-skew */ ++ for (n = 0; n < ARRAY_SIZE(de_skew->cs0_de_skew); n++) { ++ offset = ((n / 21) * 11) + ((n % 21) / 2); ++ shift = ((n % 21) % 2); ++ if ((n % 21) == 20) ++ shift = 0; ++ else ++ /* 0 => 4; 1 => 0 */ ++ shift = (shift == 0) ? 4 : 0; ++ tim->cs0_skew[offset] &= ~(0xf << shift); ++ tim->cs0_skew[offset] |= (de_skew->cs0_de_skew[n] << shift); ++ } ++ ++ /* CS1 data de-skew */ ++ for (n = 0; n < ARRAY_SIZE(de_skew->cs1_de_skew); n++) { ++ offset = ((n / 21) * 11) + ((n % 21) / 2); ++ shift = ((n % 21) % 2); ++ if ((n % 21) == 20) ++ shift = 0; ++ else ++ /* 0 => 4; 1 => 0 */ ++ shift = (shift == 0) ? 4 : 0; ++ tim->cs1_skew[offset] &= ~(0xf << shift); ++ tim->cs1_skew[offset] |= (de_skew->cs1_de_skew[n] << shift); ++ } ++} ++ ++static int rk_drm_get_lcdc_type(void) ++{ ++ u32 lcdc_type = rockchip_drm_get_sub_dev_type(); ++ ++ switch (lcdc_type) { ++ case DRM_MODE_CONNECTOR_DPI: ++ case DRM_MODE_CONNECTOR_LVDS: ++ lcdc_type = SCREEN_LVDS; ++ break; ++ case DRM_MODE_CONNECTOR_DisplayPort: ++ lcdc_type = SCREEN_DP; ++ break; ++ case DRM_MODE_CONNECTOR_HDMIA: ++ case DRM_MODE_CONNECTOR_HDMIB: ++ lcdc_type = SCREEN_HDMI; ++ break; ++ case DRM_MODE_CONNECTOR_TV: ++ lcdc_type = SCREEN_TVOUT; ++ break; ++ case DRM_MODE_CONNECTOR_eDP: ++ lcdc_type = SCREEN_EDP; ++ break; ++ case DRM_MODE_CONNECTOR_DSI: ++ lcdc_type = SCREEN_MIPI; ++ break; ++ default: ++ lcdc_type = SCREEN_NULL; ++ break; ++ } ++ ++ return lcdc_type; ++} ++ ++static int rockchip_ddr_set_rate(unsigned long target_rate) ++{ ++ struct arm_smccc_res res; ++ ++ ddr_psci_param->hz = target_rate; ++ ddr_psci_param->lcdc_type = rk_drm_get_lcdc_type(); ++ ddr_psci_param->wait_flag1 = 1; ++ ddr_psci_param->wait_flag0 = 1; ++ ++ res = sip_smc_dram(SHARE_PAGE_TYPE_DDR, 0, ++ ROCKCHIP_SIP_CONFIG_DRAM_SET_RATE); ++ ++ if ((int)res.a1 == SIP_RET_SET_RATE_TIMEOUT) ++ rockchip_dmcfreq_wait_complete(); ++ ++ return res.a0; ++} ++ ++static int rockchip_dmcfreq_target(struct device *dev, unsigned long *freq, ++ u32 flags) ++{ ++ struct rockchip_dmcfreq *dmcfreq = dev_get_drvdata(dev); ++ struct dev_pm_opp *opp; ++ struct cpufreq_policy *policy; ++ unsigned long old_clk_rate = dmcfreq->rate; ++ unsigned long target_volt, target_rate; ++ unsigned int cpu_cur, cpufreq_cur; ++ bool is_cpufreq_changed = false; ++ int err = 0; ++ ++ opp = devfreq_recommended_opp(dev, freq, flags); ++ if (IS_ERR(opp)) { ++ dev_err(dev, "Failed to find opp for %lu Hz\n", *freq); ++ return PTR_ERR(opp); ++ } ++ target_volt = dev_pm_opp_get_voltage(opp); ++ dev_pm_opp_put(opp); ++ ++ if (dmcfreq->is_set_rate_direct) { ++ target_rate = *freq; ++ } else { ++ target_rate = clk_round_rate(dmcfreq->dmc_clk, *freq); ++ if ((long)target_rate <= 0) ++ target_rate = *freq; ++ } ++ ++ if (dmcfreq->rate == target_rate) { ++ if (dmcfreq->volt == target_volt) ++ return 0; ++ err = regulator_set_voltage(dmcfreq->vdd_center, target_volt, ++ INT_MAX); ++ if (err) { ++ dev_err(dev, "Cannot set voltage %lu uV\n", ++ target_volt); ++ return err; ++ } ++ dmcfreq->volt = target_volt; ++ return 0; ++ } else if (!dmcfreq->volt) { ++ dmcfreq->volt = regulator_get_voltage(dmcfreq->vdd_center); ++ } ++ ++ /* ++ * We need to prevent cpu hotplug from happening while a dmc freq rate ++ * change is happening. ++ * ++ * Do this before taking the policy rwsem to avoid deadlocks between the ++ * mutex that is locked/unlocked in cpu_hotplug_disable/enable. And it ++ * can also avoid deadlocks between the mutex that is locked/unlocked ++ * in get/put_online_cpus (such as store_scaling_max_freq()). ++ */ ++ get_online_cpus(); ++ ++ /* ++ * Go to specified cpufreq and block other cpufreq changes since ++ * set_rate needs to complete during vblank. ++ */ ++ cpu_cur = raw_smp_processor_id(); ++ policy = cpufreq_cpu_get(cpu_cur); ++ if (!policy) { ++ dev_err(dev, "cpu%d policy NULL\n", cpu_cur); ++ goto cpufreq; ++ } ++ down_write(&policy->rwsem); ++ cpufreq_cur = cpufreq_quick_get(cpu_cur); ++ ++ /* If we're thermally throttled; don't change; */ ++ if (dmcfreq->min_cpu_freq && cpufreq_cur < dmcfreq->min_cpu_freq) { ++ if (policy->max >= dmcfreq->min_cpu_freq) { ++ __cpufreq_driver_target(policy, dmcfreq->min_cpu_freq, ++ CPUFREQ_RELATION_L); ++ is_cpufreq_changed = true; ++ } else { ++ dev_dbg(dev, "CPU may too slow for DMC (%d MHz)\n", ++ policy->max); ++ } ++ } ++ ++ /* ++ * If frequency scaling from low to high, adjust voltage first. ++ * If frequency scaling from high to low, adjust frequency first. ++ */ ++ if (old_clk_rate < target_rate) { ++ err = regulator_set_voltage(dmcfreq->vdd_center, target_volt, ++ INT_MAX); ++ if (err) { ++ dev_err(dev, "Cannot set voltage %lu uV\n", ++ target_volt); ++ goto out; ++ } ++ } ++ ++ /* ++ * Writer in rwsem may block readers even during its waiting in queue, ++ * and this may lead to a deadlock when the code path takes read sem ++ * twice (e.g. one in vop_lock() and another in rockchip_pmu_lock()). ++ * As a (suboptimal) workaround, let writer to spin until it gets the ++ * lock. ++ */ ++ while (!down_write_trylock(&rockchip_dmcfreq_sem)) ++ cond_resched(); ++ dev_dbg(dev, "%lu-->%lu\n", old_clk_rate, target_rate); ++ ++ if (dmcfreq->set_rate_params) { ++ dmcfreq->set_rate_params->lcdc_type = rk_drm_get_lcdc_type(); ++ dmcfreq->set_rate_params->wait_flag1 = 1; ++ dmcfreq->set_rate_params->wait_flag0 = 1; ++ } ++ ++ if (dmcfreq->is_set_rate_direct) ++ err = rockchip_ddr_set_rate(target_rate); ++ else ++ err = clk_set_rate(dmcfreq->dmc_clk, target_rate); ++ ++ up_write(&rockchip_dmcfreq_sem); ++ if (err) { ++ dev_err(dev, "Cannot set frequency %lu (%d)\n", ++ target_rate, err); ++ regulator_set_voltage(dmcfreq->vdd_center, dmcfreq->volt, ++ INT_MAX); ++ goto out; ++ } ++ ++ /* ++ * Check the dpll rate, ++ * There only two result we will get, ++ * 1. Ddr frequency scaling fail, we still get the old rate. ++ * 2. Ddr frequency scaling sucessful, we get the rate we set. ++ */ ++ dmcfreq->rate = clk_get_rate(dmcfreq->dmc_clk); ++ ++ /* If get the incorrect rate, set voltage to old value. */ ++ if (dmcfreq->rate != target_rate) { ++ dev_err(dev, "Get wrong frequency, Request %lu, Current %lu\n", ++ target_rate, dmcfreq->rate); ++ regulator_set_voltage(dmcfreq->vdd_center, dmcfreq->volt, ++ INT_MAX); ++ goto out; ++ } else if (old_clk_rate > target_rate) { ++ err = regulator_set_voltage(dmcfreq->vdd_center, target_volt, ++ INT_MAX); ++ if (err) { ++ dev_err(dev, "Cannot set vol %lu uV\n", target_volt); ++ goto out; ++ } ++ } ++ ++ if (dmcfreq->devfreq) ++ dmcfreq->devfreq->last_status.current_frequency = *freq; ++ ++ dmcfreq->volt = target_volt; ++out: ++ if (is_cpufreq_changed) ++ __cpufreq_driver_target(policy, cpufreq_cur, ++ CPUFREQ_RELATION_L); ++ up_write(&policy->rwsem); ++ cpufreq_cpu_put(policy); ++cpufreq: ++ put_online_cpus(); ++ return err; ++} ++ ++static int rockchip_dmcfreq_get_dev_status(struct device *dev, ++ struct devfreq_dev_status *stat) ++{ ++ struct rockchip_dmcfreq *dmcfreq = dev_get_drvdata(dev); ++ struct devfreq_event_data edata; ++ int i, j, ret = 0; ++ ++ if (!dmcfreq->auto_freq_en) ++ return -EINVAL; ++ ++ if (dmcfreq->dfi_id >= 0) { ++ ret = devfreq_event_get_event(dmcfreq->edev[dmcfreq->dfi_id], ++ &edata); ++ if (ret < 0) { ++ dev_err(dev, "failed to get dfi event\n"); ++ return ret; ++ } ++ stat->busy_time = edata.load_count; ++ stat->total_time = edata.total_count; ++ } ++ ++ for (i = 0, j = 0; i < dmcfreq->edev_count; i++) { ++ if (i == dmcfreq->dfi_id) ++ continue; ++ ret = devfreq_event_get_event(dmcfreq->edev[i], &edata); ++ if (ret < 0) { ++ dev_err(dev, "failed to get event %s\n", ++ dmcfreq->edev[i]->desc->name); ++ return ret; ++ } ++ dmcfreq->nocp_bw[j++] = edata.load_count; ++ } ++ ++ return 0; ++} ++ ++static int rockchip_dmcfreq_get_cur_freq(struct device *dev, ++ unsigned long *freq) ++{ ++ struct rockchip_dmcfreq *dmcfreq = dev_get_drvdata(dev); ++ ++ *freq = dmcfreq->rate; ++ ++ return 0; ++} ++ ++static struct devfreq_dev_profile rockchip_devfreq_dmc_profile = { ++ .polling_ms = 50, ++ .target = rockchip_dmcfreq_target, ++ .get_dev_status = rockchip_dmcfreq_get_dev_status, ++ .get_cur_freq = rockchip_dmcfreq_get_cur_freq, ++}; ++ ++ ++static inline void reset_last_status(struct devfreq *devfreq) ++{ ++ devfreq->last_status.total_time = 1; ++ devfreq->last_status.busy_time = 1; ++} ++ ++static void of_get_px30_timings(struct device *dev, ++ struct device_node *np, uint32_t *timing) ++{ ++ struct device_node *np_tim; ++ u32 *p; ++ struct px30_ddr_dts_config_timing *dts_timing; ++ struct rk3328_ddr_de_skew_setting *de_skew; ++ int ret = 0; ++ u32 i; ++ ++ dts_timing = ++ (struct px30_ddr_dts_config_timing *)(timing + ++ DTS_PAR_OFFSET / 4); ++ ++ np_tim = of_parse_phandle(np, "ddr_timing", 0); ++ if (!np_tim) { ++ ret = -EINVAL; ++ goto end; ++ } ++ de_skew = kmalloc(sizeof(*de_skew), GFP_KERNEL); ++ if (!de_skew) { ++ ret = -ENOMEM; ++ goto end; ++ } ++ p = (u32 *)dts_timing; ++ for (i = 0; i < ARRAY_SIZE(px30_dts_timing); i++) { ++ ret |= of_property_read_u32(np_tim, px30_dts_timing[i], ++ p + i); ++ } ++ p = (u32 *)de_skew->ca_de_skew; ++ for (i = 0; i < ARRAY_SIZE(rk3328_dts_ca_timing); i++) { ++ ret |= of_property_read_u32(np_tim, rk3328_dts_ca_timing[i], ++ p + i); ++ } ++ p = (u32 *)de_skew->cs0_de_skew; ++ for (i = 0; i < ARRAY_SIZE(rk3328_dts_cs0_timing); i++) { ++ ret |= of_property_read_u32(np_tim, rk3328_dts_cs0_timing[i], ++ p + i); ++ } ++ p = (u32 *)de_skew->cs1_de_skew; ++ for (i = 0; i < ARRAY_SIZE(rk3328_dts_cs1_timing); i++) { ++ ret |= of_property_read_u32(np_tim, rk3328_dts_cs1_timing[i], ++ p + i); ++ } ++ if (!ret) ++ px30_de_skew_set_2_reg(de_skew, dts_timing); ++ kfree(de_skew); ++end: ++ if (!ret) { ++ dts_timing->available = 1; ++ } else { ++ dts_timing->available = 0; ++ dev_err(dev, "of_get_ddr_timings: fail\n"); ++ } ++ ++ of_node_put(np_tim); ++} ++ ++static void of_get_rk1808_timings(struct device *dev, ++ struct device_node *np, uint32_t *timing) ++{ ++ struct device_node *np_tim; ++ u32 *p; ++ struct rk1808_ddr_dts_config_timing *dts_timing; ++ int ret = 0; ++ u32 i; ++ ++ dts_timing = ++ (struct rk1808_ddr_dts_config_timing *)(timing + ++ DTS_PAR_OFFSET / 4); ++ ++ np_tim = of_parse_phandle(np, "ddr_timing", 0); ++ if (!np_tim) { ++ ret = -EINVAL; ++ goto end; ++ } ++ ++ p = (u32 *)dts_timing; ++ for (i = 0; i < ARRAY_SIZE(px30_dts_timing); i++) { ++ ret |= of_property_read_u32(np_tim, px30_dts_timing[i], ++ p + i); ++ } ++ p = (u32 *)dts_timing->ca_de_skew; ++ for (i = 0; i < ARRAY_SIZE(rk1808_dts_ca_timing); i++) { ++ ret |= of_property_read_u32(np_tim, rk1808_dts_ca_timing[i], ++ p + i); ++ } ++ p = (u32 *)dts_timing->cs0_a_de_skew; ++ for (i = 0; i < ARRAY_SIZE(rk1808_dts_cs0_a_timing); i++) { ++ ret |= of_property_read_u32(np_tim, rk1808_dts_cs0_a_timing[i], ++ p + i); ++ } ++ p = (u32 *)dts_timing->cs0_b_de_skew; ++ for (i = 0; i < ARRAY_SIZE(rk1808_dts_cs0_b_timing); i++) { ++ ret |= of_property_read_u32(np_tim, rk1808_dts_cs0_b_timing[i], ++ p + i); ++ } ++ p = (u32 *)dts_timing->cs1_a_de_skew; ++ for (i = 0; i < ARRAY_SIZE(rk1808_dts_cs1_a_timing); i++) { ++ ret |= of_property_read_u32(np_tim, rk1808_dts_cs1_a_timing[i], ++ p + i); ++ } ++ p = (u32 *)dts_timing->cs1_b_de_skew; ++ for (i = 0; i < ARRAY_SIZE(rk1808_dts_cs1_b_timing); i++) { ++ ret |= of_property_read_u32(np_tim, rk1808_dts_cs1_b_timing[i], ++ p + i); ++ } ++ ++end: ++ if (!ret) { ++ dts_timing->available = 1; ++ } else { ++ dts_timing->available = 0; ++ dev_err(dev, "of_get_ddr_timings: fail\n"); ++ } ++ ++ of_node_put(np_tim); ++} ++ ++static void of_get_rk3128_timings(struct device *dev, ++ struct device_node *np, uint32_t *timing) ++{ ++ struct device_node *np_tim; ++ u32 *p; ++ struct rk3128_ddr_dts_config_timing *dts_timing; ++ struct share_params *init_timing; ++ int ret = 0; ++ u32 i; ++ ++ init_timing = (struct share_params *)timing; ++ ++ if (of_property_read_u32(np, "vop-dclk-mode", ++ &init_timing->vop_dclk_mode)) ++ init_timing->vop_dclk_mode = 0; ++ ++ p = timing + DTS_PAR_OFFSET / 4; ++ np_tim = of_parse_phandle(np, "rockchip,ddr_timing", 0); ++ if (!np_tim) { ++ ret = -EINVAL; ++ goto end; ++ } ++ for (i = 0; i < ARRAY_SIZE(rk3128_dts_timing); i++) { ++ ret |= of_property_read_u32(np_tim, rk3128_dts_timing[i], ++ p + i); ++ } ++end: ++ dts_timing = ++ (struct rk3128_ddr_dts_config_timing *)(timing + ++ DTS_PAR_OFFSET / 4); ++ if (!ret) { ++ dts_timing->available = 1; ++ } else { ++ dts_timing->available = 0; ++ dev_err(dev, "of_get_ddr_timings: fail\n"); ++ } ++ ++ of_node_put(np_tim); ++} ++ ++static uint32_t of_get_rk3228_timings(struct device *dev, ++ struct device_node *np, uint32_t *timing) ++{ ++ struct device_node *np_tim; ++ u32 *p; ++ int ret = 0; ++ u32 i; ++ ++ p = timing + DTS_PAR_OFFSET / 4; ++ np_tim = of_parse_phandle(np, "rockchip,dram_timing", 0); ++ if (!np_tim) { ++ ret = -EINVAL; ++ goto end; ++ } ++ for (i = 0; i < ARRAY_SIZE(rk3228_dts_timing); i++) { ++ ret |= of_property_read_u32(np_tim, rk3228_dts_timing[i], ++ p + i); ++ } ++end: ++ if (ret) ++ dev_err(dev, "of_get_ddr_timings: fail\n"); ++ ++ of_node_put(np_tim); ++ return ret; ++} ++ ++static void of_get_rk3288_timings(struct device *dev, ++ struct device_node *np, uint32_t *timing) ++{ ++ struct device_node *np_tim; ++ u32 *p; ++ struct rk3288_ddr_dts_config_timing *dts_timing; ++ struct share_params *init_timing; ++ int ret = 0; ++ u32 i; ++ ++ init_timing = (struct share_params *)timing; ++ ++ if (of_property_read_u32(np, "vop-dclk-mode", ++ &init_timing->vop_dclk_mode)) ++ init_timing->vop_dclk_mode = 0; ++ ++ p = timing + DTS_PAR_OFFSET / 4; ++ np_tim = of_parse_phandle(np, "rockchip,ddr_timing", 0); ++ if (!np_tim) { ++ ret = -EINVAL; ++ goto end; ++ } ++ for (i = 0; i < ARRAY_SIZE(rk3288_dts_timing); i++) { ++ ret |= of_property_read_u32(np_tim, rk3288_dts_timing[i], ++ p + i); ++ } ++end: ++ dts_timing = ++ (struct rk3288_ddr_dts_config_timing *)(timing + ++ DTS_PAR_OFFSET / 4); ++ if (!ret) { ++ dts_timing->available = 1; ++ } else { ++ dts_timing->available = 0; ++ dev_err(dev, "of_get_ddr_timings: fail\n"); ++ } ++ ++ of_node_put(np_tim); ++} ++ ++static void of_get_rk3328_timings(struct device *dev, ++ struct device_node *np, uint32_t *timing) ++{ ++ struct device_node *np_tim; ++ u32 *p; ++ struct rk3328_ddr_dts_config_timing *dts_timing; ++ struct rk3328_ddr_de_skew_setting *de_skew; ++ int ret = 0; ++ u32 i; ++ ++ dts_timing = ++ (struct rk3328_ddr_dts_config_timing *)(timing + ++ DTS_PAR_OFFSET / 4); ++ ++ np_tim = of_parse_phandle(np, "ddr_timing", 0); ++ if (!np_tim) { ++ ret = -EINVAL; ++ goto end; ++ } ++ de_skew = kmalloc(sizeof(*de_skew), GFP_KERNEL); ++ if (!de_skew) { ++ ret = -ENOMEM; ++ goto end; ++ } ++ p = (u32 *)dts_timing; ++ for (i = 0; i < ARRAY_SIZE(rk3328_dts_timing); i++) { ++ ret |= of_property_read_u32(np_tim, rk3328_dts_timing[i], ++ p + i); ++ } ++ p = (u32 *)de_skew->ca_de_skew; ++ for (i = 0; i < ARRAY_SIZE(rk3328_dts_ca_timing); i++) { ++ ret |= of_property_read_u32(np_tim, rk3328_dts_ca_timing[i], ++ p + i); ++ } ++ p = (u32 *)de_skew->cs0_de_skew; ++ for (i = 0; i < ARRAY_SIZE(rk3328_dts_cs0_timing); i++) { ++ ret |= of_property_read_u32(np_tim, rk3328_dts_cs0_timing[i], ++ p + i); ++ } ++ p = (u32 *)de_skew->cs1_de_skew; ++ for (i = 0; i < ARRAY_SIZE(rk3328_dts_cs1_timing); i++) { ++ ret |= of_property_read_u32(np_tim, rk3328_dts_cs1_timing[i], ++ p + i); ++ } ++ if (!ret) ++ rk3328_de_skew_setting_2_register(de_skew, dts_timing); ++ kfree(de_skew); ++end: ++ if (!ret) { ++ dts_timing->available = 1; ++ } else { ++ dts_timing->available = 0; ++ dev_err(dev, "of_get_ddr_timings: fail\n"); ++ } ++ ++ of_node_put(np_tim); ++} ++ ++static void of_get_rk3568_timings(struct device *dev, ++ struct device_node *np, uint32_t *timing) ++{ ++ struct device_node *np_tim; ++ u32 *p; ++ struct rk3568_ddr_dts_config_timing *dts_timing; ++ int ret = 0; ++ u32 i; ++ ++ dts_timing = ++ (struct rk3568_ddr_dts_config_timing *)(timing + ++ DTS_PAR_OFFSET / 4); ++ ++ np_tim = of_parse_phandle(np, "ddr_timing", 0); ++ if (!np_tim) { ++ ret = -EINVAL; ++ goto end; ++ } ++ ++ p = (u32 *)dts_timing; ++ for (i = 0; i < ARRAY_SIZE(px30_dts_timing); i++) { ++ ret |= of_property_read_u32(np_tim, px30_dts_timing[i], ++ p + i); ++ } ++ ++end: ++ if (!ret) { ++ dts_timing->available = 1; ++ } else { ++ dts_timing->available = 0; ++ dev_err(dev, "of_get_ddr_timings: fail\n"); ++ } ++ ++ of_node_put(np_tim); ++} ++ ++static void of_get_rv1126_timings(struct device *dev, ++ struct device_node *np, uint32_t *timing) ++{ ++ struct device_node *np_tim; ++ u32 *p; ++ struct rk1808_ddr_dts_config_timing *dts_timing; ++ int ret = 0; ++ u32 i; ++ ++ dts_timing = ++ (struct rk1808_ddr_dts_config_timing *)(timing + ++ DTS_PAR_OFFSET / 4); ++ ++ np_tim = of_parse_phandle(np, "ddr_timing", 0); ++ if (!np_tim) { ++ ret = -EINVAL; ++ goto end; ++ } ++ ++ p = (u32 *)dts_timing; ++ for (i = 0; i < ARRAY_SIZE(px30_dts_timing); i++) { ++ ret |= of_property_read_u32(np_tim, px30_dts_timing[i], ++ p + i); ++ } ++ p = (u32 *)dts_timing->ca_de_skew; ++ for (i = 0; i < ARRAY_SIZE(rv1126_dts_ca_timing); i++) { ++ ret |= of_property_read_u32(np_tim, rv1126_dts_ca_timing[i], ++ p + i); ++ } ++ p = (u32 *)dts_timing->cs0_a_de_skew; ++ for (i = 0; i < ARRAY_SIZE(rv1126_dts_cs0_a_timing); i++) { ++ ret |= of_property_read_u32(np_tim, rv1126_dts_cs0_a_timing[i], ++ p + i); ++ } ++ p = (u32 *)dts_timing->cs0_b_de_skew; ++ for (i = 0; i < ARRAY_SIZE(rv1126_dts_cs0_b_timing); i++) { ++ ret |= of_property_read_u32(np_tim, rv1126_dts_cs0_b_timing[i], ++ p + i); ++ } ++ p = (u32 *)dts_timing->cs1_a_de_skew; ++ for (i = 0; i < ARRAY_SIZE(rv1126_dts_cs1_a_timing); i++) { ++ ret |= of_property_read_u32(np_tim, rv1126_dts_cs1_a_timing[i], ++ p + i); ++ } ++ p = (u32 *)dts_timing->cs1_b_de_skew; ++ for (i = 0; i < ARRAY_SIZE(rv1126_dts_cs1_b_timing); i++) { ++ ret |= of_property_read_u32(np_tim, rv1126_dts_cs1_b_timing[i], ++ p + i); ++ } ++ ++end: ++ if (!ret) { ++ dts_timing->available = 1; ++ } else { ++ dts_timing->available = 0; ++ dev_err(dev, "of_get_ddr_timings: fail\n"); ++ } ++ ++ of_node_put(np_tim); ++} ++ ++static struct rk3368_dram_timing *of_get_rk3368_timings(struct device *dev, ++ struct device_node *np) ++{ ++ struct rk3368_dram_timing *timing = NULL; ++ struct device_node *np_tim; ++ int ret = 0; ++ ++ np_tim = of_parse_phandle(np, "ddr_timing", 0); ++ if (np_tim) { ++ timing = devm_kzalloc(dev, sizeof(*timing), GFP_KERNEL); ++ if (!timing) ++ goto err; ++ ++ ret |= of_property_read_u32(np_tim, "dram_spd_bin", ++ &timing->dram_spd_bin); ++ ret |= of_property_read_u32(np_tim, "sr_idle", ++ &timing->sr_idle); ++ ret |= of_property_read_u32(np_tim, "pd_idle", ++ &timing->pd_idle); ++ ret |= of_property_read_u32(np_tim, "dram_dll_disb_freq", ++ &timing->dram_dll_dis_freq); ++ ret |= of_property_read_u32(np_tim, "phy_dll_disb_freq", ++ &timing->phy_dll_dis_freq); ++ ret |= of_property_read_u32(np_tim, "dram_odt_disb_freq", ++ &timing->dram_odt_dis_freq); ++ ret |= of_property_read_u32(np_tim, "phy_odt_disb_freq", ++ &timing->phy_odt_dis_freq); ++ ret |= of_property_read_u32(np_tim, "ddr3_drv", ++ &timing->ddr3_drv); ++ ret |= of_property_read_u32(np_tim, "ddr3_odt", ++ &timing->ddr3_odt); ++ ret |= of_property_read_u32(np_tim, "lpddr3_drv", ++ &timing->lpddr3_drv); ++ ret |= of_property_read_u32(np_tim, "lpddr3_odt", ++ &timing->lpddr3_odt); ++ ret |= of_property_read_u32(np_tim, "lpddr2_drv", ++ &timing->lpddr2_drv); ++ ret |= of_property_read_u32(np_tim, "phy_clk_drv", ++ &timing->phy_clk_drv); ++ ret |= of_property_read_u32(np_tim, "phy_cmd_drv", ++ &timing->phy_cmd_drv); ++ ret |= of_property_read_u32(np_tim, "phy_dqs_drv", ++ &timing->phy_dqs_drv); ++ ret |= of_property_read_u32(np_tim, "phy_odt", ++ &timing->phy_odt); ++ ret |= of_property_read_u32(np_tim, "ddr_2t", ++ &timing->ddr_2t); ++ if (ret) { ++ devm_kfree(dev, timing); ++ goto err; ++ } ++ of_node_put(np_tim); ++ return timing; ++ } ++ ++err: ++ if (timing) { ++ devm_kfree(dev, timing); ++ timing = NULL; ++ } ++ of_node_put(np_tim); ++ return timing; ++} ++ ++static struct rk3399_dram_timing *of_get_rk3399_timings(struct device *dev, ++ struct device_node *np) ++{ ++ struct rk3399_dram_timing *timing = NULL; ++ struct device_node *np_tim; ++ int ret; ++ ++ np_tim = of_parse_phandle(np, "ddr_timing", 0); ++ if (np_tim) { ++ timing = devm_kzalloc(dev, sizeof(*timing), GFP_KERNEL); ++ if (!timing) ++ goto err; ++ ++ ret = of_property_read_u32(np_tim, "ddr3_speed_bin", ++ &timing->ddr3_speed_bin); ++ ret |= of_property_read_u32(np_tim, "pd_idle", ++ &timing->pd_idle); ++ ret |= of_property_read_u32(np_tim, "sr_idle", ++ &timing->sr_idle); ++ ret |= of_property_read_u32(np_tim, "sr_mc_gate_idle", ++ &timing->sr_mc_gate_idle); ++ ret |= of_property_read_u32(np_tim, "srpd_lite_idle", ++ &timing->srpd_lite_idle); ++ ret |= of_property_read_u32(np_tim, "standby_idle", ++ &timing->standby_idle); ++ ret |= of_property_read_u32(np_tim, "auto_lp_dis_freq", ++ &timing->auto_lp_dis_freq); ++ ret |= of_property_read_u32(np_tim, "ddr3_dll_dis_freq", ++ &timing->ddr3_dll_dis_freq); ++ ret |= of_property_read_u32(np_tim, "phy_dll_dis_freq", ++ &timing->phy_dll_dis_freq); ++ ret |= of_property_read_u32(np_tim, "ddr3_odt_dis_freq", ++ &timing->ddr3_odt_dis_freq); ++ ret |= of_property_read_u32(np_tim, "ddr3_drv", ++ &timing->ddr3_drv); ++ ret |= of_property_read_u32(np_tim, "ddr3_odt", ++ &timing->ddr3_odt); ++ ret |= of_property_read_u32(np_tim, "phy_ddr3_ca_drv", ++ &timing->phy_ddr3_ca_drv); ++ ret |= of_property_read_u32(np_tim, "phy_ddr3_dq_drv", ++ &timing->phy_ddr3_dq_drv); ++ ret |= of_property_read_u32(np_tim, "phy_ddr3_odt", ++ &timing->phy_ddr3_odt); ++ ret |= of_property_read_u32(np_tim, "lpddr3_odt_dis_freq", ++ &timing->lpddr3_odt_dis_freq); ++ ret |= of_property_read_u32(np_tim, "lpddr3_drv", ++ &timing->lpddr3_drv); ++ ret |= of_property_read_u32(np_tim, "lpddr3_odt", ++ &timing->lpddr3_odt); ++ ret |= of_property_read_u32(np_tim, "phy_lpddr3_ca_drv", ++ &timing->phy_lpddr3_ca_drv); ++ ret |= of_property_read_u32(np_tim, "phy_lpddr3_dq_drv", ++ &timing->phy_lpddr3_dq_drv); ++ ret |= of_property_read_u32(np_tim, "phy_lpddr3_odt", ++ &timing->phy_lpddr3_odt); ++ ret |= of_property_read_u32(np_tim, "lpddr4_odt_dis_freq", ++ &timing->lpddr4_odt_dis_freq); ++ ret |= of_property_read_u32(np_tim, "lpddr4_drv", ++ &timing->lpddr4_drv); ++ ret |= of_property_read_u32(np_tim, "lpddr4_dq_odt", ++ &timing->lpddr4_dq_odt); ++ ret |= of_property_read_u32(np_tim, "lpddr4_ca_odt", ++ &timing->lpddr4_ca_odt); ++ ret |= of_property_read_u32(np_tim, "phy_lpddr4_ca_drv", ++ &timing->phy_lpddr4_ca_drv); ++ ret |= of_property_read_u32(np_tim, "phy_lpddr4_ck_cs_drv", ++ &timing->phy_lpddr4_ck_cs_drv); ++ ret |= of_property_read_u32(np_tim, "phy_lpddr4_dq_drv", ++ &timing->phy_lpddr4_dq_drv); ++ ret |= of_property_read_u32(np_tim, "phy_lpddr4_odt", ++ &timing->phy_lpddr4_odt); ++ if (ret) { ++ devm_kfree(dev, timing); ++ goto err; ++ } ++ of_node_put(np_tim); ++ return timing; ++ } ++ ++err: ++ if (timing) { ++ devm_kfree(dev, timing); ++ timing = NULL; ++ } ++ of_node_put(np_tim); ++ return timing; ++} ++ ++static int rockchip_ddr_set_auto_self_refresh(uint32_t en) ++{ ++ struct arm_smccc_res res; ++ ++ ddr_psci_param->sr_idle_en = en; ++ res = sip_smc_dram(SHARE_PAGE_TYPE_DDR, 0, ++ ROCKCHIP_SIP_CONFIG_DRAM_SET_AT_SR); ++ ++ return res.a0; ++} ++ ++struct dmcfreq_wait_ctrl_t { ++ wait_queue_head_t wait_wq; ++ int complt_irq; ++ int wait_flag; ++ int wait_en; ++ int wait_time_out_ms; ++ int dcf_en; ++ struct regmap *regmap_dcf; ++}; ++ ++static struct dmcfreq_wait_ctrl_t wait_ctrl; ++ ++static irqreturn_t wait_complete_irq(int irqno, void *dev_id) ++{ ++ struct dmcfreq_wait_ctrl_t *ctrl = dev_id; ++ ++ ctrl->wait_flag = 0; ++ wake_up(&ctrl->wait_wq); ++ return IRQ_HANDLED; ++} ++ ++static irqreturn_t wait_dcf_complete_irq(int irqno, void *dev_id) ++{ ++ struct arm_smccc_res res; ++ struct dmcfreq_wait_ctrl_t *ctrl = dev_id; ++ ++ res = sip_smc_dram(SHARE_PAGE_TYPE_DDR, 0, ++ ROCKCHIP_SIP_CONFIG_DRAM_POST_SET_RATE); ++ if (res.a0) ++ pr_err("%s: dram post set rate error:%lx\n", __func__, res.a0); ++ ++ ctrl->wait_flag = 0; ++ wake_up(&ctrl->wait_wq); ++ return IRQ_HANDLED; ++} ++ ++int rockchip_dmcfreq_wait_complete(void) ++{ ++ struct arm_smccc_res res; ++ ++ if (!wait_ctrl.wait_en) { ++ pr_err("%s: Do not support time out!\n", __func__); ++ return 0; ++ } ++ wait_ctrl.wait_flag = -1; ++ ++ enable_irq(wait_ctrl.complt_irq); ++ /* ++ * CPUs only enter WFI when idle to make sure that ++ * FIQn can quick response. ++ */ ++ cpu_latency_qos_update_request(&pm_qos, 0); ++ ++ if (wait_ctrl.dcf_en == 1) { ++ /* start dcf */ ++ regmap_update_bits(wait_ctrl.regmap_dcf, 0x0, 0x1, 0x1); ++ } else if (wait_ctrl.dcf_en == 2) { ++ res = sip_smc_dram(0, 0, ROCKCHIP_SIP_CONFIG_MCU_START); ++ if (res.a0) { ++ pr_err("rockchip_sip_config_mcu_start error:%lx\n", res.a0); ++ return -ENOMEM; ++ } ++ } ++ ++ wait_event_timeout(wait_ctrl.wait_wq, (wait_ctrl.wait_flag == 0), ++ msecs_to_jiffies(wait_ctrl.wait_time_out_ms)); ++ ++ cpu_latency_qos_update_request(&pm_qos, PM_QOS_DEFAULT_VALUE); ++ disable_irq(wait_ctrl.complt_irq); ++ ++ return 0; ++} ++ ++static __maybe_unused int rockchip_get_freq_info(struct rockchip_dmcfreq *dmcfreq) ++{ ++ struct arm_smccc_res res; ++ struct dev_pm_opp *opp; ++ struct dmc_freq_table *freq_table; ++ unsigned long rate; ++ int i, j, count, ret = 0; ++ ++ res = sip_smc_dram(SHARE_PAGE_TYPE_DDR, 0, ++ ROCKCHIP_SIP_CONFIG_DRAM_GET_FREQ_INFO); ++ if (res.a0) { ++ dev_err(dmcfreq->dev, "rockchip_sip_config_dram_get_freq_info error:%lx\n", ++ res.a0); ++ return -ENOMEM; ++ } ++ ++ if (ddr_psci_param->freq_count == 0 || ddr_psci_param->freq_count > 6) { ++ dev_err(dmcfreq->dev, "it is no available frequencies!\n"); ++ return -EPERM; ++ } ++ ++ for (i = 0; i < ddr_psci_param->freq_count; i++) ++ dmcfreq->freq_info_rate[i] = ddr_psci_param->freq_info_mhz[i] * 1000000; ++ dmcfreq->freq_count = ddr_psci_param->freq_count; ++ ++ /* update dmc_opp_table */ ++ count = dev_pm_opp_get_opp_count(dmcfreq->dev); ++ if (count <= 0) { ++ ret = count ? count : -ENODATA; ++ return ret; ++ } ++ ++ freq_table = kmalloc(sizeof(struct dmc_freq_table) * count, GFP_KERNEL); ++ for (i = 0, rate = 0; i < count; i++, rate++) { ++ /* find next rate */ ++ opp = dev_pm_opp_find_freq_ceil(dmcfreq->dev, &rate); ++ if (IS_ERR(opp)) { ++ ret = PTR_ERR(opp); ++ dev_err(dmcfreq->dev, "failed to find OPP for freq %lu.\n", rate); ++ goto out; ++ } ++ freq_table[i].freq = rate; ++ freq_table[i].volt = dev_pm_opp_get_voltage(opp); ++ dev_pm_opp_put(opp); ++ ++ for (j = 0; j < dmcfreq->freq_count; j++) { ++ if (rate == dmcfreq->freq_info_rate[j]) ++ break; ++ } ++ if (j == dmcfreq->freq_count) ++ dev_pm_opp_remove(dmcfreq->dev, rate); ++ } ++ ++ for (i = 0; i < dmcfreq->freq_count; i++) { ++ for (j = 0; j < count; j++) { ++ if (dmcfreq->freq_info_rate[i] == freq_table[j].freq) { ++ break; ++ } else if (dmcfreq->freq_info_rate[i] < freq_table[j].freq) { ++ dev_pm_opp_add(dmcfreq->dev, dmcfreq->freq_info_rate[i], ++ freq_table[j].volt); ++ break; ++ } ++ } ++ if (j == count) { ++ dev_err(dmcfreq->dev, "failed to match dmc_opp_table for %ld\n", ++ dmcfreq->freq_info_rate[i]); ++ if (i == 0) ++ ret = -EPERM; ++ else ++ dmcfreq->freq_count = i; ++ goto out; ++ } ++ } ++ ++out: ++ kfree(freq_table); ++ return ret; ++} ++ ++static __maybe_unused int px30_dmc_init(struct platform_device *pdev, ++ struct rockchip_dmcfreq *dmcfreq) ++{ ++ struct arm_smccc_res res; ++ u32 size; ++ int ret; ++ int complt_irq; ++ u32 complt_hwirq; ++ struct irq_data *complt_irq_data; ++ ++ res = sip_smc_dram(0, 0, ++ ROCKCHIP_SIP_CONFIG_DRAM_GET_VERSION); ++ dev_notice(&pdev->dev, "current ATF version 0x%lx!\n", res.a1); ++ if (res.a0 || res.a1 < 0x103) { ++ dev_err(&pdev->dev, ++ "trusted firmware need to update or is invalid!\n"); ++ return -ENXIO; ++ } ++ ++ dev_notice(&pdev->dev, "read tf version 0x%lx!\n", res.a1); ++ ++ /* ++ * first 4KB is used for interface parameters ++ * after 4KB * N is dts parameters ++ */ ++ size = sizeof(struct px30_ddr_dts_config_timing); ++ res = sip_smc_request_share_mem(DIV_ROUND_UP(size, 4096) + 1, ++ SHARE_PAGE_TYPE_DDR); ++ if (res.a0 != 0) { ++ dev_err(&pdev->dev, "no ATF memory for init\n"); ++ return -ENOMEM; ++ } ++ ddr_psci_param = (struct share_params *)res.a1; ++ of_get_px30_timings(&pdev->dev, pdev->dev.of_node, ++ (uint32_t *)ddr_psci_param); ++ ++ init_waitqueue_head(&wait_ctrl.wait_wq); ++ wait_ctrl.wait_en = 1; ++ wait_ctrl.wait_time_out_ms = 17 * 5; ++ ++ complt_irq = platform_get_irq_byname(pdev, "complete_irq"); ++ if (complt_irq < 0) { ++ dev_err(&pdev->dev, "no IRQ for complete_irq: %d\n", ++ complt_irq); ++ return complt_irq; ++ } ++ wait_ctrl.complt_irq = complt_irq; ++ ++ ret = devm_request_irq(&pdev->dev, complt_irq, wait_complete_irq, ++ 0, dev_name(&pdev->dev), &wait_ctrl); ++ if (ret < 0) { ++ dev_err(&pdev->dev, "cannot request complete_irq\n"); ++ return ret; ++ } ++ disable_irq(complt_irq); ++ ++ complt_irq_data = irq_get_irq_data(complt_irq); ++ complt_hwirq = irqd_to_hwirq(complt_irq_data); ++ ddr_psci_param->complt_hwirq = complt_hwirq; ++ ++ dmcfreq->set_rate_params = ddr_psci_param; ++ rockchip_set_ddrclk_params(dmcfreq->set_rate_params); ++ rockchip_set_ddrclk_dmcfreq_wait_complete(rockchip_dmcfreq_wait_complete); ++ ++ res = sip_smc_dram(SHARE_PAGE_TYPE_DDR, 0, ++ ROCKCHIP_SIP_CONFIG_DRAM_INIT); ++ if (res.a0) { ++ dev_err(&pdev->dev, "rockchip_sip_config_dram_init error:%lx\n", ++ res.a0); ++ return -ENOMEM; ++ } ++ ++ dmcfreq->set_auto_self_refresh = rockchip_ddr_set_auto_self_refresh; ++ ++ return 0; ++} ++ ++static __maybe_unused int rk1808_dmc_init(struct platform_device *pdev, ++ struct rockchip_dmcfreq *dmcfreq) ++{ ++ struct arm_smccc_res res; ++ u32 size; ++ int ret; ++ int complt_irq; ++ struct device_node *node; ++ ++ res = sip_smc_dram(0, 0, ++ ROCKCHIP_SIP_CONFIG_DRAM_GET_VERSION); ++ dev_notice(&pdev->dev, "current ATF version 0x%lx!\n", res.a1); ++ if (res.a0 || res.a1 < 0x101) { ++ dev_err(&pdev->dev, ++ "trusted firmware need to update or is invalid!\n"); ++ return -ENXIO; ++ } ++ ++ /* ++ * first 4KB is used for interface parameters ++ * after 4KB * N is dts parameters ++ */ ++ size = sizeof(struct rk1808_ddr_dts_config_timing); ++ res = sip_smc_request_share_mem(DIV_ROUND_UP(size, 4096) + 1, ++ SHARE_PAGE_TYPE_DDR); ++ if (res.a0 != 0) { ++ dev_err(&pdev->dev, "no ATF memory for init\n"); ++ return -ENOMEM; ++ } ++ ddr_psci_param = (struct share_params *)res.a1; ++ of_get_rk1808_timings(&pdev->dev, pdev->dev.of_node, ++ (uint32_t *)ddr_psci_param); ++ ++ /* enable start dcf in kernel after dcf ready */ ++ node = of_parse_phandle(pdev->dev.of_node, "dcf_reg", 0); ++ wait_ctrl.regmap_dcf = syscon_node_to_regmap(node); ++ if (IS_ERR(wait_ctrl.regmap_dcf)) ++ return PTR_ERR(wait_ctrl.regmap_dcf); ++ wait_ctrl.dcf_en = 1; ++ ++ init_waitqueue_head(&wait_ctrl.wait_wq); ++ wait_ctrl.wait_en = 1; ++ wait_ctrl.wait_time_out_ms = 17 * 5; ++ ++ complt_irq = platform_get_irq_byname(pdev, "complete_irq"); ++ if (complt_irq < 0) { ++ dev_err(&pdev->dev, "no IRQ for complete_irq: %d\n", ++ complt_irq); ++ return complt_irq; ++ } ++ wait_ctrl.complt_irq = complt_irq; ++ ++ ret = devm_request_irq(&pdev->dev, complt_irq, wait_dcf_complete_irq, ++ 0, dev_name(&pdev->dev), &wait_ctrl); ++ if (ret < 0) { ++ dev_err(&pdev->dev, "cannot request complete_irq\n"); ++ return ret; ++ } ++ disable_irq(complt_irq); ++ ++ dmcfreq->set_rate_params = ddr_psci_param; ++ rockchip_set_ddrclk_params(dmcfreq->set_rate_params); ++ rockchip_set_ddrclk_dmcfreq_wait_complete(rockchip_dmcfreq_wait_complete); ++ ++ res = sip_smc_dram(SHARE_PAGE_TYPE_DDR, 0, ++ ROCKCHIP_SIP_CONFIG_DRAM_INIT); ++ if (res.a0) { ++ dev_err(&pdev->dev, "rockchip_sip_config_dram_init error:%lx\n", ++ res.a0); ++ return -ENOMEM; ++ } ++ ++ dmcfreq->set_auto_self_refresh = rockchip_ddr_set_auto_self_refresh; ++ ++ return 0; ++} ++ ++static __maybe_unused int rk3128_dmc_init(struct platform_device *pdev, ++ struct rockchip_dmcfreq *dmcfreq) ++{ ++ struct arm_smccc_res res; ++ ++ res = sip_smc_request_share_mem(DIV_ROUND_UP(sizeof( ++ struct rk3128_ddr_dts_config_timing), ++ 4096) + 1, SHARE_PAGE_TYPE_DDR); ++ if (res.a0) { ++ dev_err(&pdev->dev, "no ATF memory for init\n"); ++ return -ENOMEM; ++ } ++ ddr_psci_param = (struct share_params *)res.a1; ++ of_get_rk3128_timings(&pdev->dev, pdev->dev.of_node, ++ (uint32_t *)ddr_psci_param); ++ ++ ddr_psci_param->hz = 0; ++ ddr_psci_param->lcdc_type = rk_drm_get_lcdc_type(); ++ ++ dmcfreq->set_rate_params = ddr_psci_param; ++ rockchip_set_ddrclk_params(dmcfreq->set_rate_params); ++ ++ res = sip_smc_dram(SHARE_PAGE_TYPE_DDR, 0, ++ ROCKCHIP_SIP_CONFIG_DRAM_INIT); ++ ++ if (res.a0) { ++ dev_err(&pdev->dev, "rockchip_sip_config_dram_init error:%lx\n", ++ res.a0); ++ return -ENOMEM; ++ } ++ ++ dmcfreq->set_auto_self_refresh = rockchip_ddr_set_auto_self_refresh; ++ ++ return 0; ++} ++ ++static __maybe_unused int rk3228_dmc_init(struct platform_device *pdev, ++ struct rockchip_dmcfreq *dmcfreq) ++{ ++ struct arm_smccc_res res; ++ ++ res = sip_smc_request_share_mem(DIV_ROUND_UP(sizeof( ++ struct rk3228_ddr_dts_config_timing), ++ 4096) + 1, SHARE_PAGE_TYPE_DDR); ++ if (res.a0) { ++ dev_err(&pdev->dev, "no ATF memory for init\n"); ++ return -ENOMEM; ++ } ++ ++ ddr_psci_param = (struct share_params *)res.a1; ++ if (of_get_rk3228_timings(&pdev->dev, pdev->dev.of_node, ++ (uint32_t *)ddr_psci_param)) ++ return -ENOMEM; ++ ++ ddr_psci_param->hz = 0; ++ ++ dmcfreq->set_rate_params = ddr_psci_param; ++ rockchip_set_ddrclk_params(dmcfreq->set_rate_params); ++ ++ res = sip_smc_dram(SHARE_PAGE_TYPE_DDR, 0, ++ ROCKCHIP_SIP_CONFIG_DRAM_INIT); ++ ++ if (res.a0) { ++ dev_err(&pdev->dev, "rockchip_sip_config_dram_init error:%lx\n", ++ res.a0); ++ return -ENOMEM; ++ } ++ ++ dmcfreq->set_auto_self_refresh = rockchip_ddr_set_auto_self_refresh; ++ ++ return 0; ++} ++ ++static __maybe_unused int rk3288_dmc_init(struct platform_device *pdev, ++ struct rockchip_dmcfreq *dmcfreq) ++{ ++ struct device *dev = &pdev->dev; ++ struct clk *pclk_phy, *pclk_upctl, *dmc_clk; ++ struct arm_smccc_res res; ++ int ret; ++ ++ dmc_clk = devm_clk_get(dev, "dmc_clk"); ++ if (IS_ERR(dmc_clk)) { ++ dev_err(dev, "Cannot get the clk dmc_clk\n"); ++ return PTR_ERR(dmc_clk); ++ } ++ ret = clk_prepare_enable(dmc_clk); ++ if (ret < 0) { ++ dev_err(dev, "failed to prepare/enable dmc_clk\n"); ++ return ret; ++ } ++ ++ pclk_phy = devm_clk_get(dev, "pclk_phy0"); ++ if (IS_ERR(pclk_phy)) { ++ dev_err(dev, "Cannot get the clk pclk_phy0\n"); ++ return PTR_ERR(pclk_phy); ++ } ++ ret = clk_prepare_enable(pclk_phy); ++ if (ret < 0) { ++ dev_err(dev, "failed to prepare/enable pclk_phy0\n"); ++ return ret; ++ } ++ pclk_upctl = devm_clk_get(dev, "pclk_upctl0"); ++ if (IS_ERR(pclk_upctl)) { ++ dev_err(dev, "Cannot get the clk pclk_upctl0\n"); ++ return PTR_ERR(pclk_upctl); ++ } ++ ret = clk_prepare_enable(pclk_upctl); ++ if (ret < 0) { ++ dev_err(dev, "failed to prepare/enable pclk_upctl1\n"); ++ return ret; ++ } ++ ++ pclk_phy = devm_clk_get(dev, "pclk_phy1"); ++ if (IS_ERR(pclk_phy)) { ++ dev_err(dev, "Cannot get the clk pclk_phy1\n"); ++ return PTR_ERR(pclk_phy); ++ } ++ ret = clk_prepare_enable(pclk_phy); ++ if (ret < 0) { ++ dev_err(dev, "failed to prepare/enable pclk_phy1\n"); ++ return ret; ++ } ++ pclk_upctl = devm_clk_get(dev, "pclk_upctl1"); ++ if (IS_ERR(pclk_upctl)) { ++ dev_err(dev, "Cannot get the clk pclk_upctl1\n"); ++ return PTR_ERR(pclk_upctl); ++ } ++ ret = clk_prepare_enable(pclk_upctl); ++ if (ret < 0) { ++ dev_err(dev, "failed to prepare/enable pclk_upctl1\n"); ++ return ret; ++ } ++ ++ res = sip_smc_request_share_mem(DIV_ROUND_UP(sizeof( ++ struct rk3288_ddr_dts_config_timing), ++ 4096) + 1, SHARE_PAGE_TYPE_DDR); ++ if (res.a0) { ++ dev_err(&pdev->dev, "no ATF memory for init\n"); ++ return -ENOMEM; ++ } ++ ++ ddr_psci_param = (struct share_params *)res.a1; ++ of_get_rk3288_timings(&pdev->dev, pdev->dev.of_node, ++ (uint32_t *)ddr_psci_param); ++ ++ ddr_psci_param->hz = 0; ++ ddr_psci_param->lcdc_type = rk_drm_get_lcdc_type(); ++ ++ dmcfreq->set_rate_params = ddr_psci_param; ++ rockchip_set_ddrclk_params(dmcfreq->set_rate_params); ++ ++ res = sip_smc_dram(SHARE_PAGE_TYPE_DDR, 0, ++ ROCKCHIP_SIP_CONFIG_DRAM_INIT); ++ ++ if (res.a0) { ++ dev_err(&pdev->dev, "rockchip_sip_config_dram_init error:%lx\n", ++ res.a0); ++ return -ENOMEM; ++ } ++ ++ dmcfreq->set_auto_self_refresh = rockchip_ddr_set_auto_self_refresh; ++ ++ return 0; ++} ++ ++static __maybe_unused int rk3328_dmc_init(struct platform_device *pdev, ++ struct rockchip_dmcfreq *dmcfreq) ++{ ++ struct arm_smccc_res res; ++ u32 size; ++ ++ res = sip_smc_dram(0, 0, ++ ROCKCHIP_SIP_CONFIG_DRAM_GET_VERSION); ++ dev_notice(&pdev->dev, "current ATF version 0x%lx!\n", res.a1); ++ if (res.a0 || (res.a1 < 0x101)) { ++ dev_err(&pdev->dev, ++ "trusted firmware need to update or is invalid!\n"); ++ return -ENXIO; ++ } ++ ++ dev_notice(&pdev->dev, "read tf version 0x%lx!\n", res.a1); ++ ++ /* ++ * first 4KB is used for interface parameters ++ * after 4KB * N is dts parameters ++ */ ++ size = sizeof(struct rk3328_ddr_dts_config_timing); ++ res = sip_smc_request_share_mem(DIV_ROUND_UP(size, 4096) + 1, ++ SHARE_PAGE_TYPE_DDR); ++ if (res.a0 != 0) { ++ dev_err(&pdev->dev, "no ATF memory for init\n"); ++ return -ENOMEM; ++ } ++ ddr_psci_param = (struct share_params *)res.a1; ++ of_get_rk3328_timings(&pdev->dev, pdev->dev.of_node, ++ (uint32_t *)ddr_psci_param); ++ ++ dmcfreq->set_rate_params = ddr_psci_param; ++ rockchip_set_ddrclk_params(dmcfreq->set_rate_params); ++ ++ res = sip_smc_dram(SHARE_PAGE_TYPE_DDR, 0, ++ ROCKCHIP_SIP_CONFIG_DRAM_INIT); ++ if (res.a0) { ++ dev_err(&pdev->dev, "rockchip_sip_config_dram_init error:%lx\n", ++ res.a0); ++ return -ENOMEM; ++ } ++ ++ dmcfreq->set_auto_self_refresh = rockchip_ddr_set_auto_self_refresh; ++ ++ return 0; ++} ++ ++static __maybe_unused int rk3368_dmc_init(struct platform_device *pdev, ++ struct rockchip_dmcfreq *dmcfreq) ++{ ++ struct device *dev = &pdev->dev; ++ struct device_node *np = pdev->dev.of_node; ++ struct arm_smccc_res res; ++ struct rk3368_dram_timing *dram_timing; ++ struct clk *pclk_phy, *pclk_upctl; ++ int ret; ++ u32 dram_spd_bin; ++ u32 addr_mcu_el3; ++ u32 dclk_mode; ++ u32 lcdc_type; ++ ++ pclk_phy = devm_clk_get(dev, "pclk_phy"); ++ if (IS_ERR(pclk_phy)) { ++ dev_err(dev, "Cannot get the clk pclk_phy\n"); ++ return PTR_ERR(pclk_phy); ++ } ++ ret = clk_prepare_enable(pclk_phy); ++ if (ret < 0) { ++ dev_err(dev, "failed to prepare/enable pclk_phy\n"); ++ return ret; ++ } ++ pclk_upctl = devm_clk_get(dev, "pclk_upctl"); ++ if (IS_ERR(pclk_upctl)) { ++ dev_err(dev, "Cannot get the clk pclk_upctl\n"); ++ return PTR_ERR(pclk_upctl); ++ } ++ ret = clk_prepare_enable(pclk_upctl); ++ if (ret < 0) { ++ dev_err(dev, "failed to prepare/enable pclk_upctl\n"); ++ return ret; ++ } ++ ++ /* ++ * Get dram timing and pass it to arm trust firmware, ++ * the dram drvier in arm trust firmware will get these ++ * timing and to do dram initial. ++ */ ++ dram_timing = of_get_rk3368_timings(dev, np); ++ if (dram_timing) { ++ dram_spd_bin = dram_timing->dram_spd_bin; ++ if (scpi_ddr_send_timing((u32 *)dram_timing, ++ sizeof(struct rk3368_dram_timing))) ++ dev_err(dev, "send ddr timing timeout\n"); ++ } else { ++ dev_err(dev, "get ddr timing from dts error\n"); ++ dram_spd_bin = DDR3_DEFAULT; ++ } ++ ++ res = sip_smc_mcu_el3fiq(FIQ_INIT_HANDLER, ++ FIQ_NUM_FOR_DCF, ++ FIQ_CPU_TGT_BOOT); ++ if ((res.a0) || (res.a1 == 0) || (res.a1 > 0x80000)) ++ dev_err(dev, "Trust version error, pls check trust version\n"); ++ addr_mcu_el3 = res.a1; ++ ++ if (of_property_read_u32(np, "vop-dclk-mode", &dclk_mode) == 0) ++ scpi_ddr_dclk_mode(dclk_mode); ++ ++ dmcfreq->set_rate_params = ++ devm_kzalloc(dev, sizeof(struct share_params), GFP_KERNEL); ++ if (!dmcfreq->set_rate_params) ++ return -ENOMEM; ++ rockchip_set_ddrclk_params(dmcfreq->set_rate_params); ++ ++ lcdc_type = rk_drm_get_lcdc_type(); ++ ++ if (scpi_ddr_init(dram_spd_bin, 0, lcdc_type, ++ addr_mcu_el3)) ++ dev_err(dev, "ddr init error\n"); ++ else ++ dev_dbg(dev, ("%s out\n"), __func__); ++ ++ dmcfreq->set_auto_self_refresh = scpi_ddr_set_auto_self_refresh; ++ ++ return 0; ++} ++ ++static int rk3399_set_msch_readlatency(unsigned int readlatency) ++{ ++ struct arm_smccc_res res; ++ ++ arm_smccc_smc(ROCKCHIP_SIP_DRAM_FREQ, readlatency, 0, ++ ROCKCHIP_SIP_CONFIG_DRAM_SET_MSCH_RL, ++ 0, 0, 0, 0, &res); ++ ++ return res.a0; ++} ++ ++static __maybe_unused int rk3399_dmc_init(struct platform_device *pdev, ++ struct rockchip_dmcfreq *dmcfreq) ++{ ++ struct device *dev = &pdev->dev; ++ struct device_node *np = pdev->dev.of_node; ++ struct arm_smccc_res res; ++ struct rk3399_dram_timing *dram_timing; ++ int index, size; ++ u32 *timing; ++ ++ /* ++ * Get dram timing and pass it to arm trust firmware, ++ * the dram drvier in arm trust firmware will get these ++ * timing and to do dram initial. ++ */ ++ dram_timing = of_get_rk3399_timings(dev, np); ++ if (dram_timing) { ++ timing = (u32 *)dram_timing; ++ size = sizeof(struct rk3399_dram_timing) / 4; ++ for (index = 0; index < size; index++) { ++ arm_smccc_smc(ROCKCHIP_SIP_DRAM_FREQ, *timing++, index, ++ ROCKCHIP_SIP_CONFIG_DRAM_SET_PARAM, ++ 0, 0, 0, 0, &res); ++ if (res.a0) { ++ dev_err(dev, "Failed to set dram param: %ld\n", ++ res.a0); ++ return -EINVAL; ++ } ++ } ++ } ++ ++ dmcfreq->set_rate_params = ++ devm_kzalloc(dev, sizeof(struct share_params), GFP_KERNEL); ++ if (!dmcfreq->set_rate_params) ++ return -ENOMEM; ++ rockchip_set_ddrclk_params(dmcfreq->set_rate_params); ++ ++ arm_smccc_smc(ROCKCHIP_SIP_DRAM_FREQ, 0, 0, ++ ROCKCHIP_SIP_CONFIG_DRAM_INIT, ++ 0, 0, 0, 0, &res); ++ ++ dmcfreq->set_msch_readlatency = rk3399_set_msch_readlatency; ++ ++ return 0; ++} ++ ++static __maybe_unused int rk3568_dmc_init(struct platform_device *pdev, ++ struct rockchip_dmcfreq *dmcfreq) ++{ ++ struct arm_smccc_res res; ++ u32 size; ++ int ret; ++ int complt_irq; ++ ++ res = sip_smc_dram(0, 0, ++ ROCKCHIP_SIP_CONFIG_DRAM_GET_VERSION); ++ dev_notice(&pdev->dev, "current ATF version 0x%lx\n", res.a1); ++ if (res.a0 || res.a1 < 0x101) { ++ dev_err(&pdev->dev, "trusted firmware need update to V1.01 and above.\n"); ++ return -ENXIO; ++ } ++ ++ /* ++ * first 4KB is used for interface parameters ++ * after 4KB * N is dts parameters ++ */ ++ size = sizeof(struct rk1808_ddr_dts_config_timing); ++ res = sip_smc_request_share_mem(DIV_ROUND_UP(size, 4096) + 1, ++ SHARE_PAGE_TYPE_DDR); ++ if (res.a0 != 0) { ++ dev_err(&pdev->dev, "no ATF memory for init\n"); ++ return -ENOMEM; ++ } ++ ddr_psci_param = (struct share_params *)res.a1; ++ of_get_rk3568_timings(&pdev->dev, pdev->dev.of_node, ++ (uint32_t *)ddr_psci_param); ++ ++ /* start mcu with sip_smc_dram */ ++ wait_ctrl.dcf_en = 2; ++ ++ init_waitqueue_head(&wait_ctrl.wait_wq); ++ wait_ctrl.wait_en = 1; ++ wait_ctrl.wait_time_out_ms = 17 * 5; ++ ++ complt_irq = platform_get_irq_byname(pdev, "complete"); ++ if (complt_irq < 0) { ++ dev_err(&pdev->dev, "no IRQ for complt_irq: %d\n", ++ complt_irq); ++ return complt_irq; ++ } ++ wait_ctrl.complt_irq = complt_irq; ++ ++ ret = devm_request_irq(&pdev->dev, complt_irq, wait_dcf_complete_irq, ++ 0, dev_name(&pdev->dev), &wait_ctrl); ++ if (ret < 0) { ++ dev_err(&pdev->dev, "cannot request complt_irq\n"); ++ return ret; ++ } ++ disable_irq(complt_irq); ++ ++ if (of_property_read_u32(pdev->dev.of_node, "update_drv_odt_cfg", ++ &ddr_psci_param->update_drv_odt_cfg)) ++ ddr_psci_param->update_drv_odt_cfg = 0; ++ ++ if (of_property_read_u32(pdev->dev.of_node, "update_deskew_cfg", ++ &ddr_psci_param->update_deskew_cfg)) ++ ddr_psci_param->update_deskew_cfg = 0; ++ ++ res = sip_smc_dram(SHARE_PAGE_TYPE_DDR, 0, ++ ROCKCHIP_SIP_CONFIG_DRAM_INIT); ++ if (res.a0) { ++ dev_err(&pdev->dev, "rockchip_sip_config_dram_init error:%lx\n", ++ res.a0); ++ return -ENOMEM; ++ } ++ ++ ret = rockchip_get_freq_info(dmcfreq); ++ if (ret < 0) { ++ dev_err(&pdev->dev, "cannot get frequency info\n"); ++ return ret; ++ } ++ dmcfreq->is_set_rate_direct = true; ++ ++ dmcfreq->set_auto_self_refresh = rockchip_ddr_set_auto_self_refresh; ++ ++ return 0; ++} ++ ++static __maybe_unused int rv1126_dmc_init(struct platform_device *pdev, ++ struct rockchip_dmcfreq *dmcfreq) ++{ ++ struct arm_smccc_res res; ++ u32 size; ++ int ret; ++ int complt_irq; ++ struct device_node *node; ++ ++ res = sip_smc_dram(0, 0, ++ ROCKCHIP_SIP_CONFIG_DRAM_GET_VERSION); ++ dev_notice(&pdev->dev, "current ATF version 0x%lx\n", res.a1); ++ if (res.a0 || res.a1 < 0x100) { ++ dev_err(&pdev->dev, ++ "trusted firmware need to update or is invalid!\n"); ++ return -ENXIO; ++ } ++ ++ /* ++ * first 4KB is used for interface parameters ++ * after 4KB * N is dts parameters ++ */ ++ size = sizeof(struct rk1808_ddr_dts_config_timing); ++ res = sip_smc_request_share_mem(DIV_ROUND_UP(size, 4096) + 1, ++ SHARE_PAGE_TYPE_DDR); ++ if (res.a0 != 0) { ++ dev_err(&pdev->dev, "no ATF memory for init\n"); ++ return -ENOMEM; ++ } ++ ddr_psci_param = (struct share_params *)res.a1; ++ of_get_rv1126_timings(&pdev->dev, pdev->dev.of_node, ++ (uint32_t *)ddr_psci_param); ++ ++ /* enable start dcf in kernel after dcf ready */ ++ node = of_parse_phandle(pdev->dev.of_node, "dcf", 0); ++ wait_ctrl.regmap_dcf = syscon_node_to_regmap(node); ++ if (IS_ERR(wait_ctrl.regmap_dcf)) ++ return PTR_ERR(wait_ctrl.regmap_dcf); ++ wait_ctrl.dcf_en = 1; ++ ++ init_waitqueue_head(&wait_ctrl.wait_wq); ++ wait_ctrl.wait_en = 1; ++ wait_ctrl.wait_time_out_ms = 17 * 5; ++ ++ complt_irq = platform_get_irq_byname(pdev, "complete"); ++ if (complt_irq < 0) { ++ dev_err(&pdev->dev, "no IRQ for complt_irq: %d\n", ++ complt_irq); ++ return complt_irq; ++ } ++ wait_ctrl.complt_irq = complt_irq; ++ ++ ret = devm_request_irq(&pdev->dev, complt_irq, wait_dcf_complete_irq, ++ 0, dev_name(&pdev->dev), &wait_ctrl); ++ if (ret < 0) { ++ dev_err(&pdev->dev, "cannot request complt_irq\n"); ++ return ret; ++ } ++ disable_irq(complt_irq); ++ ++ if (of_property_read_u32(pdev->dev.of_node, "update_drv_odt_cfg", ++ &ddr_psci_param->update_drv_odt_cfg)) ++ ddr_psci_param->update_drv_odt_cfg = 0; ++ ++ if (of_property_read_u32(pdev->dev.of_node, "update_deskew_cfg", ++ &ddr_psci_param->update_deskew_cfg)) ++ ddr_psci_param->update_deskew_cfg = 0; ++ ++ dmcfreq->set_rate_params = ddr_psci_param; ++ rockchip_set_ddrclk_params(dmcfreq->set_rate_params); ++ rockchip_set_ddrclk_dmcfreq_wait_complete(rockchip_dmcfreq_wait_complete); ++ ++ res = sip_smc_dram(SHARE_PAGE_TYPE_DDR, 0, ++ ROCKCHIP_SIP_CONFIG_DRAM_INIT); ++ if (res.a0) { ++ dev_err(&pdev->dev, "rockchip_sip_config_dram_init error:%lx\n", ++ res.a0); ++ return -ENOMEM; ++ } ++ ++ dmcfreq->set_auto_self_refresh = rockchip_ddr_set_auto_self_refresh; ++ ++ return 0; ++} ++ ++static const struct of_device_id rockchip_dmcfreq_of_match[] = { ++#if IS_ENABLED(CONFIG_CPU_PX30) ++ { .compatible = "rockchip,px30-dmc", .data = px30_dmc_init }, ++#endif ++#if IS_ENABLED(CONFIG_CPU_RK1808) ++ { .compatible = "rockchip,rk1808-dmc", .data = rk1808_dmc_init }, ++#endif ++#if IS_ENABLED(CONFIG_CPU_RK312X) ++ { .compatible = "rockchip,rk3128-dmc", .data = rk3128_dmc_init }, ++#endif ++#if IS_ENABLED(CONFIG_CPU_RK322X) ++ { .compatible = "rockchip,rk3228-dmc", .data = rk3228_dmc_init }, ++#endif ++#if IS_ENABLED(CONFIG_CPU_RK3288) ++ { .compatible = "rockchip,rk3288-dmc", .data = rk3288_dmc_init }, ++#endif ++#if IS_ENABLED(CONFIG_CPU_RK3308) ++ { .compatible = "rockchip,rk3308-dmc", .data = NULL }, ++#endif ++#if IS_ENABLED(CONFIG_CPU_RK3328) ++ { .compatible = "rockchip,rk3328-dmc", .data = rk3328_dmc_init }, ++#endif ++#if IS_ENABLED(CONFIG_CPU_RK3368) ++ { .compatible = "rockchip,rk3368-dmc", .data = rk3368_dmc_init }, ++#endif ++#if IS_ENABLED(CONFIG_CPU_RK3399) ++ { .compatible = "rockchip,rk3399-dmc", .data = rk3399_dmc_init }, ++#endif ++#if IS_ENABLED(CONFIG_CPU_RK3568) ++ { .compatible = "rockchip,rk3568-dmc", .data = rk3568_dmc_init }, ++#endif ++#if IS_ENABLED(CONFIG_CPU_RV1126) ++ { .compatible = "rockchip,rv1126-dmc", .data = rv1126_dmc_init }, ++#endif ++ { }, ++}; ++MODULE_DEVICE_TABLE(of, rockchip_dmcfreq_of_match); ++ ++static int rockchip_get_freq_map_talbe(struct device_node *np, char *porp_name, ++ struct freq_map_table **table) ++{ ++ struct freq_map_table *tbl; ++ const struct property *prop; ++ unsigned int temp_freq = 0; ++ int count, i; ++ ++ prop = of_find_property(np, porp_name, NULL); ++ if (!prop) ++ return -EINVAL; ++ ++ if (!prop->value) ++ return -ENODATA; ++ ++ count = of_property_count_u32_elems(np, porp_name); ++ if (count < 0) ++ return -EINVAL; ++ ++ if (count % 3) ++ return -EINVAL; ++ ++ tbl = kzalloc(sizeof(*tbl) * (count / 3 + 1), GFP_KERNEL); ++ if (!tbl) ++ return -ENOMEM; ++ ++ for (i = 0; i < count / 3; i++) { ++ of_property_read_u32_index(np, porp_name, 3 * i, &tbl[i].min); ++ of_property_read_u32_index(np, porp_name, 3 * i + 1, ++ &tbl[i].max); ++ of_property_read_u32_index(np, porp_name, 3 * i + 2, ++ &temp_freq); ++ tbl[i].freq = temp_freq * 1000; ++ } ++ ++ tbl[i].min = 0; ++ tbl[i].max = 0; ++ tbl[i].freq = CPUFREQ_TABLE_END; ++ ++ *table = tbl; ++ ++ return 0; ++} ++ ++static int rockchip_get_rl_map_talbe(struct device_node *np, char *porp_name, ++ struct rl_map_table **table) ++{ ++ struct rl_map_table *tbl; ++ const struct property *prop; ++ int count, i; ++ ++ prop = of_find_property(np, porp_name, NULL); ++ if (!prop) ++ return -EINVAL; ++ ++ if (!prop->value) ++ return -ENODATA; ++ ++ count = of_property_count_u32_elems(np, porp_name); ++ if (count < 0) ++ return -EINVAL; ++ ++ if (count % 2) ++ return -EINVAL; ++ ++ tbl = kzalloc(sizeof(*tbl) * (count / 2 + 1), GFP_KERNEL); ++ if (!tbl) ++ return -ENOMEM; ++ ++ for (i = 0; i < count / 2; i++) { ++ of_property_read_u32_index(np, porp_name, 2 * i, &tbl[i].pn); ++ of_property_read_u32_index(np, porp_name, 2 * i + 1, ++ &tbl[i].rl); ++ } ++ ++ tbl[i].pn = 0; ++ tbl[i].rl = CPUFREQ_TABLE_END; ++ ++ *table = tbl; ++ ++ return 0; ++} ++ ++static int rockchip_get_system_status_rate(struct device_node *np, ++ char *porp_name, ++ struct rockchip_dmcfreq *dmcfreq) ++{ ++ const struct property *prop; ++ unsigned int status = 0, freq = 0; ++ unsigned long temp_rate = 0; ++ int count, i; ++ ++ prop = of_find_property(np, porp_name, NULL); ++ if (!prop) ++ return -ENODEV; ++ ++ if (!prop->value) ++ return -ENODATA; ++ ++ count = of_property_count_u32_elems(np, porp_name); ++ if (count < 0) ++ return -EINVAL; ++ ++ if (count % 2) ++ return -EINVAL; ++ ++ for (i = 0; i < count / 2; i++) { ++ of_property_read_u32_index(np, porp_name, 2 * i, ++ &status); ++ of_property_read_u32_index(np, porp_name, 2 * i + 1, ++ &freq); ++ switch (status) { ++ case SYS_STATUS_NORMAL: ++ dmcfreq->normal_rate = freq * 1000; ++ break; ++ case SYS_STATUS_SUSPEND: ++ dmcfreq->suspend_rate = freq * 1000; ++ break; ++ case SYS_STATUS_VIDEO_1080P: ++ dmcfreq->video_1080p_rate = freq * 1000; ++ break; ++ case SYS_STATUS_VIDEO_4K: ++ dmcfreq->video_4k_rate = freq * 1000; ++ break; ++ case SYS_STATUS_VIDEO_4K_10B: ++ dmcfreq->video_4k_10b_rate = freq * 1000; ++ break; ++ case SYS_STATUS_PERFORMANCE: ++ dmcfreq->performance_rate = freq * 1000; ++ break; ++ case SYS_STATUS_HDMI: ++ dmcfreq->hdmi_rate = freq * 1000; ++ break; ++ case SYS_STATUS_IDLE: ++ dmcfreq->idle_rate = freq * 1000; ++ break; ++ case SYS_STATUS_REBOOT: ++ dmcfreq->reboot_rate = freq * 1000; ++ break; ++ case SYS_STATUS_BOOST: ++ dmcfreq->boost_rate = freq * 1000; ++ break; ++ case SYS_STATUS_ISP: ++ case SYS_STATUS_CIF0: ++ case SYS_STATUS_CIF1: ++ case SYS_STATUS_DUALVIEW: ++ temp_rate = freq * 1000; ++ if (dmcfreq->fixed_rate < temp_rate) ++ dmcfreq->fixed_rate = temp_rate; ++ break; ++ case SYS_STATUS_LOW_POWER: ++ dmcfreq->low_power_rate = freq * 1000; ++ break; ++ default: ++ break; ++ } ++ } ++ ++ return 0; ++} ++ ++static unsigned long rockchip_freq_level_2_rate(struct rockchip_dmcfreq *dmcfreq, ++ unsigned int level) ++{ ++ unsigned long rate = 0; ++ ++ switch (level) { ++ case DMC_FREQ_LEVEL_LOW: ++ rate = dmcfreq->rate_low; ++ break; ++ case DMC_FREQ_LEVEL_MID_LOW: ++ rate = dmcfreq->rate_mid_low; ++ break; ++ case DMC_FREQ_LEVEL_MID_HIGH: ++ rate = dmcfreq->rate_mid_high; ++ break; ++ case DMC_FREQ_LEVEL_HIGH: ++ rate = dmcfreq->rate_high; ++ break; ++ default: ++ break; ++ } ++ ++ return rate; ++} ++ ++static int rockchip_get_system_status_level(struct device_node *np, ++ char *porp_name, ++ struct rockchip_dmcfreq *dmcfreq) ++{ ++ const struct property *prop; ++ unsigned int status = 0, level = 0; ++ unsigned long temp_rate = 0; ++ int count, i; ++ ++ prop = of_find_property(np, porp_name, NULL); ++ if (!prop) ++ return -ENODEV; ++ ++ if (!prop->value) ++ return -ENODATA; ++ ++ count = of_property_count_u32_elems(np, porp_name); ++ if (count < 0) ++ return -EINVAL; ++ ++ if (count % 2) ++ return -EINVAL; ++ ++ if (dmcfreq->freq_count == 1) { ++ dmcfreq->rate_low = dmcfreq->freq_info_rate[0]; ++ dmcfreq->rate_mid_low = dmcfreq->freq_info_rate[0]; ++ dmcfreq->rate_mid_high = dmcfreq->freq_info_rate[0]; ++ dmcfreq->rate_high = dmcfreq->freq_info_rate[0]; ++ } else if (dmcfreq->freq_count == 2) { ++ dmcfreq->rate_low = dmcfreq->freq_info_rate[0]; ++ dmcfreq->rate_mid_low = dmcfreq->freq_info_rate[0]; ++ dmcfreq->rate_mid_high = dmcfreq->freq_info_rate[1]; ++ dmcfreq->rate_high = dmcfreq->freq_info_rate[1]; ++ } else if (dmcfreq->freq_count == 3) { ++ dmcfreq->rate_low = dmcfreq->freq_info_rate[0]; ++ dmcfreq->rate_mid_low = dmcfreq->freq_info_rate[1]; ++ dmcfreq->rate_mid_high = dmcfreq->freq_info_rate[1]; ++ dmcfreq->rate_high = dmcfreq->freq_info_rate[2]; ++ } else if (dmcfreq->freq_count == 4) { ++ dmcfreq->rate_low = dmcfreq->freq_info_rate[0]; ++ dmcfreq->rate_mid_low = dmcfreq->freq_info_rate[1]; ++ dmcfreq->rate_mid_high = dmcfreq->freq_info_rate[2]; ++ dmcfreq->rate_high = dmcfreq->freq_info_rate[3]; ++ } else if (dmcfreq->freq_count == 5 || dmcfreq->freq_count == 6) { ++ dmcfreq->rate_low = dmcfreq->freq_info_rate[0]; ++ dmcfreq->rate_mid_low = dmcfreq->freq_info_rate[1]; ++ dmcfreq->rate_mid_high = dmcfreq->freq_info_rate[dmcfreq->freq_count - 2]; ++ dmcfreq->rate_high = dmcfreq->freq_info_rate[dmcfreq->freq_count - 1]; ++ } else { ++ return -EINVAL; ++ } ++ ++ for (i = 0; i < count / 2; i++) { ++ of_property_read_u32_index(np, porp_name, 2 * i, ++ &status); ++ of_property_read_u32_index(np, porp_name, 2 * i + 1, ++ &level); ++ switch (status) { ++ case SYS_STATUS_NORMAL: ++ dmcfreq->normal_rate = rockchip_freq_level_2_rate(dmcfreq, level); ++ dev_info(dmcfreq->dev, "normal_rate = %ld\n", dmcfreq->normal_rate); ++ break; ++ case SYS_STATUS_SUSPEND: ++ dmcfreq->suspend_rate = rockchip_freq_level_2_rate(dmcfreq, level); ++ dev_info(dmcfreq->dev, "suspend_rate = %ld\n", dmcfreq->suspend_rate); ++ break; ++ case SYS_STATUS_VIDEO_1080P: ++ dmcfreq->video_1080p_rate = rockchip_freq_level_2_rate(dmcfreq, level); ++ dev_info(dmcfreq->dev, "video_1080p_rate = %ld\n", ++ dmcfreq->video_1080p_rate); ++ break; ++ case SYS_STATUS_VIDEO_4K: ++ dmcfreq->video_4k_rate = rockchip_freq_level_2_rate(dmcfreq, level); ++ dev_info(dmcfreq->dev, "video_4k_rate = %ld\n", dmcfreq->video_4k_rate); ++ break; ++ case SYS_STATUS_VIDEO_4K_10B: ++ dmcfreq->video_4k_10b_rate = rockchip_freq_level_2_rate(dmcfreq, level); ++ dev_info(dmcfreq->dev, "video_4k_10b_rate = %ld\n", ++ dmcfreq->video_4k_10b_rate); ++ break; ++ case SYS_STATUS_PERFORMANCE: ++ dmcfreq->performance_rate = rockchip_freq_level_2_rate(dmcfreq, level); ++ dev_info(dmcfreq->dev, "performance_rate = %ld\n", ++ dmcfreq->performance_rate); ++ break; ++ case SYS_STATUS_HDMI: ++ dmcfreq->hdmi_rate = rockchip_freq_level_2_rate(dmcfreq, level); ++ dev_info(dmcfreq->dev, "hdmi_rate = %ld\n", dmcfreq->hdmi_rate); ++ break; ++ case SYS_STATUS_IDLE: ++ dmcfreq->idle_rate = rockchip_freq_level_2_rate(dmcfreq, level); ++ dev_info(dmcfreq->dev, "idle_rate = %ld\n", dmcfreq->idle_rate); ++ break; ++ case SYS_STATUS_REBOOT: ++ dmcfreq->reboot_rate = rockchip_freq_level_2_rate(dmcfreq, level); ++ dev_info(dmcfreq->dev, "reboot_rate = %ld\n", dmcfreq->reboot_rate); ++ break; ++ case SYS_STATUS_BOOST: ++ dmcfreq->boost_rate = rockchip_freq_level_2_rate(dmcfreq, level); ++ dev_info(dmcfreq->dev, "boost_rate = %ld\n", dmcfreq->boost_rate); ++ break; ++ case SYS_STATUS_ISP: ++ case SYS_STATUS_CIF0: ++ case SYS_STATUS_CIF1: ++ case SYS_STATUS_DUALVIEW: ++ temp_rate = rockchip_freq_level_2_rate(dmcfreq, level); ++ if (dmcfreq->fixed_rate < temp_rate) { ++ dmcfreq->fixed_rate = temp_rate; ++ dev_info(dmcfreq->dev, ++ "fixed_rate(isp|cif0|cif1|dualview) = %ld\n", ++ dmcfreq->fixed_rate); ++ } ++ break; ++ case SYS_STATUS_LOW_POWER: ++ dmcfreq->low_power_rate = rockchip_freq_level_2_rate(dmcfreq, level); ++ dev_info(dmcfreq->dev, "low_power_rate = %ld\n", dmcfreq->low_power_rate); ++ break; ++ default: ++ break; ++ } ++ } ++ ++ return 0; ++} ++ ++static void rockchip_dmcfreq_update_target(struct rockchip_dmcfreq *dmcfreq) ++{ ++ struct devfreq *df = dmcfreq->devfreq; ++ ++ mutex_lock(&df->lock); ++ ++ if (dmcfreq->last_refresh != dmcfreq->refresh) { ++ if (dmcfreq->set_auto_self_refresh) ++ dmcfreq->set_auto_self_refresh(dmcfreq->refresh); ++ dmcfreq->last_refresh = dmcfreq->refresh; ++ } ++ ++ update_devfreq(df); ++ ++ mutex_unlock(&df->lock); ++} ++ ++static int rockchip_dmcfreq_system_status_notifier(struct notifier_block *nb, ++ unsigned long status, ++ void *ptr) ++{ ++ struct rockchip_dmcfreq *dmcfreq = system_status_to_dmcfreq(nb); ++ unsigned long target_rate = 0; ++ unsigned int refresh = false; ++ bool is_fixed = false; ++ ++ if (dmcfreq->fixed_rate && (is_dualview(status) || is_isp(status))) { ++ if (dmcfreq->is_fixed) ++ return NOTIFY_OK; ++ is_fixed = true; ++ target_rate = dmcfreq->fixed_rate; ++ goto next; ++ } ++ ++ if (dmcfreq->reboot_rate && (status & SYS_STATUS_REBOOT)) { ++ if (dmcfreq->auto_freq_en) ++ devfreq_monitor_stop(dmcfreq->devfreq); ++ target_rate = dmcfreq->reboot_rate; ++ goto next; ++ } ++ ++ if (dmcfreq->suspend_rate && (status & SYS_STATUS_SUSPEND)) { ++ target_rate = dmcfreq->suspend_rate; ++ refresh = true; ++ goto next; ++ } ++ ++ if (dmcfreq->low_power_rate && (status & SYS_STATUS_LOW_POWER)) { ++ target_rate = dmcfreq->low_power_rate; ++ goto next; ++ } ++ ++ if (dmcfreq->performance_rate && (status & SYS_STATUS_PERFORMANCE)) { ++ if (dmcfreq->performance_rate > target_rate) ++ target_rate = dmcfreq->performance_rate; ++ } ++ ++ if (dmcfreq->hdmi_rate && (status & SYS_STATUS_HDMI)) { ++ if (dmcfreq->hdmi_rate > target_rate) ++ target_rate = dmcfreq->hdmi_rate; ++ } ++ ++ if (dmcfreq->video_4k_rate && (status & SYS_STATUS_VIDEO_4K)) { ++ if (dmcfreq->video_4k_rate > target_rate) ++ target_rate = dmcfreq->video_4k_rate; ++ } ++ ++ if (dmcfreq->video_4k_10b_rate && (status & SYS_STATUS_VIDEO_4K_10B)) { ++ if (dmcfreq->video_4k_10b_rate > target_rate) ++ target_rate = dmcfreq->video_4k_10b_rate; ++ } ++ ++ if (dmcfreq->video_1080p_rate && (status & SYS_STATUS_VIDEO_1080P)) { ++ if (dmcfreq->video_1080p_rate > target_rate) ++ target_rate = dmcfreq->video_1080p_rate; ++ } ++ ++next: ++ ++ dev_dbg(&dmcfreq->devfreq->dev, "status=0x%x\n", (unsigned int)status); ++ dmcfreq->refresh = refresh; ++ dmcfreq->is_fixed = is_fixed; ++ dmcfreq->status_rate = target_rate; ++ rockchip_dmcfreq_update_target(dmcfreq); ++ ++ return NOTIFY_OK; ++} ++ ++static ssize_t rockchip_dmcfreq_status_show(struct device *dev, ++ struct device_attribute *attr, ++ char *buf) ++{ ++ unsigned int status = rockchip_get_system_status(); ++ ++ return sprintf(buf, "0x%x\n", status); ++} ++ ++static ssize_t rockchip_dmcfreq_status_store(struct device *dev, ++ struct device_attribute *attr, ++ const char *buf, ++ size_t count) ++{ ++ if (!count) ++ return -EINVAL; ++ ++ rockchip_update_system_status(buf); ++ ++ return count; ++} ++ ++static DEVICE_ATTR(system_status, 0644, rockchip_dmcfreq_status_show, ++ rockchip_dmcfreq_status_store); ++ ++static void rockchip_dmcfreq_set_msch_rl(struct rockchip_dmcfreq *dmcfreq, ++ unsigned int readlatency) ++ ++{ ++ down_read(&rockchip_dmcfreq_sem); ++ dev_dbg(dmcfreq->dev, "rl 0x%x -> 0x%x\n", ++ dmcfreq->read_latency, readlatency); ++ if (!dmcfreq->set_msch_readlatency(readlatency)) ++ dmcfreq->read_latency = readlatency; ++ else ++ dev_err(dmcfreq->dev, "failed to set msch rl\n"); ++ up_read(&rockchip_dmcfreq_sem); ++} ++ ++static void rockchip_dmcfreq_set_msch_rl_work(struct work_struct *work) ++{ ++ struct rockchip_dmcfreq *dmcfreq = msch_rl_to_dmcfreq(work); ++ ++ rockchip_dmcfreq_set_msch_rl(dmcfreq, 0); ++ dmcfreq->is_msch_rl_work_started = false; ++} ++ ++static void rockchip_dmcfreq_msch_rl_init(struct rockchip_dmcfreq *dmcfreq) ++{ ++ if (!dmcfreq->set_msch_readlatency) ++ return; ++ INIT_DELAYED_WORK(&dmcfreq->msch_rl_work, ++ rockchip_dmcfreq_set_msch_rl_work); ++} ++ ++void rockchip_dmcfreq_vop_bandwidth_update(struct dmcfreq_vop_info *vop_info) ++{ ++ struct rockchip_dmcfreq *dmcfreq = rk_dmcfreq; ++ unsigned long vop_last_rate, target = 0; ++ unsigned int readlatency = 0; ++ int i; ++ ++ if (!dmcfreq) ++ return; ++ ++ if (!dmcfreq->vop_pn_rl_tbl || !dmcfreq->set_msch_readlatency) ++ goto vop_bw_tbl; ++ for (i = 0; dmcfreq->vop_pn_rl_tbl[i].rl != CPUFREQ_TABLE_END; i++) { ++ if (vop_info->plane_num >= dmcfreq->vop_pn_rl_tbl[i].pn) ++ readlatency = dmcfreq->vop_pn_rl_tbl[i].rl; ++ } ++ dev_dbg(dmcfreq->dev, "pn=%u\n", vop_info->plane_num); ++ if (readlatency) { ++ cancel_delayed_work_sync(&dmcfreq->msch_rl_work); ++ dmcfreq->is_msch_rl_work_started = false; ++ if (dmcfreq->read_latency != readlatency) ++ rockchip_dmcfreq_set_msch_rl(dmcfreq, readlatency); ++ } else if (dmcfreq->read_latency && ++ !dmcfreq->is_msch_rl_work_started) { ++ dmcfreq->is_msch_rl_work_started = true; ++ schedule_delayed_work(&dmcfreq->msch_rl_work, ++ msecs_to_jiffies(MSCH_RL_DELAY_TIME)); ++ } ++ ++vop_bw_tbl: ++ if (!dmcfreq->auto_freq_en || !dmcfreq->vop_bw_tbl) ++ return; ++ ++ for (i = 0; dmcfreq->vop_bw_tbl[i].freq != CPUFREQ_TABLE_END; i++) { ++ if (vop_info->bw_mbyte >= dmcfreq->vop_bw_tbl[i].min) ++ target = dmcfreq->vop_bw_tbl[i].freq; ++ } ++ ++ dev_dbg(dmcfreq->dev, "bw=%u\n", vop_info->bw_mbyte); ++ ++ if (!target || target == dmcfreq->vop_req_rate) ++ return; ++ ++ vop_last_rate = dmcfreq->vop_req_rate; ++ dmcfreq->vop_req_rate = target; ++ ++ if (target > vop_last_rate) ++ rockchip_dmcfreq_update_target(dmcfreq); ++} ++EXPORT_SYMBOL(rockchip_dmcfreq_vop_bandwidth_update); ++ ++int rockchip_dmcfreq_vop_bandwidth_request(struct dmcfreq_vop_info *vop_info) ++{ ++ struct rockchip_dmcfreq *dmcfreq = rk_dmcfreq; ++ unsigned long target = 0; ++ int i; ++ ++ if (!dmcfreq || !dmcfreq->auto_freq_en || !dmcfreq->vop_bw_tbl) ++ return 0; ++ ++ for (i = 0; dmcfreq->vop_bw_tbl[i].freq != CPUFREQ_TABLE_END; i++) { ++ if (vop_info->bw_mbyte <= dmcfreq->vop_bw_tbl[i].max) { ++ target = dmcfreq->vop_bw_tbl[i].freq; ++ break; ++ } ++ } ++ if (target) ++ return 0; ++ else ++ return -EINVAL; ++} ++EXPORT_SYMBOL(rockchip_dmcfreq_vop_bandwidth_request); ++ ++static int devfreq_dmc_ondemand_func(struct devfreq *df, ++ unsigned long *freq) ++{ ++ int err; ++ struct devfreq_dev_status *stat; ++ unsigned long long a, b; ++ struct rockchip_dmcfreq *dmcfreq = dev_get_drvdata(df->dev.parent); ++ struct devfreq_simple_ondemand_data *data = &dmcfreq->ondemand_data; ++ unsigned int upthreshold = data->upthreshold; ++ unsigned int downdifferential = data->downdifferential; ++ unsigned long target_freq = 0; ++ u64 now; ++ ++ if (dmcfreq->auto_freq_en && !dmcfreq->is_fixed) { ++ if (dmcfreq->status_rate) ++ target_freq = dmcfreq->status_rate; ++ else if (dmcfreq->auto_min_rate) ++ target_freq = dmcfreq->auto_min_rate; ++ now = ktime_to_us(ktime_get()); ++ if (now < dmcfreq->touchboostpulse_endtime) ++ target_freq = max3(target_freq, dmcfreq->vop_req_rate, ++ dmcfreq->boost_rate); ++ else ++ target_freq = max(target_freq, dmcfreq->vop_req_rate); ++ } else { ++ if (dmcfreq->status_rate) ++ target_freq = dmcfreq->status_rate; ++ else if (dmcfreq->normal_rate) ++ target_freq = dmcfreq->normal_rate; ++ if (target_freq) ++ *freq = target_freq; ++ if (dmcfreq->auto_freq_en && !devfreq_update_stats(df)) ++ return 0; ++ goto reset_last_status; ++ } ++ ++ if (!upthreshold || !downdifferential) ++ goto reset_last_status; ++ ++ if (upthreshold > 100 || ++ upthreshold < downdifferential) ++ goto reset_last_status; ++ ++ err = devfreq_update_stats(df); ++ if (err) ++ goto reset_last_status; ++ ++ stat = &df->last_status; ++ ++ /* Assume MAX if it is going to be divided by zero */ ++ if (stat->total_time == 0) { ++ *freq = DEVFREQ_MAX_FREQ; ++ return 0; ++ } ++ ++ /* Prevent overflow */ ++ if (stat->busy_time >= (1 << 24) || stat->total_time >= (1 << 24)) { ++ stat->busy_time >>= 7; ++ stat->total_time >>= 7; ++ } ++ ++ /* Set MAX if it's busy enough */ ++ if (stat->busy_time * 100 > ++ stat->total_time * upthreshold) { ++ *freq = DEVFREQ_MAX_FREQ; ++ return 0; ++ } ++ ++ /* Set MAX if we do not know the initial frequency */ ++ if (stat->current_frequency == 0) { ++ *freq = DEVFREQ_MAX_FREQ; ++ return 0; ++ } ++ ++ /* Keep the current frequency */ ++ if (stat->busy_time * 100 > ++ stat->total_time * (upthreshold - downdifferential)) { ++ *freq = max(target_freq, stat->current_frequency); ++ return 0; ++ } ++ ++ /* Set the desired frequency based on the load */ ++ a = stat->busy_time; ++ a *= stat->current_frequency; ++ b = div_u64(a, stat->total_time); ++ b *= 100; ++ b = div_u64(b, (upthreshold - downdifferential / 2)); ++ *freq = max_t(unsigned long, target_freq, b); ++ ++ return 0; ++ ++reset_last_status: ++ reset_last_status(df); ++ ++ return 0; ++} ++ ++static int devfreq_dmc_ondemand_handler(struct devfreq *devfreq, ++ unsigned int event, void *data) ++{ ++ struct rockchip_dmcfreq *dmcfreq = dev_get_drvdata(devfreq->dev.parent); ++ ++ if (!dmcfreq->auto_freq_en) ++ return 0; ++ ++ switch (event) { ++ case DEVFREQ_GOV_START: ++ devfreq_monitor_start(devfreq); ++ break; ++ ++ case DEVFREQ_GOV_STOP: ++ devfreq_monitor_stop(devfreq); ++ break; ++ ++ case DEVFREQ_GOV_UPDATE_INTERVAL: ++ devfreq_update_interval(devfreq, (unsigned int *)data); ++ break; ++ ++ case DEVFREQ_GOV_SUSPEND: ++ devfreq_monitor_suspend(devfreq); ++ break; ++ ++ case DEVFREQ_GOV_RESUME: ++ devfreq_monitor_resume(devfreq); ++ break; ++ ++ default: ++ break; ++ } ++ ++ return 0; ++} ++ ++static struct devfreq_governor devfreq_dmc_ondemand = { ++ .name = "dmc_ondemand", ++ .get_target_freq = devfreq_dmc_ondemand_func, ++ .event_handler = devfreq_dmc_ondemand_handler, ++}; ++ ++static int rockchip_dmcfreq_enable_event(struct rockchip_dmcfreq *dmcfreq) ++{ ++ int i, ret; ++ ++ if (!dmcfreq->auto_freq_en) ++ return 0; ++ ++ for (i = 0; i < dmcfreq->edev_count; i++) { ++ ret = devfreq_event_enable_edev(dmcfreq->edev[i]); ++ if (ret < 0) { ++ dev_err(dmcfreq->dev, ++ "failed to enable devfreq-event\n"); ++ return ret; ++ } ++ } ++ ++ return 0; ++} ++ ++static int rockchip_dmcfreq_disable_event(struct rockchip_dmcfreq *dmcfreq) ++{ ++ int i, ret; ++ ++ if (!dmcfreq->auto_freq_en) ++ return 0; ++ ++ for (i = 0; i < dmcfreq->edev_count; i++) { ++ ret = devfreq_event_disable_edev(dmcfreq->edev[i]); ++ if (ret < 0) { ++ dev_err(dmcfreq->dev, ++ "failed to disable devfreq-event\n"); ++ return ret; ++ } ++ } ++ ++ return 0; ++} ++ ++static int rockchip_get_edev_id(struct rockchip_dmcfreq *dmcfreq, ++ const char *name) ++{ ++ struct devfreq_event_dev *edev; ++ int i; ++ ++ for (i = 0; i < dmcfreq->edev_count; i++) { ++ edev = dmcfreq->edev[i]; ++ if (!strcmp(edev->desc->name, name)) ++ return i; ++ } ++ ++ return -EINVAL; ++} ++ ++static int rockchip_dmcfreq_get_event(struct rockchip_dmcfreq *dmcfreq) ++{ ++ struct device *dev = dmcfreq->dev; ++ struct device_node *events_np, *np = dev->of_node; ++ int i, j, count, available_count = 0; ++ ++ count = devfreq_event_get_edev_count(dev, "devfreq-events"); ++ if (count < 0) { ++ dev_dbg(dev, "failed to get count of devfreq-event dev\n"); ++ return 0; ++ } ++ for (i = 0; i < count; i++) { ++ events_np = of_parse_phandle(np, "devfreq-events", i); ++ if (!events_np) ++ continue; ++ if (of_device_is_available(events_np)) ++ available_count++; ++ of_node_put(events_np); ++ } ++ if (!available_count) { ++ dev_dbg(dev, "failed to get available devfreq-event\n"); ++ return 0; ++ } ++ dmcfreq->edev_count = available_count; ++ dmcfreq->edev = devm_kzalloc(dev, ++ sizeof(*dmcfreq->edev) * available_count, ++ GFP_KERNEL); ++ if (!dmcfreq->edev) ++ return -ENOMEM; ++ ++ for (i = 0, j = 0; i < count; i++) { ++ events_np = of_parse_phandle(np, "devfreq-events", i); ++ if (!events_np) ++ continue; ++ if (of_device_is_available(events_np)) { ++ of_node_put(events_np); ++ if (j >= available_count) { ++ dev_err(dev, "invalid event conut\n"); ++ return -EINVAL; ++ } ++ dmcfreq->edev[j] = ++ devfreq_event_get_edev_by_phandle(dev, "devfreq-events", i); ++ if (IS_ERR(dmcfreq->edev[j])) ++ return -EPROBE_DEFER; ++ j++; ++ } else { ++ of_node_put(events_np); ++ } ++ } ++ dmcfreq->auto_freq_en = true; ++ dmcfreq->dfi_id = rockchip_get_edev_id(dmcfreq, "dfi"); ++ if (dmcfreq->dfi_id >= 0) ++ available_count--; ++ if (available_count <= 0) ++ return 0; ++ dmcfreq->nocp_bw = ++ devm_kzalloc(dev, sizeof(*dmcfreq->nocp_bw) * available_count, ++ GFP_KERNEL); ++ if (!dmcfreq->nocp_bw) ++ return -ENOMEM; ++ ++ return 0; ++} ++ ++static int rockchip_dmcfreq_power_control(struct rockchip_dmcfreq *dmcfreq) ++{ ++ struct device *dev = dmcfreq->dev; ++ ++ dmcfreq->vdd_center = devm_regulator_get_optional(dev, "center"); ++ if (IS_ERR(dmcfreq->vdd_center)) { ++ dev_err(dev, "Cannot get the regulator \"center\"\n"); ++ return PTR_ERR(dmcfreq->vdd_center); ++ } ++ ++ dmcfreq->dmc_clk = devm_clk_get(dev, "dmc_clk"); ++ if (IS_ERR(dmcfreq->dmc_clk)) { ++ dev_err(dev, "Cannot get the clk dmc_clk. If using SCMI, trusted firmware need update to V1.01 and above.\n"); ++ return PTR_ERR(dmcfreq->dmc_clk); ++ } ++ dmcfreq->rate = clk_get_rate(dmcfreq->dmc_clk); ++ ++ return 0; ++} ++ ++static int rockchip_dmcfreq_dmc_init(struct platform_device *pdev, ++ struct rockchip_dmcfreq *dmcfreq) ++{ ++ const struct of_device_id *match; ++ int (*init)(struct platform_device *pdev, ++ struct rockchip_dmcfreq *data); ++ int ret; ++ ++ match = of_match_node(rockchip_dmcfreq_of_match, pdev->dev.of_node); ++ if (match) { ++ init = match->data; ++ if (init) { ++ ret = init(pdev, dmcfreq); ++ if (ret) ++ return ret; ++ } ++ } ++ ++ return 0; ++} ++ ++static void rockchip_dmcfreq_parse_dt(struct rockchip_dmcfreq *dmcfreq) ++{ ++ struct device *dev = dmcfreq->dev; ++ struct device_node *np = dev->of_node; ++ ++ if (!rockchip_get_system_status_rate(np, "system-status-freq", dmcfreq)) ++ dmcfreq->system_status_en = true; ++ else if (!rockchip_get_system_status_level(np, "system-status-level", dmcfreq)) ++ dmcfreq->system_status_en = true; ++ ++ of_property_read_u32(np, "min-cpu-freq", &dmcfreq->min_cpu_freq); ++ ++ of_property_read_u32(np, "upthreshold", ++ &dmcfreq->ondemand_data.upthreshold); ++ of_property_read_u32(np, "downdifferential", ++ &dmcfreq->ondemand_data.downdifferential); ++ if (dmcfreq->auto_freq_en) ++ of_property_read_u32(np, "auto-freq-en", ++ &dmcfreq->auto_freq_en); ++ of_property_read_u32(np, "auto-min-freq", ++ (u32 *)&dmcfreq->auto_min_rate); ++ dmcfreq->auto_min_rate *= 1000; ++ ++ if (rockchip_get_freq_map_talbe(np, "vop-bw-dmc-freq", ++ &dmcfreq->vop_bw_tbl)) ++ dev_err(dev, "failed to get vop bandwidth to dmc rate\n"); ++ if (rockchip_get_rl_map_talbe(np, "vop-pn-msch-readlatency", ++ &dmcfreq->vop_pn_rl_tbl)) ++ dev_err(dev, "failed to get vop pn to msch rl\n"); ++ ++ of_property_read_u32(np, "touchboost_duration", ++ (u32 *)&dmcfreq->touchboostpulse_duration_val); ++ if (dmcfreq->touchboostpulse_duration_val) ++ dmcfreq->touchboostpulse_duration_val *= USEC_PER_MSEC; ++ else ++ dmcfreq->touchboostpulse_duration_val = 500 * USEC_PER_MSEC; ++} ++ ++static int rockchip_dmcfreq_set_volt_only(struct rockchip_dmcfreq *dmcfreq) ++{ ++ struct device *dev = dmcfreq->dev; ++ struct dev_pm_opp *opp; ++ unsigned long opp_volt, opp_rate = dmcfreq->rate; ++ int ret; ++ ++ opp = devfreq_recommended_opp(dev, &opp_rate, 0); ++ if (IS_ERR(opp)) { ++ dev_err(dev, "Failed to find opp for %lu Hz\n", opp_rate); ++ return PTR_ERR(opp); ++ } ++ opp_volt = dev_pm_opp_get_voltage(opp); ++ dev_pm_opp_put(opp); ++ ++ ret = regulator_set_voltage(dmcfreq->vdd_center, opp_volt, INT_MAX); ++ if (ret) { ++ dev_err(dev, "Cannot set voltage %lu uV\n", opp_volt); ++ return ret; ++ } ++ ++ return 0; ++} ++ ++static int rockchip_dmcfreq_add_devfreq(struct rockchip_dmcfreq *dmcfreq) ++{ ++ struct devfreq_dev_profile *devp = &rockchip_devfreq_dmc_profile; ++ struct device *dev = dmcfreq->dev; ++ struct dev_pm_opp *opp; ++ unsigned long opp_rate = dmcfreq->rate; ++ ++ opp = devfreq_recommended_opp(dev, &opp_rate, 0); ++ if (IS_ERR(opp)) { ++ dev_err(dev, "Failed to find opp for %lu Hz\n", opp_rate); ++ return PTR_ERR(opp); ++ } ++ dev_pm_opp_put(opp); ++ ++ devp->initial_freq = dmcfreq->rate; ++ dmcfreq->devfreq = devm_devfreq_add_device(dev, devp, ++ "dmc_ondemand", ++ &dmcfreq->ondemand_data); ++ if (IS_ERR(dmcfreq->devfreq)) { ++ dev_err(dev, "failed to add devfreq\n"); ++ return PTR_ERR(dmcfreq->devfreq); ++ } ++ ++ devm_devfreq_register_opp_notifier(dev, dmcfreq->devfreq); ++ ++ dmcfreq->devfreq->last_status.current_frequency = opp_rate; ++ ++ reset_last_status(dmcfreq->devfreq); ++ ++ return 0; ++} ++ ++static struct monitor_dev_profile dmc_mdevp = { ++ .type = MONITOR_TPYE_DEV, ++ .low_temp_adjust = rockchip_monitor_dev_low_temp_adjust, ++ .high_temp_adjust = rockchip_monitor_dev_high_temp_adjust, ++}; ++ ++static void rockchip_dmcfreq_register_notifier(struct rockchip_dmcfreq *dmcfreq) ++{ ++ int ret; ++ ++ if (vop_register_dmc()) ++ dev_err(dmcfreq->dev, "fail to register notify to vop.\n"); ++ ++ dmcfreq->status_nb.notifier_call = ++ rockchip_dmcfreq_system_status_notifier; ++ ret = rockchip_register_system_status_notifier(&dmcfreq->status_nb); ++ if (ret) ++ dev_err(dmcfreq->dev, "failed to register system_status nb\n"); ++ ++ dmc_mdevp.data = dmcfreq->devfreq; ++ dmcfreq->mdev_info = rockchip_system_monitor_register(dmcfreq->dev, ++ &dmc_mdevp); ++ if (IS_ERR(dmcfreq->mdev_info)) { ++ dev_dbg(dmcfreq->dev, "without without system monitor\n"); ++ dmcfreq->mdev_info = NULL; ++ } ++} ++ ++static void rockchip_dmcfreq_add_interface(struct rockchip_dmcfreq *dmcfreq) ++{ ++ if (!rockchip_add_system_status_interface(&dmcfreq->devfreq->dev)) ++ return; ++ if (sysfs_create_file(&dmcfreq->devfreq->dev.kobj, ++ &dev_attr_system_status.attr)) ++ dev_err(dmcfreq->dev, ++ "failed to register system_status sysfs file\n"); ++} ++ ++static void rockchip_dmcfreq_boost_work(struct work_struct *work) ++{ ++ struct rockchip_dmcfreq *dmcfreq = boost_to_dmcfreq(work); ++ ++ rockchip_dmcfreq_update_target(dmcfreq); ++} ++ ++static void rockchip_dmcfreq_input_event(struct input_handle *handle, ++ unsigned int type, ++ unsigned int code, ++ int value) ++{ ++ struct rockchip_dmcfreq *dmcfreq = handle->private; ++ u64 now, endtime; ++ ++ if (type != EV_ABS && type != EV_KEY) ++ return; ++ ++ now = ktime_to_us(ktime_get()); ++ endtime = now + dmcfreq->touchboostpulse_duration_val; ++ if (endtime < (dmcfreq->touchboostpulse_endtime + 10 * USEC_PER_MSEC)) ++ return; ++ dmcfreq->touchboostpulse_endtime = endtime; ++ ++ schedule_work(&dmcfreq->boost_work); ++} ++ ++static int rockchip_dmcfreq_input_connect(struct input_handler *handler, ++ struct input_dev *dev, ++ const struct input_device_id *id) ++{ ++ int error; ++ struct input_handle *handle; ++ struct rockchip_dmcfreq *dmcfreq = input_hd_to_dmcfreq(handler); ++ ++ handle = kzalloc(sizeof(*handle), GFP_KERNEL); ++ if (!handle) ++ return -ENOMEM; ++ ++ handle->dev = dev; ++ handle->handler = handler; ++ handle->name = "dmcfreq"; ++ handle->private = dmcfreq; ++ ++ error = input_register_handle(handle); ++ if (error) ++ goto err2; ++ ++ error = input_open_device(handle); ++ if (error) ++ goto err1; ++ ++ return 0; ++err1: ++ input_unregister_handle(handle); ++err2: ++ kfree(handle); ++ return error; ++} ++ ++static void rockchip_dmcfreq_input_disconnect(struct input_handle *handle) ++{ ++ input_close_device(handle); ++ input_unregister_handle(handle); ++ kfree(handle); ++} ++ ++static const struct input_device_id rockchip_dmcfreq_input_ids[] = { ++ { ++ .flags = INPUT_DEVICE_ID_MATCH_EVBIT | ++ INPUT_DEVICE_ID_MATCH_ABSBIT, ++ .evbit = { BIT_MASK(EV_ABS) }, ++ .absbit = { [BIT_WORD(ABS_MT_POSITION_X)] = ++ BIT_MASK(ABS_MT_POSITION_X) | ++ BIT_MASK(ABS_MT_POSITION_Y) }, ++ }, ++ { ++ .flags = INPUT_DEVICE_ID_MATCH_KEYBIT | ++ INPUT_DEVICE_ID_MATCH_ABSBIT, ++ .keybit = { [BIT_WORD(BTN_TOUCH)] = BIT_MASK(BTN_TOUCH) }, ++ .absbit = { [BIT_WORD(ABS_X)] = ++ BIT_MASK(ABS_X) | BIT_MASK(ABS_Y) }, ++ }, ++ { ++ .flags = INPUT_DEVICE_ID_MATCH_EVBIT, ++ .evbit = { BIT_MASK(EV_KEY) }, ++ }, ++ { }, ++}; ++ ++static void rockchip_dmcfreq_boost_init(struct rockchip_dmcfreq *dmcfreq) ++{ ++ if (!dmcfreq->boost_rate) ++ return; ++ INIT_WORK(&dmcfreq->boost_work, rockchip_dmcfreq_boost_work); ++ dmcfreq->input_handler.event = rockchip_dmcfreq_input_event; ++ dmcfreq->input_handler.connect = rockchip_dmcfreq_input_connect; ++ dmcfreq->input_handler.disconnect = rockchip_dmcfreq_input_disconnect; ++ dmcfreq->input_handler.name = "dmcfreq"; ++ dmcfreq->input_handler.id_table = rockchip_dmcfreq_input_ids; ++ if (input_register_handler(&dmcfreq->input_handler)) ++ dev_err(dmcfreq->dev, "failed to register input handler\n"); ++} ++ ++static unsigned long model_static_power(struct devfreq *devfreq, ++ unsigned long voltage) ++{ ++ struct device *dev = devfreq->dev.parent; ++ struct rockchip_dmcfreq *dmcfreq = dev_get_drvdata(dev); ++ ++ int temperature; ++ unsigned long temp; ++ unsigned long temp_squared, temp_cubed, temp_scaling_factor; ++ const unsigned long voltage_cubed = (voltage * voltage * voltage) >> 10; ++ ++ if (!IS_ERR_OR_NULL(dmcfreq->ddr_tz) && dmcfreq->ddr_tz->ops->get_temp) { ++ int ret; ++ ++ ret = ++ dmcfreq->ddr_tz->ops->get_temp(dmcfreq->ddr_tz, ++ &temperature); ++ if (ret) { ++ dev_warn_ratelimited(dev, ++ "failed to read temp for ddr thermal zone: %d\n", ++ ret); ++ temperature = FALLBACK_STATIC_TEMPERATURE; ++ } ++ } else { ++ temperature = FALLBACK_STATIC_TEMPERATURE; ++ } ++ ++ /* ++ * Calculate the temperature scaling factor. To be applied to the ++ * voltage scaled power. ++ */ ++ temp = temperature / 1000; ++ temp_squared = temp * temp; ++ temp_cubed = temp_squared * temp; ++ temp_scaling_factor = (dmcfreq->ts[3] * temp_cubed) ++ + (dmcfreq->ts[2] * temp_squared) ++ + (dmcfreq->ts[1] * temp) ++ + dmcfreq->ts[0]; ++ ++ return (((dmcfreq->static_coefficient * voltage_cubed) >> 20) ++ * temp_scaling_factor) / 1000000; ++} ++ ++static struct devfreq_cooling_power ddr_cooling_power_data = { ++ .get_static_power = model_static_power, ++ .dyn_power_coeff = 120, ++}; ++ ++static int ddr_power_model_simple_init(struct rockchip_dmcfreq *dmcfreq) ++{ ++ struct device_node *power_model_node; ++ const char *tz_name; ++ u32 temp; ++ ++ power_model_node = of_get_child_by_name(dmcfreq->dev->of_node, ++ "ddr_power_model"); ++ if (!power_model_node) { ++ dev_err(dmcfreq->dev, "could not find power_model node\n"); ++ return -ENODEV; ++ } ++ ++ if (of_property_read_string(power_model_node, "thermal-zone", &tz_name)) { ++ dev_err(dmcfreq->dev, "ts in power_model not available\n"); ++ return -EINVAL; ++ } ++ ++ dmcfreq->ddr_tz = thermal_zone_get_zone_by_name(tz_name); ++ if (IS_ERR(dmcfreq->ddr_tz)) { ++ pr_warn_ratelimited ++ ("Error getting ddr thermal zone (%ld), not yet ready?\n", ++ PTR_ERR(dmcfreq->ddr_tz)); ++ dmcfreq->ddr_tz = NULL; ++ ++ return -EPROBE_DEFER; ++ } ++ ++ if (of_property_read_u32(power_model_node, "static-power-coefficient", ++ &dmcfreq->static_coefficient)) { ++ dev_err(dmcfreq->dev, ++ "static-power-coefficient not available\n"); ++ return -EINVAL; ++ } ++ if (of_property_read_u32(power_model_node, "dynamic-power-coefficient", ++ &temp)) { ++ dev_err(dmcfreq->dev, ++ "dynamic-power-coefficient not available\n"); ++ return -EINVAL; ++ } ++ ddr_cooling_power_data.dyn_power_coeff = (unsigned long)temp; ++ ++ if (of_property_read_u32_array ++ (power_model_node, "ts", (u32 *)dmcfreq->ts, 4)) { ++ dev_err(dmcfreq->dev, "ts in power_model not available\n"); ++ return -EINVAL; ++ } ++ ++ return 0; ++} ++ ++static void ++rockchip_dmcfreq_register_cooling_device(struct rockchip_dmcfreq *dmcfreq) ++{ ++ int ret; ++ ++ ret = ddr_power_model_simple_init(dmcfreq); ++ if (ret) ++ return; ++ dmcfreq->devfreq_cooling = ++ of_devfreq_cooling_register_power(dmcfreq->dev->of_node, ++ dmcfreq->devfreq, ++ &ddr_cooling_power_data); ++ if (IS_ERR(dmcfreq->devfreq_cooling)) { ++ ret = PTR_ERR(dmcfreq->devfreq_cooling); ++ dev_err(dmcfreq->dev, ++ "Failed to register cooling device (%d)\n", ++ ret); ++ } ++} ++ ++static int rockchip_dmcfreq_probe(struct platform_device *pdev) ++{ ++ struct device *dev = &pdev->dev; ++ struct rockchip_dmcfreq *data; ++ int ret; ++ ++ data = devm_kzalloc(dev, sizeof(struct rockchip_dmcfreq), GFP_KERNEL); ++ if (!data) ++ return -ENOMEM; ++ ++ data->dev = dev; ++ mutex_init(&data->lock); ++ INIT_LIST_HEAD(&data->video_info_list); ++ ++ ret = rockchip_dmcfreq_get_event(data); ++ if (ret) ++ return ret; ++ ++ ret = rockchip_dmcfreq_power_control(data); ++ if (ret) ++ return ret; ++ ++ ret = rockchip_init_opp_table(dev, NULL, "ddr_leakage", "center"); ++ if (ret) ++ return ret; ++ ++ ret = rockchip_dmcfreq_dmc_init(pdev, data); ++ if (ret) ++ return ret; ++ ++ rockchip_dmcfreq_parse_dt(data); ++ if (!data->system_status_en && !data->auto_freq_en) { ++ dev_info(dev, "don't add devfreq feature\n"); ++ return rockchip_dmcfreq_set_volt_only(data); ++ } ++ ++ cpu_latency_qos_add_request(&pm_qos, PM_QOS_DEFAULT_VALUE); ++ platform_set_drvdata(pdev, data); ++ ++ ret = devfreq_add_governor(&devfreq_dmc_ondemand); ++ if (ret) ++ return ret; ++ ret = rockchip_dmcfreq_enable_event(data); ++ if (ret) ++ return ret; ++ ret = rockchip_dmcfreq_add_devfreq(data); ++ if (ret) { ++ rockchip_dmcfreq_disable_event(data); ++ return ret; ++ } ++ ++ rockchip_dmcfreq_register_notifier(data); ++ rockchip_dmcfreq_add_interface(data); ++ rockchip_dmcfreq_boost_init(data); ++ rockchip_dmcfreq_msch_rl_init(data); ++ rockchip_dmcfreq_register_cooling_device(data); ++ ++ rockchip_set_system_status(SYS_STATUS_NORMAL); ++ ++ rk_dmcfreq = data; ++ ++ return 0; ++} ++ ++static __maybe_unused int rockchip_dmcfreq_suspend(struct device *dev) ++{ ++ struct rockchip_dmcfreq *dmcfreq = dev_get_drvdata(dev); ++ int ret = 0; ++ ++ if (!dmcfreq) ++ return 0; ++ ++ ret = rockchip_dmcfreq_disable_event(dmcfreq); ++ if (ret) ++ return ret; ++ ++ ret = devfreq_suspend_device(dmcfreq->devfreq); ++ if (ret < 0) { ++ dev_err(dev, "failed to suspend the devfreq devices\n"); ++ return ret; ++ } ++ ++ return 0; ++} ++ ++static __maybe_unused int rockchip_dmcfreq_resume(struct device *dev) ++{ ++ struct rockchip_dmcfreq *dmcfreq = dev_get_drvdata(dev); ++ int ret = 0; ++ ++ if (!dmcfreq) ++ return 0; ++ ++ ret = rockchip_dmcfreq_enable_event(dmcfreq); ++ if (ret) ++ return ret; ++ ++ ret = devfreq_resume_device(dmcfreq->devfreq); ++ if (ret < 0) { ++ dev_err(dev, "failed to resume the devfreq devices\n"); ++ return ret; ++ } ++ return ret; ++} ++ ++static SIMPLE_DEV_PM_OPS(rockchip_dmcfreq_pm, rockchip_dmcfreq_suspend, ++ rockchip_dmcfreq_resume); ++static struct platform_driver rockchip_dmcfreq_driver = { ++ .probe = rockchip_dmcfreq_probe, ++ .driver = { ++ .name = "rockchip-dmc", ++ .pm = &rockchip_dmcfreq_pm, ++ .of_match_table = rockchip_dmcfreq_of_match, ++ }, ++}; ++module_platform_driver(rockchip_dmcfreq_driver); ++ ++MODULE_LICENSE("GPL v2"); ++MODULE_AUTHOR("Lin Huang "); ++MODULE_DESCRIPTION("rockchip dmcfreq driver with devfreq framework"); +diff --git a/drivers/devfreq/rockchip_dmc_dbg.c b/drivers/devfreq/rockchip_dmc_dbg.c +new file mode 100755 +index 000000000000..80b25e9046d1 +--- /dev/null ++++ b/drivers/devfreq/rockchip_dmc_dbg.c +@@ -0,0 +1,1061 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * Copyright (c) 2020, Rockchip Electronics Co., Ltd. ++ */ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++ ++#include "rockchip_dmc_timing.h" ++ ++/* ++ * DMCDBG share memory request 4KB for delivery parameter ++ */ ++#define DMCDBG_PAGE_NUMS (1) ++#define DMCDBG_SHARE_MEM_SIZE ((DMCDBG_PAGE_NUMS) * 4096) ++ ++#define PROC_DMCDBG_DIR_NAME "dmcdbg" ++#define PROC_DMCDBG_DRAM_INFO "dmcinfo" ++#define PROC_DMCDBG_POWERSAVE "powersave" ++#define PROC_DMCDBG_DRVODT "drvodt" ++#define PROC_DMCDBG_DESKEW "deskew" ++#define PROC_DMCDBG_REGS_INFO "regsinfo" ++ ++#define DDRDBG_FUNC_GET_VERSION (0x01) ++#define DDRDBG_FUNC_GET_SUPPORTED (0x02) ++#define DDRDBG_FUNC_GET_DRAM_INFO (0x03) ++#define DDRDBG_FUNC_GET_DESKEW_INFO (0x04) ++#define DDRDBG_FUNC_UPDATE_DESKEW (0x05) ++#define DDRDBG_FUNC_DATA_TRAINING (0x06) ++#define DDRDBG_FUNC_UPDATE_DESKEW_TR (0x07) ++#define DDRDBG_FUNC_GET_POWERSAVE_INFO (0x08) ++#define DDRDBG_FUNC_UPDATE_POWERSAVE (0x09) ++#define DDRDBG_FUNC_GET_DRVODT_INFO (0x0a) ++#define DDRDBG_FUNC_UPDATE_DRVODT (0x0b) ++#define DDRDBG_FUNC_GET_REGISTERS_INFO (0x0c) ++ ++#define DRV_ODT_UNKNOWN (0xffff) ++#define DRV_ODT_UNSUSPEND_FIX (0x0) ++#define DRV_ODT_SUSPEND_FIX (0x1) ++ ++#define REGS_NAME_LEN_MAX (20) ++#define SKEW_GROUP_NUM_MAX (6) ++#define SKEW_TIMING_NUM_MAX (50) ++ ++struct rockchip_dmcdbg { ++ struct device *dev; ++}; ++ ++struct proc_dir_entry *proc_dmcdbg_dir; ++ ++struct dram_cap_info { ++ unsigned int rank; ++ unsigned int col; ++ unsigned int bank; ++ unsigned int buswidth; ++ unsigned int die_buswidth; ++ unsigned int row_3_4; ++ unsigned int cs0_row; ++ unsigned int cs1_row; ++ unsigned int cs0_high16bit_row; ++ unsigned int cs1_high16bit_row; ++ unsigned int bankgroup; ++ unsigned int size; ++}; ++ ++struct dram_info { ++ unsigned int version; ++ char dramtype[10]; ++ unsigned int dramfreq; ++ unsigned int channel_num; ++ struct dram_cap_info ch[2]; ++}; ++ ++static const char * const power_save_msg[] = { ++ "auto power down enable", ++ "auto power down idle cycle", ++ "auto self refresh enable", ++ "auto self refresh idle cycle", ++ "self refresh with clock gate idle cycle", ++ "self refresh and power down lite idle cycle", ++ "standby idle cycle", ++}; ++ ++struct power_save_info { ++ unsigned int pd_en; ++ unsigned int pd_idle; ++ unsigned int sr_en; ++ unsigned int sr_idle; ++ unsigned int sr_mc_gate_idle; ++ unsigned int srpd_lite_idle; ++ unsigned int standby_idle; ++}; ++ ++static const char * const drv_odt_msg[] = { ++ "dram side drv pull-up", ++ "dram side drv pull-down", ++ "dram side dq odt pull-up", ++ "dram side dq odt pull-down", ++ "dram side ca odt pull-up", ++ "dram side ca odt pull-down", ++ "soc side ca drv pull-up", ++ "soc side ca drv pull-down", ++ "soc side ck drv pull-up", ++ "soc side ck drv pull-down", ++ "soc side cs drv pull-up", ++ "soc side cs drv pull-down", ++ "soc side dq drv pull-up", ++ "soc side dq drv pull-down", ++ "soc side odt pull-up", ++ "soc side odt pull-down", ++ "phy vref inner", ++ "phy vref out", ++}; ++ ++struct drv_odt { ++ unsigned int value; ++ unsigned int ohm; ++ unsigned int flag; ++}; ++ ++struct drv_odt_vref { ++ unsigned int value; ++ unsigned int percen; ++ unsigned int flag; ++}; ++ ++struct drv_odt_info { ++ struct drv_odt dram_drv_up; ++ struct drv_odt dram_drv_down; ++ struct drv_odt dram_dq_odt_up; ++ struct drv_odt dram_dq_odt_down; ++ struct drv_odt dram_ca_odt_up; ++ struct drv_odt dram_ca_odt_down; ++ struct drv_odt phy_ca_drv_up; ++ struct drv_odt phy_ca_drv_down; ++ struct drv_odt phy_ck_drv_up; ++ struct drv_odt phy_ck_drv_down; ++ struct drv_odt phy_cs_drv_up; ++ struct drv_odt phy_cs_drv_down; ++ struct drv_odt phy_dq_drv_up; ++ struct drv_odt phy_dq_drv_down; ++ struct drv_odt phy_odt_up; ++ struct drv_odt phy_odt_down; ++ struct drv_odt_vref phy_vref_inner; ++ struct drv_odt_vref phy_vref_out; ++}; ++ ++struct dmc_registers { ++ char regs_name[REGS_NAME_LEN_MAX]; ++ unsigned int regs_addr; ++}; ++ ++struct registers_info { ++ unsigned int regs_num; ++ struct dmc_registers regs[]; ++}; ++ ++struct skew_group { ++ unsigned int skew_num; ++ unsigned int *p_skew_info; ++ char *p_skew_timing[SKEW_TIMING_NUM_MAX]; ++ char *note; ++}; ++ ++struct rockchip_dmcdbg_data { ++ unsigned int inited_flag; ++ void __iomem *share_memory; ++ unsigned int skew_group_num; ++ struct skew_group skew_group[SKEW_GROUP_NUM_MAX]; ++}; ++ ++static struct rockchip_dmcdbg_data dmcdbg_data; ++ ++struct skew_info_rv1126 { ++ unsigned int ca_skew[32]; ++ unsigned int cs0_a_skew[44]; ++ unsigned int cs0_b_skew[44]; ++ unsigned int cs1_a_skew[44]; ++ unsigned int cs1_b_skew[44]; ++}; ++ ++static int dmcinfo_proc_show(struct seq_file *m, void *v) ++{ ++ struct arm_smccc_res res; ++ struct dram_info *p_dram_info; ++ struct file *fp = NULL; ++ char cur_freq[20] = {0}; ++ char governor[20] = {0}; ++ loff_t pos; ++ u32 i; ++ ++ res = sip_smc_dram(SHARE_PAGE_TYPE_DDRDBG, DDRDBG_FUNC_GET_DRAM_INFO, ++ ROCKCHIP_SIP_CONFIG_DRAM_DEBUG); ++ if (res.a0) { ++ seq_printf(m, "rockchip_sip_config_dram_debug error:%lx\n", ++ res.a0); ++ return -ENOMEM; ++ } ++ ++ if (!dmcdbg_data.inited_flag) { ++ seq_puts(m, "dmcdbg_data no int\n"); ++ return -EPERM; ++ } ++ p_dram_info = (struct dram_info *)dmcdbg_data.share_memory; ++ ++ /* dram type information */ ++ seq_printf(m, ++ "DramType: %s\n" ++ , ++ p_dram_info->dramtype ++ ); ++ ++ /* dram capacity information */ ++ seq_printf(m, ++ "\n" ++ "DramCapacity:\n" ++ ); ++ ++ for (i = 0; i < p_dram_info->channel_num; i++) { ++ if (p_dram_info->channel_num == 2) ++ seq_printf(m, ++ "Channel [%d]:\n" ++ , ++ i ++ ); ++ ++ seq_printf(m, ++ "CS Count: %d\n" ++ "Bus Width: %d bit\n" ++ "Column: %d\n" ++ "Bank: %d\n" ++ "CS0_Row: %d\n" ++ "CS1_Row: %d\n" ++ "DieBusWidth: %d bit\n" ++ "TotalSize: %d MB\n" ++ , ++ p_dram_info->ch[i].rank, ++ p_dram_info->ch[i].buswidth, ++ p_dram_info->ch[i].col, ++ p_dram_info->ch[i].bank, ++ p_dram_info->ch[i].cs0_row, ++ p_dram_info->ch[i].cs1_row, ++ p_dram_info->ch[i].die_buswidth, ++ p_dram_info->ch[i].size ++ ); ++ } ++ ++ /* check devfreq/dmc device */ ++ fp = filp_open("/sys/class/devfreq/dmc/cur_freq", O_RDONLY, 0); ++ if (IS_ERR(fp)) { ++ seq_printf(m, ++ "\n" ++ "devfreq/dmc: Disable\n" ++ "DramFreq: %d\n" ++ , ++ p_dram_info->dramfreq ++ ); ++ } else { ++ pos = 0; ++ kernel_read(fp, cur_freq, sizeof(cur_freq), &pos); ++ filp_close(fp, NULL); ++ ++ fp = filp_open("/sys/class/devfreq/dmc/governor", O_RDONLY, 0); ++ if (IS_ERR(fp)) { ++ fp = NULL; ++ } else { ++ pos = 0; ++ kernel_read(fp, governor, sizeof(governor), &pos); ++ filp_close(fp, NULL); ++ } ++ ++ seq_printf(m, ++ "\n" ++ "devfreq/dmc: Enable\n" ++ "governor: %s\n" ++ "cur_freq: %s\n" ++ , ++ governor, ++ cur_freq ++ ); ++ seq_printf(m, ++ "NOTE:\n" ++ "more information about dmc can get from /sys/class/devfreq/dmc.\n" ++ ); ++ } ++ ++ return 0; ++} ++ ++static int dmcinfo_proc_open(struct inode *inode, struct file *file) ++{ ++ return single_open(file, dmcinfo_proc_show, NULL); ++} ++ ++static const struct file_operations dmcinfo_proc_fops = { ++ .open = dmcinfo_proc_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = single_release, ++}; ++ ++static int proc_dmcinfo_init(void) ++{ ++ /* create dmcinfo file */ ++ proc_create(PROC_DMCDBG_DRAM_INFO, 0644, proc_dmcdbg_dir, ++ &dmcinfo_proc_fops); ++ ++ return 0; ++} ++ ++static int powersave_proc_show(struct seq_file *m, void *v) ++{ ++ struct arm_smccc_res res; ++ struct power_save_info *p_power; ++ unsigned int *p_uint; ++ unsigned int i = 0; ++ ++ /* get low power information */ ++ res = sip_smc_dram(SHARE_PAGE_TYPE_DDRDBG, ++ DDRDBG_FUNC_GET_POWERSAVE_INFO, ++ ROCKCHIP_SIP_CONFIG_DRAM_DEBUG); ++ if (res.a0) { ++ seq_printf(m, "rockchip_sip_config_dram_debug error:%lx\n", ++ res.a0); ++ return -ENOMEM; ++ } ++ ++ if (!dmcdbg_data.inited_flag) { ++ seq_puts(m, "dmcdbg_data no int\n"); ++ return -EPERM; ++ } ++ p_power = (struct power_save_info *)dmcdbg_data.share_memory; ++ ++ seq_printf(m, ++ "low power information:\n" ++ "\n" ++ "[number]name: value\n" ++ ); ++ ++ p_uint = (unsigned int *)p_power; ++ for (i = 0; i < ARRAY_SIZE(power_save_msg); i++) ++ seq_printf(m, ++ "[%d]%s: %d\n" ++ , ++ i, power_save_msg[i], *(p_uint + i) ++ ); ++ ++ seq_printf(m, ++ "\n" ++ "power save setting:\n" ++ "echo number=value > /proc/dmcdbg/powersave\n" ++ "eg: set auto power down enable to 1\n" ++ " echo 0=1 > /proc/dmcdbg/powersave\n" ++ "\n" ++ "Support for setting multiple parameters at the same time.\n" ++ "echo number=value,number=value,... > /proc/dmcdbg/powersave\n" ++ "eg:\n" ++ " echo 0=1,1=32 > /proc/dmcdbg/powersave\n" ++ ); ++ ++ return 0; ++} ++ ++static int powersave_proc_open(struct inode *inode, struct file *file) ++{ ++ return single_open(file, powersave_proc_show, NULL); ++} ++ ++static ssize_t powersave_proc_write(struct file *file, ++ const char __user *buffer, ++ size_t count, loff_t *ppos) ++{ ++ struct arm_smccc_res res; ++ struct power_save_info *p_power; ++ unsigned int *p_uint; ++ char *buf, *cookie_pot, *p_char; ++ int ret = 0; ++ u32 loop, i, offset, value; ++ long long_val; ++ ++ /* get buffer data */ ++ buf = vzalloc(count); ++ cookie_pot = buf; ++ if (!cookie_pot) ++ return -ENOMEM; ++ ++ if (copy_from_user(cookie_pot, buffer, count)) { ++ ret = -EFAULT; ++ goto err; ++ } ++ ++ /* get power save setting information */ ++ res = sip_smc_dram(SHARE_PAGE_TYPE_DDRDBG, ++ DDRDBG_FUNC_GET_POWERSAVE_INFO, ++ ROCKCHIP_SIP_CONFIG_DRAM_DEBUG); ++ if (res.a0) { ++ pr_err("rockchip_sip_config_dram_debug error:%lx\n", res.a0); ++ ret = -ENOMEM; ++ goto err; ++ } ++ ++ if (!dmcdbg_data.inited_flag) { ++ pr_err("dmcdbg_data no int\n"); ++ ret = -EPERM; ++ goto err; ++ } ++ p_power = (struct power_save_info *)dmcdbg_data.share_memory; ++ ++ loop = 0; ++ for (i = 0; i < count; i++) { ++ if (*(cookie_pot + i) == '=') ++ loop++; ++ } ++ ++ p_uint = (unsigned int *)p_power; ++ for (i = 0; i < loop; i++) { ++ p_char = strsep(&cookie_pot, "="); ++ ret = kstrtol(p_char, 10, &long_val); ++ if (ret) ++ goto err; ++ offset = long_val; ++ ++ if (i == (loop - 1)) ++ p_char = strsep(&cookie_pot, "\0"); ++ else ++ p_char = strsep(&cookie_pot, ","); ++ ++ ret = kstrtol(p_char, 10, &long_val); ++ if (ret) ++ goto err; ++ value = long_val; ++ ++ if (offset >= ARRAY_SIZE(power_save_msg)) { ++ ret = -EINVAL; ++ goto err; ++ } ++ offset = array_index_nospec(offset, ARRAY_SIZE(power_save_msg)); ++ ++ *(p_uint + offset) = value; ++ } ++ ++ /* update power save setting */ ++ res = sip_smc_dram(SHARE_PAGE_TYPE_DDRDBG, DDRDBG_FUNC_UPDATE_POWERSAVE, ++ ROCKCHIP_SIP_CONFIG_DRAM_DEBUG); ++ if (res.a0) { ++ pr_err("rockchip_sip_config_dram_debug error:%lx\n", res.a0); ++ ret = -ENOMEM; ++ goto err; ++ } ++ ++ ret = count; ++err: ++ vfree(buf); ++ return ret; ++} ++ ++static const struct file_operations powersave_proc_fops = { ++ .open = powersave_proc_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = single_release, ++ .write = powersave_proc_write, ++}; ++ ++static int proc_powersave_init(void) ++{ ++ /* create dmcinfo file */ ++ proc_create(PROC_DMCDBG_POWERSAVE, 0644, proc_dmcdbg_dir, ++ &powersave_proc_fops); ++ ++ return 0; ++} ++ ++static int drvodt_proc_show(struct seq_file *m, void *v) ++{ ++ struct arm_smccc_res res; ++ struct drv_odt_info *p_drvodt; ++ unsigned int *p_uint; ++ unsigned int i; ++ ++ /* get drive strength and odt information */ ++ res = sip_smc_dram(SHARE_PAGE_TYPE_DDRDBG, DDRDBG_FUNC_GET_DRVODT_INFO, ++ ROCKCHIP_SIP_CONFIG_DRAM_DEBUG); ++ if (res.a0) { ++ seq_printf(m, "rockchip_sip_config_dram_debug error:%lx\n", ++ res.a0); ++ return -ENOMEM; ++ } ++ ++ if (!dmcdbg_data.inited_flag) { ++ seq_puts(m, "dmcdbg_data no int\n"); ++ return -EPERM; ++ } ++ p_drvodt = (struct drv_odt_info *)dmcdbg_data.share_memory; ++ ++ seq_printf(m, ++ "drv and odt information:\n" ++ "\n" ++ "[number]name: value (ohm)\n" ++ ); ++ ++ p_uint = (unsigned int *)p_drvodt; ++ for (i = 0; i < ARRAY_SIZE(drv_odt_msg); i++) { ++ if (*(p_uint + (i * 3)) == DRV_ODT_UNKNOWN) ++ seq_printf(m, ++ "[%2d]%s: NULL (unknown) %c\n" ++ , ++ i, drv_odt_msg[i], ++ (*(p_uint + (i * 3) + 2) == ++ DRV_ODT_SUSPEND_FIX) ? '\0' : '*' ++ ); ++ else if (*(p_uint + (i * 3) + 1) == DRV_ODT_UNKNOWN) ++ seq_printf(m, ++ "[%2d]%s: %d (unknown) %c\n" ++ , ++ i, drv_odt_msg[i], *(p_uint + (i * 3)), ++ (*(p_uint + (i * 3) + 2) == ++ DRV_ODT_SUSPEND_FIX) ? '\0' : '*' ++ ); ++ else if (i < (ARRAY_SIZE(drv_odt_msg) - 2)) ++ seq_printf(m, ++ "[%2d]%s: %d (%d ohm) %c\n" ++ , ++ i, drv_odt_msg[i], *(p_uint + (i * 3)), ++ *(p_uint + (i * 3) + 1), ++ (*(p_uint + (i * 3) + 2) == ++ DRV_ODT_SUSPEND_FIX) ? '\0' : '*' ++ ); ++ else ++ seq_printf(m, ++ "[%2d]%s: %d (%d %%) %c\n" ++ , ++ i, drv_odt_msg[i], *(p_uint + (i * 3)), ++ *(p_uint + (i * 3) + 1), ++ (*(p_uint + (i * 3) + 2) == ++ DRV_ODT_SUSPEND_FIX) ? '\0' : '*' ++ ); ++ } ++ ++ seq_printf(m, ++ "\n" ++ "drvodt setting:\n" ++ "echo number=value > /proc/dmcdbg/drvodt\n" ++ "eg: set soc side ca drv up to 20\n" ++ " echo 6=20 > /proc/dmcdbg/drvodt\n" ++ "\n" ++ "Support for setting multiple parameters at the same time.\n" ++ "echo number=value,number=value,... > /proc/dmcdbg/drvodt\n" ++ "eg: set soc side ca drv up and down to 20\n" ++ " echo 6=20,7=20 > /proc/dmcdbg/drvodt\n" ++ "Note: Please update both up and down at the same time.\n" ++ " (*) mean unsupported setting value\n" ++ ); ++ ++ return 0; ++} ++ ++static int drvodt_proc_open(struct inode *inode, struct file *file) ++{ ++ return single_open(file, drvodt_proc_show, NULL); ++} ++ ++static ssize_t drvodt_proc_write(struct file *file, ++ const char __user *buffer, ++ size_t count, loff_t *ppos) ++{ ++ struct arm_smccc_res res; ++ struct drv_odt_info *p_drvodt; ++ unsigned int *p_uint; ++ char *buf, *cookie_pot, *p_char; ++ int ret = 0; ++ u32 loop, i, offset, value; ++ long long_val; ++ ++ /* get buffer data */ ++ buf = vzalloc(count); ++ cookie_pot = buf; ++ if (!cookie_pot) ++ return -ENOMEM; ++ ++ if (copy_from_user(cookie_pot, buffer, count)) { ++ ret = -EFAULT; ++ goto err; ++ } ++ ++ /* get drv and odt setting */ ++ res = sip_smc_dram(SHARE_PAGE_TYPE_DDRDBG, DDRDBG_FUNC_GET_DRVODT_INFO, ++ ROCKCHIP_SIP_CONFIG_DRAM_DEBUG); ++ if (res.a0) { ++ pr_err("rockchip_sip_config_dram_debug error:%lx\n", res.a0); ++ ret = -ENOMEM; ++ goto err; ++ } ++ ++ if (!dmcdbg_data.inited_flag) { ++ pr_err("dmcdbg_data no int\n"); ++ ret = -EPERM; ++ goto err; ++ } ++ p_drvodt = (struct drv_odt_info *)dmcdbg_data.share_memory; ++ ++ loop = 0; ++ for (i = 0; i < count; i++) { ++ if (*(cookie_pot + i) == '=') ++ loop++; ++ } ++ ++ p_uint = (unsigned int *)p_drvodt; ++ for (i = 0; i < loop; i++) { ++ p_char = strsep(&cookie_pot, "="); ++ ret = kstrtol(p_char, 10, &long_val); ++ if (ret) ++ goto err; ++ offset = long_val; ++ ++ if (i == (loop - 1)) ++ p_char = strsep(&cookie_pot, "\0"); ++ else ++ p_char = strsep(&cookie_pot, ","); ++ ++ ret = kstrtol(p_char, 10, &long_val); ++ if (ret) ++ goto err; ++ value = long_val; ++ ++ if (offset >= ARRAY_SIZE(drv_odt_msg)) { ++ ret = -EINVAL; ++ goto err; ++ } ++ offset *= 3; ++ offset = array_index_nospec(offset, ARRAY_SIZE(drv_odt_msg) * 3); ++ ++ *(p_uint + offset) = value; ++ } ++ ++ /* update power save setting */ ++ res = sip_smc_dram(SHARE_PAGE_TYPE_DDRDBG, DDRDBG_FUNC_UPDATE_DRVODT, ++ ROCKCHIP_SIP_CONFIG_DRAM_DEBUG); ++ if (res.a0) { ++ pr_err("rockchip_sip_config_dram_debug error:%lx\n", res.a0); ++ ret = -ENOMEM; ++ goto err; ++ } ++ ++ ret = count; ++err: ++ vfree(buf); ++ return ret; ++} ++ ++static const struct file_operations drvodt_proc_fops = { ++ .open = drvodt_proc_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = single_release, ++ .write = drvodt_proc_write, ++}; ++ ++static int proc_drvodt_init(void) ++{ ++ /* create dmcinfo file */ ++ proc_create(PROC_DMCDBG_DRVODT, 0644, proc_dmcdbg_dir, ++ &drvodt_proc_fops); ++ ++ return 0; ++} ++ ++static int skew_proc_show(struct seq_file *m, void *v) ++{ ++ struct arm_smccc_res res; ++ unsigned int *p_uint; ++ u32 group, i; ++ ++ /* get deskew information */ ++ res = sip_smc_dram(SHARE_PAGE_TYPE_DDRDBG, DDRDBG_FUNC_GET_DESKEW_INFO, ++ ROCKCHIP_SIP_CONFIG_DRAM_DEBUG); ++ if (res.a0) { ++ seq_printf(m, "rockchip_sip_config_dram_debug error:%lx\n", ++ res.a0); ++ return -ENOMEM; ++ } ++ ++ if (!dmcdbg_data.inited_flag) { ++ seq_puts(m, "dmcdbg_data no int\n"); ++ return -EPERM; ++ } ++ ++ seq_printf(m, ++ "de-skew information:\n" ++ "\n" ++ "[group_number]name: value\n" ++ ); ++ ++ for (group = 0; group < dmcdbg_data.skew_group_num; group++) { ++ if (dmcdbg_data.skew_group[group].note != NULL) ++ seq_printf(m, ++ "%s\n" ++ , ++ dmcdbg_data.skew_group[group].note ++ ); ++ p_uint = (unsigned int *)dmcdbg_data.skew_group[group].p_skew_info; ++ for (i = 0; i < dmcdbg_data.skew_group[group].skew_num; i++) ++ seq_printf(m, ++ "[%c%d_%d]%s: %d\n" ++ , ++ (i < 10) ? ' ' : '\0', group, i, ++ dmcdbg_data.skew_group[group].p_skew_timing[i], ++ *(p_uint + i) ++ ); ++ } ++ ++ seq_printf(m, ++ "\n" ++ "de-skew setting:\n" ++ "echo group_number=value > /proc/dmcdbg/deskew\n" ++ "eg: set a1_ddr3a14_de-skew to 8\n" ++ " echo 0_1=8 > /proc/dmcdbg/deskew\n" ++ "\n" ++ "Support for setting multiple parameters simultaneously.\n" ++ "echo group_number=value,group_number=value,... > /proc/dmcdbg/deskew\n" ++ "eg:\n" ++ " echo 0_1=8,1_2=8 > /proc/dmcdbg/deskew\n" ++ ); ++ ++ return 0; ++} ++ ++static int skew_proc_open(struct inode *inode, struct file *file) ++{ ++ return single_open(file, skew_proc_show, NULL); ++} ++ ++static ssize_t skew_proc_write(struct file *file, ++ const char __user *buffer, ++ size_t count, loff_t *ppos) ++{ ++ struct arm_smccc_res res; ++ unsigned int *p_uint; ++ char *buf, *cookie_pot, *p_char; ++ int ret = 0; ++ u32 loop, i, offset_max, group, offset, value; ++ long long_val; ++ ++ /* get buffer data */ ++ buf = vzalloc(count); ++ cookie_pot = buf; ++ if (!cookie_pot) ++ return -ENOMEM; ++ ++ if (copy_from_user(cookie_pot, buffer, count)) { ++ ret = -EFAULT; ++ goto err; ++ } ++ ++ /* get skew setting */ ++ res = sip_smc_dram(SHARE_PAGE_TYPE_DDRDBG, DDRDBG_FUNC_GET_DESKEW_INFO, ++ ROCKCHIP_SIP_CONFIG_DRAM_DEBUG); ++ if (res.a0) { ++ pr_err("rockchip_sip_config_dram_debug error:%lx\n", res.a0); ++ ret = -ENOMEM; ++ goto err; ++ } ++ ++ if (!dmcdbg_data.inited_flag) { ++ pr_err("dmcdbg_data no int\n"); ++ ret = -EPERM; ++ goto err; ++ } ++ ++ loop = 0; ++ for (i = 0; i < count; i++) { ++ if (*(cookie_pot + i) == '=') ++ loop++; ++ } ++ ++ for (i = 0; i < loop; i++) { ++ p_char = strsep(&cookie_pot, "_"); ++ ret = kstrtol(p_char, 10, &long_val); ++ if (ret) ++ goto err; ++ group = long_val; ++ ++ p_char = strsep(&cookie_pot, "="); ++ ret = kstrtol(p_char, 10, &long_val); ++ if (ret) ++ goto err; ++ offset = long_val; ++ ++ if (i == (loop - 1)) ++ p_char = strsep(&cookie_pot, "\0"); ++ else ++ p_char = strsep(&cookie_pot, ","); ++ ++ ret = kstrtol(p_char, 10, &long_val); ++ if (ret) ++ goto err; ++ value = long_val; ++ ++ if (group >= dmcdbg_data.skew_group_num) { ++ ret = -EINVAL; ++ goto err; ++ } ++ group = array_index_nospec(group, dmcdbg_data.skew_group_num); ++ ++ p_uint = (unsigned int *)dmcdbg_data.skew_group[group].p_skew_info; ++ offset_max = dmcdbg_data.skew_group[group].skew_num; ++ ++ if (offset >= offset_max) { ++ ret = -EINVAL; ++ goto err; ++ } ++ offset = array_index_nospec(offset, offset_max); ++ ++ *(p_uint + offset) = value; ++ } ++ ++ /* update power save setting */ ++ res = sip_smc_dram(SHARE_PAGE_TYPE_DDRDBG, DDRDBG_FUNC_UPDATE_DESKEW, ++ ROCKCHIP_SIP_CONFIG_DRAM_DEBUG); ++ if (res.a0) { ++ pr_err("rockchip_sip_config_dram_debug error:%lx\n", res.a0); ++ ret = -ENOMEM; ++ goto err; ++ } ++ ++ ret = count; ++err: ++ vfree(buf); ++ return ret; ++} ++ ++static const struct file_operations skew_proc_fops = { ++ .open = skew_proc_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = single_release, ++ .write = skew_proc_write, ++}; ++ ++static int proc_skew_init(void) ++{ ++ /* create dmcinfo file */ ++ proc_create(PROC_DMCDBG_DESKEW, 0644, proc_dmcdbg_dir, ++ &skew_proc_fops); ++ ++ return 0; ++} ++ ++static int regsinfo_proc_show(struct seq_file *m, void *v) ++{ ++ struct arm_smccc_res res; ++ struct registers_info *p_regsinfo; ++ u32 i; ++ ++ res = sip_smc_dram(SHARE_PAGE_TYPE_DDRDBG, ++ DDRDBG_FUNC_GET_REGISTERS_INFO, ++ ROCKCHIP_SIP_CONFIG_DRAM_DEBUG); ++ if (res.a0) { ++ seq_printf(m, "rockchip_sip_config_dram_debug error:%lx\n", ++ res.a0); ++ return -ENOMEM; ++ } ++ ++ if (!dmcdbg_data.inited_flag) { ++ seq_puts(m, "dmcdbg_data no int\n"); ++ return -EPERM; ++ } ++ p_regsinfo = (struct registers_info *)dmcdbg_data.share_memory; ++ ++ seq_printf(m, ++ "registers base address information:\n" ++ "\n" ++ ); ++ ++ for (i = 0; i < p_regsinfo->regs_num; i++) { ++ seq_printf(m, ++ "%s=0x%x\n" ++ , ++ p_regsinfo->regs[i].regs_name, ++ p_regsinfo->regs[i].regs_addr ++ ); ++ } ++ ++ return 0; ++} ++ ++static int regsinfo_proc_open(struct inode *inode, struct file *file) ++{ ++ return single_open(file, regsinfo_proc_show, NULL); ++} ++ ++static const struct file_operations regsinfo_proc_fops = { ++ .open = regsinfo_proc_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = single_release, ++}; ++ ++static int proc_regsinfo_init(void) ++{ ++ /* create dmcinfo file */ ++ proc_create(PROC_DMCDBG_REGS_INFO, 0644, proc_dmcdbg_dir, ++ ®sinfo_proc_fops); ++ ++ return 0; ++} ++ ++static void rv1126_get_skew_parameter(void) ++{ ++ struct skew_info_rv1126 *p_skew; ++ u32 i; ++ ++ /* get skew parameters */ ++ p_skew = (struct skew_info_rv1126 *)dmcdbg_data.share_memory; ++ dmcdbg_data.skew_group_num = 5; ++ ++ /* ca_skew parameters */ ++ dmcdbg_data.skew_group[0].p_skew_info = (unsigned int *)p_skew->ca_skew; ++ dmcdbg_data.skew_group[0].skew_num = ARRAY_SIZE(rv1126_dts_ca_timing); ++ for (i = 0; i < dmcdbg_data.skew_group[0].skew_num; i++) ++ dmcdbg_data.skew_group[0].p_skew_timing[i] = ++ (char *)rv1126_dts_ca_timing[i]; ++ dmcdbg_data.skew_group[0].note = ++ "(ca_skew: ddr4(pad_name)_ddr3_lpddr3_lpddr4_de-skew)"; ++ ++ /* cs0_a_skew parameters */ ++ dmcdbg_data.skew_group[1].p_skew_info = (unsigned int *)p_skew->cs0_a_skew; ++ dmcdbg_data.skew_group[1].skew_num = ARRAY_SIZE(rv1126_dts_cs0_a_timing); ++ for (i = 0; i < dmcdbg_data.skew_group[1].skew_num; i++) ++ dmcdbg_data.skew_group[1].p_skew_timing[i] = ++ (char *)rv1126_dts_cs0_a_timing[i]; ++ dmcdbg_data.skew_group[1].note = "(cs0_a_skew)"; ++ ++ /* cs0_b_skew parameters */ ++ dmcdbg_data.skew_group[2].p_skew_info = (unsigned int *)p_skew->cs0_b_skew; ++ dmcdbg_data.skew_group[2].skew_num = ARRAY_SIZE(rv1126_dts_cs0_b_timing); ++ for (i = 0; i < dmcdbg_data.skew_group[2].skew_num; i++) ++ dmcdbg_data.skew_group[2].p_skew_timing[i] = ++ (char *)rv1126_dts_cs0_b_timing[i]; ++ dmcdbg_data.skew_group[2].note = "(cs0_b_skew)"; ++ ++ /* cs1_a_skew parameters */ ++ dmcdbg_data.skew_group[3].p_skew_info = (unsigned int *)p_skew->cs1_a_skew; ++ dmcdbg_data.skew_group[3].skew_num = ARRAY_SIZE(rv1126_dts_cs1_a_timing); ++ for (i = 0; i < dmcdbg_data.skew_group[3].skew_num; i++) ++ dmcdbg_data.skew_group[3].p_skew_timing[i] = ++ (char *)rv1126_dts_cs1_a_timing[i]; ++ dmcdbg_data.skew_group[3].note = "(cs1_a_skew)"; ++ ++ /* cs1_b_skew parameters */ ++ dmcdbg_data.skew_group[4].p_skew_info = (unsigned int *)p_skew->cs1_b_skew; ++ dmcdbg_data.skew_group[4].skew_num = ARRAY_SIZE(rv1126_dts_cs1_b_timing); ++ for (i = 0; i < dmcdbg_data.skew_group[3].skew_num; i++) ++ dmcdbg_data.skew_group[4].p_skew_timing[i] = ++ (char *)rv1126_dts_cs1_b_timing[i]; ++ dmcdbg_data.skew_group[4].note = "(cs1_b_skew)"; ++} ++ ++static __maybe_unused int rv1126_dmcdbg_init(struct platform_device *pdev, ++ struct rockchip_dmcdbg *dmcdbg) ++{ ++ struct arm_smccc_res res; ++ ++ /* check ddr_debug_func version */ ++ res = sip_smc_dram(0, DDRDBG_FUNC_GET_VERSION, ++ ROCKCHIP_SIP_CONFIG_DRAM_DEBUG); ++ dev_notice(&pdev->dev, "current ATF ddr_debug_func version 0x%lx.\n", ++ res.a1); ++ /* ++ * [15:8] major version, [7:0] minor version ++ * major version must match both kernel dmcdbg and ATF ddr_debug_func. ++ */ ++ if (res.a0 || res.a1 < 0x101 || ((res.a1 & 0xff00) != 0x100)) { ++ dev_err(&pdev->dev, ++ "version invalid,need update,the major version unmatch!\n"); ++ return -ENXIO; ++ } ++ ++ /* request share memory for pass parameter */ ++ res = sip_smc_request_share_mem(DMCDBG_PAGE_NUMS, ++ SHARE_PAGE_TYPE_DDRDBG); ++ if (res.a0 != 0) { ++ dev_err(&pdev->dev, "request share mem error\n"); ++ return -ENOMEM; ++ } ++ ++ dmcdbg_data.share_memory = (void __iomem *)res.a1; ++ dmcdbg_data.inited_flag = 1; ++ ++ rv1126_get_skew_parameter(); ++ ++ /* create parent dir in /proc */ ++ proc_dmcdbg_dir = proc_mkdir(PROC_DMCDBG_DIR_NAME, NULL); ++ if (!proc_dmcdbg_dir) { ++ dev_err(&pdev->dev, "create proc dir error!"); ++ return -ENOENT; ++ } ++ ++ proc_dmcinfo_init(); ++ proc_powersave_init(); ++ proc_drvodt_init(); ++ proc_skew_init(); ++ proc_regsinfo_init(); ++ return 0; ++} ++ ++static const struct of_device_id rockchip_dmcdbg_of_match[] = { ++ { .compatible = "rockchip,rv1126-dmcdbg", .data = rv1126_dmcdbg_init}, ++ { }, ++}; ++MODULE_DEVICE_TABLE(of, rockchip_dmcdbg_of_match); ++ ++static int rockchip_dmcdbg_probe(struct platform_device *pdev) ++{ ++ struct device *dev = &pdev->dev; ++ struct rockchip_dmcdbg *data; ++ const struct of_device_id *match; ++ int (*init)(struct platform_device *pdev, ++ struct rockchip_dmcdbg *data); ++ int ret = 0; ++ ++ data = devm_kzalloc(dev, sizeof(struct rockchip_dmcdbg), GFP_KERNEL); ++ if (!data) ++ return -ENOMEM; ++ ++ data->dev = dev; ++ ++ /* match soc chip init */ ++ match = of_match_node(rockchip_dmcdbg_of_match, pdev->dev.of_node); ++ if (match) { ++ init = match->data; ++ if (init) { ++ if (init(pdev, data)) ++ return -EINVAL; ++ } ++ } ++ ++ return ret; ++} ++ ++static struct platform_driver rockchip_dmcdbg_driver = { ++ .probe = rockchip_dmcdbg_probe, ++ .driver = { ++ .name = "rockchip,dmcdbg", ++ .of_match_table = rockchip_dmcdbg_of_match, ++ }, ++}; ++module_platform_driver(rockchip_dmcdbg_driver); ++ ++MODULE_LICENSE("GPL v2"); ++MODULE_AUTHOR("YouMin Chen "); ++MODULE_DESCRIPTION("rockchip dmc debug driver with devfreq framework"); +diff --git a/drivers/devfreq/rockchip_dmc_timing.h b/drivers/devfreq/rockchip_dmc_timing.h +new file mode 100755 +index 000000000000..8f2e2c02bb90 +--- /dev/null ++++ b/drivers/devfreq/rockchip_dmc_timing.h +@@ -0,0 +1,1307 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* ++ * Copyright (c) 2020, Rockchip Electronics Co., Ltd. ++ */ ++ ++#ifndef __ROCKCHIP_DMC_TIMING_H__ ++#define __ROCKCHIP_DMC_TIMING_H__ ++ ++/* hope this define can adapt all future platfor */ ++static const char * const px30_dts_timing[] = { ++ "ddr2_speed_bin", ++ "ddr3_speed_bin", ++ "ddr4_speed_bin", ++ "pd_idle", ++ "sr_idle", ++ "sr_mc_gate_idle", ++ "srpd_lite_idle", ++ "standby_idle", ++ ++ "auto_pd_dis_freq", ++ "auto_sr_dis_freq", ++ "ddr2_dll_dis_freq", ++ "ddr3_dll_dis_freq", ++ "ddr4_dll_dis_freq", ++ "phy_dll_dis_freq", ++ ++ "ddr2_odt_dis_freq", ++ "phy_ddr2_odt_dis_freq", ++ "ddr2_drv", ++ "ddr2_odt", ++ "phy_ddr2_ca_drv", ++ "phy_ddr2_ck_drv", ++ "phy_ddr2_dq_drv", ++ "phy_ddr2_odt", ++ ++ "ddr3_odt_dis_freq", ++ "phy_ddr3_odt_dis_freq", ++ "ddr3_drv", ++ "ddr3_odt", ++ "phy_ddr3_ca_drv", ++ "phy_ddr3_ck_drv", ++ "phy_ddr3_dq_drv", ++ "phy_ddr3_odt", ++ ++ "phy_lpddr2_odt_dis_freq", ++ "lpddr2_drv", ++ "phy_lpddr2_ca_drv", ++ "phy_lpddr2_ck_drv", ++ "phy_lpddr2_dq_drv", ++ "phy_lpddr2_odt", ++ ++ "lpddr3_odt_dis_freq", ++ "phy_lpddr3_odt_dis_freq", ++ "lpddr3_drv", ++ "lpddr3_odt", ++ "phy_lpddr3_ca_drv", ++ "phy_lpddr3_ck_drv", ++ "phy_lpddr3_dq_drv", ++ "phy_lpddr3_odt", ++ ++ "lpddr4_odt_dis_freq", ++ "phy_lpddr4_odt_dis_freq", ++ "lpddr4_drv", ++ "lpddr4_dq_odt", ++ "lpddr4_ca_odt", ++ "phy_lpddr4_ca_drv", ++ "phy_lpddr4_ck_cs_drv", ++ "phy_lpddr4_dq_drv", ++ "phy_lpddr4_odt", ++ ++ "ddr4_odt_dis_freq", ++ "phy_ddr4_odt_dis_freq", ++ "ddr4_drv", ++ "ddr4_odt", ++ "phy_ddr4_ca_drv", ++ "phy_ddr4_ck_drv", ++ "phy_ddr4_dq_drv", ++ "phy_ddr4_odt", ++}; ++ ++struct px30_ddr_dts_config_timing { ++ unsigned int ddr2_speed_bin; ++ unsigned int ddr3_speed_bin; ++ unsigned int ddr4_speed_bin; ++ unsigned int pd_idle; ++ unsigned int sr_idle; ++ unsigned int sr_mc_gate_idle; ++ unsigned int srpd_lite_idle; ++ unsigned int standby_idle; ++ ++ unsigned int auto_pd_dis_freq; ++ unsigned int auto_sr_dis_freq; ++ /* for ddr2 only */ ++ unsigned int ddr2_dll_dis_freq; ++ /* for ddr3 only */ ++ unsigned int ddr3_dll_dis_freq; ++ /* for ddr4 only */ ++ unsigned int ddr4_dll_dis_freq; ++ unsigned int phy_dll_dis_freq; ++ ++ unsigned int ddr2_odt_dis_freq; ++ unsigned int phy_ddr2_odt_dis_freq; ++ unsigned int ddr2_drv; ++ unsigned int ddr2_odt; ++ unsigned int phy_ddr2_ca_drv; ++ unsigned int phy_ddr2_ck_drv; ++ unsigned int phy_ddr2_dq_drv; ++ unsigned int phy_ddr2_odt; ++ ++ unsigned int ddr3_odt_dis_freq; ++ unsigned int phy_ddr3_odt_dis_freq; ++ unsigned int ddr3_drv; ++ unsigned int ddr3_odt; ++ unsigned int phy_ddr3_ca_drv; ++ unsigned int phy_ddr3_ck_drv; ++ unsigned int phy_ddr3_dq_drv; ++ unsigned int phy_ddr3_odt; ++ ++ unsigned int phy_lpddr2_odt_dis_freq; ++ unsigned int lpddr2_drv; ++ unsigned int phy_lpddr2_ca_drv; ++ unsigned int phy_lpddr2_ck_drv; ++ unsigned int phy_lpddr2_dq_drv; ++ unsigned int phy_lpddr2_odt; ++ ++ unsigned int lpddr3_odt_dis_freq; ++ unsigned int phy_lpddr3_odt_dis_freq; ++ unsigned int lpddr3_drv; ++ unsigned int lpddr3_odt; ++ unsigned int phy_lpddr3_ca_drv; ++ unsigned int phy_lpddr3_ck_drv; ++ unsigned int phy_lpddr3_dq_drv; ++ unsigned int phy_lpddr3_odt; ++ ++ unsigned int lpddr4_odt_dis_freq; ++ unsigned int phy_lpddr4_odt_dis_freq; ++ unsigned int lpddr4_drv; ++ unsigned int lpddr4_dq_odt; ++ unsigned int lpddr4_ca_odt; ++ unsigned int phy_lpddr4_ca_drv; ++ unsigned int phy_lpddr4_ck_cs_drv; ++ unsigned int phy_lpddr4_dq_drv; ++ unsigned int phy_lpddr4_odt; ++ ++ unsigned int ddr4_odt_dis_freq; ++ unsigned int phy_ddr4_odt_dis_freq; ++ unsigned int ddr4_drv; ++ unsigned int ddr4_odt; ++ unsigned int phy_ddr4_ca_drv; ++ unsigned int phy_ddr4_ck_drv; ++ unsigned int phy_ddr4_dq_drv; ++ unsigned int phy_ddr4_odt; ++ ++ unsigned int ca_skew[15]; ++ unsigned int cs0_skew[44]; ++ unsigned int cs1_skew[44]; ++ ++ unsigned int available; ++}; ++ ++static const char * const rk1808_dts_ca_timing[] = { ++ "a0_ddr3a9_de-skew", ++ "a1_ddr3a14_de-skew", ++ "a2_ddr3a13_de-skew", ++ "a3_ddr3a11_de-skew", ++ "a4_ddr3a2_de-skew", ++ "a5_ddr3a4_de-skew", ++ "a6_ddr3a3_de-skew", ++ "a7_ddr3a6_de-skew", ++ "a8_ddr3a5_de-skew", ++ "a9_ddr3a1_de-skew", ++ "a10_ddr3a0_de-skew", ++ "a11_ddr3a7_de-skew", ++ "a12_ddr3casb_de-skew", ++ "a13_ddr3a8_de-skew", ++ "a14_ddr3odt0_de-skew", ++ "a15_ddr3ba1_de-skew", ++ "a16_ddr3rasb_de-skew", ++ "a17_ddr3null_de-skew", ++ "ba0_ddr3ba2_de-skew", ++ "ba1_ddr3a12_de-skew", ++ "bg0_ddr3ba0_de-skew", ++ "bg1_ddr3web_de-skew", ++ "cke_ddr3cke_de-skew", ++ "ck_ddr3ck_de-skew", ++ "ckb_ddr3ckb_de-skew", ++ "csb0_ddr3a10_de-skew", ++ "odt0_ddr3a15_de-skew", ++ "resetn_ddr3resetn_de-skew", ++ "actn_ddr3csb0_de-skew", ++ "csb1_ddr3csb1_de-skew", ++ "odt1_ddr3odt1_de-skew", ++}; ++ ++static const char * const rk1808_dts_cs0_a_timing[] = { ++ "cs0_dm0_rx_de-skew", ++ "cs0_dm0_tx_de-skew", ++ "cs0_dq0_rx_de-skew", ++ "cs0_dq0_tx_de-skew", ++ "cs0_dq1_rx_de-skew", ++ "cs0_dq1_tx_de-skew", ++ "cs0_dq2_rx_de-skew", ++ "cs0_dq2_tx_de-skew", ++ "cs0_dq3_rx_de-skew", ++ "cs0_dq3_tx_de-skew", ++ "cs0_dq4_rx_de-skew", ++ "cs0_dq4_tx_de-skew", ++ "cs0_dq5_rx_de-skew", ++ "cs0_dq5_tx_de-skew", ++ "cs0_dq6_rx_de-skew", ++ "cs0_dq6_tx_de-skew", ++ "cs0_dq7_rx_de-skew", ++ "cs0_dq7_tx_de-skew", ++ "cs0_dqs0p_rx_de-skew", ++ "cs0_dqs0p_tx_de-skew", ++ "cs0_dqs0n_tx_de-skew", ++ "cs0_dm1_rx_de-skew", ++ "cs0_dm1_tx_de-skew", ++ "cs0_dq8_rx_de-skew", ++ "cs0_dq8_tx_de-skew", ++ "cs0_dq9_rx_de-skew", ++ "cs0_dq9_tx_de-skew", ++ "cs0_dq10_rx_de-skew", ++ "cs0_dq10_tx_de-skew", ++ "cs0_dq11_rx_de-skew", ++ "cs0_dq11_tx_de-skew", ++ "cs0_dq12_rx_de-skew", ++ "cs0_dq12_tx_de-skew", ++ "cs0_dq13_rx_de-skew", ++ "cs0_dq13_tx_de-skew", ++ "cs0_dq14_rx_de-skew", ++ "cs0_dq14_tx_de-skew", ++ "cs0_dq15_rx_de-skew", ++ "cs0_dq15_tx_de-skew", ++ "cs0_dqs1p_rx_de-skew", ++ "cs0_dqs1p_tx_de-skew", ++ "cs0_dqs1n_tx_de-skew", ++ "cs0_dqs0n_rx_de-skew", ++ "cs0_dqs1n_rx_de-skew", ++}; ++ ++static const char * const rk1808_dts_cs0_b_timing[] = { ++ "cs0_dm2_rx_de-skew", ++ "cs0_dm2_tx_de-skew", ++ "cs0_dq16_rx_de-skew", ++ "cs0_dq16_tx_de-skew", ++ "cs0_dq17_rx_de-skew", ++ "cs0_dq17_tx_de-skew", ++ "cs0_dq18_rx_de-skew", ++ "cs0_dq18_tx_de-skew", ++ "cs0_dq19_rx_de-skew", ++ "cs0_dq19_tx_de-skew", ++ "cs0_dq20_rx_de-skew", ++ "cs0_dq20_tx_de-skew", ++ "cs0_dq21_rx_de-skew", ++ "cs0_dq21_tx_de-skew", ++ "cs0_dq22_rx_de-skew", ++ "cs0_dq22_tx_de-skew", ++ "cs0_dq23_rx_de-skew", ++ "cs0_dq23_tx_de-skew", ++ "cs0_dqs2p_rx_de-skew", ++ "cs0_dqs2p_tx_de-skew", ++ "cs0_dqs2n_tx_de-skew", ++ "cs0_dm3_rx_de-skew", ++ "cs0_dm3_tx_de-skew", ++ "cs0_dq24_rx_de-skew", ++ "cs0_dq24_tx_de-skew", ++ "cs0_dq25_rx_de-skew", ++ "cs0_dq25_tx_de-skew", ++ "cs0_dq26_rx_de-skew", ++ "cs0_dq26_tx_de-skew", ++ "cs0_dq27_rx_de-skew", ++ "cs0_dq27_tx_de-skew", ++ "cs0_dq28_rx_de-skew", ++ "cs0_dq28_tx_de-skew", ++ "cs0_dq29_rx_de-skew", ++ "cs0_dq29_tx_de-skew", ++ "cs0_dq30_rx_de-skew", ++ "cs0_dq30_tx_de-skew", ++ "cs0_dq31_rx_de-skew", ++ "cs0_dq31_tx_de-skew", ++ "cs0_dqs3p_rx_de-skew", ++ "cs0_dqs3p_tx_de-skew", ++ "cs0_dqs3n_tx_de-skew", ++ "cs0_dqs2n_rx_de-skew", ++ "cs0_dqs3n_rx_de-skew", ++}; ++ ++static const char * const rk1808_dts_cs1_a_timing[] = { ++ "cs1_dm0_rx_de-skew", ++ "cs1_dm0_tx_de-skew", ++ "cs1_dq0_rx_de-skew", ++ "cs1_dq0_tx_de-skew", ++ "cs1_dq1_rx_de-skew", ++ "cs1_dq1_tx_de-skew", ++ "cs1_dq2_rx_de-skew", ++ "cs1_dq2_tx_de-skew", ++ "cs1_dq3_rx_de-skew", ++ "cs1_dq3_tx_de-skew", ++ "cs1_dq4_rx_de-skew", ++ "cs1_dq4_tx_de-skew", ++ "cs1_dq5_rx_de-skew", ++ "cs1_dq5_tx_de-skew", ++ "cs1_dq6_rx_de-skew", ++ "cs1_dq6_tx_de-skew", ++ "cs1_dq7_rx_de-skew", ++ "cs1_dq7_tx_de-skew", ++ "cs1_dqs0p_rx_de-skew", ++ "cs1_dqs0p_tx_de-skew", ++ "cs1_dqs0n_tx_de-skew", ++ "cs1_dm1_rx_de-skew", ++ "cs1_dm1_tx_de-skew", ++ "cs1_dq8_rx_de-skew", ++ "cs1_dq8_tx_de-skew", ++ "cs1_dq9_rx_de-skew", ++ "cs1_dq9_tx_de-skew", ++ "cs1_dq10_rx_de-skew", ++ "cs1_dq10_tx_de-skew", ++ "cs1_dq11_rx_de-skew", ++ "cs1_dq11_tx_de-skew", ++ "cs1_dq12_rx_de-skew", ++ "cs1_dq12_tx_de-skew", ++ "cs1_dq13_rx_de-skew", ++ "cs1_dq13_tx_de-skew", ++ "cs1_dq14_rx_de-skew", ++ "cs1_dq14_tx_de-skew", ++ "cs1_dq15_rx_de-skew", ++ "cs1_dq15_tx_de-skew", ++ "cs1_dqs1p_rx_de-skew", ++ "cs1_dqs1p_tx_de-skew", ++ "cs1_dqs1n_tx_de-skew", ++ "cs1_dqs0n_rx_de-skew", ++ "cs1_dqs1n_rx_de-skew", ++}; ++ ++static const char * const rk1808_dts_cs1_b_timing[] = { ++ "cs1_dm2_rx_de-skew", ++ "cs1_dm2_tx_de-skew", ++ "cs1_dq16_rx_de-skew", ++ "cs1_dq16_tx_de-skew", ++ "cs1_dq17_rx_de-skew", ++ "cs1_dq17_tx_de-skew", ++ "cs1_dq18_rx_de-skew", ++ "cs1_dq18_tx_de-skew", ++ "cs1_dq19_rx_de-skew", ++ "cs1_dq19_tx_de-skew", ++ "cs1_dq20_rx_de-skew", ++ "cs1_dq20_tx_de-skew", ++ "cs1_dq21_rx_de-skew", ++ "cs1_dq21_tx_de-skew", ++ "cs1_dq22_rx_de-skew", ++ "cs1_dq22_tx_de-skew", ++ "cs1_dq23_rx_de-skew", ++ "cs1_dq23_tx_de-skew", ++ "cs1_dqs2p_rx_de-skew", ++ "cs1_dqs2p_tx_de-skew", ++ "cs1_dqs2n_tx_de-skew", ++ "cs1_dm3_rx_de-skew", ++ "cs1_dm3_tx_de-skew", ++ "cs1_dq24_rx_de-skew", ++ "cs1_dq24_tx_de-skew", ++ "cs1_dq25_rx_de-skew", ++ "cs1_dq25_tx_de-skew", ++ "cs1_dq26_rx_de-skew", ++ "cs1_dq26_tx_de-skew", ++ "cs1_dq27_rx_de-skew", ++ "cs1_dq27_tx_de-skew", ++ "cs1_dq28_rx_de-skew", ++ "cs1_dq28_tx_de-skew", ++ "cs1_dq29_rx_de-skew", ++ "cs1_dq29_tx_de-skew", ++ "cs1_dq30_rx_de-skew", ++ "cs1_dq30_tx_de-skew", ++ "cs1_dq31_rx_de-skew", ++ "cs1_dq31_tx_de-skew", ++ "cs1_dqs3p_rx_de-skew", ++ "cs1_dqs3p_tx_de-skew", ++ "cs1_dqs3n_tx_de-skew", ++ "cs1_dqs2n_rx_de-skew", ++ "cs1_dqs3n_rx_de-skew", ++}; ++ ++struct rk1808_ddr_dts_config_timing { ++ unsigned int ddr2_speed_bin; ++ unsigned int ddr3_speed_bin; ++ unsigned int ddr4_speed_bin; ++ unsigned int pd_idle; ++ unsigned int sr_idle; ++ unsigned int sr_mc_gate_idle; ++ unsigned int srpd_lite_idle; ++ unsigned int standby_idle; ++ ++ unsigned int auto_pd_dis_freq; ++ unsigned int auto_sr_dis_freq; ++ /* for ddr2 only */ ++ unsigned int ddr2_dll_dis_freq; ++ /* for ddr3 only */ ++ unsigned int ddr3_dll_dis_freq; ++ /* for ddr4 only */ ++ unsigned int ddr4_dll_dis_freq; ++ unsigned int phy_dll_dis_freq; ++ ++ unsigned int ddr2_odt_dis_freq; ++ unsigned int phy_ddr2_odt_dis_freq; ++ unsigned int ddr2_drv; ++ unsigned int ddr2_odt; ++ unsigned int phy_ddr2_ca_drv; ++ unsigned int phy_ddr2_ck_drv; ++ unsigned int phy_ddr2_dq_drv; ++ unsigned int phy_ddr2_odt; ++ ++ unsigned int ddr3_odt_dis_freq; ++ unsigned int phy_ddr3_odt_dis_freq; ++ unsigned int ddr3_drv; ++ unsigned int ddr3_odt; ++ unsigned int phy_ddr3_ca_drv; ++ unsigned int phy_ddr3_ck_drv; ++ unsigned int phy_ddr3_dq_drv; ++ unsigned int phy_ddr3_odt; ++ ++ unsigned int phy_lpddr2_odt_dis_freq; ++ unsigned int lpddr2_drv; ++ unsigned int phy_lpddr2_ca_drv; ++ unsigned int phy_lpddr2_ck_drv; ++ unsigned int phy_lpddr2_dq_drv; ++ unsigned int phy_lpddr2_odt; ++ ++ unsigned int lpddr3_odt_dis_freq; ++ unsigned int phy_lpddr3_odt_dis_freq; ++ unsigned int lpddr3_drv; ++ unsigned int lpddr3_odt; ++ unsigned int phy_lpddr3_ca_drv; ++ unsigned int phy_lpddr3_ck_drv; ++ unsigned int phy_lpddr3_dq_drv; ++ unsigned int phy_lpddr3_odt; ++ ++ unsigned int lpddr4_odt_dis_freq; ++ unsigned int phy_lpddr4_odt_dis_freq; ++ unsigned int lpddr4_drv; ++ unsigned int lpddr4_dq_odt; ++ unsigned int lpddr4_ca_odt; ++ unsigned int phy_lpddr4_ca_drv; ++ unsigned int phy_lpddr4_ck_cs_drv; ++ unsigned int phy_lpddr4_dq_drv; ++ unsigned int phy_lpddr4_odt; ++ ++ unsigned int ddr4_odt_dis_freq; ++ unsigned int phy_ddr4_odt_dis_freq; ++ unsigned int ddr4_drv; ++ unsigned int ddr4_odt; ++ unsigned int phy_ddr4_ca_drv; ++ unsigned int phy_ddr4_ck_drv; ++ unsigned int phy_ddr4_dq_drv; ++ unsigned int phy_ddr4_odt; ++ ++ unsigned int ca_de_skew[31]; ++ unsigned int cs0_a_de_skew[44]; ++ unsigned int cs0_b_de_skew[44]; ++ unsigned int cs1_a_de_skew[44]; ++ unsigned int cs1_b_de_skew[44]; ++ ++ unsigned int available; ++}; ++ ++static const char * const rk3128_dts_timing[] = { ++ "ddr3_speed_bin", ++ "pd_idle", ++ "sr_idle", ++ "auto_pd_dis_freq", ++ "auto_sr_dis_freq", ++ "ddr3_dll_dis_freq", ++ "lpddr2_dll_dis_freq", ++ "phy_dll_dis_freq", ++ "ddr3_odt_dis_freq", ++ "phy_ddr3_odt_disb_freq", ++ "ddr3_drv", ++ "ddr3_odt", ++ "phy_ddr3_clk_drv", ++ "phy_ddr3_cmd_drv", ++ "phy_ddr3_dqs_drv", ++ "phy_ddr3_odt", ++ "lpddr2_drv", ++ "phy_lpddr2_clk_drv", ++ "phy_lpddr2_cmd_drv", ++ "phy_lpddr2_dqs_drv", ++ "ddr_2t", ++}; ++ ++struct rk3128_ddr_dts_config_timing { ++ u32 ddr3_speed_bin; ++ u32 pd_idle; ++ u32 sr_idle; ++ u32 auto_pd_dis_freq; ++ u32 auto_sr_dis_freq; ++ u32 ddr3_dll_dis_freq; ++ u32 lpddr2_dll_dis_freq; ++ u32 phy_dll_dis_freq; ++ u32 ddr3_odt_dis_freq; ++ u32 phy_ddr3_odt_disb_freq; ++ u32 ddr3_drv; ++ u32 ddr3_odt; ++ u32 phy_ddr3_clk_drv; ++ u32 phy_ddr3_cmd_drv; ++ u32 phy_ddr3_dqs_drv; ++ u32 phy_ddr3_odt; ++ u32 lpddr2_drv; ++ u32 phy_lpddr2_clk_drv; ++ u32 phy_lpddr2_cmd_drv; ++ u32 phy_lpddr2_dqs_drv; ++ u32 ddr_2t; ++ u32 available; ++}; ++ ++static const char * const rk3228_dts_timing[] = { ++ "dram_spd_bin", ++ "sr_idle", ++ "pd_idle", ++ "dram_dll_disb_freq", ++ "phy_dll_disb_freq", ++ "dram_odt_disb_freq", ++ "phy_odt_disb_freq", ++ "ddr3_drv", ++ "ddr3_odt", ++ "lpddr3_drv", ++ "lpddr3_odt", ++ "lpddr2_drv", ++ "phy_ddr3_clk_drv", ++ "phy_ddr3_cmd_drv", ++ "phy_ddr3_dqs_drv", ++ "phy_ddr3_odt", ++ "phy_lp23_clk_drv", ++ "phy_lp23_cmd_drv", ++ "phy_lp23_dqs_drv", ++ "phy_lp3_odt" ++}; ++ ++struct rk3228_ddr_dts_config_timing { ++ u32 dram_spd_bin; ++ u32 sr_idle; ++ u32 pd_idle; ++ u32 dram_dll_dis_freq; ++ u32 phy_dll_dis_freq; ++ u32 dram_odt_dis_freq; ++ u32 phy_odt_dis_freq; ++ u32 ddr3_drv; ++ u32 ddr3_odt; ++ u32 lpddr3_drv; ++ u32 lpddr3_odt; ++ u32 lpddr2_drv; ++ u32 phy_ddr3_clk_drv; ++ u32 phy_ddr3_cmd_drv; ++ u32 phy_ddr3_dqs_drv; ++ u32 phy_ddr3_odt; ++ u32 phy_lp23_clk_drv; ++ u32 phy_lp23_cmd_drv; ++ u32 phy_lp23_dqs_drv; ++ u32 phy_lp3_odt; ++}; ++ ++static const char * const rk3288_dts_timing[] = { ++ "ddr3_speed_bin", ++ "pd_idle", ++ "sr_idle", ++ ++ "auto_pd_dis_freq", ++ "auto_sr_dis_freq", ++ /* for ddr3 only */ ++ "ddr3_dll_dis_freq", ++ "phy_dll_dis_freq", ++ ++ "ddr3_odt_dis_freq", ++ "phy_ddr3_odt_dis_freq", ++ "ddr3_drv", ++ "ddr3_odt", ++ "phy_ddr3_drv", ++ "phy_ddr3_odt", ++ ++ "lpddr2_drv", ++ "phy_lpddr2_drv", ++ ++ "lpddr3_odt_dis_freq", ++ "phy_lpddr3_odt_dis_freq", ++ "lpddr3_drv", ++ "lpddr3_odt", ++ "phy_lpddr3_drv", ++ "phy_lpddr3_odt" ++}; ++ ++struct rk3288_ddr_dts_config_timing { ++ unsigned int ddr3_speed_bin; ++ unsigned int pd_idle; ++ unsigned int sr_idle; ++ ++ unsigned int auto_pd_dis_freq; ++ unsigned int auto_sr_dis_freq; ++ /* for ddr3 only */ ++ unsigned int ddr3_dll_dis_freq; ++ unsigned int phy_dll_dis_freq; ++ ++ unsigned int ddr3_odt_dis_freq; ++ unsigned int phy_ddr3_odt_dis_freq; ++ unsigned int ddr3_drv; ++ unsigned int ddr3_odt; ++ unsigned int phy_ddr3_drv; ++ unsigned int phy_ddr3_odt; ++ ++ unsigned int lpddr2_drv; ++ unsigned int phy_lpddr2_drv; ++ ++ unsigned int lpddr3_odt_dis_freq; ++ unsigned int phy_lpddr3_odt_dis_freq; ++ unsigned int lpddr3_drv; ++ unsigned int lpddr3_odt; ++ unsigned int phy_lpddr3_drv; ++ unsigned int phy_lpddr3_odt; ++ ++ unsigned int available; ++}; ++ ++/* hope this define can adapt all future platfor */ ++static const char * const rk3328_dts_timing[] = { ++ "ddr3_speed_bin", ++ "ddr4_speed_bin", ++ "pd_idle", ++ "sr_idle", ++ "sr_mc_gate_idle", ++ "srpd_lite_idle", ++ "standby_idle", ++ ++ "auto_pd_dis_freq", ++ "auto_sr_dis_freq", ++ "ddr3_dll_dis_freq", ++ "ddr4_dll_dis_freq", ++ "phy_dll_dis_freq", ++ ++ "ddr3_odt_dis_freq", ++ "phy_ddr3_odt_dis_freq", ++ "ddr3_drv", ++ "ddr3_odt", ++ "phy_ddr3_ca_drv", ++ "phy_ddr3_ck_drv", ++ "phy_ddr3_dq_drv", ++ "phy_ddr3_odt", ++ ++ "lpddr3_odt_dis_freq", ++ "phy_lpddr3_odt_dis_freq", ++ "lpddr3_drv", ++ "lpddr3_odt", ++ "phy_lpddr3_ca_drv", ++ "phy_lpddr3_ck_drv", ++ "phy_lpddr3_dq_drv", ++ "phy_lpddr3_odt", ++ ++ "lpddr4_odt_dis_freq", ++ "phy_lpddr4_odt_dis_freq", ++ "lpddr4_drv", ++ "lpddr4_dq_odt", ++ "lpddr4_ca_odt", ++ "phy_lpddr4_ca_drv", ++ "phy_lpddr4_ck_cs_drv", ++ "phy_lpddr4_dq_drv", ++ "phy_lpddr4_odt", ++ ++ "ddr4_odt_dis_freq", ++ "phy_ddr4_odt_dis_freq", ++ "ddr4_drv", ++ "ddr4_odt", ++ "phy_ddr4_ca_drv", ++ "phy_ddr4_ck_drv", ++ "phy_ddr4_dq_drv", ++ "phy_ddr4_odt", ++}; ++ ++static const char * const rk3328_dts_ca_timing[] = { ++ "ddr3a1_ddr4a9_de-skew", ++ "ddr3a0_ddr4a10_de-skew", ++ "ddr3a3_ddr4a6_de-skew", ++ "ddr3a2_ddr4a4_de-skew", ++ "ddr3a5_ddr4a8_de-skew", ++ "ddr3a4_ddr4a5_de-skew", ++ "ddr3a7_ddr4a11_de-skew", ++ "ddr3a6_ddr4a7_de-skew", ++ "ddr3a9_ddr4a0_de-skew", ++ "ddr3a8_ddr4a13_de-skew", ++ "ddr3a11_ddr4a3_de-skew", ++ "ddr3a10_ddr4cs0_de-skew", ++ "ddr3a13_ddr4a2_de-skew", ++ "ddr3a12_ddr4ba1_de-skew", ++ "ddr3a15_ddr4odt0_de-skew", ++ "ddr3a14_ddr4a1_de-skew", ++ "ddr3ba1_ddr4a15_de-skew", ++ "ddr3ba0_ddr4bg0_de-skew", ++ "ddr3ras_ddr4cke_de-skew", ++ "ddr3ba2_ddr4ba0_de-skew", ++ "ddr3we_ddr4bg1_de-skew", ++ "ddr3cas_ddr4a12_de-skew", ++ "ddr3ckn_ddr4ckn_de-skew", ++ "ddr3ckp_ddr4ckp_de-skew", ++ "ddr3cke_ddr4a16_de-skew", ++ "ddr3odt0_ddr4a14_de-skew", ++ "ddr3cs0_ddr4act_de-skew", ++ "ddr3reset_ddr4reset_de-skew", ++ "ddr3cs1_ddr4cs1_de-skew", ++ "ddr3odt1_ddr4odt1_de-skew", ++}; ++ ++static const char * const rk3328_dts_cs0_timing[] = { ++ "cs0_dm0_rx_de-skew", ++ "cs0_dm0_tx_de-skew", ++ "cs0_dq0_rx_de-skew", ++ "cs0_dq0_tx_de-skew", ++ "cs0_dq1_rx_de-skew", ++ "cs0_dq1_tx_de-skew", ++ "cs0_dq2_rx_de-skew", ++ "cs0_dq2_tx_de-skew", ++ "cs0_dq3_rx_de-skew", ++ "cs0_dq3_tx_de-skew", ++ "cs0_dq4_rx_de-skew", ++ "cs0_dq4_tx_de-skew", ++ "cs0_dq5_rx_de-skew", ++ "cs0_dq5_tx_de-skew", ++ "cs0_dq6_rx_de-skew", ++ "cs0_dq6_tx_de-skew", ++ "cs0_dq7_rx_de-skew", ++ "cs0_dq7_tx_de-skew", ++ "cs0_dqs0_rx_de-skew", ++ "cs0_dqs0p_tx_de-skew", ++ "cs0_dqs0n_tx_de-skew", ++ ++ "cs0_dm1_rx_de-skew", ++ "cs0_dm1_tx_de-skew", ++ "cs0_dq8_rx_de-skew", ++ "cs0_dq8_tx_de-skew", ++ "cs0_dq9_rx_de-skew", ++ "cs0_dq9_tx_de-skew", ++ "cs0_dq10_rx_de-skew", ++ "cs0_dq10_tx_de-skew", ++ "cs0_dq11_rx_de-skew", ++ "cs0_dq11_tx_de-skew", ++ "cs0_dq12_rx_de-skew", ++ "cs0_dq12_tx_de-skew", ++ "cs0_dq13_rx_de-skew", ++ "cs0_dq13_tx_de-skew", ++ "cs0_dq14_rx_de-skew", ++ "cs0_dq14_tx_de-skew", ++ "cs0_dq15_rx_de-skew", ++ "cs0_dq15_tx_de-skew", ++ "cs0_dqs1_rx_de-skew", ++ "cs0_dqs1p_tx_de-skew", ++ "cs0_dqs1n_tx_de-skew", ++ ++ "cs0_dm2_rx_de-skew", ++ "cs0_dm2_tx_de-skew", ++ "cs0_dq16_rx_de-skew", ++ "cs0_dq16_tx_de-skew", ++ "cs0_dq17_rx_de-skew", ++ "cs0_dq17_tx_de-skew", ++ "cs0_dq18_rx_de-skew", ++ "cs0_dq18_tx_de-skew", ++ "cs0_dq19_rx_de-skew", ++ "cs0_dq19_tx_de-skew", ++ "cs0_dq20_rx_de-skew", ++ "cs0_dq20_tx_de-skew", ++ "cs0_dq21_rx_de-skew", ++ "cs0_dq21_tx_de-skew", ++ "cs0_dq22_rx_de-skew", ++ "cs0_dq22_tx_de-skew", ++ "cs0_dq23_rx_de-skew", ++ "cs0_dq23_tx_de-skew", ++ "cs0_dqs2_rx_de-skew", ++ "cs0_dqs2p_tx_de-skew", ++ "cs0_dqs2n_tx_de-skew", ++ ++ "cs0_dm3_rx_de-skew", ++ "cs0_dm3_tx_de-skew", ++ "cs0_dq24_rx_de-skew", ++ "cs0_dq24_tx_de-skew", ++ "cs0_dq25_rx_de-skew", ++ "cs0_dq25_tx_de-skew", ++ "cs0_dq26_rx_de-skew", ++ "cs0_dq26_tx_de-skew", ++ "cs0_dq27_rx_de-skew", ++ "cs0_dq27_tx_de-skew", ++ "cs0_dq28_rx_de-skew", ++ "cs0_dq28_tx_de-skew", ++ "cs0_dq29_rx_de-skew", ++ "cs0_dq29_tx_de-skew", ++ "cs0_dq30_rx_de-skew", ++ "cs0_dq30_tx_de-skew", ++ "cs0_dq31_rx_de-skew", ++ "cs0_dq31_tx_de-skew", ++ "cs0_dqs3_rx_de-skew", ++ "cs0_dqs3p_tx_de-skew", ++ "cs0_dqs3n_tx_de-skew", ++}; ++ ++static const char * const rk3328_dts_cs1_timing[] = { ++ "cs1_dm0_rx_de-skew", ++ "cs1_dm0_tx_de-skew", ++ "cs1_dq0_rx_de-skew", ++ "cs1_dq0_tx_de-skew", ++ "cs1_dq1_rx_de-skew", ++ "cs1_dq1_tx_de-skew", ++ "cs1_dq2_rx_de-skew", ++ "cs1_dq2_tx_de-skew", ++ "cs1_dq3_rx_de-skew", ++ "cs1_dq3_tx_de-skew", ++ "cs1_dq4_rx_de-skew", ++ "cs1_dq4_tx_de-skew", ++ "cs1_dq5_rx_de-skew", ++ "cs1_dq5_tx_de-skew", ++ "cs1_dq6_rx_de-skew", ++ "cs1_dq6_tx_de-skew", ++ "cs1_dq7_rx_de-skew", ++ "cs1_dq7_tx_de-skew", ++ "cs1_dqs0_rx_de-skew", ++ "cs1_dqs0p_tx_de-skew", ++ "cs1_dqs0n_tx_de-skew", ++ ++ "cs1_dm1_rx_de-skew", ++ "cs1_dm1_tx_de-skew", ++ "cs1_dq8_rx_de-skew", ++ "cs1_dq8_tx_de-skew", ++ "cs1_dq9_rx_de-skew", ++ "cs1_dq9_tx_de-skew", ++ "cs1_dq10_rx_de-skew", ++ "cs1_dq10_tx_de-skew", ++ "cs1_dq11_rx_de-skew", ++ "cs1_dq11_tx_de-skew", ++ "cs1_dq12_rx_de-skew", ++ "cs1_dq12_tx_de-skew", ++ "cs1_dq13_rx_de-skew", ++ "cs1_dq13_tx_de-skew", ++ "cs1_dq14_rx_de-skew", ++ "cs1_dq14_tx_de-skew", ++ "cs1_dq15_rx_de-skew", ++ "cs1_dq15_tx_de-skew", ++ "cs1_dqs1_rx_de-skew", ++ "cs1_dqs1p_tx_de-skew", ++ "cs1_dqs1n_tx_de-skew", ++ ++ "cs1_dm2_rx_de-skew", ++ "cs1_dm2_tx_de-skew", ++ "cs1_dq16_rx_de-skew", ++ "cs1_dq16_tx_de-skew", ++ "cs1_dq17_rx_de-skew", ++ "cs1_dq17_tx_de-skew", ++ "cs1_dq18_rx_de-skew", ++ "cs1_dq18_tx_de-skew", ++ "cs1_dq19_rx_de-skew", ++ "cs1_dq19_tx_de-skew", ++ "cs1_dq20_rx_de-skew", ++ "cs1_dq20_tx_de-skew", ++ "cs1_dq21_rx_de-skew", ++ "cs1_dq21_tx_de-skew", ++ "cs1_dq22_rx_de-skew", ++ "cs1_dq22_tx_de-skew", ++ "cs1_dq23_rx_de-skew", ++ "cs1_dq23_tx_de-skew", ++ "cs1_dqs2_rx_de-skew", ++ "cs1_dqs2p_tx_de-skew", ++ "cs1_dqs2n_tx_de-skew", ++ ++ "cs1_dm3_rx_de-skew", ++ "cs1_dm3_tx_de-skew", ++ "cs1_dq24_rx_de-skew", ++ "cs1_dq24_tx_de-skew", ++ "cs1_dq25_rx_de-skew", ++ "cs1_dq25_tx_de-skew", ++ "cs1_dq26_rx_de-skew", ++ "cs1_dq26_tx_de-skew", ++ "cs1_dq27_rx_de-skew", ++ "cs1_dq27_tx_de-skew", ++ "cs1_dq28_rx_de-skew", ++ "cs1_dq28_tx_de-skew", ++ "cs1_dq29_rx_de-skew", ++ "cs1_dq29_tx_de-skew", ++ "cs1_dq30_rx_de-skew", ++ "cs1_dq30_tx_de-skew", ++ "cs1_dq31_rx_de-skew", ++ "cs1_dq31_tx_de-skew", ++ "cs1_dqs3_rx_de-skew", ++ "cs1_dqs3p_tx_de-skew", ++ "cs1_dqs3n_tx_de-skew", ++}; ++ ++struct rk3328_ddr_dts_config_timing { ++ unsigned int ddr3_speed_bin; ++ unsigned int ddr4_speed_bin; ++ unsigned int pd_idle; ++ unsigned int sr_idle; ++ unsigned int sr_mc_gate_idle; ++ unsigned int srpd_lite_idle; ++ unsigned int standby_idle; ++ ++ unsigned int auto_pd_dis_freq; ++ unsigned int auto_sr_dis_freq; ++ /* for ddr3 only */ ++ unsigned int ddr3_dll_dis_freq; ++ /* for ddr4 only */ ++ unsigned int ddr4_dll_dis_freq; ++ unsigned int phy_dll_dis_freq; ++ ++ unsigned int ddr3_odt_dis_freq; ++ unsigned int phy_ddr3_odt_dis_freq; ++ unsigned int ddr3_drv; ++ unsigned int ddr3_odt; ++ unsigned int phy_ddr3_ca_drv; ++ unsigned int phy_ddr3_ck_drv; ++ unsigned int phy_ddr3_dq_drv; ++ unsigned int phy_ddr3_odt; ++ ++ unsigned int lpddr3_odt_dis_freq; ++ unsigned int phy_lpddr3_odt_dis_freq; ++ unsigned int lpddr3_drv; ++ unsigned int lpddr3_odt; ++ unsigned int phy_lpddr3_ca_drv; ++ unsigned int phy_lpddr3_ck_drv; ++ unsigned int phy_lpddr3_dq_drv; ++ unsigned int phy_lpddr3_odt; ++ ++ unsigned int lpddr4_odt_dis_freq; ++ unsigned int phy_lpddr4_odt_dis_freq; ++ unsigned int lpddr4_drv; ++ unsigned int lpddr4_dq_odt; ++ unsigned int lpddr4_ca_odt; ++ unsigned int phy_lpddr4_ca_drv; ++ unsigned int phy_lpddr4_ck_cs_drv; ++ unsigned int phy_lpddr4_dq_drv; ++ unsigned int phy_lpddr4_odt; ++ ++ unsigned int ddr4_odt_dis_freq; ++ unsigned int phy_ddr4_odt_dis_freq; ++ unsigned int ddr4_drv; ++ unsigned int ddr4_odt; ++ unsigned int phy_ddr4_ca_drv; ++ unsigned int phy_ddr4_ck_drv; ++ unsigned int phy_ddr4_dq_drv; ++ unsigned int phy_ddr4_odt; ++ ++ unsigned int ca_skew[15]; ++ unsigned int cs0_skew[44]; ++ unsigned int cs1_skew[44]; ++ ++ unsigned int available; ++}; ++ ++struct rk3328_ddr_de_skew_setting { ++ unsigned int ca_de_skew[30]; ++ unsigned int cs0_de_skew[84]; ++ unsigned int cs1_de_skew[84]; ++}; ++ ++struct rk3368_dram_timing { ++ u32 dram_spd_bin; ++ u32 sr_idle; ++ u32 pd_idle; ++ u32 dram_dll_dis_freq; ++ u32 phy_dll_dis_freq; ++ u32 dram_odt_dis_freq; ++ u32 phy_odt_dis_freq; ++ u32 ddr3_drv; ++ u32 ddr3_odt; ++ u32 lpddr3_drv; ++ u32 lpddr3_odt; ++ u32 lpddr2_drv; ++ u32 phy_clk_drv; ++ u32 phy_cmd_drv; ++ u32 phy_dqs_drv; ++ u32 phy_odt; ++ u32 ddr_2t; ++}; ++ ++struct rk3399_dram_timing { ++ unsigned int ddr3_speed_bin; ++ unsigned int pd_idle; ++ unsigned int sr_idle; ++ unsigned int sr_mc_gate_idle; ++ unsigned int srpd_lite_idle; ++ unsigned int standby_idle; ++ unsigned int auto_lp_dis_freq; ++ unsigned int ddr3_dll_dis_freq; ++ unsigned int phy_dll_dis_freq; ++ unsigned int ddr3_odt_dis_freq; ++ unsigned int ddr3_drv; ++ unsigned int ddr3_odt; ++ unsigned int phy_ddr3_ca_drv; ++ unsigned int phy_ddr3_dq_drv; ++ unsigned int phy_ddr3_odt; ++ unsigned int lpddr3_odt_dis_freq; ++ unsigned int lpddr3_drv; ++ unsigned int lpddr3_odt; ++ unsigned int phy_lpddr3_ca_drv; ++ unsigned int phy_lpddr3_dq_drv; ++ unsigned int phy_lpddr3_odt; ++ unsigned int lpddr4_odt_dis_freq; ++ unsigned int lpddr4_drv; ++ unsigned int lpddr4_dq_odt; ++ unsigned int lpddr4_ca_odt; ++ unsigned int phy_lpddr4_ca_drv; ++ unsigned int phy_lpddr4_ck_cs_drv; ++ unsigned int phy_lpddr4_dq_drv; ++ unsigned int phy_lpddr4_odt; ++}; ++ ++struct rk3568_ddr_dts_config_timing { ++ unsigned int ddr2_speed_bin; ++ unsigned int ddr3_speed_bin; ++ unsigned int ddr4_speed_bin; ++ unsigned int pd_idle; ++ unsigned int sr_idle; ++ unsigned int sr_mc_gate_idle; ++ unsigned int srpd_lite_idle; ++ unsigned int standby_idle; ++ ++ unsigned int auto_pd_dis_freq; ++ unsigned int auto_sr_dis_freq; ++ /* for ddr2 only */ ++ unsigned int ddr2_dll_dis_freq; ++ /* for ddr3 only */ ++ unsigned int ddr3_dll_dis_freq; ++ /* for ddr4 only */ ++ unsigned int ddr4_dll_dis_freq; ++ unsigned int phy_dll_dis_freq; ++ ++ unsigned int ddr2_odt_dis_freq; ++ unsigned int phy_ddr2_odt_dis_freq; ++ unsigned int ddr2_drv; ++ unsigned int ddr2_odt; ++ unsigned int phy_ddr2_ca_drv; ++ unsigned int phy_ddr2_ck_drv; ++ unsigned int phy_ddr2_dq_drv; ++ unsigned int phy_ddr2_odt; ++ ++ unsigned int ddr3_odt_dis_freq; ++ unsigned int phy_ddr3_odt_dis_freq; ++ unsigned int ddr3_drv; ++ unsigned int ddr3_odt; ++ unsigned int phy_ddr3_ca_drv; ++ unsigned int phy_ddr3_ck_drv; ++ unsigned int phy_ddr3_dq_drv; ++ unsigned int phy_ddr3_odt; ++ ++ unsigned int phy_lpddr2_odt_dis_freq; ++ unsigned int lpddr2_drv; ++ unsigned int phy_lpddr2_ca_drv; ++ unsigned int phy_lpddr2_ck_drv; ++ unsigned int phy_lpddr2_dq_drv; ++ unsigned int phy_lpddr2_odt; ++ ++ unsigned int lpddr3_odt_dis_freq; ++ unsigned int phy_lpddr3_odt_dis_freq; ++ unsigned int lpddr3_drv; ++ unsigned int lpddr3_odt; ++ unsigned int phy_lpddr3_ca_drv; ++ unsigned int phy_lpddr3_ck_drv; ++ unsigned int phy_lpddr3_dq_drv; ++ unsigned int phy_lpddr3_odt; ++ ++ unsigned int lpddr4_odt_dis_freq; ++ unsigned int phy_lpddr4_odt_dis_freq; ++ unsigned int lpddr4_drv; ++ unsigned int lpddr4_dq_odt; ++ unsigned int lpddr4_ca_odt; ++ unsigned int phy_lpddr4_ca_drv; ++ unsigned int phy_lpddr4_ck_cs_drv; ++ unsigned int phy_lpddr4_dq_drv; ++ unsigned int phy_lpddr4_odt; ++ ++ unsigned int ddr4_odt_dis_freq; ++ unsigned int phy_ddr4_odt_dis_freq; ++ unsigned int ddr4_drv; ++ unsigned int ddr4_odt; ++ unsigned int phy_ddr4_ca_drv; ++ unsigned int phy_ddr4_ck_drv; ++ unsigned int phy_ddr4_dq_drv; ++ unsigned int phy_ddr4_odt; ++ ++ unsigned int available; ++}; ++ ++/* name rule: ddr4(pad_name)_ddr3_lpddr3_lpddr4_de-skew */ ++static const char * const rv1126_dts_ca_timing[] = { ++ "a0_a3_a3_cke1-a_de-skew", ++ "a1_ba1_null_cke0-b_de-skew", ++ "a2_a9_a9_a4-a_de-skew", ++ "a3_a15_null_a5-b_de-skew", ++ "a4_a6_a6_ck-a_de-skew", ++ "a5_a12_null_odt0-b_de-skew", ++ "a6_ba2_null_a0-a_de-skew", ++ "a7_a4_a4_odt0-a_de-skew", ++ "a8_a1_a1_cke0-a_de-skew", ++ "a9_a5_a5_a5-a_de-skew", ++ "a10_a8_a8_clkb-a_de-skew", ++ "a11_a7_a7_ca2-a_de-skew", ++ "a12_rasn_null_ca1-a_de-skew", ++ "a13_a13_null_ca3-a_de-skew", ++ "a14_a14_null_csb1-b_de-skew", ++ "a15_a10_null_ca0-b_de-skew", ++ "a16_a11_null_csb0-b_de-skew", ++ "a17_null_null_null_de-skew", ++ "ba0_csb1_csb1_csb0-a_de-skew", ++ "ba1_wen_null_cke1-b_de-skew", ++ "bg0_odt1_odt1_csb1-a_de-skew", ++ "bg1_a2_a2_odt1-a_de-skew", ++ "cke0_casb_null_ca1-b_de-skew", ++ "ck_ck_ck_ck-b_de-skew", ++ "ckb_ckb_ckb_ckb-b_de-skew", ++ "csb0_odt0_odt0_ca2-b_de-skew", ++ "odt0_csb0_csb0_ca4-b_de-skew", ++ "resetn_resetn_null-resetn_de-skew", ++ "actn_cke_cke_ca3-b_de-skew", ++ "cke1_null_null_null_de-skew", ++ "csb1_ba0_null_null_de-skew", ++ "odt1_a0_a0_odt1-b_de-skew", ++}; ++ ++static const char * const rv1126_dts_cs0_a_timing[] = { ++ "cs0_dm0_rx_de-skew", ++ "cs0_dq0_rx_de-skew", ++ "cs0_dq1_rx_de-skew", ++ "cs0_dq2_rx_de-skew", ++ "cs0_dq3_rx_de-skew", ++ "cs0_dq4_rx_de-skew", ++ "cs0_dq5_rx_de-skew", ++ "cs0_dq6_rx_de-skew", ++ "cs0_dq7_rx_de-skew", ++ "cs0_dqs0p_rx_de-skew", ++ "cs0_dqs0n_rx_de-skew", ++ "cs0_dm1_rx_de-skew", ++ "cs0_dq8_rx_de-skew", ++ "cs0_dq9_rx_de-skew", ++ "cs0_dq10_rx_de-skew", ++ "cs0_dq11_rx_de-skew", ++ "cs0_dq12_rx_de-skew", ++ "cs0_dq13_rx_de-skew", ++ "cs0_dq14_rx_de-skew", ++ "cs0_dq15_rx_de-skew", ++ "cs0_dqs1p_rx_de-skew", ++ "cs0_dqs1n_rx_de-skew", ++ "cs0_dm0_tx_de-skew", ++ "cs0_dq0_tx_de-skew", ++ "cs0_dq1_tx_de-skew", ++ "cs0_dq2_tx_de-skew", ++ "cs0_dq3_tx_de-skew", ++ "cs0_dq4_tx_de-skew", ++ "cs0_dq5_tx_de-skew", ++ "cs0_dq6_tx_de-skew", ++ "cs0_dq7_tx_de-skew", ++ "cs0_dqs0p_tx_de-skew", ++ "cs0_dqs0n_tx_de-skew", ++ "cs0_dm1_tx_de-skew", ++ "cs0_dq8_tx_de-skew", ++ "cs0_dq9_tx_de-skew", ++ "cs0_dq10_tx_de-skew", ++ "cs0_dq11_tx_de-skew", ++ "cs0_dq12_tx_de-skew", ++ "cs0_dq13_tx_de-skew", ++ "cs0_dq14_tx_de-skew", ++ "cs0_dq15_tx_de-skew", ++ "cs0_dqs1p_tx_de-skew", ++ "cs0_dqs1n_tx_de-skew", ++}; ++ ++static const char * const rv1126_dts_cs0_b_timing[] = { ++ "cs0_dm2_rx_de-skew", ++ "cs0_dq16_rx_de-skew", ++ "cs0_dq17_rx_de-skew", ++ "cs0_dq18_rx_de-skew", ++ "cs0_dq19_rx_de-skew", ++ "cs0_dq20_rx_de-skew", ++ "cs0_dq21_rx_de-skew", ++ "cs0_dq22_rx_de-skew", ++ "cs0_dq23_rx_de-skew", ++ "cs0_dqs2p_rx_de-skew", ++ "cs0_dqs2n_rx_de-skew", ++ "cs0_dm3_rx_de-skew", ++ "cs0_dq24_rx_de-skew", ++ "cs0_dq25_rx_de-skew", ++ "cs0_dq26_rx_de-skew", ++ "cs0_dq27_rx_de-skew", ++ "cs0_dq28_rx_de-skew", ++ "cs0_dq29_rx_de-skew", ++ "cs0_dq30_rx_de-skew", ++ "cs0_dq31_rx_de-skew", ++ "cs0_dqs3p_rx_de-skew", ++ "cs0_dqs3n_rx_de-skew", ++ "cs0_dm2_tx_de-skew", ++ "cs0_dq16_tx_de-skew", ++ "cs0_dq17_tx_de-skew", ++ "cs0_dq18_tx_de-skew", ++ "cs0_dq19_tx_de-skew", ++ "cs0_dq20_tx_de-skew", ++ "cs0_dq21_tx_de-skew", ++ "cs0_dq22_tx_de-skew", ++ "cs0_dq23_tx_de-skew", ++ "cs0_dqs2p_tx_de-skew", ++ "cs0_dqs2n_tx_de-skew", ++ "cs0_dm3_tx_de-skew", ++ "cs0_dq24_tx_de-skew", ++ "cs0_dq25_tx_de-skew", ++ "cs0_dq26_tx_de-skew", ++ "cs0_dq27_tx_de-skew", ++ "cs0_dq28_tx_de-skew", ++ "cs0_dq29_tx_de-skew", ++ "cs0_dq30_tx_de-skew", ++ "cs0_dq31_tx_de-skew", ++ "cs0_dqs3p_tx_de-skew", ++ "cs0_dqs3n_tx_de-skew", ++}; ++ ++static const char * const rv1126_dts_cs1_a_timing[] = { ++ "cs1_dm0_rx_de-skew", ++ "cs1_dq0_rx_de-skew", ++ "cs1_dq1_rx_de-skew", ++ "cs1_dq2_rx_de-skew", ++ "cs1_dq3_rx_de-skew", ++ "cs1_dq4_rx_de-skew", ++ "cs1_dq5_rx_de-skew", ++ "cs1_dq6_rx_de-skew", ++ "cs1_dq7_rx_de-skew", ++ "cs1_dqs0p_rx_de-skew", ++ "cs1_dqs0n_rx_de-skew", ++ "cs1_dm1_rx_de-skew", ++ "cs1_dq8_rx_de-skew", ++ "cs1_dq9_rx_de-skew", ++ "cs1_dq10_rx_de-skew", ++ "cs1_dq11_rx_de-skew", ++ "cs1_dq12_rx_de-skew", ++ "cs1_dq13_rx_de-skew", ++ "cs1_dq14_rx_de-skew", ++ "cs1_dq15_rx_de-skew", ++ "cs1_dqs1p_rx_de-skew", ++ "cs1_dqs1n_rx_de-skew", ++ "cs1_dm0_tx_de-skew", ++ "cs1_dq0_tx_de-skew", ++ "cs1_dq1_tx_de-skew", ++ "cs1_dq2_tx_de-skew", ++ "cs1_dq3_tx_de-skew", ++ "cs1_dq4_tx_de-skew", ++ "cs1_dq5_tx_de-skew", ++ "cs1_dq6_tx_de-skew", ++ "cs1_dq7_tx_de-skew", ++ "cs1_dqs0p_tx_de-skew", ++ "cs1_dqs0n_tx_de-skew", ++ "cs1_dm1_tx_de-skew", ++ "cs1_dq8_tx_de-skew", ++ "cs1_dq9_tx_de-skew", ++ "cs1_dq10_tx_de-skew", ++ "cs1_dq11_tx_de-skew", ++ "cs1_dq12_tx_de-skew", ++ "cs1_dq13_tx_de-skew", ++ "cs1_dq14_tx_de-skew", ++ "cs1_dq15_tx_de-skew", ++ "cs1_dqs1p_tx_de-skew", ++ "cs1_dqs1n_tx_de-skew", ++}; ++ ++static const char * const rv1126_dts_cs1_b_timing[] = { ++ "cs1_dm2_rx_de-skew", ++ "cs1_dq16_rx_de-skew", ++ "cs1_dq17_rx_de-skew", ++ "cs1_dq18_rx_de-skew", ++ "cs1_dq19_rx_de-skew", ++ "cs1_dq20_rx_de-skew", ++ "cs1_dq21_rx_de-skew", ++ "cs1_dq22_rx_de-skew", ++ "cs1_dq23_rx_de-skew", ++ "cs1_dqs2p_rx_de-skew", ++ "cs1_dqs2n_rx_de-skew", ++ "cs1_dm3_rx_de-skew", ++ "cs1_dq24_rx_de-skew", ++ "cs1_dq25_rx_de-skew", ++ "cs1_dq26_rx_de-skew", ++ "cs1_dq27_rx_de-skew", ++ "cs1_dq28_rx_de-skew", ++ "cs1_dq29_rx_de-skew", ++ "cs1_dq30_rx_de-skew", ++ "cs1_dq31_rx_de-skew", ++ "cs1_dqs3p_rx_de-skew", ++ "cs1_dqs3n_rx_de-skew", ++ "cs1_dm2_tx_de-skew", ++ "cs1_dq16_tx_de-skew", ++ "cs1_dq17_tx_de-skew", ++ "cs1_dq18_tx_de-skew", ++ "cs1_dq19_tx_de-skew", ++ "cs1_dq20_tx_de-skew", ++ "cs1_dq21_tx_de-skew", ++ "cs1_dq22_tx_de-skew", ++ "cs1_dq23_tx_de-skew", ++ "cs1_dqs2p_tx_de-skew", ++ "cs1_dqs2n_tx_de-skew", ++ "cs1_dm3_tx_de-skew", ++ "cs1_dq24_tx_de-skew", ++ "cs1_dq25_tx_de-skew", ++ "cs1_dq26_tx_de-skew", ++ "cs1_dq27_tx_de-skew", ++ "cs1_dq28_tx_de-skew", ++ "cs1_dq29_tx_de-skew", ++ "cs1_dq30_tx_de-skew", ++ "cs1_dq31_tx_de-skew", ++ "cs1_dqs3p_tx_de-skew", ++ "cs1_dqs3n_tx_de-skew", ++}; ++ ++#endif /* __ROCKCHIP_DMC_TIMING_H__ */ ++ +diff --git a/drivers/dma-buf/Kconfig b/drivers/dma-buf/Kconfig +index 3ca7de37dd8f..dec5d6851ac5 100644 +--- a/drivers/dma-buf/Kconfig ++++ b/drivers/dma-buf/Kconfig +@@ -21,7 +21,6 @@ config SW_SYNC + bool "Sync File Validation Framework" + default n + depends on SYNC_FILE +- depends on DEBUG_FS + help + A sync object driver that uses a 32bit counter to coordinate + synchronization. Useful when there is no hardware primitive backing +@@ -65,6 +64,17 @@ menuconfig DMABUF_HEAPS + allows userspace to allocate dma-bufs that can be shared + between drivers. + ++menuconfig DMABUF_SYSFS_STATS ++ bool "DMA-BUF sysfs statistics" ++ select DMA_SHARED_BUFFER ++ help ++ Choose this option to enable DMA-BUF sysfs statistics ++ in location /sys/kernel/dmabuf/buffers. ++ ++ /sys/kernel/dmabuf/buffers/ will contain ++ statistics for the DMA-BUF with the unique inode number ++ . ++ + source "drivers/dma-buf/heaps/Kconfig" + + endmenu +diff --git a/drivers/dma-buf/Makefile b/drivers/dma-buf/Makefile +index 995e05f609ff..40d81f23cacf 100644 +--- a/drivers/dma-buf/Makefile ++++ b/drivers/dma-buf/Makefile +@@ -6,6 +6,7 @@ obj-$(CONFIG_DMABUF_HEAPS) += heaps/ + obj-$(CONFIG_SYNC_FILE) += sync_file.o + obj-$(CONFIG_SW_SYNC) += sw_sync.o sync_debug.o + obj-$(CONFIG_UDMABUF) += udmabuf.o ++obj-$(CONFIG_DMABUF_SYSFS_STATS) += dma-buf-sysfs-stats.o + + dmabuf_selftests-y := \ + selftest.o \ +diff --git a/drivers/dma-buf/dma-buf-sysfs-stats.c b/drivers/dma-buf/dma-buf-sysfs-stats.c +new file mode 100755 +index 000000000000..943e395d1807 +--- /dev/null ++++ b/drivers/dma-buf/dma-buf-sysfs-stats.c +@@ -0,0 +1,311 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* ++ * DMA-BUF sysfs statistics. ++ * ++ * Copyright (C) 2021 Google LLC. ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "dma-buf-sysfs-stats.h" ++ ++#define to_dma_buf_entry_from_kobj(x) container_of(x, struct dma_buf_sysfs_entry, kobj) ++ ++struct dma_buf_stats_attribute { ++ struct attribute attr; ++ ssize_t (*show)(struct dma_buf *dmabuf, ++ struct dma_buf_stats_attribute *attr, char *buf); ++}; ++#define to_dma_buf_stats_attr(x) container_of(x, struct dma_buf_stats_attribute, attr) ++ ++static ssize_t dma_buf_stats_attribute_show(struct kobject *kobj, ++ struct attribute *attr, ++ char *buf) ++{ ++ struct dma_buf_stats_attribute *attribute; ++ struct dma_buf_sysfs_entry *sysfs_entry; ++ struct dma_buf *dmabuf; ++ ++ attribute = to_dma_buf_stats_attr(attr); ++ sysfs_entry = to_dma_buf_entry_from_kobj(kobj); ++ dmabuf = sysfs_entry->dmabuf; ++ ++ if (!dmabuf || !attribute->show) ++ return -EIO; ++ ++ return attribute->show(dmabuf, attribute, buf); ++} ++ ++static const struct sysfs_ops dma_buf_stats_sysfs_ops = { ++ .show = dma_buf_stats_attribute_show, ++}; ++ ++static ssize_t exporter_name_show(struct dma_buf *dmabuf, ++ struct dma_buf_stats_attribute *attr, ++ char *buf) ++{ ++ return sysfs_emit(buf, "%s\n", dmabuf->exp_name); ++} ++ ++static ssize_t mmap_count_show(struct dma_buf *dmabuf, ++ struct dma_buf_stats_attribute *attr, ++ char *buf) ++{ ++ return sysfs_emit(buf, "%d\n", dmabuf->mmap_count); ++} ++ ++static ssize_t size_show(struct dma_buf *dmabuf, ++ struct dma_buf_stats_attribute *attr, ++ char *buf) ++{ ++ return sysfs_emit(buf, "%zu\n", dmabuf->size); ++} ++ ++static struct dma_buf_stats_attribute exporter_name_attribute = ++ __ATTR_RO(exporter_name); ++static struct dma_buf_stats_attribute size_attribute = __ATTR_RO(size); ++static struct dma_buf_stats_attribute mmap_count_attribute = ++ __ATTR_RO(mmap_count); ++ ++static struct attribute *dma_buf_stats_default_attrs[] = { ++ &exporter_name_attribute.attr, ++ &size_attribute.attr, ++ &mmap_count_attribute.attr, ++ NULL, ++}; ++ATTRIBUTE_GROUPS(dma_buf_stats_default); ++ ++static void dma_buf_sysfs_release(struct kobject *kobj) ++{ ++ struct dma_buf_sysfs_entry *sysfs_entry; ++ ++ sysfs_entry = to_dma_buf_entry_from_kobj(kobj); ++ kfree(sysfs_entry); ++} ++ ++static struct kobj_type dma_buf_ktype = { ++ .sysfs_ops = &dma_buf_stats_sysfs_ops, ++ .release = dma_buf_sysfs_release, ++ .default_groups = dma_buf_stats_default_groups, ++}; ++ ++#define to_dma_buf_attach_entry_from_kobj(x) container_of(x, struct dma_buf_attach_sysfs_entry, kobj) ++ ++struct dma_buf_attach_stats_attribute { ++ struct attribute attr; ++ ssize_t (*show)(struct dma_buf_attach_sysfs_entry *sysfs_entry, ++ struct dma_buf_attach_stats_attribute *attr, char *buf); ++}; ++#define to_dma_buf_attach_stats_attr(x) container_of(x, struct dma_buf_attach_stats_attribute, attr) ++ ++static ssize_t dma_buf_attach_stats_attribute_show(struct kobject *kobj, ++ struct attribute *attr, ++ char *buf) ++{ ++ struct dma_buf_attach_stats_attribute *attribute; ++ struct dma_buf_attach_sysfs_entry *sysfs_entry; ++ ++ attribute = to_dma_buf_attach_stats_attr(attr); ++ sysfs_entry = to_dma_buf_attach_entry_from_kobj(kobj); ++ ++ if (!attribute->show) ++ return -EIO; ++ ++ return attribute->show(sysfs_entry, attribute, buf); ++} ++ ++static const struct sysfs_ops dma_buf_attach_stats_sysfs_ops = { ++ .show = dma_buf_attach_stats_attribute_show, ++}; ++ ++static ssize_t map_counter_show(struct dma_buf_attach_sysfs_entry *sysfs_entry, ++ struct dma_buf_attach_stats_attribute *attr, ++ char *buf) ++{ ++ return sysfs_emit(buf, "%u\n", sysfs_entry->map_counter); ++} ++ ++static struct dma_buf_attach_stats_attribute map_counter_attribute = ++ __ATTR_RO(map_counter); ++ ++static struct attribute *dma_buf_attach_stats_default_attrs[] = { ++ &map_counter_attribute.attr, ++ NULL, ++}; ++ATTRIBUTE_GROUPS(dma_buf_attach_stats_default); ++ ++static void dma_buf_attach_sysfs_release(struct kobject *kobj) ++{ ++ struct dma_buf_attach_sysfs_entry *sysfs_entry; ++ ++ sysfs_entry = to_dma_buf_attach_entry_from_kobj(kobj); ++ kfree(sysfs_entry); ++} ++ ++static struct kobj_type dma_buf_attach_ktype = { ++ .sysfs_ops = &dma_buf_attach_stats_sysfs_ops, ++ .release = dma_buf_attach_sysfs_release, ++ .default_groups = dma_buf_attach_stats_default_groups, ++}; ++ ++void dma_buf_attach_stats_teardown(struct dma_buf_attachment *attach) ++{ ++ struct dma_buf_attach_sysfs_entry *sysfs_entry; ++ ++ sysfs_entry = attach->sysfs_entry; ++ if (!sysfs_entry) ++ return; ++ ++ sysfs_delete_link(&sysfs_entry->kobj, &attach->dev->kobj, "device"); ++ ++ kobject_del(&sysfs_entry->kobj); ++ kobject_put(&sysfs_entry->kobj); ++} ++ ++int dma_buf_attach_stats_setup(struct dma_buf_attachment *attach, ++ unsigned int uid) ++{ ++ struct dma_buf_attach_sysfs_entry *sysfs_entry; ++ int ret; ++ struct dma_buf *dmabuf; ++ ++ if (!attach) ++ return -EINVAL; ++ ++ dmabuf = attach->dmabuf; ++ ++ sysfs_entry = kzalloc(sizeof(struct dma_buf_attach_sysfs_entry), ++ GFP_KERNEL); ++ if (!sysfs_entry) ++ return -ENOMEM; ++ ++ sysfs_entry->kobj.kset = dmabuf->sysfs_entry->attach_stats_kset; ++ ++ attach->sysfs_entry = sysfs_entry; ++ ++ ret = kobject_init_and_add(&sysfs_entry->kobj, &dma_buf_attach_ktype, ++ NULL, "%u", uid); ++ if (ret) ++ goto kobj_err; ++ ++ ret = sysfs_create_link(&sysfs_entry->kobj, &attach->dev->kobj, ++ "device"); ++ if (ret) ++ goto link_err; ++ ++ return 0; ++ ++link_err: ++ kobject_del(&sysfs_entry->kobj); ++kobj_err: ++ kobject_put(&sysfs_entry->kobj); ++ attach->sysfs_entry = NULL; ++ ++ return ret; ++} ++void dma_buf_stats_teardown(struct dma_buf *dmabuf) ++{ ++ struct dma_buf_sysfs_entry *sysfs_entry; ++ ++ sysfs_entry = dmabuf->sysfs_entry; ++ if (!sysfs_entry) ++ return; ++ ++ kset_unregister(sysfs_entry->attach_stats_kset); ++ kobject_del(&sysfs_entry->kobj); ++ kobject_put(&sysfs_entry->kobj); ++} ++ ++/* ++ * Statistics files do not need to send uevents. ++ */ ++static int dmabuf_sysfs_uevent_filter(struct kset *kset, struct kobject *kobj) ++{ ++ return 0; ++} ++ ++static const struct kset_uevent_ops dmabuf_sysfs_no_uevent_ops = { ++ .filter = dmabuf_sysfs_uevent_filter, ++}; ++ ++static struct kset *dma_buf_stats_kset; ++static struct kset *dma_buf_per_buffer_stats_kset; ++int dma_buf_init_sysfs_statistics(void) ++{ ++ dma_buf_stats_kset = kset_create_and_add("dmabuf", ++ &dmabuf_sysfs_no_uevent_ops, ++ kernel_kobj); ++ if (!dma_buf_stats_kset) ++ return -ENOMEM; ++ ++ dma_buf_per_buffer_stats_kset = kset_create_and_add("buffers", ++ &dmabuf_sysfs_no_uevent_ops, ++ &dma_buf_stats_kset->kobj); ++ if (!dma_buf_per_buffer_stats_kset) { ++ kset_unregister(dma_buf_stats_kset); ++ return -ENOMEM; ++ } ++ ++ return 0; ++} ++ ++void dma_buf_uninit_sysfs_statistics(void) ++{ ++ kset_unregister(dma_buf_per_buffer_stats_kset); ++ kset_unregister(dma_buf_stats_kset); ++} ++ ++int dma_buf_stats_setup(struct dma_buf *dmabuf) ++{ ++ struct dma_buf_sysfs_entry *sysfs_entry; ++ int ret; ++ struct kset *attach_stats_kset; ++ ++ if (!dmabuf || !dmabuf->file) ++ return -EINVAL; ++ ++ if (!dmabuf->exp_name) { ++ pr_err("exporter name must not be empty if stats needed\n"); ++ return -EINVAL; ++ } ++ ++ sysfs_entry = kzalloc(sizeof(struct dma_buf_sysfs_entry), GFP_KERNEL); ++ if (!sysfs_entry) ++ return -ENOMEM; ++ ++ sysfs_entry->kobj.kset = dma_buf_per_buffer_stats_kset; ++ sysfs_entry->dmabuf = dmabuf; ++ ++ dmabuf->sysfs_entry = sysfs_entry; ++ ++ /* create the directory for buffer stats */ ++ ret = kobject_init_and_add(&sysfs_entry->kobj, &dma_buf_ktype, NULL, ++ "%lu", file_inode(dmabuf->file)->i_ino); ++ if (ret) ++ goto err_sysfs_dmabuf; ++ ++ /* create the directory for attachment stats */ ++ attach_stats_kset = kset_create_and_add("attachments", ++ &dmabuf_sysfs_no_uevent_ops, ++ &sysfs_entry->kobj); ++ if (!attach_stats_kset) { ++ ret = -ENOMEM; ++ goto err_sysfs_attach; ++ } ++ ++ sysfs_entry->attach_stats_kset = attach_stats_kset; ++ ++ return 0; ++ ++err_sysfs_attach: ++ kobject_del(&sysfs_entry->kobj); ++err_sysfs_dmabuf: ++ kobject_put(&sysfs_entry->kobj); ++ dmabuf->sysfs_entry = NULL; ++ return ret; ++} +diff --git a/drivers/dma-buf/dma-buf-sysfs-stats.h b/drivers/dma-buf/dma-buf-sysfs-stats.h +new file mode 100755 +index 000000000000..5f4703249117 +--- /dev/null ++++ b/drivers/dma-buf/dma-buf-sysfs-stats.h +@@ -0,0 +1,62 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* ++ * DMA-BUF sysfs statistics. ++ * ++ * Copyright (C) 2021 Google LLC. ++ */ ++ ++#ifndef _DMA_BUF_SYSFS_STATS_H ++#define _DMA_BUF_SYSFS_STATS_H ++ ++#ifdef CONFIG_DMABUF_SYSFS_STATS ++ ++int dma_buf_init_sysfs_statistics(void); ++void dma_buf_uninit_sysfs_statistics(void); ++ ++int dma_buf_stats_setup(struct dma_buf *dmabuf); ++int dma_buf_attach_stats_setup(struct dma_buf_attachment *attach, ++ unsigned int uid); ++static inline void dma_buf_update_attachment_map_count(struct dma_buf_attachment *attach, ++ int delta) ++{ ++ struct dma_buf_attach_sysfs_entry *entry = attach->sysfs_entry; ++ ++ entry->map_counter += delta; ++} ++void dma_buf_stats_teardown(struct dma_buf *dmabuf); ++void dma_buf_attach_stats_teardown(struct dma_buf_attachment *attach); ++static inline unsigned int dma_buf_update_attach_uid(struct dma_buf *dmabuf) ++{ ++ struct dma_buf_sysfs_entry *entry = dmabuf->sysfs_entry; ++ ++ return entry->attachment_uid++; ++} ++#else ++ ++static inline int dma_buf_init_sysfs_statistics(void) ++{ ++ return 0; ++} ++ ++static inline void dma_buf_uninit_sysfs_statistics(void) {} ++ ++static inline int dma_buf_stats_setup(struct dma_buf *dmabuf) ++{ ++ return 0; ++} ++static inline int dma_buf_attach_stats_setup(struct dma_buf_attachment *attach, ++ unsigned int uid) ++{ ++ return 0; ++} ++ ++static inline void dma_buf_stats_teardown(struct dma_buf *dmabuf) {} ++static inline void dma_buf_attach_stats_teardown(struct dma_buf_attachment *attach) {} ++static inline void dma_buf_update_attachment_map_count(struct dma_buf_attachment *attach, ++ int delta) {} ++static inline unsigned int dma_buf_update_attach_uid(struct dma_buf *dmabuf) ++{ ++ return 0; ++} ++#endif ++#endif // _DMA_BUF_SYSFS_STATS_H +diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c +index 922416b3aace..0ab865543d1f 100644 +--- a/drivers/dma-buf/dma-buf.c ++++ b/drivers/dma-buf/dma-buf.c +@@ -29,7 +29,7 @@ + #include + #include + +-static inline int is_dma_buf_file(struct file *); ++#include "dma-buf-sysfs-stats.h" + + struct dma_buf_list { + struct list_head head; +@@ -38,6 +38,30 @@ struct dma_buf_list { + + static struct dma_buf_list db_list; + ++/* ++ * This function helps in traversing the db_list and calls the ++ * callback function which can extract required info out of each ++ * dmabuf. ++ */ ++int get_each_dmabuf(int (*callback)(const struct dma_buf *dmabuf, ++ void *private), void *private) ++{ ++ struct dma_buf *buf; ++ int ret = mutex_lock_interruptible(&db_list.lock); ++ ++ if (ret) ++ return ret; ++ ++ list_for_each_entry(buf, &db_list.head, list_node) { ++ ret = callback(buf, private); ++ if (ret) ++ break; ++ } ++ mutex_unlock(&db_list.lock); ++ return ret; ++} ++EXPORT_SYMBOL_GPL(get_each_dmabuf); ++ + static char *dmabuffs_dname(struct dentry *dentry, char *buffer, int buflen) + { + struct dma_buf *dmabuf; +@@ -79,6 +103,7 @@ static void dma_buf_release(struct dentry *dentry) + if (dmabuf->resv == (struct dma_resv *)&dmabuf[1]) + dma_resv_fini(dmabuf->resv); + ++ dma_buf_stats_teardown(dmabuf); + module_put(dmabuf->owner); + kfree(dmabuf->name); + kfree(dmabuf); +@@ -124,6 +149,54 @@ static struct file_system_type dma_buf_fs_type = { + .kill_sb = kill_anon_super, + }; + ++#ifdef CONFIG_DMABUF_SYSFS_STATS ++static void dma_buf_vma_open(struct vm_area_struct *vma) ++{ ++ struct dma_buf *dmabuf = vma->vm_file->private_data; ++ ++ dmabuf->mmap_count++; ++ /* call the heap provided vma open() op */ ++ if (dmabuf->exp_vm_ops->open) ++ dmabuf->exp_vm_ops->open(vma); ++} ++ ++static void dma_buf_vma_close(struct vm_area_struct *vma) ++{ ++ struct dma_buf *dmabuf = vma->vm_file->private_data; ++ ++ if (dmabuf->mmap_count) ++ dmabuf->mmap_count--; ++ /* call the heap provided vma close() op */ ++ if (dmabuf->exp_vm_ops->close) ++ dmabuf->exp_vm_ops->close(vma); ++} ++ ++static int dma_buf_do_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma) ++{ ++ /* call this first because the exporter might override vma->vm_ops */ ++ int ret = dmabuf->ops->mmap(dmabuf, vma); ++ ++ if (ret) ++ return ret; ++ ++ /* save the exporter provided vm_ops */ ++ dmabuf->exp_vm_ops = vma->vm_ops; ++ dmabuf->vm_ops = *(dmabuf->exp_vm_ops); ++ /* override open() and close() to provide buffer mmap count */ ++ dmabuf->vm_ops.open = dma_buf_vma_open; ++ dmabuf->vm_ops.close = dma_buf_vma_close; ++ vma->vm_ops = &dmabuf->vm_ops; ++ dmabuf->mmap_count++; ++ ++ return ret; ++} ++#else /* CONFIG_DMABUF_SYSFS_STATS */ ++static int dma_buf_do_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma) ++{ ++ return dmabuf->ops->mmap(dmabuf, vma); ++} ++#endif /* CONFIG_DMABUF_SYSFS_STATS */ ++ + static int dma_buf_mmap_internal(struct file *file, struct vm_area_struct *vma) + { + struct dma_buf *dmabuf; +@@ -142,7 +215,7 @@ static int dma_buf_mmap_internal(struct file *file, struct vm_area_struct *vma) + dmabuf->size >> PAGE_SHIFT) + return -EINVAL; + +- return dmabuf->ops->mmap(dmabuf, vma); ++ return dma_buf_do_mmap(dmabuf, vma); + } + + static loff_t dma_buf_llseek(struct file *file, loff_t offset, int whence) +@@ -437,10 +510,11 @@ static const struct file_operations dma_buf_fops = { + /* + * is_dma_buf_file - Check if struct file* is associated with dma_buf + */ +-static inline int is_dma_buf_file(struct file *file) ++int is_dma_buf_file(struct file *file) + { + return file->f_op == &dma_buf_fops; + } ++EXPORT_SYMBOL_GPL(is_dma_buf_file); + + static struct file *dma_buf_getfile(struct dma_buf *dmabuf, int flags) + { +@@ -579,6 +653,10 @@ struct dma_buf *dma_buf_export(const struct dma_buf_export_info *exp_info) + file->f_mode |= FMODE_LSEEK; + dmabuf->file = file; + ++ ret = dma_buf_stats_setup(dmabuf); ++ if (ret) ++ goto err_sysfs; ++ + mutex_init(&dmabuf->lock); + INIT_LIST_HEAD(&dmabuf->attachments); + +@@ -588,6 +666,14 @@ struct dma_buf *dma_buf_export(const struct dma_buf_export_info *exp_info) + + return dmabuf; + ++err_sysfs: ++ /* ++ * Set file->f_path.dentry->d_fsdata to NULL so that when ++ * dma_buf_release() gets invoked by dentry_ops, it exits ++ * early before calling the release() dma_buf op. ++ */ ++ file->f_path.dentry->d_fsdata = NULL; ++ fput(file); + err_dmabuf: + kfree(dmabuf); + err_module: +@@ -692,6 +778,7 @@ dma_buf_dynamic_attach(struct dma_buf *dmabuf, struct device *dev, + { + struct dma_buf_attachment *attach; + int ret; ++ unsigned int attach_uid; + + if (WARN_ON(!dmabuf || !dev)) + return ERR_PTR(-EINVAL); +@@ -717,8 +804,13 @@ dma_buf_dynamic_attach(struct dma_buf *dmabuf, struct device *dev, + } + dma_resv_lock(dmabuf->resv, NULL); + list_add(&attach->node, &dmabuf->attachments); ++ attach_uid = dma_buf_update_attach_uid(dmabuf); + dma_resv_unlock(dmabuf->resv); + ++ ret = dma_buf_attach_stats_setup(attach, attach_uid); ++ if (ret) ++ goto err_sysfs; ++ + /* When either the importer or the exporter can't handle dynamic + * mappings we cache the mapping here to avoid issues with the + * reservation object lock. +@@ -745,6 +837,7 @@ dma_buf_dynamic_attach(struct dma_buf *dmabuf, struct device *dev, + dma_resv_unlock(attach->dmabuf->resv); + attach->sgt = sgt; + attach->dir = DMA_BIDIRECTIONAL; ++ dma_buf_update_attachment_map_count(attach, 1 /* delta */); + } + + return attach; +@@ -761,6 +854,7 @@ dma_buf_dynamic_attach(struct dma_buf *dmabuf, struct device *dev, + if (dma_buf_is_dynamic(attach->dmabuf)) + dma_resv_unlock(attach->dmabuf->resv); + ++err_sysfs: + dma_buf_detach(dmabuf, attach); + return ERR_PTR(ret); + } +@@ -799,6 +893,7 @@ void dma_buf_detach(struct dma_buf *dmabuf, struct dma_buf_attachment *attach) + dma_resv_lock(attach->dmabuf->resv, NULL); + + dmabuf->ops->unmap_dma_buf(attach, attach->sgt, attach->dir); ++ dma_buf_update_attachment_map_count(attach, -1 /* delta */); + + if (dma_buf_is_dynamic(attach->dmabuf)) { + dma_buf_unpin(attach); +@@ -812,6 +907,7 @@ void dma_buf_detach(struct dma_buf *dmabuf, struct dma_buf_attachment *attach) + if (dmabuf->ops->detach) + dmabuf->ops->detach(dmabuf, attach); + ++ dma_buf_attach_stats_teardown(attach); + kfree(attach); + } + EXPORT_SYMBOL_GPL(dma_buf_detach); +@@ -917,6 +1013,9 @@ struct sg_table *dma_buf_map_attachment(struct dma_buf_attachment *attach, + attach->dir = direction; + } + ++ if (!IS_ERR(sg_table)) ++ dma_buf_update_attachment_map_count(attach, 1 /* delta */); ++ + return sg_table; + } + EXPORT_SYMBOL_GPL(dma_buf_map_attachment); +@@ -954,6 +1053,8 @@ void dma_buf_unmap_attachment(struct dma_buf_attachment *attach, + if (dma_buf_is_dynamic(attach->dmabuf) && + !IS_ENABLED(CONFIG_DMABUF_MOVE_NOTIFY)) + dma_buf_unpin(attach); ++ ++ dma_buf_update_attachment_map_count(attach, -1 /* delta */); + } + EXPORT_SYMBOL_GPL(dma_buf_unmap_attachment); + +@@ -1114,6 +1215,30 @@ int dma_buf_begin_cpu_access(struct dma_buf *dmabuf, + } + EXPORT_SYMBOL_GPL(dma_buf_begin_cpu_access); + ++int dma_buf_begin_cpu_access_partial(struct dma_buf *dmabuf, ++ enum dma_data_direction direction, ++ unsigned int offset, unsigned int len) ++{ ++ int ret = 0; ++ ++ if (WARN_ON(!dmabuf)) ++ return -EINVAL; ++ ++ if (dmabuf->ops->begin_cpu_access_partial) ++ ret = dmabuf->ops->begin_cpu_access_partial(dmabuf, direction, ++ offset, len); ++ ++ /* Ensure that all fences are waited upon - but we first allow ++ * the native handler the chance to do so more efficiently if it ++ * chooses. A double invocation here will be reasonably cheap no-op. ++ */ ++ if (ret == 0) ++ ret = __dma_buf_begin_cpu_access(dmabuf, direction); ++ ++ return ret; ++} ++EXPORT_SYMBOL_GPL(dma_buf_begin_cpu_access_partial); ++ + /** + * dma_buf_end_cpu_access - Must be called after accessing a dma_buf from the + * cpu in the kernel context. Calls end_cpu_access to allow exporter-specific +@@ -1140,6 +1265,21 @@ int dma_buf_end_cpu_access(struct dma_buf *dmabuf, + } + EXPORT_SYMBOL_GPL(dma_buf_end_cpu_access); + ++int dma_buf_end_cpu_access_partial(struct dma_buf *dmabuf, ++ enum dma_data_direction direction, ++ unsigned int offset, unsigned int len) ++{ ++ int ret = 0; ++ ++ WARN_ON(!dmabuf); ++ ++ if (dmabuf->ops->end_cpu_access_partial) ++ ret = dmabuf->ops->end_cpu_access_partial(dmabuf, direction, ++ offset, len); ++ ++ return ret; ++} ++EXPORT_SYMBOL_GPL(dma_buf_end_cpu_access_partial); + + /** + * dma_buf_mmap - Setup up a userspace mmap with the given vma +@@ -1268,6 +1408,32 @@ void dma_buf_vunmap(struct dma_buf *dmabuf, void *vaddr) + } + EXPORT_SYMBOL_GPL(dma_buf_vunmap); + ++int dma_buf_get_flags(struct dma_buf *dmabuf, unsigned long *flags) ++{ ++ int ret = 0; ++ ++ if (WARN_ON(!dmabuf) || !flags) ++ return -EINVAL; ++ ++ if (dmabuf->ops->get_flags) ++ ret = dmabuf->ops->get_flags(dmabuf, flags); ++ ++ return ret; ++} ++EXPORT_SYMBOL_GPL(dma_buf_get_flags); ++ ++int dma_buf_get_uuid(struct dma_buf *dmabuf, uuid_t *uuid) ++{ ++ if (WARN_ON(!dmabuf) || !uuid) ++ return -EINVAL; ++ ++ if (!dmabuf->ops->get_uuid) ++ return -ENODEV; ++ ++ return dmabuf->ops->get_uuid(dmabuf, uuid); ++} ++EXPORT_SYMBOL_GPL(dma_buf_get_uuid); ++ + #ifdef CONFIG_DEBUG_FS + static int dma_buf_debug_show(struct seq_file *s, void *unused) + { +@@ -1402,6 +1568,12 @@ static inline void dma_buf_uninit_debugfs(void) + + static int __init dma_buf_init(void) + { ++ int ret; ++ ++ ret = dma_buf_init_sysfs_statistics(); ++ if (ret) ++ return ret; ++ + dma_buf_mnt = kern_mount(&dma_buf_fs_type); + if (IS_ERR(dma_buf_mnt)) + return PTR_ERR(dma_buf_mnt); +@@ -1417,5 +1589,6 @@ static void __exit dma_buf_deinit(void) + { + dma_buf_uninit_debugfs(); + kern_unmount(dma_buf_mnt); ++ dma_buf_uninit_sysfs_statistics(); + } + __exitcall(dma_buf_deinit); +diff --git a/drivers/dma-buf/dma-fence.c b/drivers/dma-buf/dma-fence.c +index 7475e09b0680..d64fc03929be 100644 +--- a/drivers/dma-buf/dma-fence.c ++++ b/drivers/dma-buf/dma-fence.c +@@ -312,22 +312,25 @@ void __dma_fence_might_wait(void) + + + /** +- * dma_fence_signal_locked - signal completion of a fence ++ * dma_fence_signal_timestamp_locked - signal completion of a fence + * @fence: the fence to signal ++ * @timestamp: fence signal timestamp in kernel's CLOCK_MONOTONIC time domain + * + * Signal completion for software callbacks on a fence, this will unblock + * dma_fence_wait() calls and run all the callbacks added with + * dma_fence_add_callback(). Can be called multiple times, but since a fence + * can only go from the unsignaled to the signaled state and not back, it will +- * only be effective the first time. ++ * only be effective the first time. Set the timestamp provided as the fence ++ * signal timestamp. + * +- * Unlike dma_fence_signal(), this function must be called with &dma_fence.lock +- * held. ++ * Unlike dma_fence_signal_timestamp(), this function must be called with ++ * &dma_fence.lock held. + * + * Returns 0 on success and a negative error value when @fence has been + * signalled already. + */ +-int dma_fence_signal_locked(struct dma_fence *fence) ++int dma_fence_signal_timestamp_locked(struct dma_fence *fence, ++ ktime_t timestamp) + { + struct dma_fence_cb *cur, *tmp; + struct list_head cb_list; +@@ -341,7 +344,7 @@ int dma_fence_signal_locked(struct dma_fence *fence) + /* Stash the cb_list before replacing it with the timestamp */ + list_replace(&fence->cb_list, &cb_list); + +- fence->timestamp = ktime_get(); ++ fence->timestamp = timestamp; + set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags); + trace_dma_fence_signaled(fence); + +@@ -352,6 +355,59 @@ int dma_fence_signal_locked(struct dma_fence *fence) + + return 0; + } ++EXPORT_SYMBOL(dma_fence_signal_timestamp_locked); ++ ++/** ++ * dma_fence_signal_timestamp - signal completion of a fence ++ * @fence: the fence to signal ++ * @timestamp: fence signal timestamp in kernel's CLOCK_MONOTONIC time domain ++ * ++ * Signal completion for software callbacks on a fence, this will unblock ++ * dma_fence_wait() calls and run all the callbacks added with ++ * dma_fence_add_callback(). Can be called multiple times, but since a fence ++ * can only go from the unsignaled to the signaled state and not back, it will ++ * only be effective the first time. Set the timestamp provided as the fence ++ * signal timestamp. ++ * ++ * Returns 0 on success and a negative error value when @fence has been ++ * signalled already. ++ */ ++int dma_fence_signal_timestamp(struct dma_fence *fence, ktime_t timestamp) ++{ ++ unsigned long flags; ++ int ret; ++ ++ if (!fence) ++ return -EINVAL; ++ ++ spin_lock_irqsave(fence->lock, flags); ++ ret = dma_fence_signal_timestamp_locked(fence, timestamp); ++ spin_unlock_irqrestore(fence->lock, flags); ++ ++ return ret; ++} ++EXPORT_SYMBOL(dma_fence_signal_timestamp); ++ ++/** ++ * dma_fence_signal_locked - signal completion of a fence ++ * @fence: the fence to signal ++ * ++ * Signal completion for software callbacks on a fence, this will unblock ++ * dma_fence_wait() calls and run all the callbacks added with ++ * dma_fence_add_callback(). Can be called multiple times, but since a fence ++ * can only go from the unsignaled to the signaled state and not back, it will ++ * only be effective the first time. ++ * ++ * Unlike dma_fence_signal(), this function must be called with &dma_fence.lock ++ * held. ++ * ++ * Returns 0 on success and a negative error value when @fence has been ++ * signalled already. ++ */ ++int dma_fence_signal_locked(struct dma_fence *fence) ++{ ++ return dma_fence_signal_timestamp_locked(fence, ktime_get()); ++} + EXPORT_SYMBOL(dma_fence_signal_locked); + + /** +@@ -379,7 +435,7 @@ int dma_fence_signal(struct dma_fence *fence) + tmp = dma_fence_begin_signalling(); + + spin_lock_irqsave(fence->lock, flags); +- ret = dma_fence_signal_locked(fence); ++ ret = dma_fence_signal_timestamp_locked(fence, ktime_get()); + spin_unlock_irqrestore(fence->lock, flags); + + dma_fence_end_signalling(tmp); +diff --git a/drivers/dma-buf/dma-heap.c b/drivers/dma-buf/dma-heap.c +index afd22c9dbdcf..4fb22001b2ca 100644 +--- a/drivers/dma-buf/dma-heap.c ++++ b/drivers/dma-buf/dma-heap.c +@@ -30,6 +30,7 @@ + * @heap_devt heap device node + * @list list head connecting to list of heaps + * @heap_cdev heap char device ++ * @heap_dev heap device struct + * + * Represents a heap of memory from which buffers can be made. + */ +@@ -40,6 +41,8 @@ struct dma_heap { + dev_t heap_devt; + struct list_head list; + struct cdev heap_cdev; ++ struct kref refcount; ++ struct device *heap_dev; + }; + + static LIST_HEAD(heap_list); +@@ -48,20 +51,72 @@ static dev_t dma_heap_devt; + static struct class *dma_heap_class; + static DEFINE_XARRAY_ALLOC(dma_heap_minors); + +-static int dma_heap_buffer_alloc(struct dma_heap *heap, size_t len, +- unsigned int fd_flags, +- unsigned int heap_flags) ++struct dma_heap *dma_heap_find(const char *name) + { ++ struct dma_heap *h; ++ ++ mutex_lock(&heap_list_lock); ++ list_for_each_entry(h, &heap_list, list) { ++ if (!strcmp(h->name, name)) { ++ kref_get(&h->refcount); ++ mutex_unlock(&heap_list_lock); ++ return h; ++ } ++ } ++ mutex_unlock(&heap_list_lock); ++ return NULL; ++} ++EXPORT_SYMBOL_GPL(dma_heap_find); ++ ++ ++void dma_heap_buffer_free(struct dma_buf *dmabuf) ++{ ++ dma_buf_put(dmabuf); ++} ++EXPORT_SYMBOL_GPL(dma_heap_buffer_free); ++ ++struct dma_buf *dma_heap_buffer_alloc(struct dma_heap *heap, size_t len, ++ unsigned int fd_flags, ++ unsigned int heap_flags) ++{ ++ if (fd_flags & ~DMA_HEAP_VALID_FD_FLAGS) ++ return ERR_PTR(-EINVAL); ++ ++ if (heap_flags & ~DMA_HEAP_VALID_HEAP_FLAGS) ++ return ERR_PTR(-EINVAL); + /* + * Allocations from all heaps have to begin + * and end on page boundaries. + */ + len = PAGE_ALIGN(len); + if (!len) +- return -EINVAL; ++ return ERR_PTR(-EINVAL); + + return heap->ops->allocate(heap, len, fd_flags, heap_flags); + } ++EXPORT_SYMBOL_GPL(dma_heap_buffer_alloc); ++ ++int dma_heap_bufferfd_alloc(struct dma_heap *heap, size_t len, ++ unsigned int fd_flags, ++ unsigned int heap_flags) ++{ ++ struct dma_buf *dmabuf; ++ int fd; ++ ++ dmabuf = dma_heap_buffer_alloc(heap, len, fd_flags, heap_flags); ++ ++ if (IS_ERR(dmabuf)) ++ return PTR_ERR(dmabuf); ++ ++ fd = dma_buf_fd(dmabuf, fd_flags); ++ if (fd < 0) { ++ dma_buf_put(dmabuf); ++ /* just return, as put will call release and that will free */ ++ } ++ return fd; ++ ++} ++EXPORT_SYMBOL_GPL(dma_heap_bufferfd_alloc); + + static int dma_heap_open(struct inode *inode, struct file *file) + { +@@ -89,15 +144,9 @@ static long dma_heap_ioctl_allocate(struct file *file, void *data) + if (heap_allocation->fd) + return -EINVAL; + +- if (heap_allocation->fd_flags & ~DMA_HEAP_VALID_FD_FLAGS) +- return -EINVAL; +- +- if (heap_allocation->heap_flags & ~DMA_HEAP_VALID_HEAP_FLAGS) +- return -EINVAL; +- +- fd = dma_heap_buffer_alloc(heap, heap_allocation->len, +- heap_allocation->fd_flags, +- heap_allocation->heap_flags); ++ fd = dma_heap_bufferfd_alloc(heap, heap_allocation->len, ++ heap_allocation->fd_flags, ++ heap_allocation->heap_flags); + if (fd < 0) + return fd; + +@@ -189,11 +238,64 @@ void *dma_heap_get_drvdata(struct dma_heap *heap) + { + return heap->priv; + } ++EXPORT_SYMBOL_GPL(dma_heap_get_drvdata); ++ ++static void dma_heap_release(struct kref *ref) ++{ ++ struct dma_heap *heap = container_of(ref, struct dma_heap, refcount); ++ int minor = MINOR(heap->heap_devt); ++ ++ /* Note, we already holding the heap_list_lock here */ ++ list_del(&heap->list); ++ ++ device_destroy(dma_heap_class, heap->heap_devt); ++ cdev_del(&heap->heap_cdev); ++ xa_erase(&dma_heap_minors, minor); ++ ++ kfree(heap); ++} ++ ++void dma_heap_put(struct dma_heap *h) ++{ ++ /* ++ * Take the heap_list_lock now to avoid racing with code ++ * scanning the list and then taking a kref. ++ */ ++ mutex_lock(&heap_list_lock); ++ kref_put(&h->refcount, dma_heap_release); ++ mutex_unlock(&heap_list_lock); ++} ++EXPORT_SYMBOL_GPL(dma_heap_put); ++ ++/** ++ * dma_heap_get_dev() - get device struct for the heap ++ * @heap: DMA-Heap to retrieve device struct from ++ * ++ * Returns: ++ * The device struct for the heap. ++ */ ++struct device *dma_heap_get_dev(struct dma_heap *heap) ++{ ++ return heap->heap_dev; ++} ++EXPORT_SYMBOL_GPL(dma_heap_get_dev); ++ ++/** ++ * dma_heap_get_name() - get heap name ++ * @heap: DMA-Heap to retrieve private data for ++ * ++ * Returns: ++ * The char* for the heap name. ++ */ ++const char *dma_heap_get_name(struct dma_heap *heap) ++{ ++ return heap->name; ++} ++EXPORT_SYMBOL_GPL(dma_heap_get_name); + + struct dma_heap *dma_heap_add(const struct dma_heap_export_info *exp_info) + { +- struct dma_heap *heap, *h, *err_ret; +- struct device *dev_ret; ++ struct dma_heap *heap, *err_ret; + unsigned int minor; + int ret; + +@@ -208,21 +310,19 @@ struct dma_heap *dma_heap_add(const struct dma_heap_export_info *exp_info) + } + + /* check the name is unique */ +- mutex_lock(&heap_list_lock); +- list_for_each_entry(h, &heap_list, list) { +- if (!strcmp(h->name, exp_info->name)) { +- mutex_unlock(&heap_list_lock); +- pr_err("dma_heap: Already registered heap named %s\n", +- exp_info->name); +- return ERR_PTR(-EINVAL); +- } ++ heap = dma_heap_find(exp_info->name); ++ if (heap) { ++ pr_err("dma_heap: Already registered heap named %s\n", ++ exp_info->name); ++ dma_heap_put(heap); ++ return ERR_PTR(-EINVAL); + } +- mutex_unlock(&heap_list_lock); + + heap = kzalloc(sizeof(*heap), GFP_KERNEL); + if (!heap) + return ERR_PTR(-ENOMEM); + ++ kref_init(&heap->refcount); + heap->name = exp_info->name; + heap->ops = exp_info->ops; + heap->priv = exp_info->priv; +@@ -247,16 +347,20 @@ struct dma_heap *dma_heap_add(const struct dma_heap_export_info *exp_info) + goto err1; + } + +- dev_ret = device_create(dma_heap_class, +- NULL, +- heap->heap_devt, +- NULL, +- heap->name); +- if (IS_ERR(dev_ret)) { ++ heap->heap_dev = device_create(dma_heap_class, ++ NULL, ++ heap->heap_devt, ++ NULL, ++ heap->name); ++ if (IS_ERR(heap->heap_dev)) { + pr_err("dma_heap: Unable to create device\n"); +- err_ret = ERR_CAST(dev_ret); ++ err_ret = ERR_CAST(heap->heap_dev); + goto err2; + } ++ ++ /* Make sure it doesn't disappear on us */ ++ heap->heap_dev = get_device(heap->heap_dev); ++ + /* Add heap to the list */ + mutex_lock(&heap_list_lock); + list_add(&heap->list, &heap_list); +@@ -272,27 +376,88 @@ struct dma_heap *dma_heap_add(const struct dma_heap_export_info *exp_info) + kfree(heap); + return err_ret; + } ++EXPORT_SYMBOL_GPL(dma_heap_add); + + static char *dma_heap_devnode(struct device *dev, umode_t *mode) + { + return kasprintf(GFP_KERNEL, "dma_heap/%s", dev_name(dev)); + } + ++static ssize_t total_pools_kb_show(struct kobject *kobj, ++ struct kobj_attribute *attr, char *buf) ++{ ++ struct dma_heap *heap; ++ u64 total_pool_size = 0; ++ ++ mutex_lock(&heap_list_lock); ++ list_for_each_entry(heap, &heap_list, list) { ++ if (heap->ops->get_pool_size) ++ total_pool_size += heap->ops->get_pool_size(heap); ++ } ++ mutex_unlock(&heap_list_lock); ++ ++ return sysfs_emit(buf, "%llu\n", total_pool_size / 1024); ++} ++ ++static struct kobj_attribute total_pools_kb_attr = ++ __ATTR_RO(total_pools_kb); ++ ++static struct attribute *dma_heap_sysfs_attrs[] = { ++ &total_pools_kb_attr.attr, ++ NULL, ++}; ++ ++ATTRIBUTE_GROUPS(dma_heap_sysfs); ++ ++static struct kobject *dma_heap_kobject; ++ ++static int dma_heap_sysfs_setup(void) ++{ ++ int ret; ++ ++ dma_heap_kobject = kobject_create_and_add("dma_heap", kernel_kobj); ++ if (!dma_heap_kobject) ++ return -ENOMEM; ++ ++ ret = sysfs_create_groups(dma_heap_kobject, dma_heap_sysfs_groups); ++ if (ret) { ++ kobject_put(dma_heap_kobject); ++ return ret; ++ } ++ ++ return 0; ++} ++ ++static void dma_heap_sysfs_teardown(void) ++{ ++ kobject_put(dma_heap_kobject); ++} ++ + static int dma_heap_init(void) + { + int ret; + +- ret = alloc_chrdev_region(&dma_heap_devt, 0, NUM_HEAP_MINORS, DEVNAME); ++ ret = dma_heap_sysfs_setup(); + if (ret) + return ret; + ++ ret = alloc_chrdev_region(&dma_heap_devt, 0, NUM_HEAP_MINORS, DEVNAME); ++ if (ret) ++ goto err_chrdev; ++ + dma_heap_class = class_create(THIS_MODULE, DEVNAME); + if (IS_ERR(dma_heap_class)) { +- unregister_chrdev_region(dma_heap_devt, NUM_HEAP_MINORS); +- return PTR_ERR(dma_heap_class); ++ ret = PTR_ERR(dma_heap_class); ++ goto err_class; + } + dma_heap_class->devnode = dma_heap_devnode; + + return 0; ++ ++err_class: ++ unregister_chrdev_region(dma_heap_devt, NUM_HEAP_MINORS); ++err_chrdev: ++ dma_heap_sysfs_teardown(); ++ return ret; + } + subsys_initcall(dma_heap_init); +diff --git a/drivers/dma-buf/heaps/Kconfig b/drivers/dma-buf/heaps/Kconfig +index a5eef06c4226..ff52efa83f39 100644 +--- a/drivers/dma-buf/heaps/Kconfig ++++ b/drivers/dma-buf/heaps/Kconfig +@@ -1,12 +1,22 @@ ++menuconfig DMABUF_HEAPS_DEFERRED_FREE ++ bool "DMA-BUF heaps deferred-free library" ++ help ++ Choose this option to enable the DMA-BUF heaps deferred-free library. ++ ++menuconfig DMABUF_HEAPS_PAGE_POOL ++ bool "DMA-BUF heaps page-pool library" ++ help ++ Choose this option to enable the DMA-BUF heaps page-pool library. ++ + config DMABUF_HEAPS_SYSTEM +- bool "DMA-BUF System Heap" +- depends on DMABUF_HEAPS ++ tristate "DMA-BUF System Heap" ++ depends on DMABUF_HEAPS && DMABUF_HEAPS_DEFERRED_FREE && DMABUF_HEAPS_PAGE_POOL + help + Choose this option to enable the system dmabuf heap. The system heap + is backed by pages from the buddy allocator. If in doubt, say Y. + + config DMABUF_HEAPS_CMA +- bool "DMA-BUF CMA Heap" ++ tristate "DMA-BUF CMA Heap" + depends on DMABUF_HEAPS && DMA_CMA + help + Choose this option to enable dma-buf CMA heap. This heap is backed +diff --git a/drivers/dma-buf/heaps/Makefile b/drivers/dma-buf/heaps/Makefile +index 6e54cdec3da0..4d4cd94a3a4a 100644 +--- a/drivers/dma-buf/heaps/Makefile ++++ b/drivers/dma-buf/heaps/Makefile +@@ -1,4 +1,5 @@ + # SPDX-License-Identifier: GPL-2.0 +-obj-y += heap-helpers.o ++obj-$(CONFIG_DMABUF_HEAPS_DEFERRED_FREE) += deferred-free-helper.o ++obj-$(CONFIG_DMABUF_HEAPS_PAGE_POOL) += page_pool.o + obj-$(CONFIG_DMABUF_HEAPS_SYSTEM) += system_heap.o + obj-$(CONFIG_DMABUF_HEAPS_CMA) += cma_heap.o +diff --git a/drivers/dma-buf/heaps/cma_heap.c b/drivers/dma-buf/heaps/cma_heap.c +index e55384dc115b..4931578df815 100644 +--- a/drivers/dma-buf/heaps/cma_heap.c ++++ b/drivers/dma-buf/heaps/cma_heap.c +@@ -2,76 +2,304 @@ + /* + * DMABUF CMA heap exporter + * +- * Copyright (C) 2012, 2019 Linaro Ltd. ++ * Copyright (C) 2012, 2019, 2020 Linaro Ltd. + * Author: for ST-Ericsson. ++ * ++ * Also utilizing parts of Andrew Davis' SRAM heap: ++ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/ ++ * Andrew F. Davis + */ +- + #include +-#include + #include + #include + #include + #include +-#include + #include ++#include ++#include + #include +-#include + #include +-#include ++#include ++#include + +-#include "heap-helpers.h" + + struct cma_heap { + struct dma_heap *heap; + struct cma *cma; + }; + +-static void cma_heap_free(struct heap_helper_buffer *buffer) ++struct cma_heap_buffer { ++ struct cma_heap *heap; ++ struct list_head attachments; ++ struct mutex lock; ++ unsigned long len; ++ struct page *cma_pages; ++ struct page **pages; ++ pgoff_t pagecount; ++ int vmap_cnt; ++ void *vaddr; ++}; ++ ++struct dma_heap_attachment { ++ struct device *dev; ++ struct sg_table table; ++ struct list_head list; ++ bool mapped; ++}; ++ ++static int cma_heap_attach(struct dma_buf *dmabuf, ++ struct dma_buf_attachment *attachment) ++{ ++ struct cma_heap_buffer *buffer = dmabuf->priv; ++ struct dma_heap_attachment *a; ++ int ret; ++ ++ a = kzalloc(sizeof(*a), GFP_KERNEL); ++ if (!a) ++ return -ENOMEM; ++ ++ ret = sg_alloc_table_from_pages(&a->table, buffer->pages, ++ buffer->pagecount, 0, ++ buffer->pagecount << PAGE_SHIFT, ++ GFP_KERNEL); ++ if (ret) { ++ kfree(a); ++ return ret; ++ } ++ ++ a->dev = attachment->dev; ++ INIT_LIST_HEAD(&a->list); ++ a->mapped = false; ++ ++ attachment->priv = a; ++ ++ mutex_lock(&buffer->lock); ++ list_add(&a->list, &buffer->attachments); ++ mutex_unlock(&buffer->lock); ++ ++ return 0; ++} ++ ++static void cma_heap_detach(struct dma_buf *dmabuf, ++ struct dma_buf_attachment *attachment) ++{ ++ struct cma_heap_buffer *buffer = dmabuf->priv; ++ struct dma_heap_attachment *a = attachment->priv; ++ ++ mutex_lock(&buffer->lock); ++ list_del(&a->list); ++ mutex_unlock(&buffer->lock); ++ ++ sg_free_table(&a->table); ++ kfree(a); ++} ++ ++static struct sg_table *cma_heap_map_dma_buf(struct dma_buf_attachment *attachment, ++ enum dma_data_direction direction) + { +- struct cma_heap *cma_heap = dma_heap_get_drvdata(buffer->heap); +- unsigned long nr_pages = buffer->pagecount; +- struct page *cma_pages = buffer->priv_virt; ++ struct dma_heap_attachment *a = attachment->priv; ++ struct sg_table *table = &a->table; ++ int ret; ++ ++ ret = dma_map_sgtable(attachment->dev, table, direction, 0); ++ if (ret) ++ return ERR_PTR(-ENOMEM); ++ a->mapped = true; ++ return table; ++} ++ ++static void cma_heap_unmap_dma_buf(struct dma_buf_attachment *attachment, ++ struct sg_table *table, ++ enum dma_data_direction direction) ++{ ++ struct dma_heap_attachment *a = attachment->priv; ++ ++ a->mapped = false; ++ dma_unmap_sgtable(attachment->dev, table, direction, 0); ++} ++ ++static int cma_heap_dma_buf_begin_cpu_access(struct dma_buf *dmabuf, ++ enum dma_data_direction direction) ++{ ++ struct cma_heap_buffer *buffer = dmabuf->priv; ++ struct dma_heap_attachment *a; ++ ++ if (buffer->vmap_cnt) ++ invalidate_kernel_vmap_range(buffer->vaddr, buffer->len); ++ ++ mutex_lock(&buffer->lock); ++ list_for_each_entry(a, &buffer->attachments, list) { ++ if (!a->mapped) ++ continue; ++ dma_sync_sgtable_for_cpu(a->dev, &a->table, direction); ++ } ++ mutex_unlock(&buffer->lock); ++ ++ return 0; ++} ++ ++static int cma_heap_dma_buf_end_cpu_access(struct dma_buf *dmabuf, ++ enum dma_data_direction direction) ++{ ++ struct cma_heap_buffer *buffer = dmabuf->priv; ++ struct dma_heap_attachment *a; ++ ++ if (buffer->vmap_cnt) ++ flush_kernel_vmap_range(buffer->vaddr, buffer->len); ++ ++ mutex_lock(&buffer->lock); ++ list_for_each_entry(a, &buffer->attachments, list) { ++ if (!a->mapped) ++ continue; ++ dma_sync_sgtable_for_device(a->dev, &a->table, direction); ++ } ++ mutex_unlock(&buffer->lock); ++ ++ return 0; ++} ++ ++static vm_fault_t cma_heap_vm_fault(struct vm_fault *vmf) ++{ ++ struct vm_area_struct *vma = vmf->vma; ++ struct cma_heap_buffer *buffer = vma->vm_private_data; ++ ++ if (vmf->pgoff > buffer->pagecount) ++ return VM_FAULT_SIGBUS; ++ ++ vmf->page = buffer->pages[vmf->pgoff]; ++ get_page(vmf->page); ++ ++ return 0; ++} ++ ++static const struct vm_operations_struct dma_heap_vm_ops = { ++ .fault = cma_heap_vm_fault, ++}; ++ ++static int cma_heap_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma) ++{ ++ struct cma_heap_buffer *buffer = dmabuf->priv; ++ ++ if ((vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) == 0) ++ return -EINVAL; ++ ++ vma->vm_ops = &dma_heap_vm_ops; ++ vma->vm_private_data = buffer; ++ ++ return 0; ++} ++ ++static void *cma_heap_do_vmap(struct cma_heap_buffer *buffer) ++{ ++ void *vaddr; ++ ++ vaddr = vmap(buffer->pages, buffer->pagecount, VM_MAP, PAGE_KERNEL); ++ if (!vaddr) ++ return ERR_PTR(-ENOMEM); ++ ++ return vaddr; ++} ++ ++static void *cma_heap_vmap(struct dma_buf *dmabuf) ++{ ++ struct cma_heap_buffer *buffer = dmabuf->priv; ++ void *vaddr; ++ ++ mutex_lock(&buffer->lock); ++ if (buffer->vmap_cnt) { ++ buffer->vmap_cnt++; ++ vaddr = buffer->vaddr; ++ goto out; ++ } ++ ++ vaddr = cma_heap_do_vmap(buffer); ++ if (IS_ERR(vaddr)) ++ goto out; ++ ++ buffer->vaddr = vaddr; ++ buffer->vmap_cnt++; ++out: ++ mutex_unlock(&buffer->lock); ++ ++ return vaddr; ++} ++ ++static void cma_heap_vunmap(struct dma_buf *dmabuf, void *vaddr) ++{ ++ struct cma_heap_buffer *buffer = dmabuf->priv; ++ ++ mutex_lock(&buffer->lock); ++ if (!--buffer->vmap_cnt) { ++ vunmap(buffer->vaddr); ++ buffer->vaddr = NULL; ++ } ++ mutex_unlock(&buffer->lock); ++} ++ ++static void cma_heap_dma_buf_release(struct dma_buf *dmabuf) ++{ ++ struct cma_heap_buffer *buffer = dmabuf->priv; ++ struct cma_heap *cma_heap = buffer->heap; ++ ++ if (buffer->vmap_cnt > 0) { ++ WARN(1, "%s: buffer still mapped in the kernel\n", __func__); ++ vunmap(buffer->vaddr); ++ } + + /* free page list */ + kfree(buffer->pages); + /* release memory */ +- cma_release(cma_heap->cma, cma_pages, nr_pages); ++ cma_release(cma_heap->cma, buffer->cma_pages, buffer->pagecount); + kfree(buffer); + } + +-/* dmabuf heap CMA operations functions */ +-static int cma_heap_allocate(struct dma_heap *heap, +- unsigned long len, +- unsigned long fd_flags, +- unsigned long heap_flags) ++static const struct dma_buf_ops cma_heap_buf_ops = { ++ .attach = cma_heap_attach, ++ .detach = cma_heap_detach, ++ .map_dma_buf = cma_heap_map_dma_buf, ++ .unmap_dma_buf = cma_heap_unmap_dma_buf, ++ .begin_cpu_access = cma_heap_dma_buf_begin_cpu_access, ++ .end_cpu_access = cma_heap_dma_buf_end_cpu_access, ++ .mmap = cma_heap_mmap, ++ .vmap = cma_heap_vmap, ++ .vunmap = cma_heap_vunmap, ++ .release = cma_heap_dma_buf_release, ++}; ++ ++static struct dma_buf *cma_heap_allocate(struct dma_heap *heap, ++ unsigned long len, ++ unsigned long fd_flags, ++ unsigned long heap_flags) + { + struct cma_heap *cma_heap = dma_heap_get_drvdata(heap); +- struct heap_helper_buffer *helper_buffer; +- struct page *cma_pages; ++ struct cma_heap_buffer *buffer; ++ DEFINE_DMA_BUF_EXPORT_INFO(exp_info); + size_t size = PAGE_ALIGN(len); +- unsigned long nr_pages = size >> PAGE_SHIFT; ++ pgoff_t pagecount = size >> PAGE_SHIFT; + unsigned long align = get_order(size); ++ struct page *cma_pages; + struct dma_buf *dmabuf; + int ret = -ENOMEM; + pgoff_t pg; + +- if (align > CONFIG_CMA_ALIGNMENT) +- align = CONFIG_CMA_ALIGNMENT; ++ buffer = kzalloc(sizeof(*buffer), GFP_KERNEL); ++ if (!buffer) ++ return ERR_PTR(-ENOMEM); + +- helper_buffer = kzalloc(sizeof(*helper_buffer), GFP_KERNEL); +- if (!helper_buffer) +- return -ENOMEM; ++ INIT_LIST_HEAD(&buffer->attachments); ++ mutex_init(&buffer->lock); ++ buffer->len = size; + +- init_heap_helper_buffer(helper_buffer, cma_heap_free); +- helper_buffer->heap = heap; +- helper_buffer->size = len; ++ if (align > CONFIG_CMA_ALIGNMENT) ++ align = CONFIG_CMA_ALIGNMENT; + +- cma_pages = cma_alloc(cma_heap->cma, nr_pages, align, false); ++ cma_pages = cma_alloc(cma_heap->cma, pagecount, align, GFP_KERNEL); + if (!cma_pages) +- goto free_buf; ++ goto free_buffer; + ++ /* Clear the cma pages */ + if (PageHighMem(cma_pages)) { +- unsigned long nr_clear_pages = nr_pages; ++ unsigned long nr_clear_pages = pagecount; + struct page *page = cma_pages; + + while (nr_clear_pages > 0) { +@@ -85,7 +313,6 @@ static int cma_heap_allocate(struct dma_heap *heap, + */ + if (fatal_signal_pending(current)) + goto free_cma; +- + page++; + nr_clear_pages--; + } +@@ -93,44 +320,41 @@ static int cma_heap_allocate(struct dma_heap *heap, + memset(page_address(cma_pages), 0, size); + } + +- helper_buffer->pagecount = nr_pages; +- helper_buffer->pages = kmalloc_array(helper_buffer->pagecount, +- sizeof(*helper_buffer->pages), +- GFP_KERNEL); +- if (!helper_buffer->pages) { ++ buffer->pages = kmalloc_array(pagecount, sizeof(*buffer->pages), GFP_KERNEL); ++ if (!buffer->pages) { + ret = -ENOMEM; + goto free_cma; + } + +- for (pg = 0; pg < helper_buffer->pagecount; pg++) +- helper_buffer->pages[pg] = &cma_pages[pg]; ++ for (pg = 0; pg < pagecount; pg++) ++ buffer->pages[pg] = &cma_pages[pg]; ++ ++ buffer->cma_pages = cma_pages; ++ buffer->heap = cma_heap; ++ buffer->pagecount = pagecount; + + /* create the dmabuf */ +- dmabuf = heap_helper_export_dmabuf(helper_buffer, fd_flags); ++ exp_info.exp_name = dma_heap_get_name(heap); ++ exp_info.ops = &cma_heap_buf_ops; ++ exp_info.size = buffer->len; ++ exp_info.flags = fd_flags; ++ exp_info.priv = buffer; ++ dmabuf = dma_buf_export(&exp_info); + if (IS_ERR(dmabuf)) { + ret = PTR_ERR(dmabuf); + goto free_pages; + } + +- helper_buffer->dmabuf = dmabuf; +- helper_buffer->priv_virt = cma_pages; +- +- ret = dma_buf_fd(dmabuf, fd_flags); +- if (ret < 0) { +- dma_buf_put(dmabuf); +- /* just return, as put will call release and that will free */ +- return ret; +- } +- +- return ret; ++ return dmabuf; + + free_pages: +- kfree(helper_buffer->pages); ++ kfree(buffer->pages); + free_cma: +- cma_release(cma_heap->cma, cma_pages, nr_pages); +-free_buf: +- kfree(helper_buffer); +- return ret; ++ cma_release(cma_heap->cma, cma_pages, pagecount); ++free_buffer: ++ kfree(buffer); ++ ++ return ERR_PTR(ret); + } + + static const struct dma_heap_ops cma_heap_ops = { +diff --git a/drivers/dma-buf/heaps/deferred-free-helper.c b/drivers/dma-buf/heaps/deferred-free-helper.c +new file mode 100755 +index 000000000000..e19c8b68dfeb +--- /dev/null ++++ b/drivers/dma-buf/heaps/deferred-free-helper.c +@@ -0,0 +1,138 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * Deferred dmabuf freeing helper ++ * ++ * Copyright (C) 2020 Linaro, Ltd. ++ * ++ * Based on the ION page pool code ++ * Copyright (C) 2011 Google, Inc. ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++ ++#include "deferred-free-helper.h" ++ ++static LIST_HEAD(free_list); ++static size_t list_nr_pages; ++wait_queue_head_t freelist_waitqueue; ++struct task_struct *freelist_task; ++static DEFINE_SPINLOCK(free_list_lock); ++ ++void deferred_free(struct deferred_freelist_item *item, ++ void (*free)(struct deferred_freelist_item*, ++ enum df_reason), ++ size_t nr_pages) ++{ ++ unsigned long flags; ++ ++ INIT_LIST_HEAD(&item->list); ++ item->nr_pages = nr_pages; ++ item->free = free; ++ ++ spin_lock_irqsave(&free_list_lock, flags); ++ list_add(&item->list, &free_list); ++ list_nr_pages += nr_pages; ++ spin_unlock_irqrestore(&free_list_lock, flags); ++ wake_up(&freelist_waitqueue); ++} ++EXPORT_SYMBOL_GPL(deferred_free); ++ ++static size_t free_one_item(enum df_reason reason) ++{ ++ unsigned long flags; ++ size_t nr_pages; ++ struct deferred_freelist_item *item; ++ ++ spin_lock_irqsave(&free_list_lock, flags); ++ if (list_empty(&free_list)) { ++ spin_unlock_irqrestore(&free_list_lock, flags); ++ return 0; ++ } ++ item = list_first_entry(&free_list, struct deferred_freelist_item, list); ++ list_del(&item->list); ++ nr_pages = item->nr_pages; ++ list_nr_pages -= nr_pages; ++ spin_unlock_irqrestore(&free_list_lock, flags); ++ ++ item->free(item, reason); ++ return nr_pages; ++} ++ ++static unsigned long get_freelist_nr_pages(void) ++{ ++ unsigned long nr_pages; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&free_list_lock, flags); ++ nr_pages = list_nr_pages; ++ spin_unlock_irqrestore(&free_list_lock, flags); ++ return nr_pages; ++} ++ ++static unsigned long freelist_shrink_count(struct shrinker *shrinker, ++ struct shrink_control *sc) ++{ ++ return get_freelist_nr_pages(); ++} ++ ++static unsigned long freelist_shrink_scan(struct shrinker *shrinker, ++ struct shrink_control *sc) ++{ ++ unsigned long total_freed = 0; ++ ++ if (sc->nr_to_scan == 0) ++ return 0; ++ ++ while (total_freed < sc->nr_to_scan) { ++ size_t pages_freed = free_one_item(DF_UNDER_PRESSURE); ++ ++ if (!pages_freed) ++ break; ++ ++ total_freed += pages_freed; ++ } ++ ++ return total_freed; ++} ++ ++static struct shrinker freelist_shrinker = { ++ .count_objects = freelist_shrink_count, ++ .scan_objects = freelist_shrink_scan, ++ .seeks = DEFAULT_SEEKS, ++ .batch = 0, ++}; ++ ++static int deferred_free_thread(void *data) ++{ ++ while (true) { ++ wait_event_freezable(freelist_waitqueue, ++ get_freelist_nr_pages() > 0); ++ ++ free_one_item(DF_NORMAL); ++ } ++ ++ return 0; ++} ++ ++static int deferred_freelist_init(void) ++{ ++ list_nr_pages = 0; ++ ++ init_waitqueue_head(&freelist_waitqueue); ++ freelist_task = kthread_run(deferred_free_thread, NULL, ++ "%s", "dmabuf-deferred-free-worker"); ++ if (IS_ERR(freelist_task)) { ++ pr_err("Creating thread for deferred free failed\n"); ++ return -1; ++ } ++ sched_set_normal(freelist_task, 19); ++ ++ return register_shrinker(&freelist_shrinker); ++} ++module_init(deferred_freelist_init); ++MODULE_LICENSE("GPL v2"); ++ +diff --git a/drivers/dma-buf/heaps/deferred-free-helper.h b/drivers/dma-buf/heaps/deferred-free-helper.h +new file mode 100755 +index 000000000000..11940328ce3f +--- /dev/null ++++ b/drivers/dma-buf/heaps/deferred-free-helper.h +@@ -0,0 +1,55 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++ ++#ifndef DEFERRED_FREE_HELPER_H ++#define DEFERRED_FREE_HELPER_H ++ ++/** ++ * df_reason - enum for reason why item was freed ++ * ++ * This provides a reason for why the free function was called ++ * on the item. This is useful when deferred_free is used in ++ * combination with a pagepool, so under pressure the page can ++ * be immediately freed. ++ * ++ * DF_NORMAL: Normal deferred free ++ * ++ * DF_UNDER_PRESSURE: Free was called because the system ++ * is under memory pressure. Usually ++ * from a shrinker. Avoid allocating ++ * memory in the free call, as it may ++ * fail. ++ */ ++enum df_reason { ++ DF_NORMAL, ++ DF_UNDER_PRESSURE, ++}; ++ ++/** ++ * deferred_freelist_item - item structure for deferred freelist ++ * ++ * This is to be added to the structure for whatever you want to ++ * defer freeing on. ++ * ++ * @nr_pages: number of pages used by item to be freed ++ * @free: function pointer to be called when freeing the item ++ * @list: list entry for the deferred list ++ */ ++struct deferred_freelist_item { ++ size_t nr_pages; ++ void (*free)(struct deferred_freelist_item *i, ++ enum df_reason reason); ++ struct list_head list; ++}; ++ ++/** ++ * deferred_free - call to add item to the deferred free list ++ * ++ * @item: Pointer to deferred_freelist_item field of a structure ++ * @free: Function pointer to the free call ++ * @nr_pages: number of pages to be freed ++ */ ++void deferred_free(struct deferred_freelist_item *item, ++ void (*free)(struct deferred_freelist_item *i, ++ enum df_reason reason), ++ size_t nr_pages); ++#endif +diff --git a/drivers/dma-buf/heaps/heap-helpers.c b/drivers/dma-buf/heaps/heap-helpers.c +deleted file mode 100644 +index d0696cf937af..000000000000 +--- a/drivers/dma-buf/heaps/heap-helpers.c ++++ /dev/null +@@ -1,270 +0,0 @@ +-// SPDX-License-Identifier: GPL-2.0 +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +- +-#include "heap-helpers.h" +- +-void init_heap_helper_buffer(struct heap_helper_buffer *buffer, +- void (*free)(struct heap_helper_buffer *)) +-{ +- buffer->priv_virt = NULL; +- mutex_init(&buffer->lock); +- buffer->vmap_cnt = 0; +- buffer->vaddr = NULL; +- buffer->pagecount = 0; +- buffer->pages = NULL; +- INIT_LIST_HEAD(&buffer->attachments); +- buffer->free = free; +-} +- +-struct dma_buf *heap_helper_export_dmabuf(struct heap_helper_buffer *buffer, +- int fd_flags) +-{ +- DEFINE_DMA_BUF_EXPORT_INFO(exp_info); +- +- exp_info.ops = &heap_helper_ops; +- exp_info.size = buffer->size; +- exp_info.flags = fd_flags; +- exp_info.priv = buffer; +- +- return dma_buf_export(&exp_info); +-} +- +-static void *dma_heap_map_kernel(struct heap_helper_buffer *buffer) +-{ +- void *vaddr; +- +- vaddr = vmap(buffer->pages, buffer->pagecount, VM_MAP, PAGE_KERNEL); +- if (!vaddr) +- return ERR_PTR(-ENOMEM); +- +- return vaddr; +-} +- +-static void dma_heap_buffer_destroy(struct heap_helper_buffer *buffer) +-{ +- if (buffer->vmap_cnt > 0) { +- WARN(1, "%s: buffer still mapped in the kernel\n", __func__); +- vunmap(buffer->vaddr); +- } +- +- buffer->free(buffer); +-} +- +-static void *dma_heap_buffer_vmap_get(struct heap_helper_buffer *buffer) +-{ +- void *vaddr; +- +- if (buffer->vmap_cnt) { +- buffer->vmap_cnt++; +- return buffer->vaddr; +- } +- vaddr = dma_heap_map_kernel(buffer); +- if (IS_ERR(vaddr)) +- return vaddr; +- buffer->vaddr = vaddr; +- buffer->vmap_cnt++; +- return vaddr; +-} +- +-static void dma_heap_buffer_vmap_put(struct heap_helper_buffer *buffer) +-{ +- if (!--buffer->vmap_cnt) { +- vunmap(buffer->vaddr); +- buffer->vaddr = NULL; +- } +-} +- +-struct dma_heaps_attachment { +- struct device *dev; +- struct sg_table table; +- struct list_head list; +-}; +- +-static int dma_heap_attach(struct dma_buf *dmabuf, +- struct dma_buf_attachment *attachment) +-{ +- struct dma_heaps_attachment *a; +- struct heap_helper_buffer *buffer = dmabuf->priv; +- int ret; +- +- a = kzalloc(sizeof(*a), GFP_KERNEL); +- if (!a) +- return -ENOMEM; +- +- ret = sg_alloc_table_from_pages(&a->table, buffer->pages, +- buffer->pagecount, 0, +- buffer->pagecount << PAGE_SHIFT, +- GFP_KERNEL); +- if (ret) { +- kfree(a); +- return ret; +- } +- +- a->dev = attachment->dev; +- INIT_LIST_HEAD(&a->list); +- +- attachment->priv = a; +- +- mutex_lock(&buffer->lock); +- list_add(&a->list, &buffer->attachments); +- mutex_unlock(&buffer->lock); +- +- return 0; +-} +- +-static void dma_heap_detach(struct dma_buf *dmabuf, +- struct dma_buf_attachment *attachment) +-{ +- struct dma_heaps_attachment *a = attachment->priv; +- struct heap_helper_buffer *buffer = dmabuf->priv; +- +- mutex_lock(&buffer->lock); +- list_del(&a->list); +- mutex_unlock(&buffer->lock); +- +- sg_free_table(&a->table); +- kfree(a); +-} +- +-static +-struct sg_table *dma_heap_map_dma_buf(struct dma_buf_attachment *attachment, +- enum dma_data_direction direction) +-{ +- struct dma_heaps_attachment *a = attachment->priv; +- struct sg_table *table = &a->table; +- int ret; +- +- ret = dma_map_sgtable(attachment->dev, table, direction, 0); +- if (ret) +- table = ERR_PTR(ret); +- return table; +-} +- +-static void dma_heap_unmap_dma_buf(struct dma_buf_attachment *attachment, +- struct sg_table *table, +- enum dma_data_direction direction) +-{ +- dma_unmap_sgtable(attachment->dev, table, direction, 0); +-} +- +-static vm_fault_t dma_heap_vm_fault(struct vm_fault *vmf) +-{ +- struct vm_area_struct *vma = vmf->vma; +- struct heap_helper_buffer *buffer = vma->vm_private_data; +- +- if (vmf->pgoff > buffer->pagecount) +- return VM_FAULT_SIGBUS; +- +- vmf->page = buffer->pages[vmf->pgoff]; +- get_page(vmf->page); +- +- return 0; +-} +- +-static const struct vm_operations_struct dma_heap_vm_ops = { +- .fault = dma_heap_vm_fault, +-}; +- +-static int dma_heap_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma) +-{ +- struct heap_helper_buffer *buffer = dmabuf->priv; +- +- if ((vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) == 0) +- return -EINVAL; +- +- vma->vm_ops = &dma_heap_vm_ops; +- vma->vm_private_data = buffer; +- +- return 0; +-} +- +-static void dma_heap_dma_buf_release(struct dma_buf *dmabuf) +-{ +- struct heap_helper_buffer *buffer = dmabuf->priv; +- +- dma_heap_buffer_destroy(buffer); +-} +- +-static int dma_heap_dma_buf_begin_cpu_access(struct dma_buf *dmabuf, +- enum dma_data_direction direction) +-{ +- struct heap_helper_buffer *buffer = dmabuf->priv; +- struct dma_heaps_attachment *a; +- int ret = 0; +- +- mutex_lock(&buffer->lock); +- +- if (buffer->vmap_cnt) +- invalidate_kernel_vmap_range(buffer->vaddr, buffer->size); +- +- list_for_each_entry(a, &buffer->attachments, list) { +- dma_sync_sg_for_cpu(a->dev, a->table.sgl, a->table.nents, +- direction); +- } +- mutex_unlock(&buffer->lock); +- +- return ret; +-} +- +-static int dma_heap_dma_buf_end_cpu_access(struct dma_buf *dmabuf, +- enum dma_data_direction direction) +-{ +- struct heap_helper_buffer *buffer = dmabuf->priv; +- struct dma_heaps_attachment *a; +- +- mutex_lock(&buffer->lock); +- +- if (buffer->vmap_cnt) +- flush_kernel_vmap_range(buffer->vaddr, buffer->size); +- +- list_for_each_entry(a, &buffer->attachments, list) { +- dma_sync_sg_for_device(a->dev, a->table.sgl, a->table.nents, +- direction); +- } +- mutex_unlock(&buffer->lock); +- +- return 0; +-} +- +-static void *dma_heap_dma_buf_vmap(struct dma_buf *dmabuf) +-{ +- struct heap_helper_buffer *buffer = dmabuf->priv; +- void *vaddr; +- +- mutex_lock(&buffer->lock); +- vaddr = dma_heap_buffer_vmap_get(buffer); +- mutex_unlock(&buffer->lock); +- +- return vaddr; +-} +- +-static void dma_heap_dma_buf_vunmap(struct dma_buf *dmabuf, void *vaddr) +-{ +- struct heap_helper_buffer *buffer = dmabuf->priv; +- +- mutex_lock(&buffer->lock); +- dma_heap_buffer_vmap_put(buffer); +- mutex_unlock(&buffer->lock); +-} +- +-const struct dma_buf_ops heap_helper_ops = { +- .map_dma_buf = dma_heap_map_dma_buf, +- .unmap_dma_buf = dma_heap_unmap_dma_buf, +- .mmap = dma_heap_mmap, +- .release = dma_heap_dma_buf_release, +- .attach = dma_heap_attach, +- .detach = dma_heap_detach, +- .begin_cpu_access = dma_heap_dma_buf_begin_cpu_access, +- .end_cpu_access = dma_heap_dma_buf_end_cpu_access, +- .vmap = dma_heap_dma_buf_vmap, +- .vunmap = dma_heap_dma_buf_vunmap, +-}; +diff --git a/drivers/dma-buf/heaps/heap-helpers.h b/drivers/dma-buf/heaps/heap-helpers.h +deleted file mode 100644 +index 805d2df88024..000000000000 +--- a/drivers/dma-buf/heaps/heap-helpers.h ++++ /dev/null +@@ -1,53 +0,0 @@ +-/* SPDX-License-Identifier: GPL-2.0 */ +-/* +- * DMABUF Heaps helper code +- * +- * Copyright (C) 2011 Google, Inc. +- * Copyright (C) 2019 Linaro Ltd. +- */ +- +-#ifndef _HEAP_HELPERS_H +-#define _HEAP_HELPERS_H +- +-#include +-#include +- +-/** +- * struct heap_helper_buffer - helper buffer metadata +- * @heap: back pointer to the heap the buffer came from +- * @dmabuf: backing dma-buf for this buffer +- * @size: size of the buffer +- * @priv_virt pointer to heap specific private value +- * @lock mutext to protect the data in this structure +- * @vmap_cnt count of vmap references on the buffer +- * @vaddr vmap'ed virtual address +- * @pagecount number of pages in the buffer +- * @pages list of page pointers +- * @attachments list of device attachments +- * +- * @free heap callback to free the buffer +- */ +-struct heap_helper_buffer { +- struct dma_heap *heap; +- struct dma_buf *dmabuf; +- size_t size; +- +- void *priv_virt; +- struct mutex lock; +- int vmap_cnt; +- void *vaddr; +- pgoff_t pagecount; +- struct page **pages; +- struct list_head attachments; +- +- void (*free)(struct heap_helper_buffer *buffer); +-}; +- +-void init_heap_helper_buffer(struct heap_helper_buffer *buffer, +- void (*free)(struct heap_helper_buffer *)); +- +-struct dma_buf *heap_helper_export_dmabuf(struct heap_helper_buffer *buffer, +- int fd_flags); +- +-extern const struct dma_buf_ops heap_helper_ops; +-#endif /* _HEAP_HELPERS_H */ +diff --git a/drivers/dma-buf/heaps/page_pool.c b/drivers/dma-buf/heaps/page_pool.c +new file mode 100755 +index 000000000000..7c34a1ba48bb +--- /dev/null ++++ b/drivers/dma-buf/heaps/page_pool.c +@@ -0,0 +1,247 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * DMA BUF page pool system ++ * ++ * Copyright (C) 2020 Linaro Ltd. ++ * ++ * Based on the ION page pool code ++ * Copyright (C) 2011 Google, Inc. ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include "page_pool.h" ++ ++static LIST_HEAD(pool_list); ++static DEFINE_MUTEX(pool_list_lock); ++ ++static inline ++struct page *dmabuf_page_pool_alloc_pages(struct dmabuf_page_pool *pool) ++{ ++ if (fatal_signal_pending(current)) ++ return NULL; ++ return alloc_pages(pool->gfp_mask, pool->order); ++} ++ ++static inline void dmabuf_page_pool_free_pages(struct dmabuf_page_pool *pool, ++ struct page *page) ++{ ++ __free_pages(page, pool->order); ++} ++ ++static void dmabuf_page_pool_add(struct dmabuf_page_pool *pool, struct page *page) ++{ ++ int index; ++ ++ if (PageHighMem(page)) ++ index = POOL_HIGHPAGE; ++ else ++ index = POOL_LOWPAGE; ++ ++ mutex_lock(&pool->mutex); ++ list_add_tail(&page->lru, &pool->items[index]); ++ pool->count[index]++; ++ mutex_unlock(&pool->mutex); ++ mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE, ++ 1 << pool->order); ++} ++ ++static struct page *dmabuf_page_pool_remove(struct dmabuf_page_pool *pool, int index) ++{ ++ struct page *page; ++ ++ mutex_lock(&pool->mutex); ++ page = list_first_entry_or_null(&pool->items[index], struct page, lru); ++ if (page) { ++ pool->count[index]--; ++ list_del(&page->lru); ++ mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE, ++ -(1 << pool->order)); ++ } ++ mutex_unlock(&pool->mutex); ++ ++ return page; ++} ++ ++static struct page *dmabuf_page_pool_fetch(struct dmabuf_page_pool *pool) ++{ ++ struct page *page = NULL; ++ ++ page = dmabuf_page_pool_remove(pool, POOL_HIGHPAGE); ++ if (!page) ++ page = dmabuf_page_pool_remove(pool, POOL_LOWPAGE); ++ ++ return page; ++} ++ ++struct page *dmabuf_page_pool_alloc(struct dmabuf_page_pool *pool) ++{ ++ struct page *page = NULL; ++ ++ if (WARN_ON(!pool)) ++ return NULL; ++ ++ page = dmabuf_page_pool_fetch(pool); ++ ++ if (!page) ++ page = dmabuf_page_pool_alloc_pages(pool); ++ return page; ++} ++EXPORT_SYMBOL_GPL(dmabuf_page_pool_alloc); ++ ++void dmabuf_page_pool_free(struct dmabuf_page_pool *pool, struct page *page) ++{ ++ if (WARN_ON(pool->order != compound_order(page))) ++ return; ++ ++ dmabuf_page_pool_add(pool, page); ++} ++EXPORT_SYMBOL_GPL(dmabuf_page_pool_free); ++ ++static int dmabuf_page_pool_total(struct dmabuf_page_pool *pool, bool high) ++{ ++ int count = pool->count[POOL_LOWPAGE]; ++ ++ if (high) ++ count += pool->count[POOL_HIGHPAGE]; ++ ++ return count << pool->order; ++} ++ ++struct dmabuf_page_pool *dmabuf_page_pool_create(gfp_t gfp_mask, unsigned int order) ++{ ++ struct dmabuf_page_pool *pool = kmalloc(sizeof(*pool), GFP_KERNEL); ++ int i; ++ ++ if (!pool) ++ return NULL; ++ ++ for (i = 0; i < POOL_TYPE_SIZE; i++) { ++ pool->count[i] = 0; ++ INIT_LIST_HEAD(&pool->items[i]); ++ } ++ pool->gfp_mask = gfp_mask | __GFP_COMP; ++ pool->order = order; ++ mutex_init(&pool->mutex); ++ ++ mutex_lock(&pool_list_lock); ++ list_add(&pool->list, &pool_list); ++ mutex_unlock(&pool_list_lock); ++ ++ return pool; ++} ++EXPORT_SYMBOL_GPL(dmabuf_page_pool_create); ++ ++void dmabuf_page_pool_destroy(struct dmabuf_page_pool *pool) ++{ ++ struct page *page; ++ int i; ++ ++ /* Remove us from the pool list */ ++ mutex_lock(&pool_list_lock); ++ list_del(&pool->list); ++ mutex_unlock(&pool_list_lock); ++ ++ /* Free any remaining pages in the pool */ ++ for (i = 0; i < POOL_TYPE_SIZE; i++) { ++ while ((page = dmabuf_page_pool_remove(pool, i))) ++ dmabuf_page_pool_free_pages(pool, page); ++ } ++ ++ kfree(pool); ++} ++EXPORT_SYMBOL_GPL(dmabuf_page_pool_destroy); ++ ++static int dmabuf_page_pool_do_shrink(struct dmabuf_page_pool *pool, gfp_t gfp_mask, ++ int nr_to_scan) ++{ ++ int freed = 0; ++ bool high; ++ ++ if (current_is_kswapd()) ++ high = true; ++ else ++ high = !!(gfp_mask & __GFP_HIGHMEM); ++ ++ if (nr_to_scan == 0) ++ return dmabuf_page_pool_total(pool, high); ++ ++ while (freed < nr_to_scan) { ++ struct page *page; ++ ++ /* Try to free low pages first */ ++ page = dmabuf_page_pool_remove(pool, POOL_LOWPAGE); ++ if (!page) ++ page = dmabuf_page_pool_remove(pool, POOL_HIGHPAGE); ++ ++ if (!page) ++ break; ++ ++ dmabuf_page_pool_free_pages(pool, page); ++ freed += (1 << pool->order); ++ } ++ ++ return freed; ++} ++ ++static int dmabuf_page_pool_shrink(gfp_t gfp_mask, int nr_to_scan) ++{ ++ struct dmabuf_page_pool *pool; ++ int nr_total = 0; ++ int nr_freed; ++ int only_scan = 0; ++ ++ if (!nr_to_scan) ++ only_scan = 1; ++ ++ mutex_lock(&pool_list_lock); ++ list_for_each_entry(pool, &pool_list, list) { ++ if (only_scan) { ++ nr_total += dmabuf_page_pool_do_shrink(pool, ++ gfp_mask, ++ nr_to_scan); ++ } else { ++ nr_freed = dmabuf_page_pool_do_shrink(pool, ++ gfp_mask, ++ nr_to_scan); ++ nr_to_scan -= nr_freed; ++ nr_total += nr_freed; ++ if (nr_to_scan <= 0) ++ break; ++ } ++ } ++ mutex_unlock(&pool_list_lock); ++ ++ return nr_total; ++} ++ ++static unsigned long dmabuf_page_pool_shrink_count(struct shrinker *shrinker, ++ struct shrink_control *sc) ++{ ++ return dmabuf_page_pool_shrink(sc->gfp_mask, 0); ++} ++ ++static unsigned long dmabuf_page_pool_shrink_scan(struct shrinker *shrinker, ++ struct shrink_control *sc) ++{ ++ if (sc->nr_to_scan == 0) ++ return 0; ++ return dmabuf_page_pool_shrink(sc->gfp_mask, sc->nr_to_scan); ++} ++ ++struct shrinker pool_shrinker = { ++ .count_objects = dmabuf_page_pool_shrink_count, ++ .scan_objects = dmabuf_page_pool_shrink_scan, ++ .seeks = DEFAULT_SEEKS, ++ .batch = 0, ++}; ++ ++static int dmabuf_page_pool_init_shrinker(void) ++{ ++ return register_shrinker(&pool_shrinker); ++} ++module_init(dmabuf_page_pool_init_shrinker); ++MODULE_LICENSE("GPL v2"); +diff --git a/drivers/dma-buf/heaps/page_pool.h b/drivers/dma-buf/heaps/page_pool.h +new file mode 100755 +index 000000000000..6b083b04f195 +--- /dev/null ++++ b/drivers/dma-buf/heaps/page_pool.h +@@ -0,0 +1,55 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* ++ * DMA BUF PagePool implementation ++ * Based on earlier ION code by Google ++ * ++ * Copyright (C) 2011 Google, Inc. ++ * Copyright (C) 2020 Linaro Ltd. ++ */ ++ ++#ifndef _DMABUF_PAGE_POOL_H ++#define _DMABUF_PAGE_POOL_H ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++/* page types we track in the pool */ ++enum { ++ POOL_LOWPAGE, /* Clean lowmem pages */ ++ POOL_HIGHPAGE, /* Clean highmem pages */ ++ ++ POOL_TYPE_SIZE, ++}; ++ ++/** ++ * struct dmabuf_page_pool - pagepool struct ++ * @count[]: array of number of pages of that type in the pool ++ * @items[]: array of list of pages of the specific type ++ * @mutex: lock protecting this struct and especially the count ++ * item list ++ * @gfp_mask: gfp_mask to use from alloc ++ * @order: order of pages in the pool ++ * @list: list node for list of pools ++ * ++ * Allows you to keep a pool of pre allocated pages to use ++ */ ++struct dmabuf_page_pool { ++ int count[POOL_TYPE_SIZE]; ++ struct list_head items[POOL_TYPE_SIZE]; ++ struct mutex mutex; ++ gfp_t gfp_mask; ++ unsigned int order; ++ struct list_head list; ++}; ++ ++struct dmabuf_page_pool *dmabuf_page_pool_create(gfp_t gfp_mask, ++ unsigned int order); ++void dmabuf_page_pool_destroy(struct dmabuf_page_pool *pool); ++struct page *dmabuf_page_pool_alloc(struct dmabuf_page_pool *pool); ++void dmabuf_page_pool_free(struct dmabuf_page_pool *pool, struct page *page); ++ ++#endif /* _DMABUF_PAGE_POOL_H */ +diff --git a/drivers/dma-buf/heaps/system_heap.c b/drivers/dma-buf/heaps/system_heap.c +index 0bf688e3c023..15796bc4c033 100644 +--- a/drivers/dma-buf/heaps/system_heap.c ++++ b/drivers/dma-buf/heaps/system_heap.c +@@ -3,7 +3,11 @@ + * DMABUF System heap exporter + * + * Copyright (C) 2011 Google, Inc. +- * Copyright (C) 2019 Linaro Ltd. ++ * Copyright (C) 2019, 2020 Linaro Ltd. ++ * ++ * Portions based off of Andrew Davis' SRAM heap: ++ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/ ++ * Andrew F. Davis + */ + + #include +@@ -15,99 +19,546 @@ + #include + #include + #include +-#include +-#include ++#include ++ ++#include "page_pool.h" ++#include "deferred-free-helper.h" ++ ++static struct dma_heap *sys_heap; ++static struct dma_heap *sys_uncached_heap; ++ ++struct system_heap_buffer { ++ struct dma_heap *heap; ++ struct list_head attachments; ++ struct mutex lock; ++ unsigned long len; ++ struct sg_table sg_table; ++ int vmap_cnt; ++ void *vaddr; ++ struct deferred_freelist_item deferred_free; ++ ++ bool uncached; ++}; ++ ++struct dma_heap_attachment { ++ struct device *dev; ++ struct sg_table *table; ++ struct list_head list; ++ bool mapped; + +-#include "heap-helpers.h" ++ bool uncached; ++}; + +-struct dma_heap *sys_heap; ++#define HIGH_ORDER_GFP (((GFP_HIGHUSER | __GFP_ZERO | __GFP_NOWARN \ ++ | __GFP_NORETRY) & ~__GFP_RECLAIM) \ ++ | __GFP_COMP) ++#define LOW_ORDER_GFP (GFP_HIGHUSER | __GFP_ZERO | __GFP_COMP) ++static gfp_t order_flags[] = {HIGH_ORDER_GFP, LOW_ORDER_GFP, LOW_ORDER_GFP}; ++/* ++ * The selection of the orders used for allocation (1MB, 64K, 4K) is designed ++ * to match with the sizes often found in IOMMUs. Using order 4 pages instead ++ * of order 0 pages can significantly improve the performance of many IOMMUs ++ * by reducing TLB pressure and time spent updating page tables. ++ */ ++static const unsigned int orders[] = {8, 4, 0}; ++#define NUM_ORDERS ARRAY_SIZE(orders) ++struct dmabuf_page_pool *pools[NUM_ORDERS]; + +-static void system_heap_free(struct heap_helper_buffer *buffer) ++static struct sg_table *dup_sg_table(struct sg_table *table) + { +- pgoff_t pg; ++ struct sg_table *new_table; ++ int ret, i; ++ struct scatterlist *sg, *new_sg; + +- for (pg = 0; pg < buffer->pagecount; pg++) +- __free_page(buffer->pages[pg]); +- kfree(buffer->pages); +- kfree(buffer); ++ new_table = kzalloc(sizeof(*new_table), GFP_KERNEL); ++ if (!new_table) ++ return ERR_PTR(-ENOMEM); ++ ++ ret = sg_alloc_table(new_table, table->orig_nents, GFP_KERNEL); ++ if (ret) { ++ kfree(new_table); ++ return ERR_PTR(-ENOMEM); ++ } ++ ++ new_sg = new_table->sgl; ++ for_each_sgtable_sg(table, sg, i) { ++ sg_set_page(new_sg, sg_page(sg), sg->length, sg->offset); ++ new_sg = sg_next(new_sg); ++ } ++ ++ return new_table; + } + +-static int system_heap_allocate(struct dma_heap *heap, +- unsigned long len, +- unsigned long fd_flags, +- unsigned long heap_flags) ++static int system_heap_attach(struct dma_buf *dmabuf, ++ struct dma_buf_attachment *attachment) + { +- struct heap_helper_buffer *helper_buffer; +- struct dma_buf *dmabuf; +- int ret = -ENOMEM; +- pgoff_t pg; ++ struct system_heap_buffer *buffer = dmabuf->priv; ++ struct dma_heap_attachment *a; ++ struct sg_table *table; ++ ++ a = kzalloc(sizeof(*a), GFP_KERNEL); ++ if (!a) ++ return -ENOMEM; + +- helper_buffer = kzalloc(sizeof(*helper_buffer), GFP_KERNEL); +- if (!helper_buffer) ++ table = dup_sg_table(&buffer->sg_table); ++ if (IS_ERR(table)) { ++ kfree(a); + return -ENOMEM; ++ } ++ ++ a->table = table; ++ a->dev = attachment->dev; ++ INIT_LIST_HEAD(&a->list); ++ a->mapped = false; ++ a->uncached = buffer->uncached; ++ attachment->priv = a; ++ ++ mutex_lock(&buffer->lock); ++ list_add(&a->list, &buffer->attachments); ++ mutex_unlock(&buffer->lock); ++ ++ return 0; ++} ++ ++static void system_heap_detach(struct dma_buf *dmabuf, ++ struct dma_buf_attachment *attachment) ++{ ++ struct system_heap_buffer *buffer = dmabuf->priv; ++ struct dma_heap_attachment *a = attachment->priv; ++ ++ mutex_lock(&buffer->lock); ++ list_del(&a->list); ++ mutex_unlock(&buffer->lock); ++ ++ sg_free_table(a->table); ++ kfree(a->table); ++ kfree(a); ++} + +- init_heap_helper_buffer(helper_buffer, system_heap_free); +- helper_buffer->heap = heap; +- helper_buffer->size = len; ++static struct sg_table *system_heap_map_dma_buf(struct dma_buf_attachment *attachment, ++ enum dma_data_direction direction) ++{ ++ struct dma_heap_attachment *a = attachment->priv; ++ struct sg_table *table = a->table; ++ int attr = 0; ++ int ret; ++ ++ if (a->uncached) ++ attr = DMA_ATTR_SKIP_CPU_SYNC; ++ ++ ret = dma_map_sgtable(attachment->dev, table, direction, attr); ++ if (ret) ++ return ERR_PTR(ret); ++ ++ a->mapped = true; ++ return table; ++} ++ ++static void system_heap_unmap_dma_buf(struct dma_buf_attachment *attachment, ++ struct sg_table *table, ++ enum dma_data_direction direction) ++{ ++ struct dma_heap_attachment *a = attachment->priv; ++ int attr = 0; ++ ++ if (a->uncached) ++ attr = DMA_ATTR_SKIP_CPU_SYNC; ++ a->mapped = false; ++ dma_unmap_sgtable(attachment->dev, table, direction, attr); ++} + +- helper_buffer->pagecount = len / PAGE_SIZE; +- helper_buffer->pages = kmalloc_array(helper_buffer->pagecount, +- sizeof(*helper_buffer->pages), +- GFP_KERNEL); +- if (!helper_buffer->pages) { +- ret = -ENOMEM; +- goto err0; ++static int system_heap_dma_buf_begin_cpu_access(struct dma_buf *dmabuf, ++ enum dma_data_direction direction) ++{ ++ struct system_heap_buffer *buffer = dmabuf->priv; ++ struct dma_heap_attachment *a; ++ ++ mutex_lock(&buffer->lock); ++ ++ if (buffer->vmap_cnt) ++ invalidate_kernel_vmap_range(buffer->vaddr, buffer->len); ++ ++ if (!buffer->uncached) { ++ list_for_each_entry(a, &buffer->attachments, list) { ++ if (!a->mapped) ++ continue; ++ dma_sync_sgtable_for_cpu(a->dev, a->table, direction); ++ } + } ++ mutex_unlock(&buffer->lock); ++ ++ return 0; ++} ++ ++static int system_heap_dma_buf_end_cpu_access(struct dma_buf *dmabuf, ++ enum dma_data_direction direction) ++{ ++ struct system_heap_buffer *buffer = dmabuf->priv; ++ struct dma_heap_attachment *a; ++ ++ mutex_lock(&buffer->lock); ++ ++ if (buffer->vmap_cnt) ++ flush_kernel_vmap_range(buffer->vaddr, buffer->len); ++ ++ if (!buffer->uncached) { ++ list_for_each_entry(a, &buffer->attachments, list) { ++ if (!a->mapped) ++ continue; ++ dma_sync_sgtable_for_device(a->dev, a->table, direction); ++ } ++ } ++ mutex_unlock(&buffer->lock); ++ ++ return 0; ++} ++ ++static int system_heap_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma) ++{ ++ struct system_heap_buffer *buffer = dmabuf->priv; ++ struct sg_table *table = &buffer->sg_table; ++ unsigned long addr = vma->vm_start; ++ struct sg_page_iter piter; ++ int ret; ++ ++ if (buffer->uncached) ++ vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); ++ ++ for_each_sgtable_page(table, &piter, vma->vm_pgoff) { ++ struct page *page = sg_page_iter_page(&piter); ++ ++ ret = remap_pfn_range(vma, addr, page_to_pfn(page), PAGE_SIZE, ++ vma->vm_page_prot); ++ if (ret) ++ return ret; ++ addr += PAGE_SIZE; ++ if (addr >= vma->vm_end) ++ return 0; ++ } ++ return 0; ++} ++ ++static void *system_heap_do_vmap(struct system_heap_buffer *buffer) ++{ ++ struct sg_table *table = &buffer->sg_table; ++ int npages = PAGE_ALIGN(buffer->len) / PAGE_SIZE; ++ struct page **pages = vmalloc(sizeof(struct page *) * npages); ++ struct page **tmp = pages; ++ struct sg_page_iter piter; ++ pgprot_t pgprot = PAGE_KERNEL; ++ void *vaddr; ++ ++ if (!pages) ++ return ERR_PTR(-ENOMEM); ++ ++ if (buffer->uncached) ++ pgprot = pgprot_writecombine(PAGE_KERNEL); ++ ++ for_each_sgtable_page(table, &piter, 0) { ++ WARN_ON(tmp - pages >= npages); ++ *tmp++ = sg_page_iter_page(&piter); ++ } ++ ++ vaddr = vmap(pages, npages, VM_MAP, pgprot); ++ vfree(pages); ++ ++ if (!vaddr) ++ return ERR_PTR(-ENOMEM); ++ ++ return vaddr; ++} ++ ++static void *system_heap_vmap(struct dma_buf *dmabuf) ++{ ++ struct system_heap_buffer *buffer = dmabuf->priv; ++ void *vaddr; ++ ++ mutex_lock(&buffer->lock); ++ if (buffer->vmap_cnt) { ++ buffer->vmap_cnt++; ++ vaddr = buffer->vaddr; ++ goto out; ++ } ++ ++ vaddr = system_heap_do_vmap(buffer); ++ if (IS_ERR(vaddr)) ++ goto out; ++ ++ buffer->vaddr = vaddr; ++ buffer->vmap_cnt++; ++out: ++ mutex_unlock(&buffer->lock); + +- for (pg = 0; pg < helper_buffer->pagecount; pg++) { ++ return vaddr; ++} ++ ++static void system_heap_vunmap(struct dma_buf *dmabuf, void *vaddr) ++{ ++ struct system_heap_buffer *buffer = dmabuf->priv; ++ ++ mutex_lock(&buffer->lock); ++ if (!--buffer->vmap_cnt) { ++ vunmap(buffer->vaddr); ++ buffer->vaddr = NULL; ++ } ++ mutex_unlock(&buffer->lock); ++} ++ ++static int system_heap_zero_buffer(struct system_heap_buffer *buffer) ++{ ++ struct sg_table *sgt = &buffer->sg_table; ++ struct sg_page_iter piter; ++ struct page *p; ++ void *vaddr; ++ int ret = 0; ++ ++ for_each_sgtable_page(sgt, &piter, 0) { ++ p = sg_page_iter_page(&piter); ++ vaddr = kmap_atomic(p); ++ memset(vaddr, 0, PAGE_SIZE); ++ kunmap_atomic(vaddr); ++ } ++ ++ return ret; ++} ++ ++static void system_heap_buf_free(struct deferred_freelist_item *item, ++ enum df_reason reason) ++{ ++ struct system_heap_buffer *buffer; ++ struct sg_table *table; ++ struct scatterlist *sg; ++ int i, j; ++ ++ buffer = container_of(item, struct system_heap_buffer, deferred_free); ++ /* Zero the buffer pages before adding back to the pool */ ++ if (reason == DF_NORMAL) ++ if (system_heap_zero_buffer(buffer)) ++ reason = DF_UNDER_PRESSURE; // On failure, just free ++ ++ table = &buffer->sg_table; ++ for_each_sg(table->sgl, sg, table->nents, i) { ++ struct page *page = sg_page(sg); ++ ++ if (reason == DF_UNDER_PRESSURE) { ++ __free_pages(page, compound_order(page)); ++ } else { ++ for (j = 0; j < NUM_ORDERS; j++) { ++ if (compound_order(page) == orders[j]) ++ break; ++ } ++ dmabuf_page_pool_free(pools[j], page); ++ } ++ } ++ sg_free_table(table); ++ kfree(buffer); ++} ++ ++static void system_heap_dma_buf_release(struct dma_buf *dmabuf) ++{ ++ struct system_heap_buffer *buffer = dmabuf->priv; ++ int npages = PAGE_ALIGN(buffer->len) / PAGE_SIZE; ++ ++ deferred_free(&buffer->deferred_free, system_heap_buf_free, npages); ++} ++ ++static const struct dma_buf_ops system_heap_buf_ops = { ++ .attach = system_heap_attach, ++ .detach = system_heap_detach, ++ .map_dma_buf = system_heap_map_dma_buf, ++ .unmap_dma_buf = system_heap_unmap_dma_buf, ++ .begin_cpu_access = system_heap_dma_buf_begin_cpu_access, ++ .end_cpu_access = system_heap_dma_buf_end_cpu_access, ++ .mmap = system_heap_mmap, ++ .vmap = system_heap_vmap, ++ .vunmap = system_heap_vunmap, ++ .release = system_heap_dma_buf_release, ++}; ++ ++static struct page *alloc_largest_available(unsigned long size, ++ unsigned int max_order) ++{ ++ struct page *page; ++ int i; ++ ++ for (i = 0; i < NUM_ORDERS; i++) { ++ if (size < (PAGE_SIZE << orders[i])) ++ continue; ++ if (max_order < orders[i]) ++ continue; ++ page = dmabuf_page_pool_alloc(pools[i]); ++ if (!page) ++ continue; ++ return page; ++ } ++ return NULL; ++} ++ ++static struct dma_buf *system_heap_do_allocate(struct dma_heap *heap, ++ unsigned long len, ++ unsigned long fd_flags, ++ unsigned long heap_flags, ++ bool uncached) ++{ ++ struct system_heap_buffer *buffer; ++ DEFINE_DMA_BUF_EXPORT_INFO(exp_info); ++ unsigned long size_remaining = len; ++ unsigned int max_order = orders[0]; ++ struct dma_buf *dmabuf; ++ struct sg_table *table; ++ struct scatterlist *sg; ++ struct list_head pages; ++ struct page *page, *tmp_page; ++ int i, ret = -ENOMEM; ++ ++ buffer = kzalloc(sizeof(*buffer), GFP_KERNEL); ++ if (!buffer) ++ return ERR_PTR(-ENOMEM); ++ ++ INIT_LIST_HEAD(&buffer->attachments); ++ mutex_init(&buffer->lock); ++ buffer->heap = heap; ++ buffer->len = len; ++ buffer->uncached = uncached; ++ ++ INIT_LIST_HEAD(&pages); ++ i = 0; ++ while (size_remaining > 0) { + /* + * Avoid trying to allocate memory if the process +- * has been killed by by SIGKILL ++ * has been killed by SIGKILL + */ + if (fatal_signal_pending(current)) +- goto err1; ++ goto free_buffer; ++ ++ page = alloc_largest_available(size_remaining, max_order); ++ if (!page) ++ goto free_buffer; ++ ++ list_add_tail(&page->lru, &pages); ++ size_remaining -= page_size(page); ++ max_order = compound_order(page); ++ i++; ++ } + +- helper_buffer->pages[pg] = alloc_page(GFP_KERNEL | __GFP_ZERO); +- if (!helper_buffer->pages[pg]) +- goto err1; ++ table = &buffer->sg_table; ++ if (sg_alloc_table(table, i, GFP_KERNEL)) ++ goto free_buffer; ++ ++ sg = table->sgl; ++ list_for_each_entry_safe(page, tmp_page, &pages, lru) { ++ sg_set_page(sg, page, page_size(page), 0); ++ sg = sg_next(sg); ++ list_del(&page->lru); + } + + /* create the dmabuf */ +- dmabuf = heap_helper_export_dmabuf(helper_buffer, fd_flags); ++ exp_info.exp_name = dma_heap_get_name(heap); ++ exp_info.ops = &system_heap_buf_ops; ++ exp_info.size = buffer->len; ++ exp_info.flags = fd_flags; ++ exp_info.priv = buffer; ++ dmabuf = dma_buf_export(&exp_info); + if (IS_ERR(dmabuf)) { + ret = PTR_ERR(dmabuf); +- goto err1; ++ goto free_pages; ++ } ++ ++ /* ++ * For uncached buffers, we need to initially flush cpu cache, since ++ * the __GFP_ZERO on the allocation means the zeroing was done by the ++ * cpu and thus it is likely cached. Map (and implicitly flush) and ++ * unmap it now so we don't get corruption later on. ++ */ ++ if (buffer->uncached) { ++ dma_map_sgtable(dma_heap_get_dev(heap), table, DMA_BIDIRECTIONAL, 0); ++ dma_unmap_sgtable(dma_heap_get_dev(heap), table, DMA_BIDIRECTIONAL, 0); + } + +- helper_buffer->dmabuf = dmabuf; ++ return dmabuf; + +- ret = dma_buf_fd(dmabuf, fd_flags); +- if (ret < 0) { +- dma_buf_put(dmabuf); +- /* just return, as put will call release and that will free */ +- return ret; ++free_pages: ++ for_each_sgtable_sg(table, sg, i) { ++ struct page *p = sg_page(sg); ++ ++ __free_pages(p, compound_order(p)); + } ++ sg_free_table(table); ++free_buffer: ++ list_for_each_entry_safe(page, tmp_page, &pages, lru) ++ __free_pages(page, compound_order(page)); ++ kfree(buffer); + +- return ret; ++ return ERR_PTR(ret); ++} + +-err1: +- while (pg > 0) +- __free_page(helper_buffer->pages[--pg]); +- kfree(helper_buffer->pages); +-err0: +- kfree(helper_buffer); ++static struct dma_buf *system_heap_allocate(struct dma_heap *heap, ++ unsigned long len, ++ unsigned long fd_flags, ++ unsigned long heap_flags) ++{ ++ return system_heap_do_allocate(heap, len, fd_flags, heap_flags, false); ++} + +- return ret; ++static long system_get_pool_size(struct dma_heap *heap) ++{ ++ int i; ++ long num_pages = 0; ++ struct dmabuf_page_pool **pool; ++ ++ pool = pools; ++ for (i = 0; i < NUM_ORDERS; i++, pool++) { ++ num_pages += ((*pool)->count[POOL_LOWPAGE] + ++ (*pool)->count[POOL_HIGHPAGE]) << (*pool)->order; ++ } ++ ++ return num_pages << PAGE_SHIFT; + } + + static const struct dma_heap_ops system_heap_ops = { + .allocate = system_heap_allocate, ++ .get_pool_size = system_get_pool_size, ++}; ++ ++static struct dma_buf *system_uncached_heap_allocate(struct dma_heap *heap, ++ unsigned long len, ++ unsigned long fd_flags, ++ unsigned long heap_flags) ++{ ++ return system_heap_do_allocate(heap, len, fd_flags, heap_flags, true); ++} ++ ++/* Dummy function to be used until we can call coerce_mask_and_coherent */ ++static struct dma_buf *system_uncached_heap_not_initialized(struct dma_heap *heap, ++ unsigned long len, ++ unsigned long fd_flags, ++ unsigned long heap_flags) ++{ ++ return ERR_PTR(-EBUSY); ++} ++ ++static struct dma_heap_ops system_uncached_heap_ops = { ++ /* After system_heap_create is complete, we will swap this */ ++ .allocate = system_uncached_heap_not_initialized, + }; + + static int system_heap_create(void) + { + struct dma_heap_export_info exp_info; +- int ret = 0; ++ int i; ++ ++ for (i = 0; i < NUM_ORDERS; i++) { ++ pools[i] = dmabuf_page_pool_create(order_flags[i], orders[i]); ++ ++ if (!pools[i]) { ++ int j; ++ ++ pr_err("%s: page pool creation failed!\n", __func__); ++ for (j = 0; j < i; j++) ++ dmabuf_page_pool_destroy(pools[j]); ++ return -ENOMEM; ++ } ++ } + + exp_info.name = "system"; + exp_info.ops = &system_heap_ops; +@@ -115,9 +566,21 @@ static int system_heap_create(void) + + sys_heap = dma_heap_add(&exp_info); + if (IS_ERR(sys_heap)) +- ret = PTR_ERR(sys_heap); ++ return PTR_ERR(sys_heap); + +- return ret; ++ exp_info.name = "system-uncached"; ++ exp_info.ops = &system_uncached_heap_ops; ++ exp_info.priv = NULL; ++ ++ sys_uncached_heap = dma_heap_add(&exp_info); ++ if (IS_ERR(sys_uncached_heap)) ++ return PTR_ERR(sys_uncached_heap); ++ ++ dma_coerce_mask_and_coherent(dma_heap_get_dev(sys_uncached_heap), DMA_BIT_MASK(64)); ++ mb(); /* make sure we only set allocate after dma_mask is set */ ++ system_uncached_heap_ops.allocate = system_uncached_heap_allocate; ++ ++ return 0; + } + module_init(system_heap_create); + MODULE_LICENSE("GPL v2"); +diff --git a/drivers/dma-buf/sw_sync.c b/drivers/dma-buf/sw_sync.c +index 348b3a9170fa..3daa6c76b8dd 100644 +--- a/drivers/dma-buf/sw_sync.c ++++ b/drivers/dma-buf/sw_sync.c +@@ -7,6 +7,8 @@ + + #include + #include ++#include ++#include + #include + #include + #include +@@ -410,3 +412,13 @@ const struct file_operations sw_sync_debugfs_fops = { + .unlocked_ioctl = sw_sync_ioctl, + .compat_ioctl = compat_ptr_ioctl, + }; ++ ++static struct miscdevice sw_sync_dev = { ++ .minor = MISC_DYNAMIC_MINOR, ++ .name = "sw_sync", ++ .fops = &sw_sync_debugfs_fops, ++}; ++ ++module_misc_device(sw_sync_dev); ++ ++MODULE_LICENSE("GPL v2"); +diff --git a/drivers/dma-buf/sync_debug.c b/drivers/dma-buf/sync_debug.c +index 101394f16930..a2f906741ce0 100644 +--- a/drivers/dma-buf/sync_debug.c ++++ b/drivers/dma-buf/sync_debug.c +@@ -8,6 +8,7 @@ + #include + #include "sync_debug.h" + ++#ifdef CONFIG_DEBUG_FS + static struct dentry *dbgfs; + + static LIST_HEAD(sync_timeline_list_head); +@@ -188,3 +189,4 @@ static __init int sync_debugfs_init(void) + return 0; + } + late_initcall(sync_debugfs_init); ++#endif +diff --git a/drivers/dma-buf/sync_debug.h b/drivers/dma-buf/sync_debug.h +index 6176e52ba2d7..ee84997da6b4 100644 +--- a/drivers/dma-buf/sync_debug.h ++++ b/drivers/dma-buf/sync_debug.h +@@ -62,11 +62,18 @@ struct sync_pt { + struct rb_node node; + }; + ++#ifdef CONFIG_DEBUG_FS + extern const struct file_operations sw_sync_debugfs_fops; + + void sync_timeline_debug_add(struct sync_timeline *obj); + void sync_timeline_debug_remove(struct sync_timeline *obj); + void sync_file_debug_add(struct sync_file *fence); + void sync_file_debug_remove(struct sync_file *fence); ++#else ++static inline void sync_timeline_debug_add(struct sync_timeline *obj) {} ++static inline void sync_timeline_debug_remove(struct sync_timeline *obj) {} ++static inline void sync_file_debug_add(struct sync_file *fence) {} ++static inline void sync_file_debug_remove(struct sync_file *fence) {} ++#endif + + #endif /* _LINUX_SYNC_H */ +diff --git a/drivers/firmware/Kconfig b/drivers/firmware/Kconfig +index c08968c5ddf8..d9dbfda3c5f2 100644 +--- a/drivers/firmware/Kconfig ++++ b/drivers/firmware/Kconfig +@@ -9,7 +9,7 @@ menu "Firmware Drivers" + config ARM_SCMI_PROTOCOL + tristate "ARM System Control and Management Interface (SCMI) Message Protocol" + depends on ARM || ARM64 || COMPILE_TEST +- depends on MAILBOX || HAVE_ARM_SMCCC_DISCOVERY ++ depends on MAILBOX + help + ARM System Control and Management Interface (SCMI) protocol is a + set of operating system-independent software interfaces that are +@@ -251,6 +251,13 @@ config QCOM_SCM_DOWNLOAD_MODE_DEFAULT + + Say Y here to enable "download mode" by default. + ++config ROCKCHIP_SIP ++ tristate "Rockchip SIP interface" ++ depends on HAVE_ARM_SMCCC && ARCH_ROCKCHIP ++ help ++ Say Y here if you want to enable SIP callbacks for Rockchip platforms ++ This option enables support for communicating with the ATF. ++ + config TI_SCI_PROTOCOL + tristate "TI System Control Interface (TISCI) Message Protocol" + depends on TI_MESSAGE_MANAGER +diff --git a/drivers/firmware/Makefile b/drivers/firmware/Makefile +index 5e013b6a3692..850970fbacbc 100644 +--- a/drivers/firmware/Makefile ++++ b/drivers/firmware/Makefile +@@ -16,6 +16,7 @@ obj-$(CONFIG_ISCSI_IBFT_FIND) += iscsi_ibft_find.o + obj-$(CONFIG_ISCSI_IBFT) += iscsi_ibft.o + obj-$(CONFIG_FIRMWARE_MEMMAP) += memmap.o + obj-$(CONFIG_RASPBERRYPI_FIRMWARE) += raspberrypi.o ++obj-$(CONFIG_ROCKCHIP_SIP) += rockchip_sip.o + obj-$(CONFIG_FW_CFG_SYSFS) += qemu_fw_cfg.o + obj-$(CONFIG_QCOM_SCM) += qcom_scm.o qcom_scm-smc.o qcom_scm-legacy.o + obj-$(CONFIG_TI_SCI_PROTOCOL) += ti_sci.o +diff --git a/drivers/firmware/rockchip_sip.c b/drivers/firmware/rockchip_sip.c +new file mode 100755 +index 000000000000..4b00f33f3a05 +--- /dev/null ++++ b/drivers/firmware/rockchip_sip.c +@@ -0,0 +1,548 @@ ++/* ++ * This program is free software; you can redistribute it and/or modify ++ * it under the terms of the GNU General Public License version 2 as ++ * published by the Free Software Foundation. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Copyright (C) 2016, Fuzhou Rockchip Electronics Co., Ltd ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#ifdef CONFIG_ARM ++#include ++#endif ++#include ++#include ++#include ++#include ++#include ++ ++#ifdef CONFIG_64BIT ++#define PSCI_FN_NATIVE(version, name) PSCI_##version##_FN64_##name ++#else ++#define PSCI_FN_NATIVE(version, name) PSCI_##version##_FN_##name ++#endif ++ ++#define SIZE_PAGE(n) ((n) << 12) ++ ++static struct arm_smccc_res __invoke_sip_fn_smc(unsigned long function_id, ++ unsigned long arg0, ++ unsigned long arg1, ++ unsigned long arg2) ++{ ++ struct arm_smccc_res res; ++ ++ arm_smccc_smc(function_id, arg0, arg1, arg2, 0, 0, 0, 0, &res); ++ return res; ++} ++ ++struct arm_smccc_res sip_smc_dram(u32 arg0, u32 arg1, u32 arg2) ++{ ++ return __invoke_sip_fn_smc(SIP_DRAM_CONFIG, arg0, arg1, arg2); ++} ++EXPORT_SYMBOL_GPL(sip_smc_dram); ++ ++struct arm_smccc_res sip_smc_get_atf_version(void) ++{ ++ return __invoke_sip_fn_smc(SIP_ATF_VERSION, 0, 0, 0); ++} ++EXPORT_SYMBOL_GPL(sip_smc_get_atf_version); ++ ++struct arm_smccc_res sip_smc_get_sip_version(void) ++{ ++ return __invoke_sip_fn_smc(SIP_SIP_VERSION, 0, 0, 0); ++} ++EXPORT_SYMBOL_GPL(sip_smc_get_sip_version); ++ ++int sip_smc_set_suspend_mode(u32 ctrl, u32 config1, u32 config2) ++{ ++ struct arm_smccc_res res; ++ ++ res = __invoke_sip_fn_smc(SIP_SUSPEND_MODE, ctrl, config1, config2); ++ return res.a0; ++} ++EXPORT_SYMBOL_GPL(sip_smc_set_suspend_mode); ++ ++struct arm_smccc_res sip_smc_get_suspend_info(u32 info) ++{ ++ struct arm_smccc_res res; ++ ++ res = __invoke_sip_fn_smc(SIP_SUSPEND_MODE, info, 0, 0); ++ return res; ++} ++EXPORT_SYMBOL_GPL(sip_smc_get_suspend_info); ++ ++int sip_smc_virtual_poweroff(void) ++{ ++ struct arm_smccc_res res; ++ ++ res = __invoke_sip_fn_smc(PSCI_FN_NATIVE(1_0, SYSTEM_SUSPEND), 0, 0, 0); ++ return res.a0; ++} ++EXPORT_SYMBOL_GPL(sip_smc_virtual_poweroff); ++ ++int sip_smc_remotectl_config(u32 func, u32 data) ++{ ++ struct arm_smccc_res res; ++ ++ res = __invoke_sip_fn_smc(SIP_REMOTECTL_CFG, func, data, 0); ++ ++ return res.a0; ++} ++EXPORT_SYMBOL_GPL(sip_smc_remotectl_config); ++ ++u32 sip_smc_secure_reg_read(u32 addr_phy) ++{ ++ struct arm_smccc_res res; ++ ++ res = __invoke_sip_fn_smc(SIP_ACCESS_REG, 0, addr_phy, SECURE_REG_RD); ++ if (res.a0) ++ pr_err("%s error: %d, addr phy: 0x%x\n", ++ __func__, (int)res.a0, addr_phy); ++ ++ return res.a1; ++} ++EXPORT_SYMBOL_GPL(sip_smc_secure_reg_read); ++ ++int sip_smc_secure_reg_write(u32 addr_phy, u32 val) ++{ ++ struct arm_smccc_res res; ++ ++ res = __invoke_sip_fn_smc(SIP_ACCESS_REG, val, addr_phy, SECURE_REG_WR); ++ if (res.a0) ++ pr_err("%s error: %d, addr phy: 0x%x\n", ++ __func__, (int)res.a0, addr_phy); ++ ++ return res.a0; ++} ++EXPORT_SYMBOL_GPL(sip_smc_secure_reg_write); ++ ++static void *sip_map(phys_addr_t start, size_t size) ++{ ++ struct page **pages; ++ phys_addr_t page_start; ++ unsigned int page_count; ++ pgprot_t prot; ++ unsigned int i; ++ void *vaddr; ++ ++ if (!pfn_valid(__phys_to_pfn(start))) ++ return ioremap(start, size); ++ ++ page_start = start - offset_in_page(start); ++ page_count = DIV_ROUND_UP(size + offset_in_page(start), PAGE_SIZE); ++ ++ prot = pgprot_noncached(PAGE_KERNEL); ++ ++ pages = kmalloc_array(page_count, sizeof(struct page *), GFP_KERNEL); ++ if (!pages) { ++ pr_err("%s: Failed to allocate array for %u pages\n", ++ __func__, page_count); ++ return NULL; ++ } ++ ++ for (i = 0; i < page_count; i++) ++ pages[i] = phys_to_page(page_start + i * PAGE_SIZE); ++ ++ vaddr = vmap(pages, page_count, VM_MAP, prot); ++ kfree(pages); ++ ++ /* ++ * Since vmap() uses page granularity, we must add the offset ++ * into the page here, to get the byte granularity address ++ * into the mapping to represent the actual "start" location. ++ */ ++ return vaddr + offset_in_page(start); ++} ++ ++struct arm_smccc_res sip_smc_request_share_mem(u32 page_num, ++ share_page_type_t page_type) ++{ ++ struct arm_smccc_res res; ++ unsigned long share_mem_phy; ++ ++ res = __invoke_sip_fn_smc(SIP_SHARE_MEM, page_num, page_type, 0); ++ if (IS_SIP_ERROR(res.a0)) ++ goto error; ++ ++ share_mem_phy = res.a1; ++ res.a1 = (unsigned long)sip_map(share_mem_phy, SIZE_PAGE(page_num)); ++ ++error: ++ return res; ++} ++EXPORT_SYMBOL_GPL(sip_smc_request_share_mem); ++ ++struct arm_smccc_res sip_smc_mcu_el3fiq(u32 arg0, u32 arg1, u32 arg2) ++{ ++ return __invoke_sip_fn_smc(SIP_MCU_EL3FIQ_CFG, arg0, arg1, arg2); ++} ++EXPORT_SYMBOL_GPL(sip_smc_mcu_el3fiq); ++ ++struct arm_smccc_res sip_smc_vpu_reset(u32 arg0, u32 arg1, u32 arg2) ++{ ++ struct arm_smccc_res res; ++ ++ res = __invoke_sip_fn_smc(PSCI_SIP_VPU_RESET, arg0, arg1, arg2); ++ return res; ++} ++EXPORT_SYMBOL_GPL(sip_smc_vpu_reset); ++ ++struct arm_smccc_res sip_smc_bus_config(u32 arg0, u32 arg1, u32 arg2) ++{ ++ struct arm_smccc_res res; ++ ++ res = __invoke_sip_fn_smc(SIP_BUS_CFG, arg0, arg1, arg2); ++ return res; ++} ++EXPORT_SYMBOL_GPL(sip_smc_bus_config); ++ ++struct arm_smccc_res sip_smc_lastlog_request(void) ++{ ++ struct arm_smccc_res res; ++ void __iomem *addr1, *addr2; ++ ++ res = __invoke_sip_fn_smc(SIP_LAST_LOG, local_clock(), 0, 0); ++ if (IS_SIP_ERROR(res.a0)) ++ return res; ++ ++ addr1 = sip_map(res.a1, res.a3); ++ if (!addr1) { ++ pr_err("%s: share memory buffer0 ioremap failed\n", __func__); ++ res.a0 = SIP_RET_INVALID_ADDRESS; ++ return res; ++ } ++ addr2 = sip_map(res.a2, res.a3); ++ if (!addr2) { ++ pr_err("%s: share memory buffer1 ioremap failed\n", __func__); ++ res.a0 = SIP_RET_INVALID_ADDRESS; ++ return res; ++ } ++ ++ res.a1 = (unsigned long)addr1; ++ res.a2 = (unsigned long)addr2; ++ ++ return res; ++} ++EXPORT_SYMBOL_GPL(sip_smc_lastlog_request); ++ ++/************************** fiq debugger **************************************/ ++/* ++ * AArch32 is not allowed to call SMC64(ATF framework does not support), so we ++ * don't change SIP_UARTDBG_FN to SIP_UARTDBG_CFG64 even when cpu is AArch32 ++ * mode. Let ATF support SIP_UARTDBG_CFG, and we just initialize SIP_UARTDBG_FN ++ * depends on compile option(CONFIG_ARM or CONFIG_ARM64). ++ */ ++#ifdef CONFIG_ARM64 ++#define SIP_UARTDBG_FN SIP_UARTDBG_CFG64 ++#else ++#define SIP_UARTDBG_FN SIP_UARTDBG_CFG ++static int firmware_64_32bit; ++#endif ++ ++static int fiq_sip_enabled; ++static int fiq_target_cpu; ++static phys_addr_t ft_fiq_mem_phy; ++static void __iomem *ft_fiq_mem_base; ++static void (*sip_fiq_debugger_uart_irq_tf)(struct pt_regs _pt_regs, ++ unsigned long cpu); ++int sip_fiq_debugger_is_enabled(void) ++{ ++ return fiq_sip_enabled; ++} ++EXPORT_SYMBOL_GPL(sip_fiq_debugger_is_enabled); ++ ++static struct pt_regs sip_fiq_debugger_get_pt_regs(void *reg_base, ++ unsigned long sp_el1) ++{ ++ struct pt_regs fiq_pt_regs; ++ __maybe_unused struct sm_nsec_ctx *nsec_ctx = reg_base; ++ __maybe_unused struct gp_regs_ctx *gp_regs = reg_base; ++ ++#ifdef CONFIG_ARM64 ++ /* ++ * 64-bit ATF + 64-bit kernel ++ */ ++ /* copy cpu context: x0 ~ spsr_el3 */ ++ memcpy(&fiq_pt_regs, reg_base, 8 * 31); ++ ++ /* copy pstate: spsr_el3 */ ++ memcpy(&fiq_pt_regs.pstate, reg_base + 0x110, 8); ++ fiq_pt_regs.sp = sp_el1; ++ ++ /* copy pc: elr_el3 */ ++ memcpy(&fiq_pt_regs.pc, reg_base + 0x118, 8); ++#else ++ if (firmware_64_32bit == FIRMWARE_ATF_64BIT) { ++ /* ++ * 64-bit ATF + 32-bit kernel ++ */ ++ fiq_pt_regs.ARM_r0 = gp_regs->x0; ++ fiq_pt_regs.ARM_r1 = gp_regs->x1; ++ fiq_pt_regs.ARM_r2 = gp_regs->x2; ++ fiq_pt_regs.ARM_r3 = gp_regs->x3; ++ fiq_pt_regs.ARM_r4 = gp_regs->x4; ++ fiq_pt_regs.ARM_r5 = gp_regs->x5; ++ fiq_pt_regs.ARM_r6 = gp_regs->x6; ++ fiq_pt_regs.ARM_r7 = gp_regs->x7; ++ fiq_pt_regs.ARM_r8 = gp_regs->x8; ++ fiq_pt_regs.ARM_r9 = gp_regs->x9; ++ fiq_pt_regs.ARM_r10 = gp_regs->x10; ++ fiq_pt_regs.ARM_fp = gp_regs->x11; ++ fiq_pt_regs.ARM_ip = gp_regs->x12; ++ fiq_pt_regs.ARM_sp = gp_regs->x19; /* aarch32 svc_r13 */ ++ fiq_pt_regs.ARM_lr = gp_regs->x18; /* aarch32 svc_r14 */ ++ fiq_pt_regs.ARM_cpsr = gp_regs->spsr_el3; ++ fiq_pt_regs.ARM_pc = gp_regs->elr_el3; ++ } else { ++ /* ++ * 32-bit tee firmware + 32-bit kernel ++ */ ++ fiq_pt_regs.ARM_r0 = nsec_ctx->r0; ++ fiq_pt_regs.ARM_r1 = nsec_ctx->r1; ++ fiq_pt_regs.ARM_r2 = nsec_ctx->r2; ++ fiq_pt_regs.ARM_r3 = nsec_ctx->r3; ++ fiq_pt_regs.ARM_r4 = nsec_ctx->r4; ++ fiq_pt_regs.ARM_r5 = nsec_ctx->r5; ++ fiq_pt_regs.ARM_r6 = nsec_ctx->r6; ++ fiq_pt_regs.ARM_r7 = nsec_ctx->r7; ++ fiq_pt_regs.ARM_r8 = nsec_ctx->r8; ++ fiq_pt_regs.ARM_r9 = nsec_ctx->r9; ++ fiq_pt_regs.ARM_r10 = nsec_ctx->r10; ++ fiq_pt_regs.ARM_fp = nsec_ctx->r11; ++ fiq_pt_regs.ARM_ip = nsec_ctx->r12; ++ fiq_pt_regs.ARM_sp = nsec_ctx->svc_sp; ++ fiq_pt_regs.ARM_lr = nsec_ctx->svc_lr; ++ fiq_pt_regs.ARM_cpsr = nsec_ctx->mon_spsr; ++ ++ /* ++ * 'nsec_ctx->mon_lr' is not the fiq break point's PC, because it will ++ * be override as 'psci_fiq_debugger_uart_irq_tf_cb' for optee-os to ++ * jump to fiq_debugger handler. ++ * ++ * As 'nsec_ctx->und_lr' is not used for kernel, so optee-os uses it to ++ * deliver fiq break point's PC. ++ * ++ */ ++ fiq_pt_regs.ARM_pc = nsec_ctx->und_lr; ++ } ++#endif ++ ++ return fiq_pt_regs; ++} ++ ++static void sip_fiq_debugger_uart_irq_tf_cb(unsigned long sp_el1, ++ unsigned long offset, ++ unsigned long cpu) ++{ ++ struct pt_regs fiq_pt_regs; ++ char *cpu_context; ++ ++ /* calling fiq handler */ ++ if (ft_fiq_mem_base) { ++ cpu_context = (char *)ft_fiq_mem_base + offset; ++ fiq_pt_regs = sip_fiq_debugger_get_pt_regs(cpu_context, sp_el1); ++ sip_fiq_debugger_uart_irq_tf(fiq_pt_regs, cpu); ++ } ++ ++ /* fiq handler done, return to EL3(then EL3 return to EL1 entry) */ ++ __invoke_sip_fn_smc(SIP_UARTDBG_FN, 0, 0, UARTDBG_CFG_OSHDL_TO_OS); ++} ++ ++int sip_fiq_debugger_uart_irq_tf_init(u32 irq_id, void *callback_fn) ++{ ++ struct arm_smccc_res res; ++ ++ fiq_target_cpu = 0; ++ ++ /* init fiq debugger callback */ ++ sip_fiq_debugger_uart_irq_tf = callback_fn; ++ res = __invoke_sip_fn_smc(SIP_UARTDBG_FN, irq_id, ++ (unsigned long)sip_fiq_debugger_uart_irq_tf_cb, ++ UARTDBG_CFG_INIT); ++ if (IS_SIP_ERROR(res.a0)) { ++ pr_err("%s error: %d\n", __func__, (int)res.a0); ++ return res.a0; ++ } ++ ++ /* share memory ioremap */ ++ if (!ft_fiq_mem_base) { ++ ft_fiq_mem_phy = res.a1; ++ ft_fiq_mem_base = sip_map(ft_fiq_mem_phy, ++ FIQ_UARTDBG_SHARE_MEM_SIZE); ++ if (!ft_fiq_mem_base) { ++ pr_err("%s: share memory ioremap failed\n", __func__); ++ return -ENOMEM; ++ } ++ } ++ ++ fiq_sip_enabled = 1; ++ ++ return SIP_RET_SUCCESS; ++} ++EXPORT_SYMBOL_GPL(sip_fiq_debugger_uart_irq_tf_init); ++ ++static ulong cpu_logical_map_mpidr(u32 cpu) ++{ ++#ifdef MODULE ++ /* Empirically, local "cpu_logical_map()" for rockchip platforms */ ++ ulong mpidr = 0x00; ++ ++ if (cpu < 4) ++ /* 0x00, 0x01, 0x02, 0x03 */ ++ mpidr = cpu; ++ else if (cpu < 8) ++ /* 0x100, 0x101, 0x102, 0x103 */ ++ mpidr = 0x100 | (cpu - 4); ++ else ++ pr_err("Unsupported map cpu: %d\n", cpu); ++ ++ return mpidr; ++#else ++ return cpu_logical_map(cpu); ++#endif ++} ++ ++int sip_fiq_debugger_switch_cpu(u32 cpu) ++{ ++ struct arm_smccc_res res; ++ ++ fiq_target_cpu = cpu; ++ res = __invoke_sip_fn_smc(SIP_UARTDBG_FN, cpu_logical_map_mpidr(cpu), ++ 0, UARTDBG_CFG_OSHDL_CPUSW); ++ return res.a0; ++} ++ ++int sip_fiq_debugger_sdei_switch_cpu(u32 cur_cpu, u32 target_cpu, u32 flag) ++{ ++ struct arm_smccc_res res; ++ ++ res = __invoke_sip_fn_smc(SIP_SDEI_FIQ_DBG_SWITCH_CPU, ++ cur_cpu, target_cpu, flag); ++ return res.a0; ++} ++ ++int sip_fiq_debugger_sdei_get_event_id(u32 *fiq, u32 *sw_cpu, u32 *flag) ++{ ++ struct arm_smccc_res res; ++ ++ res = __invoke_sip_fn_smc(SIP_SDEI_FIQ_DBG_GET_EVENT_ID, ++ 0, 0, 0); ++ *fiq = res.a1; ++ *sw_cpu = res.a2; ++ if (flag) ++ *flag = res.a3; ++ ++ return res.a0; ++} ++ ++EXPORT_SYMBOL_GPL(sip_fiq_debugger_switch_cpu); ++ ++void sip_fiq_debugger_enable_debug(bool enable) ++{ ++ unsigned long val; ++ ++ val = enable ? UARTDBG_CFG_OSHDL_DEBUG_ENABLE : ++ UARTDBG_CFG_OSHDL_DEBUG_DISABLE; ++ ++ __invoke_sip_fn_smc(SIP_UARTDBG_FN, 0, 0, val); ++} ++EXPORT_SYMBOL_GPL(sip_fiq_debugger_enable_debug); ++ ++int sip_fiq_debugger_set_print_port(u32 port_phyaddr, u32 baudrate) ++{ ++ struct arm_smccc_res res; ++ ++ res = __invoke_sip_fn_smc(SIP_UARTDBG_FN, port_phyaddr, baudrate, ++ UARTDBG_CFG_PRINT_PORT); ++ return res.a0; ++} ++EXPORT_SYMBOL_GPL(sip_fiq_debugger_set_print_port); ++ ++int sip_fiq_debugger_request_share_memory(void) ++{ ++ struct arm_smccc_res res; ++ ++ /* request page share memory */ ++ res = sip_smc_request_share_mem(FIQ_UARTDBG_PAGE_NUMS, ++ SHARE_PAGE_TYPE_UARTDBG); ++ if (IS_SIP_ERROR(res.a0)) ++ return res.a0; ++ ++ return SIP_RET_SUCCESS; ++} ++EXPORT_SYMBOL_GPL(sip_fiq_debugger_request_share_memory); ++ ++int sip_fiq_debugger_get_target_cpu(void) ++{ ++ return fiq_target_cpu; ++} ++EXPORT_SYMBOL_GPL(sip_fiq_debugger_get_target_cpu); ++ ++void sip_fiq_debugger_enable_fiq(bool enable, uint32_t tgt_cpu) ++{ ++ u32 en; ++ ++ fiq_target_cpu = tgt_cpu; ++ en = enable ? UARTDBG_CFG_FIQ_ENABEL : UARTDBG_CFG_FIQ_DISABEL; ++ __invoke_sip_fn_smc(SIP_UARTDBG_FN, tgt_cpu, 0, en); ++} ++EXPORT_SYMBOL_GPL(sip_fiq_debugger_enable_fiq); ++ ++/******************************************************************************/ ++#ifdef CONFIG_ARM ++static __init int sip_firmware_init(void) ++{ ++ struct arm_smccc_res res; ++ ++ if (!psci_smp_available()) ++ return 0; ++ ++ /* ++ * OP-TEE works on kernel 3.10 and 4.4 and we have different sip ++ * implement. We should tell OP-TEE the current rockchip sip version. ++ */ ++ res = __invoke_sip_fn_smc(SIP_SIP_VERSION, SIP_IMPLEMENT_V2, ++ SECURE_REG_WR, 0); ++ if (IS_SIP_ERROR(res.a0)) ++ pr_err("%s: set rockchip sip version v2 failed\n", __func__); ++ ++ /* ++ * Currently, we support: ++ * ++ * 1. 64-bit ATF + 64-bit kernel; ++ * 2. 64-bit ATF + 32-bit kernel; ++ * 3. 32-bit TEE + 32-bit kernel; ++ * ++ * We need to detect which case of above and record in firmware_64_32bit ++ * We get info from cpuid and compare with all supported ARMv7 cpu. ++ */ ++ switch (read_cpuid_part()) { ++ case ARM_CPU_PART_CORTEX_A7: ++ case ARM_CPU_PART_CORTEX_A8: ++ case ARM_CPU_PART_CORTEX_A9: ++ case ARM_CPU_PART_CORTEX_A12: ++ case ARM_CPU_PART_CORTEX_A15: ++ case ARM_CPU_PART_CORTEX_A17: ++ firmware_64_32bit = FIRMWARE_TEE_32BIT; ++ break; ++ default: ++ firmware_64_32bit = FIRMWARE_ATF_64BIT; ++ break; ++ } ++ ++ return 0; ++} ++arch_initcall(sip_firmware_init); ++#endif ++ ++MODULE_DESCRIPTION("Rockchip SIP Call"); ++MODULE_LICENSE("GPL"); +diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig +index d1300fc003ed..9a4110ad1c20 100644 +--- a/drivers/gpio/Kconfig ++++ b/drivers/gpio/Kconfig +@@ -495,6 +495,14 @@ config GPIO_REG + A 32-bit single register GPIO fixed in/out implementation. This + can be used to represent any register as a set of GPIO signals. + ++config GPIO_ROCKCHIP ++ tristate "Rockchip GPIO support" ++ depends on ARCH_ROCKCHIP || COMPILE_TEST ++ select GPIOLIB_IRQCHIP ++ default ARCH_ROCKCHIP ++ help ++ Say yes here to support GPIO on Rockchip SoCs. ++ + config GPIO_SAMA5D2_PIOBU + tristate "SAMA5D2 PIOBU GPIO support" + depends on MFD_SYSCON +diff --git a/drivers/gpio/Makefile b/drivers/gpio/Makefile +index 09dada80ac34..ce6cafb6d3de 100644 +--- a/drivers/gpio/Makefile ++++ b/drivers/gpio/Makefile +@@ -125,6 +125,7 @@ obj-$(CONFIG_GPIO_RCAR) += gpio-rcar.o + obj-$(CONFIG_GPIO_RDA) += gpio-rda.o + obj-$(CONFIG_GPIO_RDC321X) += gpio-rdc321x.o + obj-$(CONFIG_GPIO_REG) += gpio-reg.o ++obj-$(CONFIG_GPIO_ROCKCHIP) += gpio-rockchip.o + obj-$(CONFIG_ARCH_SA1100) += gpio-sa1100.o + obj-$(CONFIG_GPIO_SAMA5D2_PIOBU) += gpio-sama5d2-piobu.o + obj-$(CONFIG_GPIO_SCH311X) += gpio-sch311x.o +diff --git a/drivers/gpio/gpio-rockchip.c b/drivers/gpio/gpio-rockchip.c +new file mode 100755 +index 000000000000..1d1e9a64cceb +--- /dev/null ++++ b/drivers/gpio/gpio-rockchip.c +@@ -0,0 +1,746 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* ++ * Copyright (c) 2020 Rockchip Electronics Co. Ltd. ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "../pinctrl/core.h" ++#include "../pinctrl/pinctrl-rockchip.h" ++ ++#define GPIO_TYPE_V1 (0) /* GPIO Version ID reserved */ ++#define GPIO_TYPE_V2 (0x01000C2B) /* GPIO Version ID 0x01000C2B */ ++ ++#define GPIO_BANK_PIN_NUM (32) ++ ++static const struct rockchip_gpio_regs gpio_regs_v1 = { ++ .port_dr = 0x00, ++ .port_ddr = 0x04, ++ .int_en = 0x30, ++ .int_mask = 0x34, ++ .int_type = 0x38, ++ .int_polarity = 0x3c, ++ .int_status = 0x40, ++ .int_rawstatus = 0x44, ++ .debounce = 0x48, ++ .port_eoi = 0x4c, ++ .ext_port = 0x50, ++}; ++ ++static const struct rockchip_gpio_regs gpio_regs_v2 = { ++ .port_dr = 0x00, ++ .port_ddr = 0x08, ++ .int_en = 0x10, ++ .int_mask = 0x18, ++ .int_type = 0x20, ++ .int_polarity = 0x28, ++ .int_bothedge = 0x30, ++ .int_status = 0x50, ++ .int_rawstatus = 0x58, ++ .debounce = 0x38, ++ .dbclk_div_en = 0x40, ++ .dbclk_div_con = 0x48, ++ .port_eoi = 0x60, ++ .ext_port = 0x70, ++ .version_id = 0x78, ++}; ++ ++static inline void gpio_writel_v2(u32 val, void __iomem *reg) ++{ ++ writel((val & 0xffff) | 0xffff0000, reg); ++ writel((val >> 16) | 0xffff0000, reg + 0x4); ++} ++ ++static inline u32 gpio_readl_v2(void __iomem *reg) ++{ ++ return readl(reg + 0x4) << 16 | readl(reg); ++} ++ ++static inline void rockchip_gpio_writel(struct rockchip_pin_bank *bank, ++ u32 value, unsigned int offset) ++{ ++ void __iomem *reg = bank->reg_base + offset; ++ ++ if (bank->gpio_type == GPIO_TYPE_V2) ++ gpio_writel_v2(value, reg); ++ else ++ writel(value, reg); ++} ++ ++static inline u32 rockchip_gpio_readl(struct rockchip_pin_bank *bank, ++ unsigned int offset) ++{ ++ void __iomem *reg = bank->reg_base + offset; ++ u32 value; ++ ++ if (bank->gpio_type == GPIO_TYPE_V2) ++ value = gpio_readl_v2(reg); ++ else ++ value = readl(reg); ++ ++ return value; ++} ++ ++static inline void rockchip_gpio_writel_bit(struct rockchip_pin_bank *bank, ++ u32 bit, u32 value, ++ unsigned int offset) ++{ ++ void __iomem *reg = bank->reg_base + offset; ++ u32 data; ++ ++ if (bank->gpio_type == GPIO_TYPE_V2) { ++ if (value) ++ data = BIT(bit % 16) | BIT(bit % 16 + 16); ++ else ++ data = BIT(bit % 16 + 16); ++ writel(data, bit >= 16 ? reg + 0x4 : reg); ++ } else { ++ data = readl(reg); ++ data &= ~BIT(bit); ++ if (value) ++ data |= BIT(bit); ++ writel(data, reg); ++ } ++} ++ ++static inline u32 rockchip_gpio_readl_bit(struct rockchip_pin_bank *bank, ++ u32 bit, unsigned int offset) ++{ ++ void __iomem *reg = bank->reg_base + offset; ++ u32 data; ++ ++ if (bank->gpio_type == GPIO_TYPE_V2) { ++ data = readl(bit >= 16 ? reg + 0x4 : reg); ++ data >>= bit % 16; ++ } else { ++ data = readl(reg); ++ data >>= bit; ++ } ++ ++ return data & (0x1); ++} ++ ++static void rockchip_gpio_set(struct gpio_chip *gc, unsigned int offset, int value) ++{ ++ struct rockchip_pin_bank *bank = gpiochip_get_data(gc); ++ unsigned long flags; ++ ++ raw_spin_lock_irqsave(&bank->slock, flags); ++ rockchip_gpio_writel_bit(bank, offset, value, bank->gpio_regs->port_dr); ++ raw_spin_unlock_irqrestore(&bank->slock, flags); ++} ++ ++static int rockchip_gpio_get(struct gpio_chip *gc, unsigned int offset) ++{ ++ struct rockchip_pin_bank *bank = gpiochip_get_data(gc); ++ u32 data; ++ ++ data = readl(bank->reg_base + bank->gpio_regs->ext_port); ++ data >>= offset; ++ data &= 1; ++ ++ return data; ++} ++ ++static int rockchip_gpio_get_direction(struct gpio_chip *chip, unsigned int offset) ++{ ++ struct rockchip_pin_bank *bank = gpiochip_get_data(chip); ++ u32 data; ++ ++ data = rockchip_gpio_readl_bit(bank, offset, bank->gpio_regs->port_ddr); ++ ++ return !data; ++} ++ ++static int rockchip_gpio_set_direction(struct gpio_chip *chip, unsigned int offset, bool input) ++{ ++ struct rockchip_pin_bank *bank = gpiochip_get_data(chip); ++ u32 data = input ? 0 : 1; ++ ++ rockchip_gpio_writel_bit(bank, offset, data, bank->gpio_regs->port_ddr); ++ ++ return 0; ++} ++ ++static int rockchip_gpio_direction_input(struct gpio_chip *gc, unsigned int offset) ++{ ++ return rockchip_gpio_set_direction(gc, offset, true); ++} ++ ++static int rockchip_gpio_direction_output(struct gpio_chip *gc, ++ unsigned int offset, int value) ++{ ++ rockchip_gpio_set(gc, offset, value); ++ ++ return rockchip_gpio_set_direction(gc, offset, false); ++} ++ ++static int rockchip_gpio_set_debounce(struct gpio_chip *gc, ++ unsigned int offset, unsigned int debounce) ++{ ++ struct rockchip_pin_bank *bank = gpiochip_get_data(gc); ++ const struct rockchip_gpio_regs *reg = bank->gpio_regs; ++ unsigned long flags, div_reg, freq, max_debounce; ++ bool div_debounce_support; ++ unsigned int cur_div_reg; ++ u64 div; ++ ++ if (!IS_ERR(bank->db_clk)) { ++ div_debounce_support = true; ++ freq = clk_get_rate(bank->db_clk); ++ max_debounce = (GENMASK(23, 0) + 1) * 2 * 1000000 / freq; ++ if (debounce > max_debounce) ++ return -EINVAL; ++ ++ div = debounce * freq; ++ div_reg = DIV_ROUND_CLOSEST_ULL(div, 2 * USEC_PER_SEC) - 1; ++ } else { ++ div_debounce_support = false; ++ } ++ ++ raw_spin_lock_irqsave(&bank->slock, flags); ++ ++ /* Only the v1 needs to configure div_en and div_con for dbclk */ ++ if (debounce) { ++ if (div_debounce_support) { ++ /* Configure the max debounce from consumers */ ++ cur_div_reg = readl(bank->reg_base + reg->dbclk_div_con); ++ if (cur_div_reg < div_reg) ++ writel(div_reg, bank->reg_base + reg->dbclk_div_con); ++ rockchip_gpio_writel_bit(bank, offset, 1, ++ reg->dbclk_div_en); ++ } ++ ++ rockchip_gpio_writel_bit(bank, offset, 1, reg->debounce); ++ } else { ++ if (div_debounce_support) ++ rockchip_gpio_writel_bit(bank, offset, 0, ++ reg->dbclk_div_en); ++ ++ rockchip_gpio_writel_bit(bank, offset, 0, reg->debounce); ++ } ++ ++ raw_spin_unlock_irqrestore(&bank->slock, flags); ++ ++ /* Enable or disable dbclk at last */ ++ if (div_debounce_support) { ++ if (debounce) ++ clk_prepare_enable(bank->db_clk); ++ else ++ clk_disable_unprepare(bank->db_clk); ++ } ++ ++ return 0; ++} ++ ++/* ++ * gpiolib set_config callback function. The setting of the pin ++ * mux function as 'gpio output' will be handled by the pinctrl subsystem ++ * interface. ++ */ ++static int rockchip_gpio_set_config(struct gpio_chip *gc, unsigned int offset, ++ unsigned long config) ++{ ++ enum pin_config_param param = pinconf_to_config_param(config); ++ unsigned int debounce = pinconf_to_config_argument(config); ++ int ret = 0; ++ ++ switch (param) { ++ case PIN_CONFIG_INPUT_DEBOUNCE: ++ /* ++ * Rockchip's gpio could only support up to one period ++ * of the debounce clock(pclk), which is far away from ++ * satisftying the requirement, as pclk is usually near ++ * 100MHz shared by all peripherals. So the fact is it ++ * has crippled debounce capability could only be useful ++ * to prevent any spurious glitches from waking up the system ++ * if the gpio is conguired as wakeup interrupt source. Let's ++ * still return -ENOTSUPP as before, to make sure the caller ++ * of gpiod_set_debounce won't change its behaviour. ++ */ ++ rockchip_gpio_set_debounce(gc, offset, debounce); ++ ret = -ENOTSUPP; ++ break; ++ default: ++ ret = -ENOTSUPP; ++ break; ++ } ++ ++ return ret; ++} ++ ++/* ++ * gpiolib gpio_to_irq callback function. Creates a mapping between a GPIO pin ++ * and a virtual IRQ, if not already present. ++ */ ++static int rockchip_gpio_to_irq(struct gpio_chip *gc, unsigned int offset) ++{ ++ struct rockchip_pin_bank *bank = gpiochip_get_data(gc); ++ unsigned int virq; ++ ++ if (!bank->domain) ++ return -ENXIO; ++ ++ virq = irq_create_mapping(bank->domain, offset); ++ ++ return (virq) ? : -ENXIO; ++} ++ ++static const struct gpio_chip rockchip_gpiolib_chip = { ++ .request = gpiochip_generic_request, ++ .free = gpiochip_generic_free, ++ .set = rockchip_gpio_set, ++ .get = rockchip_gpio_get, ++ .get_direction = rockchip_gpio_get_direction, ++ .direction_input = rockchip_gpio_direction_input, ++ .direction_output = rockchip_gpio_direction_output, ++ .set_config = rockchip_gpio_set_config, ++ .to_irq = rockchip_gpio_to_irq, ++ .owner = THIS_MODULE, ++}; ++ ++static void rockchip_irq_demux(struct irq_desc *desc) ++{ ++ struct irq_chip *chip = irq_desc_get_chip(desc); ++ struct rockchip_pin_bank *bank = irq_desc_get_handler_data(desc); ++ const struct rockchip_gpio_regs *reg = bank->gpio_regs; ++ u32 pend; ++ ++ dev_dbg(bank->dev, "got irq for bank %s\n", bank->name); ++ ++ chained_irq_enter(chip, desc); ++ ++ pend = readl_relaxed(bank->reg_base + reg->int_status); ++ ++ while (pend) { ++ unsigned int irq, virq; ++ ++ irq = __ffs(pend); ++ pend &= ~BIT(irq); ++ virq = irq_find_mapping(bank->domain, irq); ++ ++ if (!virq) { ++ dev_err(bank->dev, "unmapped irq %d\n", irq); ++ continue; ++ } ++ ++ dev_dbg(bank->dev, "handling irq %d\n", irq); ++ ++ /* ++ * Triggering IRQ on both rising and falling edge ++ * needs manual intervention. ++ */ ++ if (bank->toggle_edge_mode & BIT(irq)) { ++ u32 data, data_old, polarity; ++ unsigned long flags; ++ ++ data = readl_relaxed(bank->reg_base + reg->ext_port); ++ do { ++ raw_spin_lock_irqsave(&bank->slock, flags); ++ ++ polarity = readl_relaxed(bank->reg_base + ++ reg->int_polarity); ++ if (data & BIT(irq)) ++ polarity &= ~BIT(irq); ++ else ++ polarity |= BIT(irq); ++ writel(polarity, bank->reg_base + ++ reg->int_polarity); ++ ++ raw_spin_unlock_irqrestore(&bank->slock, flags); ++ ++ data_old = data; ++ data = readl_relaxed(bank->reg_base + ++ reg->ext_port); ++ } while ((data & BIT(irq)) != (data_old & BIT(irq))); ++ } ++ ++ generic_handle_irq(virq); ++ } ++ ++ chained_irq_exit(chip, desc); ++} ++ ++static int rockchip_irq_set_type(struct irq_data *d, unsigned int type) ++{ ++ struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); ++ struct rockchip_pin_bank *bank = gc->private; ++ u32 mask = BIT(d->hwirq); ++ u32 polarity; ++ u32 level; ++ u32 data; ++ unsigned long flags; ++ int ret = 0; ++ ++ raw_spin_lock_irqsave(&bank->slock, flags); ++ ++ rockchip_gpio_writel_bit(bank, d->hwirq, 0, ++ bank->gpio_regs->port_ddr); ++ ++ raw_spin_unlock_irqrestore(&bank->slock, flags); ++ ++ if (type & IRQ_TYPE_EDGE_BOTH) ++ irq_set_handler_locked(d, handle_edge_irq); ++ else ++ irq_set_handler_locked(d, handle_level_irq); ++ ++ raw_spin_lock_irqsave(&bank->slock, flags); ++ ++ level = rockchip_gpio_readl(bank, bank->gpio_regs->int_type); ++ polarity = rockchip_gpio_readl(bank, bank->gpio_regs->int_polarity); ++ ++ switch (type) { ++ case IRQ_TYPE_EDGE_BOTH: ++ if (bank->gpio_type == GPIO_TYPE_V2) { ++ bank->toggle_edge_mode &= ~mask; ++ rockchip_gpio_writel_bit(bank, d->hwirq, 1, ++ bank->gpio_regs->int_bothedge); ++ goto out; ++ } else { ++ bank->toggle_edge_mode |= mask; ++ level |= mask; ++ ++ /* ++ * Determine gpio state. If 1 next interrupt should be falling ++ * otherwise rising. ++ */ ++ data = readl(bank->reg_base + bank->gpio_regs->ext_port); ++ if (data & mask) ++ polarity &= ~mask; ++ else ++ polarity |= mask; ++ } ++ break; ++ case IRQ_TYPE_EDGE_RISING: ++ bank->toggle_edge_mode &= ~mask; ++ level |= mask; ++ polarity |= mask; ++ break; ++ case IRQ_TYPE_EDGE_FALLING: ++ bank->toggle_edge_mode &= ~mask; ++ level |= mask; ++ polarity &= ~mask; ++ break; ++ case IRQ_TYPE_LEVEL_HIGH: ++ bank->toggle_edge_mode &= ~mask; ++ level &= ~mask; ++ polarity |= mask; ++ break; ++ case IRQ_TYPE_LEVEL_LOW: ++ bank->toggle_edge_mode &= ~mask; ++ level &= ~mask; ++ polarity &= ~mask; ++ break; ++ default: ++ ret = -EINVAL; ++ goto out; ++ } ++ ++ rockchip_gpio_writel(bank, level, bank->gpio_regs->int_type); ++ rockchip_gpio_writel(bank, polarity, bank->gpio_regs->int_polarity); ++out: ++ raw_spin_unlock_irqrestore(&bank->slock, flags); ++ ++ return ret; ++} ++ ++static void rockchip_irq_suspend(struct irq_data *d) ++{ ++ struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); ++ struct rockchip_pin_bank *bank = gc->private; ++ ++ bank->saved_masks = irq_reg_readl(gc, bank->gpio_regs->int_mask); ++ irq_reg_writel(gc, ~gc->wake_active, bank->gpio_regs->int_mask); ++} ++ ++static void rockchip_irq_resume(struct irq_data *d) ++{ ++ struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); ++ struct rockchip_pin_bank *bank = gc->private; ++ ++ irq_reg_writel(gc, bank->saved_masks, bank->gpio_regs->int_mask); ++} ++ ++static int rockchip_interrupts_register(struct rockchip_pin_bank *bank) ++{ ++ unsigned int clr = IRQ_NOREQUEST | IRQ_NOPROBE | IRQ_NOAUTOEN; ++ struct irq_chip_generic *gc; ++ int ret; ++ ++ bank->domain = irq_domain_add_linear(bank->of_node, 32, ++ &irq_generic_chip_ops, NULL); ++ if (!bank->domain) { ++ dev_warn(bank->dev, "could not initialize irq domain for bank %s\n", bank->name); ++ return -EINVAL; ++ } ++ ++ ret = irq_alloc_domain_generic_chips(bank->domain, 32, 1, ++ bank->name, handle_level_irq, ++ clr, 0, 0); ++ if (ret) { ++ dev_err(bank->dev, "could not alloc generic chips for bank %s\n", bank->name); ++ irq_domain_remove(bank->domain); ++ return ret; ++ } ++ ++ gc = irq_get_domain_generic_chip(bank->domain, 0); ++ if (bank->gpio_type == GPIO_TYPE_V2) { ++ gc->reg_writel = gpio_writel_v2; ++ gc->reg_readl = gpio_readl_v2; ++ } ++ gc->reg_base = bank->reg_base; ++ gc->private = bank; ++ gc->chip_types[0].regs.mask = bank->gpio_regs->int_mask; ++ gc->chip_types[0].regs.ack = bank->gpio_regs->port_eoi; ++ gc->chip_types[0].chip.irq_ack = irq_gc_ack_set_bit; ++ gc->chip_types[0].chip.irq_mask = irq_gc_mask_set_bit; ++ gc->chip_types[0].chip.irq_unmask = irq_gc_mask_clr_bit; ++ gc->chip_types[0].chip.irq_enable = irq_gc_mask_clr_bit; ++ gc->chip_types[0].chip.irq_disable = irq_gc_mask_set_bit; ++ gc->chip_types[0].chip.irq_set_wake = irq_gc_set_wake; ++ gc->chip_types[0].chip.irq_suspend = rockchip_irq_suspend; ++ gc->chip_types[0].chip.irq_resume = rockchip_irq_resume; ++ gc->chip_types[0].chip.irq_set_type = rockchip_irq_set_type; ++ gc->wake_enabled = IRQ_MSK(bank->nr_pins); ++ ++ /* ++ * Linux assumes that all interrupts start out disabled/masked. ++ * Our driver only uses the concept of masked and always keeps ++ * things enabled, so for us that's all masked and all enabled. ++ */ ++ rockchip_gpio_writel(bank, 0xffffffff, bank->gpio_regs->int_mask); ++ rockchip_gpio_writel(bank, 0xffffffff, bank->gpio_regs->port_eoi); ++ rockchip_gpio_writel(bank, 0xffffffff, bank->gpio_regs->int_en); ++ gc->mask_cache = 0xffffffff; ++ ++ irq_set_chained_handler_and_data(bank->irq, ++ rockchip_irq_demux, bank); ++ ++ return 0; ++} ++ ++static int rockchip_gpiolib_register(struct rockchip_pin_bank *bank) ++{ ++ struct gpio_chip *gc; ++ int ret; ++ ++ bank->gpio_chip = rockchip_gpiolib_chip; ++ ++ gc = &bank->gpio_chip; ++ gc->base = bank->pin_base; ++ gc->ngpio = bank->nr_pins; ++ gc->label = bank->name; ++ gc->parent = bank->dev; ++#ifdef CONFIG_OF_GPIO ++ gc->of_node = of_node_get(bank->of_node); ++#endif ++ ++ ret = gpiochip_add_data(gc, bank); ++ if (ret) { ++ dev_err(bank->dev, "failed to add gpiochip %s, %d\n", gc->label, ret); ++ return ret; ++ } ++ ++ /* ++ * For DeviceTree-supported systems, the gpio core checks the ++ * pinctrl's device node for the "gpio-ranges" property. ++ * If it is present, it takes care of adding the pin ranges ++ * for the driver. In this case the driver can skip ahead. ++ * ++ * In order to remain compatible with older, existing DeviceTree ++ * files which don't set the "gpio-ranges" property or systems that ++ * utilize ACPI the driver has to call gpiochip_add_pin_range(). ++ */ ++ if (!of_property_read_bool(bank->of_node, "gpio-ranges")) { ++ struct device_node *pctlnp = of_get_parent(bank->of_node); ++ struct pinctrl_dev *pctldev = NULL; ++ ++ if (!pctlnp) ++ return -ENODATA; ++ ++ pctldev = of_pinctrl_get(pctlnp); ++ if (!pctldev) ++ return -ENODEV; ++ ++ ret = gpiochip_add_pin_range(gc, dev_name(pctldev->dev), 0, gc->base, gc->ngpio); ++ if (ret) { ++ dev_err(bank->dev, "Failed to add pin range\n"); ++ goto fail; ++ } ++ } ++ ++ ret = rockchip_interrupts_register(bank); ++ if (ret) { ++ dev_err(bank->dev, "failed to register interrupt, %d\n", ret); ++ goto fail; ++ } ++ ++ return 0; ++ ++fail: ++ gpiochip_remove(&bank->gpio_chip); ++ ++ return ret; ++} ++ ++static int rockchip_get_bank_data(struct rockchip_pin_bank *bank) ++{ ++ struct resource res; ++ int id = 0; ++ ++ if (of_address_to_resource(bank->of_node, 0, &res)) ++ return -ENOENT; ++ ++ bank->reg_base = devm_ioremap_resource(bank->dev, &res); ++ if (IS_ERR(bank->reg_base)) ++ return PTR_ERR(bank->reg_base); ++ ++ bank->irq = irq_of_parse_and_map(bank->of_node, 0); ++ if (!bank->irq) ++ return -EINVAL; ++ ++ bank->clk = of_clk_get(bank->of_node, 0); ++ if (IS_ERR(bank->clk)) ++ return PTR_ERR(bank->clk); ++ ++ clk_prepare_enable(bank->clk); ++ id = readl(bank->reg_base + gpio_regs_v2.version_id); ++ ++ /* If not gpio v2, that is default to v1. */ ++ if (id == GPIO_TYPE_V2) { ++ bank->gpio_regs = &gpio_regs_v2; ++ bank->gpio_type = GPIO_TYPE_V2; ++ bank->db_clk = of_clk_get(bank->of_node, 1); ++ if (IS_ERR(bank->db_clk)) { ++ dev_err(bank->dev, "cannot find debounce clk\n"); ++ bank->db_clk = NULL; ++ return -EINVAL; ++ } ++ } else { ++ bank->gpio_regs = &gpio_regs_v1; ++ bank->gpio_type = GPIO_TYPE_V1; ++ } ++ ++ return 0; ++} ++ ++static struct rockchip_pin_bank *rockchip_gpio_find_bank(struct pinctrl_dev *pctldev, int id) ++{ ++ struct rockchip_pinctrl *info; ++ struct rockchip_pin_bank *bank; ++ int i, found = 0; ++ ++ info = pinctrl_dev_get_drvdata(pctldev); ++ bank = info->ctrl->pin_banks; ++ for (i = 0; i < info->ctrl->nr_banks; i++, bank++) { ++ if (bank->bank_num == id) { ++ found = 1; ++ break; ++ } ++ } ++ ++ return found ? bank : NULL; ++} ++ ++static int rockchip_gpio_probe(struct platform_device *pdev) ++{ ++ struct device *dev = &pdev->dev; ++ struct device_node *np = pdev->dev.of_node; ++ struct device_node *pctlnp = of_get_parent(np); ++ struct pinctrl_dev *pctldev = NULL; ++ struct rockchip_pin_bank *bank = NULL; ++ static int gpio; ++ int id, ret; ++ ++ if (!np || !pctlnp) ++ return -ENODEV; ++ ++ pctldev = of_pinctrl_get(pctlnp); ++ if (!pctldev) ++ return -EPROBE_DEFER; ++ ++ id = of_alias_get_id(np, "gpio"); ++ if (id < 0) ++ id = gpio++; ++ ++ bank = rockchip_gpio_find_bank(pctldev, id); ++ if (!bank) ++ return -EINVAL; ++ ++ bank->dev = dev; ++ bank->of_node = dev->of_node; ++ ++ raw_spin_lock_init(&bank->slock); ++ ++ ret = rockchip_get_bank_data(bank); ++ if (ret) ++ return ret; ++ ++ ret = rockchip_gpiolib_register(bank); ++ if (ret) ++ goto err_clk; ++ ++ platform_set_drvdata(pdev, bank); ++ dev_info(dev, "probed %s (%s)\n", bank->name, dev_name(dev)); ++ ++ return 0; ++err_clk: ++ clk_disable_unprepare(bank->clk); ++ ++ return ret; ++} ++ ++static int rockchip_gpio_remove(struct platform_device *pdev) ++{ ++ struct rockchip_pin_bank *bank = platform_get_drvdata(pdev); ++ ++ clk_disable_unprepare(bank->clk); ++ gpiochip_remove(&bank->gpio_chip); ++ ++ return 0; ++} ++ ++static const struct of_device_id rockchip_gpio_match[] = { ++ { .compatible = "rockchip,gpio-bank", }, ++ { .compatible = "rockchip,rk3188-gpio-bank0" }, ++ { }, ++}; ++ ++static struct platform_driver rockchip_gpio_driver = { ++ .probe = rockchip_gpio_probe, ++ .remove = rockchip_gpio_remove, ++ .driver = { ++ .name = "rockchip-gpio", ++ .of_match_table = rockchip_gpio_match, ++ }, ++}; ++ ++static int __init rockchip_gpio_init(void) ++{ ++ return platform_driver_register(&rockchip_gpio_driver); ++} ++postcore_initcall(rockchip_gpio_init); ++ ++static void __exit rockchip_gpio_exit(void) ++{ ++ platform_driver_unregister(&rockchip_gpio_driver); ++} ++module_exit(rockchip_gpio_exit); ++ ++MODULE_DESCRIPTION("Rockchip gpio driver"); ++MODULE_ALIAS("platform:rockchip-gpio"); ++MODULE_LICENSE("GPL v2"); ++MODULE_DEVICE_TABLE(of, rockchip_gpio_match); +diff --git a/drivers/gpio/gpiolib-of.c b/drivers/gpio/gpiolib-of.c +index 2f895a2b8411..be9002d622d5 100644 +--- a/drivers/gpio/gpiolib-of.c ++++ b/drivers/gpio/gpiolib-of.c +@@ -1039,3 +1039,14 @@ void of_gpiochip_remove(struct gpio_chip *chip) + { + of_node_put(chip->of_node); + } ++ ++void of_gpio_dev_init(struct gpio_chip *gc, struct gpio_device *gdev) ++{ ++ /* If the gpiochip has an assigned OF node this takes precedence */ ++ if (gc->of_node) ++ gdev->dev.of_node = gc->of_node; ++ else ++ gc->of_node = gdev->dev.of_node; ++ if (gdev->dev.of_node) ++ gdev->dev.fwnode = of_fwnode_handle(gdev->dev.of_node); ++} +diff --git a/drivers/gpio/gpiolib-of.h b/drivers/gpio/gpiolib-of.h +index ed26664f1537..8af2bc899aab 100644 +--- a/drivers/gpio/gpiolib-of.h ++++ b/drivers/gpio/gpiolib-of.h +@@ -15,6 +15,7 @@ int of_gpiochip_add(struct gpio_chip *gc); + void of_gpiochip_remove(struct gpio_chip *gc); + int of_gpio_get_count(struct device *dev, const char *con_id); + bool of_gpio_need_valid_mask(const struct gpio_chip *gc); ++void of_gpio_dev_init(struct gpio_chip *gc, struct gpio_device *gdev); + #else + static inline struct gpio_desc *of_find_gpio(struct device *dev, + const char *con_id, +@@ -33,6 +34,10 @@ static inline bool of_gpio_need_valid_mask(const struct gpio_chip *gc) + { + return false; + } ++static inline void of_gpio_dev_init(struct gpio_chip *gc, ++ struct gpio_device *gdev) ++{ ++} + #endif /* CONFIG_OF_GPIO */ + + extern struct notifier_block gpio_of_notifier; +diff --git a/drivers/gpu/Makefile b/drivers/gpu/Makefile +index 835c88318cec..b66e520ebbdd 100644 +--- a/drivers/gpu/Makefile ++++ b/drivers/gpu/Makefile +@@ -3,6 +3,6 @@ + # taken to initialize them in the correct order. Link order is the only way + # to ensure this currently. + obj-$(CONFIG_TEGRA_HOST1X) += host1x/ +-obj-y += drm/ vga/ ++obj-y += drm/ vga/ arm/ + obj-$(CONFIG_IMX_IPUV3_CORE) += ipu-v3/ + obj-$(CONFIG_TRACE_GPU_MEM) += trace/ +diff --git a/drivers/gpu/arm/Kbuild b/drivers/gpu/arm/Kbuild +new file mode 100755 +index 000000000000..b3466f167d8e +--- /dev/null ++++ b/drivers/gpu/arm/Kbuild +@@ -0,0 +1,31 @@ ++# ++# (C) COPYRIGHT 2012 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# GNU General Public License for more details. ++# ++# You should have received a copy of the GNU General Public License ++# along with this program; if not, you can access it online at ++# http://www.gnu.org/licenses/gpl-2.0.html. ++# ++# SPDX-License-Identifier: GPL-2.0 ++# ++# ++ ++ ++# obj-$(CONFIG_MALI_MIDGARD_FOR_LINUX) += midgard_for_linux/ ++ ++obj-$(CONFIG_MALI_MIDGARD_FOR_ANDROID) += midgard/ ++ ++obj-$(CONFIG_MALI400) += mali400/ ++ ++obj-$(CONFIG_MALI_BIFROST_FOR_ANDROID) += bifrost/ ++ ++obj-$(CONFIG_MALI_BIFROST_FOR_LINUX) += bifrost_for_linux/ +diff --git a/drivers/gpu/arm/Kconfig b/drivers/gpu/arm/Kconfig +new file mode 100755 +index 000000000000..599711c18af9 +--- /dev/null ++++ b/drivers/gpu/arm/Kconfig +@@ -0,0 +1,48 @@ ++# ++# (C) COPYRIGHT 2012 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# GNU General Public License for more details. ++# ++# You should have received a copy of the GNU General Public License ++# along with this program; if not, you can access it online at ++# http://www.gnu.org/licenses/gpl-2.0.html. ++# ++# SPDX-License-Identifier: GPL-2.0 ++# ++# ++# ++source "drivers/gpu/arm/mali400/mali/Kconfig" ++ ++choice ++ prompt "Mali Midgard driver" ++ ++config MALI_MIDGARD_FOR_ANDROID ++ bool "Mali Midgard for Android" ++ ++config MALI_MIDGARD_FOR_LINUX ++ bool "Mali Midgard for Linux only" ++ ++endchoice ++ ++source "drivers/gpu/arm/midgard/Kconfig" ++ ++choice ++ prompt "Mali bifrost driver" ++ ++config MALI_BIFROST_FOR_ANDROID ++ bool "Mali Bifrost for Android" ++ ++config MALI_BIFROST_FOR_LINUX ++ bool "Mali Bifrost for Linux only" ++ ++endchoice ++ ++source "drivers/gpu/arm/bifrost/Kconfig" +diff --git a/drivers/gpu/arm/bifrost/Kbuild b/drivers/gpu/arm/bifrost/Kbuild +new file mode 100755 +index 000000000000..c05dc8399027 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/Kbuild +@@ -0,0 +1,229 @@ ++# ++# (C) COPYRIGHT 2012-2020 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# GNU General Public License for more details. ++# ++# You should have received a copy of the GNU General Public License ++# along with this program; if not, you can access it online at ++# http://www.gnu.org/licenses/gpl-2.0.html. ++# ++# SPDX-License-Identifier: GPL-2.0 ++# ++# ++ ++# Driver version string which is returned to userspace via an ioctl ++MALI_RELEASE_NAME ?= "g2p0-01eac0" ++ ++# Paths required for build ++ ++# make $(src) as absolute path if it isn't already, by prefixing $(srctree) ++src:=$(if $(patsubst /%,,$(src)),$(srctree)/$(src),$(src)) ++KBASE_PATH = $(src) ++KBASE_PLATFORM_PATH = $(KBASE_PATH)/platform_dummy ++UMP_PATH = $(src)/../../../base ++ ++# Set up defaults if not defined by build system ++MALI_CUSTOMER_RELEASE ?= 1 ++MALI_USE_CSF ?= 0 ++MALI_UNIT_TEST ?= 0 ++MALI_KERNEL_TEST_API ?= 0 ++MALI_COVERAGE ?= 0 ++MALI_JIT_PRESSURE_LIMIT_BASE ?= 1 ++CONFIG_MALI_PLATFORM_NAME ?= "devicetree" ++# Experimental features (corresponding -D definition should be appended to ++# DEFINES below, e.g. for MALI_EXPERIMENTAL_FEATURE, ++# -DMALI_EXPERIMENTAL_FEATURE=$(MALI_EXPERIMENTAL_FEATURE) should be appended) ++# ++# Experimental features must default to disabled, e.g.: ++# MALI_EXPERIMENTAL_FEATURE ?= 0 ++MALI_INCREMENTAL_RENDERING ?= 0 ++ ++# Set up our defines, which will be passed to gcc ++DEFINES = \ ++ -DMALI_CUSTOMER_RELEASE=$(MALI_CUSTOMER_RELEASE) \ ++ -DMALI_USE_CSF=$(MALI_USE_CSF) \ ++ -DMALI_KERNEL_TEST_API=$(MALI_KERNEL_TEST_API) \ ++ -DMALI_UNIT_TEST=$(MALI_UNIT_TEST) \ ++ -DMALI_COVERAGE=$(MALI_COVERAGE) \ ++ -DMALI_RELEASE_NAME=\"$(MALI_RELEASE_NAME)\" \ ++ -DMALI_JIT_PRESSURE_LIMIT_BASE=$(MALI_JIT_PRESSURE_LIMIT_BASE) \ ++ -DMALI_INCREMENTAL_RENDERING=$(MALI_INCREMENTAL_RENDERING) ++ ++ifeq ($(KBUILD_EXTMOD),) ++# in-tree ++DEFINES +=-DMALI_KBASE_PLATFORM_PATH=../../$(src)/platform/$(CONFIG_MALI_PLATFORM_NAME) ++else ++# out-of-tree ++DEFINES +=-DMALI_KBASE_PLATFORM_PATH=$(src)/platform/$(CONFIG_MALI_PLATFORM_NAME) ++endif ++ ++DEFINES += -I$(srctree)/drivers/staging/android ++ ++DEFINES += -DMALI_KBASE_BUILD ++ ++# Use our defines when compiling ++ccflags-y += $(DEFINES) -I$(KBASE_PATH) -I$(KBASE_PLATFORM_PATH) -I$(UMP_PATH) -I$(srctree)/include/linux ++subdir-ccflags-y += $(DEFINES) -I$(KBASE_PATH) -I$(KBASE_PLATFORM_PATH) -I$(UMP_PATH) -I$(srctree)/include/linux ++ ++SRC := \ ++ context/mali_kbase_context.c \ ++ debug/mali_kbase_debug_ktrace.c \ ++ device/mali_kbase_device.c \ ++ device/mali_kbase_device_hw.c \ ++ mali_kbase_cache_policy.c \ ++ mali_kbase_ccswe.c \ ++ mali_kbase_mem.c \ ++ mali_kbase_mem_pool_group.c \ ++ mali_kbase_native_mgm.c \ ++ mali_kbase_ctx_sched.c \ ++ mali_kbase_jm.c \ ++ mali_kbase_gpuprops.c \ ++ mali_kbase_pm.c \ ++ mali_kbase_config.c \ ++ mali_kbase_vinstr.c \ ++ mali_kbase_hwcnt.c \ ++ mali_kbase_hwcnt_backend_jm.c \ ++ mali_kbase_hwcnt_gpu.c \ ++ mali_kbase_hwcnt_legacy.c \ ++ mali_kbase_hwcnt_types.c \ ++ mali_kbase_hwcnt_virtualizer.c \ ++ mali_kbase_softjobs.c \ ++ mali_kbase_hw.c \ ++ mali_kbase_debug.c \ ++ mali_kbase_gpu_memory_debugfs.c \ ++ mali_kbase_mem_linux.c \ ++ mali_kbase_core_linux.c \ ++ mali_kbase_mem_profile_debugfs.c \ ++ mmu/mali_kbase_mmu.c \ ++ mmu/mali_kbase_mmu_hw_direct.c \ ++ mmu/mali_kbase_mmu_mode_lpae.c \ ++ mmu/mali_kbase_mmu_mode_aarch64.c \ ++ mali_kbase_disjoint_events.c \ ++ mali_kbase_debug_mem_view.c \ ++ mali_kbase_smc.c \ ++ mali_kbase_mem_pool.c \ ++ mali_kbase_mem_pool_debugfs.c \ ++ mali_kbase_debugfs_helper.c \ ++ mali_kbase_strings.c \ ++ mali_kbase_as_fault_debugfs.c \ ++ mali_kbase_regs_history_debugfs.c \ ++ mali_power_gpu_frequency_trace.c \ ++ mali_kbase_trace_gpu_mem.c \ ++ thirdparty/mali_kbase_mmap.c \ ++ tl/mali_kbase_timeline.c \ ++ tl/mali_kbase_timeline_io.c \ ++ tl/mali_kbase_tlstream.c \ ++ tl/mali_kbase_tracepoints.c \ ++ gpu/mali_kbase_gpu.c ++ ++ifeq ($(MALI_USE_CSF),1) ++ SRC += \ ++ debug/backend/mali_kbase_debug_ktrace_csf.c \ ++ device/backend/mali_kbase_device_csf.c \ ++ device/backend/mali_kbase_device_hw_csf.c \ ++ gpu/backend/mali_kbase_gpu_fault_csf.c \ ++ tl/backend/mali_kbase_timeline_csf.c \ ++ mmu/backend/mali_kbase_mmu_csf.c \ ++ context/backend/mali_kbase_context_csf.c ++else ++ SRC += \ ++ mali_kbase_dummy_job_wa.c \ ++ mali_kbase_debug_job_fault.c \ ++ mali_kbase_event.c \ ++ mali_kbase_jd.c \ ++ mali_kbase_jd_debugfs.c \ ++ mali_kbase_js.c \ ++ mali_kbase_js_ctx_attr.c \ ++ mali_kbase_kinstr_jm.c \ ++ debug/backend/mali_kbase_debug_ktrace_jm.c \ ++ device/backend/mali_kbase_device_jm.c \ ++ device/backend/mali_kbase_device_hw_jm.c \ ++ gpu/backend/mali_kbase_gpu_fault_jm.c \ ++ tl/backend/mali_kbase_timeline_jm.c \ ++ mmu/backend/mali_kbase_mmu_jm.c \ ++ context/backend/mali_kbase_context_jm.c ++endif ++ ++ifeq ($(CONFIG_MALI_CINSTR_GWT),y) ++ SRC += mali_kbase_gwt.c ++endif ++ ++ifeq ($(MALI_UNIT_TEST),1) ++ SRC += tl/mali_kbase_timeline_test.c ++endif ++ ++ifeq ($(MALI_CUSTOMER_RELEASE),0) ++ SRC += mali_kbase_regs_dump_debugfs.c ++endif ++ ++ ++ccflags-y += -I$(KBASE_PATH) -I$(KBASE_PATH)/debug \ ++ -I$(KBASE_PATH)/debug/backend ++ ++# Tell the Linux build system from which .o file to create the kernel module ++obj-$(CONFIG_MALI_BIFROST) += bifrost_kbase.o ++ ++# Tell the Linux build system to enable building of our .c files ++bifrost_kbase-y := $(SRC:.c=.o) ++ ++# Kconfig passes in the name with quotes for in-tree builds - remove them. ++platform_name := $(shell echo $(CONFIG_MALI_PLATFORM_NAME)) ++MALI_PLATFORM_DIR := platform/$(platform_name) ++ccflags-y += -I$(src)/$(MALI_PLATFORM_DIR) ++include $(src)/$(MALI_PLATFORM_DIR)/Kbuild ++ ++ifeq ($(CONFIG_MALI_BIFROST_DEVFREQ),y) ++ ifeq ($(CONFIG_DEVFREQ_THERMAL),y) ++ include $(src)/ipa/Kbuild ++ endif ++endif ++ ++ifeq ($(MALI_USE_CSF),1) ++ include $(src)/csf/Kbuild ++else ++# empty ++endif ++ ++ifeq ($(CONFIG_MALI_ARBITER_SUPPORT),y) ++ include $(src)/arbiter/Kbuild ++else ++# empty ++endif ++ ++ifeq ($(MALI_USE_CSF),0) ++ bifrost_kbase-$(CONFIG_MALI_BIFROST_DMA_FENCE) += \ ++ mali_kbase_fence_ops.o \ ++ mali_kbase_dma_fence.o \ ++ mali_kbase_fence.o ++ ++ bifrost_kbase-$(CONFIG_SYNC_FILE) += \ ++ mali_kbase_fence_ops.o \ ++ mali_kbase_fence.o ++endif ++ ++bifrost_kbase-$(CONFIG_SYNC) += \ ++ mali_kbase_sync_android.o \ ++ mali_kbase_sync_common.o ++ ++bifrost_kbase-$(CONFIG_SYNC_FILE) += \ ++ mali_kbase_fence_ops.o \ ++ mali_kbase_sync_file.o \ ++ mali_kbase_sync_common.o ++ ++include $(src)/backend/gpu/Kbuild ++bifrost_kbase-y += $(BACKEND:.c=.o) ++ ++ ++ccflags-y += -I$(src)/backend/gpu ++subdir-ccflags-y += -I$(src)/backend/gpu ++ ++# For kutf and mali_kutf_irq_latency_test ++obj-$(CONFIG_MALI_KUTF) += tests/ +diff --git a/drivers/gpu/arm/bifrost/Kconfig b/drivers/gpu/arm/bifrost/Kconfig +new file mode 100755 +index 000000000000..ccb16671047a +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/Kconfig +@@ -0,0 +1,308 @@ ++# ++# (C) COPYRIGHT 2012-2020 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# GNU General Public License for more details. ++# ++# You should have received a copy of the GNU General Public License ++# along with this program; if not, you can access it online at ++# http://www.gnu.org/licenses/gpl-2.0.html. ++# ++# SPDX-License-Identifier: GPL-2.0 ++# ++# ++ ++ ++menuconfig MALI_BIFROST ++ tristate "Mali Bifrost series support" ++ select GPU_TRACEPOINTS if ANDROID ++ select DMA_SHARED_BUFFER ++ default n ++ help ++ Enable this option to build support for a ARM Mali Bifrost GPU. ++ ++ To compile this driver as a module, choose M here: ++ this will generate a single module, called mali_kbase. ++ ++config MALI_BIFROST_GATOR_SUPPORT ++ bool "Enable Streamline tracing support" ++ depends on MALI_BIFROST ++ default n ++ help ++ Enables kbase tracing used by the Arm Streamline Performance Analyzer. ++ The tracepoints are used to derive GPU activity charts in Streamline. ++ ++config MALI_BIFROST_DVFS ++ bool "Enable legacy DVFS" ++ depends on MALI_BIFROST && !MALI_BIFROST_DEVFREQ ++ default n ++ help ++ Choose this option to enable legacy DVFS in the Mali Midgard DDK. ++ ++config MALI_BIFROST_ENABLE_TRACE ++ bool "Enable kbase tracing" ++ depends on MALI_BIFROST ++ default y if MALI_BIFROST_DEBUG ++ default n ++ help ++ Enables tracing in kbase. Trace log available through ++ the "mali_trace" debugfs file, when the CONFIG_DEBUG_FS is enabled ++ ++config MALI_BIFROST_DEVFREQ ++ bool "devfreq support for Mali" ++ depends on MALI_BIFROST && PM_DEVFREQ ++ help ++ Support devfreq for Mali. ++ ++ Using the devfreq framework and, by default, the simpleondemand ++ governor, the frequency of Mali will be dynamically selected from the ++ available OPPs. ++ ++config MALI_BIFROST_DMA_FENCE ++ bool "DMA_BUF fence support for Mali" ++ depends on MALI_BIFROST ++ default n ++ help ++ Support DMA_BUF fences for Mali. ++ ++ This option should only be enabled if the Linux Kernel has built in ++ support for DMA_BUF fences. ++ ++config MALI_PLATFORM_NAME ++ depends on MALI_BIFROST ++ string "Platform name" ++ default "devicetree" ++ help ++ Enter the name of the desired platform configuration directory to ++ include in the build. 'platform/$(MALI_PLATFORM_NAME)/Kbuild' must ++ exist. ++ ++config MALI_ARBITER_SUPPORT ++ bool "Enable arbiter support for Mali" ++ depends on MALI_BIFROST ++ default n ++ help ++ Enable support for the arbiter interface in the driver. ++ This allows an external arbiter to manage driver access ++ to GPU hardware in a virtualized environment ++ ++ If unsure, say N. ++ ++# MALI_BIFROST_EXPERT configuration options ++ ++menuconfig MALI_BIFROST_EXPERT ++ depends on MALI_BIFROST ++ bool "Enable Expert Settings" ++ default n ++ help ++ Enabling this option and modifying the default settings may produce a driver with performance or ++ other limitations. ++ ++config MALI_CORESTACK ++ bool "Support controlling power to the GPU core stack" ++ depends on MALI_BIFROST && MALI_BIFROST_EXPERT ++ default n ++ help ++ Enabling this feature on supported GPUs will let the driver powering ++ on/off the GPU core stack independently without involving the Power ++ Domain Controller. This should only be enabled on platforms which ++ integration of the PDC to the Mali GPU is known to be problematic. ++ This feature is currently only supported on t-Six and t-HEx GPUs. ++ ++ If unsure, say N. ++ ++config MALI_BIFROST_DEBUG ++ bool "Debug build" ++ depends on MALI_BIFROST && MALI_BIFROST_EXPERT ++ default n ++ help ++ Select this option for increased checking and reporting of errors. ++ ++config MALI_BIFROST_FENCE_DEBUG ++ bool "Debug sync fence usage" ++ depends on MALI_BIFROST && MALI_BIFROST_EXPERT && (SYNC || SYNC_FILE) ++ default y if MALI_BIFROST_DEBUG ++ help ++ Select this option to enable additional checking and reporting on the ++ use of sync fences in the Mali driver. ++ ++ This will add a 3s timeout to all sync fence waits in the Mali ++ driver, so that when work for Mali has been waiting on a sync fence ++ for a long time a debug message will be printed, detailing what fence ++ is causing the block, and which dependent Mali atoms are blocked as a ++ result of this. ++ ++ The timeout can be changed at runtime through the js_soft_timeout ++ device attribute, where the timeout is specified in milliseconds. ++ ++config MALI_BIFROST_NO_MALI ++ bool "No Mali" ++ depends on MALI_BIFROST && MALI_BIFROST_EXPERT ++ default n ++ help ++ This can be used to test the driver in a simulated environment ++ whereby the hardware is not physically present. If the hardware is physically ++ present it will not be used. This can be used to test the majority of the ++ driver without needing actual hardware or for software benchmarking. ++ All calls to the simulated hardware will complete immediately as if the hardware ++ completed the task. ++ ++config MALI_REAL_HW ++ def_bool !MALI_BIFROST_NO_MALI ++ ++config MALI_BIFROST_ERROR_INJECT ++ bool "Error injection" ++ depends on MALI_BIFROST && MALI_BIFROST_EXPERT && MALI_BIFROST_NO_MALI ++ default n ++ help ++ Enables insertion of errors to test module failure and recovery mechanisms. ++ ++config MALI_BIFROST_SYSTEM_TRACE ++ bool "Enable system event tracing support" ++ depends on MALI_BIFROST && MALI_BIFROST_EXPERT ++ default y if MALI_BIFROST_DEBUG ++ default n ++ help ++ Choose this option to enable system trace events for each ++ kbase event. This is typically used for debugging but has ++ minimal overhead when not in use. Enable only if you know what ++ you are doing. ++ ++config MALI_2MB_ALLOC ++ bool "Attempt to allocate 2MB pages" ++ depends on MALI_BIFROST && MALI_BIFROST_EXPERT ++ default n ++ help ++ Rather than allocating all GPU memory page-by-page, attempt to ++ allocate 2MB pages from the kernel. This reduces TLB pressure and ++ helps to prevent memory fragmentation. ++ ++ If in doubt, say N ++ ++config MALI_PWRSOFT_765 ++ bool "PWRSOFT-765 ticket" ++ depends on MALI_BIFROST && MALI_BIFROST_EXPERT ++ default n ++ help ++ PWRSOFT-765 fixes devfreq cooling devices issues. The fix was merged ++ in kernel v4.10, however if backported into the kernel then this ++ option must be manually selected. ++ ++ If using kernel >= v4.10 then say N, otherwise if devfreq cooling ++ changes have been backported say Y to avoid compilation errors. ++ ++config MALI_MEMORY_FULLY_BACKED ++ bool "Memory fully physically-backed" ++ depends on MALI_BIFROST && MALI_BIFROST_EXPERT ++ default n ++ help ++ This option enables full physical backing of all virtual ++ memory allocations in the kernel. Notice that this build ++ option only affects allocations of grow-on-GPU-page-fault ++ memory. ++ ++config MALI_DMA_BUF_MAP_ON_DEMAND ++ bool "Map imported dma-bufs on demand" ++ depends on MALI_BIFROST ++ default n ++ help ++ This option caused kbase to set up the GPU mapping of imported ++ dma-buf when needed to run atoms. This is the legacy behaviour. ++ ++ This is intended for testing and the option will get removed in the ++ future. ++ ++config MALI_DMA_BUF_LEGACY_COMPAT ++ bool "Enable legacy compatibility cache flush on dma-buf map" ++ depends on MALI_BIFROST && !MALI_DMA_BUF_MAP_ON_DEMAND ++ default n ++ help ++ This option enables compatibility with legacy dma-buf mapping ++ behavior, then the dma-buf is mapped on import, by adding cache ++ maintenance where MALI_DMA_BUF_MAP_ON_DEMAND would do the mapping, ++ including a cache flush. ++ ++ This option might work-around issues related to missing cache ++ flushes in other drivers. This only has an effect for clients using ++ UK 11.18 or older. For later UK versions it is not possible. ++ ++config MALI_HW_ERRATA_1485982_NOT_AFFECTED ++ bool "Disable workaround for BASE_HW_ISSUE_GPU2017_1336" ++ depends on MALI_BIFROST && MALI_BIFROST_EXPERT ++ default n ++ help ++ This option disables the default workaround for GPU2017-1336. The ++ workaround keeps the L2 cache powered up except for powerdown and reset. ++ ++ The workaround introduces a limitation that will prevent the running of ++ protected mode content on fully coherent platforms, as the switch to IO ++ coherency mode requires the L2 to be turned off. ++ ++config MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE ++ bool "Use alternative workaround for BASE_HW_ISSUE_GPU2017_1336" ++ depends on MALI_BIFROST && MALI_BIFROST_EXPERT && !MALI_HW_ERRATA_1485982_NOT_AFFECTED ++ default n ++ help ++ This option uses an alternative workaround for GPU2017-1336. Lowering ++ the GPU clock to a, platform specific, known good frequeuncy before ++ powering down the L2 cache. The clock can be specified in the device ++ tree using the property, opp-mali-errata-1485982. Otherwise the ++ slowest clock will be selected. ++ ++config MALI_GEM5_BUILD ++ bool "Enable build of Mali kernel driver for GEM5" ++ depends on MALI_BIFROST ++ default n ++ help ++ This option is to do a Mali GEM5 build. ++ If unsure, say N. ++ ++# Instrumentation options. ++ ++config MALI_JOB_DUMP ++ bool "Enable system level support needed for job dumping" ++ depends on MALI_BIFROST && MALI_BIFROST_EXPERT ++ default n ++ help ++ Choose this option to enable system level support needed for ++ job dumping. This is typically used for instrumentation but has ++ minimal overhead when not in use. Enable only if you know what ++ you are doing. ++ ++config MALI_BIFROST_PRFCNT_SET_SECONDARY ++ bool "Use secondary set of performance counters" ++ depends on MALI_BIFROST && MALI_BIFROST_EXPERT ++ default n ++ help ++ Select this option to use secondary set of performance counters. Kernel ++ features that depend on an access to the primary set of counters may ++ become unavailable. Enabling this option will prevent power management ++ from working optimally and may cause instrumentation tools to return ++ bogus results. ++ ++ If unsure, say N. ++ ++config MALI_PRFCNT_SET_SECONDARY_VIA_DEBUG_FS ++ bool "Use secondary set of performance counters" ++ depends on MALI_BIFROST && MALI_BIFROST_EXPERT && !MALI_BIFROST_PRFCNT_SET_SECONDARY && DEBUG_FS ++ default n ++ help ++ Select this option to make the secondary set of performance counters ++ available at runtime via debugfs. Kernel features that depend on an ++ access to the primary set of counters may become unavailable. ++ ++ This feature is unsupported and unstable, and may break at any time. ++ Enabling this option will prevent power management from working ++ optimally and may cause instrumentation tools to return bogus results. ++ ++ If unsure, say N. ++ ++source "drivers/gpu/arm/midgard/platform/Kconfig" ++# source "drivers/gpu/arm/midgard/tests/Kconfig" +diff --git a/drivers/gpu/arm/bifrost/Makefile b/drivers/gpu/arm/bifrost/Makefile +new file mode 100755 +index 000000000000..53a12094ec14 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/Makefile +@@ -0,0 +1,38 @@ ++# ++# (C) COPYRIGHT 2010-2019 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# GNU General Public License for more details. ++# ++# You should have received a copy of the GNU General Public License ++# along with this program; if not, you can access it online at ++# http://www.gnu.org/licenses/gpl-2.0.html. ++# ++# SPDX-License-Identifier: GPL-2.0 ++# ++# ++ ++ ++KDIR ?= /lib/modules/$(shell uname -r)/build ++ ++BUSLOG_PATH_RELATIVE = $(CURDIR)/../../../.. ++KBASE_PATH_RELATIVE = $(CURDIR) ++ ++ifeq ($(CONFIG_MALI_BUSLOG),y) ++#Add bus logger symbols ++EXTRA_SYMBOLS += $(BUSLOG_PATH_RELATIVE)/drivers/base/bus_logger/Module.symvers ++endif ++ ++# we get the symbols from modules using KBUILD_EXTRA_SYMBOLS to prevent warnings about unknown functions ++all: ++ $(MAKE) -C $(KDIR) M=$(CURDIR) EXTRA_CFLAGS="-I$(CURDIR)/../../../../include -I$(CURDIR)/../../../../tests/include $(SCONS_CFLAGS)" $(SCONS_CONFIGS) KBUILD_EXTRA_SYMBOLS="$(EXTRA_SYMBOLS)" modules ++ ++clean: ++ $(MAKE) -C $(KDIR) M=$(CURDIR) clean +diff --git a/drivers/gpu/arm/bifrost/Makefile.kbase b/drivers/gpu/arm/bifrost/Makefile.kbase +new file mode 100755 +index 000000000000..6b0f81ee76e8 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/Makefile.kbase +@@ -0,0 +1,23 @@ ++# ++# (C) COPYRIGHT 2010, 2013, 2018 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# GNU General Public License for more details. ++# ++# You should have received a copy of the GNU General Public License ++# along with this program; if not, you can access it online at ++# http://www.gnu.org/licenses/gpl-2.0.html. ++# ++# SPDX-License-Identifier: GPL-2.0 ++# ++# ++ ++EXTRA_CFLAGS += -I$(ROOT) -I$(KBASE_PATH) -I$(KBASE_PATH)/platform_$(PLATFORM) ++ +diff --git a/drivers/gpu/arm/bifrost/Mconfig b/drivers/gpu/arm/bifrost/Mconfig +new file mode 100755 +index 000000000000..99ababfc2d16 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/Mconfig +@@ -0,0 +1,277 @@ ++# ++# (C) COPYRIGHT 2012-2020 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++menuconfig MALI_BIFROST ++ bool "Mali Midgard series support" ++ default y ++ help ++ Enable this option to build support for a ARM Mali Midgard GPU. ++ ++ To compile this driver as a module, choose M here: ++ this will generate a single module, called mali_kbase. ++ ++config MALI_BIFROST_GATOR_SUPPORT ++ bool "Enable Streamline tracing support" ++ depends on MALI_BIFROST && !BACKEND_USER ++ default y ++ help ++ Enables kbase tracing used by the Arm Streamline Performance Analyzer. ++ The tracepoints are used to derive GPU activity charts in Streamline. ++ ++config MALI_BIFROST_DVFS ++ bool "Enable legacy DVFS" ++ depends on MALI_BIFROST && !MALI_BIFROST_DEVFREQ ++ default n ++ help ++ Choose this option to enable legacy DVFS in the Mali Midgard DDK. ++ ++config MALI_BIFROST_ENABLE_TRACE ++ bool "Enable kbase tracing" ++ default y if MALI_BIFROST_DEBUG ++ default n ++ help ++ Enables tracing in kbase. Trace log available through ++ the "mali_trace" debugfs file, when the CONFIG_DEBUG_FS is enabled ++ ++config MALI_BIFROST_DEVFREQ ++ bool "devfreq support for Mali" ++ depends on MALI_BIFROST ++ default y if PLATFORM_JUNO ++ default y if PLATFORM_CUSTOM ++ help ++ Support devfreq for Mali. ++ ++ Using the devfreq framework and, by default, the simpleondemand ++ governor, the frequency of Mali will be dynamically selected from the ++ available OPPs. ++ ++config MALI_BIFROST_DMA_FENCE ++ bool "DMA_BUF fence support for Mali" ++ depends on MALI_BIFROST ++ default n ++ help ++ Support DMA_BUF fences for Mali. ++ ++ This option should only be enabled if the Linux Kernel has built in ++ support for DMA_BUF fences. ++ ++config MALI_PLATFORM_NAME ++ depends on MALI_BIFROST ++ string "Platform name" ++ default "hisilicon" if PLATFORM_HIKEY960 ++ default "hisilicon" if PLATFORM_HIKEY970 ++ default "devicetree" ++ help ++ Enter the name of the desired platform configuration directory to ++ include in the build. 'platform/$(MALI_PLATFORM_NAME)/Kbuild' must ++ exist. ++ ++ When PLATFORM_CUSTOM is set, this needs to be set manually to ++ pick up the desired platform files. ++ ++config MALI_ARBITER_SUPPORT ++ bool "Enable arbiter support for Mali" ++ depends on MALI_BIFROST && !GPU_HAS_CSF ++ default n ++ help ++ Enable support for the arbiter interface in the driver. ++ This allows an external arbiter to manage driver access ++ to GPU hardware in a virtualized environment ++ ++ If unsure, say N. ++ ++# MALI_BIFROST_EXPERT configuration options ++ ++menuconfig MALI_BIFROST_EXPERT ++ depends on MALI_BIFROST ++ bool "Enable Expert Settings" ++ default y ++ help ++ Enabling this option and modifying the default settings may produce a driver with performance or ++ other limitations. ++ ++config MALI_CORESTACK ++ bool "Support controlling power to the GPU core stack" ++ depends on MALI_BIFROST && MALI_BIFROST_EXPERT ++ default n ++ help ++ Enabling this feature on supported GPUs will let the driver powering ++ on/off the GPU core stack independently without involving the Power ++ Domain Controller. This should only be enabled on platforms which ++ integration of the PDC to the Mali GPU is known to be problematic. ++ This feature is currently only supported on t-Six and t-HEx GPUs. ++ ++ If unsure, say N. ++ ++config MALI_BIFROST_DEBUG ++ bool "Debug build" ++ depends on MALI_BIFROST && MALI_BIFROST_EXPERT ++ default y if DEBUG ++ default n ++ help ++ Select this option for increased checking and reporting of errors. ++ ++config MALI_BIFROST_FENCE_DEBUG ++ bool "Debug sync fence usage" ++ depends on MALI_BIFROST && MALI_BIFROST_EXPERT ++ default y if MALI_BIFROST_DEBUG ++ help ++ Select this option to enable additional checking and reporting on the ++ use of sync fences in the Mali driver. ++ ++ This will add a 3s timeout to all sync fence waits in the Mali ++ driver, so that when work for Mali has been waiting on a sync fence ++ for a long time a debug message will be printed, detailing what fence ++ is causing the block, and which dependent Mali atoms are blocked as a ++ result of this. ++ ++ The timeout can be changed at runtime through the js_soft_timeout ++ device attribute, where the timeout is specified in milliseconds. ++ ++choice ++ prompt "Error injection level" ++ default MALI_ERROR_INJECT_NONE ++ help ++ Enables insertion of errors to test module failure and recovery mechanisms. ++ ++config MALI_ERROR_INJECT_NONE ++ bool "disabled" ++ help ++ Error injection is disabled. ++ ++config MALI_ERROR_INJECT_TRACK_LIST ++ bool "error track list" ++ depends on MALI_BIFROST && MALI_BIFROST_EXPERT && NO_MALI ++ help ++ Errors to inject are pre-configured by the user. ++ ++config MALI_ERROR_INJECT_RANDOM ++ bool "random error injection" ++ depends on MALI_BIFROST && MALI_BIFROST_EXPERT && NO_MALI ++ help ++ Injected errors are random, rather than user-driven. ++ ++endchoice ++ ++config MALI_ERROR_INJECT_ON ++ string ++ default "0" if MALI_ERROR_INJECT_NONE ++ default "1" if MALI_ERROR_INJECT_TRACK_LIST ++ default "2" if MALI_ERROR_INJECT_RANDOM ++ ++config MALI_BIFROST_ERROR_INJECT ++ bool ++ default y if !MALI_ERROR_INJECT_NONE ++ ++config MALI_BIFROST_SYSTEM_TRACE ++ bool "Enable system event tracing support" ++ depends on MALI_BIFROST && MALI_BIFROST_EXPERT ++ default y if MALI_BIFROST_DEBUG ++ default n ++ help ++ Choose this option to enable system trace events for each ++ kbase event. This is typically used for debugging but has ++ minimal overhead when not in use. Enable only if you know what ++ you are doing. ++ ++config MALI_2MB_ALLOC ++ bool "Attempt to allocate 2MB pages" ++ depends on MALI_BIFROST && MALI_BIFROST_EXPERT ++ default n ++ help ++ Rather than allocating all GPU memory page-by-page, attempt to ++ allocate 2MB pages from the kernel. This reduces TLB pressure and ++ helps to prevent memory fragmentation. ++ ++ If in doubt, say N ++ ++config MALI_PWRSOFT_765 ++ bool "PWRSOFT-765 ticket" ++ depends on MALI_BIFROST && MALI_BIFROST_EXPERT ++ default n ++ help ++ PWRSOFT-765 fixes devfreq cooling devices issues. However, they are ++ not merged in mainline kernel yet. So this define helps to guard those ++ parts of the code. ++ ++config MALI_MEMORY_FULLY_BACKED ++ bool "Memory fully physically-backed" ++ default n ++ help ++ This option enables full backing of all virtual memory allocations ++ for the kernel. This only affects grow-on-GPU-page-fault memory. ++ ++config MALI_DMA_BUF_MAP_ON_DEMAND ++ bool "Map imported dma-bufs on demand" ++ depends on MALI_BIFROST ++ default n ++ default y if !DMA_BUF_SYNC_IOCTL_SUPPORTED ++ help ++ This option caused kbase to set up the GPU mapping of imported ++ dma-buf when needed to run atoms. This is the legacy behaviour. ++ ++config MALI_DMA_BUF_LEGACY_COMPAT ++ bool "Enable legacy compatibility cache flush on dma-buf map" ++ depends on MALI_BIFROST && !MALI_DMA_BUF_MAP_ON_DEMAND ++ default n ++ help ++ This option enables compatibility with legacy dma-buf mapping ++ behavior, then the dma-buf is mapped on import, by adding cache ++ maintenance where MALI_DMA_BUF_MAP_ON_DEMAND would do the mapping, ++ including a cache flush. ++ ++config MALI_REAL_HW ++ bool ++ default y ++ default n if NO_MALI ++ ++config MALI_HW_ERRATA_1485982_NOT_AFFECTED ++ bool "Disable workaround for BASE_HW_ISSUE_GPU2017_1336" ++ depends on MALI_BIFROST && MALI_BIFROST_EXPERT ++ default n ++ default y if PLATFORM_JUNO ++ help ++ This option disables the default workaround for GPU2017-1336. The ++ workaround keeps the L2 cache powered up except for powerdown and reset. ++ ++ The workaround introduces a limitation that will prevent the running of ++ protected mode content on fully coherent platforms, as the switch to IO ++ coherency mode requires the L2 to be turned off. ++ ++config MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE ++ bool "Use alternative workaround for BASE_HW_ISSUE_GPU2017_1336" ++ depends on MALI_BIFROST && MALI_BIFROST_EXPERT && !MALI_HW_ERRATA_1485982_NOT_AFFECTED ++ default n ++ help ++ This option uses an alternative workaround for GPU2017-1336. Lowering ++ the GPU clock to a, platform specific, known good frequeuncy before ++ powering down the L2 cache. The clock can be specified in the device ++ tree using the property, opp-mali-errata-1485982. Otherwise the ++ slowest clock will be selected. ++ ++config MALI_GEM5_BUILD ++ bool "Enable build of Mali kernel driver for GEM5" ++ depends on MALI_BIFROST ++ default n ++ help ++ This option is to do a Mali GEM5 build. ++ If unsure, say N. ++ ++# Instrumentation options. ++ ++# config MALI_JOB_DUMP exists in the Kernel Kconfig but is configured using CINSTR_JOB_DUMP in Mconfig. ++# config MALI_BIFROST_PRFCNT_SET_SECONDARY exists in the Kernel Kconfig but is configured using CINSTR_SECONDARY_HWC in Mconfig. ++ ++source "kernel/drivers/gpu/arm/midgard/tests/Mconfig" +diff --git a/drivers/gpu/arm/bifrost/arbiter/Kbuild b/drivers/gpu/arm/bifrost/arbiter/Kbuild +new file mode 100755 +index 000000000000..98e47bed223a +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/arbiter/Kbuild +@@ -0,0 +1,24 @@ ++# ++# (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# GNU General Public License for more details. ++# ++# You should have received a copy of the GNU General Public License ++# along with this program; if not, you can access it online at ++# http://www.gnu.org/licenses/gpl-2.0.html. ++# ++# SPDX-License-Identifier: GPL-2.0 ++# ++# ++ ++mali_kbase-y += \ ++ arbiter/mali_kbase_arbif.o \ ++ arbiter/mali_kbase_arbiter_pm.o +diff --git a/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbif.c b/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbif.c +new file mode 100755 +index 000000000000..ddf1a0ce0b05 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbif.c +@@ -0,0 +1,175 @@ ++// SPDX-License-Identifier: GPL-2.0 ++ ++/* ++ * ++ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/** ++ * @file mali_kbase_arbif.c ++ * Mali arbiter interface APIs to share GPU between Virtual Machines ++ */ ++ ++#include ++#include "mali_kbase_arbif.h" ++#include ++#include ++#include ++#include "mali_kbase_arbiter_interface.h" ++ ++static void on_gpu_stop(struct device *dev) ++{ ++ struct kbase_device *kbdev = dev_get_drvdata(dev); ++ ++ KBASE_TLSTREAM_TL_ARBITER_STOP_REQUESTED(kbdev, kbdev); ++ kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_STOP_EVT); ++} ++ ++static void on_gpu_granted(struct device *dev) ++{ ++ struct kbase_device *kbdev = dev_get_drvdata(dev); ++ ++ KBASE_TLSTREAM_TL_ARBITER_GRANTED(kbdev, kbdev); ++ kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_GRANTED_EVT); ++} ++ ++static void on_gpu_lost(struct device *dev) ++{ ++ struct kbase_device *kbdev = dev_get_drvdata(dev); ++ ++ kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_LOST_EVT); ++} ++ ++int kbase_arbif_init(struct kbase_device *kbdev) ++{ ++#ifdef CONFIG_OF ++ struct arbiter_if_arb_vm_ops ops; ++ struct arbiter_if_dev *arb_if; ++ struct device_node *arbiter_if_node; ++ struct platform_device *pdev; ++ int err; ++ ++ dev_dbg(kbdev->dev, "%s\n", __func__); ++ ++ arbiter_if_node = of_parse_phandle(kbdev->dev->of_node, ++ "arbiter_if", 0); ++ if (!arbiter_if_node) { ++ dev_dbg(kbdev->dev, "No arbiter_if in Device Tree\n"); ++ /* no arbiter interface defined in device tree */ ++ kbdev->arb.arb_dev = NULL; ++ kbdev->arb.arb_if = NULL; ++ return 0; ++ } ++ ++ pdev = of_find_device_by_node(arbiter_if_node); ++ if (!pdev) { ++ dev_err(kbdev->dev, "Failed to find arbiter_if device\n"); ++ return -EPROBE_DEFER; ++ } ++ ++ if (!pdev->dev.driver || !try_module_get(pdev->dev.driver->owner)) { ++ dev_err(kbdev->dev, "arbiter_if driver not available\n"); ++ return -EPROBE_DEFER; ++ } ++ kbdev->arb.arb_dev = &pdev->dev; ++ arb_if = platform_get_drvdata(pdev); ++ if (!arb_if) { ++ dev_err(kbdev->dev, "arbiter_if driver not ready\n"); ++ module_put(pdev->dev.driver->owner); ++ return -EPROBE_DEFER; ++ } ++ ++ kbdev->arb.arb_if = arb_if; ++ ops.arb_vm_gpu_stop = on_gpu_stop; ++ ops.arb_vm_gpu_granted = on_gpu_granted; ++ ops.arb_vm_gpu_lost = on_gpu_lost; ++ ++ /* register kbase arbiter_if callbacks */ ++ if (arb_if->vm_ops.vm_arb_register_dev) { ++ err = arb_if->vm_ops.vm_arb_register_dev(arb_if, ++ kbdev->dev, &ops); ++ if (err) { ++ dev_err(&pdev->dev, "Failed to register with arbiter\n"); ++ module_put(pdev->dev.driver->owner); ++ return err; ++ } ++ } ++#else /* CONFIG_OF */ ++ dev_dbg(kbdev->dev, "No arbiter without Device Tree support\n"); ++ kbdev->arb.arb_dev = NULL; ++ kbdev->arb.arb_if = NULL; ++#endif ++ return 0; ++} ++ ++void kbase_arbif_destroy(struct kbase_device *kbdev) ++{ ++ struct arbiter_if_dev *arb_if = kbdev->arb.arb_if; ++ ++ if (arb_if && arb_if->vm_ops.vm_arb_unregister_dev) { ++ dev_dbg(kbdev->dev, "%s\n", __func__); ++ arb_if->vm_ops.vm_arb_unregister_dev(kbdev->arb.arb_if); ++ } ++ kbdev->arb.arb_if = NULL; ++ if (kbdev->arb.arb_dev) ++ module_put(kbdev->arb.arb_dev->driver->owner); ++ kbdev->arb.arb_dev = NULL; ++} ++ ++void kbase_arbif_gpu_request(struct kbase_device *kbdev) ++{ ++ struct arbiter_if_dev *arb_if = kbdev->arb.arb_if; ++ ++ if (arb_if && arb_if->vm_ops.vm_arb_gpu_request) { ++ dev_dbg(kbdev->dev, "%s\n", __func__); ++ arb_if->vm_ops.vm_arb_gpu_request(arb_if); ++ } ++} ++ ++void kbase_arbif_gpu_stopped(struct kbase_device *kbdev, u8 gpu_required) ++{ ++ struct arbiter_if_dev *arb_if = kbdev->arb.arb_if; ++ ++ if (arb_if && arb_if->vm_ops.vm_arb_gpu_stopped) { ++ dev_dbg(kbdev->dev, "%s\n", __func__); ++ KBASE_TLSTREAM_TL_ARBITER_STOPPED(kbdev, kbdev); ++ arb_if->vm_ops.vm_arb_gpu_stopped(arb_if, gpu_required); ++ } ++} ++ ++void kbase_arbif_gpu_active(struct kbase_device *kbdev) ++{ ++ struct arbiter_if_dev *arb_if = kbdev->arb.arb_if; ++ ++ if (arb_if && arb_if->vm_ops.vm_arb_gpu_active) { ++ dev_dbg(kbdev->dev, "%s\n", __func__); ++ arb_if->vm_ops.vm_arb_gpu_active(arb_if); ++ } ++} ++ ++void kbase_arbif_gpu_idle(struct kbase_device *kbdev) ++{ ++ struct arbiter_if_dev *arb_if = kbdev->arb.arb_if; ++ ++ if (arb_if && arb_if->vm_ops.vm_arb_gpu_idle) { ++ dev_dbg(kbdev->dev, "vm_arb_gpu_idle\n"); ++ arb_if->vm_ops.vm_arb_gpu_idle(arb_if); ++ } ++} +diff --git a/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbif.h b/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbif.h +new file mode 100755 +index 000000000000..e7e9de76c94c +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbif.h +@@ -0,0 +1,133 @@ ++/* ++ * ++ * (C) COPYRIGHT ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ *//* SPDX-License-Identifier: GPL-2.0 */ ++ ++/* ++ * ++ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU license. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * ++ */ ++ ++/** ++ * @file ++ * Mali arbiter interface APIs to share GPU between Virtual Machines ++ */ ++ ++#ifndef _MALI_KBASE_ARBIF_H_ ++#define _MALI_KBASE_ARBIF_H_ ++ ++/** ++ * enum kbase_arbif_evt - Internal Arbiter event. ++ * ++ * @KBASE_VM_GPU_INITIALIZED_EVT: KBase has finished initializing ++ * and can be stopped ++ * @KBASE_VM_GPU_STOP_EVT: Stop message received from Arbiter ++ * @KBASE_VM_GPU_GRANTED_EVT: Grant message received from Arbiter ++ * @KBASE_VM_GPU_LOST_EVT: Lost message received from Arbiter ++ * @KBASE_VM_GPU_IDLE_EVENT: KBase has transitioned into an inactive state. ++ * @KBASE_VM_REF_EVENT: KBase has transitioned into an active state. ++ * @KBASE_VM_OS_SUSPEND_EVENT: KBase is suspending ++ * @KBASE_VM_OS_RESUME_EVENT: Kbase is resuming ++ */ ++enum kbase_arbif_evt { ++ KBASE_VM_GPU_INITIALIZED_EVT = 1, ++ KBASE_VM_GPU_STOP_EVT, ++ KBASE_VM_GPU_GRANTED_EVT, ++ KBASE_VM_GPU_LOST_EVT, ++ KBASE_VM_GPU_IDLE_EVENT, ++ KBASE_VM_REF_EVENT, ++ KBASE_VM_OS_SUSPEND_EVENT, ++ KBASE_VM_OS_RESUME_EVENT, ++}; ++ ++/** ++ * kbase_arbif_init() - Initialize the arbiter interface functionality. ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * Initialize the arbiter interface and also determines ++ * if Arbiter functionality is required. ++ * ++ * Return: 0 if the Arbiter interface was successfully initialized or the ++ * Arbiter was not required. ++ */ ++int kbase_arbif_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_arbif_destroy() - Cleanups the arbiter interface functionality. ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * Cleans up the arbiter interface functionality and resets the reference count ++ * of the arbif module used ++ */ ++void kbase_arbif_destroy(struct kbase_device *kbdev); ++ ++/** ++ * kbase_arbif_gpu_request() - Send GPU request message to the arbiter ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * Sends a message to Arbiter to request GPU access. ++ */ ++void kbase_arbif_gpu_request(struct kbase_device *kbdev); ++ ++/** ++ * kbase_arbif_gpu_stopped() - Send GPU stopped message to the arbiter ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * @gpu_required: true if GPU access is still required ++ * (Arbiter will automatically send another grant message) ++ * ++ * Sends a message to Arbiter to notify that the GPU has stopped. ++ * @note Once this call has been made, KBase must not attempt to access the GPU ++ * until the #KBASE_VM_GPU_GRANTED_EVT event has been received. ++ */ ++void kbase_arbif_gpu_stopped(struct kbase_device *kbdev, u8 gpu_required); ++ ++/** ++ * kbase_arbif_gpu_active() - Send a GPU active message to the arbiter ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * Sends a message to Arbiter to report that KBase has gone active. ++ */ ++void kbase_arbif_gpu_active(struct kbase_device *kbdev); ++ ++/** ++ * kbase_arbif_gpu_idle() - Send a GPU idle message to the arbiter ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * Sends a message to Arbiter to report that KBase has gone idle. ++ */ ++void kbase_arbif_gpu_idle(struct kbase_device *kbdev); ++ ++#endif /* _MALI_KBASE_ARBIF_H_ */ +diff --git a/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_defs.h b/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_defs.h +new file mode 100755 +index 000000000000..1f53cbf1a286 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_defs.h +@@ -0,0 +1,95 @@ ++/* ++ * ++ * (C) COPYRIGHT ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ *//* SPDX-License-Identifier: GPL-2.0 */ ++ ++/* ++ * ++ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU license. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * ++ */ ++ ++/** ++ * @file ++ * Mali structures define to support arbitration feature ++ */ ++ ++#ifndef _MALI_KBASE_ARBITER_DEFS_H_ ++#define _MALI_KBASE_ARBITER_DEFS_H_ ++ ++#include "mali_kbase_arbiter_pm.h" ++ ++/** ++ * struct kbase_arbiter_vm_state - Struct representing the state and containing the ++ * data of pm work ++ * @kbdev: Pointer to kbase device structure (must be a valid pointer) ++ * @vm_state_lock: The lock protecting the VM state when arbiter is used. ++ * This lock must also be held whenever the VM state is being ++ * transitioned ++ * @vm_state_wait: Wait queue set when GPU is granted ++ * @vm_state: Current state of VM ++ * @vm_arb_wq: Work queue for resuming or stopping work on the GPU for use ++ * with the Arbiter ++ * @vm_suspend_work: Work item for vm_arb_wq to stop current work on GPU ++ * @vm_resume_work: Work item for vm_arb_wq to resume current work on GPU ++ * @vm_arb_starting: Work queue resume in progress ++ * @vm_arb_stopping: Work queue suspend in progress ++ * @vm_arb_users_waiting: Count of users waiting for GPU ++ */ ++struct kbase_arbiter_vm_state { ++ struct kbase_device *kbdev; ++ struct mutex vm_state_lock; ++ wait_queue_head_t vm_state_wait; ++ enum kbase_vm_state vm_state; ++ struct workqueue_struct *vm_arb_wq; ++ struct work_struct vm_suspend_work; ++ struct work_struct vm_resume_work; ++ bool vm_arb_starting; ++ bool vm_arb_stopping; ++ int vm_arb_users_waiting; ++}; ++ ++/** ++ * struct kbase_arbiter_device - Representing an instance of arbiter device, ++ * allocated from the probe method of Mali driver ++ * @arb_if: Pointer to the arbiter interface device ++ * @arb_dev: Pointer to the arbiter device ++ */ ++struct kbase_arbiter_device { ++ struct arbiter_if_dev *arb_if; ++ struct device *arb_dev; ++}; ++ ++#endif /* _MALI_KBASE_ARBITER_DEFS_H_ */ +diff --git a/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_interface.h b/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_interface.h +new file mode 100755 +index 000000000000..5d5d8a7d2cff +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_interface.h +@@ -0,0 +1,181 @@ ++/* ++ * ++ * (C) COPYRIGHT ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ *//* SPDX-License-Identifier: GPL-2.0 */ ++ ++/* ++ * ++ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU license. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * ++ */ ++ ++/** ++ * @file ++ * Defines the Mali arbiter interface ++ */ ++ ++#ifndef _MALI_KBASE_ARBITER_INTERFACE_H_ ++#define _MALI_KBASE_ARBITER_INTERFACE_H_ ++ ++/** ++ * @brief Mali arbiter interface version ++ * ++ * This specifies the current version of the configuration interface. Whenever ++ * the arbiter interface changes, so that integration effort is required, the ++ * version number will be increased. Each configuration must make an effort ++ * to check that it implements the correct version. ++ * ++ * Version history: ++ * 1 - Added the Mali arbiter configuration interface. ++ * 2 - Strip out reference code from header ++ * 3 - Removed DVFS utilization interface (DVFS moved to arbiter side) ++ */ ++#define MALI_KBASE_ARBITER_INTERFACE_VERSION 3 ++ ++struct arbiter_if_dev; ++ ++/** ++ * struct arbiter_if_arb_vm_ops - Interface to communicate messages to VM ++ * ++ * This struct contains callbacks used to deliver messages ++ * from the arbiter to the corresponding VM. ++ * ++ * Note that calls into these callbacks may have synchronous calls back into ++ * the arbiter arbiter_if_vm_arb_ops callbacks below. ++ * For example vm_arb_gpu_stopped() may be called as a side effect of ++ * arb_vm_gpu_stop() being called here. ++ */ ++struct arbiter_if_arb_vm_ops { ++ /** ++ * arb_vm_gpu_stop() - Ask VM to stop using GPU ++ * @dev: The arbif kernel module device. ++ * ++ * Informs KBase to stop using the GPU as soon as possible. ++ * @Note: Once the driver is no longer using the GPU, a call to ++ * vm_arb_gpu_stopped is expected by the arbiter. ++ */ ++ void (*arb_vm_gpu_stop)(struct device *dev); ++ ++ /** ++ * arb_vm_gpu_granted() - GPU has been granted to VM ++ * @dev: The arbif kernel module device. ++ * ++ * Informs KBase that the GPU can now be used by the VM. ++ */ ++ void (*arb_vm_gpu_granted)(struct device *dev); ++ ++ /** ++ * arb_vm_gpu_lost() - VM has lost the GPU ++ * @dev: The arbif kernel module device. ++ * ++ * This is called if KBase takes too long to respond to the arbiter ++ * stop request. ++ * Once this is called, KBase will assume that access to the GPU ++ * has been lost and will fail all running jobs and reset its ++ * internal state. ++ * If successful, will respond with a vm_arb_gpu_stopped message. ++ */ ++ void (*arb_vm_gpu_lost)(struct device *dev); ++}; ++ ++/** ++ * struct arbiter_if_vm_arb_ops - Interface to communicate messages to arbiter ++ * ++ * This struct contains callbacks used to request operations ++ * from the VM to the arbiter ++ * ++ * Note that we must not make any synchronous calls back in to the VM ++ * (via arbiter_if_arb_vm_ops above) in the context of these callbacks. ++ */ ++struct arbiter_if_vm_arb_ops { ++ /** ++ * vm_arb_register_dev() - Register VM device driver callbacks. ++ * @arbif_dev: The arbiter interface we are registering device callbacks ++ * @dev: The device structure to supply in the callbacks. ++ * @ops: The callbacks that the device driver supports ++ * (none are optional). ++ */ ++ int (*vm_arb_register_dev)(struct arbiter_if_dev *arbif_dev, ++ struct device *dev, struct arbiter_if_arb_vm_ops *ops); ++ ++ /** ++ * vm_arb_unregister_dev() - Unregister VM device driver callbacks. ++ * @arbif_dev: The arbiter interface we are unregistering from. ++ */ ++ void (*vm_arb_unregister_dev)(struct arbiter_if_dev *arbif_dev); ++ ++ /** ++ * vm_arb_gpu_request() - Ask the arbiter interface for GPU access. ++ * @arbif_dev: The arbiter interface we want to issue the request. ++ */ ++ void (*vm_arb_gpu_request)(struct arbiter_if_dev *arbif_dev); ++ ++ /** ++ * vm_arb_gpu_active() - Inform arbiter that the driver has gone active ++ * @arbif_dev: The arbiter interface device. ++ */ ++ void (*vm_arb_gpu_active)(struct arbiter_if_dev *arbif_dev); ++ ++ /** ++ * vm_arb_gpu_idle() - Inform the arbiter that the driver has gone idle ++ * @arbif_dev: The arbiter interface device. ++ */ ++ void (*vm_arb_gpu_idle)(struct arbiter_if_dev *arbif_dev); ++ ++ /** ++ * vm_arb_gpu_stopped() - Inform the arbiter that the driver has stopped ++ * using the GPU ++ * @arbif_dev: The arbiter interface device. ++ * @gpu_required: The GPU is still needed to do more work. ++ */ ++ void (*vm_arb_gpu_stopped)(struct arbiter_if_dev *arbif_dev, ++ u8 gpu_required); ++}; ++ ++/** ++ * struct arbiter_if_dev - Arbiter Interface ++ * @vm_ops: Callback functions for connecting KBase with ++ * arbiter interface device. ++ * @priv_data: Internal arbif data not used by KBASE. ++ * ++ * Arbiter Interface Kernel Module State used for linking KBase ++ * with an arbiter interface platform device ++ */ ++struct arbiter_if_dev { ++ struct arbiter_if_vm_arb_ops vm_ops; ++ void *priv_data; ++}; ++ ++#endif /* _MALI_KBASE_ARBITER_INTERFACE_H_ */ +diff --git a/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_pm.c b/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_pm.c +new file mode 100755 +index 000000000000..02b5de2436ea +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_pm.c +@@ -0,0 +1,676 @@ ++// SPDX-License-Identifier: GPL-2.0 ++ ++/* ++ * ++ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/** ++ * @file mali_kbase_arbiter_pm.c ++ * Mali arbiter power manager state machine and APIs ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++static void kbase_arbiter_pm_vm_wait_gpu_assignment(struct kbase_device *kbdev); ++static inline bool kbase_arbiter_pm_vm_gpu_assigned_lockheld( ++ struct kbase_device *kbdev); ++ ++static inline const char *kbase_arbiter_pm_vm_state_str( ++ enum kbase_vm_state state) ++{ ++ switch (state) { ++ case KBASE_VM_STATE_INITIALIZING: ++ return "KBASE_VM_STATE_INITIALIZING"; ++ case KBASE_VM_STATE_INITIALIZING_WITH_GPU: ++ return "KBASE_VM_STATE_INITIALIZING_WITH_GPU"; ++ case KBASE_VM_STATE_SUSPENDED: ++ return "KBASE_VM_STATE_SUSPENDED"; ++ case KBASE_VM_STATE_STOPPED: ++ return "KBASE_VM_STATE_STOPPED"; ++ case KBASE_VM_STATE_STOPPED_GPU_REQUESTED: ++ return "KBASE_VM_STATE_STOPPED_GPU_REQUESTED"; ++ case KBASE_VM_STATE_STARTING: ++ return "KBASE_VM_STATE_STARTING"; ++ case KBASE_VM_STATE_IDLE: ++ return "KBASE_VM_STATE_IDLE"; ++ case KBASE_VM_STATE_ACTIVE: ++ return "KBASE_VM_STATE_ACTIVE"; ++ case KBASE_VM_STATE_STOPPING_IDLE: ++ return "KBASE_VM_STATE_STOPPING_IDLE"; ++ case KBASE_VM_STATE_STOPPING_ACTIVE: ++ return "KBASE_VM_STATE_STOPPING_ACTIVE"; ++ case KBASE_VM_STATE_SUSPEND_PENDING: ++ return "KBASE_VM_STATE_SUSPEND_PENDING"; ++ case KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT: ++ return "KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT"; ++ default: ++ KBASE_DEBUG_ASSERT(false); ++ return "[UnknownState]"; ++ } ++} ++ ++static inline const char *kbase_arbiter_pm_vm_event_str( ++ enum kbase_arbif_evt evt) ++{ ++ switch (evt) { ++ case KBASE_VM_GPU_INITIALIZED_EVT: ++ return "KBASE_VM_GPU_INITIALIZED_EVT"; ++ case KBASE_VM_GPU_STOP_EVT: ++ return "KBASE_VM_GPU_STOP_EVT"; ++ case KBASE_VM_GPU_GRANTED_EVT: ++ return "KBASE_VM_GPU_GRANTED_EVT"; ++ case KBASE_VM_GPU_LOST_EVT: ++ return "KBASE_VM_GPU_LOST_EVT"; ++ case KBASE_VM_OS_SUSPEND_EVENT: ++ return "KBASE_VM_OS_SUSPEND_EVENT"; ++ case KBASE_VM_OS_RESUME_EVENT: ++ return "KBASE_VM_OS_RESUME_EVENT"; ++ case KBASE_VM_GPU_IDLE_EVENT: ++ return "KBASE_VM_GPU_IDLE_EVENT"; ++ case KBASE_VM_REF_EVENT: ++ return "KBASE_VM_REF_EVENT"; ++ default: ++ KBASE_DEBUG_ASSERT(false); ++ return "[UnknownEvent]"; ++ } ++} ++ ++static void kbase_arbiter_pm_vm_set_state(struct kbase_device *kbdev, ++ enum kbase_vm_state new_state) ++{ ++ struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; ++ ++ dev_dbg(kbdev->dev, "VM set_state %s -> %s", ++ kbase_arbiter_pm_vm_state_str(arb_vm_state->vm_state), ++ kbase_arbiter_pm_vm_state_str(new_state)); ++ lockdep_assert_held(&arb_vm_state->vm_state_lock); ++ arb_vm_state->vm_state = new_state; ++ wake_up(&arb_vm_state->vm_state_wait); ++} ++ ++static void kbase_arbiter_pm_suspend_wq(struct work_struct *data) ++{ ++ struct kbase_arbiter_vm_state *arb_vm_state = container_of(data, ++ struct kbase_arbiter_vm_state, ++ vm_suspend_work); ++ struct kbase_device *kbdev = arb_vm_state->kbdev; ++ ++ mutex_lock(&arb_vm_state->vm_state_lock); ++ dev_dbg(kbdev->dev, ">%s\n", __func__); ++ if (arb_vm_state->vm_state == KBASE_VM_STATE_STOPPING_IDLE || ++ arb_vm_state->vm_state == ++ KBASE_VM_STATE_STOPPING_ACTIVE || ++ arb_vm_state->vm_state == ++ KBASE_VM_STATE_SUSPEND_PENDING) { ++ mutex_unlock(&arb_vm_state->vm_state_lock); ++ dev_dbg(kbdev->dev, ">kbase_pm_driver_suspend\n"); ++ kbase_pm_driver_suspend(kbdev); ++ dev_dbg(kbdev->dev, "vm_state_lock); ++ } ++ mutex_unlock(&arb_vm_state->vm_state_lock); ++ dev_dbg(kbdev->dev, "<%s\n", __func__); ++} ++ ++static void kbase_arbiter_pm_resume_wq(struct work_struct *data) ++{ ++ struct kbase_arbiter_vm_state *arb_vm_state = container_of(data, ++ struct kbase_arbiter_vm_state, ++ vm_resume_work); ++ struct kbase_device *kbdev = arb_vm_state->kbdev; ++ ++ mutex_lock(&arb_vm_state->vm_state_lock); ++ dev_dbg(kbdev->dev, ">%s\n", __func__); ++ arb_vm_state->vm_arb_starting = true; ++ if (arb_vm_state->vm_state == KBASE_VM_STATE_STARTING) { ++ mutex_unlock(&arb_vm_state->vm_state_lock); ++ dev_dbg(kbdev->dev, ">kbase_pm_driver_resume\n"); ++ kbase_pm_driver_resume(kbdev, true); ++ dev_dbg(kbdev->dev, "vm_state_lock); ++ } else if (arb_vm_state->vm_state == KBASE_VM_STATE_STOPPING_ACTIVE) { ++ kbase_arbiter_pm_vm_stopped(kbdev); ++ } ++ arb_vm_state->vm_arb_starting = false; ++ mutex_unlock(&arb_vm_state->vm_state_lock); ++ dev_dbg(kbdev->dev, "<%s\n", __func__); ++} ++ ++int kbase_arbiter_pm_early_init(struct kbase_device *kbdev) ++{ ++ int err; ++ struct kbase_arbiter_vm_state *arb_vm_state = NULL; ++ ++ arb_vm_state = kmalloc(sizeof(struct kbase_arbiter_vm_state), ++ GFP_KERNEL); ++ if (arb_vm_state == NULL) ++ return -ENOMEM; ++ ++ arb_vm_state->kbdev = kbdev; ++ arb_vm_state->vm_state = KBASE_VM_STATE_INITIALIZING; ++ ++ mutex_init(&arb_vm_state->vm_state_lock); ++ init_waitqueue_head(&arb_vm_state->vm_state_wait); ++ arb_vm_state->vm_arb_wq = alloc_ordered_workqueue("kbase_vm_arb_wq", ++ WQ_HIGHPRI); ++ if (!arb_vm_state->vm_arb_wq) { ++ dev_err(kbdev->dev, "Failed to allocate vm_arb workqueue\n"); ++ return -ENOMEM; ++ } ++ INIT_WORK(&arb_vm_state->vm_suspend_work, kbase_arbiter_pm_suspend_wq); ++ INIT_WORK(&arb_vm_state->vm_resume_work, kbase_arbiter_pm_resume_wq); ++ arb_vm_state->vm_arb_starting = false; ++ arb_vm_state->vm_arb_users_waiting = 0; ++ kbdev->pm.arb_vm_state = arb_vm_state; ++ ++ err = kbase_arbif_init(kbdev); ++ if (err) { ++ dev_err(kbdev->dev, "Failed to initialise arbif module\n"); ++ goto arbif_init_fail; ++ } ++ if (kbdev->arb.arb_if) { ++ kbase_arbif_gpu_request(kbdev); ++ dev_dbg(kbdev->dev, "Waiting for initial GPU assignment...\n"); ++ wait_event(arb_vm_state->vm_state_wait, ++ arb_vm_state->vm_state == ++ KBASE_VM_STATE_INITIALIZING_WITH_GPU); ++ dev_dbg(kbdev->dev, ++ "Waiting for initial GPU assignment - done\n"); ++ } ++ return 0; ++ ++arbif_init_fail: ++ destroy_workqueue(arb_vm_state->vm_arb_wq); ++ kfree(arb_vm_state); ++ kbdev->pm.arb_vm_state = NULL; ++ return err; ++} ++ ++void kbase_arbiter_pm_early_term(struct kbase_device *kbdev) ++{ ++ struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; ++ ++ mutex_lock(&arb_vm_state->vm_state_lock); ++ if (arb_vm_state->vm_state > KBASE_VM_STATE_STOPPED_GPU_REQUESTED) { ++ kbase_pm_set_gpu_lost(kbdev, false); ++ kbase_arbif_gpu_stopped(kbdev, false); ++ } ++ mutex_unlock(&arb_vm_state->vm_state_lock); ++ kbase_arbif_destroy(kbdev); ++ destroy_workqueue(arb_vm_state->vm_arb_wq); ++ arb_vm_state->vm_arb_wq = NULL; ++ kfree(kbdev->pm.arb_vm_state); ++ kbdev->pm.arb_vm_state = NULL; ++} ++ ++void kbase_arbiter_pm_release_interrupts(struct kbase_device *kbdev) ++{ ++ struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; ++ ++ mutex_lock(&arb_vm_state->vm_state_lock); ++ if (!kbdev->arb.arb_if || ++ arb_vm_state->vm_state > ++ KBASE_VM_STATE_STOPPED_GPU_REQUESTED) ++ kbase_release_interrupts(kbdev); ++ ++ mutex_unlock(&arb_vm_state->vm_state_lock); ++} ++ ++void kbase_arbiter_pm_vm_stopped(struct kbase_device *kbdev) ++{ ++ bool request_gpu = false; ++ struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; ++ ++ lockdep_assert_held(&arb_vm_state->vm_state_lock); ++ ++ if (arb_vm_state->vm_arb_users_waiting > 0 && ++ arb_vm_state->vm_state == KBASE_VM_STATE_STOPPING_IDLE) ++ kbase_arbiter_pm_vm_set_state(kbdev, ++ KBASE_VM_STATE_STOPPING_ACTIVE); ++ ++ dev_dbg(kbdev->dev, "%s %s\n", __func__, ++ kbase_arbiter_pm_vm_state_str(arb_vm_state->vm_state)); ++ kbase_release_interrupts(kbdev); ++ switch (arb_vm_state->vm_state) { ++ case KBASE_VM_STATE_STOPPING_ACTIVE: ++ request_gpu = true; ++ kbase_arbiter_pm_vm_set_state(kbdev, ++ KBASE_VM_STATE_STOPPED_GPU_REQUESTED); ++ break; ++ case KBASE_VM_STATE_STOPPING_IDLE: ++ kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_STOPPED); ++ break; ++ case KBASE_VM_STATE_SUSPEND_PENDING: ++ kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_SUSPENDED); ++ break; ++ default: ++ dev_warn(kbdev->dev, "unexpected pm_stop VM state %u", ++ arb_vm_state->vm_state); ++ break; ++ } ++ ++ kbase_pm_set_gpu_lost(kbdev, false); ++ kbase_arbif_gpu_stopped(kbdev, request_gpu); ++} ++ ++static void kbase_arbiter_pm_vm_gpu_start(struct kbase_device *kbdev) ++{ ++ struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; ++ ++ lockdep_assert_held(&arb_vm_state->vm_state_lock); ++ switch (arb_vm_state->vm_state) { ++ case KBASE_VM_STATE_INITIALIZING: ++ kbase_arbiter_pm_vm_set_state(kbdev, ++ KBASE_VM_STATE_INITIALIZING_WITH_GPU); ++ break; ++ case KBASE_VM_STATE_STOPPED_GPU_REQUESTED: ++ kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_STARTING); ++ kbase_install_interrupts(kbdev); ++ queue_work(arb_vm_state->vm_arb_wq, ++ &arb_vm_state->vm_resume_work); ++ break; ++ case KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT: ++ kbase_pm_set_gpu_lost(kbdev, false); ++ kbase_arbif_gpu_stopped(kbdev, false); ++ kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_SUSPENDED); ++ break; ++ default: ++ dev_warn(kbdev->dev, ++ "GPU_GRANTED when not expected - state %s\n", ++ kbase_arbiter_pm_vm_state_str(arb_vm_state->vm_state)); ++ break; ++ } ++} ++ ++static void kbase_arbiter_pm_vm_gpu_stop(struct kbase_device *kbdev) ++{ ++ struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; ++ ++ lockdep_assert_held(&arb_vm_state->vm_state_lock); ++ if (arb_vm_state->vm_state == KBASE_VM_STATE_INITIALIZING_WITH_GPU) { ++ mutex_unlock(&arb_vm_state->vm_state_lock); ++ kbase_arbiter_pm_vm_wait_gpu_assignment(kbdev); ++ mutex_lock(&arb_vm_state->vm_state_lock); ++ } ++ ++ switch (arb_vm_state->vm_state) { ++ case KBASE_VM_STATE_IDLE: ++ kbase_arbiter_pm_vm_set_state(kbdev, ++ KBASE_VM_STATE_STOPPING_IDLE); ++ queue_work(arb_vm_state->vm_arb_wq, ++ &arb_vm_state->vm_suspend_work); ++ break; ++ case KBASE_VM_STATE_ACTIVE: ++ kbase_arbiter_pm_vm_set_state(kbdev, ++ KBASE_VM_STATE_STOPPING_ACTIVE); ++ queue_work(arb_vm_state->vm_arb_wq, ++ &arb_vm_state->vm_suspend_work); ++ break; ++ case KBASE_VM_STATE_STARTING: ++ dev_dbg(kbdev->dev, "Got GPU_STOP event while STARTING."); ++ kbase_arbiter_pm_vm_set_state(kbdev, ++ KBASE_VM_STATE_STOPPING_ACTIVE); ++ if (arb_vm_state->vm_arb_starting) ++ queue_work(arb_vm_state->vm_arb_wq, ++ &arb_vm_state->vm_suspend_work); ++ break; ++ case KBASE_VM_STATE_SUSPEND_PENDING: ++ /* Suspend finishes with a stop so nothing else to do */ ++ break; ++ default: ++ dev_warn(kbdev->dev, "GPU_STOP when not expected - state %s\n", ++ kbase_arbiter_pm_vm_state_str(arb_vm_state->vm_state)); ++ break; ++ } ++} ++ ++static void kbase_gpu_lost(struct kbase_device *kbdev) ++{ ++ struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; ++ bool handle_gpu_lost = false; ++ ++ lockdep_assert_held(&arb_vm_state->vm_state_lock); ++ ++ switch (arb_vm_state->vm_state) { ++ case KBASE_VM_STATE_STARTING: ++ case KBASE_VM_STATE_ACTIVE: ++ case KBASE_VM_STATE_IDLE: ++ dev_warn(kbdev->dev, "GPU lost in state %s", ++ kbase_arbiter_pm_vm_state_str(arb_vm_state->vm_state)); ++ kbase_arbiter_pm_vm_gpu_stop(kbdev); ++ handle_gpu_lost = true; ++ break; ++ case KBASE_VM_STATE_STOPPING_IDLE: ++ case KBASE_VM_STATE_STOPPING_ACTIVE: ++ case KBASE_VM_STATE_SUSPEND_PENDING: ++ dev_dbg(kbdev->dev, "GPU lost while stopping"); ++ handle_gpu_lost = true; ++ break; ++ case KBASE_VM_STATE_SUSPENDED: ++ case KBASE_VM_STATE_STOPPED: ++ case KBASE_VM_STATE_STOPPED_GPU_REQUESTED: ++ dev_dbg(kbdev->dev, "GPU lost while already stopped"); ++ break; ++ case KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT: ++ dev_dbg(kbdev->dev, "GPU lost while waiting to suspend"); ++ kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_SUSPENDED); ++ break; ++ default: ++ break; ++ } ++ if (handle_gpu_lost) { ++ /* Releasing the VM state lock here is safe because ++ * we are guaranteed to be in either STOPPING_IDLE, ++ * STOPPING_ACTIVE or SUSPEND_PENDING at this point. ++ * The only transitions that are valid from here are to ++ * STOPPED, STOPPED_GPU_REQUESTED or SUSPENDED which can ++ * only happen at the completion of the GPU lost handling. ++ */ ++ mutex_unlock(&arb_vm_state->vm_state_lock); ++ kbase_pm_handle_gpu_lost(kbdev); ++ mutex_lock(&arb_vm_state->vm_state_lock); ++ } ++} ++ ++static inline bool kbase_arbiter_pm_vm_os_suspend_ready_state( ++ struct kbase_device *kbdev) ++{ ++ switch (kbdev->pm.arb_vm_state->vm_state) { ++ case KBASE_VM_STATE_SUSPENDED: ++ case KBASE_VM_STATE_STOPPED: ++ case KBASE_VM_STATE_IDLE: ++ case KBASE_VM_STATE_ACTIVE: ++ return true; ++ default: ++ return false; ++ } ++} ++ ++static void kbase_arbiter_pm_vm_os_prepare_suspend(struct kbase_device *kbdev) ++{ ++ struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; ++ enum kbase_vm_state prev_state; ++ ++ lockdep_assert_held(&arb_vm_state->vm_state_lock); ++ if (kbdev->arb.arb_if) { ++ if (kbdev->pm.arb_vm_state->vm_state == ++ KBASE_VM_STATE_SUSPENDED) ++ return; ++ } ++ /* Block suspend OS function until we are in a stable state ++ * with vm_state_lock ++ */ ++ while (!kbase_arbiter_pm_vm_os_suspend_ready_state(kbdev)) { ++ prev_state = arb_vm_state->vm_state; ++ switch (arb_vm_state->vm_state) { ++ case KBASE_VM_STATE_STOPPING_ACTIVE: ++ case KBASE_VM_STATE_STOPPING_IDLE: ++ kbase_arbiter_pm_vm_set_state(kbdev, ++ KBASE_VM_STATE_SUSPEND_PENDING); ++ break; ++ case KBASE_VM_STATE_STOPPED_GPU_REQUESTED: ++ kbase_arbiter_pm_vm_set_state(kbdev, ++ KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT); ++ break; ++ case KBASE_VM_STATE_STARTING: ++ if (!arb_vm_state->vm_arb_starting) { ++ kbase_arbiter_pm_vm_set_state(kbdev, ++ KBASE_VM_STATE_SUSPEND_PENDING); ++ kbase_arbiter_pm_vm_stopped(kbdev); ++ } ++ break; ++ default: ++ break; ++ } ++ mutex_unlock(&arb_vm_state->vm_state_lock); ++ wait_event(arb_vm_state->vm_state_wait, ++ arb_vm_state->vm_state != prev_state); ++ mutex_lock(&arb_vm_state->vm_state_lock); ++ } ++ ++ switch (arb_vm_state->vm_state) { ++ case KBASE_VM_STATE_STOPPED: ++ kbase_arbiter_pm_vm_set_state(kbdev, ++ KBASE_VM_STATE_SUSPENDED); ++ break; ++ case KBASE_VM_STATE_IDLE: ++ case KBASE_VM_STATE_ACTIVE: ++ kbase_arbiter_pm_vm_set_state(kbdev, ++ KBASE_VM_STATE_SUSPEND_PENDING); ++ mutex_unlock(&arb_vm_state->vm_state_lock); ++ /* Ensure resume has completed fully before starting suspend */ ++ flush_work(&arb_vm_state->vm_resume_work); ++ kbase_pm_driver_suspend(kbdev); ++ mutex_lock(&arb_vm_state->vm_state_lock); ++ break; ++ case KBASE_VM_STATE_SUSPENDED: ++ break; ++ default: ++ KBASE_DEBUG_ASSERT_MSG(false, "Unexpected state to suspend"); ++ break; ++ } ++} ++ ++static void kbase_arbiter_pm_vm_os_resume(struct kbase_device *kbdev) ++{ ++ struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; ++ ++ lockdep_assert_held(&arb_vm_state->vm_state_lock); ++ KBASE_DEBUG_ASSERT_MSG(arb_vm_state->vm_state == ++ KBASE_VM_STATE_SUSPENDED, ++ "Unexpected state to resume"); ++ ++ kbase_arbiter_pm_vm_set_state(kbdev, ++ KBASE_VM_STATE_STOPPED_GPU_REQUESTED); ++ kbase_arbif_gpu_request(kbdev); ++ ++ /* Release lock and block resume OS function until we have ++ * asynchronously received the GRANT message from the Arbiter and ++ * fully resumed ++ */ ++ mutex_unlock(&arb_vm_state->vm_state_lock); ++ kbase_arbiter_pm_vm_wait_gpu_assignment(kbdev); ++ flush_work(&arb_vm_state->vm_resume_work); ++ mutex_lock(&arb_vm_state->vm_state_lock); ++} ++ ++void kbase_arbiter_pm_vm_event(struct kbase_device *kbdev, ++ enum kbase_arbif_evt evt) ++{ ++ struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; ++ ++ if (!kbdev->arb.arb_if) ++ return; ++ ++ mutex_lock(&arb_vm_state->vm_state_lock); ++ dev_dbg(kbdev->dev, "%s %s\n", __func__, ++ kbase_arbiter_pm_vm_event_str(evt)); ++ ++ switch (evt) { ++ case KBASE_VM_GPU_GRANTED_EVT: ++ kbase_arbiter_pm_vm_gpu_start(kbdev); ++ break; ++ case KBASE_VM_GPU_STOP_EVT: ++ kbase_arbiter_pm_vm_gpu_stop(kbdev); ++ break; ++ case KBASE_VM_GPU_LOST_EVT: ++ dev_dbg(kbdev->dev, "KBASE_ARBIF_GPU_LOST_EVT!"); ++ kbase_gpu_lost(kbdev); ++ break; ++ case KBASE_VM_OS_SUSPEND_EVENT: ++ kbase_arbiter_pm_vm_os_prepare_suspend(kbdev); ++ break; ++ case KBASE_VM_OS_RESUME_EVENT: ++ kbase_arbiter_pm_vm_os_resume(kbdev); ++ break; ++ case KBASE_VM_GPU_IDLE_EVENT: ++ switch (arb_vm_state->vm_state) { ++ case KBASE_VM_STATE_ACTIVE: ++ kbase_arbiter_pm_vm_set_state(kbdev, ++ KBASE_VM_STATE_IDLE); ++ kbase_arbif_gpu_idle(kbdev); ++ break; ++ default: ++ break; ++ } ++ break; ++ ++ case KBASE_VM_REF_EVENT: ++ switch (arb_vm_state->vm_state) { ++ case KBASE_VM_STATE_STARTING: ++ KBASE_TLSTREAM_TL_ARBITER_STARTED(kbdev, kbdev); ++ /* FALL THROUGH */ ++ case KBASE_VM_STATE_IDLE: ++ kbase_arbiter_pm_vm_set_state(kbdev, ++ KBASE_VM_STATE_ACTIVE); ++ kbase_arbif_gpu_active(kbdev); ++ break; ++ case KBASE_VM_STATE_STOPPING_IDLE: ++ kbase_arbiter_pm_vm_set_state(kbdev, ++ KBASE_VM_STATE_STOPPING_ACTIVE); ++ break; ++ default: ++ break; ++ } ++ break; ++ ++ case KBASE_VM_GPU_INITIALIZED_EVT: ++ switch (arb_vm_state->vm_state) { ++ case KBASE_VM_STATE_INITIALIZING_WITH_GPU: ++ lockdep_assert_held(&kbdev->pm.lock); ++ if (kbdev->pm.active_count > 0) { ++ kbase_arbiter_pm_vm_set_state(kbdev, ++ KBASE_VM_STATE_ACTIVE); ++ kbase_arbif_gpu_active(kbdev); ++ } else { ++ kbase_arbiter_pm_vm_set_state(kbdev, ++ KBASE_VM_STATE_IDLE); ++ kbase_arbif_gpu_idle(kbdev); ++ } ++ break; ++ default: ++ break; ++ } ++ break; ++ ++ default: ++ dev_alert(kbdev->dev, "Got Unknown Event!"); ++ break; ++ } ++ mutex_unlock(&arb_vm_state->vm_state_lock); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_arbiter_pm_vm_event); ++ ++static void kbase_arbiter_pm_vm_wait_gpu_assignment(struct kbase_device *kbdev) ++{ ++ struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; ++ ++ dev_dbg(kbdev->dev, "Waiting for GPU assignment...\n"); ++ wait_event(arb_vm_state->vm_state_wait, ++ arb_vm_state->vm_state == KBASE_VM_STATE_IDLE || ++ arb_vm_state->vm_state == KBASE_VM_STATE_ACTIVE); ++ dev_dbg(kbdev->dev, "Waiting for GPU assignment - done\n"); ++} ++ ++static inline bool kbase_arbiter_pm_vm_gpu_assigned_lockheld( ++ struct kbase_device *kbdev) ++{ ++ struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; ++ ++ lockdep_assert_held(&arb_vm_state->vm_state_lock); ++ return (arb_vm_state->vm_state == KBASE_VM_STATE_IDLE || ++ arb_vm_state->vm_state == KBASE_VM_STATE_ACTIVE); ++} ++ ++int kbase_arbiter_pm_ctx_active_handle_suspend(struct kbase_device *kbdev, ++ enum kbase_pm_suspend_handler suspend_handler) ++{ ++ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; ++ struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; ++ int res = 0; ++ ++ if (kbdev->arb.arb_if) { ++ mutex_lock(&arb_vm_state->vm_state_lock); ++ while (!kbase_arbiter_pm_vm_gpu_assigned_lockheld(kbdev)) { ++ /* Update VM state since we have GPU work to do */ ++ if (arb_vm_state->vm_state == ++ KBASE_VM_STATE_STOPPING_IDLE) ++ kbase_arbiter_pm_vm_set_state(kbdev, ++ KBASE_VM_STATE_STOPPING_ACTIVE); ++ else if (arb_vm_state->vm_state == ++ KBASE_VM_STATE_STOPPED) { ++ kbase_arbiter_pm_vm_set_state(kbdev, ++ KBASE_VM_STATE_STOPPED_GPU_REQUESTED); ++ kbase_arbif_gpu_request(kbdev); ++ } else if (arb_vm_state->vm_state == ++ KBASE_VM_STATE_INITIALIZING_WITH_GPU) ++ break; ++ ++ if (suspend_handler != ++ KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE) { ++ ++ /* In case of GPU lost, even if ++ * active_count > 0, we no longer have GPU ++ * access ++ */ ++ if (kbase_pm_is_gpu_lost(kbdev)) ++ res = 1; ++ ++ switch (suspend_handler) { ++ case KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE: ++ res = 1; ++ break; ++ case KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE: ++ if (kbdev->pm.active_count == 0) ++ res = 1; ++ break; ++ case KBASE_PM_SUSPEND_HANDLER_VM_GPU_GRANTED: ++ break; ++ default: ++ WARN(1, "Unknown suspend_handler\n"); ++ res = 1; ++ break; ++ } ++ break; ++ } ++ ++ /* Need to synchronously wait for GPU assignment */ ++ arb_vm_state->vm_arb_users_waiting++; ++ mutex_unlock(&arb_vm_state->vm_state_lock); ++ mutex_unlock(&kbdev->pm.lock); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ kbase_arbiter_pm_vm_wait_gpu_assignment(kbdev); ++ mutex_lock(&js_devdata->runpool_mutex); ++ mutex_lock(&kbdev->pm.lock); ++ mutex_lock(&arb_vm_state->vm_state_lock); ++ arb_vm_state->vm_arb_users_waiting--; ++ } ++ mutex_unlock(&arb_vm_state->vm_state_lock); ++ } ++ return res; ++} +diff --git a/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_pm.h b/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_pm.h +new file mode 100755 +index 000000000000..3c49eb1948c5 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_pm.h +@@ -0,0 +1,159 @@ ++/* ++ * ++ * (C) COPYRIGHT ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ *//* SPDX-License-Identifier: GPL-2.0 */ ++ ++/* ++ * ++ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU license. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/** ++ * @file ++ * Mali arbiter power manager state machine and APIs ++ */ ++ ++#ifndef _MALI_KBASE_ARBITER_PM_H_ ++#define _MALI_KBASE_ARBITER_PM_H_ ++ ++#include "mali_kbase_arbif.h" ++ ++/** ++ * enum kbase_vm_state - Current PM Arbitration state. ++ * ++ * @KBASE_VM_STATE_INITIALIZING: Special state before arbiter is initialized. ++ * @KBASE_VM_STATE_INITIALIZING_WITH_GPU: Initialization after GPU ++ * has been granted. ++ * @KBASE_VM_STATE_SUSPENDED: KBase is suspended by OS and GPU is not assigned. ++ * @KBASE_VM_STATE_STOPPED: GPU is not assigned to KBase and is not required. ++ * @KBASE_VM_STATE_STOPPED_GPU_REQUESTED: GPU is not assigned to KBase ++ * but a request has been made. ++ * @KBASE_VM_STATE_STARTING: GPU is assigned and KBase is getting ready to run. ++ * @KBASE_VM_STATE_IDLE: GPU is assigned but KBase has no work to do ++ * @KBASE_VM_STATE_ACTIVE: GPU is assigned and KBase is busy using it ++ * @KBASE_VM_STATE_SUSPEND_PENDING: OS is going into suspend mode. ++ * @KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT: OS is going into suspend mode but GPU ++ * has already been requested. ++ * In this situation we must wait for ++ * the Arbiter to send a GRANTED message ++ * and respond immediately with ++ * a STOPPED message before entering ++ * the suspend mode. ++ * @KBASE_VM_STATE_STOPPING_IDLE: Arbiter has sent a stopped message and there ++ * is currently no work to do on the GPU. ++ * @KBASE_VM_STATE_STOPPING_ACTIVE: Arbiter has sent a stopped message when ++ * KBase has work to do. ++ */ ++enum kbase_vm_state { ++ KBASE_VM_STATE_INITIALIZING, ++ KBASE_VM_STATE_INITIALIZING_WITH_GPU, ++ KBASE_VM_STATE_SUSPENDED, ++ KBASE_VM_STATE_STOPPED, ++ KBASE_VM_STATE_STOPPED_GPU_REQUESTED, ++ KBASE_VM_STATE_STARTING, ++ KBASE_VM_STATE_IDLE, ++ KBASE_VM_STATE_ACTIVE, ++ KBASE_VM_STATE_SUSPEND_PENDING, ++ KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT, ++ KBASE_VM_STATE_STOPPING_IDLE, ++ KBASE_VM_STATE_STOPPING_ACTIVE ++}; ++ ++/** ++ * kbase_arbiter_pm_early_init() - Initialize arbiter for VM Paravirtualized use ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * Initialize the arbiter and other required resources during the runtime ++ * and request the GPU for the VM for the first time. ++ * ++ * Return: 0 if successful, otherwise a standard Linux error code ++ */ ++int kbase_arbiter_pm_early_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_arbiter_pm_early_term() - Shutdown arbiter and free resources. ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * Clean up all the resources ++ */ ++void kbase_arbiter_pm_early_term(struct kbase_device *kbdev); ++ ++/** ++ * kbase_arbiter_pm_release_interrupts() - Release the GPU interrupts ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * Releases interrupts if needed (GPU is available) otherwise does nothing ++ */ ++void kbase_arbiter_pm_release_interrupts(struct kbase_device *kbdev); ++ ++/** ++ * kbase_arbiter_pm_vm_event() - Dispatch VM event to the state machine ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * The state machine function. Receives events and transitions states ++ * according the event received and the current state ++ */ ++void kbase_arbiter_pm_vm_event(struct kbase_device *kbdev, ++ enum kbase_arbif_evt event); ++ ++/** ++ * kbase_arbiter_pm_ctx_active_handle_suspend() - Handle suspend operation for ++ * arbitration mode ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * @suspend_handler: The handler code for how to handle a suspend ++ * that might occur ++ * ++ * This function handles a suspend event from the driver, ++ * communicating with the arbiter and waiting synchronously for the GPU ++ * to be granted again depending on the VM state. ++ * ++ * Return: 0 if success, 1 if failure due to system suspending/suspended ++ */ ++int kbase_arbiter_pm_ctx_active_handle_suspend(struct kbase_device *kbdev, ++ enum kbase_pm_suspend_handler suspend_handler); ++ ++ ++/** ++ * kbase_arbiter_pm_vm_stopped() - Handle stop event for the VM ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * This function handles a stop event for the VM. ++ * It will update the VM state and forward the stop event to the driver. ++ */ ++void kbase_arbiter_pm_vm_stopped(struct kbase_device *kbdev); ++ ++#endif /*_MALI_KBASE_ARBITER_PM_H_ */ +diff --git a/drivers/gpu/arm/bifrost/backend/gpu/Kbuild b/drivers/gpu/arm/bifrost/backend/gpu/Kbuild +new file mode 100755 +index 000000000000..b48ab4c51875 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/backend/gpu/Kbuild +@@ -0,0 +1,65 @@ ++# ++# (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# GNU General Public License for more details. ++# ++# You should have received a copy of the GNU General Public License ++# along with this program; if not, you can access it online at ++# http://www.gnu.org/licenses/gpl-2.0.html. ++# ++# SPDX-License-Identifier: GPL-2.0 ++# ++# ++ ++BACKEND += \ ++ backend/gpu/mali_kbase_cache_policy_backend.c \ ++ backend/gpu/mali_kbase_gpuprops_backend.c \ ++ backend/gpu/mali_kbase_irq_linux.c \ ++ backend/gpu/mali_kbase_instr_backend.c \ ++ backend/gpu/mali_kbase_js_backend.c \ ++ backend/gpu/mali_kbase_pm_backend.c \ ++ backend/gpu/mali_kbase_pm_driver.c \ ++ backend/gpu/mali_kbase_pm_metrics.c \ ++ backend/gpu/mali_kbase_pm_ca.c \ ++ backend/gpu/mali_kbase_pm_always_on.c \ ++ backend/gpu/mali_kbase_pm_coarse_demand.c \ ++ backend/gpu/mali_kbase_pm_policy.c \ ++ backend/gpu/mali_kbase_time.c \ ++ backend/gpu/mali_kbase_l2_mmu_config.c \ ++ backend/gpu/mali_kbase_clk_rate_trace_mgr.c ++ ++ifeq ($(MALI_USE_CSF),1) ++# empty ++else ++ BACKEND += \ ++ backend/gpu/mali_kbase_jm_as.c \ ++ backend/gpu/mali_kbase_debug_job_fault_backend.c \ ++ backend/gpu/mali_kbase_jm_hw.c \ ++ backend/gpu/mali_kbase_jm_rb.c ++endif ++ ++ifeq ($(MALI_CUSTOMER_RELEASE),0) ++BACKEND += \ ++ backend/gpu/mali_kbase_pm_always_on_demand.c ++endif ++ ++ifeq ($(CONFIG_MALI_BIFROST_DEVFREQ),y) ++BACKEND += \ ++ backend/gpu/mali_kbase_devfreq.c ++endif ++ ++ifeq ($(CONFIG_MALI_BIFROST_NO_MALI),y) ++ # Dummy model ++ BACKEND += backend/gpu/mali_kbase_model_dummy.c ++ BACKEND += backend/gpu/mali_kbase_model_linux.c ++ # HW error simulation ++ BACKEND += backend/gpu/mali_kbase_model_error_generator.c ++endif +diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_backend_config.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_backend_config.h +new file mode 100755 +index 000000000000..4a61f96c8c7d +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_backend_config.h +@@ -0,0 +1,31 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2018 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* ++ * Backend specific configuration ++ */ ++ ++#ifndef _KBASE_BACKEND_CONFIG_H_ ++#define _KBASE_BACKEND_CONFIG_H_ ++ ++#endif /* _KBASE_BACKEND_CONFIG_H_ */ ++ +diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_cache_policy_backend.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_cache_policy_backend.c +new file mode 100755 +index 000000000000..4e07a3f9d83f +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_cache_policy_backend.c +@@ -0,0 +1,34 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2016, 2018, 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include "backend/gpu/mali_kbase_cache_policy_backend.h" ++#include ++ ++void kbase_cache_set_coherency_mode(struct kbase_device *kbdev, ++ u32 mode) ++{ ++ kbdev->current_gpu_coherency_mode = mode; ++ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_COHERENCY_REG)) ++ kbase_reg_write(kbdev, COHERENCY_ENABLE, mode); ++} ++ +diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_cache_policy_backend.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_cache_policy_backend.h +new file mode 100755 +index 000000000000..f78ada74f605 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_cache_policy_backend.h +@@ -0,0 +1,39 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++#ifndef _KBASE_CACHE_POLICY_BACKEND_H_ ++#define _KBASE_CACHE_POLICY_BACKEND_H_ ++ ++#include "mali_kbase.h" ++#include "mali_base_kernel.h" ++ ++/** ++ * kbase_cache_set_coherency_mode() - Sets the system coherency mode ++ * in the GPU. ++ * @kbdev: Device pointer ++ * @mode: Coherency mode. COHERENCY_ACE/ACE_LITE ++ */ ++void kbase_cache_set_coherency_mode(struct kbase_device *kbdev, ++ u32 mode); ++ ++#endif /* _KBASE_CACHE_POLICY_H_ */ +diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_clk_rate_trace_mgr.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_clk_rate_trace_mgr.c +new file mode 100755 +index 000000000000..187d7d6f6926 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_clk_rate_trace_mgr.c +@@ -0,0 +1,287 @@ ++/* ++ * ++ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* ++ * Implementation of the GPU clock rate trace manager. ++ */ ++ ++#include ++#include ++#include ++#include ++#include "mali_kbase_clk_rate_trace_mgr.h" ++ ++#ifdef CONFIG_TRACE_POWER_GPU_FREQUENCY ++#include ++#else ++#include "mali_power_gpu_frequency_trace.h" ++#endif ++ ++#ifndef CLK_RATE_TRACE_OPS ++#define CLK_RATE_TRACE_OPS (NULL) ++#endif ++ ++static int gpu_clk_rate_change_notifier(struct notifier_block *nb, ++ unsigned long event, void *data) ++{ ++ struct kbase_gpu_clk_notifier_data *ndata = data; ++ struct kbase_clk_data *clk_data = ++ container_of(nb, struct kbase_clk_data, clk_rate_change_nb); ++ struct kbase_clk_rate_trace_manager *clk_rtm = clk_data->clk_rtm; ++ unsigned long flags; ++ ++ if (WARN_ON_ONCE(clk_data->gpu_clk_handle != ndata->gpu_clk_handle)) ++ return NOTIFY_BAD; ++ ++ spin_lock_irqsave(&clk_rtm->lock, flags); ++ if (event == POST_RATE_CHANGE) { ++ if (!clk_rtm->gpu_idle && ++ (clk_data->clock_val != ndata->new_rate)) { ++ kbase_clk_rate_trace_manager_notify_all( ++ clk_rtm, clk_data->index, ndata->new_rate); ++ } ++ ++ clk_data->clock_val = ndata->new_rate; ++ } ++ spin_unlock_irqrestore(&clk_rtm->lock, flags); ++ ++ return NOTIFY_DONE; ++} ++ ++static int gpu_clk_data_init(struct kbase_device *kbdev, ++ void *gpu_clk_handle, unsigned int index) ++{ ++ struct kbase_clk_rate_trace_op_conf *callbacks = ++ (struct kbase_clk_rate_trace_op_conf *)CLK_RATE_TRACE_OPS; ++ struct kbase_clk_data *clk_data; ++ struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm; ++ int ret = 0; ++ ++ if (WARN_ON(!callbacks) || ++ WARN_ON(!gpu_clk_handle) || ++ WARN_ON(index >= BASE_MAX_NR_CLOCKS_REGULATORS)) ++ return -EINVAL; ++ ++ clk_data = kzalloc(sizeof(*clk_data), GFP_KERNEL); ++ if (!clk_data) { ++ dev_err(kbdev->dev, "Failed to allocate data for clock enumerated at index %u", index); ++ return -ENOMEM; ++ } ++ ++ clk_data->index = (u8)index; ++ clk_data->gpu_clk_handle = gpu_clk_handle; ++ /* Store the initial value of clock */ ++ clk_data->clock_val = ++ callbacks->get_gpu_clk_rate(kbdev, gpu_clk_handle); ++ ++ { ++ /* At the initialization time, GPU is powered off. */ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&clk_rtm->lock, flags); ++ kbase_clk_rate_trace_manager_notify_all( ++ clk_rtm, clk_data->index, 0); ++ spin_unlock_irqrestore(&clk_rtm->lock, flags); ++ } ++ ++ clk_data->clk_rtm = clk_rtm; ++ clk_rtm->clks[index] = clk_data; ++ ++ clk_data->clk_rate_change_nb.notifier_call = ++ gpu_clk_rate_change_notifier; ++ ++ ret = callbacks->gpu_clk_notifier_register(kbdev, gpu_clk_handle, ++ &clk_data->clk_rate_change_nb); ++ if (ret) { ++ dev_err(kbdev->dev, "Failed to register notifier for clock enumerated at index %u", index); ++ kfree(clk_data); ++ } ++ ++ return ret; ++} ++ ++int kbase_clk_rate_trace_manager_init(struct kbase_device *kbdev) ++{ ++ struct kbase_clk_rate_trace_op_conf *callbacks = ++ (struct kbase_clk_rate_trace_op_conf *)CLK_RATE_TRACE_OPS; ++ struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm; ++ unsigned int i; ++ int ret = 0; ++ ++ /* Return early if no callbacks provided for clock rate tracing */ ++ if (!callbacks) ++ return 0; ++ ++ spin_lock_init(&clk_rtm->lock); ++ INIT_LIST_HEAD(&clk_rtm->listeners); ++ ++ clk_rtm->gpu_idle = true; ++ ++ for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) { ++ void *gpu_clk_handle = ++ callbacks->enumerate_gpu_clk(kbdev, i); ++ ++ if (!gpu_clk_handle) ++ break; ++ ++ ret = gpu_clk_data_init(kbdev, gpu_clk_handle, i); ++ if (ret) ++ goto error; ++ } ++ ++ /* Activate clock rate trace manager if at least one GPU clock was ++ * enumerated. ++ */ ++ if (i) ++ WRITE_ONCE(clk_rtm->clk_rate_trace_ops, callbacks); ++ else ++ dev_info(kbdev->dev, "No clock(s) available for rate tracing"); ++ ++ return 0; ++ ++error: ++ while (i--) { ++ clk_rtm->clk_rate_trace_ops->gpu_clk_notifier_unregister( ++ kbdev, clk_rtm->clks[i]->gpu_clk_handle, ++ &clk_rtm->clks[i]->clk_rate_change_nb); ++ kfree(clk_rtm->clks[i]); ++ } ++ ++ return ret; ++} ++ ++void kbase_clk_rate_trace_manager_term(struct kbase_device *kbdev) ++{ ++ struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm; ++ unsigned int i; ++ ++ WARN_ON(!list_empty(&clk_rtm->listeners)); ++ ++ if (!clk_rtm->clk_rate_trace_ops) ++ return; ++ ++ for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) { ++ if (!clk_rtm->clks[i]) ++ break; ++ ++ clk_rtm->clk_rate_trace_ops->gpu_clk_notifier_unregister( ++ kbdev, clk_rtm->clks[i]->gpu_clk_handle, ++ &clk_rtm->clks[i]->clk_rate_change_nb); ++ kfree(clk_rtm->clks[i]); ++ } ++ ++ WRITE_ONCE(clk_rtm->clk_rate_trace_ops, NULL); ++} ++ ++void kbase_clk_rate_trace_manager_gpu_active(struct kbase_device *kbdev) ++{ ++ struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm; ++ unsigned int i; ++ unsigned long flags; ++ ++ if (!clk_rtm->clk_rate_trace_ops) ++ return; ++ ++ spin_lock_irqsave(&clk_rtm->lock, flags); ++ ++ for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) { ++ struct kbase_clk_data *clk_data = clk_rtm->clks[i]; ++ ++ if (!clk_data) ++ break; ++ ++ if (unlikely(!clk_data->clock_val)) ++ continue; ++ ++ kbase_clk_rate_trace_manager_notify_all( ++ clk_rtm, clk_data->index, clk_data->clock_val); ++ } ++ ++ clk_rtm->gpu_idle = false; ++ spin_unlock_irqrestore(&clk_rtm->lock, flags); ++} ++ ++void kbase_clk_rate_trace_manager_gpu_idle(struct kbase_device *kbdev) ++{ ++ struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm; ++ unsigned int i; ++ unsigned long flags; ++ ++ if (!clk_rtm->clk_rate_trace_ops) ++ return; ++ ++ spin_lock_irqsave(&clk_rtm->lock, flags); ++ ++ for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) { ++ struct kbase_clk_data *clk_data = clk_rtm->clks[i]; ++ ++ if (!clk_data) ++ break; ++ ++ if (unlikely(!clk_data->clock_val)) ++ continue; ++ ++ kbase_clk_rate_trace_manager_notify_all( ++ clk_rtm, clk_data->index, 0); ++ } ++ ++ clk_rtm->gpu_idle = true; ++ spin_unlock_irqrestore(&clk_rtm->lock, flags); ++} ++ ++void kbase_clk_rate_trace_manager_notify_all( ++ struct kbase_clk_rate_trace_manager *clk_rtm, ++ u32 clk_index, ++ unsigned long new_rate) ++{ ++ struct kbase_clk_rate_listener *pos; ++ struct kbase_device *kbdev; ++ ++ lockdep_assert_held(&clk_rtm->lock); ++ ++ kbdev = container_of(clk_rtm, struct kbase_device, pm.clk_rtm); ++ ++ dev_dbg(kbdev->dev, "GPU clock %u rate changed to %lu", ++ clk_index, new_rate); ++ ++ /* Raise standard `power/gpu_frequency` ftrace event */ ++ { ++ unsigned long new_rate_khz = new_rate; ++ ++#if BITS_PER_LONG == 64 ++ do_div(new_rate_khz, 1000); ++#elif BITS_PER_LONG == 32 ++ new_rate_khz /= 1000; ++#else ++#error "unsigned long division is not supported for this architecture" ++#endif ++ ++ trace_gpu_frequency(new_rate_khz, clk_index); ++ } ++ ++ /* Notify the listeners. */ ++ list_for_each_entry(pos, &clk_rtm->listeners, node) { ++ pos->notify(pos, clk_index, new_rate); ++ } ++} ++KBASE_EXPORT_TEST_API(kbase_clk_rate_trace_manager_notify_all); ++ +diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_clk_rate_trace_mgr.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_clk_rate_trace_mgr.h +new file mode 100755 +index 000000000000..dcafb26ea4c0 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_clk_rate_trace_mgr.h +@@ -0,0 +1,155 @@ ++/* ++ * ++ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_CLK_RATE_TRACE_MGR_ ++#define _KBASE_CLK_RATE_TRACE_MGR_ ++ ++/** The index of top clock domain in kbase_clk_rate_trace_manager:clks. */ ++#define KBASE_CLOCK_DOMAIN_TOP (0) ++ ++/** The index of shader-cores clock domain in ++ * kbase_clk_rate_trace_manager:clks. ++ */ ++#define KBASE_CLOCK_DOMAIN_SHADER_CORES (1) ++ ++/** ++ * struct kbase_clk_data - Data stored per enumerated GPU clock. ++ * ++ * @clk_rtm: Pointer to clock rate trace manager object. ++ * @gpu_clk_handle: Handle unique to the enumerated GPU clock. ++ * @plat_private: Private data for the platform to store into ++ * @clk_rate_change_nb: notifier block containing the pointer to callback ++ * function that is invoked whenever the rate of ++ * enumerated GPU clock changes. ++ * @clock_val: Current rate of the enumerated GPU clock. ++ * @index: Index at which the GPU clock was enumerated. ++ */ ++struct kbase_clk_data { ++ struct kbase_clk_rate_trace_manager *clk_rtm; ++ void *gpu_clk_handle; ++ void *plat_private; ++ struct notifier_block clk_rate_change_nb; ++ unsigned long clock_val; ++ u8 index; ++}; ++ ++/** ++ * kbase_clk_rate_trace_manager_init - Initialize GPU clock rate trace manager. ++ * ++ * @kbdev: Device pointer ++ * ++ * Return: 0 if success, or an error code on failure. ++ */ ++int kbase_clk_rate_trace_manager_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_clk_rate_trace_manager_term - Terminate GPU clock rate trace manager. ++ * ++ * @kbdev: Device pointer ++ */ ++void kbase_clk_rate_trace_manager_term(struct kbase_device *kbdev); ++ ++/** ++ * kbase_clk_rate_trace_manager_gpu_active - Inform GPU clock rate trace ++ * manager of GPU becoming active. ++ * ++ * @kbdev: Device pointer ++ */ ++void kbase_clk_rate_trace_manager_gpu_active(struct kbase_device *kbdev); ++ ++/** ++ * kbase_clk_rate_trace_manager_gpu_idle - Inform GPU clock rate trace ++ * manager of GPU becoming idle. ++ * @kbdev: Device pointer ++ */ ++void kbase_clk_rate_trace_manager_gpu_idle(struct kbase_device *kbdev); ++ ++/** ++ * kbase_clk_rate_trace_manager_subscribe_no_lock() - Add freq change listener. ++ * ++ * @clk_rtm: Clock rate manager instance. ++ * @listener: Listener handle ++ * ++ * kbase_clk_rate_trace_manager:lock must be held by the caller. ++ */ ++static inline void kbase_clk_rate_trace_manager_subscribe_no_lock( ++ struct kbase_clk_rate_trace_manager *clk_rtm, ++ struct kbase_clk_rate_listener *listener) ++{ ++ lockdep_assert_held(&clk_rtm->lock); ++ list_add(&listener->node, &clk_rtm->listeners); ++} ++ ++/** ++ * kbase_clk_rate_trace_manager_subscribe() - Add freq change listener. ++ * ++ * @clk_rtm: Clock rate manager instance. ++ * @listener: Listener handle ++ */ ++static inline void kbase_clk_rate_trace_manager_subscribe( ++ struct kbase_clk_rate_trace_manager *clk_rtm, ++ struct kbase_clk_rate_listener *listener) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&clk_rtm->lock, flags); ++ kbase_clk_rate_trace_manager_subscribe_no_lock( ++ clk_rtm, listener); ++ spin_unlock_irqrestore(&clk_rtm->lock, flags); ++} ++ ++/** ++ * kbase_clk_rate_trace_manager_unsubscribe() - Remove freq change listener. ++ * ++ * @clk_rtm: Clock rate manager instance. ++ * @listener: Listener handle ++ */ ++static inline void kbase_clk_rate_trace_manager_unsubscribe( ++ struct kbase_clk_rate_trace_manager *clk_rtm, ++ struct kbase_clk_rate_listener *listener) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&clk_rtm->lock, flags); ++ list_del(&listener->node); ++ spin_unlock_irqrestore(&clk_rtm->lock, flags); ++} ++ ++/** ++ * kbase_clk_rate_trace_manager_notify_all() - Notify all clock \ ++ * rate listeners. ++ * ++ * @clk_rtm: Clock rate manager instance. ++ * @clk_index: Clock index. ++ * @new_rate: New clock frequency(Hz) ++ * ++ * kbase_clk_rate_trace_manager:lock must be locked. ++ * This function is exported to be used by clock rate trace test ++ * portal. ++ */ ++void kbase_clk_rate_trace_manager_notify_all( ++ struct kbase_clk_rate_trace_manager *clk_rtm, ++ u32 clock_index, ++ unsigned long new_rate); ++ ++#endif /* _KBASE_CLK_RATE_TRACE_MGR_ */ ++ +diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_debug_job_fault_backend.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_debug_job_fault_backend.c +new file mode 100755 +index 000000000000..3aadcb04160c +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_debug_job_fault_backend.c +@@ -0,0 +1,164 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2015, 2018-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++#include ++#include "mali_kbase_debug_job_fault.h" ++ ++#ifdef CONFIG_DEBUG_FS ++ ++/*GPU_CONTROL_REG(r)*/ ++static int gpu_control_reg_snapshot[] = { ++ GPU_ID, ++ SHADER_READY_LO, ++ SHADER_READY_HI, ++ TILER_READY_LO, ++ TILER_READY_HI, ++ L2_READY_LO, ++ L2_READY_HI ++}; ++ ++/* JOB_CONTROL_REG(r) */ ++static int job_control_reg_snapshot[] = { ++ JOB_IRQ_MASK, ++ JOB_IRQ_STATUS ++}; ++ ++/* JOB_SLOT_REG(n,r) */ ++static int job_slot_reg_snapshot[] = { ++ JS_HEAD_LO, ++ JS_HEAD_HI, ++ JS_TAIL_LO, ++ JS_TAIL_HI, ++ JS_AFFINITY_LO, ++ JS_AFFINITY_HI, ++ JS_CONFIG, ++ JS_STATUS, ++ JS_HEAD_NEXT_LO, ++ JS_HEAD_NEXT_HI, ++ JS_AFFINITY_NEXT_LO, ++ JS_AFFINITY_NEXT_HI, ++ JS_CONFIG_NEXT ++}; ++ ++/*MMU_REG(r)*/ ++static int mmu_reg_snapshot[] = { ++ MMU_IRQ_MASK, ++ MMU_IRQ_STATUS ++}; ++ ++/* MMU_AS_REG(n,r) */ ++static int as_reg_snapshot[] = { ++ AS_TRANSTAB_LO, ++ AS_TRANSTAB_HI, ++ AS_TRANSCFG_LO, ++ AS_TRANSCFG_HI, ++ AS_MEMATTR_LO, ++ AS_MEMATTR_HI, ++ AS_FAULTSTATUS, ++ AS_FAULTADDRESS_LO, ++ AS_FAULTADDRESS_HI, ++ AS_STATUS ++}; ++ ++bool kbase_debug_job_fault_reg_snapshot_init(struct kbase_context *kctx, ++ int reg_range) ++{ ++ int i, j; ++ int offset = 0; ++ int slot_number; ++ int as_number; ++ ++ if (kctx->reg_dump == NULL) ++ return false; ++ ++ slot_number = kctx->kbdev->gpu_props.num_job_slots; ++ as_number = kctx->kbdev->gpu_props.num_address_spaces; ++ ++ /* get the GPU control registers*/ ++ for (i = 0; i < sizeof(gpu_control_reg_snapshot)/4; i++) { ++ kctx->reg_dump[offset] = ++ GPU_CONTROL_REG(gpu_control_reg_snapshot[i]); ++ offset += 2; ++ } ++ ++ /* get the Job control registers*/ ++ for (i = 0; i < sizeof(job_control_reg_snapshot)/4; i++) { ++ kctx->reg_dump[offset] = ++ JOB_CONTROL_REG(job_control_reg_snapshot[i]); ++ offset += 2; ++ } ++ ++ /* get the Job Slot registers*/ ++ for (j = 0; j < slot_number; j++) { ++ for (i = 0; i < sizeof(job_slot_reg_snapshot)/4; i++) { ++ kctx->reg_dump[offset] = ++ JOB_SLOT_REG(j, job_slot_reg_snapshot[i]); ++ offset += 2; ++ } ++ } ++ ++ /* get the MMU registers*/ ++ for (i = 0; i < sizeof(mmu_reg_snapshot)/4; i++) { ++ kctx->reg_dump[offset] = MMU_REG(mmu_reg_snapshot[i]); ++ offset += 2; ++ } ++ ++ /* get the Address space registers*/ ++ for (j = 0; j < as_number; j++) { ++ for (i = 0; i < sizeof(as_reg_snapshot)/4; i++) { ++ kctx->reg_dump[offset] = ++ MMU_AS_REG(j, as_reg_snapshot[i]); ++ offset += 2; ++ } ++ } ++ ++ WARN_ON(offset >= (reg_range*2/4)); ++ ++ /* set the termination flag*/ ++ kctx->reg_dump[offset] = REGISTER_DUMP_TERMINATION_FLAG; ++ kctx->reg_dump[offset + 1] = REGISTER_DUMP_TERMINATION_FLAG; ++ ++ dev_dbg(kctx->kbdev->dev, "kbase_job_fault_reg_snapshot_init:%d\n", ++ offset); ++ ++ return true; ++} ++ ++bool kbase_job_fault_get_reg_snapshot(struct kbase_context *kctx) ++{ ++ int offset = 0; ++ ++ if (kctx->reg_dump == NULL) ++ return false; ++ ++ while (kctx->reg_dump[offset] != REGISTER_DUMP_TERMINATION_FLAG) { ++ kctx->reg_dump[offset+1] = ++ kbase_reg_read(kctx->kbdev, ++ kctx->reg_dump[offset]); ++ offset += 2; ++ } ++ return true; ++} ++ ++ ++#endif +diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_devfreq.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_devfreq.c +new file mode 100755 +index 000000000000..ff561d180247 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_devfreq.c +@@ -0,0 +1,847 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++#include ++#include ++ ++#include ++#include ++#include ++#ifdef CONFIG_DEVFREQ_THERMAL ++#include ++#endif ++ ++#include ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0) ++#include ++#else /* Linux >= 3.13 */ ++/* In 3.13 the OPP include header file, types, and functions were all ++ * renamed. Use the old filename for the include, and define the new names to ++ * the old, when an old kernel is detected. ++ */ ++#include ++#define dev_pm_opp opp ++#define dev_pm_opp_get_voltage opp_get_voltage ++#define dev_pm_opp_get_opp_count opp_get_opp_count ++#define dev_pm_opp_find_freq_ceil opp_find_freq_ceil ++#define dev_pm_opp_find_freq_floor opp_find_freq_floor ++#endif /* Linux >= 3.13 */ ++#include ++#include ++#include ++ ++static struct devfreq_simple_ondemand_data ondemand_data; ++ ++static struct monitor_dev_profile mali_mdevp = { ++ .type = MONITOR_TPYE_DEV, ++ .low_temp_adjust = rockchip_monitor_dev_low_temp_adjust, ++ .high_temp_adjust = rockchip_monitor_dev_high_temp_adjust, ++}; ++ ++/** ++ * opp_translate - Translate nominal OPP frequency from devicetree into real ++ * frequency and core mask ++ * @kbdev: Device pointer ++ * @freq: Nominal frequency ++ * @volt: Nominal voltage ++ * @core_mask: Pointer to u64 to store core mask to ++ * @freqs: Pointer to array of frequencies ++ * @volts: Pointer to array of voltages ++ * ++ * This function will only perform translation if an operating-points-v2-mali ++ * table is present in devicetree. If one is not present then it will return an ++ * untranslated frequency and all cores enabled. ++ */ ++static void opp_translate(struct kbase_device *kbdev, unsigned long freq, ++ unsigned long volt, u64 *core_mask, ++ unsigned long *freqs, unsigned long *volts) ++{ ++ unsigned int i; ++ ++ for (i = 0; i < kbdev->num_opps; i++) { ++ if (kbdev->devfreq_table[i].opp_freq == freq) { ++ unsigned int j; ++ ++ *core_mask = kbdev->devfreq_table[i].core_mask; ++ for (j = 0; j < kbdev->nr_clocks; j++) { ++ freqs[j] = ++ kbdev->devfreq_table[i].real_freqs[j]; ++ volts[j] = ++ kbdev->devfreq_table[i].opp_volts[j]; ++ } ++ ++ break; ++ } ++ } ++ ++ /* If failed to find OPP, return all cores enabled ++ * and nominal frequency ++ */ ++ if (i == kbdev->num_opps) { ++ *core_mask = kbdev->gpu_props.props.raw_props.shader_present; ++ for (i = 0; i < kbdev->nr_clocks; i++) { ++ freqs[i] = freq; ++ volts[i] = volt; ++ } ++ } ++} ++ ++static int ++kbase_devfreq_target(struct device *dev, unsigned long *target_freq, u32 flags) ++{ ++ struct kbase_device *kbdev = dev_get_drvdata(dev); ++ struct dev_pm_opp *opp; ++ unsigned long nominal_freq, nominal_volt; ++ unsigned long freqs[BASE_MAX_NR_CLOCKS_REGULATORS] = {0}; ++ unsigned long old_freqs[BASE_MAX_NR_CLOCKS_REGULATORS] = {0}; ++ unsigned long volts[BASE_MAX_NR_CLOCKS_REGULATORS] = {0}; ++ unsigned int i; ++ u64 core_mask = 0; ++ ++ nominal_freq = *target_freq; ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) ++ rcu_read_lock(); ++#endif ++ opp = devfreq_recommended_opp(dev, &nominal_freq, flags); ++ if (IS_ERR_OR_NULL(opp)) { ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) ++ rcu_read_unlock(); ++#endif ++ dev_err(dev, "Failed to get opp (%ld)\n", PTR_ERR(opp)); ++ return PTR_ERR(opp); ++ } ++ nominal_volt = dev_pm_opp_get_voltage(opp); ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) ++ rcu_read_unlock(); ++#endif ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0) ++ dev_pm_opp_put(opp); ++#endif ++ ++ opp_translate(kbdev, nominal_freq, nominal_volt, &core_mask, freqs, ++ volts); ++ ++ /* ++ * Only update if there is a change of frequency ++ */ ++ if (kbdev->current_nominal_freq == nominal_freq) { ++ unsigned int i; ++ int err; ++ ++ *target_freq = nominal_freq; ++ ++#ifdef CONFIG_REGULATOR ++ for (i = 0; i < kbdev->nr_regulators; i++) { ++ if (kbdev->current_voltages[i] == volts[i]) ++ continue; ++ ++ err = regulator_set_voltage(kbdev->regulators[i], ++ volts[i], ++ INT_MAX); ++ if (err) { ++ dev_err(dev, "Failed to set voltage (%d)\n", err); ++ return err; ++ } ++ kbdev->current_voltages[i] = volts[i]; ++ } ++#endif ++ return 0; ++ } ++ dev_dbg(dev, "%lu-->%lu\n", kbdev->current_nominal_freq, nominal_freq); ++ ++#ifdef CONFIG_REGULATOR ++ /* Regulators and clocks work in pairs: every clock has a regulator, ++ * and we never expect to have more regulators than clocks. ++ * ++ * We always need to increase the voltage before increasing ++ * the frequency of a regulator/clock pair, otherwise the clock ++ * wouldn't have enough power to perform the transition. ++ * ++ * It's always safer to decrease the frequency before decreasing ++ * voltage of a regulator/clock pair, otherwise the clock could have ++ * problems operating if it is deprived of the necessary power ++ * to sustain its current frequency (even if that happens for a short ++ * transition interval). ++ */ ++ ++ for (i = 0; i < kbdev->nr_clocks; i++) ++ old_freqs[i] = kbdev->current_freqs[i]; ++ ++ for (i = 0; i < kbdev->nr_clocks; i++) { ++ if (kbdev->regulators[i] && ++ kbdev->current_voltages[i] != volts[i] && ++ old_freqs[i] < freqs[i]) { ++ int err; ++ ++ err = regulator_set_voltage(kbdev->regulators[i], ++ volts[i], INT_MAX); ++ if (!err) { ++ kbdev->current_voltages[i] = volts[i]; ++ } else { ++ dev_err(dev, "Failed to increase voltage (%d) (target %lu)\n", ++ err, volts[i]); ++ return err; ++ } ++ } ++ } ++#endif ++ ++ for (i = 0; i < kbdev->nr_clocks; i++) { ++ if (kbdev->clocks[i]) { ++ int err; ++ ++ err = clk_set_rate(kbdev->clocks[i], freqs[i]); ++ if (!err) { ++ kbdev->current_freqs[i] = freqs[i]; ++ } else { ++ dev_err(dev, "Failed to set clock %lu (target %lu)\n", ++ freqs[i], *target_freq); ++ return err; ++ } ++ } ++ } ++ ++#ifdef CONFIG_REGULATOR ++ for (i = 0; i < kbdev->nr_clocks; i++) { ++ if (kbdev->regulators[i] && ++ kbdev->current_voltages[i] != volts[i] && ++ old_freqs[i] > freqs[i]) { ++ int err; ++ ++ err = regulator_set_voltage(kbdev->regulators[i], ++ volts[i], INT_MAX); ++ if (!err) { ++ kbdev->current_voltages[i] = volts[i]; ++ } else { ++ dev_err(dev, "Failed to decrease voltage (%d) (target %lu)\n", ++ err, volts[i]); ++ return err; ++ } ++ } ++ } ++#endif ++ ++ kbase_devfreq_set_core_mask(kbdev, core_mask); ++ ++ *target_freq = nominal_freq; ++ kbdev->current_nominal_freq = nominal_freq; ++ kbdev->current_core_mask = core_mask; ++ if (kbdev->devfreq) ++ kbdev->devfreq->last_status.current_frequency = nominal_freq; ++ ++ KBASE_TLSTREAM_AUX_DEVFREQ_TARGET(kbdev, (u64)nominal_freq); ++ ++ return 0; ++} ++ ++void kbase_devfreq_force_freq(struct kbase_device *kbdev, unsigned long freq) ++{ ++ unsigned long target_freq = freq; ++ ++ kbase_devfreq_target(kbdev->dev, &target_freq, 0); ++} ++ ++static int ++kbase_devfreq_cur_freq(struct device *dev, unsigned long *freq) ++{ ++ struct kbase_device *kbdev = dev_get_drvdata(dev); ++ ++ *freq = kbdev->current_nominal_freq; ++ ++ return 0; ++} ++ ++static int ++kbase_devfreq_status(struct device *dev, struct devfreq_dev_status *stat) ++{ ++ struct kbase_device *kbdev = dev_get_drvdata(dev); ++ struct kbasep_pm_metrics diff; ++ ++ kbase_pm_get_dvfs_metrics(kbdev, &kbdev->last_devfreq_metrics, &diff); ++ ++ stat->busy_time = diff.time_busy; ++ stat->total_time = diff.time_busy + diff.time_idle; ++ stat->current_frequency = kbdev->current_nominal_freq; ++ stat->private_data = NULL; ++ ++ return 0; ++} ++ ++static int kbase_devfreq_init_freq_table(struct kbase_device *kbdev, ++ struct devfreq_dev_profile *dp) ++{ ++ int count; ++ int i = 0; ++ unsigned long freq; ++ struct dev_pm_opp *opp; ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) ++ rcu_read_lock(); ++#endif ++ count = dev_pm_opp_get_opp_count(kbdev->dev); ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) ++ rcu_read_unlock(); ++#endif ++ if (count < 0) ++ return count; ++ ++ dp->freq_table = kmalloc_array(count, sizeof(dp->freq_table[0]), ++ GFP_KERNEL); ++ if (!dp->freq_table) ++ return -ENOMEM; ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) ++ rcu_read_lock(); ++#endif ++ for (i = 0, freq = ULONG_MAX; i < count; i++, freq--) { ++ opp = dev_pm_opp_find_freq_floor(kbdev->dev, &freq); ++ if (IS_ERR(opp)) ++ break; ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0) ++ dev_pm_opp_put(opp); ++#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0) */ ++ ++ dp->freq_table[i] = freq; ++ } ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) ++ rcu_read_unlock(); ++#endif ++ ++ if (count != i) ++ dev_warn(kbdev->dev, "Unable to enumerate all OPPs (%d!=%d\n", ++ count, i); ++ ++ dp->max_state = i; ++ ++ /* Have the lowest clock as suspend clock. ++ * It may be overridden by 'opp-mali-errata-1485982'. ++ */ ++ if (kbdev->pm.backend.gpu_clock_slow_down_wa) { ++ freq = 0; ++ opp = dev_pm_opp_find_freq_ceil(kbdev->dev, &freq); ++ if (IS_ERR(opp)) { ++ dev_err(kbdev->dev, "failed to find slowest clock"); ++ return 0; ++ } ++ dev_pm_opp_put(opp); ++ dev_info(kbdev->dev, "suspend clock %lu from slowest", freq); ++ kbdev->pm.backend.gpu_clock_suspend_freq = freq; ++ } ++ ++ return 0; ++} ++ ++static void kbase_devfreq_term_freq_table(struct kbase_device *kbdev) ++{ ++ struct devfreq_dev_profile *dp = &kbdev->devfreq_profile; ++ ++ kfree(dp->freq_table); ++} ++ ++static void kbase_devfreq_term_core_mask_table(struct kbase_device *kbdev) ++{ ++ kfree(kbdev->devfreq_table); ++} ++ ++static void kbase_devfreq_exit(struct device *dev) ++{ ++ struct kbase_device *kbdev = dev_get_drvdata(dev); ++ ++ kbase_devfreq_term_freq_table(kbdev); ++} ++ ++static void kbasep_devfreq_read_suspend_clock(struct kbase_device *kbdev, ++ struct device_node *node) ++{ ++ u64 freq = 0; ++ int err = 0; ++ ++ /* Check if this node is the opp entry having 'opp-mali-errata-1485982' ++ * to get the suspend clock, otherwise skip it. ++ */ ++ if (!of_property_read_bool(node, "opp-mali-errata-1485982")) ++ return; ++ ++ /* In kbase DevFreq, the clock will be read from 'opp-hz' ++ * and translated into the actual clock by opp_translate. ++ * ++ * In customer DVFS, the clock will be read from 'opp-hz-real' ++ * for clk driver. If 'opp-hz-real' does not exist, ++ * read from 'opp-hz'. ++ */ ++ if (IS_ENABLED(CONFIG_MALI_BIFROST_DEVFREQ)) ++ err = of_property_read_u64(node, "opp-hz", &freq); ++ else { ++ if (of_property_read_u64(node, "opp-hz-real", &freq)) ++ err = of_property_read_u64(node, "opp-hz", &freq); ++ } ++ ++ if (WARN_ON(err || !freq)) ++ return; ++ ++ kbdev->pm.backend.gpu_clock_suspend_freq = freq; ++ dev_info(kbdev->dev, ++ "suspend clock %llu by opp-mali-errata-1485982", freq); ++} ++ ++static int kbase_devfreq_init_core_mask_table(struct kbase_device *kbdev) ++{ ++#if KERNEL_VERSION(3, 18, 0) > LINUX_VERSION_CODE || !defined(CONFIG_OF) ++ /* OPP table initialization requires at least the capability to get ++ * regulators and clocks from the device tree, as well as parsing ++ * arrays of unsigned integer values. ++ * ++ * The whole initialization process shall simply be skipped if the ++ * minimum capability is not available. ++ */ ++ return 0; ++#else ++ struct device_node *opp_node = of_parse_phandle(kbdev->dev->of_node, ++ "operating-points-v2", 0); ++ struct device_node *node; ++ int i = 0; ++ int count; ++ u64 shader_present = kbdev->gpu_props.props.raw_props.shader_present; ++ ++ if (!opp_node) ++ return 0; ++ if (!of_device_is_compatible(opp_node, "operating-points-v2-mali")) ++ return 0; ++ ++ count = dev_pm_opp_get_opp_count(kbdev->dev); ++ kbdev->devfreq_table = kmalloc_array(count, ++ sizeof(struct kbase_devfreq_opp), GFP_KERNEL); ++ if (!kbdev->devfreq_table) ++ return -ENOMEM; ++ ++ for_each_available_child_of_node(opp_node, node) { ++ const void *core_count_p; ++ u64 core_mask, opp_freq, ++ real_freqs[BASE_MAX_NR_CLOCKS_REGULATORS]; ++ int err; ++#ifdef CONFIG_REGULATOR ++ u32 opp_volts[BASE_MAX_NR_CLOCKS_REGULATORS]; ++#endif ++ ++ /* Read suspend clock from opp table */ ++ if (kbdev->pm.backend.gpu_clock_slow_down_wa) ++ kbasep_devfreq_read_suspend_clock(kbdev, node); ++ ++ err = of_property_read_u64(node, "opp-hz", &opp_freq); ++ if (err) { ++ dev_warn(kbdev->dev, "Failed to read opp-hz property with error %d\n", ++ err); ++ continue; ++ } ++ ++ ++#if BASE_MAX_NR_CLOCKS_REGULATORS > 1 ++ err = of_property_read_u64_array(node, "opp-hz-real", ++ real_freqs, kbdev->nr_clocks); ++#else ++ WARN_ON(kbdev->nr_clocks != 1); ++ err = of_property_read_u64(node, "opp-hz-real", real_freqs); ++#endif ++ if (err < 0) { ++ dev_warn(kbdev->dev, "Failed to read opp-hz-real property with error %d\n", ++ err); ++ continue; ++ } ++#ifdef CONFIG_REGULATOR ++ err = of_property_read_u32_array(node, ++ "opp-microvolt", opp_volts, kbdev->nr_regulators); ++ if (err < 0) { ++ dev_warn(kbdev->dev, "Failed to read opp-microvolt property with error %d\n", ++ err); ++ continue; ++ } ++#endif ++ ++ if (of_property_read_u64(node, "opp-core-mask", &core_mask)) ++ core_mask = shader_present; ++ if (core_mask != shader_present && corestack_driver_control) { ++ ++ dev_warn(kbdev->dev, "Ignoring OPP %llu - Dynamic Core Scaling not supported on this GPU\n", ++ opp_freq); ++ continue; ++ } ++ ++ core_count_p = of_get_property(node, "opp-core-count", NULL); ++ if (core_count_p) { ++ u64 remaining_core_mask = ++ kbdev->gpu_props.props.raw_props.shader_present; ++ int core_count = be32_to_cpup(core_count_p); ++ ++ core_mask = 0; ++ ++ for (; core_count > 0; core_count--) { ++ int core = ffs(remaining_core_mask); ++ ++ if (!core) { ++ dev_err(kbdev->dev, "OPP has more cores than GPU\n"); ++ return -ENODEV; ++ } ++ ++ core_mask |= (1ull << (core-1)); ++ remaining_core_mask &= ~(1ull << (core-1)); ++ } ++ } ++ ++ if (!core_mask) { ++ dev_err(kbdev->dev, "OPP has invalid core mask of 0\n"); ++ return -ENODEV; ++ } ++ ++ kbdev->devfreq_table[i].opp_freq = opp_freq; ++ kbdev->devfreq_table[i].core_mask = core_mask; ++ if (kbdev->nr_clocks > 0) { ++ int j; ++ ++ for (j = 0; j < kbdev->nr_clocks; j++) ++ kbdev->devfreq_table[i].real_freqs[j] = ++ real_freqs[j]; ++ } ++#ifdef CONFIG_REGULATOR ++ if (kbdev->nr_regulators > 0) { ++ int j; ++ ++ for (j = 0; j < kbdev->nr_regulators; j++) ++ kbdev->devfreq_table[i].opp_volts[j] = ++ opp_volts[j]; ++ } ++#endif ++ ++ dev_info(kbdev->dev, "OPP %d : opp_freq=%llu core_mask=%llx\n", ++ i, opp_freq, core_mask); ++ ++ i++; ++ } ++ ++ kbdev->num_opps = i; ++ ++ return 0; ++#endif /* KERNEL_VERSION(3, 18, 0) > LINUX_VERSION_CODE */ ++} ++ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0) ++ ++static const char *kbase_devfreq_req_type_name(enum kbase_devfreq_work_type type) ++{ ++ const char *p; ++ ++ switch (type) { ++ case DEVFREQ_WORK_NONE: ++ p = "devfreq_none"; ++ break; ++ case DEVFREQ_WORK_SUSPEND: ++ p = "devfreq_suspend"; ++ break; ++ case DEVFREQ_WORK_RESUME: ++ p = "devfreq_resume"; ++ break; ++ default: ++ p = "Unknown devfreq_type"; ++ } ++ return p; ++} ++ ++static void kbase_devfreq_suspend_resume_worker(struct work_struct *work) ++{ ++ struct kbase_devfreq_queue_info *info = container_of(work, ++ struct kbase_devfreq_queue_info, work); ++ struct kbase_device *kbdev = container_of(info, struct kbase_device, ++ devfreq_queue); ++ unsigned long flags; ++ enum kbase_devfreq_work_type type, acted_type; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ type = kbdev->devfreq_queue.req_type; ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ acted_type = kbdev->devfreq_queue.acted_type; ++ dev_dbg(kbdev->dev, "Worker handles queued req: %s (acted: %s)\n", ++ kbase_devfreq_req_type_name(type), ++ kbase_devfreq_req_type_name(acted_type)); ++ switch (type) { ++ case DEVFREQ_WORK_SUSPEND: ++ case DEVFREQ_WORK_RESUME: ++ if (type != acted_type) { ++ if (type == DEVFREQ_WORK_RESUME) ++ devfreq_resume_device(kbdev->devfreq); ++ else ++ devfreq_suspend_device(kbdev->devfreq); ++ dev_dbg(kbdev->dev, "Devfreq transition occured: %s => %s\n", ++ kbase_devfreq_req_type_name(acted_type), ++ kbase_devfreq_req_type_name(type)); ++ kbdev->devfreq_queue.acted_type = type; ++ } ++ break; ++ default: ++ WARN_ON(1); ++ } ++} ++ ++#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0) */ ++ ++void kbase_devfreq_enqueue_work(struct kbase_device *kbdev, ++ enum kbase_devfreq_work_type work_type) ++{ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0) ++ unsigned long flags; ++ ++ WARN_ON(work_type == DEVFREQ_WORK_NONE); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbdev->devfreq_queue.req_type = work_type; ++ queue_work(kbdev->devfreq_queue.workq, &kbdev->devfreq_queue.work); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ dev_dbg(kbdev->dev, "Enqueuing devfreq req: %s\n", ++ kbase_devfreq_req_type_name(work_type)); ++#endif ++} ++ ++static int kbase_devfreq_work_init(struct kbase_device *kbdev) ++{ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0) ++ kbdev->devfreq_queue.req_type = DEVFREQ_WORK_NONE; ++ kbdev->devfreq_queue.acted_type = DEVFREQ_WORK_RESUME; ++ ++ kbdev->devfreq_queue.workq = alloc_ordered_workqueue("devfreq_workq", 0); ++ if (!kbdev->devfreq_queue.workq) ++ return -ENOMEM; ++ ++ INIT_WORK(&kbdev->devfreq_queue.work, ++ kbase_devfreq_suspend_resume_worker); ++#endif ++ return 0; ++} ++ ++static void kbase_devfreq_work_term(struct kbase_device *kbdev) ++{ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0) ++ destroy_workqueue(kbdev->devfreq_queue.workq); ++#endif ++} ++ ++static unsigned long kbase_devfreq_get_static_power(struct devfreq *devfreq, ++ unsigned long voltage) ++{ ++ struct device *dev = devfreq->dev.parent; ++ struct kbase_device *kbdev = dev_get_drvdata(dev); ++ ++ return rockchip_ipa_get_static_power(kbdev->model_data, voltage); ++} ++ ++static struct devfreq_cooling_power kbase_cooling_power = { ++ .get_static_power = &kbase_devfreq_get_static_power, ++}; ++ ++int kbase_devfreq_init(struct kbase_device *kbdev) ++{ ++ struct devfreq_cooling_power *kbase_dcp = &kbase_cooling_power; ++ struct device_node *np = kbdev->dev->of_node; ++ struct devfreq_dev_profile *dp; ++ struct dev_pm_opp *opp; ++ unsigned long opp_rate; ++ int err; ++ unsigned int i; ++ ++ if (kbdev->nr_clocks == 0) { ++ dev_err(kbdev->dev, "Clock not available for devfreq\n"); ++ return -ENODEV; ++ } ++ ++ for (i = 0; i < kbdev->nr_clocks; i++) { ++ if (kbdev->clocks[i]) ++ kbdev->current_freqs[i] = ++ clk_get_rate(kbdev->clocks[i]); ++ else ++ kbdev->current_freqs[i] = 0; ++ } ++ kbdev->current_nominal_freq = kbdev->current_freqs[0]; ++ ++ dp = &kbdev->devfreq_profile; ++ ++ dp->initial_freq = kbdev->current_freqs[0]; ++ dp->polling_ms = 100; ++ dp->target = kbase_devfreq_target; ++ dp->get_dev_status = kbase_devfreq_status; ++ dp->get_cur_freq = kbase_devfreq_cur_freq; ++ dp->exit = kbase_devfreq_exit; ++ ++ if (kbase_devfreq_init_freq_table(kbdev, dp)) ++ return -EFAULT; ++ ++ if (dp->max_state > 0) { ++ /* Record the maximum frequency possible */ ++ kbdev->gpu_props.props.core_props.gpu_freq_khz_max = ++ dp->freq_table[0] / 1000; ++ }; ++ ++ err = kbase_devfreq_init_core_mask_table(kbdev); ++ if (err) { ++ kbase_devfreq_term_freq_table(kbdev); ++ return err; ++ } ++ ++ /* Initialise devfreq suspend/resume workqueue */ ++ err = kbase_devfreq_work_init(kbdev); ++ if (err) { ++ kbase_devfreq_term_freq_table(kbdev); ++ dev_err(kbdev->dev, "Devfreq initialization failed"); ++ return err; ++ } ++ ++ of_property_read_u32(np, "upthreshold", ++ &ondemand_data.upthreshold); ++ of_property_read_u32(np, "downdifferential", ++ &ondemand_data.downdifferential); ++ kbdev->devfreq = devfreq_add_device(kbdev->dev, dp, ++ "simple_ondemand", &ondemand_data); ++ if (IS_ERR(kbdev->devfreq)) { ++ err = PTR_ERR(kbdev->devfreq); ++ kbase_devfreq_work_term(kbdev); ++ kbase_devfreq_term_freq_table(kbdev); ++ return err; ++ } ++ ++ /* devfreq_add_device only copies a few of kbdev->dev's fields, so ++ * set drvdata explicitly so IPA models can access kbdev. */ ++ dev_set_drvdata(&kbdev->devfreq->dev, kbdev); ++ ++ err = devfreq_register_opp_notifier(kbdev->dev, kbdev->devfreq); ++ if (err) { ++ dev_err(kbdev->dev, ++ "Failed to register OPP notifier (%d)\n", err); ++ goto opp_notifier_failed; ++ } ++ ++ opp_rate = kbdev->current_freqs[0]; /* Bifrost GPU has only 1 clock. */ ++ opp = devfreq_recommended_opp(kbdev->dev, &opp_rate, 0); ++ if (!IS_ERR(opp)) ++ dev_pm_opp_put(opp); ++ kbdev->devfreq->last_status.current_frequency = opp_rate; ++ ++ mali_mdevp.data = kbdev->devfreq; ++ kbdev->mdev_info = rockchip_system_monitor_register(kbdev->dev, ++ &mali_mdevp); ++ if (IS_ERR(kbdev->mdev_info)) { ++ dev_dbg(kbdev->dev, "without system monitor\n"); ++ kbdev->mdev_info = NULL; ++ } ++#ifdef CONFIG_DEVFREQ_THERMAL ++ if (of_find_compatible_node(kbdev->dev->of_node, NULL, ++ "simple-power-model")) { ++ of_property_read_u32(kbdev->dev->of_node, ++ "dynamic-power-coefficient", ++ (u32 *)&kbase_dcp->dyn_power_coeff); ++ kbdev->model_data = rockchip_ipa_power_model_init(kbdev->dev, ++ "gpu_leakage"); ++ if (IS_ERR_OR_NULL(kbdev->model_data)) { ++ kbdev->model_data = NULL; ++ dev_err(kbdev->dev, "failed to initialize power model\n"); ++ } else if (kbdev->model_data->dynamic_coefficient) { ++ kbase_dcp->dyn_power_coeff = ++ kbdev->model_data->dynamic_coefficient; ++ } ++ if (!kbase_dcp->dyn_power_coeff) { ++ err = -EINVAL; ++ dev_err(kbdev->dev, "failed to get dynamic-coefficient\n"); ++ goto cooling_failed; ++ } ++ ++ kbdev->devfreq_cooling = ++ of_devfreq_cooling_register_power(kbdev->dev->of_node, ++ kbdev->devfreq, ++ kbase_dcp); ++ if (IS_ERR(kbdev->devfreq_cooling)) { ++ err = PTR_ERR(kbdev->devfreq_cooling); ++ dev_err(kbdev->dev, "failed to register cooling device\n"); ++ goto cooling_failed; ++ } ++ } else { ++ err = kbase_ipa_init(kbdev); ++ if (err) { ++ dev_err(kbdev->dev, "IPA initialization failed\n"); ++ goto cooling_failed; ++ } ++ ++ kbdev->devfreq_cooling = of_devfreq_cooling_register_power( ++ kbdev->dev->of_node, ++ kbdev->devfreq, ++ &kbase_ipa_power_model_ops); ++ if (IS_ERR(kbdev->devfreq_cooling)) { ++ err = PTR_ERR(kbdev->devfreq_cooling); ++ dev_err(kbdev->dev, ++ "Failed to register cooling device (%d)\n", ++ err); ++ goto cooling_failed; ++ } ++ } ++#endif ++ ++ return 0; ++ ++#ifdef CONFIG_DEVFREQ_THERMAL ++cooling_failed: ++ devfreq_unregister_opp_notifier(kbdev->dev, kbdev->devfreq); ++#endif /* CONFIG_DEVFREQ_THERMAL */ ++opp_notifier_failed: ++ if (devfreq_remove_device(kbdev->devfreq)) ++ dev_err(kbdev->dev, "Failed to terminate devfreq (%d)\n", err); ++ else ++ kbdev->devfreq = NULL; ++ ++ kbase_devfreq_work_term(kbdev); ++ ++ return err; ++} ++ ++void kbase_devfreq_term(struct kbase_device *kbdev) ++{ ++ int err; ++ ++ dev_dbg(kbdev->dev, "Term Mali devfreq\n"); ++ ++ rockchip_system_monitor_unregister(kbdev->mdev_info); ++#ifdef CONFIG_DEVFREQ_THERMAL ++ if (kbdev->devfreq_cooling) ++ devfreq_cooling_unregister(kbdev->devfreq_cooling); ++ ++ if (!kbdev->model_data) ++ kbase_ipa_term(kbdev); ++ kfree(kbdev->model_data); ++#endif ++ ++ devfreq_unregister_opp_notifier(kbdev->dev, kbdev->devfreq); ++ ++ err = devfreq_remove_device(kbdev->devfreq); ++ if (err) ++ dev_err(kbdev->dev, "Failed to terminate devfreq (%d)\n", err); ++ else ++ kbdev->devfreq = NULL; ++ ++ kbase_devfreq_term_core_mask_table(kbdev); ++ ++ kbase_devfreq_work_term(kbdev); ++} +diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_devfreq.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_devfreq.h +new file mode 100755 +index 000000000000..7bcd47c70ef0 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_devfreq.h +@@ -0,0 +1,47 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014, 2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _BASE_DEVFREQ_H_ ++#define _BASE_DEVFREQ_H_ ++ ++int kbase_devfreq_init(struct kbase_device *kbdev); ++ ++void kbase_devfreq_term(struct kbase_device *kbdev); ++int kbase_platform_rk_init_opp_table(struct kbase_device *kbdev); ++ ++/** ++ * kbase_devfreq_force_freq - Set GPU frequency on L2 power on/off. ++ * @kbdev: Device pointer ++ * @freq: GPU frequency in HZ to be set when ++ * MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE is enabled ++ */ ++void kbase_devfreq_force_freq(struct kbase_device *kbdev, unsigned long freq); ++ ++/** ++ * kbase_devfreq_enqueue_work - Enqueue a work item for suspend/resume devfreq. ++ * @kbdev: Device pointer ++ * @work_type: The type of the devfreq work item, i.e. suspend or resume ++ */ ++void kbase_devfreq_enqueue_work(struct kbase_device *kbdev, ++ enum kbase_devfreq_work_type work_type); ++ ++#endif /* _BASE_DEVFREQ_H_ */ +diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_gpuprops_backend.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_gpuprops_backend.c +new file mode 100755 +index 000000000000..60ae0206d6a8 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_gpuprops_backend.c +@@ -0,0 +1,158 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * ++ * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* ++ * Base kernel property query backend APIs ++ */ ++ ++#include ++#include ++#include ++#include ++ ++int kbase_backend_gpuprops_get(struct kbase_device *kbdev, ++ struct kbase_gpuprops_regdump *regdump) ++{ ++ int i; ++ struct kbase_gpuprops_regdump registers; ++ ++ /* Fill regdump with the content of the relevant registers */ ++ registers.gpu_id = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_ID)); ++ ++ registers.l2_features = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(L2_FEATURES)); ++#if !MALI_USE_CSF ++ registers.core_features = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(CORE_FEATURES)); ++#else /* !MALI_USE_CSF */ ++ registers.core_features = 0; ++#endif /* !MALI_USE_CSF */ ++ registers.tiler_features = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(TILER_FEATURES)); ++ registers.mem_features = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(MEM_FEATURES)); ++ registers.mmu_features = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(MMU_FEATURES)); ++ registers.as_present = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(AS_PRESENT)); ++#if !MALI_USE_CSF ++ registers.js_present = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(JS_PRESENT)); ++#else /* !MALI_USE_CSF */ ++ registers.js_present = 0; ++#endif /* !MALI_USE_CSF */ ++ ++ for (i = 0; i < GPU_MAX_JOB_SLOTS; i++) ++#if !MALI_USE_CSF ++ registers.js_features[i] = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(JS_FEATURES_REG(i))); ++#else /* !MALI_USE_CSF */ ++ registers.js_features[i] = 0; ++#endif /* !MALI_USE_CSF */ ++ ++ for (i = 0; i < BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS; i++) ++ registers.texture_features[i] = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(TEXTURE_FEATURES_REG(i))); ++ ++ registers.thread_max_threads = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(THREAD_MAX_THREADS)); ++ registers.thread_max_workgroup_size = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(THREAD_MAX_WORKGROUP_SIZE)); ++ registers.thread_max_barrier_size = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(THREAD_MAX_BARRIER_SIZE)); ++ registers.thread_features = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(THREAD_FEATURES)); ++ registers.thread_tls_alloc = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(THREAD_TLS_ALLOC)); ++ ++ registers.shader_present_lo = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(SHADER_PRESENT_LO)); ++ registers.shader_present_hi = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(SHADER_PRESENT_HI)); ++ ++ registers.tiler_present_lo = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(TILER_PRESENT_LO)); ++ registers.tiler_present_hi = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(TILER_PRESENT_HI)); ++ ++ registers.l2_present_lo = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(L2_PRESENT_LO)); ++ registers.l2_present_hi = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(L2_PRESENT_HI)); ++ ++ registers.stack_present_lo = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(STACK_PRESENT_LO)); ++ registers.stack_present_hi = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(STACK_PRESENT_HI)); ++ ++ if (!kbase_is_gpu_removed(kbdev)) { ++ *regdump = registers; ++ return 0; ++ } else ++ return -EIO; ++} ++ ++int kbase_backend_gpuprops_get_features(struct kbase_device *kbdev, ++ struct kbase_gpuprops_regdump *regdump) ++{ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_COHERENCY_REG)) { ++ u32 coherency_features; ++ ++ /* Ensure we can access the GPU registers */ ++ kbase_pm_register_access_enable(kbdev); ++ ++ coherency_features = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(COHERENCY_FEATURES)); ++ ++ if (kbase_is_gpu_removed(kbdev)) ++ return -EIO; ++ ++ regdump->coherency_features = coherency_features; ++ ++ /* We're done accessing the GPU registers for now. */ ++ kbase_pm_register_access_disable(kbdev); ++ } else { ++ /* Pre COHERENCY_FEATURES we only supported ACE_LITE */ ++ regdump->coherency_features = ++ COHERENCY_FEATURE_BIT(COHERENCY_NONE) | ++ COHERENCY_FEATURE_BIT(COHERENCY_ACE_LITE); ++ } ++ ++ return 0; ++} ++ ++int kbase_backend_gpuprops_get_l2_features(struct kbase_device *kbdev, ++ struct kbase_gpuprops_regdump *regdump) ++{ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_L2_CONFIG)) { ++ u32 l2_features = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(L2_FEATURES)); ++ ++ if (kbase_is_gpu_removed(kbdev)) ++ return -EIO; ++ ++ regdump->l2_features = l2_features; ++ } ++ ++ return 0; ++} +diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_instr_backend.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_instr_backend.c +new file mode 100755 +index 000000000000..54b07483dee6 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_instr_backend.c +@@ -0,0 +1,520 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++/* ++ * GPU backend instrumentation APIs. ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++ ++ ++int kbase_instr_hwcnt_enable_internal(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ struct kbase_instr_hwcnt_enable *enable) ++{ ++ unsigned long flags; ++ int err = -EINVAL; ++#if !MALI_USE_CSF ++ u32 irq_mask; ++#endif ++ u32 prfcnt_config; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ /* alignment failure */ ++ if ((enable->dump_buffer == 0ULL) || (enable->dump_buffer & (2048 - 1))) ++ goto out_err; ++ ++ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); ++ ++ if (kbdev->hwcnt.backend.state != KBASE_INSTR_STATE_DISABLED) { ++ /* Instrumentation is already enabled */ ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++ goto out_err; ++ } ++ ++#if !MALI_USE_CSF ++ /* Enable interrupt */ ++ irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK)); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), irq_mask | ++ PRFCNT_SAMPLE_COMPLETED); ++#endif ++ ++ /* In use, this context is the owner */ ++ kbdev->hwcnt.kctx = kctx; ++ /* Remember the dump address so we can reprogram it later */ ++ kbdev->hwcnt.addr = enable->dump_buffer; ++ kbdev->hwcnt.addr_bytes = enable->dump_buffer_bytes; ++ ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++ ++ /* Configure */ ++ prfcnt_config = kctx->as_nr << PRFCNT_CONFIG_AS_SHIFT; ++#ifdef CONFIG_MALI_PRFCNT_SET_SECONDARY_VIA_DEBUG_FS ++ if (kbdev->hwcnt.backend.use_secondary_override) ++#else ++ if (enable->use_secondary) ++#endif ++ prfcnt_config |= 1 << PRFCNT_CONFIG_SETSELECT_SHIFT; ++ ++#if MALI_USE_CSF ++ kbase_reg_write(kbdev, GPU_CONTROL_MCU_REG(PRFCNT_CONFIG), ++ prfcnt_config | PRFCNT_CONFIG_MODE_OFF); ++ ++ kbase_reg_write(kbdev, GPU_CONTROL_MCU_REG(PRFCNT_BASE_LO), ++ enable->dump_buffer & 0xFFFFFFFF); ++ kbase_reg_write(kbdev, GPU_CONTROL_MCU_REG(PRFCNT_BASE_HI), ++ enable->dump_buffer >> 32); ++ ++ kbase_reg_write(kbdev, GPU_CONTROL_MCU_REG(PRFCNT_CSHW_EN), ++ enable->fe_bm); ++ ++ kbase_reg_write(kbdev, GPU_CONTROL_MCU_REG(PRFCNT_SHADER_EN), ++ enable->shader_bm); ++ kbase_reg_write(kbdev, GPU_CONTROL_MCU_REG(PRFCNT_MMU_L2_EN), ++ enable->mmu_l2_bm); ++ ++ kbase_reg_write(kbdev, GPU_CONTROL_MCU_REG(PRFCNT_TILER_EN), ++ enable->tiler_bm); ++ ++ kbase_reg_write(kbdev, GPU_CONTROL_MCU_REG(PRFCNT_CONFIG), ++ prfcnt_config | PRFCNT_CONFIG_MODE_MANUAL); ++#else ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), ++ prfcnt_config | PRFCNT_CONFIG_MODE_OFF); ++ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_LO), ++ enable->dump_buffer & 0xFFFFFFFF); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_HI), ++ enable->dump_buffer >> 32); ++ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_JM_EN), ++ enable->fe_bm); ++ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_SHADER_EN), ++ enable->shader_bm); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_MMU_L2_EN), ++ enable->mmu_l2_bm); ++ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), ++ enable->tiler_bm); ++ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), ++ prfcnt_config | PRFCNT_CONFIG_MODE_MANUAL); ++#endif ++ ++ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); ++ ++ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; ++ kbdev->hwcnt.backend.triggered = 1; ++ wake_up(&kbdev->hwcnt.backend.wait); ++ ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++ ++ err = 0; ++ ++ dev_dbg(kbdev->dev, "HW counters dumping set-up for context %p", kctx); ++ return err; ++ out_err: ++ return err; ++} ++ ++int kbase_instr_hwcnt_disable_internal(struct kbase_context *kctx) ++{ ++ unsigned long flags, pm_flags; ++ int err = -EINVAL; ++#if !MALI_USE_CSF ++ u32 irq_mask; ++#endif ++ struct kbase_device *kbdev = kctx->kbdev; ++ ++ while (1) { ++ spin_lock_irqsave(&kbdev->hwaccess_lock, pm_flags); ++ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); ++ ++ if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_DISABLED) { ++ /* Instrumentation is not enabled */ ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); ++ goto out; ++ } ++ ++ if (kbdev->hwcnt.kctx != kctx) { ++ /* Instrumentation has been setup for another context */ ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); ++ goto out; ++ } ++ ++ if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_IDLE) ++ break; ++ ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); ++ ++ /* Ongoing dump/setup - wait for its completion */ ++ wait_event(kbdev->hwcnt.backend.wait, ++ kbdev->hwcnt.backend.triggered != 0); ++ } ++ ++ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DISABLED; ++ kbdev->hwcnt.backend.triggered = 0; ++ ++#if MALI_USE_CSF ++ /* Disable the counters */ ++ kbase_reg_write(kbdev, GPU_CONTROL_MCU_REG(PRFCNT_CONFIG), 0); ++#else ++ /* Disable interrupt */ ++ irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK)); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), ++ irq_mask & ~PRFCNT_SAMPLE_COMPLETED); ++ ++ /* Disable the counters */ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), 0); ++#endif ++ ++ kbdev->hwcnt.kctx = NULL; ++ kbdev->hwcnt.addr = 0ULL; ++ kbdev->hwcnt.addr_bytes = 0ULL; ++ ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); ++ ++ dev_dbg(kbdev->dev, "HW counters dumping disabled for context %p", ++ kctx); ++ ++ err = 0; ++ ++ out: ++ return err; ++} ++ ++int kbase_instr_hwcnt_request_dump(struct kbase_context *kctx) ++{ ++ unsigned long flags; ++ int err = -EINVAL; ++ struct kbase_device *kbdev = kctx->kbdev; ++ ++ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); ++ ++ if (kbdev->hwcnt.kctx != kctx) { ++ /* The instrumentation has been setup for another context */ ++ goto unlock; ++ } ++ ++ if (kbdev->hwcnt.backend.state != KBASE_INSTR_STATE_IDLE) { ++ /* HW counters are disabled or another dump is ongoing, or we're ++ * resetting */ ++ goto unlock; ++ } ++ ++ kbdev->hwcnt.backend.triggered = 0; ++ ++ /* Mark that we're dumping - the PF handler can signal that we faulted ++ */ ++ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DUMPING; ++ ++ ++#if MALI_USE_CSF ++ /* Reconfigure the dump address */ ++ kbase_reg_write(kbdev, GPU_CONTROL_MCU_REG(PRFCNT_BASE_LO), ++ kbdev->hwcnt.addr & 0xFFFFFFFF); ++ kbase_reg_write(kbdev, GPU_CONTROL_MCU_REG(PRFCNT_BASE_HI), ++ kbdev->hwcnt.addr >> 32); ++#else ++ /* Reconfigure the dump address */ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_LO), ++ kbdev->hwcnt.addr & 0xFFFFFFFF); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_HI), ++ kbdev->hwcnt.addr >> 32); ++#endif ++ ++ /* Start dumping */ ++ KBASE_KTRACE_ADD(kbdev, CORE_GPU_PRFCNT_SAMPLE, NULL, ++ kbdev->hwcnt.addr); ++ ++#if MALI_USE_CSF ++ kbase_reg_write(kbdev, GPU_CONTROL_MCU_REG(GPU_COMMAND), ++ GPU_COMMAND_PRFCNT_SAMPLE); ++#else ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), ++ GPU_COMMAND_PRFCNT_SAMPLE); ++#endif ++ ++ dev_dbg(kbdev->dev, "HW counters dumping done for context %p", kctx); ++ ++ err = 0; ++ ++ unlock: ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++ ++#if MALI_USE_CSF ++ tasklet_schedule(&kbdev->hwcnt.backend.csf_hwc_irq_poll_tasklet); ++#endif ++ ++ return err; ++} ++KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_request_dump); ++ ++bool kbase_instr_hwcnt_dump_complete(struct kbase_context *kctx, ++ bool * const success) ++{ ++ unsigned long flags; ++ bool complete = false; ++ struct kbase_device *kbdev = kctx->kbdev; ++ ++ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); ++ ++ if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_IDLE) { ++ *success = true; ++ complete = true; ++ } else if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_FAULT) { ++ *success = false; ++ complete = true; ++ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; ++ } ++ ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++ ++ return complete; ++} ++KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_dump_complete); ++ ++void kbasep_cache_clean_worker(struct work_struct *data) ++{ ++ struct kbase_device *kbdev; ++ unsigned long flags, pm_flags; ++ ++ kbdev = container_of(data, struct kbase_device, ++ hwcnt.backend.cache_clean_work); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, pm_flags); ++ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); ++ ++ /* Clean and invalidate the caches so we're sure the mmu tables for the ++ * dump buffer is valid. ++ */ ++ KBASE_DEBUG_ASSERT(kbdev->hwcnt.backend.state == ++ KBASE_INSTR_STATE_REQUEST_CLEAN); ++ kbase_gpu_start_cache_clean_nolock(kbdev); ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); ++ ++ kbase_gpu_wait_cache_clean(kbdev); ++ ++ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); ++ KBASE_DEBUG_ASSERT(kbdev->hwcnt.backend.state == ++ KBASE_INSTR_STATE_REQUEST_CLEAN); ++ /* All finished and idle */ ++ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; ++ kbdev->hwcnt.backend.triggered = 1; ++ wake_up(&kbdev->hwcnt.backend.wait); ++ ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++} ++ ++#if MALI_USE_CSF ++/** ++ * kbasep_hwcnt_irq_poll_tasklet - tasklet to poll MCU IRQ status register ++ * ++ * @data: tasklet parameter which pointer to kbdev ++ * ++ * This tasklet poll GPU_IRQ_STATUS register in GPU_CONTROL_MCU page to check ++ * PRFCNT_SAMPLE_COMPLETED bit. ++ * ++ * Tasklet is needed here since work_queue is too slow and cuased some test ++ * cases timeout, the poll_count variable is introduced to avoid infinite ++ * loop in unexpected cases, the poll_count is 1 or 2 in normal case, 128 ++ * should be big enough to exit the tasklet in abnormal cases. ++ * ++ * Return: void ++ */ ++static void kbasep_hwcnt_irq_poll_tasklet(unsigned long int data) ++{ ++ struct kbase_device *kbdev = (struct kbase_device *)data; ++ unsigned long flags, pm_flags; ++ u32 mcu_gpu_irq_raw_status = 0; ++ u32 poll_count = 0; ++ ++ while (1) { ++ spin_lock_irqsave(&kbdev->hwaccess_lock, pm_flags); ++ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); ++ mcu_gpu_irq_raw_status = kbase_reg_read(kbdev, ++ GPU_CONTROL_MCU_REG(GPU_IRQ_RAWSTAT)); ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); ++ if (mcu_gpu_irq_raw_status & PRFCNT_SAMPLE_COMPLETED) { ++ kbase_reg_write(kbdev, ++ GPU_CONTROL_MCU_REG(GPU_IRQ_CLEAR), ++ PRFCNT_SAMPLE_COMPLETED); ++ kbase_instr_hwcnt_sample_done(kbdev); ++ break; ++ } else if (poll_count++ > 128) { ++ dev_err(kbdev->dev, ++ "Err: HWC dump timeout, count: %u", poll_count); ++ /* Still call sample_done to unblock waiting thread */ ++ kbase_instr_hwcnt_sample_done(kbdev); ++ break; ++ } ++ } ++} ++#endif ++ ++void kbase_instr_hwcnt_sample_done(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); ++ ++ if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_FAULT) { ++ kbdev->hwcnt.backend.triggered = 1; ++ wake_up(&kbdev->hwcnt.backend.wait); ++ } else if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_DUMPING) { ++ if (kbdev->mmu_mode->flags & KBASE_MMU_MODE_HAS_NON_CACHEABLE) { ++ /* All finished and idle */ ++ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; ++ kbdev->hwcnt.backend.triggered = 1; ++ wake_up(&kbdev->hwcnt.backend.wait); ++ } else { ++ int ret; ++ /* Always clean and invalidate the cache after a successful dump ++ */ ++ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_REQUEST_CLEAN; ++ ret = queue_work(kbdev->hwcnt.backend.cache_clean_wq, ++ &kbdev->hwcnt.backend.cache_clean_work); ++ KBASE_DEBUG_ASSERT(ret); ++ } ++ } ++ ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++} ++ ++int kbase_instr_hwcnt_wait_for_dump(struct kbase_context *kctx) ++{ ++ struct kbase_device *kbdev = kctx->kbdev; ++ unsigned long flags; ++ int err; ++ ++ /* Wait for dump & cache clean to complete */ ++ wait_event(kbdev->hwcnt.backend.wait, ++ kbdev->hwcnt.backend.triggered != 0); ++ ++ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); ++ ++ if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_FAULT) { ++ err = -EINVAL; ++ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; ++ } else { ++ /* Dump done */ ++ KBASE_DEBUG_ASSERT(kbdev->hwcnt.backend.state == ++ KBASE_INSTR_STATE_IDLE); ++ err = 0; ++ } ++ ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++ ++ return err; ++} ++ ++int kbase_instr_hwcnt_clear(struct kbase_context *kctx) ++{ ++ unsigned long flags; ++ int err = -EINVAL; ++ struct kbase_device *kbdev = kctx->kbdev; ++ ++ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); ++ ++ /* Check it's the context previously set up and we're not already ++ * dumping */ ++ if (kbdev->hwcnt.kctx != kctx || kbdev->hwcnt.backend.state != ++ KBASE_INSTR_STATE_IDLE) ++ goto out; ++ ++ /* Clear the counters */ ++ KBASE_KTRACE_ADD(kbdev, CORE_GPU_PRFCNT_CLEAR, NULL, 0); ++#if MALI_USE_CSF ++ kbase_reg_write(kbdev, GPU_CONTROL_MCU_REG(GPU_COMMAND), ++ GPU_COMMAND_PRFCNT_CLEAR); ++#else ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), ++ GPU_COMMAND_PRFCNT_CLEAR); ++#endif ++ ++ err = 0; ++ ++out: ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++ return err; ++} ++KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_clear); ++ ++int kbase_instr_backend_init(struct kbase_device *kbdev) ++{ ++ int ret = 0; ++ ++ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DISABLED; ++ ++ init_waitqueue_head(&kbdev->hwcnt.backend.wait); ++ INIT_WORK(&kbdev->hwcnt.backend.cache_clean_work, ++ kbasep_cache_clean_worker); ++ ++#if MALI_USE_CSF ++ tasklet_init(&kbdev->hwcnt.backend.csf_hwc_irq_poll_tasklet, ++ kbasep_hwcnt_irq_poll_tasklet, (unsigned long int)kbdev); ++#endif ++ ++ kbdev->hwcnt.backend.triggered = 0; ++ ++#ifdef CONFIG_MALI_PRFCNT_SET_SECONDARY_VIA_DEBUG_FS ++ kbdev->hwcnt.backend.use_secondary_override = false; ++#endif ++ ++ kbdev->hwcnt.backend.cache_clean_wq = ++ alloc_workqueue("Mali cache cleaning workqueue", 0, 1); ++ if (NULL == kbdev->hwcnt.backend.cache_clean_wq) ++ ret = -EINVAL; ++ ++ return ret; ++} ++ ++void kbase_instr_backend_term(struct kbase_device *kbdev) ++{ ++#if MALI_USE_CSF ++ tasklet_kill(&kbdev->hwcnt.backend.csf_hwc_irq_poll_tasklet); ++#endif ++ destroy_workqueue(kbdev->hwcnt.backend.cache_clean_wq); ++} ++ ++#ifdef CONFIG_MALI_PRFCNT_SET_SECONDARY_VIA_DEBUG_FS ++void kbase_instr_backend_debugfs_init(struct kbase_device *kbdev) ++{ ++ debugfs_create_bool("hwcnt_use_secondary", S_IRUGO | S_IWUSR, ++ kbdev->mali_debugfs_directory, ++ &kbdev->hwcnt.backend.use_secondary_override); ++} ++#endif +diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_instr_defs.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_instr_defs.h +new file mode 100755 +index 000000000000..9f785ce16e17 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_instr_defs.h +@@ -0,0 +1,63 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014, 2016, 2018, 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* ++ * Backend-specific instrumentation definitions ++ */ ++ ++#ifndef _KBASE_INSTR_DEFS_H_ ++#define _KBASE_INSTR_DEFS_H_ ++ ++/* ++ * Instrumentation State Machine States ++ */ ++enum kbase_instr_state { ++ /* State where instrumentation is not active */ ++ KBASE_INSTR_STATE_DISABLED = 0, ++ /* State machine is active and ready for a command. */ ++ KBASE_INSTR_STATE_IDLE, ++ /* Hardware is currently dumping a frame. */ ++ KBASE_INSTR_STATE_DUMPING, ++ /* We've requested a clean to occur on a workqueue */ ++ KBASE_INSTR_STATE_REQUEST_CLEAN, ++ /* An error has occured during DUMPING (page fault). */ ++ KBASE_INSTR_STATE_FAULT ++}; ++ ++/* Structure used for instrumentation and HW counters dumping */ ++struct kbase_instr_backend { ++ wait_queue_head_t wait; ++ int triggered; ++#ifdef CONFIG_MALI_PRFCNT_SET_SECONDARY_VIA_DEBUG_FS ++ bool use_secondary_override; ++#endif ++ ++ enum kbase_instr_state state; ++ struct workqueue_struct *cache_clean_wq; ++ struct work_struct cache_clean_work; ++#if MALI_USE_CSF ++ struct tasklet_struct csf_hwc_irq_poll_tasklet; ++#endif ++}; ++ ++#endif /* _KBASE_INSTR_DEFS_H_ */ ++ +diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_instr_internal.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_instr_internal.h +new file mode 100755 +index 000000000000..2254b9f30d02 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_instr_internal.h +@@ -0,0 +1,44 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014, 2018 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++/* ++ * Backend-specific HW access instrumentation APIs ++ */ ++ ++#ifndef _KBASE_INSTR_INTERNAL_H_ ++#define _KBASE_INSTR_INTERNAL_H_ ++ ++/** ++ * kbasep_cache_clean_worker() - Workqueue for handling cache cleaning ++ * @data: a &struct work_struct ++ */ ++void kbasep_cache_clean_worker(struct work_struct *data); ++ ++/** ++ * kbase_instr_hwcnt_sample_done() - Dump complete interrupt received ++ * @kbdev: Kbase device ++ */ ++void kbase_instr_hwcnt_sample_done(struct kbase_device *kbdev); ++ ++#endif /* _KBASE_INSTR_INTERNAL_H_ */ +diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_irq_internal.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_irq_internal.h +new file mode 100755 +index 000000000000..ca3c048b637a +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_irq_internal.h +@@ -0,0 +1,44 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* ++ * Backend specific IRQ APIs ++ */ ++ ++#ifndef _KBASE_IRQ_INTERNAL_H_ ++#define _KBASE_IRQ_INTERNAL_H_ ++ ++int kbase_install_interrupts(struct kbase_device *kbdev); ++ ++void kbase_release_interrupts(struct kbase_device *kbdev); ++ ++/** ++ * kbase_synchronize_irqs - Ensure that all IRQ handlers have completed ++ * execution ++ * @kbdev: The kbase device ++ */ ++void kbase_synchronize_irqs(struct kbase_device *kbdev); ++ ++int kbasep_common_test_interrupt_handlers( ++ struct kbase_device * const kbdev); ++ ++#endif /* _KBASE_IRQ_INTERNAL_H_ */ +diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_irq_linux.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_irq_linux.c +new file mode 100755 +index 000000000000..b09db552e639 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_irq_linux.c +@@ -0,0 +1,504 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2016,2018-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++#include ++#include ++ ++#include ++ ++#if !defined(CONFIG_MALI_BIFROST_NO_MALI) ++ ++/* GPU IRQ Tags */ ++#define JOB_IRQ_TAG 0 ++#define MMU_IRQ_TAG 1 ++#define GPU_IRQ_TAG 2 ++ ++static void *kbase_tag(void *ptr, u32 tag) ++{ ++ return (void *)(((uintptr_t) ptr) | tag); ++} ++ ++static void *kbase_untag(void *ptr) ++{ ++ return (void *)(((uintptr_t) ptr) & ~3); ++} ++ ++static irqreturn_t kbase_job_irq_handler(int irq, void *data) ++{ ++ unsigned long flags; ++ struct kbase_device *kbdev = kbase_untag(data); ++ u32 val; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ if (!kbdev->pm.backend.gpu_powered) { ++ /* GPU is turned off - IRQ is not for us */ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ return IRQ_NONE; ++ } ++ ++ val = kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_STATUS)); ++ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++ if (!kbdev->pm.backend.driver_ready_for_irqs) ++ dev_warn(kbdev->dev, "%s: irq %d irqstatus 0x%x before driver is ready\n", ++ __func__, irq, val); ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++ ++ if (!val) { ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ return IRQ_NONE; ++ } ++ ++ dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val); ++ ++#if MALI_USE_CSF ++ /* call the csf interrupt handler */ ++ kbase_csf_interrupt(kbdev, val); ++#else ++ kbase_job_done(kbdev, val); ++#endif ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ return IRQ_HANDLED; ++} ++ ++static irqreturn_t kbase_mmu_irq_handler(int irq, void *data) ++{ ++ unsigned long flags; ++ struct kbase_device *kbdev = kbase_untag(data); ++ u32 val; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ if (!kbdev->pm.backend.gpu_powered) { ++ /* GPU is turned off - IRQ is not for us */ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ return IRQ_NONE; ++ } ++ ++ atomic_inc(&kbdev->faults_pending); ++ ++ val = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_STATUS)); ++ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++ if (!kbdev->pm.backend.driver_ready_for_irqs) ++ dev_warn(kbdev->dev, "%s: irq %d irqstatus 0x%x before driver is ready\n", ++ __func__, irq, val); ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ if (!val) { ++ atomic_dec(&kbdev->faults_pending); ++ return IRQ_NONE; ++ } ++ ++ dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val); ++ ++ kbase_mmu_interrupt(kbdev, val); ++ ++ atomic_dec(&kbdev->faults_pending); ++ ++ return IRQ_HANDLED; ++} ++ ++static irqreturn_t kbase_gpu_irq_handler(int irq, void *data) ++{ ++ unsigned long flags; ++ struct kbase_device *kbdev = kbase_untag(data); ++ u32 val; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ if (!kbdev->pm.backend.gpu_powered) { ++ /* GPU is turned off - IRQ is not for us */ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ return IRQ_NONE; ++ } ++ ++ val = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_STATUS)); ++ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++ if (!kbdev->pm.backend.driver_ready_for_irqs) ++ dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x before driver is ready\n", ++ __func__, irq, val); ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ if (!val) ++ return IRQ_NONE; ++ ++ dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val); ++ ++ kbase_gpu_interrupt(kbdev, val); ++ ++ return IRQ_HANDLED; ++} ++ ++static irq_handler_t kbase_handler_table[] = { ++ [JOB_IRQ_TAG] = kbase_job_irq_handler, ++ [MMU_IRQ_TAG] = kbase_mmu_irq_handler, ++ [GPU_IRQ_TAG] = kbase_gpu_irq_handler, ++}; ++ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++#define JOB_IRQ_HANDLER JOB_IRQ_TAG ++#define MMU_IRQ_HANDLER MMU_IRQ_TAG ++#define GPU_IRQ_HANDLER GPU_IRQ_TAG ++ ++/** ++ * kbase_gpu_irq_test_handler - Variant (for test) of kbase_gpu_irq_handler() ++ * @irq: IRQ number ++ * @data: Data associated with this IRQ (i.e. kbdev) ++ * @val: Value of the GPU_CONTROL_REG(GPU_IRQ_STATUS) ++ * ++ * Handle the GPU device interrupt source requests reflected in the ++ * given source bit-pattern. The test code caller is responsible for ++ * undertaking the required device power maintenace. ++ * ++ * Return: IRQ_HANDLED if the requests are from the GPU device, ++ * IRQ_NONE otherwise ++ */ ++irqreturn_t kbase_gpu_irq_test_handler(int irq, void *data, u32 val) ++{ ++ struct kbase_device *kbdev = kbase_untag(data); ++ ++ if (!val) ++ return IRQ_NONE; ++ ++ dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val); ++ ++ kbase_gpu_interrupt(kbdev, val); ++ ++ return IRQ_HANDLED; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_gpu_irq_test_handler); ++ ++/** ++ * kbase_set_custom_irq_handler - Set a custom IRQ handler ++ * @kbdev: Device for which the handler is to be registered ++ * @custom_handler: Handler to be registered ++ * @irq_type: Interrupt type ++ * ++ * Registers given interrupt handler for requested interrupt type ++ * In the case where irq handler is not specified, the default handler shall be ++ * registered ++ * ++ * Return: 0 case success, error code otherwise ++ */ ++int kbase_set_custom_irq_handler(struct kbase_device *kbdev, ++ irq_handler_t custom_handler, ++ int irq_type) ++{ ++ int result = 0; ++ irq_handler_t requested_irq_handler = NULL; ++ ++ KBASE_DEBUG_ASSERT((JOB_IRQ_HANDLER <= irq_type) && ++ (GPU_IRQ_HANDLER >= irq_type)); ++ ++ /* Release previous handler */ ++ if (kbdev->irqs[irq_type].irq) ++ free_irq(kbdev->irqs[irq_type].irq, kbase_tag(kbdev, irq_type)); ++ ++ requested_irq_handler = (NULL != custom_handler) ? custom_handler : ++ kbase_handler_table[irq_type]; ++ ++ if (0 != request_irq(kbdev->irqs[irq_type].irq, ++ requested_irq_handler, ++ kbdev->irqs[irq_type].flags | IRQF_SHARED, ++ dev_name(kbdev->dev), kbase_tag(kbdev, irq_type))) { ++ result = -EINVAL; ++ dev_err(kbdev->dev, "Can't request interrupt %d (index %d)\n", ++ kbdev->irqs[irq_type].irq, irq_type); ++#ifdef CONFIG_SPARSE_IRQ ++ dev_err(kbdev->dev, "You have CONFIG_SPARSE_IRQ support enabled - is the interrupt number correct for this configuration?\n"); ++#endif /* CONFIG_SPARSE_IRQ */ ++ } ++ ++ return result; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_set_custom_irq_handler); ++ ++/* test correct interrupt assigment and reception by cpu */ ++struct kbasep_irq_test { ++ struct hrtimer timer; ++ wait_queue_head_t wait; ++ int triggered; ++ u32 timeout; ++}; ++ ++static struct kbasep_irq_test kbasep_irq_test_data; ++ ++#define IRQ_TEST_TIMEOUT 500 ++ ++static irqreturn_t kbase_job_irq_test_handler(int irq, void *data) ++{ ++ unsigned long flags; ++ struct kbase_device *kbdev = kbase_untag(data); ++ u32 val; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ if (!kbdev->pm.backend.gpu_powered) { ++ /* GPU is turned off - IRQ is not for us */ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ return IRQ_NONE; ++ } ++ ++ val = kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_STATUS)); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ if (!val) ++ return IRQ_NONE; ++ ++ dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val); ++ ++ kbasep_irq_test_data.triggered = 1; ++ wake_up(&kbasep_irq_test_data.wait); ++ ++ kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), val); ++ ++ return IRQ_HANDLED; ++} ++ ++static irqreturn_t kbase_mmu_irq_test_handler(int irq, void *data) ++{ ++ unsigned long flags; ++ struct kbase_device *kbdev = kbase_untag(data); ++ u32 val; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ if (!kbdev->pm.backend.gpu_powered) { ++ /* GPU is turned off - IRQ is not for us */ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ return IRQ_NONE; ++ } ++ ++ val = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_STATUS)); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ if (!val) ++ return IRQ_NONE; ++ ++ dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val); ++ ++ kbasep_irq_test_data.triggered = 1; ++ wake_up(&kbasep_irq_test_data.wait); ++ ++ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), val); ++ ++ return IRQ_HANDLED; ++} ++ ++static enum hrtimer_restart kbasep_test_interrupt_timeout(struct hrtimer *timer) ++{ ++ struct kbasep_irq_test *test_data = container_of(timer, ++ struct kbasep_irq_test, timer); ++ ++ test_data->timeout = 1; ++ test_data->triggered = 1; ++ wake_up(&test_data->wait); ++ return HRTIMER_NORESTART; ++} ++ ++static int kbasep_common_test_interrupt( ++ struct kbase_device * const kbdev, u32 tag) ++{ ++ int err = 0; ++ irq_handler_t test_handler; ++ ++ u32 old_mask_val; ++ u16 mask_offset; ++ u16 rawstat_offset; ++ ++ switch (tag) { ++ case JOB_IRQ_TAG: ++ test_handler = kbase_job_irq_test_handler; ++ rawstat_offset = JOB_CONTROL_REG(JOB_IRQ_RAWSTAT); ++ mask_offset = JOB_CONTROL_REG(JOB_IRQ_MASK); ++ break; ++ case MMU_IRQ_TAG: ++ test_handler = kbase_mmu_irq_test_handler; ++ rawstat_offset = MMU_REG(MMU_IRQ_RAWSTAT); ++ mask_offset = MMU_REG(MMU_IRQ_MASK); ++ break; ++ case GPU_IRQ_TAG: ++ /* already tested by pm_driver - bail out */ ++ default: ++ return 0; ++ } ++ ++ /* store old mask */ ++ old_mask_val = kbase_reg_read(kbdev, mask_offset); ++ /* mask interrupts */ ++ kbase_reg_write(kbdev, mask_offset, 0x0); ++ ++ if (kbdev->irqs[tag].irq) { ++ /* release original handler and install test handler */ ++ if (kbase_set_custom_irq_handler(kbdev, test_handler, tag) != 0) { ++ err = -EINVAL; ++ } else { ++ kbasep_irq_test_data.timeout = 0; ++ hrtimer_init(&kbasep_irq_test_data.timer, ++ CLOCK_MONOTONIC, HRTIMER_MODE_REL); ++ kbasep_irq_test_data.timer.function = ++ kbasep_test_interrupt_timeout; ++ ++ /* trigger interrupt */ ++ kbase_reg_write(kbdev, mask_offset, 0x1); ++ kbase_reg_write(kbdev, rawstat_offset, 0x1); ++ ++ hrtimer_start(&kbasep_irq_test_data.timer, ++ HR_TIMER_DELAY_MSEC(IRQ_TEST_TIMEOUT), ++ HRTIMER_MODE_REL); ++ ++ wait_event(kbasep_irq_test_data.wait, ++ kbasep_irq_test_data.triggered != 0); ++ ++ if (kbasep_irq_test_data.timeout != 0) { ++ dev_err(kbdev->dev, "Interrupt %d (index %d) didn't reach CPU.\n", ++ kbdev->irqs[tag].irq, tag); ++ err = -EINVAL; ++ } else { ++ dev_dbg(kbdev->dev, "Interrupt %d (index %d) reached CPU.\n", ++ kbdev->irqs[tag].irq, tag); ++ } ++ ++ hrtimer_cancel(&kbasep_irq_test_data.timer); ++ kbasep_irq_test_data.triggered = 0; ++ ++ /* mask interrupts */ ++ kbase_reg_write(kbdev, mask_offset, 0x0); ++ ++ /* release test handler */ ++ free_irq(kbdev->irqs[tag].irq, kbase_tag(kbdev, tag)); ++ } ++ ++ /* restore original interrupt */ ++ if (request_irq(kbdev->irqs[tag].irq, kbase_handler_table[tag], ++ kbdev->irqs[tag].flags | IRQF_SHARED, ++ dev_name(kbdev->dev), kbase_tag(kbdev, tag))) { ++ dev_err(kbdev->dev, "Can't restore original interrupt %d (index %d)\n", ++ kbdev->irqs[tag].irq, tag); ++ err = -EINVAL; ++ } ++ } ++ /* restore old mask */ ++ kbase_reg_write(kbdev, mask_offset, old_mask_val); ++ ++ return err; ++} ++ ++int kbasep_common_test_interrupt_handlers( ++ struct kbase_device * const kbdev) ++{ ++ int err; ++ ++ init_waitqueue_head(&kbasep_irq_test_data.wait); ++ kbasep_irq_test_data.triggered = 0; ++ ++ /* A suspend won't happen during startup/insmod */ ++ kbase_pm_context_active(kbdev); ++ ++ err = kbasep_common_test_interrupt(kbdev, JOB_IRQ_TAG); ++ if (err) { ++ dev_err(kbdev->dev, "Interrupt JOB_IRQ didn't reach CPU. Check interrupt assignments.\n"); ++ goto out; ++ } ++ ++ err = kbasep_common_test_interrupt(kbdev, MMU_IRQ_TAG); ++ if (err) { ++ dev_err(kbdev->dev, "Interrupt MMU_IRQ didn't reach CPU. Check interrupt assignments.\n"); ++ goto out; ++ } ++ ++ dev_dbg(kbdev->dev, "Interrupts are correctly assigned.\n"); ++ ++ out: ++ kbase_pm_context_idle(kbdev); ++ ++ return err; ++} ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++ ++int kbase_install_interrupts(struct kbase_device *kbdev) ++{ ++ u32 nr = ARRAY_SIZE(kbase_handler_table); ++ int err; ++ u32 i; ++ ++ for (i = 0; i < nr; i++) { ++ err = request_irq(kbdev->irqs[i].irq, kbase_handler_table[i], ++ kbdev->irqs[i].flags | IRQF_SHARED, ++ dev_name(kbdev->dev), ++ kbase_tag(kbdev, i)); ++ if (err) { ++ dev_err(kbdev->dev, "Can't request interrupt %d (index %d)\n", ++ kbdev->irqs[i].irq, i); ++#ifdef CONFIG_SPARSE_IRQ ++ dev_err(kbdev->dev, "You have CONFIG_SPARSE_IRQ support enabled - is the interrupt number correct for this configuration?\n"); ++#endif /* CONFIG_SPARSE_IRQ */ ++ goto release; ++ } ++ } ++ ++ return 0; ++ ++ release: ++ while (i-- > 0) ++ free_irq(kbdev->irqs[i].irq, kbase_tag(kbdev, i)); ++ ++ return err; ++} ++ ++void kbase_release_interrupts(struct kbase_device *kbdev) ++{ ++ u32 nr = ARRAY_SIZE(kbase_handler_table); ++ u32 i; ++ ++ for (i = 0; i < nr; i++) { ++ if (kbdev->irqs[i].irq) ++ free_irq(kbdev->irqs[i].irq, kbase_tag(kbdev, i)); ++ } ++} ++ ++void kbase_synchronize_irqs(struct kbase_device *kbdev) ++{ ++ u32 nr = ARRAY_SIZE(kbase_handler_table); ++ u32 i; ++ ++ for (i = 0; i < nr; i++) { ++ if (kbdev->irqs[i].irq) ++ synchronize_irq(kbdev->irqs[i].irq); ++ } ++} ++ ++KBASE_EXPORT_TEST_API(kbase_synchronize_irqs); ++ ++#endif /* !defined(CONFIG_MALI_BIFROST_NO_MALI) */ +diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_as.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_as.c +new file mode 100755 +index 000000000000..9b775898dac2 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_as.c +@@ -0,0 +1,245 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++/* ++ * Register backend context / address space management ++ */ ++ ++#include ++#include ++#include ++ ++/** ++ * assign_and_activate_kctx_addr_space - Assign an AS to a context ++ * @kbdev: Kbase device ++ * @kctx: Kbase context ++ * @current_as: Address Space to assign ++ * ++ * Assign an Address Space (AS) to a context, and add the context to the Policy. ++ * ++ * This includes ++ * setting up the global runpool_irq structure and the context on the AS, ++ * Activating the MMU on the AS, ++ * Allowing jobs to be submitted on the AS. ++ * ++ * Context: ++ * kbasep_js_kctx_info.jsctx_mutex held, ++ * kbasep_js_device_data.runpool_mutex held, ++ * AS transaction mutex held, ++ * Runpool IRQ lock held ++ */ ++static void assign_and_activate_kctx_addr_space(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ struct kbase_as *current_as) ++{ ++ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; ++ ++ lockdep_assert_held(&kctx->jctx.sched_info.ctx.jsctx_mutex); ++ lockdep_assert_held(&js_devdata->runpool_mutex); ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++#if !MALI_USE_CSF ++ /* Attribute handling */ ++ kbasep_js_ctx_attr_runpool_retain_ctx(kbdev, kctx); ++#endif ++ ++ /* Allow it to run jobs */ ++ kbasep_js_set_submit_allowed(js_devdata, kctx); ++ ++ kbase_js_runpool_inc_context_count(kbdev, kctx); ++} ++ ++bool kbase_backend_use_ctx_sched(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ int js) ++{ ++ int i; ++ ++ if (kbdev->hwaccess.active_kctx[js] == kctx) { ++ /* Context is already active */ ++ return true; ++ } ++ ++ for (i = 0; i < kbdev->nr_hw_address_spaces; i++) { ++ if (kbdev->as_to_kctx[i] == kctx) { ++ /* Context already has ASID - mark as active */ ++ return true; ++ } ++ } ++ ++ /* Context does not have address space assigned */ ++ return false; ++} ++ ++void kbase_backend_release_ctx_irq(struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++ int as_nr = kctx->as_nr; ++ ++ if (as_nr == KBASEP_AS_NR_INVALID) { ++ WARN(1, "Attempting to release context without ASID\n"); ++ return; ++ } ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (atomic_read(&kctx->refcount) != 1) { ++ WARN(1, "Attempting to release active ASID\n"); ++ return; ++ } ++ ++ kbasep_js_clear_submit_allowed(&kbdev->js_data, kctx); ++ ++ kbase_ctx_sched_release_ctx(kctx); ++ kbase_js_runpool_dec_context_count(kbdev, kctx); ++} ++ ++void kbase_backend_release_ctx_noirq(struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++} ++ ++int kbase_backend_find_and_release_free_address_space( ++ struct kbase_device *kbdev, struct kbase_context *kctx) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ unsigned long flags; ++ int i; ++ ++ js_devdata = &kbdev->js_data; ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); ++ mutex_lock(&js_devdata->runpool_mutex); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ for (i = 0; i < kbdev->nr_hw_address_spaces; i++) { ++ struct kbasep_js_kctx_info *as_js_kctx_info; ++ struct kbase_context *as_kctx; ++ ++ as_kctx = kbdev->as_to_kctx[i]; ++ as_js_kctx_info = &as_kctx->jctx.sched_info; ++ ++ /* Don't release privileged or active contexts, or contexts with ++ * jobs running. ++ * Note that a context will have at least 1 reference (which ++ * was previously taken by kbasep_js_schedule_ctx()) until ++ * descheduled. ++ */ ++ if (as_kctx && !kbase_ctx_flag(as_kctx, KCTX_PRIVILEGED) && ++ atomic_read(&as_kctx->refcount) == 1) { ++ if (!kbase_ctx_sched_inc_refcount_nolock(as_kctx)) { ++ WARN(1, "Failed to retain active context\n"); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, ++ flags); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ ++ return KBASEP_AS_NR_INVALID; ++ } ++ ++ kbasep_js_clear_submit_allowed(js_devdata, as_kctx); ++ ++ /* Drop and retake locks to take the jsctx_mutex on the ++ * context we're about to release without violating lock ++ * ordering ++ */ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ ++ ++ /* Release context from address space */ ++ mutex_lock(&as_js_kctx_info->ctx.jsctx_mutex); ++ mutex_lock(&js_devdata->runpool_mutex); ++ ++ kbasep_js_runpool_release_ctx_nolock(kbdev, as_kctx); ++ ++ if (!kbase_ctx_flag(as_kctx, KCTX_SCHEDULED)) { ++ kbasep_js_runpool_requeue_or_kill_ctx(kbdev, ++ as_kctx, ++ true); ++ ++ mutex_unlock(&js_devdata->runpool_mutex); ++ mutex_unlock(&as_js_kctx_info->ctx.jsctx_mutex); ++ ++ return i; ++ } ++ ++ /* Context was retained while locks were dropped, ++ * continue looking for free AS */ ++ ++ mutex_unlock(&js_devdata->runpool_mutex); ++ mutex_unlock(&as_js_kctx_info->ctx.jsctx_mutex); ++ ++ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); ++ mutex_lock(&js_devdata->runpool_mutex); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ } ++ } ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ mutex_unlock(&js_devdata->runpool_mutex); ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ ++ return KBASEP_AS_NR_INVALID; ++} ++ ++bool kbase_backend_use_ctx(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ int as_nr) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ struct kbase_as *new_address_space = NULL; ++ int js; ++ ++ js_devdata = &kbdev->js_data; ++ ++ for (js = 0; js < BASE_JM_MAX_NR_SLOTS; js++) { ++ if (kbdev->hwaccess.active_kctx[js] == kctx) { ++ WARN(1, "Context is already scheduled in\n"); ++ return false; ++ } ++ } ++ ++ new_address_space = &kbdev->as[as_nr]; ++ ++ lockdep_assert_held(&js_devdata->runpool_mutex); ++ lockdep_assert_held(&kbdev->mmu_hw_mutex); ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ assign_and_activate_kctx_addr_space(kbdev, kctx, new_address_space); ++ ++ if (kbase_ctx_flag(kctx, KCTX_PRIVILEGED)) { ++ /* We need to retain it to keep the corresponding address space ++ */ ++ kbase_ctx_sched_retain_ctx_refcount(kctx); ++ } ++ ++ return true; ++} ++ +diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_defs.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_defs.h +new file mode 100755 +index 000000000000..9cccf224999e +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_defs.h +@@ -0,0 +1,113 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2016, 2018-2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++/* ++ * Register-based HW access backend specific definitions ++ */ ++ ++#ifndef _KBASE_HWACCESS_GPU_DEFS_H_ ++#define _KBASE_HWACCESS_GPU_DEFS_H_ ++ ++/* SLOT_RB_SIZE must be < 256 */ ++#define SLOT_RB_SIZE 2 ++#define SLOT_RB_MASK (SLOT_RB_SIZE - 1) ++ ++/** ++ * struct rb_entry - Ringbuffer entry ++ * @katom: Atom associated with this entry ++ */ ++struct rb_entry { ++ struct kbase_jd_atom *katom; ++}; ++ ++/** ++ * struct slot_rb - Slot ringbuffer ++ * @entries: Ringbuffer entries ++ * @last_context: The last context to submit a job on this slot ++ * @read_idx: Current read index of buffer ++ * @write_idx: Current write index of buffer ++ * @job_chain_flag: Flag used to implement jobchain disambiguation ++ */ ++struct slot_rb { ++ struct rb_entry entries[SLOT_RB_SIZE]; ++ ++ struct kbase_context *last_context; ++ ++ u8 read_idx; ++ u8 write_idx; ++ ++ u8 job_chain_flag; ++}; ++ ++/** ++ * struct kbase_backend_data - GPU backend specific data for HW access layer ++ * @slot_rb: Slot ringbuffers ++ * @scheduling_timer: The timer tick used for rescheduling jobs ++ * @timer_running: Is the timer running? The runpool_mutex must be ++ * held whilst modifying this. ++ * @suspend_timer: Is the timer suspended? Set when a suspend ++ * occurs and cleared on resume. The runpool_mutex ++ * must be held whilst modifying this. ++ * @reset_gpu: Set to a KBASE_RESET_xxx value (see comments) ++ * @reset_workq: Work queue for performing the reset ++ * @reset_work: Work item for performing the reset ++ * @reset_wait: Wait event signalled when the reset is complete ++ * @reset_timer: Timeout for soft-stops before the reset ++ * @timeouts_updated: Have timeout values just been updated? ++ * ++ * The hwaccess_lock (a spinlock) must be held when accessing this structure ++ */ ++struct kbase_backend_data { ++ struct slot_rb slot_rb[BASE_JM_MAX_NR_SLOTS]; ++ ++#if !MALI_USE_CSF ++ struct hrtimer scheduling_timer; ++ ++ bool timer_running; ++#endif ++ bool suspend_timer; ++ ++ atomic_t reset_gpu; ++ ++/* The GPU reset isn't pending */ ++#define KBASE_RESET_GPU_NOT_PENDING 0 ++/* kbase_prepare_to_reset_gpu has been called */ ++#define KBASE_RESET_GPU_PREPARED 1 ++/* kbase_reset_gpu has been called - the reset will now definitely happen ++ * within the timeout period */ ++#define KBASE_RESET_GPU_COMMITTED 2 ++/* The GPU reset process is currently occuring (timeout has expired or ++ * kbasep_try_reset_gpu_early was called) */ ++#define KBASE_RESET_GPU_HAPPENING 3 ++/* Reset the GPU silently, used when resetting the GPU as part of normal ++ * behavior (e.g. when exiting protected mode). */ ++#define KBASE_RESET_GPU_SILENT 4 ++ struct workqueue_struct *reset_workq; ++ struct work_struct reset_work; ++ wait_queue_head_t reset_wait; ++ struct hrtimer reset_timer; ++ ++ bool timeouts_updated; ++}; ++ ++#endif /* _KBASE_HWACCESS_GPU_DEFS_H_ */ +diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_hw.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_hw.c +new file mode 100755 +index 000000000000..19661c9766c6 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_hw.c +@@ -0,0 +1,1462 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* ++ * Base kernel job manager APIs ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++static void kbasep_try_reset_gpu_early_locked(struct kbase_device *kbdev); ++ ++static u64 kbase_job_write_affinity(struct kbase_device *kbdev, ++ base_jd_core_req core_req, ++ int js) ++{ ++ u64 affinity; ++ ++ if ((core_req & (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T)) == ++ BASE_JD_REQ_T) { ++ /* Tiler-only atom */ ++ /* If the hardware supports XAFFINITY then we'll only enable ++ * the tiler (which is the default so this is a no-op), ++ * otherwise enable shader core 0. ++ */ ++ if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_XAFFINITY)) ++ affinity = 1; ++ else ++ affinity = 0; ++ } else if ((core_req & (BASE_JD_REQ_COHERENT_GROUP | ++ BASE_JD_REQ_SPECIFIC_COHERENT_GROUP))) { ++ unsigned int num_core_groups = kbdev->gpu_props.num_core_groups; ++ struct mali_base_gpu_coherent_group_info *coherency_info = ++ &kbdev->gpu_props.props.coherency_info; ++ ++ affinity = kbdev->pm.backend.shaders_avail & ++ kbdev->pm.debug_core_mask[js]; ++ ++ /* JS2 on a dual core group system targets core group 1. All ++ * other cases target core group 0. ++ */ ++ if (js == 2 && num_core_groups > 1) ++ affinity &= coherency_info->group[1].core_mask; ++ else ++ affinity &= coherency_info->group[0].core_mask; ++ } else { ++ /* Use all cores */ ++ affinity = kbdev->pm.backend.shaders_avail & ++ kbdev->pm.debug_core_mask[js]; ++ } ++ ++ if (unlikely(!affinity)) { ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++ u64 shaders_ready = ++ kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER); ++ ++ WARN_ON(!(shaders_ready & kbdev->pm.backend.shaders_avail)); ++#endif ++ ++ affinity = kbdev->pm.backend.shaders_avail; ++ } ++ ++ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_AFFINITY_NEXT_LO), ++ affinity & 0xFFFFFFFF); ++ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_AFFINITY_NEXT_HI), ++ affinity >> 32); ++ ++ return affinity; ++} ++ ++/** ++ * select_job_chain() - Select which job chain to submit to the GPU ++ * @katom: Pointer to the atom about to be submitted to the GPU ++ * ++ * Selects one of the fragment job chains attached to the special atom at the ++ * end of a renderpass, or returns the address of the single job chain attached ++ * to any other type of atom. ++ * ++ * Which job chain is selected depends upon whether the tiling phase of the ++ * renderpass completed normally or was soft-stopped because it used too ++ * much memory. It also depends upon whether one of the fragment job chains ++ * has already been run as part of the same renderpass. ++ * ++ * Return: GPU virtual address of the selected job chain ++ */ ++static u64 select_job_chain(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *const kctx = katom->kctx; ++ u64 jc = katom->jc; ++ struct kbase_jd_renderpass *rp; ++ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ ++ if (!(katom->core_req & BASE_JD_REQ_END_RENDERPASS)) ++ return jc; ++ ++ compiletime_assert((1ull << (sizeof(katom->renderpass_id) * 8)) <= ++ ARRAY_SIZE(kctx->jctx.renderpasses), ++ "Should check invalid access to renderpasses"); ++ ++ rp = &kctx->jctx.renderpasses[katom->renderpass_id]; ++ /* We can read a subset of renderpass state without holding ++ * higher-level locks (but not end_katom, for example). ++ * If the end-of-renderpass atom is running with as-yet indeterminate ++ * OOM state then assume that the start atom was not soft-stopped. ++ */ ++ switch (rp->state) { ++ case KBASE_JD_RP_OOM: ++ /* Tiling ran out of memory. ++ * Start of incremental rendering, used once. ++ */ ++ jc = katom->jc_fragment.norm_read_forced_write; ++ break; ++ case KBASE_JD_RP_START: ++ case KBASE_JD_RP_PEND_OOM: ++ /* Tiling completed successfully first time. ++ * Single-iteration rendering, used once. ++ */ ++ jc = katom->jc_fragment.norm_read_norm_write; ++ break; ++ case KBASE_JD_RP_RETRY_OOM: ++ /* Tiling ran out of memory again. ++ * Continuation of incremental rendering, used as ++ * many times as required. ++ */ ++ jc = katom->jc_fragment.forced_read_forced_write; ++ break; ++ case KBASE_JD_RP_RETRY: ++ case KBASE_JD_RP_RETRY_PEND_OOM: ++ /* Tiling completed successfully this time. ++ * End of incremental rendering, used once. ++ */ ++ jc = katom->jc_fragment.forced_read_norm_write; ++ break; ++ default: ++ WARN_ON(1); ++ break; ++ } ++ ++ dev_dbg(kctx->kbdev->dev, ++ "Selected job chain 0x%llx for end atom %p in state %d\n", ++ jc, (void *)katom, (int)rp->state); ++ ++ katom->jc = jc; ++ return jc; ++} ++ ++void kbase_job_hw_submit(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom, ++ int js) ++{ ++ struct kbase_context *kctx; ++ u32 cfg; ++ u64 const jc_head = select_job_chain(katom); ++ u64 affinity; ++ ++ KBASE_DEBUG_ASSERT(kbdev); ++ KBASE_DEBUG_ASSERT(katom); ++ ++ kctx = katom->kctx; ++ ++ /* Command register must be available */ ++ KBASE_DEBUG_ASSERT(kbasep_jm_is_js_free(kbdev, js, kctx)); ++ ++ dev_dbg(kctx->kbdev->dev, "Write JS_HEAD_NEXT 0x%llx for atom %p\n", ++ jc_head, (void *)katom); ++ ++ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_LO), ++ jc_head & 0xFFFFFFFF); ++ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_HI), ++ jc_head >> 32); ++ ++ affinity = kbase_job_write_affinity(kbdev, katom->core_req, js); ++ ++ /* start MMU, medium priority, cache clean/flush on end, clean/flush on ++ * start */ ++ cfg = kctx->as_nr; ++ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_FLUSH_REDUCTION) && ++ !(kbdev->serialize_jobs & KBASE_SERIALIZE_RESET)) ++ cfg |= JS_CONFIG_ENABLE_FLUSH_REDUCTION; ++ ++ if (0 != (katom->core_req & BASE_JD_REQ_SKIP_CACHE_START)) ++ cfg |= JS_CONFIG_START_FLUSH_NO_ACTION; ++ else ++ cfg |= JS_CONFIG_START_FLUSH_CLEAN_INVALIDATE; ++ ++ if (0 != (katom->core_req & BASE_JD_REQ_SKIP_CACHE_END) && ++ !(kbdev->serialize_jobs & KBASE_SERIALIZE_RESET)) ++ cfg |= JS_CONFIG_END_FLUSH_NO_ACTION; ++ else if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_CLEAN_ONLY_SAFE)) ++ cfg |= JS_CONFIG_END_FLUSH_CLEAN; ++ else ++ cfg |= JS_CONFIG_END_FLUSH_CLEAN_INVALIDATE; ++ ++ cfg |= JS_CONFIG_THREAD_PRI(8); ++ ++ if ((katom->atom_flags & KBASE_KATOM_FLAG_PROTECTED) || ++ (katom->core_req & BASE_JD_REQ_END_RENDERPASS)) ++ cfg |= JS_CONFIG_DISABLE_DESCRIPTOR_WR_BK; ++ ++ if (kbase_hw_has_feature(kbdev, ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) { ++ if (!kbdev->hwaccess.backend.slot_rb[js].job_chain_flag) { ++ cfg |= JS_CONFIG_JOB_CHAIN_FLAG; ++ katom->atom_flags |= KBASE_KATOM_FLAGS_JOBCHAIN; ++ kbdev->hwaccess.backend.slot_rb[js].job_chain_flag = ++ true; ++ } else { ++ katom->atom_flags &= ~KBASE_KATOM_FLAGS_JOBCHAIN; ++ kbdev->hwaccess.backend.slot_rb[js].job_chain_flag = ++ false; ++ } ++ } ++ ++ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_CONFIG_NEXT), cfg); ++ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_FLUSH_REDUCTION)) ++ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_FLUSH_ID_NEXT), ++ katom->flush_id); ++ ++ /* Write an approximate start timestamp. ++ * It's approximate because there might be a job in the HEAD register. ++ */ ++ katom->start_timestamp = ktime_get(); ++ ++ /* GO ! */ ++ dev_dbg(kbdev->dev, "JS: Submitting atom %p from ctx %p to js[%d] with head=0x%llx", ++ katom, kctx, js, jc_head); ++ ++ KBASE_KTRACE_ADD_JM_SLOT_INFO(kbdev, JM_SUBMIT, kctx, katom, jc_head, js, ++ (u32)affinity); ++ ++ KBASE_TLSTREAM_AUX_EVENT_JOB_SLOT(kbdev, kctx, ++ js, kbase_jd_atom_id(kctx, katom), TL_JS_EVENT_START); ++ ++ KBASE_TLSTREAM_TL_ATTRIB_ATOM_CONFIG(kbdev, katom, jc_head, ++ affinity, cfg); ++ KBASE_TLSTREAM_TL_RET_CTX_LPU( ++ kbdev, ++ kctx, ++ &kbdev->gpu_props.props.raw_props.js_features[ ++ katom->slot_nr]); ++ KBASE_TLSTREAM_TL_RET_ATOM_AS(kbdev, katom, &kbdev->as[kctx->as_nr]); ++ KBASE_TLSTREAM_TL_RET_ATOM_LPU( ++ kbdev, ++ katom, ++ &kbdev->gpu_props.props.raw_props.js_features[js], ++ "ctx_nr,atom_nr"); ++ kbase_kinstr_jm_atom_hw_submit(katom); ++#ifdef CONFIG_GPU_TRACEPOINTS ++ if (!kbase_backend_nr_atoms_submitted(kbdev, js)) { ++ /* If this is the only job on the slot, trace it as starting */ ++ char js_string[16]; ++ ++ trace_gpu_sched_switch( ++ kbasep_make_job_slot_string(js, js_string, ++ sizeof(js_string)), ++ ktime_to_ns(katom->start_timestamp), ++ (u32)katom->kctx->id, 0, katom->work_id); ++ kbdev->hwaccess.backend.slot_rb[js].last_context = katom->kctx; ++ } ++#endif ++ ++ trace_sysgraph_gpu(SGR_SUBMIT, kctx->id, ++ kbase_jd_atom_id(kctx, katom), js); ++ ++ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT), ++ JS_COMMAND_START); ++} ++ ++/** ++ * kbasep_job_slot_update_head_start_timestamp - Update timestamp ++ * @kbdev: kbase device ++ * @js: job slot ++ * @end_timestamp: timestamp ++ * ++ * Update the start_timestamp of the job currently in the HEAD, based on the ++ * fact that we got an IRQ for the previous set of completed jobs. ++ * ++ * The estimate also takes into account the time the job was submitted, to ++ * work out the best estimate (which might still result in an over-estimate to ++ * the calculated time spent) ++ */ ++static void kbasep_job_slot_update_head_start_timestamp( ++ struct kbase_device *kbdev, ++ int js, ++ ktime_t end_timestamp) ++{ ++ ktime_t timestamp_diff; ++ struct kbase_jd_atom *katom; ++ ++ /* Checking the HEAD position for the job slot */ ++ katom = kbase_gpu_inspect(kbdev, js, 0); ++ if (katom != NULL) { ++ timestamp_diff = ktime_sub(end_timestamp, ++ katom->start_timestamp); ++ if (ktime_to_ns(timestamp_diff) >= 0) { ++ /* Only update the timestamp if it's a better estimate ++ * than what's currently stored. This is because our ++ * estimate that accounts for the throttle time may be ++ * too much of an overestimate */ ++ katom->start_timestamp = end_timestamp; ++ } ++ } ++} ++ ++/** ++ * kbasep_trace_tl_event_lpu_softstop - Call event_lpu_softstop timeline ++ * tracepoint ++ * @kbdev: kbase device ++ * @js: job slot ++ * ++ * Make a tracepoint call to the instrumentation module informing that ++ * softstop happened on given lpu (job slot). ++ */ ++static void kbasep_trace_tl_event_lpu_softstop(struct kbase_device *kbdev, ++ int js) ++{ ++ KBASE_TLSTREAM_TL_EVENT_LPU_SOFTSTOP( ++ kbdev, ++ &kbdev->gpu_props.props.raw_props.js_features[js]); ++} ++ ++void kbase_job_done(struct kbase_device *kbdev, u32 done) ++{ ++ int i; ++ u32 count = 0; ++ ktime_t end_timestamp; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ KBASE_DEBUG_ASSERT(kbdev); ++ ++ KBASE_KTRACE_ADD_JM(kbdev, JM_IRQ, NULL, NULL, 0, done); ++ ++ end_timestamp = ktime_get(); ++ ++ while (done) { ++ u32 failed = done >> 16; ++ ++ /* treat failed slots as finished slots */ ++ u32 finished = (done & 0xFFFF) | failed; ++ ++ /* Note: This is inherently unfair, as we always check ++ * for lower numbered interrupts before the higher ++ * numbered ones.*/ ++ i = ffs(finished) - 1; ++ KBASE_DEBUG_ASSERT(i >= 0); ++ ++ do { ++ int nr_done; ++ u32 active; ++ u32 completion_code = BASE_JD_EVENT_DONE;/* assume OK */ ++ u64 job_tail = 0; ++ ++ if (failed & (1u << i)) { ++ /* read out the job slot status code if the job ++ * slot reported failure */ ++ completion_code = kbase_reg_read(kbdev, ++ JOB_SLOT_REG(i, JS_STATUS)); ++ ++ if (completion_code == BASE_JD_EVENT_STOPPED) { ++ KBASE_TLSTREAM_AUX_EVENT_JOB_SLOT( ++ kbdev, NULL, ++ i, 0, TL_JS_EVENT_SOFT_STOP); ++ ++ kbasep_trace_tl_event_lpu_softstop( ++ kbdev, i); ++ ++ /* Soft-stopped job - read the value of ++ * JS_TAIL so that the job chain can ++ * be resumed */ ++ job_tail = (u64)kbase_reg_read(kbdev, ++ JOB_SLOT_REG(i, JS_TAIL_LO)) | ++ ((u64)kbase_reg_read(kbdev, ++ JOB_SLOT_REG(i, JS_TAIL_HI)) ++ << 32); ++ } else if (completion_code == ++ BASE_JD_EVENT_NOT_STARTED) { ++ /* PRLAM-10673 can cause a TERMINATED ++ * job to come back as NOT_STARTED, but ++ * the error interrupt helps us detect ++ * it */ ++ completion_code = ++ BASE_JD_EVENT_TERMINATED; ++ } ++ ++ kbase_gpu_irq_evict(kbdev, i, completion_code); ++ ++ /* Some jobs that encounter a BUS FAULT may result in corrupted ++ * state causing future jobs to hang. Reset GPU before ++ * allowing any other jobs on the slot to continue. */ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_3076)) { ++ if (completion_code == BASE_JD_EVENT_JOB_BUS_FAULT) { ++ if (kbase_prepare_to_reset_gpu_locked(kbdev)) ++ kbase_reset_gpu_locked(kbdev); ++ } ++ } ++ } ++ ++ kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), ++ done & ((1 << i) | (1 << (i + 16)))); ++ active = kbase_reg_read(kbdev, ++ JOB_CONTROL_REG(JOB_IRQ_JS_STATE)); ++ ++ if (((active >> i) & 1) == 0 && ++ (((done >> (i + 16)) & 1) == 0)) { ++ /* There is a potential race we must work ++ * around: ++ * ++ * 1. A job slot has a job in both current and ++ * next registers ++ * 2. The job in current completes ++ * successfully, the IRQ handler reads ++ * RAWSTAT and calls this function with the ++ * relevant bit set in "done" ++ * 3. The job in the next registers becomes the ++ * current job on the GPU ++ * 4. Sometime before the JOB_IRQ_CLEAR line ++ * above the job on the GPU _fails_ ++ * 5. The IRQ_CLEAR clears the done bit but not ++ * the failed bit. This atomically sets ++ * JOB_IRQ_JS_STATE. However since both jobs ++ * have now completed the relevant bits for ++ * the slot are set to 0. ++ * ++ * If we now did nothing then we'd incorrectly ++ * assume that _both_ jobs had completed ++ * successfully (since we haven't yet observed ++ * the fail bit being set in RAWSTAT). ++ * ++ * So at this point if there are no active jobs ++ * left we check to see if RAWSTAT has a failure ++ * bit set for the job slot. If it does we know ++ * that there has been a new failure that we ++ * didn't previously know about, so we make sure ++ * that we record this in active (but we wait ++ * for the next loop to deal with it). ++ * ++ * If we were handling a job failure (i.e. done ++ * has the relevant high bit set) then we know ++ * that the value read back from ++ * JOB_IRQ_JS_STATE is the correct number of ++ * remaining jobs because the failed job will ++ * have prevented any futher jobs from starting ++ * execution. ++ */ ++ u32 rawstat = kbase_reg_read(kbdev, ++ JOB_CONTROL_REG(JOB_IRQ_RAWSTAT)); ++ ++ if ((rawstat >> (i + 16)) & 1) { ++ /* There is a failed job that we've ++ * missed - add it back to active */ ++ active |= (1u << i); ++ } ++ } ++ ++ dev_dbg(kbdev->dev, "Job ended with status 0x%08X\n", ++ completion_code); ++ ++ nr_done = kbase_backend_nr_atoms_submitted(kbdev, i); ++ nr_done -= (active >> i) & 1; ++ nr_done -= (active >> (i + 16)) & 1; ++ ++ if (nr_done <= 0) { ++ dev_warn(kbdev->dev, "Spurious interrupt on slot %d", ++ i); ++ ++ goto spurious; ++ } ++ ++ count += nr_done; ++ ++ while (nr_done) { ++ if (nr_done == 1) { ++ kbase_gpu_complete_hw(kbdev, i, ++ completion_code, ++ job_tail, ++ &end_timestamp); ++ kbase_jm_try_kick_all(kbdev); ++ } else { ++ /* More than one job has completed. ++ * Since this is not the last job being ++ * reported this time it must have ++ * passed. This is because the hardware ++ * will not allow further jobs in a job ++ * slot to complete until the failed job ++ * is cleared from the IRQ status. ++ */ ++ kbase_gpu_complete_hw(kbdev, i, ++ BASE_JD_EVENT_DONE, ++ 0, ++ &end_timestamp); ++ } ++ nr_done--; ++ } ++ spurious: ++ done = kbase_reg_read(kbdev, ++ JOB_CONTROL_REG(JOB_IRQ_RAWSTAT)); ++ ++ failed = done >> 16; ++ finished = (done & 0xFFFF) | failed; ++ if (done) ++ end_timestamp = ktime_get(); ++ } while (finished & (1 << i)); ++ ++ kbasep_job_slot_update_head_start_timestamp(kbdev, i, ++ end_timestamp); ++ } ++ ++ if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) == ++ KBASE_RESET_GPU_COMMITTED) { ++ /* If we're trying to reset the GPU then we might be able to do ++ * it early (without waiting for a timeout) because some jobs ++ * have completed ++ */ ++ kbasep_try_reset_gpu_early_locked(kbdev); ++ } ++ KBASE_KTRACE_ADD_JM(kbdev, JM_IRQ_END, NULL, NULL, 0, count); ++} ++ ++void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev, ++ int js, ++ u32 action, ++ base_jd_core_req core_reqs, ++ struct kbase_jd_atom *target_katom) ++{ ++#if KBASE_KTRACE_ENABLE ++ u32 status_reg_before; ++ u64 job_in_head_before; ++ u32 status_reg_after; ++ ++ KBASE_DEBUG_ASSERT(!(action & (~JS_COMMAND_MASK))); ++ ++ /* Check the head pointer */ ++ job_in_head_before = ((u64) kbase_reg_read(kbdev, ++ JOB_SLOT_REG(js, JS_HEAD_LO))) ++ | (((u64) kbase_reg_read(kbdev, ++ JOB_SLOT_REG(js, JS_HEAD_HI))) ++ << 32); ++ status_reg_before = kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_STATUS)); ++#endif ++ ++ if (action == JS_COMMAND_SOFT_STOP) { ++ if (kbase_jd_katom_is_protected(target_katom)) { ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++ dev_dbg(kbdev->dev, ++ "Attempt made to soft-stop a job that cannot be soft-stopped. core_reqs = 0x%x", ++ (unsigned int)core_reqs); ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++ return; ++ } ++ ++ /* We are about to issue a soft stop, so mark the atom as having ++ * been soft stopped */ ++ target_katom->atom_flags |= KBASE_KATOM_FLAG_BEEN_SOFT_STOPPED; ++ ++ /* Mark the point where we issue the soft-stop command */ ++ KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_ISSUE(kbdev, target_katom); ++ ++ if (kbase_hw_has_feature( ++ kbdev, ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) { ++ action = (target_katom->atom_flags & ++ KBASE_KATOM_FLAGS_JOBCHAIN) ? ++ JS_COMMAND_SOFT_STOP_1 : ++ JS_COMMAND_SOFT_STOP_0; ++ } ++ } else if (action == JS_COMMAND_HARD_STOP) { ++ target_katom->atom_flags |= KBASE_KATOM_FLAG_BEEN_HARD_STOPPED; ++ ++ if (kbase_hw_has_feature( ++ kbdev, ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) { ++ action = (target_katom->atom_flags & ++ KBASE_KATOM_FLAGS_JOBCHAIN) ? ++ JS_COMMAND_HARD_STOP_1 : ++ JS_COMMAND_HARD_STOP_0; ++ } ++ } ++ ++ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND), action); ++ ++#if KBASE_KTRACE_ENABLE ++ status_reg_after = kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_STATUS)); ++ if (status_reg_after == BASE_JD_EVENT_ACTIVE) { ++ struct kbase_jd_atom *head; ++ struct kbase_context *head_kctx; ++ ++ head = kbase_gpu_inspect(kbdev, js, 0); ++ head_kctx = head->kctx; ++ ++ if (status_reg_before == BASE_JD_EVENT_ACTIVE) ++ KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_CHECK_HEAD, head_kctx, head, job_in_head_before, js); ++ else ++ KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, 0, js); ++ ++ switch (action) { ++ case JS_COMMAND_SOFT_STOP: ++ KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_SOFTSTOP, head_kctx, head, head->jc, js); ++ break; ++ case JS_COMMAND_SOFT_STOP_0: ++ KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_SOFTSTOP_0, head_kctx, head, head->jc, js); ++ break; ++ case JS_COMMAND_SOFT_STOP_1: ++ KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_SOFTSTOP_1, head_kctx, head, head->jc, js); ++ break; ++ case JS_COMMAND_HARD_STOP: ++ KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP, head_kctx, head, head->jc, js); ++ break; ++ case JS_COMMAND_HARD_STOP_0: ++ KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP_0, head_kctx, head, head->jc, js); ++ break; ++ case JS_COMMAND_HARD_STOP_1: ++ KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP_1, head_kctx, head, head->jc, js); ++ break; ++ default: ++ BUG(); ++ break; ++ } ++ } else { ++ if (status_reg_before == BASE_JD_EVENT_ACTIVE) ++ KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, job_in_head_before, js); ++ else ++ KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, 0, js); ++ ++ switch (action) { ++ case JS_COMMAND_SOFT_STOP: ++ KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_SOFTSTOP, NULL, NULL, 0, js); ++ break; ++ case JS_COMMAND_SOFT_STOP_0: ++ KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_SOFTSTOP_0, NULL, NULL, 0, js); ++ break; ++ case JS_COMMAND_SOFT_STOP_1: ++ KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_SOFTSTOP_1, NULL, NULL, 0, js); ++ break; ++ case JS_COMMAND_HARD_STOP: ++ KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP, NULL, NULL, 0, js); ++ break; ++ case JS_COMMAND_HARD_STOP_0: ++ KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP_0, NULL, NULL, 0, js); ++ break; ++ case JS_COMMAND_HARD_STOP_1: ++ KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP_1, NULL, NULL, 0, js); ++ break; ++ default: ++ BUG(); ++ break; ++ } ++ } ++#endif ++} ++ ++void kbase_backend_jm_kill_running_jobs_from_kctx(struct kbase_context *kctx) ++{ ++ struct kbase_device *kbdev = kctx->kbdev; ++ int i; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) ++ kbase_job_slot_hardstop(kctx, i, NULL); ++} ++ ++/** ++ * kbase_is_existing_atom_submitted_later_than_ready ++ * @ready: sequence number of the ready atom ++ * @existing: sequence number of the existing atom ++ * ++ * Returns true if the existing atom has been submitted later than the ++ * ready atom. It is used to understand if an atom that is ready has been ++ * submitted earlier than the currently running atom, so that the currently ++ * running atom should be preempted to allow the ready atom to run. ++ */ ++static inline bool kbase_is_existing_atom_submitted_later_than_ready(u64 ready, u64 existing) ++{ ++ /* No seq_nr set? */ ++ if (!ready || !existing) ++ return false; ++ ++ /* Efficiently handle the unlikely case of wrapping. ++ * The following code assumes that the delta between the sequence number ++ * of the two atoms is less than INT64_MAX. ++ * In the extremely unlikely case where the delta is higher, the comparison ++ * defaults for no preemption. ++ * The code also assumes that the conversion from unsigned to signed types ++ * works because the signed integers are 2's complement. ++ */ ++ return (s64)(ready - existing) < 0; ++} ++ ++void kbase_job_slot_ctx_priority_check_locked(struct kbase_context *kctx, ++ struct kbase_jd_atom *target_katom) ++{ ++ struct kbase_device *kbdev; ++ int js = target_katom->slot_nr; ++ int priority = target_katom->sched_priority; ++ int seq_nr = target_katom->seq_nr; ++ int i; ++ bool stop_sent = false; ++ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ kbdev = kctx->kbdev; ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ for (i = 0; i < kbase_backend_nr_atoms_on_slot(kbdev, js); i++) { ++ struct kbase_jd_atom *katom; ++ ++ katom = kbase_gpu_inspect(kbdev, js, i); ++ if (!katom) ++ continue; ++ ++ if ((kbdev->js_ctx_scheduling_mode == ++ KBASE_JS_PROCESS_LOCAL_PRIORITY_MODE) && ++ (katom->kctx != kctx)) ++ continue; ++ ++ if ((katom->sched_priority > priority) || ++ (katom->kctx == kctx && kbase_is_existing_atom_submitted_later_than_ready(seq_nr, katom->seq_nr))) { ++ if (!stop_sent) ++ KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITIZED( ++ kbdev, ++ target_katom); ++ ++ kbase_job_slot_softstop(kbdev, js, katom); ++ stop_sent = true; ++ } ++ } ++} ++ ++static int softstop_start_rp_nolock( ++ struct kbase_context *kctx, struct kbase_va_region *reg) ++{ ++ struct kbase_device *const kbdev = kctx->kbdev; ++ struct kbase_jd_atom *katom; ++ struct kbase_jd_renderpass *rp; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ katom = kbase_gpu_inspect(kbdev, 1, 0); ++ ++ if (!katom) { ++ dev_dbg(kctx->kbdev->dev, "No atom on job slot\n"); ++ return -ESRCH; ++ } ++ ++ if (!(katom->core_req & BASE_JD_REQ_START_RENDERPASS)) { ++ dev_dbg(kctx->kbdev->dev, ++ "Atom %p on job slot is not start RP\n", (void *)katom); ++ return -EPERM; ++ } ++ ++ compiletime_assert((1ull << (sizeof(katom->renderpass_id) * 8)) <= ++ ARRAY_SIZE(kctx->jctx.renderpasses), ++ "Should check invalid access to renderpasses"); ++ ++ rp = &kctx->jctx.renderpasses[katom->renderpass_id]; ++ if (WARN_ON(rp->state != KBASE_JD_RP_START && ++ rp->state != KBASE_JD_RP_RETRY)) ++ return -EINVAL; ++ ++ dev_dbg(kctx->kbdev->dev, "OOM in state %d with region %p\n", ++ (int)rp->state, (void *)reg); ++ ++ if (WARN_ON(katom != rp->start_katom)) ++ return -EINVAL; ++ ++ dev_dbg(kctx->kbdev->dev, "Adding region %p to list %p\n", ++ (void *)reg, (void *)&rp->oom_reg_list); ++ list_move_tail(®->link, &rp->oom_reg_list); ++ dev_dbg(kctx->kbdev->dev, "Added region to list\n"); ++ ++ rp->state = (rp->state == KBASE_JD_RP_START ? ++ KBASE_JD_RP_PEND_OOM : KBASE_JD_RP_RETRY_PEND_OOM); ++ ++ kbase_job_slot_softstop(kbdev, 1, katom); ++ ++ return 0; ++} ++ ++int kbase_job_slot_softstop_start_rp(struct kbase_context *const kctx, ++ struct kbase_va_region *const reg) ++{ ++ struct kbase_device *const kbdev = kctx->kbdev; ++ int err; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ err = softstop_start_rp_nolock(kctx, reg); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ return err; ++} ++ ++void kbase_jm_wait_for_zero_jobs(struct kbase_context *kctx) ++{ ++ struct kbase_device *kbdev = kctx->kbdev; ++ unsigned long timeout = msecs_to_jiffies(ZAP_TIMEOUT); ++ ++ timeout = wait_event_timeout(kctx->jctx.zero_jobs_wait, ++ kctx->jctx.job_nr == 0, timeout); ++ ++ if (timeout != 0) ++ timeout = wait_event_timeout( ++ kctx->jctx.sched_info.ctx.is_scheduled_wait, ++ !kbase_ctx_flag(kctx, KCTX_SCHEDULED), ++ timeout); ++ ++ /* Neither wait timed out; all done! */ ++ if (timeout != 0) ++ goto exit; ++ ++ if (kbase_prepare_to_reset_gpu(kbdev)) { ++ dev_err(kbdev->dev, ++ "Issueing GPU soft-reset because jobs failed to be killed (within %d ms) as part of context termination (e.g. process exit)\n", ++ ZAP_TIMEOUT); ++ kbase_reset_gpu(kbdev); ++ } ++ ++ /* Wait for the reset to complete */ ++ kbase_reset_gpu_wait(kbdev); ++exit: ++ dev_dbg(kbdev->dev, "Zap: Finished Context %p", kctx); ++ ++ /* Ensure that the signallers of the waitqs have finished */ ++ mutex_lock(&kctx->jctx.lock); ++ mutex_lock(&kctx->jctx.sched_info.ctx.jsctx_mutex); ++ mutex_unlock(&kctx->jctx.sched_info.ctx.jsctx_mutex); ++ mutex_unlock(&kctx->jctx.lock); ++} ++ ++u32 kbase_backend_get_current_flush_id(struct kbase_device *kbdev) ++{ ++ u32 flush_id = 0; ++ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_FLUSH_REDUCTION)) { ++ mutex_lock(&kbdev->pm.lock); ++ if (kbdev->pm.backend.gpu_powered) ++ flush_id = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(LATEST_FLUSH)); ++ mutex_unlock(&kbdev->pm.lock); ++ } ++ ++ return flush_id; ++} ++ ++int kbase_job_slot_init(struct kbase_device *kbdev) ++{ ++ CSTD_UNUSED(kbdev); ++ return 0; ++} ++KBASE_EXPORT_TEST_API(kbase_job_slot_init); ++ ++void kbase_job_slot_halt(struct kbase_device *kbdev) ++{ ++ CSTD_UNUSED(kbdev); ++} ++ ++void kbase_job_slot_term(struct kbase_device *kbdev) ++{ ++ CSTD_UNUSED(kbdev); ++} ++KBASE_EXPORT_TEST_API(kbase_job_slot_term); ++ ++ ++/** ++ * kbase_job_slot_softstop_swflags - Soft-stop a job with flags ++ * @kbdev: The kbase device ++ * @js: The job slot to soft-stop ++ * @target_katom: The job that should be soft-stopped (or NULL for any job) ++ * @sw_flags: Flags to pass in about the soft-stop ++ * ++ * Context: ++ * The job slot lock must be held when calling this function. ++ * The job slot must not already be in the process of being soft-stopped. ++ * ++ * Soft-stop the specified job slot, with extra information about the stop ++ * ++ * Where possible any job in the next register is evicted before the soft-stop. ++ */ ++void kbase_job_slot_softstop_swflags(struct kbase_device *kbdev, int js, ++ struct kbase_jd_atom *target_katom, u32 sw_flags) ++{ ++ dev_dbg(kbdev->dev, "Soft-stop atom %p with flags 0x%x (s:%d)\n", ++ target_katom, sw_flags, js); ++ ++ KBASE_DEBUG_ASSERT(!(sw_flags & JS_COMMAND_MASK)); ++ kbase_backend_soft_hard_stop_slot(kbdev, NULL, js, target_katom, ++ JS_COMMAND_SOFT_STOP | sw_flags); ++} ++ ++/** ++ * kbase_job_slot_softstop - Soft-stop the specified job slot ++ * @kbdev: The kbase device ++ * @js: The job slot to soft-stop ++ * @target_katom: The job that should be soft-stopped (or NULL for any job) ++ * Context: ++ * The job slot lock must be held when calling this function. ++ * The job slot must not already be in the process of being soft-stopped. ++ * ++ * Where possible any job in the next register is evicted before the soft-stop. ++ */ ++void kbase_job_slot_softstop(struct kbase_device *kbdev, int js, ++ struct kbase_jd_atom *target_katom) ++{ ++ kbase_job_slot_softstop_swflags(kbdev, js, target_katom, 0u); ++} ++ ++/** ++ * kbase_job_slot_hardstop - Hard-stop the specified job slot ++ * @kctx: The kbase context that contains the job(s) that should ++ * be hard-stopped ++ * @js: The job slot to hard-stop ++ * @target_katom: The job that should be hard-stopped (or NULL for all ++ * jobs from the context) ++ * Context: ++ * The job slot lock must be held when calling this function. ++ */ ++void kbase_job_slot_hardstop(struct kbase_context *kctx, int js, ++ struct kbase_jd_atom *target_katom) ++{ ++ struct kbase_device *kbdev = kctx->kbdev; ++ bool stopped; ++ ++ stopped = kbase_backend_soft_hard_stop_slot(kbdev, kctx, js, ++ target_katom, ++ JS_COMMAND_HARD_STOP); ++} ++ ++/** ++ * kbase_job_check_enter_disjoint - potentiall enter disjoint mode ++ * @kbdev: kbase device ++ * @action: the event which has occurred ++ * @core_reqs: core requirements of the atom ++ * @target_katom: the atom which is being affected ++ * ++ * For a certain soft-stop action, work out whether to enter disjoint ++ * state. ++ * ++ * This does not register multiple disjoint events if the atom has already ++ * started a disjoint period ++ * ++ * @core_reqs can be supplied as 0 if the atom had not started on the hardware ++ * (and so a 'real' soft/hard-stop was not required, but it still interrupted ++ * flow, perhaps on another context) ++ * ++ * kbase_job_check_leave_disjoint() should be used to end the disjoint ++ * state when the soft/hard-stop action is complete ++ */ ++void kbase_job_check_enter_disjoint(struct kbase_device *kbdev, u32 action, ++ base_jd_core_req core_reqs, struct kbase_jd_atom *target_katom) ++{ ++ u32 hw_action = action & JS_COMMAND_MASK; ++ ++ /* For soft-stop, don't enter if soft-stop not allowed, or isn't ++ * causing disjoint. ++ */ ++ if (hw_action == JS_COMMAND_SOFT_STOP && ++ (kbase_jd_katom_is_protected(target_katom) || ++ (0 == (action & JS_COMMAND_SW_CAUSES_DISJOINT)))) ++ return; ++ ++ /* Nothing to do if already logged disjoint state on this atom */ ++ if (target_katom->atom_flags & KBASE_KATOM_FLAG_IN_DISJOINT) ++ return; ++ ++ target_katom->atom_flags |= KBASE_KATOM_FLAG_IN_DISJOINT; ++ kbase_disjoint_state_up(kbdev); ++} ++ ++/** ++ * kbase_job_check_enter_disjoint - potentially leave disjoint state ++ * @kbdev: kbase device ++ * @target_katom: atom which is finishing ++ * ++ * Work out whether to leave disjoint state when finishing an atom that was ++ * originated by kbase_job_check_enter_disjoint(). ++ */ ++void kbase_job_check_leave_disjoint(struct kbase_device *kbdev, ++ struct kbase_jd_atom *target_katom) ++{ ++ if (target_katom->atom_flags & KBASE_KATOM_FLAG_IN_DISJOINT) { ++ target_katom->atom_flags &= ~KBASE_KATOM_FLAG_IN_DISJOINT; ++ kbase_disjoint_state_down(kbdev); ++ } ++} ++ ++static void kbase_debug_dump_registers(struct kbase_device *kbdev) ++{ ++ int i; ++ ++ kbase_io_history_dump(kbdev); ++ ++ dev_err(kbdev->dev, "Register state:"); ++ dev_err(kbdev->dev, " GPU_IRQ_RAWSTAT=0x%08x GPU_STATUS=0x%08x", ++ kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT)), ++ kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_STATUS))); ++ dev_err(kbdev->dev, " JOB_IRQ_RAWSTAT=0x%08x JOB_IRQ_JS_STATE=0x%08x", ++ kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_RAWSTAT)), ++ kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_JS_STATE))); ++ for (i = 0; i < 3; i++) { ++ dev_err(kbdev->dev, " JS%d_STATUS=0x%08x JS%d_HEAD_LO=0x%08x", ++ i, kbase_reg_read(kbdev, JOB_SLOT_REG(i, JS_STATUS)), ++ i, kbase_reg_read(kbdev, JOB_SLOT_REG(i, JS_HEAD_LO))); ++ } ++ dev_err(kbdev->dev, " MMU_IRQ_RAWSTAT=0x%08x GPU_FAULTSTATUS=0x%08x", ++ kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_RAWSTAT)), ++ kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_FAULTSTATUS))); ++ dev_err(kbdev->dev, " GPU_IRQ_MASK=0x%08x JOB_IRQ_MASK=0x%08x MMU_IRQ_MASK=0x%08x", ++ kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK)), ++ kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK)), ++ kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK))); ++ dev_err(kbdev->dev, " PWR_OVERRIDE0=0x%08x PWR_OVERRIDE1=0x%08x", ++ kbase_reg_read(kbdev, GPU_CONTROL_REG(PWR_OVERRIDE0)), ++ kbase_reg_read(kbdev, GPU_CONTROL_REG(PWR_OVERRIDE1))); ++ dev_err(kbdev->dev, " SHADER_CONFIG=0x%08x L2_MMU_CONFIG=0x%08x", ++ kbase_reg_read(kbdev, GPU_CONTROL_REG(SHADER_CONFIG)), ++ kbase_reg_read(kbdev, GPU_CONTROL_REG(L2_MMU_CONFIG))); ++ dev_err(kbdev->dev, " TILER_CONFIG=0x%08x JM_CONFIG=0x%08x", ++ kbase_reg_read(kbdev, GPU_CONTROL_REG(TILER_CONFIG)), ++ kbase_reg_read(kbdev, GPU_CONTROL_REG(JM_CONFIG))); ++} ++ ++static void kbasep_reset_timeout_worker(struct work_struct *data) ++{ ++ unsigned long flags; ++ struct kbase_device *kbdev; ++ ktime_t end_timestamp = ktime_get(); ++ struct kbasep_js_device_data *js_devdata; ++ bool silent = false; ++ u32 max_loops = KBASE_CLEAN_CACHE_MAX_LOOPS; ++ ++ KBASE_DEBUG_ASSERT(data); ++ ++ kbdev = container_of(data, struct kbase_device, ++ hwaccess.backend.reset_work); ++ ++ KBASE_DEBUG_ASSERT(kbdev); ++ js_devdata = &kbdev->js_data; ++ ++ if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) == ++ KBASE_RESET_GPU_SILENT) ++ silent = true; ++ ++ KBASE_KTRACE_ADD_JM(kbdev, JM_BEGIN_RESET_WORKER, NULL, NULL, 0u, 0); ++ ++ /* Disable GPU hardware counters. ++ * This call will block until counters are disabled. ++ */ ++ kbase_hwcnt_context_disable(kbdev->hwcnt_gpu_ctx); ++ ++ /* Make sure the timer has completed - this cannot be done from ++ * interrupt context, so this cannot be done within ++ * kbasep_try_reset_gpu_early. */ ++ hrtimer_cancel(&kbdev->hwaccess.backend.reset_timer); ++ ++ if (kbase_pm_context_active_handle_suspend(kbdev, ++ KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) { ++ /* This would re-activate the GPU. Since it's already idle, ++ * there's no need to reset it */ ++ atomic_set(&kbdev->hwaccess.backend.reset_gpu, ++ KBASE_RESET_GPU_NOT_PENDING); ++ kbase_disjoint_state_down(kbdev); ++ wake_up(&kbdev->hwaccess.backend.reset_wait); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ return; ++ } ++ ++ KBASE_DEBUG_ASSERT(kbdev->irq_reset_flush == false); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ spin_lock(&kbdev->mmu_mask_change); ++ kbase_pm_reset_start_locked(kbdev); ++ ++ /* We're about to flush out the IRQs and their bottom half's */ ++ kbdev->irq_reset_flush = true; ++ ++ /* Disable IRQ to avoid IRQ handlers to kick in after releasing the ++ * spinlock; this also clears any outstanding interrupts */ ++ kbase_pm_disable_interrupts_nolock(kbdev); ++ ++ spin_unlock(&kbdev->mmu_mask_change); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ /* Ensure that any IRQ handlers have finished ++ * Must be done without any locks IRQ handlers will take */ ++ kbase_synchronize_irqs(kbdev); ++ ++ /* Flush out any in-flight work items */ ++ kbase_flush_mmu_wqs(kbdev); ++ ++ /* The flush has completed so reset the active indicator */ ++ kbdev->irq_reset_flush = false; ++ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TMIX_8463)) { ++ /* Ensure that L2 is not transitioning when we send the reset ++ * command */ ++ while (--max_loops && kbase_pm_get_trans_cores(kbdev, ++ KBASE_PM_CORE_L2)) ++ ; ++ ++ WARN(!max_loops, "L2 power transition timed out while trying to reset\n"); ++ } ++ ++ mutex_lock(&kbdev->pm.lock); ++ /* We hold the pm lock, so there ought to be a current policy */ ++ KBASE_DEBUG_ASSERT(kbdev->pm.backend.pm_current_policy); ++ ++ /* All slot have been soft-stopped and we've waited ++ * SOFT_STOP_RESET_TIMEOUT for the slots to clear, at this point we ++ * assume that anything that is still left on the GPU is stuck there and ++ * we'll kill it when we reset the GPU */ ++ ++ if (!silent) ++ dev_err(kbdev->dev, "Resetting GPU (allowing up to %d ms)", ++ RESET_TIMEOUT); ++ ++ /* Output the state of some interesting registers to help in the ++ * debugging of GPU resets */ ++ if (!silent) ++ kbase_debug_dump_registers(kbdev); ++ ++ /* Complete any jobs that were still on the GPU */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbdev->protected_mode = false; ++ if (!kbdev->pm.backend.protected_entry_transition_override) ++ kbase_backend_reset(kbdev, &end_timestamp); ++ kbase_pm_metrics_update(kbdev, NULL); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ /* Reset the GPU */ ++ kbase_pm_init_hw(kbdev, 0); ++ ++ mutex_unlock(&kbdev->pm.lock); ++ ++ mutex_lock(&js_devdata->runpool_mutex); ++ ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_ctx_sched_restore_all_as(kbdev); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ ++ kbase_pm_enable_interrupts(kbdev); ++ ++ kbase_disjoint_state_down(kbdev); ++ ++ mutex_unlock(&js_devdata->runpool_mutex); ++ ++ mutex_lock(&kbdev->pm.lock); ++ ++ kbase_pm_reset_complete(kbdev); ++ ++ /* Find out what cores are required now */ ++ kbase_pm_update_cores_state(kbdev); ++ ++ /* Synchronously request and wait for those cores, because if ++ * instrumentation is enabled it would need them immediately. */ ++ kbase_pm_wait_for_desired_state(kbdev); ++ ++ mutex_unlock(&kbdev->pm.lock); ++ ++ atomic_set(&kbdev->hwaccess.backend.reset_gpu, ++ KBASE_RESET_GPU_NOT_PENDING); ++ ++ wake_up(&kbdev->hwaccess.backend.reset_wait); ++ if (!silent) ++ dev_err(kbdev->dev, "Reset complete"); ++ ++ /* Try submitting some jobs to restart processing */ ++ KBASE_KTRACE_ADD_JM(kbdev, JM_SUBMIT_AFTER_RESET, NULL, NULL, 0u, 0); ++ kbase_js_sched_all(kbdev); ++ ++ /* Process any pending slot updates */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_backend_slot_update(kbdev); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ kbase_pm_context_idle(kbdev); ++ ++ /* Re-enable GPU hardware counters */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ KBASE_KTRACE_ADD_JM(kbdev, JM_END_RESET_WORKER, NULL, NULL, 0u, 0); ++} ++ ++static enum hrtimer_restart kbasep_reset_timer_callback(struct hrtimer *timer) ++{ ++ struct kbase_device *kbdev = container_of(timer, struct kbase_device, ++ hwaccess.backend.reset_timer); ++ ++ KBASE_DEBUG_ASSERT(kbdev); ++ ++ /* Reset still pending? */ ++ if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu, ++ KBASE_RESET_GPU_COMMITTED, KBASE_RESET_GPU_HAPPENING) == ++ KBASE_RESET_GPU_COMMITTED) ++ queue_work(kbdev->hwaccess.backend.reset_workq, ++ &kbdev->hwaccess.backend.reset_work); ++ ++ return HRTIMER_NORESTART; ++} ++ ++/* ++ * If all jobs are evicted from the GPU then we can reset the GPU ++ * immediately instead of waiting for the timeout to elapse ++ */ ++ ++static void kbasep_try_reset_gpu_early_locked(struct kbase_device *kbdev) ++{ ++ int i; ++ int pending_jobs = 0; ++ ++ KBASE_DEBUG_ASSERT(kbdev); ++ ++ /* Count the number of jobs */ ++ for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) ++ pending_jobs += kbase_backend_nr_atoms_submitted(kbdev, i); ++ ++ if (pending_jobs > 0) { ++ /* There are still jobs on the GPU - wait */ ++ return; ++ } ++ ++ /* To prevent getting incorrect registers when dumping failed job, ++ * skip early reset. ++ */ ++ if (atomic_read(&kbdev->job_fault_debug) > 0) ++ return; ++ ++ /* Check that the reset has been committed to (i.e. kbase_reset_gpu has ++ * been called), and that no other thread beat this thread to starting ++ * the reset */ ++ if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu, ++ KBASE_RESET_GPU_COMMITTED, KBASE_RESET_GPU_HAPPENING) != ++ KBASE_RESET_GPU_COMMITTED) { ++ /* Reset has already occurred */ ++ return; ++ } ++ ++ queue_work(kbdev->hwaccess.backend.reset_workq, ++ &kbdev->hwaccess.backend.reset_work); ++} ++ ++static void kbasep_try_reset_gpu_early(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbasep_try_reset_gpu_early_locked(kbdev); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++} ++ ++/** ++ * kbase_prepare_to_reset_gpu_locked - Prepare for resetting the GPU ++ * @kbdev: kbase device ++ * ++ * This function just soft-stops all the slots to ensure that as many jobs as ++ * possible are saved. ++ * ++ * Return: ++ * The function returns a boolean which should be interpreted as follows: ++ * true - Prepared for reset, kbase_reset_gpu_locked should be called. ++ * false - Another thread is performing a reset, kbase_reset_gpu should ++ * not be called. ++ */ ++bool kbase_prepare_to_reset_gpu_locked(struct kbase_device *kbdev) ++{ ++ int i; ++ ++ KBASE_DEBUG_ASSERT(kbdev); ++ ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++ if (kbase_pm_is_gpu_lost(kbdev)) { ++ /* GPU access has been removed, reset will be done by ++ * Arbiter instead ++ */ ++ return false; ++ } ++#endif ++ ++ if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu, ++ KBASE_RESET_GPU_NOT_PENDING, ++ KBASE_RESET_GPU_PREPARED) != ++ KBASE_RESET_GPU_NOT_PENDING) { ++ /* Some other thread is already resetting the GPU */ ++ return false; ++ } ++ ++ kbase_disjoint_state_up(kbdev); ++ ++ for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) ++ kbase_job_slot_softstop(kbdev, i, NULL); ++ ++ return true; ++} ++ ++bool kbase_prepare_to_reset_gpu(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ bool ret; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ret = kbase_prepare_to_reset_gpu_locked(kbdev); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ return ret; ++} ++KBASE_EXPORT_TEST_API(kbase_prepare_to_reset_gpu); ++ ++/* ++ * This function should be called after kbase_prepare_to_reset_gpu if it ++ * returns true. It should never be called without a corresponding call to ++ * kbase_prepare_to_reset_gpu. ++ * ++ * After this function is called (or not called if kbase_prepare_to_reset_gpu ++ * returned false), the caller should wait for ++ * kbdev->hwaccess.backend.reset_waitq to be signalled to know when the reset ++ * has completed. ++ */ ++void kbase_reset_gpu(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev); ++ ++ /* Note this is an assert/atomic_set because it is a software issue for ++ * a race to be occuring here */ ++ KBASE_DEBUG_ASSERT(atomic_read(&kbdev->hwaccess.backend.reset_gpu) == ++ KBASE_RESET_GPU_PREPARED); ++ atomic_set(&kbdev->hwaccess.backend.reset_gpu, ++ KBASE_RESET_GPU_COMMITTED); ++ ++ dev_err(kbdev->dev, "Preparing to soft-reset GPU: Waiting (upto %d ms) for all jobs to complete soft-stop\n", ++ kbdev->reset_timeout_ms); ++ ++ hrtimer_start(&kbdev->hwaccess.backend.reset_timer, ++ HR_TIMER_DELAY_MSEC(kbdev->reset_timeout_ms), ++ HRTIMER_MODE_REL); ++ ++ /* Try resetting early */ ++ kbasep_try_reset_gpu_early(kbdev); ++} ++KBASE_EXPORT_TEST_API(kbase_reset_gpu); ++ ++void kbase_reset_gpu_locked(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev); ++ ++ /* Note this is an assert/atomic_set because it is a software issue for ++ * a race to be occuring here */ ++ KBASE_DEBUG_ASSERT(atomic_read(&kbdev->hwaccess.backend.reset_gpu) == ++ KBASE_RESET_GPU_PREPARED); ++ atomic_set(&kbdev->hwaccess.backend.reset_gpu, ++ KBASE_RESET_GPU_COMMITTED); ++ ++ dev_err(kbdev->dev, "Preparing to soft-reset GPU: Waiting (upto %d ms) for all jobs to complete soft-stop\n", ++ kbdev->reset_timeout_ms); ++ hrtimer_start(&kbdev->hwaccess.backend.reset_timer, ++ HR_TIMER_DELAY_MSEC(kbdev->reset_timeout_ms), ++ HRTIMER_MODE_REL); ++ ++ /* Try resetting early */ ++ kbasep_try_reset_gpu_early_locked(kbdev); ++} ++ ++int kbase_reset_gpu_silent(struct kbase_device *kbdev) ++{ ++ if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu, ++ KBASE_RESET_GPU_NOT_PENDING, ++ KBASE_RESET_GPU_SILENT) != ++ KBASE_RESET_GPU_NOT_PENDING) { ++ /* Some other thread is already resetting the GPU */ ++ return -EAGAIN; ++ } ++ ++ kbase_disjoint_state_up(kbdev); ++ ++ queue_work(kbdev->hwaccess.backend.reset_workq, ++ &kbdev->hwaccess.backend.reset_work); ++ ++ return 0; ++} ++ ++bool kbase_reset_gpu_is_active(struct kbase_device *kbdev) ++{ ++ if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) == ++ KBASE_RESET_GPU_NOT_PENDING) ++ return false; ++ ++ return true; ++} ++ ++int kbase_reset_gpu_wait(struct kbase_device *kbdev) ++{ ++ wait_event(kbdev->hwaccess.backend.reset_wait, ++ atomic_read(&kbdev->hwaccess.backend.reset_gpu) ++ == KBASE_RESET_GPU_NOT_PENDING); ++ ++ return 0; ++} ++KBASE_EXPORT_TEST_API(kbase_reset_gpu_wait); ++ ++int kbase_reset_gpu_init(struct kbase_device *kbdev) ++{ ++ kbdev->hwaccess.backend.reset_workq = alloc_workqueue( ++ "Mali reset workqueue", 0, 1); ++ if (kbdev->hwaccess.backend.reset_workq == NULL) ++ return -ENOMEM; ++ ++ INIT_WORK(&kbdev->hwaccess.backend.reset_work, ++ kbasep_reset_timeout_worker); ++ ++ hrtimer_init(&kbdev->hwaccess.backend.reset_timer, CLOCK_MONOTONIC, ++ HRTIMER_MODE_REL); ++ kbdev->hwaccess.backend.reset_timer.function = ++ kbasep_reset_timer_callback; ++ ++ return 0; ++} ++ ++void kbase_reset_gpu_term(struct kbase_device *kbdev) ++{ ++ destroy_workqueue(kbdev->hwaccess.backend.reset_workq); ++} +diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_internal.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_internal.h +new file mode 100755 +index 000000000000..cd1f9794fdc4 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_internal.h +@@ -0,0 +1,181 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2016, 2018-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++/* ++ * Job Manager backend-specific low-level APIs. ++ */ ++ ++#ifndef _KBASE_JM_HWACCESS_H_ ++#define _KBASE_JM_HWACCESS_H_ ++ ++#include ++#include ++#include ++ ++#include ++#include ++ ++/** ++ * kbase_job_submit_nolock() - Submit a job to a certain job-slot ++ * @kbdev: Device pointer ++ * @katom: Atom to submit ++ * @js: Job slot to submit on ++ * ++ * The caller must check kbasep_jm_is_submit_slots_free() != false before ++ * calling this. ++ * ++ * The following locking conditions are made on the caller: ++ * - it must hold the hwaccess_lock ++ */ ++void kbase_job_submit_nolock(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom, int js); ++ ++/** ++ * kbase_job_done_slot() - Complete the head job on a particular job-slot ++ * @kbdev: Device pointer ++ * @s: Job slot ++ * @completion_code: Completion code of job reported by GPU ++ * @job_tail: Job tail address reported by GPU ++ * @end_timestamp: Timestamp of job completion ++ */ ++void kbase_job_done_slot(struct kbase_device *kbdev, int s, u32 completion_code, ++ u64 job_tail, ktime_t *end_timestamp); ++ ++#ifdef CONFIG_GPU_TRACEPOINTS ++static inline char *kbasep_make_job_slot_string(int js, char *js_string, ++ size_t js_size) ++{ ++ snprintf(js_string, js_size, "job_slot_%i", js); ++ return js_string; ++} ++#endif ++ ++#if !MALI_USE_CSF ++static inline int kbasep_jm_is_js_free(struct kbase_device *kbdev, int js, ++ struct kbase_context *kctx) ++{ ++ return !kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT)); ++} ++#endif ++ ++ ++/** ++ * kbase_job_hw_submit() - Submit a job to the GPU ++ * @kbdev: Device pointer ++ * @katom: Atom to submit ++ * @js: Job slot to submit on ++ * ++ * The caller must check kbasep_jm_is_submit_slots_free() != false before ++ * calling this. ++ * ++ * The following locking conditions are made on the caller: ++ * - it must hold the hwaccess_lock ++ */ ++void kbase_job_hw_submit(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom, ++ int js); ++ ++#if !MALI_USE_CSF ++/** ++ * kbasep_job_slot_soft_or_hard_stop_do_action() - Perform a soft or hard stop ++ * on the specified atom ++ * @kbdev: Device pointer ++ * @js: Job slot to stop on ++ * @action: The action to perform, either JSn_COMMAND_HARD_STOP or ++ * JSn_COMMAND_SOFT_STOP ++ * @core_reqs: Core requirements of atom to stop ++ * @target_katom: Atom to stop ++ * ++ * The following locking conditions are made on the caller: ++ * - it must hold the hwaccess_lock ++ */ ++void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev, ++ int js, ++ u32 action, ++ base_jd_core_req core_reqs, ++ struct kbase_jd_atom *target_katom); ++#endif /* !MALI_USE_CSF */ ++ ++/** ++ * kbase_backend_soft_hard_stop_slot() - Soft or hard stop jobs on a given job ++ * slot belonging to a given context. ++ * @kbdev: Device pointer ++ * @kctx: Context pointer. May be NULL ++ * @katom: Specific atom to stop. May be NULL ++ * @js: Job slot to hard stop ++ * @action: The action to perform, either JSn_COMMAND_HARD_STOP or ++ * JSn_COMMAND_SOFT_STOP ++ * ++ * If no context is provided then all jobs on the slot will be soft or hard ++ * stopped. ++ * ++ * If a katom is provided then only that specific atom will be stopped. In this ++ * case the kctx parameter is ignored. ++ * ++ * Jobs that are on the slot but are not yet on the GPU will be unpulled and ++ * returned to the job scheduler. ++ * ++ * Return: true if an atom was stopped, false otherwise ++ */ ++bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ int js, ++ struct kbase_jd_atom *katom, ++ u32 action); ++ ++/** ++ * kbase_job_slot_init - Initialise job slot framework ++ * @kbdev: Device pointer ++ * ++ * Called on driver initialisation ++ * ++ * Return: 0 on success ++ */ ++int kbase_job_slot_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_job_slot_halt - Halt the job slot framework ++ * @kbdev: Device pointer ++ * ++ * Should prevent any further job slot processing ++ */ ++void kbase_job_slot_halt(struct kbase_device *kbdev); ++ ++/** ++ * kbase_job_slot_term - Terminate job slot framework ++ * @kbdev: Device pointer ++ * ++ * Called on driver termination ++ */ ++void kbase_job_slot_term(struct kbase_device *kbdev); ++ ++/** ++ * kbase_gpu_cache_clean - Cause a GPU cache clean & flush ++ * @kbdev: Device pointer ++ * ++ * Caller must not be in IRQ context ++ */ ++void kbase_gpu_cache_clean(struct kbase_device *kbdev); ++ ++#endif /* _KBASE_JM_HWACCESS_H_ */ +diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_rb.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_rb.c +new file mode 100755 +index 000000000000..afaaef27883d +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_rb.c +@@ -0,0 +1,1659 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++/* ++ * Register-based HW access backend specific APIs ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++/* Return whether the specified ringbuffer is empty. HW access lock must be ++ * held */ ++#define SLOT_RB_EMPTY(rb) (rb->write_idx == rb->read_idx) ++/* Return number of atoms currently in the specified ringbuffer. HW access lock ++ * must be held */ ++#define SLOT_RB_ENTRIES(rb) (int)(s8)(rb->write_idx - rb->read_idx) ++ ++static void kbase_gpu_release_atom(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom, ++ ktime_t *end_timestamp); ++ ++/** ++ * kbase_gpu_enqueue_atom - Enqueue an atom in the HW access ringbuffer ++ * @kbdev: Device pointer ++ * @katom: Atom to enqueue ++ * ++ * Context: Caller must hold the HW access lock ++ */ ++static void kbase_gpu_enqueue_atom(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom) ++{ ++ struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[katom->slot_nr]; ++ ++ WARN_ON(SLOT_RB_ENTRIES(rb) >= SLOT_RB_SIZE); ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ rb->entries[rb->write_idx & SLOT_RB_MASK].katom = katom; ++ rb->write_idx++; ++ ++ katom->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_BLOCKED; ++} ++ ++/** ++ * kbase_gpu_dequeue_atom - Remove an atom from the HW access ringbuffer, once ++ * it has been completed ++ * @kbdev: Device pointer ++ * @js: Job slot to remove atom from ++ * @end_timestamp: Pointer to timestamp of atom completion. May be NULL, in ++ * which case current time will be used. ++ * ++ * Context: Caller must hold the HW access lock ++ * ++ * Return: Atom removed from ringbuffer ++ */ ++static struct kbase_jd_atom *kbase_gpu_dequeue_atom(struct kbase_device *kbdev, ++ int js, ++ ktime_t *end_timestamp) ++{ ++ struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js]; ++ struct kbase_jd_atom *katom; ++ ++ if (SLOT_RB_EMPTY(rb)) { ++ WARN(1, "GPU ringbuffer unexpectedly empty\n"); ++ return NULL; ++ } ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ katom = rb->entries[rb->read_idx & SLOT_RB_MASK].katom; ++ ++ kbase_gpu_release_atom(kbdev, katom, end_timestamp); ++ ++ rb->read_idx++; ++ ++ katom->gpu_rb_state = KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB; ++ ++ return katom; ++} ++ ++struct kbase_jd_atom *kbase_gpu_inspect(struct kbase_device *kbdev, int js, ++ int idx) ++{ ++ struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js]; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if ((SLOT_RB_ENTRIES(rb) - 1) < idx) ++ return NULL; /* idx out of range */ ++ ++ return rb->entries[(rb->read_idx + idx) & SLOT_RB_MASK].katom; ++} ++ ++struct kbase_jd_atom *kbase_backend_inspect_tail(struct kbase_device *kbdev, ++ int js) ++{ ++ struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js]; ++ ++ if (SLOT_RB_EMPTY(rb)) ++ return NULL; ++ ++ return rb->entries[(rb->write_idx - 1) & SLOT_RB_MASK].katom; ++} ++ ++bool kbase_gpu_atoms_submitted_any(struct kbase_device *kbdev) ++{ ++ int js; ++ int i; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { ++ for (i = 0; i < SLOT_RB_SIZE; i++) { ++ struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i); ++ ++ if (katom && katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED) ++ return true; ++ } ++ } ++ return false; ++} ++ ++int kbase_backend_nr_atoms_submitted(struct kbase_device *kbdev, int js) ++{ ++ int nr = 0; ++ int i; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ for (i = 0; i < SLOT_RB_SIZE; i++) { ++ struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i); ++ ++ if (katom && (katom->gpu_rb_state == ++ KBASE_ATOM_GPU_RB_SUBMITTED)) ++ nr++; ++ } ++ ++ return nr; ++} ++ ++int kbase_backend_nr_atoms_on_slot(struct kbase_device *kbdev, int js) ++{ ++ int nr = 0; ++ int i; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ for (i = 0; i < SLOT_RB_SIZE; i++) { ++ if (kbase_gpu_inspect(kbdev, js, i)) ++ nr++; ++ } ++ ++ return nr; ++} ++ ++static int kbase_gpu_nr_atoms_on_slot_min(struct kbase_device *kbdev, int js, ++ enum kbase_atom_gpu_rb_state min_rb_state) ++{ ++ int nr = 0; ++ int i; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ for (i = 0; i < SLOT_RB_SIZE; i++) { ++ struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i); ++ ++ if (katom && (katom->gpu_rb_state >= min_rb_state)) ++ nr++; ++ } ++ ++ return nr; ++} ++ ++/** ++ * check_secure_atom - Check if the given atom is in the given secure state and ++ * has a ringbuffer state of at least ++ * KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION ++ * @katom: Atom pointer ++ * @secure: Desired secure state ++ * ++ * Return: true if atom is in the given state, false otherwise ++ */ ++static bool check_secure_atom(struct kbase_jd_atom *katom, bool secure) ++{ ++ if (katom->gpu_rb_state >= ++ KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION && ++ ((kbase_jd_katom_is_protected(katom) && secure) || ++ (!kbase_jd_katom_is_protected(katom) && !secure))) ++ return true; ++ ++ return false; ++} ++ ++/** ++ * kbase_gpu_check_secure_atoms - Check if there are any atoms in the given ++ * secure state in the ringbuffers of at least ++ * state ++ * KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE ++ * @kbdev: Device pointer ++ * @secure: Desired secure state ++ * ++ * Return: true if any atoms are in the given state, false otherwise ++ */ ++static bool kbase_gpu_check_secure_atoms(struct kbase_device *kbdev, ++ bool secure) ++{ ++ int js, i; ++ ++ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { ++ for (i = 0; i < SLOT_RB_SIZE; i++) { ++ struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, ++ js, i); ++ ++ if (katom) { ++ if (check_secure_atom(katom, secure)) ++ return true; ++ } ++ } ++ } ++ ++ return false; ++} ++ ++int kbase_backend_slot_free(struct kbase_device *kbdev, int js) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) != ++ KBASE_RESET_GPU_NOT_PENDING) { ++ /* The GPU is being reset - so prevent submission */ ++ return 0; ++ } ++ ++ return SLOT_RB_SIZE - kbase_backend_nr_atoms_on_slot(kbdev, js); ++} ++ ++ ++static void kbase_gpu_release_atom(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom, ++ ktime_t *end_timestamp) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ switch (katom->gpu_rb_state) { ++ case KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB: ++ /* Should be impossible */ ++ WARN(1, "Attempting to release atom not in ringbuffer\n"); ++ break; ++ ++ case KBASE_ATOM_GPU_RB_SUBMITTED: ++ kbase_kinstr_jm_atom_hw_release(katom); ++ /* Inform power management at start/finish of atom so it can ++ * update its GPU utilisation metrics. Mark atom as not ++ * submitted beforehand. */ ++ katom->gpu_rb_state = KBASE_ATOM_GPU_RB_READY; ++ kbase_pm_metrics_update(kbdev, end_timestamp); ++ ++ if (katom->core_req & BASE_JD_REQ_PERMON) ++ kbase_pm_release_gpu_cycle_counter_nolock(kbdev); ++ /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ ++ ++ KBASE_TLSTREAM_TL_NRET_ATOM_LPU(kbdev, katom, ++ &kbdev->gpu_props.props.raw_props.js_features ++ [katom->slot_nr]); ++ KBASE_TLSTREAM_TL_NRET_ATOM_AS(kbdev, katom, &kbdev->as[kctx->as_nr]); ++ KBASE_TLSTREAM_TL_NRET_CTX_LPU(kbdev, kctx, ++ &kbdev->gpu_props.props.raw_props.js_features ++ [katom->slot_nr]); ++ ++ case KBASE_ATOM_GPU_RB_READY: ++ /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ ++ ++ case KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE: ++ break; ++ ++ case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION: ++ if (kbase_jd_katom_is_protected(katom) && ++ (katom->protected_state.enter != ++ KBASE_ATOM_ENTER_PROTECTED_CHECK) && ++ (katom->protected_state.enter != ++ KBASE_ATOM_ENTER_PROTECTED_HWCNT)) { ++ kbase_pm_protected_override_disable(kbdev); ++ kbase_pm_update_cores_state_nolock(kbdev); ++ } ++ if (kbase_jd_katom_is_protected(katom) && ++ (katom->protected_state.enter == ++ KBASE_ATOM_ENTER_PROTECTED_IDLE_L2)) ++ kbase_pm_protected_entry_override_disable(kbdev); ++ if (!kbase_jd_katom_is_protected(katom) && ++ (katom->protected_state.exit != ++ KBASE_ATOM_EXIT_PROTECTED_CHECK) && ++ (katom->protected_state.exit != ++ KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT)) { ++ kbase_pm_protected_override_disable(kbdev); ++ kbase_pm_update_cores_state_nolock(kbdev); ++ } ++ ++ if (katom->protected_state.enter != ++ KBASE_ATOM_ENTER_PROTECTED_CHECK || ++ katom->protected_state.exit != ++ KBASE_ATOM_EXIT_PROTECTED_CHECK) ++ kbdev->protected_mode_transition = false; ++ /* If the atom has suspended hwcnt but has not yet entered ++ * protected mode, then resume hwcnt now. If the GPU is now in ++ * protected mode then hwcnt will be resumed by GPU reset so ++ * don't resume it here. ++ */ ++ if (kbase_jd_katom_is_protected(katom) && ++ ((katom->protected_state.enter == ++ KBASE_ATOM_ENTER_PROTECTED_IDLE_L2) || ++ (katom->protected_state.enter == ++ KBASE_ATOM_ENTER_PROTECTED_SET_COHERENCY))) { ++ WARN_ON(!kbdev->protected_mode_hwcnt_disabled); ++ kbdev->protected_mode_hwcnt_desired = true; ++ if (kbdev->protected_mode_hwcnt_disabled) { ++ kbase_hwcnt_context_enable( ++ kbdev->hwcnt_gpu_ctx); ++ kbdev->protected_mode_hwcnt_disabled = false; ++ } ++ } ++ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TGOX_R1_1234)) { ++ if (katom->atom_flags & ++ KBASE_KATOM_FLAG_HOLDING_L2_REF_PROT) { ++ kbase_pm_protected_l2_override(kbdev, false); ++ katom->atom_flags &= ++ ~KBASE_KATOM_FLAG_HOLDING_L2_REF_PROT; ++ } ++ } ++ ++ /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ ++ ++ case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV: ++ /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ ++ ++ case KBASE_ATOM_GPU_RB_WAITING_BLOCKED: ++ /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ ++ ++ case KBASE_ATOM_GPU_RB_RETURN_TO_JS: ++ break; ++ } ++ ++ katom->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_BLOCKED; ++ katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK; ++} ++ ++static void kbase_gpu_mark_atom_for_return(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ kbase_gpu_release_atom(kbdev, katom, NULL); ++ katom->gpu_rb_state = KBASE_ATOM_GPU_RB_RETURN_TO_JS; ++} ++ ++/** ++ * other_slots_busy - Determine if any job slots other than @js are currently ++ * running atoms ++ * @kbdev: Device pointer ++ * @js: Job slot ++ * ++ * Return: true if any slots other than @js are busy, false otherwise ++ */ ++static inline bool other_slots_busy(struct kbase_device *kbdev, int js) ++{ ++ int slot; ++ ++ for (slot = 0; slot < kbdev->gpu_props.num_job_slots; slot++) { ++ if (slot == js) ++ continue; ++ ++ if (kbase_gpu_nr_atoms_on_slot_min(kbdev, slot, ++ KBASE_ATOM_GPU_RB_SUBMITTED)) ++ return true; ++ } ++ ++ return false; ++} ++ ++static inline bool kbase_gpu_in_protected_mode(struct kbase_device *kbdev) ++{ ++ return kbdev->protected_mode; ++} ++ ++static void kbase_gpu_disable_coherent(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ /* ++ * When entering into protected mode, we must ensure that the ++ * GPU is not operating in coherent mode as well. This is to ++ * ensure that no protected memory can be leaked. ++ */ ++ if (kbdev->system_coherency == COHERENCY_ACE) ++ kbase_cache_set_coherency_mode(kbdev, COHERENCY_ACE_LITE); ++} ++ ++static int kbase_gpu_protected_mode_enter(struct kbase_device *kbdev) ++{ ++ int err = -EINVAL; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ WARN_ONCE(!kbdev->protected_ops, ++ "Cannot enter protected mode: protected callbacks not specified.\n"); ++ ++ if (kbdev->protected_ops) { ++ /* Switch GPU to protected mode */ ++ err = kbdev->protected_ops->protected_mode_enable( ++ kbdev->protected_dev); ++ ++ if (err) { ++ dev_warn(kbdev->dev, "Failed to enable protected mode: %d\n", ++ err); ++ } else { ++ kbdev->protected_mode = true; ++ kbase_ipa_protection_mode_switch_event(kbdev); ++ } ++ } ++ ++ return err; ++} ++ ++static int kbase_gpu_protected_mode_reset(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ WARN_ONCE(!kbdev->protected_ops, ++ "Cannot exit protected mode: protected callbacks not specified.\n"); ++ ++ if (!kbdev->protected_ops) ++ return -EINVAL; ++ ++ /* The protected mode disable callback will be called as part of reset ++ */ ++ return kbase_reset_gpu_silent(kbdev); ++} ++ ++static int kbase_jm_protected_entry(struct kbase_device *kbdev, ++ struct kbase_jd_atom **katom, int idx, int js) ++{ ++ int err = 0; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ err = kbase_gpu_protected_mode_enter(kbdev); ++ ++ /* ++ * Regardless of result before this call, we are no longer ++ * transitioning the GPU. ++ */ ++ ++ kbdev->protected_mode_transition = false; ++ kbase_pm_protected_override_disable(kbdev); ++ kbase_pm_update_cores_state_nolock(kbdev); ++ ++ KBASE_TLSTREAM_AUX_PROTECTED_ENTER_END(kbdev, kbdev); ++ if (err) { ++ /* ++ * Failed to switch into protected mode, resume ++ * GPU hwcnt and fail atom. ++ */ ++ WARN_ON(!kbdev->protected_mode_hwcnt_disabled); ++ kbdev->protected_mode_hwcnt_desired = true; ++ if (kbdev->protected_mode_hwcnt_disabled) { ++ kbase_hwcnt_context_enable( ++ kbdev->hwcnt_gpu_ctx); ++ kbdev->protected_mode_hwcnt_disabled = false; ++ } ++ ++ katom[idx]->event_code = BASE_JD_EVENT_JOB_INVALID; ++ kbase_gpu_mark_atom_for_return(kbdev, katom[idx]); ++ /* ++ * Only return if head atom or previous atom ++ * already removed - as atoms must be returned ++ * in order. ++ */ ++ if (idx == 0 || katom[0]->gpu_rb_state == ++ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { ++ kbase_gpu_dequeue_atom(kbdev, js, NULL); ++ kbase_jm_return_atom_to_js(kbdev, katom[idx]); ++ } ++ ++ return -EINVAL; ++ } ++ ++ /* ++ * Protected mode sanity checks. ++ */ ++ KBASE_DEBUG_ASSERT_MSG( ++ kbase_jd_katom_is_protected(katom[idx]) == ++ kbase_gpu_in_protected_mode(kbdev), ++ "Protected mode of atom (%d) doesn't match protected mode of GPU (%d)", ++ kbase_jd_katom_is_protected(katom[idx]), ++ kbase_gpu_in_protected_mode(kbdev)); ++ katom[idx]->gpu_rb_state = ++ KBASE_ATOM_GPU_RB_READY; ++ ++ return err; ++} ++ ++static int kbase_jm_enter_protected_mode(struct kbase_device *kbdev, ++ struct kbase_jd_atom **katom, int idx, int js) ++{ ++ int err = 0; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ switch (katom[idx]->protected_state.enter) { ++ case KBASE_ATOM_ENTER_PROTECTED_CHECK: ++ KBASE_TLSTREAM_AUX_PROTECTED_ENTER_START(kbdev, kbdev); ++ /* The checks in KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV ++ * should ensure that we are not already transitiong, and that ++ * there are no atoms currently on the GPU. */ ++ WARN_ON(kbdev->protected_mode_transition); ++ WARN_ON(kbase_gpu_atoms_submitted_any(kbdev)); ++ /* If hwcnt is disabled, it means we didn't clean up correctly ++ * during last exit from protected mode. ++ */ ++ WARN_ON(kbdev->protected_mode_hwcnt_disabled); ++ ++ katom[idx]->protected_state.enter = ++ KBASE_ATOM_ENTER_PROTECTED_HWCNT; ++ ++ kbdev->protected_mode_transition = true; ++ ++ /* ***TRANSITION TO HIGHER STATE*** */ ++ /* fallthrough */ ++ case KBASE_ATOM_ENTER_PROTECTED_HWCNT: ++ /* See if we can get away with disabling hwcnt atomically */ ++ kbdev->protected_mode_hwcnt_desired = false; ++ if (!kbdev->protected_mode_hwcnt_disabled) { ++ if (kbase_hwcnt_context_disable_atomic( ++ kbdev->hwcnt_gpu_ctx)) ++ kbdev->protected_mode_hwcnt_disabled = true; ++ } ++ ++ /* We couldn't disable atomically, so kick off a worker */ ++ if (!kbdev->protected_mode_hwcnt_disabled) { ++#if KERNEL_VERSION(3, 16, 0) > LINUX_VERSION_CODE ++ queue_work(system_wq, ++ &kbdev->protected_mode_hwcnt_disable_work); ++#else ++ queue_work(system_highpri_wq, ++ &kbdev->protected_mode_hwcnt_disable_work); ++#endif ++ return -EAGAIN; ++ } ++ ++ /* Once reaching this point GPU must be ++ * switched to protected mode or hwcnt ++ * re-enabled. */ ++ ++ if (kbase_pm_protected_entry_override_enable(kbdev)) ++ return -EAGAIN; ++ ++ /* ++ * Not in correct mode, begin protected mode switch. ++ * Entering protected mode requires us to power down the L2, ++ * and drop out of fully coherent mode. ++ */ ++ katom[idx]->protected_state.enter = ++ KBASE_ATOM_ENTER_PROTECTED_IDLE_L2; ++ ++ kbase_pm_protected_override_enable(kbdev); ++ /* ++ * Only if the GPU reset hasn't been initiated, there is a need ++ * to invoke the state machine to explicitly power down the ++ * shader cores and L2. ++ */ ++ if (!kbdev->pm.backend.protected_entry_transition_override) ++ kbase_pm_update_cores_state_nolock(kbdev); ++ ++ /* ***TRANSITION TO HIGHER STATE*** */ ++ /* fallthrough */ ++ case KBASE_ATOM_ENTER_PROTECTED_IDLE_L2: ++ /* Avoid unnecessary waiting on non-ACE platforms. */ ++ if (kbdev->system_coherency == COHERENCY_ACE) { ++ if (kbdev->pm.backend.l2_always_on) { ++ /* ++ * If the GPU reset hasn't completed, then L2 ++ * could still be powered up. ++ */ ++ if (kbase_reset_gpu_is_active(kbdev)) ++ return -EAGAIN; ++ } ++ ++ if (kbase_pm_get_ready_cores(kbdev, ++ KBASE_PM_CORE_L2) || ++ kbase_pm_get_trans_cores(kbdev, ++ KBASE_PM_CORE_L2) || ++ kbase_is_gpu_removed(kbdev)) { ++ /* ++ * The L2 is still powered, wait for all ++ * the users to finish with it before doing ++ * the actual reset. ++ */ ++ return -EAGAIN; ++ } ++ } ++ ++ katom[idx]->protected_state.enter = ++ KBASE_ATOM_ENTER_PROTECTED_SET_COHERENCY; ++ ++ /* ***TRANSITION TO HIGHER STATE*** */ ++ /* fallthrough */ ++ case KBASE_ATOM_ENTER_PROTECTED_SET_COHERENCY: ++ /* ++ * When entering into protected mode, we must ensure that the ++ * GPU is not operating in coherent mode as well. This is to ++ * ensure that no protected memory can be leaked. ++ */ ++ kbase_gpu_disable_coherent(kbdev); ++ ++ kbase_pm_protected_entry_override_disable(kbdev); ++ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TGOX_R1_1234)) { ++ /* ++ * Power on L2 caches; this will also result in the ++ * correct value written to coherency enable register. ++ */ ++ kbase_pm_protected_l2_override(kbdev, true); ++ ++ /* ++ * Set the flag on the atom that additional ++ * L2 references are taken. ++ */ ++ katom[idx]->atom_flags |= ++ KBASE_KATOM_FLAG_HOLDING_L2_REF_PROT; ++ } ++ ++ katom[idx]->protected_state.enter = ++ KBASE_ATOM_ENTER_PROTECTED_FINISHED; ++ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TGOX_R1_1234)) ++ return -EAGAIN; ++ ++ /* ***TRANSITION TO HIGHER STATE*** */ ++ /* fallthrough */ ++ case KBASE_ATOM_ENTER_PROTECTED_FINISHED: ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TGOX_R1_1234)) { ++ /* ++ * Check that L2 caches are powered and, if so, ++ * enter protected mode. ++ */ ++ if (kbdev->pm.backend.l2_state == KBASE_L2_ON) { ++ /* ++ * Remove additional L2 reference and reset ++ * the atom flag which denotes it. ++ */ ++ if (katom[idx]->atom_flags & ++ KBASE_KATOM_FLAG_HOLDING_L2_REF_PROT) { ++ kbase_pm_protected_l2_override(kbdev, ++ false); ++ katom[idx]->atom_flags &= ++ ~KBASE_KATOM_FLAG_HOLDING_L2_REF_PROT; ++ } ++ ++ err = kbase_jm_protected_entry(kbdev, katom, idx, js); ++ ++ if (err) ++ return err; ++ } else { ++ /* ++ * still waiting for L2 caches to power up ++ */ ++ return -EAGAIN; ++ } ++ } else { ++ err = kbase_jm_protected_entry(kbdev, katom, idx, js); ++ ++ if (err) ++ return err; ++ } ++ } ++ ++ return 0; ++} ++ ++static int kbase_jm_exit_protected_mode(struct kbase_device *kbdev, ++ struct kbase_jd_atom **katom, int idx, int js) ++{ ++ int err = 0; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ switch (katom[idx]->protected_state.exit) { ++ case KBASE_ATOM_EXIT_PROTECTED_CHECK: ++ KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_START(kbdev, kbdev); ++ /* The checks in KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV ++ * should ensure that we are not already transitiong, and that ++ * there are no atoms currently on the GPU. */ ++ WARN_ON(kbdev->protected_mode_transition); ++ WARN_ON(kbase_gpu_atoms_submitted_any(kbdev)); ++ ++ /* ++ * Exiting protected mode requires a reset, but first the L2 ++ * needs to be powered down to ensure it's not active when the ++ * reset is issued. ++ */ ++ katom[idx]->protected_state.exit = ++ KBASE_ATOM_EXIT_PROTECTED_IDLE_L2; ++ ++ kbdev->protected_mode_transition = true; ++ kbase_pm_protected_override_enable(kbdev); ++ kbase_pm_update_cores_state_nolock(kbdev); ++ ++ /* ***TRANSITION TO HIGHER STATE*** */ ++ /* fallthrough */ ++ case KBASE_ATOM_EXIT_PROTECTED_IDLE_L2: ++ if (kbdev->pm.backend.l2_state != KBASE_L2_OFF) { ++ /* ++ * The L2 is still powered, wait for all the users to ++ * finish with it before doing the actual reset. ++ */ ++ return -EAGAIN; ++ } ++ katom[idx]->protected_state.exit = ++ KBASE_ATOM_EXIT_PROTECTED_RESET; ++ ++ /* ***TRANSITION TO HIGHER STATE*** */ ++ /* fallthrough */ ++ case KBASE_ATOM_EXIT_PROTECTED_RESET: ++ /* Issue the reset to the GPU */ ++ err = kbase_gpu_protected_mode_reset(kbdev); ++ ++ if (err == -EAGAIN) ++ return -EAGAIN; ++ ++ if (err) { ++ kbdev->protected_mode_transition = false; ++ kbase_pm_protected_override_disable(kbdev); ++ ++ /* Failed to exit protected mode, fail atom */ ++ katom[idx]->event_code = BASE_JD_EVENT_JOB_INVALID; ++ kbase_gpu_mark_atom_for_return(kbdev, katom[idx]); ++ /* Only return if head atom or previous atom ++ * already removed - as atoms must be returned ++ * in order */ ++ if (idx == 0 || katom[0]->gpu_rb_state == ++ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { ++ kbase_gpu_dequeue_atom(kbdev, js, NULL); ++ kbase_jm_return_atom_to_js(kbdev, katom[idx]); ++ } ++ ++ /* If we're exiting from protected mode, hwcnt must have ++ * been disabled during entry. ++ */ ++ WARN_ON(!kbdev->protected_mode_hwcnt_disabled); ++ kbdev->protected_mode_hwcnt_desired = true; ++ if (kbdev->protected_mode_hwcnt_disabled) { ++ kbase_hwcnt_context_enable( ++ kbdev->hwcnt_gpu_ctx); ++ kbdev->protected_mode_hwcnt_disabled = false; ++ } ++ ++ return -EINVAL; ++ } ++ ++ katom[idx]->protected_state.exit = ++ KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT; ++ ++ /* ***TRANSITION TO HIGHER STATE*** */ ++ /* fallthrough */ ++ case KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT: ++ /* A GPU reset is issued when exiting protected mode. Once the ++ * reset is done all atoms' state will also be reset. For this ++ * reason, if the atom is still in this state we can safely ++ * say that the reset has not completed i.e., we have not ++ * finished exiting protected mode yet. ++ */ ++ return -EAGAIN; ++ } ++ ++ return 0; ++} ++ ++void kbase_backend_slot_update(struct kbase_device *kbdev) ++{ ++ int js; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++ if (kbase_reset_gpu_is_active(kbdev) || ++ kbase_is_gpu_removed(kbdev)) ++#else ++ if (kbase_reset_gpu_is_active(kbdev)) ++#endif ++ return; ++ ++ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { ++ struct kbase_jd_atom *katom[2]; ++ int idx; ++ ++ katom[0] = kbase_gpu_inspect(kbdev, js, 0); ++ katom[1] = kbase_gpu_inspect(kbdev, js, 1); ++ WARN_ON(katom[1] && !katom[0]); ++ ++ for (idx = 0; idx < SLOT_RB_SIZE; idx++) { ++ bool cores_ready; ++ int ret; ++ ++ if (!katom[idx]) ++ continue; ++ ++ switch (katom[idx]->gpu_rb_state) { ++ case KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB: ++ /* Should be impossible */ ++ WARN(1, "Attempting to update atom not in ringbuffer\n"); ++ break; ++ ++ case KBASE_ATOM_GPU_RB_WAITING_BLOCKED: ++ if (kbase_js_atom_blocked_on_x_dep(katom[idx])) ++ break; ++ ++ katom[idx]->gpu_rb_state = ++ KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV; ++ ++ /* ***TRANSITION TO HIGHER STATE*** */ ++ /* fallthrough */ ++ case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV: ++ if (kbase_gpu_check_secure_atoms(kbdev, ++ !kbase_jd_katom_is_protected( ++ katom[idx]))) ++ break; ++ ++ if ((idx == 1) && (kbase_jd_katom_is_protected( ++ katom[0]) != ++ kbase_jd_katom_is_protected( ++ katom[1]))) ++ break; ++ ++ if (kbdev->protected_mode_transition) ++ break; ++ ++ katom[idx]->gpu_rb_state = ++ KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION; ++ ++ /* ***TRANSITION TO HIGHER STATE*** */ ++ /* fallthrough */ ++ case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION: ++ ++ /* ++ * Exiting protected mode must be done before ++ * the references on the cores are taken as ++ * a power down the L2 is required which ++ * can't happen after the references for this ++ * atom are taken. ++ */ ++ ++ if (!kbase_gpu_in_protected_mode(kbdev) && ++ kbase_jd_katom_is_protected(katom[idx])) { ++ /* Atom needs to transition into protected mode. */ ++ ret = kbase_jm_enter_protected_mode(kbdev, ++ katom, idx, js); ++ if (ret) ++ break; ++ } else if (kbase_gpu_in_protected_mode(kbdev) && ++ !kbase_jd_katom_is_protected(katom[idx])) { ++ /* Atom needs to transition out of protected mode. */ ++ ret = kbase_jm_exit_protected_mode(kbdev, ++ katom, idx, js); ++ if (ret) ++ break; ++ } ++ katom[idx]->protected_state.exit = ++ KBASE_ATOM_EXIT_PROTECTED_CHECK; ++ ++ /* Atom needs no protected mode transition. */ ++ ++ katom[idx]->gpu_rb_state = ++ KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE; ++ ++ /* ***TRANSITION TO HIGHER STATE*** */ ++ /* fallthrough */ ++ case KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE: ++ if (katom[idx]->will_fail_event_code) { ++ kbase_gpu_mark_atom_for_return(kbdev, ++ katom[idx]); ++ /* Set EVENT_DONE so this atom will be ++ completed, not unpulled. */ ++ katom[idx]->event_code = ++ BASE_JD_EVENT_DONE; ++ /* Only return if head atom or previous ++ * atom already removed - as atoms must ++ * be returned in order. */ ++ if (idx == 0 || katom[0]->gpu_rb_state == ++ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { ++ kbase_gpu_dequeue_atom(kbdev, js, NULL); ++ kbase_jm_return_atom_to_js(kbdev, katom[idx]); ++ } ++ break; ++ } ++ ++ cores_ready = kbase_pm_cores_requested(kbdev, ++ true); ++ ++ if (katom[idx]->event_code == ++ BASE_JD_EVENT_PM_EVENT) { ++ katom[idx]->gpu_rb_state = ++ KBASE_ATOM_GPU_RB_RETURN_TO_JS; ++ break; ++ } ++ ++ if (!cores_ready) ++ break; ++ ++ katom[idx]->gpu_rb_state = ++ KBASE_ATOM_GPU_RB_READY; ++ ++ /* ***TRANSITION TO HIGHER STATE*** */ ++ /* fallthrough */ ++ case KBASE_ATOM_GPU_RB_READY: ++ ++ if (idx == 1) { ++ /* Only submit if head atom or previous ++ * atom already submitted */ ++ if ((katom[0]->gpu_rb_state != ++ KBASE_ATOM_GPU_RB_SUBMITTED && ++ katom[0]->gpu_rb_state != ++ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB)) ++ break; ++ ++ /* If intra-slot serialization in use ++ * then don't submit atom to NEXT slot ++ */ ++ if (kbdev->serialize_jobs & ++ KBASE_SERIALIZE_INTRA_SLOT) ++ break; ++ } ++ ++ /* If inter-slot serialization in use then don't ++ * submit atom if any other slots are in use */ ++ if ((kbdev->serialize_jobs & ++ KBASE_SERIALIZE_INTER_SLOT) && ++ other_slots_busy(kbdev, js)) ++ break; ++ ++#ifdef CONFIG_MALI_GEM5_BUILD ++ if (!kbasep_jm_is_js_free(kbdev, js, ++ katom[idx]->kctx)) ++ break; ++#endif ++ /* Check if this job needs the cycle counter ++ * enabled before submission */ ++ if (katom[idx]->core_req & BASE_JD_REQ_PERMON) ++ kbase_pm_request_gpu_cycle_counter_l2_is_on( ++ kbdev); ++ ++ kbase_job_hw_submit(kbdev, katom[idx], js); ++ katom[idx]->gpu_rb_state = ++ KBASE_ATOM_GPU_RB_SUBMITTED; ++ ++ /* Inform power management at start/finish of ++ * atom so it can update its GPU utilisation ++ * metrics. */ ++ kbase_pm_metrics_update(kbdev, ++ &katom[idx]->start_timestamp); ++ ++ /* ***TRANSITION TO HIGHER STATE*** */ ++ /* fallthrough */ ++ case KBASE_ATOM_GPU_RB_SUBMITTED: ++ /* Atom submitted to HW, nothing else to do */ ++ break; ++ ++ case KBASE_ATOM_GPU_RB_RETURN_TO_JS: ++ /* Only return if head atom or previous atom ++ * already removed - as atoms must be returned ++ * in order */ ++ if (idx == 0 || katom[0]->gpu_rb_state == ++ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { ++ kbase_gpu_dequeue_atom(kbdev, js, NULL); ++ kbase_jm_return_atom_to_js(kbdev, ++ katom[idx]); ++ } ++ break; ++ } ++ } ++ } ++} ++ ++ ++void kbase_backend_run_atom(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ dev_dbg(kbdev->dev, "Backend running atom %p\n", (void *)katom); ++ ++ kbase_gpu_enqueue_atom(kbdev, katom); ++ kbase_backend_slot_update(kbdev); ++} ++ ++#define HAS_DEP(katom) (katom->pre_dep || katom->atom_flags & \ ++ (KBASE_KATOM_FLAG_X_DEP_BLOCKED | KBASE_KATOM_FLAG_FAIL_BLOCKER)) ++ ++bool kbase_gpu_irq_evict(struct kbase_device *kbdev, int js, ++ u32 completion_code) ++{ ++ struct kbase_jd_atom *katom; ++ struct kbase_jd_atom *next_katom; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ katom = kbase_gpu_inspect(kbdev, js, 0); ++ next_katom = kbase_gpu_inspect(kbdev, js, 1); ++ ++ if (next_katom && katom->kctx == next_katom->kctx && ++ next_katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED && ++ (HAS_DEP(next_katom) || next_katom->sched_priority == ++ katom->sched_priority) && ++ (kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_LO)) ++ != 0 || ++ kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_HI)) ++ != 0)) { ++ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT), ++ JS_COMMAND_NOP); ++ next_katom->gpu_rb_state = KBASE_ATOM_GPU_RB_READY; ++ ++ if (completion_code == BASE_JD_EVENT_STOPPED) { ++ KBASE_TLSTREAM_TL_NRET_ATOM_LPU(kbdev, next_katom, ++ &kbdev->gpu_props.props.raw_props.js_features ++ [next_katom->slot_nr]); ++ KBASE_TLSTREAM_TL_NRET_ATOM_AS(kbdev, next_katom, &kbdev->as ++ [next_katom->kctx->as_nr]); ++ KBASE_TLSTREAM_TL_NRET_CTX_LPU(kbdev, next_katom->kctx, ++ &kbdev->gpu_props.props.raw_props.js_features ++ [next_katom->slot_nr]); ++ } ++ ++ if (next_katom->core_req & BASE_JD_REQ_PERMON) ++ kbase_pm_release_gpu_cycle_counter_nolock(kbdev); ++ ++ return true; ++ } ++ ++ return false; ++} ++ ++void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js, ++ u32 completion_code, ++ u64 job_tail, ++ ktime_t *end_timestamp) ++{ ++ struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, 0); ++ struct kbase_context *kctx = katom->kctx; ++ ++ dev_dbg(kbdev->dev, ++ "Atom %p completed on hw with code 0x%x and job_tail 0x%llx (s:%d)\n", ++ (void *)katom, completion_code, job_tail, js); ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ /* ++ * When a hard-stop is followed close after a soft-stop, the completion ++ * code may be set to STOPPED, even though the job is terminated ++ */ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TMIX_8438)) { ++ if (completion_code == BASE_JD_EVENT_STOPPED && ++ (katom->atom_flags & ++ KBASE_KATOM_FLAG_BEEN_HARD_STOPPED)) { ++ completion_code = BASE_JD_EVENT_TERMINATED; ++ } ++ } ++ ++ if ((katom->core_req & BASE_JD_REQ_SKIP_CACHE_END) && ++ completion_code != BASE_JD_EVENT_DONE && ++ !(completion_code & BASE_JD_SW_EVENT)) { ++ /* When a job chain fails, on a T60x or when ++ * BASE_JD_REQ_SKIP_CACHE_END is set, the GPU cache is not ++ * flushed. To prevent future evictions causing possible memory ++ * corruption we need to flush the cache manually before any ++ * affected memory gets reused. */ ++ katom->need_cache_flush_cores_retained = true; ++ } ++ ++ katom = kbase_gpu_dequeue_atom(kbdev, js, end_timestamp); ++ ++ if (completion_code == BASE_JD_EVENT_STOPPED) { ++ struct kbase_jd_atom *next_katom = kbase_gpu_inspect(kbdev, js, ++ 0); ++ ++ /* ++ * Dequeue next atom from ringbuffers on same slot if required. ++ * This atom will already have been removed from the NEXT ++ * registers by kbase_gpu_soft_hard_stop_slot(), to ensure that ++ * the atoms on this slot are returned in the correct order. ++ */ ++ if (next_katom && katom->kctx == next_katom->kctx && ++ next_katom->sched_priority == ++ katom->sched_priority) { ++ WARN_ON(next_katom->gpu_rb_state == ++ KBASE_ATOM_GPU_RB_SUBMITTED); ++ kbase_gpu_dequeue_atom(kbdev, js, end_timestamp); ++ kbase_jm_return_atom_to_js(kbdev, next_katom); ++ } ++ } else if (completion_code != BASE_JD_EVENT_DONE) { ++ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; ++ int i; ++ ++ if (!kbase_ctx_flag(katom->kctx, KCTX_DYING)) ++ dev_warn(kbdev->dev, "error detected from slot %d, job status 0x%08x (%s)", ++ js, completion_code, ++ kbase_gpu_exception_name( ++ completion_code)); ++ ++#if KBASE_KTRACE_DUMP_ON_JOB_SLOT_ERROR != 0 ++ KBASE_KTRACE_DUMP(kbdev); ++#endif ++ kbasep_js_clear_submit_allowed(js_devdata, katom->kctx); ++ ++ /* ++ * Remove all atoms on the same context from ringbuffers. This ++ * will not remove atoms that are already on the GPU, as these ++ * are guaranteed not to have fail dependencies on the failed ++ * atom. ++ */ ++ for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) { ++ struct kbase_jd_atom *katom_idx0 = ++ kbase_gpu_inspect(kbdev, i, 0); ++ struct kbase_jd_atom *katom_idx1 = ++ kbase_gpu_inspect(kbdev, i, 1); ++ ++ if (katom_idx0 && katom_idx0->kctx == katom->kctx && ++ HAS_DEP(katom_idx0) && ++ katom_idx0->gpu_rb_state != ++ KBASE_ATOM_GPU_RB_SUBMITTED) { ++ /* Dequeue katom_idx0 from ringbuffer */ ++ kbase_gpu_dequeue_atom(kbdev, i, end_timestamp); ++ ++ if (katom_idx1 && ++ katom_idx1->kctx == katom->kctx ++ && HAS_DEP(katom_idx1) && ++ katom_idx0->gpu_rb_state != ++ KBASE_ATOM_GPU_RB_SUBMITTED) { ++ /* Dequeue katom_idx1 from ringbuffer */ ++ kbase_gpu_dequeue_atom(kbdev, i, ++ end_timestamp); ++ ++ katom_idx1->event_code = ++ BASE_JD_EVENT_STOPPED; ++ kbase_jm_return_atom_to_js(kbdev, ++ katom_idx1); ++ } ++ katom_idx0->event_code = BASE_JD_EVENT_STOPPED; ++ kbase_jm_return_atom_to_js(kbdev, katom_idx0); ++ ++ } else if (katom_idx1 && ++ katom_idx1->kctx == katom->kctx && ++ HAS_DEP(katom_idx1) && ++ katom_idx1->gpu_rb_state != ++ KBASE_ATOM_GPU_RB_SUBMITTED) { ++ /* Can not dequeue this atom yet - will be ++ * dequeued when atom at idx0 completes */ ++ katom_idx1->event_code = BASE_JD_EVENT_STOPPED; ++ kbase_gpu_mark_atom_for_return(kbdev, ++ katom_idx1); ++ } ++ } ++ } ++ ++ KBASE_KTRACE_ADD_JM_SLOT_INFO(kbdev, JM_JOB_DONE, kctx, katom, katom->jc, js, completion_code); ++ ++ if (job_tail != 0 && job_tail != katom->jc) { ++ /* Some of the job has been executed */ ++ dev_dbg(kbdev->dev, ++ "Update job chain address of atom %p to resume from 0x%llx\n", ++ (void *)katom, job_tail); ++ ++ katom->jc = job_tail; ++ KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_UPDATE_HEAD, katom->kctx, ++ katom, job_tail, js); ++ } ++ ++ /* Only update the event code for jobs that weren't cancelled */ ++ if (katom->event_code != BASE_JD_EVENT_JOB_CANCELLED) ++ katom->event_code = (enum base_jd_event_code)completion_code; ++ ++ /* Complete the job, and start new ones ++ * ++ * Also defer remaining work onto the workqueue: ++ * - Re-queue Soft-stopped jobs ++ * - For any other jobs, queue the job back into the dependency system ++ * - Schedule out the parent context if necessary, and schedule a new ++ * one in. ++ */ ++#ifdef CONFIG_GPU_TRACEPOINTS ++ { ++ /* The atom in the HEAD */ ++ struct kbase_jd_atom *next_katom = kbase_gpu_inspect(kbdev, js, ++ 0); ++ ++ if (next_katom && next_katom->gpu_rb_state == ++ KBASE_ATOM_GPU_RB_SUBMITTED) { ++ char js_string[16]; ++ ++ trace_gpu_sched_switch(kbasep_make_job_slot_string(js, ++ js_string, ++ sizeof(js_string)), ++ ktime_to_ns(*end_timestamp), ++ (u32)next_katom->kctx->id, 0, ++ next_katom->work_id); ++ kbdev->hwaccess.backend.slot_rb[js].last_context = ++ next_katom->kctx; ++ } else { ++ char js_string[16]; ++ ++ trace_gpu_sched_switch(kbasep_make_job_slot_string(js, ++ js_string, ++ sizeof(js_string)), ++ ktime_to_ns(ktime_get()), 0, 0, ++ 0); ++ kbdev->hwaccess.backend.slot_rb[js].last_context = 0; ++ } ++ } ++#endif ++ ++ if (kbdev->serialize_jobs & KBASE_SERIALIZE_RESET) ++ kbase_reset_gpu_silent(kbdev); ++ ++ if (completion_code == BASE_JD_EVENT_STOPPED) ++ katom = kbase_jm_return_atom_to_js(kbdev, katom); ++ else ++ katom = kbase_jm_complete(kbdev, katom, end_timestamp); ++ ++ if (katom) { ++ dev_dbg(kbdev->dev, ++ "Cross-slot dependency %p has become runnable.\n", ++ (void *)katom); ++ ++ /* Check if there are lower priority jobs to soft stop */ ++ kbase_job_slot_ctx_priority_check_locked(kctx, katom); ++ ++ kbase_jm_try_kick(kbdev, 1 << katom->slot_nr); ++ } ++ ++ /* For partial shader core off L2 cache flush */ ++ kbase_pm_update_state(kbdev); ++ ++ /* Job completion may have unblocked other atoms. Try to update all job ++ * slots */ ++ kbase_backend_slot_update(kbdev); ++} ++ ++void kbase_backend_reset(struct kbase_device *kbdev, ktime_t *end_timestamp) ++{ ++ int js; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ /* Reset should always take the GPU out of protected mode */ ++ WARN_ON(kbase_gpu_in_protected_mode(kbdev)); ++ ++ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { ++ int atom_idx = 0; ++ int idx; ++ ++ for (idx = 0; idx < SLOT_RB_SIZE; idx++) { ++ struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, ++ js, atom_idx); ++ bool keep_in_jm_rb = false; ++ ++ if (!katom) ++ break; ++ if (katom->protected_state.exit == ++ KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT) { ++ /* protected mode sanity checks */ ++ KBASE_DEBUG_ASSERT_MSG( ++ kbase_jd_katom_is_protected(katom) == kbase_gpu_in_protected_mode(kbdev), ++ "Protected mode of atom (%d) doesn't match protected mode of GPU (%d)", ++ kbase_jd_katom_is_protected(katom), kbase_gpu_in_protected_mode(kbdev)); ++ KBASE_DEBUG_ASSERT_MSG( ++ (kbase_jd_katom_is_protected(katom) && js == 0) || ++ !kbase_jd_katom_is_protected(katom), ++ "Protected atom on JS%d not supported", js); ++ } ++ if ((katom->gpu_rb_state < KBASE_ATOM_GPU_RB_SUBMITTED) && ++ !kbase_ctx_flag(katom->kctx, KCTX_DYING)) ++ keep_in_jm_rb = true; ++ ++ kbase_gpu_release_atom(kbdev, katom, NULL); ++ ++ /* ++ * If the atom wasn't on HW when the reset was issued ++ * then leave it in the RB and next time we're kicked ++ * it will be processed again from the starting state. ++ */ ++ if (keep_in_jm_rb) { ++ katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK; ++ /* As the atom was not removed, increment the ++ * index so that we read the correct atom in the ++ * next iteration. */ ++ atom_idx++; ++ continue; ++ } ++ ++ /* ++ * The atom was on the HW when the reset was issued ++ * all we can do is fail the atom. ++ */ ++ kbase_gpu_dequeue_atom(kbdev, js, NULL); ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ kbase_jm_complete(kbdev, katom, end_timestamp); ++ } ++ } ++ ++ /* Re-enable GPU hardware counters if we're resetting from protected ++ * mode. ++ */ ++ kbdev->protected_mode_hwcnt_desired = true; ++ if (kbdev->protected_mode_hwcnt_disabled) { ++ kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); ++ kbdev->protected_mode_hwcnt_disabled = false; ++ ++ KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_END(kbdev, kbdev); ++ } ++ ++ kbdev->protected_mode_transition = false; ++ kbase_pm_protected_override_disable(kbdev); ++} ++ ++static inline void kbase_gpu_stop_atom(struct kbase_device *kbdev, ++ int js, ++ struct kbase_jd_atom *katom, ++ u32 action) ++{ ++ u32 hw_action = action & JS_COMMAND_MASK; ++ ++ kbase_job_check_enter_disjoint(kbdev, action, katom->core_req, katom); ++ kbasep_job_slot_soft_or_hard_stop_do_action(kbdev, js, hw_action, ++ katom->core_req, katom); ++ katom->kctx->blocked_js[js][katom->sched_priority] = true; ++} ++ ++static inline void kbase_gpu_remove_atom(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom, ++ u32 action, ++ bool disjoint) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ katom->event_code = BASE_JD_EVENT_REMOVED_FROM_NEXT; ++ kbase_gpu_mark_atom_for_return(kbdev, katom); ++ katom->kctx->blocked_js[katom->slot_nr][katom->sched_priority] = true; ++ ++ if (disjoint) ++ kbase_job_check_enter_disjoint(kbdev, action, katom->core_req, ++ katom); ++} ++ ++static int should_stop_x_dep_slot(struct kbase_jd_atom *katom) ++{ ++ if (katom->x_post_dep) { ++ struct kbase_jd_atom *dep_atom = katom->x_post_dep; ++ ++ if (dep_atom->gpu_rb_state != ++ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB && ++ dep_atom->gpu_rb_state != ++ KBASE_ATOM_GPU_RB_RETURN_TO_JS) ++ return dep_atom->slot_nr; ++ } ++ return -1; ++} ++ ++bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ int js, ++ struct kbase_jd_atom *katom, ++ u32 action) ++{ ++ struct kbase_jd_atom *katom_idx0; ++ struct kbase_jd_atom *katom_idx1; ++ ++ bool katom_idx0_valid, katom_idx1_valid; ++ ++ bool ret = false; ++ ++ int stop_x_dep_idx0 = -1, stop_x_dep_idx1 = -1; ++ int prio_idx0 = 0, prio_idx1 = 0; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ katom_idx0 = kbase_gpu_inspect(kbdev, js, 0); ++ katom_idx1 = kbase_gpu_inspect(kbdev, js, 1); ++ ++ if (katom_idx0) ++ prio_idx0 = katom_idx0->sched_priority; ++ if (katom_idx1) ++ prio_idx1 = katom_idx1->sched_priority; ++ ++ if (katom) { ++ katom_idx0_valid = (katom_idx0 == katom); ++ /* If idx0 is to be removed and idx1 is on the same context, ++ * then idx1 must also be removed otherwise the atoms might be ++ * returned out of order */ ++ if (katom_idx1) ++ katom_idx1_valid = (katom_idx1 == katom) || ++ (katom_idx0_valid && ++ (katom_idx0->kctx == ++ katom_idx1->kctx)); ++ else ++ katom_idx1_valid = false; ++ } else { ++ katom_idx0_valid = (katom_idx0 && ++ (!kctx || katom_idx0->kctx == kctx)); ++ katom_idx1_valid = (katom_idx1 && ++ (!kctx || katom_idx1->kctx == kctx) && ++ prio_idx0 == prio_idx1); ++ } ++ ++ if (katom_idx0_valid) ++ stop_x_dep_idx0 = should_stop_x_dep_slot(katom_idx0); ++ if (katom_idx1_valid) ++ stop_x_dep_idx1 = should_stop_x_dep_slot(katom_idx1); ++ ++ if (katom_idx0_valid) { ++ if (katom_idx0->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) { ++ /* Simple case - just dequeue and return */ ++ kbase_gpu_dequeue_atom(kbdev, js, NULL); ++ if (katom_idx1_valid) { ++ kbase_gpu_dequeue_atom(kbdev, js, NULL); ++ katom_idx1->event_code = ++ BASE_JD_EVENT_REMOVED_FROM_NEXT; ++ kbase_jm_return_atom_to_js(kbdev, katom_idx1); ++ katom_idx1->kctx->blocked_js[js][prio_idx1] = ++ true; ++ } ++ ++ katom_idx0->event_code = ++ BASE_JD_EVENT_REMOVED_FROM_NEXT; ++ kbase_jm_return_atom_to_js(kbdev, katom_idx0); ++ katom_idx0->kctx->blocked_js[js][prio_idx0] = true; ++ } else { ++ /* katom_idx0 is on GPU */ ++ if (katom_idx1_valid && katom_idx1->gpu_rb_state == ++ KBASE_ATOM_GPU_RB_SUBMITTED) { ++ /* katom_idx0 and katom_idx1 are on GPU */ ++ ++ if (kbase_reg_read(kbdev, JOB_SLOT_REG(js, ++ JS_COMMAND_NEXT)) == 0) { ++ /* idx0 has already completed - stop ++ * idx1 if needed*/ ++ if (katom_idx1_valid) { ++ kbase_gpu_stop_atom(kbdev, js, ++ katom_idx1, ++ action); ++ ret = true; ++ } ++ } else { ++ /* idx1 is in NEXT registers - attempt ++ * to remove */ ++ kbase_reg_write(kbdev, ++ JOB_SLOT_REG(js, ++ JS_COMMAND_NEXT), ++ JS_COMMAND_NOP); ++ ++ if (kbase_reg_read(kbdev, ++ JOB_SLOT_REG(js, ++ JS_HEAD_NEXT_LO)) ++ != 0 || ++ kbase_reg_read(kbdev, ++ JOB_SLOT_REG(js, ++ JS_HEAD_NEXT_HI)) ++ != 0) { ++ /* idx1 removed successfully, ++ * will be handled in IRQ */ ++ kbase_gpu_remove_atom(kbdev, ++ katom_idx1, ++ action, true); ++ stop_x_dep_idx1 = ++ should_stop_x_dep_slot(katom_idx1); ++ ++ /* stop idx0 if still on GPU */ ++ kbase_gpu_stop_atom(kbdev, js, ++ katom_idx0, ++ action); ++ ret = true; ++ } else if (katom_idx1_valid) { ++ /* idx0 has already completed, ++ * stop idx1 if needed */ ++ kbase_gpu_stop_atom(kbdev, js, ++ katom_idx1, ++ action); ++ ret = true; ++ } ++ } ++ } else if (katom_idx1_valid) { ++ /* idx1 not on GPU but must be dequeued*/ ++ ++ /* idx1 will be handled in IRQ */ ++ kbase_gpu_remove_atom(kbdev, katom_idx1, action, ++ false); ++ /* stop idx0 */ ++ /* This will be repeated for anything removed ++ * from the next registers, since their normal ++ * flow was also interrupted, and this function ++ * might not enter disjoint state e.g. if we ++ * don't actually do a hard stop on the head ++ * atom */ ++ kbase_gpu_stop_atom(kbdev, js, katom_idx0, ++ action); ++ ret = true; ++ } else { ++ /* no atom in idx1 */ ++ /* just stop idx0 */ ++ kbase_gpu_stop_atom(kbdev, js, katom_idx0, ++ action); ++ ret = true; ++ } ++ } ++ } else if (katom_idx1_valid) { ++ if (katom_idx1->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) { ++ /* Mark for return */ ++ /* idx1 will be returned once idx0 completes */ ++ kbase_gpu_remove_atom(kbdev, katom_idx1, action, ++ false); ++ } else { ++ /* idx1 is on GPU */ ++ if (kbase_reg_read(kbdev, JOB_SLOT_REG(js, ++ JS_COMMAND_NEXT)) == 0) { ++ /* idx0 has already completed - stop idx1 */ ++ kbase_gpu_stop_atom(kbdev, js, katom_idx1, ++ action); ++ ret = true; ++ } else { ++ /* idx1 is in NEXT registers - attempt to ++ * remove */ ++ kbase_reg_write(kbdev, JOB_SLOT_REG(js, ++ JS_COMMAND_NEXT), ++ JS_COMMAND_NOP); ++ ++ if (kbase_reg_read(kbdev, JOB_SLOT_REG(js, ++ JS_HEAD_NEXT_LO)) != 0 || ++ kbase_reg_read(kbdev, JOB_SLOT_REG(js, ++ JS_HEAD_NEXT_HI)) != 0) { ++ /* idx1 removed successfully, will be ++ * handled in IRQ once idx0 completes */ ++ kbase_gpu_remove_atom(kbdev, katom_idx1, ++ action, ++ false); ++ } else { ++ /* idx0 has already completed - stop ++ * idx1 */ ++ kbase_gpu_stop_atom(kbdev, js, ++ katom_idx1, ++ action); ++ ret = true; ++ } ++ } ++ } ++ } ++ ++ ++ if (stop_x_dep_idx0 != -1) ++ kbase_backend_soft_hard_stop_slot(kbdev, kctx, stop_x_dep_idx0, ++ NULL, action); ++ ++ if (stop_x_dep_idx1 != -1) ++ kbase_backend_soft_hard_stop_slot(kbdev, kctx, stop_x_dep_idx1, ++ NULL, action); ++ ++ return ret; ++} ++ ++void kbase_backend_cache_clean(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom) ++{ ++ if (katom->need_cache_flush_cores_retained) { ++ kbase_gpu_start_cache_clean(kbdev); ++ kbase_gpu_wait_cache_clean(kbdev); ++ ++ katom->need_cache_flush_cores_retained = false; ++ } ++} ++ ++void kbase_backend_complete_wq(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom) ++{ ++ /* ++ * If cache flush required due to HW workaround then perform the flush ++ * now ++ */ ++ kbase_backend_cache_clean(kbdev, katom); ++} ++ ++void kbase_backend_complete_wq_post_sched(struct kbase_device *kbdev, ++ base_jd_core_req core_req) ++{ ++ if (!kbdev->pm.active_count) { ++ mutex_lock(&kbdev->js_data.runpool_mutex); ++ mutex_lock(&kbdev->pm.lock); ++ kbase_pm_update_active(kbdev); ++ mutex_unlock(&kbdev->pm.lock); ++ mutex_unlock(&kbdev->js_data.runpool_mutex); ++ } ++} ++ ++void kbase_gpu_dump_slots(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ int js; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ dev_info(kbdev->dev, "kbase_gpu_dump_slots:\n"); ++ ++ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { ++ int idx; ++ ++ for (idx = 0; idx < SLOT_RB_SIZE; idx++) { ++ struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, ++ js, ++ idx); ++ ++ if (katom) ++ dev_info(kbdev->dev, ++ " js%d idx%d : katom=%p gpu_rb_state=%d\n", ++ js, idx, katom, katom->gpu_rb_state); ++ else ++ dev_info(kbdev->dev, " js%d idx%d : empty\n", ++ js, idx); ++ } ++ } ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++} +diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_rb.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_rb.h +new file mode 100755 +index 000000000000..c3b9f2d85536 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_rb.h +@@ -0,0 +1,83 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2018 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++/* ++ * Register-based HW access backend specific APIs ++ */ ++ ++#ifndef _KBASE_HWACCESS_GPU_H_ ++#define _KBASE_HWACCESS_GPU_H_ ++ ++#include ++ ++/** ++ * kbase_gpu_irq_evict - Evict an atom from a NEXT slot ++ * ++ * @kbdev: Device pointer ++ * @js: Job slot to evict from ++ * @completion_code: Event code from job that was run. ++ * ++ * Evict the atom in the NEXT slot for the specified job slot. This function is ++ * called from the job complete IRQ handler when the previous job has failed. ++ * ++ * Return: true if job evicted from NEXT registers, false otherwise ++ */ ++bool kbase_gpu_irq_evict(struct kbase_device *kbdev, int js, ++ u32 completion_code); ++ ++/** ++ * kbase_gpu_complete_hw - Complete an atom on job slot js ++ * ++ * @kbdev: Device pointer ++ * @js: Job slot that has completed ++ * @completion_code: Event code from job that has completed ++ * @job_tail: The tail address from the hardware if the job has partially ++ * completed ++ * @end_timestamp: Time of completion ++ */ ++void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js, ++ u32 completion_code, ++ u64 job_tail, ++ ktime_t *end_timestamp); ++ ++/** ++ * kbase_gpu_inspect - Inspect the contents of the HW access ringbuffer ++ * ++ * @kbdev: Device pointer ++ * @js: Job slot to inspect ++ * @idx: Index into ringbuffer. 0 is the job currently running on ++ * the slot, 1 is the job waiting, all other values are invalid. ++ * Return: The atom at that position in the ringbuffer ++ * or NULL if no atom present ++ */ ++struct kbase_jd_atom *kbase_gpu_inspect(struct kbase_device *kbdev, int js, ++ int idx); ++ ++/** ++ * kbase_gpu_dump_slots - Print the contents of the slot ringbuffers ++ * ++ * @kbdev: Device pointer ++ */ ++void kbase_gpu_dump_slots(struct kbase_device *kbdev); ++ ++#endif /* _KBASE_HWACCESS_GPU_H_ */ +diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_js_backend.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_js_backend.c +new file mode 100755 +index 000000000000..8187e73767be +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_js_backend.c +@@ -0,0 +1,365 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++/* ++ * Register-based HW access backend specific job scheduler APIs ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++ ++#if !MALI_USE_CSF ++/* ++ * Hold the runpool_mutex for this ++ */ ++static inline bool timer_callback_should_run(struct kbase_device *kbdev) ++{ ++ struct kbase_backend_data *backend = &kbdev->hwaccess.backend; ++ int nr_running_ctxs; ++ ++ lockdep_assert_held(&kbdev->js_data.runpool_mutex); ++ ++ /* Timer must stop if we are suspending */ ++ if (backend->suspend_timer) ++ return false; ++ ++ /* nr_contexts_pullable is updated with the runpool_mutex. However, the ++ * locking in the caller gives us a barrier that ensures ++ * nr_contexts_pullable is up-to-date for reading */ ++ nr_running_ctxs = atomic_read(&kbdev->js_data.nr_contexts_runnable); ++ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++ if (kbdev->js_data.softstop_always) { ++ /* Debug support for allowing soft-stop on a single context */ ++ return true; ++ } ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_9435)) { ++ /* Timeouts would have to be 4x longer (due to micro- ++ * architectural design) to support OpenCL conformance tests, so ++ * only run the timer when there's: ++ * - 2 or more CL contexts ++ * - 1 or more GLES contexts ++ * ++ * NOTE: We will treat a context that has both Compute and Non- ++ * Compute jobs will be treated as an OpenCL context (hence, we ++ * don't check KBASEP_JS_CTX_ATTR_NON_COMPUTE). ++ */ ++ { ++ int nr_compute_ctxs = ++ kbasep_js_ctx_attr_count_on_runpool(kbdev, ++ KBASEP_JS_CTX_ATTR_COMPUTE); ++ int nr_noncompute_ctxs = nr_running_ctxs - ++ nr_compute_ctxs; ++ ++ return (bool) (nr_compute_ctxs >= 2 || ++ nr_noncompute_ctxs > 0); ++ } ++ } else { ++ /* Run the timer callback whenever you have at least 1 context ++ */ ++ return (bool) (nr_running_ctxs > 0); ++ } ++} ++ ++static enum hrtimer_restart timer_callback(struct hrtimer *timer) ++{ ++ unsigned long flags; ++ struct kbase_device *kbdev; ++ struct kbasep_js_device_data *js_devdata; ++ struct kbase_backend_data *backend; ++ int s; ++ bool reset_needed = false; ++ ++ KBASE_DEBUG_ASSERT(timer != NULL); ++ ++ backend = container_of(timer, struct kbase_backend_data, ++ scheduling_timer); ++ kbdev = container_of(backend, struct kbase_device, hwaccess.backend); ++ js_devdata = &kbdev->js_data; ++ ++ /* Loop through the slots */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ for (s = 0; s < kbdev->gpu_props.num_job_slots; s++) { ++ struct kbase_jd_atom *atom = NULL; ++ ++ if (kbase_backend_nr_atoms_on_slot(kbdev, s) > 0) { ++ atom = kbase_gpu_inspect(kbdev, s, 0); ++ KBASE_DEBUG_ASSERT(atom != NULL); ++ } ++ ++ if (atom != NULL) { ++ /* The current version of the model doesn't support ++ * Soft-Stop */ ++ if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_5736)) { ++ u32 ticks = atom->ticks++; ++ ++#if !defined(CONFIG_MALI_JOB_DUMP) && !defined(CONFIG_MALI_VECTOR_DUMP) ++ u32 soft_stop_ticks, hard_stop_ticks, ++ gpu_reset_ticks; ++ if (atom->core_req & BASE_JD_REQ_ONLY_COMPUTE) { ++ soft_stop_ticks = ++ js_devdata->soft_stop_ticks_cl; ++ hard_stop_ticks = ++ js_devdata->hard_stop_ticks_cl; ++ gpu_reset_ticks = ++ js_devdata->gpu_reset_ticks_cl; ++ } else { ++ soft_stop_ticks = ++ js_devdata->soft_stop_ticks; ++ hard_stop_ticks = ++ js_devdata->hard_stop_ticks_ss; ++ gpu_reset_ticks = ++ js_devdata->gpu_reset_ticks_ss; ++ } ++ ++ /* If timeouts have been changed then ensure ++ * that atom tick count is not greater than the ++ * new soft_stop timeout. This ensures that ++ * atoms do not miss any of the timeouts due to ++ * races between this worker and the thread ++ * changing the timeouts. */ ++ if (backend->timeouts_updated && ++ ticks > soft_stop_ticks) ++ ticks = atom->ticks = soft_stop_ticks; ++ ++ /* Job is Soft-Stoppable */ ++ if (ticks == soft_stop_ticks) { ++ /* Job has been scheduled for at least ++ * js_devdata->soft_stop_ticks ticks. ++ * Soft stop the slot so we can run ++ * other jobs. ++ */ ++#if !KBASE_DISABLE_SCHEDULING_SOFT_STOPS ++ int disjoint_threshold = ++ KBASE_DISJOINT_STATE_INTERLEAVED_CONTEXT_COUNT_THRESHOLD; ++ u32 softstop_flags = 0u; ++ ++ dev_dbg(kbdev->dev, "Soft-stop"); ++ /* nr_user_contexts_running is updated ++ * with the runpool_mutex, but we can't ++ * take that here. ++ * ++ * However, if it's about to be ++ * increased then the new context can't ++ * run any jobs until they take the ++ * hwaccess_lock, so it's OK to observe ++ * the older value. ++ * ++ * Similarly, if it's about to be ++ * decreased, the last job from another ++ * context has already finished, so it's ++ * not too bad that we observe the older ++ * value and register a disjoint event ++ * when we try soft-stopping */ ++ if (js_devdata->nr_user_contexts_running ++ >= disjoint_threshold) ++ softstop_flags |= ++ JS_COMMAND_SW_CAUSES_DISJOINT; ++ ++ kbase_job_slot_softstop_swflags(kbdev, ++ s, atom, softstop_flags); ++#endif ++ } else if (ticks == hard_stop_ticks) { ++ /* Job has been scheduled for at least ++ * js_devdata->hard_stop_ticks_ss ticks. ++ * It should have been soft-stopped by ++ * now. Hard stop the slot. ++ */ ++#if !KBASE_DISABLE_SCHEDULING_HARD_STOPS ++ int ms = ++ js_devdata->scheduling_period_ns ++ / 1000000u; ++ dev_warn(kbdev->dev, "JS: Job Hard-Stopped (took more than %lu ticks at %lu ms/tick)", ++ (unsigned long)ticks, ++ (unsigned long)ms); ++ kbase_job_slot_hardstop(atom->kctx, s, ++ atom); ++#endif ++ } else if (ticks == gpu_reset_ticks) { ++ /* Job has been scheduled for at least ++ * js_devdata->gpu_reset_ticks_ss ticks. ++ * It should have left the GPU by now. ++ * Signal that the GPU needs to be ++ * reset. ++ */ ++ reset_needed = true; ++ } ++#else /* !CONFIG_MALI_JOB_DUMP */ ++ /* NOTE: During CONFIG_MALI_JOB_DUMP, we use ++ * the alternate timeouts, which makes the hard- ++ * stop and GPU reset timeout much longer. We ++ * also ensure that we don't soft-stop at all. ++ */ ++ if (ticks == js_devdata->soft_stop_ticks) { ++ /* Job has been scheduled for at least ++ * js_devdata->soft_stop_ticks. We do ++ * not soft-stop during ++ * CONFIG_MALI_JOB_DUMP, however. ++ */ ++ dev_dbg(kbdev->dev, "Soft-stop"); ++ } else if (ticks == ++ js_devdata->hard_stop_ticks_dumping) { ++ /* Job has been scheduled for at least ++ * js_devdata->hard_stop_ticks_dumping ++ * ticks. Hard stop the slot. ++ */ ++#if !KBASE_DISABLE_SCHEDULING_HARD_STOPS ++ int ms = ++ js_devdata->scheduling_period_ns ++ / 1000000u; ++ dev_warn(kbdev->dev, "JS: Job Hard-Stopped (took more than %lu ticks at %lu ms/tick)", ++ (unsigned long)ticks, ++ (unsigned long)ms); ++ kbase_job_slot_hardstop(atom->kctx, s, ++ atom); ++#endif ++ } else if (ticks == ++ js_devdata->gpu_reset_ticks_dumping) { ++ /* Job has been scheduled for at least ++ * js_devdata->gpu_reset_ticks_dumping ++ * ticks. It should have left the GPU by ++ * now. Signal that the GPU needs to be ++ * reset. ++ */ ++ reset_needed = true; ++ } ++#endif /* !CONFIG_MALI_JOB_DUMP */ ++ } ++ } ++ } ++ if (reset_needed) { ++ dev_err(kbdev->dev, "JS: Job has been on the GPU for too long (JS_RESET_TICKS_SS/DUMPING timeout hit). Issueing GPU soft-reset to resolve."); ++ ++ if (kbase_prepare_to_reset_gpu_locked(kbdev)) ++ kbase_reset_gpu_locked(kbdev); ++ } ++ /* the timer is re-issued if there is contexts in the run-pool */ ++ ++ if (backend->timer_running) ++ hrtimer_start(&backend->scheduling_timer, ++ HR_TIMER_DELAY_NSEC(js_devdata->scheduling_period_ns), ++ HRTIMER_MODE_REL); ++ ++ backend->timeouts_updated = false; ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ return HRTIMER_NORESTART; ++} ++#endif /* !MALI_USE_CSF */ ++ ++void kbase_backend_ctx_count_changed(struct kbase_device *kbdev) ++{ ++#if !MALI_USE_CSF ++ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; ++ struct kbase_backend_data *backend = &kbdev->hwaccess.backend; ++ unsigned long flags; ++ ++ lockdep_assert_held(&js_devdata->runpool_mutex); ++ ++ if (!timer_callback_should_run(kbdev)) { ++ /* Take spinlock to force synchronisation with timer */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ backend->timer_running = false; ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ /* From now on, return value of timer_callback_should_run() will ++ * also cause the timer to not requeue itself. Its return value ++ * cannot change, because it depends on variables updated with ++ * the runpool_mutex held, which the caller of this must also ++ * hold */ ++ hrtimer_cancel(&backend->scheduling_timer); ++ } ++ ++ if (timer_callback_should_run(kbdev) && !backend->timer_running) { ++ /* Take spinlock to force synchronisation with timer */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ backend->timer_running = true; ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ hrtimer_start(&backend->scheduling_timer, ++ HR_TIMER_DELAY_NSEC(js_devdata->scheduling_period_ns), ++ HRTIMER_MODE_REL); ++ ++ KBASE_KTRACE_ADD_JM(kbdev, JS_POLICY_TIMER_START, NULL, NULL, 0u, 0u); ++ } ++#else /* !MALI_USE_CSF */ ++ CSTD_UNUSED(kbdev); ++#endif /* !MALI_USE_CSF */ ++} ++ ++int kbase_backend_timer_init(struct kbase_device *kbdev) ++{ ++#if !MALI_USE_CSF ++ struct kbase_backend_data *backend = &kbdev->hwaccess.backend; ++ ++ hrtimer_init(&backend->scheduling_timer, CLOCK_MONOTONIC, ++ HRTIMER_MODE_REL); ++ backend->scheduling_timer.function = timer_callback; ++ backend->timer_running = false; ++#else /* !MALI_USE_CSF */ ++ CSTD_UNUSED(kbdev); ++#endif /* !MALI_USE_CSF */ ++ ++ return 0; ++} ++ ++void kbase_backend_timer_term(struct kbase_device *kbdev) ++{ ++#if !MALI_USE_CSF ++ struct kbase_backend_data *backend = &kbdev->hwaccess.backend; ++ ++ hrtimer_cancel(&backend->scheduling_timer); ++#else /* !MALI_USE_CSF */ ++ CSTD_UNUSED(kbdev); ++#endif /* !MALI_USE_CSF */ ++} ++ ++void kbase_backend_timer_suspend(struct kbase_device *kbdev) ++{ ++ struct kbase_backend_data *backend = &kbdev->hwaccess.backend; ++ ++ backend->suspend_timer = true; ++ ++ kbase_backend_ctx_count_changed(kbdev); ++} ++ ++void kbase_backend_timer_resume(struct kbase_device *kbdev) ++{ ++ struct kbase_backend_data *backend = &kbdev->hwaccess.backend; ++ ++ backend->suspend_timer = false; ++ ++ kbase_backend_ctx_count_changed(kbdev); ++} ++ ++void kbase_backend_timeouts_changed(struct kbase_device *kbdev) ++{ ++ struct kbase_backend_data *backend = &kbdev->hwaccess.backend; ++ ++ backend->timeouts_updated = true; ++} ++ +diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_js_internal.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_js_internal.h +new file mode 100755 +index 000000000000..6576e55d2e39 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_js_internal.h +@@ -0,0 +1,74 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++/* ++ * Register-based HW access backend specific job scheduler APIs ++ */ ++ ++#ifndef _KBASE_JS_BACKEND_H_ ++#define _KBASE_JS_BACKEND_H_ ++ ++/** ++ * kbase_backend_timer_init() - Initialise the JS scheduling timer ++ * @kbdev: Device pointer ++ * ++ * This function should be called at driver initialisation ++ * ++ * Return: 0 on success ++ */ ++int kbase_backend_timer_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_backend_timer_term() - Terminate the JS scheduling timer ++ * @kbdev: Device pointer ++ * ++ * This function should be called at driver termination ++ */ ++void kbase_backend_timer_term(struct kbase_device *kbdev); ++ ++/** ++ * kbase_backend_timer_suspend - Suspend is happening, stop the JS scheduling ++ * timer ++ * @kbdev: Device pointer ++ * ++ * This function should be called on suspend, after the active count has reached ++ * zero. This is required as the timer may have been started on job submission ++ * to the job scheduler, but before jobs are submitted to the GPU. ++ * ++ * Caller must hold runpool_mutex. ++ */ ++void kbase_backend_timer_suspend(struct kbase_device *kbdev); ++ ++/** ++ * kbase_backend_timer_resume - Resume is happening, re-evaluate the JS ++ * scheduling timer ++ * @kbdev: Device pointer ++ * ++ * This function should be called on resume. Note that is is not guaranteed to ++ * re-start the timer, only evalute whether it should be re-started. ++ * ++ * Caller must hold runpool_mutex. ++ */ ++void kbase_backend_timer_resume(struct kbase_device *kbdev); ++ ++#endif /* _KBASE_JS_BACKEND_H_ */ +diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_l2_mmu_config.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_l2_mmu_config.c +new file mode 100755 +index 000000000000..d5526caa5899 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_l2_mmu_config.c +@@ -0,0 +1,133 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * ++ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++#include ++#include ++#include ++#include "mali_kbase_l2_mmu_config.h" ++ ++/** ++ * struct l2_mmu_config_limit_region ++ * ++ * @value: The default value to load into the L2_MMU_CONFIG register ++ * @mask: The shifted mask of the field in the L2_MMU_CONFIG register ++ * @shift: The shift of where the field starts in the L2_MMU_CONFIG register ++ * This should be the same value as the smaller of the two mask ++ * values ++ */ ++struct l2_mmu_config_limit_region { ++ u32 value, mask, shift; ++}; ++ ++/** ++ * struct l2_mmu_config_limit ++ * ++ * @product_model: The GPU for which this entry applies ++ * @read: Values for the read limit field ++ * @write: Values for the write limit field ++ */ ++struct l2_mmu_config_limit { ++ u32 product_model; ++ struct l2_mmu_config_limit_region read; ++ struct l2_mmu_config_limit_region write; ++}; ++ ++/* ++ * Zero represents no limit ++ * ++ * For LBEX TBEX TBAX TTRX and TNAX: ++ * The value represents the number of outstanding reads (6 bits) or writes (5 bits) ++ * ++ * For all other GPUS it is a fraction see: mali_kbase_config_defaults.h ++ */ ++static const struct l2_mmu_config_limit limits[] = { ++ /* GPU, read, write */ ++ {GPU_ID2_PRODUCT_LBEX, ++ {0, GENMASK(10, 5), 5}, ++ {0, GENMASK(16, 12), 12} }, ++ {GPU_ID2_PRODUCT_TBEX, ++ {0, GENMASK(10, 5), 5}, ++ {0, GENMASK(16, 12), 12} }, ++ {GPU_ID2_PRODUCT_TBAX, ++ {0, GENMASK(10, 5), 5}, ++ {0, GENMASK(16, 12), 12} }, ++ {GPU_ID2_PRODUCT_TTRX, ++ {0, GENMASK(12, 7), 7}, ++ {0, GENMASK(17, 13), 13} }, ++ {GPU_ID2_PRODUCT_TNAX, ++ {0, GENMASK(12, 7), 7}, ++ {0, GENMASK(17, 13), 13} }, ++ {GPU_ID2_PRODUCT_TGOX, ++ {KBASE_3BIT_AID_32, GENMASK(14, 12), 12}, ++ {KBASE_3BIT_AID_32, GENMASK(17, 15), 15} }, ++ {GPU_ID2_PRODUCT_TNOX, ++ {KBASE_3BIT_AID_32, GENMASK(14, 12), 12}, ++ {KBASE_3BIT_AID_32, GENMASK(17, 15), 15} }, ++}; ++ ++int kbase_set_mmu_quirks(struct kbase_device *kbdev) ++{ ++ /* All older GPUs had 2 bits for both fields, this is a default */ ++ struct l2_mmu_config_limit limit = { ++ 0, /* Any GPU not in the limits array defined above */ ++ {KBASE_AID_32, GENMASK(25, 24), 24}, ++ {KBASE_AID_32, GENMASK(27, 26), 26} ++ }; ++ u32 product_model, gpu_id; ++ u32 mmu_config; ++ int i; ++ ++ gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; ++ product_model = gpu_id & GPU_ID2_PRODUCT_MODEL; ++ ++ /* Limit the GPU bus bandwidth if the platform needs this. */ ++ for (i = 0; i < ARRAY_SIZE(limits); i++) { ++ if (product_model == limits[i].product_model) { ++ limit = limits[i]; ++ break; ++ } ++ } ++ ++ mmu_config = kbase_reg_read(kbdev, GPU_CONTROL_REG(L2_MMU_CONFIG)); ++ ++ if (kbase_is_gpu_removed(kbdev)) ++ return -EIO; ++ ++ mmu_config &= ~(limit.read.mask | limit.write.mask); ++ /* Can't use FIELD_PREP() macro here as the mask isn't constant */ ++ mmu_config |= (limit.read.value << limit.read.shift) | ++ (limit.write.value << limit.write.shift); ++ ++ kbdev->hw_quirks_mmu = mmu_config; ++ ++ if (kbdev->system_coherency == COHERENCY_ACE) { ++ /* Allow memory configuration disparity to be ignored, ++ * we optimize the use of shared memory and thus we ++ * expect some disparity in the memory configuration. ++ */ ++ kbdev->hw_quirks_mmu |= L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY; ++ } ++ ++ return 0; ++} +diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_l2_mmu_config.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_l2_mmu_config.h +new file mode 100755 +index 000000000000..0c779ac80d27 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_l2_mmu_config.h +@@ -0,0 +1,55 @@ ++/* ++ * ++ * (C) COPYRIGHT ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ *//* SPDX-License-Identifier: GPL-2.0 */ ++/* ++ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ */ ++ ++#ifndef _KBASE_L2_MMU_CONFIG_H_ ++#define _KBASE_L2_MMU_CONFIG_H_ ++/** ++ * kbase_set_mmu_quirks - Set the hw_quirks_mmu field of kbdev ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * Use this function to initialise the hw_quirks_mmu field, for instance to set ++ * the MAX_READS and MAX_WRITES to sane defaults for each GPU. ++ * ++ * Return: Zero for succeess or a Linux error code ++ */ ++int kbase_set_mmu_quirks(struct kbase_device *kbdev); ++ ++#endif /* _KBASE_L2_MMU_CONFIG_H */ +diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_always_on.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_always_on.c +new file mode 100755 +index 000000000000..e33fe0b8e415 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_always_on.c +@@ -0,0 +1,67 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2015, 2018-2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++/* ++ * "Always on" power management policy ++ */ ++ ++#include ++#include ++ ++static bool always_on_shaders_needed(struct kbase_device *kbdev) ++{ ++ return true; ++} ++ ++static bool always_on_get_core_active(struct kbase_device *kbdev) ++{ ++ return true; ++} ++ ++static void always_on_init(struct kbase_device *kbdev) ++{ ++ CSTD_UNUSED(kbdev); ++} ++ ++static void always_on_term(struct kbase_device *kbdev) ++{ ++ CSTD_UNUSED(kbdev); ++} ++ ++/* ++ * The struct kbase_pm_policy structure for the demand power policy. ++ * ++ * This is the static structure that defines the demand power policy's callback ++ * and name. ++ */ ++const struct kbase_pm_policy kbase_pm_always_on_policy_ops = { ++ "always_on", /* name */ ++ always_on_init, /* init */ ++ always_on_term, /* term */ ++ always_on_shaders_needed, /* shaders_needed */ ++ always_on_get_core_active, /* get_core_active */ ++ KBASE_PM_POLICY_ID_ALWAYS_ON, /* id */ ++}; ++ ++KBASE_EXPORT_TEST_API(kbase_pm_always_on_policy_ops); +diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_always_on.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_always_on.h +new file mode 100755 +index 000000000000..e7927cf82e5a +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_always_on.h +@@ -0,0 +1,81 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2015,2018 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++/* ++ * "Always on" power management policy ++ */ ++ ++#ifndef MALI_KBASE_PM_ALWAYS_ON_H ++#define MALI_KBASE_PM_ALWAYS_ON_H ++ ++/** ++ * DOC: ++ * The "Always on" power management policy has the following ++ * characteristics: ++ * ++ * - When KBase indicates that the GPU will be powered up, but we don't yet ++ * know which Job Chains are to be run: ++ * Shader Cores are powered up, regardless of whether or not they will be ++ * needed later. ++ * ++ * - When KBase indicates that Shader Cores are needed to submit the currently ++ * queued Job Chains: ++ * Shader Cores are kept powered, regardless of whether or not they will be ++ * needed ++ * ++ * - When KBase indicates that the GPU need not be powered: ++ * The Shader Cores are kept powered, regardless of whether or not they will ++ * be needed. The GPU itself is also kept powered, even though it is not ++ * needed. ++ * ++ * This policy is automatically overridden during system suspend: the desired ++ * core state is ignored, and the cores are forced off regardless of what the ++ * policy requests. After resuming from suspend, new changes to the desired ++ * core state made by the policy are honored. ++ * ++ * Note: ++ * ++ * - KBase indicates the GPU will be powered up when it has a User Process that ++ * has just started to submit Job Chains. ++ * ++ * - KBase indicates the GPU need not be powered when all the Job Chains from ++ * User Processes have finished, and it is waiting for a User Process to ++ * submit some more Job Chains. ++ */ ++ ++/** ++ * struct kbasep_pm_policy_always_on - Private struct for policy instance data ++ * @dummy: unused dummy variable ++ * ++ * This contains data that is private to the particular power policy that is ++ * active. ++ */ ++struct kbasep_pm_policy_always_on { ++ int dummy; ++}; ++ ++extern const struct kbase_pm_policy kbase_pm_always_on_policy_ops; ++ ++#endif /* MALI_KBASE_PM_ALWAYS_ON_H */ ++ +diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_backend.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_backend.c +new file mode 100755 +index 000000000000..7b10d06c5fdb +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_backend.c +@@ -0,0 +1,788 @@ ++ /* ++ * ++ * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++/* ++ * GPU backend implementation of base kernel power management APIs ++ */ ++ ++#include ++#include ++#include ++ ++#include ++#if !MALI_USE_CSF ++#include ++#include ++#include ++#endif /* !MALI_USE_CSF */ ++#include ++#include ++#include ++#include ++#include ++ ++static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data); ++static void kbase_pm_hwcnt_disable_worker(struct work_struct *data); ++static void kbase_pm_gpu_clock_control_worker(struct work_struct *data); ++ ++int kbase_pm_runtime_init(struct kbase_device *kbdev) ++{ ++ struct kbase_pm_callback_conf *callbacks; ++ ++ callbacks = (struct kbase_pm_callback_conf *)POWER_MANAGEMENT_CALLBACKS; ++ if (callbacks) { ++ kbdev->pm.backend.callback_power_on = ++ callbacks->power_on_callback; ++ kbdev->pm.backend.callback_power_off = ++ callbacks->power_off_callback; ++ kbdev->pm.backend.callback_power_suspend = ++ callbacks->power_suspend_callback; ++ kbdev->pm.backend.callback_power_resume = ++ callbacks->power_resume_callback; ++ kbdev->pm.callback_power_runtime_init = ++ callbacks->power_runtime_init_callback; ++ kbdev->pm.callback_power_runtime_term = ++ callbacks->power_runtime_term_callback; ++ kbdev->pm.backend.callback_power_runtime_on = ++ callbacks->power_runtime_on_callback; ++ kbdev->pm.backend.callback_power_runtime_off = ++ callbacks->power_runtime_off_callback; ++ kbdev->pm.backend.callback_power_runtime_idle = ++ callbacks->power_runtime_idle_callback; ++ kbdev->pm.backend.callback_soft_reset = ++ callbacks->soft_reset_callback; ++ ++ if (callbacks->power_runtime_init_callback) ++ return callbacks->power_runtime_init_callback(kbdev); ++ else ++ return 0; ++ } ++ ++ kbdev->pm.backend.callback_power_on = NULL; ++ kbdev->pm.backend.callback_power_off = NULL; ++ kbdev->pm.backend.callback_power_suspend = NULL; ++ kbdev->pm.backend.callback_power_resume = NULL; ++ kbdev->pm.callback_power_runtime_init = NULL; ++ kbdev->pm.callback_power_runtime_term = NULL; ++ kbdev->pm.backend.callback_power_runtime_on = NULL; ++ kbdev->pm.backend.callback_power_runtime_off = NULL; ++ kbdev->pm.backend.callback_power_runtime_idle = NULL; ++ kbdev->pm.backend.callback_soft_reset = NULL; ++ ++ return 0; ++} ++ ++void kbase_pm_runtime_term(struct kbase_device *kbdev) ++{ ++ if (kbdev->pm.callback_power_runtime_term) { ++ kbdev->pm.callback_power_runtime_term(kbdev); ++ } ++} ++ ++void kbase_pm_register_access_enable(struct kbase_device *kbdev) ++{ ++ struct kbase_pm_callback_conf *callbacks; ++ ++ callbacks = (struct kbase_pm_callback_conf *)POWER_MANAGEMENT_CALLBACKS; ++ ++ if (callbacks) ++ callbacks->power_on_callback(kbdev); ++ ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++ if (WARN_ON(kbase_pm_is_gpu_lost(kbdev))) ++ dev_err(kbdev->dev, "Attempting to power on while GPU lost\n"); ++#endif ++ ++ kbdev->pm.backend.gpu_powered = true; ++} ++ ++void kbase_pm_register_access_disable(struct kbase_device *kbdev) ++{ ++ struct kbase_pm_callback_conf *callbacks; ++ ++ callbacks = (struct kbase_pm_callback_conf *)POWER_MANAGEMENT_CALLBACKS; ++ ++ if (callbacks) ++ callbacks->power_off_callback(kbdev); ++ ++ kbdev->pm.backend.gpu_powered = false; ++} ++ ++int kbase_hwaccess_pm_init(struct kbase_device *kbdev) ++{ ++ int ret = 0; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ mutex_init(&kbdev->pm.lock); ++ ++ kbdev->pm.backend.gpu_poweroff_wait_wq = alloc_workqueue("kbase_pm_poweroff_wait", ++ WQ_HIGHPRI | WQ_UNBOUND, 1); ++ if (!kbdev->pm.backend.gpu_poweroff_wait_wq) ++ return -ENOMEM; ++ ++ INIT_WORK(&kbdev->pm.backend.gpu_poweroff_wait_work, ++ kbase_pm_gpu_poweroff_wait_wq); ++ ++ kbdev->pm.backend.ca_cores_enabled = ~0ull; ++ kbdev->pm.backend.gpu_powered = false; ++ kbdev->pm.backend.gpu_ready = false; ++ kbdev->pm.suspending = false; ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++ kbase_pm_set_gpu_lost(kbdev, false); ++#endif ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++ kbdev->pm.backend.driver_ready_for_irqs = false; ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++ init_waitqueue_head(&kbdev->pm.backend.gpu_in_desired_state_wait); ++ ++ /* Initialise the metrics subsystem */ ++ ret = kbasep_pm_metrics_init(kbdev); ++ if (ret) ++ return ret; ++ ++ init_waitqueue_head(&kbdev->pm.backend.reset_done_wait); ++ kbdev->pm.backend.reset_done = false; ++ ++ init_waitqueue_head(&kbdev->pm.zero_active_count_wait); ++ kbdev->pm.active_count = 0; ++ ++ spin_lock_init(&kbdev->pm.backend.gpu_cycle_counter_requests_lock); ++ ++ init_waitqueue_head(&kbdev->pm.backend.poweroff_wait); ++ ++ if (kbase_pm_ca_init(kbdev) != 0) ++ goto workq_fail; ++ ++ kbase_pm_policy_init(kbdev); ++ ++ if (kbase_pm_state_machine_init(kbdev) != 0) ++ goto pm_state_machine_fail; ++ ++ kbdev->pm.backend.hwcnt_desired = false; ++ kbdev->pm.backend.hwcnt_disabled = true; ++ INIT_WORK(&kbdev->pm.backend.hwcnt_disable_work, ++ kbase_pm_hwcnt_disable_worker); ++ kbase_hwcnt_context_disable(kbdev->hwcnt_gpu_ctx); ++ ++ if (IS_ENABLED(CONFIG_MALI_HW_ERRATA_1485982_NOT_AFFECTED)) { ++ kbdev->pm.backend.l2_always_on = false; ++ kbdev->pm.backend.gpu_clock_slow_down_wa = false; ++ ++ return 0; ++ } ++ ++ /* WA1: L2 always_on for GPUs being affected by GPU2017-1336 */ ++ if (!IS_ENABLED(CONFIG_MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE)) { ++ kbdev->pm.backend.gpu_clock_slow_down_wa = false; ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_GPU2017_1336)) ++ kbdev->pm.backend.l2_always_on = true; ++ else ++ kbdev->pm.backend.l2_always_on = false; ++ ++ return 0; ++ } ++ ++ /* WA3: Clock slow down for GPUs being affected by GPU2017-1336 */ ++ kbdev->pm.backend.l2_always_on = false; ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_GPU2017_1336)) { ++ kbdev->pm.backend.gpu_clock_slow_down_wa = true; ++ kbdev->pm.backend.gpu_clock_suspend_freq = 0; ++ kbdev->pm.backend.gpu_clock_slow_down_desired = true; ++ kbdev->pm.backend.gpu_clock_slowed_down = false; ++ INIT_WORK(&kbdev->pm.backend.gpu_clock_control_work, ++ kbase_pm_gpu_clock_control_worker); ++ } else ++ kbdev->pm.backend.gpu_clock_slow_down_wa = false; ++ ++ return 0; ++ ++pm_state_machine_fail: ++ kbase_pm_policy_term(kbdev); ++ kbase_pm_ca_term(kbdev); ++workq_fail: ++ kbasep_pm_metrics_term(kbdev); ++ return -EINVAL; ++} ++ ++void kbase_pm_do_poweron(struct kbase_device *kbdev, bool is_resume) ++{ ++ lockdep_assert_held(&kbdev->pm.lock); ++ ++ /* Turn clocks and interrupts on - no-op if we haven't done a previous ++ * kbase_pm_clock_off() */ ++ kbase_pm_clock_on(kbdev, is_resume); ++ ++ if (!is_resume) { ++ unsigned long flags; ++ ++ /* Force update of L2 state - if we have abandoned a power off ++ * then this may be required to power the L2 back on. ++ */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_pm_update_state(kbdev); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ } ++ ++ /* Update core status as required by the policy */ ++ kbase_pm_update_cores_state(kbdev); ++ ++ /* NOTE: We don't wait to reach the desired state, since running atoms ++ * will wait for that state to be reached anyway */ ++} ++ ++static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data) ++{ ++ struct kbase_device *kbdev = container_of(data, struct kbase_device, ++ pm.backend.gpu_poweroff_wait_work); ++ struct kbase_pm_device_data *pm = &kbdev->pm; ++ struct kbase_pm_backend_data *backend = &pm->backend; ++ unsigned long flags; ++ ++#if !MALI_USE_CSF ++ /* Wait for power transitions to complete. We do this with no locks held ++ * so that we don't deadlock with any pending workqueues. ++ */ ++ kbase_pm_wait_for_desired_state(kbdev); ++#endif ++ ++ kbase_pm_lock(kbdev); ++ ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++ if (kbase_pm_is_gpu_lost(kbdev)) ++ backend->poweron_required = false; ++#endif ++ ++ if (!backend->poweron_required) { ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ WARN_ON(backend->shaders_state != ++ KBASE_SHADERS_OFF_CORESTACK_OFF || ++ backend->l2_state != KBASE_L2_OFF); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ /* Disable interrupts and turn the clock off */ ++ if (!kbase_pm_clock_off(kbdev)) { ++ /* ++ * Page/bus faults are pending, must drop locks to ++ * process. Interrupts are disabled so no more faults ++ * should be generated at this point. ++ */ ++ kbase_pm_unlock(kbdev); ++ kbase_flush_mmu_wqs(kbdev); ++ kbase_pm_lock(kbdev); ++ ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++ /* poweron_required may have changed while pm lock ++ * was released. ++ */ ++ if (kbase_pm_is_gpu_lost(kbdev)) ++ backend->poweron_required = false; ++#endif ++ ++ /* Turn off clock now that fault have been handled. We ++ * dropped locks so poweron_required may have changed - ++ * power back on if this is the case (effectively only ++ * re-enabling of the interrupts would be done in this ++ * case, as the clocks to GPU were not withdrawn yet). ++ */ ++ if (backend->poweron_required) ++ kbase_pm_clock_on(kbdev, false); ++ else ++ WARN_ON(!kbase_pm_clock_off(kbdev)); ++ } ++ } ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ backend->poweroff_wait_in_progress = false; ++ if (backend->poweron_required) { ++ backend->poweron_required = false; ++ kbdev->pm.backend.l2_desired = true; ++#if MALI_USE_CSF ++ kbdev->pm.backend.mcu_desired = true; ++#endif ++ kbase_pm_update_state(kbdev); ++ kbase_pm_update_cores_state_nolock(kbdev); ++#if !MALI_USE_CSF ++ kbase_backend_slot_update(kbdev); ++#endif /* !MALI_USE_CSF */ ++ } ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ kbase_pm_unlock(kbdev); ++ ++ wake_up(&kbdev->pm.backend.poweroff_wait); ++} ++ ++static void kbase_pm_l2_clock_slow(struct kbase_device *kbdev) ++{ ++#if defined(CONFIG_MALI_BIFROST_DVFS) ++ struct clk *clk = kbdev->clocks[0]; ++#endif ++ ++ if (!kbdev->pm.backend.gpu_clock_slow_down_wa) ++ return; ++ ++ /* No suspend clock is specified */ ++ if (WARN_ON_ONCE(!kbdev->pm.backend.gpu_clock_suspend_freq)) ++ return; ++ ++#if defined(CONFIG_MALI_BIFROST_DEVFREQ) ++ ++ /* Suspend devfreq */ ++ devfreq_suspend_device(kbdev->devfreq); ++ ++ /* Keep the current freq to restore it upon resume */ ++ kbdev->previous_frequency = kbdev->current_nominal_freq; ++ ++ /* Slow down GPU clock to the suspend clock*/ ++ kbase_devfreq_force_freq(kbdev, ++ kbdev->pm.backend.gpu_clock_suspend_freq); ++ ++#elif defined(CONFIG_MALI_BIFROST_DVFS) /* CONFIG_MALI_BIFROST_DEVFREQ */ ++ ++ if (WARN_ON_ONCE(!clk)) ++ return; ++ ++ /* Stop the metrics gathering framework */ ++ if (kbase_pm_metrics_is_active(kbdev)) ++ kbase_pm_metrics_stop(kbdev); ++ ++ /* Keep the current freq to restore it upon resume */ ++ kbdev->previous_frequency = clk_get_rate(clk); ++ ++ /* Slow down GPU clock to the suspend clock*/ ++ if (WARN_ON_ONCE(clk_set_rate(clk, ++ kbdev->pm.backend.gpu_clock_suspend_freq))) ++ dev_err(kbdev->dev, "Failed to set suspend freq\n"); ++ ++#endif /* CONFIG_MALI_BIFROST_DVFS */ ++} ++ ++static void kbase_pm_l2_clock_normalize(struct kbase_device *kbdev) ++{ ++#if defined(CONFIG_MALI_BIFROST_DVFS) ++ struct clk *clk = kbdev->clocks[0]; ++#endif ++ ++ if (!kbdev->pm.backend.gpu_clock_slow_down_wa) ++ return; ++ ++#if defined(CONFIG_MALI_BIFROST_DEVFREQ) ++ ++ /* Restore GPU clock to the previous one */ ++ kbase_devfreq_force_freq(kbdev, kbdev->previous_frequency); ++ ++ /* Resume devfreq */ ++ devfreq_resume_device(kbdev->devfreq); ++ ++#elif defined(CONFIG_MALI_BIFROST_DVFS) /* CONFIG_MALI_BIFROST_DEVFREQ */ ++ ++ if (WARN_ON_ONCE(!clk)) ++ return; ++ ++ /* Restore GPU clock */ ++ if (WARN_ON_ONCE(clk_set_rate(clk, kbdev->previous_frequency))) ++ dev_err(kbdev->dev, "Failed to restore freq (%lu)\n", ++ kbdev->previous_frequency); ++ ++ /* Restart the metrics gathering framework */ ++ kbase_pm_metrics_start(kbdev); ++ ++#endif /* CONFIG_MALI_BIFROST_DVFS */ ++} ++ ++static void kbase_pm_gpu_clock_control_worker(struct work_struct *data) ++{ ++ struct kbase_device *kbdev = container_of(data, struct kbase_device, ++ pm.backend.gpu_clock_control_work); ++ struct kbase_pm_device_data *pm = &kbdev->pm; ++ struct kbase_pm_backend_data *backend = &pm->backend; ++ unsigned long flags; ++ bool slow_down = false, normalize = false; ++ ++ /* Determine if GPU clock control is required */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ if (!backend->gpu_clock_slowed_down && ++ backend->gpu_clock_slow_down_desired) { ++ slow_down = true; ++ backend->gpu_clock_slowed_down = true; ++ } else if (backend->gpu_clock_slowed_down && ++ !backend->gpu_clock_slow_down_desired) { ++ normalize = true; ++ backend->gpu_clock_slowed_down = false; ++ } ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ /* Control GPU clock according to the request of L2 state machine. ++ * The GPU clock needs to be lowered for safe L2 power down ++ * and restored to previous speed at L2 power up. ++ */ ++ if (slow_down) ++ kbase_pm_l2_clock_slow(kbdev); ++ else if (normalize) ++ kbase_pm_l2_clock_normalize(kbdev); ++ ++ /* Tell L2 state machine to transit to next state */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_pm_update_state(kbdev); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++} ++ ++static void kbase_pm_hwcnt_disable_worker(struct work_struct *data) ++{ ++ struct kbase_device *kbdev = container_of(data, struct kbase_device, ++ pm.backend.hwcnt_disable_work); ++ struct kbase_pm_device_data *pm = &kbdev->pm; ++ struct kbase_pm_backend_data *backend = &pm->backend; ++ unsigned long flags; ++ ++ bool do_disable; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ do_disable = !backend->hwcnt_desired && !backend->hwcnt_disabled; ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ if (!do_disable) ++ return; ++ ++ kbase_hwcnt_context_disable(kbdev->hwcnt_gpu_ctx); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ do_disable = !backend->hwcnt_desired && !backend->hwcnt_disabled; ++ ++ if (do_disable) { ++ /* PM state did not change while we were doing the disable, ++ * so commit the work we just performed and continue the state ++ * machine. ++ */ ++ backend->hwcnt_disabled = true; ++ kbase_pm_update_state(kbdev); ++#if !MALI_USE_CSF ++ kbase_backend_slot_update(kbdev); ++#endif /* !MALI_USE_CSF */ ++ } else { ++ /* PM state was updated while we were doing the disable, ++ * so we need to undo the disable we just performed. ++ */ ++ kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); ++ } ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++} ++ ++void kbase_pm_do_poweroff(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ ++ lockdep_assert_held(&kbdev->pm.lock); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ if (!kbdev->pm.backend.gpu_powered) ++ goto unlock_hwaccess; ++ ++ if (kbdev->pm.backend.poweroff_wait_in_progress) ++ goto unlock_hwaccess; ++ ++#if MALI_USE_CSF ++ kbdev->pm.backend.mcu_desired = false; ++#else ++ /* Force all cores off */ ++ kbdev->pm.backend.shaders_desired = false; ++#endif ++ kbdev->pm.backend.l2_desired = false; ++ ++ kbdev->pm.backend.poweroff_wait_in_progress = true; ++ kbdev->pm.backend.invoke_poweroff_wait_wq_when_l2_off = true; ++ ++ /* l2_desired being false should cause the state machine to ++ * start powering off the L2. When it actually is powered off, ++ * the interrupt handler will call kbase_pm_l2_update_state() ++ * again, which will trigger the kbase_pm_gpu_poweroff_wait_wq. ++ * Callers of this function will need to wait on poweroff_wait. ++ */ ++ kbase_pm_update_state(kbdev); ++ ++unlock_hwaccess: ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++} ++ ++static bool is_poweroff_in_progress(struct kbase_device *kbdev) ++{ ++ bool ret; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ret = (kbdev->pm.backend.poweroff_wait_in_progress == false); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ return ret; ++} ++ ++void kbase_pm_wait_for_poweroff_complete(struct kbase_device *kbdev) ++{ ++ wait_event_killable(kbdev->pm.backend.poweroff_wait, ++ is_poweroff_in_progress(kbdev)); ++} ++KBASE_EXPORT_TEST_API(kbase_pm_wait_for_poweroff_complete); ++ ++int kbase_hwaccess_pm_powerup(struct kbase_device *kbdev, ++ unsigned int flags) ++{ ++ unsigned long irq_flags; ++ int ret; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ kbase_pm_lock(kbdev); ++ ++ /* A suspend won't happen during startup/insmod */ ++ KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev)); ++ ++ /* Power up the GPU, don't enable IRQs as we are not ready to receive ++ * them. */ ++ ret = kbase_pm_init_hw(kbdev, flags); ++ if (ret) { ++ kbase_pm_unlock(kbdev); ++ return ret; ++ } ++ ++ kbdev->pm.debug_core_mask_all = kbdev->pm.debug_core_mask[0] = ++ kbdev->pm.debug_core_mask[1] = ++ kbdev->pm.debug_core_mask[2] = ++ kbdev->gpu_props.props.raw_props.shader_present; ++ ++ /* Pretend the GPU is active to prevent a power policy turning the GPU ++ * cores off */ ++ kbdev->pm.active_count = 1; ++ ++ spin_lock_irqsave(&kbdev->pm.backend.gpu_cycle_counter_requests_lock, ++ irq_flags); ++ /* Ensure cycle counter is off */ ++ kbdev->pm.backend.gpu_cycle_counter_requests = 0; ++ spin_unlock_irqrestore( ++ &kbdev->pm.backend.gpu_cycle_counter_requests_lock, ++ irq_flags); ++ ++ /* We are ready to receive IRQ's now as power policy is set up, so ++ * enable them now. */ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++ kbdev->pm.backend.driver_ready_for_irqs = true; ++#endif ++ kbase_pm_enable_interrupts(kbdev); ++ ++ WARN_ON(!kbdev->pm.backend.gpu_powered); ++ /* GPU has been powered up (by kbase_pm_init_hw) and interrupts have ++ * been enabled, so GPU is ready for use and PM state machine can be ++ * exercised from this point onwards. ++ */ ++ kbdev->pm.backend.gpu_ready = true; ++ ++ /* Turn on the GPU and any cores needed by the policy */ ++#if MALI_USE_CSF ++ /* Turn on the L2 caches, needed for firmware boot */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); ++ kbdev->pm.backend.l2_desired = true; ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); ++#endif ++ kbase_pm_do_poweron(kbdev, false); ++ kbase_pm_unlock(kbdev); ++ ++ return 0; ++} ++ ++void kbase_hwaccess_pm_halt(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ mutex_lock(&kbdev->pm.lock); ++ kbase_pm_do_poweroff(kbdev); ++ mutex_unlock(&kbdev->pm.lock); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_hwaccess_pm_halt); ++ ++void kbase_hwaccess_pm_term(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kbdev->pm.active_count == 0); ++ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests == 0); ++ ++ cancel_work_sync(&kbdev->pm.backend.hwcnt_disable_work); ++ ++ if (kbdev->pm.backend.hwcnt_disabled) { ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ } ++ ++ /* Free any resources the policy allocated */ ++ kbase_pm_state_machine_term(kbdev); ++ kbase_pm_policy_term(kbdev); ++ kbase_pm_ca_term(kbdev); ++ ++ /* Shut down the metrics subsystem */ ++ kbasep_pm_metrics_term(kbdev); ++ ++ destroy_workqueue(kbdev->pm.backend.gpu_poweroff_wait_wq); ++} ++ ++void kbase_pm_power_changed(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_pm_update_state(kbdev); ++ ++#if !MALI_USE_CSF ++ kbase_backend_slot_update(kbdev); ++#endif /* !MALI_USE_CSF */ ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++} ++ ++void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev, ++ u64 new_core_mask_js0, u64 new_core_mask_js1, ++ u64 new_core_mask_js2) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ lockdep_assert_held(&kbdev->pm.lock); ++ ++ if (kbase_dummy_job_wa_enabled(kbdev)) { ++ dev_warn(kbdev->dev, "Change of core mask not supported for slot 0 as dummy job WA is enabled"); ++ new_core_mask_js0 = kbdev->pm.debug_core_mask[0]; ++ } ++ ++ kbdev->pm.debug_core_mask[0] = new_core_mask_js0; ++ kbdev->pm.debug_core_mask[1] = new_core_mask_js1; ++ kbdev->pm.debug_core_mask[2] = new_core_mask_js2; ++ kbdev->pm.debug_core_mask_all = new_core_mask_js0 | new_core_mask_js1 | ++ new_core_mask_js2; ++ ++ kbase_pm_update_dynamic_cores_onoff(kbdev); ++} ++ ++void kbase_hwaccess_pm_gpu_active(struct kbase_device *kbdev) ++{ ++ kbase_pm_update_active(kbdev); ++} ++ ++void kbase_hwaccess_pm_gpu_idle(struct kbase_device *kbdev) ++{ ++ kbase_pm_update_active(kbdev); ++} ++ ++void kbase_hwaccess_pm_suspend(struct kbase_device *kbdev) ++{ ++ /* Force power off the GPU and all cores (regardless of policy), only ++ * after the PM active count reaches zero (otherwise, we risk turning it ++ * off prematurely) */ ++ kbase_pm_lock(kbdev); ++ ++ kbase_pm_do_poweroff(kbdev); ++ ++#if !MALI_USE_CSF ++ kbase_backend_timer_suspend(kbdev); ++#endif /* !MALI_USE_CSF */ ++ ++ kbase_pm_unlock(kbdev); ++ ++ kbase_pm_wait_for_poweroff_complete(kbdev); ++ ++ if (kbdev->pm.backend.callback_power_suspend) ++ kbdev->pm.backend.callback_power_suspend(kbdev); ++} ++ ++void kbase_hwaccess_pm_resume(struct kbase_device *kbdev) ++{ ++ kbase_pm_lock(kbdev); ++ ++ kbdev->pm.suspending = false; ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++ if (kbase_pm_is_gpu_lost(kbdev)) { ++ dev_dbg(kbdev->dev, "%s: GPU lost in progress\n", __func__); ++ kbase_pm_unlock(kbdev); ++ return; ++ } ++#endif ++ kbase_pm_do_poweron(kbdev, true); ++ ++#if !MALI_USE_CSF ++ kbase_backend_timer_resume(kbdev); ++#endif /* !MALI_USE_CSF */ ++ ++ kbase_pm_unlock(kbdev); ++} ++ ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++void kbase_pm_handle_gpu_lost(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ ktime_t end_timestamp = ktime_get(); ++ struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; ++ ++ mutex_lock(&kbdev->pm.lock); ++ mutex_lock(&arb_vm_state->vm_state_lock); ++ if (kbdev->pm.backend.gpu_powered && ++ !kbase_pm_is_gpu_lost(kbdev)) { ++ kbase_pm_set_gpu_lost(kbdev, true); ++ ++ /* GPU is no longer mapped to VM. So no interrupts will ++ * be received and Mali registers have been replaced by ++ * dummy RAM ++ */ ++ WARN(!kbase_is_gpu_removed(kbdev), ++ "GPU is still available after GPU lost event\n"); ++ ++ /* Full GPU reset will have been done by hypervisor, so ++ * cancel ++ */ ++ atomic_set(&kbdev->hwaccess.backend.reset_gpu, ++ KBASE_RESET_GPU_NOT_PENDING); ++ hrtimer_cancel(&kbdev->hwaccess.backend.reset_timer); ++ kbase_synchronize_irqs(kbdev); ++ ++ /* Clear all jobs running on the GPU */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbdev->protected_mode = false; ++ kbase_backend_reset(kbdev, &end_timestamp); ++ kbase_pm_metrics_update(kbdev, NULL); ++ kbase_pm_update_state(kbdev); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ /* Cancel any pending HWC dumps */ ++ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); ++ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; ++ kbdev->hwcnt.backend.triggered = 1; ++ wake_up(&kbdev->hwcnt.backend.wait); ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++ } ++ mutex_unlock(&arb_vm_state->vm_state_lock); ++ mutex_unlock(&kbdev->pm.lock); ++} ++ ++#endif /* CONFIG_MALI_ARBITER_SUPPORT */ +diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_ca.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_ca.c +new file mode 100755 +index 000000000000..984e12503009 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_ca.c +@@ -0,0 +1,114 @@ ++/* ++ * ++ * (C) COPYRIGHT 2013-2018, 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* ++ * Base kernel core availability APIs ++ */ ++ ++#include ++#include ++#include ++#ifdef MALI_BIFROST_NO_MALI ++#include ++#endif ++#include ++ ++int kbase_pm_ca_init(struct kbase_device *kbdev) ++{ ++#ifdef CONFIG_MALI_BIFROST_DEVFREQ ++ struct kbase_pm_backend_data *pm_backend = &kbdev->pm.backend; ++ ++ if (kbdev->current_core_mask) ++ pm_backend->ca_cores_enabled = kbdev->current_core_mask; ++ else ++ pm_backend->ca_cores_enabled = ++ kbdev->gpu_props.props.raw_props.shader_present; ++#endif ++ ++ return 0; ++} ++ ++void kbase_pm_ca_term(struct kbase_device *kbdev) ++{ ++} ++ ++#ifdef CONFIG_MALI_BIFROST_DEVFREQ ++void kbase_devfreq_set_core_mask(struct kbase_device *kbdev, u64 core_mask) ++{ ++ struct kbase_pm_backend_data *pm_backend = &kbdev->pm.backend; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ if (!(core_mask & kbdev->pm.debug_core_mask_all)) { ++ dev_err(kbdev->dev, "OPP core mask 0x%llX does not intersect with debug mask 0x%llX\n", ++ core_mask, kbdev->pm.debug_core_mask_all); ++ goto unlock; ++ } ++ ++ if (kbase_dummy_job_wa_enabled(kbdev)) { ++ dev_err(kbdev->dev, "Dynamic core scaling not supported as dummy job WA is enabled"); ++ goto unlock; ++ } ++ ++ pm_backend->ca_cores_enabled = core_mask; ++ ++ kbase_pm_update_state(kbdev); ++ ++unlock: ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ dev_dbg(kbdev->dev, "Devfreq policy : new core mask=%llX\n", ++ pm_backend->ca_cores_enabled); ++} ++#endif ++ ++u64 kbase_pm_ca_get_core_mask(struct kbase_device *kbdev) ++{ ++#ifdef CONFIG_MALI_BIFROST_DEVFREQ ++ struct kbase_pm_backend_data *pm_backend = &kbdev->pm.backend; ++#endif ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++#ifdef CONFIG_MALI_BIFROST_DEVFREQ ++ return pm_backend->ca_cores_enabled & kbdev->pm.debug_core_mask_all; ++#else ++ return kbdev->gpu_props.props.raw_props.shader_present & ++ kbdev->pm.debug_core_mask_all; ++#endif ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_ca_get_core_mask); ++ ++u64 kbase_pm_ca_get_instr_core_mask(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++#ifdef CONFIG_MALI_BIFROST_NO_MALI ++ return (((1ull) << KBASE_DUMMY_MODEL_MAX_SHADER_CORES) - 1); ++#elif MALI_USE_CSF ++ return kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER); ++#else ++ return kbdev->pm.backend.pm_shaders_core_mask; ++#endif ++} +diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_ca.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_ca.h +new file mode 100755 +index 000000000000..5423e96725b9 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_ca.h +@@ -0,0 +1,89 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2018 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* ++ * Base kernel core availability APIs ++ */ ++ ++#ifndef _KBASE_PM_CA_H_ ++#define _KBASE_PM_CA_H_ ++ ++/** ++ * kbase_pm_ca_init - Initialize core availability framework ++ * ++ * Must be called before calling any other core availability function ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * Return: 0 if the core availability framework was successfully initialized, ++ * -errno otherwise ++ */ ++int kbase_pm_ca_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_ca_term - Terminate core availability framework ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_ca_term(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_ca_get_core_mask - Get currently available shaders core mask ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * Returns a mask of the currently available shader cores. ++ * Calls into the core availability policy ++ * ++ * Return: The bit mask of available cores ++ */ ++u64 kbase_pm_ca_get_core_mask(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_ca_update_core_status - Update core status ++ * ++ * @kbdev: The kbase device structure for the device (must be ++ * a valid pointer) ++ * @cores_ready: The bit mask of cores ready for job submission ++ * @cores_transitioning: The bit mask of cores that are transitioning power ++ * state ++ * ++ * Update core availability policy with current core power status ++ * ++ * Calls into the core availability policy ++ */ ++void kbase_pm_ca_update_core_status(struct kbase_device *kbdev, u64 cores_ready, ++ u64 cores_transitioning); ++ ++/** ++ * kbase_pm_ca_get_instr_core_mask - Get the PM state sync-ed shaders core mask ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * Returns a mask of the PM state synchronised shader cores for arranging ++ * HW performance counter dumps ++ * ++ * Return: The bit mask of PM state synchronised cores ++ */ ++u64 kbase_pm_ca_get_instr_core_mask(struct kbase_device *kbdev); ++ ++#endif /* _KBASE_PM_CA_H_ */ +diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_ca_devfreq.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_ca_devfreq.h +new file mode 100755 +index 000000000000..f67ec650c981 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_ca_devfreq.h +@@ -0,0 +1,60 @@ ++/* ++ * ++ * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* ++ * A core availability policy for use with devfreq, where core masks are ++ * associated with OPPs. ++ */ ++ ++#ifndef MALI_KBASE_PM_CA_DEVFREQ_H ++#define MALI_KBASE_PM_CA_DEVFREQ_H ++ ++/** ++ * struct kbasep_pm_ca_policy_devfreq - Private structure for devfreq ca policy ++ * ++ * This contains data that is private to the devfreq core availability ++ * policy. ++ * ++ * @cores_desired: Cores that the policy wants to be available ++ * @cores_enabled: Cores that the policy is currently returning as available ++ * @cores_used: Cores currently powered or transitioning ++ */ ++struct kbasep_pm_ca_policy_devfreq { ++ u64 cores_desired; ++ u64 cores_enabled; ++ u64 cores_used; ++}; ++ ++extern const struct kbase_pm_ca_policy kbase_pm_ca_devfreq_policy_ops; ++ ++/** ++ * kbase_devfreq_set_core_mask - Set core mask for policy to use ++ * @kbdev: Device pointer ++ * @core_mask: New core mask ++ * ++ * The new core mask will have immediate effect if the GPU is powered, or will ++ * take effect when it is next powered on. ++ */ ++void kbase_devfreq_set_core_mask(struct kbase_device *kbdev, u64 core_mask); ++ ++#endif /* MALI_KBASE_PM_CA_DEVFREQ_H */ ++ +diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_coarse_demand.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_coarse_demand.c +new file mode 100755 +index 000000000000..9eef44ad877f +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_coarse_demand.c +@@ -0,0 +1,66 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2016, 2018-2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++/* ++ * "Coarse Demand" power management policy ++ */ ++ ++#include ++#include ++ ++static bool coarse_demand_shaders_needed(struct kbase_device *kbdev) ++{ ++ return kbase_pm_is_active(kbdev); ++} ++ ++static bool coarse_demand_get_core_active(struct kbase_device *kbdev) ++{ ++ return kbase_pm_is_active(kbdev); ++} ++ ++static void coarse_demand_init(struct kbase_device *kbdev) ++{ ++ CSTD_UNUSED(kbdev); ++} ++ ++static void coarse_demand_term(struct kbase_device *kbdev) ++{ ++ CSTD_UNUSED(kbdev); ++} ++ ++/* The struct kbase_pm_policy structure for the demand power policy. ++ * ++ * This is the static structure that defines the demand power policy's callback ++ * and name. ++ */ ++const struct kbase_pm_policy kbase_pm_coarse_demand_policy_ops = { ++ "coarse_demand", /* name */ ++ coarse_demand_init, /* init */ ++ coarse_demand_term, /* term */ ++ coarse_demand_shaders_needed, /* shaders_needed */ ++ coarse_demand_get_core_active, /* get_core_active */ ++ KBASE_PM_POLICY_ID_COARSE_DEMAND, /* id */ ++}; ++ ++KBASE_EXPORT_TEST_API(kbase_pm_coarse_demand_policy_ops); +diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_coarse_demand.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_coarse_demand.h +new file mode 100755 +index 000000000000..304e5d7fa32d +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_coarse_demand.h +@@ -0,0 +1,69 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2015,2018 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++/* ++ * "Coarse Demand" power management policy ++ */ ++ ++#ifndef MALI_KBASE_PM_COARSE_DEMAND_H ++#define MALI_KBASE_PM_COARSE_DEMAND_H ++ ++/** ++ * DOC: ++ * The "Coarse" demand power management policy has the following ++ * characteristics: ++ * - When KBase indicates that the GPU will be powered up, but we don't yet ++ * know which Job Chains are to be run: ++ * - Shader Cores are powered up, regardless of whether or not they will be ++ * needed later. ++ * - When KBase indicates that Shader Cores are needed to submit the currently ++ * queued Job Chains: ++ * - Shader Cores are kept powered, regardless of whether or not they will ++ * be needed ++ * - When KBase indicates that the GPU need not be powered: ++ * - The Shader Cores are powered off, and the GPU itself is powered off too. ++ * ++ * @note: ++ * - KBase indicates the GPU will be powered up when it has a User Process that ++ * has just started to submit Job Chains. ++ * - KBase indicates the GPU need not be powered when all the Job Chains from ++ * User Processes have finished, and it is waiting for a User Process to ++ * submit some more Job Chains. ++ */ ++ ++/** ++ * struct kbasep_pm_policy_coarse_demand - Private structure for coarse demand ++ * policy ++ * ++ * This contains data that is private to the coarse demand power policy. ++ * ++ * @dummy: Dummy member - no state needed ++ */ ++struct kbasep_pm_policy_coarse_demand { ++ int dummy; ++}; ++ ++extern const struct kbase_pm_policy kbase_pm_coarse_demand_policy_ops; ++ ++#endif /* MALI_KBASE_PM_COARSE_DEMAND_H */ +diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_defs.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_defs.h +new file mode 100755 +index 000000000000..7322c093c7b6 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_defs.h +@@ -0,0 +1,560 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* ++ * Backend-specific Power Manager definitions ++ */ ++ ++#ifndef _KBASE_PM_HWACCESS_DEFS_H_ ++#define _KBASE_PM_HWACCESS_DEFS_H_ ++ ++#include "mali_kbase_pm_always_on.h" ++#include "mali_kbase_pm_coarse_demand.h" ++#if !MALI_CUSTOMER_RELEASE ++#include "mali_kbase_pm_always_on_demand.h" ++#endif ++ ++/* Forward definition - see mali_kbase.h */ ++struct kbase_device; ++struct kbase_jd_atom; ++ ++/** ++ * enum kbase_pm_core_type - The types of core in a GPU. ++ * ++ * These enumerated values are used in calls to ++ * - kbase_pm_get_present_cores() ++ * - kbase_pm_get_active_cores() ++ * - kbase_pm_get_trans_cores() ++ * - kbase_pm_get_ready_cores(). ++ * ++ * They specify which type of core should be acted on. These values are set in ++ * a manner that allows core_type_to_reg() function to be simpler and more ++ * efficient. ++ * ++ * @KBASE_PM_CORE_L2: The L2 cache ++ * @KBASE_PM_CORE_SHADER: Shader cores ++ * @KBASE_PM_CORE_TILER: Tiler cores ++ * @KBASE_PM_CORE_STACK: Core stacks ++ */ ++enum kbase_pm_core_type { ++ KBASE_PM_CORE_L2 = L2_PRESENT_LO, ++ KBASE_PM_CORE_SHADER = SHADER_PRESENT_LO, ++ KBASE_PM_CORE_TILER = TILER_PRESENT_LO, ++ KBASE_PM_CORE_STACK = STACK_PRESENT_LO ++}; ++ ++/** ++ * enum kbase_l2_core_state - The states used for the L2 cache & tiler power ++ * state machine. ++ * ++ * @KBASE_L2_OFF: The L2 cache and tiler are off ++ * @KBASE_L2_PEND_ON: The L2 cache and tiler are powering on ++ * @KBASE_L2_RESTORE_CLOCKS: The GPU clock is restored. Conditionally used. ++ * @KBASE_L2_ON_HWCNT_ENABLE: The L2 cache and tiler are on, and hwcnt is being ++ * enabled ++ * @KBASE_L2_ON: The L2 cache and tiler are on, and hwcnt is enabled ++ * @KBASE_L2_ON_HWCNT_DISABLE: The L2 cache and tiler are on, and hwcnt is being ++ * disabled ++ * @KBASE_L2_SLOW_DOWN_CLOCKS: The GPU clock is set to appropriate or lowest ++ * clock. Conditionally used. ++ * @KBASE_L2_POWER_DOWN: The L2 cache and tiler are about to be powered off ++ * @KBASE_L2_PEND_OFF: The L2 cache and tiler are powering off ++ * @KBASE_L2_RESET_WAIT: The GPU is resetting, L2 cache and tiler power state ++ * are unknown ++ */ ++enum kbase_l2_core_state { ++#define KBASEP_L2_STATE(n) KBASE_L2_ ## n, ++#include "mali_kbase_pm_l2_states.h" ++#undef KBASEP_L2_STATE ++}; ++ ++#if MALI_USE_CSF ++/** ++ * enum kbase_mcu_state - The states used for the MCU state machine. ++ * ++ * @KBASE_MCU_OFF: The MCU is powered off. ++ * @KBASE_MCU_PEND_ON_RELOAD: The warm boot of MCU or cold boot of MCU (with ++ * firmware reloading) is in progress. ++ * @KBASE_MCU_ON_GLB_REINIT_PEND: The MCU is enabled and Global configuration ++ * requests have been sent to the firmware. ++ * @KBASE_MCU_ON_HWCNT_ENABLE: The Global requests have completed and MCU is ++ * now ready for use and hwcnt is being enabled. ++ * @KBASE_MCU_ON: The MCU is active and hwcnt has been enabled. ++ * @KBASE_MCU_ON_HWCNT_DISABLE: The MCU is on and hwcnt is being disabled. ++ * @KBASE_MCU_ON_HALT: The MCU is on and hwcnt has been disabled, ++ * MCU halt would be triggered. ++ * @KBASE_MCU_ON_PEND_HALT: MCU halt in progress, confirmation pending. ++ * @KBASE_MCU_POWER_DOWN: MCU halted operations, pending being disabled. ++ * @KBASE_MCU_PEND_OFF: MCU is being disabled, pending on powering off. ++ * @KBASE_MCU_RESET_WAIT: The GPU is resetting, MCU state is unknown. ++ */ ++enum kbase_mcu_state { ++#define KBASEP_MCU_STATE(n) KBASE_MCU_ ## n, ++#include "mali_kbase_pm_mcu_states.h" ++#undef KBASEP_MCU_STATE ++}; ++#endif ++ ++/** ++ * enum kbase_shader_core_state - The states used for the shaders' state machine. ++ * ++ * @KBASE_SHADERS_OFF_CORESTACK_OFF: The shaders and core stacks are off ++ * @KBASE_SHADERS_OFF_CORESTACK_PEND_ON: The shaders are off, core stacks have ++ * been requested to power on and hwcnt ++ * is being disabled ++ * @KBASE_SHADERS_PEND_ON_CORESTACK_ON: Core stacks are on, shaders have been ++ * requested to power on. Or after doing ++ * partial shader on/off, checking whether ++ * it's the desired state. ++ * @KBASE_SHADERS_ON_CORESTACK_ON: The shaders and core stacks are on, and hwcnt ++ * already enabled. ++ * @KBASE_SHADERS_ON_CORESTACK_ON_RECHECK: The shaders and core stacks ++ * are on, hwcnt disabled, and checks ++ * to powering down or re-enabling ++ * hwcnt. ++ * @KBASE_SHADERS_WAIT_OFF_CORESTACK_ON: The shaders have been requested to ++ * power off, but they remain on for the ++ * duration of the hysteresis timer ++ * @KBASE_SHADERS_WAIT_GPU_IDLE: The shaders partial poweroff needs to reach ++ * a state where jobs on the GPU are finished ++ * including jobs currently running and in the ++ * GPU queue because of GPU2017-861 ++ * @KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON: The hysteresis timer has expired ++ * @KBASE_SHADERS_L2_FLUSHING_CORESTACK_ON: The core stacks are on and the ++ * level 2 cache is being flushed. ++ * @KBASE_SHADERS_READY_OFF_CORESTACK_ON: The core stacks are on and the shaders ++ * are ready to be powered off. ++ * @KBASE_SHADERS_PEND_OFF_CORESTACK_ON: The core stacks are on, and the shaders ++ * have been requested to power off ++ * @KBASE_SHADERS_OFF_CORESTACK_PEND_OFF: The shaders are off, and the core stacks ++ * have been requested to power off ++ * @KBASE_SHADERS_OFF_CORESTACK_OFF_TIMER_PEND_OFF: Shaders and corestacks are ++ * off, but the tick timer ++ * cancellation is still ++ * pending. ++ * @KBASE_SHADERS_RESET_WAIT: The GPU is resetting, shader and core stack power ++ * states are unknown ++ */ ++enum kbase_shader_core_state { ++#define KBASEP_SHADER_STATE(n) KBASE_SHADERS_ ## n, ++#include "mali_kbase_pm_shader_states.h" ++#undef KBASEP_SHADER_STATE ++}; ++ ++/** ++ * struct kbasep_pm_metrics - Metrics data collected for use by the power ++ * management framework. ++ * ++ * @time_busy: number of ns the GPU was busy executing jobs since the ++ * @time_period_start timestamp. ++ * @time_idle: number of ns since time_period_start the GPU was not executing ++ * jobs since the @time_period_start timestamp. ++ * @busy_cl: number of ns the GPU was busy executing CL jobs. Note that ++ * if two CL jobs were active for 400ns, this value would be updated ++ * with 800. ++ * @busy_gl: number of ns the GPU was busy executing GL jobs. Note that ++ * if two GL jobs were active for 400ns, this value would be updated ++ * with 800. ++ */ ++struct kbasep_pm_metrics { ++ u32 time_busy; ++ u32 time_idle; ++ u32 busy_cl[2]; ++ u32 busy_gl; ++}; ++ ++/** ++ * struct kbasep_pm_metrics_state - State required to collect the metrics in ++ * struct kbasep_pm_metrics ++ * @time_period_start: time at which busy/idle measurements started ++ * @gpu_active: true when the GPU is executing jobs. false when ++ * not. Updated when the job scheduler informs us a job in submitted ++ * or removed from a GPU slot. ++ * @active_cl_ctx: number of CL jobs active on the GPU. Array is per-device. ++ * @active_gl_ctx: number of GL jobs active on the GPU. Array is per-slot. ++ * @lock: spinlock protecting the kbasep_pm_metrics_data structure ++ * @platform_data: pointer to data controlled by platform specific code ++ * @kbdev: pointer to kbase device for which metrics are collected ++ * @values: The current values of the power management metrics. The ++ * kbase_pm_get_dvfs_metrics() function is used to compare these ++ * current values with the saved values from a previous invocation. ++ * @timer: timer to regularly make DVFS decisions based on the power ++ * management metrics. ++ * @timer_active: boolean indicating @timer is running ++ * @dvfs_last: values of the PM metrics from the last DVFS tick ++ * @dvfs_diff: different between the current and previous PM metrics. ++ */ ++struct kbasep_pm_metrics_state { ++ ktime_t time_period_start; ++ bool gpu_active; ++ u32 active_cl_ctx[2]; ++ u32 active_gl_ctx[3]; ++ spinlock_t lock; ++ ++ void *platform_data; ++ struct kbase_device *kbdev; ++ ++ struct kbasep_pm_metrics values; ++ ++#ifdef CONFIG_MALI_BIFROST_DVFS ++ struct hrtimer timer; ++ bool timer_active; ++ struct kbasep_pm_metrics dvfs_last; ++ struct kbasep_pm_metrics dvfs_diff; ++#endif ++}; ++ ++/** ++ * struct kbasep_pm_tick_timer_state - State for the shader hysteresis timer ++ * @wq: Work queue to wait for the timer to stopped ++ * @work: Work item which cancels the timer ++ * @timer: Timer for powering off the shader cores ++ * @configured_interval: Period of GPU poweroff timer ++ * @configured_ticks: User-configured number of ticks to wait after the shader ++ * power down request is received before turning off the cores ++ * @remaining_ticks: Number of remaining timer ticks until shaders are powered off ++ * @cancel_queued: True if the cancellation work item has been queued. This is ++ * required to ensure that it is not queued twice, e.g. after ++ * a reset, which could cause the timer to be incorrectly ++ * cancelled later by a delayed workitem. ++ * @needed: Whether the timer should restart itself ++ */ ++struct kbasep_pm_tick_timer_state { ++ struct workqueue_struct *wq; ++ struct work_struct work; ++ struct hrtimer timer; ++ ++ ktime_t configured_interval; ++ unsigned int configured_ticks; ++ unsigned int remaining_ticks; ++ ++ bool cancel_queued; ++ bool needed; ++}; ++ ++union kbase_pm_policy_data { ++ struct kbasep_pm_policy_always_on always_on; ++ struct kbasep_pm_policy_coarse_demand coarse_demand; ++#if !MALI_CUSTOMER_RELEASE ++ struct kbasep_pm_policy_always_on_demand always_on_demand; ++#endif ++}; ++ ++/** ++ * struct kbase_pm_backend_data - Data stored per device for power management. ++ * ++ * This structure contains data for the power management framework. There is one ++ * instance of this structure per device in the system. ++ * ++ * @pm_current_policy: The policy that is currently actively controlling the ++ * power state. ++ * @pm_policy_data: Private data for current PM policy ++ * @reset_done: Flag when a reset is complete ++ * @reset_done_wait: Wait queue to wait for changes to @reset_done ++ * @gpu_cycle_counter_requests: The reference count of active gpu cycle counter ++ * users ++ * @gpu_cycle_counter_requests_lock: Lock to protect @gpu_cycle_counter_requests ++ * @gpu_in_desired_state_wait: Wait queue set when the GPU is in the desired ++ * state according to the L2 and shader power state ++ * machines ++ * @gpu_powered: Set to true when the GPU is powered and register ++ * accesses are possible, false otherwise. Access to this ++ * variable should be protected by: both the hwaccess_lock ++ * spinlock and the pm.lock mutex for writes; or at least ++ * one of either lock for reads. ++ * @gpu_ready: Indicates whether the GPU is in a state in which it is ++ * safe to perform PM changes. When false, the PM state ++ * machine needs to wait before making changes to the GPU ++ * power policy, DevFreq or core_mask, so as to avoid these ++ * changing while implicit GPU resets are ongoing. ++ * @pm_shaders_core_mask: Shader PM state synchronised shaders core mask. It ++ * holds the cores enabled in a hardware counters dump, ++ * and may differ from @shaders_avail when under different ++ * states and transitions. ++ * @cg1_disabled: Set if the policy wants to keep the second core group ++ * powered off ++ * @driver_ready_for_irqs: Debug state indicating whether sufficient ++ * initialization of the driver has occurred to handle ++ * IRQs ++ * @metrics: Structure to hold metrics for the GPU ++ * @shader_tick_timer: Structure to hold the shader poweroff tick timer state ++ * @poweroff_wait_in_progress: true if a wait for GPU power off is in progress. ++ * hwaccess_lock must be held when accessing ++ * @invoke_poweroff_wait_wq_when_l2_off: flag indicating that the L2 power state ++ * machine should invoke the poweroff ++ * worker after the L2 has turned off. ++ * @poweron_required: true if a GPU power on is required. Should only be set ++ * when poweroff_wait_in_progress is true, and therefore the ++ * GPU can not immediately be powered on. pm.lock must be ++ * held when accessing ++ * @gpu_poweroff_wait_wq: workqueue for waiting for GPU to power off ++ * @gpu_poweroff_wait_work: work item for use with @gpu_poweroff_wait_wq ++ * @poweroff_wait: waitqueue for waiting for @gpu_poweroff_wait_work to complete ++ * @callback_power_on: Callback when the GPU needs to be turned on. See ++ * &struct kbase_pm_callback_conf ++ * @callback_power_off: Callback when the GPU may be turned off. See ++ * &struct kbase_pm_callback_conf ++ * @callback_power_suspend: Callback when a suspend occurs and the GPU needs to ++ * be turned off. See &struct kbase_pm_callback_conf ++ * @callback_power_resume: Callback when a resume occurs and the GPU needs to ++ * be turned on. See &struct kbase_pm_callback_conf ++ * @callback_power_runtime_on: Callback when the GPU needs to be turned on. See ++ * &struct kbase_pm_callback_conf ++ * @callback_power_runtime_off: Callback when the GPU may be turned off. See ++ * &struct kbase_pm_callback_conf ++ * @callback_power_runtime_idle: Optional callback when the GPU may be idle. See ++ * &struct kbase_pm_callback_conf ++ * @callback_soft_reset: Optional callback to software reset the GPU. See ++ * &struct kbase_pm_callback_conf ++ * @ca_cores_enabled: Cores that are currently available ++ * @l2_state: The current state of the L2 cache state machine. See ++ * &enum kbase_l2_core_state ++ * @l2_desired: True if the L2 cache should be powered on by the L2 cache state ++ * machine ++ * @l2_always_on: If true, disable powering down of l2 cache. ++ * @shaders_state: The current state of the shader state machine. ++ * @shaders_avail: This is updated by the state machine when it is in a state ++ * where it can write to the SHADER_PWRON or PWROFF registers ++ * to have the same set of available cores as specified by ++ * @shaders_desired_mask. So it would eventually have the same ++ * value as @shaders_desired_mask and would precisely indicate ++ * the cores that are currently available. This is internal to ++ * shader state machine and should *not* be modified elsewhere. ++ * @shaders_desired_mask: This is updated by the state machine when it is in ++ * a state where it can handle changes to the core ++ * availability (either by DVFS or sysfs). This is ++ * internal to the shader state machine and should ++ * *not* be modified elsewhere. ++ * @shaders_desired: True if the PM active count or power policy requires the ++ * shader cores to be on. This is used as an input to the ++ * shader power state machine. The current state of the ++ * cores may be different, but there should be transitions in ++ * progress that will eventually achieve this state (assuming ++ * that the policy doesn't change its mind in the mean time). ++ * @in_reset: True if a GPU is resetting and normal power manager operation is ++ * suspended ++ * @partial_shaderoff: True if we want to partial power off shader cores, ++ * it indicates a partial shader core off case, ++ * do some special operation for such case like flush ++ * L2 cache because of GPU2017-861 ++ * @protected_entry_transition_override : True if GPU reset is being used ++ * before entering the protected mode and so ++ * the reset handling behaviour is being ++ * overridden. ++ * @protected_transition_override : True if a protected mode transition is in ++ * progress and is overriding power manager ++ * behaviour. ++ * @protected_l2_override : Non-zero if the L2 cache is required during a ++ * protected mode transition. Has no effect if not ++ * transitioning. ++ * @hwcnt_desired: True if we want GPU hardware counters to be enabled. ++ * @hwcnt_disabled: True if GPU hardware counters are not enabled. ++ * @hwcnt_disable_work: Work item to disable GPU hardware counters, used if ++ * atomic disable is not possible. ++ * @gpu_clock_suspend_freq: 'opp-mali-errata-1485982' clock in opp table ++ * for safe L2 power cycle. ++ * If no opp-mali-errata-1485982 specified, ++ * the slowest clock will be taken. ++ * @gpu_clock_slow_down_wa: If true, slow down GPU clock during L2 power cycle. ++ * @gpu_clock_slow_down_desired: True if we want lower GPU clock ++ * for safe L2 power cycle. False if want GPU clock ++ * to back to normalized one. This is updated only ++ * in L2 state machine, kbase_pm_l2_update_state. ++ * @gpu_clock_slowed_down: During L2 power cycle, ++ * True if gpu clock is set at lower frequency ++ * for safe L2 power down, False if gpu clock gets ++ * restored to previous speed. This is updated only in ++ * work function, kbase_pm_gpu_clock_control_worker. ++ * @gpu_clock_control_work: work item to set GPU clock during L2 power cycle ++ * using gpu_clock_control ++ * ++ * Note: ++ * During an IRQ, @pm_current_policy can be NULL when the policy is being ++ * changed with kbase_pm_set_policy(). The change is protected under ++ * kbase_device.pm.pcower_change_lock. Direct access to this from IRQ context ++ * must therefore check for NULL. If NULL, then kbase_pm_set_policy() will ++ * re-issue the policy functions that would have been done under IRQ. ++ */ ++struct kbase_pm_backend_data { ++ const struct kbase_pm_policy *pm_current_policy; ++ union kbase_pm_policy_data pm_policy_data; ++ bool reset_done; ++ wait_queue_head_t reset_done_wait; ++ int gpu_cycle_counter_requests; ++ spinlock_t gpu_cycle_counter_requests_lock; ++ ++ wait_queue_head_t gpu_in_desired_state_wait; ++ ++ bool gpu_powered; ++ bool gpu_ready; ++ ++ u64 pm_shaders_core_mask; ++ ++ bool cg1_disabled; ++ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++ bool driver_ready_for_irqs; ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++ ++ struct kbasep_pm_metrics_state metrics; ++ ++ struct kbasep_pm_tick_timer_state shader_tick_timer; ++ ++ bool poweroff_wait_in_progress; ++ bool invoke_poweroff_wait_wq_when_l2_off; ++ bool poweron_required; ++ ++ struct workqueue_struct *gpu_poweroff_wait_wq; ++ struct work_struct gpu_poweroff_wait_work; ++ ++ wait_queue_head_t poweroff_wait; ++ ++ int (*callback_power_on)(struct kbase_device *kbdev); ++ void (*callback_power_off)(struct kbase_device *kbdev); ++ void (*callback_power_suspend)(struct kbase_device *kbdev); ++ void (*callback_power_resume)(struct kbase_device *kbdev); ++ int (*callback_power_runtime_on)(struct kbase_device *kbdev); ++ void (*callback_power_runtime_off)(struct kbase_device *kbdev); ++ int (*callback_power_runtime_idle)(struct kbase_device *kbdev); ++ int (*callback_soft_reset)(struct kbase_device *kbdev); ++ ++ u64 ca_cores_enabled; ++ ++#if MALI_USE_CSF ++ /* The current state of the micro-control unit, only applicable ++ * to GPUs that has such a component ++ */ ++ enum kbase_mcu_state mcu_state; ++#endif ++ enum kbase_l2_core_state l2_state; ++ enum kbase_shader_core_state shaders_state; ++ u64 shaders_avail; ++ u64 shaders_desired_mask; ++#if MALI_USE_CSF ++ /* True if the micro-control unit should be powered on */ ++ bool mcu_desired; ++#endif ++ bool l2_desired; ++ bool l2_always_on; ++ bool shaders_desired; ++ ++ bool in_reset; ++ ++ bool partial_shaderoff; ++ ++ bool protected_entry_transition_override; ++ bool protected_transition_override; ++ int protected_l2_override; ++ ++ bool hwcnt_desired; ++ bool hwcnt_disabled; ++ struct work_struct hwcnt_disable_work; ++ ++ u64 gpu_clock_suspend_freq; ++ bool gpu_clock_slow_down_wa; ++ bool gpu_clock_slow_down_desired; ++ bool gpu_clock_slowed_down; ++ struct work_struct gpu_clock_control_work; ++}; ++ ++ ++/* List of policy IDs */ ++enum kbase_pm_policy_id { ++ KBASE_PM_POLICY_ID_COARSE_DEMAND, ++#if !MALI_CUSTOMER_RELEASE ++ KBASE_PM_POLICY_ID_ALWAYS_ON_DEMAND, ++#endif ++ KBASE_PM_POLICY_ID_ALWAYS_ON ++}; ++ ++/** ++ * struct kbase_pm_policy - Power policy structure. ++ * ++ * Each power policy exposes a (static) instance of this structure which ++ * contains function pointers to the policy's methods. ++ * ++ * @name: The name of this policy ++ * @init: Function called when the policy is selected ++ * @term: Function called when the policy is unselected ++ * @shaders_needed: Function called to find out if shader cores are needed ++ * @get_core_active: Function called to get the current overall GPU power ++ * state ++ * @id: Field indicating an ID for this policy. This is not ++ * necessarily the same as its index in the list returned ++ * by kbase_pm_list_policies(). ++ * It is used purely for debugging. ++ */ ++struct kbase_pm_policy { ++ char *name; ++ ++ /** ++ * Function called when the policy is selected ++ * ++ * This should initialize the kbdev->pm.pm_policy_data structure. It ++ * should not attempt to make any changes to hardware state. ++ * ++ * It is undefined what state the cores are in when the function is ++ * called. ++ * ++ * @kbdev: The kbase device structure for the device (must be a ++ * valid pointer) ++ */ ++ void (*init)(struct kbase_device *kbdev); ++ ++ /** ++ * Function called when the policy is unselected. ++ * ++ * @kbdev: The kbase device structure for the device (must be a ++ * valid pointer) ++ */ ++ void (*term)(struct kbase_device *kbdev); ++ ++ /** ++ * Function called to find out if shader cores are needed ++ * ++ * This needs to at least satisfy kbdev->pm.backend.shaders_desired, ++ * and so must never return false when shaders_desired is true. ++ * ++ * @kbdev: The kbase device structure for the device (must be a ++ * valid pointer) ++ * ++ * Return: true if shader cores are needed, false otherwise ++ */ ++ bool (*shaders_needed)(struct kbase_device *kbdev); ++ ++ /** ++ * Function called to get the current overall GPU power state ++ * ++ * This function must meet or exceed the requirements for power ++ * indicated by kbase_pm_is_active(). ++ * ++ * @kbdev: The kbase device structure for the device (must be a ++ * valid pointer) ++ * ++ * Return: true if the GPU should be powered, false otherwise ++ */ ++ bool (*get_core_active)(struct kbase_device *kbdev); ++ ++ enum kbase_pm_policy_id id; ++}; ++ ++#endif /* _KBASE_PM_HWACCESS_DEFS_H_ */ +diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_driver.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_driver.c +new file mode 100755 +index 000000000000..e9e30ebadc2d +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_driver.c +@@ -0,0 +1,2545 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * ++ * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* ++ * Base kernel Power Management hardware control ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#if !MALI_USE_CSF ++#include ++#endif /* !MALI_USE_CSF */ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++#include ++#endif /* CONFIG_MALI_ARBITER_SUPPORT */ ++ ++#include ++ ++#ifdef CONFIG_MALI_CORESTACK ++bool corestack_driver_control = true; ++#else ++bool corestack_driver_control; /* Default value of 0/false */ ++#endif ++module_param(corestack_driver_control, bool, 0444); ++MODULE_PARM_DESC(corestack_driver_control, ++ "Let the driver power on/off the GPU core stack independently " ++ "without involving the Power Domain Controller. This should " ++ "only be enabled on platforms for which integration of the PDC " ++ "to the Mali GPU is known to be problematic."); ++KBASE_EXPORT_TEST_API(corestack_driver_control); ++ ++/** ++ * enum kbasep_pm_action - Actions that can be performed on a core. ++ * ++ * This enumeration is private to the file. Its values are set to allow ++ * core_type_to_reg() function, which decodes this enumeration, to be simpler ++ * and more efficient. ++ * ++ * @ACTION_PRESENT: The cores that are present ++ * @ACTION_READY: The cores that are ready ++ * @ACTION_PWRON: Power on the cores specified ++ * @ACTION_PWROFF: Power off the cores specified ++ * @ACTION_PWRTRANS: The cores that are transitioning ++ * @ACTION_PWRACTIVE: The cores that are active ++ */ ++enum kbasep_pm_action { ++ ACTION_PRESENT = 0, ++ ACTION_READY = (SHADER_READY_LO - SHADER_PRESENT_LO), ++ ACTION_PWRON = (SHADER_PWRON_LO - SHADER_PRESENT_LO), ++ ACTION_PWROFF = (SHADER_PWROFF_LO - SHADER_PRESENT_LO), ++ ACTION_PWRTRANS = (SHADER_PWRTRANS_LO - SHADER_PRESENT_LO), ++ ACTION_PWRACTIVE = (SHADER_PWRACTIVE_LO - SHADER_PRESENT_LO) ++}; ++ ++static u64 kbase_pm_get_state( ++ struct kbase_device *kbdev, ++ enum kbase_pm_core_type core_type, ++ enum kbasep_pm_action action); ++ ++#if MALI_USE_CSF ++bool kbase_pm_is_mcu_desired(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (unlikely(!kbdev->csf.firmware_inited)) ++ return false; ++ ++ if (kbdev->csf.scheduler.pm_active_count) ++ return true; ++ ++ /* MCU is supposed to be ON, only when scheduler.pm_active_count is ++ * non zero. But for always_on policy also MCU needs to be ON. ++ * GPUCORE-24926 will add the proper handling for always_on ++ * power policy. ++ */ ++ return (kbdev->pm.backend.mcu_desired && ++ (kbdev->pm.backend.pm_current_policy == ++ &kbase_pm_always_on_policy_ops)); ++} ++#endif ++ ++bool kbase_pm_is_l2_desired(struct kbase_device *kbdev) ++{ ++ if (kbdev->pm.backend.protected_entry_transition_override) ++ return false; ++ ++ if (kbdev->pm.backend.protected_transition_override && ++ kbdev->pm.backend.protected_l2_override) ++ return true; ++ ++ if (kbdev->pm.backend.protected_transition_override && ++ !kbdev->pm.backend.shaders_desired) ++ return false; ++ ++ return kbdev->pm.backend.l2_desired; ++} ++ ++void kbase_pm_protected_override_enable(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ kbdev->pm.backend.protected_transition_override = true; ++} ++void kbase_pm_protected_override_disable(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ kbdev->pm.backend.protected_transition_override = false; ++} ++ ++int kbase_pm_protected_entry_override_enable(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ WARN_ON(!kbdev->protected_mode_transition); ++ ++ if (kbdev->pm.backend.l2_always_on && ++ (kbdev->system_coherency == COHERENCY_ACE)) { ++ WARN_ON(kbdev->pm.backend.protected_entry_transition_override); ++ ++ /* ++ * If there is already a GPU reset pending then wait for it to ++ * complete before initiating a special reset for protected ++ * mode entry. ++ */ ++ if (kbase_reset_gpu_silent(kbdev)) ++ return -EAGAIN; ++ ++ kbdev->pm.backend.protected_entry_transition_override = true; ++ } ++ ++ return 0; ++} ++ ++void kbase_pm_protected_entry_override_disable(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ WARN_ON(!kbdev->protected_mode_transition); ++ ++ if (kbdev->pm.backend.l2_always_on && ++ (kbdev->system_coherency == COHERENCY_ACE)) { ++ WARN_ON(!kbdev->pm.backend.protected_entry_transition_override); ++ ++ kbdev->pm.backend.protected_entry_transition_override = false; ++ } ++} ++ ++void kbase_pm_protected_l2_override(struct kbase_device *kbdev, bool override) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (override) { ++ kbdev->pm.backend.protected_l2_override++; ++ WARN_ON(kbdev->pm.backend.protected_l2_override <= 0); ++ } else { ++ kbdev->pm.backend.protected_l2_override--; ++ WARN_ON(kbdev->pm.backend.protected_l2_override < 0); ++ } ++ ++ kbase_pm_update_state(kbdev); ++} ++ ++/** ++ * core_type_to_reg - Decode a core type and action to a register. ++ * ++ * Given a core type (defined by kbase_pm_core_type) and an action (defined ++ * by kbasep_pm_action) this function will return the register offset that ++ * will perform the action on the core type. The register returned is the _LO ++ * register and an offset must be applied to use the _HI register. ++ * ++ * @core_type: The type of core ++ * @action: The type of action ++ * ++ * Return: The register offset of the _LO register that performs an action of ++ * type @action on a core of type @core_type. ++ */ ++static u32 core_type_to_reg(enum kbase_pm_core_type core_type, ++ enum kbasep_pm_action action) ++{ ++ if (corestack_driver_control) { ++ if (core_type == KBASE_PM_CORE_STACK) { ++ switch (action) { ++ case ACTION_PRESENT: ++ return STACK_PRESENT_LO; ++ case ACTION_READY: ++ return STACK_READY_LO; ++ case ACTION_PWRON: ++ return STACK_PWRON_LO; ++ case ACTION_PWROFF: ++ return STACK_PWROFF_LO; ++ case ACTION_PWRTRANS: ++ return STACK_PWRTRANS_LO; ++ default: ++ WARN(1, "Invalid action for core type\n"); ++ } ++ } ++ } ++ ++ return (u32)core_type + (u32)action; ++} ++ ++#ifdef CONFIG_ARM64 ++static void mali_cci_flush_l2(struct kbase_device *kbdev) ++{ ++ const u32 mask = CLEAN_CACHES_COMPLETED | RESET_COMPLETED; ++ u32 loops = KBASE_CLEAN_CACHE_MAX_LOOPS; ++ u32 raw; ++ ++ /* ++ * Note that we don't take the cache flush mutex here since ++ * we expect to be the last user of the L2, all other L2 users ++ * would have dropped their references, to initiate L2 power ++ * down, L2 power down being the only valid place for this ++ * to be called from. ++ */ ++ ++ kbase_reg_write(kbdev, ++ GPU_CONTROL_REG(GPU_COMMAND), ++ GPU_COMMAND_CLEAN_INV_CACHES); ++ ++ raw = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(GPU_IRQ_RAWSTAT)); ++ ++ /* Wait for cache flush to complete before continuing, exit on ++ * gpu resets or loop expiry. */ ++ while (((raw & mask) == 0) && --loops) { ++ raw = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(GPU_IRQ_RAWSTAT)); ++ } ++} ++#endif ++ ++/** ++ * kbase_pm_invoke - Invokes an action on a core set ++ * ++ * This function performs the action given by @action on a set of cores of a ++ * type given by @core_type. It is a static function used by ++ * kbase_pm_transition_core_type() ++ * ++ * @kbdev: The kbase device structure of the device ++ * @core_type: The type of core that the action should be performed on ++ * @cores: A bit mask of cores to perform the action on (low 32 bits) ++ * @action: The action to perform on the cores ++ */ ++static void kbase_pm_invoke(struct kbase_device *kbdev, ++ enum kbase_pm_core_type core_type, ++ u64 cores, ++ enum kbasep_pm_action action) ++{ ++ u32 reg; ++ u32 lo = cores & 0xFFFFFFFF; ++ u32 hi = (cores >> 32) & 0xFFFFFFFF; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ reg = core_type_to_reg(core_type, action); ++ ++ KBASE_DEBUG_ASSERT(reg); ++ ++ if (cores) { ++ u64 state = kbase_pm_get_state(kbdev, core_type, ACTION_READY); ++ ++ if (action == ACTION_PWRON) ++ state |= cores; ++ else if (action == ACTION_PWROFF) ++ state &= ~cores; ++ KBASE_TLSTREAM_AUX_PM_STATE(kbdev, core_type, state); ++ } ++ ++ /* Tracing */ ++ if (cores) { ++ if (action == ACTION_PWRON) ++ switch (core_type) { ++ case KBASE_PM_CORE_SHADER: ++ KBASE_KTRACE_ADD(kbdev, PM_PWRON, NULL, cores); ++ break; ++ case KBASE_PM_CORE_TILER: ++ KBASE_KTRACE_ADD(kbdev, PM_PWRON_TILER, NULL, cores); ++ break; ++ case KBASE_PM_CORE_L2: ++ KBASE_KTRACE_ADD(kbdev, PM_PWRON_L2, NULL, cores); ++ break; ++ default: ++ break; ++ } ++ else if (action == ACTION_PWROFF) ++ switch (core_type) { ++ case KBASE_PM_CORE_SHADER: ++ KBASE_KTRACE_ADD(kbdev, PM_PWROFF, NULL, cores); ++ break; ++ case KBASE_PM_CORE_TILER: ++ KBASE_KTRACE_ADD(kbdev, PM_PWROFF_TILER, NULL, cores); ++ break; ++ case KBASE_PM_CORE_L2: ++ KBASE_KTRACE_ADD(kbdev, PM_PWROFF_L2, NULL, cores); ++ /* disable snoops before L2 is turned off */ ++ kbase_pm_cache_snoop_disable(kbdev); ++ break; ++ default: ++ break; ++ } ++ } ++ ++ if (kbase_dummy_job_wa_enabled(kbdev) && ++ action == ACTION_PWRON && ++ core_type == KBASE_PM_CORE_SHADER && ++ !(kbdev->dummy_job_wa.flags & ++ KBASE_DUMMY_JOB_WA_FLAG_LOGICAL_SHADER_POWER)) { ++ kbase_dummy_job_wa_execute(kbdev, cores); ++ } else { ++ if (lo != 0) ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(reg), lo); ++ if (hi != 0) ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(reg + 4), hi); ++ } ++} ++ ++/** ++ * kbase_pm_get_state - Get information about a core set ++ * ++ * This function gets information (chosen by @action) about a set of cores of ++ * a type given by @core_type. It is a static function used by ++ * kbase_pm_get_active_cores(), kbase_pm_get_trans_cores() and ++ * kbase_pm_get_ready_cores(). ++ * ++ * @kbdev: The kbase device structure of the device ++ * @core_type: The type of core that the should be queried ++ * @action: The property of the cores to query ++ * ++ * Return: A bit mask specifying the state of the cores ++ */ ++static u64 kbase_pm_get_state(struct kbase_device *kbdev, ++ enum kbase_pm_core_type core_type, ++ enum kbasep_pm_action action) ++{ ++ u32 reg; ++ u32 lo, hi; ++ ++ reg = core_type_to_reg(core_type, action); ++ ++ KBASE_DEBUG_ASSERT(reg); ++ ++ lo = kbase_reg_read(kbdev, GPU_CONTROL_REG(reg)); ++ hi = kbase_reg_read(kbdev, GPU_CONTROL_REG(reg + 4)); ++ ++ return (((u64) hi) << 32) | ((u64) lo); ++} ++ ++/** ++ * kbase_pm_get_present_cores - Get the cores that are present ++ * ++ * @kbdev: Kbase device ++ * @type: The type of cores to query ++ * ++ * Return: Bitmask of the cores that are present ++ */ ++u64 kbase_pm_get_present_cores(struct kbase_device *kbdev, ++ enum kbase_pm_core_type type) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ switch (type) { ++ case KBASE_PM_CORE_L2: ++ return kbdev->gpu_props.props.raw_props.l2_present; ++ case KBASE_PM_CORE_SHADER: ++ return kbdev->gpu_props.props.raw_props.shader_present; ++ case KBASE_PM_CORE_TILER: ++ return kbdev->gpu_props.props.raw_props.tiler_present; ++ case KBASE_PM_CORE_STACK: ++ return kbdev->gpu_props.props.raw_props.stack_present; ++ default: ++ break; ++ } ++ KBASE_DEBUG_ASSERT(0); ++ ++ return 0; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_get_present_cores); ++ ++/** ++ * kbase_pm_get_active_cores - Get the cores that are "active" ++ * (busy processing work) ++ * ++ * @kbdev: Kbase device ++ * @type: The type of cores to query ++ * ++ * Return: Bitmask of cores that are active ++ */ ++u64 kbase_pm_get_active_cores(struct kbase_device *kbdev, ++ enum kbase_pm_core_type type) ++{ ++ return kbase_pm_get_state(kbdev, type, ACTION_PWRACTIVE); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_get_active_cores); ++ ++/** ++ * kbase_pm_get_trans_cores - Get the cores that are transitioning between ++ * power states ++ * ++ * @kbdev: Kbase device ++ * @type: The type of cores to query ++ * ++ * Return: Bitmask of cores that are transitioning ++ */ ++u64 kbase_pm_get_trans_cores(struct kbase_device *kbdev, ++ enum kbase_pm_core_type type) ++{ ++ return kbase_pm_get_state(kbdev, type, ACTION_PWRTRANS); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_get_trans_cores); ++ ++/** ++ * kbase_pm_get_ready_cores - Get the cores that are powered on ++ * ++ * @kbdev: Kbase device ++ * @type: The type of cores to query ++ * ++ * Return: Bitmask of cores that are ready (powered on) ++ */ ++u64 kbase_pm_get_ready_cores(struct kbase_device *kbdev, ++ enum kbase_pm_core_type type) ++{ ++ u64 result; ++ ++ result = kbase_pm_get_state(kbdev, type, ACTION_READY); ++ ++ switch (type) { ++ case KBASE_PM_CORE_SHADER: ++ KBASE_KTRACE_ADD(kbdev, PM_CORES_POWERED, NULL, result); ++ break; ++ case KBASE_PM_CORE_TILER: ++ KBASE_KTRACE_ADD(kbdev, PM_CORES_POWERED_TILER, NULL, result); ++ break; ++ case KBASE_PM_CORE_L2: ++ KBASE_KTRACE_ADD(kbdev, PM_CORES_POWERED_L2, NULL, result); ++ break; ++ default: ++ break; ++ } ++ ++ return result; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_get_ready_cores); ++ ++static void kbase_pm_trigger_hwcnt_disable(struct kbase_device *kbdev) ++{ ++ struct kbase_pm_backend_data *backend = &kbdev->pm.backend; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ /* See if we can get away with disabling hwcnt ++ * atomically, otherwise kick off a worker. ++ */ ++ if (kbase_hwcnt_context_disable_atomic(kbdev->hwcnt_gpu_ctx)) { ++ backend->hwcnt_disabled = true; ++ } else { ++#if KERNEL_VERSION(3, 16, 0) > LINUX_VERSION_CODE ++ queue_work(system_wq, ++ &backend->hwcnt_disable_work); ++#else ++ queue_work(system_highpri_wq, ++ &backend->hwcnt_disable_work); ++#endif ++ } ++} ++ ++static void kbase_pm_l2_config_override(struct kbase_device *kbdev) ++{ ++ u32 val; ++ ++ /* ++ * Skip if it is not supported ++ */ ++ if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_L2_CONFIG)) ++ return; ++ ++ /* ++ * Skip if size and hash are not given explicitly, ++ * which means default values are used. ++ */ ++ if ((kbdev->l2_size_override == 0) && (kbdev->l2_hash_override == 0)) ++ return; ++ ++ val = kbase_reg_read(kbdev, GPU_CONTROL_REG(L2_CONFIG)); ++ ++ if (kbdev->l2_size_override) { ++ val &= ~L2_CONFIG_SIZE_MASK; ++ val |= (kbdev->l2_size_override << L2_CONFIG_SIZE_SHIFT); ++ } ++ ++ if (kbdev->l2_hash_override) { ++ val &= ~L2_CONFIG_HASH_MASK; ++ val |= (kbdev->l2_hash_override << L2_CONFIG_HASH_SHIFT); ++ } ++ ++ dev_dbg(kbdev->dev, "Program 0x%x to L2_CONFIG\n", val); ++ ++ /* Write L2_CONFIG to override */ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(L2_CONFIG), val); ++} ++ ++static void kbase_pm_control_gpu_clock(struct kbase_device *kbdev) ++{ ++ struct kbase_pm_backend_data *const backend = &kbdev->pm.backend; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ queue_work(system_wq, &backend->gpu_clock_control_work); ++} ++ ++#if MALI_USE_CSF ++static const char *kbase_mcu_state_to_string(enum kbase_mcu_state state) ++{ ++ const char *const strings[] = { ++#define KBASEP_MCU_STATE(n) #n, ++#include "mali_kbase_pm_mcu_states.h" ++#undef KBASEP_MCU_STATE ++ }; ++ if (WARN_ON((size_t)state >= ARRAY_SIZE(strings))) ++ return "Bad MCU state"; ++ else ++ return strings[state]; ++} ++ ++static int kbase_pm_mcu_update_state(struct kbase_device *kbdev) ++{ ++ struct kbase_pm_backend_data *backend = &kbdev->pm.backend; ++ enum kbase_mcu_state prev_state; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ /* ++ * Initial load of firmare should have been done to ++ * exercise the MCU state machine. ++ */ ++ if (unlikely(!kbdev->csf.firmware_inited)) { ++ WARN_ON(backend->mcu_state != KBASE_MCU_OFF); ++ return -EIO; ++ } ++ ++ do { ++ prev_state = backend->mcu_state; ++ ++ switch (backend->mcu_state) { ++ case KBASE_MCU_OFF: ++ if (kbase_pm_is_mcu_desired(kbdev) && ++ backend->l2_state == KBASE_L2_ON) { ++ kbase_csf_firmware_trigger_reload(kbdev); ++ backend->mcu_state = KBASE_MCU_PEND_ON_RELOAD; ++ } ++ break; ++ ++ case KBASE_MCU_PEND_ON_RELOAD: ++ if (kbdev->csf.firmware_reloaded) { ++ kbase_csf_firmware_global_reinit(kbdev); ++ backend->mcu_state = ++ KBASE_MCU_ON_GLB_REINIT_PEND; ++ } ++ break; ++ ++ case KBASE_MCU_ON_GLB_REINIT_PEND: ++ if (kbase_csf_firmware_global_reinit_complete(kbdev)) ++ backend->mcu_state = KBASE_MCU_ON_HWCNT_ENABLE; ++ break; ++ ++ case KBASE_MCU_ON_HWCNT_ENABLE: ++ backend->hwcnt_desired = true; ++ if (backend->hwcnt_disabled) { ++ kbase_hwcnt_context_enable( ++ kbdev->hwcnt_gpu_ctx); ++ backend->hwcnt_disabled = false; ++ } ++ backend->mcu_state = KBASE_MCU_ON; ++ break; ++ ++ case KBASE_MCU_ON: ++ if (!kbase_pm_is_mcu_desired(kbdev)) ++ backend->mcu_state = KBASE_MCU_ON_HWCNT_DISABLE; ++ break; ++ ++ /* ToDo. Add new state(s) if shader cores mask change for DVFS ++ * has to be accommodated in the MCU state machine. ++ */ ++ ++ case KBASE_MCU_ON_HWCNT_DISABLE: ++ if (kbase_pm_is_mcu_desired(kbdev)) { ++ backend->mcu_state = KBASE_MCU_ON_HWCNT_ENABLE; ++ break; ++ } ++ ++ backend->hwcnt_desired = false; ++ if (!backend->hwcnt_disabled) ++ kbase_pm_trigger_hwcnt_disable(kbdev); ++ ++ if (backend->hwcnt_disabled) ++ backend->mcu_state = KBASE_MCU_ON_HALT; ++ break; ++ ++ case KBASE_MCU_ON_HALT: ++ if (!kbase_pm_is_mcu_desired(kbdev)) { ++ kbase_csf_firmware_trigger_mcu_halt(kbdev); ++ backend->mcu_state = KBASE_MCU_ON_PEND_HALT; ++ } else if (kbase_pm_is_mcu_desired(kbdev)) { ++ backend->mcu_state = KBASE_MCU_ON_HWCNT_ENABLE; ++ } ++ break; ++ ++ case KBASE_MCU_ON_PEND_HALT: ++ if (kbase_csf_firmware_mcu_halted(kbdev)) ++ backend->mcu_state = KBASE_MCU_POWER_DOWN; ++ break; ++ ++ case KBASE_MCU_POWER_DOWN: ++ kbase_csf_firmware_disable_mcu(kbdev); ++ backend->mcu_state = KBASE_MCU_PEND_OFF; ++ break; ++ ++ case KBASE_MCU_PEND_OFF: ++ /* wait synchronously for the MCU to get disabled */ ++ kbase_csf_firmware_disable_mcu_wait(kbdev); ++ backend->mcu_state = KBASE_MCU_OFF; ++ break; ++ ++ case KBASE_MCU_RESET_WAIT: ++ /* Reset complete */ ++ if (!backend->in_reset) ++ backend->mcu_state = KBASE_MCU_OFF; ++ break; ++ ++ default: ++ WARN(1, "Invalid state in mcu_state: %d", ++ backend->mcu_state); ++ } ++ ++ if (backend->mcu_state != prev_state) ++ dev_dbg(kbdev->dev, "MCU state transition: %s to %s\n", ++ kbase_mcu_state_to_string(prev_state), ++ kbase_mcu_state_to_string(backend->mcu_state)); ++ ++ } while (backend->mcu_state != prev_state); ++ ++ return 0; ++} ++#endif ++ ++static const char *kbase_l2_core_state_to_string(enum kbase_l2_core_state state) ++{ ++ const char *const strings[] = { ++#define KBASEP_L2_STATE(n) #n, ++#include "mali_kbase_pm_l2_states.h" ++#undef KBASEP_L2_STATE ++ }; ++ if (WARN_ON((size_t)state >= ARRAY_SIZE(strings))) ++ return "Bad level 2 cache state"; ++ else ++ return strings[state]; ++} ++ ++static int kbase_pm_l2_update_state(struct kbase_device *kbdev) ++{ ++ struct kbase_pm_backend_data *backend = &kbdev->pm.backend; ++ u64 l2_present = kbdev->gpu_props.props.raw_props.l2_present; ++ u64 tiler_present = kbdev->gpu_props.props.raw_props.tiler_present; ++ enum kbase_l2_core_state prev_state; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ do { ++ /* Get current state */ ++ u64 l2_trans = kbase_pm_get_trans_cores(kbdev, ++ KBASE_PM_CORE_L2); ++ u64 l2_ready = kbase_pm_get_ready_cores(kbdev, ++ KBASE_PM_CORE_L2); ++ u64 tiler_trans = kbase_pm_get_trans_cores(kbdev, ++ KBASE_PM_CORE_TILER); ++ u64 tiler_ready = kbase_pm_get_ready_cores(kbdev, ++ KBASE_PM_CORE_TILER); ++ ++ /* ++ * kbase_pm_get_ready_cores and kbase_pm_get_trans_cores ++ * are vulnerable to corruption if gpu is lost ++ */ ++ if (kbase_is_gpu_removed(kbdev) ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++ || kbase_pm_is_gpu_lost(kbdev)) { ++#else ++ ) { ++#endif ++ backend->shaders_state = ++ KBASE_SHADERS_OFF_CORESTACK_OFF; ++ backend->l2_state = KBASE_L2_OFF; ++ dev_dbg(kbdev->dev, "GPU lost has occurred - L2 off\n"); ++ break; ++ } ++ ++ /* mask off ready from trans in case transitions finished ++ * between the register reads ++ */ ++ l2_trans &= ~l2_ready; ++ tiler_trans &= ~tiler_ready; ++ ++ prev_state = backend->l2_state; ++ ++ switch (backend->l2_state) { ++ case KBASE_L2_OFF: ++ if (kbase_pm_is_l2_desired(kbdev)) { ++ /* ++ * Set the desired config for L2 before ++ * powering it on ++ */ ++ kbase_pm_l2_config_override(kbdev); ++ ++ /* L2 is required, power on. Powering on the ++ * tiler will also power the first L2 cache. ++ */ ++ kbase_pm_invoke(kbdev, KBASE_PM_CORE_TILER, ++ tiler_present, ACTION_PWRON); ++ ++ /* If we have more than one L2 cache then we ++ * must power them on explicitly. ++ */ ++ if (l2_present != 1) ++ kbase_pm_invoke(kbdev, KBASE_PM_CORE_L2, ++ l2_present & ~1, ++ ACTION_PWRON); ++ backend->l2_state = KBASE_L2_PEND_ON; ++ } ++ break; ++ ++ case KBASE_L2_PEND_ON: ++ if (!l2_trans && l2_ready == l2_present && !tiler_trans ++ && tiler_ready == tiler_present) { ++ KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER, NULL, tiler_ready); ++ /* ++ * Ensure snoops are enabled after L2 is powered ++ * up. Note that kbase keeps track of the snoop ++ * state, so safe to repeatedly call. ++ */ ++ kbase_pm_cache_snoop_enable(kbdev); ++ ++ /* With the L2 enabled, we can now enable ++ * hardware counters. ++ */ ++ if (kbdev->pm.backend.gpu_clock_slow_down_wa) ++ backend->l2_state = ++ KBASE_L2_RESTORE_CLOCKS; ++ else ++ backend->l2_state = ++ KBASE_L2_ON_HWCNT_ENABLE; ++ ++ /* Now that the L2 is on, the shaders can start ++ * powering on if they're required. The obvious ++ * way to do this would be to call ++ * kbase_pm_shaders_update_state() here. ++ * However, that would make the two state ++ * machines mutually recursive, as the opposite ++ * would be needed for powering down. Instead, ++ * callers of this function should use the ++ * kbase_pm_update_state() wrapper, which will ++ * call the shader state machine immediately ++ * after the L2 (for power up), or ++ * automatically re-invoke the L2 state machine ++ * when the shaders power down. ++ */ ++ } ++ break; ++ ++ case KBASE_L2_RESTORE_CLOCKS: ++ /* We always assume only GPUs being affected by ++ * BASE_HW_ISSUE_GPU2017_1336 fall into this state ++ */ ++ WARN_ON_ONCE(!kbdev->pm.backend.gpu_clock_slow_down_wa); ++ ++ /* If L2 not needed, we need to make sure cancellation ++ * of any previously issued work to restore GPU clock. ++ * For it, move to KBASE_L2_SLOW_DOWN_CLOCKS state. ++ */ ++ if (!kbase_pm_is_l2_desired(kbdev)) { ++ backend->l2_state = KBASE_L2_SLOW_DOWN_CLOCKS; ++ break; ++ } ++ ++ backend->gpu_clock_slow_down_desired = false; ++ if (backend->gpu_clock_slowed_down) ++ kbase_pm_control_gpu_clock(kbdev); ++ else ++ backend->l2_state = KBASE_L2_ON_HWCNT_ENABLE; ++ break; ++ ++ case KBASE_L2_ON_HWCNT_ENABLE: ++#if !MALI_USE_CSF ++ backend->hwcnt_desired = true; ++ if (backend->hwcnt_disabled) { ++ kbase_hwcnt_context_enable( ++ kbdev->hwcnt_gpu_ctx); ++ backend->hwcnt_disabled = false; ++ } ++#endif ++ backend->l2_state = KBASE_L2_ON; ++ break; ++ ++ case KBASE_L2_ON: ++ if (!kbase_pm_is_l2_desired(kbdev)) { ++#if !MALI_USE_CSF ++ /* Do not power off L2 until the shaders and ++ * core stacks are off. ++ */ ++ if (backend->shaders_state != KBASE_SHADERS_OFF_CORESTACK_OFF) ++ break; ++#else ++ /* Do not power off L2 until the MCU has been stopped */ ++ if (backend->mcu_state != KBASE_MCU_OFF) ++ break; ++#endif ++ ++ /* We need to make sure hardware counters are ++ * disabled before powering down the L2, to ++ * prevent loss of data. ++ * ++ * We waited until after the cores were powered ++ * down to prevent ping-ponging between hwcnt ++ * enabled and disabled, which would have ++ * happened if userspace submitted more work ++ * while we were trying to power down. ++ */ ++ backend->l2_state = KBASE_L2_ON_HWCNT_DISABLE; ++ } ++ break; ++ ++ case KBASE_L2_ON_HWCNT_DISABLE: ++#if !MALI_USE_CSF ++ /* If the L2 became desired while we were waiting on the ++ * worker to do the actual hwcnt disable (which might ++ * happen if some work was submitted immediately after ++ * the shaders powered off), then we need to early-out ++ * of this state and re-enable hwcnt. ++ * ++ * If we get lucky, the hwcnt disable might not have ++ * actually started yet, and the logic in the hwcnt ++ * enable state will prevent the worker from ++ * performing the disable entirely, preventing loss of ++ * any hardware counter data. ++ * ++ * If the hwcnt disable has started, then we'll lose ++ * a tiny amount of hardware counter data between the ++ * disable and the re-enable occurring. ++ * ++ * This loss of data is preferable to the alternative, ++ * which is to block the shader cores from doing any ++ * work until we're sure hwcnt has been re-enabled. ++ */ ++ if (kbase_pm_is_l2_desired(kbdev)) { ++ backend->l2_state = KBASE_L2_ON_HWCNT_ENABLE; ++ break; ++ } ++ ++ backend->hwcnt_desired = false; ++ if (!backend->hwcnt_disabled) { ++ kbase_pm_trigger_hwcnt_disable(kbdev); ++ } ++#endif ++ ++ if (backend->hwcnt_disabled) { ++ if (kbdev->pm.backend.gpu_clock_slow_down_wa) ++ backend->l2_state = ++ KBASE_L2_SLOW_DOWN_CLOCKS; ++ else ++ backend->l2_state = KBASE_L2_POWER_DOWN; ++ } ++ break; ++ ++ case KBASE_L2_SLOW_DOWN_CLOCKS: ++ /* We always assume only GPUs being affected by ++ * BASE_HW_ISSUE_GPU2017_1336 fall into this state ++ */ ++ WARN_ON_ONCE(!kbdev->pm.backend.gpu_clock_slow_down_wa); ++ ++ /* L2 needs to be powered up. And we need to make sure ++ * cancellation of any previously issued work to slow ++ * down GPU clock. For it, we move to the state, ++ * KBASE_L2_RESTORE_CLOCKS. ++ */ ++ if (kbase_pm_is_l2_desired(kbdev)) { ++ backend->l2_state = KBASE_L2_RESTORE_CLOCKS; ++ break; ++ } ++ ++ backend->gpu_clock_slow_down_desired = true; ++ if (!backend->gpu_clock_slowed_down) ++ kbase_pm_control_gpu_clock(kbdev); ++ else ++ backend->l2_state = KBASE_L2_POWER_DOWN; ++ ++ break; ++ ++ case KBASE_L2_POWER_DOWN: ++ if (!backend->l2_always_on) ++ /* Powering off the L2 will also power off the ++ * tiler. ++ */ ++ kbase_pm_invoke(kbdev, KBASE_PM_CORE_L2, ++ l2_present, ++ ACTION_PWROFF); ++ else ++ /* If L2 cache is powered then we must flush it ++ * before we power off the GPU. Normally this ++ * would have been handled when the L2 was ++ * powered off. ++ */ ++ kbase_gpu_start_cache_clean_nolock( ++ kbdev); ++ ++ KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER, NULL, 0u); ++ ++ backend->l2_state = KBASE_L2_PEND_OFF; ++ break; ++ ++ case KBASE_L2_PEND_OFF: ++ if (!backend->l2_always_on) { ++ /* We only need to check the L2 here - if the L2 ++ * is off then the tiler is definitely also off. ++ */ ++ if (!l2_trans && !l2_ready) ++ /* L2 is now powered off */ ++ backend->l2_state = KBASE_L2_OFF; ++ } else { ++ if (!kbdev->cache_clean_in_progress) ++ backend->l2_state = KBASE_L2_OFF; ++ } ++ break; ++ ++ case KBASE_L2_RESET_WAIT: ++ /* Reset complete */ ++ if (!backend->in_reset) ++ backend->l2_state = KBASE_L2_OFF; ++ break; ++ ++ default: ++ WARN(1, "Invalid state in l2_state: %d", ++ backend->l2_state); ++ } ++ ++ if (backend->l2_state != prev_state) ++ dev_dbg(kbdev->dev, "L2 state transition: %s to %s\n", ++ kbase_l2_core_state_to_string(prev_state), ++ kbase_l2_core_state_to_string( ++ backend->l2_state)); ++ ++ } while (backend->l2_state != prev_state); ++ ++ if (kbdev->pm.backend.invoke_poweroff_wait_wq_when_l2_off && ++ backend->l2_state == KBASE_L2_OFF) { ++ kbdev->pm.backend.invoke_poweroff_wait_wq_when_l2_off = false; ++ queue_work(kbdev->pm.backend.gpu_poweroff_wait_wq, ++ &kbdev->pm.backend.gpu_poweroff_wait_work); ++ } ++ ++ return 0; ++} ++ ++static void shader_poweroff_timer_stop_callback(struct work_struct *data) ++{ ++ unsigned long flags; ++ struct kbasep_pm_tick_timer_state *stt = container_of(data, ++ struct kbasep_pm_tick_timer_state, work); ++ struct kbase_device *kbdev = container_of(stt, struct kbase_device, ++ pm.backend.shader_tick_timer); ++ ++ hrtimer_cancel(&stt->timer); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ stt->cancel_queued = false; ++ if (kbdev->pm.backend.gpu_powered) ++ kbase_pm_update_state(kbdev); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++} ++ ++/** ++ * shader_poweroff_timer_queue_cancel - cancel the shader poweroff tick timer ++ * @kbdev: pointer to kbase device ++ * ++ * Synchronization between the shader state machine and the timer thread is ++ * difficult. This is because situations may arise where the state machine ++ * wants to start the timer, but the callback is already running, and has ++ * already passed the point at which it checks whether it is required, and so ++ * cancels itself, even though the state machine may have just tried to call ++ * hrtimer_start. ++ * ++ * This cannot be stopped by holding hwaccess_lock in the timer thread, ++ * because there are still infinitesimally small sections at the start and end ++ * of the callback where the lock is not held. ++ * ++ * Instead, a new state is added to the shader state machine, ++ * KBASE_SHADERS_OFF_CORESTACK_OFF_TIMER_PEND_OFF. This is used to guarantee ++ * that when the shaders are switched off, the timer has definitely been ++ * cancelled. As a result, when KBASE_SHADERS_ON_CORESTACK_ON is left and the ++ * timer is started, it is guaranteed that either the timer is already running ++ * (from an availability change or cancelled timer), or hrtimer_start will ++ * succeed. It is critical to avoid ending up in ++ * KBASE_SHADERS_WAIT_OFF_CORESTACK_ON without the timer running, or it could ++ * hang there forever. ++ */ ++static void shader_poweroff_timer_queue_cancel(struct kbase_device *kbdev) ++{ ++ struct kbasep_pm_tick_timer_state *stt = ++ &kbdev->pm.backend.shader_tick_timer; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ stt->needed = false; ++ ++ if (hrtimer_active(&stt->timer) && !stt->cancel_queued) { ++ stt->cancel_queued = true; ++ queue_work(stt->wq, &stt->work); ++ } ++} ++ ++#if !MALI_USE_CSF ++static const char *kbase_shader_core_state_to_string( ++ enum kbase_shader_core_state state) ++{ ++ const char *const strings[] = { ++#define KBASEP_SHADER_STATE(n) #n, ++#include "mali_kbase_pm_shader_states.h" ++#undef KBASEP_SHADER_STATE ++ }; ++ if (WARN_ON((size_t)state >= ARRAY_SIZE(strings))) ++ return "Bad shader core state"; ++ else ++ return strings[state]; ++} ++ ++static int kbase_pm_shaders_update_state(struct kbase_device *kbdev) ++{ ++ struct kbase_pm_backend_data *backend = &kbdev->pm.backend; ++ struct kbasep_pm_tick_timer_state *stt = ++ &kbdev->pm.backend.shader_tick_timer; ++ enum kbase_shader_core_state prev_state; ++ u64 stacks_avail = 0; ++ int err = 0; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (corestack_driver_control) ++ /* Always power on all the corestacks. Disabling certain ++ * corestacks when their respective shaders are not in the ++ * available bitmap is not currently supported. ++ */ ++ stacks_avail = kbase_pm_get_present_cores(kbdev, KBASE_PM_CORE_STACK); ++ ++ do { ++ u64 shaders_trans = kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_SHADER); ++ u64 shaders_ready = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER); ++ u64 stacks_trans = 0; ++ u64 stacks_ready = 0; ++ ++ if (corestack_driver_control) { ++ stacks_trans = kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_STACK); ++ stacks_ready = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_STACK); ++ } ++ ++ /* ++ * kbase_pm_get_ready_cores and kbase_pm_get_trans_cores ++ * are vulnerable to corruption if gpu is lost ++ */ ++ if (kbase_is_gpu_removed(kbdev) ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++ || kbase_pm_is_gpu_lost(kbdev)) { ++#else ++ ) { ++#endif ++ backend->shaders_state = ++ KBASE_SHADERS_OFF_CORESTACK_OFF; ++ dev_dbg(kbdev->dev, "GPU lost has occurred - shaders off\n"); ++ break; ++ } ++ ++ /* mask off ready from trans in case transitions finished ++ * between the register reads ++ */ ++ shaders_trans &= ~shaders_ready; ++ stacks_trans &= ~stacks_ready; ++ ++ prev_state = backend->shaders_state; ++ ++ switch (backend->shaders_state) { ++ case KBASE_SHADERS_OFF_CORESTACK_OFF: ++ /* Ignore changes to the shader core availability ++ * except at certain points where we can handle it, ++ * i.e. off and SHADERS_ON_CORESTACK_ON. ++ */ ++ backend->shaders_desired_mask = ++ kbase_pm_ca_get_core_mask(kbdev); ++ backend->pm_shaders_core_mask = 0; ++ ++ if (backend->shaders_desired && ++ backend->l2_state == KBASE_L2_ON) { ++ if (backend->hwcnt_desired && ++ !backend->hwcnt_disabled) { ++ /* Trigger a hwcounter dump */ ++ backend->hwcnt_desired = false; ++ kbase_pm_trigger_hwcnt_disable(kbdev); ++ } ++ ++ if (backend->hwcnt_disabled) { ++ if (corestack_driver_control) { ++ kbase_pm_invoke(kbdev, ++ KBASE_PM_CORE_STACK, ++ stacks_avail, ++ ACTION_PWRON); ++ } ++ backend->shaders_state = ++ KBASE_SHADERS_OFF_CORESTACK_PEND_ON; ++ } ++ } ++ break; ++ ++ case KBASE_SHADERS_OFF_CORESTACK_PEND_ON: ++ if (!stacks_trans && stacks_ready == stacks_avail) { ++ backend->shaders_avail = ++ backend->shaders_desired_mask; ++ kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER, ++ backend->shaders_avail, ACTION_PWRON); ++ ++ backend->shaders_state = KBASE_SHADERS_PEND_ON_CORESTACK_ON; ++ } ++ break; ++ ++ case KBASE_SHADERS_PEND_ON_CORESTACK_ON: ++ if (!shaders_trans && shaders_ready == backend->shaders_avail) { ++ KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, shaders_ready); ++ backend->pm_shaders_core_mask = shaders_ready; ++ backend->hwcnt_desired = true; ++ if (backend->hwcnt_disabled) { ++ kbase_hwcnt_context_enable( ++ kbdev->hwcnt_gpu_ctx); ++ backend->hwcnt_disabled = false; ++ } ++ ++ backend->shaders_state = KBASE_SHADERS_ON_CORESTACK_ON; ++ } ++ break; ++ ++ case KBASE_SHADERS_ON_CORESTACK_ON: ++ backend->shaders_desired_mask = ++ kbase_pm_ca_get_core_mask(kbdev); ++ ++ /* If shaders to change state, trigger a counter dump */ ++ if (!backend->shaders_desired || ++ (backend->shaders_desired_mask != shaders_ready)) { ++ backend->hwcnt_desired = false; ++ if (!backend->hwcnt_disabled) ++ kbase_pm_trigger_hwcnt_disable(kbdev); ++ backend->shaders_state = ++ KBASE_SHADERS_ON_CORESTACK_ON_RECHECK; ++ } ++ break; ++ ++ case KBASE_SHADERS_ON_CORESTACK_ON_RECHECK: ++ backend->shaders_desired_mask = ++ kbase_pm_ca_get_core_mask(kbdev); ++ ++ if (!backend->hwcnt_disabled) { ++ /* Wait for being disabled */ ++ ; ++ } else if (!backend->shaders_desired) { ++ if (kbdev->pm.backend.protected_transition_override || ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++ kbase_pm_is_suspending(kbdev) || ++ kbase_pm_is_gpu_lost(kbdev) || ++#endif /* CONFIG_MALI_ARBITER_SUPPORT */ ++ !stt->configured_ticks || ++ WARN_ON(stt->cancel_queued)) { ++ backend->shaders_state = KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON; ++ } else { ++ stt->remaining_ticks = stt->configured_ticks; ++ stt->needed = true; ++ ++ /* The shader hysteresis timer is not ++ * done the obvious way, which would be ++ * to start an hrtimer when the shader ++ * power off is requested. Instead, ++ * use a 'tick' timer, and set the ++ * remaining number of ticks on a power ++ * off request. This avoids the ++ * latency of starting, then ++ * immediately cancelling an hrtimer ++ * when the shaders are re-requested ++ * before the timeout expires. ++ */ ++ if (!hrtimer_active(&stt->timer)) ++ hrtimer_start(&stt->timer, ++ stt->configured_interval, ++ HRTIMER_MODE_REL); ++ ++ backend->shaders_state = KBASE_SHADERS_WAIT_OFF_CORESTACK_ON; ++ } ++ } else if (backend->shaders_desired_mask & ~shaders_ready) { ++ /* set cores ready but not available to ++ * meet KBASE_SHADERS_PEND_ON_CORESTACK_ON ++ * check pass ++ */ ++ backend->shaders_avail = ++ (backend->shaders_desired_mask | shaders_ready); ++ ++ kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER, ++ backend->shaders_avail & ~shaders_ready, ++ ACTION_PWRON); ++ backend->shaders_state = ++ KBASE_SHADERS_PEND_ON_CORESTACK_ON; ++ } else if (shaders_ready & ~backend->shaders_desired_mask) { ++ backend->shaders_state = ++ KBASE_SHADERS_WAIT_GPU_IDLE; ++ } else { ++ backend->shaders_state = ++ KBASE_SHADERS_PEND_ON_CORESTACK_ON; ++ } ++ break; ++ ++ case KBASE_SHADERS_WAIT_OFF_CORESTACK_ON: ++ if (WARN_ON(!hrtimer_active(&stt->timer))) { ++ stt->remaining_ticks = 0; ++ backend->shaders_state = KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON; ++ } ++ ++ if (backend->shaders_desired) { ++ stt->remaining_ticks = 0; ++ backend->shaders_state = KBASE_SHADERS_ON_CORESTACK_ON_RECHECK; ++ } else if (stt->remaining_ticks == 0) { ++ backend->shaders_state = KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON; ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++ } else if (kbase_pm_is_suspending(kbdev) || ++ kbase_pm_is_gpu_lost(kbdev)) { ++ backend->shaders_state = KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON; ++#endif /* CONFIG_MALI_ARBITER_SUPPORT */ ++ } ++ break; ++ ++ case KBASE_SHADERS_WAIT_GPU_IDLE: ++ /* If partial shader core off need to wait the job in ++ * running and next register finished then flush L2 ++ * or it might hit GPU2017-861 ++ */ ++ if (!kbase_gpu_atoms_submitted_any(kbdev)) { ++ backend->partial_shaderoff = true; ++ backend->shaders_state = KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON; ++ } ++ break; ++ ++ case KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON: ++ shader_poweroff_timer_queue_cancel(kbdev); ++ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_921)) { ++ kbase_gpu_start_cache_clean_nolock(kbdev); ++ backend->shaders_state = ++ KBASE_SHADERS_L2_FLUSHING_CORESTACK_ON; ++ } else { ++ backend->shaders_state = ++ KBASE_SHADERS_READY_OFF_CORESTACK_ON; ++ } ++ break; ++ ++ case KBASE_SHADERS_L2_FLUSHING_CORESTACK_ON: ++ if (!kbdev->cache_clean_in_progress) ++ backend->shaders_state = ++ KBASE_SHADERS_READY_OFF_CORESTACK_ON; ++ ++ break; ++ ++ case KBASE_SHADERS_READY_OFF_CORESTACK_ON: ++ if (backend->partial_shaderoff) { ++ backend->partial_shaderoff = false; ++ /* remove cores available but not ready to ++ * meet KBASE_SHADERS_PEND_ON_CORESTACK_ON ++ * check pass ++ */ ++ ++ /* shaders_desired_mask shall be a subset of ++ * shaders_ready ++ */ ++ WARN_ON(backend->shaders_desired_mask & ~shaders_ready); ++ WARN_ON(!(backend->shaders_desired_mask & shaders_ready)); ++ ++ backend->shaders_avail = ++ backend->shaders_desired_mask; ++ kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER, ++ shaders_ready & ~backend->shaders_avail, ACTION_PWROFF); ++ backend->shaders_state = KBASE_SHADERS_PEND_ON_CORESTACK_ON; ++ KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, (shaders_ready & ~backend->shaders_avail)); ++ } else { ++ kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER, ++ shaders_ready, ACTION_PWROFF); ++ ++ KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, 0u); ++ ++ backend->shaders_state = KBASE_SHADERS_PEND_OFF_CORESTACK_ON; ++ } ++ break; ++ ++ case KBASE_SHADERS_PEND_OFF_CORESTACK_ON: ++ if (!shaders_trans && !shaders_ready) { ++ if (corestack_driver_control) ++ kbase_pm_invoke(kbdev, KBASE_PM_CORE_STACK, ++ stacks_avail, ACTION_PWROFF); ++ ++ backend->shaders_state = KBASE_SHADERS_OFF_CORESTACK_PEND_OFF; ++ } ++ break; ++ ++ case KBASE_SHADERS_OFF_CORESTACK_PEND_OFF: ++ if (!stacks_trans && !stacks_ready) { ++ /* On powered off, re-enable the hwcnt */ ++ backend->pm_shaders_core_mask = 0; ++ backend->hwcnt_desired = true; ++ if (backend->hwcnt_disabled) { ++ kbase_hwcnt_context_enable( ++ kbdev->hwcnt_gpu_ctx); ++ backend->hwcnt_disabled = false; ++ } ++ backend->shaders_state = KBASE_SHADERS_OFF_CORESTACK_OFF_TIMER_PEND_OFF; ++ } ++ break; ++ ++ case KBASE_SHADERS_OFF_CORESTACK_OFF_TIMER_PEND_OFF: ++ if (!hrtimer_active(&stt->timer) && !stt->cancel_queued) ++ backend->shaders_state = KBASE_SHADERS_OFF_CORESTACK_OFF; ++ break; ++ ++ case KBASE_SHADERS_RESET_WAIT: ++ /* Reset complete */ ++ if (!backend->in_reset) ++ backend->shaders_state = KBASE_SHADERS_OFF_CORESTACK_OFF_TIMER_PEND_OFF; ++ break; ++ } ++ ++ if (backend->shaders_state != prev_state) ++ dev_dbg(kbdev->dev, "Shader state transition: %s to %s\n", ++ kbase_shader_core_state_to_string(prev_state), ++ kbase_shader_core_state_to_string( ++ backend->shaders_state)); ++ ++ } while (backend->shaders_state != prev_state); ++ ++ return err; ++} ++#endif ++ ++static bool kbase_pm_is_in_desired_state_nolock(struct kbase_device *kbdev) ++{ ++ bool in_desired_state = true; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (kbase_pm_is_l2_desired(kbdev) && ++ kbdev->pm.backend.l2_state != KBASE_L2_ON) ++ in_desired_state = false; ++ else if (!kbase_pm_is_l2_desired(kbdev) && ++ kbdev->pm.backend.l2_state != KBASE_L2_OFF) ++ in_desired_state = false; ++ ++#if !MALI_USE_CSF ++ if (kbdev->pm.backend.shaders_desired && ++ kbdev->pm.backend.shaders_state != KBASE_SHADERS_ON_CORESTACK_ON) ++ in_desired_state = false; ++ else if (!kbdev->pm.backend.shaders_desired && ++ kbdev->pm.backend.shaders_state != KBASE_SHADERS_OFF_CORESTACK_OFF) ++ in_desired_state = false; ++#else ++ if (kbase_pm_is_mcu_desired(kbdev) && ++ kbdev->pm.backend.mcu_state != KBASE_MCU_ON) ++ in_desired_state = false; ++ else if (!kbase_pm_is_mcu_desired(kbdev) && ++ kbdev->pm.backend.mcu_state != KBASE_MCU_OFF) ++ in_desired_state = false; ++#endif ++ ++ return in_desired_state; ++} ++ ++static bool kbase_pm_is_in_desired_state(struct kbase_device *kbdev) ++{ ++ bool in_desired_state; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ in_desired_state = kbase_pm_is_in_desired_state_nolock(kbdev); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ return in_desired_state; ++} ++ ++static bool kbase_pm_is_in_desired_state_with_l2_powered( ++ struct kbase_device *kbdev) ++{ ++ bool in_desired_state = false; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ if (kbase_pm_is_in_desired_state_nolock(kbdev) && ++ (kbdev->pm.backend.l2_state == KBASE_L2_ON)) ++ in_desired_state = true; ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ return in_desired_state; ++} ++ ++static void kbase_pm_trace_power_state(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ KBASE_TLSTREAM_AUX_PM_STATE( ++ kbdev, ++ KBASE_PM_CORE_L2, ++ kbase_pm_get_ready_cores( ++ kbdev, KBASE_PM_CORE_L2)); ++ KBASE_TLSTREAM_AUX_PM_STATE( ++ kbdev, ++ KBASE_PM_CORE_SHADER, ++ kbase_pm_get_ready_cores( ++ kbdev, KBASE_PM_CORE_SHADER)); ++ KBASE_TLSTREAM_AUX_PM_STATE( ++ kbdev, ++ KBASE_PM_CORE_TILER, ++ kbase_pm_get_ready_cores( ++ kbdev, ++ KBASE_PM_CORE_TILER)); ++ ++ if (corestack_driver_control) ++ KBASE_TLSTREAM_AUX_PM_STATE( ++ kbdev, ++ KBASE_PM_CORE_STACK, ++ kbase_pm_get_ready_cores( ++ kbdev, ++ KBASE_PM_CORE_STACK)); ++} ++ ++void kbase_pm_update_state(struct kbase_device *kbdev) ++{ ++#if !MALI_USE_CSF ++ enum kbase_shader_core_state prev_shaders_state = ++ kbdev->pm.backend.shaders_state; ++#else ++ enum kbase_mcu_state prev_mcu_state = kbdev->pm.backend.mcu_state; ++#endif ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (!kbdev->pm.backend.gpu_ready) ++ return; /* Do nothing if the GPU is not ready */ ++ ++ if (kbase_pm_l2_update_state(kbdev)) ++ return; ++ ++#if !MALI_USE_CSF ++ if (kbase_pm_shaders_update_state(kbdev)) ++ return; ++ ++ /* If the shaders just turned off, re-invoke the L2 state machine, in ++ * case it was waiting for the shaders to turn off before powering down ++ * the L2. ++ */ ++ if (prev_shaders_state != KBASE_SHADERS_OFF_CORESTACK_OFF && ++ kbdev->pm.backend.shaders_state == ++ KBASE_SHADERS_OFF_CORESTACK_OFF) { ++ if (kbase_pm_l2_update_state(kbdev)) ++ return; ++ } ++#else ++ if (kbase_pm_mcu_update_state(kbdev)) ++ return; ++ ++ if (prev_mcu_state != KBASE_MCU_OFF && ++ kbdev->pm.backend.mcu_state == KBASE_MCU_OFF) { ++ if (kbase_pm_l2_update_state(kbdev)) ++ return; ++ } ++#endif ++ ++ if (kbase_pm_is_in_desired_state_nolock(kbdev)) { ++ KBASE_KTRACE_ADD(kbdev, PM_DESIRED_REACHED, NULL, ++ kbdev->pm.backend.shaders_avail); ++ ++ kbase_pm_trace_power_state(kbdev); ++ ++ KBASE_KTRACE_ADD(kbdev, PM_WAKE_WAITERS, NULL, 0); ++ wake_up(&kbdev->pm.backend.gpu_in_desired_state_wait); ++ } ++} ++ ++static enum hrtimer_restart ++shader_tick_timer_callback(struct hrtimer *timer) ++{ ++ struct kbasep_pm_tick_timer_state *stt = container_of(timer, ++ struct kbasep_pm_tick_timer_state, timer); ++ struct kbase_device *kbdev = container_of(stt, struct kbase_device, ++ pm.backend.shader_tick_timer); ++ struct kbase_pm_backend_data *backend = &kbdev->pm.backend; ++ unsigned long flags; ++ enum hrtimer_restart restart = HRTIMER_NORESTART; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ if (stt->remaining_ticks && ++ backend->shaders_state == KBASE_SHADERS_WAIT_OFF_CORESTACK_ON) { ++ stt->remaining_ticks--; ++ ++ /* If the remaining ticks just changed from 1 to 0, invoke the ++ * PM state machine to power off the shader cores. ++ */ ++ if (!stt->remaining_ticks && !backend->shaders_desired) ++ kbase_pm_update_state(kbdev); ++ } ++ ++ if (stt->needed) { ++ hrtimer_forward_now(timer, stt->configured_interval); ++ restart = HRTIMER_RESTART; ++ } ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ return restart; ++} ++ ++int kbase_pm_state_machine_init(struct kbase_device *kbdev) ++{ ++ struct kbasep_pm_tick_timer_state *stt = &kbdev->pm.backend.shader_tick_timer; ++ ++ stt->wq = alloc_workqueue("kbase_pm_shader_poweroff", WQ_HIGHPRI | WQ_UNBOUND, 1); ++ if (!stt->wq) ++ return -ENOMEM; ++ ++ INIT_WORK(&stt->work, shader_poweroff_timer_stop_callback); ++ ++ stt->needed = false; ++ hrtimer_init(&stt->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); ++ stt->timer.function = shader_tick_timer_callback; ++ stt->configured_interval = HR_TIMER_DELAY_NSEC(DEFAULT_PM_GPU_POWEROFF_TICK_NS); ++ stt->configured_ticks = DEFAULT_PM_POWEROFF_TICK_SHADER; ++ ++ return 0; ++} ++ ++void kbase_pm_state_machine_term(struct kbase_device *kbdev) ++{ ++ hrtimer_cancel(&kbdev->pm.backend.shader_tick_timer.timer); ++ destroy_workqueue(kbdev->pm.backend.shader_tick_timer.wq); ++} ++ ++void kbase_pm_reset_start_locked(struct kbase_device *kbdev) ++{ ++ struct kbase_pm_backend_data *backend = &kbdev->pm.backend; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ backend->in_reset = true; ++ backend->l2_state = KBASE_L2_RESET_WAIT; ++#if !MALI_USE_CSF ++ backend->shaders_state = KBASE_SHADERS_RESET_WAIT; ++#else ++ /* MCU state machine is exercised only after the initial load/boot ++ * of the firmware. ++ */ ++ if (likely(kbdev->csf.firmware_inited)) { ++ backend->mcu_state = KBASE_MCU_RESET_WAIT; ++ kbdev->csf.firmware_reload_needed = true; ++ } else { ++ WARN_ON(backend->mcu_state != KBASE_MCU_OFF); ++ } ++#endif ++ ++ /* We're in a reset, so hwcnt will have been synchronously disabled by ++ * this function's caller as part of the reset process. We therefore ++ * know that any call to kbase_hwcnt_context_disable_atomic, if ++ * required to sync the hwcnt refcount with our internal state, is ++ * guaranteed to succeed. ++ */ ++ backend->hwcnt_desired = false; ++ if (!backend->hwcnt_disabled) { ++ WARN_ON(!kbase_hwcnt_context_disable_atomic( ++ kbdev->hwcnt_gpu_ctx)); ++ backend->hwcnt_disabled = true; ++ } ++ ++ shader_poweroff_timer_queue_cancel(kbdev); ++} ++ ++void kbase_pm_reset_complete(struct kbase_device *kbdev) ++{ ++ struct kbase_pm_backend_data *backend = &kbdev->pm.backend; ++ unsigned long flags; ++ ++ WARN_ON(!kbase_reset_gpu_is_active(kbdev)); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ /* As GPU has just been reset, that results in implicit flush of L2 ++ * cache, can safely mark the pending cache flush operation (if there ++ * was any) as complete and unblock the waiter. ++ * No work can be submitted whilst GPU reset is ongoing. ++ */ ++ kbase_gpu_cache_clean_wait_complete(kbdev); ++ backend->in_reset = false; ++ kbase_pm_update_state(kbdev); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++} ++ ++/* Timeout for kbase_pm_wait_for_desired_state when wait_event_killable has ++ * aborted due to a fatal signal. If the time spent waiting has exceeded this ++ * threshold then there is most likely a hardware issue. */ ++#define PM_TIMEOUT_MS (5000) /* 5s */ ++ ++static void kbase_pm_timed_out(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ ++ dev_err(kbdev->dev, "Power transition timed out unexpectedly\n"); ++#if !MALI_USE_CSF ++ CSTD_UNUSED(flags); ++ dev_err(kbdev->dev, "Desired state :\n"); ++ dev_err(kbdev->dev, "\tShader=%016llx\n", ++ kbdev->pm.backend.shaders_desired ? kbdev->pm.backend.shaders_avail : 0); ++#else ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ dev_err(kbdev->dev, "\tMCU desired = %d\n", ++ kbase_pm_is_mcu_desired(kbdev)); ++ dev_err(kbdev->dev, "\tMCU sw state = %d\n", ++ kbdev->pm.backend.mcu_state); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++#endif ++ dev_err(kbdev->dev, "Current state :\n"); ++ dev_err(kbdev->dev, "\tShader=%08x%08x\n", ++ kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(SHADER_READY_HI)), ++ kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(SHADER_READY_LO))); ++ dev_err(kbdev->dev, "\tTiler =%08x%08x\n", ++ kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(TILER_READY_HI)), ++ kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(TILER_READY_LO))); ++ dev_err(kbdev->dev, "\tL2 =%08x%08x\n", ++ kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(L2_READY_HI)), ++ kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(L2_READY_LO))); ++#if MALI_USE_CSF ++ dev_err(kbdev->dev, "\tMCU status = %d\n", ++ kbase_reg_read(kbdev, GPU_CONTROL_REG(MCU_STATUS))); ++#endif ++ dev_err(kbdev->dev, "Cores transitioning :\n"); ++ dev_err(kbdev->dev, "\tShader=%08x%08x\n", ++ kbase_reg_read(kbdev, GPU_CONTROL_REG( ++ SHADER_PWRTRANS_HI)), ++ kbase_reg_read(kbdev, GPU_CONTROL_REG( ++ SHADER_PWRTRANS_LO))); ++ dev_err(kbdev->dev, "\tTiler =%08x%08x\n", ++ kbase_reg_read(kbdev, GPU_CONTROL_REG( ++ TILER_PWRTRANS_HI)), ++ kbase_reg_read(kbdev, GPU_CONTROL_REG( ++ TILER_PWRTRANS_LO))); ++ dev_err(kbdev->dev, "\tL2 =%08x%08x\n", ++ kbase_reg_read(kbdev, GPU_CONTROL_REG( ++ L2_PWRTRANS_HI)), ++ kbase_reg_read(kbdev, GPU_CONTROL_REG( ++ L2_PWRTRANS_LO))); ++ ++ dev_err(kbdev->dev, "Sending reset to GPU - all running jobs will be lost\n"); ++ if (kbase_prepare_to_reset_gpu(kbdev)) ++ kbase_reset_gpu(kbdev); ++} ++ ++void kbase_pm_wait_for_l2_powered(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ unsigned long timeout; ++ int err; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_pm_update_state(kbdev); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ timeout = jiffies + msecs_to_jiffies(PM_TIMEOUT_MS); ++ ++ /* Wait for cores */ ++ err = wait_event_killable(kbdev->pm.backend.gpu_in_desired_state_wait, ++ kbase_pm_is_in_desired_state_with_l2_powered(kbdev)); ++ ++ if (err < 0 && time_after(jiffies, timeout)) ++ kbase_pm_timed_out(kbdev); ++} ++ ++int kbase_pm_wait_for_desired_state(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ long remaining; ++#if MALI_USE_CSF ++ long timeout = kbase_csf_timeout_in_jiffies(PM_TIMEOUT_MS); ++#else ++ long timeout = msecs_to_jiffies(PM_TIMEOUT_MS); ++#endif ++ int err = 0; ++ ++ /* Let the state machine latch the most recent desired state. */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_pm_update_state(kbdev); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ /* Wait for cores */ ++#if KERNEL_VERSION(4, 13, 1) <= LINUX_VERSION_CODE ++ remaining = wait_event_killable_timeout( ++ kbdev->pm.backend.gpu_in_desired_state_wait, ++ kbase_pm_is_in_desired_state(kbdev), timeout); ++#else ++ remaining = wait_event_timeout( ++ kbdev->pm.backend.gpu_in_desired_state_wait, ++ kbase_pm_is_in_desired_state(kbdev), timeout); ++#endif ++ ++ if (!remaining) { ++ kbase_pm_timed_out(kbdev); ++ err = -ETIMEDOUT; ++ } else if (remaining < 0) { ++ dev_info(kbdev->dev, ++ "Wait for desired PM state got interrupted"); ++ err = (int)remaining; ++ } ++ ++ return err; ++} ++KBASE_EXPORT_TEST_API(kbase_pm_wait_for_desired_state); ++ ++void kbase_pm_enable_interrupts(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ ++ KBASE_DEBUG_ASSERT(NULL != kbdev); ++ /* ++ * Clear all interrupts, ++ * and unmask them all. ++ */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), GPU_IRQ_REG_ALL); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), GPU_IRQ_REG_ALL); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), 0xFFFFFFFF); ++ kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), 0xFFFFFFFF); ++ ++ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), 0xFFFFFFFF); ++#if MALI_USE_CSF ++ /* Enable only the Page fault bits part */ ++ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0xFFFF); ++#else ++ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0xFFFFFFFF); ++#endif ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_enable_interrupts); ++ ++void kbase_pm_disable_interrupts_nolock(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(NULL != kbdev); ++ /* ++ * Mask all interrupts, ++ * and clear them all. ++ */ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), 0); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), GPU_IRQ_REG_ALL); ++ kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), 0); ++ kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), 0xFFFFFFFF); ++ ++ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0); ++ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), 0xFFFFFFFF); ++} ++ ++void kbase_pm_disable_interrupts(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_pm_disable_interrupts_nolock(kbdev); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_disable_interrupts); ++ ++/* ++ * pmu layout: ++ * 0x0000: PMU TAG (RO) (0xCAFECAFE) ++ * 0x0004: PMU VERSION ID (RO) (0x00000000) ++ * 0x0008: CLOCK ENABLE (RW) (31:1 SBZ, 0 CLOCK STATE) ++ */ ++void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume) ++{ ++ bool reset_required = is_resume; ++ unsigned long flags; ++ ++ KBASE_DEBUG_ASSERT(NULL != kbdev); ++#if !MALI_USE_CSF ++ lockdep_assert_held(&kbdev->js_data.runpool_mutex); ++#endif /* !MALI_USE_CSF */ ++ lockdep_assert_held(&kbdev->pm.lock); ++ ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++ if (WARN_ON(kbase_pm_is_gpu_lost(kbdev))) { ++ dev_err(kbdev->dev, ++ "%s: Cannot power up while GPU lost", __func__); ++ return; ++ } ++#endif ++ ++ if (kbdev->pm.backend.gpu_powered) { ++ /* Already turned on */ ++ if (kbdev->poweroff_pending) ++ kbase_pm_enable_interrupts(kbdev); ++ kbdev->poweroff_pending = false; ++ KBASE_DEBUG_ASSERT(!is_resume); ++ return; ++ } ++ ++ kbdev->poweroff_pending = false; ++ ++ KBASE_KTRACE_ADD(kbdev, PM_GPU_ON, NULL, 0u); ++ ++ if (is_resume && kbdev->pm.backend.callback_power_resume) { ++ kbdev->pm.backend.callback_power_resume(kbdev); ++ return; ++ } else if (kbdev->pm.backend.callback_power_on) { ++ reset_required = kbdev->pm.backend.callback_power_on(kbdev); ++ } ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbdev->pm.backend.gpu_powered = true; ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ if (reset_required) { ++ /* GPU state was lost, reset GPU to ensure it is in a ++ * consistent state */ ++ kbase_pm_init_hw(kbdev, PM_ENABLE_IRQS); ++ } ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++ else { ++ struct kbase_arbiter_vm_state *arb_vm_state = ++ kbdev->pm.arb_vm_state; ++ ++ /* In the case that the GPU has just been granted by ++ * the Arbiter, a reset will have already been done. ++ * However, it is still necessary to initialize the GPU. ++ */ ++ if (arb_vm_state->vm_arb_starting) ++ kbase_pm_init_hw(kbdev, PM_ENABLE_IRQS | ++ PM_NO_RESET); ++ } ++#endif /* CONFIG_MALI_ARBITER_SUPPORT */ ++ ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_ctx_sched_restore_all_as(kbdev); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ ++ if (kbdev->dummy_job_wa.flags & ++ KBASE_DUMMY_JOB_WA_FLAG_LOGICAL_SHADER_POWER) { ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_dummy_job_wa_execute(kbdev, ++ kbase_pm_get_present_cores(kbdev, ++ KBASE_PM_CORE_SHADER)); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ } ++ ++ /* Enable the interrupts */ ++ kbase_pm_enable_interrupts(kbdev); ++ ++ /* Turn on the L2 caches */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbdev->pm.backend.gpu_ready = true; ++ kbdev->pm.backend.l2_desired = true; ++ kbase_pm_update_state(kbdev); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_clock_on); ++ ++bool kbase_pm_clock_off(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ ++ KBASE_DEBUG_ASSERT(NULL != kbdev); ++ lockdep_assert_held(&kbdev->pm.lock); ++ ++ /* ASSERT that the cores should now be unavailable. No lock needed. */ ++ WARN_ON(kbdev->pm.backend.shaders_state != KBASE_SHADERS_OFF_CORESTACK_OFF); ++ ++ kbdev->poweroff_pending = true; ++ ++ if (!kbdev->pm.backend.gpu_powered) { ++ /* Already turned off */ ++ return true; ++ } ++ ++ KBASE_KTRACE_ADD(kbdev, PM_GPU_OFF, NULL, 0u); ++ ++ /* Disable interrupts. This also clears any outstanding interrupts */ ++ kbase_pm_disable_interrupts(kbdev); ++ /* Ensure that any IRQ handlers have finished */ ++ kbase_synchronize_irqs(kbdev); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ if (atomic_read(&kbdev->faults_pending)) { ++ /* Page/bus faults are still being processed. The GPU can not ++ * be powered off until they have completed */ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ return false; ++ } ++ ++ kbase_pm_cache_snoop_disable(kbdev); ++ ++ kbdev->pm.backend.gpu_ready = false; ++ ++ /* The GPU power may be turned off from this point */ ++ kbdev->pm.backend.gpu_powered = false; ++ ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++ if (kbase_pm_is_gpu_lost(kbdev)) { ++ /* Ensure we unblock any threads that are stuck waiting ++ * for the GPU ++ */ ++ kbase_gpu_cache_clean_wait_complete(kbdev); ++ } ++#endif ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++ kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_IDLE_EVENT); ++#endif /* CONFIG_MALI_ARBITER_SUPPORT */ ++ ++ if (kbdev->pm.backend.callback_power_off) ++ kbdev->pm.backend.callback_power_off(kbdev); ++ return true; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_clock_off); ++ ++struct kbasep_reset_timeout_data { ++ struct hrtimer timer; ++ bool timed_out; ++ struct kbase_device *kbdev; ++}; ++ ++void kbase_pm_reset_done(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ kbdev->pm.backend.reset_done = true; ++ wake_up(&kbdev->pm.backend.reset_done_wait); ++} ++ ++/** ++ * kbase_pm_wait_for_reset - Wait for a reset to happen ++ * ++ * Wait for the %RESET_COMPLETED IRQ to occur, then reset the waiting state. ++ * ++ * @kbdev: Kbase device ++ */ ++static void kbase_pm_wait_for_reset(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->pm.lock); ++ ++ wait_event(kbdev->pm.backend.reset_done_wait, ++ (kbdev->pm.backend.reset_done)); ++ kbdev->pm.backend.reset_done = false; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_reset_done); ++ ++static enum hrtimer_restart kbasep_reset_timeout(struct hrtimer *timer) ++{ ++ struct kbasep_reset_timeout_data *rtdata = ++ container_of(timer, struct kbasep_reset_timeout_data, timer); ++ ++ rtdata->timed_out = 1; ++ ++ /* Set the wait queue to wake up kbase_pm_init_hw even though the reset ++ * hasn't completed */ ++ kbase_pm_reset_done(rtdata->kbdev); ++ ++ return HRTIMER_NORESTART; ++} ++ ++static int kbase_set_jm_quirks(struct kbase_device *kbdev, const u32 prod_id) ++{ ++#if MALI_USE_CSF ++ kbdev->hw_quirks_jm = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(CSF_CONFIG)); ++#else ++ u32 hw_quirks_jm = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(JM_CONFIG)); ++ ++ if (GPU_ID2_MODEL_MATCH_VALUE(prod_id) == GPU_ID2_PRODUCT_TMIX) { ++ /* Only for tMIx */ ++ u32 coherency_features; ++ ++ coherency_features = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(COHERENCY_FEATURES)); ++ ++ /* (COHERENCY_ACE_LITE | COHERENCY_ACE) was incorrectly ++ * documented for tMIx so force correct value here. ++ */ ++ if (coherency_features == ++ COHERENCY_FEATURE_BIT(COHERENCY_ACE)) { ++ hw_quirks_jm |= (COHERENCY_ACE_LITE | ++ COHERENCY_ACE) << ++ JM_FORCE_COHERENCY_FEATURES_SHIFT; ++ } ++ } ++ ++ if (kbase_is_gpu_removed(kbdev)) ++ return -EIO; ++ ++ kbdev->hw_quirks_jm = hw_quirks_jm; ++ ++#endif /* !MALI_USE_CSF */ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_IDVS_GROUP_SIZE)) { ++ int default_idvs_group_size = 0xF; ++ u32 tmp; ++ ++ if (of_property_read_u32(kbdev->dev->of_node, ++ "idvs-group-size", &tmp)) ++ tmp = default_idvs_group_size; ++ ++ if (tmp > IDVS_GROUP_MAX_SIZE) { ++ dev_err(kbdev->dev, ++ "idvs-group-size of %d is too large. Maximum value is %d", ++ tmp, IDVS_GROUP_MAX_SIZE); ++ tmp = default_idvs_group_size; ++ } ++ ++ kbdev->hw_quirks_jm |= tmp << IDVS_GROUP_SIZE_SHIFT; ++ } ++ ++#define MANUAL_POWER_CONTROL ((u32)(1 << 8)) ++ if (corestack_driver_control) ++ kbdev->hw_quirks_jm |= MANUAL_POWER_CONTROL; ++ ++ return 0; ++} ++ ++static int kbase_set_sc_quirks(struct kbase_device *kbdev, const u32 prod_id) ++{ ++ u32 hw_quirks_sc = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(SHADER_CONFIG)); ++ ++ if (kbase_is_gpu_removed(kbdev)) ++ return -EIO; ++ ++ if (prod_id < 0x750 || prod_id == 0x6956) /* T60x, T62x, T72x */ ++ hw_quirks_sc |= SC_LS_ATTR_CHECK_DISABLE; ++ else if (prod_id >= 0x750 && prod_id <= 0x880) /* T76x, T8xx */ ++ hw_quirks_sc |= SC_LS_ALLOW_ATTR_TYPES; ++ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_2968_TTRX_3162)) ++ hw_quirks_sc |= SC_VAR_ALGORITHM; ++ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_TLS_HASHING)) ++ hw_quirks_sc |= SC_TLS_HASH_ENABLE; ++ ++ kbdev->hw_quirks_sc = hw_quirks_sc; ++ ++ return 0; ++} ++ ++static int kbase_set_tiler_quirks(struct kbase_device *kbdev) ++{ ++ u32 hw_quirks_tiler = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(TILER_CONFIG)); ++ ++ if (kbase_is_gpu_removed(kbdev)) ++ return -EIO; ++ ++ /* Set tiler clock gate override if required */ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_T76X_3953)) ++ hw_quirks_tiler |= TC_CLOCK_GATE_OVERRIDE; ++ ++ kbdev->hw_quirks_tiler = hw_quirks_tiler; ++ ++ return 0; ++} ++ ++static int kbase_pm_hw_issues_detect(struct kbase_device *kbdev) ++{ ++ struct device_node *np = kbdev->dev->of_node; ++ const u32 gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; ++ const u32 prod_id = (gpu_id & GPU_ID_VERSION_PRODUCT_ID) >> ++ GPU_ID_VERSION_PRODUCT_ID_SHIFT; ++ int error = 0; ++ ++ kbdev->hw_quirks_jm = 0; ++ kbdev->hw_quirks_sc = 0; ++ kbdev->hw_quirks_tiler = 0; ++ kbdev->hw_quirks_mmu = 0; ++ ++ if (!of_property_read_u32(np, "quirks_jm", ++ &kbdev->hw_quirks_jm)) { ++ dev_info(kbdev->dev, ++ "Found quirks_jm = [0x%x] in Devicetree\n", ++ kbdev->hw_quirks_jm); ++ } else { ++ error = kbase_set_jm_quirks(kbdev, prod_id); ++ if (error) ++ return error; ++ } ++ ++ if (!of_property_read_u32(np, "quirks_sc", ++ &kbdev->hw_quirks_sc)) { ++ dev_info(kbdev->dev, ++ "Found quirks_sc = [0x%x] in Devicetree\n", ++ kbdev->hw_quirks_sc); ++ } else { ++ error = kbase_set_sc_quirks(kbdev, prod_id); ++ if (error) ++ return error; ++ } ++ ++ if (!of_property_read_u32(np, "quirks_tiler", ++ &kbdev->hw_quirks_tiler)) { ++ dev_info(kbdev->dev, ++ "Found quirks_tiler = [0x%x] in Devicetree\n", ++ kbdev->hw_quirks_tiler); ++ } else { ++ error = kbase_set_tiler_quirks(kbdev); ++ if (error) ++ return error; ++ } ++ ++ if (!of_property_read_u32(np, "quirks_mmu", ++ &kbdev->hw_quirks_mmu)) { ++ dev_info(kbdev->dev, ++ "Found quirks_mmu = [0x%x] in Devicetree\n", ++ kbdev->hw_quirks_mmu); ++ } else { ++ error = kbase_set_mmu_quirks(kbdev); ++ } ++ ++ return error; ++} ++ ++static void kbase_pm_hw_issues_apply(struct kbase_device *kbdev) ++{ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(SHADER_CONFIG), ++ kbdev->hw_quirks_sc); ++ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(TILER_CONFIG), ++ kbdev->hw_quirks_tiler); ++ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(L2_MMU_CONFIG), ++ kbdev->hw_quirks_mmu); ++#if MALI_USE_CSF ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(CSF_CONFIG), ++ kbdev->hw_quirks_jm); ++#else ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(JM_CONFIG), ++ kbdev->hw_quirks_jm); ++#endif ++} ++ ++void kbase_pm_cache_snoop_enable(struct kbase_device *kbdev) ++{ ++ if ((kbdev->current_gpu_coherency_mode == COHERENCY_ACE) && ++ !kbdev->cci_snoop_enabled) { ++#ifdef CONFIG_ARM64 ++ if (kbdev->snoop_enable_smc != 0) ++ kbase_invoke_smc_fid(kbdev->snoop_enable_smc, 0, 0, 0); ++#endif /* CONFIG_ARM64 */ ++ dev_dbg(kbdev->dev, "MALI - CCI Snoops - Enabled\n"); ++ kbdev->cci_snoop_enabled = true; ++ } ++} ++ ++void kbase_pm_cache_snoop_disable(struct kbase_device *kbdev) ++{ ++ if (kbdev->cci_snoop_enabled) { ++#ifdef CONFIG_ARM64 ++ if (kbdev->snoop_disable_smc != 0) { ++ mali_cci_flush_l2(kbdev); ++ kbase_invoke_smc_fid(kbdev->snoop_disable_smc, 0, 0, 0); ++ } ++#endif /* CONFIG_ARM64 */ ++ dev_dbg(kbdev->dev, "MALI - CCI Snoops Disabled\n"); ++ kbdev->cci_snoop_enabled = false; ++ } ++} ++ ++static void reenable_protected_mode_hwcnt(struct kbase_device *kbdev) ++{ ++ unsigned long irq_flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); ++ kbdev->protected_mode_hwcnt_desired = true; ++ if (kbdev->protected_mode_hwcnt_disabled) { ++ kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); ++ kbdev->protected_mode_hwcnt_disabled = false; ++ } ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); ++} ++ ++static int kbase_pm_do_reset(struct kbase_device *kbdev) ++{ ++ struct kbasep_reset_timeout_data rtdata; ++ int ret; ++ ++ KBASE_KTRACE_ADD(kbdev, CORE_GPU_SOFT_RESET, NULL, 0); ++ ++ KBASE_TLSTREAM_JD_GPU_SOFT_RESET(kbdev, kbdev); ++ ++ if (kbdev->pm.backend.callback_soft_reset) { ++ ret = kbdev->pm.backend.callback_soft_reset(kbdev); ++ if (ret < 0) ++ return ret; ++ else if (ret > 0) ++ return 0; ++ } else { ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), ++ GPU_COMMAND_SOFT_RESET); ++ } ++ ++ /* Unmask the reset complete interrupt only */ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), RESET_COMPLETED); ++ ++ /* Initialize a structure for tracking the status of the reset */ ++ rtdata.kbdev = kbdev; ++ rtdata.timed_out = 0; ++ ++ /* Create a timer to use as a timeout on the reset */ ++ hrtimer_init_on_stack(&rtdata.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); ++ rtdata.timer.function = kbasep_reset_timeout; ++ ++ hrtimer_start(&rtdata.timer, HR_TIMER_DELAY_MSEC(RESET_TIMEOUT), ++ HRTIMER_MODE_REL); ++ ++ /* Wait for the RESET_COMPLETED interrupt to be raised */ ++ kbase_pm_wait_for_reset(kbdev); ++ ++ if (rtdata.timed_out == 0) { ++ /* GPU has been reset */ ++ hrtimer_cancel(&rtdata.timer); ++ destroy_hrtimer_on_stack(&rtdata.timer); ++ return 0; ++ } ++ ++ /* No interrupt has been received - check if the RAWSTAT register says ++ * the reset has completed */ ++ if ((kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT)) & ++ RESET_COMPLETED)) { ++ /* The interrupt is set in the RAWSTAT; this suggests that the ++ * interrupts are not getting to the CPU */ ++ dev_err(kbdev->dev, "Reset interrupt didn't reach CPU. Check interrupt assignments.\n"); ++ /* If interrupts aren't working we can't continue. */ ++ destroy_hrtimer_on_stack(&rtdata.timer); ++ return -EINVAL; ++ } ++ ++ if (kbase_is_gpu_removed(kbdev)) { ++ dev_dbg(kbdev->dev, "GPU has been removed, reset no longer needed.\n"); ++ destroy_hrtimer_on_stack(&rtdata.timer); ++ return -EINVAL; ++ } ++ ++ /* The GPU doesn't seem to be responding to the reset so try a hard ++ * reset */ ++ dev_err(kbdev->dev, "Failed to soft-reset GPU (timed out after %d ms), now attempting a hard reset\n", ++ RESET_TIMEOUT); ++ KBASE_KTRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, 0); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), ++ GPU_COMMAND_HARD_RESET); ++ ++ /* Restart the timer to wait for the hard reset to complete */ ++ rtdata.timed_out = 0; ++ ++ hrtimer_start(&rtdata.timer, HR_TIMER_DELAY_MSEC(RESET_TIMEOUT), ++ HRTIMER_MODE_REL); ++ ++ /* Wait for the RESET_COMPLETED interrupt to be raised */ ++ kbase_pm_wait_for_reset(kbdev); ++ ++ if (rtdata.timed_out == 0) { ++ /* GPU has been reset */ ++ hrtimer_cancel(&rtdata.timer); ++ destroy_hrtimer_on_stack(&rtdata.timer); ++ return 0; ++ } ++ ++ destroy_hrtimer_on_stack(&rtdata.timer); ++ ++ dev_err(kbdev->dev, "Failed to hard-reset the GPU (timed out after %d ms)\n", ++ RESET_TIMEOUT); ++ ++ return -EINVAL; ++} ++ ++int kbase_pm_protected_mode_enable(struct kbase_device *const kbdev) ++{ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), ++ GPU_COMMAND_SET_PROTECTED_MODE); ++ return 0; ++} ++ ++int kbase_pm_protected_mode_disable(struct kbase_device *const kbdev) ++{ ++ lockdep_assert_held(&kbdev->pm.lock); ++ ++ return kbase_pm_do_reset(kbdev); ++} ++ ++int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags) ++{ ++ unsigned long irq_flags; ++ int err = 0; ++ ++ KBASE_DEBUG_ASSERT(NULL != kbdev); ++ lockdep_assert_held(&kbdev->pm.lock); ++ ++ /* Ensure the clock is on before attempting to access the hardware */ ++ if (!kbdev->pm.backend.gpu_powered) { ++ if (kbdev->pm.backend.callback_power_on) ++ kbdev->pm.backend.callback_power_on(kbdev); ++ ++ kbdev->pm.backend.gpu_powered = true; ++ } ++ ++ /* Ensure interrupts are off to begin with, this also clears any ++ * outstanding interrupts */ ++ kbase_pm_disable_interrupts(kbdev); ++ /* Ensure cache snoops are disabled before reset. */ ++ kbase_pm_cache_snoop_disable(kbdev); ++ /* Prepare for the soft-reset */ ++ kbdev->pm.backend.reset_done = false; ++ ++ /* The cores should be made unavailable due to the reset */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); ++ if (kbdev->pm.backend.shaders_state != KBASE_SHADERS_OFF_CORESTACK_OFF) ++ KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, 0u); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); ++ ++ /* Soft reset the GPU */ ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++ if (!(flags & PM_NO_RESET)) ++#endif /* CONFIG_MALI_ARBITER_SUPPORT */ ++ err = kbdev->protected_ops->protected_mode_disable( ++ kbdev->protected_dev); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); ++ kbdev->protected_mode = false; ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); ++ ++ if (err) ++ goto exit; ++ ++ if (flags & PM_HW_ISSUES_DETECT) { ++ err = kbase_pm_hw_issues_detect(kbdev); ++ if (err) ++ goto exit; ++ } ++ ++ kbase_pm_hw_issues_apply(kbdev); ++ kbase_cache_set_coherency_mode(kbdev, kbdev->system_coherency); ++ ++ /* Sanity check protected mode was left after reset */ ++ WARN_ON(kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_STATUS)) & ++ GPU_STATUS_PROTECTED_MODE_ACTIVE); ++ ++ /* If cycle counter was in use re-enable it, enable_irqs will only be ++ * false when called from kbase_pm_powerup */ ++ if (kbdev->pm.backend.gpu_cycle_counter_requests && ++ (flags & PM_ENABLE_IRQS)) { ++ kbase_pm_enable_interrupts(kbdev); ++ ++ /* Re-enable the counters if we need to */ ++ spin_lock_irqsave( ++ &kbdev->pm.backend.gpu_cycle_counter_requests_lock, ++ irq_flags); ++ if (kbdev->pm.backend.gpu_cycle_counter_requests) ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), ++ GPU_COMMAND_CYCLE_COUNT_START); ++ spin_unlock_irqrestore( ++ &kbdev->pm.backend.gpu_cycle_counter_requests_lock, ++ irq_flags); ++ ++ kbase_pm_disable_interrupts(kbdev); ++ } ++ ++ if (flags & PM_ENABLE_IRQS) ++ kbase_pm_enable_interrupts(kbdev); ++ ++exit: ++ if (!kbdev->pm.backend.protected_entry_transition_override) { ++ /* Re-enable GPU hardware counters if we're resetting from ++ * protected mode. ++ */ ++ reenable_protected_mode_hwcnt(kbdev); ++ } ++ ++ return err; ++} ++ ++/** ++ * kbase_pm_request_gpu_cycle_counter_do_request - Request cycle counters ++ * ++ * Increase the count of cycle counter users and turn the cycle counters on if ++ * they were previously off ++ * ++ * This function is designed to be called by ++ * kbase_pm_request_gpu_cycle_counter() or ++ * kbase_pm_request_gpu_cycle_counter_l2_is_on() only ++ * ++ * When this function is called the l2 cache must be on - i.e., the GPU must be ++ * on. ++ * ++ * @kbdev: The kbase device structure of the device ++ */ ++static void ++kbase_pm_request_gpu_cycle_counter_do_request(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->pm.backend.gpu_cycle_counter_requests_lock, ++ flags); ++ ++ ++kbdev->pm.backend.gpu_cycle_counter_requests; ++ ++ if (1 == kbdev->pm.backend.gpu_cycle_counter_requests) ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), ++ GPU_COMMAND_CYCLE_COUNT_START); ++ ++ spin_unlock_irqrestore( ++ &kbdev->pm.backend.gpu_cycle_counter_requests_lock, ++ flags); ++} ++ ++void kbase_pm_request_gpu_cycle_counter(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered); ++ ++ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests < ++ INT_MAX); ++ ++ kbase_pm_request_gpu_cycle_counter_do_request(kbdev); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_request_gpu_cycle_counter); ++ ++void kbase_pm_request_gpu_cycle_counter_l2_is_on(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered); ++ ++ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests < ++ INT_MAX); ++ ++ kbase_pm_request_gpu_cycle_counter_do_request(kbdev); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_request_gpu_cycle_counter_l2_is_on); ++ ++void kbase_pm_release_gpu_cycle_counter_nolock(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ spin_lock_irqsave(&kbdev->pm.backend.gpu_cycle_counter_requests_lock, ++ flags); ++ ++ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests > 0); ++ ++ --kbdev->pm.backend.gpu_cycle_counter_requests; ++ ++ if (0 == kbdev->pm.backend.gpu_cycle_counter_requests) ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), ++ GPU_COMMAND_CYCLE_COUNT_STOP); ++ ++ spin_unlock_irqrestore( ++ &kbdev->pm.backend.gpu_cycle_counter_requests_lock, ++ flags); ++} ++ ++void kbase_pm_release_gpu_cycle_counter(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ kbase_pm_release_gpu_cycle_counter_nolock(kbdev); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_release_gpu_cycle_counter); +diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_internal.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_internal.h +new file mode 100755 +index 000000000000..50ca016bbd6d +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_internal.h +@@ -0,0 +1,739 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++/* ++ * Power management API definitions used internally by GPU backend ++ */ ++ ++#ifndef _KBASE_BACKEND_PM_INTERNAL_H_ ++#define _KBASE_BACKEND_PM_INTERNAL_H_ ++ ++#include ++ ++#include "mali_kbase_pm_ca.h" ++#include "mali_kbase_pm_policy.h" ++ ++ ++/** ++ * kbase_pm_dev_idle - The GPU is idle. ++ * ++ * The OS may choose to turn off idle devices ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_dev_idle(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_dev_activate - The GPU is active. ++ * ++ * The OS should avoid opportunistically turning off the GPU while it is active ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_dev_activate(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_get_present_cores - Get details of the cores that are present in ++ * the device. ++ * ++ * This function can be called by the active power policy to return a bitmask of ++ * the cores (of a specified type) present in the GPU device and also a count of ++ * the number of cores. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid ++ * pointer) ++ * @type: The type of core (see the enum kbase_pm_core_type enumeration) ++ * ++ * Return: The bit mask of cores present ++ */ ++u64 kbase_pm_get_present_cores(struct kbase_device *kbdev, ++ enum kbase_pm_core_type type); ++ ++/** ++ * kbase_pm_get_active_cores - Get details of the cores that are currently ++ * active in the device. ++ * ++ * This function can be called by the active power policy to return a bitmask of ++ * the cores (of a specified type) that are actively processing work (i.e. ++ * turned on *and* busy). ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * @type: The type of core (see the enum kbase_pm_core_type enumeration) ++ * ++ * Return: The bit mask of active cores ++ */ ++u64 kbase_pm_get_active_cores(struct kbase_device *kbdev, ++ enum kbase_pm_core_type type); ++ ++/** ++ * kbase_pm_get_trans_cores - Get details of the cores that are currently ++ * transitioning between power states. ++ * ++ * This function can be called by the active power policy to return a bitmask of ++ * the cores (of a specified type) that are currently transitioning between ++ * power states. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * @type: The type of core (see the enum kbase_pm_core_type enumeration) ++ * ++ * Return: The bit mask of transitioning cores ++ */ ++u64 kbase_pm_get_trans_cores(struct kbase_device *kbdev, ++ enum kbase_pm_core_type type); ++ ++/** ++ * kbase_pm_get_ready_cores - Get details of the cores that are currently ++ * powered and ready for jobs. ++ * ++ * This function can be called by the active power policy to return a bitmask of ++ * the cores (of a specified type) that are powered and ready for jobs (they may ++ * or may not be currently executing jobs). ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * @type: The type of core (see the enum kbase_pm_core_type enumeration) ++ * ++ * Return: The bit mask of ready cores ++ */ ++u64 kbase_pm_get_ready_cores(struct kbase_device *kbdev, ++ enum kbase_pm_core_type type); ++ ++/** ++ * kbase_pm_clock_on - Turn the clock for the device on, and enable device ++ * interrupts. ++ * ++ * This function can be used by a power policy to turn the clock for the GPU on. ++ * It should be modified during integration to perform the necessary actions to ++ * ensure that the GPU is fully powered and clocked. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid ++ * pointer) ++ * @is_resume: true if clock on due to resume after suspend, false otherwise ++ */ ++void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume); ++ ++/** ++ * kbase_pm_clock_off - Disable device interrupts, and turn the clock for the ++ * device off. ++ * ++ * This function can be used by a power policy to turn the clock for the GPU ++ * off. It should be modified during integration to perform the necessary ++ * actions to turn the clock off (if this is possible in the integration). ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid ++ * pointer) ++ * ++ * Return: true if clock was turned off, or ++ * false if clock can not be turned off due to pending page/bus fault ++ * workers. Caller must flush MMU workqueues and retry ++ */ ++bool kbase_pm_clock_off(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_enable_interrupts - Enable interrupts on the device. ++ * ++ * Interrupts are also enabled after a call to kbase_pm_clock_on(). ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_enable_interrupts(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_disable_interrupts - Disable interrupts on the device. ++ * ++ * This prevents delivery of Power Management interrupts to the CPU so that ++ * kbase_pm_update_state() will not be called from the IRQ handler ++ * until kbase_pm_enable_interrupts() or kbase_pm_clock_on() is called. ++ * ++ * Interrupts are also disabled after a call to kbase_pm_clock_off(). ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_disable_interrupts(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_disable_interrupts_nolock - Version of kbase_pm_disable_interrupts() ++ * that does not take the hwaccess_lock ++ * ++ * Caller must hold the hwaccess_lock. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_disable_interrupts_nolock(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_init_hw - Initialize the hardware. ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * @flags: Flags specifying the type of PM init ++ * ++ * This function checks the GPU ID register to ensure that the GPU is supported ++ * by the driver and performs a reset on the device so that it is in a known ++ * state before the device is used. ++ * ++ * Return: 0 if the device is supported and successfully reset. ++ */ ++int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags); ++ ++/** ++ * kbase_pm_reset_done - The GPU has been reset successfully. ++ * ++ * This function must be called by the GPU interrupt handler when the ++ * RESET_COMPLETED bit is set. It signals to the power management initialization ++ * code that the GPU has been successfully reset. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_reset_done(struct kbase_device *kbdev); ++ ++#if MALI_USE_CSF ++/** ++ * kbase_pm_wait_for_desired_state - Wait for the desired power state to be ++ * reached ++ * ++ * Wait for the L2 and MCU state machines to reach the states corresponding ++ * to the values of 'kbase_pm_is_l2_desired' and 'kbase_pm_is_mcu_desired'. ++ * ++ * The usual use-case for this is to ensure that all parts of GPU have been ++ * powered up after performing a GPU Reset. ++ * ++ * Unlike kbase_pm_update_state(), the caller must not hold hwaccess_lock, ++ * because this function will take that lock itself. ++ * ++ * NOTE: This may not wait until the correct state is reached if there is a ++ * power off in progress and kbase_pm_context_active() was called instead of ++ * kbase_csf_scheduler_pm_active(). ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * Return: 0 on success, error code on error ++ */ ++#else ++/** ++ * kbase_pm_wait_for_desired_state - Wait for the desired power state to be ++ * reached ++ * ++ * Wait for the L2 and shader power state machines to reach the states ++ * corresponding to the values of 'l2_desired' and 'shaders_desired'. ++ * ++ * The usual use-case for this is to ensure cores are 'READY' after performing ++ * a GPU Reset. ++ * ++ * Unlike kbase_pm_update_state(), the caller must not hold hwaccess_lock, ++ * because this function will take that lock itself. ++ * ++ * NOTE: This may not wait until the correct state is reached if there is a ++ * power off in progress. To correctly wait for the desired state the caller ++ * must ensure that this is not the case by, for example, calling ++ * kbase_pm_wait_for_poweroff_complete() ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * Return: 0 on success, error code on error ++ */ ++#endif ++int kbase_pm_wait_for_desired_state(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_wait_for_l2_powered - Wait for the L2 cache to be powered on ++ * ++ * Wait for the L2 to be powered on, and for the L2 and the state machines of ++ * its dependent stack components to stabilise. ++ * ++ * kbdev->pm.active_count must be non-zero when calling this function. ++ * ++ * Unlike kbase_pm_update_state(), the caller must not hold hwaccess_lock, ++ * because this function will take that lock itself. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_wait_for_l2_powered(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_update_dynamic_cores_onoff - Update the L2 and shader power state ++ * machines after changing shader core ++ * availability ++ * ++ * It can be called in any status, so need to check the l2 and shader core ++ * power status in this function or it will break shader/l2 state machine ++ * ++ * Caller must hold hwaccess_lock ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_update_dynamic_cores_onoff(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_update_cores_state_nolock - Variant of kbase_pm_update_cores_state() ++ * where the caller must hold ++ * kbase_device.hwaccess_lock ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_update_cores_state_nolock(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_update_state - Update the L2 and shader power state machines ++ * @kbdev: Device pointer ++ */ ++void kbase_pm_update_state(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_state_machine_init - Initialize the state machines, primarily the ++ * shader poweroff timer ++ * @kbdev: Device pointer ++ */ ++int kbase_pm_state_machine_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_state_machine_term - Clean up the PM state machines' data ++ * @kbdev: Device pointer ++ */ ++void kbase_pm_state_machine_term(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_update_cores_state - Update the desired state of shader cores from ++ * the Power Policy, and begin any power ++ * transitions. ++ * ++ * This function will update the desired_xx_state members of ++ * struct kbase_pm_device_data by calling into the current Power Policy. It will ++ * then begin power transitions to make the hardware acheive the desired shader ++ * core state. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_update_cores_state(struct kbase_device *kbdev); ++ ++/** ++ * kbasep_pm_metrics_init - Initialize the metrics gathering framework. ++ * ++ * This must be called before other metric gathering APIs are called. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * Return: 0 on success, error code on error ++ */ ++int kbasep_pm_metrics_init(struct kbase_device *kbdev); ++ ++/** ++ * kbasep_pm_metrics_term - Terminate the metrics gathering framework. ++ * ++ * This must be called when metric gathering is no longer required. It is an ++ * error to call any metrics gathering function (other than ++ * kbasep_pm_metrics_init()) after calling this function. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbasep_pm_metrics_term(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_report_vsync - Function to be called by the frame buffer driver to ++ * update the vsync metric. ++ * ++ * This function should be called by the frame buffer driver to update whether ++ * the system is hitting the vsync target or not. buffer_updated should be true ++ * if the vsync corresponded with a new frame being displayed, otherwise it ++ * should be false. This function does not need to be called every vsync, but ++ * only when the value of @buffer_updated differs from a previous call. ++ * ++ * @kbdev: The kbase device structure for the device (must be a ++ * valid pointer) ++ * @buffer_updated: True if the buffer has been updated on this VSync, ++ * false otherwise ++ */ ++void kbase_pm_report_vsync(struct kbase_device *kbdev, int buffer_updated); ++ ++/** ++ * kbase_pm_get_dvfs_action - Determine whether the DVFS system should change ++ * the clock speed of the GPU. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * This function should be called regularly by the DVFS system to check whether ++ * the clock speed of the GPU needs updating. ++ */ ++void kbase_pm_get_dvfs_action(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_request_gpu_cycle_counter - Mark that the GPU cycle counter is ++ * needed ++ * ++ * If the caller is the first caller then the GPU cycle counters will be enabled ++ * along with the l2 cache ++ * ++ * The GPU must be powered when calling this function (i.e. ++ * kbase_pm_context_active() must have been called). ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_request_gpu_cycle_counter(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_request_gpu_cycle_counter_l2_is_on - Mark GPU cycle counter is ++ * needed (l2 cache already on) ++ * ++ * This is a version of the above function ++ * (kbase_pm_request_gpu_cycle_counter()) suitable for being called when the ++ * l2 cache is known to be on and assured to be on until the subsequent call of ++ * kbase_pm_release_gpu_cycle_counter() such as when a job is submitted. It does ++ * not sleep and can be called from atomic functions. ++ * ++ * The GPU must be powered when calling this function (i.e. ++ * kbase_pm_context_active() must have been called) and the l2 cache must be ++ * powered on. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_request_gpu_cycle_counter_l2_is_on(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_release_gpu_cycle_counter - Mark that the GPU cycle counter is no ++ * longer in use ++ * ++ * If the caller is the last caller then the GPU cycle counters will be ++ * disabled. A request must have been made before a call to this. ++ * ++ * Caller must not hold the hwaccess_lock, as it will be taken in this function. ++ * If the caller is already holding this lock then ++ * kbase_pm_release_gpu_cycle_counter_nolock() must be used instead. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_release_gpu_cycle_counter(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_release_gpu_cycle_counter_nolock - Version of kbase_pm_release_gpu_cycle_counter() ++ * that does not take hwaccess_lock ++ * ++ * Caller must hold the hwaccess_lock. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_release_gpu_cycle_counter_nolock(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_wait_for_poweroff_complete - Wait for the poweroff workqueue to ++ * complete ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_wait_for_poweroff_complete(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_runtime_init - Initialize runtime-pm for Mali GPU platform device ++ * ++ * Setup the power management callbacks and initialize/enable the runtime-pm ++ * for the Mali GPU platform device, using the callback function. This must be ++ * called before the kbase_pm_register_access_enable() function. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++int kbase_pm_runtime_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_runtime_term - Disable runtime-pm for Mali GPU platform device ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_runtime_term(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_register_access_enable - Enable access to GPU registers ++ * ++ * Enables access to the GPU registers before power management has powered up ++ * the GPU with kbase_pm_powerup(). ++ * ++ * This results in the power management callbacks provided in the driver ++ * configuration to get called to turn on power and/or clocks to the GPU. See ++ * kbase_pm_callback_conf. ++ * ++ * This should only be used before power management is powered up with ++ * kbase_pm_powerup() ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_register_access_enable(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_register_access_disable - Disable early register access ++ * ++ * Disables access to the GPU registers enabled earlier by a call to ++ * kbase_pm_register_access_enable(). ++ * ++ * This results in the power management callbacks provided in the driver ++ * configuration to get called to turn off power and/or clocks to the GPU. See ++ * kbase_pm_callback_conf ++ * ++ * This should only be used before power management is powered up with ++ * kbase_pm_powerup() ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_register_access_disable(struct kbase_device *kbdev); ++ ++/* NOTE: kbase_pm_is_suspending is in mali_kbase.h, because it is an inline ++ * function */ ++ ++/** ++ * kbase_pm_metrics_is_active - Check if the power management metrics ++ * collection is active. ++ * ++ * Note that this returns if the power management metrics collection was ++ * active at the time of calling, it is possible that after the call the metrics ++ * collection enable may have changed state. ++ * ++ * The caller must handle the consequence that the state may have changed. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * Return: true if metrics collection was active else false. ++ */ ++bool kbase_pm_metrics_is_active(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_do_poweron - Power on the GPU, and any cores that are requested. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid ++ * pointer) ++ * @is_resume: true if power on due to resume after suspend, ++ * false otherwise ++ */ ++void kbase_pm_do_poweron(struct kbase_device *kbdev, bool is_resume); ++ ++/** ++ * kbase_pm_do_poweroff - Power off the GPU, and any cores that have been ++ * requested. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid ++ * pointer) ++ */ ++void kbase_pm_do_poweroff(struct kbase_device *kbdev); ++ ++#if defined(CONFIG_MALI_BIFROST_DEVFREQ) || defined(CONFIG_MALI_BIFROST_DVFS) ++void kbase_pm_get_dvfs_metrics(struct kbase_device *kbdev, ++ struct kbasep_pm_metrics *last, ++ struct kbasep_pm_metrics *diff); ++#endif /* defined(CONFIG_MALI_BIFROST_DEVFREQ) || defined(CONFIG_MALI_BIFROST_DVFS) */ ++ ++#ifdef CONFIG_MALI_BIFROST_DVFS ++ ++/** ++ * kbase_platform_dvfs_event - Report utilisation to DVFS code ++ * ++ * Function provided by platform specific code when DVFS is enabled to allow ++ * the power management metrics system to report utilisation. ++ * ++ * @kbdev: The kbase device structure for the device (must be a ++ * valid pointer) ++ * @utilisation: The current calculated utilisation by the metrics system. ++ * @util_gl_share: The current calculated gl share of utilisation. ++ * @util_cl_share: The current calculated cl share of utilisation per core ++ * group. ++ * Return: Returns 0 on failure and non zero on success. ++ */ ++ ++int kbase_platform_dvfs_event(struct kbase_device *kbdev, u32 utilisation, ++ u32 util_gl_share, u32 util_cl_share[2]); ++#endif ++ ++void kbase_pm_power_changed(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_metrics_update - Inform the metrics system that an atom is either ++ * about to be run or has just completed. ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * @now: Pointer to the timestamp of the change, or NULL to use current time ++ * ++ * Caller must hold hwaccess_lock ++ */ ++void kbase_pm_metrics_update(struct kbase_device *kbdev, ++ ktime_t *now); ++ ++/** ++ * kbase_pm_cache_snoop_enable - Allow CPU snoops on the GPU ++ * If the GPU does not have coherency this is a no-op ++ * @kbdev: Device pointer ++ * ++ * This function should be called after L2 power up. ++ */ ++ ++void kbase_pm_cache_snoop_enable(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_cache_snoop_disable - Prevent CPU snoops on the GPU ++ * If the GPU does not have coherency this is a no-op ++ * @kbdev: Device pointer ++ * ++ * This function should be called before L2 power off. ++ */ ++void kbase_pm_cache_snoop_disable(struct kbase_device *kbdev); ++ ++#ifdef CONFIG_MALI_BIFROST_DEVFREQ ++/** ++ * kbase_devfreq_set_core_mask - Set devfreq core mask ++ * @kbdev: Device pointer ++ * @core_mask: New core mask ++ * ++ * This function is used by devfreq to change the available core mask as ++ * required by Dynamic Core Scaling. ++ */ ++void kbase_devfreq_set_core_mask(struct kbase_device *kbdev, u64 core_mask); ++#endif ++ ++/** ++ * kbase_pm_reset_start_locked - Signal that GPU reset has started ++ * @kbdev: Device pointer ++ * ++ * Normal power management operation will be suspended until the reset has ++ * completed. ++ * ++ * Caller must hold hwaccess_lock. ++ */ ++void kbase_pm_reset_start_locked(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_reset_complete - Signal that GPU reset has completed ++ * @kbdev: Device pointer ++ * ++ * Normal power management operation will be resumed. The power manager will ++ * re-evaluate what cores are needed and power on or off as required. ++ */ ++void kbase_pm_reset_complete(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_protected_override_enable - Enable the protected mode override ++ * @kbdev: Device pointer ++ * ++ * When the protected mode override is enabled, all shader cores are requested ++ * to power down, and the L2 power state can be controlled by ++ * kbase_pm_protected_l2_override(). ++ * ++ * Caller must hold hwaccess_lock. ++ */ ++void kbase_pm_protected_override_enable(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_protected_override_disable - Disable the protected mode override ++ * @kbdev: Device pointer ++ * ++ * Caller must hold hwaccess_lock. ++ */ ++void kbase_pm_protected_override_disable(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_protected_l2_override - Control the protected mode L2 override ++ * @kbdev: Device pointer ++ * @override: true to enable the override, false to disable ++ * ++ * When the driver is transitioning in or out of protected mode, the L2 cache is ++ * forced to power off. This can be overridden to force the L2 cache to power ++ * on. This is required to change coherency settings on some GPUs. ++ */ ++void kbase_pm_protected_l2_override(struct kbase_device *kbdev, bool override); ++ ++/** ++ * kbase_pm_protected_entry_override_enable - Enable the protected mode entry ++ * override ++ * @kbdev: Device pointer ++ * ++ * Initiate a GPU reset and enable the protected mode entry override flag if ++ * l2_always_on WA is enabled and platform is fully coherent. If the GPU ++ * reset is already ongoing then protected mode entry override flag will not ++ * be enabled and function will have to be called again. ++ * ++ * When protected mode entry override flag is enabled to power down L2 via GPU ++ * reset, the GPU reset handling behavior gets changed. For example call to ++ * kbase_backend_reset() is skipped, Hw counters are not re-enabled and L2 ++ * isn't powered up again post reset. ++ * This is needed only as a workaround for a Hw issue where explicit power down ++ * of L2 causes a glitch. For entering protected mode on fully coherent ++ * platforms L2 needs to be powered down to switch to IO coherency mode, so to ++ * avoid the glitch GPU reset is used to power down L2. Hence, this function ++ * does nothing on systems where the glitch issue isn't present. ++ * ++ * Caller must hold hwaccess_lock. Should be only called during the transition ++ * to enter protected mode. ++ * ++ * Return: -EAGAIN if a GPU reset was required for the glitch workaround but ++ * was already ongoing, otherwise 0. ++ */ ++int kbase_pm_protected_entry_override_enable(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_protected_entry_override_disable - Disable the protected mode entry ++ * override ++ * @kbdev: Device pointer ++ * ++ * This shall be called once L2 has powered down and switch to IO coherency ++ * mode has been made. As with kbase_pm_protected_entry_override_enable(), ++ * this function does nothing on systems where the glitch issue isn't present. ++ * ++ * Caller must hold hwaccess_lock. Should be only called during the transition ++ * to enter protected mode. ++ */ ++void kbase_pm_protected_entry_override_disable(struct kbase_device *kbdev); ++ ++/* If true, the driver should explicitly control corestack power management, ++ * instead of relying on the Power Domain Controller. ++ */ ++extern bool corestack_driver_control; ++ ++/** ++ * kbase_pm_is_l2_desired - Check whether l2 is desired ++ * ++ * @kbdev: Device pointer ++ * ++ * This shall be called to check whether l2 is needed to power on ++ * ++ * Return: true if l2 need to power on ++ */ ++bool kbase_pm_is_l2_desired(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_lock - Lock all necessary mutexes to perform PM actions ++ * ++ * @kbdev: Device pointer ++ * ++ * This function locks correct mutexes independent of GPU architecture. ++ */ ++static inline void kbase_pm_lock(struct kbase_device *kbdev) ++{ ++#if !MALI_USE_CSF ++ mutex_lock(&kbdev->js_data.runpool_mutex); ++#endif /* !MALI_USE_CSF */ ++ mutex_lock(&kbdev->pm.lock); ++} ++ ++/** ++ * kbase_pm_unlock - Unlock mutexes locked by kbase_pm_lock ++ * ++ * @kbdev: Device pointer ++ */ ++static inline void kbase_pm_unlock(struct kbase_device *kbdev) ++{ ++ mutex_unlock(&kbdev->pm.lock); ++#if !MALI_USE_CSF ++ mutex_unlock(&kbdev->js_data.runpool_mutex); ++#endif /* !MALI_USE_CSF */ ++} ++ ++#endif /* _KBASE_BACKEND_PM_INTERNAL_H_ */ +diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_l2_states.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_l2_states.h +new file mode 100755 +index 000000000000..12cb051db42a +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_l2_states.h +@@ -0,0 +1,38 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* ++ * Backend-specific Power Manager level 2 cache state definitions. ++ * The function-like macro KBASEP_L2_STATE() must be defined before including ++ * this header file. This header file can be included multiple times in the ++ * same compilation unit with different definitions of KBASEP_L2_STATE(). ++ */ ++KBASEP_L2_STATE(OFF) ++KBASEP_L2_STATE(PEND_ON) ++KBASEP_L2_STATE(RESTORE_CLOCKS) ++KBASEP_L2_STATE(ON_HWCNT_ENABLE) ++KBASEP_L2_STATE(ON) ++KBASEP_L2_STATE(ON_HWCNT_DISABLE) ++KBASEP_L2_STATE(SLOW_DOWN_CLOCKS) ++KBASEP_L2_STATE(POWER_DOWN) ++KBASEP_L2_STATE(PEND_OFF) ++KBASEP_L2_STATE(RESET_WAIT) +diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_mcu_states.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_mcu_states.h +new file mode 100755 +index 000000000000..e163bd4f4094 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_mcu_states.h +@@ -0,0 +1,39 @@ ++/* ++ * ++ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* ++ * Backend-specific Power Manager MCU state definitions. ++ * The function-like macro KBASEP_MCU_STATE() must be defined before including ++ * this header file. This header file can be included multiple times in the ++ * same compilation unit with different definitions of KBASEP_MCU_STATE(). ++ */ ++KBASEP_MCU_STATE(OFF) ++KBASEP_MCU_STATE(PEND_ON_RELOAD) ++KBASEP_MCU_STATE(ON_GLB_REINIT_PEND) ++KBASEP_MCU_STATE(ON_HWCNT_ENABLE) ++KBASEP_MCU_STATE(ON) ++KBASEP_MCU_STATE(ON_HWCNT_DISABLE) ++KBASEP_MCU_STATE(ON_HALT) ++KBASEP_MCU_STATE(ON_PEND_HALT) ++KBASEP_MCU_STATE(POWER_DOWN) ++KBASEP_MCU_STATE(PEND_OFF) ++KBASEP_MCU_STATE(RESET_WAIT) +diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_metrics.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_metrics.c +new file mode 100755 +index 000000000000..b714971ba17c +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_metrics.c +@@ -0,0 +1,324 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++/* ++ * Metrics for power management ++ */ ++ ++#include ++#include ++#include ++#if !MALI_USE_CSF ++#include ++#endif /* !MALI_USE_CSF */ ++#include ++#include ++ ++/* When VSync is being hit aim for utilisation between 70-90% */ ++#define KBASE_PM_VSYNC_MIN_UTILISATION 70 ++#define KBASE_PM_VSYNC_MAX_UTILISATION 90 ++/* Otherwise aim for 10-40% */ ++#define KBASE_PM_NO_VSYNC_MIN_UTILISATION 10 ++#define KBASE_PM_NO_VSYNC_MAX_UTILISATION 40 ++ ++/* Shift used for kbasep_pm_metrics_data.time_busy/idle - units of (1 << 8) ns ++ * This gives a maximum period between samples of 2^(32+8)/100 ns = slightly ++ * under 11s. Exceeding this will cause overflow */ ++#define KBASE_PM_TIME_SHIFT 8 ++ ++#ifdef CONFIG_MALI_BIFROST_DVFS ++static enum hrtimer_restart dvfs_callback(struct hrtimer *timer) ++{ ++ unsigned long flags; ++ struct kbasep_pm_metrics_state *metrics; ++ ++ KBASE_DEBUG_ASSERT(timer != NULL); ++ ++ metrics = container_of(timer, struct kbasep_pm_metrics_state, timer); ++ kbase_pm_get_dvfs_action(metrics->kbdev); ++ ++ spin_lock_irqsave(&metrics->lock, flags); ++ ++ if (metrics->timer_active) ++ hrtimer_start(timer, ++ HR_TIMER_DELAY_MSEC(metrics->kbdev->pm.dvfs_period), ++ HRTIMER_MODE_REL); ++ ++ spin_unlock_irqrestore(&metrics->lock, flags); ++ ++ return HRTIMER_NORESTART; ++} ++#endif /* CONFIG_MALI_BIFROST_DVFS */ ++ ++int kbasep_pm_metrics_init(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ kbdev->pm.backend.metrics.kbdev = kbdev; ++ ++ kbdev->pm.backend.metrics.time_period_start = ktime_get(); ++ kbdev->pm.backend.metrics.gpu_active = false; ++ kbdev->pm.backend.metrics.active_cl_ctx[0] = 0; ++ kbdev->pm.backend.metrics.active_cl_ctx[1] = 0; ++ kbdev->pm.backend.metrics.active_gl_ctx[0] = 0; ++ kbdev->pm.backend.metrics.active_gl_ctx[1] = 0; ++ kbdev->pm.backend.metrics.active_gl_ctx[2] = 0; ++ ++ kbdev->pm.backend.metrics.values.time_busy = 0; ++ kbdev->pm.backend.metrics.values.time_idle = 0; ++ kbdev->pm.backend.metrics.values.busy_cl[0] = 0; ++ kbdev->pm.backend.metrics.values.busy_cl[1] = 0; ++ kbdev->pm.backend.metrics.values.busy_gl = 0; ++ ++ spin_lock_init(&kbdev->pm.backend.metrics.lock); ++ ++#ifdef CONFIG_MALI_BIFROST_DVFS ++ hrtimer_init(&kbdev->pm.backend.metrics.timer, CLOCK_MONOTONIC, ++ HRTIMER_MODE_REL); ++ kbdev->pm.backend.metrics.timer.function = dvfs_callback; ++ ++ kbase_pm_metrics_start(kbdev); ++#endif /* CONFIG_MALI_BIFROST_DVFS */ ++ ++ return 0; ++} ++KBASE_EXPORT_TEST_API(kbasep_pm_metrics_init); ++ ++void kbasep_pm_metrics_term(struct kbase_device *kbdev) ++{ ++#ifdef CONFIG_MALI_BIFROST_DVFS ++ unsigned long flags; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); ++ kbdev->pm.backend.metrics.timer_active = false; ++ spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); ++ ++ hrtimer_cancel(&kbdev->pm.backend.metrics.timer); ++#endif /* CONFIG_MALI_BIFROST_DVFS */ ++} ++ ++KBASE_EXPORT_TEST_API(kbasep_pm_metrics_term); ++ ++/* caller needs to hold kbdev->pm.backend.metrics.lock before calling this ++ * function ++ */ ++static void kbase_pm_get_dvfs_utilisation_calc(struct kbase_device *kbdev, ++ ktime_t now) ++{ ++ ktime_t diff; ++ ++ lockdep_assert_held(&kbdev->pm.backend.metrics.lock); ++ ++ diff = ktime_sub(now, kbdev->pm.backend.metrics.time_period_start); ++ if (ktime_to_ns(diff) < 0) ++ return; ++ ++ if (kbdev->pm.backend.metrics.gpu_active) { ++ u32 ns_time = (u32) (ktime_to_ns(diff) >> KBASE_PM_TIME_SHIFT); ++ ++ kbdev->pm.backend.metrics.values.time_busy += ns_time; ++ if (kbdev->pm.backend.metrics.active_cl_ctx[0]) ++ kbdev->pm.backend.metrics.values.busy_cl[0] += ns_time; ++ if (kbdev->pm.backend.metrics.active_cl_ctx[1]) ++ kbdev->pm.backend.metrics.values.busy_cl[1] += ns_time; ++ if (kbdev->pm.backend.metrics.active_gl_ctx[0]) ++ kbdev->pm.backend.metrics.values.busy_gl += ns_time; ++ if (kbdev->pm.backend.metrics.active_gl_ctx[1]) ++ kbdev->pm.backend.metrics.values.busy_gl += ns_time; ++ if (kbdev->pm.backend.metrics.active_gl_ctx[2]) ++ kbdev->pm.backend.metrics.values.busy_gl += ns_time; ++ } else { ++ kbdev->pm.backend.metrics.values.time_idle += (u32) (ktime_to_ns(diff) ++ >> KBASE_PM_TIME_SHIFT); ++ } ++ ++ kbdev->pm.backend.metrics.time_period_start = now; ++} ++ ++#if defined(CONFIG_MALI_BIFROST_DEVFREQ) || defined(CONFIG_MALI_BIFROST_DVFS) ++void kbase_pm_get_dvfs_metrics(struct kbase_device *kbdev, ++ struct kbasep_pm_metrics *last, ++ struct kbasep_pm_metrics *diff) ++{ ++ struct kbasep_pm_metrics *cur = &kbdev->pm.backend.metrics.values; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); ++ kbase_pm_get_dvfs_utilisation_calc(kbdev, ktime_get()); ++ ++ memset(diff, 0, sizeof(*diff)); ++ diff->time_busy = cur->time_busy - last->time_busy; ++ diff->time_idle = cur->time_idle - last->time_idle; ++ diff->busy_cl[0] = cur->busy_cl[0] - last->busy_cl[0]; ++ diff->busy_cl[1] = cur->busy_cl[1] - last->busy_cl[1]; ++ diff->busy_gl = cur->busy_gl - last->busy_gl; ++ ++ *last = *cur; ++ ++ spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); ++} ++KBASE_EXPORT_TEST_API(kbase_pm_get_dvfs_metrics); ++#endif ++ ++#ifdef CONFIG_MALI_BIFROST_DVFS ++void kbase_pm_get_dvfs_action(struct kbase_device *kbdev) ++{ ++ int utilisation, util_gl_share; ++ int util_cl_share[2]; ++ int busy; ++ struct kbasep_pm_metrics *diff; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ diff = &kbdev->pm.backend.metrics.dvfs_diff; ++ ++ kbase_pm_get_dvfs_metrics(kbdev, &kbdev->pm.backend.metrics.dvfs_last, diff); ++ ++ utilisation = (100 * diff->time_busy) / ++ max(diff->time_busy + diff->time_idle, 1u); ++ ++ busy = max(diff->busy_gl + diff->busy_cl[0] + diff->busy_cl[1], 1u); ++ util_gl_share = (100 * diff->busy_gl) / busy; ++ util_cl_share[0] = (100 * diff->busy_cl[0]) / busy; ++ util_cl_share[1] = (100 * diff->busy_cl[1]) / busy; ++ ++ kbase_platform_dvfs_event(kbdev, utilisation, util_gl_share, util_cl_share); ++} ++ ++bool kbase_pm_metrics_is_active(struct kbase_device *kbdev) ++{ ++ bool isactive; ++ unsigned long flags; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); ++ isactive = kbdev->pm.backend.metrics.timer_active; ++ spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); ++ ++ return isactive; ++} ++KBASE_EXPORT_TEST_API(kbase_pm_metrics_is_active); ++ ++void kbase_pm_metrics_start(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); ++ kbdev->pm.backend.metrics.timer_active = true; ++ spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); ++ hrtimer_start(&kbdev->pm.backend.metrics.timer, ++ HR_TIMER_DELAY_MSEC(kbdev->pm.dvfs_period), ++ HRTIMER_MODE_REL); ++} ++ ++void kbase_pm_metrics_stop(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); ++ kbdev->pm.backend.metrics.timer_active = false; ++ spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); ++ hrtimer_cancel(&kbdev->pm.backend.metrics.timer); ++} ++ ++ ++#endif /* CONFIG_MALI_BIFROST_DVFS */ ++ ++#if !MALI_USE_CSF ++/** ++ * kbase_pm_metrics_active_calc - Update PM active counts based on currently ++ * running atoms ++ * @kbdev: Device pointer ++ * ++ * The caller must hold kbdev->pm.backend.metrics.lock ++ */ ++static void kbase_pm_metrics_active_calc(struct kbase_device *kbdev) ++{ ++ int js; ++ ++ lockdep_assert_held(&kbdev->pm.backend.metrics.lock); ++ ++ kbdev->pm.backend.metrics.active_gl_ctx[0] = 0; ++ kbdev->pm.backend.metrics.active_gl_ctx[1] = 0; ++ kbdev->pm.backend.metrics.active_gl_ctx[2] = 0; ++ kbdev->pm.backend.metrics.active_cl_ctx[0] = 0; ++ kbdev->pm.backend.metrics.active_cl_ctx[1] = 0; ++ kbdev->pm.backend.metrics.gpu_active = false; ++ ++ for (js = 0; js < BASE_JM_MAX_NR_SLOTS; js++) { ++ struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, 0); ++ ++ /* Head atom may have just completed, so if it isn't running ++ * then try the next atom */ ++ if (katom && katom->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) ++ katom = kbase_gpu_inspect(kbdev, js, 1); ++ ++ if (katom && katom->gpu_rb_state == ++ KBASE_ATOM_GPU_RB_SUBMITTED) { ++ if (katom->core_req & BASE_JD_REQ_ONLY_COMPUTE) { ++ int device_nr = (katom->core_req & ++ BASE_JD_REQ_SPECIFIC_COHERENT_GROUP) ++ ? katom->device_nr : 0; ++ if (!WARN_ON(device_nr >= 2)) ++ kbdev->pm.backend.metrics. ++ active_cl_ctx[device_nr] = 1; ++ } else { ++ kbdev->pm.backend.metrics.active_gl_ctx[js] = 1; ++ trace_sysgraph(SGR_ACTIVE, 0, js); ++ } ++ kbdev->pm.backend.metrics.gpu_active = true; ++ } else { ++ trace_sysgraph(SGR_INACTIVE, 0, js); ++ } ++ } ++} ++#endif /* !MALI_USE_CSF */ ++ ++/* called when job is submitted to or removed from a GPU slot */ ++void kbase_pm_metrics_update(struct kbase_device *kbdev, ktime_t *timestamp) ++{ ++ unsigned long flags; ++ ktime_t now; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); ++ ++ if (!timestamp) { ++ now = ktime_get(); ++ timestamp = &now; ++ } ++ ++ /* Track how long CL and/or GL jobs have been busy for */ ++ kbase_pm_get_dvfs_utilisation_calc(kbdev, *timestamp); ++ ++#if !MALI_USE_CSF ++ kbase_pm_metrics_active_calc(kbdev); ++#endif /* !MALI_USE_CSF */ ++ ++ spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); ++} +diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_policy.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_policy.c +new file mode 100755 +index 000000000000..48b24b1c866e +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_policy.c +@@ -0,0 +1,268 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* ++ * Power policy API implementations ++ */ ++ ++#include ++#include ++#include ++#include ++ ++static const struct kbase_pm_policy *const all_policy_list[] = { ++#ifdef CONFIG_MALI_BIFROST_NO_MALI ++ &kbase_pm_always_on_policy_ops, ++ &kbase_pm_coarse_demand_policy_ops, ++#if !MALI_CUSTOMER_RELEASE ++ &kbase_pm_always_on_demand_policy_ops, ++#endif ++#else /* CONFIG_MALI_BIFROST_NO_MALI */ ++ &kbase_pm_coarse_demand_policy_ops, ++#if !MALI_CUSTOMER_RELEASE ++ &kbase_pm_always_on_demand_policy_ops, ++#endif ++ &kbase_pm_always_on_policy_ops ++#endif /* CONFIG_MALI_BIFROST_NO_MALI */ ++}; ++ ++void kbase_pm_policy_init(struct kbase_device *kbdev) ++{ ++ kbdev->pm.backend.pm_current_policy = all_policy_list[0]; ++ kbdev->pm.backend.pm_current_policy->init(kbdev); ++} ++ ++void kbase_pm_policy_term(struct kbase_device *kbdev) ++{ ++ kbdev->pm.backend.pm_current_policy->term(kbdev); ++} ++ ++void kbase_pm_update_active(struct kbase_device *kbdev) ++{ ++ struct kbase_pm_device_data *pm = &kbdev->pm; ++ struct kbase_pm_backend_data *backend = &pm->backend; ++ unsigned long flags; ++ bool active; ++ ++ lockdep_assert_held(&pm->lock); ++ ++ /* pm_current_policy will never be NULL while pm.lock is held */ ++ KBASE_DEBUG_ASSERT(backend->pm_current_policy); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ active = backend->pm_current_policy->get_core_active(kbdev); ++ WARN((kbase_pm_is_active(kbdev) && !active), ++ "GPU is active but policy '%s' is indicating that it can be powered off", ++ kbdev->pm.backend.pm_current_policy->name); ++ ++ if (active) { ++ /* Power on the GPU and any cores requested by the policy */ ++ if (!pm->backend.invoke_poweroff_wait_wq_when_l2_off && ++ pm->backend.poweroff_wait_in_progress) { ++ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered); ++ pm->backend.poweron_required = true; ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ } else { ++ /* Cancel the invocation of ++ * kbase_pm_gpu_poweroff_wait_wq() from the L2 state ++ * machine. This is safe - it ++ * invoke_poweroff_wait_wq_when_l2_off is true, then ++ * the poweroff work hasn't even been queued yet, ++ * meaning we can go straight to powering on. ++ */ ++ pm->backend.invoke_poweroff_wait_wq_when_l2_off = false; ++ pm->backend.poweroff_wait_in_progress = false; ++ pm->backend.l2_desired = true; ++#if MALI_USE_CSF ++ pm->backend.mcu_desired = true; ++#endif ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ kbase_pm_do_poweron(kbdev, false); ++ } ++ } else { ++ /* It is an error for the power policy to power off the GPU ++ * when there are contexts active */ ++ KBASE_DEBUG_ASSERT(pm->active_count == 0); ++ ++ pm->backend.poweron_required = false; ++ ++ /* Request power off */ ++ if (pm->backend.gpu_powered) { ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ /* Power off the GPU immediately */ ++ kbase_pm_do_poweroff(kbdev); ++ } else { ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ } ++ } ++} ++ ++void kbase_pm_update_dynamic_cores_onoff(struct kbase_device *kbdev) ++{ ++ bool shaders_desired; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ lockdep_assert_held(&kbdev->pm.lock); ++ ++#if MALI_USE_CSF ++ /* On CSF GPUs, Host driver isn't supposed to do the power management ++ * for shader cores. CSF firmware will power up the cores appropriately ++ * and so from Driver's standpoint 'shaders_desired' flag shall always ++ * remain 0. ++ */ ++ return; ++#endif ++ if (kbdev->pm.backend.pm_current_policy == NULL) ++ return; ++ if (kbdev->pm.backend.poweroff_wait_in_progress) ++ return; ++ /* In protected transition, don't allow outside shader core request ++ * affect transition, return directly ++ */ ++ if (kbdev->pm.backend.protected_transition_override) ++ return; ++ ++ shaders_desired = kbdev->pm.backend.pm_current_policy->shaders_needed(kbdev); ++ ++ if (shaders_desired && kbase_pm_is_l2_desired(kbdev)) { ++ kbase_pm_update_state(kbdev); ++ } ++} ++ ++void kbase_pm_update_cores_state_nolock(struct kbase_device *kbdev) ++{ ++ bool shaders_desired; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (kbdev->pm.backend.pm_current_policy == NULL) ++ return; ++ if (kbdev->pm.backend.poweroff_wait_in_progress) ++ return; ++ ++ if (kbdev->pm.backend.protected_transition_override) ++ /* We are trying to change in/out of protected mode - force all ++ * cores off so that the L2 powers down */ ++ shaders_desired = false; ++ else ++ shaders_desired = kbdev->pm.backend.pm_current_policy->shaders_needed(kbdev); ++ ++#if MALI_USE_CSF ++ /* On CSF GPUs, Host driver isn't supposed to do the power management ++ * for shader cores. CSF firmware will power up the cores appropriately ++ * and so from Driver's standpoint 'shaders_desired' flag shall always ++ * remain 0. ++ */ ++ shaders_desired = false; ++#endif ++ if (kbdev->pm.backend.shaders_desired != shaders_desired) { ++ KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_DESIRED, NULL, kbdev->pm.backend.shaders_desired); ++ ++ kbdev->pm.backend.shaders_desired = shaders_desired; ++ kbase_pm_update_state(kbdev); ++ } ++} ++ ++void kbase_pm_update_cores_state(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ kbase_pm_update_cores_state_nolock(kbdev); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++} ++ ++int kbase_pm_list_policies(struct kbase_device *kbdev, ++ const struct kbase_pm_policy * const **list) ++{ ++ if (list) ++ *list = all_policy_list; ++ ++ return ARRAY_SIZE(all_policy_list); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_list_policies); ++ ++const struct kbase_pm_policy *kbase_pm_get_policy(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ return kbdev->pm.backend.pm_current_policy; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_get_policy); ++ ++void kbase_pm_set_policy(struct kbase_device *kbdev, ++ const struct kbase_pm_policy *new_policy) ++{ ++ const struct kbase_pm_policy *old_policy; ++ unsigned long flags; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(new_policy != NULL); ++ ++ KBASE_KTRACE_ADD(kbdev, PM_SET_POLICY, NULL, new_policy->id); ++ ++ /* During a policy change we pretend the GPU is active */ ++ /* A suspend won't happen here, because we're in a syscall from a ++ * userspace thread */ ++ kbase_pm_context_active(kbdev); ++ ++ kbase_pm_lock(kbdev); ++ ++ /* Remove the policy to prevent IRQ handlers from working on it */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ old_policy = kbdev->pm.backend.pm_current_policy; ++ kbdev->pm.backend.pm_current_policy = NULL; ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ KBASE_KTRACE_ADD(kbdev, PM_CURRENT_POLICY_TERM, NULL, old_policy->id); ++ if (old_policy->term) ++ old_policy->term(kbdev); ++ ++ KBASE_KTRACE_ADD(kbdev, PM_CURRENT_POLICY_INIT, NULL, new_policy->id); ++ if (new_policy->init) ++ new_policy->init(kbdev); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbdev->pm.backend.pm_current_policy = new_policy; ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ /* If any core power state changes were previously attempted, but ++ * couldn't be made because the policy was changing (current_policy was ++ * NULL), then re-try them here. */ ++ kbase_pm_update_active(kbdev); ++ kbase_pm_update_cores_state(kbdev); ++ ++ kbase_pm_unlock(kbdev); ++ ++ /* Now the policy change is finished, we release our fake context active ++ * reference */ ++ kbase_pm_context_idle(kbdev); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_set_policy); +diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_policy.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_policy.h +new file mode 100755 +index 000000000000..f103ef0c01e4 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_policy.h +@@ -0,0 +1,106 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2015, 2018-2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* ++ * Power policy API definitions ++ */ ++ ++#ifndef _KBASE_PM_POLICY_H_ ++#define _KBASE_PM_POLICY_H_ ++ ++/** ++ * kbase_pm_policy_init - Initialize power policy framework ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * Must be called before calling any other policy function ++ */ ++void kbase_pm_policy_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_policy_term - Terminate power policy framework ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_policy_term(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_update_active - Update the active power state of the GPU ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * Calls into the current power policy ++ */ ++void kbase_pm_update_active(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_update_cores - Update the desired core state of the GPU ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * Calls into the current power policy ++ */ ++void kbase_pm_update_cores(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_cores_requested - Check that a power request has been locked into ++ * the HW. ++ * @kbdev: Kbase device ++ * @shader_required: true if shaders are required ++ * ++ * Called by the scheduler to check if a power on request has been locked into ++ * the HW. ++ * ++ * Note that there is no guarantee that the cores are actually ready, however ++ * when the request has been locked into the HW, then it is safe to submit work ++ * since the HW will wait for the transition to ready. ++ * ++ * A reference must first be taken prior to making this call. ++ * ++ * Caller must hold the hwaccess_lock. ++ * ++ * Return: true if the request to the HW was successfully made else false if the ++ * request is still pending. ++ */ ++static inline bool kbase_pm_cores_requested(struct kbase_device *kbdev, ++ bool shader_required) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ /* If the L2 & tiler are not on or pending, then the tiler is not yet ++ * available, and shaders are definitely not powered. ++ */ ++ if (kbdev->pm.backend.l2_state != KBASE_L2_PEND_ON && ++ kbdev->pm.backend.l2_state != KBASE_L2_ON && ++ kbdev->pm.backend.l2_state != KBASE_L2_ON_HWCNT_ENABLE) ++ return false; ++ ++ if (shader_required && ++ kbdev->pm.backend.shaders_state != KBASE_SHADERS_PEND_ON_CORESTACK_ON && ++ kbdev->pm.backend.shaders_state != KBASE_SHADERS_ON_CORESTACK_ON && ++ kbdev->pm.backend.shaders_state != KBASE_SHADERS_ON_CORESTACK_ON_RECHECK) ++ return false; ++ ++ return true; ++} ++ ++#endif /* _KBASE_PM_POLICY_H_ */ +diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_shader_states.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_shader_states.h +new file mode 100755 +index 000000000000..6cafaa171962 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_shader_states.h +@@ -0,0 +1,45 @@ ++/* ++ * ++ * (C) COPYRIGHT 2018-2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* ++ * Backend-specific Power Manager shader core state definitions. ++ * The function-like macro KBASEP_SHADER_STATE() must be defined before ++ * including this header file. This header file can be included multiple ++ * times in the same compilation unit with different definitions of ++ * KBASEP_SHADER_STATE(). ++ */ ++KBASEP_SHADER_STATE(OFF_CORESTACK_OFF) ++KBASEP_SHADER_STATE(OFF_CORESTACK_PEND_ON) ++KBASEP_SHADER_STATE(PEND_ON_CORESTACK_ON) ++KBASEP_SHADER_STATE(ON_CORESTACK_ON) ++KBASEP_SHADER_STATE(ON_CORESTACK_ON_RECHECK) ++KBASEP_SHADER_STATE(WAIT_OFF_CORESTACK_ON) ++#if !MALI_USE_CSF ++KBASEP_SHADER_STATE(WAIT_GPU_IDLE) ++#endif /* !MALI_USE_CSF */ ++KBASEP_SHADER_STATE(WAIT_FINISHED_CORESTACK_ON) ++KBASEP_SHADER_STATE(L2_FLUSHING_CORESTACK_ON) ++KBASEP_SHADER_STATE(READY_OFF_CORESTACK_ON) ++KBASEP_SHADER_STATE(PEND_OFF_CORESTACK_ON) ++KBASEP_SHADER_STATE(OFF_CORESTACK_PEND_OFF) ++KBASEP_SHADER_STATE(OFF_CORESTACK_OFF_TIMER_PEND_OFF) ++KBASEP_SHADER_STATE(RESET_WAIT) +diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_time.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_time.c +new file mode 100755 +index 000000000000..e19f53b2cbe8 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_time.c +@@ -0,0 +1,81 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2016,2018-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++#include ++#include ++#include ++ ++void kbase_backend_get_gpu_time_norequest(struct kbase_device *kbdev, ++ u64 *cycle_counter, ++ u64 *system_time, ++ struct timespec64 *ts) ++{ ++ u32 hi1, hi2; ++ ++ if (cycle_counter) { ++ /* Read hi, lo, hi to ensure a coherent u64 */ ++ do { ++ hi1 = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(CYCLE_COUNT_HI)); ++ *cycle_counter = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(CYCLE_COUNT_LO)); ++ hi2 = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(CYCLE_COUNT_HI)); ++ } while (hi1 != hi2); ++ *cycle_counter |= (((u64) hi1) << 32); ++ } ++ ++ if (system_time) { ++ /* Read hi, lo, hi to ensure a coherent u64 */ ++ do { ++ hi1 = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(TIMESTAMP_HI)); ++ *system_time = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(TIMESTAMP_LO)); ++ hi2 = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(TIMESTAMP_HI)); ++ } while (hi1 != hi2); ++ *system_time |= (((u64) hi1) << 32); ++ } ++ ++ /* Record the CPU's idea of current time */ ++ if (ts != NULL) ++#if (KERNEL_VERSION(4, 17, 0) > LINUX_VERSION_CODE) ++ *ts = ktime_to_timespec64(ktime_get_raw()); ++#else ++ ktime_get_raw_ts64(ts); ++#endif ++} ++ ++void kbase_backend_get_gpu_time(struct kbase_device *kbdev, u64 *cycle_counter, ++ u64 *system_time, struct timespec64 *ts) ++{ ++#if !MALI_USE_CSF ++ kbase_pm_request_gpu_cycle_counter(kbdev); ++#endif ++ kbase_backend_get_gpu_time_norequest( ++ kbdev, cycle_counter, system_time, ts); ++#if !MALI_USE_CSF ++ kbase_pm_release_gpu_cycle_counter(kbdev); ++#endif ++} +diff --git a/drivers/gpu/arm/bifrost/build.bp b/drivers/gpu/arm/bifrost/build.bp +new file mode 100755 +index 000000000000..b9b86184f3be +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/build.bp +@@ -0,0 +1,186 @@ ++/* ++ * ++ * (C) COPYRIGHT 2017-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++/* Kernel-side tests may include mali_kbase's headers. Therefore any config ++ * options which affect the sizes of any structs (e.g. adding extra members) ++ * must be included in these defaults, so that the structs are consistent in ++ * both mali_kbase and the test modules. */ ++bob_defaults { ++ name: "mali_kbase_shared_config_defaults", ++ no_mali: { ++ kbuild_options: ["CONFIG_MALI_BIFROST_NO_MALI=y"], ++ }, ++ mali_real_hw: { ++ kbuild_options: ["CONFIG_MALI_REAL_HW=y"], ++ }, ++ mali_dma_fence: { ++ kbuild_options: ["CONFIG_MALI_BIFROST_DMA_FENCE=y"], ++ }, ++ mali_devfreq: { ++ kbuild_options: ["CONFIG_MALI_BIFROST_DEVFREQ=y"], ++ }, ++ mali_midgard_dvfs: { ++ kbuild_options: ["CONFIG_MALI_BIFROST_DVFS=y"], ++ }, ++ mali_debug: { ++ kbuild_options: ["CONFIG_MALI_BIFROST_DEBUG=y"], ++ }, ++ buslog: { ++ kbuild_options: ["CONFIG_MALI_BUSLOG=y"], ++ }, ++ cinstr_vector_dump: { ++ kbuild_options: ["CONFIG_MALI_VECTOR_DUMP=y"], ++ }, ++ cinstr_gwt: { ++ kbuild_options: ["CONFIG_MALI_CINSTR_GWT=y"], ++ }, ++ mali_gator_support: { ++ kbuild_options: ["CONFIG_MALI_BIFROST_GATOR_SUPPORT=y"], ++ }, ++ mali_midgard_enable_trace: { ++ kbuild_options: ["CONFIG_MALI_BIFROST_ENABLE_TRACE=y"], ++ }, ++ mali_system_trace: { ++ kbuild_options: ["CONFIG_MALI_BIFROST_SYSTEM_TRACE=y"], ++ }, ++ mali_pwrsoft_765: { ++ kbuild_options: ["CONFIG_MALI_PWRSOFT_765=y"], ++ }, ++ mali_memory_fully_backed: { ++ kbuild_options: ["CONFIG_MALI_MEMORY_FULLY_BACKED=y"], ++ }, ++ mali_dma_buf_map_on_demand: { ++ kbuild_options: ["CONFIG_MALI_DMA_BUF_MAP_ON_DEMAND=y"], ++ }, ++ mali_dma_buf_legacy_compat: { ++ kbuild_options: ["CONFIG_MALI_DMA_BUF_LEGACY_COMPAT=y"], ++ }, ++ mali_arbiter_support: { ++ kbuild_options: ["CONFIG_MALI_ARBITER_SUPPORT=y"], ++ }, ++ mali_gem5_build: { ++ kbuild_options: ["CONFIG_MALI_GEM5_BUILD=y"], ++ }, ++ kbuild_options: [ ++ "MALI_UNIT_TEST={{.unit_test_code}}", ++ "MALI_CUSTOMER_RELEASE={{.release}}", ++ "MALI_USE_CSF={{.gpu_has_csf}}", ++ "MALI_KERNEL_TEST_API={{.debug}}", ++ ], ++ defaults: ["kernel_defaults"], ++} ++ ++bob_kernel_module { ++ name: "mali_kbase", ++ srcs: [ ++ "*.c", ++ "*.h", ++ "Kbuild", ++ "backend/gpu/*.c", ++ "backend/gpu/*.h", ++ "backend/gpu/Kbuild", ++ "context/*.c", ++ "context/*.h", ++ "ipa/*.c", ++ "ipa/*.h", ++ "ipa/Kbuild", ++ "platform/*.h", ++ "platform/*/*.c", ++ "platform/*/*.h", ++ "platform/*/Kbuild", ++ "thirdparty/*.c", ++ "debug/*.c", ++ "debug/*.h", ++ "device/*.c", ++ "device/*.h", ++ "gpu/*.c", ++ "gpu/*.h", ++ "tl/*.c", ++ "tl/*.h", ++ "mmu/*.c", ++ "mmu/*.h", ++ ], ++ kbuild_options: [ ++ "CONFIG_MALI_KUTF=n", ++ "CONFIG_MALI_MIDGARD=m", ++ "CONFIG_MALI_NO_MALI_DEFAULT_GPU={{.gpu}}", ++ "CONFIG_MALI_PLATFORM_NAME={{.mali_platform_name}}", ++ ], ++ buslog: { ++ extra_symbols: [ ++ "bus_logger", ++ ], ++ }, ++ mali_corestack: { ++ kbuild_options: ["CONFIG_MALI_CORESTACK=y"], ++ }, ++ mali_error_inject: { ++ kbuild_options: ["CONFIG_MALI_BIFROST_ERROR_INJECT=y"], ++ }, ++ mali_error_inject_random: { ++ kbuild_options: ["CONFIG_MALI_ERROR_INJECT_RANDOM=y"], ++ }, ++ cinstr_secondary_hwc: { ++ kbuild_options: ["CONFIG_MALI_BIFROST_PRFCNT_SET_SECONDARY=y"], ++ }, ++ cinstr_secondary_hwc_via_debug_fs: { ++ kbuild_options: ["CONFIG_MALI_PRFCNT_SET_SECONDARY_VIA_DEBUG_FS=y"], ++ }, ++ mali_2mb_alloc: { ++ kbuild_options: ["CONFIG_MALI_2MB_ALLOC=y"], ++ }, ++ mali_hw_errata_1485982_not_affected: { ++ kbuild_options: ["CONFIG_MALI_HW_ERRATA_1485982_NOT_AFFECTED=y"], ++ }, ++ mali_hw_errata_1485982_use_clock_alternative: { ++ kbuild_options: ["CONFIG_MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE=y"], ++ }, ++ gpu_has_job_manager: { ++ srcs: [ ++ "context/backend/*_jm.c", ++ "debug/backend/*_jm.c", ++ "debug/backend/*_jm.h", ++ "device/backend/*_jm.c", ++ "gpu/backend/*_jm.c", ++ "gpu/backend/*_jm.h", ++ "jm/*.h", ++ "tl/backend/*_jm.c", ++ "mmu/backend/*_jm.c", ++ ], ++ }, ++ gpu_has_csf: { ++ srcs: [ ++ "context/backend/*_csf.c", ++ "csf/*.c", ++ "csf/*.h", ++ "csf/Kbuild", ++ "debug/backend/*_csf.c", ++ "debug/backend/*_csf.h", ++ "device/backend/*_csf.c", ++ "gpu/backend/*_csf.c", ++ "gpu/backend/*_csf.h", ++ "tl/backend/*_csf.c", ++ "mmu/backend/*_csf.c", ++ ], ++ }, ++ mali_arbiter_support: { ++ srcs: [ ++ "arbiter/*.c", ++ "arbiter/*.h", ++ "arbiter/Kbuild", ++ ], ++ }, ++ defaults: ["mali_kbase_shared_config_defaults"], ++} +diff --git a/drivers/gpu/arm/bifrost/context/backend/mali_kbase_context_csf.c b/drivers/gpu/arm/bifrost/context/backend/mali_kbase_context_csf.c +new file mode 100755 +index 000000000000..7c68eb2f860a +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/context/backend/mali_kbase_context_csf.c +@@ -0,0 +1,177 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * ++ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* ++ * Base kernel context APIs for CSF GPUs ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#ifdef CONFIG_DEBUG_FS ++#include ++#include ++#include ++#include ++#include ++ ++void kbase_context_debugfs_init(struct kbase_context *const kctx) ++{ ++ kbase_debug_mem_view_init(kctx); ++ kbase_mem_pool_debugfs_init(kctx->kctx_dentry, kctx); ++ kbase_jit_debugfs_init(kctx); ++ kbase_csf_queue_group_debugfs_init(kctx); ++ kbase_csf_kcpu_debugfs_init(kctx); ++ kbase_csf_tiler_heap_debugfs_init(kctx); ++} ++KBASE_EXPORT_SYMBOL(kbase_context_debugfs_init); ++ ++void kbase_context_debugfs_term(struct kbase_context *const kctx) ++{ ++ debugfs_remove_recursive(kctx->kctx_dentry); ++} ++KBASE_EXPORT_SYMBOL(kbase_context_debugfs_term); ++#else ++void kbase_context_debugfs_init(struct kbase_context *const kctx) ++{ ++ CSTD_UNUSED(kctx); ++} ++KBASE_EXPORT_SYMBOL(kbase_context_debugfs_init); ++ ++void kbase_context_debugfs_term(struct kbase_context *const kctx) ++{ ++ CSTD_UNUSED(kctx); ++} ++KBASE_EXPORT_SYMBOL(kbase_context_debugfs_term); ++#endif /* CONFIG_DEBUG_FS */ ++ ++static const struct kbase_context_init context_init[] = { ++ {kbase_context_common_init, kbase_context_common_term, NULL}, ++ {kbase_context_mem_pool_group_init, kbase_context_mem_pool_group_term, ++ "Memory pool goup initialization failed"}, ++ {kbase_mem_evictable_init, kbase_mem_evictable_deinit, ++ "Memory evictable initialization failed"}, ++ {kbase_context_mmu_init, kbase_context_mmu_term, ++ "MMU initialization failed"}, ++ {kbase_context_mem_alloc_page, kbase_context_mem_pool_free, ++ "Memory alloc page failed"}, ++ {kbase_region_tracker_init, kbase_region_tracker_term, ++ "Region tracker initialization failed"}, ++ {kbase_sticky_resource_init, kbase_context_sticky_resource_term, ++ "Sticky resource initialization failed"}, ++ {kbase_jit_init, kbase_jit_term, ++ "JIT initialization failed"}, ++ {kbase_csf_ctx_init, kbase_csf_ctx_term, ++ "CSF context initialization failed"}, ++}; ++ ++static void kbase_context_term_partial( ++ struct kbase_context *kctx, ++ unsigned int i) ++{ ++ while (i-- > 0) { ++ if (context_init[i].term) ++ context_init[i].term(kctx); ++ } ++} ++ ++struct kbase_context *kbase_create_context(struct kbase_device *kbdev, ++ bool is_compat, ++ base_context_create_flags const flags, ++ unsigned long const api_version, ++ struct file *const filp) ++{ ++ struct kbase_context *kctx; ++ unsigned int i = 0; ++ ++ if (WARN_ON(!kbdev)) ++ return NULL; ++ ++ /* Validate flags */ ++ if (WARN_ON(flags != (flags & BASEP_CONTEXT_CREATE_KERNEL_FLAGS))) ++ return NULL; ++ ++ /* zero-inited as lot of code assume it's zero'ed out on create */ ++ kctx = vzalloc(sizeof(*kctx)); ++ if (WARN_ON(!kctx)) ++ return NULL; ++ ++ kctx->kbdev = kbdev; ++ kctx->api_version = api_version; ++ kctx->filp = filp; ++ kctx->create_flags = flags; ++ ++ if (is_compat) ++ kbase_ctx_flag_set(kctx, KCTX_COMPAT); ++#if defined(CONFIG_64BIT) ++ else ++ kbase_ctx_flag_set(kctx, KCTX_FORCE_SAME_VA); ++#endif /* !defined(CONFIG_64BIT) */ ++ ++ for (i = 0; i < ARRAY_SIZE(context_init); i++) { ++ int err = context_init[i].init(kctx); ++ ++ if (err) { ++ dev_err(kbdev->dev, "%s error = %d\n", ++ context_init[i].err_mes, err); ++ kbase_context_term_partial(kctx, i); ++ return NULL; ++ } ++ } ++ ++ return kctx; ++} ++KBASE_EXPORT_SYMBOL(kbase_create_context); ++ ++void kbase_destroy_context(struct kbase_context *kctx) ++{ ++ struct kbase_device *kbdev; ++ ++ if (WARN_ON(!kctx)) ++ return; ++ ++ kbdev = kctx->kbdev; ++ if (WARN_ON(!kbdev)) ++ return; ++ ++ /* Ensure the core is powered up for the destroy process ++ * A suspend won't happen here, because we're in a syscall ++ * from a userspace thread. ++ */ ++ kbase_pm_context_active(kbdev); ++ ++ kbase_mem_pool_group_mark_dying(&kctx->mem_pools); ++ ++ kbase_context_term_partial(kctx, ARRAY_SIZE(context_init)); ++ ++ kbase_pm_context_idle(kbdev); ++} ++KBASE_EXPORT_SYMBOL(kbase_destroy_context); +diff --git a/drivers/gpu/arm/bifrost/context/backend/mali_kbase_context_jm.c b/drivers/gpu/arm/bifrost/context/backend/mali_kbase_context_jm.c +new file mode 100755 +index 000000000000..0eb42589fe46 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/context/backend/mali_kbase_context_jm.c +@@ -0,0 +1,230 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * ++ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* ++ * Base kernel context APIs for Job Manager GPUs ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#ifdef CONFIG_DEBUG_FS ++#include ++#include ++ ++void kbase_context_debugfs_init(struct kbase_context *const kctx) ++{ ++ kbase_debug_mem_view_init(kctx); ++ kbase_mem_pool_debugfs_init(kctx->kctx_dentry, kctx); ++ kbase_jit_debugfs_init(kctx); ++ kbasep_jd_debugfs_ctx_init(kctx); ++ kbase_debug_job_fault_context_init(kctx); ++} ++KBASE_EXPORT_SYMBOL(kbase_context_debugfs_init); ++ ++void kbase_context_debugfs_term(struct kbase_context *const kctx) ++{ ++ debugfs_remove_recursive(kctx->kctx_dentry); ++ kbase_debug_job_fault_context_term(kctx); ++} ++KBASE_EXPORT_SYMBOL(kbase_context_debugfs_term); ++#else ++void kbase_context_debugfs_init(struct kbase_context *const kctx) ++{ ++ CSTD_UNUSED(kctx); ++} ++KBASE_EXPORT_SYMBOL(kbase_context_debugfs_init); ++ ++void kbase_context_debugfs_term(struct kbase_context *const kctx) ++{ ++ CSTD_UNUSED(kctx); ++} ++KBASE_EXPORT_SYMBOL(kbase_context_debugfs_term); ++#endif /* CONFIG_DEBUG_FS */ ++ ++static int kbase_context_kbase_kinstr_jm_init(struct kbase_context *kctx) ++{ ++ int ret = kbase_kinstr_jm_init(&kctx->kinstr_jm); ++ ++ if (!ret) ++ return ret; ++ ++ return 0; ++} ++ ++static void kbase_context_kbase_kinstr_jm_term(struct kbase_context *kctx) ++{ ++ kbase_kinstr_jm_term(kctx->kinstr_jm); ++} ++ ++static int kbase_context_kbase_timer_setup(struct kbase_context *kctx) ++{ ++ kbase_timer_setup(&kctx->soft_job_timeout, ++ kbasep_soft_job_timeout_worker); ++ ++ return 0; ++} ++ ++static int kbase_context_submit_check(struct kbase_context *kctx) ++{ ++ struct kbasep_js_kctx_info *js_kctx_info = &kctx->jctx.sched_info; ++ unsigned long irq_flags = 0; ++ ++ base_context_create_flags const flags = kctx->create_flags; ++ ++ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); ++ spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, irq_flags); ++ ++ /* Translate the flags */ ++ if ((flags & BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED) == 0) ++ kbase_ctx_flag_clear(kctx, KCTX_SUBMIT_DISABLED); ++ ++ spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, irq_flags); ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ ++ return 0; ++} ++ ++static const struct kbase_context_init context_init[] = { ++ { kbase_context_common_init, kbase_context_common_term, NULL }, ++ { kbase_dma_fence_init, kbase_dma_fence_term, ++ "DMA fence initialization failed" }, ++ { kbase_context_mem_pool_group_init, kbase_context_mem_pool_group_term, ++ "Memory pool goup initialization failed" }, ++ { kbase_mem_evictable_init, kbase_mem_evictable_deinit, ++ "Memory evictable initialization failed" }, ++ { kbase_context_mmu_init, kbase_context_mmu_term, ++ "MMU initialization failed" }, ++ { kbase_context_mem_alloc_page, kbase_context_mem_pool_free, ++ "Memory alloc page failed" }, ++ { kbase_region_tracker_init, kbase_region_tracker_term, ++ "Region tracker initialization failed" }, ++ { kbase_sticky_resource_init, kbase_context_sticky_resource_term, ++ "Sticky resource initialization failed" }, ++ { kbase_jit_init, kbase_jit_term, "JIT initialization failed" }, ++ { kbase_context_kbase_kinstr_jm_init, ++ kbase_context_kbase_kinstr_jm_term, ++ "JM instrumentation initialization failed" }, ++ { kbase_context_kbase_timer_setup, NULL, NULL }, ++ { kbase_event_init, kbase_event_cleanup, ++ "Event initialization failed" }, ++ { kbasep_js_kctx_init, kbasep_js_kctx_term, ++ "JS kctx initialization failed" }, ++ { kbase_jd_init, kbase_jd_exit, "JD initialization failed" }, ++ { kbase_context_submit_check, NULL, NULL }, ++}; ++ ++static void kbase_context_term_partial( ++ struct kbase_context *kctx, ++ unsigned int i) ++{ ++ while (i-- > 0) { ++ if (context_init[i].term) ++ context_init[i].term(kctx); ++ } ++} ++ ++struct kbase_context *kbase_create_context(struct kbase_device *kbdev, ++ bool is_compat, ++ base_context_create_flags const flags, ++ unsigned long const api_version, ++ struct file *const filp) ++{ ++ struct kbase_context *kctx; ++ unsigned int i = 0; ++ ++ if (WARN_ON(!kbdev)) ++ return NULL; ++ ++ /* Validate flags */ ++ if (WARN_ON(flags != (flags & BASEP_CONTEXT_CREATE_KERNEL_FLAGS))) ++ return NULL; ++ ++ /* zero-inited as lot of code assume it's zero'ed out on create */ ++ kctx = vzalloc(sizeof(*kctx)); ++ if (WARN_ON(!kctx)) ++ return NULL; ++ ++ kctx->kbdev = kbdev; ++ kctx->api_version = api_version; ++ kctx->filp = filp; ++ kctx->create_flags = flags; ++ ++ if (is_compat) ++ kbase_ctx_flag_set(kctx, KCTX_COMPAT); ++#if defined(CONFIG_64BIT) ++ else ++ kbase_ctx_flag_set(kctx, KCTX_FORCE_SAME_VA); ++#endif /* !defined(CONFIG_64BIT) */ ++ ++ for (i = 0; i < ARRAY_SIZE(context_init); i++) { ++ int err = context_init[i].init(kctx); ++ ++ if (err) { ++ dev_err(kbdev->dev, "%s error = %d\n", ++ context_init[i].err_mes, err); ++ kbase_context_term_partial(kctx, i); ++ return NULL; ++ } ++ } ++ ++ return kctx; ++} ++KBASE_EXPORT_SYMBOL(kbase_create_context); ++ ++void kbase_destroy_context(struct kbase_context *kctx) ++{ ++ struct kbase_device *kbdev; ++ ++ if (WARN_ON(!kctx)) ++ return; ++ ++ kbdev = kctx->kbdev; ++ if (WARN_ON(!kbdev)) ++ return; ++ ++ /* Ensure the core is powered up for the destroy process ++ * A suspend won't happen here, because we're in a syscall ++ * from a userspace thread. ++ */ ++ kbase_pm_context_active(kbdev); ++ ++ kbase_mem_pool_group_mark_dying(&kctx->mem_pools); ++ ++ kbase_jd_zap_context(kctx); ++ flush_workqueue(kctx->jctx.job_done_wq); ++ ++ kbase_context_term_partial(kctx, ARRAY_SIZE(context_init)); ++ ++ kbase_pm_context_idle(kbdev); ++} ++KBASE_EXPORT_SYMBOL(kbase_destroy_context); +diff --git a/drivers/gpu/arm/bifrost/context/mali_kbase_context.c b/drivers/gpu/arm/bifrost/context/mali_kbase_context.c +new file mode 100755 +index 000000000000..83182f983467 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/context/mali_kbase_context.c +@@ -0,0 +1,339 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * ++ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* ++ * Base kernel context APIs ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++/** ++ * find_process_node - Used to traverse the process rb_tree to find if ++ * process exists already in process rb_tree. ++ * ++ * @node: Pointer to root node to start search. ++ * @tgid: Thread group PID to search for. ++ * ++ * Return: Pointer to kbase_process if exists otherwise NULL. ++ */ ++static struct kbase_process *find_process_node(struct rb_node *node, pid_t tgid) ++{ ++ struct kbase_process *kprcs = NULL; ++ ++ /* Check if the kctx creation request is from a existing process.*/ ++ while (node) { ++ struct kbase_process *prcs_node = ++ rb_entry(node, struct kbase_process, kprcs_node); ++ if (prcs_node->tgid == tgid) { ++ kprcs = prcs_node; ++ break; ++ } ++ ++ if (tgid < prcs_node->tgid) ++ node = node->rb_left; ++ else ++ node = node->rb_right; ++ } ++ ++ return kprcs; ++} ++ ++/** ++ * kbase_insert_kctx_to_process - Initialise kbase process context. ++ * ++ * @kctx: Pointer to kbase context. ++ * ++ * Here we initialise per process rb_tree managed by kbase_device. ++ * We maintain a rb_tree of each unique process that gets created. ++ * and Each process maintains a list of kbase context. ++ * This setup is currently used by kernel trace functionality ++ * to trace and visualise gpu memory consumption. ++ * ++ * Return: 0 on success and error number on failure. ++ */ ++static int kbase_insert_kctx_to_process(struct kbase_context *kctx) ++{ ++ struct rb_root *const prcs_root = &kctx->kbdev->process_root; ++ const pid_t tgid = kctx->tgid; ++ struct kbase_process *kprcs = NULL; ++ ++ lockdep_assert_held(&kctx->kbdev->kctx_list_lock); ++ ++ kprcs = find_process_node(prcs_root->rb_node, tgid); ++ ++ /* if the kctx is from new process then create a new kbase_process ++ * and add it to the &kbase_device->rb_tree ++ */ ++ if (!kprcs) { ++ struct rb_node **new = &prcs_root->rb_node, *parent = NULL; ++ ++ kprcs = kzalloc(sizeof(*kprcs), GFP_KERNEL); ++ if (kprcs == NULL) ++ return -ENOMEM; ++ kprcs->tgid = tgid; ++ INIT_LIST_HEAD(&kprcs->kctx_list); ++ kprcs->dma_buf_root = RB_ROOT; ++ kprcs->total_gpu_pages = 0; ++ ++ while (*new) { ++ struct kbase_process *prcs_node; ++ ++ parent = *new; ++ prcs_node = rb_entry(parent, struct kbase_process, ++ kprcs_node); ++ if (tgid < prcs_node->tgid) ++ new = &(*new)->rb_left; ++ else ++ new = &(*new)->rb_right; ++ } ++ rb_link_node(&kprcs->kprcs_node, parent, new); ++ rb_insert_color(&kprcs->kprcs_node, prcs_root); ++ } ++ ++ kctx->kprcs = kprcs; ++ list_add(&kctx->kprcs_link, &kprcs->kctx_list); ++ ++ return 0; ++} ++ ++int kbase_context_common_init(struct kbase_context *kctx) ++{ ++ const unsigned long cookies_mask = KBASE_COOKIE_MASK; ++ int err = 0; ++ ++ /* creating a context is considered a disjoint event */ ++ kbase_disjoint_event(kctx->kbdev); ++ ++ kctx->as_nr = KBASEP_AS_NR_INVALID; ++ ++ atomic_set(&kctx->refcount, 0); ++ ++ spin_lock_init(&kctx->mm_update_lock); ++ kctx->process_mm = NULL; ++ atomic_set(&kctx->nonmapped_pages, 0); ++ atomic_set(&kctx->permanent_mapped_pages, 0); ++ kctx->tgid = current->tgid; ++ kctx->pid = current->pid; ++ ++ atomic_set(&kctx->used_pages, 0); ++ ++ mutex_init(&kctx->reg_lock); ++ ++ spin_lock_init(&kctx->mem_partials_lock); ++ INIT_LIST_HEAD(&kctx->mem_partials); ++ ++ spin_lock_init(&kctx->waiting_soft_jobs_lock); ++ INIT_LIST_HEAD(&kctx->waiting_soft_jobs); ++ ++ init_waitqueue_head(&kctx->event_queue); ++ atomic_set(&kctx->event_count, 0); ++#if !MALI_USE_CSF ++ atomic_set(&kctx->event_closed, false); ++#ifdef CONFIG_GPU_TRACEPOINTS ++ atomic_set(&kctx->jctx.work_id, 0); ++#endif ++#endif ++ ++ bitmap_copy(kctx->cookies, &cookies_mask, BITS_PER_LONG); ++ ++ kctx->id = atomic_add_return(1, &(kctx->kbdev->ctx_num)) - 1; ++ ++ mutex_init(&kctx->legacy_hwcnt_lock); ++ ++ mutex_lock(&kctx->kbdev->kctx_list_lock); ++ list_add(&kctx->kctx_list_link, &kctx->kbdev->kctx_list); ++ ++ err = kbase_insert_kctx_to_process(kctx); ++ if (err) ++ dev_err(kctx->kbdev->dev, ++ "(err:%d) failed to insert kctx to kbase_process\n", err); ++ ++ KBASE_TLSTREAM_TL_KBASE_NEW_CTX(kctx->kbdev, kctx->id, ++ kctx->kbdev->gpu_props.props.raw_props.gpu_id); ++ KBASE_TLSTREAM_TL_NEW_CTX(kctx->kbdev, kctx, kctx->id, ++ (u32)(kctx->tgid)); ++ mutex_unlock(&kctx->kbdev->kctx_list_lock); ++ ++ return err; ++} ++ ++/** ++ * kbase_remove_kctx_from_process - remove a terminating context from ++ * the process list. ++ * ++ * @kctx: Pointer to kbase context. ++ * ++ * Remove the tracking of context from the list of contexts maintained under ++ * kbase process and if the list if empty then there no outstanding contexts ++ * we can remove the process node as well. ++ */ ++ ++static void kbase_remove_kctx_from_process(struct kbase_context *kctx) ++{ ++ struct kbase_process *kprcs = kctx->kprcs; ++ ++ lockdep_assert_held(&kctx->kbdev->kctx_list_lock); ++ list_del(&kctx->kprcs_link); ++ ++ /* if there are no outstanding contexts in current process node, ++ * we can remove it from the process rb_tree. ++ */ ++ if (list_empty(&kprcs->kctx_list)) { ++ rb_erase(&kprcs->kprcs_node, &kctx->kbdev->process_root); ++ /* Add checks, so that the terminating process Should not ++ * hold any gpu_memory. ++ */ ++ WARN_ON(kprcs->total_gpu_pages); ++ WARN_ON(!RB_EMPTY_ROOT(&kprcs->dma_buf_root)); ++ kfree(kprcs); ++ } ++} ++ ++void kbase_context_common_term(struct kbase_context *kctx) ++{ ++ unsigned long flags; ++ int pages; ++ ++ mutex_lock(&kctx->kbdev->mmu_hw_mutex); ++ spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, flags); ++ kbase_ctx_sched_remove_ctx(kctx); ++ spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, flags); ++ mutex_unlock(&kctx->kbdev->mmu_hw_mutex); ++ ++ pages = atomic_read(&kctx->used_pages); ++ if (pages != 0) ++ dev_warn(kctx->kbdev->dev, ++ "%s: %d pages in use!\n", __func__, pages); ++ ++ WARN_ON(atomic_read(&kctx->nonmapped_pages) != 0); ++ ++ mutex_lock(&kctx->kbdev->kctx_list_lock); ++ kbase_remove_kctx_from_process(kctx); ++ ++ KBASE_TLSTREAM_TL_KBASE_DEL_CTX(kctx->kbdev, kctx->id); ++ ++ KBASE_TLSTREAM_TL_DEL_CTX(kctx->kbdev, kctx); ++ list_del(&kctx->kctx_list_link); ++ mutex_unlock(&kctx->kbdev->kctx_list_lock); ++ ++ KBASE_KTRACE_ADD(kctx->kbdev, CORE_CTX_DESTROY, kctx, 0u); ++ ++ /* Flush the timeline stream, so the user can see the termination ++ * tracepoints being fired. ++ * The "if" statement below is for optimization. It is safe to call ++ * kbase_timeline_streams_flush when timeline is disabled. ++ */ ++ if (atomic_read(&kctx->kbdev->timeline_flags) != 0) ++ kbase_timeline_streams_flush(kctx->kbdev->timeline); ++ ++ vfree(kctx); ++} ++ ++int kbase_context_mem_pool_group_init(struct kbase_context *kctx) ++{ ++ return kbase_mem_pool_group_init(&kctx->mem_pools, ++ kctx->kbdev, ++ &kctx->kbdev->mem_pool_defaults, ++ &kctx->kbdev->mem_pools); ++} ++ ++void kbase_context_mem_pool_group_term(struct kbase_context *kctx) ++{ ++ kbase_mem_pool_group_term(&kctx->mem_pools); ++} ++ ++int kbase_context_mmu_init(struct kbase_context *kctx) ++{ ++ kbase_mmu_init(kctx->kbdev, ++ &kctx->mmu, kctx, ++ base_context_mmu_group_id_get(kctx->create_flags)); ++ ++ return 0; ++} ++ ++void kbase_context_mmu_term(struct kbase_context *kctx) ++{ ++ kbase_mmu_term(kctx->kbdev, &kctx->mmu); ++} ++ ++int kbase_context_mem_alloc_page(struct kbase_context *kctx) ++{ ++ struct page *p; ++ ++ p = kbase_mem_alloc_page(&kctx->mem_pools.small[KBASE_MEM_GROUP_SINK]); ++ if (!p) ++ return -ENOMEM; ++ ++ kctx->aliasing_sink_page = as_tagged(page_to_phys(p)); ++ ++ return 0; ++} ++ ++void kbase_context_mem_pool_free(struct kbase_context *kctx) ++{ ++ /* drop the aliasing sink page now that it can't be mapped anymore */ ++ kbase_mem_pool_free( ++ &kctx->mem_pools.small[KBASE_MEM_GROUP_SINK], ++ as_page(kctx->aliasing_sink_page), ++ false); ++} ++ ++void kbase_context_sticky_resource_term(struct kbase_context *kctx) ++{ ++ unsigned long pending_regions_to_clean; ++ ++ kbase_gpu_vm_lock(kctx); ++ kbase_sticky_resource_term(kctx); ++ ++ /* free pending region setups */ ++ pending_regions_to_clean = KBASE_COOKIE_MASK; ++ bitmap_andnot(&pending_regions_to_clean, &pending_regions_to_clean, ++ kctx->cookies, BITS_PER_LONG); ++ while (pending_regions_to_clean) { ++ unsigned int cookie = find_first_bit(&pending_regions_to_clean, ++ BITS_PER_LONG); ++ ++ if (!WARN_ON(!kctx->pending_regions[cookie])) { ++ dev_dbg(kctx->kbdev->dev, "Freeing pending unmapped region\n"); ++ kbase_mem_phy_alloc_put( ++ kctx->pending_regions[cookie]->cpu_alloc); ++ kbase_mem_phy_alloc_put( ++ kctx->pending_regions[cookie]->gpu_alloc); ++ kfree(kctx->pending_regions[cookie]); ++ ++ kctx->pending_regions[cookie] = NULL; ++ } ++ ++ bitmap_clear(&pending_regions_to_clean, cookie, 1); ++ } ++ kbase_gpu_vm_unlock(kctx); ++} +diff --git a/drivers/gpu/arm/bifrost/context/mali_kbase_context.h b/drivers/gpu/arm/bifrost/context/mali_kbase_context.h +new file mode 100755 +index 000000000000..e4ed8944bdd2 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/context/mali_kbase_context.h +@@ -0,0 +1,157 @@ ++/* ++ * ++ * (C) COPYRIGHT ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ *//* SPDX-License-Identifier: GPL-2.0 */ ++/* ++ * ++ * (C) COPYRIGHT 2011-2017, 2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ */ ++ ++#ifndef _KBASE_CONTEXT_H_ ++#define _KBASE_CONTEXT_H_ ++ ++#include ++ ++/** ++ * kbase_context_debugfs_init - Initialize the kctx platform ++ * specific debugfs ++ * ++ * @kctx: kbase context ++ * ++ * This initializes some debugfs interfaces specific to the platform the source ++ * is compiled for. ++ */ ++void kbase_context_debugfs_init(struct kbase_context *const kctx); ++ ++/** ++ * kbase_context_debugfs_term - Terminate the kctx platform ++ * specific debugfs ++ * ++ * @kctx: kbase context ++ * ++ * This terminates some debugfs interfaces specific to the platform the source ++ * is compiled for. ++ */ ++void kbase_context_debugfs_term(struct kbase_context *const kctx); ++ ++/** ++ * kbase_create_context() - Create a kernel base context. ++ * ++ * @kbdev: Object representing an instance of GPU platform device, ++ * allocated from the probe method of the Mali driver. ++ * @is_compat: Force creation of a 32-bit context ++ * @flags: Flags to set, which shall be any combination of ++ * BASEP_CONTEXT_CREATE_KERNEL_FLAGS. ++ * @api_version: Application program interface version, as encoded in ++ * a single integer by the KBASE_API_VERSION macro. ++ * @filp: Pointer to the struct file corresponding to device file ++ * /dev/malixx instance, passed to the file's open method. ++ * ++ * Up to one context can be created for each client that opens the device file ++ * /dev/malixx. Context creation is deferred until a special ioctl() system call ++ * is made on the device file. Each context has its own GPU address space. ++ * ++ * Return: new kbase context or NULL on failure ++ */ ++struct kbase_context * ++kbase_create_context(struct kbase_device *kbdev, bool is_compat, ++ base_context_create_flags const flags, ++ unsigned long api_version, ++ struct file *filp); ++ ++/** ++ * kbase_destroy_context - Destroy a kernel base context. ++ * @kctx: Context to destroy ++ * ++ * Will release all outstanding regions. ++ */ ++void kbase_destroy_context(struct kbase_context *kctx); ++ ++/** ++ * kbase_ctx_flag - Check if @flag is set on @kctx ++ * @kctx: Pointer to kbase context to check ++ * @flag: Flag to check ++ * ++ * Return: true if @flag is set on @kctx, false if not. ++ */ ++static inline bool kbase_ctx_flag(struct kbase_context *kctx, ++ enum kbase_context_flags flag) ++{ ++ return atomic_read(&kctx->flags) & flag; ++} ++ ++/** ++ * kbase_ctx_flag_clear - Clear @flag on @kctx ++ * @kctx: Pointer to kbase context ++ * @flag: Flag to clear ++ * ++ * Clear the @flag on @kctx. This is done atomically, so other flags being ++ * cleared or set at the same time will be safe. ++ * ++ * Some flags have locking requirements, check the documentation for the ++ * respective flags. ++ */ ++static inline void kbase_ctx_flag_clear(struct kbase_context *kctx, ++ enum kbase_context_flags flag) ++{ ++#if KERNEL_VERSION(4, 3, 0) > LINUX_VERSION_CODE ++ /* ++ * Earlier kernel versions doesn't have atomic_andnot() or ++ * atomic_and(). atomic_clear_mask() was only available on some ++ * architectures and removed on arm in v3.13 on arm and arm64. ++ * ++ * Use a compare-exchange loop to clear the flag on pre 4.3 kernels, ++ * when atomic_andnot() becomes available. ++ */ ++ int old, new; ++ ++ do { ++ old = atomic_read(&kctx->flags); ++ new = old & ~flag; ++ ++ } while (atomic_cmpxchg(&kctx->flags, old, new) != old); ++#else ++ atomic_andnot(flag, &kctx->flags); ++#endif ++} ++ ++/** ++ * kbase_ctx_flag_set - Set @flag on @kctx ++ * @kctx: Pointer to kbase context ++ * @flag: Flag to set ++ * ++ * Set the @flag on @kctx. This is done atomically, so other flags being ++ * cleared or set at the same time will be safe. ++ * ++ * Some flags have locking requirements, check the documentation for the ++ * respective flags. ++ */ ++static inline void kbase_ctx_flag_set(struct kbase_context *kctx, ++ enum kbase_context_flags flag) ++{ ++ atomic_or(flag, &kctx->flags); ++} ++#endif /* _KBASE_CONTEXT_H_ */ +diff --git a/drivers/gpu/arm/bifrost/context/mali_kbase_context_internal.h b/drivers/gpu/arm/bifrost/context/mali_kbase_context_internal.h +new file mode 100755 +index 000000000000..818cdbea960d +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/context/mali_kbase_context_internal.h +@@ -0,0 +1,60 @@ ++/* ++ * ++ * (C) COPYRIGHT ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ *//* SPDX-License-Identifier: GPL-2.0 */ ++/* ++ * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ */ ++ ++#include ++ ++typedef int kbase_context_init_method(struct kbase_context *kctx); ++typedef void kbase_context_term_method(struct kbase_context *kctx); ++ ++/** ++ * struct kbase_context_init - Device init/term methods. ++ * @init: Function pointer to a initialise method. ++ * @term: Function pointer to a terminate method. ++ * @err_mes: Error message to be printed when init method fails. ++ */ ++struct kbase_context_init { ++ kbase_context_init_method *init; ++ kbase_context_term_method *term; ++ char *err_mes; ++}; ++ ++int kbase_context_common_init(struct kbase_context *kctx); ++void kbase_context_common_term(struct kbase_context *kctx); ++ ++int kbase_context_mem_pool_group_init(struct kbase_context *kctx); ++void kbase_context_mem_pool_group_term(struct kbase_context *kctx); ++ ++int kbase_context_mmu_init(struct kbase_context *kctx); ++void kbase_context_mmu_term(struct kbase_context *kctx); ++ ++int kbase_context_mem_alloc_page(struct kbase_context *kctx); ++void kbase_context_mem_pool_free(struct kbase_context *kctx); ++ ++void kbase_context_sticky_resource_term(struct kbase_context *kctx); +diff --git a/drivers/gpu/arm/bifrost/csf/Kbuild b/drivers/gpu/arm/bifrost/csf/Kbuild +new file mode 100755 +index 000000000000..bb61811e6c85 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/csf/Kbuild +@@ -0,0 +1,40 @@ ++# ++# (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# GNU General Public License for more details. ++# ++# You should have received a copy of the GNU General Public License ++# along with this program; if not, you can access it online at ++# http://www.gnu.org/licenses/gpl-2.0.html. ++# ++# SPDX-License-Identifier: GPL-2.0 ++# ++# ++ ++mali_kbase-y += \ ++ csf/mali_kbase_csf_firmware_cfg.o \ ++ csf/mali_kbase_csf_trace_buffer.o \ ++ csf/mali_kbase_csf.o \ ++ csf/mali_kbase_csf_scheduler.o \ ++ csf/mali_kbase_csf_kcpu.o \ ++ csf/mali_kbase_csf_tiler_heap.o \ ++ csf/mali_kbase_csf_timeout.o \ ++ csf/mali_kbase_csf_tl_reader.o \ ++ csf/mali_kbase_csf_heap_context_alloc.o \ ++ csf/mali_kbase_csf_reset_gpu.o \ ++ csf/mali_kbase_csf_csg_debugfs.o \ ++ csf/mali_kbase_csf_kcpu_debugfs.o \ ++ csf/mali_kbase_csf_protected_memory.o \ ++ csf/mali_kbase_csf_tiler_heap_debugfs.o ++ ++mali_kbase-$(CONFIG_MALI_REAL_HW) += csf/mali_kbase_csf_firmware.o ++ ++mali_kbase-$(CONFIG_MALI_BIFROST_NO_MALI) += csf/mali_kbase_csf_firmware_no_mali.o +diff --git a/drivers/gpu/arm/bifrost/csf/mali_base_csf_kernel.h b/drivers/gpu/arm/bifrost/csf/mali_base_csf_kernel.h +new file mode 100755 +index 000000000000..301146cbedd3 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/csf/mali_base_csf_kernel.h +@@ -0,0 +1,598 @@ ++/* ++ * ++ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _BASE_CSF_KERNEL_H_ ++#define _BASE_CSF_KERNEL_H_ ++ ++/* Memory allocation, access/hint flags. ++ * ++ * See base_mem_alloc_flags. ++ */ ++ ++/* IN */ ++/* Read access CPU side ++ */ ++#define BASE_MEM_PROT_CPU_RD ((base_mem_alloc_flags)1 << 0) ++ ++/* Write access CPU side ++ */ ++#define BASE_MEM_PROT_CPU_WR ((base_mem_alloc_flags)1 << 1) ++ ++/* Read access GPU side ++ */ ++#define BASE_MEM_PROT_GPU_RD ((base_mem_alloc_flags)1 << 2) ++ ++/* Write access GPU side ++ */ ++#define BASE_MEM_PROT_GPU_WR ((base_mem_alloc_flags)1 << 3) ++ ++/* Execute allowed on the GPU side ++ */ ++#define BASE_MEM_PROT_GPU_EX ((base_mem_alloc_flags)1 << 4) ++ ++/* Will be permanently mapped in kernel space. ++ * Flag is only allowed on allocations originating from kbase. ++ */ ++#define BASEP_MEM_PERMANENT_KERNEL_MAPPING ((base_mem_alloc_flags)1 << 5) ++ ++/* The allocation will completely reside within the same 4GB chunk in the GPU ++ * virtual space. ++ * Since this flag is primarily required only for the TLS memory which will ++ * not be used to contain executable code and also not used for Tiler heap, ++ * it can't be used along with BASE_MEM_PROT_GPU_EX and TILER_ALIGN_TOP flags. ++ */ ++#define BASE_MEM_GPU_VA_SAME_4GB_PAGE ((base_mem_alloc_flags)1 << 6) ++ ++/* Userspace is not allowed to free this memory. ++ * Flag is only allowed on allocations originating from kbase. ++ */ ++#define BASEP_MEM_NO_USER_FREE ((base_mem_alloc_flags)1 << 7) ++ ++#define BASE_MEM_RESERVED_BIT_8 ((base_mem_alloc_flags)1 << 8) ++ ++/* Grow backing store on GPU Page Fault ++ */ ++#define BASE_MEM_GROW_ON_GPF ((base_mem_alloc_flags)1 << 9) ++ ++/* Page coherence Outer shareable, if available ++ */ ++#define BASE_MEM_COHERENT_SYSTEM ((base_mem_alloc_flags)1 << 10) ++ ++/* Page coherence Inner shareable ++ */ ++#define BASE_MEM_COHERENT_LOCAL ((base_mem_alloc_flags)1 << 11) ++ ++/* IN/OUT */ ++/* Should be cached on the CPU, returned if actually cached ++ */ ++#define BASE_MEM_CACHED_CPU ((base_mem_alloc_flags)1 << 12) ++ ++/* IN/OUT */ ++/* Must have same VA on both the GPU and the CPU ++ */ ++#define BASE_MEM_SAME_VA ((base_mem_alloc_flags)1 << 13) ++ ++/* OUT */ ++/* Must call mmap to acquire a GPU address for the alloc ++ */ ++#define BASE_MEM_NEED_MMAP ((base_mem_alloc_flags)1 << 14) ++ ++/* IN */ ++/* Page coherence Outer shareable, required. ++ */ ++#define BASE_MEM_COHERENT_SYSTEM_REQUIRED ((base_mem_alloc_flags)1 << 15) ++ ++/* Protected memory ++ */ ++#define BASE_MEM_PROTECTED ((base_mem_alloc_flags)1 << 16) ++ ++/* Not needed physical memory ++ */ ++#define BASE_MEM_DONT_NEED ((base_mem_alloc_flags)1 << 17) ++ ++/* Must use shared CPU/GPU zone (SAME_VA zone) but doesn't require the ++ * addresses to be the same ++ */ ++#define BASE_MEM_IMPORT_SHARED ((base_mem_alloc_flags)1 << 18) ++ ++/* CSF event memory ++ * ++ * If Outer shareable coherence is not specified or not available, then on ++ * allocation kbase will automatically use the uncached GPU mapping. ++ * There is no need for the client to specify BASE_MEM_UNCACHED_GPU ++ * themselves when allocating memory with the BASE_MEM_CSF_EVENT flag. ++ * ++ * This memory requires a permanent mapping ++ * ++ * See also kbase_reg_needs_kernel_mapping() ++ */ ++#define BASE_MEM_CSF_EVENT ((base_mem_alloc_flags)1 << 19) ++ ++#define BASE_MEM_RESERVED_BIT_20 ((base_mem_alloc_flags)1 << 20) ++ ++/* Should be uncached on the GPU, will work only for GPUs using AARCH64 mmu ++ * mode. Some components within the GPU might only be able to access memory ++ * that is GPU cacheable. Refer to the specific GPU implementation for more ++ * details. The 3 shareability flags will be ignored for GPU uncached memory. ++ * If used while importing USER_BUFFER type memory, then the import will fail ++ * if the memory is not aligned to GPU and CPU cache line width. ++ */ ++#define BASE_MEM_UNCACHED_GPU ((base_mem_alloc_flags)1 << 21) ++ ++/* ++ * Bits [22:25] for group_id (0~15). ++ * ++ * base_mem_group_id_set() should be used to pack a memory group ID into a ++ * base_mem_alloc_flags value instead of accessing the bits directly. ++ * base_mem_group_id_get() should be used to extract the memory group ID from ++ * a base_mem_alloc_flags value. ++ */ ++#define BASEP_MEM_GROUP_ID_SHIFT 22 ++#define BASE_MEM_GROUP_ID_MASK \ ++ ((base_mem_alloc_flags)0xF << BASEP_MEM_GROUP_ID_SHIFT) ++ ++/* Must do CPU cache maintenance when imported memory is mapped/unmapped ++ * on GPU. Currently applicable to dma-buf type only. ++ */ ++#define BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP ((base_mem_alloc_flags)1 << 26) ++ ++/* OUT */ ++/* Kernel side cache sync ops required */ ++#define BASE_MEM_KERNEL_SYNC ((base_mem_alloc_flags)1 << 28) ++ ++/* Number of bits used as flags for base memory management ++ * ++ * Must be kept in sync with the base_mem_alloc_flags flags ++ */ ++#define BASE_MEM_FLAGS_NR_BITS 29 ++ ++/* A mask of all the flags which are only valid for allocations within kbase, ++ * and may not be passed from user space. ++ */ ++#define BASEP_MEM_FLAGS_KERNEL_ONLY \ ++ (BASEP_MEM_PERMANENT_KERNEL_MAPPING | BASEP_MEM_NO_USER_FREE) ++ ++/* A mask for all output bits, excluding IN/OUT bits. ++ */ ++#define BASE_MEM_FLAGS_OUTPUT_MASK BASE_MEM_NEED_MMAP ++ ++/* A mask for all input bits, including IN/OUT bits. ++ */ ++#define BASE_MEM_FLAGS_INPUT_MASK \ ++ (((1 << BASE_MEM_FLAGS_NR_BITS) - 1) & ~BASE_MEM_FLAGS_OUTPUT_MASK) ++ ++/* A mask of all currently reserved flags ++ */ ++#define BASE_MEM_FLAGS_RESERVED \ ++ BASE_MEM_RESERVED_BIT_8 | BASE_MEM_RESERVED_BIT_20 ++ ++#define BASEP_MEM_INVALID_HANDLE (0ull << 12) ++#define BASE_MEM_MMU_DUMP_HANDLE (1ull << 12) ++#define BASE_MEM_TRACE_BUFFER_HANDLE (2ull << 12) ++#define BASE_MEM_MAP_TRACKING_HANDLE (3ull << 12) ++#define BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE (4ull << 12) ++/* reserved handles ..-47< for future special handles */ ++#define BASEP_MEM_CSF_USER_REG_PAGE_HANDLE (47ul << 12) ++#define BASEP_MEM_CSF_USER_IO_PAGES_HANDLE (48ul << 12) ++#define BASE_MEM_COOKIE_BASE (64ul << 12) ++#define BASE_MEM_FIRST_FREE_ADDRESS ((BITS_PER_LONG << 12) + \ ++ BASE_MEM_COOKIE_BASE) ++ ++#define KBASE_CSF_NUM_USER_IO_PAGES_HANDLE \ ++ ((BASE_MEM_COOKIE_BASE - BASEP_MEM_CSF_USER_IO_PAGES_HANDLE) >> \ ++ LOCAL_PAGE_SHIFT) ++ ++/** ++ * Valid set of just-in-time memory allocation flags ++ */ ++#define BASE_JIT_ALLOC_VALID_FLAGS ((u8)0) ++ ++/* Flags to pass to ::base_context_init. ++ * Flags can be ORed together to enable multiple things. ++ * ++ * These share the same space as BASEP_CONTEXT_FLAG_*, and so must ++ * not collide with them. ++ */ ++typedef u32 base_context_create_flags; ++ ++/* No flags set */ ++#define BASE_CONTEXT_CREATE_FLAG_NONE ((base_context_create_flags)0) ++ ++/* Base context is embedded in a cctx object (flag used for CINSTR ++ * software counter macros) ++ */ ++#define BASE_CONTEXT_CCTX_EMBEDDED ((base_context_create_flags)1 << 0) ++ ++/* Base context is a 'System Monitor' context for Hardware counters. ++ * ++ * One important side effect of this is that job submission is disabled. ++ */ ++#define BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED \ ++ ((base_context_create_flags)1 << 1) ++ ++/* Create CSF event thread. ++ * ++ * The creation of a CSF event thread is conditional and only allowed in ++ * unit tests for the moment, in order to avoid clashes with the existing ++ * Base unit tests. ++ */ ++#define BASE_CONTEXT_CSF_EVENT_THREAD ((base_context_create_flags)1 << 2) ++ ++/* Bit-shift used to encode a memory group ID in base_context_create_flags ++ */ ++#define BASEP_CONTEXT_MMU_GROUP_ID_SHIFT (3) ++ ++/* Bitmask used to encode a memory group ID in base_context_create_flags ++ */ ++#define BASEP_CONTEXT_MMU_GROUP_ID_MASK \ ++ ((base_context_create_flags)0xF << BASEP_CONTEXT_MMU_GROUP_ID_SHIFT) ++ ++/* Bitpattern describing the base_context_create_flags that can be ++ * passed to the kernel ++ */ ++#define BASEP_CONTEXT_CREATE_KERNEL_FLAGS \ ++ (BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED | \ ++ BASEP_CONTEXT_MMU_GROUP_ID_MASK) ++ ++/* Bitpattern describing the ::base_context_create_flags that can be ++ * passed to base_context_init() ++ */ ++#define BASEP_CONTEXT_CREATE_ALLOWED_FLAGS \ ++ (BASE_CONTEXT_CCTX_EMBEDDED | \ ++ BASE_CONTEXT_CSF_EVENT_THREAD | \ ++ BASEP_CONTEXT_CREATE_KERNEL_FLAGS) ++ ++/* Enable additional tracepoints for latency measurements (TL_ATOM_READY, ++ * TL_ATOM_DONE, TL_ATOM_PRIO_CHANGE, TL_ATOM_EVENT_POST) ++ */ ++#define BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS (1 << 0) ++ ++/* Indicate that job dumping is enabled. This could affect certain timers ++ * to account for the performance impact. ++ */ ++#define BASE_TLSTREAM_JOB_DUMPING_ENABLED (1 << 1) ++ ++/* Enable KBase tracepoints for CSF builds */ ++#define BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS (1 << 2) ++ ++/* Enable additional CSF Firmware side tracepoints */ ++#define BASE_TLSTREAM_ENABLE_CSFFW_TRACEPOINTS (1 << 3) ++ ++#define BASE_TLSTREAM_FLAGS_MASK (BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS | \ ++ BASE_TLSTREAM_JOB_DUMPING_ENABLED | \ ++ BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS | \ ++ BASE_TLSTREAM_ENABLE_CSFFW_TRACEPOINTS) ++ ++/* Number of pages mapped into the process address space for a bound GPU ++ * command queue. A pair of input/output pages and a Hw doorbell page ++ * are mapped to enable direct submission of commands to Hw. ++ */ ++#define BASEP_QUEUE_NR_MMAP_USER_PAGES ((size_t)3) ++ ++#define BASE_QUEUE_MAX_PRIORITY (15U) ++ ++/* CQS Sync object is an array of u32 event_mem[2], error field index is 1 */ ++#define BASEP_EVENT_VAL_INDEX (0U) ++#define BASEP_EVENT_ERR_INDEX (1U) ++ ++/* The upper limit for number of objects that could be waited/set per command. ++ * This limit is now enforced as internally the error inherit inputs are ++ * converted to 32-bit flags in a u32 variable occupying a previously padding ++ * field. ++ */ ++#define BASEP_KCPU_CQS_MAX_NUM_OBJS ((size_t)32) ++ ++/** ++ * enum base_kcpu_command_type - Kernel CPU queue command type. ++ */ ++enum base_kcpu_command_type { ++ BASE_KCPU_COMMAND_TYPE_FENCE_SIGNAL, ++ BASE_KCPU_COMMAND_TYPE_FENCE_WAIT, ++ BASE_KCPU_COMMAND_TYPE_CQS_WAIT, ++ BASE_KCPU_COMMAND_TYPE_CQS_SET, ++ BASE_KCPU_COMMAND_TYPE_MAP_IMPORT, ++ BASE_KCPU_COMMAND_TYPE_UNMAP_IMPORT, ++ BASE_KCPU_COMMAND_TYPE_UNMAP_IMPORT_FORCE, ++ BASE_KCPU_COMMAND_TYPE_JIT_ALLOC, ++ BASE_KCPU_COMMAND_TYPE_JIT_FREE, ++ BASE_KCPU_COMMAND_TYPE_GROUP_SUSPEND, ++ BASE_KCPU_COMMAND_TYPE_ERROR_BARRIER, ++}; ++ ++/** ++ * enum base_queue_group_priority - Priority of a GPU Command Queue Group. ++ * @BASE_QUEUE_GROUP_PRIORITY_HIGH: GPU Command Queue Group is of high ++ * priority. ++ * @BASE_QUEUE_GROUP_PRIORITY_MEDIUM: GPU Command Queue Group is of medium ++ * priority. ++ * @BASE_QUEUE_GROUP_PRIORITY_LOW: GPU Command Queue Group is of low ++ * priority. ++ * @BASE_QUEUE_GROUP_PRIORITY_COUNT: Number of GPU Command Queue Group ++ * priority levels. ++ * ++ * Currently this is in order of highest to lowest, but if new levels are added ++ * then those new levels may be out of order to preserve the ABI compatibility ++ * with previous releases. At that point, ensure assignment to ++ * the 'priority' member in &kbase_queue_group is updated to ensure it remains ++ * a linear ordering. ++ * ++ * There should be no gaps in the enum, otherwise use of ++ * BASE_QUEUE_GROUP_PRIORITY_COUNT in kbase must be updated. ++ */ ++enum base_queue_group_priority { ++ BASE_QUEUE_GROUP_PRIORITY_HIGH = 0, ++ BASE_QUEUE_GROUP_PRIORITY_MEDIUM, ++ BASE_QUEUE_GROUP_PRIORITY_LOW, ++ BASE_QUEUE_GROUP_PRIORITY_COUNT ++}; ++ ++struct base_kcpu_command_fence_info { ++ u64 fence; ++}; ++ ++struct base_cqs_wait { ++ u64 addr; ++ u32 val; ++ u32 padding; ++}; ++ ++struct base_kcpu_command_cqs_wait_info { ++ u64 objs; ++ u32 nr_objs; ++ u32 inherit_err_flags; ++}; ++ ++struct base_cqs_set { ++ u64 addr; ++}; ++ ++struct base_kcpu_command_cqs_set_info { ++ u64 objs; ++ u32 nr_objs; ++ u32 propagate_flags; ++}; ++ ++/** ++ * struct base_kcpu_command_import_info - structure which contains information ++ * about the imported buffer. ++ * ++ * @handle: Address of imported user buffer. ++ */ ++struct base_kcpu_command_import_info { ++ u64 handle; ++}; ++ ++/** ++ * struct base_kcpu_command_jit_alloc_info - structure which contains ++ * information about jit memory allocation. ++ * ++ * @info: An array of elements of the ++ * struct base_jit_alloc_info type. ++ * @count: The number of elements in the info array. ++ * @padding: Padding to a multiple of 64 bits. ++ */ ++struct base_kcpu_command_jit_alloc_info { ++ u64 info; ++ u8 count; ++ u8 padding[7]; ++}; ++ ++/** ++ * struct base_kcpu_command_jit_free_info - structure which contains ++ * information about jit memory which is to be freed. ++ * ++ * @ids: An array containing the JIT IDs to free. ++ * @count: The number of elements in the ids array. ++ * @padding: Padding to a multiple of 64 bits. ++ */ ++struct base_kcpu_command_jit_free_info { ++ u64 ids; ++ u8 count; ++ u8 padding[7]; ++}; ++ ++/** ++ * struct base_kcpu_command_group_suspend_info - structure which contains ++ * suspend buffer data captured for a suspended queue group. ++ * ++ * @buffer: Pointer to an array of elements of the type char. ++ * @size: Number of elements in the @buffer array. ++ * @group_handle: Handle to the mapping of command stream group. ++ * @padding: padding to a multiple of 64 bits. ++ */ ++struct base_kcpu_command_group_suspend_info { ++ u64 buffer; ++ u32 size; ++ u8 group_handle; ++ u8 padding[3]; ++}; ++ ++/** ++ * struct base_kcpu_command - kcpu command. ++ * ++ * @type: type of the kcpu command, one enum base_kcpu_command_type ++ * @info: structure which contains information about the kcpu command; ++ * actual type is determined by @p type ++ * @padding: padding to a multiple of 64 bits ++ */ ++struct base_kcpu_command { ++ u8 type; ++ u8 padding[sizeof(u64) - sizeof(u8)]; ++ union { ++ struct base_kcpu_command_fence_info fence; ++ struct base_kcpu_command_cqs_wait_info cqs_wait; ++ struct base_kcpu_command_cqs_set_info cqs_set; ++ struct base_kcpu_command_import_info import; ++ struct base_kcpu_command_jit_alloc_info jit_alloc; ++ struct base_kcpu_command_jit_free_info jit_free; ++ struct base_kcpu_command_group_suspend_info suspend_buf_copy; ++ u64 padding[2]; /* No sub-struct should be larger */ ++ } info; ++}; ++ ++/** ++ * struct basep_cs_stream_control - Command Stream interface capabilities. ++ * ++ * @features: Features of this stream ++ * @padding: Padding to a multiple of 64 bits. ++ */ ++struct basep_cs_stream_control { ++ u32 features; ++ u32 padding; ++}; ++ ++/** ++ * struct basep_cs_group_control - Command Stream Group interface capabilities. ++ * ++ * @features: Features of this group ++ * @stream_num: Number of streams in this group ++ * @suspend_size: Size in bytes of the suspend buffer for this group ++ * @padding: Padding to a multiple of 64 bits. ++ */ ++struct basep_cs_group_control { ++ u32 features; ++ u32 stream_num; ++ u32 suspend_size; ++ u32 padding; ++}; ++ ++/** ++ * struct base_gpu_queue_group_error_fatal_payload - Unrecoverable fault ++ * error information associated with GPU command queue group. ++ * ++ * @sideband: Additional information of the unrecoverable fault. ++ * @status: Unrecoverable fault information. ++ * This consists of exception type (least significant byte) and ++ * data (remaining bytes). One example of exception type is ++ * CS_INVALID_INSTRUCTION (0x49). ++ * @padding: Padding to make multiple of 64bits ++ */ ++struct base_gpu_queue_group_error_fatal_payload { ++ u64 sideband; ++ u32 status; ++ u32 padding; ++}; ++ ++/** ++ * struct base_gpu_queue_error_fatal_payload - Unrecoverable fault ++ * error information related to GPU command queue. ++ * ++ * @sideband: Additional information about this unrecoverable fault. ++ * @status: Unrecoverable fault information. ++ * This consists of exception type (least significant byte) and ++ * data (remaining bytes). One example of exception type is ++ * CS_INVALID_INSTRUCTION (0x49). ++ * @csi_index: Index of the CSF interface the queue is bound to. ++ * @padding: Padding to make multiple of 64bits ++ */ ++struct base_gpu_queue_error_fatal_payload { ++ u64 sideband; ++ u32 status; ++ u8 csi_index; ++ u8 padding[3]; ++}; ++ ++/** ++ * enum base_gpu_queue_group_error_type - GPU Fatal error type. ++ * ++ * @BASE_GPU_QUEUE_GROUP_ERROR_FATAL: Fatal error associated with GPU ++ * command queue group. ++ * @BASE_GPU_QUEUE_GROUP_QUEUE_ERROR_FATAL: Fatal error associated with GPU ++ * command queue. ++ * @BASE_GPU_QUEUE_GROUP_ERROR_TIMEOUT: Fatal error associated with ++ * progress timeout. ++ * @BASE_GPU_QUEUE_GROUP_ERROR_TILER_HEAP_OOM: Fatal error due to running out ++ * of tiler heap memory. ++ * @BASE_GPU_QUEUE_GROUP_ERROR_FATAL_COUNT: The number of fatal error types ++ * ++ * This type is used for &struct_base_gpu_queue_group_error.error_type. ++ */ ++enum base_gpu_queue_group_error_type { ++ BASE_GPU_QUEUE_GROUP_ERROR_FATAL = 0, ++ BASE_GPU_QUEUE_GROUP_QUEUE_ERROR_FATAL, ++ BASE_GPU_QUEUE_GROUP_ERROR_TIMEOUT, ++ BASE_GPU_QUEUE_GROUP_ERROR_TILER_HEAP_OOM, ++ BASE_GPU_QUEUE_GROUP_ERROR_FATAL_COUNT ++}; ++ ++/** ++ * struct base_gpu_queue_group_error - Unrecoverable fault information ++ * ++ * @error_type: Error type of @base_gpu_queue_group_error_type ++ * indicating which field in union payload is filled ++ * @padding: Unused bytes for 64bit boundary ++ * @fatal_group: Unrecoverable fault error associated with ++ * GPU command queue group ++ * @fatal_queue: Unrecoverable fault error associated with command queue ++ * ++ * @payload: Input Payload ++ */ ++struct base_gpu_queue_group_error { ++ u8 error_type; ++ u8 padding[7]; ++ union { ++ struct base_gpu_queue_group_error_fatal_payload fatal_group; ++ struct base_gpu_queue_error_fatal_payload fatal_queue; ++ } payload; ++}; ++ ++/** ++ * enum base_csf_notification_type - Notification type ++ * ++ * @BASE_CSF_NOTIFICATION_EVENT: Notification with kernel event ++ * @BASE_CSF_NOTIFICATION_GPU_QUEUE_GROUP_ERROR: Notification with GPU fatal ++ * error ++ * @BASE_CSF_NOTIFICATION_COUNT: The number of notification type ++ * ++ * This type is used for &struct_base_csf_notification.type. ++ */ ++enum base_csf_notification_type { ++ BASE_CSF_NOTIFICATION_EVENT = 0, ++ BASE_CSF_NOTIFICATION_GPU_QUEUE_GROUP_ERROR, ++ BASE_CSF_NOTIFICATION_COUNT ++}; ++ ++/** ++ * struct base_csf_notification - Event or error notification ++ * ++ * @type: Notification type of @base_csf_notification_type ++ * @padding: Padding for 64bit boundary ++ * @handle: Handle of GPU command queue group associated with fatal error ++ * @error: Unrecoverable fault error ++ * @align: To fit the struct into a 64-byte cache line ++ * ++ * @payload: Input Payload ++ */ ++struct base_csf_notification { ++ u8 type; ++ u8 padding[7]; ++ union { ++ struct { ++ u8 handle; ++ u8 padding[7]; ++ struct base_gpu_queue_group_error error; ++ } csg_error; ++ u8 align[56]; ++ } payload; ++}; ++ ++#endif /* _BASE_CSF_KERNEL_H_ */ +diff --git a/drivers/gpu/arm/bifrost/csf/mali_gpu_csf_control_registers.h b/drivers/gpu/arm/bifrost/csf/mali_gpu_csf_control_registers.h +new file mode 100755 +index 000000000000..4fff80ca4023 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/csf/mali_gpu_csf_control_registers.h +@@ -0,0 +1,33 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* ++ * This header was autogenerated, it should not be edited. ++ */ ++ ++#ifndef _GPU_CSF_CONTROL_REGISTERS_H_ ++#define _GPU_CSF_CONTROL_REGISTERS_H_ ++ ++/* GPU_REGISTERS register offsets */ ++#define GPU_CONTROL_MCU 0x3000 /* () MCU control registers */ ++ ++#endif /* _GPU_CSF_CONTROL_REGISTERS_H_ */ +diff --git a/drivers/gpu/arm/bifrost/csf/mali_gpu_csf_registers.h b/drivers/gpu/arm/bifrost/csf/mali_gpu_csf_registers.h +new file mode 100755 +index 000000000000..5c03445f3c79 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/csf/mali_gpu_csf_registers.h +@@ -0,0 +1,1252 @@ ++/* ++ * ++ * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* ++ * This header was autogenerated, it should not be edited. ++ */ ++ ++#ifndef _GPU_CSF_REGISTERS_H_ ++#define _GPU_CSF_REGISTERS_H_ ++ ++/* ++ * Begin register sets ++ */ ++ ++/* DOORBELLS base address */ ++#define DOORBELLS_BASE 0x0080000 ++#define DOORBELLS_REG(r) (DOORBELLS_BASE + (r)) ++ ++/* CS_KERNEL_INPUT_BLOCK base address */ ++#define CS_KERNEL_INPUT_BLOCK_BASE 0x0000 ++#define CS_KERNEL_INPUT_BLOCK_REG(r) (CS_KERNEL_INPUT_BLOCK_BASE + (r)) ++ ++/* CS_KERNEL_OUTPUT_BLOCK base address */ ++#define CS_KERNEL_OUTPUT_BLOCK_BASE 0x0000 ++#define CS_KERNEL_OUTPUT_BLOCK_REG(r) (CS_KERNEL_OUTPUT_BLOCK_BASE + (r)) ++ ++/* CS_USER_INPUT_BLOCK base address */ ++#define CS_USER_INPUT_BLOCK_BASE 0x0000 ++#define CS_USER_INPUT_BLOCK_REG(r) (CS_USER_INPUT_BLOCK_BASE + (r)) ++ ++/* CS_USER_OUTPUT_BLOCK base address */ ++#define CS_USER_OUTPUT_BLOCK_BASE 0x0000 ++#define CS_USER_OUTPUT_BLOCK_REG(r) (CS_USER_OUTPUT_BLOCK_BASE + (r)) ++ ++/* CSG_INPUT_BLOCK base address */ ++#define CSG_INPUT_BLOCK_BASE 0x0000 ++#define CSG_INPUT_BLOCK_REG(r) (CSG_INPUT_BLOCK_BASE + (r)) ++ ++/* CSG_OUTPUT_BLOCK base address */ ++#define CSG_OUTPUT_BLOCK_BASE 0x0000 ++#define CSG_OUTPUT_BLOCK_REG(r) (CSG_OUTPUT_BLOCK_BASE + (r)) ++ ++/* GLB_CONTROL_BLOCK base address */ ++#define GLB_CONTROL_BLOCK_BASE 0x04000000 ++#define GLB_CONTROL_BLOCK_REG(r) (GLB_CONTROL_BLOCK_BASE + (r)) ++ ++/* GLB_INPUT_BLOCK base address */ ++#define GLB_INPUT_BLOCK_BASE 0x0000 ++#define GLB_INPUT_BLOCK_REG(r) (GLB_INPUT_BLOCK_BASE + (r)) ++ ++/* GLB_OUTPUT_BLOCK base address */ ++#define GLB_OUTPUT_BLOCK_BASE 0x0000 ++#define GLB_OUTPUT_BLOCK_REG(r) (GLB_OUTPUT_BLOCK_BASE + (r)) ++ ++/* USER base address */ ++#define USER_BASE 0x0010000 ++#define USER_REG(r) (USER_BASE + (r)) ++ ++/* End register sets */ ++ ++/* ++ * Begin register offsets ++ */ ++ ++/* DOORBELLS register offsets */ ++#define DOORBELL_0 0x0000 /* () Doorbell 0 register */ ++#define DOORBELL(n) (DOORBELL_0 + (n)*65536) ++#define DOORBELL_REG(n, r) (DOORBELL(n) + DOORBELL_BLOCK_REG(r)) ++#define DOORBELL_COUNT 1024 ++ ++/* DOORBELL_BLOCK register offsets */ ++#define DB_BLK_DOORBELL 0x0000 /* (WO) Doorbell request */ ++ ++/* CS_KERNEL_INPUT_BLOCK register offsets */ ++#define CS_REQ 0x0000 /* () Command stream request flags */ ++#define CS_CONFIG 0x0004 /* () Command stream configuration */ ++#define CS_ACK_IRQ_MASK 0x000C /* () Command steam interrupt mask */ ++#define CS_BASE_LO 0x0010 /* () Base pointer for the ring buffer, low word */ ++#define CS_BASE_HI 0x0014 /* () Base pointer for the ring buffer, high word */ ++#define CS_SIZE 0x0018 /* () Size of the ring buffer */ ++#define CS_TILER_HEAP_START_LO 0x0020 /* () Pointer to heap start, low word */ ++#define CS_TILER_HEAP_START_HI 0x0024 /* () Pointer to heap start, high word */ ++#define CS_TILER_HEAP_END_LO 0x0028 /* () Tiler heap descriptor address, low word */ ++#define CS_TILER_HEAP_END_HI 0x002C /* () Tiler heap descriptor address, high word */ ++#define CS_USER_INPUT_LO 0x0030 /* () CS user mode input page address, low word */ ++#define CS_USER_INPUT_HI 0x0034 /* () CS user mode input page address, high word */ ++#define CS_USER_OUTPUT_LO 0x0038 /* () CS user mode input page address, low word */ ++#define CS_USER_OUTPUT_HI 0x003C /* () CS user mode input page address, high word */ ++ ++/* CS_KERNEL_OUTPUT_BLOCK register offsets */ ++#define CS_ACK 0x0000 /* () Command stream acknowledge flags */ ++#define CS_STATUS_CMD_PTR_LO 0x0040 /* () Program pointer current value, low word */ ++#define CS_STATUS_CMD_PTR_HI 0x0044 /* () Program pointer current value, high word */ ++#define CS_STATUS_WAIT 0x0048 /* () Wait condition status register */ ++#define CS_STATUS_REQ_RESOURCE 0x004C /* () Indicates the resources requested by the command stream */ ++#define CS_STATUS_WAIT_SYNC_POINTER_LO 0x0050 /* () Sync object pointer, low word */ ++#define CS_STATUS_WAIT_SYNC_POINTER_HI 0x0054 /* () Sync object pointer, high word */ ++#define CS_STATUS_WAIT_SYNC_VALUE 0x0058 /* () Sync object test value */ ++#define CS_FAULT 0x0080 /* () Recoverable fault information */ ++#define CS_FATAL 0x0084 /* () Unrecoverable fault information */ ++#define CS_FAULT_INFO_LO 0x0088 /* () Additional information about a recoverable fault, low word */ ++#define CS_FAULT_INFO_HI 0x008C /* () Additional information about a recoverable fault, high word */ ++#define CS_FATAL_INFO_LO 0x0090 /* () Additional information about a non-recoverable fault, low word */ ++#define CS_FATAL_INFO_HI 0x0094 /* () Additional information about a non-recoverable fault, high word */ ++#define CS_HEAP_VT_START 0x00C0 /* () Number of vertex/tiling operations started */ ++#define CS_HEAP_VT_END 0x00C4 /* () Number of vertex/tiling operations completed */ ++#define CS_HEAP_FRAG_END 0x00CC /* () Number of fragment completed */ ++#define CS_HEAP_ADDRESS_LO 0x00D0 /* () Heap address, low word */ ++#define CS_HEAP_ADDRESS_HI 0x00D4 /* () Heap address, high word */ ++ ++/* CS_USER_INPUT_BLOCK register offsets */ ++#define CS_INSERT_LO 0x0000 /* () Current insert offset for ring buffer, low word */ ++#define CS_INSERT_HI 0x0004 /* () Current insert offset for ring buffer, high word */ ++#define CS_EXTRACT_INIT_LO 0x0008 /* () Initial extract offset for ring buffer, low word */ ++#define CS_EXTRACT_INIT_HI 0x000C /* () Initial extract offset for ring buffer, high word */ ++ ++/* CS_USER_OUTPUT_BLOCK register offsets */ ++#define CS_EXTRACT_LO 0x0000 /* () Current extract offset for ring buffer, low word */ ++#define CS_EXTRACT_HI 0x0004 /* () Current extract offset for ring buffer, high word */ ++#define CS_ACTIVE 0x0008 /* () Initial extract offset when the command stream is started */ ++ ++/* CSG_INPUT_BLOCK register offsets */ ++#define CSG_REQ 0x0000 /* () CSG request */ ++#define CSG_ACK_IRQ_MASK 0x0004 /* () Global acknowledge interrupt mask */ ++#define CSG_DB_REQ 0x0008 /* () Global doorbell request */ ++#define CSG_IRQ_ACK 0x000C /* () Command stream IRQ acknowledge */ ++#define CSG_ALLOW_COMPUTE_LO 0x0020 /* () Allowed compute endpoints, low word */ ++#define CSG_ALLOW_COMPUTE_HI 0x0024 /* () Allowed compute endpoints, high word */ ++#define CSG_ALLOW_FRAGMENT_LO 0x0028 /* () Allowed fragment endpoints, low word */ ++#define CSG_ALLOW_FRAGMENT_HI 0x002C /* () Allowed fragment endpoints, high word */ ++#define CSG_ALLOW_OTHER 0x0030 /* () Allowed other endpoints */ ++#define CSG_EP_REQ 0x0034 /* () Maximum number of endpoints allowed */ ++#define CSG_SUSPEND_BUF_LO 0x0040 /* () Normal mode suspend buffer, low word */ ++#define CSG_SUSPEND_BUF_HI 0x0044 /* () Normal mode suspend buffer, high word */ ++#define CSG_PROTM_SUSPEND_BUF_LO 0x0048 /* () Protected mode suspend buffer, low word */ ++#define CSG_PROTM_SUSPEND_BUF_HI 0x004C /* () Protected mode suspend buffer, high word */ ++#define CSG_CONFIG 0x0050 /* () CSG configuration options */ ++ ++/* CSG_OUTPUT_BLOCK register offsets */ ++#define CSG_ACK 0x0000 /* () Command stream group acknowledge flags */ ++#define CSG_DB_ACK 0x0008 /* () Command stream kernel doorbell acknowledge flags */ ++#define CSG_IRQ_REQ 0x000C /* () Command stream interrupt request flags */ ++#define CSG_STATUS_EP_CURRENT 0x0010 /* () Endpoint allocation status register */ ++#define CSG_STATUS_EP_REQ 0x0014 /* () Endpoint request status register */ ++#define CSG_RESOURCE_DEP 0x001C /* () Current resource dependencies */ ++ ++/* GLB_CONTROL_BLOCK register offsets */ ++#define GLB_VERSION 0x0000 /* () Global interface version */ ++#define GLB_FEATURES 0x0004 /* () Global interface features */ ++#define GLB_INPUT_VA 0x0008 /* () Address of GLB_INPUT_BLOCK */ ++#define GLB_OUTPUT_VA 0x000C /* () Address of GLB_OUTPUT_BLOCK */ ++#define GLB_GROUP_NUM 0x0010 /* () Number of CSG interfaces */ ++#define GLB_GROUP_STRIDE 0x0014 /* () Stride between CSG interfaces */ ++#define GLB_PRFCNT_SIZE 0x0018 /* () Size of CSF performance counters */ ++#define GROUP_CONTROL_0 0x1000 /* () CSG control and capabilities */ ++#define GROUP_CONTROL(n) (GROUP_CONTROL_0 + (n)*256) ++#define GROUP_CONTROL_REG(n, r) (GROUP_CONTROL(n) + GROUP_CONTROL_BLOCK_REG(r)) ++#define GROUP_CONTROL_COUNT 16 ++ ++/* STREAM_CONTROL_BLOCK register offsets */ ++#define STREAM_FEATURES 0x0000 /* () Command Stream interface features */ ++#define STREAM_INPUT_VA 0x0004 /* () Address of CS_KERNEL_INPUT_BLOCK */ ++#define STREAM_OUTPUT_VA 0x0008 /* () Address of CS_KERNEL_OUTPUT_BLOCK */ ++ ++/* GROUP_CONTROL_BLOCK register offsets */ ++#define GROUP_FEATURES 0x0000 /* () Command Stream Group interface features */ ++#define GROUP_INPUT_VA 0x0004 /* () Address of CSG_INPUT_BLOCK */ ++#define GROUP_OUTPUT_VA 0x0008 /* () Address of CSG_OUTPUT_BLOCK */ ++#define GROUP_SUSPEND_SIZE 0x000C /* () Size of CSG suspend buffer */ ++#define GROUP_PROTM_SUSPEND_SIZE 0x0010 /* () Size of CSG protected-mode suspend buffer */ ++#define GROUP_STREAM_NUM 0x0014 /* () Number of CS interfaces */ ++#define GROUP_STREAM_STRIDE 0x0018 /* () Stride between CS interfaces */ ++#define STREAM_CONTROL_0 0x0040 /* () CS control and capabilities */ ++#define STREAM_CONTROL(n) (STREAM_CONTROL_0 + (n)*12) ++#define STREAM_CONTROL_REG(n, r) (STREAM_CONTROL(n) + STREAM_CONTROL_BLOCK_REG(r)) ++#define STREAM_CONTROL_COUNT 16 ++ ++/* GLB_INPUT_BLOCK register offsets */ ++#define GLB_REQ 0x0000 /* () Global request */ ++#define GLB_ACK_IRQ_MASK 0x0004 /* () Global acknowledge interrupt mask */ ++#define GLB_DB_REQ 0x0008 /* () Global doorbell request */ ++#define GLB_PROGRESS_TIMER 0x0010 /* () Global progress timeout */ ++#define GLB_PWROFF_TIMER 0x0014 /* () Global shader core power off timer */ ++#define GLB_ALLOC_EN_LO 0x0018 /* () Global shader core allocation enable mask, low word */ ++#define GLB_ALLOC_EN_HI 0x001C /* () Global shader core allocation enable mask, high word */ ++#define GLB_PROTM_COHERENCY 0x0020 /* () Configure COHERENCY_ENABLE register value to use in protected mode execution */ ++ ++#define GLB_PRFCNT_JASID 0x0024 /* () Performance counter address space */ ++#define GLB_PRFCNT_BASE_LO 0x0028 /* () Performance counter buffer address, low word */ ++#define GLB_PRFCNT_BASE_HI 0x002C /* () Performance counter buffer address, high word */ ++#define GLB_PRFCNT_CONFIG 0x0040 /* () Performance counter configuration */ ++#define GLB_PRFCNT_CSG_SELECT 0x0044 /* () CSG performance counting enable */ ++#define GLB_PRFCNT_FW_EN 0x0048 /* () Performance counter enable for firmware */ ++#define GLB_PRFCNT_CSG_EN 0x004C /* () Performance counter enable for CSG */ ++#define GLB_PRFCNT_CSF_EN 0x0050 /* () Performance counter enable for CSF */ ++#define GLB_PRFCNT_SHADER_EN 0x0054 /* () Performance counter enable for shader cores */ ++#define GLB_PRFCNT_TILER_EN 0x0058 /* () Performance counter enable for tiler */ ++#define GLB_PRFCNT_MMU_L2_EN 0x005C /* () Performance counter enable for MMU/L2 cache */ ++ ++#define GLB_DEBUG_FWUTF_DESTROY 0x0FE0 /* () Test fixture destroy function address */ ++#define GLB_DEBUG_FWUTF_TEST 0x0FE4 /* () Test index */ ++#define GLB_DEBUG_FWUTF_FIXTURE 0x0FE8 /* () Test fixture index */ ++#define GLB_DEBUG_FWUTF_CREATE 0x0FEC /* () Test fixture create function address */ ++#define GLB_DEBUG_ACK_IRQ_MASK 0x0FF8 /* () Global debug acknowledge interrupt mask */ ++#define GLB_DEBUG_REQ 0x0FFC /* () Global debug request */ ++ ++/* GLB_OUTPUT_BLOCK register offsets */ ++#define GLB_ACK 0x0000 /* () Global acknowledge */ ++#define GLB_DB_ACK 0x0008 /* () Global doorbell acknowledge */ ++#define GLB_HALT_STATUS 0x0010 /* () Global halt status */ ++#define GLB_PRFCNT_STATUS 0x0014 /* () Performance counter status */ ++#define GLB_DEBUG_FWUTF_RESULT 0x0FE0 /* () Firmware debug test result */ ++#define GLB_DEBUG_ACK 0x0FFC /* () Global debug acknowledge */ ++ ++/* End register offsets */ ++ ++/* CS_KERNEL_INPUT_BLOCK register set definitions */ ++ ++/* CS_REQ register */ ++#define CS_REQ_STATE_SHIFT 0 ++#define CS_REQ_STATE_MASK (0x7 << CS_REQ_STATE_SHIFT) ++#define CS_REQ_STATE_GET(reg_val) (((reg_val)&CS_REQ_STATE_MASK) >> CS_REQ_STATE_SHIFT) ++#define CS_REQ_STATE_SET(reg_val, value) \ ++ (((reg_val) & ~CS_REQ_STATE_MASK) | (((value) << CS_REQ_STATE_SHIFT) & CS_REQ_STATE_MASK)) ++/* CS_REQ_STATE values */ ++#define CS_REQ_STATE_STOP 0x0 ++#define CS_REQ_STATE_START 0x1 ++/* End of CS_REQ_STATE values */ ++#define CS_REQ_EXTRACT_EVENT_SHIFT 4 ++#define CS_REQ_EXTRACT_EVENT_MASK (0x1 << CS_REQ_EXTRACT_EVENT_SHIFT) ++#define CS_REQ_EXTRACT_EVENT_GET(reg_val) (((reg_val)&CS_REQ_EXTRACT_EVENT_MASK) >> CS_REQ_EXTRACT_EVENT_SHIFT) ++#define CS_REQ_EXTRACT_EVENT_SET(reg_val, value) \ ++ (((reg_val) & ~CS_REQ_EXTRACT_EVENT_MASK) | (((value) << CS_REQ_EXTRACT_EVENT_SHIFT) & CS_REQ_EXTRACT_EVENT_MASK)) ++ ++/* From 10.x.5, CS_REQ_ERROR_MODE is removed but TI2 bitfile upload not finished. ++ * Need to remove on GPUCORE-23972 ++ */ ++#define CS_REQ_ERROR_MODE_SHIFT 5 ++#define CS_REQ_ERROR_MODE_MASK (0x1 << CS_REQ_ERROR_MODE_SHIFT) ++#define CS_REQ_ERROR_MODE_GET(reg_val) ((reg_val & CS_REQ_ERROR_MODE_MASK) >> CS_REQ_ERROR_MODE_SHIFT) ++#define CS_REQ_ERROR_MODE_SET(reg_val, value) \ ++ ((reg_val & ~CS_REQ_ERROR_MODE_MASK) | ((value << CS_REQ_ERROR_MODE_SHIFT) & CS_REQ_ERROR_MODE_MASK)) ++ ++#define CS_REQ_IDLE_SYNC_WAIT_SHIFT 8 ++#define CS_REQ_IDLE_SYNC_WAIT_MASK (0x1 << CS_REQ_IDLE_SYNC_WAIT_SHIFT) ++#define CS_REQ_IDLE_SYNC_WAIT_GET(reg_val) (((reg_val)&CS_REQ_IDLE_SYNC_WAIT_MASK) >> CS_REQ_IDLE_SYNC_WAIT_SHIFT) ++#define CS_REQ_IDLE_SYNC_WAIT_SET(reg_val, value) \ ++ (((reg_val) & ~CS_REQ_IDLE_SYNC_WAIT_MASK) | \ ++ (((value) << CS_REQ_IDLE_SYNC_WAIT_SHIFT) & CS_REQ_IDLE_SYNC_WAIT_MASK)) ++#define CS_REQ_IDLE_PROTM_PEND_SHIFT 9 ++#define CS_REQ_IDLE_PROTM_PEND_MASK (0x1 << CS_REQ_IDLE_PROTM_PEND_SHIFT) ++#define CS_REQ_IDLE_PROTM_PEND_GET(reg_val) (((reg_val)&CS_REQ_IDLE_PROTM_PEND_MASK) >> CS_REQ_IDLE_PROTM_PEND_SHIFT) ++#define CS_REQ_IDLE_PROTM_PEND_SET(reg_val, value) \ ++ (((reg_val) & ~CS_REQ_IDLE_PROTM_PEND_MASK) | \ ++ (((value) << CS_REQ_IDLE_PROTM_PEND_SHIFT) & CS_REQ_IDLE_PROTM_PEND_MASK)) ++#define CS_REQ_IDLE_EMPTY_SHIFT 10 ++#define CS_REQ_IDLE_EMPTY_MASK (0x1 << CS_REQ_IDLE_EMPTY_SHIFT) ++#define CS_REQ_IDLE_EMPTY_GET(reg_val) (((reg_val)&CS_REQ_IDLE_EMPTY_MASK) >> CS_REQ_IDLE_EMPTY_SHIFT) ++#define CS_REQ_IDLE_EMPTY_SET(reg_val, value) \ ++ (((reg_val) & ~CS_REQ_IDLE_EMPTY_MASK) | (((value) << CS_REQ_IDLE_EMPTY_SHIFT) & CS_REQ_IDLE_EMPTY_MASK)) ++#define CS_REQ_IDLE_RESOURCE_REQ_SHIFT 11 ++#define CS_REQ_IDLE_RESOURCE_REQ_MASK (0x1 << CS_REQ_IDLE_RESOURCE_REQ_SHIFT) ++#define CS_REQ_IDLE_RESOURCE_REQ_GET(reg_val) \ ++ (((reg_val)&CS_REQ_IDLE_RESOURCE_REQ_MASK) >> CS_REQ_IDLE_RESOURCE_REQ_SHIFT) ++#define CS_REQ_IDLE_RESOURCE_REQ_SET(reg_val, value) \ ++ (((reg_val) & ~CS_REQ_IDLE_RESOURCE_REQ_MASK) | \ ++ (((value) << CS_REQ_IDLE_RESOURCE_REQ_SHIFT) & CS_REQ_IDLE_RESOURCE_REQ_MASK)) ++#define CS_REQ_TILER_OOM_SHIFT 26 ++#define CS_REQ_TILER_OOM_MASK (0x1 << CS_REQ_TILER_OOM_SHIFT) ++#define CS_REQ_TILER_OOM_GET(reg_val) (((reg_val)&CS_REQ_TILER_OOM_MASK) >> CS_REQ_TILER_OOM_SHIFT) ++#define CS_REQ_TILER_OOM_SET(reg_val, value) \ ++ (((reg_val) & ~CS_REQ_TILER_OOM_MASK) | (((value) << CS_REQ_TILER_OOM_SHIFT) & CS_REQ_TILER_OOM_MASK)) ++#define CS_REQ_PROTM_PEND_SHIFT 27 ++#define CS_REQ_PROTM_PEND_MASK (0x1 << CS_REQ_PROTM_PEND_SHIFT) ++#define CS_REQ_PROTM_PEND_GET(reg_val) (((reg_val)&CS_REQ_PROTM_PEND_MASK) >> CS_REQ_PROTM_PEND_SHIFT) ++#define CS_REQ_PROTM_PEND_SET(reg_val, value) \ ++ (((reg_val) & ~CS_REQ_PROTM_PEND_MASK) | (((value) << CS_REQ_PROTM_PEND_SHIFT) & CS_REQ_PROTM_PEND_MASK)) ++#define CS_REQ_FATAL_SHIFT 30 ++#define CS_REQ_FATAL_MASK (0x1 << CS_REQ_FATAL_SHIFT) ++#define CS_REQ_FATAL_GET(reg_val) (((reg_val)&CS_REQ_FATAL_MASK) >> CS_REQ_FATAL_SHIFT) ++#define CS_REQ_FATAL_SET(reg_val, value) \ ++ (((reg_val) & ~CS_REQ_FATAL_MASK) | (((value) << CS_REQ_FATAL_SHIFT) & CS_REQ_FATAL_MASK)) ++#define CS_REQ_FAULT_SHIFT 31 ++#define CS_REQ_FAULT_MASK (0x1 << CS_REQ_FAULT_SHIFT) ++#define CS_REQ_FAULT_GET(reg_val) (((reg_val)&CS_REQ_FAULT_MASK) >> CS_REQ_FAULT_SHIFT) ++#define CS_REQ_FAULT_SET(reg_val, value) \ ++ (((reg_val) & ~CS_REQ_FAULT_MASK) | (((value) << CS_REQ_FAULT_SHIFT) & CS_REQ_FAULT_MASK)) ++ ++/* CS_CONFIG register */ ++#define CS_CONFIG_PRIORITY_SHIFT 0 ++#define CS_CONFIG_PRIORITY_MASK (0xF << CS_CONFIG_PRIORITY_SHIFT) ++#define CS_CONFIG_PRIORITY_GET(reg_val) (((reg_val)&CS_CONFIG_PRIORITY_MASK) >> CS_CONFIG_PRIORITY_SHIFT) ++#define CS_CONFIG_PRIORITY_SET(reg_val, value) \ ++ (((reg_val) & ~CS_CONFIG_PRIORITY_MASK) | (((value) << CS_CONFIG_PRIORITY_SHIFT) & CS_CONFIG_PRIORITY_MASK)) ++#define CS_CONFIG_USER_DOORBELL_SHIFT 8 ++#define CS_CONFIG_USER_DOORBELL_MASK (0xFF << CS_CONFIG_USER_DOORBELL_SHIFT) ++#define CS_CONFIG_USER_DOORBELL_GET(reg_val) (((reg_val)&CS_CONFIG_USER_DOORBELL_MASK) >> CS_CONFIG_USER_DOORBELL_SHIFT) ++#define CS_CONFIG_USER_DOORBELL_SET(reg_val, value) \ ++ (((reg_val) & ~CS_CONFIG_USER_DOORBELL_MASK) | \ ++ (((value) << CS_CONFIG_USER_DOORBELL_SHIFT) & CS_CONFIG_USER_DOORBELL_MASK)) ++ ++/* CS_ACK_IRQ_MASK register */ ++#define CS_ACK_IRQ_MASK_STATE_SHIFT 0 ++#define CS_ACK_IRQ_MASK_STATE_MASK (0x7 << CS_ACK_IRQ_MASK_STATE_SHIFT) ++#define CS_ACK_IRQ_MASK_STATE_GET(reg_val) (((reg_val)&CS_ACK_IRQ_MASK_STATE_MASK) >> CS_ACK_IRQ_MASK_STATE_SHIFT) ++#define CS_ACK_IRQ_MASK_STATE_SET(reg_val, value) \ ++ (((reg_val) & ~CS_ACK_IRQ_MASK_STATE_MASK) | \ ++ (((value) << CS_ACK_IRQ_MASK_STATE_SHIFT) & CS_ACK_IRQ_MASK_STATE_MASK)) ++/* CS_ACK_IRQ_MASK_STATE values */ ++#define CS_ACK_IRQ_MASK_STATE_DISABLED 0x0 ++#define CS_ACK_IRQ_MASK_STATE_ENABLED 0x7 ++/* End of CS_ACK_IRQ_MASK_STATE values */ ++#define CS_ACK_IRQ_MASK_EXTRACT_EVENT_SHIFT 4 ++#define CS_ACK_IRQ_MASK_EXTRACT_EVENT_MASK (0x1 << CS_ACK_IRQ_MASK_EXTRACT_EVENT_SHIFT) ++#define CS_ACK_IRQ_MASK_EXTRACT_EVENT_GET(reg_val) \ ++ (((reg_val)&CS_ACK_IRQ_MASK_EXTRACT_EVENT_MASK) >> CS_ACK_IRQ_MASK_EXTRACT_EVENT_SHIFT) ++#define CS_ACK_IRQ_MASK_EXTRACT_EVENT_SET(reg_val, value) \ ++ (((reg_val) & ~CS_ACK_IRQ_MASK_EXTRACT_EVENT_MASK) | \ ++ (((value) << CS_ACK_IRQ_MASK_EXTRACT_EVENT_SHIFT) & CS_ACK_IRQ_MASK_EXTRACT_EVENT_MASK)) ++#define CS_ACK_IRQ_MASK_TILER_OOM_SHIFT 26 ++#define CS_ACK_IRQ_MASK_TILER_OOM_MASK (0x1 << CS_ACK_IRQ_MASK_TILER_OOM_SHIFT) ++#define CS_ACK_IRQ_MASK_TILER_OOM_GET(reg_val) \ ++ (((reg_val)&CS_ACK_IRQ_MASK_TILER_OOM_MASK) >> CS_ACK_IRQ_MASK_TILER_OOM_SHIFT) ++#define CS_ACK_IRQ_MASK_TILER_OOM_SET(reg_val, value) \ ++ (((reg_val) & ~CS_ACK_IRQ_MASK_TILER_OOM_MASK) | \ ++ (((value) << CS_ACK_IRQ_MASK_TILER_OOM_SHIFT) & CS_ACK_IRQ_MASK_TILER_OOM_MASK)) ++#define CS_ACK_IRQ_MASK_PROTM_PEND_SHIFT 27 ++#define CS_ACK_IRQ_MASK_PROTM_PEND_MASK (0x1 << CS_ACK_IRQ_MASK_PROTM_PEND_SHIFT) ++#define CS_ACK_IRQ_MASK_PROTM_PEND_GET(reg_val) \ ++ (((reg_val)&CS_ACK_IRQ_MASK_PROTM_PEND_MASK) >> CS_ACK_IRQ_MASK_PROTM_PEND_SHIFT) ++#define CS_ACK_IRQ_MASK_PROTM_PEND_SET(reg_val, value) \ ++ (((reg_val) & ~CS_ACK_IRQ_MASK_PROTM_PEND_MASK) | \ ++ (((value) << CS_ACK_IRQ_MASK_PROTM_PEND_SHIFT) & CS_ACK_IRQ_MASK_PROTM_PEND_MASK)) ++#define CS_ACK_IRQ_MASK_FATAL_SHIFT 30 ++#define CS_ACK_IRQ_MASK_FATAL_MASK (0x1 << CS_ACK_IRQ_MASK_FATAL_SHIFT) ++#define CS_ACK_IRQ_MASK_FATAL_GET(reg_val) (((reg_val)&CS_ACK_IRQ_MASK_FATAL_MASK) >> CS_ACK_IRQ_MASK_FATAL_SHIFT) ++#define CS_ACK_IRQ_MASK_FATAL_SET(reg_val, value) \ ++ (((reg_val) & ~CS_ACK_IRQ_MASK_FATAL_MASK) | \ ++ (((value) << CS_ACK_IRQ_MASK_FATAL_SHIFT) & CS_ACK_IRQ_MASK_FATAL_MASK)) ++#define CS_ACK_IRQ_MASK_FAULT_SHIFT 31 ++#define CS_ACK_IRQ_MASK_FAULT_MASK (0x1 << CS_ACK_IRQ_MASK_FAULT_SHIFT) ++#define CS_ACK_IRQ_MASK_FAULT_GET(reg_val) (((reg_val)&CS_ACK_IRQ_MASK_FAULT_MASK) >> CS_ACK_IRQ_MASK_FAULT_SHIFT) ++#define CS_ACK_IRQ_MASK_FAULT_SET(reg_val, value) \ ++ (((reg_val) & ~CS_ACK_IRQ_MASK_FAULT_MASK) | \ ++ (((value) << CS_ACK_IRQ_MASK_FAULT_SHIFT) & CS_ACK_IRQ_MASK_FAULT_MASK)) ++ ++/* CS_BASE register */ ++#define CS_BASE_POINTER_SHIFT 0 ++#define CS_BASE_POINTER_MASK (0xFFFFFFFFFFFFFFFF << CS_BASE_POINTER_SHIFT) ++#define CS_BASE_POINTER_GET(reg_val) (((reg_val)&CS_BASE_POINTER_MASK) >> CS_BASE_POINTER_SHIFT) ++#define CS_BASE_POINTER_SET(reg_val, value) \ ++ (((reg_val) & ~CS_BASE_POINTER_MASK) | (((value) << CS_BASE_POINTER_SHIFT) & CS_BASE_POINTER_MASK)) ++ ++/* CS_SIZE register */ ++#define CS_SIZE_SIZE_SHIFT 0 ++#define CS_SIZE_SIZE_MASK (0xFFFFFFFF << CS_SIZE_SIZE_SHIFT) ++#define CS_SIZE_SIZE_GET(reg_val) (((reg_val)&CS_SIZE_SIZE_MASK) >> CS_SIZE_SIZE_SHIFT) ++#define CS_SIZE_SIZE_SET(reg_val, value) \ ++ (((reg_val) & ~CS_SIZE_SIZE_MASK) | (((value) << CS_SIZE_SIZE_SHIFT) & CS_SIZE_SIZE_MASK)) ++ ++/* CS_TILER_HEAP_START register */ ++#define CS_TILER_HEAP_START_POINTER_SHIFT 0 ++#define CS_TILER_HEAP_START_POINTER_MASK (0xFFFFFFFFFFFFFFFF << CS_TILER_HEAP_START_POINTER_SHIFT) ++#define CS_TILER_HEAP_START_POINTER_GET(reg_val) \ ++ (((reg_val)&CS_TILER_HEAP_START_POINTER_MASK) >> CS_TILER_HEAP_START_POINTER_SHIFT) ++#define CS_TILER_HEAP_START_POINTER_SET(reg_val, value) \ ++ (((reg_val) & ~CS_TILER_HEAP_START_POINTER_MASK) | \ ++ (((value) << CS_TILER_HEAP_START_POINTER_SHIFT) & CS_TILER_HEAP_START_POINTER_MASK)) ++/* HeapChunkPointer nested in CS_TILER_HEAP_START_POINTER */ ++/* End of HeapChunkPointer nested in CS_TILER_HEAP_START_POINTER */ ++ ++/* CS_TILER_HEAP_END register */ ++#define CS_TILER_HEAP_END_POINTER_SHIFT 0 ++#define CS_TILER_HEAP_END_POINTER_MASK (0xFFFFFFFFFFFFFFFF << CS_TILER_HEAP_END_POINTER_SHIFT) ++#define CS_TILER_HEAP_END_POINTER_GET(reg_val) \ ++ (((reg_val)&CS_TILER_HEAP_END_POINTER_MASK) >> CS_TILER_HEAP_END_POINTER_SHIFT) ++#define CS_TILER_HEAP_END_POINTER_SET(reg_val, value) \ ++ (((reg_val) & ~CS_TILER_HEAP_END_POINTER_MASK) | \ ++ (((value) << CS_TILER_HEAP_END_POINTER_SHIFT) & CS_TILER_HEAP_END_POINTER_MASK)) ++/* HeapChunkPointer nested in CS_TILER_HEAP_END_POINTER */ ++/* End of HeapChunkPointer nested in CS_TILER_HEAP_END_POINTER */ ++ ++/* CS_USER_INPUT register */ ++#define CS_USER_INPUT_POINTER_SHIFT 0 ++#define CS_USER_INPUT_POINTER_MASK (0xFFFFFFFFFFFFFFFF << CS_USER_INPUT_POINTER_SHIFT) ++#define CS_USER_INPUT_POINTER_GET(reg_val) (((reg_val)&CS_USER_INPUT_POINTER_MASK) >> CS_USER_INPUT_POINTER_SHIFT) ++#define CS_USER_INPUT_POINTER_SET(reg_val, value) \ ++ (((reg_val) & ~CS_USER_INPUT_POINTER_MASK) | \ ++ (((value) << CS_USER_INPUT_POINTER_SHIFT) & CS_USER_INPUT_POINTER_MASK)) ++ ++/* CS_USER_OUTPUT register */ ++#define CS_USER_OUTPUT_POINTER_SHIFT 0 ++#define CS_USER_OUTPUT_POINTER_MASK (0xFFFFFFFFFFFFFFFF << CS_USER_OUTPUT_POINTER_SHIFT) ++#define CS_USER_OUTPUT_POINTER_GET(reg_val) (((reg_val)&CS_USER_OUTPUT_POINTER_MASK) >> CS_USER_OUTPUT_POINTER_SHIFT) ++#define CS_USER_OUTPUT_POINTER_SET(reg_val, value) \ ++ (((reg_val) & ~CS_USER_OUTPUT_POINTER_MASK) | \ ++ (((value) << CS_USER_OUTPUT_POINTER_SHIFT) & CS_USER_OUTPUT_POINTER_MASK)) ++/* End of CS_KERNEL_INPUT_BLOCK register set definitions */ ++ ++/* CS_KERNEL_OUTPUT_BLOCK register set definitions */ ++ ++/* CS_ACK register */ ++#define CS_ACK_STATE_SHIFT 0 ++#define CS_ACK_STATE_MASK (0x7 << CS_ACK_STATE_SHIFT) ++#define CS_ACK_STATE_GET(reg_val) (((reg_val)&CS_ACK_STATE_MASK) >> CS_ACK_STATE_SHIFT) ++#define CS_ACK_STATE_SET(reg_val, value) \ ++ (((reg_val) & ~CS_ACK_STATE_MASK) | (((value) << CS_ACK_STATE_SHIFT) & CS_ACK_STATE_MASK)) ++/* CS_ACK_STATE values */ ++#define CS_ACK_STATE_STOP 0x0 ++#define CS_ACK_STATE_START 0x1 ++/* End of CS_ACK_STATE values */ ++#define CS_ACK_EXTRACT_EVENT_SHIFT 4 ++#define CS_ACK_EXTRACT_EVENT_MASK (0x1 << CS_ACK_EXTRACT_EVENT_SHIFT) ++#define CS_ACK_EXTRACT_EVENT_GET(reg_val) (((reg_val)&CS_ACK_EXTRACT_EVENT_MASK) >> CS_ACK_EXTRACT_EVENT_SHIFT) ++#define CS_ACK_EXTRACT_EVENT_SET(reg_val, value) \ ++ (((reg_val) & ~CS_ACK_EXTRACT_EVENT_MASK) | (((value) << CS_ACK_EXTRACT_EVENT_SHIFT) & CS_ACK_EXTRACT_EVENT_MASK)) ++#define CS_ACK_TILER_OOM_SHIFT 26 ++#define CS_ACK_TILER_OOM_MASK (0x1 << CS_ACK_TILER_OOM_SHIFT) ++#define CS_ACK_TILER_OOM_GET(reg_val) (((reg_val)&CS_ACK_TILER_OOM_MASK) >> CS_ACK_TILER_OOM_SHIFT) ++#define CS_ACK_TILER_OOM_SET(reg_val, value) \ ++ (((reg_val) & ~CS_ACK_TILER_OOM_MASK) | (((value) << CS_ACK_TILER_OOM_SHIFT) & CS_ACK_TILER_OOM_MASK)) ++#define CS_ACK_PROTM_PEND_SHIFT 27 ++#define CS_ACK_PROTM_PEND_MASK (0x1 << CS_ACK_PROTM_PEND_SHIFT) ++#define CS_ACK_PROTM_PEND_GET(reg_val) (((reg_val)&CS_ACK_PROTM_PEND_MASK) >> CS_ACK_PROTM_PEND_SHIFT) ++#define CS_ACK_PROTM_PEND_SET(reg_val, value) \ ++ (((reg_val) & ~CS_ACK_PROTM_PEND_MASK) | (((value) << CS_ACK_PROTM_PEND_SHIFT) & CS_ACK_PROTM_PEND_MASK)) ++#define CS_ACK_FATAL_SHIFT 30 ++#define CS_ACK_FATAL_MASK (0x1 << CS_ACK_FATAL_SHIFT) ++#define CS_ACK_FATAL_GET(reg_val) (((reg_val)&CS_ACK_FATAL_MASK) >> CS_ACK_FATAL_SHIFT) ++#define CS_ACK_FATAL_SET(reg_val, value) \ ++ (((reg_val) & ~CS_ACK_FATAL_MASK) | (((value) << CS_ACK_FATAL_SHIFT) & CS_ACK_FATAL_MASK)) ++#define CS_ACK_FAULT_SHIFT 31 ++#define CS_ACK_FAULT_MASK (0x1 << CS_ACK_FAULT_SHIFT) ++#define CS_ACK_FAULT_GET(reg_val) (((reg_val)&CS_ACK_FAULT_MASK) >> CS_ACK_FAULT_SHIFT) ++#define CS_ACK_FAULT_SET(reg_val, value) \ ++ (((reg_val) & ~CS_ACK_FAULT_MASK) | (((value) << CS_ACK_FAULT_SHIFT) & CS_ACK_FAULT_MASK)) ++ ++/* CS_STATUS_CMD_PTR register */ ++#define CS_STATUS_CMD_PTR_POINTER_SHIFT 0 ++#define CS_STATUS_CMD_PTR_POINTER_MASK (0xFFFFFFFFFFFFFFFF << CS_STATUS_CMD_PTR_POINTER_SHIFT) ++#define CS_STATUS_CMD_PTR_POINTER_GET(reg_val) \ ++ (((reg_val)&CS_STATUS_CMD_PTR_POINTER_MASK) >> CS_STATUS_CMD_PTR_POINTER_SHIFT) ++#define CS_STATUS_CMD_PTR_POINTER_SET(reg_val, value) \ ++ (((reg_val) & ~CS_STATUS_CMD_PTR_POINTER_MASK) | \ ++ (((value) << CS_STATUS_CMD_PTR_POINTER_SHIFT) & CS_STATUS_CMD_PTR_POINTER_MASK)) ++ ++/* CS_STATUS_WAIT register */ ++#define CS_STATUS_WAIT_SB_MASK_SHIFT 0 ++#define CS_STATUS_WAIT_SB_MASK_MASK (0xFFFF << CS_STATUS_WAIT_SB_MASK_SHIFT) ++#define CS_STATUS_WAIT_SB_MASK_GET(reg_val) (((reg_val)&CS_STATUS_WAIT_SB_MASK_MASK) >> CS_STATUS_WAIT_SB_MASK_SHIFT) ++#define CS_STATUS_WAIT_SB_MASK_SET(reg_val, value) \ ++ (((reg_val) & ~CS_STATUS_WAIT_SB_MASK_MASK) | \ ++ (((value) << CS_STATUS_WAIT_SB_MASK_SHIFT) & CS_STATUS_WAIT_SB_MASK_MASK)) ++#define CS_STATUS_WAIT_SYNC_WAIT_CONDITION_SHIFT 24 ++#define CS_STATUS_WAIT_SYNC_WAIT_CONDITION_MASK (0xF << CS_STATUS_WAIT_SYNC_WAIT_CONDITION_SHIFT) ++#define CS_STATUS_WAIT_SYNC_WAIT_CONDITION_GET(reg_val) \ ++ (((reg_val)&CS_STATUS_WAIT_SYNC_WAIT_CONDITION_MASK) >> CS_STATUS_WAIT_SYNC_WAIT_CONDITION_SHIFT) ++#define CS_STATUS_WAIT_SYNC_WAIT_CONDITION_SET(reg_val, value) \ ++ (((reg_val) & ~CS_STATUS_WAIT_SYNC_WAIT_CONDITION_MASK) | \ ++ (((value) << CS_STATUS_WAIT_SYNC_WAIT_CONDITION_SHIFT) & CS_STATUS_WAIT_SYNC_WAIT_CONDITION_MASK)) ++/* CS_STATUS_WAIT_SYNC_WAIT_CONDITION values */ ++#define CS_STATUS_WAIT_SYNC_WAIT_CONDITION_LE 0x0 ++#define CS_STATUS_WAIT_SYNC_WAIT_CONDITION_GT 0x1 ++/* End of CS_STATUS_WAIT_SYNC_WAIT_CONDITION values */ ++#define CS_STATUS_WAIT_PROGRESS_WAIT_SHIFT 28 ++#define CS_STATUS_WAIT_PROGRESS_WAIT_MASK (0x1 << CS_STATUS_WAIT_PROGRESS_WAIT_SHIFT) ++#define CS_STATUS_WAIT_PROGRESS_WAIT_GET(reg_val) \ ++ (((reg_val)&CS_STATUS_WAIT_PROGRESS_WAIT_MASK) >> CS_STATUS_WAIT_PROGRESS_WAIT_SHIFT) ++#define CS_STATUS_WAIT_PROGRESS_WAIT_SET(reg_val, value) \ ++ (((reg_val) & ~CS_STATUS_WAIT_PROGRESS_WAIT_MASK) | \ ++ (((value) << CS_STATUS_WAIT_PROGRESS_WAIT_SHIFT) & CS_STATUS_WAIT_PROGRESS_WAIT_MASK)) ++#define CS_STATUS_WAIT_PROTM_PEND_SHIFT 29 ++#define CS_STATUS_WAIT_PROTM_PEND_MASK (0x1 << CS_STATUS_WAIT_PROTM_PEND_SHIFT) ++#define CS_STATUS_WAIT_PROTM_PEND_GET(reg_val) \ ++ (((reg_val)&CS_STATUS_WAIT_PROTM_PEND_MASK) >> CS_STATUS_WAIT_PROTM_PEND_SHIFT) ++#define CS_STATUS_WAIT_PROTM_PEND_SET(reg_val, value) \ ++ (((reg_val) & ~CS_STATUS_WAIT_PROTM_PEND_MASK) | \ ++ (((value) << CS_STATUS_WAIT_PROTM_PEND_SHIFT) & CS_STATUS_WAIT_PROTM_PEND_MASK)) ++#define CS_STATUS_WAIT_SYNC_WAIT_SHIFT 31 ++#define CS_STATUS_WAIT_SYNC_WAIT_MASK (0x1 << CS_STATUS_WAIT_SYNC_WAIT_SHIFT) ++#define CS_STATUS_WAIT_SYNC_WAIT_GET(reg_val) \ ++ (((reg_val)&CS_STATUS_WAIT_SYNC_WAIT_MASK) >> CS_STATUS_WAIT_SYNC_WAIT_SHIFT) ++#define CS_STATUS_WAIT_SYNC_WAIT_SET(reg_val, value) \ ++ (((reg_val) & ~CS_STATUS_WAIT_SYNC_WAIT_MASK) | \ ++ (((value) << CS_STATUS_WAIT_SYNC_WAIT_SHIFT) & CS_STATUS_WAIT_SYNC_WAIT_MASK)) ++ ++/* CS_STATUS_REQ_RESOURCE register */ ++#define CS_STATUS_REQ_RESOURCE_COMPUTE_RESOURCES_SHIFT 0 ++#define CS_STATUS_REQ_RESOURCE_COMPUTE_RESOURCES_MASK (0x1 << CS_STATUS_REQ_RESOURCE_COMPUTE_RESOURCES_SHIFT) ++#define CS_STATUS_REQ_RESOURCE_COMPUTE_RESOURCES_GET(reg_val) \ ++ (((reg_val)&CS_STATUS_REQ_RESOURCE_COMPUTE_RESOURCES_MASK) >> CS_STATUS_REQ_RESOURCE_COMPUTE_RESOURCES_SHIFT) ++#define CS_STATUS_REQ_RESOURCE_COMPUTE_RESOURCES_SET(reg_val, value) \ ++ (((reg_val) & ~CS_STATUS_REQ_RESOURCE_COMPUTE_RESOURCES_MASK) | \ ++ (((value) << CS_STATUS_REQ_RESOURCE_COMPUTE_RESOURCES_SHIFT) & CS_STATUS_REQ_RESOURCE_COMPUTE_RESOURCES_MASK)) ++#define CS_STATUS_REQ_RESOURCE_FRAGMENT_RESOURCES_SHIFT 1 ++#define CS_STATUS_REQ_RESOURCE_FRAGMENT_RESOURCES_MASK (0x1 << CS_STATUS_REQ_RESOURCE_FRAGMENT_RESOURCES_SHIFT) ++#define CS_STATUS_REQ_RESOURCE_FRAGMENT_RESOURCES_GET(reg_val) \ ++ (((reg_val)&CS_STATUS_REQ_RESOURCE_FRAGMENT_RESOURCES_MASK) >> CS_STATUS_REQ_RESOURCE_FRAGMENT_RESOURCES_SHIFT) ++#define CS_STATUS_REQ_RESOURCE_FRAGMENT_RESOURCES_SET(reg_val, value) \ ++ (((reg_val) & ~CS_STATUS_REQ_RESOURCE_FRAGMENT_RESOURCES_MASK) | \ ++ (((value) << CS_STATUS_REQ_RESOURCE_FRAGMENT_RESOURCES_SHIFT) & CS_STATUS_REQ_RESOURCE_FRAGMENT_RESOURCES_MASK)) ++#define CS_STATUS_REQ_RESOURCE_TILER_RESOURCES_SHIFT 2 ++#define CS_STATUS_REQ_RESOURCE_TILER_RESOURCES_MASK (0x1 << CS_STATUS_REQ_RESOURCE_TILER_RESOURCES_SHIFT) ++#define CS_STATUS_REQ_RESOURCE_TILER_RESOURCES_GET(reg_val) \ ++ (((reg_val)&CS_STATUS_REQ_RESOURCE_TILER_RESOURCES_MASK) >> CS_STATUS_REQ_RESOURCE_TILER_RESOURCES_SHIFT) ++#define CS_STATUS_REQ_RESOURCE_TILER_RESOURCES_SET(reg_val, value) \ ++ (((reg_val) & ~CS_STATUS_REQ_RESOURCE_TILER_RESOURCES_MASK) | \ ++ (((value) << CS_STATUS_REQ_RESOURCE_TILER_RESOURCES_SHIFT) & CS_STATUS_REQ_RESOURCE_TILER_RESOURCES_MASK)) ++#define CS_STATUS_REQ_RESOURCE_IDVS_RESOURCES_SHIFT 3 ++#define CS_STATUS_REQ_RESOURCE_IDVS_RESOURCES_MASK (0x1 << CS_STATUS_REQ_RESOURCE_IDVS_RESOURCES_SHIFT) ++#define CS_STATUS_REQ_RESOURCE_IDVS_RESOURCES_GET(reg_val) \ ++ (((reg_val)&CS_STATUS_REQ_RESOURCE_IDVS_RESOURCES_MASK) >> CS_STATUS_REQ_RESOURCE_IDVS_RESOURCES_SHIFT) ++#define CS_STATUS_REQ_RESOURCE_IDVS_RESOURCES_SET(reg_val, value) \ ++ (((reg_val) & ~CS_STATUS_REQ_RESOURCE_IDVS_RESOURCES_MASK) | \ ++ (((value) << CS_STATUS_REQ_RESOURCE_IDVS_RESOURCES_SHIFT) & CS_STATUS_REQ_RESOURCE_IDVS_RESOURCES_MASK)) ++ ++/* CS_STATUS_WAIT_SYNC_POINTER register */ ++#define CS_STATUS_WAIT_SYNC_POINTER_POINTER_SHIFT 0 ++#define CS_STATUS_WAIT_SYNC_POINTER_POINTER_MASK (0xFFFFFFFFFFFFFFFF << CS_STATUS_WAIT_SYNC_POINTER_POINTER_SHIFT) ++#define CS_STATUS_WAIT_SYNC_POINTER_POINTER_GET(reg_val) \ ++ (((reg_val)&CS_STATUS_WAIT_SYNC_POINTER_POINTER_MASK) >> CS_STATUS_WAIT_SYNC_POINTER_POINTER_SHIFT) ++#define CS_STATUS_WAIT_SYNC_POINTER_POINTER_SET(reg_val, value) \ ++ (((reg_val) & ~CS_STATUS_WAIT_SYNC_POINTER_POINTER_MASK) | \ ++ (((value) << CS_STATUS_WAIT_SYNC_POINTER_POINTER_SHIFT) & CS_STATUS_WAIT_SYNC_POINTER_POINTER_MASK)) ++ ++/* CS_STATUS_WAIT_SYNC_VALUE register */ ++#define CS_STATUS_WAIT_SYNC_VALUE_VALUE_SHIFT 0 ++#define CS_STATUS_WAIT_SYNC_VALUE_VALUE_MASK (0xFFFFFFFF << CS_STATUS_WAIT_SYNC_VALUE_VALUE_SHIFT) ++#define CS_STATUS_WAIT_SYNC_VALUE_VALUE_GET(reg_val) \ ++ (((reg_val)&CS_STATUS_WAIT_SYNC_VALUE_VALUE_MASK) >> CS_STATUS_WAIT_SYNC_VALUE_VALUE_SHIFT) ++#define CS_STATUS_WAIT_SYNC_VALUE_VALUE_SET(reg_val, value) \ ++ (((reg_val) & ~CS_STATUS_WAIT_SYNC_VALUE_VALUE_MASK) | \ ++ (((value) << CS_STATUS_WAIT_SYNC_VALUE_VALUE_SHIFT) & CS_STATUS_WAIT_SYNC_VALUE_VALUE_MASK)) ++ ++/* CS_FAULT register */ ++#define CS_FAULT_EXCEPTION_TYPE_SHIFT 0 ++#define CS_FAULT_EXCEPTION_TYPE_MASK (0xFF << CS_FAULT_EXCEPTION_TYPE_SHIFT) ++#define CS_FAULT_EXCEPTION_TYPE_GET(reg_val) (((reg_val)&CS_FAULT_EXCEPTION_TYPE_MASK) >> CS_FAULT_EXCEPTION_TYPE_SHIFT) ++#define CS_FAULT_EXCEPTION_TYPE_SET(reg_val, value) \ ++ (((reg_val) & ~CS_FAULT_EXCEPTION_TYPE_MASK) | \ ++ (((value) << CS_FAULT_EXCEPTION_TYPE_SHIFT) & CS_FAULT_EXCEPTION_TYPE_MASK)) ++/* CS_FAULT_EXCEPTION_TYPE values */ ++#define CS_FAULT_EXCEPTION_TYPE_CS_RESOURCE_TERMINATED 0x0F ++#define CS_FAULT_EXCEPTION_TYPE_CS_INHERIT_FAULT 0x4B ++#define CS_FAULT_EXCEPTION_TYPE_INSTR_INVALID_PC 0x50 ++#define CS_FAULT_EXCEPTION_TYPE_INSTR_INVALID_ENC 0x51 ++#define CS_FAULT_EXCEPTION_TYPE_INSTR_BARRIER_FAULT 0x55 ++#define CS_FAULT_EXCEPTION_TYPE_DATA_INVALID_FAULT 0x58 ++#define CS_FAULT_EXCEPTION_TYPE_TILE_RANGE_FAULT 0x59 ++#define CS_FAULT_EXCEPTION_TYPE_ADDR_RANGE_FAULT 0x5A ++#define CS_FAULT_EXCEPTION_TYPE_IMPRECISE_FAULT 0x5B ++#define CS_FAULT_EXCEPTION_TYPE_RESOURCE_EVICTION_TIMEOUT 0x69 ++/* End of CS_FAULT_EXCEPTION_TYPE values */ ++#define CS_FAULT_EXCEPTION_DATA_SHIFT 8 ++#define CS_FAULT_EXCEPTION_DATA_MASK (0xFFFFFF << CS_FAULT_EXCEPTION_DATA_SHIFT) ++#define CS_FAULT_EXCEPTION_DATA_GET(reg_val) (((reg_val)&CS_FAULT_EXCEPTION_DATA_MASK) >> CS_FAULT_EXCEPTION_DATA_SHIFT) ++#define CS_FAULT_EXCEPTION_DATA_SET(reg_val, value) \ ++ (((reg_val) & ~CS_FAULT_EXCEPTION_DATA_MASK) | \ ++ (((value) << CS_FAULT_EXCEPTION_DATA_SHIFT) & CS_FAULT_EXCEPTION_DATA_MASK)) ++ ++/* CS_FATAL register */ ++#define CS_FATAL_EXCEPTION_TYPE_SHIFT 0 ++#define CS_FATAL_EXCEPTION_TYPE_MASK (0xFF << CS_FATAL_EXCEPTION_TYPE_SHIFT) ++#define CS_FATAL_EXCEPTION_TYPE_GET(reg_val) (((reg_val)&CS_FATAL_EXCEPTION_TYPE_MASK) >> CS_FATAL_EXCEPTION_TYPE_SHIFT) ++#define CS_FATAL_EXCEPTION_TYPE_SET(reg_val, value) \ ++ (((reg_val) & ~CS_FATAL_EXCEPTION_TYPE_MASK) | \ ++ (((value) << CS_FATAL_EXCEPTION_TYPE_SHIFT) & CS_FATAL_EXCEPTION_TYPE_MASK)) ++/* CS_FATAL_EXCEPTION_TYPE values */ ++#define CS_FATAL_EXCEPTION_TYPE_CS_CONFIG_FAULT 0x40 ++#define CS_FATAL_EXCEPTION_TYPE_CS_ENDPOINT_FAULT 0x44 ++#define CS_FATAL_EXCEPTION_TYPE_CS_BUS_FAULT 0x48 ++#define CS_FATAL_EXCEPTION_TYPE_CS_INVALID_INSTRUCTION 0x49 ++#define CS_FATAL_EXCEPTION_TYPE_CS_CALL_STACK_OVERFLOW 0x4A ++#define CS_FATAL_EXCEPTION_TYPE_FIRMWARE_INTERNAL_ERROR 0x68 ++/* End of CS_FATAL_EXCEPTION_TYPE values */ ++#define CS_FATAL_EXCEPTION_DATA_SHIFT 8 ++#define CS_FATAL_EXCEPTION_DATA_MASK (0xFFFFFF << CS_FATAL_EXCEPTION_DATA_SHIFT) ++#define CS_FATAL_EXCEPTION_DATA_GET(reg_val) (((reg_val)&CS_FATAL_EXCEPTION_DATA_MASK) >> CS_FATAL_EXCEPTION_DATA_SHIFT) ++#define CS_FATAL_EXCEPTION_DATA_SET(reg_val, value) \ ++ (((reg_val) & ~CS_FATAL_EXCEPTION_DATA_MASK) | \ ++ (((value) << CS_FATAL_EXCEPTION_DATA_SHIFT) & CS_FATAL_EXCEPTION_DATA_MASK)) ++ ++/* CS_FAULT_INFO register */ ++#define CS_FAULT_INFO_EXCEPTION_DATA_SHIFT 0 ++#define CS_FAULT_INFO_EXCEPTION_DATA_MASK (0xFFFFFFFFFFFFFFFF << CS_FAULT_INFO_EXCEPTION_DATA_SHIFT) ++#define CS_FAULT_INFO_EXCEPTION_DATA_GET(reg_val) \ ++ (((reg_val)&CS_FAULT_INFO_EXCEPTION_DATA_MASK) >> CS_FAULT_INFO_EXCEPTION_DATA_SHIFT) ++#define CS_FAULT_INFO_EXCEPTION_DATA_SET(reg_val, value) \ ++ (((reg_val) & ~CS_FAULT_INFO_EXCEPTION_DATA_MASK) | \ ++ (((value) << CS_FAULT_INFO_EXCEPTION_DATA_SHIFT) & CS_FAULT_INFO_EXCEPTION_DATA_MASK)) ++ ++/* CS_FATAL_INFO register */ ++#define CS_FATAL_INFO_EXCEPTION_DATA_SHIFT 0 ++#define CS_FATAL_INFO_EXCEPTION_DATA_MASK (0xFFFFFFFFFFFFFFFF << CS_FATAL_INFO_EXCEPTION_DATA_SHIFT) ++#define CS_FATAL_INFO_EXCEPTION_DATA_GET(reg_val) \ ++ (((reg_val)&CS_FATAL_INFO_EXCEPTION_DATA_MASK) >> CS_FATAL_INFO_EXCEPTION_DATA_SHIFT) ++#define CS_FATAL_INFO_EXCEPTION_DATA_SET(reg_val, value) \ ++ (((reg_val) & ~CS_FATAL_INFO_EXCEPTION_DATA_MASK) | \ ++ (((value) << CS_FATAL_INFO_EXCEPTION_DATA_SHIFT) & CS_FATAL_INFO_EXCEPTION_DATA_MASK)) ++ ++/* CS_HEAP_VT_START register */ ++#define CS_HEAP_VT_START_VALUE_SHIFT 0 ++#define CS_HEAP_VT_START_VALUE_MASK (0xFFFFFFFF << CS_HEAP_VT_START_VALUE_SHIFT) ++#define CS_HEAP_VT_START_VALUE_GET(reg_val) (((reg_val)&CS_HEAP_VT_START_VALUE_MASK) >> CS_HEAP_VT_START_VALUE_SHIFT) ++#define CS_HEAP_VT_START_VALUE_SET(reg_val, value) \ ++ (((reg_val) & ~CS_HEAP_VT_START_VALUE_MASK) | \ ++ (((value) << CS_HEAP_VT_START_VALUE_SHIFT) & CS_HEAP_VT_START_VALUE_MASK)) ++ ++/* CS_HEAP_VT_END register */ ++#define CS_HEAP_VT_END_VALUE_SHIFT 0 ++#define CS_HEAP_VT_END_VALUE_MASK (0xFFFFFFFF << CS_HEAP_VT_END_VALUE_SHIFT) ++#define CS_HEAP_VT_END_VALUE_GET(reg_val) (((reg_val)&CS_HEAP_VT_END_VALUE_MASK) >> CS_HEAP_VT_END_VALUE_SHIFT) ++#define CS_HEAP_VT_END_VALUE_SET(reg_val, value) \ ++ (((reg_val) & ~CS_HEAP_VT_END_VALUE_MASK) | (((value) << CS_HEAP_VT_END_VALUE_SHIFT) & CS_HEAP_VT_END_VALUE_MASK)) ++ ++/* CS_HEAP_FRAG_END register */ ++#define CS_HEAP_FRAG_END_VALUE_SHIFT 0 ++#define CS_HEAP_FRAG_END_VALUE_MASK (0xFFFFFFFF << CS_HEAP_FRAG_END_VALUE_SHIFT) ++#define CS_HEAP_FRAG_END_VALUE_GET(reg_val) (((reg_val)&CS_HEAP_FRAG_END_VALUE_MASK) >> CS_HEAP_FRAG_END_VALUE_SHIFT) ++#define CS_HEAP_FRAG_END_VALUE_SET(reg_val, value) \ ++ (((reg_val) & ~CS_HEAP_FRAG_END_VALUE_MASK) | \ ++ (((value) << CS_HEAP_FRAG_END_VALUE_SHIFT) & CS_HEAP_FRAG_END_VALUE_MASK)) ++ ++/* CS_HEAP_ADDRESS register */ ++#define CS_HEAP_ADDRESS_POINTER_SHIFT 0 ++#define CS_HEAP_ADDRESS_POINTER_MASK (0xFFFFFFFFFFFFFFFF << CS_HEAP_ADDRESS_POINTER_SHIFT) ++#define CS_HEAP_ADDRESS_POINTER_GET(reg_val) (((reg_val)&CS_HEAP_ADDRESS_POINTER_MASK) >> CS_HEAP_ADDRESS_POINTER_SHIFT) ++#define CS_HEAP_ADDRESS_POINTER_SET(reg_val, value) \ ++ (((reg_val) & ~CS_HEAP_ADDRESS_POINTER_MASK) | \ ++ (((value) << CS_HEAP_ADDRESS_POINTER_SHIFT) & CS_HEAP_ADDRESS_POINTER_MASK)) ++/* End of CS_KERNEL_OUTPUT_BLOCK register set definitions */ ++ ++/* CS_USER_INPUT_BLOCK register set definitions */ ++ ++/* CS_INSERT register */ ++#define CS_INSERT_VALUE_SHIFT 0 ++#define CS_INSERT_VALUE_MASK (0xFFFFFFFFFFFFFFFF << CS_INSERT_VALUE_SHIFT) ++#define CS_INSERT_VALUE_GET(reg_val) (((reg_val)&CS_INSERT_VALUE_MASK) >> CS_INSERT_VALUE_SHIFT) ++#define CS_INSERT_VALUE_SET(reg_val, value) \ ++ (((reg_val) & ~CS_INSERT_VALUE_MASK) | (((value) << CS_INSERT_VALUE_SHIFT) & CS_INSERT_VALUE_MASK)) ++ ++/* CS_EXTRACT_INIT register */ ++#define CS_EXTRACT_INIT_VALUE_SHIFT 0 ++#define CS_EXTRACT_INIT_VALUE_MASK (0xFFFFFFFFFFFFFFFF << CS_EXTRACT_INIT_VALUE_SHIFT) ++#define CS_EXTRACT_INIT_VALUE_GET(reg_val) (((reg_val)&CS_EXTRACT_INIT_VALUE_MASK) >> CS_EXTRACT_INIT_VALUE_SHIFT) ++#define CS_EXTRACT_INIT_VALUE_SET(reg_val, value) \ ++ (((reg_val) & ~CS_EXTRACT_INIT_VALUE_MASK) | \ ++ (((value) << CS_EXTRACT_INIT_VALUE_SHIFT) & CS_EXTRACT_INIT_VALUE_MASK)) ++/* End of CS_USER_INPUT_BLOCK register set definitions */ ++ ++/* CS_USER_OUTPUT_BLOCK register set definitions */ ++ ++/* CS_EXTRACT register */ ++#define CS_EXTRACT_VALUE_SHIFT 0 ++#define CS_EXTRACT_VALUE_MASK (0xFFFFFFFFFFFFFFFF << CS_EXTRACT_VALUE_SHIFT) ++#define CS_EXTRACT_VALUE_GET(reg_val) (((reg_val)&CS_EXTRACT_VALUE_MASK) >> CS_EXTRACT_VALUE_SHIFT) ++#define CS_EXTRACT_VALUE_SET(reg_val, value) \ ++ (((reg_val) & ~CS_EXTRACT_VALUE_MASK) | (((value) << CS_EXTRACT_VALUE_SHIFT) & CS_EXTRACT_VALUE_MASK)) ++ ++/* CS_ACTIVE register */ ++#define CS_ACTIVE_HW_ACTIVE_SHIFT 0 ++#define CS_ACTIVE_HW_ACTIVE_MASK (0x1 << CS_ACTIVE_HW_ACTIVE_SHIFT) ++#define CS_ACTIVE_HW_ACTIVE_GET(reg_val) (((reg_val)&CS_ACTIVE_HW_ACTIVE_MASK) >> CS_ACTIVE_HW_ACTIVE_SHIFT) ++#define CS_ACTIVE_HW_ACTIVE_SET(reg_val, value) \ ++ (((reg_val) & ~CS_ACTIVE_HW_ACTIVE_MASK) | (((value) << CS_ACTIVE_HW_ACTIVE_SHIFT) & CS_ACTIVE_HW_ACTIVE_MASK)) ++/* End of CS_USER_OUTPUT_BLOCK register set definitions */ ++ ++/* CSG_INPUT_BLOCK register set definitions */ ++ ++/* CSG_REQ register */ ++#define CSG_REQ_STATE_SHIFT 0 ++#define CSG_REQ_STATE_MASK (0x7 << CSG_REQ_STATE_SHIFT) ++#define CSG_REQ_STATE_GET(reg_val) (((reg_val)&CSG_REQ_STATE_MASK) >> CSG_REQ_STATE_SHIFT) ++#define CSG_REQ_STATE_SET(reg_val, value) \ ++ (((reg_val) & ~CSG_REQ_STATE_MASK) | (((value) << CSG_REQ_STATE_SHIFT) & CSG_REQ_STATE_MASK)) ++/* CSG_REQ_STATE values */ ++#define CSG_REQ_STATE_TERMINATE 0x0 ++#define CSG_REQ_STATE_START 0x1 ++#define CSG_REQ_STATE_SUSPEND 0x2 ++#define CSG_REQ_STATE_RESUME 0x3 ++/* End of CSG_REQ_STATE values */ ++#define CSG_REQ_EP_CFG_SHIFT 4 ++#define CSG_REQ_EP_CFG_MASK (0x1 << CSG_REQ_EP_CFG_SHIFT) ++#define CSG_REQ_EP_CFG_GET(reg_val) (((reg_val)&CSG_REQ_EP_CFG_MASK) >> CSG_REQ_EP_CFG_SHIFT) ++#define CSG_REQ_EP_CFG_SET(reg_val, value) \ ++ (((reg_val) & ~CSG_REQ_EP_CFG_MASK) | (((value) << CSG_REQ_EP_CFG_SHIFT) & CSG_REQ_EP_CFG_MASK)) ++#define CSG_REQ_STATUS_UPDATE_SHIFT 5 ++#define CSG_REQ_STATUS_UPDATE_MASK (0x1 << CSG_REQ_STATUS_UPDATE_SHIFT) ++#define CSG_REQ_STATUS_UPDATE_GET(reg_val) (((reg_val)&CSG_REQ_STATUS_UPDATE_MASK) >> CSG_REQ_STATUS_UPDATE_SHIFT) ++#define CSG_REQ_STATUS_UPDATE_SET(reg_val, value) \ ++ (((reg_val) & ~CSG_REQ_STATUS_UPDATE_MASK) | \ ++ (((value) << CSG_REQ_STATUS_UPDATE_SHIFT) & CSG_REQ_STATUS_UPDATE_MASK)) ++#define CSG_REQ_SYNC_UPDATE_SHIFT 28 ++#define CSG_REQ_SYNC_UPDATE_MASK (0x1 << CSG_REQ_SYNC_UPDATE_SHIFT) ++#define CSG_REQ_SYNC_UPDATE_GET(reg_val) (((reg_val)&CSG_REQ_SYNC_UPDATE_MASK) >> CSG_REQ_SYNC_UPDATE_SHIFT) ++#define CSG_REQ_SYNC_UPDATE_SET(reg_val, value) \ ++ (((reg_val) & ~CSG_REQ_SYNC_UPDATE_MASK) | (((value) << CSG_REQ_SYNC_UPDATE_SHIFT) & CSG_REQ_SYNC_UPDATE_MASK)) ++#define CSG_REQ_IDLE_SHIFT 29 ++#define CSG_REQ_IDLE_MASK (0x1 << CSG_REQ_IDLE_SHIFT) ++#define CSG_REQ_IDLE_GET(reg_val) (((reg_val)&CSG_REQ_IDLE_MASK) >> CSG_REQ_IDLE_SHIFT) ++#define CSG_REQ_IDLE_SET(reg_val, value) \ ++ (((reg_val) & ~CSG_REQ_IDLE_MASK) | (((value) << CSG_REQ_IDLE_SHIFT) & CSG_REQ_IDLE_MASK)) ++#define CSG_REQ_DOORBELL_SHIFT 30 ++#define CSG_REQ_DOORBELL_MASK (0x1 << CSG_REQ_DOORBELL_SHIFT) ++#define CSG_REQ_DOORBELL_GET(reg_val) (((reg_val)&CSG_REQ_DOORBELL_MASK) >> CSG_REQ_DOORBELL_SHIFT) ++#define CSG_REQ_DOORBELL_SET(reg_val, value) \ ++ (((reg_val) & ~CSG_REQ_DOORBELL_MASK) | (((value) << CSG_REQ_DOORBELL_SHIFT) & CSG_REQ_DOORBELL_MASK)) ++#define CSG_REQ_PROGRESS_TIMER_EVENT_SHIFT 31 ++#define CSG_REQ_PROGRESS_TIMER_EVENT_MASK (0x1 << CSG_REQ_PROGRESS_TIMER_EVENT_SHIFT) ++#define CSG_REQ_PROGRESS_TIMER_EVENT_GET(reg_val) \ ++ (((reg_val)&CSG_REQ_PROGRESS_TIMER_EVENT_MASK) >> CSG_REQ_PROGRESS_TIMER_EVENT_SHIFT) ++#define CSG_REQ_PROGRESS_TIMER_EVENT_SET(reg_val, value) \ ++ (((reg_val) & ~CSG_REQ_PROGRESS_TIMER_EVENT_MASK) | \ ++ (((value) << CSG_REQ_PROGRESS_TIMER_EVENT_SHIFT) & CSG_REQ_PROGRESS_TIMER_EVENT_MASK)) ++ ++/* CSG_ACK_IRQ_MASK register */ ++#define CSG_ACK_IRQ_MASK_STATE_SHIFT 0 ++#define CSG_ACK_IRQ_MASK_STATE_MASK (0x7 << CSG_ACK_IRQ_MASK_STATE_SHIFT) ++#define CSG_ACK_IRQ_MASK_STATE_GET(reg_val) (((reg_val)&CSG_ACK_IRQ_MASK_STATE_MASK) >> CSG_ACK_IRQ_MASK_STATE_SHIFT) ++#define CSG_ACK_IRQ_MASK_STATE_SET(reg_val, value) \ ++ (((reg_val) & ~CSG_ACK_IRQ_MASK_STATE_MASK) | \ ++ (((value) << CSG_ACK_IRQ_MASK_STATE_SHIFT) & CSG_ACK_IRQ_MASK_STATE_MASK)) ++/* CSG_ACK_IRQ_MASK_STATE values */ ++#define CSG_ACK_IRQ_MASK_STATE_DISABLED 0x0 ++#define CSG_ACK_IRQ_MASK_STATE_ENABLED 0x7 ++/* End of CSG_ACK_IRQ_MASK_STATE values */ ++#define CSG_ACK_IRQ_MASK_EP_CFG_SHIFT 4 ++#define CSG_ACK_IRQ_MASK_EP_CFG_MASK (0x1 << CSG_ACK_IRQ_MASK_EP_CFG_SHIFT) ++#define CSG_ACK_IRQ_MASK_EP_CFG_GET(reg_val) (((reg_val)&CSG_ACK_IRQ_MASK_EP_CFG_MASK) >> CSG_ACK_IRQ_MASK_EP_CFG_SHIFT) ++#define CSG_ACK_IRQ_MASK_EP_CFG_SET(reg_val, value) \ ++ (((reg_val) & ~CSG_ACK_IRQ_MASK_EP_CFG_MASK) | \ ++ (((value) << CSG_ACK_IRQ_MASK_EP_CFG_SHIFT) & CSG_ACK_IRQ_MASK_EP_CFG_MASK)) ++#define CSG_ACK_IRQ_MASK_STATUS_UPDATE_SHIFT 5 ++#define CSG_ACK_IRQ_MASK_STATUS_UPDATE_MASK (0x1 << CSG_ACK_IRQ_MASK_STATUS_UPDATE_SHIFT) ++#define CSG_ACK_IRQ_MASK_STATUS_UPDATE_GET(reg_val) \ ++ (((reg_val)&CSG_ACK_IRQ_MASK_STATUS_UPDATE_MASK) >> CSG_ACK_IRQ_MASK_STATUS_UPDATE_SHIFT) ++#define CSG_ACK_IRQ_MASK_STATUS_UPDATE_SET(reg_val, value) \ ++ (((reg_val) & ~CSG_ACK_IRQ_MASK_STATUS_UPDATE_MASK) | \ ++ (((value) << CSG_ACK_IRQ_MASK_STATUS_UPDATE_SHIFT) & CSG_ACK_IRQ_MASK_STATUS_UPDATE_MASK)) ++#define CSG_ACK_IRQ_MASK_SYNC_UPDATE_SHIFT 28 ++#define CSG_ACK_IRQ_MASK_SYNC_UPDATE_MASK (0x1 << CSG_ACK_IRQ_MASK_SYNC_UPDATE_SHIFT) ++#define CSG_ACK_IRQ_MASK_SYNC_UPDATE_GET(reg_val) \ ++ (((reg_val)&CSG_ACK_IRQ_MASK_SYNC_UPDATE_MASK) >> CSG_ACK_IRQ_MASK_SYNC_UPDATE_SHIFT) ++#define CSG_ACK_IRQ_MASK_SYNC_UPDATE_SET(reg_val, value) \ ++ (((reg_val) & ~CSG_ACK_IRQ_MASK_SYNC_UPDATE_MASK) | \ ++ (((value) << CSG_ACK_IRQ_MASK_SYNC_UPDATE_SHIFT) & CSG_ACK_IRQ_MASK_SYNC_UPDATE_MASK)) ++#define CSG_ACK_IRQ_MASK_IDLE_SHIFT 29 ++#define CSG_ACK_IRQ_MASK_IDLE_MASK (0x1 << CSG_ACK_IRQ_MASK_IDLE_SHIFT) ++#define CSG_ACK_IRQ_MASK_IDLE_GET(reg_val) (((reg_val)&CSG_ACK_IRQ_MASK_IDLE_MASK) >> CSG_ACK_IRQ_MASK_IDLE_SHIFT) ++#define CSG_ACK_IRQ_MASK_IDLE_SET(reg_val, value) \ ++ (((reg_val) & ~CSG_ACK_IRQ_MASK_IDLE_MASK) | \ ++ (((value) << CSG_ACK_IRQ_MASK_IDLE_SHIFT) & CSG_ACK_IRQ_MASK_IDLE_MASK)) ++#define CSG_ACK_IRQ_MASK_DOORBELL_SHIFT 30 ++#define CSG_ACK_IRQ_MASK_DOORBELL_MASK (0x1 << CSG_ACK_IRQ_MASK_DOORBELL_SHIFT) ++#define CSG_ACK_IRQ_MASK_DOORBELL_GET(reg_val) \ ++ (((reg_val)&CSG_ACK_IRQ_MASK_DOORBELL_MASK) >> CSG_ACK_IRQ_MASK_DOORBELL_SHIFT) ++#define CSG_ACK_IRQ_MASK_DOORBELL_SET(reg_val, value) \ ++ (((reg_val) & ~CSG_ACK_IRQ_MASK_DOORBELL_MASK) | \ ++ (((value) << CSG_ACK_IRQ_MASK_DOORBELL_SHIFT) & CSG_ACK_IRQ_MASK_DOORBELL_MASK)) ++#define CSG_ACK_IRQ_MASK_PROGRESS_TIMER_EVENT_SHIFT 31 ++#define CSG_ACK_IRQ_MASK_PROGRESS_TIMER_EVENT_MASK (0x1 << CSG_ACK_IRQ_MASK_PROGRESS_TIMER_EVENT_SHIFT) ++#define CSG_ACK_IRQ_MASK_PROGRESS_TIMER_EVENT_GET(reg_val) \ ++ (((reg_val)&CSG_ACK_IRQ_MASK_PROGRESS_TIMER_EVENT_MASK) >> CSG_ACK_IRQ_MASK_PROGRESS_TIMER_EVENT_SHIFT) ++#define CSG_ACK_IRQ_MASK_PROGRESS_TIMER_EVENT_SET(reg_val, value) \ ++ (((reg_val) & ~CSG_ACK_IRQ_MASK_PROGRESS_TIMER_EVENT_MASK) | \ ++ (((value) << CSG_ACK_IRQ_MASK_PROGRESS_TIMER_EVENT_SHIFT) & CSG_ACK_IRQ_MASK_PROGRESS_TIMER_EVENT_MASK)) ++ ++/* CSG_EP_REQ register */ ++#define CSG_EP_REQ_COMPUTE_EP_SHIFT 0 ++#define CSG_EP_REQ_COMPUTE_EP_MASK (0xFF << CSG_EP_REQ_COMPUTE_EP_SHIFT) ++#define CSG_EP_REQ_COMPUTE_EP_GET(reg_val) (((reg_val)&CSG_EP_REQ_COMPUTE_EP_MASK) >> CSG_EP_REQ_COMPUTE_EP_SHIFT) ++#define CSG_EP_REQ_COMPUTE_EP_SET(reg_val, value) \ ++ (((reg_val) & ~CSG_EP_REQ_COMPUTE_EP_MASK) | \ ++ (((value) << CSG_EP_REQ_COMPUTE_EP_SHIFT) & CSG_EP_REQ_COMPUTE_EP_MASK)) ++#define CSG_EP_REQ_FRAGMENT_EP_SHIFT 8 ++#define CSG_EP_REQ_FRAGMENT_EP_MASK (0xFF << CSG_EP_REQ_FRAGMENT_EP_SHIFT) ++#define CSG_EP_REQ_FRAGMENT_EP_GET(reg_val) (((reg_val)&CSG_EP_REQ_FRAGMENT_EP_MASK) >> CSG_EP_REQ_FRAGMENT_EP_SHIFT) ++#define CSG_EP_REQ_FRAGMENT_EP_SET(reg_val, value) \ ++ (((reg_val) & ~CSG_EP_REQ_FRAGMENT_EP_MASK) | \ ++ (((value) << CSG_EP_REQ_FRAGMENT_EP_SHIFT) & CSG_EP_REQ_FRAGMENT_EP_MASK)) ++#define CSG_EP_REQ_TILER_EP_SHIFT 16 ++#define CSG_EP_REQ_TILER_EP_MASK (0xF << CSG_EP_REQ_TILER_EP_SHIFT) ++#define CSG_EP_REQ_TILER_EP_GET(reg_val) (((reg_val)&CSG_EP_REQ_TILER_EP_MASK) >> CSG_EP_REQ_TILER_EP_SHIFT) ++#define CSG_EP_REQ_TILER_EP_SET(reg_val, value) \ ++ (((reg_val) & ~CSG_EP_REQ_TILER_EP_MASK) | (((value) << CSG_EP_REQ_TILER_EP_SHIFT) & CSG_EP_REQ_TILER_EP_MASK)) ++#define CSG_EP_REQ_EXCLUSIVE_COMPUTE_SHIFT 20 ++#define CSG_EP_REQ_EXCLUSIVE_COMPUTE_MASK (0x1 << CSG_EP_REQ_EXCLUSIVE_COMPUTE_SHIFT) ++#define CSG_EP_REQ_EXCLUSIVE_COMPUTE_GET(reg_val) \ ++ (((reg_val)&CSG_EP_REQ_EXCLUSIVE_COMPUTE_MASK) >> CSG_EP_REQ_EXCLUSIVE_COMPUTE_SHIFT) ++#define CSG_EP_REQ_EXCLUSIVE_COMPUTE_SET(reg_val, value) \ ++ (((reg_val) & ~CSG_EP_REQ_EXCLUSIVE_COMPUTE_MASK) | \ ++ (((value) << CSG_EP_REQ_EXCLUSIVE_COMPUTE_SHIFT) & CSG_EP_REQ_EXCLUSIVE_COMPUTE_MASK)) ++#define CSG_EP_REQ_EXCLUSIVE_FRAGMENT_SHIFT 21 ++#define CSG_EP_REQ_EXCLUSIVE_FRAGMENT_MASK (0x1 << CSG_EP_REQ_EXCLUSIVE_FRAGMENT_SHIFT) ++#define CSG_EP_REQ_EXCLUSIVE_FRAGMENT_GET(reg_val) \ ++ (((reg_val)&CSG_EP_REQ_EXCLUSIVE_FRAGMENT_MASK) >> CSG_EP_REQ_EXCLUSIVE_FRAGMENT_SHIFT) ++#define CSG_EP_REQ_EXCLUSIVE_FRAGMENT_SET(reg_val, value) \ ++ (((reg_val) & ~CSG_EP_REQ_EXCLUSIVE_FRAGMENT_MASK) | \ ++ (((value) << CSG_EP_REQ_EXCLUSIVE_FRAGMENT_SHIFT) & CSG_EP_REQ_EXCLUSIVE_FRAGMENT_MASK)) ++#define CSG_EP_REQ_PRIORITY_SHIFT 28 ++#define CSG_EP_REQ_PRIORITY_MASK (0xF << CSG_EP_REQ_PRIORITY_SHIFT) ++#define CSG_EP_REQ_PRIORITY_GET(reg_val) (((reg_val)&CSG_EP_REQ_PRIORITY_MASK) >> CSG_EP_REQ_PRIORITY_SHIFT) ++#define CSG_EP_REQ_PRIORITY_SET(reg_val, value) \ ++ (((reg_val) & ~CSG_EP_REQ_PRIORITY_MASK) | (((value) << CSG_EP_REQ_PRIORITY_SHIFT) & CSG_EP_REQ_PRIORITY_MASK)) ++ ++/* CSG_SUSPEND_BUF register */ ++#define CSG_SUSPEND_BUF_POINTER_SHIFT 0 ++#define CSG_SUSPEND_BUF_POINTER_MASK (0xFFFFFFFFFFFFFFFF << CSG_SUSPEND_BUF_POINTER_SHIFT) ++#define CSG_SUSPEND_BUF_POINTER_GET(reg_val) (((reg_val)&CSG_SUSPEND_BUF_POINTER_MASK) >> CSG_SUSPEND_BUF_POINTER_SHIFT) ++#define CSG_SUSPEND_BUF_POINTER_SET(reg_val, value) \ ++ (((reg_val) & ~CSG_SUSPEND_BUF_POINTER_MASK) | \ ++ (((value) << CSG_SUSPEND_BUF_POINTER_SHIFT) & CSG_SUSPEND_BUF_POINTER_MASK)) ++ ++/* CSG_PROTM_SUSPEND_BUF register */ ++#define CSG_PROTM_SUSPEND_BUF_POINTER_SHIFT 0 ++#define CSG_PROTM_SUSPEND_BUF_POINTER_MASK (0xFFFFFFFFFFFFFFFF << CSG_PROTM_SUSPEND_BUF_POINTER_SHIFT) ++#define CSG_PROTM_SUSPEND_BUF_POINTER_GET(reg_val) \ ++ (((reg_val)&CSG_PROTM_SUSPEND_BUF_POINTER_MASK) >> CSG_PROTM_SUSPEND_BUF_POINTER_SHIFT) ++#define CSG_PROTM_SUSPEND_BUF_POINTER_SET(reg_val, value) \ ++ (((reg_val) & ~CSG_PROTM_SUSPEND_BUF_POINTER_MASK) | \ ++ (((value) << CSG_PROTM_SUSPEND_BUF_POINTER_SHIFT) & CSG_PROTM_SUSPEND_BUF_POINTER_MASK)) ++ ++/* End of CSG_INPUT_BLOCK register set definitions */ ++ ++/* CSG_OUTPUT_BLOCK register set definitions */ ++ ++/* CSG_ACK register */ ++#define CSG_ACK_STATE_SHIFT 0 ++#define CSG_ACK_STATE_MASK (0x7 << CSG_ACK_STATE_SHIFT) ++#define CSG_ACK_STATE_GET(reg_val) (((reg_val)&CSG_ACK_STATE_MASK) >> CSG_ACK_STATE_SHIFT) ++#define CSG_ACK_STATE_SET(reg_val, value) \ ++ (((reg_val) & ~CSG_ACK_STATE_MASK) | (((value) << CSG_ACK_STATE_SHIFT) & CSG_ACK_STATE_MASK)) ++/* CSG_ACK_STATE values */ ++#define CSG_ACK_STATE_TERMINATE 0x0 ++#define CSG_ACK_STATE_START 0x1 ++#define CSG_ACK_STATE_SUSPEND 0x2 ++#define CSG_ACK_STATE_RESUME 0x3 ++/* End of CSG_ACK_STATE values */ ++#define CSG_ACK_EP_CFG_SHIFT 4 ++#define CSG_ACK_EP_CFG_MASK (0x1 << CSG_ACK_EP_CFG_SHIFT) ++#define CSG_ACK_EP_CFG_GET(reg_val) (((reg_val)&CSG_ACK_EP_CFG_MASK) >> CSG_ACK_EP_CFG_SHIFT) ++#define CSG_ACK_EP_CFG_SET(reg_val, value) \ ++ (((reg_val) & ~CSG_ACK_EP_CFG_MASK) | (((value) << CSG_ACK_EP_CFG_SHIFT) & CSG_ACK_EP_CFG_MASK)) ++#define CSG_ACK_STATUS_UPDATE_SHIFT 5 ++#define CSG_ACK_STATUS_UPDATE_MASK (0x1 << CSG_ACK_STATUS_UPDATE_SHIFT) ++#define CSG_ACK_STATUS_UPDATE_GET(reg_val) (((reg_val)&CSG_ACK_STATUS_UPDATE_MASK) >> CSG_ACK_STATUS_UPDATE_SHIFT) ++#define CSG_ACK_STATUS_UPDATE_SET(reg_val, value) \ ++ (((reg_val) & ~CSG_ACK_STATUS_UPDATE_MASK) | \ ++ (((value) << CSG_ACK_STATUS_UPDATE_SHIFT) & CSG_ACK_STATUS_UPDATE_MASK)) ++#define CSG_ACK_SYNC_UPDATE_SHIFT 28 ++#define CSG_ACK_SYNC_UPDATE_MASK (0x1 << CSG_ACK_SYNC_UPDATE_SHIFT) ++#define CSG_ACK_SYNC_UPDATE_GET(reg_val) (((reg_val)&CSG_ACK_SYNC_UPDATE_MASK) >> CSG_ACK_SYNC_UPDATE_SHIFT) ++#define CSG_ACK_SYNC_UPDATE_SET(reg_val, value) \ ++ (((reg_val) & ~CSG_ACK_SYNC_UPDATE_MASK) | (((value) << CSG_ACK_SYNC_UPDATE_SHIFT) & CSG_ACK_SYNC_UPDATE_MASK)) ++#define CSG_ACK_IDLE_SHIFT 29 ++#define CSG_ACK_IDLE_MASK (0x1 << CSG_ACK_IDLE_SHIFT) ++#define CSG_ACK_IDLE_GET(reg_val) (((reg_val)&CSG_ACK_IDLE_MASK) >> CSG_ACK_IDLE_SHIFT) ++#define CSG_ACK_IDLE_SET(reg_val, value) \ ++ (((reg_val) & ~CSG_ACK_IDLE_MASK) | (((value) << CSG_ACK_IDLE_SHIFT) & CSG_ACK_IDLE_MASK)) ++#define CSG_ACK_DOORBELL_SHIFT 30 ++#define CSG_ACK_DOORBELL_MASK (0x1 << CSG_ACK_DOORBELL_SHIFT) ++#define CSG_ACK_DOORBELL_GET(reg_val) (((reg_val)&CSG_ACK_DOORBELL_MASK) >> CSG_ACK_DOORBELL_SHIFT) ++#define CSG_ACK_DOORBELL_SET(reg_val, value) \ ++ (((reg_val) & ~CSG_ACK_DOORBELL_MASK) | (((value) << CSG_ACK_DOORBELL_SHIFT) & CSG_ACK_DOORBELL_MASK)) ++#define CSG_ACK_PROGRESS_TIMER_EVENT_SHIFT 31 ++#define CSG_ACK_PROGRESS_TIMER_EVENT_MASK (0x1 << CSG_ACK_PROGRESS_TIMER_EVENT_SHIFT) ++#define CSG_ACK_PROGRESS_TIMER_EVENT_GET(reg_val) \ ++ (((reg_val)&CSG_ACK_PROGRESS_TIMER_EVENT_MASK) >> CSG_ACK_PROGRESS_TIMER_EVENT_SHIFT) ++#define CSG_ACK_PROGRESS_TIMER_EVENT_SET(reg_val, value) \ ++ (((reg_val) & ~CSG_ACK_PROGRESS_TIMER_EVENT_MASK) | \ ++ (((value) << CSG_ACK_PROGRESS_TIMER_EVENT_SHIFT) & CSG_ACK_PROGRESS_TIMER_EVENT_MASK)) ++ ++/* CSG_STATUS_EP_CURRENT register */ ++#define CSG_STATUS_EP_CURRENT_COMPUTE_EP_SHIFT 0 ++#define CSG_STATUS_EP_CURRENT_COMPUTE_EP_MASK (0xFF << CSG_STATUS_EP_CURRENT_COMPUTE_EP_SHIFT) ++#define CSG_STATUS_EP_CURRENT_COMPUTE_EP_GET(reg_val) \ ++ (((reg_val)&CSG_STATUS_EP_CURRENT_COMPUTE_EP_MASK) >> CSG_STATUS_EP_CURRENT_COMPUTE_EP_SHIFT) ++#define CSG_STATUS_EP_CURRENT_COMPUTE_EP_SET(reg_val, value) \ ++ (((reg_val) & ~CSG_STATUS_EP_CURRENT_COMPUTE_EP_MASK) | \ ++ (((value) << CSG_STATUS_EP_CURRENT_COMPUTE_EP_SHIFT) & CSG_STATUS_EP_CURRENT_COMPUTE_EP_MASK)) ++#define CSG_STATUS_EP_CURRENT_FRAGMENT_EP_SHIFT 8 ++#define CSG_STATUS_EP_CURRENT_FRAGMENT_EP_MASK (0xFF << CSG_STATUS_EP_CURRENT_FRAGMENT_EP_SHIFT) ++#define CSG_STATUS_EP_CURRENT_FRAGMENT_EP_GET(reg_val) \ ++ (((reg_val)&CSG_STATUS_EP_CURRENT_FRAGMENT_EP_MASK) >> CSG_STATUS_EP_CURRENT_FRAGMENT_EP_SHIFT) ++#define CSG_STATUS_EP_CURRENT_FRAGMENT_EP_SET(reg_val, value) \ ++ (((reg_val) & ~CSG_STATUS_EP_CURRENT_FRAGMENT_EP_MASK) | \ ++ (((value) << CSG_STATUS_EP_CURRENT_FRAGMENT_EP_SHIFT) & CSG_STATUS_EP_CURRENT_FRAGMENT_EP_MASK)) ++#define CSG_STATUS_EP_CURRENT_TILER_EP_SHIFT 16 ++#define CSG_STATUS_EP_CURRENT_TILER_EP_MASK (0xF << CSG_STATUS_EP_CURRENT_TILER_EP_SHIFT) ++#define CSG_STATUS_EP_CURRENT_TILER_EP_GET(reg_val) \ ++ (((reg_val)&CSG_STATUS_EP_CURRENT_TILER_EP_MASK) >> CSG_STATUS_EP_CURRENT_TILER_EP_SHIFT) ++#define CSG_STATUS_EP_CURRENT_TILER_EP_SET(reg_val, value) \ ++ (((reg_val) & ~CSG_STATUS_EP_CURRENT_TILER_EP_MASK) | \ ++ (((value) << CSG_STATUS_EP_CURRENT_TILER_EP_SHIFT) & CSG_STATUS_EP_CURRENT_TILER_EP_MASK)) ++ ++/* CSG_STATUS_EP_REQ register */ ++#define CSG_STATUS_EP_REQ_COMPUTE_EP_SHIFT 0 ++#define CSG_STATUS_EP_REQ_COMPUTE_EP_MASK (0xFF << CSG_STATUS_EP_REQ_COMPUTE_EP_SHIFT) ++#define CSG_STATUS_EP_REQ_COMPUTE_EP_GET(reg_val) \ ++ (((reg_val)&CSG_STATUS_EP_REQ_COMPUTE_EP_MASK) >> CSG_STATUS_EP_REQ_COMPUTE_EP_SHIFT) ++#define CSG_STATUS_EP_REQ_COMPUTE_EP_SET(reg_val, value) \ ++ (((reg_val) & ~CSG_STATUS_EP_REQ_COMPUTE_EP_MASK) | \ ++ (((value) << CSG_STATUS_EP_REQ_COMPUTE_EP_SHIFT) & CSG_STATUS_EP_REQ_COMPUTE_EP_MASK)) ++#define CSG_STATUS_EP_REQ_FRAGMENT_EP_SHIFT 8 ++#define CSG_STATUS_EP_REQ_FRAGMENT_EP_MASK (0xFF << CSG_STATUS_EP_REQ_FRAGMENT_EP_SHIFT) ++#define CSG_STATUS_EP_REQ_FRAGMENT_EP_GET(reg_val) \ ++ (((reg_val)&CSG_STATUS_EP_REQ_FRAGMENT_EP_MASK) >> CSG_STATUS_EP_REQ_FRAGMENT_EP_SHIFT) ++#define CSG_STATUS_EP_REQ_FRAGMENT_EP_SET(reg_val, value) \ ++ (((reg_val) & ~CSG_STATUS_EP_REQ_FRAGMENT_EP_MASK) | \ ++ (((value) << CSG_STATUS_EP_REQ_FRAGMENT_EP_SHIFT) & CSG_STATUS_EP_REQ_FRAGMENT_EP_MASK)) ++#define CSG_STATUS_EP_REQ_TILER_EP_SHIFT 16 ++#define CSG_STATUS_EP_REQ_TILER_EP_MASK (0xF << CSG_STATUS_EP_REQ_TILER_EP_SHIFT) ++#define CSG_STATUS_EP_REQ_TILER_EP_GET(reg_val) \ ++ (((reg_val)&CSG_STATUS_EP_REQ_TILER_EP_MASK) >> CSG_STATUS_EP_REQ_TILER_EP_SHIFT) ++#define CSG_STATUS_EP_REQ_TILER_EP_SET(reg_val, value) \ ++ (((reg_val) & ~CSG_STATUS_EP_REQ_TILER_EP_MASK) | \ ++ (((value) << CSG_STATUS_EP_REQ_TILER_EP_SHIFT) & CSG_STATUS_EP_REQ_TILER_EP_MASK)) ++#define CSG_STATUS_EP_REQ_EXCLUSIVE_COMPUTE_SHIFT 20 ++#define CSG_STATUS_EP_REQ_EXCLUSIVE_COMPUTE_MASK (0x1 << CSG_STATUS_EP_REQ_EXCLUSIVE_COMPUTE_SHIFT) ++#define CSG_STATUS_EP_REQ_EXCLUSIVE_COMPUTE_GET(reg_val) \ ++ (((reg_val)&CSG_STATUS_EP_REQ_EXCLUSIVE_COMPUTE_MASK) >> CSG_STATUS_EP_REQ_EXCLUSIVE_COMPUTE_SHIFT) ++#define CSG_STATUS_EP_REQ_EXCLUSIVE_COMPUTE_SET(reg_val, value) \ ++ (((reg_val) & ~CSG_STATUS_EP_REQ_EXCLUSIVE_COMPUTE_MASK) | \ ++ (((value) << CSG_STATUS_EP_REQ_EXCLUSIVE_COMPUTE_SHIFT) & CSG_STATUS_EP_REQ_EXCLUSIVE_COMPUTE_MASK)) ++#define CSG_STATUS_EP_REQ_EXCLUSIVE_FRAGMENT_SHIFT 21 ++#define CSG_STATUS_EP_REQ_EXCLUSIVE_FRAGMENT_MASK (0x1 << CSG_STATUS_EP_REQ_EXCLUSIVE_FRAGMENT_SHIFT) ++#define CSG_STATUS_EP_REQ_EXCLUSIVE_FRAGMENT_GET(reg_val) \ ++ (((reg_val)&CSG_STATUS_EP_REQ_EXCLUSIVE_FRAGMENT_MASK) >> CSG_STATUS_EP_REQ_EXCLUSIVE_FRAGMENT_SHIFT) ++#define CSG_STATUS_EP_REQ_EXCLUSIVE_FRAGMENT_SET(reg_val, value) \ ++ (((reg_val) & ~CSG_STATUS_EP_REQ_EXCLUSIVE_FRAGMENT_MASK) | \ ++ (((value) << CSG_STATUS_EP_REQ_EXCLUSIVE_FRAGMENT_SHIFT) & CSG_STATUS_EP_REQ_EXCLUSIVE_FRAGMENT_MASK)) ++ ++/* End of CSG_OUTPUT_BLOCK register set definitions */ ++ ++/* STREAM_CONTROL_BLOCK register set definitions */ ++ ++/* STREAM_FEATURES register */ ++#define STREAM_FEATURES_WORK_REGISTERS_SHIFT 0 ++#define STREAM_FEATURES_WORK_REGISTERS_MASK (0xFF << STREAM_FEATURES_WORK_REGISTERS_SHIFT) ++#define STREAM_FEATURES_WORK_REGISTERS_GET(reg_val) \ ++ (((reg_val)&STREAM_FEATURES_WORK_REGISTERS_MASK) >> STREAM_FEATURES_WORK_REGISTERS_SHIFT) ++#define STREAM_FEATURES_WORK_REGISTERS_SET(reg_val, value) \ ++ (((reg_val) & ~STREAM_FEATURES_WORK_REGISTERS_MASK) | \ ++ (((value) << STREAM_FEATURES_WORK_REGISTERS_SHIFT) & STREAM_FEATURES_WORK_REGISTERS_MASK)) ++#define STREAM_FEATURES_SCOREBOARDS_SHIFT 8 ++#define STREAM_FEATURES_SCOREBOARDS_MASK (0xFF << STREAM_FEATURES_SCOREBOARDS_SHIFT) ++#define STREAM_FEATURES_SCOREBOARDS_GET(reg_val) \ ++ (((reg_val)&STREAM_FEATURES_SCOREBOARDS_MASK) >> STREAM_FEATURES_SCOREBOARDS_SHIFT) ++#define STREAM_FEATURES_SCOREBOARDS_SET(reg_val, value) \ ++ (((reg_val) & ~STREAM_FEATURES_SCOREBOARDS_MASK) | \ ++ (((value) << STREAM_FEATURES_SCOREBOARDS_SHIFT) & STREAM_FEATURES_SCOREBOARDS_MASK)) ++#define STREAM_FEATURES_COMPUTE_SHIFT 16 ++#define STREAM_FEATURES_COMPUTE_MASK (0x1 << STREAM_FEATURES_COMPUTE_SHIFT) ++#define STREAM_FEATURES_COMPUTE_GET(reg_val) (((reg_val)&STREAM_FEATURES_COMPUTE_MASK) >> STREAM_FEATURES_COMPUTE_SHIFT) ++#define STREAM_FEATURES_COMPUTE_SET(reg_val, value) \ ++ (((reg_val) & ~STREAM_FEATURES_COMPUTE_MASK) | \ ++ (((value) << STREAM_FEATURES_COMPUTE_SHIFT) & STREAM_FEATURES_COMPUTE_MASK)) ++#define STREAM_FEATURES_FRAGMENT_SHIFT 17 ++#define STREAM_FEATURES_FRAGMENT_MASK (0x1 << STREAM_FEATURES_FRAGMENT_SHIFT) ++#define STREAM_FEATURES_FRAGMENT_GET(reg_val) \ ++ (((reg_val)&STREAM_FEATURES_FRAGMENT_MASK) >> STREAM_FEATURES_FRAGMENT_SHIFT) ++#define STREAM_FEATURES_FRAGMENT_SET(reg_val, value) \ ++ (((reg_val) & ~STREAM_FEATURES_FRAGMENT_MASK) | \ ++ (((value) << STREAM_FEATURES_FRAGMENT_SHIFT) & STREAM_FEATURES_FRAGMENT_MASK)) ++#define STREAM_FEATURES_TILER_SHIFT 18 ++#define STREAM_FEATURES_TILER_MASK (0x1 << STREAM_FEATURES_TILER_SHIFT) ++#define STREAM_FEATURES_TILER_GET(reg_val) (((reg_val)&STREAM_FEATURES_TILER_MASK) >> STREAM_FEATURES_TILER_SHIFT) ++#define STREAM_FEATURES_TILER_SET(reg_val, value) \ ++ (((reg_val) & ~STREAM_FEATURES_TILER_MASK) | \ ++ (((value) << STREAM_FEATURES_TILER_SHIFT) & STREAM_FEATURES_TILER_MASK)) ++ ++/* STREAM_INPUT_VA register */ ++#define STREAM_INPUT_VA_VALUE_SHIFT 0 ++#define STREAM_INPUT_VA_VALUE_MASK (0xFFFFFFFF << STREAM_INPUT_VA_VALUE_SHIFT) ++#define STREAM_INPUT_VA_VALUE_GET(reg_val) (((reg_val)&STREAM_INPUT_VA_VALUE_MASK) >> STREAM_INPUT_VA_VALUE_SHIFT) ++#define STREAM_INPUT_VA_VALUE_SET(reg_val, value) \ ++ (((reg_val) & ~STREAM_INPUT_VA_VALUE_MASK) | \ ++ (((value) << STREAM_INPUT_VA_VALUE_SHIFT) & STREAM_INPUT_VA_VALUE_MASK)) ++ ++/* STREAM_OUTPUT_VA register */ ++#define STREAM_OUTPUT_VA_VALUE_SHIFT 0 ++#define STREAM_OUTPUT_VA_VALUE_MASK (0xFFFFFFFF << STREAM_OUTPUT_VA_VALUE_SHIFT) ++#define STREAM_OUTPUT_VA_VALUE_GET(reg_val) (((reg_val)&STREAM_OUTPUT_VA_VALUE_MASK) >> STREAM_OUTPUT_VA_VALUE_SHIFT) ++#define STREAM_OUTPUT_VA_VALUE_SET(reg_val, value) \ ++ (((reg_val) & ~STREAM_OUTPUT_VA_VALUE_MASK) | \ ++ (((value) << STREAM_OUTPUT_VA_VALUE_SHIFT) & STREAM_OUTPUT_VA_VALUE_MASK)) ++/* End of STREAM_CONTROL_BLOCK register set definitions */ ++ ++/* GLB_INPUT_BLOCK register set definitions */ ++ ++/* GLB_REQ register */ ++#define GLB_REQ_HALT_SHIFT 0 ++#define GLB_REQ_HALT_MASK (0x1 << GLB_REQ_HALT_SHIFT) ++#define GLB_REQ_HALT_GET(reg_val) (((reg_val)&GLB_REQ_HALT_MASK) >> GLB_REQ_HALT_SHIFT) ++#define GLB_REQ_HALT_SET(reg_val, value) \ ++ (((reg_val) & ~GLB_REQ_HALT_MASK) | (((value) << GLB_REQ_HALT_SHIFT) & GLB_REQ_HALT_MASK)) ++#define GLB_REQ_CFG_PROGRESS_TIMER_SHIFT 1 ++#define GLB_REQ_CFG_PROGRESS_TIMER_MASK (0x1 << GLB_REQ_CFG_PROGRESS_TIMER_SHIFT) ++#define GLB_REQ_CFG_PROGRESS_TIMER_GET(reg_val) \ ++ (((reg_val)&GLB_REQ_CFG_PROGRESS_TIMER_MASK) >> GLB_REQ_CFG_PROGRESS_TIMER_SHIFT) ++#define GLB_REQ_CFG_PROGRESS_TIMER_SET(reg_val, value) \ ++ (((reg_val) & ~GLB_REQ_CFG_PROGRESS_TIMER_MASK) | \ ++ (((value) << GLB_REQ_CFG_PROGRESS_TIMER_SHIFT) & GLB_REQ_CFG_PROGRESS_TIMER_MASK)) ++#define GLB_REQ_CFG_ALLOC_EN_SHIFT 2 ++#define GLB_REQ_CFG_ALLOC_EN_MASK (0x1 << GLB_REQ_CFG_ALLOC_EN_SHIFT) ++#define GLB_REQ_CFG_ALLOC_EN_GET(reg_val) (((reg_val)&GLB_REQ_CFG_ALLOC_EN_MASK) >> GLB_REQ_CFG_ALLOC_EN_SHIFT) ++#define GLB_REQ_CFG_ALLOC_EN_SET(reg_val, value) \ ++ (((reg_val) & ~GLB_REQ_CFG_ALLOC_EN_MASK) | (((value) << GLB_REQ_CFG_ALLOC_EN_SHIFT) & GLB_REQ_CFG_ALLOC_EN_MASK)) ++#define GLB_REQ_CFG_PWROFF_TIMER_SHIFT 3 ++#define GLB_REQ_CFG_PWROFF_TIMER_MASK (0x1 << GLB_REQ_CFG_PWROFF_TIMER_SHIFT) ++#define GLB_REQ_CFG_PWROFF_TIMER_GET(reg_val) \ ++ (((reg_val)&GLB_REQ_CFG_PWROFF_TIMER_MASK) >> GLB_REQ_CFG_PWROFF_TIMER_SHIFT) ++#define GLB_REQ_CFG_PWROFF_TIMER_SET(reg_val, value) \ ++ (((reg_val) & ~GLB_REQ_CFG_PWROFF_TIMER_MASK) | \ ++ (((value) << GLB_REQ_CFG_PWROFF_TIMER_SHIFT) & GLB_REQ_CFG_PWROFF_TIMER_MASK)) ++#define GLB_REQ_PROTM_ENTER_SHIFT 4 ++#define GLB_REQ_PROTM_ENTER_MASK (0x1 << GLB_REQ_PROTM_ENTER_SHIFT) ++#define GLB_REQ_PROTM_ENTER_GET(reg_val) (((reg_val)&GLB_REQ_PROTM_ENTER_MASK) >> GLB_REQ_PROTM_ENTER_SHIFT) ++#define GLB_REQ_PROTM_ENTER_SET(reg_val, value) \ ++ (((reg_val) & ~GLB_REQ_PROTM_ENTER_MASK) | (((value) << GLB_REQ_PROTM_ENTER_SHIFT) & GLB_REQ_PROTM_ENTER_MASK)) ++#define GLB_REQ_PRFCNT_ENABLE_SHIFT 5 ++#define GLB_REQ_PRFCNT_ENABLE_MASK (0x1 << GLB_REQ_PRFCNT_ENABLE_SHIFT) ++#define GLB_REQ_PRFCNT_ENABLE_GET(reg_val) (((reg_val)&GLB_REQ_PRFCNT_ENABLE_MASK) >> GLB_REQ_PRFCNT_ENABLE_SHIFT) ++#define GLB_REQ_PRFCNT_ENABLE_SET(reg_val, value) \ ++ (((reg_val) & ~GLB_REQ_PRFCNT_ENABLE_MASK) | \ ++ (((value) << GLB_REQ_PRFCNT_ENABLE_SHIFT) & GLB_REQ_PRFCNT_ENABLE_MASK)) ++#define GLB_REQ_PRFCNT_SAMPLE_SHIFT 6 ++#define GLB_REQ_PRFCNT_SAMPLE_MASK (0x1 << GLB_REQ_PRFCNT_SAMPLE_SHIFT) ++#define GLB_REQ_PRFCNT_SAMPLE_GET(reg_val) (((reg_val)&GLB_REQ_PRFCNT_SAMPLE_MASK) >> GLB_REQ_PRFCNT_SAMPLE_SHIFT) ++#define GLB_REQ_PRFCNT_SAMPLE_SET(reg_val, value) \ ++ (((reg_val) & ~GLB_REQ_PRFCNT_SAMPLE_MASK) | \ ++ (((value) << GLB_REQ_PRFCNT_SAMPLE_SHIFT) & GLB_REQ_PRFCNT_SAMPLE_MASK)) ++#define GLB_REQ_COUNTER_ENABLE_SHIFT 7 ++#define GLB_REQ_COUNTER_ENABLE_MASK (0x1 << GLB_REQ_COUNTER_ENABLE_SHIFT) ++#define GLB_REQ_COUNTER_ENABLE_GET(reg_val) (((reg_val)&GLB_REQ_COUNTER_ENABLE_MASK) >> GLB_REQ_COUNTER_ENABLE_SHIFT) ++#define GLB_REQ_COUNTER_ENABLE_SET(reg_val, value) \ ++ (((reg_val) & ~GLB_REQ_COUNTER_ENABLE_MASK) | \ ++ (((value) << GLB_REQ_COUNTER_ENABLE_SHIFT) & GLB_REQ_COUNTER_ENABLE_MASK)) ++#define GLB_REQ_PING_SHIFT 8 ++#define GLB_REQ_PING_MASK (0x1 << GLB_REQ_PING_SHIFT) ++#define GLB_REQ_PING_GET(reg_val) (((reg_val)&GLB_REQ_PING_MASK) >> GLB_REQ_PING_SHIFT) ++#define GLB_REQ_PING_SET(reg_val, value) \ ++ (((reg_val) & ~GLB_REQ_PING_MASK) | (((value) << GLB_REQ_PING_SHIFT) & GLB_REQ_PING_MASK)) ++#define GLB_REQ_INACTIVE_COMPUTE_SHIFT 20 ++#define GLB_REQ_INACTIVE_COMPUTE_MASK (0x1 << GLB_REQ_INACTIVE_COMPUTE_SHIFT) ++#define GLB_REQ_INACTIVE_COMPUTE_GET(reg_val) \ ++ (((reg_val)&GLB_REQ_INACTIVE_COMPUTE_MASK) >> GLB_REQ_INACTIVE_COMPUTE_SHIFT) ++#define GLB_REQ_INACTIVE_COMPUTE_SET(reg_val, value) \ ++ (((reg_val) & ~GLB_REQ_INACTIVE_COMPUTE_MASK) | \ ++ (((value) << GLB_REQ_INACTIVE_COMPUTE_SHIFT) & GLB_REQ_INACTIVE_COMPUTE_MASK)) ++#define GLB_REQ_INACTIVE_FRAGMENT_SHIFT 21 ++#define GLB_REQ_INACTIVE_FRAGMENT_MASK (0x1 << GLB_REQ_INACTIVE_FRAGMENT_SHIFT) ++#define GLB_REQ_INACTIVE_FRAGMENT_GET(reg_val) \ ++ (((reg_val)&GLB_REQ_INACTIVE_FRAGMENT_MASK) >> GLB_REQ_INACTIVE_FRAGMENT_SHIFT) ++#define GLB_REQ_INACTIVE_FRAGMENT_SET(reg_val, value) \ ++ (((reg_val) & ~GLB_REQ_INACTIVE_FRAGMENT_MASK) | \ ++ (((value) << GLB_REQ_INACTIVE_FRAGMENT_SHIFT) & GLB_REQ_INACTIVE_FRAGMENT_MASK)) ++#define GLB_REQ_INACTIVE_TILER_SHIFT 22 ++#define GLB_REQ_INACTIVE_TILER_MASK (0x1 << GLB_REQ_INACTIVE_TILER_SHIFT) ++#define GLB_REQ_INACTIVE_TILER_GET(reg_val) (((reg_val)&GLB_REQ_INACTIVE_TILER_MASK) >> GLB_REQ_INACTIVE_TILER_SHIFT) ++#define GLB_REQ_INACTIVE_TILER_SET(reg_val, value) \ ++ (((reg_val) & ~GLB_REQ_INACTIVE_TILER_MASK) | \ ++ (((value) << GLB_REQ_INACTIVE_TILER_SHIFT) & GLB_REQ_INACTIVE_TILER_MASK)) ++#define GLB_REQ_PROTM_EXIT_SHIFT 23 ++#define GLB_REQ_PROTM_EXIT_MASK (0x1 << GLB_REQ_PROTM_EXIT_SHIFT) ++#define GLB_REQ_PROTM_EXIT_GET(reg_val) (((reg_val)&GLB_REQ_PROTM_EXIT_MASK) >> GLB_REQ_PROTM_EXIT_SHIFT) ++#define GLB_REQ_PROTM_EXIT_SET(reg_val, value) \ ++ (((reg_val) & ~GLB_REQ_PROTM_EXIT_MASK) | (((value) << GLB_REQ_PROTM_EXIT_SHIFT) & GLB_REQ_PROTM_EXIT_MASK)) ++#define GLB_REQ_DEBUG_CSF_REQ_SHIFT 30 ++#define GLB_REQ_DEBUG_CSF_REQ_MASK (0x1 << GLB_REQ_DEBUG_CSF_REQ_SHIFT) ++#define GLB_REQ_DEBUG_CSF_REQ_GET(reg_val) (((reg_val)&GLB_REQ_DEBUG_CSF_REQ_MASK) >> GLB_REQ_DEBUG_CSF_REQ_SHIFT) ++#define GLB_REQ_DEBUG_CSF_REQ_SET(reg_val, value) \ ++ (((reg_val) & ~GLB_REQ_DEBUG_CSF_REQ_MASK) | \ ++ (((value) << GLB_REQ_DEBUG_CSF_REQ_SHIFT) & GLB_REQ_DEBUG_CSF_REQ_MASK)) ++#define GLB_REQ_DEBUG_HOST_REQ_SHIFT 31 ++#define GLB_REQ_DEBUG_HOST_REQ_MASK (0x1 << GLB_REQ_DEBUG_HOST_REQ_SHIFT) ++#define GLB_REQ_DEBUG_HOST_REQ_GET(reg_val) (((reg_val)&GLB_REQ_DEBUG_HOST_REQ_MASK) >> GLB_REQ_DEBUG_HOST_REQ_SHIFT) ++#define GLB_REQ_DEBUG_HOST_REQ_SET(reg_val, value) \ ++ (((reg_val) & ~GLB_REQ_DEBUG_HOST_REQ_MASK) | \ ++ (((value) << GLB_REQ_DEBUG_HOST_REQ_SHIFT) & GLB_REQ_DEBUG_HOST_REQ_MASK)) ++ ++/* GLB_ACK_IRQ_MASK register */ ++#define GLB_ACK_IRQ_MASK_HALT_SHIFT 0 ++#define GLB_ACK_IRQ_MASK_HALT_MASK (0x1 << GLB_ACK_IRQ_MASK_HALT_SHIFT) ++#define GLB_ACK_IRQ_MASK_HALT_GET(reg_val) (((reg_val)&GLB_ACK_IRQ_MASK_HALT_MASK) >> GLB_ACK_IRQ_MASK_HALT_SHIFT) ++#define GLB_ACK_IRQ_MASK_HALT_SET(reg_val, value) \ ++ (((reg_val) & ~GLB_ACK_IRQ_MASK_HALT_MASK) | \ ++ (((value) << GLB_ACK_IRQ_MASK_HALT_SHIFT) & GLB_ACK_IRQ_MASK_HALT_MASK)) ++#define GLB_ACK_IRQ_MASK_CFG_PROGRESS_TIMER_SHIFT 1 ++#define GLB_ACK_IRQ_MASK_CFG_PROGRESS_TIMER_MASK (0x1 << GLB_ACK_IRQ_MASK_CFG_PROGRESS_TIMER_SHIFT) ++#define GLB_ACK_IRQ_MASK_CFG_PROGRESS_TIMER_GET(reg_val) \ ++ (((reg_val)&GLB_ACK_IRQ_MASK_CFG_PROGRESS_TIMER_MASK) >> GLB_ACK_IRQ_MASK_CFG_PROGRESS_TIMER_SHIFT) ++#define GLB_ACK_IRQ_MASK_CFG_PROGRESS_TIMER_SET(reg_val, value) \ ++ (((reg_val) & ~GLB_ACK_IRQ_MASK_CFG_PROGRESS_TIMER_MASK) | \ ++ (((value) << GLB_ACK_IRQ_MASK_CFG_PROGRESS_TIMER_SHIFT) & GLB_ACK_IRQ_MASK_CFG_PROGRESS_TIMER_MASK)) ++#define GLB_ACK_IRQ_MASK_CFG_ALLOC_EN_SHIFT 2 ++#define GLB_ACK_IRQ_MASK_CFG_ALLOC_EN_MASK (0x1 << GLB_ACK_IRQ_MASK_CFG_ALLOC_EN_SHIFT) ++#define GLB_ACK_IRQ_MASK_CFG_ALLOC_EN_GET(reg_val) \ ++ (((reg_val)&GLB_ACK_IRQ_MASK_CFG_ALLOC_EN_MASK) >> GLB_ACK_IRQ_MASK_CFG_ALLOC_EN_SHIFT) ++#define GLB_ACK_IRQ_MASK_CFG_ALLOC_EN_SET(reg_val, value) \ ++ (((reg_val) & ~GLB_ACK_IRQ_MASK_CFG_ALLOC_EN_MASK) | \ ++ (((value) << GLB_ACK_IRQ_MASK_CFG_ALLOC_EN_SHIFT) & GLB_ACK_IRQ_MASK_CFG_ALLOC_EN_MASK)) ++#define GLB_ACK_IRQ_MASK_CFG_PWROFF_TIMER_SHIFT 3 ++#define GLB_ACK_IRQ_MASK_CFG_PWROFF_TIMER_MASK (0x1 << GLB_ACK_IRQ_MASK_CFG_PWROFF_TIMER_SHIFT) ++#define GLB_ACK_IRQ_MASK_CFG_PWROFF_TIMER_GET(reg_val) \ ++ (((reg_val)&GLB_ACK_IRQ_MASK_CFG_PWROFF_TIMER_MASK) >> GLB_ACK_IRQ_MASK_CFG_PWROFF_TIMER_SHIFT) ++#define GLB_ACK_IRQ_MASK_CFG_PWROFF_TIMER_SET(reg_val, value) \ ++ (((reg_val) & ~GLB_ACK_IRQ_MASK_CFG_PWROFF_TIMER_MASK) | \ ++ (((value) << GLB_ACK_IRQ_MASK_CFG_PWROFF_TIMER_SHIFT) & GLB_ACK_IRQ_MASK_CFG_PWROFF_TIMER_MASK)) ++#define GLB_ACK_IRQ_MASK_PROTM_ENTER_SHIFT 4 ++#define GLB_ACK_IRQ_MASK_PROTM_ENTER_MASK (0x1 << GLB_ACK_IRQ_MASK_PROTM_ENTER_SHIFT) ++#define GLB_ACK_IRQ_MASK_PROTM_ENTER_GET(reg_val) \ ++ (((reg_val)&GLB_ACK_IRQ_MASK_PROTM_ENTER_MASK) >> GLB_ACK_IRQ_MASK_PROTM_ENTER_SHIFT) ++#define GLB_ACK_IRQ_MASK_PROTM_ENTER_SET(reg_val, value) \ ++ (((reg_val) & ~GLB_ACK_IRQ_MASK_PROTM_ENTER_MASK) | \ ++ (((value) << GLB_ACK_IRQ_MASK_PROTM_ENTER_SHIFT) & GLB_ACK_IRQ_MASK_PROTM_ENTER_MASK)) ++#define GLB_ACK_IRQ_MASK_PRFCNT_ENABLE_SHIFT 5 ++#define GLB_ACK_IRQ_MASK_PRFCNT_ENABLE_MASK (0x1 << GLB_ACK_IRQ_MASK_PRFCNT_ENABLE_SHIFT) ++#define GLB_ACK_IRQ_MASK_PRFCNT_ENABLE_GET(reg_val) \ ++ (((reg_val)&GLB_ACK_IRQ_MASK_PRFCNT_ENABLE_MASK) >> GLB_ACK_IRQ_MASK_PRFCNT_ENABLE_SHIFT) ++#define GLB_ACK_IRQ_MASK_PRFCNT_ENABLE_SET(reg_val, value) \ ++ (((reg_val) & ~GLB_ACK_IRQ_MASK_PRFCNT_ENABLE_MASK) | \ ++ (((value) << GLB_ACK_IRQ_MASK_PRFCNT_ENABLE_SHIFT) & GLB_ACK_IRQ_MASK_PRFCNT_ENABLE_MASK)) ++#define GLB_ACK_IRQ_MASK_PRFCNT_SAMPLE_SHIFT 6 ++#define GLB_ACK_IRQ_MASK_PRFCNT_SAMPLE_MASK (0x1 << GLB_ACK_IRQ_MASK_PRFCNT_SAMPLE_SHIFT) ++#define GLB_ACK_IRQ_MASK_PRFCNT_SAMPLE_GET(reg_val) \ ++ (((reg_val)&GLB_ACK_IRQ_MASK_PRFCNT_SAMPLE_MASK) >> GLB_ACK_IRQ_MASK_PRFCNT_SAMPLE_SHIFT) ++#define GLB_ACK_IRQ_MASK_PRFCNT_SAMPLE_SET(reg_val, value) \ ++ (((reg_val) & ~GLB_ACK_IRQ_MASK_PRFCNT_SAMPLE_MASK) | \ ++ (((value) << GLB_ACK_IRQ_MASK_PRFCNT_SAMPLE_SHIFT) & GLB_ACK_IRQ_MASK_PRFCNT_SAMPLE_MASK)) ++#define GLB_ACK_IRQ_MASK_COUNTER_ENABLE_SHIFT 7 ++#define GLB_ACK_IRQ_MASK_COUNTER_ENABLE_MASK (0x1 << GLB_ACK_IRQ_MASK_COUNTER_ENABLE_SHIFT) ++#define GLB_ACK_IRQ_MASK_COUNTER_ENABLE_GET(reg_val) \ ++ (((reg_val)&GLB_ACK_IRQ_MASK_COUNTER_ENABLE_MASK) >> GLB_ACK_IRQ_MASK_COUNTER_ENABLE_SHIFT) ++#define GLB_ACK_IRQ_MASK_COUNTER_ENABLE_SET(reg_val, value) \ ++ (((reg_val) & ~GLB_ACK_IRQ_MASK_COUNTER_ENABLE_MASK) | \ ++ (((value) << GLB_ACK_IRQ_MASK_COUNTER_ENABLE_SHIFT) & GLB_ACK_IRQ_MASK_COUNTER_ENABLE_MASK)) ++#define GLB_ACK_IRQ_MASK_PING_SHIFT 8 ++#define GLB_ACK_IRQ_MASK_PING_MASK (0x1 << GLB_ACK_IRQ_MASK_PING_SHIFT) ++#define GLB_ACK_IRQ_MASK_PING_GET(reg_val) (((reg_val)&GLB_ACK_IRQ_MASK_PING_MASK) >> GLB_ACK_IRQ_MASK_PING_SHIFT) ++#define GLB_ACK_IRQ_MASK_PING_SET(reg_val, value) \ ++ (((reg_val) & ~GLB_ACK_IRQ_MASK_PING_MASK) | \ ++ (((value) << GLB_ACK_IRQ_MASK_PING_SHIFT) & GLB_ACK_IRQ_MASK_PING_MASK)) ++#define GLB_ACK_IRQ_MASK_INACTIVE_COMPUTE_SHIFT 20 ++#define GLB_ACK_IRQ_MASK_INACTIVE_COMPUTE_MASK (0x1 << GLB_ACK_IRQ_MASK_INACTIVE_COMPUTE_SHIFT) ++#define GLB_ACK_IRQ_MASK_INACTIVE_COMPUTE_GET(reg_val) \ ++ (((reg_val)&GLB_ACK_IRQ_MASK_INACTIVE_COMPUTE_MASK) >> GLB_ACK_IRQ_MASK_INACTIVE_COMPUTE_SHIFT) ++#define GLB_ACK_IRQ_MASK_INACTIVE_COMPUTE_SET(reg_val, value) \ ++ (((reg_val) & ~GLB_ACK_IRQ_MASK_INACTIVE_COMPUTE_MASK) | \ ++ (((value) << GLB_ACK_IRQ_MASK_INACTIVE_COMPUTE_SHIFT) & GLB_ACK_IRQ_MASK_INACTIVE_COMPUTE_MASK)) ++#define GLB_ACK_IRQ_MASK_INACTIVE_FRAGMENT_SHIFT 21 ++#define GLB_ACK_IRQ_MASK_INACTIVE_FRAGMENT_MASK (0x1 << GLB_ACK_IRQ_MASK_INACTIVE_FRAGMENT_SHIFT) ++#define GLB_ACK_IRQ_MASK_INACTIVE_FRAGMENT_GET(reg_val) \ ++ (((reg_val)&GLB_ACK_IRQ_MASK_INACTIVE_FRAGMENT_MASK) >> GLB_ACK_IRQ_MASK_INACTIVE_FRAGMENT_SHIFT) ++#define GLB_ACK_IRQ_MASK_INACTIVE_FRAGMENT_SET(reg_val, value) \ ++ (((reg_val) & ~GLB_ACK_IRQ_MASK_INACTIVE_FRAGMENT_MASK) | \ ++ (((value) << GLB_ACK_IRQ_MASK_INACTIVE_FRAGMENT_SHIFT) & GLB_ACK_IRQ_MASK_INACTIVE_FRAGMENT_MASK)) ++#define GLB_ACK_IRQ_MASK_INACTIVE_TILER_SHIFT 22 ++#define GLB_ACK_IRQ_MASK_INACTIVE_TILER_MASK (0x1 << GLB_ACK_IRQ_MASK_INACTIVE_TILER_SHIFT) ++#define GLB_ACK_IRQ_MASK_INACTIVE_TILER_GET(reg_val) \ ++ (((reg_val)&GLB_ACK_IRQ_MASK_INACTIVE_TILER_MASK) >> GLB_ACK_IRQ_MASK_INACTIVE_TILER_SHIFT) ++#define GLB_ACK_IRQ_MASK_INACTIVE_TILER_SET(reg_val, value) \ ++ (((reg_val) & ~GLB_ACK_IRQ_MASK_INACTIVE_TILER_MASK) | \ ++ (((value) << GLB_ACK_IRQ_MASK_INACTIVE_TILER_SHIFT) & GLB_ACK_IRQ_MASK_INACTIVE_TILER_MASK)) ++#define GLB_ACK_IRQ_MASK_PROTM_EXIT_SHIFT 23 ++#define GLB_ACK_IRQ_MASK_PROTM_EXIT_MASK (0x1 << GLB_ACK_IRQ_MASK_PROTM_EXIT_SHIFT) ++#define GLB_ACK_IRQ_MASK_PROTM_EXIT_GET(reg_val) \ ++ (((reg_val)&GLB_ACK_IRQ_MASK_PROTM_EXIT_MASK) >> GLB_ACK_IRQ_MASK_PROTM_EXIT_SHIFT) ++#define GLB_ACK_IRQ_MASK_PROTM_EXIT_SET(reg_val, value) \ ++ (((reg_val) & ~GLB_ACK_IRQ_MASK_PROTM_EXIT_MASK) | \ ++ (((value) << GLB_ACK_IRQ_MASK_PROTM_EXIT_SHIFT) & GLB_ACK_IRQ_MASK_PROTM_EXIT_MASK)) ++#define GLB_ACK_IRQ_MASK_DEBUG_CSF_REQ_SHIFT 30 ++#define GLB_ACK_IRQ_MASK_DEBUG_CSF_REQ_MASK (0x1 << GLB_ACK_IRQ_MASK_DEBUG_CSF_REQ_SHIFT) ++#define GLB_ACK_IRQ_MASK_DEBUG_CSF_REQ_GET(reg_val) \ ++ (((reg_val)&GLB_ACK_IRQ_MASK_DEBUG_CSF_REQ_MASK) >> GLB_ACK_IRQ_MASK_DEBUG_CSF_REQ_SHIFT) ++#define GLB_ACK_IRQ_MASK_DEBUG_CSF_REQ_SET(reg_val, value) \ ++ (((reg_val) & ~GLB_ACK_IRQ_MASK_DEBUG_CSF_REQ_MASK) | \ ++ (((value) << GLB_ACK_IRQ_MASK_DEBUG_CSF_REQ_SHIFT) & GLB_ACK_IRQ_MASK_DEBUG_CSF_REQ_MASK)) ++#define GLB_ACK_IRQ_MASK_DEBUG_HOST_REQ_SHIFT 31 ++#define GLB_ACK_IRQ_MASK_DEBUG_HOST_REQ_MASK (0x1 << GLB_ACK_IRQ_MASK_DEBUG_HOST_REQ_SHIFT) ++#define GLB_ACK_IRQ_MASK_DEBUG_HOST_REQ_GET(reg_val) \ ++ (((reg_val)&GLB_ACK_IRQ_MASK_DEBUG_HOST_REQ_MASK) >> GLB_ACK_IRQ_MASK_DEBUG_HOST_REQ_SHIFT) ++#define GLB_ACK_IRQ_MASK_DEBUG_HOST_REQ_SET(reg_val, value) \ ++ (((reg_val) & ~GLB_ACK_IRQ_MASK_DEBUG_HOST_REQ_MASK) | \ ++ (((value) << GLB_ACK_IRQ_MASK_DEBUG_HOST_REQ_SHIFT) & GLB_ACK_IRQ_MASK_DEBUG_HOST_REQ_MASK)) ++ ++/* GLB_PROGRESS_TIMER register */ ++#define GLB_PROGRESS_TIMER_TIMEOUT_SHIFT 0 ++#define GLB_PROGRESS_TIMER_TIMEOUT_MASK (0xFFFFFFFF << GLB_PROGRESS_TIMER_TIMEOUT_SHIFT) ++#define GLB_PROGRESS_TIMER_TIMEOUT_GET(reg_val) \ ++ (((reg_val)&GLB_PROGRESS_TIMER_TIMEOUT_MASK) >> GLB_PROGRESS_TIMER_TIMEOUT_SHIFT) ++#define GLB_PROGRESS_TIMER_TIMEOUT_SET(reg_val, value) \ ++ (((reg_val) & ~GLB_PROGRESS_TIMER_TIMEOUT_MASK) | \ ++ (((value) << GLB_PROGRESS_TIMER_TIMEOUT_SHIFT) & GLB_PROGRESS_TIMER_TIMEOUT_MASK)) ++ ++/* GLB_ALLOC_EN register */ ++#define GLB_ALLOC_EN_MASK_SHIFT 0 ++#define GLB_ALLOC_EN_MASK_MASK (0xFFFFFFFFFFFFFFFF << GLB_ALLOC_EN_MASK_SHIFT) ++#define GLB_ALLOC_EN_MASK_GET(reg_val) (((reg_val)&GLB_ALLOC_EN_MASK_MASK) >> GLB_ALLOC_EN_MASK_SHIFT) ++#define GLB_ALLOC_EN_MASK_SET(reg_val, value) \ ++ (((reg_val) & ~GLB_ALLOC_EN_MASK_MASK) | (((value) << GLB_ALLOC_EN_MASK_SHIFT) & GLB_ALLOC_EN_MASK_MASK)) ++ ++/* GLB_PROTM_COHERENCY register */ ++#define GLB_PROTM_COHERENCY_L2_CACHE_PROTOCOL_SELECT_SHIFT 0 ++#define GLB_PROTM_COHERENCY_L2_CACHE_PROTOCOL_SELECT_MASK \ ++ (0xFFFFFFFF << GLB_PROTM_COHERENCY_L2_CACHE_PROTOCOL_SELECT_SHIFT) ++#define GLB_PROTM_COHERENCY_L2_CACHE_PROTOCOL_SELECT_GET(reg_val) \ ++ (((reg_val)&GLB_PROTM_COHERENCY_L2_CACHE_PROTOCOL_SELECT_MASK) >> \ ++ GLB_PROTM_COHERENCY_L2_CACHE_PROTOCOL_SELECT_SHIFT) ++#define GLB_PROTM_COHERENCY_L2_CACHE_PROTOCOL_SELECT_SET(reg_val, value) \ ++ (((reg_val) & ~GLB_PROTM_COHERENCY_L2_CACHE_PROTOCOL_SELECT_MASK) | \ ++ (((value) << GLB_PROTM_COHERENCY_L2_CACHE_PROTOCOL_SELECT_SHIFT) & \ ++ GLB_PROTM_COHERENCY_L2_CACHE_PROTOCOL_SELECT_MASK)) ++/* End of GLB_INPUT_BLOCK register set definitions */ ++ ++/* GLB_OUTPUT_BLOCK register set definitions */ ++ ++/* GLB_ACK register */ ++#define GLB_ACK_CFG_PROGRESS_TIMER_SHIFT 1 ++#define GLB_ACK_CFG_PROGRESS_TIMER_MASK (0x1 << GLB_ACK_CFG_PROGRESS_TIMER_SHIFT) ++#define GLB_ACK_CFG_PROGRESS_TIMER_GET(reg_val) \ ++ (((reg_val)&GLB_ACK_CFG_PROGRESS_TIMER_MASK) >> GLB_ACK_CFG_PROGRESS_TIMER_SHIFT) ++#define GLB_ACK_CFG_PROGRESS_TIMER_SET(reg_val, value) \ ++ (((reg_val) & ~GLB_ACK_CFG_PROGRESS_TIMER_MASK) | \ ++ (((value) << GLB_ACK_CFG_PROGRESS_TIMER_SHIFT) & GLB_ACK_CFG_PROGRESS_TIMER_MASK)) ++#define GLB_ACK_CFG_ALLOC_EN_SHIFT 2 ++#define GLB_ACK_CFG_ALLOC_EN_MASK (0x1 << GLB_ACK_CFG_ALLOC_EN_SHIFT) ++#define GLB_ACK_CFG_ALLOC_EN_GET(reg_val) (((reg_val)&GLB_ACK_CFG_ALLOC_EN_MASK) >> GLB_ACK_CFG_ALLOC_EN_SHIFT) ++#define GLB_ACK_CFG_ALLOC_EN_SET(reg_val, value) \ ++ (((reg_val) & ~GLB_ACK_CFG_ALLOC_EN_MASK) | (((value) << GLB_ACK_CFG_ALLOC_EN_SHIFT) & GLB_ACK_CFG_ALLOC_EN_MASK)) ++/* End of GLB_OUTPUT_BLOCK register set definitions */ ++ ++#endif /* _GPU_CSF_REGISTERS_H_ */ +diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf.c +new file mode 100755 +index 000000000000..83d7513e78d9 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf.c +@@ -0,0 +1,2547 @@ ++/* ++ * ++ * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++#include ++#include ++#include ++#include "mali_kbase_csf.h" ++#include "backend/gpu/mali_kbase_pm_internal.h" ++#include ++#include ++#include "mali_gpu_csf_registers.h" ++#include "mali_kbase_csf_tiler_heap.h" ++#include ++#include ++ ++#define CS_REQ_EXCEPTION_MASK (CS_REQ_FAULT_MASK | CS_REQ_FATAL_MASK) ++#define CS_ACK_EXCEPTION_MASK (CS_ACK_FAULT_MASK | CS_ACK_FATAL_MASK) ++ ++/** ++ * struct kbase_csf_event - CSF event callback. ++ * ++ * This structure belongs to the list of events which is part of a Kbase ++ * context, and describes a callback function with a custom parameter to pass ++ * to it when a CSF event is signalled. ++ * ++ * @link: Link to the rest of the list. ++ * @kctx: Pointer to the Kbase context this event belongs to. ++ * @callback: Callback function to call when a CSF event is signalled. ++ * @param: Parameter to pass to the callback function. ++ */ ++struct kbase_csf_event { ++ struct list_head link; ++ struct kbase_context *kctx; ++ kbase_csf_event_callback *callback; ++ void *param; ++}; ++ ++static void put_user_pages_mmap_handle(struct kbase_context *kctx, ++ struct kbase_queue *queue) ++{ ++ unsigned long cookie_nr; ++ ++ lockdep_assert_held(&kctx->csf.lock); ++ ++ if (queue->handle == BASEP_MEM_INVALID_HANDLE) ++ return; ++ ++ cookie_nr = ++ PFN_DOWN(queue->handle - BASEP_MEM_CSF_USER_IO_PAGES_HANDLE); ++ ++ if (!WARN_ON(kctx->csf.user_pages_info[cookie_nr] != queue)) { ++ /* free up cookie */ ++ kctx->csf.user_pages_info[cookie_nr] = NULL; ++ bitmap_set(kctx->csf.cookies, cookie_nr, 1); ++ } ++ ++ queue->handle = BASEP_MEM_INVALID_HANDLE; ++} ++ ++/* Reserve a cookie, to be returned as a handle to userspace for creating ++ * the CPU mapping of the pair of input/output pages and Hw doorbell page. ++ * Will return 0 in case of success otherwise negative on failure. ++ */ ++static int get_user_pages_mmap_handle(struct kbase_context *kctx, ++ struct kbase_queue *queue) ++{ ++ unsigned long cookie, cookie_nr; ++ ++ lockdep_assert_held(&kctx->csf.lock); ++ ++ if (bitmap_empty(kctx->csf.cookies, ++ KBASE_CSF_NUM_USER_IO_PAGES_HANDLE)) { ++ dev_err(kctx->kbdev->dev, ++ "No csf cookies available for allocation!"); ++ return -ENOMEM; ++ } ++ ++ /* allocate a cookie */ ++ cookie_nr = find_first_bit(kctx->csf.cookies, ++ KBASE_CSF_NUM_USER_IO_PAGES_HANDLE); ++ if (kctx->csf.user_pages_info[cookie_nr]) { ++ dev_err(kctx->kbdev->dev, ++ "Inconsistent state of csf cookies!"); ++ return -EINVAL; ++ } ++ kctx->csf.user_pages_info[cookie_nr] = queue; ++ bitmap_clear(kctx->csf.cookies, cookie_nr, 1); ++ ++ /* relocate to correct base */ ++ cookie = cookie_nr + PFN_DOWN(BASEP_MEM_CSF_USER_IO_PAGES_HANDLE); ++ cookie <<= PAGE_SHIFT; ++ ++ queue->handle = (u64)cookie; ++ ++ return 0; ++} ++ ++static void gpu_munmap_user_io_pages(struct kbase_context *kctx, ++ struct kbase_va_region *reg) ++{ ++ size_t num_pages = 2; ++ ++ kbase_mmu_teardown_pages(kctx->kbdev, &kctx->kbdev->csf.mcu_mmu, ++ reg->start_pfn, num_pages, MCU_AS_NR); ++ ++ WARN_ON(reg->flags & KBASE_REG_FREE); ++ ++ mutex_lock(&kctx->kbdev->csf.reg_lock); ++ kbase_remove_va_region(reg); ++ mutex_unlock(&kctx->kbdev->csf.reg_lock); ++} ++ ++static void init_user_output_page(struct kbase_queue *queue) ++{ ++ u32 *addr = (u32 *)(queue->user_io_addr + PAGE_SIZE); ++ ++ addr[CS_EXTRACT_LO/4] = 0; ++ addr[CS_EXTRACT_HI/4] = 0; ++ ++ addr[CS_ACTIVE/4] = 0; ++} ++ ++/* Map the input/output pages in the shared interface segment of MCU firmware ++ * address space. ++ */ ++static int gpu_mmap_user_io_pages(struct kbase_device *kbdev, ++ struct tagged_addr *phys, struct kbase_va_region *reg) ++{ ++ unsigned long mem_flags = KBASE_REG_GPU_RD; ++ const size_t num_pages = 2; ++ int ret; ++ ++#if ((KERNEL_VERSION(4, 4, 147) >= LINUX_VERSION_CODE) || \ ++ ((KERNEL_VERSION(4, 6, 0) > LINUX_VERSION_CODE) && \ ++ (KERNEL_VERSION(4, 5, 0) <= LINUX_VERSION_CODE))) ++ mem_flags |= ++ KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_NON_CACHEABLE); ++#else ++ if (kbdev->system_coherency == COHERENCY_NONE) { ++ mem_flags |= ++ KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_NON_CACHEABLE); ++ } else { ++ mem_flags |= KBASE_REG_SHARE_BOTH | ++ KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_SHARED); ++ } ++#endif ++ ++ mutex_lock(&kbdev->csf.reg_lock); ++ ret = kbase_add_va_region_rbtree(kbdev, reg, 0, num_pages, 1); ++ reg->flags &= ~KBASE_REG_FREE; ++ mutex_unlock(&kbdev->csf.reg_lock); ++ ++ if (ret) ++ return ret; ++ ++ /* Map input page */ ++ ret = kbase_mmu_insert_pages(kbdev, &kbdev->csf.mcu_mmu, ++ reg->start_pfn, &phys[0], ++ 1, mem_flags, MCU_AS_NR, ++ KBASE_MEM_GROUP_CSF_IO); ++ if (ret) ++ goto bad_insert; ++ ++ /* Map output page, it needs rw access */ ++ mem_flags |= KBASE_REG_GPU_WR; ++ ret = kbase_mmu_insert_pages(kbdev, &kbdev->csf.mcu_mmu, ++ reg->start_pfn + 1, &phys[1], ++ 1, mem_flags, MCU_AS_NR, ++ KBASE_MEM_GROUP_CSF_IO); ++ if (ret) ++ goto bad_insert_output_page; ++ ++ return 0; ++ ++bad_insert_output_page: ++ kbase_mmu_teardown_pages(kbdev, &kbdev->csf.mcu_mmu, ++ reg->start_pfn, 1, MCU_AS_NR); ++bad_insert: ++ mutex_lock(&kbdev->csf.reg_lock); ++ kbase_remove_va_region(reg); ++ mutex_unlock(&kbdev->csf.reg_lock); ++ ++ return ret; ++} ++ ++static void kernel_unmap_user_io_pages(struct kbase_context *kctx, ++ struct kbase_queue *queue) ++{ ++ const size_t num_pages = 2; ++ ++ kbase_gpu_vm_lock(kctx); ++ ++ vunmap(queue->user_io_addr); ++ ++ WARN_ON(num_pages > atomic_read(&kctx->permanent_mapped_pages)); ++ atomic_sub(num_pages, &kctx->permanent_mapped_pages); ++ ++ kbase_gpu_vm_unlock(kctx); ++} ++ ++static int kernel_map_user_io_pages(struct kbase_context *kctx, ++ struct kbase_queue *queue) ++{ ++ struct page *page_list[2]; ++ pgprot_t cpu_map_prot; ++ int ret = 0; ++ size_t i; ++ ++ kbase_gpu_vm_lock(kctx); ++ ++ if (ARRAY_SIZE(page_list) > (KBASE_PERMANENTLY_MAPPED_MEM_LIMIT_PAGES - ++ atomic_read(&kctx->permanent_mapped_pages))) { ++ ret = -ENOMEM; ++ goto unlock; ++ } ++ ++ /* The pages are mapped to Userspace also, so use the same mapping ++ * attributes as used inside the CPU page fault handler. ++ */ ++#if ((KERNEL_VERSION(4, 4, 147) >= LINUX_VERSION_CODE) || \ ++ ((KERNEL_VERSION(4, 6, 0) > LINUX_VERSION_CODE) && \ ++ (KERNEL_VERSION(4, 5, 0) <= LINUX_VERSION_CODE))) ++ cpu_map_prot = pgprot_device(PAGE_KERNEL); ++#else ++ if (kctx->kbdev->system_coherency == COHERENCY_NONE) ++ cpu_map_prot = pgprot_writecombine(PAGE_KERNEL); ++ else ++ cpu_map_prot = PAGE_KERNEL; ++#endif ++ ++ for (i = 0; i < ARRAY_SIZE(page_list); i++) ++ page_list[i] = as_page(queue->phys[i]); ++ ++ queue->user_io_addr = vmap(page_list, ARRAY_SIZE(page_list), VM_MAP, cpu_map_prot); ++ ++ if (!queue->user_io_addr) ++ ret = -ENOMEM; ++ else ++ atomic_add(ARRAY_SIZE(page_list), &kctx->permanent_mapped_pages); ++ ++unlock: ++ kbase_gpu_vm_unlock(kctx); ++ return ret; ++} ++ ++static void get_queue(struct kbase_queue *queue); ++static void release_queue(struct kbase_queue *queue); ++ ++/** ++ * kbase_csf_free_command_stream_user_pages() - Free the resources allocated ++ * for a queue at the time of bind. ++ * ++ * @kctx: Address of the kbase context within which the queue was created. ++ * @queue: Pointer to the queue to be unlinked. ++ * ++ * This function will free the pair of physical pages allocated for a GPU ++ * command queue, and also release the hardware doorbell page, that were mapped ++ * into the process address space to enable direct submission of commands to ++ * the hardware. Also releases the reference taken on the queue when the mapping ++ * was created. ++ * ++ * This function will be called only when the mapping is being removed and ++ * so the resources for queue will not get freed up until the mapping is ++ * removed even though userspace could have terminated the queue. ++ * Kernel will ensure that the termination of Kbase context would only be ++ * triggered after the mapping is removed. ++ * ++ * If an explicit or implicit unbind was missed by the userspace then the ++ * mapping will persist. On process exit kernel itself will remove the mapping. ++ */ ++static void kbase_csf_free_command_stream_user_pages(struct kbase_context *kctx, ++ struct kbase_queue *queue) ++{ ++ const size_t num_pages = 2; ++ ++ gpu_munmap_user_io_pages(kctx, queue->reg); ++ kernel_unmap_user_io_pages(kctx, queue); ++ ++ kbase_mem_pool_free_pages( ++ &kctx->mem_pools.small[KBASE_MEM_GROUP_CSF_IO], ++ num_pages, queue->phys, true, false); ++ ++ kfree(queue->reg); ++ queue->reg = NULL; ++ ++ /* If the queue has already been terminated by userspace ++ * then the ref count for queue object will drop to 0 here. ++ */ ++ release_queue(queue); ++} ++ ++int kbase_csf_alloc_command_stream_user_pages(struct kbase_context *kctx, ++ struct kbase_queue *queue) ++{ ++ struct kbase_device *kbdev = kctx->kbdev; ++ struct kbase_va_region *reg; ++ const size_t num_pages = 2; ++ int ret; ++ ++ lockdep_assert_held(&kctx->csf.lock); ++ ++ reg = kbase_alloc_free_region(&kctx->kbdev->csf.shared_reg_rbtree, 0, ++ num_pages, KBASE_REG_ZONE_MCU_SHARED); ++ if (!reg) ++ return -ENOMEM; ++ ++ ret = kbase_mem_pool_alloc_pages( ++ &kctx->mem_pools.small[KBASE_MEM_GROUP_CSF_IO], ++ num_pages, queue->phys, false); ++ ++ if (ret != num_pages) ++ goto phys_alloc_failed; ++ ++ ret = kernel_map_user_io_pages(kctx, queue); ++ if (ret) ++ goto kernel_map_failed; ++ ++ init_user_output_page(queue); ++ ++ ret = gpu_mmap_user_io_pages(kctx->kbdev, queue->phys, reg); ++ if (ret) ++ goto gpu_mmap_failed; ++ ++ queue->reg = reg; ++ ++ mutex_lock(&kbdev->csf.reg_lock); ++ if (kbdev->csf.db_file_offsets > ++ (U32_MAX - BASEP_QUEUE_NR_MMAP_USER_PAGES + 1)) ++ kbdev->csf.db_file_offsets = 0; ++ ++ queue->db_file_offset = kbdev->csf.db_file_offsets; ++ kbdev->csf.db_file_offsets += BASEP_QUEUE_NR_MMAP_USER_PAGES; ++ ++ WARN(atomic_read(&queue->refcount) != 1, "Incorrect refcounting for queue object\n"); ++ /* This is the second reference taken on the queue object and ++ * would be dropped only when the IO mapping is removed either ++ * explicitly by userspace or implicitly by kernel on process exit. ++ */ ++ get_queue(queue); ++ queue->bind_state = KBASE_CSF_QUEUE_BOUND; ++ mutex_unlock(&kbdev->csf.reg_lock); ++ ++ return 0; ++ ++gpu_mmap_failed: ++ kernel_unmap_user_io_pages(kctx, queue); ++ ++kernel_map_failed: ++ kbase_mem_pool_free_pages( ++ &kctx->mem_pools.small[KBASE_MEM_GROUP_CSF_IO], ++ num_pages, queue->phys, false, false); ++ ++phys_alloc_failed: ++ kfree(reg); ++ ++ return -ENOMEM; ++} ++ ++static struct kbase_queue_group *find_queue_group(struct kbase_context *kctx, ++ u8 group_handle) ++{ ++ uint index = group_handle; ++ ++ lockdep_assert_held(&kctx->csf.lock); ++ ++ if (index < MAX_QUEUE_GROUP_NUM && kctx->csf.queue_groups[index]) { ++ if (WARN_ON(kctx->csf.queue_groups[index]->handle != index)) ++ return NULL; ++ return kctx->csf.queue_groups[index]; ++ } ++ ++ return NULL; ++} ++ ++int kbase_csf_queue_group_handle_is_valid(struct kbase_context *kctx, ++ u8 group_handle) ++{ ++ struct kbase_queue_group *group; ++ ++ mutex_lock(&kctx->csf.lock); ++ group = find_queue_group(kctx, group_handle); ++ mutex_unlock(&kctx->csf.lock); ++ ++ return group ? 0 : -EINVAL; ++} ++ ++static struct kbase_queue *find_queue(struct kbase_context *kctx, u64 base_addr) ++{ ++ struct kbase_queue *queue; ++ ++ lockdep_assert_held(&kctx->csf.lock); ++ ++ list_for_each_entry(queue, &kctx->csf.queue_list, link) { ++ if (base_addr == queue->base_addr) ++ return queue; ++ } ++ ++ return NULL; ++} ++ ++static void get_queue(struct kbase_queue *queue) ++{ ++ WARN_ON(!atomic_inc_not_zero(&queue->refcount)); ++} ++ ++static void release_queue(struct kbase_queue *queue) ++{ ++ lockdep_assert_held(&queue->kctx->csf.lock); ++ ++ WARN_ON(atomic_read(&queue->refcount) <= 0); ++ ++ if (atomic_dec_and_test(&queue->refcount)) { ++ /* The queue can't still be on the per context list. */ ++ WARN_ON(!list_empty(&queue->link)); ++ WARN_ON(queue->group); ++ kfree(queue); ++ } ++} ++ ++static void oom_event_worker(struct work_struct *data); ++static void fault_event_worker(struct work_struct *data); ++ ++int kbase_csf_queue_register(struct kbase_context *kctx, ++ struct kbase_ioctl_cs_queue_register *reg) ++{ ++ struct kbase_queue *queue; ++ int ret = 0; ++ struct kbase_va_region *region; ++ u64 queue_addr = reg->buffer_gpu_addr; ++ size_t queue_size = reg->buffer_size >> PAGE_SHIFT; ++ ++ /* Validate the queue priority */ ++ if (reg->priority > BASE_QUEUE_MAX_PRIORITY) ++ return -EINVAL; ++ ++ mutex_lock(&kctx->csf.lock); ++ ++ /* Check if queue is already registered */ ++ if (find_queue(kctx, queue_addr) != NULL) { ++ ret = -EINVAL; ++ goto out; ++ } ++ ++ /* Check if the queue address is valid */ ++ kbase_gpu_vm_lock(kctx); ++ region = kbase_region_tracker_find_region_enclosing_address(kctx, ++ queue_addr); ++ ++ if (kbase_is_region_invalid_or_free(region)) { ++ ret = -ENOENT; ++ goto out_unlock_vm; ++ } ++ ++ if (queue_size > (region->nr_pages - ++ ((queue_addr >> PAGE_SHIFT) - region->start_pfn))) { ++ ret = -EINVAL; ++ goto out_unlock_vm; ++ } ++ ++ queue = kzalloc(sizeof(struct kbase_queue), GFP_KERNEL); ++ ++ if (!queue) { ++ ret = -ENOMEM; ++ goto out_unlock_vm; ++ } ++ ++ queue->kctx = kctx; ++ queue->base_addr = queue_addr; ++ queue->queue_reg = region; ++ queue->size = (queue_size << PAGE_SHIFT); ++ queue->csi_index = KBASEP_IF_NR_INVALID; ++ queue->enabled = false; ++ ++ queue->priority = reg->priority; ++ atomic_set(&queue->refcount, 1); ++ ++ queue->group = NULL; ++ queue->bind_state = KBASE_CSF_QUEUE_UNBOUND; ++ queue->handle = BASEP_MEM_INVALID_HANDLE; ++ queue->doorbell_nr = KBASEP_USER_DB_NR_INVALID; ++ ++ queue->status_wait = 0; ++ queue->sync_ptr = 0; ++ queue->sync_value = 0; ++ ++ INIT_LIST_HEAD(&queue->link); ++ INIT_LIST_HEAD(&queue->error.link); ++ INIT_WORK(&queue->oom_event_work, oom_event_worker); ++ INIT_WORK(&queue->fault_event_work, fault_event_worker); ++ list_add(&queue->link, &kctx->csf.queue_list); ++ ++ region->flags |= KBASE_REG_NO_USER_FREE; ++ ++out_unlock_vm: ++ kbase_gpu_vm_unlock(kctx); ++out: ++ mutex_unlock(&kctx->csf.lock); ++ ++ return ret; ++} ++ ++static void unbind_queue(struct kbase_context *kctx, ++ struct kbase_queue *queue); ++ ++void kbase_csf_queue_terminate(struct kbase_context *kctx, ++ struct kbase_ioctl_cs_queue_terminate *term) ++{ ++ struct kbase_queue *queue; ++ ++ mutex_lock(&kctx->csf.lock); ++ ++ queue = find_queue(kctx, term->buffer_gpu_addr); ++ ++ if (queue) { ++ /* As the GPU queue has been terminated by the ++ * user space, undo the actions that were performed when the ++ * queue was registered i.e. remove the queue from the per ++ * context list & release the initial reference. The subsequent ++ * lookups for the queue in find_queue() would fail. ++ */ ++ list_del_init(&queue->link); ++ ++ /* Stop the CSI to which queue was bound */ ++ unbind_queue(kctx, queue); ++ ++ kbase_gpu_vm_lock(kctx); ++ if (!WARN_ON(!queue->queue_reg)) { ++ /* After this the Userspace would be able to free the ++ * memory for GPU queue. In case the Userspace missed ++ * terminating the queue, the cleanup will happen on ++ * context termination where teardown of region tracker ++ * would free up the GPU queue memory. ++ */ ++ queue->queue_reg->flags &= ~KBASE_REG_NO_USER_FREE; ++ } ++ kbase_gpu_vm_unlock(kctx); ++ ++ /* Remove any pending command queue fatal from ++ * the per-context list. ++ */ ++ list_del_init(&queue->error.link); ++ ++ release_queue(queue); ++ } ++ ++ mutex_unlock(&kctx->csf.lock); ++} ++ ++int kbase_csf_queue_bind(struct kbase_context *kctx, union kbase_ioctl_cs_queue_bind *bind) ++{ ++ struct kbase_queue *queue; ++ struct kbase_queue_group *group; ++ u8 max_streams; ++ int ret = -EINVAL; ++ ++ mutex_lock(&kctx->csf.lock); ++ ++ group = find_queue_group(kctx, bind->in.group_handle); ++ queue = find_queue(kctx, bind->in.buffer_gpu_addr); ++ ++ if (!group || !queue) ++ goto out; ++ ++ /* For the time being, all CSGs have the same number of CSs ++ * so we check CSG 0 for this number ++ */ ++ max_streams = kctx->kbdev->csf.global_iface.groups[0].stream_num; ++ ++ if (bind->in.csi_index >= max_streams) ++ goto out; ++ ++ if (group->run_state == KBASE_CSF_GROUP_TERMINATED) ++ goto out; ++ ++ if (queue->group || group->bound_queues[bind->in.csi_index]) ++ goto out; ++ ++ ret = get_user_pages_mmap_handle(kctx, queue); ++ if (ret) ++ goto out; ++ ++ bind->out.mmap_handle = queue->handle; ++ group->bound_queues[bind->in.csi_index] = queue; ++ queue->group = group; ++ queue->csi_index = bind->in.csi_index; ++ queue->bind_state = KBASE_CSF_QUEUE_BIND_IN_PROGRESS; ++ ++out: ++ mutex_unlock(&kctx->csf.lock); ++ ++ return ret; ++} ++ ++static struct kbase_queue_group *get_bound_queue_group( ++ struct kbase_queue *queue) ++{ ++ struct kbase_context *kctx = queue->kctx; ++ struct kbase_queue_group *group; ++ ++ if (queue->bind_state == KBASE_CSF_QUEUE_UNBOUND) ++ return NULL; ++ ++ if (!queue->group) ++ return NULL; ++ ++ if (queue->csi_index == KBASEP_IF_NR_INVALID) { ++ dev_warn(kctx->kbdev->dev, "CS interface index is incorrect\n"); ++ return NULL; ++ } ++ ++ group = queue->group; ++ ++ if (group->bound_queues[queue->csi_index] != queue) { ++ dev_warn(kctx->kbdev->dev, "Incorrect mapping between queues & queue groups\n"); ++ return NULL; ++ } ++ ++ return group; ++} ++ ++void kbase_csf_ring_csg_doorbell(struct kbase_device *kbdev, int slot) ++{ ++ if (WARN_ON(slot < 0)) ++ return; ++ ++ kbase_csf_ring_csg_slots_doorbell(kbdev, (u32) (1 << slot)); ++} ++ ++void kbase_csf_ring_csg_slots_doorbell(struct kbase_device *kbdev, ++ u32 slot_bitmap) ++{ ++ const struct kbase_csf_global_iface *const global_iface = ++ &kbdev->csf.global_iface; ++ const u32 allowed_bitmap = ++ (u32) ((1U << kbdev->csf.global_iface.group_num) - 1); ++ u32 value; ++ ++ if (WARN_ON(slot_bitmap > allowed_bitmap)) ++ return; ++ ++ value = kbase_csf_firmware_global_output(global_iface, GLB_DB_ACK); ++ value ^= slot_bitmap; ++ kbase_csf_firmware_global_input_mask(global_iface, GLB_DB_REQ, value, ++ slot_bitmap); ++ ++ kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); ++} ++ ++void kbase_csf_ring_cs_user_doorbell(struct kbase_device *kbdev, ++ struct kbase_queue *queue) ++{ ++ mutex_lock(&kbdev->csf.reg_lock); ++ ++ if (queue->doorbell_nr != KBASEP_USER_DB_NR_INVALID) ++ kbase_csf_ring_doorbell(kbdev, queue->doorbell_nr); ++ ++ mutex_unlock(&kbdev->csf.reg_lock); ++} ++ ++void kbase_csf_ring_cs_kernel_doorbell(struct kbase_device *kbdev, ++ struct kbase_queue *queue) ++{ ++ struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; ++ struct kbase_queue_group *group = get_bound_queue_group(queue); ++ struct kbase_csf_cmd_stream_group_info *ginfo; ++ u32 value; ++ int slot; ++ ++ if (WARN_ON(!group)) ++ return; ++ ++ slot = kbase_csf_scheduler_group_get_slot(group); ++ ++ if (WARN_ON(slot < 0)) ++ return; ++ ++ ginfo = &global_iface->groups[slot]; ++ ++ value = kbase_csf_firmware_csg_output(ginfo, CSG_DB_ACK); ++ value ^= (1 << queue->csi_index); ++ kbase_csf_firmware_csg_input_mask(ginfo, CSG_DB_REQ, value, ++ 1 << queue->csi_index); ++ ++ kbase_csf_ring_csg_doorbell(kbdev, slot); ++} ++ ++int kbase_csf_queue_kick(struct kbase_context *kctx, ++ struct kbase_ioctl_cs_queue_kick *kick) ++{ ++ struct kbase_queue_group *group; ++ struct kbase_queue *queue; ++ int err = 0; ++ ++ mutex_lock(&kctx->csf.lock); ++ ++ queue = find_queue(kctx, kick->buffer_gpu_addr); ++ if (!queue) ++ err = -EINVAL; ++ ++ if (!err) { ++ group = get_bound_queue_group(queue); ++ if (!group) { ++ dev_err(kctx->kbdev->dev, "queue not bound\n"); ++ err = -EINVAL; ++ } ++ } ++ ++ if (!err) ++ err = kbase_csf_scheduler_queue_start(queue); ++ ++ mutex_unlock(&kctx->csf.lock); ++ return err; ++} ++ ++static void unbind_stopped_queue(struct kbase_context *kctx, ++ struct kbase_queue *queue) ++{ ++ lockdep_assert_held(&kctx->csf.lock); ++ ++ if (queue->bind_state != KBASE_CSF_QUEUE_UNBOUND) { ++ unsigned long flags; ++ ++ kbase_csf_scheduler_spin_lock(kctx->kbdev, &flags); ++ bitmap_clear(queue->group->protm_pending_bitmap, ++ queue->csi_index, 1); ++ queue->group->bound_queues[queue->csi_index] = NULL; ++ queue->group = NULL; ++ kbase_csf_scheduler_spin_unlock(kctx->kbdev, flags); ++ ++ put_user_pages_mmap_handle(kctx, queue); ++ queue->bind_state = KBASE_CSF_QUEUE_UNBOUND; ++ } ++} ++/** ++ * unbind_queue() - Remove the linkage between a GPU command queue and the group ++ * to which it was bound or being bound. ++ * ++ * @kctx: Address of the kbase context within which the queue was created. ++ * @queue: Pointer to the queue to be unlinked. ++ * ++ * This function will also send the stop request to firmware for the command ++ * stream if the group to which the GPU command queue was bound is scheduled. ++ * ++ * This function would be called when :- ++ * - queue is being unbound. This would happen when the IO mapping ++ * created on bind is removed explicitly by userspace or the process ++ * is getting exited. ++ * - queue group is being terminated which still has queues bound ++ * to it. This could happen on an explicit terminate request from userspace ++ * or when the kbase context is being terminated. ++ * - queue is being terminated without completing the bind operation. ++ * This could happen if either the queue group is terminated ++ * after the CS_QUEUE_BIND ioctl but before the 2nd part of bind operation ++ * to create the IO mapping is initiated. ++ * - There is a failure in executing the 2nd part of bind operation, inside the ++ * mmap handler, which creates the IO mapping for queue. ++ */ ++ ++static void unbind_queue(struct kbase_context *kctx, struct kbase_queue *queue) ++{ ++ lockdep_assert_held(&kctx->csf.lock); ++ ++ if (queue->bind_state != KBASE_CSF_QUEUE_UNBOUND) { ++ if (queue->bind_state == KBASE_CSF_QUEUE_BOUND) ++ kbase_csf_scheduler_queue_stop(queue); ++ ++ unbind_stopped_queue(kctx, queue); ++ } ++} ++ ++void kbase_csf_queue_unbind(struct kbase_queue *queue) ++{ ++ struct kbase_context *kctx = queue->kctx; ++ ++ lockdep_assert_held(&kctx->csf.lock); ++ ++ unbind_queue(kctx, queue); ++ ++ /* Free the resources, if allocated for this queue. */ ++ if (queue->reg) ++ kbase_csf_free_command_stream_user_pages(kctx, queue); ++} ++ ++/** ++ * find_free_group_handle() - Find a free handle for a queue group ++ * ++ * @kctx: Address of the kbase context within which the queue group ++ * is to be created. ++ * ++ * Return: a queue group handle on success, or a negative error code on failure. ++ */ ++static int find_free_group_handle(struct kbase_context *const kctx) ++{ ++ /* find the available index in the array of CSGs per this context */ ++ int idx, group_handle = -ENOMEM; ++ ++ lockdep_assert_held(&kctx->csf.lock); ++ ++ for (idx = 0; ++ (idx != MAX_QUEUE_GROUP_NUM) && (group_handle < 0); ++ idx++) { ++ if (!kctx->csf.queue_groups[idx]) ++ group_handle = idx; ++ } ++ ++ return group_handle; ++} ++ ++/** ++ * iface_has_enough_streams() - Check that at least one command stream ++ * group supports a given number of streams ++ * ++ * @kbdev: Instance of a GPU platform device that implements a command ++ * stream front-end interface. ++ * @cs_min: Minimum number of command streams required. ++ * ++ * Return: true if at least one command stream group supports the given number ++ * of command streams (or more); otherwise false. ++ */ ++static bool iface_has_enough_streams(struct kbase_device *const kbdev, ++ u32 const cs_min) ++{ ++ bool has_enough = false; ++ struct kbase_csf_cmd_stream_group_info *const groups = ++ kbdev->csf.global_iface.groups; ++ const u32 group_num = kbdev->csf.global_iface.group_num; ++ u32 i; ++ ++ for (i = 0; (i < group_num) && !has_enough; i++) { ++ if (groups[i].stream_num >= cs_min) ++ has_enough = true; ++ } ++ ++ return has_enough; ++} ++ ++/** ++ * create_normal_suspend_buffer() - Create normal-mode suspend buffer per ++ * queue group ++ * ++ * @kctx: Pointer to kbase context where the queue group is created at ++ * @s_buf: Pointer to suspend buffer that is attached to queue group ++ * ++ * Return: 0 if suspend buffer is successfully allocated and reflected to GPU ++ * MMU page table. Otherwise -ENOMEM. ++ */ ++static int create_normal_suspend_buffer(struct kbase_context *const kctx, ++ struct kbase_normal_suspend_buffer *s_buf) ++{ ++ struct kbase_va_region *reg = NULL; ++ const unsigned long mem_flags = KBASE_REG_GPU_RD | KBASE_REG_GPU_WR; ++ const size_t nr_pages = ++ PFN_UP(kctx->kbdev->csf.global_iface.groups[0].suspend_size); ++ int err = 0; ++ ++ lockdep_assert_held(&kctx->csf.lock); ++ ++ /* Allocate and initialize Region Object */ ++ reg = kbase_alloc_free_region(&kctx->kbdev->csf.shared_reg_rbtree, 0, ++ nr_pages, KBASE_REG_ZONE_MCU_SHARED); ++ ++ if (!reg) ++ return -ENOMEM; ++ ++ s_buf->phy = kcalloc(nr_pages, sizeof(*s_buf->phy), GFP_KERNEL); ++ ++ if (!s_buf->phy) { ++ err = -ENOMEM; ++ goto phy_alloc_failed; ++ } ++ ++ /* Get physical page for a normal suspend buffer */ ++ err = kbase_mem_pool_alloc_pages( ++ &kctx->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], ++ nr_pages, &s_buf->phy[0], false); ++ ++ if (err < 0) ++ goto phy_pages_alloc_failed; ++ ++ /* Insert Region Object into rbtree and make virtual address available ++ * to map it to physical page ++ */ ++ mutex_lock(&kctx->kbdev->csf.reg_lock); ++ err = kbase_add_va_region_rbtree(kctx->kbdev, reg, 0, nr_pages, 1); ++ reg->flags &= ~KBASE_REG_FREE; ++ mutex_unlock(&kctx->kbdev->csf.reg_lock); ++ ++ if (err) ++ goto add_va_region_failed; ++ ++ /* Update MMU table */ ++ err = kbase_mmu_insert_pages(kctx->kbdev, &kctx->kbdev->csf.mcu_mmu, ++ reg->start_pfn, &s_buf->phy[0], ++ nr_pages, mem_flags, ++ MCU_AS_NR, KBASE_MEM_GROUP_CSF_FW); ++ if (err) ++ goto mmu_insert_failed; ++ ++ s_buf->reg = reg; ++ ++ return 0; ++ ++mmu_insert_failed: ++ mutex_lock(&kctx->kbdev->csf.reg_lock); ++ WARN_ON(kbase_remove_va_region(reg)); ++ mutex_unlock(&kctx->kbdev->csf.reg_lock); ++ ++add_va_region_failed: ++ kbase_mem_pool_free_pages( ++ &kctx->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], nr_pages, ++ &s_buf->phy[0], false, false); ++ ++phy_pages_alloc_failed: ++ kfree(s_buf->phy); ++phy_alloc_failed: ++ kfree(reg); ++ ++ return err; ++} ++ ++/** ++ * create_protected_suspend_buffer() - Create protected-mode suspend buffer ++ * per queue group ++ * ++ * @kbdev: Instance of a GPU platform device that implements a command ++ * stream front-end interface. ++ * @s_buf: Pointer to suspend buffer that is attached to queue group ++ * ++ * Return: 0 if suspend buffer is successfully allocated and reflected to GPU ++ * MMU page table. Otherwise -ENOMEM. ++ */ ++static int create_protected_suspend_buffer(struct kbase_device *const kbdev, ++ struct kbase_protected_suspend_buffer *s_buf) ++{ ++ struct kbase_va_region *reg = NULL; ++ struct tagged_addr *phys = NULL; ++ const unsigned long mem_flags = KBASE_REG_GPU_RD | KBASE_REG_GPU_WR; ++ const size_t nr_pages = ++ PFN_UP(kbdev->csf.global_iface.groups[0].suspend_size); ++ int err = 0; ++ ++ /* Allocate and initialize Region Object */ ++ reg = kbase_alloc_free_region(&kbdev->csf.shared_reg_rbtree, 0, ++ nr_pages, KBASE_REG_ZONE_MCU_SHARED); ++ ++ if (!reg) ++ return -ENOMEM; ++ ++ phys = kcalloc(nr_pages, sizeof(*phys), GFP_KERNEL); ++ if (!phys) { ++ err = -ENOMEM; ++ goto phy_alloc_failed; ++ } ++ ++ s_buf->pma = kbase_csf_protected_memory_alloc(kbdev, phys, ++ nr_pages); ++ if (s_buf->pma == NULL) { ++ err = -ENOMEM; ++ goto pma_alloc_failed; ++ } ++ ++ /* Insert Region Object into rbtree and make virtual address available ++ * to map it to physical page ++ */ ++ mutex_lock(&kbdev->csf.reg_lock); ++ err = kbase_add_va_region_rbtree(kbdev, reg, 0, nr_pages, 1); ++ reg->flags &= ~KBASE_REG_FREE; ++ mutex_unlock(&kbdev->csf.reg_lock); ++ ++ if (err) ++ goto add_va_region_failed; ++ ++ /* Update MMU table */ ++ err = kbase_mmu_insert_pages(kbdev, &kbdev->csf.mcu_mmu, ++ reg->start_pfn, phys, ++ nr_pages, mem_flags, MCU_AS_NR, ++ KBASE_MEM_GROUP_CSF_FW); ++ if (err) ++ goto mmu_insert_failed; ++ ++ s_buf->reg = reg; ++ kfree(phys); ++ return 0; ++ ++mmu_insert_failed: ++ mutex_lock(&kbdev->csf.reg_lock); ++ WARN_ON(kbase_remove_va_region(reg)); ++ mutex_unlock(&kbdev->csf.reg_lock); ++ ++add_va_region_failed: ++ kbase_csf_protected_memory_free(kbdev, s_buf->pma, nr_pages); ++pma_alloc_failed: ++ kfree(phys); ++phy_alloc_failed: ++ kfree(reg); ++ ++ return err; ++} ++ ++static void timer_event_worker(struct work_struct *data); ++static void protm_event_worker(struct work_struct *data); ++static void term_normal_suspend_buffer(struct kbase_context *const kctx, ++ struct kbase_normal_suspend_buffer *s_buf); ++ ++/** ++ * create_suspend_buffers - Setup normal and protected mode ++ * suspend buffers. ++ * ++ * @kctx: Address of the kbase context within which the queue group ++ * is to be created. ++ * @group: Pointer to GPU command queue group data. ++ * ++ * Return: 0 if suspend buffers are successfully allocated. Otherwise -ENOMEM. ++ */ ++static int create_suspend_buffers(struct kbase_context *const kctx, ++ struct kbase_queue_group * const group) ++{ ++ int err = 0; ++ ++ if (create_normal_suspend_buffer(kctx, &group->normal_suspend_buf)) { ++ dev_err(kctx->kbdev->dev, "Failed to create normal suspend buffer\n"); ++ return -ENOMEM; ++ } ++ ++ if (kctx->kbdev->csf.pma_dev) { ++ err = create_protected_suspend_buffer(kctx->kbdev, ++ &group->protected_suspend_buf); ++ if (err) { ++ term_normal_suspend_buffer(kctx, ++ &group->normal_suspend_buf); ++ dev_err(kctx->kbdev->dev, "Failed to create protected suspend buffer\n"); ++ } ++ } else { ++ group->protected_suspend_buf.reg = NULL; ++ } ++ ++ return err; ++} ++ ++/** ++ * create_queue_group() - Create a queue group ++ * ++ * @kctx: Address of the kbase context within which the queue group ++ * is to be created. ++ * @create: Address of a structure which contains details of the ++ * queue group which is to be created. ++ * ++ * Return: a queue group handle on success, or a negative error code on failure. ++ */ ++static int create_queue_group(struct kbase_context *const kctx, ++ const union kbase_ioctl_cs_queue_group_create *const create) ++{ ++ int group_handle = find_free_group_handle(kctx); ++ ++ if (group_handle < 0) { ++ dev_err(kctx->kbdev->dev, ++ "All queue group handles are already in use\n"); ++ } else { ++ struct kbase_queue_group * const group = ++ kmalloc(sizeof(struct kbase_queue_group), ++ GFP_KERNEL); ++ ++ lockdep_assert_held(&kctx->csf.lock); ++ ++ if (!group) { ++ dev_err(kctx->kbdev->dev, "Failed to allocate a queue\n"); ++ group_handle = -ENOMEM; ++ } else { ++ int err = 0; ++ ++ group->kctx = kctx; ++ group->handle = group_handle; ++ group->csg_nr = KBASEP_CSG_NR_INVALID; ++ ++ group->tiler_mask = create->in.tiler_mask; ++ group->fragment_mask = create->in.fragment_mask; ++ group->compute_mask = create->in.compute_mask; ++ ++ group->tiler_max = create->in.tiler_max; ++ group->fragment_max = create->in.fragment_max; ++ group->compute_max = create->in.compute_max; ++ group->priority = create->in.priority; ++ group->doorbell_nr = KBASEP_USER_DB_NR_INVALID; ++ group->faulted = false; ++ ++ INIT_LIST_HEAD(&group->link); ++ INIT_LIST_HEAD(&group->link_to_schedule); ++ INIT_LIST_HEAD(&group->error_fatal.link); ++ INIT_LIST_HEAD(&group->error_timeout.link); ++ INIT_LIST_HEAD(&group->error_tiler_oom.link); ++ INIT_WORK(&group->timer_event_work, timer_event_worker); ++ INIT_WORK(&group->protm_event_work, protm_event_worker); ++ bitmap_zero(group->protm_pending_bitmap, ++ MAX_SUPPORTED_STREAMS_PER_GROUP); ++ ++ group->run_state = KBASE_CSF_GROUP_INACTIVE; ++ err = create_suspend_buffers(kctx, group); ++ ++ if (err < 0) { ++ kfree(group); ++ group_handle = err; ++ } else { ++ int j; ++ ++ kctx->csf.queue_groups[group_handle] = group; ++ for (j = 0; j < MAX_SUPPORTED_STREAMS_PER_GROUP; ++ j++) ++ group->bound_queues[j] = NULL; ++ } ++ } ++ } ++ ++ return group_handle; ++} ++ ++int kbase_csf_queue_group_create(struct kbase_context *const kctx, ++ union kbase_ioctl_cs_queue_group_create *const create) ++{ ++ int err = 0; ++ const u32 tiler_count = hweight64(create->in.tiler_mask); ++ const u32 fragment_count = hweight64(create->in.fragment_mask); ++ const u32 compute_count = hweight64(create->in.compute_mask); ++ ++ mutex_lock(&kctx->csf.lock); ++ ++ if ((create->in.tiler_max > tiler_count) || ++ (create->in.fragment_max > fragment_count) || ++ (create->in.compute_max > compute_count)) { ++ dev_err(kctx->kbdev->dev, ++ "Invalid maximum number of endpoints for a queue group\n"); ++ err = -EINVAL; ++ } else if (create->in.priority >= BASE_QUEUE_GROUP_PRIORITY_COUNT) { ++ dev_err(kctx->kbdev->dev, "Invalid queue group priority %u\n", ++ (unsigned int)create->in.priority); ++ err = -EINVAL; ++ } else if (!iface_has_enough_streams(kctx->kbdev, create->in.cs_min)) { ++ dev_err(kctx->kbdev->dev, ++ "No CSG has at least %d streams\n", ++ create->in.cs_min); ++ err = -EINVAL; ++ } else { ++ /* For the CSG which satisfies the condition for having ++ * the needed number of CSs, check whether it also conforms ++ * with the requirements for at least one of its CSs having ++ * the iterator of the needed type ++ * (note: for CSF v1.0 all CSs in a CSG will have access to ++ * the same iterators) ++ */ ++ const int group_handle = create_queue_group(kctx, create); ++ ++ if (group_handle >= 0) ++ create->out.group_handle = group_handle; ++ else ++ err = group_handle; ++ } ++ ++ mutex_unlock(&kctx->csf.lock); ++ ++ return err; ++} ++ ++/** ++ * term_normal_suspend_buffer() - Free normal-mode suspend buffer of queue group ++ * ++ * @kctx: Pointer to kbase context where queue group belongs to ++ * @s_buf: Pointer to queue group suspend buffer to be freed ++ */ ++static void term_normal_suspend_buffer(struct kbase_context *const kctx, ++ struct kbase_normal_suspend_buffer *s_buf) ++{ ++ const size_t nr_pages = ++ PFN_UP(kctx->kbdev->csf.global_iface.groups[0].suspend_size); ++ ++ lockdep_assert_held(&kctx->csf.lock); ++ ++ WARN_ON(kbase_mmu_teardown_pages( ++ kctx->kbdev, &kctx->kbdev->csf.mcu_mmu, ++ s_buf->reg->start_pfn, nr_pages, MCU_AS_NR)); ++ ++ WARN_ON(s_buf->reg->flags & KBASE_REG_FREE); ++ ++ mutex_lock(&kctx->kbdev->csf.reg_lock); ++ WARN_ON(kbase_remove_va_region(s_buf->reg)); ++ mutex_unlock(&kctx->kbdev->csf.reg_lock); ++ ++ kbase_mem_pool_free_pages( ++ &kctx->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], ++ nr_pages, &s_buf->phy[0], false, false); ++ ++ kfree(s_buf->phy); ++ s_buf->phy = NULL; ++ kfree(s_buf->reg); ++ s_buf->reg = NULL; ++} ++ ++/** ++ * term_protected_suspend_buffer() - Free normal-mode suspend buffer of ++ * queue group ++ * ++ * @kbdev: Instance of a GPU platform device that implements a command ++ * stream front-end interface. ++ * @s_buf: Pointer to queue group suspend buffer to be freed ++ */ ++static void term_protected_suspend_buffer(struct kbase_device *const kbdev, ++ struct kbase_protected_suspend_buffer *s_buf) ++{ ++ const size_t nr_pages = ++ PFN_UP(kbdev->csf.global_iface.groups[0].suspend_size); ++ ++ WARN_ON(kbase_mmu_teardown_pages( ++ kbdev, &kbdev->csf.mcu_mmu, ++ s_buf->reg->start_pfn, nr_pages, MCU_AS_NR)); ++ ++ WARN_ON(s_buf->reg->flags & KBASE_REG_FREE); ++ ++ mutex_lock(&kbdev->csf.reg_lock); ++ WARN_ON(kbase_remove_va_region(s_buf->reg)); ++ mutex_unlock(&kbdev->csf.reg_lock); ++ ++ kbase_csf_protected_memory_free(kbdev, s_buf->pma, nr_pages); ++ s_buf->pma = NULL; ++ kfree(s_buf->reg); ++ s_buf->reg = NULL; ++} ++ ++void kbase_csf_term_descheduled_queue_group(struct kbase_queue_group *group) ++{ ++ struct kbase_context *kctx = group->kctx; ++ ++ /* Currently each group supports the same number of streams */ ++ u32 max_streams = ++ kctx->kbdev->csf.global_iface.groups[0].stream_num; ++ u32 i; ++ ++ lockdep_assert_held(&kctx->csf.lock); ++ ++ WARN_ON(group->run_state != KBASE_CSF_GROUP_INACTIVE && ++ group->run_state != KBASE_CSF_GROUP_FAULT_EVICTED); ++ ++ for (i = 0; i < max_streams; i++) { ++ struct kbase_queue *queue = ++ group->bound_queues[i]; ++ ++ /* The group is already being evicted from the scheduler */ ++ if (queue) ++ unbind_stopped_queue(kctx, queue); ++ } ++ ++ term_normal_suspend_buffer(kctx, &group->normal_suspend_buf); ++ if (kctx->kbdev->csf.pma_dev) ++ term_protected_suspend_buffer(kctx->kbdev, ++ &group->protected_suspend_buf); ++ ++ group->run_state = KBASE_CSF_GROUP_TERMINATED; ++} ++ ++/** ++ * term_queue_group - Terminate a GPU command queue group. ++ * ++ * @group: Pointer to GPU command queue group data. ++ * ++ * Terminates a GPU command queue group. From the userspace perspective the ++ * group will still exist but it can't bind new queues to it. Userspace can ++ * still add work in queues bound to the group but it won't be executed. (This ++ * is because the IO mapping created upon binding such queues is still intact.) ++ */ ++static void term_queue_group(struct kbase_queue_group *group) ++{ ++ struct kbase_context *kctx = group->kctx; ++ ++ lockdep_assert_held(&kctx->csf.lock); ++ ++ /* Stop the group and evict it from the scheduler */ ++ kbase_csf_scheduler_group_deschedule(group); ++ ++ if (group->run_state == KBASE_CSF_GROUP_TERMINATED) ++ return; ++ ++ dev_dbg(kctx->kbdev->dev, "group %d terminating", group->handle); ++ ++ kbase_csf_term_descheduled_queue_group(group); ++} ++ ++static void cancel_queue_group_events(struct kbase_queue_group *group) ++{ ++ cancel_work_sync(&group->timer_event_work); ++ cancel_work_sync(&group->protm_event_work); ++} ++ ++void kbase_csf_queue_group_terminate(struct kbase_context *kctx, ++ u8 group_handle) ++{ ++ struct kbase_queue_group *group; ++ ++ mutex_lock(&kctx->csf.lock); ++ ++ group = find_queue_group(kctx, group_handle); ++ ++ if (group) { ++ /* Remove any pending group fatal error from the per-context list. */ ++ list_del_init(&group->error_tiler_oom.link); ++ list_del_init(&group->error_timeout.link); ++ list_del_init(&group->error_fatal.link); ++ ++ term_queue_group(group); ++ kctx->csf.queue_groups[group_handle] = NULL; ++ } ++ ++ mutex_unlock(&kctx->csf.lock); ++ ++ if (!group) ++ return; ++ ++ /* Cancel any pending event callbacks. If one is in progress ++ * then this thread waits synchronously for it to complete (which ++ * is why we must unlock the context first). We already ensured ++ * that no more callbacks can be enqueued by terminating the group. ++ */ ++ cancel_queue_group_events(group); ++ kfree(group); ++} ++ ++int kbase_csf_queue_group_suspend(struct kbase_context *kctx, ++ struct kbase_suspend_copy_buffer *sus_buf, ++ u8 group_handle) ++{ ++ int err = -EINVAL; ++ struct kbase_queue_group *group; ++ ++ mutex_lock(&kctx->csf.lock); ++ ++ group = find_queue_group(kctx, group_handle); ++ if (group) ++ err = kbase_csf_scheduler_group_copy_suspend_buf(group, ++ sus_buf); ++ ++ mutex_unlock(&kctx->csf.lock); ++ return err; ++} ++ ++/** ++ * kbase_csf_add_fatal_error_to_kctx - Add a fatal error to per-ctx error list. ++ * ++ * @group: GPU command queue group. ++ * @err_payload: Error payload to report. ++ */ ++static void kbase_csf_add_fatal_error_to_kctx( ++ struct kbase_queue_group *const group, ++ const struct base_gpu_queue_group_error *const err_payload) ++{ ++ struct base_csf_notification error; ++ ++ if (WARN_ON(!group)) ++ return; ++ ++ if (WARN_ON(!err_payload)) ++ return; ++ ++ error = (struct base_csf_notification) { ++ .type = BASE_CSF_NOTIFICATION_GPU_QUEUE_GROUP_ERROR, ++ .payload = { ++ .csg_error = { ++ .handle = group->handle, ++ .error = *err_payload ++ } ++ } ++ }; ++ ++ lockdep_assert_held(&group->kctx->csf.lock); ++ ++ /* If this group has already been in fatal error status, ++ * subsequent fatal error on this group should never take place. ++ */ ++ if (!WARN_ON(!list_empty(&group->error_fatal.link))) { ++ group->error_fatal.data = error; ++ list_add_tail(&group->error_fatal.link, ++ &group->kctx->csf.error_list); ++ } ++} ++ ++void kbase_csf_active_queue_groups_reset(struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++ struct list_head evicted_groups; ++ struct kbase_queue_group *group; ++ int i; ++ bool fatal_error_built = false; ++ ++ INIT_LIST_HEAD(&evicted_groups); ++ ++ mutex_lock(&kctx->csf.lock); ++ ++ kbase_csf_scheduler_evict_ctx_slots(kbdev, kctx, &evicted_groups); ++ while (!list_empty(&evicted_groups)) { ++ struct kbase_csf_scheduler *scheduler = ++ &kbdev->csf.scheduler; ++ unsigned long flags; ++ ++ group = list_first_entry(&evicted_groups, ++ struct kbase_queue_group, link); ++ ++ dev_dbg(kbdev->dev, "Context %d_%d active group %d terminated", ++ kctx->tgid, kctx->id, group->handle); ++ kbase_csf_term_descheduled_queue_group(group); ++ list_del_init(&group->link); ++ ++ kbase_csf_scheduler_spin_lock(kbdev, &flags); ++ if ((group == scheduler->active_protm_grp) && ++ group->faulted) { ++ const struct base_gpu_queue_group_error err_payload = { ++ .error_type = BASE_GPU_QUEUE_GROUP_ERROR_FATAL, ++ .payload = { ++ .fatal_group = { ++ .status = GPU_EXCEPTION_TYPE_SW_FAULT_0, ++ } ++ } ++ }; ++ ++ kbase_csf_add_fatal_error_to_kctx(group, &err_payload); ++ fatal_error_built = true; ++ } ++ kbase_csf_scheduler_spin_unlock(kbdev, flags); ++ } ++ ++ if (fatal_error_built) ++ kbase_event_wakeup(kctx); ++ ++ /* Acting on the queue groups that are pending to be terminated. */ ++ for (i = 0; i < MAX_QUEUE_GROUP_NUM; i++) { ++ group = kctx->csf.queue_groups[i]; ++ if (group && ++ group->run_state == KBASE_CSF_GROUP_FAULT_EVICTED) ++ kbase_csf_term_descheduled_queue_group(group); ++ } ++ ++ mutex_unlock(&kctx->csf.lock); ++} ++ ++int kbase_csf_ctx_init(struct kbase_context *kctx) ++{ ++ int err = -ENOMEM; ++ ++ INIT_LIST_HEAD(&kctx->csf.event_callback_list); ++ INIT_LIST_HEAD(&kctx->csf.queue_list); ++ INIT_LIST_HEAD(&kctx->csf.link); ++ INIT_LIST_HEAD(&kctx->csf.error_list); ++ ++ spin_lock_init(&kctx->csf.event_lock); ++ kctx->csf.user_reg_vma = NULL; ++ ++ /* Mark all the cookies as 'free' */ ++ bitmap_fill(kctx->csf.cookies, KBASE_CSF_NUM_USER_IO_PAGES_HANDLE); ++ ++ kctx->csf.wq = alloc_workqueue("mali_kbase_csf_wq", ++ WQ_UNBOUND, 1); ++ ++ if (likely(kctx->csf.wq)) { ++ err = kbase_csf_scheduler_context_init(kctx); ++ ++ if (likely(!err)) { ++ err = kbase_csf_kcpu_queue_context_init(kctx); ++ ++ if (likely(!err)) { ++ err = kbase_csf_tiler_heap_context_init(kctx); ++ ++ if (likely(!err)) ++ mutex_init(&kctx->csf.lock); ++ else ++ kbase_csf_kcpu_queue_context_term(kctx); ++ } ++ ++ if (unlikely(err)) ++ kbase_csf_scheduler_context_term(kctx); ++ } ++ ++ if (unlikely(err)) ++ destroy_workqueue(kctx->csf.wq); ++ } ++ ++ return err; ++} ++ ++void kbase_csf_ctx_handle_fault(struct kbase_context *kctx, ++ struct kbase_fault *fault) ++{ ++ int gr; ++ bool reported = false; ++ struct base_gpu_queue_group_error err_payload; ++ ++ if (WARN_ON(!kctx)) ++ return; ++ ++ if (WARN_ON(!fault)) ++ return; ++ ++ err_payload = (struct base_gpu_queue_group_error) { ++ .error_type = BASE_GPU_QUEUE_GROUP_ERROR_FATAL, ++ .payload = { ++ .fatal_group = { ++ .sideband = fault->addr, ++ .status = fault->status, ++ } ++ } ++ }; ++ ++ mutex_lock(&kctx->csf.lock); ++ ++ for (gr = 0; gr < MAX_QUEUE_GROUP_NUM; gr++) { ++ struct kbase_queue_group *const group = ++ kctx->csf.queue_groups[gr]; ++ ++ if (group && group->run_state != KBASE_CSF_GROUP_TERMINATED) { ++ term_queue_group(group); ++ kbase_csf_add_fatal_error_to_kctx(group, &err_payload); ++ reported = true; ++ } ++ } ++ ++ mutex_unlock(&kctx->csf.lock); ++ ++ if (reported) ++ kbase_event_wakeup(kctx); ++} ++ ++void kbase_csf_ctx_term(struct kbase_context *kctx) ++{ ++ u32 i; ++ ++ /* As the kbase context is terminating, its debugfs sub-directory would ++ * have been removed already and so would be the debugfs file created ++ * for queue groups & kcpu queues, hence no need to explicitly remove ++ * those debugfs files. ++ */ ++ kbase_csf_event_wait_remove_all(kctx); ++ ++ mutex_lock(&kctx->csf.lock); ++ ++ /* Iterate through the queue groups that were not terminated by ++ * userspace and issue the term request to firmware for them. ++ */ ++ for (i = 0; i < MAX_QUEUE_GROUP_NUM; i++) { ++ if (kctx->csf.queue_groups[i]) ++ term_queue_group(kctx->csf.queue_groups[i]); ++ } ++ ++ mutex_unlock(&kctx->csf.lock); ++ ++ /* Now that all queue groups have been terminated, there can be no ++ * more OoM or timer event interrupts but there can be inflight work ++ * items. Destroying the wq will implicitly flush those work items. ++ */ ++ destroy_workqueue(kctx->csf.wq); ++ ++ mutex_lock(&kctx->csf.lock); ++ ++ for (i = 0; i < MAX_QUEUE_GROUP_NUM; i++) ++ kfree(kctx->csf.queue_groups[i]); ++ ++ /* Iterate through the queues that were not terminated by ++ * userspace and do the required cleanup for them. ++ */ ++ while (!list_empty(&kctx->csf.queue_list)) { ++ struct kbase_queue *queue; ++ ++ queue = list_first_entry(&kctx->csf.queue_list, ++ struct kbase_queue, link); ++ ++ /* The reference held when the IO mapping was created on bind ++ * would have been dropped otherwise the termination of Kbase ++ * context itself wouldn't have kicked-in. So there shall be ++ * only one reference left that was taken when queue was ++ * registered. ++ */ ++ if (atomic_read(&queue->refcount) != 1) ++ dev_warn(kctx->kbdev->dev, ++ "Releasing queue with incorrect refcounting!\n"); ++ list_del_init(&queue->link); ++ release_queue(queue); ++ } ++ ++ mutex_unlock(&kctx->csf.lock); ++ ++ kbase_csf_tiler_heap_context_term(kctx); ++ kbase_csf_kcpu_queue_context_term(kctx); ++ kbase_csf_scheduler_context_term(kctx); ++ ++ mutex_destroy(&kctx->csf.lock); ++} ++ ++int kbase_csf_event_wait_add(struct kbase_context *kctx, ++ kbase_csf_event_callback *callback, void *param) ++{ ++ int err = -ENOMEM; ++ struct kbase_csf_event *event = ++ kzalloc(sizeof(struct kbase_csf_event), GFP_KERNEL); ++ ++ if (event) { ++ unsigned long flags; ++ ++ event->kctx = kctx; ++ event->callback = callback; ++ event->param = param; ++ ++ spin_lock_irqsave(&kctx->csf.event_lock, flags); ++ list_add_tail(&event->link, &kctx->csf.event_callback_list); ++ spin_unlock_irqrestore(&kctx->csf.event_lock, flags); ++ ++ err = 0; ++ } ++ ++ return err; ++} ++ ++void kbase_csf_event_wait_remove(struct kbase_context *kctx, ++ kbase_csf_event_callback *callback, void *param) ++{ ++ struct kbase_csf_event *event; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kctx->csf.event_lock, flags); ++ ++ list_for_each_entry(event, &kctx->csf.event_callback_list, link) { ++ if ((event->callback == callback) && (event->param == param)) { ++ list_del(&event->link); ++ kfree(event); ++ break; ++ } ++ } ++ spin_unlock_irqrestore(&kctx->csf.event_lock, flags); ++} ++ ++bool kbase_csf_read_error(struct kbase_context *kctx, ++ struct base_csf_notification *event_data) ++{ ++ bool got_event = true; ++ struct kbase_csf_notification *error_data = NULL; ++ ++ mutex_lock(&kctx->csf.lock); ++ ++ if (likely(!list_empty(&kctx->csf.error_list))) { ++ error_data = list_first_entry(&kctx->csf.error_list, ++ struct kbase_csf_notification, link); ++ list_del_init(&error_data->link); ++ *event_data = error_data->data; ++ } else { ++ got_event = false; ++ } ++ ++ mutex_unlock(&kctx->csf.lock); ++ ++ return got_event; ++} ++ ++bool kbase_csf_error_pending(struct kbase_context *kctx) ++{ ++ bool event_pended = false; ++ ++ mutex_lock(&kctx->csf.lock); ++ event_pended = !list_empty(&kctx->csf.error_list); ++ mutex_unlock(&kctx->csf.lock); ++ ++ return event_pended; ++} ++ ++void kbase_csf_event_signal(struct kbase_context *kctx, bool notify_gpu) ++{ ++ struct kbase_csf_event *event, *next_event; ++ unsigned long flags; ++ ++ /* First increment the signal count and wake up event thread. ++ */ ++ atomic_set(&kctx->event_count, 1); ++ kbase_event_wakeup(kctx); ++ ++ /* Signal the CSF firmware. This is to ensure that pending command ++ * stream synch object wait operations are re-evaluated. ++ * Write to GLB_DOORBELL would suffice as spec says that all pending ++ * synch object wait operations are re-evaluated on a write to any ++ * CS_DOORBELL/GLB_DOORBELL register. ++ */ ++ if (notify_gpu) { ++ spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, flags); ++ if (kctx->kbdev->pm.backend.gpu_powered) ++ kbase_csf_ring_doorbell(kctx->kbdev, CSF_KERNEL_DOORBELL_NR); ++ spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, flags); ++ } ++ ++ /* Now invoke the callbacks registered on backend side. ++ * Allow item removal inside the loop, if requested by the callback. ++ */ ++ spin_lock_irqsave(&kctx->csf.event_lock, flags); ++ ++ list_for_each_entry_safe( ++ event, next_event, &kctx->csf.event_callback_list, link) { ++ enum kbase_csf_event_callback_action action = ++ event->callback(event->param); ++ ++ if (action == KBASE_CSF_EVENT_CALLBACK_REMOVE) { ++ list_del(&event->link); ++ kfree(event); ++ } ++ } ++ ++ spin_unlock_irqrestore(&kctx->csf.event_lock, flags); ++} ++ ++void kbase_csf_event_wait_remove_all(struct kbase_context *kctx) ++{ ++ struct kbase_csf_event *event, *next_event; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kctx->csf.event_lock, flags); ++ ++ list_for_each_entry_safe( ++ event, next_event, &kctx->csf.event_callback_list, link) { ++ list_del(&event->link); ++ kfree(event); ++ } ++ ++ spin_unlock_irqrestore(&kctx->csf.event_lock, flags); ++} ++ ++/** ++ * handle_oom_event - Handle the OoM event generated by the firmware for the ++ * command stream interface. ++ * ++ * This function will handle the OoM event request from the firmware for the ++ * command stream. It will retrieve the address of heap context and heap's ++ * statistics (like number of render passes in-flight) from the command ++ * stream's kernel output page and pass them to the tiler heap function ++ * to allocate a new chunk. ++ * It will also update the command stream's kernel input page with the address ++ * of a new chunk that was allocated. ++ * ++ * @kctx: Pointer to the kbase context in which the tiler heap was initialized. ++ * @stream: Pointer to the structure containing info provided by the firmware ++ * about the command stream interface. ++ * ++ * Return: 0 if successfully handled the request, otherwise a negative error ++ * code on failure. ++ */ ++static int handle_oom_event(struct kbase_context *const kctx, ++ struct kbase_csf_cmd_stream_info const *const stream) ++{ ++ u64 gpu_heap_va = ++ kbase_csf_firmware_cs_output(stream, CS_HEAP_ADDRESS_LO) | ++ ((u64)kbase_csf_firmware_cs_output(stream, CS_HEAP_ADDRESS_HI) << 32); ++ const u32 vt_start = ++ kbase_csf_firmware_cs_output(stream, CS_HEAP_VT_START); ++ const u32 vt_end = ++ kbase_csf_firmware_cs_output(stream, CS_HEAP_VT_END); ++ const u32 frag_end = ++ kbase_csf_firmware_cs_output(stream, CS_HEAP_FRAG_END); ++ u32 renderpasses_in_flight; ++ u64 new_chunk_ptr; ++ int err; ++ ++ if ((frag_end > vt_end) || (vt_end >= vt_start)) { ++ dev_warn(kctx->kbdev->dev, "Invalid Heap statistics provided by firmware: vt_start %d, vt_end %d, frag_end %d\n", ++ vt_start, vt_end, frag_end); ++ return -EINVAL; ++ } ++ ++ renderpasses_in_flight = vt_start - frag_end; ++ ++ err = kbase_csf_tiler_heap_alloc_new_chunk(kctx, ++ gpu_heap_va, renderpasses_in_flight, &new_chunk_ptr); ++ ++ /* It is okay to acknowledge with a NULL chunk (firmware will then wait ++ * for the fragment jobs to complete and release chunks) ++ */ ++ if (err == -EBUSY) ++ new_chunk_ptr = 0; ++ else if (err) ++ return err; ++ ++ kbase_csf_firmware_cs_input(stream, CS_TILER_HEAP_START_LO, ++ new_chunk_ptr & 0xFFFFFFFF); ++ kbase_csf_firmware_cs_input(stream, CS_TILER_HEAP_START_HI, ++ new_chunk_ptr >> 32); ++ ++ kbase_csf_firmware_cs_input(stream, CS_TILER_HEAP_END_LO, ++ new_chunk_ptr & 0xFFFFFFFF); ++ kbase_csf_firmware_cs_input(stream, CS_TILER_HEAP_END_HI, ++ new_chunk_ptr >> 32); ++ ++ return 0; ++} ++ ++/** ++ * report_tiler_oom_error - Report a CSG error due to a tiler heap OOM event ++ * ++ * @group: Pointer to the GPU command queue group that encountered the error ++ */ ++static void report_tiler_oom_error(struct kbase_queue_group *group) ++{ ++ struct base_csf_notification const ++ error = { .type = BASE_CSF_NOTIFICATION_GPU_QUEUE_GROUP_ERROR, ++ .payload = { ++ .csg_error = { ++ .handle = group->handle, ++ .error = { ++ .error_type = ++ BASE_GPU_QUEUE_GROUP_ERROR_TILER_HEAP_OOM, ++ } } } }; ++ struct kbase_context *kctx = group->kctx; ++ ++ lockdep_assert_held(&kctx->csf.lock); ++ ++ /* Ignore this error if the previous one hasn't been reported */ ++ if (!WARN_ON(!list_empty(&group->error_tiler_oom.link))) { ++ group->error_tiler_oom.data = error; ++ list_add_tail(&group->error_tiler_oom.link, ++ &kctx->csf.error_list); ++ kbase_event_wakeup(kctx); ++ } ++} ++ ++/** ++ * kbase_queue_oom_event - Handle tiler out-of-memory for a GPU command queue. ++ * ++ * @queue: Pointer to queue for which out-of-memory event was received. ++ * ++ * Called with the command-stream front-end locked for the affected GPU ++ * virtual address space. Do not call in interrupt context. ++ * ++ * Handles tiler out-of-memory for a GPU command queue and then clears the ++ * notification to allow the firmware to report out-of-memory again in future. ++ * If the out-of-memory condition was successfully handled then this function ++ * rings the relevant doorbell to notify the firmware; otherwise, it terminates ++ * the GPU command queue group to which the queue is bound. See ++ * term_queue_group() for details. ++ */ ++static void kbase_queue_oom_event(struct kbase_queue *const queue) ++{ ++ struct kbase_context *const kctx = queue->kctx; ++ struct kbase_device *const kbdev = kctx->kbdev; ++ struct kbase_queue_group *group; ++ int slot_num, err; ++ struct kbase_csf_cmd_stream_group_info const *ginfo; ++ struct kbase_csf_cmd_stream_info const *stream; ++ u32 cs_oom_ack, cs_oom_req; ++ ++ lockdep_assert_held(&kctx->csf.lock); ++ ++ group = get_bound_queue_group(queue); ++ if (!group) { ++ dev_warn(kctx->kbdev->dev, "queue not bound\n"); ++ return; ++ } ++ ++ kbase_csf_scheduler_lock(kbdev); ++ ++ slot_num = kbase_csf_scheduler_group_get_slot(group); ++ ++ /* The group could have gone off slot before this work item got ++ * a chance to execute. ++ */ ++ if (slot_num < 0) ++ goto unlock; ++ ++ /* If the bound group is on slot yet the kctx is marked with disabled ++ * on address-space fault, the group is pending to be killed. So skip ++ * the inflight oom operation. ++ */ ++ if (kbase_ctx_flag(kctx, KCTX_AS_DISABLED_ON_FAULT)) ++ goto unlock; ++ ++ ginfo = &kbdev->csf.global_iface.groups[slot_num]; ++ stream = &ginfo->streams[queue->csi_index]; ++ cs_oom_ack = kbase_csf_firmware_cs_output(stream, CS_ACK) & ++ CS_ACK_TILER_OOM_MASK; ++ cs_oom_req = kbase_csf_firmware_cs_input_read(stream, CS_REQ) & ++ CS_REQ_TILER_OOM_MASK; ++ ++ /* The group could have already undergone suspend-resume cycle before ++ * this work item got a chance to execute. On CSG resume the CS_ACK ++ * register is set by firmware to reflect the CS_REQ register, which ++ * implies that all events signaled before suspension are implicitly ++ * acknowledged. ++ * A new OoM event is expected to be generated after resume. ++ */ ++ if (cs_oom_ack == cs_oom_req) ++ goto unlock; ++ ++ err = handle_oom_event(kctx, stream); ++ ++ kbase_csf_firmware_cs_input_mask(stream, CS_REQ, cs_oom_ack, ++ CS_REQ_TILER_OOM_MASK); ++ ++ if (err) { ++ dev_warn( ++ kbdev->dev, ++ "Queue group to be terminated, couldn't handle the OoM event\n"); ++ kbase_csf_scheduler_unlock(kbdev); ++ term_queue_group(group); ++ report_tiler_oom_error(group); ++ return; ++ } ++ ++ kbase_csf_ring_cs_kernel_doorbell(kbdev, queue); ++unlock: ++ kbase_csf_scheduler_unlock(kbdev); ++} ++ ++/** ++ * oom_event_worker - Tiler out-of-memory handler called from a workqueue. ++ * ++ * @data: Pointer to a work_struct embedded in GPU command queue data. ++ * ++ * Handles a tiler out-of-memory condition for a GPU command queue and then ++ * releases a reference that was added to prevent the queue being destroyed ++ * while this work item was pending on a workqueue. ++ */ ++static void oom_event_worker(struct work_struct *data) ++{ ++ struct kbase_queue *queue = ++ container_of(data, struct kbase_queue, oom_event_work); ++ struct kbase_context *kctx = queue->kctx; ++ ++ mutex_lock(&kctx->csf.lock); ++ ++ kbase_queue_oom_event(queue); ++ release_queue(queue); ++ ++ mutex_unlock(&kctx->csf.lock); ++} ++ ++/** ++ * timer_event_worker - Timer event handler called from a workqueue. ++ * ++ * @data: Pointer to a work_struct embedded in GPU command queue group data. ++ * ++ * Notify the event notification thread of progress timeout fault ++ * for the GPU command queue group. ++ */ ++static void timer_event_worker(struct work_struct *data) ++{ ++ struct kbase_queue_group *const group = ++ container_of(data, struct kbase_queue_group, timer_event_work); ++ struct base_csf_notification const ++ error = { .type = BASE_CSF_NOTIFICATION_GPU_QUEUE_GROUP_ERROR, ++ .payload = { ++ .csg_error = { ++ .handle = group->handle, ++ .error = { ++ .error_type = ++ BASE_GPU_QUEUE_GROUP_ERROR_TIMEOUT, ++ } } } }; ++ struct kbase_context *const kctx = group->kctx; ++ ++ mutex_lock(&kctx->csf.lock); ++ ++ /* Ignore this error if the previous one hasn't been reported */ ++ if (!WARN_ON(!list_empty(&group->error_timeout.link))) { ++ group->error_timeout.data = error; ++ list_add_tail(&group->error_timeout.link, ++ &kctx->csf.error_list); ++ kbase_event_wakeup(kctx); ++ } ++ ++ mutex_unlock(&kctx->csf.lock); ++} ++ ++/** ++ * protm_event_worker - Protected mode switch request event handler ++ * called from a workqueue. ++ * ++ * @data: Pointer to a work_struct embedded in GPU command queue group data. ++ * ++ * Request to switch to protected mode. ++ */ ++static void protm_event_worker(struct work_struct *data) ++{ ++ struct kbase_queue_group *const group = ++ container_of(data, struct kbase_queue_group, protm_event_work); ++ ++ kbase_csf_scheduler_group_protm_enter(group); ++} ++ ++/** ++ * handle_fault_event - Handler for CS fault. ++ * ++ * @queue: Pointer to queue for which fault event was received. ++ * @stream: Pointer to the structure containing info provided by the ++ * firmware about the command stream interface. ++ * ++ * Prints meaningful CS fault information. ++ * ++ * Return: 0 on success, otherwise a negative system code. ++ */ ++static int handle_fault_event(struct kbase_queue const *const queue, ++ struct kbase_csf_cmd_stream_info const *const stream) ++{ ++ const u32 cs_fault = kbase_csf_firmware_cs_output(stream, CS_FAULT); ++ const u64 cs_fault_info = ++ kbase_csf_firmware_cs_output(stream, CS_FAULT_INFO_LO) | ++ ((u64)kbase_csf_firmware_cs_output(stream, CS_FAULT_INFO_HI) ++ << 32); ++ const u8 cs_fault_exception_type = ++ CS_FAULT_EXCEPTION_TYPE_GET(cs_fault); ++ const u32 cs_fault_exception_data = ++ CS_FAULT_EXCEPTION_DATA_GET(cs_fault); ++ const u64 cs_fault_info_exception_data = ++ CS_FAULT_INFO_EXCEPTION_DATA_GET(cs_fault_info); ++ struct kbase_device *const kbdev = queue->kctx->kbdev; ++ ++ dev_warn(kbdev->dev, "CSI: %d\n" ++ "CS_FAULT.EXCEPTION_TYPE: 0x%x (%s)\n" ++ "CS_FAULT.EXCEPTION_DATA: 0x%x\n" ++ "CS_FAULT_INFO.EXCEPTION_DATA: 0x%llx\n", ++ queue->csi_index, cs_fault_exception_type, ++ kbase_gpu_exception_name(cs_fault_exception_type), ++ cs_fault_exception_data, cs_fault_info_exception_data); ++ ++ return -EFAULT; ++} ++ ++/** ++ * report_queue_fatal_error - Report queue fatal error to user space ++ * ++ * @queue: Pointer to queue for which fatal event was received. ++ * @cs_fatal: Fault information ++ * @cs_fatal_info: Additional fault information ++ * ++ * If a queue has already been in fatal error status, ++ * subsequent fatal error on the queue should never take place. ++ */ ++static void report_queue_fatal_error(struct kbase_queue *const queue, ++ u32 cs_fatal, u64 cs_fatal_info) ++{ ++ struct base_csf_notification error = { ++ .type = BASE_CSF_NOTIFICATION_GPU_QUEUE_GROUP_ERROR, ++ .payload = { ++ .csg_error = { ++ .handle = queue->group->handle, ++ .error = { ++ .error_type = ++ BASE_GPU_QUEUE_GROUP_QUEUE_ERROR_FATAL, ++ .payload = { ++ .fatal_queue = { ++ .sideband = cs_fatal_info, ++ .status = cs_fatal, ++ .csi_index = queue->csi_index, ++ } ++ } ++ } ++ } ++ } ++ }; ++ ++ lockdep_assert_held(&queue->kctx->csf.lock); ++ ++ /* If a queue has already been in fatal error status, ++ * subsequent fatal error on the queue should never take place. ++ */ ++ if (!WARN_ON(!list_empty(&queue->error.link))) { ++ queue->error.data = error; ++ list_add_tail(&queue->error.link, &queue->kctx->csf.error_list); ++ kbase_event_wakeup(queue->kctx); ++ } ++} ++ ++/** ++ * handle_fatal_event - Handler for CS fatal. ++ * ++ * @queue: Pointer to queue for which fatal event was received. ++ * @stream: Pointer to the structure containing info provided by the ++ * firmware about the command stream interface. ++ * @fw_error: Return true if internal firmware fatal is handled ++ * ++ * Prints meaningful CS fatal information. ++ * Report queue fatal error to user space. ++ * ++ * Return: 0 on success otherwise a negative system error. ++ */ ++static int handle_fatal_event(struct kbase_queue *const queue, ++ struct kbase_csf_cmd_stream_info const *const stream, ++ bool *fw_error) ++{ ++ const u32 cs_fatal = kbase_csf_firmware_cs_output(stream, CS_FATAL); ++ const u64 cs_fatal_info = ++ kbase_csf_firmware_cs_output(stream, CS_FATAL_INFO_LO) | ++ ((u64)kbase_csf_firmware_cs_output(stream, CS_FATAL_INFO_HI) ++ << 32); ++ const u32 cs_fatal_exception_type = ++ CS_FATAL_EXCEPTION_TYPE_GET(cs_fatal); ++ const u32 cs_fatal_exception_data = ++ CS_FATAL_EXCEPTION_DATA_GET(cs_fatal); ++ const u64 cs_fatal_info_exception_data = ++ CS_FATAL_INFO_EXCEPTION_DATA_GET(cs_fatal_info); ++ struct kbase_device *const kbdev = queue->kctx->kbdev; ++ ++ lockdep_assert_held(&queue->kctx->csf.lock); ++ ++ dev_warn(kbdev->dev, ++ "CSG: %d, CSI: %d\n" ++ "CS_FATAL.EXCEPTION_TYPE: 0x%x (%s)\n" ++ "CS_FATAL.EXCEPTION_DATA: 0x%x\n" ++ "CS_FATAL_INFO.EXCEPTION_DATA: 0x%llx\n", ++ queue->group->handle, queue->csi_index, ++ cs_fatal_exception_type, ++ kbase_gpu_exception_name(cs_fatal_exception_type), ++ cs_fatal_exception_data, cs_fatal_info_exception_data); ++ ++ if (cs_fatal_exception_type == ++ CS_FATAL_EXCEPTION_TYPE_FIRMWARE_INTERNAL_ERROR) ++ *fw_error = true; ++ else ++ report_queue_fatal_error(queue, cs_fatal, cs_fatal_info); ++ ++ return -EFAULT; ++} ++ ++/** ++ * handle_internal_firmware_fatal - Handler for CS internal firmware fault. ++ * ++ * @kbdev: Pointer to kbase device ++ * ++ * Report group fatal error to user space for all GPU command queue groups ++ * in the device, terminate them and reset GPU. ++ */ ++static void handle_internal_firmware_fatal(struct kbase_device *const kbdev) ++{ ++ int as; ++ ++ for (as = 0; as < kbdev->nr_hw_address_spaces; as++) { ++ struct kbase_context *kctx; ++ struct kbase_fault fault = { ++ .status = GPU_EXCEPTION_TYPE_SW_FAULT_1, ++ }; ++ ++ if (as == MCU_AS_NR) ++ continue; ++ ++ kctx = kbase_ctx_sched_as_to_ctx_refcount(kbdev, as); ++ if (!kctx) ++ continue; ++ ++ kbase_csf_ctx_handle_fault(kctx, &fault); ++ kbase_ctx_sched_release_ctx_lock(kctx); ++ } ++ ++ if (kbase_prepare_to_reset_gpu(kbdev)) ++ kbase_reset_gpu(kbdev); ++} ++ ++/** ++ * fault_event_worker - Worker function for CS fault/fatal. ++ * ++ * @data: Pointer to a work_struct embedded in GPU command queue data. ++ * ++ * Handle the fault and fatal exception for a GPU command queue and then ++ * releases a reference that was added to prevent the queue being destroyed ++ * while this work item was pending on a workqueue. ++ * ++ * Report the fault and fatal exception for a GPU command queue and then ++ * clears the corresponding notification fields to allow the firmware to ++ * report other faults in future. ++ * ++ * It may also terminate the GPU command queue group(s) and reset GPU ++ * in case internal firmware CS fatal exception occurred. ++ */ ++static void fault_event_worker(struct work_struct *const data) ++{ ++ struct kbase_queue *const queue = ++ container_of(data, struct kbase_queue, fault_event_work); ++ ++ struct kbase_context *const kctx = queue->kctx; ++ struct kbase_device *const kbdev = kctx->kbdev; ++ struct kbase_queue_group *group; ++ int slot_num; ++ struct kbase_csf_cmd_stream_group_info const *ginfo; ++ struct kbase_csf_cmd_stream_info const *stream; ++ u32 cs_ack, cs_req; ++ int err = 0; ++ bool internal_fw_error = false; ++ ++ mutex_lock(&kctx->csf.lock); ++ kbase_csf_scheduler_lock(kbdev); ++ ++ group = get_bound_queue_group(queue); ++ if (!group) { ++ dev_warn(kbdev->dev, "queue not bound\n"); ++ goto unlock; ++ } ++ ++ slot_num = kbase_csf_scheduler_group_get_slot(group); ++ ++ /* The group could have gone off slot before this work item got ++ * a chance to execute. ++ */ ++ if (slot_num < 0) { ++ dev_warn(kbdev->dev, "invalid slot_num\n"); ++ goto unlock; ++ } ++ ++ /* If the bound group is on slot yet the kctx is marked with disabled ++ * on address-space fault, the group is pending to be killed. So skip ++ * the inflight queue exception event operation. ++ */ ++ if (kbase_ctx_flag(kctx, KCTX_AS_DISABLED_ON_FAULT)) { ++ dev_warn(kbdev->dev, "kctx is already disabled on fault\n"); ++ goto unlock; ++ } ++ ++ ginfo = &kbdev->csf.global_iface.groups[slot_num]; ++ stream = &ginfo->streams[queue->csi_index]; ++ cs_ack = kbase_csf_firmware_cs_output(stream, CS_ACK); ++ cs_req = kbase_csf_firmware_cs_input_read(stream, CS_REQ); ++ ++ if ((cs_ack & CS_ACK_FATAL_MASK) != (cs_req & CS_REQ_FATAL_MASK)) { ++ err = handle_fatal_event(queue, stream, &internal_fw_error); ++ kbase_csf_firmware_cs_input_mask(stream, CS_REQ, cs_ack, ++ CS_REQ_FATAL_MASK); ++ } ++ ++ if ((cs_ack & CS_ACK_FAULT_MASK) != (cs_req & CS_REQ_FAULT_MASK)) { ++ err |= handle_fault_event(queue, stream); ++ kbase_csf_firmware_cs_input_mask(stream, CS_REQ, cs_ack, ++ CS_REQ_FAULT_MASK); ++ kbase_csf_ring_cs_kernel_doorbell(kbdev, queue); ++ } ++ ++ if (err) { ++ /* From 10.x.5, CS_REQ_ERROR_MODE is removed but TI2 bitfile ++ * upload not finished. Need to remove on GPUCORE-23972 ++ */ ++ kbase_csf_firmware_cs_input_mask(stream, CS_REQ, ~cs_ack, ++ CS_REQ_ERROR_MODE_MASK); ++ dev_dbg(kbdev->dev, "Slot-%d CSI-%d entering error mode\n", ++ slot_num, queue->csi_index); ++ } ++ ++unlock: ++ release_queue(queue); ++ kbase_csf_scheduler_unlock(kbdev); ++ mutex_unlock(&kctx->csf.lock); ++ ++ if (internal_fw_error) ++ handle_internal_firmware_fatal(kbdev); ++} ++ ++/** ++ * process_cs_interrupts - Process interrupts for a command stream. ++ * ++ * @group: Pointer to GPU command queue group data. ++ * @ginfo: The command stream group interface provided by the firmware. ++ * @irqreq: CSG's IRQ request bitmask (one bit per stream). ++ * @irqack: CSG's IRQ acknowledge bitmask (one bit per stream). ++ * ++ * If the interrupt request bitmask differs from the acknowledge bitmask ++ * then the firmware is notifying the host of an event concerning those ++ * streams indicated by bits whose value differs. The actions required ++ * are then determined by examining which notification flags differ between ++ * the request and acknowledge registers for the individual stream(s). ++ */ ++static void process_cs_interrupts(struct kbase_queue_group *const group, ++ struct kbase_csf_cmd_stream_group_info const *const ginfo, ++ u32 const irqreq, u32 const irqack) ++{ ++ struct kbase_device *const kbdev = group->kctx->kbdev; ++ u32 remaining = irqreq ^ irqack; ++ bool protm_pend = false; ++ ++ kbase_csf_scheduler_spin_lock_assert_held(kbdev); ++ ++ while (remaining != 0) { ++ int const i = ffs(remaining) - 1; ++ struct kbase_queue *const queue = group->bound_queues[i]; ++ ++ /* The queue pointer can be NULL, but if it isn't NULL then it ++ * cannot disappear since scheduler spinlock is held and before ++ * freeing a bound queue it has to be first unbound which ++ * requires scheduler spinlock. ++ */ ++ if (queue && !WARN_ON(queue->csi_index != i)) { ++ struct kbase_csf_cmd_stream_info const *const stream = ++ &ginfo->streams[i]; ++ u32 const cs_req = kbase_csf_firmware_cs_input_read( ++ stream, CS_REQ); ++ u32 const cs_ack = ++ kbase_csf_firmware_cs_output(stream, CS_ACK); ++ struct workqueue_struct *wq = group->kctx->csf.wq; ++ ++ if ((cs_req & CS_REQ_EXCEPTION_MASK) ^ ++ (cs_ack & CS_ACK_EXCEPTION_MASK)) { ++ get_queue(queue); ++ KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, CSI_FAULT_INTERRUPT, group, queue, cs_req ^ cs_ack); ++ if (!queue_work(wq, &queue->fault_event_work)) ++ release_queue(queue); ++ } ++ ++ if (((cs_req & CS_REQ_TILER_OOM_MASK) ^ ++ (cs_ack & CS_ACK_TILER_OOM_MASK))) { ++ get_queue(queue); ++ KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, CSI_TILER_OOM_INTERRUPT, group, queue, cs_req ^ cs_ack); ++ if (WARN_ON(!queue_work( ++ wq, &queue->oom_event_work))) { ++ /* The work item shall not have been ++ * already queued, there can be only ++ * one pending OoM event for a ++ * queue. ++ */ ++ release_queue(queue); ++ } ++ } ++ ++ if ((cs_req & CS_REQ_PROTM_PEND_MASK) ^ ++ (cs_ack & CS_ACK_PROTM_PEND_MASK)) { ++ dev_dbg(kbdev->dev, ++ "Protected mode entry request for queue on csi %d bound to group-%d on slot %d", ++ queue->csi_index, group->handle, ++ group->csg_nr); ++ ++ bitmap_set(group->protm_pending_bitmap, i, 1); ++ protm_pend = true; ++ } ++ } ++ ++ remaining &= ~(1 << i); ++ } ++ ++ if (protm_pend) ++ queue_work(group->kctx->csf.wq, &group->protm_event_work); ++} ++ ++/** ++ * process_csg_interrupts - Process interrupts for a command stream group. ++ * ++ * @kbdev: Instance of a GPU platform device that implements a command stream ++ * front-end interface. ++ * @csg_nr: Command stream group number. ++ * ++ * Handles interrupts for a command stream group and for streams within it. ++ * ++ * If the CSG's request register value differs from its acknowledge register ++ * then the firmware is notifying the host of an event concerning the whole ++ * group. The actions required are then determined by examining which ++ * notification flags differ between those two register values. ++ * ++ * See process_cs_interrupts() for details of per-stream interrupt handling. ++ */ ++static void process_csg_interrupts(struct kbase_device *const kbdev, ++ int const csg_nr) ++{ ++ struct kbase_csf_cmd_stream_group_info *ginfo; ++ struct kbase_queue_group *group; ++ u32 req, ack, irqreq, irqack; ++ ++ kbase_csf_scheduler_spin_lock_assert_held(kbdev); ++ ++ if (WARN_ON(csg_nr >= kbdev->csf.global_iface.group_num)) ++ return; ++ ++ ginfo = &kbdev->csf.global_iface.groups[csg_nr]; ++ req = kbase_csf_firmware_csg_input_read(ginfo, CSG_REQ); ++ ack = kbase_csf_firmware_csg_output(ginfo, CSG_ACK); ++ irqreq = kbase_csf_firmware_csg_output(ginfo, CSG_IRQ_REQ); ++ irqack = kbase_csf_firmware_csg_input_read(ginfo, CSG_IRQ_ACK); ++ ++ /* There may not be any pending CSG/CS interrupts to process */ ++ if ((req == ack) && (irqreq == irqack)) ++ return; ++ ++ /* Immediately set IRQ_ACK bits to be same as the IRQ_REQ bits before ++ * examining the CS_ACK & CS_REQ bits. This would ensure that Host ++ * doesn't misses an interrupt for the CS in the race scenario where ++ * whilst Host is servicing an interrupt for the CS, firmware sends ++ * another interrupt for that CS. ++ */ ++ kbase_csf_firmware_csg_input(ginfo, CSG_IRQ_ACK, irqreq); ++ ++ group = kbase_csf_scheduler_get_group_on_slot(kbdev, csg_nr); ++ ++ /* The group pointer can be NULL here if interrupts for the group ++ * (like SYNC_UPDATE, IDLE notification) were delayed and arrived ++ * just after the suspension of group completed. However if not NULL ++ * then the group pointer cannot disappear even if User tries to ++ * terminate the group whilst this loop is running as scheduler ++ * spinlock is held and for freeing a group that is resident on a CSG ++ * slot scheduler spinlock is required. ++ */ ++ if (!group) ++ return; ++ ++ if (WARN_ON(kbase_csf_scheduler_group_get_slot_locked(group) != csg_nr)) ++ return; ++ ++ if ((req ^ ack) & CSG_REQ_SYNC_UPDATE) { ++ kbase_csf_firmware_csg_input_mask(ginfo, ++ CSG_REQ, ack, CSG_REQ_SYNC_UPDATE); ++ ++ KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_SYNC_UPDATE_INTERRUPT, group, req ^ ack); ++ kbase_csf_event_signal_cpu_only(group->kctx); ++ } ++ ++ /* IDLE and TILER_OOM can be safely ignored because they will be ++ * raised again if the group is assigned a CSG slot in future. ++ * TILER_OOM and PROGRESS_TIMER_EVENT may terminate the group. ++ */ ++ if (!kbase_csf_scheduler_group_events_enabled(kbdev, group)) ++ return; ++ ++ if ((req ^ ack) & CSG_REQ_IDLE_MASK) { ++ kbase_csf_firmware_csg_input_mask(ginfo, CSG_REQ, ack, ++ CSG_REQ_IDLE_MASK); ++ ++ set_bit(csg_nr, kbdev->csf.scheduler.csg_slots_idle_mask); ++ ++ KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_IDLE_INTERRUPT, group, req ^ ack); ++ dev_dbg(kbdev->dev, "Idle notification received for Group %u on slot %d\n", ++ group->handle, csg_nr); ++ } ++ ++ if ((req ^ ack) & CSG_REQ_PROGRESS_TIMER_EVENT_MASK) { ++ kbase_csf_firmware_csg_input_mask(ginfo, CSG_REQ, ack, ++ CSG_REQ_PROGRESS_TIMER_EVENT_MASK); ++ ++ dev_dbg(kbdev->dev, "Timeout notification received for Group %u on slot %d\n", ++ group->handle, csg_nr); ++ ++ queue_work(group->kctx->csf.wq, &group->timer_event_work); ++ } ++ ++ process_cs_interrupts(group, ginfo, irqreq, irqack); ++} ++ ++void kbase_csf_interrupt(struct kbase_device *kbdev, u32 val) ++{ ++ unsigned long flags; ++ u32 remaining = val; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), val); ++ ++ if (val & JOB_IRQ_GLOBAL_IF) { ++ const struct kbase_csf_global_iface *const global_iface = ++ &kbdev->csf.global_iface; ++ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; ++ ++ kbdev->csf.interrupt_received = true; ++ remaining &= ~JOB_IRQ_GLOBAL_IF; ++ ++ if (!kbdev->csf.firmware_reloaded) ++ kbase_csf_firmware_reload_completed(kbdev); ++ else if (kbdev->csf.glb_init_request_pending) ++ kbase_pm_update_state(kbdev); ++ ++ if (global_iface->output) { ++ u32 glb_req, glb_ack; ++ ++ kbase_csf_scheduler_spin_lock(kbdev, &flags); ++ glb_req = kbase_csf_firmware_global_input_read( ++ global_iface, GLB_REQ); ++ glb_ack = kbase_csf_firmware_global_output( ++ global_iface, GLB_ACK); ++ ++ if ((glb_req ^ glb_ack) & GLB_REQ_PROTM_EXIT_MASK) { ++ dev_dbg(kbdev->dev, "Protected mode exit interrupt received"); ++ kbase_csf_firmware_global_input_mask( ++ global_iface, GLB_REQ, glb_ack, ++ GLB_REQ_PROTM_EXIT_MASK); ++ WARN_ON(!kbase_csf_scheduler_protected_mode_in_use(kbdev)); ++ scheduler->active_protm_grp = NULL; ++ KBASE_KTRACE_ADD(kbdev, SCHEDULER_EXIT_PROTM, NULL, 0u); ++ } ++ kbase_csf_scheduler_spin_unlock(kbdev, flags); ++ } ++ ++ if (!remaining) { ++ wake_up_all(&kbdev->csf.event_wait); ++ return; ++ } ++ } ++ ++ kbase_csf_scheduler_spin_lock(kbdev, &flags); ++ while (remaining != 0) { ++ int const csg_nr = ffs(remaining) - 1; ++ ++ process_csg_interrupts(kbdev, csg_nr); ++ remaining &= ~(1 << csg_nr); ++ } ++ kbase_csf_scheduler_spin_unlock(kbdev, flags); ++ ++ wake_up_all(&kbdev->csf.event_wait); ++} ++ ++void kbase_csf_doorbell_mapping_term(struct kbase_device *kbdev) ++{ ++ if (kbdev->csf.db_filp) { ++ struct page *page = as_page(kbdev->csf.dummy_db_page); ++ ++ kbase_mem_pool_free( ++ &kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], ++ page, false); ++ ++ fput(kbdev->csf.db_filp); ++ } ++} ++ ++int kbase_csf_doorbell_mapping_init(struct kbase_device *kbdev) ++{ ++ struct tagged_addr phys; ++ struct file *filp; ++ int ret; ++ ++ filp = shmem_file_setup("mali csf", MAX_LFS_FILESIZE, VM_NORESERVE); ++ if (IS_ERR(filp)) ++ return PTR_ERR(filp); ++ ++ ret = kbase_mem_pool_alloc_pages( ++ &kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], ++ 1, &phys, false); ++ ++ if (ret <= 0) { ++ fput(filp); ++ return ret; ++ } ++ ++ kbdev->csf.db_filp = filp; ++ kbdev->csf.dummy_db_page = phys; ++ kbdev->csf.db_file_offsets = 0; ++ ++ return 0; ++} +diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf.h +new file mode 100755 +index 000000000000..c183d0a32302 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf.h +@@ -0,0 +1,444 @@ ++/* ++ * ++ * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_CSF_H_ ++#define _KBASE_CSF_H_ ++ ++#include "mali_kbase_csf_kcpu.h" ++#include "mali_kbase_csf_scheduler.h" ++#include "mali_kbase_csf_firmware.h" ++#include "mali_kbase_csf_protected_memory.h" ++ ++/* Indicate invalid command stream h/w interface ++ */ ++#define KBASEP_IF_NR_INVALID ((s8)-1) ++ ++/* Indicate invalid command stream group number for a GPU command queue group ++ */ ++#define KBASEP_CSG_NR_INVALID ((s8)-1) ++ ++/* Indicate invalid user doorbell number for a GPU command queue ++ */ ++#define KBASEP_USER_DB_NR_INVALID ((s8)-1) ++ ++/* Waiting timeout for global request completion acknowledgment */ ++#define GLB_REQ_WAIT_TIMEOUT_MS (300) /* 300 milliseconds */ ++ ++#define CSG_REQ_EP_CFG (0x1 << CSG_REQ_EP_CFG_SHIFT) ++#define CSG_REQ_SYNC_UPDATE (0x1 << CSG_REQ_SYNC_UPDATE_SHIFT) ++#define FIRMWARE_PING_INTERVAL_MS (2000) /* 2 seconds */ ++ ++/** ++ * enum kbase_csf_event_callback_action - return type for CSF event callbacks. ++ * ++ * @KBASE_CSF_EVENT_CALLBACK_FIRST: Never set explicitly. ++ * It doesn't correspond to any action or type of event callback. ++ * ++ * @KBASE_CSF_EVENT_CALLBACK_KEEP: The callback will remain registered. ++ * ++ * @KBASE_CSF_EVENT_CALLBACK_REMOVE: The callback will be removed ++ * immediately upon return. ++ * ++ * @KBASE_CSF_EVENT_CALLBACK_LAST: Never set explicitly. ++ * It doesn't correspond to any action or type of event callback. ++ */ ++enum kbase_csf_event_callback_action { ++ KBASE_CSF_EVENT_CALLBACK_FIRST = 0, ++ KBASE_CSF_EVENT_CALLBACK_KEEP, ++ KBASE_CSF_EVENT_CALLBACK_REMOVE, ++ KBASE_CSF_EVENT_CALLBACK_LAST, ++}; ++ ++/** ++ * kbase_csf_event_callback_action - type for callback functions to be ++ * called upon CSF events. ++ * ++ * This is the type of callback functions that can be registered ++ * for CSF events. These function calls shall be triggered by any call ++ * to kbase_csf_event_signal. ++ * ++ * @param: Generic parameter to pass to the callback function. ++ * ++ * Return: KBASE_CSF_EVENT_CALLBACK_KEEP if the callback should remain ++ * registered, or KBASE_CSF_EVENT_CALLBACK_REMOVE if it should be removed. ++ */ ++typedef enum kbase_csf_event_callback_action kbase_csf_event_callback(void *param); ++ ++/** ++ * kbase_csf_event_wait_add - Add a CSF event callback ++ * ++ * This function adds an event callback to the list of CSF event callbacks ++ * belonging to a given Kbase context, to be triggered when a CSF event is ++ * signalled by kbase_csf_event_signal. ++ * ++ * @kctx: The Kbase context the @callback should be registered to. ++ * @callback: The callback function to register. ++ * @param: Custom parameter to be passed to the @callback function. ++ * ++ * Return: 0 on success, or negative on failure. ++ */ ++int kbase_csf_event_wait_add(struct kbase_context *kctx, ++ kbase_csf_event_callback *callback, void *param); ++ ++/** ++ * kbase_csf_event_wait_remove - Remove a CSF event callback ++ * ++ * This function removes an event callback from the list of CSF event callbacks ++ * belonging to a given Kbase context. ++ * ++ * @kctx: The kbase context the @callback should be removed from. ++ * @callback: The callback function to remove. ++ * @param: Custom parameter that would have been passed to the @p callback ++ * function. ++ */ ++void kbase_csf_event_wait_remove(struct kbase_context *kctx, ++ kbase_csf_event_callback *callback, void *param); ++ ++/** ++ * kbase_csf_event_wait_remove_all - Removes all CSF event callbacks ++ * ++ * This function empties the list of CSF event callbacks belonging to a given ++ * Kbase context. ++ * ++ * @kctx: The kbase context for which CSF event callbacks have to be removed. ++ */ ++void kbase_csf_event_wait_remove_all(struct kbase_context *kctx); ++ ++/** ++ * kbase_csf_read_error - Read command stream fatal error ++ * ++ * This function takes the command stream fatal error from context's ordered ++ * error_list, copies its contents to @event_data. ++ * ++ * @kctx: The kbase context to read fatal error from ++ * @event_data: Caller-provided buffer to copy the fatal error to ++ * ++ * Return: true if fatal error is read successfully. ++ */ ++bool kbase_csf_read_error(struct kbase_context *kctx, ++ struct base_csf_notification *event_data); ++ ++/** ++ * kbase_csf_error_pending - Check whether fatal error is pending ++ * ++ * @kctx: The kbase context to check fatal error upon. ++ * ++ * Return: true if fatal error is pending. ++ */ ++bool kbase_csf_error_pending(struct kbase_context *kctx); ++ ++/** ++ * kbase_csf_event_signal - Signal a CSF event ++ * ++ * This function triggers all the CSF event callbacks that are registered to ++ * a given Kbase context, and also signals the thread of userspace driver ++ * (front-end), waiting for the CSF event. ++ * ++ * @kctx: The kbase context whose CSF event callbacks shall be triggered. ++ * @notify_gpu: Flag to indicate if CSF firmware should be notified of the ++ * signaling of event that happened on the Driver side, either ++ * the signal came from userspace or from kcpu queues. ++ */ ++void kbase_csf_event_signal(struct kbase_context *kctx, bool notify_gpu); ++ ++static inline void kbase_csf_event_signal_notify_gpu(struct kbase_context *kctx) ++{ ++ kbase_csf_event_signal(kctx, true); ++} ++ ++static inline void kbase_csf_event_signal_cpu_only(struct kbase_context *kctx) ++{ ++ kbase_csf_event_signal(kctx, false); ++} ++ ++/** ++ * kbase_csf_ctx_init - Initialize the command-stream front-end for a GPU ++ * address space. ++ * ++ * @kctx: Pointer to the kbase context which is being initialized. ++ * ++ * Return: 0 if successful or a negative error code on failure. ++ */ ++int kbase_csf_ctx_init(struct kbase_context *kctx); ++ ++/** ++ * kbase_csf_ctx_handle_fault - Terminate queue groups & notify fault upon ++ * GPU bus fault, MMU page fault or similar. ++ * ++ * This function terminates all GPU command queue groups in the context and ++ * notifies the event notification thread of the fault. ++ * ++ * @kctx: Pointer to faulty kbase context. ++ * @fault: Pointer to the fault. ++ */ ++void kbase_csf_ctx_handle_fault(struct kbase_context *kctx, ++ struct kbase_fault *fault); ++ ++/** ++ * kbase_csf_ctx_term - Terminate the command-stream front-end for a GPU ++ * address space. ++ * ++ * This function terminates any remaining CSGs and CSs which weren't destroyed ++ * before context termination. ++ * ++ * @kctx: Pointer to the kbase context which is being terminated. ++ */ ++void kbase_csf_ctx_term(struct kbase_context *kctx); ++ ++/** ++ * kbase_csf_queue_register - Register a GPU command queue. ++ * ++ * @kctx: Pointer to the kbase context within which the ++ * queue is to be registered. ++ * @reg: Pointer to the structure which contains details of the ++ * queue to be registered within the provided ++ * context. ++ * ++ * Return: 0 on success, or negative on failure. ++ */ ++int kbase_csf_queue_register(struct kbase_context *kctx, ++ struct kbase_ioctl_cs_queue_register *reg); ++ ++/** ++ * kbase_csf_queue_terminate - Terminate a GPU command queue. ++ * ++ * @kctx: Pointer to the kbase context within which the ++ * queue is to be terminated. ++ * @term: Pointer to the structure which identifies which ++ * queue is to be terminated. ++ */ ++void kbase_csf_queue_terminate(struct kbase_context *kctx, ++ struct kbase_ioctl_cs_queue_terminate *term); ++ ++/** ++ * kbase_csf_alloc_command_stream_user_pages - Allocate resources for a ++ * GPU command queue. ++ * ++ * This function allocates a pair of User mode input/output pages for a ++ * GPU command queue and maps them in the shared interface segment of MCU ++ * firmware address space. Also reserves a hardware doorbell page for the queue. ++ * ++ * @kctx: Pointer to the kbase context within which the resources ++ * for the queue are being allocated. ++ * @queue: Pointer to the queue for which to allocate resources. ++ * ++ * Return: 0 on success, or negative on failure. ++ */ ++int kbase_csf_alloc_command_stream_user_pages(struct kbase_context *kctx, ++ struct kbase_queue *queue); ++ ++/** ++ * kbase_csf_queue_bind - Bind a GPU command queue to a queue group. ++ * ++ * @kctx: The kbase context. ++ * @bind: Pointer to the union which specifies a queue group and a ++ * queue to be bound to that group. ++ * ++ * Return: 0 on success, or negative on failure. ++ */ ++int kbase_csf_queue_bind(struct kbase_context *kctx, ++ union kbase_ioctl_cs_queue_bind *bind); ++ ++/** ++ * kbase_csf_queue_unbind - Unbind a GPU command queue from a queue group ++ * to which it has been bound and free ++ * resources allocated for this queue if there ++ * are any. ++ * ++ * @queue: Pointer to queue to be unbound. ++ */ ++void kbase_csf_queue_unbind(struct kbase_queue *queue); ++ ++/** ++ * kbase_csf_queue_kick - Schedule a GPU command queue on the firmware ++ * ++ * @kctx: The kbase context. ++ * @kick: Pointer to the struct which specifies the queue ++ * that needs to be scheduled. ++ * ++ * Return: 0 on success, or negative on failure. ++ */ ++int kbase_csf_queue_kick(struct kbase_context *kctx, ++ struct kbase_ioctl_cs_queue_kick *kick); ++ ++/** Find if given the queue group handle is valid. ++ * ++ * This function is used to determine if the queue group handle is valid. ++ * ++ * @kctx: The kbase context under which the queue group exists. ++ * @group_handle: Handle for the group which uniquely identifies it within ++ * the context with which it was created. ++ * ++ * Return: 0 on success, or negative on failure. ++ */ ++int kbase_csf_queue_group_handle_is_valid(struct kbase_context *kctx, ++ u8 group_handle); ++ ++/** ++ * kbase_csf_queue_group_create - Create a GPU command queue group. ++ * ++ * @kctx: Pointer to the kbase context within which the ++ * queue group is to be created. ++ * @create: Pointer to the structure which contains details of the ++ * queue group which is to be created within the ++ * provided kbase context. ++ * ++ * Return: 0 on success, or negative on failure. ++ */ ++int kbase_csf_queue_group_create(struct kbase_context *kctx, ++ union kbase_ioctl_cs_queue_group_create *create); ++ ++/** ++ * kbase_csf_queue_group_terminate - Terminate a GPU command queue group. ++ * ++ * @kctx: Pointer to the kbase context within which the ++ * queue group is to be terminated. ++ * @group_handle: Pointer to the structure which identifies the queue ++ * group which is to be terminated. ++ */ ++void kbase_csf_queue_group_terminate(struct kbase_context *kctx, ++ u8 group_handle); ++ ++/** ++ * kbase_csf_term_descheduled_queue_group - Terminate a GPU command queue ++ * group that is not operational ++ * inside the scheduler. ++ * ++ * @group: Pointer to the structure which identifies the queue ++ * group to be terminated. The function assumes that the caller ++ * is sure that the given group is not operational inside the ++ * scheduler. If in doubt, use its alternative: ++ * @ref kbase_csf_queue_group_terminate(). ++ */ ++void kbase_csf_term_descheduled_queue_group(struct kbase_queue_group *group); ++ ++/** ++ * kbase_csf_queue_group_suspend - Suspend a GPU command queue group ++ * ++ * This function is used to suspend a queue group and copy the suspend buffer. ++ * ++ * @kctx: The kbase context for which the queue group is to be ++ * suspended. ++ * @sus_buf: Pointer to the structure which contains details of the ++ * user buffer and its kernel pinned pages. ++ * @size: The size in bytes for the user provided buffer. ++ * @group_handle: Handle for the group which uniquely identifies it within ++ * the context within which it was created. ++ * ++ * Return: 0 on success or negative value if failed to suspend ++ * queue group and copy suspend buffer contents. ++ */ ++int kbase_csf_queue_group_suspend(struct kbase_context *kctx, ++ struct kbase_suspend_copy_buffer *sus_buf, u8 group_handle); ++ ++/** ++ * kbase_csf_interrupt - Handle interrupts issued by CSF firmware. ++ * ++ * @kbdev: The kbase device to handle an IRQ for ++ * @val: The value of JOB IRQ status register which triggered the interrupt ++ */ ++void kbase_csf_interrupt(struct kbase_device *kbdev, u32 val); ++ ++/** ++ * kbase_csf_doorbell_mapping_init - Initialize the bitmap of Hw doorbell pages ++ * used to track their availability. ++ * ++ * @kbdev: Instance of a GPU platform device that implements a command ++ * stream front-end interface. ++ */ ++int kbase_csf_doorbell_mapping_init(struct kbase_device *kbdev); ++ ++void kbase_csf_doorbell_mapping_term(struct kbase_device *kbdev); ++ ++/** ++ * kbase_csf_ring_csg_doorbell - ring the doorbell for a command stream group ++ * interface. ++ * ++ * The function kicks a notification on the command stream group interface to ++ * firmware. ++ * ++ * @kbdev: Instance of a GPU platform device that implements a command ++ * stream front-end interface. ++ * @slot: Index of command stream group interface for ringing the door-bell. ++ */ ++void kbase_csf_ring_csg_doorbell(struct kbase_device *kbdev, int slot); ++ ++/** ++ * kbase_csf_ring_csg_slots_doorbell - ring the doorbell for a set of command ++ * stream group interfaces. ++ * ++ * The function kicks a notification on a set of command stream group ++ * interfaces to firmware. ++ * ++ * @kbdev: Instance of a GPU platform device that implements a command ++ * stream front-end interface. ++ * @slot_bitmap: bitmap for the given slots, slot-0 on bit-0, etc. ++ */ ++void kbase_csf_ring_csg_slots_doorbell(struct kbase_device *kbdev, ++ u32 slot_bitmap); ++ ++/** ++ * kbase_csf_ring_cs_kernel_doorbell - ring the kernel doorbell for a queue ++ * ++ * The function kicks a notification to the firmware for the command stream ++ * interface to which the queue is bound. ++ * ++ * @kbdev: Instance of a GPU platform device that implements a command ++ * stream front-end interface. ++ * @queue: Pointer to the queue for ringing the door-bell. ++ */ ++void kbase_csf_ring_cs_kernel_doorbell(struct kbase_device *kbdev, ++ struct kbase_queue *queue); ++ ++/** ++ * kbase_csf_ring_cs_user_doorbell - ring the user doorbell allocated for a ++ * queue. ++ * ++ * The function kicks a notification to the firmware on the doorbell assigned ++ * to the queue. ++ * ++ * @kbdev: Instance of a GPU platform device that implements a command ++ * stream front-end interface. ++ * @queue: Pointer to the queue for ringing the door-bell. ++ */ ++void kbase_csf_ring_cs_user_doorbell(struct kbase_device *kbdev, ++ struct kbase_queue *queue); ++ ++/** ++ * kbase_csf_active_queue_groups_reset - Reset the state of all active GPU ++ * command queue groups associated with the context. ++ * ++ * @kbdev: Instance of a GPU platform device that implements a command ++ * stream front-end interface. ++ * @kctx: The kbase context. ++ * ++ * This function will iterate through all the active/scheduled GPU command ++ * queue groups associated with the context, deschedule and mark them as ++ * terminated (which will then lead to unbinding of all the queues bound to ++ * them) and also no more work would be allowed to execute for them. ++ * ++ * This is similar to the action taken in response to an unexpected OoM event. ++ */ ++void kbase_csf_active_queue_groups_reset(struct kbase_device *kbdev, ++ struct kbase_context *kctx); ++ ++#endif /* _KBASE_CSF_H_ */ +diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_csg_debugfs.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_csg_debugfs.c +new file mode 100755 +index 000000000000..fd8329ba9422 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_csg_debugfs.c +@@ -0,0 +1,460 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include "mali_kbase_csf_csg_debugfs.h" ++#include ++#include ++#include ++#include ++ ++#ifdef CONFIG_DEBUG_FS ++#include "mali_kbase_csf_tl_reader.h" ++ ++static void kbasep_csf_scheduler_dump_active_queue_cs_status_wait( ++ struct seq_file *file, ++ u32 wait_status, ++ u32 wait_sync_value, ++ u64 wait_sync_live_value, ++ u64 wait_sync_pointer) ++{ ++#define WAITING "Waiting" ++#define NOT_WAITING "Not waiting" ++ ++ seq_printf(file, "SB_MASK: %d\n", ++ CS_STATUS_WAIT_SB_MASK_GET(wait_status)); ++ seq_printf(file, "PROGRESS_WAIT: %s\n", ++ CS_STATUS_WAIT_PROGRESS_WAIT_GET(wait_status) ? ++ WAITING : NOT_WAITING); ++ seq_printf(file, "PROTM_PEND: %s\n", ++ CS_STATUS_WAIT_PROTM_PEND_GET(wait_status) ? ++ WAITING : NOT_WAITING); ++ seq_printf(file, "SYNC_WAIT: %s\n", ++ CS_STATUS_WAIT_SYNC_WAIT_GET(wait_status) ? ++ WAITING : NOT_WAITING); ++ seq_printf(file, "WAIT_CONDITION: %s\n", ++ CS_STATUS_WAIT_SYNC_WAIT_CONDITION_GET(wait_status) ? ++ "greater than" : "less or equal"); ++ seq_printf(file, "SYNC_POINTER: 0x%llx\n", wait_sync_pointer); ++ seq_printf(file, "SYNC_VALUE: %d\n", wait_sync_value); ++ seq_printf(file, "SYNC_LIVE_VALUE: 0x%016llx\n", wait_sync_live_value); ++} ++ ++/** ++ * kbasep_csf_scheduler_dump_active_queue() - Print GPU command queue ++ * debug information ++ * ++ * @file: seq_file for printing to ++ * @queue: Address of a GPU command queue to examine ++ */ ++static void kbasep_csf_scheduler_dump_active_queue(struct seq_file *file, ++ struct kbase_queue *queue) ++{ ++ u32 *addr; ++ u64 cs_extract; ++ u64 cs_insert; ++ u32 cs_active; ++ u64 wait_sync_pointer; ++ u32 wait_status, wait_sync_value; ++ struct kbase_vmap_struct *mapping; ++ u64 *evt; ++ u64 wait_sync_live_value; ++ ++ if (!queue) ++ return; ++ ++ if (WARN_ON(queue->csi_index == KBASEP_IF_NR_INVALID || ++ !queue->group)) ++ return; ++ ++ /* Ring the doorbell to have firmware update CS_EXTRACT */ ++ kbase_csf_ring_cs_user_doorbell(queue->kctx->kbdev, queue); ++ msleep(100); ++ ++ addr = (u32 *)queue->user_io_addr; ++ cs_insert = addr[CS_INSERT_LO/4] | ((u64)addr[CS_INSERT_HI/4] << 32); ++ ++ addr = (u32 *)(queue->user_io_addr + PAGE_SIZE); ++ cs_extract = addr[CS_EXTRACT_LO/4] | ((u64)addr[CS_EXTRACT_HI/4] << 32); ++ cs_active = addr[CS_ACTIVE/4]; ++ ++#define KBASEP_CSF_DEBUGFS_CS_HEADER_USER_IO \ ++ "Bind Idx, Ringbuf addr, Prio, Insert offset, Extract offset, Active, Doorbell\n" ++ ++ seq_printf(file, KBASEP_CSF_DEBUGFS_CS_HEADER_USER_IO "%8d, %16llx, %4u, %16llx, %16llx, %6u, %8d\n", ++ queue->csi_index, queue->base_addr, queue->priority, ++ cs_insert, cs_extract, cs_active, queue->doorbell_nr); ++ ++ /* Print status information for blocked group waiting for sync object */ ++ if (kbase_csf_scheduler_group_get_slot(queue->group) < 0) { ++ if (CS_STATUS_WAIT_SYNC_WAIT_GET(queue->status_wait)) { ++ wait_status = queue->status_wait; ++ wait_sync_value = queue->sync_value; ++ wait_sync_pointer = queue->sync_ptr; ++ ++ evt = (u64 *)kbase_phy_alloc_mapping_get(queue->kctx, wait_sync_pointer, &mapping); ++ if (evt) { ++ wait_sync_live_value = evt[0]; ++ kbase_phy_alloc_mapping_put(queue->kctx, mapping); ++ } else { ++ wait_sync_live_value = U64_MAX; ++ } ++ ++ kbasep_csf_scheduler_dump_active_queue_cs_status_wait( ++ file, wait_status, wait_sync_value, ++ wait_sync_live_value, wait_sync_pointer); ++ } ++ } else { ++ struct kbase_device const *const kbdev = ++ queue->group->kctx->kbdev; ++ struct kbase_csf_cmd_stream_group_info const *const ginfo = ++ &kbdev->csf.global_iface.groups[queue->group->csg_nr]; ++ struct kbase_csf_cmd_stream_info const *const stream = ++ &ginfo->streams[queue->csi_index]; ++ u64 cmd_ptr; ++ u32 req_res; ++ ++ if (WARN_ON(!stream)) ++ return; ++ ++ cmd_ptr = kbase_csf_firmware_cs_output(stream, ++ CS_STATUS_CMD_PTR_LO); ++ cmd_ptr |= (u64)kbase_csf_firmware_cs_output(stream, ++ CS_STATUS_CMD_PTR_HI) << 32; ++ req_res = kbase_csf_firmware_cs_output(stream, ++ CS_STATUS_REQ_RESOURCE); ++ ++ seq_printf(file, "CMD_PTR: 0x%llx\n", cmd_ptr); ++ seq_printf(file, "REQ_RESOURCE [COMPUTE]: %d\n", ++ CS_STATUS_REQ_RESOURCE_COMPUTE_RESOURCES_GET(req_res)); ++ seq_printf(file, "REQ_RESOURCE [FRAGMENT]: %d\n", ++ CS_STATUS_REQ_RESOURCE_FRAGMENT_RESOURCES_GET(req_res)); ++ seq_printf(file, "REQ_RESOURCE [TILER]: %d\n", ++ CS_STATUS_REQ_RESOURCE_TILER_RESOURCES_GET(req_res)); ++ seq_printf(file, "REQ_RESOURCE [IDVS]: %d\n", ++ CS_STATUS_REQ_RESOURCE_IDVS_RESOURCES_GET(req_res)); ++ ++ wait_status = kbase_csf_firmware_cs_output(stream, ++ CS_STATUS_WAIT); ++ wait_sync_value = kbase_csf_firmware_cs_output(stream, ++ CS_STATUS_WAIT_SYNC_VALUE); ++ wait_sync_pointer = kbase_csf_firmware_cs_output(stream, ++ CS_STATUS_WAIT_SYNC_POINTER_LO); ++ wait_sync_pointer |= (u64)kbase_csf_firmware_cs_output(stream, ++ CS_STATUS_WAIT_SYNC_POINTER_HI) << 32; ++ ++ evt = (u64 *)kbase_phy_alloc_mapping_get(queue->kctx, wait_sync_pointer, &mapping); ++ if (evt) { ++ wait_sync_live_value = evt[0]; ++ kbase_phy_alloc_mapping_put(queue->kctx, mapping); ++ } else { ++ wait_sync_live_value = U64_MAX; ++ } ++ ++ kbasep_csf_scheduler_dump_active_queue_cs_status_wait( ++ file, wait_status, wait_sync_value, ++ wait_sync_live_value, wait_sync_pointer); ++ } ++ ++ seq_puts(file, "\n"); ++} ++ ++/* Waiting timeout for STATUS_UPDATE acknowledgment, in milliseconds */ ++#define CSF_STATUS_UPDATE_TO_MS (100) ++ ++static void kbasep_csf_scheduler_dump_active_group(struct seq_file *file, ++ struct kbase_queue_group *const group) ++{ ++ if (kbase_csf_scheduler_group_get_slot(group) >= 0) { ++ struct kbase_device *const kbdev = group->kctx->kbdev; ++ unsigned long flags; ++ u32 ep_c, ep_r; ++ char exclusive; ++ struct kbase_csf_cmd_stream_group_info const *const ginfo = ++ &kbdev->csf.global_iface.groups[group->csg_nr]; ++ long remaining = ++ kbase_csf_timeout_in_jiffies(CSF_STATUS_UPDATE_TO_MS); ++ u8 slot_priority = ++ kbdev->csf.scheduler.csg_slots[group->csg_nr].priority; ++ ++ kbase_csf_scheduler_spin_lock(kbdev, &flags); ++ kbase_csf_firmware_csg_input_mask(ginfo, CSG_REQ, ++ ~kbase_csf_firmware_csg_output(ginfo, CSG_ACK), ++ CSG_REQ_STATUS_UPDATE_MASK); ++ kbase_csf_scheduler_spin_unlock(kbdev, flags); ++ kbase_csf_ring_csg_doorbell(kbdev, group->csg_nr); ++ ++ remaining = wait_event_timeout(kbdev->csf.event_wait, ++ !((kbase_csf_firmware_csg_input_read(ginfo, CSG_REQ) ^ ++ kbase_csf_firmware_csg_output(ginfo, CSG_ACK)) & ++ CSG_REQ_STATUS_UPDATE_MASK), remaining); ++ ++ ep_c = kbase_csf_firmware_csg_output(ginfo, ++ CSG_STATUS_EP_CURRENT); ++ ep_r = kbase_csf_firmware_csg_output(ginfo, CSG_STATUS_EP_REQ); ++ ++ if (CSG_STATUS_EP_REQ_EXCLUSIVE_COMPUTE_GET(ep_r)) ++ exclusive = 'C'; ++ else if (CSG_STATUS_EP_REQ_EXCLUSIVE_FRAGMENT_GET(ep_r)) ++ exclusive = 'F'; ++ else ++ exclusive = '0'; ++ ++ if (!remaining) { ++ dev_err(kbdev->dev, ++ "Timed out for STATUS_UPDATE on group %d on slot %d", ++ group->handle, group->csg_nr); ++ ++ seq_printf(file, "*** Warn: Timed out for STATUS_UPDATE on slot %d\n", ++ group->csg_nr); ++ seq_printf(file, "*** The following group-record is likely stale\n"); ++ } ++ ++ seq_puts(file, "GroupID, CSG NR, CSG Prio, Run State, Priority, C_EP(Alloc/Req), F_EP(Alloc/Req), T_EP(Alloc/Req), Exclusive\n"); ++ seq_printf(file, "%7d, %6d, %8d, %9d, %8d, %11d/%3d, %11d/%3d, %11d/%3d, %9c\n", ++ group->handle, ++ group->csg_nr, ++ slot_priority, ++ group->run_state, ++ group->priority, ++ CSG_STATUS_EP_CURRENT_COMPUTE_EP_GET(ep_c), ++ CSG_STATUS_EP_REQ_COMPUTE_EP_GET(ep_r), ++ CSG_STATUS_EP_CURRENT_FRAGMENT_EP_GET(ep_c), ++ CSG_STATUS_EP_REQ_FRAGMENT_EP_GET(ep_r), ++ CSG_STATUS_EP_CURRENT_TILER_EP_GET(ep_c), ++ CSG_STATUS_EP_REQ_TILER_EP_GET(ep_r), ++ exclusive); ++ } else { ++ seq_puts(file, "GroupID, CSG NR, Run State, Priority\n"); ++ seq_printf(file, "%7d, %6d, %9d, %8d\n", ++ group->handle, ++ group->csg_nr, ++ group->run_state, ++ group->priority); ++ } ++ ++ if (group->run_state != KBASE_CSF_GROUP_TERMINATED) { ++ unsigned int i; ++ ++ seq_puts(file, "Bound queues:\n"); ++ ++ for (i = 0; i < MAX_SUPPORTED_STREAMS_PER_GROUP; i++) { ++ kbasep_csf_scheduler_dump_active_queue(file, ++ group->bound_queues[i]); ++ } ++ } ++ ++ seq_puts(file, "\n"); ++} ++ ++/** ++ * kbasep_csf_queue_group_debugfs_show() - Print per-context GPU command queue ++ * group debug information ++ * ++ * @file: The seq_file for printing to ++ * @data: The debugfs dentry private data, a pointer to kbase context ++ * ++ * Return: Negative error code or 0 on success. ++ */ ++static int kbasep_csf_queue_group_debugfs_show(struct seq_file *file, ++ void *data) ++{ ++ u32 gr; ++ struct kbase_context *const kctx = file->private; ++ struct kbase_device *const kbdev = kctx->kbdev; ++ ++ if (WARN_ON(!kctx)) ++ return -EINVAL; ++ ++ seq_printf(file, "MALI_CSF_CSG_DEBUGFS_VERSION: v%u\n", ++ MALI_CSF_CSG_DEBUGFS_VERSION); ++ ++ mutex_lock(&kctx->csf.lock); ++ kbase_csf_scheduler_lock(kbdev); ++ for (gr = 0; gr < MAX_QUEUE_GROUP_NUM; gr++) { ++ struct kbase_queue_group *const group = ++ kctx->csf.queue_groups[gr]; ++ ++ if (group) ++ kbasep_csf_scheduler_dump_active_group(file, group); ++ } ++ kbase_csf_scheduler_unlock(kbdev); ++ mutex_unlock(&kctx->csf.lock); ++ ++ return 0; ++} ++ ++/** ++ * kbasep_csf_scheduler_dump_active_groups() - Print debug info for active ++ * GPU command queue groups ++ * ++ * @file: The seq_file for printing to ++ * @data: The debugfs dentry private data, a pointer to kbase_device ++ * ++ * Return: Negative error code or 0 on success. ++ */ ++static int kbasep_csf_scheduler_dump_active_groups(struct seq_file *file, ++ void *data) ++{ ++ u32 csg_nr; ++ struct kbase_device *kbdev = file->private; ++ u32 num_groups = kbdev->csf.global_iface.group_num; ++ ++ seq_printf(file, "MALI_CSF_CSG_DEBUGFS_VERSION: v%u\n", ++ MALI_CSF_CSG_DEBUGFS_VERSION); ++ ++ kbase_csf_scheduler_lock(kbdev); ++ for (csg_nr = 0; csg_nr < num_groups; csg_nr++) { ++ struct kbase_queue_group *const group = ++ kbdev->csf.scheduler.csg_slots[csg_nr].resident_group; ++ ++ if (!group) ++ continue; ++ ++ seq_printf(file, "\nCtx %d_%d\n", group->kctx->tgid, ++ group->kctx->id); ++ ++ kbasep_csf_scheduler_dump_active_group(file, group); ++ } ++ kbase_csf_scheduler_unlock(kbdev); ++ ++ return 0; ++} ++ ++static int kbasep_csf_queue_group_debugfs_open(struct inode *in, ++ struct file *file) ++{ ++ return single_open(file, kbasep_csf_queue_group_debugfs_show, ++ in->i_private); ++} ++ ++static int kbasep_csf_active_queue_groups_debugfs_open(struct inode *in, ++ struct file *file) ++{ ++ return single_open(file, kbasep_csf_scheduler_dump_active_groups, ++ in->i_private); ++} ++ ++static const struct file_operations kbasep_csf_queue_group_debugfs_fops = { ++ .open = kbasep_csf_queue_group_debugfs_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = single_release, ++}; ++ ++void kbase_csf_queue_group_debugfs_init(struct kbase_context *kctx) ++{ ++ struct dentry *file; ++#if (KERNEL_VERSION(4, 7, 0) <= LINUX_VERSION_CODE) ++ const mode_t mode = 0444; ++#else ++ const mode_t mode = 0400; ++#endif ++ ++ if (WARN_ON(!kctx || IS_ERR_OR_NULL(kctx->kctx_dentry))) ++ return; ++ ++ file = debugfs_create_file("groups", mode, ++ kctx->kctx_dentry, kctx, &kbasep_csf_queue_group_debugfs_fops); ++ ++ if (IS_ERR_OR_NULL(file)) { ++ dev_warn(kctx->kbdev->dev, ++ "Unable to create per context queue groups debugfs entry"); ++ } ++} ++ ++static const struct file_operations ++ kbasep_csf_active_queue_groups_debugfs_fops = { ++ .open = kbasep_csf_active_queue_groups_debugfs_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = single_release, ++}; ++ ++static int kbasep_csf_debugfs_scheduling_timer_enabled_get( ++ void *data, u64 *val) ++{ ++ struct kbase_device *const kbdev = data; ++ ++ *val = kbase_csf_scheduler_timer_is_enabled(kbdev); ++ ++ return 0; ++} ++ ++static int kbasep_csf_debugfs_scheduling_timer_enabled_set( ++ void *data, u64 val) ++{ ++ struct kbase_device *const kbdev = data; ++ ++ kbase_csf_scheduler_timer_set_enabled(kbdev, val != 0); ++ ++ return 0; ++} ++ ++static int kbasep_csf_debugfs_scheduling_timer_kick_set( ++ void *data, u64 val) ++{ ++ struct kbase_device *const kbdev = data; ++ ++ kbase_csf_scheduler_kick(kbdev); ++ ++ return 0; ++} ++ ++DEFINE_SIMPLE_ATTRIBUTE(kbasep_csf_debugfs_scheduling_timer_enabled_fops, ++ &kbasep_csf_debugfs_scheduling_timer_enabled_get, ++ &kbasep_csf_debugfs_scheduling_timer_enabled_set, ++ "%llu\n"); ++DEFINE_SIMPLE_ATTRIBUTE(kbasep_csf_debugfs_scheduling_timer_kick_fops, ++ NULL, ++ &kbasep_csf_debugfs_scheduling_timer_kick_set, ++ "%llu\n"); ++ ++void kbase_csf_debugfs_init(struct kbase_device *kbdev) ++{ ++ debugfs_create_file("active_groups", 0444, ++ kbdev->mali_debugfs_directory, kbdev, ++ &kbasep_csf_active_queue_groups_debugfs_fops); ++ ++ debugfs_create_file("scheduling_timer_enabled", 0644, ++ kbdev->mali_debugfs_directory, kbdev, ++ &kbasep_csf_debugfs_scheduling_timer_enabled_fops); ++ debugfs_create_file("scheduling_timer_kick", 0200, ++ kbdev->mali_debugfs_directory, kbdev, ++ &kbasep_csf_debugfs_scheduling_timer_kick_fops); ++ ++ kbase_csf_tl_reader_debugfs_init(kbdev); ++ kbase_csf_firmware_trace_buffer_debugfs_init(kbdev); ++} ++ ++#else ++/* ++ * Stub functions for when debugfs is disabled ++ */ ++void kbase_csf_queue_group_debugfs_init(struct kbase_context *kctx) ++{ ++} ++ ++void kbase_csf_debugfs_init(struct kbase_device *kbdev) ++{ ++} ++ ++#endif /* CONFIG_DEBUG_FS */ +diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_csg_debugfs.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_csg_debugfs.h +new file mode 100755 +index 000000000000..c2e99d386f8c +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_csg_debugfs.h +@@ -0,0 +1,48 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_CSF_CSG_DEBUGFS_H_ ++#define _KBASE_CSF_CSG_DEBUGFS_H_ ++ ++/* Forward declarations */ ++struct kbase_device; ++struct kbase_context; ++struct kbase_queue_group; ++ ++#define MALI_CSF_CSG_DEBUGFS_VERSION 0 ++ ++/** ++ * kbase_csf_queue_group_debugfs_init() - Add debugfs entry for queue groups ++ * associated with @kctx. ++ * ++ * @kctx: Pointer to kbase_context ++ */ ++void kbase_csf_queue_group_debugfs_init(struct kbase_context *kctx); ++ ++/** ++ * kbase_csf_debugfs_init() - Add a global debugfs entry for queue groups ++ * ++ * @kbdev: Pointer to the device ++ */ ++void kbase_csf_debugfs_init(struct kbase_device *kbdev); ++ ++#endif /* _KBASE_CSF_CSG_DEBUGFS_H_ */ +diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_defs.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_defs.h +new file mode 100755 +index 000000000000..3829572a1aeb +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_defs.h +@@ -0,0 +1,883 @@ ++/* ++ * ++ * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* Definitions (types, defines, etcs) common to the command stream frontend. ++ * They are placed here to allow the hierarchy of header files to work. ++ */ ++ ++#ifndef _KBASE_CSF_DEFS_H_ ++#define _KBASE_CSF_DEFS_H_ ++ ++#include ++#include ++ ++#include "mali_kbase_csf_firmware.h" ++ ++/* Maximum number of KCPU command queues to be created per GPU address space. ++ */ ++#define KBASEP_MAX_KCPU_QUEUES ((size_t)256) ++ ++/* Maximum number of GPU command queue groups to be created per GPU address ++ * space. ++ */ ++#define MAX_QUEUE_GROUP_NUM (256) ++ ++/* Maximum number of GPU tiler heaps to allow to be created per GPU address ++ * space. ++ */ ++#define MAX_TILER_HEAPS (128) ++ ++/** ++ * enum kbase_csf_bind_state - bind state of the queue ++ * ++ * @KBASE_CSF_QUEUE_UNBOUND: Set when the queue is registered or when the link ++ * between queue and the group to which it was bound or being bound is removed. ++ * @KBASE_CSF_QUEUE_BIND_IN_PROGRESS: Set when the first part of bind operation ++ * has completed i.e. CS_QUEUE_BIND ioctl. ++ * @KBASE_CSF_QUEUE_BOUND: Set when the bind operation has completed i.e. IO ++ * pages have been mapped in the process address space. ++ */ ++enum kbase_csf_queue_bind_state { ++ KBASE_CSF_QUEUE_UNBOUND, ++ KBASE_CSF_QUEUE_BIND_IN_PROGRESS, ++ KBASE_CSF_QUEUE_BOUND, ++}; ++ ++/** ++ * enum kbase_csf_reset_gpu_state - state of the gpu reset ++ * ++ * @KBASE_CSF_RESET_GPU_NOT_PENDING: Set when the GPU reset isn't pending ++ * @KBASE_CSF_RESET_GPU_HAPPENING: Set when the GPU reset process is occurring ++ * @KBASE_CSF_RESET_GPU_SILENT: Set when the GPU reset process is occurring, ++ * used when resetting the GPU as part of normal behavior (e.g. when exiting ++ * protected mode). ++ * @KBASE_CSF_RESET_GPU_FAILED: Set when an error is encountered during the ++ * GPU reset process. No more work could then be executed on GPU, unloading ++ * the Driver module is the only option. ++ */ ++enum kbase_csf_reset_gpu_state { ++ KBASE_CSF_RESET_GPU_NOT_PENDING, ++ KBASE_CSF_RESET_GPU_HAPPENING, ++ KBASE_CSF_RESET_GPU_SILENT, ++ KBASE_CSF_RESET_GPU_FAILED, ++}; ++ ++/** ++ * enum kbase_csf_group_state - state of the GPU command queue group ++ * ++ * @KBASE_CSF_GROUP_INACTIVE: Group is inactive and won't be ++ * considered by scheduler for running on ++ * command stream group slot. ++ * @KBASE_CSF_GROUP_RUNNABLE: Group is in the list of runnable groups ++ * and is subjected to time-slice based ++ * scheduling. A start request would be ++ * sent (or already has been sent) if the ++ * group is assigned the command stream ++ * group slot for the fist time. ++ * @KBASE_CSF_GROUP_IDLE: Group is currently on a command stream ++ * group slot but all the command streams ++ * bound to the group have become either ++ * idle or waiting on sync object. ++ * Group could be evicted from the slot on ++ * the next tick if there are no spare ++ * slots left after scheduling non-idle ++ * queue groups. If the group is kept on ++ * slot then it would be moved to the ++ * RUNNABLE state, also if one of the ++ * queues bound to the group is kicked it ++ * would be moved to the RUNNABLE state. ++ * If the group is evicted from the slot it ++ * would be moved to either ++ * KBASE_CSF_GROUP_SUSPENDED_ON_IDLE or ++ * KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC ++ * state. ++ * @KBASE_CSF_GROUP_SUSPENDED: Group was evicted from the command ++ * stream group slot and is not running but ++ * is still in the list of runnable groups ++ * and subjected to time-slice based ++ * scheduling. A resume request would be ++ * sent when a command stream group slot is ++ * re-assigned to the group and once the ++ * resume is complete group would be moved ++ * back to the RUNNABLE state. ++ * @KBASE_CSF_GROUP_SUSPENDED_ON_IDLE: Same as KBASE_CSF_GROUP_SUSPENDED except ++ * that queue group also became idle before ++ * the suspension. This state helps ++ * Scheduler avoid scheduling the idle ++ * groups over the non-idle groups in the ++ * subsequent ticks. If one of the queues ++ * bound to the group is kicked it would be ++ * moved to the SUSPENDED state. ++ * @KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC: Same as GROUP_SUSPENDED_ON_IDLE ++ * except that at least one command ++ * stream bound to this group was ++ * waiting for synchronization object ++ * before the suspension. ++ * @KBASE_CSF_GROUP_FAULT_EVICTED: Group is evicted from the scheduler due ++ * to a fault condition, pending to be ++ * terminated. ++ * @KBASE_CSF_GROUP_TERMINATED: Group is no longer schedulable and is ++ * pending to be deleted by Client, all the ++ * queues bound to it have been unbound. ++ */ ++enum kbase_csf_group_state { ++ KBASE_CSF_GROUP_INACTIVE, ++ KBASE_CSF_GROUP_RUNNABLE, ++ KBASE_CSF_GROUP_IDLE, ++ KBASE_CSF_GROUP_SUSPENDED, ++ KBASE_CSF_GROUP_SUSPENDED_ON_IDLE, ++ KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC, ++ KBASE_CSF_GROUP_FAULT_EVICTED, ++ KBASE_CSF_GROUP_TERMINATED, ++}; ++ ++/** ++ * enum kbase_csf_csg_slot_state - state of the command queue group slots under ++ * the scheduler control. ++ * ++ * @CSG_SLOT_READY: The slot is clean and ready to be programmed with a ++ * queue group. ++ * @CSG_SLOT_READY2RUN: The slot has been programmed with a queue group, i.e. a ++ * start or resume request has been sent to the firmware. ++ * @CSG_SLOT_RUNNING: The queue group is running on the slot, acknowledgment ++ * of a start or resume request has been obtained from the ++ * firmware. ++ * @CSG_SLOT_DOWN2STOP: The suspend or terminate request for the queue group on ++ * the slot has been sent to the firmware. ++ * @CSG_SLOT_STOPPED: The queue group is removed from the slot, acknowledgment ++ * of suspend or terminate request has been obtained from ++ * the firmware. ++ * @CSG_SLOT_READY2RUN_TIMEDOUT: The start or resume request sent on the slot ++ * for the queue group timed out. ++ * @CSG_SLOT_DOWN2STOP_TIMEDOUT: The suspend or terminate request for queue ++ * group on the slot timed out. ++ */ ++enum kbase_csf_csg_slot_state { ++ CSG_SLOT_READY, ++ CSG_SLOT_READY2RUN, ++ CSG_SLOT_RUNNING, ++ CSG_SLOT_DOWN2STOP, ++ CSG_SLOT_STOPPED, ++ CSG_SLOT_READY2RUN_TIMEDOUT, ++ CSG_SLOT_DOWN2STOP_TIMEDOUT, ++}; ++ ++/** ++ * enum kbase_csf_scheduler_state - state of the scheduler operational phases. ++ * ++ * @SCHED_BUSY: The scheduler is busy performing on tick schedule ++ * operations, the state of command stream group slots ++ * can't be changed. ++ * @SCHED_INACTIVE: The scheduler is inactive, it is allowed to modify the ++ * state of command stream group slots by in-cycle ++ * priority scheduling. ++ * @SCHED_SUSPENDED: The scheduler is in low-power mode with scheduling ++ * operations suspended and is not holding the power ++ * management reference. This can happen if the GPU ++ * becomes idle for a duration exceeding a threshold, ++ * or due to a system triggered suspend action. ++ */ ++enum kbase_csf_scheduler_state { ++ SCHED_BUSY, ++ SCHED_INACTIVE, ++ SCHED_SUSPENDED, ++}; ++ ++/** ++ * struct kbase_csf_notification - Event or error generated as part of command ++ * queue execution ++ * ++ * @data: Event or error data returned to userspace ++ * @link: Link to the linked list, &struct_kbase_csf_context.error_list. ++ */ ++struct kbase_csf_notification { ++ struct base_csf_notification data; ++ struct list_head link; ++}; ++ ++/** ++ * struct kbase_queue - Object representing a GPU command queue. ++ * ++ * @kctx: Pointer to the base context with which this GPU command queue ++ * is associated. ++ * @reg: Pointer to the region allocated from the shared ++ * interface segment for mapping the User mode ++ * input/output pages in MCU firmware address space. ++ * @phys: Pointer to the physical pages allocated for the ++ * pair or User mode input/output page ++ * @user_io_addr: Pointer to the permanent kernel mapping of User mode ++ * input/output pages. The pages can be accessed through ++ * the mapping without any cache maintenance. ++ * @handle: Handle returned with bind ioctl for creating a ++ * contiguous User mode mapping of input/output pages & ++ * the hardware doorbell page. ++ * @doorbell_nr: Index of the hardware doorbell page assigned to the ++ * queue. ++ * @db_file_offset: File offset value that is assigned to userspace mapping ++ * created on bind to access the doorbell page. ++ * It is in page units. ++ * @link: Link to the linked list of GPU command queues created per ++ * GPU address space. ++ * @refcount: Reference count, stands for the number of times the queue ++ * has been referenced. The reference is taken when it is ++ * created, when it is bound to the group and also when the ++ * @oom_event_work or @fault_event_work work item is queued ++ * for it. ++ * @group: Pointer to the group to which this queue is bound. ++ * @queue_reg: Pointer to the VA region allocated for command ++ * stream buffer. ++ * @oom_event_work: Work item corresponding to the out of memory event for ++ * chunked tiler heap being used for this queue. ++ * @fault_event_work: Work item corresponding to the firmware fault event. ++ * @base_addr: Base address of the command stream buffer. ++ * @size: Size of the command stream buffer. ++ * @priority: Priority of this queue within the group. ++ * @bind_state: Bind state of the queue. ++ * @csi_index: The ID of the assigned command stream hardware interface. ++ * @enabled: Indicating whether the command stream is running, or not. ++ * @status_wait: Value of CS_STATUS_WAIT register of the command stream will ++ * be kept when the command stream gets blocked by sync wait. ++ * CS_STATUS_WAIT provides information on conditions queue is ++ * blocking on. This is set when the group, to which queue is ++ * bound, is suspended after getting blocked, i.e. in ++ * KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC state. ++ * @sync_ptr: Value of CS_STATUS_WAIT_SYNC_POINTER register of the command ++ * stream will be kept when the command stream gets blocked by ++ * sync wait. CS_STATUS_WAIT_SYNC_POINTER contains the address ++ * of synchronization object being waited on. ++ * Valid only when @status_wait is set. ++ * @sync_value: Value of CS_STATUS_WAIT_SYNC_VALUE register of the command ++ * stream will be kept when the command stream gets blocked by ++ * sync wait. CS_STATUS_WAIT_SYNC_VALUE contains the value ++ * tested against the synchronization object. ++ * Valid only when @status_wait is set. ++ * @error: GPU command queue fatal information to pass to user space. ++ */ ++struct kbase_queue { ++ struct kbase_context *kctx; ++ struct kbase_va_region *reg; ++ struct tagged_addr phys[2]; ++ char *user_io_addr; ++ u64 handle; ++ int doorbell_nr; ++ unsigned long db_file_offset; ++ struct list_head link; ++ atomic_t refcount; ++ struct kbase_queue_group *group; ++ struct kbase_va_region *queue_reg; ++ struct work_struct oom_event_work; ++ struct work_struct fault_event_work; ++ u64 base_addr; ++ u32 size; ++ u8 priority; ++ u8 bind_state; ++ s8 csi_index; ++ bool enabled; ++ u32 status_wait; ++ u64 sync_ptr; ++ u32 sync_value; ++ struct kbase_csf_notification error; ++}; ++ ++/** ++ * struct kbase_normal_suspend_buffer - Object representing a normal ++ * suspend buffer for queue group. ++ * @reg: Memory region allocated for the normal-mode suspend buffer. ++ * @phy: Array of physical memory pages allocated for the normal- ++ * mode suspend buffer. ++ */ ++struct kbase_normal_suspend_buffer { ++ struct kbase_va_region *reg; ++ struct tagged_addr *phy; ++}; ++ ++/** ++ * struct kbase_protected_suspend_buffer - Object representing a protected ++ * suspend buffer for queue group. ++ * @reg: Memory region allocated for the protected-mode suspend buffer. ++ * @pma: Array of pointer to protected mode allocations containing ++ * information about memory pages allocated for protected mode ++ * suspend buffer. ++ */ ++struct kbase_protected_suspend_buffer { ++ struct kbase_va_region *reg; ++ struct protected_memory_allocation **pma; ++}; ++ ++/** ++ * struct kbase_queue_group - Object representing a GPU command queue group. ++ * ++ * @kctx: Pointer to the kbase context with which this queue group ++ * is associated. ++ * @normal_suspend_buf: Object representing the normal suspend buffer. ++ * Normal-mode suspend buffer that is used for ++ * group context switch. ++ * @protected_suspend_buf: Object representing the protected suspend ++ * buffer. Protected-mode suspend buffer that is ++ * used for group context switch. ++ * @handle: Handle which identifies this queue group. ++ * @csg_nr: Number/index of the command stream group to ++ * which this queue group is mapped; KBASEP_CSG_NR_INVALID ++ * indicates that the queue group is not scheduled. ++ * @priority: Priority of the queue group, 0 being the highest, ++ * BASE_QUEUE_GROUP_PRIORITY_COUNT - 1 being the lowest. ++ * @tiler_max: Maximum number of tiler endpoints the group is allowed ++ * to use. ++ * @fragment_max: Maximum number of fragment endpoints the group is ++ * allowed to use. ++ * @compute_max: Maximum number of compute endpoints the group is ++ * allowed to use. ++ * @tiler_mask: Mask of tiler endpoints the group is allowed to use. ++ * @fragment_mask: Mask of fragment endpoints the group is allowed to use. ++ * @compute_mask: Mask of compute endpoints the group is allowed to use. ++ * @link: Link to this queue group in the 'runnable_groups' list of ++ * the corresponding kctx. ++ * @link_to_schedule: Link to this queue group in the list of prepared groups ++ * to be scheduled, if the group is runnable/suspended. ++ * If the group is idle or waiting for CQS, it would be a ++ * link to the list of idle/blocked groups list. ++ * @timer_event_work: Work item corresponding to the event generated when a task ++ * started by a queue in this group takes too long to execute ++ * on an endpoint. ++ * @run_state: Current state of the queue group. ++ * @prepared_seq_num: Indicates the position of queue group in the list of ++ * prepared groups to be scheduled. ++ * @faulted: Indicates that a GPU fault occurred for the queue group. ++ * This flag persists until the fault has been queued to be ++ * reported to userspace. ++ * @bound_queues: Array of registered queues bound to this queue group. ++ * @doorbell_nr: Index of the hardware doorbell page assigned to the ++ * group. ++ * @protm_event_work: Work item corresponding to the protected mode entry ++ * event for this queue. ++ * @protm_pending_bitmap: Bit array to keep a track of command streams that ++ * have pending protected mode entry requests. ++ * @error_fatal: An error of type BASE_GPU_QUEUE_GROUP_ERROR_FATAL to be ++ * returned to userspace if such an error has occurred. ++ * @error_timeout: An error of type BASE_GPU_QUEUE_GROUP_ERROR_TIMEOUT ++ * to be returned to userspace if such an error has occurred. ++ * @error_tiler_oom: An error of type BASE_GPU_QUEUE_GROUP_ERROR_TILER_HEAP_OOM ++ * to be returned to userspace if such an error has occurred. ++ */ ++struct kbase_queue_group { ++ struct kbase_context *kctx; ++ struct kbase_normal_suspend_buffer normal_suspend_buf; ++ struct kbase_protected_suspend_buffer protected_suspend_buf; ++ u8 handle; ++ s8 csg_nr; ++ u8 priority; ++ ++ u8 tiler_max; ++ u8 fragment_max; ++ u8 compute_max; ++ ++ u64 tiler_mask; ++ u64 fragment_mask; ++ u64 compute_mask; ++ ++ struct list_head link; ++ struct list_head link_to_schedule; ++ struct work_struct timer_event_work; ++ enum kbase_csf_group_state run_state; ++ u32 prepared_seq_num; ++ bool faulted; ++ ++ struct kbase_queue *bound_queues[MAX_SUPPORTED_STREAMS_PER_GROUP]; ++ ++ int doorbell_nr; ++ struct work_struct protm_event_work; ++ DECLARE_BITMAP(protm_pending_bitmap, MAX_SUPPORTED_STREAMS_PER_GROUP); ++ ++ struct kbase_csf_notification error_fatal; ++ struct kbase_csf_notification error_timeout; ++ struct kbase_csf_notification error_tiler_oom; ++}; ++ ++/** ++ * struct kbase_csf_kcpu_queue_context - Object representing the kernel CPU ++ * queues for a GPU address space. ++ * ++ * @lock: Lock preventing concurrent access to @array and the @in_use bitmap. ++ * @array: Array of pointers to kernel CPU command queues. ++ * @in_use: Bitmap which indicates which kernel CPU command queues are in use. ++ * @wq: Dedicated workqueue for processing kernel CPU command queues. ++ * @num_cmds: The number of commands that have been enqueued across ++ * all the KCPU command queues. This could be used as a ++ * timestamp to determine the command's enqueueing time. ++ * @jit_cmds_head: A list of the just-in-time memory commands, both ++ * allocate & free, in submission order, protected ++ * by kbase_csf_kcpu_queue_context.lock. ++ * @jit_blocked_queues: A list of KCPU command queues blocked by a pending ++ * just-in-time memory allocation command which will be ++ * reattempted after the impending free of other active ++ * allocations. ++ */ ++struct kbase_csf_kcpu_queue_context { ++ struct mutex lock; ++ struct kbase_kcpu_command_queue *array[KBASEP_MAX_KCPU_QUEUES]; ++ DECLARE_BITMAP(in_use, KBASEP_MAX_KCPU_QUEUES); ++ struct workqueue_struct *wq; ++ u64 num_cmds; ++ ++ struct list_head jit_cmds_head; ++ struct list_head jit_blocked_queues; ++}; ++ ++/** ++ * struct kbase_csf_heap_context_allocator - Allocator of heap contexts ++ * ++ * Heap context structures are allocated by the kernel for use by the firmware. ++ * The current implementation subdivides a single GPU memory region for use as ++ * a sparse array. ++ * ++ * @kctx: Pointer to the kbase context with which this allocator is ++ * associated. ++ * @region: Pointer to a GPU memory region from which heap context structures ++ * are allocated. NULL if no heap contexts have been allocated. ++ * @gpu_va: GPU virtual address of the start of the region from which heap ++ * context structures are allocated. 0 if no heap contexts have been ++ * allocated. ++ * @lock: Lock preventing concurrent access to the @in_use bitmap. ++ * @in_use: Bitmap that indicates which heap context structures are currently ++ * allocated (in @region). ++ */ ++struct kbase_csf_heap_context_allocator { ++ struct kbase_context *kctx; ++ struct kbase_va_region *region; ++ u64 gpu_va; ++ struct mutex lock; ++ DECLARE_BITMAP(in_use, MAX_TILER_HEAPS); ++}; ++ ++/** ++ * struct kbase_csf_tiler_heap_context - Object representing the tiler heaps ++ * context for a GPU address space. ++ * ++ * This contains all of the command-stream front-end state relating to chunked ++ * tiler heaps for one @kbase_context. It is not the same as a heap context ++ * structure allocated by the kernel for use by the firmware. ++ * ++ * @lock: Lock preventing concurrent access to the tiler heaps. ++ * @list: List of tiler heaps. ++ * @ctx_alloc: Allocator for heap context structures. ++ */ ++struct kbase_csf_tiler_heap_context { ++ struct mutex lock; ++ struct list_head list; ++ struct kbase_csf_heap_context_allocator ctx_alloc; ++}; ++ ++/** ++ * struct kbase_csf_scheduler_context - Object representing the scheduler's ++ * context for a GPU address space. ++ * ++ * @runnable_groups: Lists of runnable GPU command queue groups in the kctx, ++ * one per queue group priority level. ++ * @num_runnable_grps: Total number of runnable groups across all priority ++ * levels in @runnable_groups. ++ * @idle_wait_groups: A list of GPU command queue groups in which all enabled ++ * GPU command queues are idle and at least one of them ++ * is blocked on a sync wait operation. ++ * @num_idle_wait_grps: Length of the @idle_wait_groups list. ++ * @sync_update_wq: Dedicated workqueue to process work items corresponding ++ * to the sync_update events by sync_set/sync_add ++ * instruction execution on command streams bound to groups ++ * of @idle_wait_groups list. ++ * @sync_update_work: work item to process the sync_update events by ++ * sync_set / sync_add instruction execution on command ++ * streams bound to groups of @idle_wait_groups list. ++ * @ngrp_to_schedule: Number of groups added for the context to the ++ * 'groups_to_schedule' list of scheduler instance. ++ */ ++struct kbase_csf_scheduler_context { ++ struct list_head runnable_groups[BASE_QUEUE_GROUP_PRIORITY_COUNT]; ++ u32 num_runnable_grps; ++ struct list_head idle_wait_groups; ++ u32 num_idle_wait_grps; ++ struct workqueue_struct *sync_update_wq; ++ struct work_struct sync_update_work; ++ u32 ngrp_to_schedule; ++}; ++ ++/** ++ * struct kbase_csf_context - Object representing command-stream front-end ++ * for a GPU address space. ++ * ++ * @event_pages_head: A list of pages allocated for the event memory used by ++ * the synchronization objects. A separate list would help ++ * in the fast lookup, since the list is expected to be short ++ * as one page would provide the memory for up to 1K ++ * synchronization objects. ++ * KBASE_PERMANENTLY_MAPPED_MEM_LIMIT_PAGES is the upper ++ * bound on the size of event memory. ++ * @cookies: Bitmask containing of KBASE_CSF_NUM_USER_IO_PAGES_HANDLE ++ * bits, used for creating the User mode CPU mapping in a ++ * deferred manner of a pair of User mode input/output pages ++ * & a hardware doorbell page. ++ * The pages are allocated when a GPU command queue is ++ * bound to a command stream group in kbase_csf_queue_bind. ++ * This helps returning unique handles to Userspace from ++ * kbase_csf_queue_bind and later retrieving the pointer to ++ * queue in the mmap handler. ++ * @user_pages_info: Array containing pointers to queue ++ * structures, used in conjunction with cookies bitmask for ++ * providing a mechansim to create a CPU mapping of ++ * input/output pages & hardware doorbell page. ++ * @lock: Serializes accesses to all members, except for ones that ++ * have their own locks. ++ * @queue_groups: Array of registered GPU command queue groups. ++ * @queue_list: Linked list of GPU command queues not yet deregistered. ++ * Note that queues can persist after deregistration if the ++ * userspace mapping created for them on bind operation ++ * hasn't been removed. ++ * @kcpu_queues: Kernel CPU command queues. ++ * @event_lock: Lock protecting access to @event_callback_list ++ * @event_callback_list: List of callbacks which are registered to serve CSF ++ * events. ++ * @tiler_heaps: Chunked tiler memory heaps. ++ * @wq: Dedicated workqueue to process work items corresponding ++ * to the OoM events raised for chunked tiler heaps being ++ * used by GPU command queues, and progress timeout events. ++ * @link: Link to this csf context in the 'runnable_kctxs' list of ++ * the scheduler instance ++ * @user_reg_vma: Pointer to the vma corresponding to the virtual mapping ++ * of the USER register page. Currently used only for sanity ++ * checking. ++ * @sched: Object representing the scheduler's context ++ * @error_list: List for command stream fatal errors in this context. ++ * Link of fatal error is ++ * &struct_kbase_csf_notification.link. ++ * @lock needs to be held to access to this list. ++ */ ++struct kbase_csf_context { ++ struct list_head event_pages_head; ++ DECLARE_BITMAP(cookies, KBASE_CSF_NUM_USER_IO_PAGES_HANDLE); ++ struct kbase_queue *user_pages_info[ ++ KBASE_CSF_NUM_USER_IO_PAGES_HANDLE]; ++ struct mutex lock; ++ struct kbase_queue_group *queue_groups[MAX_QUEUE_GROUP_NUM]; ++ struct list_head queue_list; ++ struct kbase_csf_kcpu_queue_context kcpu_queues; ++ spinlock_t event_lock; ++ struct list_head event_callback_list; ++ struct kbase_csf_tiler_heap_context tiler_heaps; ++ struct workqueue_struct *wq; ++ struct list_head link; ++ struct vm_area_struct *user_reg_vma; ++ struct kbase_csf_scheduler_context sched; ++ struct list_head error_list; ++}; ++ ++/** ++ * struct kbase_csf_reset_gpu - Object containing the members required for ++ * GPU reset handling. ++ * @workq: Workqueue to execute the GPU reset work item @work. ++ * @work: Work item for performing the GPU reset. ++ * @wait: Wait queue used to wait for the GPU reset completion. ++ * @state: Tracks if the GPU reset is in progress or not. ++ */ ++struct kbase_csf_reset_gpu { ++ struct workqueue_struct *workq; ++ struct work_struct work; ++ wait_queue_head_t wait; ++ atomic_t state; ++}; ++ ++/** ++ * struct kbase_csf_csg_slot - Object containing members for tracking the state ++ * of command stream group slots. ++ * @resident_group: pointer to the queue group that is resident on the ++ * command stream group slot. ++ * @state: state of the slot as per enum kbase_csf_csg_slot_state. ++ * @trigger_jiffies: value of jiffies when change in slot state is recorded. ++ * @priority: dynamic priority assigned to command stream group slot. ++ */ ++struct kbase_csf_csg_slot { ++ struct kbase_queue_group *resident_group; ++ atomic_t state; ++ unsigned long trigger_jiffies; ++ u8 priority; ++}; ++ ++/** ++ * struct kbase_csf_scheduler - Object representing the scheduler used for ++ * command-stream front-end for an instance of ++ * GPU platform device. ++ * @lock: Lock to serialize the scheduler operations and ++ * access to the data members. ++ * @interrupt_lock: Lock to protect members accessed by interrupt ++ * handler. ++ * @state: The operational phase the scheduler is in. Primarily ++ * used for indicating what in-cycle schedule actions ++ * are allowed. ++ * @doorbell_inuse_bitmap: Bitmap of hardware doorbell pages keeping track of ++ * which pages are currently available for assignment ++ * to clients. ++ * @csg_inuse_bitmap: Bitmap to keep a track of command stream group slots ++ * that are currently in use. ++ * @csg_slots: The array for tracking the state of command stream ++ * group slots. ++ * @runnable_kctxs: List of Kbase contexts that have runnable command ++ * queue groups. ++ * @groups_to_schedule: List of runnable queue groups prepared on every ++ * scheduler tick. The dynamic priority of the command ++ * stream group slot assigned to a group will depend ++ * upon the position of group in the list. ++ * @ngrp_to_schedule: Number of groups in the @groups_to_schedule list, ++ * incremented when a group is added to the list, used ++ * to record the position of group in the list. ++ * @num_active_address_spaces: Number of GPU address space slots that would get ++ * used to program the groups in @groups_to_schedule ++ * list on all the available command stream group ++ * slots. ++ * @num_csg_slots_for_tick: Number of command stream group slots that can be ++ * active in the given tick/tock. This depends on the ++ * value of @num_active_address_spaces. ++ * @idle_groups_to_schedule: List of runnable queue groups, in which all GPU ++ * command queues became idle or are waiting for ++ * synchronization object, prepared on every ++ * scheduler tick. The groups in this list are ++ * appended to the tail of @groups_to_schedule list ++ * after the scan out so that the idle groups aren't ++ * preferred for scheduling over the non-idle ones. ++ * @total_runnable_grps: Total number of runnable groups across all KCTXs. ++ * @csgs_events_enable_mask: Use for temporary masking off asynchronous events ++ * from firmware (such as OoM events) before a group ++ * is suspended. ++ * @csg_slots_idle_mask: Bit array for storing the mask of command stream ++ * group slots for which idle notification was ++ * received. ++ * @csg_slots_prio_update: Bit array for tracking slots that have an on-slot ++ * priority update operation. ++ * @last_schedule: Time in jiffies recorded when the last "tick" or ++ * "tock" schedule operation concluded. Used for ++ * evaluating the exclusion window for in-cycle ++ * schedule operation. ++ * @timer_enabled: Whether the CSF scheduler wakes itself up for ++ * periodic scheduling tasks. If this value is 0 ++ * then it will only perform scheduling under the ++ * influence of external factors e.g., IRQs, IOCTLs. ++ * @wq: Dedicated workqueue to execute the @tick_work. ++ * @tick_work: Work item that would perform the schedule on tick ++ * operation to implement the time slice based ++ * scheduling. ++ * @tock_work: Work item that would perform the schedule on tock ++ * operation to implement the asynchronous scheduling. ++ * @ping_work: Work item that would ping the firmware at regular ++ * intervals, only if there is a single active command ++ * stream group slot, to check if firmware is alive ++ * and would initiate a reset if the ping request ++ * isn't acknowledged. ++ * @top_ctx: Pointer to the Kbase context corresponding to the ++ * @top_grp. ++ * @top_grp: Pointer to queue group inside @groups_to_schedule ++ * list that was assigned the highest slot priority. ++ * @head_slot_priority: The dynamic slot priority to be used for the ++ * queue group at the head of @groups_to_schedule ++ * list. Once the queue group is assigned a command ++ * stream group slot, it is removed from the list and ++ * priority is decremented. ++ * @tock_pending_request: A "tock" request is pending: a group that is not ++ * currently on the GPU demands to be scheduled. ++ * @active_protm_grp: Indicates if firmware has been permitted to let GPU ++ * enter protected mode with the given group. On exit ++ * from protected mode the pointer is reset to NULL. ++ * @gpu_idle_work: Work item for facilitating the scheduler to bring ++ * the GPU to a low-power mode on becoming idle. ++ * @non_idle_suspended_grps: Count of suspended queue groups not idle. ++ * @pm_active_count: Count indicating if the scheduler is owning a power ++ * management reference count. Reference is taken when ++ * the count becomes 1 and is dropped when the count ++ * becomes 0. It is used to enable the power up of MCU ++ * after GPU and L2 cache have been powered up. So when ++ * this count is zero, MCU will not be powered up. ++ */ ++struct kbase_csf_scheduler { ++ struct mutex lock; ++ spinlock_t interrupt_lock; ++ enum kbase_csf_scheduler_state state; ++ DECLARE_BITMAP(doorbell_inuse_bitmap, CSF_NUM_DOORBELL); ++ DECLARE_BITMAP(csg_inuse_bitmap, MAX_SUPPORTED_CSGS); ++ struct kbase_csf_csg_slot *csg_slots; ++ struct list_head runnable_kctxs; ++ struct list_head groups_to_schedule; ++ u32 ngrp_to_schedule; ++ u32 num_active_address_spaces; ++ u32 num_csg_slots_for_tick; ++ struct list_head idle_groups_to_schedule; ++ u32 total_runnable_grps; ++ DECLARE_BITMAP(csgs_events_enable_mask, MAX_SUPPORTED_CSGS); ++ DECLARE_BITMAP(csg_slots_idle_mask, MAX_SUPPORTED_CSGS); ++ DECLARE_BITMAP(csg_slots_prio_update, MAX_SUPPORTED_CSGS); ++ unsigned long last_schedule; ++ bool timer_enabled; ++ struct workqueue_struct *wq; ++ struct delayed_work tick_work; ++ struct delayed_work tock_work; ++ struct delayed_work ping_work; ++ struct kbase_context *top_ctx; ++ struct kbase_queue_group *top_grp; ++ u8 head_slot_priority; ++ bool tock_pending_request; ++ struct kbase_queue_group *active_protm_grp; ++ struct delayed_work gpu_idle_work; ++ atomic_t non_idle_suspended_grps; ++ u32 pm_active_count; ++}; ++ ++/** ++ * Number of GPU cycles per unit of the global progress timeout. ++ */ ++#define GLB_PROGRESS_TIMER_TIMEOUT_SCALE ((u64)1024) ++ ++/** ++ * Maximum value of the global progress timeout. ++ */ ++#define GLB_PROGRESS_TIMER_TIMEOUT_MAX \ ++ ((GLB_PROGRESS_TIMER_TIMEOUT_MASK >> \ ++ GLB_PROGRESS_TIMER_TIMEOUT_SHIFT) * \ ++ GLB_PROGRESS_TIMER_TIMEOUT_SCALE) ++ ++/** ++ * struct kbase_csf - Object representing command-stream front-end for an ++ * instance of GPU platform device. ++ * ++ * @mcu_mmu: MMU page tables for the MCU firmware ++ * @firmware_interfaces: List of interfaces defined in the firmware image ++ * @firmware_config: List of configuration options within the firmware ++ * image ++ * @firmware_timeline_metadata: List of timeline meta-data within the firmware ++ * image ++ * @fw_cfg_kobj: Pointer to the kobject corresponding to the sysf ++ * directory that contains a sub-directory for each ++ * of the configuration option present in the ++ * firmware image. ++ * @firmware_trace_buffers: List of trace buffers described in the firmware ++ * image. ++ * @shared_interface: Pointer to the interface object containing info for ++ * the memory area shared between firmware & host. ++ * @shared_reg_rbtree: RB tree of the memory regions allocated from the ++ * shared interface segment in MCU firmware address ++ * space. ++ * @db_filp: Pointer to a dummy file, that alongwith ++ * @db_file_offsets, facilitates the use of unqiue ++ * file offset for the userspace mapping created ++ * for Hw Doorbell pages. The userspace mapping ++ * is made to point to this file inside the mmap ++ * handler. ++ * @db_file_offsets: Counter that is incremented every time a GPU ++ * command queue is bound to provide a unique file ++ * offset range for @db_filp file, so that pte of ++ * Doorbell page can be zapped through the kernel ++ * function unmap_mapping_range(). It is incremented ++ * in page units. ++ * @dummy_db_page: Address of the dummy page that is mapped in place ++ * of the real Hw doorbell page for the active GPU ++ * command queues after they are stopped or after the ++ * GPU is powered down. ++ * @reg_lock: Lock to serialize the MCU firmware related actions ++ * that affect all contexts such as allocation of ++ * regions from shared interface area, assignment of ++ * of hardware doorbell pages, assignment of CSGs, ++ * sending global requests. ++ * @event_wait: Wait queue to wait for receiving csf events, i.e. ++ * the interrupt from CSF firmware, or scheduler state ++ * changes. ++ * @interrupt_received: Flag set when the interrupt is received from CSF fw ++ * @global_iface: The result of parsing the global interface ++ * structure set up by the firmware, including the ++ * CSGs, CSs, and their properties ++ * @scheduler: The command stream scheduler instance. ++ * @reset: Contain members required for GPU reset handling. ++ * @progress_timeout: Maximum number of GPU clock cycles without forward ++ * progress to allow, for all tasks running on ++ * hardware endpoints (e.g. shader cores), before ++ * terminating a GPU command queue group. ++ * Must not exceed @GLB_PROGRESS_TIMER_TIMEOUT_MAX. ++ * @pma_dev: Pointer to protected memory allocator device. ++ * @firmware_inited: Flag for indicating that the cold-boot stage of ++ * the MCU has completed. ++ * @firmware_reloaded: Flag for indicating a firmware reload operation ++ * in GPU reset has completed. ++ * @firmware_reload_needed: Flag for indicating that the firmware needs to be ++ * reloaded as part of the GPU reset action. ++ * @firmware_reload_work: Work item for facilitating the procedural actions ++ * on reloading the firmware. ++ * @glb_init_request_pending: Flag to indicate that Global requests have been ++ * sent to the FW after MCU was re-enabled and their ++ * acknowledgement is pending. ++ */ ++struct kbase_csf_device { ++ struct kbase_mmu_table mcu_mmu; ++ struct list_head firmware_interfaces; ++ struct list_head firmware_config; ++ struct list_head firmware_timeline_metadata; ++ struct kobject *fw_cfg_kobj; ++ struct kbase_csf_trace_buffers firmware_trace_buffers; ++ void *shared_interface; ++ struct rb_root shared_reg_rbtree; ++ struct file *db_filp; ++ u32 db_file_offsets; ++ struct tagged_addr dummy_db_page; ++ struct mutex reg_lock; ++ wait_queue_head_t event_wait; ++ bool interrupt_received; ++ struct kbase_csf_global_iface global_iface; ++ struct kbase_csf_scheduler scheduler; ++ struct kbase_csf_reset_gpu reset; ++ atomic64_t progress_timeout; ++ struct protected_memory_allocator_device *pma_dev; ++ bool firmware_inited; ++ bool firmware_reloaded; ++ bool firmware_reload_needed; ++ struct work_struct firmware_reload_work; ++ bool glb_init_request_pending; ++}; ++ ++/** ++ * struct kbase_as - Object representing an address space of GPU. ++ * @number: Index at which this address space structure is present ++ * in an array of address space structures embedded inside ++ * the &struct kbase_device. ++ * @pf_wq: Workqueue for processing work items related to ++ * Page fault, Bus fault and GPU fault handling. ++ * @work_pagefault: Work item for the Page fault handling. ++ * @work_busfault: Work item for the Bus fault handling. ++ * @work_gpufault: Work item for the GPU fault handling. ++ * @pf_data: Data relating to Page fault. ++ * @bf_data: Data relating to Bus fault. ++ * @gf_data: Data relating to GPU fault. ++ * @current_setup: Stores the MMU configuration for this address space. ++ */ ++struct kbase_as { ++ int number; ++ struct workqueue_struct *pf_wq; ++ struct work_struct work_pagefault; ++ struct work_struct work_busfault; ++ struct work_struct work_gpufault; ++ struct kbase_fault pf_data; ++ struct kbase_fault bf_data; ++ struct kbase_fault gf_data; ++ struct kbase_mmu_setup current_setup; ++}; ++ ++#endif /* _KBASE_CSF_DEFS_H_ */ +diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.c +new file mode 100755 +index 000000000000..4a924f346685 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.c +@@ -0,0 +1,1993 @@ ++/* ++ * ++ * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include "mali_kbase.h" ++#include "mali_kbase_csf_firmware_cfg.h" ++#include "mali_kbase_csf_trace_buffer.h" ++#include "mali_kbase_csf_timeout.h" ++#include "mali_kbase_mem.h" ++#include ++#include "mali_kbase_csf_scheduler.h" ++#include "device/mali_kbase_device.h" ++#include "backend/gpu/mali_kbase_pm_internal.h" ++#include "tl/mali_kbase_timeline_priv.h" ++#include "mali_kbase_csf_tl_reader.h" ++ ++#include ++#include ++#include ++#include ++#include ++#if (KERNEL_VERSION(4, 13, 0) <= LINUX_VERSION_CODE) ++#include ++#endif ++#include ++ ++#define MALI_MAX_FIRMWARE_NAME_LEN ((size_t)20) ++ ++static char fw_name[MALI_MAX_FIRMWARE_NAME_LEN] = "mali_csffw.bin"; ++module_param_string(fw_name, fw_name, sizeof(fw_name), 0644); ++MODULE_PARM_DESC(fw_name, "firmware image"); ++ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++/* Makes Driver wait indefinitely for an acknowledgment for the different ++ * requests it sends to firmware. Otherwise the timeouts interfere with the ++ * use of debugger for source-level debugging of firmware as Driver initiates ++ * a GPU reset when a request times out, which always happen when a debugger ++ * is connected. ++ */ ++bool fw_debug; /* Default value of 0/false */ ++module_param(fw_debug, bool, 0444); ++MODULE_PARM_DESC(fw_debug, ++ "Enables effective use of a debugger for debugging firmware code."); ++#endif ++ ++#define FIRMWARE_HEADER_MAGIC (0xC3F13A6Eul) ++#define FIRMWARE_HEADER_VERSION (0ul) ++#define FIRMWARE_HEADER_LENGTH (0x14ul) ++ ++#define CSF_FIRMWARE_ENTRY_READ (1ul << 0) ++#define CSF_FIRMWARE_ENTRY_WRITE (1ul << 1) ++#define CSF_FIRMWARE_ENTRY_EXECUTE (1ul << 2) ++#define CSF_FIRMWARE_ENTRY_CACHE_MODE (3ul << 3) ++#define CSF_FIRMWARE_ENTRY_PROTECTED (1ul << 5) ++#define CSF_FIRMWARE_ENTRY_SHARED (1ul << 30) ++#define CSF_FIRMWARE_ENTRY_ZERO (1ul << 31) ++ ++#define CSF_FIRMWARE_ENTRY_SUPPORTED_FLAGS \ ++ (CSF_FIRMWARE_ENTRY_READ | \ ++ CSF_FIRMWARE_ENTRY_WRITE | \ ++ CSF_FIRMWARE_ENTRY_EXECUTE | \ ++ CSF_FIRMWARE_ENTRY_PROTECTED | \ ++ CSF_FIRMWARE_ENTRY_SHARED | \ ++ CSF_FIRMWARE_ENTRY_ZERO | \ ++ CSF_FIRMWARE_ENTRY_CACHE_MODE) ++ ++#define CSF_FIRMWARE_ENTRY_TYPE_INTERFACE (0) ++#define CSF_FIRMWARE_ENTRY_TYPE_CONFIGURATION (1) ++#define CSF_FIRMWARE_ENTRY_TYPE_FUTF_TEST (2) ++#define CSF_FIRMWARE_ENTRY_TYPE_TRACE_BUFFER (3) ++#define CSF_FIRMWARE_ENTRY_TYPE_TIMELINE_METADATA (4) ++ ++#define CSF_FIRMWARE_CACHE_MODE_NONE (0ul << 3) ++#define CSF_FIRMWARE_CACHE_MODE_CACHED (1ul << 3) ++#define CSF_FIRMWARE_CACHE_MODE_UNCACHED_COHERENT (2ul << 3) ++#define CSF_FIRMWARE_CACHE_MODE_CACHED_COHERENT (3ul << 3) ++ ++#define INTERFACE_ENTRY_NAME_OFFSET (0x14) ++ ++#define TL_METADATA_ENTRY_NAME_OFFSET (0x8) ++ ++#define CSF_FIRMWARE_BOOT_TIMEOUT_MS (500) ++#define CSF_MAX_FW_STOP_LOOPS (100000) ++ ++#define CSF_GLB_REQ_CFG_MASK \ ++ (GLB_REQ_CFG_ALLOC_EN_MASK | GLB_REQ_CFG_PROGRESS_TIMER_MASK) ++ ++static inline u32 input_page_read(const u32 *const input, const u32 offset) ++{ ++ WARN_ON(offset % sizeof(u32)); ++ ++ return input[offset / sizeof(u32)]; ++} ++ ++static inline void input_page_write(u32 *const input, const u32 offset, ++ const u32 value) ++{ ++ WARN_ON(offset % sizeof(u32)); ++ ++ input[offset / sizeof(u32)] = value; ++} ++ ++static inline void input_page_partial_write(u32 *const input, const u32 offset, ++ u32 value, u32 mask) ++{ ++ WARN_ON(offset % sizeof(u32)); ++ ++ input[offset / sizeof(u32)] = ++ (input_page_read(input, offset) & ~mask) | (value & mask); ++} ++ ++static inline u32 output_page_read(const u32 *const output, const u32 offset) ++{ ++ WARN_ON(offset % sizeof(u32)); ++ ++ return output[offset / sizeof(u32)]; ++} ++ ++static unsigned int entry_type(u32 header) ++{ ++ return header & 0xFF; ++} ++static unsigned int entry_size(u32 header) ++{ ++ return (header >> 8) & 0xFF; ++} ++static bool entry_optional(u32 header) ++{ ++ return (header >> 31) & 0x1; ++} ++ ++/** ++ * struct firmware_interface - Represents an interface in the MCU firmware ++ * ++ * @node: Interface objects are on the kbase_device:csf.firmware_interfaces ++ * list using this list_head to link them ++ * @phys: Array of the physical (tagged) addresses making up this interface ++ * @name: NUL-terminated string naming the interface ++ * @num_pages: Number of entries in @phys (and length of the interface) ++ * @virtual: Virtual address that this interface is mapped at for the GPU ++ * @flags: bitmask of CSF_FIRMWARE_ENTRY_* conveying the interface attributes ++ * @data_start: Offset into firmware image at which the interface data starts ++ * @data_end: Offset into firmware image at which the interface data ends ++ * @kernel_map: A kernel mapping of the memory or NULL if not required to be ++ * mapped in the kernel ++ * @pma: Array of pointers to protected memory allocations. ++ */ ++struct firmware_interface { ++ struct list_head node; ++ struct tagged_addr *phys; ++ char *name; ++ u32 num_pages; ++ u32 virtual; ++ u32 flags; ++ u32 data_start; ++ u32 data_end; ++ void *kernel_map; ++ struct protected_memory_allocation **pma; ++}; ++ ++/** ++ * Timeline metadata item within the MCU firmware ++ * ++ * @node: List head linking all timeline metadata to ++ * kbase_device:csf.firmware_timeline_metadata. ++ * @name: NUL-terminated string naming the metadata. ++ * @data: Metadata content. ++ * @size: Metadata size. ++ */ ++struct firmware_timeline_metadata { ++ struct list_head node; ++ char *name; ++ char *data; ++ size_t size; ++}; ++ ++/* The shared interface area, used for communicating with firmware, is managed ++ * like a virtual memory zone. Reserve the virtual space from that zone ++ * corresponding to shared interface entry parsed from the firmware image. ++ * The shared_reg_rbtree should have been initialized before calling this ++ * function. ++ */ ++static int setup_shared_iface_static_region(struct kbase_device *kbdev) ++{ ++ struct firmware_interface *interface = kbdev->csf.shared_interface; ++ struct kbase_va_region *reg; ++ int ret = -ENOMEM; ++ ++ if (!interface) ++ return -EINVAL; ++ ++ reg = kbase_alloc_free_region(&kbdev->csf.shared_reg_rbtree, 0, ++ interface->num_pages, KBASE_REG_ZONE_MCU_SHARED); ++ if (reg) { ++ ret = kbase_add_va_region_rbtree(kbdev, reg, ++ interface->virtual, interface->num_pages, 1); ++ if (ret) ++ kfree(reg); ++ else ++ reg->flags &= ~KBASE_REG_FREE; ++ } ++ ++ return ret; ++} ++ ++static int wait_mcu_status_value(struct kbase_device *kbdev, u32 val) ++{ ++ u32 max_loops = CSF_MAX_FW_STOP_LOOPS; ++ ++ /* wait for the MCU_STATUS register to reach the given status value */ ++ while (--max_loops && ++ (kbase_reg_read(kbdev, GPU_CONTROL_REG(MCU_STATUS)) != val)) { ++ } ++ ++ return (max_loops == 0) ? -1 : 0; ++} ++ ++void kbase_csf_firmware_disable_mcu_wait(struct kbase_device *kbdev) ++{ ++ if (wait_mcu_status_value(kbdev, MCU_CNTRL_DISABLE) < 0) ++ dev_err(kbdev->dev, "MCU failed to get disabled"); ++} ++ ++static void wait_for_firmware_stop(struct kbase_device *kbdev) ++{ ++ if (wait_mcu_status_value(kbdev, MCU_CNTRL_DISABLE) < 0) { ++ /* This error shall go away once MIDJM-2371 is closed */ ++ dev_err(kbdev->dev, "Firmware failed to stop"); ++ } ++} ++ ++static void stop_csf_firmware(struct kbase_device *kbdev) ++{ ++ /* Stop the MCU firmware */ ++ kbase_csf_firmware_disable_mcu(kbdev); ++ ++ wait_for_firmware_stop(kbdev); ++} ++ ++static void wait_for_firmware_boot(struct kbase_device *kbdev) ++{ ++ const long wait_timeout = ++ kbase_csf_timeout_in_jiffies(CSF_FIRMWARE_BOOT_TIMEOUT_MS); ++ long remaining; ++ ++ /* Firmware will generate a global interface interrupt once booting ++ * is complete ++ */ ++ remaining = wait_event_timeout(kbdev->csf.event_wait, ++ kbdev->csf.interrupt_received == true, wait_timeout); ++ ++ if (!remaining) ++ dev_err(kbdev->dev, "Timed out waiting for fw boot completion"); ++ ++ kbdev->csf.interrupt_received = false; ++} ++ ++static void boot_csf_firmware(struct kbase_device *kbdev) ++{ ++ kbase_csf_firmware_enable_mcu(kbdev); ++ ++ wait_for_firmware_boot(kbdev); ++} ++ ++static void wait_ready(struct kbase_device *kbdev) ++{ ++ u32 max_loops = KBASE_AS_INACTIVE_MAX_LOOPS; ++ u32 val; ++ ++ val = kbase_reg_read(kbdev, MMU_AS_REG(MCU_AS_NR, AS_STATUS)); ++ ++ /* Wait for a while for the update command to take effect */ ++ while (--max_loops && (val & AS_STATUS_AS_ACTIVE)) ++ val = kbase_reg_read(kbdev, MMU_AS_REG(MCU_AS_NR, AS_STATUS)); ++ ++ if (max_loops == 0) ++ dev_err(kbdev->dev, "AS_ACTIVE bit stuck, might be caused by slow/unstable GPU clock or possible faulty FPGA connector\n"); ++} ++ ++static void unload_mmu_tables(struct kbase_device *kbdev) ++{ ++ unsigned long irq_flags; ++ ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); ++ if (kbdev->pm.backend.gpu_powered) ++ kbase_mmu_disable_as(kbdev, MCU_AS_NR); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++} ++ ++static void load_mmu_tables(struct kbase_device *kbdev) ++{ ++ unsigned long irq_flags; ++ ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); ++ kbase_mmu_update(kbdev, &kbdev->csf.mcu_mmu, MCU_AS_NR); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ ++ /* Wait for a while for the update command to take effect */ ++ wait_ready(kbdev); ++} ++ ++/** ++ * convert_mem_flags() - Convert firmware memory flags to GPU region flags ++ * ++ * Return: GPU memory region flags ++ * ++ * @kbdev: Instance of GPU platform device (used to determine system coherency) ++ * @flags: Flags of an "interface memory setup" section in a firmware image ++ * @cm: appropriate cache mode chosen for the "interface memory setup" ++ * section, which could be different from the cache mode requested by ++ * firmware. ++ */ ++static unsigned long convert_mem_flags(const struct kbase_device * const kbdev, ++ const u32 flags, u32 *cm) ++{ ++ unsigned long mem_flags = 0; ++ u32 cache_mode = flags & CSF_FIRMWARE_ENTRY_CACHE_MODE; ++ bool is_shared = (flags & CSF_FIRMWARE_ENTRY_SHARED) ? true : false; ++ ++ /* The memory flags control the access permissions for the MCU, the ++ * shader cores/tiler are not expected to access this memory ++ */ ++ if (flags & CSF_FIRMWARE_ENTRY_READ) ++ mem_flags |= KBASE_REG_GPU_RD; ++ ++ if (flags & CSF_FIRMWARE_ENTRY_WRITE) ++ mem_flags |= KBASE_REG_GPU_WR; ++ ++ if ((flags & CSF_FIRMWARE_ENTRY_EXECUTE) == 0) ++ mem_flags |= KBASE_REG_GPU_NX; ++ ++ if (flags & CSF_FIRMWARE_ENTRY_PROTECTED) ++ mem_flags |= KBASE_REG_PROTECTED; ++ ++ /* Substitute uncached coherent memory for cached coherent memory if ++ * the system does not support ACE coherency. ++ */ ++ if ((cache_mode == CSF_FIRMWARE_CACHE_MODE_CACHED_COHERENT) && ++ (kbdev->system_coherency != COHERENCY_ACE)) ++ cache_mode = CSF_FIRMWARE_CACHE_MODE_UNCACHED_COHERENT; ++ ++ /* Substitute uncached incoherent memory for uncached coherent memory ++ * if the system does not support ACE-Lite coherency. ++ */ ++ if ((cache_mode == CSF_FIRMWARE_CACHE_MODE_UNCACHED_COHERENT) && ++ (kbdev->system_coherency == COHERENCY_NONE)) ++ cache_mode = CSF_FIRMWARE_CACHE_MODE_NONE; ++ ++ *cm = cache_mode; ++ ++ switch (cache_mode) { ++ case CSF_FIRMWARE_CACHE_MODE_NONE: ++ mem_flags |= ++ KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_NON_CACHEABLE); ++ break; ++ case CSF_FIRMWARE_CACHE_MODE_CACHED: ++ mem_flags |= ++ KBASE_REG_MEMATTR_INDEX( ++ AS_MEMATTR_INDEX_IMPL_DEF_CACHE_POLICY); ++ break; ++ case CSF_FIRMWARE_CACHE_MODE_UNCACHED_COHERENT: ++ case CSF_FIRMWARE_CACHE_MODE_CACHED_COHERENT: ++ WARN_ON(!is_shared); ++ mem_flags |= KBASE_REG_SHARE_BOTH | ++ KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_SHARED); ++ break; ++ default: ++ dev_err(kbdev->dev, ++ "Firmware contains interface with unsupported cache mode\n"); ++ break; ++ } ++ return mem_flags; ++} ++ ++static void load_fw_image_section(struct kbase_device *kbdev, const u8 *data, ++ struct tagged_addr *phys, u32 num_pages, u32 flags, ++ u32 data_start, u32 data_end) ++{ ++ u32 data_pos = data_start; ++ u32 data_len = data_end - data_start; ++ u32 page_num; ++ u32 page_limit; ++ ++ if (flags & CSF_FIRMWARE_ENTRY_ZERO) ++ page_limit = num_pages; ++ else ++ page_limit = (data_len + PAGE_SIZE - 1) / PAGE_SIZE; ++ ++ for (page_num = 0; page_num < page_limit; ++page_num) { ++ struct page *const page = as_page(phys[page_num]); ++ char *const p = kmap_atomic(page); ++ u32 const copy_len = min_t(u32, PAGE_SIZE, data_len); ++ ++ if (copy_len > 0) { ++ memcpy(p, data + data_pos, copy_len); ++ data_pos += copy_len; ++ data_len -= copy_len; ++ } ++ ++ if (flags & CSF_FIRMWARE_ENTRY_ZERO) { ++ u32 const zi_len = PAGE_SIZE - copy_len; ++ ++ memset(p + copy_len, 0, zi_len); ++ } ++ ++ kbase_sync_single_for_device(kbdev, kbase_dma_addr(page), ++ PAGE_SIZE, DMA_TO_DEVICE); ++ kunmap_atomic(p); ++ } ++} ++ ++static int reload_fw_data_sections(struct kbase_device *kbdev) ++{ ++ const u32 magic = FIRMWARE_HEADER_MAGIC; ++ struct firmware_interface *interface; ++ const struct firmware *firmware; ++ int ret = 0; ++ ++ if (request_firmware(&firmware, fw_name, kbdev->dev) != 0) { ++ dev_err(kbdev->dev, ++ "Failed to reload firmware image '%s'\n", ++ fw_name); ++ return -ENOENT; ++ } ++ ++ /* Do couple of basic sanity checks */ ++ if (firmware->size < FIRMWARE_HEADER_LENGTH) { ++ dev_err(kbdev->dev, "Firmware image unexpectedly too small\n"); ++ ret = -EINVAL; ++ goto out; ++ } ++ ++ if (memcmp(firmware->data, &magic, sizeof(magic)) != 0) { ++ dev_err(kbdev->dev, "Incorrect magic value, firmware image could have been corrupted\n"); ++ ret = -EINVAL; ++ goto out; ++ } ++ ++ list_for_each_entry(interface, &kbdev->csf.firmware_interfaces, node) { ++ /* Skip reload of text & read only data sections */ ++ if ((interface->flags & CSF_FIRMWARE_ENTRY_EXECUTE) || ++ !(interface->flags & CSF_FIRMWARE_ENTRY_WRITE)) ++ continue; ++ ++ load_fw_image_section(kbdev, firmware->data, interface->phys, ++ interface->num_pages, interface->flags, ++ interface->data_start, interface->data_end); ++ } ++ ++ kbase_csf_firmware_reload_trace_buffers_data(kbdev); ++ ++out: ++ release_firmware(firmware); ++ return ret; ++} ++ ++/** ++ * parse_memory_setup_entry() - Process an "interface memory setup" section ++ * ++ * Read an "interface memory setup" section from the firmware image and create ++ * the necessary memory region including the MMU page tables. If successful ++ * the interface will be added to the kbase_device:csf.firmware_interfaces list. ++ * ++ * Return: 0 if successful, negative error code on failure ++ * ++ * @kbdev: Kbase device structure ++ * @fw: The firmware image containing the section ++ * @entry: Pointer to the start of the section ++ * @size: Size (in bytes) of the section ++ */ ++static int parse_memory_setup_entry(struct kbase_device *kbdev, ++ const struct firmware *fw, ++ const u32 *entry, unsigned int size) ++{ ++ int ret = 0; ++ const u32 flags = entry[0]; ++ const u32 virtual_start = entry[1]; ++ const u32 virtual_end = entry[2]; ++ const u32 data_start = entry[3]; ++ const u32 data_end = entry[4]; ++ u32 num_pages; ++ char *name; ++ struct tagged_addr *phys = NULL; ++ struct firmware_interface *interface = NULL; ++ bool allocated_pages = false, protected_mode = false; ++ unsigned long mem_flags = 0; ++ u32 cache_mode = 0; ++ struct protected_memory_allocation **pma = NULL; ++ ++ if (data_end < data_start) { ++ dev_err(kbdev->dev, "Firmware corrupt, data_end < data_start (0x%x<0x%x)\n", ++ data_end, data_start); ++ return -EINVAL; ++ } ++ if (virtual_end < virtual_start) { ++ dev_err(kbdev->dev, "Firmware corrupt, virtual_end < virtual_start (0x%x<0x%x)\n", ++ virtual_end, virtual_start); ++ return -EINVAL; ++ } ++ if (data_end > fw->size) { ++ dev_err(kbdev->dev, "Firmware corrupt, file truncated? data_end=0x%x > fw->size=0x%zx\n", ++ data_end, fw->size); ++ return -EINVAL; ++ } ++ ++ if ((virtual_start & ~PAGE_MASK) != 0 || ++ (virtual_end & ~PAGE_MASK) != 0) { ++ dev_err(kbdev->dev, "Firmware corrupt: virtual addresses not page aligned: 0x%x-0x%x\n", ++ virtual_start, virtual_end); ++ return -EINVAL; ++ } ++ ++ if ((flags & CSF_FIRMWARE_ENTRY_SUPPORTED_FLAGS) != flags) { ++ dev_err(kbdev->dev, "Firmware contains interface with unsupported flags (0x%x)\n", ++ flags); ++ return -EINVAL; ++ } ++ ++ if (flags & CSF_FIRMWARE_ENTRY_PROTECTED) ++ protected_mode = true; ++ ++ if (protected_mode && kbdev->csf.pma_dev == NULL) { ++ dev_err(kbdev->dev, ++ "Protected memory allocator not found, Firmware protected mode entry will not be supported"); ++ return 0; ++ } ++ ++ num_pages = (virtual_end - virtual_start) ++ >> PAGE_SHIFT; ++ ++ phys = kmalloc_array(num_pages, sizeof(*phys), GFP_KERNEL); ++ if (!phys) ++ return -ENOMEM; ++ ++ if (protected_mode) { ++ pma = kbase_csf_protected_memory_alloc(kbdev, phys, num_pages); ++ ++ if (pma == NULL) { ++ ret = -ENOMEM; ++ goto out; ++ } ++ } else { ++ ret = kbase_mem_pool_alloc_pages( ++ &kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], ++ num_pages, phys, false); ++ if (ret < 0) ++ goto out; ++ } ++ ++ allocated_pages = true; ++ load_fw_image_section(kbdev, fw->data, phys, num_pages, flags, ++ data_start, data_end); ++ ++ /* Allocate enough memory for the struct firmware_interface and ++ * the name of the interface. An extra byte is allocated to place a ++ * NUL-terminator in. This should already be included according to the ++ * specification but here we add it anyway to be robust against a ++ * corrupt firmware image. ++ */ ++ interface = kmalloc(sizeof(*interface) + ++ size - INTERFACE_ENTRY_NAME_OFFSET + 1, GFP_KERNEL); ++ if (!interface) { ++ ret = -ENOMEM; ++ goto out; ++ } ++ name = (void *)(interface + 1); ++ memcpy(name, entry + (INTERFACE_ENTRY_NAME_OFFSET / sizeof(*entry)), ++ size - INTERFACE_ENTRY_NAME_OFFSET); ++ name[size - INTERFACE_ENTRY_NAME_OFFSET] = 0; ++ ++ interface->name = name; ++ interface->phys = phys; ++ interface->num_pages = num_pages; ++ interface->virtual = virtual_start; ++ interface->kernel_map = NULL; ++ interface->flags = flags; ++ interface->data_start = data_start; ++ interface->data_end = data_end; ++ interface->pma = pma; ++ ++ mem_flags = convert_mem_flags(kbdev, flags, &cache_mode); ++ ++ if (flags & CSF_FIRMWARE_ENTRY_SHARED) { ++ struct page **page_list; ++ u32 i; ++ pgprot_t cpu_map_prot; ++ u32 mem_attr_index = KBASE_REG_MEMATTR_VALUE(mem_flags); ++ ++ /* Since SHARED memory type was used for mapping shared memory ++ * on GPU side, it can be mapped as cached on CPU side on both ++ * types of coherent platforms. ++ */ ++ if ((cache_mode == CSF_FIRMWARE_CACHE_MODE_CACHED_COHERENT) || ++ (cache_mode == CSF_FIRMWARE_CACHE_MODE_UNCACHED_COHERENT)) { ++ WARN_ON(mem_attr_index != ++ AS_MEMATTR_INDEX_SHARED); ++ cpu_map_prot = PAGE_KERNEL; ++ } else { ++ WARN_ON(mem_attr_index != ++ AS_MEMATTR_INDEX_NON_CACHEABLE); ++ cpu_map_prot = pgprot_writecombine(PAGE_KERNEL); ++ } ++ ++ page_list = kmalloc_array(num_pages, sizeof(*page_list), ++ GFP_KERNEL); ++ if (!page_list) { ++ ret = -ENOMEM; ++ goto out; ++ } ++ ++ for (i = 0; i < num_pages; i++) ++ page_list[i] = as_page(phys[i]); ++ ++ interface->kernel_map = vmap(page_list, num_pages, VM_MAP, ++ cpu_map_prot); ++ ++ kfree(page_list); ++ ++ if (!interface->kernel_map) { ++ ret = -ENOMEM; ++ goto out; ++ } ++ } ++ ++ /* Start location of the shared interface area is fixed and is ++ * specified in firmware spec, and so there shall only be a ++ * single entry with that start address. ++ */ ++ if (virtual_start == (KBASE_REG_ZONE_MCU_SHARED_BASE << PAGE_SHIFT)) ++ kbdev->csf.shared_interface = interface; ++ ++ list_add(&interface->node, &kbdev->csf.firmware_interfaces); ++ ++ ret = kbase_mmu_insert_pages_no_flush(kbdev, &kbdev->csf.mcu_mmu, ++ virtual_start >> PAGE_SHIFT, phys, num_pages, mem_flags, ++ KBASE_MEM_GROUP_CSF_FW); ++ ++ if (ret != 0) { ++ dev_err(kbdev->dev, "Failed to insert firmware pages\n"); ++ /* The interface has been added to the list, so cleanup will ++ * be handled by firmware unloading ++ */ ++ } ++ ++ dev_dbg(kbdev->dev, "Processed section '%s'", name); ++ ++ return ret; ++ ++out: ++ if (allocated_pages) { ++ if (protected_mode) { ++ if (interface) { ++ kbase_csf_protected_memory_free(kbdev, ++ interface->pma, num_pages); ++ } ++ } else { ++ kbase_mem_pool_free_pages( ++ &kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], ++ num_pages, phys, false, false); ++ } ++ } ++ ++ kfree(phys); ++ kfree(interface); ++ return ret; ++} ++ ++/** ++ * parse_timeline_metadata_entry() - Process a "timeline metadata" section ++ * ++ * Return: 0 if successful, negative error code on failure ++ * ++ * @kbdev: Kbase device structure ++ * @fw: Firmware image containing the section ++ * @entry: Pointer to the section ++ * @size: Size (in bytes) of the section ++ */ ++static int parse_timeline_metadata_entry(struct kbase_device *kbdev, ++ const struct firmware *fw, const u32 *entry, unsigned int size) ++{ ++ const u32 data_start = entry[0]; ++ const u32 data_size = entry[1]; ++ const u32 data_end = data_start + data_size; ++ const char *name = (char *)&entry[2]; ++ struct firmware_timeline_metadata *metadata; ++ const unsigned int name_len = ++ size - TL_METADATA_ENTRY_NAME_OFFSET; ++ size_t allocation_size = sizeof(*metadata) + name_len + 1 + data_size; ++ ++ if (data_end > fw->size) { ++ dev_err(kbdev->dev, ++ "Firmware corrupt, file truncated? data_end=0x%x > fw->size=0x%zx", ++ data_end, fw->size); ++ return -EINVAL; ++ } ++ ++ /* Allocate enough space for firmware_timeline_metadata, ++ * its name and the content. ++ */ ++ metadata = kmalloc(allocation_size, GFP_KERNEL); ++ if (!metadata) ++ return -ENOMEM; ++ ++ metadata->name = (char *)(metadata + 1); ++ metadata->data = (char *)(metadata + 1) + name_len + 1; ++ metadata->size = data_size; ++ ++ memcpy(metadata->name, name, name_len); ++ metadata->name[name_len] = 0; ++ ++ /* Copy metadata's content. */ ++ memcpy(metadata->data, fw->data + data_start, data_size); ++ ++ list_add(&metadata->node, &kbdev->csf.firmware_timeline_metadata); ++ ++ dev_dbg(kbdev->dev, "Timeline metadata '%s'", metadata->name); ++ ++ return 0; ++} ++ ++/** ++ * load_firmware_entry() - Process an entry from a firmware image ++ * ++ * Read an entry from a firmware image and do any necessary work (e.g. loading ++ * the data into page accessible to the MCU). ++ * ++ * Unknown entries are ignored if the 'optional' flag is set within the entry, ++ * otherwise the function will fail with -EINVAL ++ * ++ * Return: 0 if successful, negative error code on failure ++ * ++ * @kbdev: Kbase device ++ * @fw: Firmware image containing the entry ++ * @offset: Byte offset within the image of the entry to load ++ * @header: Header word of the entry ++ */ ++static int load_firmware_entry(struct kbase_device *kbdev, ++ const struct firmware *fw, ++ u32 offset, u32 header) ++{ ++ const unsigned int type = entry_type(header); ++ unsigned int size = entry_size(header); ++ const bool optional = entry_optional(header); ++ const u32 *entry = (void *)(fw->data + offset); ++ ++ if ((offset % sizeof(*entry)) || (size % sizeof(*entry))) { ++ dev_err(kbdev->dev, "Firmware entry isn't 32 bit aligned, offset=0x%x size=0x%x\n", ++ offset, size); ++ return -EINVAL; ++ } ++ ++ if (size < sizeof(*entry)) { ++ dev_err(kbdev->dev, "Size field too small: %u\n", size); ++ return -EINVAL; ++ } ++ ++ /* Remove the header */ ++ entry++; ++ size -= sizeof(*entry); ++ ++ switch (type) { ++ case CSF_FIRMWARE_ENTRY_TYPE_INTERFACE: ++ /* Interface memory setup */ ++ if (size < INTERFACE_ENTRY_NAME_OFFSET + sizeof(*entry)) { ++ dev_err(kbdev->dev, "Interface memory setup entry too short (size=%u)\n", ++ size); ++ return -EINVAL; ++ } ++ return parse_memory_setup_entry(kbdev, fw, entry, size); ++ case CSF_FIRMWARE_ENTRY_TYPE_CONFIGURATION: ++ /* Configuration option */ ++ if (size < CONFIGURATION_ENTRY_NAME_OFFSET + sizeof(*entry)) { ++ dev_err(kbdev->dev, "Configuration option entry too short (size=%u)\n", ++ size); ++ return -EINVAL; ++ } ++ return kbase_csf_firmware_cfg_option_entry_parse( ++ kbdev, fw, entry, size); ++ case CSF_FIRMWARE_ENTRY_TYPE_FUTF_TEST: ++#ifndef MALI_KBASE_BUILD ++ /* FW UTF option */ ++ if (size < 2*sizeof(*entry)) { ++ dev_err(kbdev->dev, "FW UTF entry too short (size=%u)\n", ++ size); ++ return -EINVAL; ++ } ++ return mali_kutf_process_fw_utf_entry(kbdev, fw->data, ++ fw->size, entry); ++#endif ++ break; ++ case CSF_FIRMWARE_ENTRY_TYPE_TRACE_BUFFER: ++ /* Trace buffer */ ++ if (size < TRACE_BUFFER_ENTRY_NAME_OFFSET + sizeof(*entry)) { ++ dev_err(kbdev->dev, "Trace Buffer entry too short (size=%u)\n", ++ size); ++ return -EINVAL; ++ } ++ return kbase_csf_firmware_parse_trace_buffer_entry( ++ kbdev, entry, size); ++ case CSF_FIRMWARE_ENTRY_TYPE_TIMELINE_METADATA: ++ /* Meta data section */ ++ if (size < TL_METADATA_ENTRY_NAME_OFFSET + sizeof(*entry)) { ++ dev_err(kbdev->dev, "Timeline metadata entry too short (size=%u)\n", ++ size); ++ return -EINVAL; ++ } ++ return parse_timeline_metadata_entry(kbdev, fw, entry, size); ++ } ++ ++ if (!optional) { ++ dev_err(kbdev->dev, ++ "Unsupported non-optional entry type %u in firmware\n", ++ type); ++ return -EINVAL; ++ } ++ ++ return 0; ++} ++ ++static void free_global_iface(struct kbase_device *kbdev) ++{ ++ struct kbase_csf_global_iface *iface = &kbdev->csf.global_iface; ++ ++ if (iface->groups) { ++ unsigned int gid; ++ ++ for (gid = 0; gid < iface->group_num; ++gid) ++ kfree(iface->groups[gid].streams); ++ ++ kfree(iface->groups); ++ iface->groups = NULL; ++ } ++} ++ ++/** ++ * iface_gpu_va_to_cpu - Convert a GPU VA address within the shared interface ++ * region to a CPU address, using the existing mapping. ++ * @kbdev: Device pointer ++ * @gpu_va: GPU VA to convert ++ * ++ * Return: A CPU pointer to the location within the shared interface region, or ++ * NULL on failure. ++ */ ++static inline void *iface_gpu_va_to_cpu(struct kbase_device *kbdev, u32 gpu_va) ++{ ++ struct firmware_interface *interface = kbdev->csf.shared_interface; ++ u8 *kernel_base = interface->kernel_map; ++ ++ if (gpu_va < interface->virtual || ++ gpu_va >= interface->virtual + interface->num_pages * PAGE_SIZE) { ++ dev_err(kbdev->dev, ++ "Interface address 0x%x not within %u-page region at 0x%x", ++ gpu_va, interface->num_pages, ++ interface->virtual); ++ return NULL; ++ } ++ ++ return (void *)(kernel_base + (gpu_va - interface->virtual)); ++} ++ ++static int parse_cmd_stream_info(struct kbase_device *kbdev, ++ struct kbase_csf_cmd_stream_info *sinfo, ++ u32 *stream_base) ++{ ++ sinfo->kbdev = kbdev; ++ sinfo->features = stream_base[STREAM_FEATURES/4]; ++ sinfo->input = iface_gpu_va_to_cpu(kbdev, ++ stream_base[STREAM_INPUT_VA/4]); ++ sinfo->output = iface_gpu_va_to_cpu(kbdev, ++ stream_base[STREAM_OUTPUT_VA/4]); ++ ++ if (sinfo->input == NULL || sinfo->output == NULL) ++ return -EINVAL; ++ ++ return 0; ++} ++ ++static int parse_cmd_stream_group_info(struct kbase_device *kbdev, ++ struct kbase_csf_cmd_stream_group_info *ginfo, ++ u32 *group_base, u32 group_stride) ++{ ++ unsigned int sid; ++ ++ ginfo->kbdev = kbdev; ++ ginfo->features = group_base[GROUP_FEATURES/4]; ++ ginfo->input = iface_gpu_va_to_cpu(kbdev, ++ group_base[GROUP_INPUT_VA/4]); ++ ginfo->output = iface_gpu_va_to_cpu(kbdev, ++ group_base[GROUP_OUTPUT_VA/4]); ++ ++ if (ginfo->input == NULL || ginfo->output == NULL) ++ return -ENOMEM; ++ ++ ginfo->suspend_size = group_base[GROUP_SUSPEND_SIZE/4]; ++ ginfo->protm_suspend_size = group_base[GROUP_PROTM_SUSPEND_SIZE/4]; ++ ginfo->stream_num = group_base[GROUP_STREAM_NUM/4]; ++ ++ if (ginfo->stream_num < MIN_SUPPORTED_STREAMS_PER_GROUP || ++ ginfo->stream_num > MAX_SUPPORTED_STREAMS_PER_GROUP) { ++ dev_err(kbdev->dev, "CSG with %u streams out of range %u-%u", ++ ginfo->stream_num, ++ MIN_SUPPORTED_STREAMS_PER_GROUP, ++ MAX_SUPPORTED_STREAMS_PER_GROUP); ++ return -EINVAL; ++ } ++ ++ ginfo->stream_stride = group_base[GROUP_STREAM_STRIDE/4]; ++ ++ if (ginfo->stream_num * ginfo->stream_stride > group_stride) { ++ dev_err(kbdev->dev, ++ "group stride of 0x%x exceeded by %u streams with stride 0x%x", ++ group_stride, ginfo->stream_num, ++ ginfo->stream_stride); ++ return -EINVAL; ++ } ++ ++ ginfo->streams = kmalloc_array(ginfo->stream_num, ++ sizeof(*ginfo->streams), GFP_KERNEL); ++ ++ if (!ginfo->streams) ++ return -ENOMEM; ++ ++ for (sid = 0; sid < ginfo->stream_num; sid++) { ++ int err; ++ u32 *stream_base = group_base + (STREAM_CONTROL_0 + ++ ginfo->stream_stride * sid) / 4; ++ ++ err = parse_cmd_stream_info(kbdev, &ginfo->streams[sid], ++ stream_base); ++ if (err < 0) { ++ /* caller will free the memory for streams array */ ++ return err; ++ } ++ } ++ ++ return 0; ++} ++ ++static u32 get_firmware_version(struct kbase_device *kbdev) ++{ ++ struct firmware_interface *interface = kbdev->csf.shared_interface; ++ u32 *shared_info = interface->kernel_map; ++ ++ return shared_info[GLB_VERSION/4]; ++} ++ ++static int parse_capabilities(struct kbase_device *kbdev) ++{ ++ struct firmware_interface *interface = kbdev->csf.shared_interface; ++ u32 *shared_info = interface->kernel_map; ++ struct kbase_csf_global_iface *iface = &kbdev->csf.global_iface; ++ unsigned int gid; ++ ++ /* All offsets are in bytes, so divide by 4 for access via a u32 pointer ++ */ ++ ++ /* The version number of the global interface is expected to be a ++ * non-zero value. If it's not, the firmware may not have booted. ++ */ ++ iface->version = get_firmware_version(kbdev); ++ if (!iface->version) { ++ dev_err(kbdev->dev, "Version check failed. Firmware may have failed to boot."); ++ return -EINVAL; ++ } ++ ++ ++ iface->kbdev = kbdev; ++ iface->features = shared_info[GLB_FEATURES/4]; ++ iface->input = iface_gpu_va_to_cpu(kbdev, shared_info[GLB_INPUT_VA/4]); ++ iface->output = iface_gpu_va_to_cpu(kbdev, ++ shared_info[GLB_OUTPUT_VA/4]); ++ ++ if (iface->input == NULL || iface->output == NULL) ++ return -ENOMEM; ++ ++ iface->group_num = shared_info[GLB_GROUP_NUM/4]; ++ ++ if (iface->group_num < MIN_SUPPORTED_CSGS || ++ iface->group_num > MAX_SUPPORTED_CSGS) { ++ dev_err(kbdev->dev, ++ "Interface containing %u CSGs outside of range %u-%u", ++ iface->group_num, MIN_SUPPORTED_CSGS, ++ MAX_SUPPORTED_CSGS); ++ return -EINVAL; ++ } ++ ++ iface->group_stride = shared_info[GLB_GROUP_STRIDE/4]; ++ iface->prfcnt_size = shared_info[GLB_PRFCNT_SIZE/4]; ++ ++ if ((GROUP_CONTROL_0 + ++ (unsigned long)iface->group_num * iface->group_stride) > ++ (interface->num_pages * PAGE_SIZE)) { ++ dev_err(kbdev->dev, ++ "interface size of %u pages exceeded by %u CSGs with stride 0x%x", ++ interface->num_pages, iface->group_num, ++ iface->group_stride); ++ return -EINVAL; ++ } ++ ++ WARN_ON(iface->groups); ++ ++ iface->groups = kcalloc(iface->group_num, sizeof(*iface->groups), ++ GFP_KERNEL); ++ if (!iface->groups) ++ return -ENOMEM; ++ ++ for (gid = 0; gid < iface->group_num; gid++) { ++ int err; ++ u32 *group_base = shared_info + (GROUP_CONTROL_0 + ++ iface->group_stride * gid) / 4; ++ ++ err = parse_cmd_stream_group_info(kbdev, &iface->groups[gid], ++ group_base, iface->group_stride); ++ if (err < 0) { ++ free_global_iface(kbdev); ++ return err; ++ } ++ } ++ ++ return 0; ++} ++ ++static inline void access_firmware_memory(struct kbase_device *kbdev, ++ u32 gpu_addr, u32 *value, const bool read) ++{ ++ struct firmware_interface *interface; ++ ++ list_for_each_entry(interface, &kbdev->csf.firmware_interfaces, node) { ++ if ((gpu_addr >= interface->virtual) && ++ (gpu_addr < interface->virtual + (interface->num_pages << PAGE_SHIFT))) { ++ u32 offset_bytes = gpu_addr - interface->virtual; ++ u32 page_num = offset_bytes >> PAGE_SHIFT; ++ u32 offset_in_page = offset_bytes & ~PAGE_MASK; ++ struct page *target_page = as_page( ++ interface->phys[page_num]); ++ u32 *cpu_addr = kmap_atomic(target_page); ++ ++ if (read) { ++ kbase_sync_single_for_device(kbdev, ++ kbase_dma_addr(target_page) + offset_in_page, ++ sizeof(u32), DMA_BIDIRECTIONAL); ++ ++ *value = cpu_addr[offset_in_page >> 2]; ++ } else { ++ cpu_addr[offset_in_page >> 2] = *value; ++ ++ kbase_sync_single_for_device(kbdev, ++ kbase_dma_addr(target_page) + offset_in_page, ++ sizeof(u32), DMA_BIDIRECTIONAL); ++ } ++ ++ kunmap_atomic(cpu_addr); ++ return; ++ } ++ } ++ dev_warn(kbdev->dev, "Invalid GPU VA %x passed\n", gpu_addr); ++} ++ ++void kbase_csf_read_firmware_memory(struct kbase_device *kbdev, ++ u32 gpu_addr, u32 *value) ++{ ++ access_firmware_memory(kbdev, gpu_addr, value, true); ++} ++ ++void kbase_csf_update_firmware_memory(struct kbase_device *kbdev, ++ u32 gpu_addr, u32 value) ++{ ++ access_firmware_memory(kbdev, gpu_addr, &value, false); ++} ++ ++void kbase_csf_firmware_cs_input( ++ const struct kbase_csf_cmd_stream_info *const info, const u32 offset, ++ const u32 value) ++{ ++ const struct kbase_device * const kbdev = info->kbdev; ++ ++ dev_dbg(kbdev->dev, "cs input w: reg %08x val %08x\n", offset, value); ++ input_page_write(info->input, offset, value); ++} ++ ++u32 kbase_csf_firmware_cs_input_read( ++ const struct kbase_csf_cmd_stream_info *const info, ++ const u32 offset) ++{ ++ const struct kbase_device * const kbdev = info->kbdev; ++ u32 const val = input_page_read(info->input, offset); ++ ++ dev_dbg(kbdev->dev, "cs input r: reg %08x val %08x\n", offset, val); ++ return val; ++} ++ ++void kbase_csf_firmware_cs_input_mask( ++ const struct kbase_csf_cmd_stream_info *const info, const u32 offset, ++ const u32 value, const u32 mask) ++{ ++ const struct kbase_device * const kbdev = info->kbdev; ++ ++ dev_dbg(kbdev->dev, "cs input w: reg %08x val %08x mask %08x\n", ++ offset, value, mask); ++ input_page_partial_write(info->input, offset, value, mask); ++} ++ ++u32 kbase_csf_firmware_cs_output( ++ const struct kbase_csf_cmd_stream_info *const info, const u32 offset) ++{ ++ const struct kbase_device * const kbdev = info->kbdev; ++ u32 const val = output_page_read(info->output, offset); ++ ++ dev_dbg(kbdev->dev, "cs output r: reg %08x val %08x\n", offset, val); ++ return val; ++} ++ ++void kbase_csf_firmware_csg_input( ++ const struct kbase_csf_cmd_stream_group_info *const info, ++ const u32 offset, const u32 value) ++{ ++ const struct kbase_device * const kbdev = info->kbdev; ++ ++ dev_dbg(kbdev->dev, "csg input w: reg %08x val %08x\n", ++ offset, value); ++ input_page_write(info->input, offset, value); ++} ++ ++u32 kbase_csf_firmware_csg_input_read( ++ const struct kbase_csf_cmd_stream_group_info *const info, ++ const u32 offset) ++{ ++ const struct kbase_device * const kbdev = info->kbdev; ++ u32 const val = input_page_read(info->input, offset); ++ ++ dev_dbg(kbdev->dev, "csg input r: reg %08x val %08x\n", offset, val); ++ return val; ++} ++ ++void kbase_csf_firmware_csg_input_mask( ++ const struct kbase_csf_cmd_stream_group_info *const info, ++ const u32 offset, const u32 value, const u32 mask) ++{ ++ const struct kbase_device * const kbdev = info->kbdev; ++ ++ dev_dbg(kbdev->dev, "csg input w: reg %08x val %08x mask %08x\n", ++ offset, value, mask); ++ input_page_partial_write(info->input, offset, value, mask); ++} ++ ++u32 kbase_csf_firmware_csg_output( ++ const struct kbase_csf_cmd_stream_group_info *const info, ++ const u32 offset) ++{ ++ const struct kbase_device * const kbdev = info->kbdev; ++ u32 const val = output_page_read(info->output, offset); ++ ++ dev_dbg(kbdev->dev, "csg output r: reg %08x val %08x\n", offset, val); ++ return val; ++} ++ ++void kbase_csf_firmware_global_input( ++ const struct kbase_csf_global_iface *const iface, const u32 offset, ++ const u32 value) ++{ ++ const struct kbase_device * const kbdev = iface->kbdev; ++ ++ dev_dbg(kbdev->dev, "glob input w: reg %08x val %08x\n", offset, value); ++ input_page_write(iface->input, offset, value); ++} ++ ++void kbase_csf_firmware_global_input_mask( ++ const struct kbase_csf_global_iface *const iface, const u32 offset, ++ const u32 value, const u32 mask) ++{ ++ const struct kbase_device * const kbdev = iface->kbdev; ++ ++ dev_dbg(kbdev->dev, "glob input w: reg %08x val %08x mask %08x\n", ++ offset, value, mask); ++ input_page_partial_write(iface->input, offset, value, mask); ++} ++ ++u32 kbase_csf_firmware_global_input_read( ++ const struct kbase_csf_global_iface *const iface, const u32 offset) ++{ ++ const struct kbase_device * const kbdev = iface->kbdev; ++ u32 const val = input_page_read(iface->input, offset); ++ ++ dev_dbg(kbdev->dev, "glob input r: reg %08x val %08x\n", offset, val); ++ return val; ++} ++ ++u32 kbase_csf_firmware_global_output( ++ const struct kbase_csf_global_iface *const iface, const u32 offset) ++{ ++ const struct kbase_device * const kbdev = iface->kbdev; ++ u32 const val = output_page_read(iface->output, offset); ++ ++ dev_dbg(kbdev->dev, "glob output r: reg %08x val %08x\n", offset, val); ++ return val; ++} ++ ++static bool global_request_complete(struct kbase_device *const kbdev, ++ u32 const req_mask) ++{ ++ struct kbase_csf_global_iface *global_iface = ++ &kbdev->csf.global_iface; ++ bool complete = false; ++ unsigned long flags; ++ ++ kbase_csf_scheduler_spin_lock(kbdev, &flags); ++ ++ if ((kbase_csf_firmware_global_output(global_iface, GLB_ACK) & ++ req_mask) == ++ (kbase_csf_firmware_global_input_read(global_iface, GLB_REQ) & ++ req_mask)) ++ complete = true; ++ ++ kbase_csf_scheduler_spin_unlock(kbdev, flags); ++ ++ return complete; ++} ++ ++static int wait_for_global_request(struct kbase_device *const kbdev, ++ u32 const req_mask) ++{ ++ const long wait_timeout = ++ kbase_csf_timeout_in_jiffies(GLB_REQ_WAIT_TIMEOUT_MS); ++ long remaining; ++ int err = 0; ++ ++ remaining = wait_event_timeout(kbdev->csf.event_wait, ++ global_request_complete(kbdev, req_mask), ++ wait_timeout); ++ ++ if (!remaining) { ++ dev_warn(kbdev->dev, "Timed out waiting for global request %x to complete", ++ req_mask); ++ err = -ETIMEDOUT; ++ } ++ ++ return err; ++} ++ ++static void set_global_request( ++ const struct kbase_csf_global_iface *const global_iface, ++ u32 const req_mask) ++{ ++ u32 glb_req; ++ ++ kbase_csf_scheduler_spin_lock_assert_held(global_iface->kbdev); ++ ++ glb_req = kbase_csf_firmware_global_output(global_iface, GLB_ACK); ++ glb_req ^= req_mask; ++ kbase_csf_firmware_global_input_mask(global_iface, GLB_REQ, glb_req, ++ req_mask); ++} ++ ++static void enable_endpoints_global( ++ const struct kbase_csf_global_iface *const global_iface, ++ u64 const shader_core_mask) ++{ ++ kbase_csf_firmware_global_input(global_iface, GLB_ALLOC_EN_LO, ++ shader_core_mask & U32_MAX); ++ kbase_csf_firmware_global_input(global_iface, GLB_ALLOC_EN_HI, ++ shader_core_mask >> 32); ++ ++ set_global_request(global_iface, GLB_REQ_CFG_ALLOC_EN_MASK); ++} ++ ++static void set_timeout_global( ++ const struct kbase_csf_global_iface *const global_iface, ++ u64 const timeout) ++{ ++ kbase_csf_firmware_global_input(global_iface, GLB_PROGRESS_TIMER, ++ timeout / GLB_PROGRESS_TIMER_TIMEOUT_SCALE); ++ ++ set_global_request(global_iface, GLB_REQ_CFG_PROGRESS_TIMER_MASK); ++} ++ ++static void set_coherency_mode(struct kbase_device *const kbdev) ++{ ++ const struct kbase_csf_global_iface *const global_iface = ++ &kbdev->csf.global_iface; ++ u32 protected_mode_coherency = kbdev->system_coherency; ++ ++ /* GPU is supposed to use ACE-Lite coherency mode on a fully coherent ++ * system during protected mode execution. ++ */ ++ if (kbdev->system_coherency == COHERENCY_ACE) ++ protected_mode_coherency = COHERENCY_ACE_LITE; ++ ++ kbase_csf_firmware_global_input(global_iface, GLB_PROTM_COHERENCY, ++ protected_mode_coherency); ++} ++ ++static void global_init(struct kbase_device *const kbdev, u32 req_mask) ++{ ++ u32 const ack_irq_mask = GLB_ACK_IRQ_MASK_CFG_ALLOC_EN_MASK | ++ GLB_ACK_IRQ_MASK_PING_MASK | ++ GLB_ACK_IRQ_MASK_CFG_PROGRESS_TIMER_MASK | ++ GLB_ACK_IRQ_MASK_PROTM_ENTER_MASK | ++ GLB_ACK_IRQ_MASK_PROTM_EXIT_MASK; ++ ++ const struct kbase_csf_global_iface *const global_iface = ++ &kbdev->csf.global_iface; ++ unsigned long flags; ++ ++ kbase_csf_scheduler_spin_lock(kbdev, &flags); ++ ++ /* Set the cohereny mode for protected mode execution */ ++ set_coherency_mode(kbdev); ++ ++ /* Enable endpoints on all present shader cores */ ++ enable_endpoints_global(global_iface, ++ kbase_pm_get_present_cores(kbdev, KBASE_PM_CORE_SHADER)); ++ ++ set_timeout_global(global_iface, kbase_csf_timeout_get(kbdev)); ++ ++ /* Unmask the interrupts */ ++ kbase_csf_firmware_global_input(global_iface, ++ GLB_ACK_IRQ_MASK, ack_irq_mask); ++ ++ kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); ++ ++ kbase_csf_scheduler_spin_unlock(kbdev, flags); ++} ++ ++/** ++ * global_init_on_boot - Sends a global request to control various features. ++ * ++ * @kbdev: Instance of a GPU platform device that implements a command ++ * stream front-end interface. ++ * ++ * Currently only the request to enable endpoints and timeout for GPU progress ++ * timer is sent. ++ * ++ * Return: 0 on success, or negative on failure. ++ */ ++static int global_init_on_boot(struct kbase_device *const kbdev) ++{ ++ u32 const req_mask = CSF_GLB_REQ_CFG_MASK; ++ ++ global_init(kbdev, req_mask); ++ ++ return wait_for_global_request(kbdev, req_mask); ++} ++ ++void kbase_csf_firmware_global_reinit(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ kbdev->csf.glb_init_request_pending = true; ++ global_init(kbdev, CSF_GLB_REQ_CFG_MASK); ++} ++ ++bool kbase_csf_firmware_global_reinit_complete(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ WARN_ON(!kbdev->csf.glb_init_request_pending); ++ ++ if (global_request_complete(kbdev, CSF_GLB_REQ_CFG_MASK)) ++ kbdev->csf.glb_init_request_pending = false; ++ ++ return !kbdev->csf.glb_init_request_pending; ++} ++ ++/** ++ * This helper function will reload the firmware image and re-enable the MCU. ++ * It is supposed to be called after MCU(GPU) has been reset. ++ * Unlike the initial boot the firmware binary image is not parsed completely. ++ * Only the data sections, which were loaded in memory during the initial boot, ++ * are re-initialized either by zeroing them or copying their data from the ++ * firmware binary image. The memory allocation for the firmware pages and ++ * MMU programming is not needed for the reboot, presuming the firmware binary ++ * file on the filesystem would not change. ++ */ ++static void kbase_csf_firmware_reload_worker(struct work_struct *work) ++{ ++ struct kbase_device *kbdev = container_of(work, struct kbase_device, ++ csf.firmware_reload_work); ++ int err; ++ ++ dev_info(kbdev->dev, "reloading firmware"); ++ ++ /* Reload just the data sections from firmware binary image */ ++ err = reload_fw_data_sections(kbdev); ++ if (err) ++ return; ++ ++ kbase_csf_tl_reader_reset(&kbdev->timeline->csf_tl_reader); ++ ++ /* Reboot the firmware */ ++ kbase_csf_firmware_enable_mcu(kbdev); ++} ++ ++void kbase_csf_firmware_trigger_reload(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ kbdev->csf.firmware_reloaded = false; ++ ++ if (kbdev->csf.firmware_reload_needed) { ++ kbdev->csf.firmware_reload_needed = false; ++ queue_work(system_wq, &kbdev->csf.firmware_reload_work); ++ } else { ++ kbase_csf_firmware_enable_mcu(kbdev); ++ } ++} ++ ++void kbase_csf_firmware_reload_completed(struct kbase_device *kbdev) ++{ ++ u32 version; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (unlikely(!kbdev->csf.firmware_inited)) ++ return; ++ ++ /* Check firmware rebooted properly: we do not expect ++ * the version number to change with a running reboot. ++ */ ++ version = get_firmware_version(kbdev); ++ ++ if (version != kbdev->csf.global_iface.version) ++ dev_err(kbdev->dev, "Version check failed in firmware reboot."); ++ ++ KBASE_KTRACE_ADD(kbdev, FIRMWARE_REBOOT, NULL, 0u); ++ ++ /* Tell MCU state machine to transit to next state */ ++ kbdev->csf.firmware_reloaded = true; ++ kbase_pm_update_state(kbdev); ++} ++ ++int kbase_csf_firmware_init(struct kbase_device *kbdev) ++{ ++ const struct firmware *firmware; ++ const u32 magic = FIRMWARE_HEADER_MAGIC; ++ u8 version_major, version_minor; ++ u32 version_hash; ++ u32 entry_end_offset; ++ u32 entry_offset; ++ int ret; ++ ++ if (WARN_ON((kbdev->as_free & MCU_AS_BITMASK) == 0)) ++ return -EINVAL; ++ kbdev->as_free &= ~MCU_AS_BITMASK; ++ ++ ret = kbase_mmu_init(kbdev, &kbdev->csf.mcu_mmu, NULL, ++ BASE_MEM_GROUP_DEFAULT); ++ ++ if (ret != 0) { ++ /* Release the address space */ ++ kbdev->as_free |= MCU_AS_BITMASK; ++ return ret; ++ } ++ ++ init_waitqueue_head(&kbdev->csf.event_wait); ++ kbdev->csf.interrupt_received = false; ++ ++ INIT_LIST_HEAD(&kbdev->csf.firmware_interfaces); ++ INIT_LIST_HEAD(&kbdev->csf.firmware_config); ++ INIT_LIST_HEAD(&kbdev->csf.firmware_timeline_metadata); ++ INIT_LIST_HEAD(&kbdev->csf.firmware_trace_buffers.list); ++ INIT_WORK(&kbdev->csf.firmware_reload_work, ++ kbase_csf_firmware_reload_worker); ++ ++ mutex_init(&kbdev->csf.reg_lock); ++ ++ ret = kbase_mcu_shared_interface_region_tracker_init(kbdev); ++ if (ret != 0) { ++ dev_err(kbdev->dev, "Failed to setup the rb tree for managing shared interface segment\n"); ++ goto error; ++ } ++ ++ if (request_firmware(&firmware, fw_name, kbdev->dev) != 0) { ++ dev_err(kbdev->dev, ++ "Failed to load firmware image '%s'\n", ++ fw_name); ++ ret = -ENOENT; ++ goto error; ++ } ++ ++ if (firmware->size < FIRMWARE_HEADER_LENGTH) { ++ dev_err(kbdev->dev, "Firmware too small\n"); ++ ret = -EINVAL; ++ goto error; ++ } ++ ++ if (memcmp(firmware->data, &magic, sizeof(magic)) != 0) { ++ dev_err(kbdev->dev, "Incorrect firmware magic\n"); ++ ret = -EINVAL; ++ goto error; ++ } ++ ++ version_major = firmware->data[4]; ++ version_minor = firmware->data[5]; ++ ++ if (version_major != FIRMWARE_HEADER_VERSION) { ++ dev_err(kbdev->dev, ++ "Firmware header version %d.%d not understood\n", ++ version_major, version_minor); ++ ret = -EINVAL; ++ goto error; ++ } ++ ++ memcpy(&version_hash, &firmware->data[8], sizeof(version_hash)); ++ ++ dev_notice(kbdev->dev, "Loading Mali firmware 0x%x", version_hash); ++ ++ memcpy(&entry_end_offset, &firmware->data[0x10], ++ sizeof(entry_end_offset)); ++ ++ if (entry_end_offset > firmware->size) { ++ dev_err(kbdev->dev, "Firmware image is truncated\n"); ++ ret = -EINVAL; ++ goto error; ++ } ++ ++ entry_offset = FIRMWARE_HEADER_LENGTH; ++ while (entry_offset < entry_end_offset) { ++ u32 header; ++ unsigned int size; ++ ++ memcpy(&header, &firmware->data[entry_offset], sizeof(header)); ++ ++ size = entry_size(header); ++ ++ ret = load_firmware_entry(kbdev, firmware, entry_offset, ++ header); ++ if (ret != 0) { ++ dev_err(kbdev->dev, "Failed to load firmware image\n"); ++ goto error; ++ } ++ entry_offset += size; ++ } ++ ++ if (!kbdev->csf.shared_interface) { ++ dev_err(kbdev->dev, "Shared interface region not found\n"); ++ ret = -EINVAL; ++ goto error; ++ } else { ++ ret = setup_shared_iface_static_region(kbdev); ++ if (ret != 0) { ++ dev_err(kbdev->dev, "Failed to insert a region for shared iface entry parsed from fw image\n"); ++ goto error; ++ } ++ } ++ ++ ret = kbase_csf_firmware_trace_buffers_init(kbdev); ++ if (ret != 0) { ++ dev_err(kbdev->dev, "Failed to initialize trace buffers\n"); ++ goto error; ++ } ++ ++ /* Make sure L2 cache is powered up */ ++ kbase_pm_wait_for_l2_powered(kbdev); ++ ++ /* Load the MMU tables into the selected address space */ ++ load_mmu_tables(kbdev); ++ ++ boot_csf_firmware(kbdev); ++ ++ ret = parse_capabilities(kbdev); ++ if (ret != 0) ++ goto error; ++ ++ ret = kbase_csf_doorbell_mapping_init(kbdev); ++ if (ret != 0) ++ goto error; ++ ++ ret = kbase_csf_scheduler_init(kbdev); ++ if (ret != 0) ++ goto error; ++ ++ ret = kbase_csf_timeout_init(kbdev); ++ if (ret != 0) ++ goto error; ++ ++ ret = global_init_on_boot(kbdev); ++ if (ret != 0) ++ goto error; ++ ++ ret = kbase_csf_firmware_cfg_init(kbdev); ++ if (ret != 0) ++ goto error; ++ ++ /* Firmware loaded successfully */ ++ release_firmware(firmware); ++ KBASE_KTRACE_ADD(kbdev, FIRMWARE_BOOT, NULL, ++ (((u64)version_hash) << 32) | ++ (((u64)version_major) << 8) | version_minor); ++ return 0; ++ ++error: ++ kbase_csf_firmware_term(kbdev); ++ release_firmware(firmware); ++ return ret; ++} ++ ++void kbase_csf_firmware_term(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ int ret = 0; ++ ++ while (kbase_reset_gpu_is_active(kbdev) && !ret) ++ ret = kbase_reset_gpu_wait(kbdev); ++ ++ WARN(ret, "failed to wait for GPU reset"); ++ ++ /* Make sure ongoing transitions have completed */ ++ kbase_pm_wait_for_desired_state(kbdev); ++ ++ kbase_csf_firmware_cfg_term(kbdev); ++ ++ kbase_csf_timeout_term(kbdev); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbdev->csf.firmware_inited = false; ++ if (kbdev->pm.backend.mcu_state != KBASE_MCU_OFF) { ++ kbdev->pm.backend.mcu_state = KBASE_MCU_OFF; ++ stop_csf_firmware(kbdev); ++ } ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ unload_mmu_tables(kbdev); ++ ++ kbase_mmu_term(kbdev, &kbdev->csf.mcu_mmu); ++ ++ kbase_csf_scheduler_term(kbdev); ++ ++ kbase_csf_doorbell_mapping_term(kbdev); ++ ++ free_global_iface(kbdev); ++ ++ /* Release the address space */ ++ kbdev->as_free |= MCU_AS_BITMASK; ++ ++ while (!list_empty(&kbdev->csf.firmware_interfaces)) { ++ struct firmware_interface *interface; ++ ++ interface = list_first_entry(&kbdev->csf.firmware_interfaces, ++ struct firmware_interface, node); ++ list_del(&interface->node); ++ ++ vunmap(interface->kernel_map); ++ if (interface->flags & CSF_FIRMWARE_ENTRY_PROTECTED) { ++ kbase_csf_protected_memory_free(kbdev, interface->pma, ++ interface->num_pages); ++ } else { ++ kbase_mem_pool_free_pages( ++ &kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], ++ interface->num_pages, interface->phys, ++ true, false); ++ } ++ ++ kfree(interface->phys); ++ kfree(interface); ++ } ++ ++ while (!list_empty(&kbdev->csf.firmware_timeline_metadata)) { ++ struct firmware_timeline_metadata *metadata; ++ ++ metadata = list_first_entry( ++ &kbdev->csf.firmware_timeline_metadata, ++ struct firmware_timeline_metadata, ++ node); ++ list_del(&metadata->node); ++ ++ kfree(metadata); ++ } ++ ++ kbase_csf_firmware_trace_buffers_term(kbdev); ++ ++#ifndef MALI_KBASE_BUILD ++ mali_kutf_fw_utf_entry_cleanup(kbdev); ++#endif ++ ++ mutex_destroy(&kbdev->csf.reg_lock); ++ ++ /* This will also free up the region allocated for the shared interface ++ * entry parsed from the firmware image. ++ */ ++ kbase_mcu_shared_interface_region_tracker_term(kbdev); ++} ++ ++int kbase_csf_firmware_ping(struct kbase_device *const kbdev) ++{ ++ const struct kbase_csf_global_iface *const global_iface = ++ &kbdev->csf.global_iface; ++ unsigned long flags; ++ ++ kbase_csf_scheduler_spin_lock(kbdev, &flags); ++ set_global_request(global_iface, GLB_REQ_PING_MASK); ++ kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); ++ kbase_csf_scheduler_spin_unlock(kbdev, flags); ++ ++ return wait_for_global_request(kbdev, GLB_REQ_PING_MASK); ++} ++ ++int kbase_csf_firmware_set_timeout(struct kbase_device *const kbdev, ++ u64 const timeout) ++{ ++ const struct kbase_csf_global_iface *const global_iface = ++ &kbdev->csf.global_iface; ++ unsigned long flags; ++ int err; ++ ++ /* The 'reg_lock' is also taken and is held till the update is not ++ * complete, to ensure the update of timeout value by multiple Users ++ * gets serialized. ++ */ ++ mutex_lock(&kbdev->csf.reg_lock); ++ kbase_csf_scheduler_spin_lock(kbdev, &flags); ++ set_timeout_global(global_iface, timeout); ++ kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); ++ kbase_csf_scheduler_spin_unlock(kbdev, flags); ++ ++ err = wait_for_global_request(kbdev, GLB_REQ_CFG_PROGRESS_TIMER_MASK); ++ mutex_unlock(&kbdev->csf.reg_lock); ++ ++ return err; ++} ++ ++void kbase_csf_enter_protected_mode(struct kbase_device *kbdev) ++{ ++ struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; ++ unsigned long flags; ++ unsigned int value; ++ ++ kbase_csf_scheduler_spin_lock(kbdev, &flags); ++ value = kbase_csf_firmware_global_output(global_iface, GLB_ACK); ++ value ^= GLB_REQ_PROTM_ENTER_MASK; ++ kbase_csf_firmware_global_input_mask(global_iface, GLB_REQ, value, ++ GLB_REQ_PROTM_ENTER_MASK); ++ dev_dbg(kbdev->dev, "Sending request to enter protected mode"); ++ kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); ++ kbase_csf_scheduler_spin_unlock(kbdev, flags); ++ ++ wait_for_global_request(kbdev, GLB_REQ_PROTM_ENTER_MASK); ++} ++ ++void kbase_csf_firmware_trigger_mcu_halt(struct kbase_device *kbdev) ++{ ++ struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; ++ unsigned long flags; ++ unsigned int value; ++ ++ kbase_csf_scheduler_spin_lock(kbdev, &flags); ++ value = kbase_csf_firmware_global_output(global_iface, GLB_ACK); ++ value ^= GLB_REQ_HALT_MASK; ++ kbase_csf_firmware_global_input_mask(global_iface, GLB_REQ, value, ++ GLB_REQ_HALT_MASK); ++ dev_dbg(kbdev->dev, "Sending request to HALT MCU"); ++ kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); ++ kbase_csf_scheduler_spin_unlock(kbdev, flags); ++} ++ ++/** ++ * copy_grp_and_stm - Copy command stream and/or group data ++ * ++ * @iface: Global command stream front-end interface provided by ++ * the firmware. ++ * @group_data: Pointer where to store all the group data ++ * (sequentially). ++ * @max_group_num: The maximum number of groups to be read. Can be 0, in ++ * which case group_data is unused. ++ * @stream_data: Pointer where to store all the stream data ++ * (sequentially). ++ * @max_total_stream_num: The maximum number of streams to be read. ++ * Can be 0, in which case stream_data is unused. ++ * ++ * Return: Total number of command streams, summed across all groups. ++ */ ++static u32 copy_grp_and_stm( ++ const struct kbase_csf_global_iface * const iface, ++ struct basep_cs_group_control * const group_data, ++ u32 max_group_num, ++ struct basep_cs_stream_control * const stream_data, ++ u32 max_total_stream_num) ++{ ++ u32 i, total_stream_num = 0; ++ ++ if (WARN_ON((max_group_num > 0) && !group_data)) ++ max_group_num = 0; ++ ++ if (WARN_ON((max_total_stream_num > 0) && !stream_data)) ++ max_total_stream_num = 0; ++ ++ for (i = 0; i < iface->group_num; i++) { ++ u32 j; ++ ++ if (i < max_group_num) { ++ group_data[i].features = iface->groups[i].features; ++ group_data[i].stream_num = iface->groups[i].stream_num; ++ group_data[i].suspend_size = ++ iface->groups[i].suspend_size; ++ } ++ for (j = 0; j < iface->groups[i].stream_num; j++) { ++ if (total_stream_num < max_total_stream_num) ++ stream_data[total_stream_num].features = ++ iface->groups[i].streams[j].features; ++ total_stream_num++; ++ } ++ } ++ ++ return total_stream_num; ++} ++ ++u32 kbase_csf_firmware_get_glb_iface(struct kbase_device *kbdev, ++ struct basep_cs_group_control *const group_data, ++ u32 const max_group_num, ++ struct basep_cs_stream_control *const stream_data, ++ u32 const max_total_stream_num, u32 *const glb_version, ++ u32 *const features, u32 *const group_num, u32 *const prfcnt_size) ++{ ++ const struct kbase_csf_global_iface * const iface = ++ &kbdev->csf.global_iface; ++ ++ if (WARN_ON(!glb_version) || ++ WARN_ON(!features) || ++ WARN_ON(!group_num) || ++ WARN_ON(!prfcnt_size)) ++ return 0; ++ ++ *glb_version = iface->version; ++ *features = iface->features; ++ *group_num = iface->group_num; ++ *prfcnt_size = iface->prfcnt_size; ++ ++ return copy_grp_and_stm(iface, group_data, max_group_num, ++ stream_data, max_total_stream_num); ++} ++ ++const char *kbase_csf_firmware_get_timeline_metadata( ++ struct kbase_device *kbdev, const char *name, size_t *size) ++{ ++ struct firmware_timeline_metadata *metadata; ++ ++ list_for_each_entry( ++ metadata, &kbdev->csf.firmware_timeline_metadata, node) { ++ if (!strcmp(metadata->name, name)) { ++ *size = metadata->size; ++ return metadata->data; ++ } ++ } ++ ++ *size = 0; ++ return NULL; ++} ++ ++int kbase_csf_firmware_mcu_shared_mapping_init( ++ struct kbase_device *kbdev, ++ unsigned int num_pages, ++ unsigned long cpu_map_properties, ++ unsigned long gpu_map_properties, ++ struct kbase_csf_mapping *csf_mapping) ++{ ++ struct tagged_addr *phys; ++ struct kbase_va_region *va_reg; ++ struct page **page_list; ++ void *cpu_addr; ++ int i, ret = 0; ++ pgprot_t cpu_map_prot = PAGE_KERNEL; ++ unsigned long gpu_map_prot; ++ ++ if (cpu_map_properties & PROT_READ) ++ cpu_map_prot = PAGE_KERNEL_RO; ++ ++ if (kbdev->system_coherency == COHERENCY_ACE) { ++ gpu_map_prot = ++ KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_DEFAULT_ACE); ++ } else { ++ gpu_map_prot = ++ KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_NON_CACHEABLE); ++ cpu_map_prot = pgprot_writecombine(cpu_map_prot); ++ }; ++ ++ phys = kmalloc_array(num_pages, sizeof(*phys), GFP_KERNEL); ++ if (!phys) ++ goto out; ++ ++ page_list = kmalloc_array(num_pages, sizeof(*page_list), GFP_KERNEL); ++ if (!page_list) ++ goto page_list_alloc_error; ++ ++ ret = kbase_mem_pool_alloc_pages( ++ &kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], ++ num_pages, phys, false); ++ if (ret <= 0) ++ goto phys_mem_pool_alloc_error; ++ ++ for (i = 0; i < num_pages; i++) ++ page_list[i] = as_page(phys[i]); ++ ++ cpu_addr = vmap(page_list, num_pages, VM_MAP, cpu_map_prot); ++ if (!cpu_addr) ++ goto vmap_error; ++ ++ va_reg = kbase_alloc_free_region(&kbdev->csf.shared_reg_rbtree, 0, ++ num_pages, KBASE_REG_ZONE_MCU_SHARED); ++ if (!va_reg) ++ goto va_region_alloc_error; ++ ++ mutex_lock(&kbdev->csf.reg_lock); ++ ret = kbase_add_va_region_rbtree(kbdev, va_reg, 0, num_pages, 1); ++ va_reg->flags &= ~KBASE_REG_FREE; ++ mutex_unlock(&kbdev->csf.reg_lock); ++ if (ret) ++ goto va_region_add_error; ++ ++ gpu_map_properties &= (KBASE_REG_GPU_RD | KBASE_REG_GPU_WR); ++ gpu_map_properties |= gpu_map_prot; ++ ++ ret = kbase_mmu_insert_pages_no_flush(kbdev, &kbdev->csf.mcu_mmu, ++ va_reg->start_pfn, &phys[0], num_pages, ++ gpu_map_properties, KBASE_MEM_GROUP_CSF_FW); ++ if (ret) ++ goto mmu_insert_pages_error; ++ ++ kfree(page_list); ++ csf_mapping->phys = phys; ++ csf_mapping->cpu_addr = cpu_addr; ++ csf_mapping->va_reg = va_reg; ++ csf_mapping->num_pages = num_pages; ++ ++ return 0; ++ ++mmu_insert_pages_error: ++ mutex_lock(&kbdev->csf.reg_lock); ++ kbase_remove_va_region(va_reg); ++ mutex_unlock(&kbdev->csf.reg_lock); ++va_region_add_error: ++ kbase_free_alloced_region(va_reg); ++va_region_alloc_error: ++ vunmap(cpu_addr); ++vmap_error: ++ kbase_mem_pool_free_pages( ++ &kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], ++ num_pages, phys, false, false); ++ ++phys_mem_pool_alloc_error: ++ kfree(page_list); ++page_list_alloc_error: ++ kfree(phys); ++out: ++ /* Zero-initialize the mapping to make sure that the termination ++ * function doesn't try to unmap or free random addresses. */ ++ csf_mapping->phys = NULL; ++ csf_mapping->cpu_addr = NULL; ++ csf_mapping->va_reg = NULL; ++ csf_mapping->num_pages = 0; ++ ++ return -ENOMEM; ++} ++ ++void kbase_csf_firmware_mcu_shared_mapping_term( ++ struct kbase_device *kbdev, struct kbase_csf_mapping *csf_mapping) ++{ ++ if (csf_mapping->va_reg) { ++ mutex_lock(&kbdev->csf.reg_lock); ++ kbase_remove_va_region(csf_mapping->va_reg); ++ mutex_unlock(&kbdev->csf.reg_lock); ++ kbase_free_alloced_region(csf_mapping->va_reg); ++ } ++ ++ if (csf_mapping->phys) { ++ kbase_mem_pool_free_pages( ++ &kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], ++ csf_mapping->num_pages, csf_mapping->phys, false, ++ false); ++ } ++ ++ vunmap(csf_mapping->cpu_addr); ++ kfree(csf_mapping->phys); ++} +diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.h +new file mode 100755 +index 000000000000..03a5217cffb0 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.h +@@ -0,0 +1,663 @@ ++/* ++ * ++ * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_CSF_FIRMWARE_H_ ++#define _KBASE_CSF_FIRMWARE_H_ ++ ++#include "device/mali_kbase_device.h" ++#include "mali_gpu_csf_registers.h" ++ ++/* ++ * PAGE_KERNEL_RO was only defined on 32bit ARM in 4.19 in: ++ * Commit a3266bd49c721e2e0a71f352d83713fbd60caadb ++ * Author: Luis R. Rodriguez ++ * Date: Fri Aug 17 15:46:29 2018 -0700 ++ * ++ * mm: provide a fallback for PAGE_KERNEL_RO for architectures ++ * ++ * Some architectures do not define certain PAGE_KERNEL_* flags, this is ++ * either because: ++ * ++ * a) The way to implement some of these flags is *not yet ported*, or ++ * b) The architecture *has no way* to describe them ++ * ++ * [snip] ++ * ++ * This can be removed once support of 32bit ARM kernels predating 4.19 is no ++ * longer required. ++ */ ++#ifndef PAGE_KERNEL_RO ++#define PAGE_KERNEL_RO PAGE_KERNEL ++#endif ++ ++/* Address space number to claim for the firmware. */ ++#define MCU_AS_NR 0 ++#define MCU_AS_BITMASK (1 << MCU_AS_NR) ++ ++/* Number of available Doorbells */ ++#define CSF_NUM_DOORBELL ((u8)24) ++ ++/* Offset to the first HW doorbell page */ ++#define CSF_HW_DOORBELL_PAGE_OFFSET ((u32)0x80000) ++ ++/* Size of HW Doorbell page, used to calculate the offset to subsequent pages */ ++#define CSF_HW_DOORBELL_PAGE_SIZE ((u32)0x10000) ++ ++/* Doorbell 0 is used by the driver. */ ++#define CSF_KERNEL_DOORBELL_NR ((u32)0) ++ ++/* Offset of name inside a trace buffer entry in the firmware image */ ++#define TRACE_BUFFER_ENTRY_NAME_OFFSET (0x1C) ++ ++/* All implementations of the host interface with major version 0 must comply ++ * with these restrictions: ++ */ ++/* GLB_GROUP_NUM: At least 3 command stream groups, but no more than 31 */ ++#define MIN_SUPPORTED_CSGS 3 ++#define MAX_SUPPORTED_CSGS 31 ++/* GROUP_STREAM_NUM: At least 8 command streams per CSG, but no more than 32 */ ++#define MIN_SUPPORTED_STREAMS_PER_GROUP 8 ++/* Maximum command streams per csg. */ ++#define MAX_SUPPORTED_STREAMS_PER_GROUP 32 ++ ++struct kbase_device; ++ ++ ++/** ++ * struct kbase_csf_mapping - Memory mapping for CSF memory. ++ * @phys: Physical memory allocation used by the mapping. ++ * @cpu_addr: Starting CPU address for the mapping. ++ * @va_reg: GPU virtual address region for the mapping. ++ * @num_pages: Size of the mapping, in memory pages. ++ */ ++struct kbase_csf_mapping { ++ struct tagged_addr *phys; ++ void *cpu_addr; ++ struct kbase_va_region *va_reg; ++ unsigned int num_pages; ++}; ++ ++/** ++ * struct kbase_csf_trace_buffers - List and state of firmware trace buffers. ++ * @list: List of trace buffers descriptors. ++ * @mcu_rw: Metadata for the MCU shared memory mapping used for ++ * GPU-readable,writable/CPU-writable variables. ++ * @mcu_write: Metadata for the MCU shared memory mapping used for ++ * GPU-writable/CPU-readable variables. ++ */ ++struct kbase_csf_trace_buffers { ++ struct list_head list; ++ struct kbase_csf_mapping mcu_rw; ++ struct kbase_csf_mapping mcu_write; ++}; ++ ++/** ++ * struct kbase_csf_cmd_stream_info - Command stream interface provided by the ++ * firmware. ++ * ++ * @kbdev: Address of the instance of a GPU platform device that implements ++ * this interface. ++ * @features: Bit field of command stream features (e.g. which types of jobs ++ * are supported). Bits 7:0 specify the number of work registers(-1). ++ * Bits 11:8 specify the number of scoreboard entries(-1). ++ * @input: Address of command stream interface input page. ++ * @output: Address of command stream interface output page. ++ */ ++struct kbase_csf_cmd_stream_info { ++ struct kbase_device *kbdev; ++ u32 features; ++ void *input; ++ void *output; ++}; ++ ++/** ++ * kbase_csf_firmware_cs_input() - Set a word in a command stream's input page ++ * ++ * @info: Command stream interface provided by the firmware. ++ * @offset: Offset of the word to be written, in bytes. ++ * @value: Value to be written. ++ */ ++void kbase_csf_firmware_cs_input( ++ const struct kbase_csf_cmd_stream_info *info, u32 offset, u32 value); ++ ++/** ++ * kbase_csf_firmware_cs_input_read() - Read a word in a command stream's input ++ * page ++ * ++ * Return: Value of the word read from the command stream's input page. ++ * ++ * @info: Command stream interface provided by the firmware. ++ * @offset: Offset of the word to be read, in bytes. ++ */ ++u32 kbase_csf_firmware_cs_input_read( ++ const struct kbase_csf_cmd_stream_info *const info, const u32 offset); ++ ++/** ++ * kbase_csf_firmware_cs_input_mask() - Set part of a word in a command stream's ++ * input page ++ * ++ * @info: Command stream interface provided by the firmware. ++ * @offset: Offset of the word to be modified, in bytes. ++ * @value: Value to be written. ++ * @mask: Bitmask with the bits to be modified set. ++ */ ++void kbase_csf_firmware_cs_input_mask( ++ const struct kbase_csf_cmd_stream_info *info, u32 offset, ++ u32 value, u32 mask); ++ ++/** ++ * kbase_csf_firmware_cs_output() - Read a word in a command stream's output ++ * page ++ * ++ * Return: Value of the word read from the command stream's output page. ++ * ++ * @info: Command stream interface provided by the firmware. ++ * @offset: Offset of the word to be read, in bytes. ++ */ ++u32 kbase_csf_firmware_cs_output( ++ const struct kbase_csf_cmd_stream_info *info, u32 offset); ++/** ++ * struct kbase_csf_cmd_stream_group_info - Command stream group interface ++ * provided by the firmware. ++ * ++ * @kbdev: Address of the instance of a GPU platform device that implements ++ * this interface. ++ * @features: Bit mask of features. Reserved bits should be 0, and should ++ * be ignored. ++ * @input: Address of global interface input page. ++ * @output: Address of global interface output page. ++ * @suspend_size: Size in bytes for normal suspend buffer for the command ++ * stream group. ++ * @protm_suspend_size: Size in bytes for protected mode suspend buffer ++ * for the command stream group. ++ * @stream_num: Number of command streams in the command stream group. ++ * @stream_stride: Stride in bytes in JASID0 virtual address between ++ * command stream capability structures. ++ * @streams: Address of an array of command stream capability structures. ++ */ ++struct kbase_csf_cmd_stream_group_info { ++ struct kbase_device *kbdev; ++ u32 features; ++ void *input; ++ void *output; ++ u32 suspend_size; ++ u32 protm_suspend_size; ++ u32 stream_num; ++ u32 stream_stride; ++ struct kbase_csf_cmd_stream_info *streams; ++}; ++ ++/** ++ * kbase_csf_firmware_csg_input() - Set a word in a command stream group's ++ * input page ++ * ++ * @info: Command stream group interface provided by the firmware. ++ * @offset: Offset of the word to be written, in bytes. ++ * @value: Value to be written. ++ */ ++void kbase_csf_firmware_csg_input( ++ const struct kbase_csf_cmd_stream_group_info *info, u32 offset, ++ u32 value); ++ ++/** ++ * kbase_csf_firmware_csg_input_read() - Read a word in a command stream group's ++ * input page ++ * ++ * Return: Value of the word read from the command stream group's input page. ++ * ++ * @info: Command stream group interface provided by the firmware. ++ * @offset: Offset of the word to be read, in bytes. ++ */ ++u32 kbase_csf_firmware_csg_input_read( ++ const struct kbase_csf_cmd_stream_group_info *info, u32 offset); ++ ++/** ++ * kbase_csf_firmware_csg_input_mask() - Set part of a word in a command stream ++ * group's input page ++ * ++ * @info: Command stream group interface provided by the firmware. ++ * @offset: Offset of the word to be modified, in bytes. ++ * @value: Value to be written. ++ * @mask: Bitmask with the bits to be modified set. ++ */ ++void kbase_csf_firmware_csg_input_mask( ++ const struct kbase_csf_cmd_stream_group_info *info, u32 offset, ++ u32 value, u32 mask); ++ ++/** ++ * kbase_csf_firmware_csg_output()- Read a word in a command stream group's ++ * output page ++ * ++ * Return: Value of the word read from the command stream group's output page. ++ * ++ * @info: Command stream group interface provided by the firmware. ++ * @offset: Offset of the word to be read, in bytes. ++ */ ++u32 kbase_csf_firmware_csg_output( ++ const struct kbase_csf_cmd_stream_group_info *info, u32 offset); ++ ++/** ++ * struct kbase_csf_global_iface - Global command stream front-end interface ++ * provided by the firmware. ++ * ++ * @kbdev: Address of the instance of a GPU platform device that implements ++ * this interface. ++ * @version: Bits 31:16 hold the major version number and 15:0 hold the minor ++ * version number. A higher minor version is backwards-compatible ++ * with a lower minor version for the same major version. ++ * @features: Bit mask of features (e.g. whether certain types of job can ++ * be suspended). Reserved bits should be 0, and should be ignored. ++ * @input: Address of global interface input page. ++ * @output: Address of global interface output page. ++ * @group_num: Number of command stream groups supported. ++ * @group_stride: Stride in bytes in JASID0 virtual address between ++ * command stream group capability structures. ++ * @prfcnt_size: Performance counters size. ++ * @groups: Address of an array of command stream group capability structures. ++ */ ++struct kbase_csf_global_iface { ++ struct kbase_device *kbdev; ++ u32 version; ++ u32 features; ++ void *input; ++ void *output; ++ u32 group_num; ++ u32 group_stride; ++ u32 prfcnt_size; ++ struct kbase_csf_cmd_stream_group_info *groups; ++}; ++ ++/** ++ * kbase_csf_firmware_global_input() - Set a word in the global input page ++ * ++ * @iface: Command stream front-end interface provided by the firmware. ++ * @offset: Offset of the word to be written, in bytes. ++ * @value: Value to be written. ++ */ ++void kbase_csf_firmware_global_input( ++ const struct kbase_csf_global_iface *iface, u32 offset, u32 value); ++ ++/** ++ * kbase_csf_firmware_global_input_mask() - Set part of a word in the global ++ * input page ++ * ++ * @iface: Command stream front-end interface provided by the firmware. ++ * @offset: Offset of the word to be modified, in bytes. ++ * @value: Value to be written. ++ * @mask: Bitmask with the bits to be modified set. ++ */ ++void kbase_csf_firmware_global_input_mask( ++ const struct kbase_csf_global_iface *iface, u32 offset, ++ u32 value, u32 mask); ++ ++/** ++ * kbase_csf_firmware_global_input_read() - Read a word in a global input page ++ * ++ * Return: Value of the word read from the global input page. ++ * ++ * @info: Command stream group interface provided by the firmware. ++ * @offset: Offset of the word to be read, in bytes. ++ */ ++u32 kbase_csf_firmware_global_input_read( ++ const struct kbase_csf_global_iface *info, u32 offset); ++ ++/** ++ * kbase_csf_firmware_global_output() - Read a word in the global output page ++ * ++ * Return: Value of the word read from the global output page. ++ * ++ * @iface: Command stream front-end interface provided by the firmware. ++ * @offset: Offset of the word to be read, in bytes. ++ */ ++u32 kbase_csf_firmware_global_output( ++ const struct kbase_csf_global_iface *iface, u32 offset); ++ ++/* Calculate the offset to the Hw doorbell page corresponding to the ++ * doorbell number. ++ */ ++static u32 csf_doorbell_offset(int doorbell_nr) ++{ ++ WARN_ON(doorbell_nr >= CSF_NUM_DOORBELL); ++ ++ return CSF_HW_DOORBELL_PAGE_OFFSET + ++ (doorbell_nr * CSF_HW_DOORBELL_PAGE_SIZE); ++} ++ ++static inline void kbase_csf_ring_doorbell(struct kbase_device *kbdev, ++ int doorbell_nr) ++{ ++ WARN_ON(doorbell_nr >= CSF_NUM_DOORBELL); ++ ++ kbase_reg_write(kbdev, csf_doorbell_offset(doorbell_nr), (u32)1); ++} ++ ++/** ++ * kbase_csf_read_firmware_memory - Read a value in a GPU address ++ * ++ * This function read a value in a GPU address that belongs to ++ * a private firmware memory region. The function assumes that the location ++ * is not permanently mapped on the CPU address space, therefore it maps it ++ * and then unmaps it to access it independently. ++ * ++ * @kbdev: Device pointer ++ * @gpu_addr: GPU address to read ++ * @value: output pointer to which the read value will be written. ++ */ ++void kbase_csf_read_firmware_memory(struct kbase_device *kbdev, ++ u32 gpu_addr, u32 *value); ++ ++/** ++ * kbase_csf_update_firmware_memory - Write a value in a GPU address ++ * ++ * This function writes a given value in a GPU address that belongs to ++ * a private firmware memory region. The function assumes that the destination ++ * is not permanently mapped on the CPU address space, therefore it maps it ++ * and then unmaps it to access it independently. ++ * ++ * @kbdev: Device pointer ++ * @gpu_addr: GPU address to write ++ * @value: Value to write ++ */ ++void kbase_csf_update_firmware_memory(struct kbase_device *kbdev, ++ u32 gpu_addr, u32 value); ++ ++/** ++ * kbase_csf_firmware_init() - Load the firmware for the CSF MCU ++ * ++ * Request the firmware from user space and load it into memory. ++ * ++ * Return: 0 if successful, negative error code on failure ++ * ++ * @kbdev: Kbase device ++ */ ++int kbase_csf_firmware_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_csf_firmware_term() - Unload the firmware ++ * ++ * Frees the memory allocated by kbase_csf_firmware_init() ++ * ++ * @kbdev: Kbase device ++ */ ++void kbase_csf_firmware_term(struct kbase_device *kbdev); ++ ++/** ++ * kbase_csf_firmware_ping - Send the ping request to firmware. ++ * ++ * The function sends the ping request to firmware to confirm it is alive. ++ * ++ * @kbdev: Instance of a GPU platform device that implements a command ++ * stream front-end interface. ++ * ++ * Return: 0 on success, or negative on failure. ++ */ ++int kbase_csf_firmware_ping(struct kbase_device *kbdev); ++ ++/** ++ * kbase_csf_firmware_set_timeout - Set a hardware endpoint progress timeout. ++ * ++ * @kbdev: Instance of a GPU platform device that implements a command ++ * stream front-end interface. ++ * @timeout: The maximum number of GPU cycles that is allowed to elapse ++ * without forward progress before the driver terminates a GPU ++ * command queue group. ++ * ++ * Configures the progress timeout value used by the firmware to decide ++ * when to report that a task is not making progress on an endpoint. ++ * ++ * Return: 0 on success, or negative on failure. ++ */ ++int kbase_csf_firmware_set_timeout(struct kbase_device *kbdev, u64 timeout); ++ ++/** ++ * kbase_csf_enter_protected_mode - Send the Global request to firmware to ++ * enter protected mode and wait for its ++ * completion. ++ * ++ * @kbdev: Instance of a GPU platform device that implements a command ++ * stream front-end interface. ++ */ ++void kbase_csf_enter_protected_mode(struct kbase_device *kbdev); ++ ++static inline bool kbase_csf_firmware_mcu_halted(struct kbase_device *kbdev) ++{ ++#ifndef CONFIG_MALI_BIFROST_NO_MALI ++ return (kbase_reg_read(kbdev, GPU_CONTROL_REG(MCU_STATUS)) == ++ MCU_STATUS_HALTED); ++#else ++ return true; ++#endif ++} ++ ++/** ++ * kbase_csf_firmware_trigger_mcu_halt - Send the Global request to firmware to ++ * halt its operation and bring itself ++ * into a known internal state for warm ++ * boot later. ++ * ++ * @kbdev: Instance of a GPU platform device that implements a command ++ * stream front-end interface. ++ */ ++void kbase_csf_firmware_trigger_mcu_halt(struct kbase_device *kbdev); ++ ++/** ++ * kbase_csf_firmware_enable_mcu - Send the command to enable MCU ++ * ++ * @kbdev: Instance of a GPU platform device that implements a command ++ * stream front-end interface. ++ */ ++static inline void kbase_csf_firmware_enable_mcu(struct kbase_device *kbdev) ++{ ++ /* Trigger the boot of MCU firmware, Use the AUTO mode as ++ * otherwise on fast reset, to exit protected mode, MCU will ++ * not reboot by itself to enter normal mode. ++ */ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(MCU_CONTROL), MCU_CNTRL_AUTO); ++} ++ ++/** ++ * kbase_csf_firmware_disable_mcu - Send the command to disable MCU ++ * ++ * @kbdev: Instance of a GPU platform device that implements a command ++ * stream front-end interface. ++ */ ++static inline void kbase_csf_firmware_disable_mcu(struct kbase_device *kbdev) ++{ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(MCU_CONTROL), MCU_CNTRL_DISABLE); ++} ++ ++/** ++ * kbase_csf_firmware_disable_mcu_wait - Wait for the MCU to reach disabled ++ * status. ++ * ++ * @kbdev: Instance of a GPU platform device that implements a command ++ * stream front-end interface. ++ */ ++void kbase_csf_firmware_disable_mcu_wait(struct kbase_device *kbdev); ++ ++/** ++ * kbase_trigger_firmware_reload - Trigger the reboot of MCU firmware, for the ++ * cold boot case firmware image would be ++ * reloaded from filesystem into memory. ++ * ++ * @kbdev: Instance of a GPU platform device that implements a command ++ * stream front-end interface. ++ */ ++void kbase_csf_firmware_trigger_reload(struct kbase_device *kbdev); ++ ++/** ++ * kbase_csf_firmware_reload_completed - The reboot of MCU firmware has ++ * completed. ++ * ++ * @kbdev: Instance of a GPU platform device that implements a command ++ * stream front-end interface. ++ */ ++void kbase_csf_firmware_reload_completed(struct kbase_device *kbdev); ++ ++/** ++ * kbase_csf_firmware_global_reinit - Send the Global configuration requests ++ * after the reboot of MCU firmware. ++ * ++ * @kbdev: Instance of a GPU platform device that implements a command ++ * stream front-end interface. ++ */ ++void kbase_csf_firmware_global_reinit(struct kbase_device *kbdev); ++ ++/** ++ * kbase_csf_firmware_global_reinit_complete - Check the Global configuration ++ * requests, sent after the reboot of MCU firmware, have ++ * completed or not. ++ * ++ * Return: true if the Global configuration requests completed otherwise false. ++ * ++ * @kbdev: Instance of a GPU platform device that implements a command ++ * stream front-end interface. ++ */ ++bool kbase_csf_firmware_global_reinit_complete(struct kbase_device *kbdev); ++ ++/** ++ * Request the global control block of CSF interface capabilities ++ * ++ * Return: Total number of command streams, summed across all groups. ++ * ++ * @kbdev: Kbase device. ++ * @group_data: Pointer where to store all the group data ++ * (sequentially). ++ * @max_group_num: The maximum number of groups to be read. ++ * Can be 0, in which case group_data is unused. ++ * @stream_data: Pointer where to store all the stream data ++ * (sequentially). ++ * @max_total_stream_num: The maximum number of streams to be read. ++ * Can be 0, in which case stream_data is unused. ++ * @glb_version: Where to store the global interface version. ++ * Bits 31:16 hold the major version number and ++ * 15:0 hold the minor version number. ++ * A higher minor version is backwards-compatible ++ * with a lower minor version for the same major ++ * version. ++ * @features: Where to store a bit mask of features (e.g. ++ * whether certain types of job can be suspended). ++ * @group_num: Where to store the number of command stream groups ++ * supported. ++ * @prfcnt_size: Where to store the size of CSF performance counters, ++ * in bytes. Bits 31:16 hold the size of firmware ++ * performance counter data and 15:0 hold the size of ++ * hardware performance counter data. ++ */ ++u32 kbase_csf_firmware_get_glb_iface(struct kbase_device *kbdev, ++ struct basep_cs_group_control *group_data, u32 max_group_num, ++ struct basep_cs_stream_control *stream_data, u32 max_total_stream_num, ++ u32 *glb_version, u32 *features, u32 *group_num, u32 *prfcnt_size); ++ ++ ++/** ++ * Get CSF firmware header timeline metadata content ++ * ++ * Return: The firmware timeline metadata content which match @p name. ++ * ++ * @kbdev: Kbase device. ++ * @name: Name of the metadata which metadata content to be returned. ++ * @size: Metadata size if specified metadata found. ++ */ ++const char *kbase_csf_firmware_get_timeline_metadata(struct kbase_device *kbdev, ++ const char *name, size_t *size); ++ ++/** ++ * kbase_csf_firmware_mcu_shared_mapping_init - ++ * Allocate and map MCU shared memory. ++ * ++ * This helper function allocates memory and maps it on both the CPU ++ * and the GPU address spaces. Most of the properties of the mapping ++ * are implicit and will be automatically determined by the function, ++ * e.g. whether memory is cacheable. ++ * ++ * The client is only expected to specify whether the mapping is readable ++ * or writable in the CPU and the GPU address spaces; any other flag ++ * will be ignored by the function. ++ * ++ * Return: 0 if success, or an error code on failure. ++ * ++ * @kbdev: Kbase device the memory mapping shall belong to. ++ * @num_pages: Number of memory pages to map. ++ * @cpu_map_properties: Either PROT_READ or PROT_WRITE. ++ * @gpu_map_properties: Either KBASE_REG_GPU_RD or KBASE_REG_GPU_WR. ++ * @csf_mapping: Object where to write metadata for the memory mapping. ++ */ ++int kbase_csf_firmware_mcu_shared_mapping_init( ++ struct kbase_device *kbdev, ++ unsigned int num_pages, ++ unsigned long cpu_map_properties, ++ unsigned long gpu_map_properties, ++ struct kbase_csf_mapping *csf_mapping); ++ ++/** ++ * kbase_csf_firmware_mcu_shared_mapping_term - Unmap and free MCU shared memory. ++ * ++ * @kbdev: Device pointer. ++ * @csf_mapping: Metadata of the memory mapping to terminate. ++ */ ++void kbase_csf_firmware_mcu_shared_mapping_term( ++ struct kbase_device *kbdev, struct kbase_csf_mapping *csf_mapping); ++ ++#ifndef MALI_KBASE_BUILD ++/** ++ * mali_kutf_process_fw_utf_entry() - Process the "Firmware UTF tests" section ++ * ++ * Read "Firmware UTF tests" section from the firmware image and create ++ * necessary kutf app+suite+tests. ++ * ++ * Return: 0 if successful, negative error code on failure. In both cases ++ * caller will have to invoke mali_kutf_fw_utf_entry_cleanup for the cleanup ++ * ++ * @kbdev: Kbase device structure ++ * @fw_data: Pointer to the start of firmware binary image loaded from disk ++ * @fw_size: Size (in bytes) of the firmware image ++ * @entry: Pointer to the start of the section ++ */ ++int mali_kutf_process_fw_utf_entry(struct kbase_device *kbdev, ++ const void *fw_data, size_t fw_size, const u32 *entry); ++ ++/** ++ * mali_kutf_fw_utf_entry_cleanup() - Remove the Fw UTF tests debugfs entries ++ * ++ * Destroy the kutf apps+suites+tests created on parsing "Firmware UTF tests" ++ * section from the firmware image. ++ * ++ * @kbdev: Kbase device structure ++ */ ++void mali_kutf_fw_utf_entry_cleanup(struct kbase_device *kbdev); ++#endif ++ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++extern bool fw_debug; ++#endif ++ ++static inline long kbase_csf_timeout_in_jiffies(const unsigned int msecs) ++{ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++ return (fw_debug ? MAX_SCHEDULE_TIMEOUT : msecs_to_jiffies(msecs)); ++#else ++ return msecs_to_jiffies(msecs); ++#endif ++} ++ ++#endif +diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_cfg.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_cfg.c +new file mode 100755 +index 000000000000..d282d5ca7fc2 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_cfg.c +@@ -0,0 +1,306 @@ ++/* ++ * ++ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++#include "mali_kbase_csf_firmware_cfg.h" ++#include ++ ++#if CONFIG_SYSFS ++#define CSF_FIRMWARE_CFG_SYSFS_DIR_NAME "firmware_config" ++ ++/** ++ * struct firmware_config - Configuration item within the MCU firmware ++ * ++ * The firmware may expose configuration options. Each option has a name, the ++ * address where the option is controlled and the minimum and maximum values ++ * that the option can take. ++ * ++ * @node: List head linking all options to ++ * kbase_device:csf.firmware_config ++ * @kbdev: Pointer to the Kbase device ++ * @kobj: Kobject corresponding to the sysfs sub-directory, ++ * inside CSF_FIRMWARE_CFG_SYSFS_DIR_NAME directory, ++ * representing the configuration option @name. ++ * @kobj_inited: kobject initialization state ++ * @name: NUL-terminated string naming the option ++ * @address: The address in the firmware image of the configuration option ++ * @min: The lowest legal value of the configuration option ++ * @max: The maximum legal value of the configuration option ++ * @cur_val: The current value of the configuration option ++ */ ++struct firmware_config { ++ struct list_head node; ++ struct kbase_device *kbdev; ++ struct kobject kobj; ++ bool kobj_inited; ++ char *name; ++ u32 address; ++ u32 min; ++ u32 max; ++ u32 cur_val; ++}; ++ ++#define FW_CFG_ATTR(_name, _mode) \ ++ struct attribute fw_cfg_attr_##_name = { \ ++ .name = __stringify(_name), \ ++ .mode = VERIFY_OCTAL_PERMISSIONS(_mode), \ ++ } ++ ++static FW_CFG_ATTR(min, S_IRUGO); ++static FW_CFG_ATTR(max, S_IRUGO); ++static FW_CFG_ATTR(cur, S_IRUGO | S_IWUSR); ++ ++static void fw_cfg_kobj_release(struct kobject *kobj) ++{ ++ struct firmware_config *config = ++ container_of(kobj, struct firmware_config, kobj); ++ ++ kfree(config); ++} ++ ++static ssize_t show_fw_cfg(struct kobject *kobj, ++ struct attribute *attr, char *buf) ++{ ++ struct firmware_config *config = ++ container_of(kobj, struct firmware_config, kobj); ++ struct kbase_device *kbdev = config->kbdev; ++ u32 val = 0; ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ if (attr == &fw_cfg_attr_max) ++ val = config->max; ++ else if (attr == &fw_cfg_attr_min) ++ val = config->min; ++ else if (attr == &fw_cfg_attr_cur) { ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ val = config->cur_val; ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ } else { ++ dev_warn(kbdev->dev, ++ "Unexpected read from entry %s/%s", ++ config->name, attr->name); ++ return -EINVAL; ++ } ++ ++ return snprintf(buf, PAGE_SIZE, "%u\n", val); ++} ++ ++static ssize_t store_fw_cfg(struct kobject *kobj, ++ struct attribute *attr, ++ const char *buf, ++ size_t count) ++{ ++ struct firmware_config *config = ++ container_of(kobj, struct firmware_config, kobj); ++ struct kbase_device *kbdev = config->kbdev; ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ if (attr == &fw_cfg_attr_cur) { ++ unsigned long flags; ++ u32 val; ++ int ret = kstrtouint(buf, 0, &val); ++ ++ if (ret) { ++ dev_err(kbdev->dev, ++ "Couldn't process %s/%s write operation.\n" ++ "Use format \n", ++ config->name, attr->name); ++ return -EINVAL; ++ } ++ ++ if ((val < config->min) || (val > config->max)) ++ return -EINVAL; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ if (config->cur_val == val) { ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ return count; ++ } ++ ++ /* ++ * If there is already a GPU reset pending then inform ++ * the User to retry the write. ++ */ ++ if (kbase_reset_gpu_silent(kbdev)) { ++ spin_unlock_irqrestore( ++ &kbdev->hwaccess_lock, flags); ++ return -EAGAIN; ++ } ++ ++ /* ++ * GPU reset request has been placed, now update the ++ * firmware image. GPU reset will take place only after ++ * hwaccess_lock is released. ++ * Update made to firmware image in memory would not ++ * be lost on GPU reset as configuration entries reside ++ * in the RONLY section of firmware image, which is not ++ * reloaded on firmware reboot due to GPU reset. ++ */ ++ kbase_csf_update_firmware_memory( ++ kbdev, config->address, val); ++ ++ config->cur_val = val; ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ /* Wait for the config update to take effect */ ++ kbase_reset_gpu_wait(kbdev); ++ } else { ++ dev_warn(kbdev->dev, ++ "Unexpected write to entry %s/%s", ++ config->name, attr->name); ++ return -EINVAL; ++ } ++ ++ return count; ++} ++ ++static const struct sysfs_ops fw_cfg_ops = { ++ .show = &show_fw_cfg, ++ .store = &store_fw_cfg, ++}; ++ ++static struct attribute *fw_cfg_attrs[] = { ++ &fw_cfg_attr_min, ++ &fw_cfg_attr_max, ++ &fw_cfg_attr_cur, ++ NULL, ++}; ++ ++static struct kobj_type fw_cfg_kobj_type = { ++ .release = &fw_cfg_kobj_release, ++ .sysfs_ops = &fw_cfg_ops, ++ .default_attrs = fw_cfg_attrs, ++}; ++ ++int kbase_csf_firmware_cfg_init(struct kbase_device *kbdev) ++{ ++ struct firmware_config *config; ++ ++ kbdev->csf.fw_cfg_kobj = kobject_create_and_add( ++ CSF_FIRMWARE_CFG_SYSFS_DIR_NAME, &kbdev->dev->kobj); ++ if (!kbdev->csf.fw_cfg_kobj) { ++ kobject_put(kbdev->csf.fw_cfg_kobj); ++ dev_err(kbdev->dev, ++ "Creation of %s sysfs sub-directory failed\n", ++ CSF_FIRMWARE_CFG_SYSFS_DIR_NAME); ++ return -ENOMEM; ++ } ++ ++ list_for_each_entry(config, &kbdev->csf.firmware_config, node) { ++ int err; ++ ++ kbase_csf_read_firmware_memory(kbdev, config->address, ++ &config->cur_val); ++ ++ err = kobject_init_and_add(&config->kobj, &fw_cfg_kobj_type, ++ kbdev->csf.fw_cfg_kobj, "%s", config->name); ++ if (err) { ++ kobject_put(&config->kobj); ++ dev_err(kbdev->dev, ++ "Creation of %s sysfs sub-directory failed\n", ++ config->name); ++ return err; ++ } ++ ++ config->kobj_inited = true; ++ } ++ ++ return 0; ++} ++ ++void kbase_csf_firmware_cfg_term(struct kbase_device *kbdev) ++{ ++ while (!list_empty(&kbdev->csf.firmware_config)) { ++ struct firmware_config *config; ++ ++ config = list_first_entry(&kbdev->csf.firmware_config, ++ struct firmware_config, node); ++ list_del(&config->node); ++ ++ if (config->kobj_inited) { ++ kobject_del(&config->kobj); ++ kobject_put(&config->kobj); ++ } else ++ kfree(config); ++ } ++ ++ kobject_del(kbdev->csf.fw_cfg_kobj); ++ kobject_put(kbdev->csf.fw_cfg_kobj); ++} ++ ++int kbase_csf_firmware_cfg_option_entry_parse(struct kbase_device *kbdev, ++ const struct firmware *fw, ++ const u32 *entry, unsigned int size) ++{ ++ const char *name = (char *)&entry[3]; ++ struct firmware_config *config; ++ const unsigned int name_len = size - CONFIGURATION_ENTRY_NAME_OFFSET; ++ ++ /* Allocate enough space for struct firmware_config and the ++ * configuration option name (with NULL termination) ++ */ ++ config = kzalloc(sizeof(*config) + name_len + 1, GFP_KERNEL); ++ ++ if (!config) ++ return -ENOMEM; ++ ++ config->kbdev = kbdev; ++ config->name = (char *)(config+1); ++ config->address = entry[0]; ++ config->min = entry[1]; ++ config->max = entry[2]; ++ ++ memcpy(config->name, name, name_len); ++ config->name[name_len] = 0; ++ ++ list_add(&config->node, &kbdev->csf.firmware_config); ++ ++ dev_dbg(kbdev->dev, "Configuration option '%s' at 0x%x range %u-%u", ++ config->name, config->address, ++ config->min, config->max); ++ ++ return 0; ++} ++#else ++int kbase_csf_firmware_cfg_init(struct kbase_device *kbdev) ++{ ++ return 0; ++} ++ ++void kbase_csf_firmware_cfg_term(struct kbase_device *kbdev) ++{ ++ /* !CONFIG_SYSFS: Nothing to do here */ ++} ++ ++int kbase_csf_firmware_cfg_option_entry_parse(struct kbase_device *kbdev, ++ const struct firmware *fw, ++ const u32 *entry, unsigned int size) ++{ ++ return 0; ++} ++#endif /* CONFIG_SYSFS */ +diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_cfg.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_cfg.h +new file mode 100755 +index 000000000000..ab4b6ebc5296 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_cfg.h +@@ -0,0 +1,72 @@ ++/* ++ * ++ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_CSF_FIRMWARE_CFG_H_ ++#define _KBASE_CSF_FIRMWARE_CFG_H_ ++ ++#include ++#include "mali_kbase_csf_firmware.h" ++#include ++ ++#define CONFIGURATION_ENTRY_NAME_OFFSET (0xC) ++ ++/** ++ * kbase_csf_firmware_cfg_init - Create the sysfs directory for configuration ++ * options present in firmware image. ++ * ++ * This function would create a sysfs directory and populate it with a ++ * sub-directory, that would contain a file per attribute, for every ++ * configuration option parsed from firmware image. ++ * ++ * @kbdev: Pointer to the Kbase device ++ * ++ * Return: The initialization error code. ++ */ ++int kbase_csf_firmware_cfg_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_csf_firmware_cfg_term - Delete the sysfs directory that was created ++ * for firmware configuration options. ++ * ++ * @kbdev: Pointer to the Kbase device ++ * ++ */ ++void kbase_csf_firmware_cfg_term(struct kbase_device *kbdev); ++ ++/** ++ * kbase_csf_firmware_cfg_option_entry_parse() - Process a ++ * "configuration option" section. ++ * ++ * Read a "configuration option" section adding it to the ++ * kbase_device:csf.firmware_config list. ++ * ++ * Return: 0 if successful, negative error code on failure ++ * ++ * @kbdev: Kbase device structure ++ * @fw: Firmware image containing the section ++ * @entry: Pointer to the section ++ * @size: Size (in bytes) of the section ++ */ ++int kbase_csf_firmware_cfg_option_entry_parse(struct kbase_device *kbdev, ++ const struct firmware *fw, ++ const u32 *entry, unsigned int size); ++#endif /* _KBASE_CSF_FIRMWARE_CFG_H_ */ +diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_no_mali.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_no_mali.c +new file mode 100755 +index 000000000000..7401113c5d6a +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_no_mali.c +@@ -0,0 +1,1012 @@ ++/* ++ * ++ * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include "mali_kbase.h" ++#include "mali_kbase_csf_firmware.h" ++#include "mali_kbase_csf_trace_buffer.h" ++#include "mali_kbase_csf_timeout.h" ++#include "mali_kbase_mem.h" ++#include "mali_kbase_reset_gpu.h" ++#include "device/mali_kbase_device.h" ++#include "backend/gpu/mali_kbase_pm_internal.h" ++#include "mali_kbase_csf_scheduler.h" ++#include "mmu/mali_kbase_mmu.h" ++ ++#include ++#include ++#include ++#include ++#include ++#if (KERNEL_VERSION(4, 13, 0) <= LINUX_VERSION_CODE) ++#include ++#endif ++ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++/* Makes Driver wait indefinitely for an acknowledgment for the different ++ * requests it sends to firmware. Otherwise the timeouts interfere with the ++ * use of debugger for source-level debugging of firmware as Driver initiates ++ * a GPU reset when a request times out, which always happen when a debugger ++ * is connected. ++ */ ++bool fw_debug; /* Default value of 0/false */ ++module_param(fw_debug, bool, 0444); ++MODULE_PARM_DESC(fw_debug, ++ "Enables effective use of a debugger for debugging firmware code."); ++#endif ++ ++#define DUMMY_FW_PAGE_SIZE SZ_4K ++ ++/** ++ * struct dummy_firmware_csi - Represents a dummy interface for MCU firmware streams ++ * ++ * @cs_kernel_input: CS kernel input memory region ++ * @cs_kernel_output: CS kernel output memory region ++ */ ++struct dummy_firmware_csi { ++ u8 cs_kernel_input[DUMMY_FW_PAGE_SIZE]; ++ u8 cs_kernel_output[DUMMY_FW_PAGE_SIZE]; ++}; ++ ++/** ++ * struct dummy_firmware_csg - Represents a dummy interface for MCU firmware stream groups ++ * ++ * @csg_input: CSG kernel input memory region ++ * @csg_output: CSG kernel output memory region ++ * @csi: Dummy firmware CSIs ++ */ ++struct dummy_firmware_csg { ++ u8 csg_input[DUMMY_FW_PAGE_SIZE]; ++ u8 csg_output[DUMMY_FW_PAGE_SIZE]; ++ struct dummy_firmware_csi csi[8]; ++} dummy_firmware_csg; ++ ++/** ++ * struct dummy_firmware_interface - Represents a dummy interface in the MCU firmware ++ * ++ * @global_input: Global input memory region ++ * @global_output: Global output memory region ++ * @csg: Dummy firmware CSGs ++ * @node: Interface objects are on the kbase_device:csf.firmware_interfaces ++ * list using this list_head to link them ++ */ ++struct dummy_firmware_interface { ++ u8 global_input[DUMMY_FW_PAGE_SIZE]; ++ u8 global_output[DUMMY_FW_PAGE_SIZE]; ++ struct dummy_firmware_csg csg[8]; ++ struct list_head node; ++} dummy_firmware_interface; ++ ++#define CSF_GLB_REQ_CFG_MASK \ ++ (GLB_REQ_CFG_ALLOC_EN_MASK | GLB_REQ_CFG_PROGRESS_TIMER_MASK) ++ ++static inline u32 input_page_read(const u32 *const input, const u32 offset) ++{ ++ WARN_ON(offset % sizeof(u32)); ++ ++ return input[offset / sizeof(u32)]; ++} ++ ++static inline void input_page_write(u32 *const input, const u32 offset, ++ const u32 value) ++{ ++ WARN_ON(offset % sizeof(u32)); ++ ++ input[offset / sizeof(u32)] = value; ++} ++ ++static inline void input_page_partial_write(u32 *const input, const u32 offset, ++ u32 value, u32 mask) ++{ ++ WARN_ON(offset % sizeof(u32)); ++ ++ input[offset / sizeof(u32)] = ++ (input_page_read(input, offset) & ~mask) | (value & mask); ++} ++ ++static inline u32 output_page_read(const u32 *const output, const u32 offset) ++{ ++ WARN_ON(offset % sizeof(u32)); ++ ++ return output[offset / sizeof(u32)]; ++} ++ ++static inline void output_page_write(u32 *const output, const u32 offset, ++ const u32 value) ++{ ++ WARN_ON(offset % sizeof(u32)); ++ ++ output[offset / sizeof(u32)] = value; ++} ++ ++/** ++ * invent_memory_setup_entry() - Invent an "interface memory setup" section ++ * ++ * Invent an "interface memory setup" section similar to one from a firmware ++ * image. If successful the interface will be added to the ++ * kbase_device:csf.firmware_interfaces list. ++ * ++ * Return: 0 if successful, negative error code on failure ++ * ++ * @kbdev: Kbase device structure ++ */ ++static int invent_memory_setup_entry(struct kbase_device *kbdev) ++{ ++ struct dummy_firmware_interface *interface = NULL; ++ ++ /* Allocate enough memory for the struct dummy_firmware_interface. ++ */ ++ interface = kmalloc(sizeof(*interface), GFP_KERNEL); ++ if (!interface) ++ return -ENOMEM; ++ ++ kbdev->csf.shared_interface = interface; ++ list_add(&interface->node, &kbdev->csf.firmware_interfaces); ++ ++ /* NO_MALI: Don't insert any firmware pages */ ++ return 0; ++} ++ ++static void free_global_iface(struct kbase_device *kbdev) ++{ ++ struct kbase_csf_global_iface *iface = &kbdev->csf.global_iface; ++ ++ if (iface->groups) { ++ unsigned int gid; ++ ++ for (gid = 0; gid < iface->group_num; ++gid) ++ kfree(iface->groups[gid].streams); ++ ++ kfree(iface->groups); ++ iface->groups = NULL; ++ } ++} ++ ++static int invent_cmd_stream_group_info(struct kbase_device *kbdev, ++ struct kbase_csf_cmd_stream_group_info *ginfo, ++ struct dummy_firmware_csg *csg) ++{ ++ unsigned int sid; ++ ++ ginfo->input = csg->csg_input; ++ ginfo->output = csg->csg_output; ++ ++ ginfo->kbdev = kbdev; ++ ginfo->features = 0; ++ ginfo->suspend_size = 64; ++ ginfo->protm_suspend_size = 64; ++ ginfo->stream_num = ARRAY_SIZE(csg->csi); ++ ginfo->stream_stride = 0; ++ ++ ginfo->streams = kcalloc(ginfo->stream_num, sizeof(*ginfo->streams), GFP_KERNEL); ++ if (ginfo->streams == NULL) { ++ return -ENOMEM; ++ } ++ ++ for (sid = 0; sid < ginfo->stream_num; ++sid) { ++ struct kbase_csf_cmd_stream_info *stream = &ginfo->streams[sid]; ++ struct dummy_firmware_csi *csi = &csg->csi[sid]; ++ ++ stream->input = csi->cs_kernel_input; ++ stream->output = csi->cs_kernel_output; ++ ++ stream->kbdev = kbdev; ++ stream->features = ++ STREAM_FEATURES_WORK_REGISTERS_SET(0, 80) | ++ STREAM_FEATURES_SCOREBOARDS_SET(0, 8) | ++ STREAM_FEATURES_COMPUTE_SET(0, 1) | ++ STREAM_FEATURES_FRAGMENT_SET(0, 1) | ++ STREAM_FEATURES_TILER_SET(0, 1); ++ } ++ ++ return 0; ++} ++ ++static int invent_capabilities(struct kbase_device *kbdev) ++{ ++ struct dummy_firmware_interface *interface = kbdev->csf.shared_interface; ++ struct kbase_csf_global_iface *iface = &kbdev->csf.global_iface; ++ unsigned int gid; ++ ++ iface->input = interface->global_input; ++ iface->output = interface->global_output; ++ ++ iface->version = 1; ++ iface->kbdev = kbdev; ++ iface->features = 0; ++ iface->prfcnt_size = 64; ++ iface->group_num = ARRAY_SIZE(interface->csg); ++ iface->group_stride = 0; ++ ++ iface->groups = kcalloc(iface->group_num, sizeof(*iface->groups), GFP_KERNEL); ++ if (iface->groups == NULL) { ++ return -ENOMEM; ++ } ++ ++ for (gid = 0; gid < iface->group_num; ++gid) { ++ int err; ++ ++ err = invent_cmd_stream_group_info(kbdev, &iface->groups[gid], ++ &interface->csg[gid]); ++ if (err < 0) { ++ free_global_iface(kbdev); ++ return err; ++ } ++ } ++ ++ return 0; ++} ++ ++void kbase_csf_read_firmware_memory(struct kbase_device *kbdev, ++ u32 gpu_addr, u32 *value) ++{ ++ /* NO_MALI: Nothing to do here */ ++} ++ ++ ++void kbase_csf_update_firmware_memory(struct kbase_device *kbdev, ++ u32 gpu_addr, u32 value) ++{ ++ /* NO_MALI: Nothing to do here */ ++} ++ ++void kbase_csf_firmware_cs_input( ++ const struct kbase_csf_cmd_stream_info *const info, const u32 offset, ++ const u32 value) ++{ ++ const struct kbase_device * const kbdev = info->kbdev; ++ ++ dev_dbg(kbdev->dev, "cs input w: reg %08x val %08x\n", offset, value); ++ input_page_write(info->input, offset, value); ++ ++ if (offset == CS_REQ) { ++ /* NO_MALI: Immediately acknowledge requests */ ++ output_page_write(info->output, CS_ACK, value); ++ } ++} ++ ++u32 kbase_csf_firmware_cs_input_read( ++ const struct kbase_csf_cmd_stream_info *const info, ++ const u32 offset) ++{ ++ const struct kbase_device * const kbdev = info->kbdev; ++ u32 const val = input_page_read(info->input, offset); ++ ++ dev_dbg(kbdev->dev, "cs input r: reg %08x val %08x\n", offset, val); ++ return val; ++} ++ ++void kbase_csf_firmware_cs_input_mask( ++ const struct kbase_csf_cmd_stream_info *const info, const u32 offset, ++ const u32 value, const u32 mask) ++{ ++ const struct kbase_device * const kbdev = info->kbdev; ++ ++ dev_dbg(kbdev->dev, "cs input w: reg %08x val %08x mask %08x\n", ++ offset, value, mask); ++ ++ /* NO_MALI: Go through kbase_csf_firmware_cs_input to capture writes */ ++ kbase_csf_firmware_cs_input(info, offset, (input_page_read(info->input, offset) & ~mask) | (value & mask)); ++} ++ ++u32 kbase_csf_firmware_cs_output( ++ const struct kbase_csf_cmd_stream_info *const info, const u32 offset) ++{ ++ const struct kbase_device * const kbdev = info->kbdev; ++ u32 const val = output_page_read(info->output, offset); ++ ++ dev_dbg(kbdev->dev, "cs output r: reg %08x val %08x\n", offset, val); ++ return val; ++} ++ ++void kbase_csf_firmware_csg_input( ++ const struct kbase_csf_cmd_stream_group_info *const info, ++ const u32 offset, const u32 value) ++{ ++ const struct kbase_device * const kbdev = info->kbdev; ++ ++ dev_dbg(kbdev->dev, "csg input w: reg %08x val %08x\n", ++ offset, value); ++ input_page_write(info->input, offset, value); ++ ++ if (offset == CSG_REQ) { ++ /* NO_MALI: Immediately acknowledge requests */ ++ output_page_write(info->output, CSG_ACK, value); ++ } ++} ++ ++u32 kbase_csf_firmware_csg_input_read( ++ const struct kbase_csf_cmd_stream_group_info *const info, ++ const u32 offset) ++{ ++ const struct kbase_device * const kbdev = info->kbdev; ++ u32 const val = input_page_read(info->input, offset); ++ ++ dev_dbg(kbdev->dev, "csg input r: reg %08x val %08x\n", offset, val); ++ return val; ++} ++ ++void kbase_csf_firmware_csg_input_mask( ++ const struct kbase_csf_cmd_stream_group_info *const info, ++ const u32 offset, const u32 value, const u32 mask) ++{ ++ const struct kbase_device * const kbdev = info->kbdev; ++ ++ dev_dbg(kbdev->dev, "csg input w: reg %08x val %08x mask %08x\n", ++ offset, value, mask); ++ ++ /* NO_MALI: Go through kbase_csf_firmware_csg_input to capture writes */ ++ kbase_csf_firmware_csg_input(info, offset, (input_page_read(info->input, offset) & ~mask) | (value & mask)); ++} ++ ++u32 kbase_csf_firmware_csg_output( ++ const struct kbase_csf_cmd_stream_group_info *const info, ++ const u32 offset) ++{ ++ const struct kbase_device * const kbdev = info->kbdev; ++ u32 const val = output_page_read(info->output, offset); ++ ++ dev_dbg(kbdev->dev, "csg output r: reg %08x val %08x\n", offset, val); ++ return val; ++} ++ ++void kbase_csf_firmware_global_input( ++ const struct kbase_csf_global_iface *const iface, const u32 offset, ++ const u32 value) ++{ ++ const struct kbase_device * const kbdev = iface->kbdev; ++ ++ dev_dbg(kbdev->dev, "glob input w: reg %08x val %08x\n", offset, value); ++ input_page_write(iface->input, offset, value); ++ ++ if (offset == GLB_REQ) { ++ /* NO_MALI: Immediately acknowledge requests */ ++ output_page_write(iface->output, GLB_ACK, value); ++ } ++} ++ ++void kbase_csf_firmware_global_input_mask( ++ const struct kbase_csf_global_iface *const iface, const u32 offset, ++ const u32 value, const u32 mask) ++{ ++ const struct kbase_device * const kbdev = iface->kbdev; ++ ++ dev_dbg(kbdev->dev, "glob input w: reg %08x val %08x mask %08x\n", ++ offset, value, mask); ++ ++ /* NO_MALI: Go through kbase_csf_firmware_global_input to capture writes */ ++ kbase_csf_firmware_global_input(iface, offset, (input_page_read(iface->input, offset) & ~mask) | (value & mask)); ++} ++ ++u32 kbase_csf_firmware_global_input_read( ++ const struct kbase_csf_global_iface *const iface, const u32 offset) ++{ ++ const struct kbase_device * const kbdev = iface->kbdev; ++ u32 const val = input_page_read(iface->input, offset); ++ ++ dev_dbg(kbdev->dev, "glob input r: reg %08x val %08x\n", offset, val); ++ return val; ++} ++ ++u32 kbase_csf_firmware_global_output( ++ const struct kbase_csf_global_iface *const iface, const u32 offset) ++{ ++ const struct kbase_device * const kbdev = iface->kbdev; ++ u32 const val = output_page_read(iface->output, offset); ++ ++ dev_dbg(kbdev->dev, "glob output r: reg %08x val %08x\n", offset, val); ++ return val; ++} ++ ++static bool global_request_complete(struct kbase_device *const kbdev, ++ u32 const req_mask) ++{ ++ struct kbase_csf_global_iface *global_iface = ++ &kbdev->csf.global_iface; ++ bool complete = false; ++ unsigned long flags; ++ ++ kbase_csf_scheduler_spin_lock(kbdev, &flags); ++ ++ if ((kbase_csf_firmware_global_output(global_iface, GLB_ACK) & ++ req_mask) == ++ (kbase_csf_firmware_global_input_read(global_iface, GLB_REQ) & ++ req_mask)) ++ complete = true; ++ ++ kbase_csf_scheduler_spin_unlock(kbdev, flags); ++ ++ return complete; ++} ++ ++static int wait_for_global_request(struct kbase_device *const kbdev, ++ u32 const req_mask) ++{ ++ const long wait_timeout = ++ kbase_csf_timeout_in_jiffies(GLB_REQ_WAIT_TIMEOUT_MS); ++ long remaining; ++ int err = 0; ++ ++ remaining = wait_event_timeout(kbdev->csf.event_wait, ++ global_request_complete(kbdev, req_mask), ++ wait_timeout); ++ ++ if (!remaining) { ++ dev_warn(kbdev->dev, "Timed out waiting for global request %x to complete", ++ req_mask); ++ err = -ETIMEDOUT; ++ } ++ ++ return err; ++} ++ ++static void set_global_request( ++ const struct kbase_csf_global_iface *const global_iface, ++ u32 const req_mask) ++{ ++ u32 glb_req; ++ ++ lockdep_assert_held(&global_iface->kbdev->csf.reg_lock); ++ ++ glb_req = kbase_csf_firmware_global_output(global_iface, GLB_ACK); ++ glb_req ^= req_mask; ++ kbase_csf_firmware_global_input_mask(global_iface, GLB_REQ, glb_req, ++ req_mask); ++} ++ ++static void enable_endpoints_global( ++ const struct kbase_csf_global_iface *const global_iface, ++ u64 const shader_core_mask) ++{ ++ kbase_csf_firmware_global_input(global_iface, GLB_ALLOC_EN_LO, ++ shader_core_mask & U32_MAX); ++ kbase_csf_firmware_global_input(global_iface, GLB_ALLOC_EN_HI, ++ shader_core_mask >> 32); ++ ++ set_global_request(global_iface, GLB_REQ_CFG_ALLOC_EN_MASK); ++} ++ ++static void set_timeout_global( ++ const struct kbase_csf_global_iface *const global_iface, ++ u64 const timeout) ++{ ++ kbase_csf_firmware_global_input(global_iface, GLB_PROGRESS_TIMER, ++ timeout / GLB_PROGRESS_TIMER_TIMEOUT_SCALE); ++ ++ set_global_request(global_iface, GLB_REQ_CFG_PROGRESS_TIMER_MASK); ++} ++ ++static void global_init(struct kbase_device *const kbdev, u32 req_mask) ++{ ++ u32 const ack_irq_mask = GLB_ACK_IRQ_MASK_CFG_ALLOC_EN_MASK | ++ GLB_ACK_IRQ_MASK_PING_MASK | ++ GLB_ACK_IRQ_MASK_CFG_PROGRESS_TIMER_MASK | ++ GLB_ACK_IRQ_MASK_PROTM_ENTER_MASK | ++ GLB_ACK_IRQ_MASK_PROTM_EXIT_MASK; ++ ++ const struct kbase_csf_global_iface *const global_iface = ++ &kbdev->csf.global_iface; ++ unsigned long flags; ++ ++ kbase_csf_scheduler_spin_lock(kbdev, &flags); ++ ++ /* Enable endpoints on all present shader cores */ ++ enable_endpoints_global(global_iface, ++ kbase_pm_get_present_cores(kbdev, KBASE_PM_CORE_SHADER)); ++ ++ set_timeout_global(global_iface, kbase_csf_timeout_get(kbdev)); ++ ++ /* Unmask the interrupts */ ++ kbase_csf_firmware_global_input(global_iface, ++ GLB_ACK_IRQ_MASK, ack_irq_mask); ++ ++ kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); ++ ++ kbase_csf_scheduler_spin_unlock(kbdev, flags); ++} ++ ++/** ++ * global_init_on_boot - Sends a global request to control various features. ++ * ++ * @kbdev: Instance of a GPU platform device that implements a command ++ * stream front-end interface. ++ * ++ * Currently only the request to enable endpoints and cycle counter is sent. ++ * ++ * Return: 0 on success, or negative on failure. ++ */ ++static int global_init_on_boot(struct kbase_device *const kbdev) ++{ ++ u32 const req_mask = CSF_GLB_REQ_CFG_MASK; ++ ++ global_init(kbdev, req_mask); ++ ++ return wait_for_global_request(kbdev, req_mask); ++} ++ ++void kbase_csf_firmware_global_reinit(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ kbdev->csf.glb_init_request_pending = true; ++ global_init(kbdev, CSF_GLB_REQ_CFG_MASK); ++} ++ ++bool kbase_csf_firmware_global_reinit_complete(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ WARN_ON(!kbdev->csf.glb_init_request_pending); ++ ++ if (global_request_complete(kbdev, CSF_GLB_REQ_CFG_MASK)) ++ kbdev->csf.glb_init_request_pending = false; ++ ++ return !kbdev->csf.glb_init_request_pending; ++} ++ ++static void kbase_csf_firmware_reload_worker(struct work_struct *work) ++{ ++ struct kbase_device *kbdev = container_of(work, struct kbase_device, ++ csf.firmware_reload_work); ++ unsigned long flags; ++ ++ /* Reboot the firmware */ ++ kbase_csf_firmware_enable_mcu(kbdev); ++ ++ /* Tell MCU state machine to transit to next state */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbdev->csf.firmware_reloaded = true; ++ kbase_pm_update_state(kbdev); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++} ++ ++void kbase_csf_firmware_trigger_reload(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ kbdev->csf.firmware_reloaded = false; ++ ++ if (kbdev->csf.firmware_reload_needed) { ++ kbdev->csf.firmware_reload_needed = false; ++ queue_work(system_wq, &kbdev->csf.firmware_reload_work); ++ } else { ++ kbase_csf_firmware_enable_mcu(kbdev); ++ kbdev->csf.firmware_reloaded = true; ++ } ++} ++ ++void kbase_csf_firmware_reload_completed(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (unlikely(!kbdev->csf.firmware_inited)) ++ return; ++ ++ /* Tell MCU state machine to transit to next state */ ++ kbdev->csf.firmware_reloaded = true; ++ kbase_pm_update_state(kbdev); ++} ++ ++int kbase_csf_firmware_init(struct kbase_device *kbdev) ++{ ++ int ret; ++ ++ if (WARN_ON((kbdev->as_free & MCU_AS_BITMASK) == 0)) ++ return -EINVAL; ++ kbdev->as_free &= ~MCU_AS_BITMASK; ++ ++ ret = kbase_mmu_init(kbdev, &kbdev->csf.mcu_mmu, NULL, ++ BASE_MEM_GROUP_DEFAULT); ++ ++ if (ret != 0) { ++ /* Release the address space */ ++ kbdev->as_free |= MCU_AS_BITMASK; ++ return ret; ++ } ++ ++ init_waitqueue_head(&kbdev->csf.event_wait); ++ kbdev->csf.interrupt_received = false; ++ ++ INIT_LIST_HEAD(&kbdev->csf.firmware_interfaces); ++ INIT_LIST_HEAD(&kbdev->csf.firmware_config); ++ INIT_LIST_HEAD(&kbdev->csf.firmware_trace_buffers.list); ++ INIT_WORK(&kbdev->csf.firmware_reload_work, ++ kbase_csf_firmware_reload_worker); ++ ++ mutex_init(&kbdev->csf.reg_lock); ++ ++ ret = kbase_mcu_shared_interface_region_tracker_init(kbdev); ++ if (ret != 0) { ++ dev_err(kbdev->dev, "Failed to setup the rb tree for managing shared interface segment\n"); ++ goto error; ++ } ++ ++ ret = invent_memory_setup_entry(kbdev); ++ if (ret != 0) { ++ dev_err(kbdev->dev, "Failed to load firmware entry\n"); ++ goto error; ++ } ++ ++ /* Make sure L2 cache is powered up */ ++ kbase_pm_wait_for_l2_powered(kbdev); ++ ++ /* NO_MALI: Don't init trace buffers */ ++ ++ /* NO_MALI: Don't load the MMU tables or boot CSF firmware */ ++ ++ ret = invent_capabilities(kbdev); ++ if (ret != 0) ++ goto error; ++ ++ ret = kbase_csf_doorbell_mapping_init(kbdev); ++ if (ret != 0) ++ goto error; ++ ++ ret = kbase_csf_scheduler_init(kbdev); ++ if (ret != 0) ++ goto error; ++ ++ ret = kbase_csf_timeout_init(kbdev); ++ if (ret != 0) ++ goto error; ++ ++ ret = global_init_on_boot(kbdev); ++ if (ret != 0) ++ goto error; ++ ++ return 0; ++ ++error: ++ kbase_csf_firmware_term(kbdev); ++ return ret; ++} ++ ++void kbase_csf_firmware_term(struct kbase_device *kbdev) ++{ ++ kbase_csf_timeout_term(kbdev); ++ ++ /* NO_MALI: Don't stop firmware or unload MMU tables */ ++ ++ kbase_mmu_term(kbdev, &kbdev->csf.mcu_mmu); ++ ++ kbase_csf_scheduler_term(kbdev); ++ ++ kbase_csf_doorbell_mapping_term(kbdev); ++ ++ free_global_iface(kbdev); ++ ++ /* Release the address space */ ++ kbdev->as_free |= MCU_AS_BITMASK; ++ ++ while (!list_empty(&kbdev->csf.firmware_interfaces)) { ++ struct dummy_firmware_interface *interface; ++ ++ interface = list_first_entry(&kbdev->csf.firmware_interfaces, ++ struct dummy_firmware_interface, node); ++ list_del(&interface->node); ++ ++ /* NO_MALI: No cleanup in dummy interface necessary */ ++ ++ kfree(interface); ++ } ++ ++ /* NO_MALI: No trace buffers to terminate */ ++ ++#ifndef MALI_KBASE_BUILD ++ mali_kutf_fw_utf_entry_cleanup(kbdev); ++#endif ++ ++ mutex_destroy(&kbdev->csf.reg_lock); ++ ++ /* This will also free up the region allocated for the shared interface ++ * entry parsed from the firmware image. ++ */ ++ kbase_mcu_shared_interface_region_tracker_term(kbdev); ++} ++ ++int kbase_csf_firmware_ping(struct kbase_device *const kbdev) ++{ ++ const struct kbase_csf_global_iface *const global_iface = ++ &kbdev->csf.global_iface; ++ unsigned long flags; ++ ++ kbase_csf_scheduler_spin_lock(kbdev, &flags); ++ set_global_request(global_iface, GLB_REQ_PING_MASK); ++ kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); ++ kbase_csf_scheduler_spin_unlock(kbdev, flags); ++ ++ return wait_for_global_request(kbdev, GLB_REQ_PING_MASK); ++} ++ ++int kbase_csf_firmware_set_timeout(struct kbase_device *const kbdev, ++ u64 const timeout) ++{ ++ const struct kbase_csf_global_iface *const global_iface = ++ &kbdev->csf.global_iface; ++ unsigned long flags; ++ int err; ++ ++ /* The 'reg_lock' is also taken and is held till the update is not ++ * complete, to ensure the update of timeout value by multiple Users ++ * gets serialized. ++ */ ++ mutex_lock(&kbdev->csf.reg_lock); ++ kbase_csf_scheduler_spin_lock(kbdev, &flags); ++ set_timeout_global(global_iface, timeout); ++ kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); ++ kbase_csf_scheduler_spin_unlock(kbdev, flags); ++ ++ err = wait_for_global_request(kbdev, GLB_REQ_CFG_PROGRESS_TIMER_MASK); ++ mutex_unlock(&kbdev->csf.reg_lock); ++ ++ return err; ++} ++ ++void kbase_csf_enter_protected_mode(struct kbase_device *kbdev) ++{ ++ struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; ++ unsigned long flags; ++ unsigned int value; ++ ++ kbase_csf_scheduler_spin_lock(kbdev, &flags); ++ value = kbase_csf_firmware_global_output(global_iface, GLB_ACK); ++ value ^= GLB_REQ_PROTM_ENTER_MASK; ++ kbase_csf_firmware_global_input_mask(global_iface, GLB_REQ, value, ++ GLB_REQ_PROTM_ENTER_MASK); ++ dev_dbg(kbdev->dev, "Sending request to enter protected mode"); ++ kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); ++ kbase_csf_scheduler_spin_unlock(kbdev, flags); ++ ++ wait_for_global_request(kbdev, GLB_REQ_PROTM_ENTER_MASK); ++} ++ ++void kbase_csf_firmware_trigger_mcu_halt(struct kbase_device *kbdev) ++{ ++ struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; ++ unsigned long flags; ++ unsigned int value; ++ ++ kbase_csf_scheduler_spin_lock(kbdev, &flags); ++ value = kbase_csf_firmware_global_output(global_iface, GLB_ACK); ++ value ^= GLB_REQ_HALT_MASK; ++ kbase_csf_firmware_global_input_mask(global_iface, GLB_REQ, value, ++ GLB_REQ_HALT_MASK); ++ dev_dbg(kbdev->dev, "Sending request to HALT MCU"); ++ kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); ++ kbase_csf_scheduler_spin_unlock(kbdev, flags); ++} ++ ++/** ++ * copy_grp_and_stm - Copy command stream and/or group data ++ * ++ * @iface: Global command stream front-end interface provided by ++ * the firmware. ++ * @group_data: Pointer where to store all the group data ++ * (sequentially). ++ * @max_group_num: The maximum number of groups to be read. Can be 0, in ++ * which case group_data is unused. ++ * @stream_data: Pointer where to store all the stream data ++ * (sequentially). ++ * @max_total_stream_num: The maximum number of streams to be read. ++ * Can be 0, in which case stream_data is unused. ++ * ++ * Return: Total number of command streams, summed across all groups. ++ */ ++static u32 copy_grp_and_stm( ++ const struct kbase_csf_global_iface * const iface, ++ struct basep_cs_group_control * const group_data, ++ u32 max_group_num, ++ struct basep_cs_stream_control * const stream_data, ++ u32 max_total_stream_num) ++{ ++ u32 i, total_stream_num = 0; ++ ++ if (WARN_ON((max_group_num > 0) && !group_data)) ++ max_group_num = 0; ++ ++ if (WARN_ON((max_total_stream_num > 0) && !stream_data)) ++ max_total_stream_num = 0; ++ ++ for (i = 0; i < iface->group_num; i++) { ++ u32 j; ++ ++ if (i < max_group_num) { ++ group_data[i].features = iface->groups[i].features; ++ group_data[i].stream_num = iface->groups[i].stream_num; ++ } ++ for (j = 0; j < iface->groups[i].stream_num; j++) { ++ if (total_stream_num < max_total_stream_num) ++ stream_data[total_stream_num].features = ++ iface->groups[i].streams[j].features; ++ total_stream_num++; ++ } ++ } ++ ++ return total_stream_num; ++} ++ ++u32 kbase_csf_firmware_get_glb_iface(struct kbase_device *kbdev, ++ struct basep_cs_group_control *const group_data, ++ u32 const max_group_num, ++ struct basep_cs_stream_control *const stream_data, ++ u32 const max_total_stream_num, u32 *const glb_version, ++ u32 *const features, u32 *const group_num, u32 *const prfcnt_size) ++{ ++ const struct kbase_csf_global_iface * const iface = ++ &kbdev->csf.global_iface; ++ ++ if (WARN_ON(!glb_version) || ++ WARN_ON(!features) || ++ WARN_ON(!group_num) || ++ WARN_ON(!prfcnt_size)) ++ return 0; ++ ++ *glb_version = iface->version; ++ *features = iface->features; ++ *group_num = iface->group_num; ++ *prfcnt_size = iface->prfcnt_size; ++ ++ return copy_grp_and_stm(iface, group_data, max_group_num, ++ stream_data, max_total_stream_num); ++} ++ ++const char *kbase_csf_firmware_get_timeline_metadata( ++ struct kbase_device *kbdev, const char *name, size_t *size) ++{ ++ if (WARN_ON(!kbdev) || ++ WARN_ON(!name) || ++ WARN_ON(!size)) { ++ return NULL; ++ } ++ ++ *size = 0; ++ return NULL; ++} ++ ++void kbase_csf_firmware_disable_mcu_wait(struct kbase_device *kbdev) ++{ ++ /* NO_MALI: Nothing to do here */ ++} ++ ++int kbase_csf_firmware_mcu_shared_mapping_init( ++ struct kbase_device *kbdev, ++ unsigned int num_pages, ++ unsigned long cpu_map_properties, ++ unsigned long gpu_map_properties, ++ struct kbase_csf_mapping *csf_mapping) ++{ ++ struct tagged_addr *phys; ++ struct kbase_va_region *va_reg; ++ struct page **page_list; ++ void *cpu_addr; ++ int i, ret = 0; ++ pgprot_t cpu_map_prot = PAGE_KERNEL; ++ unsigned long gpu_map_prot; ++ ++ if (cpu_map_properties & PROT_READ) ++ cpu_map_prot = PAGE_KERNEL_RO; ++ ++ if (kbdev->system_coherency == COHERENCY_ACE) { ++ gpu_map_prot = ++ KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_DEFAULT_ACE); ++ } else { ++ gpu_map_prot = ++ KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_NON_CACHEABLE); ++ cpu_map_prot = pgprot_writecombine(cpu_map_prot); ++ }; ++ ++ phys = kmalloc_array(num_pages, sizeof(*phys), GFP_KERNEL); ++ if (!phys) ++ goto out; ++ ++ page_list = kmalloc_array(num_pages, sizeof(*page_list), GFP_KERNEL); ++ if (!page_list) ++ goto page_list_alloc_error; ++ ++ ret = kbase_mem_pool_alloc_pages( ++ &kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], ++ num_pages, phys, false); ++ if (ret <= 0) ++ goto phys_mem_pool_alloc_error; ++ ++ for (i = 0; i < num_pages; i++) ++ page_list[i] = as_page(phys[i]); ++ ++ cpu_addr = vmap(page_list, num_pages, VM_MAP, cpu_map_prot); ++ if (!cpu_addr) ++ goto vmap_error; ++ ++ va_reg = kbase_alloc_free_region(&kbdev->csf.shared_reg_rbtree, 0, ++ num_pages, KBASE_REG_ZONE_MCU_SHARED); ++ if (!va_reg) ++ goto va_region_alloc_error; ++ ++ mutex_lock(&kbdev->csf.reg_lock); ++ ret = kbase_add_va_region_rbtree(kbdev, va_reg, 0, num_pages, 1); ++ va_reg->flags &= ~KBASE_REG_FREE; ++ mutex_unlock(&kbdev->csf.reg_lock); ++ if (ret) ++ goto va_region_add_error; ++ ++ gpu_map_properties &= (KBASE_REG_GPU_RD | KBASE_REG_GPU_WR); ++ gpu_map_properties |= gpu_map_prot; ++ ++ ret = kbase_mmu_insert_pages_no_flush(kbdev, &kbdev->csf.mcu_mmu, ++ va_reg->start_pfn, &phys[0], num_pages, ++ gpu_map_properties, KBASE_MEM_GROUP_CSF_FW); ++ if (ret) ++ goto mmu_insert_pages_error; ++ ++ kfree(page_list); ++ csf_mapping->phys = phys; ++ csf_mapping->cpu_addr = cpu_addr; ++ csf_mapping->va_reg = va_reg; ++ csf_mapping->num_pages = num_pages; ++ ++ return 0; ++ ++mmu_insert_pages_error: ++ mutex_lock(&kbdev->csf.reg_lock); ++ kbase_remove_va_region(va_reg); ++ mutex_unlock(&kbdev->csf.reg_lock); ++va_region_add_error: ++ kbase_free_alloced_region(va_reg); ++va_region_alloc_error: ++ vunmap(cpu_addr); ++vmap_error: ++ kbase_mem_pool_free_pages( ++ &kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], ++ num_pages, phys, false, false); ++ ++phys_mem_pool_alloc_error: ++ kfree(page_list); ++page_list_alloc_error: ++ kfree(phys); ++out: ++ /* Zero-initialize the mapping to make sure that the termination ++ * function doesn't try to unmap or free random addresses. */ ++ csf_mapping->phys = NULL; ++ csf_mapping->cpu_addr = NULL; ++ csf_mapping->va_reg = NULL; ++ csf_mapping->num_pages = 0; ++ ++ return -ENOMEM; ++} ++ ++void kbase_csf_firmware_mcu_shared_mapping_term( ++ struct kbase_device *kbdev, struct kbase_csf_mapping *csf_mapping) ++{ ++ if (csf_mapping->va_reg) { ++ mutex_lock(&kbdev->csf.reg_lock); ++ kbase_remove_va_region(csf_mapping->va_reg); ++ mutex_unlock(&kbdev->csf.reg_lock); ++ kbase_free_alloced_region(csf_mapping->va_reg); ++ } ++ ++ if (csf_mapping->phys) { ++ kbase_mem_pool_free_pages( ++ &kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], ++ csf_mapping->num_pages, csf_mapping->phys, false, ++ false); ++ } ++ ++ vunmap(csf_mapping->cpu_addr); ++ kfree(csf_mapping->phys); ++} +diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_heap_context_alloc.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_heap_context_alloc.c +new file mode 100755 +index 000000000000..087cc858c2b8 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_heap_context_alloc.c +@@ -0,0 +1,196 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++#include "mali_kbase_csf_heap_context_alloc.h" ++ ++/* Size of one heap context structure, in bytes. */ ++#define HEAP_CTX_SIZE ((size_t)32) ++ ++/* Total size of the GPU memory region allocated for heap contexts, in bytes. */ ++#define HEAP_CTX_REGION_SIZE (MAX_TILER_HEAPS * HEAP_CTX_SIZE) ++ ++/** ++ * sub_alloc - Sub-allocate a heap context from a GPU memory region ++ * ++ * @ctx_alloc: Pointer to the heap context allocator. ++ * ++ * Return: GPU virtual address of the allocated heap context or 0 on failure. ++ */ ++static u64 sub_alloc(struct kbase_csf_heap_context_allocator *const ctx_alloc) ++{ ++ struct kbase_context *const kctx = ctx_alloc->kctx; ++ int heap_nr = 0; ++ size_t ctx_offset = 0; ++ u64 heap_gpu_va = 0; ++ struct kbase_vmap_struct mapping; ++ void *ctx_ptr = NULL; ++ ++ lockdep_assert_held(&ctx_alloc->lock); ++ ++ heap_nr = find_first_zero_bit(ctx_alloc->in_use, ++ MAX_TILER_HEAPS); ++ ++ if (unlikely(heap_nr >= MAX_TILER_HEAPS)) { ++ dev_err(kctx->kbdev->dev, ++ "No free tiler heap contexts in the pool\n"); ++ return 0; ++ } ++ ++ ctx_offset = heap_nr * HEAP_CTX_SIZE; ++ heap_gpu_va = ctx_alloc->gpu_va + ctx_offset; ++ ctx_ptr = kbase_vmap_prot(kctx, heap_gpu_va, ++ HEAP_CTX_SIZE, KBASE_REG_CPU_WR, &mapping); ++ ++ if (unlikely(!ctx_ptr)) { ++ dev_err(kctx->kbdev->dev, ++ "Failed to map tiler heap context %d (0x%llX)\n", ++ heap_nr, heap_gpu_va); ++ return 0; ++ } ++ ++ memset(ctx_ptr, 0, HEAP_CTX_SIZE); ++ kbase_vunmap(ctx_ptr, &mapping); ++ ++ bitmap_set(ctx_alloc->in_use, heap_nr, 1); ++ ++ dev_dbg(kctx->kbdev->dev, "Allocated tiler heap context %d (0x%llX)\n", ++ heap_nr, heap_gpu_va); ++ ++ return heap_gpu_va; ++} ++ ++/** ++ * sub_free - Free a heap context sub-allocated from a GPU memory region ++ * ++ * @ctx_alloc: Pointer to the heap context allocator. ++ * @heap_gpu_va: The GPU virtual address of a heap context structure to free. ++ */ ++static void sub_free(struct kbase_csf_heap_context_allocator *const ctx_alloc, ++ u64 const heap_gpu_va) ++{ ++ struct kbase_context *const kctx = ctx_alloc->kctx; ++ u64 ctx_offset = 0; ++ unsigned int heap_nr = 0; ++ ++ lockdep_assert_held(&ctx_alloc->lock); ++ ++ if (WARN_ON(!ctx_alloc->region)) ++ return; ++ ++ if (WARN_ON(heap_gpu_va < ctx_alloc->gpu_va)) ++ return; ++ ++ ctx_offset = heap_gpu_va - ctx_alloc->gpu_va; ++ ++ if (WARN_ON(ctx_offset >= HEAP_CTX_REGION_SIZE) || ++ WARN_ON(ctx_offset % HEAP_CTX_SIZE)) ++ return; ++ ++ heap_nr = ctx_offset / HEAP_CTX_SIZE; ++ dev_dbg(kctx->kbdev->dev, ++ "Freed tiler heap context %d (0x%llX)\n", heap_nr, heap_gpu_va); ++ ++ bitmap_clear(ctx_alloc->in_use, heap_nr, 1); ++} ++ ++int kbase_csf_heap_context_allocator_init( ++ struct kbase_csf_heap_context_allocator *const ctx_alloc, ++ struct kbase_context *const kctx) ++{ ++ /* We cannot pre-allocate GPU memory here because the ++ * custom VA zone may not have been created yet. ++ */ ++ ctx_alloc->kctx = kctx; ++ ctx_alloc->region = NULL; ++ ctx_alloc->gpu_va = 0; ++ ++ mutex_init(&ctx_alloc->lock); ++ bitmap_zero(ctx_alloc->in_use, MAX_TILER_HEAPS); ++ ++ dev_dbg(kctx->kbdev->dev, ++ "Initialized a tiler heap context allocator\n"); ++ ++ return 0; ++} ++ ++void kbase_csf_heap_context_allocator_term( ++ struct kbase_csf_heap_context_allocator *const ctx_alloc) ++{ ++ struct kbase_context *const kctx = ctx_alloc->kctx; ++ ++ dev_dbg(kctx->kbdev->dev, ++ "Terminating tiler heap context allocator\n"); ++ ++ if (ctx_alloc->region) { ++ kbase_gpu_vm_lock(kctx); ++ ctx_alloc->region->flags &= ~KBASE_REG_NO_USER_FREE; ++ kbase_mem_free_region(kctx, ctx_alloc->region); ++ kbase_gpu_vm_unlock(kctx); ++ } ++ ++ mutex_destroy(&ctx_alloc->lock); ++} ++ ++u64 kbase_csf_heap_context_allocator_alloc( ++ struct kbase_csf_heap_context_allocator *const ctx_alloc) ++{ ++ struct kbase_context *const kctx = ctx_alloc->kctx; ++ u64 flags = BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR | ++ BASE_MEM_PROT_CPU_WR | BASEP_MEM_NO_USER_FREE; ++ u64 nr_pages = PFN_UP(HEAP_CTX_REGION_SIZE); ++ u64 heap_gpu_va = 0; ++ ++#ifdef CONFIG_MALI_VECTOR_DUMP ++ flags |= BASE_MEM_PROT_CPU_RD; ++#endif ++ ++ mutex_lock(&ctx_alloc->lock); ++ ++ /* If the pool of heap contexts wasn't already allocated then ++ * allocate it. ++ */ ++ if (!ctx_alloc->region) { ++ ctx_alloc->region = kbase_mem_alloc(kctx, nr_pages, nr_pages, ++ 0, &flags, &ctx_alloc->gpu_va); ++ } ++ ++ /* If the pool still isn't allocated then an error occurred. */ ++ if (unlikely(!ctx_alloc->region)) { ++ dev_err(kctx->kbdev->dev, "Failed to allocate a pool of tiler heap contexts\n"); ++ } else { ++ heap_gpu_va = sub_alloc(ctx_alloc); ++ } ++ ++ mutex_unlock(&ctx_alloc->lock); ++ ++ return heap_gpu_va; ++} ++ ++void kbase_csf_heap_context_allocator_free( ++ struct kbase_csf_heap_context_allocator *const ctx_alloc, ++ u64 const heap_gpu_va) ++{ ++ mutex_lock(&ctx_alloc->lock); ++ sub_free(ctx_alloc, heap_gpu_va); ++ mutex_unlock(&ctx_alloc->lock); ++} +diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_heap_context_alloc.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_heap_context_alloc.h +new file mode 100755 +index 000000000000..f71ea01ed8c0 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_heap_context_alloc.h +@@ -0,0 +1,76 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++ ++#ifndef _KBASE_CSF_HEAP_CONTEXT_ALLOC_H_ ++#define _KBASE_CSF_HEAP_CONTEXT_ALLOC_H_ ++ ++/** ++ * kbase_csf_heap_context_allocator_init - Initialize an allocator for heap ++ * contexts ++ * @ctx_alloc: Pointer to the heap context allocator to initialize. ++ * @kctx: Pointer to the kbase context. ++ * ++ * Return: 0 if successful or a negative error code on failure. ++ */ ++int kbase_csf_heap_context_allocator_init( ++ struct kbase_csf_heap_context_allocator *const ctx_alloc, ++ struct kbase_context *const kctx); ++ ++/** ++ * kbase_csf_heap_context_allocator_term - Terminate an allocator for heap ++ * contexts ++ * @ctx_alloc: Pointer to the heap context allocator to terminate. ++ */ ++void kbase_csf_heap_context_allocator_term( ++ struct kbase_csf_heap_context_allocator *const ctx_alloc); ++ ++/** ++ * kbase_csf_heap_context_allocator_alloc - Allocate a heap context structure ++ * ++ * If this function is successful then it returns the address of a ++ * zero-initialized heap context structure for use by the firmware. ++ * ++ * @ctx_alloc: Pointer to the heap context allocator. ++ * ++ * Return: GPU virtual address of the allocated heap context or 0 on failure. ++ */ ++u64 kbase_csf_heap_context_allocator_alloc( ++ struct kbase_csf_heap_context_allocator *const ctx_alloc); ++ ++/** ++ * kbase_csf_heap_context_allocator_free - Free a heap context structure ++ * ++ * This function returns a heap context structure to the free pool of unused ++ * contexts for possible reuse by a future call to ++ * @kbase_csf_heap_context_allocator_alloc. ++ * ++ * @ctx_alloc: Pointer to the heap context allocator. ++ * @heap_gpu_va: The GPU virtual address of a heap context structure that ++ * was allocated for the firmware. ++ */ ++void kbase_csf_heap_context_allocator_free( ++ struct kbase_csf_heap_context_allocator *const ctx_alloc, ++ u64 const heap_gpu_va); ++ ++#endif /* _KBASE_CSF_HEAP_CONTEXT_ALLOC_H_ */ +diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_ioctl.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_ioctl.h +new file mode 100755 +index 000000000000..e9bb8d299754 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_ioctl.h +@@ -0,0 +1,379 @@ ++/* ++ * ++ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_CSF_IOCTL_H_ ++#define _KBASE_CSF_IOCTL_H_ ++ ++#include ++#include ++ ++/* ++ * 1.0: ++ * - CSF IOCTL header separated from JM ++ */ ++ ++#define BASE_UK_VERSION_MAJOR 1 ++#define BASE_UK_VERSION_MINOR 0 ++ ++/** ++ * struct kbase_ioctl_version_check - Check version compatibility between ++ * kernel and userspace ++ * ++ * @major: Major version number ++ * @minor: Minor version number ++ */ ++struct kbase_ioctl_version_check { ++ __u16 major; ++ __u16 minor; ++}; ++ ++#define KBASE_IOCTL_VERSION_CHECK \ ++ _IOWR(KBASE_IOCTL_TYPE, 52, struct kbase_ioctl_version_check) ++ ++#define KBASE_IOCTL_VERSION_CHECK_RESERVED \ ++ _IOWR(KBASE_IOCTL_TYPE, 0, struct kbase_ioctl_version_check) ++ ++ ++/** ++ * struct kbase_ioctl_cs_queue_register - Register a GPU command queue with the ++ * base back-end ++ * ++ * @buffer_gpu_addr: GPU address of the buffer backing the queue ++ * @buffer_size: Size of the buffer in bytes ++ * @priority: Priority of the queue within a group when run within a process ++ * @padding: Currently unused, must be zero ++ */ ++struct kbase_ioctl_cs_queue_register { ++ __u64 buffer_gpu_addr; ++ __u32 buffer_size; ++ __u8 priority; ++ __u8 padding[3]; ++}; ++ ++#define KBASE_IOCTL_CS_QUEUE_REGISTER \ ++ _IOW(KBASE_IOCTL_TYPE, 36, struct kbase_ioctl_cs_queue_register) ++ ++/** ++ * struct kbase_ioctl_cs_queue_kick - Kick the GPU command queue group scheduler ++ * to notify that a queue has been updated ++ * ++ * @buffer_gpu_addr: GPU address of the buffer backing the queue ++ */ ++struct kbase_ioctl_cs_queue_kick { ++ __u64 buffer_gpu_addr; ++}; ++ ++#define KBASE_IOCTL_CS_QUEUE_KICK \ ++ _IOW(KBASE_IOCTL_TYPE, 37, struct kbase_ioctl_cs_queue_kick) ++ ++/** ++ * union kbase_ioctl_cs_queue_bind - Bind a GPU command queue to a group ++ * ++ * @buffer_gpu_addr: GPU address of the buffer backing the queue ++ * @group_handle: Handle of the group to which the queue should be bound ++ * @csi_index: Index of the CSF interface the queue should be bound to ++ * @padding: Currently unused, must be zero ++ * @mmap_handle: Handle to be used for creating the mapping of command stream ++ * input/output pages ++ * ++ * @in: Input parameters ++ * @out: Output parameters ++ * ++ */ ++union kbase_ioctl_cs_queue_bind { ++ struct { ++ __u64 buffer_gpu_addr; ++ __u8 group_handle; ++ __u8 csi_index; ++ __u8 padding[6]; ++ } in; ++ struct { ++ __u64 mmap_handle; ++ } out; ++}; ++ ++#define KBASE_IOCTL_CS_QUEUE_BIND \ ++ _IOWR(KBASE_IOCTL_TYPE, 39, union kbase_ioctl_cs_queue_bind) ++ ++/* ioctl 40 is free to use */ ++ ++/** ++ * struct kbase_ioctl_cs_queue_terminate - Terminate a GPU command queue ++ * ++ * @buffer_gpu_addr: GPU address of the buffer backing the queue ++ */ ++struct kbase_ioctl_cs_queue_terminate { ++ __u64 buffer_gpu_addr; ++}; ++ ++#define KBASE_IOCTL_CS_QUEUE_TERMINATE \ ++ _IOW(KBASE_IOCTL_TYPE, 41, struct kbase_ioctl_cs_queue_terminate) ++ ++/** ++ * union kbase_ioctl_cs_queue_group_create - Create a GPU command queue group ++ * ++ * @tiler_mask: Mask of tiler endpoints the group is allowed to use. ++ * @fragment_mask: Mask of fragment endpoints the group is allowed to use. ++ * @compute_mask: Mask of compute endpoints the group is allowed to use. ++ * @cs_min: Minimum number of command streams required. ++ * @priority: Queue group's priority within a process. ++ * @tiler_max: Maximum number of tiler endpoints the group is allowed ++ * to use. ++ * @fragment_max: Maximum number of fragment endpoints the group is ++ * allowed to use. ++ * @compute_max: Maximum number of compute endpoints the group is allowed ++ * to use. ++ * @padding: Currently unused, must be zero ++ * @group_handle: Handle of a newly created queue group. ++ * ++ * @in: Input parameters ++ * @out: Output parameters ++ * ++ */ ++union kbase_ioctl_cs_queue_group_create { ++ struct { ++ __u64 tiler_mask; ++ __u64 fragment_mask; ++ __u64 compute_mask; ++ __u8 cs_min; ++ __u8 priority; ++ __u8 tiler_max; ++ __u8 fragment_max; ++ __u8 compute_max; ++ __u8 padding[3]; ++ ++ } in; ++ struct { ++ __u8 group_handle; ++ __u8 padding[7]; ++ } out; ++}; ++ ++#define KBASE_IOCTL_CS_QUEUE_GROUP_CREATE \ ++ _IOWR(KBASE_IOCTL_TYPE, 42, union kbase_ioctl_cs_queue_group_create) ++ ++/** ++ * struct kbase_ioctl_cs_queue_group_term - Terminate a GPU command queue group ++ * ++ * @group_handle: Handle of the queue group to be terminated ++ * @padding: Padding to round up to a multiple of 8 bytes, must be zero ++ */ ++struct kbase_ioctl_cs_queue_group_term { ++ __u8 group_handle; ++ __u8 padding[7]; ++}; ++ ++#define KBASE_IOCTL_CS_QUEUE_GROUP_TERMINATE \ ++ _IOW(KBASE_IOCTL_TYPE, 43, struct kbase_ioctl_cs_queue_group_term) ++ ++#define KBASE_IOCTL_CS_EVENT_SIGNAL \ ++ _IO(KBASE_IOCTL_TYPE, 44) ++ ++typedef __u8 base_kcpu_queue_id; /* We support up to 256 active KCPU queues */ ++ ++/** ++ * struct kbase_ioctl_kcpu_queue_new - Create a KCPU command queue ++ * ++ * @id: ID of the new command queue returned by the kernel ++ * @padding: Padding to round up to a multiple of 8 bytes, must be zero ++ */ ++struct kbase_ioctl_kcpu_queue_new { ++ base_kcpu_queue_id id; ++ __u8 padding[7]; ++}; ++ ++#define KBASE_IOCTL_KCPU_QUEUE_CREATE \ ++ _IOR(KBASE_IOCTL_TYPE, 45, struct kbase_ioctl_kcpu_queue_new) ++ ++/** ++ * struct kbase_ioctl_kcpu_queue_delete - Destroy a KCPU command queue ++ * ++ * @id: ID of the command queue to be destroyed ++ * @padding: Padding to round up to a multiple of 8 bytes, must be zero ++ */ ++struct kbase_ioctl_kcpu_queue_delete { ++ base_kcpu_queue_id id; ++ __u8 padding[7]; ++}; ++ ++#define KBASE_IOCTL_KCPU_QUEUE_DELETE \ ++ _IOW(KBASE_IOCTL_TYPE, 46, struct kbase_ioctl_kcpu_queue_delete) ++ ++/** ++ * struct kbase_ioctl_kcpu_queue_enqueue - Enqueue commands into the KCPU queue ++ * ++ * @addr: Memory address of an array of struct base_kcpu_queue_command ++ * @nr_commands: Number of commands in the array ++ * @id: kcpu queue identifier, returned by KBASE_IOCTL_KCPU_QUEUE_CREATE ioctl ++ * @padding: Padding to round up to a multiple of 8 bytes, must be zero ++ */ ++struct kbase_ioctl_kcpu_queue_enqueue { ++ __u64 addr; ++ __u32 nr_commands; ++ base_kcpu_queue_id id; ++ __u8 padding[3]; ++}; ++ ++#define KBASE_IOCTL_KCPU_QUEUE_ENQUEUE \ ++ _IOW(KBASE_IOCTL_TYPE, 47, struct kbase_ioctl_kcpu_queue_enqueue) ++ ++/** ++ * union kbase_ioctl_cs_tiler_heap_init - Initialize chunked tiler memory heap ++ * ++ * @chunk_size: Size of each chunk. ++ * @initial_chunks: Initial number of chunks that heap will be created with. ++ * @max_chunks: Maximum number of chunks that the heap is allowed to use. ++ * @target_in_flight: Number of render-passes that the driver should attempt to ++ * keep in flight for which allocation of new chunks is ++ * allowed. ++ * @group_id: Group ID to be used for physical allocations. ++ * @gpu_heap_va: GPU VA (virtual address) of Heap context that was set up for ++ * the heap. ++ * @first_chunk_va: GPU VA of the first chunk allocated for the heap, actually ++ * points to the header of heap chunk and not to the low ++ * address of free memory in the chunk. ++ * ++ * @in: Input parameters ++ * @out: Output parameters ++ * ++ */ ++union kbase_ioctl_cs_tiler_heap_init { ++ struct { ++ __u32 chunk_size; ++ __u32 initial_chunks; ++ __u32 max_chunks; ++ __u16 target_in_flight; ++ __u8 group_id; ++ __u8 padding; ++ } in; ++ struct { ++ __u64 gpu_heap_va; ++ __u64 first_chunk_va; ++ } out; ++}; ++ ++#define KBASE_IOCTL_CS_TILER_HEAP_INIT \ ++ _IOWR(KBASE_IOCTL_TYPE, 48, union kbase_ioctl_cs_tiler_heap_init) ++ ++/** ++ * struct kbase_ioctl_cs_tiler_heap_term - Terminate a chunked tiler heap ++ * instance ++ * ++ * @gpu_heap_va: GPU VA of Heap context that was set up for the heap. ++ */ ++struct kbase_ioctl_cs_tiler_heap_term { ++ __u64 gpu_heap_va; ++}; ++ ++#define KBASE_IOCTL_CS_TILER_HEAP_TERM \ ++ _IOW(KBASE_IOCTL_TYPE, 49, struct kbase_ioctl_cs_tiler_heap_term) ++ ++/** ++ * union kbase_ioctl_cs_get_glb_iface - Request the global control block ++ * of CSF interface capabilities ++ * ++ * @max_group_num: The maximum number of groups to be read. Can be 0, in ++ * which case groups_ptr is unused. ++ * @max_total_stream_num: The maximum number of streams to be read. Can be 0, in ++ * which case streams_ptr is unused. ++ * @groups_ptr: Pointer where to store all the group data (sequentially). ++ * @streams_ptr: Pointer where to store all the stream data (sequentially). ++ * @glb_version: Global interface version. Bits 31:16 hold the major ++ * version number and 15:0 hold the minor version number. ++ * A higher minor version is backwards-compatible with a ++ * lower minor version for the same major version. ++ * @features: Bit mask of features (e.g. whether certain types of job ++ * can be suspended). ++ * @group_num: Number of command stream groups supported. ++ * @prfcnt_size: Size of CSF performance counters, in bytes. Bits 31:16 ++ * hold the size of firmware performance counter data ++ * and 15:0 hold the size of hardware performance counter ++ * data. ++ * @total_stream_num: Total number of command streams, summed across all groups. ++ * @padding: Will be zeroed. ++ * ++ * @in: Input parameters ++ * @out: Output parameters ++ * ++ */ ++union kbase_ioctl_cs_get_glb_iface { ++ struct { ++ __u32 max_group_num; ++ __u32 max_total_stream_num; ++ __u64 groups_ptr; ++ __u64 streams_ptr; ++ } in; ++ struct { ++ __u32 glb_version; ++ __u32 features; ++ __u32 group_num; ++ __u32 prfcnt_size; ++ __u32 total_stream_num; ++ __u32 padding; ++ } out; ++}; ++ ++#define KBASE_IOCTL_CS_GET_GLB_IFACE \ ++ _IOWR(KBASE_IOCTL_TYPE, 51, union kbase_ioctl_cs_get_glb_iface) ++ ++/*************** ++ * test ioctls * ++ ***************/ ++#if MALI_UNIT_TEST ++/* These ioctls are purely for test purposes and are not used in the production ++ * driver, they therefore may change without notice ++ */ ++ ++/** ++ * struct kbase_ioctl_cs_event_memory_write - Write an event memory address ++ * @cpu_addr: Memory address to write ++ * @value: Value to write ++ * @padding: Currently unused, must be zero ++ */ ++struct kbase_ioctl_cs_event_memory_write { ++ __u64 cpu_addr; ++ __u8 value; ++ __u8 padding[7]; ++}; ++ ++/** ++ * union kbase_ioctl_cs_event_memory_read - Read an event memory address ++ * @cpu_addr: Memory address to read ++ * @value: Value read ++ * @padding: Currently unused, must be zero ++ * ++ * @in: Input parameters ++ * @out: Output parameters ++ */ ++union kbase_ioctl_cs_event_memory_read { ++ struct { ++ __u64 cpu_addr; ++ } in; ++ struct { ++ __u8 value; ++ __u8 padding[7]; ++ } out; ++}; ++ ++#endif /* MALI_UNIT_TEST */ ++ ++#endif /* _KBASE_CSF_IOCTL_H_ */ +diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.c +new file mode 100755 +index 000000000000..e1263d535918 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.c +@@ -0,0 +1,1737 @@ ++/* ++ * ++ * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++#include ++#include ++#include "device/mali_kbase_device.h" ++#include "mali_kbase_csf.h" ++#include ++ ++#ifdef CONFIG_SYNC_FILE ++#include "mali_kbase_fence.h" ++#include "mali_kbase_sync.h" ++ ++static DEFINE_SPINLOCK(kbase_csf_fence_lock); ++#endif ++ ++static void kcpu_queue_process(struct kbase_kcpu_command_queue *kcpu_queue, ++ bool ignore_waits); ++ ++static void kcpu_queue_process_worker(struct work_struct *data); ++ ++static int kbase_kcpu_map_import_prepare( ++ struct kbase_kcpu_command_queue *kcpu_queue, ++ struct base_kcpu_command_import_info *import_info, ++ struct kbase_kcpu_command *current_command) ++{ ++ struct kbase_context *const kctx = kcpu_queue->kctx; ++ struct kbase_va_region *reg; ++ int ret = 0; ++ ++ lockdep_assert_held(&kctx->csf.kcpu_queues.lock); ++ ++ /* Take the processes mmap lock */ ++ down_read(kbase_mem_get_process_mmap_lock()); ++ kbase_gpu_vm_lock(kctx); ++ ++ reg = kbase_region_tracker_find_region_enclosing_address(kctx, ++ import_info->handle); ++ ++ if (kbase_is_region_invalid_or_free(reg) || ++ !kbase_mem_is_imported(reg->gpu_alloc->type)) { ++ ret = -EINVAL; ++ goto out; ++ } ++ ++ if (reg->gpu_alloc->type == KBASE_MEM_TYPE_IMPORTED_USER_BUF) { ++ /* Pin the physical pages backing the user buffer while ++ * we are in the process context and holding the mmap lock. ++ * The dma mapping & GPU mapping of the pages would be done ++ * when the MAP_IMPORT operation is executed. ++ * ++ * Though the pages would be pinned, no reference is taken ++ * on the physical pages tracking object. When the last ++ * reference to the tracking object is dropped the pages ++ * would be unpinned if they weren't unpinned before. ++ */ ++ ret = kbase_jd_user_buf_pin_pages(kctx, reg); ++ if (ret) ++ goto out; ++ } ++ ++ current_command->type = BASE_KCPU_COMMAND_TYPE_MAP_IMPORT; ++ current_command->info.import.gpu_va = import_info->handle; ++ ++out: ++ kbase_gpu_vm_unlock(kctx); ++ /* Release the processes mmap lock */ ++ up_read(kbase_mem_get_process_mmap_lock()); ++ ++ return ret; ++} ++ ++static int kbase_kcpu_unmap_import_prepare_internal( ++ struct kbase_kcpu_command_queue *kcpu_queue, ++ struct base_kcpu_command_import_info *import_info, ++ struct kbase_kcpu_command *current_command, ++ enum base_kcpu_command_type type) ++{ ++ struct kbase_context *const kctx = kcpu_queue->kctx; ++ struct kbase_va_region *reg; ++ int ret = 0; ++ ++ lockdep_assert_held(&kctx->csf.kcpu_queues.lock); ++ ++ kbase_gpu_vm_lock(kctx); ++ ++ reg = kbase_region_tracker_find_region_enclosing_address(kctx, ++ import_info->handle); ++ ++ if (kbase_is_region_invalid_or_free(reg) || ++ !kbase_mem_is_imported(reg->gpu_alloc->type)) { ++ ret = -EINVAL; ++ goto out; ++ } ++ ++ if (reg->gpu_alloc->type == KBASE_MEM_TYPE_IMPORTED_USER_BUF) { ++ /* The pages should have been pinned when MAP_IMPORT ++ * was enqueued previously. ++ */ ++ if (reg->gpu_alloc->nents != ++ reg->gpu_alloc->imported.user_buf.nr_pages) { ++ ret = -EINVAL; ++ goto out; ++ } ++ } ++ ++ current_command->type = type; ++ current_command->info.import.gpu_va = import_info->handle; ++ ++out: ++ kbase_gpu_vm_unlock(kctx); ++ ++ return ret; ++} ++ ++static int kbase_kcpu_unmap_import_prepare( ++ struct kbase_kcpu_command_queue *kcpu_queue, ++ struct base_kcpu_command_import_info *import_info, ++ struct kbase_kcpu_command *current_command) ++{ ++ return kbase_kcpu_unmap_import_prepare_internal(kcpu_queue, ++ import_info, current_command, ++ BASE_KCPU_COMMAND_TYPE_UNMAP_IMPORT); ++} ++ ++static int kbase_kcpu_unmap_import_force_prepare( ++ struct kbase_kcpu_command_queue *kcpu_queue, ++ struct base_kcpu_command_import_info *import_info, ++ struct kbase_kcpu_command *current_command) ++{ ++ return kbase_kcpu_unmap_import_prepare_internal(kcpu_queue, ++ import_info, current_command, ++ BASE_KCPU_COMMAND_TYPE_UNMAP_IMPORT_FORCE); ++} ++ ++/** ++ * kbase_jit_add_to_pending_alloc_list() - Pend JIT allocation ++ * ++ * @queue: The queue containing this JIT allocation ++ * @cmd: The JIT allocation that is blocking this queue ++ */ ++static void kbase_jit_add_to_pending_alloc_list( ++ struct kbase_kcpu_command_queue *queue, ++ struct kbase_kcpu_command *cmd) ++{ ++ struct kbase_context *const kctx = queue->kctx; ++ struct list_head *target_list_head = ++ &kctx->csf.kcpu_queues.jit_blocked_queues; ++ struct kbase_kcpu_command_queue *blocked_queue; ++ ++ lockdep_assert_held(&kctx->csf.kcpu_queues.lock); ++ ++ list_for_each_entry(blocked_queue, ++ &kctx->csf.kcpu_queues.jit_blocked_queues, ++ jit_blocked) { ++ struct kbase_kcpu_command const*const jit_alloc_cmd = ++ &blocked_queue->commands[blocked_queue->start_offset]; ++ ++ WARN_ON(jit_alloc_cmd->type != BASE_KCPU_COMMAND_TYPE_JIT_ALLOC); ++ if (cmd->enqueue_ts < jit_alloc_cmd->enqueue_ts) { ++ target_list_head = &blocked_queue->jit_blocked; ++ break; ++ } ++ } ++ ++ list_add_tail(&queue->jit_blocked, target_list_head); ++} ++ ++/** ++ * kbase_kcpu_jit_allocate_process() - Process JIT allocation ++ * ++ * @queue: The queue containing this JIT allocation ++ * @cmd: The JIT allocation command ++ */ ++static int kbase_kcpu_jit_allocate_process( ++ struct kbase_kcpu_command_queue *queue, ++ struct kbase_kcpu_command *cmd) ++{ ++ struct kbase_context *const kctx = queue->kctx; ++ struct kbase_kcpu_command_jit_alloc_info *alloc_info = ++ &cmd->info.jit_alloc; ++ struct base_jit_alloc_info *info = alloc_info->info; ++ struct kbase_vmap_struct mapping; ++ struct kbase_va_region *reg; ++ u32 count = alloc_info->count; ++ u64 *ptr, new_addr; ++ u32 i; ++ int ret; ++ ++ lockdep_assert_held(&kctx->csf.kcpu_queues.lock); ++ ++ if (alloc_info->blocked) { ++ list_del(&queue->jit_blocked); ++ alloc_info->blocked = false; ++ } ++ ++ if (WARN_ON(!info)) ++ return -EINVAL; ++ ++ /* Check if all JIT IDs are not in use */ ++ for (i = 0; i < count; i++, info++) { ++ /* The JIT ID is still in use so fail the allocation */ ++ if (kctx->jit_alloc[info->id]) { ++ dev_warn(kctx->kbdev->dev, "JIT ID still in use\n"); ++ return -EINVAL; ++ } ++ } ++ ++ /* Now start the allocation loop */ ++ for (i = 0, info = alloc_info->info; i < count; i++, info++) { ++ if (kctx->jit_alloc[info->id]) { ++ /* The JIT ID is duplicated in this command. Roll back ++ * previous allocations and fail. ++ */ ++ dev_warn(kctx->kbdev->dev, "JIT ID is duplicated\n"); ++ ret = -EINVAL; ++ goto fail; ++ } ++ ++ /* Create a JIT allocation */ ++ reg = kbase_jit_allocate(kctx, info, true); ++ if (!reg) { ++ bool can_block = false; ++ struct kbase_kcpu_command const *jit_cmd; ++ ++ list_for_each_entry(jit_cmd, &kctx->csf.kcpu_queues.jit_cmds_head, info.jit_alloc.node) { ++ if (jit_cmd == cmd) ++ break; ++ ++ if (jit_cmd->type == BASE_KCPU_COMMAND_TYPE_JIT_FREE) { ++ u8 const*const free_ids = jit_cmd->info.jit_free.ids; ++ ++ if (free_ids && *free_ids && kctx->jit_alloc[*free_ids]) { ++ /** ++ * A JIT free which is active ++ * and submitted before this ++ * command. ++ */ ++ can_block = true; ++ break; ++ } ++ } ++ } ++ ++ if (!can_block) { ++ /** ++ * No prior JIT_FREE command is active. Roll ++ * back previous allocations and fail. ++ */ ++ dev_warn_ratelimited(kctx->kbdev->dev, "JIT alloc command failed: %p\n", cmd); ++ ret = -ENOMEM; ++ goto fail; ++ } ++ ++ /* There are pending frees for an active allocation ++ * so we should wait to see whether they free the ++ * memory. Add to the list of atoms for which JIT ++ * allocation is pending. ++ */ ++ kbase_jit_add_to_pending_alloc_list(queue, cmd); ++ alloc_info->blocked = true; ++ ++ /* Rollback, the whole set will be re-attempted */ ++ while (i-- > 0) { ++ info--; ++ kbase_jit_free(kctx, kctx->jit_alloc[info->id]); ++ kctx->jit_alloc[info->id] = NULL; ++ } ++ ++ return -EAGAIN; ++ } ++ ++ /* Bind it to the user provided ID. */ ++ kctx->jit_alloc[info->id] = reg; ++ } ++ ++ for (i = 0, info = alloc_info->info; i < count; i++, info++) { ++ /* ++ * Write the address of the JIT allocation to the user provided ++ * GPU allocation. ++ */ ++ ptr = kbase_vmap(kctx, info->gpu_alloc_addr, sizeof(*ptr), ++ &mapping); ++ if (!ptr) { ++ ret = -ENOMEM; ++ goto fail; ++ } ++ ++ reg = kctx->jit_alloc[info->id]; ++ new_addr = reg->start_pfn << PAGE_SHIFT; ++ *ptr = new_addr; ++ kbase_vunmap(kctx, &mapping); ++ } ++ ++ return 0; ++ ++fail: ++ /* Roll back completely */ ++ for (i = 0, info = alloc_info->info; i < count; i++, info++) { ++ /* Free the allocations that were successful. ++ * Mark all the allocations including the failed one and the ++ * other un-attempted allocations in the set, so we know they ++ * are in use. ++ */ ++ if (kctx->jit_alloc[info->id]) ++ kbase_jit_free(kctx, kctx->jit_alloc[info->id]); ++ ++ kctx->jit_alloc[info->id] = KBASE_RESERVED_REG_JIT_ALLOC; ++ } ++ ++ return ret; ++} ++ ++static int kbase_kcpu_jit_allocate_prepare( ++ struct kbase_kcpu_command_queue *kcpu_queue, ++ struct base_kcpu_command_jit_alloc_info *alloc_info, ++ struct kbase_kcpu_command *current_command) ++{ ++ struct kbase_context *const kctx = kcpu_queue->kctx; ++ void __user *data = u64_to_user_ptr(alloc_info->info); ++ struct base_jit_alloc_info *info; ++ u32 count = alloc_info->count; ++ int ret = 0; ++ u32 i; ++ ++ lockdep_assert_held(&kctx->csf.kcpu_queues.lock); ++ ++ if (!data || count > kcpu_queue->kctx->jit_max_allocations || ++ count > ARRAY_SIZE(kctx->jit_alloc)) { ++ ret = -EINVAL; ++ goto out; ++ } ++ ++ info = kmalloc_array(count, sizeof(*info), GFP_KERNEL); ++ if (!info) { ++ ret = -ENOMEM; ++ goto out; ++ } ++ ++ if (copy_from_user(info, data, sizeof(*info) * count) != 0) { ++ ret = -EINVAL; ++ goto out_free; ++ } ++ ++ for (i = 0; i < count; i++) { ++ ret = kbasep_jit_alloc_validate(kctx, &info[i]); ++ if (ret) ++ goto out_free; ++ } ++ ++ current_command->type = BASE_KCPU_COMMAND_TYPE_JIT_ALLOC; ++ list_add_tail(¤t_command->info.jit_alloc.node, ++ &kctx->csf.kcpu_queues.jit_cmds_head); ++ current_command->info.jit_alloc.info = info; ++ current_command->info.jit_alloc.count = count; ++ current_command->info.jit_alloc.blocked = false; ++ ++ return 0; ++out_free: ++ kfree(info); ++out: ++ return ret; ++} ++ ++/** ++ * kbase_kcpu_jit_allocate_finish() - Finish handling the JIT_ALLOC command ++ * ++ * @queue: The queue containing this JIT allocation ++ * @cmd: The JIT allocation command ++ */ ++static void kbase_kcpu_jit_allocate_finish( ++ struct kbase_kcpu_command_queue *queue, ++ struct kbase_kcpu_command *cmd) ++{ ++ lockdep_assert_held(&queue->kctx->csf.kcpu_queues.lock); ++ ++ /* Remove this command from the jit_cmds_head list */ ++ list_del(&cmd->info.jit_alloc.node); ++ ++ /** ++ * If we get to this point we must have already cleared the blocked ++ * flag, otherwise it'd be a bug. ++ */ ++ if (WARN_ON(cmd->info.jit_alloc.blocked)) { ++ list_del(&queue->jit_blocked); ++ cmd->info.jit_alloc.blocked = false; ++ } ++ ++ kfree(cmd->info.jit_alloc.info); ++} ++ ++/** ++ * kbase_kcpu_jit_retry_pending_allocs() - Retry blocked JIT_ALLOC commands ++ * ++ * @kctx: The context containing the blocked JIT_ALLOC commands ++ */ ++static void kbase_kcpu_jit_retry_pending_allocs(struct kbase_context *kctx) ++{ ++ struct kbase_kcpu_command_queue *blocked_queue; ++ ++ lockdep_assert_held(&kctx->csf.kcpu_queues.lock); ++ ++ /** ++ * Reschedule all queues blocked by JIT_ALLOC commands. ++ * NOTE: This code traverses the list of blocked queues directly. It ++ * only works as long as the queued works are not executed at the same ++ * time. This precondition is true since we're holding the ++ * kbase_csf_kcpu_queue_context.lock . ++ */ ++ list_for_each_entry(blocked_queue, ++ &kctx->csf.kcpu_queues.jit_blocked_queues, jit_blocked) ++ queue_work(kctx->csf.kcpu_queues.wq, &blocked_queue->work); ++} ++ ++static int kbase_kcpu_jit_free_process(struct kbase_context *kctx, ++ struct kbase_kcpu_command *const cmd) ++{ ++ struct kbase_kcpu_command_jit_free_info *const free_info = ++ &cmd->info.jit_free; ++ u8 *ids = free_info->ids; ++ u32 count = free_info->count; ++ u32 i; ++ ++ if (WARN_ON(!ids)) ++ return -EINVAL; ++ ++ lockdep_assert_held(&kctx->csf.kcpu_queues.lock); ++ ++ for (i = 0; i < count; i++, ids++) { ++ if ((*ids == 0) || (kctx->jit_alloc[*ids] == NULL)) { ++ dev_warn(kctx->kbdev->dev, "invalid JIT free ID\n"); ++ } else { ++ /* If the ID is valid but the allocation request ++ * failed, still succeed this command but don't ++ * try and free the allocation. ++ */ ++ if (kctx->jit_alloc[*ids] != ++ KBASE_RESERVED_REG_JIT_ALLOC) ++ kbase_jit_free(kctx, kctx->jit_alloc[*ids]); ++ ++ kctx->jit_alloc[*ids] = NULL; ++ } ++ } ++ ++ /* Free the list of ids */ ++ kfree(free_info->ids); ++ ++ /** ++ * Remove this command from the jit_cmds_head list and retry pending ++ * allocations. ++ */ ++ list_del(&cmd->info.jit_free.node); ++ kbase_kcpu_jit_retry_pending_allocs(kctx); ++ ++ return 0; ++} ++ ++static int kbase_kcpu_jit_free_prepare( ++ struct kbase_kcpu_command_queue *kcpu_queue, ++ struct base_kcpu_command_jit_free_info *free_info, ++ struct kbase_kcpu_command *current_command) ++{ ++ struct kbase_context *const kctx = kcpu_queue->kctx; ++ void __user *data = u64_to_user_ptr(free_info->ids); ++ u8 *ids; ++ u32 count = free_info->count; ++ int ret; ++ u32 i; ++ ++ lockdep_assert_held(&kctx->csf.kcpu_queues.lock); ++ ++ /* Sanity checks */ ++ if (!count || count > ARRAY_SIZE(kctx->jit_alloc)) { ++ ret = -EINVAL; ++ goto out; ++ } ++ ++ /* Copy the information for safe access and future storage */ ++ ids = kmalloc_array(count, sizeof(*ids), GFP_KERNEL); ++ if (!ids) { ++ ret = -ENOMEM; ++ goto out; ++ } ++ ++ if (!data) { ++ ret = -EINVAL; ++ goto out_free; ++ } ++ ++ if (copy_from_user(ids, data, sizeof(*ids) * count)) { ++ ret = -EINVAL; ++ goto out_free; ++ } ++ ++ for (i = 0; i < count; i++) { ++ /* Fail the command if ID sent is zero */ ++ if (!ids[i]) { ++ ret = -EINVAL; ++ goto out_free; ++ } ++ } ++ ++ current_command->type = BASE_KCPU_COMMAND_TYPE_JIT_FREE; ++ list_add_tail(¤t_command->info.jit_free.node, ++ &kctx->csf.kcpu_queues.jit_cmds_head); ++ current_command->info.jit_free.ids = ids; ++ current_command->info.jit_free.count = count; ++ ++ return 0; ++out_free: ++ kfree(ids); ++out: ++ return ret; ++} ++ ++static int kbase_csf_queue_group_suspend_prepare( ++ struct kbase_kcpu_command_queue *kcpu_queue, ++ struct base_kcpu_command_group_suspend_info *suspend_buf, ++ struct kbase_kcpu_command *current_command) ++{ ++ struct kbase_context *const kctx = kcpu_queue->kctx; ++ struct kbase_suspend_copy_buffer *sus_buf = NULL; ++ u64 addr = suspend_buf->buffer; ++ u64 page_addr = addr & PAGE_MASK; ++ u64 end_addr = addr + suspend_buf->size - 1; ++ u64 last_page_addr = end_addr & PAGE_MASK; ++ int nr_pages = (last_page_addr - page_addr) / PAGE_SIZE + 1; ++ int pinned_pages; ++ int ret = 0; ++ ++ lockdep_assert_held(&kctx->csf.kcpu_queues.lock); ++ ++ if (suspend_buf->size < ++ kctx->kbdev->csf.global_iface.groups[0].suspend_size) ++ return -EINVAL; ++ ++ ret = kbase_csf_queue_group_handle_is_valid(kctx, ++ suspend_buf->group_handle); ++ if (ret) ++ return ret; ++ ++ sus_buf = kzalloc(sizeof(*sus_buf), GFP_KERNEL); ++ if (!sus_buf) ++ return -ENOMEM; ++ ++ sus_buf->size = suspend_buf->size; ++ sus_buf->nr_pages = nr_pages; ++ sus_buf->offset = addr & ~PAGE_MASK; ++ ++ sus_buf->pages = kcalloc(nr_pages, sizeof(struct page *), GFP_KERNEL); ++ if (!sus_buf->pages) { ++ ret = -ENOMEM; ++ goto out_clean_sus_buf; ++ } ++ ++ pinned_pages = get_user_pages_fast(page_addr, nr_pages, 1, ++ sus_buf->pages); ++ if (pinned_pages < 0) { ++ ret = pinned_pages; ++ goto out_clean_pages; ++ } ++ if (pinned_pages != nr_pages) { ++ ret = -EINVAL; ++ goto out_clean_pages; ++ } ++ ++ current_command->type = BASE_KCPU_COMMAND_TYPE_GROUP_SUSPEND; ++ current_command->info.suspend_buf_copy.sus_buf = sus_buf; ++ current_command->info.suspend_buf_copy.group_handle = ++ suspend_buf->group_handle; ++ return ret; ++ ++out_clean_pages: ++ kfree(sus_buf->pages); ++out_clean_sus_buf: ++ kfree(sus_buf); ++ return ret; ++} ++ ++static int kbase_csf_queue_group_suspend_process(struct kbase_context *kctx, ++ struct kbase_suspend_copy_buffer *sus_buf, ++ u8 group_handle) ++{ ++ return kbase_csf_queue_group_suspend(kctx, sus_buf, group_handle); ++} ++ ++static enum kbase_csf_event_callback_action event_cqs_callback(void *param) ++{ ++ struct kbase_kcpu_command_queue *kcpu_queue = ++ (struct kbase_kcpu_command_queue *)param; ++ struct kbase_context *const kctx = kcpu_queue->kctx; ++ ++ queue_work(kctx->csf.kcpu_queues.wq, &kcpu_queue->work); ++ ++ return KBASE_CSF_EVENT_CALLBACK_KEEP; ++} ++ ++static void cleanup_cqs_wait(struct kbase_kcpu_command_queue *queue, ++ struct kbase_kcpu_command_cqs_wait_info *cqs_wait) ++{ ++ WARN_ON(!cqs_wait->nr_objs); ++ WARN_ON(!cqs_wait->objs); ++ WARN_ON(!cqs_wait->signaled); ++ WARN_ON(!queue->cqs_wait_count); ++ ++ if (--queue->cqs_wait_count == 0) { ++ kbase_csf_event_wait_remove(queue->kctx, ++ event_cqs_callback, queue); ++ } ++ ++ kfree(cqs_wait->signaled); ++ kfree(cqs_wait->objs); ++ cqs_wait->signaled = NULL; ++ cqs_wait->objs = NULL; ++} ++ ++static int kbase_kcpu_cqs_wait_process(struct kbase_device *kbdev, ++ struct kbase_kcpu_command_queue *queue, ++ struct kbase_kcpu_command_cqs_wait_info *cqs_wait) ++{ ++ u32 i; ++ ++ lockdep_assert_held(&queue->kctx->csf.kcpu_queues.lock); ++ ++ if (WARN_ON(!cqs_wait->nr_objs)) ++ return -EINVAL; ++ ++ if (WARN_ON(!cqs_wait->objs)) ++ return -EINVAL; ++ ++ ++ /* Skip the CQS waits that have already been signaled when processing */ ++ for (i = find_first_zero_bit(cqs_wait->signaled, cqs_wait->nr_objs); i < cqs_wait->nr_objs; i++) { ++ if (!test_bit(i, cqs_wait->signaled)) { ++ struct kbase_vmap_struct *mapping; ++ bool sig_set; ++ u32 *evt = (u32 *)kbase_phy_alloc_mapping_get(queue->kctx, ++ cqs_wait->objs[i].addr, &mapping); ++ ++ if (!queue->command_started) { ++ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_START( ++ kbdev, queue); ++ queue->command_started = true; ++ } ++ ++ if (WARN_ON(!evt)) { ++ queue->has_error = true; ++ return -EINVAL; ++ } ++ ++ sig_set = evt[BASEP_EVENT_VAL_INDEX] > cqs_wait->objs[i].val; ++ if (sig_set) { ++ bitmap_set(cqs_wait->signaled, i, 1); ++ if ((cqs_wait->inherit_err_flags & (1U << i)) && ++ evt[BASEP_EVENT_ERR_INDEX] > 0) ++ queue->has_error = true; ++ ++ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_END( ++ kbdev, queue); ++ queue->command_started = false; ++ } ++ ++ kbase_phy_alloc_mapping_put(queue->kctx, mapping); ++ ++ if (!sig_set) ++ break; ++ } ++ } ++ ++ /* For the queue to progress further, all cqs objects should get ++ * signaled. ++ */ ++ return bitmap_full(cqs_wait->signaled, cqs_wait->nr_objs); ++} ++ ++static int kbase_kcpu_cqs_wait_prepare(struct kbase_kcpu_command_queue *queue, ++ struct base_kcpu_command_cqs_wait_info *cqs_wait_info, ++ struct kbase_kcpu_command *current_command) ++{ ++ struct base_cqs_wait *objs; ++ unsigned int nr_objs = cqs_wait_info->nr_objs; ++ ++ lockdep_assert_held(&queue->kctx->csf.kcpu_queues.lock); ++ ++ if (cqs_wait_info->nr_objs > BASEP_KCPU_CQS_MAX_NUM_OBJS) ++ return -EINVAL; ++ ++ objs = kcalloc(nr_objs, sizeof(*objs), GFP_KERNEL); ++ if (!objs) ++ return -ENOMEM; ++ ++ if (copy_from_user(objs, u64_to_user_ptr(cqs_wait_info->objs), ++ nr_objs * sizeof(*objs))) { ++ kfree(objs); ++ return -ENOMEM; ++ } ++ ++ if (++queue->cqs_wait_count == 1) { ++ if (kbase_csf_event_wait_add(queue->kctx, ++ event_cqs_callback, queue)) { ++ kfree(objs); ++ return -ENOMEM; ++ } ++ } ++ ++ current_command->type = BASE_KCPU_COMMAND_TYPE_CQS_WAIT; ++ current_command->info.cqs_wait.nr_objs = nr_objs; ++ current_command->info.cqs_wait.objs = objs; ++ current_command->info.cqs_wait.inherit_err_flags = ++ cqs_wait_info->inherit_err_flags; ++ ++ current_command->info.cqs_wait.signaled = kcalloc(BITS_TO_LONGS(nr_objs), ++ sizeof(*current_command->info.cqs_wait.signaled), GFP_KERNEL); ++ if (!current_command->info.cqs_wait.signaled) ++ return -ENOMEM; ++ ++ return 0; ++} ++ ++static void kbase_kcpu_cqs_set_process(struct kbase_device *kbdev, ++ struct kbase_kcpu_command_queue *queue, ++ struct kbase_kcpu_command_cqs_set_info *cqs_set) ++{ ++ unsigned int i; ++ ++ lockdep_assert_held(&queue->kctx->csf.kcpu_queues.lock); ++ ++ WARN_ON(!cqs_set->nr_objs); ++ WARN_ON(!cqs_set->objs); ++ ++ for (i = 0; i < cqs_set->nr_objs; i++) { ++ struct kbase_vmap_struct *mapping; ++ u32 *evt = (u32 *)kbase_phy_alloc_mapping_get(queue->kctx, ++ cqs_set->objs[i].addr, &mapping); ++ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_SET(kbdev, queue); ++ if (WARN_ON(!evt)) ++ queue->has_error = true; ++ else { ++ if (cqs_set->propagate_flags & (1 << i)) ++ evt[BASEP_EVENT_ERR_INDEX] = queue->has_error; ++ else ++ evt[BASEP_EVENT_ERR_INDEX] = false; ++ /* Set to signaled */ ++ evt[BASEP_EVENT_VAL_INDEX]++; ++ kbase_phy_alloc_mapping_put(queue->kctx, mapping); ++ } ++ } ++ ++ kbase_csf_event_signal_notify_gpu(queue->kctx); ++ ++ kfree(cqs_set->objs); ++ cqs_set->objs = NULL; ++} ++ ++static int kbase_kcpu_cqs_set_prepare( ++ struct kbase_kcpu_command_queue *kcpu_queue, ++ struct base_kcpu_command_cqs_set_info *cqs_set_info, ++ struct kbase_kcpu_command *current_command) ++{ ++ struct kbase_context *const kctx = kcpu_queue->kctx; ++ struct base_cqs_set *objs; ++ unsigned int nr_objs = cqs_set_info->nr_objs; ++ ++ lockdep_assert_held(&kctx->csf.kcpu_queues.lock); ++ ++ if (cqs_set_info->nr_objs > BASEP_KCPU_CQS_MAX_NUM_OBJS) ++ return -EINVAL; ++ ++ objs = kcalloc(nr_objs, sizeof(*objs), GFP_KERNEL); ++ if (!objs) ++ return -ENOMEM; ++ ++ if (copy_from_user(objs, u64_to_user_ptr(cqs_set_info->objs), ++ nr_objs * sizeof(*objs))) { ++ kfree(objs); ++ return -ENOMEM; ++ } ++ ++ current_command->type = BASE_KCPU_COMMAND_TYPE_CQS_SET; ++ current_command->info.cqs_set.nr_objs = nr_objs; ++ current_command->info.cqs_set.objs = objs; ++ current_command->info.cqs_set.propagate_flags = ++ cqs_set_info->propagate_flags; ++ ++ return 0; ++} ++ ++#ifdef CONFIG_SYNC_FILE ++#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) ++static void kbase_csf_fence_wait_callback(struct fence *fence, ++ struct fence_cb *cb) ++#else ++static void kbase_csf_fence_wait_callback(struct dma_fence *fence, ++ struct dma_fence_cb *cb) ++#endif ++{ ++ struct kbase_kcpu_command_fence_info *fence_info = container_of(cb, ++ struct kbase_kcpu_command_fence_info, fence_cb); ++ struct kbase_kcpu_command_queue *kcpu_queue = fence_info->kcpu_queue; ++ struct kbase_context *const kctx = kcpu_queue->kctx; ++ ++ /* Resume kcpu command queue processing. */ ++ queue_work(kctx->csf.kcpu_queues.wq, &kcpu_queue->work); ++} ++ ++static void kbase_kcpu_fence_wait_cancel( ++ struct kbase_kcpu_command_queue *kcpu_queue, ++ struct kbase_kcpu_command_fence_info *fence_info) ++{ ++ struct kbase_context *const kctx = kcpu_queue->kctx; ++ ++ lockdep_assert_held(&kctx->csf.kcpu_queues.lock); ++ ++ if (WARN_ON(!fence_info->fence)) ++ return; ++ ++ if (kcpu_queue->fence_wait_processed) { ++ dma_fence_remove_callback(fence_info->fence, ++ &fence_info->fence_cb); ++ } ++ ++ /* Release the reference which is kept by the kcpu_queue */ ++ kbase_fence_put(fence_info->fence); ++ kcpu_queue->fence_wait_processed = false; ++ ++ fence_info->fence = NULL; ++} ++ ++/** ++ * kbase_kcpu_fence_wait_process() - Process the kcpu fence wait command ++ * ++ * @kcpu_queue: The queue containing the fence wait command ++ * @fence_info: Reference to a fence for which the command is waiting ++ * ++ * Return: 0 if fence wait is blocked, 1 if it is unblocked, negative error if ++ * an error has occurred and fence should no longer be waited on. ++ */ ++static int kbase_kcpu_fence_wait_process( ++ struct kbase_kcpu_command_queue *kcpu_queue, ++ struct kbase_kcpu_command_fence_info *fence_info) ++{ ++ int fence_status = 0; ++#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) ++ struct fence *fence; ++#else ++ struct dma_fence *fence; ++#endif ++ ++ lockdep_assert_held(&kcpu_queue->kctx->csf.kcpu_queues.lock); ++ ++ if (WARN_ON(!fence_info->fence)) ++ return -EINVAL; ++ ++ fence = fence_info->fence; ++ ++ if (kcpu_queue->fence_wait_processed) { ++ fence_status = dma_fence_get_status(fence); ++ } else { ++ int cb_err = dma_fence_add_callback(fence, ++ &fence_info->fence_cb, ++ kbase_csf_fence_wait_callback); ++ ++ fence_status = cb_err; ++ if (cb_err == 0) ++ kcpu_queue->fence_wait_processed = true; ++ else if (cb_err == -ENOENT) ++ fence_status = dma_fence_get_status(fence); ++ } ++ ++ /* ++ * At this point fence status can contain 3 types of values: ++ * - Value 0 to represent that fence in question is not signalled yet ++ * - Value 1 to represent that fence in question is signalled without ++ * errors ++ * - Negative error code to represent that some error has occurred such ++ * that waiting on it is no longer valid. ++ */ ++ ++ if (fence_status) ++ kbase_kcpu_fence_wait_cancel(kcpu_queue, fence_info); ++ ++ return fence_status; ++} ++ ++static int kbase_kcpu_fence_wait_prepare( ++ struct kbase_kcpu_command_queue *kcpu_queue, ++ struct base_kcpu_command_fence_info *fence_info, ++ struct kbase_kcpu_command *current_command) ++{ ++ struct kbase_context *const kctx = kcpu_queue->kctx; ++#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) ++ struct fence *fence_in; ++#else ++ struct dma_fence *fence_in; ++#endif ++ struct base_fence fence; ++ ++ lockdep_assert_held(&kctx->csf.kcpu_queues.lock); ++ ++ if (copy_from_user(&fence, u64_to_user_ptr(fence_info->fence), ++ sizeof(fence))) ++ return -ENOMEM; ++ ++ fence_in = sync_file_get_fence(fence.basep.fd); ++ ++ if (!fence_in) ++ return -ENOENT; ++ ++ current_command->type = BASE_KCPU_COMMAND_TYPE_FENCE_WAIT; ++ current_command->info.fence.fence = fence_in; ++ current_command->info.fence.kcpu_queue = kcpu_queue; ++ ++ return 0; ++} ++ ++static int kbase_kcpu_fence_signal_process( ++ struct kbase_kcpu_command_queue *kcpu_queue, ++ struct kbase_kcpu_command_fence_info *fence_info) ++{ ++ struct kbase_context *const kctx = kcpu_queue->kctx; ++ int ret; ++ ++ if (WARN_ON(!fence_info->fence)) ++ return -EINVAL; ++ ++ ret = dma_fence_signal(fence_info->fence); ++ ++ if (unlikely(ret < 0)) { ++ dev_warn(kctx->kbdev->dev, ++ "fence_signal() failed with %d\n", ret); ++ } ++ ++ dma_fence_put(fence_info->fence); ++ fence_info->fence = NULL; ++ ++ return ret; ++} ++ ++static int kbase_kcpu_fence_signal_prepare( ++ struct kbase_kcpu_command_queue *kcpu_queue, ++ struct base_kcpu_command_fence_info *fence_info, ++ struct kbase_kcpu_command *current_command) ++{ ++ struct kbase_context *const kctx = kcpu_queue->kctx; ++#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) ++ struct fence *fence_out; ++#else ++ struct dma_fence *fence_out; ++#endif ++ struct base_fence fence; ++ struct sync_file *sync_file; ++ int ret = 0; ++ int fd; ++ ++ lockdep_assert_held(&kctx->csf.kcpu_queues.lock); ++ ++ if (copy_from_user(&fence, u64_to_user_ptr(fence_info->fence), ++ sizeof(fence))) ++ return -EFAULT; ++ ++ fence_out = kzalloc(sizeof(*fence_out), GFP_KERNEL); ++ if (!fence_out) ++ return -ENOMEM; ++ ++ dma_fence_init(fence_out, ++ &kbase_fence_ops, ++ &kbase_csf_fence_lock, ++ kcpu_queue->fence_context, ++ ++kcpu_queue->fence_seqno); ++ ++#if (KERNEL_VERSION(4, 9, 67) >= LINUX_VERSION_CODE) ++ /* Take an extra reference to the fence on behalf of the sync file. ++ * This is only needded on older kernels where sync_file_create() ++ * does not take its own reference. This was changed in v4.9.68 ++ * where sync_file_create() now takes its own reference. ++ */ ++ dma_fence_get(fence_out); ++#endif ++ ++ /* create a sync_file fd representing the fence */ ++ sync_file = sync_file_create(fence_out); ++ if (!sync_file) { ++#if (KERNEL_VERSION(4, 9, 67) >= LINUX_VERSION_CODE) ++ dma_fence_put(fence_out); ++#endif ++ ret = -ENOMEM; ++ goto file_create_fail; ++ } ++ ++ fd = get_unused_fd_flags(O_CLOEXEC); ++ if (fd < 0) { ++ ret = fd; ++ goto fd_flags_fail; ++ } ++ ++ fd_install(fd, sync_file->file); ++ ++ fence.basep.fd = fd; ++ ++ current_command->type = BASE_KCPU_COMMAND_TYPE_FENCE_SIGNAL; ++ current_command->info.fence.fence = fence_out; ++ ++ if (copy_to_user(u64_to_user_ptr(fence_info->fence), &fence, ++ sizeof(fence))) { ++ ret = -EFAULT; ++ goto fd_flags_fail; ++ } ++ ++ return 0; ++ ++fd_flags_fail: ++ fput(sync_file->file); ++file_create_fail: ++ dma_fence_put(fence_out); ++ ++ return ret; ++} ++#endif /* CONFIG_SYNC_FILE */ ++ ++static void kcpu_queue_process_worker(struct work_struct *data) ++{ ++ struct kbase_kcpu_command_queue *queue = container_of(data, ++ struct kbase_kcpu_command_queue, work); ++ ++ mutex_lock(&queue->kctx->csf.kcpu_queues.lock); ++ ++ kcpu_queue_process(queue, false); ++ ++ mutex_unlock(&queue->kctx->csf.kcpu_queues.lock); ++} ++ ++static int delete_queue(struct kbase_context *kctx, u32 id) ++{ ++ int err = 0; ++ ++ mutex_lock(&kctx->csf.kcpu_queues.lock); ++ ++ if ((id < KBASEP_MAX_KCPU_QUEUES) && kctx->csf.kcpu_queues.array[id]) { ++ struct kbase_kcpu_command_queue *queue = ++ kctx->csf.kcpu_queues.array[id]; ++ ++ /* Drain the remaining work for this queue first and go past ++ * all the waits. ++ */ ++ kcpu_queue_process(queue, true); ++ ++ /* All commands should have been processed */ ++ WARN_ON(queue->num_pending_cmds); ++ ++ /* All CQS wait commands should have been cleaned up */ ++ WARN_ON(queue->cqs_wait_count); ++ ++ kctx->csf.kcpu_queues.array[id] = NULL; ++ bitmap_clear(kctx->csf.kcpu_queues.in_use, id, 1); ++ ++ /* Fire the tracepoint with the mutex held to enforce correct ++ * ordering with the summary stream. ++ */ ++ KBASE_TLSTREAM_TL_KBASE_DEL_KCPUQUEUE(kctx->kbdev, queue); ++ ++ mutex_unlock(&kctx->csf.kcpu_queues.lock); ++ ++ cancel_work_sync(&queue->work); ++ ++ kfree(queue); ++ } else { ++ dev_warn(kctx->kbdev->dev, ++ "Attempt to delete a non-existent KCPU queue\n"); ++ mutex_unlock(&kctx->csf.kcpu_queues.lock); ++ err = -EINVAL; ++ } ++ return err; ++} ++ ++static void KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_INFO( ++ struct kbase_device *kbdev, ++ const struct kbase_kcpu_command_queue *queue, ++ const struct kbase_kcpu_command_jit_alloc_info *jit_alloc, ++ bool alloc_success) ++{ ++ u8 i; ++ ++ KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( ++ kbdev, queue); ++ for (i = 0; i < jit_alloc->count; i++) { ++ const u8 id = jit_alloc->info[i].id; ++ const struct kbase_va_region *reg = queue->kctx->jit_alloc[id]; ++ u64 gpu_alloc_addr = 0; ++ u64 mmu_flags = 0; ++ ++ if (alloc_success && !WARN_ON(!reg) && ++ !WARN_ON(reg == KBASE_RESERVED_REG_JIT_ALLOC)) { ++#ifdef CONFIG_MALI_VECTOR_DUMP ++ struct tagged_addr phy = {0}; ++#endif /* CONFIG_MALI_VECTOR_DUMP */ ++ ++ gpu_alloc_addr = reg->start_pfn << PAGE_SHIFT; ++#ifdef CONFIG_MALI_VECTOR_DUMP ++ mmu_flags = kbase_mmu_create_ate(kbdev, ++ phy, reg->flags, ++ MIDGARD_MMU_BOTTOMLEVEL, ++ queue->kctx->jit_group_id); ++#endif /* CONFIG_MALI_VECTOR_DUMP */ ++ } ++ KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( ++ kbdev, queue, gpu_alloc_addr, mmu_flags); ++ } ++} ++ ++static void KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( ++ struct kbase_device *kbdev, ++ const struct kbase_kcpu_command_queue *queue) ++{ ++ KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( ++ kbdev, queue); ++} ++ ++static void KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_INFO( ++ struct kbase_device *kbdev, ++ const struct kbase_kcpu_command_queue *queue, ++ const struct kbase_kcpu_command_jit_free_info *jit_free) ++{ ++ u8 i; ++ ++ KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_FREE_END( ++ kbdev, queue); ++ for (i = 0; i < jit_free->count; i++) { ++ const u8 id = jit_free->ids[i]; ++ u64 pages_used = 0; ++ ++ if (id != 0) { ++ const struct kbase_va_region *reg = ++ queue->kctx->jit_alloc[id]; ++ if (reg && (reg != KBASE_RESERVED_REG_JIT_ALLOC)) ++ pages_used = reg->gpu_alloc->nents; ++ } ++ KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_FREE_END( ++ kbdev, queue, pages_used); ++ } ++} ++ ++static void KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_END( ++ struct kbase_device *kbdev, ++ const struct kbase_kcpu_command_queue *queue) ++{ ++ KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_FREE_END( ++ kbdev, queue); ++} ++ ++static void kcpu_queue_process(struct kbase_kcpu_command_queue *queue, ++ bool ignore_waits) ++{ ++ struct kbase_device *kbdev = queue->kctx->kbdev; ++ bool process_next = true; ++ size_t i; ++ ++ lockdep_assert_held(&queue->kctx->csf.kcpu_queues.lock); ++ ++ for (i = 0; i != queue->num_pending_cmds; ++i) { ++ struct kbase_kcpu_command *cmd = ++ &queue->commands[(u8)(queue->start_offset + i)]; ++ int status; ++ ++ switch (cmd->type) { ++ case BASE_KCPU_COMMAND_TYPE_FENCE_WAIT: ++ if (!queue->command_started) { ++ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_START( ++ kbdev, queue); ++ queue->command_started = true; ++ } ++ ++#ifdef CONFIG_SYNC_FILE ++ status = 0; ++ ++ ++ if (ignore_waits) { ++ kbase_kcpu_fence_wait_cancel(queue, ++ &cmd->info.fence); ++ } else { ++ status = kbase_kcpu_fence_wait_process(queue, ++ &cmd->info.fence); ++ ++ if (status == 0) ++ process_next = false; ++ else if (status < 0) ++ queue->has_error = true; ++ } ++#else ++ dev_warn(kbdev->dev, ++ "unexpected fence wait command found\n"); ++#endif ++ ++ if (process_next) { ++ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_END( ++ kbdev, queue); ++ queue->command_started = false; ++ } ++ break; ++ case BASE_KCPU_COMMAND_TYPE_FENCE_SIGNAL: ++ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_START( ++ kbdev, queue); ++ ++#ifdef CONFIG_SYNC_FILE ++ kbase_kcpu_fence_signal_process(queue, ++ &cmd->info.fence); ++#else ++ dev_warn(kbdev->dev, ++ "unexpected fence signal command found\n"); ++#endif ++ ++ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_END( ++ kbdev, queue); ++ break; ++ case BASE_KCPU_COMMAND_TYPE_CQS_WAIT: ++ status = kbase_kcpu_cqs_wait_process(kbdev, queue, ++ &cmd->info.cqs_wait); ++ ++ if (!status && !ignore_waits) { ++ process_next = false; ++ } else { ++ /* Either all CQS objects were signaled or ++ * there was an error or the queue itself is ++ * being deleted. ++ * In all cases can move to the next command. ++ * TBD: handle the error ++ */ ++ cleanup_cqs_wait(queue, &cmd->info.cqs_wait); ++ } ++ ++ break; ++ case BASE_KCPU_COMMAND_TYPE_CQS_SET: ++ kbase_kcpu_cqs_set_process(kbdev, queue, ++ &cmd->info.cqs_set); ++ ++ /* CQS sets are only traced before execution */ ++ break; ++ case BASE_KCPU_COMMAND_TYPE_ERROR_BARRIER: ++ /* Clear the queue's error state */ ++ queue->has_error = false; ++ break; ++ case BASE_KCPU_COMMAND_TYPE_MAP_IMPORT: ++ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_START( ++ kbdev, queue); ++ ++ kbase_gpu_vm_lock(queue->kctx); ++ kbase_sticky_resource_acquire(queue->kctx, ++ cmd->info.import.gpu_va); ++ kbase_gpu_vm_unlock(queue->kctx); ++ ++ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_END( ++ kbdev, queue); ++ break; ++ case BASE_KCPU_COMMAND_TYPE_UNMAP_IMPORT: ++ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_START( ++ kbdev, queue); ++ ++ kbase_gpu_vm_lock(queue->kctx); ++ kbase_sticky_resource_release(queue->kctx, NULL, ++ cmd->info.import.gpu_va); ++ kbase_gpu_vm_unlock(queue->kctx); ++ ++ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_END( ++ kbdev, queue); ++ break; ++ case BASE_KCPU_COMMAND_TYPE_UNMAP_IMPORT_FORCE: ++ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_START( ++ kbdev, queue); ++ ++ kbase_gpu_vm_lock(queue->kctx); ++ kbase_sticky_resource_release_force(queue->kctx, NULL, ++ cmd->info.import.gpu_va); ++ kbase_gpu_vm_unlock(queue->kctx); ++ ++ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_END( ++ kbdev, queue); ++ break; ++ case BASE_KCPU_COMMAND_TYPE_JIT_ALLOC: ++ { ++ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_START( ++ kbdev, queue); ++ ++ status = kbase_kcpu_jit_allocate_process(queue, cmd); ++ if (status == -EAGAIN) { ++ process_next = false; ++ } else { ++ if (status != 0) ++ queue->has_error = true; ++ ++ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_INFO( ++ kbdev, queue, &cmd->info.jit_alloc, (status == 0)); ++ ++ kbase_kcpu_jit_allocate_finish(queue, cmd); ++ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( ++ kbdev, queue); ++ } ++ break; ++ } ++ case BASE_KCPU_COMMAND_TYPE_JIT_FREE: ++ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_START( ++ kbdev, queue); ++ ++ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_INFO( ++ kbdev, queue, &cmd->info.jit_free); ++ ++ status = kbase_kcpu_jit_free_process(queue->kctx, cmd); ++ if (status) ++ queue->has_error = true; ++ ++ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_END( ++ kbdev, queue); ++ break; ++ case BASE_KCPU_COMMAND_TYPE_GROUP_SUSPEND: ++ status = kbase_csf_queue_group_suspend_process( ++ queue->kctx, ++ cmd->info.suspend_buf_copy.sus_buf, ++ cmd->info.suspend_buf_copy.group_handle); ++ if (status) ++ queue->has_error = true; ++ ++ kfree(cmd->info.suspend_buf_copy.sus_buf->pages); ++ kfree(cmd->info.suspend_buf_copy.sus_buf); ++ break; ++ default: ++ dev_warn(kbdev->dev, ++ "Unrecognized command type\n"); ++ break; ++ } /* switch */ ++ ++ /*TBD: error handling */ ++ ++ if (!process_next) ++ break; ++ } ++ ++ if (i > 0) { ++ queue->start_offset += i; ++ queue->num_pending_cmds -= i; ++ ++ /* If an attempt to enqueue commands failed then we must raise ++ * an event in case the client wants to retry now that there is ++ * free space in the buffer. ++ */ ++ if (queue->enqueue_failed) { ++ queue->enqueue_failed = false; ++ kbase_csf_event_signal_cpu_only(queue->kctx); ++ } ++ } ++} ++ ++static size_t kcpu_queue_get_space(struct kbase_kcpu_command_queue *queue) ++{ ++ return KBASEP_KCPU_QUEUE_SIZE - queue->num_pending_cmds; ++} ++ ++static void KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_COMMAND( ++ const struct kbase_kcpu_command_queue *queue, ++ const struct kbase_kcpu_command *cmd) ++{ ++ struct kbase_device *kbdev = queue->kctx->kbdev; ++ ++ switch (cmd->type) { ++ case BASE_KCPU_COMMAND_TYPE_FENCE_WAIT: ++ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_WAIT( ++ kbdev, queue, cmd->info.fence.fence); ++ break; ++ case BASE_KCPU_COMMAND_TYPE_FENCE_SIGNAL: ++ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_SIGNAL( ++ kbdev, queue, cmd->info.fence.fence); ++ break; ++ case BASE_KCPU_COMMAND_TYPE_CQS_WAIT: ++ { ++ const struct base_cqs_wait *waits = cmd->info.cqs_wait.objs; ++ unsigned int i; ++ ++ for (i = 0; i < cmd->info.cqs_wait.nr_objs; i++) { ++ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_WAIT( ++ kbdev, queue, waits[i].addr, waits[i].val); ++ } ++ break; ++ } ++ case BASE_KCPU_COMMAND_TYPE_CQS_SET: ++ { ++ const struct base_cqs_set *sets = cmd->info.cqs_set.objs; ++ unsigned int i; ++ ++ for (i = 0; i < cmd->info.cqs_set.nr_objs; i++) { ++ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_SET( ++ kbdev, queue, sets[i].addr); ++ } ++ break; ++ } ++ case BASE_KCPU_COMMAND_TYPE_ERROR_BARRIER: ++ /* No implemented tracepoint */ ++ break; ++ case BASE_KCPU_COMMAND_TYPE_MAP_IMPORT: ++ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_MAP_IMPORT( ++ kbdev, queue, cmd->info.import.gpu_va); ++ break; ++ case BASE_KCPU_COMMAND_TYPE_UNMAP_IMPORT: ++ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT( ++ kbdev, queue, cmd->info.import.gpu_va); ++ break; ++ case BASE_KCPU_COMMAND_TYPE_UNMAP_IMPORT_FORCE: ++ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT_FORCE( ++ kbdev, queue, cmd->info.import.gpu_va); ++ break; ++ case BASE_KCPU_COMMAND_TYPE_JIT_ALLOC: ++ { ++ u8 i; ++ ++ KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_ALLOC( ++ kbdev, queue); ++ for (i = 0; i < cmd->info.jit_alloc.count; i++) { ++ const struct base_jit_alloc_info *info = ++ &cmd->info.jit_alloc.info[i]; ++ ++ KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_ALLOC( ++ kbdev, queue, ++ info->gpu_alloc_addr, info->va_pages, ++ info->commit_pages, info->extent, info->id, ++ info->bin_id, info->max_allocations, ++ info->flags, info->usage_id); ++ } ++ KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_ALLOC( ++ kbdev, queue); ++ break; ++ } ++ case BASE_KCPU_COMMAND_TYPE_JIT_FREE: ++ { ++ u8 i; ++ ++ KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_FREE( ++ kbdev, queue); ++ for (i = 0; i < cmd->info.jit_free.count; i++) { ++ KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_FREE( ++ kbdev, queue, cmd->info.jit_free.ids[i]); ++ } ++ KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_FREE( ++ kbdev, queue); ++ break; ++ } ++ case BASE_KCPU_COMMAND_TYPE_GROUP_SUSPEND: ++ /* No implemented tracepoint */ ++ break; ++ } ++} ++ ++int kbase_csf_kcpu_queue_enqueue(struct kbase_context *kctx, ++ struct kbase_ioctl_kcpu_queue_enqueue *enq) ++{ ++ struct kbase_kcpu_command_queue *queue = NULL; ++ void __user *user_cmds = u64_to_user_ptr(enq->addr); ++ int ret = 0; ++ u32 i; ++ ++ /* The offset to the first command that is being processed or yet to ++ * be processed is of u8 type, so the number of commands inside the ++ * queue cannot be more than 256. ++ */ ++ BUILD_BUG_ON(KBASEP_KCPU_QUEUE_SIZE > 256); ++ ++ /* Whilst the backend interface allows enqueueing multiple commands in ++ * a single operation, the Base interface does not expose any mechanism ++ * to do so. And also right now the handling is missing for the case ++ * where multiple commands are submitted and the enqueue of one of the ++ * command in the set fails after successfully enqueuing other commands ++ * in the set. ++ */ ++ if (enq->nr_commands != 1) { ++ dev_err(kctx->kbdev->dev, ++ "More than one commands enqueued\n"); ++ return -EINVAL; ++ } ++ ++ mutex_lock(&kctx->csf.kcpu_queues.lock); ++ ++ if (!kctx->csf.kcpu_queues.array[enq->id]) { ++ ret = -EINVAL; ++ goto out; ++ } ++ ++ queue = kctx->csf.kcpu_queues.array[enq->id]; ++ ++ if (kcpu_queue_get_space(queue) < enq->nr_commands) { ++ ret = -EBUSY; ++ queue->enqueue_failed = true; ++ goto out; ++ } ++ ++ /* Copy all command's info to the command buffer. ++ * Note: it would be more efficient to process all commands in-line ++ * until we encounter an unresolved CQS_ / FENCE_WAIT, however, the ++ * interface allows multiple commands to be enqueued so we must account ++ * for the possibility to roll back. ++ */ ++ ++ for (i = 0; (i != enq->nr_commands) && !ret; ++i, ++kctx->csf.kcpu_queues.num_cmds) { ++ struct kbase_kcpu_command *kcpu_cmd = ++ &queue->commands[(u8)(queue->start_offset + queue->num_pending_cmds + i)]; ++ struct base_kcpu_command command; ++ unsigned int j; ++ ++ if (copy_from_user(&command, user_cmds, sizeof(command))) { ++ ret = -EFAULT; ++ goto out; ++ } ++ ++ user_cmds = (void __user *)((uintptr_t)user_cmds + ++ sizeof(struct base_kcpu_command)); ++ ++ for (j = 0; j < sizeof(command.padding); j++) { ++ if (command.padding[j] != 0) { ++ dev_dbg(kctx->kbdev->dev, ++ "base_kcpu_command padding not 0\n"); ++ ret = -EINVAL; ++ goto out; ++ } ++ } ++ ++ kcpu_cmd->enqueue_ts = kctx->csf.kcpu_queues.num_cmds; ++ switch (command.type) { ++ case BASE_KCPU_COMMAND_TYPE_FENCE_WAIT: ++#ifdef CONFIG_SYNC_FILE ++ ret = kbase_kcpu_fence_wait_prepare(queue, ++ &command.info.fence, kcpu_cmd); ++#else ++ ret = -EINVAL; ++ dev_warn(kctx->kbdev->dev, "fence wait command unsupported\n"); ++#endif ++ break; ++ case BASE_KCPU_COMMAND_TYPE_FENCE_SIGNAL: ++#ifdef CONFIG_SYNC_FILE ++ ret = kbase_kcpu_fence_signal_prepare(queue, ++ &command.info.fence, kcpu_cmd); ++#else ++ ret = -EINVAL; ++ dev_warn(kctx->kbdev->dev, "fence signal command unsupported\n"); ++#endif ++ break; ++ case BASE_KCPU_COMMAND_TYPE_CQS_WAIT: ++ ret = kbase_kcpu_cqs_wait_prepare(queue, ++ &command.info.cqs_wait, kcpu_cmd); ++ break; ++ case BASE_KCPU_COMMAND_TYPE_CQS_SET: ++ ret = kbase_kcpu_cqs_set_prepare(queue, ++ &command.info.cqs_set, kcpu_cmd); ++ break; ++ case BASE_KCPU_COMMAND_TYPE_ERROR_BARRIER: ++ kcpu_cmd->type = BASE_KCPU_COMMAND_TYPE_ERROR_BARRIER; ++ ret = 0; ++ break; ++ case BASE_KCPU_COMMAND_TYPE_MAP_IMPORT: ++ ret = kbase_kcpu_map_import_prepare(queue, ++ &command.info.import, kcpu_cmd); ++ break; ++ case BASE_KCPU_COMMAND_TYPE_UNMAP_IMPORT: ++ ret = kbase_kcpu_unmap_import_prepare(queue, ++ &command.info.import, kcpu_cmd); ++ break; ++ case BASE_KCPU_COMMAND_TYPE_UNMAP_IMPORT_FORCE: ++ ret = kbase_kcpu_unmap_import_force_prepare(queue, ++ &command.info.import, kcpu_cmd); ++ break; ++ case BASE_KCPU_COMMAND_TYPE_JIT_ALLOC: ++ ret = kbase_kcpu_jit_allocate_prepare(queue, ++ &command.info.jit_alloc, kcpu_cmd); ++ break; ++ case BASE_KCPU_COMMAND_TYPE_JIT_FREE: ++ ret = kbase_kcpu_jit_free_prepare(queue, ++ &command.info.jit_free, kcpu_cmd); ++ break; ++ case BASE_KCPU_COMMAND_TYPE_GROUP_SUSPEND: ++ ret = kbase_csf_queue_group_suspend_prepare(queue, ++ &command.info.suspend_buf_copy, ++ kcpu_cmd); ++ break; ++ ++ default: ++ dev_warn(queue->kctx->kbdev->dev, ++ "Unknown command type %u\n", command.type); ++ ret = -EINVAL; ++ break; ++ } ++ } ++ ++ if (!ret) { ++ /* We only instrument the enqueues after all commands have been ++ * successfully enqueued, as if we do them during the enqueue ++ * and there is an error, we won't be able to roll them back ++ * like is done for the command enqueues themselves. ++ */ ++ for (i = 0; i != enq->nr_commands; ++i) { ++ u8 cmd_idx = (u8)(queue->start_offset + queue->num_pending_cmds + i); ++ ++ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_COMMAND( ++ queue, &queue->commands[cmd_idx]); ++ } ++ ++ queue->num_pending_cmds += enq->nr_commands; ++ kcpu_queue_process(queue, false); ++ } else { ++ /* Roll back the number of enqueued commands */ ++ kctx->csf.kcpu_queues.num_cmds -= i; ++ } ++ ++out: ++ mutex_unlock(&kctx->csf.kcpu_queues.lock); ++ ++ return ret; ++} ++ ++int kbase_csf_kcpu_queue_context_init(struct kbase_context *kctx) ++{ ++ int idx; ++ ++ bitmap_zero(kctx->csf.kcpu_queues.in_use, KBASEP_MAX_KCPU_QUEUES); ++ ++ for (idx = 0; idx < KBASEP_MAX_KCPU_QUEUES; ++idx) ++ kctx->csf.kcpu_queues.array[idx] = NULL; ++ ++ kctx->csf.kcpu_queues.wq = alloc_workqueue("mali_kbase_csf_kcpu", ++ WQ_UNBOUND | WQ_HIGHPRI, 0); ++ if (!kctx->csf.kcpu_queues.wq) ++ return -ENOMEM; ++ ++ mutex_init(&kctx->csf.kcpu_queues.lock); ++ ++ kctx->csf.kcpu_queues.num_cmds = 0; ++ ++ return 0; ++} ++ ++void kbase_csf_kcpu_queue_context_term(struct kbase_context *kctx) ++{ ++ while (!bitmap_empty(kctx->csf.kcpu_queues.in_use, ++ KBASEP_MAX_KCPU_QUEUES)) { ++ int id = find_first_bit(kctx->csf.kcpu_queues.in_use, ++ KBASEP_MAX_KCPU_QUEUES); ++ ++ if (WARN_ON(!kctx->csf.kcpu_queues.array[id])) ++ clear_bit(id, kctx->csf.kcpu_queues.in_use); ++ else ++ (void)delete_queue(kctx, id); ++ } ++ ++ destroy_workqueue(kctx->csf.kcpu_queues.wq); ++ mutex_destroy(&kctx->csf.kcpu_queues.lock); ++} ++ ++int kbase_csf_kcpu_queue_delete(struct kbase_context *kctx, ++ struct kbase_ioctl_kcpu_queue_delete *del) ++{ ++ return delete_queue(kctx, (u32)del->id); ++} ++ ++int kbase_csf_kcpu_queue_new(struct kbase_context *kctx, ++ struct kbase_ioctl_kcpu_queue_new *newq) ++{ ++ struct kbase_kcpu_command_queue *queue; ++ int idx; ++ int ret = 0; ++ ++ /* The queue id is of u8 type and we use the index of the kcpu_queues ++ * array as an id, so the number of elements in the array can't be ++ * more than 256. ++ */ ++ BUILD_BUG_ON(KBASEP_MAX_KCPU_QUEUES > 256); ++ ++ mutex_lock(&kctx->csf.kcpu_queues.lock); ++ ++ idx = find_first_zero_bit(kctx->csf.kcpu_queues.in_use, ++ KBASEP_MAX_KCPU_QUEUES); ++ if (idx >= (int)KBASEP_MAX_KCPU_QUEUES) { ++ ret = -ENOMEM; ++ goto out; ++ } ++ ++ if (WARN_ON(kctx->csf.kcpu_queues.array[idx])) { ++ ret = -EINVAL; ++ goto out; ++ } ++ ++ queue = kzalloc(sizeof(*queue), GFP_KERNEL); ++ ++ if (!queue) { ++ ret = -ENOMEM; ++ goto out; ++ } ++ ++ bitmap_set(kctx->csf.kcpu_queues.in_use, idx, 1); ++ kctx->csf.kcpu_queues.array[idx] = queue; ++ queue->kctx = kctx; ++ queue->start_offset = 0; ++ queue->num_pending_cmds = 0; ++#ifdef CONFIG_SYNC_FILE ++ queue->fence_context = dma_fence_context_alloc(1); ++ queue->fence_seqno = 0; ++ queue->fence_wait_processed = false; ++#endif ++ queue->enqueue_failed = false; ++ queue->command_started = false; ++ INIT_LIST_HEAD(&queue->jit_blocked); ++ queue->has_error = false; ++ INIT_WORK(&queue->work, kcpu_queue_process_worker); ++ ++ newq->id = idx; ++ ++ /* Fire the tracepoint with the mutex held to enforce correct ordering ++ * with the summary stream. ++ */ ++ KBASE_TLSTREAM_TL_KBASE_NEW_KCPUQUEUE( ++ kctx->kbdev, queue, kctx->id, queue->num_pending_cmds); ++out: ++ mutex_unlock(&kctx->csf.kcpu_queues.lock); ++ ++ return ret; ++} +diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.h +new file mode 100755 +index 000000000000..45c76af04c0f +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.h +@@ -0,0 +1,305 @@ ++/* ++ * ++ * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_CSF_KCPU_H_ ++#define _KBASE_CSF_KCPU_H_ ++ ++#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) ++#include ++#else ++#include ++#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) */ ++ ++/* The maximum number of KCPU commands in flight, enqueueing more commands ++ * than this value shall block. ++ */ ++#define KBASEP_KCPU_QUEUE_SIZE ((size_t)256) ++ ++/** ++ * struct kbase_kcpu_command_import_info - Structure which holds information ++ * about the buffer to be imported ++ * ++ * @gpu_va: Address of the buffer to be imported. ++ */ ++struct kbase_kcpu_command_import_info { ++ u64 gpu_va; ++}; ++ ++/** ++ * struct kbase_kcpu_command_fence_info - Structure which holds information ++ * about the fence object enqueued in the kcpu command queue ++ * ++ * @fence_cb: ++ * @fence: ++ * @kcpu_queue: ++ */ ++struct kbase_kcpu_command_fence_info { ++#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) ++ struct fence_cb fence_cb; ++ struct fence *fence; ++#else ++ struct dma_fence_cb fence_cb; ++ struct dma_fence *fence; ++#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) */ ++ struct kbase_kcpu_command_queue *kcpu_queue; ++}; ++ ++/** ++ * struct kbase_kcpu_command_cqs_set_info - Structure which holds information ++ * about CQS objects for the kcpu CQS set command ++ * ++ * @objs: Array of structures which define CQS objects to be used by ++ * the kcpu command. ++ * @nr_objs: Number of CQS objects in the array. ++ * @propagate_flags: Bit-pattern for the CQSs in the array that are set ++ * to propagate queue error-state to the flagged CQSs. ++ */ ++struct kbase_kcpu_command_cqs_set_info { ++ struct base_cqs_set *objs; ++ unsigned int nr_objs; ++ u32 propagate_flags; ++}; ++ ++/** ++ * struct kbase_kcpu_command_cqs_wait_info - Structure which holds information ++ * about CQS objects for the kcpu CQS wait command ++ * ++ * @objs: Array of structures which define CQS objects to be used by ++ * the kcpu command. ++ * @signaled: Bit array used to report the status of the CQS wait objects. ++ * 1 is signaled, 0 otherwise. ++ * @nr_objs: Number of CQS objects in the array. ++ * @inherit_err_flags: Bit-pattern for the CQSs in the array who's error field ++ * to be served as the source for importing into the ++ * queue's error-state. ++ */ ++struct kbase_kcpu_command_cqs_wait_info { ++ struct base_cqs_wait *objs; ++ unsigned long *signaled; ++ unsigned int nr_objs; ++ u32 inherit_err_flags; ++}; ++ ++/** ++ * struct kbase_kcpu_command_jit_alloc_info - Structure which holds information ++ * needed for the kcpu command for jit allocations ++ * ++ * @node Used to keep track of all JIT free/alloc commands in submission ++ * order. This must be located in the front of this struct to ++ * match that of kbase_kcpu_command_jit_free_info. ++ * @info: Array of objects of the struct base_jit_alloc_info type which ++ * specify jit allocations to be made by the kcpu command. ++ * @count: Number of jit alloc objects in the array. ++ * @blocked: Whether this allocation has been put into the pending list to ++ * be retried later. ++ */ ++struct kbase_kcpu_command_jit_alloc_info { ++ struct list_head node; ++ struct base_jit_alloc_info *info; ++ u8 count; ++ bool blocked; ++}; ++ ++/** ++ * struct kbase_kcpu_command_jit_free_info - Structure which holds information ++ * needed for the kcpu jit free command ++ * ++ * @node: Used to keep track of all JIT free/alloc commands in submission ++ * order. This must be located in the front of this struct to ++ * match that of kbase_kcpu_command_jit_alloc_info. ++ * @ids: Array of identifiers of jit allocations which are to be freed ++ * by the kcpu command. ++ * @count: Number of elements in the array. ++ */ ++struct kbase_kcpu_command_jit_free_info { ++ struct list_head node; ++ u8 *ids; ++ u8 count; ++}; ++ ++/** ++ * struct kbase_suspend_copy_buffer - information about the suspend buffer ++ * to be copied. ++ * ++ * @size: size of the suspend buffer in bytes. ++ * @pages: pointer to an array of pointers to the pages which contain ++ * the user buffer. ++ * @nr_pages: number of pages. ++ * @offset: offset into the pages ++ */ ++struct kbase_suspend_copy_buffer { ++ size_t size; ++ struct page **pages; ++ int nr_pages; ++ size_t offset; ++}; ++ ++/** ++ * struct base_kcpu_command_group_suspend - structure which contains ++ * suspend buffer data captured for a suspended queue group. ++ * ++ * @sus_buf: Pointer to the structure which contains details of the ++ * user buffer and its kernel pinned pages. ++ * @group_handle: Handle to the mapping of command stream group. ++ */ ++struct kbase_kcpu_command_group_suspend_info { ++ struct kbase_suspend_copy_buffer *sus_buf; ++ u8 group_handle; ++}; ++ ++/** ++ * struct kbase_cpu_command - Command which is to be part of the kernel ++ * command queue ++ * ++ * @type: Type of the command. ++ * @enqueue_ts: Denotes the relative time of enqueueing, a smaller value ++ * indicates that it has been enqueued earlier. ++ * @info: Structure which holds information about the command ++ * dependent on the command type. ++ */ ++struct kbase_kcpu_command { ++ enum base_kcpu_command_type type; ++ u64 enqueue_ts; ++ union { ++ struct kbase_kcpu_command_fence_info fence; ++ struct kbase_kcpu_command_cqs_wait_info cqs_wait; ++ struct kbase_kcpu_command_cqs_set_info cqs_set; ++ struct kbase_kcpu_command_import_info import; ++ struct kbase_kcpu_command_jit_alloc_info jit_alloc; ++ struct kbase_kcpu_command_jit_free_info jit_free; ++ struct kbase_kcpu_command_group_suspend_info suspend_buf_copy; ++ } info; ++}; ++ ++/** ++ * struct kbase_kcpu_command_queue - a command queue executed by the kernel ++ * ++ * @kctx: The context to which this command queue belongs. ++ * @commands: Array of commands which have been successfully ++ * enqueued to this command queue. ++ * @work: struct work_struct which contains a pointer to ++ * the function which handles processing of kcpu ++ * commands enqueued into a kcpu command queue; ++ * part of kernel API for processing workqueues ++ * @start_offset: Index of the command to be executed next ++ * @num_pending_cmds: The number of commands enqueued but not yet ++ * executed or pending ++ * @cqs_wait_count: Tracks the number of CQS wait commands enqueued ++ * @fence_context: The dma-buf fence context number for this kcpu ++ * queue. A unique context number is allocated for ++ * each kcpu queue. ++ * @fence_seqno: The dma-buf fence sequence number for the fence ++ * that is returned on the enqueue of fence signal ++ * command. This is increased every time the ++ * fence signal command is queued. ++ * @fence_wait_processed: Used to avoid reprocessing of the fence wait ++ * command which has blocked the processing of ++ * commands that follow it. ++ * @enqueue_failed: Indicates that no space has become available in ++ * the buffer since an enqueue operation failed ++ * because of insufficient free space. ++ * @command_started: Indicates that the command at the front of the ++ * queue has been started in a previous queue ++ * process, but was not completed due to some ++ * unmet dependencies. Ensures that instrumentation ++ * of the execution start of these commands is only ++ * fired exactly once. ++ * @has_error: Indicates that the kcpu queue is in error mode ++ * or without errors since last cleaned. ++ * @jit_blocked: Used to keep track of command queues blocked ++ * by a pending JIT allocation command. ++ */ ++struct kbase_kcpu_command_queue { ++ struct kbase_context *kctx; ++ struct kbase_kcpu_command commands[KBASEP_KCPU_QUEUE_SIZE]; ++ struct work_struct work; ++ u8 start_offset; ++ u16 num_pending_cmds; ++ u32 cqs_wait_count; ++ u64 fence_context; ++ unsigned int fence_seqno; ++ bool fence_wait_processed; ++ bool enqueue_failed; ++ bool command_started; ++ struct list_head jit_blocked; ++ bool has_error; ++}; ++ ++/** ++ * kbase_csf_kcpu_queue_new - Create new KCPU command queue. ++ * ++ * @kctx: Pointer to the kbase context within which the KCPU command ++ * queue will be created. ++ * @newq: Pointer to the structure which contains information about ++ * the new KCPU command queue to be created. ++ */ ++int kbase_csf_kcpu_queue_new(struct kbase_context *kctx, ++ struct kbase_ioctl_kcpu_queue_new *newq); ++ ++/** ++ * kbase_csf_kcpu_queue_delete - Delete KCPU command queue. ++ * ++ * Return: 0 if successful, -EINVAL if the queue ID is invalid. ++ * ++ * @kctx: Pointer to the kbase context from which the KCPU command ++ * queue is to be deleted. ++ * @del: Pointer to the structure which specifies the KCPU command ++ * queue to be deleted. ++ */ ++int kbase_csf_kcpu_queue_delete(struct kbase_context *kctx, ++ struct kbase_ioctl_kcpu_queue_delete *del); ++ ++/** ++ * kbase_csf_kcpu_queue_enqueue - Enqueue a KCPU command into a KCPU command ++ * queue. ++ * ++ * @kctx: Pointer to the kbase context within which the KCPU command ++ * is to be enqueued into the KCPU command queue. ++ * @enq: Pointer to the structure which specifies the KCPU command ++ * as well as the KCPU command queue into which the command ++ * is to be enqueued. ++ */ ++int kbase_csf_kcpu_queue_enqueue(struct kbase_context *kctx, ++ struct kbase_ioctl_kcpu_queue_enqueue *enq); ++ ++/** ++ * kbase_csf_kcpu_queue_context_init - Initialize the kernel CPU queues context ++ * for a GPU address space ++ * ++ * @kctx: Pointer to the kbase context being initialized. ++ * ++ * Return: 0 if successful or a negative error code on failure. ++ */ ++int kbase_csf_kcpu_queue_context_init(struct kbase_context *kctx); ++ ++/** ++ * kbase_csf_kcpu_queue_context_term - Terminate the kernel CPU queues context ++ * for a GPU address space ++ * ++ * This function deletes any kernel CPU queues that weren't deleted before ++ * context termination. ++ * ++ * @kctx: Pointer to the kbase context being terminated. ++ */ ++void kbase_csf_kcpu_queue_context_term(struct kbase_context *kctx); ++ ++#endif /* _KBASE_CSF_KCPU_H_ */ +diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu_debugfs.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu_debugfs.c +new file mode 100755 +index 000000000000..55e3b64cbe71 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu_debugfs.c +@@ -0,0 +1,199 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include "mali_kbase_csf_kcpu_debugfs.h" ++#include ++#include ++ ++#ifdef CONFIG_SYNC_FILE ++#include "mali_kbase_sync.h" ++#endif ++ ++#ifdef CONFIG_DEBUG_FS ++ ++/** ++ * kbasep_csf_kcpu_debugfs_print_queue() - Print additional info for KCPU ++ * queues blocked on CQS wait commands. ++ * ++ * @file: The seq_file to print to ++ * @kctx: The context of the KCPU queue ++ * @waits: Pointer to the KCPU CQS wait command info ++ */ ++static void kbasep_csf_kcpu_debugfs_print_cqs_waits(struct seq_file *file, ++ struct kbase_context *kctx, ++ struct kbase_kcpu_command_cqs_wait_info *waits) ++{ ++ unsigned int i; ++ ++ for (i = 0; i < waits->nr_objs; i++) { ++ struct kbase_vmap_struct *mapping; ++ u32 val; ++ char const *msg; ++ u32 *const cpu_ptr = (u32 *)kbase_phy_alloc_mapping_get(kctx, ++ waits->objs[i].addr, &mapping); ++ ++ if (!cpu_ptr) ++ return; ++ ++ val = *cpu_ptr; ++ ++ kbase_phy_alloc_mapping_put(kctx, mapping); ++ ++ msg = (waits->inherit_err_flags && (1U << i)) ? "true" : ++ "false"; ++ seq_printf(file, " %llx(%u > %u, inherit_err: %s), ", ++ waits->objs[i].addr, val, waits->objs[i].val, msg); ++ } ++} ++ ++/** ++ * kbasep_csf_kcpu_debugfs_print_queue() - Print debug data for a KCPU queue ++ * ++ * @file: The seq_file to print to ++ * @kctx: The context of the KCPU queue ++ * @queue: Pointer to the KCPU queue ++ */ ++static void kbasep_csf_kcpu_debugfs_print_queue(struct seq_file *file, ++ struct kbase_context *kctx, ++ struct kbase_kcpu_command_queue *queue) ++{ ++ if (WARN_ON(!queue)) ++ return; ++ ++ lockdep_assert_held(&kctx->csf.kcpu_queues.lock); ++ ++ seq_printf(file, "%16u, %11u, %7u, %13llu %8u", ++ queue->num_pending_cmds, queue->enqueue_failed, ++ queue->command_started ? 1 : 0, ++ queue->fence_context, queue->fence_seqno); ++ ++ if (queue->command_started) { ++ struct kbase_kcpu_command *cmd = ++ &queue->commands[queue->start_offset]; ++ switch (cmd->type) { ++#ifdef CONFIG_SYNC_FILE ++ case BASE_KCPU_COMMAND_TYPE_FENCE_WAIT: ++ { ++ struct kbase_sync_fence_info info; ++ ++ kbase_sync_fence_info_get(cmd->info.fence.fence, &info); ++ seq_printf(file, ", Fence %p %s %s", ++ info.fence, info.name, ++ kbase_sync_status_string(info.status)); ++ break; ++ } ++#endif ++ case BASE_KCPU_COMMAND_TYPE_CQS_WAIT: ++ seq_puts(file, ", CQS "); ++ kbasep_csf_kcpu_debugfs_print_cqs_waits(file, kctx, ++ &cmd->info.cqs_wait); ++ break; ++ default: ++ seq_puts(file, ", U, Unknown blocking command"); ++ break; ++ } ++ } ++ ++ seq_puts(file, "\n"); ++} ++ ++/** ++ * kbasep_csf_kcpu_debugfs_show() - Print the KCPU queues debug information ++ * ++ * @file: The seq_file for printing to ++ * @data: The debugfs dentry private data, a pointer to kbase_context ++ * ++ * Return: Negative error code or 0 on success. ++ */ ++static int kbasep_csf_kcpu_debugfs_show(struct seq_file *file, void *data) ++{ ++ struct kbase_context *kctx = file->private; ++ unsigned long idx; ++ ++ seq_printf(file, "MALI_CSF_KCPU_DEBUGFS_VERSION: v%u\n", MALI_CSF_KCPU_DEBUGFS_VERSION); ++ seq_puts(file, "Queue Idx(err-mode), Pending Commands, Enqueue err, Blocked, Fence context & seqno, (Wait Type, Additional info)\n"); ++ mutex_lock(&kctx->csf.kcpu_queues.lock); ++ ++ idx = find_first_bit(kctx->csf.kcpu_queues.in_use, ++ KBASEP_MAX_KCPU_QUEUES); ++ ++ while (idx < KBASEP_MAX_KCPU_QUEUES) { ++ struct kbase_kcpu_command_queue *queue = ++ kctx->csf.kcpu_queues.array[idx]; ++ ++ seq_printf(file, "%9lu( %s ), ", idx, ++ queue->has_error ? "InErr" : "NoErr"); ++ kbasep_csf_kcpu_debugfs_print_queue(file, kctx, ++ kctx->csf.kcpu_queues.array[idx]); ++ ++ idx = find_next_bit(kctx->csf.kcpu_queues.in_use, ++ KBASEP_MAX_KCPU_QUEUES, idx + 1); ++ } ++ ++ mutex_unlock(&kctx->csf.kcpu_queues.lock); ++ return 0; ++} ++ ++static int kbasep_csf_kcpu_debugfs_open(struct inode *in, struct file *file) ++{ ++ return single_open(file, kbasep_csf_kcpu_debugfs_show, in->i_private); ++} ++ ++static const struct file_operations kbasep_csf_kcpu_debugfs_fops = { ++ .open = kbasep_csf_kcpu_debugfs_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = single_release, ++}; ++ ++void kbase_csf_kcpu_debugfs_init(struct kbase_context *kctx) ++{ ++ struct dentry *file; ++#if (KERNEL_VERSION(4, 7, 0) <= LINUX_VERSION_CODE) ++ const mode_t mode = 0444; ++#else ++ const mode_t mode = 0400; ++#endif ++ ++ if (WARN_ON(!kctx || IS_ERR_OR_NULL(kctx->kctx_dentry))) ++ return; ++ ++ file = debugfs_create_file("kcpu_queues", mode, kctx->kctx_dentry, ++ kctx, &kbasep_csf_kcpu_debugfs_fops); ++ ++ if (IS_ERR_OR_NULL(file)) { ++ dev_warn(kctx->kbdev->dev, ++ "Unable to create KCPU debugfs entry"); ++ } ++} ++ ++ ++#else ++/* ++ * Stub functions for when debugfs is disabled ++ */ ++void kbase_csf_kcpu_debugfs_init(struct kbase_context *kctx) ++{ ++} ++ ++#endif /* CONFIG_DEBUG_FS */ ++ +diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu_debugfs.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu_debugfs.h +new file mode 100755 +index 000000000000..359fe2cb0168 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu_debugfs.h +@@ -0,0 +1,38 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_CSF_KCPU_DEBUGFS_H_ ++#define _KBASE_CSF_KCPU_DEBUGFS_H_ ++ ++/* Forward declaration */ ++struct kbase_context; ++ ++#define MALI_CSF_KCPU_DEBUGFS_VERSION 0 ++ ++/** ++ * kbase_csf_kcpu_debugfs_init() - Create a debugfs entry for KCPU queues ++ * ++ * @kctx: The kbase_context for which to create the debugfs entry ++ */ ++void kbase_csf_kcpu_debugfs_init(struct kbase_context *kctx); ++ ++#endif /* _KBASE_CSF_KCPU_DEBUGFS_H_ */ +diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_protected_memory.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_protected_memory.c +new file mode 100755 +index 000000000000..987cbc2fc201 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_protected_memory.c +@@ -0,0 +1,120 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include "mali_kbase_csf_protected_memory.h" ++#include ++ ++#ifdef CONFIG_OF ++#include ++#endif ++ ++int kbase_csf_protected_memory_init(struct kbase_device *const kbdev) ++{ ++ int err = 0; ++ ++#if CONFIG_OF ++ struct device_node *pma_node = of_parse_phandle(kbdev->dev->of_node, ++ "protected-memory-allocator", 0); ++ if (!pma_node) { ++ dev_info(kbdev->dev, "Protected memory allocator not available\n"); ++ } else { ++ struct platform_device *const pdev = ++ of_find_device_by_node(pma_node); ++ ++ kbdev->csf.pma_dev = NULL; ++ if (!pdev) { ++ dev_err(kbdev->dev, "Platform device for Protected memory allocator not found\n"); ++ } else { ++ kbdev->csf.pma_dev = platform_get_drvdata(pdev); ++ if (!kbdev->csf.pma_dev) { ++ dev_info(kbdev->dev, "Protected memory allocator is not ready\n"); ++ err = -EPROBE_DEFER; ++ } else if (!try_module_get(kbdev->csf.pma_dev->owner)) { ++ dev_err(kbdev->dev, "Failed to get Protected memory allocator module\n"); ++ err = -ENODEV; ++ } else { ++ dev_info(kbdev->dev, "Protected memory allocator successfully loaded\n"); ++ } ++ } ++ of_node_put(pma_node); ++ } ++#endif ++ ++ return err; ++} ++ ++void kbase_csf_protected_memory_term(struct kbase_device *const kbdev) ++{ ++ if (kbdev->csf.pma_dev) ++ module_put(kbdev->csf.pma_dev->owner); ++} ++ ++struct protected_memory_allocation ** ++ kbase_csf_protected_memory_alloc( ++ struct kbase_device *const kbdev, ++ struct tagged_addr *phys, ++ size_t num_pages) ++{ ++ size_t i; ++ struct protected_memory_allocator_device *pma_dev = ++ kbdev->csf.pma_dev; ++ struct protected_memory_allocation **pma = ++ kmalloc_array(num_pages, sizeof(*pma), GFP_KERNEL); ++ ++ if (WARN_ON(!pma_dev) || WARN_ON(!phys) || !pma) ++ return NULL; ++ ++ for (i = 0; i < num_pages; i++) { ++ pma[i] = pma_dev->ops.pma_alloc_page(pma_dev, ++ KBASE_MEM_POOL_4KB_PAGE_TABLE_ORDER); ++ if (!pma[i]) ++ break; ++ ++ phys[i] = as_tagged(pma_dev->ops.pma_get_phys_addr(pma_dev, ++ pma[i])); ++ } ++ ++ if (i != num_pages) { ++ kbase_csf_protected_memory_free(kbdev, pma, i); ++ return NULL; ++ } ++ ++ return pma; ++} ++ ++void kbase_csf_protected_memory_free( ++ struct kbase_device *const kbdev, ++ struct protected_memory_allocation **pma, ++ size_t num_pages) ++{ ++ size_t i; ++ struct protected_memory_allocator_device *pma_dev = ++ kbdev->csf.pma_dev; ++ ++ if (WARN_ON(!pma_dev) || WARN_ON(!pma)) ++ return; ++ ++ for (i = 0; i < num_pages; i++) ++ pma_dev->ops.pma_free_page(pma_dev, pma[i]); ++ ++ kfree(pma); ++} +diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_protected_memory.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_protected_memory.h +new file mode 100755 +index 000000000000..2b459911d834 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_protected_memory.h +@@ -0,0 +1,72 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_CSF_PROTECTED_MEMORY_H_ ++#define _KBASE_CSF_PROTECTED_MEMORY_H_ ++ ++#include "mali_kbase.h" ++/** ++ * kbase_csf_protected_memory_init - Initilaise protected memory allocator. ++ * ++ * @kbdev: Device pointer. ++ * ++ * Return: 0 if success, or an error code on failure. ++ */ ++int kbase_csf_protected_memory_init(struct kbase_device *const kbdev); ++ ++/** ++ * kbase_csf_protected_memory_term - Terminate prtotected memory allocator. ++ * ++ * @kbdev: Device pointer. ++ */ ++void kbase_csf_protected_memory_term(struct kbase_device *const kbdev); ++ ++/** ++ * kbase_csf_protected_memory_alloc - Allocate protected memory pages. ++ * ++ * @kbdev: Device pointer. ++ * @phys: Array of physical addresses to be filled in by the protected ++ * memory allocator. ++ * @num_pages: Number of pages requested to be allocated. ++ * ++ * Return: Pointer to an array of protected memory allocations on success, ++ * or NULL on failure. ++ */ ++struct protected_memory_allocation ** ++ kbase_csf_protected_memory_alloc( ++ struct kbase_device *const kbdev, ++ struct tagged_addr *phys, ++ size_t num_pages); ++ ++/** ++ * kbase_csf_protected_memory_free - Free the allocated ++ * protected memory pages ++ * ++ * @kbdev: Device pointer. ++ * @pma: Array of pointer to protected memory allocations. ++ * @num_pages: Number of pages to be freed. ++ */ ++void kbase_csf_protected_memory_free( ++ struct kbase_device *const kbdev, ++ struct protected_memory_allocation **pma, ++ size_t num_pages); ++#endif +diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_reset_gpu.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_reset_gpu.c +new file mode 100755 +index 000000000000..f1a318d26f43 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_reset_gpu.c +@@ -0,0 +1,355 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++/* Waiting timeout for GPU reset to complete */ ++#define GPU_RESET_TIMEOUT_MS (5000) /* 5 seconds */ ++#define DUMP_DWORDS_PER_LINE (4) ++/* 16 characters needed for a 8 byte value in hex & 1 character for space */ ++#define DUMP_HEX_CHARS_PER_DWORD ((2 * 8) + 1) ++#define DUMP_HEX_CHARS_PER_LINE \ ++ (DUMP_DWORDS_PER_LINE * DUMP_HEX_CHARS_PER_DWORD) ++ ++static void kbase_csf_debug_dump_registers(struct kbase_device *kbdev) ++{ ++ kbase_io_history_dump(kbdev); ++ ++ dev_err(kbdev->dev, "Register state:"); ++ dev_err(kbdev->dev, " GPU_IRQ_RAWSTAT=0x%08x GPU_STATUS=0x%08x MCU_STATUS=0x%08x", ++ kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT)), ++ kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_STATUS)), ++ kbase_reg_read(kbdev, GPU_CONTROL_REG(MCU_STATUS))); ++ dev_err(kbdev->dev, " JOB_IRQ_RAWSTAT=0x%08x MMU_IRQ_RAWSTAT=0x%08x GPU_FAULTSTATUS=0x%08x", ++ kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_RAWSTAT)), ++ kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_RAWSTAT)), ++ kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_FAULTSTATUS))); ++ dev_err(kbdev->dev, " GPU_IRQ_MASK=0x%08x JOB_IRQ_MASK=0x%08x MMU_IRQ_MASK=0x%08x", ++ kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK)), ++ kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK)), ++ kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK))); ++ dev_err(kbdev->dev, " PWR_OVERRIDE0=0x%08x PWR_OVERRIDE1=0x%08x", ++ kbase_reg_read(kbdev, GPU_CONTROL_REG(PWR_OVERRIDE0)), ++ kbase_reg_read(kbdev, GPU_CONTROL_REG(PWR_OVERRIDE1))); ++ dev_err(kbdev->dev, " SHADER_CONFIG=0x%08x L2_MMU_CONFIG=0x%08x TILER_CONFIG=0x%08x", ++ kbase_reg_read(kbdev, GPU_CONTROL_REG(SHADER_CONFIG)), ++ kbase_reg_read(kbdev, GPU_CONTROL_REG(L2_MMU_CONFIG)), ++ kbase_reg_read(kbdev, GPU_CONTROL_REG(TILER_CONFIG))); ++} ++ ++static void kbase_csf_dump_firmware_trace_buffer(struct kbase_device *kbdev) ++{ ++ u8 *buf, *line_str; ++ unsigned int read_size; ++ struct firmware_trace_buffer *tb = ++ kbase_csf_firmware_get_trace_buffer(kbdev, FW_TRACE_BUF_NAME); ++ ++ if (tb == NULL) { ++ dev_dbg(kbdev->dev, "Can't get the trace buffer, firmware trace dump skipped"); ++ return; ++ } ++ ++ buf = kmalloc(PAGE_SIZE + DUMP_HEX_CHARS_PER_LINE + 1, GFP_KERNEL); ++ if (buf == NULL) { ++ dev_err(kbdev->dev, "Short of memory, firmware trace dump skipped"); ++ return; ++ } ++ line_str = &buf[PAGE_SIZE]; ++ ++ dev_err(kbdev->dev, "Firmware trace buffer dump:"); ++ while ((read_size = kbase_csf_firmware_trace_buffer_read_data(tb, buf, ++ PAGE_SIZE))) { ++ u64 *ptr = (u64 *)buf; ++ u32 num_dwords; ++ ++ for (num_dwords = read_size / sizeof(u64); ++ num_dwords >= DUMP_DWORDS_PER_LINE; ++ num_dwords -= DUMP_DWORDS_PER_LINE) { ++ dev_err(kbdev->dev, "%016llx %016llx %016llx %016llx", ++ ptr[0], ptr[1], ptr[2], ptr[3]); ++ ptr += DUMP_DWORDS_PER_LINE; ++ } ++ ++ if (num_dwords) { ++ int pos = 0; ++ ++ while (num_dwords--) { ++ pos += snprintf(line_str + pos, ++ DUMP_HEX_CHARS_PER_DWORD + 1, ++ "%016llx ", ptr[0]); ++ ptr++; ++ } ++ ++ dev_err(kbdev->dev, "%s", line_str); ++ } ++ } ++ ++ kfree(buf); ++} ++ ++static int kbase_csf_reset_gpu_now(struct kbase_device *kbdev, ++ bool firmware_inited) ++{ ++ unsigned long flags; ++ bool silent = false; ++ int err; ++ ++ if (atomic_read(&kbdev->csf.reset.state) == KBASE_CSF_RESET_GPU_SILENT) ++ silent = true; ++ ++ WARN_ON(kbdev->irq_reset_flush); ++ ++ /* Reset the scheduler state before disabling the interrupts as suspend of active ++ * CSG slots would also be done as a part of reset. ++ */ ++ if (likely(firmware_inited)) ++ kbase_csf_scheduler_reset(kbdev); ++ cancel_work_sync(&kbdev->csf.firmware_reload_work); ++ ++ /* Disable GPU hardware counters. ++ * This call will block until counters are disabled. ++ */ ++ kbase_hwcnt_context_disable(kbdev->hwcnt_gpu_ctx); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ spin_lock(&kbdev->mmu_mask_change); ++ kbase_pm_reset_start_locked(kbdev); ++ ++ /* We're about to flush out the IRQs and their bottom halves */ ++ kbdev->irq_reset_flush = true; ++ ++ /* Disable IRQ to avoid IRQ handlers to kick in after releasing the ++ * spinlock; this also clears any outstanding interrupts ++ */ ++ kbase_pm_disable_interrupts_nolock(kbdev); ++ ++ spin_unlock(&kbdev->mmu_mask_change); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ /* Ensure that any IRQ handlers have finished ++ * Must be done without any locks IRQ handlers will take. ++ */ ++ kbase_synchronize_irqs(kbdev); ++ ++ /* Flush out any in-flight work items */ ++ kbase_flush_mmu_wqs(kbdev); ++ ++ /* The flush has completed so reset the active indicator */ ++ kbdev->irq_reset_flush = false; ++ ++ mutex_lock(&kbdev->pm.lock); ++ if (!silent) ++ dev_err(kbdev->dev, "Resetting GPU (allowing up to %d ms)", ++ RESET_TIMEOUT); ++ ++ /* Output the state of some interesting registers to help in the ++ * debugging of GPU resets, and dump the firmware trace buffer ++ */ ++ if (!silent) { ++ kbase_csf_debug_dump_registers(kbdev); ++ if (likely(firmware_inited)) ++ kbase_csf_dump_firmware_trace_buffer(kbdev); ++ } ++ ++ /* Reset the GPU */ ++ err = kbase_pm_init_hw(kbdev, 0); ++ ++ mutex_unlock(&kbdev->pm.lock); ++ ++ if (WARN_ON(err)) ++ return err; ++ ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_ctx_sched_restore_all_as(kbdev); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ ++ kbase_pm_enable_interrupts(kbdev); ++ ++ mutex_lock(&kbdev->pm.lock); ++ kbase_pm_reset_complete(kbdev); ++ /* Synchronously wait for the reload of firmware to complete */ ++ err = kbase_pm_wait_for_desired_state(kbdev); ++ mutex_unlock(&kbdev->pm.lock); ++ ++ if (err) ++ return err; ++ ++ /* Re-enable GPU hardware counters */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ if (!silent) ++ dev_err(kbdev->dev, "Reset complete"); ++ ++ return 0; ++} ++ ++static void kbase_csf_reset_gpu_worker(struct work_struct *data) ++{ ++ struct kbase_device *kbdev = container_of(data, struct kbase_device, ++ csf.reset.work); ++ bool firmware_inited; ++ unsigned long flags; ++ int err = 0; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ firmware_inited = kbdev->csf.firmware_inited; ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ if (!kbase_pm_context_active_handle_suspend(kbdev, ++ KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) { ++ err = kbase_csf_reset_gpu_now(kbdev, firmware_inited); ++ kbase_pm_context_idle(kbdev); ++ } ++ ++ kbase_disjoint_state_down(kbdev); ++ ++ if (!err) { ++ atomic_set(&kbdev->csf.reset.state, ++ KBASE_CSF_RESET_GPU_NOT_PENDING); ++ if (likely(firmware_inited)) ++ kbase_csf_scheduler_enable_tick_timer(kbdev); ++ } else { ++ dev_err(kbdev->dev, "Reset failed to complete"); ++ atomic_set(&kbdev->csf.reset.state, ++ KBASE_CSF_RESET_GPU_FAILED); ++ } ++ ++ wake_up(&kbdev->csf.reset.wait); ++} ++ ++bool kbase_prepare_to_reset_gpu(struct kbase_device *kbdev) ++{ ++ if (atomic_cmpxchg(&kbdev->csf.reset.state, ++ KBASE_CSF_RESET_GPU_NOT_PENDING, ++ KBASE_CSF_RESET_GPU_HAPPENING) != ++ KBASE_CSF_RESET_GPU_NOT_PENDING) { ++ /* Some other thread is already resetting the GPU */ ++ return false; ++ } ++ ++ return true; ++} ++KBASE_EXPORT_TEST_API(kbase_prepare_to_reset_gpu); ++ ++bool kbase_prepare_to_reset_gpu_locked(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ return kbase_prepare_to_reset_gpu(kbdev); ++} ++ ++int kbase_reset_gpu(struct kbase_device *kbdev) ++{ ++ dev_err(kbdev->dev, "Preparing to soft-reset GPU\n"); ++ ++ kbase_disjoint_state_up(kbdev); ++ ++ queue_work(kbdev->csf.reset.workq, &kbdev->csf.reset.work); ++ ++ return 0; ++} ++KBASE_EXPORT_TEST_API(kbase_reset_gpu); ++ ++void kbase_reset_gpu_locked(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ kbase_reset_gpu(kbdev); ++} ++ ++int kbase_reset_gpu_silent(struct kbase_device *kbdev) ++{ ++ if (atomic_cmpxchg(&kbdev->csf.reset.state, ++ KBASE_CSF_RESET_GPU_NOT_PENDING, ++ KBASE_CSF_RESET_GPU_SILENT) != ++ KBASE_CSF_RESET_GPU_NOT_PENDING) { ++ /* Some other thread is already resetting the GPU */ ++ return -EAGAIN; ++ } ++ ++ kbase_disjoint_state_up(kbdev); ++ ++ queue_work(kbdev->csf.reset.workq, &kbdev->csf.reset.work); ++ ++ return 0; ++} ++ ++bool kbase_reset_gpu_is_active(struct kbase_device *kbdev) ++{ ++ if (atomic_read(&kbdev->csf.reset.state) == ++ KBASE_CSF_RESET_GPU_NOT_PENDING) ++ return false; ++ ++ return true; ++} ++ ++int kbase_reset_gpu_wait(struct kbase_device *kbdev) ++{ ++ const long wait_timeout = ++ kbase_csf_timeout_in_jiffies(GPU_RESET_TIMEOUT_MS); ++ long remaining = wait_event_timeout(kbdev->csf.reset.wait, ++ (atomic_read(&kbdev->csf.reset.state) == ++ KBASE_CSF_RESET_GPU_NOT_PENDING) || ++ (atomic_read(&kbdev->csf.reset.state) == ++ KBASE_CSF_RESET_GPU_FAILED), ++ wait_timeout); ++ ++ if (!remaining) { ++ dev_warn(kbdev->dev, "Timed out waiting for the GPU reset to complete"); ++ return -ETIMEDOUT; ++ } else if (atomic_read(&kbdev->csf.reset.state) == ++ KBASE_CSF_RESET_GPU_FAILED) { ++ return -ENOMEM; ++ } ++ ++ return 0; ++} ++KBASE_EXPORT_TEST_API(kbase_reset_gpu_wait); ++ ++int kbase_reset_gpu_init(struct kbase_device *kbdev) ++{ ++ kbdev->csf.reset.workq = alloc_workqueue("Mali reset workqueue", 0, 1); ++ if (kbdev->csf.reset.workq == NULL) ++ return -ENOMEM; ++ ++ INIT_WORK(&kbdev->csf.reset.work, kbase_csf_reset_gpu_worker); ++ ++ init_waitqueue_head(&kbdev->csf.reset.wait); ++ ++ return 0; ++} ++ ++void kbase_reset_gpu_term(struct kbase_device *kbdev) ++{ ++ destroy_workqueue(kbdev->csf.reset.workq); ++} +diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.c +new file mode 100755 +index 000000000000..a3017a7f25ba +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.c +@@ -0,0 +1,4135 @@ ++/* ++ * ++ * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++#include "mali_kbase_config_defaults.h" ++#include ++#include ++#include ++#include ++#include "mali_kbase_csf.h" ++#include "../tl/mali_kbase_tracepoints.h" ++#include "backend/gpu/mali_kbase_pm_internal.h" ++#include ++#include "mali_gpu_csf_registers.h" ++#include ++ ++/* Value to indicate that a queue group is not groups_to_schedule list */ ++#define KBASEP_GROUP_PREPARED_SEQ_NUM_INVALID (U32_MAX) ++ ++/* Waiting timeout for status change acknowledgment, in milliseconds */ ++#define CSF_STATE_WAIT_TIMEOUT_MS (800) /* Relaxed to 800ms from 100ms */ ++ ++/* Waiting timeout for scheduler state change for descheduling a CSG */ ++#define CSG_SCHED_STOP_TIMEOUT_MS (50) ++ ++#define CSG_SUSPEND_ON_RESET_WAIT_TIMEOUT_MS DEFAULT_RESET_TIMEOUT_MS ++ ++/* Maximum number of endpoints which may run tiler jobs. */ ++#define CSG_TILER_MAX ((u8)1) ++ ++/* Maximum dynamic CSG slot priority value */ ++#define MAX_CSG_SLOT_PRIORITY ((u8)15) ++ ++/* CSF scheduler time slice value */ ++#define CSF_SCHEDULER_TIME_TICK_MS (100) /* 100 milliseconds */ ++#define CSF_SCHEDULER_TIME_TICK_JIFFIES \ ++ msecs_to_jiffies(CSF_SCHEDULER_TIME_TICK_MS) ++ ++/* ++ * CSF scheduler time threshold for converting "tock" requests into "tick" if ++ * they come too close to the end of a tick interval. This avoids scheduling ++ * twice in a row. ++ */ ++#define CSF_SCHEDULER_TIME_TICK_THRESHOLD_MS \ ++ CSF_SCHEDULER_TIME_TICK_MS ++ ++#define CSF_SCHEDULER_TIME_TICK_THRESHOLD_JIFFIES \ ++ msecs_to_jiffies(CSF_SCHEDULER_TIME_TICK_THRESHOLD_MS) ++ ++/* Nanoseconds per millisecond */ ++#define NS_PER_MS ((u64)1000 * 1000) ++ ++/* ++ * CSF minimum time to reschedule for a new "tock" request. Bursts of "tock" ++ * requests are not serviced immediately, but shall wait for a minimum time in ++ * order to reduce load on the CSF scheduler thread. ++ */ ++#define CSF_SCHEDULER_TIME_TOCK_JIFFIES 1 /* 1 jiffies-time */ ++ ++/* Command stream suspended and is idle (empty ring buffer) */ ++#define CS_IDLE_FLAG (1 << 0) ++ ++/* Command stream suspended and is wait for a CQS condition */ ++#define CS_WAIT_SYNC_FLAG (1 << 1) ++ ++/* This is to avoid the immediate power down of GPU when then are no groups ++ * left for scheduling. GPUCORE-24250 would add the proper GPU idle detection ++ * logic. ++ */ ++#define GPU_IDLE_POWEROFF_HYSTERESIS_DELAY msecs_to_jiffies((u32)10) ++ ++static int scheduler_group_schedule(struct kbase_queue_group *group); ++static void remove_group_from_idle_wait(struct kbase_queue_group *const group); ++static ++void insert_group_to_runnable(struct kbase_csf_scheduler *const scheduler, ++ struct kbase_queue_group *const group, ++ enum kbase_csf_group_state run_state); ++static struct kbase_queue_group *scheduler_get_protm_enter_async_group( ++ struct kbase_device *const kbdev, ++ struct kbase_queue_group *const group); ++static struct kbase_queue_group *get_tock_top_group( ++ struct kbase_csf_scheduler *const scheduler); ++static void scheduler_enable_tick_timer_nolock(struct kbase_device *kbdev); ++static int suspend_active_queue_groups(struct kbase_device *kbdev, ++ unsigned long *slot_mask); ++ ++#define kctx_as_enabled(kctx) (!kbase_ctx_flag(kctx, KCTX_AS_DISABLED_ON_FAULT)) ++ ++static void release_doorbell(struct kbase_device *kbdev, int doorbell_nr) ++{ ++ WARN_ON(doorbell_nr >= CSF_NUM_DOORBELL); ++ ++ lockdep_assert_held(&kbdev->csf.scheduler.lock); ++ clear_bit(doorbell_nr, kbdev->csf.scheduler.doorbell_inuse_bitmap); ++} ++ ++static int acquire_doorbell(struct kbase_device *kbdev) ++{ ++ int doorbell_nr; ++ ++ lockdep_assert_held(&kbdev->csf.scheduler.lock); ++ ++ doorbell_nr = find_first_zero_bit( ++ kbdev->csf.scheduler.doorbell_inuse_bitmap, ++ CSF_NUM_DOORBELL); ++ ++ if (doorbell_nr >= CSF_NUM_DOORBELL) ++ return KBASEP_USER_DB_NR_INVALID; ++ ++ set_bit(doorbell_nr, kbdev->csf.scheduler.doorbell_inuse_bitmap); ++ ++ return doorbell_nr; ++} ++ ++static void unassign_user_doorbell_from_group(struct kbase_device *kbdev, ++ struct kbase_queue_group *group) ++{ ++ lockdep_assert_held(&kbdev->csf.scheduler.lock); ++ ++ if (group->doorbell_nr != KBASEP_USER_DB_NR_INVALID) { ++ release_doorbell(kbdev, group->doorbell_nr); ++ group->doorbell_nr = KBASEP_USER_DB_NR_INVALID; ++ } ++} ++ ++static void unassign_user_doorbell_from_queue(struct kbase_device *kbdev, ++ struct kbase_queue *queue) ++{ ++ lockdep_assert_held(&kbdev->csf.scheduler.lock); ++ ++ mutex_lock(&kbdev->csf.reg_lock); ++ ++ if (queue->doorbell_nr != KBASEP_USER_DB_NR_INVALID) { ++ queue->doorbell_nr = KBASEP_USER_DB_NR_INVALID; ++ /* After this the dummy page would be mapped in */ ++ unmap_mapping_range(kbdev->csf.db_filp->f_inode->i_mapping, ++ queue->db_file_offset << PAGE_SHIFT, PAGE_SIZE, 1); ++ } ++ ++ mutex_unlock(&kbdev->csf.reg_lock); ++} ++ ++static void assign_user_doorbell_to_group(struct kbase_device *kbdev, ++ struct kbase_queue_group *group) ++{ ++ lockdep_assert_held(&kbdev->csf.scheduler.lock); ++ ++ if (group->doorbell_nr == KBASEP_USER_DB_NR_INVALID) ++ group->doorbell_nr = acquire_doorbell(kbdev); ++} ++ ++static void assign_user_doorbell_to_queue(struct kbase_device *kbdev, ++ struct kbase_queue *const queue) ++{ ++ lockdep_assert_held(&kbdev->csf.scheduler.lock); ++ ++ mutex_lock(&kbdev->csf.reg_lock); ++ ++ /* If bind operation for the queue hasn't completed yet, then the ++ * the command stream interface can't be programmed for the queue ++ * (even in stopped state) and so the doorbell also can't be assigned ++ * to it. ++ */ ++ if ((queue->bind_state == KBASE_CSF_QUEUE_BOUND) && ++ (queue->doorbell_nr == KBASEP_USER_DB_NR_INVALID)) { ++ WARN_ON(queue->group->doorbell_nr == KBASEP_USER_DB_NR_INVALID); ++ queue->doorbell_nr = queue->group->doorbell_nr; ++ ++ /* After this the real Hw doorbell page would be mapped in */ ++ unmap_mapping_range( ++ kbdev->csf.db_filp->f_inode->i_mapping, ++ queue->db_file_offset << PAGE_SHIFT, ++ PAGE_SIZE, 1); ++ } ++ ++ mutex_unlock(&kbdev->csf.reg_lock); ++} ++ ++static void scheduler_doorbell_init(struct kbase_device *kbdev) ++{ ++ int doorbell_nr; ++ ++ bitmap_zero(kbdev->csf.scheduler.doorbell_inuse_bitmap, ++ CSF_NUM_DOORBELL); ++ ++ mutex_lock(&kbdev->csf.scheduler.lock); ++ /* Reserve doorbell 0 for use by kernel driver */ ++ doorbell_nr = acquire_doorbell(kbdev); ++ mutex_unlock(&kbdev->csf.scheduler.lock); ++ ++ WARN_ON(doorbell_nr != CSF_KERNEL_DOORBELL_NR); ++} ++ ++static u32 get_nr_active_csgs(struct kbase_device *kbdev) ++{ ++ u32 nr_active_csgs; ++ ++ lockdep_assert_held(&kbdev->csf.scheduler.lock); ++ ++ nr_active_csgs = bitmap_weight(kbdev->csf.scheduler.csg_inuse_bitmap, ++ kbdev->csf.global_iface.group_num); ++ ++ return nr_active_csgs; ++} ++ ++/** ++ * csgs_active - returns true if any of CSG slots are in use ++ * ++ * @kbdev: Instance of a GPU platform device that implements a command ++ * stream front-end interface. ++ * ++ * Return: the interface is actively engaged flag. ++ */ ++bool csgs_active(struct kbase_device *kbdev) ++{ ++ u32 nr_active_csgs; ++ ++ mutex_lock(&kbdev->csf.scheduler.lock); ++ nr_active_csgs = get_nr_active_csgs(kbdev); ++ mutex_unlock(&kbdev->csf.scheduler.lock); ++ ++ /* Right now if any of the command stream group interfaces are in use ++ * then we need to assume that there is some work pending. ++ * In future when we have IDLE notifications from firmware implemented ++ * then we would have a better idea of the pending work. ++ */ ++ return (nr_active_csgs != 0); ++} ++ ++/** ++ * csg_slot_in_use - returns true if a queue group has been programmed on a ++ * given CSG slot. ++ * ++ * @kbdev: Instance of a GPU platform device that implements a command ++ * stream front-end interface. ++ * @slot: Index/number of the CSG slot in question. ++ * ++ * Return: the interface is actively engaged flag. ++ * ++ * Note: Caller must hold the scheduler lock. ++ */ ++static inline bool csg_slot_in_use(struct kbase_device *kbdev, int slot) ++{ ++ lockdep_assert_held(&kbdev->csf.scheduler.lock); ++ ++ return (kbdev->csf.scheduler.csg_slots[slot].resident_group != NULL); ++} ++ ++static bool queue_group_suspended_locked(struct kbase_queue_group *group) ++{ ++ lockdep_assert_held(&group->kctx->kbdev->csf.scheduler.lock); ++ ++ return (group->run_state == KBASE_CSF_GROUP_SUSPENDED || ++ group->run_state == KBASE_CSF_GROUP_SUSPENDED_ON_IDLE || ++ group->run_state == KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC); ++} ++ ++static bool queue_group_idle_locked(struct kbase_queue_group *group) ++{ ++ lockdep_assert_held(&group->kctx->kbdev->csf.scheduler.lock); ++ ++ return (group->run_state == KBASE_CSF_GROUP_IDLE || ++ group->run_state == KBASE_CSF_GROUP_SUSPENDED_ON_IDLE); ++} ++ ++static bool queue_group_scheduled(struct kbase_queue_group *group) ++{ ++ return (group->run_state != KBASE_CSF_GROUP_INACTIVE && ++ group->run_state != KBASE_CSF_GROUP_TERMINATED && ++ group->run_state != KBASE_CSF_GROUP_FAULT_EVICTED); ++} ++ ++static bool queue_group_scheduled_locked(struct kbase_queue_group *group) ++{ ++ lockdep_assert_held(&group->kctx->kbdev->csf.scheduler.lock); ++ ++ return queue_group_scheduled(group); ++} ++ ++/** ++ * scheduler_timer_is_enabled_nolock() - Check if the scheduler wakes up ++ * automatically for periodic tasks. ++ * ++ * @kbdev: Pointer to the device ++ * ++ * This is a variant of kbase_csf_scheduler_timer_is_enabled() that assumes the ++ * CSF scheduler lock to already have been held. ++ * ++ * Return: true if the scheduler is configured to wake up periodically ++ */ ++static bool scheduler_timer_is_enabled_nolock(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->csf.scheduler.lock); ++ ++ return kbdev->csf.scheduler.timer_enabled; ++} ++ ++static void scheduler_wakeup(struct kbase_device *kbdev, bool kick) ++{ ++ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; ++ ++ lockdep_assert_held(&scheduler->lock); ++ ++ if (scheduler->state == SCHED_SUSPENDED) { ++ dev_info(kbdev->dev, "Re-activating the Scheduler"); ++ kbase_csf_scheduler_pm_active(kbdev); ++ scheduler->state = SCHED_INACTIVE; ++ ++ if (kick) ++ scheduler_enable_tick_timer_nolock(kbdev); ++ } ++} ++ ++static void scheduler_suspend(struct kbase_device *kbdev) ++{ ++ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; ++ ++ lockdep_assert_held(&scheduler->lock); ++ ++ if (!WARN_ON(scheduler->state == SCHED_SUSPENDED)) { ++ dev_dbg(kbdev->dev, "Suspending the Scheduler"); ++ kbase_csf_scheduler_pm_idle(kbdev); ++ scheduler->state = SCHED_SUSPENDED; ++ } ++} ++ ++/** ++ * update_idle_suspended_group_state() - Move the queue group to a non-idle ++ * suspended state. ++ * @group: Pointer to the queue group. ++ * ++ * This function is called to change the state of queue group to non-idle ++ * suspended state, if the group was suspended when all the queues bound to it ++ * became empty or when some queues got blocked on a sync wait & others became ++ * empty. The group is also moved to the runnbale list from idle wait list in ++ * the latter case. ++ * So the function gets called when a queue is kicked or sync wait condition ++ * gets satisfied. ++ */ ++static void update_idle_suspended_group_state(struct kbase_queue_group *group) ++{ ++ struct kbase_csf_scheduler *scheduler = ++ &group->kctx->kbdev->csf.scheduler; ++ ++ lockdep_assert_held(&scheduler->lock); ++ ++ if (group->run_state == KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC) { ++ remove_group_from_idle_wait(group); ++ insert_group_to_runnable(scheduler, group, ++ KBASE_CSF_GROUP_SUSPENDED); ++ } else { ++ if (group->run_state == KBASE_CSF_GROUP_SUSPENDED_ON_IDLE) ++ group->run_state = KBASE_CSF_GROUP_SUSPENDED; ++ else ++ return; ++ } ++ ++ atomic_inc(&scheduler->non_idle_suspended_grps); ++} ++ ++int kbase_csf_scheduler_group_get_slot_locked(struct kbase_queue_group *group) ++{ ++ struct kbase_csf_scheduler *scheduler = ++ &group->kctx->kbdev->csf.scheduler; ++ int slot_num = group->csg_nr; ++ ++ lockdep_assert_held(&scheduler->interrupt_lock); ++ ++ if (slot_num >= 0) { ++ if (WARN_ON(scheduler->csg_slots[slot_num].resident_group != ++ group)) ++ return -1; ++ } ++ ++ return slot_num; ++} ++ ++int kbase_csf_scheduler_group_get_slot(struct kbase_queue_group *group) ++{ ++ struct kbase_csf_scheduler *scheduler = ++ &group->kctx->kbdev->csf.scheduler; ++ unsigned long flags; ++ int slot_num; ++ ++ spin_lock_irqsave(&scheduler->interrupt_lock, flags); ++ slot_num = kbase_csf_scheduler_group_get_slot_locked(group); ++ spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); ++ ++ return slot_num; ++} ++ ++static bool kbasep_csf_scheduler_group_is_on_slot_locked( ++ struct kbase_queue_group *group) ++{ ++ struct kbase_csf_scheduler *scheduler = ++ &group->kctx->kbdev->csf.scheduler; ++ int slot_num = group->csg_nr; ++ ++ lockdep_assert_held(&scheduler->lock); ++ ++ if (slot_num >= 0) { ++ if (!WARN_ON(scheduler->csg_slots[slot_num].resident_group != ++ group)) ++ return true; ++ } ++ ++ return false; ++} ++ ++bool kbase_csf_scheduler_group_events_enabled(struct kbase_device *kbdev, ++ struct kbase_queue_group *group) ++{ ++ struct kbase_csf_scheduler *scheduler = ++ &group->kctx->kbdev->csf.scheduler; ++ int slot_num = group->csg_nr; ++ ++ lockdep_assert_held(&scheduler->interrupt_lock); ++ ++ if (WARN_ON(slot_num < 0)) ++ return false; ++ ++ return test_bit(slot_num, scheduler->csgs_events_enable_mask); ++} ++ ++struct kbase_queue_group *kbase_csf_scheduler_get_group_on_slot( ++ struct kbase_device *kbdev, int slot) ++{ ++ lockdep_assert_held(&kbdev->csf.scheduler.interrupt_lock); ++ ++ return kbdev->csf.scheduler.csg_slots[slot].resident_group; ++} ++ ++static int halt_stream_sync(struct kbase_queue *queue) ++{ ++ struct kbase_queue_group *group = queue->group; ++ struct kbase_device *kbdev = queue->kctx->kbdev; ++ struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; ++ struct kbase_csf_cmd_stream_group_info *ginfo; ++ struct kbase_csf_cmd_stream_info *stream; ++ long remaining = ++ kbase_csf_timeout_in_jiffies(CSF_STATE_WAIT_TIMEOUT_MS); ++ ++ if (WARN_ON(!group) || ++ WARN_ON(!kbasep_csf_scheduler_group_is_on_slot_locked(group))) ++ return -EINVAL; ++ ++ lockdep_assert_held(&kbdev->csf.scheduler.lock); ++ ginfo = &global_iface->groups[group->csg_nr]; ++ stream = &ginfo->streams[queue->csi_index]; ++ ++ if (CS_REQ_STATE_GET(kbase_csf_firmware_cs_input_read(stream, CS_REQ)) == ++ CS_REQ_STATE_START) { ++ ++ remaining = wait_event_timeout(kbdev->csf.event_wait, ++ (CS_ACK_STATE_GET(kbase_csf_firmware_cs_output(stream, CS_ACK)) ++ == CS_ACK_STATE_START), remaining); ++ ++ if (!remaining) { ++ dev_warn(kbdev->dev, "Timed out waiting for queue to start on csi %d bound to group %d on slot %d", ++ queue->csi_index, group->handle, group->csg_nr); ++ if (kbase_prepare_to_reset_gpu(kbdev)) ++ kbase_reset_gpu(kbdev); ++ ++ return -ETIMEDOUT; ++ } ++ ++ remaining = ++ kbase_csf_timeout_in_jiffies(CSF_STATE_WAIT_TIMEOUT_MS); ++ } ++ ++ /* Set state to STOP */ ++ kbase_csf_firmware_cs_input_mask(stream, CS_REQ, CS_REQ_STATE_STOP, ++ CS_REQ_STATE_MASK); ++ ++ KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, CSI_STOP_REQUESTED, group, queue, 0u); ++ kbase_csf_ring_cs_kernel_doorbell(kbdev, queue); ++ ++ /* Timed wait */ ++ remaining = wait_event_timeout(kbdev->csf.event_wait, ++ (CS_ACK_STATE_GET(kbase_csf_firmware_cs_output(stream, CS_ACK)) ++ == CS_ACK_STATE_STOP), remaining); ++ ++ if (!remaining) { ++ dev_warn(kbdev->dev, "Timed out waiting for queue to stop on csi %d bound to group %d on slot %d", ++ queue->csi_index, group->handle, group->csg_nr); ++ if (kbase_prepare_to_reset_gpu(kbdev)) ++ kbase_reset_gpu(kbdev); ++ } ++ return (remaining) ? 0 : -ETIMEDOUT; ++} ++ ++static bool can_halt_stream(struct kbase_device *kbdev, ++ struct kbase_queue_group *group) ++{ ++ struct kbase_csf_csg_slot *const csg_slot = ++ kbdev->csf.scheduler.csg_slots; ++ unsigned long flags; ++ bool can_halt; ++ int slot; ++ ++ if (!queue_group_scheduled(group)) ++ return true; ++ ++ spin_lock_irqsave(&kbdev->csf.scheduler.interrupt_lock, flags); ++ slot = kbase_csf_scheduler_group_get_slot_locked(group); ++ can_halt = (slot >= 0) && ++ (atomic_read(&csg_slot[slot].state) == CSG_SLOT_RUNNING); ++ spin_unlock_irqrestore(&kbdev->csf.scheduler.interrupt_lock, ++ flags); ++ ++ return can_halt; ++} ++ ++/** ++ * sched_halt_stream() - Stop a GPU queue when its queue group is not running ++ * on a CSG slot. ++ * @queue: Pointer to the GPU queue to stop. ++ * ++ * This function handles stopping gpu queues for groups that are either not on ++ * a command stream group slot or are on the slot but undergoing transition to ++ * resume or suspend states. ++ * It waits until the queue group is scheduled on a slot and starts running, ++ * which is needed as groups that were suspended may need to resume all queues ++ * that were enabled and running at the time of suspension. ++ * ++ * Return: 0 on success, or negative on failure. ++ */ ++static int sched_halt_stream(struct kbase_queue *queue) ++{ ++ struct kbase_queue_group *group = queue->group; ++ struct kbase_device *kbdev = queue->kctx->kbdev; ++ struct kbase_csf_scheduler *const scheduler = ++ &kbdev->csf.scheduler; ++ struct kbase_csf_csg_slot *const csg_slot = ++ kbdev->csf.scheduler.csg_slots; ++ bool retry_needed = false; ++ bool retried = false; ++ long remaining; ++ int slot; ++ int err = 0; ++ ++ if (WARN_ON(!group)) ++ return -EINVAL; ++ ++ lockdep_assert_held(&queue->kctx->csf.lock); ++ lockdep_assert_held(&scheduler->lock); ++ ++ slot = kbase_csf_scheduler_group_get_slot(group); ++ ++ if (slot >= 0) { ++ WARN_ON(atomic_read(&csg_slot[slot].state) == CSG_SLOT_RUNNING); ++ ++ if (atomic_read(&csg_slot[slot].state) == CSG_SLOT_READY2RUN) { ++ dev_dbg(kbdev->dev, "Stopping a queue on csi %d when Group-%d is in under transition to running state", ++ queue->csi_index, group->handle); ++ retry_needed = true; ++ } ++ } ++retry: ++ /* First wait for the group to reach a stable state. IDLE state is ++ * an intermediate state that is only set by Scheduler at the start ++ * of a tick (prior to scanout) for groups that received idle ++ * notification, then later the idle group is moved to one of the ++ * suspended states or the runnable state. ++ */ ++ while (group->run_state == KBASE_CSF_GROUP_IDLE) { ++ mutex_unlock(&scheduler->lock); ++ remaining = wait_event_timeout(kbdev->csf.event_wait, ++ group->run_state != KBASE_CSF_GROUP_IDLE, ++ CSF_STATE_WAIT_TIMEOUT_MS); ++ mutex_lock(&scheduler->lock); ++ if (!remaining) { ++ dev_warn(kbdev->dev, ++ "Timed out waiting for state change of Group-%d when stopping a queue on csi %d", ++ group->handle, queue->csi_index); ++ } ++ } ++ ++ WARN_ON(group->run_state == KBASE_CSF_GROUP_IDLE); ++ /* Update the group state so that it can get scheduled soon */ ++ update_idle_suspended_group_state(group); ++ ++ mutex_unlock(&scheduler->lock); ++ ++ /* This function is called when the queue group is either not on a CSG ++ * slot or is on the slot but undergoing transition. ++ * ++ * To stop the queue, the function needs to wait either for the queue ++ * group to be assigned a CSG slot (and that slot has to reach the ++ * running state) or for the eviction of the queue group from the ++ * scheduler's list. ++ * ++ * In order to evaluate the latter condition, the function doesn't ++ * really need to lock the scheduler, as any update to the run_state ++ * of the queue group by sched_evict_group() would be visible due ++ * to implicit barriers provided by the kernel waitqueue macros. ++ * ++ * The group pointer cannot disappear meanwhile, as the high level ++ * CSF context is locked. Therefore, the scheduler would be ++ * the only one to update the run_state of the group. ++ */ ++ remaining = wait_event_timeout(kbdev->csf.event_wait, ++ can_halt_stream(kbdev, group), ++ kbase_csf_timeout_in_jiffies(20 * CSF_SCHEDULER_TIME_TICK_MS)); ++ ++ mutex_lock(&scheduler->lock); ++ ++ if (remaining && queue_group_scheduled_locked(group)) { ++ slot = kbase_csf_scheduler_group_get_slot(group); ++ ++ /* If the group is still on slot and slot is in running state ++ * then explicitly stop the command stream interface of the ++ * queue. Otherwise there are different cases to consider ++ * ++ * - If the queue group was already undergoing transition to ++ * resume/start state when this function was entered then it ++ * would not have disabled the command stream interface of the ++ * queue being stopped and the previous wait would have ended ++ * once the slot was in a running state with command stream ++ * interface still enabled. ++ * Now the group is going through another transition either ++ * to a suspend state or to a resume state (it could have ++ * been suspended before the scheduler lock was grabbed). ++ * In both scenarios need to wait again for the group to ++ * come on a slot and that slot to reach the running state, ++ * as that would guarantee that firmware will observe the ++ * command stream interface as disabled. ++ * ++ * - If the queue group was either off the slot or was ++ * undergoing transition to suspend state on entering this ++ * function, then the group would have been resumed with the ++ * queue's command stream interface in disabled state. ++ * So now if the group is undergoing another transition ++ * (after the resume) then just need to wait for the state ++ * bits in the ACK register of command stream interface to be ++ * set to STOP value. It is expected that firmware will ++ * process the stop/disable request of the command stream ++ * interface after resuming the group before it processes ++ * another state change request of the group. ++ */ ++ if ((slot >= 0) && ++ (atomic_read(&csg_slot[slot].state) == CSG_SLOT_RUNNING)) { ++ err = halt_stream_sync(queue); ++ } else if (retry_needed && !retried) { ++ retried = true; ++ goto retry; ++ } else if (slot >= 0) { ++ struct kbase_csf_global_iface *global_iface = ++ &kbdev->csf.global_iface; ++ struct kbase_csf_cmd_stream_group_info *ginfo = ++ &global_iface->groups[slot]; ++ struct kbase_csf_cmd_stream_info *stream = ++ &ginfo->streams[queue->csi_index]; ++ u32 cs_req = ++ kbase_csf_firmware_cs_input_read(stream, CS_REQ); ++ ++ if (!WARN_ON(CS_REQ_STATE_GET(cs_req) != ++ CS_REQ_STATE_STOP)) { ++ /* Timed wait */ ++ remaining = wait_event_timeout( ++ kbdev->csf.event_wait, ++ (CS_ACK_STATE_GET(kbase_csf_firmware_cs_output(stream, CS_ACK)) ++ == CS_ACK_STATE_STOP), ++ CSF_STATE_WAIT_TIMEOUT_MS); ++ ++ if (!remaining) { ++ dev_warn(kbdev->dev, ++ "Timed out waiting for queue stop ack on csi %d bound to group %d on slot %d", ++ queue->csi_index, ++ group->handle, group->csg_nr); ++ err = -ETIMEDOUT; ++ } ++ } ++ } ++ } else if (!remaining) { ++ dev_warn(kbdev->dev, "Group-%d failed to get a slot for stopping the queue on csi %d", ++ group->handle, queue->csi_index); ++ err = -ETIMEDOUT; ++ } ++ ++ return err; ++} ++ ++static int wait_gpu_reset(struct kbase_device *kbdev) ++{ ++ int ret = 0; ++ ++ lockdep_assert_held(&kbdev->csf.scheduler.lock); ++ ++ while (kbase_reset_gpu_is_active(kbdev) && !ret) { ++ mutex_unlock(&kbdev->csf.scheduler.lock); ++ ret = kbase_reset_gpu_wait(kbdev); ++ mutex_lock(&kbdev->csf.scheduler.lock); ++ } ++ ++ return ret; ++} ++ ++int kbase_csf_scheduler_queue_stop(struct kbase_queue *queue) ++{ ++ struct kbase_device *kbdev = queue->kctx->kbdev; ++ struct kbase_queue_group *group = queue->group; ++ bool const cs_enabled = queue->enabled; ++ int err = 0; ++ ++ if (WARN_ON(!group)) ++ return -EINVAL; ++ ++ lockdep_assert_held(&queue->kctx->csf.lock); ++ mutex_lock(&kbdev->csf.scheduler.lock); ++ ++ queue->enabled = false; ++ KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, CSI_STOP, group, queue, cs_enabled); ++ ++ wait_gpu_reset(kbdev); ++ ++ if (cs_enabled && queue_group_scheduled_locked(group)) { ++ struct kbase_csf_csg_slot *const csg_slot = ++ kbdev->csf.scheduler.csg_slots; ++ int slot = kbase_csf_scheduler_group_get_slot(group); ++ ++ /* Since the group needs to be resumed in order to stop the queue, ++ * check if GPU needs to be powered up. ++ */ ++ scheduler_wakeup(kbdev, true); ++ ++ if ((slot >= 0) && ++ (atomic_read(&csg_slot[slot].state) == CSG_SLOT_RUNNING)) ++ err = halt_stream_sync(queue); ++ else ++ err = sched_halt_stream(queue); ++ ++ unassign_user_doorbell_from_queue(kbdev, queue); ++ } ++ ++ mutex_unlock(&kbdev->csf.scheduler.lock); ++ return err; ++} ++ ++static void update_hw_active(struct kbase_queue *queue, bool active) ++{ ++#ifdef CONFIG_MALI_BIFROST_NO_MALI ++ if (queue && queue->enabled) { ++ u32 *output_addr = (u32 *)(queue->user_io_addr + PAGE_SIZE); ++ ++ output_addr[CS_ACTIVE / sizeof(u32)] = active; ++ } ++#else ++ CSTD_UNUSED(queue); ++ CSTD_UNUSED(active); ++#endif ++} ++ ++static void program_cs_extract_init(struct kbase_queue *queue) ++{ ++ u64 *input_addr = (u64 *)queue->user_io_addr; ++ u64 *output_addr = (u64 *)(queue->user_io_addr + PAGE_SIZE); ++ ++ input_addr[CS_EXTRACT_INIT_LO / sizeof(u64)] = ++ output_addr[CS_EXTRACT_LO / sizeof(u64)]; ++} ++ ++static void program_cs(struct kbase_device *kbdev, ++ struct kbase_queue *queue) ++{ ++ struct kbase_queue_group *group = queue->group; ++ struct kbase_csf_cmd_stream_group_info *ginfo; ++ struct kbase_csf_cmd_stream_info *stream; ++ u64 user_input; ++ u64 user_output; ++ ++ if (WARN_ON(!group)) ++ return; ++ ++ lockdep_assert_held(&kbdev->csf.scheduler.lock); ++ ++ if (WARN_ON(!kbasep_csf_scheduler_group_is_on_slot_locked(group))) ++ return; ++ ++ ginfo = &kbdev->csf.global_iface.groups[group->csg_nr]; ++ ++ if (WARN_ON(queue->csi_index < 0) || ++ WARN_ON(queue->csi_index >= ginfo->stream_num)) ++ return; ++ ++ assign_user_doorbell_to_queue(kbdev, queue); ++ if (queue->doorbell_nr == KBASEP_USER_DB_NR_INVALID) ++ return; ++ ++ WARN_ON(queue->doorbell_nr != queue->group->doorbell_nr); ++ ++ if (queue->enabled && queue_group_suspended_locked(group)) ++ program_cs_extract_init(queue); ++ ++ stream = &ginfo->streams[queue->csi_index]; ++ ++ kbase_csf_firmware_cs_input(stream, CS_BASE_LO, ++ queue->base_addr & 0xFFFFFFFF); ++ kbase_csf_firmware_cs_input(stream, CS_BASE_HI, ++ queue->base_addr >> 32); ++ kbase_csf_firmware_cs_input(stream, CS_SIZE, ++ queue->size); ++ ++ user_input = (queue->reg->start_pfn << PAGE_SHIFT); ++ kbase_csf_firmware_cs_input(stream, CS_USER_INPUT_LO, ++ user_input & 0xFFFFFFFF); ++ kbase_csf_firmware_cs_input(stream, CS_USER_INPUT_HI, ++ user_input >> 32); ++ ++ user_output = ((queue->reg->start_pfn + 1) << PAGE_SHIFT); ++ kbase_csf_firmware_cs_input(stream, CS_USER_OUTPUT_LO, ++ user_output & 0xFFFFFFFF); ++ kbase_csf_firmware_cs_input(stream, CS_USER_OUTPUT_HI, ++ user_output >> 32); ++ ++ kbase_csf_firmware_cs_input(stream, CS_CONFIG, ++ (queue->doorbell_nr << 8) | (queue->priority & 0xF)); ++ ++ /* Enable all interrupts for now */ ++ kbase_csf_firmware_cs_input(stream, CS_ACK_IRQ_MASK, ~((u32)0)); ++ ++ /* ++ * Enable the CSG idle notification once the stream's ringbuffer ++ * becomes empty or the stream becomes sync_idle, waiting sync update ++ * or protected mode switch. ++ */ ++ kbase_csf_firmware_cs_input_mask(stream, CS_REQ, ++ CS_REQ_IDLE_EMPTY_MASK | CS_REQ_IDLE_SYNC_WAIT_MASK, ++ CS_REQ_IDLE_EMPTY_MASK | CS_REQ_IDLE_SYNC_WAIT_MASK); ++ ++ /* Set state to START/STOP */ ++ kbase_csf_firmware_cs_input_mask(stream, CS_REQ, ++ queue->enabled ? CS_REQ_STATE_START : CS_REQ_STATE_STOP, ++ CS_REQ_STATE_MASK); ++ ++ KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, CSI_START, group, queue, queue->enabled); ++ ++ kbase_csf_ring_cs_kernel_doorbell(kbdev, queue); ++ update_hw_active(queue, true); ++} ++ ++int kbase_csf_scheduler_queue_start(struct kbase_queue *queue) ++{ ++ struct kbase_queue_group *group = queue->group; ++ struct kbase_device *kbdev = queue->kctx->kbdev; ++ bool const cs_enabled = queue->enabled; ++ int err = 0; ++ bool evicted = false; ++ ++ lockdep_assert_held(&queue->kctx->csf.lock); ++ ++ if (WARN_ON(!group || queue->bind_state != KBASE_CSF_QUEUE_BOUND)) ++ return -EINVAL; ++ ++ mutex_lock(&kbdev->csf.scheduler.lock); ++ ++ KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, QUEUE_START, group, queue, group->run_state); ++ err = wait_gpu_reset(kbdev); ++ ++ if (err) { ++ dev_warn(kbdev->dev, "Unsuccessful GPU reset detected when kicking queue (csi_index=%d) of group %d", ++ queue->csi_index, group->handle); ++ } else if (group->run_state == KBASE_CSF_GROUP_FAULT_EVICTED) { ++ err = -EIO; ++ evicted = true; ++ } else if ((group->run_state == KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC) ++ && CS_STATUS_WAIT_SYNC_WAIT_GET(queue->status_wait)) { ++ dev_dbg(kbdev->dev, "blocked queue(csi_index=%d) of group %d was kicked", ++ queue->csi_index, group->handle); ++ } else { ++ err = scheduler_group_schedule(group); ++ ++ if (!err) { ++ queue->enabled = true; ++ if (kbasep_csf_scheduler_group_is_on_slot_locked(group)) { ++ if (cs_enabled) { ++ /* In normal situation, when a queue is ++ * already running, the queue update ++ * would be a doorbell kick on user ++ * side. However, if such a kick is ++ * shortly following a start or resume, ++ * the queue may actually in transition ++ * hence the said kick would enter the ++ * kernel as the hw_active flag is yet ++ * to be set. The sheduler needs to ++ * give a kick to the corresponding ++ * user door-bell on such a case. ++ */ ++ kbase_csf_ring_cs_user_doorbell(kbdev, queue); ++ } else ++ program_cs(kbdev, queue); ++ } ++ queue_delayed_work(system_long_wq, ++ &kbdev->csf.scheduler.ping_work, ++ msecs_to_jiffies(FIRMWARE_PING_INTERVAL_MS)); ++ } ++ } ++ ++ mutex_unlock(&kbdev->csf.scheduler.lock); ++ ++ if (evicted) ++ kbase_csf_term_descheduled_queue_group(group); ++ ++ return err; ++} ++ ++static enum kbase_csf_csg_slot_state update_csg_slot_status( ++ struct kbase_device *kbdev, s8 slot) ++{ ++ struct kbase_csf_csg_slot *csg_slot = ++ &kbdev->csf.scheduler.csg_slots[slot]; ++ struct kbase_csf_cmd_stream_group_info *ginfo = ++ &kbdev->csf.global_iface.groups[slot]; ++ u32 state; ++ enum kbase_csf_csg_slot_state slot_state; ++ ++ lockdep_assert_held(&kbdev->csf.scheduler.lock); ++ ++ state = CSG_ACK_STATE_GET(kbase_csf_firmware_csg_output(ginfo, ++ CSG_ACK)); ++ slot_state = atomic_read(&csg_slot->state); ++ ++ switch (slot_state) { ++ case CSG_SLOT_READY2RUN: ++ if ((state == CSG_ACK_STATE_START) || ++ (state == CSG_ACK_STATE_RESUME)) { ++ slot_state = CSG_SLOT_RUNNING; ++ atomic_set(&csg_slot->state, slot_state); ++ csg_slot->trigger_jiffies = jiffies; ++ KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_SLOT_STARTED, csg_slot->resident_group, state); ++ dev_dbg(kbdev->dev, "Group %u running on slot %d\n", ++ csg_slot->resident_group->handle, slot); ++ } ++ break; ++ case CSG_SLOT_DOWN2STOP: ++ if ((state == CSG_ACK_STATE_SUSPEND) || ++ (state == CSG_ACK_STATE_TERMINATE)) { ++ slot_state = CSG_SLOT_STOPPED; ++ atomic_set(&csg_slot->state, slot_state); ++ csg_slot->trigger_jiffies = jiffies; ++ KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_SLOT_STOPPED, csg_slot->resident_group, state); ++ dev_dbg(kbdev->dev, "Group %u stopped on slot %d\n", ++ csg_slot->resident_group->handle, slot); ++ } ++ break; ++ case CSG_SLOT_DOWN2STOP_TIMEDOUT: ++ case CSG_SLOT_READY2RUN_TIMEDOUT: ++ case CSG_SLOT_READY: ++ case CSG_SLOT_RUNNING: ++ case CSG_SLOT_STOPPED: ++ break; ++ default: ++ dev_warn(kbdev->dev, "Unknown CSG slot state %d", slot_state); ++ break; ++ } ++ ++ return slot_state; ++} ++ ++static bool csg_slot_running(struct kbase_device *kbdev, s8 slot) ++{ ++ lockdep_assert_held(&kbdev->csf.scheduler.lock); ++ ++ return (update_csg_slot_status(kbdev, slot) == CSG_SLOT_RUNNING); ++} ++ ++static bool csg_slot_stopped_locked(struct kbase_device *kbdev, s8 slot) ++{ ++ enum kbase_csf_csg_slot_state slot_state; ++ ++ lockdep_assert_held(&kbdev->csf.scheduler.lock); ++ ++ slot_state = update_csg_slot_status(kbdev, slot); ++ ++ return (slot_state == CSG_SLOT_STOPPED || ++ slot_state == CSG_SLOT_READY); ++} ++ ++static bool csg_slot_stopped_raw(struct kbase_device *kbdev, s8 slot) ++{ ++ struct kbase_csf_cmd_stream_group_info *ginfo = ++ &kbdev->csf.global_iface.groups[slot]; ++ u32 state; ++ ++ state = CSG_ACK_STATE_GET(kbase_csf_firmware_csg_output(ginfo, ++ CSG_ACK)); ++ ++ if (state == CSG_ACK_STATE_SUSPEND || state == CSG_ACK_STATE_TERMINATE) { ++ KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_SLOT_STOPPED, kbdev->csf.scheduler.csg_slots[slot].resident_group, state); ++ dev_dbg(kbdev->dev, "(raw status) slot %d stopped\n", slot); ++ return true; ++ } ++ ++ return false; ++} ++ ++static void halt_csg_slot(struct kbase_queue_group *group, bool suspend) ++{ ++ struct kbase_device *kbdev = group->kctx->kbdev; ++ struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; ++ struct kbase_csf_csg_slot *csg_slot = ++ kbdev->csf.scheduler.csg_slots; ++ s8 slot; ++ ++ lockdep_assert_held(&kbdev->csf.scheduler.lock); ++ ++ if (WARN_ON(!kbasep_csf_scheduler_group_is_on_slot_locked(group))) ++ return; ++ ++ slot = group->csg_nr; ++ ++ /* When in transition, wait for it to complete */ ++ if (atomic_read(&csg_slot[slot].state) == CSG_SLOT_READY2RUN) { ++ long remaining = ++ kbase_csf_timeout_in_jiffies(CSF_STATE_WAIT_TIMEOUT_MS); ++ ++ dev_dbg(kbdev->dev, "slot %d wait for up-running\n", slot); ++ remaining = wait_event_timeout(kbdev->csf.event_wait, ++ csg_slot_running(kbdev, slot), remaining); ++ if (!remaining) ++ dev_warn(kbdev->dev, ++ "slot %d timed out on up-running\n", slot); ++ } ++ ++ if (csg_slot_running(kbdev, slot)) { ++ unsigned long flags; ++ struct kbase_csf_cmd_stream_group_info *ginfo = ++ &global_iface->groups[slot]; ++ u32 halt_cmd = suspend ? CSG_REQ_STATE_SUSPEND : ++ CSG_REQ_STATE_TERMINATE; ++ ++ dev_dbg(kbdev->dev, "Halting(suspend=%d) group %d of context %d_%d on slot %d", ++ suspend, group->handle, group->kctx->tgid, group->kctx->id, slot); ++ ++ spin_lock_irqsave(&kbdev->csf.scheduler.interrupt_lock, flags); ++ /* Set state to SUSPEND/TERMINATE */ ++ kbase_csf_firmware_csg_input_mask(ginfo, CSG_REQ, halt_cmd, ++ CSG_REQ_STATE_MASK); ++ spin_unlock_irqrestore(&kbdev->csf.scheduler.interrupt_lock, ++ flags); ++ atomic_set(&csg_slot[slot].state, CSG_SLOT_DOWN2STOP); ++ csg_slot[slot].trigger_jiffies = jiffies; ++ KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_SLOT_STOP, group, halt_cmd); ++ ++ kbase_csf_ring_csg_doorbell(kbdev, slot); ++ } ++} ++ ++static void term_csg_slot(struct kbase_queue_group *group) ++{ ++ halt_csg_slot(group, false); ++} ++ ++static void suspend_csg_slot(struct kbase_queue_group *group) ++{ ++ halt_csg_slot(group, true); ++} ++ ++/** ++ * evaluate_sync_update() - Evaluate the sync wait condition the GPU command ++ * queue has been blocked on. ++ * ++ * @queue: Pointer to the GPU command queue ++ * ++ * Return: true if sync wait condition is satisfied. ++ */ ++static bool evaluate_sync_update(struct kbase_queue *queue) ++{ ++ enum kbase_csf_group_state run_state; ++ struct kbase_vmap_struct *mapping; ++ bool updated = false; ++ u32 *sync_ptr; ++ u32 sync_wait_cond; ++ ++ if (WARN_ON(!queue)) ++ return false; ++ ++ run_state = queue->group->run_state; ++ ++ if (WARN_ON((run_state != KBASE_CSF_GROUP_IDLE) && ++ (run_state != KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC))) ++ return false; ++ ++ lockdep_assert_held(&queue->kctx->kbdev->csf.scheduler.lock); ++ ++ sync_ptr = kbase_phy_alloc_mapping_get(queue->kctx, queue->sync_ptr, ++ &mapping); ++ ++ if (!sync_ptr) { ++ dev_dbg(queue->kctx->kbdev->dev, "sync memory VA 0x%016llX already freed", ++ queue->sync_ptr); ++ return false; ++ } ++ ++ sync_wait_cond = ++ CS_STATUS_WAIT_SYNC_WAIT_CONDITION_GET(queue->status_wait); ++ ++ WARN_ON((sync_wait_cond != CS_STATUS_WAIT_SYNC_WAIT_CONDITION_GT) && ++ (sync_wait_cond != CS_STATUS_WAIT_SYNC_WAIT_CONDITION_LE)); ++ ++ if (((sync_wait_cond == CS_STATUS_WAIT_SYNC_WAIT_CONDITION_GT) && ++ (*sync_ptr > queue->sync_value)) || ++ ((sync_wait_cond == CS_STATUS_WAIT_SYNC_WAIT_CONDITION_LE) && ++ (*sync_ptr <= queue->sync_value))) { ++ /* The sync wait condition is satisfied so the group to which ++ * queue is bound can be re-scheduled. ++ */ ++ updated = true; ++ } else { ++ dev_dbg(queue->kctx->kbdev->dev, "sync memory not updated yet(%u)", ++ *sync_ptr); ++ } ++ ++ kbase_phy_alloc_mapping_put(queue->kctx, mapping); ++ ++ return updated; ++} ++ ++/** ++ * save_slot_cs() - Save the state for blocked GPU command queue. ++ * ++ * @ginfo: Pointer to the command stream group interface used by the group ++ * the queue is bound to. ++ * @queue: Pointer to the GPU command queue. ++ * ++ * This function will check if GPU command queue is blocked on a sync wait and ++ * evaluate the wait condition. If the wait condition isn't satisfied it would ++ * save the state needed to reevaluate the condition in future. ++ * The group to which queue is bound shall be in idle state. ++ * ++ * Return: true if the queue is blocked on a sync wait operation. ++ */ ++static ++bool save_slot_cs(struct kbase_csf_cmd_stream_group_info const *const ginfo, ++ struct kbase_queue *queue) ++{ ++ struct kbase_csf_cmd_stream_info *const stream = ++ &ginfo->streams[queue->csi_index]; ++ u32 status = kbase_csf_firmware_cs_output(stream, CS_STATUS_WAIT); ++ bool is_waiting = false; ++ ++ WARN_ON(queue->group->run_state != KBASE_CSF_GROUP_IDLE); ++ ++ if (CS_STATUS_WAIT_SYNC_WAIT_GET(status)) { ++ queue->status_wait = status; ++ queue->sync_ptr = kbase_csf_firmware_cs_output(stream, ++ CS_STATUS_WAIT_SYNC_POINTER_LO); ++ queue->sync_ptr |= (u64)kbase_csf_firmware_cs_output(stream, ++ CS_STATUS_WAIT_SYNC_POINTER_HI) << 32; ++ queue->sync_value = kbase_csf_firmware_cs_output(stream, ++ CS_STATUS_WAIT_SYNC_VALUE); ++ ++ if (!evaluate_sync_update(queue)) { ++ is_waiting = true; ++ } else { ++ /* Sync object already got updated & met the condition ++ * thus it doesn't need to be reevaluated and so can ++ * clear the 'status_wait' here. ++ */ ++ queue->status_wait = 0; ++ } ++ } else { ++ /* Invalidate wait status info that would have been recorded if ++ * this queue was blocked when the group (in idle state) was ++ * suspended previously. After that the group could have been ++ * unblocked due to the kicking of another queue bound to it & ++ * so the wait status info would have stuck with this queue. ++ */ ++ queue->status_wait = 0; ++ } ++ ++ return is_waiting; ++} ++ ++/** ++ * Calculate how far in the future an event should be scheduled. ++ * ++ * The objective of this function is making sure that a minimum period of ++ * time is guaranteed between handling two consecutive events. ++ * ++ * This function guarantees a minimum period of time between two consecutive ++ * events: given the minimum period and the distance between the current time ++ * and the last event, the function returns the difference between the two. ++ * However, if more time than the minimum period has already elapsed ++ * since the last event, the function will return 0 to schedule work to handle ++ * the event with the lowest latency possible. ++ * ++ * @last_event: Timestamp of the last event, in jiffies. ++ * @time_now: Timestamp of the new event to handle, in jiffies. ++ * Must be successive to last_event. ++ * @period: Minimum period between two events, in jiffies. ++ * ++ * Return: Time to delay work to handle the current event, in jiffies ++ */ ++static unsigned long get_schedule_delay(unsigned long last_event, ++ unsigned long time_now, ++ unsigned long period) ++{ ++ const unsigned long t_distance = time_now - last_event; ++ const unsigned long delay_t = (t_distance < period) ? ++ (period - t_distance) : 0; ++ ++ return delay_t; ++} ++ ++static void schedule_in_cycle(struct kbase_queue_group *group, bool force) ++{ ++ struct kbase_context *kctx = group->kctx; ++ struct kbase_device *kbdev = kctx->kbdev; ++ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; ++ ++ lockdep_assert_held(&scheduler->lock); ++ ++ /* Only try to schedule work for this event if no requests are pending, ++ * otherwise the function will end up canceling previous work requests, ++ * and scheduler is configured to wake up periodically (or the schedule ++ * of work needs to be enforced in situation such as entering into ++ * protected mode). ++ */ ++ if ((likely(scheduler_timer_is_enabled_nolock(kbdev)) || force) && ++ !scheduler->tock_pending_request) { ++ const unsigned long delay = ++ get_schedule_delay(scheduler->last_schedule, jiffies, ++ CSF_SCHEDULER_TIME_TOCK_JIFFIES); ++ scheduler->tock_pending_request = true; ++ dev_dbg(kbdev->dev, "Kicking async for group %d\n", ++ group->handle); ++ mod_delayed_work(scheduler->wq, &scheduler->tock_work, delay); ++ } ++} ++ ++static ++void insert_group_to_runnable(struct kbase_csf_scheduler *const scheduler, ++ struct kbase_queue_group *const group, ++ enum kbase_csf_group_state run_state) ++{ ++ struct kbase_context *const kctx = group->kctx; ++ struct kbase_device *const kbdev = kctx->kbdev; ++ ++ lockdep_assert_held(&scheduler->lock); ++ ++ WARN_ON(group->run_state != KBASE_CSF_GROUP_INACTIVE); ++ ++ if (WARN_ON(group->priority >= BASE_QUEUE_GROUP_PRIORITY_COUNT)) ++ return; ++ ++ group->run_state = run_state; ++ ++ if (run_state == KBASE_CSF_GROUP_RUNNABLE) ++ group->prepared_seq_num = KBASEP_GROUP_PREPARED_SEQ_NUM_INVALID; ++ ++ list_add_tail(&group->link, ++ &kctx->csf.sched.runnable_groups[group->priority]); ++ kctx->csf.sched.num_runnable_grps++; ++ /* Add the kctx if not yet in runnable kctxs */ ++ if (kctx->csf.sched.num_runnable_grps == 1) { ++ /* First runnable csg, adds to the runnable_kctxs */ ++ INIT_LIST_HEAD(&kctx->csf.link); ++ list_add_tail(&kctx->csf.link, &scheduler->runnable_kctxs); ++ } ++ ++ scheduler->total_runnable_grps++; ++ ++ if (likely(scheduler_timer_is_enabled_nolock(kbdev)) && ++ (scheduler->total_runnable_grps == 1 || ++ scheduler->state == SCHED_SUSPENDED)) { ++ dev_dbg(kbdev->dev, "Kicking scheduler on first runnable group\n"); ++ /* Fire a scheduling to start the time-slice */ ++ mod_delayed_work(kbdev->csf.scheduler.wq, ++ &kbdev->csf.scheduler.tick_work, 0); ++ } else ++ schedule_in_cycle(group, false); ++ ++ /* Since a new group has become runnable, check if GPU needs to be ++ * powered up. ++ */ ++ scheduler_wakeup(kbdev, false); ++} ++ ++static ++void remove_group_from_runnable(struct kbase_csf_scheduler *const scheduler, ++ struct kbase_queue_group *group, ++ enum kbase_csf_group_state run_state) ++{ ++ struct kbase_context *kctx = group->kctx; ++ ++ lockdep_assert_held(&scheduler->lock); ++ ++ WARN_ON(!queue_group_scheduled_locked(group)); ++ ++ group->run_state = run_state; ++ list_del_init(&group->link); ++ ++ if (scheduler->top_grp == group) { ++ /* ++ * Note: this disables explicit rotation in the next scheduling ++ * cycle. However, removing the top_grp is the same as an ++ * implicit rotation (e.g. if we instead rotated the top_ctx ++ * and then remove top_grp) ++ * ++ * This implicit rotation is assumed by the scheduler rotate ++ * functions. ++ */ ++ scheduler->top_grp = NULL; ++ ++ /* ++ * Trigger a scheduling tock for a CSG containing protected ++ * content in case there has been any in order to minimise ++ * latency. ++ */ ++ group = scheduler_get_protm_enter_async_group(kctx->kbdev, ++ NULL); ++ if (group) ++ schedule_in_cycle(group, true); ++ } ++ ++ kctx->csf.sched.num_runnable_grps--; ++ if (kctx->csf.sched.num_runnable_grps == 0) { ++ /* drop the kctx */ ++ list_del_init(&kctx->csf.link); ++ if (scheduler->top_ctx == kctx) ++ scheduler->top_ctx = NULL; ++ } ++ ++ WARN_ON(scheduler->total_runnable_grps == 0); ++ scheduler->total_runnable_grps--; ++ if (!scheduler->total_runnable_grps && ++ scheduler->state != SCHED_SUSPENDED) { ++ dev_dbg(kctx->kbdev->dev, "Scheduler idle as no runnable groups"); ++ mod_delayed_work(system_wq, &scheduler->gpu_idle_work, ++ GPU_IDLE_POWEROFF_HYSTERESIS_DELAY); ++ } ++ KBASE_KTRACE_ADD_CSF_GRP(kctx->kbdev, SCHEDULER_TOP_GRP, scheduler->top_grp, ++ scheduler->num_active_address_spaces | ++ (((u64)scheduler->total_runnable_grps) << 32)); ++} ++ ++static void insert_group_to_idle_wait(struct kbase_queue_group *const group) ++{ ++ struct kbase_context *kctx = group->kctx; ++ ++ lockdep_assert_held(&kctx->kbdev->csf.scheduler.lock); ++ ++ WARN_ON(group->run_state != KBASE_CSF_GROUP_IDLE); ++ ++ list_add_tail(&group->link, &kctx->csf.sched.idle_wait_groups); ++ kctx->csf.sched.num_idle_wait_grps++; ++ group->run_state = KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC; ++ dev_dbg(kctx->kbdev->dev, ++ "Group-%d suspended on sync_wait, total wait_groups: %u\n", ++ group->handle, kctx->csf.sched.num_idle_wait_grps); ++} ++ ++static void remove_group_from_idle_wait(struct kbase_queue_group *const group) ++{ ++ struct kbase_context *kctx = group->kctx; ++ ++ lockdep_assert_held(&kctx->kbdev->csf.scheduler.lock); ++ ++ WARN_ON(group->run_state != KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC); ++ ++ list_del_init(&group->link); ++ WARN_ON(kctx->csf.sched.num_idle_wait_grps == 0); ++ kctx->csf.sched.num_idle_wait_grps--; ++ group->run_state = KBASE_CSF_GROUP_INACTIVE; ++} ++ ++static void deschedule_idle_wait_group(struct kbase_csf_scheduler *scheduler, ++ struct kbase_queue_group *group) ++{ ++ lockdep_assert_held(&scheduler->lock); ++ ++ if (WARN_ON(!group)) ++ return; ++ ++ remove_group_from_runnable(scheduler, group, KBASE_CSF_GROUP_IDLE); ++ insert_group_to_idle_wait(group); ++} ++ ++static bool confirm_cs_idle(struct kbase_queue *queue) ++{ ++ u64 *input_addr = (u64 *)queue->user_io_addr; ++ u64 *output_addr = (u64 *)(queue->user_io_addr + PAGE_SIZE); ++ ++ return (input_addr[CS_INSERT_LO / sizeof(u64)] == ++ output_addr[CS_EXTRACT_LO / sizeof(u64)]); ++} ++ ++static void save_csg_slot(struct kbase_queue_group *group) ++{ ++ struct kbase_device *kbdev = group->kctx->kbdev; ++ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; ++ struct kbase_csf_cmd_stream_group_info *ginfo; ++ u32 state; ++ ++ lockdep_assert_held(&scheduler->lock); ++ ++ if (WARN_ON(!kbasep_csf_scheduler_group_is_on_slot_locked(group))) ++ return; ++ ++ ginfo = &kbdev->csf.global_iface.groups[group->csg_nr]; ++ ++ state = ++ CSG_ACK_STATE_GET(kbase_csf_firmware_csg_output(ginfo, CSG_ACK)); ++ ++ if (!WARN_ON((state != CSG_ACK_STATE_SUSPEND) && ++ (state != CSG_ACK_STATE_TERMINATE))) { ++ int i; ++ ++#ifdef CONFIG_MALI_BIFROST_NO_MALI ++ for (i = 0; i < MAX_SUPPORTED_STREAMS_PER_GROUP; i++) ++ update_hw_active(group->bound_queues[i], false); ++#endif ++ if (group->run_state == KBASE_CSF_GROUP_IDLE) { ++ bool sync_wait = false; ++ bool idle = true; ++ ++ /* Loop through all bound CSs & save their context */ ++ for (i = 0; i < MAX_SUPPORTED_STREAMS_PER_GROUP; i++) { ++ struct kbase_queue *const queue = ++ group->bound_queues[i]; ++ ++ if (queue && queue->enabled) { ++ if (save_slot_cs(ginfo, queue)) ++ sync_wait = true; ++ else if (idle) ++ idle = confirm_cs_idle(queue); ++ } ++ } ++ ++ /* Take the suspended group out of the runnable_groups ++ * list of the context and move it to the ++ * idle_wait_groups list. ++ */ ++ if (sync_wait && idle) ++ deschedule_idle_wait_group(scheduler, group); ++ else if (idle) { ++ group->run_state = ++ KBASE_CSF_GROUP_SUSPENDED_ON_IDLE; ++ dev_dbg(kbdev->dev, "Group-%d suspended: idle\n", ++ group->handle); ++ } else { ++ group->run_state = KBASE_CSF_GROUP_SUSPENDED; ++ atomic_inc(&scheduler->non_idle_suspended_grps); ++ } ++ } else { ++ group->run_state = KBASE_CSF_GROUP_SUSPENDED; ++ atomic_inc(&scheduler->non_idle_suspended_grps); ++ } ++ } ++} ++ ++/* Cleanup_csg_slot after it has been vacated, ready for next csg run. ++ * Return whether there is a kctx address fault associated with the group ++ * for which the clean-up is done. ++ */ ++static bool cleanup_csg_slot(struct kbase_queue_group *group) ++{ ++ struct kbase_context *kctx = group->kctx; ++ struct kbase_device *kbdev = kctx->kbdev; ++ struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; ++ struct kbase_csf_cmd_stream_group_info *ginfo; ++ s8 slot; ++ struct kbase_csf_csg_slot *csg_slot; ++ unsigned long flags; ++ u32 i; ++ bool as_fault = false; ++ ++ lockdep_assert_held(&kbdev->csf.scheduler.lock); ++ ++ if (WARN_ON(!kbasep_csf_scheduler_group_is_on_slot_locked(group))) ++ return as_fault; ++ ++ slot = group->csg_nr; ++ csg_slot = &kbdev->csf.scheduler.csg_slots[slot]; ++ ginfo = &global_iface->groups[slot]; ++ ++ /* Now loop through all the bound CSs, and clean them via a stop */ ++ for (i = 0; i < ginfo->stream_num; i++) { ++ struct kbase_csf_cmd_stream_info *stream = &ginfo->streams[i]; ++ ++ if (group->bound_queues[i]) { ++ if (group->bound_queues[i]->enabled) { ++ kbase_csf_firmware_cs_input_mask(stream, ++ CS_REQ, CS_REQ_STATE_STOP, ++ CS_REQ_STATE_MASK); ++ } ++ ++ unassign_user_doorbell_from_queue(kbdev, ++ group->bound_queues[i]); ++ } ++ } ++ ++ unassign_user_doorbell_from_group(kbdev, group); ++ ++ /* The csg does not need cleanup other than drop its AS */ ++ spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, flags); ++ as_fault = kbase_ctx_flag(kctx, KCTX_AS_DISABLED_ON_FAULT); ++ kbase_ctx_sched_release_ctx(kctx); ++ if (unlikely(group->faulted)) ++ as_fault = true; ++ spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, flags); ++ ++ /* now marking the slot is vacant */ ++ spin_lock_irqsave(&kbdev->csf.scheduler.interrupt_lock, flags); ++ kbdev->csf.scheduler.csg_slots[slot].resident_group = NULL; ++ group->csg_nr = KBASEP_CSG_NR_INVALID; ++ clear_bit(slot, kbdev->csf.scheduler.csg_slots_idle_mask); ++ set_bit(slot, kbdev->csf.scheduler.csgs_events_enable_mask); ++ clear_bit(slot, kbdev->csf.scheduler.csg_inuse_bitmap); ++ spin_unlock_irqrestore(&kbdev->csf.scheduler.interrupt_lock, flags); ++ ++ csg_slot->trigger_jiffies = jiffies; ++ atomic_set(&csg_slot->state, CSG_SLOT_READY); ++ ++ KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_SLOT_CLEANED, group, slot); ++ dev_dbg(kbdev->dev, "Cleanup done for group %d on slot %d\n", ++ group->handle, slot); ++ ++ KBASE_TLSTREAM_TL_KBASE_DEVICE_DEPROGRAM_CSG(kbdev, ++ kbdev->gpu_props.props.raw_props.gpu_id, slot); ++ ++ return as_fault; ++} ++ ++static void update_csg_slot_priority(struct kbase_queue_group *group, u8 prio) ++{ ++ struct kbase_device *kbdev = group->kctx->kbdev; ++ struct kbase_csf_csg_slot *csg_slot; ++ struct kbase_csf_cmd_stream_group_info *ginfo; ++ s8 slot; ++ u8 prev_prio; ++ u32 ep_cfg; ++ u32 csg_req; ++ unsigned long flags; ++ ++ lockdep_assert_held(&kbdev->csf.scheduler.lock); ++ ++ if (WARN_ON(!kbasep_csf_scheduler_group_is_on_slot_locked(group))) ++ return; ++ ++ slot = group->csg_nr; ++ csg_slot = &kbdev->csf.scheduler.csg_slots[slot]; ++ ginfo = &kbdev->csf.global_iface.groups[slot]; ++ ++ WARN_ON(!((group->run_state == KBASE_CSF_GROUP_RUNNABLE) || ++ (group->run_state == KBASE_CSF_GROUP_IDLE))); ++ ++ group->run_state = KBASE_CSF_GROUP_RUNNABLE; ++ ++ if (csg_slot->priority == prio) ++ return; ++ ++ /* Read the csg_ep_cfg back for updating the priority field */ ++ ep_cfg = kbase_csf_firmware_csg_input_read(ginfo, CSG_EP_REQ); ++ prev_prio = CSG_EP_REQ_PRIORITY_GET(ep_cfg); ++ ep_cfg = CSG_EP_REQ_PRIORITY_SET(ep_cfg, prio); ++ kbase_csf_firmware_csg_input(ginfo, CSG_EP_REQ, ep_cfg); ++ ++ spin_lock_irqsave(&kbdev->csf.scheduler.interrupt_lock, flags); ++ csg_req = kbase_csf_firmware_csg_output(ginfo, CSG_ACK); ++ csg_req ^= CSG_REQ_EP_CFG; ++ kbase_csf_firmware_csg_input_mask(ginfo, CSG_REQ, csg_req, ++ CSG_REQ_EP_CFG); ++ spin_unlock_irqrestore(&kbdev->csf.scheduler.interrupt_lock, flags); ++ ++ csg_slot->priority = prio; ++ ++ dev_dbg(kbdev->dev, "Priority for group %d of context %d_%d on slot %d to be updated from %u to %u\n", ++ group->handle, group->kctx->tgid, group->kctx->id, slot, ++ prev_prio, prio); ++ ++ KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_PRIO_UPDATE, group, prev_prio); ++ ++ kbase_csf_ring_csg_doorbell(kbdev, slot); ++ set_bit(slot, kbdev->csf.scheduler.csg_slots_prio_update); ++} ++ ++static void program_csg_slot(struct kbase_queue_group *group, s8 slot, ++ u8 prio) ++{ ++ struct kbase_context *kctx = group->kctx; ++ struct kbase_device *kbdev = kctx->kbdev; ++ struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; ++ const u64 shader_core_mask = ++ kbase_pm_get_present_cores(kbdev, KBASE_PM_CORE_SHADER); ++ const u64 tiler_core_mask = ++ kbase_pm_get_present_cores(kbdev, KBASE_PM_CORE_TILER); ++ const u64 compute_mask = shader_core_mask & group->compute_mask; ++ const u64 fragment_mask = shader_core_mask & group->fragment_mask; ++ const u64 tiler_mask = tiler_core_mask & group->tiler_mask; ++ const u8 num_cores = kbdev->gpu_props.num_cores; ++ const u8 compute_max = min(num_cores, group->compute_max); ++ const u8 fragment_max = min(num_cores, group->fragment_max); ++ const u8 tiler_max = min(CSG_TILER_MAX, group->tiler_max); ++ struct kbase_csf_cmd_stream_group_info *ginfo; ++ u32 ep_cfg = 0; ++ u32 csg_req; ++ u32 state; ++ int i; ++ unsigned long flags; ++ const u64 normal_suspend_buf = ++ group->normal_suspend_buf.reg->start_pfn << PAGE_SHIFT; ++ struct kbase_csf_csg_slot *csg_slot = ++ &kbdev->csf.scheduler.csg_slots[slot]; ++ ++ lockdep_assert_held(&kbdev->csf.scheduler.lock); ++ ++ if (WARN_ON(slot < 0) && ++ WARN_ON(slot >= global_iface->group_num)) ++ return; ++ ++ WARN_ON(atomic_read(&csg_slot->state) != CSG_SLOT_READY); ++ ++ ginfo = &global_iface->groups[slot]; ++ ++ /* Pick an available address space for this context */ ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_ctx_sched_retain_ctx(kctx); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ ++ if (kctx->as_nr == KBASEP_AS_NR_INVALID) { ++ dev_dbg(kbdev->dev, "Could not get a valid AS for group %d of context %d_%d on slot %d\n", ++ group->handle, kctx->tgid, kctx->id, slot); ++ return; ++ } ++ ++ spin_lock_irqsave(&kbdev->csf.scheduler.interrupt_lock, flags); ++ set_bit(slot, kbdev->csf.scheduler.csg_inuse_bitmap); ++ kbdev->csf.scheduler.csg_slots[slot].resident_group = group; ++ group->csg_nr = slot; ++ spin_unlock_irqrestore(&kbdev->csf.scheduler.interrupt_lock, flags); ++ ++ assign_user_doorbell_to_group(kbdev, group); ++ ++ /* Now loop through all the bound & kicked CSs, and program them */ ++ for (i = 0; i < MAX_SUPPORTED_STREAMS_PER_GROUP; i++) { ++ struct kbase_queue *queue = group->bound_queues[i]; ++ ++ if (queue) ++ program_cs(kbdev, queue); ++ } ++ ++ ++ /* Endpoint programming for CSG */ ++ kbase_csf_firmware_csg_input(ginfo, CSG_ALLOW_COMPUTE_LO, ++ compute_mask & U32_MAX); ++ kbase_csf_firmware_csg_input(ginfo, CSG_ALLOW_COMPUTE_HI, ++ compute_mask >> 32); ++ kbase_csf_firmware_csg_input(ginfo, CSG_ALLOW_FRAGMENT_LO, ++ fragment_mask & U32_MAX); ++ kbase_csf_firmware_csg_input(ginfo, CSG_ALLOW_FRAGMENT_HI, ++ fragment_mask >> 32); ++ kbase_csf_firmware_csg_input(ginfo, CSG_ALLOW_OTHER, ++ tiler_mask & U32_MAX); ++ ++ ep_cfg = CSG_EP_REQ_COMPUTE_EP_SET(ep_cfg, compute_max); ++ ep_cfg = CSG_EP_REQ_FRAGMENT_EP_SET(ep_cfg, fragment_max); ++ ep_cfg = CSG_EP_REQ_TILER_EP_SET(ep_cfg, tiler_max); ++ ep_cfg = CSG_EP_REQ_PRIORITY_SET(ep_cfg, prio); ++ kbase_csf_firmware_csg_input(ginfo, CSG_EP_REQ, ep_cfg); ++ ++ /* Program the address space number assigned to the context */ ++ kbase_csf_firmware_csg_input(ginfo, CSG_CONFIG, kctx->as_nr); ++ ++ kbase_csf_firmware_csg_input(ginfo, CSG_SUSPEND_BUF_LO, ++ normal_suspend_buf & U32_MAX); ++ kbase_csf_firmware_csg_input(ginfo, CSG_SUSPEND_BUF_HI, ++ normal_suspend_buf >> 32); ++ ++ if (group->protected_suspend_buf.reg) { ++ const u64 protm_suspend_buf = ++ group->protected_suspend_buf.reg->start_pfn << ++ PAGE_SHIFT; ++ kbase_csf_firmware_csg_input(ginfo, CSG_PROTM_SUSPEND_BUF_LO, ++ protm_suspend_buf & U32_MAX); ++ kbase_csf_firmware_csg_input(ginfo, CSG_PROTM_SUSPEND_BUF_HI, ++ protm_suspend_buf >> 32); ++ } ++ ++ /* Enable all interrupts for now */ ++ kbase_csf_firmware_csg_input(ginfo, CSG_ACK_IRQ_MASK, ~((u32)0)); ++ ++ spin_lock_irqsave(&kbdev->csf.scheduler.interrupt_lock, flags); ++ csg_req = kbase_csf_firmware_csg_output(ginfo, CSG_ACK); ++ csg_req ^= CSG_REQ_EP_CFG; ++ kbase_csf_firmware_csg_input_mask(ginfo, CSG_REQ, csg_req, ++ CSG_REQ_EP_CFG); ++ ++ /* Set state to START/RESUME */ ++ if (queue_group_suspended_locked(group)) { ++ state = CSG_REQ_STATE_RESUME; ++ if (group->run_state == KBASE_CSF_GROUP_SUSPENDED) ++ atomic_dec( ++ &kbdev->csf.scheduler.non_idle_suspended_grps); ++ } else { ++ WARN_ON(group->run_state != KBASE_CSF_GROUP_RUNNABLE); ++ state = CSG_REQ_STATE_START; ++ } ++ ++ kbase_csf_firmware_csg_input_mask(ginfo, CSG_REQ, ++ state, CSG_REQ_STATE_MASK); ++ spin_unlock_irqrestore(&kbdev->csf.scheduler.interrupt_lock, flags); ++ ++ /* Update status before rings the door-bell, marking ready => run */ ++ atomic_set(&csg_slot->state, CSG_SLOT_READY2RUN); ++ csg_slot->trigger_jiffies = jiffies; ++ csg_slot->priority = prio; ++ ++ /* Trace the programming of the CSG on the slot */ ++ KBASE_TLSTREAM_TL_KBASE_DEVICE_PROGRAM_CSG(kbdev, ++ kbdev->gpu_props.props.raw_props.gpu_id, group->handle, slot); ++ ++ dev_dbg(kbdev->dev, "Starting group %d of context %d_%d on slot %d with priority %u\n", ++ group->handle, kctx->tgid, kctx->id, slot, prio); ++ ++ KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_SLOT_START, group, ++ (((u64)ep_cfg) << 32) | ++ ((((u32)kctx->as_nr) & 0xF) << 16) | ++ (state & (CSG_REQ_STATE_MASK >> CS_REQ_STATE_SHIFT))); ++ ++ kbase_csf_ring_csg_doorbell(kbdev, slot); ++} ++ ++static void remove_scheduled_group(struct kbase_device *kbdev, ++ struct kbase_queue_group *group) ++{ ++ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; ++ ++ lockdep_assert_held(&scheduler->lock); ++ ++ WARN_ON(group->prepared_seq_num == ++ KBASEP_GROUP_PREPARED_SEQ_NUM_INVALID); ++ WARN_ON(list_empty(&group->link_to_schedule)); ++ ++ list_del_init(&group->link_to_schedule); ++ scheduler->ngrp_to_schedule--; ++ group->prepared_seq_num = KBASEP_GROUP_PREPARED_SEQ_NUM_INVALID; ++ group->kctx->csf.sched.ngrp_to_schedule--; ++} ++ ++static void sched_evict_group(struct kbase_queue_group *group, bool fault) ++{ ++ struct kbase_context *kctx = group->kctx; ++ struct kbase_device *kbdev = kctx->kbdev; ++ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; ++ ++ lockdep_assert_held(&kbdev->csf.scheduler.lock); ++ ++ if (queue_group_scheduled_locked(group)) { ++ u32 i; ++ ++ if (group->run_state == KBASE_CSF_GROUP_SUSPENDED) ++ atomic_dec(&scheduler->non_idle_suspended_grps); ++ ++ for (i = 0; i < MAX_SUPPORTED_STREAMS_PER_GROUP; i++) { ++ if (group->bound_queues[i]) ++ group->bound_queues[i]->enabled = false; ++ } ++ ++ if (group->prepared_seq_num != ++ KBASEP_GROUP_PREPARED_SEQ_NUM_INVALID) ++ remove_scheduled_group(kbdev, group); ++ ++ if (group->run_state == KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC) ++ remove_group_from_idle_wait(group); ++ else { ++ remove_group_from_runnable(scheduler, group, ++ KBASE_CSF_GROUP_INACTIVE); ++ } ++ ++ WARN_ON(group->run_state != KBASE_CSF_GROUP_INACTIVE); ++ ++ if (fault) ++ group->run_state = KBASE_CSF_GROUP_FAULT_EVICTED; ++ ++ KBASE_KTRACE_ADD_CSF_GRP(kbdev, GROUP_EVICT_SCHED, group, ++ (((u64)scheduler->total_runnable_grps) << 32) | ++ ((u32)group->run_state)); ++ dev_dbg(kbdev->dev, "group %d exited scheduler, num_runnable_grps %d\n", ++ group->handle, scheduler->total_runnable_grps); ++ /* Notify a group has been evicted */ ++ wake_up_all(&kbdev->csf.event_wait); ++ } ++} ++ ++static int term_group_sync(struct kbase_queue_group *group) ++{ ++ struct kbase_device *kbdev = group->kctx->kbdev; ++ long remaining = ++ kbase_csf_timeout_in_jiffies(CSF_STATE_WAIT_TIMEOUT_MS); ++ int err = 0; ++ ++ term_csg_slot(group); ++ ++ remaining = wait_event_timeout(kbdev->csf.event_wait, ++ csg_slot_stopped_locked(kbdev, group->csg_nr), remaining); ++ ++ if (!remaining) { ++ dev_warn(kbdev->dev, "term request timed out for group %d on slot %d", ++ group->handle, group->csg_nr); ++ if (kbase_prepare_to_reset_gpu(kbdev)) ++ kbase_reset_gpu(kbdev); ++ err = -ETIMEDOUT; ++ } ++ ++ return err; ++} ++ ++void kbase_csf_scheduler_group_deschedule(struct kbase_queue_group *group) ++{ ++ struct kbase_device *kbdev = group->kctx->kbdev; ++ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; ++ long remaining = ++ kbase_csf_timeout_in_jiffies(CSG_SCHED_STOP_TIMEOUT_MS); ++ bool force = false; ++ ++ lockdep_assert_held(&group->kctx->csf.lock); ++ mutex_lock(&scheduler->lock); ++ ++ KBASE_KTRACE_ADD_CSF_GRP(kbdev, GROUP_DESCHEDULE, group, group->run_state); ++ while (queue_group_scheduled_locked(group)) { ++ u32 saved_state = scheduler->state; ++ bool reset = kbase_reset_gpu_is_active(kbdev); ++ ++ if (!kbasep_csf_scheduler_group_is_on_slot_locked(group)) { ++ sched_evict_group(group, false); ++ } else if (reset || saved_state == SCHED_INACTIVE || force) { ++ bool as_faulty; ++ ++ if (!reset) ++ term_group_sync(group); ++ /* Treat the csg been terminated */ ++ as_faulty = cleanup_csg_slot(group); ++ /* remove from the scheduler list */ ++ sched_evict_group(group, as_faulty); ++ } ++ ++ /* waiting scheduler state to change */ ++ if (queue_group_scheduled_locked(group)) { ++ mutex_unlock(&scheduler->lock); ++ remaining = wait_event_timeout( ++ kbdev->csf.event_wait, ++ saved_state != scheduler->state, ++ remaining); ++ if (!remaining) { ++ dev_warn(kbdev->dev, "Scheduler state change wait timed out for group %d on slot %d", ++ group->handle, group->csg_nr); ++ force = true; ++ } ++ mutex_lock(&scheduler->lock); ++ } ++ } ++ ++ mutex_unlock(&scheduler->lock); ++} ++ ++/** ++ * scheduler_group_schedule() - Schedule a GPU command queue group on firmware ++ * ++ * @group: Pointer to the queue group to be scheduled. ++ * ++ * This function would enable the scheduling of GPU command queue group on ++ * firmware. ++ * ++ * Return: 0 on success, or negative on failure. ++ */ ++static int scheduler_group_schedule(struct kbase_queue_group *group) ++{ ++ struct kbase_context *kctx = group->kctx; ++ struct kbase_device *kbdev = kctx->kbdev; ++ ++ lockdep_assert_held(&kctx->csf.lock); ++ lockdep_assert_held(&kbdev->csf.scheduler.lock); ++ ++ KBASE_KTRACE_ADD_CSF_GRP(kbdev, GROUP_SCHEDULE, group, group->run_state); ++ if (group->run_state == KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC) ++ update_idle_suspended_group_state(group); ++ else if (queue_group_idle_locked(group)) { ++ WARN_ON(kctx->csf.sched.num_runnable_grps == 0); ++ WARN_ON(kbdev->csf.scheduler.total_runnable_grps == 0); ++ ++ if (group->run_state == KBASE_CSF_GROUP_SUSPENDED_ON_IDLE) ++ update_idle_suspended_group_state(group); ++ else ++ group->run_state = KBASE_CSF_GROUP_RUNNABLE; ++ } else if (!queue_group_scheduled_locked(group)) { ++ insert_group_to_runnable(&kbdev->csf.scheduler, group, ++ KBASE_CSF_GROUP_RUNNABLE); ++ } ++ ++ /* Since a group has become active now, check if GPU needs to be ++ * powered up. Also rekick the Scheduler. ++ */ ++ scheduler_wakeup(kbdev, true); ++ ++ return 0; ++} ++ ++/** ++ * set_max_csg_slots() - Set the number of available command stream group slots ++ * ++ * @kbdev: Pointer of the GPU device. ++ * ++ * This function would set/limit the number of command stream group slots that ++ * can be used in the given tick/tock. It would be less than the total command ++ * stream group slots supported by firmware if the number of GPU address space ++ * slots required to utilize all the CSG slots is more than the available ++ * address space slots. ++ */ ++static inline void set_max_csg_slots(struct kbase_device *kbdev) ++{ ++ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; ++ unsigned int total_csg_slots = kbdev->csf.global_iface.group_num; ++ unsigned int max_address_space_slots = kbdev->nr_hw_address_spaces - 1; ++ ++ WARN_ON(scheduler->num_active_address_spaces > total_csg_slots); ++ ++ if (likely(scheduler->num_active_address_spaces <= ++ max_address_space_slots)) ++ scheduler->num_csg_slots_for_tick = total_csg_slots; ++} ++ ++/** ++ * count_active_address_space() - Count the number of GPU address space slots ++ * ++ * @kbdev: Pointer of the GPU device. ++ * @kctx: Pointer of the Kbase context. ++ * ++ * This function would update the counter that is tracking the number of GPU ++ * address space slots that would be required to program the command stream ++ * group slots from the groups at the head of groups_to_schedule list. ++ */ ++static inline void count_active_address_space(struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; ++ unsigned int total_csg_slots = kbdev->csf.global_iface.group_num; ++ unsigned int max_address_space_slots = kbdev->nr_hw_address_spaces - 1; ++ ++ if (scheduler->ngrp_to_schedule <= total_csg_slots) { ++ if (kctx->csf.sched.ngrp_to_schedule == 1) { ++ scheduler->num_active_address_spaces++; ++ ++ if (scheduler->num_active_address_spaces <= ++ max_address_space_slots) ++ scheduler->num_csg_slots_for_tick++; ++ } ++ } ++} ++ ++/** ++ * update_resident_groups_priority() - Update the priority of resident groups ++ * ++ * @kbdev: The GPU device. ++ * ++ * This function will update the priority of all resident queue groups ++ * that are at the head of groups_to_schedule list, preceding the first ++ * non-resident group. ++ * ++ * This function will also adjust kbase_csf_scheduler.head_slot_priority on ++ * the priority update. ++ */ ++static void update_resident_groups_priority(struct kbase_device *kbdev) ++{ ++ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; ++ u32 num_groups = scheduler->num_csg_slots_for_tick; ++ ++ lockdep_assert_held(&kbdev->csf.scheduler.lock); ++ while (!list_empty(&scheduler->groups_to_schedule)) { ++ struct kbase_queue_group *group = ++ list_first_entry(&scheduler->groups_to_schedule, ++ struct kbase_queue_group, ++ link_to_schedule); ++ bool resident = ++ kbasep_csf_scheduler_group_is_on_slot_locked(group); ++ ++ if ((group->prepared_seq_num >= num_groups) || !resident) ++ break; ++ ++ update_csg_slot_priority(group, ++ scheduler->head_slot_priority); ++ ++ /* Drop the head group from the list */ ++ remove_scheduled_group(kbdev, group); ++ scheduler->head_slot_priority--; ++ } ++} ++ ++/** ++ * program_group_on_vacant_csg_slot() - Program a non-resident group on the ++ * given vacant CSG slot. ++ * @kbdev: Pointer to the GPU device. ++ * @slot: Vacant command stream group slot number. ++ * ++ * This function will program a non-resident group at the head of ++ * kbase_csf_scheduler.groups_to_schedule list on the given vacant command ++ * stream group slot, provided the initial position of the non-resident ++ * group in the list is less than the number of CSG slots and there is ++ * an available GPU address space slot. ++ * kbase_csf_scheduler.head_slot_priority would also be adjusted after ++ * programming the slot. ++ */ ++static void program_group_on_vacant_csg_slot(struct kbase_device *kbdev, ++ s8 slot) ++{ ++ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; ++ struct kbase_queue_group *const group = ++ list_empty(&scheduler->groups_to_schedule) ? NULL : ++ list_first_entry(&scheduler->groups_to_schedule, ++ struct kbase_queue_group, ++ link_to_schedule); ++ u32 num_groups = scheduler->num_csg_slots_for_tick; ++ ++ lockdep_assert_held(&kbdev->csf.scheduler.lock); ++ if (group && (group->prepared_seq_num < num_groups)) { ++ bool ret = kbasep_csf_scheduler_group_is_on_slot_locked(group); ++ ++ if (!WARN_ON(ret)) { ++ if (kctx_as_enabled(group->kctx) && !group->faulted) { ++ program_csg_slot(group, ++ slot, ++ scheduler->head_slot_priority); ++ ++ if (likely(csg_slot_in_use(kbdev, slot))) { ++ /* Drop the head group from the list */ ++ remove_scheduled_group(kbdev, group); ++ scheduler->head_slot_priority--; ++ } ++ } else ++ remove_scheduled_group(kbdev, group); ++ } ++ } ++} ++ ++/** ++ * program_vacant_csg_slot() - Program the vacant CSG slot with a non-resident ++ * group and update the priority of resident groups. ++ * ++ * @kbdev: Pointer to the GPU device. ++ * @slot: Vacant command stream group slot number. ++ * ++ * This function will first update the priority of all resident queue groups ++ * that are at the head of groups_to_schedule list, preceding the first ++ * non-resident group, it will then try to program the given command stream ++ * group slot with the non-resident group. Finally update the priority of all ++ * resident queue groups following the non-resident group. ++ * ++ * kbase_csf_scheduler.head_slot_priority would also be adjusted. ++ */ ++static void program_vacant_csg_slot(struct kbase_device *kbdev, s8 slot) ++{ ++ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; ++ struct kbase_csf_csg_slot *const csg_slot = ++ scheduler->csg_slots; ++ ++ lockdep_assert_held(&kbdev->csf.scheduler.lock); ++ WARN_ON(atomic_read(&csg_slot[slot].state) != CSG_SLOT_READY); ++ ++ /* First update priority for already resident groups (if any) ++ * before the non-resident group ++ */ ++ update_resident_groups_priority(kbdev); ++ ++ /* Now consume the vacant slot for the non-resident group */ ++ program_group_on_vacant_csg_slot(kbdev, slot); ++ ++ /* Now update priority for already resident groups (if any) ++ * following the non-resident group ++ */ ++ update_resident_groups_priority(kbdev); ++} ++ ++static bool slots_state_changed(struct kbase_device *kbdev, ++ unsigned long *slots_mask, ++ bool (*state_check_func)(struct kbase_device *, s8)) ++{ ++ u32 num_groups = kbdev->csf.global_iface.group_num; ++ DECLARE_BITMAP(changed_slots, MAX_SUPPORTED_CSGS) = {0}; ++ bool changed = false; ++ u32 i; ++ ++ for_each_set_bit(i, slots_mask, num_groups) { ++ if (state_check_func(kbdev, (s8)i)) { ++ set_bit(i, changed_slots); ++ changed = true; ++ } ++ } ++ ++ if (changed) ++ bitmap_copy(slots_mask, changed_slots, MAX_SUPPORTED_CSGS); ++ ++ return changed; ++} ++ ++/** ++ * program_suspending_csg_slots() - Program the CSG slots vacated on suspension ++ * of queue groups running on them. ++ * ++ * @kbdev: Pointer to the GPU device. ++ * ++ * This function will first wait for the ongoing suspension to complete on a ++ * command stream group slot and will then program the vacant slot with the ++ * non-resident queue group inside the groups_to_schedule list. ++ * The programming of the non-resident queue group on the vacant slot could ++ * fail due to unavailability of free GPU address space slot and so the ++ * programming is re-attempted after the ongoing suspension has completed ++ * for all the command stream group slots. ++ * The priority of resident groups before and after the non-resident group ++ * in the groups_to_schedule list would also be updated. ++ * This would be repeated for all the slots undergoing suspension. ++ * GPU reset would be initiated if the wait for suspend times out. ++ */ ++static void program_suspending_csg_slots(struct kbase_device *kbdev) ++{ ++ u32 num_groups = kbdev->csf.global_iface.group_num; ++ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; ++ DECLARE_BITMAP(slot_mask, MAX_SUPPORTED_CSGS); ++ DECLARE_BITMAP(evicted_mask, MAX_SUPPORTED_CSGS) = {0}; ++ bool suspend_wait_failed = false; ++ long remaining = ++ kbase_csf_timeout_in_jiffies(CSF_STATE_WAIT_TIMEOUT_MS); ++ ++ lockdep_assert_held(&kbdev->csf.scheduler.lock); ++ ++ bitmap_complement(slot_mask, scheduler->csgs_events_enable_mask, ++ MAX_SUPPORTED_CSGS); ++ ++ while (!bitmap_empty(slot_mask, MAX_SUPPORTED_CSGS)) { ++ DECLARE_BITMAP(changed, MAX_SUPPORTED_CSGS); ++ ++ bitmap_copy(changed, slot_mask, MAX_SUPPORTED_CSGS); ++ ++ remaining = wait_event_timeout(kbdev->csf.event_wait, ++ slots_state_changed(kbdev, changed, ++ csg_slot_stopped_raw), ++ remaining); ++ ++ if (remaining) { ++ u32 i; ++ ++ for_each_set_bit(i, changed, num_groups) { ++ struct kbase_queue_group *group = ++ scheduler->csg_slots[i].resident_group; ++ ++ if (WARN_ON(!csg_slot_stopped_locked(kbdev, (s8)i))) { ++ continue; ++ } ++ /* The on slot csg is now stopped */ ++ clear_bit(i, slot_mask); ++ ++ if (likely(group)) { ++ bool as_fault; ++ /* Only do save/cleanup if the ++ * group is not terminated during ++ * the sleep. ++ */ ++ save_csg_slot(group); ++ as_fault = cleanup_csg_slot(group); ++ /* If AS fault detected, evict it */ ++ if (as_fault) { ++ sched_evict_group(group, true); ++ set_bit(i, evicted_mask); ++ } ++ } ++ ++ program_vacant_csg_slot(kbdev, (s8)i); ++ } ++ } else { ++ dev_warn(kbdev->dev, "Timed out waiting for CSG slots to suspend, slot_mask: 0x%*pb\n", ++ num_groups, slot_mask); ++ ++ if (kbase_prepare_to_reset_gpu(kbdev)) ++ kbase_reset_gpu(kbdev); ++ suspend_wait_failed = true; ++ break; ++ } ++ } ++ ++ if (!bitmap_empty(evicted_mask, MAX_SUPPORTED_CSGS)) ++ dev_info(kbdev->dev, "Scheduler evicting slots: 0x%*pb\n", ++ num_groups, evicted_mask); ++ ++ if (unlikely(!suspend_wait_failed)) { ++ u32 i; ++ ++ while (scheduler->ngrp_to_schedule && ++ (scheduler->head_slot_priority > (MAX_CSG_SLOT_PRIORITY ++ - scheduler->num_csg_slots_for_tick))) { ++ i = find_first_zero_bit(scheduler->csg_inuse_bitmap, ++ num_groups); ++ if (WARN_ON(i == num_groups)) ++ break; ++ program_vacant_csg_slot(kbdev, (s8)i); ++ if (WARN_ON(!csg_slot_in_use(kbdev, (int)i))) ++ break; ++ } ++ } ++} ++ ++static void suspend_queue_group(struct kbase_queue_group *group) ++{ ++ unsigned long flags; ++ struct kbase_csf_scheduler *const scheduler = ++ &group->kctx->kbdev->csf.scheduler; ++ ++ spin_lock_irqsave(&scheduler->interrupt_lock, flags); ++ clear_bit(group->csg_nr, scheduler->csgs_events_enable_mask); ++ spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); ++ ++ /* If AS fault detected, terminate the group */ ++ if (!kctx_as_enabled(group->kctx) || group->faulted) ++ term_csg_slot(group); ++ else ++ suspend_csg_slot(group); ++} ++ ++static void wait_csg_slots_start(struct kbase_device *kbdev) ++{ ++ u32 num_groups = kbdev->csf.global_iface.group_num; ++ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; ++ long remaining = ++ kbase_csf_timeout_in_jiffies(CSF_STATE_WAIT_TIMEOUT_MS); ++ DECLARE_BITMAP(slot_mask, MAX_SUPPORTED_CSGS) = {0}; ++ u32 i; ++ ++ lockdep_assert_held(&kbdev->csf.scheduler.lock); ++ ++ /* extract start slot flags for check */ ++ for (i = 0; i < num_groups; i++) { ++ if (atomic_read(&scheduler->csg_slots[i].state) == ++ CSG_SLOT_READY2RUN) ++ set_bit(i, slot_mask); ++ } ++ ++ while (!bitmap_empty(slot_mask, MAX_SUPPORTED_CSGS)) { ++ DECLARE_BITMAP(changed, MAX_SUPPORTED_CSGS); ++ ++ bitmap_copy(changed, slot_mask, MAX_SUPPORTED_CSGS); ++ ++ remaining = wait_event_timeout(kbdev->csf.event_wait, ++ slots_state_changed(kbdev, changed, csg_slot_running), ++ remaining); ++ ++ if (remaining) { ++ for_each_set_bit(i, changed, num_groups) { ++ struct kbase_queue_group *group = ++ scheduler->csg_slots[i].resident_group; ++ ++ /* The on slot csg is now running */ ++ clear_bit(i, slot_mask); ++ group->run_state = KBASE_CSF_GROUP_RUNNABLE; ++ } ++ } else { ++ dev_warn(kbdev->dev, "Timed out waiting for CSG slots to start, slots: 0x%*pb\n", ++ num_groups, slot_mask); ++ ++ if (kbase_prepare_to_reset_gpu(kbdev)) ++ kbase_reset_gpu(kbdev); ++ break; ++ } ++ } ++} ++ ++/** ++ * group_on_slot_is_idle() - Check if the queue group resident on a command ++ * stream group slot is idle. ++ * ++ * This function is called at the start of scheduling tick to check the ++ * idle status of a queue group resident on a command sream group slot. ++ * The group's idleness is determined by looping over all the bound command ++ * queues and checking their respective CS_STATUS_WAIT register as well as ++ * the insert and extract offsets. ++ ++ * This function would be simplified in future after the changes under ++ * consideration with MIDHARC-3065 are introduced. ++ * ++ * @kbdev: Pointer to the GPU device. ++ * @group: Pointer to the resident group on the given slot. ++ * @slot: The slot that the given group is resident on. ++ * ++ * Return: true if the group resident on slot is idle, otherwise false. ++ */ ++static bool group_on_slot_is_idle(struct kbase_device *kbdev, ++ struct kbase_queue_group *group, unsigned long slot) ++{ ++ struct kbase_csf_cmd_stream_group_info *ginfo = ++ &kbdev->csf.global_iface.groups[slot]; ++ u32 i; ++ ++ lockdep_assert_held(&kbdev->csf.scheduler.lock); ++ for (i = 0; i < MAX_SUPPORTED_STREAMS_PER_GROUP; i++) { ++ struct kbase_queue *queue = group->bound_queues[i]; ++ ++ if (queue && queue->enabled) { ++ struct kbase_csf_cmd_stream_info *stream = ++ &ginfo->streams[queue->csi_index]; ++ u32 status = kbase_csf_firmware_cs_output(stream, ++ CS_STATUS_WAIT); ++ ++ if (!CS_STATUS_WAIT_SYNC_WAIT_GET(status) && ++ !confirm_cs_idle(group->bound_queues[i])) ++ return false; ++ } ++ } ++ ++ return true; ++} ++ ++/** ++ * slots_update_state_changed() - Check the handshake state of a subset of ++ * command group slots. ++ * ++ * Checks the state of a subset of slots selected through the slots_mask ++ * bit_map. Records which slots' handshake completed and send it back in the ++ * slots_done bit_map. ++ * ++ * @kbdev: The GPU device. ++ * @field_mask: The field mask for checking the state in the csg_req/ack. ++ * @slots_mask: A bit_map specifying the slots to check. ++ * @slots_done: A cleared bit_map for returning the slots that ++ * have finished update. ++ * ++ * Return: true if the slots_done is set for at least one slot. ++ * Otherwise false. ++ */ ++static ++bool slots_update_state_changed(struct kbase_device *kbdev, u32 field_mask, ++ const unsigned long *slots_mask, unsigned long *slots_done) ++{ ++ u32 num_groups = kbdev->csf.global_iface.group_num; ++ bool changed = false; ++ u32 i; ++ ++ lockdep_assert_held(&kbdev->csf.scheduler.lock); ++ ++ for_each_set_bit(i, slots_mask, num_groups) { ++ struct kbase_csf_cmd_stream_group_info const *const ginfo = ++ &kbdev->csf.global_iface.groups[i]; ++ u32 state = kbase_csf_firmware_csg_input_read(ginfo, CSG_REQ); ++ ++ state ^= kbase_csf_firmware_csg_output(ginfo, CSG_ACK); ++ ++ if (!(state & field_mask)) { ++ set_bit(i, slots_done); ++ changed = true; ++ } ++ } ++ ++ return changed; ++} ++ ++/** ++ * wait_csg_slots_handshake_ack - Wait the req/ack handshakes to complete on ++ * the specified groups. ++ * ++ * This function waits for the acknowledgement of the request that have ++ * already been placed for the CSG slots by the caller. Currently used for ++ * the CSG priority update and status update requests. ++ * ++ * @kbdev: Pointer to the GPU device. ++ * @field_mask: The field mask for checking the state in the csg_req/ack. ++ * @slot_mask: Bitmap reflecting the slots, the function will modify ++ * the acknowledged slots by clearing their corresponding ++ * bits. ++ * @wait_in_jiffies: Wait duration in jiffies, controlling the time-out. ++ * ++ * Return: 0 on all specified slots acknowledged; otherwise -ETIMEDOUT. For ++ * timed out condition with unacknowledged slots, their bits remain ++ * set in the slot_mask. ++ */ ++static int wait_csg_slots_handshake_ack(struct kbase_device *kbdev, ++ u32 field_mask, unsigned long *slot_mask, long wait_in_jiffies) ++{ ++ const u32 num_groups = kbdev->csf.global_iface.group_num; ++ long remaining = wait_in_jiffies; ++ ++ lockdep_assert_held(&kbdev->csf.scheduler.lock); ++ ++ while (!bitmap_empty(slot_mask, num_groups) && ++ !kbase_reset_gpu_is_active(kbdev)) { ++ DECLARE_BITMAP(dones, MAX_SUPPORTED_CSGS) = { 0 }; ++ ++ remaining = wait_event_timeout(kbdev->csf.event_wait, ++ slots_update_state_changed(kbdev, field_mask, ++ slot_mask, dones), ++ remaining); ++ ++ if (remaining) ++ bitmap_andnot(slot_mask, slot_mask, dones, num_groups); ++ else ++ /* Timed-out on the wait */ ++ return -ETIMEDOUT; ++ } ++ ++ return 0; ++} ++ ++static void wait_csg_slots_finish_prio_update(struct kbase_device *kbdev) ++{ ++ unsigned long *slot_mask = ++ kbdev->csf.scheduler.csg_slots_prio_update; ++ long wait_time = ++ kbase_csf_timeout_in_jiffies(CSF_STATE_WAIT_TIMEOUT_MS); ++ int ret = wait_csg_slots_handshake_ack(kbdev, CSG_REQ_EP_CFG_MASK, ++ slot_mask, wait_time); ++ ++ lockdep_assert_held(&kbdev->csf.scheduler.lock); ++ ++ if (ret != 0) { ++ /* The update timeout is not regarded as a serious ++ * issue, no major consequences are expected as a ++ * result, so just warn the case. ++ */ ++ dev_warn(kbdev->dev, "Timeout, skipping the update wait: slot mask=0x%lx", ++ slot_mask[0]); ++ } ++} ++ ++void kbase_csf_scheduler_evict_ctx_slots(struct kbase_device *kbdev, ++ struct kbase_context *kctx, struct list_head *evicted_groups) ++{ ++ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; ++ struct kbase_queue_group *group; ++ u32 num_groups = kbdev->csf.global_iface.group_num; ++ u32 slot; ++ DECLARE_BITMAP(slot_mask, MAX_SUPPORTED_CSGS) = {0}; ++ DECLARE_BITMAP(terminated_slot_mask, MAX_SUPPORTED_CSGS); ++ long remaining = ++ kbase_csf_timeout_in_jiffies(DEFAULT_RESET_TIMEOUT_MS); ++ ++ lockdep_assert_held(&kctx->csf.lock); ++ mutex_lock(&scheduler->lock); ++ ++ KBASE_KTRACE_ADD(kbdev, EVICT_CTX_SLOTS, kctx, 0u); ++ for (slot = 0; slot < num_groups; slot++) { ++ group = kbdev->csf.scheduler.csg_slots[slot].resident_group; ++ if (group && group->kctx == kctx) { ++ term_csg_slot(group); ++ set_bit(slot, slot_mask); ++ } ++ } ++ ++ dev_info(kbdev->dev, "Evicting context %d_%d slots: 0x%*pb\n", ++ kctx->tgid, kctx->id, num_groups, slot_mask); ++ ++ bitmap_copy(terminated_slot_mask, slot_mask, MAX_SUPPORTED_CSGS); ++ /* Only check for GPU reset once - this thread has the scheduler lock, ++ * so even if the return value of kbase_reset_gpu_is_active changes, ++ * no reset work would be done anyway until the scheduler lock was ++ * released. ++ */ ++ if (!kbase_reset_gpu_is_active(kbdev)) { ++ while (remaining ++ && !bitmap_empty(slot_mask, MAX_SUPPORTED_CSGS)) { ++ DECLARE_BITMAP(changed, MAX_SUPPORTED_CSGS); ++ ++ bitmap_copy(changed, slot_mask, MAX_SUPPORTED_CSGS); ++ ++ remaining = wait_event_timeout(kbdev->csf.event_wait, ++ slots_state_changed(kbdev, changed, ++ csg_slot_stopped_raw), ++ remaining); ++ ++ if (remaining) ++ bitmap_andnot(slot_mask, slot_mask, changed, ++ MAX_SUPPORTED_CSGS); ++ } ++ } ++ ++ for_each_set_bit(slot, terminated_slot_mask, num_groups) { ++ bool as_fault; ++ ++ group = scheduler->csg_slots[slot].resident_group; ++ as_fault = cleanup_csg_slot(group); ++ /* remove the group from the scheduler list */ ++ sched_evict_group(group, as_fault); ++ /* return the evicted group to the caller */ ++ list_add_tail(&group->link, evicted_groups); ++ } ++ ++ if (!remaining) { ++ dev_warn(kbdev->dev, "Timeout on evicting ctx slots: 0x%*pb\n", ++ num_groups, slot_mask); ++ if (kbase_prepare_to_reset_gpu(kbdev)) ++ kbase_reset_gpu(kbdev); ++ } ++ ++ mutex_unlock(&scheduler->lock); ++} ++ ++/** ++ * scheduler_slot_protm_ack - Acknowledging the protected region requests ++ * from the resident group on a given slot. ++ * ++ * The function assumes that the given slot is in stable running state and ++ * has already been judged by the caller on that any pending protected region ++ * requests of the resident group should be acknowledged. ++ * ++ * @kbdev: Pointer to the GPU device. ++ * @group: Pointer to the resident group on the given slot. ++ * @slot: The slot that the given group is actively operating on. ++ * ++ * Return: true if the group has pending protm request(s) and is acknowledged. ++ * The caller should arrange to enter the protected mode for servicing ++ * it. Otherwise return false, indicating the group has no pending protm ++ * request. ++ */ ++static bool scheduler_slot_protm_ack(struct kbase_device *const kbdev, ++ struct kbase_queue_group *const group, ++ const int slot) ++{ ++ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; ++ bool protm_ack = false; ++ struct kbase_csf_cmd_stream_group_info *ginfo = ++ &kbdev->csf.global_iface.groups[slot]; ++ u32 max_csi; ++ int i; ++ ++ if (WARN_ON(scheduler->csg_slots[slot].resident_group != group)) ++ return protm_ack; ++ ++ lockdep_assert_held(&scheduler->lock); ++ lockdep_assert_held(&group->kctx->kbdev->csf.scheduler.interrupt_lock); ++ ++ max_csi = ginfo->stream_num; ++ for (i = find_first_bit(group->protm_pending_bitmap, max_csi); ++ i < max_csi; ++ i = find_next_bit(group->protm_pending_bitmap, max_csi, i + 1)) { ++ struct kbase_queue *queue = group->bound_queues[i]; ++ ++ clear_bit(i, group->protm_pending_bitmap); ++ ++ if (!WARN_ON(!queue) && queue->enabled) { ++ struct kbase_csf_cmd_stream_info *stream = ++ &ginfo->streams[i]; ++ u32 cs_protm_ack = kbase_csf_firmware_cs_output( ++ stream, CS_ACK) & ++ CS_ACK_PROTM_PEND_MASK; ++ u32 cs_protm_req = kbase_csf_firmware_cs_input_read( ++ stream, CS_REQ) & ++ CS_REQ_PROTM_PEND_MASK; ++ ++ if (cs_protm_ack == cs_protm_req) { ++ dev_dbg(kbdev->dev, ++ "PROTM-ack already done for queue-%d group-%d slot-%d", ++ queue->csi_index, group->handle, slot); ++ continue; ++ } ++ ++ kbase_csf_firmware_cs_input_mask(stream, CS_REQ, ++ cs_protm_ack, ++ CS_ACK_PROTM_PEND_MASK); ++ protm_ack = true; ++ dev_dbg(kbdev->dev, ++ "PROTM-ack for queue-%d, group-%d slot-%d", ++ queue->csi_index, group->handle, slot); ++ } ++ } ++ ++ return protm_ack; ++} ++ ++/** ++ * scheduler_group_check_protm_enter - Request the given group to be evaluated ++ * for triggering the protected mode. ++ * ++ * The function assumes the given group is either an active running group or ++ * the scheduler internally maintained field scheduler->top_grp. ++ * ++ * If the GPU is not already running in protected mode and the input group ++ * has protected region requests from its bound queues, the requests are ++ * acknowledged and the GPU is instructed to enter the protected mode. ++ * ++ * @kbdev: Pointer to the GPU device. ++ * @input_grp: Pointer to the GPU queue group. ++ */ ++static void scheduler_group_check_protm_enter(struct kbase_device *const kbdev, ++ struct kbase_queue_group *const input_grp) ++{ ++ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; ++ unsigned long flags; ++ ++ lockdep_assert_held(&scheduler->lock); ++ ++ spin_lock_irqsave(&scheduler->interrupt_lock, flags); ++ ++ /* Firmware samples the PROTM_PEND ACK bit for command streams when ++ * Host sends PROTM_ENTER global request. So if PROTM_PEND ACK bit ++ * is set for a command stream after Host has sent the PROTM_ENTER ++ * Global request, then there is no guarantee that firmware will ++ * notice that prior to switching to protected mode. And firmware ++ * may not again raise the PROTM_PEND interrupt for that command ++ * stream later on. To avoid that uncertainty PROTM_PEND ACK bit ++ * is not set for a command stream if the request to enter protected ++ * mode has already been sent. It will be set later (after the exit ++ * from protected mode has taken place) when the group to which ++ * command stream is bound becomes the top group. ++ * ++ * The actual decision of entering protected mode is hinging on the ++ * input group is the top priority group, or, in case the previous ++ * top-group is evicted from the scheduler during the tick, its would ++ * be replacement, and that it is currently in a stable state (i.e. the ++ * slot state is running). ++ */ ++ if (!kbase_csf_scheduler_protected_mode_in_use(kbdev)) { ++ if (!WARN_ON(!input_grp)) { ++ const int slot = ++ kbase_csf_scheduler_group_get_slot_locked( ++ input_grp); ++ ++ /* check the input_grp is running and requesting ++ * protected mode ++ */ ++ if (slot >= 0 && ++ atomic_read( ++ &scheduler->csg_slots[slot].state) == ++ CSG_SLOT_RUNNING) { ++ if (kctx_as_enabled(input_grp->kctx) && ++ scheduler_slot_protm_ack(kbdev, ++ input_grp, slot)) { ++ /* Option of acknowledging to multiple ++ * CSGs from the same kctx is dropped, ++ * after consulting with the ++ * architecture team. See the comment in ++ * GPUCORE-21394. ++ */ ++ ++ /* Switch to protected mode */ ++ scheduler->active_protm_grp = input_grp; ++ KBASE_KTRACE_ADD_CSF_GRP(kbdev, SCHEDULER_ENTER_PROTM, input_grp, 0u); ++ spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); ++ kbase_csf_enter_protected_mode(kbdev); ++ return; ++ } ++ } ++ } ++ } ++ ++ spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); ++} ++ ++static void scheduler_apply(struct kbase_device *kbdev) ++{ ++ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; ++ const u32 total_csg_slots = kbdev->csf.global_iface.group_num; ++ const u32 available_csg_slots = scheduler->num_csg_slots_for_tick; ++ u32 suspend_cnt = 0; ++ u32 remain_cnt = 0; ++ u32 resident_cnt = 0; ++ struct kbase_queue_group *group; ++ u32 i; ++ u32 spare; ++ ++ lockdep_assert_held(&scheduler->lock); ++ ++ /* Suspend those resident groups not in the run list */ ++ for (i = 0; i < total_csg_slots; i++) { ++ group = scheduler->csg_slots[i].resident_group; ++ if (group) { ++ resident_cnt++; ++ if (group->prepared_seq_num >= available_csg_slots) { ++ suspend_queue_group(group); ++ suspend_cnt++; ++ } else ++ remain_cnt++; ++ } ++ } ++ ++ /* If there are spare slots, apply heads in the list */ ++ spare = (available_csg_slots > resident_cnt) ? ++ (available_csg_slots - resident_cnt) : 0; ++ while (!list_empty(&scheduler->groups_to_schedule)) { ++ group = list_first_entry(&scheduler->groups_to_schedule, ++ struct kbase_queue_group, ++ link_to_schedule); ++ ++ if (kbasep_csf_scheduler_group_is_on_slot_locked(group) && ++ group->prepared_seq_num < available_csg_slots) { ++ /* One of the resident remainders */ ++ update_csg_slot_priority(group, ++ scheduler->head_slot_priority); ++ } else if (spare != 0) { ++ s8 slot = (s8)find_first_zero_bit( ++ kbdev->csf.scheduler.csg_inuse_bitmap, ++ total_csg_slots); ++ ++ if (WARN_ON(slot >= (s8)total_csg_slots)) ++ break; ++ ++ if (!kctx_as_enabled(group->kctx) || group->faulted) { ++ /* Drop the head group and continue */ ++ remove_scheduled_group(kbdev, group); ++ continue; ++ } ++ program_csg_slot(group, slot, ++ scheduler->head_slot_priority); ++ if (unlikely(!csg_slot_in_use(kbdev, slot))) ++ break; ++ ++ spare--; ++ } else ++ break; ++ ++ /* Drop the head csg from the list */ ++ remove_scheduled_group(kbdev, group); ++ if (scheduler->head_slot_priority) ++ scheduler->head_slot_priority--; ++ } ++ ++ /* Dealing with groups currently going through suspend */ ++ program_suspending_csg_slots(kbdev); ++} ++ ++static void scheduler_ctx_scan_groups(struct kbase_device *kbdev, ++ struct kbase_context *kctx, int priority) ++{ ++ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; ++ struct kbase_queue_group *group; ++ ++ lockdep_assert_held(&scheduler->lock); ++ if (WARN_ON(priority < 0) || ++ WARN_ON(priority >= BASE_QUEUE_GROUP_PRIORITY_COUNT)) ++ return; ++ ++ if (!kctx_as_enabled(kctx)) ++ return; ++ ++ list_for_each_entry(group, &kctx->csf.sched.runnable_groups[priority], ++ link) { ++ if (WARN_ON(!list_empty(&group->link_to_schedule))) ++ /* This would be a bug */ ++ list_del_init(&group->link_to_schedule); ++ ++ if (unlikely(group->faulted)) ++ continue; ++ ++ if (queue_group_idle_locked(group)) { ++ list_add_tail(&group->link_to_schedule, ++ &scheduler->idle_groups_to_schedule); ++ continue; ++ } ++ ++ if (!scheduler->ngrp_to_schedule) { ++ /* keep the top csg's origin */ ++ scheduler->top_ctx = kctx; ++ scheduler->top_grp = group; ++ } ++ ++ list_add_tail(&group->link_to_schedule, ++ &scheduler->groups_to_schedule); ++ group->prepared_seq_num = scheduler->ngrp_to_schedule++; ++ ++ kctx->csf.sched.ngrp_to_schedule++; ++ count_active_address_space(kbdev, kctx); ++ } ++} ++ ++/** ++ * scheduler_rotate_groups() - Rotate the runnable queue groups to provide ++ * fairness of scheduling within a single ++ * kbase_context. ++ * ++ * Since only kbase_csf_scheduler's top_grp (i.e. the queue group assigned ++ * the highest slot priority) is guaranteed to get the resources that it ++ * needs we only rotate the kbase_context corresponding to it - ++ * kbase_csf_scheduler's top_ctx. ++ * ++ * The priority level chosen for rotation is the one containing the previous ++ * scheduling cycle's kbase_csf_scheduler's top_grp. ++ * ++ * In a 'fresh-slice-cycle' this always corresponds to the highest group ++ * priority in use by kbase_csf_scheduler's top_ctx. That is, it's the priority ++ * level of the previous scheduling cycle's first runnable kbase_context. ++ * ++ * We choose this priority level because when higher priority work is ++ * scheduled, we should always cause the scheduler to run and do a scan. The ++ * scan always enumerates the highest priority work first (whether that be ++ * based on process priority or group priority), and thus ++ * kbase_csf_scheduler's top_grp will point to the first of those high priority ++ * groups, which necessarily must be the highest priority group in ++ * kbase_csf_scheduler's top_ctx. The fresh-slice-cycle will run later and pick ++ * up that group appropriately. ++ * ++ * If kbase_csf_scheduler's top_grp was instead evicted (and thus is NULL), ++ * then no explicit rotation occurs on the next fresh-slice-cycle schedule, but ++ * will set up kbase_csf_scheduler's top_ctx again for the next scheduling ++ * cycle. Implicitly, a rotation had already occurred by removing ++ * the kbase_csf_scheduler's top_grp ++ * ++ * If kbase_csf_scheduler's top_grp became idle and all other groups belonging ++ * to kbase_csf_scheduler's top_grp's priority level in kbase_csf_scheduler's ++ * top_ctx are also idle, then the effect of this will be to rotate idle ++ * groups, which might not actually become resident in the next ++ * scheduling slice. However this is acceptable since a queue group becoming ++ * idle is implicitly a rotation (as above with evicted queue groups), as it ++ * automatically allows a new queue group to take the maximum slot priority ++ * whilst the idle kbase_csf_scheduler's top_grp ends up near the back of ++ * the kbase_csf_scheduler's groups_to_schedule list. In this example, it will ++ * be for a group in the next lowest priority level or in absence of those the ++ * next kbase_context's queue groups. ++ * ++ * @kbdev: Pointer to the GPU device. ++ */ ++static void scheduler_rotate_groups(struct kbase_device *kbdev) ++{ ++ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; ++ struct kbase_context *const top_ctx = scheduler->top_ctx; ++ struct kbase_queue_group *const top_grp = scheduler->top_grp; ++ ++ lockdep_assert_held(&scheduler->lock); ++ if (top_ctx && top_grp) { ++ struct list_head *list = ++ &top_ctx->csf.sched.runnable_groups[top_grp->priority]; ++ ++ WARN_ON(top_grp->kctx != top_ctx); ++ if (!WARN_ON(list_empty(list))) { ++ list_move_tail(&top_grp->link, list); ++ dev_dbg(kbdev->dev, ++ "groups rotated for a context, num_runnable_groups: %u\n", ++ scheduler->top_ctx->csf.sched.num_runnable_grps); ++ } ++ } ++} ++ ++static void scheduler_rotate_ctxs(struct kbase_device *kbdev) ++{ ++ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; ++ struct list_head *list = &scheduler->runnable_kctxs; ++ ++ lockdep_assert_held(&scheduler->lock); ++ if (scheduler->top_ctx) { ++ if (!WARN_ON(list_empty(list))) { ++ struct kbase_context *pos; ++ bool found = false; ++ ++ /* Locate the ctx on the list */ ++ list_for_each_entry(pos, list, csf.link) { ++ if (scheduler->top_ctx == pos) { ++ found = true; ++ break; ++ } ++ } ++ ++ if (!WARN_ON(!found)) { ++ list_move_tail(&pos->csf.link, list); ++ dev_dbg(kbdev->dev, "contexts rotated\n"); ++ } ++ } ++ } ++} ++ ++/** ++ * scheduler_update_idle_slots_status() - Get the status update for the command ++ * stream group slots for which the IDLE notification was ++ * received previously. ++ * ++ * This function sends a CSG status update request for all the command stream ++ * group slots present in the bitmap scheduler->csg_slots_idle_mask and wait ++ * for the request to complete. ++ * The bits set in the scheduler->csg_slots_idle_mask bitmap are cleared by ++ * this function. ++ * ++ * @kbdev: Pointer to the GPU device. ++ * @csg_bitmap: Bitmap of the command stream group slots for which ++ * the status update request completed successfully. ++ * @failed_csg_bitmap: Bitmap of the command stream group slots for which ++ * the status update request timedout. ++ */ ++static void scheduler_update_idle_slots_status(struct kbase_device *kbdev, ++ unsigned long *csg_bitmap, unsigned long *failed_csg_bitmap) ++{ ++ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; ++ const u32 num_groups = kbdev->csf.global_iface.group_num; ++ struct kbase_csf_global_iface *const global_iface = ++ &kbdev->csf.global_iface; ++ unsigned long flags, i; ++ ++ lockdep_assert_held(&scheduler->lock); ++ ++ spin_lock_irqsave(&scheduler->interrupt_lock, flags); ++ for_each_set_bit(i, scheduler->csg_slots_idle_mask, num_groups) { ++ struct kbase_csf_csg_slot *csg_slot = &scheduler->csg_slots[i]; ++ struct kbase_queue_group *group = csg_slot->resident_group; ++ struct kbase_csf_cmd_stream_group_info *const ginfo = ++ &global_iface->groups[i]; ++ u32 csg_req; ++ ++ clear_bit(i, scheduler->csg_slots_idle_mask); ++ ++ if (WARN_ON(!group)) ++ continue; ++ ++ csg_req = kbase_csf_firmware_csg_output(ginfo, CSG_ACK); ++ csg_req ^= CSG_REQ_STATUS_UPDATE_MASK; ++ kbase_csf_firmware_csg_input_mask(ginfo, CSG_REQ, csg_req, ++ CSG_REQ_STATUS_UPDATE_MASK); ++ ++ set_bit(i, csg_bitmap); ++ } ++ spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); ++ ++ /* The groups are aggregated into a single kernel doorbell request */ ++ if (!bitmap_empty(csg_bitmap, num_groups)) { ++ long wt = ++ kbase_csf_timeout_in_jiffies(CSF_STATE_WAIT_TIMEOUT_MS); ++ u32 db_slots = (u32)csg_bitmap[0]; ++ ++ kbase_csf_ring_csg_slots_doorbell(kbdev, db_slots); ++ ++ if (wait_csg_slots_handshake_ack(kbdev, ++ CSG_REQ_STATUS_UPDATE_MASK, csg_bitmap, wt)) { ++ dev_warn(kbdev->dev, "Timeout, treat groups as not idle: slot mask=0x%lx", ++ csg_bitmap[0]); ++ ++ /* Store the bitmap of timed out slots */ ++ bitmap_copy(failed_csg_bitmap, csg_bitmap, num_groups); ++ csg_bitmap[0] = ~csg_bitmap[0] & db_slots; ++ } else { ++ csg_bitmap[0] = db_slots; ++ } ++ } ++} ++ ++/** ++ * scheduler_handle_idle_slots() - Update the idle status of queue groups ++ * resident on command stream group slots for which the ++ * IDLE notification was received previously. ++ * ++ * This function is called at the start of scheduling tick/tock to reconfirm ++ * the idle status of queue groups resident on command sream group slots for ++ * which idle notification was received previously, i.e. all the CSG slots ++ * present in the bitmap scheduler->csg_slots_idle_mask. ++ * The confirmation is done by sending the CSG status update request to the ++ * firmware. The idleness of a CSG is determined by looping over all the ++ * bound command streams and checking their respective CS_STATUS_WAIT register ++ * as well as the insert and extract offset. ++ * The run state of the groups resident on still idle CSG slots is changed to ++ * KBASE_CSF_GROUP_IDLE and the bitmap scheduler->csg_slots_idle_mask is ++ * updated accordingly. ++ * The bits corresponding to slots for which the status update request timedout ++ * remain set in scheduler->csg_slots_idle_mask. ++ * ++ * @kbdev: Pointer to the GPU device. ++ */ ++static void scheduler_handle_idle_slots(struct kbase_device *kbdev) ++{ ++ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; ++ u32 num_groups = kbdev->csf.global_iface.group_num; ++ unsigned long flags, i; ++ DECLARE_BITMAP(csg_bitmap, MAX_SUPPORTED_CSGS) = { 0 }; ++ DECLARE_BITMAP(failed_csg_bitmap, MAX_SUPPORTED_CSGS) = { 0 }; ++ ++ lockdep_assert_held(&scheduler->lock); ++ ++ scheduler_update_idle_slots_status(kbdev, csg_bitmap, ++ failed_csg_bitmap); ++ ++ spin_lock_irqsave(&scheduler->interrupt_lock, flags); ++ for_each_set_bit(i, csg_bitmap, num_groups) { ++ struct kbase_csf_csg_slot *csg_slot = &scheduler->csg_slots[i]; ++ struct kbase_queue_group *group = csg_slot->resident_group; ++ ++ if (WARN_ON(atomic_read(&csg_slot->state) != CSG_SLOT_RUNNING)) ++ continue; ++ if (WARN_ON(!group)) ++ continue; ++ if (WARN_ON(group->run_state != KBASE_CSF_GROUP_RUNNABLE)) ++ continue; ++ if (WARN_ON(group->priority >= BASE_QUEUE_GROUP_PRIORITY_COUNT)) ++ continue; ++ ++ if (group_on_slot_is_idle(kbdev, group, i)) { ++ group->run_state = KBASE_CSF_GROUP_IDLE; ++ set_bit(i, scheduler->csg_slots_idle_mask); ++ } ++ } ++ ++ bitmap_or(scheduler->csg_slots_idle_mask, ++ scheduler->csg_slots_idle_mask, ++ failed_csg_bitmap, num_groups); ++ spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); ++} ++ ++static void scheduler_scan_idle_groups(struct kbase_device *kbdev) ++{ ++ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; ++ struct kbase_queue_group *group, *n; ++ ++ list_for_each_entry_safe(group, n, &scheduler->idle_groups_to_schedule, ++ link_to_schedule) { ++ ++ WARN_ON(!queue_group_idle_locked(group)); ++ ++ if (!scheduler->ngrp_to_schedule) { ++ /* keep the top csg's origin */ ++ scheduler->top_ctx = group->kctx; ++ scheduler->top_grp = group; ++ } ++ ++ group->prepared_seq_num = scheduler->ngrp_to_schedule++; ++ list_move_tail(&group->link_to_schedule, ++ &scheduler->groups_to_schedule); ++ ++ group->kctx->csf.sched.ngrp_to_schedule++; ++ count_active_address_space(kbdev, group->kctx); ++ } ++} ++ ++static void scheduler_rotate(struct kbase_device *kbdev) ++{ ++ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; ++ ++ lockdep_assert_held(&scheduler->lock); ++ ++ /* Dealing with rotation */ ++ scheduler_rotate_groups(kbdev); ++ scheduler_rotate_ctxs(kbdev); ++} ++ ++static struct kbase_queue_group *get_tock_top_group( ++ struct kbase_csf_scheduler *const scheduler) ++{ ++ struct kbase_context *kctx; ++ int i; ++ ++ lockdep_assert_held(&scheduler->lock); ++ for (i = 0; i < BASE_QUEUE_GROUP_PRIORITY_COUNT; ++i) { ++ list_for_each_entry(kctx, ++ &scheduler->runnable_kctxs, csf.link) { ++ struct kbase_queue_group *group; ++ ++ list_for_each_entry(group, ++ &kctx->csf.sched.runnable_groups[i], ++ link) { ++ if (queue_group_idle_locked(group)) ++ continue; ++ ++ return group; ++ } ++ } ++ } ++ ++ return NULL; ++} ++ ++static int suspend_active_groups_on_powerdown(struct kbase_device *kbdev, ++ bool is_suspend) ++{ ++ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; ++ DECLARE_BITMAP(slot_mask, MAX_SUPPORTED_CSGS) = { 0 }; ++ ++ int ret = suspend_active_queue_groups(kbdev, slot_mask); ++ ++ if (ret) { ++ /* The suspend of CSGs failed, trigger the GPU reset and wait ++ * for it to complete to be in a deterministic state. ++ */ ++ dev_warn(kbdev->dev, "Timed out waiting for CSG slots to suspend on power down, slot_mask: 0x%*pb\n", ++ kbdev->csf.global_iface.group_num, slot_mask); ++ ++ if (kbase_prepare_to_reset_gpu(kbdev)) ++ kbase_reset_gpu(kbdev); ++ ++ if (is_suspend) { ++ mutex_unlock(&scheduler->lock); ++ kbase_reset_gpu_wait(kbdev); ++ mutex_lock(&scheduler->lock); ++ } ++ return -1; ++ } ++ ++ /* Check if the groups became active whilst the suspend was ongoing, ++ * but only for the case where the system suspend is not in progress ++ */ ++ if (!is_suspend && atomic_read(&scheduler->non_idle_suspended_grps)) ++ return -1; ++ ++ return 0; ++} ++ ++static void gpu_idle_worker(struct work_struct *work) ++{ ++ struct kbase_device *kbdev = container_of( ++ work, struct kbase_device, csf.scheduler.gpu_idle_work.work); ++ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; ++ ++ mutex_lock(&scheduler->lock); ++ ++ if (!scheduler->total_runnable_grps) { ++ if (scheduler->state != SCHED_SUSPENDED) { ++ scheduler_suspend(kbdev); ++ dev_info(kbdev->dev, "Scheduler now suspended"); ++ } ++ } else { ++ dev_dbg(kbdev->dev, "Scheduler couldn't be suspended"); ++ } ++ ++ mutex_unlock(&scheduler->lock); ++} ++ ++static int scheduler_prepare(struct kbase_device *kbdev) ++{ ++ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; ++ int i; ++ ++ lockdep_assert_held(&scheduler->lock); ++ ++ /* Empty the groups_to_schedule */ ++ while (!list_empty(&scheduler->groups_to_schedule)) { ++ struct kbase_queue_group *grp = ++ list_first_entry(&scheduler->groups_to_schedule, ++ struct kbase_queue_group, ++ link_to_schedule); ++ ++ remove_scheduled_group(kbdev, grp); ++ } ++ ++ /* Pre-scan init scheduler fields */ ++ if (WARN_ON(scheduler->ngrp_to_schedule != 0)) ++ scheduler->ngrp_to_schedule = 0; ++ scheduler->top_ctx = NULL; ++ scheduler->top_grp = NULL; ++ scheduler->head_slot_priority = MAX_CSG_SLOT_PRIORITY; ++ WARN_ON(!list_empty(&scheduler->idle_groups_to_schedule)); ++ scheduler->num_active_address_spaces = 0; ++ scheduler->num_csg_slots_for_tick = 0; ++ bitmap_zero(scheduler->csg_slots_prio_update, MAX_SUPPORTED_CSGS); ++ ++ /* Scan out to run groups */ ++ for (i = 0; i < BASE_QUEUE_GROUP_PRIORITY_COUNT; ++i) { ++ struct kbase_context *kctx; ++ ++ list_for_each_entry(kctx, &scheduler->runnable_kctxs, csf.link) ++ scheduler_ctx_scan_groups(kbdev, kctx, i); ++ } ++ ++ scheduler_scan_idle_groups(kbdev); ++ KBASE_KTRACE_ADD_CSF_GRP(kbdev, SCHEDULER_TOP_GRP, scheduler->top_grp, ++ scheduler->num_active_address_spaces | ++ (((u64)scheduler->ngrp_to_schedule) << 32)); ++ set_max_csg_slots(kbdev); ++ dev_dbg(kbdev->dev, "prepared groups length: %u, num_active_address_spaces: %u\n", ++ scheduler->ngrp_to_schedule, scheduler->num_active_address_spaces); ++ return 0; ++} ++ ++static void scheduler_wait_protm_quit(struct kbase_device *kbdev) ++{ ++ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; ++ long wt = kbase_csf_timeout_in_jiffies(CSF_STATE_WAIT_TIMEOUT_MS); ++ long remaining; ++ ++ lockdep_assert_held(&scheduler->lock); ++ ++ remaining = wait_event_timeout(kbdev->csf.event_wait, ++ !kbase_csf_scheduler_protected_mode_in_use(kbdev), wt); ++ ++ if (!remaining) ++ dev_warn(kbdev->dev, "Timeout, protm_quit wait skipped"); ++} ++ ++static void schedule_actions(struct kbase_device *kbdev) ++{ ++ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; ++ unsigned long flags; ++ struct kbase_queue_group *protm_grp; ++ int ret; ++ ++ lockdep_assert_held(&scheduler->lock); ++ ++ ret = kbase_pm_wait_for_desired_state(kbdev); ++ if (ret) { ++ dev_err(kbdev->dev, "Wait for MCU power on failed"); ++ return; ++ } ++ ++ scheduler_handle_idle_slots(kbdev); ++ scheduler_prepare(kbdev); ++ spin_lock_irqsave(&scheduler->interrupt_lock, flags); ++ protm_grp = scheduler->active_protm_grp; ++ ++ /* Avoid update if the top-group remains unchanged and in protected ++ * mode. For the said case, all the slots update is effectively ++ * competing against the active protected mode group (typically the ++ * top-group). If we update other slots, even on leaving the ++ * top-group slot untouched, the firmware would exit the protected mode ++ * for interacting with the host-driver. After it, as the top-group ++ * would again raise the request for entering protected mode, we would ++ * be actively doing the switching over twice without progressing the ++ * queue jobs. ++ */ ++ if (protm_grp && scheduler->top_grp == protm_grp) { ++ dev_dbg(kbdev->dev, "Scheduler keep protm exec: group-%d", ++ protm_grp->handle); ++ } else if (scheduler->top_grp) { ++ if (protm_grp) ++ dev_dbg(kbdev->dev, "Scheduler drop protm exec: group-%d", ++ protm_grp->handle); ++ ++ if (!bitmap_empty(scheduler->top_grp->protm_pending_bitmap, ++ kbdev->csf.global_iface.groups[0].stream_num)) { ++ dev_dbg(kbdev->dev, "Scheduler prepare protm exec: group-%d of context %d_%d", ++ scheduler->top_grp->handle, ++ scheduler->top_grp->kctx->tgid, ++ scheduler->top_grp->kctx->id); ++ ++ /* Due to GPUCORE-24491 only the top-group is allowed ++ * to be on slot and all other on slot groups have to ++ * be suspended before entering protected mode. ++ * This would change in GPUCORE-24492. ++ */ ++ scheduler->num_csg_slots_for_tick = 1; ++ } ++ ++ spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); ++ ++ scheduler_apply(kbdev); ++ /* Scheduler is dropping the exec of the previous protm_grp, ++ * Until the protm quit completes, the GPU is effectively ++ * locked in the secure mode. ++ */ ++ if (protm_grp) ++ scheduler_wait_protm_quit(kbdev); ++ ++ wait_csg_slots_start(kbdev); ++ wait_csg_slots_finish_prio_update(kbdev); ++ ++ if (scheduler->num_csg_slots_for_tick == 1) { ++ scheduler_group_check_protm_enter(kbdev, ++ scheduler->top_grp); ++ } ++ ++ return; ++ } ++ ++ spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); ++ return; ++} ++ ++static void schedule_on_tock(struct work_struct *work) ++{ ++ struct kbase_device *kbdev = container_of(work, struct kbase_device, ++ csf.scheduler.tock_work.work); ++ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; ++ ++ mutex_lock(&scheduler->lock); ++ ++ if (kbase_reset_gpu_is_active(kbdev) || ++ (scheduler->state == SCHED_SUSPENDED)) { ++ mutex_unlock(&scheduler->lock); ++ return; ++ } ++ ++ WARN_ON(!(scheduler->state == SCHED_INACTIVE)); ++ scheduler->state = SCHED_BUSY; ++ ++ /* Undertaking schedule action steps */ ++ KBASE_KTRACE_ADD(kbdev, SCHEDULER_TOCK, NULL, 0u); ++ schedule_actions(kbdev); ++ ++ /* Record time information */ ++ scheduler->last_schedule = jiffies; ++ ++ /* Tock is serviced */ ++ scheduler->tock_pending_request = false; ++ ++ scheduler->state = SCHED_INACTIVE; ++ mutex_unlock(&scheduler->lock); ++ ++ dev_dbg(kbdev->dev, ++ "Waking up for event after schedule-on-tock completes."); ++ wake_up_all(&kbdev->csf.event_wait); ++} ++ ++static void schedule_on_tick(struct work_struct *work) ++{ ++ struct kbase_device *kbdev = container_of(work, struct kbase_device, ++ csf.scheduler.tick_work.work); ++ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; ++ ++ mutex_lock(&scheduler->lock); ++ ++ if (kbase_reset_gpu_is_active(kbdev) || ++ (scheduler->state == SCHED_SUSPENDED)) { ++ mutex_unlock(&scheduler->lock); ++ return; ++ } ++ ++ scheduler->state = SCHED_BUSY; ++ ++ /* Do scheduling stuff */ ++ scheduler_rotate(kbdev); ++ ++ /* Undertaking schedule action steps */ ++ KBASE_KTRACE_ADD(kbdev, SCHEDULER_TICK, NULL, 0u); ++ schedule_actions(kbdev); ++ ++ /* Record time information */ ++ scheduler->last_schedule = jiffies; ++ ++ /* Kicking next scheduling if needed */ ++ if (likely(scheduler_timer_is_enabled_nolock(kbdev)) && ++ (scheduler->total_runnable_grps > 0)) { ++ mod_delayed_work(scheduler->wq, &scheduler->tick_work, ++ CSF_SCHEDULER_TIME_TICK_JIFFIES); ++ dev_dbg(kbdev->dev, "scheduling for next tick, num_runnable_groups:%u\n", ++ scheduler->total_runnable_grps); ++ } ++ ++ scheduler->state = SCHED_INACTIVE; ++ mutex_unlock(&scheduler->lock); ++ ++ dev_dbg(kbdev->dev, "Waking up for event after schedule-on-tick completes."); ++ wake_up_all(&kbdev->csf.event_wait); ++} ++ ++int wait_csg_slots_suspend(struct kbase_device *kbdev, ++ const unsigned long *slot_mask, ++ unsigned int timeout_ms) ++{ ++ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; ++ long remaining = kbase_csf_timeout_in_jiffies(timeout_ms); ++ u32 num_groups = kbdev->csf.global_iface.group_num; ++ int err = 0; ++ DECLARE_BITMAP(slot_mask_local, MAX_SUPPORTED_CSGS); ++ ++ lockdep_assert_held(&scheduler->lock); ++ ++ bitmap_copy(slot_mask_local, slot_mask, MAX_SUPPORTED_CSGS); ++ ++ while (!bitmap_empty(slot_mask_local, MAX_SUPPORTED_CSGS) ++ && remaining) { ++ DECLARE_BITMAP(changed, MAX_SUPPORTED_CSGS); ++ ++ bitmap_copy(changed, slot_mask_local, MAX_SUPPORTED_CSGS); ++ ++ remaining = wait_event_timeout(kbdev->csf.event_wait, ++ slots_state_changed(kbdev, changed, ++ csg_slot_stopped_locked), ++ remaining); ++ ++ if (remaining) { ++ u32 i; ++ ++ for_each_set_bit(i, changed, num_groups) { ++ struct kbase_queue_group *group; ++ ++ if (WARN_ON(!csg_slot_stopped_locked(kbdev, (s8)i))) ++ continue; ++ ++ /* The on slot csg is now stopped */ ++ clear_bit(i, slot_mask_local); ++ ++ group = scheduler->csg_slots[i].resident_group; ++ if (likely(group)) { ++ /* Only do save/cleanup if the ++ * group is not terminated during ++ * the sleep. ++ */ ++ save_csg_slot(group); ++ if (cleanup_csg_slot(group)) ++ sched_evict_group(group, true); ++ } ++ } ++ } else { ++ dev_warn(kbdev->dev, "Timed out waiting for CSG slots to suspend, slot_mask: 0x%*pb\n", ++ num_groups, slot_mask_local); ++ err = -ETIMEDOUT; ++ } ++ } ++ ++ return err; ++} ++ ++static int suspend_active_queue_groups(struct kbase_device *kbdev, ++ unsigned long *slot_mask) ++{ ++ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; ++ u32 num_groups = kbdev->csf.global_iface.group_num; ++ u32 slot_num; ++ int ret; ++ ++ lockdep_assert_held(&scheduler->lock); ++ ++ for (slot_num = 0; slot_num < num_groups; slot_num++) { ++ struct kbase_queue_group *group = ++ scheduler->csg_slots[slot_num].resident_group; ++ ++ if (group) { ++ suspend_queue_group(group); ++ set_bit(slot_num, slot_mask); ++ } ++ } ++ ++ ret = wait_csg_slots_suspend(kbdev, slot_mask, ++ CSG_SUSPEND_ON_RESET_WAIT_TIMEOUT_MS); ++ return ret; ++} ++ ++static int suspend_active_queue_groups_on_reset(struct kbase_device *kbdev) ++{ ++ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; ++ DECLARE_BITMAP(slot_mask, MAX_SUPPORTED_CSGS) = { 0 }; ++ int ret; ++ ++ mutex_lock(&scheduler->lock); ++ ++ ret = suspend_active_queue_groups(kbdev, slot_mask); ++ if (ret) { ++ dev_warn(kbdev->dev, "Timed out waiting for CSG slots to suspend before reset, slot_mask: 0x%*pb\n", ++ kbdev->csf.global_iface.group_num, slot_mask); ++ } ++ ++ if (!bitmap_empty(slot_mask, MAX_SUPPORTED_CSGS)) { ++ int ret2; ++ ++ /* Need to flush the GPU cache to ensure suspend buffer ++ * contents are not lost on reset of GPU. ++ * Do this even if suspend operation had timedout for some of ++ * the CSG slots. ++ */ ++ kbase_gpu_start_cache_clean(kbdev); ++ ret2 = kbase_gpu_wait_cache_clean_timeout(kbdev, ++ DEFAULT_RESET_TIMEOUT_MS); ++ if (ret2) { ++ dev_warn(kbdev->dev, "Timed out waiting for cache clean to complete before reset"); ++ if (!ret) ++ ret = ret2; ++ } ++ } ++ ++ mutex_unlock(&scheduler->lock); ++ ++ return ret; ++} ++ ++static void scheduler_inner_reset(struct kbase_device *kbdev) ++{ ++ u32 const num_groups = kbdev->csf.global_iface.group_num; ++ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; ++ unsigned long flags; ++ ++ WARN_ON(csgs_active(kbdev)); ++ ++ /* Cancel any potential queued delayed work(s) */ ++ cancel_delayed_work_sync(&scheduler->tick_work); ++ cancel_delayed_work_sync(&scheduler->tock_work); ++ cancel_delayed_work_sync(&scheduler->ping_work); ++ ++ mutex_lock(&scheduler->lock); ++ ++ spin_lock_irqsave(&scheduler->interrupt_lock, flags); ++ bitmap_fill(scheduler->csgs_events_enable_mask, MAX_SUPPORTED_CSGS); ++ scheduler->active_protm_grp = NULL; ++ memset(kbdev->csf.scheduler.csg_slots, 0, ++ num_groups * sizeof(struct kbase_csf_csg_slot)); ++ bitmap_zero(kbdev->csf.scheduler.csg_inuse_bitmap, num_groups); ++ spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); ++ ++ scheduler->top_ctx = NULL; ++ scheduler->top_grp = NULL; ++ ++ KBASE_KTRACE_ADD_CSF_GRP(kbdev, SCHEDULER_TOP_GRP, scheduler->top_grp, ++ scheduler->num_active_address_spaces | ++ (((u64)scheduler->total_runnable_grps) << 32)); ++ ++ mutex_unlock(&scheduler->lock); ++} ++ ++void kbase_csf_scheduler_reset(struct kbase_device *kbdev) ++{ ++ struct kbase_context *kctx; ++ ++ WARN_ON(!kbase_reset_gpu_is_active(kbdev)); ++ ++ KBASE_KTRACE_ADD(kbdev, SCHEDULER_RESET, NULL, 0u); ++ if (!kbase_csf_scheduler_protected_mode_in_use(kbdev) && ++ !suspend_active_queue_groups_on_reset(kbdev)) { ++ /* As all groups have been successfully evicted from the CSG ++ * slots, clear out thee scheduler data fields and return ++ */ ++ scheduler_inner_reset(kbdev); ++ return; ++ } ++ ++ mutex_lock(&kbdev->kctx_list_lock); ++ ++ /* The loop to iterate over the kbase contexts is present due to lock ++ * ordering issue between kctx->csf.lock & kbdev->csf.scheduler.lock. ++ * CSF ioctls first take kctx->csf.lock which is context-specific and ++ * then take kbdev->csf.scheduler.lock for global actions like assigning ++ * a CSG slot. ++ * If the lock ordering constraint was not there then could have ++ * directly looped over the active queue groups. ++ */ ++ list_for_each_entry(kctx, &kbdev->kctx_list, kctx_list_link) { ++ /* Firmware reload would reinitialize the CSG & CS interface IO ++ * pages, so just need to internally mark the currently active ++ * queue groups as terminated (similar to the unexpected OoM ++ * event case). ++ * No further work can now get executed for the active groups ++ * (new groups would have to be created to execute work) and ++ * in near future Clients would be duly informed of this ++ * reset. The resources (like User IO pages, GPU queue memory) ++ * allocated for the associated queues would be freed when the ++ * Clients do the teardown when they become aware of the reset. ++ */ ++ kbase_csf_active_queue_groups_reset(kbdev, kctx); ++ } ++ ++ mutex_unlock(&kbdev->kctx_list_lock); ++ ++ /* After queue groups reset, the scheduler data fields clear out */ ++ scheduler_inner_reset(kbdev); ++} ++ ++static void firmware_aliveness_monitor(struct work_struct *work) ++{ ++ struct kbase_device *kbdev = container_of(work, struct kbase_device, ++ csf.scheduler.ping_work.work); ++ int err; ++ ++ /* Get the scheduler mutex to ensure that reset will not change while ++ * this function is being executed as otherwise calling kbase_reset_gpu ++ * when reset is already occurring is a programming error. ++ */ ++ mutex_lock(&kbdev->csf.scheduler.lock); ++ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++ if (fw_debug) { ++ /* ping requests cause distraction in firmware debugging */ ++ goto exit; ++ } ++#endif ++ ++ if (kbdev->csf.scheduler.state == SCHED_SUSPENDED) ++ goto exit; ++ ++ if (kbase_reset_gpu_is_active(kbdev)) ++ goto exit; ++ ++ if (get_nr_active_csgs(kbdev) != 1) ++ goto exit; ++ ++ if (kbase_csf_scheduler_protected_mode_in_use(kbdev)) ++ goto exit; ++ ++ if (kbase_pm_context_active_handle_suspend(kbdev, ++ KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE)) { ++ /* Suspend pending - no real need to ping */ ++ goto exit; ++ } ++ ++ kbase_pm_wait_for_desired_state(kbdev); ++ ++ err = kbase_csf_firmware_ping(kbdev); ++ ++ if (err) { ++ if (kbase_prepare_to_reset_gpu(kbdev)) ++ kbase_reset_gpu(kbdev); ++ } else if (get_nr_active_csgs(kbdev) == 1) { ++ queue_delayed_work(system_long_wq, ++ &kbdev->csf.scheduler.ping_work, ++ msecs_to_jiffies(FIRMWARE_PING_INTERVAL_MS)); ++ } ++ ++ kbase_pm_context_idle(kbdev); ++exit: ++ mutex_unlock(&kbdev->csf.scheduler.lock); ++} ++ ++int kbase_csf_scheduler_group_copy_suspend_buf(struct kbase_queue_group *group, ++ struct kbase_suspend_copy_buffer *sus_buf) ++{ ++ struct kbase_context *const kctx = group->kctx; ++ struct kbase_device *const kbdev = kctx->kbdev; ++ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; ++ int err; ++ ++ lockdep_assert_held(&kctx->csf.lock); ++ mutex_lock(&scheduler->lock); ++ ++ err = wait_gpu_reset(kbdev); ++ if (err) { ++ dev_warn(kbdev->dev, "Error while waiting for the GPU reset to complete when suspending group %d on slot %d", ++ group->handle, group->csg_nr); ++ goto exit; ++ } ++ ++ if (kbasep_csf_scheduler_group_is_on_slot_locked(group)) { ++ DECLARE_BITMAP(slot_mask, MAX_SUPPORTED_CSGS) = {0}; ++ ++ set_bit(kbase_csf_scheduler_group_get_slot(group), slot_mask); ++ ++ if (!WARN_ON(scheduler->state == SCHED_SUSPENDED)) ++ suspend_queue_group(group); ++ err = wait_csg_slots_suspend(kbdev, slot_mask, ++ CSF_STATE_WAIT_TIMEOUT_MS); ++ if (err) { ++ dev_warn(kbdev->dev, "Timed out waiting for the group %d to suspend on slot %d", ++ group->handle, group->csg_nr); ++ goto exit; ++ } ++ } ++ ++ if (queue_group_suspended_locked(group)) { ++ unsigned int target_page_nr = 0, i = 0; ++ u64 offset = sus_buf->offset; ++ size_t to_copy = sus_buf->size; ++ ++ if (scheduler->state != SCHED_SUSPENDED) { ++ /* Similar to the case of HW counters, need to flush ++ * the GPU cache before reading from the suspend buffer ++ * pages as they are mapped and cached on GPU side. ++ */ ++ kbase_gpu_start_cache_clean(kbdev); ++ kbase_gpu_wait_cache_clean(kbdev); ++ } else { ++ /* Make sure power down transitions have completed, ++ * i.e. L2 has been powered off as that would ensure ++ * its contents are flushed to memory. ++ * This is needed as Scheduler doesn't wait for the ++ * power down to finish. ++ */ ++ kbase_pm_wait_for_desired_state(kbdev); ++ } ++ ++ for (i = 0; i < PFN_UP(sus_buf->size) && ++ target_page_nr < sus_buf->nr_pages; i++) { ++ struct page *pg = ++ as_page(group->normal_suspend_buf.phy[i]); ++ void *sus_page = kmap(pg); ++ ++ if (sus_page) { ++ kbase_sync_single_for_cpu(kbdev, ++ kbase_dma_addr(pg), ++ PAGE_SIZE, DMA_BIDIRECTIONAL); ++ ++ err = kbase_mem_copy_to_pinned_user_pages( ++ sus_buf->pages, sus_page, ++ &to_copy, sus_buf->nr_pages, ++ &target_page_nr, offset); ++ kunmap(pg); ++ if (err) ++ break; ++ } else { ++ err = -ENOMEM; ++ break; ++ } ++ } ++ schedule_in_cycle(group, false); ++ } else { ++ /* If addr-space fault, the group may have been evicted */ ++ err = -EIO; ++ } ++ ++exit: ++ mutex_unlock(&scheduler->lock); ++ return err; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_csf_scheduler_group_copy_suspend_buf); ++ ++/** ++ * group_sync_updated() - Evaluate sync wait condition of all blocked command ++ * queues of the group. ++ * ++ * @group: Pointer to the command queue group that has blocked command queue(s) ++ * bound to it. ++ * ++ * Return: true if sync wait condition is satisfied for at least one blocked ++ * queue of the group. ++ */ ++static bool group_sync_updated(struct kbase_queue_group *group) ++{ ++ bool updated = false; ++ int stream; ++ ++ WARN_ON(group->run_state != KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC); ++ ++ for (stream = 0; stream < MAX_SUPPORTED_STREAMS_PER_GROUP; ++stream) { ++ struct kbase_queue *const queue = group->bound_queues[stream]; ++ ++ /* To check the necessity of sync-wait evaluation, ++ * we rely on the cached 'status_wait' instead of reading it ++ * directly from shared memory as the CSG has been already ++ * evicted from the CSG slot, thus this CSG doesn't have ++ * valid information in the shared memory. ++ */ ++ if (queue && queue->enabled && ++ CS_STATUS_WAIT_SYNC_WAIT_GET(queue->status_wait)) ++ if (evaluate_sync_update(queue)) { ++ updated = true; ++ queue->status_wait = 0; ++ } ++ } ++ ++ return updated; ++} ++ ++/** ++ * scheduler_get_protm_enter_async_group() - Check if the GPU queue group ++ * can be now allowed to execute in protected mode. ++ * ++ * @kbdev: Pointer to the GPU device. ++ * @group: Pointer to the GPU queue group. ++ * ++ * This function is called outside the scheduling tick/tock to determine ++ * if the given GPU queue group can now execute in protected mode or not. ++ * If the group pointer passed is NULL then the evaluation is done for the ++ * scheduler->top_grp (or the second top-group). ++ * ++ * It returns the same group pointer, that was passed as an argument, if that ++ * group matches the scheduler->top_grp and has pending protected region ++ * requests otherwise NULL is returned. ++ * ++ * If the group pointer passed is NULL then the pointer to scheduler->top_grp ++ * is returned if that has pending protected region requests otherwise NULL is ++ * returned. ++ * ++ * If the scheduler->top_grp is NULL, which may happen when the top-group is ++ * evicted during the tick, the second top-group (as a replacement of the ++ * top-group) is used for the match check and also for the evaluation of ++ * pending protected region requests if the group pointer passed is NULL. ++ * ++ * Return: the pointer to queue group that can currently execute in protected ++ * mode or NULL. ++ */ ++static struct kbase_queue_group *scheduler_get_protm_enter_async_group( ++ struct kbase_device *const kbdev, ++ struct kbase_queue_group *const group) ++{ ++ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; ++ struct kbase_queue_group *match_grp, *input_grp; ++ ++ lockdep_assert_held(&scheduler->lock); ++ ++ if (scheduler->state != SCHED_INACTIVE) ++ return NULL; ++ ++ match_grp = scheduler->top_grp ? scheduler->top_grp : ++ get_tock_top_group(scheduler); ++ input_grp = group ? group : match_grp; ++ ++ if (input_grp && (input_grp == match_grp)) { ++ struct kbase_csf_cmd_stream_group_info *ginfo = ++ &kbdev->csf.global_iface.groups[0]; ++ unsigned long *pending = ++ input_grp->protm_pending_bitmap; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&scheduler->interrupt_lock, flags); ++ ++ if (kbase_csf_scheduler_protected_mode_in_use(kbdev) || ++ bitmap_empty(pending, ginfo->stream_num)) ++ input_grp = NULL; ++ ++ spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); ++ } else { ++ input_grp = NULL; ++ } ++ ++ return input_grp; ++} ++ ++void kbase_csf_scheduler_group_protm_enter(struct kbase_queue_group *group) ++{ ++ struct kbase_device *const kbdev = group->kctx->kbdev; ++ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; ++ ++ mutex_lock(&scheduler->lock); ++ ++ /* Check if the group is now eligible for execution in protected mode ++ * and accordingly undertake full scheduling actions as due to ++ * GPUCORE-24491 the on slot groups other than the top group have to ++ * be suspended first before entering protected mode. ++ */ ++ if (scheduler_get_protm_enter_async_group(kbdev, group)) ++ schedule_actions(kbdev); ++ ++ mutex_unlock(&scheduler->lock); ++} ++ ++/** ++ * check_group_sync_update_worker() - Check the sync wait condition for all the ++ * blocked queue groups ++ * ++ * @work: Pointer to the context-specific work item for evaluating the wait ++ * condition for all the queue groups in idle_wait_groups list. ++ * ++ * This function checks the gpu queues of all the groups present in ++ * idle_wait_groups list of a context. If the sync wait condition ++ * for at least one queue bound to the group has been satisfied then ++ * the group is moved to the per context list of runnable groups so ++ * that Scheduler can consider scheduling the group in next tick. ++ */ ++static void check_group_sync_update_worker(struct work_struct *work) ++{ ++ struct kbase_context *const kctx = container_of(work, ++ struct kbase_context, csf.sched.sync_update_work); ++ struct kbase_csf_scheduler *const scheduler = ++ &kctx->kbdev->csf.scheduler; ++ ++ mutex_lock(&scheduler->lock); ++ ++ if (kctx->csf.sched.num_idle_wait_grps != 0) { ++ struct kbase_queue_group *group, *temp; ++ ++ list_for_each_entry_safe(group, temp, ++ &kctx->csf.sched.idle_wait_groups, link) { ++ if (group_sync_updated(group)) { ++ /* Move this group back in to the runnable ++ * groups list of the context. ++ */ ++ update_idle_suspended_group_state(group); ++ KBASE_KTRACE_ADD_CSF_GRP(kctx->kbdev, GROUP_SYNC_UPDATE_DONE, group, 0u); ++ } ++ } ++ } else { ++ WARN_ON(!list_empty(&kctx->csf.sched.idle_wait_groups)); ++ } ++ ++ mutex_unlock(&scheduler->lock); ++} ++ ++static ++enum kbase_csf_event_callback_action check_group_sync_update_cb(void *param) ++{ ++ struct kbase_context *const kctx = param; ++ ++ KBASE_KTRACE_ADD(kctx->kbdev, SYNC_UPDATE_EVENT, kctx, 0u); ++ queue_work(kctx->csf.sched.sync_update_wq, ++ &kctx->csf.sched.sync_update_work); ++ ++ return KBASE_CSF_EVENT_CALLBACK_KEEP; ++} ++ ++int kbase_csf_scheduler_context_init(struct kbase_context *kctx) ++{ ++ int priority; ++ int err; ++ ++ for (priority = 0; priority < BASE_QUEUE_GROUP_PRIORITY_COUNT; ++ ++priority) { ++ INIT_LIST_HEAD(&kctx->csf.sched.runnable_groups[priority]); ++ } ++ ++ kctx->csf.sched.num_runnable_grps = 0; ++ INIT_LIST_HEAD(&kctx->csf.sched.idle_wait_groups); ++ kctx->csf.sched.num_idle_wait_grps = 0; ++ kctx->csf.sched.ngrp_to_schedule = 0; ++ ++ kctx->csf.sched.sync_update_wq = ++ alloc_ordered_workqueue("mali_kbase_csf_sync_update_wq", ++ WQ_HIGHPRI); ++ if (!kctx->csf.sched.sync_update_wq) { ++ dev_err(kctx->kbdev->dev, ++ "Failed to initialize scheduler context workqueue"); ++ return -ENOMEM; ++ } ++ ++ INIT_WORK(&kctx->csf.sched.sync_update_work, ++ check_group_sync_update_worker); ++ ++ err = kbase_csf_event_wait_add(kctx, check_group_sync_update_cb, kctx); ++ ++ if (err) { ++ dev_err(kctx->kbdev->dev, ++ "Failed to register a sync update callback"); ++ destroy_workqueue(kctx->csf.sched.sync_update_wq); ++ } ++ ++ return err; ++} ++ ++void kbase_csf_scheduler_context_term(struct kbase_context *kctx) ++{ ++ kbase_csf_event_wait_remove(kctx, check_group_sync_update_cb, kctx); ++ cancel_work_sync(&kctx->csf.sched.sync_update_work); ++ destroy_workqueue(kctx->csf.sched.sync_update_wq); ++} ++ ++int kbase_csf_scheduler_init(struct kbase_device *kbdev) ++{ ++ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; ++ u32 num_groups = kbdev->csf.global_iface.group_num; ++ ++ bitmap_zero(scheduler->csg_inuse_bitmap, num_groups); ++ bitmap_zero(scheduler->csg_slots_idle_mask, num_groups); ++ ++ scheduler->csg_slots = kcalloc(num_groups, ++ sizeof(*scheduler->csg_slots), GFP_KERNEL); ++ if (!scheduler->csg_slots) { ++ dev_err(kbdev->dev, ++ "Failed to allocate memory for csg slot status array\n"); ++ return -ENOMEM; ++ } ++ ++ scheduler->timer_enabled = true; ++ ++ scheduler->wq = alloc_ordered_workqueue("csf_scheduler_wq", WQ_HIGHPRI); ++ if (!scheduler->wq) { ++ dev_err(kbdev->dev, "Failed to allocate scheduler workqueue\n"); ++ ++ kfree(scheduler->csg_slots); ++ scheduler->csg_slots = NULL; ++ ++ return -ENOMEM; ++ } ++ ++ INIT_DEFERRABLE_WORK(&scheduler->tick_work, schedule_on_tick); ++ INIT_DEFERRABLE_WORK(&scheduler->tock_work, schedule_on_tock); ++ ++ INIT_DEFERRABLE_WORK(&scheduler->ping_work, firmware_aliveness_monitor); ++ BUILD_BUG_ON(GLB_REQ_WAIT_TIMEOUT_MS >= FIRMWARE_PING_INTERVAL_MS); ++ ++ mutex_init(&scheduler->lock); ++ spin_lock_init(&scheduler->interrupt_lock); ++ ++ /* Internal lists */ ++ INIT_LIST_HEAD(&scheduler->runnable_kctxs); ++ INIT_LIST_HEAD(&scheduler->groups_to_schedule); ++ INIT_LIST_HEAD(&scheduler->idle_groups_to_schedule); ++ ++ BUILD_BUG_ON(MAX_SUPPORTED_CSGS > ++ (sizeof(scheduler->csgs_events_enable_mask) * BITS_PER_BYTE)); ++ bitmap_fill(scheduler->csgs_events_enable_mask, MAX_SUPPORTED_CSGS); ++ scheduler->state = SCHED_SUSPENDED; ++ scheduler->pm_active_count = 0; ++ scheduler->ngrp_to_schedule = 0; ++ scheduler->total_runnable_grps = 0; ++ scheduler->top_ctx = NULL; ++ scheduler->top_grp = NULL; ++ scheduler->last_schedule = 0; ++ scheduler->tock_pending_request = false; ++ scheduler->active_protm_grp = NULL; ++ scheduler_doorbell_init(kbdev); ++ ++ INIT_DEFERRABLE_WORK(&scheduler->gpu_idle_work, gpu_idle_worker); ++ atomic_set(&scheduler->non_idle_suspended_grps, 0); ++ ++ return 0; ++} ++ ++void kbase_csf_scheduler_term(struct kbase_device *kbdev) ++{ ++ if (kbdev->csf.scheduler.csg_slots) { ++ WARN_ON(csgs_active(kbdev)); ++ cancel_delayed_work_sync(&kbdev->csf.scheduler.gpu_idle_work); ++ cancel_delayed_work_sync(&kbdev->csf.scheduler.ping_work); ++ destroy_workqueue(kbdev->csf.scheduler.wq); ++ mutex_destroy(&kbdev->csf.scheduler.lock); ++ kfree(kbdev->csf.scheduler.csg_slots); ++ kbdev->csf.scheduler.csg_slots = NULL; ++ } ++} ++ ++/** ++ * scheduler_enable_tick_timer_nolock - Enable the scheduler tick timer. ++ * ++ * @kbdev: Instance of a GPU platform device that implements a command ++ * stream front-end interface. ++ * ++ * This function will restart the scheduler tick so that regular scheduling can ++ * be resumed without any explicit trigger (like kicking of GPU queues). This ++ * is a variant of kbase_csf_scheduler_enable_tick_timer() that assumes the ++ * CSF scheduler lock to already have been held. ++ */ ++static void scheduler_enable_tick_timer_nolock(struct kbase_device *kbdev) ++{ ++ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; ++ ++ lockdep_assert_held(&kbdev->csf.scheduler.lock); ++ ++ if (unlikely(!scheduler_timer_is_enabled_nolock(kbdev))) ++ return; ++ ++ WARN_ON((scheduler->state != SCHED_INACTIVE) && ++ (scheduler->state != SCHED_SUSPENDED)); ++ WARN_ON(delayed_work_pending(&scheduler->tick_work)); ++ ++ if (scheduler->total_runnable_grps > 0) { ++ mod_delayed_work(scheduler->wq, &scheduler->tick_work, 0); ++ dev_dbg(kbdev->dev, "Re-enabling the scheduler timer\n"); ++ } ++} ++ ++void kbase_csf_scheduler_enable_tick_timer(struct kbase_device *kbdev) ++{ ++ mutex_lock(&kbdev->csf.scheduler.lock); ++ scheduler_enable_tick_timer_nolock(kbdev); ++ mutex_unlock(&kbdev->csf.scheduler.lock); ++} ++ ++bool kbase_csf_scheduler_timer_is_enabled(struct kbase_device *kbdev) ++{ ++ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; ++ bool enabled; ++ ++ mutex_lock(&scheduler->lock); ++ enabled = scheduler_timer_is_enabled_nolock(kbdev); ++ mutex_unlock(&scheduler->lock); ++ ++ return enabled; ++} ++ ++void kbase_csf_scheduler_timer_set_enabled(struct kbase_device *kbdev, ++ bool enable) ++{ ++ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; ++ bool currently_enabled; ++ ++ mutex_lock(&scheduler->lock); ++ ++ currently_enabled = scheduler_timer_is_enabled_nolock(kbdev); ++ if (currently_enabled && !enable) { ++ scheduler->timer_enabled = false; ++ ++ cancel_delayed_work(&scheduler->tick_work); ++ cancel_delayed_work(&scheduler->tock_work); ++ } else if (!currently_enabled && enable) { ++ scheduler->timer_enabled = true; ++ ++ scheduler_enable_tick_timer_nolock(kbdev); ++ } ++ ++ mutex_unlock(&scheduler->lock); ++} ++ ++void kbase_csf_scheduler_kick(struct kbase_device *kbdev) ++{ ++ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; ++ ++ mutex_lock(&scheduler->lock); ++ ++ if (unlikely(scheduler_timer_is_enabled_nolock(kbdev))) ++ goto out; ++ ++ if (scheduler->total_runnable_grps > 0) { ++ mod_delayed_work(scheduler->wq, &scheduler->tick_work, 0); ++ dev_dbg(kbdev->dev, "Kicking the scheduler manually\n"); ++ } ++ ++out: ++ mutex_unlock(&scheduler->lock); ++} ++ ++void kbase_csf_scheduler_pm_suspend(struct kbase_device *kbdev) ++{ ++ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; ++ ++ /* Cancel any potential queued delayed work(s) */ ++ cancel_delayed_work_sync(&scheduler->tick_work); ++ cancel_delayed_work_sync(&scheduler->tock_work); ++ ++ mutex_lock(&scheduler->lock); ++ ++ WARN_ON(!kbase_pm_is_suspending(kbdev)); ++ ++ if (scheduler->state != SCHED_SUSPENDED) { ++ suspend_active_groups_on_powerdown(kbdev, true); ++ dev_info(kbdev->dev, "Scheduler PM suspend"); ++ scheduler_suspend(kbdev); ++ } ++ mutex_unlock(&scheduler->lock); ++} ++ ++void kbase_csf_scheduler_pm_resume(struct kbase_device *kbdev) ++{ ++ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; ++ ++ mutex_lock(&scheduler->lock); ++ ++ WARN_ON(kbase_pm_is_suspending(kbdev)); ++ ++ if (scheduler->total_runnable_grps > 0) { ++ WARN_ON(scheduler->state != SCHED_SUSPENDED); ++ dev_info(kbdev->dev, "Scheduler PM resume"); ++ scheduler_wakeup(kbdev, true); ++ } ++ mutex_unlock(&scheduler->lock); ++} ++ ++void kbase_csf_scheduler_pm_active(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ u32 prev_count; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ prev_count = kbdev->csf.scheduler.pm_active_count++; ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ /* On 0 => 1, make a pm_ctx_active request */ ++ if (!prev_count) ++ kbase_pm_context_active(kbdev); ++ else ++ WARN_ON(prev_count == U32_MAX); ++} ++ ++void kbase_csf_scheduler_pm_idle(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ u32 prev_count; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ prev_count = kbdev->csf.scheduler.pm_active_count--; ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ if (prev_count == 1) ++ kbase_pm_context_idle(kbdev); ++ else ++ WARN_ON(prev_count == 0); ++} +diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.h +new file mode 100755 +index 000000000000..1b1c0681f64d +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.h +@@ -0,0 +1,408 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_CSF_SCHEDULER_H_ ++#define _KBASE_CSF_SCHEDULER_H_ ++ ++#include "mali_kbase_csf.h" ++ ++/** ++ * kbase_csf_scheduler_queue_start() - Enable the running of GPU command queue ++ * on firmware. ++ * ++ * @queue: Pointer to the GPU command queue to be started. ++ * ++ * This function would enable the start of a command stream interface, within a ++ * command stream group, to which the @queue was bound. ++ * If the command stream group is already scheduled and resident, the command ++ * stream interface will be started right away, otherwise once the group is ++ * made resident. ++ * ++ * Return: 0 on success, or negative on failure. ++ */ ++int kbase_csf_scheduler_queue_start(struct kbase_queue *queue); ++ ++/** ++ * kbase_csf_scheduler_queue_stop() - Disable the running of GPU command queue ++ * on firmware. ++ * ++ * @queue: Pointer to the GPU command queue to be stopped. ++ * ++ * This function would stop the command stream interface, within a command ++ * stream group, to which the @queue was bound. ++ * ++ * Return: 0 on success, or negative on failure. ++ */ ++int kbase_csf_scheduler_queue_stop(struct kbase_queue *queue); ++ ++/** ++ * kbase_csf_scheduler_group_protm_enter - Handle the protm enter event for the ++ * GPU command queue group. ++ * ++ * @group: The command queue group. ++ * ++ * This function could request the firmware to enter the protected mode ++ * and allow the execution of protected region instructions for all the ++ * bound queues of the group that have protm pending bit set in their ++ * respective CS_ACK register. ++ */ ++void kbase_csf_scheduler_group_protm_enter(struct kbase_queue_group *group); ++ ++/** ++ * kbase_csf_scheduler_group_get_slot() - Checks if a queue group is ++ * programmed on a firmware Command Stream Group slot ++ * and returns the slot number. ++ * ++ * @group: The command queue group. ++ * ++ * Return: The slot number, if the group is programmed on a slot. ++ * Otherwise returns a negative number. ++ * ++ * Note: This function should not be used if the interrupt_lock is held. Use ++ * kbase_csf_scheduler_group_get_slot_locked() instead. ++ */ ++int kbase_csf_scheduler_group_get_slot(struct kbase_queue_group *group); ++ ++/** ++ * kbase_csf_scheduler_group_get_slot_locked() - Checks if a queue group is ++ * programmed on a firmware Command Stream Group slot ++ * and returns the slot number. ++ * ++ * @group: The command queue group. ++ * ++ * Return: The slot number, if the group is programmed on a slot. ++ * Otherwise returns a negative number. ++ * ++ * Note: Caller must hold the interrupt_lock. ++ */ ++int kbase_csf_scheduler_group_get_slot_locked(struct kbase_queue_group *group); ++ ++/** ++ * kbase_csf_scheduler_group_events_enabled() - Checks if interrupt events ++ * should be handled for a queue group. ++ * ++ * @kbdev: The device of the group. ++ * @group: The queue group. ++ * ++ * Return: true if interrupt events should be handled. ++ * ++ * Note: Caller must hold the interrupt_lock. ++ */ ++bool kbase_csf_scheduler_group_events_enabled(struct kbase_device *kbdev, ++ struct kbase_queue_group *group); ++ ++/** ++ * kbase_csf_scheduler_get_group_on_slot()- Gets the queue group that has been ++ * programmed to a firmware Command Stream Group slot. ++ * ++ * @kbdev: The GPU device. ++ * @slot: The slot for which to get the queue group. ++ * ++ * Return: Pointer to the programmed queue group. ++ * ++ * Note: Caller must hold the interrupt_lock. ++ */ ++struct kbase_queue_group *kbase_csf_scheduler_get_group_on_slot( ++ struct kbase_device *kbdev, int slot); ++ ++/** ++ * kbase_csf_scheduler_group_deschedule() - Deschedule a GPU command queue ++ * group from the firmware. ++ * ++ * @group: Pointer to the queue group to be scheduled. ++ * ++ * This function would disable the scheduling of GPU command queue group on ++ * firmware. ++ */ ++void kbase_csf_scheduler_group_deschedule(struct kbase_queue_group *group); ++ ++/** ++ * kbase_csf_scheduler_evict_ctx_slots() - Evict all GPU command queue groups ++ * of a given context that are active ++ * running from the firmware. ++ * ++ * @kbdev: The GPU device. ++ * @kctx: Kbase context for the evict operation. ++ * @evicted_groups: List_head for returning evicted active queue groups. ++ * ++ * This function would disable the scheduling of GPU command queue groups active ++ * on firmware slots from the given Kbase context. The affected groups are ++ * added to the supplied list_head argument. ++ */ ++void kbase_csf_scheduler_evict_ctx_slots(struct kbase_device *kbdev, ++ struct kbase_context *kctx, struct list_head *evicted_groups); ++ ++/** ++ * kbase_csf_scheduler_context_init() - Initialize the context-specific part ++ * for CSF scheduler. ++ * ++ * @kctx: Pointer to kbase context that is being created. ++ * ++ * This function must be called during Kbase context creation. ++ * ++ * Return: 0 on success, or negative on failure. ++ */ ++int kbase_csf_scheduler_context_init(struct kbase_context *kctx); ++ ++/** ++ * kbase_csf_scheduler_init - Initialize the CSF scheduler ++ * ++ * @kbdev: Instance of a GPU platform device that implements a command ++ * stream front-end interface. ++ * ++ * The scheduler does the arbitration for the command stream group slots ++ * provided by the firmware between the GPU command queue groups created ++ * by the Clients. ++ * ++ * Return: 0 on success, or negative on failure. ++ */ ++int kbase_csf_scheduler_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_csf_scheduler_context_init() - Terminate the context-specific part ++ * for CSF scheduler. ++ * ++ * @kctx: Pointer to kbase context that is being terminated. ++ * ++ * This function must be called during Kbase context termination. ++ */ ++void kbase_csf_scheduler_context_term(struct kbase_context *kctx); ++ ++/** ++ * kbase_csf_scheduler_term - Terminate the CSF scheduler. ++ * ++ * @kbdev: Instance of a GPU platform device that implements a command ++ * stream front-end interface. ++ * ++ * This should be called when unload of firmware is done on device ++ * termination. ++ */ ++void kbase_csf_scheduler_term(struct kbase_device *kbdev); ++ ++/** ++ * kbase_csf_scheduler_reset - Reset the state of all active GPU command ++ * queue groups. ++ * ++ * @kbdev: Instance of a GPU platform device that implements a command ++ * stream front-end interface. ++ * ++ * This function will first iterate through all the active/scheduled GPU ++ * command queue groups and suspend them (to avoid losing work for groups ++ * that are not stuck). The groups that could not get suspended would be ++ * descheduled and marked as terminated (which will then lead to unbinding ++ * of all the queues bound to them) and also no more work would be allowed ++ * to execute for them. ++ * ++ * This is similar to the action taken in response to an unexpected OoM event. ++ * No explicit re-initialization is done for CSG & CS interface I/O pages; ++ * instead, that happens implicitly on firmware reload. ++ * ++ * Should be called only after initiating the GPU reset. ++ */ ++void kbase_csf_scheduler_reset(struct kbase_device *kbdev); ++ ++/** ++ * kbase_csf_scheduler_enable_tick_timer - Enable the scheduler tick timer. ++ * ++ * @kbdev: Instance of a GPU platform device that implements a command ++ * stream front-end interface. ++ * ++ * This function will restart the scheduler tick so that regular scheduling can ++ * be resumed without any explicit trigger (like kicking of GPU queues). ++ */ ++void kbase_csf_scheduler_enable_tick_timer(struct kbase_device *kbdev); ++ ++/** ++ * kbase_csf_scheduler_group_copy_suspend_buf - Suspend a queue ++ * group and copy suspend buffer. ++ * ++ * This function is called to suspend a queue group and copy the suspend_buffer ++ * contents to the input buffer provided. ++ * ++ * @group: Pointer to the queue group to be suspended. ++ * @sus_buf: Pointer to the structure which contains details of the ++ * user buffer and its kernel pinned pages to which we need to copy ++ * the group suspend buffer. ++ * ++ * Return: 0 on success, or negative on failure. ++ */ ++int kbase_csf_scheduler_group_copy_suspend_buf(struct kbase_queue_group *group, ++ struct kbase_suspend_copy_buffer *sus_buf); ++ ++/** ++ * kbase_csf_scheduler_lock - Acquire the global Scheduler lock. ++ * ++ * @kbdev: Instance of a GPU platform device that implements a command ++ * stream front-end interface. ++ * ++ * This function will take the global scheduler lock, in order to serialize ++ * against the Scheduler actions, for access to CS IO pages. ++ */ ++static inline void kbase_csf_scheduler_lock(struct kbase_device *kbdev) ++{ ++ mutex_lock(&kbdev->csf.scheduler.lock); ++} ++ ++/** ++ * kbase_csf_scheduler_unlock - Release the global Scheduler lock. ++ * ++ * @kbdev: Instance of a GPU platform device that implements a command ++ * stream front-end interface. ++ */ ++static inline void kbase_csf_scheduler_unlock(struct kbase_device *kbdev) ++{ ++ mutex_unlock(&kbdev->csf.scheduler.lock); ++} ++ ++/** ++ * kbase_csf_scheduler_spin_lock - Acquire Scheduler interrupt spinlock. ++ * ++ * @kbdev: Instance of a GPU platform device that implements a command ++ * stream front-end interface. ++ * @flags: Pointer to the memory location that would store the previous ++ * interrupt state. ++ * ++ * This function will take the global scheduler lock, in order to serialize ++ * against the Scheduler actions, for access to CS IO pages. ++ */ ++static inline void kbase_csf_scheduler_spin_lock(struct kbase_device *kbdev, ++ unsigned long *flags) ++{ ++ spin_lock_irqsave(&kbdev->csf.scheduler.interrupt_lock, *flags); ++} ++ ++/** ++ * kbase_csf_scheduler_spin_unlock - Release Scheduler interrupt spinlock. ++ * ++ * @kbdev: Instance of a GPU platform device that implements a command ++ * stream front-end interface. ++ * @flags: Previously stored interrupt state when Scheduler interrupt ++ * spinlock was acquired. ++ */ ++static inline void kbase_csf_scheduler_spin_unlock(struct kbase_device *kbdev, ++ unsigned long flags) ++{ ++ spin_unlock_irqrestore(&kbdev->csf.scheduler.interrupt_lock, flags); ++} ++ ++/** ++ * kbase_csf_scheduler_spin_lock_assert_held - Assert if the Scheduler ++ * interrupt spinlock is held. ++ * ++ * @kbdev: Instance of a GPU platform device that implements a command ++ * stream front-end interface. ++ */ ++static inline void ++kbase_csf_scheduler_spin_lock_assert_held(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->csf.scheduler.interrupt_lock); ++} ++ ++/** ++ * kbase_csf_scheduler_timer_is_enabled() - Check if the scheduler wakes up ++ * automatically for periodic tasks. ++ * ++ * @kbdev: Pointer to the device ++ * ++ * Return: true if the scheduler is configured to wake up periodically ++ */ ++bool kbase_csf_scheduler_timer_is_enabled(struct kbase_device *kbdev); ++ ++/** ++ * kbase_csf_scheduler_timer_set_enabled() - Enable/disable periodic ++ * scheduler tasks. ++ * ++ * @kbdev: Pointer to the device ++ * @enable: Whether to enable periodic scheduler tasks ++ */ ++void kbase_csf_scheduler_timer_set_enabled(struct kbase_device *kbdev, ++ bool enable); ++ ++/** ++ * kbase_csf_scheduler_kick - Perform pending scheduling tasks once. ++ * ++ * Note: This function is only effective if the scheduling timer is disabled. ++ * ++ * @kbdev: Instance of a GPU platform device that implements a command ++ * stream front-end interface. ++ */ ++void kbase_csf_scheduler_kick(struct kbase_device *kbdev); ++ ++/** ++ * kbase_csf_scheduler_protected_mode_in_use() - Check if the scheduler is ++ * running with protected mode tasks. ++ * ++ * @kbdev: Pointer to the device ++ * ++ * Return: true if the scheduler is running with protected mode tasks ++ */ ++static inline bool kbase_csf_scheduler_protected_mode_in_use( ++ struct kbase_device *kbdev) ++{ ++ return (kbdev->csf.scheduler.active_protm_grp != NULL); ++} ++ ++/** ++ * kbase_csf_scheduler_pm_active - Perform scheduler power active operation ++ * ++ * Note: This function will increase the scheduler's internal pm_active_count ++ * value, ensuring that both GPU and MCU are powered for access. ++ * ++ * @kbdev: Instance of a GPU platform device that implements a command ++ * stream front-end interface. ++ */ ++void kbase_csf_scheduler_pm_active(struct kbase_device *kbdev); ++ ++/** ++ * kbase_csf_scheduler_pm_idle - Perform the scheduler power idle operation ++ * ++ * Note: This function will decrease the scheduler's internal pm_active_count ++ * value. On reaching 0, the MCU and GPU could be powered off. ++ * ++ * @kbdev: Instance of a GPU platform device that implements a command ++ * stream front-end interface. ++ */ ++void kbase_csf_scheduler_pm_idle(struct kbase_device *kbdev); ++ ++/** ++ * kbase_csf_scheduler_pm_resume - Reactivate the scheduler on system resume ++ * ++ * @kbdev: Instance of a GPU platform device that implements a command ++ * stream front-end interface. ++ * ++ * This function will make the scheduler resume the scheduling of queue groups ++ * and take the power managemenet reference, if there are any runnable groups. ++ */ ++void kbase_csf_scheduler_pm_resume(struct kbase_device *kbdev); ++ ++/** ++ * kbase_csf_scheduler_pm_suspend - Idle the scheduler on system suspend ++ * ++ * @kbdev: Instance of a GPU platform device that implements a command ++ * stream front-end interface. ++ * ++ * This function will make the scheduler suspend all the running queue groups ++ * and drop its power managemenet reference. ++ */ ++void kbase_csf_scheduler_pm_suspend(struct kbase_device *kbdev); ++ ++#endif /* _KBASE_CSF_SCHEDULER_H_ */ +diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap.c +new file mode 100755 +index 000000000000..60cae15bc8ef +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap.c +@@ -0,0 +1,584 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include "mali_kbase_csf_tiler_heap.h" ++#include "mali_kbase_csf_tiler_heap_def.h" ++#include "mali_kbase_csf_heap_context_alloc.h" ++ ++/** ++ * encode_chunk_ptr - Encode the address and size of a chunk as an integer. ++ * ++ * The size and address of the next chunk in a list are packed into a single ++ * 64-bit value for storage in a chunk's header. This function returns that ++ * value. ++ * ++ * @chunk_size: Size of a tiler heap chunk, in bytes. ++ * @chunk_addr: GPU virtual address of the same tiler heap chunk. ++ * ++ * Return: Next chunk pointer suitable for writing into a chunk header. ++ */ ++static u64 encode_chunk_ptr(u32 const chunk_size, u64 const chunk_addr) ++{ ++ u64 encoded_size, encoded_addr; ++ ++ WARN_ON(chunk_size & ~CHUNK_SIZE_MASK); ++ WARN_ON(chunk_addr & ~CHUNK_ADDR_MASK); ++ ++ encoded_size = ++ (u64)(chunk_size >> CHUNK_HDR_NEXT_SIZE_ENCODE_SHIFT) << ++ CHUNK_HDR_NEXT_SIZE_POS; ++ ++ encoded_addr = ++ (chunk_addr >> CHUNK_HDR_NEXT_ADDR_ENCODE_SHIFT) << ++ CHUNK_HDR_NEXT_ADDR_POS; ++ ++ return (encoded_size & CHUNK_HDR_NEXT_SIZE_MASK) | ++ (encoded_addr & CHUNK_HDR_NEXT_ADDR_MASK); ++} ++ ++/** ++ * get_last_chunk - Get the last chunk of a tiler heap ++ * ++ * @heap: Pointer to the tiler heap. ++ * ++ * Return: The address of the most recently-linked chunk, or NULL if none. ++ */ ++static struct kbase_csf_tiler_heap_chunk *get_last_chunk( ++ struct kbase_csf_tiler_heap *const heap) ++{ ++ lockdep_assert_held(&heap->kctx->csf.tiler_heaps.lock); ++ ++ if (list_empty(&heap->chunks_list)) ++ return NULL; ++ ++ return list_last_entry(&heap->chunks_list, ++ struct kbase_csf_tiler_heap_chunk, link); ++} ++ ++/** ++ * link_chunk - Link a chunk into a tiler heap ++ * ++ * Unless the @chunk is the first in the kernel's list of chunks belonging to ++ * a given tiler heap, this function stores the size and address of the @chunk ++ * in the header of the preceding chunk. This requires the GPU memory region ++ * containing the header to be be mapped temporarily, which can fail. ++ * ++ * @heap: Pointer to the tiler heap. ++ * @chunk: Pointer to the heap chunk to be linked. ++ * ++ * Return: 0 if successful or a negative error code on failure. ++ */ ++static int link_chunk(struct kbase_csf_tiler_heap *const heap, ++ struct kbase_csf_tiler_heap_chunk *const chunk) ++{ ++ struct kbase_csf_tiler_heap_chunk *const prev = get_last_chunk(heap); ++ ++ if (prev) { ++ struct kbase_context *const kctx = heap->kctx; ++ struct kbase_vmap_struct map; ++ u64 *const prev_hdr = kbase_vmap_prot(kctx, prev->gpu_va, ++ sizeof(*prev_hdr), KBASE_REG_CPU_WR, &map); ++ ++ if (unlikely(!prev_hdr)) { ++ dev_err(kctx->kbdev->dev, ++ "Failed to map tiler heap chunk 0x%llX\n", ++ prev->gpu_va); ++ return -ENOMEM; ++ } ++ ++ *prev_hdr = encode_chunk_ptr(heap->chunk_size, chunk->gpu_va); ++ kbase_vunmap(kctx, &map); ++ ++ dev_dbg(kctx->kbdev->dev, ++ "Linked tiler heap chunks, 0x%llX -> 0x%llX\n", ++ prev->gpu_va, chunk->gpu_va); ++ } ++ ++ return 0; ++} ++ ++/** ++ * init_chunk - Initialize and link a tiler heap chunk ++ * ++ * Zero-initialize a new chunk's header (including its pointer to the next ++ * chunk, which doesn't exist yet) and then update the previous chunk's ++ * header to link the new chunk into the chunk list. ++ * ++ * @heap: Pointer to the tiler heap. ++ * @chunk: Pointer to the heap chunk to be initialized and linked. ++ * @link_with_prev: Flag to indicate if the new chunk needs to be linked with ++ * the previously allocated chunk. ++ * ++ * Return: 0 if successful or a negative error code on failure. ++ */ ++static int init_chunk(struct kbase_csf_tiler_heap *const heap, ++ struct kbase_csf_tiler_heap_chunk *const chunk, bool link_with_prev) ++{ ++ struct kbase_vmap_struct map; ++ struct u64 *chunk_hdr = NULL; ++ struct kbase_context *const kctx = heap->kctx; ++ ++ if (unlikely(chunk->gpu_va & ~CHUNK_ADDR_MASK)) { ++ dev_err(kctx->kbdev->dev, ++ "Tiler heap chunk address is unusable\n"); ++ return -EINVAL; ++ } ++ ++ chunk_hdr = kbase_vmap_prot(kctx, ++ chunk->gpu_va, CHUNK_HDR_SIZE, KBASE_REG_CPU_WR, &map); ++ ++ if (unlikely(!chunk_hdr)) { ++ dev_err(kctx->kbdev->dev, ++ "Failed to map a tiler heap chunk header\n"); ++ return -ENOMEM; ++ } ++ ++ memset(chunk_hdr, 0, CHUNK_HDR_SIZE); ++ kbase_vunmap(kctx, &map); ++ ++ if (link_with_prev) ++ return link_chunk(heap, chunk); ++ else ++ return 0; ++} ++ ++/** ++ * create_chunk - Create a tiler heap chunk ++ * ++ * This function allocates a chunk of memory for a tiler heap and adds it to ++ * the end of the list of chunks associated with that heap. The size of the ++ * chunk is not a parameter because it is configured per-heap not per-chunk. ++ * ++ * @heap: Pointer to the tiler heap for which to allocate memory. ++ * @link_with_prev: Flag to indicate if the chunk to be allocated needs to be ++ * linked with the previously allocated chunk. ++ * ++ * Return: 0 if successful or a negative error code on failure. ++ */ ++static int create_chunk(struct kbase_csf_tiler_heap *const heap, ++ bool link_with_prev) ++{ ++ int err = 0; ++ struct kbase_context *const kctx = heap->kctx; ++ u64 nr_pages = PFN_UP(heap->chunk_size); ++ u64 flags = BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR | ++ BASE_MEM_PROT_CPU_WR | BASEP_MEM_NO_USER_FREE | ++ BASE_MEM_COHERENT_LOCAL; ++ struct kbase_csf_tiler_heap_chunk *chunk = NULL; ++ ++ flags |= base_mem_group_id_set(kctx->jit_group_id); ++ ++#if defined(CONFIG_MALI_BIFROST_DEBUG) || defined(CONFIG_MALI_VECTOR_DUMP) ++ flags |= BASE_MEM_PROT_CPU_RD; ++#endif ++ ++ lockdep_assert_held(&kctx->csf.tiler_heaps.lock); ++ ++ chunk = kzalloc(sizeof(*chunk), GFP_KERNEL); ++ if (unlikely(!chunk)) { ++ dev_err(kctx->kbdev->dev, ++ "No kernel memory for a new tiler heap chunk\n"); ++ return -ENOMEM; ++ } ++ ++ /* Allocate GPU memory for the new chunk. */ ++ INIT_LIST_HEAD(&chunk->link); ++ chunk->region = kbase_mem_alloc(kctx, nr_pages, nr_pages, 0, ++ &flags, &chunk->gpu_va); ++ ++ if (unlikely(!chunk->region)) { ++ dev_err(kctx->kbdev->dev, ++ "Failed to allocate a tiler heap chunk\n"); ++ err = -ENOMEM; ++ } else { ++ err = init_chunk(heap, chunk, link_with_prev); ++ if (unlikely(err)) { ++ kbase_gpu_vm_lock(kctx); ++ chunk->region->flags &= ~KBASE_REG_NO_USER_FREE; ++ kbase_mem_free_region(kctx, chunk->region); ++ kbase_gpu_vm_unlock(kctx); ++ } ++ } ++ ++ if (unlikely(err)) { ++ kfree(chunk); ++ } else { ++ list_add_tail(&chunk->link, &heap->chunks_list); ++ heap->chunk_count++; ++ ++ dev_dbg(kctx->kbdev->dev, "Created tiler heap chunk 0x%llX\n", ++ chunk->gpu_va); ++ } ++ ++ return err; ++} ++ ++/** ++ * delete_chunk - Delete a tiler heap chunk ++ * ++ * This function frees a tiler heap chunk previously allocated by @create_chunk ++ * and removes it from the list of chunks associated with the heap. ++ * ++ * WARNING: The deleted chunk is not unlinked from the list of chunks used by ++ * the GPU, therefore it is only safe to use this function when ++ * deleting a heap. ++ * ++ * @heap: Pointer to the tiler heap for which @chunk was allocated. ++ * @chunk: Pointer to a chunk to be deleted. ++ */ ++static void delete_chunk(struct kbase_csf_tiler_heap *const heap, ++ struct kbase_csf_tiler_heap_chunk *const chunk) ++{ ++ struct kbase_context *const kctx = heap->kctx; ++ ++ lockdep_assert_held(&kctx->csf.tiler_heaps.lock); ++ ++ kbase_gpu_vm_lock(kctx); ++ chunk->region->flags &= ~KBASE_REG_NO_USER_FREE; ++ kbase_mem_free_region(kctx, chunk->region); ++ kbase_gpu_vm_unlock(kctx); ++ list_del(&chunk->link); ++ heap->chunk_count--; ++ kfree(chunk); ++} ++ ++/** ++ * delete_all_chunks - Delete all chunks belonging to a tiler heap ++ * ++ * This function empties the list of chunks associated with a tiler heap by ++ * freeing all chunks previously allocated by @create_chunk. ++ * ++ * @heap: Pointer to a tiler heap. ++ */ ++static void delete_all_chunks(struct kbase_csf_tiler_heap *heap) ++{ ++ struct list_head *entry = NULL, *tmp = NULL; ++ struct kbase_context *const kctx = heap->kctx; ++ ++ lockdep_assert_held(&kctx->csf.tiler_heaps.lock); ++ ++ list_for_each_safe(entry, tmp, &heap->chunks_list) { ++ struct kbase_csf_tiler_heap_chunk *chunk = list_entry( ++ entry, struct kbase_csf_tiler_heap_chunk, link); ++ ++ delete_chunk(heap, chunk); ++ } ++} ++ ++/** ++ * create_initial_chunks - Create the initial list of chunks for a tiler heap ++ * ++ * This function allocates a given number of chunks for a tiler heap and ++ * adds them to the list of chunks associated with that heap. ++ * ++ * @heap: Pointer to the tiler heap for which to allocate memory. ++ * @nchunks: Number of chunks to create. ++ * ++ * Return: 0 if successful or a negative error code on failure. ++ */ ++static int create_initial_chunks(struct kbase_csf_tiler_heap *const heap, ++ u32 const nchunks) ++{ ++ int err = 0; ++ u32 i; ++ ++ for (i = 0; (i < nchunks) && likely(!err); i++) ++ err = create_chunk(heap, true); ++ ++ if (unlikely(err)) ++ delete_all_chunks(heap); ++ ++ return err; ++} ++ ++/** ++ * delete_heap - Delete a tiler heap ++ * ++ * This function frees any chunks allocated for a tiler heap previously ++ * initialized by @kbase_csf_tiler_heap_init and removes it from the list of ++ * heaps associated with the kbase context. The heap context structure used by ++ * the firmware is also freed. ++ * ++ * @heap: Pointer to a tiler heap to be deleted. ++ */ ++static void delete_heap(struct kbase_csf_tiler_heap *heap) ++{ ++ struct kbase_context *const kctx = heap->kctx; ++ ++ dev_dbg(kctx->kbdev->dev, "Deleting tiler heap 0x%llX\n", heap->gpu_va); ++ ++ lockdep_assert_held(&kctx->csf.tiler_heaps.lock); ++ ++ delete_all_chunks(heap); ++ ++ /* We could optimize context destruction by not freeing leaked heap ++ * contexts but it doesn't seem worth the extra complexity. ++ */ ++ kbase_csf_heap_context_allocator_free(&kctx->csf.tiler_heaps.ctx_alloc, ++ heap->gpu_va); ++ ++ list_del(&heap->link); ++ kfree(heap); ++} ++ ++/** ++ * find_tiler_heap - Find a tiler heap from the address of its heap context ++ * ++ * Each tiler heap managed by the kernel has an associated heap context ++ * structure used by the firmware. This function finds a tiler heap object from ++ * the GPU virtual address of its associated heap context. The heap context ++ * should have been allocated by @kbase_csf_heap_context_allocator_alloc in the ++ * same @kctx. ++ * ++ * @kctx: Pointer to the kbase context to search for a tiler heap. ++ * @heap_gpu_va: GPU virtual address of a heap context structure. ++ * ++ * Return: pointer to the tiler heap object, or NULL if not found. ++ */ ++static struct kbase_csf_tiler_heap *find_tiler_heap( ++ struct kbase_context *const kctx, u64 const heap_gpu_va) ++{ ++ struct kbase_csf_tiler_heap *heap = NULL; ++ ++ lockdep_assert_held(&kctx->csf.tiler_heaps.lock); ++ ++ list_for_each_entry(heap, &kctx->csf.tiler_heaps.list, link) { ++ if (heap_gpu_va == heap->gpu_va) ++ return heap; ++ } ++ ++ dev_dbg(kctx->kbdev->dev, "Tiler heap 0x%llX was not found\n", ++ heap_gpu_va); ++ ++ return NULL; ++} ++ ++int kbase_csf_tiler_heap_context_init(struct kbase_context *const kctx) ++{ ++ int err = kbase_csf_heap_context_allocator_init( ++ &kctx->csf.tiler_heaps.ctx_alloc, kctx); ++ ++ if (unlikely(err)) ++ return err; ++ ++ INIT_LIST_HEAD(&kctx->csf.tiler_heaps.list); ++ mutex_init(&kctx->csf.tiler_heaps.lock); ++ ++ dev_dbg(kctx->kbdev->dev, "Initialized a context for tiler heaps\n"); ++ ++ return 0; ++} ++ ++void kbase_csf_tiler_heap_context_term(struct kbase_context *const kctx) ++{ ++ struct list_head *entry = NULL, *tmp = NULL; ++ ++ dev_dbg(kctx->kbdev->dev, "Terminating a context for tiler heaps\n"); ++ ++ mutex_lock(&kctx->csf.tiler_heaps.lock); ++ ++ list_for_each_safe(entry, tmp, &kctx->csf.tiler_heaps.list) { ++ struct kbase_csf_tiler_heap *heap = list_entry( ++ entry, struct kbase_csf_tiler_heap, link); ++ delete_heap(heap); ++ } ++ ++ mutex_unlock(&kctx->csf.tiler_heaps.lock); ++ mutex_destroy(&kctx->csf.tiler_heaps.lock); ++ ++ kbase_csf_heap_context_allocator_term(&kctx->csf.tiler_heaps.ctx_alloc); ++} ++ ++int kbase_csf_tiler_heap_init(struct kbase_context *const kctx, ++ u32 const chunk_size, u32 const initial_chunks, u32 const max_chunks, ++ u16 const target_in_flight, u64 *const heap_gpu_va, ++ u64 *const first_chunk_va) ++{ ++ int err = 0; ++ struct kbase_csf_tiler_heap *heap = NULL; ++ struct kbase_csf_heap_context_allocator *const ctx_alloc = ++ &kctx->csf.tiler_heaps.ctx_alloc; ++ ++ dev_dbg(kctx->kbdev->dev, ++ "Creating a tiler heap with %u chunks (limit: %u) of size %u\n", ++ initial_chunks, max_chunks, chunk_size); ++ ++ if (chunk_size == 0) ++ return -EINVAL; ++ ++ if (chunk_size & ~CHUNK_SIZE_MASK) ++ return -EINVAL; ++ ++ if (initial_chunks == 0) ++ return -EINVAL; ++ ++ if (initial_chunks > max_chunks) ++ return -EINVAL; ++ ++ if (target_in_flight == 0) ++ return -EINVAL; ++ ++ heap = kzalloc(sizeof(*heap), GFP_KERNEL); ++ if (unlikely(!heap)) { ++ dev_err(kctx->kbdev->dev, ++ "No kernel memory for a new tiler heap\n"); ++ return -ENOMEM; ++ } ++ ++ heap->kctx = kctx; ++ heap->chunk_size = chunk_size; ++ heap->max_chunks = max_chunks; ++ heap->target_in_flight = target_in_flight; ++ INIT_LIST_HEAD(&heap->chunks_list); ++ ++ heap->gpu_va = kbase_csf_heap_context_allocator_alloc(ctx_alloc); ++ ++ mutex_lock(&kctx->csf.tiler_heaps.lock); ++ ++ if (unlikely(!heap->gpu_va)) { ++ dev_err(kctx->kbdev->dev, ++ "Failed to allocate a tiler heap context\n"); ++ err = -ENOMEM; ++ } else { ++ err = create_initial_chunks(heap, initial_chunks); ++ if (unlikely(err)) { ++ kbase_csf_heap_context_allocator_free(ctx_alloc, ++ heap->gpu_va); ++ } ++ } ++ ++ if (unlikely(err)) { ++ kfree(heap); ++ } else { ++ struct kbase_csf_tiler_heap_chunk const *first_chunk = ++ list_first_entry(&heap->chunks_list, ++ struct kbase_csf_tiler_heap_chunk, link); ++ ++ list_add(&heap->link, &kctx->csf.tiler_heaps.list); ++ ++ *heap_gpu_va = heap->gpu_va; ++ *first_chunk_va = first_chunk->gpu_va; ++ ++ dev_dbg(kctx->kbdev->dev, "Created tiler heap 0x%llX\n", ++ heap->gpu_va); ++ } ++ ++ mutex_unlock(&kctx->csf.tiler_heaps.lock); ++ ++ return err; ++} ++ ++int kbase_csf_tiler_heap_term(struct kbase_context *const kctx, ++ u64 const heap_gpu_va) ++{ ++ int err = 0; ++ struct kbase_csf_tiler_heap *heap = NULL; ++ ++ mutex_lock(&kctx->csf.tiler_heaps.lock); ++ ++ heap = find_tiler_heap(kctx, heap_gpu_va); ++ if (likely(heap)) ++ delete_heap(heap); ++ else ++ err = -EINVAL; ++ ++ mutex_unlock(&kctx->csf.tiler_heaps.lock); ++ ++ return err; ++} ++ ++/** ++ * alloc_new_chunk - Allocate a new chunk for the tiler heap. ++ * ++ * This function will allocate a new chunk for the chunked tiler heap depending ++ * on the settings provided by userspace when the heap was created and the ++ * heap's statistics (like number of render passes in-flight). ++ * ++ * @heap: Pointer to the tiler heap. ++ * @nr_in_flight: Number of render passes that are in-flight, must not be zero. ++ * @new_chunk_ptr: Where to store the GPU virtual address & size of the new ++ * chunk allocated for the heap. ++ * ++ * Return: 0 if a new chunk was allocated otherwise an appropriate negative ++ * error code. ++ */ ++static int alloc_new_chunk(struct kbase_csf_tiler_heap *heap, ++ u32 nr_in_flight, u64 *new_chunk_ptr) ++{ ++ int err = -ENOMEM; ++ ++ lockdep_assert_held(&heap->kctx->csf.tiler_heaps.lock); ++ ++ if (!nr_in_flight) ++ return -EINVAL; ++ ++ if ((nr_in_flight <= heap->target_in_flight) && ++ (heap->chunk_count < heap->max_chunks)) { ++ /* Not exceeded the target number of render passes yet so be ++ * generous with memory. ++ */ ++ err = create_chunk(heap, false); ++ ++ if (likely(!err)) { ++ struct kbase_csf_tiler_heap_chunk *new_chunk = ++ get_last_chunk(heap); ++ if (!WARN_ON(!new_chunk)) { ++ *new_chunk_ptr = ++ encode_chunk_ptr(heap->chunk_size, ++ new_chunk->gpu_va); ++ return 0; ++ } ++ } ++ } ++ ++ /* A new chunk wasn't allocated this time, check if the allocation can ++ * be retried later. ++ */ ++ if (nr_in_flight > 1) { ++ /* Can retry as there are some ongoing fragment ++ * jobs which are expected to free up chunks. ++ */ ++ err = -EBUSY; ++ } ++ ++ return err; ++} ++ ++int kbase_csf_tiler_heap_alloc_new_chunk(struct kbase_context *kctx, ++ u64 gpu_heap_va, u32 nr_in_flight, u64 *new_chunk_ptr) ++{ ++ struct kbase_csf_tiler_heap *heap; ++ int err = -EINVAL; ++ ++ mutex_lock(&kctx->csf.tiler_heaps.lock); ++ ++ heap = find_tiler_heap(kctx, gpu_heap_va); ++ ++ if (likely(heap)) { ++ err = alloc_new_chunk(heap, nr_in_flight, ++ new_chunk_ptr); ++ } ++ ++ mutex_unlock(&kctx->csf.tiler_heaps.lock); ++ ++ return err; ++} +diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap.h +new file mode 100755 +index 000000000000..1a4729df6ca3 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap.h +@@ -0,0 +1,113 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_CSF_TILER_HEAP_H_ ++#define _KBASE_CSF_TILER_HEAP_H_ ++ ++#include ++ ++/** ++ * kbase_csf_tiler_heap_context_init - Initialize the tiler heaps context for a ++ * GPU address space ++ * ++ * @kctx: Pointer to the kbase context being initialized. ++ * ++ * Return: 0 if successful or a negative error code on failure. ++ */ ++int kbase_csf_tiler_heap_context_init(struct kbase_context *kctx); ++ ++/** ++ * kbase_csf_tiler_heap_context_term - Terminate the tiler heaps context for a ++ * GPU address space ++ * ++ * This function deletes any chunked tiler heaps that weren't deleted before ++ * context termination. ++ * ++ * @kctx: Pointer to the kbase context being terminated. ++ */ ++void kbase_csf_tiler_heap_context_term(struct kbase_context *kctx); ++ ++/** ++ * kbase_csf_tiler_heap_init - Initialize a chunked tiler memory heap. ++ * ++ * @kctx: Pointer to the kbase context in which to allocate resources for the ++ * tiler heap. ++ * @chunk_size: Size of each chunk, in bytes. Must be page-aligned. ++ * @initial_chunks: The initial number of chunks to allocate. Must not be ++ * zero or greater than @max_chunks. ++ * @max_chunks: The maximum number of chunks that the heap should be allowed ++ * to use. Must not be less than @initial_chunks. ++ * @target_in_flight: Number of render-passes that the driver should attempt to ++ * keep in flight for which allocation of new chunks is ++ * allowed. Must not be zero. ++ * @gpu_heap_va: Where to store the GPU virtual address of the context that was ++ * set up for the tiler heap. ++ * @first_chunk_va: Where to store the GPU virtual address of the first chunk ++ * allocated for the heap. This points to the header of the ++ * heap chunk and not to the low address of free memory in it. ++ * ++ * Return: 0 if successful or a negative error code on failure. ++ */ ++int kbase_csf_tiler_heap_init(struct kbase_context *kctx, ++ u32 chunk_size, u32 initial_chunks, u32 max_chunks, ++ u16 target_in_flight, u64 *gpu_heap_va, ++ u64 *first_chunk_va); ++ ++/** ++ * kbasep_cs_tiler_heap_term - Terminate a chunked tiler memory heap. ++ * ++ * This function will terminate a chunked tiler heap and cause all the chunks ++ * (initial and those added during out-of-memory processing) to be freed. ++ * It is the caller's responsibility to ensure no further operations on this ++ * heap will happen before calling this function. ++ * ++ * @kctx: Pointer to the kbase context in which the tiler heap was initialized. ++ * @gpu_heap_va: The GPU virtual address of the context that was set up for the ++ * tiler heap. ++ * ++ * Return: 0 if successful or a negative error code on failure. ++ */ ++int kbase_csf_tiler_heap_term(struct kbase_context *kctx, u64 gpu_heap_va); ++ ++/** ++ * kbase_csf_tiler_heap_alloc_new_chunk - Allocate a new chunk for tiler heap. ++ * ++ * This function will allocate a new chunk for the chunked tiler heap depending ++ * on the settings provided by userspace when the heap was created and the ++ * heap's statistics (like number of render passes in-flight). ++ * It would return an appropriate error code if a new chunk couldn't be ++ * allocated. ++ * ++ * @kctx: Pointer to the kbase context in which the tiler heap was initialized. ++ * @gpu_heap_va: GPU virtual address of the heap context. ++ * @nr_in_flight: Number of render passes that are in-flight, must not be zero. ++ * @new_chunk_ptr: Where to store the GPU virtual address & size of the new ++ * chunk allocated for the heap. ++ * ++ * Return: 0 if a new chunk was allocated otherwise an appropriate negative ++ * error code (like -EBUSY when a free chunk is expected to be ++ * available upon completion of a render pass and -EINVAL when ++ * invalid value was passed for one of the argument). ++ */ ++int kbase_csf_tiler_heap_alloc_new_chunk(struct kbase_context *kctx, ++ u64 gpu_heap_va, u32 nr_in_flight, u64 *new_chunk_ptr); ++#endif +diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap_debugfs.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap_debugfs.c +new file mode 100755 +index 000000000000..5d744b81fe4a +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap_debugfs.c +@@ -0,0 +1,107 @@ ++/* ++ * ++ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include "mali_kbase_csf_tiler_heap_debugfs.h" ++#include "mali_kbase_csf_tiler_heap_def.h" ++#include ++#include ++ ++#ifdef CONFIG_DEBUG_FS ++ ++/** ++ * kbasep_csf_tiler_heap_debugfs_show() - Print tiler heap information for per context ++ * ++ * @file: The seq_file for printing to ++ * @data: The debugfs dentry private data, a pointer to kbase_context ++ * ++ * Return: Negative error code or 0 on success. ++ */ ++static int kbasep_csf_tiler_heap_debugfs_show(struct seq_file *file, void *data) ++{ ++ struct kbase_context *kctx = file->private; ++ struct kbase_csf_tiler_heap_context *tiler_heaps_p = &kctx->csf.tiler_heaps; ++ struct kbase_csf_tiler_heap *heap; ++ struct kbase_csf_tiler_heap_chunk *chunk; ++ ++ seq_printf(file, "MALI_CSF_TILER_HEAP_DEBUGFS_VERSION: v%u\n", MALI_CSF_TILER_HEAP_DEBUGFS_VERSION); ++ ++ mutex_lock(&tiler_heaps_p->lock); ++ ++ list_for_each_entry(heap, &tiler_heaps_p->list, link) { ++ if (heap->kctx != kctx) ++ continue; ++ ++ seq_printf(file, "HEAP(gpu_va = 0x%llx):\n", heap->gpu_va); ++ seq_printf(file, "\tchunk_size = %u\n", heap->chunk_size); ++ seq_printf(file, "\tchunk_count = %u\n", heap->chunk_count); ++ seq_printf(file, "\tmax_chunks = %u\n", heap->max_chunks); ++ seq_printf(file, "\ttarget_in_flight = %u\n", heap->target_in_flight); ++ ++ list_for_each_entry(chunk, &heap->chunks_list, link) ++ seq_printf(file, "\t\tchunk gpu_va = 0x%llx\n", ++ chunk->gpu_va); ++ } ++ ++ mutex_unlock(&tiler_heaps_p->lock); ++ ++ return 0; ++} ++ ++static int kbasep_csf_tiler_heap_debugfs_open(struct inode *in, struct file *file) ++{ ++ return single_open(file, kbasep_csf_tiler_heap_debugfs_show, in->i_private); ++} ++ ++static const struct file_operations kbasep_csf_tiler_heap_debugfs_fops = { ++ .open = kbasep_csf_tiler_heap_debugfs_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = single_release, ++}; ++ ++void kbase_csf_tiler_heap_debugfs_init(struct kbase_context *kctx) ++{ ++ struct dentry *file; ++ ++ if (WARN_ON(!kctx || IS_ERR_OR_NULL(kctx->kctx_dentry))) ++ return; ++ ++ file = debugfs_create_file("tiler_heaps", 0444, kctx->kctx_dentry, ++ kctx, &kbasep_csf_tiler_heap_debugfs_fops); ++ ++ if (IS_ERR_OR_NULL(file)) { ++ dev_warn(kctx->kbdev->dev, ++ "Unable to create tiler heap debugfs entry"); ++ } ++} ++ ++ ++#else ++/* ++ * Stub functions for when debugfs is disabled ++ */ ++void kbase_csf_tiler_heap_debugfs_init(struct kbase_context *kctx) ++{ ++} ++ ++#endif /* CONFIG_DEBUG_FS */ ++ +diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap_debugfs.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap_debugfs.h +new file mode 100755 +index 000000000000..44c580d82068 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap_debugfs.h +@@ -0,0 +1,38 @@ ++/* ++ * ++ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_CSF_TILER_HEAP_DEBUGFS_H_ ++#define _KBASE_CSF_TILER_HEAP_DEBUGFS_H_ ++ ++/* Forward declaration */ ++struct kbase_context; ++ ++#define MALI_CSF_TILER_HEAP_DEBUGFS_VERSION 0 ++ ++/** ++ * kbase_csf_tiler_heap_debugfs_init() - Create a debugfs entry for per context tiler heap ++ * ++ * @kctx: The kbase_context for which to create the debugfs entry ++ */ ++void kbase_csf_tiler_heap_debugfs_init(struct kbase_context *kctx); ++ ++#endif /* _KBASE_CSF_TILER_HEAP_DEBUGFS_H_ */ +diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap_def.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap_def.h +new file mode 100755 +index 000000000000..1f9e208904a9 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap_def.h +@@ -0,0 +1,112 @@ ++/* ++ * ++ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_CSF_TILER_HEAP_DEF_H_ ++#define _KBASE_CSF_TILER_HEAP_DEF_H_ ++ ++#include ++ ++/* Size of a tiler heap chunk header, in bytes. */ ++#define CHUNK_HDR_SIZE ((size_t)64) ++ ++/* Bit-position of the next chunk's size when stored in a chunk header. */ ++#define CHUNK_HDR_NEXT_SIZE_POS (0) ++ ++/* Bit-position of the next chunk's address when stored in a chunk header. */ ++#define CHUNK_HDR_NEXT_ADDR_POS (12) ++ ++/* Bitmask of the next chunk's size when stored in a chunk header. */ ++#define CHUNK_HDR_NEXT_SIZE_MASK (((u64)1 << CHUNK_HDR_NEXT_ADDR_POS) - 1u) ++ ++/* Bitmask of the address of the next chunk when stored in a chunk header. */ ++#define CHUNK_HDR_NEXT_ADDR_MASK (~CHUNK_HDR_NEXT_SIZE_MASK) ++ ++/* Right-shift before storing the next chunk's size in a chunk header. */ ++#define CHUNK_HDR_NEXT_SIZE_ENCODE_SHIFT (12) ++ ++/* Right-shift before storing the next chunk's address in a chunk header. */ ++#define CHUNK_HDR_NEXT_ADDR_ENCODE_SHIFT (12) ++ ++/* Bitmask of valid chunk sizes. This is also the maximum chunk size, in bytes. ++ */ ++#define CHUNK_SIZE_MASK \ ++ ((CHUNK_HDR_NEXT_SIZE_MASK >> CHUNK_HDR_NEXT_SIZE_POS) << \ ++ CHUNK_HDR_NEXT_SIZE_ENCODE_SHIFT) ++ ++/* Bitmask of valid chunk addresses. This is also the highest address. */ ++#define CHUNK_ADDR_MASK \ ++ ((CHUNK_HDR_NEXT_ADDR_MASK >> CHUNK_HDR_NEXT_ADDR_POS) << \ ++ CHUNK_HDR_NEXT_ADDR_ENCODE_SHIFT) ++ ++/** ++ * struct kbase_csf_tiler_heap_chunk - A tiler heap chunk managed by the kernel ++ * ++ * Chunks are allocated upon initialization of a tiler heap or in response to ++ * out-of-memory events from the firmware. Chunks are always fully backed by ++ * physical memory to avoid the overhead of processing GPU page faults. The ++ * allocated GPU memory regions are linked together independent of the list of ++ * kernel objects of this type. ++ * ++ * @link: Link to this chunk in a list of chunks belonging to a ++ * @kbase_csf_tiler_heap. ++ * @region: Pointer to the GPU memory region allocated for the chunk. ++ * @gpu_va: GPU virtual address of the start of the memory region. ++ * This points to the header of the chunk and not to the low address ++ * of free memory within it. ++ */ ++struct kbase_csf_tiler_heap_chunk { ++ struct list_head link; ++ struct kbase_va_region *region; ++ u64 gpu_va; ++}; ++ ++/** ++ * struct kbase_csf_tiler_heap - A tiler heap managed by the kernel ++ * ++ * @kctx: Pointer to the kbase context with which this heap is ++ * associated. ++ * @link: Link to this heap in a list of tiler heaps belonging to ++ * the @kbase_csf_tiler_heap_context. ++ * @chunk_size: Size of each chunk, in bytes. Must be page-aligned. ++ * @chunk_count: The number of chunks currently allocated. Must not be ++ * zero or greater than @max_chunks. ++ * @max_chunks: The maximum number of chunks that the heap should be ++ * allowed to use. Must not be less than @chunk_count. ++ * @target_in_flight: Number of render-passes that the driver should attempt ++ * to keep in flight for which allocation of new chunks is ++ * allowed. Must not be zero. ++ * @gpu_va: The GPU virtual address of the heap context structure that ++ * was allocated for the firmware. This is also used to ++ * uniquely identify the heap. ++ * @chunks_list: Linked list of allocated chunks. ++ */ ++struct kbase_csf_tiler_heap { ++ struct kbase_context *kctx; ++ struct list_head link; ++ u32 chunk_size; ++ u32 chunk_count; ++ u32 max_chunks; ++ u16 target_in_flight; ++ u64 gpu_va; ++ struct list_head chunks_list; ++}; ++#endif /* !_KBASE_CSF_TILER_HEAP_DEF_H_ */ +diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_timeout.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_timeout.c +new file mode 100755 +index 000000000000..495ff2850500 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_timeout.c +@@ -0,0 +1,169 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++ ++#include "mali_kbase.h" ++#include "mali_kbase_config_defaults.h" ++#include "mali_kbase_csf_firmware.h" ++#include "mali_kbase_csf_timeout.h" ++#include "backend/gpu/mali_kbase_pm_internal.h" ++ ++/** ++ * set_timeout - set a new global progress timeout. ++ * ++ * @kbdev: Instance of a GPU platform device that implements a command ++ * stream front-end interface. ++ * @timeout: the maximum number of GPU cycles without forward progress to allow ++ * to elapse before terminating a GPU command queue group. ++ * ++ * Return: 0 on success, or negative on failure ++ * (e.g. -ERANGE if the requested timeout is too large). ++ */ ++static int set_timeout(struct kbase_device *const kbdev, u64 const timeout) ++{ ++ if (timeout > GLB_PROGRESS_TIMER_TIMEOUT_MAX) { ++ dev_err(kbdev->dev, "Timeout %llu is too large.\n", timeout); ++ return -ERANGE; ++ } ++ ++ dev_dbg(kbdev->dev, "New progress timeout: %llu cycles\n", timeout); ++ ++ atomic64_set(&kbdev->csf.progress_timeout, timeout); ++ ++ return 0; ++} ++ ++/** ++ * progress_timeout_store - Store the progress_timeout device attribute. ++ * @dev: The device that has the attribute. ++ * @attr: The attributes of the sysfs file. ++ * @buf: The value written to the sysfs file. ++ * @count: The number of bytes written to the sysfs file. ++ * ++ * This function is called when the progress_timeout sysfs file is written to. ++ * It checks the data written, and if valid updates the progress timeout value. ++ * ++ * Return: @count if the function succeeded. An error code on failure. ++ */ ++static ssize_t progress_timeout_store(struct device * const dev, ++ struct device_attribute * const attr, const char * const buf, ++ size_t const count) ++{ ++ struct kbase_device *const kbdev = dev_get_drvdata(dev); ++ int err; ++ u64 timeout; ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ err = kstrtou64(buf, 0, &timeout); ++ if (err) { ++ dev_err(kbdev->dev, ++ "Couldn't process progress_timeout write operation.\n" ++ "Use format \n"); ++ return err; ++ } ++ ++ err = set_timeout(kbdev, timeout); ++ if (!err) { ++ kbase_csf_scheduler_pm_active(kbdev); ++ ++ err = kbase_pm_wait_for_desired_state(kbdev); ++ if (!err) ++ err = kbase_csf_firmware_set_timeout(kbdev, timeout); ++ ++ kbase_csf_scheduler_pm_idle(kbdev); ++ } ++ ++ if (err) ++ return err; ++ ++ return count; ++} ++ ++/** ++ * progress_timeout_show - Show the progress_timeout device attribute. ++ * @dev: The device that has the attribute. ++ * @attr: The attributes of the sysfs file. ++ * @buf: The output buffer to receive the global timeout. ++ * ++ * This function is called to get the progress timeout value. ++ * ++ * Return: The number of bytes output to @buf. ++ */ ++static ssize_t progress_timeout_show(struct device * const dev, ++ struct device_attribute * const attr, char * const buf) ++{ ++ struct kbase_device *const kbdev = dev_get_drvdata(dev); ++ int err; ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ err = scnprintf(buf, PAGE_SIZE, "%llu\n", kbase_csf_timeout_get(kbdev)); ++ ++ return err; ++ ++} ++ ++static DEVICE_ATTR(progress_timeout, 0644, progress_timeout_show, ++ progress_timeout_store); ++ ++int kbase_csf_timeout_init(struct kbase_device *const kbdev) ++{ ++ u64 timeout = DEFAULT_PROGRESS_TIMEOUT; ++ int err; ++ ++#ifdef CONFIG_OF ++ err = of_property_read_u64(kbdev->dev->of_node, ++ "progress_timeout", &timeout); ++ if (!err) ++ dev_info(kbdev->dev, "Found progress_timeout = %llu in Devicetree\n", ++ timeout); ++#endif ++ ++ err = set_timeout(kbdev, timeout); ++ if (err) ++ return err; ++ ++ err = sysfs_create_file(&kbdev->dev->kobj, ++ &dev_attr_progress_timeout.attr); ++ if (err) ++ dev_err(kbdev->dev, "SysFS file creation failed\n"); ++ ++ return err; ++} ++ ++void kbase_csf_timeout_term(struct kbase_device * const kbdev) ++{ ++ sysfs_remove_file(&kbdev->dev->kobj, &dev_attr_progress_timeout.attr); ++} ++ ++u64 kbase_csf_timeout_get(struct kbase_device *const kbdev) ++{ ++ return atomic64_read(&kbdev->csf.progress_timeout); ++} +diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_timeout.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_timeout.h +new file mode 100755 +index 000000000000..d0156c09a60f +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_timeout.h +@@ -0,0 +1,69 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_CSF_TIMEOUT_H_ ++#define _KBASE_CSF_TIMEOUT_H_ ++ ++struct kbase_device; ++ ++/** ++ * kbase_csf_timeout_init - Initialize the progress timeout. ++ * ++ * @kbdev: Instance of a GPU platform device that implements a command ++ * stream front-end interface. Must be zero-initialized. ++ * ++ * The progress timeout is the number of GPU clock cycles allowed to elapse ++ * before the driver terminates a GPU command queue group in which a task is ++ * making no forward progress on an endpoint (e.g. a shader core). This function ++ * determines the initial value and also creates a sysfs file to allow the ++ * timeout to be reconfigured later. ++ * ++ * Reconfigures the global firmware interface to enable the current timeout. ++ * ++ * Return: 0 on success, or negative on failure. ++ */ ++int kbase_csf_timeout_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_csf_timeout_term - Terminate the progress timeout. ++ * ++ * @kbdev: Instance of a GPU platform device that implements a command ++ * stream front-end interface. ++ * ++ * Removes the sysfs file which allowed the timeout to be reconfigured. ++ * Does nothing if called on a zero-initialized object. ++ */ ++void kbase_csf_timeout_term(struct kbase_device *kbdev); ++ ++/** ++ * kbase_csf_timeout_get - get the current global progress timeout. ++ * ++ * @kbdev: Instance of a GPU platform device that implements a command ++ * stream front-end interface. ++ * ++ * Return: the maximum number of GPU cycles that is allowed to elapse without ++ * forward progress before the driver terminates a GPU command queue ++ * group. ++ */ ++u64 kbase_csf_timeout_get(struct kbase_device *const kbdev); ++ ++#endif /* _KBASE_CSF_TIMEOUT_H_ */ +diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tl_reader.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tl_reader.c +new file mode 100755 +index 000000000000..5079a8e5af8c +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tl_reader.c +@@ -0,0 +1,555 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include "mali_kbase_csf_tl_reader.h" ++ ++#include "mali_kbase_csf_trace_buffer.h" ++#include "mali_kbase_reset_gpu.h" ++ ++#include "tl/mali_kbase_tlstream.h" ++#include "tl/mali_kbase_tl_serialize.h" ++#include "tl/mali_kbase_tracepoints.h" ++ ++#include "mali_kbase_pm.h" ++#include "mali_kbase_hwaccess_time.h" ++ ++#include ++#include ++#include ++ ++#ifdef CONFIG_DEBUG_FS ++#include "tl/mali_kbase_timeline_priv.h" ++#include ++ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0)) ++#define DEFINE_DEBUGFS_ATTRIBUTE DEFINE_SIMPLE_ATTRIBUTE ++#endif ++#endif ++ ++/** Name of the CSFFW timeline tracebuffer. */ ++#define KBASE_CSFFW_TRACEBUFFER_NAME "timeline" ++/** Name of the timeline header metatadata */ ++#define KBASE_CSFFW_TIMELINE_HEADER_NAME "timeline_header" ++ ++/** ++ * CSFFW timeline message. ++ * ++ * @msg_id: Message ID. ++ * @timestamp: Timestamp of the event. ++ * @cycle_counter: Cycle number of the event. ++ * ++ * Contain fields that are common for all CSFFW timeline messages. ++ */ ++struct kbase_csffw_tl_message { ++ u32 msg_id; ++ u64 timestamp; ++ u64 cycle_counter; ++} __packed __aligned(4); ++ ++#ifdef CONFIG_DEBUG_FS ++static int kbase_csf_tl_debugfs_poll_interval_read(void *data, u64 *val) ++{ ++ struct kbase_device *kbdev = (struct kbase_device *)data; ++ struct kbase_csf_tl_reader *self = &kbdev->timeline->csf_tl_reader; ++ ++ *val = self->timer_interval; ++ ++ return 0; ++} ++ ++static int kbase_csf_tl_debugfs_poll_interval_write(void *data, u64 val) ++{ ++ struct kbase_device *kbdev = (struct kbase_device *)data; ++ struct kbase_csf_tl_reader *self = &kbdev->timeline->csf_tl_reader; ++ ++ if (val > KBASE_CSF_TL_READ_INTERVAL_MAX || val < KBASE_CSF_TL_READ_INTERVAL_MIN) { ++ return -EINVAL; ++ } ++ ++ self->timer_interval = (u32)val; ++ ++ return 0; ++} ++ ++DEFINE_DEBUGFS_ATTRIBUTE(kbase_csf_tl_poll_interval_fops, ++ kbase_csf_tl_debugfs_poll_interval_read, ++ kbase_csf_tl_debugfs_poll_interval_write, "%llu\n"); ++ ++void kbase_csf_tl_reader_debugfs_init(struct kbase_device *kbdev) ++{ ++ debugfs_create_file("csf_tl_poll_interval_in_ms", S_IRUGO | S_IWUSR, ++ kbdev->debugfs_instr_directory, kbdev, ++ &kbase_csf_tl_poll_interval_fops); ++} ++#endif ++ ++/** ++ * get_cpu_gpu_time() - Get current CPU and GPU timestamps. ++ * ++ * @kbdev: Kbase device. ++ * @cpu_ts: Output CPU timestamp. ++ * @gpu_ts: Output GPU timestamp. ++ * @gpu_cycle: Output GPU cycle counts. ++ */ ++static void get_cpu_gpu_time( ++ struct kbase_device *kbdev, ++ u64 *cpu_ts, ++ u64 *gpu_ts, ++ u64 *gpu_cycle) ++{ ++ struct timespec64 ts; ++ ++ kbase_pm_context_active(kbdev); ++ kbase_backend_get_gpu_time(kbdev, gpu_cycle, gpu_ts, &ts); ++ kbase_pm_context_idle(kbdev); ++ ++ if (cpu_ts) ++ *cpu_ts = ts.tv_sec * NSEC_PER_SEC + ts.tv_nsec; ++} ++ ++/** ++ * kbase_ts_converter_init() - Initialize system timestamp converter. ++ * ++ * @self: System Timestamp Converter instance. ++ * ++ * Return: Zero on success, -1 otherwise. ++ */ ++static int kbase_ts_converter_init( ++ struct kbase_ts_converter *self, ++ struct kbase_device *kbdev) ++{ ++ u64 cpu_ts = 0; ++ u64 gpu_ts = 0; ++ u64 freq; ++ u64 common_factor; ++ ++ get_cpu_gpu_time(kbdev, &cpu_ts, &gpu_ts, NULL); ++ freq = arch_timer_get_cntfrq(); ++ ++ if (!freq) { ++ dev_warn(kbdev->dev, "arch_timer_get_rate() is zero!"); ++ return -1; ++ } ++ ++ common_factor = gcd(NSEC_PER_SEC, freq); ++ ++ self->multiplier = div64_u64(NSEC_PER_SEC, common_factor); ++ self->divisor = div64_u64(freq, common_factor); ++ self->offset = ++ cpu_ts - div64_u64(gpu_ts * self->multiplier, self->divisor); ++ ++ return 0; ++} ++ ++/** ++ * kbase_ts_converter_convert() - Convert GPU timestamp to CPU timestamp. ++ * ++ * @self: System Timestamp Converter instance. ++ * @gpu_ts: System timestamp value to converter. ++ * ++ * Return: The CPU timestamp. ++ */ ++void kbase_ts_converter_convert( ++ const struct kbase_ts_converter *self, ++ u64 *gpu_ts) ++{ ++ u64 old_gpu_ts = *gpu_ts; ++ *gpu_ts = div64_u64(old_gpu_ts * self->multiplier, ++ self->divisor) + self->offset; ++} ++ ++/** ++ * tl_reader_overflow_notify() - Emit stream overflow tracepoint. ++ * ++ * @self: CSFFW TL Reader instance. ++ * @msg_buf_start: Start of the message. ++ * @msg_buf_end: End of the message buffer. ++ */ ++static void tl_reader_overflow_notify( ++ const struct kbase_csf_tl_reader *self, ++ u8 *const msg_buf_start, ++ u8 *const msg_buf_end) ++{ ++ struct kbase_device *kbdev = self->kbdev; ++ struct kbase_csffw_tl_message message = {0}; ++ ++ /* Reuse the timestamp and cycle count from current event if possible */ ++ if (msg_buf_start + sizeof(message) <= msg_buf_end) ++ memcpy(&message, msg_buf_start, sizeof(message)); ++ ++ KBASE_TLSTREAM_TL_KBASE_CSFFW_TLSTREAM_OVERFLOW( ++ kbdev, message.timestamp, message.cycle_counter); ++} ++ ++/** ++ * tl_reader_overflow_check() - Check if an overflow has happened ++ * ++ * @self: CSFFW TL Reader instance. ++ * @event_id: Incoming event id. ++ * ++ * Return: True, if an overflow has happened, False otherwise. ++ */ ++static bool tl_reader_overflow_check( ++ struct kbase_csf_tl_reader *self, ++ u16 event_id) ++{ ++ struct kbase_device *kbdev = self->kbdev; ++ bool has_overflow = false; ++ ++ /* 0 is a special event_id and reserved for the very first tracepoint ++ * after reset, we should skip overflow check when reset happened. ++ */ ++ if (event_id != 0) { ++ has_overflow = self->got_first_event ++ && self->expected_event_id != event_id; ++ ++ if (has_overflow) ++ dev_warn(kbdev->dev, ++ "CSFFW overflow, event_id: %u, expected: %u.", ++ event_id, self->expected_event_id); ++ } ++ ++ self->got_first_event = true; ++ self->expected_event_id = event_id + 1; ++ /* When event_id reaches its max value, it skips 0 and wraps to 1. */ ++ if (self->expected_event_id == 0) ++ self->expected_event_id++; ++ ++ return has_overflow; ++} ++ ++/** ++ * tl_reader_reset() - Reset timeline tracebuffer reader state machine. ++ * ++ * @self: CSFFW TL Reader instance. ++ * ++ * Reset the reader to the default state, i.e. set all the ++ * mutable fields to zero. ++ */ ++static void tl_reader_reset(struct kbase_csf_tl_reader *self) ++{ ++ self->got_first_event = false; ++ self->is_active = false; ++ self->expected_event_id = 0; ++ self->tl_header.btc = 0; ++} ++ ++void kbase_csf_tl_reader_flush_buffer(struct kbase_csf_tl_reader *self) ++{ ++ struct kbase_device *kbdev = self->kbdev; ++ struct kbase_tlstream *stream = self->stream; ++ ++ u8 *read_buffer = self->read_buffer; ++ const size_t read_buffer_size = sizeof(self->read_buffer); ++ ++ u32 bytes_read; ++ u8 *csffw_data_begin; ++ u8 *csffw_data_end; ++ u8 *csffw_data_it; ++ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&self->read_lock, flags); ++ ++ /* If not running, early exit. */ ++ if (!self->is_active) { ++ spin_unlock_irqrestore(&self->read_lock, flags); ++ return; ++ } ++ ++ /* Copying the whole buffer in a single shot. We assume ++ * that the buffer will not contain partially written messages. ++ */ ++ bytes_read = kbase_csf_firmware_trace_buffer_read_data( ++ self->trace_buffer, read_buffer, read_buffer_size); ++ csffw_data_begin = read_buffer; ++ csffw_data_end = read_buffer + bytes_read; ++ ++ for (csffw_data_it = csffw_data_begin; ++ csffw_data_it < csffw_data_end;) { ++ u32 event_header; ++ u16 event_id; ++ u16 event_size; ++ unsigned long acq_flags; ++ char *buffer; ++ ++ /* Can we safely read event_id? */ ++ if (csffw_data_it + sizeof(event_header) > csffw_data_end) { ++ dev_warn( ++ kbdev->dev, ++ "Unable to parse CSFFW tracebuffer event header."); ++ break; ++ } ++ ++ /* Read and parse the event header. */ ++ memcpy(&event_header, csffw_data_it, sizeof(event_header)); ++ event_id = (event_header >> 0) & 0xFFFF; ++ event_size = (event_header >> 16) & 0xFFFF; ++ csffw_data_it += sizeof(event_header); ++ ++ /* Detect if an overflow has happened. */ ++ if (tl_reader_overflow_check(self, event_id)) ++ tl_reader_overflow_notify(self, ++ csffw_data_it, ++ csffw_data_end); ++ ++ /* Can we safely read the message body? */ ++ if (csffw_data_it + event_size > csffw_data_end) { ++ dev_warn(kbdev->dev, ++ "event_id: %u, can't read with event_size: %u.", ++ event_id, event_size); ++ break; ++ } ++ ++ /* Convert GPU timestamp to CPU timestamp. */ ++ { ++ struct kbase_csffw_tl_message *msg = ++ (struct kbase_csffw_tl_message *) csffw_data_it; ++ kbase_ts_converter_convert( ++ &self->ts_converter, ++ &msg->timestamp); ++ } ++ ++ /* Copy the message out to the tl_stream. */ ++ buffer = kbase_tlstream_msgbuf_acquire( ++ stream, event_size, &acq_flags); ++ kbasep_serialize_bytes(buffer, 0, csffw_data_it, event_size); ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++ csffw_data_it += event_size; ++ } ++ ++ spin_unlock_irqrestore(&self->read_lock, flags); ++} ++ ++static void kbasep_csf_tl_reader_read_callback(struct timer_list *timer) ++{ ++ struct kbase_csf_tl_reader *self = ++ container_of(timer, struct kbase_csf_tl_reader, read_timer); ++ ++ int rcode; ++ ++ kbase_csf_tl_reader_flush_buffer(self); ++ ++ rcode = mod_timer(&self->read_timer, ++ jiffies + msecs_to_jiffies(self->timer_interval)); ++ ++ CSTD_UNUSED(rcode); ++} ++ ++/** ++ * tl_reader_init_late() - Late CSFFW TL Reader initialization. ++ * ++ * @self: CSFFW TL Reader instance. ++ * @kbdev: Kbase device. ++ * ++ * Late initialization is done once at kbase_csf_tl_reader_start() time. ++ * This is because the firmware image is not parsed ++ * by the kbase_csf_tl_reader_init() time. ++ * ++ * Return: Zero on success, -1 otherwise. ++ */ ++static int tl_reader_init_late( ++ struct kbase_csf_tl_reader *self, ++ struct kbase_device *kbdev) ++{ ++ struct firmware_trace_buffer *tb; ++ size_t hdr_size = 0; ++ const char *hdr = NULL; ++ ++ if (self->kbdev) ++ return 0; ++ ++ tb = kbase_csf_firmware_get_trace_buffer( ++ kbdev, KBASE_CSFFW_TRACEBUFFER_NAME); ++ hdr = kbase_csf_firmware_get_timeline_metadata( ++ kbdev, KBASE_CSFFW_TIMELINE_HEADER_NAME, &hdr_size); ++ ++ if (!tb) { ++ dev_warn( ++ kbdev->dev, ++ "'%s' tracebuffer is not present in the firmware image.", ++ KBASE_CSFFW_TRACEBUFFER_NAME); ++ return -1; ++ } ++ ++ if (!hdr) { ++ dev_warn( ++ kbdev->dev, ++ "'%s' timeline metadata is not present in the firmware image.", ++ KBASE_CSFFW_TIMELINE_HEADER_NAME); ++ return -1; ++ } ++ ++ if (kbase_ts_converter_init(&self->ts_converter, kbdev)) { ++ return -1; ++ } ++ ++ self->kbdev = kbdev; ++ self->trace_buffer = tb; ++ self->tl_header.data = hdr; ++ self->tl_header.size = hdr_size; ++ ++ return 0; ++} ++ ++/** ++ * tl_reader_update_enable_bit() - Update the first bit of a CSFFW tracebuffer. ++ * ++ * @self: CSFFW TL Reader instance. ++ * @value: The value to set. ++ * ++ * Update the first bit of a CSFFW tracebufer and then reset the GPU. ++ * This is to make these changes visible to the MCU. ++ * ++ * Return: 0 on success, -EAGAIN if a GPU reset was in progress. ++ */ ++static int tl_reader_update_enable_bit( ++ struct kbase_csf_tl_reader *self, ++ bool value) ++{ ++ struct kbase_device *kbdev = self->kbdev; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ /* If there is already a GPU reset pending then inform ++ * the User to retry the update. ++ */ ++ if (kbase_reset_gpu_silent(kbdev)) { ++ spin_unlock_irqrestore( ++ &kbdev->hwaccess_lock, flags); ++ dev_warn( ++ kbdev->dev, ++ "GPU reset already in progress when enabling firmware timeline."); ++ return -EAGAIN; ++ } ++ ++ /* GPU reset request has been placed, now update the ++ * firmware image. GPU reset will take place only after ++ * hwaccess_lock is released. ++ */ ++ kbase_csf_firmware_trace_buffer_update_trace_enable_bit( ++ self->trace_buffer, 0, value); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ return 0; ++} ++ ++void kbase_csf_tl_reader_init(struct kbase_csf_tl_reader *self, ++ struct kbase_tlstream *stream) ++{ ++ self->timer_interval = KBASE_CSF_TL_READ_INTERVAL_DEFAULT; ++ ++ kbase_timer_setup(&self->read_timer, ++ kbasep_csf_tl_reader_read_callback); ++ ++ self->stream = stream; ++ ++ /* This will be initialized by tl_reader_init_late() */ ++ self->kbdev = NULL; ++ self->trace_buffer = NULL; ++ self->tl_header.data = NULL; ++ self->tl_header.size = 0; ++ ++ spin_lock_init(&self->read_lock); ++ ++ tl_reader_reset(self); ++} ++ ++void kbase_csf_tl_reader_term(struct kbase_csf_tl_reader *self) ++{ ++ del_timer_sync(&self->read_timer); ++} ++ ++int kbase_csf_tl_reader_start(struct kbase_csf_tl_reader *self, ++ struct kbase_device *kbdev) ++{ ++ int rcode; ++ ++ /* If already running, early exit. */ ++ if (self->is_active) ++ return 0; ++ ++ if (tl_reader_init_late(self, kbdev)) { ++#if defined(CONFIG_MALI_BIFROST_NO_MALI) ++ dev_warn( ++ kbdev->dev, ++ "CSFFW timeline is not available for MALI_BIFROST_NO_MALI builds!"); ++ return 0; ++#else ++ return -EINVAL; ++#endif ++ } ++ ++ tl_reader_reset(self); ++ ++ self->is_active = true; ++ /* Set bytes to copy to the header size. This is to trigger copying ++ * of the header to the user space. ++ */ ++ self->tl_header.btc = self->tl_header.size; ++ ++ /* Enable the tracebuffer on the CSFFW side. */ ++ rcode = tl_reader_update_enable_bit(self, true); ++ if (rcode != 0) ++ return rcode; ++ ++ rcode = mod_timer(&self->read_timer, ++ jiffies + msecs_to_jiffies(self->timer_interval)); ++ ++ return 0; ++} ++ ++void kbase_csf_tl_reader_stop(struct kbase_csf_tl_reader *self) ++{ ++ unsigned long flags; ++ ++ /* If is not running, early exit. */ ++ if (!self->is_active) ++ return; ++ ++ /* Disable the tracebuffer on the CSFFW side. */ ++ tl_reader_update_enable_bit(self, false); ++ ++ del_timer_sync(&self->read_timer); ++ ++ spin_lock_irqsave(&self->read_lock, flags); ++ ++ tl_reader_reset(self); ++ ++ spin_unlock_irqrestore(&self->read_lock, flags); ++} ++ ++void kbase_csf_tl_reader_reset(struct kbase_csf_tl_reader *self) ++{ ++ u64 gpu_cycle = 0; ++ struct kbase_device *kbdev = self->kbdev; ++ ++ if (!kbdev) ++ return; ++ ++ kbase_csf_tl_reader_flush_buffer(self); ++ ++ get_cpu_gpu_time(kbdev, NULL, NULL, &gpu_cycle); ++ KBASE_TLSTREAM_TL_KBASE_CSFFW_RESET(kbdev, gpu_cycle); ++} +diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tl_reader.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tl_reader.h +new file mode 100755 +index 000000000000..f5ce9d629f55 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tl_reader.h +@@ -0,0 +1,181 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_CSFFW_TL_READER_H_ ++#define _KBASE_CSFFW_TL_READER_H_ ++ ++#include ++#include ++#include ++ ++/** The number of pages used for CSFFW trace buffer. Can be tweaked. */ ++#define KBASE_CSF_TL_BUFFER_NR_PAGES 4 ++/** CSFFW Timeline read polling minimum period in milliseconds. */ ++#define KBASE_CSF_TL_READ_INTERVAL_MIN 20 ++/** CSFFW Timeline read polling default period in milliseconds. */ ++#define KBASE_CSF_TL_READ_INTERVAL_DEFAULT 200 ++/** CSFFW Timeline read polling maximum period in milliseconds. */ ++#define KBASE_CSF_TL_READ_INTERVAL_MAX (60*1000) ++ ++struct firmware_trace_buffer; ++struct kbase_tlstream; ++struct kbase_device; ++ ++/** ++ * System timestamp to CPU timestamp converter state. ++ * ++ * @multiplier: Numerator of the converter's fraction. ++ * @divisor: Denominator of the converter's fraction. ++ * @offset: Converter's offset term. ++ * ++ * According to Generic timer spec, system timer: ++ * - Increments at a fixed frequency ++ * - Starts operating from zero ++ * ++ * Hence CPU time is a linear function of System Time. ++ * ++ * CPU_ts = alpha * SYS_ts + beta ++ * ++ * Where ++ * - alpha = 10^9/SYS_ts_freq ++ * - beta is calculated by two timer samples taken at the same time: ++ * beta = CPU_ts_s - SYS_ts_s * alpha ++ * ++ * Since alpha is a rational number, we minimizing possible ++ * rounding error by simplifying the ratio. Thus alpha is stored ++ * as a simple `multiplier / divisor` ratio. ++ * ++ */ ++struct kbase_ts_converter { ++ u64 multiplier; ++ u64 divisor; ++ s64 offset; ++}; ++ ++/** ++ * struct kbase_csf_tl_reader - CSFFW timeline reader state. ++ * ++ * @read_timer: Timer used for periodical tracebufer reading. ++ * @timer_interval: Timer polling period in milliseconds. ++ * @stream: Timeline stream where to the tracebuffer content ++ * is copied. ++ * @kbdev: KBase device. ++ * @trace_buffer: CSF Firmware timeline tracebuffer. ++ * @tl_header.data: CSFFW Timeline header content. ++ * @tl_header.size: CSFFW Timeline header size. ++ * @tl_header.btc: CSFFW Timeline header remaining bytes to copy to ++ * the user space. ++ * @ts_converter: Timestamp converter state. ++ * @got_first_event: True, if a CSFFW timelime session has been enabled ++ * and the first event was received. ++ * @is_active: True, if a CSFFW timelime session has been enabled. ++ * @expected_event_id: The last 16 bit event ID received from CSFFW. It ++ * is only valid when got_first_event is true. ++ * @read_buffer: Temporary buffer used for CSFFW timeline data ++ * reading from the tracebufer. ++ */ ++struct kbase_csf_tl_reader { ++ struct timer_list read_timer; ++ u32 timer_interval; ++ struct kbase_tlstream *stream; ++ ++ struct kbase_device *kbdev; ++ struct firmware_trace_buffer *trace_buffer; ++ struct { ++ const char *data; ++ size_t size; ++ size_t btc; ++ } tl_header; ++ struct kbase_ts_converter ts_converter; ++ ++ bool got_first_event; ++ bool is_active; ++ u16 expected_event_id; ++ ++ u8 read_buffer[PAGE_SIZE * KBASE_CSF_TL_BUFFER_NR_PAGES]; ++ spinlock_t read_lock; ++}; ++ ++/** ++ * kbase_csf_tl_reader_init() - Initialize CSFFW Timelime Stream Reader. ++ * ++ * @self: CSFFW TL Reader instance. ++ * @stream: Destination timeline stream. ++ */ ++void kbase_csf_tl_reader_init(struct kbase_csf_tl_reader *self, ++ struct kbase_tlstream *stream); ++ ++/** ++ * kbase_csf_tl_reader_term() - Terminate CSFFW Timelime Stream Reader. ++ * ++ * @self: CSFFW TL Reader instance. ++ */ ++void kbase_csf_tl_reader_term(struct kbase_csf_tl_reader *self); ++ ++/** ++ * kbase_csf_tl_reader_flush_buffer() - ++ * Flush trace from buffer into CSFFW timeline stream. ++ * ++ * @self: CSFFW TL Reader instance. ++ */ ++ ++void kbase_csf_tl_reader_flush_buffer(struct kbase_csf_tl_reader *self); ++ ++/** ++ * kbase_csf_tl_reader_start() - ++ * Start asynchronous copying of CSFFW timeline stream. ++ * ++ * @self: CSFFW TL Reader instance. ++ * @kbdev: Kbase device. ++ * ++ * Return: zero on success, a negative error code otherwise. ++ */ ++int kbase_csf_tl_reader_start(struct kbase_csf_tl_reader *self, ++ struct kbase_device *kbdev); ++ ++/** ++ * kbase_csf_tl_reader_stop() - ++ * Stop asynchronous copying of CSFFW timeline stream. ++ * ++ * @self: CSFFW TL Reader instance. ++ */ ++void kbase_csf_tl_reader_stop(struct kbase_csf_tl_reader *self); ++ ++#ifdef CONFIG_DEBUG_FS ++/** ++ * kbase_csf_tl_reader_debugfs_init() - ++ * Initialize debugfs for CSFFW Timelime Stream Reader. ++ * ++ * @kbdev: Kbase device. ++ */ ++void kbase_csf_tl_reader_debugfs_init(struct kbase_device *kbdev); ++#endif ++ ++/** ++ * kbase_csf_tl_reader_reset() - ++ * Reset CSFFW timeline reader, it should be called before reset CSFFW. ++ * ++ * @self: CSFFW TL Reader instance. ++ */ ++void kbase_csf_tl_reader_reset(struct kbase_csf_tl_reader *self); ++ ++#endif /* _KBASE_CSFFW_TL_READER_H_ */ +diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_trace_buffer.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_trace_buffer.c +new file mode 100755 +index 000000000000..4d68766b8b9a +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_trace_buffer.c +@@ -0,0 +1,623 @@ ++/* ++ * ++ * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include "mali_kbase.h" ++#include "mali_kbase_defs.h" ++#include "mali_kbase_csf_firmware.h" ++#include "mali_kbase_csf_trace_buffer.h" ++#include "mali_kbase_reset_gpu.h" ++#include "mali_kbase_csf_tl_reader.h" ++ ++#include ++#include ++ ++/** ++ * struct firmware_trace_buffer - Trace Buffer within the MCU firmware ++ * ++ * The firmware relays information to the host by writing on memory buffers ++ * which are allocated and partially configured by the host. These buffers ++ * are called Trace Buffers: each of them has a specific purpose and is ++ * identified by a name and a set of memory addresses where the host can ++ * set pointers to host-allocated structures. ++ * ++ * @kbdev: Pointer to the Kbase device. ++ * @node: List head linking all trace buffers to ++ * kbase_device:csf.firmware_trace_buffers ++ * @data_mapping: MCU shared memory mapping used for the data buffer. ++ * @type: The type of the trace buffer. ++ * @trace_enable_entry_count: Number of Trace Enable bits. ++ * @gpu_va: Structure containing all the Firmware addresses ++ * that are accessed by the MCU. ++ * @size_address: The address where the MCU shall read the size of ++ * the data buffer. ++ * @insert_address: The address that shall be dereferenced by the MCU ++ * to write the Insert offset. ++ * @extract_address: The address that shall be dereferenced by the MCU ++ * to read the Extract offset. ++ * @data_address: The address that shall be dereferenced by the MCU ++ * to write the Trace Buffer. ++ * @trace_enable: The address where the MCU shall read the array of ++ * Trace Enable bits describing which trace points ++ * and features shall be enabled. ++ * @cpu_va: Structure containing CPU addresses of variables which ++ * are permanently mapped on the CPU address space. ++ * @insert_cpu_va: CPU virtual address of the Insert variable. ++ * @extract_cpu_va: CPU virtual address of the Extract variable. ++ * @num_pages: Size of the data buffer, in pages. ++ * @trace_enable_init_mask: Initial value for the trace enable bit mask. ++ * @name: NULL terminated string which contains the name of the trace buffer. ++ */ ++struct firmware_trace_buffer { ++ struct kbase_device *kbdev; ++ struct list_head node; ++ struct kbase_csf_mapping data_mapping; ++ u32 type; ++ u32 trace_enable_entry_count; ++ struct gpu_va { ++ u32 size_address; ++ u32 insert_address; ++ u32 extract_address; ++ u32 data_address; ++ u32 trace_enable; ++ } gpu_va; ++ struct cpu_va { ++ u32 *insert_cpu_va; ++ u32 *extract_cpu_va; ++ } cpu_va; ++ u32 num_pages; ++ u32 trace_enable_init_mask[CSF_FIRMWARE_TRACE_ENABLE_INIT_MASK_MAX]; ++ char name[1]; /* this field must be last */ ++}; ++ ++/** ++ * struct firmware_trace_buffer_data - Configuration data for trace buffers ++ * ++ * Describe how to set up a trace buffer interface. ++ * Trace buffers are identified by name and they require a data buffer and ++ * an initial mask of values for the trace enable bits. ++ * ++ * @name: Name identifier of the trace buffer ++ * @trace_enable_init_mask: Initial value to assign to the trace enable bits ++ * @size: Size of the data buffer to allocate for the trace buffer, in pages. ++ * The size of a data buffer must always be a power of 2. ++ */ ++struct firmware_trace_buffer_data { ++ char name[64]; ++ u32 trace_enable_init_mask[CSF_FIRMWARE_TRACE_ENABLE_INIT_MASK_MAX]; ++ size_t size; ++}; ++ ++/** ++ * Table of configuration data for trace buffers. ++ * ++ * This table contains the configuration data for the trace buffers that are ++ * expected to be parsed from the firmware. ++ */ ++static const struct firmware_trace_buffer_data ++trace_buffer_data[] = { ++#ifndef MALI_KBASE_BUILD ++ { "fwutf", {0}, 1 }, ++#endif ++ { FW_TRACE_BUF_NAME, {0}, 4 }, ++ { "benchmark", {0}, 2 }, ++ { "timeline", {0}, KBASE_CSF_TL_BUFFER_NR_PAGES }, ++}; ++ ++int kbase_csf_firmware_trace_buffers_init(struct kbase_device *kbdev) ++{ ++ struct firmware_trace_buffer *trace_buffer; ++ int ret = 0; ++ u32 mcu_rw_offset = 0, mcu_write_offset = 0; ++ const u32 cache_line_alignment = kbase_get_cache_line_alignment(kbdev); ++ ++ if (list_empty(&kbdev->csf.firmware_trace_buffers.list)) { ++ dev_dbg(kbdev->dev, "No trace buffers to initialise\n"); ++ return 0; ++ } ++ ++ /* GPU-readable,writable memory used for Extract variables */ ++ ret = kbase_csf_firmware_mcu_shared_mapping_init( ++ kbdev, 1, PROT_WRITE, ++ KBASE_REG_GPU_RD | KBASE_REG_GPU_WR, ++ &kbdev->csf.firmware_trace_buffers.mcu_rw); ++ if (ret != 0) { ++ dev_err(kbdev->dev, "Failed to map GPU-rw MCU shared memory\n"); ++ goto out; ++ } ++ ++ /* GPU-writable memory used for Insert variables */ ++ ret = kbase_csf_firmware_mcu_shared_mapping_init( ++ kbdev, 1, PROT_READ, KBASE_REG_GPU_WR, ++ &kbdev->csf.firmware_trace_buffers.mcu_write); ++ if (ret != 0) { ++ dev_err(kbdev->dev, "Failed to map GPU-writable MCU shared memory\n"); ++ goto out; ++ } ++ ++ list_for_each_entry(trace_buffer, &kbdev->csf.firmware_trace_buffers.list, node) { ++ u32 extract_gpu_va, insert_gpu_va, data_buffer_gpu_va, ++ trace_enable_size_dwords; ++ u32 *extract_cpu_va, *insert_cpu_va; ++ unsigned int i; ++ ++ /* GPU-writable data buffer for the individual trace buffer */ ++ ret = kbase_csf_firmware_mcu_shared_mapping_init( ++ kbdev, trace_buffer->num_pages, PROT_READ, KBASE_REG_GPU_WR, ++ &trace_buffer->data_mapping); ++ if (ret) { ++ dev_err(kbdev->dev, "Failed to map GPU-writable MCU shared memory for a trace buffer\n"); ++ goto out; ++ } ++ ++ extract_gpu_va = ++ (kbdev->csf.firmware_trace_buffers.mcu_rw.va_reg->start_pfn << PAGE_SHIFT) + ++ mcu_rw_offset; ++ extract_cpu_va = (u32*)( ++ kbdev->csf.firmware_trace_buffers.mcu_rw.cpu_addr + ++ mcu_rw_offset); ++ insert_gpu_va = ++ (kbdev->csf.firmware_trace_buffers.mcu_write.va_reg->start_pfn << PAGE_SHIFT) + ++ mcu_write_offset; ++ insert_cpu_va = (u32*)( ++ kbdev->csf.firmware_trace_buffers.mcu_write.cpu_addr + ++ mcu_write_offset); ++ data_buffer_gpu_va = ++ (trace_buffer->data_mapping.va_reg->start_pfn << PAGE_SHIFT); ++ ++ /* Initialize the Extract variable */ ++ *extract_cpu_va = 0; ++ ++ /* Each FW address shall be mapped and set individually, as we can't ++ * assume anything about their location in the memory address space. ++ */ ++ kbase_csf_update_firmware_memory( ++ kbdev, trace_buffer->gpu_va.data_address, data_buffer_gpu_va); ++ kbase_csf_update_firmware_memory( ++ kbdev, trace_buffer->gpu_va.insert_address, insert_gpu_va); ++ kbase_csf_update_firmware_memory( ++ kbdev, trace_buffer->gpu_va.extract_address, extract_gpu_va); ++ kbase_csf_update_firmware_memory( ++ kbdev, trace_buffer->gpu_va.size_address, ++ trace_buffer->num_pages << PAGE_SHIFT); ++ ++ trace_enable_size_dwords = ++ (trace_buffer->trace_enable_entry_count + 31) >> 5; ++ ++ for (i = 0; i < trace_enable_size_dwords; i++) { ++ kbase_csf_update_firmware_memory( ++ kbdev, trace_buffer->gpu_va.trace_enable + i*4, ++ trace_buffer->trace_enable_init_mask[i]); ++ } ++ ++ /* Store CPU virtual addresses for permanently mapped variables */ ++ trace_buffer->cpu_va.insert_cpu_va = insert_cpu_va; ++ trace_buffer->cpu_va.extract_cpu_va = extract_cpu_va; ++ ++ /* Update offsets */ ++ mcu_write_offset += cache_line_alignment; ++ mcu_rw_offset += cache_line_alignment; ++ } ++ ++out: ++ return ret; ++} ++ ++void kbase_csf_firmware_trace_buffers_term(struct kbase_device *kbdev) ++{ ++ if (list_empty(&kbdev->csf.firmware_trace_buffers.list)) ++ return; ++ ++ while (!list_empty(&kbdev->csf.firmware_trace_buffers.list)) { ++ struct firmware_trace_buffer *trace_buffer; ++ ++ trace_buffer = list_first_entry(&kbdev->csf.firmware_trace_buffers.list, ++ struct firmware_trace_buffer, node); ++ kbase_csf_firmware_mcu_shared_mapping_term(kbdev, &trace_buffer->data_mapping); ++ list_del(&trace_buffer->node); ++ ++ kfree(trace_buffer); ++ } ++ ++ kbase_csf_firmware_mcu_shared_mapping_term( ++ kbdev, &kbdev->csf.firmware_trace_buffers.mcu_rw); ++ kbase_csf_firmware_mcu_shared_mapping_term( ++ kbdev, &kbdev->csf.firmware_trace_buffers.mcu_write); ++} ++ ++int kbase_csf_firmware_parse_trace_buffer_entry(struct kbase_device *kbdev, ++ const u32 *entry, unsigned int size) ++{ ++ const char *name = (char *)&entry[7]; ++ const unsigned int name_len = size - TRACE_BUFFER_ENTRY_NAME_OFFSET; ++ struct firmware_trace_buffer *trace_buffer; ++ unsigned int i; ++ ++ /* Allocate enough space for struct firmware_trace_buffer and the ++ * trace buffer name (with NULL termination). ++ */ ++ trace_buffer = ++ kmalloc(sizeof(*trace_buffer) + name_len + 1, GFP_KERNEL); ++ ++ if (!trace_buffer) ++ return -ENOMEM; ++ ++ memcpy(&trace_buffer->name, name, name_len); ++ trace_buffer->name[name_len] = '\0'; ++ ++ for (i = 0; i < ARRAY_SIZE(trace_buffer_data); i++) { ++ if (!strcmp(trace_buffer_data[i].name, trace_buffer->name)) { ++ unsigned int j; ++ ++ trace_buffer->kbdev = kbdev; ++ trace_buffer->type = entry[0]; ++ trace_buffer->gpu_va.size_address = entry[1]; ++ trace_buffer->gpu_va.insert_address = entry[2]; ++ trace_buffer->gpu_va.extract_address = entry[3]; ++ trace_buffer->gpu_va.data_address = entry[4]; ++ trace_buffer->gpu_va.trace_enable = entry[5]; ++ trace_buffer->trace_enable_entry_count = entry[6]; ++ trace_buffer->num_pages = trace_buffer_data[i].size; ++ ++ for (j = 0; j < CSF_FIRMWARE_TRACE_ENABLE_INIT_MASK_MAX; j++) { ++ trace_buffer->trace_enable_init_mask[j] = ++ trace_buffer_data[i].trace_enable_init_mask[j]; ++ } ++ break; ++ } ++ } ++ ++ if (i < ARRAY_SIZE(trace_buffer_data)) { ++ list_add(&trace_buffer->node, &kbdev->csf.firmware_trace_buffers.list); ++ dev_dbg(kbdev->dev, "Trace buffer '%s'", trace_buffer->name); ++ } else { ++ dev_dbg(kbdev->dev, "Unknown trace buffer '%s'", trace_buffer->name); ++ kfree(trace_buffer); ++ } ++ ++ return 0; ++} ++ ++void kbase_csf_firmware_reload_trace_buffers_data(struct kbase_device *kbdev) ++{ ++ struct firmware_trace_buffer *trace_buffer; ++ u32 mcu_rw_offset = 0, mcu_write_offset = 0; ++ const u32 cache_line_alignment = kbase_get_cache_line_alignment(kbdev); ++ ++ list_for_each_entry(trace_buffer, &kbdev->csf.firmware_trace_buffers.list, node) { ++ u32 extract_gpu_va, insert_gpu_va, data_buffer_gpu_va, ++ trace_enable_size_dwords; ++ u32 *extract_cpu_va, *insert_cpu_va; ++ unsigned int i; ++ ++ /* Rely on the fact that all required mappings already exist */ ++ extract_gpu_va = ++ (kbdev->csf.firmware_trace_buffers.mcu_rw.va_reg->start_pfn << PAGE_SHIFT) + ++ mcu_rw_offset; ++ extract_cpu_va = (u32*)( ++ kbdev->csf.firmware_trace_buffers.mcu_rw.cpu_addr + ++ mcu_rw_offset); ++ insert_gpu_va = ++ (kbdev->csf.firmware_trace_buffers.mcu_write.va_reg->start_pfn << PAGE_SHIFT) + ++ mcu_write_offset; ++ insert_cpu_va = (u32*)( ++ kbdev->csf.firmware_trace_buffers.mcu_write.cpu_addr + ++ mcu_write_offset); ++ data_buffer_gpu_va = ++ (trace_buffer->data_mapping.va_reg->start_pfn << PAGE_SHIFT); ++ ++ /* Notice that the function only re-updates firmware memory locations ++ * with information that allows access to the trace buffers without ++ * really resetting their state. For instance, the Insert offset will ++ * not change and, as a consequence, the Extract offset is not going ++ * to be reset to keep consistency. ++ */ ++ ++ /* Each FW address shall be mapped and set individually, as we can't ++ * assume anything about their location in the memory address space. ++ */ ++ kbase_csf_update_firmware_memory( ++ kbdev, trace_buffer->gpu_va.data_address, data_buffer_gpu_va); ++ kbase_csf_update_firmware_memory( ++ kbdev, trace_buffer->gpu_va.insert_address, insert_gpu_va); ++ kbase_csf_update_firmware_memory( ++ kbdev, trace_buffer->gpu_va.extract_address, extract_gpu_va); ++ kbase_csf_update_firmware_memory( ++ kbdev, trace_buffer->gpu_va.size_address, ++ trace_buffer->num_pages << PAGE_SHIFT); ++ ++ trace_enable_size_dwords = ++ (trace_buffer->trace_enable_entry_count + 31) >> 5; ++ ++ for (i = 0; i < trace_enable_size_dwords; i++) { ++ kbase_csf_update_firmware_memory( ++ kbdev, trace_buffer->gpu_va.trace_enable + i*4, ++ trace_buffer->trace_enable_init_mask[i]); ++ } ++ ++ /* Store CPU virtual addresses for permanently mapped variables, ++ * as they might have slightly changed. ++ */ ++ trace_buffer->cpu_va.insert_cpu_va = insert_cpu_va; ++ trace_buffer->cpu_va.extract_cpu_va = extract_cpu_va; ++ ++ /* Update offsets */ ++ mcu_write_offset += cache_line_alignment; ++ mcu_rw_offset += cache_line_alignment; ++ } ++} ++ ++struct firmware_trace_buffer *kbase_csf_firmware_get_trace_buffer( ++ struct kbase_device *kbdev, const char *name) ++{ ++ struct firmware_trace_buffer *trace_buffer; ++ ++ list_for_each_entry(trace_buffer, &kbdev->csf.firmware_trace_buffers.list, node) { ++ if (!strcmp(trace_buffer->name, name)) ++ return trace_buffer; ++ } ++ ++ return NULL; ++} ++EXPORT_SYMBOL(kbase_csf_firmware_get_trace_buffer); ++ ++unsigned int kbase_csf_firmware_trace_buffer_get_trace_enable_bits_count( ++ const struct firmware_trace_buffer *trace_buffer) ++{ ++ return trace_buffer->trace_enable_entry_count; ++} ++EXPORT_SYMBOL(kbase_csf_firmware_trace_buffer_get_trace_enable_bits_count); ++ ++void kbase_csf_firmware_trace_buffer_update_trace_enable_bit( ++ struct firmware_trace_buffer *tb, unsigned int bit, bool value) ++{ ++ if (bit < tb->trace_enable_entry_count) { ++ unsigned int trace_enable_reg_offset = bit >> 5; ++ u32 trace_enable_bit_mask = 1u << (bit & 0x1F); ++ ++ if (value) { ++ tb->trace_enable_init_mask[trace_enable_reg_offset] |= ++ trace_enable_bit_mask; ++ } else { ++ tb->trace_enable_init_mask[trace_enable_reg_offset] &= ++ ~trace_enable_bit_mask; ++ } ++ ++ /* This is not strictly needed as the caller is supposed to ++ * reload the firmware image (through GPU reset) after updating ++ * the bitmask. Otherwise there is no guarantee that firmware ++ * will take into account the updated bitmask for all types of ++ * trace buffers, since firmware could continue to use the ++ * value of bitmask it cached after the boot. ++ */ ++ kbase_csf_update_firmware_memory(tb->kbdev, ++ tb->gpu_va.trace_enable + trace_enable_reg_offset*4, ++ tb->trace_enable_init_mask[trace_enable_reg_offset]); ++ } ++} ++EXPORT_SYMBOL(kbase_csf_firmware_trace_buffer_update_trace_enable_bit); ++ ++bool kbase_csf_firmware_trace_buffer_is_empty( ++ const struct firmware_trace_buffer *trace_buffer) ++{ ++ return *(trace_buffer->cpu_va.insert_cpu_va) == ++ *(trace_buffer->cpu_va.extract_cpu_va); ++} ++EXPORT_SYMBOL(kbase_csf_firmware_trace_buffer_is_empty); ++ ++unsigned int kbase_csf_firmware_trace_buffer_read_data( ++ struct firmware_trace_buffer *trace_buffer, u8 *data, unsigned int num_bytes) ++{ ++ unsigned int bytes_copied; ++ u8 *data_cpu_va = trace_buffer->data_mapping.cpu_addr; ++ u32 extract_offset = *(trace_buffer->cpu_va.extract_cpu_va); ++ u32 insert_offset = *(trace_buffer->cpu_va.insert_cpu_va); ++ u32 buffer_size = trace_buffer->num_pages << PAGE_SHIFT; ++ ++ if (insert_offset >= extract_offset) { ++ bytes_copied = min_t(unsigned int, num_bytes, ++ (insert_offset - extract_offset)); ++ memcpy(data, &data_cpu_va[extract_offset], bytes_copied); ++ extract_offset += bytes_copied; ++ } else { ++ unsigned int bytes_copied_head, bytes_copied_tail; ++ ++ bytes_copied_tail = min_t(unsigned int, num_bytes, ++ (buffer_size - extract_offset)); ++ memcpy(data, &data_cpu_va[extract_offset], bytes_copied_tail); ++ ++ bytes_copied_head = min_t(unsigned int, ++ (num_bytes - bytes_copied_tail), insert_offset); ++ memcpy(&data[bytes_copied_tail], data_cpu_va, bytes_copied_head); ++ ++ bytes_copied = bytes_copied_head + bytes_copied_tail; ++ extract_offset += bytes_copied; ++ if (extract_offset >= buffer_size) ++ extract_offset = bytes_copied_head; ++ } ++ ++ *(trace_buffer->cpu_va.extract_cpu_va) = extract_offset; ++ ++ return bytes_copied; ++} ++EXPORT_SYMBOL(kbase_csf_firmware_trace_buffer_read_data); ++ ++#ifdef CONFIG_DEBUG_FS ++ ++#define U32_BITS 32 ++static u64 get_trace_buffer_active_mask64(struct firmware_trace_buffer *tb) ++{ ++ u64 active_mask = tb->trace_enable_init_mask[0]; ++ ++ if (tb->trace_enable_entry_count > U32_BITS) ++ active_mask |= (u64)tb->trace_enable_init_mask[1] << U32_BITS; ++ ++ return active_mask; ++} ++ ++static void update_trace_buffer_active_mask64(struct firmware_trace_buffer *tb, ++ u64 mask) ++{ ++ unsigned int i; ++ ++ for (i = 0; i < tb->trace_enable_entry_count; i++) ++ kbase_csf_firmware_trace_buffer_update_trace_enable_bit(tb, i, ++ (mask >> i) & 1); ++} ++ ++static int set_trace_buffer_active_mask64(struct firmware_trace_buffer *tb, ++ u64 mask) ++{ ++ struct kbase_device *kbdev = tb->kbdev; ++ unsigned long flags; ++ int err = 0; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ /* If there is already a GPU reset pending, need a retry */ ++ if (kbase_reset_gpu_silent(kbdev)) ++ err = -EAGAIN; ++ else ++ update_trace_buffer_active_mask64(tb, mask); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ return err; ++} ++ ++static int kbase_csf_firmware_trace_enable_mask_read(void *data, u64 *val) ++{ ++ struct kbase_device *kbdev = (struct kbase_device *)data; ++ struct firmware_trace_buffer *tb = ++ kbase_csf_firmware_get_trace_buffer(kbdev, FW_TRACE_BUF_NAME); ++ ++ if (tb == NULL) { ++ dev_err(kbdev->dev, "Couldn't get the firmware trace buffer"); ++ return -EIO; ++ } ++ /* The enabled traces limited to u64 here, regarded practical */ ++ *val = get_trace_buffer_active_mask64(tb); ++ return 0; ++} ++ ++static int kbase_csf_firmware_trace_enable_mask_write(void *data, u64 val) ++{ ++ struct kbase_device *kbdev = (struct kbase_device *)data; ++ struct firmware_trace_buffer *tb = ++ kbase_csf_firmware_get_trace_buffer(kbdev, FW_TRACE_BUF_NAME); ++ u64 new_mask; ++ unsigned int enable_bits_count; ++ ++ if (tb == NULL) { ++ dev_err(kbdev->dev, "Couldn't get the firmware trace buffer"); ++ return -EIO; ++ } ++ ++ /* Ignore unsupported types */ ++ enable_bits_count = ++ kbase_csf_firmware_trace_buffer_get_trace_enable_bits_count(tb); ++ if (enable_bits_count > 64) { ++ dev_dbg(kbdev->dev, "Limit enabled bits count from %u to 64", ++ enable_bits_count); ++ enable_bits_count = 64; ++ } ++ new_mask = val & ((1 << enable_bits_count) - 1); ++ ++ if (new_mask != get_trace_buffer_active_mask64(tb)) ++ return set_trace_buffer_active_mask64(tb, new_mask); ++ else ++ return 0; ++} ++ ++static int kbasep_csf_firmware_trace_debugfs_open(struct inode *in, ++ struct file *file) ++{ ++ struct kbase_device *kbdev = in->i_private; ++ ++ file->private_data = kbdev; ++ dev_dbg(kbdev->dev, "Opened firmware trace buffer dump debugfs file"); ++ ++ return 0; ++} ++ ++static ssize_t kbasep_csf_firmware_trace_debugfs_read(struct file *file, ++ char __user *buf, size_t size, loff_t *ppos) ++{ ++ struct kbase_device *kbdev = file->private_data; ++ u8 *pbyte; ++ unsigned int n_read; ++ unsigned long not_copied; ++ /* Limit the kernel buffer to no more than two pages */ ++ size_t mem = MIN(size, 2 * PAGE_SIZE); ++ unsigned long flags; ++ ++ struct firmware_trace_buffer *tb = ++ kbase_csf_firmware_get_trace_buffer(kbdev, FW_TRACE_BUF_NAME); ++ ++ if (tb == NULL) { ++ dev_err(kbdev->dev, "Couldn't get the firmware trace buffer"); ++ return -EIO; ++ } ++ ++ pbyte = kmalloc(mem, GFP_KERNEL); ++ if (pbyte == NULL) { ++ dev_err(kbdev->dev, "Couldn't allocate memory for trace buffer dump"); ++ return -ENOMEM; ++ } ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ n_read = kbase_csf_firmware_trace_buffer_read_data(tb, pbyte, mem); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ /* Do the copy, if we have obtained some trace data */ ++ not_copied = (n_read) ? copy_to_user(buf, pbyte, n_read) : 0; ++ kfree(pbyte); ++ ++ if (!not_copied) { ++ *ppos += n_read; ++ return n_read; ++ } ++ ++ dev_err(kbdev->dev, "Couldn't copy trace buffer data to user space buffer"); ++ return -EFAULT; ++} ++ ++ ++DEFINE_SIMPLE_ATTRIBUTE(kbase_csf_firmware_trace_enable_mask_fops, ++ kbase_csf_firmware_trace_enable_mask_read, ++ kbase_csf_firmware_trace_enable_mask_write, "%llx\n"); ++ ++static const struct file_operations kbasep_csf_firmware_trace_debugfs_fops = { ++ .owner = THIS_MODULE, ++ .open = kbasep_csf_firmware_trace_debugfs_open, ++ .read = kbasep_csf_firmware_trace_debugfs_read, ++ .llseek = no_llseek, ++}; ++ ++void kbase_csf_firmware_trace_buffer_debugfs_init(struct kbase_device *kbdev) ++{ ++ debugfs_create_file("fw_trace_enable_mask", 0644, ++ kbdev->mali_debugfs_directory, kbdev, ++ &kbase_csf_firmware_trace_enable_mask_fops); ++ ++ debugfs_create_file("fw_traces", 0444, ++ kbdev->mali_debugfs_directory, kbdev, ++ &kbasep_csf_firmware_trace_debugfs_fops); ++} ++#endif /* CONFIG_DEBUG_FS */ +diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_trace_buffer.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_trace_buffer.h +new file mode 100755 +index 000000000000..2cac55e0664d +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_trace_buffer.h +@@ -0,0 +1,177 @@ ++/* ++ * ++ * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_CSF_TRACE_BUFFER_H_ ++#define _KBASE_CSF_TRACE_BUFFER_H_ ++ ++#include ++ ++#define CSF_FIRMWARE_TRACE_ENABLE_INIT_MASK_MAX (4) ++#define FW_TRACE_BUF_NAME "fwlog" ++ ++/* Forward declarations */ ++struct firmware_trace_buffer; ++struct kbase_device; ++ ++/** ++ * kbase_csf_firmware_trace_buffers_init - Initialize trace buffers ++ * ++ * Allocate resources for trace buffers. In particular: ++ * - One memory page of GPU-readable, CPU-writable memory is used for ++ * the Extract variables of all trace buffers. ++ * - One memory page of GPU-writable, CPU-readable memory is used for ++ * the Insert variables of all trace buffers. ++ * - A data buffer of GPU-writable, CPU-readable memory is allocated ++ * for each trace buffer. ++ * ++ * After that, firmware addresses are written with pointers to the ++ * insert, extract and data buffer variables. The size and the trace ++ * enable bits are not dereferenced by the GPU and shall be written ++ * in the firmware addresses directly. ++ * ++ * This function relies on the assumption that the list of ++ * firmware_trace_buffer elements in the device has already been ++ * populated with data from the firmware image parsing. ++ * ++ * Return: 0 if success, or an error code on failure. ++ * ++ * @kbdev: Device pointer ++ */ ++int kbase_csf_firmware_trace_buffers_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_csf_firmware_trace_buffer_term - Terminate trace buffers ++ * ++ * @kbdev: Device pointer ++ */ ++void kbase_csf_firmware_trace_buffers_term(struct kbase_device *kbdev); ++ ++/** ++ * kbase_csf_firmware_parse_trace_buffer_entry - Process a "trace buffer" section ++ * ++ * Read a "trace buffer" section adding metadata for the related trace buffer ++ * to the kbase_device:csf.firmware_trace_buffers list. ++ * ++ * Unexpected trace buffers will not be parsed and, as a consequence, ++ * will not be initialized. ++ * ++ * Return: 0 if successful, negative error code on failure. ++ * ++ * @kbdev: Kbase device structure ++ * @entry: Pointer to the section ++ * @size: Size (in bytes) of the section ++ */ ++int kbase_csf_firmware_parse_trace_buffer_entry(struct kbase_device *kbdev, ++ const u32 *entry, unsigned int size); ++ ++/** ++ * kbase_csf_firmware_reload_trace_buffers_data - ++ * Reload trace buffers data for firmware reboot ++ * ++ * Helper function used when rebooting the firmware to reload the initial setup ++ * for all the trace buffers which have been previously parsed and initialized. ++ * ++ * Almost all of the operations done in the initialization process are ++ * replicated, with the difference that they might be done in a different order ++ * and that the variables of a given trace buffer may be mapped to different ++ * offsets within the same existing mappings. ++ * ++ * In other words, the re-initialization done by this function will be ++ * equivalent but not necessarily identical to the original initialization. ++ * ++ * @kbdev: Device pointer ++ */ ++void kbase_csf_firmware_reload_trace_buffers_data(struct kbase_device *kbdev); ++ ++/** ++ * kbase_csf_firmware_get_trace_buffer - Get a trace buffer ++ * ++ * Return: handle to a trace buffer, given the name, or NULL if a trace buffer ++ * with that name couldn't be found. ++ * ++ * @kbdev: Device pointer ++ * @name: Name of the trace buffer to find ++ */ ++struct firmware_trace_buffer *kbase_csf_firmware_get_trace_buffer( ++ struct kbase_device *kbdev, const char *name); ++ ++/** ++ * kbase_csf_firmware_trace_buffer_get_trace_enable_bits_count - ++ * Get number of trace enable bits for a trace buffer ++ * ++ * Return: Number of trace enable bits in a trace buffer. ++ * ++ * @trace_buffer: Trace buffer handle ++ */ ++unsigned int kbase_csf_firmware_trace_buffer_get_trace_enable_bits_count( ++ const struct firmware_trace_buffer *trace_buffer); ++ ++/** ++ * kbase_csf_firmware_trace_buffer_update_trace_enable_bit - ++ * Update a trace enable bit ++ * ++ * Update the value of a given trace enable bit. ++ * ++ * @trace_buffer: Trace buffer handle ++ * @bit: Bit to update ++ * @value: New value for the given bit ++ */ ++void kbase_csf_firmware_trace_buffer_update_trace_enable_bit( ++ struct firmware_trace_buffer *trace_buffer, unsigned int bit, bool value); ++ ++/** ++ * kbase_csf_firmware_trace_buffer_is_empty - Empty trace buffer predicate ++ * ++ * Return: True if the trace buffer is empty, or false otherwise. ++ * ++ * @trace_buffer: Trace buffer handle ++ */ ++bool kbase_csf_firmware_trace_buffer_is_empty( ++ const struct firmware_trace_buffer *trace_buffer); ++ ++/** ++ * kbase_csf_firmware_trace_buffer_read_data - Read data from a trace buffer ++ * ++ * Read available data from a trace buffer. The client provides a data buffer ++ * of a given size and the maximum number of bytes to read. ++ * ++ * Return: Number of bytes read from the trace buffer. ++ * ++ * @trace_buffer: Trace buffer handle ++ * @data: Pointer to a client-allocated where data shall be written. ++ * @num_bytes: Maximum number of bytes to read from the trace buffer. ++ */ ++unsigned int kbase_csf_firmware_trace_buffer_read_data( ++ struct firmware_trace_buffer *trace_buffer, u8 *data, unsigned int num_bytes); ++ ++#ifdef CONFIG_DEBUG_FS ++/** ++ * kbase_csf_fw_trace_buffer_debugfs_init() - Add debugfs entries for setting ++ * enable mask and dumping the binary ++ * firmware trace buffer ++ * ++ * @kbdev: Pointer to the device ++ */ ++void kbase_csf_firmware_trace_buffer_debugfs_init(struct kbase_device *kbdev); ++#endif /* CONFIG_DEBUG_FS */ ++ ++#endif /* _KBASE_CSF_TRACE_BUFFER_H_ */ +diff --git a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_codes_csf.h b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_codes_csf.h +new file mode 100755 +index 000000000000..32181d711193 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_codes_csf.h +@@ -0,0 +1,116 @@ ++/* ++ * ++ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* ++ * ***** IMPORTANT: THIS IS NOT A NORMAL HEADER FILE ***** ++ * ***** DO NOT INCLUDE DIRECTLY ***** ++ * ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** ++ */ ++ ++/* ++ * The purpose of this header file is just to contain a list of trace code ++ * identifiers ++ * ++ * When updating this file, also remember to update ++ * mali_kbase_debug_linux_ktrace_csf.h ++ * ++ * IMPORTANT: THIS FILE MUST NOT BE USED FOR ANY OTHER PURPOSE OTHER THAN THAT ++ * DESCRIBED IN mali_kbase_debug_ktrace_codes.h ++ */ ++ ++#if 0 /* Dummy section to avoid breaking formatting */ ++int dummy_array[] = { ++#endif ++ /* ++ * Generic CSF events ++ */ ++ KBASE_KTRACE_CODE_MAKE_CODE(EVICT_CTX_SLOTS), ++ /* info_val[0:7] == fw version_minor ++ * info_val[15:8] == fw version_major ++ * info_val[63:32] == fw version_hash ++ */ ++ KBASE_KTRACE_CODE_MAKE_CODE(FIRMWARE_BOOT), ++ KBASE_KTRACE_CODE_MAKE_CODE(FIRMWARE_REBOOT), ++ KBASE_KTRACE_CODE_MAKE_CODE(SCHEDULER_TOCK), ++ KBASE_KTRACE_CODE_MAKE_CODE(SCHEDULER_TICK), ++ KBASE_KTRACE_CODE_MAKE_CODE(SCHEDULER_RESET), ++ KBASE_KTRACE_CODE_MAKE_CODE(SCHEDULER_EXIT_PROTM), ++ KBASE_KTRACE_CODE_MAKE_CODE(SYNC_UPDATE_EVENT), ++ ++ /* ++ * Group events ++ */ ++ /* info_val[2:0] == CSG_REQ state issued ++ * info_val[19:16] == as_nr ++ * info_val[63:32] == endpoint config (max number of endpoints allowed) ++ */ ++ KBASE_KTRACE_CODE_MAKE_CODE(CSG_SLOT_START), ++ /* info_val == CSG_REQ state issued */ ++ KBASE_KTRACE_CODE_MAKE_CODE(CSG_SLOT_STOP), ++ /* info_val == CSG_ACK state */ ++ KBASE_KTRACE_CODE_MAKE_CODE(CSG_SLOT_STARTED), ++ /* info_val == CSG_ACK state */ ++ KBASE_KTRACE_CODE_MAKE_CODE(CSG_SLOT_STOPPED), ++ /* info_val == slot cleaned */ ++ KBASE_KTRACE_CODE_MAKE_CODE(CSG_SLOT_CLEANED), ++ /* info_val == previous priority */ ++ KBASE_KTRACE_CODE_MAKE_CODE(CSG_PRIO_UPDATE), ++ /* info_val == CSG_REQ ^ CSG_ACK */ ++ KBASE_KTRACE_CODE_MAKE_CODE(CSG_SYNC_UPDATE_INTERRUPT), ++ /* info_val == CSG_REQ ^ CSG_ACK */ ++ KBASE_KTRACE_CODE_MAKE_CODE(CSG_IDLE_INTERRUPT), ++ KBASE_KTRACE_CODE_MAKE_CODE(GROUP_SYNC_UPDATE_DONE), ++ /* info_val == run state of the group */ ++ KBASE_KTRACE_CODE_MAKE_CODE(GROUP_DESCHEDULE), ++ /* info_val == run state of the group */ ++ KBASE_KTRACE_CODE_MAKE_CODE(GROUP_SCHEDULE), ++ /* info_val[31:0] == new run state of the evicted group ++ * info_val[63:32] == number of runnable groups ++ */ ++ KBASE_KTRACE_CODE_MAKE_CODE(GROUP_EVICT_SCHED), ++ KBASE_KTRACE_CODE_MAKE_CODE(SCHEDULER_ENTER_PROTM), ++ /* info_val[31:0] == number of GPU address space slots in use ++ * info_val[63:32] == number of runnable groups ++ */ ++ KBASE_KTRACE_CODE_MAKE_CODE(SCHEDULER_TOP_GRP), ++ ++ /* ++ * Group + Queue events ++ */ ++ /* info_val == queue->enabled */ ++ KBASE_KTRACE_CODE_MAKE_CODE(CSI_START), ++ /* info_val == queue->enabled before stop */ ++ KBASE_KTRACE_CODE_MAKE_CODE(CSI_STOP), ++ KBASE_KTRACE_CODE_MAKE_CODE(CSI_STOP_REQUESTED), ++ /* info_val == CS_REQ ^ CS_ACK */ ++ KBASE_KTRACE_CODE_MAKE_CODE(CSI_FAULT_INTERRUPT), ++ /* info_val == CS_REQ ^ CS_ACK */ ++ KBASE_KTRACE_CODE_MAKE_CODE(CSI_TILER_OOM_INTERRUPT), ++ /* info_val == group->run_State (for group the queue is bound to) */ ++ KBASE_KTRACE_CODE_MAKE_CODE(QUEUE_START), ++ KBASE_KTRACE_CODE_MAKE_CODE(QUEUE_STOP), ++ ++#if 0 /* Dummy section to avoid breaking formatting */ ++}; ++#endif ++ ++/* ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** */ +diff --git a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_codes_jm.h b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_codes_jm.h +new file mode 100755 +index 000000000000..b201e49bd0f2 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_codes_jm.h +@@ -0,0 +1,173 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2015,2018-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* ++ * ***** IMPORTANT: THIS IS NOT A NORMAL HEADER FILE ***** ++ * ***** DO NOT INCLUDE DIRECTLY ***** ++ * ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** ++ */ ++ ++/* ++ * The purpose of this header file is just to contain a list of trace code ++ * identifiers ++ * ++ * When updating this file, also remember to update ++ * mali_kbase_debug_linux_ktrace_jm.h ++ * ++ * IMPORTANT: THIS FILE MUST NOT BE USED FOR ANY OTHER PURPOSE OTHER THAN THAT ++ * DESCRIBED IN mali_kbase_debug_ktrace_codes.h ++ */ ++ ++#if 0 /* Dummy section to avoid breaking formatting */ ++int dummy_array[] = { ++#endif ++ ++ /* ++ * Job Slot management events ++ */ ++ /* info_val==irq rawstat at start */ ++ KBASE_KTRACE_CODE_MAKE_CODE(JM_IRQ), ++ /* info_val==jobs processed */ ++ KBASE_KTRACE_CODE_MAKE_CODE(JM_IRQ_END), ++ /* In the following: ++ * ++ * - ctx is set if a corresponding job found (NULL otherwise, e.g. some ++ * soft-stop cases) ++ * - uatom==kernel-side mapped uatom address (for correlation with ++ * user-side) ++ */ ++ /* info_val==exit code; gpu_addr==chain gpuaddr */ ++ KBASE_KTRACE_CODE_MAKE_CODE(JM_JOB_DONE), ++ /* gpu_addr==JS_HEAD_NEXT written, info_val==lower 32 bits of ++ * affinity ++ */ ++ KBASE_KTRACE_CODE_MAKE_CODE(JM_SUBMIT), ++ /* gpu_addr is as follows: ++ * - If JS_STATUS active after soft-stop, val==gpu addr written to ++ * JS_HEAD on submit ++ * - otherwise gpu_addr==0 ++ */ ++ KBASE_KTRACE_CODE_MAKE_CODE(JM_SOFTSTOP), ++ KBASE_KTRACE_CODE_MAKE_CODE(JM_SOFTSTOP_0), ++ KBASE_KTRACE_CODE_MAKE_CODE(JM_SOFTSTOP_1), ++ /* gpu_addr==JS_HEAD read */ ++ KBASE_KTRACE_CODE_MAKE_CODE(JM_HARDSTOP), ++ /* gpu_addr==JS_HEAD read */ ++ KBASE_KTRACE_CODE_MAKE_CODE(JM_HARDSTOP_0), ++ /* gpu_addr==JS_HEAD read */ ++ KBASE_KTRACE_CODE_MAKE_CODE(JM_HARDSTOP_1), ++ /* gpu_addr==JS_TAIL read */ ++ KBASE_KTRACE_CODE_MAKE_CODE(JM_UPDATE_HEAD), ++ /* gpu_addr is as follows: ++ * - If JS_STATUS active before soft-stop, val==JS_HEAD ++ * - otherwise gpu_addr==0 ++ */ ++ /* gpu_addr==JS_HEAD read */ ++ KBASE_KTRACE_CODE_MAKE_CODE(JM_CHECK_HEAD), ++ KBASE_KTRACE_CODE_MAKE_CODE(JM_FLUSH_WORKQS), ++ KBASE_KTRACE_CODE_MAKE_CODE(JM_FLUSH_WORKQS_DONE), ++ /* info_val == is_scheduled */ ++ KBASE_KTRACE_CODE_MAKE_CODE(JM_ZAP_NON_SCHEDULED), ++ /* info_val == is_scheduled */ ++ KBASE_KTRACE_CODE_MAKE_CODE(JM_ZAP_SCHEDULED), ++ KBASE_KTRACE_CODE_MAKE_CODE(JM_ZAP_DONE), ++ /* info_val == nr jobs submitted */ ++ KBASE_KTRACE_CODE_MAKE_CODE(JM_SLOT_SOFT_OR_HARD_STOP), ++ /* gpu_addr==JS_HEAD_NEXT last written */ ++ KBASE_KTRACE_CODE_MAKE_CODE(JM_SLOT_EVICT), ++ KBASE_KTRACE_CODE_MAKE_CODE(JM_SUBMIT_AFTER_RESET), ++ KBASE_KTRACE_CODE_MAKE_CODE(JM_BEGIN_RESET_WORKER), ++ KBASE_KTRACE_CODE_MAKE_CODE(JM_END_RESET_WORKER), ++ /* ++ * Job dispatch events ++ */ ++ /* gpu_addr==value to write into JS_HEAD */ ++ KBASE_KTRACE_CODE_MAKE_CODE(JD_DONE), ++ /* gpu_addr==value to write into JS_HEAD */ ++ KBASE_KTRACE_CODE_MAKE_CODE(JD_DONE_WORKER), ++ /* gpu_addr==value to write into JS_HEAD */ ++ KBASE_KTRACE_CODE_MAKE_CODE(JD_DONE_WORKER_END), ++ /* gpu_addr==value to write into JS_HEAD */ ++ KBASE_KTRACE_CODE_MAKE_CODE(JD_DONE_TRY_RUN_NEXT_JOB), ++ /* gpu_addr==0, info_val==0, uatom==0 */ ++ KBASE_KTRACE_CODE_MAKE_CODE(JD_ZAP_CONTEXT), ++ /* gpu_addr==value to write into JS_HEAD */ ++ KBASE_KTRACE_CODE_MAKE_CODE(JD_CANCEL), ++ /* gpu_addr==value to write into JS_HEAD */ ++ KBASE_KTRACE_CODE_MAKE_CODE(JD_CANCEL_WORKER), ++ /* ++ * Scheduler Core events ++ */ ++ /* gpu_addr==value to write into JS_HEAD */ ++ KBASE_KTRACE_CODE_MAKE_CODE(JS_ADD_JOB), ++ /* gpu_addr==last value written/would be written to JS_HEAD */ ++ KBASE_KTRACE_CODE_MAKE_CODE(JS_REMOVE_JOB), ++ KBASE_KTRACE_CODE_MAKE_CODE(JS_TRY_SCHEDULE_HEAD_CTX), ++ /* gpu_addr==value to write into JS_HEAD */ ++ KBASE_KTRACE_CODE_MAKE_CODE(JS_JOB_DONE_TRY_RUN_NEXT_JOB), ++ /* gpu_addr==value to write into JS_HEAD */ ++ KBASE_KTRACE_CODE_MAKE_CODE(JS_JOB_DONE_RETRY_NEEDED), ++ KBASE_KTRACE_CODE_MAKE_CODE(JS_AFFINITY_SUBMIT_TO_BLOCKED), ++ /* info_val == lower 32 bits of affinity */ ++ KBASE_KTRACE_CODE_MAKE_CODE(JS_AFFINITY_CURRENT), ++ /* info_val == lower 32 bits of affinity */ ++ KBASE_KTRACE_CODE_MAKE_CODE(JS_CORE_REF_REQUEST_CORES_FAILED), ++ /* info_val == lower 32 bits of affinity */ ++ KBASE_KTRACE_CODE_MAKE_CODE(JS_CORE_REF_REGISTER_INUSE_FAILED), ++ /* info_val == lower 32 bits of rechecked affinity */ ++ KBASE_KTRACE_CODE_MAKE_CODE(JS_CORE_REF_REQUEST_ON_RECHECK_FAILED), ++ /* info_val == lower 32 bits of rechecked affinity */ ++ KBASE_KTRACE_CODE_MAKE_CODE(JS_CORE_REF_REGISTER_ON_RECHECK_FAILED), ++ /* info_val == lower 32 bits of affinity */ ++ KBASE_KTRACE_CODE_MAKE_CODE(JS_CORE_REF_AFFINITY_WOULD_VIOLATE), ++ /* info_val == the ctx attribute now on ctx */ ++ KBASE_KTRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_ON_CTX), ++ /* info_val == the ctx attribute now on runpool */ ++ KBASE_KTRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_ON_RUNPOOL), ++ /* info_val == the ctx attribute now off ctx */ ++ KBASE_KTRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_OFF_CTX), ++ /* info_val == the ctx attribute now off runpool */ ++ KBASE_KTRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_OFF_RUNPOOL), ++ /* ++ * Scheduler Policy events ++ */ ++ KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_INIT_CTX), ++ KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_TERM_CTX), ++ /* info_val == whether it was evicted */ ++ KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_TRY_EVICT_CTX), ++ KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_FOREACH_CTX_JOBS), ++ KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_ENQUEUE_CTX), ++ KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_DEQUEUE_HEAD_CTX), ++ KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_RUNPOOL_ADD_CTX), ++ KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_RUNPOOL_REMOVE_CTX), ++ KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_DEQUEUE_JOB), ++ KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_DEQUEUE_JOB_IRQ), ++ /* gpu_addr==JS_HEAD to write if the job were run */ ++ KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_ENQUEUE_JOB), ++ KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_TIMER_START), ++ KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_TIMER_END), ++ ++#if 0 /* Dummy section to avoid breaking formatting */ ++}; ++#endif ++ ++/* ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** */ +diff --git a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_csf.c b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_csf.c +new file mode 100755 +index 000000000000..2ea901b666c2 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_csf.c +@@ -0,0 +1,143 @@ ++/* ++ * ++ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++#include ++#include "debug/mali_kbase_debug_ktrace_internal.h" ++#include "debug/backend/mali_kbase_debug_ktrace_csf.h" ++ ++#if KBASE_KTRACE_TARGET_RBUF ++ ++void kbasep_ktrace_backend_format_header(char *buffer, int sz, s32 *written) ++{ ++ *written += MAX(snprintf(buffer + *written, MAX(sz - *written, 0), ++ "group,slot,prio,csi"), 0); ++} ++ ++void kbasep_ktrace_backend_format_msg(struct kbase_ktrace_msg *trace_msg, ++ char *buffer, int sz, s32 *written) ++{ ++ const struct kbase_ktrace_backend * const be_msg = &trace_msg->backend; ++ /* At present, no need to check for KBASE_KTRACE_FLAG_BACKEND, as the ++ * other backend-specific flags currently imply this anyway ++ */ ++ ++ /* group parts */ ++ if (be_msg->flags & KBASE_KTRACE_FLAG_CSF_GROUP) { ++ const s8 slot = be_msg->csg_nr; ++ /* group,slot, */ ++ *written += MAX(snprintf(buffer + *written, ++ MAX(sz - *written, 0), ++ "%u,%d,", be_msg->group_handle, slot), 0); ++ ++ /* prio */ ++ if (slot >= 0) ++ *written += MAX(snprintf(buffer + *written, ++ MAX(sz - *written, 0), ++ "%u", be_msg->slot_prio), 0); ++ ++ /* , */ ++ *written += MAX(snprintf(buffer + *written, ++ MAX(sz - *written, 0), ++ ","), 0); ++ } else { ++ /* No group,slot,prio fields, but ensure ending with "," */ ++ *written += MAX(snprintf(buffer + *written, ++ MAX(sz - *written, 0), ++ ",,,"), 0); ++ } ++ ++ /* queue parts: csi */ ++ if (trace_msg->backend.flags & KBASE_KTRACE_FLAG_CSF_QUEUE) ++ *written += MAX(snprintf(buffer + *written, ++ MAX(sz - *written, 0), ++ "%d", be_msg->csi_index), 0); ++ ++ /* Don't end with a trailing "," - this is a 'standalone' formatted ++ * msg, caller will handle the delimiters ++ */ ++} ++ ++void kbasep_ktrace_add_csf(struct kbase_device *kbdev, ++ enum kbase_ktrace_code code, struct kbase_queue_group *group, ++ struct kbase_queue *queue, kbase_ktrace_flag_t flags, ++ u64 info_val) ++{ ++ unsigned long irqflags; ++ struct kbase_ktrace_msg *trace_msg; ++ struct kbase_context *kctx = NULL; ++ ++ spin_lock_irqsave(&kbdev->ktrace.lock, irqflags); ++ ++ /* Reserve and update indices */ ++ trace_msg = kbasep_ktrace_reserve(&kbdev->ktrace); ++ ++ /* Determine the kctx */ ++ if (group) ++ kctx = group->kctx; ++ else if (queue) ++ kctx = queue->kctx; ++ ++ /* Fill the common part of the message (including backend.flags) */ ++ kbasep_ktrace_msg_init(&kbdev->ktrace, trace_msg, code, kctx, flags, ++ info_val); ++ ++ /* Indicate to the common code that backend-specific parts will be ++ * valid ++ */ ++ trace_msg->backend.flags |= KBASE_KTRACE_FLAG_BACKEND; ++ ++ /* Fill the CSF-specific parts of the message ++ * ++ * Generally, no need to use default initializers when queue/group not ++ * present - can usually check the flags instead. ++ */ ++ ++ if (queue) { ++ trace_msg->backend.flags |= KBASE_KTRACE_FLAG_CSF_QUEUE; ++ trace_msg->backend.csi_index = queue->csi_index; ++ } ++ ++ if (group) { ++ const s8 slot = group->csg_nr; ++ ++ trace_msg->backend.flags |= KBASE_KTRACE_FLAG_CSF_GROUP; ++ ++ trace_msg->backend.csg_nr = slot; ++ ++ if (slot >= 0) { ++ struct kbase_csf_csg_slot *csg_slot = &kbdev->csf.scheduler.csg_slots[slot]; ++ ++ trace_msg->backend.slot_prio = csg_slot->priority; ++ } ++ /* slot >=0 indicates whether slot_prio valid, so no need to ++ * initialize in the case where it's invalid ++ */ ++ ++ trace_msg->backend.group_handle = group->handle; ++ } ++ ++ WARN_ON((trace_msg->backend.flags & ~KBASE_KTRACE_FLAG_ALL)); ++ ++ /* Done */ ++ spin_unlock_irqrestore(&kbdev->ktrace.lock, irqflags); ++} ++ ++#endif /* KBASE_KTRACE_TARGET_RBUF */ +diff --git a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_csf.h b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_csf.h +new file mode 100755 +index 000000000000..b055ff82a116 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_csf.h +@@ -0,0 +1,148 @@ ++/* ++ * ++ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_DEBUG_KTRACE_CSF_H_ ++#define _KBASE_DEBUG_KTRACE_CSF_H_ ++ ++/* ++ * KTrace target for internal ringbuffer ++ */ ++#if KBASE_KTRACE_TARGET_RBUF ++/** ++ * kbasep_ktrace_add_csf - internal function to add trace about Command Stream ++ * Frontend ++ * @kbdev: kbase device ++ * @code: trace code ++ * @group: queue group, or NULL if no queue group ++ * @queue: queue, or NULL if no queue ++ * @flags: flags about the message ++ * @info_val: generic information about @code to add to the trace ++ * ++ * PRIVATE: do not use directly. Use KBASE_KTRACE_ADD_CSF() instead. ++ */ ++ ++void kbasep_ktrace_add_csf(struct kbase_device *kbdev, ++ enum kbase_ktrace_code code, struct kbase_queue_group *group, ++ struct kbase_queue *queue, kbase_ktrace_flag_t flags, ++ u64 info_val); ++ ++#define KBASE_KTRACE_RBUF_ADD_CSF(kbdev, code, group, queue, flags, info_val) \ ++ kbasep_ktrace_add_csf(kbdev, KBASE_KTRACE_CODE(code), group, queue, \ ++ flags, info_val) ++ ++#else /* KBASE_KTRACE_TARGET_RBUF */ ++ ++#define KBASE_KTRACE_RBUF_ADD_CSF(kbdev, code, group, queue, flags, info_val) \ ++ do {\ ++ CSTD_UNUSED(kbdev);\ ++ CSTD_NOP(code);\ ++ CSTD_UNUSED(group);\ ++ CSTD_UNUSED(queue); \ ++ CSTD_UNUSED(flags);\ ++ CSTD_UNUSED(info_val);\ ++ CSTD_NOP(0);\ ++ } while (0) ++ ++#endif /* KBASE_KTRACE_TARGET_RBUF */ ++ ++/* ++ * KTrace target for Linux's ftrace ++ * ++ * Note: the header file(s) that define the trace_mali_<...> tracepoints are ++ * included by the parent header file ++ */ ++#if KBASE_KTRACE_TARGET_FTRACE ++ ++#define KBASE_KTRACE_FTRACE_ADD_CSF(kbdev, code, group, queue, info_val) \ ++ trace_mali_##code(kbdev, group, queue, info_val) ++ ++#else /* KBASE_KTRACE_TARGET_FTRACE */ ++ ++#define KBASE_KTRACE_FTRACE_ADD_CSF(kbdev, code, group, queue, info_val) \ ++ do {\ ++ CSTD_UNUSED(kbdev);\ ++ CSTD_NOP(code);\ ++ CSTD_UNUSED(group);\ ++ CSTD_UNUSED(queue);\ ++ CSTD_UNUSED(info_val);\ ++ CSTD_NOP(0);\ ++ } while (0) ++ ++#endif /* KBASE_KTRACE_TARGET_FTRACE */ ++ ++/* ++ * Master set of macros to route KTrace to any of the targets ++ */ ++ ++/** ++ * KBASE_KTRACE_ADD_CSF_GRP - Add trace values about a group, with info ++ * @kbdev: kbase device ++ * @code: trace code ++ * @group: queue group, or NULL if no queue group ++ * @info_val: generic information about @code to add to the trace ++ * ++ * Note: Any functions called through this macro will still be evaluated in ++ * Release builds (CONFIG_MALI_BIFROST_DEBUG not defined). Therefore, when ++ * KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied ++ * to this macro must: ++ * a) be static or static inline, and ++ * b) just return 0 and have no other statements present in the body. ++ */ ++#define KBASE_KTRACE_ADD_CSF_GRP(kbdev, code, group, info_val) \ ++ do { \ ++ /* capture values that could come from non-pure function calls */ \ ++ struct kbase_queue_group *__group = group; \ ++ u64 __info_val = info_val; \ ++ KBASE_KTRACE_RBUF_ADD_CSF(kbdev, code, __group, NULL, 0u, \ ++ __info_val); \ ++ KBASE_KTRACE_FTRACE_ADD_CSF(kbdev, code, __group, NULL, \ ++ __info_val); \ ++ } while (0) ++ ++/** ++ * KBASE_KTRACE_ADD_CSF_GRP_Q - Add trace values about a group, queue, with info ++ * @kbdev: kbase device ++ * @code: trace code ++ * @group: queue group, or NULL if no queue group ++ * @queue: queue, or NULL if no queue ++ * @info_val: generic information about @code to add to the trace ++ * ++ * Note: Any functions called through this macro will still be evaluated in ++ * Release builds (CONFIG_MALI_BIFROST_DEBUG not defined). Therefore, when ++ * KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied ++ * to this macro must: ++ * a) be static or static inline, and ++ * b) just return 0 and have no other statements present in the body. ++ */ ++#define KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, code, group, queue, info_val) \ ++ do { \ ++ /* capture values that could come from non-pure function calls */ \ ++ struct kbase_queue_group *__group = group; \ ++ struct kbase_queue *__queue = queue; \ ++ u64 __info_val = info_val; \ ++ KBASE_KTRACE_RBUF_ADD_CSF(kbdev, code, __group, __queue, 0u, \ ++ __info_val); \ ++ KBASE_KTRACE_FTRACE_ADD_CSF(kbdev, code, __group, \ ++ __queue, __info_val); \ ++ } while (0) ++ ++#endif /* _KBASE_DEBUG_KTRACE_CSF_H_ */ +diff --git a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_defs_csf.h b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_defs_csf.h +new file mode 100755 +index 000000000000..f265fe9a9753 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_defs_csf.h +@@ -0,0 +1,85 @@ ++/* ++ * ++ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_DEBUG_KTRACE_DEFS_CSF_H_ ++#define _KBASE_DEBUG_KTRACE_DEFS_CSF_H_ ++ ++#if KBASE_KTRACE_TARGET_RBUF ++/** ++ * DOC: KTrace version history, CSF variant ++ * ++ * 1.0: ++ * First version, with version information in the header. ++ * ++ * 1.1: ++ * kctx field is no longer a pointer, and is now an ID of the format %d_%u as ++ * used by kctx directories in mali debugfs entries: (tgid creating the kctx), ++ * (unique kctx id) ++ * ++ * ftrace backend now outputs kctx field (as %d_%u format). ++ * ++ * Add fields group, slot, prio, csi into backend-specific part. ++ */ ++#define KBASE_KTRACE_VERSION_MAJOR 1 ++#define KBASE_KTRACE_VERSION_MINOR 1 ++ ++/* indicates if the trace message has valid queue-group related info. */ ++#define KBASE_KTRACE_FLAG_CSF_GROUP (((kbase_ktrace_flag_t)1) << 0) ++ ++/* indicates if the trace message has valid queue related info. */ ++#define KBASE_KTRACE_FLAG_CSF_QUEUE (((kbase_ktrace_flag_t)1) << 1) ++ ++/* Collect all the flags together for debug checking */ ++#define KBASE_KTRACE_FLAG_BACKEND_ALL \ ++ (KBASE_KTRACE_FLAG_CSF_GROUP | KBASE_KTRACE_FLAG_CSF_QUEUE) ++ ++ ++/** ++ * struct kbase_ktrace_backend - backend specific part of a trace message ++ * ++ * @code: Identifies the event, refer to enum kbase_ktrace_code. ++ * @flags: indicates information about the trace message itself. Used ++ * during dumping of the message. ++ * @group_handle: Handle identifying the associated queue group. Only valid ++ * when @flags contains KBASE_KTRACE_FLAG_CSF_GROUP. ++ * @csg_nr: Number/index of the associated queue group's command stream ++ * group to which it is mapped, or negative if none associated. ++ * Only valid when @flags contains KBASE_KTRACE_FLAG_CSF_GROUP. ++ * @slot_prio: The priority of the slot for the associated group, if it was ++ * scheduled. Hence, only valid when @csg_nr >=0 and @flags ++ * contains KBASE_KTRACE_FLAG_CSF_GROUP. ++ * @csi_index: ID of the associated queue's Command Stream HW interface. ++ * Only valid when @flags contains KBASE_KTRACE_FLAG_CSF_QUEUE. ++ */ ++struct kbase_ktrace_backend { ++ /* Place 64 and 32-bit members together */ ++ /* Pack smaller members together */ ++ kbase_ktrace_code_t code; ++ kbase_ktrace_flag_t flags; ++ u8 group_handle; ++ s8 csg_nr; ++ u8 slot_prio; ++ s8 csi_index; ++}; ++ ++#endif /* KBASE_KTRACE_TARGET_RBUF */ ++#endif /* _KBASE_DEBUG_KTRACE_DEFS_CSF_H_ */ +diff --git a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_defs_jm.h b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_defs_jm.h +new file mode 100755 +index 000000000000..ea8e01a87f3f +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_defs_jm.h +@@ -0,0 +1,102 @@ ++/* ++ * ++ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_DEBUG_KTRACE_DEFS_JM_H_ ++#define _KBASE_DEBUG_KTRACE_DEFS_JM_H_ ++ ++#if KBASE_KTRACE_TARGET_RBUF ++/** ++ * DOC: KTrace version history, JM variant ++ * ++ * 1.0: ++ * Original version (implicit, header did not carry version information). ++ * ++ * 2.0: ++ * Introduced version information into the header. ++ * ++ * Some changes of parameter names in header. ++ * ++ * Trace now uses all 64-bits of info_val. ++ * ++ * Non-JM specific parts moved to using info_val instead of refcount/gpu_addr. ++ * ++ * 2.1: ++ * kctx field is no longer a pointer, and is now an ID of the format %d_%u as ++ * used by kctx directories in mali debugfs entries: (tgid creating the kctx), ++ * (unique kctx id). ++ * ++ * ftrace backend now outputs kctx field (as %d_%u format). ++ * ++ */ ++#define KBASE_KTRACE_VERSION_MAJOR 2 ++#define KBASE_KTRACE_VERSION_MINOR 1 ++#endif /* KBASE_KTRACE_TARGET_RBUF */ ++ ++/* ++ * Note: mali_kbase_debug_ktrace_jm.h needs these value even if the RBUF target ++ * is disabled (they get discarded with CSTD_UNUSED(), but they're still ++ * referenced) ++ */ ++ ++/* indicates if the trace message has a valid refcount member */ ++#define KBASE_KTRACE_FLAG_JM_REFCOUNT (((kbase_ktrace_flag_t)1) << 0) ++/* indicates if the trace message has a valid jobslot member */ ++#define KBASE_KTRACE_FLAG_JM_JOBSLOT (((kbase_ktrace_flag_t)1) << 1) ++/* indicates if the trace message has valid atom related info. */ ++#define KBASE_KTRACE_FLAG_JM_ATOM (((kbase_ktrace_flag_t)1) << 2) ++ ++#if KBASE_KTRACE_TARGET_RBUF ++/* Collect all the flags together for debug checking */ ++#define KBASE_KTRACE_FLAG_BACKEND_ALL \ ++ (KBASE_KTRACE_FLAG_JM_REFCOUNT | KBASE_KTRACE_FLAG_JM_JOBSLOT \ ++ | KBASE_KTRACE_FLAG_JM_ATOM) ++ ++/** ++ * struct kbase_ktrace_backend - backend specific part of a trace message ++ * ++ * @atom_udata: Copy of the user data sent for the atom in base_jd_submit. ++ * Only valid if KBASE_KTRACE_FLAG_JM_ATOM is set in @flags ++ * @gpu_addr: GPU address, usually of the job-chain represented by an atom. ++ * @atom_number: id of the atom for which trace message was added. Only valid ++ * if KBASE_KTRACE_FLAG_JM_ATOM is set in @flags ++ * @code: Identifies the event, refer to enum kbase_ktrace_code. ++ * @flags: indicates information about the trace message itself. Used ++ * during dumping of the message. ++ * @jobslot: job-slot for which trace message was added, valid only for ++ * job-slot management events. ++ * @refcount: reference count for the context, valid for certain events ++ * related to scheduler core and policy. ++ */ ++struct kbase_ktrace_backend { ++ /* Place 64 and 32-bit members together */ ++ u64 atom_udata[2]; /* Only valid for KBASE_KTRACE_FLAG_JM_ATOM */ ++ u64 gpu_addr; ++ int atom_number; /* Only valid for KBASE_KTRACE_FLAG_JM_ATOM */ ++ /* Pack smaller members together */ ++ kbase_ktrace_code_t code; ++ kbase_ktrace_flag_t flags; ++ u8 jobslot; ++ u8 refcount; ++}; ++#endif /* KBASE_KTRACE_TARGET_RBUF */ ++ ++#endif /* _KBASE_DEBUG_KTRACE_DEFS_JM_H_ */ +diff --git a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_jm.c b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_jm.c +new file mode 100755 +index 000000000000..1b821281f09f +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_jm.c +@@ -0,0 +1,115 @@ ++/* ++ * ++ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++#include ++#include "debug/mali_kbase_debug_ktrace_internal.h" ++#include "debug/backend/mali_kbase_debug_ktrace_jm.h" ++ ++#if KBASE_KTRACE_TARGET_RBUF ++ ++void kbasep_ktrace_backend_format_header(char *buffer, int sz, s32 *written) ++{ ++ *written += MAX(snprintf(buffer + *written, MAX(sz - *written, 0), ++ "katom,gpu_addr,jobslot,refcount"), 0); ++} ++ ++void kbasep_ktrace_backend_format_msg(struct kbase_ktrace_msg *trace_msg, ++ char *buffer, int sz, s32 *written) ++{ ++ /* katom */ ++ if (trace_msg->backend.flags & KBASE_KTRACE_FLAG_JM_ATOM) ++ *written += MAX(snprintf(buffer + *written, ++ MAX(sz - *written, 0), ++ "atom %d (ud: 0x%llx 0x%llx)", ++ trace_msg->backend.atom_number, ++ trace_msg->backend.atom_udata[0], ++ trace_msg->backend.atom_udata[1]), 0); ++ ++ /* gpu_addr */ ++ if (trace_msg->backend.flags & KBASE_KTRACE_FLAG_BACKEND) ++ *written += MAX(snprintf(buffer + *written, ++ MAX(sz - *written, 0), ++ ",%.8llx,", trace_msg->backend.gpu_addr), 0); ++ else ++ *written += MAX(snprintf(buffer + *written, ++ MAX(sz - *written, 0), ++ ",,"), 0); ++ ++ /* jobslot */ ++ if (trace_msg->backend.flags & KBASE_KTRACE_FLAG_JM_JOBSLOT) ++ *written += MAX(snprintf(buffer + *written, ++ MAX(sz - *written, 0), ++ "%d", trace_msg->backend.jobslot), 0); ++ ++ *written += MAX(snprintf(buffer + *written, MAX(sz - *written, 0), ++ ","), 0); ++ ++ /* refcount */ ++ if (trace_msg->backend.flags & KBASE_KTRACE_FLAG_JM_REFCOUNT) ++ *written += MAX(snprintf(buffer + *written, ++ MAX(sz - *written, 0), ++ "%d", trace_msg->backend.refcount), 0); ++} ++ ++void kbasep_ktrace_add_jm(struct kbase_device *kbdev, ++ enum kbase_ktrace_code code, struct kbase_context *kctx, ++ struct kbase_jd_atom *katom, u64 gpu_addr, ++ kbase_ktrace_flag_t flags, int refcount, int jobslot, ++ u64 info_val) ++{ ++ unsigned long irqflags; ++ struct kbase_ktrace_msg *trace_msg; ++ ++ spin_lock_irqsave(&kbdev->ktrace.lock, irqflags); ++ ++ /* Reserve and update indices */ ++ trace_msg = kbasep_ktrace_reserve(&kbdev->ktrace); ++ ++ /* Fill the common part of the message (including backend.flags) */ ++ kbasep_ktrace_msg_init(&kbdev->ktrace, trace_msg, code, kctx, flags, ++ info_val); ++ ++ /* Indicate to the common code that backend-specific parts will be ++ * valid ++ */ ++ trace_msg->backend.flags |= KBASE_KTRACE_FLAG_BACKEND; ++ ++ /* Fill the JM-specific parts of the message */ ++ if (katom) { ++ trace_msg->backend.flags |= KBASE_KTRACE_FLAG_JM_ATOM; ++ ++ trace_msg->backend.atom_number = kbase_jd_atom_id(katom->kctx, katom); ++ trace_msg->backend.atom_udata[0] = katom->udata.blob[0]; ++ trace_msg->backend.atom_udata[1] = katom->udata.blob[1]; ++ } ++ ++ trace_msg->backend.gpu_addr = gpu_addr; ++ trace_msg->backend.jobslot = jobslot; ++ /* Clamp refcount */ ++ trace_msg->backend.refcount = MIN((unsigned int)refcount, 0xFF); ++ ++ WARN_ON((trace_msg->backend.flags & ~KBASE_KTRACE_FLAG_ALL)); ++ ++ /* Done */ ++ spin_unlock_irqrestore(&kbdev->ktrace.lock, irqflags); ++} ++ ++#endif /* KBASE_KTRACE_TARGET_RBUF */ +diff --git a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_jm.h b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_jm.h +new file mode 100755 +index 000000000000..adfcb1aa556e +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_jm.h +@@ -0,0 +1,309 @@ ++/* ++ * ++ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_DEBUG_KTRACE_JM_H_ ++#define _KBASE_DEBUG_KTRACE_JM_H_ ++ ++/* ++ * KTrace target for internal ringbuffer ++ */ ++#if KBASE_KTRACE_TARGET_RBUF ++/** ++ * kbasep_ktrace_add_jm - internal function to add trace about Job Management ++ * @kbdev: kbase device ++ * @code: trace code ++ * @kctx: kbase context, or NULL if no context ++ * @katom: kbase atom, or NULL if no atom ++ * @gpu_addr: GPU address, usually related to @katom ++ * @flags: flags about the message ++ * @refcount: reference count information to add to the trace ++ * @jobslot: jobslot information to add to the trace ++ * @info_val: generic information about @code to add to the trace ++ * ++ * PRIVATE: do not use directly. Use KBASE_KTRACE_ADD_JM() instead. ++ */ ++void kbasep_ktrace_add_jm(struct kbase_device *kbdev, ++ enum kbase_ktrace_code code, struct kbase_context *kctx, ++ struct kbase_jd_atom *katom, u64 gpu_addr, ++ kbase_ktrace_flag_t flags, int refcount, int jobslot, ++ u64 info_val); ++ ++#define KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, gpu_addr, flags, \ ++ refcount, jobslot, info_val) \ ++ kbasep_ktrace_add_jm(kbdev, KBASE_KTRACE_CODE(code), kctx, katom, \ ++ gpu_addr, flags, refcount, jobslot, info_val) ++ ++#else /* KBASE_KTRACE_TARGET_RBUF */ ++ ++#define KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, gpu_addr, flags, \ ++ refcount, jobslot, info_val) \ ++ do {\ ++ CSTD_UNUSED(kbdev);\ ++ CSTD_NOP(code);\ ++ CSTD_UNUSED(kctx);\ ++ CSTD_UNUSED(katom);\ ++ CSTD_UNUSED(gpu_addr);\ ++ CSTD_UNUSED(flags);\ ++ CSTD_UNUSED(refcount);\ ++ CSTD_UNUSED(jobslot);\ ++ CSTD_UNUSED(info_val);\ ++ CSTD_NOP(0);\ ++ } while (0) ++#endif /* KBASE_KTRACE_TARGET_RBUF */ ++ ++/* ++ * KTrace target for Linux's ftrace ++ * ++ * Note: the header file(s) that define the trace_mali_<...> tracepoints are ++ * included by the parent header file ++ */ ++#if KBASE_KTRACE_TARGET_FTRACE ++#define KBASE_KTRACE_FTRACE_ADD_JM_SLOT(kbdev, code, kctx, katom, gpu_addr, \ ++ jobslot) \ ++ trace_mali_##code(kctx, jobslot, 0) ++ ++#define KBASE_KTRACE_FTRACE_ADD_JM_SLOT_INFO(kbdev, code, kctx, katom, \ ++ gpu_addr, jobslot, info_val) \ ++ trace_mali_##code(kctx, jobslot, info_val) ++ ++#define KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, \ ++ gpu_addr, refcount) \ ++ trace_mali_##code(kctx, refcount, 0) ++ ++#define KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT_INFO(kbdev, code, kctx, katom, \ ++ gpu_addr, refcount, info_val) \ ++ trace_mali_##code(kctx, refcount, info_val) ++ ++#define KBASE_KTRACE_FTRACE_ADD_JM(kbdev, code, kctx, katom, gpu_addr, \ ++ info_val) \ ++ trace_mali_##code(kctx, gpu_addr, info_val) ++#else /* KBASE_KTRACE_TARGET_FTRACE */ ++#define KBASE_KTRACE_FTRACE_ADD_JM_SLOT(kbdev, code, kctx, katom, gpu_addr, \ ++ jobslot) \ ++ do {\ ++ CSTD_UNUSED(kbdev);\ ++ CSTD_NOP(code);\ ++ CSTD_UNUSED(kctx);\ ++ CSTD_UNUSED(katom);\ ++ CSTD_UNUSED(gpu_addr);\ ++ CSTD_UNUSED(jobslot);\ ++ CSTD_NOP(0);\ ++ } while (0) ++ ++#define KBASE_KTRACE_FTRACE_ADD_JM_SLOT_INFO(kbdev, code, kctx, katom, \ ++ gpu_addr, jobslot, info_val) \ ++ do {\ ++ CSTD_UNUSED(kbdev);\ ++ CSTD_NOP(code);\ ++ CSTD_UNUSED(kctx);\ ++ CSTD_UNUSED(katom);\ ++ CSTD_UNUSED(gpu_addr);\ ++ CSTD_UNUSED(jobslot);\ ++ CSTD_UNUSED(info_val);\ ++ CSTD_NOP(0);\ ++ } while (0) ++ ++#define KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, \ ++ gpu_addr, refcount) \ ++ do {\ ++ CSTD_UNUSED(kbdev);\ ++ CSTD_NOP(code);\ ++ CSTD_UNUSED(kctx);\ ++ CSTD_UNUSED(katom);\ ++ CSTD_UNUSED(gpu_addr);\ ++ CSTD_UNUSED(refcount);\ ++ CSTD_NOP(0);\ ++ } while (0) ++ ++#define KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT_INFO(kbdev, code, kctx, katom, \ ++ gpu_addr, refcount, info_val) \ ++ do {\ ++ CSTD_UNUSED(kbdev);\ ++ CSTD_NOP(code);\ ++ CSTD_UNUSED(kctx);\ ++ CSTD_UNUSED(katom);\ ++ CSTD_UNUSED(gpu_addr);\ ++ CSTD_UNUSED(info_val);\ ++ CSTD_NOP(0);\ ++ } while (0) ++ ++#define KBASE_KTRACE_FTRACE_ADD_JM(kbdev, code, kctx, katom, gpu_addr, \ ++ info_val)\ ++ do {\ ++ CSTD_UNUSED(kbdev);\ ++ CSTD_NOP(code);\ ++ CSTD_UNUSED(kctx);\ ++ CSTD_UNUSED(katom);\ ++ CSTD_UNUSED(gpu_addr);\ ++ CSTD_UNUSED(info_val);\ ++ CSTD_NOP(0);\ ++ } while (0) ++#endif /* KBASE_KTRACE_TARGET_FTRACE */ ++ ++/* ++ * Master set of macros to route KTrace to any of the targets ++ */ ++ ++/** ++ * KBASE_KTRACE_ADD_JM_SLOT - Add trace values about a job-slot ++ * @kbdev: kbase device ++ * @code: trace code ++ * @kctx: kbase context, or NULL if no context ++ * @katom: kbase atom, or NULL if no atom ++ * @gpu_addr: GPU address, usually related to @katom ++ * @jobslot: jobslot information to add to the trace ++ * ++ * Note: Any functions called through this macro will still be evaluated in ++ * Release builds (CONFIG_MALI_BIFROST_DEBUG not defined). Therefore, when ++ * KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied ++ * to this macro must: ++ * a) be static or static inline, and ++ * b) just return 0 and have no other statements present in the body. ++ */ ++#define KBASE_KTRACE_ADD_JM_SLOT(kbdev, code, kctx, katom, gpu_addr, \ ++ jobslot) \ ++ do { \ ++ /* capture values that could come from non-pure function calls */ \ ++ u64 __gpu_addr = gpu_addr; \ ++ int __jobslot = jobslot; \ ++ KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, __gpu_addr, \ ++ KBASE_KTRACE_FLAG_JM_JOBSLOT, 0, __jobslot, \ ++ 0); \ ++ KBASE_KTRACE_FTRACE_ADD_JM_SLOT(kbdev, code, kctx, katom, __gpu_addr, __jobslot); \ ++ } while (0) ++ ++/** ++ * KBASE_KTRACE_ADD_JM_SLOT_INFO - Add trace values about a job-slot, with info ++ * @kbdev: kbase device ++ * @code: trace code ++ * @kctx: kbase context, or NULL if no context ++ * @katom: kbase atom, or NULL if no atom ++ * @gpu_addr: GPU address, usually related to @katom ++ * @jobslot: jobslot information to add to the trace ++ * @info_val: generic information about @code to add to the trace ++ * ++ * Note: Any functions called through this macro will still be evaluated in ++ * Release builds (CONFIG_MALI_BIFROST_DEBUG not defined). Therefore, when ++ * KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied ++ * to this macro must: ++ * a) be static or static inline, and ++ * b) just return 0 and have no other statements present in the body. ++ */ ++#define KBASE_KTRACE_ADD_JM_SLOT_INFO(kbdev, code, kctx, katom, gpu_addr, \ ++ jobslot, info_val) \ ++ do { \ ++ /* capture values that could come from non-pure function calls */ \ ++ u64 __gpu_addr = gpu_addr; \ ++ int __jobslot = jobslot; \ ++ u64 __info_val = info_val; \ ++ KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, __gpu_addr, \ ++ KBASE_KTRACE_FLAG_JM_JOBSLOT, 0, __jobslot, \ ++ __info_val); \ ++ KBASE_KTRACE_FTRACE_ADD_JM_SLOT_INFO(kbdev, code, kctx, katom, __gpu_addr, __jobslot, __info_val); \ ++ } while (0) ++ ++/** ++ * KBASE_KTRACE_ADD_JM_REFCOUNT - Add trace values about a kctx refcount ++ * @kbdev: kbase device ++ * @code: trace code ++ * @kctx: kbase context, or NULL if no context ++ * @katom: kbase atom, or NULL if no atom ++ * @gpu_addr: GPU address, usually related to @katom ++ * @refcount: reference count information to add to the trace ++ * ++ * Note: Any functions called through this macro will still be evaluated in ++ * Release builds (CONFIG_MALI_BIFROST_DEBUG not defined). Therefore, when ++ * KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied ++ * to this macro must: ++ * a) be static or static inline, and ++ * b) just return 0 and have no other statements present in the body. ++ */ ++#define KBASE_KTRACE_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, gpu_addr, \ ++ refcount) \ ++ do { \ ++ /* capture values that could come from non-pure function calls */ \ ++ u64 __gpu_addr = gpu_addr; \ ++ int __refcount = refcount; \ ++ KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, __gpu_addr, \ ++ KBASE_KTRACE_FLAG_JM_REFCOUNT, __refcount, 0, \ ++ 0u); \ ++ KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, __gpu_addr, __refcount); \ ++ } while (0) ++ ++/** ++ * KBASE_KTRACE_ADD_JM_REFCOUNT_INFO - Add trace values about a kctx refcount, ++ * and info ++ * @kbdev: kbase device ++ * @code: trace code ++ * @kctx: kbase context, or NULL if no context ++ * @katom: kbase atom, or NULL if no atom ++ * @gpu_addr: GPU address, usually related to @katom ++ * @refcount: reference count information to add to the trace ++ * @info_val: generic information about @code to add to the trace ++ * ++ * Note: Any functions called through this macro will still be evaluated in ++ * Release builds (CONFIG_MALI_BIFROST_DEBUG not defined). Therefore, when ++ * KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied ++ * to this macro must: ++ * a) be static or static inline, and ++ * b) just return 0 and have no other statements present in the body. ++ */ ++#define KBASE_KTRACE_ADD_JM_REFCOUNT_INFO(kbdev, code, kctx, katom, \ ++ gpu_addr, refcount, info_val) \ ++ do { \ ++ /* capture values that could come from non-pure function calls */ \ ++ u64 __gpu_addr = gpu_addr; \ ++ int __refcount = refcount; \ ++ u64 __info_val = info_val; \ ++ KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, __gpu_addr, \ ++ KBASE_KTRACE_FLAG_JM_REFCOUNT, __refcount, 0, \ ++ __info_val); \ ++ KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, __gpu_addr, __refcount, __info_val); \ ++ } while (0) ++ ++/** ++ * KBASE_KTRACE_ADD_JM - Add trace values (no slot or refcount) ++ * @kbdev: kbase device ++ * @code: trace code ++ * @kctx: kbase context, or NULL if no context ++ * @katom: kbase atom, or NULL if no atom ++ * @gpu_addr: GPU address, usually related to @katom ++ * @info_val: generic information about @code to add to the trace ++ * ++ * Note: Any functions called through this macro will still be evaluated in ++ * Release builds (CONFIG_MALI_BIFROST_DEBUG not defined). Therefore, when ++ * KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied ++ * to this macro must: ++ * a) be static or static inline, and ++ * b) just return 0 and have no other statements present in the body. ++ */ ++#define KBASE_KTRACE_ADD_JM(kbdev, code, kctx, katom, gpu_addr, info_val) \ ++ do { \ ++ /* capture values that could come from non-pure function calls */ \ ++ u64 __gpu_addr = gpu_addr; \ ++ u64 __info_val = info_val; \ ++ KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, __gpu_addr, \ ++ 0u, 0, 0, __info_val); \ ++ KBASE_KTRACE_FTRACE_ADD_JM(kbdev, code, kctx, katom, __gpu_addr, __info_val); \ ++ } while (0) ++ ++#endif /* _KBASE_DEBUG_KTRACE_JM_H_ */ +diff --git a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_linux_ktrace_csf.h b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_linux_ktrace_csf.h +new file mode 100755 +index 000000000000..d103e5766456 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_linux_ktrace_csf.h +@@ -0,0 +1,147 @@ ++/* ++ * ++ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* ++ * NOTE: This must **only** be included through mali_linux_trace.h, ++ * otherwise it will fail to setup tracepoints correctly ++ */ ++ ++#if !defined(_KBASE_DEBUG_LINUX_KTRACE_CSF_H_) || defined(TRACE_HEADER_MULTI_READ) ++#define _KBASE_DEBUG_LINUX_KTRACE_CSF_H_ ++ ++/* ++ * Generic CSF events - using the common DEFINE_MALI_ADD_EVENT ++ */ ++DEFINE_MALI_ADD_EVENT(EVICT_CTX_SLOTS); ++DEFINE_MALI_ADD_EVENT(FIRMWARE_BOOT); ++DEFINE_MALI_ADD_EVENT(FIRMWARE_REBOOT); ++DEFINE_MALI_ADD_EVENT(SCHEDULER_TOCK); ++DEFINE_MALI_ADD_EVENT(SCHEDULER_TICK); ++DEFINE_MALI_ADD_EVENT(SCHEDULER_RESET); ++DEFINE_MALI_ADD_EVENT(SCHEDULER_EXIT_PROTM); ++DEFINE_MALI_ADD_EVENT(SYNC_UPDATE_EVENT); ++ ++DECLARE_EVENT_CLASS(mali_csf_grp_q_template, ++ TP_PROTO(struct kbase_device *kbdev, struct kbase_queue_group *group, ++ struct kbase_queue *queue, u64 info_val), ++ TP_ARGS(kbdev, group, queue, info_val), ++ TP_STRUCT__entry( ++ __field(u64, info_val) ++ __field(pid_t, kctx_tgid) ++ __field(u32, kctx_id) ++ __field(u8, group_handle) ++ __field(s8, csg_nr) ++ __field(u8, slot_prio) ++ __field(s8, csi_index) ++ ), ++ TP_fast_assign( ++ { ++ struct kbase_context *kctx = NULL; ++ ++ __entry->info_val = info_val; ++ /* Note: if required in future, we could record some ++ * flags in __entry about whether the group/queue parts ++ * are valid, and add that to the trace message e.g. ++ * by using __print_flags()/__print_symbolic() ++ */ ++ if (queue) { ++ /* Note: kctx overridden by group->kctx later if group is valid */ ++ kctx = queue->kctx; ++ __entry->csi_index = queue->csi_index; ++ } else { ++ __entry->csi_index = -1; ++ } ++ ++ if (group) { ++ kctx = group->kctx; ++ __entry->group_handle = group->handle; ++ __entry->csg_nr = group->csg_nr; ++ if (group->csg_nr >= 0) ++ __entry->slot_prio = kbdev->csf.scheduler.csg_slots[group->csg_nr].priority; ++ else ++ __entry->slot_prio = 0u; ++ } else { ++ __entry->group_handle = 0u; ++ __entry->csg_nr = -1; ++ __entry->slot_prio = 0u; ++ } ++ __entry->kctx_id = (kctx) ? kctx->id : 0u; ++ __entry->kctx_tgid = (kctx) ? kctx->tgid : 0; ++ } ++ ++ ), ++ TP_printk("kctx=%d_%u group=%u slot=%d prio=%u csi=%d info=0x%llx", ++ __entry->kctx_tgid, __entry->kctx_id, ++ __entry->group_handle, __entry->csg_nr, ++ __entry->slot_prio, __entry->csi_index, ++ __entry->info_val) ++); ++ ++/* ++ * Group events ++ */ ++#define DEFINE_MALI_CSF_GRP_EVENT(name) \ ++ DEFINE_EVENT_PRINT(mali_csf_grp_q_template, mali_##name, \ ++ TP_PROTO(struct kbase_device *kbdev, struct kbase_queue_group *group, \ ++ struct kbase_queue *queue, u64 info_val), \ ++ TP_ARGS(kbdev, group, queue, info_val), \ ++ TP_printk("kctx=%d_%u group=%u slot=%d prio=%u info=0x%llx", \ ++ __entry->kctx_tgid, __entry->kctx_id, __entry->group_handle, \ ++ __entry->csg_nr, __entry->slot_prio, __entry->info_val)) ++ ++DEFINE_MALI_CSF_GRP_EVENT(CSG_SLOT_START); ++DEFINE_MALI_CSF_GRP_EVENT(CSG_SLOT_STOP); ++DEFINE_MALI_CSF_GRP_EVENT(CSG_SLOT_STARTED); ++DEFINE_MALI_CSF_GRP_EVENT(CSG_SLOT_STOPPED); ++DEFINE_MALI_CSF_GRP_EVENT(CSG_SLOT_CLEANED); ++DEFINE_MALI_CSF_GRP_EVENT(CSG_PRIO_UPDATE); ++DEFINE_MALI_CSF_GRP_EVENT(CSG_SYNC_UPDATE_INTERRUPT); ++DEFINE_MALI_CSF_GRP_EVENT(CSG_IDLE_INTERRUPT); ++DEFINE_MALI_CSF_GRP_EVENT(GROUP_SYNC_UPDATE_DONE); ++DEFINE_MALI_CSF_GRP_EVENT(GROUP_DESCHEDULE); ++DEFINE_MALI_CSF_GRP_EVENT(GROUP_SCHEDULE); ++DEFINE_MALI_CSF_GRP_EVENT(GROUP_EVICT_SCHED); ++DEFINE_MALI_CSF_GRP_EVENT(SCHEDULER_ENTER_PROTM); ++DEFINE_MALI_CSF_GRP_EVENT(SCHEDULER_TOP_GRP); ++ ++#undef DEFINE_MALI_CSF_GRP_EVENT ++ ++/* ++ * Group + Queue events ++ */ ++#define DEFINE_MALI_CSF_GRP_Q_EVENT(name) \ ++ DEFINE_EVENT(mali_csf_grp_q_template, mali_##name, \ ++ TP_PROTO(struct kbase_device *kbdev, struct kbase_queue_group *group, \ ++ struct kbase_queue *queue, u64 info_val), \ ++ TP_ARGS(kbdev, group, queue, info_val)) ++ ++DEFINE_MALI_CSF_GRP_Q_EVENT(CSI_START); ++DEFINE_MALI_CSF_GRP_Q_EVENT(CSI_STOP); ++DEFINE_MALI_CSF_GRP_Q_EVENT(CSI_STOP_REQUESTED); ++DEFINE_MALI_CSF_GRP_Q_EVENT(CSI_FAULT_INTERRUPT); ++DEFINE_MALI_CSF_GRP_Q_EVENT(CSI_TILER_OOM_INTERRUPT); ++DEFINE_MALI_CSF_GRP_Q_EVENT(QUEUE_START); ++DEFINE_MALI_CSF_GRP_Q_EVENT(QUEUE_STOP); ++ ++#undef DEFINE_MALI_CSF_GRP_Q_EVENT ++ ++#endif /* !defined(_KBASE_DEBUG_LINUX_KTRACE_CSF_H_) || defined(TRACE_HEADER_MULTI_READ) */ +diff --git a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_linux_ktrace_jm.h b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_linux_ktrace_jm.h +new file mode 100755 +index 000000000000..037b1edecd8e +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_linux_ktrace_jm.h +@@ -0,0 +1,165 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014,2018,2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* ++ * NOTE: This must **only** be included through mali_linux_trace.h, ++ * otherwise it will fail to setup tracepoints correctly ++ */ ++ ++#if !defined(_KBASE_DEBUG_LINUX_KTRACE_JM_H_) || defined(TRACE_HEADER_MULTI_READ) ++#define _KBASE_DEBUG_LINUX_KTRACE_JM_H_ ++ ++DECLARE_EVENT_CLASS(mali_jm_slot_template, ++ TP_PROTO(struct kbase_context *kctx, int jobslot, u64 info_val), ++ TP_ARGS(kctx, jobslot, info_val), ++ TP_STRUCT__entry( ++ __field(pid_t, kctx_tgid) ++ __field(u32, kctx_id) ++ __field(unsigned int, jobslot) ++ __field(u64, info_val) ++ ), ++ TP_fast_assign( ++ __entry->kctx_id = (kctx) ? kctx->id : 0u; ++ __entry->kctx_tgid = (kctx) ? kctx->tgid : 0; ++ __entry->jobslot = jobslot; ++ __entry->info_val = info_val; ++ ), ++ TP_printk("kctx=%d_%u jobslot=%u info=0x%llx", __entry->kctx_tgid, ++ __entry->kctx_id, __entry->jobslot, __entry->info_val) ++); ++ ++#define DEFINE_MALI_JM_SLOT_EVENT(name) \ ++DEFINE_EVENT(mali_jm_slot_template, mali_##name, \ ++ TP_PROTO(struct kbase_context *kctx, int jobslot, u64 info_val), \ ++ TP_ARGS(kctx, jobslot, info_val)) ++DEFINE_MALI_JM_SLOT_EVENT(JM_SUBMIT); ++DEFINE_MALI_JM_SLOT_EVENT(JM_JOB_DONE); ++DEFINE_MALI_JM_SLOT_EVENT(JM_UPDATE_HEAD); ++DEFINE_MALI_JM_SLOT_EVENT(JM_CHECK_HEAD); ++DEFINE_MALI_JM_SLOT_EVENT(JM_SOFTSTOP); ++DEFINE_MALI_JM_SLOT_EVENT(JM_SOFTSTOP_0); ++DEFINE_MALI_JM_SLOT_EVENT(JM_SOFTSTOP_1); ++DEFINE_MALI_JM_SLOT_EVENT(JM_HARDSTOP); ++DEFINE_MALI_JM_SLOT_EVENT(JM_HARDSTOP_0); ++DEFINE_MALI_JM_SLOT_EVENT(JM_HARDSTOP_1); ++DEFINE_MALI_JM_SLOT_EVENT(JM_SLOT_SOFT_OR_HARD_STOP); ++DEFINE_MALI_JM_SLOT_EVENT(JM_SLOT_EVICT); ++DEFINE_MALI_JM_SLOT_EVENT(JM_BEGIN_RESET_WORKER); ++DEFINE_MALI_JM_SLOT_EVENT(JM_END_RESET_WORKER); ++DEFINE_MALI_JM_SLOT_EVENT(JS_CORE_REF_REGISTER_ON_RECHECK_FAILED); ++DEFINE_MALI_JM_SLOT_EVENT(JS_AFFINITY_SUBMIT_TO_BLOCKED); ++DEFINE_MALI_JM_SLOT_EVENT(JS_AFFINITY_CURRENT); ++DEFINE_MALI_JM_SLOT_EVENT(JD_DONE_TRY_RUN_NEXT_JOB); ++DEFINE_MALI_JM_SLOT_EVENT(JS_CORE_REF_REQUEST_CORES_FAILED); ++DEFINE_MALI_JM_SLOT_EVENT(JS_CORE_REF_REGISTER_INUSE_FAILED); ++DEFINE_MALI_JM_SLOT_EVENT(JS_CORE_REF_REQUEST_ON_RECHECK_FAILED); ++DEFINE_MALI_JM_SLOT_EVENT(JS_CORE_REF_AFFINITY_WOULD_VIOLATE); ++DEFINE_MALI_JM_SLOT_EVENT(JS_JOB_DONE_TRY_RUN_NEXT_JOB); ++DEFINE_MALI_JM_SLOT_EVENT(JS_JOB_DONE_RETRY_NEEDED); ++DEFINE_MALI_JM_SLOT_EVENT(JS_POLICY_DEQUEUE_JOB); ++DEFINE_MALI_JM_SLOT_EVENT(JS_POLICY_DEQUEUE_JOB_IRQ); ++#undef DEFINE_MALI_JM_SLOT_EVENT ++ ++DECLARE_EVENT_CLASS(mali_jm_refcount_template, ++ TP_PROTO(struct kbase_context *kctx, int refcount, u64 info_val), ++ TP_ARGS(kctx, refcount, info_val), ++ TP_STRUCT__entry( ++ __field(pid_t, kctx_tgid) ++ __field(u32, kctx_id) ++ __field(unsigned int, refcount) ++ __field(u64, info_val) ++ ), ++ TP_fast_assign( ++ __entry->kctx_id = (kctx) ? kctx->id : 0u; ++ __entry->kctx_tgid = (kctx) ? kctx->tgid : 0; ++ __entry->refcount = refcount; ++ __entry->info_val = info_val; ++ ), ++ TP_printk("kctx=%d_%u refcount=%u info=0x%llx", __entry->kctx_tgid, ++ __entry->kctx_id, __entry->refcount, __entry->info_val) ++); ++ ++#define DEFINE_MALI_JM_REFCOUNT_EVENT(name) \ ++DEFINE_EVENT(mali_jm_refcount_template, mali_##name, \ ++ TP_PROTO(struct kbase_context *kctx, int refcount, u64 info_val), \ ++ TP_ARGS(kctx, refcount, info_val)) ++DEFINE_MALI_JM_REFCOUNT_EVENT(JS_ADD_JOB); ++DEFINE_MALI_JM_REFCOUNT_EVENT(JS_REMOVE_JOB); ++DEFINE_MALI_JM_REFCOUNT_EVENT(JS_TRY_SCHEDULE_HEAD_CTX); ++DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_INIT_CTX); ++DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_TERM_CTX); ++DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_ENQUEUE_CTX); ++DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_DEQUEUE_HEAD_CTX); ++DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_TRY_EVICT_CTX); ++DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_RUNPOOL_ADD_CTX); ++DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_RUNPOOL_REMOVE_CTX); ++DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_FOREACH_CTX_JOBS); ++#undef DEFINE_MALI_JM_REFCOUNT_EVENT ++ ++DECLARE_EVENT_CLASS(mali_jm_add_template, ++ TP_PROTO(struct kbase_context *kctx, u64 gpu_addr, u64 info_val), ++ TP_ARGS(kctx, gpu_addr, info_val), ++ TP_STRUCT__entry( ++ __field(pid_t, kctx_tgid) ++ __field(u32, kctx_id) ++ __field(u64, gpu_addr) ++ __field(u64, info_val) ++ ), ++ TP_fast_assign( ++ __entry->kctx_id = (kctx) ? kctx->id : 0u; ++ __entry->kctx_tgid = (kctx) ? kctx->tgid : 0; ++ __entry->gpu_addr = gpu_addr; ++ __entry->info_val = info_val; ++ ), ++ TP_printk("kctx=%d_%u gpu_addr=0x%llx info=0x%llx", __entry->kctx_tgid, ++ __entry->kctx_id, __entry->gpu_addr, __entry->info_val) ++); ++ ++#define DEFINE_MALI_JM_ADD_EVENT(name) \ ++DEFINE_EVENT(mali_jm_add_template, mali_##name, \ ++ TP_PROTO(struct kbase_context *kctx, u64 gpu_addr, u64 info_val), \ ++ TP_ARGS(kctx, gpu_addr, info_val)) ++DEFINE_MALI_JM_ADD_EVENT(JD_DONE_WORKER); ++DEFINE_MALI_JM_ADD_EVENT(JD_DONE_WORKER_END); ++DEFINE_MALI_JM_ADD_EVENT(JD_CANCEL_WORKER); ++DEFINE_MALI_JM_ADD_EVENT(JD_DONE); ++DEFINE_MALI_JM_ADD_EVENT(JD_CANCEL); ++DEFINE_MALI_JM_ADD_EVENT(JD_ZAP_CONTEXT); ++DEFINE_MALI_JM_ADD_EVENT(JM_IRQ); ++DEFINE_MALI_JM_ADD_EVENT(JM_IRQ_END); ++DEFINE_MALI_JM_ADD_EVENT(JM_FLUSH_WORKQS); ++DEFINE_MALI_JM_ADD_EVENT(JM_FLUSH_WORKQS_DONE); ++DEFINE_MALI_JM_ADD_EVENT(JM_ZAP_NON_SCHEDULED); ++DEFINE_MALI_JM_ADD_EVENT(JM_ZAP_SCHEDULED); ++DEFINE_MALI_JM_ADD_EVENT(JM_ZAP_DONE); ++DEFINE_MALI_JM_ADD_EVENT(JM_SUBMIT_AFTER_RESET); ++DEFINE_MALI_JM_ADD_EVENT(JM_JOB_COMPLETE); ++DEFINE_MALI_JM_ADD_EVENT(JS_CTX_ATTR_NOW_ON_RUNPOOL); ++DEFINE_MALI_JM_ADD_EVENT(JS_CTX_ATTR_NOW_OFF_RUNPOOL); ++DEFINE_MALI_JM_ADD_EVENT(JS_CTX_ATTR_NOW_ON_CTX); ++DEFINE_MALI_JM_ADD_EVENT(JS_CTX_ATTR_NOW_OFF_CTX); ++DEFINE_MALI_JM_ADD_EVENT(JS_POLICY_TIMER_END); ++DEFINE_MALI_JM_ADD_EVENT(JS_POLICY_TIMER_START); ++DEFINE_MALI_JM_ADD_EVENT(JS_POLICY_ENQUEUE_JOB); ++#undef DEFINE_MALI_JM_ADD_EVENT ++ ++#endif /* !defined(_KBASE_DEBUG_LINUX_KTRACE_JM_H_) || defined(TRACE_HEADER_MULTI_READ)*/ +diff --git a/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace.c b/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace.c +new file mode 100755 +index 000000000000..a13c0ba20c94 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace.c +@@ -0,0 +1,356 @@ ++/* ++ * ++ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++#include ++#include "debug/mali_kbase_debug_ktrace_internal.h" ++ ++int kbase_ktrace_init(struct kbase_device *kbdev) ++{ ++#if KBASE_KTRACE_TARGET_RBUF ++ struct kbase_ktrace_msg *rbuf; ++ ++ rbuf = kmalloc_array(KBASE_KTRACE_SIZE, sizeof(*rbuf), GFP_KERNEL); ++ ++ if (!rbuf) ++ return -EINVAL; ++ ++ kbdev->ktrace.rbuf = rbuf; ++ spin_lock_init(&kbdev->ktrace.lock); ++#endif /* KBASE_KTRACE_TARGET_RBUF */ ++ return 0; ++} ++ ++void kbase_ktrace_term(struct kbase_device *kbdev) ++{ ++#if KBASE_KTRACE_TARGET_RBUF ++ kfree(kbdev->ktrace.rbuf); ++#endif /* KBASE_KTRACE_TARGET_RBUF */ ++} ++ ++void kbase_ktrace_hook_wrapper(void *param) ++{ ++ struct kbase_device *kbdev = (struct kbase_device *)param; ++ ++ KBASE_KTRACE_DUMP(kbdev); ++} ++ ++#if KBASE_KTRACE_TARGET_RBUF ++ ++static const char * const kbasep_ktrace_code_string[] = { ++ /* ++ * IMPORTANT: USE OF SPECIAL #INCLUDE OF NON-STANDARD HEADER FILE ++ * THIS MUST BE USED AT THE START OF THE ARRAY ++ */ ++#define KBASE_KTRACE_CODE_MAKE_CODE(X) # X ++#include "debug/mali_kbase_debug_ktrace_codes.h" ++#undef KBASE_KTRACE_CODE_MAKE_CODE ++}; ++ ++static void kbasep_ktrace_format_header(char *buffer, int sz, s32 written) ++{ ++ written += MAX(snprintf(buffer + written, MAX(sz - written, 0), ++ "secs,thread_id,cpu,code,kctx,"), 0); ++ ++ kbasep_ktrace_backend_format_header(buffer, sz, &written); ++ ++ written += MAX(snprintf(buffer + written, MAX(sz - written, 0), ++ ",info_val,ktrace_version=%u.%u", ++ KBASE_KTRACE_VERSION_MAJOR, ++ KBASE_KTRACE_VERSION_MINOR), 0); ++ ++ buffer[sz - 1] = 0; ++} ++ ++static void kbasep_ktrace_format_msg(struct kbase_ktrace_msg *trace_msg, ++ char *buffer, int sz) ++{ ++ s32 written = 0; ++ ++ /* Initial part of message: ++ * ++ * secs,thread_id,cpu,code, ++ */ ++ written += MAX(snprintf(buffer + written, MAX(sz - written, 0), ++ "%d.%.6d,%d,%d,%s,", ++ (int)trace_msg->timestamp.tv_sec, ++ (int)(trace_msg->timestamp.tv_nsec / 1000), ++ trace_msg->thread_id, trace_msg->cpu, ++ kbasep_ktrace_code_string[trace_msg->backend.code]), 0); ++ ++ /* kctx part: */ ++ if (trace_msg->kctx_tgid) { ++ written += MAX(snprintf(buffer + written, MAX(sz - written, 0), ++ "%d_%u", ++ trace_msg->kctx_tgid, trace_msg->kctx_id), 0); ++ } ++ /* Trailing comma */ ++ written += MAX(snprintf(buffer + written, MAX(sz - written, 0), ++ ","), 0); ++ ++ /* Backend parts */ ++ kbasep_ktrace_backend_format_msg(trace_msg, buffer, sz, ++ &written); ++ ++ /* Rest of message: ++ * ++ * ,info_val ++ * ++ * Note that the last column is empty, it's simply to hold the ktrace ++ * version in the header ++ */ ++ written += MAX(snprintf(buffer + written, MAX(sz - written, 0), ++ ",0x%.16llx", ++ (unsigned long long)trace_msg->info_val), 0); ++ buffer[sz - 1] = 0; ++} ++ ++static void kbasep_ktrace_dump_msg(struct kbase_device *kbdev, ++ struct kbase_ktrace_msg *trace_msg) ++{ ++ char buffer[KTRACE_DUMP_MESSAGE_SIZE]; ++ ++ lockdep_assert_held(&kbdev->ktrace.lock); ++ ++ kbasep_ktrace_format_msg(trace_msg, buffer, sizeof(buffer)); ++ dev_dbg(kbdev->dev, "%s", buffer); ++} ++ ++struct kbase_ktrace_msg *kbasep_ktrace_reserve(struct kbase_ktrace *ktrace) ++{ ++ struct kbase_ktrace_msg *trace_msg; ++ ++ lockdep_assert_held(&ktrace->lock); ++ ++ trace_msg = &ktrace->rbuf[ktrace->next_in]; ++ ++ /* Update the ringbuffer indices */ ++ ktrace->next_in = (ktrace->next_in + 1) & KBASE_KTRACE_MASK; ++ if (ktrace->next_in == ktrace->first_out) ++ ktrace->first_out = (ktrace->first_out + 1) & KBASE_KTRACE_MASK; ++ ++ return trace_msg; ++} ++void kbasep_ktrace_msg_init(struct kbase_ktrace *ktrace, ++ struct kbase_ktrace_msg *trace_msg, enum kbase_ktrace_code code, ++ struct kbase_context *kctx, kbase_ktrace_flag_t flags, ++ u64 info_val) ++{ ++ lockdep_assert_held(&ktrace->lock); ++ ++ trace_msg->thread_id = task_pid_nr(current); ++ trace_msg->cpu = task_cpu(current); ++ ++ ktime_get_real_ts64(&trace_msg->timestamp); ++ ++ /* No need to store a flag about whether there was a kctx, tgid==0 is ++ * sufficient ++ */ ++ if (kctx) { ++ trace_msg->kctx_tgid = kctx->tgid; ++ trace_msg->kctx_id = kctx->id; ++ } else { ++ trace_msg->kctx_tgid = 0; ++ trace_msg->kctx_id = 0; ++ } ++ trace_msg->info_val = info_val; ++ trace_msg->backend.code = code; ++ trace_msg->backend.flags = flags; ++} ++ ++void kbasep_ktrace_add(struct kbase_device *kbdev, enum kbase_ktrace_code code, ++ struct kbase_context *kctx, kbase_ktrace_flag_t flags, ++ u64 info_val) ++{ ++ unsigned long irqflags; ++ struct kbase_ktrace_msg *trace_msg; ++ ++ WARN_ON((flags & ~KBASE_KTRACE_FLAG_COMMON_ALL)); ++ ++ spin_lock_irqsave(&kbdev->ktrace.lock, irqflags); ++ ++ /* Reserve and update indices */ ++ trace_msg = kbasep_ktrace_reserve(&kbdev->ktrace); ++ ++ /* Fill the common part of the message (including backend.flags) */ ++ kbasep_ktrace_msg_init(&kbdev->ktrace, trace_msg, code, kctx, flags, ++ info_val); ++ ++ /* Done */ ++ spin_unlock_irqrestore(&kbdev->ktrace.lock, irqflags); ++} ++ ++static void kbasep_ktrace_clear_locked(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->ktrace.lock); ++ kbdev->ktrace.first_out = kbdev->ktrace.next_in; ++} ++void kbasep_ktrace_clear(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->ktrace.lock, flags); ++ kbasep_ktrace_clear_locked(kbdev); ++ spin_unlock_irqrestore(&kbdev->ktrace.lock, flags); ++} ++ ++void kbasep_ktrace_dump(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ u32 start; ++ u32 end; ++ char buffer[KTRACE_DUMP_MESSAGE_SIZE] = "Dumping trace:\n"; ++ ++ kbasep_ktrace_format_header(buffer, sizeof(buffer), strlen(buffer)); ++ dev_dbg(kbdev->dev, "%s", buffer); ++ ++ spin_lock_irqsave(&kbdev->ktrace.lock, flags); ++ start = kbdev->ktrace.first_out; ++ end = kbdev->ktrace.next_in; ++ ++ while (start != end) { ++ struct kbase_ktrace_msg *trace_msg = &kbdev->ktrace.rbuf[start]; ++ ++ kbasep_ktrace_dump_msg(kbdev, trace_msg); ++ ++ start = (start + 1) & KBASE_KTRACE_MASK; ++ } ++ dev_dbg(kbdev->dev, "TRACE_END"); ++ ++ kbasep_ktrace_clear_locked(kbdev); ++ ++ spin_unlock_irqrestore(&kbdev->ktrace.lock, flags); ++} ++ ++#ifdef CONFIG_DEBUG_FS ++struct trace_seq_state { ++ struct kbase_ktrace_msg trace_buf[KBASE_KTRACE_SIZE]; ++ u32 start; ++ u32 end; ++}; ++ ++static void *kbasep_ktrace_seq_start(struct seq_file *s, loff_t *pos) ++{ ++ struct trace_seq_state *state = s->private; ++ int i; ++ ++ if (*pos == 0) ++ /* See Documentation/filesystems/seq_file.txt */ ++ return SEQ_START_TOKEN; ++ ++ if (*pos > KBASE_KTRACE_SIZE) ++ return NULL; ++ i = state->start + *pos; ++ if ((state->end >= state->start && i >= state->end) || ++ i >= state->end + KBASE_KTRACE_SIZE) ++ return NULL; ++ ++ i &= KBASE_KTRACE_MASK; ++ ++ return &state->trace_buf[i]; ++} ++ ++static void kbasep_ktrace_seq_stop(struct seq_file *s, void *data) ++{ ++} ++ ++static void *kbasep_ktrace_seq_next(struct seq_file *s, void *data, loff_t *pos) ++{ ++ struct trace_seq_state *state = s->private; ++ int i; ++ ++ if (data != SEQ_START_TOKEN) ++ (*pos)++; ++ ++ i = (state->start + *pos) & KBASE_KTRACE_MASK; ++ if (i == state->end) ++ return NULL; ++ ++ return &state->trace_buf[i]; ++} ++ ++static int kbasep_ktrace_seq_show(struct seq_file *s, void *data) ++{ ++ struct kbase_ktrace_msg *trace_msg = data; ++ char buffer[KTRACE_DUMP_MESSAGE_SIZE]; ++ ++ /* If this is the start, print a header */ ++ if (data == SEQ_START_TOKEN) ++ kbasep_ktrace_format_header(buffer, sizeof(buffer), 0); ++ else ++ kbasep_ktrace_format_msg(trace_msg, buffer, sizeof(buffer)); ++ ++ seq_printf(s, "%s\n", buffer); ++ return 0; ++} ++ ++static const struct seq_operations kbasep_ktrace_seq_ops = { ++ .start = kbasep_ktrace_seq_start, ++ .next = kbasep_ktrace_seq_next, ++ .stop = kbasep_ktrace_seq_stop, ++ .show = kbasep_ktrace_seq_show, ++}; ++ ++static int kbasep_ktrace_debugfs_open(struct inode *inode, struct file *file) ++{ ++ struct kbase_device *kbdev = inode->i_private; ++ unsigned long flags; ++ ++ struct trace_seq_state *state; ++ ++ state = __seq_open_private(file, &kbasep_ktrace_seq_ops, ++ sizeof(*state)); ++ if (!state) ++ return -ENOMEM; ++ ++ spin_lock_irqsave(&kbdev->ktrace.lock, flags); ++ state->start = kbdev->ktrace.first_out; ++ state->end = kbdev->ktrace.next_in; ++ memcpy(state->trace_buf, kbdev->ktrace.rbuf, sizeof(state->trace_buf)); ++ spin_unlock_irqrestore(&kbdev->ktrace.lock, flags); ++ ++ return 0; ++} ++ ++static const struct file_operations kbasep_ktrace_debugfs_fops = { ++ .owner = THIS_MODULE, ++ .open = kbasep_ktrace_debugfs_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = seq_release_private, ++}; ++ ++void kbase_ktrace_debugfs_init(struct kbase_device *kbdev) ++{ ++ debugfs_create_file("mali_trace", 0444, ++ kbdev->mali_debugfs_directory, kbdev, ++ &kbasep_ktrace_debugfs_fops); ++} ++#endif /* CONFIG_DEBUG_FS */ ++ ++#else /* KBASE_KTRACE_TARGET_RBUF */ ++ ++#ifdef CONFIG_DEBUG_FS ++void kbase_ktrace_debugfs_init(struct kbase_device *kbdev) ++{ ++ CSTD_UNUSED(kbdev); ++} ++#endif /* CONFIG_DEBUG_FS */ ++#endif /* KBASE_KTRACE_TARGET_RBUF */ +diff --git a/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace.h b/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace.h +new file mode 100755 +index 000000000000..e4e2e8c35001 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace.h +@@ -0,0 +1,226 @@ ++/* ++ * ++ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* ++ * DOC: Kbase's own trace, 'KTrace' ++ * ++ * Low overhead trace specific to kbase, aimed at: ++ * - common use-cases for tracing kbase specific functionality to do with ++ * running work on the GPU ++ * - easy 1-line addition of new types of trace ++ * ++ * KTrace can be recorded in one or more of the following targets: ++ * - KBASE_KTRACE_TARGET_RBUF: low overhead ringbuffer protected by an ++ * irq-spinlock, output available via dev_dbg() and debugfs file ++ * - KBASE_KTRACE_TARGET_FTRACE: ftrace based tracepoints under 'mali' events ++ */ ++ ++#ifndef _KBASE_DEBUG_KTRACE_H_ ++#define _KBASE_DEBUG_KTRACE_H_ ++ ++#if KBASE_KTRACE_TARGET_FTRACE ++#include "mali_linux_trace.h" ++#endif ++ ++#if MALI_USE_CSF ++#include "debug/backend/mali_kbase_debug_ktrace_csf.h" ++#else ++#include "debug/backend/mali_kbase_debug_ktrace_jm.h" ++#endif ++ ++/** ++ * kbase_ktrace_init - initialize kbase ktrace. ++ * @kbdev: kbase device ++ */ ++int kbase_ktrace_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_ktrace_term - terminate kbase ktrace. ++ * @kbdev: kbase device ++ */ ++void kbase_ktrace_term(struct kbase_device *kbdev); ++ ++/** ++ * kbase_ktrace_hook_wrapper - wrapper so that dumping ktrace can be done via a ++ * callback. ++ * @param: kbase device, cast to void pointer ++ */ ++void kbase_ktrace_hook_wrapper(void *param); ++ ++#ifdef CONFIG_DEBUG_FS ++/** ++ * kbase_ktrace_debugfs_init - initialize kbase ktrace for debugfs usage, if ++ * the selected targets support it. ++ * @kbdev: kbase device ++ * ++ * There is no matching 'term' call, debugfs_remove_recursive() is sufficient. ++ */ ++void kbase_ktrace_debugfs_init(struct kbase_device *kbdev); ++#endif /* CONFIG_DEBUG_FS */ ++ ++/* ++ * KTrace target for internal ringbuffer ++ */ ++#if KBASE_KTRACE_TARGET_RBUF ++/** ++ * kbasep_ktrace_add - internal function to add trace to the ringbuffer. ++ * @kbdev: kbase device ++ * @code: ktrace code ++ * @kctx: kbase context, or NULL if no context ++ * @flags: flags about the message ++ * @info_val: generic information about @code to add to the trace ++ * ++ * PRIVATE: do not use directly. Use KBASE_KTRACE_ADD() instead. ++ */ ++void kbasep_ktrace_add(struct kbase_device *kbdev, enum kbase_ktrace_code code, ++ struct kbase_context *kctx, kbase_ktrace_flag_t flags, ++ u64 info_val); ++ ++/** ++ * kbasep_ktrace_clear - clear the trace ringbuffer ++ * @kbdev: kbase device ++ * ++ * PRIVATE: do not use directly. Use KBASE_KTRACE_CLEAR() instead. ++ */ ++void kbasep_ktrace_clear(struct kbase_device *kbdev); ++ ++/** ++ * kbasep_ktrace_dump - dump ktrace ringbuffer to dev_dbg(), then clear it ++ * @kbdev: kbase device ++ * ++ * PRIVATE: do not use directly. Use KBASE_KTRACE_DUMP() instead. ++ */ ++void kbasep_ktrace_dump(struct kbase_device *kbdev); ++ ++#define KBASE_KTRACE_RBUF_ADD(kbdev, code, kctx, info_val) \ ++ kbasep_ktrace_add(kbdev, KBASE_KTRACE_CODE(code), kctx, 0, \ ++ info_val) \ ++ ++#define KBASE_KTRACE_RBUF_CLEAR(kbdev) \ ++ kbasep_ktrace_clear(kbdev) ++ ++#define KBASE_KTRACE_RBUF_DUMP(kbdev) \ ++ kbasep_ktrace_dump(kbdev) ++ ++#else /* KBASE_KTRACE_TARGET_RBUF */ ++ ++#define KBASE_KTRACE_RBUF_ADD(kbdev, code, kctx, info_val) \ ++ do { \ ++ CSTD_UNUSED(kbdev); \ ++ CSTD_NOP(code); \ ++ CSTD_UNUSED(kctx); \ ++ CSTD_UNUSED(info_val); \ ++ CSTD_NOP(0); \ ++ } while (0) ++ ++#define KBASE_KTRACE_RBUF_CLEAR(kbdev) \ ++ do { \ ++ CSTD_UNUSED(kbdev); \ ++ CSTD_NOP(0); \ ++ } while (0) ++#define KBASE_KTRACE_RBUF_DUMP(kbdev) \ ++ do { \ ++ CSTD_UNUSED(kbdev); \ ++ CSTD_NOP(0); \ ++ } while (0) ++#endif /* KBASE_KTRACE_TARGET_RBUF */ ++ ++/* ++ * KTrace target for Linux's ftrace ++ */ ++#if KBASE_KTRACE_TARGET_FTRACE ++ ++#define KBASE_KTRACE_FTRACE_ADD(kbdev, code, kctx, info_val) \ ++ trace_mali_##code(kctx, info_val) ++ ++#else /* KBASE_KTRACE_TARGET_FTRACE */ ++#define KBASE_KTRACE_FTRACE_ADD(kbdev, code, kctx, info_val) \ ++ do { \ ++ CSTD_UNUSED(kbdev); \ ++ CSTD_NOP(code); \ ++ CSTD_UNUSED(kctx); \ ++ CSTD_UNUSED(info_val); \ ++ CSTD_NOP(0); \ ++ } while (0) ++#endif /* KBASE_KTRACE_TARGET_FTRACE */ ++ ++/* No 'clear' implementation for ftrace yet */ ++#define KBASE_KTRACE_FTRACE_CLEAR(kbdev) \ ++ do { \ ++ CSTD_UNUSED(kbdev); \ ++ CSTD_NOP(0); \ ++ } while (0) ++ ++/* No 'dump' implementation for ftrace yet */ ++#define KBASE_KTRACE_FTRACE_DUMP(kbdev) \ ++ do { \ ++ CSTD_UNUSED(kbdev); \ ++ CSTD_NOP(0); \ ++ } while (0) ++ ++/* ++ * Master set of macros to route KTrace to any of the targets ++ */ ++ ++/** ++ * KBASE_KTRACE_ADD - Add trace values ++ * @kbdev: kbase device ++ * @code: trace code ++ * @kctx: kbase context, or NULL if no context ++ * @info_val: generic information about @code to add to the trace ++ * ++ * Note: Any functions called through this macro will still be evaluated in ++ * Release builds (CONFIG_MALI_BIFROST_DEBUG not defined). Therefore, when ++ * KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied ++ * to this macro must: ++ * a) be static or static inline, and ++ * b) just return 0 and have no other statements present in the body. ++ */ ++#define KBASE_KTRACE_ADD(kbdev, code, kctx, info_val) \ ++ do { \ ++ /* capture values that could come from non-pure function calls */ \ ++ u64 __info_val = info_val; \ ++ KBASE_KTRACE_RBUF_ADD(kbdev, code, kctx, __info_val); \ ++ KBASE_KTRACE_FTRACE_ADD(kbdev, code, kctx, __info_val); \ ++ } while (0) ++ ++/** ++ * KBASE_KTRACE_CLEAR - Clear the trace, if applicable to the target(s) ++ * @kbdev: kbase device ++ */ ++#define KBASE_KTRACE_CLEAR(kbdev) \ ++ do { \ ++ KBASE_KTRACE_RBUF_CLEAR(kbdev); \ ++ KBASE_KTRACE_FTRACE_CLEAR(kbdev); \ ++ } while (0) ++ ++/** ++ * KBASE_KTRACE_DUMP - Dump the trace, if applicable to the target(s) ++ * @kbdev: kbase device ++ */ ++#define KBASE_KTRACE_DUMP(kbdev) \ ++ do { \ ++ KBASE_KTRACE_RBUF_DUMP(kbdev); \ ++ KBASE_KTRACE_FTRACE_DUMP(kbdev); \ ++ } while (0) ++ ++#endif /* _KBASE_DEBUG_KTRACE_H_ */ +diff --git a/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace_codes.h b/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace_codes.h +new file mode 100755 +index 000000000000..b50bceee4244 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace_codes.h +@@ -0,0 +1,165 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2015,2018-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* ++ * ***** IMPORTANT: THIS IS NOT A NORMAL HEADER FILE ***** ++ * ***** DO NOT INCLUDE DIRECTLY ***** ++ * ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** ++ */ ++ ++/* ++ * The purpose of this header file is just to contain a list of trace code ++ * identifiers ++ * ++ * When updating this file, also remember to update ++ * mali_kbase_debug_linux_ktrace.h ++ * ++ * Each identifier is wrapped in a macro, so that its string form and enum form ++ * can be created ++ * ++ * Each macro is separated with a comma, to allow insertion into an array ++ * initializer or enum definition block. ++ * ++ * This allows automatic creation of an enum and a corresponding array of ++ * strings ++ * ++ * Before #including, the includer MUST #define KBASE_KTRACE_CODE_MAKE_CODE. ++ * After #including, the includer MUST #under KBASE_KTRACE_CODE_MAKE_CODE. ++ * ++ * e.g.: ++ * #define KBASE_KTRACE_CODE( X ) KBASE_KTRACE_CODE_ ## X ++ * typedef enum ++ * { ++ * #define KBASE_KTRACE_CODE_MAKE_CODE( X ) KBASE_KTRACE_CODE( X ) ++ * #include "mali_kbase_debug_ktrace_codes.h" ++ * #undef KBASE_KTRACE_CODE_MAKE_CODE ++ * } kbase_ktrace_code; ++ * ++ * IMPORTANT: THIS FILE MUST NOT BE USED FOR ANY OTHER PURPOSE OTHER THAN THE ABOVE ++ * ++ * ++ * The use of the macro here is: ++ * - KBASE_KTRACE_CODE_MAKE_CODE( X ) ++ * ++ * Which produces: ++ * - For an enum, KBASE_KTRACE_CODE_X ++ * - For a string, "X" ++ * ++ * ++ * For example: ++ * - KBASE_KTRACE_CODE_MAKE_CODE( JM_JOB_COMPLETE ) expands to: ++ * - KBASE_KTRACE_CODE_JM_JOB_COMPLETE for the enum ++ * - "JM_JOB_COMPLETE" for the string ++ * - To use it to trace an event, do: ++ * - KBASE_KTRACE_ADD( kbdev, JM_JOB_COMPLETE, subcode, kctx, uatom, val ); ++ */ ++ ++#if 0 /* Dummy section to avoid breaking formatting */ ++int dummy_array[] = { ++#endif ++ ++ /* ++ * Core events ++ */ ++ /* no info_val */ ++ KBASE_KTRACE_CODE_MAKE_CODE(CORE_CTX_DESTROY), ++ /* no info_val */ ++ KBASE_KTRACE_CODE_MAKE_CODE(CORE_CTX_HWINSTR_TERM), ++ /* info_val == GPU_IRQ_STATUS register */ ++ KBASE_KTRACE_CODE_MAKE_CODE(CORE_GPU_IRQ), ++ /* info_val == bits cleared */ ++ KBASE_KTRACE_CODE_MAKE_CODE(CORE_GPU_IRQ_CLEAR), ++ /* info_val == GPU_IRQ_STATUS register */ ++ KBASE_KTRACE_CODE_MAKE_CODE(CORE_GPU_IRQ_DONE), ++ KBASE_KTRACE_CODE_MAKE_CODE(CORE_GPU_SOFT_RESET), ++ KBASE_KTRACE_CODE_MAKE_CODE(CORE_GPU_HARD_RESET), ++ KBASE_KTRACE_CODE_MAKE_CODE(CORE_GPU_PRFCNT_CLEAR), ++ /* info_val == dump address */ ++ KBASE_KTRACE_CODE_MAKE_CODE(CORE_GPU_PRFCNT_SAMPLE), ++ KBASE_KTRACE_CODE_MAKE_CODE(CORE_GPU_CLEAN_INV_CACHES), ++ ++ /* ++ * Power Management Events ++ */ ++ KBASE_KTRACE_CODE_MAKE_CODE(PM_JOB_SUBMIT_AFTER_POWERING_UP), ++ KBASE_KTRACE_CODE_MAKE_CODE(PM_JOB_SUBMIT_AFTER_POWERED_UP), ++ KBASE_KTRACE_CODE_MAKE_CODE(PM_PWRON), ++ KBASE_KTRACE_CODE_MAKE_CODE(PM_PWRON_TILER), ++ KBASE_KTRACE_CODE_MAKE_CODE(PM_PWRON_L2), ++ KBASE_KTRACE_CODE_MAKE_CODE(PM_PWROFF), ++ KBASE_KTRACE_CODE_MAKE_CODE(PM_PWROFF_TILER), ++ KBASE_KTRACE_CODE_MAKE_CODE(PM_PWROFF_L2), ++ KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_POWERED), ++ KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_POWERED_TILER), ++ KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_POWERED_L2), ++ KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_DESIRED), ++ KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_DESIRED_TILER), ++ KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_AVAILABLE), ++ KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_AVAILABLE_TILER), ++ KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_AVAILABLE), ++ KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_AVAILABLE_TILER), ++ KBASE_KTRACE_CODE_MAKE_CODE(PM_DESIRED_REACHED), ++ KBASE_KTRACE_CODE_MAKE_CODE(PM_DESIRED_REACHED_TILER), ++ KBASE_KTRACE_CODE_MAKE_CODE(PM_RELEASE_CHANGE_SHADER_NEEDED), ++ KBASE_KTRACE_CODE_MAKE_CODE(PM_RELEASE_CHANGE_TILER_NEEDED), ++ KBASE_KTRACE_CODE_MAKE_CODE(PM_REQUEST_CHANGE_SHADER_NEEDED), ++ KBASE_KTRACE_CODE_MAKE_CODE(PM_REQUEST_CHANGE_TILER_NEEDED), ++ KBASE_KTRACE_CODE_MAKE_CODE(PM_WAKE_WAITERS), ++ /* info_val == kbdev->pm.active_count*/ ++ KBASE_KTRACE_CODE_MAKE_CODE(PM_CONTEXT_ACTIVE), ++ /* info_val == kbdev->pm.active_count*/ ++ KBASE_KTRACE_CODE_MAKE_CODE(PM_CONTEXT_IDLE), ++ KBASE_KTRACE_CODE_MAKE_CODE(PM_GPU_ON), ++ KBASE_KTRACE_CODE_MAKE_CODE(PM_GPU_OFF), ++ /* info_val == policy number, or -1 for "Already changing" */ ++ KBASE_KTRACE_CODE_MAKE_CODE(PM_SET_POLICY), ++ KBASE_KTRACE_CODE_MAKE_CODE(PM_CA_SET_POLICY), ++ /* info_val == policy number */ ++ KBASE_KTRACE_CODE_MAKE_CODE(PM_CURRENT_POLICY_INIT), ++ /* info_val == policy number */ ++ KBASE_KTRACE_CODE_MAKE_CODE(PM_CURRENT_POLICY_TERM), ++ ++ /* ++ * Context Scheduler events ++ */ ++ /* info_val == kctx->refcount */ ++ KBASE_KTRACE_CODE_MAKE_CODE(SCHED_RETAIN_CTX_NOLOCK), ++ /* info_val == kctx->refcount */ ++ KBASE_KTRACE_CODE_MAKE_CODE(SCHED_RELEASE_CTX), ++ ++ ++#if MALI_USE_CSF ++#include "debug/backend/mali_kbase_debug_ktrace_codes_csf.h" ++#else ++#include "debug/backend/mali_kbase_debug_ktrace_codes_jm.h" ++#endif ++ /* ++ * Unused code just to make it easier to not have a comma at the end. ++ * All other codes MUST come before this ++ */ ++ KBASE_KTRACE_CODE_MAKE_CODE(DUMMY) ++ ++#if 0 /* Dummy section to avoid breaking formatting */ ++}; ++#endif ++ ++/* ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** */ +diff --git a/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace_defs.h b/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace_defs.h +new file mode 100755 +index 000000000000..c680feb86387 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace_defs.h +@@ -0,0 +1,183 @@ ++/* ++ * ++ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_DEBUG_KTRACE_DEFS_H_ ++#define _KBASE_DEBUG_KTRACE_DEFS_H_ ++ ++/* Enable SW tracing when set */ ++#if defined(CONFIG_MALI_BIFROST_ENABLE_TRACE) || defined(CONFIG_MALI_BIFROST_SYSTEM_TRACE) ++#define KBASE_KTRACE_ENABLE 1 ++#endif ++ ++#ifndef KBASE_KTRACE_ENABLE ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++#define KBASE_KTRACE_ENABLE 1 ++#else /* CONFIG_MALI_BIFROST_DEBUG */ ++#define KBASE_KTRACE_ENABLE 0 ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++#endif /* KBASE_KTRACE_ENABLE */ ++ ++/* Select targets for recording of trace: ++ * ++ */ ++#if KBASE_KTRACE_ENABLE ++ ++#ifdef CONFIG_MALI_BIFROST_SYSTEM_TRACE ++#define KBASE_KTRACE_TARGET_FTRACE 1 ++#else /* CONFIG_MALI_BIFROST_SYSTEM_TRACE */ ++#define KBASE_KTRACE_TARGET_FTRACE 0 ++#endif /* CONFIG_MALI_BIFROST_SYSTEM_TRACE */ ++ ++#ifdef CONFIG_MALI_BIFROST_ENABLE_TRACE ++#define KBASE_KTRACE_TARGET_RBUF 1 ++#else /* CONFIG_MALI_BIFROST_ENABLE_TRACE*/ ++#define KBASE_KTRACE_TARGET_RBUF 0 ++#endif /* CONFIG_MALI_BIFROST_ENABLE_TRACE */ ++ ++#else /* KBASE_KTRACE_ENABLE */ ++#define KBASE_KTRACE_TARGET_FTRACE 0 ++#define KBASE_KTRACE_TARGET_RBUF 0 ++#endif /* KBASE_KTRACE_ENABLE */ ++ ++/* ++ * Note: Some backends define flags in this type even if the RBUF target is ++ * disabled (they get discarded with CSTD_UNUSED(), but they're still ++ * referenced) ++ */ ++typedef u8 kbase_ktrace_flag_t; ++ ++#if KBASE_KTRACE_TARGET_RBUF ++typedef u8 kbase_ktrace_code_t; ++ ++/* ++ * NOTE: KBASE_KTRACE_VERSION_MAJOR, KBASE_KTRACE_VERSION_MINOR are kept in ++ * the backend, since updates can be made to one backend in a way that doesn't ++ * affect the other. ++ * ++ * However, modifying the common part could require both backend versions to be ++ * updated. ++ */ ++ ++/* ++ * struct kbase_ktrace_backend - backend specific part of a trace message ++ * ++ * At the very least, this must contain a kbase_ktrace_code_t 'code' member and ++ * a kbase_ktrace_flag_t 'flags' member ++ */ ++struct kbase_ktrace_backend; ++#endif /* KBASE_KTRACE_TARGET_RBUF */ ++ ++#if MALI_USE_CSF ++#include "debug/backend/mali_kbase_debug_ktrace_defs_csf.h" ++#else ++#include "debug/backend/mali_kbase_debug_ktrace_defs_jm.h" ++#endif ++ ++#if KBASE_KTRACE_TARGET_RBUF ++/* Indicates if the trace message has backend related info. ++ * ++ * If not set, consider the &kbase_ktrace_backend part of a &kbase_ktrace_msg ++ * as uninitialized, apart from the mandatory parts: ++ * - code ++ * - flags ++ */ ++#define KBASE_KTRACE_FLAG_BACKEND (((kbase_ktrace_flag_t)1) << 7) ++ ++/* Collect all the common flags together for debug checking */ ++#define KBASE_KTRACE_FLAG_COMMON_ALL \ ++ (KBASE_KTRACE_FLAG_BACKEND) ++ ++#define KBASE_KTRACE_FLAG_ALL \ ++ (KBASE_KTRACE_FLAG_COMMON_ALL | KBASE_KTRACE_FLAG_BACKEND_ALL) ++ ++#define KBASE_KTRACE_SHIFT 8 /* 256 entries */ ++#define KBASE_KTRACE_SIZE (1 << KBASE_KTRACE_SHIFT) ++#define KBASE_KTRACE_MASK ((1 << KBASE_KTRACE_SHIFT)-1) ++ ++#define KBASE_KTRACE_CODE(X) KBASE_KTRACE_CODE_ ## X ++ ++/* Note: compiletime_assert() about this against kbase_ktrace_code_t is in ++ * kbase_ktrace_init() ++ */ ++enum kbase_ktrace_code { ++ /* ++ * IMPORTANT: USE OF SPECIAL #INCLUDE OF NON-STANDARD HEADER FILE ++ * THIS MUST BE USED AT THE START OF THE ENUM ++ */ ++#define KBASE_KTRACE_CODE_MAKE_CODE(X) KBASE_KTRACE_CODE(X) ++#include ++#undef KBASE_KTRACE_CODE_MAKE_CODE ++ /* Comma on its own, to extend the list */ ++ , ++ /* Must be the last in the enum */ ++ KBASE_KTRACE_CODE_COUNT ++}; ++ ++/** ++ * struct kbase_ktrace - object representing a trace message added to trace ++ * buffer trace_rbuf in &kbase_device ++ * @timestamp: CPU timestamp at which the trace message was added. ++ * @thread_id: id of the thread in the context of which trace message was ++ * added. ++ * @cpu: indicates which CPU the @thread_id was scheduled on when the ++ * trace message was added. ++ * @kctx_tgid: Thread group ID of the &kbase_context associated with the ++ * message, or 0 if none associated. ++ * @kctx_id: Unique identifier of the &kbase_context associated with the ++ * message. Only valid if @kctx_tgid != 0. ++ * @info_val: value specific to the type of event being traced. Refer to the ++ * specific code in enum kbase_ktrace_code ++ * @backend: backend-specific trace information. All backends must implement ++ * a minimum common set of members ++ */ ++struct kbase_ktrace_msg { ++ struct timespec64 timestamp; ++ u32 thread_id; ++ u32 cpu; ++ pid_t kctx_tgid; ++ u32 kctx_id; ++ u64 info_val; ++ ++ struct kbase_ktrace_backend backend; ++}; ++ ++struct kbase_ktrace { ++ spinlock_t lock; ++ u16 first_out; ++ u16 next_in; ++ struct kbase_ktrace_msg *rbuf; ++}; ++ ++ ++static inline void kbase_ktrace_compiletime_asserts(void) ++{ ++ /* See also documentation of enum kbase_ktrace_code */ ++ compiletime_assert(sizeof(kbase_ktrace_code_t) == sizeof(unsigned long long) || ++ KBASE_KTRACE_CODE_COUNT <= (1ull << (sizeof(kbase_ktrace_code_t) * BITS_PER_BYTE)), ++ "kbase_ktrace_code_t not wide enough for KBASE_KTRACE_CODE_COUNT"); ++ compiletime_assert((KBASE_KTRACE_FLAG_BACKEND_ALL & KBASE_KTRACE_FLAG_COMMON_ALL) == 0, ++ "KTrace backend flags intersect with KTrace common flags"); ++ ++} ++ ++#endif /* KBASE_KTRACE_TARGET_RBUF */ ++#endif /* _KBASE_DEBUG_KTRACE_DEFS_H_ */ +diff --git a/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace_internal.h b/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace_internal.h +new file mode 100755 +index 000000000000..e450760e3426 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace_internal.h +@@ -0,0 +1,89 @@ ++/* ++ * ++ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_DEBUG_KTRACE_INTERNAL_H_ ++#define _KBASE_DEBUG_KTRACE_INTERNAL_H_ ++ ++#if KBASE_KTRACE_TARGET_RBUF ++ ++#define KTRACE_DUMP_MESSAGE_SIZE 256 ++ ++/** ++ * kbasep_ktrace_backend_format_header - format the backend part of the header ++ * @buffer: buffer to write to ++ * @sz: size of @buffer in bytes ++ * @written: pointer to storage for updating bytes written so far to @buffer ++ * ++ * The backend must format only the non-common backend specific parts of the ++ * header. It must format them as though they were standalone. The caller will ++ * handle adding any delimiters around this. ++ */ ++void kbasep_ktrace_backend_format_header(char *buffer, int sz, s32 *written); ++ ++/** ++ * kbasep_ktrace_backend_format_msg - format the backend part of the message ++ * @trace_msg: ktrace message ++ * @buffer: buffer to write to ++ * @sz: size of @buffer in bytes ++ * @written: pointer to storage for updating bytes written so far to @buffer ++ * ++ * The backend must format only the non-common backend specific parts of the ++ * message. It must format them as though they were standalone. The caller will ++ * handle adding any delimiters around this. ++ * ++ * A caller may have the flags member of @trace_msg with ++ * %KBASE_KTRACE_FLAG_BACKEND clear. The backend must handle that setting ++ * appropriately. ++ */ ++void kbasep_ktrace_backend_format_msg(struct kbase_ktrace_msg *trace_msg, ++ char *buffer, int sz, s32 *written); ++ ++ ++/** ++ * kbasep_ktrace_reserve - internal function to reserve space for a ktrace ++ * message ++ * @ktrace: kbase device's ktrace ++ * ++ * This may also empty the oldest entry in the ringbuffer to make space. ++ */ ++struct kbase_ktrace_msg *kbasep_ktrace_reserve(struct kbase_ktrace *ktrace); ++ ++/** ++ * kbasep_ktrace_msg_init - internal function to initialize just the common ++ * part of a ktrace message ++ * @ktrace: kbase device's ktrace ++ * @trace_msg: ktrace message to initialize ++ * @code: ktrace code ++ * @kctx: kbase context, or NULL if no context ++ * @flags: flags about the message ++ * @info_val: generic information about @code to add to the trace ++ * ++ * The common part includes the mandatory parts of the backend part ++ */ ++void kbasep_ktrace_msg_init(struct kbase_ktrace *ktrace, ++ struct kbase_ktrace_msg *trace_msg, enum kbase_ktrace_code code, ++ struct kbase_context *kctx, kbase_ktrace_flag_t flags, ++ u64 info_val); ++ ++#endif /* KBASE_KTRACE_TARGET_RBUF */ ++ ++#endif /* _KBASE_DEBUG_KTRACE_INTERNAL_H_ */ +diff --git a/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_linux_ktrace.h b/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_linux_ktrace.h +new file mode 100755 +index 000000000000..27f687faf072 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_linux_ktrace.h +@@ -0,0 +1,111 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014,2018,2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* ++ * NOTE: This must **only** be included through mali_linux_trace.h, ++ * otherwise it will fail to setup tracepoints correctly ++ */ ++ ++#if !defined(_KBASE_DEBUG_LINUX_KTRACE_H_) || defined(TRACE_HEADER_MULTI_READ) ++#define _KBASE_DEBUG_LINUX_KTRACE_H_ ++ ++#if KBASE_KTRACE_TARGET_FTRACE ++ ++DECLARE_EVENT_CLASS(mali_add_template, ++ TP_PROTO(struct kbase_context *kctx, u64 info_val), ++ TP_ARGS(kctx, info_val), ++ TP_STRUCT__entry( ++ __field(pid_t, kctx_tgid) ++ __field(u32, kctx_id) ++ __field(u64, info_val) ++ ), ++ TP_fast_assign( ++ __entry->kctx_id = (kctx) ? kctx->id : 0u; ++ __entry->kctx_tgid = (kctx) ? kctx->tgid : 0; ++ __entry->info_val = info_val; ++ ), ++ TP_printk("kctx=%d_%u info=0x%llx", __entry->kctx_tgid, ++ __entry->kctx_id, __entry->info_val) ++); ++ ++/* DEFINE_MALI_ADD_EVENT is available also to backends for backend-specific ++ * simple trace codes ++ */ ++#define DEFINE_MALI_ADD_EVENT(name) \ ++DEFINE_EVENT(mali_add_template, mali_##name, \ ++ TP_PROTO(struct kbase_context *kctx, u64 info_val), \ ++ TP_ARGS(kctx, info_val)) ++DEFINE_MALI_ADD_EVENT(CORE_CTX_DESTROY); ++DEFINE_MALI_ADD_EVENT(CORE_CTX_HWINSTR_TERM); ++DEFINE_MALI_ADD_EVENT(CORE_GPU_IRQ); ++DEFINE_MALI_ADD_EVENT(CORE_GPU_IRQ_CLEAR); ++DEFINE_MALI_ADD_EVENT(CORE_GPU_IRQ_DONE); ++DEFINE_MALI_ADD_EVENT(CORE_GPU_SOFT_RESET); ++DEFINE_MALI_ADD_EVENT(CORE_GPU_HARD_RESET); ++DEFINE_MALI_ADD_EVENT(CORE_GPU_PRFCNT_SAMPLE); ++DEFINE_MALI_ADD_EVENT(CORE_GPU_PRFCNT_CLEAR); ++DEFINE_MALI_ADD_EVENT(CORE_GPU_CLEAN_INV_CACHES); ++DEFINE_MALI_ADD_EVENT(PM_CORES_CHANGE_DESIRED); ++DEFINE_MALI_ADD_EVENT(PM_JOB_SUBMIT_AFTER_POWERING_UP); ++DEFINE_MALI_ADD_EVENT(PM_JOB_SUBMIT_AFTER_POWERED_UP); ++DEFINE_MALI_ADD_EVENT(PM_PWRON); ++DEFINE_MALI_ADD_EVENT(PM_PWRON_TILER); ++DEFINE_MALI_ADD_EVENT(PM_PWRON_L2); ++DEFINE_MALI_ADD_EVENT(PM_PWROFF); ++DEFINE_MALI_ADD_EVENT(PM_PWROFF_TILER); ++DEFINE_MALI_ADD_EVENT(PM_PWROFF_L2); ++DEFINE_MALI_ADD_EVENT(PM_CORES_POWERED); ++DEFINE_MALI_ADD_EVENT(PM_CORES_POWERED_TILER); ++DEFINE_MALI_ADD_EVENT(PM_CORES_POWERED_L2); ++DEFINE_MALI_ADD_EVENT(PM_DESIRED_REACHED); ++DEFINE_MALI_ADD_EVENT(PM_DESIRED_REACHED_TILER); ++DEFINE_MALI_ADD_EVENT(PM_REQUEST_CHANGE_SHADER_NEEDED); ++DEFINE_MALI_ADD_EVENT(PM_REQUEST_CHANGE_TILER_NEEDED); ++DEFINE_MALI_ADD_EVENT(PM_RELEASE_CHANGE_SHADER_NEEDED); ++DEFINE_MALI_ADD_EVENT(PM_RELEASE_CHANGE_TILER_NEEDED); ++DEFINE_MALI_ADD_EVENT(PM_CORES_AVAILABLE); ++DEFINE_MALI_ADD_EVENT(PM_CORES_AVAILABLE_TILER); ++DEFINE_MALI_ADD_EVENT(PM_CORES_CHANGE_AVAILABLE); ++DEFINE_MALI_ADD_EVENT(PM_CORES_CHANGE_AVAILABLE_TILER); ++DEFINE_MALI_ADD_EVENT(PM_GPU_ON); ++DEFINE_MALI_ADD_EVENT(PM_GPU_OFF); ++DEFINE_MALI_ADD_EVENT(PM_SET_POLICY); ++DEFINE_MALI_ADD_EVENT(PM_CURRENT_POLICY_INIT); ++DEFINE_MALI_ADD_EVENT(PM_CURRENT_POLICY_TERM); ++DEFINE_MALI_ADD_EVENT(PM_CA_SET_POLICY); ++DEFINE_MALI_ADD_EVENT(PM_CONTEXT_ACTIVE); ++DEFINE_MALI_ADD_EVENT(PM_CONTEXT_IDLE); ++DEFINE_MALI_ADD_EVENT(PM_WAKE_WAITERS); ++DEFINE_MALI_ADD_EVENT(SCHED_RETAIN_CTX_NOLOCK); ++DEFINE_MALI_ADD_EVENT(SCHED_RELEASE_CTX); ++ ++#if MALI_USE_CSF ++#include "mali_kbase_debug_linux_ktrace_csf.h" ++#else ++#include "mali_kbase_debug_linux_ktrace_jm.h" ++#endif ++ ++#undef DEFINE_MALI_ADD_EVENT ++ ++#endif /* KBASE_KTRACE_TARGET_FTRACE */ ++ ++#endif /* !defined(_KBASE_DEBUG_LINUX_KTRACE_H_) || defined(TRACE_HEADER_MULTI_READ) */ +diff --git a/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_csf.c b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_csf.c +new file mode 100755 +index 000000000000..d8b3fff6a214 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_csf.c +@@ -0,0 +1,274 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * ++ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include "../mali_kbase_device_internal.h" ++#include "../mali_kbase_device.h" ++ ++#include ++#include ++#include ++#include ++#include ++ ++#ifdef CONFIG_MALI_BIFROST_NO_MALI ++#include ++#endif ++ ++#include ++#include ++#include ++#include ++#include ++ ++static void kbase_device_csf_firmware_term(struct kbase_device *kbdev) ++{ ++ kbase_clk_rate_trace_manager_term(kbdev); ++ kbase_csf_firmware_term(kbdev); ++} ++ ++static int kbase_device_csf_firmware_init(struct kbase_device *kbdev) ++{ ++ int err = kbase_csf_firmware_init(kbdev); ++ ++ if (!err) { ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbdev->pm.backend.mcu_state = KBASE_MCU_ON; ++ kbdev->csf.firmware_inited = true; ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ } ++ ++ /* Post firmware init, idle condition is restored. Note this is ++ * a deferral action step from the late init stage for CSF. ++ */ ++ kbase_pm_context_idle(kbdev); ++ ++ if (!err) ++ kbase_clk_rate_trace_manager_init(kbdev); ++ ++ return err; ++} ++ ++/** ++ * kbase_backend_late_init - Perform any backend-specific initialization. ++ * @kbdev: Device pointer ++ * ++ * Return: 0 on success, or an error code on failure. ++ */ ++static int kbase_backend_late_init(struct kbase_device *kbdev) ++{ ++ int err; ++ ++ err = kbase_hwaccess_pm_init(kbdev); ++ if (err) ++ return err; ++ ++ err = kbase_reset_gpu_init(kbdev); ++ if (err) ++ goto fail_reset_gpu_init; ++ ++ err = kbase_hwaccess_pm_powerup(kbdev, PM_HW_ISSUES_DETECT); ++ if (err) ++ goto fail_pm_powerup; ++ ++ err = kbase_backend_timer_init(kbdev); ++ if (err) ++ goto fail_timer; ++ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++#ifndef CONFIG_MALI_BIFROST_NO_MALI ++ if (kbasep_common_test_interrupt_handlers(kbdev) != 0) { ++ dev_err(kbdev->dev, "Interrupt assignment check failed.\n"); ++ err = -EINVAL; ++ goto fail_interrupt_test; ++ } ++#endif /* !CONFIG_MALI_BIFROST_NO_MALI */ ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++ ++ /* Do the initialisation of devfreq. ++ * Devfreq needs backend_timer_init() for completion of its ++ * initialisation and it also needs to catch the first callback ++ * occurrence of the runtime_suspend event for maintaining state ++ * coherence with the backend power management, hence needs to be ++ * placed before the kbase_pm_context_idle(). ++ */ ++ err = kbase_backend_devfreq_init(kbdev); ++ if (err) ++ goto fail_devfreq_init; ++ ++ /* Update gpuprops with L2_FEATURES if applicable */ ++ err = kbase_gpuprops_update_l2_features(kbdev); ++ if (err) ++ goto fail_update_l2_features; ++ ++ init_waitqueue_head(&kbdev->hwaccess.backend.reset_wait); ++ ++ return 0; ++ ++fail_update_l2_features: ++fail_devfreq_init: ++ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++#ifndef CONFIG_MALI_BIFROST_NO_MALI ++fail_interrupt_test: ++#endif /* !CONFIG_MALI_BIFROST_NO_MALI */ ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++ ++ kbase_backend_timer_term(kbdev); ++fail_timer: ++ kbase_hwaccess_pm_halt(kbdev); ++fail_pm_powerup: ++ kbase_reset_gpu_term(kbdev); ++fail_reset_gpu_init: ++ kbase_hwaccess_pm_term(kbdev); ++ ++ return err; ++} ++ ++/** ++ * kbase_backend_late_term - Perform any backend-specific termination. ++ * @kbdev: Device pointer ++ */ ++static void kbase_backend_late_term(struct kbase_device *kbdev) ++{ ++ kbase_backend_devfreq_term(kbdev); ++ kbase_hwaccess_pm_halt(kbdev); ++ kbase_reset_gpu_term(kbdev); ++ kbase_hwaccess_pm_term(kbdev); ++} ++ ++static const struct kbase_device_init dev_init[] = { ++#ifdef CONFIG_MALI_BIFROST_NO_MALI ++ {kbase_gpu_device_create, kbase_gpu_device_destroy, ++ "Dummy model initialization failed"}, ++#else ++ {assign_irqs, NULL, ++ "IRQ search failed"}, ++ {registers_map, registers_unmap, ++ "Register map failed"}, ++#endif ++ {power_control_init, power_control_term, ++ "Power control initialization failed"}, ++ {kbase_device_io_history_init, kbase_device_io_history_term, ++ "Register access history initialization failed"}, ++ {kbase_device_early_init, kbase_device_early_term, ++ "Early device initialization failed"}, ++ {kbase_device_populate_max_freq, NULL, ++ "Populating max frequency failed"}, ++ {kbase_device_misc_init, kbase_device_misc_term, ++ "Miscellaneous device initialization failed"}, ++ {kbase_ctx_sched_init, kbase_ctx_sched_term, ++ "Context scheduler initialization failed"}, ++ {kbase_mem_init, kbase_mem_term, ++ "Memory subsystem initialization failed"}, ++ {kbase_csf_protected_memory_init, kbase_csf_protected_memory_term, ++ "Protected memory allocator initialization failed"}, ++ {kbase_device_coherency_init, NULL, ++ "Device coherency init failed"}, ++ {kbase_protected_mode_init, kbase_protected_mode_term, ++ "Protected mode subsystem initialization failed"}, ++ {kbase_device_list_init, kbase_device_list_term, ++ "Device list setup failed"}, ++ {kbase_device_timeline_init, kbase_device_timeline_term, ++ "Timeline stream initialization failed"}, ++ {kbase_clk_rate_trace_manager_init, ++ kbase_clk_rate_trace_manager_term, ++ "Clock rate trace manager initialization failed"}, ++ {kbase_device_hwcnt_backend_jm_init, ++ kbase_device_hwcnt_backend_jm_term, ++ "GPU hwcnt backend creation failed"}, ++ {kbase_device_hwcnt_context_init, kbase_device_hwcnt_context_term, ++ "GPU hwcnt context initialization failed"}, ++ {kbase_device_hwcnt_virtualizer_init, ++ kbase_device_hwcnt_virtualizer_term, ++ "GPU hwcnt virtualizer initialization failed"}, ++ {kbase_device_vinstr_init, kbase_device_vinstr_term, ++ "Virtual instrumentation initialization failed"}, ++ {kbase_backend_late_init, kbase_backend_late_term, ++ "Late backend initialization failed"}, ++ {kbase_device_csf_firmware_init, kbase_device_csf_firmware_term, ++ "Firmware initialization failed"}, ++#ifdef MALI_KBASE_BUILD ++ {kbase_device_debugfs_init, kbase_device_debugfs_term, ++ "DebugFS initialization failed"}, ++ /* Sysfs init needs to happen before registering the device with ++ * misc_register(), otherwise it causes a race condition between ++ * registering the device and a uevent event being generated for ++ * userspace, causing udev rules to run which might expect certain ++ * sysfs attributes present. As a result of the race condition ++ * we avoid, some Mali sysfs entries may have appeared to udev ++ * to not exist. ++ * For more information, see ++ * https://www.kernel.org/doc/Documentation/driver-model/device.txt, the ++ * paragraph that starts with "Word of warning", currently the ++ * second-last paragraph. ++ */ ++ {kbase_sysfs_init, kbase_sysfs_term, "SysFS group creation failed"}, ++ {kbase_device_misc_register, kbase_device_misc_deregister, ++ "Misc device registration failed"}, ++#ifdef CONFIG_MALI_BUSLOG ++ {buslog_init, buslog_term, "Bus log client registration failed"}, ++#endif ++ {kbase_gpuprops_populate_user_buffer, kbase_gpuprops_free_user_buffer, ++ "GPU property population failed"}, ++#endif ++}; ++ ++static void kbase_device_term_partial(struct kbase_device *kbdev, ++ unsigned int i) ++{ ++ while (i-- > 0) { ++ if (dev_init[i].term) ++ dev_init[i].term(kbdev); ++ } ++} ++ ++void kbase_device_term(struct kbase_device *kbdev) ++{ ++ kbase_device_term_partial(kbdev, ARRAY_SIZE(dev_init)); ++ kbase_mem_halt(kbdev); ++} ++ ++int kbase_device_init(struct kbase_device *kbdev) ++{ ++ int err = 0; ++ unsigned int i = 0; ++ ++ dev_info(kbdev->dev, "Kernel DDK version %s", MALI_RELEASE_NAME); ++ ++ kbase_device_id_init(kbdev); ++ kbase_disjoint_init(kbdev); ++ ++ for (i = 0; i < ARRAY_SIZE(dev_init); i++) { ++ err = dev_init[i].init(kbdev); ++ if (err) { ++ dev_err(kbdev->dev, "%s error = %d\n", ++ dev_init[i].err_mes, err); ++ kbase_device_term_partial(kbdev, i); ++ break; ++ } ++ } ++ ++ return err; ++} +diff --git a/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_csf.c b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_csf.c +new file mode 100755 +index 000000000000..97bcc1d23aa3 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_csf.c +@@ -0,0 +1,161 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * ++ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++/** ++ * kbase_report_gpu_fault - Report a GPU fault of the device. ++ * ++ * @kbdev: Kbase device pointer ++ * @status: Fault status ++ * @as_nr: Faulty address space ++ * @as_valid: true if address space is valid ++ * ++ * This function is called from the interrupt handler when a GPU fault occurs. ++ */ ++static void kbase_report_gpu_fault(struct kbase_device *kbdev, u32 status, ++ u32 as_nr, bool as_valid) ++{ ++ u64 address = (u64) kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(GPU_FAULTADDRESS_HI)) << 32; ++ ++ address |= kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(GPU_FAULTADDRESS_LO)); ++ ++ /* Report GPU fault for all contexts in case either ++ * the address space is invalid or it's MCU address space. ++ */ ++ kbase_mmu_gpu_fault_interrupt(kbdev, status, as_nr, address, as_valid); ++} ++ ++static bool kbase_gpu_fault_interrupt(struct kbase_device *kbdev) ++{ ++ const u32 status = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(GPU_FAULTSTATUS)); ++ const bool as_valid = status & GPU_FAULTSTATUS_JASID_VALID_FLAG; ++ const u32 as_nr = (status & GPU_FAULTSTATUS_JASID_MASK) >> ++ GPU_FAULTSTATUS_JASID_SHIFT; ++ bool bus_fault = (status & GPU_FAULTSTATUS_EXCEPTION_TYPE_MASK) == ++ GPU_FAULTSTATUS_EXCEPTION_TYPE_GPU_BUS_FAULT; ++ bool clear_gpu_fault = true; ++ ++ if (bus_fault) { ++ /* If as_valid, reset gpu when ASID is for MCU. */ ++ if (!as_valid || (as_nr == MCU_AS_NR)) { ++ kbase_report_gpu_fault(kbdev, status, as_nr, as_valid); ++ ++ dev_err(kbdev->dev, "GPU bus fault triggering gpu-reset ...\n"); ++ if (kbase_prepare_to_reset_gpu(kbdev)) ++ kbase_reset_gpu(kbdev); ++ } else { ++ /* Handle Bus fault */ ++ if (kbase_mmu_bus_fault_interrupt(kbdev, status, as_nr)) ++ clear_gpu_fault = false; ++ } ++ } else ++ kbase_report_gpu_fault(kbdev, status, as_nr, as_valid); ++ ++ return clear_gpu_fault; ++} ++ ++void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val) ++{ ++ bool clear_gpu_fault = false; ++ ++ KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ, NULL, val); ++ if (val & GPU_FAULT) ++ clear_gpu_fault = kbase_gpu_fault_interrupt(kbdev); ++ ++ if (val & GPU_PROTECTED_FAULT) { ++ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; ++ unsigned long flags; ++ ++ dev_err_ratelimited(kbdev->dev, "GPU fault in protected mode"); ++ ++ /* Mask the protected fault interrupt to avoid the potential ++ * deluge of such interrupts. It will be unmasked on GPU reset. ++ */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), ++ GPU_IRQ_REG_ALL & ~GPU_PROTECTED_FAULT); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ kbase_csf_scheduler_spin_lock(kbdev, &flags); ++ if (!WARN_ON(!kbase_csf_scheduler_protected_mode_in_use(kbdev))) ++ scheduler->active_protm_grp->faulted = true; ++ kbase_csf_scheduler_spin_unlock(kbdev, flags); ++ ++ if (kbase_prepare_to_reset_gpu(kbdev)) ++ kbase_reset_gpu(kbdev); ++ } ++ ++ if (val & RESET_COMPLETED) ++ kbase_pm_reset_done(kbdev); ++ ++ KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, val); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), val); ++ ++ /* kbase_pm_check_transitions (called by kbase_pm_power_changed) must ++ * be called after the IRQ has been cleared. This is because it might ++ * trigger further power transitions and we don't want to miss the ++ * interrupt raised to notify us that these further transitions have ++ * finished. The same applies to kbase_clean_caches_done() - if another ++ * clean was queued, it might trigger another clean, which might ++ * generate another interrupt which shouldn't be missed. ++ */ ++ ++ if (val & CLEAN_CACHES_COMPLETED) ++ kbase_clean_caches_done(kbdev); ++ ++ if (val & (POWER_CHANGED_ALL | MCU_STATUS_GPU_IRQ)) { ++ kbase_pm_power_changed(kbdev); ++ } else if (val & CLEAN_CACHES_COMPLETED) { ++ /* If cache line evict messages can be lost when shader cores ++ * power down then we need to flush the L2 cache before powering ++ * down cores. When the flush completes, the shaders' state ++ * machine needs to be re-invoked to proceed with powering down ++ * cores. ++ */ ++ if (kbdev->pm.backend.l2_always_on || ++ kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_921)) ++ kbase_pm_power_changed(kbdev); ++ } ++ ++ if (clear_gpu_fault) { ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), ++ GPU_COMMAND_CLEAR_FAULT); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ } ++ ++ KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ_DONE, NULL, val); ++} +diff --git a/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_jm.c b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_jm.c +new file mode 100755 +index 000000000000..a11d778071b5 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_jm.c +@@ -0,0 +1,100 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * ++ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++/** ++ * kbase_report_gpu_fault - Report a GPU fault. ++ * @kbdev: Kbase device pointer ++ * @multiple: Zero if only GPU_FAULT was raised, non-zero if MULTIPLE_GPU_FAULTS ++ * was also set ++ * ++ * This function is called from the interrupt handler when a GPU fault occurs. ++ * It reports the details of the fault using dev_warn(). ++ */ ++static void kbase_report_gpu_fault(struct kbase_device *kbdev, int multiple) ++{ ++ u32 status = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_FAULTSTATUS)); ++ u64 address = (u64) kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(GPU_FAULTADDRESS_HI)) << 32; ++ ++ address |= kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(GPU_FAULTADDRESS_LO)); ++ ++ dev_warn(kbdev->dev, "GPU Fault 0x%08x (%s) at 0x%016llx", ++ status, ++ kbase_gpu_exception_name(status & 0xFF), ++ address); ++ if (multiple) ++ dev_warn(kbdev->dev, "There were multiple GPU faults - some have not been reported\n"); ++} ++ ++void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val) ++{ ++ KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ, NULL, val); ++ if (val & GPU_FAULT) ++ kbase_report_gpu_fault(kbdev, val & MULTIPLE_GPU_FAULTS); ++ ++ if (val & RESET_COMPLETED) ++ kbase_pm_reset_done(kbdev); ++ ++ if (val & PRFCNT_SAMPLE_COMPLETED) ++ kbase_instr_hwcnt_sample_done(kbdev); ++ ++ KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, val); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), val); ++ ++ /* kbase_pm_check_transitions (called by kbase_pm_power_changed) must ++ * be called after the IRQ has been cleared. This is because it might ++ * trigger further power transitions and we don't want to miss the ++ * interrupt raised to notify us that these further transitions have ++ * finished. The same applies to kbase_clean_caches_done() - if another ++ * clean was queued, it might trigger another clean, which might ++ * generate another interrupt which shouldn't be missed. ++ */ ++ ++ if (val & CLEAN_CACHES_COMPLETED) ++ kbase_clean_caches_done(kbdev); ++ ++ if (val & POWER_CHANGED_ALL) { ++ kbase_pm_power_changed(kbdev); ++ } else if (val & CLEAN_CACHES_COMPLETED) { ++ /* If cache line evict messages can be lost when shader cores ++ * power down then we need to flush the L2 cache before powering ++ * down cores. When the flush completes, the shaders' state ++ * machine needs to be re-invoked to proceed with powering down ++ * cores. ++ */ ++ if (kbdev->pm.backend.l2_always_on || ++ kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_921)) ++ kbase_pm_power_changed(kbdev); ++ } ++ ++ KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ_DONE, NULL, val); ++} +diff --git a/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_jm.c b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_jm.c +new file mode 100755 +index 000000000000..8e853eb82fa1 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_jm.c +@@ -0,0 +1,264 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * ++ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include "../mali_kbase_device_internal.h" ++#include "../mali_kbase_device.h" ++ ++#include ++#include ++#include ++#include ++ ++#ifdef CONFIG_MALI_BIFROST_NO_MALI ++#include ++#endif ++ ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++#include ++#endif ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++/** ++ * kbase_backend_late_init - Perform any backend-specific initialization. ++ * @kbdev: Device pointer ++ * ++ * Return: 0 on success, or an error code on failure. ++ */ ++static int kbase_backend_late_init(struct kbase_device *kbdev) ++{ ++ int err; ++ ++ err = kbase_hwaccess_pm_init(kbdev); ++ if (err) ++ return err; ++ ++ err = kbase_reset_gpu_init(kbdev); ++ if (err) ++ goto fail_reset_gpu_init; ++ ++ err = kbase_hwaccess_pm_powerup(kbdev, PM_HW_ISSUES_DETECT); ++ if (err) ++ goto fail_pm_powerup; ++ ++ err = kbase_backend_timer_init(kbdev); ++ if (err) ++ goto fail_timer; ++ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++#ifndef CONFIG_MALI_BIFROST_NO_MALI ++ if (kbasep_common_test_interrupt_handlers(kbdev) != 0) { ++ dev_err(kbdev->dev, "Interrupt assignment check failed.\n"); ++ err = -EINVAL; ++ goto fail_interrupt_test; ++ } ++#endif /* !CONFIG_MALI_BIFROST_NO_MALI */ ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++ ++ err = kbase_job_slot_init(kbdev); ++ if (err) ++ goto fail_job_slot; ++ ++ /* Do the initialisation of devfreq. ++ * Devfreq needs backend_timer_init() for completion of its ++ * initialisation and it also needs to catch the first callback ++ * occurrence of the runtime_suspend event for maintaining state ++ * coherence with the backend power management, hence needs to be ++ * placed before the kbase_pm_context_idle(). ++ */ ++ err = kbase_backend_devfreq_init(kbdev); ++ if (err) ++ goto fail_devfreq_init; ++ ++ /* Idle the GPU and/or cores, if the policy wants it to */ ++ kbase_pm_context_idle(kbdev); ++ ++ /* Update gpuprops with L2_FEATURES if applicable */ ++ err = kbase_gpuprops_update_l2_features(kbdev); ++ if (err) ++ goto fail_update_l2_features; ++ ++ init_waitqueue_head(&kbdev->hwaccess.backend.reset_wait); ++ ++ return 0; ++ ++fail_update_l2_features: ++fail_devfreq_init: ++ kbase_job_slot_term(kbdev); ++fail_job_slot: ++ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++#ifndef CONFIG_MALI_BIFROST_NO_MALI ++fail_interrupt_test: ++#endif /* !CONFIG_MALI_BIFROST_NO_MALI */ ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++ ++ kbase_backend_timer_term(kbdev); ++fail_timer: ++ kbase_hwaccess_pm_halt(kbdev); ++fail_pm_powerup: ++ kbase_reset_gpu_term(kbdev); ++fail_reset_gpu_init: ++ kbase_hwaccess_pm_term(kbdev); ++ ++ return err; ++} ++ ++/** ++ * kbase_backend_late_term - Perform any backend-specific termination. ++ * @kbdev: Device pointer ++ */ ++static void kbase_backend_late_term(struct kbase_device *kbdev) ++{ ++ kbase_backend_devfreq_term(kbdev); ++ kbase_job_slot_halt(kbdev); ++ kbase_job_slot_term(kbdev); ++ kbase_backend_timer_term(kbdev); ++ kbase_hwaccess_pm_halt(kbdev); ++ kbase_reset_gpu_term(kbdev); ++ kbase_hwaccess_pm_term(kbdev); ++} ++ ++static const struct kbase_device_init dev_init[] = { ++#ifdef CONFIG_MALI_BIFROST_NO_MALI ++ {kbase_gpu_device_create, kbase_gpu_device_destroy, ++ "Dummy model initialization failed"}, ++#else ++ {assign_irqs, NULL, ++ "IRQ search failed"}, ++ {registers_map, registers_unmap, ++ "Register map failed"}, ++#endif ++ {kbase_device_io_history_init, kbase_device_io_history_term, ++ "Register access history initialization failed"}, ++ {kbase_device_pm_init, kbase_device_pm_term, ++ "Power management initialization failed"}, ++ {kbase_device_early_init, kbase_device_early_term, ++ "Early device initialization failed"}, ++ {kbase_device_populate_max_freq, NULL, ++ "Populating max frequency failed"}, ++ {kbase_device_misc_init, kbase_device_misc_term, ++ "Miscellaneous device initialization failed"}, ++ {kbase_ctx_sched_init, kbase_ctx_sched_term, ++ "Context scheduler initialization failed"}, ++ {kbase_mem_init, kbase_mem_term, ++ "Memory subsystem initialization failed"}, ++ {kbase_device_coherency_init, NULL, ++ "Device coherency init failed"}, ++ {kbase_protected_mode_init, kbase_protected_mode_term, ++ "Protected mode subsystem initialization failed"}, ++ {kbase_device_list_init, kbase_device_list_term, ++ "Device list setup failed"}, ++ {kbasep_js_devdata_init, kbasep_js_devdata_term, ++ "Job JS devdata initialization failed"}, ++ {kbase_device_timeline_init, kbase_device_timeline_term, ++ "Timeline stream initialization failed"}, ++ {kbase_clk_rate_trace_manager_init, ++ kbase_clk_rate_trace_manager_term, ++ "Clock rate trace manager initialization failed"}, ++ {kbase_device_hwcnt_backend_jm_init, ++ kbase_device_hwcnt_backend_jm_term, ++ "GPU hwcnt backend creation failed"}, ++ {kbase_device_hwcnt_context_init, kbase_device_hwcnt_context_term, ++ "GPU hwcnt context initialization failed"}, ++ {kbase_device_hwcnt_virtualizer_init, ++ kbase_device_hwcnt_virtualizer_term, ++ "GPU hwcnt virtualizer initialization failed"}, ++ {kbase_device_vinstr_init, kbase_device_vinstr_term, ++ "Virtual instrumentation initialization failed"}, ++ {kbase_backend_late_init, kbase_backend_late_term, ++ "Late backend initialization failed"}, ++#ifdef MALI_KBASE_BUILD ++ {kbase_debug_job_fault_dev_init, kbase_debug_job_fault_dev_term, ++ "Job fault debug initialization failed"}, ++ {kbase_device_debugfs_init, kbase_device_debugfs_term, ++ "DebugFS initialization failed"}, ++ /* Sysfs init needs to happen before registering the device with ++ * misc_register(), otherwise it causes a race condition between ++ * registering the device and a uevent event being generated for ++ * userspace, causing udev rules to run which might expect certain ++ * sysfs attributes present. As a result of the race condition ++ * we avoid, some Mali sysfs entries may have appeared to udev ++ * to not exist. ++ * For more information, see ++ * https://www.kernel.org/doc/Documentation/driver-model/device.txt, the ++ * paragraph that starts with "Word of warning", currently the ++ * second-last paragraph. ++ */ ++ {kbase_sysfs_init, kbase_sysfs_term, "SysFS group creation failed"}, ++ {kbase_device_misc_register, kbase_device_misc_deregister, ++ "Misc device registration failed"}, ++#ifdef CONFIG_MALI_BUSLOG ++ {buslog_init, buslog_term, "Bus log client registration failed"}, ++#endif ++ {kbase_gpuprops_populate_user_buffer, kbase_gpuprops_free_user_buffer, ++ "GPU property population failed"}, ++#endif ++ {kbase_dummy_job_wa_load, kbase_dummy_job_wa_cleanup, ++ "Dummy job workaround load failed"}, ++}; ++ ++static void kbase_device_term_partial(struct kbase_device *kbdev, ++ unsigned int i) ++{ ++ while (i-- > 0) { ++ if (dev_init[i].term) ++ dev_init[i].term(kbdev); ++ } ++} ++ ++void kbase_device_term(struct kbase_device *kbdev) ++{ ++ kbase_device_term_partial(kbdev, ARRAY_SIZE(dev_init)); ++ kbasep_js_devdata_halt(kbdev); ++ kbase_mem_halt(kbdev); ++} ++ ++int kbase_device_init(struct kbase_device *kbdev) ++{ ++ int err = 0; ++ unsigned int i = 0; ++ ++ dev_info(kbdev->dev, "Kernel DDK version %s", MALI_RELEASE_NAME); ++ ++ kbase_device_id_init(kbdev); ++ kbase_disjoint_init(kbdev); ++ ++ for (i = 0; i < ARRAY_SIZE(dev_init); i++) { ++ err = dev_init[i].init(kbdev); ++ if (err) { ++ dev_err(kbdev->dev, "%s error = %d\n", ++ dev_init[i].err_mes, err); ++ kbase_device_term_partial(kbdev, i); ++ break; ++ } ++ } ++ ++ return err; ++} +diff --git a/drivers/gpu/arm/bifrost/device/mali_kbase_device.c b/drivers/gpu/arm/bifrost/device/mali_kbase_device.c +new file mode 100755 +index 000000000000..76fb33a5e881 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/device/mali_kbase_device.c +@@ -0,0 +1,411 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * ++ * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* ++ * Base kernel device APIs ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#include "mali_kbase_vinstr.h" ++#include "mali_kbase_hwcnt_context.h" ++#include "mali_kbase_hwcnt_virtualizer.h" ++ ++#include "mali_kbase_device.h" ++#include "backend/gpu/mali_kbase_pm_internal.h" ++#include "backend/gpu/mali_kbase_irq_internal.h" ++#include "mali_kbase_regs_history_debugfs.h" ++ ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++#include "arbiter/mali_kbase_arbiter_pm.h" ++#endif /* CONFIG_MALI_ARBITER_SUPPORT */ ++ ++/* NOTE: Magic - 0x45435254 (TRCE in ASCII). ++ * Supports tracing feature provided in the base module. ++ * Please keep it in sync with the value of base module. ++ */ ++#define TRACE_BUFFER_HEADER_SPECIAL 0x45435254 ++ ++/* Number of register accesses for the buffer that we allocate during ++ * initialization time. The buffer size can be changed later via debugfs. ++ */ ++#define KBASEP_DEFAULT_REGISTER_HISTORY_SIZE ((u16)512) ++ ++static DEFINE_MUTEX(kbase_dev_list_lock); ++static LIST_HEAD(kbase_dev_list); ++static int kbase_dev_nr; ++ ++struct kbase_device *kbase_device_alloc(void) ++{ ++ return kzalloc(sizeof(struct kbase_device), GFP_KERNEL); ++} ++ ++/** ++ * kbase_device_all_as_init() - Initialise address space objects of the device. ++ * ++ * @kbdev: Pointer to kbase device. ++ * ++ * Return: 0 on success otherwise non-zero. ++ */ ++static int kbase_device_all_as_init(struct kbase_device *kbdev) ++{ ++ int i, err = 0; ++ ++ for (i = 0; i < kbdev->nr_hw_address_spaces; i++) { ++ err = kbase_mmu_as_init(kbdev, i); ++ if (err) ++ break; ++ } ++ ++ if (err) { ++ while (i-- > 0) ++ kbase_mmu_as_term(kbdev, i); ++ } ++ ++ return err; ++} ++ ++static void kbase_device_all_as_term(struct kbase_device *kbdev) ++{ ++ int i; ++ ++ for (i = 0; i < kbdev->nr_hw_address_spaces; i++) ++ kbase_mmu_as_term(kbdev, i); ++} ++ ++int kbase_device_misc_init(struct kbase_device * const kbdev) ++{ ++ int err; ++#ifdef CONFIG_ARM64 ++ struct device_node *np = NULL; ++#endif /* CONFIG_ARM64 */ ++ ++ spin_lock_init(&kbdev->mmu_mask_change); ++ mutex_init(&kbdev->mmu_hw_mutex); ++#ifdef CONFIG_ARM64 ++ kbdev->cci_snoop_enabled = false; ++ np = kbdev->dev->of_node; ++ if (np != NULL) { ++ if (of_property_read_u32(np, "snoop_enable_smc", ++ &kbdev->snoop_enable_smc)) ++ kbdev->snoop_enable_smc = 0; ++ if (of_property_read_u32(np, "snoop_disable_smc", ++ &kbdev->snoop_disable_smc)) ++ kbdev->snoop_disable_smc = 0; ++ /* Either both or none of the calls should be provided. */ ++ if (!((kbdev->snoop_disable_smc == 0 ++ && kbdev->snoop_enable_smc == 0) ++ || (kbdev->snoop_disable_smc != 0 ++ && kbdev->snoop_enable_smc != 0))) { ++ WARN_ON(1); ++ err = -EINVAL; ++ goto fail; ++ } ++ } ++#endif /* CONFIG_ARM64 */ ++ /* Get the list of workarounds for issues on the current HW ++ * (identified by the GPU_ID register) ++ */ ++ err = kbase_hw_set_issues_mask(kbdev); ++ if (err) ++ goto fail; ++ ++ /* Set the list of features available on the current HW ++ * (identified by the GPU_ID register) ++ */ ++ kbase_hw_set_features_mask(kbdev); ++ ++ err = kbase_gpuprops_set_features(kbdev); ++ if (err) ++ goto fail; ++ ++ /* On Linux 4.0+, dma coherency is determined from device tree */ ++#if defined(CONFIG_ARM64) && LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0) ++ set_dma_ops(kbdev->dev, &noncoherent_swiotlb_dma_ops); ++#endif ++ ++ /* Workaround a pre-3.13 Linux issue, where dma_mask is NULL when our ++ * device structure was created by device-tree ++ */ ++ if (!kbdev->dev->dma_mask) ++ kbdev->dev->dma_mask = &kbdev->dev->coherent_dma_mask; ++ ++ err = dma_set_mask(kbdev->dev, ++ DMA_BIT_MASK(kbdev->gpu_props.mmu.pa_bits)); ++ if (err) ++ goto dma_set_mask_failed; ++ ++ err = dma_set_coherent_mask(kbdev->dev, ++ DMA_BIT_MASK(kbdev->gpu_props.mmu.pa_bits)); ++ if (err) ++ goto dma_set_mask_failed; ++ ++ kbdev->nr_hw_address_spaces = kbdev->gpu_props.num_address_spaces; ++ ++ err = kbase_device_all_as_init(kbdev); ++ if (err) ++ goto dma_set_mask_failed; ++ ++ spin_lock_init(&kbdev->hwcnt.lock); ++ ++ err = kbase_ktrace_init(kbdev); ++ if (err) ++ goto term_as; ++ ++ init_waitqueue_head(&kbdev->cache_clean_wait); ++ ++ kbase_debug_assert_register_hook(&kbase_ktrace_hook_wrapper, kbdev); ++ ++ atomic_set(&kbdev->ctx_num, 0); ++ ++ err = kbase_instr_backend_init(kbdev); ++ if (err) ++ goto term_trace; ++ ++ kbdev->pm.dvfs_period = DEFAULT_PM_DVFS_PERIOD; ++ ++ kbdev->reset_timeout_ms = DEFAULT_RESET_TIMEOUT_MS; ++ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) ++ kbdev->mmu_mode = kbase_mmu_mode_get_aarch64(); ++ else ++ kbdev->mmu_mode = kbase_mmu_mode_get_lpae(); ++ ++ mutex_init(&kbdev->kctx_list_lock); ++ INIT_LIST_HEAD(&kbdev->kctx_list); ++ ++ spin_lock_init(&kbdev->hwaccess_lock); ++ ++ return 0; ++term_trace: ++ kbase_ktrace_term(kbdev); ++term_as: ++ kbase_device_all_as_term(kbdev); ++dma_set_mask_failed: ++fail: ++ return err; ++} ++ ++void kbase_device_misc_term(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev); ++ ++ WARN_ON(!list_empty(&kbdev->kctx_list)); ++ ++#if KBASE_KTRACE_ENABLE ++ kbase_debug_assert_register_hook(NULL, NULL); ++#endif ++ ++ kbase_instr_backend_term(kbdev); ++ ++ kbase_ktrace_term(kbdev); ++ ++ kbase_device_all_as_term(kbdev); ++} ++ ++void kbase_device_free(struct kbase_device *kbdev) ++{ ++ kfree(kbdev); ++} ++ ++void kbase_device_id_init(struct kbase_device *kbdev) ++{ ++ scnprintf(kbdev->devname, DEVNAME_SIZE, "%s%d", kbase_drv_name, ++ kbase_dev_nr); ++ kbdev->id = kbase_dev_nr; ++} ++ ++void kbase_increment_device_id(void) ++{ ++ kbase_dev_nr++; ++} ++ ++int kbase_device_hwcnt_backend_jm_init(struct kbase_device *kbdev) ++{ ++ return kbase_hwcnt_backend_jm_create(kbdev, &kbdev->hwcnt_gpu_iface); ++} ++ ++void kbase_device_hwcnt_backend_jm_term(struct kbase_device *kbdev) ++{ ++ kbase_hwcnt_backend_jm_destroy(&kbdev->hwcnt_gpu_iface); ++} ++ ++int kbase_device_hwcnt_context_init(struct kbase_device *kbdev) ++{ ++ return kbase_hwcnt_context_init(&kbdev->hwcnt_gpu_iface, ++ &kbdev->hwcnt_gpu_ctx); ++} ++ ++void kbase_device_hwcnt_context_term(struct kbase_device *kbdev) ++{ ++ kbase_hwcnt_context_term(kbdev->hwcnt_gpu_ctx); ++} ++ ++int kbase_device_hwcnt_virtualizer_init(struct kbase_device *kbdev) ++{ ++ return kbase_hwcnt_virtualizer_init(kbdev->hwcnt_gpu_ctx, ++ KBASE_HWCNT_GPU_VIRTUALIZER_DUMP_THRESHOLD_NS, ++ &kbdev->hwcnt_gpu_virt); ++} ++ ++void kbase_device_hwcnt_virtualizer_term(struct kbase_device *kbdev) ++{ ++ kbase_hwcnt_virtualizer_term(kbdev->hwcnt_gpu_virt); ++} ++ ++int kbase_device_timeline_init(struct kbase_device *kbdev) ++{ ++ atomic_set(&kbdev->timeline_flags, 0); ++ return kbase_timeline_init(&kbdev->timeline, &kbdev->timeline_flags); ++} ++ ++void kbase_device_timeline_term(struct kbase_device *kbdev) ++{ ++ kbase_timeline_term(kbdev->timeline); ++} ++ ++int kbase_device_vinstr_init(struct kbase_device *kbdev) ++{ ++ return kbase_vinstr_init(kbdev->hwcnt_gpu_virt, &kbdev->vinstr_ctx); ++} ++ ++void kbase_device_vinstr_term(struct kbase_device *kbdev) ++{ ++ kbase_vinstr_term(kbdev->vinstr_ctx); ++} ++ ++int kbase_device_io_history_init(struct kbase_device *kbdev) ++{ ++ return kbase_io_history_init(&kbdev->io_history, ++ KBASEP_DEFAULT_REGISTER_HISTORY_SIZE); ++} ++ ++void kbase_device_io_history_term(struct kbase_device *kbdev) ++{ ++ kbase_io_history_term(&kbdev->io_history); ++} ++ ++int kbase_device_misc_register(struct kbase_device *kbdev) ++{ ++ return misc_register(&kbdev->mdev); ++} ++ ++void kbase_device_misc_deregister(struct kbase_device *kbdev) ++{ ++ misc_deregister(&kbdev->mdev); ++} ++ ++int kbase_device_list_init(struct kbase_device *kbdev) ++{ ++ const struct list_head *dev_list; ++ ++ dev_list = kbase_device_get_list(); ++ list_add(&kbdev->entry, &kbase_dev_list); ++ kbase_device_put_list(dev_list); ++ ++ return 0; ++} ++ ++void kbase_device_list_term(struct kbase_device *kbdev) ++{ ++ const struct list_head *dev_list; ++ ++ dev_list = kbase_device_get_list(); ++ list_del(&kbdev->entry); ++ kbase_device_put_list(dev_list); ++} ++ ++const struct list_head *kbase_device_get_list(void) ++{ ++ mutex_lock(&kbase_dev_list_lock); ++ return &kbase_dev_list; ++} ++KBASE_EXPORT_TEST_API(kbase_device_get_list); ++ ++void kbase_device_put_list(const struct list_head *dev_list) ++{ ++ mutex_unlock(&kbase_dev_list_lock); ++} ++KBASE_EXPORT_TEST_API(kbase_device_put_list); ++ ++int kbase_device_early_init(struct kbase_device *kbdev) ++{ ++ int err; ++ ++ err = kbasep_platform_device_init(kbdev); ++ if (err) ++ return err; ++ ++ err = kbase_pm_runtime_init(kbdev); ++ if (err) ++ goto fail_runtime_pm; ++ ++ /* Ensure we can access the GPU registers */ ++ kbase_pm_register_access_enable(kbdev); ++ ++ /* Find out GPU properties based on the GPU feature registers */ ++ kbase_gpuprops_set(kbdev); ++ ++ /* We're done accessing the GPU registers for now. */ ++ kbase_pm_register_access_disable(kbdev); ++ ++ err = kbase_install_interrupts(kbdev); ++ if (err) ++ goto fail_interrupts; ++ ++ return 0; ++ ++fail_interrupts: ++ kbase_pm_runtime_term(kbdev); ++fail_runtime_pm: ++ kbasep_platform_device_term(kbdev); ++ ++ return err; ++} ++ ++void kbase_device_early_term(struct kbase_device *kbdev) ++{ ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++ if (kbdev->arb.arb_if) ++ kbase_arbiter_pm_release_interrupts(kbdev); ++ else ++ kbase_release_interrupts(kbdev); ++#else ++ kbase_release_interrupts(kbdev); ++#endif /* CONFIG_MALI_ARBITER_SUPPORT */ ++ kbase_pm_runtime_term(kbdev); ++ kbasep_platform_device_term(kbdev); ++} +diff --git a/drivers/gpu/arm/bifrost/device/mali_kbase_device.h b/drivers/gpu/arm/bifrost/device/mali_kbase_device.h +new file mode 100755 +index 000000000000..33264bcc0464 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/device/mali_kbase_device.h +@@ -0,0 +1,177 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++ ++/** ++ * kbase_device_get_list - get device list. ++ * ++ * Get access to device list. ++ * ++ * Return: Pointer to the linked list head. ++ */ ++const struct list_head *kbase_device_get_list(void); ++ ++/** ++ * kbase_device_put_list - put device list. ++ * ++ * @dev_list: head of linked list containing device list. ++ * ++ * Put access to the device list. ++ */ ++void kbase_device_put_list(const struct list_head *dev_list); ++ ++/** ++ * Kbase_increment_device_id - increment device id. ++ * ++ * Used to increment device id on successful initialization of the device. ++ */ ++void kbase_increment_device_id(void); ++ ++/** ++ * kbase_device_init - Device initialisation. ++ * ++ * This is called from device probe to initialise various other ++ * components needed. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * Return: 0 on success and non-zero value on failure. ++ */ ++int kbase_device_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_device_term - Device termination. ++ * ++ * This is called from device remove to terminate various components that ++ * were initialised during kbase_device_init. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ */ ++void kbase_device_term(struct kbase_device *kbdev); ++ ++/** ++ * kbase_reg_write - write to GPU register ++ * @kbdev: Kbase device pointer ++ * @offset: Offset of register ++ * @value: Value to write ++ * ++ * Caller must ensure the GPU is powered (@kbdev->pm.gpu_powered != false). ++ */ ++void kbase_reg_write(struct kbase_device *kbdev, u32 offset, u32 value); ++ ++/** ++ * kbase_reg_read - read from GPU register ++ * @kbdev: Kbase device pointer ++ * @offset: Offset of register ++ * ++ * Caller must ensure the GPU is powered (@kbdev->pm.gpu_powered != false). ++ * ++ * Return: Value in desired register ++ */ ++u32 kbase_reg_read(struct kbase_device *kbdev, u32 offset); ++ ++/** ++ * kbase_is_gpu_removed() - Has the GPU been removed. ++ * @kbdev: Kbase device pointer ++ * ++ * When Kbase takes too long to give up the GPU, the Arbiter ++ * can remove it. This will then be followed by a GPU lost event. ++ * This function will return true if the GPU has been removed. ++ * When this happens register reads will be zero. A zero GPU_ID is ++ * invalid so this is used to detect when GPU is removed. ++ * ++ * Return: True if GPU removed ++ */ ++bool kbase_is_gpu_removed(struct kbase_device *kbdev); ++ ++/** ++ * kbase_gpu_start_cache_clean - Start a cache clean ++ * @kbdev: Kbase device ++ * ++ * Issue a cache clean and invalidate command to hardware. This function will ++ * take hwaccess_lock. ++ */ ++void kbase_gpu_start_cache_clean(struct kbase_device *kbdev); ++ ++/** ++ * kbase_gpu_start_cache_clean_nolock - Start a cache clean ++ * @kbdev: Kbase device ++ * ++ * Issue a cache clean and invalidate command to hardware. hwaccess_lock ++ * must be held by the caller. ++ */ ++void kbase_gpu_start_cache_clean_nolock(struct kbase_device *kbdev); ++ ++/** ++ * kbase_gpu_wait_cache_clean - Wait for cache cleaning to finish ++ * @kbdev: Kbase device ++ * ++ * This function will take hwaccess_lock, and may sleep. ++ */ ++void kbase_gpu_wait_cache_clean(struct kbase_device *kbdev); ++ ++/** ++ * kbase_gpu_wait_cache_clean_timeout - Wait for certain time for cache ++ * cleaning to finish ++ * @kbdev: Kbase device ++ * @wait_timeout_ms: Time in milliseconds, to wait for cache clean to complete. ++ * ++ * This function will take hwaccess_lock, and may sleep. This is supposed to be ++ * called from paths (like GPU reset) where an indefinite wait for the ++ * completion of cache clean operation can cause deadlock, as the operation may ++ * never complete. ++ * ++ * Return: 0 if successful or a negative error code on failure. ++ */ ++int kbase_gpu_wait_cache_clean_timeout(struct kbase_device *kbdev, ++ unsigned int wait_timeout_ms); ++ ++/** ++ * kbase_gpu_cache_clean_wait_complete - Called after the cache cleaning is ++ * finished. Would also be called after ++ * the GPU reset. ++ * @kbdev: Kbase device ++ * ++ * Caller must hold the hwaccess_lock. ++ */ ++void kbase_gpu_cache_clean_wait_complete(struct kbase_device *kbdev); ++ ++/** ++ * kbase_clean_caches_done - Issue preiously queued cache clean request or ++ * wake up the requester that issued cache clean. ++ * @kbdev: Kbase device ++ * ++ * Caller must hold the hwaccess_lock. ++ */ ++void kbase_clean_caches_done(struct kbase_device *kbdev); ++ ++/** ++ * kbase_gpu_interrupt - GPU interrupt handler ++ * @kbdev: Kbase device pointer ++ * @val: The value of the GPU IRQ status register which triggered the call ++ * ++ * This function is called from the interrupt handler when a GPU irq is to be ++ * handled. ++ */ ++void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val); +diff --git a/drivers/gpu/arm/bifrost/device/mali_kbase_device_hw.c b/drivers/gpu/arm/bifrost/device/mali_kbase_device_hw.c +new file mode 100755 +index 000000000000..3a75c6c05cfa +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/device/mali_kbase_device_hw.c +@@ -0,0 +1,184 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * ++ * (C) COPYRIGHT 2014-2016, 2018-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#if !defined(CONFIG_MALI_BIFROST_NO_MALI) ++void kbase_reg_write(struct kbase_device *kbdev, u32 offset, u32 value) ++{ ++ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered); ++ KBASE_DEBUG_ASSERT(kbdev->dev != NULL); ++ ++ writel(value, kbdev->reg + offset); ++ ++#ifdef CONFIG_DEBUG_FS ++ if (unlikely(kbdev->io_history.enabled)) ++ kbase_io_history_add(&kbdev->io_history, kbdev->reg + offset, ++ value, 1); ++#endif /* CONFIG_DEBUG_FS */ ++ dev_dbg(kbdev->dev, "w: reg %08x val %08x", offset, value); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_reg_write); ++ ++u32 kbase_reg_read(struct kbase_device *kbdev, u32 offset) ++{ ++ u32 val; ++ ++ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered); ++ KBASE_DEBUG_ASSERT(kbdev->dev != NULL); ++ ++ val = readl(kbdev->reg + offset); ++ ++#ifdef CONFIG_DEBUG_FS ++ if (unlikely(kbdev->io_history.enabled)) ++ kbase_io_history_add(&kbdev->io_history, kbdev->reg + offset, ++ val, 0); ++#endif /* CONFIG_DEBUG_FS */ ++ dev_dbg(kbdev->dev, "r: reg %08x val %08x", offset, val); ++ ++ return val; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_reg_read); ++ ++bool kbase_is_gpu_removed(struct kbase_device *kbdev) ++{ ++ u32 val; ++ ++ val = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_ID)); ++ ++ return val == 0; ++} ++#endif /* !defined(CONFIG_MALI_BIFROST_NO_MALI) */ ++ ++void kbase_gpu_start_cache_clean_nolock(struct kbase_device *kbdev) ++{ ++ u32 irq_mask; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (kbdev->cache_clean_in_progress) { ++ /* If this is called while another clean is in progress, we ++ * can't rely on the current one to flush any new changes in ++ * the cache. Instead, trigger another cache clean immediately ++ * after this one finishes. ++ */ ++ kbdev->cache_clean_queued = true; ++ return; ++ } ++ ++ /* Enable interrupt */ ++ irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK)); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), ++ irq_mask | CLEAN_CACHES_COMPLETED); ++ ++ KBASE_KTRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, 0); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), ++ GPU_COMMAND_CLEAN_INV_CACHES); ++ ++ kbdev->cache_clean_in_progress = true; ++} ++ ++void kbase_gpu_start_cache_clean(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_gpu_start_cache_clean_nolock(kbdev); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++} ++ ++void kbase_gpu_cache_clean_wait_complete(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ kbdev->cache_clean_queued = false; ++ kbdev->cache_clean_in_progress = false; ++ wake_up(&kbdev->cache_clean_wait); ++} ++ ++void kbase_clean_caches_done(struct kbase_device *kbdev) ++{ ++ u32 irq_mask; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ if (kbdev->cache_clean_queued) { ++ kbdev->cache_clean_queued = false; ++ ++ KBASE_KTRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, 0); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), ++ GPU_COMMAND_CLEAN_INV_CACHES); ++ } else { ++ /* Disable interrupt */ ++ irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK)); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), ++ irq_mask & ~CLEAN_CACHES_COMPLETED); ++ ++ kbase_gpu_cache_clean_wait_complete(kbdev); ++ } ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++} ++ ++static inline bool get_cache_clean_flag(struct kbase_device *kbdev) ++{ ++ bool cache_clean_in_progress; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ cache_clean_in_progress = kbdev->cache_clean_in_progress; ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ return cache_clean_in_progress; ++} ++ ++void kbase_gpu_wait_cache_clean(struct kbase_device *kbdev) ++{ ++ while (get_cache_clean_flag(kbdev)) { ++ wait_event_interruptible(kbdev->cache_clean_wait, ++ !kbdev->cache_clean_in_progress); ++ } ++} ++ ++int kbase_gpu_wait_cache_clean_timeout(struct kbase_device *kbdev, ++ unsigned int wait_timeout_ms) ++{ ++ long remaining = msecs_to_jiffies(wait_timeout_ms); ++ ++ while (remaining && get_cache_clean_flag(kbdev)) { ++ remaining = wait_event_timeout(kbdev->cache_clean_wait, ++ !kbdev->cache_clean_in_progress, ++ remaining); ++ } ++ ++ return (remaining ? 0 : -ETIMEDOUT); ++} +diff --git a/drivers/gpu/arm/bifrost/device/mali_kbase_device_internal.h b/drivers/gpu/arm/bifrost/device/mali_kbase_device_internal.h +new file mode 100755 +index 000000000000..54644582eac5 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/device/mali_kbase_device_internal.h +@@ -0,0 +1,78 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++ ++typedef int kbase_device_init_method(struct kbase_device *kbdev); ++typedef void kbase_device_term_method(struct kbase_device *kbdev); ++ ++/** ++ * struct kbase_device_init - Device init/term methods. ++ * @init: Function pointer to a initialise method. ++ * @term: Function pointer to a terminate method. ++ * @err_mes: Error message to be printed when init method fails. ++ */ ++struct kbase_device_init { ++ kbase_device_init_method *init; ++ kbase_device_term_method *term; ++ char *err_mes; ++}; ++ ++int kbase_device_vinstr_init(struct kbase_device *kbdev); ++void kbase_device_vinstr_term(struct kbase_device *kbdev); ++ ++int kbase_device_timeline_init(struct kbase_device *kbdev); ++void kbase_device_timeline_term(struct kbase_device *kbdev); ++ ++int kbase_device_hwcnt_backend_jm_init(struct kbase_device *kbdev); ++void kbase_device_hwcnt_backend_jm_term(struct kbase_device *kbdev); ++ ++int kbase_device_hwcnt_context_init(struct kbase_device *kbdev); ++void kbase_device_hwcnt_context_term(struct kbase_device *kbdev); ++ ++int kbase_device_hwcnt_virtualizer_init(struct kbase_device *kbdev); ++void kbase_device_hwcnt_virtualizer_term(struct kbase_device *kbdev); ++ ++int kbase_device_list_init(struct kbase_device *kbdev); ++void kbase_device_list_term(struct kbase_device *kbdev); ++ ++int kbase_device_io_history_init(struct kbase_device *kbdev); ++void kbase_device_io_history_term(struct kbase_device *kbdev); ++ ++int kbase_device_misc_register(struct kbase_device *kbdev); ++void kbase_device_misc_deregister(struct kbase_device *kbdev); ++ ++void kbase_device_id_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_device_early_init - Perform any device-specific initialization. ++ * @kbdev: Device pointer ++ * ++ * Return: 0 on success, or an error code on failure. ++ */ ++int kbase_device_early_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_device_early_term - Perform any device-specific termination. ++ * @kbdev: Device pointer ++ */ ++void kbase_device_early_term(struct kbase_device *kbdev); +diff --git a/drivers/gpu/arm/bifrost/gpu/backend/mali_kbase_gpu_fault_csf.c b/drivers/gpu/arm/bifrost/gpu/backend/mali_kbase_gpu_fault_csf.c +new file mode 100755 +index 000000000000..f7e9b125ba8b +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/gpu/backend/mali_kbase_gpu_fault_csf.c +@@ -0,0 +1,105 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++#include "csf/mali_gpu_csf_registers.h" ++#include "../mali_kbase_gpu_fault.h" ++ ++const char *kbase_gpu_exception_name(u32 const exception_code) ++{ ++ const char *e; ++ ++ switch (exception_code) { ++ /* Command Stream exceptions */ ++ case CS_FAULT_EXCEPTION_TYPE_CS_RESOURCE_TERMINATED: ++ e = "CS_RESOURCE_TERMINATED"; ++ break; ++ case CS_FAULT_EXCEPTION_TYPE_CS_INHERIT_FAULT: ++ e = "CS_INHERIT_FAULT"; ++ break; ++ /* Command Stream fatal exceptions */ ++ case CS_FATAL_EXCEPTION_TYPE_CS_CONFIG_FAULT: ++ e = "CS_CONFIG_FAULT"; ++ break; ++ case CS_FATAL_EXCEPTION_TYPE_CS_ENDPOINT_FAULT: ++ e = "FATAL_CS_ENDPOINT_FAULT"; ++ break; ++ case CS_FATAL_EXCEPTION_TYPE_CS_BUS_FAULT: ++ e = "FATAL_CS_BUS_FAULT"; ++ break; ++ case CS_FATAL_EXCEPTION_TYPE_CS_INVALID_INSTRUCTION: ++ e = "FATAL_CS_INVALID_INSTRUCTION"; ++ break; ++ case CS_FATAL_EXCEPTION_TYPE_CS_CALL_STACK_OVERFLOW: ++ e = "FATAL_CS_CALL_STACK_OVERFLOW"; ++ break; ++ /* Shader exceptions */ ++ case CS_FAULT_EXCEPTION_TYPE_INSTR_INVALID_PC: ++ e = "INSTR_INVALID_PC"; ++ break; ++ case CS_FAULT_EXCEPTION_TYPE_INSTR_INVALID_ENC: ++ e = "INSTR_INVALID_ENC"; ++ break; ++ case CS_FAULT_EXCEPTION_TYPE_INSTR_BARRIER_FAULT: ++ e = "INSTR_BARRIER_FAULT"; ++ break; ++ /* Misc exceptions */ ++ case CS_FAULT_EXCEPTION_TYPE_DATA_INVALID_FAULT: ++ e = "DATA_INVALID_FAULT"; ++ break; ++ case CS_FAULT_EXCEPTION_TYPE_TILE_RANGE_FAULT: ++ e = "TILE_RANGE_FAULT"; ++ break; ++ case CS_FAULT_EXCEPTION_TYPE_ADDR_RANGE_FAULT: ++ e = "ADDR_RANGE_FAULT"; ++ break; ++ case CS_FAULT_EXCEPTION_TYPE_IMPRECISE_FAULT: ++ e = "IMPRECISE_FAULT"; ++ break; ++ /* FW exceptions */ ++ case CS_FATAL_EXCEPTION_TYPE_FIRMWARE_INTERNAL_ERROR: ++ e = "FIRMWARE_INTERNAL_ERROR"; ++ break; ++ case CS_FAULT_EXCEPTION_TYPE_RESOURCE_EVICTION_TIMEOUT: ++ e = "RESOURCE_EVICTION_TIMEOUT"; ++ break; ++ /* GPU Fault */ ++ case GPU_FAULTSTATUS_EXCEPTION_TYPE_GPU_BUS_FAULT: ++ e = "GPU_BUS_FAULT"; ++ break; ++ case GPU_FAULTSTATUS_EXCEPTION_TYPE_GPU_SHAREABILITY_FAULT: ++ e = "GPU_SHAREABILITY_FAULT"; ++ break; ++ case GPU_FAULTSTATUS_EXCEPTION_TYPE_SYSTEM_SHAREABILITY_FAULT: ++ e = "SYSTEM_SHAREABILITY_FAULT"; ++ break; ++ case GPU_FAULTSTATUS_EXCEPTION_TYPE_GPU_CACHEABILITY_FAULT: ++ e = "GPU_CACHEABILITY_FAULT"; ++ break; ++ /* Any other exception code is unknown */ ++ default: ++ e = "UNKNOWN"; ++ break; ++ } ++ ++ return e; ++} +diff --git a/drivers/gpu/arm/bifrost/gpu/backend/mali_kbase_gpu_fault_jm.c b/drivers/gpu/arm/bifrost/gpu/backend/mali_kbase_gpu_fault_jm.c +new file mode 100755 +index 000000000000..56f541516489 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/gpu/backend/mali_kbase_gpu_fault_jm.c +@@ -0,0 +1,177 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++ ++#include "../mali_kbase_gpu_fault.h" ++ ++const char *kbase_gpu_exception_name(u32 const exception_code) ++{ ++ const char *e; ++ ++ switch (exception_code) { ++ /* Non-Fault Status code */ ++ case 0x00: ++ e = "NOT_STARTED/IDLE/OK"; ++ break; ++ case 0x01: ++ e = "DONE"; ++ break; ++ case 0x02: ++ e = "INTERRUPTED"; ++ break; ++ case 0x03: ++ e = "STOPPED"; ++ break; ++ case 0x04: ++ e = "TERMINATED"; ++ break; ++ case 0x08: ++ e = "ACTIVE"; ++ break; ++ /* Job exceptions */ ++ case 0x40: ++ e = "JOB_CONFIG_FAULT"; ++ break; ++ case 0x41: ++ e = "JOB_POWER_FAULT"; ++ break; ++ case 0x42: ++ e = "JOB_READ_FAULT"; ++ break; ++ case 0x43: ++ e = "JOB_WRITE_FAULT"; ++ break; ++ case 0x44: ++ e = "JOB_AFFINITY_FAULT"; ++ break; ++ case 0x48: ++ e = "JOB_BUS_FAULT"; ++ break; ++ case 0x50: ++ e = "INSTR_INVALID_PC"; ++ break; ++ case 0x51: ++ e = "INSTR_INVALID_ENC"; ++ break; ++ case 0x52: ++ e = "INSTR_TYPE_MISMATCH"; ++ break; ++ case 0x53: ++ e = "INSTR_OPERAND_FAULT"; ++ break; ++ case 0x54: ++ e = "INSTR_TLS_FAULT"; ++ break; ++ case 0x55: ++ e = "INSTR_BARRIER_FAULT"; ++ break; ++ case 0x56: ++ e = "INSTR_ALIGN_FAULT"; ++ break; ++ case 0x58: ++ e = "DATA_INVALID_FAULT"; ++ break; ++ case 0x59: ++ e = "TILE_RANGE_FAULT"; ++ break; ++ case 0x5A: ++ e = "ADDR_RANGE_FAULT"; ++ break; ++ case 0x60: ++ e = "OUT_OF_MEMORY"; ++ break; ++ /* GPU exceptions */ ++ case 0x80: ++ e = "DELAYED_BUS_FAULT"; ++ break; ++ case 0x88: ++ e = "SHAREABILITY_FAULT"; ++ break; ++ /* MMU exceptions */ ++ case 0xC0: ++ case 0xC1: ++ case 0xC2: ++ case 0xC3: ++ case 0xC4: ++ case 0xC5: ++ case 0xC6: ++ case 0xC7: ++ e = "TRANSLATION_FAULT"; ++ break; ++ case 0xC8: ++ case 0xC9: ++ case 0xCA: ++ case 0xCB: ++ case 0xCC: ++ case 0xCD: ++ case 0xCE: ++ case 0xCF: ++ e = "PERMISSION_FAULT"; ++ break; ++ case 0xD0: ++ case 0xD1: ++ case 0xD2: ++ case 0xD3: ++ case 0xD4: ++ case 0xD5: ++ case 0xD6: ++ case 0xD7: ++ e = "TRANSTAB_BUS_FAULT"; ++ break; ++ case 0xD8: ++ case 0xD9: ++ case 0xDA: ++ case 0xDB: ++ case 0xDC: ++ case 0xDD: ++ case 0xDE: ++ case 0xDF: ++ e = "ACCESS_FLAG"; ++ break; ++ case 0xE0: ++ case 0xE1: ++ case 0xE2: ++ case 0xE3: ++ case 0xE4: ++ case 0xE5: ++ case 0xE6: ++ case 0xE7: ++ e = "ADDRESS_SIZE_FAULT"; ++ break; ++ case 0xE8: ++ case 0xE9: ++ case 0xEA: ++ case 0xEB: ++ case 0xEC: ++ case 0xED: ++ case 0xEE: ++ case 0xEF: ++ e = "MEMORY_ATTRIBUTES_FAULT"; ++ break; ++ default: ++ e = "UNKNOWN"; ++ break; ++ }; ++ ++ return e; ++} +diff --git a/drivers/gpu/arm/bifrost/gpu/backend/mali_kbase_gpu_regmap_csf.h b/drivers/gpu/arm/bifrost/gpu/backend/mali_kbase_gpu_regmap_csf.h +new file mode 100755 +index 000000000000..ff6e4ae47184 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/gpu/backend/mali_kbase_gpu_regmap_csf.h +@@ -0,0 +1,297 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_GPU_REGMAP_CSF_H_ ++#define _KBASE_GPU_REGMAP_CSF_H_ ++ ++#if !MALI_USE_CSF ++#error "Cannot be compiled with JM" ++#endif ++ ++#include "csf/mali_gpu_csf_control_registers.h" ++#define GPU_CONTROL_MCU_REG(r) (GPU_CONTROL_MCU + (r)) ++ ++ ++/* Set to implementation defined, outer caching */ ++#define AS_MEMATTR_AARCH64_OUTER_IMPL_DEF 0x88ull ++/* Set to write back memory, outer caching */ ++#define AS_MEMATTR_AARCH64_OUTER_WA 0x8Dull ++/* Set to inner non-cacheable, outer-non-cacheable ++ * Setting defined by the alloc bits is ignored, but set to a valid encoding: ++ * - no-alloc on read ++ * - no alloc on write ++ */ ++#define AS_MEMATTR_AARCH64_NON_CACHEABLE 0x4Cull ++/* Set to shared memory, that is inner cacheable on ACE and inner or outer ++ * shared, otherwise inner non-cacheable. ++ * Outer cacheable if inner or outer shared, otherwise outer non-cacheable. ++ */ ++#define AS_MEMATTR_AARCH64_SHARED 0x8ull ++ ++/* Symbols for default MEMATTR to use ++ * Default is - HW implementation defined caching ++ */ ++#define AS_MEMATTR_INDEX_DEFAULT 0 ++#define AS_MEMATTR_INDEX_DEFAULT_ACE 3 ++ ++/* HW implementation defined caching */ ++#define AS_MEMATTR_INDEX_IMPL_DEF_CACHE_POLICY 0 ++/* Force cache on */ ++#define AS_MEMATTR_INDEX_FORCE_TO_CACHE_ALL 1 ++/* Write-alloc */ ++#define AS_MEMATTR_INDEX_WRITE_ALLOC 2 ++/* Outer coherent, inner implementation defined policy */ ++#define AS_MEMATTR_INDEX_OUTER_IMPL_DEF 3 ++/* Outer coherent, write alloc inner */ ++#define AS_MEMATTR_INDEX_OUTER_WA 4 ++/* Normal memory, inner non-cacheable, outer non-cacheable (ARMv8 mode only) */ ++#define AS_MEMATTR_INDEX_NON_CACHEABLE 5 ++/* Normal memory, shared between MCU and Host */ ++#define AS_MEMATTR_INDEX_SHARED 6 ++ ++/* Configuration bits for the Command Stream Frontend. */ ++#define CSF_CONFIG 0xF00 ++ ++/* CSF_CONFIG register */ ++#define CSF_CONFIG_FORCE_COHERENCY_FEATURES_SHIFT 2 ++ ++/* GPU control registers */ ++#define MCU_CONTROL 0x700 ++#define MCU_STATUS 0x704 ++ ++#define MCU_CNTRL_ENABLE (1 << 0) ++#define MCU_CNTRL_AUTO (1 << 1) ++#define MCU_CNTRL_DISABLE (0) ++ ++#define MCU_STATUS_HALTED (1 << 1) ++ ++#define PRFCNT_BASE_LO 0x060 /* (RW) Performance counter memory ++ * region base address, low word ++ */ ++#define PRFCNT_BASE_HI 0x064 /* (RW) Performance counter memory ++ * region base address, high word ++ */ ++#define PRFCNT_CONFIG 0x068 /* (RW) Performance counter ++ * configuration ++ */ ++ ++#define PRFCNT_CSHW_EN 0x06C /* (RW) Performance counter ++ * enable for Command Stream Hardware ++ */ ++ ++#define PRFCNT_SHADER_EN 0x070 /* (RW) Performance counter enable ++ * flags for shader cores ++ */ ++#define PRFCNT_TILER_EN 0x074 /* (RW) Performance counter enable ++ * flags for tiler ++ */ ++#define PRFCNT_MMU_L2_EN 0x07C /* (RW) Performance counter enable ++ * flags for MMU/L2 cache ++ */ ++ ++/* JOB IRQ flags */ ++#define JOB_IRQ_GLOBAL_IF (1 << 31) /* Global interface interrupt received */ ++ ++/* GPU_COMMAND codes */ ++#define GPU_COMMAND_CODE_NOP 0x00 /* No operation, nothing happens */ ++#define GPU_COMMAND_CODE_RESET 0x01 /* Reset the GPU */ ++#define GPU_COMMAND_CODE_PRFCNT 0x02 /* Clear or sample performance counters */ ++#define GPU_COMMAND_CODE_TIME 0x03 /* Configure time sources */ ++#define GPU_COMMAND_CODE_FLUSH_CACHES 0x04 /* Flush caches */ ++#define GPU_COMMAND_CODE_SET_PROTECTED_MODE 0x05 /* Places the GPU in protected mode */ ++#define GPU_COMMAND_CODE_FINISH_HALT 0x06 /* Halt CSF */ ++#define GPU_COMMAND_CODE_CLEAR_FAULT 0x07 /* Clear GPU_FAULTSTATUS and GPU_FAULTADDRESS, TODX */ ++ ++/* GPU_COMMAND_RESET payloads */ ++ ++/* This will leave the state of active jobs UNDEFINED, but will leave the external bus in a defined and idle state. ++ * Power domains will remain powered on. ++ */ ++#define GPU_COMMAND_RESET_PAYLOAD_FAST_RESET 0x00 ++ ++/* This will leave the state of active command streams UNDEFINED, but will leave the external bus in a defined and ++ * idle state. ++ */ ++#define GPU_COMMAND_RESET_PAYLOAD_SOFT_RESET 0x01 ++ ++/* This reset will leave the state of currently active streams UNDEFINED, will likely lose data, and may leave ++ * the system bus in an inconsistent state. Use only as a last resort when nothing else works. ++ */ ++#define GPU_COMMAND_RESET_PAYLOAD_HARD_RESET 0x02 ++ ++/* GPU_COMMAND_PRFCNT payloads */ ++#define GPU_COMMAND_PRFCNT_PAYLOAD_SAMPLE 0x01 /* Sample performance counters */ ++#define GPU_COMMAND_PRFCNT_PAYLOAD_CLEAR 0x02 /* Clear performance counters */ ++ ++/* GPU_COMMAND_TIME payloads */ ++#define GPU_COMMAND_TIME_DISABLE 0x00 /* Disable cycle counter */ ++#define GPU_COMMAND_TIME_ENABLE 0x01 /* Enable cycle counter */ ++ ++/* GPU_COMMAND_FLUSH_CACHES payloads */ ++#define GPU_COMMAND_FLUSH_PAYLOAD_NONE 0x00 /* No flush */ ++#define GPU_COMMAND_FLUSH_PAYLOAD_CLEAN 0x01 /* Clean the caches */ ++#define GPU_COMMAND_FLUSH_PAYLOAD_INVALIDATE 0x02 /* Invalidate the caches */ ++#define GPU_COMMAND_FLUSH_PAYLOAD_CLEAN_INVALIDATE 0x03 /* Clean and invalidate the caches */ ++ ++/* GPU_COMMAND command + payload */ ++#define GPU_COMMAND_CODE_PAYLOAD(opcode, payload) \ ++ ((u32)opcode | ((u32)payload << 8)) ++ ++/* Final GPU_COMMAND form */ ++/* No operation, nothing happens */ ++#define GPU_COMMAND_NOP \ ++ GPU_COMMAND_CODE_PAYLOAD(GPU_COMMAND_CODE_NOP, 0) ++ ++/* Stop all external bus interfaces, and then reset the entire GPU. */ ++#define GPU_COMMAND_SOFT_RESET \ ++ GPU_COMMAND_CODE_PAYLOAD(GPU_COMMAND_CODE_RESET, GPU_COMMAND_RESET_PAYLOAD_SOFT_RESET) ++ ++/* Immediately reset the entire GPU. */ ++#define GPU_COMMAND_HARD_RESET \ ++ GPU_COMMAND_CODE_PAYLOAD(GPU_COMMAND_CODE_RESET, GPU_COMMAND_RESET_PAYLOAD_HARD_RESET) ++ ++/* Clear all performance counters, setting them all to zero. */ ++#define GPU_COMMAND_PRFCNT_CLEAR \ ++ GPU_COMMAND_CODE_PAYLOAD(GPU_COMMAND_CODE_PRFCNT, GPU_COMMAND_PRFCNT_PAYLOAD_CLEAR) ++ ++/* Sample all performance counters, writing them out to memory */ ++#define GPU_COMMAND_PRFCNT_SAMPLE \ ++ GPU_COMMAND_CODE_PAYLOAD(GPU_COMMAND_CODE_PRFCNT, GPU_COMMAND_PRFCNT_PAYLOAD_SAMPLE) ++ ++/* Starts the cycle counter, and system timestamp propagation */ ++#define GPU_COMMAND_CYCLE_COUNT_START \ ++ GPU_COMMAND_CODE_PAYLOAD(GPU_COMMAND_CODE_TIME, GPU_COMMAND_TIME_ENABLE) ++ ++/* Stops the cycle counter, and system timestamp propagation */ ++#define GPU_COMMAND_CYCLE_COUNT_STOP \ ++ GPU_COMMAND_CODE_PAYLOAD(GPU_COMMAND_CODE_TIME, GPU_COMMAND_TIME_DISABLE) ++ ++/* Clean all caches */ ++#define GPU_COMMAND_CLEAN_CACHES \ ++ GPU_COMMAND_CODE_PAYLOAD(GPU_COMMAND_CODE_FLUSH_CACHES, GPU_COMMAND_FLUSH_PAYLOAD_CLEAN) ++ ++/* Clean and invalidate all caches */ ++#define GPU_COMMAND_CLEAN_INV_CACHES \ ++ GPU_COMMAND_CODE_PAYLOAD(GPU_COMMAND_CODE_FLUSH_CACHES, GPU_COMMAND_FLUSH_PAYLOAD_CLEAN_INVALIDATE) ++ ++/* Places the GPU in protected mode */ ++#define GPU_COMMAND_SET_PROTECTED_MODE \ ++ GPU_COMMAND_CODE_PAYLOAD(GPU_COMMAND_CODE_SET_PROTECTED_MODE, 0) ++ ++/* Halt CSF */ ++#define GPU_COMMAND_FINISH_HALT \ ++ GPU_COMMAND_CODE_PAYLOAD(GPU_COMMAND_CODE_FINISH_HALT, 0) ++ ++/* Clear GPU faults */ ++#define GPU_COMMAND_CLEAR_FAULT \ ++ GPU_COMMAND_CODE_PAYLOAD(GPU_COMMAND_CODE_CLEAR_FAULT, 0) ++ ++/* End Command Values */ ++ ++/* GPU_FAULTSTATUS register */ ++#define GPU_FAULTSTATUS_EXCEPTION_TYPE_SHIFT 0 ++#define GPU_FAULTSTATUS_EXCEPTION_TYPE_MASK (0xFFul) ++#define GPU_FAULTSTATUS_EXCEPTION_TYPE_GET(reg_val) \ ++ (((reg_val)&GPU_FAULTSTATUS_EXCEPTION_TYPE_MASK) \ ++ >> GPU_FAULTSTATUS_EXCEPTION_TYPE_SHIFT) ++#define GPU_FAULTSTATUS_ACCESS_TYPE_SHIFT 8 ++#define GPU_FAULTSTATUS_ACCESS_TYPE_MASK \ ++ (0x3ul << GPU_FAULTSTATUS_ACCESS_TYPE_SHIFT) ++ ++#define GPU_FAULTSTATUS_ADDR_VALID_SHIFT 10 ++#define GPU_FAULTSTATUS_ADDR_VALID_FLAG \ ++ (1ul << GPU_FAULTSTATUS_ADDR_VALID_SHIFT) ++ ++#define GPU_FAULTSTATUS_JASID_VALID_SHIFT 11 ++#define GPU_FAULTSTATUS_JASID_VALID_FLAG \ ++ (1ul << GPU_FAULTSTATUS_JASID_VALID_SHIFT) ++ ++#define GPU_FAULTSTATUS_JASID_SHIFT 12 ++#define GPU_FAULTSTATUS_JASID_MASK (0xF << GPU_FAULTSTATUS_JASID_SHIFT) ++#define GPU_FAULTSTATUS_JASID_GET(reg_val) \ ++ (((reg_val)&GPU_FAULTSTATUS_JASID_MASK) >> GPU_FAULTSTATUS_JASID_SHIFT) ++#define GPU_FAULTSTATUS_JASID_SET(reg_val, value) \ ++ (((reg_val) & ~GPU_FAULTSTATUS_JASID_MASK) | \ ++ (((value) << GPU_FAULTSTATUS_JASID_SHIFT) & GPU_FAULTSTATUS_JASID_MASK)) ++ ++#define GPU_FAULTSTATUS_SOURCE_ID_SHIFT 16 ++#define GPU_FAULTSTATUS_SOURCE_ID_MASK \ ++ (0xFFFFul << GPU_FAULTSTATUS_SOURCE_ID_SHIFT) ++/* End GPU_FAULTSTATUS register */ ++ ++/* GPU_FAULTSTATUS_ACCESS_TYPE values */ ++#define GPU_FAULTSTATUS_ACCESS_TYPE_ATOMIC 0x0 ++#define GPU_FAULTSTATUS_ACCESS_TYPE_EXECUTE 0x1 ++#define GPU_FAULTSTATUS_ACCESS_TYPE_READ 0x2 ++#define GPU_FAULTSTATUS_ACCESS_TYPE_WRITE 0x3 ++/* End of GPU_FAULTSTATUS_ACCESS_TYPE values */ ++ ++/* TODO: Remove once 10.x.6 headers became available */ ++#define GPU_EXCEPTION_TYPE_SW_FAULT_0 ((u8)0x70) ++#define GPU_EXCEPTION_TYPE_SW_FAULT_1 ((u8)0x71) ++ ++/* GPU_FAULTSTATUS_EXCEPTION_TYPE values */ ++#define GPU_FAULTSTATUS_EXCEPTION_TYPE_OK 0x00 ++#define GPU_FAULTSTATUS_EXCEPTION_TYPE_GPU_BUS_FAULT 0x80 ++#define GPU_FAULTSTATUS_EXCEPTION_TYPE_GPU_SHAREABILITY_FAULT 0x88 ++#define GPU_FAULTSTATUS_EXCEPTION_TYPE_SYSTEM_SHAREABILITY_FAULT 0x89 ++#define GPU_FAULTSTATUS_EXCEPTION_TYPE_GPU_CACHEABILITY_FAULT 0x8A ++/* End of GPU_FAULTSTATUS_EXCEPTION_TYPE values */ ++ ++#define GPU_FAULTSTATUS_ADDRESS_VALID_SHIFT GPU_U(10) ++#define GPU_FAULTSTATUS_ADDRESS_VALID_MASK (GPU_U(0x1) << GPU_FAULTSTATUS_ADDRESS_VALID_SHIFT) ++#define GPU_FAULTSTATUS_ADDRESS_VALID_GET(reg_val) \ ++ (((reg_val)&GPU_FAULTSTATUS_ADDRESS_VALID_MASK) >> GPU_FAULTSTATUS_ADDRESS_VALID_SHIFT) ++#define GPU_FAULTSTATUS_ADDRESS_VALID_SET(reg_val, value) \ ++ (((reg_val) & ~GPU_FAULTSTATUS_ADDRESS_VALID_MASK) | \ ++ (((value) << GPU_FAULTSTATUS_ADDRESS_VALID_SHIFT) & GPU_FAULTSTATUS_ADDRESS_VALID_MASK)) ++ ++/* IRQ flags */ ++#define GPU_FAULT (1 << 0) /* A GPU Fault has occurred */ ++#define GPU_PROTECTED_FAULT (1 << 1) /* A GPU fault has occurred in protected mode */ ++#define RESET_COMPLETED (1 << 8) /* Set when a reset has completed. */ ++#define POWER_CHANGED_SINGLE (1 << 9) /* Set when a single core has finished powering up or down. */ ++#define POWER_CHANGED_ALL (1 << 10) /* Set when all cores have finished powering up or down. */ ++#define CLEAN_CACHES_COMPLETED (1 << 17) /* Set when a cache clean operation has completed. */ ++#define DOORBELL_MIRROR (1 << 18) /* Mirrors the doorbell interrupt line to the CPU */ ++#define MCU_STATUS_GPU_IRQ (1 << 19) /* MCU requires attention */ ++ ++/* ++ * In Debug build, ++ * GPU_IRQ_REG_COMMON | POWER_CHANGED_SINGLE is used to clear and unmask interupts sources of GPU_IRQ ++ * by writing it onto GPU_IRQ_CLEAR/MASK registers. ++ * ++ * In Release build, ++ * GPU_IRQ_REG_COMMON is used. ++ * ++ * Note: ++ * CLEAN_CACHES_COMPLETED - Used separately for cache operation. ++ * DOORBELL_MIRROR - Do not have it included for GPU_IRQ_REG_COMMON ++ * as it can't be cleared by GPU_IRQ_CLEAR, thus interrupt storm might happen ++ */ ++#define GPU_IRQ_REG_COMMON (GPU_FAULT | GPU_PROTECTED_FAULT | RESET_COMPLETED \ ++ | POWER_CHANGED_ALL | MCU_STATUS_GPU_IRQ) ++ ++/* GPU_CONTROL_MCU.GPU_IRQ_RAWSTAT */ ++#define PRFCNT_SAMPLE_COMPLETED (1 << 16) /* Set when performance count sample has completed */ ++ ++#endif /* _KBASE_GPU_REGMAP_CSF_H_ */ +diff --git a/drivers/gpu/arm/bifrost/gpu/backend/mali_kbase_gpu_regmap_jm.h b/drivers/gpu/arm/bifrost/gpu/backend/mali_kbase_gpu_regmap_jm.h +new file mode 100755 +index 000000000000..c9c2fbd49058 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/gpu/backend/mali_kbase_gpu_regmap_jm.h +@@ -0,0 +1,288 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_GPU_REGMAP_JM_H_ ++#define _KBASE_GPU_REGMAP_JM_H_ ++ ++#if MALI_USE_CSF ++#error "Cannot be compiled with CSF" ++#endif ++ ++/* Set to implementation defined, outer caching */ ++#define AS_MEMATTR_AARCH64_OUTER_IMPL_DEF 0x88ull ++/* Set to write back memory, outer caching */ ++#define AS_MEMATTR_AARCH64_OUTER_WA 0x8Dull ++/* Set to inner non-cacheable, outer-non-cacheable ++ * Setting defined by the alloc bits is ignored, but set to a valid encoding: ++ * - no-alloc on read ++ * - no alloc on write ++ */ ++#define AS_MEMATTR_AARCH64_NON_CACHEABLE 0x4Cull ++ ++/* Symbols for default MEMATTR to use ++ * Default is - HW implementation defined caching ++ */ ++#define AS_MEMATTR_INDEX_DEFAULT 0 ++#define AS_MEMATTR_INDEX_DEFAULT_ACE 3 ++ ++/* HW implementation defined caching */ ++#define AS_MEMATTR_INDEX_IMPL_DEF_CACHE_POLICY 0 ++/* Force cache on */ ++#define AS_MEMATTR_INDEX_FORCE_TO_CACHE_ALL 1 ++/* Write-alloc */ ++#define AS_MEMATTR_INDEX_WRITE_ALLOC 2 ++/* Outer coherent, inner implementation defined policy */ ++#define AS_MEMATTR_INDEX_OUTER_IMPL_DEF 3 ++/* Outer coherent, write alloc inner */ ++#define AS_MEMATTR_INDEX_OUTER_WA 4 ++/* Normal memory, inner non-cacheable, outer non-cacheable (ARMv8 mode only) */ ++#define AS_MEMATTR_INDEX_NON_CACHEABLE 5 ++ ++/* GPU control registers */ ++ ++#define CORE_FEATURES 0x008 /* (RO) Shader Core Features */ ++#define JS_PRESENT 0x01C /* (RO) Job slots present */ ++#define LATEST_FLUSH 0x038 /* (RO) Flush ID of latest ++ * clean-and-invalidate operation ++ */ ++ ++#define PRFCNT_BASE_LO 0x060 /* (RW) Performance counter memory ++ * region base address, low word ++ */ ++#define PRFCNT_BASE_HI 0x064 /* (RW) Performance counter memory ++ * region base address, high word ++ */ ++#define PRFCNT_CONFIG 0x068 /* (RW) Performance counter ++ * configuration ++ */ ++#define PRFCNT_JM_EN 0x06C /* (RW) Performance counter enable ++ * flags for Job Manager ++ */ ++#define PRFCNT_SHADER_EN 0x070 /* (RW) Performance counter enable ++ * flags for shader cores ++ */ ++#define PRFCNT_TILER_EN 0x074 /* (RW) Performance counter enable ++ * flags for tiler ++ */ ++#define PRFCNT_MMU_L2_EN 0x07C /* (RW) Performance counter enable ++ * flags for MMU/L2 cache ++ */ ++ ++#define JS0_FEATURES 0x0C0 /* (RO) Features of job slot 0 */ ++#define JS1_FEATURES 0x0C4 /* (RO) Features of job slot 1 */ ++#define JS2_FEATURES 0x0C8 /* (RO) Features of job slot 2 */ ++#define JS3_FEATURES 0x0CC /* (RO) Features of job slot 3 */ ++#define JS4_FEATURES 0x0D0 /* (RO) Features of job slot 4 */ ++#define JS5_FEATURES 0x0D4 /* (RO) Features of job slot 5 */ ++#define JS6_FEATURES 0x0D8 /* (RO) Features of job slot 6 */ ++#define JS7_FEATURES 0x0DC /* (RO) Features of job slot 7 */ ++#define JS8_FEATURES 0x0E0 /* (RO) Features of job slot 8 */ ++#define JS9_FEATURES 0x0E4 /* (RO) Features of job slot 9 */ ++#define JS10_FEATURES 0x0E8 /* (RO) Features of job slot 10 */ ++#define JS11_FEATURES 0x0EC /* (RO) Features of job slot 11 */ ++#define JS12_FEATURES 0x0F0 /* (RO) Features of job slot 12 */ ++#define JS13_FEATURES 0x0F4 /* (RO) Features of job slot 13 */ ++#define JS14_FEATURES 0x0F8 /* (RO) Features of job slot 14 */ ++#define JS15_FEATURES 0x0FC /* (RO) Features of job slot 15 */ ++ ++#define JS_FEATURES_REG(n) GPU_CONTROL_REG(JS0_FEATURES + ((n) << 2)) ++ ++#define JM_CONFIG 0xF00 /* (RW) Job manager configuration (implementation-specific) */ ++ ++/* Job control registers */ ++ ++#define JOB_IRQ_JS_STATE 0x010 /* status==active and _next == busy snapshot from last JOB_IRQ_CLEAR */ ++#define JOB_IRQ_THROTTLE 0x014 /* cycles to delay delivering an interrupt externally. The JOB_IRQ_STATUS is NOT affected by this, just the delivery of the interrupt. */ ++ ++#define JOB_SLOT0 0x800 /* Configuration registers for job slot 0 */ ++#define JOB_SLOT1 0x880 /* Configuration registers for job slot 1 */ ++#define JOB_SLOT2 0x900 /* Configuration registers for job slot 2 */ ++#define JOB_SLOT3 0x980 /* Configuration registers for job slot 3 */ ++#define JOB_SLOT4 0xA00 /* Configuration registers for job slot 4 */ ++#define JOB_SLOT5 0xA80 /* Configuration registers for job slot 5 */ ++#define JOB_SLOT6 0xB00 /* Configuration registers for job slot 6 */ ++#define JOB_SLOT7 0xB80 /* Configuration registers for job slot 7 */ ++#define JOB_SLOT8 0xC00 /* Configuration registers for job slot 8 */ ++#define JOB_SLOT9 0xC80 /* Configuration registers for job slot 9 */ ++#define JOB_SLOT10 0xD00 /* Configuration registers for job slot 10 */ ++#define JOB_SLOT11 0xD80 /* Configuration registers for job slot 11 */ ++#define JOB_SLOT12 0xE00 /* Configuration registers for job slot 12 */ ++#define JOB_SLOT13 0xE80 /* Configuration registers for job slot 13 */ ++#define JOB_SLOT14 0xF00 /* Configuration registers for job slot 14 */ ++#define JOB_SLOT15 0xF80 /* Configuration registers for job slot 15 */ ++ ++#define JOB_SLOT_REG(n, r) (JOB_CONTROL_REG(JOB_SLOT0 + ((n) << 7)) + (r)) ++ ++#define JS_HEAD_LO 0x00 /* (RO) Job queue head pointer for job slot n, low word */ ++#define JS_HEAD_HI 0x04 /* (RO) Job queue head pointer for job slot n, high word */ ++#define JS_TAIL_LO 0x08 /* (RO) Job queue tail pointer for job slot n, low word */ ++#define JS_TAIL_HI 0x0C /* (RO) Job queue tail pointer for job slot n, high word */ ++#define JS_AFFINITY_LO 0x10 /* (RO) Core affinity mask for job slot n, low word */ ++#define JS_AFFINITY_HI 0x14 /* (RO) Core affinity mask for job slot n, high word */ ++#define JS_CONFIG 0x18 /* (RO) Configuration settings for job slot n */ ++#define JS_XAFFINITY 0x1C /* (RO) Extended affinity mask for job ++ slot n */ ++ ++#define JS_COMMAND 0x20 /* (WO) Command register for job slot n */ ++#define JS_STATUS 0x24 /* (RO) Status register for job slot n */ ++ ++#define JS_HEAD_NEXT_LO 0x40 /* (RW) Next job queue head pointer for job slot n, low word */ ++#define JS_HEAD_NEXT_HI 0x44 /* (RW) Next job queue head pointer for job slot n, high word */ ++ ++#define JS_AFFINITY_NEXT_LO 0x50 /* (RW) Next core affinity mask for job slot n, low word */ ++#define JS_AFFINITY_NEXT_HI 0x54 /* (RW) Next core affinity mask for job slot n, high word */ ++#define JS_CONFIG_NEXT 0x58 /* (RW) Next configuration settings for job slot n */ ++#define JS_XAFFINITY_NEXT 0x5C /* (RW) Next extended affinity mask for ++ job slot n */ ++ ++#define JS_COMMAND_NEXT 0x60 /* (RW) Next command register for job slot n */ ++ ++#define JS_FLUSH_ID_NEXT 0x70 /* (RW) Next job slot n cache flush ID */ ++ ++/* No JM-specific MMU control registers */ ++/* No JM-specific MMU address space control registers */ ++ ++/* JS_COMMAND register commands */ ++#define JS_COMMAND_NOP 0x00 /* NOP Operation. Writing this value is ignored */ ++#define JS_COMMAND_START 0x01 /* Start processing a job chain. Writing this value is ignored */ ++#define JS_COMMAND_SOFT_STOP 0x02 /* Gently stop processing a job chain */ ++#define JS_COMMAND_HARD_STOP 0x03 /* Rudely stop processing a job chain */ ++#define JS_COMMAND_SOFT_STOP_0 0x04 /* Execute SOFT_STOP if JOB_CHAIN_FLAG is 0 */ ++#define JS_COMMAND_HARD_STOP_0 0x05 /* Execute HARD_STOP if JOB_CHAIN_FLAG is 0 */ ++#define JS_COMMAND_SOFT_STOP_1 0x06 /* Execute SOFT_STOP if JOB_CHAIN_FLAG is 1 */ ++#define JS_COMMAND_HARD_STOP_1 0x07 /* Execute HARD_STOP if JOB_CHAIN_FLAG is 1 */ ++ ++#define JS_COMMAND_MASK 0x07 /* Mask of bits currently in use by the HW */ ++ ++/* Possible values of JS_CONFIG and JS_CONFIG_NEXT registers */ ++#define JS_CONFIG_START_FLUSH_NO_ACTION (0u << 0) ++#define JS_CONFIG_START_FLUSH_CLEAN (1u << 8) ++#define JS_CONFIG_START_FLUSH_CLEAN_INVALIDATE (3u << 8) ++#define JS_CONFIG_START_MMU (1u << 10) ++#define JS_CONFIG_JOB_CHAIN_FLAG (1u << 11) ++#define JS_CONFIG_END_FLUSH_NO_ACTION JS_CONFIG_START_FLUSH_NO_ACTION ++#define JS_CONFIG_END_FLUSH_CLEAN (1u << 12) ++#define JS_CONFIG_END_FLUSH_CLEAN_INVALIDATE (3u << 12) ++#define JS_CONFIG_ENABLE_FLUSH_REDUCTION (1u << 14) ++#define JS_CONFIG_DISABLE_DESCRIPTOR_WR_BK (1u << 15) ++#define JS_CONFIG_THREAD_PRI(n) ((n) << 16) ++ ++/* JS_XAFFINITY register values */ ++#define JS_XAFFINITY_XAFFINITY_ENABLE (1u << 0) ++#define JS_XAFFINITY_TILER_ENABLE (1u << 8) ++#define JS_XAFFINITY_CACHE_ENABLE (1u << 16) ++ ++/* JS_STATUS register values */ ++ ++/* NOTE: Please keep this values in sync with enum base_jd_event_code in mali_base_kernel.h. ++ * The values are separated to avoid dependency of userspace and kernel code. ++ */ ++ ++/* Group of values representing the job status instead of a particular fault */ ++#define JS_STATUS_NO_EXCEPTION_BASE 0x00 ++#define JS_STATUS_INTERRUPTED (JS_STATUS_NO_EXCEPTION_BASE + 0x02) /* 0x02 means INTERRUPTED */ ++#define JS_STATUS_STOPPED (JS_STATUS_NO_EXCEPTION_BASE + 0x03) /* 0x03 means STOPPED */ ++#define JS_STATUS_TERMINATED (JS_STATUS_NO_EXCEPTION_BASE + 0x04) /* 0x04 means TERMINATED */ ++ ++/* General fault values */ ++#define JS_STATUS_FAULT_BASE 0x40 ++#define JS_STATUS_CONFIG_FAULT (JS_STATUS_FAULT_BASE) /* 0x40 means CONFIG FAULT */ ++#define JS_STATUS_POWER_FAULT (JS_STATUS_FAULT_BASE + 0x01) /* 0x41 means POWER FAULT */ ++#define JS_STATUS_READ_FAULT (JS_STATUS_FAULT_BASE + 0x02) /* 0x42 means READ FAULT */ ++#define JS_STATUS_WRITE_FAULT (JS_STATUS_FAULT_BASE + 0x03) /* 0x43 means WRITE FAULT */ ++#define JS_STATUS_AFFINITY_FAULT (JS_STATUS_FAULT_BASE + 0x04) /* 0x44 means AFFINITY FAULT */ ++#define JS_STATUS_BUS_FAULT (JS_STATUS_FAULT_BASE + 0x08) /* 0x48 means BUS FAULT */ ++ ++/* Instruction or data faults */ ++#define JS_STATUS_INSTRUCTION_FAULT_BASE 0x50 ++#define JS_STATUS_INSTR_INVALID_PC (JS_STATUS_INSTRUCTION_FAULT_BASE) /* 0x50 means INSTR INVALID PC */ ++#define JS_STATUS_INSTR_INVALID_ENC (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x01) /* 0x51 means INSTR INVALID ENC */ ++#define JS_STATUS_INSTR_TYPE_MISMATCH (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x02) /* 0x52 means INSTR TYPE MISMATCH */ ++#define JS_STATUS_INSTR_OPERAND_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x03) /* 0x53 means INSTR OPERAND FAULT */ ++#define JS_STATUS_INSTR_TLS_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x04) /* 0x54 means INSTR TLS FAULT */ ++#define JS_STATUS_INSTR_BARRIER_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x05) /* 0x55 means INSTR BARRIER FAULT */ ++#define JS_STATUS_INSTR_ALIGN_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x06) /* 0x56 means INSTR ALIGN FAULT */ ++/* NOTE: No fault with 0x57 code defined in spec. */ ++#define JS_STATUS_DATA_INVALID_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x08) /* 0x58 means DATA INVALID FAULT */ ++#define JS_STATUS_TILE_RANGE_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x09) /* 0x59 means TILE RANGE FAULT */ ++#define JS_STATUS_ADDRESS_RANGE_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x0A) /* 0x5A means ADDRESS RANGE FAULT */ ++ ++/* Other faults */ ++#define JS_STATUS_MEMORY_FAULT_BASE 0x60 ++#define JS_STATUS_OUT_OF_MEMORY (JS_STATUS_MEMORY_FAULT_BASE) /* 0x60 means OUT OF MEMORY */ ++#define JS_STATUS_UNKNOWN 0x7F /* 0x7F means UNKNOWN */ ++ ++/* JS_FEATURES register */ ++#define JS_FEATURE_NULL_JOB (1u << 1) ++#define JS_FEATURE_SET_VALUE_JOB (1u << 2) ++#define JS_FEATURE_CACHE_FLUSH_JOB (1u << 3) ++#define JS_FEATURE_COMPUTE_JOB (1u << 4) ++#define JS_FEATURE_VERTEX_JOB (1u << 5) ++#define JS_FEATURE_GEOMETRY_JOB (1u << 6) ++#define JS_FEATURE_TILER_JOB (1u << 7) ++#define JS_FEATURE_FUSED_JOB (1u << 8) ++#define JS_FEATURE_FRAGMENT_JOB (1u << 9) ++ ++/* JM_CONFIG register */ ++#define JM_TIMESTAMP_OVERRIDE (1ul << 0) ++#define JM_CLOCK_GATE_OVERRIDE (1ul << 1) ++#define JM_JOB_THROTTLE_ENABLE (1ul << 2) ++#define JM_JOB_THROTTLE_LIMIT_SHIFT (3) ++#define JM_MAX_JOB_THROTTLE_LIMIT (0x3F) ++#define JM_FORCE_COHERENCY_FEATURES_SHIFT (2) ++ ++/* GPU_COMMAND values */ ++#define GPU_COMMAND_NOP 0x00 /* No operation, nothing happens */ ++#define GPU_COMMAND_SOFT_RESET 0x01 /* Stop all external bus interfaces, and then reset the entire GPU. */ ++#define GPU_COMMAND_HARD_RESET 0x02 /* Immediately reset the entire GPU. */ ++#define GPU_COMMAND_PRFCNT_CLEAR 0x03 /* Clear all performance counters, setting them all to zero. */ ++#define GPU_COMMAND_PRFCNT_SAMPLE 0x04 /* Sample all performance counters, writing them out to memory */ ++#define GPU_COMMAND_CYCLE_COUNT_START 0x05 /* Starts the cycle counter, and system timestamp propagation */ ++#define GPU_COMMAND_CYCLE_COUNT_STOP 0x06 /* Stops the cycle counter, and system timestamp propagation */ ++#define GPU_COMMAND_CLEAN_CACHES 0x07 /* Clean all caches */ ++#define GPU_COMMAND_CLEAN_INV_CACHES 0x08 /* Clean and invalidate all caches */ ++#define GPU_COMMAND_SET_PROTECTED_MODE 0x09 /* Places the GPU in protected mode */ ++ ++/* IRQ flags */ ++#define GPU_FAULT (1 << 0) /* A GPU Fault has occurred */ ++#define MULTIPLE_GPU_FAULTS (1 << 7) /* More than one GPU Fault occurred. */ ++#define RESET_COMPLETED (1 << 8) /* Set when a reset has completed. */ ++#define POWER_CHANGED_SINGLE (1 << 9) /* Set when a single core has finished powering up or down. */ ++#define POWER_CHANGED_ALL (1 << 10) /* Set when all cores have finished powering up or down. */ ++#define PRFCNT_SAMPLE_COMPLETED (1 << 16) /* Set when a performance count sample has completed. */ ++#define CLEAN_CACHES_COMPLETED (1 << 17) /* Set when a cache clean operation has completed. */ ++ ++/* ++ * In Debug build, ++ * GPU_IRQ_REG_COMMON | POWER_CHANGED_SINGLE is used to clear and enable interupts sources of GPU_IRQ ++ * by writing it onto GPU_IRQ_CLEAR/MASK registers. ++ * ++ * In Release build, ++ * GPU_IRQ_REG_COMMON is used. ++ * ++ * Note: ++ * CLEAN_CACHES_COMPLETED - Used separately for cache operation. ++ */ ++#define GPU_IRQ_REG_COMMON (GPU_FAULT | MULTIPLE_GPU_FAULTS | RESET_COMPLETED \ ++ | POWER_CHANGED_ALL | PRFCNT_SAMPLE_COMPLETED) ++ ++#endif /* _KBASE_GPU_REGMAP_JM_H_ */ +diff --git a/drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu.c b/drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu.c +new file mode 100755 +index 000000000000..3128db4cabfc +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu.c +@@ -0,0 +1,41 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++#include ++ ++const char *kbase_gpu_access_type_name(u32 fault_status) ++{ ++ switch (AS_FAULTSTATUS_ACCESS_TYPE_GET(fault_status)) { ++ case AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC: ++ return "ATOMIC"; ++ case AS_FAULTSTATUS_ACCESS_TYPE_READ: ++ return "READ"; ++ case AS_FAULTSTATUS_ACCESS_TYPE_WRITE: ++ return "WRITE"; ++ case AS_FAULTSTATUS_ACCESS_TYPE_EX: ++ return "EXECUTE"; ++ default: ++ WARN_ON(1); ++ return NULL; ++ } ++} +diff --git a/drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu.h b/drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu.h +new file mode 100755 +index 000000000000..9516e56eda01 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu.h +@@ -0,0 +1,31 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_GPU_H_ ++#define _KBASE_GPU_H_ ++ ++#include "mali_kbase_gpu_regmap.h" ++#include "mali_kbase_gpu_fault.h" ++#include "mali_kbase_gpu_coherency.h" ++#include "mali_kbase_gpu_id.h" ++ ++#endif /* _KBASE_GPU_H_ */ +diff --git a/drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu_coherency.h b/drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu_coherency.h +new file mode 100755 +index 000000000000..bb2b1613aa47 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu_coherency.h +@@ -0,0 +1,31 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_GPU_COHERENCY_H_ ++#define _KBASE_GPU_COHERENCY_H_ ++ ++#define COHERENCY_ACE_LITE 0 ++#define COHERENCY_ACE 1 ++#define COHERENCY_NONE 31 ++#define COHERENCY_FEATURE_BIT(x) (1 << (x)) ++ ++#endif /* _KBASE_GPU_COHERENCY_H_ */ +diff --git a/drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu_fault.h b/drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu_fault.h +new file mode 100755 +index 000000000000..e63c3881a3ca +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu_fault.h +@@ -0,0 +1,48 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_GPU_FAULT_H_ ++#define _KBASE_GPU_FAULT_H_ ++ ++/** Returns the name associated with a Mali exception code ++ * ++ * @exception_code: exception code ++ * ++ * This function is called from the interrupt handler when a GPU fault occurs. ++ * ++ * Return: name associated with the exception code ++ */ ++const char *kbase_gpu_exception_name(u32 exception_code); ++ ++/** ++ * kbase_gpu_access_type_name - Convert MMU_AS_CONTROL.FAULTSTATUS.ACCESS_TYPE ++ * into string. ++ * @fault_status: value of FAULTSTATUS register. ++ * ++ * After MMU fault, this function can be used to get readable information about ++ * access_type of the MMU fault. ++ * ++ * Return: String of the access type. ++ */ ++const char *kbase_gpu_access_type_name(u32 fault_status); ++ ++#endif /* _KBASE_GPU_FAULT_H_ */ +diff --git a/drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu_id.h b/drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu_id.h +new file mode 100755 +index 000000000000..31d55264c67f +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu_id.h +@@ -0,0 +1,119 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_GPU_ID_H_ ++#define _KBASE_GPU_ID_H_ ++ ++/* GPU_ID register */ ++#define GPU_ID_VERSION_STATUS_SHIFT 0 ++#define GPU_ID_VERSION_MINOR_SHIFT 4 ++#define GPU_ID_VERSION_MAJOR_SHIFT 12 ++#define GPU_ID_VERSION_PRODUCT_ID_SHIFT 16 ++#define GPU_ID_VERSION_STATUS (0xFu << GPU_ID_VERSION_STATUS_SHIFT) ++#define GPU_ID_VERSION_MINOR (0xFFu << GPU_ID_VERSION_MINOR_SHIFT) ++#define GPU_ID_VERSION_MAJOR (0xFu << GPU_ID_VERSION_MAJOR_SHIFT) ++#define GPU_ID_VERSION_PRODUCT_ID (0xFFFFu << GPU_ID_VERSION_PRODUCT_ID_SHIFT) ++ ++#define GPU_ID2_VERSION_STATUS_SHIFT 0 ++#define GPU_ID2_VERSION_MINOR_SHIFT 4 ++#define GPU_ID2_VERSION_MAJOR_SHIFT 12 ++#define GPU_ID2_PRODUCT_MAJOR_SHIFT 16 ++#define GPU_ID2_ARCH_REV_SHIFT 20 ++#define GPU_ID2_ARCH_MINOR_SHIFT 24 ++#define GPU_ID2_ARCH_MAJOR_SHIFT 28 ++#define GPU_ID2_VERSION_STATUS (0xFu << GPU_ID2_VERSION_STATUS_SHIFT) ++#define GPU_ID2_VERSION_MINOR (0xFFu << GPU_ID2_VERSION_MINOR_SHIFT) ++#define GPU_ID2_VERSION_MAJOR (0xFu << GPU_ID2_VERSION_MAJOR_SHIFT) ++#define GPU_ID2_PRODUCT_MAJOR (0xFu << GPU_ID2_PRODUCT_MAJOR_SHIFT) ++#define GPU_ID2_ARCH_REV (0xFu << GPU_ID2_ARCH_REV_SHIFT) ++#define GPU_ID2_ARCH_MINOR (0xFu << GPU_ID2_ARCH_MINOR_SHIFT) ++#define GPU_ID2_ARCH_MAJOR (0xFu << GPU_ID2_ARCH_MAJOR_SHIFT) ++#define GPU_ID2_PRODUCT_MODEL (GPU_ID2_ARCH_MAJOR | GPU_ID2_PRODUCT_MAJOR) ++#define GPU_ID2_VERSION (GPU_ID2_VERSION_MAJOR | \ ++ GPU_ID2_VERSION_MINOR | \ ++ GPU_ID2_VERSION_STATUS) ++ ++/* Helper macro to create a partial GPU_ID (new format) that defines ++ a product ignoring its version. */ ++#define GPU_ID2_PRODUCT_MAKE(arch_major, arch_minor, arch_rev, product_major) \ ++ ((((u32)arch_major) << GPU_ID2_ARCH_MAJOR_SHIFT) | \ ++ (((u32)arch_minor) << GPU_ID2_ARCH_MINOR_SHIFT) | \ ++ (((u32)arch_rev) << GPU_ID2_ARCH_REV_SHIFT) | \ ++ (((u32)product_major) << GPU_ID2_PRODUCT_MAJOR_SHIFT)) ++ ++/* Helper macro to create a partial GPU_ID (new format) that specifies the ++ revision (major, minor, status) of a product */ ++#define GPU_ID2_VERSION_MAKE(version_major, version_minor, version_status) \ ++ ((((u32)version_major) << GPU_ID2_VERSION_MAJOR_SHIFT) | \ ++ (((u32)version_minor) << GPU_ID2_VERSION_MINOR_SHIFT) | \ ++ (((u32)version_status) << GPU_ID2_VERSION_STATUS_SHIFT)) ++ ++/* Helper macro to create a complete GPU_ID (new format) */ ++#define GPU_ID2_MAKE(arch_major, arch_minor, arch_rev, product_major, \ ++ version_major, version_minor, version_status) \ ++ (GPU_ID2_PRODUCT_MAKE(arch_major, arch_minor, arch_rev, \ ++ product_major) | \ ++ GPU_ID2_VERSION_MAKE(version_major, version_minor, \ ++ version_status)) ++ ++/* Helper macro to create a partial GPU_ID (new format) that identifies ++ a particular GPU model by its arch_major and product_major. */ ++#define GPU_ID2_MODEL_MAKE(arch_major, product_major) \ ++ ((((u32)arch_major) << GPU_ID2_ARCH_MAJOR_SHIFT) | \ ++ (((u32)product_major) << GPU_ID2_PRODUCT_MAJOR_SHIFT)) ++ ++/* Strip off the non-relevant bits from a product_id value and make it suitable ++ for comparison against the GPU_ID2_PRODUCT_xxx values which identify a GPU ++ model. */ ++#define GPU_ID2_MODEL_MATCH_VALUE(product_id) \ ++ ((((u32)product_id) << GPU_ID2_PRODUCT_MAJOR_SHIFT) & \ ++ GPU_ID2_PRODUCT_MODEL) ++ ++#define GPU_ID2_PRODUCT_TMIX GPU_ID2_MODEL_MAKE(6, 0) ++#define GPU_ID2_PRODUCT_THEX GPU_ID2_MODEL_MAKE(6, 1) ++#define GPU_ID2_PRODUCT_TSIX GPU_ID2_MODEL_MAKE(7, 0) ++#define GPU_ID2_PRODUCT_TDVX GPU_ID2_MODEL_MAKE(7, 3) ++#define GPU_ID2_PRODUCT_TNOX GPU_ID2_MODEL_MAKE(7, 1) ++#define GPU_ID2_PRODUCT_TGOX GPU_ID2_MODEL_MAKE(7, 2) ++#define GPU_ID2_PRODUCT_TTRX GPU_ID2_MODEL_MAKE(9, 0) ++#define GPU_ID2_PRODUCT_TNAX GPU_ID2_MODEL_MAKE(9, 1) ++#define GPU_ID2_PRODUCT_TBEX GPU_ID2_MODEL_MAKE(9, 2) ++#define GPU_ID2_PRODUCT_LBEX GPU_ID2_MODEL_MAKE(9, 4) ++#define GPU_ID2_PRODUCT_TBAX GPU_ID2_MODEL_MAKE(9, 5) ++#define GPU_ID2_PRODUCT_TDUX GPU_ID2_MODEL_MAKE(10, 1) ++#define GPU_ID2_PRODUCT_TODX GPU_ID2_MODEL_MAKE(10, 2) ++#define GPU_ID2_PRODUCT_TGRX GPU_ID2_MODEL_MAKE(10, 3) ++#define GPU_ID2_PRODUCT_TVAX GPU_ID2_MODEL_MAKE(10, 4) ++#define GPU_ID2_PRODUCT_LODX GPU_ID2_MODEL_MAKE(10, 7) ++#define GPU_ID2_PRODUCT_TTUX GPU_ID2_MODEL_MAKE(11, 2) ++#define GPU_ID2_PRODUCT_LTUX GPU_ID2_MODEL_MAKE(11, 3) ++#define GPU_ID2_PRODUCT_TE2X GPU_ID2_MODEL_MAKE(11, 1) ++ ++/* Helper macro to create a GPU_ID assuming valid values for id, major, ++ minor, status */ ++#define GPU_ID_MAKE(id, major, minor, status) \ ++ ((((u32)id) << GPU_ID_VERSION_PRODUCT_ID_SHIFT) | \ ++ (((u32)major) << GPU_ID_VERSION_MAJOR_SHIFT) | \ ++ (((u32)minor) << GPU_ID_VERSION_MINOR_SHIFT) | \ ++ (((u32)status) << GPU_ID_VERSION_STATUS_SHIFT)) ++ ++#endif /* _KBASE_GPU_ID_H_ */ +diff --git a/drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu_regmap.h b/drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu_regmap.h +new file mode 100755 +index 000000000000..d8066f43768b +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu_regmap.h +@@ -0,0 +1,428 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_GPU_REGMAP_H_ ++#define _KBASE_GPU_REGMAP_H_ ++ ++#include "mali_kbase_gpu_coherency.h" ++#include "mali_kbase_gpu_id.h" ++#if MALI_USE_CSF ++#include "backend/mali_kbase_gpu_regmap_csf.h" ++#else ++#include "backend/mali_kbase_gpu_regmap_jm.h" ++#endif ++ ++/* Begin Register Offsets */ ++/* GPU control registers */ ++ ++#define GPU_CONTROL_BASE 0x0000 ++#define GPU_CONTROL_REG(r) (GPU_CONTROL_BASE + (r)) ++#define GPU_ID 0x000 /* (RO) GPU and revision identifier */ ++#define L2_FEATURES 0x004 /* (RO) Level 2 cache features */ ++#define TILER_FEATURES 0x00C /* (RO) Tiler Features */ ++#define MEM_FEATURES 0x010 /* (RO) Memory system features */ ++#define MMU_FEATURES 0x014 /* (RO) MMU features */ ++#define AS_PRESENT 0x018 /* (RO) Address space slots present */ ++#define GPU_IRQ_RAWSTAT 0x020 /* (RW) */ ++#define GPU_IRQ_CLEAR 0x024 /* (WO) */ ++#define GPU_IRQ_MASK 0x028 /* (RW) */ ++#define GPU_IRQ_STATUS 0x02C /* (RO) */ ++ ++#define GPU_COMMAND 0x030 /* (WO) */ ++#define GPU_STATUS 0x034 /* (RO) */ ++ ++#define GPU_DBGEN (1 << 8) /* DBGEN wire status */ ++ ++#define GPU_FAULTSTATUS 0x03C /* (RO) GPU exception type and fault status */ ++#define GPU_FAULTADDRESS_LO 0x040 /* (RO) GPU exception fault address, low word */ ++#define GPU_FAULTADDRESS_HI 0x044 /* (RO) GPU exception fault address, high word */ ++ ++#define L2_CONFIG 0x048 /* (RW) Level 2 cache configuration */ ++ ++#define GROUPS_L2_COHERENT (1 << 0) /* Cores groups are l2 coherent */ ++#define SUPER_L2_COHERENT (1 << 1) /* Shader cores within a core ++ * supergroup are l2 coherent ++ */ ++ ++#define PWR_KEY 0x050 /* (WO) Power manager key register */ ++#define PWR_OVERRIDE0 0x054 /* (RW) Power manager override settings */ ++#define PWR_OVERRIDE1 0x058 /* (RW) Power manager override settings */ ++ ++#define CYCLE_COUNT_LO 0x090 /* (RO) Cycle counter, low word */ ++#define CYCLE_COUNT_HI 0x094 /* (RO) Cycle counter, high word */ ++#define TIMESTAMP_LO 0x098 /* (RO) Global time stamp counter, low word */ ++#define TIMESTAMP_HI 0x09C /* (RO) Global time stamp counter, high word */ ++ ++#define THREAD_MAX_THREADS 0x0A0 /* (RO) Maximum number of threads per core */ ++#define THREAD_MAX_WORKGROUP_SIZE 0x0A4 /* (RO) Maximum workgroup size */ ++#define THREAD_MAX_BARRIER_SIZE 0x0A8 /* (RO) Maximum threads waiting at a barrier */ ++#define THREAD_FEATURES 0x0AC /* (RO) Thread features */ ++#define THREAD_TLS_ALLOC 0x310 /* (RO) Number of threads per core that TLS must be allocated for */ ++ ++#define TEXTURE_FEATURES_0 0x0B0 /* (RO) Support flags for indexed texture formats 0..31 */ ++#define TEXTURE_FEATURES_1 0x0B4 /* (RO) Support flags for indexed texture formats 32..63 */ ++#define TEXTURE_FEATURES_2 0x0B8 /* (RO) Support flags for indexed texture formats 64..95 */ ++#define TEXTURE_FEATURES_3 0x0BC /* (RO) Support flags for texture order */ ++ ++#define TEXTURE_FEATURES_REG(n) GPU_CONTROL_REG(TEXTURE_FEATURES_0 + ((n) << 2)) ++ ++#define SHADER_PRESENT_LO 0x100 /* (RO) Shader core present bitmap, low word */ ++#define SHADER_PRESENT_HI 0x104 /* (RO) Shader core present bitmap, high word */ ++ ++#define TILER_PRESENT_LO 0x110 /* (RO) Tiler core present bitmap, low word */ ++#define TILER_PRESENT_HI 0x114 /* (RO) Tiler core present bitmap, high word */ ++ ++#define L2_PRESENT_LO 0x120 /* (RO) Level 2 cache present bitmap, low word */ ++#define L2_PRESENT_HI 0x124 /* (RO) Level 2 cache present bitmap, high word */ ++ ++#define STACK_PRESENT_LO 0xE00 /* (RO) Core stack present bitmap, low word */ ++#define STACK_PRESENT_HI 0xE04 /* (RO) Core stack present bitmap, high word */ ++ ++#define SHADER_READY_LO 0x140 /* (RO) Shader core ready bitmap, low word */ ++#define SHADER_READY_HI 0x144 /* (RO) Shader core ready bitmap, high word */ ++ ++#define TILER_READY_LO 0x150 /* (RO) Tiler core ready bitmap, low word */ ++#define TILER_READY_HI 0x154 /* (RO) Tiler core ready bitmap, high word */ ++ ++#define L2_READY_LO 0x160 /* (RO) Level 2 cache ready bitmap, low word */ ++#define L2_READY_HI 0x164 /* (RO) Level 2 cache ready bitmap, high word */ ++ ++#define STACK_READY_LO 0xE10 /* (RO) Core stack ready bitmap, low word */ ++#define STACK_READY_HI 0xE14 /* (RO) Core stack ready bitmap, high word */ ++ ++#define SHADER_PWRON_LO 0x180 /* (WO) Shader core power on bitmap, low word */ ++#define SHADER_PWRON_HI 0x184 /* (WO) Shader core power on bitmap, high word */ ++ ++#define TILER_PWRON_LO 0x190 /* (WO) Tiler core power on bitmap, low word */ ++#define TILER_PWRON_HI 0x194 /* (WO) Tiler core power on bitmap, high word */ ++ ++#define L2_PWRON_LO 0x1A0 /* (WO) Level 2 cache power on bitmap, low word */ ++#define L2_PWRON_HI 0x1A4 /* (WO) Level 2 cache power on bitmap, high word */ ++ ++#define STACK_PWRON_LO 0xE20 /* (RO) Core stack power on bitmap, low word */ ++#define STACK_PWRON_HI 0xE24 /* (RO) Core stack power on bitmap, high word */ ++ ++#define SHADER_PWROFF_LO 0x1C0 /* (WO) Shader core power off bitmap, low word */ ++#define SHADER_PWROFF_HI 0x1C4 /* (WO) Shader core power off bitmap, high word */ ++ ++#define TILER_PWROFF_LO 0x1D0 /* (WO) Tiler core power off bitmap, low word */ ++#define TILER_PWROFF_HI 0x1D4 /* (WO) Tiler core power off bitmap, high word */ ++ ++#define L2_PWROFF_LO 0x1E0 /* (WO) Level 2 cache power off bitmap, low word */ ++#define L2_PWROFF_HI 0x1E4 /* (WO) Level 2 cache power off bitmap, high word */ ++ ++#define STACK_PWROFF_LO 0xE30 /* (RO) Core stack power off bitmap, low word */ ++#define STACK_PWROFF_HI 0xE34 /* (RO) Core stack power off bitmap, high word */ ++ ++#define SHADER_PWRTRANS_LO 0x200 /* (RO) Shader core power transition bitmap, low word */ ++#define SHADER_PWRTRANS_HI 0x204 /* (RO) Shader core power transition bitmap, high word */ ++ ++#define TILER_PWRTRANS_LO 0x210 /* (RO) Tiler core power transition bitmap, low word */ ++#define TILER_PWRTRANS_HI 0x214 /* (RO) Tiler core power transition bitmap, high word */ ++ ++#define L2_PWRTRANS_LO 0x220 /* (RO) Level 2 cache power transition bitmap, low word */ ++#define L2_PWRTRANS_HI 0x224 /* (RO) Level 2 cache power transition bitmap, high word */ ++ ++#define STACK_PWRTRANS_LO 0xE40 /* (RO) Core stack power transition bitmap, low word */ ++#define STACK_PWRTRANS_HI 0xE44 /* (RO) Core stack power transition bitmap, high word */ ++ ++#define SHADER_PWRACTIVE_LO 0x240 /* (RO) Shader core active bitmap, low word */ ++#define SHADER_PWRACTIVE_HI 0x244 /* (RO) Shader core active bitmap, high word */ ++ ++#define TILER_PWRACTIVE_LO 0x250 /* (RO) Tiler core active bitmap, low word */ ++#define TILER_PWRACTIVE_HI 0x254 /* (RO) Tiler core active bitmap, high word */ ++ ++#define L2_PWRACTIVE_LO 0x260 /* (RO) Level 2 cache active bitmap, low word */ ++#define L2_PWRACTIVE_HI 0x264 /* (RO) Level 2 cache active bitmap, high word */ ++ ++#define COHERENCY_FEATURES 0x300 /* (RO) Coherency features present */ ++#define COHERENCY_ENABLE 0x304 /* (RW) Coherency enable */ ++ ++#define SHADER_CONFIG 0xF04 /* (RW) Shader core configuration (implementation-specific) */ ++#define TILER_CONFIG 0xF08 /* (RW) Tiler core configuration (implementation-specific) */ ++#define L2_MMU_CONFIG 0xF0C /* (RW) L2 cache and MMU configuration (implementation-specific) */ ++ ++/* Job control registers */ ++ ++#define JOB_CONTROL_BASE 0x1000 ++ ++#define JOB_CONTROL_REG(r) (JOB_CONTROL_BASE + (r)) ++ ++#define JOB_IRQ_RAWSTAT 0x000 /* Raw interrupt status register */ ++#define JOB_IRQ_CLEAR 0x004 /* Interrupt clear register */ ++#define JOB_IRQ_MASK 0x008 /* Interrupt mask register */ ++#define JOB_IRQ_STATUS 0x00C /* Interrupt status register */ ++ ++/* MMU control registers */ ++ ++#define MEMORY_MANAGEMENT_BASE 0x2000 ++#define MMU_REG(r) (MEMORY_MANAGEMENT_BASE + (r)) ++ ++#define MMU_IRQ_RAWSTAT 0x000 /* (RW) Raw interrupt status register */ ++#define MMU_IRQ_CLEAR 0x004 /* (WO) Interrupt clear register */ ++#define MMU_IRQ_MASK 0x008 /* (RW) Interrupt mask register */ ++#define MMU_IRQ_STATUS 0x00C /* (RO) Interrupt status register */ ++ ++#define MMU_AS0 0x400 /* Configuration registers for address space 0 */ ++#define MMU_AS1 0x440 /* Configuration registers for address space 1 */ ++#define MMU_AS2 0x480 /* Configuration registers for address space 2 */ ++#define MMU_AS3 0x4C0 /* Configuration registers for address space 3 */ ++#define MMU_AS4 0x500 /* Configuration registers for address space 4 */ ++#define MMU_AS5 0x540 /* Configuration registers for address space 5 */ ++#define MMU_AS6 0x580 /* Configuration registers for address space 6 */ ++#define MMU_AS7 0x5C0 /* Configuration registers for address space 7 */ ++#define MMU_AS8 0x600 /* Configuration registers for address space 8 */ ++#define MMU_AS9 0x640 /* Configuration registers for address space 9 */ ++#define MMU_AS10 0x680 /* Configuration registers for address space 10 */ ++#define MMU_AS11 0x6C0 /* Configuration registers for address space 11 */ ++#define MMU_AS12 0x700 /* Configuration registers for address space 12 */ ++#define MMU_AS13 0x740 /* Configuration registers for address space 13 */ ++#define MMU_AS14 0x780 /* Configuration registers for address space 14 */ ++#define MMU_AS15 0x7C0 /* Configuration registers for address space 15 */ ++ ++/* MMU address space control registers */ ++ ++#define MMU_AS_REG(n, r) (MMU_REG(MMU_AS0 + ((n) << 6)) + (r)) ++ ++#define AS_TRANSTAB_LO 0x00 /* (RW) Translation Table Base Address for address space n, low word */ ++#define AS_TRANSTAB_HI 0x04 /* (RW) Translation Table Base Address for address space n, high word */ ++#define AS_MEMATTR_LO 0x08 /* (RW) Memory attributes for address space n, low word. */ ++#define AS_MEMATTR_HI 0x0C /* (RW) Memory attributes for address space n, high word. */ ++#define AS_LOCKADDR_LO 0x10 /* (RW) Lock region address for address space n, low word */ ++#define AS_LOCKADDR_HI 0x14 /* (RW) Lock region address for address space n, high word */ ++#define AS_COMMAND 0x18 /* (WO) MMU command register for address space n */ ++#define AS_FAULTSTATUS 0x1C /* (RO) MMU fault status register for address space n */ ++#define AS_FAULTADDRESS_LO 0x20 /* (RO) Fault Address for address space n, low word */ ++#define AS_FAULTADDRESS_HI 0x24 /* (RO) Fault Address for address space n, high word */ ++#define AS_STATUS 0x28 /* (RO) Status flags for address space n */ ++ ++/* (RW) Translation table configuration for address space n, low word */ ++#define AS_TRANSCFG_LO 0x30 ++/* (RW) Translation table configuration for address space n, high word */ ++#define AS_TRANSCFG_HI 0x34 ++/* (RO) Secondary fault address for address space n, low word */ ++#define AS_FAULTEXTRA_LO 0x38 ++/* (RO) Secondary fault address for address space n, high word */ ++#define AS_FAULTEXTRA_HI 0x3C ++ ++/* End Register Offsets */ ++ ++/* Include POWER_CHANGED_SINGLE in debug builds for use in irq latency test. */ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++#define GPU_IRQ_REG_ALL (GPU_IRQ_REG_COMMON | POWER_CHANGED_SINGLE) ++#else /* CONFIG_MALI_BIFROST_DEBUG */ ++#define GPU_IRQ_REG_ALL (GPU_IRQ_REG_COMMON) ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++ ++/* ++ * MMU_IRQ_RAWSTAT register values. Values are valid also for ++ * MMU_IRQ_CLEAR, MMU_IRQ_MASK, MMU_IRQ_STATUS registers. ++ */ ++ ++#define MMU_PAGE_FAULT_FLAGS 16 ++ ++/* Macros returning a bitmask to retrieve page fault or bus error flags from ++ * MMU registers */ ++#define MMU_PAGE_FAULT(n) (1UL << (n)) ++#define MMU_BUS_ERROR(n) (1UL << ((n) + MMU_PAGE_FAULT_FLAGS)) ++ ++/* ++ * Begin LPAE MMU TRANSTAB register values ++ */ ++#define AS_TRANSTAB_LPAE_ADDR_SPACE_MASK 0xfffff000 ++#define AS_TRANSTAB_LPAE_ADRMODE_UNMAPPED (0u << 0) ++#define AS_TRANSTAB_LPAE_ADRMODE_IDENTITY (1u << 1) ++#define AS_TRANSTAB_LPAE_ADRMODE_TABLE (3u << 0) ++#define AS_TRANSTAB_LPAE_READ_INNER (1u << 2) ++#define AS_TRANSTAB_LPAE_SHARE_OUTER (1u << 4) ++ ++#define AS_TRANSTAB_LPAE_ADRMODE_MASK 0x00000003 ++ ++/* ++ * Begin AARCH64 MMU TRANSTAB register values ++ */ ++#define MMU_HW_OUTA_BITS 40 ++#define AS_TRANSTAB_BASE_MASK ((1ULL << MMU_HW_OUTA_BITS) - (1ULL << 4)) ++ ++/* ++ * Begin MMU STATUS register values ++ */ ++#define AS_STATUS_AS_ACTIVE 0x01 ++ ++#define AS_FAULTSTATUS_EXCEPTION_CODE_MASK (0x7<<3) ++#define AS_FAULTSTATUS_EXCEPTION_CODE_TRANSLATION_FAULT (0x0<<3) ++#define AS_FAULTSTATUS_EXCEPTION_CODE_PERMISSION_FAULT (0x1<<3) ++#define AS_FAULTSTATUS_EXCEPTION_CODE_TRANSTAB_BUS_FAULT (0x2<<3) ++#define AS_FAULTSTATUS_EXCEPTION_CODE_ACCESS_FLAG (0x3<<3) ++#define AS_FAULTSTATUS_EXCEPTION_CODE_ADDRESS_SIZE_FAULT (0x4<<3) ++#define AS_FAULTSTATUS_EXCEPTION_CODE_MEMORY_ATTRIBUTES_FAULT (0x5<<3) ++ ++#define AS_FAULTSTATUS_EXCEPTION_TYPE_SHIFT 0 ++#define AS_FAULTSTATUS_EXCEPTION_TYPE_MASK (0xFF << AS_FAULTSTATUS_EXCEPTION_TYPE_SHIFT) ++#define AS_FAULTSTATUS_EXCEPTION_TYPE_GET(reg_val) \ ++ (((reg_val)&AS_FAULTSTATUS_EXCEPTION_TYPE_MASK) >> AS_FAULTSTATUS_EXCEPTION_TYPE_SHIFT) ++#define AS_FAULTSTATUS_EXCEPTION_TYPE_TRANSLATION_FAULT_0 0xC0 ++ ++#define AS_FAULTSTATUS_ACCESS_TYPE_SHIFT 8 ++#define AS_FAULTSTATUS_ACCESS_TYPE_MASK (0x3 << AS_FAULTSTATUS_ACCESS_TYPE_SHIFT) ++#define AS_FAULTSTATUS_ACCESS_TYPE_GET(reg_val) \ ++ (((reg_val)&AS_FAULTSTATUS_ACCESS_TYPE_MASK) >> AS_FAULTSTATUS_ACCESS_TYPE_SHIFT) ++ ++#define AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC (0x0) ++#define AS_FAULTSTATUS_ACCESS_TYPE_EX (0x1) ++#define AS_FAULTSTATUS_ACCESS_TYPE_READ (0x2) ++#define AS_FAULTSTATUS_ACCESS_TYPE_WRITE (0x3) ++ ++#define AS_FAULTSTATUS_SOURCE_ID_SHIFT 16 ++#define AS_FAULTSTATUS_SOURCE_ID_MASK (0xFFFF << AS_FAULTSTATUS_SOURCE_ID_SHIFT) ++#define AS_FAULTSTATUS_SOURCE_ID_GET(reg_val) \ ++ (((reg_val)&AS_FAULTSTATUS_SOURCE_ID_MASK) >> AS_FAULTSTATUS_SOURCE_ID_SHIFT) ++ ++/* ++ * Begin MMU TRANSCFG register values ++ */ ++#define AS_TRANSCFG_ADRMODE_LEGACY 0 ++#define AS_TRANSCFG_ADRMODE_UNMAPPED 1 ++#define AS_TRANSCFG_ADRMODE_IDENTITY 2 ++#define AS_TRANSCFG_ADRMODE_AARCH64_4K 6 ++#define AS_TRANSCFG_ADRMODE_AARCH64_64K 8 ++ ++#define AS_TRANSCFG_ADRMODE_MASK 0xF ++ ++/* ++ * Begin TRANSCFG register values ++ */ ++#define AS_TRANSCFG_PTW_MEMATTR_MASK (3ull << 24) ++#define AS_TRANSCFG_PTW_MEMATTR_NON_CACHEABLE (1ull << 24) ++#define AS_TRANSCFG_PTW_MEMATTR_WRITE_BACK (2ull << 24) ++ ++#define AS_TRANSCFG_PTW_SH_MASK ((3ull << 28)) ++#define AS_TRANSCFG_PTW_SH_OS (2ull << 28) ++#define AS_TRANSCFG_PTW_SH_IS (3ull << 28) ++#define AS_TRANSCFG_R_ALLOCATE (1ull << 30) ++ ++/* ++ * Begin Command Values ++ */ ++ ++/* AS_COMMAND register commands */ ++#define AS_COMMAND_NOP 0x00 /* NOP Operation */ ++#define AS_COMMAND_UPDATE 0x01 /* Broadcasts the values in AS_TRANSTAB and ASn_MEMATTR to all MMUs */ ++#define AS_COMMAND_LOCK 0x02 /* Issue a lock region command to all MMUs */ ++#define AS_COMMAND_UNLOCK 0x03 /* Issue a flush region command to all MMUs */ ++#define AS_COMMAND_FLUSH 0x04 /* Flush all L2 caches then issue a flush region command to all MMUs ++ (deprecated - only for use with T60x) */ ++#define AS_COMMAND_FLUSH_PT 0x04 /* Flush all L2 caches then issue a flush region command to all MMUs */ ++#define AS_COMMAND_FLUSH_MEM 0x05 /* Wait for memory accesses to complete, flush all the L1s cache then ++ flush all L2 caches then issue a flush region command to all MMUs */ ++ ++/* GPU_STATUS values */ ++#define GPU_STATUS_PRFCNT_ACTIVE (1 << 2) /* Set if the performance counters are active. */ ++#define GPU_STATUS_PROTECTED_MODE_ACTIVE (1 << 7) /* Set if protected mode is active */ ++ ++/* PRFCNT_CONFIG register values */ ++#define PRFCNT_CONFIG_MODE_SHIFT 0 /* Counter mode position. */ ++#define PRFCNT_CONFIG_AS_SHIFT 4 /* Address space bitmap position. */ ++#define PRFCNT_CONFIG_SETSELECT_SHIFT 8 /* Set select position. */ ++ ++/* The performance counters are disabled. */ ++#define PRFCNT_CONFIG_MODE_OFF 0 ++/* The performance counters are enabled, but are only written out when a ++ * PRFCNT_SAMPLE command is issued using the GPU_COMMAND register. ++ */ ++#define PRFCNT_CONFIG_MODE_MANUAL 1 ++/* The performance counters are enabled, and are written out each time a tile ++ * finishes rendering. ++ */ ++#define PRFCNT_CONFIG_MODE_TILE 2 ++ ++/* AS_MEMATTR values from MMU_MEMATTR_STAGE1: */ ++/* Use GPU implementation-defined caching policy. */ ++#define AS_MEMATTR_IMPL_DEF_CACHE_POLICY 0x88ull ++/* The attribute set to force all resources to be cached. */ ++#define AS_MEMATTR_FORCE_TO_CACHE_ALL 0x8Full ++/* Inner write-alloc cache setup, no outer caching */ ++#define AS_MEMATTR_WRITE_ALLOC 0x8Dull ++ ++/* Use GPU implementation-defined caching policy. */ ++#define AS_MEMATTR_LPAE_IMPL_DEF_CACHE_POLICY 0x48ull ++/* The attribute set to force all resources to be cached. */ ++#define AS_MEMATTR_LPAE_FORCE_TO_CACHE_ALL 0x4Full ++/* Inner write-alloc cache setup, no outer caching */ ++#define AS_MEMATTR_LPAE_WRITE_ALLOC 0x4Dull ++/* Set to implementation defined, outer caching */ ++#define AS_MEMATTR_LPAE_OUTER_IMPL_DEF 0x88ull ++/* Set to write back memory, outer caching */ ++#define AS_MEMATTR_LPAE_OUTER_WA 0x8Dull ++/* There is no LPAE support for non-cacheable, since the memory type is always ++ * write-back. ++ * Marking this setting as reserved for LPAE ++ */ ++#define AS_MEMATTR_LPAE_NON_CACHEABLE_RESERVED ++ ++/* L2_MMU_CONFIG register */ ++#define L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY_SHIFT (23) ++#define L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY (0x1 << L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY_SHIFT) ++ ++/* End L2_MMU_CONFIG register */ ++ ++/* THREAD_* registers */ ++ ++/* THREAD_FEATURES IMPLEMENTATION_TECHNOLOGY values */ ++#define IMPLEMENTATION_UNSPECIFIED 0 ++#define IMPLEMENTATION_SILICON 1 ++#define IMPLEMENTATION_FPGA 2 ++#define IMPLEMENTATION_MODEL 3 ++ ++/* Default values when registers are not supported by the implemented hardware */ ++#define THREAD_MT_DEFAULT 256 ++#define THREAD_MWS_DEFAULT 256 ++#define THREAD_MBS_DEFAULT 256 ++#define THREAD_MR_DEFAULT 1024 ++#define THREAD_MTQ_DEFAULT 4 ++#define THREAD_MTGS_DEFAULT 10 ++ ++/* End THREAD_* registers */ ++ ++/* SHADER_CONFIG register */ ++#define SC_LS_ALLOW_ATTR_TYPES (1ul << 16) ++#define SC_TLS_HASH_ENABLE (1ul << 17) ++#define SC_LS_ATTR_CHECK_DISABLE (1ul << 18) ++#define SC_VAR_ALGORITHM (1ul << 29) ++/* End SHADER_CONFIG register */ ++ ++/* TILER_CONFIG register */ ++#define TC_CLOCK_GATE_OVERRIDE (1ul << 0) ++/* End TILER_CONFIG register */ ++ ++/* L2_CONFIG register */ ++#define L2_CONFIG_SIZE_SHIFT 16 ++#define L2_CONFIG_SIZE_MASK (0xFFul << L2_CONFIG_SIZE_SHIFT) ++#define L2_CONFIG_HASH_SHIFT 24 ++#define L2_CONFIG_HASH_MASK (0xFFul << L2_CONFIG_HASH_SHIFT) ++/* End L2_CONFIG register */ ++ ++/* IDVS_GROUP register */ ++#define IDVS_GROUP_SIZE_SHIFT (16) ++#define IDVS_GROUP_MAX_SIZE (0x3F) ++ ++#endif /* _KBASE_GPU_REGMAP_H_ */ +diff --git a/drivers/gpu/arm/bifrost/ipa/Kbuild b/drivers/gpu/arm/bifrost/ipa/Kbuild +new file mode 100755 +index 000000000000..04aa9d82d7c5 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/ipa/Kbuild +@@ -0,0 +1,28 @@ ++# ++# (C) COPYRIGHT 2016-2018 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# GNU General Public License for more details. ++# ++# You should have received a copy of the GNU General Public License ++# along with this program; if not, you can access it online at ++# http://www.gnu.org/licenses/gpl-2.0.html. ++# ++# SPDX-License-Identifier: GPL-2.0 ++# ++# ++ ++bifrost_kbase-y += \ ++ ipa/mali_kbase_ipa_simple.o \ ++ ipa/mali_kbase_ipa.o \ ++ ipa/mali_kbase_ipa_vinstr_g7x.o \ ++ ipa/mali_kbase_ipa_vinstr_common.o ++ ++bifrost_kbase-$(CONFIG_DEBUG_FS) += ipa/mali_kbase_ipa_debugfs.o +diff --git a/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa.c b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa.c +new file mode 100755 +index 000000000000..67adb65306dd +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa.c +@@ -0,0 +1,672 @@ ++/* ++ * ++ * (C) COPYRIGHT 2016-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++#include ++#include ++#include ++#include "mali_kbase.h" ++#include "mali_kbase_ipa.h" ++#include "mali_kbase_ipa_debugfs.h" ++#include "mali_kbase_ipa_simple.h" ++#include "backend/gpu/mali_kbase_pm_internal.h" ++ ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0)) ++#include ++#else ++#include ++#define dev_pm_opp_find_freq_exact opp_find_freq_exact ++#define dev_pm_opp_get_voltage opp_get_voltage ++#define dev_pm_opp opp ++#endif ++ ++#define KBASE_IPA_FALLBACK_MODEL_NAME "mali-simple-power-model" ++ ++static const struct kbase_ipa_model_ops *kbase_ipa_all_model_ops[] = { ++ &kbase_simple_ipa_model_ops, ++ &kbase_g71_ipa_model_ops, ++ &kbase_g72_ipa_model_ops, ++ &kbase_g76_ipa_model_ops, ++ &kbase_g52_ipa_model_ops, ++ &kbase_g52_r1_ipa_model_ops, ++ &kbase_g51_ipa_model_ops, ++ &kbase_g77_ipa_model_ops, ++ &kbase_tnax_ipa_model_ops, ++ &kbase_tbex_ipa_model_ops, ++ &kbase_tbax_ipa_model_ops ++}; ++ ++int kbase_ipa_model_recalculate(struct kbase_ipa_model *model) ++{ ++ int err = 0; ++ ++ lockdep_assert_held(&model->kbdev->ipa.lock); ++ ++ if (model->ops->recalculate) { ++ err = model->ops->recalculate(model); ++ if (err) { ++ dev_err(model->kbdev->dev, ++ "recalculation of power model %s returned error %d\n", ++ model->ops->name, err); ++ } ++ } ++ ++ return err; ++} ++ ++const struct kbase_ipa_model_ops *kbase_ipa_model_ops_find(struct kbase_device *kbdev, ++ const char *name) ++{ ++ int i; ++ ++ for (i = 0; i < ARRAY_SIZE(kbase_ipa_all_model_ops); ++i) { ++ const struct kbase_ipa_model_ops *ops = kbase_ipa_all_model_ops[i]; ++ ++ if (!strcmp(ops->name, name)) ++ return ops; ++ } ++ ++ dev_err(kbdev->dev, "power model \'%s\' not found\n", name); ++ ++ return NULL; ++} ++KBASE_EXPORT_TEST_API(kbase_ipa_model_ops_find); ++ ++const char *kbase_ipa_model_name_from_id(u32 gpu_id) ++{ ++ const u32 prod_id = (gpu_id & GPU_ID_VERSION_PRODUCT_ID) >> ++ GPU_ID_VERSION_PRODUCT_ID_SHIFT; ++ ++ switch (GPU_ID2_MODEL_MATCH_VALUE(prod_id)) { ++ case GPU_ID2_PRODUCT_TMIX: ++ return "mali-g71-power-model"; ++ case GPU_ID2_PRODUCT_THEX: ++ return "mali-g72-power-model"; ++ case GPU_ID2_PRODUCT_TNOX: ++ return "mali-g76-power-model"; ++ case GPU_ID2_PRODUCT_TSIX: ++ return "mali-g51-power-model"; ++ case GPU_ID2_PRODUCT_TGOX: ++ if ((gpu_id & GPU_ID2_VERSION_MAJOR) == ++ (0 << GPU_ID2_VERSION_MAJOR_SHIFT)) ++ /* g52 aliased to g76 power-model's ops */ ++ return "mali-g52-power-model"; ++ else ++ return "mali-g52_r1-power-model"; ++ case GPU_ID2_PRODUCT_TNAX: ++ return "mali-tnax-power-model"; ++ case GPU_ID2_PRODUCT_TTRX: ++ return "mali-g77-power-model"; ++ case GPU_ID2_PRODUCT_TBEX: ++ return "mali-tbex-power-model"; ++ case GPU_ID2_PRODUCT_TBAX: ++ return "mali-tbax-power-model"; ++ default: ++ return KBASE_IPA_FALLBACK_MODEL_NAME; ++ } ++} ++KBASE_EXPORT_TEST_API(kbase_ipa_model_name_from_id); ++ ++static struct device_node *get_model_dt_node(struct kbase_ipa_model *model, ++ bool dt_required) ++{ ++ struct device_node *model_dt_node; ++ char compat_string[64]; ++ ++ snprintf(compat_string, sizeof(compat_string), "arm,%s", ++ model->ops->name); ++ ++ /* of_find_compatible_node() will call of_node_put() on the root node, ++ * so take a reference on it first. ++ */ ++ of_node_get(model->kbdev->dev->of_node); ++ model_dt_node = of_find_compatible_node(model->kbdev->dev->of_node, ++ NULL, compat_string); ++ if (!model_dt_node && !model->missing_dt_node_warning) { ++ if (dt_required) ++ dev_warn(model->kbdev->dev, ++ "Couldn't find power_model DT node matching \'%s\'\n", ++ compat_string); ++ model->missing_dt_node_warning = true; ++ } ++ ++ return model_dt_node; ++} ++ ++int kbase_ipa_model_add_param_s32(struct kbase_ipa_model *model, ++ const char *name, s32 *addr, ++ size_t num_elems, bool dt_required) ++{ ++ int err, i; ++ struct device_node *model_dt_node = get_model_dt_node(model, ++ dt_required); ++ char *origin; ++ ++ err = of_property_read_u32_array(model_dt_node, name, addr, num_elems); ++ /* We're done with model_dt_node now, so drop the reference taken in ++ * get_model_dt_node()/of_find_compatible_node(). ++ */ ++ of_node_put(model_dt_node); ++ ++ if (err && dt_required) { ++ memset(addr, 0, sizeof(s32) * num_elems); ++ dev_warn(model->kbdev->dev, ++ "Error %d, no DT entry: %s.%s = %zu*[0]\n", ++ err, model->ops->name, name, num_elems); ++ origin = "zero"; ++ } else if (err && !dt_required) { ++ origin = "default"; ++ } else /* !err */ { ++ origin = "DT"; ++ } ++ ++ /* Create a unique debugfs entry for each element */ ++ for (i = 0; i < num_elems; ++i) { ++ char elem_name[32]; ++ ++ if (num_elems == 1) ++ snprintf(elem_name, sizeof(elem_name), "%s", name); ++ else ++ snprintf(elem_name, sizeof(elem_name), "%s.%d", ++ name, i); ++ ++ dev_dbg(model->kbdev->dev, "%s.%s = %d (%s)\n", ++ model->ops->name, elem_name, addr[i], origin); ++ ++ err = kbase_ipa_model_param_add(model, elem_name, ++ &addr[i], sizeof(s32), ++ PARAM_TYPE_S32); ++ if (err) ++ goto exit; ++ } ++exit: ++ return err; ++} ++ ++int kbase_ipa_model_add_param_string(struct kbase_ipa_model *model, ++ const char *name, char *addr, ++ size_t size, bool dt_required) ++{ ++ int err; ++ struct device_node *model_dt_node = get_model_dt_node(model, ++ dt_required); ++ const char *string_prop_value; ++ char *origin; ++ ++ err = of_property_read_string(model_dt_node, name, ++ &string_prop_value); ++ ++ /* We're done with model_dt_node now, so drop the reference taken in ++ * get_model_dt_node()/of_find_compatible_node(). ++ */ ++ of_node_put(model_dt_node); ++ ++ if (err && dt_required) { ++ strncpy(addr, "", size - 1); ++ dev_warn(model->kbdev->dev, ++ "Error %d, no DT entry: %s.%s = \'%s\'\n", ++ err, model->ops->name, name, addr); ++ err = 0; ++ origin = "zero"; ++ } else if (err && !dt_required) { ++ origin = "default"; ++ } else /* !err */ { ++ strncpy(addr, string_prop_value, size - 1); ++ origin = "DT"; ++ } ++ ++ addr[size - 1] = '\0'; ++ ++ dev_dbg(model->kbdev->dev, "%s.%s = \'%s\' (%s)\n", ++ model->ops->name, name, string_prop_value, origin); ++ ++ err = kbase_ipa_model_param_add(model, name, addr, size, ++ PARAM_TYPE_STRING); ++ return err; ++} ++ ++void kbase_ipa_term_model(struct kbase_ipa_model *model) ++{ ++ if (!model) ++ return; ++ ++ lockdep_assert_held(&model->kbdev->ipa.lock); ++ ++ if (model->ops->term) ++ model->ops->term(model); ++ ++ kbase_ipa_model_param_free_all(model); ++ ++ kfree(model); ++} ++KBASE_EXPORT_TEST_API(kbase_ipa_term_model); ++ ++struct kbase_ipa_model *kbase_ipa_init_model(struct kbase_device *kbdev, ++ const struct kbase_ipa_model_ops *ops) ++{ ++ struct kbase_ipa_model *model; ++ int err; ++ ++ lockdep_assert_held(&kbdev->ipa.lock); ++ ++ if (!ops || !ops->name) ++ return NULL; ++ ++ model = kzalloc(sizeof(struct kbase_ipa_model), GFP_KERNEL); ++ if (!model) ++ return NULL; ++ ++ model->kbdev = kbdev; ++ model->ops = ops; ++ INIT_LIST_HEAD(&model->params); ++ ++ err = model->ops->init(model); ++ if (err) { ++ dev_err(kbdev->dev, ++ "init of power model \'%s\' returned error %d\n", ++ ops->name, err); ++ kfree(model); ++ return NULL; ++ } ++ ++ err = kbase_ipa_model_recalculate(model); ++ if (err) { ++ kbase_ipa_term_model(model); ++ return NULL; ++ } ++ ++ return model; ++} ++KBASE_EXPORT_TEST_API(kbase_ipa_init_model); ++ ++static void kbase_ipa_term_locked(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->ipa.lock); ++ ++ /* Clean up the models */ ++ if (kbdev->ipa.configured_model != kbdev->ipa.fallback_model) ++ kbase_ipa_term_model(kbdev->ipa.configured_model); ++ kbase_ipa_term_model(kbdev->ipa.fallback_model); ++ ++ kbdev->ipa.configured_model = NULL; ++ kbdev->ipa.fallback_model = NULL; ++} ++ ++int kbase_ipa_init(struct kbase_device *kbdev) ++{ ++ ++ const char *model_name; ++ const struct kbase_ipa_model_ops *ops; ++ struct kbase_ipa_model *default_model = NULL; ++ int err; ++ ++ mutex_init(&kbdev->ipa.lock); ++ /* ++ * Lock during init to avoid warnings from lockdep_assert_held (there ++ * shouldn't be any concurrent access yet). ++ */ ++ mutex_lock(&kbdev->ipa.lock); ++ ++ /* The simple IPA model must *always* be present.*/ ++ ops = kbase_ipa_model_ops_find(kbdev, KBASE_IPA_FALLBACK_MODEL_NAME); ++ ++ default_model = kbase_ipa_init_model(kbdev, ops); ++ if (!default_model) { ++ err = -EINVAL; ++ goto end; ++ } ++ ++ kbdev->ipa.fallback_model = default_model; ++ err = of_property_read_string(kbdev->dev->of_node, ++ "ipa-model", ++ &model_name); ++ if (err) { ++ /* Attempt to load a match from GPU-ID */ ++ u32 gpu_id; ++ ++ gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; ++ model_name = kbase_ipa_model_name_from_id(gpu_id); ++ dev_dbg(kbdev->dev, ++ "Inferring model from GPU ID 0x%x: \'%s\'\n", ++ gpu_id, model_name); ++ err = 0; ++ } else { ++ dev_dbg(kbdev->dev, ++ "Using ipa-model parameter from DT: \'%s\'\n", ++ model_name); ++ } ++ ++ if (strcmp(KBASE_IPA_FALLBACK_MODEL_NAME, model_name) != 0) { ++ ops = kbase_ipa_model_ops_find(kbdev, model_name); ++ kbdev->ipa.configured_model = kbase_ipa_init_model(kbdev, ops); ++ if (!kbdev->ipa.configured_model) { ++ dev_warn(kbdev->dev, ++ "Failed to initialize ipa-model: \'%s\'\n" ++ "Falling back on default model\n", ++ model_name); ++ kbdev->ipa.configured_model = default_model; ++ } ++ } else { ++ kbdev->ipa.configured_model = default_model; ++ } ++ ++end: ++ if (err) ++ kbase_ipa_term_locked(kbdev); ++ else ++ dev_info(kbdev->dev, ++ "Using configured power model %s, and fallback %s\n", ++ kbdev->ipa.configured_model->ops->name, ++ kbdev->ipa.fallback_model->ops->name); ++ ++ mutex_unlock(&kbdev->ipa.lock); ++ return err; ++} ++KBASE_EXPORT_TEST_API(kbase_ipa_init); ++ ++void kbase_ipa_term(struct kbase_device *kbdev) ++{ ++ mutex_lock(&kbdev->ipa.lock); ++ kbase_ipa_term_locked(kbdev); ++ mutex_unlock(&kbdev->ipa.lock); ++ ++ mutex_destroy(&kbdev->ipa.lock); ++} ++KBASE_EXPORT_TEST_API(kbase_ipa_term); ++ ++/** ++ * kbase_scale_dynamic_power() - Scale a dynamic power coefficient to an OPP ++ * @c: Dynamic model coefficient, in pW/(Hz V^2). Should be in range ++ * 0 < c < 2^26 to prevent overflow. ++ * @freq: Frequency, in Hz. Range: 2^23 < freq < 2^30 (~8MHz to ~1GHz) ++ * @voltage: Voltage, in mV. Range: 2^9 < voltage < 2^13 (~0.5V to ~8V) ++ * ++ * Keep a record of the approximate range of each value at every stage of the ++ * calculation, to ensure we don't overflow. This makes heavy use of the ++ * approximations 1000 = 2^10 and 1000000 = 2^20, but does the actual ++ * calculations in decimal for increased accuracy. ++ * ++ * Return: Power consumption, in mW. Range: 0 < p < 2^13 (0W to ~8W) ++ */ ++static u32 kbase_scale_dynamic_power(const u32 c, const u32 freq, ++ const u32 voltage) ++{ ++ /* Range: 2^8 < v2 < 2^16 m(V^2) */ ++ const u32 v2 = (voltage * voltage) / 1000; ++ ++ /* Range: 2^3 < f_MHz < 2^10 MHz */ ++ const u32 f_MHz = freq / 1000000; ++ ++ /* Range: 2^11 < v2f_big < 2^26 kHz V^2 */ ++ const u32 v2f_big = v2 * f_MHz; ++ ++ /* Range: 2^1 < v2f < 2^16 MHz V^2 */ ++ const u32 v2f = v2f_big / 1000; ++ ++ /* Range (working backwards from next line): 0 < v2fc < 2^23 uW. ++ * Must be < 2^42 to avoid overflowing the return value. */ ++ const u64 v2fc = (u64) c * (u64) v2f; ++ ++ /* Range: 0 < v2fc / 1000 < 2^13 mW */ ++ return div_u64(v2fc, 1000); ++} ++ ++/** ++ * kbase_scale_static_power() - Scale a static power coefficient to an OPP ++ * @c: Static model coefficient, in uW/V^3. Should be in range ++ * 0 < c < 2^32 to prevent overflow. ++ * @voltage: Voltage, in mV. Range: 2^9 < voltage < 2^13 (~0.5V to ~8V) ++ * ++ * Return: Power consumption, in mW. Range: 0 < p < 2^13 (0W to ~8W) ++ */ ++u32 kbase_scale_static_power(const u32 c, const u32 voltage) ++{ ++ /* Range: 2^8 < v2 < 2^16 m(V^2) */ ++ const u32 v2 = (voltage * voltage) / 1000; ++ ++ /* Range: 2^17 < v3_big < 2^29 m(V^2) mV */ ++ const u32 v3_big = v2 * voltage; ++ ++ /* Range: 2^7 < v3 < 2^19 m(V^3) */ ++ const u32 v3 = v3_big / 1000; ++ ++ /* ++ * Range (working backwards from next line): 0 < v3c_big < 2^33 nW. ++ * The result should be < 2^52 to avoid overflowing the return value. ++ */ ++ const u64 v3c_big = (u64) c * (u64) v3; ++ ++ /* Range: 0 < v3c_big / 1000000 < 2^13 mW */ ++ return div_u64(v3c_big, 1000000); ++} ++ ++void kbase_ipa_protection_mode_switch_event(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ /* Record the event of GPU entering protected mode. */ ++ kbdev->ipa_protection_mode_switched = true; ++} ++ ++static struct kbase_ipa_model *get_current_model(struct kbase_device *kbdev) ++{ ++ struct kbase_ipa_model *model; ++ unsigned long flags; ++ ++ lockdep_assert_held(&kbdev->ipa.lock); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ if (kbdev->ipa_protection_mode_switched || ++ kbdev->ipa.force_fallback_model) ++ model = kbdev->ipa.fallback_model; ++ else ++ model = kbdev->ipa.configured_model; ++ ++ /* ++ * Having taken cognizance of the fact that whether GPU earlier ++ * protected mode or not, the event can be now reset (if GPU is not ++ * currently in protected mode) so that configured model is used ++ * for the next sample. ++ */ ++ if (!kbdev->protected_mode) ++ kbdev->ipa_protection_mode_switched = false; ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ return model; ++} ++ ++static u32 get_static_power_locked(struct kbase_device *kbdev, ++ struct kbase_ipa_model *model, ++ unsigned long voltage) ++{ ++ u32 power = 0; ++ int err; ++ u32 power_coeff; ++ ++ lockdep_assert_held(&model->kbdev->ipa.lock); ++ ++ if (!model->ops->get_static_coeff) ++ model = kbdev->ipa.fallback_model; ++ ++ if (model->ops->get_static_coeff) { ++ err = model->ops->get_static_coeff(model, &power_coeff); ++ if (!err) ++ power = kbase_scale_static_power(power_coeff, ++ (u32) voltage); ++ } ++ ++ return power; ++} ++ ++#if defined(CONFIG_MALI_PWRSOFT_765) || \ ++ LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++static unsigned long kbase_get_static_power(struct devfreq *df, ++ unsigned long voltage) ++#else ++static unsigned long kbase_get_static_power(unsigned long voltage) ++#endif ++{ ++ struct kbase_ipa_model *model; ++ u32 power = 0; ++#if defined(CONFIG_MALI_PWRSOFT_765) || \ ++ LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++ struct kbase_device *kbdev = dev_get_drvdata(&df->dev); ++#else ++ struct kbase_device *kbdev = kbase_find_device(-1); ++#endif ++ ++ if (!kbdev) ++ return 0ul; ++ ++ mutex_lock(&kbdev->ipa.lock); ++ ++ model = get_current_model(kbdev); ++ power = get_static_power_locked(kbdev, model, voltage); ++ ++ mutex_unlock(&kbdev->ipa.lock); ++ ++#if !(defined(CONFIG_MALI_PWRSOFT_765) || \ ++ LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)) ++ kbase_release_device(kbdev); ++#endif ++ ++ return power; ++} ++ ++#if defined(CONFIG_MALI_PWRSOFT_765) || \ ++ LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++static unsigned long kbase_get_dynamic_power(struct devfreq *df, ++ unsigned long freq, ++ unsigned long voltage) ++#else ++static unsigned long kbase_get_dynamic_power(unsigned long freq, ++ unsigned long voltage) ++#endif ++{ ++ struct kbase_ipa_model *model; ++ u32 power_coeff = 0, power = 0; ++ int err = 0; ++#if defined(CONFIG_MALI_PWRSOFT_765) || \ ++ LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++ struct kbase_device *kbdev = dev_get_drvdata(&df->dev); ++#else ++ struct kbase_device *kbdev = kbase_find_device(-1); ++#endif ++ ++ if (!kbdev) ++ return 0ul; ++ ++ mutex_lock(&kbdev->ipa.lock); ++ ++ model = kbdev->ipa.fallback_model; ++ ++ err = model->ops->get_dynamic_coeff(model, &power_coeff); ++ ++ if (!err) ++ power = kbase_scale_dynamic_power(power_coeff, freq, voltage); ++ else ++ dev_err_ratelimited(kbdev->dev, ++ "Model %s returned error code %d\n", ++ model->ops->name, err); ++ ++ mutex_unlock(&kbdev->ipa.lock); ++ ++#if !(defined(CONFIG_MALI_PWRSOFT_765) || \ ++ LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)) ++ kbase_release_device(kbdev); ++#endif ++ ++ return power; ++} ++ ++int kbase_get_real_power_locked(struct kbase_device *kbdev, u32 *power, ++ unsigned long freq, ++ unsigned long voltage) ++{ ++ struct kbase_ipa_model *model; ++ u32 power_coeff = 0; ++ int err = 0; ++ struct kbasep_pm_metrics diff; ++ u64 total_time; ++ ++ lockdep_assert_held(&kbdev->ipa.lock); ++ ++ kbase_pm_get_dvfs_metrics(kbdev, &kbdev->ipa.last_metrics, &diff); ++ ++ model = get_current_model(kbdev); ++ ++ err = model->ops->get_dynamic_coeff(model, &power_coeff); ++ ++ /* If the counter model returns an error (e.g. switching back to ++ * protected mode and failing to read counters, or a counter sample ++ * with too few cycles), revert to the fallback model. ++ */ ++ if (err && model != kbdev->ipa.fallback_model) { ++ model = kbdev->ipa.fallback_model; ++ err = model->ops->get_dynamic_coeff(model, &power_coeff); ++ } ++ ++ if (err) ++ return err; ++ ++ *power = kbase_scale_dynamic_power(power_coeff, freq, voltage); ++ ++ /* time_busy / total_time cannot be >1, so assigning the 64-bit ++ * result of div_u64 to *power cannot overflow. ++ */ ++ total_time = diff.time_busy + (u64) diff.time_idle; ++ *power = div_u64(*power * (u64) diff.time_busy, ++ max(total_time, 1ull)); ++ ++ *power += get_static_power_locked(kbdev, model, voltage); ++ ++ return err; ++} ++KBASE_EXPORT_TEST_API(kbase_get_real_power_locked); ++ ++int kbase_get_real_power(struct devfreq *df, u32 *power, ++ unsigned long freq, ++ unsigned long voltage) ++{ ++ int ret; ++ struct kbase_device *kbdev = dev_get_drvdata(&df->dev); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ mutex_lock(&kbdev->ipa.lock); ++ ret = kbase_get_real_power_locked(kbdev, power, freq, voltage); ++ mutex_unlock(&kbdev->ipa.lock); ++ ++ return ret; ++} ++KBASE_EXPORT_TEST_API(kbase_get_real_power); ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) ++struct devfreq_cooling_ops kbase_ipa_power_model_ops = { ++#else ++struct devfreq_cooling_power kbase_ipa_power_model_ops = { ++#endif ++ .get_static_power = &kbase_get_static_power, ++ .get_dynamic_power = &kbase_get_dynamic_power, ++}; ++KBASE_EXPORT_TEST_API(kbase_ipa_power_model_ops); +diff --git a/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa.h b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa.h +new file mode 100755 +index 000000000000..f43f3d9416b4 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa.h +@@ -0,0 +1,254 @@ ++/* ++ * ++ * (C) COPYRIGHT 2016-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_IPA_H_ ++#define _KBASE_IPA_H_ ++ ++#if defined(CONFIG_MALI_BIFROST_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL) ++ ++struct devfreq; ++ ++/** ++ * struct kbase_ipa_model - Object describing a particular IPA model. ++ * @kbdev: pointer to kbase device ++ * @model_data: opaque pointer to model specific data, accessed ++ * only by model specific methods. ++ * @ops: pointer to object containing model specific methods. ++ * @params: head of the list of debugfs params added for model ++ * @missing_dt_node_warning: flag to limit the matching power model DT not found ++ * warning to once. ++ */ ++struct kbase_ipa_model { ++ struct kbase_device *kbdev; ++ void *model_data; ++ const struct kbase_ipa_model_ops *ops; ++ struct list_head params; ++ bool missing_dt_node_warning; ++}; ++ ++/** ++ * kbase_ipa_model_add_param_s32 - Add an integer model parameter ++ * @model: pointer to IPA model ++ * @name: name of corresponding debugfs entry ++ * @addr: address where the value is stored ++ * @num_elems: number of elements (1 if not an array) ++ * @dt_required: if false, a corresponding devicetree entry is not required, ++ * and the current value will be used. If true, a warning is ++ * output and the data is zeroed ++ * ++ * Return: 0 on success, or an error code ++ */ ++int kbase_ipa_model_add_param_s32(struct kbase_ipa_model *model, ++ const char *name, s32 *addr, ++ size_t num_elems, bool dt_required); ++ ++/** ++ * kbase_ipa_model_add_param_string - Add a string model parameter ++ * @model: pointer to IPA model ++ * @name: name of corresponding debugfs entry ++ * @addr: address where the value is stored ++ * @size: size, in bytes, of the value storage (so the maximum string ++ * length is size - 1) ++ * @dt_required: if false, a corresponding devicetree entry is not required, ++ * and the current value will be used. If true, a warning is ++ * output and the data is zeroed ++ * ++ * Return: 0 on success, or an error code ++ */ ++int kbase_ipa_model_add_param_string(struct kbase_ipa_model *model, ++ const char *name, char *addr, ++ size_t size, bool dt_required); ++ ++struct kbase_ipa_model_ops { ++ char *name; ++ /* The init, recalculate and term ops on the default model are always ++ * called. However, all the other models are only invoked if the model ++ * is selected in the device tree. Otherwise they are never ++ * initialized. Additional resources can be acquired by models in ++ * init(), however they must be terminated in the term(). ++ */ ++ int (*init)(struct kbase_ipa_model *model); ++ /* Called immediately after init(), or when a parameter is changed, so ++ * that any coefficients derived from model parameters can be ++ * recalculated. */ ++ int (*recalculate)(struct kbase_ipa_model *model); ++ void (*term)(struct kbase_ipa_model *model); ++ /* ++ * get_dynamic_coeff() - calculate dynamic power coefficient ++ * @model: pointer to model ++ * @coeffp: pointer to return value location ++ * ++ * Calculate a dynamic power coefficient, with units pW/(Hz V^2), which ++ * is then scaled by the IPA framework according to the current OPP's ++ * frequency and voltage. ++ * ++ * Return: 0 on success, or an error code. ++ */ ++ int (*get_dynamic_coeff)(struct kbase_ipa_model *model, u32 *coeffp); ++ /* ++ * get_static_coeff() - calculate static power coefficient ++ * @model: pointer to model ++ * @coeffp: pointer to return value location ++ * ++ * Calculate a static power coefficient, with units uW/(V^3), which is ++ * scaled by the IPA framework according to the current OPP's voltage. ++ * ++ * Return: 0 on success, or an error code. ++ */ ++ int (*get_static_coeff)(struct kbase_ipa_model *model, u32 *coeffp); ++}; ++ ++/** ++ * kbase_ipa_init - Initialize the IPA feature ++ * @kbdev: pointer to kbase device ++ * ++ * simple IPA power model is initialized as a fallback model and if that ++ * initialization fails then IPA is not used. ++ * The device tree is read for the name of ipa model to be used, by using the ++ * property string "ipa-model". If that ipa model is supported then it is ++ * initialized but if the initialization fails then simple power model is used. ++ * ++ * Return: 0 on success, negative -errno on error ++ */ ++int kbase_ipa_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_ipa_term - Terminate the IPA feature ++ * @kbdev: pointer to kbase device ++ * ++ * Both simple IPA power model and model retrieved from device tree are ++ * terminated. ++ */ ++void kbase_ipa_term(struct kbase_device *kbdev); ++ ++/** ++ * kbase_ipa_model_recalculate - Recalculate the model coefficients ++ * @model: pointer to the IPA model object, already initialized ++ * ++ * It shall be called immediately after the model has been initialized ++ * or when the model parameter has changed, so that any coefficients ++ * derived from parameters can be recalculated. ++ * Its a wrapper for the module specific recalculate() method. ++ * ++ * Return: 0 on success, negative -errno on error ++ */ ++int kbase_ipa_model_recalculate(struct kbase_ipa_model *model); ++ ++/** ++ * kbase_ipa_model_ops_find - Lookup an IPA model using its name ++ * @kbdev: pointer to kbase device ++ * @name: name of model to lookup ++ * ++ * Return: Pointer to model's 'ops' structure, or NULL if the lookup failed. ++ */ ++const struct kbase_ipa_model_ops *kbase_ipa_model_ops_find(struct kbase_device *kbdev, ++ const char *name); ++ ++/** ++ * kbase_ipa_model_name_from_id - Find the best model for a given GPU ID ++ * @gpu_id: GPU ID of GPU the model will be used for ++ * ++ * Return: The name of the appropriate counter-based model, or the name of the ++ * fallback model if no counter model exists. ++ */ ++const char *kbase_ipa_model_name_from_id(u32 gpu_id); ++ ++/** ++ * kbase_ipa_init_model - Initilaize the particular IPA model ++ * @kbdev: pointer to kbase device ++ * @ops: pointer to object containing model specific methods. ++ * ++ * Initialize the model corresponding to the @ops pointer passed. ++ * The init() method specified in @ops would be called. ++ * ++ * Return: pointer to kbase_ipa_model on success, NULL on error ++ */ ++struct kbase_ipa_model *kbase_ipa_init_model(struct kbase_device *kbdev, ++ const struct kbase_ipa_model_ops *ops); ++/** ++ * kbase_ipa_term_model - Terminate the particular IPA model ++ * @model: pointer to the IPA model object, already initialized ++ * ++ * Terminate the model, using the term() method. ++ * Module specific parameters would be freed. ++ */ ++void kbase_ipa_term_model(struct kbase_ipa_model *model); ++ ++/** ++ * kbase_ipa_protection_mode_switch_event - Inform IPA of the GPU's entry into ++ * protected mode ++ * @kbdev: pointer to kbase device ++ * ++ * Makes IPA aware of the GPU switching to protected mode. ++ */ ++void kbase_ipa_protection_mode_switch_event(struct kbase_device *kbdev); ++ ++extern const struct kbase_ipa_model_ops kbase_g71_ipa_model_ops; ++extern const struct kbase_ipa_model_ops kbase_g72_ipa_model_ops; ++extern const struct kbase_ipa_model_ops kbase_g76_ipa_model_ops; ++extern const struct kbase_ipa_model_ops kbase_g52_ipa_model_ops; ++extern const struct kbase_ipa_model_ops kbase_g52_r1_ipa_model_ops; ++extern const struct kbase_ipa_model_ops kbase_g51_ipa_model_ops; ++extern const struct kbase_ipa_model_ops kbase_g77_ipa_model_ops; ++extern const struct kbase_ipa_model_ops kbase_tnax_ipa_model_ops; ++extern const struct kbase_ipa_model_ops kbase_tbex_ipa_model_ops; ++extern const struct kbase_ipa_model_ops kbase_tbax_ipa_model_ops; ++ ++/** ++ * kbase_get_real_power() - get the real power consumption of the GPU ++ * @df: dynamic voltage and frequency scaling information for the GPU. ++ * @power: where to store the power consumption, in mW. ++ * @freq: a frequency, in HZ. ++ * @voltage: a voltage, in mV. ++ * ++ * The returned value incorporates both static and dynamic power consumption. ++ * ++ * Return: 0 on success, or an error code. ++ */ ++int kbase_get_real_power(struct devfreq *df, u32 *power, ++ unsigned long freq, ++ unsigned long voltage); ++ ++#if MALI_UNIT_TEST ++/* Called by kbase_get_real_power() to invoke the power models. ++ * Must be called with kbdev->ipa.lock held. ++ * This function is only exposed for use by unit tests. ++ */ ++int kbase_get_real_power_locked(struct kbase_device *kbdev, u32 *power, ++ unsigned long freq, ++ unsigned long voltage); ++#endif /* MALI_UNIT_TEST */ ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) ++extern struct devfreq_cooling_ops kbase_ipa_power_model_ops; ++#else ++extern struct devfreq_cooling_power kbase_ipa_power_model_ops; ++#endif ++ ++#else /* !(defined(CONFIG_MALI_BIFROST_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL)) */ ++ ++static inline void kbase_ipa_protection_mode_switch_event(struct kbase_device *kbdev) ++{ } ++ ++#endif /* (defined(CONFIG_MALI_BIFROST_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL)) */ ++ ++#endif +diff --git a/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_debugfs.c b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_debugfs.c +new file mode 100755 +index 000000000000..30a3b7d1b3be +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_debugfs.c +@@ -0,0 +1,322 @@ ++/* ++ * ++ * (C) COPYRIGHT 2017-2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++#include ++#include ++ ++#include "mali_kbase.h" ++#include "mali_kbase_ipa.h" ++#include "mali_kbase_ipa_debugfs.h" ++ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0)) ++#define DEFINE_DEBUGFS_ATTRIBUTE DEFINE_SIMPLE_ATTRIBUTE ++#endif ++ ++struct kbase_ipa_model_param { ++ char *name; ++ union { ++ void *voidp; ++ s32 *s32p; ++ char *str; ++ } addr; ++ size_t size; ++ enum kbase_ipa_model_param_type type; ++ struct kbase_ipa_model *model; ++ struct list_head link; ++}; ++ ++static int param_int_get(void *data, u64 *val) ++{ ++ struct kbase_ipa_model_param *param = data; ++ ++ mutex_lock(¶m->model->kbdev->ipa.lock); ++ *(s64 *) val = *param->addr.s32p; ++ mutex_unlock(¶m->model->kbdev->ipa.lock); ++ ++ return 0; ++} ++ ++static int param_int_set(void *data, u64 val) ++{ ++ struct kbase_ipa_model_param *param = data; ++ struct kbase_ipa_model *model = param->model; ++ s64 sval = (s64) val; ++ s32 old_val; ++ int err = 0; ++ ++ if (sval < S32_MIN || sval > S32_MAX) ++ return -ERANGE; ++ ++ mutex_lock(¶m->model->kbdev->ipa.lock); ++ old_val = *param->addr.s32p; ++ *param->addr.s32p = val; ++ err = kbase_ipa_model_recalculate(model); ++ if (err < 0) ++ *param->addr.s32p = old_val; ++ mutex_unlock(¶m->model->kbdev->ipa.lock); ++ ++ return err; ++} ++ ++DEFINE_DEBUGFS_ATTRIBUTE(fops_s32, param_int_get, param_int_set, "%lld\n"); ++ ++static ssize_t param_string_get(struct file *file, char __user *user_buf, ++ size_t count, loff_t *ppos) ++{ ++ struct kbase_ipa_model_param *param = file->private_data; ++ ssize_t ret; ++ size_t len; ++ ++ mutex_lock(¶m->model->kbdev->ipa.lock); ++ len = strnlen(param->addr.str, param->size - 1) + 1; ++ ret = simple_read_from_buffer(user_buf, count, ppos, ++ param->addr.str, len); ++ mutex_unlock(¶m->model->kbdev->ipa.lock); ++ ++ return ret; ++} ++ ++static ssize_t param_string_set(struct file *file, const char __user *user_buf, ++ size_t count, loff_t *ppos) ++{ ++ struct kbase_ipa_model_param *param = file->private_data; ++ struct kbase_ipa_model *model = param->model; ++ char *old_str = NULL; ++ ssize_t ret = count; ++ size_t buf_size; ++ int err; ++ ++ mutex_lock(&model->kbdev->ipa.lock); ++ ++ if (count > param->size) { ++ ret = -EINVAL; ++ goto end; ++ } ++ ++ old_str = kstrndup(param->addr.str, param->size, GFP_KERNEL); ++ if (!old_str) { ++ ret = -ENOMEM; ++ goto end; ++ } ++ ++ buf_size = min(param->size - 1, count); ++ if (copy_from_user(param->addr.str, user_buf, buf_size)) { ++ ret = -EFAULT; ++ goto end; ++ } ++ ++ param->addr.str[buf_size] = '\0'; ++ ++ err = kbase_ipa_model_recalculate(model); ++ if (err < 0) { ++ ret = err; ++ strlcpy(param->addr.str, old_str, param->size); ++ } ++ ++end: ++ kfree(old_str); ++ mutex_unlock(&model->kbdev->ipa.lock); ++ ++ return ret; ++} ++ ++static const struct file_operations fops_string = { ++ .owner = THIS_MODULE, ++ .read = param_string_get, ++ .write = param_string_set, ++ .open = simple_open, ++ .llseek = default_llseek, ++}; ++ ++int kbase_ipa_model_param_add(struct kbase_ipa_model *model, const char *name, ++ void *addr, size_t size, ++ enum kbase_ipa_model_param_type type) ++{ ++ struct kbase_ipa_model_param *param; ++ ++ param = kzalloc(sizeof(*param), GFP_KERNEL); ++ ++ if (!param) ++ return -ENOMEM; ++ ++ /* 'name' is stack-allocated for array elements, so copy it into ++ * heap-allocated storage */ ++ param->name = kstrdup(name, GFP_KERNEL); ++ ++ if (!param->name) { ++ kfree(param); ++ return -ENOMEM; ++ } ++ ++ param->addr.voidp = addr; ++ param->size = size; ++ param->type = type; ++ param->model = model; ++ ++ list_add(¶m->link, &model->params); ++ ++ return 0; ++} ++ ++void kbase_ipa_model_param_free_all(struct kbase_ipa_model *model) ++{ ++ struct kbase_ipa_model_param *param_p, *param_n; ++ ++ list_for_each_entry_safe(param_p, param_n, &model->params, link) { ++ list_del(¶m_p->link); ++ kfree(param_p->name); ++ kfree(param_p); ++ } ++} ++ ++static int force_fallback_model_get(void *data, u64 *val) ++{ ++ struct kbase_device *kbdev = data; ++ ++ mutex_lock(&kbdev->ipa.lock); ++ *val = kbdev->ipa.force_fallback_model; ++ mutex_unlock(&kbdev->ipa.lock); ++ ++ return 0; ++} ++ ++static int force_fallback_model_set(void *data, u64 val) ++{ ++ struct kbase_device *kbdev = data; ++ ++ mutex_lock(&kbdev->ipa.lock); ++ kbdev->ipa.force_fallback_model = (val ? true : false); ++ mutex_unlock(&kbdev->ipa.lock); ++ ++ return 0; ++} ++ ++DEFINE_DEBUGFS_ATTRIBUTE(force_fallback_model, ++ force_fallback_model_get, ++ force_fallback_model_set, ++ "%llu\n"); ++ ++static int current_power_get(void *data, u64 *val) ++{ ++ struct kbase_device *kbdev = data; ++ struct devfreq *df = kbdev->devfreq; ++ u32 power; ++ ++ kbase_pm_context_active(kbdev); ++ /* The current model assumes that there's no more than one voltage ++ * regulator currently available in the system. ++ */ ++ kbase_get_real_power(df, &power, ++ kbdev->current_nominal_freq, ++ (kbdev->current_voltages[0] / 1000)); ++ kbase_pm_context_idle(kbdev); ++ ++ *val = power; ++ ++ return 0; ++} ++DEFINE_DEBUGFS_ATTRIBUTE(current_power, current_power_get, NULL, "%llu\n"); ++ ++static void kbase_ipa_model_debugfs_init(struct kbase_ipa_model *model) ++{ ++ struct list_head *it; ++ struct dentry *dir; ++ ++ lockdep_assert_held(&model->kbdev->ipa.lock); ++ ++ dir = debugfs_create_dir(model->ops->name, ++ model->kbdev->mali_debugfs_directory); ++ ++ if (!dir) { ++ dev_err(model->kbdev->dev, ++ "Couldn't create mali debugfs %s directory", ++ model->ops->name); ++ return; ++ } ++ ++ list_for_each(it, &model->params) { ++ struct kbase_ipa_model_param *param = ++ list_entry(it, ++ struct kbase_ipa_model_param, ++ link); ++ const struct file_operations *fops = NULL; ++ ++ switch (param->type) { ++ case PARAM_TYPE_S32: ++ fops = &fops_s32; ++ break; ++ case PARAM_TYPE_STRING: ++ fops = &fops_string; ++ break; ++ } ++ ++ if (unlikely(!fops)) { ++ dev_err(model->kbdev->dev, ++ "Type not set for %s parameter %s\n", ++ model->ops->name, param->name); ++ } else { ++ debugfs_create_file(param->name, S_IRUGO | S_IWUSR, ++ dir, param, fops); ++ } ++ } ++} ++ ++void kbase_ipa_model_param_set_s32(struct kbase_ipa_model *model, ++ const char *name, s32 val) ++{ ++ struct kbase_ipa_model_param *param; ++ ++ mutex_lock(&model->kbdev->ipa.lock); ++ ++ list_for_each_entry(param, &model->params, link) { ++ if (!strcmp(param->name, name)) { ++ if (param->type == PARAM_TYPE_S32) { ++ *param->addr.s32p = val; ++ } else { ++ dev_err(model->kbdev->dev, ++ "Wrong type for %s parameter %s\n", ++ model->ops->name, param->name); ++ } ++ break; ++ } ++ } ++ ++ mutex_unlock(&model->kbdev->ipa.lock); ++} ++KBASE_EXPORT_TEST_API(kbase_ipa_model_param_set_s32); ++ ++void kbase_ipa_debugfs_init(struct kbase_device *kbdev) ++{ ++ mutex_lock(&kbdev->ipa.lock); ++ ++ if (kbdev->ipa.configured_model != kbdev->ipa.fallback_model) ++ kbase_ipa_model_debugfs_init(kbdev->ipa.configured_model); ++ kbase_ipa_model_debugfs_init(kbdev->ipa.fallback_model); ++ ++ debugfs_create_file("ipa_current_power", 0444, ++ kbdev->mali_debugfs_directory, kbdev, ¤t_power); ++ debugfs_create_file("ipa_force_fallback_model", 0644, ++ kbdev->mali_debugfs_directory, kbdev, &force_fallback_model); ++ ++ mutex_unlock(&kbdev->ipa.lock); ++} +diff --git a/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_debugfs.h b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_debugfs.h +new file mode 100755 +index 000000000000..a983d9c14216 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_debugfs.h +@@ -0,0 +1,68 @@ ++/* ++ * ++ * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_IPA_DEBUGFS_H_ ++#define _KBASE_IPA_DEBUGFS_H_ ++ ++enum kbase_ipa_model_param_type { ++ PARAM_TYPE_S32 = 1, ++ PARAM_TYPE_STRING, ++}; ++ ++#ifdef CONFIG_DEBUG_FS ++ ++void kbase_ipa_debugfs_init(struct kbase_device *kbdev); ++int kbase_ipa_model_param_add(struct kbase_ipa_model *model, const char *name, ++ void *addr, size_t size, ++ enum kbase_ipa_model_param_type type); ++void kbase_ipa_model_param_free_all(struct kbase_ipa_model *model); ++ ++/** ++ * kbase_ipa_model_param_set_s32 - Set an integer model parameter ++ * ++ * @model: pointer to IPA model ++ * @name: name of corresponding debugfs entry ++ * @val: new value of the parameter ++ * ++ * This function is only exposed for use by unit tests running in ++ * kernel space. Normally it is expected that parameter values will ++ * instead be set via debugfs. ++ */ ++void kbase_ipa_model_param_set_s32(struct kbase_ipa_model *model, ++ const char *name, s32 val); ++ ++#else /* CONFIG_DEBUG_FS */ ++ ++static inline int kbase_ipa_model_param_add(struct kbase_ipa_model *model, ++ const char *name, void *addr, ++ size_t size, ++ enum kbase_ipa_model_param_type type) ++{ ++ return 0; ++} ++ ++static inline void kbase_ipa_model_param_free_all(struct kbase_ipa_model *model) ++{ } ++ ++#endif /* CONFIG_DEBUG_FS */ ++ ++#endif /* _KBASE_IPA_DEBUGFS_H_ */ +diff --git a/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_simple.c b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_simple.c +new file mode 100755 +index 000000000000..9a11ee5b1e74 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_simple.c +@@ -0,0 +1,356 @@ ++/* ++ * ++ * (C) COPYRIGHT 2016-2018 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++#include ++#include ++#ifdef CONFIG_DEVFREQ_THERMAL ++#include ++#endif ++#include ++#include ++#include ++ ++#include "mali_kbase.h" ++#include "mali_kbase_defs.h" ++#include "mali_kbase_ipa_simple.h" ++#include "mali_kbase_ipa_debugfs.h" ++ ++#if MALI_UNIT_TEST ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) ++static unsigned long dummy_temp; ++ ++static int kbase_simple_power_model_get_dummy_temp( ++ struct thermal_zone_device *tz, ++ unsigned long *temp) ++{ ++ *temp = READ_ONCE(dummy_temp); ++ return 0; ++} ++ ++#else ++static int dummy_temp; ++ ++static int kbase_simple_power_model_get_dummy_temp( ++ struct thermal_zone_device *tz, ++ int *temp) ++{ ++ *temp = READ_ONCE(dummy_temp); ++ return 0; ++} ++#endif ++ ++/* Intercept calls to the kernel function using a macro */ ++#ifdef thermal_zone_get_temp ++#undef thermal_zone_get_temp ++#endif ++#define thermal_zone_get_temp(tz, temp) \ ++ kbase_simple_power_model_get_dummy_temp(tz, temp) ++ ++void kbase_simple_power_model_set_dummy_temp(int temp) ++{ ++ WRITE_ONCE(dummy_temp, temp); ++} ++KBASE_EXPORT_TEST_API(kbase_simple_power_model_set_dummy_temp); ++ ++#endif /* MALI_UNIT_TEST */ ++ ++/* ++ * This model is primarily designed for the Juno platform. It may not be ++ * suitable for other platforms. The additional resources in this model ++ * should preferably be minimal, as this model is rarely used when a dynamic ++ * model is available. ++ */ ++ ++/** ++ * struct kbase_ipa_model_simple_data - IPA context per device ++ * @dynamic_coefficient: dynamic coefficient of the model ++ * @static_coefficient: static coefficient of the model ++ * @ts: Thermal scaling coefficients of the model ++ * @tz_name: Thermal zone name ++ * @gpu_tz: thermal zone device ++ * @poll_temperature_thread: Handle for temperature polling thread ++ * @current_temperature: Most recent value of polled temperature ++ * @temperature_poll_interval_ms: How often temperature should be checked, in ms ++ */ ++ ++struct kbase_ipa_model_simple_data { ++ u32 dynamic_coefficient; ++ u32 static_coefficient; ++ s32 ts[4]; ++ char tz_name[THERMAL_NAME_LENGTH]; ++ struct thermal_zone_device *gpu_tz; ++ struct task_struct *poll_temperature_thread; ++ int current_temperature; ++ int temperature_poll_interval_ms; ++}; ++#define FALLBACK_STATIC_TEMPERATURE 55000 ++ ++/** ++ * calculate_temp_scaling_factor() - Calculate temperature scaling coefficient ++ * @ts: Signed coefficients, in order t^0 to t^3, with units Deg^-N ++ * @t: Temperature, in mDeg C. Range: -2^17 < t < 2^17 ++ * ++ * Scale the temperature according to a cubic polynomial whose coefficients are ++ * provided in the device tree. The result is used to scale the static power ++ * coefficient, where 1000000 means no change. ++ * ++ * Return: Temperature scaling factor. Range 0 <= ret <= 10,000,000. ++ */ ++static u32 calculate_temp_scaling_factor(s32 ts[4], s64 t) ++{ ++ /* Range: -2^24 < t2 < 2^24 m(Deg^2) */ ++ const s64 t2 = div_s64((t * t), 1000); ++ ++ /* Range: -2^31 < t3 < 2^31 m(Deg^3) */ ++ const s64 t3 = div_s64((t * t2), 1000); ++ ++ /* ++ * Sum the parts. t^[1-3] are in m(Deg^N), but the coefficients are in ++ * Deg^-N, so we need to multiply the last coefficient by 1000. ++ * Range: -2^63 < res_big < 2^63 ++ */ ++ const s64 res_big = ts[3] * t3 /* +/- 2^62 */ ++ + ts[2] * t2 /* +/- 2^55 */ ++ + ts[1] * t /* +/- 2^48 */ ++ + ts[0] * (s64)1000; /* +/- 2^41 */ ++ ++ /* Range: -2^60 < res_unclamped < 2^60 */ ++ s64 res_unclamped = div_s64(res_big, 1000); ++ ++ /* Clamp to range of 0x to 10x the static power */ ++ return clamp(res_unclamped, (s64) 0, (s64) 10000000); ++} ++ ++/* We can't call thermal_zone_get_temp() directly in model_static_coeff(), ++ * because we don't know if tz->lock is held in the same thread. So poll it in ++ * a separate thread to get around this. */ ++static int poll_temperature(void *data) ++{ ++ struct kbase_ipa_model_simple_data *model_data = ++ (struct kbase_ipa_model_simple_data *) data; ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) ++ unsigned long temp; ++#else ++ int temp; ++#endif ++ ++ set_freezable(); ++ ++ while (!kthread_should_stop()) { ++ struct thermal_zone_device *tz = READ_ONCE(model_data->gpu_tz); ++ ++ if (tz) { ++ int ret; ++ ++ ret = thermal_zone_get_temp(tz, &temp); ++ if (ret) { ++ pr_warn_ratelimited("Error reading temperature for gpu thermal zone: %d\n", ++ ret); ++ temp = FALLBACK_STATIC_TEMPERATURE; ++ } ++ } else { ++ temp = FALLBACK_STATIC_TEMPERATURE; ++ } ++ ++ WRITE_ONCE(model_data->current_temperature, temp); ++ ++ msleep_interruptible(READ_ONCE(model_data->temperature_poll_interval_ms)); ++ ++ try_to_freeze(); ++ } ++ ++ return 0; ++} ++ ++static int model_static_coeff(struct kbase_ipa_model *model, u32 *coeffp) ++{ ++ u32 temp_scaling_factor; ++ struct kbase_ipa_model_simple_data *model_data = ++ (struct kbase_ipa_model_simple_data *) model->model_data; ++ u64 coeff_big; ++ int temp; ++ ++ temp = READ_ONCE(model_data->current_temperature); ++ ++ /* Range: 0 <= temp_scaling_factor < 2^24 */ ++ temp_scaling_factor = calculate_temp_scaling_factor(model_data->ts, ++ temp); ++ ++ /* ++ * Range: 0 <= coeff_big < 2^52 to avoid overflowing *coeffp. This ++ * means static_coefficient must be in range ++ * 0 <= static_coefficient < 2^28. ++ */ ++ coeff_big = (u64) model_data->static_coefficient * (u64) temp_scaling_factor; ++ *coeffp = div_u64(coeff_big, 1000000); ++ ++ return 0; ++} ++ ++static int model_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp) ++{ ++ struct kbase_ipa_model_simple_data *model_data = ++ (struct kbase_ipa_model_simple_data *) model->model_data; ++ ++ *coeffp = model_data->dynamic_coefficient; ++ ++ return 0; ++} ++ ++static int add_params(struct kbase_ipa_model *model) ++{ ++ int err = 0; ++ struct kbase_ipa_model_simple_data *model_data = ++ (struct kbase_ipa_model_simple_data *)model->model_data; ++ ++ err = kbase_ipa_model_add_param_s32(model, "static-coefficient", ++ &model_data->static_coefficient, ++ 1, true); ++ if (err) ++ goto end; ++ ++ err = kbase_ipa_model_add_param_s32(model, "dynamic-coefficient", ++ &model_data->dynamic_coefficient, ++ 1, true); ++ if (err) ++ goto end; ++ ++ err = kbase_ipa_model_add_param_s32(model, "ts", ++ model_data->ts, 4, true); ++ if (err) ++ goto end; ++ ++ err = kbase_ipa_model_add_param_string(model, "thermal-zone", ++ model_data->tz_name, ++ sizeof(model_data->tz_name), true); ++ if (err) ++ goto end; ++ ++ model_data->temperature_poll_interval_ms = 200; ++ err = kbase_ipa_model_add_param_s32(model, "temp-poll-interval-ms", ++ &model_data->temperature_poll_interval_ms, ++ 1, false); ++ ++end: ++ return err; ++} ++ ++static int kbase_simple_power_model_init(struct kbase_ipa_model *model) ++{ ++ int err; ++ struct kbase_ipa_model_simple_data *model_data; ++ ++ model_data = kzalloc(sizeof(struct kbase_ipa_model_simple_data), ++ GFP_KERNEL); ++ if (!model_data) ++ return -ENOMEM; ++ ++ model->model_data = (void *) model_data; ++ ++ model_data->current_temperature = FALLBACK_STATIC_TEMPERATURE; ++ model_data->poll_temperature_thread = kthread_run(poll_temperature, ++ (void *) model_data, ++ "mali-simple-power-model-temp-poll"); ++ if (IS_ERR(model_data->poll_temperature_thread)) { ++ err = PTR_ERR(model_data->poll_temperature_thread); ++ kfree(model_data); ++ return err; ++ } ++ ++ err = add_params(model); ++ if (err) { ++ kbase_ipa_model_param_free_all(model); ++ kthread_stop(model_data->poll_temperature_thread); ++ kfree(model_data); ++ } ++ ++ return err; ++} ++ ++static int kbase_simple_power_model_recalculate(struct kbase_ipa_model *model) ++{ ++ struct kbase_ipa_model_simple_data *model_data = ++ (struct kbase_ipa_model_simple_data *)model->model_data; ++ struct thermal_zone_device *tz; ++ ++ lockdep_assert_held(&model->kbdev->ipa.lock); ++ ++ if (!strnlen(model_data->tz_name, sizeof(model_data->tz_name))) { ++ model_data->gpu_tz = NULL; ++ } else { ++ char tz_name[THERMAL_NAME_LENGTH]; ++ ++ strlcpy(tz_name, model_data->tz_name, sizeof(tz_name)); ++ ++ /* Release ipa.lock so that thermal_list_lock is not acquired ++ * with ipa.lock held, thereby avoid lock ordering violation ++ * lockdep warning. The warning comes as a chain of locks ++ * ipa.lock --> thermal_list_lock --> tz->lock gets formed ++ * on registering devfreq cooling device when probe method ++ * of mali platform driver is invoked. ++ */ ++ mutex_unlock(&model->kbdev->ipa.lock); ++ tz = thermal_zone_get_zone_by_name(tz_name); ++ mutex_lock(&model->kbdev->ipa.lock); ++ ++ if (IS_ERR_OR_NULL(tz)) { ++ pr_warn_ratelimited("Error %ld getting thermal zone \'%s\', not yet ready?\n", ++ PTR_ERR(tz), tz_name); ++ return -EPROBE_DEFER; ++ } ++ ++ /* Check if another thread raced against us & updated the ++ * thermal zone name string. Update the gpu_tz pointer only if ++ * the name string did not change whilst we retrieved the new ++ * thermal_zone_device pointer, otherwise model_data->tz_name & ++ * model_data->gpu_tz would become inconsistent with each other. ++ * The below check will succeed only for the thread which last ++ * updated the name string. ++ */ ++ if (strncmp(tz_name, model_data->tz_name, sizeof(tz_name)) == 0) ++ model_data->gpu_tz = tz; ++ } ++ ++ return 0; ++} ++ ++static void kbase_simple_power_model_term(struct kbase_ipa_model *model) ++{ ++ struct kbase_ipa_model_simple_data *model_data = ++ (struct kbase_ipa_model_simple_data *)model->model_data; ++ ++ kthread_stop(model_data->poll_temperature_thread); ++ ++ kfree(model_data); ++} ++ ++struct kbase_ipa_model_ops kbase_simple_ipa_model_ops = { ++ .name = "mali-simple-power-model", ++ .init = &kbase_simple_power_model_init, ++ .recalculate = &kbase_simple_power_model_recalculate, ++ .term = &kbase_simple_power_model_term, ++ .get_dynamic_coeff = &model_dynamic_coeff, ++ .get_static_coeff = &model_static_coeff, ++}; ++KBASE_EXPORT_TEST_API(kbase_simple_ipa_model_ops); +diff --git a/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_simple.h b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_simple.h +new file mode 100755 +index 000000000000..84534e07ec55 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_simple.h +@@ -0,0 +1,45 @@ ++/* ++ * ++ * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_IPA_SIMPLE_H_ ++#define _KBASE_IPA_SIMPLE_H_ ++ ++#if defined(CONFIG_MALI_BIFROST_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL) ++ ++extern struct kbase_ipa_model_ops kbase_simple_ipa_model_ops; ++ ++#if MALI_UNIT_TEST ++/** ++ * kbase_simple_power_model_set_dummy_temp() - set a dummy temperature value ++ * @temp: Temperature of the thermal zone, in millidegrees celsius. ++ * ++ * This is only intended for use in unit tests, to ensure that the temperature ++ * values used by the simple power model are predictable. Deterministic ++ * behavior is necessary to allow validation of the static power values ++ * computed by this model. ++ */ ++void kbase_simple_power_model_set_dummy_temp(int temp); ++#endif /* MALI_UNIT_TEST */ ++ ++#endif /* (defined(CONFIG_MALI_BIFROST_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL)) */ ++ ++#endif /* _KBASE_IPA_SIMPLE_H_ */ +diff --git a/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_vinstr_common.c b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_vinstr_common.c +new file mode 100755 +index 000000000000..702db1623101 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_vinstr_common.c +@@ -0,0 +1,349 @@ ++/* ++ * ++ * (C) COPYRIGHT 2017-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include "mali_kbase_ipa_vinstr_common.h" ++#include "mali_kbase_ipa_debugfs.h" ++ ++#define DEFAULT_SCALING_FACTOR 5 ++ ++/* If the value of GPU_ACTIVE is below this, use the simple model ++ * instead, to avoid extrapolating small amounts of counter data across ++ * large sample periods. ++ */ ++#define DEFAULT_MIN_SAMPLE_CYCLES 10000 ++ ++/** ++ * read_hwcnt() - read a counter value ++ * @model_data: pointer to model data ++ * @offset: offset, in bytes, into vinstr buffer ++ * ++ * Return: A 32-bit counter value. Range: 0 < value < 2^27 (worst case would be ++ * incrementing every cycle over a ~100ms sample period at a high frequency, ++ * e.g. 1 GHz: 2^30 * 0.1seconds ~= 2^27. ++ */ ++static inline u32 kbase_ipa_read_hwcnt( ++ struct kbase_ipa_model_vinstr_data *model_data, ++ u32 offset) ++{ ++ u8 *p = (u8 *)model_data->dump_buf.dump_buf; ++ ++ return *(u32 *)&p[offset]; ++} ++ ++static inline s64 kbase_ipa_add_saturate(s64 a, s64 b) ++{ ++ s64 rtn; ++ ++ if (a > 0 && (S64_MAX - a) < b) ++ rtn = S64_MAX; ++ else if (a < 0 && (S64_MIN - a) > b) ++ rtn = S64_MIN; ++ else ++ rtn = a + b; ++ ++ return rtn; ++} ++ ++s64 kbase_ipa_sum_all_shader_cores( ++ struct kbase_ipa_model_vinstr_data *model_data, ++ s32 coeff, u32 counter) ++{ ++ struct kbase_device *kbdev = model_data->kbdev; ++ u64 core_mask; ++ u32 base = 0; ++ s64 ret = 0; ++ ++ core_mask = kbdev->gpu_props.props.coherency_info.group[0].core_mask; ++ while (core_mask != 0ull) { ++ if ((core_mask & 1ull) != 0ull) { ++ /* 0 < counter_value < 2^27 */ ++ u32 counter_value = kbase_ipa_read_hwcnt(model_data, ++ base + counter); ++ ++ /* 0 < ret < 2^27 * max_num_cores = 2^32 */ ++ ret = kbase_ipa_add_saturate(ret, counter_value); ++ } ++ base += KBASE_IPA_NR_BYTES_PER_BLOCK; ++ core_mask >>= 1; ++ } ++ ++ /* Range: -2^54 < ret * coeff < 2^54 */ ++ return ret * coeff; ++} ++ ++s64 kbase_ipa_sum_all_memsys_blocks( ++ struct kbase_ipa_model_vinstr_data *model_data, ++ s32 coeff, u32 counter) ++{ ++ struct kbase_device *kbdev = model_data->kbdev; ++ const u32 num_blocks = kbdev->gpu_props.props.l2_props.num_l2_slices; ++ u32 base = 0; ++ s64 ret = 0; ++ u32 i; ++ ++ for (i = 0; i < num_blocks; i++) { ++ /* 0 < counter_value < 2^27 */ ++ u32 counter_value = kbase_ipa_read_hwcnt(model_data, ++ base + counter); ++ ++ /* 0 < ret < 2^27 * max_num_memsys_blocks = 2^29 */ ++ ret = kbase_ipa_add_saturate(ret, counter_value); ++ base += KBASE_IPA_NR_BYTES_PER_BLOCK; ++ } ++ ++ /* Range: -2^51 < ret * coeff < 2^51 */ ++ return ret * coeff; ++} ++ ++s64 kbase_ipa_single_counter( ++ struct kbase_ipa_model_vinstr_data *model_data, ++ s32 coeff, u32 counter) ++{ ++ /* Range: 0 < counter_value < 2^27 */ ++ const u32 counter_value = kbase_ipa_read_hwcnt(model_data, counter); ++ ++ /* Range: -2^49 < ret < 2^49 */ ++ return counter_value * (s64) coeff; ++} ++ ++int kbase_ipa_attach_vinstr(struct kbase_ipa_model_vinstr_data *model_data) ++{ ++ int errcode; ++ struct kbase_device *kbdev = model_data->kbdev; ++ struct kbase_hwcnt_virtualizer *hvirt = kbdev->hwcnt_gpu_virt; ++ struct kbase_hwcnt_enable_map enable_map; ++ const struct kbase_hwcnt_metadata *metadata = ++ kbase_hwcnt_virtualizer_metadata(hvirt); ++ ++ if (!metadata) ++ return -1; ++ ++ errcode = kbase_hwcnt_enable_map_alloc(metadata, &enable_map); ++ if (errcode) { ++ dev_err(kbdev->dev, "Failed to allocate IPA enable map"); ++ return errcode; ++ } ++ ++ kbase_hwcnt_enable_map_enable_all(&enable_map); ++ ++ /* Disable cycle counter only. */ ++ enable_map.clk_enable_map = 0; ++ ++ errcode = kbase_hwcnt_virtualizer_client_create( ++ hvirt, &enable_map, &model_data->hvirt_cli); ++ kbase_hwcnt_enable_map_free(&enable_map); ++ if (errcode) { ++ dev_err(kbdev->dev, "Failed to register IPA with virtualizer"); ++ model_data->hvirt_cli = NULL; ++ return errcode; ++ } ++ ++ errcode = kbase_hwcnt_dump_buffer_alloc( ++ metadata, &model_data->dump_buf); ++ if (errcode) { ++ dev_err(kbdev->dev, "Failed to allocate IPA dump buffer"); ++ kbase_hwcnt_virtualizer_client_destroy(model_data->hvirt_cli); ++ model_data->hvirt_cli = NULL; ++ return errcode; ++ } ++ ++ return 0; ++} ++ ++void kbase_ipa_detach_vinstr(struct kbase_ipa_model_vinstr_data *model_data) ++{ ++ if (model_data->hvirt_cli) { ++ kbase_hwcnt_virtualizer_client_destroy(model_data->hvirt_cli); ++ kbase_hwcnt_dump_buffer_free(&model_data->dump_buf); ++ model_data->hvirt_cli = NULL; ++ } ++} ++ ++int kbase_ipa_vinstr_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp) ++{ ++ struct kbase_ipa_model_vinstr_data *model_data = ++ (struct kbase_ipa_model_vinstr_data *)model->model_data; ++ s64 energy = 0; ++ size_t i; ++ u64 coeff = 0, coeff_mul = 0; ++ u64 start_ts_ns, end_ts_ns; ++ u32 active_cycles; ++ int err = 0; ++ ++ err = kbase_hwcnt_virtualizer_client_dump(model_data->hvirt_cli, ++ &start_ts_ns, &end_ts_ns, &model_data->dump_buf); ++ if (err) ++ goto err0; ++ ++ /* Range: 0 (GPU not used at all), to the max sampling interval, say ++ * 1s, * max GPU frequency (GPU 100% utilized). ++ * 0 <= active_cycles <= 1 * ~2GHz ++ * 0 <= active_cycles < 2^31 ++ */ ++ active_cycles = model_data->get_active_cycles(model_data); ++ ++ if (active_cycles < (u32) max(model_data->min_sample_cycles, 0)) { ++ err = -ENODATA; ++ goto err0; ++ } ++ ++ /* Range: 1 <= active_cycles < 2^31 */ ++ active_cycles = max(1u, active_cycles); ++ ++ /* Range of 'energy' is +/- 2^54 * number of IPA groups (~8), so around ++ * -2^57 < energy < 2^57 ++ */ ++ for (i = 0; i < model_data->groups_def_num; i++) { ++ const struct kbase_ipa_group *group = &model_data->groups_def[i]; ++ s32 coeff = model_data->group_values[i]; ++ s64 group_energy = group->op(model_data, coeff, ++ group->counter_block_offset); ++ ++ energy = kbase_ipa_add_saturate(energy, group_energy); ++ } ++ ++ /* Range: 0 <= coeff < 2^57 */ ++ if (energy > 0) ++ coeff = energy; ++ ++ /* Range: 0 <= coeff < 2^57 (because active_cycles >= 1). However, this ++ * can be constrained further: Counter values can only be increased by ++ * a theoretical maximum of about 64k per clock cycle. Beyond this, ++ * we'd have to sample every 1ms to avoid them overflowing at the ++ * lowest clock frequency (say 100MHz). Therefore, we can write the ++ * range of 'coeff' in terms of active_cycles: ++ * ++ * coeff = SUM(coeffN * counterN * num_cores_for_counterN) ++ * coeff <= SUM(coeffN * counterN) * max_num_cores ++ * coeff <= num_IPA_groups * max_coeff * max_counter * max_num_cores ++ * (substitute max_counter = 2^16 * active_cycles) ++ * coeff <= num_IPA_groups * max_coeff * 2^16 * active_cycles * max_num_cores ++ * coeff <= 2^3 * 2^22 * 2^16 * active_cycles * 2^5 ++ * coeff <= 2^46 * active_cycles ++ * ++ * So after the division: 0 <= coeff <= 2^46 ++ */ ++ coeff = div_u64(coeff, active_cycles); ++ ++ /* Not all models were derived at the same reference voltage. Voltage ++ * scaling is done by multiplying by V^2, so we need to *divide* by ++ * Vref^2 here. ++ * Range: 0 <= coeff <= 2^49 ++ */ ++ coeff = div_u64(coeff * 1000, max(model_data->reference_voltage, 1)); ++ /* Range: 0 <= coeff <= 2^52 */ ++ coeff = div_u64(coeff * 1000, max(model_data->reference_voltage, 1)); ++ ++ /* Scale by user-specified integer factor. ++ * Range: 0 <= coeff_mul < 2^57 ++ */ ++ coeff_mul = coeff * model_data->scaling_factor; ++ ++ /* The power models have results with units ++ * mW/(MHz V^2), i.e. nW/(Hz V^2). With precision of 1/1000000, this ++ * becomes fW/(Hz V^2), which are the units of coeff_mul. However, ++ * kbase_scale_dynamic_power() expects units of pW/(Hz V^2), so divide ++ * by 1000. ++ * Range: 0 <= coeff_mul < 2^47 ++ */ ++ coeff_mul = div_u64(coeff_mul, 1000u); ++ ++err0: ++ /* Clamp to a sensible range - 2^16 gives about 14W at 400MHz/750mV */ ++ *coeffp = clamp(coeff_mul, (u64) 0, (u64) 1 << 16); ++ return err; ++} ++ ++int kbase_ipa_vinstr_common_model_init(struct kbase_ipa_model *model, ++ const struct kbase_ipa_group *ipa_groups_def, ++ size_t ipa_group_size, ++ kbase_ipa_get_active_cycles_callback get_active_cycles, ++ s32 reference_voltage) ++{ ++ int err = 0; ++ size_t i; ++ struct kbase_ipa_model_vinstr_data *model_data; ++ ++ if (!model || !ipa_groups_def || !ipa_group_size || !get_active_cycles) ++ return -EINVAL; ++ ++ model_data = kzalloc(sizeof(*model_data), GFP_KERNEL); ++ if (!model_data) ++ return -ENOMEM; ++ ++ model_data->kbdev = model->kbdev; ++ model_data->groups_def = ipa_groups_def; ++ model_data->groups_def_num = ipa_group_size; ++ model_data->get_active_cycles = get_active_cycles; ++ ++ model->model_data = (void *) model_data; ++ ++ for (i = 0; i < model_data->groups_def_num; ++i) { ++ const struct kbase_ipa_group *group = &model_data->groups_def[i]; ++ ++ model_data->group_values[i] = group->default_value; ++ err = kbase_ipa_model_add_param_s32(model, group->name, ++ &model_data->group_values[i], ++ 1, false); ++ if (err) ++ goto exit; ++ } ++ ++ model_data->scaling_factor = DEFAULT_SCALING_FACTOR; ++ err = kbase_ipa_model_add_param_s32(model, "scale", ++ &model_data->scaling_factor, ++ 1, false); ++ if (err) ++ goto exit; ++ ++ model_data->min_sample_cycles = DEFAULT_MIN_SAMPLE_CYCLES; ++ err = kbase_ipa_model_add_param_s32(model, "min_sample_cycles", ++ &model_data->min_sample_cycles, ++ 1, false); ++ if (err) ++ goto exit; ++ ++ model_data->reference_voltage = reference_voltage; ++ err = kbase_ipa_model_add_param_s32(model, "reference_voltage", ++ &model_data->reference_voltage, ++ 1, false); ++ if (err) ++ goto exit; ++ ++ err = kbase_ipa_attach_vinstr(model_data); ++ ++exit: ++ if (err) { ++ kbase_ipa_model_param_free_all(model); ++ kfree(model_data); ++ } ++ return err; ++} ++ ++void kbase_ipa_vinstr_common_model_term(struct kbase_ipa_model *model) ++{ ++ struct kbase_ipa_model_vinstr_data *model_data = ++ (struct kbase_ipa_model_vinstr_data *)model->model_data; ++ ++ kbase_ipa_detach_vinstr(model_data); ++ kfree(model_data); ++} +diff --git a/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_vinstr_common.h b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_vinstr_common.h +new file mode 100755 +index 000000000000..46e3cd4bc6e1 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_vinstr_common.h +@@ -0,0 +1,217 @@ ++/* ++ * ++ * (C) COPYRIGHT 2017-2018 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_IPA_VINSTR_COMMON_H_ ++#define _KBASE_IPA_VINSTR_COMMON_H_ ++ ++#include "mali_kbase.h" ++#include "mali_kbase_hwcnt_virtualizer.h" ++#include "mali_kbase_hwcnt_types.h" ++ ++/* Maximum number of IPA groups for an IPA model. */ ++#define KBASE_IPA_MAX_GROUP_DEF_NUM 16 ++ ++/* Number of bytes per hardware counter in a vinstr_buffer. */ ++#define KBASE_IPA_NR_BYTES_PER_CNT 4 ++ ++/* Number of hardware counters per block in a vinstr_buffer. */ ++#define KBASE_IPA_NR_CNT_PER_BLOCK 64 ++ ++/* Number of bytes per block in a vinstr_buffer. */ ++#define KBASE_IPA_NR_BYTES_PER_BLOCK \ ++ (KBASE_IPA_NR_CNT_PER_BLOCK * KBASE_IPA_NR_BYTES_PER_CNT) ++ ++struct kbase_ipa_model_vinstr_data; ++ ++typedef u32 (*kbase_ipa_get_active_cycles_callback)(struct kbase_ipa_model_vinstr_data *); ++ ++/** ++ * struct kbase_ipa_model_vinstr_data - IPA context per device ++ * @kbdev: pointer to kbase device ++ * @groups_def: Array of IPA groups. ++ * @groups_def_num: Number of elements in the array of IPA groups. ++ * @get_active_cycles: Callback to return number of active cycles during ++ * counter sample period ++ * @hvirt_cli: hardware counter virtualizer client handle ++ * @dump_buf: buffer to dump hardware counters onto ++ * @reference_voltage: voltage, in mV, of the operating point used when ++ * deriving the power model coefficients. Range approx ++ * 0.1V - 5V (~= 8V): 2^7 <= reference_voltage <= 2^13 ++ * @scaling_factor: User-specified power scaling factor. This is an ++ * integer, which is multiplied by the power coefficient ++ * just before OPP scaling. ++ * Range approx 0-32: 0 < scaling_factor < 2^5 ++ * @min_sample_cycles: If the value of the GPU_ACTIVE counter (the number of ++ * cycles the GPU was working) is less than ++ * min_sample_cycles, the counter model will return an ++ * error, causing the IPA framework to approximate using ++ * the cached simple model results instead. This may be ++ * more accurate than extrapolating using a very small ++ * counter dump. ++ */ ++struct kbase_ipa_model_vinstr_data { ++ struct kbase_device *kbdev; ++ s32 group_values[KBASE_IPA_MAX_GROUP_DEF_NUM]; ++ const struct kbase_ipa_group *groups_def; ++ size_t groups_def_num; ++ kbase_ipa_get_active_cycles_callback get_active_cycles; ++ struct kbase_hwcnt_virtualizer_client *hvirt_cli; ++ struct kbase_hwcnt_dump_buffer dump_buf; ++ s32 reference_voltage; ++ s32 scaling_factor; ++ s32 min_sample_cycles; ++}; ++ ++/** ++ * struct ipa_group - represents a single IPA group ++ * @name: name of the IPA group ++ * @default_value: default value of coefficient for IPA group. ++ * Coefficients are interpreted as fractions where the ++ * denominator is 1000000. ++ * @op: which operation to be performed on the counter values ++ * @counter_block_offset: block offset in bytes of the counter used to calculate energy for IPA group ++ */ ++struct kbase_ipa_group { ++ const char *name; ++ s32 default_value; ++ s64 (*op)(struct kbase_ipa_model_vinstr_data *, s32, u32); ++ u32 counter_block_offset; ++}; ++ ++/** ++ * kbase_ipa_sum_all_shader_cores() - sum a counter over all cores ++ * @model_data: pointer to model data ++ * @coeff: model coefficient. Unity is ~2^20, so range approx ++ * +/- 4.0: -2^22 < coeff < 2^22 ++ * @counter offset in bytes of the counter used to calculate energy ++ * for IPA group ++ * ++ * Calculate energy estimation based on hardware counter `counter' ++ * across all shader cores. ++ * ++ * Return: Sum of counter values. Range: -2^54 < ret < 2^54 ++ */ ++s64 kbase_ipa_sum_all_shader_cores( ++ struct kbase_ipa_model_vinstr_data *model_data, ++ s32 coeff, u32 counter); ++ ++/** ++ * kbase_ipa_sum_all_memsys_blocks() - sum a counter over all mem system blocks ++ * @model_data: pointer to model data ++ * @coeff: model coefficient. Unity is ~2^20, so range approx ++ * +/- 4.0: -2^22 < coeff < 2^22 ++ * @counter: offset in bytes of the counter used to calculate energy ++ * for IPA group ++ * ++ * Calculate energy estimation based on hardware counter `counter' across all ++ * memory system blocks. ++ * ++ * Return: Sum of counter values. Range: -2^51 < ret < 2^51 ++ */ ++s64 kbase_ipa_sum_all_memsys_blocks( ++ struct kbase_ipa_model_vinstr_data *model_data, ++ s32 coeff, u32 counter); ++ ++/** ++ * kbase_ipa_single_counter() - sum a single counter ++ * @model_data: pointer to model data ++ * @coeff: model coefficient. Unity is ~2^20, so range approx ++ * +/- 4.0: -2^22 < coeff < 2^22 ++ * @counter: offset in bytes of the counter used to calculate energy ++ * for IPA group ++ * ++ * Calculate energy estimation based on hardware counter `counter'. ++ * ++ * Return: Counter value. Range: -2^49 < ret < 2^49 ++ */ ++s64 kbase_ipa_single_counter( ++ struct kbase_ipa_model_vinstr_data *model_data, ++ s32 coeff, u32 counter); ++ ++/** ++ * attach_vinstr() - attach a vinstr_buffer to an IPA model. ++ * @model_data pointer to model data ++ * ++ * Attach a vinstr_buffer to an IPA model. The vinstr_buffer ++ * allows access to the hardware counters used to calculate ++ * energy consumption. ++ * ++ * Return: 0 on success, or an error code. ++ */ ++int kbase_ipa_attach_vinstr(struct kbase_ipa_model_vinstr_data *model_data); ++ ++/** ++ * detach_vinstr() - detach a vinstr_buffer from an IPA model. ++ * @model_data pointer to model data ++ * ++ * Detach a vinstr_buffer from an IPA model. ++ */ ++void kbase_ipa_detach_vinstr(struct kbase_ipa_model_vinstr_data *model_data); ++ ++/** ++ * kbase_ipa_vinstr_dynamic_coeff() - calculate dynamic power based on HW counters ++ * @model: pointer to instantiated model ++ * @coeffp: pointer to location where calculated power, in ++ * pW/(Hz V^2), is stored. ++ * ++ * This is a GPU-agnostic implementation of the get_dynamic_coeff() ++ * function of an IPA model. It relies on the model being populated ++ * with GPU-specific attributes at initialization time. ++ * ++ * Return: 0 on success, or an error code. ++ */ ++int kbase_ipa_vinstr_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp); ++ ++/** ++ * kbase_ipa_vinstr_common_model_init() - initialize ipa power model ++ * @model: ipa power model to initialize ++ * @ipa_groups_def: array of ipa groups which sets coefficients for ++ * the corresponding counters used in the ipa model ++ * @ipa_group_size: number of elements in the array @ipa_groups_def ++ * @get_active_cycles: callback to return the number of cycles the GPU was ++ * active during the counter sample period. ++ * @reference_voltage: voltage, in mV, of the operating point used when ++ * deriving the power model coefficients. ++ * ++ * This initialization function performs initialization steps common ++ * for ipa models based on counter values. In each call, the model ++ * passes its specific coefficient values per ipa counter group via ++ * @ipa_groups_def array. ++ * ++ * Return: 0 on success, error code otherwise ++ */ ++int kbase_ipa_vinstr_common_model_init(struct kbase_ipa_model *model, ++ const struct kbase_ipa_group *ipa_groups_def, ++ size_t ipa_group_size, ++ kbase_ipa_get_active_cycles_callback get_active_cycles, ++ s32 reference_voltage); ++ ++/** ++ * kbase_ipa_vinstr_common_model_term() - terminate ipa power model ++ * @model: ipa power model to terminate ++ * ++ * This function performs all necessary steps to terminate ipa power model ++ * including clean up of resources allocated to hold model data. ++ */ ++void kbase_ipa_vinstr_common_model_term(struct kbase_ipa_model *model); ++ ++#endif /* _KBASE_IPA_VINSTR_COMMON_H_ */ +diff --git a/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_vinstr_g7x.c b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_vinstr_g7x.c +new file mode 100755 +index 000000000000..83174eb66ded +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_vinstr_g7x.c +@@ -0,0 +1,490 @@ ++/* ++ * ++ * (C) COPYRIGHT 2016-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++#include ++ ++#include "mali_kbase_ipa_vinstr_common.h" ++#include "mali_kbase.h" ++ ++ ++/* Performance counter blocks base offsets */ ++#define JM_BASE (0 * KBASE_IPA_NR_BYTES_PER_BLOCK) ++#define TILER_BASE (1 * KBASE_IPA_NR_BYTES_PER_BLOCK) ++#define MEMSYS_BASE (2 * KBASE_IPA_NR_BYTES_PER_BLOCK) ++ ++/* JM counter block offsets */ ++#define JM_GPU_ACTIVE (KBASE_IPA_NR_BYTES_PER_CNT * 6) ++ ++/* Tiler counter block offsets */ ++#define TILER_ACTIVE (KBASE_IPA_NR_BYTES_PER_CNT * 45) ++ ++/* MEMSYS counter block offsets */ ++#define MEMSYS_L2_ANY_LOOKUP (KBASE_IPA_NR_BYTES_PER_CNT * 25) ++ ++/* SC counter block offsets */ ++#define SC_FRAG_ACTIVE (KBASE_IPA_NR_BYTES_PER_CNT * 4) ++#define SC_EXEC_CORE_ACTIVE (KBASE_IPA_NR_BYTES_PER_CNT * 26) ++#define SC_EXEC_INSTR_FMA (KBASE_IPA_NR_BYTES_PER_CNT * 27) ++#define SC_EXEC_INSTR_COUNT (KBASE_IPA_NR_BYTES_PER_CNT * 28) ++#define SC_EXEC_INSTR_MSG (KBASE_IPA_NR_BYTES_PER_CNT * 30) ++#define SC_TEX_FILT_NUM_OPERATIONS (KBASE_IPA_NR_BYTES_PER_CNT * 39) ++#define SC_TEX_COORD_ISSUE (KBASE_IPA_NR_BYTES_PER_CNT * 40) ++#define SC_TEX_TFCH_NUM_OPERATIONS (KBASE_IPA_NR_BYTES_PER_CNT * 42) ++#define SC_VARY_INSTR (KBASE_IPA_NR_BYTES_PER_CNT * 49) ++#define SC_VARY_SLOT_32 (KBASE_IPA_NR_BYTES_PER_CNT * 50) ++#define SC_VARY_SLOT_16 (KBASE_IPA_NR_BYTES_PER_CNT * 51) ++#define SC_BEATS_RD_LSC (KBASE_IPA_NR_BYTES_PER_CNT * 56) ++#define SC_BEATS_WR_LSC (KBASE_IPA_NR_BYTES_PER_CNT * 61) ++#define SC_BEATS_WR_TIB (KBASE_IPA_NR_BYTES_PER_CNT * 62) ++ ++/** ++ * get_jm_counter() - get performance counter offset inside the Job Manager block ++ * @model_data: pointer to GPU model data. ++ * @counter_block_offset: offset in bytes of the performance counter inside the Job Manager block. ++ * ++ * Return: Block offset in bytes of the required performance counter. ++ */ ++static u32 kbase_g7x_power_model_get_jm_counter(struct kbase_ipa_model_vinstr_data *model_data, ++ u32 counter_block_offset) ++{ ++ return JM_BASE + counter_block_offset; ++} ++ ++/** ++ * get_memsys_counter() - get performance counter offset inside the Memory System block ++ * @model_data: pointer to GPU model data. ++ * @counter_block_offset: offset in bytes of the performance counter inside the (first) Memory System block. ++ * ++ * Return: Block offset in bytes of the required performance counter. ++ */ ++static u32 kbase_g7x_power_model_get_memsys_counter(struct kbase_ipa_model_vinstr_data *model_data, ++ u32 counter_block_offset) ++{ ++ /* The base address of Memory System performance counters is always the same, although their number ++ * may vary based on the number of cores. For the moment it's ok to return a constant. ++ */ ++ return MEMSYS_BASE + counter_block_offset; ++} ++ ++/** ++ * get_sc_counter() - get performance counter offset inside the Shader Cores block ++ * @model_data: pointer to GPU model data. ++ * @counter_block_offset: offset in bytes of the performance counter inside the (first) Shader Cores block. ++ * ++ * Return: Block offset in bytes of the required performance counter. ++ */ ++static u32 kbase_g7x_power_model_get_sc_counter(struct kbase_ipa_model_vinstr_data *model_data, ++ u32 counter_block_offset) ++{ ++ const u32 sc_base = MEMSYS_BASE + ++ (model_data->kbdev->gpu_props.props.l2_props.num_l2_slices * ++ KBASE_IPA_NR_BYTES_PER_BLOCK); ++ ++ return sc_base + counter_block_offset; ++} ++ ++/** ++ * memsys_single_counter() - calculate energy for a single Memory System performance counter. ++ * @model_data: pointer to GPU model data. ++ * @coeff: default value of coefficient for IPA group. ++ * @offset: offset in bytes of the counter inside the block it belongs to. ++ * ++ * Return: Energy estimation for a single Memory System performance counter. ++ */ ++static s64 kbase_g7x_sum_all_memsys_blocks( ++ struct kbase_ipa_model_vinstr_data *model_data, ++ s32 coeff, ++ u32 offset) ++{ ++ u32 counter; ++ ++ counter = kbase_g7x_power_model_get_memsys_counter(model_data, offset); ++ return kbase_ipa_sum_all_memsys_blocks(model_data, coeff, counter); ++} ++ ++/** ++ * sum_all_shader_cores() - calculate energy for a Shader Cores performance counter for all cores. ++ * @model_data: pointer to GPU model data. ++ * @coeff: default value of coefficient for IPA group. ++ * @counter_block_offset: offset in bytes of the counter inside the block it belongs to. ++ * ++ * Return: Energy estimation for a Shader Cores performance counter for all cores. ++ */ ++static s64 kbase_g7x_sum_all_shader_cores( ++ struct kbase_ipa_model_vinstr_data *model_data, ++ s32 coeff, ++ u32 counter_block_offset) ++{ ++ u32 counter; ++ ++ counter = kbase_g7x_power_model_get_sc_counter(model_data, ++ counter_block_offset); ++ return kbase_ipa_sum_all_shader_cores(model_data, coeff, counter); ++} ++ ++/** ++ * jm_single_counter() - calculate energy for a single Job Manager performance counter. ++ * @model_data: pointer to GPU model data. ++ * @coeff: default value of coefficient for IPA group. ++ * @counter_block_offset: offset in bytes of the counter inside the block it belongs to. ++ * ++ * Return: Energy estimation for a single Job Manager performance counter. ++ */ ++static s64 kbase_g7x_jm_single_counter( ++ struct kbase_ipa_model_vinstr_data *model_data, ++ s32 coeff, ++ u32 counter_block_offset) ++{ ++ u32 counter; ++ ++ counter = kbase_g7x_power_model_get_jm_counter(model_data, ++ counter_block_offset); ++ return kbase_ipa_single_counter(model_data, coeff, counter); ++} ++ ++/** ++ * get_active_cycles() - return the GPU_ACTIVE counter ++ * @model_data: pointer to GPU model data. ++ * ++ * Return: the number of cycles the GPU was active during the counter sampling ++ * period. ++ */ ++static u32 kbase_g7x_get_active_cycles( ++ struct kbase_ipa_model_vinstr_data *model_data) ++{ ++ u32 counter = kbase_g7x_power_model_get_jm_counter(model_data, JM_GPU_ACTIVE); ++ ++ /* Counters are only 32-bit, so we can safely multiply by 1 then cast ++ * the 64-bit result back to a u32. ++ */ ++ return kbase_ipa_single_counter(model_data, 1, counter); ++} ++ ++/** Table of IPA group definitions. ++ * ++ * For each IPA group, this table defines a function to access the given performance block counter (or counters, ++ * if the operation needs to be iterated on multiple blocks) and calculate energy estimation. ++ */ ++ ++static const struct kbase_ipa_group ipa_groups_def_g71[] = { ++ { ++ .name = "l2_access", ++ .default_value = 526300, ++ .op = kbase_g7x_sum_all_memsys_blocks, ++ .counter_block_offset = MEMSYS_L2_ANY_LOOKUP, ++ }, ++ { ++ .name = "exec_instr_count", ++ .default_value = 301100, ++ .op = kbase_g7x_sum_all_shader_cores, ++ .counter_block_offset = SC_EXEC_INSTR_COUNT, ++ }, ++ { ++ .name = "tex_issue", ++ .default_value = 197400, ++ .op = kbase_g7x_sum_all_shader_cores, ++ .counter_block_offset = SC_TEX_COORD_ISSUE, ++ }, ++ { ++ .name = "tile_wb", ++ .default_value = -156400, ++ .op = kbase_g7x_sum_all_shader_cores, ++ .counter_block_offset = SC_BEATS_WR_TIB, ++ }, ++ { ++ .name = "gpu_active", ++ .default_value = 115800, ++ .op = kbase_g7x_jm_single_counter, ++ .counter_block_offset = JM_GPU_ACTIVE, ++ }, ++}; ++ ++static const struct kbase_ipa_group ipa_groups_def_g72[] = { ++ { ++ .name = "l2_access", ++ .default_value = 393000, ++ .op = kbase_g7x_sum_all_memsys_blocks, ++ .counter_block_offset = MEMSYS_L2_ANY_LOOKUP, ++ }, ++ { ++ .name = "exec_instr_count", ++ .default_value = 227000, ++ .op = kbase_g7x_sum_all_shader_cores, ++ .counter_block_offset = SC_EXEC_INSTR_COUNT, ++ }, ++ { ++ .name = "tex_issue", ++ .default_value = 181900, ++ .op = kbase_g7x_sum_all_shader_cores, ++ .counter_block_offset = SC_TEX_COORD_ISSUE, ++ }, ++ { ++ .name = "tile_wb", ++ .default_value = -120200, ++ .op = kbase_g7x_sum_all_shader_cores, ++ .counter_block_offset = SC_BEATS_WR_TIB, ++ }, ++ { ++ .name = "gpu_active", ++ .default_value = 133100, ++ .op = kbase_g7x_jm_single_counter, ++ .counter_block_offset = JM_GPU_ACTIVE, ++ }, ++}; ++ ++static const struct kbase_ipa_group ipa_groups_def_g76[] = { ++ { ++ .name = "gpu_active", ++ .default_value = 122000, ++ .op = kbase_g7x_jm_single_counter, ++ .counter_block_offset = JM_GPU_ACTIVE, ++ }, ++ { ++ .name = "exec_instr_count", ++ .default_value = 488900, ++ .op = kbase_g7x_sum_all_shader_cores, ++ .counter_block_offset = SC_EXEC_INSTR_COUNT, ++ }, ++ { ++ .name = "vary_instr", ++ .default_value = 212100, ++ .op = kbase_g7x_sum_all_shader_cores, ++ .counter_block_offset = SC_VARY_INSTR, ++ }, ++ { ++ .name = "tex_tfch_num_operations", ++ .default_value = 288000, ++ .op = kbase_g7x_sum_all_shader_cores, ++ .counter_block_offset = SC_TEX_TFCH_NUM_OPERATIONS, ++ }, ++ { ++ .name = "l2_access", ++ .default_value = 378100, ++ .op = kbase_g7x_sum_all_memsys_blocks, ++ .counter_block_offset = MEMSYS_L2_ANY_LOOKUP, ++ }, ++}; ++ ++static const struct kbase_ipa_group ipa_groups_def_g52_r1[] = { ++ { ++ .name = "gpu_active", ++ .default_value = 224200, ++ .op = kbase_g7x_jm_single_counter, ++ .counter_block_offset = JM_GPU_ACTIVE, ++ }, ++ { ++ .name = "exec_instr_count", ++ .default_value = 384700, ++ .op = kbase_g7x_sum_all_shader_cores, ++ .counter_block_offset = SC_EXEC_INSTR_COUNT, ++ }, ++ { ++ .name = "vary_instr", ++ .default_value = 271900, ++ .op = kbase_g7x_sum_all_shader_cores, ++ .counter_block_offset = SC_VARY_INSTR, ++ }, ++ { ++ .name = "tex_tfch_num_operations", ++ .default_value = 477700, ++ .op = kbase_g7x_sum_all_shader_cores, ++ .counter_block_offset = SC_TEX_TFCH_NUM_OPERATIONS, ++ }, ++ { ++ .name = "l2_access", ++ .default_value = 551400, ++ .op = kbase_g7x_sum_all_memsys_blocks, ++ .counter_block_offset = MEMSYS_L2_ANY_LOOKUP, ++ }, ++}; ++ ++static const struct kbase_ipa_group ipa_groups_def_g51[] = { ++ { ++ .name = "gpu_active", ++ .default_value = 201400, ++ .op = kbase_g7x_jm_single_counter, ++ .counter_block_offset = JM_GPU_ACTIVE, ++ }, ++ { ++ .name = "exec_instr_count", ++ .default_value = 392700, ++ .op = kbase_g7x_sum_all_shader_cores, ++ .counter_block_offset = SC_EXEC_INSTR_COUNT, ++ }, ++ { ++ .name = "vary_instr", ++ .default_value = 274000, ++ .op = kbase_g7x_sum_all_shader_cores, ++ .counter_block_offset = SC_VARY_INSTR, ++ }, ++ { ++ .name = "tex_tfch_num_operations", ++ .default_value = 528000, ++ .op = kbase_g7x_sum_all_shader_cores, ++ .counter_block_offset = SC_TEX_TFCH_NUM_OPERATIONS, ++ }, ++ { ++ .name = "l2_access", ++ .default_value = 506400, ++ .op = kbase_g7x_sum_all_memsys_blocks, ++ .counter_block_offset = MEMSYS_L2_ANY_LOOKUP, ++ }, ++}; ++ ++static const struct kbase_ipa_group ipa_groups_def_g77[] = { ++ { ++ .name = "l2_access", ++ .default_value = 710800, ++ .op = kbase_g7x_sum_all_memsys_blocks, ++ .counter_block_offset = MEMSYS_L2_ANY_LOOKUP, ++ }, ++ { ++ .name = "exec_instr_msg", ++ .default_value = 2375300, ++ .op = kbase_g7x_sum_all_shader_cores, ++ .counter_block_offset = SC_EXEC_INSTR_MSG, ++ }, ++ { ++ .name = "exec_instr_fma", ++ .default_value = 656100, ++ .op = kbase_g7x_sum_all_shader_cores, ++ .counter_block_offset = SC_EXEC_INSTR_FMA, ++ }, ++ { ++ .name = "tex_filt_num_operations", ++ .default_value = 318800, ++ .op = kbase_g7x_sum_all_shader_cores, ++ .counter_block_offset = SC_TEX_FILT_NUM_OPERATIONS, ++ }, ++ { ++ .name = "gpu_active", ++ .default_value = 172800, ++ .op = kbase_g7x_jm_single_counter, ++ .counter_block_offset = JM_GPU_ACTIVE, ++ }, ++}; ++ ++static const struct kbase_ipa_group ipa_groups_def_tbex[] = { ++ { ++ .name = "l2_access", ++ .default_value = 599800, ++ .op = kbase_g7x_sum_all_memsys_blocks, ++ .counter_block_offset = MEMSYS_L2_ANY_LOOKUP, ++ }, ++ { ++ .name = "exec_instr_msg", ++ .default_value = 1830200, ++ .op = kbase_g7x_sum_all_shader_cores, ++ .counter_block_offset = SC_EXEC_INSTR_MSG, ++ }, ++ { ++ .name = "exec_instr_fma", ++ .default_value = 407300, ++ .op = kbase_g7x_sum_all_shader_cores, ++ .counter_block_offset = SC_EXEC_INSTR_FMA, ++ }, ++ { ++ .name = "tex_filt_num_operations", ++ .default_value = 224500, ++ .op = kbase_g7x_sum_all_shader_cores, ++ .counter_block_offset = SC_TEX_FILT_NUM_OPERATIONS, ++ }, ++ { ++ .name = "gpu_active", ++ .default_value = 153800, ++ .op = kbase_g7x_jm_single_counter, ++ .counter_block_offset = JM_GPU_ACTIVE, ++ }, ++}; ++ ++static const struct kbase_ipa_group ipa_groups_def_tbax[] = { ++ { ++ .name = "l2_access", ++ .default_value = 599800, ++ .op = kbase_g7x_sum_all_memsys_blocks, ++ .counter_block_offset = MEMSYS_L2_ANY_LOOKUP, ++ }, ++ { ++ .name = "exec_instr_msg", ++ .default_value = 1830200, ++ .op = kbase_g7x_sum_all_shader_cores, ++ .counter_block_offset = SC_EXEC_INSTR_MSG, ++ }, ++ { ++ .name = "exec_instr_fma", ++ .default_value = 407300, ++ .op = kbase_g7x_sum_all_shader_cores, ++ .counter_block_offset = SC_EXEC_INSTR_FMA, ++ }, ++ { ++ .name = "tex_filt_num_operations", ++ .default_value = 224500, ++ .op = kbase_g7x_sum_all_shader_cores, ++ .counter_block_offset = SC_TEX_FILT_NUM_OPERATIONS, ++ }, ++ { ++ .name = "gpu_active", ++ .default_value = 153800, ++ .op = kbase_g7x_jm_single_counter, ++ .counter_block_offset = JM_GPU_ACTIVE, ++ }, ++}; ++ ++ ++#define IPA_POWER_MODEL_OPS(gpu, init_token) \ ++ const struct kbase_ipa_model_ops kbase_ ## gpu ## _ipa_model_ops = { \ ++ .name = "mali-" #gpu "-power-model", \ ++ .init = kbase_ ## init_token ## _power_model_init, \ ++ .term = kbase_ipa_vinstr_common_model_term, \ ++ .get_dynamic_coeff = kbase_ipa_vinstr_dynamic_coeff, \ ++ }; \ ++ KBASE_EXPORT_TEST_API(kbase_ ## gpu ## _ipa_model_ops) ++ ++#define STANDARD_POWER_MODEL(gpu, reference_voltage) \ ++ static int kbase_ ## gpu ## _power_model_init(\ ++ struct kbase_ipa_model *model) \ ++ { \ ++ BUILD_BUG_ON(ARRAY_SIZE(ipa_groups_def_ ## gpu) > \ ++ KBASE_IPA_MAX_GROUP_DEF_NUM); \ ++ return kbase_ipa_vinstr_common_model_init(model, \ ++ ipa_groups_def_ ## gpu, \ ++ ARRAY_SIZE(ipa_groups_def_ ## gpu), \ ++ kbase_g7x_get_active_cycles, \ ++ (reference_voltage)); \ ++ } \ ++ IPA_POWER_MODEL_OPS(gpu, gpu) ++ ++#define ALIAS_POWER_MODEL(gpu, as_gpu) \ ++ IPA_POWER_MODEL_OPS(gpu, as_gpu) ++ ++STANDARD_POWER_MODEL(g71, 800); ++STANDARD_POWER_MODEL(g72, 800); ++STANDARD_POWER_MODEL(g76, 800); ++STANDARD_POWER_MODEL(g52_r1, 1000); ++STANDARD_POWER_MODEL(g51, 1000); ++STANDARD_POWER_MODEL(g77, 1000); ++STANDARD_POWER_MODEL(tbex, 1000); ++STANDARD_POWER_MODEL(tbax, 1000); ++ ++/* g52 is an alias of g76 (TNOX) for IPA */ ++ALIAS_POWER_MODEL(g52, g76); ++/* tnax is an alias of g77 (TTRX) for IPA */ ++ALIAS_POWER_MODEL(tnax, g77); +diff --git a/drivers/gpu/arm/bifrost/jm/mali_base_jm_kernel.h b/drivers/gpu/arm/bifrost/jm/mali_base_jm_kernel.h +new file mode 100755 +index 000000000000..9367cc5431cf +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/jm/mali_base_jm_kernel.h +@@ -0,0 +1,1079 @@ ++/* ++ * ++ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++#ifndef _BASE_JM_KERNEL_H_ ++#define _BASE_JM_KERNEL_H_ ++ ++/* Memory allocation, access/hint flags. ++ * ++ * See base_mem_alloc_flags. ++ */ ++ ++/* IN */ ++/* Read access CPU side ++ */ ++#define BASE_MEM_PROT_CPU_RD ((base_mem_alloc_flags)1 << 0) ++ ++/* Write access CPU side ++ */ ++#define BASE_MEM_PROT_CPU_WR ((base_mem_alloc_flags)1 << 1) ++ ++/* Read access GPU side ++ */ ++#define BASE_MEM_PROT_GPU_RD ((base_mem_alloc_flags)1 << 2) ++ ++/* Write access GPU side ++ */ ++#define BASE_MEM_PROT_GPU_WR ((base_mem_alloc_flags)1 << 3) ++ ++/* Execute allowed on the GPU side ++ */ ++#define BASE_MEM_PROT_GPU_EX ((base_mem_alloc_flags)1 << 4) ++ ++/* Will be permanently mapped in kernel space. ++ * Flag is only allowed on allocations originating from kbase. ++ */ ++#define BASEP_MEM_PERMANENT_KERNEL_MAPPING ((base_mem_alloc_flags)1 << 5) ++ ++/* The allocation will completely reside within the same 4GB chunk in the GPU ++ * virtual space. ++ * Since this flag is primarily required only for the TLS memory which will ++ * not be used to contain executable code and also not used for Tiler heap, ++ * it can't be used along with BASE_MEM_PROT_GPU_EX and TILER_ALIGN_TOP flags. ++ */ ++#define BASE_MEM_GPU_VA_SAME_4GB_PAGE ((base_mem_alloc_flags)1 << 6) ++ ++/* Userspace is not allowed to free this memory. ++ * Flag is only allowed on allocations originating from kbase. ++ */ ++#define BASEP_MEM_NO_USER_FREE ((base_mem_alloc_flags)1 << 7) ++ ++#define BASE_MEM_RESERVED_BIT_8 ((base_mem_alloc_flags)1 << 8) ++ ++/* Grow backing store on GPU Page Fault ++ */ ++#define BASE_MEM_GROW_ON_GPF ((base_mem_alloc_flags)1 << 9) ++ ++/* Page coherence Outer shareable, if available ++ */ ++#define BASE_MEM_COHERENT_SYSTEM ((base_mem_alloc_flags)1 << 10) ++ ++/* Page coherence Inner shareable ++ */ ++#define BASE_MEM_COHERENT_LOCAL ((base_mem_alloc_flags)1 << 11) ++ ++/* IN/OUT */ ++/* Should be cached on the CPU, returned if actually cached ++ */ ++#define BASE_MEM_CACHED_CPU ((base_mem_alloc_flags)1 << 12) ++ ++/* IN/OUT */ ++/* Must have same VA on both the GPU and the CPU ++ */ ++#define BASE_MEM_SAME_VA ((base_mem_alloc_flags)1 << 13) ++ ++/* OUT */ ++/* Must call mmap to acquire a GPU address for the allocation ++ */ ++#define BASE_MEM_NEED_MMAP ((base_mem_alloc_flags)1 << 14) ++ ++/* IN */ ++/* Page coherence Outer shareable, required. ++ */ ++#define BASE_MEM_COHERENT_SYSTEM_REQUIRED ((base_mem_alloc_flags)1 << 15) ++ ++/* Protected memory ++ */ ++#define BASE_MEM_PROTECTED ((base_mem_alloc_flags)1 << 16) ++ ++/* Not needed physical memory ++ */ ++#define BASE_MEM_DONT_NEED ((base_mem_alloc_flags)1 << 17) ++ ++/* Must use shared CPU/GPU zone (SAME_VA zone) but doesn't require the ++ * addresses to be the same ++ */ ++#define BASE_MEM_IMPORT_SHARED ((base_mem_alloc_flags)1 << 18) ++ ++/** ++ * Bit 19 is reserved. ++ * ++ * Do not remove, use the next unreserved bit for new flags ++ */ ++#define BASE_MEM_RESERVED_BIT_19 ((base_mem_alloc_flags)1 << 19) ++ ++/** ++ * Memory starting from the end of the initial commit is aligned to 'extent' ++ * pages, where 'extent' must be a power of 2 and no more than ++ * BASE_MEM_TILER_ALIGN_TOP_EXTENT_MAX_PAGES ++ */ ++#define BASE_MEM_TILER_ALIGN_TOP ((base_mem_alloc_flags)1 << 20) ++ ++/* Should be uncached on the GPU, will work only for GPUs using AARCH64 mmu ++ * mode. Some components within the GPU might only be able to access memory ++ * that is GPU cacheable. Refer to the specific GPU implementation for more ++ * details. The 3 shareability flags will be ignored for GPU uncached memory. ++ * If used while importing USER_BUFFER type memory, then the import will fail ++ * if the memory is not aligned to GPU and CPU cache line width. ++ */ ++#define BASE_MEM_UNCACHED_GPU ((base_mem_alloc_flags)1 << 21) ++ ++/* ++ * Bits [22:25] for group_id (0~15). ++ * ++ * base_mem_group_id_set() should be used to pack a memory group ID into a ++ * base_mem_alloc_flags value instead of accessing the bits directly. ++ * base_mem_group_id_get() should be used to extract the memory group ID from ++ * a base_mem_alloc_flags value. ++ */ ++#define BASEP_MEM_GROUP_ID_SHIFT 22 ++#define BASE_MEM_GROUP_ID_MASK \ ++ ((base_mem_alloc_flags)0xF << BASEP_MEM_GROUP_ID_SHIFT) ++ ++/* Must do CPU cache maintenance when imported memory is mapped/unmapped ++ * on GPU. Currently applicable to dma-buf type only. ++ */ ++#define BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP ((base_mem_alloc_flags)1 << 26) ++ ++/* Use the GPU VA chosen by the kernel client */ ++#define BASE_MEM_FLAG_MAP_FIXED ((base_mem_alloc_flags)1 << 27) ++ ++/* OUT */ ++/* Kernel side cache sync ops required */ ++#define BASE_MEM_KERNEL_SYNC ((base_mem_alloc_flags)1 << 28) ++ ++/* Force trimming of JIT allocations when creating a new allocation */ ++#define BASEP_MEM_PERFORM_JIT_TRIM ((base_mem_alloc_flags)1 << 29) ++ ++/* Number of bits used as flags for base memory management ++ * ++ * Must be kept in sync with the base_mem_alloc_flags flags ++ */ ++#define BASE_MEM_FLAGS_NR_BITS 30 ++ ++/* A mask of all the flags which are only valid for allocations within kbase, ++ * and may not be passed from user space. ++ */ ++#define BASEP_MEM_FLAGS_KERNEL_ONLY \ ++ (BASEP_MEM_PERMANENT_KERNEL_MAPPING | BASEP_MEM_NO_USER_FREE | \ ++ BASE_MEM_FLAG_MAP_FIXED | BASEP_MEM_PERFORM_JIT_TRIM) ++ ++/* A mask for all output bits, excluding IN/OUT bits. ++ */ ++#define BASE_MEM_FLAGS_OUTPUT_MASK BASE_MEM_NEED_MMAP ++ ++/* A mask for all input bits, including IN/OUT bits. ++ */ ++#define BASE_MEM_FLAGS_INPUT_MASK \ ++ (((1 << BASE_MEM_FLAGS_NR_BITS) - 1) & ~BASE_MEM_FLAGS_OUTPUT_MASK) ++ ++/* A mask of all currently reserved flags ++ */ ++#define BASE_MEM_FLAGS_RESERVED \ ++ (BASE_MEM_RESERVED_BIT_8 | BASE_MEM_RESERVED_BIT_19) ++ ++#define BASEP_MEM_INVALID_HANDLE (0ull << 12) ++#define BASE_MEM_MMU_DUMP_HANDLE (1ull << 12) ++#define BASE_MEM_TRACE_BUFFER_HANDLE (2ull << 12) ++#define BASE_MEM_MAP_TRACKING_HANDLE (3ull << 12) ++#define BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE (4ull << 12) ++/* reserved handles ..-47< for future special handles */ ++#define BASE_MEM_COOKIE_BASE (64ul << 12) ++#define BASE_MEM_FIRST_FREE_ADDRESS ((BITS_PER_LONG << 12) + \ ++ BASE_MEM_COOKIE_BASE) ++ ++/* Similar to BASE_MEM_TILER_ALIGN_TOP, memory starting from the end of the ++ * initial commit is aligned to 'extent' pages, where 'extent' must be a power ++ * of 2 and no more than BASE_MEM_TILER_ALIGN_TOP_EXTENT_MAX_PAGES ++ */ ++#define BASE_JIT_ALLOC_MEM_TILER_ALIGN_TOP (1 << 0) ++ ++/** ++ * If set, the heap info address points to a u32 holding the used size in bytes; ++ * otherwise it points to a u64 holding the lowest address of unused memory. ++ */ ++#define BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE (1 << 1) ++ ++/** ++ * Valid set of just-in-time memory allocation flags ++ * ++ * Note: BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE cannot be set if heap_info_gpu_addr ++ * in %base_jit_alloc_info is 0 (atom with BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE set ++ * and heap_info_gpu_addr being 0 will be rejected). ++ */ ++#define BASE_JIT_ALLOC_VALID_FLAGS \ ++ (BASE_JIT_ALLOC_MEM_TILER_ALIGN_TOP | BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE) ++ ++/** ++ * typedef base_context_create_flags - Flags to pass to ::base_context_init. ++ * ++ * Flags can be ORed together to enable multiple things. ++ * ++ * These share the same space as BASEP_CONTEXT_FLAG_*, and so must ++ * not collide with them. ++ */ ++typedef u32 base_context_create_flags; ++ ++/* No flags set */ ++#define BASE_CONTEXT_CREATE_FLAG_NONE ((base_context_create_flags)0) ++ ++/* Base context is embedded in a cctx object (flag used for CINSTR ++ * software counter macros) ++ */ ++#define BASE_CONTEXT_CCTX_EMBEDDED ((base_context_create_flags)1 << 0) ++ ++/* Base context is a 'System Monitor' context for Hardware counters. ++ * ++ * One important side effect of this is that job submission is disabled. ++ */ ++#define BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED \ ++ ((base_context_create_flags)1 << 1) ++ ++/* Bit-shift used to encode a memory group ID in base_context_create_flags ++ */ ++#define BASEP_CONTEXT_MMU_GROUP_ID_SHIFT (3) ++ ++/* Bitmask used to encode a memory group ID in base_context_create_flags ++ */ ++#define BASEP_CONTEXT_MMU_GROUP_ID_MASK \ ++ ((base_context_create_flags)0xF << BASEP_CONTEXT_MMU_GROUP_ID_SHIFT) ++ ++/* Bitpattern describing the base_context_create_flags that can be ++ * passed to the kernel ++ */ ++#define BASEP_CONTEXT_CREATE_KERNEL_FLAGS \ ++ (BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED | \ ++ BASEP_CONTEXT_MMU_GROUP_ID_MASK) ++ ++/* Bitpattern describing the ::base_context_create_flags that can be ++ * passed to base_context_init() ++ */ ++#define BASEP_CONTEXT_CREATE_ALLOWED_FLAGS \ ++ (BASE_CONTEXT_CCTX_EMBEDDED | BASEP_CONTEXT_CREATE_KERNEL_FLAGS) ++ ++/* ++ * Private flags used on the base context ++ * ++ * These start at bit 31, and run down to zero. ++ * ++ * They share the same space as base_context_create_flags, and so must ++ * not collide with them. ++ */ ++ ++/* Private flag tracking whether job descriptor dumping is disabled */ ++#define BASEP_CONTEXT_FLAG_JOB_DUMP_DISABLED \ ++ ((base_context_create_flags)(1 << 31)) ++ ++/* Enable additional tracepoints for latency measurements (TL_ATOM_READY, ++ * TL_ATOM_DONE, TL_ATOM_PRIO_CHANGE, TL_ATOM_EVENT_POST) ++ */ ++#define BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS (1 << 0) ++ ++/* Indicate that job dumping is enabled. This could affect certain timers ++ * to account for the performance impact. ++ */ ++#define BASE_TLSTREAM_JOB_DUMPING_ENABLED (1 << 1) ++ ++#define BASE_TLSTREAM_FLAGS_MASK (BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS | \ ++ BASE_TLSTREAM_JOB_DUMPING_ENABLED) ++/* ++ * Dependency stuff, keep it private for now. May want to expose it if ++ * we decide to make the number of semaphores a configurable ++ * option. ++ */ ++#define BASE_JD_ATOM_COUNT 256 ++ ++/* Maximum number of concurrent render passes. ++ */ ++#define BASE_JD_RP_COUNT (256) ++ ++/* Set/reset values for a software event */ ++#define BASE_JD_SOFT_EVENT_SET ((unsigned char)1) ++#define BASE_JD_SOFT_EVENT_RESET ((unsigned char)0) ++ ++/** ++ * struct base_jd_udata - Per-job data ++ * ++ * This structure is used to store per-job data, and is completely unused ++ * by the Base driver. It can be used to store things such as callback ++ * function pointer, data to handle job completion. It is guaranteed to be ++ * untouched by the Base driver. ++ * ++ * @blob: per-job data array ++ */ ++struct base_jd_udata { ++ u64 blob[2]; ++}; ++ ++/** ++ * typedef base_jd_dep_type - Job dependency type. ++ * ++ * A flags field will be inserted into the atom structure to specify whether a ++ * dependency is a data or ordering dependency (by putting it before/after ++ * 'core_req' in the structure it should be possible to add without changing ++ * the structure size). ++ * When the flag is set for a particular dependency to signal that it is an ++ * ordering only dependency then errors will not be propagated. ++ */ ++typedef u8 base_jd_dep_type; ++ ++#define BASE_JD_DEP_TYPE_INVALID (0) /**< Invalid dependency */ ++#define BASE_JD_DEP_TYPE_DATA (1U << 0) /**< Data dependency */ ++#define BASE_JD_DEP_TYPE_ORDER (1U << 1) /**< Order dependency */ ++ ++/** ++ * typedef base_jd_core_req - Job chain hardware requirements. ++ * ++ * A job chain must specify what GPU features it needs to allow the ++ * driver to schedule the job correctly. By not specifying the ++ * correct settings can/will cause an early job termination. Multiple ++ * values can be ORed together to specify multiple requirements. ++ * Special case is ::BASE_JD_REQ_DEP, which is used to express complex ++ * dependencies, and that doesn't execute anything on the hardware. ++ */ ++typedef u32 base_jd_core_req; ++ ++/* Requirements that come from the HW */ ++ ++/* No requirement, dependency only ++ */ ++#define BASE_JD_REQ_DEP ((base_jd_core_req)0) ++ ++/* Requires fragment shaders ++ */ ++#define BASE_JD_REQ_FS ((base_jd_core_req)1 << 0) ++ ++/* Requires compute shaders ++ * ++ * This covers any of the following GPU job types: ++ * - Vertex Shader Job ++ * - Geometry Shader Job ++ * - An actual Compute Shader Job ++ * ++ * Compare this with BASE_JD_REQ_ONLY_COMPUTE, which specifies that the ++ * job is specifically just the "Compute Shader" job type, and not the "Vertex ++ * Shader" nor the "Geometry Shader" job type. ++ */ ++#define BASE_JD_REQ_CS ((base_jd_core_req)1 << 1) ++ ++/* Requires tiling */ ++#define BASE_JD_REQ_T ((base_jd_core_req)1 << 2) ++ ++/* Requires cache flushes */ ++#define BASE_JD_REQ_CF ((base_jd_core_req)1 << 3) ++ ++/* Requires value writeback */ ++#define BASE_JD_REQ_V ((base_jd_core_req)1 << 4) ++ ++/* SW-only requirements - the HW does not expose these as part of the job slot ++ * capabilities ++ */ ++ ++/* Requires fragment job with AFBC encoding */ ++#define BASE_JD_REQ_FS_AFBC ((base_jd_core_req)1 << 13) ++ ++/* SW-only requirement: coalesce completion events. ++ * If this bit is set then completion of this atom will not cause an event to ++ * be sent to userspace, whether successful or not; completion events will be ++ * deferred until an atom completes which does not have this bit set. ++ * ++ * This bit may not be used in combination with BASE_JD_REQ_EXTERNAL_RESOURCES. ++ */ ++#define BASE_JD_REQ_EVENT_COALESCE ((base_jd_core_req)1 << 5) ++ ++/* SW Only requirement: the job chain requires a coherent core group. We don't ++ * mind which coherent core group is used. ++ */ ++#define BASE_JD_REQ_COHERENT_GROUP ((base_jd_core_req)1 << 6) ++ ++/* SW Only requirement: The performance counters should be enabled only when ++ * they are needed, to reduce power consumption. ++ */ ++#define BASE_JD_REQ_PERMON ((base_jd_core_req)1 << 7) ++ ++/* SW Only requirement: External resources are referenced by this atom. ++ * ++ * This bit may not be used in combination with BASE_JD_REQ_EVENT_COALESCE and ++ * BASE_JD_REQ_SOFT_EVENT_WAIT. ++ */ ++#define BASE_JD_REQ_EXTERNAL_RESOURCES ((base_jd_core_req)1 << 8) ++ ++/* SW Only requirement: Software defined job. Jobs with this bit set will not be ++ * submitted to the hardware but will cause some action to happen within the ++ * driver ++ */ ++#define BASE_JD_REQ_SOFT_JOB ((base_jd_core_req)1 << 9) ++ ++#define BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME (BASE_JD_REQ_SOFT_JOB | 0x1) ++#define BASE_JD_REQ_SOFT_FENCE_TRIGGER (BASE_JD_REQ_SOFT_JOB | 0x2) ++#define BASE_JD_REQ_SOFT_FENCE_WAIT (BASE_JD_REQ_SOFT_JOB | 0x3) ++ ++/* 0x4 RESERVED for now */ ++ ++/* SW only requirement: event wait/trigger job. ++ * ++ * - BASE_JD_REQ_SOFT_EVENT_WAIT: this job will block until the event is set. ++ * - BASE_JD_REQ_SOFT_EVENT_SET: this job sets the event, thus unblocks the ++ * other waiting jobs. It completes immediately. ++ * - BASE_JD_REQ_SOFT_EVENT_RESET: this job resets the event, making it ++ * possible for other jobs to wait upon. It completes immediately. ++ */ ++#define BASE_JD_REQ_SOFT_EVENT_WAIT (BASE_JD_REQ_SOFT_JOB | 0x5) ++#define BASE_JD_REQ_SOFT_EVENT_SET (BASE_JD_REQ_SOFT_JOB | 0x6) ++#define BASE_JD_REQ_SOFT_EVENT_RESET (BASE_JD_REQ_SOFT_JOB | 0x7) ++ ++#define BASE_JD_REQ_SOFT_DEBUG_COPY (BASE_JD_REQ_SOFT_JOB | 0x8) ++ ++/* SW only requirement: Just In Time allocation ++ * ++ * This job requests a single or multiple just-in-time allocations through a ++ * list of base_jit_alloc_info structure which is passed via the jc element of ++ * the atom. The number of base_jit_alloc_info structures present in the ++ * list is passed via the nr_extres element of the atom ++ * ++ * It should be noted that the id entry in base_jit_alloc_info must not ++ * be reused until it has been released via BASE_JD_REQ_SOFT_JIT_FREE. ++ * ++ * Should this soft job fail it is expected that a BASE_JD_REQ_SOFT_JIT_FREE ++ * soft job to free the JIT allocation is still made. ++ * ++ * The job will complete immediately. ++ */ ++#define BASE_JD_REQ_SOFT_JIT_ALLOC (BASE_JD_REQ_SOFT_JOB | 0x9) ++ ++/* SW only requirement: Just In Time free ++ * ++ * This job requests a single or multiple just-in-time allocations created by ++ * BASE_JD_REQ_SOFT_JIT_ALLOC to be freed. The ID list of the just-in-time ++ * allocations is passed via the jc element of the atom. ++ * ++ * The job will complete immediately. ++ */ ++#define BASE_JD_REQ_SOFT_JIT_FREE (BASE_JD_REQ_SOFT_JOB | 0xa) ++ ++/* SW only requirement: Map external resource ++ * ++ * This job requests external resource(s) are mapped once the dependencies ++ * of the job have been satisfied. The list of external resources are ++ * passed via the jc element of the atom which is a pointer to a ++ * base_external_resource_list. ++ */ ++#define BASE_JD_REQ_SOFT_EXT_RES_MAP (BASE_JD_REQ_SOFT_JOB | 0xb) ++ ++/* SW only requirement: Unmap external resource ++ * ++ * This job requests external resource(s) are unmapped once the dependencies ++ * of the job has been satisfied. The list of external resources are ++ * passed via the jc element of the atom which is a pointer to a ++ * base_external_resource_list. ++ */ ++#define BASE_JD_REQ_SOFT_EXT_RES_UNMAP (BASE_JD_REQ_SOFT_JOB | 0xc) ++ ++/* HW Requirement: Requires Compute shaders (but not Vertex or Geometry Shaders) ++ * ++ * This indicates that the Job Chain contains GPU jobs of the 'Compute ++ * Shaders' type. ++ * ++ * In contrast to BASE_JD_REQ_CS, this does not indicate that the Job ++ * Chain contains 'Geometry Shader' or 'Vertex Shader' jobs. ++ */ ++#define BASE_JD_REQ_ONLY_COMPUTE ((base_jd_core_req)1 << 10) ++ ++/* HW Requirement: Use the base_jd_atom::device_nr field to specify a ++ * particular core group ++ * ++ * If both BASE_JD_REQ_COHERENT_GROUP and this flag are set, this flag ++ * takes priority ++ * ++ * This is only guaranteed to work for BASE_JD_REQ_ONLY_COMPUTE atoms. ++ * ++ * If the core availability policy is keeping the required core group turned ++ * off, then the job will fail with a BASE_JD_EVENT_PM_EVENT error code. ++ */ ++#define BASE_JD_REQ_SPECIFIC_COHERENT_GROUP ((base_jd_core_req)1 << 11) ++ ++/* SW Flag: If this bit is set then the successful completion of this atom ++ * will not cause an event to be sent to userspace ++ */ ++#define BASE_JD_REQ_EVENT_ONLY_ON_FAILURE ((base_jd_core_req)1 << 12) ++ ++/* SW Flag: If this bit is set then completion of this atom will not cause an ++ * event to be sent to userspace, whether successful or not. ++ */ ++#define BASEP_JD_REQ_EVENT_NEVER ((base_jd_core_req)1 << 14) ++ ++/* SW Flag: Skip GPU cache clean and invalidation before starting a GPU job. ++ * ++ * If this bit is set then the GPU's cache will not be cleaned and invalidated ++ * until a GPU job starts which does not have this bit set or a job completes ++ * which does not have the BASE_JD_REQ_SKIP_CACHE_END bit set. Do not use ++ * if the CPU may have written to memory addressed by the job since the last job ++ * without this bit set was submitted. ++ */ ++#define BASE_JD_REQ_SKIP_CACHE_START ((base_jd_core_req)1 << 15) ++ ++/* SW Flag: Skip GPU cache clean and invalidation after a GPU job completes. ++ * ++ * If this bit is set then the GPU's cache will not be cleaned and invalidated ++ * until a GPU job completes which does not have this bit set or a job starts ++ * which does not have the BASE_JD_REQ_SKIP_CACHE_START bit set. Do not use ++ * if the CPU may read from or partially overwrite memory addressed by the job ++ * before the next job without this bit set completes. ++ */ ++#define BASE_JD_REQ_SKIP_CACHE_END ((base_jd_core_req)1 << 16) ++ ++/* Request the atom be executed on a specific job slot. ++ * ++ * When this flag is specified, it takes precedence over any existing job slot ++ * selection logic. ++ */ ++#define BASE_JD_REQ_JOB_SLOT ((base_jd_core_req)1 << 17) ++ ++/* SW-only requirement: The atom is the start of a renderpass. ++ * ++ * If this bit is set then the job chain will be soft-stopped if it causes the ++ * GPU to write beyond the end of the physical pages backing the tiler heap, and ++ * committing more memory to the heap would exceed an internal threshold. It may ++ * be resumed after running one of the job chains attached to an atom with ++ * BASE_JD_REQ_END_RENDERPASS set and the same renderpass ID. It may be ++ * resumed multiple times until it completes without memory usage exceeding the ++ * threshold. ++ * ++ * Usually used with BASE_JD_REQ_T. ++ */ ++#define BASE_JD_REQ_START_RENDERPASS ((base_jd_core_req)1 << 18) ++ ++/* SW-only requirement: The atom is the end of a renderpass. ++ * ++ * If this bit is set then the atom incorporates the CPU address of a ++ * base_jd_fragment object instead of the GPU address of a job chain. ++ * ++ * Which job chain is run depends upon whether the atom with the same renderpass ++ * ID and the BASE_JD_REQ_START_RENDERPASS bit set completed normally or ++ * was soft-stopped when it exceeded an upper threshold for tiler heap memory ++ * usage. ++ * ++ * It also depends upon whether one of the job chains attached to the atom has ++ * already been run as part of the same renderpass (in which case it would have ++ * written unresolved multisampled and otherwise-discarded output to temporary ++ * buffers that need to be read back). The job chain for doing a forced read and ++ * forced write (from/to temporary buffers) is run as many times as necessary. ++ * ++ * Usually used with BASE_JD_REQ_FS. ++ */ ++#define BASE_JD_REQ_END_RENDERPASS ((base_jd_core_req)1 << 19) ++ ++/* These requirement bits are currently unused in base_jd_core_req ++ */ ++#define BASEP_JD_REQ_RESERVED \ ++ (~(BASE_JD_REQ_ATOM_TYPE | BASE_JD_REQ_EXTERNAL_RESOURCES | \ ++ BASE_JD_REQ_EVENT_ONLY_ON_FAILURE | BASEP_JD_REQ_EVENT_NEVER | \ ++ BASE_JD_REQ_EVENT_COALESCE | \ ++ BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_SPECIFIC_COHERENT_GROUP | \ ++ BASE_JD_REQ_FS_AFBC | BASE_JD_REQ_PERMON | \ ++ BASE_JD_REQ_SKIP_CACHE_START | BASE_JD_REQ_SKIP_CACHE_END | \ ++ BASE_JD_REQ_JOB_SLOT | BASE_JD_REQ_START_RENDERPASS | \ ++ BASE_JD_REQ_END_RENDERPASS)) ++ ++/* Mask of all bits in base_jd_core_req that control the type of the atom. ++ * ++ * This allows dependency only atoms to have flags set ++ */ ++#define BASE_JD_REQ_ATOM_TYPE \ ++ (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T | BASE_JD_REQ_CF | \ ++ BASE_JD_REQ_V | BASE_JD_REQ_SOFT_JOB | BASE_JD_REQ_ONLY_COMPUTE) ++ ++/** ++ * Mask of all bits in base_jd_core_req that control the type of a soft job. ++ */ ++#define BASE_JD_REQ_SOFT_JOB_TYPE (BASE_JD_REQ_SOFT_JOB | 0x1f) ++ ++/* Returns non-zero value if core requirements passed define a soft job or ++ * a dependency only job. ++ */ ++#define BASE_JD_REQ_SOFT_JOB_OR_DEP(core_req) \ ++ (((core_req) & BASE_JD_REQ_SOFT_JOB) || \ ++ ((core_req) & BASE_JD_REQ_ATOM_TYPE) == BASE_JD_REQ_DEP) ++ ++/** ++ * enum kbase_jd_atom_state ++ * ++ * @KBASE_JD_ATOM_STATE_UNUSED: Atom is not used. ++ * @KBASE_JD_ATOM_STATE_QUEUED: Atom is queued in JD. ++ * @KBASE_JD_ATOM_STATE_IN_JS: Atom has been given to JS (is runnable/running). ++ * @KBASE_JD_ATOM_STATE_HW_COMPLETED: Atom has been completed, but not yet ++ * handed back to job dispatcher for ++ * dependency resolution. ++ * @KBASE_JD_ATOM_STATE_COMPLETED: Atom has been completed, but not yet handed ++ * back to userspace. ++ */ ++enum kbase_jd_atom_state { ++ KBASE_JD_ATOM_STATE_UNUSED, ++ KBASE_JD_ATOM_STATE_QUEUED, ++ KBASE_JD_ATOM_STATE_IN_JS, ++ KBASE_JD_ATOM_STATE_HW_COMPLETED, ++ KBASE_JD_ATOM_STATE_COMPLETED ++}; ++ ++/** ++ * typedef base_atom_id - Type big enough to store an atom number in. ++ */ ++typedef u8 base_atom_id; ++ ++/** ++ * struct base_dependency - ++ * ++ * @atom_id: An atom number ++ * @dependency_type: Dependency type ++ */ ++struct base_dependency { ++ base_atom_id atom_id; ++ base_jd_dep_type dependency_type; ++}; ++ ++/** ++ * struct base_jd_fragment - Set of GPU fragment job chains used for rendering. ++ * ++ * @norm_read_norm_write: Job chain for full rendering. ++ * GPU address of a fragment job chain to render in the ++ * circumstance where the tiler job chain did not exceed ++ * its memory usage threshold and no fragment job chain ++ * was previously run for the same renderpass. ++ * It is used no more than once per renderpass. ++ * @norm_read_forced_write: Job chain for starting incremental ++ * rendering. ++ * GPU address of a fragment job chain to render in ++ * the circumstance where the tiler job chain exceeded ++ * its memory usage threshold for the first time and ++ * no fragment job chain was previously run for the ++ * same renderpass. ++ * Writes unresolved multisampled and normally- ++ * discarded output to temporary buffers that must be ++ * read back by a subsequent forced_read job chain ++ * before the renderpass is complete. ++ * It is used no more than once per renderpass. ++ * @forced_read_forced_write: Job chain for continuing incremental ++ * rendering. ++ * GPU address of a fragment job chain to render in ++ * the circumstance where the tiler job chain ++ * exceeded its memory usage threshold again ++ * and a fragment job chain was previously run for ++ * the same renderpass. ++ * Reads unresolved multisampled and ++ * normally-discarded output from temporary buffers ++ * written by a previous forced_write job chain and ++ * writes the same to temporary buffers again. ++ * It is used as many times as required until ++ * rendering completes. ++ * @forced_read_norm_write: Job chain for ending incremental rendering. ++ * GPU address of a fragment job chain to render in the ++ * circumstance where the tiler job chain did not ++ * exceed its memory usage threshold this time and a ++ * fragment job chain was previously run for the same ++ * renderpass. ++ * Reads unresolved multisampled and normally-discarded ++ * output from temporary buffers written by a previous ++ * forced_write job chain in order to complete a ++ * renderpass. ++ * It is used no more than once per renderpass. ++ * ++ * This structure is referenced by the main atom structure if ++ * BASE_JD_REQ_END_RENDERPASS is set in the base_jd_core_req. ++ */ ++struct base_jd_fragment { ++ u64 norm_read_norm_write; ++ u64 norm_read_forced_write; ++ u64 forced_read_forced_write; ++ u64 forced_read_norm_write; ++}; ++ ++/** ++ * typedef base_jd_prio - Base Atom priority. ++ * ++ * Only certain priority levels are actually implemented, as specified by the ++ * BASE_JD_PRIO_<...> definitions below. It is undefined to use a priority ++ * level that is not one of those defined below. ++ * ++ * Priority levels only affect scheduling after the atoms have had dependencies ++ * resolved. For example, a low priority atom that has had its dependencies ++ * resolved might run before a higher priority atom that has not had its ++ * dependencies resolved. ++ * ++ * In general, fragment atoms do not affect non-fragment atoms with ++ * lower priorities, and vice versa. One exception is that there is only one ++ * priority value for each context. So a high-priority (e.g.) fragment atom ++ * could increase its context priority, causing its non-fragment atoms to also ++ * be scheduled sooner. ++ * ++ * The atoms are scheduled as follows with respect to their priorities: ++ * * Let atoms 'X' and 'Y' be for the same job slot who have dependencies ++ * resolved, and atom 'X' has a higher priority than atom 'Y' ++ * * If atom 'Y' is currently running on the HW, then it is interrupted to ++ * allow atom 'X' to run soon after ++ * * If instead neither atom 'Y' nor atom 'X' are running, then when choosing ++ * the next atom to run, atom 'X' will always be chosen instead of atom 'Y' ++ * * Any two atoms that have the same priority could run in any order with ++ * respect to each other. That is, there is no ordering constraint between ++ * atoms of the same priority. ++ * ++ * The sysfs file 'js_ctx_scheduling_mode' is used to control how atoms are ++ * scheduled between contexts. The default value, 0, will cause higher-priority ++ * atoms to be scheduled first, regardless of their context. The value 1 will ++ * use a round-robin algorithm when deciding which context's atoms to schedule ++ * next, so higher-priority atoms can only preempt lower priority atoms within ++ * the same context. See KBASE_JS_SYSTEM_PRIORITY_MODE and ++ * KBASE_JS_PROCESS_LOCAL_PRIORITY_MODE for more details. ++ */ ++typedef u8 base_jd_prio; ++ ++/* Medium atom priority. This is a priority higher than BASE_JD_PRIO_LOW */ ++#define BASE_JD_PRIO_MEDIUM ((base_jd_prio)0) ++/* High atom priority. This is a priority higher than BASE_JD_PRIO_MEDIUM and ++ * BASE_JD_PRIO_LOW ++ */ ++#define BASE_JD_PRIO_HIGH ((base_jd_prio)1) ++/* Low atom priority. */ ++#define BASE_JD_PRIO_LOW ((base_jd_prio)2) ++ ++/* Count of the number of priority levels. This itself is not a valid ++ * base_jd_prio setting ++ */ ++#define BASE_JD_NR_PRIO_LEVELS 3 ++ ++/** ++ * struct base_jd_atom_v2 - Node of a dependency graph used to submit a ++ * GPU job chain or soft-job to the kernel driver. ++ * ++ * @jc: GPU address of a job chain or (if BASE_JD_REQ_END_RENDERPASS ++ * is set in the base_jd_core_req) the CPU address of a ++ * base_jd_fragment object. ++ * @udata: User data. ++ * @extres_list: List of external resources. ++ * @nr_extres: Number of external resources or JIT allocations. ++ * @jit_id: Zero-terminated array of IDs of just-in-time memory ++ * allocations written to by the atom. When the atom ++ * completes, the value stored at the ++ * &struct_base_jit_alloc_info.heap_info_gpu_addr of ++ * each allocation is read in order to enforce an ++ * overall physical memory usage limit. ++ * @pre_dep: Pre-dependencies. One need to use SETTER function to assign ++ * this field; this is done in order to reduce possibility of ++ * improper assignment of a dependency field. ++ * @atom_number: Unique number to identify the atom. ++ * @prio: Atom priority. Refer to base_jd_prio for more details. ++ * @device_nr: Core group when BASE_JD_REQ_SPECIFIC_COHERENT_GROUP ++ * specified. ++ * @jobslot: Job slot to use when BASE_JD_REQ_JOB_SLOT is specified. ++ * @core_req: Core requirements. ++ * @renderpass_id: Renderpass identifier used to associate an atom that has ++ * BASE_JD_REQ_START_RENDERPASS set in its core requirements ++ * with an atom that has BASE_JD_REQ_END_RENDERPASS set. ++ * @padding: Unused. Must be zero. ++ * ++ * This structure has changed since UK 10.2 for which base_jd_core_req was a ++ * u16 value. ++ * ++ * In UK 10.3 a core_req field of a u32 type was added to the end of the ++ * structure, and the place in the structure previously occupied by u16 ++ * core_req was kept but renamed to compat_core_req. ++ * ++ * From UK 11.20 - compat_core_req is now occupied by u8 jit_id[2]. ++ * Compatibility with UK 10.x from UK 11.y is not handled because ++ * the major version increase prevents this. ++ * ++ * For UK 11.20 jit_id[2] must be initialized to zero. ++ */ ++struct base_jd_atom_v2 { ++ u64 jc; ++ struct base_jd_udata udata; ++ u64 extres_list; ++ u16 nr_extres; ++ u8 jit_id[2]; ++ struct base_dependency pre_dep[2]; ++ base_atom_id atom_number; ++ base_jd_prio prio; ++ u8 device_nr; ++ u8 jobslot; ++ base_jd_core_req core_req; ++ u8 renderpass_id; ++ u8 padding[7]; ++}; ++ ++/** ++ * struct base_jd_atom - Same as base_jd_atom_v2, but has an extra seq_nr ++ * at the beginning. ++ * ++ * @seq_nr: Sequence number of logical grouping of atoms. ++ * @jc: GPU address of a job chain or (if BASE_JD_REQ_END_RENDERPASS ++ * is set in the base_jd_core_req) the CPU address of a ++ * base_jd_fragment object. ++ * @udata: User data. ++ * @extres_list: List of external resources. ++ * @nr_extres: Number of external resources or JIT allocations. ++ * @jit_id: Zero-terminated array of IDs of just-in-time memory ++ * allocations written to by the atom. When the atom ++ * completes, the value stored at the ++ * &struct_base_jit_alloc_info.heap_info_gpu_addr of ++ * each allocation is read in order to enforce an ++ * overall physical memory usage limit. ++ * @pre_dep: Pre-dependencies. One need to use SETTER function to assign ++ * this field; this is done in order to reduce possibility of ++ * improper assignment of a dependency field. ++ * @atom_number: Unique number to identify the atom. ++ * @prio: Atom priority. Refer to base_jd_prio for more details. ++ * @device_nr: Core group when BASE_JD_REQ_SPECIFIC_COHERENT_GROUP ++ * specified. ++ * @jobslot: Job slot to use when BASE_JD_REQ_JOB_SLOT is specified. ++ * @core_req: Core requirements. ++ * @renderpass_id: Renderpass identifier used to associate an atom that has ++ * BASE_JD_REQ_START_RENDERPASS set in its core requirements ++ * with an atom that has BASE_JD_REQ_END_RENDERPASS set. ++ * @padding: Unused. Must be zero. ++ */ ++typedef struct base_jd_atom { ++ u64 seq_nr; ++ u64 jc; ++ struct base_jd_udata udata; ++ u64 extres_list; ++ u16 nr_extres; ++ u8 jit_id[2]; ++ struct base_dependency pre_dep[2]; ++ base_atom_id atom_number; ++ base_jd_prio prio; ++ u8 device_nr; ++ u8 jobslot; ++ base_jd_core_req core_req; ++ u8 renderpass_id; ++ u8 padding[7]; ++} base_jd_atom; ++ ++/* Job chain event code bits ++ * Defines the bits used to create ::base_jd_event_code ++ */ ++enum { ++ BASE_JD_SW_EVENT_KERNEL = (1u << 15), /* Kernel side event */ ++ BASE_JD_SW_EVENT = (1u << 14), /* SW defined event */ ++ /* Event indicates success (SW events only) */ ++ BASE_JD_SW_EVENT_SUCCESS = (1u << 13), ++ BASE_JD_SW_EVENT_JOB = (0u << 11), /* Job related event */ ++ BASE_JD_SW_EVENT_BAG = (1u << 11), /* Bag related event */ ++ BASE_JD_SW_EVENT_INFO = (2u << 11), /* Misc/info event */ ++ BASE_JD_SW_EVENT_RESERVED = (3u << 11), /* Reserved event type */ ++ /* Mask to extract the type from an event code */ ++ BASE_JD_SW_EVENT_TYPE_MASK = (3u << 11) ++}; ++ ++/** ++ * enum base_jd_event_code - Job chain event codes ++ * ++ * @BASE_JD_EVENT_RANGE_HW_NONFAULT_START: Start of hardware non-fault status ++ * codes. ++ * Obscurely, BASE_JD_EVENT_TERMINATED ++ * indicates a real fault, because the ++ * job was hard-stopped. ++ * @BASE_JD_EVENT_NOT_STARTED: Can't be seen by userspace, treated as ++ * 'previous job done'. ++ * @BASE_JD_EVENT_STOPPED: Can't be seen by userspace, becomes ++ * TERMINATED, DONE or JOB_CANCELLED. ++ * @BASE_JD_EVENT_TERMINATED: This is actually a fault status code - the job ++ * was hard stopped. ++ * @BASE_JD_EVENT_ACTIVE: Can't be seen by userspace, jobs only returned on ++ * complete/fail/cancel. ++ * @BASE_JD_EVENT_RANGE_HW_NONFAULT_END: End of hardware non-fault status codes. ++ * Obscurely, BASE_JD_EVENT_TERMINATED ++ * indicates a real fault, ++ * because the job was hard-stopped. ++ * @BASE_JD_EVENT_RANGE_HW_FAULT_OR_SW_ERROR_START: Start of hardware fault and ++ * software error status codes. ++ * @BASE_JD_EVENT_RANGE_HW_FAULT_OR_SW_ERROR_END: End of hardware fault and ++ * software error status codes. ++ * @BASE_JD_EVENT_RANGE_SW_SUCCESS_START: Start of software success status ++ * codes. ++ * @BASE_JD_EVENT_RANGE_SW_SUCCESS_END: End of software success status codes. ++ * @BASE_JD_EVENT_RANGE_KERNEL_ONLY_START: Start of kernel-only status codes. ++ * Such codes are never returned to ++ * user-space. ++ * @BASE_JD_EVENT_RANGE_KERNEL_ONLY_END: End of kernel-only status codes. ++ * ++ * HW and low-level SW events are represented by event codes. ++ * The status of jobs which succeeded are also represented by ++ * an event code (see @BASE_JD_EVENT_DONE). ++ * Events are usually reported as part of a &struct base_jd_event. ++ * ++ * The event codes are encoded in the following way: ++ * * 10:0 - subtype ++ * * 12:11 - type ++ * * 13 - SW success (only valid if the SW bit is set) ++ * * 14 - SW event (HW event if not set) ++ * * 15 - Kernel event (should never be seen in userspace) ++ * ++ * Events are split up into ranges as follows: ++ * * BASE_JD_EVENT_RANGE__START ++ * * BASE_JD_EVENT_RANGE__END ++ * ++ * code is in 's range when: ++ * BASE_JD_EVENT_RANGE__START <= code < ++ * BASE_JD_EVENT_RANGE__END ++ * ++ * Ranges can be asserted for adjacency by testing that the END of the previous ++ * is equal to the START of the next. This is useful for optimizing some tests ++ * for range. ++ * ++ * A limitation is that the last member of this enum must explicitly be handled ++ * (with an assert-unreachable statement) in switch statements that use ++ * variables of this type. Otherwise, the compiler warns that we have not ++ * handled that enum value. ++ */ ++enum base_jd_event_code { ++ /* HW defined exceptions */ ++ BASE_JD_EVENT_RANGE_HW_NONFAULT_START = 0, ++ ++ /* non-fatal exceptions */ ++ BASE_JD_EVENT_NOT_STARTED = 0x00, ++ BASE_JD_EVENT_DONE = 0x01, ++ BASE_JD_EVENT_STOPPED = 0x03, ++ BASE_JD_EVENT_TERMINATED = 0x04, ++ BASE_JD_EVENT_ACTIVE = 0x08, ++ ++ BASE_JD_EVENT_RANGE_HW_NONFAULT_END = 0x40, ++ BASE_JD_EVENT_RANGE_HW_FAULT_OR_SW_ERROR_START = 0x40, ++ ++ /* job exceptions */ ++ BASE_JD_EVENT_JOB_CONFIG_FAULT = 0x40, ++ BASE_JD_EVENT_JOB_POWER_FAULT = 0x41, ++ BASE_JD_EVENT_JOB_READ_FAULT = 0x42, ++ BASE_JD_EVENT_JOB_WRITE_FAULT = 0x43, ++ BASE_JD_EVENT_JOB_AFFINITY_FAULT = 0x44, ++ BASE_JD_EVENT_JOB_BUS_FAULT = 0x48, ++ BASE_JD_EVENT_INSTR_INVALID_PC = 0x50, ++ BASE_JD_EVENT_INSTR_INVALID_ENC = 0x51, ++ BASE_JD_EVENT_INSTR_TYPE_MISMATCH = 0x52, ++ BASE_JD_EVENT_INSTR_OPERAND_FAULT = 0x53, ++ BASE_JD_EVENT_INSTR_TLS_FAULT = 0x54, ++ BASE_JD_EVENT_INSTR_BARRIER_FAULT = 0x55, ++ BASE_JD_EVENT_INSTR_ALIGN_FAULT = 0x56, ++ BASE_JD_EVENT_DATA_INVALID_FAULT = 0x58, ++ BASE_JD_EVENT_TILE_RANGE_FAULT = 0x59, ++ BASE_JD_EVENT_STATE_FAULT = 0x5A, ++ BASE_JD_EVENT_OUT_OF_MEMORY = 0x60, ++ BASE_JD_EVENT_UNKNOWN = 0x7F, ++ ++ /* GPU exceptions */ ++ BASE_JD_EVENT_DELAYED_BUS_FAULT = 0x80, ++ BASE_JD_EVENT_SHAREABILITY_FAULT = 0x88, ++ ++ /* MMU exceptions */ ++ BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL1 = 0xC1, ++ BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL2 = 0xC2, ++ BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL3 = 0xC3, ++ BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL4 = 0xC4, ++ BASE_JD_EVENT_PERMISSION_FAULT = 0xC8, ++ BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL1 = 0xD1, ++ BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL2 = 0xD2, ++ BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL3 = 0xD3, ++ BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL4 = 0xD4, ++ BASE_JD_EVENT_ACCESS_FLAG = 0xD8, ++ ++ /* SW defined exceptions */ ++ BASE_JD_EVENT_MEM_GROWTH_FAILED = ++ BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x000, ++ BASE_JD_EVENT_TIMED_OUT = ++ BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x001, ++ BASE_JD_EVENT_JOB_CANCELLED = ++ BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x002, ++ BASE_JD_EVENT_JOB_INVALID = ++ BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x003, ++ BASE_JD_EVENT_PM_EVENT = ++ BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x004, ++ ++ BASE_JD_EVENT_BAG_INVALID = ++ BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_BAG | 0x003, ++ ++ BASE_JD_EVENT_RANGE_HW_FAULT_OR_SW_ERROR_END = BASE_JD_SW_EVENT | ++ BASE_JD_SW_EVENT_RESERVED | 0x3FF, ++ ++ BASE_JD_EVENT_RANGE_SW_SUCCESS_START = BASE_JD_SW_EVENT | ++ BASE_JD_SW_EVENT_SUCCESS | 0x000, ++ ++ BASE_JD_EVENT_PROGRESS_REPORT = BASE_JD_SW_EVENT | ++ BASE_JD_SW_EVENT_SUCCESS | BASE_JD_SW_EVENT_JOB | 0x000, ++ BASE_JD_EVENT_BAG_DONE = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_SUCCESS | ++ BASE_JD_SW_EVENT_BAG | 0x000, ++ BASE_JD_EVENT_DRV_TERMINATED = BASE_JD_SW_EVENT | ++ BASE_JD_SW_EVENT_SUCCESS | BASE_JD_SW_EVENT_INFO | 0x000, ++ ++ BASE_JD_EVENT_RANGE_SW_SUCCESS_END = BASE_JD_SW_EVENT | ++ BASE_JD_SW_EVENT_SUCCESS | BASE_JD_SW_EVENT_RESERVED | 0x3FF, ++ ++ BASE_JD_EVENT_RANGE_KERNEL_ONLY_START = BASE_JD_SW_EVENT | ++ BASE_JD_SW_EVENT_KERNEL | 0x000, ++ BASE_JD_EVENT_REMOVED_FROM_NEXT = BASE_JD_SW_EVENT | ++ BASE_JD_SW_EVENT_KERNEL | BASE_JD_SW_EVENT_JOB | 0x000, ++ BASE_JD_EVENT_END_RP_DONE = BASE_JD_SW_EVENT | ++ BASE_JD_SW_EVENT_KERNEL | BASE_JD_SW_EVENT_JOB | 0x001, ++ ++ BASE_JD_EVENT_RANGE_KERNEL_ONLY_END = BASE_JD_SW_EVENT | ++ BASE_JD_SW_EVENT_KERNEL | BASE_JD_SW_EVENT_RESERVED | 0x3FF ++}; ++ ++/** ++ * struct base_jd_event_v2 - Event reporting structure ++ * ++ * @event_code: event code. ++ * @atom_number: the atom number that has completed. ++ * @udata: user data. ++ * ++ * This structure is used by the kernel driver to report information ++ * about GPU events. They can either be HW-specific events or low-level ++ * SW events, such as job-chain completion. ++ * ++ * The event code contains an event type field which can be extracted ++ * by ANDing with BASE_JD_SW_EVENT_TYPE_MASK. ++ */ ++struct base_jd_event_v2 { ++ enum base_jd_event_code event_code; ++ base_atom_id atom_number; ++ struct base_jd_udata udata; ++}; ++ ++/** ++ * struct base_dump_cpu_gpu_counters - Structure for ++ * BASE_JD_REQ_SOFT_DUMP_CPU_GPU_COUNTERS ++ * jobs. ++ * ++ * This structure is stored into the memory pointed to by the @jc field ++ * of &struct base_jd_atom. ++ * ++ * It must not occupy the same CPU cache line(s) as any neighboring data. ++ * This is to avoid cases where access to pages containing the structure ++ * is shared between cached and un-cached memory regions, which would ++ * cause memory corruption. ++ */ ++ ++struct base_dump_cpu_gpu_counters { ++ u64 system_time; ++ u64 cycle_counter; ++ u64 sec; ++ u32 usec; ++ u8 padding[36]; ++}; ++ ++#endif /* _BASE_JM_KERNEL_H_ */ +diff --git a/drivers/gpu/arm/bifrost/jm/mali_kbase_jm_defs.h b/drivers/gpu/arm/bifrost/jm/mali_kbase_jm_defs.h +new file mode 100755 +index 000000000000..4fb5d1d9c410 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/jm/mali_kbase_jm_defs.h +@@ -0,0 +1,844 @@ ++/* ++ * ++ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++/* ++ * Definitions (types, defines, etcs) specific to Job Manager Kbase. ++ * They are placed here to allow the hierarchy of header files to work. ++ */ ++ ++#ifndef _KBASE_JM_DEFS_H_ ++#define _KBASE_JM_DEFS_H_ ++ ++#include "mali_kbase_js_defs.h" ++ ++/* Dump Job slot trace on error (only active if KBASE_KTRACE_ENABLE != 0) */ ++#define KBASE_KTRACE_DUMP_ON_JOB_SLOT_ERROR 1 ++ ++/* ++ * Number of milliseconds before resetting the GPU when a job cannot be "zapped" ++ * from the hardware. Note that the time is actually ++ * ZAP_TIMEOUT+SOFT_STOP_RESET_TIMEOUT between the context zap starting and ++ * the GPU actually being reset to give other contexts time for their jobs ++ * to be soft-stopped and removed from the hardware before resetting. ++ */ ++#define ZAP_TIMEOUT 1000 ++ ++/* ++ * Prevent soft-stops from occurring in scheduling situations ++ * ++ * This is not due to HW issues, but when scheduling is desired to be more ++ * predictable. ++ * ++ * Therefore, soft stop may still be disabled due to HW issues. ++ * ++ * Soft stop will still be used for non-scheduling purposes e.g. when ++ * terminating a context. ++ * ++ * if not in use, define this value to 0 instead of being undefined. ++ */ ++#define KBASE_DISABLE_SCHEDULING_SOFT_STOPS 0 ++ ++/* ++ * Prevent hard-stops from occurring in scheduling situations ++ * ++ * This is not due to HW issues, but when scheduling is desired to be more ++ * predictable. ++ * ++ * Hard stop will still be used for non-scheduling purposes e.g. when ++ * terminating a context. ++ * ++ * if not in use, define this value to 0 instead of being undefined. ++ */ ++#define KBASE_DISABLE_SCHEDULING_HARD_STOPS 0 ++ ++/* Atom has been previously soft-stopped */ ++#define KBASE_KATOM_FLAG_BEEN_SOFT_STOPPED (1<<1) ++/* Atom has been previously retried to execute */ ++#define KBASE_KATOM_FLAGS_RERUN (1<<2) ++/* Atom submitted with JOB_CHAIN_FLAG bit set in JS_CONFIG_NEXT register, helps ++ * to disambiguate short-running job chains during soft/hard stopping of jobs ++ */ ++#define KBASE_KATOM_FLAGS_JOBCHAIN (1<<3) ++/* Atom has been previously hard-stopped. */ ++#define KBASE_KATOM_FLAG_BEEN_HARD_STOPPED (1<<4) ++/* Atom has caused us to enter disjoint state */ ++#define KBASE_KATOM_FLAG_IN_DISJOINT (1<<5) ++/* Atom blocked on cross-slot dependency */ ++#define KBASE_KATOM_FLAG_X_DEP_BLOCKED (1<<7) ++/* Atom has fail dependency on cross-slot dependency */ ++#define KBASE_KATOM_FLAG_FAIL_BLOCKER (1<<8) ++/* Atom is currently in the list of atoms blocked on cross-slot dependencies */ ++#define KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST (1<<9) ++/* Atom is currently holding a context reference */ ++#define KBASE_KATOM_FLAG_HOLDING_CTX_REF (1<<10) ++/* Atom requires GPU to be in protected mode */ ++#define KBASE_KATOM_FLAG_PROTECTED (1<<11) ++/* Atom has been stored in runnable_tree */ ++#define KBASE_KATOM_FLAG_JSCTX_IN_TREE (1<<12) ++/* Atom is waiting for L2 caches to power up in order to enter protected mode */ ++#define KBASE_KATOM_FLAG_HOLDING_L2_REF_PROT (1<<13) ++ ++/* SW related flags about types of JS_COMMAND action ++ * NOTE: These must be masked off by JS_COMMAND_MASK ++ */ ++ ++/* This command causes a disjoint event */ ++#define JS_COMMAND_SW_CAUSES_DISJOINT 0x100 ++ ++/* Bitmask of all SW related flags */ ++#define JS_COMMAND_SW_BITS (JS_COMMAND_SW_CAUSES_DISJOINT) ++ ++#if (JS_COMMAND_SW_BITS & JS_COMMAND_MASK) ++#error "JS_COMMAND_SW_BITS not masked off by JS_COMMAND_MASK." \ ++ "Must update JS_COMMAND_SW_<..> bitmasks" ++#endif ++ ++/* Soft-stop command that causes a Disjoint event. This of course isn't ++ * entirely masked off by JS_COMMAND_MASK ++ */ ++#define JS_COMMAND_SOFT_STOP_WITH_SW_DISJOINT \ ++ (JS_COMMAND_SW_CAUSES_DISJOINT | JS_COMMAND_SOFT_STOP) ++ ++#define KBASEP_ATOM_ID_INVALID BASE_JD_ATOM_COUNT ++ ++/* Serialize atoms within a slot (ie only one atom per job slot) */ ++#define KBASE_SERIALIZE_INTRA_SLOT (1 << 0) ++/* Serialize atoms between slots (ie only one job slot running at any time) */ ++#define KBASE_SERIALIZE_INTER_SLOT (1 << 1) ++/* Reset the GPU after each atom completion */ ++#define KBASE_SERIALIZE_RESET (1 << 2) ++ ++#ifdef CONFIG_DEBUG_FS ++/** ++ * struct base_job_fault_event - keeps track of the atom which faulted or which ++ * completed after the faulty atom but before the ++ * debug data for faulty atom was dumped. ++ * ++ * @event_code: event code for the atom, should != BASE_JD_EVENT_DONE for ++ * the atom which faulted. ++ * @katom: pointer to the atom for which job fault occurred or which ++ * completed after the faulty atom. ++ * @job_fault_work: work item, queued only for the faulty atom, which waits for ++ * the dumping to get completed and then does the bottom half ++ * of job done for the atoms which followed the faulty atom. ++ * @head: List head used to store the atom in the global list of ++ * faulty atoms or context specific list of atoms which got ++ * completed during the dump. ++ * @reg_offset: offset of the register to be dumped next, only applicable ++ * for the faulty atom. ++ */ ++struct base_job_fault_event { ++ ++ u32 event_code; ++ struct kbase_jd_atom *katom; ++ struct work_struct job_fault_work; ++ struct list_head head; ++ int reg_offset; ++}; ++#endif ++ ++/** ++ * struct kbase_jd_atom_dependency - Contains the dependency info for an atom. ++ * @atom: pointer to the dependee atom. ++ * @dep_type: type of dependency on the dependee @atom, i.e. order or data ++ * dependency. BASE_JD_DEP_TYPE_INVALID indicates no dependency. ++ */ ++struct kbase_jd_atom_dependency { ++ struct kbase_jd_atom *atom; ++ u8 dep_type; ++}; ++ ++/** ++ * kbase_jd_katom_dep_atom - Retrieves a read-only reference to the ++ * dependee atom. ++ * @dep: pointer to the dependency info structure. ++ * ++ * Return: readonly reference to dependee atom. ++ */ ++static inline const struct kbase_jd_atom * ++kbase_jd_katom_dep_atom(const struct kbase_jd_atom_dependency *dep) ++{ ++ LOCAL_ASSERT(dep != NULL); ++ ++ return (const struct kbase_jd_atom *)(dep->atom); ++} ++ ++/** ++ * kbase_jd_katom_dep_type - Retrieves the dependency type info ++ * ++ * @dep: pointer to the dependency info structure. ++ * ++ * Return: the type of dependency there is on the dependee atom. ++ */ ++static inline u8 kbase_jd_katom_dep_type( ++ const struct kbase_jd_atom_dependency *dep) ++{ ++ LOCAL_ASSERT(dep != NULL); ++ ++ return dep->dep_type; ++} ++ ++/** ++ * kbase_jd_katom_dep_set - sets up the dependency info structure ++ * as per the values passed. ++ * @const_dep: pointer to the dependency info structure to be setup. ++ * @a: pointer to the dependee atom. ++ * @type: type of dependency there is on the dependee atom. ++ */ ++static inline void kbase_jd_katom_dep_set( ++ const struct kbase_jd_atom_dependency *const_dep, ++ struct kbase_jd_atom *a, u8 type) ++{ ++ struct kbase_jd_atom_dependency *dep; ++ ++ LOCAL_ASSERT(const_dep != NULL); ++ ++ dep = (struct kbase_jd_atom_dependency *)const_dep; ++ ++ dep->atom = a; ++ dep->dep_type = type; ++} ++ ++/** ++ * kbase_jd_katom_dep_clear - resets the dependency info structure ++ * ++ * @const_dep: pointer to the dependency info structure to be setup. ++ */ ++static inline void kbase_jd_katom_dep_clear( ++ const struct kbase_jd_atom_dependency *const_dep) ++{ ++ struct kbase_jd_atom_dependency *dep; ++ ++ LOCAL_ASSERT(const_dep != NULL); ++ ++ dep = (struct kbase_jd_atom_dependency *)const_dep; ++ ++ dep->atom = NULL; ++ dep->dep_type = BASE_JD_DEP_TYPE_INVALID; ++} ++ ++/** ++ * enum kbase_atom_gpu_rb_state - The state of an atom, pertinent after it ++ * becomes runnable, with respect to job slot ++ * ringbuffer/fifo. ++ * @KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB: Atom not currently present in slot fifo, ++ * which implies that either atom has not become ++ * runnable due to dependency or has completed ++ * the execution on GPU. ++ * @KBASE_ATOM_GPU_RB_WAITING_BLOCKED: Atom has been added to slot fifo but is ++ * blocked due to cross slot dependency, ++ * can't be submitted to GPU. ++ * @KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV: Atom has been added to slot ++ * fifo but is waiting for the completion of ++ * previously added atoms in current & other ++ * slots, as their protected mode requirements ++ * do not match with the current atom. ++ * @KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION: Atom is in slot fifo ++ * and is waiting for completion of protected ++ * mode transition, needed before the atom is ++ * submitted to GPU. ++ * @KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE: Atom is in slot fifo but is ++ * waiting for the cores, which are needed to ++ * execute the job chain represented by the atom, ++ * to become available ++ * @KBASE_ATOM_GPU_RB_READY: Atom is in slot fifo and can be submitted to ++ * GPU. ++ * @KBASE_ATOM_GPU_RB_SUBMITTED: Atom is in slot fifo and has been submitted ++ * to GPU. ++ * @KBASE_ATOM_GPU_RB_RETURN_TO_JS: Atom must be returned to JS due to some ++ * failure, but only after the previously added ++ * atoms in fifo have completed or have also ++ * been returned to JS. ++ */ ++enum kbase_atom_gpu_rb_state { ++ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB, ++ KBASE_ATOM_GPU_RB_WAITING_BLOCKED, ++ KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV, ++ KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION, ++ KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE, ++ KBASE_ATOM_GPU_RB_READY, ++ KBASE_ATOM_GPU_RB_SUBMITTED, ++ KBASE_ATOM_GPU_RB_RETURN_TO_JS = -1 ++}; ++ ++/** ++ * enum kbase_atom_enter_protected_state - The state of an atom with respect to ++ * the preparation for GPU's entry into protected mode, ++ * becomes pertinent only after atom's state with respect ++ * to slot ringbuffer is ++ * KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION ++ * @KBASE_ATOM_ENTER_PROTECTED_CHECK: Starting state. Check if there are any ++ * atoms currently submitted to GPU and protected mode ++ * transition is not already in progress. ++ * @KBASE_ATOM_ENTER_PROTECTED_HWCNT: Wait for hardware counter context to ++ * become disabled before entry into protected mode. ++ * @KBASE_ATOM_ENTER_PROTECTED_IDLE_L2: Wait for the L2 to become idle in ++ * preparation for the coherency change. L2 shall be ++ * powered down and GPU shall come out of fully ++ * coherent mode before entering protected mode. ++ * @KBASE_ATOM_ENTER_PROTECTED_SET_COHERENCY: Prepare coherency change; ++ * for BASE_HW_ISSUE_TGOX_R1_1234 also request L2 power on ++ * so that coherency register contains correct value when ++ * GPU enters protected mode. ++ * @KBASE_ATOM_ENTER_PROTECTED_FINISHED: End state; for ++ * BASE_HW_ISSUE_TGOX_R1_1234 check ++ * that L2 is powered up and switch GPU to protected mode. ++ */ ++enum kbase_atom_enter_protected_state { ++ /* ++ * NOTE: The integer value of this must match ++ * KBASE_ATOM_EXIT_PROTECTED_CHECK. ++ */ ++ KBASE_ATOM_ENTER_PROTECTED_CHECK = 0, ++ KBASE_ATOM_ENTER_PROTECTED_HWCNT, ++ KBASE_ATOM_ENTER_PROTECTED_IDLE_L2, ++ KBASE_ATOM_ENTER_PROTECTED_SET_COHERENCY, ++ KBASE_ATOM_ENTER_PROTECTED_FINISHED, ++}; ++ ++/** ++ * enum kbase_atom_exit_protected_state - The state of an atom with respect to ++ * the preparation for GPU's exit from protected mode, ++ * becomes pertinent only after atom's state with respect ++ * to slot ngbuffer is ++ * KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION ++ * @KBASE_ATOM_EXIT_PROTECTED_CHECK: Starting state. Check if there are any ++ * atoms currently submitted to GPU and protected mode ++ * transition is not already in progress. ++ * @KBASE_ATOM_EXIT_PROTECTED_IDLE_L2: Wait for the L2 to become idle in ++ * preparation for the reset, as exiting protected mode ++ * requires a reset. ++ * @KBASE_ATOM_EXIT_PROTECTED_RESET: Issue the reset to trigger exit from ++ * protected mode ++ * @KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT: End state, Wait for the reset to ++ * complete ++ */ ++enum kbase_atom_exit_protected_state { ++ /* ++ * NOTE: The integer value of this must match ++ * KBASE_ATOM_ENTER_PROTECTED_CHECK. ++ */ ++ KBASE_ATOM_EXIT_PROTECTED_CHECK = 0, ++ KBASE_ATOM_EXIT_PROTECTED_IDLE_L2, ++ KBASE_ATOM_EXIT_PROTECTED_RESET, ++ KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT, ++}; ++ ++/** ++ * struct kbase_ext_res - Contains the info for external resources referred ++ * by an atom, which have been mapped on GPU side. ++ * @gpu_address: Start address of the memory region allocated for ++ * the resource from GPU virtual address space. ++ * @alloc: pointer to physical pages tracking object, set on ++ * mapping the external resource on GPU side. ++ */ ++struct kbase_ext_res { ++ u64 gpu_address; ++ struct kbase_mem_phy_alloc *alloc; ++}; ++ ++/** ++ * struct kbase_jd_atom - object representing the atom, containing the complete ++ * state and attributes of an atom. ++ * @work: work item for the bottom half processing of the atom, ++ * by JD or JS, after it got executed on GPU or the ++ * input fence got signaled ++ * @start_timestamp: time at which the atom was submitted to the GPU, by ++ * updating the JS_HEAD_NEXTn register. ++ * @udata: copy of the user data sent for the atom in ++ * base_jd_submit. ++ * @kctx: Pointer to the base context with which the atom is ++ * associated. ++ * @dep_head: Array of 2 list heads, pointing to the two list of ++ * atoms ++ * which are blocked due to dependency on this atom. ++ * @dep_item: Array of 2 list heads, used to store the atom in the ++ * list of other atoms depending on the same dependee ++ * atom. ++ * @dep: Array containing the dependency info for the 2 atoms ++ * on which the atom depends upon. ++ * @jd_item: List head used during job dispatch job_done ++ * processing - as dependencies may not be entirely ++ * resolved at this point, ++ * we need to use a separate list head. ++ * @in_jd_list: flag set to true if atom's @jd_item is currently on ++ * a list, prevents atom being processed twice. ++ * @jit_ids: Zero-terminated array of IDs of just-in-time memory ++ * allocations written to by the atom. When the atom ++ * completes, the value stored at the ++ * &struct_base_jit_alloc_info.heap_info_gpu_addr of ++ * each allocation is read in order to enforce an ++ * overall physical memory usage limit. ++ * @nr_extres: number of external resources referenced by the atom. ++ * @extres: pointer to the location containing info about ++ * @nr_extres external resources referenced by the atom. ++ * @device_nr: indicates the coregroup with which the atom is ++ * associated, when ++ * BASE_JD_REQ_SPECIFIC_COHERENT_GROUP specified. ++ * @jc: GPU address of the job-chain. ++ * @softjob_data: Copy of data read from the user space buffer that @jc ++ * points to. ++ * @fence: Stores either an input or output sync fence, ++ * depending on soft-job type ++ * @sync_waiter: Pointer to the sync fence waiter structure passed to ++ * the callback function on signaling of the input ++ * fence. ++ * @dma_fence: object containing pointers to both input & output ++ * fences and other related members used for explicit ++ * sync through soft jobs and for the implicit ++ * synchronization required on access to external ++ * resources. ++ * @event_code: Event code for the job chain represented by the atom, ++ * both HW and low-level SW events are represented by ++ * event codes. ++ * @core_req: bitmask of BASE_JD_REQ_* flags specifying either ++ * Hw or Sw requirements for the job chain represented ++ * by the atom. ++ * @ticks: Number of scheduling ticks for which atom has been ++ * running on the GPU. ++ * @sched_priority: Priority of the atom for Job scheduling, as per the ++ * KBASE_JS_ATOM_SCHED_PRIO_*. ++ * @completed: Wait queue to wait upon for the completion of atom. ++ * @status: Indicates at high level at what stage the atom is in, ++ * as per KBASE_JD_ATOM_STATE_*, that whether it is not ++ * in use or its queued in JD or given to JS or ++ * submitted to Hw or it completed the execution on Hw. ++ * @work_id: used for GPU tracepoints, its a snapshot of the ++ * 'work_id' counter in kbase_jd_context which is ++ * incremented on every call to base_jd_submit. ++ * @slot_nr: Job slot chosen for the atom. ++ * @atom_flags: bitmask of KBASE_KATOM_FLAG* flags capturing the ++ * excat low level state of the atom. ++ * @gpu_rb_state: bitmnask of KBASE_ATOM_GPU_RB_* flags, precisely ++ * tracking atom's state after it has entered ++ * Job scheduler on becoming runnable. Atom ++ * could be blocked due to cross slot dependency ++ * or waiting for the shader cores to become available ++ * or waiting for protected mode transitions to ++ * complete. ++ * @need_cache_flush_cores_retained: flag indicating that manual flush of GPU ++ * cache is needed for the atom and the shader cores ++ * used for atom have been kept on. ++ * @blocked: flag indicating that atom's resubmission to GPU is ++ * blocked till the work item is scheduled to return the ++ * atom to JS. ++ * @pre_dep: Pointer to atom that this atom has same-slot ++ * dependency on ++ * @post_dep: Pointer to atom that has same-slot dependency on ++ * this atom ++ * @x_pre_dep: Pointer to atom that this atom has cross-slot ++ * dependency on ++ * @x_post_dep: Pointer to atom that has cross-slot dependency on ++ * this atom ++ * @flush_id: The GPU's flush count recorded at the time of ++ * submission, ++ * used for the cache flush optimization ++ * @fault_event: Info for dumping the debug data on Job fault. ++ * @queue: List head used for 4 different purposes : ++ * Adds atom to the list of dma-buf fence waiting atoms. ++ * Adds atom to the list of atoms blocked due to cross ++ * slot dependency. ++ * Adds atom to the list of softjob atoms for which JIT ++ * allocation has been deferred ++ * Adds atom to the list of softjob atoms waiting for ++ * the signaling of fence. ++ * @jit_node: Used to keep track of all JIT free/alloc jobs in ++ * submission order ++ * @jit_blocked: Flag indicating that JIT allocation requested through ++ * softjob atom will be reattempted after the impending ++ * free of other active JIT allocations. ++ * @will_fail_event_code: If non-zero, this indicates that the atom will fail ++ * with the set event_code when the atom is processed. ++ * Used for special handling of atoms, which have a data ++ * dependency on the failed atoms. ++ * @protected_state: State of the atom, as per ++ * KBASE_ATOM_(ENTER|EXIT)_PROTECTED_*, ++ * when transitioning into or out of protected mode. ++ * Atom will be either entering or exiting the ++ * protected mode. ++ * @runnable_tree_node: The node added to context's job slot specific rb tree ++ * when the atom becomes runnable. ++ * @age: Age of atom relative to other atoms in the context, ++ * is snapshot of the age_count counter in kbase ++ * context. ++ */ ++struct kbase_jd_atom { ++ struct work_struct work; ++ ktime_t start_timestamp; ++ ++ struct base_jd_udata udata; ++ struct kbase_context *kctx; ++ ++ struct list_head dep_head[2]; ++ struct list_head dep_item[2]; ++ const struct kbase_jd_atom_dependency dep[2]; ++ struct list_head jd_item; ++ bool in_jd_list; ++ ++#if MALI_JIT_PRESSURE_LIMIT_BASE ++ u8 jit_ids[2]; ++#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ ++ ++ u16 nr_extres; ++ struct kbase_ext_res *extres; ++ ++ u32 device_nr; ++ u64 jc; ++ void *softjob_data; ++#if defined(CONFIG_SYNC) ++ struct sync_fence *fence; ++ struct sync_fence_waiter sync_waiter; ++#endif /* CONFIG_SYNC */ ++#if defined(CONFIG_MALI_BIFROST_DMA_FENCE) || defined(CONFIG_SYNC_FILE) ++ struct { ++ /* Use the functions/API defined in mali_kbase_fence.h to ++ * when working with this sub struct ++ */ ++#if defined(CONFIG_SYNC_FILE) ++ /* Input fence */ ++#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) ++ struct fence *fence_in; ++#else ++ struct dma_fence *fence_in; ++#endif ++#endif ++ /* This points to the dma-buf output fence for this atom. If ++ * this is NULL then there is no fence for this atom and the ++ * following fields related to dma_fence may have invalid data. ++ * ++ * The context and seqno fields contain the details for this ++ * fence. ++ * ++ * This fence is signaled when the katom is completed, ++ * regardless of the event_code of the katom (signal also on ++ * failure). ++ */ ++#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) ++ struct fence *fence; ++#else ++ struct dma_fence *fence; ++#endif ++ /* The dma-buf fence context number for this atom. A unique ++ * context number is allocated to each katom in the context on ++ * context creation. ++ */ ++ unsigned int context; ++ /* The dma-buf fence sequence number for this atom. This is ++ * increased every time this katom uses dma-buf fence. ++ */ ++ atomic_t seqno; ++ /* This contains a list of all callbacks set up to wait on ++ * other fences. This atom must be held back from JS until all ++ * these callbacks have been called and dep_count have reached ++ * 0. The initial value of dep_count must be equal to the ++ * number of callbacks on this list. ++ * ++ * This list is protected by jctx.lock. Callbacks are added to ++ * this list when the atom is built and the wait are set up. ++ * All the callbacks then stay on the list until all callbacks ++ * have been called and the atom is queued, or cancelled, and ++ * then all callbacks are taken off the list and freed. ++ */ ++ struct list_head callbacks; ++ /* Atomic counter of number of outstandind dma-buf fence ++ * dependencies for this atom. When dep_count reaches 0 the ++ * atom may be queued. ++ * ++ * The special value "-1" may only be set after the count ++ * reaches 0, while holding jctx.lock. This indicates that the ++ * atom has been handled, either queued in JS or cancelled. ++ * ++ * If anyone but the dma-fence worker sets this to -1 they must ++ * ensure that any potentially queued worker must have ++ * completed before allowing the atom to be marked as unused. ++ * This can be done by flushing the fence work queue: ++ * kctx->dma_fence.wq. ++ */ ++ atomic_t dep_count; ++ } dma_fence; ++#endif /* CONFIG_MALI_BIFROST_DMA_FENCE || CONFIG_SYNC_FILE */ ++ ++ /* Note: refer to kbasep_js_atom_retained_state, which will take a copy ++ * of some of the following members ++ */ ++ enum base_jd_event_code event_code; ++ base_jd_core_req core_req; ++ u8 jobslot; ++ u8 renderpass_id; ++ struct base_jd_fragment jc_fragment; ++ ++ u32 ticks; ++ int sched_priority; ++ ++ wait_queue_head_t completed; ++ enum kbase_jd_atom_state status; ++#ifdef CONFIG_GPU_TRACEPOINTS ++ int work_id; ++#endif ++ int slot_nr; ++ ++ u32 atom_flags; ++ ++ int retry_count; ++ ++ enum kbase_atom_gpu_rb_state gpu_rb_state; ++ ++ bool need_cache_flush_cores_retained; ++ ++ atomic_t blocked; ++ ++ /* user-space sequence number, to order atoms in some temporal order */ ++ u64 seq_nr; ++ ++ struct kbase_jd_atom *pre_dep; ++ struct kbase_jd_atom *post_dep; ++ ++ struct kbase_jd_atom *x_pre_dep; ++ struct kbase_jd_atom *x_post_dep; ++ ++ u32 flush_id; ++ ++#ifdef CONFIG_DEBUG_FS ++ struct base_job_fault_event fault_event; ++#endif ++ struct list_head queue; ++ ++ struct list_head jit_node; ++ bool jit_blocked; ++ ++ enum base_jd_event_code will_fail_event_code; ++ ++ union { ++ enum kbase_atom_enter_protected_state enter; ++ enum kbase_atom_exit_protected_state exit; ++ } protected_state; ++ ++ struct rb_node runnable_tree_node; ++ ++ u32 age; ++}; ++ ++static inline bool kbase_jd_katom_is_protected( ++ const struct kbase_jd_atom *katom) ++{ ++ return (bool)(katom->atom_flags & KBASE_KATOM_FLAG_PROTECTED); ++} ++ ++/* ++ * Theory of operations: ++ * ++ * Atom objects are statically allocated within the context structure. ++ * ++ * Each atom is the head of two lists, one for the "left" set of dependencies, ++ * one for the "right" set. ++ */ ++ ++#define KBASE_JD_DEP_QUEUE_SIZE 256 ++ ++/** ++ * enum kbase_jd_renderpass_state - State of a renderpass ++ * @KBASE_JD_RP_COMPLETE: Unused or completed renderpass. Can only transition to ++ * START. ++ * @KBASE_JD_RP_START: Renderpass making a first attempt at tiling. ++ * Can transition to PEND_OOM or COMPLETE. ++ * @KBASE_JD_RP_PEND_OOM: Renderpass whose first attempt at tiling used too much ++ * memory and has a soft-stop pending. Can transition to ++ * OOM or COMPLETE. ++ * @KBASE_JD_RP_OOM: Renderpass whose first attempt at tiling used too much ++ * memory and therefore switched to incremental ++ * rendering. The fragment job chain is forced to run. ++ * Can only transition to RETRY. ++ * @KBASE_JD_RP_RETRY: Renderpass making a second or subsequent attempt at ++ * tiling. Can transition to RETRY_PEND_OOM or COMPLETE. ++ * @KBASE_JD_RP_RETRY_PEND_OOM: Renderpass whose second or subsequent attempt at ++ * tiling used too much memory again and has a ++ * soft-stop pending. Can transition to RETRY_OOM ++ * or COMPLETE. ++ * @KBASE_JD_RP_RETRY_OOM: Renderpass whose second or subsequent attempt at ++ * tiling used too much memory again. The fragment job ++ * chain is forced to run. Can only transition to RETRY. ++ * ++ * A state machine is used to control incremental rendering. ++ */ ++enum kbase_jd_renderpass_state { ++ KBASE_JD_RP_COMPLETE, /* COMPLETE => START */ ++ KBASE_JD_RP_START, /* START => PEND_OOM or COMPLETE */ ++ KBASE_JD_RP_PEND_OOM, /* PEND_OOM => OOM or COMPLETE */ ++ KBASE_JD_RP_OOM, /* OOM => RETRY */ ++ KBASE_JD_RP_RETRY, /* RETRY => RETRY_PEND_OOM or ++ * COMPLETE ++ */ ++ KBASE_JD_RP_RETRY_PEND_OOM, /* RETRY_PEND_OOM => RETRY_OOM or ++ * COMPLETE ++ */ ++ KBASE_JD_RP_RETRY_OOM, /* RETRY_OOM => RETRY */ ++}; ++ ++/** ++ * struct kbase_jd_renderpass - Data for a renderpass ++ * @state: Current state of the renderpass. If KBASE_JD_RP_COMPLETE then ++ * all other members are invalid. ++ * Both the job dispatcher context and hwaccess_lock must be ++ * locked to modify this so that it can be read with either ++ * (or both) locked. ++ * @start_katom: Address of the atom that is the start of a renderpass. ++ * Both the job dispatcher context and hwaccess_lock must be ++ * locked to modify this so that it can be read with either ++ * (or both) locked. ++ * @end_katom: Address of the atom that is the end of a renderpass, or NULL ++ * if that atom hasn't been added to the job scheduler yet. ++ * The job dispatcher context and hwaccess_lock must be ++ * locked to modify this so that it can be read with either ++ * (or both) locked. ++ * @oom_reg_list: A list of region structures which triggered out-of-memory. ++ * The hwaccess_lock must be locked to access this. ++ * ++ * Atoms tagged with BASE_JD_REQ_START_RENDERPASS or BASE_JD_REQ_END_RENDERPASS ++ * are associated with an object of this type, which is created and maintained ++ * by kbase to keep track of each renderpass. ++ */ ++struct kbase_jd_renderpass { ++ enum kbase_jd_renderpass_state state; ++ struct kbase_jd_atom *start_katom; ++ struct kbase_jd_atom *end_katom; ++ struct list_head oom_reg_list; ++}; ++ ++/** ++ * struct kbase_jd_context - per context object encapsulating all the ++ * Job dispatcher related state. ++ * @lock: lock to serialize the updates made to the ++ * Job dispatcher state and kbase_jd_atom objects. ++ * @sched_info: Structure encapsulating all the Job scheduling ++ * info. ++ * @atoms: Array of the objects representing atoms, ++ * containing the complete state and attributes ++ * of an atom. ++ * @renderpasses: Array of renderpass state for incremental ++ * rendering, indexed by user-specified renderpass ++ * ID. ++ * @job_nr: Tracks the number of atoms being processed by the ++ * kbase. This includes atoms that are not tracked by ++ * scheduler: 'not ready to run' & 'dependency-only' ++ * jobs. ++ * @zero_jobs_wait: Waitq that reflects whether there are no jobs ++ * (including SW-only dependency jobs). This is set ++ * when no jobs are present on the ctx, and clear ++ * when there are jobs. ++ * This must be updated atomically with @job_nr. ++ * note: Job Dispatcher knows about more jobs than ++ * the Job Scheduler as it is unaware of jobs that ++ * are blocked on dependencies and SW-only dependency ++ * jobs. This waitq can be waited upon to find out ++ * when the context jobs are all done/cancelled ++ * (including those that might've been blocked ++ * on dependencies) - and so, whether it can be ++ * terminated. However, it should only be terminated ++ * once it is not present in the run-pool. ++ * Since the waitq is only set under @lock, ++ * the waiter should also briefly obtain and drop ++ * @lock to guarantee that the setter has completed ++ * its work on the kbase_context ++ * @job_done_wq: Workqueue to which the per atom work item is ++ * queued for bottom half processing when the ++ * atom completes ++ * execution on GPU or the input fence get signaled. ++ * @tb_lock: Lock to serialize the write access made to @tb to ++ * to store the register access trace messages. ++ * @tb: Pointer to the Userspace accessible buffer storing ++ * the trace messages for register read/write ++ * accesses made by the Kbase. The buffer is filled ++ * in circular fashion. ++ * @tb_wrap_offset: Offset to the end location in the trace buffer, ++ * the write pointer is moved to the beginning on ++ * reaching this offset. ++ * @work_id: atomic variable used for GPU tracepoints, ++ * incremented on every call to base_jd_submit. ++ * @jit_atoms_head: A list of the just-in-time memory soft-jobs, both ++ * allocate & free, in submission order, protected ++ * by kbase_jd_context.lock. ++ * @jit_pending_alloc: A list of just-in-time memory allocation ++ * soft-jobs which will be reattempted after the ++ * impending free of other active allocations. ++ */ ++struct kbase_jd_context { ++ struct mutex lock; ++ struct kbasep_js_kctx_info sched_info; ++ struct kbase_jd_atom atoms[BASE_JD_ATOM_COUNT]; ++ struct kbase_jd_renderpass renderpasses[BASE_JD_RP_COUNT]; ++ struct workqueue_struct *job_done_wq; ++ ++ wait_queue_head_t zero_jobs_wait; ++ spinlock_t tb_lock; ++ u32 *tb; ++ u32 job_nr; ++ size_t tb_wrap_offset; ++ ++#ifdef CONFIG_GPU_TRACEPOINTS ++ atomic_t work_id; ++#endif ++ ++ struct list_head jit_atoms_head; ++ struct list_head jit_pending_alloc; ++}; ++ ++/** ++ * struct jsctx_queue - JS context atom queue ++ * @runnable_tree: Root of RB-tree containing currently runnable atoms on this ++ * job slot. ++ * @x_dep_head: Head item of the linked list of atoms blocked on cross-slot ++ * dependencies. Atoms on this list will be moved to the ++ * runnable_tree when the blocking atom completes. ++ * ++ * hwaccess_lock must be held when accessing this structure. ++ */ ++struct jsctx_queue { ++ struct rb_root runnable_tree; ++ struct list_head x_dep_head; ++}; ++ ++/** ++ * struct kbase_as - Object representing an address space of GPU. ++ * @number: Index at which this address space structure is present ++ * in an array of address space structures embedded inside ++ * the &struct kbase_device. ++ * @pf_wq: Workqueue for processing work items related to ++ * Page fault and Bus fault handling. ++ * @work_pagefault: Work item for the Page fault handling. ++ * @work_busfault: Work item for the Bus fault handling. ++ * @pf_data: Data relating to Page fault. ++ * @bf_data: Data relating to Bus fault. ++ * @current_setup: Stores the MMU configuration for this address space. ++ */ ++struct kbase_as { ++ int number; ++ struct workqueue_struct *pf_wq; ++ struct work_struct work_pagefault; ++ struct work_struct work_busfault; ++ struct kbase_fault pf_data; ++ struct kbase_fault bf_data; ++ struct kbase_mmu_setup current_setup; ++}; ++ ++#endif /* _KBASE_JM_DEFS_H_ */ +diff --git a/drivers/gpu/arm/bifrost/jm/mali_kbase_jm_ioctl.h b/drivers/gpu/arm/bifrost/jm/mali_kbase_jm_ioctl.h +new file mode 100755 +index 000000000000..305a9eb221ae +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/jm/mali_kbase_jm_ioctl.h +@@ -0,0 +1,216 @@ ++/* ++ * ++ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_JM_IOCTL_H_ ++#define _KBASE_JM_IOCTL_H_ ++ ++#include ++#include ++ ++/* ++ * 11.1: ++ * - Add BASE_MEM_TILER_ALIGN_TOP under base_mem_alloc_flags ++ * 11.2: ++ * - KBASE_MEM_QUERY_FLAGS can return KBASE_REG_PF_GROW and KBASE_REG_PROTECTED, ++ * which some user-side clients prior to 11.2 might fault if they received ++ * them ++ * 11.3: ++ * - New ioctls KBASE_IOCTL_STICKY_RESOURCE_MAP and ++ * KBASE_IOCTL_STICKY_RESOURCE_UNMAP ++ * 11.4: ++ * - New ioctl KBASE_IOCTL_MEM_FIND_GPU_START_AND_OFFSET ++ * 11.5: ++ * - New ioctl: KBASE_IOCTL_MEM_JIT_INIT (old ioctl renamed to _OLD) ++ * 11.6: ++ * - Added flags field to base_jit_alloc_info structure, which can be used to ++ * specify pseudo chunked tiler alignment for JIT allocations. ++ * 11.7: ++ * - Removed UMP support ++ * 11.8: ++ * - Added BASE_MEM_UNCACHED_GPU under base_mem_alloc_flags ++ * 11.9: ++ * - Added BASE_MEM_PERMANENT_KERNEL_MAPPING and BASE_MEM_FLAGS_KERNEL_ONLY ++ * under base_mem_alloc_flags ++ * 11.10: ++ * - Enabled the use of nr_extres field of base_jd_atom_v2 structure for ++ * JIT_ALLOC and JIT_FREE type softjobs to enable multiple JIT allocations ++ * with one softjob. ++ * 11.11: ++ * - Added BASE_MEM_GPU_VA_SAME_4GB_PAGE under base_mem_alloc_flags ++ * 11.12: ++ * - Removed ioctl: KBASE_IOCTL_GET_PROFILING_CONTROLS ++ * 11.13: ++ * - New ioctl: KBASE_IOCTL_MEM_EXEC_INIT ++ * 11.14: ++ * - Add BASE_MEM_GROUP_ID_MASK, base_mem_group_id_get, base_mem_group_id_set ++ * under base_mem_alloc_flags ++ * 11.15: ++ * - Added BASEP_CONTEXT_MMU_GROUP_ID_MASK under base_context_create_flags. ++ * - Require KBASE_IOCTL_SET_FLAGS before BASE_MEM_MAP_TRACKING_HANDLE can be ++ * passed to mmap(). ++ * 11.16: ++ * - Extended ioctl KBASE_IOCTL_MEM_SYNC to accept imported dma-buf. ++ * - Modified (backwards compatible) ioctl KBASE_IOCTL_MEM_IMPORT behavior for ++ * dma-buf. Now, buffers are mapped on GPU when first imported, no longer ++ * requiring external resource or sticky resource tracking. UNLESS, ++ * CONFIG_MALI_DMA_BUF_MAP_ON_DEMAND is enabled. ++ * 11.17: ++ * - Added BASE_JD_REQ_JOB_SLOT. ++ * - Reused padding field in base_jd_atom_v2 to pass job slot number. ++ * - New ioctl: KBASE_IOCTL_GET_CPU_GPU_TIMEINFO ++ * 11.18: ++ * - Added BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP under base_mem_alloc_flags ++ * 11.19: ++ * - Extended base_jd_atom_v2 to allow a renderpass ID to be specified. ++ * 11.20: ++ * - Added new phys_pages member to kbase_ioctl_mem_jit_init for ++ * KBASE_IOCTL_MEM_JIT_INIT, previous variants of this renamed to use _10_2 ++ * (replacing '_OLD') and _11_5 suffixes ++ * - Replaced compat_core_req (deprecated in 10.3) with jit_id[2] in ++ * base_jd_atom_v2. It must currently be initialized to zero. ++ * - Added heap_info_gpu_addr to base_jit_alloc_info, and ++ * BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE allowable in base_jit_alloc_info's ++ * flags member. Previous variants of this structure are kept and given _10_2 ++ * and _11_5 suffixes. ++ * - The above changes are checked for safe values in usual builds ++ * 11.21: ++ * - v2.0 of mali_trace debugfs file, which now versions the file separately ++ * 11.22: ++ * - Added base_jd_atom (v3), which is seq_nr + base_jd_atom_v2. ++ * KBASE_IOCTL_JOB_SUBMIT supports both in parallel. ++ * 11.23: ++ * - Modified KBASE_IOCTL_MEM_COMMIT behavior to reject requests to modify ++ * the physical memory backing of JIT allocations. This was not supposed ++ * to be a valid use case, but it was allowed by the previous implementation. ++ * 11.24: ++ * - Added a sysfs file 'serialize_jobs' inside a new sub-directory ++ * 'scheduling'. ++ * 11.25: ++ * - Enabled JIT pressure limit in base/kbase by default ++ * 11.26: ++ * - Added kinstr_jm API ++ * 11.27: ++ * - Backwards compatible extension to HWC ioctl. ++ * 11.28: ++ * - Added kernel side cache ops needed hint ++ * 11.29: ++ * - Reserve ioctl 52 ++ */ ++#define BASE_UK_VERSION_MAJOR 11 ++#define BASE_UK_VERSION_MINOR 29 ++ ++/** ++ * struct kbase_ioctl_version_check - Check version compatibility between ++ * kernel and userspace ++ * ++ * @major: Major version number ++ * @minor: Minor version number ++ */ ++struct kbase_ioctl_version_check { ++ __u16 major; ++ __u16 minor; ++}; ++ ++#define KBASE_IOCTL_VERSION_CHECK \ ++ _IOWR(KBASE_IOCTL_TYPE, 0, struct kbase_ioctl_version_check) ++ ++#define KBASE_IOCTL_VERSION_CHECK_RESERVED \ ++ _IOWR(KBASE_IOCTL_TYPE, 52, struct kbase_ioctl_version_check) ++ ++/** ++ * struct kbase_ioctl_job_submit - Submit jobs/atoms to the kernel ++ * ++ * @addr: Memory address of an array of struct base_jd_atom_v2 or v3 ++ * @nr_atoms: Number of entries in the array ++ * @stride: sizeof(struct base_jd_atom_v2) or sizeof(struct base_jd_atom) ++ */ ++struct kbase_ioctl_job_submit { ++ __u64 addr; ++ __u32 nr_atoms; ++ __u32 stride; ++}; ++ ++#define KBASE_IOCTL_JOB_SUBMIT \ ++ _IOW(KBASE_IOCTL_TYPE, 2, struct kbase_ioctl_job_submit) ++ ++#define KBASE_IOCTL_POST_TERM \ ++ _IO(KBASE_IOCTL_TYPE, 4) ++ ++/** ++ * struct kbase_ioctl_soft_event_update - Update the status of a soft-event ++ * @event: GPU address of the event which has been updated ++ * @new_status: The new status to set ++ * @flags: Flags for future expansion ++ */ ++struct kbase_ioctl_soft_event_update { ++ __u64 event; ++ __u32 new_status; ++ __u32 flags; ++}; ++ ++#define KBASE_IOCTL_SOFT_EVENT_UPDATE \ ++ _IOW(KBASE_IOCTL_TYPE, 28, struct kbase_ioctl_soft_event_update) ++ ++/** ++ * struct kbase_kinstr_jm_fd_out - Explains the compatibility information for ++ * the `struct kbase_kinstr_jm_atom_state_change` structure returned from the ++ * kernel ++ * ++ * @size: The size of the `struct kbase_kinstr_jm_atom_state_change` ++ * @version: Represents a breaking change in the ++ * `struct kbase_kinstr_jm_atom_state_change` ++ * @padding: Explicit padding to get the structure up to 64bits. See ++ * https://www.kernel.org/doc/Documentation/ioctl/botching-up-ioctls.rst ++ * ++ * The `struct kbase_kinstr_jm_atom_state_change` may have extra members at the ++ * end of the structure that older user space might not understand. If the ++ * `version` is the same, the structure is still compatible with newer kernels. ++ * The `size` can be used to cast the opaque memory returned from the kernel. ++ */ ++struct kbase_kinstr_jm_fd_out { ++ __u16 size; ++ __u8 version; ++ __u8 padding[5]; ++}; ++ ++/** ++ * struct kbase_kinstr_jm_fd_in - Options when creating the file descriptor ++ * ++ * @count: Number of atom states that can be stored in the kernel circular ++ * buffer. Must be a power of two ++ * @padding: Explicit padding to get the structure up to 64bits. See ++ * https://www.kernel.org/doc/Documentation/ioctl/botching-up-ioctls.rst ++ */ ++struct kbase_kinstr_jm_fd_in { ++ __u16 count; ++ __u8 padding[6]; ++}; ++ ++union kbase_kinstr_jm_fd { ++ struct kbase_kinstr_jm_fd_in in; ++ struct kbase_kinstr_jm_fd_out out; ++}; ++ ++#define KBASE_IOCTL_KINSTR_JM_FD \ ++ _IOWR(KBASE_IOCTL_TYPE, 51, union kbase_kinstr_jm_fd) ++ ++#endif /* _KBASE_JM_IOCTL_H_ */ +diff --git a/drivers/gpu/arm/bifrost/jm/mali_kbase_jm_js.h b/drivers/gpu/arm/bifrost/jm/mali_kbase_jm_js.h +new file mode 100755 +index 000000000000..6c222ceae8ee +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/jm/mali_kbase_jm_js.h +@@ -0,0 +1,892 @@ ++/* ++ * ++ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* ++ * Job Scheduler Interface. ++ * These interfaces are Internal to KBase. ++ */ ++ ++#ifndef _KBASE_JM_JS_H_ ++#define _KBASE_JM_JS_H_ ++ ++#include "mali_kbase_js_ctx_attr.h" ++ ++/** ++ * kbasep_js_devdata_init - Initialize the Job Scheduler ++ * ++ * The struct kbasep_js_device_data sub-structure of kbdev must be zero ++ * initialized before passing to the kbasep_js_devdata_init() function. This is ++ * to give efficient error path code. ++ */ ++int kbasep_js_devdata_init(struct kbase_device * const kbdev); ++ ++/** ++ * kbasep_js_devdata_halt - Halt the Job Scheduler. ++ * ++ * It is safe to call this on kbdev even if it the kbasep_js_device_data ++ * sub-structure was never initialized/failed initialization, to give efficient ++ * error-path code. ++ * ++ * For this to work, the struct kbasep_js_device_data sub-structure of kbdev ++ * must be zero initialized before passing to the kbasep_js_devdata_init() ++ * function. This is to give efficient error path code. ++ * ++ * It is a programming error to call this whilst there are still kbase_context ++ * structures registered with this scheduler. ++ * ++ */ ++void kbasep_js_devdata_halt(struct kbase_device *kbdev); ++ ++/** ++ * kbasep_js_devdata_term - Terminate the Job Scheduler ++ * ++ * It is safe to call this on kbdev even if it the kbasep_js_device_data ++ * sub-structure was never initialized/failed initialization, to give efficient ++ * error-path code. ++ * ++ * For this to work, the struct kbasep_js_device_data sub-structure of kbdev ++ * must be zero initialized before passing to the kbasep_js_devdata_init() ++ * function. This is to give efficient error path code. ++ * ++ * It is a programming error to call this whilst there are still kbase_context ++ * structures registered with this scheduler. ++ */ ++void kbasep_js_devdata_term(struct kbase_device *kbdev); ++ ++/** ++ * kbasep_js_kctx_init - Initialize the Scheduling Component of a ++ * struct kbase_context on the Job Scheduler. ++ * ++ * This effectively registers a struct kbase_context with a Job Scheduler. ++ * ++ * It does not register any jobs owned by the struct kbase_context with ++ * the scheduler. Those must be separately registered by kbasep_js_add_job(). ++ * ++ * The struct kbase_context must be zero initialized before passing to the ++ * kbase_js_init() function. This is to give efficient error path code. ++ */ ++int kbasep_js_kctx_init(struct kbase_context *const kctx); ++ ++/** ++ * kbasep_js_kctx_term - Terminate the Scheduling Component of a ++ * struct kbase_context on the Job Scheduler ++ * ++ * This effectively de-registers a struct kbase_context from its Job Scheduler ++ * ++ * It is safe to call this on a struct kbase_context that has never had or ++ * failed initialization of its jctx.sched_info member, to give efficient ++ * error-path code. ++ * ++ * For this to work, the struct kbase_context must be zero intitialized before ++ * passing to the kbase_js_init() function. ++ * ++ * It is a Programming Error to call this whilst there are still jobs ++ * registered with this context. ++ */ ++void kbasep_js_kctx_term(struct kbase_context *kctx); ++ ++/** ++ * kbasep_js_add_job - Add a job chain to the Job Scheduler, ++ * and take necessary actions to ++ * schedule the context/run the job. ++ * ++ * This atomically does the following: ++ * * Update the numbers of jobs information ++ * * Add the job to the run pool if necessary (part of init_job) ++ * ++ * Once this is done, then an appropriate action is taken: ++ * * If the ctx is scheduled, it attempts to start the next job (which might be ++ * this added job) ++ * * Otherwise, and if this is the first job on the context, it enqueues it on ++ * the Policy Queue ++ * ++ * The Policy's Queue can be updated by this in the following ways: ++ * * In the above case that this is the first job on the context ++ * * If the context is high priority and the context is not scheduled, then it ++ * could cause the Policy to schedule out a low-priority context, allowing ++ * this context to be scheduled in. ++ * ++ * If the context is already scheduled on the RunPool, then adding a job to it ++ * is guaranteed not to update the Policy Queue. And so, the caller is ++ * guaranteed to not need to try scheduling a context from the Run Pool - it ++ * can safely assert that the result is false. ++ * ++ * It is a programming error to have more than U32_MAX jobs in flight at a time. ++ * ++ * The following locking conditions are made on the caller: ++ * * it must not hold kbasep_js_kctx_info::ctx::jsctx_mutex. ++ * * it must not hold hwaccess_lock (as this will be obtained internally) ++ * * it must not hold kbasep_js_device_data::runpool_mutex (as this will be ++ * obtained internally) ++ * * it must not hold kbasep_jd_device_data::queue_mutex (again, it's used ++ * internally). ++ * ++ * Return: true indicates that the Policy Queue was updated, and so the ++ * caller will need to try scheduling a context onto the Run Pool, ++ * false indicates that no updates were made to the Policy Queue, ++ * so no further action is required from the caller. This is always returned ++ * when the context is currently scheduled. ++ */ ++bool kbasep_js_add_job(struct kbase_context *kctx, struct kbase_jd_atom *atom); ++ ++/** ++ * kbasep_js_remove_job - Remove a job chain from the Job Scheduler, ++ * except for its 'retained state'. ++ * ++ * Completely removing a job requires several calls: ++ * * kbasep_js_copy_atom_retained_state(), to capture the 'retained state' of ++ * the atom ++ * * kbasep_js_remove_job(), to partially remove the atom from the Job Scheduler ++ * * kbasep_js_runpool_release_ctx_and_katom_retained_state(), to release the ++ * remaining state held as part of the job having been run. ++ * ++ * In the common case of atoms completing normally, this set of actions is more ++ * optimal for spinlock purposes than having kbasep_js_remove_job() handle all ++ * of the actions. ++ * ++ * In the case of canceling atoms, it is easier to call ++ * kbasep_js_remove_cancelled_job(), which handles all the necessary actions. ++ * ++ * It is a programming error to call this when: ++ * * a atom is not a job belonging to kctx. ++ * * a atom has already been removed from the Job Scheduler. ++ * * a atom is still in the runpool ++ * ++ * Do not use this for removing jobs being killed by kbase_jd_cancel() - use ++ * kbasep_js_remove_cancelled_job() instead. ++ * ++ * The following locking conditions are made on the caller: ++ * * it must hold kbasep_js_kctx_info::ctx::jsctx_mutex. ++ * ++ */ ++void kbasep_js_remove_job(struct kbase_device *kbdev, ++ struct kbase_context *kctx, struct kbase_jd_atom *atom); ++ ++/** ++ * kbasep_js_remove_cancelled_job - Completely remove a job chain from the ++ * Job Scheduler, in the case ++ * where the job chain was cancelled. ++ * ++ * This is a variant of kbasep_js_remove_job() that takes care of removing all ++ * of the retained state too. This is generally useful for cancelled atoms, ++ * which need not be handled in an optimal way. ++ * ++ * It is a programming error to call this when: ++ * * a atom is not a job belonging to kctx. ++ * * a atom has already been removed from the Job Scheduler. ++ * * a atom is still in the runpool: ++ * * it is not being killed with kbasep_jd_cancel() ++ * ++ * The following locking conditions are made on the caller: ++ * * it must hold kbasep_js_kctx_info::ctx::jsctx_mutex. ++ * * it must not hold the hwaccess_lock, (as this will be obtained ++ * internally) ++ * * it must not hold kbasep_js_device_data::runpool_mutex (as this could be ++ * obtained internally) ++ * ++ * Return: true indicates that ctx attributes have changed and the caller ++ * should call kbase_js_sched_all() to try to run more jobs and ++ * false otherwise. ++ */ ++bool kbasep_js_remove_cancelled_job(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ struct kbase_jd_atom *katom); ++ ++/** ++ * kbasep_js_runpool_requeue_or_kill_ctx - Handling the requeuing/killing of a ++ * context that was evicted from the ++ * policy queue or runpool. ++ * ++ * This should be used whenever handing off a context that has been evicted ++ * from the policy queue or the runpool: ++ * * If the context is not dying and has jobs, it gets re-added to the policy ++ * queue ++ * * Otherwise, it is not added ++ * ++ * In addition, if the context is dying the jobs are killed asynchronously. ++ * ++ * In all cases, the Power Manager active reference is released ++ * (kbase_pm_context_idle()) whenever the has_pm_ref parameter is true. ++ * has_pm_ref must be set to false whenever the context was not previously in ++ * the runpool and does not hold a Power Manager active refcount. Note that ++ * contexts in a rollback of kbasep_js_try_schedule_head_ctx() might have an ++ * active refcount even though they weren't in the runpool. ++ * ++ * The following locking conditions are made on the caller: ++ * * it must hold kbasep_js_kctx_info::ctx::jsctx_mutex. ++ * * it must not hold kbasep_jd_device_data::queue_mutex (as this will be ++ * obtained internally) ++ */ ++void kbasep_js_runpool_requeue_or_kill_ctx(struct kbase_device *kbdev, ++ struct kbase_context *kctx, bool has_pm_ref); ++ ++/** ++ * kbasep_js_runpool_release_ctx - Release a refcount of a context being busy, ++ * allowing it to be scheduled out. ++ * ++ * When the refcount reaches zero and the context might be scheduled out ++ * (depending on whether the Scheduling Policy has deemed it so, or if it has ++ * run out of jobs). ++ * ++ * If the context does get scheduled out, then The following actions will be ++ * taken as part of deschduling a context: ++ * For the context being descheduled: ++ * * If the context is in the processing of dying (all the jobs are being ++ * removed from it), then descheduling also kills off any jobs remaining in the ++ * context. ++ * * If the context is not dying, and any jobs remain after descheduling the ++ * context then it is re-enqueued to the Policy's Queue. ++ * * Otherwise, the context is still known to the scheduler, but remains absent ++ * from the Policy Queue until a job is next added to it. ++ * * In all descheduling cases, the Power Manager active reference (obtained ++ * during kbasep_js_try_schedule_head_ctx()) is released ++ * (kbase_pm_context_idle()). ++ * ++ * Whilst the context is being descheduled, this also handles actions that ++ * cause more atoms to be run: ++ * * Attempt submitting atoms when the Context Attributes on the Runpool have ++ * changed. This is because the context being scheduled out could mean that ++ * there are more opportunities to run atoms. ++ * * Attempt submitting to a slot that was previously blocked due to affinity ++ * restrictions. This is usually only necessary when releasing a context ++ * happens as part of completing a previous job, but is harmless nonetheless. ++ * * Attempt scheduling in a new context (if one is available), and if ++ * necessary, running a job from that new context. ++ * ++ * Unlike retaining a context in the runpool, this function cannot be called ++ * from IRQ context. ++ * ++ * It is a programming error to call this on a kctx that is not currently ++ * scheduled, or that already has a zero refcount. ++ * ++ * The following locking conditions are made on the caller: ++ * * it must not hold the hwaccess_lock, because it will be used internally. ++ * * it must not hold kbasep_js_kctx_info::ctx::jsctx_mutex. ++ * * it must not hold kbasep_js_device_data::runpool_mutex (as this will be ++ * obtained internally) ++ * * it must not hold the kbase_device::mmu_hw_mutex (as this will be ++ * obtained internally) ++ * * it must not hold kbasep_jd_device_data::queue_mutex (as this will be ++ * obtained internally) ++ * ++ */ ++void kbasep_js_runpool_release_ctx(struct kbase_device *kbdev, ++ struct kbase_context *kctx); ++ ++/** ++ * kbasep_js_runpool_release_ctx_and_katom_retained_state - Variant of ++ * kbasep_js_runpool_release_ctx() that handles additional ++ * actions from completing an atom. ++ * ++ * This is usually called as part of completing an atom and releasing the ++ * refcount on the context held by the atom. ++ * ++ * Therefore, the extra actions carried out are part of handling actions queued ++ * on a completed atom, namely: ++ * * Releasing the atom's context attributes ++ * * Retrying the submission on a particular slot, because we couldn't submit ++ * on that slot from an IRQ handler. ++ * ++ * The locking conditions of this function are the same as those for ++ * kbasep_js_runpool_release_ctx() ++ */ ++void kbasep_js_runpool_release_ctx_and_katom_retained_state( ++ struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ struct kbasep_js_atom_retained_state *katom_retained_state); ++ ++/** ++ * kbasep_js_runpool_release_ctx_nolock - Variant of ++ * kbase_js_runpool_release_ctx() that assumes that ++ * kbasep_js_device_data::runpool_mutex and ++ * kbasep_js_kctx_info::ctx::jsctx_mutex are held by the caller, and does not ++ * attempt to schedule new contexts. ++ */ ++void kbasep_js_runpool_release_ctx_nolock(struct kbase_device *kbdev, ++ struct kbase_context *kctx); ++ ++/** ++ * kbasep_js_schedule_privileged_ctx - Schedule in a privileged context ++ * ++ * This schedules a context in regardless of the context priority. ++ * If the runpool is full, a context will be forced out of the runpool and the ++ * function will wait for the new context to be scheduled in. ++ * The context will be kept scheduled in (and the corresponding address space ++ * reserved) until kbasep_js_release_privileged_ctx is called). ++ * ++ * The following locking conditions are made on the caller: ++ * * it must not hold the hwaccess_lock, because it will be used internally. ++ * * it must not hold kbasep_js_device_data::runpool_mutex (as this will be ++ * obtained internally) ++ * * it must not hold the kbase_device::mmu_hw_mutex (as this will be ++ * obtained internally) ++ * * it must not hold kbasep_jd_device_data::queue_mutex (again, it's used ++ * internally). ++ * * it must not hold kbasep_js_kctx_info::ctx::jsctx_mutex, because it will ++ * be used internally. ++ * ++ */ ++void kbasep_js_schedule_privileged_ctx(struct kbase_device *kbdev, ++ struct kbase_context *kctx); ++ ++/** ++ * kbasep_js_release_privileged_ctx - Release a privileged context, ++ * allowing it to be scheduled out. ++ * ++ * See kbasep_js_runpool_release_ctx for potential side effects. ++ * ++ * The following locking conditions are made on the caller: ++ * * it must not hold the hwaccess_lock, because it will be used internally. ++ * * it must not hold kbasep_js_kctx_info::ctx::jsctx_mutex. ++ * * it must not hold kbasep_js_device_data::runpool_mutex (as this will be ++ * obtained internally) ++ * * it must not hold the kbase_device::mmu_hw_mutex (as this will be ++ * obtained internally) ++ * ++ */ ++void kbasep_js_release_privileged_ctx(struct kbase_device *kbdev, ++ struct kbase_context *kctx); ++ ++/** ++ * kbase_js_try_run_jobs - Try to submit the next job on each slot ++ * ++ * The following locks may be used: ++ * * kbasep_js_device_data::runpool_mutex ++ * * hwaccess_lock ++ */ ++void kbase_js_try_run_jobs(struct kbase_device *kbdev); ++ ++/** ++ * kbasep_js_suspend - Suspend the job scheduler during a Power Management ++ * Suspend event. ++ * ++ * Causes all contexts to be removed from the runpool, and prevents any ++ * contexts from (re)entering the runpool. ++ * ++ * This does not handle suspending the one privileged context: the caller must ++ * instead do this by by suspending the GPU HW Counter Instrumentation. ++ * ++ * This will eventually cause all Power Management active references held by ++ * contexts on the runpool to be released, without running any more atoms. ++ * ++ * The caller must then wait for all Power Management active refcount to become ++ * zero before completing the suspend. ++ * ++ * The emptying mechanism may take some time to complete, since it can wait for ++ * jobs to complete naturally instead of forcing them to end quickly. However, ++ * this is bounded by the Job Scheduler's Job Timeouts. Hence, this ++ * function is guaranteed to complete in a finite time. ++ */ ++void kbasep_js_suspend(struct kbase_device *kbdev); ++ ++/** ++ * kbasep_js_resume - Resume the Job Scheduler after a Power Management ++ * Resume event. ++ * ++ * This restores the actions from kbasep_js_suspend(): ++ * * Schedules contexts back into the runpool ++ * * Resumes running atoms on the GPU ++ */ ++void kbasep_js_resume(struct kbase_device *kbdev); ++ ++/** ++ * kbase_js_dep_resolved_submit - Submit an atom to the job scheduler. ++ * ++ * @kctx: Context pointer ++ * @atom: Pointer to the atom to submit ++ * ++ * The atom is enqueued on the context's ringbuffer. The caller must have ++ * ensured that all dependencies can be represented in the ringbuffer. ++ * ++ * Caller must hold jctx->lock ++ * ++ * Return: true if the context requires to be enqueued, otherwise false. ++ */ ++bool kbase_js_dep_resolved_submit(struct kbase_context *kctx, ++ struct kbase_jd_atom *katom); ++ ++/** ++ * jsctx_ll_flush_to_rb() - Pushes atoms from the linked list to ringbuffer. ++ * @kctx: Context Pointer ++ * @prio: Priority (specifies the queue together with js). ++ * @js: Job slot (specifies the queue together with prio). ++ * ++ * Pushes all possible atoms from the linked list to the ringbuffer. ++ * Number of atoms are limited to free space in the ringbuffer and ++ * number of available atoms in the linked list. ++ * ++ */ ++void jsctx_ll_flush_to_rb(struct kbase_context *kctx, int prio, int js); ++ ++/** ++ * kbase_js_pull - Pull an atom from a context in the job scheduler for ++ * execution. ++ * ++ * @kctx: Context to pull from ++ * @js: Job slot to pull from ++ * ++ * The atom will not be removed from the ringbuffer at this stage. ++ * ++ * The HW access lock must be held when calling this function. ++ * ++ * Return: a pointer to an atom, or NULL if there are no atoms for this ++ * slot that can be currently run. ++ */ ++struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, int js); ++ ++/** ++ * kbase_js_unpull - Return an atom to the job scheduler ringbuffer. ++ * ++ * @kctx: Context pointer ++ * @atom: Pointer to the atom to unpull ++ * ++ * An atom is 'unpulled' if execution is stopped but intended to be returned to ++ * later. The most common reason for this is that the atom has been ++ * soft-stopped. Another reason is if an end-of-renderpass atom completed ++ * but will need to be run again as part of the same renderpass. ++ * ++ * Note that if multiple atoms are to be 'unpulled', they must be returned in ++ * the reverse order to which they were originally pulled. It is a programming ++ * error to return atoms in any other order. ++ * ++ * The HW access lock must be held when calling this function. ++ * ++ */ ++void kbase_js_unpull(struct kbase_context *kctx, struct kbase_jd_atom *katom); ++ ++/** ++ * kbase_js_complete_atom_wq - Complete an atom from jd_done_worker(), ++ * removing it from the job ++ * scheduler ringbuffer. ++ * @kctx: Context pointer ++ * @katom: Pointer to the atom to complete ++ * ++ * If the atom failed then all dependee atoms marked for failure propagation ++ * will also fail. ++ * ++ * Return: true if the context is now idle (no jobs pulled) false otherwise. ++ */ ++bool kbase_js_complete_atom_wq(struct kbase_context *kctx, ++ struct kbase_jd_atom *katom); ++ ++/** ++ * kbase_js_complete_atom - Complete an atom. ++ * ++ * @katom: Pointer to the atom to complete ++ * @end_timestamp: The time that the atom completed (may be NULL) ++ * ++ * Most of the work required to complete an atom will be performed by ++ * jd_done_worker(). ++ * ++ * The HW access lock must be held when calling this function. ++ * ++ * Return: a atom that has now been unblocked and can now be run, or NULL ++ * if none ++ */ ++struct kbase_jd_atom *kbase_js_complete_atom(struct kbase_jd_atom *katom, ++ ktime_t *end_timestamp); ++ ++/** ++ * kbase_js_atom_blocked_on_x_dep - Decide whether to ignore a cross-slot ++ * dependency ++ * @katom: Pointer to an atom in the slot ringbuffer ++ * ++ * A cross-slot dependency is ignored if necessary to unblock incremental ++ * rendering. If the atom at the start of a renderpass used too much memory ++ * and was soft-stopped then the atom at the end of a renderpass is submitted ++ * to hardware regardless of its dependency on the start-of-renderpass atom. ++ * This can happen multiple times for the same pair of atoms. ++ * ++ * Return: true to block the atom or false to allow it to be submitted to ++ * hardware. ++ */ ++bool kbase_js_atom_blocked_on_x_dep(struct kbase_jd_atom *katom); ++ ++/** ++ * kbase_js_sched - Submit atoms from all available contexts. ++ * ++ * @kbdev: Device pointer ++ * @js_mask: Mask of job slots to submit to ++ * ++ * This will attempt to submit as many jobs as possible to the provided job ++ * slots. It will exit when either all job slots are full, or all contexts have ++ * been used. ++ * ++ */ ++void kbase_js_sched(struct kbase_device *kbdev, int js_mask); ++ ++/** ++ * kbase_jd_zap_context - Attempt to deschedule a context that is being ++ * destroyed ++ * @kctx: Context pointer ++ * ++ * This will attempt to remove a context from any internal job scheduler queues ++ * and perform any other actions to ensure a context will not be submitted ++ * from. ++ * ++ * If the context is currently scheduled, then the caller must wait for all ++ * pending jobs to complete before taking any further action. ++ */ ++void kbase_js_zap_context(struct kbase_context *kctx); ++ ++/** ++ * kbase_js_is_atom_valid - Validate an atom ++ * ++ * @kbdev: Device pointer ++ * @katom: Atom to validate ++ * ++ * This will determine whether the atom can be scheduled onto the GPU. Atoms ++ * with invalid combinations of core requirements will be rejected. ++ * ++ * Return: true if atom is valid false otherwise. ++ */ ++bool kbase_js_is_atom_valid(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom); ++ ++/** ++ * kbase_js_set_timeouts - update all JS timeouts with user specified data ++ * ++ * @kbdev: Device pointer ++ * ++ * Timeouts are specified through the 'js_timeouts' sysfs file. If a timeout is ++ * set to a positive number then that becomes the new value used, if a timeout ++ * is negative then the default is set. ++ */ ++void kbase_js_set_timeouts(struct kbase_device *kbdev); ++ ++/** ++ * kbase_js_set_ctx_priority - set the context priority ++ * ++ * @kctx: Context pointer ++ * @new_priority: New priority value for the Context ++ * ++ * The context priority is set to a new value and it is moved to the ++ * pullable/unpullable list as per the new priority. ++ */ ++void kbase_js_set_ctx_priority(struct kbase_context *kctx, int new_priority); ++ ++ ++/** ++ * kbase_js_update_ctx_priority - update the context priority ++ * ++ * @kctx: Context pointer ++ * ++ * The context priority gets updated as per the priority of atoms currently in ++ * use for that context, but only if system priority mode for context scheduling ++ * is being used. ++ */ ++void kbase_js_update_ctx_priority(struct kbase_context *kctx); ++ ++/* ++ * Helpers follow ++ */ ++ ++/** ++ * kbasep_js_is_submit_allowed - Check that a context is allowed to submit ++ * jobs on this policy ++ * ++ * The purpose of this abstraction is to hide the underlying data size, ++ * and wrap up the long repeated line of code. ++ * ++ * As with any bool, never test the return value with true. ++ * ++ * The caller must hold hwaccess_lock. ++ */ ++static inline bool kbasep_js_is_submit_allowed( ++ struct kbasep_js_device_data *js_devdata, ++ struct kbase_context *kctx) ++{ ++ u16 test_bit; ++ bool is_allowed; ++ ++ /* Ensure context really is scheduled in */ ++ KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); ++ KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ ++ test_bit = (u16) (1u << kctx->as_nr); ++ ++ is_allowed = (bool) (js_devdata->runpool_irq.submit_allowed & test_bit); ++ dev_dbg(kctx->kbdev->dev, "JS: submit %s allowed on %p (as=%d)", ++ is_allowed ? "is" : "isn't", (void *)kctx, kctx->as_nr); ++ return is_allowed; ++} ++ ++/** ++ * kbasep_js_set_submit_allowed - Allow a context to submit jobs on this policy ++ * ++ * The purpose of this abstraction is to hide the underlying data size, ++ * and wrap up the long repeated line of code. ++ * ++ * The caller must hold hwaccess_lock. ++ */ ++static inline void kbasep_js_set_submit_allowed( ++ struct kbasep_js_device_data *js_devdata, ++ struct kbase_context *kctx) ++{ ++ u16 set_bit; ++ ++ /* Ensure context really is scheduled in */ ++ KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); ++ KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ ++ set_bit = (u16) (1u << kctx->as_nr); ++ ++ dev_dbg(kctx->kbdev->dev, "JS: Setting Submit Allowed on %p (as=%d)", ++ kctx, kctx->as_nr); ++ ++ js_devdata->runpool_irq.submit_allowed |= set_bit; ++} ++ ++/** ++ * kbasep_js_clear_submit_allowed - Prevent a context from submitting more ++ * jobs on this policy ++ * ++ * The purpose of this abstraction is to hide the underlying data size, ++ * and wrap up the long repeated line of code. ++ * ++ * The caller must hold hwaccess_lock. ++ */ ++static inline void kbasep_js_clear_submit_allowed( ++ struct kbasep_js_device_data *js_devdata, ++ struct kbase_context *kctx) ++{ ++ u16 clear_bit; ++ u16 clear_mask; ++ ++ /* Ensure context really is scheduled in */ ++ KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); ++ KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ ++ clear_bit = (u16) (1u << kctx->as_nr); ++ clear_mask = ~clear_bit; ++ ++ dev_dbg(kctx->kbdev->dev, "JS: Clearing Submit Allowed on %p (as=%d)", ++ kctx, kctx->as_nr); ++ ++ js_devdata->runpool_irq.submit_allowed &= clear_mask; ++} ++ ++/** ++ * Create an initial 'invalid' atom retained state, that requires no ++ * atom-related work to be done on releasing with ++ * kbasep_js_runpool_release_ctx_and_katom_retained_state() ++ */ ++static inline void kbasep_js_atom_retained_state_init_invalid( ++ struct kbasep_js_atom_retained_state *retained_state) ++{ ++ retained_state->event_code = BASE_JD_EVENT_NOT_STARTED; ++ retained_state->core_req = ++ KBASEP_JS_ATOM_RETAINED_STATE_CORE_REQ_INVALID; ++} ++ ++/** ++ * Copy atom state that can be made available after jd_done_nolock() is called ++ * on that atom. ++ */ ++static inline void kbasep_js_atom_retained_state_copy( ++ struct kbasep_js_atom_retained_state *retained_state, ++ const struct kbase_jd_atom *katom) ++{ ++ retained_state->event_code = katom->event_code; ++ retained_state->core_req = katom->core_req; ++ retained_state->sched_priority = katom->sched_priority; ++ retained_state->device_nr = katom->device_nr; ++} ++ ++/** ++ * kbasep_js_has_atom_finished - Determine whether an atom has finished ++ * (given its retained state), ++ * and so should be given back to ++ * userspace/removed from the system. ++ * ++ * @katom_retained_state: the retained state of the atom to check ++ * ++ * Reasons for an atom not finishing include: ++ * * Being soft-stopped (and so, the atom should be resubmitted sometime later) ++ * * It is an end of renderpass atom that was run to consume the output of a ++ * start-of-renderpass atom that was soft-stopped because it used too much ++ * memory. In this case, it will have to be run again later. ++ * ++ * Return: false if the atom has not finished, true otherwise. ++ */ ++static inline bool kbasep_js_has_atom_finished( ++ const struct kbasep_js_atom_retained_state *katom_retained_state) ++{ ++ return (bool) (katom_retained_state->event_code != ++ BASE_JD_EVENT_STOPPED && ++ katom_retained_state->event_code != ++ BASE_JD_EVENT_REMOVED_FROM_NEXT && ++ katom_retained_state->event_code != ++ BASE_JD_EVENT_END_RP_DONE); ++} ++ ++/** ++ * kbasep_js_atom_retained_state_is_valid - Determine whether a struct ++ * kbasep_js_atom_retained_state ++ * is valid ++ * @katom_retained_state the atom's retained state to check ++ * ++ * An invalid struct kbasep_js_atom_retained_state is allowed, and indicates ++ * that the code should just ignore it. ++ * ++ * Return: false if the retained state is invalid, true otherwise. ++ */ ++static inline bool kbasep_js_atom_retained_state_is_valid( ++ const struct kbasep_js_atom_retained_state *katom_retained_state) ++{ ++ return (bool) (katom_retained_state->core_req != ++ KBASEP_JS_ATOM_RETAINED_STATE_CORE_REQ_INVALID); ++} ++ ++/** ++ * kbase_js_runpool_inc_context_count - Increment number of running contexts. ++ * ++ * The following locking conditions are made on the caller: ++ * * The caller must hold the kbasep_js_kctx_info::ctx::jsctx_mutex. ++ * * The caller must hold the kbasep_js_device_data::runpool_mutex ++ */ ++static inline void kbase_js_runpool_inc_context_count( ++ struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ ++ js_devdata = &kbdev->js_data; ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); ++ lockdep_assert_held(&js_devdata->runpool_mutex); ++ ++ /* Track total contexts */ ++ KBASE_DEBUG_ASSERT(js_devdata->nr_all_contexts_running < S8_MAX); ++ ++(js_devdata->nr_all_contexts_running); ++ ++ if (!kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) { ++ /* Track contexts that can submit jobs */ ++ KBASE_DEBUG_ASSERT(js_devdata->nr_user_contexts_running < ++ S8_MAX); ++ ++(js_devdata->nr_user_contexts_running); ++ } ++} ++ ++/** ++ * kbase_js_runpool_dec_context_count - decrement number of running contexts. ++ * ++ * The following locking conditions are made on the caller: ++ * * The caller must hold the kbasep_js_kctx_info::ctx::jsctx_mutex. ++ * * The caller must hold the kbasep_js_device_data::runpool_mutex ++ */ ++static inline void kbase_js_runpool_dec_context_count( ++ struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ ++ js_devdata = &kbdev->js_data; ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); ++ lockdep_assert_held(&js_devdata->runpool_mutex); ++ ++ /* Track total contexts */ ++ --(js_devdata->nr_all_contexts_running); ++ KBASE_DEBUG_ASSERT(js_devdata->nr_all_contexts_running >= 0); ++ ++ if (!kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) { ++ /* Track contexts that can submit jobs */ ++ --(js_devdata->nr_user_contexts_running); ++ KBASE_DEBUG_ASSERT(js_devdata->nr_user_contexts_running >= 0); ++ } ++} ++ ++/** ++ * kbase_js_sched_all - Submit atoms from all available contexts to all ++ * job slots. ++ * ++ * @kbdev: Device pointer ++ * ++ * This will attempt to submit as many jobs as possible. It will exit when ++ * either all job slots are full, or all contexts have been used. ++ */ ++static inline void kbase_js_sched_all(struct kbase_device *kbdev) ++{ ++ kbase_js_sched(kbdev, (1 << kbdev->gpu_props.num_job_slots) - 1); ++} ++ ++extern const int ++kbasep_js_atom_priority_to_relative[BASE_JD_NR_PRIO_LEVELS]; ++ ++extern const base_jd_prio ++kbasep_js_relative_priority_to_atom[KBASE_JS_ATOM_SCHED_PRIO_COUNT]; ++ ++/** ++ * kbasep_js_atom_prio_to_sched_prio(): - Convert atom priority (base_jd_prio) ++ * to relative ordering ++ * @atom_prio: Priority ID to translate. ++ * ++ * Atom priority values for @ref base_jd_prio cannot be compared directly to ++ * find out which are higher or lower. ++ * ++ * This function will convert base_jd_prio values for successively lower ++ * priorities into a monotonically increasing sequence. That is, the lower the ++ * base_jd_prio priority, the higher the value produced by this function. This ++ * is in accordance with how the rest of the kernel treats priority. ++ * ++ * The mapping is 1:1 and the size of the valid input range is the same as the ++ * size of the valid output range, i.e. ++ * KBASE_JS_ATOM_SCHED_PRIO_COUNT == BASE_JD_NR_PRIO_LEVELS ++ * ++ * Note This must be kept in sync with BASE_JD_PRIO_<...> definitions ++ * ++ * Return: On success: a value in the inclusive range ++ * 0..KBASE_JS_ATOM_SCHED_PRIO_COUNT-1. On failure: ++ * KBASE_JS_ATOM_SCHED_PRIO_INVALID ++ */ ++static inline int kbasep_js_atom_prio_to_sched_prio(base_jd_prio atom_prio) ++{ ++ if (atom_prio >= BASE_JD_NR_PRIO_LEVELS) ++ return KBASE_JS_ATOM_SCHED_PRIO_INVALID; ++ ++ return kbasep_js_atom_priority_to_relative[atom_prio]; ++} ++ ++static inline base_jd_prio kbasep_js_sched_prio_to_atom_prio(int sched_prio) ++{ ++ unsigned int prio_idx; ++ ++ KBASE_DEBUG_ASSERT(sched_prio >= 0 && ++ sched_prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT); ++ ++ prio_idx = (unsigned int)sched_prio; ++ ++ return kbasep_js_relative_priority_to_atom[prio_idx]; ++} ++ ++#endif /* _KBASE_JM_JS_H_ */ +diff --git a/drivers/gpu/arm/bifrost/jm/mali_kbase_js_defs.h b/drivers/gpu/arm/bifrost/jm/mali_kbase_js_defs.h +new file mode 100755 +index 000000000000..900ecd2c1b8d +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/jm/mali_kbase_js_defs.h +@@ -0,0 +1,409 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2018, 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++/** ++ * @file mali_kbase_js.h ++ * Job Scheduler Type Definitions ++ */ ++ ++#ifndef _KBASE_JS_DEFS_H_ ++#define _KBASE_JS_DEFS_H_ ++ ++/** ++ * @addtogroup base_api ++ * @{ ++ */ ++ ++/** ++ * @addtogroup base_kbase_api ++ * @{ ++ */ ++ ++/** ++ * @addtogroup kbase_js ++ * @{ ++ */ ++/* Forward decls */ ++struct kbase_device; ++struct kbase_jd_atom; ++ ++ ++typedef u32 kbase_context_flags; ++ ++/** Callback function run on all of a context's jobs registered with the Job ++ * Scheduler */ ++typedef void (*kbasep_js_ctx_job_cb)(struct kbase_device *kbdev, struct kbase_jd_atom *katom); ++ ++/** ++ * @brief Maximum number of jobs that can be submitted to a job slot whilst ++ * inside the IRQ handler. ++ * ++ * This is important because GPU NULL jobs can complete whilst the IRQ handler ++ * is running. Otherwise, it potentially allows an unlimited number of GPU NULL ++ * jobs to be submitted inside the IRQ handler, which increases IRQ latency. ++ */ ++#define KBASE_JS_MAX_JOB_SUBMIT_PER_SLOT_PER_IRQ 2 ++ ++/** ++ * @brief Context attributes ++ * ++ * Each context attribute can be thought of as a boolean value that caches some ++ * state information about either the runpool, or the context: ++ * - In the case of the runpool, it is a cache of "Do any contexts owned by ++ * the runpool have attribute X?" ++ * - In the case of a context, it is a cache of "Do any atoms owned by the ++ * context have attribute X?" ++ * ++ * The boolean value of the context attributes often affect scheduling ++ * decisions, such as affinities to use and job slots to use. ++ * ++ * To accomodate changes of state in the context, each attribute is refcounted ++ * in the context, and in the runpool for all running contexts. Specifically: ++ * - The runpool holds a refcount of how many contexts in the runpool have this ++ * attribute. ++ * - The context holds a refcount of how many atoms have this attribute. ++ */ ++enum kbasep_js_ctx_attr { ++ /** Attribute indicating a context that contains Compute jobs. That is, ++ * the context has jobs of type @ref BASE_JD_REQ_ONLY_COMPUTE ++ * ++ * @note A context can be both 'Compute' and 'Non Compute' if it contains ++ * both types of jobs. ++ */ ++ KBASEP_JS_CTX_ATTR_COMPUTE, ++ ++ /** Attribute indicating a context that contains Non-Compute jobs. That is, ++ * the context has some jobs that are \b not of type @ref ++ * BASE_JD_REQ_ONLY_COMPUTE. ++ * ++ * @note A context can be both 'Compute' and 'Non Compute' if it contains ++ * both types of jobs. ++ */ ++ KBASEP_JS_CTX_ATTR_NON_COMPUTE, ++ ++ /** Attribute indicating that a context contains compute-job atoms that ++ * aren't restricted to a coherent group, and can run on all cores. ++ * ++ * Specifically, this is when the atom's \a core_req satisfy: ++ * - (\a core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE | BASE_JD_REQ_T) // uses slot 1 or slot 2 ++ * - && !(\a core_req & BASE_JD_REQ_COHERENT_GROUP) // not restricted to coherent groups ++ * ++ * Such atoms could be blocked from running if one of the coherent groups ++ * is being used by another job slot, so tracking this context attribute ++ * allows us to prevent such situations. ++ * ++ * @note This doesn't take into account the 1-coregroup case, where all ++ * compute atoms would effectively be able to run on 'all cores', but ++ * contexts will still not always get marked with this attribute. Instead, ++ * it is the caller's responsibility to take into account the number of ++ * coregroups when interpreting this attribute. ++ * ++ * @note Whilst Tiler atoms are normally combined with ++ * BASE_JD_REQ_COHERENT_GROUP, it is possible to send such atoms without ++ * BASE_JD_REQ_COHERENT_GROUP set. This is an unlikely case, but it's easy ++ * enough to handle anyway. ++ */ ++ KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES, ++ ++ /** Must be the last in the enum */ ++ KBASEP_JS_CTX_ATTR_COUNT ++}; ++ ++enum { ++ /** Bit indicating that new atom should be started because this atom completed */ ++ KBASE_JS_ATOM_DONE_START_NEW_ATOMS = (1u << 0), ++ /** Bit indicating that the atom was evicted from the JS_NEXT registers */ ++ KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT = (1u << 1) ++}; ++ ++/** Combination of KBASE_JS_ATOM_DONE_<...> bits */ ++typedef u32 kbasep_js_atom_done_code; ++ ++/* ++ * Context scheduling mode defines for kbase_device::js_ctx_scheduling_mode ++ */ ++enum { ++ /* ++ * In this mode, higher priority atoms will be scheduled first, ++ * regardless of the context they belong to. Newly-runnable higher ++ * priority atoms can preempt lower priority atoms currently running on ++ * the GPU, even if they belong to a different context. ++ */ ++ KBASE_JS_SYSTEM_PRIORITY_MODE = 0, ++ ++ /* ++ * In this mode, the highest-priority atom will be chosen from each ++ * context in turn using a round-robin algorithm, so priority only has ++ * an effect within the context an atom belongs to. Newly-runnable ++ * higher priority atoms can preempt the lower priority atoms currently ++ * running on the GPU, but only if they belong to the same context. ++ */ ++ KBASE_JS_PROCESS_LOCAL_PRIORITY_MODE, ++ ++ /* Must be the last in the enum */ ++ KBASE_JS_PRIORITY_MODE_COUNT, ++}; ++ ++/* ++ * Internal atom priority defines for kbase_jd_atom::sched_prio ++ */ ++enum { ++ KBASE_JS_ATOM_SCHED_PRIO_HIGH = 0, ++ KBASE_JS_ATOM_SCHED_PRIO_MED, ++ KBASE_JS_ATOM_SCHED_PRIO_LOW, ++ KBASE_JS_ATOM_SCHED_PRIO_COUNT, ++}; ++ ++/* Invalid priority for kbase_jd_atom::sched_prio */ ++#define KBASE_JS_ATOM_SCHED_PRIO_INVALID -1 ++ ++/* Default priority in the case of contexts with no atoms, or being lenient ++ * about invalid priorities from userspace. ++ */ ++#define KBASE_JS_ATOM_SCHED_PRIO_DEFAULT KBASE_JS_ATOM_SCHED_PRIO_MED ++ ++/** ++ * @brief KBase Device Data Job Scheduler sub-structure ++ * ++ * This encapsulates the current context of the Job Scheduler on a particular ++ * device. This context is global to the device, and is not tied to any ++ * particular struct kbase_context running on the device. ++ * ++ * nr_contexts_running and as_free are optimized for packing together (by making ++ * them smaller types than u32). The operations on them should rarely involve ++ * masking. The use of signed types for arithmetic indicates to the compiler that ++ * the value will not rollover (which would be undefined behavior), and so under ++ * the Total License model, it is free to make optimizations based on that (i.e. ++ * to remove masking). ++ */ ++struct kbasep_js_device_data { ++ /* Sub-structure to collect together Job Scheduling data used in IRQ ++ * context. The hwaccess_lock must be held when accessing. */ ++ struct runpool_irq { ++ /** Bitvector indicating whether a currently scheduled context is allowed to submit jobs. ++ * When bit 'N' is set in this, it indicates whether the context bound to address space ++ * 'N' is allowed to submit jobs. ++ */ ++ u16 submit_allowed; ++ ++ /** Context Attributes: ++ * Each is large enough to hold a refcount of the number of contexts ++ * that can fit into the runpool. This is currently BASE_MAX_NR_AS ++ * ++ * Note that when BASE_MAX_NR_AS==16 we need 5 bits (not 4) to store ++ * the refcount. Hence, it's not worthwhile reducing this to ++ * bit-manipulation on u32s to save space (where in contrast, 4 bit ++ * sub-fields would be easy to do and would save space). ++ * ++ * Whilst this must not become negative, the sign bit is used for: ++ * - error detection in debug builds ++ * - Optimization: it is undefined for a signed int to overflow, and so ++ * the compiler can optimize for that never happening (thus, no masking ++ * is required on updating the variable) */ ++ s8 ctx_attr_ref_count[KBASEP_JS_CTX_ATTR_COUNT]; ++ ++ /* ++ * Affinity management and tracking ++ */ ++ /** Bitvector to aid affinity checking. Element 'n' bit 'i' indicates ++ * that slot 'n' is using core i (i.e. slot_affinity_refcount[n][i] > 0) */ ++ u64 slot_affinities[BASE_JM_MAX_NR_SLOTS]; ++ /** Refcount for each core owned by each slot. Used to generate the ++ * slot_affinities array of bitvectors ++ * ++ * The value of the refcount will not exceed BASE_JM_SUBMIT_SLOTS, ++ * because it is refcounted only when a job is definitely about to be ++ * submitted to a slot, and is de-refcounted immediately after a job ++ * finishes */ ++ s8 slot_affinity_refcount[BASE_JM_MAX_NR_SLOTS][64]; ++ } runpool_irq; ++ ++ /** ++ * Scheduling semaphore. This must be held when calling ++ * kbase_jm_kick() ++ */ ++ struct semaphore schedule_sem; ++ ++ /** ++ * List of contexts that can currently be pulled from ++ */ ++ struct list_head ctx_list_pullable[BASE_JM_MAX_NR_SLOTS][KBASE_JS_ATOM_SCHED_PRIO_COUNT]; ++ /** ++ * List of contexts that can not currently be pulled from, but have ++ * jobs currently running. ++ */ ++ struct list_head ctx_list_unpullable[BASE_JM_MAX_NR_SLOTS][KBASE_JS_ATOM_SCHED_PRIO_COUNT]; ++ ++ /** Number of currently scheduled user contexts (excluding ones that are not submitting jobs) */ ++ s8 nr_user_contexts_running; ++ /** Number of currently scheduled contexts (including ones that are not submitting jobs) */ ++ s8 nr_all_contexts_running; ++ ++ /** Core Requirements to match up with base_js_atom's core_req memeber ++ * @note This is a write-once member, and so no locking is required to read */ ++ base_jd_core_req js_reqs[BASE_JM_MAX_NR_SLOTS]; ++ ++ u32 scheduling_period_ns; /*< Value for JS_SCHEDULING_PERIOD_NS */ ++ u32 soft_stop_ticks; /*< Value for JS_SOFT_STOP_TICKS */ ++ u32 soft_stop_ticks_cl; /*< Value for JS_SOFT_STOP_TICKS_CL */ ++ u32 hard_stop_ticks_ss; /*< Value for JS_HARD_STOP_TICKS_SS */ ++ u32 hard_stop_ticks_cl; /*< Value for JS_HARD_STOP_TICKS_CL */ ++ u32 hard_stop_ticks_dumping; /*< Value for JS_HARD_STOP_TICKS_DUMPING */ ++ u32 gpu_reset_ticks_ss; /*< Value for JS_RESET_TICKS_SS */ ++ u32 gpu_reset_ticks_cl; /*< Value for JS_RESET_TICKS_CL */ ++ u32 gpu_reset_ticks_dumping; /*< Value for JS_RESET_TICKS_DUMPING */ ++ u32 ctx_timeslice_ns; /**< Value for JS_CTX_TIMESLICE_NS */ ++ ++ /** List of suspended soft jobs */ ++ struct list_head suspended_soft_jobs_list; ++ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++ /* Support soft-stop on a single context */ ++ bool softstop_always; ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++ ++ /** The initalized-flag is placed at the end, to avoid cache-pollution (we should ++ * only be using this during init/term paths). ++ * @note This is a write-once member, and so no locking is required to read */ ++ int init_status; ++ ++ /* Number of contexts that can currently be pulled from */ ++ u32 nr_contexts_pullable; ++ ++ /* Number of contexts that can either be pulled from or are currently ++ * running */ ++ atomic_t nr_contexts_runnable; ++ ++ /** Value for JS_SOFT_JOB_TIMEOUT */ ++ atomic_t soft_job_timeout_ms; ++ ++ /** ++ * Queue Lock, used to access the Policy's queue of contexts ++ * independently of the Run Pool. ++ * ++ * Of course, you don't need the Run Pool lock to access this. ++ */ ++ struct mutex queue_mutex; ++ ++ /** ++ * Run Pool mutex, for managing contexts within the runpool. ++ * Unless otherwise specified, you must hold this lock whilst accessing ++ * any members that follow ++ * ++ * In addition, this is used to access: ++ * * the kbasep_js_kctx_info::runpool substructure ++ */ ++ struct mutex runpool_mutex; ++}; ++ ++/** ++ * @brief KBase Context Job Scheduling information structure ++ * ++ * This is a substructure in the struct kbase_context that encapsulates all the ++ * scheduling information. ++ */ ++struct kbasep_js_kctx_info { ++ ++ /** ++ * Job Scheduler Context information sub-structure. These members are ++ * accessed regardless of whether the context is: ++ * - In the Policy's Run Pool ++ * - In the Policy's Queue ++ * - Not queued nor in the Run Pool. ++ * ++ * You must obtain the jsctx_mutex before accessing any other members of ++ * this substructure. ++ * ++ * You may not access any of these members from IRQ context. ++ */ ++ struct kbase_jsctx { ++ struct mutex jsctx_mutex; /**< Job Scheduler Context lock */ ++ ++ /** Number of jobs ready to run - does \em not include the jobs waiting in ++ * the dispatcher, and dependency-only jobs. See kbase_jd_context::job_nr ++ * for such jobs*/ ++ u32 nr_jobs; ++ ++ /** Context Attributes: ++ * Each is large enough to hold a refcount of the number of atoms on ++ * the context. **/ ++ u32 ctx_attr_ref_count[KBASEP_JS_CTX_ATTR_COUNT]; ++ ++ /** ++ * Wait queue to wait for KCTX_SHEDULED flag state changes. ++ * */ ++ wait_queue_head_t is_scheduled_wait; ++ ++ /** Link implementing JS queues. Context can be present on one ++ * list per job slot ++ */ ++ struct list_head ctx_list_entry[BASE_JM_MAX_NR_SLOTS]; ++ } ctx; ++ ++ /* The initalized-flag is placed at the end, to avoid cache-pollution (we should ++ * only be using this during init/term paths) */ ++ int init_status; ++}; ++ ++/** Subset of atom state that can be available after jd_done_nolock() is called ++ * on that atom. A copy must be taken via kbasep_js_atom_retained_state_copy(), ++ * because the original atom could disappear. */ ++struct kbasep_js_atom_retained_state { ++ /** Event code - to determine whether the atom has finished */ ++ enum base_jd_event_code event_code; ++ /** core requirements */ ++ base_jd_core_req core_req; ++ /* priority */ ++ int sched_priority; ++ /* Core group atom was executed on */ ++ u32 device_nr; ++ ++}; ++ ++/** ++ * Value signifying 'no retry on a slot required' for: ++ * - kbase_js_atom_retained_state::retry_submit_on_slot ++ * - kbase_jd_atom::retry_submit_on_slot ++ */ ++#define KBASEP_JS_RETRY_SUBMIT_SLOT_INVALID (-1) ++ ++/** ++ * base_jd_core_req value signifying 'invalid' for a kbase_jd_atom_retained_state. ++ * ++ * @see kbase_atom_retained_state_is_valid() ++ */ ++#define KBASEP_JS_ATOM_RETAINED_STATE_CORE_REQ_INVALID BASE_JD_REQ_DEP ++ ++/** ++ * @brief The JS timer resolution, in microseconds ++ * ++ * Any non-zero difference in time will be at least this size. ++ */ ++#define KBASEP_JS_TICK_RESOLUTION_US 1 ++ ++ ++ /** @} *//* end group kbase_js */ ++ /** @} *//* end group base_kbase_api */ ++ /** @} *//* end group base_api */ ++ ++#endif /* _KBASE_JS_DEFS_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_base_hwconfig_features.h b/drivers/gpu/arm/bifrost/mali_base_hwconfig_features.h +new file mode 100755 +index 000000000000..0dc08381bee6 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_base_hwconfig_features.h +@@ -0,0 +1,515 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* AUTOMATICALLY GENERATED FILE. If you want to amend the issues/features, ++ * please update base/tools/hwconfig_generator/hwc_{issues,features}.py ++ * For more information see base/tools/hwconfig_generator/README ++ */ ++ ++#ifndef _BASE_HWCONFIG_FEATURES_H_ ++#define _BASE_HWCONFIG_FEATURES_H_ ++ ++enum base_hw_feature { ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, ++ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, ++ BASE_HW_FEATURE_XAFFINITY, ++ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, ++ BASE_HW_FEATURE_MRT, ++ BASE_HW_FEATURE_BRNDOUT_CC, ++ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, ++ BASE_HW_FEATURE_LD_ST_TILEBUFFER, ++ BASE_HW_FEATURE_MSAA_16X, ++ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, ++ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, ++ BASE_HW_FEATURE_T7XX_PAIRING_RULES, ++ BASE_HW_FEATURE_LD_ST_LEA_TEX, ++ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, ++ BASE_HW_FEATURE_TEST4_DATUM_MODE, ++ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, ++ BASE_HW_FEATURE_BRNDOUT_KILL, ++ BASE_HW_FEATURE_WARPING, ++ BASE_HW_FEATURE_FLUSH_REDUCTION, ++ BASE_HW_FEATURE_COHERENCY_REG, ++ BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, ++ BASE_HW_FEATURE_AARCH64_MMU, ++ BASE_HW_FEATURE_TLS_HASHING, ++ BASE_HW_FEATURE_THREAD_GROUP_SPLIT, ++ BASE_HW_FEATURE_CLEAN_ONLY_SAFE, ++ BASE_HW_FEATURE_IDVS_GROUP_SIZE, ++ BASE_HW_FEATURE_L2_CONFIG, ++ BASE_HW_FEATURE_END ++}; ++ ++static const enum base_hw_feature base_hw_features_generic[] = { ++ BASE_HW_FEATURE_END ++}; ++ ++static const enum base_hw_feature base_hw_features_tMIx[] = { ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, ++ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, ++ BASE_HW_FEATURE_XAFFINITY, ++ BASE_HW_FEATURE_WARPING, ++ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, ++ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, ++ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, ++ BASE_HW_FEATURE_BRNDOUT_CC, ++ BASE_HW_FEATURE_BRNDOUT_KILL, ++ BASE_HW_FEATURE_LD_ST_LEA_TEX, ++ BASE_HW_FEATURE_LD_ST_TILEBUFFER, ++ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, ++ BASE_HW_FEATURE_MRT, ++ BASE_HW_FEATURE_MSAA_16X, ++ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, ++ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, ++ BASE_HW_FEATURE_T7XX_PAIRING_RULES, ++ BASE_HW_FEATURE_TEST4_DATUM_MODE, ++ BASE_HW_FEATURE_THREAD_GROUP_SPLIT, ++ BASE_HW_FEATURE_FLUSH_REDUCTION, ++ BASE_HW_FEATURE_COHERENCY_REG, ++ BASE_HW_FEATURE_AARCH64_MMU, ++ BASE_HW_FEATURE_END ++}; ++ ++static const enum base_hw_feature base_hw_features_tHEx[] = { ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, ++ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, ++ BASE_HW_FEATURE_XAFFINITY, ++ BASE_HW_FEATURE_WARPING, ++ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, ++ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, ++ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, ++ BASE_HW_FEATURE_BRNDOUT_CC, ++ BASE_HW_FEATURE_BRNDOUT_KILL, ++ BASE_HW_FEATURE_LD_ST_LEA_TEX, ++ BASE_HW_FEATURE_LD_ST_TILEBUFFER, ++ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, ++ BASE_HW_FEATURE_MRT, ++ BASE_HW_FEATURE_MSAA_16X, ++ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, ++ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, ++ BASE_HW_FEATURE_T7XX_PAIRING_RULES, ++ BASE_HW_FEATURE_TEST4_DATUM_MODE, ++ BASE_HW_FEATURE_THREAD_GROUP_SPLIT, ++ BASE_HW_FEATURE_FLUSH_REDUCTION, ++ BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, ++ BASE_HW_FEATURE_COHERENCY_REG, ++ BASE_HW_FEATURE_AARCH64_MMU, ++ BASE_HW_FEATURE_END ++}; ++ ++static const enum base_hw_feature base_hw_features_tSIx[] = { ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, ++ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, ++ BASE_HW_FEATURE_XAFFINITY, ++ BASE_HW_FEATURE_WARPING, ++ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, ++ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, ++ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, ++ BASE_HW_FEATURE_BRNDOUT_CC, ++ BASE_HW_FEATURE_BRNDOUT_KILL, ++ BASE_HW_FEATURE_LD_ST_LEA_TEX, ++ BASE_HW_FEATURE_LD_ST_TILEBUFFER, ++ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, ++ BASE_HW_FEATURE_MRT, ++ BASE_HW_FEATURE_MSAA_16X, ++ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, ++ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, ++ BASE_HW_FEATURE_T7XX_PAIRING_RULES, ++ BASE_HW_FEATURE_TEST4_DATUM_MODE, ++ BASE_HW_FEATURE_THREAD_GROUP_SPLIT, ++ BASE_HW_FEATURE_FLUSH_REDUCTION, ++ BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, ++ BASE_HW_FEATURE_COHERENCY_REG, ++ BASE_HW_FEATURE_AARCH64_MMU, ++ BASE_HW_FEATURE_END ++}; ++ ++static const enum base_hw_feature base_hw_features_tDVx[] = { ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, ++ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, ++ BASE_HW_FEATURE_XAFFINITY, ++ BASE_HW_FEATURE_WARPING, ++ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, ++ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, ++ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, ++ BASE_HW_FEATURE_BRNDOUT_CC, ++ BASE_HW_FEATURE_BRNDOUT_KILL, ++ BASE_HW_FEATURE_LD_ST_LEA_TEX, ++ BASE_HW_FEATURE_LD_ST_TILEBUFFER, ++ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, ++ BASE_HW_FEATURE_MRT, ++ BASE_HW_FEATURE_MSAA_16X, ++ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, ++ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, ++ BASE_HW_FEATURE_T7XX_PAIRING_RULES, ++ BASE_HW_FEATURE_TEST4_DATUM_MODE, ++ BASE_HW_FEATURE_THREAD_GROUP_SPLIT, ++ BASE_HW_FEATURE_FLUSH_REDUCTION, ++ BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, ++ BASE_HW_FEATURE_COHERENCY_REG, ++ BASE_HW_FEATURE_AARCH64_MMU, ++ BASE_HW_FEATURE_END ++}; ++ ++static const enum base_hw_feature base_hw_features_tNOx[] = { ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, ++ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, ++ BASE_HW_FEATURE_XAFFINITY, ++ BASE_HW_FEATURE_WARPING, ++ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, ++ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, ++ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, ++ BASE_HW_FEATURE_BRNDOUT_CC, ++ BASE_HW_FEATURE_BRNDOUT_KILL, ++ BASE_HW_FEATURE_LD_ST_LEA_TEX, ++ BASE_HW_FEATURE_LD_ST_TILEBUFFER, ++ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, ++ BASE_HW_FEATURE_MRT, ++ BASE_HW_FEATURE_MSAA_16X, ++ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, ++ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, ++ BASE_HW_FEATURE_T7XX_PAIRING_RULES, ++ BASE_HW_FEATURE_TEST4_DATUM_MODE, ++ BASE_HW_FEATURE_THREAD_GROUP_SPLIT, ++ BASE_HW_FEATURE_FLUSH_REDUCTION, ++ BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, ++ BASE_HW_FEATURE_COHERENCY_REG, ++ BASE_HW_FEATURE_AARCH64_MMU, ++ BASE_HW_FEATURE_TLS_HASHING, ++ BASE_HW_FEATURE_IDVS_GROUP_SIZE, ++ BASE_HW_FEATURE_END ++}; ++ ++static const enum base_hw_feature base_hw_features_tGOx[] = { ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, ++ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, ++ BASE_HW_FEATURE_XAFFINITY, ++ BASE_HW_FEATURE_WARPING, ++ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, ++ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, ++ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, ++ BASE_HW_FEATURE_BRNDOUT_CC, ++ BASE_HW_FEATURE_BRNDOUT_KILL, ++ BASE_HW_FEATURE_LD_ST_LEA_TEX, ++ BASE_HW_FEATURE_LD_ST_TILEBUFFER, ++ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, ++ BASE_HW_FEATURE_MRT, ++ BASE_HW_FEATURE_MSAA_16X, ++ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, ++ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, ++ BASE_HW_FEATURE_T7XX_PAIRING_RULES, ++ BASE_HW_FEATURE_TEST4_DATUM_MODE, ++ BASE_HW_FEATURE_THREAD_GROUP_SPLIT, ++ BASE_HW_FEATURE_FLUSH_REDUCTION, ++ BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, ++ BASE_HW_FEATURE_COHERENCY_REG, ++ BASE_HW_FEATURE_AARCH64_MMU, ++ BASE_HW_FEATURE_TLS_HASHING, ++ BASE_HW_FEATURE_IDVS_GROUP_SIZE, ++ BASE_HW_FEATURE_END ++}; ++ ++static const enum base_hw_feature base_hw_features_tTRx[] = { ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, ++ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, ++ BASE_HW_FEATURE_XAFFINITY, ++ BASE_HW_FEATURE_WARPING, ++ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, ++ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, ++ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, ++ BASE_HW_FEATURE_BRNDOUT_CC, ++ BASE_HW_FEATURE_BRNDOUT_KILL, ++ BASE_HW_FEATURE_LD_ST_LEA_TEX, ++ BASE_HW_FEATURE_LD_ST_TILEBUFFER, ++ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, ++ BASE_HW_FEATURE_MRT, ++ BASE_HW_FEATURE_MSAA_16X, ++ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, ++ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, ++ BASE_HW_FEATURE_T7XX_PAIRING_RULES, ++ BASE_HW_FEATURE_TEST4_DATUM_MODE, ++ BASE_HW_FEATURE_FLUSH_REDUCTION, ++ BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, ++ BASE_HW_FEATURE_COHERENCY_REG, ++ BASE_HW_FEATURE_AARCH64_MMU, ++ BASE_HW_FEATURE_IDVS_GROUP_SIZE, ++ BASE_HW_FEATURE_CLEAN_ONLY_SAFE, ++ BASE_HW_FEATURE_END ++}; ++ ++static const enum base_hw_feature base_hw_features_tNAx[] = { ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, ++ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, ++ BASE_HW_FEATURE_XAFFINITY, ++ BASE_HW_FEATURE_WARPING, ++ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, ++ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, ++ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, ++ BASE_HW_FEATURE_BRNDOUT_CC, ++ BASE_HW_FEATURE_BRNDOUT_KILL, ++ BASE_HW_FEATURE_LD_ST_LEA_TEX, ++ BASE_HW_FEATURE_LD_ST_TILEBUFFER, ++ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, ++ BASE_HW_FEATURE_MRT, ++ BASE_HW_FEATURE_MSAA_16X, ++ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, ++ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, ++ BASE_HW_FEATURE_T7XX_PAIRING_RULES, ++ BASE_HW_FEATURE_TEST4_DATUM_MODE, ++ BASE_HW_FEATURE_FLUSH_REDUCTION, ++ BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, ++ BASE_HW_FEATURE_COHERENCY_REG, ++ BASE_HW_FEATURE_AARCH64_MMU, ++ BASE_HW_FEATURE_IDVS_GROUP_SIZE, ++ BASE_HW_FEATURE_CLEAN_ONLY_SAFE, ++ BASE_HW_FEATURE_END ++}; ++ ++static const enum base_hw_feature base_hw_features_tBEx[] = { ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, ++ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, ++ BASE_HW_FEATURE_XAFFINITY, ++ BASE_HW_FEATURE_WARPING, ++ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, ++ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, ++ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, ++ BASE_HW_FEATURE_BRNDOUT_CC, ++ BASE_HW_FEATURE_BRNDOUT_KILL, ++ BASE_HW_FEATURE_LD_ST_LEA_TEX, ++ BASE_HW_FEATURE_LD_ST_TILEBUFFER, ++ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, ++ BASE_HW_FEATURE_MRT, ++ BASE_HW_FEATURE_MSAA_16X, ++ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, ++ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, ++ BASE_HW_FEATURE_T7XX_PAIRING_RULES, ++ BASE_HW_FEATURE_TEST4_DATUM_MODE, ++ BASE_HW_FEATURE_FLUSH_REDUCTION, ++ BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, ++ BASE_HW_FEATURE_COHERENCY_REG, ++ BASE_HW_FEATURE_AARCH64_MMU, ++ BASE_HW_FEATURE_IDVS_GROUP_SIZE, ++ BASE_HW_FEATURE_L2_CONFIG, ++ BASE_HW_FEATURE_CLEAN_ONLY_SAFE, ++ BASE_HW_FEATURE_END ++}; ++ ++static const enum base_hw_feature base_hw_features_tBAx[] = { ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, ++ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, ++ BASE_HW_FEATURE_XAFFINITY, ++ BASE_HW_FEATURE_WARPING, ++ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, ++ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, ++ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, ++ BASE_HW_FEATURE_BRNDOUT_CC, ++ BASE_HW_FEATURE_BRNDOUT_KILL, ++ BASE_HW_FEATURE_LD_ST_LEA_TEX, ++ BASE_HW_FEATURE_LD_ST_TILEBUFFER, ++ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, ++ BASE_HW_FEATURE_MRT, ++ BASE_HW_FEATURE_MSAA_16X, ++ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, ++ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, ++ BASE_HW_FEATURE_T7XX_PAIRING_RULES, ++ BASE_HW_FEATURE_TEST4_DATUM_MODE, ++ BASE_HW_FEATURE_FLUSH_REDUCTION, ++ BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, ++ BASE_HW_FEATURE_COHERENCY_REG, ++ BASE_HW_FEATURE_AARCH64_MMU, ++ BASE_HW_FEATURE_IDVS_GROUP_SIZE, ++ BASE_HW_FEATURE_L2_CONFIG, ++ BASE_HW_FEATURE_CLEAN_ONLY_SAFE, ++ BASE_HW_FEATURE_END ++}; ++ ++static const enum base_hw_feature base_hw_features_tDUx[] = { ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, ++ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, ++ BASE_HW_FEATURE_XAFFINITY, ++ BASE_HW_FEATURE_WARPING, ++ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, ++ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, ++ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, ++ BASE_HW_FEATURE_BRNDOUT_CC, ++ BASE_HW_FEATURE_BRNDOUT_KILL, ++ BASE_HW_FEATURE_LD_ST_LEA_TEX, ++ BASE_HW_FEATURE_LD_ST_TILEBUFFER, ++ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, ++ BASE_HW_FEATURE_MRT, ++ BASE_HW_FEATURE_MSAA_16X, ++ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, ++ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, ++ BASE_HW_FEATURE_T7XX_PAIRING_RULES, ++ BASE_HW_FEATURE_TEST4_DATUM_MODE, ++ BASE_HW_FEATURE_FLUSH_REDUCTION, ++ BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, ++ BASE_HW_FEATURE_COHERENCY_REG, ++ BASE_HW_FEATURE_AARCH64_MMU, ++ BASE_HW_FEATURE_IDVS_GROUP_SIZE, ++ BASE_HW_FEATURE_L2_CONFIG, ++ BASE_HW_FEATURE_CLEAN_ONLY_SAFE, ++ BASE_HW_FEATURE_END ++}; ++ ++static const enum base_hw_feature base_hw_features_tODx[] = { ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, ++ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, ++ BASE_HW_FEATURE_XAFFINITY, ++ BASE_HW_FEATURE_WARPING, ++ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, ++ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, ++ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, ++ BASE_HW_FEATURE_BRNDOUT_CC, ++ BASE_HW_FEATURE_BRNDOUT_KILL, ++ BASE_HW_FEATURE_LD_ST_LEA_TEX, ++ BASE_HW_FEATURE_LD_ST_TILEBUFFER, ++ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, ++ BASE_HW_FEATURE_MRT, ++ BASE_HW_FEATURE_MSAA_16X, ++ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, ++ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, ++ BASE_HW_FEATURE_T7XX_PAIRING_RULES, ++ BASE_HW_FEATURE_TEST4_DATUM_MODE, ++ BASE_HW_FEATURE_FLUSH_REDUCTION, ++ BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, ++ BASE_HW_FEATURE_COHERENCY_REG, ++ BASE_HW_FEATURE_AARCH64_MMU, ++ BASE_HW_FEATURE_L2_CONFIG, ++ BASE_HW_FEATURE_CLEAN_ONLY_SAFE, ++ BASE_HW_FEATURE_END ++}; ++ ++static const enum base_hw_feature base_hw_features_tGRx[] = { ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, ++ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, ++ BASE_HW_FEATURE_XAFFINITY, ++ BASE_HW_FEATURE_WARPING, ++ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, ++ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, ++ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, ++ BASE_HW_FEATURE_BRNDOUT_CC, ++ BASE_HW_FEATURE_BRNDOUT_KILL, ++ BASE_HW_FEATURE_LD_ST_LEA_TEX, ++ BASE_HW_FEATURE_LD_ST_TILEBUFFER, ++ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, ++ BASE_HW_FEATURE_MRT, ++ BASE_HW_FEATURE_MSAA_16X, ++ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, ++ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, ++ BASE_HW_FEATURE_T7XX_PAIRING_RULES, ++ BASE_HW_FEATURE_TEST4_DATUM_MODE, ++ BASE_HW_FEATURE_FLUSH_REDUCTION, ++ BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, ++ BASE_HW_FEATURE_COHERENCY_REG, ++ BASE_HW_FEATURE_AARCH64_MMU, ++ BASE_HW_FEATURE_L2_CONFIG, ++ BASE_HW_FEATURE_CLEAN_ONLY_SAFE, ++ BASE_HW_FEATURE_END ++}; ++ ++static const enum base_hw_feature base_hw_features_tVAx[] = { ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, ++ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, ++ BASE_HW_FEATURE_XAFFINITY, ++ BASE_HW_FEATURE_WARPING, ++ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, ++ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, ++ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, ++ BASE_HW_FEATURE_BRNDOUT_CC, ++ BASE_HW_FEATURE_BRNDOUT_KILL, ++ BASE_HW_FEATURE_LD_ST_LEA_TEX, ++ BASE_HW_FEATURE_LD_ST_TILEBUFFER, ++ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, ++ BASE_HW_FEATURE_MRT, ++ BASE_HW_FEATURE_MSAA_16X, ++ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, ++ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, ++ BASE_HW_FEATURE_T7XX_PAIRING_RULES, ++ BASE_HW_FEATURE_TEST4_DATUM_MODE, ++ BASE_HW_FEATURE_FLUSH_REDUCTION, ++ BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, ++ BASE_HW_FEATURE_COHERENCY_REG, ++ BASE_HW_FEATURE_AARCH64_MMU, ++ BASE_HW_FEATURE_L2_CONFIG, ++ BASE_HW_FEATURE_CLEAN_ONLY_SAFE, ++ BASE_HW_FEATURE_END ++}; ++ ++static const enum base_hw_feature base_hw_features_tTUx[] = { ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, ++ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, ++ BASE_HW_FEATURE_XAFFINITY, ++ BASE_HW_FEATURE_WARPING, ++ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, ++ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, ++ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, ++ BASE_HW_FEATURE_BRNDOUT_CC, ++ BASE_HW_FEATURE_BRNDOUT_KILL, ++ BASE_HW_FEATURE_LD_ST_LEA_TEX, ++ BASE_HW_FEATURE_LD_ST_TILEBUFFER, ++ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, ++ BASE_HW_FEATURE_MRT, ++ BASE_HW_FEATURE_MSAA_16X, ++ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, ++ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, ++ BASE_HW_FEATURE_T7XX_PAIRING_RULES, ++ BASE_HW_FEATURE_TEST4_DATUM_MODE, ++ BASE_HW_FEATURE_FLUSH_REDUCTION, ++ BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, ++ BASE_HW_FEATURE_COHERENCY_REG, ++ BASE_HW_FEATURE_AARCH64_MMU, ++ BASE_HW_FEATURE_L2_CONFIG, ++ BASE_HW_FEATURE_CLEAN_ONLY_SAFE, ++ BASE_HW_FEATURE_END ++}; ++ ++static const enum base_hw_feature base_hw_features_tE2x[] = { ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, ++ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, ++ BASE_HW_FEATURE_XAFFINITY, ++ BASE_HW_FEATURE_WARPING, ++ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, ++ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, ++ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, ++ BASE_HW_FEATURE_BRNDOUT_CC, ++ BASE_HW_FEATURE_BRNDOUT_KILL, ++ BASE_HW_FEATURE_LD_ST_LEA_TEX, ++ BASE_HW_FEATURE_LD_ST_TILEBUFFER, ++ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, ++ BASE_HW_FEATURE_MRT, ++ BASE_HW_FEATURE_MSAA_16X, ++ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, ++ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, ++ BASE_HW_FEATURE_T7XX_PAIRING_RULES, ++ BASE_HW_FEATURE_TEST4_DATUM_MODE, ++ BASE_HW_FEATURE_FLUSH_REDUCTION, ++ BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, ++ BASE_HW_FEATURE_COHERENCY_REG, ++ BASE_HW_FEATURE_AARCH64_MMU, ++ BASE_HW_FEATURE_IDVS_GROUP_SIZE, ++ BASE_HW_FEATURE_L2_CONFIG, ++ BASE_HW_FEATURE_CLEAN_ONLY_SAFE, ++ BASE_HW_FEATURE_END ++}; ++ ++#endif /* _BASE_HWCONFIG_FEATURES_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_base_hwconfig_issues.h b/drivers/gpu/arm/bifrost/mali_base_hwconfig_issues.h +new file mode 100755 +index 000000000000..c1ad3ac40705 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_base_hwconfig_issues.h +@@ -0,0 +1,684 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* AUTOMATICALLY GENERATED FILE. If you want to amend the issues/features, ++ * please update base/tools/hwconfig_generator/hwc_{issues,features}.py ++ * For more information see base/tools/hwconfig_generator/README ++ */ ++ ++#ifndef _BASE_HWCONFIG_ISSUES_H_ ++#define _BASE_HWCONFIG_ISSUES_H_ ++ ++enum base_hw_issue { ++ BASE_HW_ISSUE_5736, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10682, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T76X_3953, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_7940, ++ BASE_HW_ISSUE_TMIX_8042, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TMIX_8138, ++ BASE_HW_ISSUE_TMIX_8206, ++ BASE_HW_ISSUE_TMIX_8343, ++ BASE_HW_ISSUE_TMIX_8463, ++ BASE_HW_ISSUE_TMIX_8456, ++ BASE_HW_ISSUE_TSIX_1116, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_TNOX_1194, ++ BASE_HW_ISSUE_TGOX_R1_1234, ++ BASE_HW_ISSUE_TTRX_1337, ++ BASE_HW_ISSUE_TSIX_1792, ++ BASE_HW_ISSUE_TTRX_2968_TTRX_3162, ++ BASE_HW_ISSUE_TTRX_3076, ++ BASE_HW_ISSUE_TTRX_921, ++ BASE_HW_ISSUE_TTRX_3414, ++ BASE_HW_ISSUE_GPU2017_1336, ++ BASE_HW_ISSUE_TTRX_3083, ++ BASE_HW_ISSUE_TTRX_3470, ++ BASE_HW_ISSUE_TTRX_3464, ++ BASE_HW_ISSUE_TTRX_3485, ++ BASE_HW_ISSUE_GPU2019_3212, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_generic[] = { ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tMIx_r0p0_05dev0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10682, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T76X_3953, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8042, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TMIX_8138, ++ BASE_HW_ISSUE_TMIX_8206, ++ BASE_HW_ISSUE_TMIX_8343, ++ BASE_HW_ISSUE_TMIX_8463, ++ BASE_HW_ISSUE_TMIX_8456, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TTRX_921, ++ BASE_HW_ISSUE_GPU2017_1336, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tMIx_r0p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10682, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_7940, ++ BASE_HW_ISSUE_TMIX_8042, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TMIX_8138, ++ BASE_HW_ISSUE_TMIX_8206, ++ BASE_HW_ISSUE_TMIX_8343, ++ BASE_HW_ISSUE_TMIX_8463, ++ BASE_HW_ISSUE_TMIX_8456, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TTRX_921, ++ BASE_HW_ISSUE_GPU2017_1336, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tMIx_r0p1[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10682, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_7940, ++ BASE_HW_ISSUE_TMIX_8042, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TMIX_8138, ++ BASE_HW_ISSUE_TMIX_8206, ++ BASE_HW_ISSUE_TMIX_8343, ++ BASE_HW_ISSUE_TMIX_8463, ++ BASE_HW_ISSUE_TMIX_8456, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TTRX_921, ++ BASE_HW_ISSUE_GPU2017_1336, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_model_tMIx[] = { ++ BASE_HW_ISSUE_5736, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_7940, ++ BASE_HW_ISSUE_TMIX_8042, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TMIX_8138, ++ BASE_HW_ISSUE_TMIX_8206, ++ BASE_HW_ISSUE_TMIX_8343, ++ BASE_HW_ISSUE_TMIX_8456, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tHEx_r0p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10682, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8042, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TTRX_921, ++ BASE_HW_ISSUE_GPU2017_1336, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tHEx_r0p1[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10682, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8042, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TTRX_921, ++ BASE_HW_ISSUE_GPU2017_1336, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tHEx_r0p2[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10682, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8042, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TTRX_921, ++ BASE_HW_ISSUE_GPU2017_1336, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tHEx_r0p3[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10682, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8042, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TTRX_921, ++ BASE_HW_ISSUE_GPU2017_1336, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_model_tHEx[] = { ++ BASE_HW_ISSUE_5736, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8042, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tSIx_r0p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TSIX_1116, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TSIX_1792, ++ BASE_HW_ISSUE_TTRX_921, ++ BASE_HW_ISSUE_GPU2017_1336, ++ BASE_HW_ISSUE_TTRX_3464, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tSIx_r0p1[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TSIX_1116, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TSIX_1792, ++ BASE_HW_ISSUE_TTRX_921, ++ BASE_HW_ISSUE_GPU2017_1336, ++ BASE_HW_ISSUE_TTRX_3464, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tSIx_r1p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TSIX_1116, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TTRX_921, ++ BASE_HW_ISSUE_GPU2017_1336, ++ BASE_HW_ISSUE_TTRX_3464, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tSIx_r1p1[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TSIX_1116, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TTRX_921, ++ BASE_HW_ISSUE_GPU2017_1336, ++ BASE_HW_ISSUE_TTRX_3464, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_model_tSIx[] = { ++ BASE_HW_ISSUE_5736, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TSIX_1116, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TTRX_3464, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tDVx_r0p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TSIX_1116, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TTRX_921, ++ BASE_HW_ISSUE_GPU2017_1336, ++ BASE_HW_ISSUE_TTRX_3464, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_model_tDVx[] = { ++ BASE_HW_ISSUE_5736, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TSIX_1116, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TTRX_3464, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tNOx_r0p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TSIX_1116, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TNOX_1194, ++ BASE_HW_ISSUE_TTRX_921, ++ BASE_HW_ISSUE_GPU2017_1336, ++ BASE_HW_ISSUE_TTRX_3464, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_model_tNOx[] = { ++ BASE_HW_ISSUE_5736, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TSIX_1116, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TTRX_3464, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tGOx_r0p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TSIX_1116, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TNOX_1194, ++ BASE_HW_ISSUE_TTRX_921, ++ BASE_HW_ISSUE_GPU2017_1336, ++ BASE_HW_ISSUE_TTRX_3464, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tGOx_r1p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TSIX_1116, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TGOX_R1_1234, ++ BASE_HW_ISSUE_TTRX_921, ++ BASE_HW_ISSUE_GPU2017_1336, ++ BASE_HW_ISSUE_TTRX_3464, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_model_tGOx[] = { ++ BASE_HW_ISSUE_5736, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TSIX_1116, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TTRX_3464, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tTRx_r0p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TTRX_1337, ++ BASE_HW_ISSUE_TTRX_2968_TTRX_3162, ++ BASE_HW_ISSUE_TTRX_3076, ++ BASE_HW_ISSUE_TTRX_921, ++ BASE_HW_ISSUE_TTRX_3414, ++ BASE_HW_ISSUE_GPU2017_1336, ++ BASE_HW_ISSUE_TTRX_3083, ++ BASE_HW_ISSUE_TTRX_3470, ++ BASE_HW_ISSUE_TTRX_3464, ++ BASE_HW_ISSUE_TTRX_3485, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tTRx_r0p1[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TTRX_1337, ++ BASE_HW_ISSUE_TTRX_2968_TTRX_3162, ++ BASE_HW_ISSUE_TTRX_3076, ++ BASE_HW_ISSUE_TTRX_921, ++ BASE_HW_ISSUE_TTRX_3414, ++ BASE_HW_ISSUE_GPU2017_1336, ++ BASE_HW_ISSUE_TTRX_3083, ++ BASE_HW_ISSUE_TTRX_3470, ++ BASE_HW_ISSUE_TTRX_3464, ++ BASE_HW_ISSUE_TTRX_3485, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tTRx_r0p2[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TTRX_1337, ++ BASE_HW_ISSUE_TTRX_2968_TTRX_3162, ++ BASE_HW_ISSUE_TTRX_3076, ++ BASE_HW_ISSUE_TTRX_921, ++ BASE_HW_ISSUE_TTRX_3414, ++ BASE_HW_ISSUE_GPU2017_1336, ++ BASE_HW_ISSUE_TTRX_3083, ++ BASE_HW_ISSUE_TTRX_3470, ++ BASE_HW_ISSUE_TTRX_3464, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_model_tTRx[] = { ++ BASE_HW_ISSUE_5736, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TTRX_1337, ++ BASE_HW_ISSUE_TTRX_3414, ++ BASE_HW_ISSUE_TTRX_3083, ++ BASE_HW_ISSUE_TTRX_3470, ++ BASE_HW_ISSUE_TTRX_3464, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tNAx_r0p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TTRX_1337, ++ BASE_HW_ISSUE_TTRX_2968_TTRX_3162, ++ BASE_HW_ISSUE_TTRX_3076, ++ BASE_HW_ISSUE_TTRX_921, ++ BASE_HW_ISSUE_TTRX_3414, ++ BASE_HW_ISSUE_GPU2017_1336, ++ BASE_HW_ISSUE_TTRX_3083, ++ BASE_HW_ISSUE_TTRX_3470, ++ BASE_HW_ISSUE_TTRX_3464, ++ BASE_HW_ISSUE_TTRX_3485, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tNAx_r0p1[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TTRX_1337, ++ BASE_HW_ISSUE_TTRX_2968_TTRX_3162, ++ BASE_HW_ISSUE_TTRX_3076, ++ BASE_HW_ISSUE_TTRX_921, ++ BASE_HW_ISSUE_TTRX_3414, ++ BASE_HW_ISSUE_GPU2017_1336, ++ BASE_HW_ISSUE_TTRX_3083, ++ BASE_HW_ISSUE_TTRX_3470, ++ BASE_HW_ISSUE_TTRX_3464, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_model_tNAx[] = { ++ BASE_HW_ISSUE_5736, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TTRX_1337, ++ BASE_HW_ISSUE_TTRX_3414, ++ BASE_HW_ISSUE_TTRX_3083, ++ BASE_HW_ISSUE_TTRX_3470, ++ BASE_HW_ISSUE_TTRX_3464, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tBEx_r0p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TTRX_1337, ++ BASE_HW_ISSUE_TTRX_2968_TTRX_3162, ++ BASE_HW_ISSUE_TTRX_921, ++ BASE_HW_ISSUE_TTRX_3414, ++ BASE_HW_ISSUE_TTRX_3083, ++ BASE_HW_ISSUE_TTRX_3470, ++ BASE_HW_ISSUE_TTRX_3464, ++ BASE_HW_ISSUE_TTRX_3485, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tBEx_r0p1[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TTRX_1337, ++ BASE_HW_ISSUE_TTRX_2968_TTRX_3162, ++ BASE_HW_ISSUE_TTRX_921, ++ BASE_HW_ISSUE_TTRX_3414, ++ BASE_HW_ISSUE_TTRX_3083, ++ BASE_HW_ISSUE_TTRX_3470, ++ BASE_HW_ISSUE_TTRX_3464, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tBEx_r1p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TTRX_1337, ++ BASE_HW_ISSUE_TTRX_2968_TTRX_3162, ++ BASE_HW_ISSUE_TTRX_921, ++ BASE_HW_ISSUE_TTRX_3414, ++ BASE_HW_ISSUE_TTRX_3083, ++ BASE_HW_ISSUE_TTRX_3470, ++ BASE_HW_ISSUE_TTRX_3464, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tBEx_r1p1[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TTRX_1337, ++ BASE_HW_ISSUE_TTRX_2968_TTRX_3162, ++ BASE_HW_ISSUE_TTRX_921, ++ BASE_HW_ISSUE_TTRX_3414, ++ BASE_HW_ISSUE_TTRX_3083, ++ BASE_HW_ISSUE_TTRX_3470, ++ BASE_HW_ISSUE_TTRX_3464, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_model_tBEx[] = { ++ BASE_HW_ISSUE_5736, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TTRX_1337, ++ BASE_HW_ISSUE_TTRX_3414, ++ BASE_HW_ISSUE_TTRX_3083, ++ BASE_HW_ISSUE_TTRX_3470, ++ BASE_HW_ISSUE_TTRX_3464, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_lBEx_r1p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TTRX_1337, ++ BASE_HW_ISSUE_TTRX_2968_TTRX_3162, ++ BASE_HW_ISSUE_TTRX_921, ++ BASE_HW_ISSUE_TTRX_3414, ++ BASE_HW_ISSUE_TTRX_3083, ++ BASE_HW_ISSUE_TTRX_3470, ++ BASE_HW_ISSUE_TTRX_3464, ++ BASE_HW_ISSUE_TTRX_3485, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_lBEx_r1p1[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TTRX_1337, ++ BASE_HW_ISSUE_TTRX_2968_TTRX_3162, ++ BASE_HW_ISSUE_TTRX_921, ++ BASE_HW_ISSUE_TTRX_3414, ++ BASE_HW_ISSUE_TTRX_3083, ++ BASE_HW_ISSUE_TTRX_3470, ++ BASE_HW_ISSUE_TTRX_3464, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tBAx_r0p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TTRX_1337, ++ BASE_HW_ISSUE_TTRX_2968_TTRX_3162, ++ BASE_HW_ISSUE_TTRX_921, ++ BASE_HW_ISSUE_TTRX_3414, ++ BASE_HW_ISSUE_TTRX_3083, ++ BASE_HW_ISSUE_TTRX_3470, ++ BASE_HW_ISSUE_TTRX_3464, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tBAx_r1p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TTRX_1337, ++ BASE_HW_ISSUE_TTRX_2968_TTRX_3162, ++ BASE_HW_ISSUE_TTRX_921, ++ BASE_HW_ISSUE_TTRX_3414, ++ BASE_HW_ISSUE_TTRX_3083, ++ BASE_HW_ISSUE_TTRX_3470, ++ BASE_HW_ISSUE_TTRX_3464, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_model_tBAx[] = { ++ BASE_HW_ISSUE_5736, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TTRX_1337, ++ BASE_HW_ISSUE_TTRX_3414, ++ BASE_HW_ISSUE_TTRX_3083, ++ BASE_HW_ISSUE_TTRX_3470, ++ BASE_HW_ISSUE_TTRX_3464, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tDUx_r0p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TTRX_1337, ++ BASE_HW_ISSUE_TTRX_921, ++ BASE_HW_ISSUE_TTRX_3414, ++ BASE_HW_ISSUE_TTRX_3083, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_model_tDUx[] = { ++ BASE_HW_ISSUE_5736, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TTRX_1337, ++ BASE_HW_ISSUE_TTRX_3414, ++ BASE_HW_ISSUE_TTRX_3083, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tODx_r0p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TTRX_1337, ++ BASE_HW_ISSUE_GPU2019_3212, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_model_tODx[] = { ++ BASE_HW_ISSUE_5736, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TTRX_1337, ++ BASE_HW_ISSUE_GPU2019_3212, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tGRx_r0p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TTRX_1337, ++ BASE_HW_ISSUE_GPU2019_3212, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_model_tGRx[] = { ++ BASE_HW_ISSUE_5736, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TTRX_1337, ++ BASE_HW_ISSUE_GPU2019_3212, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tVAx_r0p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TTRX_1337, ++ BASE_HW_ISSUE_GPU2019_3212, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_model_tVAx[] = { ++ BASE_HW_ISSUE_5736, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TTRX_1337, ++ BASE_HW_ISSUE_GPU2019_3212, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tTUx_r0p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TTRX_1337, ++ BASE_HW_ISSUE_GPU2019_3212, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_model_tTUx[] = { ++ BASE_HW_ISSUE_5736, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TTRX_1337, ++ BASE_HW_ISSUE_GPU2019_3212, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tE2x_r0p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TTRX_1337, ++ BASE_HW_ISSUE_TTRX_921, ++ BASE_HW_ISSUE_TTRX_3414, ++ BASE_HW_ISSUE_TTRX_3083, ++ BASE_HW_ISSUE_GPU2019_3212, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_model_tE2x[] = { ++ BASE_HW_ISSUE_5736, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TSIX_2033, ++ BASE_HW_ISSUE_TTRX_1337, ++ BASE_HW_ISSUE_TTRX_3414, ++ BASE_HW_ISSUE_TTRX_3083, ++ BASE_HW_ISSUE_GPU2019_3212, ++ BASE_HW_ISSUE_END ++}; ++ ++#endif /* _BASE_HWCONFIG_ISSUES_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_base_kernel.h b/drivers/gpu/arm/bifrost/mali_base_kernel.h +new file mode 100755 +index 000000000000..086171adb6e5 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_base_kernel.h +@@ -0,0 +1,807 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++/* ++ * Base structures shared with the kernel. ++ */ ++ ++#ifndef _BASE_KERNEL_H_ ++#define _BASE_KERNEL_H_ ++ ++struct base_mem_handle { ++ struct { ++ u64 handle; ++ } basep; ++}; ++ ++#include "mali_base_mem_priv.h" ++#include "gpu/mali_kbase_gpu_coherency.h" ++#include "gpu/mali_kbase_gpu_id.h" ++ ++#define BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS 4 ++ ++#define BASE_MAX_COHERENT_GROUPS 16 ++ ++#if defined CDBG_ASSERT ++#define LOCAL_ASSERT CDBG_ASSERT ++#elif defined KBASE_DEBUG_ASSERT ++#define LOCAL_ASSERT KBASE_DEBUG_ASSERT ++#else ++#error assert macro not defined! ++#endif ++ ++#if defined(PAGE_MASK) && defined(PAGE_SHIFT) ++#define LOCAL_PAGE_SHIFT PAGE_SHIFT ++#define LOCAL_PAGE_LSB ~PAGE_MASK ++#else ++#include ++ ++#if defined OSU_CONFIG_CPU_PAGE_SIZE_LOG2 ++#define LOCAL_PAGE_SHIFT OSU_CONFIG_CPU_PAGE_SIZE_LOG2 ++#define LOCAL_PAGE_LSB ((1ul << OSU_CONFIG_CPU_PAGE_SIZE_LOG2) - 1) ++#else ++#error Failed to find page size ++#endif ++#endif ++ ++/* Physical memory group ID for normal usage. ++ */ ++#define BASE_MEM_GROUP_DEFAULT (0) ++ ++/* Number of physical memory groups. ++ */ ++#define BASE_MEM_GROUP_COUNT (16) ++ ++/** ++ * typedef base_mem_alloc_flags - Memory allocation, access/hint flags. ++ * ++ * A combination of MEM_PROT/MEM_HINT flags must be passed to each allocator ++ * in order to determine the best cache policy. Some combinations are ++ * of course invalid (e.g. MEM_PROT_CPU_WR | MEM_HINT_CPU_RD), ++ * which defines a write-only region on the CPU side, which is ++ * heavily read by the CPU... ++ * Other flags are only meaningful to a particular allocator. ++ * More flags can be added to this list, as long as they don't clash ++ * (see BASE_MEM_FLAGS_NR_BITS for the number of the first free bit). ++ */ ++typedef u32 base_mem_alloc_flags; ++ ++/* A mask for all the flags which are modifiable via the base_mem_set_flags ++ * interface. ++ */ ++#define BASE_MEM_FLAGS_MODIFIABLE \ ++ (BASE_MEM_DONT_NEED | BASE_MEM_COHERENT_SYSTEM | \ ++ BASE_MEM_COHERENT_LOCAL) ++ ++/* A mask of all the flags that can be returned via the base_mem_get_flags() ++ * interface. ++ */ ++#define BASE_MEM_FLAGS_QUERYABLE \ ++ (BASE_MEM_FLAGS_INPUT_MASK & ~(BASE_MEM_SAME_VA | \ ++ BASE_MEM_COHERENT_SYSTEM_REQUIRED | BASE_MEM_DONT_NEED | \ ++ BASE_MEM_IMPORT_SHARED | BASE_MEM_FLAGS_RESERVED | \ ++ BASEP_MEM_FLAGS_KERNEL_ONLY)) ++ ++/** ++ * enum base_mem_import_type - Memory types supported by @a base_mem_import ++ * ++ * @BASE_MEM_IMPORT_TYPE_INVALID: Invalid type ++ * @BASE_MEM_IMPORT_TYPE_UMM: UMM import. Handle type is a file descriptor (int) ++ * @BASE_MEM_IMPORT_TYPE_USER_BUFFER: User buffer import. Handle is a ++ * base_mem_import_user_buffer ++ * ++ * Each type defines what the supported handle type is. ++ * ++ * If any new type is added here ARM must be contacted ++ * to allocate a numeric value for it. ++ * Do not just add a new type without synchronizing with ARM ++ * as future releases from ARM might include other new types ++ * which could clash with your custom types. ++ */ ++enum base_mem_import_type { ++ BASE_MEM_IMPORT_TYPE_INVALID = 0, ++ /** ++ * Import type with value 1 is deprecated. ++ */ ++ BASE_MEM_IMPORT_TYPE_UMM = 2, ++ BASE_MEM_IMPORT_TYPE_USER_BUFFER = 3 ++}; ++ ++/** ++ * struct base_mem_import_user_buffer - Handle of an imported user buffer ++ * ++ * @ptr: address of imported user buffer ++ * @length: length of imported user buffer in bytes ++ * ++ * This structure is used to represent a handle of an imported user buffer. ++ */ ++ ++struct base_mem_import_user_buffer { ++ u64 ptr; ++ u64 length; ++}; ++ ++/* Mask to detect 4GB boundary alignment */ ++#define BASE_MEM_MASK_4GB 0xfffff000UL ++/* Mask to detect 4GB boundary (in page units) alignment */ ++#define BASE_MEM_PFN_MASK_4GB (BASE_MEM_MASK_4GB >> LOCAL_PAGE_SHIFT) ++ ++/* Limit on the 'extent' parameter for an allocation with the ++ * BASE_MEM_TILER_ALIGN_TOP flag set ++ * ++ * This is the same as the maximum limit for a Buffer Descriptor's chunk size ++ */ ++#define BASE_MEM_TILER_ALIGN_TOP_EXTENT_MAX_PAGES_LOG2 \ ++ (21u - (LOCAL_PAGE_SHIFT)) ++#define BASE_MEM_TILER_ALIGN_TOP_EXTENT_MAX_PAGES \ ++ (1ull << (BASE_MEM_TILER_ALIGN_TOP_EXTENT_MAX_PAGES_LOG2)) ++ ++/* Bit mask of cookies used for for memory allocation setup */ ++#define KBASE_COOKIE_MASK ~1UL /* bit 0 is reserved */ ++ ++/* Maximum size allowed in a single KBASE_IOCTL_MEM_ALLOC call */ ++#define KBASE_MEM_ALLOC_MAX_SIZE ((8ull << 30) >> PAGE_SHIFT) /* 8 GB */ ++ ++/** ++ * struct base_fence - Cross-device synchronisation fence. ++ * ++ * A fence is used to signal when the GPU has finished accessing a resource that ++ * may be shared with other devices, and also to delay work done asynchronously ++ * by the GPU until other devices have finished accessing a shared resource. ++ */ ++struct base_fence { ++ struct { ++ int fd; ++ int stream_fd; ++ } basep; ++}; ++ ++/** ++ * struct base_mem_aliasing_info - Memory aliasing info ++ * ++ * Describes a memory handle to be aliased. ++ * A subset of the handle can be chosen for aliasing, given an offset and a ++ * length. ++ * A special handle BASE_MEM_WRITE_ALLOC_PAGES_HANDLE is used to represent a ++ * region where a special page is mapped with a write-alloc cache setup, ++ * typically used when the write result of the GPU isn't needed, but the GPU ++ * must write anyway. ++ * ++ * Offset and length are specified in pages. ++ * Offset must be within the size of the handle. ++ * Offset+length must not overrun the size of the handle. ++ * ++ * @handle: Handle to alias, can be BASE_MEM_WRITE_ALLOC_PAGES_HANDLE ++ * @offset: Offset within the handle to start aliasing from, in pages. ++ * Not used with BASE_MEM_WRITE_ALLOC_PAGES_HANDLE. ++ * @length: Length to alias, in pages. For BASE_MEM_WRITE_ALLOC_PAGES_HANDLE ++ * specifies the number of times the special page is needed. ++ */ ++struct base_mem_aliasing_info { ++ struct base_mem_handle handle; ++ u64 offset; ++ u64 length; ++}; ++ ++/* Maximum percentage of just-in-time memory allocation trimming to perform ++ * on free. ++ */ ++#define BASE_JIT_MAX_TRIM_LEVEL (100) ++ ++/* Maximum number of concurrent just-in-time memory allocations. ++ */ ++#define BASE_JIT_ALLOC_COUNT (255) ++ ++/* base_jit_alloc_info in use for kernel driver versions 10.2 to early 11.5 ++ * ++ * jit_version is 1 ++ * ++ * Due to the lack of padding specified, user clients between 32 and 64-bit ++ * may have assumed a different size of the struct ++ * ++ * An array of structures was not supported ++ */ ++struct base_jit_alloc_info_10_2 { ++ u64 gpu_alloc_addr; ++ u64 va_pages; ++ u64 commit_pages; ++ u64 extent; ++ u8 id; ++}; ++ ++/* base_jit_alloc_info introduced by kernel driver version 11.5, and in use up ++ * to 11.19 ++ * ++ * This structure had a number of modifications during and after kernel driver ++ * version 11.5, but remains size-compatible throughout its version history, and ++ * with earlier variants compatible with future variants by requiring ++ * zero-initialization to the unused space in the structure. ++ * ++ * jit_version is 2 ++ * ++ * Kernel driver version history: ++ * 11.5: Initial introduction with 'usage_id' and padding[5]. All padding bytes ++ * must be zero. Kbase minor version was not incremented, so some ++ * versions of 11.5 do not have this change. ++ * 11.5: Added 'bin_id' and 'max_allocations', replacing 2 padding bytes (Kbase ++ * minor version not incremented) ++ * 11.6: Added 'flags', replacing 1 padding byte ++ * 11.10: Arrays of this structure are supported ++ */ ++struct base_jit_alloc_info_11_5 { ++ u64 gpu_alloc_addr; ++ u64 va_pages; ++ u64 commit_pages; ++ u64 extent; ++ u8 id; ++ u8 bin_id; ++ u8 max_allocations; ++ u8 flags; ++ u8 padding[2]; ++ u16 usage_id; ++}; ++ ++/** ++ * struct base_jit_alloc_info - Structure which describes a JIT allocation ++ * request. ++ * @gpu_alloc_addr: The GPU virtual address to write the JIT ++ * allocated GPU virtual address to. ++ * @va_pages: The minimum number of virtual pages required. ++ * @commit_pages: The minimum number of physical pages which ++ * should back the allocation. ++ * @extent: Granularity of physical pages to grow the ++ * allocation by during a fault. ++ * @id: Unique ID provided by the caller, this is used ++ * to pair allocation and free requests. ++ * Zero is not a valid value. ++ * @bin_id: The JIT allocation bin, used in conjunction with ++ * @max_allocations to limit the number of each ++ * type of JIT allocation. ++ * @max_allocations: The maximum number of allocations allowed within ++ * the bin specified by @bin_id. Should be the same ++ * for all allocations within the same bin. ++ * @flags: flags specifying the special requirements for ++ * the JIT allocation, see ++ * %BASE_JIT_ALLOC_VALID_FLAGS ++ * @padding: Expansion space - should be initialised to zero ++ * @usage_id: A hint about which allocation should be reused. ++ * The kernel should attempt to use a previous ++ * allocation with the same usage_id ++ * @heap_info_gpu_addr: Pointer to an object in GPU memory describing ++ * the actual usage of the region. ++ * ++ * jit_version is 3. ++ * ++ * When modifications are made to this structure, it is still compatible with ++ * jit_version 3 when: a) the size is unchanged, and b) new members only ++ * replace the padding bytes. ++ * ++ * Previous jit_version history: ++ * jit_version == 1, refer to &base_jit_alloc_info_10_2 ++ * jit_version == 2, refer to &base_jit_alloc_info_11_5 ++ * ++ * Kbase version history: ++ * 11.20: added @heap_info_gpu_addr ++ */ ++struct base_jit_alloc_info { ++ u64 gpu_alloc_addr; ++ u64 va_pages; ++ u64 commit_pages; ++ u64 extent; ++ u8 id; ++ u8 bin_id; ++ u8 max_allocations; ++ u8 flags; ++ u8 padding[2]; ++ u16 usage_id; ++ u64 heap_info_gpu_addr; ++}; ++ ++enum base_external_resource_access { ++ BASE_EXT_RES_ACCESS_SHARED, ++ BASE_EXT_RES_ACCESS_EXCLUSIVE ++}; ++ ++struct base_external_resource { ++ u64 ext_resource; ++}; ++ ++ ++/** ++ * The maximum number of external resources which can be mapped/unmapped ++ * in a single request. ++ */ ++#define BASE_EXT_RES_COUNT_MAX 10 ++ ++/** ++ * struct base_external_resource_list - Structure which describes a list of ++ * external resources. ++ * @count: The number of resources. ++ * @ext_res: Array of external resources which is ++ * sized at allocation time. ++ */ ++struct base_external_resource_list { ++ u64 count; ++ struct base_external_resource ext_res[1]; ++}; ++ ++struct base_jd_debug_copy_buffer { ++ u64 address; ++ u64 size; ++ struct base_external_resource extres; ++}; ++ ++#define GPU_MAX_JOB_SLOTS 16 ++ ++/** ++ * User-side Base GPU Property Queries ++ * ++ * The User-side Base GPU Property Query interface encapsulates two ++ * sub-modules: ++ * ++ * - "Dynamic GPU Properties" ++ * - "Base Platform Config GPU Properties" ++ * ++ * Base only deals with properties that vary between different GPU ++ * implementations - the Dynamic GPU properties and the Platform Config ++ * properties. ++ * ++ * For properties that are constant for the GPU Architecture, refer to the ++ * GPU module. However, we will discuss their relevance here just to ++ * provide background information. ++ * ++ * About the GPU Properties in Base and GPU modules ++ * ++ * The compile-time properties (Platform Config, GPU Compile-time ++ * properties) are exposed as pre-processor macros. ++ * ++ * Complementing the compile-time properties are the Dynamic GPU ++ * Properties, which act as a conduit for the GPU Configuration ++ * Discovery. ++ * ++ * In general, the dynamic properties are present to verify that the platform ++ * has been configured correctly with the right set of Platform Config ++ * Compile-time Properties. ++ * ++ * As a consistent guide across the entire DDK, the choice for dynamic or ++ * compile-time should consider the following, in order: ++ * 1. Can the code be written so that it doesn't need to know the ++ * implementation limits at all? ++ * 2. If you need the limits, get the information from the Dynamic Property ++ * lookup. This should be done once as you fetch the context, and then cached ++ * as part of the context data structure, so it's cheap to access. ++ * 3. If there's a clear and arguable inefficiency in using Dynamic Properties, ++ * then use a Compile-Time Property (Platform Config, or GPU Compile-time ++ * property). Examples of where this might be sensible follow: ++ * - Part of a critical inner-loop ++ * - Frequent re-use throughout the driver, causing significant extra load ++ * instructions or control flow that would be worthwhile optimizing out. ++ * ++ * We cannot provide an exhaustive set of examples, neither can we provide a ++ * rule for every possible situation. Use common sense, and think about: what ++ * the rest of the driver will be doing; how the compiler might represent the ++ * value if it is a compile-time constant; whether an OEM shipping multiple ++ * devices would benefit much more from a single DDK binary, instead of ++ * insignificant micro-optimizations. ++ * ++ * Dynamic GPU Properties ++ * ++ * Dynamic GPU properties are presented in two sets: ++ * 1. the commonly used properties in @ref base_gpu_props, which have been ++ * unpacked from GPU register bitfields. ++ * 2. The full set of raw, unprocessed properties in gpu_raw_gpu_props ++ * (also a member of base_gpu_props). All of these are presented in ++ * the packed form, as presented by the GPU registers themselves. ++ * ++ * The raw properties in gpu_raw_gpu_props are necessary to ++ * allow a user of the Mali Tools (e.g. PAT) to determine "Why is this device ++ * behaving differently?". In this case, all information about the ++ * configuration is potentially useful, but it does not need to be processed ++ * by the driver. Instead, the raw registers can be processed by the Mali ++ * Tools software on the host PC. ++ * ++ * The properties returned extend the GPU Configuration Discovery ++ * registers. For example, GPU clock speed is not specified in the GPU ++ * Architecture, but is necessary for OpenCL's clGetDeviceInfo() function. ++ * ++ * The GPU properties are obtained by a call to ++ * base_get_gpu_props(). This simply returns a pointer to a const ++ * base_gpu_props structure. It is constant for the life of a base ++ * context. Multiple calls to base_get_gpu_props() to a base context ++ * return the same pointer to a constant structure. This avoids cache pollution ++ * of the common data. ++ * ++ * This pointer must not be freed, because it does not point to the start of a ++ * region allocated by the memory allocator; instead, just close the @ref ++ * base_context. ++ * ++ * ++ * Kernel Operation ++ * ++ * During Base Context Create time, user-side makes a single kernel call: ++ * - A call to fill user memory with GPU information structures ++ * ++ * The kernel-side will fill the provided the entire processed base_gpu_props ++ * structure, because this information is required in both ++ * user and kernel side; it does not make sense to decode it twice. ++ * ++ * Coherency groups must be derived from the bitmasks, but this can be done ++ * kernel side, and just once at kernel startup: Coherency groups must already ++ * be known kernel-side, to support chains that specify a 'Only Coherent Group' ++ * SW requirement, or 'Only Coherent Group with Tiler' SW requirement. ++ * ++ * Coherency Group calculation ++ * ++ * Creation of the coherent group data is done at device-driver startup, and so ++ * is one-time. This will most likely involve a loop with CLZ, shifting, and ++ * bit clearing on the L2_PRESENT mask, depending on whether the ++ * system is L2 Coherent. The number of shader cores is done by a ++ * population count, since faulty cores may be disabled during production, ++ * producing a non-contiguous mask. ++ * ++ * The memory requirements for this algorithm can be determined either by a u64 ++ * population count on the L2_PRESENT mask (a LUT helper already is ++ * required for the above), or simple assumption that there can be no more than ++ * 16 coherent groups, since core groups are typically 4 cores. ++ */ ++ ++#define BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS 4 ++ ++#define BASE_MAX_COHERENT_GROUPS 16 ++ ++struct mali_base_gpu_core_props { ++ /** ++ * Product specific value. ++ */ ++ u32 product_id; ++ ++ /** ++ * Status of the GPU release. ++ * No defined values, but starts at 0 and increases by one for each ++ * release status (alpha, beta, EAC, etc.). ++ * 4 bit values (0-15). ++ */ ++ u16 version_status; ++ ++ /** ++ * Minor release number of the GPU. "P" part of an "RnPn" release number. ++ * 8 bit values (0-255). ++ */ ++ u16 minor_revision; ++ ++ /** ++ * Major release number of the GPU. "R" part of an "RnPn" release number. ++ * 4 bit values (0-15). ++ */ ++ u16 major_revision; ++ ++ u16 padding; ++ ++ /* The maximum GPU frequency. Reported to applications by ++ * clGetDeviceInfo() ++ */ ++ u32 gpu_freq_khz_max; ++ ++ /** ++ * Size of the shader program counter, in bits. ++ */ ++ u32 log2_program_counter_size; ++ ++ /** ++ * TEXTURE_FEATURES_x registers, as exposed by the GPU. This is a ++ * bitpattern where a set bit indicates that the format is supported. ++ * ++ * Before using a texture format, it is recommended that the corresponding ++ * bit be checked. ++ */ ++ u32 texture_features[BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS]; ++ ++ /** ++ * Theoretical maximum memory available to the GPU. It is unlikely that a ++ * client will be able to allocate all of this memory for their own ++ * purposes, but this at least provides an upper bound on the memory ++ * available to the GPU. ++ * ++ * This is required for OpenCL's clGetDeviceInfo() call when ++ * CL_DEVICE_GLOBAL_MEM_SIZE is requested, for OpenCL GPU devices. The ++ * client will not be expecting to allocate anywhere near this value. ++ */ ++ u64 gpu_available_memory_size; ++ ++ /** ++ * The number of execution engines. ++ */ ++ u8 num_exec_engines; ++}; ++ ++/** ++ * ++ * More information is possible - but associativity and bus width are not ++ * required by upper-level apis. ++ */ ++struct mali_base_gpu_l2_cache_props { ++ u8 log2_line_size; ++ u8 log2_cache_size; ++ u8 num_l2_slices; /* Number of L2C slices. 1 or higher */ ++ u8 padding[5]; ++}; ++ ++struct mali_base_gpu_tiler_props { ++ u32 bin_size_bytes; /* Max is 4*2^15 */ ++ u32 max_active_levels; /* Max is 2^15 */ ++}; ++ ++/** ++ * GPU threading system details. ++ */ ++struct mali_base_gpu_thread_props { ++ u32 max_threads; /* Max. number of threads per core */ ++ u32 max_workgroup_size; /* Max. number of threads per workgroup */ ++ u32 max_barrier_size; /* Max. number of threads that can synchronize on a simple barrier */ ++ u16 max_registers; /* Total size [1..65535] of the register file available per core. */ ++ u8 max_task_queue; /* Max. tasks [1..255] which may be sent to a core before it becomes blocked. */ ++ u8 max_thread_group_split; /* Max. allowed value [1..15] of the Thread Group Split field. */ ++ u8 impl_tech; /* 0 = Not specified, 1 = Silicon, 2 = FPGA, 3 = SW Model/Emulation */ ++ u8 padding[3]; ++ u32 tls_alloc; /* Number of threads per core that TLS must ++ * be allocated for ++ */ ++}; ++ ++/** ++ * struct mali_base_gpu_coherent_group - descriptor for a coherent group ++ * ++ * \c core_mask exposes all cores in that coherent group, and \c num_cores ++ * provides a cached population-count for that mask. ++ * ++ * @note Whilst all cores are exposed in the mask, not all may be available to ++ * the application, depending on the Kernel Power policy. ++ * ++ * @note if u64s must be 8-byte aligned, then this structure has 32-bits of wastage. ++ */ ++struct mali_base_gpu_coherent_group { ++ u64 core_mask; /**< Core restriction mask required for the group */ ++ u16 num_cores; /**< Number of cores in the group */ ++ u16 padding[3]; ++}; ++ ++/** ++ * struct mali_base_gpu_coherent_group_info - Coherency group information ++ * ++ * Note that the sizes of the members could be reduced. However, the \c group ++ * member might be 8-byte aligned to ensure the u64 core_mask is 8-byte ++ * aligned, thus leading to wastage if the other members sizes were reduced. ++ * ++ * The groups are sorted by core mask. The core masks are non-repeating and do ++ * not intersect. ++ */ ++struct mali_base_gpu_coherent_group_info { ++ u32 num_groups; ++ ++ /** ++ * Number of core groups (coherent or not) in the GPU. Equivalent to the number of L2 Caches. ++ * ++ * The GPU Counter dumping writes 2048 bytes per core group, regardless of ++ * whether the core groups are coherent or not. Hence this member is needed ++ * to calculate how much memory is required for dumping. ++ * ++ * @note Do not use it to work out how many valid elements are in the ++ * group[] member. Use num_groups instead. ++ */ ++ u32 num_core_groups; ++ ++ /** ++ * Coherency features of the memory, accessed by gpu_mem_features ++ * methods ++ */ ++ u32 coherency; ++ ++ u32 padding; ++ ++ /** ++ * Descriptors of coherent groups ++ */ ++ struct mali_base_gpu_coherent_group group[BASE_MAX_COHERENT_GROUPS]; ++}; ++ ++/** ++ * struct gpu_raw_gpu_props - A complete description of the GPU's Hardware ++ * Configuration Discovery registers. ++ * ++ * The information is presented inefficiently for access. For frequent access, ++ * the values should be better expressed in an unpacked form in the ++ * base_gpu_props structure. ++ * ++ * The raw properties in gpu_raw_gpu_props are necessary to ++ * allow a user of the Mali Tools (e.g. PAT) to determine "Why is this device ++ * behaving differently?". In this case, all information about the ++ * configuration is potentially useful, but it does not need to be processed ++ * by the driver. Instead, the raw registers can be processed by the Mali ++ * Tools software on the host PC. ++ * ++ */ ++struct gpu_raw_gpu_props { ++ u64 shader_present; ++ u64 tiler_present; ++ u64 l2_present; ++ u64 stack_present; ++ ++ u32 l2_features; ++ u32 core_features; ++ u32 mem_features; ++ u32 mmu_features; ++ ++ u32 as_present; ++ ++ u32 js_present; ++ u32 js_features[GPU_MAX_JOB_SLOTS]; ++ u32 tiler_features; ++ u32 texture_features[BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS]; ++ ++ u32 gpu_id; ++ ++ u32 thread_max_threads; ++ u32 thread_max_workgroup_size; ++ u32 thread_max_barrier_size; ++ u32 thread_features; ++ ++ /* ++ * Note: This is the _selected_ coherency mode rather than the ++ * available modes as exposed in the coherency_features register. ++ */ ++ u32 coherency_mode; ++ ++ u32 thread_tls_alloc; ++}; ++ ++/** ++ * struct base_gpu_props - Return structure for base_get_gpu_props(). ++ * ++ * NOTE: the raw_props member in this data structure contains the register ++ * values from which the value of the other members are derived. The derived ++ * members exist to allow for efficient access and/or shielding the details ++ * of the layout of the registers. ++ * ++ * @unused_1: Keep for backwards compatibility. ++ * @raw_props: This member is large, likely to be 128 bytes. ++ * @coherency_info: This must be last member of the structure. ++ */ ++struct base_gpu_props { ++ struct mali_base_gpu_core_props core_props; ++ struct mali_base_gpu_l2_cache_props l2_props; ++ u64 unused_1; ++ struct mali_base_gpu_tiler_props tiler_props; ++ struct mali_base_gpu_thread_props thread_props; ++ struct gpu_raw_gpu_props raw_props; ++ struct mali_base_gpu_coherent_group_info coherency_info; ++}; ++ ++#if MALI_USE_CSF ++#include "csf/mali_base_csf_kernel.h" ++#else ++#include "jm/mali_base_jm_kernel.h" ++#endif ++ ++/** ++ * base_mem_group_id_get() - Get group ID from flags ++ * @flags: Flags to pass to base_mem_alloc ++ * ++ * This inline function extracts the encoded group ID from flags ++ * and converts it into numeric value (0~15). ++ * ++ * Return: group ID(0~15) extracted from the parameter ++ */ ++static inline int base_mem_group_id_get(base_mem_alloc_flags flags) ++{ ++ LOCAL_ASSERT((flags & ~BASE_MEM_FLAGS_INPUT_MASK) == 0); ++ return (int)((flags & BASE_MEM_GROUP_ID_MASK) >> ++ BASEP_MEM_GROUP_ID_SHIFT); ++} ++ ++/** ++ * base_mem_group_id_set() - Set group ID into base_mem_alloc_flags ++ * @id: group ID(0~15) you want to encode ++ * ++ * This inline function encodes specific group ID into base_mem_alloc_flags. ++ * Parameter 'id' should lie in-between 0 to 15. ++ * ++ * Return: base_mem_alloc_flags with the group ID (id) encoded ++ * ++ * The return value can be combined with other flags against base_mem_alloc ++ * to identify a specific memory group. ++ */ ++static inline base_mem_alloc_flags base_mem_group_id_set(int id) ++{ ++ if ((id < 0) || (id >= BASE_MEM_GROUP_COUNT)) { ++ /* Set to default value when id is out of range. */ ++ id = BASE_MEM_GROUP_DEFAULT; ++ } ++ ++ return ((base_mem_alloc_flags)id << BASEP_MEM_GROUP_ID_SHIFT) & ++ BASE_MEM_GROUP_ID_MASK; ++} ++ ++/** ++ * base_context_mmu_group_id_set - Encode a memory group ID in ++ * base_context_create_flags ++ * ++ * Memory allocated for GPU page tables will come from the specified group. ++ * ++ * @group_id: Physical memory group ID. Range is 0..(BASE_MEM_GROUP_COUNT-1). ++ * ++ * Return: Bitmask of flags to pass to base_context_init. ++ */ ++static inline base_context_create_flags base_context_mmu_group_id_set( ++ int const group_id) ++{ ++ LOCAL_ASSERT(group_id >= 0); ++ LOCAL_ASSERT(group_id < BASE_MEM_GROUP_COUNT); ++ return BASEP_CONTEXT_MMU_GROUP_ID_MASK & ++ ((base_context_create_flags)group_id << ++ BASEP_CONTEXT_MMU_GROUP_ID_SHIFT); ++} ++ ++/** ++ * base_context_mmu_group_id_get - Decode a memory group ID from ++ * base_context_create_flags ++ * ++ * Memory allocated for GPU page tables will come from the returned group. ++ * ++ * @flags: Bitmask of flags to pass to base_context_init. ++ * ++ * Return: Physical memory group ID. Valid range is 0..(BASE_MEM_GROUP_COUNT-1). ++ */ ++static inline int base_context_mmu_group_id_get( ++ base_context_create_flags const flags) ++{ ++ LOCAL_ASSERT(flags == (flags & BASEP_CONTEXT_CREATE_ALLOWED_FLAGS)); ++ return (int)((flags & BASEP_CONTEXT_MMU_GROUP_ID_MASK) >> ++ BASEP_CONTEXT_MMU_GROUP_ID_SHIFT); ++} ++ ++/* ++ * A number of bit flags are defined for requesting cpu_gpu_timeinfo. These ++ * flags are also used, where applicable, for specifying which fields ++ * are valid following the request operation. ++ */ ++ ++/* For monotonic (counter) timefield */ ++#define BASE_TIMEINFO_MONOTONIC_FLAG (1UL << 0) ++/* For system wide timestamp */ ++#define BASE_TIMEINFO_TIMESTAMP_FLAG (1UL << 1) ++/* For GPU cycle counter */ ++#define BASE_TIMEINFO_CYCLE_COUNTER_FLAG (1UL << 2) ++/* Specify kernel GPU register timestamp */ ++#define BASE_TIMEINFO_KERNEL_SOURCE_FLAG (1UL << 30) ++/* Specify userspace cntvct_el0 timestamp source */ ++#define BASE_TIMEINFO_USER_SOURCE_FLAG (1UL << 31) ++ ++#define BASE_TIMEREQUEST_ALLOWED_FLAGS (\ ++ BASE_TIMEINFO_MONOTONIC_FLAG | \ ++ BASE_TIMEINFO_TIMESTAMP_FLAG | \ ++ BASE_TIMEINFO_CYCLE_COUNTER_FLAG | \ ++ BASE_TIMEINFO_KERNEL_SOURCE_FLAG | \ ++ BASE_TIMEINFO_USER_SOURCE_FLAG) ++ ++#endif /* _BASE_KERNEL_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_base_mem_priv.h b/drivers/gpu/arm/bifrost/mali_base_mem_priv.h +new file mode 100755 +index 000000000000..844a025b715d +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_base_mem_priv.h +@@ -0,0 +1,57 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2015, 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++#ifndef _BASE_MEM_PRIV_H_ ++#define _BASE_MEM_PRIV_H_ ++ ++#define BASE_SYNCSET_OP_MSYNC (1U << 0) ++#define BASE_SYNCSET_OP_CSYNC (1U << 1) ++ ++/* ++ * This structure describe a basic memory coherency operation. ++ * It can either be: ++ * @li a sync from CPU to Memory: ++ * - type = ::BASE_SYNCSET_OP_MSYNC ++ * - mem_handle = a handle to the memory object on which the operation ++ * is taking place ++ * - user_addr = the address of the range to be synced ++ * - size = the amount of data to be synced, in bytes ++ * - offset is ignored. ++ * @li a sync from Memory to CPU: ++ * - type = ::BASE_SYNCSET_OP_CSYNC ++ * - mem_handle = a handle to the memory object on which the operation ++ * is taking place ++ * - user_addr = the address of the range to be synced ++ * - size = the amount of data to be synced, in bytes. ++ * - offset is ignored. ++ */ ++struct basep_syncset { ++ struct base_mem_handle mem_handle; ++ u64 user_addr; ++ u64 size; ++ u8 type; ++ u8 padding[7]; ++}; ++ ++#endif +diff --git a/drivers/gpu/arm/bifrost/mali_kbase.h b/drivers/gpu/arm/bifrost/mali_kbase.h +new file mode 100755 +index 000000000000..8189d02ab910 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase.h +@@ -0,0 +1,614 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++#ifndef _KBASE_H_ ++#define _KBASE_H_ ++ ++#include ++ ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)) ++#include ++#endif ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "mali_base_kernel.h" ++#include ++ ++/* ++ * Include mali_kbase_defs.h first as this provides types needed by other local ++ * header files. ++ */ ++#include "mali_kbase_defs.h" ++ ++#include "debug/mali_kbase_debug_ktrace.h" ++#include "context/mali_kbase_context.h" ++#include "mali_kbase_strings.h" ++#include "mali_kbase_mem_lowlevel.h" ++#include "mali_kbase_utility.h" ++#include "mali_kbase_mem.h" ++#include "mmu/mali_kbase_mmu.h" ++#include "mali_kbase_gpu_memory_debugfs.h" ++#include "mali_kbase_mem_profile_debugfs.h" ++#include "mali_kbase_gpuprops.h" ++#include "mali_kbase_ioctl.h" ++#if !MALI_USE_CSF ++#include "mali_kbase_debug_job_fault.h" ++#include "mali_kbase_jd_debugfs.h" ++#include "mali_kbase_jm.h" ++#include "mali_kbase_js.h" ++#endif /* !MALI_USE_CSF */ ++ ++#include "ipa/mali_kbase_ipa.h" ++ ++#ifdef CONFIG_GPU_TRACEPOINTS ++#include ++#endif ++ ++#include "mali_linux_trace.h" ++ ++#if MALI_USE_CSF ++#include "csf/mali_kbase_csf.h" ++#endif ++ ++#ifndef u64_to_user_ptr ++/* Introduced in Linux v4.6 */ ++#define u64_to_user_ptr(x) ((void __user *)(uintptr_t)x) ++#endif ++ ++#if MALI_USE_CSF ++/* Physical memory group ID for command stream frontend user I/O. ++ */ ++#define KBASE_MEM_GROUP_CSF_IO BASE_MEM_GROUP_DEFAULT ++ ++/* Physical memory group ID for command stream frontend firmware. ++ */ ++#define KBASE_MEM_GROUP_CSF_FW BASE_MEM_GROUP_DEFAULT ++#endif ++ ++/* Physical memory group ID for a special page which can alias several regions. ++ */ ++#define KBASE_MEM_GROUP_SINK BASE_MEM_GROUP_DEFAULT ++ ++/* ++ * Kernel-side Base (KBase) APIs ++ */ ++ ++struct kbase_device *kbase_device_alloc(void); ++/* ++* note: configuration attributes member of kbdev needs to have ++* been setup before calling kbase_device_init ++*/ ++ ++int kbase_device_misc_init(struct kbase_device *kbdev); ++void kbase_device_misc_term(struct kbase_device *kbdev); ++void kbase_device_free(struct kbase_device *kbdev); ++int kbase_device_has_feature(struct kbase_device *kbdev, u32 feature); ++ ++/* Needed for gator integration and for reporting vsync information */ ++struct kbase_device *kbase_find_device(int minor); ++void kbase_release_device(struct kbase_device *kbdev); ++ ++/** ++ * kbase_context_get_unmapped_area() - get an address range which is currently ++ * unmapped. ++ * @kctx: A kernel base context (which has its own GPU address space). ++ * @addr: CPU mapped address (set to 0 since MAP_FIXED mapping is not allowed ++ * as Mali GPU driver decides about the mapping). ++ * @len: Length of the address range. ++ * @pgoff: Page offset within the GPU address space of the kbase context. ++ * @flags: Flags for the allocation. ++ * ++ * Finds the unmapped address range which satisfies requirements specific to ++ * GPU and those provided by the call parameters. ++ * ++ * 1) Requirement for allocations greater than 2MB: ++ * - alignment offset is set to 2MB and the alignment mask to 2MB decremented ++ * by 1. ++ * ++ * 2) Requirements imposed for the shader memory alignment: ++ * - alignment is decided by the number of GPU pc bits which can be read from ++ * GPU properties of the device associated with this kbase context; alignment ++ * offset is set to this value in bytes and the alignment mask to the offset ++ * decremented by 1. ++ * - allocations must not to be at 4GB boundaries. Such cases are indicated ++ * by the flag KBASE_REG_GPU_NX not being set (check the flags of the kbase ++ * region). 4GB boundaries can be checked against @ref BASE_MEM_MASK_4GB. ++ * ++ * 3) Requirements imposed for tiler memory alignment, cases indicated by ++ * the flag @ref KBASE_REG_TILER_ALIGN_TOP (check the flags of the kbase ++ * region): ++ * - alignment offset is set to the difference between the kbase region ++ * extent (converted from the original value in pages to bytes) and the kbase ++ * region initial_commit (also converted from the original value in pages to ++ * bytes); alignment mask is set to the kbase region extent in bytes and ++ * decremented by 1. ++ * ++ * Return: if successful, address of the unmapped area aligned as required; ++ * error code (negative) in case of failure; ++ */ ++unsigned long kbase_context_get_unmapped_area(struct kbase_context *kctx, ++ const unsigned long addr, const unsigned long len, ++ const unsigned long pgoff, const unsigned long flags); ++ ++ ++int assign_irqs(struct kbase_device *kbdev); ++ ++int kbase_sysfs_init(struct kbase_device *kbdev); ++void kbase_sysfs_term(struct kbase_device *kbdev); ++ ++ ++int kbase_protected_mode_init(struct kbase_device *kbdev); ++void kbase_protected_mode_term(struct kbase_device *kbdev); ++ ++/** ++ * kbase_device_pm_init() - Performs power management initialization and ++ * Verifies device tree configurations. ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * Return: 0 if successful, otherwise a standard Linux error code ++ */ ++int kbase_device_pm_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_device_pm_term() - Performs power management deinitialization and ++ * Free resources. ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * Clean up all the resources ++ */ ++void kbase_device_pm_term(struct kbase_device *kbdev); ++ ++ ++int power_control_init(struct kbase_device *kbdev); ++void power_control_term(struct kbase_device *kbdev); ++ ++#ifdef CONFIG_DEBUG_FS ++void kbase_device_debugfs_term(struct kbase_device *kbdev); ++int kbase_device_debugfs_init(struct kbase_device *kbdev); ++#else /* CONFIG_DEBUG_FS */ ++static inline int kbase_device_debugfs_init(struct kbase_device *kbdev) ++{ ++ return 0; ++} ++ ++static inline void kbase_device_debugfs_term(struct kbase_device *kbdev) { } ++#endif /* CONFIG_DEBUG_FS */ ++ ++int registers_map(struct kbase_device *kbdev); ++void registers_unmap(struct kbase_device *kbdev); ++ ++int kbase_device_coherency_init(struct kbase_device *kbdev); ++ ++#ifdef CONFIG_MALI_BUSLOG ++int buslog_init(struct kbase_device *kbdev); ++void buslog_term(struct kbase_device *kbdev); ++#endif ++ ++#if !MALI_USE_CSF ++int kbase_jd_init(struct kbase_context *kctx); ++void kbase_jd_exit(struct kbase_context *kctx); ++ ++/** ++ * kbase_jd_submit - Submit atoms to the job dispatcher ++ * ++ * @kctx: The kbase context to submit to ++ * @user_addr: The address in user space of the struct base_jd_atom array ++ * @nr_atoms: The number of atoms in the array ++ * @stride: sizeof(struct base_jd_atom) ++ * @uk6_atom: true if the atoms are legacy atoms (struct base_jd_atom_v2_uk6) ++ * ++ * Return: 0 on success or error code ++ */ ++int kbase_jd_submit(struct kbase_context *kctx, ++ void __user *user_addr, u32 nr_atoms, u32 stride, ++ bool uk6_atom); ++ ++/** ++ * kbase_jd_done_worker - Handle a job completion ++ * @data: a &struct work_struct ++ * ++ * This function requeues the job from the runpool (if it was soft-stopped or ++ * removed from NEXT registers). ++ * ++ * Removes it from the system if it finished/failed/was cancelled. ++ * ++ * Resolves dependencies to add dependent jobs to the context, potentially ++ * starting them if necessary (which may add more references to the context) ++ * ++ * Releases the reference to the context from the no-longer-running job. ++ * ++ * Handles retrying submission outside of IRQ context if it failed from within ++ * IRQ context. ++ */ ++void kbase_jd_done_worker(struct work_struct *data); ++ ++void kbase_jd_done(struct kbase_jd_atom *katom, int slot_nr, ktime_t *end_timestamp, ++ kbasep_js_atom_done_code done_code); ++void kbase_jd_cancel(struct kbase_device *kbdev, struct kbase_jd_atom *katom); ++void kbase_jd_zap_context(struct kbase_context *kctx); ++bool jd_done_nolock(struct kbase_jd_atom *katom, ++ struct list_head *completed_jobs_ctx); ++void kbase_jd_free_external_resources(struct kbase_jd_atom *katom); ++void kbase_jd_dep_clear_locked(struct kbase_jd_atom *katom); ++ ++/** ++ * kbase_job_done - Process completed jobs from job interrupt ++ * @kbdev: Pointer to the kbase device. ++ * @done: Bitmask of done or failed jobs, from JOB_IRQ_STAT register ++ * ++ * This function processes the completed, or failed, jobs from the GPU job ++ * slots, for the bits set in the @done bitmask. ++ * ++ * The hwaccess_lock must be held when calling this function. ++ */ ++void kbase_job_done(struct kbase_device *kbdev, u32 done); ++ ++/** ++ * kbase_job_slot_ctx_priority_check_locked(): - Check for lower priority atoms ++ * and soft stop them ++ * @kctx: Pointer to context to check. ++ * @katom: Pointer to priority atom. ++ * ++ * Atoms from @kctx on the same job slot as @katom, which have lower priority ++ * than @katom will be soft stopped and put back in the queue, so that atoms ++ * with higher priority can run. ++ * ++ * The hwaccess_lock must be held when calling this function. ++ */ ++void kbase_job_slot_ctx_priority_check_locked(struct kbase_context *kctx, ++ struct kbase_jd_atom *katom); ++ ++/** ++ * kbase_job_slot_softstop_start_rp() - Soft-stop the atom at the start ++ * of a renderpass. ++ * @kctx: Pointer to a kernel base context. ++ * @reg: Reference of a growable GPU memory region in the same context. ++ * Takes ownership of the reference if successful. ++ * ++ * Used to switch to incremental rendering if we have nearly run out of ++ * virtual address space in a growable memory region and the atom currently ++ * executing on a job slot is the tiler job chain at the start of a renderpass. ++ * ++ * Return 0 if successful, otherwise a negative error code. ++ */ ++int kbase_job_slot_softstop_start_rp(struct kbase_context *kctx, ++ struct kbase_va_region *reg); ++ ++void kbase_job_slot_softstop(struct kbase_device *kbdev, int js, ++ struct kbase_jd_atom *target_katom); ++void kbase_job_slot_softstop_swflags(struct kbase_device *kbdev, int js, ++ struct kbase_jd_atom *target_katom, u32 sw_flags); ++void kbase_job_slot_hardstop(struct kbase_context *kctx, int js, ++ struct kbase_jd_atom *target_katom); ++void kbase_job_check_enter_disjoint(struct kbase_device *kbdev, u32 action, ++ base_jd_core_req core_reqs, struct kbase_jd_atom *target_katom); ++void kbase_job_check_leave_disjoint(struct kbase_device *kbdev, ++ struct kbase_jd_atom *target_katom); ++ ++#endif /* !MALI_USE_CSF */ ++ ++void kbase_event_post(struct kbase_context *ctx, struct kbase_jd_atom *event); ++#if !MALI_USE_CSF ++int kbase_event_dequeue(struct kbase_context *ctx, struct base_jd_event_v2 *uevent); ++#endif /* !MALI_USE_CSF */ ++int kbase_event_pending(struct kbase_context *ctx); ++int kbase_event_init(struct kbase_context *kctx); ++void kbase_event_close(struct kbase_context *kctx); ++void kbase_event_cleanup(struct kbase_context *kctx); ++void kbase_event_wakeup(struct kbase_context *kctx); ++ ++/** ++ * kbasep_jit_alloc_validate() - Validate the JIT allocation info. ++ * ++ * @kctx: Pointer to the kbase context within which the JIT ++ * allocation is to be validated. ++ * @info: Pointer to struct @base_jit_alloc_info ++ * which is to be validated. ++ * @return: 0 if jit allocation is valid; negative error code otherwise ++ */ ++int kbasep_jit_alloc_validate(struct kbase_context *kctx, ++ struct base_jit_alloc_info *info); ++ ++/** ++ * kbase_jit_retry_pending_alloc() - Retry blocked just-in-time memory ++ * allocations. ++ * ++ * @kctx: Pointer to the kbase context within which the just-in-time ++ * memory allocations are to be retried. ++ */ ++void kbase_jit_retry_pending_alloc(struct kbase_context *kctx); ++ ++/** ++ * kbase_free_user_buffer() - Free memory allocated for struct ++ * @kbase_debug_copy_buffer. ++ * ++ * @buffer: Pointer to the memory location allocated for the object ++ * of the type struct @kbase_debug_copy_buffer. ++ */ ++static inline void kbase_free_user_buffer( ++ struct kbase_debug_copy_buffer *buffer) ++{ ++ struct page **pages = buffer->extres_pages; ++ int nr_pages = buffer->nr_extres_pages; ++ ++ if (pages) { ++ int i; ++ ++ for (i = 0; i < nr_pages; i++) { ++ struct page *pg = pages[i]; ++ ++ if (pg) ++ put_page(pg); ++ } ++ kfree(pages); ++ } ++} ++ ++/** ++ * kbase_mem_copy_from_extres() - Copy from external resources. ++ * ++ * @kctx: kbase context within which the copying is to take place. ++ * @buf_data: Pointer to the information about external resources: ++ * pages pertaining to the external resource, number of ++ * pages to copy. ++ */ ++int kbase_mem_copy_from_extres(struct kbase_context *kctx, ++ struct kbase_debug_copy_buffer *buf_data); ++#if !MALI_USE_CSF ++int kbase_process_soft_job(struct kbase_jd_atom *katom); ++int kbase_prepare_soft_job(struct kbase_jd_atom *katom); ++void kbase_finish_soft_job(struct kbase_jd_atom *katom); ++void kbase_cancel_soft_job(struct kbase_jd_atom *katom); ++void kbase_resume_suspended_soft_jobs(struct kbase_device *kbdev); ++void kbasep_remove_waiting_soft_job(struct kbase_jd_atom *katom); ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++void kbase_soft_event_wait_callback(struct kbase_jd_atom *katom); ++#endif ++int kbase_soft_event_update(struct kbase_context *kctx, ++ u64 event, ++ unsigned char new_status); ++ ++void kbasep_soft_job_timeout_worker(struct timer_list *timer); ++void kbasep_complete_triggered_soft_events(struct kbase_context *kctx, u64 evt); ++#endif /* !MALI_USE_CSF */ ++ ++void kbasep_as_do_poke(struct work_struct *work); ++ ++/** ++ * Check whether a system suspend is in progress, or has already been suspended ++ * ++ * The caller should ensure that either kbdev->pm.active_count_lock is held, or ++ * a dmb was executed recently (to ensure the value is most ++ * up-to-date). However, without a lock the value could change afterwards. ++ * ++ * @return false if a suspend is not in progress ++ * @return !=false otherwise ++ */ ++static inline bool kbase_pm_is_suspending(struct kbase_device *kbdev) ++{ ++ return kbdev->pm.suspending; ++} ++ ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++/* ++ * Check whether a gpu lost is in progress ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * Indicates whether a gpu lost has been received and jobs are no longer ++ * being scheduled ++ * ++ * Return: false if gpu is lost ++ * Return: != false otherwise ++ */ ++static inline bool kbase_pm_is_gpu_lost(struct kbase_device *kbdev) ++{ ++ return (atomic_read(&kbdev->pm.gpu_lost) == 0 ? false : true); ++} ++ ++/* ++ * Set or clear gpu lost state ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * @gpu_lost: true to activate GPU lost state, FALSE is deactive it ++ * ++ * Puts power management code into gpu lost state or takes it out of the ++ * state. Once in gpu lost state new GPU jobs will no longer be ++ * scheduled. ++ */ ++static inline void kbase_pm_set_gpu_lost(struct kbase_device *kbdev, ++ bool gpu_lost) ++{ ++ atomic_set(&kbdev->pm.gpu_lost, (gpu_lost ? 1 : 0)); ++} ++#endif ++ ++/** ++ * kbase_pm_is_active - Determine whether the GPU is active ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * This takes into account whether there is an active context reference. ++ * ++ * Return: true if the GPU is active, false otherwise ++ */ ++static inline bool kbase_pm_is_active(struct kbase_device *kbdev) ++{ ++ return kbdev->pm.active_count > 0; ++} ++ ++/** ++ * kbase_pm_metrics_start - Start the utilization metrics timer ++ * @kbdev: Pointer to the kbase device for which to start the utilization ++ * metrics calculation thread. ++ * ++ * Start the timer that drives the metrics calculation, runs the custom DVFS. ++ */ ++void kbase_pm_metrics_start(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_metrics_stop - Stop the utilization metrics timer ++ * @kbdev: Pointer to the kbase device for which to stop the utilization ++ * metrics calculation thread. ++ * ++ * Stop the timer that drives the metrics calculation, runs the custom DVFS. ++ */ ++void kbase_pm_metrics_stop(struct kbase_device *kbdev); ++ ++#if !MALI_USE_CSF ++/** ++ * Return the atom's ID, as was originally supplied by userspace in ++ * base_jd_atom::atom_number ++ */ ++static inline int kbase_jd_atom_id(struct kbase_context *kctx, struct kbase_jd_atom *katom) ++{ ++ int result; ++ ++ KBASE_DEBUG_ASSERT(kctx); ++ KBASE_DEBUG_ASSERT(katom); ++ KBASE_DEBUG_ASSERT(katom->kctx == kctx); ++ ++ result = katom - &kctx->jctx.atoms[0]; ++ KBASE_DEBUG_ASSERT(result >= 0 && result <= BASE_JD_ATOM_COUNT); ++ return result; ++} ++ ++/** ++ * kbase_jd_atom_from_id - Return the atom structure for the given atom ID ++ * @kctx: Context pointer ++ * @id: ID of atom to retrieve ++ * ++ * Return: Pointer to struct kbase_jd_atom associated with the supplied ID ++ */ ++static inline struct kbase_jd_atom *kbase_jd_atom_from_id( ++ struct kbase_context *kctx, int id) ++{ ++ return &kctx->jctx.atoms[id]; ++} ++#endif /* !MALI_USE_CSF */ ++ ++/** ++ * Initialize the disjoint state ++ * ++ * The disjoint event count and state are both set to zero. ++ * ++ * Disjoint functions usage: ++ * ++ * The disjoint event count should be incremented whenever a disjoint event occurs. ++ * ++ * There are several cases which are regarded as disjoint behavior. Rather than just increment ++ * the counter during disjoint events we also increment the counter when jobs may be affected ++ * by what the GPU is currently doing. To facilitate this we have the concept of disjoint state. ++ * ++ * Disjoint state is entered during GPU reset. Increasing the disjoint state also increases ++ * the count of disjoint events. ++ * ++ * The disjoint state is then used to increase the count of disjoint events during job submission ++ * and job completion. Any atom submitted or completed while the disjoint state is greater than ++ * zero is regarded as a disjoint event. ++ * ++ * The disjoint event counter is also incremented immediately whenever a job is soft stopped ++ * and during context creation. ++ * ++ * @param kbdev The kbase device ++ * ++ * Return: 0 on success and non-zero value on failure. ++ */ ++void kbase_disjoint_init(struct kbase_device *kbdev); ++ ++/** ++ * Increase the count of disjoint events ++ * called when a disjoint event has happened ++ * ++ * @param kbdev The kbase device ++ */ ++void kbase_disjoint_event(struct kbase_device *kbdev); ++ ++/** ++ * Increase the count of disjoint events only if the GPU is in a disjoint state ++ * ++ * This should be called when something happens which could be disjoint if the GPU ++ * is in a disjoint state. The state refcount keeps track of this. ++ * ++ * @param kbdev The kbase device ++ */ ++void kbase_disjoint_event_potential(struct kbase_device *kbdev); ++ ++/** ++ * Returns the count of disjoint events ++ * ++ * @param kbdev The kbase device ++ * @return the count of disjoint events ++ */ ++u32 kbase_disjoint_event_get(struct kbase_device *kbdev); ++ ++/** ++ * Increment the refcount state indicating that the GPU is in a disjoint state. ++ * ++ * Also Increment the disjoint event count (calls @ref kbase_disjoint_event) ++ * eventually after the disjoint state has completed @ref kbase_disjoint_state_down ++ * should be called ++ * ++ * @param kbdev The kbase device ++ */ ++void kbase_disjoint_state_up(struct kbase_device *kbdev); ++ ++/** ++ * Decrement the refcount state ++ * ++ * Also Increment the disjoint event count (calls @ref kbase_disjoint_event) ++ * ++ * Called after @ref kbase_disjoint_state_up once the disjoint state is over ++ * ++ * @param kbdev The kbase device ++ */ ++void kbase_disjoint_state_down(struct kbase_device *kbdev); ++ ++/** ++ * If a job is soft stopped and the number of contexts is >= this value ++ * it is reported as a disjoint event ++ */ ++#define KBASE_DISJOINT_STATE_INTERLEAVED_CONTEXT_COUNT_THRESHOLD 2 ++ ++#if !defined(UINT64_MAX) ++ #define UINT64_MAX ((uint64_t)0xFFFFFFFFFFFFFFFFULL) ++#endif ++ ++#endif +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_as_fault_debugfs.c b/drivers/gpu/arm/bifrost/mali_kbase_as_fault_debugfs.c +new file mode 100755 +index 000000000000..76bbfffe03a0 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_as_fault_debugfs.c +@@ -0,0 +1,113 @@ ++/* ++ * ++ * (C) COPYRIGHT 2016-2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++ ++#include ++#include ++#include ++ ++#ifdef CONFIG_DEBUG_FS ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++ ++static int kbase_as_fault_read(struct seq_file *sfile, void *data) ++{ ++ uintptr_t as_no = (uintptr_t) sfile->private; ++ ++ struct list_head *entry; ++ const struct list_head *kbdev_list; ++ struct kbase_device *kbdev = NULL; ++ ++ kbdev_list = kbase_device_get_list(); ++ ++ list_for_each(entry, kbdev_list) { ++ kbdev = list_entry(entry, struct kbase_device, entry); ++ ++ if (kbdev->debugfs_as_read_bitmap & (1ULL << as_no)) { ++ ++ /* don't show this one again until another fault occors */ ++ kbdev->debugfs_as_read_bitmap &= ~(1ULL << as_no); ++ ++ /* output the last page fault addr */ ++ seq_printf(sfile, "%llu\n", ++ (u64) kbdev->as[as_no].pf_data.addr); ++ } ++ ++ } ++ ++ kbase_device_put_list(kbdev_list); ++ ++ return 0; ++} ++ ++static int kbase_as_fault_debugfs_open(struct inode *in, struct file *file) ++{ ++ return single_open(file, kbase_as_fault_read, in->i_private); ++} ++ ++static const struct file_operations as_fault_fops = { ++ .owner = THIS_MODULE, ++ .open = kbase_as_fault_debugfs_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = single_release, ++}; ++ ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++#endif /* CONFIG_DEBUG_FS */ ++ ++/* ++ * Initialize debugfs entry for each address space ++ */ ++void kbase_as_fault_debugfs_init(struct kbase_device *kbdev) ++{ ++#ifdef CONFIG_DEBUG_FS ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++ uint i; ++ char as_name[64]; ++ struct dentry *debugfs_directory; ++ ++ kbdev->debugfs_as_read_bitmap = 0ULL; ++ ++ KBASE_DEBUG_ASSERT(kbdev->nr_hw_address_spaces); ++ KBASE_DEBUG_ASSERT(sizeof(kbdev->as[0].pf_data.addr) == sizeof(u64)); ++ ++ debugfs_directory = debugfs_create_dir("address_spaces", ++ kbdev->mali_debugfs_directory); ++ ++ if (debugfs_directory) { ++ for (i = 0; i < kbdev->nr_hw_address_spaces; i++) { ++ snprintf(as_name, ARRAY_SIZE(as_name), "as%u", i); ++ debugfs_create_file(as_name, S_IRUGO, ++ debugfs_directory, ++ (void *)(uintptr_t)i, ++ &as_fault_fops); ++ } ++ } else { ++ dev_warn(kbdev->dev, ++ "unable to create address_spaces debugfs directory"); ++ } ++ ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++#endif /* CONFIG_DEBUG_FS */ ++ return; ++} +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_as_fault_debugfs.h b/drivers/gpu/arm/bifrost/mali_kbase_as_fault_debugfs.h +new file mode 100755 +index 000000000000..58d7fcf030a4 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_as_fault_debugfs.h +@@ -0,0 +1,50 @@ ++/* ++ * ++ * (C) COPYRIGHT 2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_AS_FAULT_DEBUG_FS_H ++#define _KBASE_AS_FAULT_DEBUG_FS_H ++ ++/** ++ * kbase_as_fault_debugfs_init() - Add debugfs files for reporting page faults ++ * ++ * @kbdev: Pointer to kbase_device ++ */ ++void kbase_as_fault_debugfs_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_as_fault_debugfs_new() - make the last fault available on debugfs ++ * ++ * @kbdev: Pointer to kbase_device ++ * @as_no: The address space the fault occurred on ++ */ ++static inline void ++kbase_as_fault_debugfs_new(struct kbase_device *kbdev, int as_no) ++{ ++#ifdef CONFIG_DEBUG_FS ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++ kbdev->debugfs_as_read_bitmap |= (1ULL << as_no); ++#endif /* CONFIG_DEBUG_FS */ ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++ return; ++} ++ ++#endif /*_KBASE_AS_FAULT_DEBUG_FS_H*/ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_bits.h b/drivers/gpu/arm/bifrost/mali_kbase_bits.h +new file mode 100755 +index 000000000000..2c110937a792 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_bits.h +@@ -0,0 +1,41 @@ ++/* ++ * ++ * (C) COPYRIGHT ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ *//* SPDX-License-Identifier: GPL-2.0 */ ++/* ++ * ++ * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ */ ++ ++#ifndef _KBASE_BITS_H_ ++#define _KBASE_BITS_H_ ++ ++#if (KERNEL_VERSION(4, 19, 0) <= LINUX_VERSION_CODE) ++#include ++#else ++#include ++#endif ++ ++#endif /* _KBASE_BITS_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_cache_policy.c b/drivers/gpu/arm/bifrost/mali_kbase_cache_policy.c +new file mode 100755 +index 000000000000..27a03cf02138 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_cache_policy.c +@@ -0,0 +1,67 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2018 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++/* ++ * Cache Policy API. ++ */ ++ ++#include "mali_kbase_cache_policy.h" ++ ++/* ++ * The output flags should be a combination of the following values: ++ * KBASE_REG_CPU_CACHED: CPU cache should be enabled ++ * KBASE_REG_GPU_CACHED: GPU cache should be enabled ++ * ++ * NOTE: Some components within the GPU might only be able to access memory ++ * that is KBASE_REG_GPU_CACHED. Refer to the specific GPU implementation for ++ * more details. ++ */ ++u32 kbase_cache_enabled(u32 flags, u32 nr_pages) ++{ ++ u32 cache_flags = 0; ++ ++ CSTD_UNUSED(nr_pages); ++ ++ if (!(flags & BASE_MEM_UNCACHED_GPU)) ++ cache_flags |= KBASE_REG_GPU_CACHED; ++ ++ if (flags & BASE_MEM_CACHED_CPU) ++ cache_flags |= KBASE_REG_CPU_CACHED; ++ ++ return cache_flags; ++} ++ ++ ++void kbase_sync_single_for_device(struct kbase_device *kbdev, dma_addr_t handle, ++ size_t size, enum dma_data_direction dir) ++{ ++ dma_sync_single_for_device(kbdev->dev, handle, size, dir); ++} ++ ++ ++void kbase_sync_single_for_cpu(struct kbase_device *kbdev, dma_addr_t handle, ++ size_t size, enum dma_data_direction dir) ++{ ++ dma_sync_single_for_cpu(kbdev->dev, handle, size, dir); ++} +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_cache_policy.h b/drivers/gpu/arm/bifrost/mali_kbase_cache_policy.h +new file mode 100755 +index 000000000000..8a1e5291bf5f +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_cache_policy.h +@@ -0,0 +1,50 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2013, 2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++/* ++ * Cache Policy API. ++ */ ++ ++#ifndef _KBASE_CACHE_POLICY_H_ ++#define _KBASE_CACHE_POLICY_H_ ++ ++#include "mali_kbase.h" ++#include "mali_base_kernel.h" ++ ++/** ++ * kbase_cache_enabled - Choose the cache policy for a specific region ++ * @flags: flags describing attributes of the region ++ * @nr_pages: total number of pages (backed or not) for the region ++ * ++ * Tells whether the CPU and GPU caches should be enabled or not for a specific ++ * region. ++ * This function can be modified to customize the cache policy depending on the ++ * flags and size of the region. ++ * ++ * Return: a combination of %KBASE_REG_CPU_CACHED and %KBASE_REG_GPU_CACHED ++ * depending on the cache policy ++ */ ++u32 kbase_cache_enabled(u32 flags, u32 nr_pages); ++ ++#endif /* _KBASE_CACHE_POLICY_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_caps.h b/drivers/gpu/arm/bifrost/mali_kbase_caps.h +new file mode 100755 +index 000000000000..b201a60fa6e3 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_caps.h +@@ -0,0 +1,65 @@ ++/* ++ * ++ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++/** ++ * @file mali_kbase_caps.h ++ * ++ * Driver Capability Queries. ++ */ ++ ++#ifndef _KBASE_CAPS_H_ ++#define _KBASE_CAPS_H_ ++ ++#include ++ ++typedef enum mali_kbase_cap { ++ MALI_KBASE_CAP_SYSTEM_MONITOR = 0, ++ MALI_KBASE_CAP_JIT_PRESSURE_LIMIT, ++ MALI_KBASE_CAP_MEM_GROW_ON_GPF, ++ MALI_KBASE_CAP_MEM_PROTECTED, ++ MALI_KBASE_NUM_CAPS ++} mali_kbase_cap; ++ ++extern bool mali_kbase_supports_cap(unsigned long api_version, mali_kbase_cap cap); ++ ++static inline bool mali_kbase_supports_system_monitor(unsigned long api_version) ++{ ++ return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_SYSTEM_MONITOR); ++} ++ ++static inline bool mali_kbase_supports_jit_pressure_limit(unsigned long api_version) ++{ ++ return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_JIT_PRESSURE_LIMIT); ++} ++ ++static inline bool mali_kbase_supports_mem_grow_on_gpf(unsigned long api_version) ++{ ++ return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_MEM_GROW_ON_GPF); ++} ++ ++static inline bool mali_kbase_supports_mem_protected(unsigned long api_version) ++{ ++ return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_MEM_PROTECTED); ++} ++ ++#endif /* __KBASE_CAPS_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_ccswe.c b/drivers/gpu/arm/bifrost/mali_kbase_ccswe.c +new file mode 100755 +index 000000000000..87d5aaa6bb5d +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_ccswe.c +@@ -0,0 +1,105 @@ ++/* ++ * ++ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include "mali_kbase_ccswe.h" ++#include "mali_kbase_linux.h" ++ ++#include ++#include ++ ++static u64 kbasep_ccswe_cycle_at_no_lock( ++ struct kbase_ccswe *self, u64 timestamp_ns) ++{ ++ s64 diff_s, diff_ns; ++ u32 gpu_freq; ++ ++ lockdep_assert_held(&self->access); ++ ++ diff_ns = timestamp_ns - self->timestamp_ns; ++ gpu_freq = diff_ns > 0 ? self->gpu_freq : self->prev_gpu_freq; ++ ++ diff_s = div_s64(diff_ns, NSEC_PER_SEC); ++ diff_ns -= diff_s * NSEC_PER_SEC; ++ ++ return self->cycles_elapsed + diff_s * gpu_freq ++ + div_s64(diff_ns * gpu_freq, NSEC_PER_SEC); ++} ++ ++void kbase_ccswe_init(struct kbase_ccswe *self) ++{ ++ memset(self, 0, sizeof(*self)); ++ ++ spin_lock_init(&self->access); ++} ++KBASE_EXPORT_TEST_API(kbase_ccswe_init); ++ ++u64 kbase_ccswe_cycle_at(struct kbase_ccswe *self, u64 timestamp_ns) ++{ ++ unsigned long flags; ++ u64 result; ++ ++ spin_lock_irqsave(&self->access, flags); ++ result = kbasep_ccswe_cycle_at_no_lock(self, timestamp_ns); ++ spin_unlock_irqrestore(&self->access, flags); ++ ++ return result; ++} ++KBASE_EXPORT_TEST_API(kbase_ccswe_cycle_at); ++ ++void kbase_ccswe_freq_change( ++ struct kbase_ccswe *self, u64 timestamp_ns, u32 gpu_freq) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&self->access, flags); ++ ++ /* The time must go only forward. */ ++ if (WARN_ON(timestamp_ns < self->timestamp_ns)) ++ goto exit; ++ ++ /* If this is the first frequency change, cycles_elapsed is zero. */ ++ if (self->timestamp_ns) ++ self->cycles_elapsed = kbasep_ccswe_cycle_at_no_lock( ++ self, timestamp_ns); ++ ++ self->timestamp_ns = timestamp_ns; ++ self->prev_gpu_freq = self->gpu_freq; ++ self->gpu_freq = gpu_freq; ++exit: ++ spin_unlock_irqrestore(&self->access, flags); ++} ++KBASE_EXPORT_TEST_API(kbase_ccswe_freq_change); ++ ++void kbase_ccswe_reset(struct kbase_ccswe *self) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&self->access, flags); ++ ++ self->timestamp_ns = 0; ++ self->cycles_elapsed = 0; ++ self->gpu_freq = 0; ++ self->prev_gpu_freq = 0; ++ ++ spin_unlock_irqrestore(&self->access, flags); ++} ++ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_ccswe.h b/drivers/gpu/arm/bifrost/mali_kbase_ccswe.h +new file mode 100755 +index 000000000000..3a7cf73d9eac +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_ccswe.h +@@ -0,0 +1,97 @@ ++/* ++ * ++ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_CCSWE_H_ ++#define _KBASE_CCSWE_H_ ++ ++#include ++ ++/** ++ * struct kbase_ccswe - Cycle count software estimator. ++ * ++ * @access: Spinlock protecting this structure access. ++ * @timestamp_ns: Timestamp(ns) when the last frequency change ++ * occurred. ++ * @cycles_elapsed: Number of cycles elapsed before the last frequency ++ * change ++ * @gpu_freq: Current GPU frequency(Hz) value. ++ * @prev_gpu_freq: Previous GPU frequency(Hz) before the last frequency ++ * change. ++ */ ++struct kbase_ccswe { ++ spinlock_t access; ++ u64 timestamp_ns; ++ u64 cycles_elapsed; ++ u32 gpu_freq; ++ u32 prev_gpu_freq; ++}; ++ ++/** ++ * kbase_ccswe_init() - initialize the cycle count estimator. ++ * ++ * @self: Cycles count software estimator instance. ++ */ ++void kbase_ccswe_init(struct kbase_ccswe *self); ++ ++ ++/** ++ * kbase_ccswe_cycle_at() - Estimate cycle count at given timestamp. ++ * ++ * @self: Cycles count software estimator instance. ++ * @timestamp_ns: The timestamp(ns) for cycle count estimation. ++ * ++ * The timestamp must be bigger than the timestamp of the penultimate ++ * frequency change. If only one frequency change occurred, the ++ * timestamp must be bigger than the timestamp of the frequency change. ++ * This is to allow the following code to be executed w/o synchronization. ++ * If lines below executed atomically, it is safe to assume that only ++ * one frequency change may happen in between. ++ * ++ * u64 ts = ktime_get_raw_ns(); ++ * u64 cycle = kbase_ccswe_cycle_at(&ccswe, ts) ++ * ++ * Returns: estimated value of cycle count at a given time. ++ */ ++u64 kbase_ccswe_cycle_at(struct kbase_ccswe *self, u64 timestamp_ns); ++ ++/** ++ * kbase_ccswe_freq_change() - update GPU frequency. ++ * ++ * @self: Cycles count software estimator instance. ++ * @timestamp_ns: Timestamp(ns) when frequency change occurred. ++ * @gpu_freq: New GPU frequency value. ++ * ++ * The timestamp must be bigger than the timestamp of the previous ++ * frequency change. The function is to be called at the frequency ++ * change moment (not later). ++ */ ++void kbase_ccswe_freq_change( ++ struct kbase_ccswe *self, u64 timestamp_ns, u32 gpu_freq); ++ ++/** ++ * kbase_ccswe_reset() - reset estimator state ++ * ++ * @self: Cycles count software estimator instance. ++ */ ++void kbase_ccswe_reset(struct kbase_ccswe *self); ++ ++#endif /* _KBASE_CCSWE_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_config.c b/drivers/gpu/arm/bifrost/mali_kbase_config.c +new file mode 100755 +index 000000000000..ce7070d1d634 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_config.c +@@ -0,0 +1,48 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2015,2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++#include ++#include ++#include ++ ++int kbasep_platform_device_init(struct kbase_device *kbdev) ++{ ++ struct kbase_platform_funcs_conf *platform_funcs_p; ++ ++ platform_funcs_p = (struct kbase_platform_funcs_conf *)PLATFORM_FUNCS; ++ if (platform_funcs_p && platform_funcs_p->platform_init_func) ++ return platform_funcs_p->platform_init_func(kbdev); ++ ++ return 0; ++} ++ ++void kbasep_platform_device_term(struct kbase_device *kbdev) ++{ ++ struct kbase_platform_funcs_conf *platform_funcs_p; ++ ++ platform_funcs_p = (struct kbase_platform_funcs_conf *)PLATFORM_FUNCS; ++ if (platform_funcs_p && platform_funcs_p->platform_term_func) ++ platform_funcs_p->platform_term_func(kbdev); ++} ++ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_config.h b/drivers/gpu/arm/bifrost/mali_kbase_config.h +new file mode 100755 +index 000000000000..57456e2b90db +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_config.h +@@ -0,0 +1,393 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2017, 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++/** ++ * @file mali_kbase_config.h ++ * Configuration API and Attributes for KBase ++ */ ++ ++#ifndef _KBASE_CONFIG_H_ ++#define _KBASE_CONFIG_H_ ++ ++#include ++#include ++#include ++#include ++ ++/** ++ * @addtogroup base_api ++ * @{ ++ */ ++ ++/** ++ * @addtogroup base_kbase_api ++ * @{ ++ */ ++ ++/** ++ * @addtogroup kbase_config Configuration API and Attributes ++ * @{ ++ */ ++ ++/* Forward declaration of struct kbase_device */ ++struct kbase_device; ++ ++/** ++ * kbase_platform_funcs_conf - Specifies platform init/term function pointers ++ * ++ * Specifies the functions pointers for platform specific initialization and ++ * termination. By default no functions are required. No additional platform ++ * specific control is necessary. ++ */ ++struct kbase_platform_funcs_conf { ++ /** ++ * platform_init_func - platform specific init function pointer ++ * @kbdev - kbase_device pointer ++ * ++ * Returns 0 on success, negative error code otherwise. ++ * ++ * Function pointer for platform specific initialization or NULL if no ++ * initialization function is required. At the point this the GPU is ++ * not active and its power and clocks are in unknown (platform specific ++ * state) as kbase doesn't yet have control of power and clocks. ++ * ++ * The platform specific private pointer kbase_device::platform_context ++ * can be accessed (and possibly initialized) in here. ++ */ ++ int (*platform_init_func)(struct kbase_device *kbdev); ++ /** ++ * platform_term_func - platform specific termination function pointer ++ * @kbdev - kbase_device pointer ++ * ++ * Function pointer for platform specific termination or NULL if no ++ * termination function is required. At the point this the GPU will be ++ * idle but still powered and clocked. ++ * ++ * The platform specific private pointer kbase_device::platform_context ++ * can be accessed (and possibly terminated) in here. ++ */ ++ void (*platform_term_func)(struct kbase_device *kbdev); ++}; ++ ++/* ++ * @brief Specifies the callbacks for power management ++ * ++ * By default no callbacks will be made and the GPU must not be powered off. ++ */ ++struct kbase_pm_callback_conf { ++ /** Callback for when the GPU is idle and the power to it can be switched off. ++ * ++ * The system integrator can decide whether to either do nothing, just switch off ++ * the clocks to the GPU, or to completely power down the GPU. ++ * The platform specific private pointer kbase_device::platform_context can be accessed and modified in here. It is the ++ * platform \em callbacks responsibility to initialize and terminate this pointer if used (see @ref kbase_platform_funcs_conf). ++ */ ++ void (*power_off_callback)(struct kbase_device *kbdev); ++ ++ /** Callback for when the GPU is about to become active and power must be supplied. ++ * ++ * This function must not return until the GPU is powered and clocked sufficiently for register access to ++ * succeed. The return value specifies whether the GPU was powered down since the call to power_off_callback. ++ * If the GPU state has been lost then this function must return 1, otherwise it should return 0. ++ * The platform specific private pointer kbase_device::platform_context can be accessed and modified in here. It is the ++ * platform \em callbacks responsibility to initialize and terminate this pointer if used (see @ref kbase_platform_funcs_conf). ++ * ++ * The return value of the first call to this function is ignored. ++ * ++ * @return 1 if the GPU state may have been lost, 0 otherwise. ++ */ ++ int (*power_on_callback)(struct kbase_device *kbdev); ++ ++ /** Callback for when the system is requesting a suspend and GPU power ++ * must be switched off. ++ * ++ * Note that if this callback is present, then this may be called ++ * without a preceding call to power_off_callback. Therefore this ++ * callback must be able to take any action that might otherwise happen ++ * in power_off_callback. ++ * ++ * The platform specific private pointer kbase_device::platform_context ++ * can be accessed and modified in here. It is the platform \em ++ * callbacks responsibility to initialize and terminate this pointer if ++ * used (see @ref kbase_platform_funcs_conf). ++ */ ++ void (*power_suspend_callback)(struct kbase_device *kbdev); ++ ++ /** Callback for when the system is resuming from a suspend and GPU ++ * power must be switched on. ++ * ++ * Note that if this callback is present, then this may be called ++ * without a following call to power_on_callback. Therefore this ++ * callback must be able to take any action that might otherwise happen ++ * in power_on_callback. ++ * ++ * The platform specific private pointer kbase_device::platform_context ++ * can be accessed and modified in here. It is the platform \em ++ * callbacks responsibility to initialize and terminate this pointer if ++ * used (see @ref kbase_platform_funcs_conf). ++ */ ++ void (*power_resume_callback)(struct kbase_device *kbdev); ++ ++ /** Callback for handling runtime power management initialization. ++ * ++ * The runtime power management callbacks @ref power_runtime_off_callback and @ref power_runtime_on_callback ++ * will become active from calls made to the OS from within this function. ++ * The runtime calls can be triggered by calls from @ref power_off_callback and @ref power_on_callback. ++ * Note: for linux the kernel must have CONFIG_PM_RUNTIME enabled to use this feature. ++ * ++ * @return 0 on success, else int error code. ++ */ ++ int (*power_runtime_init_callback)(struct kbase_device *kbdev); ++ ++ /** Callback for handling runtime power management termination. ++ * ++ * The runtime power management callbacks @ref power_runtime_off_callback and @ref power_runtime_on_callback ++ * should no longer be called by the OS on completion of this function. ++ * Note: for linux the kernel must have CONFIG_PM_RUNTIME enabled to use this feature. ++ */ ++ void (*power_runtime_term_callback)(struct kbase_device *kbdev); ++ ++ /** Callback for runtime power-off power management callback ++ * ++ * For linux this callback will be called by the kernel runtime_suspend callback. ++ * Note: for linux the kernel must have CONFIG_PM_RUNTIME enabled to use this feature. ++ * ++ * @return 0 on success, else OS error code. ++ */ ++ void (*power_runtime_off_callback)(struct kbase_device *kbdev); ++ ++ /** Callback for runtime power-on power management callback ++ * ++ * For linux this callback will be called by the kernel runtime_resume callback. ++ * Note: for linux the kernel must have CONFIG_PM_RUNTIME enabled to use this feature. ++ */ ++ int (*power_runtime_on_callback)(struct kbase_device *kbdev); ++ ++ /* ++ * Optional callback for checking if GPU can be suspended when idle ++ * ++ * This callback will be called by the runtime power management core ++ * when the reference count goes to 0 to provide notification that the ++ * GPU now seems idle. ++ * ++ * If this callback finds that the GPU can't be powered off, or handles ++ * suspend by powering off directly or queueing up a power off, a ++ * non-zero value must be returned to prevent the runtime PM core from ++ * also triggering a suspend. ++ * ++ * Returning 0 will cause the runtime PM core to conduct a regular ++ * autosuspend. ++ * ++ * This callback is optional and if not provided regular autosuspend ++ * will be triggered. ++ * ++ * Note: The Linux kernel must have CONFIG_PM_RUNTIME enabled to use ++ * this feature. ++ * ++ * Return 0 if GPU can be suspended, positive value if it can not be ++ * suspeneded by runtime PM, else OS error code ++ */ ++ int (*power_runtime_idle_callback)(struct kbase_device *kbdev); ++ ++ /* ++ * Optional callback for software reset ++ * ++ * This callback will be called by the power management core to trigger ++ * a GPU soft reset. ++ * ++ * Return 0 if the soft reset was successful and the RESET_COMPLETED ++ * interrupt will be raised, or a positive value if the interrupt won't ++ * be raised. On error, return the corresponding OS error code. ++ */ ++ int (*soft_reset_callback)(struct kbase_device *kbdev); ++}; ++ ++/* struct kbase_gpu_clk_notifier_data - Data for clock rate change notifier. ++ * ++ * Pointer to this structure is supposed to be passed to the gpu clock rate ++ * change notifier function. This structure is deliberately aligned with the ++ * common clock framework notification structure 'struct clk_notifier_data' ++ * and such alignment should be maintained. ++ * ++ * @gpu_clk_handle: Handle of the GPU clock for which notifier was registered. ++ * @old_rate: Previous rate of this GPU clock. ++ * @new_rate: New rate of this GPU clock. ++ */ ++struct kbase_gpu_clk_notifier_data { ++ void *gpu_clk_handle; ++ unsigned long old_rate; ++ unsigned long new_rate; ++}; ++ ++/** ++ * kbase_clk_rate_trace_op_conf - Specifies GPU clock rate trace operations. ++ * ++ * Specifies the functions pointers for platform specific GPU clock rate trace ++ * operations. By default no functions are required. ++ */ ++struct kbase_clk_rate_trace_op_conf { ++ /** ++ * enumerate_gpu_clk - Enumerate a GPU clock on the given index ++ * @kbdev - kbase_device pointer ++ * @index - GPU clock index ++ * ++ * Returns a handle unique to the given GPU clock, or NULL if the clock ++ * array has been exhausted at the given index value. ++ * ++ * Kbase will use this function pointer to enumerate the existence of a ++ * GPU clock on the given index. ++ */ ++ void *(*enumerate_gpu_clk)(struct kbase_device *kbdev, ++ unsigned int index); ++ ++ /** ++ * get_gpu_clk_rate - Get the current rate for an enumerated clock. ++ * @kbdev - kbase_device pointer ++ * @gpu_clk_handle - Handle unique to the enumerated GPU clock ++ * ++ * Returns current rate of the GPU clock in unit of Hz. ++ */ ++ unsigned long (*get_gpu_clk_rate)(struct kbase_device *kbdev, ++ void *gpu_clk_handle); ++ ++ /** ++ * gpu_clk_notifier_register - Register a clock rate change notifier. ++ * @kbdev - kbase_device pointer ++ * @gpu_clk_handle - Handle unique to the enumerated GPU clock ++ * @nb - notifier block containing the callback function ++ * pointer ++ * ++ * Returns 0 on success, negative error code otherwise. ++ * ++ * This function pointer is used to register a callback function that ++ * is supposed to be invoked whenever the rate of clock corresponding ++ * to @gpu_clk_handle changes. ++ * @nb contains the pointer to callback function. ++ * The callback function expects the pointer of type ++ * 'struct kbase_gpu_clk_notifier_data' as the third argument. ++ */ ++ int (*gpu_clk_notifier_register)(struct kbase_device *kbdev, ++ void *gpu_clk_handle, struct notifier_block *nb); ++ ++ /** ++ * gpu_clk_notifier_unregister - Unregister clock rate change notifier ++ * @kbdev - kbase_device pointer ++ * @gpu_clk_handle - Handle unique to the enumerated GPU clock ++ * @nb - notifier block containing the callback function ++ * pointer ++ * ++ * This function pointer is used to unregister a callback function that ++ * was previously registered to get notified of the change in rate ++ * of clock corresponding to @gpu_clk_handle. ++ */ ++ void (*gpu_clk_notifier_unregister)(struct kbase_device *kbdev, ++ void *gpu_clk_handle, struct notifier_block *nb); ++}; ++ ++#ifdef CONFIG_OF ++struct kbase_platform_config { ++}; ++#else ++ ++/* ++ * @brief Specifies start and end of I/O memory region. ++ */ ++struct kbase_io_memory_region { ++ u64 start; ++ u64 end; ++}; ++ ++/* ++ * @brief Specifies I/O related resources like IRQs and memory region for I/O operations. ++ */ ++struct kbase_io_resources { ++ u32 job_irq_number; ++ u32 mmu_irq_number; ++ u32 gpu_irq_number; ++ struct kbase_io_memory_region io_memory_region; ++}; ++ ++struct kbase_platform_config { ++ const struct kbase_io_resources *io_resources; ++}; ++ ++#endif /* CONFIG_OF */ ++ ++/** ++ * @brief Gets the pointer to platform config. ++ * ++ * @return Pointer to the platform config ++ */ ++struct kbase_platform_config *kbase_get_platform_config(void); ++ ++/** ++ * kbasep_platform_device_init: - Platform specific call to initialize hardware ++ * @kbdev: kbase device pointer ++ * ++ * Function calls a platform defined routine if specified in the configuration ++ * attributes. The routine can initialize any hardware and context state that ++ * is required for the GPU block to function. ++ * ++ * Return: 0 if no errors have been found in the config. ++ * Negative error code otherwise. ++ */ ++int kbasep_platform_device_init(struct kbase_device *kbdev); ++ ++/** ++ * kbasep_platform_device_term - Platform specific call to terminate hardware ++ * @kbdev: Kbase device pointer ++ * ++ * Function calls a platform defined routine if specified in the configuration ++ * attributes. The routine can destroy any platform specific context state and ++ * shut down any hardware functionality that are outside of the Power Management ++ * callbacks. ++ * ++ */ ++void kbasep_platform_device_term(struct kbase_device *kbdev); ++ ++#ifndef CONFIG_OF ++/** ++ * kbase_platform_register - Register a platform device for the GPU ++ * ++ * This can be used to register a platform device on systems where device tree ++ * is not enabled and the platform initialisation code in the kernel doesn't ++ * create the GPU device. Where possible device tree should be used instead. ++ * ++ * Return: 0 for success, any other fail causes module initialisation to fail ++ */ ++int kbase_platform_register(void); ++ ++/** ++ * kbase_platform_unregister - Unregister a fake platform device ++ * ++ * Unregister the platform device created with kbase_platform_register() ++ */ ++void kbase_platform_unregister(void); ++#endif ++ ++ /** @} *//* end group kbase_config */ ++ /** @} *//* end group base_kbase_api */ ++ /** @} *//* end group base_api */ ++ ++#endif /* _KBASE_CONFIG_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_config_defaults.h b/drivers/gpu/arm/bifrost/mali_kbase_config_defaults.h +new file mode 100755 +index 000000000000..e079281127ab +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_config_defaults.h +@@ -0,0 +1,213 @@ ++/* ++ * ++ * (C) COPYRIGHT 2013-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/** ++ * @file mali_kbase_config_defaults.h ++ * ++ * Default values for configuration settings ++ * ++ */ ++ ++#ifndef _KBASE_CONFIG_DEFAULTS_H_ ++#define _KBASE_CONFIG_DEFAULTS_H_ ++ ++/* Include mandatory definitions per platform */ ++#include ++ ++enum { ++ /** ++ * Use unrestricted Address ID width on the AXI bus. ++ */ ++ KBASE_AID_32 = 0x0, ++ ++ /** ++ * Restrict GPU to a half of maximum Address ID count. ++ * This will reduce performance, but reduce bus load due to GPU. ++ */ ++ KBASE_AID_16 = 0x3, ++ ++ /** ++ * Restrict GPU to a quarter of maximum Address ID count. ++ * This will reduce performance, but reduce bus load due to GPU. ++ */ ++ KBASE_AID_8 = 0x2, ++ ++ /** ++ * Restrict GPU to an eighth of maximum Address ID count. ++ * This will reduce performance, but reduce bus load due to GPU. ++ */ ++ KBASE_AID_4 = 0x1 ++}; ++ ++enum { ++ /** ++ * Use unrestricted Address ID width on the AXI bus. ++ * Restricting ID width will reduce performance & bus load due to GPU. ++ */ ++ KBASE_3BIT_AID_32 = 0x0, ++ ++ /* Restrict GPU to 7/8 of maximum Address ID count. */ ++ KBASE_3BIT_AID_28 = 0x1, ++ ++ /* Restrict GPU to 3/4 of maximum Address ID count. */ ++ KBASE_3BIT_AID_24 = 0x2, ++ ++ /* Restrict GPU to 5/8 of maximum Address ID count. */ ++ KBASE_3BIT_AID_20 = 0x3, ++ ++ /* Restrict GPU to 1/2 of maximum Address ID count. */ ++ KBASE_3BIT_AID_16 = 0x4, ++ ++ /* Restrict GPU to 3/8 of maximum Address ID count. */ ++ KBASE_3BIT_AID_12 = 0x5, ++ ++ /* Restrict GPU to 1/4 of maximum Address ID count. */ ++ KBASE_3BIT_AID_8 = 0x6, ++ ++ /* Restrict GPU to 1/8 of maximum Address ID count. */ ++ KBASE_3BIT_AID_4 = 0x7 ++}; ++ ++/** ++ * Default period for DVFS sampling ++ */ ++#define DEFAULT_PM_DVFS_PERIOD 100 /* 100ms */ ++ ++/** ++ * Power Management poweroff tick granuality. This is in nanoseconds to ++ * allow HR timer support. ++ * ++ * On each scheduling tick, the power manager core may decide to: ++ * -# Power off one or more shader cores ++ * -# Power off the entire GPU ++ */ ++#define DEFAULT_PM_GPU_POWEROFF_TICK_NS (400000) /* 400us */ ++ ++/** ++ * Power Manager number of ticks before shader cores are powered off ++ */ ++#define DEFAULT_PM_POWEROFF_TICK_SHADER (2) /* 400-800us */ ++ ++/** ++ * Default scheduling tick granuality ++ */ ++#define DEFAULT_JS_SCHEDULING_PERIOD_NS (100000000u) /* 100ms */ ++ ++/** ++ * Default minimum number of scheduling ticks before jobs are soft-stopped. ++ * ++ * This defines the time-slice for a job (which may be different from that of a ++ * context) ++ */ ++#define DEFAULT_JS_SOFT_STOP_TICKS (1) /* 100ms-200ms */ ++ ++/** ++ * Default minimum number of scheduling ticks before CL jobs are soft-stopped. ++ */ ++#define DEFAULT_JS_SOFT_STOP_TICKS_CL (1) /* 100ms-200ms */ ++ ++/** ++ * Default minimum number of scheduling ticks before jobs are hard-stopped ++ */ ++#define DEFAULT_JS_HARD_STOP_TICKS_SS (50) /* 5s */ ++ ++/** ++ * Default minimum number of scheduling ticks before CL jobs are hard-stopped. ++ */ ++#define DEFAULT_JS_HARD_STOP_TICKS_CL (50) /* 5s */ ++ ++/** ++ * Default minimum number of scheduling ticks before jobs are hard-stopped ++ * during dumping ++ */ ++#define DEFAULT_JS_HARD_STOP_TICKS_DUMPING (15000) /* 1500s */ ++ ++/** ++ * Default timeout for some software jobs, after which the software event wait ++ * jobs will be cancelled. ++ */ ++#define DEFAULT_JS_SOFT_JOB_TIMEOUT (3000) /* 3s */ ++ ++/** ++ * Default minimum number of scheduling ticks before the GPU is reset to clear a ++ * "stuck" job ++ */ ++#define DEFAULT_JS_RESET_TICKS_SS (55) /* 5.5s */ ++ ++/** ++ * Default minimum number of scheduling ticks before the GPU is reset to clear a ++ * "stuck" CL job. ++ */ ++#define DEFAULT_JS_RESET_TICKS_CL (55) /* 5.5s */ ++ ++/** ++ * Default minimum number of scheduling ticks before the GPU is reset to clear a ++ * "stuck" job during dumping. ++ */ ++#define DEFAULT_JS_RESET_TICKS_DUMPING (15020) /* 1502s */ ++ ++/** ++ * Default number of milliseconds given for other jobs on the GPU to be ++ * soft-stopped when the GPU needs to be reset. ++ */ ++#define DEFAULT_RESET_TIMEOUT_MS (3000) /* 3s */ ++ ++/** ++ * Default timeslice that a context is scheduled in for, in nanoseconds. ++ * ++ * When a context has used up this amount of time across its jobs, it is ++ * scheduled out to let another run. ++ * ++ * @note the resolution is nanoseconds (ns) here, because that's the format ++ * often used by the OS. ++ */ ++#define DEFAULT_JS_CTX_TIMESLICE_NS (50000000) /* 50ms */ ++ ++/** ++ * Maximum frequency (in kHz) that the GPU can be clocked. For some platforms ++ * this isn't available, so we simply define a dummy value here. If devfreq ++ * is enabled the value will be read from there, otherwise this should be ++ * overridden by defining GPU_FREQ_KHZ_MAX in the platform file. ++ */ ++#define DEFAULT_GPU_FREQ_KHZ_MAX (5000) ++ ++/** ++ * Default timeout for task execution on an endpoint ++ * ++ * Number of GPU clock cycles before the driver terminates a task that is ++ * making no forward progress on an endpoint (e.g. shader core). ++ * Value chosen is equivalent to the time after which a job is hard stopped ++ * which is 5 seconds (assuming the GPU is usually clocked at ~500 MHZ). ++ */ ++#define DEFAULT_PROGRESS_TIMEOUT ((u64)5 * 500 * 1024 * 1024) ++ ++/** ++ * Default threshold at which to switch to incremental rendering ++ * ++ * Fraction of the maximum size of an allocation that grows on GPU page fault ++ * that can be used up before the driver switches to incremental rendering, ++ * in 256ths. 0 means disable incremental rendering. ++ */ ++#define DEFAULT_IR_THRESHOLD (192) ++ ++#endif /* _KBASE_CONFIG_DEFAULTS_H_ */ ++ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_core_linux.c b/drivers/gpu/arm/bifrost/mali_kbase_core_linux.c +new file mode 100755 +index 000000000000..071b9236dee0 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_core_linux.c +@@ -0,0 +1,5001 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#ifdef CONFIG_MALI_BIFROST_DEVFREQ ++#include ++#include ++#ifdef CONFIG_DEVFREQ_THERMAL ++#include ++#endif /* CONFIG_DEVFREQ_THERMAL */ ++#endif /* CONFIG_MALI_BIFROST_DEVFREQ */ ++#ifdef CONFIG_MALI_BIFROST_NO_MALI ++#include "mali_kbase_model_linux.h" ++#include ++#endif /* CONFIG_MALI_BIFROST_NO_MALI */ ++#include "mali_kbase_mem_profile_debugfs_buf_size.h" ++#include "mali_kbase_debug_mem_view.h" ++#include "mali_kbase_mem.h" ++#include "mali_kbase_mem_pool_debugfs.h" ++#include "mali_kbase_debugfs_helper.h" ++#if !MALI_CUSTOMER_RELEASE ++#include "mali_kbase_regs_dump_debugfs.h" ++#endif /* !MALI_CUSTOMER_RELEASE */ ++#include "mali_kbase_regs_history_debugfs.h" ++#include ++#include ++#if !MALI_USE_CSF ++#include ++#endif /* !MALI_USE_CSF */ ++#ifdef CONFIG_MALI_PRFCNT_SET_SECONDARY_VIA_DEBUG_FS ++#include ++#endif ++#include ++#include ++#include "mali_kbase_ioctl.h" ++#if !MALI_USE_CSF ++#include "mali_kbase_kinstr_jm.h" ++#endif ++#include "mali_kbase_hwcnt_context.h" ++#include "mali_kbase_hwcnt_virtualizer.h" ++#include "mali_kbase_hwcnt_legacy.h" ++#include "mali_kbase_vinstr.h" ++#if MALI_USE_CSF ++#include "csf/mali_kbase_csf_firmware.h" ++#include "csf/mali_kbase_csf_tiler_heap.h" ++#include "csf/mali_kbase_csf_kcpu_debugfs.h" ++#include "csf/mali_kbase_csf_csg_debugfs.h" ++#endif ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++#include "arbiter/mali_kbase_arbiter_pm.h" ++#endif ++ ++#include "mali_kbase_cs_experimental.h" ++ ++#ifdef CONFIG_MALI_CINSTR_GWT ++#include "mali_kbase_gwt.h" ++#endif ++#include "mali_kbase_pm_internal.h" ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include /* is_compat_task/in_compat_syscall */ ++#include ++#include ++#include ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++#include ++#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ ++#include ++#include ++#include ++#include ++ ++#include ++ ++ ++#if (KERNEL_VERSION(3, 13, 0) <= LINUX_VERSION_CODE) ++#include ++#include ++#else ++#include ++#endif ++ ++#include ++ ++#include ++ ++#include ++#include ++#include ++ ++#include ++ ++/* GPU IRQ Tags */ ++#define JOB_IRQ_TAG 0 ++#define MMU_IRQ_TAG 1 ++#define GPU_IRQ_TAG 2 ++ ++#define KERNEL_SIDE_DDK_VERSION_STRING "K:" MALI_RELEASE_NAME "(GPL)" ++ ++/** ++ * Kernel min/maj <=> API Version ++ */ ++#define KBASE_API_VERSION(major, minor) ((((major) & 0xFFF) << 20) | \ ++ (((minor) & 0xFFF) << 8) | \ ++ ((0 & 0xFF) << 0)) ++ ++#define KBASE_API_MIN(api_version) ((api_version >> 8) & 0xFFF) ++#define KBASE_API_MAJ(api_version) ((api_version >> 20) & 0xFFF) ++ ++/** ++ * mali_kbase_api_version_to_maj_min - convert an api_version to a min/maj pair ++ * ++ * @api_version: API version to convert ++ * @major: Major version number (must not exceed 12 bits) ++ * @minor: Major version number (must not exceed 12 bits) ++ */ ++void mali_kbase_api_version_to_maj_min(unsigned long api_version, u16 *maj, u16 *min) ++{ ++ if (WARN_ON(!maj)) ++ return; ++ ++ if (WARN_ON(!min)) ++ return; ++ ++ *maj = KBASE_API_MAJ(api_version); ++ *min = KBASE_API_MIN(api_version); ++} ++ ++/** ++ * kbase capabilities table ++ */ ++typedef struct mali_kbase_capability_def { ++ u16 required_major; ++ u16 required_minor; ++} mali_kbase_capability_def; ++ ++/** ++ * This must be kept in-sync with mali_kbase_cap ++ * ++ * TODO: The alternative approach would be to embed the cap enum values ++ * in the table. Less efficient but potentially safer. ++ */ ++static mali_kbase_capability_def kbase_caps_table[MALI_KBASE_NUM_CAPS] = { ++#if MALI_USE_CSF ++ { 1, 0 }, /* SYSTEM_MONITOR */ ++ { 1, 0 }, /* JIT_PRESSURE_LIMIT */ ++ { 1, 0 }, /* MEM_GROW_ON_GPF */ ++ { 1, 0 } /* MEM_PROTECTED */ ++#else ++ { 11, 15 }, /* SYSTEM_MONITOR */ ++ { 11, 25 }, /* JIT_PRESSURE_LIMIT */ ++ { 11, 2 }, /* MEM_GROW_ON_GPF */ ++ { 11, 2 } /* MEM_PROTECTED */ ++#endif ++}; ++ ++/** ++ * mali_kbase_supports_cap - Query whether a kbase capability is supported ++ * ++ * @api_version: API version to convert ++ * @cap: Capability to query for - see mali_kbase_caps.h ++ */ ++bool mali_kbase_supports_cap(unsigned long api_version, mali_kbase_cap cap) ++{ ++ bool supported = false; ++ unsigned long required_ver; ++ ++ mali_kbase_capability_def const *cap_def; ++ ++ if (WARN_ON(cap < 0)) ++ return false; ++ ++ if (WARN_ON(cap >= MALI_KBASE_NUM_CAPS)) ++ return false; ++ ++ cap_def = &kbase_caps_table[(int)cap]; ++ required_ver = KBASE_API_VERSION(cap_def->required_major, cap_def->required_minor); ++ supported = (api_version >= required_ver); ++ ++ return supported; ++} ++ ++/** ++ * kbase_file_new - Create an object representing a device file ++ * ++ * @kbdev: An instance of the GPU platform device, allocated from the probe ++ * method of the driver. ++ * @filp: Pointer to the struct file corresponding to device file ++ * /dev/malixx instance, passed to the file's open method. ++ * ++ * In its initial state, the device file has no context (i.e. no GPU ++ * address space) and no API version number. Both must be assigned before ++ * kbase_file_get_kctx_if_setup_complete() can be used successfully. ++ * ++ * @return Address of an object representing a simulated device file, or NULL ++ * on failure. ++ */ ++static struct kbase_file *kbase_file_new(struct kbase_device *const kbdev, ++ struct file *const filp) ++{ ++ struct kbase_file *const kfile = kmalloc(sizeof(*kfile), GFP_KERNEL); ++ ++ if (kfile) { ++ kfile->kbdev = kbdev; ++ kfile->filp = filp; ++ kfile->kctx = NULL; ++ kfile->api_version = 0; ++ atomic_set(&kfile->setup_state, KBASE_FILE_NEED_VSN); ++ } ++ return kfile; ++} ++ ++/** ++ * kbase_file_set_api_version - Set the application programmer interface version ++ * ++ * @kfile: A device file created by kbase_file_new() ++ * @major: Major version number (must not exceed 12 bits) ++ * @minor: Major version number (must not exceed 12 bits) ++ * ++ * An application programmer interface (API) version must be specified ++ * before calling kbase_file_create_kctx(), otherwise an error is returned. ++ * ++ * If a version number was already set for the given @kfile (or is in the ++ * process of being set by another thread) then an error is returned. ++ * ++ * Return: 0 if successful, otherwise a negative error code. ++ */ ++static int kbase_file_set_api_version(struct kbase_file *const kfile, ++ u16 const major, u16 const minor) ++{ ++ if (WARN_ON(!kfile)) ++ return -EINVAL; ++ ++ /* setup pending, try to signal that we'll do the setup, ++ * if setup was already in progress, err this call ++ */ ++ if (atomic_cmpxchg(&kfile->setup_state, KBASE_FILE_NEED_VSN, ++ KBASE_FILE_VSN_IN_PROGRESS) != KBASE_FILE_NEED_VSN) ++ return -EPERM; ++ ++ /* save the proposed version number for later use */ ++ kfile->api_version = KBASE_API_VERSION(major, minor); ++ ++ atomic_set(&kfile->setup_state, KBASE_FILE_NEED_CTX); ++ return 0; ++} ++ ++/** ++ * kbase_file_get_api_version - Get the application programmer interface version ++ * ++ * @kfile: A device file created by kbase_file_new() ++ * ++ * Return: The version number (encoded with KBASE_API_VERSION) or 0 if none has ++ * been set. ++ */ ++static unsigned long kbase_file_get_api_version(struct kbase_file *const kfile) ++{ ++ if (WARN_ON(!kfile)) ++ return 0; ++ ++ if (atomic_read(&kfile->setup_state) < KBASE_FILE_NEED_CTX) ++ return 0; ++ ++ return kfile->api_version; ++} ++ ++/** ++ * kbase_file_create_kctx - Create a kernel base context ++ * ++ * @kfile: A device file created by kbase_file_new() ++ * @flags: Flags to set, which can be any combination of ++ * BASEP_CONTEXT_CREATE_KERNEL_FLAGS. ++ * ++ * This creates a new context for the GPU platform device instance that was ++ * specified when kbase_file_new() was called. Each context has its own GPU ++ * address space. If a context was already created for the given @kfile (or is ++ * in the process of being created for it by another thread) then an error is ++ * returned. ++ * ++ * An API version number must have been set by kbase_file_set_api_version() ++ * before calling this function, otherwise an error is returned. ++ * ++ * Return: 0 if a new context was created, otherwise a negative error code. ++ */ ++static int kbase_file_create_kctx(struct kbase_file *kfile, ++ base_context_create_flags flags); ++ ++/** ++ * kbase_file_get_kctx_if_setup_complete - Get a kernel base context ++ * pointer from a device file ++ * ++ * @kfile: A device file created by kbase_file_new() ++ * ++ * This function returns an error code (encoded with ERR_PTR) if no context ++ * has been created for the given @kfile. This makes it safe to use in ++ * circumstances where the order of initialization cannot be enforced, but ++ * only if the caller checks the return value. ++ * ++ * Return: Address of the kernel base context associated with the @kfile, or ++ * NULL if no context exists. ++ */ ++static struct kbase_context *kbase_file_get_kctx_if_setup_complete( ++ struct kbase_file *const kfile) ++{ ++ if (WARN_ON(!kfile) || ++ atomic_read(&kfile->setup_state) != KBASE_FILE_COMPLETE || ++ WARN_ON(!kfile->kctx)) ++ return NULL; ++ ++ return kfile->kctx; ++} ++ ++/** ++ * kbase_file_delete - Destroy an object representing a device file ++ * ++ * @kfile: A device file created by kbase_file_new() ++ * ++ * If any context was created for the @kfile then it is destroyed. ++ */ ++static void kbase_file_delete(struct kbase_file *const kfile) ++{ ++ struct kbase_device *kbdev = NULL; ++ ++ if (WARN_ON(!kfile)) ++ return; ++ ++ kfile->filp->private_data = NULL; ++ kbdev = kfile->kbdev; ++ ++ if (atomic_read(&kfile->setup_state) == KBASE_FILE_COMPLETE) { ++ struct kbase_context *kctx = kfile->kctx; ++ ++#ifdef CONFIG_DEBUG_FS ++ kbasep_mem_profile_debugfs_remove(kctx); ++#endif ++ ++ mutex_lock(&kctx->legacy_hwcnt_lock); ++ /* If this client was performing hardware counter dumping and ++ * did not explicitly detach itself, destroy it now ++ */ ++ kbase_hwcnt_legacy_client_destroy(kctx->legacy_hwcnt_cli); ++ kctx->legacy_hwcnt_cli = NULL; ++ mutex_unlock(&kctx->legacy_hwcnt_lock); ++ ++ kbase_context_debugfs_term(kctx); ++ ++ kbase_destroy_context(kctx); ++ ++ dev_dbg(kbdev->dev, "deleted base context\n"); ++ } ++ ++ kbase_release_device(kbdev); ++ ++ kfree(kfile); ++} ++ ++static int kbase_api_handshake(struct kbase_file *kfile, ++ struct kbase_ioctl_version_check *version) ++{ ++ int err = 0; ++ ++ switch (version->major) { ++ case BASE_UK_VERSION_MAJOR: ++ /* set minor to be the lowest common */ ++ version->minor = min_t(int, BASE_UK_VERSION_MINOR, ++ (int)version->minor); ++ break; ++ default: ++ /* We return our actual version regardless if it ++ * matches the version returned by userspace - ++ * userspace can bail if it can't handle this ++ * version ++ */ ++ version->major = BASE_UK_VERSION_MAJOR; ++ version->minor = BASE_UK_VERSION_MINOR; ++ break; ++ } ++ ++ /* save the proposed version number for later use */ ++ err = kbase_file_set_api_version(kfile, version->major, version->minor); ++ if (unlikely(err)) ++ return err; ++ ++ /* For backward compatibility, we may need to create the context before ++ * the flags have been set. Originally it was created on file open ++ * (with job submission disabled) but we don't support that usage. ++ */ ++ if (!mali_kbase_supports_system_monitor(kbase_file_get_api_version(kfile))) ++ err = kbase_file_create_kctx(kfile, ++ BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED); ++ ++ return err; ++} ++ ++static int kbase_api_handshake_dummy(struct kbase_file *kfile, ++ struct kbase_ioctl_version_check *version) ++{ ++ return -EPERM; ++} ++ ++/** ++ * enum mali_error - Mali error codes shared with userspace ++ * ++ * This is subset of those common Mali errors that can be returned to userspace. ++ * Values of matching user and kernel space enumerators MUST be the same. ++ * MALI_ERROR_NONE is guaranteed to be 0. ++ * ++ * @MALI_ERROR_NONE: Success ++ * @MALI_ERROR_OUT_OF_GPU_MEMORY: Not used in the kernel driver ++ * @MALI_ERROR_OUT_OF_MEMORY: Memory allocation failure ++ * @MALI_ERROR_FUNCTION_FAILED: Generic error code ++ */ ++enum mali_error { ++ MALI_ERROR_NONE = 0, ++ MALI_ERROR_OUT_OF_GPU_MEMORY, ++ MALI_ERROR_OUT_OF_MEMORY, ++ MALI_ERROR_FUNCTION_FAILED, ++}; ++ ++static struct kbase_device *to_kbase_device(struct device *dev) ++{ ++ return dev_get_drvdata(dev); ++} ++ ++int assign_irqs(struct kbase_device *kbdev) ++{ ++ struct platform_device *pdev; ++ int i; ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ pdev = to_platform_device(kbdev->dev); ++ /* 3 IRQ resources */ ++ for (i = 0; i < 3; i++) { ++ struct resource *irq_res; ++ int irqtag; ++ ++ irq_res = platform_get_resource(pdev, IORESOURCE_IRQ, i); ++ if (!irq_res) { ++ dev_err(kbdev->dev, "No IRQ resource at index %d\n", i); ++ return -ENOENT; ++ } ++ ++#ifdef CONFIG_OF ++ if (!strncasecmp(irq_res->name, "JOB", 4)) { ++ irqtag = JOB_IRQ_TAG; ++ } else if (!strncasecmp(irq_res->name, "MMU", 4)) { ++ irqtag = MMU_IRQ_TAG; ++ } else if (!strncasecmp(irq_res->name, "GPU", 4)) { ++ irqtag = GPU_IRQ_TAG; ++ } else { ++ dev_err(&pdev->dev, "Invalid irq res name: '%s'\n", ++ irq_res->name); ++ return -EINVAL; ++ } ++#else ++ irqtag = i; ++#endif /* CONFIG_OF */ ++ kbdev->irqs[irqtag].irq = irq_res->start; ++ kbdev->irqs[irqtag].flags = irq_res->flags & IRQF_TRIGGER_MASK; ++ } ++ ++ return 0; ++} ++ ++/* Find a particular kbase device (as specified by minor number), or find the "first" device if -1 is specified */ ++struct kbase_device *kbase_find_device(int minor) ++{ ++ struct kbase_device *kbdev = NULL; ++ struct list_head *entry; ++ const struct list_head *dev_list = kbase_device_get_list(); ++ ++ list_for_each(entry, dev_list) { ++ struct kbase_device *tmp; ++ ++ tmp = list_entry(entry, struct kbase_device, entry); ++ if (tmp->mdev.minor == minor || minor == -1) { ++ kbdev = tmp; ++ get_device(kbdev->dev); ++ break; ++ } ++ } ++ kbase_device_put_list(dev_list); ++ ++ return kbdev; ++} ++EXPORT_SYMBOL(kbase_find_device); ++ ++void kbase_release_device(struct kbase_device *kbdev) ++{ ++ put_device(kbdev->dev); ++} ++EXPORT_SYMBOL(kbase_release_device); ++ ++#ifdef CONFIG_DEBUG_FS ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) && \ ++ !(LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 28) && \ ++ LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0)) ++/* ++ * Older versions, before v4.6, of the kernel doesn't have ++ * kstrtobool_from_user(), except longterm 4.4.y which had it added in 4.4.28 ++ */ ++static int kstrtobool_from_user(const char __user *s, size_t count, bool *res) ++{ ++ char buf[4]; ++ ++ count = min(count, sizeof(buf) - 1); ++ ++ if (copy_from_user(buf, s, count)) ++ return -EFAULT; ++ buf[count] = '\0'; ++ ++ return strtobool(buf, res); ++} ++#endif ++ ++static ssize_t write_ctx_infinite_cache(struct file *f, const char __user *ubuf, size_t size, loff_t *off) ++{ ++ struct kbase_context *kctx = f->private_data; ++ int err; ++ bool value; ++ ++ err = kstrtobool_from_user(ubuf, size, &value); ++ if (err) ++ return err; ++ ++ if (value) ++ kbase_ctx_flag_set(kctx, KCTX_INFINITE_CACHE); ++ else ++ kbase_ctx_flag_clear(kctx, KCTX_INFINITE_CACHE); ++ ++ return size; ++} ++ ++static ssize_t read_ctx_infinite_cache(struct file *f, char __user *ubuf, size_t size, loff_t *off) ++{ ++ struct kbase_context *kctx = f->private_data; ++ char buf[32]; ++ int count; ++ bool value; ++ ++ value = kbase_ctx_flag(kctx, KCTX_INFINITE_CACHE); ++ ++ count = scnprintf(buf, sizeof(buf), "%s\n", value ? "Y" : "N"); ++ ++ return simple_read_from_buffer(ubuf, size, off, buf, count); ++} ++ ++static const struct file_operations kbase_infinite_cache_fops = { ++ .owner = THIS_MODULE, ++ .open = simple_open, ++ .write = write_ctx_infinite_cache, ++ .read = read_ctx_infinite_cache, ++}; ++ ++static ssize_t write_ctx_force_same_va(struct file *f, const char __user *ubuf, ++ size_t size, loff_t *off) ++{ ++ struct kbase_context *kctx = f->private_data; ++ int err; ++ bool value; ++ ++ err = kstrtobool_from_user(ubuf, size, &value); ++ if (err) ++ return err; ++ ++ if (value) { ++#if defined(CONFIG_64BIT) ++ /* 32-bit clients cannot force SAME_VA */ ++ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) ++ return -EINVAL; ++ kbase_ctx_flag_set(kctx, KCTX_FORCE_SAME_VA); ++#else /* defined(CONFIG_64BIT) */ ++ /* 32-bit clients cannot force SAME_VA */ ++ return -EINVAL; ++#endif /* defined(CONFIG_64BIT) */ ++ } else { ++ kbase_ctx_flag_clear(kctx, KCTX_FORCE_SAME_VA); ++ } ++ ++ return size; ++} ++ ++static ssize_t read_ctx_force_same_va(struct file *f, char __user *ubuf, ++ size_t size, loff_t *off) ++{ ++ struct kbase_context *kctx = f->private_data; ++ char buf[32]; ++ int count; ++ bool value; ++ ++ value = kbase_ctx_flag(kctx, KCTX_FORCE_SAME_VA); ++ ++ count = scnprintf(buf, sizeof(buf), "%s\n", value ? "Y" : "N"); ++ ++ return simple_read_from_buffer(ubuf, size, off, buf, count); ++} ++ ++static const struct file_operations kbase_force_same_va_fops = { ++ .owner = THIS_MODULE, ++ .open = simple_open, ++ .write = write_ctx_force_same_va, ++ .read = read_ctx_force_same_va, ++}; ++#endif /* CONFIG_DEBUG_FS */ ++ ++static int kbase_file_create_kctx(struct kbase_file *const kfile, ++ base_context_create_flags const flags) ++{ ++ struct kbase_device *kbdev = NULL; ++ struct kbase_context *kctx = NULL; ++#ifdef CONFIG_DEBUG_FS ++ char kctx_name[64]; ++#endif ++ ++ if (WARN_ON(!kfile)) ++ return -EINVAL; ++ ++ /* setup pending, try to signal that we'll do the setup, ++ * if setup was already in progress, err this call ++ */ ++ if (atomic_cmpxchg(&kfile->setup_state, KBASE_FILE_NEED_CTX, ++ KBASE_FILE_CTX_IN_PROGRESS) != KBASE_FILE_NEED_CTX) ++ return -EPERM; ++ ++ kbdev = kfile->kbdev; ++ ++#if (KERNEL_VERSION(4, 6, 0) <= LINUX_VERSION_CODE) ++ kctx = kbase_create_context(kbdev, in_compat_syscall(), ++ flags, kfile->api_version, kfile->filp); ++#else ++ kctx = kbase_create_context(kbdev, is_compat_task(), ++ flags, kfile->api_version, kfile->filp); ++#endif /* (KERNEL_VERSION(4, 6, 0) <= LINUX_VERSION_CODE) */ ++ ++ /* if bad flags, will stay stuck in setup mode */ ++ if (!kctx) ++ return -ENOMEM; ++ ++ if (kbdev->infinite_cache_active_default) ++ kbase_ctx_flag_set(kctx, KCTX_INFINITE_CACHE); ++ ++#ifdef CONFIG_DEBUG_FS ++ snprintf(kctx_name, 64, "%d_%d", kctx->tgid, kctx->id); ++ ++ mutex_init(&kctx->mem_profile_lock); ++ ++ kctx->kctx_dentry = debugfs_create_dir(kctx_name, ++ kbdev->debugfs_ctx_directory); ++ ++ if (IS_ERR_OR_NULL(kctx->kctx_dentry)) { ++ /* we don't treat this as a fail - just warn about it */ ++ dev_warn(kbdev->dev, "couldn't create debugfs dir for kctx\n"); ++ } else { ++#if (KERNEL_VERSION(4, 7, 0) > LINUX_VERSION_CODE) ++ /* prevent unprivileged use of debug file system ++ * in old kernel version ++ */ ++ debugfs_create_file("infinite_cache", 0600, kctx->kctx_dentry, ++ kctx, &kbase_infinite_cache_fops); ++#else ++ debugfs_create_file("infinite_cache", 0644, kctx->kctx_dentry, ++ kctx, &kbase_infinite_cache_fops); ++#endif ++ debugfs_create_file("force_same_va", 0600, kctx->kctx_dentry, ++ kctx, &kbase_force_same_va_fops); ++ ++ kbase_context_debugfs_init(kctx); ++ } ++#endif /* CONFIG_DEBUG_FS */ ++ ++ dev_dbg(kbdev->dev, "created base context\n"); ++ ++ kfile->kctx = kctx; ++ atomic_set(&kfile->setup_state, KBASE_FILE_COMPLETE); ++ ++ return 0; ++} ++ ++static int kbase_open(struct inode *inode, struct file *filp) ++{ ++ struct kbase_device *kbdev = NULL; ++ struct kbase_file *kfile; ++ int ret = 0; ++ ++ kbdev = kbase_find_device(iminor(inode)); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ kfile = kbase_file_new(kbdev, filp); ++ if (!kfile) { ++ ret = -ENOMEM; ++ goto out; ++ } ++ ++ filp->private_data = kfile; ++ filp->f_mode |= FMODE_UNSIGNED_OFFSET; ++ ++ return 0; ++ ++ out: ++ kbase_release_device(kbdev); ++ return ret; ++} ++ ++static int kbase_release(struct inode *inode, struct file *filp) ++{ ++ struct kbase_file *const kfile = filp->private_data; ++ ++ kbase_file_delete(kfile); ++ return 0; ++} ++ ++static int kbase_api_set_flags(struct kbase_file *kfile, ++ struct kbase_ioctl_set_flags *flags) ++{ ++ int err = 0; ++ unsigned long const api_version = kbase_file_get_api_version(kfile); ++ struct kbase_context *kctx = NULL; ++ ++ /* Validate flags */ ++ if (flags->create_flags != ++ (flags->create_flags & BASEP_CONTEXT_CREATE_KERNEL_FLAGS)) ++ return -EINVAL; ++ ++ /* For backward compatibility, the context may have been created before ++ * the flags were set. ++ */ ++ if (mali_kbase_supports_system_monitor(api_version)) { ++ err = kbase_file_create_kctx(kfile, flags->create_flags); ++ } else { ++#if !MALI_USE_CSF ++ struct kbasep_js_kctx_info *js_kctx_info = NULL; ++ unsigned long irq_flags = 0; ++#endif ++ ++ /* If setup is incomplete (e.g. because the API version ++ * wasn't set) then we have to give up. ++ */ ++ kctx = kbase_file_get_kctx_if_setup_complete(kfile); ++ if (unlikely(!kctx)) ++ return -EPERM; ++ ++#if MALI_USE_CSF ++ /* On CSF GPUs Job Manager interface isn't used to submit jobs ++ * (there are no job slots). So the legacy job manager path to ++ * submit jobs needs to remain disabled for CSF GPUs. ++ */ ++#else ++ js_kctx_info = &kctx->jctx.sched_info; ++ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); ++ spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, irq_flags); ++ /* Translate the flags */ ++ if ((flags->create_flags & ++ BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED) == 0) ++ kbase_ctx_flag_clear(kctx, KCTX_SUBMIT_DISABLED); ++ ++ ++ spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, irq_flags); ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++#endif ++ } ++ ++ return err; ++} ++ ++#if !MALI_USE_CSF ++static int kbase_api_job_submit(struct kbase_context *kctx, ++ struct kbase_ioctl_job_submit *submit) ++{ ++ return kbase_jd_submit(kctx, u64_to_user_ptr(submit->addr), ++ submit->nr_atoms, ++ submit->stride, false); ++} ++#endif /* !MALI_USE_CSF */ ++ ++static int kbase_api_get_gpuprops(struct kbase_context *kctx, ++ struct kbase_ioctl_get_gpuprops *get_props) ++{ ++ struct kbase_gpu_props *kprops = &kctx->kbdev->gpu_props; ++ int err; ++ ++ if (get_props->flags != 0) { ++ dev_err(kctx->kbdev->dev, "Unsupported flags to get_gpuprops"); ++ return -EINVAL; ++ } ++ ++ if (get_props->size == 0) ++ return kprops->prop_buffer_size; ++ if (get_props->size < kprops->prop_buffer_size) ++ return -EINVAL; ++ ++ err = copy_to_user(u64_to_user_ptr(get_props->buffer), ++ kprops->prop_buffer, ++ kprops->prop_buffer_size); ++ if (err) ++ return -EFAULT; ++ return kprops->prop_buffer_size; ++} ++ ++#if !MALI_USE_CSF ++static int kbase_api_post_term(struct kbase_context *kctx) ++{ ++ kbase_event_close(kctx); ++ return 0; ++} ++#endif /* !MALI_USE_CSF */ ++ ++static int kbase_api_mem_alloc(struct kbase_context *kctx, ++ union kbase_ioctl_mem_alloc *alloc) ++{ ++ struct kbase_va_region *reg; ++ u64 flags = alloc->in.flags; ++ u64 gpu_va; ++ ++ rcu_read_lock(); ++ /* Don't allow memory allocation until user space has set up the ++ * tracking page (which sets kctx->process_mm). Also catches when we've ++ * forked. ++ */ ++ if (rcu_dereference(kctx->process_mm) != current->mm) { ++ rcu_read_unlock(); ++ return -EINVAL; ++ } ++ rcu_read_unlock(); ++ ++ if (flags & BASEP_MEM_FLAGS_KERNEL_ONLY) ++ return -ENOMEM; ++ ++ /* Force SAME_VA if a 64-bit client. ++ * The only exception is GPU-executable memory if an EXEC_VA zone ++ * has been initialized. In that case, GPU-executable memory may ++ * or may not be SAME_VA. ++ */ ++ if ((!kbase_ctx_flag(kctx, KCTX_COMPAT)) && ++ kbase_ctx_flag(kctx, KCTX_FORCE_SAME_VA)) { ++ if (!(flags & BASE_MEM_PROT_GPU_EX) || !kbase_has_exec_va_zone(kctx)) ++ flags |= BASE_MEM_SAME_VA; ++ } ++ ++#if MALI_USE_CSF ++ /* If CSF event memory allocation, need to force certain flags. ++ * SAME_VA - GPU address needs to be used as a CPU address, explicit ++ * mmap has to be avoided. ++ * CACHED_CPU - Frequent access to the event memory by CPU. ++ * COHERENT_SYSTEM - No explicit cache maintenance around the access ++ * to event memory so need to leverage the coherency support. ++ */ ++ if (flags & BASE_MEM_CSF_EVENT) { ++ flags |= (BASE_MEM_SAME_VA | ++ BASE_MEM_CACHED_CPU | ++ BASE_MEM_COHERENT_SYSTEM); ++ } ++#endif ++ ++ reg = kbase_mem_alloc(kctx, alloc->in.va_pages, ++ alloc->in.commit_pages, ++ alloc->in.extent, ++ &flags, &gpu_va); ++ ++ if (!reg) ++ return -ENOMEM; ++ ++ alloc->out.flags = flags; ++ alloc->out.gpu_va = gpu_va; ++ ++ return 0; ++} ++ ++static int kbase_api_mem_query(struct kbase_context *kctx, ++ union kbase_ioctl_mem_query *query) ++{ ++ return kbase_mem_query(kctx, query->in.gpu_addr, ++ query->in.query, &query->out.value); ++} ++ ++static int kbase_api_mem_free(struct kbase_context *kctx, ++ struct kbase_ioctl_mem_free *free) ++{ ++ return kbase_mem_free(kctx, free->gpu_addr); ++} ++ ++#if !MALI_USE_CSF ++static int kbase_api_kinstr_jm_fd(struct kbase_context *kctx, ++ union kbase_kinstr_jm_fd *arg) ++{ ++ return kbase_kinstr_jm_get_fd(kctx->kinstr_jm, arg); ++} ++#endif ++ ++static int kbase_api_hwcnt_reader_setup(struct kbase_context *kctx, ++ struct kbase_ioctl_hwcnt_reader_setup *setup) ++{ ++ return kbase_vinstr_hwcnt_reader_setup(kctx->kbdev->vinstr_ctx, setup); ++} ++ ++static int kbase_api_hwcnt_enable(struct kbase_context *kctx, ++ struct kbase_ioctl_hwcnt_enable *enable) ++{ ++ int ret; ++ ++ mutex_lock(&kctx->legacy_hwcnt_lock); ++ if (enable->dump_buffer != 0) { ++ /* Non-zero dump buffer, so user wants to create the client */ ++ if (kctx->legacy_hwcnt_cli == NULL) { ++ ret = kbase_hwcnt_legacy_client_create( ++ kctx->kbdev->hwcnt_gpu_virt, ++ enable, ++ &kctx->legacy_hwcnt_cli); ++ } else { ++ /* This context already has a client */ ++ ret = -EBUSY; ++ } ++ } else { ++ /* Zero dump buffer, so user wants to destroy the client */ ++ if (kctx->legacy_hwcnt_cli != NULL) { ++ kbase_hwcnt_legacy_client_destroy( ++ kctx->legacy_hwcnt_cli); ++ kctx->legacy_hwcnt_cli = NULL; ++ ret = 0; ++ } else { ++ /* This context has no client to destroy */ ++ ret = -EINVAL; ++ } ++ } ++ mutex_unlock(&kctx->legacy_hwcnt_lock); ++ ++ return ret; ++} ++ ++static int kbase_api_hwcnt_dump(struct kbase_context *kctx) ++{ ++ int ret; ++ ++ mutex_lock(&kctx->legacy_hwcnt_lock); ++ ret = kbase_hwcnt_legacy_client_dump(kctx->legacy_hwcnt_cli); ++ mutex_unlock(&kctx->legacy_hwcnt_lock); ++ ++ return ret; ++} ++ ++static int kbase_api_hwcnt_clear(struct kbase_context *kctx) ++{ ++ int ret; ++ ++ mutex_lock(&kctx->legacy_hwcnt_lock); ++ ret = kbase_hwcnt_legacy_client_clear(kctx->legacy_hwcnt_cli); ++ mutex_unlock(&kctx->legacy_hwcnt_lock); ++ ++ return ret; ++} ++ ++static int kbase_api_get_cpu_gpu_timeinfo(struct kbase_context *kctx, ++ union kbase_ioctl_get_cpu_gpu_timeinfo *timeinfo) ++{ ++ u32 flags = timeinfo->in.request_flags; ++ struct timespec64 ts; ++ u64 timestamp; ++ u64 cycle_cnt; ++ ++ kbase_pm_context_active(kctx->kbdev); ++ ++ kbase_backend_get_gpu_time(kctx->kbdev, ++ (flags & BASE_TIMEINFO_CYCLE_COUNTER_FLAG) ? &cycle_cnt : NULL, ++ (flags & BASE_TIMEINFO_TIMESTAMP_FLAG) ? ×tamp : NULL, ++ (flags & BASE_TIMEINFO_MONOTONIC_FLAG) ? &ts : NULL); ++ ++ if (flags & BASE_TIMEINFO_TIMESTAMP_FLAG) ++ timeinfo->out.timestamp = timestamp; ++ ++ if (flags & BASE_TIMEINFO_CYCLE_COUNTER_FLAG) ++ timeinfo->out.cycle_counter = cycle_cnt; ++ ++ if (flags & BASE_TIMEINFO_MONOTONIC_FLAG) { ++ timeinfo->out.sec = ts.tv_sec; ++ timeinfo->out.nsec = ts.tv_nsec; ++ } ++ ++ kbase_pm_context_idle(kctx->kbdev); ++ ++ return 0; ++} ++ ++#ifdef CONFIG_MALI_BIFROST_NO_MALI ++static int kbase_api_hwcnt_set(struct kbase_context *kctx, ++ struct kbase_ioctl_hwcnt_values *values) ++{ ++ gpu_model_set_dummy_prfcnt_sample( ++ (u32 __user *)(uintptr_t)values->data, ++ values->size); ++ ++ return 0; ++} ++#endif ++ ++static int kbase_api_disjoint_query(struct kbase_context *kctx, ++ struct kbase_ioctl_disjoint_query *query) ++{ ++ query->counter = kbase_disjoint_event_get(kctx->kbdev); ++ ++ return 0; ++} ++ ++static int kbase_api_get_ddk_version(struct kbase_context *kctx, ++ struct kbase_ioctl_get_ddk_version *version) ++{ ++ int ret; ++ int len = sizeof(KERNEL_SIDE_DDK_VERSION_STRING); ++ ++ if (version->version_buffer == 0) ++ return len; ++ ++ if (version->size < len) ++ return -EOVERFLOW; ++ ++ ret = copy_to_user(u64_to_user_ptr(version->version_buffer), ++ KERNEL_SIDE_DDK_VERSION_STRING, ++ sizeof(KERNEL_SIDE_DDK_VERSION_STRING)); ++ ++ if (ret) ++ return -EFAULT; ++ ++ return len; ++} ++ ++/* Defaults for legacy just-in-time memory allocator initialization ++ * kernel calls ++ */ ++#define DEFAULT_MAX_JIT_ALLOCATIONS 255 ++#define JIT_LEGACY_TRIM_LEVEL (0) /* No trimming */ ++ ++static int kbase_api_mem_jit_init_10_2(struct kbase_context *kctx, ++ struct kbase_ioctl_mem_jit_init_10_2 *jit_init) ++{ ++ kctx->jit_version = 1; ++ ++ /* since no phys_pages parameter, use the maximum: va_pages */ ++ return kbase_region_tracker_init_jit(kctx, jit_init->va_pages, ++ DEFAULT_MAX_JIT_ALLOCATIONS, ++ JIT_LEGACY_TRIM_LEVEL, BASE_MEM_GROUP_DEFAULT, ++ jit_init->va_pages); ++} ++ ++static int kbase_api_mem_jit_init_11_5(struct kbase_context *kctx, ++ struct kbase_ioctl_mem_jit_init_11_5 *jit_init) ++{ ++ int i; ++ ++ kctx->jit_version = 2; ++ ++ for (i = 0; i < sizeof(jit_init->padding); i++) { ++ /* Ensure all padding bytes are 0 for potential future ++ * extension ++ */ ++ if (jit_init->padding[i]) ++ return -EINVAL; ++ } ++ ++ /* since no phys_pages parameter, use the maximum: va_pages */ ++ return kbase_region_tracker_init_jit(kctx, jit_init->va_pages, ++ jit_init->max_allocations, jit_init->trim_level, ++ jit_init->group_id, jit_init->va_pages); ++} ++ ++static int kbase_api_mem_jit_init(struct kbase_context *kctx, ++ struct kbase_ioctl_mem_jit_init *jit_init) ++{ ++ int i; ++ ++ kctx->jit_version = 3; ++ ++ for (i = 0; i < sizeof(jit_init->padding); i++) { ++ /* Ensure all padding bytes are 0 for potential future ++ * extension ++ */ ++ if (jit_init->padding[i]) ++ return -EINVAL; ++ } ++ ++ return kbase_region_tracker_init_jit(kctx, jit_init->va_pages, ++ jit_init->max_allocations, jit_init->trim_level, ++ jit_init->group_id, jit_init->phys_pages); ++} ++ ++static int kbase_api_mem_exec_init(struct kbase_context *kctx, ++ struct kbase_ioctl_mem_exec_init *exec_init) ++{ ++ return kbase_region_tracker_init_exec(kctx, exec_init->va_pages); ++} ++ ++static int kbase_api_mem_sync(struct kbase_context *kctx, ++ struct kbase_ioctl_mem_sync *sync) ++{ ++ struct basep_syncset sset = { ++ .mem_handle.basep.handle = sync->handle, ++ .user_addr = sync->user_addr, ++ .size = sync->size, ++ .type = sync->type ++ }; ++ ++ return kbase_sync_now(kctx, &sset); ++} ++ ++static int kbase_api_mem_find_cpu_offset(struct kbase_context *kctx, ++ union kbase_ioctl_mem_find_cpu_offset *find) ++{ ++ return kbasep_find_enclosing_cpu_mapping_offset( ++ kctx, ++ find->in.cpu_addr, ++ find->in.size, ++ &find->out.offset); ++} ++ ++static int kbase_api_mem_find_gpu_start_and_offset(struct kbase_context *kctx, ++ union kbase_ioctl_mem_find_gpu_start_and_offset *find) ++{ ++ return kbasep_find_enclosing_gpu_mapping_start_and_offset( ++ kctx, ++ find->in.gpu_addr, ++ find->in.size, ++ &find->out.start, ++ &find->out.offset); ++} ++ ++static int kbase_api_get_context_id(struct kbase_context *kctx, ++ struct kbase_ioctl_get_context_id *info) ++{ ++ info->id = kctx->id; ++ ++ return 0; ++} ++ ++static int kbase_api_tlstream_acquire(struct kbase_context *kctx, ++ struct kbase_ioctl_tlstream_acquire *acquire) ++{ ++ return kbase_timeline_io_acquire(kctx->kbdev, acquire->flags); ++} ++ ++static int kbase_api_tlstream_flush(struct kbase_context *kctx) ++{ ++ kbase_timeline_streams_flush(kctx->kbdev->timeline); ++ ++ return 0; ++} ++ ++static int kbase_api_mem_commit(struct kbase_context *kctx, ++ struct kbase_ioctl_mem_commit *commit) ++{ ++ return kbase_mem_commit(kctx, commit->gpu_addr, commit->pages); ++} ++ ++static int kbase_api_mem_alias(struct kbase_context *kctx, ++ union kbase_ioctl_mem_alias *alias) ++{ ++ struct base_mem_aliasing_info *ai; ++ u64 flags; ++ int err; ++ ++ if (alias->in.nents == 0 || alias->in.nents > 2048) ++ return -EINVAL; ++ ++ if (alias->in.stride > (U64_MAX / 2048)) ++ return -EINVAL; ++ ++ ai = vmalloc(sizeof(*ai) * alias->in.nents); ++ if (!ai) ++ return -ENOMEM; ++ ++ err = copy_from_user(ai, ++ u64_to_user_ptr(alias->in.aliasing_info), ++ sizeof(*ai) * alias->in.nents); ++ if (err) { ++ vfree(ai); ++ return -EFAULT; ++ } ++ ++ flags = alias->in.flags; ++ if (flags & BASEP_MEM_FLAGS_KERNEL_ONLY) { ++ vfree(ai); ++ return -EINVAL; ++ } ++ ++ alias->out.gpu_va = kbase_mem_alias(kctx, &flags, ++ alias->in.stride, alias->in.nents, ++ ai, &alias->out.va_pages); ++ ++ alias->out.flags = flags; ++ ++ vfree(ai); ++ ++ if (alias->out.gpu_va == 0) ++ return -ENOMEM; ++ ++ return 0; ++} ++ ++static int kbase_api_mem_import(struct kbase_context *kctx, ++ union kbase_ioctl_mem_import *import) ++{ ++ int ret; ++ u64 flags = import->in.flags; ++ ++ if (flags & BASEP_MEM_FLAGS_KERNEL_ONLY) ++ return -ENOMEM; ++ ++ ret = kbase_mem_import(kctx, ++ import->in.type, ++ u64_to_user_ptr(import->in.phandle), ++ import->in.padding, ++ &import->out.gpu_va, ++ &import->out.va_pages, ++ &flags); ++ ++ import->out.flags = flags; ++ ++ return ret; ++} ++ ++static int kbase_api_mem_flags_change(struct kbase_context *kctx, ++ struct kbase_ioctl_mem_flags_change *change) ++{ ++ if (change->flags & BASEP_MEM_FLAGS_KERNEL_ONLY) ++ return -ENOMEM; ++ ++ return kbase_mem_flags_change(kctx, change->gpu_va, ++ change->flags, change->mask); ++} ++ ++static int kbase_api_stream_create(struct kbase_context *kctx, ++ struct kbase_ioctl_stream_create *stream) ++{ ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ int fd, ret; ++ ++ /* Name must be NULL-terminated and padded with NULLs, so check last ++ * character is NULL ++ */ ++ if (stream->name[sizeof(stream->name)-1] != 0) ++ return -EINVAL; ++ ++ ret = kbase_sync_fence_stream_create(stream->name, &fd); ++ ++ if (ret) ++ return ret; ++ return fd; ++#else ++ return -ENOENT; ++#endif ++} ++ ++static int kbase_api_fence_validate(struct kbase_context *kctx, ++ struct kbase_ioctl_fence_validate *validate) ++{ ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ return kbase_sync_fence_validate(validate->fd); ++#else ++ return -ENOENT; ++#endif ++} ++ ++static int kbase_api_mem_profile_add(struct kbase_context *kctx, ++ struct kbase_ioctl_mem_profile_add *data) ++{ ++ char *buf; ++ int err; ++ ++ if (data->len > KBASE_MEM_PROFILE_MAX_BUF_SIZE) { ++ dev_err(kctx->kbdev->dev, "mem_profile_add: buffer too big\n"); ++ return -EINVAL; ++ } ++ ++ buf = kmalloc(data->len, GFP_KERNEL); ++ if (ZERO_OR_NULL_PTR(buf)) ++ return -ENOMEM; ++ ++ err = copy_from_user(buf, u64_to_user_ptr(data->buffer), ++ data->len); ++ if (err) { ++ kfree(buf); ++ return -EFAULT; ++ } ++ ++ return kbasep_mem_profile_debugfs_insert(kctx, buf, data->len); ++} ++ ++#if !MALI_USE_CSF ++static int kbase_api_soft_event_update(struct kbase_context *kctx, ++ struct kbase_ioctl_soft_event_update *update) ++{ ++ if (update->flags != 0) ++ return -EINVAL; ++ ++ return kbase_soft_event_update(kctx, update->event, update->new_status); ++} ++#endif /* !MALI_USE_CSF */ ++ ++static int kbase_api_sticky_resource_map(struct kbase_context *kctx, ++ struct kbase_ioctl_sticky_resource_map *map) ++{ ++ int ret; ++ u64 i; ++ u64 gpu_addr[BASE_EXT_RES_COUNT_MAX]; ++ ++ if (!map->count || map->count > BASE_EXT_RES_COUNT_MAX) ++ return -EOVERFLOW; ++ ++ ret = copy_from_user(gpu_addr, u64_to_user_ptr(map->address), ++ sizeof(u64) * map->count); ++ ++ if (ret != 0) ++ return -EFAULT; ++ ++ kbase_gpu_vm_lock(kctx); ++ ++ for (i = 0; i < map->count; i++) { ++ if (!kbase_sticky_resource_acquire(kctx, gpu_addr[i])) { ++ /* Invalid resource */ ++ ret = -EINVAL; ++ break; ++ } ++ } ++ ++ if (ret != 0) { ++ while (i > 0) { ++ i--; ++ kbase_sticky_resource_release_force(kctx, NULL, gpu_addr[i]); ++ } ++ } ++ ++ kbase_gpu_vm_unlock(kctx); ++ ++ return ret; ++} ++ ++static int kbase_api_sticky_resource_unmap(struct kbase_context *kctx, ++ struct kbase_ioctl_sticky_resource_unmap *unmap) ++{ ++ int ret; ++ u64 i; ++ u64 gpu_addr[BASE_EXT_RES_COUNT_MAX]; ++ ++ if (!unmap->count || unmap->count > BASE_EXT_RES_COUNT_MAX) ++ return -EOVERFLOW; ++ ++ ret = copy_from_user(gpu_addr, u64_to_user_ptr(unmap->address), ++ sizeof(u64) * unmap->count); ++ ++ if (ret != 0) ++ return -EFAULT; ++ ++ kbase_gpu_vm_lock(kctx); ++ ++ for (i = 0; i < unmap->count; i++) { ++ if (!kbase_sticky_resource_release_force(kctx, NULL, gpu_addr[i])) { ++ /* Invalid resource, but we keep going anyway */ ++ ret = -EINVAL; ++ } ++ } ++ ++ kbase_gpu_vm_unlock(kctx); ++ ++ return ret; ++} ++ ++#if MALI_UNIT_TEST ++static int kbase_api_tlstream_test(struct kbase_context *kctx, ++ struct kbase_ioctl_tlstream_test *test) ++{ ++ kbase_timeline_test( ++ kctx->kbdev, ++ test->tpw_count, ++ test->msg_delay, ++ test->msg_count, ++ test->aux_msg); ++ ++ return 0; ++} ++ ++static int kbase_api_tlstream_stats(struct kbase_context *kctx, ++ struct kbase_ioctl_tlstream_stats *stats) ++{ ++ kbase_timeline_stats(kctx->kbdev->timeline, ++ &stats->bytes_collected, ++ &stats->bytes_generated); ++ ++ return 0; ++} ++#endif /* MALI_UNIT_TEST */ ++ ++#if MALI_USE_CSF ++static int kbasep_cs_event_signal(struct kbase_context *kctx) ++{ ++ kbase_csf_event_signal_notify_gpu(kctx); ++ return 0; ++} ++ ++static int kbasep_cs_queue_register(struct kbase_context *kctx, ++ struct kbase_ioctl_cs_queue_register *reg) ++{ ++ kctx->jit_group_id = BASE_MEM_GROUP_DEFAULT; ++ ++ return kbase_csf_queue_register(kctx, reg); ++} ++ ++static int kbasep_cs_queue_terminate(struct kbase_context *kctx, ++ struct kbase_ioctl_cs_queue_terminate *term) ++{ ++ kbase_csf_queue_terminate(kctx, term); ++ ++ return 0; ++} ++ ++static int kbasep_cs_queue_bind(struct kbase_context *kctx, ++ union kbase_ioctl_cs_queue_bind *bind) ++{ ++ return kbase_csf_queue_bind(kctx, bind); ++} ++ ++static int kbasep_cs_queue_kick(struct kbase_context *kctx, ++ struct kbase_ioctl_cs_queue_kick *kick) ++{ ++ return kbase_csf_queue_kick(kctx, kick); ++} ++ ++static int kbasep_cs_queue_group_create(struct kbase_context *kctx, ++ union kbase_ioctl_cs_queue_group_create *create) ++{ ++ return kbase_csf_queue_group_create(kctx, create); ++} ++ ++static int kbasep_cs_queue_group_terminate(struct kbase_context *kctx, ++ struct kbase_ioctl_cs_queue_group_term *term) ++{ ++ kbase_csf_queue_group_terminate(kctx, term->group_handle); ++ ++ return 0; ++} ++ ++static int kbasep_kcpu_queue_new(struct kbase_context *kctx, ++ struct kbase_ioctl_kcpu_queue_new *new) ++{ ++ return kbase_csf_kcpu_queue_new(kctx, new); ++} ++ ++static int kbasep_kcpu_queue_delete(struct kbase_context *kctx, ++ struct kbase_ioctl_kcpu_queue_delete *delete) ++{ ++ return kbase_csf_kcpu_queue_delete(kctx, delete); ++} ++ ++static int kbasep_kcpu_queue_enqueue(struct kbase_context *kctx, ++ struct kbase_ioctl_kcpu_queue_enqueue *enqueue) ++{ ++ return kbase_csf_kcpu_queue_enqueue(kctx, enqueue); ++} ++ ++static int kbasep_cs_tiler_heap_init(struct kbase_context *kctx, ++ union kbase_ioctl_cs_tiler_heap_init *heap_init) ++{ ++ kctx->jit_group_id = heap_init->in.group_id; ++ ++ return kbase_csf_tiler_heap_init(kctx, heap_init->in.chunk_size, ++ heap_init->in.initial_chunks, heap_init->in.max_chunks, ++ heap_init->in.target_in_flight, ++ &heap_init->out.gpu_heap_va, &heap_init->out.first_chunk_va); ++} ++ ++static int kbasep_cs_tiler_heap_term(struct kbase_context *kctx, ++ struct kbase_ioctl_cs_tiler_heap_term *heap_term) ++{ ++ return kbase_csf_tiler_heap_term(kctx, heap_term->gpu_heap_va); ++} ++ ++static int kbase_ioctl_cs_get_glb_iface(struct kbase_context *kctx, ++ union kbase_ioctl_cs_get_glb_iface *param) ++{ ++ struct basep_cs_stream_control *stream_data = NULL; ++ struct basep_cs_group_control *group_data = NULL; ++ void __user *user_groups, *user_streams; ++ int err = 0; ++ u32 const max_group_num = param->in.max_group_num; ++ u32 const max_total_stream_num = param->in.max_total_stream_num; ++ ++ if (max_group_num > MAX_SUPPORTED_CSGS) ++ return -EINVAL; ++ ++ if (max_total_stream_num > ++ MAX_SUPPORTED_CSGS * MAX_SUPPORTED_STREAMS_PER_GROUP) ++ return -EINVAL; ++ ++ user_groups = u64_to_user_ptr(param->in.groups_ptr); ++ user_streams = u64_to_user_ptr(param->in.streams_ptr); ++ ++ if (max_group_num > 0) { ++ if (!user_groups) ++ err = -EINVAL; ++ else { ++ group_data = kcalloc(max_group_num, ++ sizeof(*group_data), GFP_KERNEL); ++ if (!group_data) ++ err = -ENOMEM; ++ } ++ } ++ ++ if (max_total_stream_num > 0) { ++ if (!user_streams) ++ err = -EINVAL; ++ else { ++ stream_data = kcalloc(max_total_stream_num, ++ sizeof(*stream_data), GFP_KERNEL); ++ if (!stream_data) ++ err = -ENOMEM; ++ } ++ } ++ ++ if (!err) { ++ param->out.total_stream_num = ++ kbase_csf_firmware_get_glb_iface(kctx->kbdev, ++ group_data, max_group_num, ++ stream_data, max_total_stream_num, ++ ¶m->out.glb_version, ¶m->out.features, ++ ¶m->out.group_num, ¶m->out.prfcnt_size); ++ ++ param->out.padding = 0; ++ ++ if (copy_to_user(user_groups, group_data, ++ MIN(max_group_num, param->out.group_num) * ++ sizeof(*group_data))) ++ err = -EFAULT; ++ } ++ ++ if (!err) ++ if (copy_to_user(user_streams, stream_data, ++ MIN(max_total_stream_num, param->out.total_stream_num) * ++ sizeof(*stream_data))) ++ err = -EFAULT; ++ ++ kfree(group_data); ++ kfree(stream_data); ++ return err; ++} ++#endif /* MALI_USE_CSF */ ++ ++#define KBASE_HANDLE_IOCTL(cmd, function, arg) \ ++ do { \ ++ BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_NONE); \ ++ return function(arg); \ ++ } while (0) ++ ++#define KBASE_HANDLE_IOCTL_IN(cmd, function, type, arg) \ ++ do { \ ++ type param; \ ++ int err; \ ++ BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_WRITE); \ ++ BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ ++ err = copy_from_user(¶m, uarg, sizeof(param)); \ ++ if (err) \ ++ return -EFAULT; \ ++ return function(arg, ¶m); \ ++ } while (0) ++ ++#define KBASE_HANDLE_IOCTL_OUT(cmd, function, type, arg) \ ++ do { \ ++ type param; \ ++ int ret, err; \ ++ BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_READ); \ ++ BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ ++ memset(¶m, 0, sizeof(param)); \ ++ ret = function(arg, ¶m); \ ++ err = copy_to_user(uarg, ¶m, sizeof(param)); \ ++ if (err) \ ++ return -EFAULT; \ ++ return ret; \ ++ } while (0) ++ ++#define KBASE_HANDLE_IOCTL_INOUT(cmd, function, type, arg) \ ++ do { \ ++ type param; \ ++ int ret, err; \ ++ BUILD_BUG_ON(_IOC_DIR(cmd) != (_IOC_WRITE|_IOC_READ)); \ ++ BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ ++ err = copy_from_user(¶m, uarg, sizeof(param)); \ ++ if (err) \ ++ return -EFAULT; \ ++ ret = function(arg, ¶m); \ ++ err = copy_to_user(uarg, ¶m, sizeof(param)); \ ++ if (err) \ ++ return -EFAULT; \ ++ return ret; \ ++ } while (0) ++ ++static long kbase_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) ++{ ++ struct kbase_file *const kfile = filp->private_data; ++ struct kbase_context *kctx = NULL; ++ struct kbase_device *kbdev = kfile->kbdev; ++ void __user *uarg = (void __user *)arg; ++ ++ /* Only these ioctls are available until setup is complete */ ++ switch (cmd) { ++ case KBASE_IOCTL_VERSION_CHECK: ++ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_VERSION_CHECK, ++ kbase_api_handshake, ++ struct kbase_ioctl_version_check, ++ kfile); ++ break; ++ ++ case KBASE_IOCTL_VERSION_CHECK_RESERVED: ++ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_VERSION_CHECK_RESERVED, ++ kbase_api_handshake_dummy, ++ struct kbase_ioctl_version_check, ++ kfile); ++ break; ++ ++ case KBASE_IOCTL_SET_FLAGS: ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_SET_FLAGS, ++ kbase_api_set_flags, ++ struct kbase_ioctl_set_flags, ++ kfile); ++ break; ++ } ++ ++ kctx = kbase_file_get_kctx_if_setup_complete(kfile); ++ if (unlikely(!kctx)) ++ return -EPERM; ++ ++ /* Normal ioctls */ ++ switch (cmd) { ++#if !MALI_USE_CSF ++ case KBASE_IOCTL_JOB_SUBMIT: ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_JOB_SUBMIT, ++ kbase_api_job_submit, ++ struct kbase_ioctl_job_submit, ++ kctx); ++ break; ++#endif /* !MALI_USE_CSF */ ++ case KBASE_IOCTL_GET_GPUPROPS: ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_GET_GPUPROPS, ++ kbase_api_get_gpuprops, ++ struct kbase_ioctl_get_gpuprops, ++ kctx); ++ break; ++#if !MALI_USE_CSF ++ case KBASE_IOCTL_POST_TERM: ++ KBASE_HANDLE_IOCTL(KBASE_IOCTL_POST_TERM, ++ kbase_api_post_term, ++ kctx); ++ break; ++#endif /* !MALI_USE_CSF */ ++ case KBASE_IOCTL_MEM_ALLOC: ++ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_ALLOC, ++ kbase_api_mem_alloc, ++ union kbase_ioctl_mem_alloc, ++ kctx); ++ break; ++ case KBASE_IOCTL_MEM_QUERY: ++ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_QUERY, ++ kbase_api_mem_query, ++ union kbase_ioctl_mem_query, ++ kctx); ++ break; ++ case KBASE_IOCTL_MEM_FREE: ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_FREE, ++ kbase_api_mem_free, ++ struct kbase_ioctl_mem_free, ++ kctx); ++ break; ++ case KBASE_IOCTL_DISJOINT_QUERY: ++ KBASE_HANDLE_IOCTL_OUT(KBASE_IOCTL_DISJOINT_QUERY, ++ kbase_api_disjoint_query, ++ struct kbase_ioctl_disjoint_query, ++ kctx); ++ break; ++ case KBASE_IOCTL_GET_DDK_VERSION: ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_GET_DDK_VERSION, ++ kbase_api_get_ddk_version, ++ struct kbase_ioctl_get_ddk_version, ++ kctx); ++ break; ++ case KBASE_IOCTL_MEM_JIT_INIT_10_2: ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_JIT_INIT_10_2, ++ kbase_api_mem_jit_init_10_2, ++ struct kbase_ioctl_mem_jit_init_10_2, ++ kctx); ++ break; ++ case KBASE_IOCTL_MEM_JIT_INIT_11_5: ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_JIT_INIT_11_5, ++ kbase_api_mem_jit_init_11_5, ++ struct kbase_ioctl_mem_jit_init_11_5, ++ kctx); ++ break; ++ case KBASE_IOCTL_MEM_JIT_INIT: ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_JIT_INIT, ++ kbase_api_mem_jit_init, ++ struct kbase_ioctl_mem_jit_init, ++ kctx); ++ break; ++ case KBASE_IOCTL_MEM_EXEC_INIT: ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_EXEC_INIT, ++ kbase_api_mem_exec_init, ++ struct kbase_ioctl_mem_exec_init, ++ kctx); ++ break; ++ case KBASE_IOCTL_MEM_SYNC: ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_SYNC, ++ kbase_api_mem_sync, ++ struct kbase_ioctl_mem_sync, ++ kctx); ++ break; ++ case KBASE_IOCTL_MEM_FIND_CPU_OFFSET: ++ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_FIND_CPU_OFFSET, ++ kbase_api_mem_find_cpu_offset, ++ union kbase_ioctl_mem_find_cpu_offset, ++ kctx); ++ break; ++ case KBASE_IOCTL_MEM_FIND_GPU_START_AND_OFFSET: ++ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_FIND_GPU_START_AND_OFFSET, ++ kbase_api_mem_find_gpu_start_and_offset, ++ union kbase_ioctl_mem_find_gpu_start_and_offset, ++ kctx); ++ break; ++ case KBASE_IOCTL_GET_CONTEXT_ID: ++ KBASE_HANDLE_IOCTL_OUT(KBASE_IOCTL_GET_CONTEXT_ID, ++ kbase_api_get_context_id, ++ struct kbase_ioctl_get_context_id, ++ kctx); ++ break; ++ case KBASE_IOCTL_TLSTREAM_ACQUIRE: ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_TLSTREAM_ACQUIRE, ++ kbase_api_tlstream_acquire, ++ struct kbase_ioctl_tlstream_acquire, ++ kctx); ++ break; ++ case KBASE_IOCTL_TLSTREAM_FLUSH: ++ KBASE_HANDLE_IOCTL(KBASE_IOCTL_TLSTREAM_FLUSH, ++ kbase_api_tlstream_flush, ++ kctx); ++ break; ++ case KBASE_IOCTL_MEM_COMMIT: ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_COMMIT, ++ kbase_api_mem_commit, ++ struct kbase_ioctl_mem_commit, ++ kctx); ++ break; ++ case KBASE_IOCTL_MEM_ALIAS: ++ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_ALIAS, ++ kbase_api_mem_alias, ++ union kbase_ioctl_mem_alias, ++ kctx); ++ break; ++ case KBASE_IOCTL_MEM_IMPORT: ++ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_IMPORT, ++ kbase_api_mem_import, ++ union kbase_ioctl_mem_import, ++ kctx); ++ break; ++ case KBASE_IOCTL_MEM_FLAGS_CHANGE: ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_FLAGS_CHANGE, ++ kbase_api_mem_flags_change, ++ struct kbase_ioctl_mem_flags_change, ++ kctx); ++ break; ++ case KBASE_IOCTL_STREAM_CREATE: ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_STREAM_CREATE, ++ kbase_api_stream_create, ++ struct kbase_ioctl_stream_create, ++ kctx); ++ break; ++ case KBASE_IOCTL_FENCE_VALIDATE: ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_FENCE_VALIDATE, ++ kbase_api_fence_validate, ++ struct kbase_ioctl_fence_validate, ++ kctx); ++ break; ++ case KBASE_IOCTL_MEM_PROFILE_ADD: ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_PROFILE_ADD, ++ kbase_api_mem_profile_add, ++ struct kbase_ioctl_mem_profile_add, ++ kctx); ++ break; ++ ++#if !MALI_USE_CSF ++ case KBASE_IOCTL_SOFT_EVENT_UPDATE: ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_SOFT_EVENT_UPDATE, ++ kbase_api_soft_event_update, ++ struct kbase_ioctl_soft_event_update, ++ kctx); ++ break; ++#endif /* !MALI_USE_CSF */ ++ ++ case KBASE_IOCTL_STICKY_RESOURCE_MAP: ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_STICKY_RESOURCE_MAP, ++ kbase_api_sticky_resource_map, ++ struct kbase_ioctl_sticky_resource_map, ++ kctx); ++ break; ++ case KBASE_IOCTL_STICKY_RESOURCE_UNMAP: ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_STICKY_RESOURCE_UNMAP, ++ kbase_api_sticky_resource_unmap, ++ struct kbase_ioctl_sticky_resource_unmap, ++ kctx); ++ break; ++ ++ /* Instrumentation. */ ++#if !MALI_USE_CSF ++ case KBASE_IOCTL_KINSTR_JM_FD: ++ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_KINSTR_JM_FD, ++ kbase_api_kinstr_jm_fd, ++ union kbase_kinstr_jm_fd, ++ kctx); ++ break; ++#endif ++ case KBASE_IOCTL_HWCNT_READER_SETUP: ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_HWCNT_READER_SETUP, ++ kbase_api_hwcnt_reader_setup, ++ struct kbase_ioctl_hwcnt_reader_setup, ++ kctx); ++ break; ++ case KBASE_IOCTL_HWCNT_ENABLE: ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_HWCNT_ENABLE, ++ kbase_api_hwcnt_enable, ++ struct kbase_ioctl_hwcnt_enable, ++ kctx); ++ break; ++ case KBASE_IOCTL_HWCNT_DUMP: ++ KBASE_HANDLE_IOCTL(KBASE_IOCTL_HWCNT_DUMP, ++ kbase_api_hwcnt_dump, ++ kctx); ++ break; ++ case KBASE_IOCTL_HWCNT_CLEAR: ++ KBASE_HANDLE_IOCTL(KBASE_IOCTL_HWCNT_CLEAR, ++ kbase_api_hwcnt_clear, ++ kctx); ++ break; ++ case KBASE_IOCTL_GET_CPU_GPU_TIMEINFO: ++ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_GET_CPU_GPU_TIMEINFO, ++ kbase_api_get_cpu_gpu_timeinfo, ++ union kbase_ioctl_get_cpu_gpu_timeinfo, ++ kctx); ++ break; ++#ifdef CONFIG_MALI_BIFROST_NO_MALI ++ case KBASE_IOCTL_HWCNT_SET: ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_HWCNT_SET, ++ kbase_api_hwcnt_set, ++ struct kbase_ioctl_hwcnt_values, ++ kctx); ++ break; ++#endif ++#ifdef CONFIG_MALI_CINSTR_GWT ++ case KBASE_IOCTL_CINSTR_GWT_START: ++ KBASE_HANDLE_IOCTL(KBASE_IOCTL_CINSTR_GWT_START, ++ kbase_gpu_gwt_start, ++ kctx); ++ break; ++ case KBASE_IOCTL_CINSTR_GWT_STOP: ++ KBASE_HANDLE_IOCTL(KBASE_IOCTL_CINSTR_GWT_STOP, ++ kbase_gpu_gwt_stop, ++ kctx); ++ break; ++ case KBASE_IOCTL_CINSTR_GWT_DUMP: ++ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_CINSTR_GWT_DUMP, ++ kbase_gpu_gwt_dump, ++ union kbase_ioctl_cinstr_gwt_dump, ++ kctx); ++ break; ++#endif ++#if MALI_USE_CSF ++ case KBASE_IOCTL_CS_EVENT_SIGNAL: ++ KBASE_HANDLE_IOCTL(KBASE_IOCTL_CS_EVENT_SIGNAL, ++ kbasep_cs_event_signal, ++ kctx); ++ break; ++ case KBASE_IOCTL_CS_QUEUE_REGISTER: ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_CS_QUEUE_REGISTER, ++ kbasep_cs_queue_register, ++ struct kbase_ioctl_cs_queue_register, ++ kctx); ++ break; ++ case KBASE_IOCTL_CS_QUEUE_TERMINATE: ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_CS_QUEUE_TERMINATE, ++ kbasep_cs_queue_terminate, ++ struct kbase_ioctl_cs_queue_terminate, ++ kctx); ++ break; ++ case KBASE_IOCTL_CS_QUEUE_BIND: ++ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_CS_QUEUE_BIND, ++ kbasep_cs_queue_bind, ++ union kbase_ioctl_cs_queue_bind, ++ kctx); ++ break; ++ case KBASE_IOCTL_CS_QUEUE_KICK: ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_CS_QUEUE_KICK, ++ kbasep_cs_queue_kick, ++ struct kbase_ioctl_cs_queue_kick, ++ kctx); ++ break; ++ case KBASE_IOCTL_CS_QUEUE_GROUP_CREATE: ++ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_CS_QUEUE_GROUP_CREATE, ++ kbasep_cs_queue_group_create, ++ union kbase_ioctl_cs_queue_group_create, ++ kctx); ++ break; ++ case KBASE_IOCTL_CS_QUEUE_GROUP_TERMINATE: ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_CS_QUEUE_GROUP_TERMINATE, ++ kbasep_cs_queue_group_terminate, ++ struct kbase_ioctl_cs_queue_group_term, ++ kctx); ++ break; ++ case KBASE_IOCTL_KCPU_QUEUE_CREATE: ++ KBASE_HANDLE_IOCTL_OUT(KBASE_IOCTL_KCPU_QUEUE_CREATE, ++ kbasep_kcpu_queue_new, ++ struct kbase_ioctl_kcpu_queue_new, ++ kctx); ++ break; ++ case KBASE_IOCTL_KCPU_QUEUE_DELETE: ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_KCPU_QUEUE_DELETE, ++ kbasep_kcpu_queue_delete, ++ struct kbase_ioctl_kcpu_queue_delete, ++ kctx); ++ break; ++ case KBASE_IOCTL_KCPU_QUEUE_ENQUEUE: ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_KCPU_QUEUE_ENQUEUE, ++ kbasep_kcpu_queue_enqueue, ++ struct kbase_ioctl_kcpu_queue_enqueue, ++ kctx); ++ break; ++ case KBASE_IOCTL_CS_TILER_HEAP_INIT: ++ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_CS_TILER_HEAP_INIT, ++ kbasep_cs_tiler_heap_init, ++ union kbase_ioctl_cs_tiler_heap_init, ++ kctx); ++ break; ++ case KBASE_IOCTL_CS_TILER_HEAP_TERM: ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_CS_TILER_HEAP_TERM, ++ kbasep_cs_tiler_heap_term, ++ struct kbase_ioctl_cs_tiler_heap_term, ++ kctx); ++ break; ++ case KBASE_IOCTL_CS_GET_GLB_IFACE: ++ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_CS_GET_GLB_IFACE, ++ kbase_ioctl_cs_get_glb_iface, ++ union kbase_ioctl_cs_get_glb_iface, ++ kctx); ++ break; ++#endif /* MALI_USE_CSF */ ++#if MALI_UNIT_TEST ++ case KBASE_IOCTL_TLSTREAM_TEST: ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_TLSTREAM_TEST, ++ kbase_api_tlstream_test, ++ struct kbase_ioctl_tlstream_test, ++ kctx); ++ break; ++ case KBASE_IOCTL_TLSTREAM_STATS: ++ KBASE_HANDLE_IOCTL_OUT(KBASE_IOCTL_TLSTREAM_STATS, ++ kbase_api_tlstream_stats, ++ struct kbase_ioctl_tlstream_stats, ++ kctx); ++ break; ++#endif /* MALI_UNIT_TEST */ ++ } ++ ++ dev_warn(kbdev->dev, "Unknown ioctl 0x%x nr:%d", cmd, _IOC_NR(cmd)); ++ ++ return -ENOIOCTLCMD; ++} ++ ++#if MALI_USE_CSF ++static ssize_t kbase_read(struct file *filp, char __user *buf, size_t count, loff_t *f_pos) ++{ ++ struct kbase_file *const kfile = filp->private_data; ++ struct kbase_context *const kctx = ++ kbase_file_get_kctx_if_setup_complete(kfile); ++ struct base_csf_notification event_data = { ++ .type = BASE_CSF_NOTIFICATION_EVENT }; ++ const size_t data_size = sizeof(event_data); ++ bool read_event = false, read_error = false; ++ ++ if (unlikely(!kctx)) ++ return -EPERM; ++ ++ if (atomic_read(&kctx->event_count)) ++ read_event = true; ++ else ++ read_error = kbase_csf_read_error(kctx, &event_data); ++ ++ if (!read_event && !read_error) { ++ /* This condition is not treated as an error. ++ * It is possible that event handling thread was woken up due ++ * to a fault/error that occurred for a queue group, but before ++ * the corresponding fault data was read by the thread the ++ * queue group was already terminated by the userspace. ++ */ ++ dev_dbg(kctx->kbdev->dev, "Neither event nor error signaled"); ++ } ++ ++ if (copy_to_user(buf, &event_data, data_size) != 0) { ++ dev_warn(kctx->kbdev->dev, ++ "Failed to copy data\n"); ++ return -EFAULT; ++ } ++ ++ if (read_event) ++ atomic_set(&kctx->event_count, 0); ++ ++ return data_size; ++} ++#else /* MALI_USE_CSF */ ++static ssize_t kbase_read(struct file *filp, char __user *buf, size_t count, loff_t *f_pos) ++{ ++ struct kbase_file *const kfile = filp->private_data; ++ struct kbase_context *const kctx = ++ kbase_file_get_kctx_if_setup_complete(kfile); ++ struct base_jd_event_v2 uevent; ++ int out_count = 0; ++ ++ if (unlikely(!kctx)) ++ return -EPERM; ++ ++ if (count < sizeof(uevent)) ++ return -ENOBUFS; ++ ++ do { ++ while (kbase_event_dequeue(kctx, &uevent)) { ++ if (out_count > 0) ++ goto out; ++ ++ if (filp->f_flags & O_NONBLOCK) ++ return -EAGAIN; ++ ++ if (wait_event_interruptible(kctx->event_queue, ++ kbase_event_pending(kctx)) != 0) ++ return -ERESTARTSYS; ++ } ++ if (uevent.event_code == BASE_JD_EVENT_DRV_TERMINATED) { ++ if (out_count == 0) ++ return -EPIPE; ++ goto out; ++ } ++ ++ if (copy_to_user(buf, &uevent, sizeof(uevent)) != 0) ++ return -EFAULT; ++ ++ buf += sizeof(uevent); ++ out_count++; ++ count -= sizeof(uevent); ++ } while (count >= sizeof(uevent)); ++ ++ out: ++ return out_count * sizeof(uevent); ++} ++#endif /* MALI_USE_CSF */ ++ ++static unsigned int kbase_poll(struct file *filp, poll_table *wait) ++{ ++ struct kbase_file *const kfile = filp->private_data; ++ struct kbase_context *const kctx = ++ kbase_file_get_kctx_if_setup_complete(kfile); ++ ++ if (unlikely(!kctx)) ++ return POLLERR; ++ ++ poll_wait(filp, &kctx->event_queue, wait); ++ if (kbase_event_pending(kctx)) ++ return POLLIN | POLLRDNORM; ++ ++ return 0; ++} ++ ++void kbase_event_wakeup(struct kbase_context *kctx) ++{ ++ KBASE_DEBUG_ASSERT(kctx); ++ ++ wake_up_interruptible(&kctx->event_queue); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_event_wakeup); ++ ++#if MALI_USE_CSF ++int kbase_event_pending(struct kbase_context *ctx) ++{ ++ WARN_ON_ONCE(!ctx); ++ ++ return (atomic_read(&ctx->event_count) != 0) || ++ kbase_csf_error_pending(ctx); ++} ++#else ++int kbase_event_pending(struct kbase_context *ctx) ++{ ++ KBASE_DEBUG_ASSERT(ctx); ++ ++ return (atomic_read(&ctx->event_count) != 0) || ++ (atomic_read(&ctx->event_closed) != 0); ++} ++#endif ++ ++KBASE_EXPORT_TEST_API(kbase_event_pending); ++ ++static int kbase_mmap(struct file *const filp, struct vm_area_struct *const vma) ++{ ++ struct kbase_file *const kfile = filp->private_data; ++ struct kbase_context *const kctx = ++ kbase_file_get_kctx_if_setup_complete(kfile); ++ ++ if (unlikely(!kctx)) ++ return -EPERM; ++ ++ return kbase_context_mmap(kctx, vma); ++} ++ ++static int kbase_check_flags(int flags) ++{ ++ /* Enforce that the driver keeps the O_CLOEXEC flag so that execve() always ++ * closes the file descriptor in a child process. ++ */ ++ if (0 == (flags & O_CLOEXEC)) ++ return -EINVAL; ++ ++ return 0; ++} ++ ++static unsigned long kbase_get_unmapped_area(struct file *const filp, ++ const unsigned long addr, const unsigned long len, ++ const unsigned long pgoff, const unsigned long flags) ++{ ++ struct kbase_file *const kfile = filp->private_data; ++ struct kbase_context *const kctx = ++ kbase_file_get_kctx_if_setup_complete(kfile); ++ ++ if (unlikely(!kctx)) ++ return -EPERM; ++ ++ return kbase_context_get_unmapped_area(kctx, addr, len, pgoff, flags); ++} ++ ++static const struct file_operations kbase_fops = { ++ .owner = THIS_MODULE, ++ .open = kbase_open, ++ .release = kbase_release, ++ .read = kbase_read, ++ .poll = kbase_poll, ++ .unlocked_ioctl = kbase_ioctl, ++ .compat_ioctl = kbase_ioctl, ++ .mmap = kbase_mmap, ++ .check_flags = kbase_check_flags, ++ .get_unmapped_area = kbase_get_unmapped_area, ++}; ++ ++/** ++ * show_policy - Show callback for the power_policy sysfs file. ++ * ++ * This function is called to get the contents of the power_policy sysfs ++ * file. This is a list of the available policies with the currently active one ++ * surrounded by square brackets. ++ * ++ * @dev: The device this sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The output buffer for the sysfs file contents ++ * ++ * Return: The number of bytes output to @buf. ++ */ ++static ssize_t show_policy(struct device *dev, struct device_attribute *attr, char *const buf) ++{ ++ struct kbase_device *kbdev; ++ const struct kbase_pm_policy *current_policy; ++ const struct kbase_pm_policy *const *policy_list; ++ int policy_count; ++ int i; ++ ssize_t ret = 0; ++ ++ kbdev = to_kbase_device(dev); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ current_policy = kbase_pm_get_policy(kbdev); ++ ++ policy_count = kbase_pm_list_policies(kbdev, &policy_list); ++ ++ for (i = 0; i < policy_count && ret < PAGE_SIZE; i++) { ++ if (policy_list[i] == current_policy) ++ ret += scnprintf(buf + ret, PAGE_SIZE - ret, "[%s] ", policy_list[i]->name); ++ else ++ ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s ", policy_list[i]->name); ++ } ++ ++ if (ret < PAGE_SIZE - 1) { ++ ret += scnprintf(buf + ret, PAGE_SIZE - ret, "\n"); ++ } else { ++ buf[PAGE_SIZE - 2] = '\n'; ++ buf[PAGE_SIZE - 1] = '\0'; ++ ret = PAGE_SIZE - 1; ++ } ++ ++ return ret; ++} ++ ++/** ++ * set_policy - Store callback for the power_policy sysfs file. ++ * ++ * This function is called when the power_policy sysfs file is written to. ++ * It matches the requested policy against the available policies and if a ++ * matching policy is found calls kbase_pm_set_policy() to change the ++ * policy. ++ * ++ * @dev: The device with sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The value written to the sysfs file ++ * @count: The number of bytes to write to the sysfs file ++ * ++ * Return: @count if the function succeeded. An error code on failure. ++ */ ++static ssize_t set_policy(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) ++{ ++ struct kbase_device *kbdev; ++ const struct kbase_pm_policy *new_policy = NULL; ++ const struct kbase_pm_policy *const *policy_list; ++ int policy_count; ++ int i; ++ ++ kbdev = to_kbase_device(dev); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ policy_count = kbase_pm_list_policies(kbdev, &policy_list); ++ ++ for (i = 0; i < policy_count; i++) { ++ if (sysfs_streq(policy_list[i]->name, buf)) { ++ new_policy = policy_list[i]; ++ break; ++ } ++ } ++ ++ if (!new_policy) { ++ dev_err(dev, "power_policy: policy not found\n"); ++ return -EINVAL; ++ } ++ ++ kbase_pm_set_policy(kbdev, new_policy); ++ ++ return count; ++} ++ ++/* ++ * The sysfs file power_policy. ++ * ++ * This is used for obtaining information about the available policies, ++ * determining which policy is currently active, and changing the active ++ * policy. ++ */ ++static DEVICE_ATTR(power_policy, S_IRUGO | S_IWUSR, show_policy, set_policy); ++ ++/* ++ * show_core_mask - Show callback for the core_mask sysfs file. ++ * ++ * This function is called to get the contents of the core_mask sysfs file. ++ * ++ * @dev: The device this sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The output buffer for the sysfs file contents ++ * ++ * Return: The number of bytes output to @buf. ++ */ ++static ssize_t show_core_mask(struct device *dev, struct device_attribute *attr, char * const buf) ++{ ++ struct kbase_device *kbdev; ++ ssize_t ret = 0; ++ ++ kbdev = to_kbase_device(dev); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ ret += scnprintf(buf + ret, PAGE_SIZE - ret, ++ "Current core mask (JS0) : 0x%llX\n", ++ kbdev->pm.debug_core_mask[0]); ++ ret += scnprintf(buf + ret, PAGE_SIZE - ret, ++ "Current core mask (JS1) : 0x%llX\n", ++ kbdev->pm.debug_core_mask[1]); ++ ret += scnprintf(buf + ret, PAGE_SIZE - ret, ++ "Current core mask (JS2) : 0x%llX\n", ++ kbdev->pm.debug_core_mask[2]); ++ ret += scnprintf(buf + ret, PAGE_SIZE - ret, ++ "Available core mask : 0x%llX\n", ++ kbdev->gpu_props.props.raw_props.shader_present); ++ ++ return ret; ++} ++ ++/** ++ * set_core_mask - Store callback for the core_mask sysfs file. ++ * ++ * This function is called when the core_mask sysfs file is written to. ++ * ++ * @dev: The device with sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The value written to the sysfs file ++ * @count: The number of bytes to write to the sysfs file ++ * ++ * Return: @count if the function succeeded. An error code on failure. ++ */ ++static ssize_t set_core_mask(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) ++{ ++ struct kbase_device *kbdev; ++ u64 new_core_mask[3]; ++ int items, i; ++ ssize_t err = count; ++ unsigned long flags; ++ u64 shader_present, group0_core_mask; ++ ++ kbdev = to_kbase_device(dev); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ items = sscanf(buf, "%llx %llx %llx", ++ &new_core_mask[0], &new_core_mask[1], ++ &new_core_mask[2]); ++ ++ if (items != 1 && items != 3) { ++ dev_err(kbdev->dev, "Couldn't process core mask write operation.\n" ++ "Use format \n" ++ "or \n"); ++ err = -EINVAL; ++ goto end; ++ } ++ ++ if (items == 1) ++ new_core_mask[1] = new_core_mask[2] = new_core_mask[0]; ++ ++ mutex_lock(&kbdev->pm.lock); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ shader_present = kbdev->gpu_props.props.raw_props.shader_present; ++ group0_core_mask = kbdev->gpu_props.props.coherency_info.group[0].core_mask; ++ ++ for (i = 0; i < 3; ++i) { ++ if ((new_core_mask[i] & shader_present) != new_core_mask[i]) { ++ dev_err(dev, "Invalid core mask 0x%llX for JS %d: Includes non-existent cores (present = 0x%llX)", ++ new_core_mask[i], i, shader_present); ++ err = -EINVAL; ++ goto unlock; ++ ++ } else if (!(new_core_mask[i] & shader_present & kbdev->pm.backend.ca_cores_enabled)) { ++ dev_err(dev, "Invalid core mask 0x%llX for JS %d: No intersection with currently available cores (present = 0x%llX, CA enabled = 0x%llX\n", ++ new_core_mask[i], i, ++ kbdev->gpu_props.props.raw_props.shader_present, ++ kbdev->pm.backend.ca_cores_enabled); ++ err = -EINVAL; ++ goto unlock; ++ ++ } else if (!(new_core_mask[i] & group0_core_mask)) { ++ dev_err(dev, "Invalid core mask 0x%llX for JS %d: No intersection with group 0 core mask 0x%llX\n", ++ new_core_mask[i], i, group0_core_mask); ++ err = -EINVAL; ++ goto unlock; ++ } ++ } ++ ++ if (kbdev->pm.debug_core_mask[0] != new_core_mask[0] || ++ kbdev->pm.debug_core_mask[1] != ++ new_core_mask[1] || ++ kbdev->pm.debug_core_mask[2] != ++ new_core_mask[2]) { ++ ++ kbase_pm_set_debug_core_mask(kbdev, new_core_mask[0], ++ new_core_mask[1], new_core_mask[2]); ++ } ++ ++unlock: ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&kbdev->pm.lock); ++end: ++ return err; ++} ++ ++/* ++ * The sysfs file core_mask. ++ * ++ * This is used to restrict shader core availability for debugging purposes. ++ * Reading it will show the current core mask and the mask of cores available. ++ * Writing to it will set the current core mask. ++ */ ++static DEVICE_ATTR(core_mask, S_IRUGO | S_IWUSR, show_core_mask, set_core_mask); ++ ++#if !MALI_USE_CSF ++/** ++ * set_soft_job_timeout - Store callback for the soft_job_timeout sysfs ++ * file. ++ * ++ * @dev: The device this sysfs file is for. ++ * @attr: The attributes of the sysfs file. ++ * @buf: The value written to the sysfs file. ++ * @count: The number of bytes to write to the sysfs file. ++ * ++ * This allows setting the timeout for software jobs. Waiting soft event wait ++ * jobs will be cancelled after this period expires, while soft fence wait jobs ++ * will print debug information if the fence debug feature is enabled. ++ * ++ * This is expressed in milliseconds. ++ * ++ * Return: count if the function succeeded. An error code on failure. ++ */ ++static ssize_t set_soft_job_timeout(struct device *dev, ++ struct device_attribute *attr, ++ const char *buf, size_t count) ++{ ++ struct kbase_device *kbdev; ++ int soft_job_timeout_ms; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ if ((kstrtoint(buf, 0, &soft_job_timeout_ms) != 0) || ++ (soft_job_timeout_ms <= 0)) ++ return -EINVAL; ++ ++ atomic_set(&kbdev->js_data.soft_job_timeout_ms, ++ soft_job_timeout_ms); ++ ++ return count; ++} ++ ++/** ++ * show_soft_job_timeout - Show callback for the soft_job_timeout sysfs ++ * file. ++ * ++ * This will return the timeout for the software jobs. ++ * ++ * @dev: The device this sysfs file is for. ++ * @attr: The attributes of the sysfs file. ++ * @buf: The output buffer for the sysfs file contents. ++ * ++ * Return: The number of bytes output to buf. ++ */ ++static ssize_t show_soft_job_timeout(struct device *dev, ++ struct device_attribute *attr, ++ char * const buf) ++{ ++ struct kbase_device *kbdev; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ return scnprintf(buf, PAGE_SIZE, "%i\n", ++ atomic_read(&kbdev->js_data.soft_job_timeout_ms)); ++} ++ ++static DEVICE_ATTR(soft_job_timeout, S_IRUGO | S_IWUSR, ++ show_soft_job_timeout, set_soft_job_timeout); ++ ++static u32 timeout_ms_to_ticks(struct kbase_device *kbdev, long timeout_ms, ++ int default_ticks, u32 old_ticks) ++{ ++ if (timeout_ms > 0) { ++ u64 ticks = timeout_ms * 1000000ULL; ++ do_div(ticks, kbdev->js_data.scheduling_period_ns); ++ if (!ticks) ++ return 1; ++ return ticks; ++ } else if (timeout_ms < 0) { ++ return default_ticks; ++ } else { ++ return old_ticks; ++ } ++} ++ ++/** ++ * set_js_timeouts - Store callback for the js_timeouts sysfs file. ++ * ++ * This function is called to get the contents of the js_timeouts sysfs ++ * file. This file contains five values separated by whitespace. The values ++ * are basically the same as %JS_SOFT_STOP_TICKS, %JS_HARD_STOP_TICKS_SS, ++ * %JS_HARD_STOP_TICKS_DUMPING, %JS_RESET_TICKS_SS, %JS_RESET_TICKS_DUMPING ++ * configuration values (in that order), with the difference that the js_timeout ++ * values are expressed in MILLISECONDS. ++ * ++ * The js_timeouts sysfile file allows the current values in ++ * use by the job scheduler to get override. Note that a value needs to ++ * be other than 0 for it to override the current job scheduler value. ++ * ++ * @dev: The device with sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The value written to the sysfs file ++ * @count: The number of bytes to write to the sysfs file ++ * ++ * Return: @count if the function succeeded. An error code on failure. ++ */ ++static ssize_t set_js_timeouts(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) ++{ ++ struct kbase_device *kbdev; ++ int items; ++ long js_soft_stop_ms; ++ long js_soft_stop_ms_cl; ++ long js_hard_stop_ms_ss; ++ long js_hard_stop_ms_cl; ++ long js_hard_stop_ms_dumping; ++ long js_reset_ms_ss; ++ long js_reset_ms_cl; ++ long js_reset_ms_dumping; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ items = sscanf(buf, "%ld %ld %ld %ld %ld %ld %ld %ld", ++ &js_soft_stop_ms, &js_soft_stop_ms_cl, ++ &js_hard_stop_ms_ss, &js_hard_stop_ms_cl, ++ &js_hard_stop_ms_dumping, &js_reset_ms_ss, ++ &js_reset_ms_cl, &js_reset_ms_dumping); ++ ++ if (items == 8) { ++ struct kbasep_js_device_data *js_data = &kbdev->js_data; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++#define UPDATE_TIMEOUT(ticks_name, ms_name, default) do {\ ++ js_data->ticks_name = timeout_ms_to_ticks(kbdev, ms_name, \ ++ default, js_data->ticks_name); \ ++ dev_dbg(kbdev->dev, "Overriding " #ticks_name \ ++ " with %lu ticks (%lu ms)\n", \ ++ (unsigned long)js_data->ticks_name, \ ++ ms_name); \ ++ } while (0) ++ ++ UPDATE_TIMEOUT(soft_stop_ticks, js_soft_stop_ms, ++ DEFAULT_JS_SOFT_STOP_TICKS); ++ UPDATE_TIMEOUT(soft_stop_ticks_cl, js_soft_stop_ms_cl, ++ DEFAULT_JS_SOFT_STOP_TICKS_CL); ++ UPDATE_TIMEOUT(hard_stop_ticks_ss, js_hard_stop_ms_ss, ++ DEFAULT_JS_HARD_STOP_TICKS_SS); ++ UPDATE_TIMEOUT(hard_stop_ticks_cl, js_hard_stop_ms_cl, ++ DEFAULT_JS_HARD_STOP_TICKS_CL); ++ UPDATE_TIMEOUT(hard_stop_ticks_dumping, ++ js_hard_stop_ms_dumping, ++ DEFAULT_JS_HARD_STOP_TICKS_DUMPING); ++ UPDATE_TIMEOUT(gpu_reset_ticks_ss, js_reset_ms_ss, ++ DEFAULT_JS_RESET_TICKS_SS); ++ UPDATE_TIMEOUT(gpu_reset_ticks_cl, js_reset_ms_cl, ++ DEFAULT_JS_RESET_TICKS_CL); ++ UPDATE_TIMEOUT(gpu_reset_ticks_dumping, js_reset_ms_dumping, ++ DEFAULT_JS_RESET_TICKS_DUMPING); ++ ++ kbase_js_set_timeouts(kbdev); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ return count; ++ } ++ ++ dev_err(kbdev->dev, "Couldn't process js_timeouts write operation.\n" ++ "Use format \n" ++ "Write 0 for no change, -1 to restore default timeout\n"); ++ return -EINVAL; ++} ++ ++static unsigned long get_js_timeout_in_ms( ++ u32 scheduling_period_ns, ++ u32 ticks) ++{ ++ u64 ms = (u64)ticks * scheduling_period_ns; ++ ++ do_div(ms, 1000000UL); ++ return ms; ++} ++ ++/** ++ * show_js_timeouts - Show callback for the js_timeouts sysfs file. ++ * ++ * This function is called to get the contents of the js_timeouts sysfs ++ * file. It returns the last set values written to the js_timeouts sysfs file. ++ * If the file didn't get written yet, the values will be current setting in ++ * use. ++ * @dev: The device this sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The output buffer for the sysfs file contents ++ * ++ * Return: The number of bytes output to @buf. ++ */ ++static ssize_t show_js_timeouts(struct device *dev, struct device_attribute *attr, char * const buf) ++{ ++ struct kbase_device *kbdev; ++ ssize_t ret; ++ unsigned long js_soft_stop_ms; ++ unsigned long js_soft_stop_ms_cl; ++ unsigned long js_hard_stop_ms_ss; ++ unsigned long js_hard_stop_ms_cl; ++ unsigned long js_hard_stop_ms_dumping; ++ unsigned long js_reset_ms_ss; ++ unsigned long js_reset_ms_cl; ++ unsigned long js_reset_ms_dumping; ++ u32 scheduling_period_ns; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ scheduling_period_ns = kbdev->js_data.scheduling_period_ns; ++ ++#define GET_TIMEOUT(name) get_js_timeout_in_ms(\ ++ scheduling_period_ns, \ ++ kbdev->js_data.name) ++ ++ js_soft_stop_ms = GET_TIMEOUT(soft_stop_ticks); ++ js_soft_stop_ms_cl = GET_TIMEOUT(soft_stop_ticks_cl); ++ js_hard_stop_ms_ss = GET_TIMEOUT(hard_stop_ticks_ss); ++ js_hard_stop_ms_cl = GET_TIMEOUT(hard_stop_ticks_cl); ++ js_hard_stop_ms_dumping = GET_TIMEOUT(hard_stop_ticks_dumping); ++ js_reset_ms_ss = GET_TIMEOUT(gpu_reset_ticks_ss); ++ js_reset_ms_cl = GET_TIMEOUT(gpu_reset_ticks_cl); ++ js_reset_ms_dumping = GET_TIMEOUT(gpu_reset_ticks_dumping); ++ ++#undef GET_TIMEOUT ++ ++ ret = scnprintf(buf, PAGE_SIZE, "%lu %lu %lu %lu %lu %lu %lu %lu\n", ++ js_soft_stop_ms, js_soft_stop_ms_cl, ++ js_hard_stop_ms_ss, js_hard_stop_ms_cl, ++ js_hard_stop_ms_dumping, js_reset_ms_ss, ++ js_reset_ms_cl, js_reset_ms_dumping); ++ ++ if (ret >= PAGE_SIZE) { ++ buf[PAGE_SIZE - 2] = '\n'; ++ buf[PAGE_SIZE - 1] = '\0'; ++ ret = PAGE_SIZE - 1; ++ } ++ ++ return ret; ++} ++ ++/* ++ * The sysfs file js_timeouts. ++ * ++ * This is used to override the current job scheduler values for ++ * JS_STOP_STOP_TICKS_SS ++ * JS_STOP_STOP_TICKS_CL ++ * JS_HARD_STOP_TICKS_SS ++ * JS_HARD_STOP_TICKS_CL ++ * JS_HARD_STOP_TICKS_DUMPING ++ * JS_RESET_TICKS_SS ++ * JS_RESET_TICKS_CL ++ * JS_RESET_TICKS_DUMPING. ++ */ ++static DEVICE_ATTR(js_timeouts, S_IRUGO | S_IWUSR, show_js_timeouts, set_js_timeouts); ++ ++static u32 get_new_js_timeout( ++ u32 old_period, ++ u32 old_ticks, ++ u32 new_scheduling_period_ns) ++{ ++ u64 ticks = (u64)old_period * (u64)old_ticks; ++ do_div(ticks, new_scheduling_period_ns); ++ return ticks?ticks:1; ++} ++ ++/** ++ * set_js_scheduling_period - Store callback for the js_scheduling_period sysfs ++ * file ++ * @dev: The device the sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The value written to the sysfs file ++ * @count: The number of bytes to write to the sysfs file ++ * ++ * This function is called when the js_scheduling_period sysfs file is written ++ * to. It checks the data written, and if valid updates the js_scheduling_period ++ * value ++ * ++ * Return: @count if the function succeeded. An error code on failure. ++ */ ++static ssize_t set_js_scheduling_period(struct device *dev, ++ struct device_attribute *attr, const char *buf, size_t count) ++{ ++ struct kbase_device *kbdev; ++ int ret; ++ unsigned int js_scheduling_period; ++ u32 new_scheduling_period_ns; ++ u32 old_period; ++ struct kbasep_js_device_data *js_data; ++ unsigned long flags; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ js_data = &kbdev->js_data; ++ ++ ret = kstrtouint(buf, 0, &js_scheduling_period); ++ if (ret || !js_scheduling_period) { ++ dev_err(kbdev->dev, "Couldn't process js_scheduling_period write operation.\n" ++ "Use format \n"); ++ return -EINVAL; ++ } ++ ++ new_scheduling_period_ns = js_scheduling_period * 1000000; ++ ++ /* Update scheduling timeouts */ ++ mutex_lock(&js_data->runpool_mutex); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ /* If no contexts have been scheduled since js_timeouts was last written ++ * to, the new timeouts might not have been latched yet. So check if an ++ * update is pending and use the new values if necessary. */ ++ ++ /* Use previous 'new' scheduling period as a base if present. */ ++ old_period = js_data->scheduling_period_ns; ++ ++#define SET_TIMEOUT(name) \ ++ (js_data->name = get_new_js_timeout(\ ++ old_period, \ ++ kbdev->js_data.name, \ ++ new_scheduling_period_ns)) ++ ++ SET_TIMEOUT(soft_stop_ticks); ++ SET_TIMEOUT(soft_stop_ticks_cl); ++ SET_TIMEOUT(hard_stop_ticks_ss); ++ SET_TIMEOUT(hard_stop_ticks_cl); ++ SET_TIMEOUT(hard_stop_ticks_dumping); ++ SET_TIMEOUT(gpu_reset_ticks_ss); ++ SET_TIMEOUT(gpu_reset_ticks_cl); ++ SET_TIMEOUT(gpu_reset_ticks_dumping); ++ ++#undef SET_TIMEOUT ++ ++ js_data->scheduling_period_ns = new_scheduling_period_ns; ++ ++ kbase_js_set_timeouts(kbdev); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&js_data->runpool_mutex); ++ ++ dev_dbg(kbdev->dev, "JS scheduling period: %dms\n", ++ js_scheduling_period); ++ ++ return count; ++} ++ ++/** ++ * show_js_scheduling_period - Show callback for the js_scheduling_period sysfs ++ * entry. ++ * @dev: The device this sysfs file is for. ++ * @attr: The attributes of the sysfs file. ++ * @buf: The output buffer to receive the GPU information. ++ * ++ * This function is called to get the current period used for the JS scheduling ++ * period. ++ * ++ * Return: The number of bytes output to @buf. ++ */ ++static ssize_t show_js_scheduling_period(struct device *dev, ++ struct device_attribute *attr, char * const buf) ++{ ++ struct kbase_device *kbdev; ++ u32 period; ++ ssize_t ret; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ period = kbdev->js_data.scheduling_period_ns; ++ ++ ret = scnprintf(buf, PAGE_SIZE, "%d\n", ++ period / 1000000); ++ ++ return ret; ++} ++ ++static DEVICE_ATTR(js_scheduling_period, S_IRUGO | S_IWUSR, ++ show_js_scheduling_period, set_js_scheduling_period); ++ ++ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++static ssize_t set_js_softstop_always(struct device *dev, ++ struct device_attribute *attr, const char *buf, size_t count) ++{ ++ struct kbase_device *kbdev; ++ int ret; ++ int softstop_always; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ ret = kstrtoint(buf, 0, &softstop_always); ++ if (ret || ((softstop_always != 0) && (softstop_always != 1))) { ++ dev_err(kbdev->dev, "Couldn't process js_softstop_always write operation.\n" ++ "Use format \n"); ++ return -EINVAL; ++ } ++ ++ kbdev->js_data.softstop_always = (bool) softstop_always; ++ dev_dbg(kbdev->dev, "Support for softstop on a single context: %s\n", ++ (kbdev->js_data.softstop_always) ? ++ "Enabled" : "Disabled"); ++ return count; ++} ++ ++static ssize_t show_js_softstop_always(struct device *dev, ++ struct device_attribute *attr, char * const buf) ++{ ++ struct kbase_device *kbdev; ++ ssize_t ret; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ ret = scnprintf(buf, PAGE_SIZE, "%d\n", kbdev->js_data.softstop_always); ++ ++ if (ret >= PAGE_SIZE) { ++ buf[PAGE_SIZE - 2] = '\n'; ++ buf[PAGE_SIZE - 1] = '\0'; ++ ret = PAGE_SIZE - 1; ++ } ++ ++ return ret; ++} ++ ++/* ++ * By default, soft-stops are disabled when only a single context is present. ++ * The ability to enable soft-stop when only a single context is present can be ++ * used for debug and unit-testing purposes. ++ * (see CL t6xx_stress_1 unit-test as an example whereby this feature is used.) ++ */ ++static DEVICE_ATTR(js_softstop_always, S_IRUGO | S_IWUSR, show_js_softstop_always, set_js_softstop_always); ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++#endif /* !MALI_USE_CSF */ ++ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++typedef void (kbasep_debug_command_func) (struct kbase_device *); ++ ++enum kbasep_debug_command_code { ++ KBASEP_DEBUG_COMMAND_DUMPTRACE, ++ ++ /* This must be the last enum */ ++ KBASEP_DEBUG_COMMAND_COUNT ++}; ++ ++struct kbasep_debug_command { ++ char *str; ++ kbasep_debug_command_func *func; ++}; ++ ++void kbasep_ktrace_dump_wrapper(struct kbase_device *kbdev) ++{ ++ KBASE_KTRACE_DUMP(kbdev); ++} ++ ++/* Debug commands supported by the driver */ ++static const struct kbasep_debug_command debug_commands[] = { ++ { ++ .str = "dumptrace", ++ .func = &kbasep_ktrace_dump_wrapper, ++ } ++}; ++ ++/** ++ * show_debug - Show callback for the debug_command sysfs file. ++ * ++ * This function is called to get the contents of the debug_command sysfs ++ * file. This is a list of the available debug commands, separated by newlines. ++ * ++ * @dev: The device this sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The output buffer for the sysfs file contents ++ * ++ * Return: The number of bytes output to @buf. ++ */ ++static ssize_t show_debug(struct device *dev, struct device_attribute *attr, char * const buf) ++{ ++ struct kbase_device *kbdev; ++ int i; ++ ssize_t ret = 0; ++ ++ kbdev = to_kbase_device(dev); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ for (i = 0; i < KBASEP_DEBUG_COMMAND_COUNT && ret < PAGE_SIZE; i++) ++ ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s\n", debug_commands[i].str); ++ ++ if (ret >= PAGE_SIZE) { ++ buf[PAGE_SIZE - 2] = '\n'; ++ buf[PAGE_SIZE - 1] = '\0'; ++ ret = PAGE_SIZE - 1; ++ } ++ ++ return ret; ++} ++ ++/** ++ * issue_debug - Store callback for the debug_command sysfs file. ++ * ++ * This function is called when the debug_command sysfs file is written to. ++ * It matches the requested command against the available commands, and if ++ * a matching command is found calls the associated function from ++ * @debug_commands to issue the command. ++ * ++ * @dev: The device with sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The value written to the sysfs file ++ * @count: The number of bytes written to the sysfs file ++ * ++ * Return: @count if the function succeeded. An error code on failure. ++ */ ++static ssize_t issue_debug(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) ++{ ++ struct kbase_device *kbdev; ++ int i; ++ ++ kbdev = to_kbase_device(dev); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ for (i = 0; i < KBASEP_DEBUG_COMMAND_COUNT; i++) { ++ if (sysfs_streq(debug_commands[i].str, buf)) { ++ debug_commands[i].func(kbdev); ++ return count; ++ } ++ } ++ ++ /* Debug Command not found */ ++ dev_err(dev, "debug_command: command not known\n"); ++ return -EINVAL; ++} ++ ++/* The sysfs file debug_command. ++ * ++ * This is used to issue general debug commands to the device driver. ++ * Reading it will produce a list of debug commands, separated by newlines. ++ * Writing to it with one of those commands will issue said command. ++ */ ++static DEVICE_ATTR(debug_command, S_IRUGO | S_IWUSR, show_debug, issue_debug); ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++ ++/** ++ * kbase_show_gpuinfo - Show callback for the gpuinfo sysfs entry. ++ * @dev: The device this sysfs file is for. ++ * @attr: The attributes of the sysfs file. ++ * @buf: The output buffer to receive the GPU information. ++ * ++ * This function is called to get a description of the present Mali ++ * GPU via the gpuinfo sysfs entry. This includes the GPU family, the ++ * number of cores, the hardware version and the raw product id. For ++ * example ++ * ++ * Mali-T60x MP4 r0p0 0x6956 ++ * ++ * Return: The number of bytes output to @buf. ++ */ ++static ssize_t kbase_show_gpuinfo(struct device *dev, ++ struct device_attribute *attr, char *buf) ++{ ++ static const struct gpu_product_id_name { ++ unsigned id; ++ char *name; ++ } gpu_product_id_names[] = { ++ { .id = GPU_ID2_PRODUCT_TMIX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, ++ .name = "Mali-G71" }, ++ { .id = GPU_ID2_PRODUCT_THEX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, ++ .name = "Mali-G72" }, ++ { .id = GPU_ID2_PRODUCT_TSIX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, ++ .name = "Mali-G51" }, ++ { .id = GPU_ID2_PRODUCT_TNOX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, ++ .name = "Mali-G76" }, ++ { .id = GPU_ID2_PRODUCT_TDVX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, ++ .name = "Mali-G31" }, ++ { .id = GPU_ID2_PRODUCT_TGOX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, ++ .name = "Mali-G52" }, ++ { .id = GPU_ID2_PRODUCT_TTRX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, ++ .name = "Mali-G77" }, ++ { .id = GPU_ID2_PRODUCT_TBEX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, ++ .name = "Mali-G78" }, ++ { .id = GPU_ID2_PRODUCT_TBAX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, ++ .name = "Mali-TBAX" }, ++ { .id = GPU_ID2_PRODUCT_LBEX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, ++ .name = "Mali-G68" }, ++ { .id = GPU_ID2_PRODUCT_TNAX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, ++ .name = "Mali-G57" }, ++ { .id = GPU_ID2_PRODUCT_TODX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, ++ .name = "Mali-TODX" }, ++ { .id = GPU_ID2_PRODUCT_TGRX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, ++ .name = "Mali-TGRX" }, ++ { .id = GPU_ID2_PRODUCT_TVAX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, ++ .name = "Mali-TVAX" }, ++ { .id = GPU_ID2_PRODUCT_LODX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, ++ .name = "Mali-LODX" }, ++ { .id = GPU_ID2_PRODUCT_TTUX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, ++ .name = "Mali-TTUX" }, ++ { .id = GPU_ID2_PRODUCT_LTUX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, ++ .name = "Mali-LTUX" }, ++ { .id = GPU_ID2_PRODUCT_TE2X >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, ++ .name = "Mali-TE2X" }, ++ }; ++ const char *product_name = "(Unknown Mali GPU)"; ++ struct kbase_device *kbdev; ++ u32 gpu_id; ++ unsigned product_id, product_id_mask; ++ unsigned i; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; ++ product_id = gpu_id >> GPU_ID_VERSION_PRODUCT_ID_SHIFT; ++ product_id_mask = GPU_ID2_PRODUCT_MODEL >> GPU_ID_VERSION_PRODUCT_ID_SHIFT; ++ ++ for (i = 0; i < ARRAY_SIZE(gpu_product_id_names); ++i) { ++ const struct gpu_product_id_name *p = &gpu_product_id_names[i]; ++ ++ if ((p->id & product_id_mask) == ++ (product_id & product_id_mask)) { ++ product_name = p->name; ++ break; ++ } ++ } ++ ++ return scnprintf(buf, PAGE_SIZE, "%s %d cores r%dp%d 0x%04X\n", ++ product_name, kbdev->gpu_props.num_cores, ++ (gpu_id & GPU_ID_VERSION_MAJOR) >> GPU_ID_VERSION_MAJOR_SHIFT, ++ (gpu_id & GPU_ID_VERSION_MINOR) >> GPU_ID_VERSION_MINOR_SHIFT, ++ product_id); ++} ++static DEVICE_ATTR(gpuinfo, S_IRUGO, kbase_show_gpuinfo, NULL); ++ ++/** ++ * set_dvfs_period - Store callback for the dvfs_period sysfs file. ++ * @dev: The device with sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The value written to the sysfs file ++ * @count: The number of bytes written to the sysfs file ++ * ++ * This function is called when the dvfs_period sysfs file is written to. It ++ * checks the data written, and if valid updates the DVFS period variable, ++ * ++ * Return: @count if the function succeeded. An error code on failure. ++ */ ++static ssize_t set_dvfs_period(struct device *dev, ++ struct device_attribute *attr, const char *buf, size_t count) ++{ ++ struct kbase_device *kbdev; ++ int ret; ++ int dvfs_period; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ ret = kstrtoint(buf, 0, &dvfs_period); ++ if (ret || dvfs_period <= 0) { ++ dev_err(kbdev->dev, "Couldn't process dvfs_period write operation.\n" ++ "Use format \n"); ++ return -EINVAL; ++ } ++ ++ kbdev->pm.dvfs_period = dvfs_period; ++ dev_dbg(kbdev->dev, "DVFS period: %dms\n", dvfs_period); ++ ++ return count; ++} ++ ++/** ++ * show_dvfs_period - Show callback for the dvfs_period sysfs entry. ++ * @dev: The device this sysfs file is for. ++ * @attr: The attributes of the sysfs file. ++ * @buf: The output buffer to receive the GPU information. ++ * ++ * This function is called to get the current period used for the DVFS sample ++ * timer. ++ * ++ * Return: The number of bytes output to @buf. ++ */ ++static ssize_t show_dvfs_period(struct device *dev, ++ struct device_attribute *attr, char * const buf) ++{ ++ struct kbase_device *kbdev; ++ ssize_t ret; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ ret = scnprintf(buf, PAGE_SIZE, "%d\n", kbdev->pm.dvfs_period); ++ ++ return ret; ++} ++ ++static DEVICE_ATTR(dvfs_period, S_IRUGO | S_IWUSR, show_dvfs_period, ++ set_dvfs_period); ++ ++/** ++ * set_pm_poweroff - Store callback for the pm_poweroff sysfs file. ++ * @dev: The device with sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The value written to the sysfs file ++ * @count: The number of bytes written to the sysfs file ++ * ++ * This function is called when the pm_poweroff sysfs file is written to. ++ * ++ * This file contains three values separated by whitespace. The values ++ * are gpu_poweroff_time (the period of the poweroff timer, in ns), ++ * poweroff_shader_ticks (the number of poweroff timer ticks before an idle ++ * shader is powered off), and poweroff_gpu_ticks (the number of poweroff timer ++ * ticks before the GPU is powered off), in that order. ++ * ++ * Return: @count if the function succeeded. An error code on failure. ++ */ ++static ssize_t set_pm_poweroff(struct device *dev, ++ struct device_attribute *attr, const char *buf, size_t count) ++{ ++ struct kbase_device *kbdev; ++ struct kbasep_pm_tick_timer_state *stt; ++ int items; ++ u64 gpu_poweroff_time; ++ unsigned int poweroff_shader_ticks, poweroff_gpu_ticks; ++ unsigned long flags; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ items = sscanf(buf, "%llu %u %u", &gpu_poweroff_time, ++ &poweroff_shader_ticks, ++ &poweroff_gpu_ticks); ++ if (items != 3) { ++ dev_err(kbdev->dev, "Couldn't process pm_poweroff write operation.\n" ++ "Use format \n"); ++ return -EINVAL; ++ } ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ stt = &kbdev->pm.backend.shader_tick_timer; ++ stt->configured_interval = HR_TIMER_DELAY_NSEC(gpu_poweroff_time); ++ stt->configured_ticks = poweroff_shader_ticks; ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ if (poweroff_gpu_ticks != 0) ++ dev_warn(kbdev->dev, "Separate GPU poweroff delay no longer supported.\n"); ++ ++ return count; ++} ++ ++/** ++ * show_pm_poweroff - Show callback for the pm_poweroff sysfs entry. ++ * @dev: The device this sysfs file is for. ++ * @attr: The attributes of the sysfs file. ++ * @buf: The output buffer to receive the GPU information. ++ * ++ * This function is called to get the current period used for the DVFS sample ++ * timer. ++ * ++ * Return: The number of bytes output to @buf. ++ */ ++static ssize_t show_pm_poweroff(struct device *dev, ++ struct device_attribute *attr, char * const buf) ++{ ++ struct kbase_device *kbdev; ++ struct kbasep_pm_tick_timer_state *stt; ++ ssize_t ret; ++ unsigned long flags; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ stt = &kbdev->pm.backend.shader_tick_timer; ++ ret = scnprintf(buf, PAGE_SIZE, "%llu %u 0\n", ++ ktime_to_ns(stt->configured_interval), ++ stt->configured_ticks); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ return ret; ++} ++ ++static DEVICE_ATTR(pm_poweroff, S_IRUGO | S_IWUSR, show_pm_poweroff, ++ set_pm_poweroff); ++ ++/** ++ * set_reset_timeout - Store callback for the reset_timeout sysfs file. ++ * @dev: The device with sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The value written to the sysfs file ++ * @count: The number of bytes written to the sysfs file ++ * ++ * This function is called when the reset_timeout sysfs file is written to. It ++ * checks the data written, and if valid updates the reset timeout. ++ * ++ * Return: @count if the function succeeded. An error code on failure. ++ */ ++static ssize_t set_reset_timeout(struct device *dev, ++ struct device_attribute *attr, const char *buf, size_t count) ++{ ++ struct kbase_device *kbdev; ++ int ret; ++ int reset_timeout; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ ret = kstrtoint(buf, 0, &reset_timeout); ++ if (ret || reset_timeout <= 0) { ++ dev_err(kbdev->dev, "Couldn't process reset_timeout write operation.\n" ++ "Use format \n"); ++ return -EINVAL; ++ } ++ ++ kbdev->reset_timeout_ms = reset_timeout; ++ dev_dbg(kbdev->dev, "Reset timeout: %dms\n", reset_timeout); ++ ++ return count; ++} ++ ++/** ++ * show_reset_timeout - Show callback for the reset_timeout sysfs entry. ++ * @dev: The device this sysfs file is for. ++ * @attr: The attributes of the sysfs file. ++ * @buf: The output buffer to receive the GPU information. ++ * ++ * This function is called to get the current reset timeout. ++ * ++ * Return: The number of bytes output to @buf. ++ */ ++static ssize_t show_reset_timeout(struct device *dev, ++ struct device_attribute *attr, char * const buf) ++{ ++ struct kbase_device *kbdev; ++ ssize_t ret; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ ret = scnprintf(buf, PAGE_SIZE, "%d\n", kbdev->reset_timeout_ms); ++ ++ return ret; ++} ++ ++static DEVICE_ATTR(reset_timeout, S_IRUGO | S_IWUSR, show_reset_timeout, ++ set_reset_timeout); ++ ++ ++static ssize_t show_mem_pool_size(struct device *dev, ++ struct device_attribute *attr, char * const buf) ++{ ++ struct kbase_device *const kbdev = to_kbase_device(dev); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ return kbase_debugfs_helper_get_attr_to_string(buf, PAGE_SIZE, ++ kbdev->mem_pools.small, MEMORY_GROUP_MANAGER_NR_GROUPS, ++ kbase_mem_pool_debugfs_size); ++} ++ ++static ssize_t set_mem_pool_size(struct device *dev, ++ struct device_attribute *attr, const char *buf, size_t count) ++{ ++ struct kbase_device *const kbdev = to_kbase_device(dev); ++ int err; ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ err = kbase_debugfs_helper_set_attr_from_string(buf, ++ kbdev->mem_pools.small, MEMORY_GROUP_MANAGER_NR_GROUPS, ++ kbase_mem_pool_debugfs_trim); ++ ++ return err ? err : count; ++} ++ ++static DEVICE_ATTR(mem_pool_size, S_IRUGO | S_IWUSR, show_mem_pool_size, ++ set_mem_pool_size); ++ ++static ssize_t show_mem_pool_max_size(struct device *dev, ++ struct device_attribute *attr, char * const buf) ++{ ++ struct kbase_device *const kbdev = to_kbase_device(dev); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ return kbase_debugfs_helper_get_attr_to_string(buf, PAGE_SIZE, ++ kbdev->mem_pools.small, MEMORY_GROUP_MANAGER_NR_GROUPS, ++ kbase_mem_pool_debugfs_max_size); ++} ++ ++static ssize_t set_mem_pool_max_size(struct device *dev, ++ struct device_attribute *attr, const char *buf, size_t count) ++{ ++ struct kbase_device *const kbdev = to_kbase_device(dev); ++ int err; ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ err = kbase_debugfs_helper_set_attr_from_string(buf, ++ kbdev->mem_pools.small, MEMORY_GROUP_MANAGER_NR_GROUPS, ++ kbase_mem_pool_debugfs_set_max_size); ++ ++ return err ? err : count; ++} ++ ++static DEVICE_ATTR(mem_pool_max_size, S_IRUGO | S_IWUSR, show_mem_pool_max_size, ++ set_mem_pool_max_size); ++ ++/** ++ * show_lp_mem_pool_size - Show size of the large memory pages pool. ++ * @dev: The device this sysfs file is for. ++ * @attr: The attributes of the sysfs file. ++ * @buf: The output buffer to receive the pool size. ++ * ++ * This function is called to get the number of large memory pages which currently populate the kbdev pool. ++ * ++ * Return: The number of bytes output to @buf. ++ */ ++static ssize_t show_lp_mem_pool_size(struct device *dev, ++ struct device_attribute *attr, char * const buf) ++{ ++ struct kbase_device *const kbdev = to_kbase_device(dev); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ return kbase_debugfs_helper_get_attr_to_string(buf, PAGE_SIZE, ++ kbdev->mem_pools.large, MEMORY_GROUP_MANAGER_NR_GROUPS, ++ kbase_mem_pool_debugfs_size); ++} ++ ++/** ++ * set_lp_mem_pool_size - Set size of the large memory pages pool. ++ * @dev: The device this sysfs file is for. ++ * @attr: The attributes of the sysfs file. ++ * @buf: The value written to the sysfs file. ++ * @count: The number of bytes written to the sysfs file. ++ * ++ * This function is called to set the number of large memory pages which should populate the kbdev pool. ++ * This may cause existing pages to be removed from the pool, or new pages to be created and then added to the pool. ++ * ++ * Return: @count if the function succeeded. An error code on failure. ++ */ ++static ssize_t set_lp_mem_pool_size(struct device *dev, ++ struct device_attribute *attr, const char *buf, size_t count) ++{ ++ struct kbase_device *const kbdev = to_kbase_device(dev); ++ int err; ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ err = kbase_debugfs_helper_set_attr_from_string(buf, ++ kbdev->mem_pools.large, MEMORY_GROUP_MANAGER_NR_GROUPS, ++ kbase_mem_pool_debugfs_trim); ++ ++ return err ? err : count; ++} ++ ++static DEVICE_ATTR(lp_mem_pool_size, S_IRUGO | S_IWUSR, show_lp_mem_pool_size, ++ set_lp_mem_pool_size); ++ ++/** ++ * show_lp_mem_pool_max_size - Show maximum size of the large memory pages pool. ++ * @dev: The device this sysfs file is for. ++ * @attr: The attributes of the sysfs file. ++ * @buf: The output buffer to receive the pool size. ++ * ++ * This function is called to get the maximum number of large memory pages that the kbdev pool can possibly contain. ++ * ++ * Return: The number of bytes output to @buf. ++ */ ++static ssize_t show_lp_mem_pool_max_size(struct device *dev, ++ struct device_attribute *attr, char * const buf) ++{ ++ struct kbase_device *const kbdev = to_kbase_device(dev); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ return kbase_debugfs_helper_get_attr_to_string(buf, PAGE_SIZE, ++ kbdev->mem_pools.large, MEMORY_GROUP_MANAGER_NR_GROUPS, ++ kbase_mem_pool_debugfs_max_size); ++} ++ ++/** ++ * set_lp_mem_pool_max_size - Set maximum size of the large memory pages pool. ++ * @dev: The device this sysfs file is for. ++ * @attr: The attributes of the sysfs file. ++ * @buf: The value written to the sysfs file. ++ * @count: The number of bytes written to the sysfs file. ++ * ++ * This function is called to set the maximum number of large memory pages that the kbdev pool can possibly contain. ++ * ++ * Return: @count if the function succeeded. An error code on failure. ++ */ ++static ssize_t set_lp_mem_pool_max_size(struct device *dev, ++ struct device_attribute *attr, const char *buf, size_t count) ++{ ++ struct kbase_device *const kbdev = to_kbase_device(dev); ++ int err; ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ err = kbase_debugfs_helper_set_attr_from_string(buf, ++ kbdev->mem_pools.large, MEMORY_GROUP_MANAGER_NR_GROUPS, ++ kbase_mem_pool_debugfs_set_max_size); ++ ++ return err ? err : count; ++} ++ ++static DEVICE_ATTR(lp_mem_pool_max_size, S_IRUGO | S_IWUSR, show_lp_mem_pool_max_size, ++ set_lp_mem_pool_max_size); ++ ++#if !MALI_USE_CSF ++/** ++ * show_js_ctx_scheduling_mode - Show callback for js_ctx_scheduling_mode sysfs ++ * entry. ++ * @dev: The device this sysfs file is for. ++ * @attr: The attributes of the sysfs file. ++ * @buf: The output buffer to receive the context scheduling mode information. ++ * ++ * This function is called to get the context scheduling mode being used by JS. ++ * ++ * Return: The number of bytes output to @buf. ++ */ ++static ssize_t show_js_ctx_scheduling_mode(struct device *dev, ++ struct device_attribute *attr, char * const buf) ++{ ++ struct kbase_device *kbdev; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ return scnprintf(buf, PAGE_SIZE, "%u\n", kbdev->js_ctx_scheduling_mode); ++} ++ ++/** ++ * set_js_ctx_scheduling_mode - Set callback for js_ctx_scheduling_mode sysfs ++ * entry. ++ * @dev: The device this sysfs file is for. ++ * @attr: The attributes of the sysfs file. ++ * @buf: The value written to the sysfs file. ++ * @count: The number of bytes written to the sysfs file. ++ * ++ * This function is called when the js_ctx_scheduling_mode sysfs file is written ++ * to. It checks the data written, and if valid updates the ctx scheduling mode ++ * being by JS. ++ * ++ * Return: @count if the function succeeded. An error code on failure. ++ */ ++static ssize_t set_js_ctx_scheduling_mode(struct device *dev, ++ struct device_attribute *attr, const char *buf, size_t count) ++{ ++ struct kbase_context *kctx; ++ u32 new_js_ctx_scheduling_mode; ++ struct kbase_device *kbdev; ++ unsigned long flags; ++ int ret; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ ret = kstrtouint(buf, 0, &new_js_ctx_scheduling_mode); ++ if (ret || new_js_ctx_scheduling_mode >= KBASE_JS_PRIORITY_MODE_COUNT) { ++ dev_err(kbdev->dev, "Couldn't process js_ctx_scheduling_mode" ++ " write operation.\n" ++ "Use format \n"); ++ return -EINVAL; ++ } ++ ++ if (new_js_ctx_scheduling_mode == kbdev->js_ctx_scheduling_mode) ++ return count; ++ ++ mutex_lock(&kbdev->kctx_list_lock); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ /* Update the context priority mode */ ++ kbdev->js_ctx_scheduling_mode = new_js_ctx_scheduling_mode; ++ ++ /* Adjust priority of all the contexts as per the new mode */ ++ list_for_each_entry(kctx, &kbdev->kctx_list, kctx_list_link) ++ kbase_js_update_ctx_priority(kctx); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&kbdev->kctx_list_lock); ++ ++ dev_dbg(kbdev->dev, "JS ctx scheduling mode: %u\n", new_js_ctx_scheduling_mode); ++ ++ return count; ++} ++ ++static DEVICE_ATTR(js_ctx_scheduling_mode, S_IRUGO | S_IWUSR, ++ show_js_ctx_scheduling_mode, ++ set_js_ctx_scheduling_mode); ++ ++#ifdef MALI_KBASE_BUILD ++ ++/* Number of entries in serialize_jobs_settings[] */ ++#define NR_SERIALIZE_JOBS_SETTINGS 5 ++/* Maximum string length in serialize_jobs_settings[].name */ ++#define MAX_SERIALIZE_JOBS_NAME_LEN 16 ++ ++static struct ++{ ++ char *name; ++ u8 setting; ++} serialize_jobs_settings[NR_SERIALIZE_JOBS_SETTINGS] = { ++ {"none", 0}, ++ {"intra-slot", KBASE_SERIALIZE_INTRA_SLOT}, ++ {"inter-slot", KBASE_SERIALIZE_INTER_SLOT}, ++ {"full", KBASE_SERIALIZE_INTRA_SLOT | KBASE_SERIALIZE_INTER_SLOT}, ++ {"full-reset", KBASE_SERIALIZE_INTRA_SLOT | KBASE_SERIALIZE_INTER_SLOT | ++ KBASE_SERIALIZE_RESET} ++}; ++ ++/** ++ * update_serialize_jobs_setting - Update the serialization setting for the ++ * submission of GPU jobs. ++ * ++ * This function is called when the serialize_jobs sysfs/debugfs file is ++ * written to. It matches the requested setting against the available settings ++ * and if a matching setting is found updates kbdev->serialize_jobs. ++ * ++ * @kbdev: An instance of the GPU platform device, allocated from the probe ++ * method of the driver. ++ * @buf: Buffer containing the value written to the sysfs/debugfs file. ++ * @count: The number of bytes to write to the sysfs/debugfs file. ++ * ++ * Return: @count if the function succeeded. An error code on failure. ++ */ ++static ssize_t update_serialize_jobs_setting(struct kbase_device *kbdev, ++ const char *buf, size_t count) ++{ ++ int i; ++ bool valid = false; ++ ++ for (i = 0; i < NR_SERIALIZE_JOBS_SETTINGS; i++) { ++ if (sysfs_streq(serialize_jobs_settings[i].name, buf)) { ++ kbdev->serialize_jobs = ++ serialize_jobs_settings[i].setting; ++ valid = true; ++ break; ++ } ++ } ++ ++ if (!valid) { ++ dev_err(kbdev->dev, "serialize_jobs: invalid setting"); ++ return -EINVAL; ++ } ++ ++ return count; ++} ++ ++#ifdef CONFIG_DEBUG_FS ++/** ++ * kbasep_serialize_jobs_seq_debugfs_show - Show callback for the serialize_jobs ++ * debugfs file ++ * @sfile: seq_file pointer ++ * @data: Private callback data ++ * ++ * This function is called to get the contents of the serialize_jobs debugfs ++ * file. This is a list of the available settings with the currently active one ++ * surrounded by square brackets. ++ * ++ * Return: 0 on success, or an error code on error ++ */ ++static int kbasep_serialize_jobs_seq_debugfs_show(struct seq_file *sfile, ++ void *data) ++{ ++ struct kbase_device *kbdev = sfile->private; ++ int i; ++ ++ CSTD_UNUSED(data); ++ ++ for (i = 0; i < NR_SERIALIZE_JOBS_SETTINGS; i++) { ++ if (kbdev->serialize_jobs == serialize_jobs_settings[i].setting) ++ seq_printf(sfile, "[%s] ", ++ serialize_jobs_settings[i].name); ++ else ++ seq_printf(sfile, "%s ", ++ serialize_jobs_settings[i].name); ++ } ++ ++ seq_puts(sfile, "\n"); ++ ++ return 0; ++} ++ ++/** ++ * kbasep_serialize_jobs_debugfs_write - Store callback for the serialize_jobs ++ * debugfs file. ++ * @file: File pointer ++ * @ubuf: User buffer containing data to store ++ * @count: Number of bytes in user buffer ++ * @ppos: File position ++ * ++ * This function is called when the serialize_jobs debugfs file is written to. ++ * It matches the requested setting against the available settings and if a ++ * matching setting is found updates kbdev->serialize_jobs. ++ * ++ * Return: @count if the function succeeded. An error code on failure. ++ */ ++static ssize_t kbasep_serialize_jobs_debugfs_write(struct file *file, ++ const char __user *ubuf, size_t count, loff_t *ppos) ++{ ++ struct seq_file *s = file->private_data; ++ struct kbase_device *kbdev = s->private; ++ char buf[MAX_SERIALIZE_JOBS_NAME_LEN]; ++ ++ CSTD_UNUSED(ppos); ++ ++ count = min_t(size_t, sizeof(buf) - 1, count); ++ if (copy_from_user(buf, ubuf, count)) ++ return -EFAULT; ++ ++ buf[count] = 0; ++ ++ return update_serialize_jobs_setting(kbdev, buf, count); ++} ++ ++/** ++ * kbasep_serialize_jobs_debugfs_open - Open callback for the serialize_jobs ++ * debugfs file ++ * @in: inode pointer ++ * @file: file pointer ++ * ++ * Return: Zero on success, error code on failure ++ */ ++static int kbasep_serialize_jobs_debugfs_open(struct inode *in, ++ struct file *file) ++{ ++ return single_open(file, kbasep_serialize_jobs_seq_debugfs_show, ++ in->i_private); ++} ++ ++static const struct file_operations kbasep_serialize_jobs_debugfs_fops = { ++ .owner = THIS_MODULE, ++ .open = kbasep_serialize_jobs_debugfs_open, ++ .read = seq_read, ++ .write = kbasep_serialize_jobs_debugfs_write, ++ .llseek = seq_lseek, ++ .release = single_release, ++}; ++ ++#endif /* CONFIG_DEBUG_FS */ ++ ++/** ++ * show_serialize_jobs_sysfs - Show callback for serialize_jobs sysfs file. ++ * ++ * This function is called to get the contents of the serialize_jobs sysfs ++ * file. This is a list of the available settings with the currently active ++ * one surrounded by square brackets. ++ * ++ * @dev: The device this sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The output buffer for the sysfs file contents ++ * ++ * Return: The number of bytes output to @buf. ++ */ ++static ssize_t show_serialize_jobs_sysfs(struct device *dev, ++ struct device_attribute *attr, ++ char *buf) ++{ ++ struct kbase_device *kbdev = to_kbase_device(dev); ++ ssize_t ret = 0; ++ int i; ++ ++ for (i = 0; i < NR_SERIALIZE_JOBS_SETTINGS; i++) { ++ if (kbdev->serialize_jobs == ++ serialize_jobs_settings[i].setting) ++ ret += scnprintf(buf + ret, PAGE_SIZE - ret, "[%s]", ++ serialize_jobs_settings[i].name); ++ else ++ ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s ", ++ serialize_jobs_settings[i].name); ++ } ++ ++ if (ret < PAGE_SIZE - 1) { ++ ret += scnprintf(buf + ret, PAGE_SIZE - ret, "\n"); ++ } else { ++ buf[PAGE_SIZE - 2] = '\n'; ++ buf[PAGE_SIZE - 1] = '\0'; ++ ret = PAGE_SIZE - 1; ++ } ++ ++ return ret; ++} ++ ++/** ++ * store_serialize_jobs_sysfs - Store callback for serialize_jobs sysfs file. ++ * ++ * This function is called when the serialize_jobs sysfs file is written to. ++ * It matches the requested setting against the available settings and if a ++ * matching setting is found updates kbdev->serialize_jobs. ++ * ++ * @dev: The device this sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The value written to the sysfs file ++ * @count: The number of bytes to write to the sysfs file ++ * ++ * Return: @count if the function succeeded. An error code on failure. ++ */ ++static ssize_t store_serialize_jobs_sysfs(struct device *dev, ++ struct device_attribute *attr, ++ const char *buf, size_t count) ++{ ++ return update_serialize_jobs_setting(to_kbase_device(dev), buf, count); ++} ++ ++static DEVICE_ATTR(serialize_jobs, 0600, show_serialize_jobs_sysfs, ++ store_serialize_jobs_sysfs); ++#endif /* MALI_KBASE_BUILD */ ++#endif /* !MALI_USE_CSF */ ++ ++static void kbasep_protected_mode_hwcnt_disable_worker(struct work_struct *data) ++{ ++ struct kbase_device *kbdev = container_of(data, struct kbase_device, ++ protected_mode_hwcnt_disable_work); ++ unsigned long flags; ++ ++ bool do_disable; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ do_disable = !kbdev->protected_mode_hwcnt_desired && ++ !kbdev->protected_mode_hwcnt_disabled; ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ if (!do_disable) ++ return; ++ ++ kbase_hwcnt_context_disable(kbdev->hwcnt_gpu_ctx); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ do_disable = !kbdev->protected_mode_hwcnt_desired && ++ !kbdev->protected_mode_hwcnt_disabled; ++ ++ if (do_disable) { ++ /* Protected mode state did not change while we were doing the ++ * disable, so commit the work we just performed and continue ++ * the state machine. ++ */ ++ kbdev->protected_mode_hwcnt_disabled = true; ++#if !MALI_USE_CSF ++ kbase_backend_slot_update(kbdev); ++#endif /* !MALI_USE_CSF */ ++ } else { ++ /* Protected mode state was updated while we were doing the ++ * disable, so we need to undo the disable we just performed. ++ */ ++ kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); ++ } ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++} ++ ++static int kbasep_protected_mode_enable(struct protected_mode_device *pdev) ++{ ++ struct kbase_device *kbdev = pdev->data; ++ ++ return kbase_pm_protected_mode_enable(kbdev); ++} ++ ++static int kbasep_protected_mode_disable(struct protected_mode_device *pdev) ++{ ++ struct kbase_device *kbdev = pdev->data; ++ ++ return kbase_pm_protected_mode_disable(kbdev); ++} ++ ++static const struct protected_mode_ops kbasep_native_protected_ops = { ++ .protected_mode_enable = kbasep_protected_mode_enable, ++ .protected_mode_disable = kbasep_protected_mode_disable ++}; ++ ++int kbase_protected_mode_init(struct kbase_device *kbdev) ++{ ++ /* Use native protected ops */ ++ kbdev->protected_dev = kzalloc(sizeof(*kbdev->protected_dev), ++ GFP_KERNEL); ++ if (!kbdev->protected_dev) ++ return -ENOMEM; ++ kbdev->protected_dev->data = kbdev; ++ kbdev->protected_ops = &kbasep_native_protected_ops; ++ INIT_WORK(&kbdev->protected_mode_hwcnt_disable_work, ++ kbasep_protected_mode_hwcnt_disable_worker); ++ kbdev->protected_mode_hwcnt_desired = true; ++ kbdev->protected_mode_hwcnt_disabled = false; ++ return 0; ++} ++ ++void kbase_protected_mode_term(struct kbase_device *kbdev) ++{ ++ cancel_work_sync(&kbdev->protected_mode_hwcnt_disable_work); ++ kfree(kbdev->protected_dev); ++} ++ ++#ifdef CONFIG_MALI_BIFROST_NO_MALI ++static int kbase_common_reg_map(struct kbase_device *kbdev) ++{ ++ return 0; ++} ++static void kbase_common_reg_unmap(struct kbase_device * const kbdev) ++{ ++} ++#else /* CONFIG_MALI_BIFROST_NO_MALI */ ++static int kbase_common_reg_map(struct kbase_device *kbdev) ++{ ++ int err = 0; ++ ++ if (!request_mem_region(kbdev->reg_start, kbdev->reg_size, dev_name(kbdev->dev))) { ++ dev_err(kbdev->dev, "Register window unavailable\n"); ++ err = -EIO; ++ goto out_region; ++ } ++ ++ kbdev->reg = ioremap(kbdev->reg_start, kbdev->reg_size); ++ if (!kbdev->reg) { ++ dev_err(kbdev->dev, "Can't remap register window\n"); ++ err = -EINVAL; ++ goto out_ioremap; ++ } ++ ++ return err; ++ ++out_ioremap: ++ release_mem_region(kbdev->reg_start, kbdev->reg_size); ++out_region: ++ return err; ++} ++ ++static void kbase_common_reg_unmap(struct kbase_device * const kbdev) ++{ ++ if (kbdev->reg) { ++ iounmap(kbdev->reg); ++ release_mem_region(kbdev->reg_start, kbdev->reg_size); ++ kbdev->reg = NULL; ++ kbdev->reg_start = 0; ++ kbdev->reg_size = 0; ++ } ++} ++#endif /* CONFIG_MALI_BIFROST_NO_MALI */ ++ ++int registers_map(struct kbase_device * const kbdev) ++{ ++ /* the first memory resource is the physical address of the GPU ++ * registers. ++ */ ++ struct platform_device *pdev = to_platform_device(kbdev->dev); ++ struct resource *reg_res; ++ int err; ++ ++ reg_res = platform_get_resource(pdev, IORESOURCE_MEM, 0); ++ if (!reg_res) { ++ dev_err(kbdev->dev, "Invalid register resource\n"); ++ return -ENOENT; ++ } ++ ++ kbdev->reg_start = reg_res->start; ++ kbdev->reg_size = resource_size(reg_res); ++ ++#if MALI_USE_CSF ++ if (kbdev->reg_size < ++ (CSF_HW_DOORBELL_PAGE_OFFSET + ++ CSF_NUM_DOORBELL * CSF_HW_DOORBELL_PAGE_SIZE)) { ++ dev_err(kbdev->dev, "Insufficient register space, will override to the required size\n"); ++ kbdev->reg_size = CSF_HW_DOORBELL_PAGE_OFFSET + ++ CSF_NUM_DOORBELL * CSF_HW_DOORBELL_PAGE_SIZE; ++ } ++#endif ++ ++ err = kbase_common_reg_map(kbdev); ++ if (err) { ++ dev_err(kbdev->dev, "Failed to map registers\n"); ++ return err; ++ } ++ ++ return 0; ++} ++ ++void registers_unmap(struct kbase_device *kbdev) ++{ ++ kbase_common_reg_unmap(kbdev); ++} ++ ++#if defined(CONFIG_MALI_ARBITER_SUPPORT) && defined(CONFIG_OF) ++ ++static bool kbase_is_pm_enabled(const struct device_node *gpu_node) ++{ ++ const struct device_node *power_model_node; ++ const void *cooling_cells_node; ++ const void *operating_point_node; ++ bool is_pm_enable = false; ++ ++ power_model_node = of_get_child_by_name(gpu_node, ++ "power_model"); ++ if (power_model_node) ++ is_pm_enable = true; ++ ++ cooling_cells_node = of_get_property(gpu_node, ++ "#cooling-cells", NULL); ++ if (cooling_cells_node) ++ is_pm_enable = true; ++ ++ operating_point_node = of_get_property(gpu_node, ++ "operating-points", NULL); ++ if (operating_point_node) ++ is_pm_enable = true; ++ ++ return is_pm_enable; ++} ++ ++static bool kbase_is_pv_enabled(const struct device_node *gpu_node) ++{ ++ const void *arbiter_if_node; ++ ++ arbiter_if_node = of_get_property(gpu_node, ++ "arbiter_if", NULL); ++ ++ return arbiter_if_node ? true : false; ++} ++ ++static bool kbase_is_full_coherency_enabled(const struct device_node *gpu_node) ++{ ++ const void *coherency_dts; ++ u32 coherency; ++ ++ coherency_dts = of_get_property(gpu_node, ++ "system-coherency", ++ NULL); ++ if (coherency_dts) { ++ coherency = be32_to_cpup(coherency_dts); ++ if (coherency == COHERENCY_ACE) ++ return true; ++ } ++ return false; ++} ++ ++#endif /* CONFIG_MALI_ARBITER_SUPPORT && CONFIG_OF */ ++ ++int kbase_device_pm_init(struct kbase_device *kbdev) ++{ ++ int err = 0; ++ ++#if defined(CONFIG_MALI_ARBITER_SUPPORT) && defined(CONFIG_OF) ++ ++ u32 gpu_id; ++ u32 product_id; ++ u32 gpu_model_id; ++ ++ if (kbase_is_pv_enabled(kbdev->dev->of_node)) { ++ if (kbase_is_pm_enabled(kbdev->dev->of_node)) { ++ /* Arbitration AND power management invalid */ ++ dev_err(kbdev->dev, "Invalid combination of arbitration AND power management\n"); ++ return -EPERM; ++ } ++ if (kbase_is_full_coherency_enabled(kbdev->dev->of_node)) { ++ /* Arbitration AND full coherency invalid */ ++ dev_err(kbdev->dev, "Invalid combination of arbitration AND full coherency\n"); ++ return -EPERM; ++ } ++ err = kbase_arbiter_pm_early_init(kbdev); ++ if (err == 0) { ++ /* Check if Arbitration is running on ++ * supported GPU platform ++ */ ++ kbase_pm_register_access_enable(kbdev); ++ gpu_id = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_ID)); ++ kbase_pm_register_access_disable(kbdev); ++ product_id = KBASE_UBFX32(gpu_id, ++ GPU_ID_VERSION_PRODUCT_ID_SHIFT, 16); ++ gpu_model_id = GPU_ID2_MODEL_MATCH_VALUE(product_id); ++ ++ if (gpu_model_id != GPU_ID2_PRODUCT_TGOX ++ && gpu_model_id != GPU_ID2_PRODUCT_TNOX) { ++ kbase_arbiter_pm_early_term(kbdev); ++ dev_err(kbdev->dev, "GPU platform not suitable for arbitration\n"); ++ return -EPERM; ++ } ++ } ++ } else { ++ err = power_control_init(kbdev); ++ } ++#else ++ err = power_control_init(kbdev); ++#endif /* CONFIG_MALI_ARBITER_SUPPORT && CONFIG_OF */ ++ return err; ++} ++ ++void kbase_device_pm_term(struct kbase_device *kbdev) ++{ ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++#ifdef CONFIG_OF ++ if (kbase_is_pv_enabled(kbdev->dev->of_node)) ++ kbase_arbiter_pm_early_term(kbdev); ++ else ++ power_control_term(kbdev); ++#endif /* CONFIG_OF */ ++#else ++ power_control_term(kbdev); ++#endif ++} ++ ++int power_control_init(struct kbase_device *kbdev) ++{ ++#if KERNEL_VERSION(3, 18, 0) > LINUX_VERSION_CODE || !defined(CONFIG_OF) ++ /* Power control initialization requires at least the capability to get ++ * regulators and clocks from the device tree, as well as parsing ++ * arrays of unsigned integer values. ++ * ++ * The whole initialization process shall simply be skipped if the ++ * minimum capability is not available. ++ */ ++ return 0; ++#else ++ struct platform_device *pdev; ++ int err = 0; ++ unsigned int i; ++#if defined(CONFIG_REGULATOR) ++ static const char *regulator_names[] = { ++ "mali", "shadercores" ++ }; ++ BUILD_BUG_ON(ARRAY_SIZE(regulator_names) < BASE_MAX_NR_CLOCKS_REGULATORS); ++#endif /* CONFIG_REGULATOR */ ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ pdev = to_platform_device(kbdev->dev); ++ ++#if defined(CONFIG_REGULATOR) ++ /* Since the error code EPROBE_DEFER causes the entire probing ++ * procedure to be restarted from scratch at a later time, ++ * all regulators will be released before returning. ++ * ++ * Any other error is ignored and the driver will continue ++ * operating with a partial initialization of regulators. ++ */ ++ for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) { ++ kbdev->regulators[i] = regulator_get_optional(kbdev->dev, ++ regulator_names[i]); ++ if (IS_ERR_OR_NULL(kbdev->regulators[i])) { ++ err = PTR_ERR(kbdev->regulators[i]); ++ kbdev->regulators[i] = NULL; ++ break; ++ } ++ } ++ if (err == -EPROBE_DEFER) { ++ while ((i > 0) && (i < BASE_MAX_NR_CLOCKS_REGULATORS)) ++ regulator_put(kbdev->regulators[--i]); ++ return err; ++ } ++ ++ kbdev->nr_regulators = i; ++ dev_dbg(&pdev->dev, "Regulators probed: %u\n", kbdev->nr_regulators); ++#endif ++ ++ /* Having more clocks than regulators is acceptable, while the ++ * opposite shall not happen. ++ * ++ * Since the error code EPROBE_DEFER causes the entire probing ++ * procedure to be restarted from scratch at a later time, ++ * all clocks and regulators will be released before returning. ++ * ++ * Any other error is ignored and the driver will continue ++ * operating with a partial initialization of clocks. ++ */ ++ for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) { ++ kbdev->clocks[i] = of_clk_get(kbdev->dev->of_node, i); ++ if (IS_ERR_OR_NULL(kbdev->clocks[i])) { ++ err = PTR_ERR(kbdev->clocks[i]); ++ kbdev->clocks[i] = NULL; ++ break; ++ } ++ ++ err = clk_prepare(kbdev->clocks[i]); ++ if (err) { ++ dev_err(kbdev->dev, ++ "Failed to prepare and enable clock (%d)\n", ++ err); ++ clk_put(kbdev->clocks[i]); ++ break; ++ } ++ } ++ if (err == -EPROBE_DEFER) { ++ while ((i > 0) && (i < BASE_MAX_NR_CLOCKS_REGULATORS)) { ++ clk_unprepare(kbdev->clocks[--i]); ++ clk_put(kbdev->clocks[i]); ++ } ++ goto clocks_probe_defer; ++ } ++ ++ kbdev->nr_clocks = i; ++ dev_dbg(&pdev->dev, "Clocks probed: %u\n", kbdev->nr_clocks); ++ ++#if defined(CONFIG_PM_OPP) ++#if ((KERNEL_VERSION(4, 10, 0) <= LINUX_VERSION_CODE) && \ ++ defined(CONFIG_REGULATOR)) ++ if (kbdev->nr_regulators > 0) { ++ kbdev->opp_table = dev_pm_opp_set_regulators(kbdev->dev, ++ regulator_names, BASE_MAX_NR_CLOCKS_REGULATORS); ++ } ++#endif /* (KERNEL_VERSION(4, 10, 0) <= LINUX_VERSION_CODE */ ++#ifdef CONFIG_ARCH_ROCKCHIP ++ err = kbase_platform_rk_init_opp_table(kbdev); ++ if (err) ++ dev_err(kbdev->dev, "Failed to init_opp_table (%d)\n", err); ++#else ++ err = dev_pm_opp_of_add_table(kbdev->dev); ++ CSTD_UNUSED(err); ++#endif ++#endif /* CONFIG_PM_OPP */ ++ ++#endif /* KERNEL_VERSION(4, 4, 0) > LINUX_VERSION_CODE */ ++ return 0; ++ ++clocks_probe_defer: ++#if defined(CONFIG_REGULATOR) ++ for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) ++ regulator_put(kbdev->regulators[i]); ++#endif ++ return err; ++} ++ ++void power_control_term(struct kbase_device *kbdev) ++{ ++ unsigned int i; ++ ++#if (KERNEL_VERSION(4, 4, 0) > LINUX_VERSION_CODE && \ ++ !defined(LSK_OPPV2_BACKPORT)) ++#if KERNEL_VERSION(3, 19, 0) <= LINUX_VERSION_CODE ++ of_free_opp_table(kbdev->dev); ++#endif ++#else ++ ++#if defined(CONFIG_PM_OPP) ++ dev_pm_opp_of_remove_table(kbdev->dev); ++#if ((KERNEL_VERSION(4, 10, 0) <= LINUX_VERSION_CODE) && \ ++ defined(CONFIG_REGULATOR)) ++ if (!IS_ERR_OR_NULL(kbdev->opp_table)) ++ dev_pm_opp_put_regulators(kbdev->opp_table); ++#endif /* (KERNEL_VERSION(4, 10, 0) <= LINUX_VERSION_CODE */ ++#endif /* CONFIG_PM_OPP */ ++ ++#endif /* KERNEL_VERSION(4, 4, 0) > LINUX_VERSION_CODE */ ++ ++ for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) { ++ if (kbdev->clocks[i]) { ++ clk_unprepare(kbdev->clocks[i]); ++ clk_put(kbdev->clocks[i]); ++ kbdev->clocks[i] = NULL; ++ } else ++ break; ++ } ++ ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0)) && defined(CONFIG_OF) \ ++ && defined(CONFIG_REGULATOR) ++ for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) { ++ if (kbdev->regulators[i]) { ++ regulator_put(kbdev->regulators[i]); ++ kbdev->regulators[i] = NULL; ++ } ++ } ++#endif /* LINUX_VERSION_CODE >= 3, 12, 0 */ ++} ++ ++#ifdef MALI_KBASE_BUILD ++#ifdef CONFIG_DEBUG_FS ++ ++static void trigger_reset(struct kbase_device *kbdev) ++{ ++ kbase_pm_context_active(kbdev); ++ if (kbase_prepare_to_reset_gpu(kbdev)) ++ kbase_reset_gpu(kbdev); ++ kbase_pm_context_idle(kbdev); ++} ++ ++#define MAKE_QUIRK_ACCESSORS(type) \ ++static int type##_quirks_set(void *data, u64 val) \ ++{ \ ++ struct kbase_device *kbdev; \ ++ kbdev = (struct kbase_device *)data; \ ++ kbdev->hw_quirks_##type = (u32)val; \ ++ trigger_reset(kbdev); \ ++ return 0;\ ++} \ ++\ ++static int type##_quirks_get(void *data, u64 *val) \ ++{ \ ++ struct kbase_device *kbdev;\ ++ kbdev = (struct kbase_device *)data;\ ++ *val = kbdev->hw_quirks_##type;\ ++ return 0;\ ++} \ ++DEFINE_SIMPLE_ATTRIBUTE(fops_##type##_quirks, type##_quirks_get,\ ++ type##_quirks_set, "%llu\n") ++ ++MAKE_QUIRK_ACCESSORS(sc); ++MAKE_QUIRK_ACCESSORS(tiler); ++MAKE_QUIRK_ACCESSORS(mmu); ++MAKE_QUIRK_ACCESSORS(jm); ++ ++static ssize_t kbase_device_debugfs_reset_write(struct file *file, ++ const char __user *ubuf, size_t count, loff_t *ppos) ++{ ++ struct kbase_device *kbdev = file->private_data; ++ CSTD_UNUSED(ubuf); ++ CSTD_UNUSED(count); ++ CSTD_UNUSED(ppos); ++ ++ trigger_reset(kbdev); ++ ++ return count; ++} ++ ++static const struct file_operations fops_trigger_reset = { ++ .owner = THIS_MODULE, ++ .open = simple_open, ++ .write = kbase_device_debugfs_reset_write, ++ .llseek = default_llseek, ++}; ++ ++/** ++ * debugfs_protected_debug_mode_read - "protected_debug_mode" debugfs read ++ * @file: File object to read is for ++ * @buf: User buffer to populate with data ++ * @len: Length of user buffer ++ * @ppos: Offset within file object ++ * ++ * Retrieves the current status of protected debug mode ++ * (0 = disabled, 1 = enabled) ++ * ++ * Return: Number of bytes added to user buffer ++ */ ++static ssize_t debugfs_protected_debug_mode_read(struct file *file, ++ char __user *buf, size_t len, loff_t *ppos) ++{ ++ struct kbase_device *kbdev = (struct kbase_device *)file->private_data; ++ u32 gpu_status; ++ ssize_t ret_val; ++ ++ kbase_pm_context_active(kbdev); ++ gpu_status = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_STATUS)); ++ kbase_pm_context_idle(kbdev); ++ ++ if (gpu_status & GPU_DBGEN) ++ ret_val = simple_read_from_buffer(buf, len, ppos, "1\n", 2); ++ else ++ ret_val = simple_read_from_buffer(buf, len, ppos, "0\n", 2); ++ ++ return ret_val; ++} ++ ++/* ++ * struct fops_protected_debug_mode - "protected_debug_mode" debugfs fops ++ * ++ * Contains the file operations for the "protected_debug_mode" debugfs file ++ */ ++static const struct file_operations fops_protected_debug_mode = { ++ .owner = THIS_MODULE, ++ .open = simple_open, ++ .read = debugfs_protected_debug_mode_read, ++ .llseek = default_llseek, ++}; ++ ++static int kbase_device_debugfs_mem_pool_max_size_show(struct seq_file *sfile, ++ void *data) ++{ ++ CSTD_UNUSED(data); ++ return kbase_debugfs_helper_seq_read(sfile, ++ MEMORY_GROUP_MANAGER_NR_GROUPS, ++ kbase_mem_pool_config_debugfs_max_size); ++} ++ ++static ssize_t kbase_device_debugfs_mem_pool_max_size_write(struct file *file, ++ const char __user *ubuf, size_t count, loff_t *ppos) ++{ ++ int err = 0; ++ ++ CSTD_UNUSED(ppos); ++ err = kbase_debugfs_helper_seq_write(file, ubuf, count, ++ MEMORY_GROUP_MANAGER_NR_GROUPS, ++ kbase_mem_pool_config_debugfs_set_max_size); ++ ++ return err ? err : count; ++} ++ ++static int kbase_device_debugfs_mem_pool_max_size_open(struct inode *in, ++ struct file *file) ++{ ++ return single_open(file, kbase_device_debugfs_mem_pool_max_size_show, ++ in->i_private); ++} ++ ++static const struct file_operations ++ kbase_device_debugfs_mem_pool_max_size_fops = { ++ .owner = THIS_MODULE, ++ .open = kbase_device_debugfs_mem_pool_max_size_open, ++ .read = seq_read, ++ .write = kbase_device_debugfs_mem_pool_max_size_write, ++ .llseek = seq_lseek, ++ .release = single_release, ++}; ++ ++int kbase_device_debugfs_init(struct kbase_device *kbdev) ++{ ++ struct dentry *debugfs_ctx_defaults_directory; ++ int err; ++ /* prevent unprivileged use of debug file system ++ * in old kernel version ++ */ ++#if (KERNEL_VERSION(4, 7, 0) <= LINUX_VERSION_CODE) ++ /* only for newer kernel version debug file system is safe */ ++ const mode_t mode = 0644; ++#else ++ const mode_t mode = 0600; ++#endif ++ ++ kbdev->mali_debugfs_directory = debugfs_create_dir(kbdev->devname, ++ NULL); ++ if (!kbdev->mali_debugfs_directory) { ++ dev_err(kbdev->dev, "Couldn't create mali debugfs directory\n"); ++ err = -ENOMEM; ++ goto out; ++ } ++ ++ kbdev->debugfs_ctx_directory = debugfs_create_dir("ctx", ++ kbdev->mali_debugfs_directory); ++ if (!kbdev->debugfs_ctx_directory) { ++ dev_err(kbdev->dev, "Couldn't create mali debugfs ctx directory\n"); ++ err = -ENOMEM; ++ goto out; ++ } ++ ++ kbdev->debugfs_instr_directory = debugfs_create_dir("instrumentation", ++ kbdev->mali_debugfs_directory); ++ if (!kbdev->debugfs_instr_directory) { ++ dev_err(kbdev->dev, "Couldn't create mali debugfs instrumentation directory\n"); ++ err = -ENOMEM; ++ goto out; ++ } ++ ++ debugfs_ctx_defaults_directory = debugfs_create_dir("defaults", ++ kbdev->debugfs_ctx_directory); ++ if (!debugfs_ctx_defaults_directory) { ++ dev_err(kbdev->dev, "Couldn't create mali debugfs ctx defaults directory\n"); ++ err = -ENOMEM; ++ goto out; ++ } ++ ++#if !MALI_CUSTOMER_RELEASE ++ kbasep_regs_dump_debugfs_init(kbdev); ++#endif /* !MALI_CUSTOMER_RELEASE */ ++ kbasep_regs_history_debugfs_init(kbdev); ++ ++#if !MALI_USE_CSF ++ kbase_debug_job_fault_debugfs_init(kbdev); ++#endif /* !MALI_USE_CSF */ ++ ++ kbasep_gpu_memory_debugfs_init(kbdev); ++ kbase_as_fault_debugfs_init(kbdev); ++#ifdef CONFIG_MALI_PRFCNT_SET_SECONDARY_VIA_DEBUG_FS ++ kbase_instr_backend_debugfs_init(kbdev); ++#endif ++ /* fops_* variables created by invocations of macro ++ * MAKE_QUIRK_ACCESSORS() above. */ ++ debugfs_create_file("quirks_sc", 0644, ++ kbdev->mali_debugfs_directory, kbdev, ++ &fops_sc_quirks); ++ debugfs_create_file("quirks_tiler", 0644, ++ kbdev->mali_debugfs_directory, kbdev, ++ &fops_tiler_quirks); ++ debugfs_create_file("quirks_mmu", 0644, ++ kbdev->mali_debugfs_directory, kbdev, ++ &fops_mmu_quirks); ++ debugfs_create_file("quirks_jm", 0644, ++ kbdev->mali_debugfs_directory, kbdev, ++ &fops_jm_quirks); ++ ++ debugfs_create_bool("infinite_cache", mode, ++ debugfs_ctx_defaults_directory, ++ &kbdev->infinite_cache_active_default); ++ ++ debugfs_create_file("mem_pool_max_size", mode, ++ debugfs_ctx_defaults_directory, ++ &kbdev->mem_pool_defaults.small, ++ &kbase_device_debugfs_mem_pool_max_size_fops); ++ ++ debugfs_create_file("lp_mem_pool_max_size", mode, ++ debugfs_ctx_defaults_directory, ++ &kbdev->mem_pool_defaults.large, ++ &kbase_device_debugfs_mem_pool_max_size_fops); ++ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE)) { ++ debugfs_create_file("protected_debug_mode", S_IRUGO, ++ kbdev->mali_debugfs_directory, kbdev, ++ &fops_protected_debug_mode); ++ } ++ ++ debugfs_create_file("reset", 0644, ++ kbdev->mali_debugfs_directory, kbdev, ++ &fops_trigger_reset); ++ ++ kbase_ktrace_debugfs_init(kbdev); ++ ++#ifdef CONFIG_MALI_BIFROST_DEVFREQ ++#ifdef CONFIG_DEVFREQ_THERMAL ++ if (kbdev->devfreq && !kbdev->model_data) ++ kbase_ipa_debugfs_init(kbdev); ++#endif /* CONFIG_DEVFREQ_THERMAL */ ++#endif /* CONFIG_MALI_BIFROST_DEVFREQ */ ++ ++#if MALI_USE_CSF ++ kbase_csf_debugfs_init(kbdev); ++#else ++ debugfs_create_file("serialize_jobs", S_IRUGO | S_IWUSR, ++ kbdev->mali_debugfs_directory, kbdev, ++ &kbasep_serialize_jobs_debugfs_fops); ++#endif ++ ++ return 0; ++ ++out: ++ debugfs_remove_recursive(kbdev->mali_debugfs_directory); ++ return err; ++} ++ ++void kbase_device_debugfs_term(struct kbase_device *kbdev) ++{ ++ debugfs_remove_recursive(kbdev->mali_debugfs_directory); ++} ++#endif /* CONFIG_DEBUG_FS */ ++#endif /* MALI_KBASE_BUILD */ ++ ++int kbase_device_coherency_init(struct kbase_device *kbdev) ++{ ++#ifdef CONFIG_OF ++ u32 supported_coherency_bitmap = ++ kbdev->gpu_props.props.raw_props.coherency_mode; ++ const void *coherency_override_dts; ++ u32 override_coherency, gpu_id; ++ unsigned int prod_id; ++ ++ gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; ++ gpu_id &= GPU_ID_VERSION_PRODUCT_ID; ++ prod_id = gpu_id >> GPU_ID_VERSION_PRODUCT_ID_SHIFT; ++ ++ /* Only for tMIx : ++ * (COHERENCY_ACE_LITE | COHERENCY_ACE) was incorrectly ++ * documented for tMIx so force correct value here. ++ */ ++ if (GPU_ID2_MODEL_MATCH_VALUE(prod_id) == ++ GPU_ID2_PRODUCT_TMIX) ++ if (supported_coherency_bitmap == ++ COHERENCY_FEATURE_BIT(COHERENCY_ACE)) ++ supported_coherency_bitmap |= ++ COHERENCY_FEATURE_BIT(COHERENCY_ACE_LITE); ++ ++#endif /* CONFIG_OF */ ++ ++ kbdev->system_coherency = COHERENCY_NONE; ++ ++ /* device tree may override the coherency */ ++#ifdef CONFIG_OF ++ coherency_override_dts = of_get_property(kbdev->dev->of_node, ++ "system-coherency", ++ NULL); ++ if (coherency_override_dts) { ++ ++ override_coherency = be32_to_cpup(coherency_override_dts); ++ ++ if ((override_coherency <= COHERENCY_NONE) && ++ (supported_coherency_bitmap & ++ COHERENCY_FEATURE_BIT(override_coherency))) { ++ ++ kbdev->system_coherency = override_coherency; ++ ++ dev_info(kbdev->dev, ++ "Using coherency mode %u set from dtb", ++ override_coherency); ++ } else ++ dev_warn(kbdev->dev, ++ "Ignoring unsupported coherency mode %u set from dtb", ++ override_coherency); ++ } ++ ++#endif /* CONFIG_OF */ ++ ++ kbdev->gpu_props.props.raw_props.coherency_mode = ++ kbdev->system_coherency; ++ ++ return 0; ++} ++ ++#ifdef CONFIG_MALI_BUSLOG ++ ++/* Callback used by the kbase bus logger client, to initiate a GPU reset ++ * when the bus log is restarted. GPU reset is used as reference point ++ * in HW bus log analyses. ++ */ ++static void kbase_logging_started_cb(void *data) ++{ ++ struct kbase_device *kbdev = (struct kbase_device *)data; ++ ++ if (kbase_prepare_to_reset_gpu(kbdev)) ++ kbase_reset_gpu(kbdev); ++ dev_info(kbdev->dev, "KBASE - Bus logger restarted\n"); ++} ++ ++int buslog_init(struct kbase_device *kbdev) ++{ ++ int err = 0; ++ ++ err = bl_core_client_register(kbdev->devname, ++ kbase_logging_started_cb, ++ kbdev, &kbdev->buslogger, ++ THIS_MODULE, NULL); ++ if (err == 0) ++ bl_core_set_threshold(kbdev->buslogger, 1024*1024*1024); ++ ++ return err; ++} ++ ++void buslog_term(struct kbase_device *kbdev) ++{ ++ bl_core_client_unregister(kbdev->buslogger); ++} ++#endif ++ ++static struct attribute *kbase_scheduling_attrs[] = { ++#if !MALI_USE_CSF ++ &dev_attr_serialize_jobs.attr, ++#endif /* !MALI_USE_CSF */ ++ NULL ++}; ++ ++static struct attribute *kbase_attrs[] = { ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++ &dev_attr_debug_command.attr, ++#if !MALI_USE_CSF ++ &dev_attr_js_softstop_always.attr, ++#endif /* !MALI_USE_CSF */ ++#endif ++#if !MALI_USE_CSF ++ &dev_attr_js_timeouts.attr, ++ &dev_attr_soft_job_timeout.attr, ++#endif /* !MALI_USE_CSF */ ++ &dev_attr_gpuinfo.attr, ++ &dev_attr_dvfs_period.attr, ++ &dev_attr_pm_poweroff.attr, ++ &dev_attr_reset_timeout.attr, ++#if !MALI_USE_CSF ++ &dev_attr_js_scheduling_period.attr, ++#endif /* !MALI_USE_CSF */ ++ &dev_attr_power_policy.attr, ++ &dev_attr_core_mask.attr, ++ &dev_attr_mem_pool_size.attr, ++ &dev_attr_mem_pool_max_size.attr, ++ &dev_attr_lp_mem_pool_size.attr, ++ &dev_attr_lp_mem_pool_max_size.attr, ++#if !MALI_USE_CSF ++ &dev_attr_js_ctx_scheduling_mode.attr, ++#endif /* !MALI_USE_CSF */ ++ NULL ++}; ++ ++#define SYSFS_SCHEDULING_GROUP "scheduling" ++static const struct attribute_group kbase_scheduling_attr_group = { ++ .name = SYSFS_SCHEDULING_GROUP, ++ .attrs = kbase_scheduling_attrs, ++}; ++ ++static const struct attribute_group kbase_attr_group = { ++ .attrs = kbase_attrs, ++}; ++ ++int kbase_sysfs_init(struct kbase_device *kbdev) ++{ ++ int err = 0; ++ ++ kbdev->mdev.minor = MISC_DYNAMIC_MINOR; ++ kbdev->mdev.name = kbdev->devname; ++ kbdev->mdev.fops = &kbase_fops; ++ kbdev->mdev.parent = get_device(kbdev->dev); ++ kbdev->mdev.mode = 0666; ++ ++ err = sysfs_create_group(&kbdev->dev->kobj, &kbase_attr_group); ++ if (!err) { ++ err = sysfs_create_group(&kbdev->dev->kobj, ++ &kbase_scheduling_attr_group); ++ if (err) { ++ dev_err(kbdev->dev, "Creation of %s sysfs group failed", ++ SYSFS_SCHEDULING_GROUP); ++ sysfs_remove_group(&kbdev->dev->kobj, ++ &kbase_attr_group); ++ } ++ } ++ ++ return err; ++} ++ ++void kbase_sysfs_term(struct kbase_device *kbdev) ++{ ++ sysfs_remove_group(&kbdev->dev->kobj, &kbase_scheduling_attr_group); ++ sysfs_remove_group(&kbdev->dev->kobj, &kbase_attr_group); ++ put_device(kbdev->dev); ++} ++ ++static int kbase_platform_device_remove(struct platform_device *pdev) ++{ ++ struct kbase_device *kbdev = to_kbase_device(&pdev->dev); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ kbase_device_term(kbdev); ++ dev_set_drvdata(kbdev->dev, NULL); ++ kbase_device_free(kbdev); ++ ++ return 0; ++} ++ ++void kbase_backend_devfreq_term(struct kbase_device *kbdev) ++{ ++#ifdef CONFIG_MALI_BIFROST_DEVFREQ ++ if (kbdev->devfreq) ++ kbase_devfreq_term(kbdev); ++#endif ++} ++ ++int kbase_backend_devfreq_init(struct kbase_device *kbdev) ++{ ++#ifdef CONFIG_MALI_BIFROST_DEVFREQ ++ /* Devfreq uses hardware counters, so must be initialized after it. */ ++ int err = kbase_devfreq_init(kbdev); ++ ++ if (err) ++ dev_err(kbdev->dev, "Continuing without devfreq\n"); ++#endif /* CONFIG_MALI_BIFROST_DEVFREQ */ ++ return 0; ++} ++ ++static int kbase_platform_device_probe(struct platform_device *pdev) ++{ ++ struct kbase_device *kbdev; ++ int err = 0; ++ ++ mali_kbase_print_cs_experimental(); ++ ++ kbdev = kbase_device_alloc(); ++ if (!kbdev) { ++ dev_err(&pdev->dev, "Allocate device failed\n"); ++ return -ENOMEM; ++ } ++ ++ kbdev->dev = &pdev->dev; ++ dev_set_drvdata(kbdev->dev, kbdev); ++ ++ err = kbase_device_init(kbdev); ++ ++ if (err) { ++ if (err == -EPROBE_DEFER) ++ dev_err(kbdev->dev, "Device initialization Deferred\n"); ++ else ++ dev_err(kbdev->dev, "Device initialization failed\n"); ++ ++ dev_set_drvdata(kbdev->dev, NULL); ++ kbase_device_free(kbdev); ++ } else { ++#ifdef MALI_KBASE_BUILD ++ dev_info(kbdev->dev, ++ "Probed as %s\n", dev_name(kbdev->mdev.this_device)); ++#endif /* MALI_KBASE_BUILD */ ++ kbase_increment_device_id(); ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++ mutex_lock(&kbdev->pm.lock); ++ kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_INITIALIZED_EVT); ++ mutex_unlock(&kbdev->pm.lock); ++#endif ++ } ++ ++ return err; ++} ++ ++#undef KBASEP_DEFAULT_REGISTER_HISTORY_SIZE ++ ++/** ++ * kbase_device_suspend - Suspend callback from the OS. ++ * ++ * This is called by Linux when the device should suspend. ++ * ++ * @dev: The device to suspend ++ * ++ * Return: A standard Linux error code ++ */ ++static int kbase_device_suspend(struct device *dev) ++{ ++ struct kbase_device *kbdev = to_kbase_device(dev); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ kbase_pm_suspend(kbdev); ++ ++#if defined(CONFIG_MALI_BIFROST_DEVFREQ) && \ ++ (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) ++ dev_dbg(dev, "Callback %s\n", __func__); ++ if (kbdev->devfreq) { ++ kbase_devfreq_enqueue_work(kbdev, DEVFREQ_WORK_SUSPEND); ++ flush_workqueue(kbdev->devfreq_queue.workq); ++ } ++#endif ++ return 0; ++} ++ ++/** ++ * kbase_device_resume - Resume callback from the OS. ++ * ++ * This is called by Linux when the device should resume from suspension. ++ * ++ * @dev: The device to resume ++ * ++ * Return: A standard Linux error code ++ */ ++static int kbase_device_resume(struct device *dev) ++{ ++ struct kbase_device *kbdev = to_kbase_device(dev); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ kbase_pm_resume(kbdev); ++ ++#if defined(CONFIG_MALI_BIFROST_DEVFREQ) && \ ++ (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) ++ dev_dbg(dev, "Callback %s\n", __func__); ++ if (kbdev->devfreq) { ++ mutex_lock(&kbdev->pm.lock); ++ if (kbdev->pm.active_count > 0) ++ kbase_devfreq_enqueue_work(kbdev, DEVFREQ_WORK_RESUME); ++ mutex_unlock(&kbdev->pm.lock); ++ flush_workqueue(kbdev->devfreq_queue.workq); ++ } ++#endif ++ return 0; ++} ++ ++/** ++ * kbase_device_runtime_suspend - Runtime suspend callback from the OS. ++ * ++ * This is called by Linux when the device should prepare for a condition in ++ * which it will not be able to communicate with the CPU(s) and RAM due to ++ * power management. ++ * ++ * @dev: The device to suspend ++ * ++ * Return: A standard Linux error code ++ */ ++#ifdef KBASE_PM_RUNTIME ++static int kbase_device_runtime_suspend(struct device *dev) ++{ ++ struct kbase_device *kbdev = to_kbase_device(dev); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++#if defined(CONFIG_MALI_BIFROST_DEVFREQ) && \ ++ (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) ++ if (kbdev->devfreq) ++ kbase_devfreq_enqueue_work(kbdev, DEVFREQ_WORK_SUSPEND); ++#endif ++ ++ if (kbdev->pm.backend.callback_power_runtime_off) { ++ kbdev->pm.backend.callback_power_runtime_off(kbdev); ++ dev_dbg(dev, "runtime suspend\n"); ++ } ++ return 0; ++} ++#endif /* KBASE_PM_RUNTIME */ ++ ++/** ++ * kbase_device_runtime_resume - Runtime resume callback from the OS. ++ * ++ * This is called by Linux when the device should go into a fully active state. ++ * ++ * @dev: The device to suspend ++ * ++ * Return: A standard Linux error code ++ */ ++ ++#ifdef KBASE_PM_RUNTIME ++static int kbase_device_runtime_resume(struct device *dev) ++{ ++ int ret = 0; ++ struct kbase_device *kbdev = to_kbase_device(dev); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ dev_dbg(dev, "Callback %s\n", __func__); ++ if (kbdev->pm.backend.callback_power_runtime_on) { ++ ret = kbdev->pm.backend.callback_power_runtime_on(kbdev); ++ dev_dbg(dev, "runtime resume\n"); ++ } ++ ++#if defined(CONFIG_MALI_BIFROST_DEVFREQ) && \ ++ (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) ++ if (kbdev->devfreq) ++ kbase_devfreq_enqueue_work(kbdev, DEVFREQ_WORK_RESUME); ++#endif ++ ++ return ret; ++} ++#endif /* KBASE_PM_RUNTIME */ ++ ++ ++#ifdef KBASE_PM_RUNTIME ++/** ++ * kbase_device_runtime_idle - Runtime idle callback from the OS. ++ * @dev: The device to suspend ++ * ++ * This is called by Linux when the device appears to be inactive and it might ++ * be placed into a low power state. ++ * ++ * Return: 0 if device can be suspended, non-zero to avoid runtime autosuspend, ++ * otherwise a standard Linux error code ++ */ ++static int kbase_device_runtime_idle(struct device *dev) ++{ ++ struct kbase_device *kbdev = to_kbase_device(dev); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ dev_dbg(dev, "Callback %s\n", __func__); ++ /* Use platform specific implementation if it exists. */ ++ if (kbdev->pm.backend.callback_power_runtime_idle) ++ return kbdev->pm.backend.callback_power_runtime_idle(kbdev); ++ ++ /* Just need to update the device's last busy mark. Kernel will respect ++ * the autosuspend delay and so won't suspend the device immediately. ++ */ ++ pm_runtime_mark_last_busy(kbdev->dev); ++ return 0; ++} ++#endif /* KBASE_PM_RUNTIME */ ++ ++/* The power management operations for the platform driver. ++ */ ++static const struct dev_pm_ops kbase_pm_ops = { ++ .suspend = kbase_device_suspend, ++ .resume = kbase_device_resume, ++#ifdef KBASE_PM_RUNTIME ++ .runtime_suspend = kbase_device_runtime_suspend, ++ .runtime_resume = kbase_device_runtime_resume, ++ .runtime_idle = kbase_device_runtime_idle, ++#endif /* KBASE_PM_RUNTIME */ ++}; ++ ++#ifdef CONFIG_OF ++static const struct of_device_id kbase_dt_ids[] = { ++ { .compatible = "arm,mali-bifrost" }, ++ { /* sentinel */ } ++}; ++MODULE_DEVICE_TABLE(of, kbase_dt_ids); ++#endif ++ ++static struct platform_driver kbase_platform_driver = { ++ .probe = kbase_platform_device_probe, ++ .remove = kbase_platform_device_remove, ++ .driver = { ++ .name = kbase_drv_name, ++ .owner = THIS_MODULE, ++ .pm = &kbase_pm_ops, ++ .of_match_table = of_match_ptr(kbase_dt_ids), ++ }, ++}; ++ ++/* ++ * The driver will not provide a shortcut to create the Mali platform device ++ * anymore when using Device Tree. ++ */ ++#ifdef CONFIG_OF ++module_platform_driver(kbase_platform_driver); ++#else ++ ++static int __init kbase_driver_init(void) ++{ ++ int ret; ++ ++ ret = kbase_platform_register(); ++ if (ret) ++ return ret; ++ ++ ret = platform_driver_register(&kbase_platform_driver); ++ ++ if (ret) ++ kbase_platform_unregister(); ++ ++ return ret; ++} ++ ++static void __exit kbase_driver_exit(void) ++{ ++ platform_driver_unregister(&kbase_platform_driver); ++ kbase_platform_unregister(); ++} ++ ++module_init(kbase_driver_init); ++module_exit(kbase_driver_exit); ++ ++#endif /* CONFIG_OF */ ++ ++MODULE_LICENSE("GPL"); ++MODULE_VERSION(MALI_RELEASE_NAME " (UK version " \ ++ __stringify(BASE_UK_VERSION_MAJOR) "." \ ++ __stringify(BASE_UK_VERSION_MINOR) ")"); ++ ++#define CREATE_TRACE_POINTS ++/* Create the trace points (otherwise we just get code to call a tracepoint) */ ++#include "mali_linux_trace.h" ++ ++#ifdef CONFIG_MALI_BIFROST_GATOR_SUPPORT ++EXPORT_TRACEPOINT_SYMBOL_GPL(mali_job_slots_event); ++EXPORT_TRACEPOINT_SYMBOL_GPL(mali_pm_status); ++EXPORT_TRACEPOINT_SYMBOL_GPL(mali_page_fault_insert_pages); ++EXPORT_TRACEPOINT_SYMBOL_GPL(mali_total_alloc_pages_change); ++ ++void kbase_trace_mali_pm_status(u32 dev_id, u32 event, u64 value) ++{ ++ trace_mali_pm_status(dev_id, event, value); ++} ++ ++void kbase_trace_mali_job_slots_event(u32 dev_id, u32 event, const struct kbase_context *kctx, u8 atom_id) ++{ ++ trace_mali_job_slots_event(dev_id, event, ++ (kctx != NULL ? kctx->tgid : 0), ++ (kctx != NULL ? kctx->pid : 0), ++ atom_id); ++} ++ ++void kbase_trace_mali_page_fault_insert_pages(u32 dev_id, int event, u32 value) ++{ ++ trace_mali_page_fault_insert_pages(dev_id, event, value); ++} ++ ++void kbase_trace_mali_total_alloc_pages_change(u32 dev_id, long long int event) ++{ ++ trace_mali_total_alloc_pages_change(dev_id, event); ++} ++#endif /* CONFIG_MALI_BIFROST_GATOR_SUPPORT */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_cs_experimental.h b/drivers/gpu/arm/bifrost/mali_kbase_cs_experimental.h +new file mode 100755 +index 000000000000..caba2cd7a0e3 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_cs_experimental.h +@@ -0,0 +1,51 @@ ++/* ++ * ++ * (C) COPYRIGHT ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ *//* SPDX-License-Identifier: GPL-2.0 */ ++ ++/* ++ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ */ ++ ++#ifndef _KBASE_CS_EXPERIMENTAL_H_ ++#define _KBASE_CS_EXPERIMENTAL_H_ ++ ++#include ++ ++/** ++ * mali_kbase_print_cs_experimental() - Print a string for every Core Services ++ * experimental feature that is enabled ++ */ ++static inline void mali_kbase_print_cs_experimental(void) ++{ ++#if MALI_INCREMENTAL_RENDERING ++ pr_info("mali_kbase: INCREMENTAL_RENDERING (experimental) enabled"); ++#endif /* MALI_INCREMENTAL_RENDERING */ ++} ++ ++#endif /* _KBASE_CS_EXPERIMENTAL_H_ */ ++ ++ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_ctx_sched.c b/drivers/gpu/arm/bifrost/mali_kbase_ctx_sched.c +new file mode 100755 +index 000000000000..750dbd8c3924 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_ctx_sched.c +@@ -0,0 +1,355 @@ ++/* ++ * ++ * (C) COPYRIGHT 2017-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++#include ++ ++#include ++#include "mali_kbase_ctx_sched.h" ++#include "tl/mali_kbase_tracepoints.h" ++ ++/* Helper for ktrace */ ++#if KBASE_KTRACE_ENABLE ++static int kbase_ktrace_get_ctx_refcnt(struct kbase_context *kctx) ++{ ++ return atomic_read(&kctx->refcount); ++} ++#else /* KBASE_KTRACE_ENABLE */ ++static int kbase_ktrace_get_ctx_refcnt(struct kbase_context *kctx) ++{ ++ CSTD_UNUSED(kctx); ++ return 0; ++} ++#endif /* KBASE_KTRACE_ENABLE */ ++ ++int kbase_ctx_sched_init(struct kbase_device *kbdev) ++{ ++ int as_present = (1U << kbdev->nr_hw_address_spaces) - 1; ++ ++ /* These two must be recalculated if nr_hw_address_spaces changes ++ * (e.g. for HW workarounds) */ ++ kbdev->nr_user_address_spaces = kbdev->nr_hw_address_spaces; ++ kbdev->as_free = as_present; /* All ASs initially free */ ++ ++ memset(kbdev->as_to_kctx, 0, sizeof(kbdev->as_to_kctx)); ++ ++ return 0; ++} ++ ++void kbase_ctx_sched_term(struct kbase_device *kbdev) ++{ ++ s8 i; ++ ++ /* Sanity checks */ ++ for (i = 0; i != kbdev->nr_hw_address_spaces; ++i) { ++ WARN_ON(kbdev->as_to_kctx[i] != NULL); ++ WARN_ON(!(kbdev->as_free & (1u << i))); ++ } ++} ++ ++/* kbasep_ctx_sched_find_as_for_ctx - Find a free address space ++ * ++ * @kbdev: The context for which to find a free address space ++ * ++ * Return: A valid AS if successful, otherwise KBASEP_AS_NR_INVALID ++ * ++ * This function returns an address space available for use. It would prefer ++ * returning an AS that has been previously assigned to the context to ++ * avoid having to reprogram the MMU. ++ */ ++static int kbasep_ctx_sched_find_as_for_ctx(struct kbase_context *kctx) ++{ ++ struct kbase_device *const kbdev = kctx->kbdev; ++ int free_as; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ /* First check if the previously assigned AS is available */ ++ if ((kctx->as_nr != KBASEP_AS_NR_INVALID) && ++ (kbdev->as_free & (1u << kctx->as_nr))) ++ return kctx->as_nr; ++ ++ /* The previously assigned AS was taken, we'll be returning any free ++ * AS at this point. ++ */ ++ free_as = ffs(kbdev->as_free) - 1; ++ if (free_as >= 0 && free_as < kbdev->nr_hw_address_spaces) ++ return free_as; ++ ++ return KBASEP_AS_NR_INVALID; ++} ++ ++int kbase_ctx_sched_retain_ctx(struct kbase_context *kctx) ++{ ++ struct kbase_device *const kbdev = kctx->kbdev; ++ ++ lockdep_assert_held(&kbdev->mmu_hw_mutex); ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ WARN_ON(!kbdev->pm.backend.gpu_powered); ++ ++ if (atomic_inc_return(&kctx->refcount) == 1) { ++ int const free_as = kbasep_ctx_sched_find_as_for_ctx(kctx); ++ ++ if (free_as != KBASEP_AS_NR_INVALID) { ++ kbdev->as_free &= ~(1u << free_as); ++ /* Only program the MMU if the context has not been ++ * assigned the same address space before. ++ */ ++ if (free_as != kctx->as_nr) { ++ struct kbase_context *const prev_kctx = ++ kbdev->as_to_kctx[free_as]; ++ ++ if (prev_kctx) { ++ WARN_ON(atomic_read(&prev_kctx->refcount) != 0); ++ kbase_mmu_disable(prev_kctx); ++ KBASE_TLSTREAM_TL_KBASE_CTX_UNASSIGN_AS( ++ kbdev, prev_kctx->id); ++ prev_kctx->as_nr = KBASEP_AS_NR_INVALID; ++ } ++ ++ kctx->as_nr = free_as; ++ kbdev->as_to_kctx[free_as] = kctx; ++ KBASE_TLSTREAM_TL_KBASE_CTX_ASSIGN_AS( ++ kbdev, kctx->id, free_as); ++ kbase_mmu_update(kbdev, &kctx->mmu, ++ kctx->as_nr); ++ } ++ } else { ++ atomic_dec(&kctx->refcount); ++ ++ /* Failed to find an available address space, we must ++ * be returning an error at this point. ++ */ ++ WARN_ON(kctx->as_nr != KBASEP_AS_NR_INVALID); ++ } ++ } ++ ++ return kctx->as_nr; ++} ++ ++void kbase_ctx_sched_retain_ctx_refcount(struct kbase_context *kctx) ++{ ++ struct kbase_device *const kbdev = kctx->kbdev; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ WARN_ON(atomic_read(&kctx->refcount) == 0); ++ WARN_ON(kctx->as_nr == KBASEP_AS_NR_INVALID); ++ WARN_ON(kbdev->as_to_kctx[kctx->as_nr] != kctx); ++ ++ atomic_inc(&kctx->refcount); ++} ++ ++void kbase_ctx_sched_release_ctx(struct kbase_context *kctx) ++{ ++ struct kbase_device *const kbdev = kctx->kbdev; ++ int new_ref_count; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ new_ref_count = atomic_dec_return(&kctx->refcount); ++ if (new_ref_count == 0) { ++ kbdev->as_free |= (1u << kctx->as_nr); ++ if (kbase_ctx_flag(kctx, KCTX_AS_DISABLED_ON_FAULT)) { ++ KBASE_TLSTREAM_TL_KBASE_CTX_UNASSIGN_AS( ++ kbdev, kctx->id); ++ kbdev->as_to_kctx[kctx->as_nr] = NULL; ++ kctx->as_nr = KBASEP_AS_NR_INVALID; ++ kbase_ctx_flag_clear(kctx, KCTX_AS_DISABLED_ON_FAULT); ++ } ++ } ++ ++ KBASE_KTRACE_ADD(kbdev, SCHED_RELEASE_CTX, kctx, new_ref_count); ++} ++ ++void kbase_ctx_sched_remove_ctx(struct kbase_context *kctx) ++{ ++ struct kbase_device *const kbdev = kctx->kbdev; ++ ++ lockdep_assert_held(&kbdev->mmu_hw_mutex); ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ WARN_ON(atomic_read(&kctx->refcount) != 0); ++ ++ if (kctx->as_nr != KBASEP_AS_NR_INVALID) { ++ if (kbdev->pm.backend.gpu_powered) ++ kbase_mmu_disable(kctx); ++ ++ KBASE_TLSTREAM_TL_KBASE_CTX_UNASSIGN_AS(kbdev, kctx->id); ++ kbdev->as_to_kctx[kctx->as_nr] = NULL; ++ kctx->as_nr = KBASEP_AS_NR_INVALID; ++ } ++} ++ ++void kbase_ctx_sched_restore_all_as(struct kbase_device *kbdev) ++{ ++ s8 i; ++ ++ lockdep_assert_held(&kbdev->mmu_hw_mutex); ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ WARN_ON(!kbdev->pm.backend.gpu_powered); ++ ++ for (i = 0; i != kbdev->nr_hw_address_spaces; ++i) { ++ struct kbase_context *kctx; ++ ++#if MALI_USE_CSF ++ if ((i == MCU_AS_NR) && kbdev->csf.firmware_inited) { ++ kbase_mmu_update(kbdev, &kbdev->csf.mcu_mmu, ++ MCU_AS_NR); ++ continue; ++ } ++#endif ++ kctx = kbdev->as_to_kctx[i]; ++ if (kctx) { ++ if (atomic_read(&kctx->refcount)) { ++ WARN_ON(kctx->as_nr != i); ++ ++ kbase_mmu_update(kbdev, &kctx->mmu, ++ kctx->as_nr); ++ kbase_ctx_flag_clear(kctx, ++ KCTX_AS_DISABLED_ON_FAULT); ++ } else { ++ /* This context might have been assigned an ++ * AS before, clear it. ++ */ ++ if (kctx->as_nr != KBASEP_AS_NR_INVALID) { ++ KBASE_TLSTREAM_TL_KBASE_CTX_UNASSIGN_AS( ++ kbdev, kctx->id); ++ kbdev->as_to_kctx[kctx->as_nr] = NULL; ++ kctx->as_nr = KBASEP_AS_NR_INVALID; ++ } ++ } ++ } else { ++ kbase_mmu_disable_as(kbdev, i); ++ } ++ } ++} ++ ++struct kbase_context *kbase_ctx_sched_as_to_ctx_refcount( ++ struct kbase_device *kbdev, size_t as_nr) ++{ ++ unsigned long flags; ++ struct kbase_context *found_kctx = NULL; ++ ++ if (WARN_ON(kbdev == NULL)) ++ return NULL; ++ ++ if (WARN_ON(as_nr >= BASE_MAX_NR_AS)) ++ return NULL; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ found_kctx = kbdev->as_to_kctx[as_nr]; ++ ++ if (found_kctx != NULL) ++ kbase_ctx_sched_retain_ctx_refcount(found_kctx); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ return found_kctx; ++} ++ ++struct kbase_context *kbase_ctx_sched_as_to_ctx(struct kbase_device *kbdev, ++ size_t as_nr) ++{ ++ unsigned long flags; ++ struct kbase_context *found_kctx; ++ ++ if (WARN_ON(kbdev == NULL)) ++ return NULL; ++ ++ if (WARN_ON(as_nr >= BASE_MAX_NR_AS)) ++ return NULL; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ found_kctx = kbdev->as_to_kctx[as_nr]; ++ ++ if (found_kctx) { ++ if (WARN_ON(atomic_read(&found_kctx->refcount) <= 0)) ++ found_kctx = NULL; ++ } ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ return found_kctx; ++} ++ ++bool kbase_ctx_sched_inc_refcount_nolock(struct kbase_context *kctx) ++{ ++ bool result = false; ++ int as_nr; ++ ++ if (WARN_ON(kctx == NULL)) ++ return result; ++ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ ++ as_nr = kctx->as_nr; ++ if (atomic_read(&kctx->refcount) > 0) { ++ KBASE_DEBUG_ASSERT(as_nr >= 0); ++ ++ kbase_ctx_sched_retain_ctx_refcount(kctx); ++ KBASE_KTRACE_ADD(kctx->kbdev, SCHED_RETAIN_CTX_NOLOCK, kctx, ++ kbase_ktrace_get_ctx_refcnt(kctx)); ++ result = true; ++ } ++ ++ return result; ++} ++ ++bool kbase_ctx_sched_inc_refcount(struct kbase_context *kctx) ++{ ++ unsigned long flags; ++ bool result = false; ++ ++ if (WARN_ON(kctx == NULL)) ++ return result; ++ ++ if (WARN_ON(kctx->kbdev == NULL)) ++ return result; ++ ++ mutex_lock(&kctx->kbdev->mmu_hw_mutex); ++ spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, flags); ++ result = kbase_ctx_sched_inc_refcount_nolock(kctx); ++ spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, flags); ++ mutex_unlock(&kctx->kbdev->mmu_hw_mutex); ++ ++ return result; ++} ++ ++void kbase_ctx_sched_release_ctx_lock(struct kbase_context *kctx) ++{ ++ unsigned long flags; ++ ++ if (WARN_ON(!kctx)) ++ return; ++ ++ spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, flags); ++ ++ if (!WARN_ON(kctx->as_nr == KBASEP_AS_NR_INVALID) && ++ !WARN_ON(atomic_read(&kctx->refcount) <= 0)) ++ kbase_ctx_sched_release_ctx(kctx); ++ ++ spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, flags); ++} +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_ctx_sched.h b/drivers/gpu/arm/bifrost/mali_kbase_ctx_sched.h +new file mode 100755 +index 000000000000..1affa719e6dc +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_ctx_sched.h +@@ -0,0 +1,209 @@ ++/* ++ * ++ * (C) COPYRIGHT 2017-2018, 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_CTX_SCHED_H_ ++#define _KBASE_CTX_SCHED_H_ ++ ++#include ++ ++/** ++ * The Context Scheduler manages address space assignment and reference ++ * counting to kbase_context. The interface has been designed to minimise ++ * interactions between the Job Scheduler and Power Management/MMU to support ++ * the existing Job Scheduler interface. ++ * ++ * The initial implementation of the Context Scheduler does not schedule ++ * contexts. Instead it relies on the Job Scheduler to make decisions of ++ * when to schedule/evict contexts if address spaces are starved. In the ++ * future, once an interface between the CS and JS has been devised to ++ * provide enough information about how each context is consuming GPU resources, ++ * those decisions can be made in the CS itself, thereby reducing duplicated ++ * code. ++ */ ++ ++/** ++ * kbase_ctx_sched_init - Initialise the context scheduler ++ * @kbdev: The device for which the context scheduler needs to be initialised ++ * ++ * This must be called during device initialisation. The number of hardware ++ * address spaces must already be established before calling this function. ++ * ++ * Return: 0 for success, otherwise failure ++ */ ++int kbase_ctx_sched_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_ctx_sched_term - Terminate the context scheduler ++ * @kbdev: The device for which the context scheduler needs to be terminated ++ * ++ * This must be called during device termination after all contexts have been ++ * destroyed. ++ */ ++void kbase_ctx_sched_term(struct kbase_device *kbdev); ++ ++/** ++ * kbase_ctx_sched_retain_ctx - Retain a reference to the @ref kbase_context ++ * @kctx: The context to which to retain a reference ++ * ++ * This function should be called whenever an address space should be assigned ++ * to a context and programmed onto the MMU. It should typically be called ++ * when jobs are ready to be submitted to the GPU. ++ * ++ * It can be called as many times as necessary. The address space will be ++ * assigned to the context for as long as there is a reference to said context. ++ * ++ * The kbase_device::mmu_hw_mutex and kbase_device::hwaccess_lock locks must be ++ * held whilst calling this function. ++ * ++ * Return: The address space that the context has been assigned to or ++ * KBASEP_AS_NR_INVALID if no address space was available. ++ */ ++int kbase_ctx_sched_retain_ctx(struct kbase_context *kctx); ++ ++/** ++ * kbase_ctx_sched_retain_ctx_refcount ++ * @kctx: The context to which to retain a reference ++ * ++ * This function only retains a reference to the context. It must be called ++ * only when the context already has a reference. ++ * ++ * This is typically called inside an atomic session where we know the context ++ * is already scheduled in but want to take an extra reference to ensure that ++ * it doesn't get descheduled. ++ * ++ * The kbase_device::hwaccess_lock must be held whilst calling this function ++ */ ++void kbase_ctx_sched_retain_ctx_refcount(struct kbase_context *kctx); ++ ++/** ++ * kbase_ctx_sched_release_ctx - Release a reference to the @ref kbase_context ++ * @kctx: The context from which to release a reference ++ * ++ * This function should be called whenever an address space could be unassigned ++ * from a context. When there are no more references to said context, the ++ * address space previously assigned to this context shall be reassigned to ++ * other contexts as needed. ++ * ++ * The kbase_device::hwaccess_lock must be held whilst calling this function ++ */ ++void kbase_ctx_sched_release_ctx(struct kbase_context *kctx); ++ ++/** ++ * kbase_ctx_sched_remove_ctx - Unassign previously assigned address space ++ * @kctx: The context to be removed ++ * ++ * This function should be called when a context is being destroyed. The ++ * context must no longer have any reference. If it has been assigned an ++ * address space before then the AS will be unprogrammed. ++ * ++ * The kbase_device::mmu_hw_mutex and kbase_device::hwaccess_lock locks must be ++ * held whilst calling this function. ++ */ ++void kbase_ctx_sched_remove_ctx(struct kbase_context *kctx); ++ ++/** ++ * kbase_ctx_sched_restore_all_as - Reprogram all address spaces ++ * @kbdev: The device for which address spaces to be reprogrammed ++ * ++ * This function shall reprogram all address spaces previously assigned to ++ * contexts. It can be used after the GPU is reset. ++ * ++ * The kbase_device::mmu_hw_mutex and kbase_device::hwaccess_lock locks must be ++ * held whilst calling this function. ++ */ ++void kbase_ctx_sched_restore_all_as(struct kbase_device *kbdev); ++ ++/** ++ * kbase_ctx_sched_as_to_ctx_refcount - Lookup a context based on its current ++ * address space and ensure that is stays scheduled in ++ * @kbdev: The device for which the returned context must belong ++ * @as_nr: address space assigned to the context of interest ++ * ++ * The context is refcounted as being busy to prevent it from scheduling ++ * out. It must be released with kbase_ctx_sched_release_ctx() when it is no ++ * longer required to stay scheduled in. ++ * ++ * This function can safely be called from IRQ context. ++ * ++ * The following locking conditions are made on the caller: ++ * * it must not hold the kbase_device::hwaccess_lock, because it will be used ++ * internally. ++ * ++ * Return: a valid struct kbase_context on success, which has been refcounted ++ * as being busy or return NULL on failure, indicating that no context was found ++ * in as_nr. ++ */ ++struct kbase_context *kbase_ctx_sched_as_to_ctx_refcount( ++ struct kbase_device *kbdev, size_t as_nr); ++ ++/** ++ * kbase_ctx_sched_as_to_ctx - Lookup a context based on its current address ++ * space ++ * @kbdev: The device for which the returned context must belong ++ * @as_nr: address space assigned to the context of interest ++ * ++ * Return: a valid struct kbase_context on success or NULL on failure, ++ * indicating that no context was found in as_nr. ++ */ ++struct kbase_context *kbase_ctx_sched_as_to_ctx(struct kbase_device *kbdev, ++ size_t as_nr); ++ ++/** ++ * kbase_ctx_sched_inc_refcount_nolock - Refcount a context as being busy, ++ * preventing it from being scheduled out. ++ * @kctx: Context to be refcounted ++ * ++ * The following locks must be held by the caller: ++ * * kbase_device::mmu_hw_mutex ++ * * kbase_device::hwaccess_lock ++ * ++ * Return: true if refcount succeeded, and the context will not be scheduled ++ * out, false if the refcount failed (because the context is being/has been ++ * scheduled out). ++ */ ++bool kbase_ctx_sched_inc_refcount_nolock(struct kbase_context *kctx); ++ ++/** ++ * kbase_ctx_sched_inc_refcount - Refcount a context as being busy, preventing ++ * it from being scheduled out. ++ * @kctx: Context to be refcounted ++ * ++ * The following locking conditions are made on the caller: ++ * * it must not hold kbase_device::mmu_hw_mutex and ++ * kbase_device::hwaccess_lock, because they will be used internally. ++ * ++ * Return: true if refcount succeeded, and the context will not be scheduled ++ * out, false if the refcount failed (because the context is being/has been ++ * scheduled out). ++ */ ++bool kbase_ctx_sched_inc_refcount(struct kbase_context *kctx); ++ ++/** ++ * kbase_ctx_sched_release_ctx_lock - Release a reference count of a context ++ * @kctx: Context for which refcount should be decreased ++ * ++ * Effectivelly, this is a wrapper for kbase_ctx_sched_release_ctx, but ++ * kbase_device::hwaccess_lock is required NOT to be locked. ++ */ ++void kbase_ctx_sched_release_ctx_lock(struct kbase_context *kctx); ++ ++#endif /* _KBASE_CTX_SCHED_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_debug.c b/drivers/gpu/arm/bifrost/mali_kbase_debug.c +new file mode 100755 +index 000000000000..118f787fb74c +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_debug.c +@@ -0,0 +1,44 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2014 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++#include ++ ++static struct kbasep_debug_assert_cb kbasep_debug_assert_registered_cb = { ++ NULL, ++ NULL ++}; ++ ++void kbase_debug_assert_register_hook(kbase_debug_assert_hook *func, void *param) ++{ ++ kbasep_debug_assert_registered_cb.func = func; ++ kbasep_debug_assert_registered_cb.param = param; ++} ++ ++void kbasep_debug_assert_call_hook(void) ++{ ++ if (kbasep_debug_assert_registered_cb.func != NULL) ++ kbasep_debug_assert_registered_cb.func(kbasep_debug_assert_registered_cb.param); ++} ++KBASE_EXPORT_SYMBOL(kbasep_debug_assert_call_hook); ++ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_debug.h b/drivers/gpu/arm/bifrost/mali_kbase_debug.h +new file mode 100755 +index 000000000000..f33413908405 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_debug.h +@@ -0,0 +1,169 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2015, 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++#ifndef _KBASE_DEBUG_H ++#define _KBASE_DEBUG_H ++ ++#include ++ ++/** @brief If equals to 0, a trace containing the file, line, and function will be displayed before each message. */ ++#define KBASE_DEBUG_SKIP_TRACE 0 ++ ++/** @brief If different from 0, the trace will only contain the file and line. */ ++#define KBASE_DEBUG_SKIP_FUNCTION_NAME 0 ++ ++/** @brief Disable the asserts tests if set to 1. Default is to disable the asserts in release. */ ++#ifndef KBASE_DEBUG_DISABLE_ASSERTS ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++#define KBASE_DEBUG_DISABLE_ASSERTS 0 ++#else ++#define KBASE_DEBUG_DISABLE_ASSERTS 1 ++#endif ++#endif /* KBASE_DEBUG_DISABLE_ASSERTS */ ++ ++/** Function type that is called on an KBASE_DEBUG_ASSERT() or KBASE_DEBUG_ASSERT_MSG() */ ++typedef void (kbase_debug_assert_hook) (void *); ++ ++struct kbasep_debug_assert_cb { ++ kbase_debug_assert_hook *func; ++ void *param; ++}; ++ ++/** ++ * @def KBASEP_DEBUG_PRINT_TRACE ++ * @brief Private macro containing the format of the trace to display before every message ++ * @sa KBASE_DEBUG_SKIP_TRACE, KBASE_DEBUG_SKIP_FUNCTION_NAME ++ */ ++#if !KBASE_DEBUG_SKIP_TRACE ++#define KBASEP_DEBUG_PRINT_TRACE \ ++ "In file: " __FILE__ " line: " CSTD_STR2(__LINE__) ++#if !KBASE_DEBUG_SKIP_FUNCTION_NAME ++#define KBASEP_DEBUG_PRINT_FUNCTION __func__ ++#else ++#define KBASEP_DEBUG_PRINT_FUNCTION "" ++#endif ++#else ++#define KBASEP_DEBUG_PRINT_TRACE "" ++#endif ++ ++/** ++ * @def KBASEP_DEBUG_ASSERT_OUT(trace, function, ...) ++ * @brief (Private) system printing function associated to the @ref KBASE_DEBUG_ASSERT_MSG event. ++ * @param trace location in the code from where the message is printed ++ * @param function function from where the message is printed ++ * @param ... Format string followed by format arguments. ++ * @note function parameter cannot be concatenated with other strings ++ */ ++/* Select the correct system output function*/ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++#define KBASEP_DEBUG_ASSERT_OUT(trace, function, ...)\ ++ do { \ ++ pr_err("Mali: %s function:%s ", trace, function);\ ++ pr_err(__VA_ARGS__);\ ++ pr_err("\n");\ ++ } while (false) ++#else ++#define KBASEP_DEBUG_ASSERT_OUT(trace, function, ...) CSTD_NOP() ++#endif ++ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++#define KBASE_CALL_ASSERT_HOOK() kbasep_debug_assert_call_hook() ++#else ++#define KBASE_CALL_ASSERT_HOOK() CSTD_NOP() ++#endif ++ ++/** ++ * @def KBASE_DEBUG_ASSERT(expr) ++ * @brief Calls @ref KBASE_PRINT_ASSERT and prints the expression @a expr if @a expr is false ++ * ++ * @note This macro does nothing if the flag @ref KBASE_DEBUG_DISABLE_ASSERTS is set to 1 ++ * ++ * @param expr Boolean expression ++ */ ++#define KBASE_DEBUG_ASSERT(expr) \ ++ KBASE_DEBUG_ASSERT_MSG(expr, #expr) ++ ++#if KBASE_DEBUG_DISABLE_ASSERTS ++#define KBASE_DEBUG_ASSERT_MSG(expr, ...) CSTD_NOP() ++#else ++ /** ++ * @def KBASE_DEBUG_ASSERT_MSG(expr, ...) ++ * @brief Calls @ref KBASEP_DEBUG_ASSERT_OUT and prints the given message if @a expr is false ++ * ++ * @note This macro does nothing if the flag @ref KBASE_DEBUG_DISABLE_ASSERTS is set to 1 ++ * ++ * @param expr Boolean expression ++ * @param ... Message to display when @a expr is false, as a format string followed by format arguments. ++ */ ++#define KBASE_DEBUG_ASSERT_MSG(expr, ...) \ ++ do { \ ++ if (!(expr)) { \ ++ KBASEP_DEBUG_ASSERT_OUT(KBASEP_DEBUG_PRINT_TRACE, KBASEP_DEBUG_PRINT_FUNCTION, __VA_ARGS__);\ ++ KBASE_CALL_ASSERT_HOOK();\ ++ BUG();\ ++ } \ ++ } while (false) ++#endif /* KBASE_DEBUG_DISABLE_ASSERTS */ ++ ++/** ++ * @def KBASE_DEBUG_CODE( X ) ++ * @brief Executes the code inside the macro only in debug mode ++ * ++ * @param X Code to compile only in debug mode. ++ */ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++#define KBASE_DEBUG_CODE(X) X ++#else ++#define KBASE_DEBUG_CODE(X) CSTD_NOP() ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++ ++/** @} */ ++ ++/** ++ * @brief Register a function to call on ASSERT ++ * ++ * Such functions will \b only be called during Debug mode, and for debugging ++ * features \b only. Do not rely on them to be called in general use. ++ * ++ * To disable the hook, supply NULL to \a func. ++ * ++ * @note This function is not thread-safe, and should only be used to ++ * register/deregister once in the module's lifetime. ++ * ++ * @param[in] func the function to call when an assert is triggered. ++ * @param[in] param the parameter to pass to \a func when calling it ++ */ ++void kbase_debug_assert_register_hook(kbase_debug_assert_hook *func, void *param); ++ ++/** ++ * @brief Call a debug assert hook previously registered with kbase_debug_assert_register_hook() ++ * ++ * @note This function is not thread-safe with respect to multiple threads ++ * registering functions and parameters with ++ * kbase_debug_assert_register_hook(). Otherwise, thread safety is the ++ * responsibility of the registered hook. ++ */ ++void kbasep_debug_assert_call_hook(void); ++ ++#endif /* _KBASE_DEBUG_H */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_debug_job_fault.c b/drivers/gpu/arm/bifrost/mali_kbase_debug_job_fault.c +new file mode 100755 +index 000000000000..dbc774d56ab4 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_debug_job_fault.c +@@ -0,0 +1,566 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2016, 2018-2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++#include ++#include ++ ++#ifdef CONFIG_DEBUG_FS ++ ++static bool kbase_is_job_fault_event_pending(struct kbase_device *kbdev) ++{ ++ struct list_head *event_list = &kbdev->job_fault_event_list; ++ unsigned long flags; ++ bool ret; ++ ++ spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); ++ ret = !list_empty(event_list); ++ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); ++ ++ return ret; ++} ++ ++static void kbase_ctx_remove_pending_event(struct kbase_context *kctx) ++{ ++ struct list_head *event_list = &kctx->kbdev->job_fault_event_list; ++ struct base_job_fault_event *event; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kctx->kbdev->job_fault_event_lock, flags); ++ list_for_each_entry(event, event_list, head) { ++ if (event->katom->kctx == kctx) { ++ list_del(&event->head); ++ spin_unlock_irqrestore(&kctx->kbdev->job_fault_event_lock, flags); ++ ++ wake_up(&kctx->kbdev->job_fault_resume_wq); ++ flush_work(&event->job_fault_work); ++ ++ /* job_fault_event_list can only have a single atom for ++ * each context. ++ */ ++ return; ++ } ++ } ++ spin_unlock_irqrestore(&kctx->kbdev->job_fault_event_lock, flags); ++} ++ ++static bool kbase_ctx_has_no_event_pending(struct kbase_context *kctx) ++{ ++ struct kbase_device *kbdev = kctx->kbdev; ++ struct list_head *event_list = &kctx->kbdev->job_fault_event_list; ++ struct base_job_fault_event *event; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); ++ if (list_empty(event_list)) { ++ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); ++ return true; ++ } ++ list_for_each_entry(event, event_list, head) { ++ if (event->katom->kctx == kctx) { ++ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, ++ flags); ++ return false; ++ } ++ } ++ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); ++ return true; ++} ++ ++static int wait_for_job_fault(struct kbase_device *kbdev) ++{ ++#if KERNEL_VERSION(4, 7, 0) <= LINUX_VERSION_CODE && \ ++ KERNEL_VERSION(4, 15, 0) > LINUX_VERSION_CODE ++ int ret = wait_event_interruptible_timeout(kbdev->job_fault_wq, ++ kbase_is_job_fault_event_pending(kbdev), ++ msecs_to_jiffies(2000)); ++ if (ret == 0) ++ return -EAGAIN; ++ else if (ret > 0) ++ return 0; ++ else ++ return ret; ++#else ++ return wait_event_interruptible(kbdev->job_fault_wq, ++ kbase_is_job_fault_event_pending(kbdev)); ++#endif ++} ++ ++/* wait until the fault happen and copy the event */ ++static int kbase_job_fault_event_wait(struct kbase_device *kbdev, ++ struct base_job_fault_event *event) ++{ ++ struct list_head *event_list = &kbdev->job_fault_event_list; ++ struct base_job_fault_event *event_in; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); ++ while (list_empty(event_list)) { ++ int err; ++ ++ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); ++ ++ err = wait_for_job_fault(kbdev); ++ if (err) ++ return err; ++ ++ spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); ++ } ++ ++ event_in = list_entry(event_list->next, ++ struct base_job_fault_event, head); ++ event->event_code = event_in->event_code; ++ event->katom = event_in->katom; ++ ++ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); ++ ++ return 0; ++ ++} ++ ++/* remove the event from the queue */ ++static struct base_job_fault_event *kbase_job_fault_event_dequeue( ++ struct kbase_device *kbdev, struct list_head *event_list) ++{ ++ struct base_job_fault_event *event; ++ ++ event = list_entry(event_list->next, ++ struct base_job_fault_event, head); ++ list_del(event_list->next); ++ ++ return event; ++ ++} ++ ++/* Remove all the following atoms after the failed atom in the same context ++ * Call the postponed bottom half of job done. ++ * Then, this context could be rescheduled. ++ */ ++static void kbase_job_fault_resume_event_cleanup(struct kbase_context *kctx) ++{ ++ struct list_head *event_list = &kctx->job_fault_resume_event_list; ++ ++ while (!list_empty(event_list)) { ++ struct base_job_fault_event *event; ++ ++ event = kbase_job_fault_event_dequeue(kctx->kbdev, ++ &kctx->job_fault_resume_event_list); ++ kbase_jd_done_worker(&event->katom->work); ++ } ++ ++} ++ ++static void kbase_job_fault_resume_worker(struct work_struct *data) ++{ ++ struct base_job_fault_event *event = container_of(data, ++ struct base_job_fault_event, job_fault_work); ++ struct kbase_context *kctx; ++ struct kbase_jd_atom *katom; ++ ++ katom = event->katom; ++ kctx = katom->kctx; ++ ++ dev_info(kctx->kbdev->dev, "Job dumping wait\n"); ++ ++ /* When it was waked up, it need to check if queue is empty or the ++ * failed atom belongs to different context. If yes, wake up. Both ++ * of them mean the failed job has been dumped. Please note, it ++ * should never happen that the job_fault_event_list has the two ++ * atoms belong to the same context. ++ */ ++ wait_event(kctx->kbdev->job_fault_resume_wq, ++ kbase_ctx_has_no_event_pending(kctx)); ++ ++ atomic_set(&kctx->job_fault_count, 0); ++ kbase_jd_done_worker(&katom->work); ++ ++ /* In case the following atoms were scheduled during failed job dump ++ * the job_done_worker was held. We need to rerun it after the dump ++ * was finished ++ */ ++ kbase_job_fault_resume_event_cleanup(kctx); ++ ++ dev_info(kctx->kbdev->dev, "Job dumping finish, resume scheduler\n"); ++} ++ ++static struct base_job_fault_event *kbase_job_fault_event_queue( ++ struct list_head *event_list, ++ struct kbase_jd_atom *atom, ++ u32 completion_code) ++{ ++ struct base_job_fault_event *event; ++ ++ event = &atom->fault_event; ++ ++ event->katom = atom; ++ event->event_code = completion_code; ++ ++ list_add_tail(&event->head, event_list); ++ ++ return event; ++ ++} ++ ++static void kbase_job_fault_event_post(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom, u32 completion_code) ++{ ++ struct base_job_fault_event *event; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); ++ event = kbase_job_fault_event_queue(&kbdev->job_fault_event_list, ++ katom, completion_code); ++ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); ++ ++ wake_up_interruptible(&kbdev->job_fault_wq); ++ ++ INIT_WORK(&event->job_fault_work, kbase_job_fault_resume_worker); ++ queue_work(kbdev->job_fault_resume_workq, &event->job_fault_work); ++ ++ dev_info(katom->kctx->kbdev->dev, "Job fault happen, start dump: %d_%d", ++ katom->kctx->tgid, katom->kctx->id); ++ ++} ++ ++/* ++ * This function will process the job fault ++ * Get the register copy ++ * Send the failed job dump event ++ * Create a Wait queue to wait until the job dump finish ++ */ ++ ++bool kbase_debug_job_fault_process(struct kbase_jd_atom *katom, ++ u32 completion_code) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ ++ /* Check if dumping is in the process ++ * only one atom of each context can be dumped at the same time ++ * If the atom belongs to different context, it can be dumped ++ */ ++ if (atomic_read(&kctx->job_fault_count) > 0) { ++ kbase_job_fault_event_queue( ++ &kctx->job_fault_resume_event_list, ++ katom, completion_code); ++ dev_info(kctx->kbdev->dev, "queue:%d\n", ++ kbase_jd_atom_id(kctx, katom)); ++ return true; ++ } ++ ++ if (kbase_ctx_flag(kctx, KCTX_DYING)) ++ return false; ++ ++ if (atomic_read(&kctx->kbdev->job_fault_debug) > 0) { ++ ++ if (completion_code != BASE_JD_EVENT_DONE) { ++ ++ if (kbase_job_fault_get_reg_snapshot(kctx) == false) { ++ dev_warn(kctx->kbdev->dev, "get reg dump failed\n"); ++ return false; ++ } ++ ++ kbase_job_fault_event_post(kctx->kbdev, katom, ++ completion_code); ++ atomic_inc(&kctx->job_fault_count); ++ dev_info(kctx->kbdev->dev, "post:%d\n", ++ kbase_jd_atom_id(kctx, katom)); ++ return true; ++ ++ } ++ } ++ return false; ++ ++} ++ ++static int debug_job_fault_show(struct seq_file *m, void *v) ++{ ++ struct kbase_device *kbdev = m->private; ++ struct base_job_fault_event *event = (struct base_job_fault_event *)v; ++ struct kbase_context *kctx = event->katom->kctx; ++ int i; ++ ++ dev_info(kbdev->dev, "debug job fault seq show:%d_%d, %d", ++ kctx->tgid, kctx->id, event->reg_offset); ++ ++ if (kctx->reg_dump == NULL) { ++ dev_warn(kbdev->dev, "reg dump is NULL"); ++ return -1; ++ } ++ ++ if (kctx->reg_dump[event->reg_offset] == ++ REGISTER_DUMP_TERMINATION_FLAG) { ++ /* Return the error here to stop the read. And the ++ * following next() will not be called. The stop can ++ * get the real event resource and release it ++ */ ++ return -1; ++ } ++ ++ if (event->reg_offset == 0) ++ seq_printf(m, "%d_%d\n", kctx->tgid, kctx->id); ++ ++ for (i = 0; i < 50; i++) { ++ if (kctx->reg_dump[event->reg_offset] == ++ REGISTER_DUMP_TERMINATION_FLAG) { ++ break; ++ } ++ seq_printf(m, "%08x: %08x\n", ++ kctx->reg_dump[event->reg_offset], ++ kctx->reg_dump[1+event->reg_offset]); ++ event->reg_offset += 2; ++ ++ } ++ ++ ++ return 0; ++} ++static void *debug_job_fault_next(struct seq_file *m, void *v, loff_t *pos) ++{ ++ struct kbase_device *kbdev = m->private; ++ struct base_job_fault_event *event = (struct base_job_fault_event *)v; ++ ++ dev_info(kbdev->dev, "debug job fault seq next:%d, %d", ++ event->reg_offset, (int)*pos); ++ ++ return event; ++} ++ ++static void *debug_job_fault_start(struct seq_file *m, loff_t *pos) ++{ ++ struct kbase_device *kbdev = m->private; ++ struct base_job_fault_event *event; ++ ++ dev_info(kbdev->dev, "fault job seq start:%d", (int)*pos); ++ ++ /* The condition is trick here. It needs make sure the ++ * fault hasn't happened and the dumping hasn't been started, ++ * or the dumping has finished ++ */ ++ if (*pos == 0) { ++ event = kmalloc(sizeof(*event), GFP_KERNEL); ++ if (!event) ++ return NULL; ++ event->reg_offset = 0; ++ if (kbase_job_fault_event_wait(kbdev, event)) { ++ kfree(event); ++ return NULL; ++ } ++ ++ /* The cache flush workaround is called in bottom half of ++ * job done but we delayed it. Now we should clean cache ++ * earlier. Then the GPU memory dump should be correct. ++ */ ++ kbase_backend_cache_clean(kbdev, event->katom); ++ } else ++ return NULL; ++ ++ return event; ++} ++ ++static void debug_job_fault_stop(struct seq_file *m, void *v) ++{ ++ struct kbase_device *kbdev = m->private; ++ ++ /* here we wake up the kbase_jd_done_worker after stop, it needs ++ * get the memory dump before the register dump in debug daemon, ++ * otherwise, the memory dump may be incorrect. ++ */ ++ ++ if (v != NULL) { ++ kfree(v); ++ dev_info(kbdev->dev, "debug job fault seq stop stage 1"); ++ ++ } else { ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); ++ if (!list_empty(&kbdev->job_fault_event_list)) { ++ kbase_job_fault_event_dequeue(kbdev, ++ &kbdev->job_fault_event_list); ++ wake_up(&kbdev->job_fault_resume_wq); ++ } ++ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); ++ dev_info(kbdev->dev, "debug job fault seq stop stage 2"); ++ } ++ ++} ++ ++static const struct seq_operations ops = { ++ .start = debug_job_fault_start, ++ .next = debug_job_fault_next, ++ .stop = debug_job_fault_stop, ++ .show = debug_job_fault_show, ++}; ++ ++static int debug_job_fault_open(struct inode *in, struct file *file) ++{ ++ struct kbase_device *kbdev = in->i_private; ++ ++ if (atomic_cmpxchg(&kbdev->job_fault_debug, 0, 1) == 1) { ++ dev_warn(kbdev->dev, "debug job fault is busy, only a single client is allowed"); ++ return -EBUSY; ++ } ++ ++ seq_open(file, &ops); ++ ++ ((struct seq_file *)file->private_data)->private = kbdev; ++ dev_info(kbdev->dev, "debug job fault seq open"); ++ ++ ++ return 0; ++ ++} ++ ++static int debug_job_fault_release(struct inode *in, struct file *file) ++{ ++ struct kbase_device *kbdev = in->i_private; ++ struct list_head *event_list = &kbdev->job_fault_event_list; ++ unsigned long flags; ++ ++ seq_release(in, file); ++ ++ spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); ++ ++ /* Disable job fault dumping. This will let kbase run jobs as normal, ++ * without blocking waiting for a job_fault client to read failed jobs. ++ * ++ * After this a new client may open the file, and may re-enable job ++ * fault dumping, but the job_fault_event_lock we hold here will block ++ * that from interfering until after we've completed the cleanup. ++ */ ++ atomic_dec(&kbdev->job_fault_debug); ++ ++ /* Clean the unprocessed job fault. After that, all the suspended ++ * contexts could be rescheduled. Remove all the failed atoms that ++ * belong to different contexts Resume all the contexts that were ++ * suspend due to failed job. ++ */ ++ while (!list_empty(event_list)) { ++ kbase_job_fault_event_dequeue(kbdev, event_list); ++ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); ++ wake_up(&kbdev->job_fault_resume_wq); ++ spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); ++ } ++ ++ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); ++ ++ dev_info(kbdev->dev, "debug job fault seq close"); ++ ++ return 0; ++} ++ ++static const struct file_operations kbasep_debug_job_fault_fops = { ++ .owner = THIS_MODULE, ++ .open = debug_job_fault_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = debug_job_fault_release, ++}; ++ ++/* ++ * Initialize debugfs entry for job fault dump ++ */ ++void kbase_debug_job_fault_debugfs_init(struct kbase_device *kbdev) ++{ ++ debugfs_create_file("job_fault", 0400, ++ kbdev->mali_debugfs_directory, kbdev, ++ &kbasep_debug_job_fault_fops); ++} ++ ++ ++int kbase_debug_job_fault_dev_init(struct kbase_device *kbdev) ++{ ++ ++ INIT_LIST_HEAD(&kbdev->job_fault_event_list); ++ ++ init_waitqueue_head(&(kbdev->job_fault_wq)); ++ init_waitqueue_head(&(kbdev->job_fault_resume_wq)); ++ spin_lock_init(&kbdev->job_fault_event_lock); ++ ++ kbdev->job_fault_resume_workq = alloc_workqueue( ++ "kbase_job_fault_resume_work_queue", WQ_MEM_RECLAIM, 1); ++ if (!kbdev->job_fault_resume_workq) ++ return -ENOMEM; ++ ++ atomic_set(&kbdev->job_fault_debug, 0); ++ ++ return 0; ++} ++ ++/* ++ * Release the relevant resource per device ++ */ ++void kbase_debug_job_fault_dev_term(struct kbase_device *kbdev) ++{ ++ destroy_workqueue(kbdev->job_fault_resume_workq); ++} ++ ++ ++/* ++ * Initialize the relevant data structure per context ++ */ ++void kbase_debug_job_fault_context_init(struct kbase_context *kctx) ++{ ++ ++ /* We need allocate double size register range ++ * Because this memory will keep the register address and value ++ */ ++ kctx->reg_dump = vmalloc(0x4000 * 2); ++ if (kctx->reg_dump == NULL) ++ return; ++ ++ if (kbase_debug_job_fault_reg_snapshot_init(kctx, 0x4000) == false) { ++ vfree(kctx->reg_dump); ++ kctx->reg_dump = NULL; ++ } ++ INIT_LIST_HEAD(&kctx->job_fault_resume_event_list); ++ atomic_set(&kctx->job_fault_count, 0); ++ ++} ++ ++/* ++ * release the relevant resource per context ++ */ ++void kbase_debug_job_fault_context_term(struct kbase_context *kctx) ++{ ++ vfree(kctx->reg_dump); ++} ++ ++void kbase_debug_job_fault_kctx_unblock(struct kbase_context *kctx) ++{ ++ WARN_ON(!kbase_ctx_flag(kctx, KCTX_DYING)); ++ ++ kbase_ctx_remove_pending_event(kctx); ++} ++ ++#else /* CONFIG_DEBUG_FS */ ++ ++int kbase_debug_job_fault_dev_init(struct kbase_device *kbdev) ++{ ++ return 0; ++} ++ ++void kbase_debug_job_fault_dev_term(struct kbase_device *kbdev) ++{ ++} ++ ++#endif /* CONFIG_DEBUG_FS */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_debug_job_fault.h b/drivers/gpu/arm/bifrost/mali_kbase_debug_job_fault.h +new file mode 100755 +index 000000000000..ef69627cdce8 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_debug_job_fault.h +@@ -0,0 +1,116 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2016, 2018 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_DEBUG_JOB_FAULT_H ++#define _KBASE_DEBUG_JOB_FAULT_H ++ ++#include ++#include ++ ++#define REGISTER_DUMP_TERMINATION_FLAG 0xFFFFFFFF ++ ++/** ++ * kbase_debug_job_fault_dev_init - Create the fault event wait queue ++ * per device and initialize the required lists. ++ * @kbdev: Device pointer ++ * ++ * Return: Zero on success or a negative error code. ++ */ ++int kbase_debug_job_fault_dev_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_debug_job_fault_debugfs_init - Initialize job fault debug sysfs ++ * @kbdev: Device pointer ++ */ ++void kbase_debug_job_fault_debugfs_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_debug_job_fault_dev_term - Clean up resources created in ++ * kbase_debug_job_fault_dev_init. ++ * @kbdev: Device pointer ++ */ ++void kbase_debug_job_fault_dev_term(struct kbase_device *kbdev); ++ ++/** ++ * kbase_debug_job_fault_context_init - Initialize the relevant ++ * data structure per context ++ * @kctx: KBase context pointer ++ */ ++void kbase_debug_job_fault_context_init(struct kbase_context *kctx); ++ ++/** ++ * kbase_debug_job_fault_context_term - Release the relevant ++ * resource per context ++ * @kctx: KBase context pointer ++ */ ++void kbase_debug_job_fault_context_term(struct kbase_context *kctx); ++ ++/** ++ * kbase_debug_job_fault_kctx_unblock - Unblock the atoms blocked on job fault ++ * dumping on context termination. ++ * ++ * This function is called during context termination to unblock the atom for ++ * which the job fault occurred and also the atoms following it. This is needed ++ * otherwise the wait for zero jobs could timeout (leading to an assertion ++ * failure, kernel panic in debug builds) in the pathological case where ++ * although the thread/daemon capturing the job fault events is running, ++ * but for some reasons has stopped consuming the events. ++ * ++ * @kctx: KBase context pointer ++ */ ++void kbase_debug_job_fault_kctx_unblock(struct kbase_context *kctx); ++ ++/** ++ * kbase_debug_job_fault_process - Process the failed job. ++ * It will send a event and wake up the job fault waiting queue ++ * Then create a work queue to wait for job dump finish ++ * This function should be called in the interrupt handler and before ++ * jd_done that make sure the jd_done_worker will be delayed until the ++ * job dump finish ++ * @katom: The failed atom pointer ++ * @completion_code: the job status ++ * @return true if dump is going on ++ */ ++bool kbase_debug_job_fault_process(struct kbase_jd_atom *katom, ++ u32 completion_code); ++ ++ ++/** ++ * kbase_debug_job_fault_reg_snapshot_init - Set the interested registers ++ * address during the job fault process, the relevant registers will ++ * be saved when a job fault happen ++ * @kctx: KBase context pointer ++ * @reg_range: Maximum register address space ++ * @return true if initializing successfully ++ */ ++bool kbase_debug_job_fault_reg_snapshot_init(struct kbase_context *kctx, ++ int reg_range); ++ ++/** ++ * kbase_job_fault_get_reg_snapshot - Read the interested registers for ++ * failed job dump ++ * @kctx: KBase context pointer ++ * @return true if getting registers successfully ++ */ ++bool kbase_job_fault_get_reg_snapshot(struct kbase_context *kctx); ++ ++#endif /*_KBASE_DEBUG_JOB_FAULT_H*/ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_debug_mem_view.c b/drivers/gpu/arm/bifrost/mali_kbase_debug_mem_view.c +new file mode 100755 +index 000000000000..478813705a41 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_debug_mem_view.c +@@ -0,0 +1,313 @@ ++/* ++ * ++ * (C) COPYRIGHT 2013-2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* ++ * Debugfs interface to dump the memory visible to the GPU ++ */ ++ ++#include "mali_kbase_debug_mem_view.h" ++#include "mali_kbase.h" ++ ++#include ++#include ++ ++#ifdef CONFIG_DEBUG_FS ++ ++#if (KERNEL_VERSION(4, 1, 0) > LINUX_VERSION_CODE) ++#define get_file_rcu(x) atomic_long_inc_not_zero(&(x)->f_count) ++#endif ++ ++struct debug_mem_mapping { ++ struct list_head node; ++ ++ struct kbase_mem_phy_alloc *alloc; ++ unsigned long flags; ++ ++ u64 start_pfn; ++ size_t nr_pages; ++}; ++ ++struct debug_mem_data { ++ struct list_head mapping_list; ++ struct kbase_context *kctx; ++}; ++ ++struct debug_mem_seq_off { ++ struct list_head *lh; ++ size_t offset; ++}; ++ ++static void *debug_mem_start(struct seq_file *m, loff_t *_pos) ++{ ++ struct debug_mem_data *mem_data = m->private; ++ struct debug_mem_seq_off *data; ++ struct debug_mem_mapping *map; ++ loff_t pos = *_pos; ++ ++ list_for_each_entry(map, &mem_data->mapping_list, node) { ++ if (pos >= map->nr_pages) { ++ pos -= map->nr_pages; ++ } else { ++ data = kmalloc(sizeof(*data), GFP_KERNEL); ++ if (!data) ++ return NULL; ++ data->lh = &map->node; ++ data->offset = pos; ++ return data; ++ } ++ } ++ ++ /* Beyond the end */ ++ return NULL; ++} ++ ++static void debug_mem_stop(struct seq_file *m, void *v) ++{ ++ kfree(v); ++} ++ ++static void *debug_mem_next(struct seq_file *m, void *v, loff_t *pos) ++{ ++ struct debug_mem_data *mem_data = m->private; ++ struct debug_mem_seq_off *data = v; ++ struct debug_mem_mapping *map; ++ ++ map = list_entry(data->lh, struct debug_mem_mapping, node); ++ ++ if (data->offset < map->nr_pages - 1) { ++ data->offset++; ++ ++*pos; ++ return data; ++ } ++ ++ if (list_is_last(data->lh, &mem_data->mapping_list)) { ++ kfree(data); ++ return NULL; ++ } ++ ++ data->lh = data->lh->next; ++ data->offset = 0; ++ ++*pos; ++ ++ return data; ++} ++ ++static int debug_mem_show(struct seq_file *m, void *v) ++{ ++ struct debug_mem_data *mem_data = m->private; ++ struct debug_mem_seq_off *data = v; ++ struct debug_mem_mapping *map; ++ int i, j; ++ struct page *page; ++ uint32_t *mapping; ++ pgprot_t prot = PAGE_KERNEL; ++ ++ map = list_entry(data->lh, struct debug_mem_mapping, node); ++ ++ kbase_gpu_vm_lock(mem_data->kctx); ++ ++ if (data->offset >= map->alloc->nents) { ++ seq_printf(m, "%016llx: Unbacked page\n\n", (map->start_pfn + ++ data->offset) << PAGE_SHIFT); ++ goto out; ++ } ++ ++ if (!(map->flags & KBASE_REG_CPU_CACHED)) ++ prot = pgprot_writecombine(prot); ++ ++ page = as_page(map->alloc->pages[data->offset]); ++ mapping = vmap(&page, 1, VM_MAP, prot); ++ if (!mapping) ++ goto out; ++ ++ for (i = 0; i < PAGE_SIZE; i += 4*sizeof(*mapping)) { ++ seq_printf(m, "%016llx:", i + ((map->start_pfn + ++ data->offset) << PAGE_SHIFT)); ++ ++ for (j = 0; j < 4*sizeof(*mapping); j += sizeof(*mapping)) ++ seq_printf(m, " %08x", mapping[(i+j)/sizeof(*mapping)]); ++ seq_putc(m, '\n'); ++ } ++ ++ vunmap(mapping); ++ ++ seq_putc(m, '\n'); ++ ++out: ++ kbase_gpu_vm_unlock(mem_data->kctx); ++ return 0; ++} ++ ++static const struct seq_operations ops = { ++ .start = debug_mem_start, ++ .next = debug_mem_next, ++ .stop = debug_mem_stop, ++ .show = debug_mem_show, ++}; ++ ++static int debug_mem_zone_open(struct rb_root *rbtree, ++ struct debug_mem_data *mem_data) ++{ ++ int ret = 0; ++ struct rb_node *p; ++ struct kbase_va_region *reg; ++ struct debug_mem_mapping *mapping; ++ ++ for (p = rb_first(rbtree); p; p = rb_next(p)) { ++ reg = rb_entry(p, struct kbase_va_region, rblink); ++ ++ if (reg->gpu_alloc == NULL) ++ /* Empty region - ignore */ ++ continue; ++ ++ mapping = kmalloc(sizeof(*mapping), GFP_KERNEL); ++ if (!mapping) { ++ ret = -ENOMEM; ++ goto out; ++ } ++ ++ mapping->alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); ++ mapping->start_pfn = reg->start_pfn; ++ mapping->nr_pages = reg->nr_pages; ++ mapping->flags = reg->flags; ++ list_add_tail(&mapping->node, &mem_data->mapping_list); ++ } ++ ++out: ++ return ret; ++} ++ ++static int debug_mem_open(struct inode *i, struct file *file) ++{ ++ struct kbase_context *const kctx = i->i_private; ++ struct debug_mem_data *mem_data; ++ int ret; ++ ++ if (get_file_rcu(kctx->filp) == 0) ++ return -ENOENT; ++ ++ ret = seq_open(file, &ops); ++ if (ret) ++ goto open_fail; ++ ++ mem_data = kmalloc(sizeof(*mem_data), GFP_KERNEL); ++ if (!mem_data) { ++ ret = -ENOMEM; ++ goto out; ++ } ++ ++ mem_data->kctx = kctx; ++ ++ INIT_LIST_HEAD(&mem_data->mapping_list); ++ ++ kbase_gpu_vm_lock(kctx); ++ ++ ret = debug_mem_zone_open(&kctx->reg_rbtree_same, mem_data); ++ if (0 != ret) { ++ kbase_gpu_vm_unlock(kctx); ++ goto out; ++ } ++ ++ ret = debug_mem_zone_open(&kctx->reg_rbtree_custom, mem_data); ++ if (0 != ret) { ++ kbase_gpu_vm_unlock(kctx); ++ goto out; ++ } ++ ++ ret = debug_mem_zone_open(&kctx->reg_rbtree_exec, mem_data); ++ if (0 != ret) { ++ kbase_gpu_vm_unlock(kctx); ++ goto out; ++ } ++ ++ kbase_gpu_vm_unlock(kctx); ++ ++ ((struct seq_file *)file->private_data)->private = mem_data; ++ ++ return 0; ++ ++out: ++ if (mem_data) { ++ while (!list_empty(&mem_data->mapping_list)) { ++ struct debug_mem_mapping *mapping; ++ ++ mapping = list_first_entry(&mem_data->mapping_list, ++ struct debug_mem_mapping, node); ++ kbase_mem_phy_alloc_put(mapping->alloc); ++ list_del(&mapping->node); ++ kfree(mapping); ++ } ++ kfree(mem_data); ++ } ++ seq_release(i, file); ++open_fail: ++ fput(kctx->filp); ++ ++ return ret; ++} ++ ++static int debug_mem_release(struct inode *inode, struct file *file) ++{ ++ struct kbase_context *const kctx = inode->i_private; ++ struct seq_file *sfile = file->private_data; ++ struct debug_mem_data *mem_data = sfile->private; ++ struct debug_mem_mapping *mapping; ++ ++ seq_release(inode, file); ++ ++ while (!list_empty(&mem_data->mapping_list)) { ++ mapping = list_first_entry(&mem_data->mapping_list, ++ struct debug_mem_mapping, node); ++ kbase_mem_phy_alloc_put(mapping->alloc); ++ list_del(&mapping->node); ++ kfree(mapping); ++ } ++ ++ kfree(mem_data); ++ ++ fput(kctx->filp); ++ ++ return 0; ++} ++ ++static const struct file_operations kbase_debug_mem_view_fops = { ++ .owner = THIS_MODULE, ++ .open = debug_mem_open, ++ .release = debug_mem_release, ++ .read = seq_read, ++ .llseek = seq_lseek ++}; ++ ++void kbase_debug_mem_view_init(struct kbase_context *const kctx) ++{ ++ /* Caller already ensures this, but we keep the pattern for ++ * maintenance safety. ++ */ ++ if (WARN_ON(!kctx) || ++ WARN_ON(IS_ERR_OR_NULL(kctx->kctx_dentry))) ++ return; ++ ++ debugfs_create_file("mem_view", 0400, kctx->kctx_dentry, kctx, ++ &kbase_debug_mem_view_fops); ++} ++ ++#endif +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_debug_mem_view.h b/drivers/gpu/arm/bifrost/mali_kbase_debug_mem_view.h +new file mode 100755 +index 000000000000..b948b7cd9dd4 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_debug_mem_view.h +@@ -0,0 +1,40 @@ ++/* ++ * ++ * (C) COPYRIGHT 2013-2015, 2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_DEBUG_MEM_VIEW_H ++#define _KBASE_DEBUG_MEM_VIEW_H ++ ++#include ++ ++/** ++ * kbase_debug_mem_view_init - Initialize the mem_view sysfs file ++ * @kctx: Pointer to kernel base context ++ * ++ * This function creates a "mem_view" file which can be used to get a view of ++ * the context's memory as the GPU sees it (i.e. using the GPU's page tables). ++ * ++ * The file is cleaned up by a call to debugfs_remove_recursive() deleting the ++ * parent directory. ++ */ ++void kbase_debug_mem_view_init(struct kbase_context *kctx); ++ ++#endif +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_debugfs_helper.c b/drivers/gpu/arm/bifrost/mali_kbase_debugfs_helper.c +new file mode 100755 +index 000000000000..37e507b164c5 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_debugfs_helper.c +@@ -0,0 +1,183 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++#include ++#include ++#include ++ ++#include "mali_kbase_debugfs_helper.h" ++ ++/* Arbitrary maximum size to prevent user space allocating too much kernel ++ * memory ++ */ ++#define DEBUGFS_MEM_POOLS_MAX_WRITE_SIZE (256u) ++ ++/** ++ * set_attr_from_string - Parse a string to set elements of an array ++ * ++ * This is the core of the implementation of ++ * kbase_debugfs_helper_set_attr_from_string. The only difference between the ++ * two functions is that this one requires the input string to be writable. ++ * ++ * @buf: Input string to parse. Must be nul-terminated! ++ * @array: Address of an object that can be accessed like an array. ++ * @nelems: Number of elements in the array. ++ * @set_attr_fn: Function to be called back for each array element. ++ * ++ * Return: 0 if success, negative error code otherwise. ++ */ ++static int set_attr_from_string( ++ char *const buf, ++ void *const array, size_t const nelems, ++ kbase_debugfs_helper_set_attr_fn const set_attr_fn) ++{ ++ size_t index, err = 0; ++ char *ptr = buf; ++ ++ for (index = 0; index < nelems && *ptr; ++index) { ++ unsigned long new_size; ++ size_t len; ++ char sep; ++ ++ /* Drop leading spaces */ ++ while (*ptr == ' ') ++ ptr++; ++ ++ len = strcspn(ptr, "\n "); ++ if (len == 0) { ++ /* No more values (allow this) */ ++ break; ++ } ++ ++ /* Substitute a nul terminator for a space character ++ * to make the substring valid for kstrtoul. ++ */ ++ sep = ptr[len]; ++ if (sep == ' ') ++ ptr[len++] = '\0'; ++ ++ err = kstrtoul(ptr, 0, &new_size); ++ if (err) ++ break; ++ ++ /* Skip the substring (including any premature nul terminator) ++ */ ++ ptr += len; ++ ++ set_attr_fn(array, index, new_size); ++ } ++ ++ return err; ++} ++ ++int kbase_debugfs_helper_set_attr_from_string( ++ const char *const buf, void *const array, size_t const nelems, ++ kbase_debugfs_helper_set_attr_fn const set_attr_fn) ++{ ++ char *const wbuf = kstrdup(buf, GFP_KERNEL); ++ int err = 0; ++ ++ if (!wbuf) ++ return -ENOMEM; ++ ++ err = set_attr_from_string(wbuf, array, nelems, ++ set_attr_fn); ++ ++ kfree(wbuf); ++ return err; ++} ++ ++ssize_t kbase_debugfs_helper_get_attr_to_string( ++ char *const buf, size_t const size, ++ void *const array, size_t const nelems, ++ kbase_debugfs_helper_get_attr_fn const get_attr_fn) ++{ ++ ssize_t total = 0; ++ size_t index; ++ ++ for (index = 0; index < nelems; ++index) { ++ const char *postfix = " "; ++ ++ if (index == (nelems-1)) ++ postfix = "\n"; ++ ++ total += scnprintf(buf + total, size - total, "%zu%s", ++ get_attr_fn(array, index), postfix); ++ } ++ ++ return total; ++} ++ ++int kbase_debugfs_helper_seq_write(struct file *const file, ++ const char __user *const ubuf, size_t const count, ++ size_t const nelems, ++ kbase_debugfs_helper_set_attr_fn const set_attr_fn) ++{ ++ const struct seq_file *const sfile = file->private_data; ++ void *const array = sfile->private; ++ int err = 0; ++ char *buf; ++ ++ if (WARN_ON(!array)) ++ return -EINVAL; ++ ++ if (WARN_ON(count > DEBUGFS_MEM_POOLS_MAX_WRITE_SIZE)) ++ return -EINVAL; ++ ++ buf = kmalloc(count + 1, GFP_KERNEL); ++ if (buf == NULL) ++ return -ENOMEM; ++ ++ if (copy_from_user(buf, ubuf, count)) { ++ kfree(buf); ++ return -EFAULT; ++ } ++ ++ buf[count] = '\0'; ++ err = set_attr_from_string(buf, ++ array, nelems, set_attr_fn); ++ kfree(buf); ++ ++ return err; ++} ++ ++int kbase_debugfs_helper_seq_read(struct seq_file *const sfile, ++ size_t const nelems, ++ kbase_debugfs_helper_get_attr_fn const get_attr_fn) ++{ ++ void *const array = sfile->private; ++ size_t index; ++ ++ if (WARN_ON(!array)) ++ return -EINVAL; ++ ++ for (index = 0; index < nelems; ++index) { ++ const char *postfix = " "; ++ ++ if (index == (nelems-1)) ++ postfix = "\n"; ++ ++ seq_printf(sfile, "%zu%s", get_attr_fn(array, index), postfix); ++ } ++ return 0; ++} +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_debugfs_helper.h b/drivers/gpu/arm/bifrost/mali_kbase_debugfs_helper.h +new file mode 100755 +index 000000000000..c3c9efa14e65 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_debugfs_helper.h +@@ -0,0 +1,141 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_DEBUGFS_HELPER_H_ ++#define _KBASE_DEBUGFS_HELPER_H_ ++ ++/** ++ * typedef kbase_debugfs_helper_set_attr_fn - Type of function to set an ++ * attribute value from an array ++ * ++ * @array: Address of an object that can be accessed like an array. ++ * @index: An element index. The valid range depends on the use-case. ++ * @value: Attribute value to be set. ++ */ ++typedef void (*kbase_debugfs_helper_set_attr_fn)( ++ void *array, size_t index, size_t value); ++ ++/** ++ * kbase_debugfs_helper_set_attr_from_string - Parse a string to reconfigure an ++ * array ++ * ++ * The given function is called once for each attribute value found in the ++ * input string. It is not an error if the string specifies fewer attribute ++ * values than the specified number of array elements. ++ * ++ * The number base of each attribute value is detected automatically ++ * according to the standard rules (e.g. prefix "0x" for hexadecimal). ++ * Attribute values are separated by one or more space characters. ++ * Additional leading and trailing spaces are ignored. ++ * ++ * @buf: Input string to parse. Must be nul-terminated! ++ * @array: Address of an object that can be accessed like an array. ++ * @nelems: Number of elements in the array. ++ * @set_attr_fn: Function to be called back for each array element. ++ * ++ * Return: 0 if success, negative error code otherwise. ++ */ ++int kbase_debugfs_helper_set_attr_from_string( ++ const char *buf, void *array, size_t nelems, ++ kbase_debugfs_helper_set_attr_fn set_attr_fn); ++ ++/** ++ * typedef kbase_debugfs_helper_get_attr_fn - Type of function to get an ++ * attribute value from an array ++ * ++ * @array: Address of an object that can be accessed like an array. ++ * @index: An element index. The valid range depends on the use-case. ++ * ++ * Return: Value of attribute. ++ */ ++typedef size_t (*kbase_debugfs_helper_get_attr_fn)( ++ void *array, size_t index); ++ ++/** ++ * kbase_debugfs_helper_get_attr_to_string - Construct a formatted string ++ * from elements in an array ++ * ++ * The given function is called once for each array element to get the ++ * value of the attribute to be inspected. The attribute values are ++ * written to the buffer as a formatted string of decimal numbers ++ * separated by spaces and terminated by a linefeed. ++ * ++ * @buf: Buffer in which to store the formatted output string. ++ * @size: The size of the buffer, in bytes. ++ * @array: Address of an object that can be accessed like an array. ++ * @nelems: Number of elements in the array. ++ * @get_attr_fn: Function to be called back for each array element. ++ * ++ * Return: Number of characters written excluding the nul terminator. ++ */ ++ssize_t kbase_debugfs_helper_get_attr_to_string( ++ char *buf, size_t size, void *array, size_t nelems, ++ kbase_debugfs_helper_get_attr_fn get_attr_fn); ++ ++/** ++ * kbase_debugfs_helper_seq_read - Implements reads from a virtual file for an ++ * array ++ * ++ * The virtual file must have been opened by calling single_open and passing ++ * the address of an object that can be accessed like an array. ++ * ++ * The given function is called once for each array element to get the ++ * value of the attribute to be inspected. The attribute values are ++ * written to the buffer as a formatted string of decimal numbers ++ * separated by spaces and terminated by a linefeed. ++ * ++ * @sfile: A virtual file previously opened by calling single_open. ++ * @nelems: Number of elements in the array. ++ * @get_attr_fn: Function to be called back for each array element. ++ * ++ * Return: 0 if success, negative error code otherwise. ++ */ ++int kbase_debugfs_helper_seq_read( ++ struct seq_file *const sfile, size_t const nelems, ++ kbase_debugfs_helper_get_attr_fn const get_attr_fn); ++ ++/** ++ * kbase_debugfs_helper_seq_write - Implements writes to a virtual file for an ++ * array ++ * ++ * The virtual file must have been opened by calling single_open and passing ++ * the address of an object that can be accessed like an array. ++ * ++ * The given function is called once for each attribute value found in the ++ * data written to the virtual file. For further details, refer to the ++ * description of set_attr_from_string. ++ * ++ * @file: A virtual file previously opened by calling single_open. ++ * @ubuf: Source address in user space. ++ * @count: Number of bytes written to the virtual file. ++ * @nelems: Number of elements in the array. ++ * @set_attr_fn: Function to be called back for each array element. ++ * ++ * Return: 0 if success, negative error code otherwise. ++ */ ++int kbase_debugfs_helper_seq_write(struct file *const file, ++ const char __user *const ubuf, size_t const count, ++ size_t const nelems, ++ kbase_debugfs_helper_set_attr_fn const set_attr_fn); ++ ++#endif /*_KBASE_DEBUGFS_HELPER_H_ */ ++ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_defs.h b/drivers/gpu/arm/bifrost/mali_kbase_defs.h +new file mode 100755 +index 000000000000..980cf09500ef +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_defs.h +@@ -0,0 +1,1807 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++/** ++ * @file mali_kbase_defs.h ++ * ++ * Defintions (types, defines, etcs) common to Kbase. They are placed here to ++ * allow the hierarchy of header files to work. ++ */ ++ ++#ifndef _KBASE_DEFS_H_ ++#define _KBASE_DEFS_H_ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#include ++#include ++#include ++#include ++ ++#ifdef CONFIG_MALI_BUSLOG ++#include ++#endif ++ ++#if defined(CONFIG_SYNC) ++#include ++#else ++#include "mali_kbase_fence_defs.h" ++#endif ++ ++#ifdef CONFIG_DEBUG_FS ++#include ++#endif /* CONFIG_DEBUG_FS */ ++ ++#ifdef CONFIG_MALI_BIFROST_DEVFREQ ++#include ++#endif /* CONFIG_MALI_BIFROST_DEVFREQ */ ++ ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++#include ++#endif /* CONFIG_MALI_ARBITER_SUPPORT */ ++ ++#include ++#include ++#include ++ ++#if defined(CONFIG_PM_RUNTIME) || \ ++ (defined(CONFIG_PM) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0)) ++#define KBASE_PM_RUNTIME 1 ++#endif ++ ++#include "debug/mali_kbase_debug_ktrace_defs.h" ++ ++/** Number of milliseconds before we time out on a GPU soft/hard reset */ ++#define RESET_TIMEOUT 500 ++ ++/** ++ * The maximum number of Job Slots to support in the Hardware. ++ * ++ * You can optimize this down if your target devices will only ever support a ++ * small number of job slots. ++ */ ++#define BASE_JM_MAX_NR_SLOTS 3 ++ ++/** ++ * The maximum number of Address Spaces to support in the Hardware. ++ * ++ * You can optimize this down if your target devices will only ever support a ++ * small number of Address Spaces ++ */ ++#define BASE_MAX_NR_AS 16 ++ ++/* mmu */ ++#define MIDGARD_MMU_LEVEL(x) (x) ++ ++#define MIDGARD_MMU_TOPLEVEL MIDGARD_MMU_LEVEL(0) ++ ++#define MIDGARD_MMU_BOTTOMLEVEL MIDGARD_MMU_LEVEL(3) ++ ++#define GROWABLE_FLAGS_REQUIRED (KBASE_REG_PF_GROW | KBASE_REG_GPU_WR) ++ ++/** setting in kbase_context::as_nr that indicates it's invalid */ ++#define KBASEP_AS_NR_INVALID (-1) ++ ++/** ++ * Maximum size in bytes of a MMU lock region, as a logarithm ++ */ ++#define KBASE_LOCK_REGION_MAX_SIZE_LOG2 (64) ++ ++/** ++ * Minimum size in bytes of a MMU lock region, as a logarithm ++ */ ++#define KBASE_LOCK_REGION_MIN_SIZE_LOG2 (15) ++ ++#include "mali_kbase_hwaccess_defs.h" ++ ++/* Maximum number of pages of memory that require a permanent mapping, per ++ * kbase_context ++ */ ++#define KBASE_PERMANENTLY_MAPPED_MEM_LIMIT_PAGES ((32 * 1024ul * 1024ul) >> \ ++ PAGE_SHIFT) ++/* Minimum threshold period for hwcnt dumps between different hwcnt virtualizer ++ * clients, to reduce undesired system load. ++ * If a virtualizer client requests a dump within this threshold period after ++ * some other client has performed a dump, a new dump won't be performed and ++ * the accumulated counter values for that client will be returned instead. ++ */ ++#define KBASE_HWCNT_GPU_VIRTUALIZER_DUMP_THRESHOLD_NS (200 * NSEC_PER_USEC) ++ ++/* Maximum number of clock/regulator pairs that may be referenced by ++ * the device node. ++ * This is dependent on support for of_property_read_u64_array() in the ++ * kernel. ++ */ ++#if (KERNEL_VERSION(4, 0, 0) <= LINUX_VERSION_CODE) || \ ++ defined(LSK_OPPV2_BACKPORT) ++#define BASE_MAX_NR_CLOCKS_REGULATORS (2) ++#else ++#define BASE_MAX_NR_CLOCKS_REGULATORS (1) ++#endif ++ ++/* Forward declarations */ ++struct kbase_context; ++struct kbase_device; ++struct kbase_as; ++struct kbase_mmu_setup; ++struct kbase_ipa_model_vinstr_data; ++struct kbase_kinstr_jm; ++ ++/** ++ * struct kbase_io_access - holds information about 1 register access ++ * ++ * @addr: first bit indicates r/w (r=0, w=1) ++ * @value: value written or read ++ */ ++struct kbase_io_access { ++ uintptr_t addr; ++ u32 value; ++}; ++ ++/** ++ * struct kbase_io_history - keeps track of all recent register accesses ++ * ++ * @enabled: true if register accesses are recorded, false otherwise ++ * @lock: spinlock protecting kbase_io_access array ++ * @count: number of registers read/written ++ * @size: number of elements in kbase_io_access array ++ * @buf: array of kbase_io_access ++ */ ++struct kbase_io_history { ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0)) ++ bool enabled; ++#else ++ u32 enabled; ++#endif ++ ++ spinlock_t lock; ++ size_t count; ++ u16 size; ++ struct kbase_io_access *buf; ++}; ++ ++/** ++ * struct kbase_debug_copy_buffer - information about the buffer to be copied. ++ * ++ * @size: size of the buffer in bytes ++ * @pages: pointer to an array of pointers to the pages which contain ++ * the buffer ++ * @is_vmalloc: true if @pages was allocated with vzalloc. false if @pages was ++ * allocated with kcalloc ++ * @nr_pages: number of pages ++ * @offset: offset into the pages ++ * @gpu_alloc: pointer to physical memory allocated by the GPU ++ * @extres_pages: array of pointers to the pages containing external resources ++ * for this buffer ++ * @nr_extres_pages: number of pages in @extres_pages ++ */ ++struct kbase_debug_copy_buffer { ++ size_t size; ++ struct page **pages; ++ bool is_vmalloc; ++ int nr_pages; ++ size_t offset; ++ struct kbase_mem_phy_alloc *gpu_alloc; ++ ++ struct page **extres_pages; ++ int nr_extres_pages; ++}; ++ ++struct kbase_device_info { ++ u32 features; ++}; ++ ++struct kbase_mmu_setup { ++ u64 transtab; ++ u64 memattr; ++ u64 transcfg; ++}; ++ ++/** ++ * struct kbase_fault - object containing data relating to a page or bus fault. ++ * @addr: Records the faulting address. ++ * @extra_addr: Records the secondary fault address. ++ * @status: Records the fault status as reported by Hw. ++ * @protected_mode: Flag indicating whether the fault occurred in protected mode ++ * or not. ++ */ ++struct kbase_fault { ++ u64 addr; ++ u64 extra_addr; ++ u32 status; ++ bool protected_mode; ++}; ++ ++/** ++ * struct kbase_mmu_table - object representing a set of GPU page tables ++ * @mmu_teardown_pages: Buffer of 4 Pages in size, used to cache the entries ++ * of top & intermediate level page tables to avoid ++ * repeated calls to kmap_atomic during the MMU teardown. ++ * @mmu_lock: Lock to serialize the accesses made to multi level GPU ++ * page tables ++ * @pgd: Physical address of the page allocated for the top ++ * level page table of the context, this is used for ++ * MMU HW programming as the address translation will ++ * start from the top level page table. ++ * @group_id: A memory group ID to be passed to a platform-specific ++ * memory group manager. ++ * Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). ++ * @kctx: If this set of MMU tables belongs to a context then ++ * this is a back-reference to the context, otherwise ++ * it is NULL ++ */ ++struct kbase_mmu_table { ++ u64 *mmu_teardown_pages; ++ struct mutex mmu_lock; ++ phys_addr_t pgd; ++ u8 group_id; ++ struct kbase_context *kctx; ++}; ++ ++#if MALI_USE_CSF ++#include "csf/mali_kbase_csf_defs.h" ++#else ++#include "jm/mali_kbase_jm_defs.h" ++#endif ++ ++static inline int kbase_as_has_bus_fault(struct kbase_as *as, ++ struct kbase_fault *fault) ++{ ++ return (fault == &as->bf_data); ++} ++ ++static inline int kbase_as_has_page_fault(struct kbase_as *as, ++ struct kbase_fault *fault) ++{ ++ return (fault == &as->pf_data); ++} ++ ++/** ++ * struct kbasep_mem_device - Data stored per device for memory allocation ++ * ++ * @used_pages: Tracks usage of OS shared memory. Updated when OS memory is ++ * allocated/freed. ++ * @ir_threshold: Fraction of the maximum size of an allocation that grows ++ * on GPU page fault that can be used before the driver ++ * switches to incremental rendering, in 1/256ths. ++ * 0 means disabled. ++ */ ++struct kbasep_mem_device { ++ atomic_t used_pages; ++ atomic_t ir_threshold; ++}; ++ ++struct kbase_clk_rate_listener; ++ ++/** ++ * kbase_clk_rate_listener_on_change_t() - Frequency change callback ++ * ++ * @listener: Clock frequency change listener. ++ * @clk_index: Index of the clock for which the change has occurred. ++ * @clk_rate_hz: Clock frequency(Hz). ++ * ++ * A callback to call when clock rate changes. The function must not ++ * sleep. No clock rate manager functions must be called from here, as ++ * its lock is taken. ++ */ ++typedef void (*kbase_clk_rate_listener_on_change_t)( ++ struct kbase_clk_rate_listener *listener, ++ u32 clk_index, ++ u32 clk_rate_hz); ++ ++/** ++ * struct kbase_clk_rate_listener - Clock frequency listener ++ * ++ * @node: List node. ++ * @notify: Callback to be called when GPU frequency changes. ++ */ ++struct kbase_clk_rate_listener { ++ struct list_head node; ++ kbase_clk_rate_listener_on_change_t notify; ++}; ++ ++/** ++ * struct kbase_clk_rate_trace_manager - Data stored per device for GPU clock ++ * rate trace manager. ++ * ++ * @gpu_idle: Tracks the idle state of GPU. ++ * @clks: Array of pointer to structures storing data for every ++ * enumerated GPU clock. ++ * @clk_rate_trace_ops: Pointer to the platform specific GPU clock rate trace ++ * operations. ++ * @gpu_clk_rate_trace_write: Pointer to the function that would emit the ++ * tracepoint for the clock rate change. ++ * @listeners: List of listener attached. ++ * @lock: Lock to serialize the actions of GPU clock rate trace ++ * manager. ++ */ ++struct kbase_clk_rate_trace_manager { ++ bool gpu_idle; ++ struct kbase_clk_data *clks[BASE_MAX_NR_CLOCKS_REGULATORS]; ++ struct kbase_clk_rate_trace_op_conf *clk_rate_trace_ops; ++ struct list_head listeners; ++ spinlock_t lock; ++}; ++ ++/** ++ * Data stored per device for power management. ++ * ++ * This structure contains data for the power management framework. There is one ++ * instance of this structure per device in the system. ++ */ ++struct kbase_pm_device_data { ++ /** ++ * The lock protecting Power Management structures accessed outside of ++ * IRQ. ++ * ++ * This lock must also be held whenever the GPU is being powered on or ++ * off. ++ */ ++ struct mutex lock; ++ ++ /** ++ * The reference count of active contexts on this device. Note that ++ * some code paths keep shaders/the tiler powered whilst this is 0. Use ++ * kbase_pm_is_active() instead to check for such cases. ++ */ ++ int active_count; ++ /** Flag indicating suspending/suspended */ ++ bool suspending; ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++ /* Flag indicating gpu lost */ ++ atomic_t gpu_lost; ++#endif /* CONFIG_MALI_ARBITER_SUPPORT */ ++ /* Wait queue set when active_count == 0 */ ++ wait_queue_head_t zero_active_count_wait; ++ ++ /** ++ * Bit masks identifying the available shader cores that are specified ++ * via sysfs. One mask per job slot. ++ */ ++ u64 debug_core_mask[BASE_JM_MAX_NR_SLOTS]; ++ u64 debug_core_mask_all; ++ ++ /** ++ * Callback for initializing the runtime power management. ++ * ++ * @param kbdev The kbase device ++ * ++ * @return 0 on success, else error code ++ */ ++ int (*callback_power_runtime_init)(struct kbase_device *kbdev); ++ ++ /** ++ * Callback for terminating the runtime power management. ++ * ++ * @param kbdev The kbase device ++ */ ++ void (*callback_power_runtime_term)(struct kbase_device *kbdev); ++ ++ /* Time in milliseconds between each dvfs sample */ ++ u32 dvfs_period; ++ ++ struct kbase_pm_backend_data backend; ++ ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++ /** ++ * The state of the arbiter VM machine ++ */ ++ struct kbase_arbiter_vm_state *arb_vm_state; ++#endif /* CONFIG_MALI_ARBITER_SUPPORT */ ++ ++ /** ++ * The state of the GPU clock rate trace manager ++ */ ++ struct kbase_clk_rate_trace_manager clk_rtm; ++}; ++ ++/** ++ * struct kbase_mem_pool - Page based memory pool for kctx/kbdev ++ * @kbdev: Kbase device where memory is used ++ * @cur_size: Number of free pages currently in the pool (may exceed ++ * @max_size in some corner cases) ++ * @max_size: Maximum number of free pages in the pool ++ * @order: order = 0 refers to a pool of 4 KB pages ++ * order = 9 refers to a pool of 2 MB pages (2^9 * 4KB = 2 MB) ++ * @group_id: A memory group ID to be passed to a platform-specific ++ * memory group manager, if present. Immutable. ++ * Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). ++ * @pool_lock: Lock protecting the pool - must be held when modifying ++ * @cur_size and @page_list ++ * @page_list: List of free pages in the pool ++ * @reclaim: Shrinker for kernel reclaim of free pages ++ * @next_pool: Pointer to next pool where pages can be allocated when this ++ * pool is empty. Pages will spill over to the next pool when ++ * this pool is full. Can be NULL if there is no next pool. ++ * @dying: true if the pool is being terminated, and any ongoing ++ * operations should be abandoned ++ * @dont_reclaim: true if the shrinker is forbidden from reclaiming memory from ++ * this pool, eg during a grow operation ++ */ ++struct kbase_mem_pool { ++ struct kbase_device *kbdev; ++ size_t cur_size; ++ size_t max_size; ++ u8 order; ++ u8 group_id; ++ spinlock_t pool_lock; ++ struct list_head page_list; ++ struct shrinker reclaim; ++ ++ struct kbase_mem_pool *next_pool; ++ ++ bool dying; ++ bool dont_reclaim; ++}; ++ ++/** ++ * struct kbase_mem_pool_group - a complete set of physical memory pools. ++ * ++ * Memory pools are used to allow efficient reallocation of previously-freed ++ * physical pages. A pair of memory pools is initialized for each physical ++ * memory group: one for 4 KiB pages and one for 2 MiB pages. These arrays ++ * should be indexed by physical memory group ID, the meaning of which is ++ * defined by the systems integrator. ++ * ++ * @small: Array of objects containing the state for pools of 4 KiB size ++ * physical pages. ++ * @large: Array of objects containing the state for pools of 2 MiB size ++ * physical pages. ++ */ ++struct kbase_mem_pool_group { ++ struct kbase_mem_pool small[MEMORY_GROUP_MANAGER_NR_GROUPS]; ++ struct kbase_mem_pool large[MEMORY_GROUP_MANAGER_NR_GROUPS]; ++}; ++ ++/** ++ * struct kbase_mem_pool_config - Initial configuration for a physical memory ++ * pool ++ * ++ * @max_size: Maximum number of free pages that the pool can hold. ++ */ ++struct kbase_mem_pool_config { ++ size_t max_size; ++}; ++ ++/** ++ * struct kbase_mem_pool_group_config - Initial configuration for a complete ++ * set of physical memory pools ++ * ++ * This array should be indexed by physical memory group ID, the meaning ++ * of which is defined by the systems integrator. ++ * ++ * @small: Array of initial configuration for pools of 4 KiB pages. ++ * @large: Array of initial configuration for pools of 2 MiB pages. ++ */ ++struct kbase_mem_pool_group_config { ++ struct kbase_mem_pool_config small[MEMORY_GROUP_MANAGER_NR_GROUPS]; ++ struct kbase_mem_pool_config large[MEMORY_GROUP_MANAGER_NR_GROUPS]; ++}; ++ ++/** ++ * struct kbase_devfreq_opp - Lookup table for converting between nominal OPP ++ * frequency, real frequencies and core mask ++ * @real_freqs: Real GPU frequencies. ++ * @opp_volts: OPP voltages. ++ * @opp_freq: Nominal OPP frequency ++ * @core_mask: Shader core mask ++ */ ++struct kbase_devfreq_opp { ++ u64 opp_freq; ++ u64 core_mask; ++ u64 real_freqs[BASE_MAX_NR_CLOCKS_REGULATORS]; ++ u32 opp_volts[BASE_MAX_NR_CLOCKS_REGULATORS]; ++}; ++ ++/* MMU mode flags */ ++#define KBASE_MMU_MODE_HAS_NON_CACHEABLE (1ul << 0) /* Has NON_CACHEABLE MEMATTR */ ++ ++/** ++ * struct kbase_mmu_mode - object containing pointer to methods invoked for ++ * programming the MMU, as per the MMU mode supported ++ * by Hw. ++ * @update: enable & setup/configure one of the GPU address space. ++ * @get_as_setup: retrieve the configuration of one of the GPU address space. ++ * @disable_as: disable one of the GPU address space. ++ * @pte_to_phy_addr: retrieve the physical address encoded in the page table entry. ++ * @ate_is_valid: check if the pte is a valid address translation entry ++ * encoding the physical address of the actual mapped page. ++ * @pte_is_valid: check if the pte is a valid entry encoding the physical ++ * address of the next lower level page table. ++ * @entry_set_ate: program the pte to be a valid address translation entry to ++ * encode the physical address of the actual page being mapped. ++ * @entry_set_pte: program the pte to be a valid entry to encode the physical ++ * address of the next lower level page table. ++ * @entry_invalidate: clear out or invalidate the pte. ++ * @flags: bitmask of MMU mode flags. Refer to KBASE_MMU_MODE_ constants. ++ */ ++struct kbase_mmu_mode { ++ void (*update)(struct kbase_device *kbdev, ++ struct kbase_mmu_table *mmut, ++ int as_nr); ++ void (*get_as_setup)(struct kbase_mmu_table *mmut, ++ struct kbase_mmu_setup * const setup); ++ void (*disable_as)(struct kbase_device *kbdev, int as_nr); ++ phys_addr_t (*pte_to_phy_addr)(u64 entry); ++ int (*ate_is_valid)(u64 ate, int level); ++ int (*pte_is_valid)(u64 pte, int level); ++ void (*entry_set_ate)(u64 *entry, struct tagged_addr phy, ++ unsigned long flags, int level); ++ void (*entry_set_pte)(u64 *entry, phys_addr_t phy); ++ void (*entry_invalidate)(u64 *entry); ++ unsigned long flags; ++}; ++ ++struct kbase_mmu_mode const *kbase_mmu_mode_get_lpae(void); ++struct kbase_mmu_mode const *kbase_mmu_mode_get_aarch64(void); ++ ++#define DEVNAME_SIZE 16 ++ ++/** ++ * enum kbase_devfreq_work_type - The type of work to perform in the devfreq ++ * suspend/resume worker. ++ * @DEVFREQ_WORK_NONE: Initilisation state. ++ * @DEVFREQ_WORK_SUSPEND: Call devfreq_suspend_device(). ++ * @DEVFREQ_WORK_RESUME: Call devfreq_resume_device(). ++ */ ++enum kbase_devfreq_work_type { ++ DEVFREQ_WORK_NONE, ++ DEVFREQ_WORK_SUSPEND, ++ DEVFREQ_WORK_RESUME ++}; ++ ++/** ++ * struct kbase_devfreq_queue_info - Object representing an instance for managing ++ * the queued devfreq suspend/resume works. ++ * @workq: Workqueue for devfreq suspend/resume requests ++ * @work: Work item for devfreq suspend & resume ++ * @req_type: Requested work type to be performed by the devfreq ++ * suspend/resume worker ++ * @acted_type: Work type has been acted on by the worker, i.e. the ++ * internal recorded state of the suspend/resume ++ */ ++struct kbase_devfreq_queue_info { ++ struct workqueue_struct *workq; ++ struct work_struct work; ++ enum kbase_devfreq_work_type req_type; ++ enum kbase_devfreq_work_type acted_type; ++}; ++ ++/** ++ * struct kbase_process - Representing an object of a kbase process instantiated ++ * when the first kbase context is created under it. ++ * @tgid: Thread group ID. ++ * @total_gpu_pages: Total gpu pages allocated across all the contexts ++ * of this process, it accounts for both native allocations ++ * and dma_buf imported allocations. ++ * @kctx_list: List of kbase contexts created for the process. ++ * @kprcs_node: Node to a rb_tree, kbase_device will maintain a rb_tree ++ * based on key tgid, kprcs_node is the node link to ++ * &struct_kbase_device.process_root. ++ * @dma_buf_root: RB tree of the dma-buf imported allocations, imported ++ * across all the contexts created for this process. ++ * Used to ensure that pages of allocation are accounted ++ * only once for the process, even if the allocation gets ++ * imported multiple times for the process. ++ */ ++struct kbase_process { ++ pid_t tgid; ++ size_t total_gpu_pages; ++ struct list_head kctx_list; ++ ++ struct rb_node kprcs_node; ++ struct rb_root dma_buf_root; ++}; ++ ++/** ++ * struct kbase_device - Object representing an instance of GPU platform device, ++ * allocated from the probe method of mali driver. ++ * @hw_quirks_sc: Configuration to be used for the shader cores as per ++ * the HW issues present in the GPU. ++ * @hw_quirks_tiler: Configuration to be used for the Tiler as per the HW ++ * issues present in the GPU. ++ * @hw_quirks_mmu: Configuration to be used for the MMU as per the HW ++ * issues present in the GPU. ++ * @hw_quirks_jm: Configuration to be used for the Job Manager as per ++ * the HW issues present in the GPU. ++ * @entry: Links the device instance to the global list of GPU ++ * devices. The list would have as many entries as there ++ * are GPU device instances. ++ * @dev: Pointer to the kernel's generic/base representation ++ * of the GPU platform device. ++ * @mdev: Pointer to the miscellaneous device registered to ++ * provide Userspace access to kernel driver through the ++ * device file /dev/malixx. ++ * @reg_start: Base address of the region in physical address space ++ * where GPU registers have been mapped. ++ * @reg_size: Size of the region containing GPU registers ++ * @reg: Kernel virtual address of the region containing GPU ++ * registers, using which Driver will access the registers. ++ * @irqs: Array containing IRQ resource info for 3 types of ++ * interrupts : Job scheduling, MMU & GPU events (like ++ * power management, cache etc.) ++ * @clocks: Pointer to the input clock resources referenced by ++ * the GPU device node. ++ * @nr_clocks: Number of clocks set in the clocks array. ++ * @regulators: Pointer to the structs corresponding to the ++ * regulators referenced by the GPU device node. ++ * @nr_regulators: Number of regulators set in the regulators array. ++ * @opp_table: Pointer to the device OPP structure maintaining the ++ * link to OPPs attached to a device. This is obtained ++ * after setting regulator names for the device. ++ * @devname: string containing the name used for GPU device instance, ++ * miscellaneous device is registered using the same name. ++ * @id: Unique identifier for the device, indicates the number of ++ * devices which have been created so far. ++ * @model: Pointer, valid only when Driver is compiled to not access ++ * the real GPU Hw, to the dummy model which tries to mimic ++ * to some extent the state & behavior of GPU Hw in response ++ * to the register accesses made by the Driver. ++ * @irq_slab: slab cache for allocating the work items queued when ++ * model mimics raising of IRQ to cause an interrupt on CPU. ++ * @irq_workq: workqueue for processing the irq work items. ++ * @serving_job_irq: function to execute work items queued when model mimics ++ * the raising of JS irq, mimics the interrupt handler ++ * processing JS interrupts. ++ * @serving_gpu_irq: function to execute work items queued when model mimics ++ * the raising of GPU irq, mimics the interrupt handler ++ * processing GPU interrupts. ++ * @serving_mmu_irq: function to execute work items queued when model mimics ++ * the raising of MMU irq, mimics the interrupt handler ++ * processing MMU interrupts. ++ * @reg_op_lock: lock used by model to serialize the handling of register ++ * accesses made by the driver. ++ * @pm: Per device object for storing data for power management ++ * framework. ++ * @js_data: Per device object encapsulating the current context of ++ * Job Scheduler, which is global to the device and is not ++ * tied to any particular struct kbase_context running on ++ * the device ++ * @mem_pools: Global pools of free physical memory pages which can ++ * be used by all the contexts. ++ * @memdev: keeps track of the in use physical pages allocated by ++ * the Driver. ++ * @mmu_mode: Pointer to the object containing methods for programming ++ * the MMU, depending on the type of MMU supported by Hw. ++ * @mgm_dev: Pointer to the memory group manager device attached ++ * to the GPU device. This points to an internal memory ++ * group manager if no platform-specific memory group ++ * manager was retrieved through device tree. ++ * @as: Array of objects representing address spaces of GPU. ++ * @as_free: Bitpattern of free/available GPU address spaces. ++ * @as_to_kctx: Array of pointers to struct kbase_context, having ++ * GPU adrress spaces assigned to them. ++ * @mmu_mask_change: Lock to serialize the access to MMU interrupt mask ++ * register used in the handling of Bus & Page faults. ++ * @gpu_props: Object containing complete information about the ++ * configuration/properties of GPU HW device in use. ++ * @hw_issues_mask: List of SW workarounds for HW issues ++ * @hw_features_mask: List of available HW features. ++ * @disjoint_event: struct for keeping track of the disjoint information, ++ * that whether the GPU is in a disjoint state and the ++ * number of disjoint events that have occurred on GPU. ++ * @nr_hw_address_spaces: Number of address spaces actually available in the ++ * GPU, remains constant after driver initialisation. ++ * @nr_user_address_spaces: Number of address spaces available to user contexts ++ * @hwcnt: Structure used for instrumentation and HW counters ++ * dumping ++ * @hwcnt_gpu_iface: Backend interface for GPU hardware counter access. ++ * @hwcnt_gpu_ctx: Context for GPU hardware counter access. ++ * @hwaccess_lock must be held when calling ++ * kbase_hwcnt_context_enable() with @hwcnt_gpu_ctx. ++ * @hwcnt_gpu_virt: Virtualizer for GPU hardware counters. ++ * @vinstr_ctx: vinstr context created per device. ++ * @timeline_flags: Bitmask defining which sets of timeline tracepoints ++ * are enabled. If zero, there is no timeline client and ++ * therefore timeline is disabled. ++ * @timeline: Timeline context created per device. ++ * @trace_lock: Lock to serialize the access to trace buffer. ++ * @trace_first_out: Index/offset in the trace buffer at which the first ++ * unread message is present. ++ * @trace_next_in: Index/offset in the trace buffer at which the new ++ * message will be written. ++ * @trace_rbuf: Pointer to the buffer storing debug messages/prints ++ * tracing the various events in Driver. ++ * The buffer is filled in circular fashion. ++ * @reset_timeout_ms: Number of milliseconds to wait for the soft stop to ++ * complete for the GPU jobs before proceeding with the ++ * GPU reset. ++ * @cache_clean_in_progress: Set when a cache clean has been started, and ++ * cleared when it has finished. This prevents multiple ++ * cache cleans being done simultaneously. ++ * @cache_clean_queued: Set if a cache clean is invoked while another is in ++ * progress. If this happens, another cache clean needs ++ * to be triggered immediately after completion of the ++ * current one. ++ * @cache_clean_wait: Signalled when a cache clean has finished. ++ * @platform_context: Platform specific private data to be accessed by ++ * platform specific config files only. ++ * @kctx_list: List of kbase_contexts created for the device, ++ * including any contexts that might be created for ++ * hardware counters. ++ * @kctx_list_lock: Lock protecting concurrent accesses to @kctx_list. ++ * @devfreq_profile: Describes devfreq profile for the Mali GPU device, passed ++ * to devfreq_add_device() to add devfreq feature to Mali ++ * GPU device. ++ * @devfreq: Pointer to devfreq structure for Mali GPU device, ++ * returned on the call to devfreq_add_device(). ++ * @current_freqs: The real frequencies, corresponding to ++ * @current_nominal_freq, at which the Mali GPU device ++ * is currently operating, as retrieved from ++ * @devfreq_table in the target callback of ++ * @devfreq_profile. ++ * @current_nominal_freq: The nominal frequency currently used for the Mali GPU ++ * device as retrieved through devfreq_recommended_opp() ++ * using the freq value passed as an argument to target ++ * callback of @devfreq_profile ++ * @current_voltages: The voltages corresponding to @current_nominal_freq, ++ * as retrieved from @devfreq_table in the target ++ * callback of @devfreq_profile. ++ * @current_core_mask: bitmask of shader cores that are currently desired & ++ * enabled, corresponding to @current_nominal_freq as ++ * retrieved from @devfreq_table in the target callback ++ * of @devfreq_profile. ++ * @devfreq_table: Pointer to the lookup table for converting between ++ * nominal OPP (operating performance point) frequency, ++ * and real frequency and core mask. This table is ++ * constructed according to operating-points-v2-mali ++ * table in devicetree. ++ * @num_opps: Number of operating performance points available for the Mali ++ * GPU device. ++ * @devfreq_queue: Per device object for storing data that manages devfreq ++ * suspend & resume request queue and the related items. ++ * @devfreq_cooling: Pointer returned on registering devfreq cooling device ++ * corresponding to @devfreq. ++ * @ipa_protection_mode_switched: is set to TRUE when GPU is put into protected ++ * mode. It is a sticky flag which is cleared by IPA ++ * once it has made use of information that GPU had ++ * previously entered protected mode. ++ * @ipa: Top level structure for IPA, containing pointers to both ++ * configured & fallback models. ++ * @previous_frequency: Previous frequency of GPU clock used for ++ * BASE_HW_ISSUE_GPU2017_1336 workaround, This clock is ++ * restored when L2 is powered on. ++ * @job_fault_debug: Flag to control the dumping of debug data for job faults, ++ * set when the 'job_fault' debugfs file is opened. ++ * @mali_debugfs_directory: Root directory for the debugfs files created by the driver ++ * @debugfs_ctx_directory: Directory inside the @mali_debugfs_directory containing ++ * a sub-directory for every context. ++ * @debugfs_as_read_bitmap: bitmap of address spaces for which the bus or page fault ++ * has occurred. ++ * @job_fault_wq: Waitqueue to block the job fault dumping daemon till the ++ * occurrence of a job fault. ++ * @job_fault_resume_wq: Waitqueue on which every context with a faulty job wait ++ * for the job fault dumping to complete before they can ++ * do bottom half of job done for the atoms which followed ++ * the faulty atom. ++ * @job_fault_resume_workq: workqueue to process the work items queued for the faulty ++ * atoms, whereby the work item function waits for the dumping ++ * to get completed. ++ * @job_fault_event_list: List of atoms, each belonging to a different context, which ++ * generated a job fault. ++ * @job_fault_event_lock: Lock to protect concurrent accesses to @job_fault_event_list ++ * @regs_dump_debugfs_data: Contains the offset of register to be read through debugfs ++ * file "read_register". ++ * @ctx_num: Total number of contexts created for the device. ++ * @io_history: Pointer to an object keeping a track of all recent ++ * register accesses. The history of register accesses ++ * can be read through "regs_history" debugfs file. ++ * @hwaccess: Contains a pointer to active kbase context and GPU ++ * backend specific data for HW access layer. ++ * @faults_pending: Count of page/bus faults waiting for bottom half processing ++ * via workqueues. ++ * @poweroff_pending: Set when power off operation for GPU is started, reset when ++ * power on for GPU is started. ++ * @infinite_cache_active_default: Set to enable using infinite cache for all the ++ * allocations of a new context. ++ * @mem_pool_defaults: Default configuration for the group of memory pools ++ * created for a new context. ++ * @current_gpu_coherency_mode: coherency mode in use, which can be different ++ * from @system_coherency, when using protected mode. ++ * @system_coherency: coherency mode as retrieved from the device tree. ++ * @cci_snoop_enabled: Flag to track when CCI snoops have been enabled. ++ * @snoop_enable_smc: SMC function ID to call into Trusted firmware to ++ * enable cache snooping. Value of 0 indicates that it ++ * is not used. ++ * @snoop_disable_smc: SMC function ID to call disable cache snooping. ++ * @protected_ops: Pointer to the methods for switching in or out of the ++ * protected mode, as per the @protected_dev being used. ++ * @protected_dev: Pointer to the protected mode switcher device attached ++ * to the GPU device retrieved through device tree if ++ * GPU do not support protected mode switching natively. ++ * @protected_mode: set to TRUE when GPU is put into protected mode ++ * @protected_mode_transition: set to TRUE when GPU is transitioning into or ++ * out of protected mode. ++ * @protected_mode_hwcnt_desired: True if we want GPU hardware counters to be ++ * enabled. Counters must be disabled before transition ++ * into protected mode. ++ * @protected_mode_hwcnt_disabled: True if GPU hardware counters are not ++ * enabled. ++ * @protected_mode_hwcnt_disable_work: Work item to disable GPU hardware ++ * counters, used if atomic disable is not possible. ++ * @buslogger: Pointer to the structure required for interfacing ++ * with the bus logger module to set the size of buffer ++ * used by the module for capturing bus logs. ++ * @irq_reset_flush: Flag to indicate that GPU reset is in-flight and flush of ++ * IRQ + bottom half is being done, to prevent the writes ++ * to MMU_IRQ_CLEAR & MMU_IRQ_MASK registers. ++ * @inited_subsys: Bitmap of inited sub systems at the time of device probe. ++ * Used during device remove or for handling error in probe. ++ * @hwaccess_lock: Lock, which can be taken from IRQ context, to serialize ++ * the updates made to Job dispatcher + scheduler states. ++ * @mmu_hw_mutex: Protects access to MMU operations and address space ++ * related state. ++ * @serialize_jobs: Currently used mode for serialization of jobs, both ++ * intra & inter slots serialization is supported. ++ * @backup_serialize_jobs: Copy of the original value of @serialize_jobs taken ++ * when GWT is enabled. Used to restore the original value ++ * on disabling of GWT. ++ * @js_ctx_scheduling_mode: Context scheduling mode currently being used by ++ * Job Scheduler ++ * @l2_size_override: Used to set L2 cache size via device tree blob ++ * @l2_hash_override: Used to set L2 cache hash via device tree blob ++ * @process_root: rb_tree root node for maintaining a rb_tree of ++ * kbase_process based on key tgid(thread group ID). ++ * @dma_buf_root: rb_tree root node for maintaining a rb_tree of ++ * &struct kbase_dma_buf based on key dma_buf. ++ * We maintain a rb_tree of dma_buf mappings under ++ * kbase_device and kbase_process, one indicates a ++ * mapping and gpu memory usage at device level and ++ * other one at process level. ++ * @total_gpu_pages: Total GPU pages used for the complete GPU device. ++ * @dma_buf_lock: This mutex should be held while accounting for ++ * @total_gpu_pages from imported dma buffers. ++ * @gpu_mem_usage_lock: This spinlock should be held while accounting ++ * @total_gpu_pages for both native and dma-buf imported ++ * allocations. ++ */ ++struct kbase_device { ++ u32 hw_quirks_sc; ++ u32 hw_quirks_tiler; ++ u32 hw_quirks_mmu; ++ u32 hw_quirks_jm; ++ ++ struct list_head entry; ++ struct device *dev; ++ struct miscdevice mdev; ++ u64 reg_start; ++ size_t reg_size; ++ void __iomem *reg; ++ ++ struct { ++ int irq; ++ int flags; ++ } irqs[3]; ++ ++ struct clk *clocks[BASE_MAX_NR_CLOCKS_REGULATORS]; ++ unsigned int nr_clocks; ++#ifdef CONFIG_REGULATOR ++ struct regulator *regulators[BASE_MAX_NR_CLOCKS_REGULATORS]; ++ unsigned int nr_regulators; ++#if (KERNEL_VERSION(4, 10, 0) <= LINUX_VERSION_CODE) ++ struct opp_table *opp_table; ++#endif /* (KERNEL_VERSION(4, 10, 0) <= LINUX_VERSION_CODE */ ++#endif /* CONFIG_REGULATOR */ ++ char devname[DEVNAME_SIZE]; ++ u32 id; ++ ++#ifdef CONFIG_MALI_BIFROST_NO_MALI ++ void *model; ++ struct kmem_cache *irq_slab; ++ struct workqueue_struct *irq_workq; ++ atomic_t serving_job_irq; ++ atomic_t serving_gpu_irq; ++ atomic_t serving_mmu_irq; ++ spinlock_t reg_op_lock; ++#endif /* CONFIG_MALI_BIFROST_NO_MALI */ ++ ++ struct kbase_pm_device_data pm; ++ ++ struct kbase_mem_pool_group mem_pools; ++ struct kbasep_mem_device memdev; ++ struct kbase_mmu_mode const *mmu_mode; ++ ++ struct memory_group_manager_device *mgm_dev; ++ ++ struct kbase_as as[BASE_MAX_NR_AS]; ++ u16 as_free; /* Bitpattern of free Address Spaces */ ++ struct kbase_context *as_to_kctx[BASE_MAX_NR_AS]; ++ ++ spinlock_t mmu_mask_change; ++ ++ struct kbase_gpu_props gpu_props; ++ ++ unsigned long hw_issues_mask[(BASE_HW_ISSUE_END + BITS_PER_LONG - 1) / BITS_PER_LONG]; ++ unsigned long hw_features_mask[(BASE_HW_FEATURE_END + BITS_PER_LONG - 1) / BITS_PER_LONG]; ++ ++ struct { ++ atomic_t count; ++ atomic_t state; ++ } disjoint_event; ++ ++ s8 nr_hw_address_spaces; ++ s8 nr_user_address_spaces; ++ ++ struct kbase_hwcnt { ++ /* The lock should be used when accessing any of the following members */ ++ spinlock_t lock; ++ ++ struct kbase_context *kctx; ++ u64 addr; ++ u64 addr_bytes; ++ ++ struct kbase_instr_backend backend; ++ } hwcnt; ++ ++ struct kbase_hwcnt_backend_interface hwcnt_gpu_iface; ++ struct kbase_hwcnt_context *hwcnt_gpu_ctx; ++ struct kbase_hwcnt_virtualizer *hwcnt_gpu_virt; ++ struct kbase_vinstr_context *vinstr_ctx; ++ ++ atomic_t timeline_flags; ++ struct kbase_timeline *timeline; ++ ++#if KBASE_KTRACE_TARGET_RBUF ++ struct kbase_ktrace ktrace; ++#endif ++ u32 reset_timeout_ms; ++ ++ bool cache_clean_in_progress; ++ bool cache_clean_queued; ++ wait_queue_head_t cache_clean_wait; ++ ++ void *platform_context; ++ ++ struct list_head kctx_list; ++ struct mutex kctx_list_lock; ++ ++#ifdef CONFIG_MALI_BIFROST_DEVFREQ ++ struct devfreq_dev_profile devfreq_profile; ++ struct devfreq *devfreq; ++ unsigned long current_freqs[BASE_MAX_NR_CLOCKS_REGULATORS]; ++ unsigned long current_nominal_freq; ++ unsigned long current_voltages[BASE_MAX_NR_CLOCKS_REGULATORS]; ++ u64 current_core_mask; ++ struct kbase_devfreq_opp *devfreq_table; ++ int num_opps; ++ struct kbasep_pm_metrics last_devfreq_metrics; ++ struct monitor_dev_info *mdev_info; ++ struct ipa_power_model_data *model_data; ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0) ++ struct kbase_devfreq_queue_info devfreq_queue; ++#endif ++ ++#ifdef CONFIG_DEVFREQ_THERMAL ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) ++ struct devfreq_cooling_device *devfreq_cooling; ++#else ++ struct thermal_cooling_device *devfreq_cooling; ++#endif ++ bool ipa_protection_mode_switched; ++ struct { ++ /* Access to this struct must be with ipa.lock held */ ++ struct mutex lock; ++ struct kbase_ipa_model *configured_model; ++ struct kbase_ipa_model *fallback_model; ++ ++ /* Values of the PM utilization metrics from last time the ++ * power model was invoked. The utilization is calculated as ++ * the difference between last_metrics and the current values. ++ */ ++ struct kbasep_pm_metrics last_metrics; ++ /* Model data to pass to ipa_gpu_active/idle() */ ++ struct kbase_ipa_model_vinstr_data *model_data; ++ ++ /* true if use of fallback model has been forced by the User */ ++ bool force_fallback_model; ++ } ipa; ++#endif /* CONFIG_DEVFREQ_THERMAL */ ++#endif /* CONFIG_MALI_BIFROST_DEVFREQ */ ++ unsigned long previous_frequency; ++ ++ atomic_t job_fault_debug; ++ ++#ifdef CONFIG_DEBUG_FS ++ struct dentry *mali_debugfs_directory; ++ struct dentry *debugfs_ctx_directory; ++ struct dentry *debugfs_instr_directory; ++ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++ u64 debugfs_as_read_bitmap; ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++ ++ wait_queue_head_t job_fault_wq; ++ wait_queue_head_t job_fault_resume_wq; ++ struct workqueue_struct *job_fault_resume_workq; ++ struct list_head job_fault_event_list; ++ spinlock_t job_fault_event_lock; ++ ++#if !MALI_CUSTOMER_RELEASE ++ struct { ++ u16 reg_offset; ++ } regs_dump_debugfs_data; ++#endif /* !MALI_CUSTOMER_RELEASE */ ++#endif /* CONFIG_DEBUG_FS */ ++ ++ atomic_t ctx_num; ++ ++#ifdef CONFIG_DEBUG_FS ++ struct kbase_io_history io_history; ++#endif /* CONFIG_DEBUG_FS */ ++ ++ struct kbase_hwaccess_data hwaccess; ++ ++ atomic_t faults_pending; ++ ++ bool poweroff_pending; ++ ++ ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0)) ++ bool infinite_cache_active_default; ++#else ++ u32 infinite_cache_active_default; ++#endif ++ struct kbase_mem_pool_group_config mem_pool_defaults; ++ ++ u32 current_gpu_coherency_mode; ++ u32 system_coherency; ++ ++ bool cci_snoop_enabled; ++ ++ u32 snoop_enable_smc; ++ u32 snoop_disable_smc; ++ ++ const struct protected_mode_ops *protected_ops; ++ ++ struct protected_mode_device *protected_dev; ++ ++ bool protected_mode; ++ ++ bool protected_mode_transition; ++ ++ bool protected_mode_hwcnt_desired; ++ ++ bool protected_mode_hwcnt_disabled; ++ ++ struct work_struct protected_mode_hwcnt_disable_work; ++ ++#ifdef CONFIG_MALI_BUSLOG ++ struct bus_logger_client *buslogger; ++#endif ++ ++ bool irq_reset_flush; ++ ++ u32 inited_subsys; ++ ++ spinlock_t hwaccess_lock; ++ ++ struct mutex mmu_hw_mutex; ++ ++ u8 l2_size_override; ++ u8 l2_hash_override; ++ ++#if MALI_USE_CSF ++ /* Command-stream front-end for the device. */ ++ struct kbase_csf_device csf; ++#else ++ struct kbasep_js_device_data js_data; ++ ++ /* See KBASE_JS_*_PRIORITY_MODE for details. */ ++ u32 js_ctx_scheduling_mode; ++ ++ /* See KBASE_SERIALIZE_* for details */ ++ u8 serialize_jobs; ++ ++#ifdef CONFIG_MALI_CINSTR_GWT ++ u8 backup_serialize_jobs; ++#endif /* CONFIG_MALI_CINSTR_GWT */ ++ ++#endif /* MALI_USE_CSF */ ++ ++ struct rb_root process_root; ++ struct rb_root dma_buf_root; ++ ++ size_t total_gpu_pages; ++ struct mutex dma_buf_lock; ++ spinlock_t gpu_mem_usage_lock; ++ ++ struct { ++ struct kbase_context *ctx; ++ u64 jc; ++ int slot; ++ u64 flags; ++ } dummy_job_wa; ++ ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++ /* Pointer to the arbiter device */ ++ struct kbase_arbiter_device arb; ++#endif ++}; ++ ++/** ++ * enum kbase_file_state - Initialization state of a file opened by @kbase_open ++ * ++ * @KBASE_FILE_NEED_VSN: Initial state, awaiting API version. ++ * @KBASE_FILE_VSN_IN_PROGRESS: Indicates if setting an API version is in ++ * progress and other setup calls shall be ++ * rejected. ++ * @KBASE_FILE_NEED_CTX: Indicates if the API version handshake has ++ * completed, awaiting context creation flags. ++ * @KBASE_FILE_CTX_IN_PROGRESS: Indicates if the context's setup is in progress ++ * and other setup calls shall be rejected. ++ * @KBASE_FILE_COMPLETE: Indicates if the setup for context has ++ * completed, i.e. flags have been set for the ++ * context. ++ * ++ * The driver allows only limited interaction with user-space until setup ++ * is complete. ++ */ ++enum kbase_file_state { ++ KBASE_FILE_NEED_VSN, ++ KBASE_FILE_VSN_IN_PROGRESS, ++ KBASE_FILE_NEED_CTX, ++ KBASE_FILE_CTX_IN_PROGRESS, ++ KBASE_FILE_COMPLETE ++}; ++ ++/** ++ * struct kbase_file - Object representing a file opened by @kbase_open ++ * ++ * @kbdev: Object representing an instance of GPU platform device, ++ * allocated from the probe method of the Mali driver. ++ * @filp: Pointer to the struct file corresponding to device file ++ * /dev/malixx instance, passed to the file's open method. ++ * @kctx: Object representing an entity, among which GPU is ++ * scheduled and which gets its own GPU address space. ++ * Invalid until @setup_state is KBASE_FILE_COMPLETE. ++ * @api_version: Contains the version number for User/kernel interface, ++ * used for compatibility check. Invalid until ++ * @setup_state is KBASE_FILE_NEED_CTX. ++ * @setup_state: Initialization state of the file. Values come from ++ * the kbase_file_state enumeration. ++ */ ++struct kbase_file { ++ struct kbase_device *kbdev; ++ struct file *filp; ++ struct kbase_context *kctx; ++ unsigned long api_version; ++ atomic_t setup_state; ++}; ++ ++/** ++ * enum kbase_context_flags - Flags for kbase contexts ++ * ++ * @KCTX_COMPAT: Set when the context process is a compat process, 32-bit ++ * process on a 64-bit kernel. ++ * ++ * @KCTX_RUNNABLE_REF: Set when context is counted in ++ * kbdev->js_data.nr_contexts_runnable. Must hold queue_mutex when accessing. ++ * ++ * @KCTX_ACTIVE: Set when the context is active. ++ * ++ * @KCTX_PULLED: Set when last kick() caused atoms to be pulled from this ++ * context. ++ * ++ * @KCTX_MEM_PROFILE_INITIALIZED: Set when the context's memory profile has been ++ * initialized. ++ * ++ * @KCTX_INFINITE_CACHE: Set when infinite cache is to be enabled for new ++ * allocations. Existing allocations will not change. ++ * ++ * @KCTX_SUBMIT_DISABLED: Set to prevent context from submitting any jobs. ++ * ++ * @KCTX_PRIVILEGED:Set if the context uses an address space and should be kept ++ * scheduled in. ++ * ++ * @KCTX_SCHEDULED: Set when the context is scheduled on the Run Pool. ++ * This is only ever updated whilst the jsctx_mutex is held. ++ * ++ * @KCTX_DYING: Set when the context process is in the process of being evicted. ++ * ++ * @KCTX_NO_IMPLICIT_SYNC: Set when explicit Android fences are in use on this ++ * context, to disable use of implicit dma-buf fences. This is used to avoid ++ * potential synchronization deadlocks. ++ * ++ * @KCTX_FORCE_SAME_VA: Set when BASE_MEM_SAME_VA should be forced on memory ++ * allocations. For 64-bit clients it is enabled by default, and disabled by ++ * default on 32-bit clients. Being able to clear this flag is only used for ++ * testing purposes of the custom zone allocation on 64-bit user-space builds, ++ * where we also require more control than is available through e.g. the JIT ++ * allocation mechanism. However, the 64-bit user-space client must still ++ * reserve a JIT region using KBASE_IOCTL_MEM_JIT_INIT ++ * ++ * @KCTX_PULLED_SINCE_ACTIVE_JS0: Set when the context has had an atom pulled ++ * from it for job slot 0. This is reset when the context first goes active or ++ * is re-activated on that slot. ++ * ++ * @KCTX_PULLED_SINCE_ACTIVE_JS1: Set when the context has had an atom pulled ++ * from it for job slot 1. This is reset when the context first goes active or ++ * is re-activated on that slot. ++ * ++ * @KCTX_PULLED_SINCE_ACTIVE_JS2: Set when the context has had an atom pulled ++ * from it for job slot 2. This is reset when the context first goes active or ++ * is re-activated on that slot. ++ * ++ * @KCTX_AS_DISABLED_ON_FAULT: Set when the GPU address space is disabled for ++ * the context due to unhandled page(or bus) fault. It is cleared when the ++ * refcount for the context drops to 0 or on when the address spaces are ++ * re-enabled on GPU reset or power cycle. ++ * ++ * All members need to be separate bits. This enum is intended for use in a ++ * bitmask where multiple values get OR-ed together. ++ */ ++enum kbase_context_flags { ++ KCTX_COMPAT = 1U << 0, ++ KCTX_RUNNABLE_REF = 1U << 1, ++ KCTX_ACTIVE = 1U << 2, ++ KCTX_PULLED = 1U << 3, ++ KCTX_MEM_PROFILE_INITIALIZED = 1U << 4, ++ KCTX_INFINITE_CACHE = 1U << 5, ++ KCTX_SUBMIT_DISABLED = 1U << 6, ++ KCTX_PRIVILEGED = 1U << 7, ++ KCTX_SCHEDULED = 1U << 8, ++ KCTX_DYING = 1U << 9, ++ KCTX_NO_IMPLICIT_SYNC = 1U << 10, ++ KCTX_FORCE_SAME_VA = 1U << 11, ++ KCTX_PULLED_SINCE_ACTIVE_JS0 = 1U << 12, ++ KCTX_PULLED_SINCE_ACTIVE_JS1 = 1U << 13, ++ KCTX_PULLED_SINCE_ACTIVE_JS2 = 1U << 14, ++ KCTX_AS_DISABLED_ON_FAULT = 1U << 15, ++#if MALI_JIT_PRESSURE_LIMIT_BASE ++ /* ++ * Set when JIT physical page limit is less than JIT virtual address ++ * page limit, so we must take care to not exceed the physical limit ++ */ ++ KCTX_JPL_ENABLED = 1U << 16, ++#endif /* !MALI_JIT_PRESSURE_LIMIT_BASE */ ++}; ++ ++struct kbase_sub_alloc { ++ struct list_head link; ++ struct page *page; ++ DECLARE_BITMAP(sub_pages, SZ_2M / SZ_4K); ++}; ++ ++/** ++ * struct kbase_context - Kernel base context ++ * ++ * @filp: Pointer to the struct file corresponding to device file ++ * /dev/malixx instance, passed to the file's open method. ++ * @kbdev: Pointer to the Kbase device for which the context is created. ++ * @kctx_list_link: Node into Kbase device list of contexts. ++ * @mmu: Structure holding details of the MMU tables for this ++ * context ++ * @id: Unique identifier for the context, indicates the number of ++ * contexts which have been created for the device so far. ++ * @api_version: contains the version number for User/kernel interface, ++ * used for compatibility check. ++ * @event_list: list of posted events about completed atoms, to be sent to ++ * event handling thread of Userpsace. ++ * @event_coalesce_list: list containing events corresponding to successive atoms ++ * which have requested deferred delivery of the completion ++ * events to Userspace. ++ * @event_mutex: Lock to protect the concurrent access to @event_list & ++ * @event_mutex. ++ * @event_closed: Flag set through POST_TERM ioctl, indicates that Driver ++ * should stop posting events and also inform event handling ++ * thread that context termination is in progress. ++ * @event_workq: Workqueue for processing work items corresponding to atoms ++ * that do not return an event to userspace. ++ * @event_count: Count of the posted events to be consumed by Userspace. ++ * @event_coalesce_count: Count of the events present in @event_coalesce_list. ++ * @flags: bitmap of enums from kbase_context_flags, indicating the ++ * state & attributes for the context. ++ * @aliasing_sink_page: Special page used for KBASE_MEM_TYPE_ALIAS allocations, ++ * which can alias number of memory regions. The page is ++ * represent a region where it is mapped with a write-alloc ++ * cache setup, typically used when the write result of the ++ * GPU isn't needed, but the GPU must write anyway. ++ * @mem_partials_lock: Lock for protecting the operations done on the elements ++ * added to @mem_partials list. ++ * @mem_partials: List head for the list of large pages, 2MB in size, which ++ * which have been split into 4 KB pages and are used ++ * partially for the allocations >= 2 MB in size. ++ * @reg_lock: Lock used for GPU virtual address space management operations, ++ * like adding/freeing a memory region in the address space. ++ * Can be converted to a rwlock ?. ++ * @reg_rbtree_same: RB tree of the memory regions allocated from the SAME_VA ++ * zone of the GPU virtual address space. Used for allocations ++ * having the same value for GPU & CPU virtual address. ++ * @reg_rbtree_custom: RB tree of the memory regions allocated from the CUSTOM_VA ++ * zone of the GPU virtual address space. ++ * @reg_rbtree_exec: RB tree of the memory regions allocated from the EXEC_VA ++ * zone of the GPU virtual address space. Used for GPU-executable ++ * allocations which don't need the SAME_VA property. ++ * @cookies: Bitmask containing of BITS_PER_LONG bits, used mainly for ++ * SAME_VA allocations to defer the reservation of memory region ++ * (from the GPU virtual address space) from base_mem_alloc ++ * ioctl to mmap system call. This helps returning unique ++ * handles, disguised as GPU VA, to Userspace from base_mem_alloc ++ * and later retrieving the pointer to memory region structure ++ * in the mmap handler. ++ * @pending_regions: Array containing pointers to memory region structures, ++ * used in conjunction with @cookies bitmask mainly for ++ * providing a mechansim to have the same value for CPU & ++ * GPU virtual address. ++ * @event_queue: Wait queue used for blocking the thread, which consumes ++ * the base_jd_event corresponding to an atom, when there ++ * are no more posted events. ++ * @tgid: Thread group ID of the process whose thread created ++ * the context (by calling KBASE_IOCTL_VERSION_CHECK or ++ * KBASE_IOCTL_SET_FLAGS, depending on the @api_version). ++ * This is usually, but not necessarily, the same as the ++ * process whose thread opened the device file ++ * /dev/malixx instance. ++ * @pid: ID of the thread, corresponding to process @tgid, ++ * which actually created the context. This is usually, ++ * but not necessarily, the same as the thread which ++ * opened the device file /dev/malixx instance. ++ * @jctx: object encapsulating all the Job dispatcher related state, ++ * including the array of atoms. ++ * @used_pages: Keeps a track of the number of 4KB physical pages in use ++ * for the context. ++ * @nonmapped_pages: Updated in the same way as @used_pages, except for the case ++ * when special tracking page is freed by userspace where it ++ * is reset to 0. ++ * @permanent_mapped_pages: Usage count of permanently mapped memory ++ * @mem_pools: Context-specific pools of free physical memory pages. ++ * @reclaim: Shrinker object registered with the kernel containing ++ * the pointer to callback function which is invoked under ++ * low memory conditions. In the callback function Driver ++ * frees up the memory for allocations marked as ++ * evictable/reclaimable. ++ * @evict_list: List head for the list containing the allocations which ++ * can be evicted or freed up in the shrinker callback. ++ * @waiting_soft_jobs: List head for the list containing softjob atoms, which ++ * are either waiting for the event set operation, or waiting ++ * for the signaling of input fence or waiting for the GPU ++ * device to powered on so as to dump the CPU/GPU timestamps. ++ * @waiting_soft_jobs_lock: Lock to protect @waiting_soft_jobs list from concurrent ++ * accesses. ++ * @dma_fence: Object containing list head for the list of dma-buf fence ++ * waiting atoms and the waitqueue to process the work item ++ * queued for the atoms blocked on the signaling of dma-buf ++ * fences. ++ * @as_nr: id of the address space being used for the scheduled in ++ * context. This is effectively part of the Run Pool, because ++ * it only has a valid setting (!=KBASEP_AS_NR_INVALID) whilst ++ * the context is scheduled in. The hwaccess_lock must be held ++ * whilst accessing this. ++ * If the context relating to this value of as_nr is required, ++ * then the context must be retained to ensure that it doesn't ++ * disappear whilst it is being used. Alternatively, hwaccess_lock ++ * can be held to ensure the context doesn't disappear (but this ++ * has restrictions on what other locks can be taken simutaneously). ++ * @refcount: Keeps track of the number of users of this context. A user ++ * can be a job that is available for execution, instrumentation ++ * needing to 'pin' a context for counter collection, etc. ++ * If the refcount reaches 0 then this context is considered ++ * inactive and the previously programmed AS might be cleared ++ * at any point. ++ * Generally the reference count is incremented when the context ++ * is scheduled in and an atom is pulled from the context's per ++ * slot runnable tree in JM GPU or GPU command queue ++ * group is programmed on CSG slot in CSF GPU. ++ * @mm_update_lock: lock used for handling of special tracking page. ++ * @process_mm: Pointer to the memory descriptor of the process which ++ * created the context. Used for accounting the physical ++ * pages used for GPU allocations, done for the context, ++ * to the memory consumed by the process. ++ * @same_va_end: End address of the SAME_VA zone (in 4KB page units) ++ * @exec_va_start: Start address of the EXEC_VA zone (in 4KB page units) ++ * or U64_MAX if the EXEC_VA zone is uninitialized. ++ * @gpu_va_end: End address of the GPU va space (in 4KB page units) ++ * @jit_va: Indicates if a JIT_VA zone has been created. ++ * @mem_profile_data: Buffer containing the profiling information provided by ++ * Userspace, can be read through the mem_profile debugfs file. ++ * @mem_profile_size: Size of the @mem_profile_data. ++ * @mem_profile_lock: Lock to serialize the operations related to mem_profile ++ * debugfs file. ++ * @kctx_dentry: Pointer to the debugfs directory created for every context, ++ * inside kbase_device::debugfs_ctx_directory, containing ++ * context specific files. ++ * @reg_dump: Buffer containing a register offset & value pair, used ++ * for dumping job fault debug info. ++ * @job_fault_count: Indicates that a job fault occurred for the context and ++ * dumping of its debug info is in progress. ++ * @job_fault_resume_event_list: List containing atoms completed after the faulty ++ * atom but before the debug data for faulty atom was dumped. ++ * @jsctx_queue: Per slot & priority arrays of object containing the root ++ * of RB-tree holding currently runnable atoms on the job slot ++ * and the head item of the linked list of atoms blocked on ++ * cross-slot dependencies. ++ * @atoms_pulled: Total number of atoms currently pulled from the context. ++ * @atoms_pulled_slot: Per slot count of the number of atoms currently pulled ++ * from the context. ++ * @atoms_pulled_slot_pri: Per slot & priority count of the number of atoms currently ++ * pulled from the context. hwaccess_lock shall be held when ++ * accessing it. ++ * @blocked_js: Indicates if the context is blocked from submitting atoms ++ * on a slot at a given priority. This is set to true, when ++ * the atom corresponding to context is soft/hard stopped or ++ * removed from the HEAD_NEXT register in response to ++ * soft/hard stop. ++ * @slots_pullable: Bitmask of slots, indicating the slots for which the ++ * context has pullable atoms in the runnable tree. ++ * @work: Work structure used for deferred ASID assignment. ++ * @legacy_hwcnt_cli: Pointer to the legacy userspace hardware counters ++ * client, there can be only such client per kbase ++ * context. ++ * @legacy_hwcnt_lock: Lock used to prevent concurrent access to ++ * @legacy_hwcnt_cli. ++ * @completed_jobs: List containing completed atoms for which base_jd_event is ++ * to be posted. ++ * @work_count: Number of work items, corresponding to atoms, currently ++ * pending on job_done workqueue of @jctx. ++ * @soft_job_timeout: Timer object used for failing/cancelling the waiting ++ * soft-jobs which have been blocked for more than the ++ * timeout value used for the soft-jobs ++ * @jit_alloc: Array of 256 pointers to GPU memory regions, used for ++ * just-in-time memory allocations. ++ * @jit_max_allocations: Maximum allowed number of in-flight ++ * just-in-time memory allocations. ++ * @jit_current_allocations: Current number of in-flight just-in-time ++ * memory allocations. ++ * @jit_current_allocations_per_bin: Current number of in-flight just-in-time ++ * memory allocations per bin. ++ * @jit_version: Version number indicating whether userspace is using ++ * old or new version of interface for just-in-time ++ * memory allocations. ++ * 1 -> client used KBASE_IOCTL_MEM_JIT_INIT_10_2 ++ * 2 -> client used KBASE_IOCTL_MEM_JIT_INIT_11_5 ++ * 3 -> client used KBASE_IOCTL_MEM_JIT_INIT ++ * @jit_group_id: A memory group ID to be passed to a platform-specific ++ * memory group manager. ++ * Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). ++ * @jit_phys_pages_limit: Limit of physical pages to apply across all ++ * just-in-time memory allocations, applied to ++ * @jit_current_phys_pressure. ++ * @jit_current_phys_pressure: Current 'pressure' on physical pages, which is ++ * the sum of the worst case estimate of pages that ++ * could be used (i.e. the ++ * &struct_kbase_va_region.nr_pages for all in-use ++ * just-in-time memory regions that have not yet had ++ * a usage report) and the actual number of pages ++ * that were used (i.e. the ++ * &struct_kbase_va_region.used_pages for regions ++ * that have had a usage report). ++ * @jit_phys_pages_to_be_allocated: Count of the physical pages that are being ++ * now allocated for just-in-time memory ++ * allocations of a context (across all the ++ * threads). This is supposed to be updated ++ * with @reg_lock held before allocating ++ * the backing pages. This helps ensure that ++ * total physical memory usage for just in ++ * time memory allocation remains within the ++ * @jit_phys_pages_limit in multi-threaded ++ * scenarios. ++ * @jit_active_head: List containing the just-in-time memory allocations ++ * which are in use. ++ * @jit_pool_head: List containing the just-in-time memory allocations ++ * which have been freed up by userspace and so not being ++ * used by them. ++ * Driver caches them to quickly fulfill requests for new ++ * JIT allocations. They are released in case of memory ++ * pressure as they are put on the @evict_list when they ++ * are freed up by userspace. ++ * @jit_destroy_head: List containing the just-in-time memory allocations ++ * which were moved to it from @jit_pool_head, in the ++ * shrinker callback, after freeing their backing ++ * physical pages. ++ * @jit_evict_lock: Lock used for operations done on just-in-time memory ++ * allocations and also for accessing @evict_list. ++ * @jit_work: Work item queued to defer the freeing of a memory ++ * region when a just-in-time memory allocation is moved ++ * to @jit_destroy_head. ++ * @ext_res_meta_head: A list of sticky external resources which were requested to ++ * be mapped on GPU side, through a softjob atom of type ++ * EXT_RES_MAP or STICKY_RESOURCE_MAP ioctl. ++ * @age_count: Counter incremented on every call to jd_submit_atom, ++ * atom is assigned the snapshot of this counter, which ++ * is used to determine the atom's age when it is added to ++ * the runnable RB-tree. ++ * @trim_level: Level of JIT allocation trimming to perform on free (0-100%) ++ * @kprcs: Reference to @struct kbase_process that the current ++ * kbase_context belongs to. ++ * @kprcs_link: List link for the list of kbase context maintained ++ * under kbase_process. ++ * @gwt_enabled: Indicates if tracking of GPU writes is enabled, protected by ++ * kbase_context.reg_lock. ++ * @gwt_was_enabled: Simple sticky bit flag to know if GWT was ever enabled. ++ * @gwt_current_list: A list of addresses for which GPU has generated write faults, ++ * after the last snapshot of it was sent to userspace. ++ * @gwt_snapshot_list: Snapshot of the @gwt_current_list for sending to user space. ++ * @priority: Indicates the context priority. Used along with @atoms_count ++ * for context scheduling, protected by hwaccess_lock. ++ * @atoms_count: Number of GPU atoms currently in use, per priority ++ * @create_flags: Flags used in context creation. ++ * @kinstr_jm: Kernel job manager instrumentation context handle ++ * ++ * A kernel base context is an entity among which the GPU is scheduled. ++ * Each context has its own GPU address space. ++ * Up to one context can be created for each client that opens the device file ++ * /dev/malixx. Context creation is deferred until a special ioctl() system call ++ * is made on the device file. ++ */ ++struct kbase_context { ++ struct file *filp; ++ struct kbase_device *kbdev; ++ struct list_head kctx_list_link; ++ struct kbase_mmu_table mmu; ++ ++ u32 id; ++ unsigned long api_version; ++ struct list_head event_list; ++ struct list_head event_coalesce_list; ++ struct mutex event_mutex; ++#if !MALI_USE_CSF ++ atomic_t event_closed; ++#endif ++ struct workqueue_struct *event_workq; ++ atomic_t event_count; ++ int event_coalesce_count; ++ ++ atomic_t flags; ++ ++ struct tagged_addr aliasing_sink_page; ++ ++ spinlock_t mem_partials_lock; ++ struct list_head mem_partials; ++ ++ struct mutex reg_lock; ++ ++ struct rb_root reg_rbtree_same; ++ struct rb_root reg_rbtree_custom; ++ struct rb_root reg_rbtree_exec; ++ ++#if MALI_USE_CSF ++ struct kbase_csf_context csf; ++#else ++ struct kbase_jd_context jctx; ++ struct jsctx_queue jsctx_queue ++ [KBASE_JS_ATOM_SCHED_PRIO_COUNT][BASE_JM_MAX_NR_SLOTS]; ++ ++ struct list_head completed_jobs; ++ atomic_t work_count; ++ struct timer_list soft_job_timeout; ++ ++ atomic_t atoms_pulled; ++ atomic_t atoms_pulled_slot[BASE_JM_MAX_NR_SLOTS]; ++ int atoms_pulled_slot_pri[BASE_JM_MAX_NR_SLOTS][ ++ KBASE_JS_ATOM_SCHED_PRIO_COUNT]; ++ int priority; ++ bool blocked_js[BASE_JM_MAX_NR_SLOTS][KBASE_JS_ATOM_SCHED_PRIO_COUNT]; ++ s16 atoms_count[KBASE_JS_ATOM_SCHED_PRIO_COUNT]; ++ u32 slots_pullable; ++ u32 age_count; ++#endif /* MALI_USE_CSF */ ++ ++ DECLARE_BITMAP(cookies, BITS_PER_LONG); ++ struct kbase_va_region *pending_regions[BITS_PER_LONG]; ++ ++ wait_queue_head_t event_queue; ++ pid_t tgid; ++ pid_t pid; ++ atomic_t used_pages; ++ atomic_t nonmapped_pages; ++ atomic_t permanent_mapped_pages; ++ ++ struct kbase_mem_pool_group mem_pools; ++ ++ struct shrinker reclaim; ++ struct list_head evict_list; ++ ++ struct list_head waiting_soft_jobs; ++ spinlock_t waiting_soft_jobs_lock; ++#ifdef CONFIG_MALI_BIFROST_DMA_FENCE ++ struct { ++ struct list_head waiting_resource; ++ struct workqueue_struct *wq; ++ } dma_fence; ++#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ ++ ++ int as_nr; ++ ++ atomic_t refcount; ++ ++ spinlock_t mm_update_lock; ++ struct mm_struct __rcu *process_mm; ++ u64 same_va_end; ++ u64 exec_va_start; ++ u64 gpu_va_end; ++ bool jit_va; ++ ++#ifdef CONFIG_DEBUG_FS ++ char *mem_profile_data; ++ size_t mem_profile_size; ++ struct mutex mem_profile_lock; ++ struct dentry *kctx_dentry; ++ ++ unsigned int *reg_dump; ++ atomic_t job_fault_count; ++ struct list_head job_fault_resume_event_list; ++ ++#endif /* CONFIG_DEBUG_FS */ ++ ++ struct kbase_hwcnt_legacy_client *legacy_hwcnt_cli; ++ struct mutex legacy_hwcnt_lock; ++ ++ struct kbase_va_region *jit_alloc[1 + BASE_JIT_ALLOC_COUNT]; ++ u8 jit_max_allocations; ++ u8 jit_current_allocations; ++ u8 jit_current_allocations_per_bin[256]; ++ u8 jit_version; ++ u8 jit_group_id; ++#if MALI_JIT_PRESSURE_LIMIT_BASE ++ u64 jit_phys_pages_limit; ++ u64 jit_current_phys_pressure; ++ u64 jit_phys_pages_to_be_allocated; ++#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ ++ struct list_head jit_active_head; ++ struct list_head jit_pool_head; ++ struct list_head jit_destroy_head; ++ struct mutex jit_evict_lock; ++ struct work_struct jit_work; ++ ++ struct list_head ext_res_meta_head; ++ ++ u8 trim_level; ++ ++ struct kbase_process *kprcs; ++ struct list_head kprcs_link; ++ ++#ifdef CONFIG_MALI_CINSTR_GWT ++ bool gwt_enabled; ++ bool gwt_was_enabled; ++ struct list_head gwt_current_list; ++ struct list_head gwt_snapshot_list; ++#endif ++ ++ base_context_create_flags create_flags; ++ ++#if !MALI_USE_CSF ++ struct kbase_kinstr_jm *kinstr_jm; ++#endif ++}; ++ ++#ifdef CONFIG_MALI_CINSTR_GWT ++/** ++ * struct kbasep_gwt_list_element - Structure used to collect GPU ++ * write faults. ++ * @link: List head for adding write faults. ++ * @region: Details of the region where we have the ++ * faulting page address. ++ * @page_addr: Page address where GPU write fault occurred. ++ * @num_pages: The number of pages modified. ++ * ++ * Using this structure all GPU write faults are stored in a list. ++ */ ++struct kbasep_gwt_list_element { ++ struct list_head link; ++ struct kbase_va_region *region; ++ u64 page_addr; ++ u64 num_pages; ++}; ++ ++#endif ++ ++/** ++ * struct kbase_ctx_ext_res_meta - Structure which binds an external resource ++ * to a @kbase_context. ++ * @ext_res_node: List head for adding the metadata to a ++ * @kbase_context. ++ * @alloc: The physical memory allocation structure ++ * which is mapped. ++ * @gpu_addr: The GPU virtual address the resource is ++ * mapped to. ++ * @ref: Reference count. ++ * ++ * External resources can be mapped into multiple contexts as well as the same ++ * context multiple times. ++ * As kbase_va_region itself isn't refcounted we can't attach our extra ++ * information to it as it could be removed under our feet leaving external ++ * resources pinned. ++ * This metadata structure binds a single external resource to a single ++ * context, ensuring that per context mapping is tracked separately so it can ++ * be overridden when needed and abuses by the application (freeing the resource ++ * multiple times) don't effect the refcount of the physical allocation. ++ */ ++struct kbase_ctx_ext_res_meta { ++ struct list_head ext_res_node; ++ struct kbase_mem_phy_alloc *alloc; ++ u64 gpu_addr; ++ u32 ref; ++}; ++ ++enum kbase_reg_access_type { ++ REG_READ, ++ REG_WRITE ++}; ++ ++enum kbase_share_attr_bits { ++ /* (1ULL << 8) bit is reserved */ ++ SHARE_BOTH_BITS = (2ULL << 8), /* inner and outer shareable coherency */ ++ SHARE_INNER_BITS = (3ULL << 8) /* inner shareable coherency */ ++}; ++ ++/** ++ * kbase_device_is_cpu_coherent - Returns if the device is CPU coherent. ++ * @kbdev: kbase device ++ * ++ * Return: true if the device access are coherent, false if not. ++ */ ++static inline bool kbase_device_is_cpu_coherent(struct kbase_device *kbdev) ++{ ++ if ((kbdev->system_coherency == COHERENCY_ACE_LITE) || ++ (kbdev->system_coherency == COHERENCY_ACE)) ++ return true; ++ ++ return false; ++} ++ ++/* Conversion helpers for setting up high resolution timers */ ++#define HR_TIMER_DELAY_MSEC(x) (ns_to_ktime(((u64)(x))*1000000U)) ++#define HR_TIMER_DELAY_NSEC(x) (ns_to_ktime(x)) ++ ++/* Maximum number of loops polling the GPU for a cache flush before we assume it must have completed */ ++#define KBASE_CLEAN_CACHE_MAX_LOOPS 100000 ++/* Maximum number of loops polling the GPU for an AS command to complete before we assume the GPU has hung */ ++#define KBASE_AS_INACTIVE_MAX_LOOPS 100000000 ++ ++/* JobDescriptorHeader - taken from the architecture specifications, the layout ++ * is currently identical for all GPU archs. */ ++struct job_descriptor_header { ++ u32 exception_status; ++ u32 first_incomplete_task; ++ u64 fault_pointer; ++ u8 job_descriptor_size : 1; ++ u8 job_type : 7; ++ u8 job_barrier : 1; ++ u8 _reserved_01 : 1; ++ u8 _reserved_1 : 1; ++ u8 _reserved_02 : 1; ++ u8 _reserved_03 : 1; ++ u8 _reserved_2 : 1; ++ u8 _reserved_04 : 1; ++ u8 _reserved_05 : 1; ++ u16 job_index; ++ u16 job_dependency_index_1; ++ u16 job_dependency_index_2; ++ union { ++ u64 _64; ++ u32 _32; ++ } next_job; ++}; ++ ++#endif /* _KBASE_DEFS_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_disjoint_events.c b/drivers/gpu/arm/bifrost/mali_kbase_disjoint_events.c +new file mode 100755 +index 000000000000..b5ac414b1223 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_disjoint_events.c +@@ -0,0 +1,81 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014, 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* ++ * Base kernel disjoint events helper functions ++ */ ++ ++#include ++ ++void kbase_disjoint_init(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ atomic_set(&kbdev->disjoint_event.count, 0); ++ atomic_set(&kbdev->disjoint_event.state, 0); ++} ++ ++/* increment the disjoint event count */ ++void kbase_disjoint_event(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ atomic_inc(&kbdev->disjoint_event.count); ++} ++ ++/* increment the state and the event counter */ ++void kbase_disjoint_state_up(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ atomic_inc(&kbdev->disjoint_event.state); ++ ++ kbase_disjoint_event(kbdev); ++} ++ ++/* decrement the state */ ++void kbase_disjoint_state_down(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(atomic_read(&kbdev->disjoint_event.state) > 0); ++ ++ kbase_disjoint_event(kbdev); ++ ++ atomic_dec(&kbdev->disjoint_event.state); ++} ++ ++/* increments the count only if the state is > 0 */ ++void kbase_disjoint_event_potential(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ if (atomic_read(&kbdev->disjoint_event.state)) ++ kbase_disjoint_event(kbdev); ++} ++ ++u32 kbase_disjoint_event_get(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ return atomic_read(&kbdev->disjoint_event.count); ++} ++KBASE_EXPORT_TEST_API(kbase_disjoint_event_get); +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_dma_fence.c b/drivers/gpu/arm/bifrost/mali_kbase_dma_fence.c +new file mode 100755 +index 000000000000..1fac5e3e68f1 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_dma_fence.c +@@ -0,0 +1,473 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2016, 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* Include mali_kbase_dma_fence.h before checking for CONFIG_MALI_BIFROST_DMA_FENCE as ++ * it will be set there. ++ */ ++#include "mali_kbase_dma_fence.h" ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++static void ++kbase_dma_fence_work(struct work_struct *pwork); ++ ++static void ++kbase_dma_fence_waiters_add(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ ++ list_add_tail(&katom->queue, &kctx->dma_fence.waiting_resource); ++} ++ ++static void ++kbase_dma_fence_waiters_remove(struct kbase_jd_atom *katom) ++{ ++ list_del(&katom->queue); ++} ++ ++static int ++kbase_dma_fence_lock_reservations(struct kbase_dma_fence_resv_info *info, ++ struct ww_acquire_ctx *ctx) ++{ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)) ++ struct reservation_object *content_res = NULL; ++#else ++ struct dma_resv *content_res = NULL; ++#endif ++ unsigned int content_res_idx = 0; ++ unsigned int r; ++ int err = 0; ++ ++ ww_acquire_init(ctx, &reservation_ww_class); ++ ++retry: ++ for (r = 0; r < info->dma_fence_resv_count; r++) { ++ if (info->resv_objs[r] == content_res) { ++ content_res = NULL; ++ continue; ++ } ++ ++ err = ww_mutex_lock(&info->resv_objs[r]->lock, ctx); ++ if (err) ++ goto error; ++ } ++ ++ ww_acquire_done(ctx); ++ return err; ++ ++error: ++ content_res_idx = r; ++ ++ /* Unlock the locked one ones */ ++ while (r--) ++ ww_mutex_unlock(&info->resv_objs[r]->lock); ++ ++ if (content_res) ++ ww_mutex_unlock(&content_res->lock); ++ ++ /* If we deadlock try with lock_slow and retry */ ++ if (err == -EDEADLK) { ++ content_res = info->resv_objs[content_res_idx]; ++ ww_mutex_lock_slow(&content_res->lock, ctx); ++ goto retry; ++ } ++ ++ /* If we are here the function failed */ ++ ww_acquire_fini(ctx); ++ return err; ++} ++ ++static void ++kbase_dma_fence_unlock_reservations(struct kbase_dma_fence_resv_info *info, ++ struct ww_acquire_ctx *ctx) ++{ ++ unsigned int r; ++ ++ for (r = 0; r < info->dma_fence_resv_count; r++) ++ ww_mutex_unlock(&info->resv_objs[r]->lock); ++ ww_acquire_fini(ctx); ++} ++ ++ ++ ++/** ++ * kbase_dma_fence_queue_work() - Queue work to handle @katom ++ * @katom: Pointer to atom for which to queue work ++ * ++ * Queue kbase_dma_fence_work() for @katom to clean up the fence callbacks and ++ * submit the atom. ++ */ ++static void ++kbase_dma_fence_queue_work(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ bool ret; ++ ++ INIT_WORK(&katom->work, kbase_dma_fence_work); ++ ret = queue_work(kctx->dma_fence.wq, &katom->work); ++ /* Warn if work was already queued, that should not happen. */ ++ WARN_ON(!ret); ++} ++ ++/** ++ * kbase_dma_fence_cancel_atom() - Cancels waiting on an atom ++ * @katom: Katom to cancel ++ * ++ * Locking: katom->dma_fence.callbacks list assumes jctx.lock is held. ++ */ ++static void ++kbase_dma_fence_cancel_atom(struct kbase_jd_atom *katom) ++{ ++ lockdep_assert_held(&katom->kctx->jctx.lock); ++ ++ /* Cancel callbacks and clean up. */ ++ kbase_fence_free_callbacks(katom); ++ ++ /* Mark the atom as handled in case all fences signaled just before ++ * canceling the callbacks and the worker was queued. ++ */ ++ kbase_fence_dep_count_set(katom, -1); ++ ++ /* Prevent job_done_nolock from being called twice on an atom when ++ * there is a race between job completion and cancellation. ++ */ ++ ++ if (katom->status == KBASE_JD_ATOM_STATE_QUEUED) { ++ /* Wait was cancelled - zap the atom */ ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ if (jd_done_nolock(katom, NULL)) ++ kbase_js_sched_all(katom->kctx->kbdev); ++ } ++} ++ ++/** ++ * kbase_dma_fence_work() - Worker thread called when a fence is signaled ++ * @pwork: work_struct containing a pointer to a katom ++ * ++ * This function will clean and mark all dependencies as satisfied ++ */ ++static void ++kbase_dma_fence_work(struct work_struct *pwork) ++{ ++ struct kbase_jd_atom *katom; ++ struct kbase_jd_context *ctx; ++ ++ katom = container_of(pwork, struct kbase_jd_atom, work); ++ ctx = &katom->kctx->jctx; ++ ++ mutex_lock(&ctx->lock); ++ if (kbase_fence_dep_count_read(katom) != 0) ++ goto out; ++ ++ kbase_fence_dep_count_set(katom, -1); ++ ++ /* Remove atom from list of dma-fence waiting atoms. */ ++ kbase_dma_fence_waiters_remove(katom); ++ /* Cleanup callbacks. */ ++ kbase_fence_free_callbacks(katom); ++ /* ++ * Queue atom on GPU, unless it has already completed due to a failing ++ * dependency. Run jd_done_nolock() on the katom if it is completed. ++ */ ++ if (unlikely(katom->status == KBASE_JD_ATOM_STATE_COMPLETED)) ++ jd_done_nolock(katom, NULL); ++ else ++ kbase_jd_dep_clear_locked(katom); ++ ++out: ++ mutex_unlock(&ctx->lock); ++} ++ ++static void ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++kbase_dma_fence_cb(struct fence *fence, struct fence_cb *cb) ++#else ++kbase_dma_fence_cb(struct dma_fence *fence, struct dma_fence_cb *cb) ++#endif ++{ ++ struct kbase_fence_cb *kcb = container_of(cb, ++ struct kbase_fence_cb, ++ fence_cb); ++ struct kbase_jd_atom *katom = kcb->katom; ++ ++ /* If the atom is zapped dep_count will be forced to a negative number ++ * preventing this callback from ever scheduling work. Which in turn ++ * would reschedule the atom. ++ */ ++ ++ if (kbase_fence_dep_count_dec_and_test(katom)) ++ kbase_dma_fence_queue_work(katom); ++} ++ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)) ++static int ++kbase_dma_fence_add_reservation_callback(struct kbase_jd_atom *katom, ++ struct reservation_object *resv, ++ bool exclusive) ++#else ++static int ++kbase_dma_fence_add_reservation_callback(struct kbase_jd_atom *katom, ++ struct dma_resv *resv, ++ bool exclusive) ++#endif ++{ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ struct fence *excl_fence = NULL; ++ struct fence **shared_fences = NULL; ++#else ++ struct dma_fence *excl_fence = NULL; ++ struct dma_fence **shared_fences = NULL; ++#endif ++ unsigned int shared_count = 0; ++ int err, i; ++ ++ err = reservation_object_get_fences_rcu(resv, ++ &excl_fence, ++ &shared_count, ++ &shared_fences); ++ if (err) ++ return err; ++ ++ if (excl_fence) { ++ err = kbase_fence_add_callback(katom, ++ excl_fence, ++ kbase_dma_fence_cb); ++ ++ /* Release our reference, taken by reservation_object_get_fences_rcu(), ++ * to the fence. We have set up our callback (if that was possible), ++ * and it's the fence's owner is responsible for singling the fence ++ * before allowing it to disappear. ++ */ ++ dma_fence_put(excl_fence); ++ ++ if (err) ++ goto out; ++ } ++ ++ if (exclusive) { ++ for (i = 0; i < shared_count; i++) { ++ err = kbase_fence_add_callback(katom, ++ shared_fences[i], ++ kbase_dma_fence_cb); ++ if (err) ++ goto out; ++ } ++ } ++ ++ /* Release all our references to the shared fences, taken by ++ * reservation_object_get_fences_rcu(). We have set up our callback (if ++ * that was possible), and it's the fence's owner is responsible for ++ * signaling the fence before allowing it to disappear. ++ */ ++out: ++ for (i = 0; i < shared_count; i++) ++ dma_fence_put(shared_fences[i]); ++ kfree(shared_fences); ++ ++ if (err) { ++ /* ++ * On error, cancel and clean up all callbacks that was set up ++ * before the error. ++ */ ++ kbase_fence_free_callbacks(katom); ++ } ++ ++ return err; ++} ++ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)) ++void kbase_dma_fence_add_reservation(struct reservation_object *resv, ++ struct kbase_dma_fence_resv_info *info, ++ bool exclusive) ++#else ++void kbase_dma_fence_add_reservation(struct dma_resv *resv, ++ struct kbase_dma_fence_resv_info *info, ++ bool exclusive) ++#endif ++{ ++ unsigned int i; ++ ++ for (i = 0; i < info->dma_fence_resv_count; i++) { ++ /* Duplicate resource, ignore */ ++ if (info->resv_objs[i] == resv) ++ return; ++ } ++ ++ info->resv_objs[info->dma_fence_resv_count] = resv; ++ if (exclusive) ++ set_bit(info->dma_fence_resv_count, ++ info->dma_fence_excl_bitmap); ++ (info->dma_fence_resv_count)++; ++} ++ ++int kbase_dma_fence_wait(struct kbase_jd_atom *katom, ++ struct kbase_dma_fence_resv_info *info) ++{ ++ int err, i; ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ struct fence *fence; ++#else ++ struct dma_fence *fence; ++#endif ++ struct ww_acquire_ctx ww_ctx; ++ ++ lockdep_assert_held(&katom->kctx->jctx.lock); ++ ++ fence = kbase_fence_out_new(katom); ++ if (!fence) { ++ err = -ENOMEM; ++ dev_err(katom->kctx->kbdev->dev, ++ "Error %d creating fence.\n", err); ++ return err; ++ } ++ ++ kbase_fence_dep_count_set(katom, 1); ++ ++ err = kbase_dma_fence_lock_reservations(info, &ww_ctx); ++ if (err) { ++ dev_err(katom->kctx->kbdev->dev, ++ "Error %d locking reservations.\n", err); ++ kbase_fence_dep_count_set(katom, -1); ++ kbase_fence_out_remove(katom); ++ return err; ++ } ++ ++ for (i = 0; i < info->dma_fence_resv_count; i++) { ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)) ++ struct reservation_object *obj = info->resv_objs[i]; ++#else ++ struct dma_resv *obj = info->resv_objs[i]; ++#endif ++ if (!test_bit(i, info->dma_fence_excl_bitmap)) { ++ err = reservation_object_reserve_shared(obj); ++ if (err) { ++ dev_err(katom->kctx->kbdev->dev, ++ "Error %d reserving space for shared fence.\n", err); ++ goto end; ++ } ++ ++ err = kbase_dma_fence_add_reservation_callback(katom, obj, false); ++ if (err) { ++ dev_err(katom->kctx->kbdev->dev, ++ "Error %d adding reservation to callback.\n", err); ++ goto end; ++ } ++ ++ reservation_object_add_shared_fence(obj, fence); ++ } else { ++ err = kbase_dma_fence_add_reservation_callback(katom, obj, true); ++ if (err) { ++ dev_err(katom->kctx->kbdev->dev, ++ "Error %d adding reservation to callback.\n", err); ++ goto end; ++ } ++ ++ reservation_object_add_excl_fence(obj, fence); ++ } ++ } ++ ++end: ++ kbase_dma_fence_unlock_reservations(info, &ww_ctx); ++ ++ if (likely(!err)) { ++ /* Test if the callbacks are already triggered */ ++ if (kbase_fence_dep_count_dec_and_test(katom)) { ++ kbase_fence_dep_count_set(katom, -1); ++ kbase_fence_free_callbacks(katom); ++ } else { ++ /* Add katom to the list of dma-buf fence waiting atoms ++ * only if it is still waiting. ++ */ ++ kbase_dma_fence_waiters_add(katom); ++ } ++ } else { ++ /* There was an error, cancel callbacks, set dep_count to -1 to ++ * indicate that the atom has been handled (the caller will ++ * kill it for us), signal the fence, free callbacks and the ++ * fence. ++ */ ++ kbase_fence_free_callbacks(katom); ++ kbase_fence_dep_count_set(katom, -1); ++ kbase_dma_fence_signal(katom); ++ } ++ ++ return err; ++} ++ ++void kbase_dma_fence_cancel_all_atoms(struct kbase_context *kctx) ++{ ++ struct list_head *list = &kctx->dma_fence.waiting_resource; ++ ++ while (!list_empty(list)) { ++ struct kbase_jd_atom *katom; ++ ++ katom = list_first_entry(list, struct kbase_jd_atom, queue); ++ kbase_dma_fence_waiters_remove(katom); ++ kbase_dma_fence_cancel_atom(katom); ++ } ++} ++ ++void kbase_dma_fence_cancel_callbacks(struct kbase_jd_atom *katom) ++{ ++ /* Cancel callbacks and clean up. */ ++ if (kbase_fence_free_callbacks(katom)) ++ kbase_dma_fence_queue_work(katom); ++} ++ ++void kbase_dma_fence_signal(struct kbase_jd_atom *katom) ++{ ++ if (!katom->dma_fence.fence) ++ return; ++ ++ /* Signal the atom's fence. */ ++ dma_fence_signal(katom->dma_fence.fence); ++ ++ kbase_fence_out_remove(katom); ++ ++ kbase_fence_free_callbacks(katom); ++} ++ ++void kbase_dma_fence_term(struct kbase_context *kctx) ++{ ++ destroy_workqueue(kctx->dma_fence.wq); ++ kctx->dma_fence.wq = NULL; ++} ++ ++int kbase_dma_fence_init(struct kbase_context *kctx) ++{ ++ INIT_LIST_HEAD(&kctx->dma_fence.waiting_resource); ++ ++ kctx->dma_fence.wq = alloc_workqueue("mali-fence-%d", ++ WQ_UNBOUND, 1, kctx->pid); ++ if (!kctx->dma_fence.wq) ++ return -ENOMEM; ++ ++ return 0; ++} +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_dma_fence.h b/drivers/gpu/arm/bifrost/mali_kbase_dma_fence.h +new file mode 100755 +index 000000000000..3ac8186328a1 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_dma_fence.h +@@ -0,0 +1,144 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2016, 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_DMA_FENCE_H_ ++#define _KBASE_DMA_FENCE_H_ ++ ++#ifdef CONFIG_MALI_BIFROST_DMA_FENCE ++ ++#include ++#include ++#include ++ ++/* Forward declaration from mali_kbase_defs.h */ ++struct kbase_jd_atom; ++struct kbase_context; ++ ++/** ++ * struct kbase_dma_fence_resv_info - Structure with list of reservation objects ++ * @resv_objs: Array of reservation objects to attach the ++ * new fence to. ++ * @dma_fence_resv_count: Number of reservation objects in the array. ++ * @dma_fence_excl_bitmap: Specifies which resv_obj are exclusive. ++ * ++ * This is used by some functions to pass around a collection of data about ++ * reservation objects. ++ */ ++struct kbase_dma_fence_resv_info { ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)) ++ struct reservation_object **resv_objs; ++#else ++ struct dma_resv **resv_objs; ++#endif ++ unsigned int dma_fence_resv_count; ++ unsigned long *dma_fence_excl_bitmap; ++}; ++ ++/** ++ * kbase_dma_fence_add_reservation() - Adds a resv to the array of resv_objs ++ * @resv: Reservation object to add to the array. ++ * @info: Pointer to struct with current reservation info ++ * @exclusive: Boolean indicating if exclusive access is needed ++ * ++ * The function adds a new reservation_object to an existing array of ++ * reservation_objects. At the same time keeps track of which objects require ++ * exclusive access in dma_fence_excl_bitmap. ++ */ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)) ++void kbase_dma_fence_add_reservation(struct reservation_object *resv, ++ struct kbase_dma_fence_resv_info *info, ++ bool exclusive); ++#else ++void kbase_dma_fence_add_reservation(struct dma_resv *resv, ++ struct kbase_dma_fence_resv_info *info, ++ bool exclusive); ++#endif ++ ++/** ++ * kbase_dma_fence_wait() - Creates a new fence and attaches it to the resv_objs ++ * @katom: Katom with the external dependency. ++ * @info: Pointer to struct with current reservation info ++ * ++ * Return: An error code or 0 if succeeds ++ */ ++int kbase_dma_fence_wait(struct kbase_jd_atom *katom, ++ struct kbase_dma_fence_resv_info *info); ++ ++/** ++ * kbase_dma_fence_cancel_ctx() - Cancel all dma-fences blocked atoms on kctx ++ * @kctx: Pointer to kbase context ++ * ++ * This function will cancel and clean up all katoms on @kctx that is waiting ++ * on dma-buf fences. ++ * ++ * Locking: jctx.lock needs to be held when calling this function. ++ */ ++void kbase_dma_fence_cancel_all_atoms(struct kbase_context *kctx); ++ ++/** ++ * kbase_dma_fence_cancel_callbacks() - Cancel only callbacks on katom ++ * @katom: Pointer to katom whose callbacks are to be canceled ++ * ++ * This function cancels all dma-buf fence callbacks on @katom, but does not ++ * cancel the katom itself. ++ * ++ * The caller is responsible for ensuring that jd_done_nolock is called on ++ * @katom. ++ * ++ * Locking: jctx.lock must be held when calling this function. ++ */ ++void kbase_dma_fence_cancel_callbacks(struct kbase_jd_atom *katom); ++ ++/** ++ * kbase_dma_fence_signal() - Signal katom's fence and clean up after wait ++ * @katom: Pointer to katom to signal and clean up ++ * ++ * This function will signal the @katom's fence, if it has one, and clean up ++ * the callback data from the katom's wait on earlier fences. ++ * ++ * Locking: jctx.lock must be held while calling this function. ++ */ ++void kbase_dma_fence_signal(struct kbase_jd_atom *katom); ++ ++/** ++ * kbase_dma_fence_term() - Terminate Mali dma-fence context ++ * @kctx: kbase context to terminate ++ */ ++void kbase_dma_fence_term(struct kbase_context *kctx); ++ ++/** ++ * kbase_dma_fence_init() - Initialize Mali dma-fence context ++ * @kctx: kbase context to initialize ++ */ ++int kbase_dma_fence_init(struct kbase_context *kctx); ++ ++#else /* CONFIG_MALI_BIFROST_DMA_FENCE */ ++/* Dummy functions for when dma-buf fence isn't enabled. */ ++ ++static inline int kbase_dma_fence_init(struct kbase_context *kctx) ++{ ++ return 0; ++} ++ ++static inline void kbase_dma_fence_term(struct kbase_context *kctx) {} ++#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ ++#endif +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_dummy_job_wa.c b/drivers/gpu/arm/bifrost/mali_kbase_dummy_job_wa.c +new file mode 100755 +index 000000000000..a5a7ad744a8e +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_dummy_job_wa.c +@@ -0,0 +1,442 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* ++ * Implementation of the dummy job execution workaround for the GPU hang issue. ++ */ ++ ++#include ++#include ++#include ++ ++#include ++#include ++ ++#define DUMMY_JOB_WA_BINARY_NAME "valhall-1691526.wa" ++ ++struct wa_header { ++ u16 signature; ++ u16 version; ++ u32 info_offset; ++} __packed; ++ ++struct wa_v2_info { ++ u64 jc; ++ u32 js; ++ u32 blob_offset; ++ u64 flags; ++} __packed; ++ ++struct wa_blob { ++ u64 base; ++ u32 size; ++ u32 map_flags; ++ u32 payload_offset; ++ u32 blob_offset; ++} __packed; ++ ++static bool in_range(const u8 *base, const u8 *end, off_t off, size_t sz) ++{ ++ return !(end - base - off < sz); ++} ++ ++static u32 wait_any(struct kbase_device *kbdev, off_t offset, u32 bits) ++{ ++ int loop; ++ const int timeout = 100; ++ u32 val; ++ ++ for (loop = 0; loop < timeout; loop++) { ++ val = kbase_reg_read(kbdev, offset); ++ if (val & bits) ++ break; ++ udelay(10); ++ } ++ ++ if (loop == timeout) { ++ dev_err(kbdev->dev, ++ "Timeout reading register 0x%lx, bits 0x%lx, last read was 0x%lx\n", ++ (unsigned long)offset, (unsigned long)bits, ++ (unsigned long)val); ++ } ++ ++ return (val & bits); ++} ++ ++static int wait(struct kbase_device *kbdev, off_t offset, u32 bits, bool set) ++{ ++ int loop; ++ const int timeout = 100; ++ u32 val; ++ u32 target = 0; ++ ++ if (set) ++ target = bits; ++ ++ for (loop = 0; loop < timeout; loop++) { ++ val = kbase_reg_read(kbdev, (offset)); ++ if ((val & bits) == target) ++ break; ++ ++ udelay(10); ++ } ++ ++ if (loop == timeout) { ++ dev_err(kbdev->dev, ++ "Timeout reading register 0x%lx, bits 0x%lx, last read was 0x%lx\n", ++ (unsigned long)offset, (unsigned long)bits, ++ (unsigned long)val); ++ return -ETIMEDOUT; ++ } ++ ++ return 0; ++} ++ ++static inline int run_job(struct kbase_device *kbdev, int as, int slot, ++ u64 cores, u64 jc) ++{ ++ u32 done; ++ ++ /* setup job */ ++ kbase_reg_write(kbdev, JOB_SLOT_REG(slot, JS_HEAD_NEXT_LO), ++ jc & U32_MAX); ++ kbase_reg_write(kbdev, JOB_SLOT_REG(slot, JS_HEAD_NEXT_HI), ++ jc >> 32); ++ kbase_reg_write(kbdev, JOB_SLOT_REG(slot, JS_AFFINITY_NEXT_LO), ++ cores & U32_MAX); ++ kbase_reg_write(kbdev, JOB_SLOT_REG(slot, JS_AFFINITY_NEXT_HI), ++ cores >> 32); ++ kbase_reg_write(kbdev, JOB_SLOT_REG(slot, JS_CONFIG_NEXT), ++ JS_CONFIG_DISABLE_DESCRIPTOR_WR_BK | as); ++ ++ /* go */ ++ kbase_reg_write(kbdev, JOB_SLOT_REG(slot, JS_COMMAND_NEXT), ++ JS_COMMAND_START); ++ ++ /* wait for the slot to finish (done, error) */ ++ done = wait_any(kbdev, JOB_CONTROL_REG(JOB_IRQ_RAWSTAT), ++ (1ul << (16+slot)) | (1ul << slot)); ++ kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), done); ++ ++ if (done != (1ul << slot)) { ++ dev_err(kbdev->dev, ++ "Failed to run WA job on slot %d cores 0x%llx: done 0x%lx\n", ++ slot, (unsigned long long)cores, ++ (unsigned long)done); ++ dev_err(kbdev->dev, "JS_STATUS on failure: 0x%x\n", ++ kbase_reg_read(kbdev, JOB_SLOT_REG(slot, JS_STATUS))); ++ ++ return -EFAULT; ++ } else { ++ return 0; ++ } ++} ++ ++/* To be called after power up & MMU init, but before everything else */ ++int kbase_dummy_job_wa_execute(struct kbase_device *kbdev, u64 cores) ++{ ++ int as; ++ int slot; ++ u64 jc; ++ int failed = 0; ++ int runs = 0; ++ u32 old_gpu_mask; ++ u32 old_job_mask; ++ ++ if (!kbdev) ++ return -EFAULT; ++ ++ if (!kbdev->dummy_job_wa.ctx) ++ return -EFAULT; ++ ++ as = kbdev->dummy_job_wa.ctx->as_nr; ++ slot = kbdev->dummy_job_wa.slot; ++ jc = kbdev->dummy_job_wa.jc; ++ ++ /* mask off all but MMU IRQs */ ++ old_gpu_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK)); ++ old_job_mask = kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK)); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), 0); ++ kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), 0); ++ ++ /* power up requested cores */ ++ kbase_reg_write(kbdev, SHADER_PWRON_LO, (cores & U32_MAX)); ++ kbase_reg_write(kbdev, SHADER_PWRON_HI, (cores >> 32)); ++ ++ if (kbdev->dummy_job_wa.flags & KBASE_DUMMY_JOB_WA_FLAG_WAIT_POWERUP) { ++ /* wait for power-ups */ ++ wait(kbdev, SHADER_READY_LO, (cores & U32_MAX), true); ++ if (cores >> 32) ++ wait(kbdev, SHADER_READY_HI, (cores >> 32), true); ++ } ++ ++ if (kbdev->dummy_job_wa.flags & KBASE_DUMMY_JOB_WA_FLAG_SERIALIZE) { ++ int i; ++ ++ /* do for each requested core */ ++ for (i = 0; i < sizeof(cores) * 8; i++) { ++ u64 affinity; ++ ++ affinity = 1ull << i; ++ ++ if (!(cores & affinity)) ++ continue; ++ ++ if (run_job(kbdev, as, slot, affinity, jc)) ++ failed++; ++ runs++; ++ } ++ ++ } else { ++ if (run_job(kbdev, as, slot, cores, jc)) ++ failed++; ++ runs++; ++ } ++ ++ if (kbdev->dummy_job_wa.flags & ++ KBASE_DUMMY_JOB_WA_FLAG_LOGICAL_SHADER_POWER) { ++ /* power off shader cores (to reduce any dynamic leakage) */ ++ kbase_reg_write(kbdev, SHADER_PWROFF_LO, (cores & U32_MAX)); ++ kbase_reg_write(kbdev, SHADER_PWROFF_HI, (cores >> 32)); ++ ++ /* wait for power off complete */ ++ wait(kbdev, SHADER_READY_LO, (cores & U32_MAX), false); ++ wait(kbdev, SHADER_PWRTRANS_LO, (cores & U32_MAX), false); ++ if (cores >> 32) { ++ wait(kbdev, SHADER_READY_HI, (cores >> 32), false); ++ wait(kbdev, SHADER_PWRTRANS_HI, (cores >> 32), false); ++ } ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), U32_MAX); ++ } ++ ++ /* restore IRQ masks */ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), old_gpu_mask); ++ kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), old_job_mask); ++ ++ if (failed) ++ dev_err(kbdev->dev, ++ "WA complete with %d failures out of %d runs\n", failed, ++ runs); ++ ++ return failed ? -EFAULT : 0; ++} ++ ++static ssize_t show_dummy_job_wa_info(struct device * const dev, ++ struct device_attribute * const attr, char * const buf) ++{ ++ struct kbase_device *const kbdev = dev_get_drvdata(dev); ++ int err; ++ ++ if (!kbdev || !kbdev->dummy_job_wa.ctx) ++ return -ENODEV; ++ ++ err = scnprintf(buf, PAGE_SIZE, "slot %u flags %llx\n", ++ kbdev->dummy_job_wa.slot, kbdev->dummy_job_wa.flags); ++ ++ return err; ++} ++ ++static DEVICE_ATTR(dummy_job_wa_info, 0444, show_dummy_job_wa_info, NULL); ++ ++static bool wa_blob_load_needed(struct kbase_device *kbdev) ++{ ++ if (of_machine_is_compatible("arm,juno")) ++ return false; ++ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_3485)) ++ return true; ++ ++ return false; ++} ++ ++int kbase_dummy_job_wa_load(struct kbase_device *kbdev) ++{ ++ const struct firmware *firmware; ++ static const char wa_name[] = DUMMY_JOB_WA_BINARY_NAME; ++ const u32 signature = 0x4157; ++ const u32 version = 2; ++ const u8 *fw_end; ++ const u8 *fw; ++ const struct wa_header *header; ++ const struct wa_v2_info *v2_info; ++ u32 blob_offset; ++ int err; ++ struct kbase_context *kctx; ++ ++ if (!wa_blob_load_needed(kbdev)) ++ return 0; ++ ++ /* load the wa */ ++ err = request_firmware(&firmware, wa_name, kbdev->dev); ++ ++ if (err) { ++ dev_err(kbdev->dev, "WA blob missing. Please refer to the Arm Mali DDK Valhall Release Notes, " ++ "Part number DC-06002 or contact support-mali@arm.com - driver probe will be failed"); ++ return -ENODEV; ++ } ++ ++ kctx = kbase_create_context(kbdev, true, ++ BASE_CONTEXT_CREATE_FLAG_NONE, 0, ++ NULL); ++ ++ if (!kctx) { ++ dev_err(kbdev->dev, "Failed to create WA context\n"); ++ goto no_ctx; ++ } ++ ++ fw = firmware->data; ++ fw_end = fw + firmware->size; ++ ++ dev_dbg(kbdev->dev, "Loaded firmware of size %zu bytes\n", ++ firmware->size); ++ ++ if (!in_range(fw, fw_end, 0, sizeof(*header))) { ++ dev_err(kbdev->dev, "WA too small\n"); ++ goto bad_fw; ++ } ++ ++ header = (const struct wa_header *)(fw + 0); ++ ++ if (header->signature != signature) { ++ dev_err(kbdev->dev, "WA signature failure: 0x%lx\n", ++ (unsigned long)header->signature); ++ goto bad_fw; ++ } ++ ++ if (header->version != version) { ++ dev_err(kbdev->dev, "WA version 0x%lx not supported\n", ++ (unsigned long)header->version); ++ goto bad_fw; ++ } ++ ++ if (!in_range(fw, fw_end, header->info_offset, sizeof(*v2_info))) { ++ dev_err(kbdev->dev, "WA info offset out of bounds\n"); ++ goto bad_fw; ++ } ++ ++ v2_info = (const struct wa_v2_info *)(fw + header->info_offset); ++ ++ if (v2_info->flags & ~KBASE_DUMMY_JOB_WA_FLAGS) { ++ dev_err(kbdev->dev, "Unsupported WA flag(s): 0x%llx\n", ++ (unsigned long long)v2_info->flags); ++ goto bad_fw; ++ } ++ ++ kbdev->dummy_job_wa.slot = v2_info->js; ++ kbdev->dummy_job_wa.jc = v2_info->jc; ++ kbdev->dummy_job_wa.flags = v2_info->flags; ++ ++ blob_offset = v2_info->blob_offset; ++ ++ while (blob_offset) { ++ const struct wa_blob *blob; ++ size_t nr_pages; ++ u64 flags; ++ u64 gpu_va; ++ struct kbase_va_region *va_region; ++ ++ if (!in_range(fw, fw_end, blob_offset, sizeof(*blob))) { ++ dev_err(kbdev->dev, "Blob offset out-of-range: 0x%lx\n", ++ (unsigned long)blob_offset); ++ goto bad_fw; ++ } ++ ++ blob = (const struct wa_blob *)(fw + blob_offset); ++ if (!in_range(fw, fw_end, blob->payload_offset, blob->size)) { ++ dev_err(kbdev->dev, "Payload out-of-bounds\n"); ++ goto bad_fw; ++ } ++ ++ gpu_va = blob->base; ++ if (PAGE_ALIGN(gpu_va) != gpu_va) { ++ dev_err(kbdev->dev, "blob not page aligned\n"); ++ goto bad_fw; ++ } ++ nr_pages = PFN_UP(blob->size); ++ flags = blob->map_flags | BASE_MEM_FLAG_MAP_FIXED; ++ ++ va_region = kbase_mem_alloc(kctx, nr_pages, nr_pages, ++ 0, &flags, &gpu_va); ++ ++ if (!va_region) { ++ dev_err(kbdev->dev, "Failed to allocate for blob\n"); ++ } else { ++ struct kbase_vmap_struct vmap = { 0 }; ++ const u8 *payload; ++ void *dst; ++ ++ /* copy the payload, */ ++ payload = fw + blob->payload_offset; ++ ++ dst = kbase_vmap(kctx, ++ va_region->start_pfn << PAGE_SHIFT, ++ nr_pages << PAGE_SHIFT, &vmap); ++ ++ if (dst) { ++ memcpy(dst, payload, blob->size); ++ kbase_vunmap(kctx, &vmap); ++ } else { ++ dev_err(kbdev->dev, ++ "Failed to copy payload\n"); ++ } ++ ++ } ++ blob_offset = blob->blob_offset; /* follow chain */ ++ } ++ ++ release_firmware(firmware); ++ ++ kbasep_js_schedule_privileged_ctx(kbdev, kctx); ++ ++ kbdev->dummy_job_wa.ctx = kctx; ++ ++ err = sysfs_create_file(&kbdev->dev->kobj, ++ &dev_attr_dummy_job_wa_info.attr); ++ if (err) ++ dev_err(kbdev->dev, "SysFS file creation for dummy job wa failed\n"); ++ ++ return 0; ++ ++bad_fw: ++ kbase_destroy_context(kctx); ++no_ctx: ++ release_firmware(firmware); ++ return -EFAULT; ++} ++ ++void kbase_dummy_job_wa_cleanup(struct kbase_device *kbdev) ++{ ++ struct kbase_context *wa_ctx; ++ ++ /* Can be safely called even if the file wasn't created on probe */ ++ sysfs_remove_file(&kbdev->dev->kobj, &dev_attr_dummy_job_wa_info.attr); ++ ++ wa_ctx = READ_ONCE(kbdev->dummy_job_wa.ctx); ++ WRITE_ONCE(kbdev->dummy_job_wa.ctx, NULL); ++ /* make this write visible before we tear down the ctx */ ++ smp_mb(); ++ ++ if (wa_ctx) { ++ kbasep_js_release_privileged_ctx(kbdev, wa_ctx); ++ kbase_destroy_context(wa_ctx); ++ } ++} +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_dummy_job_wa.h b/drivers/gpu/arm/bifrost/mali_kbase_dummy_job_wa.h +new file mode 100755 +index 000000000000..e19495055b48 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_dummy_job_wa.h +@@ -0,0 +1,74 @@ ++/* ++ * ++ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_DUMMY_JOB_WORKAROUND_ ++#define _KBASE_DUMMY_JOB_WORKAROUND_ ++ ++#define KBASE_DUMMY_JOB_WA_FLAG_SERIALIZE (1ull << 0) ++#define KBASE_DUMMY_JOB_WA_FLAG_WAIT_POWERUP (1ull << 1) ++#define KBASE_DUMMY_JOB_WA_FLAG_LOGICAL_SHADER_POWER (1ull << 2) ++ ++#define KBASE_DUMMY_JOB_WA_FLAGS (KBASE_DUMMY_JOB_WA_FLAG_SERIALIZE | \ ++ KBASE_DUMMY_JOB_WA_FLAG_WAIT_POWERUP | \ ++ KBASE_DUMMY_JOB_WA_FLAG_LOGICAL_SHADER_POWER) ++ ++#if MALI_USE_CSF ++ ++static inline int kbase_dummy_job_wa_load(struct kbase_device *kbdev) ++{ ++ CSTD_UNUSED(kbdev); ++ return 0; ++} ++ ++static inline void kbase_dummy_job_wa_cleanup(struct kbase_device *kbdev) ++{ ++ CSTD_UNUSED(kbdev); ++} ++ ++static inline int kbase_dummy_job_wa_execute(struct kbase_device *kbdev, ++ u64 cores) ++{ ++ CSTD_UNUSED(kbdev); ++ CSTD_UNUSED(cores); ++ return 0; ++} ++ ++static inline bool kbase_dummy_job_wa_enabled(struct kbase_device *kbdev) ++{ ++ CSTD_UNUSED(kbdev); ++ return false; ++} ++ ++#else ++ ++int kbase_dummy_job_wa_load(struct kbase_device *kbdev); ++void kbase_dummy_job_wa_cleanup(struct kbase_device *kbdev); ++int kbase_dummy_job_wa_execute(struct kbase_device *kbdev, u64 cores); ++ ++static inline bool kbase_dummy_job_wa_enabled(struct kbase_device *kbdev) ++{ ++ return (kbdev->dummy_job_wa.ctx != NULL); ++} ++ ++#endif /* MALI_USE_CSF */ ++ ++#endif /* _KBASE_DUMMY_JOB_WORKAROUND_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_event.c b/drivers/gpu/arm/bifrost/mali_kbase_event.c +new file mode 100755 +index 000000000000..5adb80f9bbd2 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_event.c +@@ -0,0 +1,274 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2016,2018-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++#include ++#include ++#include ++#include ++ ++static struct base_jd_udata kbase_event_process(struct kbase_context *kctx, struct kbase_jd_atom *katom) ++{ ++ struct base_jd_udata data; ++ struct kbase_device *kbdev; ++ ++ lockdep_assert_held(&kctx->jctx.lock); ++ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ KBASE_DEBUG_ASSERT(katom != NULL); ++ KBASE_DEBUG_ASSERT(katom->status == KBASE_JD_ATOM_STATE_COMPLETED); ++ ++ kbdev = kctx->kbdev; ++ data = katom->udata; ++ ++ KBASE_TLSTREAM_TL_NRET_ATOM_CTX(kbdev, katom, kctx); ++ KBASE_TLSTREAM_TL_DEL_ATOM(kbdev, katom); ++ ++ katom->status = KBASE_JD_ATOM_STATE_UNUSED; ++ dev_dbg(kbdev->dev, "Atom %p status to unused\n", (void *)katom); ++ wake_up(&katom->completed); ++ ++ return data; ++} ++ ++int kbase_event_dequeue(struct kbase_context *ctx, struct base_jd_event_v2 *uevent) ++{ ++ struct kbase_jd_atom *atom; ++ ++ KBASE_DEBUG_ASSERT(ctx); ++ ++ mutex_lock(&ctx->event_mutex); ++ ++ if (list_empty(&ctx->event_list)) { ++ if (!atomic_read(&ctx->event_closed)) { ++ mutex_unlock(&ctx->event_mutex); ++ return -1; ++ } ++ ++ /* generate the BASE_JD_EVENT_DRV_TERMINATED message on the fly */ ++ mutex_unlock(&ctx->event_mutex); ++ uevent->event_code = BASE_JD_EVENT_DRV_TERMINATED; ++ memset(&uevent->udata, 0, sizeof(uevent->udata)); ++ dev_dbg(ctx->kbdev->dev, ++ "event system closed, returning BASE_JD_EVENT_DRV_TERMINATED(0x%X)\n", ++ BASE_JD_EVENT_DRV_TERMINATED); ++ return 0; ++ } ++ ++ /* normal event processing */ ++ atomic_dec(&ctx->event_count); ++ atom = list_entry(ctx->event_list.next, struct kbase_jd_atom, dep_item[0]); ++ list_del(ctx->event_list.next); ++ ++ mutex_unlock(&ctx->event_mutex); ++ ++ dev_dbg(ctx->kbdev->dev, "event dequeuing %p\n", (void *)atom); ++ uevent->event_code = atom->event_code; ++ ++ uevent->atom_number = (atom - ctx->jctx.atoms); ++ ++ if (atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) ++ kbase_jd_free_external_resources(atom); ++ ++ mutex_lock(&ctx->jctx.lock); ++ uevent->udata = kbase_event_process(ctx, atom); ++ mutex_unlock(&ctx->jctx.lock); ++ ++ return 0; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_event_dequeue); ++ ++/** ++ * kbase_event_process_noreport_worker - Worker for processing atoms that do not ++ * return an event but do have external ++ * resources ++ * @data: Work structure ++ */ ++static void kbase_event_process_noreport_worker(struct work_struct *data) ++{ ++ struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, ++ work); ++ struct kbase_context *kctx = katom->kctx; ++ ++ if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) ++ kbase_jd_free_external_resources(katom); ++ ++ mutex_lock(&kctx->jctx.lock); ++ kbase_event_process(kctx, katom); ++ mutex_unlock(&kctx->jctx.lock); ++} ++ ++/** ++ * kbase_event_process_noreport - Process atoms that do not return an event ++ * @kctx: Context pointer ++ * @katom: Atom to be processed ++ * ++ * Atoms that do not have external resources will be processed immediately. ++ * Atoms that do have external resources will be processed on a workqueue, in ++ * order to avoid locking issues. ++ */ ++static void kbase_event_process_noreport(struct kbase_context *kctx, ++ struct kbase_jd_atom *katom) ++{ ++ if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) { ++ INIT_WORK(&katom->work, kbase_event_process_noreport_worker); ++ queue_work(kctx->event_workq, &katom->work); ++ } else { ++ kbase_event_process(kctx, katom); ++ } ++} ++ ++/** ++ * kbase_event_coalesce - Move pending events to the main event list ++ * @kctx: Context pointer ++ * ++ * kctx->event_list and kctx->event_coalesce_count must be protected ++ * by a lock unless this is the last thread using them ++ * (and we're about to terminate the lock). ++ * ++ * Return: The number of pending events moved to the main event list ++ */ ++static int kbase_event_coalesce(struct kbase_context *kctx) ++{ ++ const int event_count = kctx->event_coalesce_count; ++ ++ /* Join the list of pending events onto the tail of the main list ++ and reset it */ ++ list_splice_tail_init(&kctx->event_coalesce_list, &kctx->event_list); ++ kctx->event_coalesce_count = 0; ++ ++ /* Return the number of events moved */ ++ return event_count; ++} ++ ++void kbase_event_post(struct kbase_context *ctx, struct kbase_jd_atom *atom) ++{ ++ struct kbase_device *kbdev = ctx->kbdev; ++ ++ dev_dbg(kbdev->dev, "Posting event for atom %p\n", (void *)atom); ++ ++ if (WARN_ON(atom->status != KBASE_JD_ATOM_STATE_COMPLETED)) { ++ dev_warn(kbdev->dev, ++ "%s: Atom %d (%p) not completed (status %d)\n", ++ __func__, ++ kbase_jd_atom_id(atom->kctx, atom), ++ atom->kctx, ++ atom->status); ++ return; ++ } ++ ++ if (atom->core_req & BASE_JD_REQ_EVENT_ONLY_ON_FAILURE) { ++ if (atom->event_code == BASE_JD_EVENT_DONE) { ++ dev_dbg(kbdev->dev, "Suppressing event (atom done)\n"); ++ kbase_event_process_noreport(ctx, atom); ++ return; ++ } ++ } ++ ++ if (atom->core_req & BASEP_JD_REQ_EVENT_NEVER) { ++ dev_dbg(kbdev->dev, "Suppressing event (never)\n"); ++ kbase_event_process_noreport(ctx, atom); ++ return; ++ } ++ KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(kbdev, atom, TL_ATOM_STATE_POSTED); ++ if (atom->core_req & BASE_JD_REQ_EVENT_COALESCE) { ++ /* Don't report the event until other event(s) have completed */ ++ dev_dbg(kbdev->dev, "Deferring event (coalesced)\n"); ++ mutex_lock(&ctx->event_mutex); ++ list_add_tail(&atom->dep_item[0], &ctx->event_coalesce_list); ++ ++ctx->event_coalesce_count; ++ mutex_unlock(&ctx->event_mutex); ++ } else { ++ /* Report the event and any pending events now */ ++ int event_count = 1; ++ ++ mutex_lock(&ctx->event_mutex); ++ event_count += kbase_event_coalesce(ctx); ++ list_add_tail(&atom->dep_item[0], &ctx->event_list); ++ atomic_add(event_count, &ctx->event_count); ++ mutex_unlock(&ctx->event_mutex); ++ dev_dbg(kbdev->dev, "Reporting %d events\n", event_count); ++ ++ kbase_event_wakeup(ctx); ++ ++ /* Post-completion latency */ ++ trace_sysgraph(SGR_POST, ctx->id, ++ kbase_jd_atom_id(ctx, atom)); ++ } ++} ++KBASE_EXPORT_TEST_API(kbase_event_post); ++ ++void kbase_event_close(struct kbase_context *kctx) ++{ ++ mutex_lock(&kctx->event_mutex); ++ atomic_set(&kctx->event_closed, true); ++ mutex_unlock(&kctx->event_mutex); ++ kbase_event_wakeup(kctx); ++} ++ ++int kbase_event_init(struct kbase_context *kctx) ++{ ++ KBASE_DEBUG_ASSERT(kctx); ++ ++ INIT_LIST_HEAD(&kctx->event_list); ++ INIT_LIST_HEAD(&kctx->event_coalesce_list); ++ mutex_init(&kctx->event_mutex); ++ kctx->event_coalesce_count = 0; ++ kctx->event_workq = alloc_workqueue("kbase_event", WQ_MEM_RECLAIM, 1); ++ ++ if (NULL == kctx->event_workq) ++ return -EINVAL; ++ ++ return 0; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_event_init); ++ ++void kbase_event_cleanup(struct kbase_context *kctx) ++{ ++ int event_count; ++ ++ KBASE_DEBUG_ASSERT(kctx); ++ KBASE_DEBUG_ASSERT(kctx->event_workq); ++ ++ flush_workqueue(kctx->event_workq); ++ destroy_workqueue(kctx->event_workq); ++ ++ /* We use kbase_event_dequeue to remove the remaining events as that ++ * deals with all the cleanup needed for the atoms. ++ * ++ * Note: use of kctx->event_list without a lock is safe because this must be the last ++ * thread using it (because we're about to terminate the lock) ++ */ ++ event_count = kbase_event_coalesce(kctx); ++ atomic_add(event_count, &kctx->event_count); ++ ++ while (!list_empty(&kctx->event_list)) { ++ struct base_jd_event_v2 event; ++ ++ kbase_event_dequeue(kctx, &event); ++ } ++} ++ ++KBASE_EXPORT_TEST_API(kbase_event_cleanup); +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_fence.c b/drivers/gpu/arm/bifrost/mali_kbase_fence.c +new file mode 100755 +index 000000000000..5e04acf87892 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_fence.c +@@ -0,0 +1,154 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++ ++/* Spin lock protecting all Mali fences as fence->lock. */ ++static DEFINE_SPINLOCK(kbase_fence_lock); ++ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++struct fence * ++kbase_fence_out_new(struct kbase_jd_atom *katom) ++#else ++struct dma_fence * ++kbase_fence_out_new(struct kbase_jd_atom *katom) ++#endif ++{ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ struct fence *fence; ++#else ++ struct dma_fence *fence; ++#endif ++ ++ WARN_ON(katom->dma_fence.fence); ++ ++ fence = kzalloc(sizeof(*fence), GFP_KERNEL); ++ if (!fence) ++ return NULL; ++ ++ dma_fence_init(fence, ++ &kbase_fence_ops, ++ &kbase_fence_lock, ++ katom->dma_fence.context, ++ atomic_inc_return(&katom->dma_fence.seqno)); ++ ++ katom->dma_fence.fence = fence; ++ ++ return fence; ++} ++ ++bool ++kbase_fence_free_callbacks(struct kbase_jd_atom *katom) ++{ ++ struct kbase_fence_cb *cb, *tmp; ++ bool res = false; ++ ++ lockdep_assert_held(&katom->kctx->jctx.lock); ++ ++ /* Clean up and free callbacks. */ ++ list_for_each_entry_safe(cb, tmp, &katom->dma_fence.callbacks, node) { ++ bool ret; ++ ++ /* Cancel callbacks that hasn't been called yet. */ ++ ret = dma_fence_remove_callback(cb->fence, &cb->fence_cb); ++ if (ret) { ++ int ret; ++ ++ /* Fence had not signaled, clean up after ++ * canceling. ++ */ ++ ret = atomic_dec_return(&katom->dma_fence.dep_count); ++ ++ if (unlikely(ret == 0)) ++ res = true; ++ } ++ ++ /* ++ * Release the reference taken in ++ * kbase_fence_add_callback(). ++ */ ++ dma_fence_put(cb->fence); ++ list_del(&cb->node); ++ kfree(cb); ++ } ++ ++ return res; ++} ++ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++int ++kbase_fence_add_callback(struct kbase_jd_atom *katom, ++ struct fence *fence, ++ fence_func_t callback) ++#else ++int ++kbase_fence_add_callback(struct kbase_jd_atom *katom, ++ struct dma_fence *fence, ++ dma_fence_func_t callback) ++#endif ++{ ++ int err = 0; ++ struct kbase_fence_cb *kbase_fence_cb; ++ ++ if (!fence) ++ return -EINVAL; ++ ++ kbase_fence_cb = kmalloc(sizeof(*kbase_fence_cb), GFP_KERNEL); ++ if (!kbase_fence_cb) ++ return -ENOMEM; ++ ++ kbase_fence_cb->fence = fence; ++ kbase_fence_cb->katom = katom; ++ INIT_LIST_HEAD(&kbase_fence_cb->node); ++ atomic_inc(&katom->dma_fence.dep_count); ++ ++ err = dma_fence_add_callback(fence, &kbase_fence_cb->fence_cb, ++ callback); ++ if (err == -ENOENT) { ++ /* Fence signaled, get the completion result */ ++ err = dma_fence_get_status(fence); ++ ++ /* remap success completion to err code */ ++ if (err == 1) ++ err = 0; ++ ++ kfree(kbase_fence_cb); ++ atomic_dec(&katom->dma_fence.dep_count); ++ } else if (err) { ++ kfree(kbase_fence_cb); ++ atomic_dec(&katom->dma_fence.dep_count); ++ } else { ++ /* ++ * Get reference to fence that will be kept until callback gets ++ * cleaned up in kbase_fence_free_callbacks(). ++ */ ++ dma_fence_get(fence); ++ /* Add callback to katom's list of callbacks */ ++ list_add(&kbase_fence_cb->node, &katom->dma_fence.callbacks); ++ } ++ ++ return err; ++} +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_fence.h b/drivers/gpu/arm/bifrost/mali_kbase_fence.h +new file mode 100755 +index 000000000000..f319d9e1dce6 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_fence.h +@@ -0,0 +1,284 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2018, 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_FENCE_H_ ++#define _KBASE_FENCE_H_ ++ ++/* ++ * mali_kbase_fence.[hc] has common fence code used by both ++ * - CONFIG_MALI_BIFROST_DMA_FENCE - implicit DMA fences ++ * - CONFIG_SYNC_FILE - explicit fences beginning with 4.9 kernel ++ */ ++ ++#if defined(CONFIG_MALI_BIFROST_DMA_FENCE) || defined(CONFIG_SYNC_FILE) ++ ++#include ++#include "mali_kbase_fence_defs.h" ++#include "mali_kbase.h" ++ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++extern const struct fence_ops kbase_fence_ops; ++#else ++extern const struct dma_fence_ops kbase_fence_ops; ++#endif ++ ++/** ++* struct kbase_fence_cb - Mali dma-fence callback data struct ++* @fence_cb: Callback function ++* @katom: Pointer to katom that is waiting on this callback ++* @fence: Pointer to the fence object on which this callback is waiting ++* @node: List head for linking this callback to the katom ++*/ ++struct kbase_fence_cb { ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ struct fence_cb fence_cb; ++ struct fence *fence; ++#else ++ struct dma_fence_cb fence_cb; ++ struct dma_fence *fence; ++#endif ++ struct kbase_jd_atom *katom; ++ struct list_head node; ++}; ++ ++/** ++ * kbase_fence_out_new() - Creates a new output fence and puts it on the atom ++ * @katom: Atom to create an output fence for ++ * ++ * return: A new fence object on success, NULL on failure. ++ */ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++struct fence *kbase_fence_out_new(struct kbase_jd_atom *katom); ++#else ++struct dma_fence *kbase_fence_out_new(struct kbase_jd_atom *katom); ++#endif ++ ++#if defined(CONFIG_SYNC_FILE) ++/** ++ * kbase_fence_fence_in_set() - Assign input fence to atom ++ * @katom: Atom to assign input fence to ++ * @fence: Input fence to assign to atom ++ * ++ * This function will take ownership of one fence reference! ++ */ ++#define kbase_fence_fence_in_set(katom, fence) \ ++ do { \ ++ WARN_ON((katom)->dma_fence.fence_in); \ ++ (katom)->dma_fence.fence_in = fence; \ ++ } while (0) ++#endif ++ ++ ++#if !MALI_USE_CSF ++/** ++ * kbase_fence_out_remove() - Removes the output fence from atom ++ * @katom: Atom to remove output fence for ++ * ++ * This will also release the reference to this fence which the atom keeps ++ */ ++static inline void kbase_fence_out_remove(struct kbase_jd_atom *katom) ++{ ++ if (katom->dma_fence.fence) { ++ dma_fence_put(katom->dma_fence.fence); ++ katom->dma_fence.fence = NULL; ++ } ++} ++ ++#if defined(CONFIG_SYNC_FILE) ++/** ++ * kbase_fence_out_remove() - Removes the input fence from atom ++ * @katom: Atom to remove input fence for ++ * ++ * This will also release the reference to this fence which the atom keeps ++ */ ++static inline void kbase_fence_in_remove(struct kbase_jd_atom *katom) ++{ ++ if (katom->dma_fence.fence_in) { ++ dma_fence_put(katom->dma_fence.fence_in); ++ katom->dma_fence.fence_in = NULL; ++ } ++} ++#endif ++ ++/** ++ * kbase_fence_out_is_ours() - Check if atom has a valid fence created by us ++ * @katom: Atom to check output fence for ++ * ++ * Return: true if fence exists and is valid, otherwise false ++ */ ++static inline bool kbase_fence_out_is_ours(struct kbase_jd_atom *katom) ++{ ++ return katom->dma_fence.fence && ++ katom->dma_fence.fence->ops == &kbase_fence_ops; ++} ++ ++/** ++ * kbase_fence_out_signal() - Signal output fence of atom ++ * @katom: Atom to signal output fence for ++ * @status: Status to signal with (0 for success, < 0 for error) ++ * ++ * Return: 0 on success, < 0 on error ++ */ ++static inline int kbase_fence_out_signal(struct kbase_jd_atom *katom, ++ int status) ++{ ++ if (status) { ++#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE && \ ++ KERNEL_VERSION(4, 9, 68) <= LINUX_VERSION_CODE) ++ fence_set_error(katom->dma_fence.fence, status); ++#elif (KERNEL_VERSION(4, 11, 0) <= LINUX_VERSION_CODE) ++ dma_fence_set_error(katom->dma_fence.fence, status); ++#else ++ katom->dma_fence.fence->status = status; ++#endif ++ } ++ return dma_fence_signal(katom->dma_fence.fence); ++} ++ ++/** ++ * kbase_fence_add_callback() - Add callback on @fence to block @katom ++ * @katom: Pointer to katom that will be blocked by @fence ++ * @fence: Pointer to fence on which to set up the callback ++ * @callback: Pointer to function to be called when fence is signaled ++ * ++ * Caller needs to hold a reference to @fence when calling this function, and ++ * the caller is responsible for releasing that reference. An additional ++ * reference to @fence will be taken when the callback was successfully set up ++ * and @fence needs to be kept valid until the callback has been called and ++ * cleanup have been done. ++ * ++ * Return: 0 on success: fence was either already signaled, or callback was ++ * set up. Negative error code is returned on error. ++ */ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++int kbase_fence_add_callback(struct kbase_jd_atom *katom, ++ struct fence *fence, ++ fence_func_t callback); ++#else ++int kbase_fence_add_callback(struct kbase_jd_atom *katom, ++ struct dma_fence *fence, ++ dma_fence_func_t callback); ++#endif ++ ++/** ++ * kbase_fence_dep_count_set() - Set dep_count value on atom to specified value ++ * @katom: Atom to set dep_count for ++ * @val: value to set dep_count to ++ * ++ * The dep_count is available to the users of this module so that they can ++ * synchronize completion of the wait with cancellation and adding of more ++ * callbacks. For instance, a user could do the following: ++ * ++ * dep_count set to 1 ++ * callback #1 added, dep_count is increased to 2 ++ * callback #1 happens, dep_count decremented to 1 ++ * since dep_count > 0, no completion is done ++ * callback #2 is added, dep_count is increased to 2 ++ * dep_count decremented to 1 ++ * callback #2 happens, dep_count decremented to 0 ++ * since dep_count now is zero, completion executes ++ * ++ * The dep_count can also be used to make sure that the completion only ++ * executes once. This is typically done by setting dep_count to -1 for the ++ * thread that takes on this responsibility. ++ */ ++static inline void ++kbase_fence_dep_count_set(struct kbase_jd_atom *katom, int val) ++{ ++ atomic_set(&katom->dma_fence.dep_count, val); ++} ++ ++/** ++ * kbase_fence_dep_count_dec_and_test() - Decrements dep_count ++ * @katom: Atom to decrement dep_count for ++ * ++ * See @kbase_fence_dep_count_set for general description about dep_count ++ * ++ * Return: true if value was decremented to zero, otherwise false ++ */ ++static inline bool ++kbase_fence_dep_count_dec_and_test(struct kbase_jd_atom *katom) ++{ ++ return atomic_dec_and_test(&katom->dma_fence.dep_count); ++} ++ ++/** ++ * kbase_fence_dep_count_read() - Returns the current dep_count value ++ * @katom: Pointer to katom ++ * ++ * See @kbase_fence_dep_count_set for general description about dep_count ++ * ++ * Return: The current dep_count value ++ */ ++static inline int kbase_fence_dep_count_read(struct kbase_jd_atom *katom) ++{ ++ return atomic_read(&katom->dma_fence.dep_count); ++} ++ ++/** ++ * kbase_fence_free_callbacks() - Free dma-fence callbacks on a katom ++ * @katom: Pointer to katom ++ * ++ * This function will free all fence callbacks on the katom's list of ++ * callbacks. Callbacks that have not yet been called, because their fence ++ * hasn't yet signaled, will first be removed from the fence. ++ * ++ * Locking: katom->dma_fence.callbacks list assumes jctx.lock is held. ++ * ++ * Return: true if dep_count reached 0, otherwise false. ++ */ ++bool kbase_fence_free_callbacks(struct kbase_jd_atom *katom); ++ ++#if defined(CONFIG_SYNC_FILE) ++/** ++ * kbase_fence_in_get() - Retrieve input fence for atom. ++ * @katom: Atom to get input fence from ++ * ++ * A ref will be taken for the fence, so use @kbase_fence_put() to release it ++ * ++ * Return: The fence, or NULL if there is no input fence for atom ++ */ ++#define kbase_fence_in_get(katom) dma_fence_get((katom)->dma_fence.fence_in) ++#endif ++ ++/** ++ * kbase_fence_out_get() - Retrieve output fence for atom. ++ * @katom: Atom to get output fence from ++ * ++ * A ref will be taken for the fence, so use @kbase_fence_put() to release it ++ * ++ * Return: The fence, or NULL if there is no output fence for atom ++ */ ++#define kbase_fence_out_get(katom) dma_fence_get((katom)->dma_fence.fence) ++ ++#endif /* !MALI_USE_CSF */ ++ ++/** ++ * kbase_fence_put() - Releases a reference to a fence ++ * @fence: Fence to release reference for. ++ */ ++#define kbase_fence_put(fence) dma_fence_put(fence) ++ ++ ++#endif /* CONFIG_MALI_BIFROST_DMA_FENCE || defined(CONFIG_SYNC_FILE */ ++ ++#endif /* _KBASE_FENCE_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_fence_defs.h b/drivers/gpu/arm/bifrost/mali_kbase_fence_defs.h +new file mode 100755 +index 000000000000..303029639d38 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_fence_defs.h +@@ -0,0 +1,64 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2018, 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_FENCE_DEFS_H_ ++#define _KBASE_FENCE_DEFS_H_ ++ ++/* ++ * There was a big rename in the 4.10 kernel (fence* -> dma_fence*) ++ * This file hides the compatibility issues with this for the rest the driver ++ */ ++ ++#include ++ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ ++#include ++ ++#define dma_fence_context_alloc(a) fence_context_alloc(a) ++#define dma_fence_init(a, b, c, d, e) fence_init(a, b, c, d, e) ++#define dma_fence_get(a) fence_get(a) ++#define dma_fence_put(a) fence_put(a) ++#define dma_fence_signal(a) fence_signal(a) ++#define dma_fence_is_signaled(a) fence_is_signaled(a) ++#define dma_fence_add_callback(a, b, c) fence_add_callback(a, b, c) ++#define dma_fence_remove_callback(a, b) fence_remove_callback(a, b) ++ ++#if (KERNEL_VERSION(4, 9, 68) <= LINUX_VERSION_CODE) ++#define dma_fence_get_status(a) (fence_is_signaled(a) ? (a)->error ?: 1 : 0) ++#else ++#define dma_fence_get_status(a) (fence_is_signaled(a) ? (a)->status ?: 1 : 0) ++#endif ++ ++#else ++ ++#include ++ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0)) ++#define dma_fence_get_status(a) (dma_fence_is_signaled(a) ? \ ++ (a)->status ?: 1 \ ++ : 0) ++#endif ++ ++#endif /* < 4.10.0 */ ++ ++#endif /* _KBASE_FENCE_DEFS_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_fence_ops.c b/drivers/gpu/arm/bifrost/mali_kbase_fence_ops.c +new file mode 100755 +index 000000000000..c4703748bec6 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_fence_ops.c +@@ -0,0 +1,84 @@ ++/* ++ * ++ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++#include ++#include ++#include ++ ++static const char * ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++kbase_fence_get_driver_name(struct fence *fence) ++#else ++kbase_fence_get_driver_name(struct dma_fence *fence) ++#endif ++{ ++ return kbase_drv_name; ++} ++ ++static const char * ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++kbase_fence_get_timeline_name(struct fence *fence) ++#else ++kbase_fence_get_timeline_name(struct dma_fence *fence) ++#endif ++{ ++ return kbase_timeline_name; ++} ++ ++static bool ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++kbase_fence_enable_signaling(struct fence *fence) ++#else ++kbase_fence_enable_signaling(struct dma_fence *fence) ++#endif ++{ ++ return true; ++} ++ ++static void ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++kbase_fence_fence_value_str(struct fence *fence, char *str, int size) ++#else ++kbase_fence_fence_value_str(struct dma_fence *fence, char *str, int size) ++#endif ++{ ++#if (KERNEL_VERSION(5, 1, 0) > LINUX_VERSION_CODE) ++ snprintf(str, size, "%u", fence->seqno); ++#else ++ snprintf(str, size, "%llu", fence->seqno); ++#endif ++} ++ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++const struct fence_ops kbase_fence_ops = { ++ .wait = fence_default_wait, ++#else ++const struct dma_fence_ops kbase_fence_ops = { ++ .wait = dma_fence_default_wait, ++#endif ++ .get_driver_name = kbase_fence_get_driver_name, ++ .get_timeline_name = kbase_fence_get_timeline_name, ++ .enable_signaling = kbase_fence_enable_signaling, ++ .fence_value_str = kbase_fence_fence_value_str ++}; ++ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_gator.h b/drivers/gpu/arm/bifrost/mali_kbase_gator.h +new file mode 100755 +index 000000000000..579c7b6ff3aa +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_gator.h +@@ -0,0 +1,53 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* NB taken from gator */ ++/* ++ * List of possible actions to be controlled by DS-5 Streamline. ++ * The following numbers are used by gator to control the frame buffer dumping ++ * and s/w counter reporting. We cannot use the enums in mali_uk_types.h because ++ * they are unknown inside gator. ++ */ ++ ++#ifndef _KBASE_GATOR_H_ ++#define _KBASE_GATOR_H_ ++ ++#include ++ ++#define GATOR_JOB_SLOT_START 1 ++#define GATOR_JOB_SLOT_STOP 2 ++#define GATOR_JOB_SLOT_SOFT_STOPPED 3 ++ ++#ifdef CONFIG_MALI_BIFROST_GATOR_SUPPORT ++ ++#define GATOR_MAKE_EVENT(type, number) (((type) << 24) | ((number) << 16)) ++ ++struct kbase_context; ++ ++void kbase_trace_mali_job_slots_event(u32 dev_id, u32 event, const struct kbase_context *kctx, u8 atom_id); ++void kbase_trace_mali_pm_status(u32 dev_id, u32 event, u64 value); ++void kbase_trace_mali_page_fault_insert_pages(u32 dev_id, int event, u32 value); ++void kbase_trace_mali_total_alloc_pages_change(u32 dev_id, long long int event); ++ ++#endif /* CONFIG_MALI_BIFROST_GATOR_SUPPORT */ ++ ++#endif /* _KBASE_GATOR_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_gpu_memory_debugfs.c b/drivers/gpu/arm/bifrost/mali_kbase_gpu_memory_debugfs.c +new file mode 100755 +index 000000000000..569abd920fde +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_gpu_memory_debugfs.c +@@ -0,0 +1,104 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2017, 2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++#include ++ ++#ifdef CONFIG_DEBUG_FS ++/** Show callback for the @c gpu_memory debugfs file. ++ * ++ * This function is called to get the contents of the @c gpu_memory debugfs ++ * file. This is a report of current gpu memory usage. ++ * ++ * @param sfile The debugfs entry ++ * @param data Data associated with the entry ++ * ++ * @return 0 if successfully prints data in debugfs entry file ++ * -1 if it encountered an error ++ */ ++ ++static int kbasep_gpu_memory_seq_show(struct seq_file *sfile, void *data) ++{ ++ struct list_head *entry; ++ const struct list_head *kbdev_list; ++ ++ kbdev_list = kbase_device_get_list(); ++ list_for_each(entry, kbdev_list) { ++ struct kbase_device *kbdev = NULL; ++ struct kbase_context *kctx; ++ ++ kbdev = list_entry(entry, struct kbase_device, entry); ++ /* output the total memory usage and cap for this device */ ++ seq_printf(sfile, "%-16s %10u\n", ++ kbdev->devname, ++ atomic_read(&(kbdev->memdev.used_pages))); ++ mutex_lock(&kbdev->kctx_list_lock); ++ list_for_each_entry(kctx, &kbdev->kctx_list, kctx_list_link) { ++ /* output the memory usage and cap for each kctx ++ * opened on this device */ ++ seq_printf(sfile, " %s-0x%p %10u\n", ++ "kctx", ++ kctx, ++ atomic_read(&(kctx->used_pages))); ++ } ++ mutex_unlock(&kbdev->kctx_list_lock); ++ } ++ kbase_device_put_list(kbdev_list); ++ return 0; ++} ++ ++/* ++ * File operations related to debugfs entry for gpu_memory ++ */ ++static int kbasep_gpu_memory_debugfs_open(struct inode *in, struct file *file) ++{ ++ return single_open(file, kbasep_gpu_memory_seq_show, NULL); ++} ++ ++static const struct file_operations kbasep_gpu_memory_debugfs_fops = { ++ .owner = THIS_MODULE, ++ .open = kbasep_gpu_memory_debugfs_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = single_release, ++}; ++ ++/* ++ * Initialize debugfs entry for gpu_memory ++ */ ++void kbasep_gpu_memory_debugfs_init(struct kbase_device *kbdev) ++{ ++ debugfs_create_file("gpu_memory", S_IRUGO, ++ kbdev->mali_debugfs_directory, NULL, ++ &kbasep_gpu_memory_debugfs_fops); ++ return; ++} ++ ++#else ++/* ++ * Stub functions for when debugfs is disabled ++ */ ++void kbasep_gpu_memory_debugfs_init(struct kbase_device *kbdev) ++{ ++ return; ++} ++#endif +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_gpu_memory_debugfs.h b/drivers/gpu/arm/bifrost/mali_kbase_gpu_memory_debugfs.h +new file mode 100755 +index 000000000000..a45dabbb680f +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_gpu_memory_debugfs.h +@@ -0,0 +1,54 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2014, 2016, 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++/** ++ * @file mali_kbase_gpu_memory_debugfs.h ++ * Header file for gpu_memory entry in debugfs ++ * ++ */ ++ ++#ifndef _KBASE_GPU_MEMORY_DEBUGFS_H ++#define _KBASE_GPU_MEMORY_DEBUGFS_H ++ ++#include ++#include ++ ++/* kbase_io_history_add - add new entry to the register access history ++ * ++ * @h: Pointer to the history data structure ++ * @addr: Register address ++ * @value: The value that is either read from or written to the register ++ * @write: 1 if it's a register write, 0 if it's a read ++ */ ++void kbase_io_history_add(struct kbase_io_history *h, void __iomem const *addr, ++ u32 value, u8 write); ++ ++/** ++ * kbasep_gpu_memory_debugfs_init - Initialize gpu_memory debugfs entry ++ * ++ * @kbdev: Device pointer ++ */ ++void kbasep_gpu_memory_debugfs_init(struct kbase_device *kbdev); ++ ++#endif /*_KBASE_GPU_MEMORY_DEBUGFS_H*/ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_gpuprops.c b/drivers/gpu/arm/bifrost/mali_kbase_gpuprops.c +new file mode 100755 +index 000000000000..020b5d853608 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_gpuprops.c +@@ -0,0 +1,649 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * ++ * (C) COPYRIGHT 2011-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++/* ++ * Base kernel property query APIs ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include "mali_kbase_ioctl.h" ++#include ++#include ++#include ++#include ++ ++ ++static void kbase_gpuprops_construct_coherent_groups( ++ struct base_gpu_props * const props) ++{ ++ struct mali_base_gpu_coherent_group *current_group; ++ u64 group_present; ++ u64 group_mask; ++ u64 first_set, first_set_prev; ++ u32 num_groups = 0; ++ ++ KBASE_DEBUG_ASSERT(NULL != props); ++ ++ props->coherency_info.coherency = props->raw_props.mem_features; ++ props->coherency_info.num_core_groups = hweight64(props->raw_props.l2_present); ++ ++ if (props->coherency_info.coherency & GROUPS_L2_COHERENT) { ++ /* Group is l2 coherent */ ++ group_present = props->raw_props.l2_present; ++ } else { ++ /* Group is l1 coherent */ ++ group_present = props->raw_props.shader_present; ++ } ++ ++ /* ++ * The coherent group mask can be computed from the l2 present ++ * register. ++ * ++ * For the coherent group n: ++ * group_mask[n] = (first_set[n] - 1) & ~(first_set[n-1] - 1) ++ * where first_set is group_present with only its nth set-bit kept ++ * (i.e. the position from where a new group starts). ++ * ++ * For instance if the groups are l2 coherent and l2_present=0x0..01111: ++ * The first mask is: ++ * group_mask[1] = (first_set[1] - 1) & ~(first_set[0] - 1) ++ * = (0x0..010 - 1) & ~(0x0..01 - 1) ++ * = 0x0..00f ++ * The second mask is: ++ * group_mask[2] = (first_set[2] - 1) & ~(first_set[1] - 1) ++ * = (0x0..100 - 1) & ~(0x0..010 - 1) ++ * = 0x0..0f0 ++ * And so on until all the bits from group_present have been cleared ++ * (i.e. there is no group left). ++ */ ++ ++ current_group = props->coherency_info.group; ++ first_set = group_present & ~(group_present - 1); ++ ++ while (group_present != 0 && num_groups < BASE_MAX_COHERENT_GROUPS) { ++ group_present -= first_set; /* Clear the current group bit */ ++ first_set_prev = first_set; ++ ++ first_set = group_present & ~(group_present - 1); ++ group_mask = (first_set - 1) & ~(first_set_prev - 1); ++ ++ /* Populate the coherent_group structure for each group */ ++ current_group->core_mask = group_mask & props->raw_props.shader_present; ++ current_group->num_cores = hweight64(current_group->core_mask); ++ ++ num_groups++; ++ current_group++; ++ } ++ ++ if (group_present != 0) ++ pr_warn("Too many coherent groups (keeping only %d groups).\n", BASE_MAX_COHERENT_GROUPS); ++ ++ props->coherency_info.num_groups = num_groups; ++} ++ ++/** ++ * kbase_gpuprops_get_props - Get the GPU configuration ++ * @gpu_props: The &struct base_gpu_props structure ++ * @kbdev: The &struct kbase_device structure for the device ++ * ++ * Fill the &struct base_gpu_props structure with values from the GPU ++ * configuration registers. Only the raw properties are filled in this function. ++ * ++ * Return: Zero on success, Linux error code on failure ++ */ ++static int kbase_gpuprops_get_props(struct base_gpu_props * const gpu_props, ++ struct kbase_device *kbdev) ++{ ++ struct kbase_gpuprops_regdump regdump; ++ int i; ++ int err; ++ ++ KBASE_DEBUG_ASSERT(NULL != kbdev); ++ KBASE_DEBUG_ASSERT(NULL != gpu_props); ++ ++ /* Dump relevant registers */ ++ err = kbase_backend_gpuprops_get(kbdev, ®dump); ++ if (err) ++ return err; ++ ++ gpu_props->raw_props.gpu_id = regdump.gpu_id; ++ gpu_props->raw_props.tiler_features = regdump.tiler_features; ++ gpu_props->raw_props.mem_features = regdump.mem_features; ++ gpu_props->raw_props.mmu_features = regdump.mmu_features; ++ gpu_props->raw_props.l2_features = regdump.l2_features; ++ gpu_props->raw_props.core_features = regdump.core_features; ++ ++ gpu_props->raw_props.as_present = regdump.as_present; ++ gpu_props->raw_props.js_present = regdump.js_present; ++ gpu_props->raw_props.shader_present = ++ ((u64) regdump.shader_present_hi << 32) + ++ regdump.shader_present_lo; ++ gpu_props->raw_props.tiler_present = ++ ((u64) regdump.tiler_present_hi << 32) + ++ regdump.tiler_present_lo; ++ gpu_props->raw_props.l2_present = ++ ((u64) regdump.l2_present_hi << 32) + ++ regdump.l2_present_lo; ++ gpu_props->raw_props.stack_present = ++ ((u64) regdump.stack_present_hi << 32) + ++ regdump.stack_present_lo; ++ ++ for (i = 0; i < GPU_MAX_JOB_SLOTS; i++) ++ gpu_props->raw_props.js_features[i] = regdump.js_features[i]; ++ ++ for (i = 0; i < BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS; i++) ++ gpu_props->raw_props.texture_features[i] = regdump.texture_features[i]; ++ ++ gpu_props->raw_props.thread_max_barrier_size = regdump.thread_max_barrier_size; ++ gpu_props->raw_props.thread_max_threads = regdump.thread_max_threads; ++ gpu_props->raw_props.thread_max_workgroup_size = regdump.thread_max_workgroup_size; ++ gpu_props->raw_props.thread_features = regdump.thread_features; ++ gpu_props->raw_props.thread_tls_alloc = regdump.thread_tls_alloc; ++ ++ return 0; ++} ++ ++void kbase_gpuprops_update_core_props_gpu_id( ++ struct base_gpu_props * const gpu_props) ++{ ++ gpu_props->core_props.version_status = ++ KBASE_UBFX32(gpu_props->raw_props.gpu_id, 0U, 4); ++ gpu_props->core_props.minor_revision = ++ KBASE_UBFX32(gpu_props->raw_props.gpu_id, 4U, 8); ++ gpu_props->core_props.major_revision = ++ KBASE_UBFX32(gpu_props->raw_props.gpu_id, 12U, 4); ++ gpu_props->core_props.product_id = ++ KBASE_UBFX32(gpu_props->raw_props.gpu_id, 16U, 16); ++} ++ ++/** ++ * kbase_gpuprops_calculate_props - Calculate the derived properties ++ * @gpu_props: The &struct base_gpu_props structure ++ * @kbdev: The &struct kbase_device structure for the device ++ * ++ * Fill the &struct base_gpu_props structure with values derived from the GPU ++ * configuration registers ++ */ ++static void kbase_gpuprops_calculate_props( ++ struct base_gpu_props * const gpu_props, struct kbase_device *kbdev) ++{ ++ int i; ++ u32 gpu_id; ++ u32 product_id; ++ ++ /* Populate the base_gpu_props structure */ ++ kbase_gpuprops_update_core_props_gpu_id(gpu_props); ++ gpu_props->core_props.log2_program_counter_size = KBASE_GPU_PC_SIZE_LOG2; ++#if KERNEL_VERSION(5, 0, 0) > LINUX_VERSION_CODE ++ gpu_props->core_props.gpu_available_memory_size = totalram_pages << PAGE_SHIFT; ++#else ++ gpu_props->core_props.gpu_available_memory_size = ++ totalram_pages() << PAGE_SHIFT; ++#endif ++ ++ gpu_props->core_props.num_exec_engines = ++ KBASE_UBFX32(gpu_props->raw_props.core_features, 0, 4); ++ ++ for (i = 0; i < BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS; i++) ++ gpu_props->core_props.texture_features[i] = gpu_props->raw_props.texture_features[i]; ++ ++ gpu_props->l2_props.log2_line_size = KBASE_UBFX32(gpu_props->raw_props.l2_features, 0U, 8); ++ gpu_props->l2_props.log2_cache_size = KBASE_UBFX32(gpu_props->raw_props.l2_features, 16U, 8); ++ ++ /* Field with number of l2 slices is added to MEM_FEATURES register ++ * since t76x. Below code assumes that for older GPU reserved bits will ++ * be read as zero. */ ++ gpu_props->l2_props.num_l2_slices = ++ KBASE_UBFX32(gpu_props->raw_props.mem_features, 8U, 4) + 1; ++ ++ gpu_props->tiler_props.bin_size_bytes = 1 << KBASE_UBFX32(gpu_props->raw_props.tiler_features, 0U, 6); ++ gpu_props->tiler_props.max_active_levels = KBASE_UBFX32(gpu_props->raw_props.tiler_features, 8U, 4); ++ ++ if (gpu_props->raw_props.thread_max_threads == 0) ++ gpu_props->thread_props.max_threads = THREAD_MT_DEFAULT; ++ else ++ gpu_props->thread_props.max_threads = gpu_props->raw_props.thread_max_threads; ++ ++ if (gpu_props->raw_props.thread_max_workgroup_size == 0) ++ gpu_props->thread_props.max_workgroup_size = THREAD_MWS_DEFAULT; ++ else ++ gpu_props->thread_props.max_workgroup_size = gpu_props->raw_props.thread_max_workgroup_size; ++ ++ if (gpu_props->raw_props.thread_max_barrier_size == 0) ++ gpu_props->thread_props.max_barrier_size = THREAD_MBS_DEFAULT; ++ else ++ gpu_props->thread_props.max_barrier_size = gpu_props->raw_props.thread_max_barrier_size; ++ ++ if (gpu_props->raw_props.thread_tls_alloc == 0) ++ gpu_props->thread_props.tls_alloc = ++ gpu_props->thread_props.max_threads; ++ else ++ gpu_props->thread_props.tls_alloc = ++ gpu_props->raw_props.thread_tls_alloc; ++ ++ /* MIDHARC-2364 was intended for tULx. ++ * Workaround for the incorrectly applied THREAD_FEATURES to tDUx. ++ */ ++ gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; ++ product_id = gpu_id & GPU_ID_VERSION_PRODUCT_ID; ++ product_id >>= GPU_ID_VERSION_PRODUCT_ID_SHIFT; ++ ++#if MALI_USE_CSF ++ gpu_props->thread_props.max_registers = ++ KBASE_UBFX32(gpu_props->raw_props.thread_features, ++ 0U, 22); ++ gpu_props->thread_props.impl_tech = ++ KBASE_UBFX32(gpu_props->raw_props.thread_features, ++ 22U, 2); ++ gpu_props->thread_props.max_task_queue = ++ KBASE_UBFX32(gpu_props->raw_props.thread_features, ++ 24U, 8); ++ gpu_props->thread_props.max_thread_group_split = 0; ++#else ++ if ((gpu_id & GPU_ID2_PRODUCT_MODEL) == GPU_ID2_PRODUCT_TDUX) { ++ gpu_props->thread_props.max_registers = ++ KBASE_UBFX32(gpu_props->raw_props.thread_features, ++ 0U, 22); ++ gpu_props->thread_props.impl_tech = ++ KBASE_UBFX32(gpu_props->raw_props.thread_features, ++ 22U, 2); ++ gpu_props->thread_props.max_task_queue = ++ KBASE_UBFX32(gpu_props->raw_props.thread_features, ++ 24U, 8); ++ gpu_props->thread_props.max_thread_group_split = 0; ++ } else { ++ gpu_props->thread_props.max_registers = ++ KBASE_UBFX32(gpu_props->raw_props.thread_features, ++ 0U, 16); ++ gpu_props->thread_props.max_task_queue = ++ KBASE_UBFX32(gpu_props->raw_props.thread_features, ++ 16U, 8); ++ gpu_props->thread_props.max_thread_group_split = ++ KBASE_UBFX32(gpu_props->raw_props.thread_features, ++ 24U, 6); ++ gpu_props->thread_props.impl_tech = ++ KBASE_UBFX32(gpu_props->raw_props.thread_features, ++ 30U, 2); ++ } ++#endif ++ ++ /* If values are not specified, then use defaults */ ++ if (gpu_props->thread_props.max_registers == 0) { ++ gpu_props->thread_props.max_registers = THREAD_MR_DEFAULT; ++ gpu_props->thread_props.max_task_queue = THREAD_MTQ_DEFAULT; ++ gpu_props->thread_props.max_thread_group_split = THREAD_MTGS_DEFAULT; ++ } ++ /* Initialize the coherent_group structure for each group */ ++ kbase_gpuprops_construct_coherent_groups(gpu_props); ++} ++ ++void kbase_gpuprops_set(struct kbase_device *kbdev) ++{ ++ struct kbase_gpu_props *gpu_props; ++ struct gpu_raw_gpu_props *raw; ++ ++ KBASE_DEBUG_ASSERT(NULL != kbdev); ++ gpu_props = &kbdev->gpu_props; ++ raw = &gpu_props->props.raw_props; ++ ++ /* Initialize the base_gpu_props structure from the hardware */ ++ kbase_gpuprops_get_props(&gpu_props->props, kbdev); ++ ++ /* Populate the derived properties */ ++ kbase_gpuprops_calculate_props(&gpu_props->props, kbdev); ++ ++ /* Populate kbase-only fields */ ++ gpu_props->l2_props.associativity = KBASE_UBFX32(raw->l2_features, 8U, 8); ++ gpu_props->l2_props.external_bus_width = KBASE_UBFX32(raw->l2_features, 24U, 8); ++ ++ gpu_props->mem.core_group = KBASE_UBFX32(raw->mem_features, 0U, 1); ++ ++ gpu_props->mmu.va_bits = KBASE_UBFX32(raw->mmu_features, 0U, 8); ++ gpu_props->mmu.pa_bits = KBASE_UBFX32(raw->mmu_features, 8U, 8); ++ ++ gpu_props->num_cores = hweight64(raw->shader_present); ++ gpu_props->num_core_groups = hweight64(raw->l2_present); ++ gpu_props->num_address_spaces = hweight32(raw->as_present); ++ gpu_props->num_job_slots = hweight32(raw->js_present); ++} ++ ++int kbase_gpuprops_set_features(struct kbase_device *kbdev) ++{ ++ struct base_gpu_props *gpu_props; ++ struct kbase_gpuprops_regdump regdump; ++ int err; ++ ++ gpu_props = &kbdev->gpu_props.props; ++ ++ /* Dump relevant registers */ ++ err = kbase_backend_gpuprops_get_features(kbdev, ®dump); ++ if (err) ++ return err; ++ ++ /* ++ * Copy the raw value from the register, later this will get turned ++ * into the selected coherency mode. ++ * Additionally, add non-coherent mode, as this is always supported. ++ */ ++ gpu_props->raw_props.coherency_mode = regdump.coherency_features | ++ COHERENCY_FEATURE_BIT(COHERENCY_NONE); ++ ++ if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_THREAD_GROUP_SPLIT)) ++ gpu_props->thread_props.max_thread_group_split = 0; ++ ++ return err; ++} ++ ++/* ++ * Module parameters to allow the L2 size and hash configuration to be ++ * overridden. ++ * ++ * These parameters must be set on insmod to take effect, and are not visible ++ * in sysfs. ++ */ ++static u8 override_l2_size; ++module_param(override_l2_size, byte, 0); ++MODULE_PARM_DESC(override_l2_size, "Override L2 size config for testing"); ++ ++static u8 override_l2_hash; ++module_param(override_l2_hash, byte, 0); ++MODULE_PARM_DESC(override_l2_hash, "Override L2 hash config for testing"); ++ ++/** ++ * kbase_read_l2_config_from_dt - Read L2 configuration ++ * @kbdev: The kbase device for which to get the L2 configuration. ++ * ++ * Check for L2 configuration overrides in module parameters and device tree. ++ * Override values in module parameters take priority over override values in ++ * device tree. ++ * ++ * Return: true if either size or hash was overridden, false if no overrides ++ * were found. ++ */ ++static bool kbase_read_l2_config_from_dt(struct kbase_device * const kbdev) ++{ ++ struct device_node *np = kbdev->dev->of_node; ++ ++ if (!np) ++ return false; ++ ++ if (override_l2_size) ++ kbdev->l2_size_override = override_l2_size; ++ else if (of_property_read_u8(np, "l2-size", &kbdev->l2_size_override)) ++ kbdev->l2_size_override = 0; ++ ++ if (override_l2_hash) ++ kbdev->l2_hash_override = override_l2_hash; ++ else if (of_property_read_u8(np, "l2-hash", &kbdev->l2_hash_override)) ++ kbdev->l2_hash_override = 0; ++ ++ if (kbdev->l2_size_override || kbdev->l2_hash_override) ++ return true; ++ ++ return false; ++} ++ ++int kbase_gpuprops_update_l2_features(struct kbase_device *kbdev) ++{ ++ int err = 0; ++ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_L2_CONFIG)) { ++ struct kbase_gpuprops_regdump regdump; ++ struct base_gpu_props *gpu_props = &kbdev->gpu_props.props; ++ ++ /* Check for L2 cache size & hash overrides */ ++ if (!kbase_read_l2_config_from_dt(kbdev)) ++ return 0; ++ ++ /* Need L2 to get powered to reflect to L2_FEATURES */ ++ kbase_pm_context_active(kbdev); ++ ++ /* Wait for the completion of L2 power transition */ ++ kbase_pm_wait_for_l2_powered(kbdev); ++ ++ /* Dump L2_FEATURES register */ ++ err = kbase_backend_gpuprops_get_l2_features(kbdev, ®dump); ++ if (err) ++ goto idle_gpu; ++ ++ dev_info(kbdev->dev, "Reflected L2_FEATURES is 0x%x\n", ++ regdump.l2_features); ++ ++ /* Update gpuprops with reflected L2_FEATURES */ ++ gpu_props->raw_props.l2_features = regdump.l2_features; ++ gpu_props->l2_props.log2_cache_size = ++ KBASE_UBFX32(gpu_props->raw_props.l2_features, 16U, 8); ++ ++idle_gpu: ++ /* Let GPU idle */ ++ kbase_pm_context_idle(kbdev); ++ } ++ ++ return err; ++} ++ ++static struct { ++ u32 type; ++ size_t offset; ++ int size; ++} gpu_property_mapping[] = { ++#define PROP(name, member) \ ++ {KBASE_GPUPROP_ ## name, offsetof(struct base_gpu_props, member), \ ++ sizeof(((struct base_gpu_props *)0)->member)} ++ PROP(PRODUCT_ID, core_props.product_id), ++ PROP(VERSION_STATUS, core_props.version_status), ++ PROP(MINOR_REVISION, core_props.minor_revision), ++ PROP(MAJOR_REVISION, core_props.major_revision), ++ PROP(GPU_FREQ_KHZ_MAX, core_props.gpu_freq_khz_max), ++ PROP(LOG2_PROGRAM_COUNTER_SIZE, core_props.log2_program_counter_size), ++ PROP(TEXTURE_FEATURES_0, core_props.texture_features[0]), ++ PROP(TEXTURE_FEATURES_1, core_props.texture_features[1]), ++ PROP(TEXTURE_FEATURES_2, core_props.texture_features[2]), ++ PROP(TEXTURE_FEATURES_3, core_props.texture_features[3]), ++ PROP(GPU_AVAILABLE_MEMORY_SIZE, core_props.gpu_available_memory_size), ++ PROP(NUM_EXEC_ENGINES, core_props.num_exec_engines), ++ ++ PROP(L2_LOG2_LINE_SIZE, l2_props.log2_line_size), ++ PROP(L2_LOG2_CACHE_SIZE, l2_props.log2_cache_size), ++ PROP(L2_NUM_L2_SLICES, l2_props.num_l2_slices), ++ ++ PROP(TILER_BIN_SIZE_BYTES, tiler_props.bin_size_bytes), ++ PROP(TILER_MAX_ACTIVE_LEVELS, tiler_props.max_active_levels), ++ ++ PROP(MAX_THREADS, thread_props.max_threads), ++ PROP(MAX_WORKGROUP_SIZE, thread_props.max_workgroup_size), ++ PROP(MAX_BARRIER_SIZE, thread_props.max_barrier_size), ++ PROP(MAX_REGISTERS, thread_props.max_registers), ++ PROP(MAX_TASK_QUEUE, thread_props.max_task_queue), ++ PROP(MAX_THREAD_GROUP_SPLIT, thread_props.max_thread_group_split), ++ PROP(IMPL_TECH, thread_props.impl_tech), ++ PROP(TLS_ALLOC, thread_props.tls_alloc), ++ ++ PROP(RAW_SHADER_PRESENT, raw_props.shader_present), ++ PROP(RAW_TILER_PRESENT, raw_props.tiler_present), ++ PROP(RAW_L2_PRESENT, raw_props.l2_present), ++ PROP(RAW_STACK_PRESENT, raw_props.stack_present), ++ PROP(RAW_L2_FEATURES, raw_props.l2_features), ++ PROP(RAW_CORE_FEATURES, raw_props.core_features), ++ PROP(RAW_MEM_FEATURES, raw_props.mem_features), ++ PROP(RAW_MMU_FEATURES, raw_props.mmu_features), ++ PROP(RAW_AS_PRESENT, raw_props.as_present), ++ PROP(RAW_JS_PRESENT, raw_props.js_present), ++ PROP(RAW_JS_FEATURES_0, raw_props.js_features[0]), ++ PROP(RAW_JS_FEATURES_1, raw_props.js_features[1]), ++ PROP(RAW_JS_FEATURES_2, raw_props.js_features[2]), ++ PROP(RAW_JS_FEATURES_3, raw_props.js_features[3]), ++ PROP(RAW_JS_FEATURES_4, raw_props.js_features[4]), ++ PROP(RAW_JS_FEATURES_5, raw_props.js_features[5]), ++ PROP(RAW_JS_FEATURES_6, raw_props.js_features[6]), ++ PROP(RAW_JS_FEATURES_7, raw_props.js_features[7]), ++ PROP(RAW_JS_FEATURES_8, raw_props.js_features[8]), ++ PROP(RAW_JS_FEATURES_9, raw_props.js_features[9]), ++ PROP(RAW_JS_FEATURES_10, raw_props.js_features[10]), ++ PROP(RAW_JS_FEATURES_11, raw_props.js_features[11]), ++ PROP(RAW_JS_FEATURES_12, raw_props.js_features[12]), ++ PROP(RAW_JS_FEATURES_13, raw_props.js_features[13]), ++ PROP(RAW_JS_FEATURES_14, raw_props.js_features[14]), ++ PROP(RAW_JS_FEATURES_15, raw_props.js_features[15]), ++ PROP(RAW_TILER_FEATURES, raw_props.tiler_features), ++ PROP(RAW_TEXTURE_FEATURES_0, raw_props.texture_features[0]), ++ PROP(RAW_TEXTURE_FEATURES_1, raw_props.texture_features[1]), ++ PROP(RAW_TEXTURE_FEATURES_2, raw_props.texture_features[2]), ++ PROP(RAW_TEXTURE_FEATURES_3, raw_props.texture_features[3]), ++ PROP(RAW_GPU_ID, raw_props.gpu_id), ++ PROP(RAW_THREAD_MAX_THREADS, raw_props.thread_max_threads), ++ PROP(RAW_THREAD_MAX_WORKGROUP_SIZE, ++ raw_props.thread_max_workgroup_size), ++ PROP(RAW_THREAD_MAX_BARRIER_SIZE, raw_props.thread_max_barrier_size), ++ PROP(RAW_THREAD_FEATURES, raw_props.thread_features), ++ PROP(RAW_THREAD_TLS_ALLOC, raw_props.thread_tls_alloc), ++ PROP(RAW_COHERENCY_MODE, raw_props.coherency_mode), ++ ++ PROP(COHERENCY_NUM_GROUPS, coherency_info.num_groups), ++ PROP(COHERENCY_NUM_CORE_GROUPS, coherency_info.num_core_groups), ++ PROP(COHERENCY_COHERENCY, coherency_info.coherency), ++ PROP(COHERENCY_GROUP_0, coherency_info.group[0].core_mask), ++ PROP(COHERENCY_GROUP_1, coherency_info.group[1].core_mask), ++ PROP(COHERENCY_GROUP_2, coherency_info.group[2].core_mask), ++ PROP(COHERENCY_GROUP_3, coherency_info.group[3].core_mask), ++ PROP(COHERENCY_GROUP_4, coherency_info.group[4].core_mask), ++ PROP(COHERENCY_GROUP_5, coherency_info.group[5].core_mask), ++ PROP(COHERENCY_GROUP_6, coherency_info.group[6].core_mask), ++ PROP(COHERENCY_GROUP_7, coherency_info.group[7].core_mask), ++ PROP(COHERENCY_GROUP_8, coherency_info.group[8].core_mask), ++ PROP(COHERENCY_GROUP_9, coherency_info.group[9].core_mask), ++ PROP(COHERENCY_GROUP_10, coherency_info.group[10].core_mask), ++ PROP(COHERENCY_GROUP_11, coherency_info.group[11].core_mask), ++ PROP(COHERENCY_GROUP_12, coherency_info.group[12].core_mask), ++ PROP(COHERENCY_GROUP_13, coherency_info.group[13].core_mask), ++ PROP(COHERENCY_GROUP_14, coherency_info.group[14].core_mask), ++ PROP(COHERENCY_GROUP_15, coherency_info.group[15].core_mask), ++ ++#undef PROP ++}; ++ ++int kbase_gpuprops_populate_user_buffer(struct kbase_device *kbdev) ++{ ++ struct kbase_gpu_props *kprops = &kbdev->gpu_props; ++ struct base_gpu_props *props = &kprops->props; ++ u32 count = ARRAY_SIZE(gpu_property_mapping); ++ u32 i; ++ u32 size = 0; ++ u8 *p; ++ ++ for (i = 0; i < count; i++) { ++ /* 4 bytes for the ID, and the size of the property */ ++ size += 4 + gpu_property_mapping[i].size; ++ } ++ ++ kprops->prop_buffer_size = size; ++ kprops->prop_buffer = kmalloc(size, GFP_KERNEL); ++ ++ if (!kprops->prop_buffer) { ++ kprops->prop_buffer_size = 0; ++ return -ENOMEM; ++ } ++ ++ p = kprops->prop_buffer; ++ ++#define WRITE_U8(v) (*p++ = (v) & 0xFF) ++#define WRITE_U16(v) do { WRITE_U8(v); WRITE_U8((v) >> 8); } while (0) ++#define WRITE_U32(v) do { WRITE_U16(v); WRITE_U16((v) >> 16); } while (0) ++#define WRITE_U64(v) do { WRITE_U32(v); WRITE_U32((v) >> 32); } while (0) ++ ++ for (i = 0; i < count; i++) { ++ u32 type = gpu_property_mapping[i].type; ++ u8 type_size; ++ void *field = ((u8 *)props) + gpu_property_mapping[i].offset; ++ ++ switch (gpu_property_mapping[i].size) { ++ case 1: ++ type_size = KBASE_GPUPROP_VALUE_SIZE_U8; ++ break; ++ case 2: ++ type_size = KBASE_GPUPROP_VALUE_SIZE_U16; ++ break; ++ case 4: ++ type_size = KBASE_GPUPROP_VALUE_SIZE_U32; ++ break; ++ case 8: ++ type_size = KBASE_GPUPROP_VALUE_SIZE_U64; ++ break; ++ default: ++ dev_err(kbdev->dev, ++ "Invalid gpu_property_mapping type=%d size=%d", ++ type, gpu_property_mapping[i].size); ++ return -EINVAL; ++ } ++ ++ WRITE_U32((type<<2) | type_size); ++ ++ switch (type_size) { ++ case KBASE_GPUPROP_VALUE_SIZE_U8: ++ WRITE_U8(*((u8 *)field)); ++ break; ++ case KBASE_GPUPROP_VALUE_SIZE_U16: ++ WRITE_U16(*((u16 *)field)); ++ break; ++ case KBASE_GPUPROP_VALUE_SIZE_U32: ++ WRITE_U32(*((u32 *)field)); ++ break; ++ case KBASE_GPUPROP_VALUE_SIZE_U64: ++ WRITE_U64(*((u64 *)field)); ++ break; ++ default: /* Cannot be reached */ ++ WARN_ON(1); ++ return -EINVAL; ++ } ++ } ++ ++ return 0; ++} ++ ++void kbase_gpuprops_free_user_buffer(struct kbase_device *kbdev) ++{ ++ kfree(kbdev->gpu_props.prop_buffer); ++} ++ ++int kbase_device_populate_max_freq(struct kbase_device *kbdev) ++{ ++ struct mali_base_gpu_core_props *core_props; ++ ++ /* obtain max configured gpu frequency, if devfreq is enabled then ++ * this will be overridden by the highest operating point found ++ */ ++ core_props = &(kbdev->gpu_props.props.core_props); ++#ifdef GPU_FREQ_KHZ_MAX ++ core_props->gpu_freq_khz_max = GPU_FREQ_KHZ_MAX; ++#else ++ core_props->gpu_freq_khz_max = DEFAULT_GPU_FREQ_KHZ_MAX; ++#endif ++ ++ return 0; ++} +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_gpuprops.h b/drivers/gpu/arm/bifrost/mali_kbase_gpuprops.h +new file mode 100755 +index 000000000000..5eee7948381a +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_gpuprops.h +@@ -0,0 +1,135 @@ ++/* ++ * ++ * (C) COPYRIGHT ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ *//* SPDX-License-Identifier: GPL-2.0 */ ++/* ++ * ++ * (C) COPYRIGHT 2011-2015, 2017, 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/** ++ * @file mali_kbase_gpuprops.h ++ * Base kernel property query APIs ++ */ ++ ++#ifndef _KBASE_GPUPROPS_H_ ++#define _KBASE_GPUPROPS_H_ ++ ++#include "mali_kbase_gpuprops_types.h" ++ ++/* Forward definition - see mali_kbase.h */ ++struct kbase_device; ++ ++/** ++ * KBASE_UBFX32 - Extracts bits from a 32-bit bitfield. ++ * @value: The value from which to extract bits. ++ * @offset: The first bit to extract (0 being the LSB). ++ * @size: The number of bits to extract. ++ * ++ * Context: @offset + @size <= 32. ++ * ++ * Return: Bits [@offset, @offset + @size) from @value. ++ */ ++/* from mali_cdsb.h */ ++#define KBASE_UBFX32(value, offset, size) \ ++ (((u32)(value) >> (u32)(offset)) & (u32)((1ULL << (u32)(size)) - 1)) ++ ++/** ++ * @brief Set up Kbase GPU properties. ++ * ++ * Set up Kbase GPU properties with information from the GPU registers ++ * ++ * @param kbdev The struct kbase_device structure for the device ++ */ ++void kbase_gpuprops_set(struct kbase_device *kbdev); ++ ++/** ++ * kbase_gpuprops_set_features - Set up Kbase GPU properties ++ * @kbdev: Device pointer ++ * ++ * This function sets up GPU properties that are dependent on the hardware ++ * features bitmask. This function must be preceeded by a call to ++ * kbase_hw_set_features_mask(). ++ * ++ * Return: Zero on success, Linux error code on failure ++ */ ++int kbase_gpuprops_set_features(struct kbase_device *kbdev); ++ ++/** ++ * kbase_gpuprops_update_l2_features - Update GPU property of L2_FEATURES ++ * @kbdev: Device pointer ++ * ++ * This function updates l2_features and the log2 cache size. ++ * ++ * Return: Zero on success, Linux error code for failure ++ */ ++int kbase_gpuprops_update_l2_features(struct kbase_device *kbdev); ++ ++/** ++ * kbase_gpuprops_populate_user_buffer - Populate the GPU properties buffer ++ * @kbdev: The kbase device ++ * ++ * Fills prop_buffer with the GPU properties for user space to read. ++ */ ++int kbase_gpuprops_populate_user_buffer(struct kbase_device *kbdev); ++ ++/** ++ * kbase_gpuprops_free_user_buffer - Free the GPU properties buffer. ++ * @kbdev: kbase device pointer ++ * ++ * Free the GPU properties buffer allocated from ++ * kbase_gpuprops_populate_user_buffer. ++ */ ++void kbase_gpuprops_free_user_buffer(struct kbase_device *kbdev); ++ ++/** ++ * kbase_device_populate_max_freq - Populate max gpu frequency. ++ * @kbdev: kbase device pointer ++ * ++ * Populate the maximum gpu frequency to be used when devfreq is disabled. ++ * ++ * Return: 0 on success and non-zero value on failure. ++ */ ++int kbase_device_populate_max_freq(struct kbase_device *kbdev); ++ ++/** ++ * kbase_gpuprops_update_core_props_gpu_id - break down gpu id value ++ * @gpu_props: the &base_gpu_props structure ++ * ++ * Break down gpu_id value stored in base_gpu_props::raw_props.gpu_id into ++ * separate fields (version_status, minor_revision, major_revision, product_id) ++ * stored in base_gpu_props::core_props. ++ */ ++void kbase_gpuprops_update_core_props_gpu_id( ++ struct base_gpu_props * const gpu_props); ++ ++#endif /* _KBASE_GPUPROPS_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_gpuprops_types.h b/drivers/gpu/arm/bifrost/mali_kbase_gpuprops_types.h +new file mode 100755 +index 000000000000..ec6f1c39ccb0 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_gpuprops_types.h +@@ -0,0 +1,98 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2018, 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++/** ++ * @file mali_kbase_gpuprops_types.h ++ * Base kernel property query APIs ++ */ ++ ++#ifndef _KBASE_GPUPROPS_TYPES_H_ ++#define _KBASE_GPUPROPS_TYPES_H_ ++ ++#include "mali_base_kernel.h" ++ ++#define KBASE_GPU_SPEED_MHZ 123 ++#define KBASE_GPU_PC_SIZE_LOG2 24U ++ ++struct kbase_gpuprops_regdump { ++ u32 gpu_id; ++ u32 l2_features; ++ u32 core_features; ++ u32 tiler_features; ++ u32 mem_features; ++ u32 mmu_features; ++ u32 as_present; ++ u32 js_present; ++ u32 thread_max_threads; ++ u32 thread_max_workgroup_size; ++ u32 thread_max_barrier_size; ++ u32 thread_features; ++ u32 thread_tls_alloc; ++ u32 texture_features[BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS]; ++ u32 js_features[GPU_MAX_JOB_SLOTS]; ++ u32 shader_present_lo; ++ u32 shader_present_hi; ++ u32 tiler_present_lo; ++ u32 tiler_present_hi; ++ u32 l2_present_lo; ++ u32 l2_present_hi; ++ u32 stack_present_lo; ++ u32 stack_present_hi; ++ u32 coherency_features; ++}; ++ ++struct kbase_gpu_cache_props { ++ u8 associativity; ++ u8 external_bus_width; ++}; ++ ++struct kbase_gpu_mem_props { ++ u8 core_group; ++}; ++ ++struct kbase_gpu_mmu_props { ++ u8 va_bits; ++ u8 pa_bits; ++}; ++ ++struct kbase_gpu_props { ++ /* kernel-only properties */ ++ u8 num_cores; ++ u8 num_core_groups; ++ u8 num_address_spaces; ++ u8 num_job_slots; ++ ++ struct kbase_gpu_cache_props l2_props; ++ ++ struct kbase_gpu_mem_props mem; ++ struct kbase_gpu_mmu_props mmu; ++ ++ /* Properties shared with userspace */ ++ struct base_gpu_props props; ++ ++ u32 prop_buffer_size; ++ void *prop_buffer; ++}; ++ ++#endif /* _KBASE_GPUPROPS_TYPES_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_gwt.c b/drivers/gpu/arm/bifrost/mali_kbase_gwt.c +new file mode 100755 +index 000000000000..91dc4dbc0800 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_gwt.c +@@ -0,0 +1,273 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include "mali_kbase_gwt.h" ++#include ++ ++static inline void kbase_gpu_gwt_setup_page_permission( ++ struct kbase_context *kctx, ++ unsigned long flag, ++ struct rb_node *node) ++{ ++ struct rb_node *rbnode = node; ++ ++ while (rbnode) { ++ struct kbase_va_region *reg; ++ int err = 0; ++ ++ reg = rb_entry(rbnode, struct kbase_va_region, rblink); ++ if (reg->nr_pages && !kbase_is_region_invalid_or_free(reg) && ++ (reg->flags & KBASE_REG_GPU_WR)) { ++ err = kbase_mmu_update_pages(kctx, reg->start_pfn, ++ kbase_get_gpu_phy_pages(reg), ++ reg->gpu_alloc->nents, ++ reg->flags & flag, ++ reg->gpu_alloc->group_id); ++ if (err) ++ dev_warn(kctx->kbdev->dev, "kbase_mmu_update_pages failure\n"); ++ } ++ ++ rbnode = rb_next(rbnode); ++ } ++} ++ ++static void kbase_gpu_gwt_setup_pages(struct kbase_context *kctx, ++ unsigned long flag) ++{ ++ kbase_gpu_gwt_setup_page_permission(kctx, flag, ++ rb_first(&(kctx->reg_rbtree_same))); ++ kbase_gpu_gwt_setup_page_permission(kctx, flag, ++ rb_first(&(kctx->reg_rbtree_custom))); ++} ++ ++ ++int kbase_gpu_gwt_start(struct kbase_context *kctx) ++{ ++ kbase_gpu_vm_lock(kctx); ++ if (kctx->gwt_enabled) { ++ kbase_gpu_vm_unlock(kctx); ++ return -EBUSY; ++ } ++ ++ INIT_LIST_HEAD(&kctx->gwt_current_list); ++ INIT_LIST_HEAD(&kctx->gwt_snapshot_list); ++ ++#if !MALI_USE_CSF ++ /* If GWT is enabled using new vector dumping format ++ * from user space, back up status of the job serialization flag and ++ * use full serialisation of jobs for dumping. ++ * Status will be restored on end of dumping in gwt_stop. ++ */ ++ kctx->kbdev->backup_serialize_jobs = kctx->kbdev->serialize_jobs; ++ kctx->kbdev->serialize_jobs = KBASE_SERIALIZE_INTRA_SLOT | ++ KBASE_SERIALIZE_INTER_SLOT; ++ ++#endif ++ /* Mark gwt enabled before making pages read only in case a ++ write page fault is triggered while we're still in this loop. ++ (kbase_gpu_vm_lock() doesn't prevent this!) ++ */ ++ kctx->gwt_enabled = true; ++ kctx->gwt_was_enabled = true; ++ ++ kbase_gpu_gwt_setup_pages(kctx, ~KBASE_REG_GPU_WR); ++ ++ kbase_gpu_vm_unlock(kctx); ++ return 0; ++} ++ ++int kbase_gpu_gwt_stop(struct kbase_context *kctx) ++{ ++ struct kbasep_gwt_list_element *pos, *n; ++ ++ kbase_gpu_vm_lock(kctx); ++ if (!kctx->gwt_enabled) { ++ kbase_gpu_vm_unlock(kctx); ++ return -EINVAL; ++ } ++ ++ list_for_each_entry_safe(pos, n, &kctx->gwt_current_list, link) { ++ list_del(&pos->link); ++ kfree(pos); ++ } ++ ++ list_for_each_entry_safe(pos, n, &kctx->gwt_snapshot_list, link) { ++ list_del(&pos->link); ++ kfree(pos); ++ } ++ ++#if !MALI_USE_CSF ++ kctx->kbdev->serialize_jobs = kctx->kbdev->backup_serialize_jobs; ++#endif ++ ++ kbase_gpu_gwt_setup_pages(kctx, ~0UL); ++ ++ kctx->gwt_enabled = false; ++ kbase_gpu_vm_unlock(kctx); ++ return 0; ++} ++ ++ ++static int list_cmp_function(void *priv, struct list_head *a, ++ struct list_head *b) ++{ ++ struct kbasep_gwt_list_element *elementA = container_of(a, ++ struct kbasep_gwt_list_element, link); ++ struct kbasep_gwt_list_element *elementB = container_of(b, ++ struct kbasep_gwt_list_element, link); ++ ++ CSTD_UNUSED(priv); ++ ++ if (elementA->page_addr > elementB->page_addr) ++ return 1; ++ return -1; ++} ++ ++static void kbase_gpu_gwt_collate(struct kbase_context *kctx, ++ struct list_head *snapshot_list) ++{ ++ struct kbasep_gwt_list_element *pos, *n; ++ struct kbasep_gwt_list_element *collated = NULL; ++ ++ /* Sort the list */ ++ list_sort(NULL, snapshot_list, list_cmp_function); ++ ++ /* Combine contiguous areas. */ ++ list_for_each_entry_safe(pos, n, snapshot_list, link) { ++ if (collated == NULL || collated->region != ++ pos->region || ++ (collated->page_addr + ++ (collated->num_pages * PAGE_SIZE)) != ++ pos->page_addr) { ++ /* This is the first time through, a new region or ++ * is not contiguous - start collating to this element ++ */ ++ collated = pos; ++ } else { ++ /* contiguous so merge */ ++ collated->num_pages += pos->num_pages; ++ /* remove element from list */ ++ list_del(&pos->link); ++ kfree(pos); ++ } ++ } ++} ++ ++int kbase_gpu_gwt_dump(struct kbase_context *kctx, ++ union kbase_ioctl_cinstr_gwt_dump *gwt_dump) ++{ ++ const u32 ubuf_size = gwt_dump->in.len; ++ u32 ubuf_count = 0; ++ __user void *user_addr = (__user void *) ++ (uintptr_t)gwt_dump->in.addr_buffer; ++ __user void *user_sizes = (__user void *) ++ (uintptr_t)gwt_dump->in.size_buffer; ++ ++ kbase_gpu_vm_lock(kctx); ++ ++ if (!kctx->gwt_enabled) { ++ kbase_gpu_vm_unlock(kctx); ++ /* gwt_dump shouldn't be called when gwt is disabled */ ++ return -EPERM; ++ } ++ ++ if (!gwt_dump->in.len || !gwt_dump->in.addr_buffer ++ || !gwt_dump->in.size_buffer) { ++ kbase_gpu_vm_unlock(kctx); ++ /* We don't have any valid user space buffer to copy the ++ * write modified addresses. ++ */ ++ return -EINVAL; ++ } ++ ++ if (list_empty(&kctx->gwt_snapshot_list) && ++ !list_empty(&kctx->gwt_current_list)) { ++ ++ list_replace_init(&kctx->gwt_current_list, ++ &kctx->gwt_snapshot_list); ++ ++ /* We have collected all write faults so far ++ * and they will be passed on to user space. ++ * Reset the page flags state to allow collection of ++ * further write faults. ++ */ ++ kbase_gpu_gwt_setup_pages(kctx, ~KBASE_REG_GPU_WR); ++ ++ /* Sort and combine consecutive pages in the dump list*/ ++ kbase_gpu_gwt_collate(kctx, &kctx->gwt_snapshot_list); ++ } ++ ++ while ((!list_empty(&kctx->gwt_snapshot_list))) { ++ u64 addr_buffer[32]; ++ u64 num_page_buffer[32]; ++ u32 count = 0; ++ int err; ++ struct kbasep_gwt_list_element *dump_info, *n; ++ ++ list_for_each_entry_safe(dump_info, n, ++ &kctx->gwt_snapshot_list, link) { ++ addr_buffer[count] = dump_info->page_addr; ++ num_page_buffer[count] = dump_info->num_pages; ++ count++; ++ list_del(&dump_info->link); ++ kfree(dump_info); ++ if (ARRAY_SIZE(addr_buffer) == count || ++ ubuf_size == (ubuf_count + count)) ++ break; ++ } ++ ++ if (count) { ++ err = copy_to_user((user_addr + ++ (ubuf_count * sizeof(u64))), ++ (void *)addr_buffer, ++ count * sizeof(u64)); ++ if (err) { ++ dev_err(kctx->kbdev->dev, "Copy to user failure\n"); ++ kbase_gpu_vm_unlock(kctx); ++ return err; ++ } ++ err = copy_to_user((user_sizes + ++ (ubuf_count * sizeof(u64))), ++ (void *)num_page_buffer, ++ count * sizeof(u64)); ++ if (err) { ++ dev_err(kctx->kbdev->dev, "Copy to user failure\n"); ++ kbase_gpu_vm_unlock(kctx); ++ return err; ++ } ++ ++ ubuf_count += count; ++ } ++ ++ if (ubuf_count == ubuf_size) ++ break; ++ } ++ ++ if (!list_empty(&kctx->gwt_snapshot_list)) ++ gwt_dump->out.more_data_available = 1; ++ else ++ gwt_dump->out.more_data_available = 0; ++ ++ gwt_dump->out.no_of_addr_collected = ubuf_count; ++ kbase_gpu_vm_unlock(kctx); ++ return 0; ++} +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_gwt.h b/drivers/gpu/arm/bifrost/mali_kbase_gwt.h +new file mode 100755 +index 000000000000..7e7746e64915 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_gwt.h +@@ -0,0 +1,55 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#if !defined(_KBASE_GWT_H) ++#define _KBASE_GWT_H ++ ++#include ++#include ++ ++/** ++ * kbase_gpu_gwt_start - Start the GPU write tracking ++ * @kctx: Pointer to kernel context ++ * ++ * @return 0 on success, error on failure. ++ */ ++int kbase_gpu_gwt_start(struct kbase_context *kctx); ++ ++/** ++ * kbase_gpu_gwt_stop - Stop the GPU write tracking ++ * @kctx: Pointer to kernel context ++ * ++ * @return 0 on success, error on failure. ++ */ ++int kbase_gpu_gwt_stop(struct kbase_context *kctx); ++ ++/** ++ * kbase_gpu_gwt_dump - Pass page address of faulting addresses to user space. ++ * @kctx: Pointer to kernel context ++ * @gwt_dump: User space data to be passed. ++ * ++ * @return 0 on success, error on failure. ++ */ ++int kbase_gpu_gwt_dump(struct kbase_context *kctx, ++ union kbase_ioctl_cinstr_gwt_dump *gwt_dump); ++ ++#endif /* _KBASE_GWT_H */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hw.c b/drivers/gpu/arm/bifrost/mali_kbase_hw.c +new file mode 100755 +index 000000000000..dc58ffb931be +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_hw.c +@@ -0,0 +1,437 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++/* ++ * Run-time work-arounds helpers ++ */ ++ ++#include ++#include ++#include "gpu/mali_kbase_gpu_regmap.h" ++#include "mali_kbase.h" ++#include "mali_kbase_hw.h" ++ ++void kbase_hw_set_features_mask(struct kbase_device *kbdev) ++{ ++ const enum base_hw_feature *features; ++ u32 gpu_id; ++ ++ gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; ++ ++ switch (gpu_id & GPU_ID2_PRODUCT_MODEL) { ++ case GPU_ID2_PRODUCT_TMIX: ++ features = base_hw_features_tMIx; ++ break; ++ case GPU_ID2_PRODUCT_THEX: ++ features = base_hw_features_tHEx; ++ break; ++ case GPU_ID2_PRODUCT_TSIX: ++ features = base_hw_features_tSIx; ++ break; ++ case GPU_ID2_PRODUCT_TDVX: ++ features = base_hw_features_tDVx; ++ break; ++ case GPU_ID2_PRODUCT_TNOX: ++ features = base_hw_features_tNOx; ++ break; ++ case GPU_ID2_PRODUCT_TGOX: ++ features = base_hw_features_tGOx; ++ break; ++ case GPU_ID2_PRODUCT_TTRX: ++ features = base_hw_features_tTRx; ++ break; ++ case GPU_ID2_PRODUCT_TNAX: ++ features = base_hw_features_tNAx; ++ break; ++ case GPU_ID2_PRODUCT_LBEX: ++ case GPU_ID2_PRODUCT_TBEX: ++ features = base_hw_features_tBEx; ++ break; ++ case GPU_ID2_PRODUCT_TBAX: ++ features = base_hw_features_tBAx; ++ break; ++ case GPU_ID2_PRODUCT_TDUX: ++ features = base_hw_features_tDUx; ++ break; ++ case GPU_ID2_PRODUCT_TODX: ++ case GPU_ID2_PRODUCT_LODX: ++ features = base_hw_features_tODx; ++ break; ++ case GPU_ID2_PRODUCT_TGRX: ++ features = base_hw_features_tGRx; ++ break; ++ case GPU_ID2_PRODUCT_TVAX: ++ features = base_hw_features_tVAx; ++ break; ++ case GPU_ID2_PRODUCT_TTUX: ++ /* Fallthrough */ ++ case GPU_ID2_PRODUCT_LTUX: ++ features = base_hw_features_tTUx; ++ break; ++ case GPU_ID2_PRODUCT_TE2X: ++ features = base_hw_features_tE2x; ++ break; ++ default: ++ features = base_hw_features_generic; ++ break; ++ } ++ ++ for (; *features != BASE_HW_FEATURE_END; features++) ++ set_bit(*features, &kbdev->hw_features_mask[0]); ++ ++#if defined(CONFIG_MALI_VECTOR_DUMP) ++ /* When dumping is enabled, need to disable flush reduction optimization ++ * for GPUs on which it is safe to have only cache clean operation at ++ * the end of job chain. ++ * This is required to make vector dump work. There is some discrepancy ++ * in the implementation of flush reduction optimization due to ++ * unclear or ambiguous ARCH spec. ++ */ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_CLEAN_ONLY_SAFE)) ++ clear_bit(BASE_HW_FEATURE_FLUSH_REDUCTION, ++ &kbdev->hw_features_mask[0]); ++#endif ++} ++ ++/** ++ * kbase_hw_get_issues_for_new_id - Get the hardware issues for a new GPU ID ++ * @kbdev: Device pointer ++ * ++ * Return: pointer to an array of hardware issues, terminated by ++ * BASE_HW_ISSUE_END. ++ * ++ * In debugging versions of the driver, unknown versions of a known GPU will ++ * be treated as the most recent known version not later than the actual ++ * version. In such circumstances, the GPU ID in @kbdev will also be replaced ++ * with the most recent known version. ++ * ++ * Note: The GPU configuration must have been read by kbase_gpuprops_get_props() ++ * before calling this function. ++ */ ++static const enum base_hw_issue *kbase_hw_get_issues_for_new_id( ++ struct kbase_device *kbdev) ++{ ++ const enum base_hw_issue *issues = NULL; ++ ++ struct base_hw_product { ++ u32 product_model; ++ struct { ++ u32 version; ++ const enum base_hw_issue *issues; ++ } map[7]; ++ }; ++ ++ static const struct base_hw_product base_hw_products[] = { ++ {GPU_ID2_PRODUCT_TMIX, ++ {{GPU_ID2_VERSION_MAKE(0, 0, 1), ++ base_hw_issues_tMIx_r0p0_05dev0}, ++ {GPU_ID2_VERSION_MAKE(0, 0, 2), base_hw_issues_tMIx_r0p0}, ++ {GPU_ID2_VERSION_MAKE(0, 1, 0), base_hw_issues_tMIx_r0p1}, ++ {U32_MAX /* sentinel value */, NULL} } }, ++ ++ {GPU_ID2_PRODUCT_THEX, ++ {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tHEx_r0p0}, ++ {GPU_ID2_VERSION_MAKE(0, 0, 1), base_hw_issues_tHEx_r0p0}, ++ {GPU_ID2_VERSION_MAKE(0, 1, 0), base_hw_issues_tHEx_r0p1}, ++ {GPU_ID2_VERSION_MAKE(0, 1, 1), base_hw_issues_tHEx_r0p1}, ++ {GPU_ID2_VERSION_MAKE(0, 2, 0), base_hw_issues_tHEx_r0p2}, ++ {GPU_ID2_VERSION_MAKE(0, 3, 0), base_hw_issues_tHEx_r0p3}, ++ {U32_MAX, NULL} } }, ++ ++ {GPU_ID2_PRODUCT_TSIX, ++ {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tSIx_r0p0}, ++ {GPU_ID2_VERSION_MAKE(0, 0, 1), base_hw_issues_tSIx_r0p0}, ++ {GPU_ID2_VERSION_MAKE(0, 1, 0), base_hw_issues_tSIx_r0p1}, ++ {GPU_ID2_VERSION_MAKE(1, 0, 0), base_hw_issues_tSIx_r1p0}, ++ {GPU_ID2_VERSION_MAKE(1, 1, 0), base_hw_issues_tSIx_r1p1}, ++ {U32_MAX, NULL} } }, ++ ++ {GPU_ID2_PRODUCT_TDVX, ++ {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tDVx_r0p0}, ++ {U32_MAX, NULL} } }, ++ ++ {GPU_ID2_PRODUCT_TNOX, ++ {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tNOx_r0p0}, ++ {U32_MAX, NULL} } }, ++ ++ {GPU_ID2_PRODUCT_TGOX, ++ {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tGOx_r0p0}, ++ {GPU_ID2_VERSION_MAKE(1, 0, 0), base_hw_issues_tGOx_r1p0}, ++ {U32_MAX, NULL} } }, ++ ++ {GPU_ID2_PRODUCT_TTRX, ++ {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tTRx_r0p0}, ++ {GPU_ID2_VERSION_MAKE(0, 0, 3), base_hw_issues_tTRx_r0p0}, ++ {GPU_ID2_VERSION_MAKE(0, 1, 0), base_hw_issues_tTRx_r0p1}, ++ {GPU_ID2_VERSION_MAKE(0, 1, 1), base_hw_issues_tTRx_r0p1}, ++ {GPU_ID2_VERSION_MAKE(0, 2, 0), base_hw_issues_tTRx_r0p2}, ++ {U32_MAX, NULL} } }, ++ ++ {GPU_ID2_PRODUCT_TNAX, ++ {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tNAx_r0p0}, ++ {GPU_ID2_VERSION_MAKE(0, 0, 3), base_hw_issues_tNAx_r0p0}, ++ {GPU_ID2_VERSION_MAKE(0, 0, 4), base_hw_issues_tNAx_r0p0}, ++ {GPU_ID2_VERSION_MAKE(0, 0, 5), base_hw_issues_tNAx_r0p0}, ++ {GPU_ID2_VERSION_MAKE(0, 1, 0), base_hw_issues_tNAx_r0p1}, ++ {GPU_ID2_VERSION_MAKE(0, 1, 1), base_hw_issues_tNAx_r0p1}, ++ {U32_MAX, NULL} } }, ++ ++ {GPU_ID2_PRODUCT_LBEX, ++ {{GPU_ID2_VERSION_MAKE(1, 0, 0), base_hw_issues_lBEx_r1p0}, ++ {GPU_ID2_VERSION_MAKE(1, 1, 0), base_hw_issues_lBEx_r1p1}, ++ {U32_MAX, NULL} } }, ++ ++ {GPU_ID2_PRODUCT_TBEX, ++ {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tBEx_r0p0}, ++ {GPU_ID2_VERSION_MAKE(0, 0, 3), base_hw_issues_tBEx_r0p0}, ++ {GPU_ID2_VERSION_MAKE(0, 1, 0), base_hw_issues_tBEx_r0p1}, ++ {GPU_ID2_VERSION_MAKE(1, 0, 0), base_hw_issues_tBEx_r1p0}, ++ {U32_MAX, NULL} } }, ++ ++ {GPU_ID2_PRODUCT_TBAX, ++ {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tBAx_r0p0}, ++ {GPU_ID2_VERSION_MAKE(0, 0, 3), base_hw_issues_tBAx_r0p0}, ++ {GPU_ID2_VERSION_MAKE(1, 0, 0), base_hw_issues_tBAx_r1p0}, ++ {U32_MAX, NULL} } }, ++ ++ {GPU_ID2_PRODUCT_TDUX, ++ {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tDUx_r0p0}, ++ {U32_MAX, NULL} } }, ++ ++ {GPU_ID2_PRODUCT_TODX, ++ {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tODx_r0p0}, ++ {U32_MAX, NULL} } }, ++ ++ {GPU_ID2_PRODUCT_LODX, ++ {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tODx_r0p0}, ++ {U32_MAX, NULL} } }, ++ ++ {GPU_ID2_PRODUCT_TGRX, ++ {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tGRx_r0p0}, ++ {U32_MAX, NULL} } }, ++ ++ {GPU_ID2_PRODUCT_TVAX, ++ {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tVAx_r0p0}, ++ {U32_MAX, NULL} } }, ++ ++ {GPU_ID2_PRODUCT_TTUX, ++ {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tTUx_r0p0}, ++ {U32_MAX, NULL} } }, ++ ++ {GPU_ID2_PRODUCT_LTUX, ++ {{GPU_ID2_VERSION_MAKE(3, 0, 0), base_hw_issues_tTUx_r0p0}, ++ {U32_MAX, NULL} } }, ++ ++ {GPU_ID2_PRODUCT_TE2X, ++ {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tE2x_r0p0}, ++ {U32_MAX, NULL} } }, ++ }; ++ ++ u32 gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; ++ const u32 product_model = gpu_id & GPU_ID2_PRODUCT_MODEL; ++ const struct base_hw_product *product = NULL; ++ size_t p; ++ ++ /* Stop when we reach the end of the products array. */ ++ for (p = 0; p < ARRAY_SIZE(base_hw_products); ++p) { ++ if (product_model == base_hw_products[p].product_model) { ++ product = &base_hw_products[p]; ++ break; ++ } ++ } ++ ++ if (product != NULL) { ++ /* Found a matching product. */ ++ const u32 version = gpu_id & GPU_ID2_VERSION; ++ u32 fallback_version = 0; ++ const enum base_hw_issue *fallback_issues = NULL; ++ size_t v; ++ ++ /* Stop when we reach the end of the map. */ ++ for (v = 0; product->map[v].version != U32_MAX; ++v) { ++ ++ if (version == product->map[v].version) { ++ /* Exact match so stop. */ ++ issues = product->map[v].issues; ++ break; ++ } ++ ++ /* Check whether this is a candidate for most recent ++ known version not later than the actual ++ version. */ ++ if ((version > product->map[v].version) && ++ (product->map[v].version >= fallback_version)) { ++#if MALI_CUSTOMER_RELEASE ++ /* Match on version's major and minor fields */ ++ if (((version ^ product->map[v].version) >> ++ GPU_ID2_VERSION_MINOR_SHIFT) == 0) ++#endif ++ { ++ fallback_version = product->map[v].version; ++ fallback_issues = product->map[v].issues; ++ } ++ } ++ } ++ ++ if ((issues == NULL) && (fallback_issues != NULL)) { ++ /* Fall back to the issue set of the most recent known ++ version not later than the actual version. */ ++ issues = fallback_issues; ++ ++#if MALI_CUSTOMER_RELEASE ++ dev_warn(kbdev->dev, ++ "GPU hardware issue table may need updating:\n" ++#else ++ dev_info(kbdev->dev, ++#endif ++ "r%dp%d status %d is unknown; treating as r%dp%d status %d", ++ (gpu_id & GPU_ID2_VERSION_MAJOR) >> ++ GPU_ID2_VERSION_MAJOR_SHIFT, ++ (gpu_id & GPU_ID2_VERSION_MINOR) >> ++ GPU_ID2_VERSION_MINOR_SHIFT, ++ (gpu_id & GPU_ID2_VERSION_STATUS) >> ++ GPU_ID2_VERSION_STATUS_SHIFT, ++ (fallback_version & GPU_ID2_VERSION_MAJOR) >> ++ GPU_ID2_VERSION_MAJOR_SHIFT, ++ (fallback_version & GPU_ID2_VERSION_MINOR) >> ++ GPU_ID2_VERSION_MINOR_SHIFT, ++ (fallback_version & GPU_ID2_VERSION_STATUS) >> ++ GPU_ID2_VERSION_STATUS_SHIFT); ++ ++ gpu_id &= ~GPU_ID2_VERSION; ++ gpu_id |= fallback_version; ++ kbdev->gpu_props.props.raw_props.gpu_id = gpu_id; ++ ++ kbase_gpuprops_update_core_props_gpu_id( ++ &kbdev->gpu_props.props); ++ } ++ } ++ return issues; ++} ++ ++int kbase_hw_set_issues_mask(struct kbase_device *kbdev) ++{ ++ const enum base_hw_issue *issues; ++ u32 gpu_id; ++ u32 impl_tech; ++ ++ gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; ++ impl_tech = kbdev->gpu_props.props.thread_props.impl_tech; ++ ++ if (impl_tech != IMPLEMENTATION_MODEL) { ++ issues = kbase_hw_get_issues_for_new_id(kbdev); ++ if (issues == NULL) { ++ dev_err(kbdev->dev, ++ "Unknown GPU ID %x", gpu_id); ++ return -EINVAL; ++ } ++ ++#if !MALI_CUSTOMER_RELEASE ++ /* The GPU ID might have been replaced with the last ++ known version of the same GPU. */ ++ gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; ++#endif ++ } else { ++ /* Software model */ ++ switch (gpu_id & GPU_ID2_PRODUCT_MODEL) { ++ case GPU_ID2_PRODUCT_TMIX: ++ issues = base_hw_issues_model_tMIx; ++ break; ++ case GPU_ID2_PRODUCT_THEX: ++ issues = base_hw_issues_model_tHEx; ++ break; ++ case GPU_ID2_PRODUCT_TSIX: ++ issues = base_hw_issues_model_tSIx; ++ break; ++ case GPU_ID2_PRODUCT_TDVX: ++ issues = base_hw_issues_model_tDVx; ++ break; ++ case GPU_ID2_PRODUCT_TNOX: ++ issues = base_hw_issues_model_tNOx; ++ break; ++ case GPU_ID2_PRODUCT_TGOX: ++ issues = base_hw_issues_model_tGOx; ++ break; ++ case GPU_ID2_PRODUCT_TTRX: ++ issues = base_hw_issues_model_tTRx; ++ break; ++ case GPU_ID2_PRODUCT_TNAX: ++ issues = base_hw_issues_model_tNAx; ++ break; ++ case GPU_ID2_PRODUCT_LBEX: ++ case GPU_ID2_PRODUCT_TBEX: ++ issues = base_hw_issues_model_tBEx; ++ break; ++ case GPU_ID2_PRODUCT_TBAX: ++ issues = base_hw_issues_model_tBAx; ++ break; ++ case GPU_ID2_PRODUCT_TDUX: ++ issues = base_hw_issues_model_tDUx; ++ break; ++ case GPU_ID2_PRODUCT_TODX: ++ case GPU_ID2_PRODUCT_LODX: ++ issues = base_hw_issues_model_tODx; ++ break; ++ case GPU_ID2_PRODUCT_TGRX: ++ issues = base_hw_issues_model_tGRx; ++ break; ++ case GPU_ID2_PRODUCT_TVAX: ++ issues = base_hw_issues_model_tVAx; ++ break; ++ case GPU_ID2_PRODUCT_TTUX: ++ case GPU_ID2_PRODUCT_LTUX: ++ issues = base_hw_issues_model_tTUx; ++ break; ++ case GPU_ID2_PRODUCT_TE2X: ++ issues = base_hw_issues_model_tE2x; ++ break; ++ default: ++ dev_err(kbdev->dev, ++ "Unknown GPU ID %x", gpu_id); ++ return -EINVAL; ++ } ++ } ++ ++ dev_info(kbdev->dev, ++ "GPU identified as 0x%x arch %d.%d.%d r%dp%d status %d", ++ (gpu_id & GPU_ID2_PRODUCT_MAJOR) >> ++ GPU_ID2_PRODUCT_MAJOR_SHIFT, ++ (gpu_id & GPU_ID2_ARCH_MAJOR) >> ++ GPU_ID2_ARCH_MAJOR_SHIFT, ++ (gpu_id & GPU_ID2_ARCH_MINOR) >> ++ GPU_ID2_ARCH_MINOR_SHIFT, ++ (gpu_id & GPU_ID2_ARCH_REV) >> ++ GPU_ID2_ARCH_REV_SHIFT, ++ (gpu_id & GPU_ID2_VERSION_MAJOR) >> ++ GPU_ID2_VERSION_MAJOR_SHIFT, ++ (gpu_id & GPU_ID2_VERSION_MINOR) >> ++ GPU_ID2_VERSION_MINOR_SHIFT, ++ (gpu_id & GPU_ID2_VERSION_STATUS) >> ++ GPU_ID2_VERSION_STATUS_SHIFT); ++ ++ for (; *issues != BASE_HW_ISSUE_END; issues++) ++ set_bit(*issues, &kbdev->hw_issues_mask[0]); ++ ++ return 0; ++} +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hw.h b/drivers/gpu/arm/bifrost/mali_kbase_hw.h +new file mode 100755 +index 000000000000..f386b1624317 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_hw.h +@@ -0,0 +1,70 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++/** ++ * @file ++ * Run-time work-arounds helpers ++ */ ++ ++#ifndef _KBASE_HW_H_ ++#define _KBASE_HW_H_ ++ ++#include "mali_kbase_defs.h" ++ ++/** ++ * @brief Tell whether a work-around should be enabled ++ */ ++#define kbase_hw_has_issue(kbdev, issue)\ ++ test_bit(issue, &(kbdev)->hw_issues_mask[0]) ++ ++/** ++ * @brief Tell whether a feature is supported ++ */ ++#define kbase_hw_has_feature(kbdev, feature)\ ++ test_bit(feature, &(kbdev)->hw_features_mask[0]) ++ ++/** ++ * kbase_hw_set_issues_mask - Set the hardware issues mask based on the GPU ID ++ * @kbdev: Device pointer ++ * ++ * Return: 0 if the GPU ID was recognized, otherwise -EINVAL. ++ * ++ * The GPU ID is read from the @kbdev. ++ * ++ * In debugging versions of the driver, unknown versions of a known GPU with a ++ * new-format ID will be treated as the most recent known version not later ++ * than the actual version. In such circumstances, the GPU ID in @kbdev will ++ * also be replaced with the most recent known version. ++ * ++ * Note: The GPU configuration must have been read by ++ * kbase_gpuprops_get_props() before calling this function. ++ */ ++int kbase_hw_set_issues_mask(struct kbase_device *kbdev); ++ ++/** ++ * @brief Set the features mask depending on the GPU ID ++ */ ++void kbase_hw_set_features_mask(struct kbase_device *kbdev); ++ ++#endif /* _KBASE_HW_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_backend.h b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_backend.h +new file mode 100755 +index 000000000000..89df2519ab97 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_backend.h +@@ -0,0 +1,45 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2015, 2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++/* ++ * HW access backend common APIs ++ */ ++ ++#ifndef _KBASE_HWACCESS_BACKEND_H_ ++#define _KBASE_HWACCESS_BACKEND_H_ ++ ++/** ++ * kbase_backend_devfreq_init - Perform backend devfreq related initialization. ++ * @kbdev: Device pointer ++ * ++ * Return: 0 on success, or an error code on failure. ++ */ ++int kbase_backend_devfreq_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_backend_devfreq_term - Perform backend-devfreq termination. ++ * @kbdev: Device pointer ++ */ ++void kbase_backend_devfreq_term(struct kbase_device *kbdev); ++ ++#endif /* _KBASE_HWACCESS_BACKEND_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_defs.h b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_defs.h +new file mode 100755 +index 000000000000..124a2d9cf0c3 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_defs.h +@@ -0,0 +1,51 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014, 2016, 2018 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++/** ++ * @file mali_kbase_hwaccess_gpu_defs.h ++ * HW access common definitions ++ */ ++ ++#ifndef _KBASE_HWACCESS_DEFS_H_ ++#define _KBASE_HWACCESS_DEFS_H_ ++ ++#include ++ ++/** ++ * struct kbase_hwaccess_data - object encapsulating the GPU backend specific ++ * data for the HW access layer. ++ * hwaccess_lock (a spinlock) must be held when ++ * accessing this structure. ++ * @active_kctx: pointer to active kbase context which last submitted an ++ * atom to GPU and while the context is active it can ++ * submit new atoms to GPU from the irq context also, without ++ * going through the bottom half of job completion path. ++ * @backend: GPU backend specific data for HW access layer ++ */ ++struct kbase_hwaccess_data { ++ struct kbase_context *active_kctx[BASE_JM_MAX_NR_SLOTS]; ++ ++ struct kbase_backend_data backend; ++}; ++ ++#endif /* _KBASE_HWACCESS_DEFS_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_gpuprops.h b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_gpuprops.h +new file mode 100755 +index 000000000000..3ae0dbe6886d +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_gpuprops.h +@@ -0,0 +1,87 @@ ++/* ++ * ++ * (C) COPYRIGHT ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ *//* SPDX-License-Identifier: GPL-2.0 */ ++/* ++ * ++ * (C) COPYRIGHT 2014-2015, 2018, 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++/** ++ * Base kernel property query backend APIs ++ */ ++ ++#ifndef _KBASE_HWACCESS_GPUPROPS_H_ ++#define _KBASE_HWACCESS_GPUPROPS_H_ ++ ++/** ++ * kbase_backend_gpuprops_get() - Fill @regdump with GPU properties read from ++ * GPU ++ * @kbdev: Device pointer ++ * @regdump: Pointer to struct kbase_gpuprops_regdump structure ++ * ++ * The caller should ensure that GPU remains powered-on during this function. ++ * ++ * Return: Zero for succeess or a Linux error code ++ */ ++int kbase_backend_gpuprops_get(struct kbase_device *kbdev, ++ struct kbase_gpuprops_regdump *regdump); ++ ++/** ++ * kbase_backend_gpuprops_get_features - Fill @regdump with GPU properties read ++ * from GPU ++ * @kbdev: Device pointer ++ * @regdump: Pointer to struct kbase_gpuprops_regdump structure ++ * ++ * This function reads GPU properties that are dependent on the hardware ++ * features bitmask. It will power-on the GPU if required. ++ * ++ * Return: Zero for succeess or a Linux error code ++ */ ++int kbase_backend_gpuprops_get_features(struct kbase_device *kbdev, ++ struct kbase_gpuprops_regdump *regdump); ++ ++/** ++ * kbase_backend_gpuprops_get_l2_features - Fill @regdump with L2_FEATURES read ++ * from GPU ++ * @kbdev: Device pointer ++ * @regdump: Pointer to struct kbase_gpuprops_regdump structure ++ * ++ * This function reads L2_FEATURES register that is dependent on the hardware ++ * features bitmask. It will power-on the GPU if required. ++ * ++ * Return: Zero on success, Linux error code on failure ++ */ ++int kbase_backend_gpuprops_get_l2_features(struct kbase_device *kbdev, ++ struct kbase_gpuprops_regdump *regdump); ++ ++ ++#endif /* _KBASE_HWACCESS_GPUPROPS_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_instr.h b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_instr.h +new file mode 100755 +index 000000000000..4fd2e3549268 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_instr.h +@@ -0,0 +1,151 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2015, 2017-2018, 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++/* ++ * HW Access instrumentation common APIs ++ */ ++ ++#ifndef _KBASE_HWACCESS_INSTR_H_ ++#define _KBASE_HWACCESS_INSTR_H_ ++ ++#include ++ ++/** ++ * struct kbase_instr_hwcnt_enable - Enable hardware counter collection. ++ * @dump_buffer: GPU address to write counters to. ++ * @dump_buffer_bytes: Size in bytes of the buffer pointed to by dump_buffer. ++ * @fe_bm: counters selection bitmask (Front End). ++ * @shader_bm: counters selection bitmask (Shader). ++ * @tiler_bm: counters selection bitmask (Tiler). ++ * @mmu_l2_bm: counters selection bitmask (MMU_L2). ++ * @use_secondary: use secondary performance counters set for applicable ++ * counter blocks. ++ */ ++struct kbase_instr_hwcnt_enable { ++ u64 dump_buffer; ++ u64 dump_buffer_bytes; ++ u32 fe_bm; ++ u32 shader_bm; ++ u32 tiler_bm; ++ u32 mmu_l2_bm; ++ bool use_secondary; ++}; ++ ++/** ++ * kbase_instr_hwcnt_enable_internal() - Enable HW counters collection ++ * @kbdev: Kbase device ++ * @kctx: Kbase context ++ * @enable: HW counter setup parameters ++ * ++ * Context: might sleep, waiting for reset to complete ++ * ++ * Return: 0 on success ++ */ ++int kbase_instr_hwcnt_enable_internal(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ struct kbase_instr_hwcnt_enable *enable); ++ ++/** ++ * kbase_instr_hwcnt_disable_internal() - Disable HW counters collection ++ * @kctx: Kbase context ++ * ++ * Context: might sleep, waiting for an ongoing dump to complete ++ * ++ * Return: 0 on success ++ */ ++int kbase_instr_hwcnt_disable_internal(struct kbase_context *kctx); ++ ++/** ++ * kbase_instr_hwcnt_request_dump() - Request HW counter dump from GPU ++ * @kctx: Kbase context ++ * ++ * Caller must either wait for kbase_instr_hwcnt_dump_complete() to return true, ++ * of call kbase_instr_hwcnt_wait_for_dump(). ++ * ++ * Return: 0 on success ++ */ ++int kbase_instr_hwcnt_request_dump(struct kbase_context *kctx); ++ ++/** ++ * kbase_instr_hwcnt_wait_for_dump() - Wait until pending HW counter dump has ++ * completed. ++ * @kctx: Kbase context ++ * ++ * Context: will sleep, waiting for dump to complete ++ * ++ * Return: 0 on success ++ */ ++int kbase_instr_hwcnt_wait_for_dump(struct kbase_context *kctx); ++ ++/** ++ * kbase_instr_hwcnt_dump_complete - Tell whether the HW counters dump has ++ * completed ++ * @kctx: Kbase context ++ * @success: Set to true if successful ++ * ++ * Context: does not sleep. ++ * ++ * Return: true if the dump is complete ++ */ ++bool kbase_instr_hwcnt_dump_complete(struct kbase_context *kctx, ++ bool * const success); ++ ++/** ++ * kbase_instr_hwcnt_clear() - Clear HW counters ++ * @kctx: Kbase context ++ * ++ * Context: might sleep, waiting for reset to complete ++ * ++ * Return: 0 on success ++ */ ++int kbase_instr_hwcnt_clear(struct kbase_context *kctx); ++ ++/** ++ * kbase_instr_backend_init() - Initialise the instrumentation backend ++ * @kbdev: Kbase device ++ * ++ * This function should be called during driver initialization. ++ * ++ * Return: 0 on success ++ */ ++int kbase_instr_backend_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_instr_backend_init() - Terminate the instrumentation backend ++ * @kbdev: Kbase device ++ * ++ * This function should be called during driver termination. ++ */ ++void kbase_instr_backend_term(struct kbase_device *kbdev); ++ ++#ifdef CONFIG_MALI_PRFCNT_SET_SECONDARY_VIA_DEBUG_FS ++/** ++ * kbase_instr_backend_debugfs_init() - Add a debugfs entry for the ++ * hardware counter set. ++ * @kbdev: kbase device ++ */ ++void kbase_instr_backend_debugfs_init(struct kbase_device *kbdev); ++#endif ++ ++#endif /* _KBASE_HWACCESS_INSTR_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_jm.h b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_jm.h +new file mode 100755 +index 000000000000..f6ce17e4180f +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_jm.h +@@ -0,0 +1,304 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++/* ++ * HW access job manager common APIs ++ */ ++ ++#ifndef _KBASE_HWACCESS_JM_H_ ++#define _KBASE_HWACCESS_JM_H_ ++ ++/** ++ * kbase_backend_run_atom() - Run an atom on the GPU ++ * @kbdev: Device pointer ++ * @atom: Atom to run ++ * ++ * Caller must hold the HW access lock ++ */ ++void kbase_backend_run_atom(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom); ++ ++/** ++ * kbase_backend_slot_update - Update state based on slot ringbuffers ++ * ++ * @kbdev: Device pointer ++ * ++ * Inspect the jobs in the slot ringbuffers and update state. ++ * ++ * This will cause jobs to be submitted to hardware if they are unblocked ++ */ ++void kbase_backend_slot_update(struct kbase_device *kbdev); ++ ++/** ++ * kbase_backend_find_and_release_free_address_space() - Release a free AS ++ * @kbdev: Device pointer ++ * @kctx: Context pointer ++ * ++ * This function can evict an idle context from the runpool, freeing up the ++ * address space it was using. ++ * ++ * The address space is marked as in use. The caller must either assign a ++ * context using kbase_gpu_use_ctx(), or release it using ++ * kbase_ctx_sched_release() ++ * ++ * Return: Number of free address space, or KBASEP_AS_NR_INVALID if none ++ * available ++ */ ++int kbase_backend_find_and_release_free_address_space( ++ struct kbase_device *kbdev, struct kbase_context *kctx); ++ ++/** ++ * kbase_backend_use_ctx() - Activate a currently unscheduled context, using the ++ * provided address space. ++ * @kbdev: Device pointer ++ * @kctx: Context pointer. May be NULL ++ * @as_nr: Free address space to use ++ * ++ * kbase_gpu_next_job() will pull atoms from the active context. ++ * ++ * Return: true if successful, false if ASID not assigned. ++ */ ++bool kbase_backend_use_ctx(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ int as_nr); ++ ++/** ++ * kbase_backend_use_ctx_sched() - Activate a context. ++ * @kbdev: Device pointer ++ * @kctx: Context pointer ++ * @js: Job slot to activate context on ++ * ++ * kbase_gpu_next_job() will pull atoms from the active context. ++ * ++ * The context must already be scheduled and assigned to an address space. If ++ * the context is not scheduled, then kbase_gpu_use_ctx() should be used ++ * instead. ++ * ++ * Caller must hold hwaccess_lock ++ * ++ * Return: true if context is now active, false otherwise (ie if context does ++ * not have an address space assigned) ++ */ ++bool kbase_backend_use_ctx_sched(struct kbase_device *kbdev, ++ struct kbase_context *kctx, int js); ++ ++/** ++ * kbase_backend_release_ctx_irq - Release a context from the GPU. This will ++ * de-assign the assigned address space. ++ * @kbdev: Device pointer ++ * @kctx: Context pointer ++ * ++ * Caller must hold kbase_device->mmu_hw_mutex and hwaccess_lock ++ */ ++void kbase_backend_release_ctx_irq(struct kbase_device *kbdev, ++ struct kbase_context *kctx); ++ ++/** ++ * kbase_backend_release_ctx_noirq - Release a context from the GPU. This will ++ * de-assign the assigned address space. ++ * @kbdev: Device pointer ++ * @kctx: Context pointer ++ * ++ * Caller must hold kbase_device->mmu_hw_mutex ++ * ++ * This function must perform any operations that could not be performed in IRQ ++ * context by kbase_backend_release_ctx_irq(). ++ */ ++void kbase_backend_release_ctx_noirq(struct kbase_device *kbdev, ++ struct kbase_context *kctx); ++ ++/** ++ * kbase_backend_cache_clean - Perform a cache clean if the given atom requires ++ * one ++ * @kbdev: Device pointer ++ * @katom: Pointer to the failed atom ++ * ++ * On some GPUs, the GPU cache must be cleaned following a failed atom. This ++ * function performs a clean if it is required by @katom. ++ */ ++void kbase_backend_cache_clean(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom); ++ ++ ++/** ++ * kbase_backend_complete_wq() - Perform backend-specific actions required on ++ * completing an atom. ++ * @kbdev: Device pointer ++ * @katom: Pointer to the atom to complete ++ * ++ * This function should only be called from kbase_jd_done_worker() or ++ * js_return_worker(). ++ * ++ * Return: true if atom has completed, false if atom should be re-submitted ++ */ ++void kbase_backend_complete_wq(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom); ++ ++#if !MALI_USE_CSF ++/** ++ * kbase_backend_complete_wq_post_sched - Perform backend-specific actions ++ * required on completing an atom, after ++ * any scheduling has taken place. ++ * @kbdev: Device pointer ++ * @core_req: Core requirements of atom ++ * ++ * This function should only be called from kbase_jd_done_worker() or ++ * js_return_worker(). ++ */ ++void kbase_backend_complete_wq_post_sched(struct kbase_device *kbdev, ++ base_jd_core_req core_req); ++#endif /* !MALI_USE_CSF */ ++ ++/** ++ * kbase_backend_reset() - The GPU is being reset. Cancel all jobs on the GPU ++ * and remove any others from the ringbuffers. ++ * @kbdev: Device pointer ++ * @end_timestamp: Timestamp of reset ++ */ ++void kbase_backend_reset(struct kbase_device *kbdev, ktime_t *end_timestamp); ++ ++/** ++ * kbase_backend_inspect_tail - Return the atom currently at the tail of slot ++ * @js ++ * @kbdev: Device pointer ++ * @js: Job slot to inspect ++ * ++ * Return : Atom currently at the head of slot @js, or NULL ++ */ ++struct kbase_jd_atom *kbase_backend_inspect_tail(struct kbase_device *kbdev, ++ int js); ++ ++/** ++ * kbase_backend_nr_atoms_on_slot() - Return the number of atoms currently on a ++ * slot. ++ * @kbdev: Device pointer ++ * @js: Job slot to inspect ++ * ++ * Return : Number of atoms currently on slot ++ */ ++int kbase_backend_nr_atoms_on_slot(struct kbase_device *kbdev, int js); ++ ++/** ++ * kbase_backend_nr_atoms_submitted() - Return the number of atoms on a slot ++ * that are currently on the GPU. ++ * @kbdev: Device pointer ++ * @js: Job slot to inspect ++ * ++ * Return : Number of atoms currently on slot @js that are currently on the GPU. ++ */ ++int kbase_backend_nr_atoms_submitted(struct kbase_device *kbdev, int js); ++ ++/** ++ * kbase_backend_ctx_count_changed() - Number of contexts ready to submit jobs ++ * has changed. ++ * @kbdev: Device pointer ++ * ++ * Perform any required backend-specific actions (eg starting/stopping ++ * scheduling timers). ++ */ ++void kbase_backend_ctx_count_changed(struct kbase_device *kbdev); ++ ++/** ++ * kbase_backend_timeouts_changed() - Job Scheduler timeouts have changed. ++ * @kbdev: Device pointer ++ * ++ * Perform any required backend-specific actions (eg updating timeouts of ++ * currently running atoms). ++ */ ++void kbase_backend_timeouts_changed(struct kbase_device *kbdev); ++ ++/** ++ * kbase_backend_slot_free() - Return the number of jobs that can be currently ++ * submitted to slot @js. ++ * @kbdev: Device pointer ++ * @js: Job slot to inspect ++ * ++ * Return : Number of jobs that can be submitted. ++ */ ++int kbase_backend_slot_free(struct kbase_device *kbdev, int js); ++ ++/** ++ * kbase_job_check_enter_disjoint - potentially leave disjoint state ++ * @kbdev: kbase device ++ * @target_katom: atom which is finishing ++ * ++ * Work out whether to leave disjoint state when finishing an atom that was ++ * originated by kbase_job_check_enter_disjoint(). ++ */ ++void kbase_job_check_leave_disjoint(struct kbase_device *kbdev, ++ struct kbase_jd_atom *target_katom); ++ ++/** ++ * kbase_backend_jm_kill_running_jobs_from_kctx - Kill all jobs that are ++ * currently running on GPU from a context ++ * @kctx: Context pointer ++ * ++ * This is used in response to a page fault to remove all jobs from the faulting ++ * context from the hardware. ++ * ++ * Caller must hold hwaccess_lock. ++ */ ++void kbase_backend_jm_kill_running_jobs_from_kctx(struct kbase_context *kctx); ++ ++/** ++ * kbase_jm_wait_for_zero_jobs - Wait for context to have zero jobs running, and ++ * to be descheduled. ++ * @kctx: Context pointer ++ * ++ * This should be called following kbase_js_zap_context(), to ensure the context ++ * can be safely destroyed. ++ */ ++void kbase_jm_wait_for_zero_jobs(struct kbase_context *kctx); ++ ++/** ++ * kbase_backend_get_current_flush_id - Return the current flush ID ++ * ++ * @kbdev: Device pointer ++ * ++ * Return: the current flush ID to be recorded for each job chain ++ */ ++u32 kbase_backend_get_current_flush_id(struct kbase_device *kbdev); ++ ++/** ++ * kbase_job_slot_hardstop - Hard-stop the specified job slot ++ * @kctx: The kbase context that contains the job(s) that should ++ * be hard-stopped ++ * @js: The job slot to hard-stop ++ * @target_katom: The job that should be hard-stopped (or NULL for all ++ * jobs from the context) ++ * Context: ++ * The job slot lock must be held when calling this function. ++ */ ++void kbase_job_slot_hardstop(struct kbase_context *kctx, int js, ++ struct kbase_jd_atom *target_katom); ++ ++/** ++ * kbase_gpu_atoms_submitted_any() - Inspect whether there are any atoms ++ * currently on the GPU ++ * @kbdev: Device pointer ++ * ++ * Return: true if there are any atoms on the GPU, false otherwise ++ */ ++bool kbase_gpu_atoms_submitted_any(struct kbase_device *kbdev); ++ ++#endif /* _KBASE_HWACCESS_JM_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_pm.h b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_pm.h +new file mode 100755 +index 000000000000..bbaf6eaf8d88 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_pm.h +@@ -0,0 +1,229 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2015, 2018-2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++/** ++ * @file mali_kbase_hwaccess_pm.h ++ * HW access power manager common APIs ++ */ ++ ++#ifndef _KBASE_HWACCESS_PM_H_ ++#define _KBASE_HWACCESS_PM_H_ ++ ++#include ++#include ++ ++#include ++ ++/* Forward definition - see mali_kbase.h */ ++struct kbase_device; ++ ++/* Functions common to all HW access backends */ ++ ++/** ++ * Initialize the power management framework. ++ * ++ * Must be called before any other power management function ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * Return: 0 if the power management framework was successfully initialized. ++ */ ++int kbase_hwaccess_pm_init(struct kbase_device *kbdev); ++ ++/** ++ * Terminate the power management framework. ++ * ++ * No power management functions may be called after this ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_hwaccess_pm_term(struct kbase_device *kbdev); ++ ++/** ++ * kbase_hwaccess_pm_powerup - Power up the GPU. ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * @flags: Flags to pass on to kbase_pm_init_hw ++ * ++ * Power up GPU after all modules have been initialized and interrupt handlers ++ * installed. ++ * ++ * Return: 0 if powerup was successful. ++ */ ++int kbase_hwaccess_pm_powerup(struct kbase_device *kbdev, ++ unsigned int flags); ++ ++/** ++ * Halt the power management framework. ++ * ++ * Should ensure that no new interrupts are generated, but allow any currently ++ * running interrupt handlers to complete successfully. The GPU is forced off by ++ * the time this function returns, regardless of whether or not the active power ++ * policy asks for the GPU to be powered off. ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid ++ * pointer) ++ */ ++void kbase_hwaccess_pm_halt(struct kbase_device *kbdev); ++ ++/** ++ * Perform any backend-specific actions to suspend the GPU ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid ++ * pointer) ++ */ ++void kbase_hwaccess_pm_suspend(struct kbase_device *kbdev); ++ ++/** ++ * Perform any backend-specific actions to resume the GPU from a suspend ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid ++ * pointer) ++ */ ++void kbase_hwaccess_pm_resume(struct kbase_device *kbdev); ++ ++/** ++ * Perform any required actions for activating the GPU. Called when the first ++ * context goes active. ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid ++ * pointer) ++ */ ++void kbase_hwaccess_pm_gpu_active(struct kbase_device *kbdev); ++ ++/** ++ * Perform any required actions for idling the GPU. Called when the last ++ * context goes idle. ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid ++ * pointer) ++ */ ++void kbase_hwaccess_pm_gpu_idle(struct kbase_device *kbdev); ++ ++ ++/** ++ * Set the debug core mask. ++ * ++ * This determines which cores the power manager is allowed to use. ++ * ++ * @param kbdev The kbase device structure for the device (must be a ++ * valid pointer) ++ * @param new_core_mask_js0 The core mask to use for job slot 0 ++ * @param new_core_mask_js0 The core mask to use for job slot 1 ++ * @param new_core_mask_js0 The core mask to use for job slot 2 ++ */ ++void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev, ++ u64 new_core_mask_js0, u64 new_core_mask_js1, ++ u64 new_core_mask_js2); ++ ++ ++/** ++ * Get the current policy. ++ * ++ * Returns the policy that is currently active. ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid ++ * pointer) ++ * ++ * @return The current policy ++ */ ++const struct kbase_pm_ca_policy ++*kbase_pm_ca_get_policy(struct kbase_device *kbdev); ++ ++/** ++ * Change the policy to the one specified. ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid ++ * pointer) ++ * @param policy The policy to change to (valid pointer returned from ++ * @ref kbase_pm_ca_list_policies) ++ */ ++void kbase_pm_ca_set_policy(struct kbase_device *kbdev, ++ const struct kbase_pm_ca_policy *policy); ++ ++/** ++ * Retrieve a static list of the available policies. ++ * ++ * @param[out] policies An array pointer to take the list of policies. This may ++ * be NULL. The contents of this array must not be ++ * modified. ++ * ++ * @return The number of policies ++ */ ++int ++kbase_pm_ca_list_policies(const struct kbase_pm_ca_policy * const **policies); ++ ++ ++/** ++ * Get the current policy. ++ * ++ * Returns the policy that is currently active. ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid ++ * pointer) ++ * ++ * @return The current policy ++ */ ++const struct kbase_pm_policy *kbase_pm_get_policy(struct kbase_device *kbdev); ++ ++/** ++ * Change the policy to the one specified. ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid ++ * pointer) ++ * @param policy The policy to change to (valid pointer returned from ++ * @ref kbase_pm_list_policies) ++ */ ++void kbase_pm_set_policy(struct kbase_device *kbdev, ++ const struct kbase_pm_policy *policy); ++ ++/** ++ * kbase_pm_list_policies - Retrieve a static list of the available policies. ++ * ++ * @kbdev: The kbase device structure for the device. ++ * @list: An array pointer to take the list of policies. This may be NULL. ++ * The contents of this array must not be modified. ++ * ++ * Return: The number of policies ++ */ ++int kbase_pm_list_policies(struct kbase_device *kbdev, ++ const struct kbase_pm_policy * const **list); ++ ++/** ++ * kbase_protected_most_enable - Enable protected mode ++ * ++ * @kbdev: Address of the instance of a GPU platform device. ++ * ++ * Return: Zero on success or an error code ++ */ ++int kbase_pm_protected_mode_enable(struct kbase_device *kbdev); ++ ++/** ++ * kbase_protected_mode_disable - Disable protected mode ++ * ++ * @kbdev: Address of the instance of a GPU platform device. ++ * ++ * Return: Zero on success or an error code ++ */ ++int kbase_pm_protected_mode_disable(struct kbase_device *kbdev); ++ ++#endif /* _KBASE_HWACCESS_PM_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_time.h b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_time.h +new file mode 100755 +index 000000000000..94b7551b865e +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_time.h +@@ -0,0 +1,56 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014,2018-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++/** ++ * ++ */ ++ ++#ifndef _KBASE_BACKEND_TIME_H_ ++#define _KBASE_BACKEND_TIME_H_ ++ ++/** ++ * kbase_backend_get_gpu_time() - Get current GPU time ++ * @kbdev: Device pointer ++ * @cycle_counter: Pointer to u64 to store cycle counter in ++ * @system_time: Pointer to u64 to store system time in ++ * @ts: Pointer to struct timespec to store current monotonic ++ * time in ++ */ ++void kbase_backend_get_gpu_time(struct kbase_device *kbdev, u64 *cycle_counter, ++ u64 *system_time, struct timespec64 *ts); ++ ++/** ++ * kbase_backend_get_gpu_time_norequest() - Get current GPU time without ++ * request/release cycle counter ++ * @kbdev: Device pointer ++ * @cycle_counter: Pointer to u64 to store cycle counter in ++ * @system_time: Pointer to u64 to store system time in ++ * @ts: Pointer to struct timespec to store current monotonic ++ * time in ++ */ ++void kbase_backend_get_gpu_time_norequest(struct kbase_device *kbdev, ++ u64 *cycle_counter, ++ u64 *system_time, ++ struct timespec64 *ts); ++ ++#endif /* _KBASE_BACKEND_TIME_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwcnt.c b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt.c +new file mode 100755 +index 000000000000..2708af78b292 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt.c +@@ -0,0 +1,794 @@ ++/* ++ * ++ * (C) COPYRIGHT 2018, 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* ++ * Implementation of hardware counter context and accumulator APIs. ++ */ ++ ++#include "mali_kbase_hwcnt_context.h" ++#include "mali_kbase_hwcnt_accumulator.h" ++#include "mali_kbase_hwcnt_backend.h" ++#include "mali_kbase_hwcnt_types.h" ++#include "mali_malisw.h" ++#include "mali_kbase_debug.h" ++#include "mali_kbase_linux.h" ++ ++#include ++#include ++#include ++ ++/** ++ * enum kbase_hwcnt_accum_state - Hardware counter accumulator states. ++ * @ACCUM_STATE_ERROR: Error state, where all accumulator operations fail. ++ * @ACCUM_STATE_DISABLED: Disabled state, where dumping is always disabled. ++ * @ACCUM_STATE_ENABLED: Enabled state, where dumping is enabled if there are ++ * any enabled counters. ++ */ ++enum kbase_hwcnt_accum_state { ++ ACCUM_STATE_ERROR, ++ ACCUM_STATE_DISABLED, ++ ACCUM_STATE_ENABLED ++}; ++ ++/** ++ * struct kbase_hwcnt_accumulator - Hardware counter accumulator structure. ++ * @backend: Pointer to created counter backend. ++ * @state: The current state of the accumulator. ++ * - State transition from disabled->enabled or ++ * disabled->error requires state_lock. ++ * - State transition from enabled->disabled or ++ * enabled->error requires both accum_lock and ++ * state_lock. ++ * - Error state persists until next disable. ++ * @enable_map: The current set of enabled counters. ++ * - Must only be modified while holding both ++ * accum_lock and state_lock. ++ * - Can be read while holding either lock. ++ * - Must stay in sync with enable_map_any_enabled. ++ * @enable_map_any_enabled: True if any counters in the map are enabled, else ++ * false. If true, and state is ACCUM_STATE_ENABLED, ++ * then the counter backend will be enabled. ++ * - Must only be modified while holding both ++ * accum_lock and state_lock. ++ * - Can be read while holding either lock. ++ * - Must stay in sync with enable_map. ++ * @scratch_map: Scratch enable map, used as temporary enable map ++ * storage during dumps. ++ * - Must only be read or modified while holding ++ * accum_lock. ++ * @accum_buf: Accumulation buffer, where dumps will be accumulated ++ * into on transition to a disable state. ++ * - Must only be read or modified while holding ++ * accum_lock. ++ * @accumulated: True if the accumulation buffer has been accumulated ++ * into and not subsequently read from yet, else false. ++ * - Must only be read or modified while holding ++ * accum_lock. ++ * @ts_last_dump_ns: Timestamp (ns) of the end time of the most recent ++ * dump that was requested by the user. ++ * - Must only be read or modified while holding ++ * accum_lock. ++ */ ++struct kbase_hwcnt_accumulator { ++ struct kbase_hwcnt_backend *backend; ++ enum kbase_hwcnt_accum_state state; ++ struct kbase_hwcnt_enable_map enable_map; ++ bool enable_map_any_enabled; ++ struct kbase_hwcnt_enable_map scratch_map; ++ struct kbase_hwcnt_dump_buffer accum_buf; ++ bool accumulated; ++ u64 ts_last_dump_ns; ++}; ++ ++/** ++ * struct kbase_hwcnt_context - Hardware counter context structure. ++ * @iface: Pointer to hardware counter backend interface. ++ * @state_lock: Spinlock protecting state. ++ * @disable_count: Disable count of the context. Initialised to 1. ++ * Decremented when the accumulator is acquired, and incremented ++ * on release. Incremented on calls to ++ * kbase_hwcnt_context_disable[_atomic], and decremented on ++ * calls to kbase_hwcnt_context_enable. ++ * - Must only be read or modified while holding state_lock. ++ * @accum_lock: Mutex protecting accumulator. ++ * @accum_inited: Flag to prevent concurrent accumulator initialisation and/or ++ * termination. Set to true before accumulator initialisation, ++ * and false after accumulator termination. ++ * - Must only be modified while holding both accum_lock and ++ * state_lock. ++ * - Can be read while holding either lock. ++ * @accum: Hardware counter accumulator structure. ++ */ ++struct kbase_hwcnt_context { ++ const struct kbase_hwcnt_backend_interface *iface; ++ spinlock_t state_lock; ++ size_t disable_count; ++ struct mutex accum_lock; ++ bool accum_inited; ++ struct kbase_hwcnt_accumulator accum; ++}; ++ ++int kbase_hwcnt_context_init( ++ const struct kbase_hwcnt_backend_interface *iface, ++ struct kbase_hwcnt_context **out_hctx) ++{ ++ struct kbase_hwcnt_context *hctx = NULL; ++ ++ if (!iface || !out_hctx) ++ return -EINVAL; ++ ++ hctx = kzalloc(sizeof(*hctx), GFP_KERNEL); ++ if (!hctx) ++ return -ENOMEM; ++ ++ hctx->iface = iface; ++ spin_lock_init(&hctx->state_lock); ++ hctx->disable_count = 1; ++ mutex_init(&hctx->accum_lock); ++ hctx->accum_inited = false; ++ ++ *out_hctx = hctx; ++ ++ return 0; ++} ++KBASE_EXPORT_TEST_API(kbase_hwcnt_context_init); ++ ++void kbase_hwcnt_context_term(struct kbase_hwcnt_context *hctx) ++{ ++ if (!hctx) ++ return; ++ ++ /* Make sure we didn't leak the accumulator */ ++ WARN_ON(hctx->accum_inited); ++ kfree(hctx); ++} ++KBASE_EXPORT_TEST_API(kbase_hwcnt_context_term); ++ ++/** ++ * kbasep_hwcnt_accumulator_term() - Terminate the accumulator for the context. ++ * @hctx: Non-NULL pointer to hardware counter context. ++ */ ++static void kbasep_hwcnt_accumulator_term(struct kbase_hwcnt_context *hctx) ++{ ++ WARN_ON(!hctx); ++ WARN_ON(!hctx->accum_inited); ++ ++ kbase_hwcnt_enable_map_free(&hctx->accum.scratch_map); ++ kbase_hwcnt_dump_buffer_free(&hctx->accum.accum_buf); ++ kbase_hwcnt_enable_map_free(&hctx->accum.enable_map); ++ hctx->iface->term(hctx->accum.backend); ++ memset(&hctx->accum, 0, sizeof(hctx->accum)); ++} ++ ++/** ++ * kbasep_hwcnt_accumulator_init() - Initialise the accumulator for the context. ++ * @hctx: Non-NULL pointer to hardware counter context. ++ * ++ * Return: 0 on success, else error code. ++ */ ++static int kbasep_hwcnt_accumulator_init(struct kbase_hwcnt_context *hctx) ++{ ++ int errcode; ++ ++ WARN_ON(!hctx); ++ WARN_ON(!hctx->accum_inited); ++ ++ errcode = hctx->iface->init( ++ hctx->iface->info, &hctx->accum.backend); ++ if (errcode) ++ goto error; ++ ++ hctx->accum.state = ACCUM_STATE_ERROR; ++ ++ errcode = kbase_hwcnt_enable_map_alloc( ++ hctx->iface->metadata, &hctx->accum.enable_map); ++ if (errcode) ++ goto error; ++ ++ hctx->accum.enable_map_any_enabled = false; ++ ++ errcode = kbase_hwcnt_dump_buffer_alloc( ++ hctx->iface->metadata, &hctx->accum.accum_buf); ++ if (errcode) ++ goto error; ++ ++ errcode = kbase_hwcnt_enable_map_alloc( ++ hctx->iface->metadata, &hctx->accum.scratch_map); ++ if (errcode) ++ goto error; ++ ++ hctx->accum.accumulated = false; ++ ++ hctx->accum.ts_last_dump_ns = ++ hctx->iface->timestamp_ns(hctx->accum.backend); ++ ++ return 0; ++ ++error: ++ kbasep_hwcnt_accumulator_term(hctx); ++ return errcode; ++} ++ ++/** ++ * kbasep_hwcnt_accumulator_disable() - Transition the accumulator into the ++ * disabled state, from the enabled or ++ * error states. ++ * @hctx: Non-NULL pointer to hardware counter context. ++ * @accumulate: True if we should accumulate before disabling, else false. ++ */ ++static void kbasep_hwcnt_accumulator_disable( ++ struct kbase_hwcnt_context *hctx, bool accumulate) ++{ ++ int errcode = 0; ++ bool backend_enabled = false; ++ struct kbase_hwcnt_accumulator *accum; ++ unsigned long flags; ++ u64 dump_time_ns; ++ ++ WARN_ON(!hctx); ++ lockdep_assert_held(&hctx->accum_lock); ++ WARN_ON(!hctx->accum_inited); ++ ++ accum = &hctx->accum; ++ ++ spin_lock_irqsave(&hctx->state_lock, flags); ++ ++ WARN_ON(hctx->disable_count != 0); ++ WARN_ON(hctx->accum.state == ACCUM_STATE_DISABLED); ++ ++ if ((hctx->accum.state == ACCUM_STATE_ENABLED) && ++ (accum->enable_map_any_enabled)) ++ backend_enabled = true; ++ ++ if (!backend_enabled) ++ hctx->accum.state = ACCUM_STATE_DISABLED; ++ ++ spin_unlock_irqrestore(&hctx->state_lock, flags); ++ ++ /* Early out if the backend is not already enabled */ ++ if (!backend_enabled) ++ return; ++ ++ if (!accumulate) ++ goto disable; ++ ++ /* Try and accumulate before disabling */ ++ errcode = hctx->iface->dump_request(accum->backend, &dump_time_ns); ++ if (errcode) ++ goto disable; ++ ++ errcode = hctx->iface->dump_wait(accum->backend); ++ if (errcode) ++ goto disable; ++ ++ errcode = hctx->iface->dump_get(accum->backend, ++ &accum->accum_buf, &accum->enable_map, accum->accumulated); ++ if (errcode) ++ goto disable; ++ ++ accum->accumulated = true; ++ ++disable: ++ hctx->iface->dump_disable(accum->backend); ++ ++ /* Regardless of any errors during the accumulate, put the accumulator ++ * in the disabled state. ++ */ ++ spin_lock_irqsave(&hctx->state_lock, flags); ++ ++ hctx->accum.state = ACCUM_STATE_DISABLED; ++ ++ spin_unlock_irqrestore(&hctx->state_lock, flags); ++} ++ ++/** ++ * kbasep_hwcnt_accumulator_enable() - Transition the accumulator into the ++ * enabled state, from the disabled state. ++ * @hctx: Non-NULL pointer to hardware counter context. ++ */ ++static void kbasep_hwcnt_accumulator_enable(struct kbase_hwcnt_context *hctx) ++{ ++ int errcode = 0; ++ struct kbase_hwcnt_accumulator *accum; ++ ++ WARN_ON(!hctx); ++ lockdep_assert_held(&hctx->state_lock); ++ WARN_ON(!hctx->accum_inited); ++ WARN_ON(hctx->accum.state != ACCUM_STATE_DISABLED); ++ ++ accum = &hctx->accum; ++ ++ /* The backend only needs enabling if any counters are enabled */ ++ if (accum->enable_map_any_enabled) ++ errcode = hctx->iface->dump_enable_nolock( ++ accum->backend, &accum->enable_map); ++ ++ if (!errcode) ++ accum->state = ACCUM_STATE_ENABLED; ++ else ++ accum->state = ACCUM_STATE_ERROR; ++} ++ ++/** ++ * kbasep_hwcnt_accumulator_dump() - Perform a dump with the most up-to-date ++ * values of enabled counters possible, and ++ * optionally update the set of enabled ++ * counters. ++ * @hctx : Non-NULL pointer to the hardware counter context ++ * @ts_start_ns: Non-NULL pointer where the start timestamp of the dump will ++ * be written out to on success ++ * @ts_end_ns: Non-NULL pointer where the end timestamp of the dump will ++ * be written out to on success ++ * @dump_buf: Pointer to the buffer where the dump will be written out to on ++ * success. If non-NULL, must have the same metadata as the ++ * accumulator. If NULL, the dump will be discarded. ++ * @new_map: Pointer to the new counter enable map. If non-NULL, must have ++ * the same metadata as the accumulator. If NULL, the set of ++ * enabled counters will be unchanged. ++ */ ++static int kbasep_hwcnt_accumulator_dump( ++ struct kbase_hwcnt_context *hctx, ++ u64 *ts_start_ns, ++ u64 *ts_end_ns, ++ struct kbase_hwcnt_dump_buffer *dump_buf, ++ const struct kbase_hwcnt_enable_map *new_map) ++{ ++ int errcode = 0; ++ unsigned long flags; ++ enum kbase_hwcnt_accum_state state; ++ bool dump_requested = false; ++ bool dump_written = false; ++ bool cur_map_any_enabled; ++ struct kbase_hwcnt_enable_map *cur_map; ++ bool new_map_any_enabled = false; ++ u64 dump_time_ns; ++ struct kbase_hwcnt_accumulator *accum; ++ ++ WARN_ON(!hctx); ++ WARN_ON(!ts_start_ns); ++ WARN_ON(!ts_end_ns); ++ WARN_ON(dump_buf && (dump_buf->metadata != hctx->iface->metadata)); ++ WARN_ON(new_map && (new_map->metadata != hctx->iface->metadata)); ++ WARN_ON(!hctx->accum_inited); ++ lockdep_assert_held(&hctx->accum_lock); ++ ++ accum = &hctx->accum; ++ cur_map = &accum->scratch_map; ++ ++ /* Save out info about the current enable map */ ++ cur_map_any_enabled = accum->enable_map_any_enabled; ++ kbase_hwcnt_enable_map_copy(cur_map, &accum->enable_map); ++ ++ if (new_map) ++ new_map_any_enabled = ++ kbase_hwcnt_enable_map_any_enabled(new_map); ++ ++ /* ++ * We're holding accum_lock, so the accumulator state might transition ++ * from disabled to enabled during this function (as enabling is lock ++ * free), but it will never disable (as disabling needs to hold the ++ * accum_lock), nor will it ever transition from enabled to error (as ++ * an enable while we're already enabled is impossible). ++ * ++ * If we're already disabled, we'll only look at the accumulation buffer ++ * rather than do a real dump, so a concurrent enable does not affect ++ * us. ++ * ++ * If a concurrent enable fails, we might transition to the error ++ * state, but again, as we're only looking at the accumulation buffer, ++ * it's not an issue. ++ */ ++ spin_lock_irqsave(&hctx->state_lock, flags); ++ ++ state = accum->state; ++ ++ /* ++ * Update the new map now, such that if an enable occurs during this ++ * dump then that enable will set the new map. If we're already enabled, ++ * then we'll do it ourselves after the dump. ++ */ ++ if (new_map) { ++ kbase_hwcnt_enable_map_copy( ++ &accum->enable_map, new_map); ++ accum->enable_map_any_enabled = new_map_any_enabled; ++ } ++ ++ spin_unlock_irqrestore(&hctx->state_lock, flags); ++ ++ /* Error state, so early out. No need to roll back any map updates */ ++ if (state == ACCUM_STATE_ERROR) ++ return -EIO; ++ ++ /* Initiate the dump if the backend is enabled. */ ++ if ((state == ACCUM_STATE_ENABLED) && cur_map_any_enabled) { ++ if (dump_buf) { ++ errcode = hctx->iface->dump_request( ++ accum->backend, &dump_time_ns); ++ dump_requested = true; ++ } else { ++ dump_time_ns = hctx->iface->timestamp_ns( ++ accum->backend); ++ errcode = hctx->iface->dump_clear(accum->backend); ++ } ++ ++ if (errcode) ++ goto error; ++ } else { ++ dump_time_ns = hctx->iface->timestamp_ns(accum->backend); ++ } ++ ++ /* Copy any accumulation into the dest buffer */ ++ if (accum->accumulated && dump_buf) { ++ kbase_hwcnt_dump_buffer_copy( ++ dump_buf, &accum->accum_buf, cur_map); ++ dump_written = true; ++ } ++ ++ /* Wait for any requested dumps to complete */ ++ if (dump_requested) { ++ WARN_ON(state != ACCUM_STATE_ENABLED); ++ errcode = hctx->iface->dump_wait(accum->backend); ++ if (errcode) ++ goto error; ++ } ++ ++ /* If we're enabled and there's a new enable map, change the enabled set ++ * as soon after the dump has completed as possible. ++ */ ++ if ((state == ACCUM_STATE_ENABLED) && new_map) { ++ /* Backend is only enabled if there were any enabled counters */ ++ if (cur_map_any_enabled) ++ hctx->iface->dump_disable(accum->backend); ++ ++ /* (Re-)enable the backend if the new map has enabled counters. ++ * No need to acquire the spinlock, as concurrent enable while ++ * we're already enabled and holding accum_lock is impossible. ++ */ ++ if (new_map_any_enabled) { ++ errcode = hctx->iface->dump_enable( ++ accum->backend, new_map); ++ if (errcode) ++ goto error; ++ } ++ } ++ ++ /* Copy, accumulate, or zero into the dest buffer to finish */ ++ if (dump_buf) { ++ /* If we dumped, copy or accumulate it into the destination */ ++ if (dump_requested) { ++ WARN_ON(state != ACCUM_STATE_ENABLED); ++ errcode = hctx->iface->dump_get( ++ accum->backend, ++ dump_buf, ++ cur_map, ++ dump_written); ++ if (errcode) ++ goto error; ++ dump_written = true; ++ } ++ ++ /* If we've not written anything into the dump buffer so far, it ++ * means there was nothing to write. Zero any enabled counters. ++ */ ++ if (!dump_written) ++ kbase_hwcnt_dump_buffer_zero(dump_buf, cur_map); ++ } ++ ++ /* Write out timestamps */ ++ *ts_start_ns = accum->ts_last_dump_ns; ++ *ts_end_ns = dump_time_ns; ++ ++ accum->accumulated = false; ++ accum->ts_last_dump_ns = dump_time_ns; ++ ++ return 0; ++error: ++ /* An error was only physically possible if the backend was enabled */ ++ WARN_ON(state != ACCUM_STATE_ENABLED); ++ ++ /* Disable the backend, and transition to the error state */ ++ hctx->iface->dump_disable(accum->backend); ++ spin_lock_irqsave(&hctx->state_lock, flags); ++ ++ accum->state = ACCUM_STATE_ERROR; ++ ++ spin_unlock_irqrestore(&hctx->state_lock, flags); ++ ++ return errcode; ++} ++ ++/** ++ * kbasep_hwcnt_context_disable() - Increment the disable count of the context. ++ * @hctx: Non-NULL pointer to hardware counter context. ++ * @accumulate: True if we should accumulate before disabling, else false. ++ */ ++static void kbasep_hwcnt_context_disable( ++ struct kbase_hwcnt_context *hctx, bool accumulate) ++{ ++ unsigned long flags; ++ ++ WARN_ON(!hctx); ++ lockdep_assert_held(&hctx->accum_lock); ++ ++ if (!kbase_hwcnt_context_disable_atomic(hctx)) { ++ kbasep_hwcnt_accumulator_disable(hctx, accumulate); ++ ++ spin_lock_irqsave(&hctx->state_lock, flags); ++ ++ /* Atomic disable failed and we're holding the mutex, so current ++ * disable count must be 0. ++ */ ++ WARN_ON(hctx->disable_count != 0); ++ hctx->disable_count++; ++ ++ spin_unlock_irqrestore(&hctx->state_lock, flags); ++ } ++} ++ ++int kbase_hwcnt_accumulator_acquire( ++ struct kbase_hwcnt_context *hctx, ++ struct kbase_hwcnt_accumulator **accum) ++{ ++ int errcode = 0; ++ unsigned long flags; ++ ++ if (!hctx || !accum) ++ return -EINVAL; ++ ++ mutex_lock(&hctx->accum_lock); ++ spin_lock_irqsave(&hctx->state_lock, flags); ++ ++ if (!hctx->accum_inited) ++ /* Set accum initing now to prevent concurrent init */ ++ hctx->accum_inited = true; ++ else ++ /* Already have an accum, or already being inited */ ++ errcode = -EBUSY; ++ ++ spin_unlock_irqrestore(&hctx->state_lock, flags); ++ mutex_unlock(&hctx->accum_lock); ++ ++ if (errcode) ++ return errcode; ++ ++ errcode = kbasep_hwcnt_accumulator_init(hctx); ++ ++ if (errcode) { ++ mutex_lock(&hctx->accum_lock); ++ spin_lock_irqsave(&hctx->state_lock, flags); ++ ++ hctx->accum_inited = false; ++ ++ spin_unlock_irqrestore(&hctx->state_lock, flags); ++ mutex_unlock(&hctx->accum_lock); ++ ++ return errcode; ++ } ++ ++ spin_lock_irqsave(&hctx->state_lock, flags); ++ ++ WARN_ON(hctx->disable_count == 0); ++ WARN_ON(hctx->accum.enable_map_any_enabled); ++ ++ /* Decrement the disable count to allow the accumulator to be accessible ++ * now that it's fully constructed. ++ */ ++ hctx->disable_count--; ++ ++ /* ++ * Make sure the accumulator is initialised to the correct state. ++ * Regardless of initial state, counters don't need to be enabled via ++ * the backend, as the initial enable map has no enabled counters. ++ */ ++ hctx->accum.state = (hctx->disable_count == 0) ? ++ ACCUM_STATE_ENABLED : ++ ACCUM_STATE_DISABLED; ++ ++ spin_unlock_irqrestore(&hctx->state_lock, flags); ++ ++ *accum = &hctx->accum; ++ ++ return 0; ++} ++KBASE_EXPORT_TEST_API(kbase_hwcnt_accumulator_acquire); ++ ++void kbase_hwcnt_accumulator_release(struct kbase_hwcnt_accumulator *accum) ++{ ++ unsigned long flags; ++ struct kbase_hwcnt_context *hctx; ++ ++ if (!accum) ++ return; ++ ++ hctx = container_of(accum, struct kbase_hwcnt_context, accum); ++ ++ mutex_lock(&hctx->accum_lock); ++ ++ /* Double release is a programming error */ ++ WARN_ON(!hctx->accum_inited); ++ ++ /* Disable the context to ensure the accumulator is inaccesible while ++ * we're destroying it. This performs the corresponding disable count ++ * increment to the decrement done during acquisition. ++ */ ++ kbasep_hwcnt_context_disable(hctx, false); ++ ++ mutex_unlock(&hctx->accum_lock); ++ ++ kbasep_hwcnt_accumulator_term(hctx); ++ ++ mutex_lock(&hctx->accum_lock); ++ spin_lock_irqsave(&hctx->state_lock, flags); ++ ++ hctx->accum_inited = false; ++ ++ spin_unlock_irqrestore(&hctx->state_lock, flags); ++ mutex_unlock(&hctx->accum_lock); ++} ++KBASE_EXPORT_TEST_API(kbase_hwcnt_accumulator_release); ++ ++void kbase_hwcnt_context_disable(struct kbase_hwcnt_context *hctx) ++{ ++ if (WARN_ON(!hctx)) ++ return; ++ ++ /* Try and atomically disable first, so we can avoid locking the mutex ++ * if we don't need to. ++ */ ++ if (kbase_hwcnt_context_disable_atomic(hctx)) ++ return; ++ ++ mutex_lock(&hctx->accum_lock); ++ ++ kbasep_hwcnt_context_disable(hctx, true); ++ ++ mutex_unlock(&hctx->accum_lock); ++} ++KBASE_EXPORT_TEST_API(kbase_hwcnt_context_disable); ++ ++bool kbase_hwcnt_context_disable_atomic(struct kbase_hwcnt_context *hctx) ++{ ++ unsigned long flags; ++ bool atomic_disabled = false; ++ ++ if (WARN_ON(!hctx)) ++ return false; ++ ++ spin_lock_irqsave(&hctx->state_lock, flags); ++ ++ if (!WARN_ON(hctx->disable_count == SIZE_MAX)) { ++ /* ++ * If disable count is non-zero, we can just bump the disable ++ * count. ++ * ++ * Otherwise, we can't disable in an atomic context. ++ */ ++ if (hctx->disable_count != 0) { ++ hctx->disable_count++; ++ atomic_disabled = true; ++ } ++ } ++ ++ spin_unlock_irqrestore(&hctx->state_lock, flags); ++ ++ return atomic_disabled; ++} ++KBASE_EXPORT_TEST_API(kbase_hwcnt_context_disable_atomic); ++ ++void kbase_hwcnt_context_enable(struct kbase_hwcnt_context *hctx) ++{ ++ unsigned long flags; ++ ++ if (WARN_ON(!hctx)) ++ return; ++ ++ spin_lock_irqsave(&hctx->state_lock, flags); ++ ++ if (!WARN_ON(hctx->disable_count == 0)) { ++ if (hctx->disable_count == 1) ++ kbasep_hwcnt_accumulator_enable(hctx); ++ ++ hctx->disable_count--; ++ } ++ ++ spin_unlock_irqrestore(&hctx->state_lock, flags); ++} ++KBASE_EXPORT_TEST_API(kbase_hwcnt_context_enable); ++ ++const struct kbase_hwcnt_metadata *kbase_hwcnt_context_metadata( ++ struct kbase_hwcnt_context *hctx) ++{ ++ if (!hctx) ++ return NULL; ++ ++ return hctx->iface->metadata; ++} ++KBASE_EXPORT_TEST_API(kbase_hwcnt_context_metadata); ++ ++int kbase_hwcnt_accumulator_set_counters( ++ struct kbase_hwcnt_accumulator *accum, ++ const struct kbase_hwcnt_enable_map *new_map, ++ u64 *ts_start_ns, ++ u64 *ts_end_ns, ++ struct kbase_hwcnt_dump_buffer *dump_buf) ++{ ++ int errcode; ++ struct kbase_hwcnt_context *hctx; ++ ++ if (!accum || !new_map || !ts_start_ns || !ts_end_ns) ++ return -EINVAL; ++ ++ hctx = container_of(accum, struct kbase_hwcnt_context, accum); ++ ++ if ((new_map->metadata != hctx->iface->metadata) || ++ (dump_buf && (dump_buf->metadata != hctx->iface->metadata))) ++ return -EINVAL; ++ ++ mutex_lock(&hctx->accum_lock); ++ ++ errcode = kbasep_hwcnt_accumulator_dump( ++ hctx, ts_start_ns, ts_end_ns, dump_buf, new_map); ++ ++ mutex_unlock(&hctx->accum_lock); ++ ++ return errcode; ++} ++KBASE_EXPORT_TEST_API(kbase_hwcnt_accumulator_set_counters); ++ ++int kbase_hwcnt_accumulator_dump( ++ struct kbase_hwcnt_accumulator *accum, ++ u64 *ts_start_ns, ++ u64 *ts_end_ns, ++ struct kbase_hwcnt_dump_buffer *dump_buf) ++{ ++ int errcode; ++ struct kbase_hwcnt_context *hctx; ++ ++ if (!accum || !ts_start_ns || !ts_end_ns) ++ return -EINVAL; ++ ++ hctx = container_of(accum, struct kbase_hwcnt_context, accum); ++ ++ if (dump_buf && (dump_buf->metadata != hctx->iface->metadata)) ++ return -EINVAL; ++ ++ mutex_lock(&hctx->accum_lock); ++ ++ errcode = kbasep_hwcnt_accumulator_dump( ++ hctx, ts_start_ns, ts_end_ns, dump_buf, NULL); ++ ++ mutex_unlock(&hctx->accum_lock); ++ ++ return errcode; ++} ++KBASE_EXPORT_TEST_API(kbase_hwcnt_accumulator_dump); ++ ++u64 kbase_hwcnt_accumulator_timestamp_ns(struct kbase_hwcnt_accumulator *accum) ++{ ++ struct kbase_hwcnt_context *hctx; ++ ++ if (WARN_ON(!accum)) ++ return 0; ++ ++ hctx = container_of(accum, struct kbase_hwcnt_context, accum); ++ return hctx->iface->timestamp_ns(accum->backend); ++} +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_accumulator.h b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_accumulator.h +new file mode 100755 +index 000000000000..eb82ea4bfd14 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_accumulator.h +@@ -0,0 +1,146 @@ ++/* ++ * ++ * (C) COPYRIGHT 2018 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/** ++ * Hardware counter accumulator API. ++ */ ++ ++#ifndef _KBASE_HWCNT_ACCUMULATOR_H_ ++#define _KBASE_HWCNT_ACCUMULATOR_H_ ++ ++#include ++ ++struct kbase_hwcnt_context; ++struct kbase_hwcnt_accumulator; ++struct kbase_hwcnt_enable_map; ++struct kbase_hwcnt_dump_buffer; ++ ++/** ++ * kbase_hwcnt_accumulator_acquire() - Acquire the hardware counter accumulator ++ * for a hardware counter context. ++ * @hctx: Non-NULL pointer to a hardware counter context. ++ * @accum: Non-NULL pointer to where the pointer to the created accumulator ++ * will be stored on success. ++ * ++ * There can exist at most one instance of the hardware counter accumulator per ++ * context at a time. ++ * ++ * If multiple clients need access to the hardware counters at the same time, ++ * then an abstraction built on top of the single instance to the hardware ++ * counter accumulator is required. ++ * ++ * No counters will be enabled with the returned accumulator. A subsequent call ++ * to kbase_hwcnt_accumulator_set_counters must be used to turn them on. ++ * ++ * There are four components to a hardware counter dump: ++ * - A set of enabled counters ++ * - A start time ++ * - An end time ++ * - A dump buffer containing the accumulated counter values for all enabled ++ * counters between the start and end times. ++ * ++ * For each dump, it is guaranteed that all enabled counters were active for the ++ * entirety of the period between the start and end times. ++ * ++ * It is also guaranteed that the start time of dump "n" is always equal to the ++ * end time of dump "n - 1". ++ * ++ * For all dumps, the values of any counters that were not enabled is undefined. ++ * ++ * Return: 0 on success or error code. ++ */ ++int kbase_hwcnt_accumulator_acquire( ++ struct kbase_hwcnt_context *hctx, ++ struct kbase_hwcnt_accumulator **accum); ++ ++/** ++ * kbase_hwcnt_accumulator_release() - Release a hardware counter accumulator. ++ * @accum: Non-NULL pointer to the hardware counter accumulator. ++ * ++ * The accumulator must be released before the context the accumulator was ++ * created from is terminated. ++ */ ++void kbase_hwcnt_accumulator_release(struct kbase_hwcnt_accumulator *accum); ++ ++/** ++ * kbase_hwcnt_accumulator_set_counters() - Perform a dump of the currently ++ * enabled counters, and enable a new ++ * set of counters that will be used ++ * for subsequent dumps. ++ * @accum: Non-NULL pointer to the hardware counter accumulator. ++ * @new_map: Non-NULL pointer to the new counter enable map. Must have the ++ * same metadata as the accumulator. ++ * @ts_start_ns: Non-NULL pointer where the start timestamp of the dump will ++ * be written out to on success. ++ * @ts_end_ns: Non-NULL pointer where the end timestamp of the dump will ++ * be written out to on success. ++ * @dump_buf: Pointer to the buffer where the dump will be written out to on ++ * success. If non-NULL, must have the same metadata as the ++ * accumulator. If NULL, the dump will be discarded. ++ * ++ * If this function fails for some unexpected reason (i.e. anything other than ++ * invalid args), then the accumulator will be put into the error state until ++ * the parent context is next disabled. ++ * ++ * Return: 0 on success or error code. ++ */ ++int kbase_hwcnt_accumulator_set_counters( ++ struct kbase_hwcnt_accumulator *accum, ++ const struct kbase_hwcnt_enable_map *new_map, ++ u64 *ts_start_ns, ++ u64 *ts_end_ns, ++ struct kbase_hwcnt_dump_buffer *dump_buf); ++ ++/** ++ * kbase_hwcnt_accumulator_dump() - Perform a dump of the currently enabled ++ * counters. ++ * @accum: Non-NULL pointer to the hardware counter accumulator. ++ * @ts_start_ns: Non-NULL pointer where the start timestamp of the dump will ++ * be written out to on success. ++ * @ts_end_ns: Non-NULL pointer where the end timestamp of the dump will ++ * be written out to on success. ++ * @dump_buf: Pointer to the buffer where the dump will be written out to on ++ * success. If non-NULL, must have the same metadata as the ++ * accumulator. If NULL, the dump will be discarded. ++ * ++ * If this function fails for some unexpected reason (i.e. anything other than ++ * invalid args), then the accumulator will be put into the error state until ++ * the parent context is next disabled. ++ * ++ * Return: 0 on success or error code. ++ */ ++int kbase_hwcnt_accumulator_dump( ++ struct kbase_hwcnt_accumulator *accum, ++ u64 *ts_start_ns, ++ u64 *ts_end_ns, ++ struct kbase_hwcnt_dump_buffer *dump_buf); ++ ++/** ++ * kbase_hwcnt_accumulator_timestamp_ns() - Get the current accumulator backend ++ * timestamp. ++ * @accum: Non-NULL pointer to the hardware counter accumulator. ++ * ++ * Return: Accumulator backend timestamp in nanoseconds. ++ */ ++u64 kbase_hwcnt_accumulator_timestamp_ns(struct kbase_hwcnt_accumulator *accum); ++ ++#endif /* _KBASE_HWCNT_ACCUMULATOR_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_backend.h b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_backend.h +new file mode 100755 +index 000000000000..3a921b754b55 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_backend.h +@@ -0,0 +1,220 @@ ++/* ++ * ++ * (C) COPYRIGHT 2018, 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* ++ * Virtual interface for hardware counter backends. ++ */ ++ ++#ifndef _KBASE_HWCNT_BACKEND_H_ ++#define _KBASE_HWCNT_BACKEND_H_ ++ ++#include ++ ++struct kbase_hwcnt_metadata; ++struct kbase_hwcnt_enable_map; ++struct kbase_hwcnt_dump_buffer; ++ ++/* ++ * struct kbase_hwcnt_backend_info - Opaque pointer to information used to ++ * create an instance of a hardware counter ++ * backend. ++ */ ++struct kbase_hwcnt_backend_info; ++ ++/* ++ * struct kbase_hwcnt_backend_info - Opaque pointer to a hardware counter ++ * backend, used to perform dumps. ++ */ ++struct kbase_hwcnt_backend; ++ ++/** ++ * typedef kbase_hwcnt_backend_init_fn - Initialise a counter backend. ++ * @info: Non-NULL pointer to backend info. ++ * @out_backend: Non-NULL pointer to where backend is stored on success. ++ * ++ * All uses of the created hardware counter backend must be externally ++ * synchronised. ++ * ++ * Return: 0 on success, else error code. ++ */ ++typedef int (*kbase_hwcnt_backend_init_fn)( ++ const struct kbase_hwcnt_backend_info *info, ++ struct kbase_hwcnt_backend **out_backend); ++ ++/** ++ * typedef kbase_hwcnt_backend_term_fn - Terminate a counter backend. ++ * @backend: Pointer to backend to be terminated. ++ */ ++typedef void (*kbase_hwcnt_backend_term_fn)( ++ struct kbase_hwcnt_backend *backend); ++ ++/** ++ * typedef kbase_hwcnt_backend_timestamp_ns_fn - Get the current backend ++ * timestamp. ++ * @backend: Non-NULL pointer to backend. ++ * ++ * Return: Backend timestamp in nanoseconds. ++ */ ++typedef u64 (*kbase_hwcnt_backend_timestamp_ns_fn)( ++ struct kbase_hwcnt_backend *backend); ++ ++/** ++ * typedef kbase_hwcnt_backend_dump_enable_fn - Start counter dumping with the ++ * backend. ++ * @backend: Non-NULL pointer to backend. ++ * @enable_map: Non-NULL pointer to enable map specifying enabled counters. ++ * ++ * The enable_map must have been created using the interface's metadata. ++ * If the backend has already been enabled, an error is returned. ++ * ++ * May be called in an atomic context. ++ * ++ * Return: 0 on success, else error code. ++ */ ++typedef int (*kbase_hwcnt_backend_dump_enable_fn)( ++ struct kbase_hwcnt_backend *backend, ++ const struct kbase_hwcnt_enable_map *enable_map); ++ ++/** ++ * typedef kbase_hwcnt_backend_dump_enable_nolock_fn - Start counter dumping ++ * with the backend. ++ * @backend: Non-NULL pointer to backend. ++ * @enable_map: Non-NULL pointer to enable map specifying enabled counters. ++ * ++ * Exactly the same as kbase_hwcnt_backend_dump_enable_fn(), except must be ++ * called in an atomic context with the spinlock documented by the specific ++ * backend interface held. ++ * ++ * Return: 0 on success, else error code. ++ */ ++typedef int (*kbase_hwcnt_backend_dump_enable_nolock_fn)( ++ struct kbase_hwcnt_backend *backend, ++ const struct kbase_hwcnt_enable_map *enable_map); ++ ++/** ++ * typedef kbase_hwcnt_backend_dump_disable_fn - Disable counter dumping with ++ * the backend. ++ * @backend: Non-NULL pointer to backend. ++ * ++ * If the backend is already disabled, does nothing. ++ * Any undumped counter values since the last dump get will be lost. ++ */ ++typedef void (*kbase_hwcnt_backend_dump_disable_fn)( ++ struct kbase_hwcnt_backend *backend); ++ ++/** ++ * typedef kbase_hwcnt_backend_dump_clear_fn - Reset all the current undumped ++ * counters. ++ * @backend: Non-NULL pointer to backend. ++ * ++ * If the backend is not enabled, returns an error. ++ * ++ * Return: 0 on success, else error code. ++ */ ++typedef int (*kbase_hwcnt_backend_dump_clear_fn)( ++ struct kbase_hwcnt_backend *backend); ++ ++/** ++ * typedef kbase_hwcnt_backend_dump_request_fn - Request an asynchronous counter ++ * dump. ++ * @backend: Non-NULL pointer to backend. ++ * @dump_time_ns: Non-NULL pointer where the timestamp of when the dump was ++ * requested will be written out to on success. ++ * ++ * If the backend is not enabled or another dump is already in progress, ++ * returns an error. ++ * ++ * Return: 0 on success, else error code. ++ */ ++typedef int (*kbase_hwcnt_backend_dump_request_fn)( ++ struct kbase_hwcnt_backend *backend, ++ u64 *dump_time_ns); ++ ++/** ++ * typedef kbase_hwcnt_backend_dump_wait_fn - Wait until the last requested ++ * counter dump has completed. ++ * @backend: Non-NULL pointer to backend. ++ * ++ * If the backend is not enabled, returns an error. ++ * ++ * Return: 0 on success, else error code. ++ */ ++typedef int (*kbase_hwcnt_backend_dump_wait_fn)( ++ struct kbase_hwcnt_backend *backend); ++ ++/** ++ * typedef kbase_hwcnt_backend_dump_get_fn - Copy or accumulate enable the ++ * counters dumped after the last dump ++ * request into the dump buffer. ++ * @backend: Non-NULL pointer to backend. ++ * @dump_buffer: Non-NULL pointer to destination dump buffer. ++ * @enable_map: Non-NULL pointer to enable map specifying enabled values. ++ * @accumulate: True if counters should be accumulated into dump_buffer, rather ++ * than copied. ++ * ++ * If the backend is not enabled, returns an error. ++ * If a dump is in progress (i.e. dump_wait has not yet returned successfully) ++ * then the resultant contents of the dump buffer will be undefined. ++ * ++ * Return: 0 on success, else error code. ++ */ ++typedef int (*kbase_hwcnt_backend_dump_get_fn)( ++ struct kbase_hwcnt_backend *backend, ++ struct kbase_hwcnt_dump_buffer *dump_buffer, ++ const struct kbase_hwcnt_enable_map *enable_map, ++ bool accumulate); ++ ++/** ++ * struct kbase_hwcnt_backend_interface - Hardware counter backend virtual ++ * interface. ++ * @metadata: Immutable hardware counter metadata. ++ * @info: Immutable info used to initialise an instance of the ++ * backend. ++ * @init: Function ptr to initialise an instance of the backend. ++ * @term: Function ptr to terminate an instance of the backend. ++ * @timestamp_ns: Function ptr to get the current backend timestamp. ++ * @dump_enable: Function ptr to enable dumping. ++ * @dump_enable_nolock: Function ptr to enable dumping while the ++ * backend-specific spinlock is already held. ++ * @dump_disable: Function ptr to disable dumping. ++ * @dump_clear: Function ptr to clear counters. ++ * @dump_request: Function ptr to request a dump. ++ * @dump_wait: Function ptr to wait until dump to complete. ++ * @dump_get: Function ptr to copy or accumulate dump into a dump ++ * buffer. ++ */ ++struct kbase_hwcnt_backend_interface { ++ const struct kbase_hwcnt_metadata *metadata; ++ const struct kbase_hwcnt_backend_info *info; ++ kbase_hwcnt_backend_init_fn init; ++ kbase_hwcnt_backend_term_fn term; ++ kbase_hwcnt_backend_timestamp_ns_fn timestamp_ns; ++ kbase_hwcnt_backend_dump_enable_fn dump_enable; ++ kbase_hwcnt_backend_dump_enable_nolock_fn dump_enable_nolock; ++ kbase_hwcnt_backend_dump_disable_fn dump_disable; ++ kbase_hwcnt_backend_dump_clear_fn dump_clear; ++ kbase_hwcnt_backend_dump_request_fn dump_request; ++ kbase_hwcnt_backend_dump_wait_fn dump_wait; ++ kbase_hwcnt_backend_dump_get_fn dump_get; ++}; ++ ++#endif /* _KBASE_HWCNT_BACKEND_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_backend_jm.c b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_backend_jm.c +new file mode 100755 +index 000000000000..9f65de41694f +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_backend_jm.c +@@ -0,0 +1,736 @@ ++/* ++ * ++ * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include "mali_kbase_hwcnt_backend_jm.h" ++#include "mali_kbase_hwcnt_gpu.h" ++#include "mali_kbase_hwcnt_types.h" ++#include "mali_kbase.h" ++#include "mali_kbase_pm_ca.h" ++#include "mali_kbase_hwaccess_instr.h" ++#include "mali_kbase_hwaccess_time.h" ++#include "mali_kbase_ccswe.h" ++ ++#ifdef CONFIG_MALI_BIFROST_NO_MALI ++#include "backend/gpu/mali_kbase_model_dummy.h" ++#endif ++#include "backend/gpu/mali_kbase_clk_rate_trace_mgr.h" ++ ++#if MALI_USE_CSF ++#include "mali_kbase_ctx_sched.h" ++#else ++#include "backend/gpu/mali_kbase_pm_internal.h" ++#endif ++ ++/** ++ * struct kbase_hwcnt_backend_jm_info - Information used to create an instance ++ * of a JM hardware counter backend. ++ * @kbdev: KBase device. ++ * @use_secondary: True if secondary performance counters should be used, ++ * else false. Ignored if secondary counters are not supported. ++ * @metadata: Hardware counter metadata. ++ * @dump_bytes: Bytes of GPU memory required to perform a ++ * hardware counter dump. ++ */ ++struct kbase_hwcnt_backend_jm_info { ++ struct kbase_device *kbdev; ++ bool use_secondary; ++ const struct kbase_hwcnt_metadata *metadata; ++ size_t dump_bytes; ++}; ++ ++/** ++ * struct kbase_hwcnt_backend_jm - Instance of a JM hardware counter backend. ++ * @info: Info used to create the backend. ++ * @kctx: KBase context used for GPU memory allocation and ++ * counter dumping. ++ * @gpu_dump_va: GPU hardware counter dump buffer virtual address. ++ * @cpu_dump_va: CPU mapping of gpu_dump_va. ++ * @vmap: Dump buffer vmap. ++ * @enabled: True if dumping has been enabled, else false. ++ * @pm_core_mask: PM state sync-ed shaders core mask for the enabled ++ * dumping. ++ * @clk_enable_map: The enable map specifying enabled clock domains. ++ * @cycle_count_elapsed: ++ * Cycle count elapsed for a given sample period. ++ * The top clock cycle, index 0, is read directly from ++ * hardware, but the other clock domains need to be ++ * calculated with software estimation. ++ * @prev_cycle_count: Previous cycle count to calculate the cycle count for ++ * sample period. ++ * @rate_listener: Clock rate listener callback state. ++ * @ccswe_shader_cores: Shader cores cycle count software estimator. ++ */ ++struct kbase_hwcnt_backend_jm { ++ const struct kbase_hwcnt_backend_jm_info *info; ++ struct kbase_context *kctx; ++ u64 gpu_dump_va; ++ void *cpu_dump_va; ++ struct kbase_vmap_struct *vmap; ++ bool enabled; ++ u64 pm_core_mask; ++ u64 clk_enable_map; ++ u64 cycle_count_elapsed[BASE_MAX_NR_CLOCKS_REGULATORS]; ++ u64 prev_cycle_count[BASE_MAX_NR_CLOCKS_REGULATORS]; ++ struct kbase_clk_rate_listener rate_listener; ++ struct kbase_ccswe ccswe_shader_cores; ++}; ++ ++/** ++ * kbasep_hwcnt_backend_jm_on_freq_change() - On freq change callback ++ * ++ * @rate_listener: Callback state ++ * @clk_index: Clock index ++ * @clk_rate_hz: Clock frequency(hz) ++ */ ++static void kbasep_hwcnt_backend_jm_on_freq_change( ++ struct kbase_clk_rate_listener *rate_listener, ++ u32 clk_index, ++ u32 clk_rate_hz) ++{ ++ struct kbase_hwcnt_backend_jm *backend_jm = container_of( ++ rate_listener, struct kbase_hwcnt_backend_jm, rate_listener); ++ u64 timestamp_ns; ++ ++ if (clk_index != KBASE_CLOCK_DOMAIN_SHADER_CORES) ++ return; ++ ++ timestamp_ns = ktime_get_raw_ns(); ++ kbase_ccswe_freq_change( ++ &backend_jm->ccswe_shader_cores, timestamp_ns, clk_rate_hz); ++} ++ ++/** ++ * kbasep_hwcnt_backend_jm_cc_enable() - Enable cycle count tracking ++ * ++ * @backend: Non-NULL pointer to backend. ++ * @enable_map: Non-NULL pointer to enable map specifying enabled counters. ++ * @timestamp_ns: Timestamp(ns) when HWCNT were enabled. ++ */ ++static void kbasep_hwcnt_backend_jm_cc_enable( ++ struct kbase_hwcnt_backend_jm *backend_jm, ++ const struct kbase_hwcnt_enable_map *enable_map, ++ u64 timestamp_ns) ++{ ++ struct kbase_device *kbdev = backend_jm->kctx->kbdev; ++ u64 clk_enable_map = enable_map->clk_enable_map; ++ u64 cycle_count; ++ ++ if (kbase_hwcnt_clk_enable_map_enabled( ++ clk_enable_map, KBASE_CLOCK_DOMAIN_TOP)) { ++#if !MALI_USE_CSF ++ /* turn on the cycle counter */ ++ kbase_pm_request_gpu_cycle_counter_l2_is_on(kbdev); ++#endif ++ /* Read cycle count for top clock domain. */ ++ kbase_backend_get_gpu_time_norequest( ++ kbdev, &cycle_count, NULL, NULL); ++ ++ backend_jm->prev_cycle_count[KBASE_CLOCK_DOMAIN_TOP] = ++ cycle_count; ++ } ++ ++ if (kbase_hwcnt_clk_enable_map_enabled( ++ clk_enable_map, KBASE_CLOCK_DOMAIN_SHADER_CORES)) { ++ /* software estimation for non-top clock domains */ ++ struct kbase_clk_rate_trace_manager *rtm = &kbdev->pm.clk_rtm; ++ const struct kbase_clk_data *clk_data = ++ rtm->clks[KBASE_CLOCK_DOMAIN_SHADER_CORES]; ++ u32 cur_freq; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&rtm->lock, flags); ++ ++ cur_freq = (u32) clk_data->clock_val; ++ kbase_ccswe_reset(&backend_jm->ccswe_shader_cores); ++ kbase_ccswe_freq_change( ++ &backend_jm->ccswe_shader_cores, ++ timestamp_ns, ++ cur_freq); ++ ++ kbase_clk_rate_trace_manager_subscribe_no_lock( ++ rtm, &backend_jm->rate_listener); ++ ++ spin_unlock_irqrestore(&rtm->lock, flags); ++ ++ /* ccswe was reset. The estimated cycle is zero. */ ++ backend_jm->prev_cycle_count[ ++ KBASE_CLOCK_DOMAIN_SHADER_CORES] = 0; ++ } ++ ++ /* Keep clk_enable_map for dump_request. */ ++ backend_jm->clk_enable_map = clk_enable_map; ++} ++ ++/** ++ * kbasep_hwcnt_backend_jm_cc_disable() - Disable cycle count tracking ++ * ++ * @backend: Non-NULL pointer to backend. ++ */ ++static void kbasep_hwcnt_backend_jm_cc_disable( ++ struct kbase_hwcnt_backend_jm *backend_jm) ++{ ++ struct kbase_device *kbdev = backend_jm->kctx->kbdev; ++ struct kbase_clk_rate_trace_manager *rtm = &kbdev->pm.clk_rtm; ++ u64 clk_enable_map = backend_jm->clk_enable_map; ++ ++#if !MALI_USE_CSF ++ if (kbase_hwcnt_clk_enable_map_enabled( ++ clk_enable_map, KBASE_CLOCK_DOMAIN_TOP)) { ++ /* turn off the cycle counter */ ++ kbase_pm_release_gpu_cycle_counter(kbdev); ++ } ++#endif ++ if (kbase_hwcnt_clk_enable_map_enabled( ++ clk_enable_map, KBASE_CLOCK_DOMAIN_SHADER_CORES)) { ++ ++ kbase_clk_rate_trace_manager_unsubscribe( ++ rtm, &backend_jm->rate_listener); ++ } ++} ++ ++ ++/* JM backend implementation of kbase_hwcnt_backend_timestamp_ns_fn */ ++static u64 kbasep_hwcnt_backend_jm_timestamp_ns( ++ struct kbase_hwcnt_backend *backend) ++{ ++ (void)backend; ++ return ktime_get_raw_ns(); ++} ++ ++/* JM backend implementation of kbase_hwcnt_backend_dump_enable_nolock_fn */ ++static int kbasep_hwcnt_backend_jm_dump_enable_nolock( ++ struct kbase_hwcnt_backend *backend, ++ const struct kbase_hwcnt_enable_map *enable_map) ++{ ++ int errcode; ++ struct kbase_hwcnt_backend_jm *backend_jm = ++ (struct kbase_hwcnt_backend_jm *)backend; ++ struct kbase_context *kctx; ++ struct kbase_device *kbdev; ++ struct kbase_hwcnt_physical_enable_map phys; ++ struct kbase_instr_hwcnt_enable enable; ++ u64 timestamp_ns; ++ ++ if (!backend_jm || !enable_map || backend_jm->enabled || ++ (enable_map->metadata != backend_jm->info->metadata)) ++ return -EINVAL; ++ ++ kctx = backend_jm->kctx; ++ kbdev = backend_jm->kctx->kbdev; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ kbase_hwcnt_gpu_enable_map_to_physical(&phys, enable_map); ++ ++ enable.fe_bm = phys.fe_bm; ++ enable.shader_bm = phys.shader_bm; ++ enable.tiler_bm = phys.tiler_bm; ++ enable.mmu_l2_bm = phys.mmu_l2_bm; ++ enable.use_secondary = backend_jm->info->use_secondary; ++ enable.dump_buffer = backend_jm->gpu_dump_va; ++ enable.dump_buffer_bytes = backend_jm->info->dump_bytes; ++ ++ timestamp_ns = kbasep_hwcnt_backend_jm_timestamp_ns(backend); ++ ++ errcode = kbase_instr_hwcnt_enable_internal(kbdev, kctx, &enable); ++ if (errcode) ++ goto error; ++ ++ backend_jm->pm_core_mask = kbase_pm_ca_get_instr_core_mask(kbdev); ++ backend_jm->enabled = true; ++ ++ kbasep_hwcnt_backend_jm_cc_enable(backend_jm, enable_map, timestamp_ns); ++ ++ return 0; ++error: ++ return errcode; ++} ++ ++/* JM backend implementation of kbase_hwcnt_backend_dump_enable_fn */ ++static int kbasep_hwcnt_backend_jm_dump_enable( ++ struct kbase_hwcnt_backend *backend, ++ const struct kbase_hwcnt_enable_map *enable_map) ++{ ++ unsigned long flags; ++ int errcode; ++ struct kbase_hwcnt_backend_jm *backend_jm = ++ (struct kbase_hwcnt_backend_jm *)backend; ++ struct kbase_device *kbdev; ++ ++ if (!backend_jm) ++ return -EINVAL; ++ ++ kbdev = backend_jm->kctx->kbdev; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ errcode = kbasep_hwcnt_backend_jm_dump_enable_nolock( ++ backend, enable_map); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ return errcode; ++} ++ ++/* JM backend implementation of kbase_hwcnt_backend_dump_disable_fn */ ++static void kbasep_hwcnt_backend_jm_dump_disable( ++ struct kbase_hwcnt_backend *backend) ++{ ++ int errcode; ++ struct kbase_hwcnt_backend_jm *backend_jm = ++ (struct kbase_hwcnt_backend_jm *)backend; ++ ++ if (WARN_ON(!backend_jm) || !backend_jm->enabled) ++ return; ++ ++ kbasep_hwcnt_backend_jm_cc_disable(backend_jm); ++ ++ errcode = kbase_instr_hwcnt_disable_internal(backend_jm->kctx); ++ WARN_ON(errcode); ++ ++ backend_jm->enabled = false; ++} ++ ++/* JM backend implementation of kbase_hwcnt_backend_dump_clear_fn */ ++static int kbasep_hwcnt_backend_jm_dump_clear( ++ struct kbase_hwcnt_backend *backend) ++{ ++ struct kbase_hwcnt_backend_jm *backend_jm = ++ (struct kbase_hwcnt_backend_jm *)backend; ++ ++ if (!backend_jm || !backend_jm->enabled) ++ return -EINVAL; ++ ++ return kbase_instr_hwcnt_clear(backend_jm->kctx); ++} ++ ++/* JM backend implementation of kbase_hwcnt_backend_dump_request_fn */ ++static int kbasep_hwcnt_backend_jm_dump_request( ++ struct kbase_hwcnt_backend *backend, ++ u64 *dump_time_ns) ++{ ++ struct kbase_hwcnt_backend_jm *backend_jm = ++ (struct kbase_hwcnt_backend_jm *)backend; ++ struct kbase_device *kbdev; ++ const struct kbase_hwcnt_metadata *metadata; ++ u64 current_cycle_count; ++ size_t clk; ++ int ret; ++ ++ if (!backend_jm || !backend_jm->enabled) ++ return -EINVAL; ++ ++ kbdev = backend_jm->kctx->kbdev; ++ metadata = backend_jm->info->metadata; ++ ++ /* Disable pre-emption, to make the timestamp as accurate as possible */ ++ preempt_disable(); ++ { ++ *dump_time_ns = kbasep_hwcnt_backend_jm_timestamp_ns(backend); ++ ret = kbase_instr_hwcnt_request_dump(backend_jm->kctx); ++ ++ kbase_hwcnt_metadata_for_each_clock(metadata, clk) { ++ if (!kbase_hwcnt_clk_enable_map_enabled( ++ backend_jm->clk_enable_map, clk)) ++ continue; ++ ++ if (clk == KBASE_CLOCK_DOMAIN_TOP) { ++ /* Read cycle count for top clock domain. */ ++ kbase_backend_get_gpu_time_norequest( ++ kbdev, ¤t_cycle_count, ++ NULL, NULL); ++ } else { ++ /* ++ * Estimate cycle count for non-top clock ++ * domain. ++ */ ++ current_cycle_count = kbase_ccswe_cycle_at( ++ &backend_jm->ccswe_shader_cores, ++ *dump_time_ns); ++ } ++ backend_jm->cycle_count_elapsed[clk] = ++ current_cycle_count - ++ backend_jm->prev_cycle_count[clk]; ++ ++ /* ++ * Keep the current cycle count for later calculation. ++ */ ++ backend_jm->prev_cycle_count[clk] = current_cycle_count; ++ } ++ } ++ preempt_enable(); ++ ++ return ret; ++} ++ ++/* JM backend implementation of kbase_hwcnt_backend_dump_wait_fn */ ++static int kbasep_hwcnt_backend_jm_dump_wait( ++ struct kbase_hwcnt_backend *backend) ++{ ++ struct kbase_hwcnt_backend_jm *backend_jm = ++ (struct kbase_hwcnt_backend_jm *)backend; ++ ++ if (!backend_jm || !backend_jm->enabled) ++ return -EINVAL; ++ ++ return kbase_instr_hwcnt_wait_for_dump(backend_jm->kctx); ++} ++ ++/* JM backend implementation of kbase_hwcnt_backend_dump_get_fn */ ++static int kbasep_hwcnt_backend_jm_dump_get( ++ struct kbase_hwcnt_backend *backend, ++ struct kbase_hwcnt_dump_buffer *dst, ++ const struct kbase_hwcnt_enable_map *dst_enable_map, ++ bool accumulate) ++{ ++ struct kbase_hwcnt_backend_jm *backend_jm = ++ (struct kbase_hwcnt_backend_jm *)backend; ++ size_t clk; ++ ++ if (!backend_jm || !dst || !dst_enable_map || ++ (backend_jm->info->metadata != dst->metadata) || ++ (dst_enable_map->metadata != dst->metadata)) ++ return -EINVAL; ++ ++ /* Invalidate the kernel buffer before reading from it. */ ++ kbase_sync_mem_regions( ++ backend_jm->kctx, backend_jm->vmap, KBASE_SYNC_TO_CPU); ++ ++ kbase_hwcnt_metadata_for_each_clock(dst_enable_map->metadata, clk) { ++ if (!kbase_hwcnt_clk_enable_map_enabled( ++ dst_enable_map->clk_enable_map, clk)) ++ continue; ++ ++ /* Extract elapsed cycle count for each clock domain. */ ++ dst->clk_cnt_buf[clk] = backend_jm->cycle_count_elapsed[clk]; ++ } ++ ++ return kbase_hwcnt_gpu_dump_get( ++ dst, backend_jm->cpu_dump_va, dst_enable_map, ++ backend_jm->pm_core_mask, accumulate); ++} ++ ++/** ++ * kbasep_hwcnt_backend_jm_dump_alloc() - Allocate a GPU dump buffer. ++ * @info: Non-NULL pointer to JM backend info. ++ * @kctx: Non-NULL pointer to kbase context. ++ * @gpu_dump_va: Non-NULL pointer to where GPU dump buffer virtual address ++ * is stored on success. ++ * ++ * Return: 0 on success, else error code. ++ */ ++static int kbasep_hwcnt_backend_jm_dump_alloc( ++ const struct kbase_hwcnt_backend_jm_info *info, ++ struct kbase_context *kctx, ++ u64 *gpu_dump_va) ++{ ++ struct kbase_va_region *reg; ++ u64 flags; ++ u64 nr_pages; ++ ++ WARN_ON(!info); ++ WARN_ON(!kctx); ++ WARN_ON(!gpu_dump_va); ++ ++ flags = BASE_MEM_PROT_CPU_RD | ++ BASE_MEM_PROT_GPU_WR | ++ BASEP_MEM_PERMANENT_KERNEL_MAPPING | ++ BASE_MEM_CACHED_CPU; ++ ++ if (kctx->kbdev->mmu_mode->flags & KBASE_MMU_MODE_HAS_NON_CACHEABLE) ++ flags |= BASE_MEM_UNCACHED_GPU; ++ ++ nr_pages = PFN_UP(info->dump_bytes); ++ ++ reg = kbase_mem_alloc(kctx, nr_pages, nr_pages, 0, &flags, gpu_dump_va); ++ ++ if (!reg) ++ return -ENOMEM; ++ ++ return 0; ++} ++ ++/** ++ * kbasep_hwcnt_backend_jm_dump_free() - Free an allocated GPU dump buffer. ++ * @kctx: Non-NULL pointer to kbase context. ++ * @gpu_dump_va: GPU dump buffer virtual address. ++ */ ++static void kbasep_hwcnt_backend_jm_dump_free( ++ struct kbase_context *kctx, ++ u64 gpu_dump_va) ++{ ++ WARN_ON(!kctx); ++ if (gpu_dump_va) ++ kbase_mem_free(kctx, gpu_dump_va); ++} ++ ++/** ++ * kbasep_hwcnt_backend_jm_destroy() - Destroy a JM backend. ++ * @backend: Pointer to JM backend to destroy. ++ * ++ * Can be safely called on a backend in any state of partial construction. ++ */ ++static void kbasep_hwcnt_backend_jm_destroy( ++ struct kbase_hwcnt_backend_jm *backend) ++{ ++ if (!backend) ++ return; ++ ++ if (backend->kctx) { ++#if MALI_USE_CSF ++ unsigned long flags; ++#endif ++ struct kbase_context *kctx = backend->kctx; ++ struct kbase_device *kbdev = kctx->kbdev; ++ ++ if (backend->cpu_dump_va) ++ kbase_phy_alloc_mapping_put(kctx, backend->vmap); ++ ++ if (backend->gpu_dump_va) ++ kbasep_hwcnt_backend_jm_dump_free( ++ kctx, backend->gpu_dump_va); ++ ++#if MALI_USE_CSF ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_ctx_sched_release_ctx(kctx); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++#else ++ kbasep_js_release_privileged_ctx(kbdev, kctx); ++#endif ++ kbase_destroy_context(kctx); ++ } ++ ++ kfree(backend); ++} ++ ++/** ++ * kbasep_hwcnt_backend_jm_create() - Create a JM backend. ++ * @info: Non-NULL pointer to backend info. ++ * @out_backend: Non-NULL pointer to where backend is stored on success. ++ * ++ * Return: 0 on success, else error code. ++ */ ++static int kbasep_hwcnt_backend_jm_create( ++ const struct kbase_hwcnt_backend_jm_info *info, ++ struct kbase_hwcnt_backend_jm **out_backend) ++{ ++#if MALI_USE_CSF ++ unsigned long flags; ++#endif ++ int errcode; ++ struct kbase_device *kbdev; ++ struct kbase_hwcnt_backend_jm *backend = NULL; ++ ++ WARN_ON(!info); ++ WARN_ON(!out_backend); ++ ++ kbdev = info->kbdev; ++ ++ backend = kzalloc(sizeof(*backend), GFP_KERNEL); ++ if (!backend) ++ goto alloc_error; ++ ++ backend->info = info; ++ ++ backend->kctx = kbase_create_context(kbdev, true, ++ BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED, 0, NULL); ++ if (!backend->kctx) ++ goto alloc_error; ++ ++#if MALI_USE_CSF ++ kbase_pm_context_active(kbdev); ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_ctx_sched_retain_ctx(backend->kctx); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ kbase_pm_context_idle(kbdev); ++#else ++ kbasep_js_schedule_privileged_ctx(kbdev, backend->kctx); ++#endif ++ ++ errcode = kbasep_hwcnt_backend_jm_dump_alloc( ++ info, backend->kctx, &backend->gpu_dump_va); ++ if (errcode) ++ goto error; ++ ++ backend->cpu_dump_va = kbase_phy_alloc_mapping_get(backend->kctx, ++ backend->gpu_dump_va, &backend->vmap); ++ if (!backend->cpu_dump_va) ++ goto alloc_error; ++ ++ kbase_ccswe_init(&backend->ccswe_shader_cores); ++ backend->rate_listener.notify = kbasep_hwcnt_backend_jm_on_freq_change; ++ ++#ifdef CONFIG_MALI_BIFROST_NO_MALI ++ /* The dummy model needs the CPU mapping. */ ++ gpu_model_set_dummy_prfcnt_base_cpu(backend->cpu_dump_va); ++#endif ++ ++ *out_backend = backend; ++ return 0; ++ ++alloc_error: ++ errcode = -ENOMEM; ++error: ++ kbasep_hwcnt_backend_jm_destroy(backend); ++ return errcode; ++} ++ ++/* JM backend implementation of kbase_hwcnt_backend_init_fn */ ++static int kbasep_hwcnt_backend_jm_init( ++ const struct kbase_hwcnt_backend_info *info, ++ struct kbase_hwcnt_backend **out_backend) ++{ ++ int errcode; ++ struct kbase_hwcnt_backend_jm *backend = NULL; ++ ++ if (!info || !out_backend) ++ return -EINVAL; ++ ++ errcode = kbasep_hwcnt_backend_jm_create( ++ (const struct kbase_hwcnt_backend_jm_info *) info, &backend); ++ if (errcode) ++ return errcode; ++ ++ *out_backend = (struct kbase_hwcnt_backend *)backend; ++ ++ return 0; ++} ++ ++/* JM backend implementation of kbase_hwcnt_backend_term_fn */ ++static void kbasep_hwcnt_backend_jm_term(struct kbase_hwcnt_backend *backend) ++{ ++ if (!backend) ++ return; ++ ++ kbasep_hwcnt_backend_jm_dump_disable(backend); ++ kbasep_hwcnt_backend_jm_destroy( ++ (struct kbase_hwcnt_backend_jm *)backend); ++} ++ ++/** ++ * kbasep_hwcnt_backend_jm_info_destroy() - Destroy a JM backend info. ++ * @info: Pointer to info to destroy. ++ * ++ * Can be safely called on a backend info in any state of partial construction. ++ */ ++static void kbasep_hwcnt_backend_jm_info_destroy( ++ const struct kbase_hwcnt_backend_jm_info *info) ++{ ++ if (!info) ++ return; ++ ++ kbase_hwcnt_gpu_metadata_destroy(info->metadata); ++ kfree(info); ++} ++ ++/** ++ * kbasep_hwcnt_backend_jm_info_create() - Create a JM backend info. ++ * @kbdev: Non_NULL pointer to kbase device. ++ * @out_info: Non-NULL pointer to where info is stored on success. ++ * ++ * Return 0 on success, else error code. ++ */ ++static int kbasep_hwcnt_backend_jm_info_create( ++ struct kbase_device *kbdev, ++ const struct kbase_hwcnt_backend_jm_info **out_info) ++{ ++ int errcode = -ENOMEM; ++ struct kbase_hwcnt_gpu_info hwcnt_gpu_info; ++ struct kbase_hwcnt_backend_jm_info *info = NULL; ++ ++ WARN_ON(!kbdev); ++ WARN_ON(!out_info); ++ ++ errcode = kbase_hwcnt_gpu_info_init(kbdev, &hwcnt_gpu_info); ++ if (errcode) ++ return errcode; ++ ++ info = kzalloc(sizeof(*info), GFP_KERNEL); ++ if (!info) ++ goto error; ++ ++ info->kbdev = kbdev; ++ ++#ifdef CONFIG_MALI_BIFROST_PRFCNT_SET_SECONDARY ++ info->use_secondary = true; ++#else ++ info->use_secondary = false; ++#endif ++ ++ errcode = kbase_hwcnt_gpu_metadata_create( ++ &hwcnt_gpu_info, info->use_secondary, ++ &info->metadata, ++ &info->dump_bytes); ++ if (errcode) ++ goto error; ++ ++ *out_info = info; ++ ++ return 0; ++error: ++ kbasep_hwcnt_backend_jm_info_destroy(info); ++ return errcode; ++} ++ ++int kbase_hwcnt_backend_jm_create( ++ struct kbase_device *kbdev, ++ struct kbase_hwcnt_backend_interface *iface) ++{ ++ int errcode; ++ const struct kbase_hwcnt_backend_jm_info *info = NULL; ++ ++ if (!kbdev || !iface) ++ return -EINVAL; ++ ++ errcode = kbasep_hwcnt_backend_jm_info_create(kbdev, &info); ++ ++ if (errcode) ++ return errcode; ++ ++ iface->metadata = info->metadata; ++ iface->info = (struct kbase_hwcnt_backend_info *)info; ++ iface->init = kbasep_hwcnt_backend_jm_init; ++ iface->term = kbasep_hwcnt_backend_jm_term; ++ iface->timestamp_ns = kbasep_hwcnt_backend_jm_timestamp_ns; ++ iface->dump_enable = kbasep_hwcnt_backend_jm_dump_enable; ++ iface->dump_enable_nolock = kbasep_hwcnt_backend_jm_dump_enable_nolock; ++ iface->dump_disable = kbasep_hwcnt_backend_jm_dump_disable; ++ iface->dump_clear = kbasep_hwcnt_backend_jm_dump_clear; ++ iface->dump_request = kbasep_hwcnt_backend_jm_dump_request; ++ iface->dump_wait = kbasep_hwcnt_backend_jm_dump_wait; ++ iface->dump_get = kbasep_hwcnt_backend_jm_dump_get; ++ ++ return 0; ++} ++ ++void kbase_hwcnt_backend_jm_destroy( ++ struct kbase_hwcnt_backend_interface *iface) ++{ ++ if (!iface) ++ return; ++ ++ kbasep_hwcnt_backend_jm_info_destroy( ++ (const struct kbase_hwcnt_backend_jm_info *)iface->info); ++ memset(iface, 0, sizeof(*iface)); ++} +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_backend_jm.h b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_backend_jm.h +new file mode 100755 +index 000000000000..f15faeba704a +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_backend_jm.h +@@ -0,0 +1,61 @@ ++/* ++ * ++ * (C) COPYRIGHT 2018, 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/** ++ * Concrete implementation of mali_kbase_hwcnt_backend interface for JM ++ * backend. ++ */ ++ ++#ifndef _KBASE_HWCNT_BACKEND_JM_H_ ++#define _KBASE_HWCNT_BACKEND_JM_H_ ++ ++#include "mali_kbase_hwcnt_backend.h" ++ ++struct kbase_device; ++ ++/** ++ * kbase_hwcnt_backend_jm_create() - Create a JM hardware counter backend ++ * interface. ++ * @kbdev: Non-NULL pointer to kbase device. ++ * @iface: Non-NULL pointer to backend interface structure that is filled in ++ * on creation success. ++ * ++ * Calls to iface->dump_enable_nolock() require kbdev->hwaccess_lock held. ++ * ++ * Return: 0 on success, else error code. ++ */ ++int kbase_hwcnt_backend_jm_create( ++ struct kbase_device *kbdev, ++ struct kbase_hwcnt_backend_interface *iface); ++ ++/** ++ * kbase_hwcnt_backend_jm_destroy() - Destroy a JM hardware counter backend ++ * interface. ++ * @iface: Pointer to interface to destroy. ++ * ++ * Can be safely called on an all-zeroed interface, or on an already destroyed ++ * interface. ++ */ ++void kbase_hwcnt_backend_jm_destroy( ++ struct kbase_hwcnt_backend_interface *iface); ++ ++#endif /* _KBASE_HWCNT_BACKEND_JM_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_context.h b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_context.h +new file mode 100755 +index 000000000000..bc50ad12c2f4 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_context.h +@@ -0,0 +1,119 @@ ++/* ++ * ++ * (C) COPYRIGHT 2018 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/** ++ * Hardware counter context API. ++ */ ++ ++#ifndef _KBASE_HWCNT_CONTEXT_H_ ++#define _KBASE_HWCNT_CONTEXT_H_ ++ ++#include ++ ++struct kbase_hwcnt_backend_interface; ++struct kbase_hwcnt_context; ++ ++/** ++ * kbase_hwcnt_context_init() - Initialise a hardware counter context. ++ * @iface: Non-NULL pointer to a hardware counter backend interface. ++ * @out_hctx: Non-NULL pointer to where the pointer to the created context will ++ * be stored on success. ++ * ++ * On creation, the disable count of the context will be 0. ++ * A hardware counter accumulator can be acquired using a created context. ++ * ++ * Return: 0 on success, else error code. ++ */ ++int kbase_hwcnt_context_init( ++ const struct kbase_hwcnt_backend_interface *iface, ++ struct kbase_hwcnt_context **out_hctx); ++ ++/** ++ * kbase_hwcnt_context_term() - Terminate a hardware counter context. ++ * @hctx: Pointer to context to be terminated. ++ */ ++void kbase_hwcnt_context_term(struct kbase_hwcnt_context *hctx); ++ ++/** ++ * kbase_hwcnt_context_metadata() - Get the hardware counter metadata used by ++ * the context, so related counter data ++ * structures can be created. ++ * @hctx: Non-NULL pointer to the hardware counter context. ++ * ++ * Return: Non-NULL pointer to metadata, or NULL on error. ++ */ ++const struct kbase_hwcnt_metadata *kbase_hwcnt_context_metadata( ++ struct kbase_hwcnt_context *hctx); ++ ++/** ++ * kbase_hwcnt_context_disable() - Increment the disable count of the context. ++ * @hctx: Pointer to the hardware counter context. ++ * ++ * If a call to this function increments the disable count from 0 to 1, and ++ * an accumulator has been acquired, then a counter dump will be performed ++ * before counters are disabled via the backend interface. ++ * ++ * Subsequent dumps via the accumulator while counters are disabled will first ++ * return the accumulated dump, then will return dumps with zeroed counters. ++ * ++ * After this function call returns, it is guaranteed that counters will not be ++ * enabled via the backend interface. ++ */ ++void kbase_hwcnt_context_disable(struct kbase_hwcnt_context *hctx); ++ ++/** ++ * kbase_hwcnt_context_disable_atomic() - Increment the disable count of the ++ * context if possible in an atomic ++ * context. ++ * @hctx: Pointer to the hardware counter context. ++ * ++ * This function will only succeed if hardware counters are effectively already ++ * disabled, i.e. there is no accumulator, the disable count is already ++ * non-zero, or the accumulator has no counters set. ++ * ++ * After this function call returns true, it is guaranteed that counters will ++ * not be enabled via the backend interface. ++ * ++ * Return: True if the disable count was incremented, else False. ++ */ ++bool kbase_hwcnt_context_disable_atomic(struct kbase_hwcnt_context *hctx); ++ ++/** ++ * kbase_hwcnt_context_enable() - Decrement the disable count of the context. ++ * @hctx: Pointer to the hardware counter context. ++ * ++ * If a call to this function decrements the disable count from 1 to 0, and ++ * an accumulator has been acquired, then counters will be re-enabled via the ++ * backend interface. ++ * ++ * If an accumulator has been acquired and enabling counters fails for some ++ * reason, the accumulator will be placed into an error state. ++ * ++ * It is only valid to call this function one time for each prior returned call ++ * to kbase_hwcnt_context_disable. ++ * ++ * The spinlock documented in the backend interface that was passed in to ++ * kbase_hwcnt_context_init() must be held before calling this function. ++ */ ++void kbase_hwcnt_context_enable(struct kbase_hwcnt_context *hctx); ++ ++#endif /* _KBASE_HWCNT_CONTEXT_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_gpu.c b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_gpu.c +new file mode 100755 +index 000000000000..499f3bc23bec +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_gpu.c +@@ -0,0 +1,571 @@ ++/* ++ * ++ * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include "mali_kbase_hwcnt_gpu.h" ++#include "mali_kbase_hwcnt_types.h" ++#include "mali_kbase.h" ++#ifdef CONFIG_MALI_BIFROST_NO_MALI ++#include "backend/gpu/mali_kbase_model_dummy.h" ++#endif ++ ++#define KBASE_HWCNT_V5_BLOCK_TYPE_COUNT 4 ++#define KBASE_HWCNT_V5_HEADERS_PER_BLOCK 4 ++#define KBASE_HWCNT_V5_COUNTERS_PER_BLOCK 60 ++#define KBASE_HWCNT_V5_VALUES_PER_BLOCK \ ++ (KBASE_HWCNT_V5_HEADERS_PER_BLOCK + KBASE_HWCNT_V5_COUNTERS_PER_BLOCK) ++/* Index of the PRFCNT_EN header into a V5 counter block */ ++#define KBASE_HWCNT_V5_PRFCNT_EN_HEADER 2 ++ ++/** ++ * kbasep_hwcnt_backend_gpu_metadata_v5_create() - Create hardware counter ++ * metadata for a v5 GPU. ++ * @v5_info: Non-NULL pointer to hwcnt info for a v5 GPU. ++ * @use_secondary: True if secondary performance counters should be used, else ++ * false. Ignored if secondary counters are not supported. ++ * @metadata: Non-NULL pointer to where created metadata is stored ++ * on success. ++ * ++ * Return: 0 on success, else error code. ++ */ ++static int kbasep_hwcnt_backend_gpu_metadata_v5_create( ++ const struct kbase_hwcnt_gpu_v5_info *v5_info, ++ bool use_secondary, ++ const struct kbase_hwcnt_metadata **metadata) ++{ ++ struct kbase_hwcnt_description desc; ++ struct kbase_hwcnt_group_description group; ++ struct kbase_hwcnt_block_description ++ blks[KBASE_HWCNT_V5_BLOCK_TYPE_COUNT]; ++ size_t non_sc_block_count; ++ size_t sc_block_count; ++ ++ WARN_ON(!v5_info); ++ WARN_ON(!metadata); ++ ++ /* Calculate number of block instances that aren't shader cores */ ++ non_sc_block_count = 2 + v5_info->l2_count; ++ /* Calculate number of block instances that are shader cores */ ++ sc_block_count = fls64(v5_info->core_mask); ++ ++ /* ++ * A system can have up to 64 shader cores, but the 64-bit ++ * availability mask can't physically represent that many cores as well ++ * as the other hardware blocks. ++ * Error out if there are more blocks than our implementation can ++ * support. ++ */ ++ if ((sc_block_count + non_sc_block_count) > KBASE_HWCNT_AVAIL_MASK_BITS) ++ return -EINVAL; ++ ++ /* One Job Manager block */ ++ blks[0].type = KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_JM; ++ blks[0].inst_cnt = 1; ++ blks[0].hdr_cnt = KBASE_HWCNT_V5_HEADERS_PER_BLOCK; ++ blks[0].ctr_cnt = KBASE_HWCNT_V5_COUNTERS_PER_BLOCK; ++ ++ /* One Tiler block */ ++ blks[1].type = KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_TILER; ++ blks[1].inst_cnt = 1; ++ blks[1].hdr_cnt = KBASE_HWCNT_V5_HEADERS_PER_BLOCK; ++ blks[1].ctr_cnt = KBASE_HWCNT_V5_COUNTERS_PER_BLOCK; ++ ++ /* l2_count memsys blks */ ++ blks[2].type = use_secondary ? ++ KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_MEMSYS2 : ++ KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_MEMSYS; ++ blks[2].inst_cnt = v5_info->l2_count; ++ blks[2].hdr_cnt = KBASE_HWCNT_V5_HEADERS_PER_BLOCK; ++ blks[2].ctr_cnt = KBASE_HWCNT_V5_COUNTERS_PER_BLOCK; ++ ++ /* ++ * There are as many shader cores in the system as there are bits set in ++ * the core mask. However, the dump buffer memory requirements need to ++ * take into account the fact that the core mask may be non-contiguous. ++ * ++ * For example, a system with a core mask of 0b1011 has the same dump ++ * buffer memory requirements as a system with 0b1111, but requires more ++ * memory than a system with 0b0111. However, core 2 of the system with ++ * 0b1011 doesn't physically exist, and the dump buffer memory that ++ * accounts for that core will never be written to when we do a counter ++ * dump. ++ * ++ * We find the core mask's last set bit to determine the memory ++ * requirements, and embed the core mask into the availability mask so ++ * we can determine later which shader cores physically exist. ++ */ ++ blks[3].type = use_secondary ? ++ KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC2 : ++ KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC; ++ blks[3].inst_cnt = sc_block_count; ++ blks[3].hdr_cnt = KBASE_HWCNT_V5_HEADERS_PER_BLOCK; ++ blks[3].ctr_cnt = KBASE_HWCNT_V5_COUNTERS_PER_BLOCK; ++ ++ WARN_ON(KBASE_HWCNT_V5_BLOCK_TYPE_COUNT != 4); ++ ++ group.type = KBASE_HWCNT_GPU_GROUP_TYPE_V5; ++ group.blk_cnt = KBASE_HWCNT_V5_BLOCK_TYPE_COUNT; ++ group.blks = blks; ++ ++ desc.grp_cnt = 1; ++ desc.grps = &group; ++ desc.clk_cnt = v5_info->clk_cnt; ++ ++ /* The JM, Tiler, and L2s are always available, and are before cores */ ++ desc.avail_mask = (1ull << non_sc_block_count) - 1; ++ /* Embed the core mask directly in the availability mask */ ++ desc.avail_mask |= (v5_info->core_mask << non_sc_block_count); ++ ++ return kbase_hwcnt_metadata_create(&desc, metadata); ++} ++ ++/** ++ * kbasep_hwcnt_backend_gpu_v5_dump_bytes() - Get the raw dump buffer size for a ++ * V5 GPU. ++ * @v5_info: Non-NULL pointer to hwcnt info for a v5 GPU. ++ * ++ * Return: Size of buffer the V5 GPU needs to perform a counter dump. ++ */ ++static size_t kbasep_hwcnt_backend_gpu_v5_dump_bytes( ++ const struct kbase_hwcnt_gpu_v5_info *v5_info) ++{ ++ WARN_ON(!v5_info); ++ return (2 + v5_info->l2_count + fls64(v5_info->core_mask)) * ++ KBASE_HWCNT_V5_VALUES_PER_BLOCK * ++ KBASE_HWCNT_VALUE_BYTES; ++} ++ ++int kbase_hwcnt_gpu_info_init( ++ struct kbase_device *kbdev, ++ struct kbase_hwcnt_gpu_info *info) ++{ ++ size_t clk; ++ ++ if (!kbdev || !info) ++ return -EINVAL; ++ ++#ifdef CONFIG_MALI_BIFROST_NO_MALI ++ /* NO_MALI uses V5 layout, regardless of the underlying platform. */ ++ info->type = KBASE_HWCNT_GPU_GROUP_TYPE_V5; ++ info->v5.l2_count = KBASE_DUMMY_MODEL_MAX_MEMSYS_BLOCKS; ++ info->v5.core_mask = (1ull << KBASE_DUMMY_MODEL_MAX_SHADER_CORES) - 1; ++#else ++ { ++ const struct base_gpu_props *props = &kbdev->gpu_props.props; ++ const size_t l2_count = props->l2_props.num_l2_slices; ++ const size_t core_mask = ++ props->coherency_info.group[0].core_mask; ++ ++ info->type = KBASE_HWCNT_GPU_GROUP_TYPE_V5; ++ info->v5.l2_count = l2_count; ++ info->v5.core_mask = core_mask; ++ } ++#endif ++ ++ /* Determine the number of available clock domains. */ ++ for (clk = 0; clk < BASE_MAX_NR_CLOCKS_REGULATORS; clk++) { ++ if (kbdev->pm.clk_rtm.clks[clk] == NULL) ++ break; ++ } ++ info->v5.clk_cnt = clk; ++ ++ return 0; ++} ++ ++int kbase_hwcnt_gpu_metadata_create( ++ const struct kbase_hwcnt_gpu_info *info, ++ bool use_secondary, ++ const struct kbase_hwcnt_metadata **out_metadata, ++ size_t *out_dump_bytes) ++{ ++ int errcode; ++ const struct kbase_hwcnt_metadata *metadata; ++ size_t dump_bytes; ++ ++ if (!info || !out_metadata || !out_dump_bytes) ++ return -EINVAL; ++ ++ if (info->type == KBASE_HWCNT_GPU_GROUP_TYPE_V5) { ++ dump_bytes = kbasep_hwcnt_backend_gpu_v5_dump_bytes(&info->v5); ++ errcode = kbasep_hwcnt_backend_gpu_metadata_v5_create( ++ &info->v5, use_secondary, &metadata); ++ } else { ++ return -EINVAL; ++ } ++ if (errcode) ++ return errcode; ++ ++ /* ++ * Dump abstraction size should be exactly the same size and layout as ++ * the physical dump size, for backwards compatibility. ++ */ ++ WARN_ON(dump_bytes != metadata->dump_buf_bytes); ++ ++ *out_metadata = metadata; ++ *out_dump_bytes = dump_bytes; ++ ++ return 0; ++} ++KBASE_EXPORT_TEST_API(kbase_hwcnt_gpu_metadata_create); ++ ++void kbase_hwcnt_gpu_metadata_destroy( ++ const struct kbase_hwcnt_metadata *metadata) ++{ ++ if (!metadata) ++ return; ++ ++ kbase_hwcnt_metadata_destroy(metadata); ++} ++KBASE_EXPORT_TEST_API(kbase_hwcnt_gpu_metadata_destroy); ++ ++static bool is_block_type_shader( ++ const u64 grp_type, ++ const u64 blk_type, ++ const size_t blk) ++{ ++ bool is_shader = false; ++ ++ /* Warn on unknown group type */ ++ if (WARN_ON(grp_type != KBASE_HWCNT_GPU_GROUP_TYPE_V5)) ++ return false; ++ ++ if (blk_type == KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC || ++ blk_type == KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC2) ++ is_shader = true; ++ ++ return is_shader; ++} ++ ++int kbase_hwcnt_gpu_dump_get( ++ struct kbase_hwcnt_dump_buffer *dst, ++ void *src, ++ const struct kbase_hwcnt_enable_map *dst_enable_map, ++ u64 pm_core_mask, ++ bool accumulate) ++{ ++ const struct kbase_hwcnt_metadata *metadata; ++ const u32 *dump_src; ++ size_t src_offset, grp, blk, blk_inst; ++ u64 core_mask = pm_core_mask; ++ ++ if (!dst || !src || !dst_enable_map || ++ (dst_enable_map->metadata != dst->metadata)) ++ return -EINVAL; ++ ++ metadata = dst->metadata; ++ dump_src = (const u32 *)src; ++ src_offset = 0; ++ ++ kbase_hwcnt_metadata_for_each_block( ++ metadata, grp, blk, blk_inst) { ++ const size_t hdr_cnt = ++ kbase_hwcnt_metadata_block_headers_count( ++ metadata, grp, blk); ++ const size_t ctr_cnt = ++ kbase_hwcnt_metadata_block_counters_count( ++ metadata, grp, blk); ++ const u64 blk_type = kbase_hwcnt_metadata_block_type( ++ metadata, grp, blk); ++ const bool is_shader_core = is_block_type_shader( ++ kbase_hwcnt_metadata_group_type(metadata, grp), ++ blk_type, blk); ++ ++ /* Early out if no values in the dest block are enabled */ ++ if (kbase_hwcnt_enable_map_block_enabled( ++ dst_enable_map, grp, blk, blk_inst)) { ++ u32 *dst_blk = kbase_hwcnt_dump_buffer_block_instance( ++ dst, grp, blk, blk_inst); ++ const u32 *src_blk = dump_src + src_offset; ++ ++ if (!is_shader_core || (core_mask & 1)) { ++ if (accumulate) { ++ kbase_hwcnt_dump_buffer_block_accumulate( ++ dst_blk, src_blk, hdr_cnt, ++ ctr_cnt); ++ } else { ++ kbase_hwcnt_dump_buffer_block_copy( ++ dst_blk, src_blk, ++ (hdr_cnt + ctr_cnt)); ++ } ++ } else if (!accumulate) { ++ kbase_hwcnt_dump_buffer_block_zero( ++ dst_blk, (hdr_cnt + ctr_cnt)); ++ } ++ } ++ ++ src_offset += (hdr_cnt + ctr_cnt); ++ if (is_shader_core) ++ core_mask = core_mask >> 1; ++ } ++ ++ return 0; ++} ++KBASE_EXPORT_TEST_API(kbase_hwcnt_gpu_dump_get); ++ ++/** ++ * kbasep_hwcnt_backend_gpu_block_map_to_physical() - Convert from a block ++ * enable map abstraction to ++ * a physical block enable ++ * map. ++ * @lo: Low 64 bits of block enable map abstraction. ++ * @hi: High 64 bits of block enable map abstraction. ++ * ++ * The abstraction uses 128 bits to enable 128 block values, whereas the ++ * physical uses just 32 bits, as bit n enables values [n*4, n*4+3]. ++ * Therefore, this conversion is lossy. ++ * ++ * Return: 32-bit physical block enable map. ++ */ ++static inline u32 kbasep_hwcnt_backend_gpu_block_map_to_physical( ++ u64 lo, ++ u64 hi) ++{ ++ u32 phys = 0; ++ u64 dwords[2] = {lo, hi}; ++ size_t dword_idx; ++ ++ for (dword_idx = 0; dword_idx < 2; dword_idx++) { ++ const u64 dword = dwords[dword_idx]; ++ u16 packed = 0; ++ ++ size_t hword_bit; ++ ++ for (hword_bit = 0; hword_bit < 16; hword_bit++) { ++ const size_t dword_bit = hword_bit * 4; ++ const u16 mask = ++ ((dword >> (dword_bit + 0)) & 0x1) | ++ ((dword >> (dword_bit + 1)) & 0x1) | ++ ((dword >> (dword_bit + 2)) & 0x1) | ++ ((dword >> (dword_bit + 3)) & 0x1); ++ packed |= (mask << hword_bit); ++ } ++ phys |= ((u32)packed) << (16 * dword_idx); ++ } ++ return phys; ++} ++ ++/** ++ * kbasep_hwcnt_backend_gpu_block_map_from_physical() - Convert from a physical ++ * block enable map to a ++ * block enable map ++ * abstraction. ++ * @phys: Physical 32-bit block enable map ++ * @lo: Non-NULL pointer to where low 64 bits of block enable map abstraction ++ * will be stored. ++ * @hi: Non-NULL pointer to where high 64 bits of block enable map abstraction ++ * will be stored. ++ */ ++static inline void kbasep_hwcnt_backend_gpu_block_map_from_physical( ++ u32 phys, ++ u64 *lo, ++ u64 *hi) ++{ ++ u64 dwords[2] = {0, 0}; ++ ++ size_t dword_idx; ++ ++ for (dword_idx = 0; dword_idx < 2; dword_idx++) { ++ const u16 packed = phys >> (16 * dword_idx); ++ u64 dword = 0; ++ ++ size_t hword_bit; ++ ++ for (hword_bit = 0; hword_bit < 16; hword_bit++) { ++ const size_t dword_bit = hword_bit * 4; ++ const u64 mask = (packed >> (hword_bit)) & 0x1; ++ ++ dword |= mask << (dword_bit + 0); ++ dword |= mask << (dword_bit + 1); ++ dword |= mask << (dword_bit + 2); ++ dword |= mask << (dword_bit + 3); ++ } ++ dwords[dword_idx] = dword; ++ } ++ *lo = dwords[0]; ++ *hi = dwords[1]; ++} ++ ++void kbase_hwcnt_gpu_enable_map_to_physical( ++ struct kbase_hwcnt_physical_enable_map *dst, ++ const struct kbase_hwcnt_enable_map *src) ++{ ++ const struct kbase_hwcnt_metadata *metadata; ++ ++ u64 fe_bm = 0; ++ u64 shader_bm = 0; ++ u64 tiler_bm = 0; ++ u64 mmu_l2_bm = 0; ++ ++ size_t grp, blk, blk_inst; ++ ++ if (WARN_ON(!src) || WARN_ON(!dst)) ++ return; ++ ++ metadata = src->metadata; ++ ++ kbase_hwcnt_metadata_for_each_block( ++ metadata, grp, blk, blk_inst) { ++ const u64 grp_type = kbase_hwcnt_metadata_group_type( ++ metadata, grp); ++ const u64 blk_type = kbase_hwcnt_metadata_block_type( ++ metadata, grp, blk); ++ const size_t blk_val_cnt = ++ kbase_hwcnt_metadata_block_values_count( ++ metadata, grp, blk); ++ const u64 *blk_map = kbase_hwcnt_enable_map_block_instance( ++ src, grp, blk, blk_inst); ++ ++ if ((enum kbase_hwcnt_gpu_group_type)grp_type == ++ KBASE_HWCNT_GPU_GROUP_TYPE_V5) { ++ WARN_ON(blk_val_cnt != KBASE_HWCNT_V5_VALUES_PER_BLOCK); ++ switch ((enum kbase_hwcnt_gpu_v5_block_type)blk_type) { ++ case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_JM: ++ fe_bm |= *blk_map; ++ break; ++ case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_TILER: ++ tiler_bm |= *blk_map; ++ break; ++ case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC: ++ case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC2: ++ shader_bm |= *blk_map; ++ break; ++ case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_MEMSYS: ++ case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_MEMSYS2: ++ mmu_l2_bm |= *blk_map; ++ break; ++ default: ++ WARN_ON(true); ++ } ++ } else { ++ WARN_ON(true); ++ } ++ } ++ ++ dst->fe_bm = ++ kbasep_hwcnt_backend_gpu_block_map_to_physical(fe_bm, 0); ++ dst->shader_bm = ++ kbasep_hwcnt_backend_gpu_block_map_to_physical(shader_bm, 0); ++ dst->tiler_bm = ++ kbasep_hwcnt_backend_gpu_block_map_to_physical(tiler_bm, 0); ++ dst->mmu_l2_bm = ++ kbasep_hwcnt_backend_gpu_block_map_to_physical(mmu_l2_bm, 0); ++} ++KBASE_EXPORT_TEST_API(kbase_hwcnt_gpu_enable_map_to_physical); ++ ++void kbase_hwcnt_gpu_enable_map_from_physical( ++ struct kbase_hwcnt_enable_map *dst, ++ const struct kbase_hwcnt_physical_enable_map *src) ++{ ++ const struct kbase_hwcnt_metadata *metadata; ++ ++ u64 ignored_hi; ++ u64 fe_bm; ++ u64 shader_bm; ++ u64 tiler_bm; ++ u64 mmu_l2_bm; ++ size_t grp, blk, blk_inst; ++ ++ if (WARN_ON(!src) || WARN_ON(!dst)) ++ return; ++ ++ metadata = dst->metadata; ++ ++ kbasep_hwcnt_backend_gpu_block_map_from_physical( ++ src->fe_bm, &fe_bm, &ignored_hi); ++ kbasep_hwcnt_backend_gpu_block_map_from_physical( ++ src->shader_bm, &shader_bm, &ignored_hi); ++ kbasep_hwcnt_backend_gpu_block_map_from_physical( ++ src->tiler_bm, &tiler_bm, &ignored_hi); ++ kbasep_hwcnt_backend_gpu_block_map_from_physical( ++ src->mmu_l2_bm, &mmu_l2_bm, &ignored_hi); ++ ++ kbase_hwcnt_metadata_for_each_block(metadata, grp, blk, blk_inst) { ++ const u64 grp_type = kbase_hwcnt_metadata_group_type( ++ metadata, grp); ++ const u64 blk_type = kbase_hwcnt_metadata_block_type( ++ metadata, grp, blk); ++ const size_t blk_val_cnt = ++ kbase_hwcnt_metadata_block_values_count( ++ metadata, grp, blk); ++ u64 *blk_map = kbase_hwcnt_enable_map_block_instance( ++ dst, grp, blk, blk_inst); ++ ++ if ((enum kbase_hwcnt_gpu_group_type)grp_type == ++ KBASE_HWCNT_GPU_GROUP_TYPE_V5) { ++ WARN_ON(blk_val_cnt != KBASE_HWCNT_V5_VALUES_PER_BLOCK); ++ switch ((enum kbase_hwcnt_gpu_v5_block_type)blk_type) { ++ case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_JM: ++ *blk_map = fe_bm; ++ break; ++ case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_TILER: ++ *blk_map = tiler_bm; ++ break; ++ case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC: ++ case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC2: ++ *blk_map = shader_bm; ++ break; ++ case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_MEMSYS: ++ case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_MEMSYS2: ++ *blk_map = mmu_l2_bm; ++ break; ++ default: ++ WARN_ON(true); ++ } ++ } else { ++ WARN_ON(true); ++ } ++ } ++} ++KBASE_EXPORT_TEST_API(kbase_hwcnt_gpu_enable_map_from_physical); ++ ++void kbase_hwcnt_gpu_patch_dump_headers( ++ struct kbase_hwcnt_dump_buffer *buf, ++ const struct kbase_hwcnt_enable_map *enable_map) ++{ ++ const struct kbase_hwcnt_metadata *metadata; ++ size_t grp, blk, blk_inst; ++ ++ if (WARN_ON(!buf) || WARN_ON(!enable_map) || ++ WARN_ON(buf->metadata != enable_map->metadata)) ++ return; ++ ++ metadata = buf->metadata; ++ ++ kbase_hwcnt_metadata_for_each_block(metadata, grp, blk, blk_inst) { ++ const u64 grp_type = ++ kbase_hwcnt_metadata_group_type(metadata, grp); ++ u32 *buf_blk = kbase_hwcnt_dump_buffer_block_instance( ++ buf, grp, blk, blk_inst); ++ const u64 *blk_map = kbase_hwcnt_enable_map_block_instance( ++ enable_map, grp, blk, blk_inst); ++ const u32 prfcnt_en = ++ kbasep_hwcnt_backend_gpu_block_map_to_physical( ++ blk_map[0], 0); ++ ++ if ((enum kbase_hwcnt_gpu_group_type)grp_type == ++ KBASE_HWCNT_GPU_GROUP_TYPE_V5) { ++ buf_blk[KBASE_HWCNT_V5_PRFCNT_EN_HEADER] = prfcnt_en; ++ } else { ++ WARN_ON(true); ++ } ++ } ++} ++KBASE_EXPORT_TEST_API(kbase_hwcnt_gpu_patch_dump_headers); +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_gpu.h b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_gpu.h +new file mode 100755 +index 000000000000..f0d51763f7f7 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_gpu.h +@@ -0,0 +1,217 @@ ++/* ++ * ++ * (C) COPYRIGHT 2018, 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_HWCNT_GPU_H_ ++#define _KBASE_HWCNT_GPU_H_ ++ ++#include ++ ++struct kbase_device; ++struct kbase_hwcnt_metadata; ++struct kbase_hwcnt_enable_map; ++struct kbase_hwcnt_dump_buffer; ++ ++/** ++ * enum kbase_hwcnt_gpu_group_type - GPU hardware counter group types, used to ++ * identify metadata groups. ++ * @KBASE_HWCNT_GPU_GROUP_TYPE_V5: GPU V5 group type. ++ */ ++enum kbase_hwcnt_gpu_group_type { ++ KBASE_HWCNT_GPU_GROUP_TYPE_V5 = 0x10, ++}; ++ ++/** ++ * enum kbase_hwcnt_gpu_v5_block_type - GPU V5 hardware counter block types, ++ * used to identify metadata blocks. ++ * @KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_JM: Job Manager block. ++ * @KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_TILER: Tiler block. ++ * @KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC: Shader Core block. ++ * @KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC2: Secondary Shader Core block. ++ * @KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_MEMSYS: Memsys block. ++ * @KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_MEMSYS2: Secondary Memsys block. ++ */ ++enum kbase_hwcnt_gpu_v5_block_type { ++ KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_JM = 0x40, ++ KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_TILER, ++ KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC, ++ KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC2, ++ KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_MEMSYS, ++ KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_MEMSYS2, ++}; ++ ++/** ++ * struct kbase_hwcnt_physical_enable_map - Representation of enable map ++ * directly used by GPU. ++ * @fe_bm: Front end (JM/CSHW) counters selection bitmask. ++ * @shader_bm: Shader counters selection bitmask. ++ * @tiler_bm: Tiler counters selection bitmask. ++ * @mmu_l2_bm: MMU_L2 counters selection bitmask. ++ */ ++struct kbase_hwcnt_physical_enable_map { ++ u32 fe_bm; ++ u32 shader_bm; ++ u32 tiler_bm; ++ u32 mmu_l2_bm; ++}; ++ ++/** ++ * struct kbase_hwcnt_gpu_v5_info - Information about hwcnt blocks on v5 GPUs. ++ * @l2_count: L2 cache count. ++ * @core_mask: Shader core mask. May be sparse. ++ * @clk_cnt: Number of clock domains available. ++ */ ++struct kbase_hwcnt_gpu_v5_info { ++ size_t l2_count; ++ u64 core_mask; ++ u8 clk_cnt; ++}; ++ ++/** ++ * struct kbase_hwcnt_gpu_info - Tagged union with information about the current ++ * GPU's hwcnt blocks. ++ * @type: GPU type. ++ * @v5: Info filled in if a v5 GPU. ++ */ ++struct kbase_hwcnt_gpu_info { ++ enum kbase_hwcnt_gpu_group_type type; ++ struct kbase_hwcnt_gpu_v5_info v5; ++}; ++ ++/** ++ * kbase_hwcnt_gpu_info_init() - Initialise an info structure used to create the ++ * hwcnt metadata. ++ * @kbdev: Non-NULL pointer to kbase device. ++ * @info: Non-NULL pointer to data structure to be filled in. ++ * ++ * The initialised info struct will only be valid for use while kbdev is valid. ++ */ ++int kbase_hwcnt_gpu_info_init( ++ struct kbase_device *kbdev, ++ struct kbase_hwcnt_gpu_info *info); ++ ++/** ++ * kbase_hwcnt_gpu_metadata_create() - Create hardware counter metadata for the ++ * current GPU. ++ * @info: Non-NULL pointer to info struct initialised by ++ * kbase_hwcnt_gpu_info_init. ++ * @use_secondary: True if secondary performance counters should be used, else ++ * false. Ignored if secondary counters are not supported. ++ * @out_metadata: Non-NULL pointer to where created metadata is stored on ++ * success. ++ * @out_dump_bytes: Non-NULL pointer to where the size of the GPU counter dump ++ * buffer is stored on success. ++ * ++ * Return: 0 on success, else error code. ++ */ ++int kbase_hwcnt_gpu_metadata_create( ++ const struct kbase_hwcnt_gpu_info *info, ++ bool use_secondary, ++ const struct kbase_hwcnt_metadata **out_metadata, ++ size_t *out_dump_bytes); ++ ++/** ++ * kbase_hwcnt_gpu_metadata_destroy() - Destroy GPU hardware counter metadata. ++ * @metadata: Pointer to metadata to destroy. ++ */ ++void kbase_hwcnt_gpu_metadata_destroy( ++ const struct kbase_hwcnt_metadata *metadata); ++ ++/** ++ * kbase_hwcnt_gpu_dump_get() - Copy or accumulate enabled counters from the raw ++ * dump buffer in src into the dump buffer ++ * abstraction in dst. ++ * @dst: Non-NULL pointer to dst dump buffer. ++ * @src: Non-NULL pointer to src raw dump buffer, of same length ++ * as returned in out_dump_bytes parameter of ++ * kbase_hwcnt_gpu_metadata_create. ++ * @dst_enable_map: Non-NULL pointer to enable map specifying enabled values. ++ * @pm_core_mask: PM state synchronized shaders core mask with the dump. ++ * @accumulate: True if counters in src should be accumulated into dst, ++ * rather than copied. ++ * ++ * The dst and dst_enable_map MUST have been created from the same metadata as ++ * returned from the call to kbase_hwcnt_gpu_metadata_create as was used to get ++ * the length of src. ++ * ++ * Return: 0 on success, else error code. ++ */ ++int kbase_hwcnt_gpu_dump_get( ++ struct kbase_hwcnt_dump_buffer *dst, ++ void *src, ++ const struct kbase_hwcnt_enable_map *dst_enable_map, ++ const u64 pm_core_mask, ++ bool accumulate); ++ ++/** ++ * kbase_hwcnt_gpu_enable_map_to_physical() - Convert an enable map abstraction ++ * into a physical enable map. ++ * @dst: Non-NULL pointer to dst physical enable map. ++ * @src: Non-NULL pointer to src enable map abstraction. ++ * ++ * The src must have been created from a metadata returned from a call to ++ * kbase_hwcnt_gpu_metadata_create. ++ * ++ * This is a lossy conversion, as the enable map abstraction has one bit per ++ * individual counter block value, but the physical enable map uses 1 bit for ++ * every 4 counters, shared over all instances of a block. ++ */ ++void kbase_hwcnt_gpu_enable_map_to_physical( ++ struct kbase_hwcnt_physical_enable_map *dst, ++ const struct kbase_hwcnt_enable_map *src); ++ ++/** ++ * kbase_hwcnt_gpu_enable_map_from_physical() - Convert a physical enable map to ++ * an enable map abstraction. ++ * @dst: Non-NULL pointer to dst enable map abstraction. ++ * @src: Non-NULL pointer to src physical enable map. ++ * ++ * The dst must have been created from a metadata returned from a call to ++ * kbase_hwcnt_gpu_metadata_create. ++ * ++ * This is a lossy conversion, as the physical enable map can technically ++ * support counter blocks with 128 counters each, but no hardware actually uses ++ * more than 64, so the enable map abstraction has nowhere to store the enable ++ * information for the 64 non-existent counters. ++ */ ++void kbase_hwcnt_gpu_enable_map_from_physical( ++ struct kbase_hwcnt_enable_map *dst, ++ const struct kbase_hwcnt_physical_enable_map *src); ++ ++/** ++ * kbase_hwcnt_gpu_patch_dump_headers() - Patch all the performance counter ++ * enable headers in a dump buffer to ++ * reflect the specified enable map. ++ * @buf: Non-NULL pointer to dump buffer to patch. ++ * @enable_map: Non-NULL pointer to enable map. ++ * ++ * The buf and enable_map must have been created from a metadata returned from ++ * a call to kbase_hwcnt_gpu_metadata_create. ++ * ++ * This function should be used before handing off a dump buffer over the ++ * kernel-user boundary, to ensure the header is accurate for the enable map ++ * used by the user. ++ */ ++void kbase_hwcnt_gpu_patch_dump_headers( ++ struct kbase_hwcnt_dump_buffer *buf, ++ const struct kbase_hwcnt_enable_map *enable_map); ++ ++#endif /* _KBASE_HWCNT_GPU_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_legacy.c b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_legacy.c +new file mode 100755 +index 000000000000..794ef39e365c +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_legacy.c +@@ -0,0 +1,152 @@ ++/* ++ * ++ * (C) COPYRIGHT 2018, 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include "mali_kbase_hwcnt_legacy.h" ++#include "mali_kbase_hwcnt_virtualizer.h" ++#include "mali_kbase_hwcnt_types.h" ++#include "mali_kbase_hwcnt_gpu.h" ++#include "mali_kbase_ioctl.h" ++ ++#include ++#include ++ ++/** ++ * struct kbase_hwcnt_legacy_client - Legacy hardware counter client. ++ * @user_dump_buf: Pointer to a non-NULL user buffer, where dumps are returned. ++ * @enable_map: Counter enable map. ++ * @dump_buf: Dump buffer used to manipulate dumps before copied to user. ++ * @hvcli: Hardware counter virtualizer client. ++ */ ++struct kbase_hwcnt_legacy_client { ++ void __user *user_dump_buf; ++ struct kbase_hwcnt_enable_map enable_map; ++ struct kbase_hwcnt_dump_buffer dump_buf; ++ struct kbase_hwcnt_virtualizer_client *hvcli; ++}; ++ ++int kbase_hwcnt_legacy_client_create( ++ struct kbase_hwcnt_virtualizer *hvirt, ++ struct kbase_ioctl_hwcnt_enable *enable, ++ struct kbase_hwcnt_legacy_client **out_hlcli) ++{ ++ int errcode; ++ struct kbase_hwcnt_legacy_client *hlcli; ++ const struct kbase_hwcnt_metadata *metadata; ++ struct kbase_hwcnt_physical_enable_map phys_em; ++ ++ if (!hvirt || !enable || !enable->dump_buffer || !out_hlcli) ++ return -EINVAL; ++ ++ metadata = kbase_hwcnt_virtualizer_metadata(hvirt); ++ ++ hlcli = kzalloc(sizeof(*hlcli), GFP_KERNEL); ++ if (!hlcli) ++ return -ENOMEM; ++ ++ hlcli->user_dump_buf = (void __user *)(uintptr_t)enable->dump_buffer; ++ ++ errcode = kbase_hwcnt_enable_map_alloc(metadata, &hlcli->enable_map); ++ if (errcode) ++ goto error; ++ ++ /* Translate from the ioctl enable map to the internal one */ ++ phys_em.fe_bm = enable->fe_bm; ++ phys_em.shader_bm = enable->shader_bm; ++ phys_em.tiler_bm = enable->tiler_bm; ++ phys_em.mmu_l2_bm = enable->mmu_l2_bm; ++ kbase_hwcnt_gpu_enable_map_from_physical(&hlcli->enable_map, &phys_em); ++ ++ errcode = kbase_hwcnt_dump_buffer_alloc(metadata, &hlcli->dump_buf); ++ if (errcode) ++ goto error; ++ ++ errcode = kbase_hwcnt_virtualizer_client_create( ++ hvirt, &hlcli->enable_map, &hlcli->hvcli); ++ if (errcode) ++ goto error; ++ ++ *out_hlcli = hlcli; ++ return 0; ++ ++error: ++ kbase_hwcnt_legacy_client_destroy(hlcli); ++ return errcode; ++} ++ ++void kbase_hwcnt_legacy_client_destroy(struct kbase_hwcnt_legacy_client *hlcli) ++{ ++ if (!hlcli) ++ return; ++ ++ kbase_hwcnt_virtualizer_client_destroy(hlcli->hvcli); ++ kbase_hwcnt_dump_buffer_free(&hlcli->dump_buf); ++ kbase_hwcnt_enable_map_free(&hlcli->enable_map); ++ kfree(hlcli); ++} ++ ++int kbase_hwcnt_legacy_client_dump(struct kbase_hwcnt_legacy_client *hlcli) ++{ ++ int errcode; ++ u64 ts_start_ns; ++ u64 ts_end_ns; ++ ++ if (!hlcli) ++ return -EINVAL; ++ ++ /* Dump into the kernel buffer */ ++ errcode = kbase_hwcnt_virtualizer_client_dump(hlcli->hvcli, ++ &ts_start_ns, &ts_end_ns, &hlcli->dump_buf); ++ if (errcode) ++ return errcode; ++ ++ /* Patch the dump buf headers, to hide the counters that other hwcnt ++ * clients are using. ++ */ ++ kbase_hwcnt_gpu_patch_dump_headers( ++ &hlcli->dump_buf, &hlcli->enable_map); ++ ++ /* Zero all non-enabled counters (current values are undefined) */ ++ kbase_hwcnt_dump_buffer_zero_non_enabled( ++ &hlcli->dump_buf, &hlcli->enable_map); ++ ++ /* Copy into the user's buffer */ ++ errcode = copy_to_user(hlcli->user_dump_buf, hlcli->dump_buf.dump_buf, ++ hlcli->dump_buf.metadata->dump_buf_bytes); ++ /* Non-zero errcode implies user buf was invalid or too small */ ++ if (errcode) ++ return -EFAULT; ++ ++ return 0; ++} ++ ++int kbase_hwcnt_legacy_client_clear(struct kbase_hwcnt_legacy_client *hlcli) ++{ ++ u64 ts_start_ns; ++ u64 ts_end_ns; ++ ++ if (!hlcli) ++ return -EINVAL; ++ ++ /* Dump with a NULL buffer to clear this client's counters */ ++ return kbase_hwcnt_virtualizer_client_dump(hlcli->hvcli, ++ &ts_start_ns, &ts_end_ns, NULL); ++} +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_legacy.h b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_legacy.h +new file mode 100755 +index 000000000000..7a610ae378a2 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_legacy.h +@@ -0,0 +1,94 @@ ++/* ++ * ++ * (C) COPYRIGHT 2018 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/** ++ * Legacy hardware counter interface, giving userspace clients simple, ++ * synchronous access to hardware counters. ++ * ++ * Any functions operating on an single legacy hardware counter client instance ++ * must be externally synchronised. ++ * Different clients may safely be used concurrently. ++ */ ++ ++#ifndef _KBASE_HWCNT_LEGACY_H_ ++#define _KBASE_HWCNT_LEGACY_H_ ++ ++struct kbase_hwcnt_legacy_client; ++struct kbase_ioctl_hwcnt_enable; ++struct kbase_hwcnt_virtualizer; ++ ++/** ++ * kbase_hwcnt_legacy_client_create() - Create a legacy hardware counter client. ++ * @hvirt: Non-NULL pointer to hardware counter virtualizer the client ++ * should be attached to. ++ * @enable: Non-NULL pointer to hwcnt_enable structure, containing a valid ++ * pointer to a user dump buffer large enough to hold a dump, and ++ * the counters that should be enabled. ++ * @out_hlcli: Non-NULL pointer to where the pointer to the created client will ++ * be stored on success. ++ * ++ * Return: 0 on success, else error code. ++ */ ++int kbase_hwcnt_legacy_client_create( ++ struct kbase_hwcnt_virtualizer *hvirt, ++ struct kbase_ioctl_hwcnt_enable *enable, ++ struct kbase_hwcnt_legacy_client **out_hlcli); ++ ++/** ++ * kbase_hwcnt_legacy_client_destroy() - Destroy a legacy hardware counter ++ * client. ++ * @hlcli: Pointer to the legacy hardware counter client. ++ * ++ * Will safely destroy a client in any partial state of construction. ++ */ ++void kbase_hwcnt_legacy_client_destroy(struct kbase_hwcnt_legacy_client *hlcli); ++ ++/** ++ * kbase_hwcnt_legacy_client_dump() - Perform a hardware counter dump into the ++ * client's user buffer. ++ * @hlcli: Non-NULL pointer to the legacy hardware counter client. ++ * ++ * This function will synchronously dump hardware counters into the user buffer ++ * specified on client creation, with the counters specified on client creation. ++ * ++ * The counters are automatically cleared after each dump, such that the next ++ * dump performed will return the counter values accumulated between the time of ++ * this function call and the next dump. ++ * ++ * Return: 0 on success, else error code. ++ */ ++int kbase_hwcnt_legacy_client_dump(struct kbase_hwcnt_legacy_client *hlcli); ++ ++/** ++ * kbase_hwcnt_legacy_client_clear() - Perform and discard a hardware counter ++ * dump. ++ * @hlcli: Non-NULL pointer to the legacy hardware counter client. ++ * ++ * This function will synchronously clear the hardware counters, such that the ++ * next dump performed will return the counter values accumulated between the ++ * time of this function call and the next dump. ++ * ++ * Return: 0 on success, else error code. ++ */ ++int kbase_hwcnt_legacy_client_clear(struct kbase_hwcnt_legacy_client *hlcli); ++ ++#endif /* _KBASE_HWCNT_LEGACY_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_reader.h b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_reader.h +new file mode 100755 +index 000000000000..8cd3835595f7 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_reader.h +@@ -0,0 +1,106 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015, 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_HWCNT_READER_H_ ++#define _KBASE_HWCNT_READER_H_ ++ ++#include ++ ++/* The ids of ioctl commands. */ ++#define KBASE_HWCNT_READER 0xBE ++#define KBASE_HWCNT_READER_GET_HWVER _IOR(KBASE_HWCNT_READER, 0x00, u32) ++#define KBASE_HWCNT_READER_GET_BUFFER_SIZE _IOR(KBASE_HWCNT_READER, 0x01, u32) ++#define KBASE_HWCNT_READER_DUMP _IOW(KBASE_HWCNT_READER, 0x10, u32) ++#define KBASE_HWCNT_READER_CLEAR _IOW(KBASE_HWCNT_READER, 0x11, u32) ++#define KBASE_HWCNT_READER_GET_BUFFER _IOC(_IOC_READ, KBASE_HWCNT_READER, 0x20,\ ++ offsetof(struct kbase_hwcnt_reader_metadata, cycles)) ++#define KBASE_HWCNT_READER_GET_BUFFER_WITH_CYCLES _IOR(KBASE_HWCNT_READER, 0x20,\ ++ struct kbase_hwcnt_reader_metadata) ++#define KBASE_HWCNT_READER_PUT_BUFFER _IOC(_IOC_WRITE, KBASE_HWCNT_READER, 0x21,\ ++ offsetof(struct kbase_hwcnt_reader_metadata, cycles)) ++#define KBASE_HWCNT_READER_PUT_BUFFER_WITH_CYCLES _IOW(KBASE_HWCNT_READER, 0x21,\ ++ struct kbase_hwcnt_reader_metadata) ++#define KBASE_HWCNT_READER_SET_INTERVAL _IOW(KBASE_HWCNT_READER, 0x30, u32) ++#define KBASE_HWCNT_READER_ENABLE_EVENT _IOW(KBASE_HWCNT_READER, 0x40, u32) ++#define KBASE_HWCNT_READER_DISABLE_EVENT _IOW(KBASE_HWCNT_READER, 0x41, u32) ++#define KBASE_HWCNT_READER_GET_API_VERSION _IOW(KBASE_HWCNT_READER, 0xFF, u32) ++#define KBASE_HWCNT_READER_GET_API_VERSION_WITH_FEATURES \ ++ _IOW(KBASE_HWCNT_READER, 0xFF, \ ++ struct kbase_hwcnt_reader_api_version) ++ ++/** ++ * struct kbase_hwcnt_reader_metadata_cycles - GPU clock cycles ++ * @top: the number of cycles associated with the main clock for the ++ * GPU ++ * @shader_cores: the cycles that have elapsed on the GPU shader cores ++ */ ++struct kbase_hwcnt_reader_metadata_cycles { ++ u64 top; ++ u64 shader_cores; ++}; ++ ++/** ++ * struct kbase_hwcnt_reader_metadata - hwcnt reader sample buffer metadata ++ * @timestamp: time when sample was collected ++ * @event_id: id of an event that triggered sample collection ++ * @buffer_idx: position in sampling area where sample buffer was stored ++ * @cycles: the GPU cycles that occurred since the last sample ++ */ ++struct kbase_hwcnt_reader_metadata { ++ u64 timestamp; ++ u32 event_id; ++ u32 buffer_idx; ++ struct kbase_hwcnt_reader_metadata_cycles cycles; ++}; ++ ++/** ++ * enum base_hwcnt_reader_event - hwcnt dumping events ++ * @BASE_HWCNT_READER_EVENT_MANUAL: manual request for dump ++ * @BASE_HWCNT_READER_EVENT_PERIODIC: periodic dump ++ * @BASE_HWCNT_READER_EVENT_PREJOB: prejob dump request ++ * @BASE_HWCNT_READER_EVENT_POSTJOB: postjob dump request ++ * @BASE_HWCNT_READER_EVENT_COUNT: number of supported events ++ */ ++enum base_hwcnt_reader_event { ++ BASE_HWCNT_READER_EVENT_MANUAL, ++ BASE_HWCNT_READER_EVENT_PERIODIC, ++ BASE_HWCNT_READER_EVENT_PREJOB, ++ BASE_HWCNT_READER_EVENT_POSTJOB, ++ ++ BASE_HWCNT_READER_EVENT_COUNT ++}; ++ ++/** ++ * struct kbase_hwcnt_reader_api_version - hwcnt reader API version ++ * @versoin: API version ++ * @features: available features in this API version ++ */ ++#define KBASE_HWCNT_READER_API_VERSION_NO_FEATURE (0) ++#define KBASE_HWCNT_READER_API_VERSION_FEATURE_CYCLES_TOP (1 << 0) ++#define KBASE_HWCNT_READER_API_VERSION_FEATURE_CYCLES_SHADER_CORES (1 << 1) ++struct kbase_hwcnt_reader_api_version { ++ u32 version; ++ u32 features; ++}; ++ ++#endif /* _KBASE_HWCNT_READER_H_ */ ++ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_types.c b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_types.c +new file mode 100755 +index 000000000000..2b9fe02acd75 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_types.c +@@ -0,0 +1,604 @@ ++/* ++ * ++ * (C) COPYRIGHT 2018, 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include "mali_kbase_hwcnt_types.h" ++#include "mali_kbase.h" ++ ++/* Minimum alignment of each block of hardware counters */ ++#define KBASE_HWCNT_BLOCK_BYTE_ALIGNMENT \ ++ (KBASE_HWCNT_BITFIELD_BITS * KBASE_HWCNT_VALUE_BYTES) ++ ++/** ++ * KBASE_HWCNT_ALIGN_UPWARDS() - Align a value to an alignment. ++ * @value: The value to align upwards. ++ * @alignment: The alignment. ++ * ++ * Return: A number greater than or equal to value that is aligned to alignment. ++ */ ++#define KBASE_HWCNT_ALIGN_UPWARDS(value, alignment) \ ++ (value + ((alignment - (value % alignment)) % alignment)) ++ ++int kbase_hwcnt_metadata_create( ++ const struct kbase_hwcnt_description *desc, ++ const struct kbase_hwcnt_metadata **out_metadata) ++{ ++ char *buf; ++ struct kbase_hwcnt_metadata *metadata; ++ struct kbase_hwcnt_group_metadata *grp_mds; ++ size_t grp; ++ size_t enable_map_count; /* Number of u64 bitfields (inc padding) */ ++ size_t dump_buf_count; /* Number of u32 values (inc padding) */ ++ size_t avail_mask_bits; /* Number of availability mask bits */ ++ ++ size_t size; ++ size_t offset; ++ ++ if (!desc || !out_metadata) ++ return -EINVAL; ++ ++ /* The maximum number of clock domains is 64. */ ++ if (desc->clk_cnt > (sizeof(u64) * BITS_PER_BYTE)) ++ return -EINVAL; ++ ++ /* Calculate the bytes needed to tightly pack the metadata */ ++ ++ /* Top level metadata */ ++ size = 0; ++ size += sizeof(struct kbase_hwcnt_metadata); ++ ++ /* Group metadata */ ++ size += sizeof(struct kbase_hwcnt_group_metadata) * desc->grp_cnt; ++ ++ /* Block metadata */ ++ for (grp = 0; grp < desc->grp_cnt; grp++) { ++ size += sizeof(struct kbase_hwcnt_block_metadata) * ++ desc->grps[grp].blk_cnt; ++ } ++ ++ /* Single allocation for the entire metadata */ ++ buf = kmalloc(size, GFP_KERNEL); ++ if (!buf) ++ return -ENOMEM; ++ ++ /* Use the allocated memory for the metadata and its members */ ++ ++ /* Bump allocate the top level metadata */ ++ offset = 0; ++ metadata = (struct kbase_hwcnt_metadata *)(buf + offset); ++ offset += sizeof(struct kbase_hwcnt_metadata); ++ ++ /* Bump allocate the group metadata */ ++ grp_mds = (struct kbase_hwcnt_group_metadata *)(buf + offset); ++ offset += sizeof(struct kbase_hwcnt_group_metadata) * desc->grp_cnt; ++ ++ enable_map_count = 0; ++ dump_buf_count = 0; ++ avail_mask_bits = 0; ++ ++ for (grp = 0; grp < desc->grp_cnt; grp++) { ++ size_t blk; ++ ++ const struct kbase_hwcnt_group_description *grp_desc = ++ desc->grps + grp; ++ struct kbase_hwcnt_group_metadata *grp_md = grp_mds + grp; ++ ++ size_t group_enable_map_count = 0; ++ size_t group_dump_buffer_count = 0; ++ size_t group_avail_mask_bits = 0; ++ ++ /* Bump allocate this group's block metadata */ ++ struct kbase_hwcnt_block_metadata *blk_mds = ++ (struct kbase_hwcnt_block_metadata *)(buf + offset); ++ offset += sizeof(struct kbase_hwcnt_block_metadata) * ++ grp_desc->blk_cnt; ++ ++ /* Fill in each block in the group's information */ ++ for (blk = 0; blk < grp_desc->blk_cnt; blk++) { ++ const struct kbase_hwcnt_block_description *blk_desc = ++ grp_desc->blks + blk; ++ struct kbase_hwcnt_block_metadata *blk_md = ++ blk_mds + blk; ++ const size_t n_values = ++ blk_desc->hdr_cnt + blk_desc->ctr_cnt; ++ ++ blk_md->type = blk_desc->type; ++ blk_md->inst_cnt = blk_desc->inst_cnt; ++ blk_md->hdr_cnt = blk_desc->hdr_cnt; ++ blk_md->ctr_cnt = blk_desc->ctr_cnt; ++ blk_md->enable_map_index = group_enable_map_count; ++ blk_md->enable_map_stride = ++ kbase_hwcnt_bitfield_count(n_values); ++ blk_md->dump_buf_index = group_dump_buffer_count; ++ blk_md->dump_buf_stride = ++ KBASE_HWCNT_ALIGN_UPWARDS( ++ n_values, ++ (KBASE_HWCNT_BLOCK_BYTE_ALIGNMENT / ++ KBASE_HWCNT_VALUE_BYTES)); ++ blk_md->avail_mask_index = group_avail_mask_bits; ++ ++ group_enable_map_count += ++ blk_md->enable_map_stride * blk_md->inst_cnt; ++ group_dump_buffer_count += ++ blk_md->dump_buf_stride * blk_md->inst_cnt; ++ group_avail_mask_bits += blk_md->inst_cnt; ++ } ++ ++ /* Fill in the group's information */ ++ grp_md->type = grp_desc->type; ++ grp_md->blk_cnt = grp_desc->blk_cnt; ++ grp_md->blk_metadata = blk_mds; ++ grp_md->enable_map_index = enable_map_count; ++ grp_md->dump_buf_index = dump_buf_count; ++ grp_md->avail_mask_index = avail_mask_bits; ++ ++ enable_map_count += group_enable_map_count; ++ dump_buf_count += group_dump_buffer_count; ++ avail_mask_bits += group_avail_mask_bits; ++ } ++ ++ /* Fill in the top level metadata's information */ ++ metadata->grp_cnt = desc->grp_cnt; ++ metadata->grp_metadata = grp_mds; ++ metadata->enable_map_bytes = ++ enable_map_count * KBASE_HWCNT_BITFIELD_BYTES; ++ metadata->dump_buf_bytes = dump_buf_count * KBASE_HWCNT_VALUE_BYTES; ++ metadata->avail_mask = desc->avail_mask; ++ metadata->clk_cnt = desc->clk_cnt; ++ ++ WARN_ON(size != offset); ++ /* Due to the block alignment, there should be exactly one enable map ++ * bit per 4 bytes in the dump buffer. ++ */ ++ WARN_ON(metadata->dump_buf_bytes != ++ (metadata->enable_map_bytes * ++ BITS_PER_BYTE * KBASE_HWCNT_VALUE_BYTES)); ++ ++ *out_metadata = metadata; ++ return 0; ++} ++KBASE_EXPORT_TEST_API(kbase_hwcnt_metadata_create); ++ ++void kbase_hwcnt_metadata_destroy(const struct kbase_hwcnt_metadata *metadata) ++{ ++ kfree(metadata); ++} ++KBASE_EXPORT_TEST_API(kbase_hwcnt_metadata_destroy); ++ ++int kbase_hwcnt_enable_map_alloc( ++ const struct kbase_hwcnt_metadata *metadata, ++ struct kbase_hwcnt_enable_map *enable_map) ++{ ++ u64 *enable_map_buf; ++ ++ if (!metadata || !enable_map) ++ return -EINVAL; ++ ++ if (metadata->enable_map_bytes > 0) { ++ enable_map_buf = ++ kzalloc(metadata->enable_map_bytes, GFP_KERNEL); ++ if (!enable_map_buf) ++ return -ENOMEM; ++ } else { ++ enable_map_buf = NULL; ++ } ++ ++ enable_map->metadata = metadata; ++ enable_map->hwcnt_enable_map = enable_map_buf; ++ return 0; ++} ++KBASE_EXPORT_TEST_API(kbase_hwcnt_enable_map_alloc); ++ ++void kbase_hwcnt_enable_map_free(struct kbase_hwcnt_enable_map *enable_map) ++{ ++ if (!enable_map) ++ return; ++ ++ kfree(enable_map->hwcnt_enable_map); ++ enable_map->hwcnt_enable_map = NULL; ++ enable_map->metadata = NULL; ++} ++KBASE_EXPORT_TEST_API(kbase_hwcnt_enable_map_free); ++ ++int kbase_hwcnt_dump_buffer_alloc( ++ const struct kbase_hwcnt_metadata *metadata, ++ struct kbase_hwcnt_dump_buffer *dump_buf) ++{ ++ size_t dump_buf_bytes; ++ size_t clk_cnt_buf_bytes; ++ u8 *buf; ++ ++ if (!metadata || !dump_buf) ++ return -EINVAL; ++ ++ dump_buf_bytes = metadata->dump_buf_bytes; ++ clk_cnt_buf_bytes = sizeof(*dump_buf->clk_cnt_buf) * metadata->clk_cnt; ++ ++ /* Make a single allocation for both dump_buf and clk_cnt_buf. */ ++ buf = kmalloc(dump_buf_bytes + clk_cnt_buf_bytes, GFP_KERNEL); ++ if (!buf) ++ return -ENOMEM; ++ ++ dump_buf->metadata = metadata; ++ dump_buf->dump_buf = (u32 *)buf; ++ dump_buf->clk_cnt_buf = (u64 *)(buf + dump_buf_bytes); ++ ++ return 0; ++} ++KBASE_EXPORT_TEST_API(kbase_hwcnt_dump_buffer_alloc); ++ ++void kbase_hwcnt_dump_buffer_free(struct kbase_hwcnt_dump_buffer *dump_buf) ++{ ++ if (!dump_buf) ++ return; ++ ++ kfree(dump_buf->dump_buf); ++ memset(dump_buf, 0, sizeof(*dump_buf)); ++} ++KBASE_EXPORT_TEST_API(kbase_hwcnt_dump_buffer_free); ++ ++int kbase_hwcnt_dump_buffer_array_alloc( ++ const struct kbase_hwcnt_metadata *metadata, ++ size_t n, ++ struct kbase_hwcnt_dump_buffer_array *dump_bufs) ++{ ++ struct kbase_hwcnt_dump_buffer *buffers; ++ size_t buf_idx; ++ unsigned int order; ++ unsigned long addr; ++ size_t dump_buf_bytes; ++ size_t clk_cnt_buf_bytes; ++ ++ if (!metadata || !dump_bufs) ++ return -EINVAL; ++ ++ dump_buf_bytes = metadata->dump_buf_bytes; ++ clk_cnt_buf_bytes = ++ sizeof(*dump_bufs->bufs->clk_cnt_buf) * metadata->clk_cnt; ++ ++ /* Allocate memory for the dump buffer struct array */ ++ buffers = kmalloc_array(n, sizeof(*buffers), GFP_KERNEL); ++ if (!buffers) ++ return -ENOMEM; ++ ++ /* Allocate pages for the actual dump buffers, as they tend to be fairly ++ * large. ++ */ ++ order = get_order((dump_buf_bytes + clk_cnt_buf_bytes) * n); ++ addr = __get_free_pages(GFP_KERNEL | __GFP_ZERO, order); ++ ++ if (!addr) { ++ kfree(buffers); ++ return -ENOMEM; ++ } ++ ++ dump_bufs->page_addr = addr; ++ dump_bufs->page_order = order; ++ dump_bufs->buf_cnt = n; ++ dump_bufs->bufs = buffers; ++ ++ /* Set the buffer of each dump buf */ ++ for (buf_idx = 0; buf_idx < n; buf_idx++) { ++ const size_t dump_buf_offset = dump_buf_bytes * buf_idx; ++ const size_t clk_cnt_buf_offset = ++ (dump_buf_bytes * n) + (clk_cnt_buf_bytes * buf_idx); ++ ++ buffers[buf_idx].metadata = metadata; ++ buffers[buf_idx].dump_buf = (u32 *)(addr + dump_buf_offset); ++ buffers[buf_idx].clk_cnt_buf = ++ (u64 *)(addr + clk_cnt_buf_offset); ++ } ++ ++ return 0; ++} ++KBASE_EXPORT_TEST_API(kbase_hwcnt_dump_buffer_array_alloc); ++ ++void kbase_hwcnt_dump_buffer_array_free( ++ struct kbase_hwcnt_dump_buffer_array *dump_bufs) ++{ ++ if (!dump_bufs) ++ return; ++ ++ kfree(dump_bufs->bufs); ++ free_pages(dump_bufs->page_addr, dump_bufs->page_order); ++ memset(dump_bufs, 0, sizeof(*dump_bufs)); ++} ++KBASE_EXPORT_TEST_API(kbase_hwcnt_dump_buffer_array_free); ++ ++void kbase_hwcnt_dump_buffer_zero( ++ struct kbase_hwcnt_dump_buffer *dst, ++ const struct kbase_hwcnt_enable_map *dst_enable_map) ++{ ++ const struct kbase_hwcnt_metadata *metadata; ++ size_t grp, blk, blk_inst; ++ ++ if (WARN_ON(!dst) || ++ WARN_ON(!dst_enable_map) || ++ WARN_ON(dst->metadata != dst_enable_map->metadata)) ++ return; ++ ++ metadata = dst->metadata; ++ ++ kbase_hwcnt_metadata_for_each_block(metadata, grp, blk, blk_inst) { ++ u32 *dst_blk; ++ size_t val_cnt; ++ ++ if (!kbase_hwcnt_enable_map_block_enabled( ++ dst_enable_map, grp, blk, blk_inst)) ++ continue; ++ ++ dst_blk = kbase_hwcnt_dump_buffer_block_instance( ++ dst, grp, blk, blk_inst); ++ val_cnt = kbase_hwcnt_metadata_block_values_count( ++ metadata, grp, blk); ++ ++ kbase_hwcnt_dump_buffer_block_zero(dst_blk, val_cnt); ++ } ++ ++ memset(dst->clk_cnt_buf, 0, ++ sizeof(*dst->clk_cnt_buf) * metadata->clk_cnt); ++} ++KBASE_EXPORT_TEST_API(kbase_hwcnt_dump_buffer_zero); ++ ++void kbase_hwcnt_dump_buffer_zero_strict( ++ struct kbase_hwcnt_dump_buffer *dst) ++{ ++ if (WARN_ON(!dst)) ++ return; ++ ++ memset(dst->dump_buf, 0, dst->metadata->dump_buf_bytes); ++ ++ memset(dst->clk_cnt_buf, 0, ++ sizeof(*dst->clk_cnt_buf) * dst->metadata->clk_cnt); ++} ++KBASE_EXPORT_TEST_API(kbase_hwcnt_dump_buffer_zero_strict); ++ ++void kbase_hwcnt_dump_buffer_zero_non_enabled( ++ struct kbase_hwcnt_dump_buffer *dst, ++ const struct kbase_hwcnt_enable_map *dst_enable_map) ++{ ++ const struct kbase_hwcnt_metadata *metadata; ++ size_t grp, blk, blk_inst; ++ ++ if (WARN_ON(!dst) || ++ WARN_ON(!dst_enable_map) || ++ WARN_ON(dst->metadata != dst_enable_map->metadata)) ++ return; ++ ++ metadata = dst->metadata; ++ ++ kbase_hwcnt_metadata_for_each_block(metadata, grp, blk, blk_inst) { ++ u32 *dst_blk = kbase_hwcnt_dump_buffer_block_instance( ++ dst, grp, blk, blk_inst); ++ const u64 *blk_em = kbase_hwcnt_enable_map_block_instance( ++ dst_enable_map, grp, blk, blk_inst); ++ size_t val_cnt = kbase_hwcnt_metadata_block_values_count( ++ metadata, grp, blk); ++ ++ /* Align upwards to include padding bytes */ ++ val_cnt = KBASE_HWCNT_ALIGN_UPWARDS(val_cnt, ++ (KBASE_HWCNT_BLOCK_BYTE_ALIGNMENT / ++ KBASE_HWCNT_VALUE_BYTES)); ++ ++ if (kbase_hwcnt_metadata_block_instance_avail( ++ metadata, grp, blk, blk_inst)) { ++ /* Block available, so only zero non-enabled values */ ++ kbase_hwcnt_dump_buffer_block_zero_non_enabled( ++ dst_blk, blk_em, val_cnt); ++ } else { ++ /* Block not available, so zero the entire thing */ ++ kbase_hwcnt_dump_buffer_block_zero(dst_blk, val_cnt); ++ } ++ } ++} ++KBASE_EXPORT_TEST_API(kbase_hwcnt_dump_buffer_zero_non_enabled); ++ ++void kbase_hwcnt_dump_buffer_copy( ++ struct kbase_hwcnt_dump_buffer *dst, ++ const struct kbase_hwcnt_dump_buffer *src, ++ const struct kbase_hwcnt_enable_map *dst_enable_map) ++{ ++ const struct kbase_hwcnt_metadata *metadata; ++ size_t grp, blk, blk_inst; ++ size_t clk; ++ ++ if (WARN_ON(!dst) || ++ WARN_ON(!src) || ++ WARN_ON(!dst_enable_map) || ++ WARN_ON(dst == src) || ++ WARN_ON(dst->metadata != src->metadata) || ++ WARN_ON(dst->metadata != dst_enable_map->metadata)) ++ return; ++ ++ metadata = dst->metadata; ++ ++ kbase_hwcnt_metadata_for_each_block(metadata, grp, blk, blk_inst) { ++ u32 *dst_blk; ++ const u32 *src_blk; ++ size_t val_cnt; ++ ++ if (!kbase_hwcnt_enable_map_block_enabled( ++ dst_enable_map, grp, blk, blk_inst)) ++ continue; ++ ++ dst_blk = kbase_hwcnt_dump_buffer_block_instance( ++ dst, grp, blk, blk_inst); ++ src_blk = kbase_hwcnt_dump_buffer_block_instance( ++ src, grp, blk, blk_inst); ++ val_cnt = kbase_hwcnt_metadata_block_values_count( ++ metadata, grp, blk); ++ ++ kbase_hwcnt_dump_buffer_block_copy(dst_blk, src_blk, val_cnt); ++ } ++ ++ kbase_hwcnt_metadata_for_each_clock(metadata, clk) { ++ if (kbase_hwcnt_clk_enable_map_enabled( ++ dst_enable_map->clk_enable_map, clk)) ++ dst->clk_cnt_buf[clk] = src->clk_cnt_buf[clk]; ++ } ++} ++KBASE_EXPORT_TEST_API(kbase_hwcnt_dump_buffer_copy); ++ ++void kbase_hwcnt_dump_buffer_copy_strict( ++ struct kbase_hwcnt_dump_buffer *dst, ++ const struct kbase_hwcnt_dump_buffer *src, ++ const struct kbase_hwcnt_enable_map *dst_enable_map) ++{ ++ const struct kbase_hwcnt_metadata *metadata; ++ size_t grp, blk, blk_inst; ++ size_t clk; ++ ++ if (WARN_ON(!dst) || ++ WARN_ON(!src) || ++ WARN_ON(!dst_enable_map) || ++ WARN_ON(dst == src) || ++ WARN_ON(dst->metadata != src->metadata) || ++ WARN_ON(dst->metadata != dst_enable_map->metadata)) ++ return; ++ ++ metadata = dst->metadata; ++ ++ kbase_hwcnt_metadata_for_each_block(metadata, grp, blk, blk_inst) { ++ u32 *dst_blk = kbase_hwcnt_dump_buffer_block_instance( ++ dst, grp, blk, blk_inst); ++ const u32 *src_blk = kbase_hwcnt_dump_buffer_block_instance( ++ src, grp, blk, blk_inst); ++ const u64 *blk_em = kbase_hwcnt_enable_map_block_instance( ++ dst_enable_map, grp, blk, blk_inst); ++ size_t val_cnt = kbase_hwcnt_metadata_block_values_count( ++ metadata, grp, blk); ++ /* Align upwards to include padding bytes */ ++ val_cnt = KBASE_HWCNT_ALIGN_UPWARDS(val_cnt, ++ (KBASE_HWCNT_BLOCK_BYTE_ALIGNMENT / ++ KBASE_HWCNT_VALUE_BYTES)); ++ ++ kbase_hwcnt_dump_buffer_block_copy_strict( ++ dst_blk, src_blk, blk_em, val_cnt); ++ } ++ ++ kbase_hwcnt_metadata_for_each_clock(metadata, clk) { ++ bool clk_enabled = ++ kbase_hwcnt_clk_enable_map_enabled( ++ dst_enable_map->clk_enable_map, clk); ++ ++ dst->clk_cnt_buf[clk] = clk_enabled ? src->clk_cnt_buf[clk] : 0; ++ } ++} ++KBASE_EXPORT_TEST_API(kbase_hwcnt_dump_buffer_copy_strict); ++ ++void kbase_hwcnt_dump_buffer_accumulate( ++ struct kbase_hwcnt_dump_buffer *dst, ++ const struct kbase_hwcnt_dump_buffer *src, ++ const struct kbase_hwcnt_enable_map *dst_enable_map) ++{ ++ const struct kbase_hwcnt_metadata *metadata; ++ size_t grp, blk, blk_inst; ++ size_t clk; ++ ++ if (WARN_ON(!dst) || ++ WARN_ON(!src) || ++ WARN_ON(!dst_enable_map) || ++ WARN_ON(dst == src) || ++ WARN_ON(dst->metadata != src->metadata) || ++ WARN_ON(dst->metadata != dst_enable_map->metadata)) ++ return; ++ ++ metadata = dst->metadata; ++ ++ kbase_hwcnt_metadata_for_each_block(metadata, grp, blk, blk_inst) { ++ u32 *dst_blk; ++ const u32 *src_blk; ++ size_t hdr_cnt; ++ size_t ctr_cnt; ++ ++ if (!kbase_hwcnt_enable_map_block_enabled( ++ dst_enable_map, grp, blk, blk_inst)) ++ continue; ++ ++ dst_blk = kbase_hwcnt_dump_buffer_block_instance( ++ dst, grp, blk, blk_inst); ++ src_blk = kbase_hwcnt_dump_buffer_block_instance( ++ src, grp, blk, blk_inst); ++ hdr_cnt = kbase_hwcnt_metadata_block_headers_count( ++ metadata, grp, blk); ++ ctr_cnt = kbase_hwcnt_metadata_block_counters_count( ++ metadata, grp, blk); ++ ++ kbase_hwcnt_dump_buffer_block_accumulate( ++ dst_blk, src_blk, hdr_cnt, ctr_cnt); ++ } ++ ++ kbase_hwcnt_metadata_for_each_clock(metadata, clk) { ++ if (kbase_hwcnt_clk_enable_map_enabled( ++ dst_enable_map->clk_enable_map, clk)) ++ dst->clk_cnt_buf[clk] += src->clk_cnt_buf[clk]; ++ } ++} ++KBASE_EXPORT_TEST_API(kbase_hwcnt_dump_buffer_accumulate); ++ ++void kbase_hwcnt_dump_buffer_accumulate_strict( ++ struct kbase_hwcnt_dump_buffer *dst, ++ const struct kbase_hwcnt_dump_buffer *src, ++ const struct kbase_hwcnt_enable_map *dst_enable_map) ++{ ++ const struct kbase_hwcnt_metadata *metadata; ++ size_t grp, blk, blk_inst; ++ size_t clk; ++ ++ if (WARN_ON(!dst) || ++ WARN_ON(!src) || ++ WARN_ON(!dst_enable_map) || ++ WARN_ON(dst == src) || ++ WARN_ON(dst->metadata != src->metadata) || ++ WARN_ON(dst->metadata != dst_enable_map->metadata)) ++ return; ++ ++ metadata = dst->metadata; ++ ++ kbase_hwcnt_metadata_for_each_block(metadata, grp, blk, blk_inst) { ++ u32 *dst_blk = kbase_hwcnt_dump_buffer_block_instance( ++ dst, grp, blk, blk_inst); ++ const u32 *src_blk = kbase_hwcnt_dump_buffer_block_instance( ++ src, grp, blk, blk_inst); ++ const u64 *blk_em = kbase_hwcnt_enable_map_block_instance( ++ dst_enable_map, grp, blk, blk_inst); ++ size_t hdr_cnt = kbase_hwcnt_metadata_block_headers_count( ++ metadata, grp, blk); ++ size_t ctr_cnt = kbase_hwcnt_metadata_block_counters_count( ++ metadata, grp, blk); ++ /* Align upwards to include padding bytes */ ++ ctr_cnt = KBASE_HWCNT_ALIGN_UPWARDS(hdr_cnt + ctr_cnt, ++ (KBASE_HWCNT_BLOCK_BYTE_ALIGNMENT / ++ KBASE_HWCNT_VALUE_BYTES) - hdr_cnt); ++ ++ kbase_hwcnt_dump_buffer_block_accumulate_strict( ++ dst_blk, src_blk, blk_em, hdr_cnt, ctr_cnt); ++ } ++ ++ kbase_hwcnt_metadata_for_each_clock(metadata, clk) { ++ if (kbase_hwcnt_clk_enable_map_enabled( ++ dst_enable_map->clk_enable_map, clk)) ++ dst->clk_cnt_buf[clk] += src->clk_cnt_buf[clk]; ++ else ++ dst->clk_cnt_buf[clk] = 0; ++ } ++} ++KBASE_EXPORT_TEST_API(kbase_hwcnt_dump_buffer_accumulate_strict); +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_types.h b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_types.h +new file mode 100755 +index 000000000000..3394b1271cc8 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_types.h +@@ -0,0 +1,1142 @@ ++/* ++ * ++ * (C) COPYRIGHT 2018, 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/** ++ * Hardware counter types. ++ * Contains structures for describing the physical layout of hardware counter ++ * dump buffers and enable maps within a system. ++ * ++ * Also contains helper functions for manipulation of these dump buffers and ++ * enable maps. ++ * ++ * Through use of these structures and functions, hardware counters can be ++ * enabled, copied, accumulated, and generally manipulated in a generic way, ++ * regardless of the physical counter dump layout. ++ * ++ * Terminology: ++ * ++ * Hardware Counter System: ++ * A collection of hardware counter groups, making a full hardware counter ++ * system. ++ * Hardware Counter Group: ++ * A group of Hardware Counter Blocks (e.g. a t62x might have more than one ++ * core group, so has one counter group per core group, where each group ++ * may have a different number and layout of counter blocks). ++ * Hardware Counter Block: ++ * A block of hardware counters (e.g. shader block, tiler block). ++ * Hardware Counter Block Instance: ++ * An instance of a Hardware Counter Block (e.g. an MP4 GPU might have ++ * 4 shader block instances). ++ * ++ * Block Header: ++ * A header value inside a counter block. Headers don't count anything, ++ * so it is only valid to copy or zero them. Headers are always the first ++ * values in the block. ++ * Block Counter: ++ * A counter value inside a counter block. Counters can be zeroed, copied, ++ * or accumulated. Counters are always immediately after the headers in the ++ * block. ++ * Block Value: ++ * A catch-all term for block headers and block counters. ++ * ++ * Enable Map: ++ * An array of u64 bitfields, where each bit either enables exactly one ++ * block value, or is unused (padding). ++ * Dump Buffer: ++ * An array of u32 values, where each u32 corresponds either to one block ++ * value, or is unused (padding). ++ * Availability Mask: ++ * A bitfield, where each bit corresponds to whether a block instance is ++ * physically available (e.g. an MP3 GPU may have a sparse core mask of ++ * 0b1011, meaning it only has 3 cores but for hardware counter dumps has the ++ * same dump buffer layout as an MP4 GPU with a core mask of 0b1111. In this ++ * case, the availability mask might be 0b1011111 (the exact layout will ++ * depend on the specific hardware architecture), with the 3 extra early bits ++ * corresponding to other block instances in the hardware counter system). ++ * Metadata: ++ * Structure describing the physical layout of the enable map and dump buffers ++ * for a specific hardware counter system. ++ * ++ */ ++ ++#ifndef _KBASE_HWCNT_TYPES_H_ ++#define _KBASE_HWCNT_TYPES_H_ ++ ++#include ++#include ++#include ++#include ++#include "mali_malisw.h" ++ ++/* Number of bytes in each bitfield */ ++#define KBASE_HWCNT_BITFIELD_BYTES (sizeof(u64)) ++ ++/* Number of bits in each bitfield */ ++#define KBASE_HWCNT_BITFIELD_BITS (KBASE_HWCNT_BITFIELD_BYTES * BITS_PER_BYTE) ++ ++/* Number of bytes for each counter value */ ++#define KBASE_HWCNT_VALUE_BYTES (sizeof(u32)) ++ ++/* Number of bits in an availability mask (i.e. max total number of block ++ * instances supported in a Hardware Counter System) ++ */ ++#define KBASE_HWCNT_AVAIL_MASK_BITS (sizeof(u64) * BITS_PER_BYTE) ++ ++/** ++ * struct kbase_hwcnt_block_description - Description of one or more identical, ++ * contiguous, Hardware Counter Blocks. ++ * @type: The arbitrary identifier used to identify the type of the block. ++ * @inst_cnt: The number of Instances of the block. ++ * @hdr_cnt: The number of 32-bit Block Headers in the block. ++ * @ctr_cnt: The number of 32-bit Block Counters in the block. ++ */ ++struct kbase_hwcnt_block_description { ++ u64 type; ++ size_t inst_cnt; ++ size_t hdr_cnt; ++ size_t ctr_cnt; ++}; ++ ++/** ++ * struct kbase_hwcnt_group_description - Description of one or more identical, ++ * contiguous Hardware Counter Groups. ++ * @type: The arbitrary identifier used to identify the type of the group. ++ * @blk_cnt: The number of types of Hardware Counter Block in the group. ++ * @blks: Non-NULL pointer to an array of blk_cnt block descriptions, ++ * describing each type of Hardware Counter Block in the group. ++ */ ++struct kbase_hwcnt_group_description { ++ u64 type; ++ size_t blk_cnt; ++ const struct kbase_hwcnt_block_description *blks; ++}; ++ ++/** ++ * struct kbase_hwcnt_description - Description of a Hardware Counter System. ++ * @grp_cnt: The number of Hardware Counter Groups. ++ * @grps: Non-NULL pointer to an array of grp_cnt group descriptions, ++ * describing each Hardware Counter Group in the system. ++ * @avail_mask: Flat Availability Mask for all block instances in the system. ++ * @clk_cnt: The number of clock domains in the system. The maximum is 64. ++ */ ++struct kbase_hwcnt_description { ++ size_t grp_cnt; ++ const struct kbase_hwcnt_group_description *grps; ++ u64 avail_mask; ++ u8 clk_cnt; ++}; ++ ++/** ++ * struct kbase_hwcnt_block_metadata - Metadata describing the physical layout ++ * of a block in a Hardware Counter System's ++ * Dump Buffers and Enable Maps. ++ * @type: The arbitrary identifier used to identify the type of the ++ * block. ++ * @inst_cnt: The number of Instances of the block. ++ * @hdr_cnt: The number of 32-bit Block Headers in the block. ++ * @ctr_cnt: The number of 32-bit Block Counters in the block. ++ * @enable_map_index: Index in u64s into the parent's Enable Map where the ++ * Enable Map bitfields of the Block Instances described by ++ * this metadata start. ++ * @enable_map_stride: Stride in u64s between the Enable Maps of each of the ++ * Block Instances described by this metadata. ++ * @dump_buf_index: Index in u32s into the parent's Dump Buffer where the ++ * Dump Buffers of the Block Instances described by this ++ * metadata start. ++ * @dump_buf_stride: Stride in u32s between the Dump Buffers of each of the ++ * Block Instances described by this metadata. ++ * @avail_mask_index: Index in bits into the parent's Availability Mask where ++ * the Availability Masks of the Block Instances described ++ * by this metadata start. ++ */ ++struct kbase_hwcnt_block_metadata { ++ u64 type; ++ size_t inst_cnt; ++ size_t hdr_cnt; ++ size_t ctr_cnt; ++ size_t enable_map_index; ++ size_t enable_map_stride; ++ size_t dump_buf_index; ++ size_t dump_buf_stride; ++ size_t avail_mask_index; ++}; ++ ++/** ++ * struct kbase_hwcnt_group_metadata - Metadata describing the physical layout ++ * of a group of blocks in a Hardware ++ * Counter System's Dump Buffers and Enable ++ * Maps. ++ * @type: The arbitrary identifier used to identify the type of the ++ * group. ++ * @blk_cnt: The number of types of Hardware Counter Block in the ++ * group. ++ * @blk_metadata: Non-NULL pointer to an array of blk_cnt block metadata, ++ * describing the physical layout of each type of Hardware ++ * Counter Block in the group. ++ * @enable_map_index: Index in u64s into the parent's Enable Map where the ++ * Enable Maps of the blocks within the group described by ++ * this metadata start. ++ * @dump_buf_index: Index in u32s into the parent's Dump Buffer where the ++ * Dump Buffers of the blocks within the group described by ++ * metadata start. ++ * @avail_mask_index: Index in bits into the parent's Availability Mask where ++ * the Availability Masks of the blocks within the group ++ * described by this metadata start. ++ */ ++struct kbase_hwcnt_group_metadata { ++ u64 type; ++ size_t blk_cnt; ++ const struct kbase_hwcnt_block_metadata *blk_metadata; ++ size_t enable_map_index; ++ size_t dump_buf_index; ++ size_t avail_mask_index; ++}; ++ ++/** ++ * struct kbase_hwcnt_metadata - Metadata describing the physical layout ++ * of Dump Buffers and Enable Maps within a ++ * Hardware Counter System. ++ * @grp_cnt: The number of Hardware Counter Groups. ++ * @grp_metadata: Non-NULL pointer to an array of grp_cnt group metadata, ++ * describing the physical layout of each Hardware Counter ++ * Group in the system. ++ * @enable_map_bytes: The size in bytes of an Enable Map needed for the system. ++ * @dump_buf_bytes: The size in bytes of a Dump Buffer needed for the system. ++ * @avail_mask: The Availability Mask for the system. ++ * @clk_cnt: The number of clock domains in the system. ++ */ ++struct kbase_hwcnt_metadata { ++ size_t grp_cnt; ++ const struct kbase_hwcnt_group_metadata *grp_metadata; ++ size_t enable_map_bytes; ++ size_t dump_buf_bytes; ++ u64 avail_mask; ++ u8 clk_cnt; ++}; ++ ++/** ++ * struct kbase_hwcnt_enable_map - Hardware Counter Enable Map. Array of u64 ++ * bitfields. ++ * @metadata: Non-NULL pointer to metadata used to identify, and to describe ++ * the layout of the enable map. ++ * @hwcnt_enable_map: Non-NULL pointer of size metadata->enable_map_bytes to an ++ * array of u64 bitfields, each bit of which enables one hardware ++ * counter. ++ * @clk_enable_map: An array of u64 bitfields, each bit of which enables cycle ++ * counter for a given clock domain. ++ */ ++struct kbase_hwcnt_enable_map { ++ const struct kbase_hwcnt_metadata *metadata; ++ u64 *hwcnt_enable_map; ++ u64 clk_enable_map; ++}; ++ ++/** ++ * struct kbase_hwcnt_dump_buffer - Hardware Counter Dump Buffer. Array of u32 ++ * values. ++ * @metadata: Non-NULL pointer to metadata used to identify, and to describe ++ * the layout of the Dump Buffer. ++ * @dump_buf: Non-NULL pointer of size metadata->dump_buf_bytes to an array ++ * of u32 values. ++ * @clk_cnt_buf: A pointer to an array of u64 values for cycle count elapsed ++ * for each clock domain. ++ */ ++struct kbase_hwcnt_dump_buffer { ++ const struct kbase_hwcnt_metadata *metadata; ++ u32 *dump_buf; ++ u64 *clk_cnt_buf; ++}; ++ ++/** ++ * struct kbase_hwcnt_dump_buffer_array - Hardware Counter Dump Buffer array. ++ * @page_addr: Address of allocated pages. A single allocation is used for all ++ * Dump Buffers in the array. ++ * @page_order: The allocation order of the pages. ++ * @buf_cnt: The number of allocated Dump Buffers. ++ * @bufs: Non-NULL pointer to the array of Dump Buffers. ++ */ ++struct kbase_hwcnt_dump_buffer_array { ++ unsigned long page_addr; ++ unsigned int page_order; ++ size_t buf_cnt; ++ struct kbase_hwcnt_dump_buffer *bufs; ++}; ++ ++/** ++ * kbase_hwcnt_metadata_create() - Create a hardware counter metadata object ++ * from a description. ++ * @desc: Non-NULL pointer to a hardware counter description. ++ * @metadata: Non-NULL pointer to where created metadata will be stored on ++ * success. ++ * ++ * Return: 0 on success, else error code. ++ */ ++int kbase_hwcnt_metadata_create( ++ const struct kbase_hwcnt_description *desc, ++ const struct kbase_hwcnt_metadata **metadata); ++ ++/** ++ * kbase_hwcnt_metadata_destroy() - Destroy a hardware counter metadata object. ++ * @metadata: Pointer to hardware counter metadata ++ */ ++void kbase_hwcnt_metadata_destroy(const struct kbase_hwcnt_metadata *metadata); ++ ++/** ++ * kbase_hwcnt_metadata_group_count() - Get the number of groups. ++ * @metadata: Non-NULL pointer to metadata. ++ * ++ * Return: Number of hardware counter groups described by metadata. ++ */ ++#define kbase_hwcnt_metadata_group_count(metadata) \ ++ ((metadata)->grp_cnt) ++ ++/** ++ * kbase_hwcnt_metadata_group_type() - Get the arbitrary type of a group. ++ * @metadata: Non-NULL pointer to metadata. ++ * @grp: Index of the group in the metadata. ++ * ++ * Return: Type of the group grp. ++ */ ++#define kbase_hwcnt_metadata_group_type(metadata, grp) \ ++ ((metadata)->grp_metadata[(grp)].type) ++ ++/** ++ * kbase_hwcnt_metadata_block_count() - Get the number of blocks in a group. ++ * @metadata: Non-NULL pointer to metadata. ++ * @grp: Index of the group in the metadata. ++ * ++ * Return: Number of blocks in group grp. ++ */ ++#define kbase_hwcnt_metadata_block_count(metadata, grp) \ ++ ((metadata)->grp_metadata[(grp)].blk_cnt) ++ ++/** ++ * kbase_hwcnt_metadata_block_type() - Get the arbitrary type of a block. ++ * @metadata: Non-NULL pointer to metadata. ++ * @grp: Index of the group in the metadata. ++ * @blk: Index of the block in the group. ++ * ++ * Return: Type of the block blk in group grp. ++ */ ++#define kbase_hwcnt_metadata_block_type(metadata, grp, blk) \ ++ ((metadata)->grp_metadata[(grp)].blk_metadata[(blk)].type) ++ ++/** ++ * kbase_hwcnt_metadata_block_instance_count() - Get the number of instances of ++ * a block. ++ * @metadata: Non-NULL pointer to metadata. ++ * @grp: Index of the group in the metadata. ++ * @blk: Index of the block in the group. ++ * ++ * Return: Number of instances of block blk in group grp. ++ */ ++#define kbase_hwcnt_metadata_block_instance_count(metadata, grp, blk) \ ++ ((metadata)->grp_metadata[(grp)].blk_metadata[(blk)].inst_cnt) ++ ++/** ++ * kbase_hwcnt_metadata_block_headers_count() - Get the number of counter ++ * headers. ++ * @metadata: Non-NULL pointer to metadata. ++ * @grp: Index of the group in the metadata. ++ * @blk: Index of the block in the group. ++ * ++ * Return: Number of u32 counter headers in each instance of block blk in ++ * group grp. ++ */ ++#define kbase_hwcnt_metadata_block_headers_count(metadata, grp, blk) \ ++ ((metadata)->grp_metadata[(grp)].blk_metadata[(blk)].hdr_cnt) ++ ++/** ++ * kbase_hwcnt_metadata_block_counters_count() - Get the number of counters. ++ * @metadata: Non-NULL pointer to metadata. ++ * @grp: Index of the group in the metadata. ++ * @blk: Index of the block in the group. ++ * ++ * Return: Number of u32 counters in each instance of block blk in group ++ * grp. ++ */ ++#define kbase_hwcnt_metadata_block_counters_count(metadata, grp, blk) \ ++ ((metadata)->grp_metadata[(grp)].blk_metadata[(blk)].ctr_cnt) ++ ++/** ++ * kbase_hwcnt_metadata_block_values_count() - Get the number of values. ++ * @metadata: Non-NULL pointer to metadata. ++ * @grp: Index of the group in the metadata. ++ * @blk: Index of the block in the group. ++ * ++ * Return: Number of u32 headers plus counters in each instance of block blk ++ * in group grp. ++ */ ++#define kbase_hwcnt_metadata_block_values_count(metadata, grp, blk) \ ++ (kbase_hwcnt_metadata_block_counters_count((metadata), (grp), (blk)) \ ++ + kbase_hwcnt_metadata_block_headers_count((metadata), (grp), (blk))) ++ ++/** ++ * kbase_hwcnt_metadata_for_each_block() - Iterate over each block instance in ++ * the metadata. ++ * @md: Non-NULL pointer to metadata. ++ * @grp: size_t variable used as group iterator. ++ * @blk: size_t variable used as block iterator. ++ * @blk_inst: size_t variable used as block instance iterator. ++ * ++ * Iteration order is group, then block, then block instance (i.e. linearly ++ * through memory). ++ */ ++#define kbase_hwcnt_metadata_for_each_block(md, grp, blk, blk_inst) \ ++ for ((grp) = 0; (grp) < kbase_hwcnt_metadata_group_count((md)); (grp)++) \ ++ for ((blk) = 0; (blk) < kbase_hwcnt_metadata_block_count((md), (grp)); (blk)++) \ ++ for ((blk_inst) = 0; (blk_inst) < kbase_hwcnt_metadata_block_instance_count((md), (grp), (blk)); (blk_inst)++) ++ ++/** ++ * kbase_hwcnt_metadata_block_avail_bit() - Get the bit index into the avail ++ * mask corresponding to the block. ++ * @metadata: Non-NULL pointer to metadata. ++ * @grp: Index of the group in the metadata. ++ * @blk: Index of the block in the group. ++ * ++ * Return: The bit index into the avail mask for the block. ++ */ ++static inline size_t kbase_hwcnt_metadata_block_avail_bit( ++ const struct kbase_hwcnt_metadata *metadata, ++ size_t grp, ++ size_t blk) ++{ ++ const size_t bit = ++ metadata->grp_metadata[grp].avail_mask_index + ++ metadata->grp_metadata[grp].blk_metadata[blk].avail_mask_index; ++ ++ return bit; ++} ++ ++/** ++ * kbase_hwcnt_metadata_block_instance_avail() - Check if a block instance is ++ * available. ++ * @metadata: Non-NULL pointer to metadata. ++ * @grp: Index of the group in the metadata. ++ * @blk: Index of the block in the group. ++ * @blk_inst: Index of the block instance in the block. ++ * ++ * Return: true if the block instance is available, else false. ++ */ ++static inline bool kbase_hwcnt_metadata_block_instance_avail( ++ const struct kbase_hwcnt_metadata *metadata, ++ size_t grp, ++ size_t blk, ++ size_t blk_inst) ++{ ++ const size_t bit = kbase_hwcnt_metadata_block_avail_bit( ++ metadata, grp, blk) + blk_inst; ++ const u64 mask = 1ull << bit; ++ ++ return (metadata->avail_mask & mask) != 0; ++} ++ ++/** ++ * kbase_hwcnt_enable_map_alloc() - Allocate an enable map. ++ * @metadata: Non-NULL pointer to metadata describing the system. ++ * @enable_map: Non-NULL pointer to enable map to be initialised. Will be ++ * initialised to all zeroes (i.e. all counters disabled). ++ * ++ * Return: 0 on success, else error code. ++ */ ++int kbase_hwcnt_enable_map_alloc( ++ const struct kbase_hwcnt_metadata *metadata, ++ struct kbase_hwcnt_enable_map *enable_map); ++ ++/** ++ * kbase_hwcnt_enable_map_free() - Free an enable map. ++ * @enable_map: Enable map to be freed. ++ * ++ * Can be safely called on an all-zeroed enable map structure, or on an already ++ * freed enable map. ++ */ ++void kbase_hwcnt_enable_map_free(struct kbase_hwcnt_enable_map *enable_map); ++ ++/** ++ * kbase_hwcnt_enable_map_block_instance() - Get the pointer to a block ++ * instance's enable map. ++ * @map: Non-NULL pointer to (const) enable map. ++ * @grp: Index of the group in the metadata. ++ * @blk: Index of the block in the group. ++ * @blk_inst: Index of the block instance in the block. ++ * ++ * Return: (const) u64* to the bitfield(s) used as the enable map for the ++ * block instance. ++ */ ++#define kbase_hwcnt_enable_map_block_instance(map, grp, blk, blk_inst) \ ++ ((map)->hwcnt_enable_map + \ ++ (map)->metadata->grp_metadata[(grp)].enable_map_index + \ ++ (map)->metadata->grp_metadata[(grp)].blk_metadata[(blk)].enable_map_index + \ ++ (map)->metadata->grp_metadata[(grp)].blk_metadata[(blk)].enable_map_stride * (blk_inst)) ++ ++/** ++ * kbase_hwcnt_bitfield_count() - Calculate the number of u64 bitfields required ++ * to have at minimum one bit per value. ++ * @val_cnt: Number of values. ++ * ++ * Return: Number of required bitfields. ++ */ ++static inline size_t kbase_hwcnt_bitfield_count(size_t val_cnt) ++{ ++ return (val_cnt + KBASE_HWCNT_BITFIELD_BITS - 1) / ++ KBASE_HWCNT_BITFIELD_BITS; ++} ++ ++/** ++ * kbase_hwcnt_enable_map_block_disable_all() - Disable all values in a block. ++ * @dst: Non-NULL pointer to enable map. ++ * @grp: Index of the group in the metadata. ++ * @blk: Index of the block in the group. ++ * @blk_inst: Index of the block instance in the block. ++ */ ++static inline void kbase_hwcnt_enable_map_block_disable_all( ++ struct kbase_hwcnt_enable_map *dst, ++ size_t grp, ++ size_t blk, ++ size_t blk_inst) ++{ ++ const size_t val_cnt = kbase_hwcnt_metadata_block_values_count( ++ dst->metadata, grp, blk); ++ const size_t bitfld_cnt = kbase_hwcnt_bitfield_count(val_cnt); ++ u64 *block_enable_map = kbase_hwcnt_enable_map_block_instance( ++ dst, grp, blk, blk_inst); ++ ++ memset(block_enable_map, 0, bitfld_cnt * KBASE_HWCNT_BITFIELD_BYTES); ++} ++ ++/** ++ * kbase_hwcnt_enable_map_disable_all() - Disable all values in the enable map. ++ * @dst: Non-NULL pointer to enable map to zero. ++ */ ++static inline void kbase_hwcnt_enable_map_disable_all( ++ struct kbase_hwcnt_enable_map *dst) ++{ ++ if (dst->hwcnt_enable_map != NULL) ++ memset(dst->hwcnt_enable_map, 0, ++ dst->metadata->enable_map_bytes); ++ ++ dst->clk_enable_map = 0; ++} ++ ++/** ++ * kbase_hwcnt_enable_map_block_enable_all() - Enable all values in a block. ++ * @dst: Non-NULL pointer to enable map. ++ * @grp: Index of the group in the metadata. ++ * @blk: Index of the block in the group. ++ * @blk_inst: Index of the block instance in the block. ++ */ ++static inline void kbase_hwcnt_enable_map_block_enable_all( ++ struct kbase_hwcnt_enable_map *dst, ++ size_t grp, ++ size_t blk, ++ size_t blk_inst) ++{ ++ const size_t val_cnt = kbase_hwcnt_metadata_block_values_count( ++ dst->metadata, grp, blk); ++ const size_t bitfld_cnt = kbase_hwcnt_bitfield_count(val_cnt); ++ u64 *block_enable_map = kbase_hwcnt_enable_map_block_instance( ++ dst, grp, blk, blk_inst); ++ ++ size_t bitfld_idx; ++ ++ for (bitfld_idx = 0; bitfld_idx < bitfld_cnt; bitfld_idx++) { ++ const u64 remaining_values = val_cnt - ++ (bitfld_idx * KBASE_HWCNT_BITFIELD_BITS); ++ u64 block_enable_map_mask = U64_MAX; ++ ++ if (remaining_values < KBASE_HWCNT_BITFIELD_BITS) ++ block_enable_map_mask = (1ull << remaining_values) - 1; ++ ++ block_enable_map[bitfld_idx] = block_enable_map_mask; ++ } ++} ++ ++/** ++ * kbase_hwcnt_enable_map_block_enable_all() - Enable all values in an enable ++ * map. ++ * @dst: Non-NULL pointer to enable map. ++ */ ++static inline void kbase_hwcnt_enable_map_enable_all( ++ struct kbase_hwcnt_enable_map *dst) ++{ ++ size_t grp, blk, blk_inst; ++ ++ kbase_hwcnt_metadata_for_each_block(dst->metadata, grp, blk, blk_inst) ++ kbase_hwcnt_enable_map_block_enable_all( ++ dst, grp, blk, blk_inst); ++ ++ dst->clk_enable_map = (1ull << dst->metadata->clk_cnt) - 1; ++} ++ ++/** ++ * kbase_hwcnt_enable_map_copy() - Copy an enable map to another. ++ * @dst: Non-NULL pointer to destination enable map. ++ * @src: Non-NULL pointer to source enable map. ++ * ++ * The dst and src MUST have been created from the same metadata. ++ */ ++static inline void kbase_hwcnt_enable_map_copy( ++ struct kbase_hwcnt_enable_map *dst, ++ const struct kbase_hwcnt_enable_map *src) ++{ ++ if (dst->hwcnt_enable_map != NULL) { ++ memcpy(dst->hwcnt_enable_map, ++ src->hwcnt_enable_map, ++ dst->metadata->enable_map_bytes); ++ } ++ ++ dst->clk_enable_map = src->clk_enable_map; ++} ++ ++/** ++ * kbase_hwcnt_enable_map_union() - Union dst and src enable maps into dst. ++ * @dst: Non-NULL pointer to destination enable map. ++ * @src: Non-NULL pointer to source enable map. ++ * ++ * The dst and src MUST have been created from the same metadata. ++ */ ++static inline void kbase_hwcnt_enable_map_union( ++ struct kbase_hwcnt_enable_map *dst, ++ const struct kbase_hwcnt_enable_map *src) ++{ ++ const size_t bitfld_count = ++ dst->metadata->enable_map_bytes / KBASE_HWCNT_BITFIELD_BYTES; ++ size_t i; ++ ++ if (dst->hwcnt_enable_map != NULL) { ++ for (i = 0; i < bitfld_count; i++) ++ dst->hwcnt_enable_map[i] |= src->hwcnt_enable_map[i]; ++ } ++ ++ dst->clk_enable_map |= src->clk_enable_map; ++} ++ ++/** ++ * kbase_hwcnt_enable_map_block_enabled() - Check if any values in a block ++ * instance are enabled. ++ * @enable_map: Non-NULL pointer to enable map. ++ * @grp: Index of the group in the metadata. ++ * @blk: Index of the block in the group. ++ * @blk_inst: Index of the block instance in the block. ++ * ++ * Return: true if any values in the block are enabled, else false. ++ */ ++static inline bool kbase_hwcnt_enable_map_block_enabled( ++ const struct kbase_hwcnt_enable_map *enable_map, ++ size_t grp, ++ size_t blk, ++ size_t blk_inst) ++{ ++ bool any_enabled = false; ++ const size_t val_cnt = kbase_hwcnt_metadata_block_values_count( ++ enable_map->metadata, grp, blk); ++ const size_t bitfld_cnt = kbase_hwcnt_bitfield_count(val_cnt); ++ const u64 *block_enable_map = kbase_hwcnt_enable_map_block_instance( ++ enable_map, grp, blk, blk_inst); ++ ++ size_t bitfld_idx; ++ ++ for (bitfld_idx = 0; bitfld_idx < bitfld_cnt; bitfld_idx++) { ++ const u64 remaining_values = val_cnt - ++ (bitfld_idx * KBASE_HWCNT_BITFIELD_BITS); ++ u64 block_enable_map_mask = U64_MAX; ++ ++ if (remaining_values < KBASE_HWCNT_BITFIELD_BITS) ++ block_enable_map_mask = (1ull << remaining_values) - 1; ++ ++ any_enabled = any_enabled || ++ (block_enable_map[bitfld_idx] & block_enable_map_mask); ++ } ++ ++ return any_enabled; ++} ++ ++/** ++ * kbase_hwcnt_enable_map_any_enabled() - Check if any values are enabled. ++ * @enable_map: Non-NULL pointer to enable map. ++ * ++ * Return: true if any values are enabled, else false. ++ */ ++static inline bool kbase_hwcnt_enable_map_any_enabled( ++ const struct kbase_hwcnt_enable_map *enable_map) ++{ ++ size_t grp, blk, blk_inst; ++ const u64 clk_enable_map_mask = ++ (1ull << enable_map->metadata->clk_cnt) - 1; ++ ++ if (enable_map->metadata->clk_cnt > 0 && ++ (enable_map->clk_enable_map & clk_enable_map_mask)) ++ return true; ++ ++ kbase_hwcnt_metadata_for_each_block( ++ enable_map->metadata, grp, blk, blk_inst) { ++ if (kbase_hwcnt_enable_map_block_enabled( ++ enable_map, grp, blk, blk_inst)) ++ return true; ++ } ++ ++ return false; ++} ++ ++/** ++ * kbase_hwcnt_enable_map_block_value_enabled() - Check if a value in a block ++ * instance is enabled. ++ * @bitfld: Non-NULL pointer to the block bitfield(s) obtained from a call to ++ * kbase_hwcnt_enable_map_block_instance. ++ * @val_idx: Index of the value to check in the block instance. ++ * ++ * Return: true if the value was enabled, else false. ++ */ ++static inline bool kbase_hwcnt_enable_map_block_value_enabled( ++ const u64 *bitfld, ++ size_t val_idx) ++{ ++ const size_t idx = val_idx / KBASE_HWCNT_BITFIELD_BITS; ++ const size_t bit = val_idx % KBASE_HWCNT_BITFIELD_BITS; ++ const u64 mask = 1ull << bit; ++ ++ return (bitfld[idx] & mask) != 0; ++} ++ ++/** ++ * kbase_hwcnt_enable_map_block_enable_value() - Enable a value in a block ++ * instance. ++ * @bitfld: Non-NULL pointer to the block bitfield(s) obtained from a call to ++ * kbase_hwcnt_enable_map_block_instance. ++ * @val_idx: Index of the value to enable in the block instance. ++ */ ++static inline void kbase_hwcnt_enable_map_block_enable_value( ++ u64 *bitfld, ++ size_t val_idx) ++{ ++ const size_t idx = val_idx / KBASE_HWCNT_BITFIELD_BITS; ++ const size_t bit = val_idx % KBASE_HWCNT_BITFIELD_BITS; ++ const u64 mask = 1ull << bit; ++ ++ bitfld[idx] |= mask; ++} ++ ++/** ++ * kbase_hwcnt_enable_map_block_disable_value() - Disable a value in a block ++ * instance. ++ * @bitfld: Non-NULL pointer to the block bitfield(s) obtained from a call to ++ * kbase_hwcnt_enable_map_block_instance. ++ * @val_idx: Index of the value to disable in the block instance. ++ */ ++static inline void kbase_hwcnt_enable_map_block_disable_value( ++ u64 *bitfld, ++ size_t val_idx) ++{ ++ const size_t idx = val_idx / KBASE_HWCNT_BITFIELD_BITS; ++ const size_t bit = val_idx % KBASE_HWCNT_BITFIELD_BITS; ++ const u64 mask = 1ull << bit; ++ ++ bitfld[idx] &= ~mask; ++} ++ ++/** ++ * kbase_hwcnt_dump_buffer_alloc() - Allocate a dump buffer. ++ * @metadata: Non-NULL pointer to metadata describing the system. ++ * @dump_buf: Non-NULL pointer to dump buffer to be initialised. Will be ++ * initialised to undefined values, so must be used as a copy dest, ++ * or cleared before use. ++ * ++ * Return: 0 on success, else error code. ++ */ ++int kbase_hwcnt_dump_buffer_alloc( ++ const struct kbase_hwcnt_metadata *metadata, ++ struct kbase_hwcnt_dump_buffer *dump_buf); ++ ++/** ++ * kbase_hwcnt_dump_buffer_free() - Free a dump buffer. ++ * @dump_buf: Dump buffer to be freed. ++ * ++ * Can be safely called on an all-zeroed dump buffer structure, or on an already ++ * freed dump buffer. ++ */ ++void kbase_hwcnt_dump_buffer_free(struct kbase_hwcnt_dump_buffer *dump_buf); ++ ++/** ++ * kbase_hwcnt_dump_buffer_array_alloc() - Allocate an array of dump buffers. ++ * @metadata: Non-NULL pointer to metadata describing the system. ++ * @n: Number of dump buffers to allocate ++ * @dump_bufs: Non-NULL pointer to dump buffer array to be initialised. Each ++ * dump buffer in the array will be initialised to undefined values, ++ * so must be used as a copy dest, or cleared before use. ++ * ++ * A single zeroed contiguous page allocation will be used for all of the ++ * buffers inside the array, where: ++ * dump_bufs[n].dump_buf == page_addr + n * metadata.dump_buf_bytes ++ * ++ * Return: 0 on success, else error code. ++ */ ++int kbase_hwcnt_dump_buffer_array_alloc( ++ const struct kbase_hwcnt_metadata *metadata, ++ size_t n, ++ struct kbase_hwcnt_dump_buffer_array *dump_bufs); ++ ++/** ++ * kbase_hwcnt_dump_buffer_array_free() - Free a dump buffer array. ++ * @dump_bufs: Dump buffer array to be freed. ++ * ++ * Can be safely called on an all-zeroed dump buffer array structure, or on an ++ * already freed dump buffer array. ++ */ ++void kbase_hwcnt_dump_buffer_array_free( ++ struct kbase_hwcnt_dump_buffer_array *dump_bufs); ++ ++/** ++ * kbase_hwcnt_dump_buffer_block_instance() - Get the pointer to a block ++ * instance's dump buffer. ++ * @buf: Non-NULL pointer to (const) dump buffer. ++ * @grp: Index of the group in the metadata. ++ * @blk: Index of the block in the group. ++ * @blk_inst: Index of the block instance in the block. ++ * ++ * Return: (const) u32* to the dump buffer for the block instance. ++ */ ++#define kbase_hwcnt_dump_buffer_block_instance(buf, grp, blk, blk_inst) \ ++ ((buf)->dump_buf + \ ++ (buf)->metadata->grp_metadata[(grp)].dump_buf_index + \ ++ (buf)->metadata->grp_metadata[(grp)].blk_metadata[(blk)].dump_buf_index + \ ++ (buf)->metadata->grp_metadata[(grp)].blk_metadata[(blk)].dump_buf_stride * (blk_inst)) ++ ++/** ++ * kbase_hwcnt_dump_buffer_zero() - Zero all enabled values in dst. ++ * After the operation, all non-enabled values ++ * will be undefined. ++ * @dst: Non-NULL pointer to dump buffer. ++ * @dst_enable_map: Non-NULL pointer to enable map specifying enabled values. ++ * ++ * The dst and dst_enable_map MUST have been created from the same metadata. ++ */ ++void kbase_hwcnt_dump_buffer_zero( ++ struct kbase_hwcnt_dump_buffer *dst, ++ const struct kbase_hwcnt_enable_map *dst_enable_map); ++ ++/** ++ * kbase_hwcnt_dump_buffer_block_zero() - Zero all values in a block. ++ * @dst_blk: Non-NULL pointer to dst block obtained from a call to ++ * kbase_hwcnt_dump_buffer_block_instance. ++ * @val_cnt: Number of values in the block. ++ */ ++static inline void kbase_hwcnt_dump_buffer_block_zero( ++ u32 *dst_blk, ++ size_t val_cnt) ++{ ++ memset(dst_blk, 0, (val_cnt * KBASE_HWCNT_VALUE_BYTES)); ++} ++ ++/** ++ * kbase_hwcnt_dump_buffer_zero_strict() - Zero all values in dst. ++ * After the operation, all values ++ * (including padding bytes) will be ++ * zero. ++ * Slower than the non-strict variant. ++ * @dst: Non-NULL pointer to dump buffer. ++ */ ++void kbase_hwcnt_dump_buffer_zero_strict( ++ struct kbase_hwcnt_dump_buffer *dst); ++ ++/** ++ * kbase_hwcnt_dump_buffer_zero_non_enabled() - Zero all non-enabled values in ++ * dst (including padding bytes and ++ * unavailable blocks). ++ * After the operation, all enabled ++ * values will be unchanged. ++ * @dst: Non-NULL pointer to dump buffer. ++ * @dst_enable_map: Non-NULL pointer to enable map specifying enabled values. ++ * ++ * The dst and dst_enable_map MUST have been created from the same metadata. ++ */ ++void kbase_hwcnt_dump_buffer_zero_non_enabled( ++ struct kbase_hwcnt_dump_buffer *dst, ++ const struct kbase_hwcnt_enable_map *dst_enable_map); ++ ++/** ++ * kbase_hwcnt_dump_buffer_block_zero_non_enabled() - Zero all non-enabled ++ * values in a block. ++ * After the operation, all ++ * enabled values will be ++ * unchanged. ++ * @dst_blk: Non-NULL pointer to dst block obtained from a call to ++ * kbase_hwcnt_dump_buffer_block_instance. ++ * @blk_em: Non-NULL pointer to the block bitfield(s) obtained from a call to ++ * kbase_hwcnt_enable_map_block_instance. ++ * @val_cnt: Number of values in the block. ++ */ ++static inline void kbase_hwcnt_dump_buffer_block_zero_non_enabled( ++ u32 *dst_blk, ++ const u64 *blk_em, ++ size_t val_cnt) ++{ ++ size_t val; ++ ++ for (val = 0; val < val_cnt; val++) { ++ if (!kbase_hwcnt_enable_map_block_value_enabled(blk_em, val)) ++ dst_blk[val] = 0; ++ } ++} ++ ++/** ++ * kbase_hwcnt_dump_buffer_copy() - Copy all enabled values from src to dst. ++ * After the operation, all non-enabled values ++ * will be undefined. ++ * @dst: Non-NULL pointer to dst dump buffer. ++ * @src: Non-NULL pointer to src dump buffer. ++ * @dst_enable_map: Non-NULL pointer to enable map specifying enabled values. ++ * ++ * The dst, src, and dst_enable_map MUST have been created from the same ++ * metadata. ++ */ ++void kbase_hwcnt_dump_buffer_copy( ++ struct kbase_hwcnt_dump_buffer *dst, ++ const struct kbase_hwcnt_dump_buffer *src, ++ const struct kbase_hwcnt_enable_map *dst_enable_map); ++ ++/** ++ * kbase_hwcnt_dump_buffer_block_copy() - Copy all block values from src to dst. ++ * @dst_blk: Non-NULL pointer to dst block obtained from a call to ++ * kbase_hwcnt_dump_buffer_block_instance. ++ * @src_blk: Non-NULL pointer to src block obtained from a call to ++ * kbase_hwcnt_dump_buffer_block_instance. ++ * @val_cnt: Number of values in the block. ++ */ ++static inline void kbase_hwcnt_dump_buffer_block_copy( ++ u32 *dst_blk, ++ const u32 *src_blk, ++ size_t val_cnt) ++{ ++ /* Copy all the counters in the block instance. ++ * Values of non-enabled counters are undefined. ++ */ ++ memcpy(dst_blk, src_blk, (val_cnt * KBASE_HWCNT_VALUE_BYTES)); ++} ++ ++/** ++ * kbase_hwcnt_dump_buffer_copy_strict() - Copy all enabled values from src to ++ * dst. ++ * After the operation, all non-enabled ++ * values (including padding bytes) will ++ * be zero. ++ * Slower than the non-strict variant. ++ * @dst: Non-NULL pointer to dst dump buffer. ++ * @src: Non-NULL pointer to src dump buffer. ++ * @dst_enable_map: Non-NULL pointer to enable map specifying enabled values. ++ * ++ * The dst, src, and dst_enable_map MUST have been created from the same ++ * metadata. ++ */ ++void kbase_hwcnt_dump_buffer_copy_strict( ++ struct kbase_hwcnt_dump_buffer *dst, ++ const struct kbase_hwcnt_dump_buffer *src, ++ const struct kbase_hwcnt_enable_map *dst_enable_map); ++ ++/** ++ * kbase_hwcnt_dump_buffer_block_copy_strict() - Copy all enabled block values ++ * from src to dst. ++ * After the operation, all ++ * non-enabled values will be ++ * zero. ++ * @dst_blk: Non-NULL pointer to dst block obtained from a call to ++ * kbase_hwcnt_dump_buffer_block_instance. ++ * @src_blk: Non-NULL pointer to src block obtained from a call to ++ * kbase_hwcnt_dump_buffer_block_instance. ++ * @blk_em: Non-NULL pointer to the block bitfield(s) obtained from a call to ++ * kbase_hwcnt_enable_map_block_instance. ++ * @val_cnt: Number of values in the block. ++ * ++ * After the copy, any disabled values in dst will be zero. ++ */ ++static inline void kbase_hwcnt_dump_buffer_block_copy_strict( ++ u32 *dst_blk, ++ const u32 *src_blk, ++ const u64 *blk_em, ++ size_t val_cnt) ++{ ++ size_t val; ++ ++ for (val = 0; val < val_cnt; val++) { ++ bool val_enabled = kbase_hwcnt_enable_map_block_value_enabled( ++ blk_em, val); ++ ++ dst_blk[val] = val_enabled ? src_blk[val] : 0; ++ } ++} ++ ++/** ++ * kbase_hwcnt_dump_buffer_accumulate() - Copy all enabled headers and ++ * accumulate all enabled counters from ++ * src to dst. ++ * After the operation, all non-enabled ++ * values will be undefined. ++ * @dst: Non-NULL pointer to dst dump buffer. ++ * @src: Non-NULL pointer to src dump buffer. ++ * @dst_enable_map: Non-NULL pointer to enable map specifying enabled values. ++ * ++ * The dst, src, and dst_enable_map MUST have been created from the same ++ * metadata. ++ */ ++void kbase_hwcnt_dump_buffer_accumulate( ++ struct kbase_hwcnt_dump_buffer *dst, ++ const struct kbase_hwcnt_dump_buffer *src, ++ const struct kbase_hwcnt_enable_map *dst_enable_map); ++ ++/** ++ * kbase_hwcnt_dump_buffer_block_accumulate() - Copy all block headers and ++ * accumulate all block counters ++ * from src to dst. ++ * @dst_blk: Non-NULL pointer to dst block obtained from a call to ++ * kbase_hwcnt_dump_buffer_block_instance. ++ * @src_blk: Non-NULL pointer to src block obtained from a call to ++ * kbase_hwcnt_dump_buffer_block_instance. ++ * @hdr_cnt: Number of headers in the block. ++ * @ctr_cnt: Number of counters in the block. ++ */ ++static inline void kbase_hwcnt_dump_buffer_block_accumulate( ++ u32 *dst_blk, ++ const u32 *src_blk, ++ size_t hdr_cnt, ++ size_t ctr_cnt) ++{ ++ size_t ctr; ++ /* Copy all the headers in the block instance. ++ * Values of non-enabled headers are undefined. ++ */ ++ memcpy(dst_blk, src_blk, hdr_cnt * KBASE_HWCNT_VALUE_BYTES); ++ ++ /* Accumulate all the counters in the block instance. ++ * Values of non-enabled counters are undefined. ++ */ ++ for (ctr = hdr_cnt; ctr < ctr_cnt + hdr_cnt; ctr++) { ++ u32 *dst_ctr = dst_blk + ctr; ++ const u32 *src_ctr = src_blk + ctr; ++ ++ const u32 src_counter = *src_ctr; ++ const u32 dst_counter = *dst_ctr; ++ ++ /* Saturating add */ ++ u32 accumulated = src_counter + dst_counter; ++ ++ if (accumulated < src_counter) ++ accumulated = U32_MAX; ++ ++ *dst_ctr = accumulated; ++ } ++} ++ ++/** ++ * kbase_hwcnt_dump_buffer_accumulate_strict() - Copy all enabled headers and ++ * accumulate all enabled counters ++ * from src to dst. ++ * After the operation, all ++ * non-enabled values (including ++ * padding bytes) will be zero. ++ * Slower than the non-strict ++ * variant. ++ * @dst: Non-NULL pointer to dst dump buffer. ++ * @src: Non-NULL pointer to src dump buffer. ++ * @dst_enable_map: Non-NULL pointer to enable map specifying enabled values. ++ * ++ * The dst, src, and dst_enable_map MUST have been created from the same ++ * metadata. ++ */ ++void kbase_hwcnt_dump_buffer_accumulate_strict( ++ struct kbase_hwcnt_dump_buffer *dst, ++ const struct kbase_hwcnt_dump_buffer *src, ++ const struct kbase_hwcnt_enable_map *dst_enable_map); ++ ++/** ++ * kbase_hwcnt_dump_buffer_block_accumulate_strict() - Copy all enabled block ++ * headers and accumulate ++ * all block counters from ++ * src to dst. ++ * After the operation, all ++ * non-enabled values will ++ * be zero. ++ * @dst_blk: Non-NULL pointer to dst block obtained from a call to ++ * kbase_hwcnt_dump_buffer_block_instance. ++ * @src_blk: Non-NULL pointer to src block obtained from a call to ++ * kbase_hwcnt_dump_buffer_block_instance. ++ * @blk_em: Non-NULL pointer to the block bitfield(s) obtained from a call to ++ * kbase_hwcnt_enable_map_block_instance. ++ * @hdr_cnt: Number of headers in the block. ++ * @ctr_cnt: Number of counters in the block. ++ */ ++static inline void kbase_hwcnt_dump_buffer_block_accumulate_strict( ++ u32 *dst_blk, ++ const u32 *src_blk, ++ const u64 *blk_em, ++ size_t hdr_cnt, ++ size_t ctr_cnt) ++{ ++ size_t ctr; ++ ++ kbase_hwcnt_dump_buffer_block_copy_strict( ++ dst_blk, src_blk, blk_em, hdr_cnt); ++ ++ for (ctr = hdr_cnt; ctr < ctr_cnt + hdr_cnt; ctr++) { ++ bool ctr_enabled = kbase_hwcnt_enable_map_block_value_enabled( ++ blk_em, ctr); ++ ++ u32 *dst_ctr = dst_blk + ctr; ++ const u32 *src_ctr = src_blk + ctr; ++ ++ const u32 src_counter = *src_ctr; ++ const u32 dst_counter = *dst_ctr; ++ ++ /* Saturating add */ ++ u32 accumulated = src_counter + dst_counter; ++ ++ if (accumulated < src_counter) ++ accumulated = U32_MAX; ++ ++ *dst_ctr = ctr_enabled ? accumulated : 0; ++ } ++} ++ ++/** ++ * @brief Iterate over each clock domain in the metadata. ++ * ++ * @param[in] md Non-NULL pointer to metadata. ++ * @param[in] clk size_t variable used as clock iterator. ++ */ ++#define kbase_hwcnt_metadata_for_each_clock(md, clk) \ ++ for ((clk) = 0; (clk) < (md)->clk_cnt; (clk)++) ++ ++/** ++ * kbase_hwcnt_clk_enable_map_enabled() - Check if the given index is enabled ++ * in clk_enable_map. ++ * @clk_enable_map: An enable map for clock domains. ++ * @index: Index of the enable map for clock domain. ++ * ++ * Return: true if the index of the clock domain is enabled, else false. ++ */ ++static inline bool kbase_hwcnt_clk_enable_map_enabled( ++ const u64 clk_enable_map, const size_t index) ++{ ++ if (clk_enable_map & (1ull << index)) ++ return true; ++ return false; ++} ++ ++#endif /* _KBASE_HWCNT_TYPES_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_virtualizer.c b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_virtualizer.c +new file mode 100755 +index 000000000000..917e47cda0f9 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_virtualizer.c +@@ -0,0 +1,790 @@ ++/* ++ * ++ * (C) COPYRIGHT 2018 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include "mali_kbase_hwcnt_virtualizer.h" ++#include "mali_kbase_hwcnt_accumulator.h" ++#include "mali_kbase_hwcnt_context.h" ++#include "mali_kbase_hwcnt_types.h" ++#include "mali_malisw.h" ++#include "mali_kbase_debug.h" ++#include "mali_kbase_linux.h" ++ ++#include ++#include ++ ++/** ++ * struct kbase_hwcnt_virtualizer - Hardware counter virtualizer structure. ++ * @hctx: Hardware counter context being virtualized. ++ * @dump_threshold_ns: Minimum threshold period for dumps between different ++ * clients where a new accumulator dump will not be ++ * performed, and instead accumulated values will be used. ++ * If 0, rate limiting is disabled. ++ * @metadata: Hardware counter metadata. ++ * @lock: Lock acquired at all entrypoints, to protect mutable ++ * state. ++ * @client_count: Current number of virtualizer clients. ++ * @clients: List of virtualizer clients. ++ * @accum: Hardware counter accumulator. NULL if no clients. ++ * @scratch_map: Enable map used as scratch space during counter changes. ++ * @scratch_buf: Dump buffer used as scratch space during dumps. ++ * @ts_last_dump_ns: End time of most recent dump across all clients. ++ */ ++struct kbase_hwcnt_virtualizer { ++ struct kbase_hwcnt_context *hctx; ++ u64 dump_threshold_ns; ++ const struct kbase_hwcnt_metadata *metadata; ++ struct mutex lock; ++ size_t client_count; ++ struct list_head clients; ++ struct kbase_hwcnt_accumulator *accum; ++ struct kbase_hwcnt_enable_map scratch_map; ++ struct kbase_hwcnt_dump_buffer scratch_buf; ++ u64 ts_last_dump_ns; ++}; ++ ++/** ++ * struct kbase_hwcnt_virtualizer_client - Virtualizer client structure. ++ * @node: List node used for virtualizer client list. ++ * @hvirt: Hardware counter virtualizer. ++ * @enable_map: Enable map with client's current enabled counters. ++ * @accum_buf: Dump buffer with client's current accumulated counters. ++ * @has_accum: True if accum_buf contains any accumulated counters. ++ * @ts_start_ns: Counter collection start time of current dump. ++ */ ++struct kbase_hwcnt_virtualizer_client { ++ struct list_head node; ++ struct kbase_hwcnt_virtualizer *hvirt; ++ struct kbase_hwcnt_enable_map enable_map; ++ struct kbase_hwcnt_dump_buffer accum_buf; ++ bool has_accum; ++ u64 ts_start_ns; ++}; ++ ++const struct kbase_hwcnt_metadata *kbase_hwcnt_virtualizer_metadata( ++ struct kbase_hwcnt_virtualizer *hvirt) ++{ ++ if (!hvirt) ++ return NULL; ++ ++ return hvirt->metadata; ++} ++KBASE_EXPORT_TEST_API(kbase_hwcnt_virtualizer_metadata); ++ ++/** ++ * kbasep_hwcnt_virtualizer_client_free - Free a virtualizer client's memory. ++ * @hvcli: Pointer to virtualizer client. ++ * ++ * Will safely free a client in any partial state of construction. ++ */ ++static void kbasep_hwcnt_virtualizer_client_free( ++ struct kbase_hwcnt_virtualizer_client *hvcli) ++{ ++ if (!hvcli) ++ return; ++ ++ kbase_hwcnt_dump_buffer_free(&hvcli->accum_buf); ++ kbase_hwcnt_enable_map_free(&hvcli->enable_map); ++ kfree(hvcli); ++} ++ ++/** ++ * kbasep_hwcnt_virtualizer_client_alloc - Allocate memory for a virtualizer ++ * client. ++ * @metadata: Non-NULL pointer to counter metadata. ++ * @out_hvcli: Non-NULL pointer to where created client will be stored on ++ * success. ++ * ++ * Return: 0 on success, else error code. ++ */ ++static int kbasep_hwcnt_virtualizer_client_alloc( ++ const struct kbase_hwcnt_metadata *metadata, ++ struct kbase_hwcnt_virtualizer_client **out_hvcli) ++{ ++ int errcode; ++ struct kbase_hwcnt_virtualizer_client *hvcli = NULL; ++ ++ WARN_ON(!metadata); ++ WARN_ON(!out_hvcli); ++ ++ hvcli = kzalloc(sizeof(*hvcli), GFP_KERNEL); ++ if (!hvcli) ++ return -ENOMEM; ++ ++ errcode = kbase_hwcnt_enable_map_alloc(metadata, &hvcli->enable_map); ++ if (errcode) ++ goto error; ++ ++ errcode = kbase_hwcnt_dump_buffer_alloc(metadata, &hvcli->accum_buf); ++ if (errcode) ++ goto error; ++ ++ *out_hvcli = hvcli; ++ return 0; ++error: ++ kbasep_hwcnt_virtualizer_client_free(hvcli); ++ return errcode; ++} ++ ++/** ++ * kbasep_hwcnt_virtualizer_client_accumulate - Accumulate a dump buffer into a ++ * client's accumulation buffer. ++ * @hvcli: Non-NULL pointer to virtualizer client. ++ * @dump_buf: Non-NULL pointer to dump buffer to accumulate from. ++ */ ++static void kbasep_hwcnt_virtualizer_client_accumulate( ++ struct kbase_hwcnt_virtualizer_client *hvcli, ++ const struct kbase_hwcnt_dump_buffer *dump_buf) ++{ ++ WARN_ON(!hvcli); ++ WARN_ON(!dump_buf); ++ lockdep_assert_held(&hvcli->hvirt->lock); ++ ++ if (hvcli->has_accum) { ++ /* If already some accumulation, accumulate */ ++ kbase_hwcnt_dump_buffer_accumulate( ++ &hvcli->accum_buf, dump_buf, &hvcli->enable_map); ++ } else { ++ /* If no accumulation, copy */ ++ kbase_hwcnt_dump_buffer_copy( ++ &hvcli->accum_buf, dump_buf, &hvcli->enable_map); ++ } ++ hvcli->has_accum = true; ++} ++ ++/** ++ * kbasep_hwcnt_virtualizer_accumulator_term - Terminate the hardware counter ++ * accumulator after final client ++ * removal. ++ * @hvirt: Non-NULL pointer to the hardware counter virtualizer. ++ * ++ * Will safely terminate the accumulator in any partial state of initialisation. ++ */ ++static void kbasep_hwcnt_virtualizer_accumulator_term( ++ struct kbase_hwcnt_virtualizer *hvirt) ++{ ++ WARN_ON(!hvirt); ++ lockdep_assert_held(&hvirt->lock); ++ WARN_ON(hvirt->client_count); ++ ++ kbase_hwcnt_dump_buffer_free(&hvirt->scratch_buf); ++ kbase_hwcnt_enable_map_free(&hvirt->scratch_map); ++ kbase_hwcnt_accumulator_release(hvirt->accum); ++ hvirt->accum = NULL; ++} ++ ++/** ++ * kbasep_hwcnt_virtualizer_accumulator_init - Initialise the hardware counter ++ * accumulator before first client ++ * addition. ++ * @hvirt: Non-NULL pointer to the hardware counter virtualizer. ++ * ++ * Return: 0 on success, else error code. ++ */ ++static int kbasep_hwcnt_virtualizer_accumulator_init( ++ struct kbase_hwcnt_virtualizer *hvirt) ++{ ++ int errcode; ++ ++ WARN_ON(!hvirt); ++ lockdep_assert_held(&hvirt->lock); ++ WARN_ON(hvirt->client_count); ++ WARN_ON(hvirt->accum); ++ ++ errcode = kbase_hwcnt_accumulator_acquire( ++ hvirt->hctx, &hvirt->accum); ++ if (errcode) ++ goto error; ++ ++ errcode = kbase_hwcnt_enable_map_alloc( ++ hvirt->metadata, &hvirt->scratch_map); ++ if (errcode) ++ goto error; ++ ++ errcode = kbase_hwcnt_dump_buffer_alloc( ++ hvirt->metadata, &hvirt->scratch_buf); ++ if (errcode) ++ goto error; ++ ++ return 0; ++error: ++ kbasep_hwcnt_virtualizer_accumulator_term(hvirt); ++ return errcode; ++} ++ ++/** ++ * kbasep_hwcnt_virtualizer_client_add - Add a newly allocated client to the ++ * virtualizer. ++ * @hvirt: Non-NULL pointer to the hardware counter virtualizer. ++ * @hvcli: Non-NULL pointer to the virtualizer client to add. ++ * @enable_map: Non-NULL pointer to client's initial enable map. ++ * ++ * Return: 0 on success, else error code. ++ */ ++static int kbasep_hwcnt_virtualizer_client_add( ++ struct kbase_hwcnt_virtualizer *hvirt, ++ struct kbase_hwcnt_virtualizer_client *hvcli, ++ const struct kbase_hwcnt_enable_map *enable_map) ++{ ++ int errcode = 0; ++ u64 ts_start_ns; ++ u64 ts_end_ns; ++ ++ WARN_ON(!hvirt); ++ WARN_ON(!hvcli); ++ WARN_ON(!enable_map); ++ lockdep_assert_held(&hvirt->lock); ++ ++ if (hvirt->client_count == 0) ++ /* First client added, so initialise the accumulator */ ++ errcode = kbasep_hwcnt_virtualizer_accumulator_init(hvirt); ++ if (errcode) ++ return errcode; ++ ++ hvirt->client_count += 1; ++ ++ if (hvirt->client_count == 1) { ++ /* First client, so just pass the enable map onwards as is */ ++ errcode = kbase_hwcnt_accumulator_set_counters(hvirt->accum, ++ enable_map, &ts_start_ns, &ts_end_ns, NULL); ++ } else { ++ struct kbase_hwcnt_virtualizer_client *pos; ++ ++ /* Make the scratch enable map the union of all enable maps */ ++ kbase_hwcnt_enable_map_copy( ++ &hvirt->scratch_map, enable_map); ++ list_for_each_entry(pos, &hvirt->clients, node) ++ kbase_hwcnt_enable_map_union( ++ &hvirt->scratch_map, &pos->enable_map); ++ ++ /* Set the counters with the new union enable map */ ++ errcode = kbase_hwcnt_accumulator_set_counters(hvirt->accum, ++ &hvirt->scratch_map, ++ &ts_start_ns, &ts_end_ns, ++ &hvirt->scratch_buf); ++ /* Accumulate into only existing clients' accumulation bufs */ ++ if (!errcode) ++ list_for_each_entry(pos, &hvirt->clients, node) ++ kbasep_hwcnt_virtualizer_client_accumulate( ++ pos, &hvirt->scratch_buf); ++ } ++ if (errcode) ++ goto error; ++ ++ list_add(&hvcli->node, &hvirt->clients); ++ hvcli->hvirt = hvirt; ++ kbase_hwcnt_enable_map_copy(&hvcli->enable_map, enable_map); ++ hvcli->has_accum = false; ++ hvcli->ts_start_ns = ts_end_ns; ++ ++ /* Store the most recent dump time for rate limiting */ ++ hvirt->ts_last_dump_ns = ts_end_ns; ++ ++ return 0; ++error: ++ hvirt->client_count -= 1; ++ if (hvirt->client_count == 0) ++ kbasep_hwcnt_virtualizer_accumulator_term(hvirt); ++ return errcode; ++} ++ ++/** ++ * kbasep_hwcnt_virtualizer_client_remove - Remove a client from the ++ * virtualizer. ++ * @hvirt: Non-NULL pointer to the hardware counter virtualizer. ++ * @hvcli: Non-NULL pointer to the virtualizer client to remove. ++ */ ++static void kbasep_hwcnt_virtualizer_client_remove( ++ struct kbase_hwcnt_virtualizer *hvirt, ++ struct kbase_hwcnt_virtualizer_client *hvcli) ++{ ++ int errcode = 0; ++ u64 ts_start_ns; ++ u64 ts_end_ns; ++ ++ WARN_ON(!hvirt); ++ WARN_ON(!hvcli); ++ lockdep_assert_held(&hvirt->lock); ++ ++ list_del(&hvcli->node); ++ hvirt->client_count -= 1; ++ ++ if (hvirt->client_count == 0) { ++ /* Last client removed, so terminate the accumulator */ ++ kbasep_hwcnt_virtualizer_accumulator_term(hvirt); ++ } else { ++ struct kbase_hwcnt_virtualizer_client *pos; ++ /* Make the scratch enable map the union of all enable maps */ ++ kbase_hwcnt_enable_map_disable_all(&hvirt->scratch_map); ++ list_for_each_entry(pos, &hvirt->clients, node) ++ kbase_hwcnt_enable_map_union( ++ &hvirt->scratch_map, &pos->enable_map); ++ /* Set the counters with the new union enable map */ ++ errcode = kbase_hwcnt_accumulator_set_counters(hvirt->accum, ++ &hvirt->scratch_map, ++ &ts_start_ns, &ts_end_ns, ++ &hvirt->scratch_buf); ++ /* Accumulate into remaining clients' accumulation bufs */ ++ if (!errcode) ++ list_for_each_entry(pos, &hvirt->clients, node) ++ kbasep_hwcnt_virtualizer_client_accumulate( ++ pos, &hvirt->scratch_buf); ++ ++ /* Store the most recent dump time for rate limiting */ ++ hvirt->ts_last_dump_ns = ts_end_ns; ++ } ++ WARN_ON(errcode); ++} ++ ++/** ++ * kbasep_hwcnt_virtualizer_client_set_counters - Perform a dump of the client's ++ * currently enabled counters, ++ * and enable a new set of ++ * counters that will be used for ++ * subsequent dumps. ++ * @hvirt: Non-NULL pointer to the hardware counter virtualizer. ++ * @hvcli: Non-NULL pointer to the virtualizer client. ++ * @enable_map: Non-NULL pointer to the new counter enable map for the client. ++ * Must have the same metadata as the virtualizer. ++ * @ts_start_ns: Non-NULL pointer where the start timestamp of the dump will ++ * be written out to on success. ++ * @ts_end_ns: Non-NULL pointer where the end timestamp of the dump will ++ * be written out to on success. ++ * @dump_buf: Pointer to the buffer where the dump will be written out to on ++ * success. If non-NULL, must have the same metadata as the ++ * accumulator. If NULL, the dump will be discarded. ++ * ++ * Return: 0 on success or error code. ++ */ ++static int kbasep_hwcnt_virtualizer_client_set_counters( ++ struct kbase_hwcnt_virtualizer *hvirt, ++ struct kbase_hwcnt_virtualizer_client *hvcli, ++ const struct kbase_hwcnt_enable_map *enable_map, ++ u64 *ts_start_ns, ++ u64 *ts_end_ns, ++ struct kbase_hwcnt_dump_buffer *dump_buf) ++{ ++ int errcode; ++ struct kbase_hwcnt_virtualizer_client *pos; ++ ++ WARN_ON(!hvirt); ++ WARN_ON(!hvcli); ++ WARN_ON(!enable_map); ++ WARN_ON(!ts_start_ns); ++ WARN_ON(!ts_end_ns); ++ WARN_ON(enable_map->metadata != hvirt->metadata); ++ WARN_ON(dump_buf && (dump_buf->metadata != hvirt->metadata)); ++ lockdep_assert_held(&hvirt->lock); ++ ++ /* Make the scratch enable map the union of all enable maps */ ++ kbase_hwcnt_enable_map_copy(&hvirt->scratch_map, enable_map); ++ list_for_each_entry(pos, &hvirt->clients, node) ++ /* Ignore the enable map of the selected client */ ++ if (pos != hvcli) ++ kbase_hwcnt_enable_map_union( ++ &hvirt->scratch_map, &pos->enable_map); ++ ++ /* Set the counters with the new union enable map */ ++ errcode = kbase_hwcnt_accumulator_set_counters(hvirt->accum, ++ &hvirt->scratch_map, ts_start_ns, ts_end_ns, ++ &hvirt->scratch_buf); ++ if (errcode) ++ return errcode; ++ ++ /* Accumulate into all accumulation bufs except the selected client's */ ++ list_for_each_entry(pos, &hvirt->clients, node) ++ if (pos != hvcli) ++ kbasep_hwcnt_virtualizer_client_accumulate( ++ pos, &hvirt->scratch_buf); ++ ++ /* Finally, write into the dump buf */ ++ if (dump_buf) { ++ const struct kbase_hwcnt_dump_buffer *src = &hvirt->scratch_buf; ++ ++ if (hvcli->has_accum) { ++ kbase_hwcnt_dump_buffer_accumulate( ++ &hvcli->accum_buf, src, &hvcli->enable_map); ++ src = &hvcli->accum_buf; ++ } ++ kbase_hwcnt_dump_buffer_copy(dump_buf, src, &hvcli->enable_map); ++ } ++ hvcli->has_accum = false; ++ ++ /* Update the selected client's enable map */ ++ kbase_hwcnt_enable_map_copy(&hvcli->enable_map, enable_map); ++ ++ /* Fix up the timestamps */ ++ *ts_start_ns = hvcli->ts_start_ns; ++ hvcli->ts_start_ns = *ts_end_ns; ++ ++ /* Store the most recent dump time for rate limiting */ ++ hvirt->ts_last_dump_ns = *ts_end_ns; ++ ++ return errcode; ++} ++ ++int kbase_hwcnt_virtualizer_client_set_counters( ++ struct kbase_hwcnt_virtualizer_client *hvcli, ++ const struct kbase_hwcnt_enable_map *enable_map, ++ u64 *ts_start_ns, ++ u64 *ts_end_ns, ++ struct kbase_hwcnt_dump_buffer *dump_buf) ++{ ++ int errcode; ++ struct kbase_hwcnt_virtualizer *hvirt; ++ ++ if (!hvcli || !enable_map || !ts_start_ns || !ts_end_ns) ++ return -EINVAL; ++ ++ hvirt = hvcli->hvirt; ++ ++ if ((enable_map->metadata != hvirt->metadata) || ++ (dump_buf && (dump_buf->metadata != hvirt->metadata))) ++ return -EINVAL; ++ ++ mutex_lock(&hvirt->lock); ++ ++ if ((hvirt->client_count == 1) && (!hvcli->has_accum)) { ++ /* ++ * If there's only one client with no prior accumulation, we can ++ * completely skip the virtualize and just pass through the call ++ * to the accumulator, saving a fair few copies and ++ * accumulations. ++ */ ++ errcode = kbase_hwcnt_accumulator_set_counters( ++ hvirt->accum, enable_map, ++ ts_start_ns, ts_end_ns, dump_buf); ++ ++ if (!errcode) { ++ /* Update the selected client's enable map */ ++ kbase_hwcnt_enable_map_copy( ++ &hvcli->enable_map, enable_map); ++ ++ /* Fix up the timestamps */ ++ *ts_start_ns = hvcli->ts_start_ns; ++ hvcli->ts_start_ns = *ts_end_ns; ++ ++ /* Store the most recent dump time for rate limiting */ ++ hvirt->ts_last_dump_ns = *ts_end_ns; ++ } ++ } else { ++ /* Otherwise, do the full virtualize */ ++ errcode = kbasep_hwcnt_virtualizer_client_set_counters( ++ hvirt, hvcli, enable_map, ++ ts_start_ns, ts_end_ns, dump_buf); ++ } ++ ++ mutex_unlock(&hvirt->lock); ++ ++ return errcode; ++} ++KBASE_EXPORT_TEST_API(kbase_hwcnt_virtualizer_client_set_counters); ++ ++/** ++ * kbasep_hwcnt_virtualizer_client_dump - Perform a dump of the client's ++ * currently enabled counters. ++ * @hvirt: Non-NULL pointer to the hardware counter virtualizer. ++ * @hvcli: Non-NULL pointer to the virtualizer client. ++ * @ts_start_ns: Non-NULL pointer where the start timestamp of the dump will ++ * be written out to on success. ++ * @ts_end_ns: Non-NULL pointer where the end timestamp of the dump will ++ * be written out to on success. ++ * @dump_buf: Pointer to the buffer where the dump will be written out to on ++ * success. If non-NULL, must have the same metadata as the ++ * accumulator. If NULL, the dump will be discarded. ++ * ++ * Return: 0 on success or error code. ++ */ ++static int kbasep_hwcnt_virtualizer_client_dump( ++ struct kbase_hwcnt_virtualizer *hvirt, ++ struct kbase_hwcnt_virtualizer_client *hvcli, ++ u64 *ts_start_ns, ++ u64 *ts_end_ns, ++ struct kbase_hwcnt_dump_buffer *dump_buf) ++{ ++ int errcode; ++ struct kbase_hwcnt_virtualizer_client *pos; ++ ++ WARN_ON(!hvirt); ++ WARN_ON(!hvcli); ++ WARN_ON(!ts_start_ns); ++ WARN_ON(!ts_end_ns); ++ WARN_ON(dump_buf && (dump_buf->metadata != hvirt->metadata)); ++ lockdep_assert_held(&hvirt->lock); ++ ++ /* Perform the dump */ ++ errcode = kbase_hwcnt_accumulator_dump(hvirt->accum, ++ ts_start_ns, ts_end_ns, &hvirt->scratch_buf); ++ if (errcode) ++ return errcode; ++ ++ /* Accumulate into all accumulation bufs except the selected client's */ ++ list_for_each_entry(pos, &hvirt->clients, node) ++ if (pos != hvcli) ++ kbasep_hwcnt_virtualizer_client_accumulate( ++ pos, &hvirt->scratch_buf); ++ ++ /* Finally, write into the dump buf */ ++ if (dump_buf) { ++ const struct kbase_hwcnt_dump_buffer *src = &hvirt->scratch_buf; ++ ++ if (hvcli->has_accum) { ++ kbase_hwcnt_dump_buffer_accumulate( ++ &hvcli->accum_buf, src, &hvcli->enable_map); ++ src = &hvcli->accum_buf; ++ } ++ kbase_hwcnt_dump_buffer_copy(dump_buf, src, &hvcli->enable_map); ++ } ++ hvcli->has_accum = false; ++ ++ /* Fix up the timestamps */ ++ *ts_start_ns = hvcli->ts_start_ns; ++ hvcli->ts_start_ns = *ts_end_ns; ++ ++ /* Store the most recent dump time for rate limiting */ ++ hvirt->ts_last_dump_ns = *ts_end_ns; ++ ++ return errcode; ++} ++ ++/** ++ * kbasep_hwcnt_virtualizer_client_dump_rate_limited - Perform a dump of the ++ * client's currently enabled counters ++ * if it hasn't been rate limited, ++ * otherwise return the client's most ++ * recent accumulation. ++ * @hvirt: Non-NULL pointer to the hardware counter virtualizer. ++ * @hvcli: Non-NULL pointer to the virtualizer client. ++ * @ts_start_ns: Non-NULL pointer where the start timestamp of the dump will ++ * be written out to on success. ++ * @ts_end_ns: Non-NULL pointer where the end timestamp of the dump will ++ * be written out to on success. ++ * @dump_buf: Pointer to the buffer where the dump will be written out to on ++ * success. If non-NULL, must have the same metadata as the ++ * accumulator. If NULL, the dump will be discarded. ++ * ++ * Return: 0 on success or error code. ++ */ ++static int kbasep_hwcnt_virtualizer_client_dump_rate_limited( ++ struct kbase_hwcnt_virtualizer *hvirt, ++ struct kbase_hwcnt_virtualizer_client *hvcli, ++ u64 *ts_start_ns, ++ u64 *ts_end_ns, ++ struct kbase_hwcnt_dump_buffer *dump_buf) ++{ ++ bool rate_limited = true; ++ ++ WARN_ON(!hvirt); ++ WARN_ON(!hvcli); ++ WARN_ON(!ts_start_ns); ++ WARN_ON(!ts_end_ns); ++ WARN_ON(dump_buf && (dump_buf->metadata != hvirt->metadata)); ++ lockdep_assert_held(&hvirt->lock); ++ ++ if (hvirt->dump_threshold_ns == 0) { ++ /* Threshold == 0, so rate limiting disabled */ ++ rate_limited = false; ++ } else if (hvirt->ts_last_dump_ns == hvcli->ts_start_ns) { ++ /* Last dump was performed by this client, and dumps from an ++ * individual client are never rate limited ++ */ ++ rate_limited = false; ++ } else { ++ const u64 ts_ns = ++ kbase_hwcnt_accumulator_timestamp_ns(hvirt->accum); ++ const u64 time_since_last_dump_ns = ++ ts_ns - hvirt->ts_last_dump_ns; ++ ++ /* Dump period equals or exceeds the threshold */ ++ if (time_since_last_dump_ns >= hvirt->dump_threshold_ns) ++ rate_limited = false; ++ } ++ ++ if (!rate_limited) ++ return kbasep_hwcnt_virtualizer_client_dump( ++ hvirt, hvcli, ts_start_ns, ts_end_ns, dump_buf); ++ ++ /* If we've gotten this far, the client must have something accumulated ++ * otherwise it is a logic error ++ */ ++ WARN_ON(!hvcli->has_accum); ++ ++ if (dump_buf) ++ kbase_hwcnt_dump_buffer_copy( ++ dump_buf, &hvcli->accum_buf, &hvcli->enable_map); ++ hvcli->has_accum = false; ++ ++ *ts_start_ns = hvcli->ts_start_ns; ++ *ts_end_ns = hvirt->ts_last_dump_ns; ++ hvcli->ts_start_ns = hvirt->ts_last_dump_ns; ++ ++ return 0; ++} ++ ++int kbase_hwcnt_virtualizer_client_dump( ++ struct kbase_hwcnt_virtualizer_client *hvcli, ++ u64 *ts_start_ns, ++ u64 *ts_end_ns, ++ struct kbase_hwcnt_dump_buffer *dump_buf) ++{ ++ int errcode; ++ struct kbase_hwcnt_virtualizer *hvirt; ++ ++ if (!hvcli || !ts_start_ns || !ts_end_ns) ++ return -EINVAL; ++ ++ hvirt = hvcli->hvirt; ++ ++ if (dump_buf && (dump_buf->metadata != hvirt->metadata)) ++ return -EINVAL; ++ ++ mutex_lock(&hvirt->lock); ++ ++ if ((hvirt->client_count == 1) && (!hvcli->has_accum)) { ++ /* ++ * If there's only one client with no prior accumulation, we can ++ * completely skip the virtualize and just pass through the call ++ * to the accumulator, saving a fair few copies and ++ * accumulations. ++ */ ++ errcode = kbase_hwcnt_accumulator_dump( ++ hvirt->accum, ts_start_ns, ts_end_ns, dump_buf); ++ ++ if (!errcode) { ++ /* Fix up the timestamps */ ++ *ts_start_ns = hvcli->ts_start_ns; ++ hvcli->ts_start_ns = *ts_end_ns; ++ ++ /* Store the most recent dump time for rate limiting */ ++ hvirt->ts_last_dump_ns = *ts_end_ns; ++ } ++ } else { ++ /* Otherwise, do the full virtualize */ ++ errcode = kbasep_hwcnt_virtualizer_client_dump_rate_limited( ++ hvirt, hvcli, ts_start_ns, ts_end_ns, dump_buf); ++ } ++ ++ mutex_unlock(&hvirt->lock); ++ ++ return errcode; ++} ++KBASE_EXPORT_TEST_API(kbase_hwcnt_virtualizer_client_dump); ++ ++int kbase_hwcnt_virtualizer_client_create( ++ struct kbase_hwcnt_virtualizer *hvirt, ++ const struct kbase_hwcnt_enable_map *enable_map, ++ struct kbase_hwcnt_virtualizer_client **out_hvcli) ++{ ++ int errcode; ++ struct kbase_hwcnt_virtualizer_client *hvcli; ++ ++ if (!hvirt || !enable_map || !out_hvcli || ++ (enable_map->metadata != hvirt->metadata)) ++ return -EINVAL; ++ ++ errcode = kbasep_hwcnt_virtualizer_client_alloc( ++ hvirt->metadata, &hvcli); ++ if (errcode) ++ return errcode; ++ ++ mutex_lock(&hvirt->lock); ++ ++ errcode = kbasep_hwcnt_virtualizer_client_add(hvirt, hvcli, enable_map); ++ ++ mutex_unlock(&hvirt->lock); ++ ++ if (errcode) { ++ kbasep_hwcnt_virtualizer_client_free(hvcli); ++ return errcode; ++ } ++ ++ *out_hvcli = hvcli; ++ return 0; ++} ++KBASE_EXPORT_TEST_API(kbase_hwcnt_virtualizer_client_create); ++ ++void kbase_hwcnt_virtualizer_client_destroy( ++ struct kbase_hwcnt_virtualizer_client *hvcli) ++{ ++ if (!hvcli) ++ return; ++ ++ mutex_lock(&hvcli->hvirt->lock); ++ ++ kbasep_hwcnt_virtualizer_client_remove(hvcli->hvirt, hvcli); ++ ++ mutex_unlock(&hvcli->hvirt->lock); ++ ++ kbasep_hwcnt_virtualizer_client_free(hvcli); ++} ++KBASE_EXPORT_TEST_API(kbase_hwcnt_virtualizer_client_destroy); ++ ++int kbase_hwcnt_virtualizer_init( ++ struct kbase_hwcnt_context *hctx, ++ u64 dump_threshold_ns, ++ struct kbase_hwcnt_virtualizer **out_hvirt) ++{ ++ struct kbase_hwcnt_virtualizer *virt; ++ const struct kbase_hwcnt_metadata *metadata; ++ ++ if (!hctx || !out_hvirt) ++ return -EINVAL; ++ ++ metadata = kbase_hwcnt_context_metadata(hctx); ++ if (!metadata) ++ return -EINVAL; ++ ++ virt = kzalloc(sizeof(*virt), GFP_KERNEL); ++ if (!virt) ++ return -ENOMEM; ++ ++ virt->hctx = hctx; ++ virt->dump_threshold_ns = dump_threshold_ns; ++ virt->metadata = metadata; ++ ++ mutex_init(&virt->lock); ++ INIT_LIST_HEAD(&virt->clients); ++ ++ *out_hvirt = virt; ++ return 0; ++} ++KBASE_EXPORT_TEST_API(kbase_hwcnt_virtualizer_init); ++ ++void kbase_hwcnt_virtualizer_term( ++ struct kbase_hwcnt_virtualizer *hvirt) ++{ ++ if (!hvirt) ++ return; ++ ++ /* Non-zero client count implies client leak */ ++ if (WARN_ON(hvirt->client_count != 0)) { ++ struct kbase_hwcnt_virtualizer_client *pos, *n; ++ ++ list_for_each_entry_safe(pos, n, &hvirt->clients, node) ++ kbase_hwcnt_virtualizer_client_destroy(pos); ++ } ++ ++ WARN_ON(hvirt->client_count != 0); ++ WARN_ON(hvirt->accum); ++ ++ kfree(hvirt); ++} ++KBASE_EXPORT_TEST_API(kbase_hwcnt_virtualizer_term); +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_virtualizer.h b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_virtualizer.h +new file mode 100755 +index 000000000000..8f628c3306fc +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_virtualizer.h +@@ -0,0 +1,145 @@ ++/* ++ * ++ * (C) COPYRIGHT 2018 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/** ++ * Hardware counter virtualizer API. ++ * ++ * Virtualizes a hardware counter context, so multiple clients can access ++ * a single hardware counter resource as though each was the exclusive user. ++ */ ++ ++#ifndef _KBASE_HWCNT_VIRTUALIZER_H_ ++#define _KBASE_HWCNT_VIRTUALIZER_H_ ++ ++#include ++ ++struct kbase_hwcnt_context; ++struct kbase_hwcnt_virtualizer; ++struct kbase_hwcnt_virtualizer_client; ++struct kbase_hwcnt_enable_map; ++struct kbase_hwcnt_dump_buffer; ++ ++/** ++ * kbase_hwcnt_virtualizer_init - Initialise a hardware counter virtualizer. ++ * @hctx: Non-NULL pointer to the hardware counter context to ++ * virtualize. ++ * @dump_threshold_ns: Minimum threshold period for dumps between different ++ * clients where a new accumulator dump will not be ++ * performed, and instead accumulated values will be used. ++ * If 0, rate limiting will be disabled. ++ * @out_hvirt: Non-NULL pointer to where the pointer to the created ++ * virtualizer will be stored on success. ++ * ++ * Return: 0 on success, else error code. ++ */ ++int kbase_hwcnt_virtualizer_init( ++ struct kbase_hwcnt_context *hctx, ++ u64 dump_threshold_ns, ++ struct kbase_hwcnt_virtualizer **out_hvirt); ++ ++/** ++ * kbase_hwcnt_virtualizer_term - Terminate a hardware counter virtualizer. ++ * @hvirt: Pointer to virtualizer to be terminated. ++ */ ++void kbase_hwcnt_virtualizer_term( ++ struct kbase_hwcnt_virtualizer *hvirt); ++ ++/** ++ * kbase_hwcnt_virtualizer_metadata - Get the hardware counter metadata used by ++ * the virtualizer, so related counter data ++ * structures can be created. ++ * @hvirt: Non-NULL pointer to the hardware counter virtualizer. ++ * ++ * Return: Non-NULL pointer to metadata, or NULL on error. ++ */ ++const struct kbase_hwcnt_metadata *kbase_hwcnt_virtualizer_metadata( ++ struct kbase_hwcnt_virtualizer *hvirt); ++ ++/** ++ * kbase_hwcnt_virtualizer_client_create - Create a new virtualizer client. ++ * @hvirt: Non-NULL pointer to the hardware counter virtualizer. ++ * @enable_map: Non-NULL pointer to the enable map for the client. Must have the ++ * same metadata as the virtualizer. ++ * @out_hvcli: Non-NULL pointer to where the pointer to the created client will ++ * be stored on success. ++ * ++ * Return: 0 on success, else error code. ++ */ ++int kbase_hwcnt_virtualizer_client_create( ++ struct kbase_hwcnt_virtualizer *hvirt, ++ const struct kbase_hwcnt_enable_map *enable_map, ++ struct kbase_hwcnt_virtualizer_client **out_hvcli); ++ ++/** ++ * kbase_hwcnt_virtualizer_client_destroy() - Destroy a virtualizer client. ++ * @hvcli: Pointer to the hardware counter client. ++ */ ++void kbase_hwcnt_virtualizer_client_destroy( ++ struct kbase_hwcnt_virtualizer_client *hvcli); ++ ++/** ++ * kbase_hwcnt_virtualizer_client_set_counters - Perform a dump of the client's ++ * currently enabled counters, and ++ * enable a new set of counters ++ * that will be used for ++ * subsequent dumps. ++ * @hvcli: Non-NULL pointer to the virtualizer client. ++ * @enable_map: Non-NULL pointer to the new counter enable map for the client. ++ * Must have the same metadata as the virtualizer. ++ * @ts_start_ns: Non-NULL pointer where the start timestamp of the dump will ++ * be written out to on success. ++ * @ts_end_ns: Non-NULL pointer where the end timestamp of the dump will ++ * be written out to on success. ++ * @dump_buf: Pointer to the buffer where the dump will be written out to on ++ * success. If non-NULL, must have the same metadata as the ++ * accumulator. If NULL, the dump will be discarded. ++ * ++ * Return: 0 on success or error code. ++ */ ++int kbase_hwcnt_virtualizer_client_set_counters( ++ struct kbase_hwcnt_virtualizer_client *hvcli, ++ const struct kbase_hwcnt_enable_map *enable_map, ++ u64 *ts_start_ns, ++ u64 *ts_end_ns, ++ struct kbase_hwcnt_dump_buffer *dump_buf); ++ ++/** ++ * kbase_hwcnt_virtualizer_client_dump - Perform a dump of the client's ++ * currently enabled counters. ++ * @hvcli: Non-NULL pointer to the virtualizer client. ++ * @ts_start_ns: Non-NULL pointer where the start timestamp of the dump will ++ * be written out to on success. ++ * @ts_end_ns: Non-NULL pointer where the end timestamp of the dump will ++ * be written out to on success. ++ * @dump_buf: Pointer to the buffer where the dump will be written out to on ++ * success. If non-NULL, must have the same metadata as the ++ * accumulator. If NULL, the dump will be discarded. ++ * ++ * Return: 0 on success or error code. ++ */ ++int kbase_hwcnt_virtualizer_client_dump( ++ struct kbase_hwcnt_virtualizer_client *hvcli, ++ u64 *ts_start_ns, ++ u64 *ts_end_ns, ++ struct kbase_hwcnt_dump_buffer *dump_buf); ++ ++#endif /* _KBASE_HWCNT_VIRTUALIZER_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_ioctl.h b/drivers/gpu/arm/bifrost/mali_kbase_ioctl.h +new file mode 100755 +index 000000000000..fed45100b4be +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_ioctl.h +@@ -0,0 +1,838 @@ ++/* ++ * ++ * (C) COPYRIGHT 2017-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_IOCTL_H_ ++#define _KBASE_IOCTL_H_ ++ ++#ifdef __cpluscplus ++extern "C" { ++#endif ++ ++#include ++#include ++ ++#if MALI_USE_CSF ++#include "csf/mali_kbase_csf_ioctl.h" ++#else ++#include "jm/mali_kbase_jm_ioctl.h" ++#endif /* MALI_USE_CSF */ ++ ++#define KBASE_IOCTL_TYPE 0x80 ++ ++/** ++ * struct kbase_ioctl_set_flags - Set kernel context creation flags ++ * ++ * @create_flags: Flags - see base_context_create_flags ++ */ ++struct kbase_ioctl_set_flags { ++ __u32 create_flags; ++}; ++ ++#define KBASE_IOCTL_SET_FLAGS \ ++ _IOW(KBASE_IOCTL_TYPE, 1, struct kbase_ioctl_set_flags) ++ ++/** ++ * struct kbase_ioctl_get_gpuprops - Read GPU properties from the kernel ++ * ++ * @buffer: Pointer to the buffer to store properties into ++ * @size: Size of the buffer ++ * @flags: Flags - must be zero for now ++ * ++ * The ioctl will return the number of bytes stored into @buffer or an error ++ * on failure (e.g. @size is too small). If @size is specified as 0 then no ++ * data will be written but the return value will be the number of bytes needed ++ * for all the properties. ++ * ++ * @flags may be used in the future to request a different format for the ++ * buffer. With @flags == 0 the following format is used. ++ * ++ * The buffer will be filled with pairs of values, a u32 key identifying the ++ * property followed by the value. The size of the value is identified using ++ * the bottom bits of the key. The value then immediately followed the key and ++ * is tightly packed (there is no padding). All keys and values are ++ * little-endian. ++ * ++ * 00 = u8 ++ * 01 = u16 ++ * 10 = u32 ++ * 11 = u64 ++ */ ++struct kbase_ioctl_get_gpuprops { ++ __u64 buffer; ++ __u32 size; ++ __u32 flags; ++}; ++ ++#define KBASE_IOCTL_GET_GPUPROPS \ ++ _IOW(KBASE_IOCTL_TYPE, 3, struct kbase_ioctl_get_gpuprops) ++ ++/** ++ * union kbase_ioctl_mem_alloc - Allocate memory on the GPU ++ * ++ * @va_pages: The number of pages of virtual address space to reserve ++ * @commit_pages: The number of physical pages to allocate ++ * @extent: The number of extra pages to allocate on each GPU fault which grows ++ * the region ++ * @flags: Flags ++ * @gpu_va: The GPU virtual address which is allocated ++ * ++ * @in: Input parameters ++ * @out: Output parameters ++ */ ++union kbase_ioctl_mem_alloc { ++ struct { ++ __u64 va_pages; ++ __u64 commit_pages; ++ __u64 extent; ++ __u64 flags; ++ } in; ++ struct { ++ __u64 flags; ++ __u64 gpu_va; ++ } out; ++}; ++ ++#define KBASE_IOCTL_MEM_ALLOC \ ++ _IOWR(KBASE_IOCTL_TYPE, 5, union kbase_ioctl_mem_alloc) ++ ++/** ++ * struct kbase_ioctl_mem_query - Query properties of a GPU memory region ++ * @gpu_addr: A GPU address contained within the region ++ * @query: The type of query ++ * @value: The result of the query ++ * ++ * Use a %KBASE_MEM_QUERY_xxx flag as input for @query. ++ * ++ * @in: Input parameters ++ * @out: Output parameters ++ */ ++union kbase_ioctl_mem_query { ++ struct { ++ __u64 gpu_addr; ++ __u64 query; ++ } in; ++ struct { ++ __u64 value; ++ } out; ++}; ++ ++#define KBASE_IOCTL_MEM_QUERY \ ++ _IOWR(KBASE_IOCTL_TYPE, 6, union kbase_ioctl_mem_query) ++ ++#define KBASE_MEM_QUERY_COMMIT_SIZE ((u64)1) ++#define KBASE_MEM_QUERY_VA_SIZE ((u64)2) ++#define KBASE_MEM_QUERY_FLAGS ((u64)3) ++ ++/** ++ * struct kbase_ioctl_mem_free - Free a memory region ++ * @gpu_addr: Handle to the region to free ++ */ ++struct kbase_ioctl_mem_free { ++ __u64 gpu_addr; ++}; ++ ++#define KBASE_IOCTL_MEM_FREE \ ++ _IOW(KBASE_IOCTL_TYPE, 7, struct kbase_ioctl_mem_free) ++ ++/** ++ * struct kbase_ioctl_hwcnt_reader_setup - Setup HWC dumper/reader ++ * @buffer_count: requested number of dumping buffers ++ * @fe_bm: counters selection bitmask (Front end) ++ * @shader_bm: counters selection bitmask (Shader) ++ * @tiler_bm: counters selection bitmask (Tiler) ++ * @mmu_l2_bm: counters selection bitmask (MMU_L2) ++ * ++ * A fd is returned from the ioctl if successful, or a negative value on error ++ */ ++struct kbase_ioctl_hwcnt_reader_setup { ++ __u32 buffer_count; ++ __u32 fe_bm; ++ __u32 shader_bm; ++ __u32 tiler_bm; ++ __u32 mmu_l2_bm; ++}; ++ ++#define KBASE_IOCTL_HWCNT_READER_SETUP \ ++ _IOW(KBASE_IOCTL_TYPE, 8, struct kbase_ioctl_hwcnt_reader_setup) ++ ++/** ++ * struct kbase_ioctl_hwcnt_enable - Enable hardware counter collection ++ * @dump_buffer: GPU address to write counters to ++ * @fe_bm: counters selection bitmask (Front end) ++ * @shader_bm: counters selection bitmask (Shader) ++ * @tiler_bm: counters selection bitmask (Tiler) ++ * @mmu_l2_bm: counters selection bitmask (MMU_L2) ++ */ ++struct kbase_ioctl_hwcnt_enable { ++ __u64 dump_buffer; ++ __u32 fe_bm; ++ __u32 shader_bm; ++ __u32 tiler_bm; ++ __u32 mmu_l2_bm; ++}; ++ ++#define KBASE_IOCTL_HWCNT_ENABLE \ ++ _IOW(KBASE_IOCTL_TYPE, 9, struct kbase_ioctl_hwcnt_enable) ++ ++#define KBASE_IOCTL_HWCNT_DUMP \ ++ _IO(KBASE_IOCTL_TYPE, 10) ++ ++#define KBASE_IOCTL_HWCNT_CLEAR \ ++ _IO(KBASE_IOCTL_TYPE, 11) ++ ++/** ++ * struct kbase_ioctl_hwcnt_values - Values to set dummy the dummy counters to. ++ * @data: Counter samples for the dummy model. ++ * @size: Size of the counter sample data. ++ * @padding: Padding. ++ */ ++struct kbase_ioctl_hwcnt_values { ++ __u64 data; ++ __u32 size; ++ __u32 padding; ++}; ++ ++#define KBASE_IOCTL_HWCNT_SET \ ++ _IOW(KBASE_IOCTL_TYPE, 32, struct kbase_ioctl_hwcnt_values) ++ ++/** ++ * struct kbase_ioctl_disjoint_query - Query the disjoint counter ++ * @counter: A counter of disjoint events in the kernel ++ */ ++struct kbase_ioctl_disjoint_query { ++ __u32 counter; ++}; ++ ++#define KBASE_IOCTL_DISJOINT_QUERY \ ++ _IOR(KBASE_IOCTL_TYPE, 12, struct kbase_ioctl_disjoint_query) ++ ++/** ++ * struct kbase_ioctl_get_ddk_version - Query the kernel version ++ * @version_buffer: Buffer to receive the kernel version string ++ * @size: Size of the buffer ++ * @padding: Padding ++ * ++ * The ioctl will return the number of bytes written into version_buffer ++ * (which includes a NULL byte) or a negative error code ++ * ++ * The ioctl request code has to be _IOW because the data in ioctl struct is ++ * being copied to the kernel, even though the kernel then writes out the ++ * version info to the buffer specified in the ioctl. ++ */ ++struct kbase_ioctl_get_ddk_version { ++ __u64 version_buffer; ++ __u32 size; ++ __u32 padding; ++}; ++ ++#define KBASE_IOCTL_GET_DDK_VERSION \ ++ _IOW(KBASE_IOCTL_TYPE, 13, struct kbase_ioctl_get_ddk_version) ++ ++/** ++ * struct kbase_ioctl_mem_jit_init_10_2 - Initialize the just-in-time memory ++ * allocator (between kernel driver ++ * version 10.2--11.4) ++ * @va_pages: Number of VA pages to reserve for JIT ++ * ++ * Note that depending on the VA size of the application and GPU, the value ++ * specified in @va_pages may be ignored. ++ * ++ * New code should use KBASE_IOCTL_MEM_JIT_INIT instead, this is kept for ++ * backwards compatibility. ++ */ ++struct kbase_ioctl_mem_jit_init_10_2 { ++ __u64 va_pages; ++}; ++ ++#define KBASE_IOCTL_MEM_JIT_INIT_10_2 \ ++ _IOW(KBASE_IOCTL_TYPE, 14, struct kbase_ioctl_mem_jit_init_10_2) ++ ++/** ++ * struct kbase_ioctl_mem_jit_init_11_5 - Initialize the just-in-time memory ++ * allocator (between kernel driver ++ * version 11.5--11.19) ++ * @va_pages: Number of VA pages to reserve for JIT ++ * @max_allocations: Maximum number of concurrent allocations ++ * @trim_level: Level of JIT allocation trimming to perform on free (0 - 100%) ++ * @group_id: Group ID to be used for physical allocations ++ * @padding: Currently unused, must be zero ++ * ++ * Note that depending on the VA size of the application and GPU, the value ++ * specified in @va_pages may be ignored. ++ * ++ * New code should use KBASE_IOCTL_MEM_JIT_INIT instead, this is kept for ++ * backwards compatibility. ++ */ ++struct kbase_ioctl_mem_jit_init_11_5 { ++ __u64 va_pages; ++ __u8 max_allocations; ++ __u8 trim_level; ++ __u8 group_id; ++ __u8 padding[5]; ++}; ++ ++#define KBASE_IOCTL_MEM_JIT_INIT_11_5 \ ++ _IOW(KBASE_IOCTL_TYPE, 14, struct kbase_ioctl_mem_jit_init_11_5) ++ ++/** ++ * struct kbase_ioctl_mem_jit_init - Initialize the just-in-time memory ++ * allocator ++ * @va_pages: Number of GPU virtual address pages to reserve for just-in-time ++ * memory allocations ++ * @max_allocations: Maximum number of concurrent allocations ++ * @trim_level: Level of JIT allocation trimming to perform on free (0 - 100%) ++ * @group_id: Group ID to be used for physical allocations ++ * @padding: Currently unused, must be zero ++ * @phys_pages: Maximum number of physical pages to allocate just-in-time ++ * ++ * Note that depending on the VA size of the application and GPU, the value ++ * specified in @va_pages may be ignored. ++ */ ++struct kbase_ioctl_mem_jit_init { ++ __u64 va_pages; ++ __u8 max_allocations; ++ __u8 trim_level; ++ __u8 group_id; ++ __u8 padding[5]; ++ __u64 phys_pages; ++}; ++ ++#define KBASE_IOCTL_MEM_JIT_INIT \ ++ _IOW(KBASE_IOCTL_TYPE, 14, struct kbase_ioctl_mem_jit_init) ++ ++/** ++ * struct kbase_ioctl_mem_sync - Perform cache maintenance on memory ++ * ++ * @handle: GPU memory handle (GPU VA) ++ * @user_addr: The address where it is mapped in user space ++ * @size: The number of bytes to synchronise ++ * @type: The direction to synchronise: 0 is sync to memory (clean), ++ * 1 is sync from memory (invalidate). Use the BASE_SYNCSET_OP_xxx constants. ++ * @padding: Padding to round up to a multiple of 8 bytes, must be zero ++ */ ++struct kbase_ioctl_mem_sync { ++ __u64 handle; ++ __u64 user_addr; ++ __u64 size; ++ __u8 type; ++ __u8 padding[7]; ++}; ++ ++#define KBASE_IOCTL_MEM_SYNC \ ++ _IOW(KBASE_IOCTL_TYPE, 15, struct kbase_ioctl_mem_sync) ++ ++/** ++ * union kbase_ioctl_mem_find_cpu_offset - Find the offset of a CPU pointer ++ * ++ * @gpu_addr: The GPU address of the memory region ++ * @cpu_addr: The CPU address to locate ++ * @size: A size in bytes to validate is contained within the region ++ * @offset: The offset from the start of the memory region to @cpu_addr ++ * ++ * @in: Input parameters ++ * @out: Output parameters ++ */ ++union kbase_ioctl_mem_find_cpu_offset { ++ struct { ++ __u64 gpu_addr; ++ __u64 cpu_addr; ++ __u64 size; ++ } in; ++ struct { ++ __u64 offset; ++ } out; ++}; ++ ++#define KBASE_IOCTL_MEM_FIND_CPU_OFFSET \ ++ _IOWR(KBASE_IOCTL_TYPE, 16, union kbase_ioctl_mem_find_cpu_offset) ++ ++/** ++ * struct kbase_ioctl_get_context_id - Get the kernel context ID ++ * ++ * @id: The kernel context ID ++ */ ++struct kbase_ioctl_get_context_id { ++ __u32 id; ++}; ++ ++#define KBASE_IOCTL_GET_CONTEXT_ID \ ++ _IOR(KBASE_IOCTL_TYPE, 17, struct kbase_ioctl_get_context_id) ++ ++/** ++ * struct kbase_ioctl_tlstream_acquire - Acquire a tlstream fd ++ * ++ * @flags: Flags ++ * ++ * The ioctl returns a file descriptor when successful ++ */ ++struct kbase_ioctl_tlstream_acquire { ++ __u32 flags; ++}; ++ ++#define KBASE_IOCTL_TLSTREAM_ACQUIRE \ ++ _IOW(KBASE_IOCTL_TYPE, 18, struct kbase_ioctl_tlstream_acquire) ++ ++#define KBASE_IOCTL_TLSTREAM_FLUSH \ ++ _IO(KBASE_IOCTL_TYPE, 19) ++ ++/** ++ * struct kbase_ioctl_mem_commit - Change the amount of memory backing a region ++ * ++ * @gpu_addr: The memory region to modify ++ * @pages: The number of physical pages that should be present ++ * ++ * The ioctl may return on the following error codes or 0 for success: ++ * -ENOMEM: Out of memory ++ * -EINVAL: Invalid arguments ++ */ ++struct kbase_ioctl_mem_commit { ++ __u64 gpu_addr; ++ __u64 pages; ++}; ++ ++#define KBASE_IOCTL_MEM_COMMIT \ ++ _IOW(KBASE_IOCTL_TYPE, 20, struct kbase_ioctl_mem_commit) ++ ++/** ++ * union kbase_ioctl_mem_alias - Create an alias of memory regions ++ * @flags: Flags, see BASE_MEM_xxx ++ * @stride: Bytes between start of each memory region ++ * @nents: The number of regions to pack together into the alias ++ * @aliasing_info: Pointer to an array of struct base_mem_aliasing_info ++ * @gpu_va: Address of the new alias ++ * @va_pages: Size of the new alias ++ * ++ * @in: Input parameters ++ * @out: Output parameters ++ */ ++union kbase_ioctl_mem_alias { ++ struct { ++ __u64 flags; ++ __u64 stride; ++ __u64 nents; ++ __u64 aliasing_info; ++ } in; ++ struct { ++ __u64 flags; ++ __u64 gpu_va; ++ __u64 va_pages; ++ } out; ++}; ++ ++#define KBASE_IOCTL_MEM_ALIAS \ ++ _IOWR(KBASE_IOCTL_TYPE, 21, union kbase_ioctl_mem_alias) ++ ++/** ++ * union kbase_ioctl_mem_import - Import memory for use by the GPU ++ * @flags: Flags, see BASE_MEM_xxx ++ * @phandle: Handle to the external memory ++ * @type: Type of external memory, see base_mem_import_type ++ * @padding: Amount of extra VA pages to append to the imported buffer ++ * @gpu_va: Address of the new alias ++ * @va_pages: Size of the new alias ++ * ++ * @in: Input parameters ++ * @out: Output parameters ++ */ ++union kbase_ioctl_mem_import { ++ struct { ++ __u64 flags; ++ __u64 phandle; ++ __u32 type; ++ __u32 padding; ++ } in; ++ struct { ++ __u64 flags; ++ __u64 gpu_va; ++ __u64 va_pages; ++ } out; ++}; ++ ++#define KBASE_IOCTL_MEM_IMPORT \ ++ _IOWR(KBASE_IOCTL_TYPE, 22, union kbase_ioctl_mem_import) ++ ++/** ++ * struct kbase_ioctl_mem_flags_change - Change the flags for a memory region ++ * @gpu_va: The GPU region to modify ++ * @flags: The new flags to set ++ * @mask: Mask of the flags to modify ++ */ ++struct kbase_ioctl_mem_flags_change { ++ __u64 gpu_va; ++ __u64 flags; ++ __u64 mask; ++}; ++ ++#define KBASE_IOCTL_MEM_FLAGS_CHANGE \ ++ _IOW(KBASE_IOCTL_TYPE, 23, struct kbase_ioctl_mem_flags_change) ++ ++/** ++ * struct kbase_ioctl_stream_create - Create a synchronisation stream ++ * @name: A name to identify this stream. Must be NULL-terminated. ++ * ++ * Note that this is also called a "timeline", but is named stream to avoid ++ * confusion with other uses of the word. ++ * ++ * Unused bytes in @name (after the first NULL byte) must be also be NULL bytes. ++ * ++ * The ioctl returns a file descriptor. ++ */ ++struct kbase_ioctl_stream_create { ++ char name[32]; ++}; ++ ++#define KBASE_IOCTL_STREAM_CREATE \ ++ _IOW(KBASE_IOCTL_TYPE, 24, struct kbase_ioctl_stream_create) ++ ++/** ++ * struct kbase_ioctl_fence_validate - Validate a fd refers to a fence ++ * @fd: The file descriptor to validate ++ */ ++struct kbase_ioctl_fence_validate { ++ int fd; ++}; ++ ++#define KBASE_IOCTL_FENCE_VALIDATE \ ++ _IOW(KBASE_IOCTL_TYPE, 25, struct kbase_ioctl_fence_validate) ++ ++/** ++ * struct kbase_ioctl_mem_profile_add - Provide profiling information to kernel ++ * @buffer: Pointer to the information ++ * @len: Length ++ * @padding: Padding ++ * ++ * The data provided is accessible through a debugfs file ++ */ ++struct kbase_ioctl_mem_profile_add { ++ __u64 buffer; ++ __u32 len; ++ __u32 padding; ++}; ++ ++#define KBASE_IOCTL_MEM_PROFILE_ADD \ ++ _IOW(KBASE_IOCTL_TYPE, 27, struct kbase_ioctl_mem_profile_add) ++ ++/** ++ * struct kbase_ioctl_sticky_resource_map - Permanently map an external resource ++ * @count: Number of resources ++ * @address: Array of u64 GPU addresses of the external resources to map ++ */ ++struct kbase_ioctl_sticky_resource_map { ++ __u64 count; ++ __u64 address; ++}; ++ ++#define KBASE_IOCTL_STICKY_RESOURCE_MAP \ ++ _IOW(KBASE_IOCTL_TYPE, 29, struct kbase_ioctl_sticky_resource_map) ++ ++/** ++ * struct kbase_ioctl_sticky_resource_map - Unmap a resource mapped which was ++ * previously permanently mapped ++ * @count: Number of resources ++ * @address: Array of u64 GPU addresses of the external resources to unmap ++ */ ++struct kbase_ioctl_sticky_resource_unmap { ++ __u64 count; ++ __u64 address; ++}; ++ ++#define KBASE_IOCTL_STICKY_RESOURCE_UNMAP \ ++ _IOW(KBASE_IOCTL_TYPE, 30, struct kbase_ioctl_sticky_resource_unmap) ++ ++/** ++ * union kbase_ioctl_mem_find_gpu_start_and_offset - Find the start address of ++ * the GPU memory region for ++ * the given gpu address and ++ * the offset of that address ++ * into the region ++ * ++ * @gpu_addr: GPU virtual address ++ * @size: Size in bytes within the region ++ * @start: Address of the beginning of the memory region enclosing @gpu_addr ++ * for the length of @offset bytes ++ * @offset: The offset from the start of the memory region to @gpu_addr ++ * ++ * @in: Input parameters ++ * @out: Output parameters ++ */ ++union kbase_ioctl_mem_find_gpu_start_and_offset { ++ struct { ++ __u64 gpu_addr; ++ __u64 size; ++ } in; ++ struct { ++ __u64 start; ++ __u64 offset; ++ } out; ++}; ++ ++#define KBASE_IOCTL_MEM_FIND_GPU_START_AND_OFFSET \ ++ _IOWR(KBASE_IOCTL_TYPE, 31, union kbase_ioctl_mem_find_gpu_start_and_offset) ++ ++ ++#define KBASE_IOCTL_CINSTR_GWT_START \ ++ _IO(KBASE_IOCTL_TYPE, 33) ++ ++#define KBASE_IOCTL_CINSTR_GWT_STOP \ ++ _IO(KBASE_IOCTL_TYPE, 34) ++ ++/** ++ * union kbase_ioctl_gwt_dump - Used to collect all GPU write fault addresses. ++ * @addr_buffer: Address of buffer to hold addresses of gpu modified areas. ++ * @size_buffer: Address of buffer to hold size of modified areas (in pages) ++ * @len: Number of addresses the buffers can hold. ++ * @more_data_available: Status indicating if more addresses are available. ++ * @no_of_addr_collected: Number of addresses collected into addr_buffer. ++ * ++ * @in: Input parameters ++ * @out: Output parameters ++ * ++ * This structure is used when performing a call to dump GPU write fault ++ * addresses. ++ */ ++union kbase_ioctl_cinstr_gwt_dump { ++ struct { ++ __u64 addr_buffer; ++ __u64 size_buffer; ++ __u32 len; ++ __u32 padding; ++ ++ } in; ++ struct { ++ __u32 no_of_addr_collected; ++ __u8 more_data_available; ++ __u8 padding[27]; ++ } out; ++}; ++ ++#define KBASE_IOCTL_CINSTR_GWT_DUMP \ ++ _IOWR(KBASE_IOCTL_TYPE, 35, union kbase_ioctl_cinstr_gwt_dump) ++ ++/** ++ * struct kbase_ioctl_mem_exec_init - Initialise the EXEC_VA memory zone ++ * ++ * @va_pages: Number of VA pages to reserve for EXEC_VA ++ */ ++struct kbase_ioctl_mem_exec_init { ++ __u64 va_pages; ++}; ++ ++#define KBASE_IOCTL_MEM_EXEC_INIT \ ++ _IOW(KBASE_IOCTL_TYPE, 38, struct kbase_ioctl_mem_exec_init) ++ ++/** ++ * union kbase_ioctl_get_cpu_gpu_timeinfo - Request zero or more types of ++ * cpu/gpu time (counter values) ++ * ++ * @request_flags: Bit-flags indicating the requested types. ++ * @paddings: Unused, size alignment matching the out. ++ * @sec: Integer field of the monotonic time, unit in seconds. ++ * @nsec: Fractional sec of the monotonic time, in nano-seconds. ++ * @padding: Unused, for u64 alignment ++ * @timestamp: System wide timestamp (counter) value. ++ * @cycle_counter: GPU cycle counter value. ++ * ++ * @in: Input parameters ++ * @out: Output parameters ++ * ++ */ ++union kbase_ioctl_get_cpu_gpu_timeinfo { ++ struct { ++ __u32 request_flags; ++ __u32 paddings[7]; ++ } in; ++ struct { ++ __u64 sec; ++ __u32 nsec; ++ __u32 padding; ++ __u64 timestamp; ++ __u64 cycle_counter; ++ } out; ++}; ++ ++#define KBASE_IOCTL_GET_CPU_GPU_TIMEINFO \ ++ _IOWR(KBASE_IOCTL_TYPE, 50, union kbase_ioctl_get_cpu_gpu_timeinfo) ++ ++/*************** ++ * test ioctls * ++ ***************/ ++#if MALI_UNIT_TEST ++/* These ioctls are purely for test purposes and are not used in the production ++ * driver, they therefore may change without notice ++ */ ++ ++#define KBASE_IOCTL_TEST_TYPE (KBASE_IOCTL_TYPE + 1) ++ ++/** ++ * struct kbase_ioctl_tlstream_test - Start a timeline stream test ++ * ++ * @tpw_count: number of trace point writers in each context ++ * @msg_delay: time delay between tracepoints from one writer in milliseconds ++ * @msg_count: number of trace points written by one writer ++ * @aux_msg: if non-zero aux messages will be included ++ */ ++struct kbase_ioctl_tlstream_test { ++ __u32 tpw_count; ++ __u32 msg_delay; ++ __u32 msg_count; ++ __u32 aux_msg; ++}; ++ ++#define KBASE_IOCTL_TLSTREAM_TEST \ ++ _IOW(KBASE_IOCTL_TEST_TYPE, 1, struct kbase_ioctl_tlstream_test) ++ ++/** ++ * struct kbase_ioctl_tlstream_stats - Read tlstream stats for test purposes ++ * @bytes_collected: number of bytes read by user ++ * @bytes_generated: number of bytes generated by tracepoints ++ */ ++struct kbase_ioctl_tlstream_stats { ++ __u32 bytes_collected; ++ __u32 bytes_generated; ++}; ++ ++#define KBASE_IOCTL_TLSTREAM_STATS \ ++ _IOR(KBASE_IOCTL_TEST_TYPE, 2, struct kbase_ioctl_tlstream_stats) ++ ++#endif /* MALI_UNIT_TEST */ ++ ++/* Customer extension range */ ++#define KBASE_IOCTL_EXTRA_TYPE (KBASE_IOCTL_TYPE + 2) ++ ++/* If the integration needs extra ioctl add them there ++ * like this: ++ * ++ * struct my_ioctl_args { ++ * .... ++ * } ++ * ++ * #define KBASE_IOCTL_MY_IOCTL \ ++ * _IOWR(KBASE_IOCTL_EXTRA_TYPE, 0, struct my_ioctl_args) ++ */ ++ ++ ++/********************************** ++ * Definitions for GPU properties * ++ **********************************/ ++#define KBASE_GPUPROP_VALUE_SIZE_U8 (0x0) ++#define KBASE_GPUPROP_VALUE_SIZE_U16 (0x1) ++#define KBASE_GPUPROP_VALUE_SIZE_U32 (0x2) ++#define KBASE_GPUPROP_VALUE_SIZE_U64 (0x3) ++ ++#define KBASE_GPUPROP_PRODUCT_ID 1 ++#define KBASE_GPUPROP_VERSION_STATUS 2 ++#define KBASE_GPUPROP_MINOR_REVISION 3 ++#define KBASE_GPUPROP_MAJOR_REVISION 4 ++/* 5 previously used for GPU speed */ ++#define KBASE_GPUPROP_GPU_FREQ_KHZ_MAX 6 ++/* 7 previously used for minimum GPU speed */ ++#define KBASE_GPUPROP_LOG2_PROGRAM_COUNTER_SIZE 8 ++#define KBASE_GPUPROP_TEXTURE_FEATURES_0 9 ++#define KBASE_GPUPROP_TEXTURE_FEATURES_1 10 ++#define KBASE_GPUPROP_TEXTURE_FEATURES_2 11 ++#define KBASE_GPUPROP_GPU_AVAILABLE_MEMORY_SIZE 12 ++ ++#define KBASE_GPUPROP_L2_LOG2_LINE_SIZE 13 ++#define KBASE_GPUPROP_L2_LOG2_CACHE_SIZE 14 ++#define KBASE_GPUPROP_L2_NUM_L2_SLICES 15 ++ ++#define KBASE_GPUPROP_TILER_BIN_SIZE_BYTES 16 ++#define KBASE_GPUPROP_TILER_MAX_ACTIVE_LEVELS 17 ++ ++#define KBASE_GPUPROP_MAX_THREADS 18 ++#define KBASE_GPUPROP_MAX_WORKGROUP_SIZE 19 ++#define KBASE_GPUPROP_MAX_BARRIER_SIZE 20 ++#define KBASE_GPUPROP_MAX_REGISTERS 21 ++#define KBASE_GPUPROP_MAX_TASK_QUEUE 22 ++#define KBASE_GPUPROP_MAX_THREAD_GROUP_SPLIT 23 ++#define KBASE_GPUPROP_IMPL_TECH 24 ++ ++#define KBASE_GPUPROP_RAW_SHADER_PRESENT 25 ++#define KBASE_GPUPROP_RAW_TILER_PRESENT 26 ++#define KBASE_GPUPROP_RAW_L2_PRESENT 27 ++#define KBASE_GPUPROP_RAW_STACK_PRESENT 28 ++#define KBASE_GPUPROP_RAW_L2_FEATURES 29 ++#define KBASE_GPUPROP_RAW_CORE_FEATURES 30 ++#define KBASE_GPUPROP_RAW_MEM_FEATURES 31 ++#define KBASE_GPUPROP_RAW_MMU_FEATURES 32 ++#define KBASE_GPUPROP_RAW_AS_PRESENT 33 ++#define KBASE_GPUPROP_RAW_JS_PRESENT 34 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_0 35 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_1 36 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_2 37 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_3 38 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_4 39 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_5 40 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_6 41 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_7 42 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_8 43 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_9 44 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_10 45 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_11 46 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_12 47 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_13 48 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_14 49 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_15 50 ++#define KBASE_GPUPROP_RAW_TILER_FEATURES 51 ++#define KBASE_GPUPROP_RAW_TEXTURE_FEATURES_0 52 ++#define KBASE_GPUPROP_RAW_TEXTURE_FEATURES_1 53 ++#define KBASE_GPUPROP_RAW_TEXTURE_FEATURES_2 54 ++#define KBASE_GPUPROP_RAW_GPU_ID 55 ++#define KBASE_GPUPROP_RAW_THREAD_MAX_THREADS 56 ++#define KBASE_GPUPROP_RAW_THREAD_MAX_WORKGROUP_SIZE 57 ++#define KBASE_GPUPROP_RAW_THREAD_MAX_BARRIER_SIZE 58 ++#define KBASE_GPUPROP_RAW_THREAD_FEATURES 59 ++#define KBASE_GPUPROP_RAW_COHERENCY_MODE 60 ++ ++#define KBASE_GPUPROP_COHERENCY_NUM_GROUPS 61 ++#define KBASE_GPUPROP_COHERENCY_NUM_CORE_GROUPS 62 ++#define KBASE_GPUPROP_COHERENCY_COHERENCY 63 ++#define KBASE_GPUPROP_COHERENCY_GROUP_0 64 ++#define KBASE_GPUPROP_COHERENCY_GROUP_1 65 ++#define KBASE_GPUPROP_COHERENCY_GROUP_2 66 ++#define KBASE_GPUPROP_COHERENCY_GROUP_3 67 ++#define KBASE_GPUPROP_COHERENCY_GROUP_4 68 ++#define KBASE_GPUPROP_COHERENCY_GROUP_5 69 ++#define KBASE_GPUPROP_COHERENCY_GROUP_6 70 ++#define KBASE_GPUPROP_COHERENCY_GROUP_7 71 ++#define KBASE_GPUPROP_COHERENCY_GROUP_8 72 ++#define KBASE_GPUPROP_COHERENCY_GROUP_9 73 ++#define KBASE_GPUPROP_COHERENCY_GROUP_10 74 ++#define KBASE_GPUPROP_COHERENCY_GROUP_11 75 ++#define KBASE_GPUPROP_COHERENCY_GROUP_12 76 ++#define KBASE_GPUPROP_COHERENCY_GROUP_13 77 ++#define KBASE_GPUPROP_COHERENCY_GROUP_14 78 ++#define KBASE_GPUPROP_COHERENCY_GROUP_15 79 ++ ++#define KBASE_GPUPROP_TEXTURE_FEATURES_3 80 ++#define KBASE_GPUPROP_RAW_TEXTURE_FEATURES_3 81 ++ ++#define KBASE_GPUPROP_NUM_EXEC_ENGINES 82 ++ ++#define KBASE_GPUPROP_RAW_THREAD_TLS_ALLOC 83 ++#define KBASE_GPUPROP_TLS_ALLOC 84 ++ ++#ifdef __cpluscplus ++} ++#endif ++ ++#endif +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_jd.c b/drivers/gpu/arm/bifrost/mali_kbase_jd.c +new file mode 100755 +index 000000000000..d0674d1bd8f4 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_jd.c +@@ -0,0 +1,1819 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++#include ++#ifdef CONFIG_COMPAT ++#include ++#endif ++#include ++#include ++#include ++#include ++ ++#include ++#include ++#include ++#include ++#include ++ ++#include "mali_kbase_dma_fence.h" ++#include ++ ++#include ++ ++#define beenthere(kctx, f, a...) dev_dbg(kctx->kbdev->dev, "%s:" f, __func__, ##a) ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 8, 0) ++/* random32 was renamed to prandom_u32 in 3.8 */ ++#define prandom_u32 random32 ++#endif ++ ++/* Return whether katom will run on the GPU or not. Currently only soft jobs and ++ * dependency-only atoms do not run on the GPU */ ++#define IS_GPU_ATOM(katom) (!((katom->core_req & BASE_JD_REQ_SOFT_JOB) || \ ++ ((katom->core_req & BASE_JD_REQ_ATOM_TYPE) == \ ++ BASE_JD_REQ_DEP))) ++ ++/* ++ * This is the kernel side of the API. Only entry points are: ++ * - kbase_jd_submit(): Called from userspace to submit a single bag ++ * - kbase_jd_done(): Called from interrupt context to track the ++ * completion of a job. ++ * Callouts: ++ * - to the job manager (enqueue a job) ++ * - to the event subsystem (signals the completion/failure of bag/job-chains). ++ */ ++ ++static void __user * ++get_compat_pointer(struct kbase_context *kctx, const u64 p) ++{ ++#ifdef CONFIG_COMPAT ++ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) ++ return compat_ptr(p); ++#endif ++ return u64_to_user_ptr(p); ++} ++ ++/* Mark an atom as complete, and trace it in kinstr_jm */ ++static void jd_mark_atom_complete(struct kbase_jd_atom *katom) ++{ ++ katom->status = KBASE_JD_ATOM_STATE_COMPLETED; ++ kbase_kinstr_jm_atom_complete(katom); ++ dev_dbg(katom->kctx->kbdev->dev, "Atom %p status to completed\n", ++ (void *)katom); ++} ++ ++/* Runs an atom, either by handing to the JS or by immediately running it in the case of soft-jobs ++ * ++ * Returns whether the JS needs a reschedule. ++ * ++ * Note that the caller must also check the atom status and ++ * if it is KBASE_JD_ATOM_STATE_COMPLETED must call jd_done_nolock ++ */ ++static bool jd_run_atom(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ ++ dev_dbg(kctx->kbdev->dev, "JD run atom %p in kctx %p\n", ++ (void *)katom, (void *)kctx); ++ ++ KBASE_DEBUG_ASSERT(katom->status != KBASE_JD_ATOM_STATE_UNUSED); ++ ++ if ((katom->core_req & BASE_JD_REQ_ATOM_TYPE) == BASE_JD_REQ_DEP) { ++ /* Dependency only atom */ ++ trace_sysgraph(SGR_SUBMIT, kctx->id, ++ kbase_jd_atom_id(katom->kctx, katom)); ++ jd_mark_atom_complete(katom); ++ return 0; ++ } else if (katom->core_req & BASE_JD_REQ_SOFT_JOB) { ++ /* Soft-job */ ++ if (katom->will_fail_event_code) { ++ kbase_finish_soft_job(katom); ++ jd_mark_atom_complete(katom); ++ return 0; ++ } ++ if (kbase_process_soft_job(katom) == 0) { ++ kbase_finish_soft_job(katom); ++ jd_mark_atom_complete(katom); ++ } ++ return 0; ++ } ++ ++ katom->status = KBASE_JD_ATOM_STATE_IN_JS; ++ dev_dbg(kctx->kbdev->dev, "Atom %p status to in JS\n", (void *)katom); ++ /* Queue an action about whether we should try scheduling a context */ ++ return kbasep_js_add_job(kctx, katom); ++} ++ ++void kbase_jd_dep_clear_locked(struct kbase_jd_atom *katom) ++{ ++ struct kbase_device *kbdev; ++ ++ KBASE_DEBUG_ASSERT(katom); ++ kbdev = katom->kctx->kbdev; ++ KBASE_DEBUG_ASSERT(kbdev); ++ ++ /* Check whether the atom's other dependencies were already met. If ++ * katom is a GPU atom then the job scheduler may be able to represent ++ * the dependencies, hence we may attempt to submit it before they are ++ * met. Other atoms must have had both dependencies resolved. ++ */ ++ if (IS_GPU_ATOM(katom) || ++ (!kbase_jd_katom_dep_atom(&katom->dep[0]) && ++ !kbase_jd_katom_dep_atom(&katom->dep[1]))) { ++ /* katom dep complete, attempt to run it */ ++ bool resched = false; ++ ++ resched = jd_run_atom(katom); ++ ++ if (katom->status == KBASE_JD_ATOM_STATE_COMPLETED) { ++ /* The atom has already finished */ ++ resched |= jd_done_nolock(katom, NULL); ++ } ++ ++ if (resched) ++ kbase_js_sched_all(kbdev); ++ } ++} ++ ++void kbase_jd_free_external_resources(struct kbase_jd_atom *katom) ++{ ++#ifdef CONFIG_MALI_BIFROST_DMA_FENCE ++ /* Flush dma-fence workqueue to ensure that any callbacks that may have ++ * been queued are done before continuing. ++ * Any successfully completed atom would have had all it's callbacks ++ * completed before the atom was run, so only flush for failed atoms. ++ */ ++ if (katom->event_code != BASE_JD_EVENT_DONE) ++ flush_workqueue(katom->kctx->dma_fence.wq); ++#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ ++} ++ ++static void kbase_jd_post_external_resources(struct kbase_jd_atom *katom) ++{ ++ KBASE_DEBUG_ASSERT(katom); ++ KBASE_DEBUG_ASSERT(katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES); ++ ++#ifdef CONFIG_MALI_BIFROST_DMA_FENCE ++ kbase_dma_fence_signal(katom); ++#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ ++ ++ kbase_gpu_vm_lock(katom->kctx); ++ /* only roll back if extres is non-NULL */ ++ if (katom->extres) { ++ u32 res_no; ++ ++ res_no = katom->nr_extres; ++ while (res_no-- > 0) { ++ struct kbase_mem_phy_alloc *alloc = katom->extres[res_no].alloc; ++ struct kbase_va_region *reg; ++ ++ reg = kbase_region_tracker_find_region_base_address( ++ katom->kctx, ++ katom->extres[res_no].gpu_address); ++ kbase_unmap_external_resource(katom->kctx, reg, alloc); ++ } ++ kfree(katom->extres); ++ katom->extres = NULL; ++ } ++ kbase_gpu_vm_unlock(katom->kctx); ++} ++ ++/* ++ * Set up external resources needed by this job. ++ * ++ * jctx.lock must be held when this is called. ++ */ ++ ++static int kbase_jd_pre_external_resources(struct kbase_jd_atom *katom, const struct base_jd_atom *user_atom) ++{ ++ int err_ret_val = -EINVAL; ++ u32 res_no; ++#ifdef CONFIG_MALI_BIFROST_DMA_FENCE ++ struct kbase_dma_fence_resv_info info = { ++ .resv_objs = NULL, ++ .dma_fence_resv_count = 0, ++ .dma_fence_excl_bitmap = NULL ++ }; ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ /* ++ * When both dma-buf fence and Android native sync is enabled, we ++ * disable dma-buf fence for contexts that are using Android native ++ * fences. ++ */ ++ const bool implicit_sync = !kbase_ctx_flag(katom->kctx, ++ KCTX_NO_IMPLICIT_SYNC); ++#else /* CONFIG_SYNC || CONFIG_SYNC_FILE*/ ++ const bool implicit_sync = true; ++#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ ++#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ ++ struct base_external_resource *input_extres; ++ ++ KBASE_DEBUG_ASSERT(katom); ++ KBASE_DEBUG_ASSERT(katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES); ++ ++ /* no resources encoded, early out */ ++ if (!katom->nr_extres) ++ return -EINVAL; ++ ++ katom->extres = kmalloc_array(katom->nr_extres, sizeof(*katom->extres), GFP_KERNEL); ++ if (!katom->extres) ++ return -ENOMEM; ++ ++ /* copy user buffer to the end of our real buffer. ++ * Make sure the struct sizes haven't changed in a way ++ * we don't support */ ++ BUILD_BUG_ON(sizeof(*input_extres) > sizeof(*katom->extres)); ++ input_extres = (struct base_external_resource *) ++ (((unsigned char *)katom->extres) + ++ (sizeof(*katom->extres) - sizeof(*input_extres)) * ++ katom->nr_extres); ++ ++ if (copy_from_user(input_extres, ++ get_compat_pointer(katom->kctx, user_atom->extres_list), ++ sizeof(*input_extres) * katom->nr_extres) != 0) { ++ err_ret_val = -EINVAL; ++ goto early_err_out; ++ } ++ ++#ifdef CONFIG_MALI_BIFROST_DMA_FENCE ++ if (implicit_sync) { ++ info.resv_objs = kmalloc_array(katom->nr_extres, ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)) ++ sizeof(struct reservation_object *), ++#else ++ sizeof(struct dma_resv *), ++#endif ++ GFP_KERNEL); ++ if (!info.resv_objs) { ++ err_ret_val = -ENOMEM; ++ goto early_err_out; ++ } ++ ++ info.dma_fence_excl_bitmap = ++ kcalloc(BITS_TO_LONGS(katom->nr_extres), ++ sizeof(unsigned long), GFP_KERNEL); ++ if (!info.dma_fence_excl_bitmap) { ++ err_ret_val = -ENOMEM; ++ goto early_err_out; ++ } ++ } ++#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ ++ ++ /* Take the processes mmap lock */ ++ down_read(kbase_mem_get_process_mmap_lock()); ++ ++ /* need to keep the GPU VM locked while we set up UMM buffers */ ++ kbase_gpu_vm_lock(katom->kctx); ++ for (res_no = 0; res_no < katom->nr_extres; res_no++) { ++ struct base_external_resource *res = &input_extres[res_no]; ++ struct kbase_va_region *reg; ++ struct kbase_mem_phy_alloc *alloc; ++#ifdef CONFIG_MALI_BIFROST_DMA_FENCE ++ bool exclusive; ++ exclusive = (res->ext_resource & BASE_EXT_RES_ACCESS_EXCLUSIVE) ++ ? true : false; ++#endif ++ reg = kbase_region_tracker_find_region_enclosing_address( ++ katom->kctx, ++ res->ext_resource & ~BASE_EXT_RES_ACCESS_EXCLUSIVE); ++ /* did we find a matching region object? */ ++ if (kbase_is_region_invalid_or_free(reg)) { ++ /* roll back */ ++ goto failed_loop; ++ } ++ ++ if (!(katom->core_req & BASE_JD_REQ_SOFT_JOB) && ++ (reg->flags & KBASE_REG_PROTECTED)) { ++ katom->atom_flags |= KBASE_KATOM_FLAG_PROTECTED; ++ } ++ ++ alloc = kbase_map_external_resource(katom->kctx, reg, ++ current->mm); ++ if (!alloc) { ++ err_ret_val = -EINVAL; ++ goto failed_loop; ++ } ++ ++#ifdef CONFIG_MALI_BIFROST_DMA_FENCE ++ if (implicit_sync && ++ reg->gpu_alloc->type == KBASE_MEM_TYPE_IMPORTED_UMM) { ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)) ++ struct reservation_object *resv; ++#else ++ struct dma_resv *resv; ++#endif ++ resv = reg->gpu_alloc->imported.umm.dma_buf->resv; ++ if (resv) ++ kbase_dma_fence_add_reservation(resv, &info, ++ exclusive); ++ } ++#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ ++ ++ /* finish with updating out array with the data we found */ ++ /* NOTE: It is important that this is the last thing we do (or ++ * at least not before the first write) as we overwrite elements ++ * as we loop and could be overwriting ourself, so no writes ++ * until the last read for an element. ++ * */ ++ katom->extres[res_no].gpu_address = reg->start_pfn << PAGE_SHIFT; /* save the start_pfn (as an address, not pfn) to use fast lookup later */ ++ katom->extres[res_no].alloc = alloc; ++ } ++ /* successfully parsed the extres array */ ++ /* drop the vm lock now */ ++ kbase_gpu_vm_unlock(katom->kctx); ++ ++ /* Release the processes mmap lock */ ++ up_read(kbase_mem_get_process_mmap_lock()); ++ ++#ifdef CONFIG_MALI_BIFROST_DMA_FENCE ++ if (implicit_sync) { ++ if (info.dma_fence_resv_count) { ++ int ret; ++ ++ ret = kbase_dma_fence_wait(katom, &info); ++ if (ret < 0) ++ goto failed_dma_fence_setup; ++ } ++ ++ kfree(info.resv_objs); ++ kfree(info.dma_fence_excl_bitmap); ++ } ++#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ ++ ++ /* all done OK */ ++ return 0; ++ ++/* error handling section */ ++ ++#ifdef CONFIG_MALI_BIFROST_DMA_FENCE ++failed_dma_fence_setup: ++ /* Lock the processes mmap lock */ ++ down_read(kbase_mem_get_process_mmap_lock()); ++ ++ /* lock before we unmap */ ++ kbase_gpu_vm_lock(katom->kctx); ++#endif ++ ++ failed_loop: ++ /* undo the loop work */ ++ while (res_no-- > 0) { ++ struct kbase_mem_phy_alloc *alloc = katom->extres[res_no].alloc; ++ ++ kbase_unmap_external_resource(katom->kctx, NULL, alloc); ++ } ++ kbase_gpu_vm_unlock(katom->kctx); ++ ++ /* Release the processes mmap lock */ ++ up_read(kbase_mem_get_process_mmap_lock()); ++ ++ early_err_out: ++ kfree(katom->extres); ++ katom->extres = NULL; ++#ifdef CONFIG_MALI_BIFROST_DMA_FENCE ++ if (implicit_sync) { ++ kfree(info.resv_objs); ++ kfree(info.dma_fence_excl_bitmap); ++ } ++#endif ++ return err_ret_val; ++} ++ ++static inline void jd_resolve_dep(struct list_head *out_list, ++ struct kbase_jd_atom *katom, ++ u8 d, bool ctx_is_dying) ++{ ++ u8 other_d = !d; ++ ++ while (!list_empty(&katom->dep_head[d])) { ++ struct kbase_jd_atom *dep_atom; ++ struct kbase_jd_atom *other_dep_atom; ++ u8 dep_type; ++ ++ dep_atom = list_entry(katom->dep_head[d].next, ++ struct kbase_jd_atom, dep_item[d]); ++ list_del(katom->dep_head[d].next); ++ ++ dep_type = kbase_jd_katom_dep_type(&dep_atom->dep[d]); ++ kbase_jd_katom_dep_clear(&dep_atom->dep[d]); ++ ++ if (katom->event_code != BASE_JD_EVENT_DONE && ++ (dep_type != BASE_JD_DEP_TYPE_ORDER)) { ++#ifdef CONFIG_MALI_BIFROST_DMA_FENCE ++ kbase_dma_fence_cancel_callbacks(dep_atom); ++#endif ++ ++ dep_atom->event_code = katom->event_code; ++ KBASE_DEBUG_ASSERT(dep_atom->status != ++ KBASE_JD_ATOM_STATE_UNUSED); ++ ++ dep_atom->will_fail_event_code = dep_atom->event_code; ++ } ++ other_dep_atom = (struct kbase_jd_atom *) ++ kbase_jd_katom_dep_atom(&dep_atom->dep[other_d]); ++ ++ if (!dep_atom->in_jd_list && (!other_dep_atom || ++ (IS_GPU_ATOM(dep_atom) && !ctx_is_dying && ++ !dep_atom->will_fail_event_code && ++ !other_dep_atom->will_fail_event_code))) { ++ bool dep_satisfied = true; ++#ifdef CONFIG_MALI_BIFROST_DMA_FENCE ++ int dep_count; ++ ++ dep_count = kbase_fence_dep_count_read(dep_atom); ++ if (likely(dep_count == -1)) { ++ dep_satisfied = true; ++ } else { ++ /* ++ * There are either still active callbacks, or ++ * all fences for this @dep_atom has signaled, ++ * but the worker that will queue the atom has ++ * not yet run. ++ * ++ * Wait for the fences to signal and the fence ++ * worker to run and handle @dep_atom. If ++ * @dep_atom was completed due to error on ++ * @katom, then the fence worker will pick up ++ * the complete status and error code set on ++ * @dep_atom above. ++ */ ++ dep_satisfied = false; ++ } ++#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ ++ ++ if (dep_satisfied) { ++ trace_sysgraph(SGR_DEP_RES, ++ dep_atom->kctx->id, ++ kbase_jd_atom_id(katom->kctx, dep_atom)); ++ dep_atom->in_jd_list = true; ++ list_add_tail(&dep_atom->jd_item, out_list); ++ } ++ } ++ } ++} ++ ++/** ++ * is_dep_valid - Validate that a dependency is valid for early dependency ++ * submission ++ * @katom: Dependency atom to validate ++ * ++ * A dependency is valid if any of the following are true : ++ * - It does not exist (a non-existent dependency does not block submission) ++ * - It is in the job scheduler ++ * - It has completed, does not have a failure event code, and has not been ++ * marked to fail in the future ++ * ++ * Return: true if valid, false otherwise ++ */ ++static bool is_dep_valid(struct kbase_jd_atom *katom) ++{ ++ /* If there's no dependency then this is 'valid' from the perspective of ++ * early dependency submission */ ++ if (!katom) ++ return true; ++ ++ /* Dependency must have reached the job scheduler */ ++ if (katom->status < KBASE_JD_ATOM_STATE_IN_JS) ++ return false; ++ ++ /* If dependency has completed and has failed or will fail then it is ++ * not valid */ ++ if (katom->status >= KBASE_JD_ATOM_STATE_HW_COMPLETED && ++ (katom->event_code != BASE_JD_EVENT_DONE || ++ katom->will_fail_event_code)) ++ return false; ++ ++ return true; ++} ++ ++static void jd_try_submitting_deps(struct list_head *out_list, ++ struct kbase_jd_atom *node) ++{ ++ int i; ++ ++ for (i = 0; i < 2; i++) { ++ struct list_head *pos; ++ ++ list_for_each(pos, &node->dep_head[i]) { ++ struct kbase_jd_atom *dep_atom = list_entry(pos, ++ struct kbase_jd_atom, dep_item[i]); ++ ++ if (IS_GPU_ATOM(dep_atom) && !dep_atom->in_jd_list) { ++ /*Check if atom deps look sane*/ ++ bool dep0_valid = is_dep_valid( ++ dep_atom->dep[0].atom); ++ bool dep1_valid = is_dep_valid( ++ dep_atom->dep[1].atom); ++ bool dep_satisfied = true; ++#ifdef CONFIG_MALI_BIFROST_DMA_FENCE ++ int dep_count; ++ ++ dep_count = kbase_fence_dep_count_read( ++ dep_atom); ++ if (likely(dep_count == -1)) { ++ dep_satisfied = true; ++ } else { ++ /* ++ * There are either still active callbacks, or ++ * all fences for this @dep_atom has signaled, ++ * but the worker that will queue the atom has ++ * not yet run. ++ * ++ * Wait for the fences to signal and the fence ++ * worker to run and handle @dep_atom. If ++ * @dep_atom was completed due to error on ++ * @katom, then the fence worker will pick up ++ * the complete status and error code set on ++ * @dep_atom above. ++ */ ++ dep_satisfied = false; ++ } ++#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ ++ ++ if (dep0_valid && dep1_valid && dep_satisfied) { ++ trace_sysgraph(SGR_DEP_RES, ++ dep_atom->kctx->id, ++ kbase_jd_atom_id(dep_atom->kctx, ++ dep_atom)); ++ dep_atom->in_jd_list = true; ++ list_add(&dep_atom->jd_item, out_list); ++ } ++ } ++ } ++ } ++} ++ ++#if MALI_JIT_PRESSURE_LIMIT_BASE ++/** ++ * jd_update_jit_usage - Update just-in-time physical memory usage for an atom. ++ * ++ * @katom: An atom that has just finished. ++ * ++ * Read back actual just-in-time memory region usage from atoms that provide ++ * this information, and update the current physical page pressure. ++ * ++ * The caller must hold the kbase_jd_context.lock. ++ */ ++static void jd_update_jit_usage(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ struct kbase_va_region *reg; ++ struct kbase_vmap_struct mapping; ++ u64 *ptr; ++ u64 used_pages; ++ unsigned int idx; ++ ++ lockdep_assert_held(&kctx->jctx.lock); ++ ++ /* If this atom wrote to JIT memory, find out how much it has written ++ * and update the usage information in the region. ++ */ ++ for (idx = 0; ++ idx < ARRAY_SIZE(katom->jit_ids) && katom->jit_ids[idx]; ++ idx++) { ++ enum heap_pointer { LOW = 0, HIGH, COUNT }; ++ size_t size_to_read; ++ u64 read_val; ++ ++ reg = kctx->jit_alloc[katom->jit_ids[idx]]; ++ ++ if (!reg) { ++ dev_warn(kctx->kbdev->dev, ++ "%s: JIT id[%u]=%u has no region\n", ++ __func__, idx, katom->jit_ids[idx]); ++ continue; ++ } ++ ++ if (reg == KBASE_RESERVED_REG_JIT_ALLOC) { ++ dev_warn(kctx->kbdev->dev, ++ "%s: JIT id[%u]=%u has failed to allocate a region\n", ++ __func__, idx, katom->jit_ids[idx]); ++ continue; ++ } ++ ++ if (!reg->heap_info_gpu_addr) ++ continue; ++ ++ size_to_read = sizeof(*ptr); ++ if (reg->flags & KBASE_REG_HEAP_INFO_IS_SIZE) ++ size_to_read = sizeof(u32); ++ else if (reg->flags & KBASE_REG_TILER_ALIGN_TOP) ++ size_to_read = sizeof(u64[COUNT]); ++ ++ ptr = kbase_vmap(kctx, reg->heap_info_gpu_addr, size_to_read, ++ &mapping); ++ ++ if (!ptr) { ++ dev_warn(kctx->kbdev->dev, ++ "%s: JIT id[%u]=%u start=0x%llx unable to map end marker %llx\n", ++ __func__, idx, katom->jit_ids[idx], ++ reg->start_pfn << PAGE_SHIFT, ++ reg->heap_info_gpu_addr); ++ continue; ++ } ++ ++ if (reg->flags & KBASE_REG_HEAP_INFO_IS_SIZE) { ++ read_val = READ_ONCE(*(u32 *)ptr); ++ used_pages = PFN_UP(read_val); ++ } else { ++ u64 addr_end; ++ ++ if (reg->flags & KBASE_REG_TILER_ALIGN_TOP) { ++ const unsigned long extent_bytes = reg->extent ++ << PAGE_SHIFT; ++ const u64 low_ptr = ptr[LOW]; ++ const u64 high_ptr = ptr[HIGH]; ++ ++ /* As either the low or high pointer could ++ * consume their partition and move onto the ++ * next chunk, we need to account for both. ++ * In the case where nothing has been allocated ++ * from the high pointer the whole chunk could ++ * be backed unnecessarily - but the granularity ++ * is the chunk size anyway and any non-zero ++ * offset of low pointer from the start of the ++ * chunk would result in the whole chunk being ++ * backed. ++ */ ++ read_val = max(high_ptr, low_ptr); ++ ++ /* kbase_check_alloc_sizes() already satisfies ++ * this, but here to avoid future maintenance ++ * hazards ++ */ ++ WARN_ON(!is_power_of_2(extent_bytes)); ++ addr_end = ALIGN(read_val, extent_bytes); ++ } else { ++ addr_end = read_val = READ_ONCE(*ptr); ++ } ++ ++ if (addr_end >= (reg->start_pfn << PAGE_SHIFT)) ++ used_pages = PFN_UP(addr_end) - reg->start_pfn; ++ else ++ used_pages = reg->used_pages; ++ } ++ ++ trace_mali_jit_report(katom, reg, idx, read_val, used_pages); ++ kbase_trace_jit_report_gpu_mem(kctx, reg, 0u); ++ ++ /* We can never have used more pages than the VA size of the ++ * region ++ */ ++ if (used_pages > reg->nr_pages) { ++ dev_warn(kctx->kbdev->dev, ++ "%s: JIT id[%u]=%u start=0x%llx used_pages %llx > %zx (read 0x%llx as %s%s)\n", ++ __func__, idx, katom->jit_ids[idx], ++ reg->start_pfn << PAGE_SHIFT, ++ used_pages, reg->nr_pages, read_val, ++ (reg->flags & KBASE_REG_HEAP_INFO_IS_SIZE) ? ++ "size" : "addr", ++ (reg->flags & KBASE_REG_TILER_ALIGN_TOP) ? ++ " with align" : ""); ++ used_pages = reg->nr_pages; ++ } ++ /* Note: one real use case has an atom correctly reporting 0 ++ * pages in use. This happens in normal use-cases but may only ++ * happen for a few of the application's frames. ++ */ ++ ++ kbase_vunmap(kctx, &mapping); ++ ++ kbase_jit_report_update_pressure(kctx, reg, used_pages, 0u); ++ } ++ ++ kbase_jit_retry_pending_alloc(kctx); ++} ++#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ ++ ++/* ++ * Perform the necessary handling of an atom that has finished running ++ * on the GPU. ++ * ++ * Note that if this is a soft-job that has had kbase_prepare_soft_job called on it then the caller ++ * is responsible for calling kbase_finish_soft_job *before* calling this function. ++ * ++ * The caller must hold the kbase_jd_context.lock. ++ */ ++bool jd_done_nolock(struct kbase_jd_atom *katom, ++ struct list_head *completed_jobs_ctx) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ struct list_head completed_jobs; ++ struct list_head runnable_jobs; ++ bool need_to_try_schedule_context = false; ++ int i; ++ ++ INIT_LIST_HEAD(&completed_jobs); ++ INIT_LIST_HEAD(&runnable_jobs); ++ ++ KBASE_DEBUG_ASSERT(katom->status != KBASE_JD_ATOM_STATE_UNUSED); ++ ++#if MALI_JIT_PRESSURE_LIMIT_BASE ++ if (kbase_ctx_flag(kctx, KCTX_JPL_ENABLED)) ++ jd_update_jit_usage(katom); ++#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ ++ ++ /* This is needed in case an atom is failed due to being invalid, this ++ * can happen *before* the jobs that the atom depends on have completed */ ++ for (i = 0; i < 2; i++) { ++ if (kbase_jd_katom_dep_atom(&katom->dep[i])) { ++ list_del(&katom->dep_item[i]); ++ kbase_jd_katom_dep_clear(&katom->dep[i]); ++ } ++ } ++ ++ jd_mark_atom_complete(katom); ++ list_add_tail(&katom->jd_item, &completed_jobs); ++ ++ while (!list_empty(&completed_jobs)) { ++ katom = list_entry(completed_jobs.prev, struct kbase_jd_atom, jd_item); ++ list_del(completed_jobs.prev); ++ KBASE_DEBUG_ASSERT(katom->status == KBASE_JD_ATOM_STATE_COMPLETED); ++ ++ for (i = 0; i < 2; i++) ++ jd_resolve_dep(&runnable_jobs, katom, i, ++ kbase_ctx_flag(kctx, KCTX_DYING)); ++ ++ if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) ++ kbase_jd_post_external_resources(katom); ++ ++ while (!list_empty(&runnable_jobs)) { ++ struct kbase_jd_atom *node; ++ ++ node = list_entry(runnable_jobs.next, ++ struct kbase_jd_atom, jd_item); ++ list_del(runnable_jobs.next); ++ node->in_jd_list = false; ++ ++ dev_dbg(kctx->kbdev->dev, "List node %p has status %d\n", ++ node, node->status); ++ ++ KBASE_DEBUG_ASSERT(node->status != KBASE_JD_ATOM_STATE_UNUSED); ++ if (node->status == KBASE_JD_ATOM_STATE_IN_JS) ++ continue; ++ ++ if (node->status != KBASE_JD_ATOM_STATE_COMPLETED && ++ !kbase_ctx_flag(kctx, KCTX_DYING)) { ++ need_to_try_schedule_context |= jd_run_atom(node); ++ } else { ++ node->event_code = katom->event_code; ++ ++ if (node->core_req & ++ BASE_JD_REQ_SOFT_JOB) { ++ WARN_ON(!list_empty(&node->queue)); ++ kbase_finish_soft_job(node); ++ } ++ node->status = KBASE_JD_ATOM_STATE_COMPLETED; ++ } ++ ++ if (node->status == KBASE_JD_ATOM_STATE_COMPLETED) { ++ list_add_tail(&node->jd_item, &completed_jobs); ++ } else if (node->status == KBASE_JD_ATOM_STATE_IN_JS && ++ !node->will_fail_event_code) { ++ /* Node successfully submitted, try submitting ++ * dependencies as they may now be representable ++ * in JS */ ++ jd_try_submitting_deps(&runnable_jobs, node); ++ } ++ } ++ ++ /* Register a completed job as a disjoint event when the GPU ++ * is in a disjoint state (ie. being reset). ++ */ ++ kbase_disjoint_event_potential(kctx->kbdev); ++ if (completed_jobs_ctx) ++ list_add_tail(&katom->jd_item, completed_jobs_ctx); ++ else ++ kbase_event_post(kctx, katom); ++ ++ /* Decrement and check the TOTAL number of jobs. This includes ++ * those not tracked by the scheduler: 'not ready to run' and ++ * 'dependency-only' jobs. */ ++ if (--kctx->jctx.job_nr == 0) ++ wake_up(&kctx->jctx.zero_jobs_wait); /* All events are safely queued now, and we can signal any waiter ++ * that we've got no more jobs (so we can be safely terminated) */ ++ } ++ ++ return need_to_try_schedule_context; ++} ++ ++KBASE_EXPORT_TEST_API(jd_done_nolock); ++ ++#ifdef CONFIG_GPU_TRACEPOINTS ++enum { ++ CORE_REQ_DEP_ONLY, ++ CORE_REQ_SOFT, ++ CORE_REQ_COMPUTE, ++ CORE_REQ_FRAGMENT, ++ CORE_REQ_VERTEX, ++ CORE_REQ_TILER, ++ CORE_REQ_FRAGMENT_VERTEX, ++ CORE_REQ_FRAGMENT_VERTEX_TILER, ++ CORE_REQ_FRAGMENT_TILER, ++ CORE_REQ_VERTEX_TILER, ++ CORE_REQ_UNKNOWN ++}; ++static const char * const core_req_strings[] = { ++ "Dependency Only Job", ++ "Soft Job", ++ "Compute Shader Job", ++ "Fragment Shader Job", ++ "Vertex/Geometry Shader Job", ++ "Tiler Job", ++ "Fragment Shader + Vertex/Geometry Shader Job", ++ "Fragment Shader + Vertex/Geometry Shader Job + Tiler Job", ++ "Fragment Shader + Tiler Job", ++ "Vertex/Geometry Shader Job + Tiler Job", ++ "Unknown Job" ++}; ++static const char *kbasep_map_core_reqs_to_string(base_jd_core_req core_req) ++{ ++ if (core_req & BASE_JD_REQ_SOFT_JOB) ++ return core_req_strings[CORE_REQ_SOFT]; ++ if (core_req & BASE_JD_REQ_ONLY_COMPUTE) ++ return core_req_strings[CORE_REQ_COMPUTE]; ++ switch (core_req & (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T)) { ++ case BASE_JD_REQ_DEP: ++ return core_req_strings[CORE_REQ_DEP_ONLY]; ++ case BASE_JD_REQ_FS: ++ return core_req_strings[CORE_REQ_FRAGMENT]; ++ case BASE_JD_REQ_CS: ++ return core_req_strings[CORE_REQ_VERTEX]; ++ case BASE_JD_REQ_T: ++ return core_req_strings[CORE_REQ_TILER]; ++ case (BASE_JD_REQ_FS | BASE_JD_REQ_CS): ++ return core_req_strings[CORE_REQ_FRAGMENT_VERTEX]; ++ case (BASE_JD_REQ_FS | BASE_JD_REQ_T): ++ return core_req_strings[CORE_REQ_FRAGMENT_TILER]; ++ case (BASE_JD_REQ_CS | BASE_JD_REQ_T): ++ return core_req_strings[CORE_REQ_VERTEX_TILER]; ++ case (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T): ++ return core_req_strings[CORE_REQ_FRAGMENT_VERTEX_TILER]; ++ } ++ return core_req_strings[CORE_REQ_UNKNOWN]; ++} ++#endif ++ ++/* Trace an atom submission. */ ++static void jd_trace_atom_submit(struct kbase_context *const kctx, ++ struct kbase_jd_atom *const katom, ++ int *priority) ++{ ++ struct kbase_device *const kbdev = kctx->kbdev; ++ ++ KBASE_TLSTREAM_TL_NEW_ATOM(kbdev, katom, kbase_jd_atom_id(kctx, katom)); ++ KBASE_TLSTREAM_TL_RET_ATOM_CTX(kbdev, katom, kctx); ++ if (priority) ++ KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY(kbdev, katom, *priority); ++ KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(kbdev, katom, TL_ATOM_STATE_IDLE); ++ kbase_kinstr_jm_atom_queue(katom); ++} ++ ++static bool jd_submit_atom(struct kbase_context *const kctx, ++ const struct base_jd_atom *const user_atom, ++ const struct base_jd_fragment *const user_jc_incr, ++ struct kbase_jd_atom *const katom) ++{ ++ struct kbase_device *kbdev = kctx->kbdev; ++ struct kbase_jd_context *jctx = &kctx->jctx; ++ int queued = 0; ++ int i; ++ int sched_prio; ++ bool will_fail = false; ++ unsigned long flags; ++ enum kbase_jd_atom_state status; ++ ++ dev_dbg(kbdev->dev, "User did JD submit atom %p\n", (void *)katom); ++ ++ /* Update the TOTAL number of jobs. This includes those not tracked by ++ * the scheduler: 'not ready to run' and 'dependency-only' jobs. */ ++ jctx->job_nr++; ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) ++ katom->start_timestamp.tv64 = 0; ++#else ++ katom->start_timestamp = 0; ++#endif ++ katom->udata = user_atom->udata; ++ katom->kctx = kctx; ++ katom->nr_extres = user_atom->nr_extres; ++ katom->extres = NULL; ++ katom->device_nr = user_atom->device_nr; ++ katom->jc = user_atom->jc; ++ katom->core_req = user_atom->core_req; ++ katom->jobslot = user_atom->jobslot; ++ katom->seq_nr = user_atom->seq_nr; ++ katom->atom_flags = 0; ++ katom->retry_count = 0; ++ katom->need_cache_flush_cores_retained = 0; ++ katom->pre_dep = NULL; ++ katom->post_dep = NULL; ++ katom->x_pre_dep = NULL; ++ katom->x_post_dep = NULL; ++ katom->will_fail_event_code = BASE_JD_EVENT_NOT_STARTED; ++ katom->softjob_data = NULL; ++ ++ trace_sysgraph(SGR_ARRIVE, kctx->id, user_atom->atom_number); ++ ++#if MALI_JIT_PRESSURE_LIMIT_BASE ++ /* Older API version atoms might have random values where jit_id now ++ * lives, but we must maintain backwards compatibility - handle the ++ * issue. ++ */ ++ if (!mali_kbase_supports_jit_pressure_limit(kctx->api_version)) { ++ katom->jit_ids[0] = 0; ++ katom->jit_ids[1] = 0; ++ } else { ++ katom->jit_ids[0] = user_atom->jit_id[0]; ++ katom->jit_ids[1] = user_atom->jit_id[1]; ++ } ++#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ ++ ++ katom->renderpass_id = user_atom->renderpass_id; ++ ++ /* Implicitly sets katom->protected_state.enter as well. */ ++ katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK; ++ ++ katom->age = kctx->age_count++; ++ ++ INIT_LIST_HEAD(&katom->queue); ++ INIT_LIST_HEAD(&katom->jd_item); ++#ifdef CONFIG_MALI_BIFROST_DMA_FENCE ++ kbase_fence_dep_count_set(katom, -1); ++#endif ++ ++ /* Don't do anything if there is a mess up with dependencies. ++ This is done in a separate cycle to check both the dependencies at ones, otherwise ++ it will be extra complexity to deal with 1st dependency ( just added to the list ) ++ if only the 2nd one has invalid config. ++ */ ++ for (i = 0; i < 2; i++) { ++ int dep_atom_number = user_atom->pre_dep[i].atom_id; ++ base_jd_dep_type dep_atom_type = user_atom->pre_dep[i].dependency_type; ++ ++ if (dep_atom_number) { ++ if (dep_atom_type != BASE_JD_DEP_TYPE_ORDER && ++ dep_atom_type != BASE_JD_DEP_TYPE_DATA) { ++ katom->event_code = BASE_JD_EVENT_JOB_CONFIG_FAULT; ++ katom->status = KBASE_JD_ATOM_STATE_COMPLETED; ++ dev_dbg(kbdev->dev, ++ "Atom %p status to completed\n", ++ (void *)katom); ++ ++ /* Wrong dependency setup. Atom will be sent ++ * back to user space. Do not record any ++ * dependencies. */ ++ jd_trace_atom_submit(kctx, katom, NULL); ++ ++ return jd_done_nolock(katom, NULL); ++ } ++ } ++ } ++ ++ /* Add dependencies */ ++ for (i = 0; i < 2; i++) { ++ int dep_atom_number = user_atom->pre_dep[i].atom_id; ++ base_jd_dep_type dep_atom_type; ++ struct kbase_jd_atom *dep_atom = &jctx->atoms[dep_atom_number]; ++ ++ dep_atom_type = user_atom->pre_dep[i].dependency_type; ++ kbase_jd_katom_dep_clear(&katom->dep[i]); ++ ++ if (!dep_atom_number) ++ continue; ++ ++ if (dep_atom->status == KBASE_JD_ATOM_STATE_UNUSED || ++ dep_atom->status == KBASE_JD_ATOM_STATE_COMPLETED) { ++ ++ if (dep_atom->event_code == BASE_JD_EVENT_DONE) ++ continue; ++ /* don't stop this atom if it has an order dependency ++ * only to the failed one, try to submit it through ++ * the normal path ++ */ ++ if (dep_atom_type == BASE_JD_DEP_TYPE_ORDER && ++ dep_atom->event_code > BASE_JD_EVENT_ACTIVE) { ++ continue; ++ } ++ ++ /* Atom has completed, propagate the error code if any */ ++ katom->event_code = dep_atom->event_code; ++ katom->status = KBASE_JD_ATOM_STATE_QUEUED; ++ dev_dbg(kbdev->dev, "Atom %p status to queued\n", ++ (void *)katom); ++ ++ /* This atom will be sent back to user space. ++ * Do not record any dependencies. ++ */ ++ jd_trace_atom_submit(kctx, katom, NULL); ++ ++ will_fail = true; ++ ++ } else { ++ /* Atom is in progress, add this atom to the list */ ++ list_add_tail(&katom->dep_item[i], &dep_atom->dep_head[i]); ++ kbase_jd_katom_dep_set(&katom->dep[i], dep_atom, dep_atom_type); ++ queued = 1; ++ } ++ } ++ ++ if (will_fail) { ++ if (!queued) { ++ if (katom->core_req & BASE_JD_REQ_SOFT_JOB) { ++ /* This softjob has failed due to a previous ++ * dependency, however we should still run the ++ * prepare & finish functions ++ */ ++ int err = kbase_prepare_soft_job(katom); ++ ++ if (err >= 0) ++ kbase_finish_soft_job(katom); ++ } ++ ++ return jd_done_nolock(katom, NULL); ++ } ++ } ++ ++ /* These must occur after the above loop to ensure that an atom ++ * that depends on a previous atom with the same number behaves ++ * as expected ++ */ ++ katom->event_code = BASE_JD_EVENT_DONE; ++ katom->status = KBASE_JD_ATOM_STATE_QUEUED; ++ dev_dbg(kbdev->dev, "Atom %p status to queued\n", (void *)katom); ++ ++ /* For invalid priority, be most lenient and choose the default */ ++ sched_prio = kbasep_js_atom_prio_to_sched_prio(user_atom->prio); ++ if (sched_prio == KBASE_JS_ATOM_SCHED_PRIO_INVALID) ++ sched_prio = KBASE_JS_ATOM_SCHED_PRIO_DEFAULT; ++ katom->sched_priority = sched_prio; ++ ++ /* Create a new atom. */ ++ jd_trace_atom_submit(kctx, katom, &katom->sched_priority); ++ ++#if !MALI_INCREMENTAL_RENDERING ++ /* Reject atoms for incremental rendering if not supported */ ++ if (katom->core_req & ++ (BASE_JD_REQ_START_RENDERPASS|BASE_JD_REQ_END_RENDERPASS)) { ++ dev_err(kctx->kbdev->dev, ++ "Rejecting atom with unsupported core_req 0x%x\n", ++ katom->core_req); ++ katom->event_code = BASE_JD_EVENT_JOB_INVALID; ++ return jd_done_nolock(katom, NULL); ++ } ++#endif /* !MALI_INCREMENTAL_RENDERING */ ++ ++ if (katom->core_req & BASE_JD_REQ_END_RENDERPASS) { ++ WARN_ON(katom->jc != 0); ++ katom->jc_fragment = *user_jc_incr; ++ } else if (!katom->jc && ++ (katom->core_req & BASE_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_DEP) { ++ /* Reject atoms with job chain = NULL, as these cause issues ++ * with soft-stop ++ */ ++ dev_err(kctx->kbdev->dev, "Rejecting atom with jc = NULL\n"); ++ katom->event_code = BASE_JD_EVENT_JOB_INVALID; ++ return jd_done_nolock(katom, NULL); ++ } ++ ++ /* Reject atoms with an invalid device_nr */ ++ if ((katom->core_req & BASE_JD_REQ_SPECIFIC_COHERENT_GROUP) && ++ (katom->device_nr >= kctx->kbdev->gpu_props.num_core_groups)) { ++ dev_err(kctx->kbdev->dev, ++ "Rejecting atom with invalid device_nr %d\n", ++ katom->device_nr); ++ katom->event_code = BASE_JD_EVENT_JOB_INVALID; ++ return jd_done_nolock(katom, NULL); ++ } ++ ++ /* Reject atoms with invalid core requirements */ ++ if ((katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) && ++ (katom->core_req & BASE_JD_REQ_EVENT_COALESCE)) { ++ dev_err(kctx->kbdev->dev, ++ "Rejecting atom with invalid core requirements\n"); ++ katom->event_code = BASE_JD_EVENT_JOB_INVALID; ++ katom->core_req &= ~BASE_JD_REQ_EVENT_COALESCE; ++ return jd_done_nolock(katom, NULL); ++ } ++ ++ /* Reject soft-job atom of certain types from accessing external resources */ ++ if ((katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) && ++ (((katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) == BASE_JD_REQ_SOFT_FENCE_WAIT) || ++ ((katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) == BASE_JD_REQ_SOFT_JIT_ALLOC) || ++ ((katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) == BASE_JD_REQ_SOFT_JIT_FREE))) { ++ dev_err(kctx->kbdev->dev, ++ "Rejecting soft-job atom accessing external resources\n"); ++ katom->event_code = BASE_JD_EVENT_JOB_INVALID; ++ return jd_done_nolock(katom, NULL); ++ } ++ ++ if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) { ++ /* handle what we need to do to access the external resources */ ++ if (kbase_jd_pre_external_resources(katom, user_atom) != 0) { ++ /* setup failed (no access, bad resource, unknown resource types, etc.) */ ++ katom->event_code = BASE_JD_EVENT_JOB_INVALID; ++ return jd_done_nolock(katom, NULL); ++ } ++ } ++ ++#if !MALI_JIT_PRESSURE_LIMIT_BASE ++ if (mali_kbase_supports_jit_pressure_limit(kctx->api_version) && ++ (user_atom->jit_id[0] || user_atom->jit_id[1])) { ++ /* JIT pressure limit is disabled, but we are receiving non-0 ++ * JIT IDs - atom is invalid. ++ */ ++ katom->event_code = BASE_JD_EVENT_JOB_INVALID; ++ return jd_done_nolock(katom, NULL); ++ } ++#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ ++ ++ /* Validate the atom. Function will return error if the atom is ++ * malformed. ++ * ++ * Soft-jobs never enter the job scheduler but have their own initialize method. ++ * ++ * If either fail then we immediately complete the atom with an error. ++ */ ++ if ((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0) { ++ if (!kbase_js_is_atom_valid(kctx->kbdev, katom)) { ++ katom->event_code = BASE_JD_EVENT_JOB_INVALID; ++ return jd_done_nolock(katom, NULL); ++ } ++ } else { ++ /* Soft-job */ ++ if (kbase_prepare_soft_job(katom) != 0) { ++ katom->event_code = BASE_JD_EVENT_JOB_INVALID; ++ return jd_done_nolock(katom, NULL); ++ } ++ } ++ ++#ifdef CONFIG_GPU_TRACEPOINTS ++ katom->work_id = atomic_inc_return(&jctx->work_id); ++ trace_gpu_job_enqueue(kctx->id, katom->work_id, ++ kbasep_map_core_reqs_to_string(katom->core_req)); ++#endif ++ ++ if (queued && !IS_GPU_ATOM(katom)) ++ return false; ++ ++#ifdef CONFIG_MALI_BIFROST_DMA_FENCE ++ if (kbase_fence_dep_count_read(katom) != -1) ++ return false; ++ ++#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ ++ ++ if (katom->core_req & BASE_JD_REQ_SOFT_JOB) { ++ if (kbase_process_soft_job(katom) == 0) { ++ kbase_finish_soft_job(katom); ++ return jd_done_nolock(katom, NULL); ++ } ++ return false; ++ } ++ ++ if ((katom->core_req & BASE_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_DEP) { ++ bool need_to_try_schedule_context; ++ ++ katom->status = KBASE_JD_ATOM_STATE_IN_JS; ++ dev_dbg(kctx->kbdev->dev, "Atom %p status to in JS\n", ++ (void *)katom); ++ ++ need_to_try_schedule_context = kbasep_js_add_job(kctx, katom); ++ /* If job was cancelled then resolve immediately */ ++ if (katom->event_code != BASE_JD_EVENT_JOB_CANCELLED) ++ return need_to_try_schedule_context; ++ ++ /* Synchronize with backend reset */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ status = katom->status; ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ if (status == KBASE_JD_ATOM_STATE_HW_COMPLETED) { ++ dev_dbg(kctx->kbdev->dev, ++ "Atom %d cancelled on HW\n", ++ kbase_jd_atom_id(katom->kctx, katom)); ++ return need_to_try_schedule_context; ++ } ++ } ++ ++ /* This is a pure dependency. Resolve it immediately */ ++ return jd_done_nolock(katom, NULL); ++} ++ ++int kbase_jd_submit(struct kbase_context *kctx, ++ void __user *user_addr, u32 nr_atoms, u32 stride, ++ bool uk6_atom) ++{ ++ struct kbase_jd_context *jctx = &kctx->jctx; ++ int err = 0; ++ int i; ++ bool need_to_try_schedule_context = false; ++ struct kbase_device *kbdev; ++ u32 latest_flush; ++ ++ bool jd_atom_is_v2 = (stride == sizeof(struct base_jd_atom_v2) || ++ stride == offsetof(struct base_jd_atom_v2, renderpass_id)); ++ ++ /* ++ * kbase_jd_submit isn't expected to fail and so all errors with the ++ * jobs are reported by immediately failing them (through event system) ++ */ ++ kbdev = kctx->kbdev; ++ ++ beenthere(kctx, "%s", "Enter"); ++ ++ if (kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) { ++ dev_err(kbdev->dev, "Attempt to submit to a context that has SUBMIT_DISABLED set on it\n"); ++ return -EINVAL; ++ } ++ ++ if (stride != offsetof(struct base_jd_atom_v2, renderpass_id) && ++ stride != sizeof(struct base_jd_atom_v2) && ++ stride != offsetof(struct base_jd_atom, renderpass_id) && ++ stride != sizeof(struct base_jd_atom)) { ++ dev_err(kbdev->dev, ++ "Stride %u passed to job_submit isn't supported by the kernel\n", ++ stride); ++ return -EINVAL; ++ } ++ ++ /* All atoms submitted in this call have the same flush ID */ ++ latest_flush = kbase_backend_get_current_flush_id(kbdev); ++ ++ for (i = 0; i < nr_atoms; i++) { ++ struct base_jd_atom user_atom; ++ struct base_jd_fragment user_jc_incr; ++ struct kbase_jd_atom *katom; ++ ++ if (unlikely(jd_atom_is_v2)) { ++ if (copy_from_user(&user_atom.jc, user_addr, sizeof(struct base_jd_atom_v2)) != 0) { ++ dev_err(kbdev->dev, ++ "Invalid atom address %p passed to job_submit\n", ++ user_addr); ++ err = -EFAULT; ++ break; ++ } ++ ++ /* no seq_nr in v2 */ ++ user_atom.seq_nr = 0; ++ } else { ++ if (copy_from_user(&user_atom, user_addr, stride) != 0) { ++ dev_err(kbdev->dev, ++ "Invalid atom address %p passed to job_submit\n", ++ user_addr); ++ err = -EFAULT; ++ break; ++ } ++ } ++ ++ if (stride == offsetof(struct base_jd_atom_v2, renderpass_id)) { ++ dev_dbg(kbdev->dev, "No renderpass ID: use 0\n"); ++ user_atom.renderpass_id = 0; ++ } else { ++ /* Ensure all padding bytes are 0 for potential future ++ * extension ++ */ ++ size_t j; ++ ++ dev_dbg(kbdev->dev, "Renderpass ID is %d\n", ++ user_atom.renderpass_id); ++ for (j = 0; j < sizeof(user_atom.padding); j++) { ++ if (user_atom.padding[j]) { ++ dev_err(kbdev->dev, ++ "Bad padding byte %zu: %d\n", ++ j, user_atom.padding[j]); ++ err = -EINVAL; ++ break; ++ } ++ } ++ if (err) ++ break; ++ } ++ ++ /* In this case 'jc' is the CPU address of a struct ++ * instead of a GPU address of a job chain. ++ */ ++ if (user_atom.core_req & BASE_JD_REQ_END_RENDERPASS) { ++ if (copy_from_user(&user_jc_incr, ++ u64_to_user_ptr(user_atom.jc), ++ sizeof(user_jc_incr))) { ++ dev_err(kbdev->dev, ++ "Invalid jc address 0x%llx passed to job_submit\n", ++ user_atom.jc); ++ err = -EFAULT; ++ break; ++ } ++ dev_dbg(kbdev->dev, "Copied IR jobchain addresses\n"); ++ user_atom.jc = 0; ++ } ++ ++ user_addr = (void __user *)((uintptr_t) user_addr + stride); ++ ++ mutex_lock(&jctx->lock); ++#ifndef compiletime_assert ++#define compiletime_assert_defined ++#define compiletime_assert(x, msg) do { switch (0) { case 0: case (x):; } } \ ++while (false) ++#endif ++ compiletime_assert((1 << (8*sizeof(user_atom.atom_number))) == ++ BASE_JD_ATOM_COUNT, ++ "BASE_JD_ATOM_COUNT and base_atom_id type out of sync"); ++ compiletime_assert(sizeof(user_atom.pre_dep[0].atom_id) == ++ sizeof(user_atom.atom_number), ++ "BASE_JD_ATOM_COUNT and base_atom_id type out of sync"); ++#ifdef compiletime_assert_defined ++#undef compiletime_assert ++#undef compiletime_assert_defined ++#endif ++ katom = &jctx->atoms[user_atom.atom_number]; ++ ++ /* Record the flush ID for the cache flush optimisation */ ++ katom->flush_id = latest_flush; ++ ++ while (katom->status != KBASE_JD_ATOM_STATE_UNUSED) { ++ /* Atom number is already in use, wait for the atom to ++ * complete ++ */ ++ mutex_unlock(&jctx->lock); ++ ++ /* This thread will wait for the atom to complete. Due ++ * to thread scheduling we are not sure that the other ++ * thread that owns the atom will also schedule the ++ * context, so we force the scheduler to be active and ++ * hence eventually schedule this context at some point ++ * later. ++ */ ++ kbase_js_sched_all(kbdev); ++ ++ if (wait_event_killable(katom->completed, ++ katom->status == ++ KBASE_JD_ATOM_STATE_UNUSED) != 0) { ++ /* We're being killed so the result code ++ * doesn't really matter ++ */ ++ return 0; ++ } ++ mutex_lock(&jctx->lock); ++ } ++ ++ need_to_try_schedule_context |= jd_submit_atom(kctx, &user_atom, ++ &user_jc_incr, katom); ++ ++ /* Register a completed job as a disjoint event when the GPU is in a disjoint state ++ * (ie. being reset). ++ */ ++ kbase_disjoint_event_potential(kbdev); ++ ++ mutex_unlock(&jctx->lock); ++ } ++ ++ if (need_to_try_schedule_context) ++ kbase_js_sched_all(kbdev); ++ ++ return err; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_jd_submit); ++ ++void kbase_jd_done_worker(struct work_struct *data) ++{ ++ struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, work); ++ struct kbase_jd_context *jctx; ++ struct kbase_context *kctx; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ struct kbase_device *kbdev; ++ struct kbasep_js_device_data *js_devdata; ++ u64 cache_jc = katom->jc; ++ struct kbasep_js_atom_retained_state katom_retained_state; ++ bool context_idle; ++ base_jd_core_req core_req = katom->core_req; ++ ++ /* Soft jobs should never reach this function */ ++ KBASE_DEBUG_ASSERT((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0); ++ ++ kctx = katom->kctx; ++ jctx = &kctx->jctx; ++ kbdev = kctx->kbdev; ++ js_kctx_info = &kctx->jctx.sched_info; ++ js_devdata = &kbdev->js_data; ++ ++ dev_dbg(kbdev->dev, "Enter atom %p done worker for kctx %p\n", ++ (void *)katom, (void *)kctx); ++ ++ KBASE_KTRACE_ADD_JM(kbdev, JD_DONE_WORKER, kctx, katom, katom->jc, 0); ++ ++ kbase_backend_complete_wq(kbdev, katom); ++ ++ /* ++ * Begin transaction on JD context and JS context ++ */ ++ mutex_lock(&jctx->lock); ++ KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(kbdev, katom, TL_ATOM_STATE_DONE); ++ mutex_lock(&js_devdata->queue_mutex); ++ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); ++ ++ /* This worker only gets called on contexts that are scheduled *in*. This is ++ * because it only happens in response to an IRQ from a job that was ++ * running. ++ */ ++ KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ ++ if (katom->event_code == BASE_JD_EVENT_STOPPED) { ++ unsigned long flags; ++ ++ dev_dbg(kbdev->dev, "Atom %p has been promoted to stopped\n", ++ (void *)katom); ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ mutex_unlock(&js_devdata->queue_mutex); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ katom->status = KBASE_JD_ATOM_STATE_IN_JS; ++ dev_dbg(kctx->kbdev->dev, "Atom %p status to in JS\n", ++ (void *)katom); ++ kbase_js_unpull(kctx, katom); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&jctx->lock); ++ ++ return; ++ } ++ ++ if ((katom->event_code != BASE_JD_EVENT_DONE) && ++ (!kbase_ctx_flag(katom->kctx, KCTX_DYING))) ++ dev_err(kbdev->dev, ++ "t6xx: GPU fault 0x%02lx from job slot %d\n", ++ (unsigned long)katom->event_code, ++ katom->slot_nr); ++ ++ /* Retain state before the katom disappears */ ++ kbasep_js_atom_retained_state_copy(&katom_retained_state, katom); ++ ++ context_idle = kbase_js_complete_atom_wq(kctx, katom); ++ ++ KBASE_DEBUG_ASSERT(kbasep_js_has_atom_finished(&katom_retained_state)); ++ ++ kbasep_js_remove_job(kbdev, kctx, katom); ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ mutex_unlock(&js_devdata->queue_mutex); ++ katom->atom_flags &= ~KBASE_KATOM_FLAG_HOLDING_CTX_REF; ++ /* jd_done_nolock() requires the jsctx_mutex lock to be dropped */ ++ jd_done_nolock(katom, &kctx->completed_jobs); ++ ++ /* katom may have been freed now, do not use! */ ++ ++ if (context_idle) { ++ unsigned long flags; ++ ++ context_idle = false; ++ mutex_lock(&js_devdata->queue_mutex); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ /* If kbase_sched() has scheduled this context back in then ++ * KCTX_ACTIVE will have been set after we marked it as ++ * inactive, and another pm reference will have been taken, so ++ * drop our reference. But do not call kbase_jm_idle_ctx(), as ++ * the context is active and fast-starting is allowed. ++ * ++ * If an atom has been fast-started then kctx->atoms_pulled will ++ * be non-zero but KCTX_ACTIVE will still be false (as the ++ * previous pm reference has been inherited). Do NOT drop our ++ * reference, as it has been re-used, and leave the context as ++ * active. ++ * ++ * If no new atoms have been started then KCTX_ACTIVE will still ++ * be false and atoms_pulled will be zero, so drop the reference ++ * and call kbase_jm_idle_ctx(). ++ * ++ * As the checks are done under both the queue_mutex and ++ * hwaccess_lock is should be impossible for this to race ++ * with the scheduler code. ++ */ ++ if (kbase_ctx_flag(kctx, KCTX_ACTIVE) || ++ !atomic_read(&kctx->atoms_pulled)) { ++ /* Calling kbase_jm_idle_ctx() here will ensure that ++ * atoms are not fast-started when we drop the ++ * hwaccess_lock. This is not performed if ++ * KCTX_ACTIVE is set as in that case another pm ++ * reference has been taken and a fast-start would be ++ * valid. ++ */ ++ if (!kbase_ctx_flag(kctx, KCTX_ACTIVE)) ++ kbase_jm_idle_ctx(kbdev, kctx); ++ context_idle = true; ++ } else { ++ kbase_ctx_flag_set(kctx, KCTX_ACTIVE); ++ } ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&js_devdata->queue_mutex); ++ } ++ ++ /* ++ * Transaction complete ++ */ ++ mutex_unlock(&jctx->lock); ++ ++ /* Job is now no longer running, so can now safely release the context ++ * reference, and handle any actions that were logged against the atom's retained state */ ++ ++ kbasep_js_runpool_release_ctx_and_katom_retained_state(kbdev, kctx, &katom_retained_state); ++ ++ kbase_js_sched_all(kbdev); ++ ++ if (!atomic_dec_return(&kctx->work_count)) { ++ /* If worker now idle then post all events that jd_done_nolock() ++ * has queued */ ++ mutex_lock(&jctx->lock); ++ while (!list_empty(&kctx->completed_jobs)) { ++ struct kbase_jd_atom *atom = list_entry( ++ kctx->completed_jobs.next, ++ struct kbase_jd_atom, jd_item); ++ list_del(kctx->completed_jobs.next); ++ ++ kbase_event_post(kctx, atom); ++ } ++ mutex_unlock(&jctx->lock); ++ } ++ ++ kbase_backend_complete_wq_post_sched(kbdev, core_req); ++ ++ if (context_idle) ++ kbase_pm_context_idle(kbdev); ++ ++ KBASE_KTRACE_ADD_JM(kbdev, JD_DONE_WORKER_END, kctx, NULL, cache_jc, 0); ++ ++ dev_dbg(kbdev->dev, "Leave atom %p done worker for kctx %p\n", ++ (void *)katom, (void *)kctx); ++} ++ ++/** ++ * jd_cancel_worker - Work queue job cancel function. ++ * @data: a &struct work_struct ++ * ++ * Only called as part of 'Zapping' a context (which occurs on termination). ++ * Operates serially with the kbase_jd_done_worker() on the work queue. ++ * ++ * This can only be called on contexts that aren't scheduled. ++ * ++ * We don't need to release most of the resources that would occur on ++ * kbase_jd_done() or kbase_jd_done_worker(), because the atoms here must not be ++ * running (by virtue of only being called on contexts that aren't ++ * scheduled). ++ */ ++static void jd_cancel_worker(struct work_struct *data) ++{ ++ struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, work); ++ struct kbase_jd_context *jctx; ++ struct kbase_context *kctx; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ bool need_to_try_schedule_context; ++ bool attr_state_changed; ++ struct kbase_device *kbdev; ++ ++ /* Soft jobs should never reach this function */ ++ KBASE_DEBUG_ASSERT((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0); ++ ++ kctx = katom->kctx; ++ kbdev = kctx->kbdev; ++ jctx = &kctx->jctx; ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ KBASE_KTRACE_ADD_JM(kbdev, JD_CANCEL_WORKER, kctx, katom, katom->jc, 0); ++ ++ /* This only gets called on contexts that are scheduled out. Hence, we must ++ * make sure we don't de-ref the number of running jobs (there aren't ++ * any), nor must we try to schedule out the context (it's already ++ * scheduled out). ++ */ ++ KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ ++ /* Scheduler: Remove the job from the system */ ++ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); ++ attr_state_changed = kbasep_js_remove_cancelled_job(kbdev, kctx, katom); ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ ++ mutex_lock(&jctx->lock); ++ ++ need_to_try_schedule_context = jd_done_nolock(katom, NULL); ++ /* Because we're zapping, we're not adding any more jobs to this ctx, so no need to ++ * schedule the context. There's also no need for the jsctx_mutex to have been taken ++ * around this too. */ ++ KBASE_DEBUG_ASSERT(!need_to_try_schedule_context); ++ ++ /* katom may have been freed now, do not use! */ ++ mutex_unlock(&jctx->lock); ++ ++ if (attr_state_changed) ++ kbase_js_sched_all(kbdev); ++} ++ ++/** ++ * kbase_jd_done - Complete a job that has been removed from the Hardware ++ * @katom: atom which has been completed ++ * @slot_nr: slot the atom was on ++ * @end_timestamp: completion time ++ * @done_code: completion code ++ * ++ * This must be used whenever a job has been removed from the Hardware, e.g.: ++ * An IRQ indicates that the job finished (for both error and 'done' codes), or ++ * the job was evicted from the JS_HEAD_NEXT registers during a Soft/Hard stop. ++ * ++ * Some work is carried out immediately, and the rest is deferred onto a ++ * workqueue ++ * ++ * Context: ++ * This can be called safely from atomic context. ++ * The caller must hold kbdev->hwaccess_lock ++ */ ++void kbase_jd_done(struct kbase_jd_atom *katom, int slot_nr, ++ ktime_t *end_timestamp, kbasep_js_atom_done_code done_code) ++{ ++ struct kbase_context *kctx; ++ struct kbase_device *kbdev; ++ ++ KBASE_DEBUG_ASSERT(katom); ++ kctx = katom->kctx; ++ KBASE_DEBUG_ASSERT(kctx); ++ kbdev = kctx->kbdev; ++ KBASE_DEBUG_ASSERT(kbdev); ++ ++ if (done_code & KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT) ++ katom->event_code = BASE_JD_EVENT_REMOVED_FROM_NEXT; ++ ++ KBASE_KTRACE_ADD_JM(kbdev, JD_DONE, kctx, katom, katom->jc, 0); ++ ++ kbase_job_check_leave_disjoint(kbdev, katom); ++ ++ katom->slot_nr = slot_nr; ++ ++ atomic_inc(&kctx->work_count); ++ ++#ifdef CONFIG_DEBUG_FS ++ /* a failed job happened and is waiting for dumping*/ ++ if (!katom->will_fail_event_code && ++ kbase_debug_job_fault_process(katom, katom->event_code)) ++ return; ++#endif ++ ++ WARN_ON(work_pending(&katom->work)); ++ INIT_WORK(&katom->work, kbase_jd_done_worker); ++ queue_work(kctx->jctx.job_done_wq, &katom->work); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_jd_done); ++ ++void kbase_jd_cancel(struct kbase_device *kbdev, struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx; ++ ++ KBASE_DEBUG_ASSERT(NULL != kbdev); ++ KBASE_DEBUG_ASSERT(NULL != katom); ++ kctx = katom->kctx; ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ ++ dev_dbg(kbdev->dev, "JD: cancelling atom %p\n", (void *)katom); ++ KBASE_KTRACE_ADD_JM(kbdev, JD_CANCEL, kctx, katom, katom->jc, 0); ++ ++ /* This should only be done from a context that is not scheduled */ ++ KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ ++ WARN_ON(work_pending(&katom->work)); ++ ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ ++ INIT_WORK(&katom->work, jd_cancel_worker); ++ queue_work(kctx->jctx.job_done_wq, &katom->work); ++} ++ ++ ++void kbase_jd_zap_context(struct kbase_context *kctx) ++{ ++ struct kbase_jd_atom *katom; ++ struct list_head *entry, *tmp; ++ struct kbase_device *kbdev; ++ ++ KBASE_DEBUG_ASSERT(kctx); ++ ++ kbdev = kctx->kbdev; ++ ++ KBASE_KTRACE_ADD_JM(kbdev, JD_ZAP_CONTEXT, kctx, NULL, 0u, 0u); ++ ++ kbase_js_zap_context(kctx); ++ ++ mutex_lock(&kctx->jctx.lock); ++ ++ /* ++ * While holding the struct kbase_jd_context lock clean up jobs which are known to kbase but are ++ * queued outside the job scheduler. ++ */ ++ ++ del_timer_sync(&kctx->soft_job_timeout); ++ list_for_each_safe(entry, tmp, &kctx->waiting_soft_jobs) { ++ katom = list_entry(entry, struct kbase_jd_atom, queue); ++ kbase_cancel_soft_job(katom); ++ } ++ ++ ++#ifdef CONFIG_MALI_BIFROST_DMA_FENCE ++ kbase_dma_fence_cancel_all_atoms(kctx); ++#endif ++ ++ mutex_unlock(&kctx->jctx.lock); ++ ++#ifdef CONFIG_MALI_BIFROST_DMA_FENCE ++ /* Flush dma-fence workqueue to ensure that any callbacks that may have ++ * been queued are done before continuing. ++ */ ++ flush_workqueue(kctx->dma_fence.wq); ++#endif ++ ++#ifdef CONFIG_DEBUG_FS ++ kbase_debug_job_fault_kctx_unblock(kctx); ++#endif ++ ++ kbase_jm_wait_for_zero_jobs(kctx); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_jd_zap_context); ++ ++int kbase_jd_init(struct kbase_context *kctx) ++{ ++ int i; ++ int mali_err = 0; ++ ++ KBASE_DEBUG_ASSERT(kctx); ++ ++ kctx->jctx.job_done_wq = alloc_workqueue("mali_jd", ++ WQ_HIGHPRI | WQ_UNBOUND, 1); ++ if (NULL == kctx->jctx.job_done_wq) { ++ mali_err = -ENOMEM; ++ goto out1; ++ } ++ ++ for (i = 0; i < BASE_JD_ATOM_COUNT; i++) { ++ init_waitqueue_head(&kctx->jctx.atoms[i].completed); ++ ++ INIT_LIST_HEAD(&kctx->jctx.atoms[i].dep_head[0]); ++ INIT_LIST_HEAD(&kctx->jctx.atoms[i].dep_head[1]); ++ ++ /* Catch userspace attempting to use an atom which doesn't exist as a pre-dependency */ ++ kctx->jctx.atoms[i].event_code = BASE_JD_EVENT_JOB_INVALID; ++ kctx->jctx.atoms[i].status = KBASE_JD_ATOM_STATE_UNUSED; ++ ++#if defined(CONFIG_MALI_BIFROST_DMA_FENCE) || defined(CONFIG_SYNC_FILE) ++ kctx->jctx.atoms[i].dma_fence.context = ++ dma_fence_context_alloc(1); ++ atomic_set(&kctx->jctx.atoms[i].dma_fence.seqno, 0); ++ INIT_LIST_HEAD(&kctx->jctx.atoms[i].dma_fence.callbacks); ++#endif ++ } ++ ++ for (i = 0; i < BASE_JD_RP_COUNT; i++) ++ kctx->jctx.renderpasses[i].state = KBASE_JD_RP_COMPLETE; ++ ++ mutex_init(&kctx->jctx.lock); ++ ++ init_waitqueue_head(&kctx->jctx.zero_jobs_wait); ++ ++ spin_lock_init(&kctx->jctx.tb_lock); ++ ++ kctx->jctx.job_nr = 0; ++ INIT_LIST_HEAD(&kctx->completed_jobs); ++ atomic_set(&kctx->work_count, 0); ++ ++ return 0; ++ ++ out1: ++ return mali_err; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_jd_init); ++ ++void kbase_jd_exit(struct kbase_context *kctx) ++{ ++ KBASE_DEBUG_ASSERT(kctx); ++ ++ /* Work queue is emptied by this */ ++ destroy_workqueue(kctx->jctx.job_done_wq); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_jd_exit); +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_jd_debugfs.c b/drivers/gpu/arm/bifrost/mali_kbase_jd_debugfs.c +new file mode 100755 +index 000000000000..6b0c36d6b93f +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_jd_debugfs.c +@@ -0,0 +1,250 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifdef CONFIG_DEBUG_FS ++ ++#include ++#include ++#include ++#include ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++#include ++#endif ++#include ++ ++struct kbase_jd_debugfs_depinfo { ++ u8 id; ++ char type; ++}; ++ ++static void kbase_jd_debugfs_fence_info(struct kbase_jd_atom *atom, ++ struct seq_file *sfile) ++{ ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ struct kbase_sync_fence_info info; ++ int res; ++ ++ switch (atom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { ++ case BASE_JD_REQ_SOFT_FENCE_TRIGGER: ++ res = kbase_sync_fence_out_info_get(atom, &info); ++ if (res == 0) ++ seq_printf(sfile, "Sa([%p]%d) ", ++ info.fence, info.status); ++ break; ++ case BASE_JD_REQ_SOFT_FENCE_WAIT: ++ res = kbase_sync_fence_in_info_get(atom, &info); ++ if (res == 0) ++ seq_printf(sfile, "Wa([%p]%d) ", ++ info.fence, info.status); ++ break; ++ default: ++ break; ++ } ++#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ ++ ++#ifdef CONFIG_MALI_BIFROST_DMA_FENCE ++ if (atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) { ++ struct kbase_fence_cb *cb; ++ ++ if (atom->dma_fence.fence) { ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ struct fence *fence = atom->dma_fence.fence; ++#else ++ struct dma_fence *fence = atom->dma_fence.fence; ++#endif ++ ++ seq_printf(sfile, ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0)) ++ "Sd(%u#%u: %s) ", ++#else ++ "Sd(%llu#%u: %s) ", ++#endif ++ fence->context, ++ fence->seqno, ++ dma_fence_is_signaled(fence) ? ++ "signaled" : "active"); ++ } ++ ++ list_for_each_entry(cb, &atom->dma_fence.callbacks, ++ node) { ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ struct fence *fence = cb->fence; ++#else ++ struct dma_fence *fence = cb->fence; ++#endif ++ ++ seq_printf(sfile, ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0)) ++ "Wd(%u#%u: %s) ", ++#else ++ "Wd(%llu#%u: %s) ", ++#endif ++ fence->context, ++ fence->seqno, ++ dma_fence_is_signaled(fence) ? ++ "signaled" : "active"); ++ } ++ } ++#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ ++ ++} ++ ++static void kbasep_jd_debugfs_atom_deps( ++ struct kbase_jd_debugfs_depinfo *deps, ++ struct kbase_jd_atom *atom) ++{ ++ struct kbase_context *kctx = atom->kctx; ++ int i; ++ ++ for (i = 0; i < 2; i++) { ++ deps[i].id = (unsigned)(atom->dep[i].atom ? ++ kbase_jd_atom_id(kctx, atom->dep[i].atom) : 0); ++ ++ switch (atom->dep[i].dep_type) { ++ case BASE_JD_DEP_TYPE_INVALID: ++ deps[i].type = ' '; ++ break; ++ case BASE_JD_DEP_TYPE_DATA: ++ deps[i].type = 'D'; ++ break; ++ case BASE_JD_DEP_TYPE_ORDER: ++ deps[i].type = '>'; ++ break; ++ default: ++ deps[i].type = '?'; ++ break; ++ } ++ } ++} ++/** ++ * kbasep_jd_debugfs_atoms_show - Show callback for the JD atoms debugfs file. ++ * @sfile: The debugfs entry ++ * @data: Data associated with the entry ++ * ++ * This function is called to get the contents of the JD atoms debugfs file. ++ * This is a report of all atoms managed by kbase_jd_context.atoms ++ * ++ * Return: 0 if successfully prints data in debugfs entry file, failure ++ * otherwise ++ */ ++static int kbasep_jd_debugfs_atoms_show(struct seq_file *sfile, void *data) ++{ ++ struct kbase_context *kctx = sfile->private; ++ struct kbase_jd_atom *atoms; ++ unsigned long irq_flags; ++ int i; ++ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ ++ /* Print version */ ++ seq_printf(sfile, "v%u\n", MALI_JD_DEBUGFS_VERSION); ++ ++ /* Print U/K API version */ ++ seq_printf(sfile, "ukv%u.%u\n", BASE_UK_VERSION_MAJOR, ++ BASE_UK_VERSION_MINOR); ++ ++ /* Print table heading */ ++ seq_puts(sfile, " ID, Core req, St, CR, Predeps, Start time, Additional info...\n"); ++ ++ atoms = kctx->jctx.atoms; ++ /* General atom states */ ++ mutex_lock(&kctx->jctx.lock); ++ /* JS-related states */ ++ spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, irq_flags); ++ for (i = 0; i != BASE_JD_ATOM_COUNT; ++i) { ++ struct kbase_jd_atom *atom = &atoms[i]; ++ s64 start_timestamp = 0; ++ struct kbase_jd_debugfs_depinfo deps[2]; ++ ++ if (atom->status == KBASE_JD_ATOM_STATE_UNUSED) ++ continue; ++ ++ /* start_timestamp is cleared as soon as the atom leaves UNUSED state ++ * and set before a job is submitted to the h/w, a non-zero value means ++ * it is valid */ ++ if (ktime_to_ns(atom->start_timestamp)) ++ start_timestamp = ktime_to_ns( ++ ktime_sub(ktime_get(), atom->start_timestamp)); ++ ++ kbasep_jd_debugfs_atom_deps(deps, atom); ++ ++ seq_printf(sfile, ++ "%3u, %8x, %2u, %c%3u %c%3u, %20lld, ", ++ i, atom->core_req, atom->status, ++ deps[0].type, deps[0].id, ++ deps[1].type, deps[1].id, ++ start_timestamp); ++ ++ ++ kbase_jd_debugfs_fence_info(atom, sfile); ++ ++ seq_puts(sfile, "\n"); ++ } ++ spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, irq_flags); ++ mutex_unlock(&kctx->jctx.lock); ++ ++ return 0; ++} ++ ++ ++/** ++ * kbasep_jd_debugfs_atoms_open - open operation for atom debugfs file ++ * @in: &struct inode pointer ++ * @file: &struct file pointer ++ * ++ * Return: file descriptor ++ */ ++static int kbasep_jd_debugfs_atoms_open(struct inode *in, struct file *file) ++{ ++ return single_open(file, kbasep_jd_debugfs_atoms_show, in->i_private); ++} ++ ++static const struct file_operations kbasep_jd_debugfs_atoms_fops = { ++ .owner = THIS_MODULE, ++ .open = kbasep_jd_debugfs_atoms_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = single_release, ++}; ++ ++void kbasep_jd_debugfs_ctx_init(struct kbase_context *kctx) ++{ ++#if (KERNEL_VERSION(4, 7, 0) <= LINUX_VERSION_CODE) ++ const mode_t mode = S_IRUGO; ++#else ++ const mode_t mode = S_IRUSR; ++#endif ++ ++ /* Caller already ensures this, but we keep the pattern for ++ * maintenance safety. ++ */ ++ if (WARN_ON(!kctx) || ++ WARN_ON(IS_ERR_OR_NULL(kctx->kctx_dentry))) ++ return; ++ ++ /* Expose all atoms */ ++ debugfs_create_file("atoms", mode, kctx->kctx_dentry, kctx, ++ &kbasep_jd_debugfs_atoms_fops); ++ ++} ++ ++#endif /* CONFIG_DEBUG_FS */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_jd_debugfs.h b/drivers/gpu/arm/bifrost/mali_kbase_jd_debugfs.h +new file mode 100755 +index 000000000000..697bdef4d434 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_jd_debugfs.h +@@ -0,0 +1,45 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2018 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/** ++ * @file mali_kbase_jd_debugfs.h ++ * Header file for job dispatcher-related entries in debugfs ++ */ ++ ++#ifndef _KBASE_JD_DEBUGFS_H ++#define _KBASE_JD_DEBUGFS_H ++ ++#include ++ ++#define MALI_JD_DEBUGFS_VERSION 3 ++ ++/* Forward declarations */ ++struct kbase_context; ++ ++/** ++ * kbasep_jd_debugfs_ctx_init() - Add debugfs entries for JD system ++ * ++ * @kctx Pointer to kbase_context ++ */ ++void kbasep_jd_debugfs_ctx_init(struct kbase_context *kctx); ++ ++#endif /*_KBASE_JD_DEBUGFS_H*/ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_jm.c b/drivers/gpu/arm/bifrost/mali_kbase_jm.c +new file mode 100755 +index 000000000000..fb15a8c1727a +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_jm.c +@@ -0,0 +1,155 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++/* ++ * HW access job manager common APIs ++ */ ++ ++#include ++#include "mali_kbase_hwaccess_jm.h" ++#include "mali_kbase_jm.h" ++ ++#if !MALI_USE_CSF ++/** ++ * kbase_jm_next_job() - Attempt to run the next @nr_jobs_to_submit jobs on slot ++ * @js on the active context. ++ * @kbdev: Device pointer ++ * @js: Job slot to run on ++ * @nr_jobs_to_submit: Number of jobs to attempt to submit ++ * ++ * Return: true if slot can still be submitted on, false if slot is now full. ++ */ ++static bool kbase_jm_next_job(struct kbase_device *kbdev, int js, ++ int nr_jobs_to_submit) ++{ ++ struct kbase_context *kctx; ++ int i; ++ ++ kctx = kbdev->hwaccess.active_kctx[js]; ++ dev_dbg(kbdev->dev, ++ "Trying to run the next %d jobs in kctx %p (s:%d)\n", ++ nr_jobs_to_submit, (void *)kctx, js); ++ ++ if (!kctx) ++ return true; ++ ++ for (i = 0; i < nr_jobs_to_submit; i++) { ++ struct kbase_jd_atom *katom = kbase_js_pull(kctx, js); ++ ++ if (!katom) ++ return true; /* Context has no jobs on this slot */ ++ ++ kbase_backend_run_atom(kbdev, katom); ++ } ++ ++ dev_dbg(kbdev->dev, "Slot ringbuffer should now be full (s:%d)\n", js); ++ return false; ++} ++ ++u32 kbase_jm_kick(struct kbase_device *kbdev, u32 js_mask) ++{ ++ u32 ret_mask = 0; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ dev_dbg(kbdev->dev, "JM kick slot mask 0x%x\n", js_mask); ++ ++ while (js_mask) { ++ int js = ffs(js_mask) - 1; ++ int nr_jobs_to_submit = kbase_backend_slot_free(kbdev, js); ++ ++ if (kbase_jm_next_job(kbdev, js, nr_jobs_to_submit)) ++ ret_mask |= (1 << js); ++ ++ js_mask &= ~(1 << js); ++ } ++ ++ dev_dbg(kbdev->dev, "Can still submit to mask 0x%x\n", ret_mask); ++ return ret_mask; ++} ++ ++void kbase_jm_try_kick(struct kbase_device *kbdev, u32 js_mask) ++{ ++ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (!down_trylock(&js_devdata->schedule_sem)) { ++ kbase_jm_kick(kbdev, js_mask); ++ up(&js_devdata->schedule_sem); ++ } ++} ++ ++void kbase_jm_try_kick_all(struct kbase_device *kbdev) ++{ ++ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (!down_trylock(&js_devdata->schedule_sem)) { ++ kbase_jm_kick_all(kbdev); ++ up(&js_devdata->schedule_sem); ++ } ++} ++#endif /* !MALI_USE_CSF */ ++ ++void kbase_jm_idle_ctx(struct kbase_device *kbdev, struct kbase_context *kctx) ++{ ++ int js; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ for (js = 0; js < BASE_JM_MAX_NR_SLOTS; js++) { ++ if (kbdev->hwaccess.active_kctx[js] == kctx) { ++ dev_dbg(kbdev->dev, "Marking kctx %p as inactive (s:%d)\n", ++ (void *)kctx, js); ++ kbdev->hwaccess.active_kctx[js] = NULL; ++ } ++ } ++} ++ ++#if !MALI_USE_CSF ++struct kbase_jd_atom *kbase_jm_return_atom_to_js(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ dev_dbg(kbdev->dev, "Atom %p is returning with event code 0x%x\n", ++ (void *)katom, katom->event_code); ++ ++ if (katom->event_code != BASE_JD_EVENT_STOPPED && ++ katom->event_code != BASE_JD_EVENT_REMOVED_FROM_NEXT) { ++ return kbase_js_complete_atom(katom, NULL); ++ } else { ++ kbase_js_unpull(katom->kctx, katom); ++ return NULL; ++ } ++} ++ ++struct kbase_jd_atom *kbase_jm_complete(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom, ktime_t *end_timestamp) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ return kbase_js_complete_atom(katom, end_timestamp); ++} ++#endif /* !MALI_USE_CSF */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_jm.h b/drivers/gpu/arm/bifrost/mali_kbase_jm.h +new file mode 100755 +index 000000000000..b3fd421a1ff3 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_jm.h +@@ -0,0 +1,119 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014, 2016, 2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++/* ++ * Job manager common APIs ++ */ ++ ++#ifndef _KBASE_JM_H_ ++#define _KBASE_JM_H_ ++ ++#if !MALI_USE_CSF ++/** ++ * kbase_jm_kick() - Indicate that there are jobs ready to run. ++ * @kbdev: Device pointer ++ * @js_mask: Mask of the job slots that can be pulled from. ++ * ++ * Caller must hold the hwaccess_lock and schedule_sem semaphore ++ * ++ * Return: Mask of the job slots that can still be submitted to. ++ */ ++u32 kbase_jm_kick(struct kbase_device *kbdev, u32 js_mask); ++ ++/** ++ * kbase_jm_kick_all() - Indicate that there are jobs ready to run on all job ++ * slots. ++ * @kbdev: Device pointer ++ * ++ * Caller must hold the hwaccess_lock and schedule_sem semaphore ++ * ++ * Return: Mask of the job slots that can still be submitted to. ++ */ ++static inline u32 kbase_jm_kick_all(struct kbase_device *kbdev) ++{ ++ return kbase_jm_kick(kbdev, (1 << kbdev->gpu_props.num_job_slots) - 1); ++} ++ ++/** ++ * kbase_jm_try_kick - Attempt to call kbase_jm_kick ++ * @kbdev: Device pointer ++ * @js_mask: Mask of the job slots that can be pulled from ++ * Context: Caller must hold hwaccess_lock ++ * ++ * If schedule_sem can be immediately obtained then this function will call ++ * kbase_jm_kick() otherwise it will do nothing. ++ */ ++void kbase_jm_try_kick(struct kbase_device *kbdev, u32 js_mask); ++ ++/** ++ * kbase_jm_try_kick_all() - Attempt to call kbase_jm_kick_all ++ * @kbdev: Device pointer ++ * Context: Caller must hold hwaccess_lock ++ * ++ * If schedule_sem can be immediately obtained then this function will call ++ * kbase_jm_kick_all() otherwise it will do nothing. ++ */ ++void kbase_jm_try_kick_all(struct kbase_device *kbdev); ++#endif /* !MALI_USE_CSF */ ++ ++/** ++ * kbase_jm_idle_ctx() - Mark a context as idle. ++ * @kbdev: Device pointer ++ * @kctx: Context to mark as idle ++ * ++ * No more atoms will be pulled from this context until it is marked as active ++ * by kbase_js_use_ctx(). ++ * ++ * The context should have no atoms currently pulled from it ++ * (kctx->atoms_pulled == 0). ++ * ++ * Caller must hold the hwaccess_lock ++ */ ++void kbase_jm_idle_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); ++ ++#if !MALI_USE_CSF ++/** ++ * kbase_jm_return_atom_to_js() - Return an atom to the job scheduler that has ++ * been soft-stopped or will fail due to a ++ * dependency ++ * @kbdev: Device pointer ++ * @katom: Atom that has been stopped or will be failed ++ * ++ * Return: Atom that has now been unblocked and can now be run, or NULL if none ++ */ ++struct kbase_jd_atom *kbase_jm_return_atom_to_js(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom); ++ ++/** ++ * kbase_jm_complete() - Complete an atom ++ * @kbdev: Device pointer ++ * @katom: Atom that has completed ++ * @end_timestamp: Timestamp of atom completion ++ * ++ * Return: Atom that has now been unblocked and can now be run, or NULL if none ++ */ ++struct kbase_jd_atom *kbase_jm_complete(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom, ktime_t *end_timestamp); ++#endif /* !MALI_USE_CSF */ ++ ++#endif /* _KBASE_JM_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_js.c b/drivers/gpu/arm/bifrost/mali_kbase_js.c +new file mode 100755 +index 000000000000..9b338eb66531 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_js.c +@@ -0,0 +1,3760 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++/* ++ * Job Scheduler Implementation ++ */ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#include ++ ++#include "mali_kbase_jm.h" ++#include "mali_kbase_hwaccess_jm.h" ++ ++/* ++ * Private types ++ */ ++ ++/* Bitpattern indicating the result of releasing a context */ ++enum { ++ /* The context was descheduled - caller should try scheduling in a new ++ * one to keep the runpool full */ ++ KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED = (1u << 0), ++ /* Ctx attributes were changed - caller should try scheduling all ++ * contexts */ ++ KBASEP_JS_RELEASE_RESULT_SCHED_ALL = (1u << 1) ++}; ++ ++typedef u32 kbasep_js_release_result; ++ ++const int kbasep_js_atom_priority_to_relative[BASE_JD_NR_PRIO_LEVELS] = { ++ KBASE_JS_ATOM_SCHED_PRIO_MED, /* BASE_JD_PRIO_MEDIUM */ ++ KBASE_JS_ATOM_SCHED_PRIO_HIGH, /* BASE_JD_PRIO_HIGH */ ++ KBASE_JS_ATOM_SCHED_PRIO_LOW /* BASE_JD_PRIO_LOW */ ++}; ++ ++const base_jd_prio ++kbasep_js_relative_priority_to_atom[KBASE_JS_ATOM_SCHED_PRIO_COUNT] = { ++ BASE_JD_PRIO_HIGH, /* KBASE_JS_ATOM_SCHED_PRIO_HIGH */ ++ BASE_JD_PRIO_MEDIUM, /* KBASE_JS_ATOM_SCHED_PRIO_MED */ ++ BASE_JD_PRIO_LOW /* KBASE_JS_ATOM_SCHED_PRIO_LOW */ ++}; ++ ++ ++/* ++ * Private function prototypes ++ */ ++static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal( ++ struct kbase_device *kbdev, struct kbase_context *kctx, ++ struct kbasep_js_atom_retained_state *katom_retained_state); ++ ++static int kbase_js_get_slot(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom); ++ ++static void kbase_js_foreach_ctx_job(struct kbase_context *kctx, ++ kbasep_js_ctx_job_cb callback); ++ ++/* Helper for ktrace */ ++#if KBASE_KTRACE_ENABLE ++static int kbase_ktrace_get_ctx_refcnt(struct kbase_context *kctx) ++{ ++ return atomic_read(&kctx->refcount); ++} ++#else /* KBASE_KTRACE_ENABLE */ ++static int kbase_ktrace_get_ctx_refcnt(struct kbase_context *kctx) ++{ ++ CSTD_UNUSED(kctx); ++ return 0; ++} ++#endif /* KBASE_KTRACE_ENABLE */ ++ ++/* ++ * Private functions ++ */ ++ ++/** ++ * core_reqs_from_jsn_features - Convert JSn_FEATURES to core requirements ++ * @features: JSn_FEATURE register value ++ * ++ * Given a JSn_FEATURE register value returns the core requirements that match ++ * ++ * Return: Core requirement bit mask ++ */ ++static base_jd_core_req core_reqs_from_jsn_features(u16 features) ++{ ++ base_jd_core_req core_req = 0u; ++ ++ if ((features & JS_FEATURE_SET_VALUE_JOB) != 0) ++ core_req |= BASE_JD_REQ_V; ++ ++ if ((features & JS_FEATURE_CACHE_FLUSH_JOB) != 0) ++ core_req |= BASE_JD_REQ_CF; ++ ++ if ((features & JS_FEATURE_COMPUTE_JOB) != 0) ++ core_req |= BASE_JD_REQ_CS; ++ ++ if ((features & JS_FEATURE_TILER_JOB) != 0) ++ core_req |= BASE_JD_REQ_T; ++ ++ if ((features & JS_FEATURE_FRAGMENT_JOB) != 0) ++ core_req |= BASE_JD_REQ_FS; ++ ++ return core_req; ++} ++ ++static void kbase_js_sync_timers(struct kbase_device *kbdev) ++{ ++ mutex_lock(&kbdev->js_data.runpool_mutex); ++ kbase_backend_ctx_count_changed(kbdev); ++ mutex_unlock(&kbdev->js_data.runpool_mutex); ++} ++ ++/** ++ * jsctx_rb_none_to_pull_prio(): - Check if there are no pullable atoms ++ * @kctx: Pointer to kbase context with ring buffer. ++ * @js: Job slot id to check. ++ * @prio: Priority to check. ++ * ++ * Return true if there are no atoms to pull. There may be running atoms in the ++ * ring buffer even if there are no atoms to pull. It is also possible for the ++ * ring buffer to be full (with running atoms) when this functions returns ++ * true. ++ * ++ * Return: true if there are no atoms to pull, false otherwise. ++ */ ++static inline bool ++jsctx_rb_none_to_pull_prio(struct kbase_context *kctx, int js, int prio) ++{ ++ bool none_to_pull; ++ struct jsctx_queue *rb = &kctx->jsctx_queue[prio][js]; ++ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ ++ none_to_pull = RB_EMPTY_ROOT(&rb->runnable_tree); ++ ++ dev_dbg(kctx->kbdev->dev, ++ "Slot %d (prio %d) is %spullable in kctx %p\n", ++ js, prio, none_to_pull ? "not " : "", kctx); ++ ++ return none_to_pull; ++} ++ ++/** ++ * jsctx_rb_none_to_pull(): - Check if all priority ring buffers have no ++ * pullable atoms ++ * @kctx: Pointer to kbase context with ring buffer. ++ * @js: Job slot id to check. ++ * ++ * Caller must hold hwaccess_lock ++ * ++ * Return: true if the ring buffers for all priorities have no pullable atoms, ++ * false otherwise. ++ */ ++static inline bool ++jsctx_rb_none_to_pull(struct kbase_context *kctx, int js) ++{ ++ int prio; ++ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ ++ for (prio = KBASE_JS_ATOM_SCHED_PRIO_HIGH; ++ prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) { ++ if (!jsctx_rb_none_to_pull_prio(kctx, js, prio)) ++ return false; ++ } ++ ++ return true; ++} ++ ++/** ++ * jsctx_queue_foreach_prio(): - Execute callback for each entry in the queue. ++ * @kctx: Pointer to kbase context with the queue. ++ * @js: Job slot id to iterate. ++ * @prio: Priority id to iterate. ++ * @callback: Function pointer to callback. ++ * ++ * Iterate over a queue and invoke @callback for each entry in the queue, and ++ * remove the entry from the queue. ++ * ++ * If entries are added to the queue while this is running those entries may, or ++ * may not be covered. To ensure that all entries in the buffer have been ++ * enumerated when this function returns jsctx->lock must be held when calling ++ * this function. ++ * ++ * The HW access lock must always be held when calling this function. ++ */ ++static void ++jsctx_queue_foreach_prio(struct kbase_context *kctx, int js, int prio, ++ kbasep_js_ctx_job_cb callback) ++{ ++ struct jsctx_queue *queue = &kctx->jsctx_queue[prio][js]; ++ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ ++ while (!RB_EMPTY_ROOT(&queue->runnable_tree)) { ++ struct rb_node *node = rb_first(&queue->runnable_tree); ++ struct kbase_jd_atom *entry = rb_entry(node, ++ struct kbase_jd_atom, runnable_tree_node); ++ ++ rb_erase(node, &queue->runnable_tree); ++ callback(kctx->kbdev, entry); ++ ++ /* Runnable end-of-renderpass atoms can also be in the linked ++ * list of atoms blocked on cross-slot dependencies. Remove them ++ * to avoid calling the callback twice. ++ */ ++ if (entry->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST) { ++ WARN_ON(!(entry->core_req & ++ BASE_JD_REQ_END_RENDERPASS)); ++ dev_dbg(kctx->kbdev->dev, ++ "Del runnable atom %p from X_DEP list\n", ++ (void *)entry); ++ ++ list_del(&entry->queue); ++ entry->atom_flags &= ++ ~KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST; ++ } ++ } ++ ++ while (!list_empty(&queue->x_dep_head)) { ++ struct kbase_jd_atom *entry = list_entry(queue->x_dep_head.next, ++ struct kbase_jd_atom, queue); ++ ++ WARN_ON(!(entry->atom_flags & ++ KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST)); ++ dev_dbg(kctx->kbdev->dev, ++ "Del blocked atom %p from X_DEP list\n", ++ (void *)entry); ++ ++ list_del(queue->x_dep_head.next); ++ entry->atom_flags &= ++ ~KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST; ++ ++ callback(kctx->kbdev, entry); ++ } ++} ++ ++/** ++ * jsctx_queue_foreach(): - Execute callback for each entry in every queue ++ * @kctx: Pointer to kbase context with queue. ++ * @js: Job slot id to iterate. ++ * @callback: Function pointer to callback. ++ * ++ * Iterate over all the different priorities, and for each call ++ * jsctx_queue_foreach_prio() to iterate over the queue and invoke @callback ++ * for each entry, and remove the entry from the queue. ++ */ ++static inline void ++jsctx_queue_foreach(struct kbase_context *kctx, int js, ++ kbasep_js_ctx_job_cb callback) ++{ ++ int prio; ++ ++ for (prio = KBASE_JS_ATOM_SCHED_PRIO_HIGH; ++ prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) ++ jsctx_queue_foreach_prio(kctx, js, prio, callback); ++} ++ ++/** ++ * jsctx_rb_peek_prio(): - Check buffer and get next atom ++ * @kctx: Pointer to kbase context with ring buffer. ++ * @js: Job slot id to check. ++ * @prio: Priority id to check. ++ * ++ * Check the ring buffer for the specified @js and @prio and return a pointer to ++ * the next atom, unless the ring buffer is empty. ++ * ++ * Return: Pointer to next atom in buffer, or NULL if there is no atom. ++ */ ++static inline struct kbase_jd_atom * ++jsctx_rb_peek_prio(struct kbase_context *kctx, int js, int prio) ++{ ++ struct jsctx_queue *rb = &kctx->jsctx_queue[prio][js]; ++ struct rb_node *node; ++ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ dev_dbg(kctx->kbdev->dev, ++ "Peeking runnable tree of kctx %p for prio %d (s:%d)\n", ++ (void *)kctx, prio, js); ++ ++ node = rb_first(&rb->runnable_tree); ++ if (!node) { ++ dev_dbg(kctx->kbdev->dev, "Tree is empty\n"); ++ return NULL; ++ } ++ ++ return rb_entry(node, struct kbase_jd_atom, runnable_tree_node); ++} ++ ++/** ++ * jsctx_rb_peek(): - Check all priority buffers and get next atom ++ * @kctx: Pointer to kbase context with ring buffer. ++ * @js: Job slot id to check. ++ * ++ * Check the ring buffers for all priorities, starting from ++ * KBASE_JS_ATOM_SCHED_PRIO_HIGH, for the specified @js and @prio and return a ++ * pointer to the next atom, unless all the priority's ring buffers are empty. ++ * ++ * Caller must hold the hwaccess_lock. ++ * ++ * Return: Pointer to next atom in buffer, or NULL if there is no atom. ++ */ ++static inline struct kbase_jd_atom * ++jsctx_rb_peek(struct kbase_context *kctx, int js) ++{ ++ int prio; ++ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ ++ for (prio = KBASE_JS_ATOM_SCHED_PRIO_HIGH; ++ prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) { ++ struct kbase_jd_atom *katom; ++ ++ katom = jsctx_rb_peek_prio(kctx, js, prio); ++ if (katom) ++ return katom; ++ } ++ ++ return NULL; ++} ++ ++/** ++ * jsctx_rb_pull(): - Mark atom in list as running ++ * @kctx: Pointer to kbase context with ring buffer. ++ * @katom: Pointer to katom to pull. ++ * ++ * Mark an atom previously obtained from jsctx_rb_peek() as running. ++ * ++ * @katom must currently be at the head of the ring buffer. ++ */ ++static inline void ++jsctx_rb_pull(struct kbase_context *kctx, struct kbase_jd_atom *katom) ++{ ++ int prio = katom->sched_priority; ++ int js = katom->slot_nr; ++ struct jsctx_queue *rb = &kctx->jsctx_queue[prio][js]; ++ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ ++ dev_dbg(kctx->kbdev->dev, "Erasing atom %p from runnable tree of kctx %p\n", ++ (void *)katom, (void *)kctx); ++ ++ /* Atoms must be pulled in the correct order. */ ++ WARN_ON(katom != jsctx_rb_peek_prio(kctx, js, prio)); ++ ++ rb_erase(&katom->runnable_tree_node, &rb->runnable_tree); ++} ++ ++#define LESS_THAN_WRAP(a, b) ((s32)(a - b) < 0) ++ ++static void ++jsctx_tree_add(struct kbase_context *kctx, struct kbase_jd_atom *katom) ++{ ++ struct kbase_device *kbdev = kctx->kbdev; ++ int prio = katom->sched_priority; ++ int js = katom->slot_nr; ++ struct jsctx_queue *queue = &kctx->jsctx_queue[prio][js]; ++ struct rb_node **new = &(queue->runnable_tree.rb_node), *parent = NULL; ++ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ ++ dev_dbg(kbdev->dev, "Adding atom %p to runnable tree of kctx %p (s:%d)\n", ++ (void *)katom, (void *)kctx, js); ++ ++ while (*new) { ++ struct kbase_jd_atom *entry = container_of(*new, ++ struct kbase_jd_atom, runnable_tree_node); ++ ++ parent = *new; ++ if (LESS_THAN_WRAP(katom->age, entry->age)) ++ new = &((*new)->rb_left); ++ else ++ new = &((*new)->rb_right); ++ } ++ ++ /* Add new node and rebalance tree. */ ++ rb_link_node(&katom->runnable_tree_node, parent, new); ++ rb_insert_color(&katom->runnable_tree_node, &queue->runnable_tree); ++ ++ KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(kbdev, katom, TL_ATOM_STATE_READY); ++} ++ ++/** ++ * jsctx_rb_unpull(): - Undo marking of atom in list as running ++ * @kctx: Pointer to kbase context with ring buffer. ++ * @katom: Pointer to katom to unpull. ++ * ++ * Undo jsctx_rb_pull() and put @katom back in the queue. ++ * ++ * jsctx_rb_unpull() must be called on atoms in the same order the atoms were ++ * pulled. ++ */ ++static inline void ++jsctx_rb_unpull(struct kbase_context *kctx, struct kbase_jd_atom *katom) ++{ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ ++ jsctx_tree_add(kctx, katom); ++} ++ ++static bool kbase_js_ctx_pullable(struct kbase_context *kctx, ++ int js, ++ bool is_scheduled); ++static bool kbase_js_ctx_list_add_pullable_nolock(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ int js); ++static bool kbase_js_ctx_list_add_unpullable_nolock(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ int js); ++ ++/* ++ * Functions private to KBase ('Protected' functions) ++ */ ++int kbasep_js_devdata_init(struct kbase_device * const kbdev) ++{ ++ struct kbasep_js_device_data *jsdd; ++ int i, j; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ jsdd = &kbdev->js_data; ++ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++ /* Soft-stop will be disabled on a single context by default unless ++ * softstop_always is set */ ++ jsdd->softstop_always = false; ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++ jsdd->nr_all_contexts_running = 0; ++ jsdd->nr_user_contexts_running = 0; ++ jsdd->nr_contexts_pullable = 0; ++ atomic_set(&jsdd->nr_contexts_runnable, 0); ++ /* No ctx allowed to submit */ ++ jsdd->runpool_irq.submit_allowed = 0u; ++ memset(jsdd->runpool_irq.ctx_attr_ref_count, 0, ++ sizeof(jsdd->runpool_irq.ctx_attr_ref_count)); ++ memset(jsdd->runpool_irq.slot_affinities, 0, ++ sizeof(jsdd->runpool_irq.slot_affinities)); ++ memset(jsdd->runpool_irq.slot_affinity_refcount, 0, ++ sizeof(jsdd->runpool_irq.slot_affinity_refcount)); ++ INIT_LIST_HEAD(&jsdd->suspended_soft_jobs_list); ++ ++ /* Config attributes */ ++ jsdd->scheduling_period_ns = DEFAULT_JS_SCHEDULING_PERIOD_NS; ++ jsdd->soft_stop_ticks = DEFAULT_JS_SOFT_STOP_TICKS; ++ jsdd->soft_stop_ticks_cl = DEFAULT_JS_SOFT_STOP_TICKS_CL; ++ jsdd->hard_stop_ticks_ss = DEFAULT_JS_HARD_STOP_TICKS_SS; ++ jsdd->hard_stop_ticks_cl = DEFAULT_JS_HARD_STOP_TICKS_CL; ++ jsdd->hard_stop_ticks_dumping = DEFAULT_JS_HARD_STOP_TICKS_DUMPING; ++ jsdd->gpu_reset_ticks_ss = DEFAULT_JS_RESET_TICKS_SS; ++ jsdd->gpu_reset_ticks_cl = DEFAULT_JS_RESET_TICKS_CL; ++ jsdd->gpu_reset_ticks_dumping = DEFAULT_JS_RESET_TICKS_DUMPING; ++ jsdd->ctx_timeslice_ns = DEFAULT_JS_CTX_TIMESLICE_NS; ++ atomic_set(&jsdd->soft_job_timeout_ms, DEFAULT_JS_SOFT_JOB_TIMEOUT); ++ ++ dev_dbg(kbdev->dev, "JS Config Attribs: "); ++ dev_dbg(kbdev->dev, "\tscheduling_period_ns:%u", ++ jsdd->scheduling_period_ns); ++ dev_dbg(kbdev->dev, "\tsoft_stop_ticks:%u", ++ jsdd->soft_stop_ticks); ++ dev_dbg(kbdev->dev, "\tsoft_stop_ticks_cl:%u", ++ jsdd->soft_stop_ticks_cl); ++ dev_dbg(kbdev->dev, "\thard_stop_ticks_ss:%u", ++ jsdd->hard_stop_ticks_ss); ++ dev_dbg(kbdev->dev, "\thard_stop_ticks_cl:%u", ++ jsdd->hard_stop_ticks_cl); ++ dev_dbg(kbdev->dev, "\thard_stop_ticks_dumping:%u", ++ jsdd->hard_stop_ticks_dumping); ++ dev_dbg(kbdev->dev, "\tgpu_reset_ticks_ss:%u", ++ jsdd->gpu_reset_ticks_ss); ++ dev_dbg(kbdev->dev, "\tgpu_reset_ticks_cl:%u", ++ jsdd->gpu_reset_ticks_cl); ++ dev_dbg(kbdev->dev, "\tgpu_reset_ticks_dumping:%u", ++ jsdd->gpu_reset_ticks_dumping); ++ dev_dbg(kbdev->dev, "\tctx_timeslice_ns:%u", ++ jsdd->ctx_timeslice_ns); ++ dev_dbg(kbdev->dev, "\tsoft_job_timeout:%i", ++ atomic_read(&jsdd->soft_job_timeout_ms)); ++ ++ if (!(jsdd->soft_stop_ticks < jsdd->hard_stop_ticks_ss && ++ jsdd->hard_stop_ticks_ss < jsdd->gpu_reset_ticks_ss && ++ jsdd->soft_stop_ticks < jsdd->hard_stop_ticks_dumping && ++ jsdd->hard_stop_ticks_dumping < ++ jsdd->gpu_reset_ticks_dumping)) { ++ dev_err(kbdev->dev, "Job scheduler timeouts invalid; soft/hard/reset tick counts should be in increasing order\n"); ++ return -EINVAL; ++ } ++ ++#if KBASE_DISABLE_SCHEDULING_SOFT_STOPS ++ dev_dbg(kbdev->dev, "Job Scheduling Soft-stops disabled, ignoring value for soft_stop_ticks==%u at %uns per tick. Other soft-stops may still occur.", ++ jsdd->soft_stop_ticks, ++ jsdd->scheduling_period_ns); ++#endif ++#if KBASE_DISABLE_SCHEDULING_HARD_STOPS ++ dev_dbg(kbdev->dev, "Job Scheduling Hard-stops disabled, ignoring values for hard_stop_ticks_ss==%d and hard_stop_ticks_dumping==%u at %uns per tick. Other hard-stops may still occur.", ++ jsdd->hard_stop_ticks_ss, ++ jsdd->hard_stop_ticks_dumping, ++ jsdd->scheduling_period_ns); ++#endif ++#if KBASE_DISABLE_SCHEDULING_SOFT_STOPS && KBASE_DISABLE_SCHEDULING_HARD_STOPS ++ dev_dbg(kbdev->dev, "Note: The JS tick timer (if coded) will still be run, but do nothing."); ++#endif ++ ++ for (i = 0; i < kbdev->gpu_props.num_job_slots; ++i) ++ jsdd->js_reqs[i] = core_reqs_from_jsn_features( ++ kbdev->gpu_props.props.raw_props.js_features[i]); ++ ++ /* On error, we could continue on: providing none of the below resources ++ * rely on the ones above */ ++ ++ mutex_init(&jsdd->runpool_mutex); ++ mutex_init(&jsdd->queue_mutex); ++ sema_init(&jsdd->schedule_sem, 1); ++ ++ for (i = 0; i < kbdev->gpu_props.num_job_slots; ++i) { ++ for (j = 0; j < KBASE_JS_ATOM_SCHED_PRIO_COUNT; ++j) { ++ INIT_LIST_HEAD(&jsdd->ctx_list_pullable[i][j]); ++ INIT_LIST_HEAD(&jsdd->ctx_list_unpullable[i][j]); ++ } ++ } ++ ++ return 0; ++} ++ ++void kbasep_js_devdata_halt(struct kbase_device *kbdev) ++{ ++ CSTD_UNUSED(kbdev); ++} ++ ++void kbasep_js_devdata_term(struct kbase_device *kbdev) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ s8 zero_ctx_attr_ref_count[KBASEP_JS_CTX_ATTR_COUNT] = { 0, }; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ js_devdata = &kbdev->js_data; ++ ++ /* The caller must de-register all contexts before calling this ++ */ ++ KBASE_DEBUG_ASSERT(js_devdata->nr_all_contexts_running == 0); ++ KBASE_DEBUG_ASSERT(memcmp( ++ js_devdata->runpool_irq.ctx_attr_ref_count, ++ zero_ctx_attr_ref_count, ++ sizeof(zero_ctx_attr_ref_count)) == 0); ++ CSTD_UNUSED(zero_ctx_attr_ref_count); ++} ++ ++int kbasep_js_kctx_init(struct kbase_context *const kctx) ++{ ++ struct kbase_device *kbdev; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ int i, j; ++ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ ++ kbdev = kctx->kbdev; ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ for (i = 0; i < BASE_JM_MAX_NR_SLOTS; ++i) ++ INIT_LIST_HEAD(&kctx->jctx.sched_info.ctx.ctx_list_entry[i]); ++ ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ kctx->slots_pullable = 0; ++ js_kctx_info->ctx.nr_jobs = 0; ++ kbase_ctx_flag_clear(kctx, KCTX_SCHEDULED); ++ kbase_ctx_flag_clear(kctx, KCTX_DYING); ++ memset(js_kctx_info->ctx.ctx_attr_ref_count, 0, ++ sizeof(js_kctx_info->ctx.ctx_attr_ref_count)); ++ ++ /* Initially, the context is disabled from submission until the create ++ * flags are set */ ++ kbase_ctx_flag_set(kctx, KCTX_SUBMIT_DISABLED); ++ ++ /* On error, we could continue on: providing none of the below resources ++ * rely on the ones above */ ++ mutex_init(&js_kctx_info->ctx.jsctx_mutex); ++ ++ init_waitqueue_head(&js_kctx_info->ctx.is_scheduled_wait); ++ ++ for (i = 0; i < KBASE_JS_ATOM_SCHED_PRIO_COUNT; i++) { ++ for (j = 0; j < BASE_JM_MAX_NR_SLOTS; j++) { ++ INIT_LIST_HEAD(&kctx->jsctx_queue[i][j].x_dep_head); ++ kctx->jsctx_queue[i][j].runnable_tree = RB_ROOT; ++ } ++ } ++ ++ return 0; ++} ++ ++void kbasep_js_kctx_term(struct kbase_context *kctx) ++{ ++ struct kbase_device *kbdev; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ int js; ++ bool update_ctx_count = false; ++ unsigned long flags; ++ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ ++ kbdev = kctx->kbdev; ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ /* The caller must de-register all jobs before calling this */ ++ KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ KBASE_DEBUG_ASSERT(js_kctx_info->ctx.nr_jobs == 0); ++ ++ mutex_lock(&kbdev->js_data.queue_mutex); ++ mutex_lock(&kctx->jctx.sched_info.ctx.jsctx_mutex); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) ++ list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ if (kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)) { ++ WARN_ON(atomic_read(&kbdev->js_data.nr_contexts_runnable) <= 0); ++ atomic_dec(&kbdev->js_data.nr_contexts_runnable); ++ update_ctx_count = true; ++ kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); ++ } ++ ++ mutex_unlock(&kctx->jctx.sched_info.ctx.jsctx_mutex); ++ mutex_unlock(&kbdev->js_data.queue_mutex); ++ ++ if (update_ctx_count) { ++ mutex_lock(&kbdev->js_data.runpool_mutex); ++ kbase_backend_ctx_count_changed(kbdev); ++ mutex_unlock(&kbdev->js_data.runpool_mutex); ++ } ++} ++ ++/** ++ * kbase_js_ctx_list_add_pullable_nolock - Variant of ++ * kbase_jd_ctx_list_add_pullable() ++ * where the caller must hold ++ * hwaccess_lock ++ * @kbdev: Device pointer ++ * @kctx: Context to add to queue ++ * @js: Job slot to use ++ * ++ * Caller must hold hwaccess_lock ++ * ++ * Return: true if caller should call kbase_backend_ctx_count_changed() ++ */ ++static bool kbase_js_ctx_list_add_pullable_nolock(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ int js) ++{ ++ bool ret = false; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ dev_dbg(kbdev->dev, "Add pullable tail kctx %p (s:%d)\n", ++ (void *)kctx, js); ++ ++ if (!list_empty(&kctx->jctx.sched_info.ctx.ctx_list_entry[js])) ++ list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); ++ ++ list_add_tail(&kctx->jctx.sched_info.ctx.ctx_list_entry[js], ++ &kbdev->js_data.ctx_list_pullable[js][kctx->priority]); ++ ++ if (!kctx->slots_pullable) { ++ kbdev->js_data.nr_contexts_pullable++; ++ ret = true; ++ if (!atomic_read(&kctx->atoms_pulled)) { ++ WARN_ON(kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); ++ kbase_ctx_flag_set(kctx, KCTX_RUNNABLE_REF); ++ atomic_inc(&kbdev->js_data.nr_contexts_runnable); ++ } ++ } ++ kctx->slots_pullable |= (1 << js); ++ ++ return ret; ++} ++ ++/** ++ * kbase_js_ctx_list_add_pullable_head_nolock - Variant of ++ * kbase_js_ctx_list_add_pullable_head() ++ * where the caller must hold ++ * hwaccess_lock ++ * @kbdev: Device pointer ++ * @kctx: Context to add to queue ++ * @js: Job slot to use ++ * ++ * Caller must hold hwaccess_lock ++ * ++ * Return: true if caller should call kbase_backend_ctx_count_changed() ++ */ ++static bool kbase_js_ctx_list_add_pullable_head_nolock( ++ struct kbase_device *kbdev, struct kbase_context *kctx, int js) ++{ ++ bool ret = false; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ dev_dbg(kbdev->dev, "Add pullable head kctx %p (s:%d)\n", ++ (void *)kctx, js); ++ ++ if (!list_empty(&kctx->jctx.sched_info.ctx.ctx_list_entry[js])) ++ list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); ++ ++ list_add(&kctx->jctx.sched_info.ctx.ctx_list_entry[js], ++ &kbdev->js_data.ctx_list_pullable[js][kctx->priority]); ++ ++ if (!kctx->slots_pullable) { ++ kbdev->js_data.nr_contexts_pullable++; ++ ret = true; ++ if (!atomic_read(&kctx->atoms_pulled)) { ++ WARN_ON(kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); ++ kbase_ctx_flag_set(kctx, KCTX_RUNNABLE_REF); ++ atomic_inc(&kbdev->js_data.nr_contexts_runnable); ++ } ++ } ++ kctx->slots_pullable |= (1 << js); ++ ++ return ret; ++} ++ ++/** ++ * kbase_js_ctx_list_add_pullable_head - Add context to the head of the ++ * per-slot pullable context queue ++ * @kbdev: Device pointer ++ * @kctx: Context to add to queue ++ * @js: Job slot to use ++ * ++ * If the context is on either the pullable or unpullable queues, then it is ++ * removed before being added to the head. ++ * ++ * This function should be used when a context has been scheduled, but no jobs ++ * can currently be pulled from it. ++ * ++ * Return: true if caller should call kbase_backend_ctx_count_changed() ++ */ ++static bool kbase_js_ctx_list_add_pullable_head(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ int js) ++{ ++ bool ret; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ret = kbase_js_ctx_list_add_pullable_head_nolock(kbdev, kctx, js); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ return ret; ++} ++ ++/** ++ * kbase_js_ctx_list_add_unpullable_nolock - Add context to the tail of the ++ * per-slot unpullable context queue ++ * @kbdev: Device pointer ++ * @kctx: Context to add to queue ++ * @js: Job slot to use ++ * ++ * The context must already be on the per-slot pullable queue. It will be ++ * removed from the pullable queue before being added to the unpullable queue. ++ * ++ * This function should be used when a context has been pulled from, and there ++ * are no jobs remaining on the specified slot. ++ * ++ * Caller must hold hwaccess_lock ++ * ++ * Return: true if caller should call kbase_backend_ctx_count_changed() ++ */ ++static bool kbase_js_ctx_list_add_unpullable_nolock(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ int js) ++{ ++ bool ret = false; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ dev_dbg(kbdev->dev, "Add unpullable tail kctx %p (s:%d)\n", ++ (void *)kctx, js); ++ ++ list_move_tail(&kctx->jctx.sched_info.ctx.ctx_list_entry[js], ++ &kbdev->js_data.ctx_list_unpullable[js][kctx->priority]); ++ ++ if (kctx->slots_pullable == (1 << js)) { ++ kbdev->js_data.nr_contexts_pullable--; ++ ret = true; ++ if (!atomic_read(&kctx->atoms_pulled)) { ++ WARN_ON(!kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); ++ kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); ++ atomic_dec(&kbdev->js_data.nr_contexts_runnable); ++ } ++ } ++ kctx->slots_pullable &= ~(1 << js); ++ ++ return ret; ++} ++ ++/** ++ * kbase_js_ctx_list_remove_nolock - Remove context from the per-slot pullable ++ * or unpullable context queues ++ * @kbdev: Device pointer ++ * @kctx: Context to remove from queue ++ * @js: Job slot to use ++ * ++ * The context must already be on one of the queues. ++ * ++ * This function should be used when a context has no jobs on the GPU, and no ++ * jobs remaining for the specified slot. ++ * ++ * Caller must hold hwaccess_lock ++ * ++ * Return: true if caller should call kbase_backend_ctx_count_changed() ++ */ ++static bool kbase_js_ctx_list_remove_nolock(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ int js) ++{ ++ bool ret = false; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ WARN_ON(list_empty(&kctx->jctx.sched_info.ctx.ctx_list_entry[js])); ++ ++ list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); ++ ++ if (kctx->slots_pullable == (1 << js)) { ++ kbdev->js_data.nr_contexts_pullable--; ++ ret = true; ++ if (!atomic_read(&kctx->atoms_pulled)) { ++ WARN_ON(!kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); ++ kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); ++ atomic_dec(&kbdev->js_data.nr_contexts_runnable); ++ } ++ } ++ kctx->slots_pullable &= ~(1 << js); ++ ++ return ret; ++} ++ ++/** ++ * kbase_js_ctx_list_pop_head_nolock - Variant of kbase_js_ctx_list_pop_head() ++ * where the caller must hold ++ * hwaccess_lock ++ * @kbdev: Device pointer ++ * @js: Job slot to use ++ * ++ * Caller must hold hwaccess_lock ++ * ++ * Return: Context to use for specified slot. ++ * NULL if no contexts present for specified slot ++ */ ++static struct kbase_context *kbase_js_ctx_list_pop_head_nolock( ++ struct kbase_device *kbdev, ++ int js) ++{ ++ struct kbase_context *kctx; ++ int i; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ for (i = 0; i < KBASE_JS_ATOM_SCHED_PRIO_COUNT; i++) { ++ if (list_empty(&kbdev->js_data.ctx_list_pullable[js][i])) ++ continue; ++ ++ kctx = list_entry(kbdev->js_data.ctx_list_pullable[js][i].next, ++ struct kbase_context, ++ jctx.sched_info.ctx.ctx_list_entry[js]); ++ ++ list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); ++ dev_dbg(kbdev->dev, ++ "Popped %p from the pullable queue (s:%d)\n", ++ (void *)kctx, js); ++ return kctx; ++ } ++ return NULL; ++} ++ ++/** ++ * kbase_js_ctx_list_pop_head - Pop the head context off the per-slot pullable ++ * queue. ++ * @kbdev: Device pointer ++ * @js: Job slot to use ++ * ++ * Return: Context to use for specified slot. ++ * NULL if no contexts present for specified slot ++ */ ++static struct kbase_context *kbase_js_ctx_list_pop_head( ++ struct kbase_device *kbdev, int js) ++{ ++ struct kbase_context *kctx; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kctx = kbase_js_ctx_list_pop_head_nolock(kbdev, js); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ return kctx; ++} ++ ++/** ++ * kbase_js_ctx_pullable - Return if a context can be pulled from on the ++ * specified slot ++ * @kctx: Context pointer ++ * @js: Job slot to use ++ * @is_scheduled: true if the context is currently scheduled ++ * ++ * Caller must hold hwaccess_lock ++ * ++ * Return: true if context can be pulled from on specified slot ++ * false otherwise ++ */ ++static bool kbase_js_ctx_pullable(struct kbase_context *kctx, int js, ++ bool is_scheduled) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ struct kbase_jd_atom *katom; ++ struct kbase_device *kbdev = kctx->kbdev; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ js_devdata = &kbdev->js_data; ++ ++ if (is_scheduled) { ++ if (!kbasep_js_is_submit_allowed(js_devdata, kctx)) { ++ dev_dbg(kbdev->dev, "JS: No submit allowed for kctx %p\n", ++ (void *)kctx); ++ return false; ++ } ++ } ++ katom = jsctx_rb_peek(kctx, js); ++ if (!katom) { ++ dev_dbg(kbdev->dev, "JS: No pullable atom in kctx %p (s:%d)\n", ++ (void *)kctx, js); ++ return false; /* No pullable atoms */ ++ } ++ if (kctx->blocked_js[js][katom->sched_priority]) { ++ dev_dbg(kbdev->dev, ++ "JS: kctx %p is blocked from submitting atoms at priority %d (s:%d)\n", ++ (void *)kctx, katom->sched_priority, js); ++ return false; ++ } ++ if (atomic_read(&katom->blocked)) { ++ dev_dbg(kbdev->dev, "JS: Atom %p is blocked in js_ctx_pullable\n", ++ (void *)katom); ++ return false; /* next atom blocked */ ++ } ++ if (kbase_js_atom_blocked_on_x_dep(katom)) { ++ if (katom->x_pre_dep->gpu_rb_state == ++ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB || ++ katom->x_pre_dep->will_fail_event_code) { ++ dev_dbg(kbdev->dev, ++ "JS: X pre-dep %p is not present in slot FIFO or will fail\n", ++ (void *)katom->x_pre_dep); ++ return false; ++ } ++ if ((katom->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER) && ++ kbase_backend_nr_atoms_on_slot(kctx->kbdev, js)) { ++ dev_dbg(kbdev->dev, ++ "JS: Atom %p has cross-slot fail dependency and atoms on slot (s:%d)\n", ++ (void *)katom, js); ++ return false; ++ } ++ } ++ ++ dev_dbg(kbdev->dev, "JS: Atom %p is pullable in kctx %p (s:%d)\n", ++ (void *)katom, (void *)kctx, js); ++ ++ return true; ++} ++ ++static bool kbase_js_dep_validate(struct kbase_context *kctx, ++ struct kbase_jd_atom *katom) ++{ ++ struct kbase_device *kbdev = kctx->kbdev; ++ bool ret = true; ++ bool has_dep = false, has_x_dep = false; ++ int js = kbase_js_get_slot(kbdev, katom); ++ int prio = katom->sched_priority; ++ int i; ++ ++ for (i = 0; i < 2; i++) { ++ struct kbase_jd_atom *dep_atom = katom->dep[i].atom; ++ ++ if (dep_atom) { ++ int dep_js = kbase_js_get_slot(kbdev, dep_atom); ++ int dep_prio = dep_atom->sched_priority; ++ ++ dev_dbg(kbdev->dev, ++ "Checking dep %d of atom %p (s:%d) on %p (s:%d)\n", ++ i, (void *)katom, js, (void *)dep_atom, dep_js); ++ ++ /* Dependent atom must already have been submitted */ ++ if (!(dep_atom->atom_flags & ++ KBASE_KATOM_FLAG_JSCTX_IN_TREE)) { ++ dev_dbg(kbdev->dev, ++ "Blocker not submitted yet\n"); ++ ret = false; ++ break; ++ } ++ ++ /* Dependencies with different priorities can't ++ be represented in the ringbuffer */ ++ if (prio != dep_prio) { ++ dev_dbg(kbdev->dev, ++ "Different atom priorities\n"); ++ ret = false; ++ break; ++ } ++ ++ if (js == dep_js) { ++ /* Only one same-slot dependency can be ++ * represented in the ringbuffer */ ++ if (has_dep) { ++ dev_dbg(kbdev->dev, ++ "Too many same-slot deps\n"); ++ ret = false; ++ break; ++ } ++ /* Each dependee atom can only have one ++ * same-slot dependency */ ++ if (dep_atom->post_dep) { ++ dev_dbg(kbdev->dev, ++ "Too many same-slot successors\n"); ++ ret = false; ++ break; ++ } ++ has_dep = true; ++ } else { ++ /* Only one cross-slot dependency can be ++ * represented in the ringbuffer */ ++ if (has_x_dep) { ++ dev_dbg(kbdev->dev, ++ "Too many cross-slot deps\n"); ++ ret = false; ++ break; ++ } ++ /* Each dependee atom can only have one ++ * cross-slot dependency */ ++ if (dep_atom->x_post_dep) { ++ dev_dbg(kbdev->dev, ++ "Too many cross-slot successors\n"); ++ ret = false; ++ break; ++ } ++ /* The dependee atom can not already be in the ++ * HW access ringbuffer */ ++ if (dep_atom->gpu_rb_state != ++ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { ++ dev_dbg(kbdev->dev, ++ "Blocker already in ringbuffer (state:%d)\n", ++ dep_atom->gpu_rb_state); ++ ret = false; ++ break; ++ } ++ /* The dependee atom can not already have ++ * completed */ ++ if (dep_atom->status != ++ KBASE_JD_ATOM_STATE_IN_JS) { ++ dev_dbg(kbdev->dev, ++ "Blocker already completed (status:%d)\n", ++ dep_atom->status); ++ ret = false; ++ break; ++ } ++ ++ has_x_dep = true; ++ } ++ ++ /* Dependency can be represented in ringbuffers */ ++ } ++ } ++ ++ /* If dependencies can be represented by ringbuffer then clear them from ++ * atom structure */ ++ if (ret) { ++ for (i = 0; i < 2; i++) { ++ struct kbase_jd_atom *dep_atom = katom->dep[i].atom; ++ ++ if (dep_atom) { ++ int dep_js = kbase_js_get_slot(kbdev, dep_atom); ++ ++ dev_dbg(kbdev->dev, ++ "Clearing dep %d of atom %p (s:%d) on %p (s:%d)\n", ++ i, (void *)katom, js, (void *)dep_atom, ++ dep_js); ++ ++ if ((js != dep_js) && ++ (dep_atom->status != ++ KBASE_JD_ATOM_STATE_COMPLETED) ++ && (dep_atom->status != ++ KBASE_JD_ATOM_STATE_HW_COMPLETED) ++ && (dep_atom->status != ++ KBASE_JD_ATOM_STATE_UNUSED)) { ++ ++ katom->atom_flags |= ++ KBASE_KATOM_FLAG_X_DEP_BLOCKED; ++ ++ dev_dbg(kbdev->dev, "Set X_DEP flag on atom %p\n", ++ (void *)katom); ++ ++ katom->x_pre_dep = dep_atom; ++ dep_atom->x_post_dep = katom; ++ if (kbase_jd_katom_dep_type( ++ &katom->dep[i]) == ++ BASE_JD_DEP_TYPE_DATA) ++ katom->atom_flags |= ++ KBASE_KATOM_FLAG_FAIL_BLOCKER; ++ } ++ if ((kbase_jd_katom_dep_type(&katom->dep[i]) ++ == BASE_JD_DEP_TYPE_DATA) && ++ (js == dep_js)) { ++ katom->pre_dep = dep_atom; ++ dep_atom->post_dep = katom; ++ } ++ ++ list_del(&katom->dep_item[i]); ++ kbase_jd_katom_dep_clear(&katom->dep[i]); ++ } ++ } ++ } else { ++ dev_dbg(kbdev->dev, ++ "Deps of atom %p (s:%d) could not be represented\n", ++ (void *)katom, js); ++ } ++ ++ return ret; ++} ++ ++void kbase_js_set_ctx_priority(struct kbase_context *kctx, int new_priority) ++{ ++ struct kbase_device *kbdev = kctx->kbdev; ++ int js; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ /* Move kctx to the pullable/upullable list as per the new priority */ ++ if (new_priority != kctx->priority) { ++ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { ++ if (kctx->slots_pullable & (1 << js)) ++ list_move_tail(&kctx->jctx.sched_info.ctx.ctx_list_entry[js], ++ &kbdev->js_data.ctx_list_pullable[js][new_priority]); ++ else ++ list_move_tail(&kctx->jctx.sched_info.ctx.ctx_list_entry[js], ++ &kbdev->js_data.ctx_list_unpullable[js][new_priority]); ++ } ++ ++ kctx->priority = new_priority; ++ } ++} ++ ++void kbase_js_update_ctx_priority(struct kbase_context *kctx) ++{ ++ struct kbase_device *kbdev = kctx->kbdev; ++ int new_priority = KBASE_JS_ATOM_SCHED_PRIO_LOW; ++ int prio; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (kbdev->js_ctx_scheduling_mode == KBASE_JS_SYSTEM_PRIORITY_MODE) { ++ /* Determine the new priority for context, as per the priority ++ * of currently in-use atoms. ++ */ ++ for (prio = KBASE_JS_ATOM_SCHED_PRIO_HIGH; ++ prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) { ++ if (kctx->atoms_count[prio]) { ++ new_priority = prio; ++ break; ++ } ++ } ++ } ++ ++ kbase_js_set_ctx_priority(kctx, new_priority); ++} ++ ++/** ++ * js_add_start_rp() - Add an atom that starts a renderpass to the job scheduler ++ * @start_katom: Pointer to the atom to be added. ++ * Return: 0 if successful or a negative value on failure. ++ */ ++static int js_add_start_rp(struct kbase_jd_atom *const start_katom) ++{ ++ struct kbase_context *const kctx = start_katom->kctx; ++ struct kbase_jd_renderpass *rp; ++ struct kbase_device *const kbdev = kctx->kbdev; ++ unsigned long flags; ++ ++ lockdep_assert_held(&kctx->jctx.lock); ++ ++ if (WARN_ON(!(start_katom->core_req & BASE_JD_REQ_START_RENDERPASS))) ++ return -EINVAL; ++ ++ if (start_katom->core_req & BASE_JD_REQ_END_RENDERPASS) ++ return -EINVAL; ++ ++ compiletime_assert((1ull << (sizeof(start_katom->renderpass_id) * 8)) <= ++ ARRAY_SIZE(kctx->jctx.renderpasses), ++ "Should check invalid access to renderpasses"); ++ ++ rp = &kctx->jctx.renderpasses[start_katom->renderpass_id]; ++ ++ if (rp->state != KBASE_JD_RP_COMPLETE) ++ return -EINVAL; ++ ++ dev_dbg(kctx->kbdev->dev, "JS add start atom %p of RP %d\n", ++ (void *)start_katom, start_katom->renderpass_id); ++ ++ /* The following members are read when updating the job slot ++ * ringbuffer/fifo therefore they require additional locking. ++ */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ rp->state = KBASE_JD_RP_START; ++ rp->start_katom = start_katom; ++ rp->end_katom = NULL; ++ INIT_LIST_HEAD(&rp->oom_reg_list); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ return 0; ++} ++ ++/** ++ * js_add_end_rp() - Add an atom that ends a renderpass to the job scheduler ++ * @end_katom: Pointer to the atom to be added. ++ * Return: 0 if successful or a negative value on failure. ++ */ ++static int js_add_end_rp(struct kbase_jd_atom *const end_katom) ++{ ++ struct kbase_context *const kctx = end_katom->kctx; ++ struct kbase_jd_renderpass *rp; ++ struct kbase_device *const kbdev = kctx->kbdev; ++ ++ lockdep_assert_held(&kctx->jctx.lock); ++ ++ if (WARN_ON(!(end_katom->core_req & BASE_JD_REQ_END_RENDERPASS))) ++ return -EINVAL; ++ ++ if (end_katom->core_req & BASE_JD_REQ_START_RENDERPASS) ++ return -EINVAL; ++ ++ compiletime_assert((1ull << (sizeof(end_katom->renderpass_id) * 8)) <= ++ ARRAY_SIZE(kctx->jctx.renderpasses), ++ "Should check invalid access to renderpasses"); ++ ++ rp = &kctx->jctx.renderpasses[end_katom->renderpass_id]; ++ ++ dev_dbg(kbdev->dev, "JS add end atom %p in state %d of RP %d\n", ++ (void *)end_katom, (int)rp->state, end_katom->renderpass_id); ++ ++ if (rp->state == KBASE_JD_RP_COMPLETE) ++ return -EINVAL; ++ ++ if (rp->end_katom == NULL) { ++ /* We can't be in a retry state until the fragment job chain ++ * has completed. ++ */ ++ unsigned long flags; ++ ++ WARN_ON(rp->state == KBASE_JD_RP_RETRY); ++ WARN_ON(rp->state == KBASE_JD_RP_RETRY_PEND_OOM); ++ WARN_ON(rp->state == KBASE_JD_RP_RETRY_OOM); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ rp->end_katom = end_katom; ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ } else ++ WARN_ON(rp->end_katom != end_katom); ++ ++ return 0; ++} ++ ++bool kbasep_js_add_job(struct kbase_context *kctx, ++ struct kbase_jd_atom *atom) ++{ ++ unsigned long flags; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ struct kbase_device *kbdev; ++ struct kbasep_js_device_data *js_devdata; ++ int err = 0; ++ ++ bool enqueue_required = false; ++ bool timer_sync = false; ++ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ KBASE_DEBUG_ASSERT(atom != NULL); ++ lockdep_assert_held(&kctx->jctx.lock); ++ ++ kbdev = kctx->kbdev; ++ js_devdata = &kbdev->js_data; ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ mutex_lock(&js_devdata->queue_mutex); ++ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); ++ ++ if (atom->core_req & BASE_JD_REQ_START_RENDERPASS) ++ err = js_add_start_rp(atom); ++ else if (atom->core_req & BASE_JD_REQ_END_RENDERPASS) ++ err = js_add_end_rp(atom); ++ ++ if (err < 0) { ++ atom->event_code = BASE_JD_EVENT_JOB_INVALID; ++ atom->status = KBASE_JD_ATOM_STATE_COMPLETED; ++ goto out_unlock; ++ } ++ ++ /* ++ * Begin Runpool transaction ++ */ ++ mutex_lock(&js_devdata->runpool_mutex); ++ ++ /* Refcount ctx.nr_jobs */ ++ KBASE_DEBUG_ASSERT(js_kctx_info->ctx.nr_jobs < U32_MAX); ++ ++(js_kctx_info->ctx.nr_jobs); ++ dev_dbg(kbdev->dev, "Add atom %p to kctx %p; now %d in ctx\n", ++ (void *)atom, (void *)kctx, js_kctx_info->ctx.nr_jobs); ++ ++ /* Lock for state available during IRQ */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ if (++kctx->atoms_count[atom->sched_priority] == 1) ++ kbase_js_update_ctx_priority(kctx); ++ ++ if (!kbase_js_dep_validate(kctx, atom)) { ++ /* Dependencies could not be represented */ ++ --(js_kctx_info->ctx.nr_jobs); ++ dev_dbg(kbdev->dev, ++ "Remove atom %p from kctx %p; now %d in ctx\n", ++ (void *)atom, (void *)kctx, js_kctx_info->ctx.nr_jobs); ++ ++ /* Setting atom status back to queued as it still has unresolved ++ * dependencies */ ++ atom->status = KBASE_JD_ATOM_STATE_QUEUED; ++ dev_dbg(kbdev->dev, "Atom %p status to queued\n", (void *)atom); ++ ++ /* Undo the count, as the atom will get added again later but ++ * leave the context priority adjusted or boosted, in case if ++ * this was the first higher priority atom received for this ++ * context. ++ * This will prevent the scenario of priority inversion, where ++ * another context having medium priority atoms keeps getting ++ * scheduled over this context, which is having both lower and ++ * higher priority atoms, but higher priority atoms are blocked ++ * due to dependency on lower priority atoms. With priority ++ * boost the high priority atom will get to run at earliest. ++ */ ++ kctx->atoms_count[atom->sched_priority]--; ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ ++ goto out_unlock; ++ } ++ ++ enqueue_required = kbase_js_dep_resolved_submit(kctx, atom); ++ ++ KBASE_KTRACE_ADD_JM_REFCOUNT(kbdev, JS_ADD_JOB, kctx, atom, atom->jc, ++ kbase_ktrace_get_ctx_refcnt(kctx)); ++ ++ /* Context Attribute Refcounting */ ++ kbasep_js_ctx_attr_ctx_retain_atom(kbdev, kctx, atom); ++ ++ if (enqueue_required) { ++ if (kbase_js_ctx_pullable(kctx, atom->slot_nr, false)) ++ timer_sync = kbase_js_ctx_list_add_pullable_nolock( ++ kbdev, kctx, atom->slot_nr); ++ else ++ timer_sync = kbase_js_ctx_list_add_unpullable_nolock( ++ kbdev, kctx, atom->slot_nr); ++ } ++ /* If this context is active and the atom is the first on its slot, ++ * kick the job manager to attempt to fast-start the atom */ ++ if (enqueue_required && kctx == ++ kbdev->hwaccess.active_kctx[atom->slot_nr]) ++ kbase_jm_try_kick(kbdev, 1 << atom->slot_nr); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ if (timer_sync) ++ kbase_backend_ctx_count_changed(kbdev); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ /* End runpool transaction */ ++ ++ if (!kbase_ctx_flag(kctx, KCTX_SCHEDULED)) { ++ if (kbase_ctx_flag(kctx, KCTX_DYING)) { ++ /* A job got added while/after kbase_job_zap_context() ++ * was called on a non-scheduled context. Kill that job ++ * by killing the context. */ ++ kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, ++ false); ++ } else if (js_kctx_info->ctx.nr_jobs == 1) { ++ /* Handle Refcount going from 0 to 1: schedule the ++ * context on the Queue */ ++ KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ dev_dbg(kbdev->dev, "JS: Enqueue Context %p", kctx); ++ ++ /* Queue was updated - caller must try to ++ * schedule the head context */ ++ WARN_ON(!enqueue_required); ++ } ++ } ++out_unlock: ++ dev_dbg(kbdev->dev, "Enqueue of kctx %p is %srequired\n", ++ kctx, enqueue_required ? "" : "not "); ++ ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ ++ mutex_unlock(&js_devdata->queue_mutex); ++ ++ return enqueue_required; ++} ++ ++void kbasep_js_remove_job(struct kbase_device *kbdev, ++ struct kbase_context *kctx, struct kbase_jd_atom *atom) ++{ ++ struct kbasep_js_kctx_info *js_kctx_info; ++ unsigned long flags; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ KBASE_DEBUG_ASSERT(atom != NULL); ++ ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ KBASE_KTRACE_ADD_JM_REFCOUNT(kbdev, JS_REMOVE_JOB, kctx, atom, atom->jc, ++ kbase_ktrace_get_ctx_refcnt(kctx)); ++ ++ /* De-refcount ctx.nr_jobs */ ++ KBASE_DEBUG_ASSERT(js_kctx_info->ctx.nr_jobs > 0); ++ --(js_kctx_info->ctx.nr_jobs); ++ dev_dbg(kbdev->dev, ++ "Remove atom %p from kctx %p; now %d in ctx\n", ++ (void *)atom, (void *)kctx, js_kctx_info->ctx.nr_jobs); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ if (--kctx->atoms_count[atom->sched_priority] == 0) ++ kbase_js_update_ctx_priority(kctx); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++} ++ ++bool kbasep_js_remove_cancelled_job(struct kbase_device *kbdev, ++ struct kbase_context *kctx, struct kbase_jd_atom *katom) ++{ ++ unsigned long flags; ++ struct kbasep_js_atom_retained_state katom_retained_state; ++ bool attr_state_changed; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ KBASE_DEBUG_ASSERT(katom != NULL); ++ ++ kbasep_js_atom_retained_state_copy(&katom_retained_state, katom); ++ kbasep_js_remove_job(kbdev, kctx, katom); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ /* The atom has 'finished' (will not be re-run), so no need to call ++ * kbasep_js_has_atom_finished(). ++ * ++ * This is because it returns false for soft-stopped atoms, but we ++ * want to override that, because we're cancelling an atom regardless of ++ * whether it was soft-stopped or not */ ++ attr_state_changed = kbasep_js_ctx_attr_ctx_release_atom(kbdev, kctx, ++ &katom_retained_state); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ return attr_state_changed; ++} ++ ++/** ++ * kbasep_js_run_jobs_after_ctx_and_atom_release - Try running more jobs after ++ * releasing a context and/or atom ++ * @kbdev: The kbase_device to operate on ++ * @kctx: The kbase_context to operate on ++ * @katom_retained_state: Retained state from the atom ++ * @runpool_ctx_attr_change: True if the runpool context attributes have changed ++ * ++ * This collates a set of actions that must happen whilst hwaccess_lock is held. ++ * ++ * This includes running more jobs when: ++ * - The previously released kctx caused a ctx attribute change, ++ * - The released atom caused a ctx attribute change, ++ * - Slots were previously blocked due to affinity restrictions, ++ * - Submission during IRQ handling failed. ++ * ++ * Return: %KBASEP_JS_RELEASE_RESULT_SCHED_ALL if context attributes were ++ * changed. The caller should try scheduling all contexts ++ */ ++static kbasep_js_release_result kbasep_js_run_jobs_after_ctx_and_atom_release( ++ struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ struct kbasep_js_atom_retained_state *katom_retained_state, ++ bool runpool_ctx_attr_change) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ kbasep_js_release_result result = 0; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ KBASE_DEBUG_ASSERT(katom_retained_state != NULL); ++ js_devdata = &kbdev->js_data; ++ ++ lockdep_assert_held(&kctx->jctx.sched_info.ctx.jsctx_mutex); ++ lockdep_assert_held(&js_devdata->runpool_mutex); ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (js_devdata->nr_user_contexts_running != 0 && runpool_ctx_attr_change) { ++ /* A change in runpool ctx attributes might mean we can ++ * run more jobs than before */ ++ result = KBASEP_JS_RELEASE_RESULT_SCHED_ALL; ++ ++ KBASE_KTRACE_ADD_JM_SLOT(kbdev, JD_DONE_TRY_RUN_NEXT_JOB, ++ kctx, NULL, 0u, 0); ++ } ++ return result; ++} ++ ++/** ++ * kbasep_js_runpool_release_ctx_internal - Internal function to release the reference ++ * on a ctx and an atom's "retained state", only ++ * taking the runpool and as transaction mutexes ++ * @kbdev: The kbase_device to operate on ++ * @kctx: The kbase_context to operate on ++ * @katom_retained_state: Retained state from the atom ++ * ++ * This also starts more jobs running in the case of an ctx-attribute state change ++ * ++ * This does none of the followup actions for scheduling: ++ * - It does not schedule in a new context ++ * - It does not requeue or handle dying contexts ++ * ++ * For those tasks, just call kbasep_js_runpool_release_ctx() instead ++ * ++ * Has following requirements ++ * - Context is scheduled in, and kctx->as_nr matches kctx_as_nr ++ * - Context has a non-zero refcount ++ * - Caller holds js_kctx_info->ctx.jsctx_mutex ++ * - Caller holds js_devdata->runpool_mutex ++ * ++ * Return: A bitpattern, containing KBASEP_JS_RELEASE_RESULT_* flags, indicating ++ * the result of releasing a context that whether the caller should try ++ * scheduling a new context or should try scheduling all contexts. ++ */ ++static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal( ++ struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ struct kbasep_js_atom_retained_state *katom_retained_state) ++{ ++ unsigned long flags; ++ struct kbasep_js_device_data *js_devdata; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ ++ kbasep_js_release_result release_result = 0u; ++ bool runpool_ctx_attr_change = false; ++ int kctx_as_nr; ++ int new_ref_count; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ js_kctx_info = &kctx->jctx.sched_info; ++ js_devdata = &kbdev->js_data; ++ ++ /* Ensure context really is scheduled in */ ++ KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ ++ kctx_as_nr = kctx->as_nr; ++ KBASE_DEBUG_ASSERT(kctx_as_nr != KBASEP_AS_NR_INVALID); ++ KBASE_DEBUG_ASSERT(atomic_read(&kctx->refcount) > 0); ++ ++ /* ++ * Transaction begins on AS and runpool_irq ++ * ++ * Assert about out calling contract ++ */ ++ mutex_lock(&kbdev->pm.lock); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ KBASE_DEBUG_ASSERT(kctx_as_nr == kctx->as_nr); ++ KBASE_DEBUG_ASSERT(atomic_read(&kctx->refcount) > 0); ++ ++ /* Update refcount */ ++ kbase_ctx_sched_release_ctx(kctx); ++ new_ref_count = atomic_read(&kctx->refcount); ++ ++ /* Release the atom if it finished (i.e. wasn't soft-stopped) */ ++ if (kbasep_js_has_atom_finished(katom_retained_state)) ++ runpool_ctx_attr_change |= kbasep_js_ctx_attr_ctx_release_atom( ++ kbdev, kctx, katom_retained_state); ++ ++ if (new_ref_count == 2 && kbase_ctx_flag(kctx, KCTX_PRIVILEGED) && ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++ !kbase_pm_is_gpu_lost(kbdev) && ++#endif ++ !kbase_pm_is_suspending(kbdev)) { ++ /* Context is kept scheduled into an address space even when ++ * there are no jobs, in this case we have to handle the ++ * situation where all jobs have been evicted from the GPU and ++ * submission is disabled. ++ * ++ * At this point we re-enable submission to allow further jobs ++ * to be executed ++ */ ++ kbasep_js_set_submit_allowed(js_devdata, kctx); ++ } ++ ++ /* Make a set of checks to see if the context should be scheduled out. ++ * Note that there'll always be at least 1 reference to the context ++ * which was previously acquired by kbasep_js_schedule_ctx(). */ ++ if (new_ref_count == 1 && ++ (!kbasep_js_is_submit_allowed(js_devdata, kctx) || ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++ kbase_pm_is_gpu_lost(kbdev) || ++#endif ++ kbase_pm_is_suspending(kbdev))) { ++ int num_slots = kbdev->gpu_props.num_job_slots; ++ int slot; ++ ++ /* Last reference, and we've been told to remove this context ++ * from the Run Pool */ ++ dev_dbg(kbdev->dev, "JS: RunPool Remove Context %p because refcount=%d, jobs=%d, allowed=%d", ++ kctx, new_ref_count, js_kctx_info->ctx.nr_jobs, ++ kbasep_js_is_submit_allowed(js_devdata, kctx)); ++ ++ KBASE_TLSTREAM_TL_NRET_AS_CTX(kbdev, &kbdev->as[kctx->as_nr], kctx); ++ ++ kbase_backend_release_ctx_irq(kbdev, kctx); ++ ++ for (slot = 0; slot < num_slots; slot++) { ++ if (kbdev->hwaccess.active_kctx[slot] == kctx) { ++ dev_dbg(kbdev->dev, "Marking kctx %p as inactive (s:%d)\n", ++ (void *)kctx, slot); ++ kbdev->hwaccess.active_kctx[slot] = NULL; ++ } ++ } ++ ++ /* Ctx Attribute handling ++ * ++ * Releasing atoms attributes must either happen before this, or ++ * after the KCTX_SHEDULED flag is changed, otherwise we ++ * double-decount the attributes ++ */ ++ runpool_ctx_attr_change |= ++ kbasep_js_ctx_attr_runpool_release_ctx(kbdev, kctx); ++ ++ /* Releasing the context and katom retained state can allow ++ * more jobs to run */ ++ release_result |= ++ kbasep_js_run_jobs_after_ctx_and_atom_release(kbdev, ++ kctx, katom_retained_state, ++ runpool_ctx_attr_change); ++ ++ /* ++ * Transaction ends on AS and runpool_irq: ++ * ++ * By this point, the AS-related data is now clear and ready ++ * for re-use. ++ * ++ * Since releases only occur once for each previous successful ++ * retain, and no more retains are allowed on this context, no ++ * other thread will be operating in this ++ * code whilst we are ++ */ ++ ++ /* Recalculate pullable status for all slots */ ++ for (slot = 0; slot < num_slots; slot++) { ++ if (kbase_js_ctx_pullable(kctx, slot, false)) ++ kbase_js_ctx_list_add_pullable_nolock(kbdev, ++ kctx, slot); ++ } ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ kbase_backend_release_ctx_noirq(kbdev, kctx); ++ ++ mutex_unlock(&kbdev->pm.lock); ++ ++ /* Note: Don't reuse kctx_as_nr now */ ++ ++ /* Synchronize with any timers */ ++ kbase_backend_ctx_count_changed(kbdev); ++ ++ /* update book-keeping info */ ++ kbase_ctx_flag_clear(kctx, KCTX_SCHEDULED); ++ /* Signal any waiter that the context is not scheduled, so is ++ * safe for termination - once the jsctx_mutex is also dropped, ++ * and jobs have finished. */ ++ wake_up(&js_kctx_info->ctx.is_scheduled_wait); ++ ++ /* Queue an action to occur after we've dropped the lock */ ++ release_result |= KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED | ++ KBASEP_JS_RELEASE_RESULT_SCHED_ALL; ++ } else { ++ kbasep_js_run_jobs_after_ctx_and_atom_release(kbdev, kctx, ++ katom_retained_state, runpool_ctx_attr_change); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&kbdev->pm.lock); ++ } ++ ++ return release_result; ++} ++ ++void kbasep_js_runpool_release_ctx_nolock(struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++ struct kbasep_js_atom_retained_state katom_retained_state; ++ ++ /* Setup a dummy katom_retained_state */ ++ kbasep_js_atom_retained_state_init_invalid(&katom_retained_state); ++ ++ kbasep_js_runpool_release_ctx_internal(kbdev, kctx, ++ &katom_retained_state); ++} ++ ++void kbasep_js_runpool_requeue_or_kill_ctx(struct kbase_device *kbdev, ++ struct kbase_context *kctx, bool has_pm_ref) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ ++ /* This is called if and only if you've you've detached the context from ++ * the Runpool Queue, and not added it back to the Runpool ++ */ ++ KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ ++ if (kbase_ctx_flag(kctx, KCTX_DYING)) { ++ /* Dying: don't requeue, but kill all jobs on the context. This ++ * happens asynchronously */ ++ dev_dbg(kbdev->dev, ++ "JS: ** Killing Context %p on RunPool Remove **", kctx); ++ kbase_js_foreach_ctx_job(kctx, &kbase_jd_cancel); ++ } ++} ++ ++void kbasep_js_runpool_release_ctx_and_katom_retained_state( ++ struct kbase_device *kbdev, struct kbase_context *kctx, ++ struct kbasep_js_atom_retained_state *katom_retained_state) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ kbasep_js_release_result release_result; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ js_kctx_info = &kctx->jctx.sched_info; ++ js_devdata = &kbdev->js_data; ++ ++ mutex_lock(&js_devdata->queue_mutex); ++ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); ++ mutex_lock(&js_devdata->runpool_mutex); ++ ++ release_result = kbasep_js_runpool_release_ctx_internal(kbdev, kctx, ++ katom_retained_state); ++ ++ /* Drop the runpool mutex to allow requeing kctx */ ++ mutex_unlock(&js_devdata->runpool_mutex); ++ ++ if ((release_result & KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED) != 0u) ++ kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, true); ++ ++ /* Drop the jsctx_mutex to allow scheduling in a new context */ ++ ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ mutex_unlock(&js_devdata->queue_mutex); ++ ++ if (release_result & KBASEP_JS_RELEASE_RESULT_SCHED_ALL) ++ kbase_js_sched_all(kbdev); ++} ++ ++void kbasep_js_runpool_release_ctx(struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++ struct kbasep_js_atom_retained_state katom_retained_state; ++ ++ kbasep_js_atom_retained_state_init_invalid(&katom_retained_state); ++ ++ kbasep_js_runpool_release_ctx_and_katom_retained_state(kbdev, kctx, ++ &katom_retained_state); ++} ++ ++/* Variant of kbasep_js_runpool_release_ctx() that doesn't call into ++ * kbase_js_sched_all() */ ++static void kbasep_js_runpool_release_ctx_no_schedule( ++ struct kbase_device *kbdev, struct kbase_context *kctx) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ kbasep_js_release_result release_result; ++ struct kbasep_js_atom_retained_state katom_retained_state_struct; ++ struct kbasep_js_atom_retained_state *katom_retained_state = ++ &katom_retained_state_struct; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ js_kctx_info = &kctx->jctx.sched_info; ++ js_devdata = &kbdev->js_data; ++ kbasep_js_atom_retained_state_init_invalid(katom_retained_state); ++ ++ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); ++ mutex_lock(&js_devdata->runpool_mutex); ++ ++ release_result = kbasep_js_runpool_release_ctx_internal(kbdev, kctx, ++ katom_retained_state); ++ ++ /* Drop the runpool mutex to allow requeing kctx */ ++ mutex_unlock(&js_devdata->runpool_mutex); ++ if ((release_result & KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED) != 0u) ++ kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, true); ++ ++ /* Drop the jsctx_mutex to allow scheduling in a new context */ ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ ++ /* NOTE: could return release_result if the caller would like to know ++ * whether it should schedule a new context, but currently no callers do ++ */ ++} ++ ++void kbase_js_set_timeouts(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ kbase_backend_timeouts_changed(kbdev); ++} ++ ++static bool kbasep_js_schedule_ctx(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ int js) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ unsigned long flags; ++ bool kctx_suspended = false; ++ int as_nr; ++ ++ dev_dbg(kbdev->dev, "Scheduling kctx %p (s:%d)\n", kctx, js); ++ ++ js_devdata = &kbdev->js_data; ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ /* Pick available address space for this context */ ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ as_nr = kbase_ctx_sched_retain_ctx(kctx); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ if (as_nr == KBASEP_AS_NR_INVALID) { ++ as_nr = kbase_backend_find_and_release_free_address_space( ++ kbdev, kctx); ++ if (as_nr != KBASEP_AS_NR_INVALID) { ++ /* Attempt to retain the context again, this should ++ * succeed */ ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ as_nr = kbase_ctx_sched_retain_ctx(kctx); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ ++ WARN_ON(as_nr == KBASEP_AS_NR_INVALID); ++ } ++ } ++ if (as_nr == KBASEP_AS_NR_INVALID) ++ return false; /* No address spaces currently available */ ++ ++ /* ++ * Atomic transaction on the Context and Run Pool begins ++ */ ++ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); ++ mutex_lock(&js_devdata->runpool_mutex); ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ /* Check to see if context is dying due to kbase_job_zap_context() */ ++ if (kbase_ctx_flag(kctx, KCTX_DYING)) { ++ /* Roll back the transaction so far and return */ ++ kbase_ctx_sched_release_ctx(kctx); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ ++ return false; ++ } ++ ++ KBASE_KTRACE_ADD_JM_REFCOUNT(kbdev, JS_TRY_SCHEDULE_HEAD_CTX, kctx, NULL, ++ 0u, ++ kbase_ktrace_get_ctx_refcnt(kctx)); ++ ++ kbase_ctx_flag_set(kctx, KCTX_SCHEDULED); ++ ++ /* Assign context to previously chosen address space */ ++ if (!kbase_backend_use_ctx(kbdev, kctx, as_nr)) { ++ /* Roll back the transaction so far and return */ ++ kbase_ctx_sched_release_ctx(kctx); ++ kbase_ctx_flag_clear(kctx, KCTX_SCHEDULED); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ ++ return false; ++ } ++ ++ kbdev->hwaccess.active_kctx[js] = kctx; ++ ++ KBASE_TLSTREAM_TL_RET_AS_CTX(kbdev, &kbdev->as[kctx->as_nr], kctx); ++ ++ /* Cause any future waiter-on-termination to wait until the context is ++ * descheduled */ ++ wake_up(&js_kctx_info->ctx.is_scheduled_wait); ++ ++ /* Re-check for suspending: a suspend could've occurred, and all the ++ * contexts could've been removed from the runpool before we took this ++ * lock. In this case, we don't want to allow this context to run jobs, ++ * we just want it out immediately. ++ * ++ * The DMB required to read the suspend flag was issued recently as part ++ * of the hwaccess_lock locking. If a suspend occurs *after* that lock ++ * was taken (i.e. this condition doesn't execute), then the ++ * kbasep_js_suspend() code will cleanup this context instead (by virtue ++ * of it being called strictly after the suspend flag is set, and will ++ * wait for this lock to drop) */ ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++ if (kbase_pm_is_suspending(kbdev) || kbase_pm_is_gpu_lost(kbdev)) { ++#else ++ if (kbase_pm_is_suspending(kbdev)) { ++#endif ++ /* Cause it to leave at some later point */ ++ bool retained; ++ ++ retained = kbase_ctx_sched_inc_refcount_nolock(kctx); ++ KBASE_DEBUG_ASSERT(retained); ++ ++ kbasep_js_clear_submit_allowed(js_devdata, kctx); ++ kctx_suspended = true; ++ } ++ ++ kbase_ctx_flag_clear(kctx, KCTX_PULLED_SINCE_ACTIVE_JS0 << js); ++ ++ /* Transaction complete */ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ ++ /* Synchronize with any timers */ ++ kbase_backend_ctx_count_changed(kbdev); ++ ++ mutex_unlock(&js_devdata->runpool_mutex); ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ /* Note: after this point, the context could potentially get scheduled ++ * out immediately */ ++ ++ if (kctx_suspended) { ++ /* Finishing forcing out the context due to a suspend. Use a ++ * variant of kbasep_js_runpool_release_ctx() that doesn't ++ * schedule a new context, to prevent a risk of recursion back ++ * into this function */ ++ kbasep_js_runpool_release_ctx_no_schedule(kbdev, kctx); ++ return false; ++ } ++ return true; ++} ++ ++static bool kbase_js_use_ctx(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ int js) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ if (kbase_ctx_flag(kctx, KCTX_SCHEDULED) && ++ kbase_backend_use_ctx_sched(kbdev, kctx, js)) { ++ ++ dev_dbg(kbdev->dev, ++ "kctx %p already has ASID - mark as active (s:%d)\n", ++ (void *)kctx, js); ++ ++ if (kbdev->hwaccess.active_kctx[js] != kctx) { ++ kbdev->hwaccess.active_kctx[js] = kctx; ++ kbase_ctx_flag_clear(kctx, ++ KCTX_PULLED_SINCE_ACTIVE_JS0 << js); ++ } ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ return true; /* Context already scheduled */ ++ } ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ return kbasep_js_schedule_ctx(kbdev, kctx, js); ++} ++ ++void kbasep_js_schedule_privileged_ctx(struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++ struct kbasep_js_kctx_info *js_kctx_info; ++ struct kbasep_js_device_data *js_devdata; ++ bool is_scheduled; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ ++ js_devdata = &kbdev->js_data; ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++ /* This should only happen in response to a system call ++ * from a user-space thread. ++ * In a non-arbitrated environment this can never happen ++ * whilst suspending. ++ * ++ * In an arbitrated environment, user-space threads can run ++ * while we are suspended (for example GPU not available ++ * to this VM), however in that case we will block on ++ * the wait event for KCTX_SCHEDULED, since no context ++ * can be scheduled until we have the GPU again. ++ */ ++ if (kbdev->arb.arb_if == NULL) ++ if (WARN_ON(kbase_pm_is_suspending(kbdev))) ++ return; ++#else ++ /* This should only happen in response to a system call ++ * from a user-space thread. ++ * In a non-arbitrated environment this can never happen ++ * whilst suspending. ++ */ ++ if (WARN_ON(kbase_pm_is_suspending(kbdev))) ++ return; ++#endif ++ ++ mutex_lock(&js_devdata->queue_mutex); ++ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); ++ ++ /* Mark the context as privileged */ ++ kbase_ctx_flag_set(kctx, KCTX_PRIVILEGED); ++ ++ is_scheduled = kbase_ctx_flag(kctx, KCTX_SCHEDULED); ++ if (!is_scheduled) { ++ /* Add the context to the pullable list */ ++ if (kbase_js_ctx_list_add_pullable_head(kbdev, kctx, 0)) ++ kbase_js_sync_timers(kbdev); ++ ++ /* Fast-starting requires the jsctx_mutex to be dropped, ++ * because it works on multiple ctxs */ ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ mutex_unlock(&js_devdata->queue_mutex); ++ ++ /* Try to schedule the context in */ ++ kbase_js_sched_all(kbdev); ++ ++ /* Wait for the context to be scheduled in */ ++ wait_event(kctx->jctx.sched_info.ctx.is_scheduled_wait, ++ kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ } else { ++ /* Already scheduled in - We need to retain it to keep the ++ * corresponding address space */ ++ WARN_ON(!kbase_ctx_sched_inc_refcount(kctx)); ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ mutex_unlock(&js_devdata->queue_mutex); ++ } ++} ++KBASE_EXPORT_TEST_API(kbasep_js_schedule_privileged_ctx); ++ ++void kbasep_js_release_privileged_ctx(struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++ struct kbasep_js_kctx_info *js_kctx_info; ++ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ /* We don't need to use the address space anymore */ ++ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); ++ kbase_ctx_flag_clear(kctx, KCTX_PRIVILEGED); ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ ++ /* Release the context - it will be scheduled out */ ++ kbasep_js_runpool_release_ctx(kbdev, kctx); ++ ++ kbase_js_sched_all(kbdev); ++} ++KBASE_EXPORT_TEST_API(kbasep_js_release_privileged_ctx); ++ ++void kbasep_js_suspend(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ struct kbasep_js_device_data *js_devdata; ++ int i; ++ u16 retained = 0u; ++ ++ KBASE_DEBUG_ASSERT(kbdev); ++ KBASE_DEBUG_ASSERT(kbase_pm_is_suspending(kbdev)); ++ js_devdata = &kbdev->js_data; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ /* Prevent all contexts from submitting */ ++ js_devdata->runpool_irq.submit_allowed = 0; ++ ++ /* Retain each of the contexts, so we can cause it to leave even if it ++ * had no refcount to begin with */ ++ for (i = BASE_MAX_NR_AS - 1; i >= 0; --i) { ++ struct kbase_context *kctx = kbdev->as_to_kctx[i]; ++ ++ retained = retained << 1; ++ ++ if (kctx && !(kbdev->as_free & (1u << i))) { ++ kbase_ctx_sched_retain_ctx_refcount(kctx); ++ retained |= 1u; ++ /* This loop will not have an effect on the privileged ++ * contexts as they would have an extra ref count ++ * compared to the normal contexts, so they will hold ++ * on to their address spaces. MMU will re-enabled for ++ * them on resume. ++ */ ++ } ++ } ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ /* De-ref the previous retain to ensure each context gets pulled out ++ * sometime later. */ ++ for (i = 0; ++ i < BASE_MAX_NR_AS; ++ ++i, retained = retained >> 1) { ++ struct kbase_context *kctx = kbdev->as_to_kctx[i]; ++ ++ if (retained & 1u) ++ kbasep_js_runpool_release_ctx(kbdev, kctx); ++ } ++ ++ /* Caller must wait for all Power Manager active references to be ++ * dropped */ ++} ++ ++void kbasep_js_resume(struct kbase_device *kbdev) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ int js, prio; ++ ++ KBASE_DEBUG_ASSERT(kbdev); ++ js_devdata = &kbdev->js_data; ++ KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev)); ++ ++ mutex_lock(&js_devdata->queue_mutex); ++ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { ++ for (prio = KBASE_JS_ATOM_SCHED_PRIO_HIGH; ++ prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) { ++ struct kbase_context *kctx, *n; ++ unsigned long flags; ++ ++#ifndef CONFIG_MALI_ARBITER_SUPPORT ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ list_for_each_entry_safe(kctx, n, ++ &kbdev->js_data.ctx_list_unpullable[js][prio], ++ jctx.sched_info.ctx.ctx_list_entry[js]) { ++ struct kbasep_js_kctx_info *js_kctx_info; ++ bool timer_sync = false; ++ ++ /* Drop lock so we can take kctx mutexes */ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, ++ flags); ++ ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); ++ mutex_lock(&js_devdata->runpool_mutex); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ if (!kbase_ctx_flag(kctx, KCTX_SCHEDULED) && ++ kbase_js_ctx_pullable(kctx, js, false)) ++ timer_sync = ++ kbase_js_ctx_list_add_pullable_nolock( ++ kbdev, kctx, js); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, ++ flags); ++ ++ if (timer_sync) ++ kbase_backend_ctx_count_changed(kbdev); ++ ++ mutex_unlock(&js_devdata->runpool_mutex); ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ ++ /* Take lock before accessing list again */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ } ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++#else ++ bool timer_sync = false; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ list_for_each_entry_safe(kctx, n, ++ &kbdev->js_data.ctx_list_unpullable[js][prio], ++ jctx.sched_info.ctx.ctx_list_entry[js]) { ++ ++ if (!kbase_ctx_flag(kctx, KCTX_SCHEDULED) && ++ kbase_js_ctx_pullable(kctx, js, false)) ++ timer_sync |= ++ kbase_js_ctx_list_add_pullable_nolock( ++ kbdev, kctx, js); ++ } ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ if (timer_sync) { ++ mutex_lock(&js_devdata->runpool_mutex); ++ kbase_backend_ctx_count_changed(kbdev); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ } ++#endif ++ } ++ } ++ mutex_unlock(&js_devdata->queue_mutex); ++ ++ /* Restart atom processing */ ++ kbase_js_sched_all(kbdev); ++ ++ /* JS Resume complete */ ++} ++ ++bool kbase_js_is_atom_valid(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom) ++{ ++ if ((katom->core_req & BASE_JD_REQ_FS) && ++ (katom->core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE | ++ BASE_JD_REQ_T))) ++ return false; ++ ++ if ((katom->core_req & BASE_JD_REQ_JOB_SLOT) && ++ (katom->jobslot >= BASE_JM_MAX_NR_SLOTS)) ++ return false; ++ ++ return true; ++} ++ ++static int kbase_js_get_slot(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom) ++{ ++ if (katom->core_req & BASE_JD_REQ_JOB_SLOT) ++ return katom->jobslot; ++ ++ if (katom->core_req & BASE_JD_REQ_FS) ++ return 0; ++ ++ if (katom->core_req & BASE_JD_REQ_ONLY_COMPUTE) { ++ if (katom->device_nr == 1 && ++ kbdev->gpu_props.num_core_groups == 2) ++ return 2; ++ } ++ ++ return 1; ++} ++ ++bool kbase_js_dep_resolved_submit(struct kbase_context *kctx, ++ struct kbase_jd_atom *katom) ++{ ++ bool enqueue_required, add_required = true; ++ ++ katom->slot_nr = kbase_js_get_slot(kctx->kbdev, katom); ++ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ lockdep_assert_held(&kctx->jctx.lock); ++ ++ /* If slot will transition from unpullable to pullable then add to ++ * pullable list */ ++ if (jsctx_rb_none_to_pull(kctx, katom->slot_nr)) { ++ enqueue_required = true; ++ } else { ++ enqueue_required = false; ++ } ++ ++ if ((katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED) || ++ (katom->pre_dep && (katom->pre_dep->atom_flags & ++ KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST))) { ++ int prio = katom->sched_priority; ++ int js = katom->slot_nr; ++ struct jsctx_queue *queue = &kctx->jsctx_queue[prio][js]; ++ ++ dev_dbg(kctx->kbdev->dev, "Add atom %p to X_DEP list (s:%d)\n", ++ (void *)katom, js); ++ ++ list_add_tail(&katom->queue, &queue->x_dep_head); ++ katom->atom_flags |= KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST; ++ if (kbase_js_atom_blocked_on_x_dep(katom)) { ++ enqueue_required = false; ++ add_required = false; ++ } ++ } else { ++ dev_dbg(kctx->kbdev->dev, "Atom %p not added to X_DEP list\n", ++ (void *)katom); ++ } ++ ++ if (add_required) { ++ /* Check if there are lower priority jobs to soft stop */ ++ kbase_job_slot_ctx_priority_check_locked(kctx, katom); ++ ++ /* Add atom to ring buffer. */ ++ jsctx_tree_add(kctx, katom); ++ katom->atom_flags |= KBASE_KATOM_FLAG_JSCTX_IN_TREE; ++ } ++ ++ dev_dbg(kctx->kbdev->dev, ++ "Enqueue of kctx %p is %srequired to submit atom %p\n", ++ kctx, enqueue_required ? "" : "not ", katom); ++ ++ return enqueue_required; ++} ++ ++/** ++ * kbase_js_move_to_tree - Move atom (and any dependent atoms) to the ++ * runnable_tree, ready for execution ++ * @katom: Atom to submit ++ * ++ * It is assumed that @katom does not have KBASE_KATOM_FLAG_X_DEP_BLOCKED set, ++ * but is still present in the x_dep list. If @katom has a same-slot dependent ++ * atom then that atom (and any dependents) will also be moved. ++ */ ++static void kbase_js_move_to_tree(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *const kctx = katom->kctx; ++ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ ++ while (katom) { ++ WARN_ON(!(katom->atom_flags & ++ KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST)); ++ ++ if (!kbase_js_atom_blocked_on_x_dep(katom)) { ++ dev_dbg(kctx->kbdev->dev, ++ "Del atom %p from X_DEP list in js_move_to_tree\n", ++ (void *)katom); ++ ++ list_del(&katom->queue); ++ katom->atom_flags &= ++ ~KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST; ++ /* For incremental rendering, an end-of-renderpass atom ++ * may have had its dependency on start-of-renderpass ++ * ignored and may therefore already be in the tree. ++ */ ++ if (!(katom->atom_flags & ++ KBASE_KATOM_FLAG_JSCTX_IN_TREE)) { ++ jsctx_tree_add(kctx, katom); ++ katom->atom_flags |= ++ KBASE_KATOM_FLAG_JSCTX_IN_TREE; ++ } ++ } else { ++ dev_dbg(kctx->kbdev->dev, ++ "Atom %p blocked on x-dep in js_move_to_tree\n", ++ (void *)katom); ++ break; ++ } ++ ++ katom = katom->post_dep; ++ } ++} ++ ++ ++/** ++ * kbase_js_evict_deps - Evict dependencies of a failed atom. ++ * @kctx: Context pointer ++ * @katom: Pointer to the atom that has failed. ++ * @js: The job slot the katom was run on. ++ * @prio: Priority of the katom. ++ * ++ * Remove all post dependencies of an atom from the context ringbuffers. ++ * ++ * The original atom's event_code will be propogated to all dependent atoms. ++ * ++ * Context: Caller must hold the HW access lock ++ */ ++static void kbase_js_evict_deps(struct kbase_context *kctx, ++ struct kbase_jd_atom *katom, int js, int prio) ++{ ++ struct kbase_jd_atom *x_dep = katom->x_post_dep; ++ struct kbase_jd_atom *next_katom = katom->post_dep; ++ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ ++ if (next_katom) { ++ KBASE_DEBUG_ASSERT(next_katom->status != ++ KBASE_JD_ATOM_STATE_HW_COMPLETED); ++ next_katom->will_fail_event_code = katom->event_code; ++ ++ } ++ ++ /* Has cross slot depenency. */ ++ if (x_dep && (x_dep->atom_flags & (KBASE_KATOM_FLAG_JSCTX_IN_TREE | ++ KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST))) { ++ /* Remove dependency.*/ ++ x_dep->atom_flags &= ~KBASE_KATOM_FLAG_X_DEP_BLOCKED; ++ trace_sysgraph(SGR_DEP_RES, kctx->id, ++ kbase_jd_atom_id(kctx, x_dep)); ++ ++ dev_dbg(kctx->kbdev->dev, "Cleared X_DEP flag on atom %p\n", ++ (void *)x_dep); ++ ++ /* Fail if it had a data dependency. */ ++ if (x_dep->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER) { ++ x_dep->will_fail_event_code = katom->event_code; ++ } ++ if (x_dep->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST) ++ kbase_js_move_to_tree(x_dep); ++ } ++} ++ ++struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, int js) ++{ ++ struct kbase_jd_atom *katom; ++ struct kbasep_js_device_data *js_devdata; ++ struct kbase_device *kbdev; ++ int pulled; ++ ++ KBASE_DEBUG_ASSERT(kctx); ++ ++ kbdev = kctx->kbdev; ++ dev_dbg(kbdev->dev, "JS: pulling an atom from kctx %p (s:%d)\n", ++ (void *)kctx, js); ++ ++ js_devdata = &kbdev->js_data; ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (!kbasep_js_is_submit_allowed(js_devdata, kctx)) { ++ dev_dbg(kbdev->dev, "JS: No submit allowed for kctx %p\n", ++ (void *)kctx); ++ return NULL; ++ } ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++ if (kbase_pm_is_suspending(kbdev) || kbase_pm_is_gpu_lost(kbdev)) ++#else ++ if (kbase_pm_is_suspending(kbdev)) ++#endif ++ return NULL; ++ ++ katom = jsctx_rb_peek(kctx, js); ++ if (!katom) { ++ dev_dbg(kbdev->dev, "JS: No pullable atom in kctx %p (s:%d)\n", ++ (void *)kctx, js); ++ return NULL; ++ } ++ if (kctx->blocked_js[js][katom->sched_priority]) { ++ dev_dbg(kbdev->dev, ++ "JS: kctx %p is blocked from submitting atoms at priority %d (s:%d)\n", ++ (void *)kctx, katom->sched_priority, js); ++ return NULL; ++ } ++ if (atomic_read(&katom->blocked)) { ++ dev_dbg(kbdev->dev, "JS: Atom %p is blocked in js_pull\n", ++ (void *)katom); ++ return NULL; ++ } ++ ++ /* Due to ordering restrictions when unpulling atoms on failure, we do ++ * not allow multiple runs of fail-dep atoms from the same context to be ++ * present on the same slot */ ++ if (katom->pre_dep && atomic_read(&kctx->atoms_pulled_slot[js])) { ++ struct kbase_jd_atom *prev_atom = ++ kbase_backend_inspect_tail(kbdev, js); ++ ++ if (prev_atom && prev_atom->kctx != kctx) ++ return NULL; ++ } ++ ++ if (kbase_js_atom_blocked_on_x_dep(katom)) { ++ if (katom->x_pre_dep->gpu_rb_state == ++ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB || ++ katom->x_pre_dep->will_fail_event_code) { ++ dev_dbg(kbdev->dev, ++ "JS: X pre-dep %p is not present in slot FIFO or will fail\n", ++ (void *)katom->x_pre_dep); ++ return NULL; ++ } ++ if ((katom->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER) && ++ kbase_backend_nr_atoms_on_slot(kbdev, js)) { ++ dev_dbg(kbdev->dev, ++ "JS: Atom %p has cross-slot fail dependency and atoms on slot (s:%d)\n", ++ (void *)katom, js); ++ return NULL; ++ } ++ } ++ ++ kbase_ctx_flag_set(kctx, KCTX_PULLED); ++ kbase_ctx_flag_set(kctx, (KCTX_PULLED_SINCE_ACTIVE_JS0 << js)); ++ ++ pulled = atomic_inc_return(&kctx->atoms_pulled); ++ if (pulled == 1 && !kctx->slots_pullable) { ++ WARN_ON(kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); ++ kbase_ctx_flag_set(kctx, KCTX_RUNNABLE_REF); ++ atomic_inc(&kbdev->js_data.nr_contexts_runnable); ++ } ++ atomic_inc(&kctx->atoms_pulled_slot[katom->slot_nr]); ++ kctx->atoms_pulled_slot_pri[katom->slot_nr][katom->sched_priority]++; ++ jsctx_rb_pull(kctx, katom); ++ ++ kbase_ctx_sched_retain_ctx_refcount(kctx); ++ ++ katom->atom_flags |= KBASE_KATOM_FLAG_HOLDING_CTX_REF; ++ ++ katom->ticks = 0; ++ ++ dev_dbg(kbdev->dev, "JS: successfully pulled atom %p from kctx %p (s:%d)\n", ++ (void *)katom, (void *)kctx, js); ++ ++ return katom; ++} ++ ++/** ++ * js_return_of_start_rp() - Handle soft-stop of an atom that starts a ++ * renderpass ++ * @start_katom: Pointer to the start-of-renderpass atom that was soft-stopped ++ * ++ * This function is called to switch to incremental rendering if the tiler job ++ * chain at the start of a renderpass has used too much memory. It prevents the ++ * tiler job being pulled for execution in the job scheduler again until the ++ * next phase of incremental rendering is complete. ++ * ++ * If the end-of-renderpass atom is already in the job scheduler (because a ++ * previous attempt at tiling used too much memory during the same renderpass) ++ * then it is unblocked; otherwise, it is run by handing it to the scheduler. ++ */ ++static void js_return_of_start_rp(struct kbase_jd_atom *const start_katom) ++{ ++ struct kbase_context *const kctx = start_katom->kctx; ++ struct kbase_device *const kbdev = kctx->kbdev; ++ struct kbase_jd_renderpass *rp; ++ struct kbase_jd_atom *end_katom; ++ unsigned long flags; ++ ++ lockdep_assert_held(&kctx->jctx.lock); ++ ++ if (WARN_ON(!(start_katom->core_req & BASE_JD_REQ_START_RENDERPASS))) ++ return; ++ ++ compiletime_assert((1ull << (sizeof(start_katom->renderpass_id) * 8)) <= ++ ARRAY_SIZE(kctx->jctx.renderpasses), ++ "Should check invalid access to renderpasses"); ++ ++ rp = &kctx->jctx.renderpasses[start_katom->renderpass_id]; ++ ++ if (WARN_ON(rp->start_katom != start_katom)) ++ return; ++ ++ dev_dbg(kctx->kbdev->dev, ++ "JS return start atom %p in state %d of RP %d\n", ++ (void *)start_katom, (int)rp->state, ++ start_katom->renderpass_id); ++ ++ if (WARN_ON(rp->state == KBASE_JD_RP_COMPLETE)) ++ return; ++ ++ /* The tiler job might have been soft-stopped for some reason other ++ * than running out of memory. ++ */ ++ if (rp->state == KBASE_JD_RP_START || rp->state == KBASE_JD_RP_RETRY) { ++ dev_dbg(kctx->kbdev->dev, ++ "JS return isn't OOM in state %d of RP %d\n", ++ (int)rp->state, start_katom->renderpass_id); ++ return; ++ } ++ ++ dev_dbg(kctx->kbdev->dev, ++ "JS return confirm OOM in state %d of RP %d\n", ++ (int)rp->state, start_katom->renderpass_id); ++ ++ if (WARN_ON(rp->state != KBASE_JD_RP_PEND_OOM && ++ rp->state != KBASE_JD_RP_RETRY_PEND_OOM)) ++ return; ++ ++ /* Prevent the tiler job being pulled for execution in the ++ * job scheduler again. ++ */ ++ dev_dbg(kbdev->dev, "Blocking start atom %p\n", ++ (void *)start_katom); ++ atomic_inc(&start_katom->blocked); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ rp->state = (rp->state == KBASE_JD_RP_PEND_OOM) ? ++ KBASE_JD_RP_OOM : KBASE_JD_RP_RETRY_OOM; ++ ++ /* Was the fragment job chain submitted to kbase yet? */ ++ end_katom = rp->end_katom; ++ if (end_katom) { ++ dev_dbg(kctx->kbdev->dev, "JS return add end atom %p\n", ++ (void *)end_katom); ++ ++ if (rp->state == KBASE_JD_RP_RETRY_OOM) { ++ /* Allow the end of the renderpass to be pulled for ++ * execution again to continue incremental rendering. ++ */ ++ dev_dbg(kbdev->dev, "Unblocking end atom %p\n", ++ (void *)end_katom); ++ atomic_dec(&end_katom->blocked); ++ WARN_ON(!(end_katom->atom_flags & ++ KBASE_KATOM_FLAG_JSCTX_IN_TREE)); ++ WARN_ON(end_katom->status != KBASE_JD_ATOM_STATE_IN_JS); ++ ++ kbase_js_ctx_list_add_pullable_nolock(kbdev, kctx, ++ end_katom->slot_nr); ++ ++ /* Expect the fragment job chain to be scheduled without ++ * further action because this function is called when ++ * returning an atom to the job scheduler ringbuffer. ++ */ ++ end_katom = NULL; ++ } else { ++ WARN_ON(end_katom->status != ++ KBASE_JD_ATOM_STATE_QUEUED && ++ end_katom->status != KBASE_JD_ATOM_STATE_IN_JS); ++ } ++ } ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ if (end_katom) ++ kbase_jd_dep_clear_locked(end_katom); ++} ++ ++/** ++ * js_return_of_end_rp() - Handle completion of an atom that ends a renderpass ++ * @end_katom: Pointer to the end-of-renderpass atom that was completed ++ * ++ * This function is called to continue incremental rendering if the tiler job ++ * chain at the start of a renderpass used too much memory. It resets the ++ * mechanism for detecting excessive memory usage then allows the soft-stopped ++ * tiler job chain to be pulled for execution again. ++ * ++ * The start-of-renderpass atom must already been submitted to kbase. ++ */ ++static void js_return_of_end_rp(struct kbase_jd_atom *const end_katom) ++{ ++ struct kbase_context *const kctx = end_katom->kctx; ++ struct kbase_device *const kbdev = kctx->kbdev; ++ struct kbase_jd_renderpass *rp; ++ struct kbase_jd_atom *start_katom; ++ unsigned long flags; ++ ++ lockdep_assert_held(&kctx->jctx.lock); ++ ++ if (WARN_ON(!(end_katom->core_req & BASE_JD_REQ_END_RENDERPASS))) ++ return; ++ ++ compiletime_assert((1ull << (sizeof(end_katom->renderpass_id) * 8)) <= ++ ARRAY_SIZE(kctx->jctx.renderpasses), ++ "Should check invalid access to renderpasses"); ++ ++ rp = &kctx->jctx.renderpasses[end_katom->renderpass_id]; ++ ++ if (WARN_ON(rp->end_katom != end_katom)) ++ return; ++ ++ dev_dbg(kctx->kbdev->dev, ++ "JS return end atom %p in state %d of RP %d\n", ++ (void *)end_katom, (int)rp->state, end_katom->renderpass_id); ++ ++ if (WARN_ON(rp->state != KBASE_JD_RP_OOM && ++ rp->state != KBASE_JD_RP_RETRY_OOM)) ++ return; ++ ++ /* Reduce the number of mapped pages in the memory regions that ++ * triggered out-of-memory last time so that we can detect excessive ++ * memory usage again. ++ */ ++ kbase_gpu_vm_lock(kctx); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ while (!list_empty(&rp->oom_reg_list)) { ++ struct kbase_va_region *reg = ++ list_first_entry(&rp->oom_reg_list, ++ struct kbase_va_region, link); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ dev_dbg(kbdev->dev, ++ "Reset backing to %zu pages for region %p\n", ++ reg->threshold_pages, (void *)reg); ++ ++ if (!WARN_ON(reg->flags & KBASE_REG_VA_FREED)) ++ kbase_mem_shrink(kctx, reg, reg->threshold_pages); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ dev_dbg(kbdev->dev, "Deleting region %p from list\n", ++ (void *)reg); ++ list_del_init(®->link); ++ kbase_va_region_alloc_put(kctx, reg); ++ } ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ kbase_gpu_vm_unlock(kctx); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ rp->state = KBASE_JD_RP_RETRY; ++ dev_dbg(kbdev->dev, "Changed state to %d for retry\n", rp->state); ++ ++ /* Allow the start of the renderpass to be pulled for execution again ++ * to begin/continue incremental rendering. ++ */ ++ start_katom = rp->start_katom; ++ if (!WARN_ON(!start_katom)) { ++ dev_dbg(kbdev->dev, "Unblocking start atom %p\n", ++ (void *)start_katom); ++ atomic_dec(&start_katom->blocked); ++ (void)kbase_js_ctx_list_add_pullable_head_nolock(kbdev, kctx, ++ start_katom->slot_nr); ++ } ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++} ++ ++static void js_return_worker(struct work_struct *data) ++{ ++ struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, ++ work); ++ struct kbase_context *kctx = katom->kctx; ++ struct kbase_device *kbdev = kctx->kbdev; ++ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; ++ struct kbasep_js_kctx_info *js_kctx_info = &kctx->jctx.sched_info; ++ struct kbasep_js_atom_retained_state retained_state; ++ int js = katom->slot_nr; ++ int prio = katom->sched_priority; ++ bool timer_sync = false; ++ bool context_idle = false; ++ unsigned long flags; ++ base_jd_core_req core_req = katom->core_req; ++ ++ dev_dbg(kbdev->dev, "%s for atom %p with event code 0x%x\n", ++ __func__, (void *)katom, katom->event_code); ++ ++ if (katom->event_code != BASE_JD_EVENT_END_RP_DONE) ++ KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_EX(kbdev, katom); ++ ++ kbase_backend_complete_wq(kbdev, katom); ++ ++ kbasep_js_atom_retained_state_copy(&retained_state, katom); ++ ++ mutex_lock(&js_devdata->queue_mutex); ++ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); ++ ++ atomic_dec(&kctx->atoms_pulled); ++ atomic_dec(&kctx->atoms_pulled_slot[js]); ++ ++ if (katom->event_code != BASE_JD_EVENT_END_RP_DONE) ++ atomic_dec(&katom->blocked); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ kctx->atoms_pulled_slot_pri[js][katom->sched_priority]--; ++ ++ if (!atomic_read(&kctx->atoms_pulled_slot[js]) && ++ jsctx_rb_none_to_pull(kctx, js)) ++ timer_sync |= kbase_js_ctx_list_remove_nolock(kbdev, kctx, js); ++ ++ /* If this slot has been blocked due to soft-stopped atoms, and all ++ * atoms have now been processed, then unblock the slot */ ++ if (!kctx->atoms_pulled_slot_pri[js][prio] && ++ kctx->blocked_js[js][prio]) { ++ kctx->blocked_js[js][prio] = false; ++ ++ /* Only mark the slot as pullable if the context is not idle - ++ * that case is handled below */ ++ if (atomic_read(&kctx->atoms_pulled) && ++ kbase_js_ctx_pullable(kctx, js, true)) ++ timer_sync |= kbase_js_ctx_list_add_pullable_nolock( ++ kbdev, kctx, js); ++ } ++ ++ if (!atomic_read(&kctx->atoms_pulled)) { ++ dev_dbg(kbdev->dev, ++ "No atoms currently pulled from context %p\n", ++ (void *)kctx); ++ ++ if (!kctx->slots_pullable) { ++ dev_dbg(kbdev->dev, ++ "Context %p %s counted as runnable\n", ++ (void *)kctx, ++ kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF) ? ++ "is" : "isn't"); ++ ++ WARN_ON(!kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); ++ kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); ++ atomic_dec(&kbdev->js_data.nr_contexts_runnable); ++ timer_sync = true; ++ } ++ ++ if (kctx->as_nr != KBASEP_AS_NR_INVALID && ++ !kbase_ctx_flag(kctx, KCTX_DYING)) { ++ int num_slots = kbdev->gpu_props.num_job_slots; ++ int slot; ++ ++ if (!kbasep_js_is_submit_allowed(js_devdata, kctx)) ++ kbasep_js_set_submit_allowed(js_devdata, kctx); ++ ++ for (slot = 0; slot < num_slots; slot++) { ++ if (kbase_js_ctx_pullable(kctx, slot, true)) ++ timer_sync |= ++ kbase_js_ctx_list_add_pullable_nolock( ++ kbdev, kctx, slot); ++ } ++ } ++ ++ kbase_jm_idle_ctx(kbdev, kctx); ++ ++ context_idle = true; ++ } ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ if (context_idle) { ++ dev_dbg(kbdev->dev, ++ "Context %p %s counted as active\n", ++ (void *)kctx, ++ kbase_ctx_flag(kctx, KCTX_ACTIVE) ? ++ "is" : "isn't"); ++ WARN_ON(!kbase_ctx_flag(kctx, KCTX_ACTIVE)); ++ kbase_ctx_flag_clear(kctx, KCTX_ACTIVE); ++ kbase_pm_context_idle(kbdev); ++ } ++ ++ if (timer_sync) ++ kbase_js_sync_timers(kbdev); ++ ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ mutex_unlock(&js_devdata->queue_mutex); ++ ++ if (katom->core_req & BASE_JD_REQ_START_RENDERPASS) { ++ mutex_lock(&kctx->jctx.lock); ++ js_return_of_start_rp(katom); ++ mutex_unlock(&kctx->jctx.lock); ++ } else if (katom->event_code == BASE_JD_EVENT_END_RP_DONE) { ++ mutex_lock(&kctx->jctx.lock); ++ js_return_of_end_rp(katom); ++ mutex_unlock(&kctx->jctx.lock); ++ } ++ ++ katom->atom_flags &= ~KBASE_KATOM_FLAG_HOLDING_CTX_REF; ++ dev_dbg(kbdev->dev, "JS: retained state %s finished", ++ kbasep_js_has_atom_finished(&retained_state) ? ++ "has" : "hasn't"); ++ ++ WARN_ON(kbasep_js_has_atom_finished(&retained_state)); ++ ++ kbasep_js_runpool_release_ctx_and_katom_retained_state(kbdev, kctx, ++ &retained_state); ++ ++ kbase_js_sched_all(kbdev); ++ ++ kbase_backend_complete_wq_post_sched(kbdev, core_req); ++ ++ dev_dbg(kbdev->dev, "Leaving %s for atom %p\n", ++ __func__, (void *)katom); ++} ++ ++void kbase_js_unpull(struct kbase_context *kctx, struct kbase_jd_atom *katom) ++{ ++ dev_dbg(kctx->kbdev->dev, "Unpulling atom %p in kctx %p\n", ++ (void *)katom, (void *)kctx); ++ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ ++ jsctx_rb_unpull(kctx, katom); ++ ++ WARN_ON(work_pending(&katom->work)); ++ ++ /* Block re-submission until workqueue has run */ ++ atomic_inc(&katom->blocked); ++ ++ kbase_job_check_leave_disjoint(kctx->kbdev, katom); ++ ++ INIT_WORK(&katom->work, js_return_worker); ++ queue_work(kctx->jctx.job_done_wq, &katom->work); ++} ++ ++/** ++ * js_complete_start_rp() - Handle completion of atom that starts a renderpass ++ * @kctx: Context pointer ++ * @start_katom: Pointer to the atom that completed ++ * ++ * Put any references to virtual memory regions that might have been added by ++ * kbase_job_slot_softstop_start_rp() because the tiler job chain completed ++ * despite any pending soft-stop request. ++ * ++ * If the atom that just completed was soft-stopped during a previous attempt to ++ * run it then there should be a blocked end-of-renderpass atom waiting for it, ++ * which we must unblock to process the output of the tiler job chain. ++ * ++ * Return: true if caller should call kbase_backend_ctx_count_changed() ++ */ ++static bool js_complete_start_rp(struct kbase_context *kctx, ++ struct kbase_jd_atom *const start_katom) ++{ ++ struct kbase_device *const kbdev = kctx->kbdev; ++ struct kbase_jd_renderpass *rp; ++ bool timer_sync = false; ++ ++ lockdep_assert_held(&kctx->jctx.lock); ++ ++ if (WARN_ON(!(start_katom->core_req & BASE_JD_REQ_START_RENDERPASS))) ++ return false; ++ ++ compiletime_assert((1ull << (sizeof(start_katom->renderpass_id) * 8)) <= ++ ARRAY_SIZE(kctx->jctx.renderpasses), ++ "Should check invalid access to renderpasses"); ++ ++ rp = &kctx->jctx.renderpasses[start_katom->renderpass_id]; ++ ++ if (WARN_ON(rp->start_katom != start_katom)) ++ return false; ++ ++ dev_dbg(kctx->kbdev->dev, ++ "Start atom %p is done in state %d of RP %d\n", ++ (void *)start_katom, (int)rp->state, ++ start_katom->renderpass_id); ++ ++ if (WARN_ON(rp->state == KBASE_JD_RP_COMPLETE)) ++ return false; ++ ++ if (rp->state == KBASE_JD_RP_PEND_OOM || ++ rp->state == KBASE_JD_RP_RETRY_PEND_OOM) { ++ unsigned long flags; ++ ++ dev_dbg(kctx->kbdev->dev, ++ "Start atom %p completed before soft-stop\n", ++ (void *)start_katom); ++ ++ kbase_gpu_vm_lock(kctx); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ while (!list_empty(&rp->oom_reg_list)) { ++ struct kbase_va_region *reg = ++ list_first_entry(&rp->oom_reg_list, ++ struct kbase_va_region, link); ++ ++ WARN_ON(reg->flags & KBASE_REG_VA_FREED); ++ dev_dbg(kctx->kbdev->dev, "Deleting region %p from list\n", ++ (void *)reg); ++ list_del_init(®->link); ++ kbase_va_region_alloc_put(kctx, reg); ++ } ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ kbase_gpu_vm_unlock(kctx); ++ } else { ++ dev_dbg(kctx->kbdev->dev, ++ "Start atom %p did not exceed memory threshold\n", ++ (void *)start_katom); ++ ++ WARN_ON(rp->state != KBASE_JD_RP_START && ++ rp->state != KBASE_JD_RP_RETRY); ++ } ++ ++ if (rp->state == KBASE_JD_RP_RETRY || ++ rp->state == KBASE_JD_RP_RETRY_PEND_OOM) { ++ struct kbase_jd_atom *const end_katom = rp->end_katom; ++ ++ if (!WARN_ON(!end_katom)) { ++ unsigned long flags; ++ ++ /* Allow the end of the renderpass to be pulled for ++ * execution again to continue incremental rendering. ++ */ ++ dev_dbg(kbdev->dev, "Unblocking end atom %p!\n", ++ (void *)end_katom); ++ atomic_dec(&end_katom->blocked); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ timer_sync = kbase_js_ctx_list_add_pullable_nolock( ++ kbdev, kctx, end_katom->slot_nr); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ } ++ } ++ ++ return timer_sync; ++} ++ ++/** ++ * js_complete_end_rp() - Handle final completion of atom that ends a renderpass ++ * @kctx: Context pointer ++ * @end_katom: Pointer to the atom that completed for the last time ++ * ++ * This function must only be called if the renderpass actually completed ++ * without the tiler job chain at the start using too much memory; otherwise ++ * completion of the end-of-renderpass atom is handled similarly to a soft-stop. ++ */ ++static void js_complete_end_rp(struct kbase_context *kctx, ++ struct kbase_jd_atom *const end_katom) ++{ ++ struct kbase_device *const kbdev = kctx->kbdev; ++ unsigned long flags; ++ struct kbase_jd_renderpass *rp; ++ ++ lockdep_assert_held(&kctx->jctx.lock); ++ ++ if (WARN_ON(!(end_katom->core_req & BASE_JD_REQ_END_RENDERPASS))) ++ return; ++ ++ compiletime_assert((1ull << (sizeof(end_katom->renderpass_id) * 8)) <= ++ ARRAY_SIZE(kctx->jctx.renderpasses), ++ "Should check invalid access to renderpasses"); ++ ++ rp = &kctx->jctx.renderpasses[end_katom->renderpass_id]; ++ ++ if (WARN_ON(rp->end_katom != end_katom)) ++ return; ++ ++ dev_dbg(kbdev->dev, "End atom %p is done in state %d of RP %d\n", ++ (void *)end_katom, (int)rp->state, end_katom->renderpass_id); ++ ++ if (WARN_ON(rp->state == KBASE_JD_RP_COMPLETE) || ++ WARN_ON(rp->state == KBASE_JD_RP_OOM) || ++ WARN_ON(rp->state == KBASE_JD_RP_RETRY_OOM)) ++ return; ++ ++ /* Rendering completed without running out of memory. ++ */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ WARN_ON(!list_empty(&rp->oom_reg_list)); ++ rp->state = KBASE_JD_RP_COMPLETE; ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ dev_dbg(kbdev->dev, "Renderpass %d is complete\n", ++ end_katom->renderpass_id); ++} ++ ++bool kbase_js_complete_atom_wq(struct kbase_context *kctx, ++ struct kbase_jd_atom *katom) ++{ ++ struct kbasep_js_kctx_info *js_kctx_info; ++ struct kbasep_js_device_data *js_devdata; ++ struct kbase_device *kbdev; ++ unsigned long flags; ++ bool timer_sync = false; ++ int atom_slot; ++ bool context_idle = false; ++ int prio = katom->sched_priority; ++ ++ kbdev = kctx->kbdev; ++ atom_slot = katom->slot_nr; ++ ++ dev_dbg(kbdev->dev, "%s for atom %p (s:%d)\n", ++ __func__, (void *)katom, atom_slot); ++ ++ /* Update the incremental rendering state machine. ++ */ ++ if (katom->core_req & BASE_JD_REQ_START_RENDERPASS) ++ timer_sync |= js_complete_start_rp(kctx, katom); ++ else if (katom->core_req & BASE_JD_REQ_END_RENDERPASS) ++ js_complete_end_rp(kctx, katom); ++ ++ js_kctx_info = &kctx->jctx.sched_info; ++ js_devdata = &kbdev->js_data; ++ ++ lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); ++ ++ mutex_lock(&js_devdata->runpool_mutex); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ if (katom->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_TREE) { ++ dev_dbg(kbdev->dev, "Atom %p is in runnable_tree\n", ++ (void *)katom); ++ ++ context_idle = !atomic_dec_return(&kctx->atoms_pulled); ++ atomic_dec(&kctx->atoms_pulled_slot[atom_slot]); ++ kctx->atoms_pulled_slot_pri[atom_slot][prio]--; ++ ++ if (!atomic_read(&kctx->atoms_pulled) && ++ !kctx->slots_pullable) { ++ WARN_ON(!kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); ++ kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); ++ atomic_dec(&kbdev->js_data.nr_contexts_runnable); ++ timer_sync = true; ++ } ++ ++ /* If this slot has been blocked due to soft-stopped atoms, and ++ * all atoms have now been processed, then unblock the slot */ ++ if (!kctx->atoms_pulled_slot_pri[atom_slot][prio] ++ && kctx->blocked_js[atom_slot][prio]) { ++ dev_dbg(kbdev->dev, ++ "kctx %p is no longer blocked from submitting on slot %d at priority %d\n", ++ (void *)kctx, atom_slot, prio); ++ ++ kctx->blocked_js[atom_slot][prio] = false; ++ if (kbase_js_ctx_pullable(kctx, atom_slot, true)) ++ timer_sync |= ++ kbase_js_ctx_list_add_pullable_nolock( ++ kbdev, kctx, atom_slot); ++ } ++ } ++ WARN_ON(!(katom->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_TREE)); ++ ++ if (!atomic_read(&kctx->atoms_pulled_slot[atom_slot]) && ++ jsctx_rb_none_to_pull(kctx, atom_slot)) { ++ if (!list_empty( ++ &kctx->jctx.sched_info.ctx.ctx_list_entry[atom_slot])) ++ timer_sync |= kbase_js_ctx_list_remove_nolock( ++ kctx->kbdev, kctx, atom_slot); ++ } ++ ++ /* ++ * If submission is disabled on this context (most likely due to an ++ * atom failure) and there are now no atoms left in the system then ++ * re-enable submission so that context can be scheduled again. ++ */ ++ if (!kbasep_js_is_submit_allowed(js_devdata, kctx) && ++ !atomic_read(&kctx->atoms_pulled) && ++ !kbase_ctx_flag(kctx, KCTX_DYING)) { ++ int js; ++ ++ kbasep_js_set_submit_allowed(js_devdata, kctx); ++ ++ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { ++ if (kbase_js_ctx_pullable(kctx, js, true)) ++ timer_sync |= ++ kbase_js_ctx_list_add_pullable_nolock( ++ kbdev, kctx, js); ++ } ++ } else if (katom->x_post_dep && ++ kbasep_js_is_submit_allowed(js_devdata, kctx)) { ++ int js; ++ ++ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { ++ if (kbase_js_ctx_pullable(kctx, js, true)) ++ timer_sync |= ++ kbase_js_ctx_list_add_pullable_nolock( ++ kbdev, kctx, js); ++ } ++ } ++ ++ /* Mark context as inactive. The pm reference will be dropped later in ++ * jd_done_worker(). ++ */ ++ if (context_idle) { ++ dev_dbg(kbdev->dev, "kctx %p is no longer active\n", ++ (void *)kctx); ++ kbase_ctx_flag_clear(kctx, KCTX_ACTIVE); ++ } ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ if (timer_sync) ++ kbase_backend_ctx_count_changed(kbdev); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ ++ dev_dbg(kbdev->dev, "Leaving %s\n", __func__); ++ return context_idle; ++} ++ ++/** ++ * js_end_rp_is_complete() - Check whether an atom that ends a renderpass has ++ * completed for the last time. ++ * ++ * @end_katom: Pointer to the atom that completed on the hardware. ++ * ++ * An atom that ends a renderpass may be run on the hardware several times ++ * before notifying userspace or allowing dependent atoms to be executed. ++ * ++ * This function is used to decide whether or not to allow end-of-renderpass ++ * atom completion. It only returns false if the atom at the start of the ++ * renderpass was soft-stopped because it used too much memory during the most ++ * recent attempt at tiling. ++ * ++ * Return: True if the atom completed for the last time. ++ */ ++static bool js_end_rp_is_complete(struct kbase_jd_atom *const end_katom) ++{ ++ struct kbase_context *const kctx = end_katom->kctx; ++ struct kbase_device *const kbdev = kctx->kbdev; ++ struct kbase_jd_renderpass *rp; ++ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ ++ if (WARN_ON(!(end_katom->core_req & BASE_JD_REQ_END_RENDERPASS))) ++ return true; ++ ++ compiletime_assert((1ull << (sizeof(end_katom->renderpass_id) * 8)) <= ++ ARRAY_SIZE(kctx->jctx.renderpasses), ++ "Should check invalid access to renderpasses"); ++ ++ rp = &kctx->jctx.renderpasses[end_katom->renderpass_id]; ++ ++ if (WARN_ON(rp->end_katom != end_katom)) ++ return true; ++ ++ dev_dbg(kbdev->dev, ++ "JS complete end atom %p in state %d of RP %d\n", ++ (void *)end_katom, (int)rp->state, ++ end_katom->renderpass_id); ++ ++ if (WARN_ON(rp->state == KBASE_JD_RP_COMPLETE)) ++ return true; ++ ++ /* Failure of end-of-renderpass atoms must not return to the ++ * start of the renderpass. ++ */ ++ if (end_katom->event_code != BASE_JD_EVENT_DONE) ++ return true; ++ ++ if (rp->state != KBASE_JD_RP_OOM && ++ rp->state != KBASE_JD_RP_RETRY_OOM) ++ return true; ++ ++ dev_dbg(kbdev->dev, "Suppressing end atom completion\n"); ++ return false; ++} ++ ++struct kbase_jd_atom *kbase_js_complete_atom(struct kbase_jd_atom *katom, ++ ktime_t *end_timestamp) ++{ ++ struct kbase_device *kbdev; ++ struct kbase_context *kctx = katom->kctx; ++ struct kbase_jd_atom *x_dep = katom->x_post_dep; ++ ++ kbdev = kctx->kbdev; ++ dev_dbg(kbdev->dev, "Atom %p complete in kctx %p (post-dep %p)\n", ++ (void *)katom, (void *)kctx, (void *)x_dep); ++ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ ++ if ((katom->core_req & BASE_JD_REQ_END_RENDERPASS) && ++ !js_end_rp_is_complete(katom)) { ++ katom->event_code = BASE_JD_EVENT_END_RP_DONE; ++ kbase_js_unpull(kctx, katom); ++ return NULL; ++ } ++ ++ if (katom->will_fail_event_code) ++ katom->event_code = katom->will_fail_event_code; ++ ++ katom->status = KBASE_JD_ATOM_STATE_HW_COMPLETED; ++ dev_dbg(kbdev->dev, "Atom %p status to HW completed\n", (void *)katom); ++ ++ if (katom->event_code != BASE_JD_EVENT_DONE) { ++ kbase_js_evict_deps(kctx, katom, katom->slot_nr, ++ katom->sched_priority); ++ } ++ ++ KBASE_TLSTREAM_AUX_EVENT_JOB_SLOT(kbdev, NULL, ++ katom->slot_nr, 0, TL_JS_EVENT_STOP); ++ ++ trace_sysgraph_gpu(SGR_COMPLETE, kctx->id, ++ kbase_jd_atom_id(katom->kctx, katom), katom->slot_nr); ++ ++ kbase_jd_done(katom, katom->slot_nr, end_timestamp, 0); ++ ++ /* Unblock cross dependency if present */ ++ if (x_dep && (katom->event_code == BASE_JD_EVENT_DONE || ++ !(x_dep->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER)) && ++ (x_dep->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST)) { ++ bool was_pullable = kbase_js_ctx_pullable(kctx, x_dep->slot_nr, ++ false); ++ x_dep->atom_flags &= ~KBASE_KATOM_FLAG_X_DEP_BLOCKED; ++ trace_sysgraph(SGR_DEP_RES, kctx->id, ++ kbase_jd_atom_id(katom->kctx, x_dep)); ++ dev_dbg(kbdev->dev, "Cleared X_DEP flag on atom %p\n", ++ (void *)x_dep); ++ ++ kbase_js_move_to_tree(x_dep); ++ ++ if (!was_pullable && kbase_js_ctx_pullable(kctx, x_dep->slot_nr, ++ false)) ++ kbase_js_ctx_list_add_pullable_nolock(kbdev, kctx, ++ x_dep->slot_nr); ++ ++ if (x_dep->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_TREE) { ++ dev_dbg(kbdev->dev, "Atom %p is in runnable tree\n", ++ (void *)x_dep); ++ return x_dep; ++ } ++ } else { ++ dev_dbg(kbdev->dev, ++ "No cross-slot dep to unblock for atom %p\n", ++ (void *)katom); ++ } ++ ++ return NULL; ++} ++ ++/** ++ * kbase_js_atom_blocked_on_x_dep - Decide whether to ignore a cross-slot ++ * dependency ++ * @katom: Pointer to an atom in the slot ringbuffer ++ * ++ * A cross-slot dependency is ignored if necessary to unblock incremental ++ * rendering. If the atom at the start of a renderpass used too much memory ++ * and was soft-stopped then the atom at the end of a renderpass is submitted ++ * to hardware regardless of its dependency on the start-of-renderpass atom. ++ * This can happen multiple times for the same pair of atoms. ++ * ++ * Return: true to block the atom or false to allow it to be submitted to ++ * hardware ++ */ ++bool kbase_js_atom_blocked_on_x_dep(struct kbase_jd_atom *const katom) ++{ ++ struct kbase_context *const kctx = katom->kctx; ++ struct kbase_device *kbdev = kctx->kbdev; ++ struct kbase_jd_renderpass *rp; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (!(katom->atom_flags & ++ KBASE_KATOM_FLAG_X_DEP_BLOCKED)) { ++ dev_dbg(kbdev->dev, "Atom %p is not blocked on a cross-slot dependency", ++ (void *)katom); ++ return false; ++ } ++ ++ if (!(katom->core_req & BASE_JD_REQ_END_RENDERPASS)) { ++ dev_dbg(kbdev->dev, "Atom %p is blocked on a cross-slot dependency", ++ (void *)katom); ++ return true; ++ } ++ ++ compiletime_assert((1ull << (sizeof(katom->renderpass_id) * 8)) <= ++ ARRAY_SIZE(kctx->jctx.renderpasses), ++ "Should check invalid access to renderpasses"); ++ ++ rp = &kctx->jctx.renderpasses[katom->renderpass_id]; ++ /* We can read a subset of renderpass state without holding ++ * higher-level locks (but not end_katom, for example). ++ */ ++ ++ WARN_ON(rp->state == KBASE_JD_RP_COMPLETE); ++ ++ dev_dbg(kbdev->dev, "End atom has cross-slot dep in state %d\n", ++ (int)rp->state); ++ ++ if (rp->state != KBASE_JD_RP_OOM && rp->state != KBASE_JD_RP_RETRY_OOM) ++ return true; ++ ++ /* Tiler ran out of memory so allow the fragment job chain to run ++ * if it only depends on the tiler job chain. ++ */ ++ if (katom->x_pre_dep != rp->start_katom) { ++ dev_dbg(kbdev->dev, "Dependency is on %p not start atom %p\n", ++ (void *)katom->x_pre_dep, (void *)rp->start_katom); ++ return true; ++ } ++ ++ dev_dbg(kbdev->dev, "Ignoring cross-slot dep on atom %p\n", ++ (void *)katom->x_pre_dep); ++ ++ return false; ++} ++ ++void kbase_js_sched(struct kbase_device *kbdev, int js_mask) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ struct kbase_context *last_active[BASE_JM_MAX_NR_SLOTS]; ++ bool timer_sync = false; ++ bool ctx_waiting[BASE_JM_MAX_NR_SLOTS]; ++ int js; ++ ++ dev_dbg(kbdev->dev, "%s kbdev %p mask 0x%x\n", ++ __func__, (void *)kbdev, (unsigned int)js_mask); ++ ++ js_devdata = &kbdev->js_data; ++ ++ down(&js_devdata->schedule_sem); ++ mutex_lock(&js_devdata->queue_mutex); ++ ++ for (js = 0; js < BASE_JM_MAX_NR_SLOTS; js++) { ++ last_active[js] = kbdev->hwaccess.active_kctx[js]; ++ ctx_waiting[js] = false; ++ } ++ ++ while (js_mask) { ++ js = ffs(js_mask) - 1; ++ ++ while (1) { ++ struct kbase_context *kctx; ++ unsigned long flags; ++ bool context_idle = false; ++ ++ kctx = kbase_js_ctx_list_pop_head(kbdev, js); ++ ++ if (!kctx) { ++ js_mask &= ~(1 << js); ++ dev_dbg(kbdev->dev, ++ "No kctx on pullable list (s:%d)\n", ++ js); ++ break; ++ } ++ ++ if (!kbase_ctx_flag(kctx, KCTX_ACTIVE)) { ++ context_idle = true; ++ ++ dev_dbg(kbdev->dev, ++ "kctx %p is not active (s:%d)\n", ++ (void *)kctx, js); ++ ++ if (kbase_pm_context_active_handle_suspend( ++ kbdev, ++ KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE)) { ++ dev_dbg(kbdev->dev, ++ "Suspend pending (s:%d)\n", js); ++ /* Suspend pending - return context to ++ * queue and stop scheduling */ ++ mutex_lock( ++ &kctx->jctx.sched_info.ctx.jsctx_mutex); ++ if (kbase_js_ctx_list_add_pullable_head( ++ kctx->kbdev, kctx, js)) ++ kbase_js_sync_timers(kbdev); ++ mutex_unlock( ++ &kctx->jctx.sched_info.ctx.jsctx_mutex); ++ mutex_unlock(&js_devdata->queue_mutex); ++ up(&js_devdata->schedule_sem); ++ return; ++ } ++ kbase_ctx_flag_set(kctx, KCTX_ACTIVE); ++ } ++ ++ if (!kbase_js_use_ctx(kbdev, kctx, js)) { ++ mutex_lock( ++ &kctx->jctx.sched_info.ctx.jsctx_mutex); ++ ++ dev_dbg(kbdev->dev, ++ "kctx %p cannot be used at this time\n", ++ kctx); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ if (kbase_js_ctx_pullable(kctx, js, false) ++ || kbase_ctx_flag(kctx, KCTX_PRIVILEGED)) ++ timer_sync |= ++ kbase_js_ctx_list_add_pullable_head_nolock( ++ kctx->kbdev, kctx, js); ++ else ++ timer_sync |= ++ kbase_js_ctx_list_add_unpullable_nolock( ++ kctx->kbdev, kctx, js); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, ++ flags); ++ mutex_unlock( ++ &kctx->jctx.sched_info.ctx.jsctx_mutex); ++ if (context_idle) { ++ WARN_ON(!kbase_ctx_flag(kctx, KCTX_ACTIVE)); ++ kbase_ctx_flag_clear(kctx, KCTX_ACTIVE); ++ kbase_pm_context_idle(kbdev); ++ } ++ ++ /* No more jobs can be submitted on this slot */ ++ js_mask &= ~(1 << js); ++ break; ++ } ++ mutex_lock(&kctx->jctx.sched_info.ctx.jsctx_mutex); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ kbase_ctx_flag_clear(kctx, KCTX_PULLED); ++ ++ if (!kbase_jm_kick(kbdev, 1 << js)) { ++ dev_dbg(kbdev->dev, ++ "No more jobs can be submitted (s:%d)\n", ++ js); ++ js_mask &= ~(1 << js); ++ } ++ if (!kbase_ctx_flag(kctx, KCTX_PULLED)) { ++ bool pullable; ++ ++ dev_dbg(kbdev->dev, ++ "No atoms pulled from kctx %p (s:%d)\n", ++ (void *)kctx, js); ++ ++ pullable = kbase_js_ctx_pullable(kctx, js, ++ true); ++ ++ /* Failed to pull jobs - push to head of list. ++ * Unless this context is already 'active', in ++ * which case it's effectively already scheduled ++ * so push it to the back of the list. */ ++ if (pullable && kctx == last_active[js] && ++ kbase_ctx_flag(kctx, ++ (KCTX_PULLED_SINCE_ACTIVE_JS0 << ++ js))) ++ timer_sync |= ++ kbase_js_ctx_list_add_pullable_nolock( ++ kctx->kbdev, ++ kctx, js); ++ else if (pullable) ++ timer_sync |= ++ kbase_js_ctx_list_add_pullable_head_nolock( ++ kctx->kbdev, ++ kctx, js); ++ else ++ timer_sync |= ++ kbase_js_ctx_list_add_unpullable_nolock( ++ kctx->kbdev, ++ kctx, js); ++ ++ /* If this context is not the active context, ++ * but the active context is pullable on this ++ * slot, then we need to remove the active ++ * marker to prevent it from submitting atoms in ++ * the IRQ handler, which would prevent this ++ * context from making progress. */ ++ if (last_active[js] && kctx != last_active[js] ++ && kbase_js_ctx_pullable( ++ last_active[js], js, true)) ++ ctx_waiting[js] = true; ++ ++ if (context_idle) { ++ kbase_jm_idle_ctx(kbdev, kctx); ++ spin_unlock_irqrestore( ++ &kbdev->hwaccess_lock, ++ flags); ++ WARN_ON(!kbase_ctx_flag(kctx, KCTX_ACTIVE)); ++ kbase_ctx_flag_clear(kctx, KCTX_ACTIVE); ++ kbase_pm_context_idle(kbdev); ++ } else { ++ spin_unlock_irqrestore( ++ &kbdev->hwaccess_lock, ++ flags); ++ } ++ mutex_unlock( ++ &kctx->jctx.sched_info.ctx.jsctx_mutex); ++ ++ js_mask &= ~(1 << js); ++ break; /* Could not run atoms on this slot */ ++ } ++ ++ dev_dbg(kbdev->dev, "Push kctx %p to back of list\n", ++ (void *)kctx); ++ if (kbase_js_ctx_pullable(kctx, js, true)) ++ timer_sync |= ++ kbase_js_ctx_list_add_pullable_nolock( ++ kctx->kbdev, kctx, js); ++ else ++ timer_sync |= ++ kbase_js_ctx_list_add_unpullable_nolock( ++ kctx->kbdev, kctx, js); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&kctx->jctx.sched_info.ctx.jsctx_mutex); ++ } ++ } ++ ++ if (timer_sync) ++ kbase_js_sync_timers(kbdev); ++ ++ for (js = 0; js < BASE_JM_MAX_NR_SLOTS; js++) { ++ if (kbdev->hwaccess.active_kctx[js] == last_active[js] && ++ ctx_waiting[js]) { ++ dev_dbg(kbdev->dev, "Marking kctx %p as inactive (s:%d)\n", ++ (void *)last_active[js], js); ++ kbdev->hwaccess.active_kctx[js] = NULL; ++ } ++ } ++ ++ mutex_unlock(&js_devdata->queue_mutex); ++ up(&js_devdata->schedule_sem); ++} ++ ++void kbase_js_zap_context(struct kbase_context *kctx) ++{ ++ struct kbase_device *kbdev = kctx->kbdev; ++ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; ++ struct kbasep_js_kctx_info *js_kctx_info = &kctx->jctx.sched_info; ++ ++ /* ++ * Critical assumption: No more submission is possible outside of the ++ * workqueue. This is because the OS *must* prevent U/K calls (IOCTLs) ++ * whilst the struct kbase_context is terminating. ++ */ ++ ++ /* First, atomically do the following: ++ * - mark the context as dying ++ * - try to evict it from the queue */ ++ mutex_lock(&kctx->jctx.lock); ++ mutex_lock(&js_devdata->queue_mutex); ++ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); ++ kbase_ctx_flag_set(kctx, KCTX_DYING); ++ ++ dev_dbg(kbdev->dev, "Zap: Try Evict Ctx %p", kctx); ++ ++ /* ++ * At this point we know: ++ * - If eviction succeeded, it was in the queue, but now no ++ * longer is ++ * - We must cancel the jobs here. No Power Manager active reference to ++ * release. ++ * - This happens asynchronously - kbase_jd_zap_context() will wait for ++ * those jobs to be killed. ++ * - If eviction failed, then it wasn't in the queue. It is one ++ * of the following: ++ * - a. it didn't have any jobs, and so is not in the Queue or ++ * the Run Pool (not scheduled) ++ * - Hence, no more work required to cancel jobs. No Power Manager ++ * active reference to release. ++ * - b. it was in the middle of a scheduling transaction (and thus must ++ * have at least 1 job). This can happen from a syscall or a ++ * kernel thread. We still hold the jsctx_mutex, and so the thread ++ * must be waiting inside kbasep_js_try_schedule_head_ctx(), ++ * before checking whether the runpool is full. That thread will ++ * continue after we drop the mutex, and will notice the context ++ * is dying. It will rollback the transaction, killing all jobs at ++ * the same time. kbase_jd_zap_context() will wait for those jobs ++ * to be killed. ++ * - Hence, no more work required to cancel jobs, or to release the ++ * Power Manager active reference. ++ * - c. it is scheduled, and may or may not be running jobs ++ * - We must cause it to leave the runpool by stopping it from ++ * submitting any more jobs. When it finally does leave, ++ * kbasep_js_runpool_requeue_or_kill_ctx() will kill all remaining jobs ++ * (because it is dying), release the Power Manager active reference, ++ * and will not requeue the context in the queue. ++ * kbase_jd_zap_context() will wait for those jobs to be killed. ++ * - Hence, work required just to make it leave the runpool. Cancelling ++ * jobs and releasing the Power manager active reference will be ++ * handled when it leaves the runpool. ++ */ ++ if (!kbase_ctx_flag(kctx, KCTX_SCHEDULED)) { ++ unsigned long flags; ++ int js; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { ++ if (!list_empty( ++ &kctx->jctx.sched_info.ctx.ctx_list_entry[js])) ++ list_del_init( ++ &kctx->jctx.sched_info.ctx.ctx_list_entry[js]); ++ } ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ /* The following events require us to kill off remaining jobs ++ * and update PM book-keeping: ++ * - we evicted it correctly (it must have jobs to be in the ++ * Queue) ++ * ++ * These events need no action, but take this path anyway: ++ * - Case a: it didn't have any jobs, and was never in the Queue ++ * - Case b: scheduling transaction will be partially rolled- ++ * back (this already cancels the jobs) ++ */ ++ ++ KBASE_KTRACE_ADD_JM(kbdev, JM_ZAP_NON_SCHEDULED, kctx, NULL, 0u, kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ ++ dev_dbg(kbdev->dev, "Zap: Ctx %p scheduled=0", kctx); ++ ++ /* Only cancel jobs when we evicted from the ++ * queue. No Power Manager active reference was held. ++ * ++ * Having is_dying set ensures that this kills, and ++ * doesn't requeue */ ++ kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, false); ++ ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ mutex_unlock(&js_devdata->queue_mutex); ++ mutex_unlock(&kctx->jctx.lock); ++ } else { ++ unsigned long flags; ++ bool was_retained; ++ ++ /* Case c: didn't evict, but it is scheduled - it's in the Run ++ * Pool */ ++ KBASE_KTRACE_ADD_JM(kbdev, JM_ZAP_SCHEDULED, kctx, NULL, 0u, kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ dev_dbg(kbdev->dev, "Zap: Ctx %p is in RunPool", kctx); ++ ++ /* Disable the ctx from submitting any more jobs */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ kbasep_js_clear_submit_allowed(js_devdata, kctx); ++ ++ /* Retain and (later) release the context whilst it is is now ++ * disallowed from submitting jobs - ensures that someone ++ * somewhere will be removing the context later on */ ++ was_retained = kbase_ctx_sched_inc_refcount_nolock(kctx); ++ ++ /* Since it's scheduled and we have the jsctx_mutex, it must be ++ * retained successfully */ ++ KBASE_DEBUG_ASSERT(was_retained); ++ ++ dev_dbg(kbdev->dev, "Zap: Ctx %p Kill Any Running jobs", kctx); ++ ++ /* Cancel any remaining running jobs for this kctx - if any. ++ * Submit is disallowed which takes effect immediately, so no ++ * more new jobs will appear after we do this. */ ++ kbase_backend_jm_kill_running_jobs_from_kctx(kctx); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ mutex_unlock(&js_devdata->queue_mutex); ++ mutex_unlock(&kctx->jctx.lock); ++ ++ dev_dbg(kbdev->dev, "Zap: Ctx %p Release (may or may not schedule out immediately)", ++ kctx); ++ ++ kbasep_js_runpool_release_ctx(kbdev, kctx); ++ } ++ ++ KBASE_KTRACE_ADD_JM(kbdev, JM_ZAP_DONE, kctx, NULL, 0u, 0u); ++ ++ /* After this, you must wait on both the ++ * kbase_jd_context::zero_jobs_wait and the ++ * kbasep_js_kctx_info::ctx::is_scheduled_waitq - to wait for the jobs ++ * to be destroyed, and the context to be de-scheduled (if it was on the ++ * runpool). ++ * ++ * kbase_jd_zap_context() will do this. */ ++} ++ ++static inline int trace_get_refcnt(struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++ return atomic_read(&kctx->refcount); ++} ++ ++/** ++ * kbase_js_foreach_ctx_job(): - Call a function on all jobs in context ++ * @kctx: Pointer to context. ++ * @callback: Pointer to function to call for each job. ++ * ++ * Call a function on all jobs belonging to a non-queued, non-running ++ * context, and detach the jobs from the context as it goes. ++ * ++ * Due to the locks that might be held at the time of the call, the callback ++ * may need to defer work on a workqueue to complete its actions (e.g. when ++ * cancelling jobs) ++ * ++ * Atoms will be removed from the queue, so this must only be called when ++ * cancelling jobs (which occurs as part of context destruction). ++ * ++ * The locking conditions on the caller are as follows: ++ * - it will be holding kbasep_js_kctx_info::ctx::jsctx_mutex. ++ */ ++static void kbase_js_foreach_ctx_job(struct kbase_context *kctx, ++ kbasep_js_ctx_job_cb callback) ++{ ++ struct kbase_device *kbdev; ++ unsigned long flags; ++ u32 js; ++ ++ kbdev = kctx->kbdev; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ KBASE_KTRACE_ADD_JM_REFCOUNT(kbdev, JS_POLICY_FOREACH_CTX_JOBS, kctx, NULL, ++ 0u, trace_get_refcnt(kbdev, kctx)); ++ ++ /* Invoke callback on jobs on each slot in turn */ ++ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) ++ jsctx_queue_foreach(kctx, js, callback); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++} +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_js.h b/drivers/gpu/arm/bifrost/mali_kbase_js.h +new file mode 100755 +index 000000000000..541acd4afed7 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_js.h +@@ -0,0 +1,40 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++/** ++ * @file mali_kbase_js.h ++ * Job Scheduler APIs. ++ */ ++ ++#ifndef _KBASE_JS_H_ ++#define _KBASE_JS_H_ ++ ++#include "context/mali_kbase_context.h" ++#include "mali_kbase_defs.h" ++#include "mali_kbase_debug.h" ++#include ++#include "jm/mali_kbase_jm_js.h" ++#include "jm/mali_kbase_js_defs.h" ++ ++#endif /* _KBASE_JS_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_js_ctx_attr.c b/drivers/gpu/arm/bifrost/mali_kbase_js_ctx_attr.c +new file mode 100755 +index 000000000000..141d04a385cb +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_js_ctx_attr.c +@@ -0,0 +1,283 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2016, 2018, 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++#include ++#include ++ ++/* ++ * Private functions follow ++ */ ++ ++/** ++ * @brief Check whether a ctx has a certain attribute, and if so, retain that ++ * attribute on the runpool. ++ * ++ * Requires: ++ * - jsctx mutex ++ * - runpool_irq spinlock ++ * - ctx is scheduled on the runpool ++ * ++ * @return true indicates a change in ctx attributes state of the runpool. ++ * In this state, the scheduler might be able to submit more jobs than ++ * previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock() ++ * or similar is called sometime later. ++ * @return false indicates no change in ctx attributes state of the runpool. ++ */ ++static bool kbasep_js_ctx_attr_runpool_retain_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ bool runpool_state_changed = false; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); ++ js_devdata = &kbdev->js_data; ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ ++ if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, attribute) != false) { ++ KBASE_DEBUG_ASSERT(js_devdata->runpool_irq.ctx_attr_ref_count[attribute] < S8_MAX); ++ ++(js_devdata->runpool_irq.ctx_attr_ref_count[attribute]); ++ ++ if (js_devdata->runpool_irq.ctx_attr_ref_count[attribute] == 1) { ++ /* First refcount indicates a state change */ ++ runpool_state_changed = true; ++ KBASE_KTRACE_ADD_JM(kbdev, JS_CTX_ATTR_NOW_ON_RUNPOOL, kctx, NULL, 0u, attribute); ++ } ++ } ++ ++ return runpool_state_changed; ++} ++ ++/** ++ * @brief Check whether a ctx has a certain attribute, and if so, release that ++ * attribute on the runpool. ++ * ++ * Requires: ++ * - jsctx mutex ++ * - runpool_irq spinlock ++ * - ctx is scheduled on the runpool ++ * ++ * @return true indicates a change in ctx attributes state of the runpool. ++ * In this state, the scheduler might be able to submit more jobs than ++ * previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock() ++ * or similar is called sometime later. ++ * @return false indicates no change in ctx attributes state of the runpool. ++ */ ++static bool kbasep_js_ctx_attr_runpool_release_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ bool runpool_state_changed = false; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); ++ js_devdata = &kbdev->js_data; ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ ++ if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, attribute) != false) { ++ KBASE_DEBUG_ASSERT(js_devdata->runpool_irq.ctx_attr_ref_count[attribute] > 0); ++ --(js_devdata->runpool_irq.ctx_attr_ref_count[attribute]); ++ ++ if (js_devdata->runpool_irq.ctx_attr_ref_count[attribute] == 0) { ++ /* Last de-refcount indicates a state change */ ++ runpool_state_changed = true; ++ KBASE_KTRACE_ADD_JM(kbdev, JS_CTX_ATTR_NOW_OFF_RUNPOOL, kctx, NULL, 0u, attribute); ++ } ++ } ++ ++ return runpool_state_changed; ++} ++ ++/** ++ * @brief Retain a certain attribute on a ctx, also retaining it on the runpool ++ * if the context is scheduled. ++ * ++ * Requires: ++ * - jsctx mutex ++ * - If the context is scheduled, then runpool_irq spinlock must also be held ++ * ++ * @return true indicates a change in ctx attributes state of the runpool. ++ * This may allow the scheduler to submit more jobs than previously. ++ * @return false indicates no change in ctx attributes state of the runpool. ++ */ ++static bool kbasep_js_ctx_attr_ctx_retain_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute) ++{ ++ struct kbasep_js_kctx_info *js_kctx_info; ++ bool runpool_state_changed = false; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); ++ KBASE_DEBUG_ASSERT(js_kctx_info->ctx.ctx_attr_ref_count[attribute] < U32_MAX); ++ ++ ++(js_kctx_info->ctx.ctx_attr_ref_count[attribute]); ++ ++ if (kbase_ctx_flag(kctx, KCTX_SCHEDULED) && js_kctx_info->ctx.ctx_attr_ref_count[attribute] == 1) { ++ /* Only ref-count the attribute on the runpool for the first time this contexts sees this attribute */ ++ KBASE_KTRACE_ADD_JM(kbdev, JS_CTX_ATTR_NOW_ON_CTX, kctx, NULL, 0u, attribute); ++ runpool_state_changed = kbasep_js_ctx_attr_runpool_retain_attr(kbdev, kctx, attribute); ++ } ++ ++ return runpool_state_changed; ++} ++ ++/* ++ * @brief Release a certain attribute on a ctx, also releasing it from the runpool ++ * if the context is scheduled. ++ * ++ * Requires: ++ * - jsctx mutex ++ * - If the context is scheduled, then runpool_irq spinlock must also be held ++ * ++ * @return true indicates a change in ctx attributes state of the runpool. ++ * This may allow the scheduler to submit more jobs than previously. ++ * @return false indicates no change in ctx attributes state of the runpool. ++ */ ++static bool kbasep_js_ctx_attr_ctx_release_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute) ++{ ++ struct kbasep_js_kctx_info *js_kctx_info; ++ bool runpool_state_changed = false; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); ++ KBASE_DEBUG_ASSERT(js_kctx_info->ctx.ctx_attr_ref_count[attribute] > 0); ++ ++ if (kbase_ctx_flag(kctx, KCTX_SCHEDULED) && js_kctx_info->ctx.ctx_attr_ref_count[attribute] == 1) { ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ /* Only de-ref-count the attribute on the runpool when this is the last ctx-reference to it */ ++ runpool_state_changed = kbasep_js_ctx_attr_runpool_release_attr(kbdev, kctx, attribute); ++ KBASE_KTRACE_ADD_JM(kbdev, JS_CTX_ATTR_NOW_OFF_CTX, kctx, NULL, 0u, attribute); ++ } ++ ++ /* De-ref must happen afterwards, because kbasep_js_ctx_attr_runpool_release() needs to check it too */ ++ --(js_kctx_info->ctx.ctx_attr_ref_count[attribute]); ++ ++ return runpool_state_changed; ++} ++ ++/* ++ * More commonly used public functions ++ */ ++ ++void kbasep_js_ctx_attr_runpool_retain_ctx(struct kbase_device *kbdev, struct kbase_context *kctx) ++{ ++ bool runpool_state_changed; ++ int i; ++ ++ /* Retain any existing attributes */ ++ for (i = 0; i < KBASEP_JS_CTX_ATTR_COUNT; ++i) { ++ if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, (enum kbasep_js_ctx_attr) i) != false) { ++ /* The context is being scheduled in, so update the runpool with the new attributes */ ++ runpool_state_changed = kbasep_js_ctx_attr_runpool_retain_attr(kbdev, kctx, (enum kbasep_js_ctx_attr) i); ++ ++ /* We don't need to know about state changed, because retaining a ++ * context occurs on scheduling it, and that itself will also try ++ * to run new atoms */ ++ CSTD_UNUSED(runpool_state_changed); ++ } ++ } ++} ++ ++bool kbasep_js_ctx_attr_runpool_release_ctx(struct kbase_device *kbdev, struct kbase_context *kctx) ++{ ++ bool runpool_state_changed = false; ++ int i; ++ ++ /* Release any existing attributes */ ++ for (i = 0; i < KBASEP_JS_CTX_ATTR_COUNT; ++i) { ++ if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, (enum kbasep_js_ctx_attr) i) != false) { ++ /* The context is being scheduled out, so update the runpool on the removed attributes */ ++ runpool_state_changed |= kbasep_js_ctx_attr_runpool_release_attr(kbdev, kctx, (enum kbasep_js_ctx_attr) i); ++ } ++ } ++ ++ return runpool_state_changed; ++} ++ ++void kbasep_js_ctx_attr_ctx_retain_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom) ++{ ++ bool runpool_state_changed = false; ++ base_jd_core_req core_req; ++ ++ KBASE_DEBUG_ASSERT(katom); ++ core_req = katom->core_req; ++ ++ if (core_req & BASE_JD_REQ_ONLY_COMPUTE) ++ runpool_state_changed |= kbasep_js_ctx_attr_ctx_retain_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE); ++ else ++ runpool_state_changed |= kbasep_js_ctx_attr_ctx_retain_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_NON_COMPUTE); ++ ++ if ((core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE | BASE_JD_REQ_T)) != 0 && (core_req & (BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_SPECIFIC_COHERENT_GROUP)) == 0) { ++ /* Atom that can run on slot1 or slot2, and can use all cores */ ++ runpool_state_changed |= kbasep_js_ctx_attr_ctx_retain_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES); ++ } ++ ++ /* We don't need to know about state changed, because retaining an ++ * atom occurs on adding it, and that itself will also try to run ++ * new atoms */ ++ CSTD_UNUSED(runpool_state_changed); ++} ++ ++bool kbasep_js_ctx_attr_ctx_release_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbasep_js_atom_retained_state *katom_retained_state) ++{ ++ bool runpool_state_changed = false; ++ base_jd_core_req core_req; ++ ++ KBASE_DEBUG_ASSERT(katom_retained_state); ++ core_req = katom_retained_state->core_req; ++ ++ /* No-op for invalid atoms */ ++ if (kbasep_js_atom_retained_state_is_valid(katom_retained_state) == false) ++ return false; ++ ++ if (core_req & BASE_JD_REQ_ONLY_COMPUTE) ++ runpool_state_changed |= kbasep_js_ctx_attr_ctx_release_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE); ++ else ++ runpool_state_changed |= kbasep_js_ctx_attr_ctx_release_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_NON_COMPUTE); ++ ++ if ((core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE | BASE_JD_REQ_T)) != 0 && (core_req & (BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_SPECIFIC_COHERENT_GROUP)) == 0) { ++ /* Atom that can run on slot1 or slot2, and can use all cores */ ++ runpool_state_changed |= kbasep_js_ctx_attr_ctx_release_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES); ++ } ++ ++ return runpool_state_changed; ++} +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_js_ctx_attr.h b/drivers/gpu/arm/bifrost/mali_kbase_js_ctx_attr.h +new file mode 100755 +index 000000000000..25fd39787c71 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_js_ctx_attr.h +@@ -0,0 +1,155 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2015, 2018 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++/** ++ * @file mali_kbase_js_ctx_attr.h ++ * Job Scheduler Context Attribute APIs ++ */ ++ ++#ifndef _KBASE_JS_CTX_ATTR_H_ ++#define _KBASE_JS_CTX_ATTR_H_ ++ ++/** ++ * @addtogroup base_api ++ * @{ ++ */ ++ ++/** ++ * @addtogroup base_kbase_api ++ * @{ ++ */ ++ ++/** ++ * @addtogroup kbase_js ++ * @{ ++ */ ++ ++/** ++ * Retain all attributes of a context ++ * ++ * This occurs on scheduling in the context on the runpool (but after ++ * is_scheduled is set) ++ * ++ * Requires: ++ * - jsctx mutex ++ * - runpool_irq spinlock ++ * - ctx->is_scheduled is true ++ */ ++void kbasep_js_ctx_attr_runpool_retain_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); ++ ++/** ++ * Release all attributes of a context ++ * ++ * This occurs on scheduling out the context from the runpool (but before ++ * is_scheduled is cleared) ++ * ++ * Requires: ++ * - jsctx mutex ++ * - runpool_irq spinlock ++ * - ctx->is_scheduled is true ++ * ++ * @return true indicates a change in ctx attributes state of the runpool. ++ * In this state, the scheduler might be able to submit more jobs than ++ * previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock() ++ * or similar is called sometime later. ++ * @return false indicates no change in ctx attributes state of the runpool. ++ */ ++bool kbasep_js_ctx_attr_runpool_release_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); ++ ++/** ++ * Retain all attributes of an atom ++ * ++ * This occurs on adding an atom to a context ++ * ++ * Requires: ++ * - jsctx mutex ++ * - If the context is scheduled, then runpool_irq spinlock must also be held ++ */ ++void kbasep_js_ctx_attr_ctx_retain_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom); ++ ++/** ++ * Release all attributes of an atom, given its retained state. ++ * ++ * This occurs after (permanently) removing an atom from a context ++ * ++ * Requires: ++ * - jsctx mutex ++ * - If the context is scheduled, then runpool_irq spinlock must also be held ++ * ++ * This is a no-op when \a katom_retained_state is invalid. ++ * ++ * @return true indicates a change in ctx attributes state of the runpool. ++ * In this state, the scheduler might be able to submit more jobs than ++ * previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock() ++ * or similar is called sometime later. ++ * @return false indicates no change in ctx attributes state of the runpool. ++ */ ++bool kbasep_js_ctx_attr_ctx_release_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbasep_js_atom_retained_state *katom_retained_state); ++ ++/** ++ * Requires: ++ * - runpool_irq spinlock ++ */ ++static inline s8 kbasep_js_ctx_attr_count_on_runpool(struct kbase_device *kbdev, enum kbasep_js_ctx_attr attribute) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); ++ js_devdata = &kbdev->js_data; ++ ++ return js_devdata->runpool_irq.ctx_attr_ref_count[attribute]; ++} ++ ++/** ++ * Requires: ++ * - runpool_irq spinlock ++ */ ++static inline bool kbasep_js_ctx_attr_is_attr_on_runpool(struct kbase_device *kbdev, enum kbasep_js_ctx_attr attribute) ++{ ++ /* In general, attributes are 'on' when they have a non-zero refcount (note: the refcount will never be < 0) */ ++ return (bool) kbasep_js_ctx_attr_count_on_runpool(kbdev, attribute); ++} ++ ++/** ++ * Requires: ++ * - jsctx mutex ++ */ ++static inline bool kbasep_js_ctx_attr_is_attr_on_ctx(struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute) ++{ ++ struct kbasep_js_kctx_info *js_kctx_info; ++ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ /* In general, attributes are 'on' when they have a refcount (which should never be < 0) */ ++ return (bool) (js_kctx_info->ctx.ctx_attr_ref_count[attribute]); ++} ++ ++ /** @} *//* end group kbase_js */ ++ /** @} *//* end group base_kbase_api */ ++ /** @} *//* end group base_api */ ++ ++#endif /* _KBASE_JS_DEFS_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_kinstr_jm.c b/drivers/gpu/arm/bifrost/mali_kbase_kinstr_jm.c +new file mode 100755 +index 000000000000..fd1ea8815b16 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_kinstr_jm.c +@@ -0,0 +1,895 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* ++ * mali_kbase_kinstr_jm.c ++ * Kernel driver public interface to job manager atom tracing ++ */ ++ ++#include "mali_kbase_kinstr_jm.h" ++#include "mali_kbase_kinstr_jm_reader.h" ++ ++#include "mali_kbase.h" ++#include "mali_kbase_linux.h" ++ ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#if KERNEL_VERSION(5, 1, 0) <= LINUX_VERSION_CODE ++#include ++#else ++// Stringify the expression if no message is given. ++#define static_assert(e, ...) __static_assert(e, #__VA_ARGS__, #e) ++#define __static_assert(e, msg, ...) _Static_assert(e, msg) ++#endif ++ ++#if KERNEL_VERSION(4, 16, 0) >= LINUX_VERSION_CODE ++typedef unsigned int __poll_t; ++#endif ++ ++#ifndef ENOTSUP ++#define ENOTSUP EOPNOTSUPP ++#endif ++ ++/* The module printing prefix */ ++#define PR_ "mali_kbase_kinstr_jm: " ++ ++/* Allows us to perform ASM goto for the tracing ++ * https://www.kernel.org/doc/Documentation/static-keys.txt ++ */ ++#if KERNEL_VERSION(4, 3, 0) <= LINUX_VERSION_CODE ++DEFINE_STATIC_KEY_FALSE(basep_kinstr_jm_reader_static_key); ++#else ++struct static_key basep_kinstr_jm_reader_static_key = STATIC_KEY_INIT_FALSE; ++#define static_branch_inc(key) static_key_slow_inc(key) ++#define static_branch_dec(key) static_key_slow_dec(key) ++#endif /* KERNEL_VERSION(4 ,3, 0) <= LINUX_VERSION_CODE */ ++ ++#define KBASE_KINSTR_JM_VERSION 1 ++ ++/** ++ * struct kbase_kinstr_jm - The context for the kernel job manager atom tracing ++ * @readers: a bitlocked list of opened readers. Readers are attached to the ++ * private data of a file descriptor that the user opens with the ++ * KBASE_IOCTL_KINSTR_JM_FD IO control call. ++ * @refcount: reference count for the context. Any reader will have a link ++ * back to the context so that they can remove themselves from the ++ * list. ++ * ++ * This is opaque outside this compilation unit ++ */ ++struct kbase_kinstr_jm { ++ struct hlist_bl_head readers; ++ struct kref refcount; ++}; ++ ++/** ++ * struct kbase_kinstr_jm_atom_state_change - Represents an atom changing to a ++ * new state ++ * @timestamp: Raw monotonic nanoseconds of the state change ++ * @state: The state that the atom has moved to ++ * @atom: The atom number that has changed state ++ * @flags: Flags associated with the state change. See ++ * KBASE_KINSTR_JM_ATOM_STATE_FLAG_* defines. ++ * @reserved: Reserved for future use. ++ * @data: Extra data for the state change. Active member depends on state. ++ * ++ * We can add new fields to the structure and old user code will gracefully ++ * ignore the new fields. ++ * ++ * We can change the size of the structure and old user code will gracefully ++ * skip over the new size via `struct kbase_kinstr_jm_fd_out->size`. ++ * ++ * If we remove fields, the version field in `struct ++ * kbase_kinstr_jm_fd_out->version` will be incremented and old user code will ++ * gracefully fail and tell the user that the kernel API is too new and has ++ * backwards-incompatible changes. Note that one userspace can opt to handle ++ * multiple kernel major versions of the structure. ++ * ++ * If we need to change the _meaning_ of one of the fields, i.e. the state ++ * machine has had a incompatible change, we can keep the same members in the ++ * structure and update the version as above. User code will no longer ++ * recognise that it has the supported field and can gracefully explain to the ++ * user that the kernel API is no longer supported. ++ * ++ * When making changes to this structure, make sure they are either: ++ * - additions to the end (for minor version bumps (i.e. only a size increase)) ++ * such that the layout of existing fields doesn't change, or; ++ * - update the version reported to userspace so that it can fail explicitly. ++ */ ++struct kbase_kinstr_jm_atom_state_change { ++ u64 timestamp; ++ s8 state; /* enum kbase_kinstr_jm_reader_atom_state */ ++ u8 atom; ++ u8 flags; ++ u8 reserved[1]; ++ /* Tagged union based on state. Ensure members are aligned correctly! */ ++ union { ++ struct { ++ u8 slot; ++ } start; ++ u8 padding[4]; ++ } data; ++}; ++static_assert( ++ ((1 << 8 * sizeof(((struct kbase_kinstr_jm_atom_state_change *)0)->state)) - 1) >= ++ KBASE_KINSTR_JM_READER_ATOM_STATE_COUNT); ++ ++#define KBASE_KINSTR_JM_ATOM_STATE_FLAG_OVERFLOW BIT(0) ++ ++/** ++ * struct reader_changes - The circular buffer of kernel atom state changes ++ * @data: The allocated buffer. This is allocated when the user requests ++ * the reader file descriptor. It is released when the user calls ++ * close() on the fd. When accessing this, lock the producer spin ++ * lock to prevent races on the allocated memory. The consume lock ++ * does not need to be held because newly-inserted data will always ++ * be outside the currenly-read range. ++ * @producer: The producing spinlock which allows us to push changes into the ++ * buffer at the same time as a user read occurring. This needs to ++ * be locked when saving/restoring the IRQ because we can receive an ++ * interrupt from the GPU when an atom completes. The CPU could have ++ * a task preempted that is holding this lock. ++ * @consumer: The consuming mutex which locks around the user read(). ++ * Must be held when updating the tail of the circular buffer. ++ * @head: The head of the circular buffer. Can be used with Linux @c CIRC_ ++ * helpers. The producer should lock and update this with an SMP ++ * store when a new change lands. The consumer can read with an ++ * SMP load. This allows the producer to safely insert new changes ++ * into the circular buffer. ++ * @tail: The tail of the circular buffer. Can be used with Linux @c CIRC_ ++ * helpers. The producer should do a READ_ONCE load and the consumer ++ * should SMP store. ++ * @size: The number of changes that are allowed in @c data. Can be used ++ * with Linux @c CIRC_ helpers. Will always be a power of two. The ++ * producer lock should be held when updating this and stored with ++ * an SMP release memory barrier. This means that the consumer can ++ * do an SMP load. ++ * @threshold: The number of changes above which threads polling on the reader ++ * file descriptor will be woken up. ++ */ ++struct reader_changes { ++ struct kbase_kinstr_jm_atom_state_change *data; ++ spinlock_t producer; ++ struct mutex consumer; ++ u32 head; ++ u32 tail; ++ u32 size; ++ u32 threshold; ++}; ++ ++/** ++ * reader_changes_is_valid_size() - Determines if requested changes buffer size ++ * is valid. ++ * @size: The requested memory size ++ * ++ * We have a constraint that the underlying physical buffer must be a ++ * power of two so that we can use the efficient circular buffer helpers that ++ * the kernel provides. It also needs to be representable within a u32. ++ * ++ * Return: ++ * * true - the size is valid ++ * * false - the size is invalid ++ */ ++static inline bool reader_changes_is_valid_size(const size_t size) ++{ ++ typedef struct reader_changes changes_t; ++ const size_t elem_size = sizeof(*((changes_t *)0)->data); ++ const size_t size_size = sizeof(((changes_t *)0)->size); ++ const size_t size_max = (1ull << (size_size * 8)) - 1; ++ ++ return is_power_of_2(size) && /* Is a power of two */ ++ ((size / elem_size) <= size_max); /* Small enough */ ++} ++ ++/** ++ * reader_changes_init() - Initializes the reader changes and allocates the ++ * changes buffer ++ * @changes: The context pointer, must point to a zero-inited allocated reader ++ * changes structure. We may support allocating the structure in the ++ * future. ++ * @size: The requested changes buffer size ++ * ++ * Return: ++ * (0, U16_MAX] - the number of data elements allocated ++ * -EINVAL - a pointer was invalid ++ * -ENOTSUP - we do not support allocation of the context ++ * -ERANGE - the requested memory size was invalid ++ * -ENOMEM - could not allocate the memory ++ * -EADDRINUSE - the buffer memory was already allocated ++ */ ++static int reader_changes_init(struct reader_changes *const changes, ++ const size_t size) ++{ ++ BUILD_BUG_ON((PAGE_SIZE % sizeof(*changes->data)) != 0); ++ ++ if (!reader_changes_is_valid_size(size)) { ++ pr_warn(PR_ "invalid size %zu\n", size); ++ return -ERANGE; ++ } ++ ++ changes->data = vmalloc(size); ++ if (!changes->data) ++ return -ENOMEM; ++ ++ spin_lock_init(&changes->producer); ++ mutex_init(&changes->consumer); ++ ++ changes->size = size / sizeof(*changes->data); ++ changes->threshold = min(((size_t)(changes->size)) / 4, ++ ((size_t)(PAGE_SIZE)) / sizeof(*changes->data)); ++ ++ return changes->size; ++} ++ ++/** ++ * reader_changes_term() - Cleans up a reader changes structure ++ * @changes: The context to clean up ++ * ++ * Releases the allocated state changes memory ++ */ ++static void reader_changes_term(struct reader_changes *const changes) ++{ ++ struct kbase_kinstr_jm_atom_state_change *data = NULL; ++ unsigned long irq; ++ ++ /* ++ * Although changes->data is used on the consumer side, too, no active ++ * consumer is possible by the time we clean up the reader changes, so ++ * no need to take the consumer lock. However, we do need the producer ++ * lock because the list removal can race with list traversal. ++ */ ++ spin_lock_irqsave(&changes->producer, irq); ++ swap(changes->data, data); ++ spin_unlock_irqrestore(&changes->producer, irq); ++ ++ mutex_destroy(&changes->consumer); ++ vfree(data); ++} ++ ++/** ++ * reader_changes_count_locked() - Retrieves the count of state changes from the ++ * tail to the physical end of the buffer ++ * @changes: The state changes context ++ * ++ * The consumer mutex must be held. Uses the CIRC_CNT_TO_END macro to ++ * determine the count, so there may be more items. However, that's the maximum ++ * number that can be read in one contiguous read. ++ * ++ * Return: the number of changes in the circular buffer until the end of the ++ * allocation ++ */ ++static u32 reader_changes_count_locked(struct reader_changes *const changes) ++{ ++ u32 head; ++ ++ lockdep_assert_held_once(&changes->consumer); ++ ++ head = smp_load_acquire(&changes->head); ++ ++ return CIRC_CNT_TO_END(head, changes->tail, changes->size); ++} ++ ++/** ++ * reader_changes_count() - Retrieves the count of state changes from the ++ * tail to the physical end of the buffer ++ * @changes: The state changes context ++ * ++ * Return: the number of changes in the circular buffer until the end of the ++ * allocation ++ */ ++static u32 reader_changes_count(struct reader_changes *const changes) ++{ ++ u32 ret; ++ ++ mutex_lock(&changes->consumer); ++ ret = reader_changes_count_locked(changes); ++ mutex_unlock(&changes->consumer); ++ return ret; ++} ++ ++/** ++ * reader_changes_push() - Pushes a change into the reader circular buffer. ++ * @changes: The buffer to insert the change into ++ * @change: Kernel atom change to insert ++ * @wait_queue: The queue to be kicked when changes should be read from ++ * userspace. Kicked when a threshold is reached or there is ++ * overflow. ++ */ ++static void reader_changes_push( ++ struct reader_changes *const changes, ++ const struct kbase_kinstr_jm_atom_state_change *const change, ++ wait_queue_head_t *const wait_queue) ++{ ++ u32 head, tail, size, space; ++ unsigned long irq; ++ struct kbase_kinstr_jm_atom_state_change *data; ++ ++ spin_lock_irqsave(&changes->producer, irq); ++ ++ /* We may be called for a reader_changes that's awaiting cleanup. */ ++ data = changes->data; ++ if (!data) ++ goto unlock; ++ ++ size = changes->size; ++ head = changes->head; ++ tail = smp_load_acquire(&changes->tail); ++ ++ space = CIRC_SPACE(head, tail, size); ++ if (space >= 1) { ++ data[head] = *change; ++ if (space == 1) { ++ data[head].flags |= ++ KBASE_KINSTR_JM_ATOM_STATE_FLAG_OVERFLOW; ++ pr_warn(PR_ "overflow of circular buffer\n"); ++ } ++ smp_store_release(&changes->head, (head + 1) & (size - 1)); ++ } ++ ++ /* Wake for either overflow or over-threshold cases. */ ++ if (CIRC_CNT(head + 1, tail, size) >= changes->threshold) ++ wake_up_interruptible(wait_queue); ++ ++unlock: ++ spin_unlock_irqrestore(&changes->producer, irq); ++} ++ ++/** ++ * struct reader - Allows the kernel state changes to be read by user space. ++ * @node: The node in the @c readers locked list ++ * @rcu_head: storage for the RCU callback to free this reader (see kfree_rcu) ++ * @changes: The circular buffer of user changes ++ * @wait_queue: A wait queue for poll ++ * @context: a pointer to the parent context that created this reader. Can be ++ * used to remove the reader from the list of readers. Reference ++ * counted. ++ * ++ * The reader is a circular buffer in kernel space. State changes are pushed ++ * into the buffer. The flow from user space is: ++ * ++ * * Request file descriptor with KBASE_IOCTL_KINSTR_JM_FD. This will ++ * allocate the kernel side circular buffer with a size specified in the ++ * ioctl argument. ++ * * The user will then poll the file descriptor for data ++ * * Upon receiving POLLIN, perform a read() on the file descriptor to get ++ * the data out. ++ * * The buffer memory will be freed when the file descriptor is closed ++ */ ++struct reader { ++ struct hlist_bl_node node; ++ struct rcu_head rcu_head; ++ struct reader_changes changes; ++ wait_queue_head_t wait_queue; ++ struct kbase_kinstr_jm *context; ++}; ++ ++static struct kbase_kinstr_jm * ++kbase_kinstr_jm_ref_get(struct kbase_kinstr_jm *const ctx); ++static void kbase_kinstr_jm_ref_put(struct kbase_kinstr_jm *const ctx); ++static int kbase_kinstr_jm_readers_add(struct kbase_kinstr_jm *const ctx, ++ struct reader *const reader); ++static void kbase_kinstr_jm_readers_del(struct kbase_kinstr_jm *const ctx, ++ struct reader *const reader); ++ ++/** ++ * reader_term() - Terminate a instrumentation job manager reader context. ++ * @reader: Pointer to context to be terminated. ++ */ ++static void reader_term(struct reader *const reader) ++{ ++ if (!reader) ++ return; ++ ++ kbase_kinstr_jm_readers_del(reader->context, reader); ++ reader_changes_term(&reader->changes); ++ kbase_kinstr_jm_ref_put(reader->context); ++ ++ kfree_rcu(reader, rcu_head); ++} ++ ++/** ++ * reader_init() - Initialise a instrumentation job manager reader context. ++ * @out_reader: Non-NULL pointer to where the pointer to the created context ++ * will be stored on success. ++ * @ctx: the pointer to the parent context. Reference count will be ++ * increased if initialization is successful ++ * @num_changes: The number of changes to allocate a buffer for ++ * ++ * Return: 0 on success, else error code. ++ */ ++static int reader_init(struct reader **const out_reader, ++ struct kbase_kinstr_jm *const ctx, ++ size_t const num_changes) ++{ ++ struct reader *reader = NULL; ++ const size_t change_size = sizeof(struct kbase_kinstr_jm_atom_state_change); ++ int status; ++ ++ if (!out_reader || !ctx || !num_changes) ++ return -EINVAL; ++ ++ reader = kzalloc(sizeof(*reader), GFP_KERNEL); ++ if (!reader) ++ return -ENOMEM; ++ ++ INIT_HLIST_BL_NODE(&reader->node); ++ init_waitqueue_head(&reader->wait_queue); ++ ++ reader->context = kbase_kinstr_jm_ref_get(ctx); ++ ++ status = reader_changes_init(&reader->changes, num_changes * change_size); ++ if (status < 0) ++ goto fail; ++ ++ status = kbase_kinstr_jm_readers_add(ctx, reader); ++ if (status < 0) ++ goto fail; ++ ++ *out_reader = reader; ++ ++ return 0; ++ ++fail: ++ kbase_kinstr_jm_ref_put(reader->context); ++ kfree(reader); ++ return status; ++} ++ ++/** ++ * reader_release() - Invoked when the reader file descriptor is released ++ * @node: The inode that the file descriptor that the file corresponds to. In ++ * our case our reader file descriptor is backed by an anonymous node so ++ * not much is in this. ++ * @file: the file data. Our reader context is held in the private data ++ * Return: zero on success ++ */ ++static int reader_release(struct inode *const node, struct file *const file) ++{ ++ struct reader *const reader = file->private_data; ++ ++ reader_term(reader); ++ file->private_data = NULL; ++ ++ return 0; ++} ++ ++/** ++ * reader_changes_copy_to_user() - Copy any changes from a changes structure to ++ * the user-provided buffer. ++ * @changes: The changes structure from which to copy. ++ * @buffer: The user buffer to copy the data to. ++ * @buffer_size: The number of bytes in the buffer. ++ * Return: The number of bytes copied or negative errno on failure. ++ */ ++static ssize_t reader_changes_copy_to_user(struct reader_changes *const changes, ++ char __user *buffer, ++ size_t buffer_size) ++{ ++ ssize_t ret = 0; ++ struct kbase_kinstr_jm_atom_state_change const *src_buf = READ_ONCE( ++ changes->data); ++ size_t const entry_size = sizeof(*src_buf); ++ size_t changes_tail, changes_count, read_size; ++ ++ /* Needed for the quick buffer capacity calculation below. ++ * Note that we can't use is_power_of_2() since old compilers don't ++ * understand it's a constant expression. ++ */ ++#define is_power_of_two(x) ((x) && !((x) & ((x) - 1))) ++ static_assert(is_power_of_two( ++ sizeof(struct kbase_kinstr_jm_atom_state_change))); ++#undef is_power_of_two ++ ++ lockdep_assert_held_once(&changes->consumer); ++ ++ /* Read continuously until either: ++ * - we've filled the output buffer, or ++ * - there are no changes when we check. ++ * ++ * If more changes arrive while we're copying to the user, we can copy ++ * those as well, space permitting. ++ */ ++ do { ++ changes_tail = changes->tail; ++ changes_count = reader_changes_count_locked(changes); ++ read_size = min(changes_count * entry_size, ++ buffer_size & ~(entry_size - 1)); ++ ++ if (!read_size) ++ break; ++ ++ if (copy_to_user(buffer, &(src_buf[changes_tail]), read_size)) ++ return -EFAULT; ++ ++ buffer += read_size; ++ buffer_size -= read_size; ++ ret += read_size; ++ changes_tail = (changes_tail + read_size / entry_size) & ++ (changes->size - 1); ++ smp_store_release(&changes->tail, changes_tail); ++ } while (read_size); ++ ++ return ret; ++} ++ ++/** ++ * reader_read() - Handles a read call on the reader file descriptor ++ * ++ * @filp: The file that the read was performed on ++ * @buffer: The destination buffer ++ * @buffer_size: The maximum number of bytes to read ++ * @offset: The offset into the 'file' to read from. ++ * ++ * Note the destination buffer needs to be fully mapped in userspace or the read ++ * will fault. ++ * ++ * Return: ++ * * The number of bytes read or: ++ * * -EBADF - the file descriptor did not have an attached reader ++ * * -EFAULT - memory access fault ++ * * -EAGAIN - if the file is set to nonblocking reads with O_NONBLOCK and there ++ * is no data available ++ * ++ * Note: The number of bytes read will always be a multiple of the size of an ++ * entry. ++ */ ++static ssize_t reader_read(struct file *const filp, ++ char __user *const buffer, ++ size_t const buffer_size, ++ loff_t *const offset) ++{ ++ struct reader *const reader = filp->private_data; ++ struct reader_changes *changes; ++ ssize_t ret; ++ ++ if (!reader) ++ return -EBADF; ++ ++ if (buffer_size < sizeof(struct kbase_kinstr_jm_atom_state_change)) ++ return -ENOBUFS; ++ ++#if KERNEL_VERSION(5, 0, 0) <= LINUX_VERSION_CODE ++ if (!access_ok(buffer, buffer_size)) ++ return -EIO; ++#else ++ if (!access_ok(VERIFY_WRITE, buffer, buffer_size)) ++ return -EIO; ++#endif ++ ++ changes = &reader->changes; ++ ++ mutex_lock(&changes->consumer); ++ if (!reader_changes_count_locked(changes)) { ++ if (filp->f_flags & O_NONBLOCK) { ++ ret = -EAGAIN; ++ goto exit; ++ } ++ ++ if (wait_event_interruptible( ++ reader->wait_queue, ++ !!reader_changes_count_locked(changes))) { ++ ret = -EINTR; ++ goto exit; ++ } ++ } ++ ++ ret = reader_changes_copy_to_user(changes, buffer, buffer_size); ++ ++exit: ++ mutex_unlock(&changes->consumer); ++ return ret; ++} ++ ++/** ++ * reader_poll() - Handles a poll call on the reader file descriptor ++ * @file: The file that the poll was performed on ++ * @wait: The poll table ++ * ++ * The results of the poll will be unreliable if there is no mapped memory as ++ * there is no circular buffer to push atom state changes into. ++ * ++ * Return: ++ * * 0 - no data ready ++ * * POLLIN - state changes have been buffered ++ * * -EBADF - the file descriptor did not have an attached reader ++ * * -EINVAL - the IO control arguments were invalid ++ */ ++static __poll_t reader_poll(struct file *const file, ++ struct poll_table_struct *const wait) ++{ ++ struct reader *reader; ++ struct reader_changes *changes; ++ ++ if (unlikely(!file || !wait)) ++ return -EINVAL; ++ ++ reader = file->private_data; ++ if (unlikely(!reader)) ++ return -EBADF; ++ ++ changes = &reader->changes; ++ ++ if (reader_changes_count(changes) >= changes->threshold) ++ return POLLIN; ++ ++ poll_wait(file, &reader->wait_queue, wait); ++ ++ return (reader_changes_count(changes) > 0) ? POLLIN : 0; ++} ++ ++/* The file operations virtual function table */ ++static const struct file_operations file_operations = { ++ .owner = THIS_MODULE, ++ .llseek = no_llseek, ++ .read = reader_read, ++ .poll = reader_poll, ++ .release = reader_release ++}; ++ ++/* The maximum amount of readers that can be created on a context. */ ++static const size_t kbase_kinstr_jm_readers_max = 16; ++ ++/** ++ * kbasep_kinstr_jm_release() - Invoked when the reference count is dropped ++ * @ref: the context reference count ++ */ ++static void kbase_kinstr_jm_release(struct kref *const ref) ++{ ++ struct kbase_kinstr_jm *const ctx = ++ container_of(ref, struct kbase_kinstr_jm, refcount); ++ ++ kfree(ctx); ++} ++ ++/** ++ * kbase_kinstr_jm_ref_get() - Reference counts the instrumentation context ++ * @ctx: the context to reference count ++ * Return: the reference counted context ++ */ ++static struct kbase_kinstr_jm * ++kbase_kinstr_jm_ref_get(struct kbase_kinstr_jm *const ctx) ++{ ++ if (likely(ctx)) ++ kref_get(&ctx->refcount); ++ return ctx; ++} ++ ++/** ++ * kbase_kinstr_jm_ref_put() - Dereferences the instrumentation context ++ * @ctx: the context to lower the reference count on ++ */ ++static void kbase_kinstr_jm_ref_put(struct kbase_kinstr_jm *const ctx) ++{ ++ if (likely(ctx)) ++ kref_put(&ctx->refcount, kbase_kinstr_jm_release); ++} ++ ++/** ++ * kbase_kinstr_jm_readers_add() - Adds a reader to the list of readers ++ * @ctx: the instrumentation context ++ * @reader: the reader to add ++ * ++ * Return: ++ * 0 - success ++ * -ENOMEM - too many readers already added. ++ */ ++static int kbase_kinstr_jm_readers_add(struct kbase_kinstr_jm *const ctx, ++ struct reader *const reader) ++{ ++ struct hlist_bl_head *const readers = &ctx->readers; ++ struct hlist_bl_node *node; ++ struct reader *temp; ++ size_t count = 0; ++ ++ hlist_bl_lock(readers); ++ ++ hlist_bl_for_each_entry_rcu(temp, node, readers, node) ++ ++count; ++ ++ if (kbase_kinstr_jm_readers_max < count) { ++ hlist_bl_unlock(readers); ++ return -ENOMEM; ++ } ++ ++ hlist_bl_add_head_rcu(&reader->node, readers); ++ ++ hlist_bl_unlock(readers); ++ ++ static_branch_inc(&basep_kinstr_jm_reader_static_key); ++ ++ return 0; ++} ++ ++/** ++ * readers_del() - Deletes a reader from the list of readers ++ * @ctx: the instrumentation context ++ * @reader: the reader to delete ++ */ ++static void kbase_kinstr_jm_readers_del(struct kbase_kinstr_jm *const ctx, ++ struct reader *const reader) ++{ ++ struct hlist_bl_head *const readers = &ctx->readers; ++ ++ hlist_bl_lock(readers); ++ hlist_bl_del_rcu(&reader->node); ++ hlist_bl_unlock(readers); ++ ++ static_branch_dec(&basep_kinstr_jm_reader_static_key); ++} ++ ++int kbase_kinstr_jm_get_fd(struct kbase_kinstr_jm *const ctx, ++ union kbase_kinstr_jm_fd *jm_fd_arg) ++{ ++ struct kbase_kinstr_jm_fd_in const *in; ++ struct reader *reader; ++ size_t const change_size = sizeof(struct ++ kbase_kinstr_jm_atom_state_change); ++ int status; ++ int fd; ++ int i; ++ ++ if (!ctx || !jm_fd_arg) ++ return -EINVAL; ++ ++ in = &jm_fd_arg->in; ++ ++ if (!is_power_of_2(in->count)) ++ return -EINVAL; ++ ++ for (i = 0; i < sizeof(in->padding); ++i) ++ if (in->padding[i]) ++ return -EINVAL; ++ ++ status = reader_init(&reader, ctx, in->count); ++ if (status < 0) ++ return status; ++ ++ jm_fd_arg->out.version = KBASE_KINSTR_JM_VERSION; ++ jm_fd_arg->out.size = change_size; ++ memset(&jm_fd_arg->out.padding, 0, sizeof(jm_fd_arg->out.padding)); ++ ++ fd = anon_inode_getfd("[mali_kinstr_jm]", &file_operations, reader, ++ O_CLOEXEC); ++ if (fd < 0) ++ reader_term(reader); ++ ++ return fd; ++} ++ ++int kbase_kinstr_jm_init(struct kbase_kinstr_jm **const out_ctx) ++{ ++ struct kbase_kinstr_jm *ctx = NULL; ++ ++ if (!out_ctx) ++ return -EINVAL; ++ ++ ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); ++ if (!ctx) ++ return -ENOMEM; ++ ++ INIT_HLIST_BL_HEAD(&ctx->readers); ++ kref_init(&ctx->refcount); ++ ++ *out_ctx = ctx; ++ ++ return 0; ++} ++ ++void kbase_kinstr_jm_term(struct kbase_kinstr_jm *const ctx) ++{ ++ kbase_kinstr_jm_ref_put(ctx); ++} ++ ++void kbasep_kinstr_jm_atom_state( ++ struct kbase_jd_atom *const katom, ++ const enum kbase_kinstr_jm_reader_atom_state state) ++{ ++ struct kbase_context *const kctx = katom->kctx; ++ struct kbase_kinstr_jm *const ctx = kctx->kinstr_jm; ++ const u8 id = kbase_jd_atom_id(kctx, katom); ++ struct kbase_kinstr_jm_atom_state_change change = { ++ .timestamp = ktime_get_raw_ns(), .atom = id, .state = state ++ }; ++ struct reader *reader; ++ struct hlist_bl_node *node; ++ ++ WARN(KBASE_KINSTR_JM_READER_ATOM_STATE_COUNT < state || 0 > state, ++ PR_ "unsupported katom (%u) state (%i)", id, state); ++ ++ switch (state) { ++ case KBASE_KINSTR_JM_READER_ATOM_STATE_START: ++ change.data.start.slot = katom->jobslot; ++ break; ++ default: ++ break; ++ } ++ ++ rcu_read_lock(); ++ hlist_bl_for_each_entry_rcu(reader, node, &ctx->readers, node) ++ reader_changes_push( ++ &reader->changes, &change, &reader->wait_queue); ++ rcu_read_unlock(); ++} ++ ++KBASE_EXPORT_TEST_API(kbasep_kinstr_jm_atom_state); ++ ++void kbasep_kinstr_jm_atom_hw_submit(struct kbase_jd_atom *const katom) ++{ ++ struct kbase_context *const kctx = katom->kctx; ++ struct kbase_device *const kbdev = kctx->kbdev; ++ const int slot = katom->slot_nr; ++ struct kbase_jd_atom *const submitted = kbase_gpu_inspect(kbdev, slot, 0); ++ ++ BUILD_BUG_ON(SLOT_RB_SIZE != 2); ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (WARN_ON(slot < 0 || slot >= GPU_MAX_JOB_SLOTS)) ++ return; ++ if (WARN_ON(!submitted)) ++ return; ++ ++ if (submitted == katom) ++ kbase_kinstr_jm_atom_state_start(katom); ++} ++ ++void kbasep_kinstr_jm_atom_hw_release(struct kbase_jd_atom *const katom) ++{ ++ struct kbase_context *const kctx = katom->kctx; ++ struct kbase_device *const kbdev = kctx->kbdev; ++ const int slot = katom->slot_nr; ++ struct kbase_jd_atom *const submitted = kbase_gpu_inspect(kbdev, slot, 0); ++ struct kbase_jd_atom *const queued = kbase_gpu_inspect(kbdev, slot, 1); ++ ++ BUILD_BUG_ON(SLOT_RB_SIZE != 2); ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (WARN_ON(slot < 0 || slot >= GPU_MAX_JOB_SLOTS)) ++ return; ++ if (WARN_ON(!submitted)) ++ return; ++ if (WARN_ON((submitted != katom) && (queued != katom))) ++ return; ++ ++ if (queued == katom) ++ return; ++ ++ if (katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED) ++ kbase_kinstr_jm_atom_state_stop(katom); ++ if (queued && queued->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED) ++ kbase_kinstr_jm_atom_state_start(queued); ++} +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_kinstr_jm.h b/drivers/gpu/arm/bifrost/mali_kbase_kinstr_jm.h +new file mode 100755 +index 000000000000..555edfeef77c +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_kinstr_jm.h +@@ -0,0 +1,283 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019,2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* ++ * mali_kbase_kinstr_jm.h ++ * Kernel driver public interface to job manager atom tracing. This API provides ++ * a method to get the atom state changes into user space. ++ * ++ * The flow of operation is: ++ * ++ * | kernel | user | ++ * | ----------------------------------- | ----------------------------------- | ++ * | Initialize API with | | ++ * | kbase_kinstr_jm_init() | | ++ * | | | ++ * | Kernel code injects states with | | ++ * | kbase_kinstr_jm_atom_state_*() APIs | | ++ * | | Call ioctl() to get file descriptor | ++ * | | via KBASE_IOCTL_KINSTR_JM_FD | ++ * | Allocates a reader attached to FD | | ++ * | Allocates circular buffer and | | ++ * | patches, via ASM goto, the | | ++ * | kbase_kinstr_jm_atom_state_*() | | ++ * | | loop: | ++ * | | Call poll() on FD for POLLIN | ++ * | When threshold of changes is hit, | | ++ * | the poll is interrupted with | | ++ * | POLLIN. If circular buffer is | | ++ * | full then store the missed count | | ++ * | and interrupt poll | Call read() to get data from | ++ * | | circular buffer via the fd | ++ * | Kernel advances tail of circular | | ++ * | buffer | | ++ * | | Close file descriptor | ++ * | Deallocates circular buffer | | ++ * | | | ++ * | Terminate API with | | ++ * | kbase_kinstr_jm_term() | | ++ * ++ * All tracepoints are guarded on a static key. The static key is activated when ++ * a user space reader gets created. This means that there is negligible cost ++ * inserting the tracepoints into code when there are no readers. ++ */ ++ ++#ifndef _KBASE_KINSTR_JM_H_ ++#define _KBASE_KINSTR_JM_H_ ++ ++#include "mali_kbase_kinstr_jm_reader.h" ++ ++#ifdef __KERNEL__ ++#include ++#include ++#else ++/* empty wrapper macros for userspace */ ++#define static_branch_unlikely(key) (1) ++#define KERNEL_VERSION(a, b, c) (0) ++#define LINUX_VERSION_CODE (1) ++#endif /* __KERNEL__ */ ++ ++/* Forward declarations */ ++struct kbase_context; ++struct kbase_kinstr_jm; ++struct kbase_jd_atom; ++union kbase_kinstr_jm_fd; ++ ++/** ++ * kbase_kinstr_jm_init() - Initialise an instrumentation job manager context. ++ * @ctx: Non-NULL pointer to where the pointer to the created context will ++ * be stored on success. ++ * ++ * Return: 0 on success, else error code. ++ */ ++int kbase_kinstr_jm_init(struct kbase_kinstr_jm **ctx); ++ ++/** ++ * kbase_kinstr_jm_term() - Terminate an instrumentation job manager context. ++ * @ctx: Pointer to context to be terminated. ++ */ ++void kbase_kinstr_jm_term(struct kbase_kinstr_jm *ctx); ++ ++/** ++ * kbase_kinstr_jm_get_fd() - Retrieves a file descriptor that can be used to ++ * read the atom state changes from userspace ++ * ++ * @ctx: Pointer to the initialized context ++ * @jm_fd_arg: Pointer to the union containing the in/out params ++ * Return: -1 on failure, valid file descriptor on success ++ */ ++int kbase_kinstr_jm_get_fd(struct kbase_kinstr_jm *const ctx, ++ union kbase_kinstr_jm_fd *jm_fd_arg); ++ ++/** ++ * kbasep_kinstr_jm_atom_state() - Signifies that an atom has changed state ++ * @atom: The atom that has changed state ++ * @state: The new state of the atom ++ * ++ * This performs the actual storage of the state ready for user space to ++ * read the data. It is only called when the static key is enabled from ++ * kbase_kinstr_jm_atom_state(). There is almost never a need to invoke this ++ * function directly. ++ */ ++void kbasep_kinstr_jm_atom_state( ++ struct kbase_jd_atom *const atom, ++ const enum kbase_kinstr_jm_reader_atom_state state); ++ ++/* Allows ASM goto patching to reduce tracing overhead. This is ++ * incremented/decremented when readers are created and terminated. This really ++ * shouldn't be changed externally, but if you do, make sure you use ++ * a static_key_inc()/static_key_dec() pair. ++ */ ++#if KERNEL_VERSION(4, 3, 0) <= LINUX_VERSION_CODE ++extern struct static_key_false basep_kinstr_jm_reader_static_key; ++#else ++/* Pre-4.3 kernels have a different API for static keys, but work ++ * mostly the same with less type safety. */ ++extern struct static_key basep_kinstr_jm_reader_static_key; ++#define static_branch_unlikely(key) static_key_false(key) ++#endif /* KERNEL_VERSION(4, 3, 0) <= LINUX_VERSION_CODE */ ++ ++/** ++ * kbase_kinstr_jm_atom_state() - Signifies that an atom has changed state ++ * @atom: The atom that has changed state ++ * @state: The new state of the atom ++ * ++ * This uses a static key to reduce overhead when tracing is disabled ++ */ ++static inline void kbase_kinstr_jm_atom_state( ++ struct kbase_jd_atom *const atom, ++ const enum kbase_kinstr_jm_reader_atom_state state) ++{ ++ if (static_branch_unlikely(&basep_kinstr_jm_reader_static_key)) ++ kbasep_kinstr_jm_atom_state(atom, state); ++} ++ ++/** ++ * kbase_kinstr_jm_atom_state_queue() - Signifies that an atom has entered a ++ * hardware or software queue. ++ * @atom: The atom that has changed state ++ */ ++static inline void kbase_kinstr_jm_atom_state_queue( ++ struct kbase_jd_atom *const atom) ++{ ++ kbase_kinstr_jm_atom_state( ++ atom, KBASE_KINSTR_JM_READER_ATOM_STATE_QUEUE); ++} ++ ++/** ++ * kbase_kinstr_jm_atom_state_start() - Signifies that work has started on an ++ * atom ++ * @atom: The atom that has changed state ++ */ ++static inline void kbase_kinstr_jm_atom_state_start( ++ struct kbase_jd_atom *const atom) ++{ ++ kbase_kinstr_jm_atom_state( ++ atom, KBASE_KINSTR_JM_READER_ATOM_STATE_START); ++} ++ ++/** ++ * kbase_kinstr_jm_atom_state_stop() - Signifies that work has stopped on an ++ * atom ++ * @atom: The atom that has changed state ++ */ ++static inline void kbase_kinstr_jm_atom_state_stop( ++ struct kbase_jd_atom *const atom) ++{ ++ kbase_kinstr_jm_atom_state( ++ atom, KBASE_KINSTR_JM_READER_ATOM_STATE_STOP); ++} ++ ++/** ++ * kbase_kinstr_jm_atom_state_complete() - Signifies that all work has completed ++ * on an atom ++ * @atom: The atom that has changed state ++ */ ++static inline void kbase_kinstr_jm_atom_state_complete( ++ struct kbase_jd_atom *const atom) ++{ ++ kbase_kinstr_jm_atom_state( ++ atom, KBASE_KINSTR_JM_READER_ATOM_STATE_COMPLETE); ++} ++ ++/** ++ * kbase_kinstr_jm_atom_queue() - A software *or* hardware atom is queued for ++ * execution ++ * @atom: The atom that has changed state ++ */ ++static inline void kbase_kinstr_jm_atom_queue(struct kbase_jd_atom *const atom) ++{ ++ kbase_kinstr_jm_atom_state_queue(atom); ++} ++ ++/** ++ * kbase_kinstr_jm_atom_complete() - A software *or* hardware atom is fully ++ * completed ++ * @atom: The atom that has changed state ++ */ ++static inline void kbase_kinstr_jm_atom_complete( ++ struct kbase_jd_atom *const atom) ++{ ++ kbase_kinstr_jm_atom_state_complete(atom); ++} ++ ++/** ++ * kbase_kinstr_jm_atom_sw_start() - A software atom has started work ++ * @atom: The atom that has changed state ++ */ ++static inline void kbase_kinstr_jm_atom_sw_start( ++ struct kbase_jd_atom *const atom) ++{ ++ kbase_kinstr_jm_atom_state_start(atom); ++} ++ ++/** ++ * kbase_kinstr_jm_atom_sw_stop() - A software atom has stopped work ++ * @atom: The atom that has changed state ++ */ ++static inline void kbase_kinstr_jm_atom_sw_stop( ++ struct kbase_jd_atom *const atom) ++{ ++ kbase_kinstr_jm_atom_state_stop(atom); ++} ++ ++/** ++ * kbasep_kinstr_jm_atom_hw_submit() - A hardware atom has been submitted ++ * @atom: The atom that has been submitted ++ * ++ * This private implementation should not be called directly, it is protected ++ * by a static key in kbase_kinstr_jm_atom_hw_submit(). Use that instead. ++ */ ++void kbasep_kinstr_jm_atom_hw_submit(struct kbase_jd_atom *const atom); ++ ++/** ++ * kbase_kinstr_jm_atom_hw_submit() - A hardware atom has been submitted ++ * @atom: The atom that has been submitted ++ */ ++static inline void kbase_kinstr_jm_atom_hw_submit( ++ struct kbase_jd_atom *const atom) ++{ ++ if (static_branch_unlikely(&basep_kinstr_jm_reader_static_key)) ++ kbasep_kinstr_jm_atom_hw_submit(atom); ++} ++ ++/** ++ * kbasep_kinstr_jm_atom_hw_release() - A hardware atom has been released ++ * @atom: The atom that has been released ++ * ++ * This private implementation should not be called directly, it is protected ++ * by a static key in kbase_kinstr_jm_atom_hw_release(). Use that instead. ++ */ ++void kbasep_kinstr_jm_atom_hw_release(struct kbase_jd_atom *const atom); ++ ++/** ++ * kbase_kinstr_jm_atom_hw_release() - A hardware atom has been released ++ * @atom: The atom that has been released ++ */ ++static inline void kbase_kinstr_jm_atom_hw_release( ++ struct kbase_jd_atom *const atom) ++{ ++ if (static_branch_unlikely(&basep_kinstr_jm_reader_static_key)) ++ kbasep_kinstr_jm_atom_hw_release(atom); ++} ++ ++#endif /* _KBASE_KINSTR_JM_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_kinstr_jm_reader.h b/drivers/gpu/arm/bifrost/mali_kbase_kinstr_jm_reader.h +new file mode 100755 +index 000000000000..e267e6bc44de +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_kinstr_jm_reader.h +@@ -0,0 +1,70 @@ ++/* ++ * ++ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* ++ * mali_kbase_kinstr_jm_reader.h ++ * Provides an ioctl API to read kernel atom state changes. The flow of the ++ * API is: ++ * 1. Obtain the file descriptor with ``KBASE_IOCTL_KINSTR_JM_FD`` ++ * 2. Determine the buffer structure layout via the above ioctl's returned ++ * size and version fields in ``struct kbase_kinstr_jm_fd_out`` ++ * 4. Poll the file descriptor for ``POLLIN`` ++ * 5. Get data with read() on the fd ++ * 6. Use the structure version to understand how to read the data from the ++ * buffer ++ * 7. Repeat 4-6 ++ * 8. Close the file descriptor ++ */ ++ ++#ifndef _KBASE_KINSTR_JM_READER_H_ ++#define _KBASE_KINSTR_JM_READER_H_ ++ ++/** ++ * enum kbase_kinstr_jm_reader_atom_state - Determines the work state of an atom ++ * @KBASE_KINSTR_JM_READER_ATOM_STATE_QUEUE: Signifies that an atom has ++ * entered a hardware queue ++ * @KBASE_KINSTR_JM_READER_ATOM_STATE_START: Signifies that work has started ++ * on an atom ++ * @KBASE_KINSTR_JM_READER_ATOM_STATE_STOP: Signifies that work has stopped ++ * on an atom ++ * @KBASE_KINSTR_JM_READER_ATOM_STATE_COMPLETE: Signifies that work has ++ * completed on an atom ++ * @KBASE_KINSTR_JM_READER_ATOM_STATE_COUNT: The number of state enumerations ++ * ++ * We can add new states to the end of this if they do not break the existing ++ * state machine. Old user mode code can gracefully ignore states they do not ++ * understand. ++ * ++ * If we need to make a breaking change to the state machine, we can do that by ++ * changing the version reported by KBASE_IOCTL_KINSTR_JM_FD. This will ++ * mean that old user mode code will fail to understand the new state field in ++ * the structure and gracefully not use the state change API. ++ */ ++enum kbase_kinstr_jm_reader_atom_state { ++ KBASE_KINSTR_JM_READER_ATOM_STATE_QUEUE, ++ KBASE_KINSTR_JM_READER_ATOM_STATE_START, ++ KBASE_KINSTR_JM_READER_ATOM_STATE_STOP, ++ KBASE_KINSTR_JM_READER_ATOM_STATE_COMPLETE, ++ KBASE_KINSTR_JM_READER_ATOM_STATE_COUNT ++}; ++ ++#endif /* _KBASE_KINSTR_JM_READER_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_linux.h b/drivers/gpu/arm/bifrost/mali_kbase_linux.h +new file mode 100755 +index 000000000000..003ac9e68a76 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_linux.h +@@ -0,0 +1,48 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2014 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++/** ++ * @file mali_kbase_linux.h ++ * Base kernel APIs, Linux implementation. ++ */ ++ ++#ifndef _KBASE_LINUX_H_ ++#define _KBASE_LINUX_H_ ++ ++/* All things that are needed for the Linux port. */ ++#include ++#include ++#include ++#include ++#include ++ ++#if (defined(MALI_KERNEL_TEST_API) && (1 == MALI_KERNEL_TEST_API)) ++ #define KBASE_EXPORT_TEST_API(func) EXPORT_SYMBOL(func) ++#else ++ #define KBASE_EXPORT_TEST_API(func) ++#endif ++ ++#define KBASE_EXPORT_SYMBOL(func) EXPORT_SYMBOL(func) ++ ++#endif /* _KBASE_LINUX_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem.c b/drivers/gpu/arm/bifrost/mali_kbase_mem.c +new file mode 100755 +index 000000000000..82a799c2d673 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_mem.c +@@ -0,0 +1,4734 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/** ++ * Base kernel memory APIs ++ */ ++#include ++#include ++#include ++#include ++#include ++#include ++#ifdef CONFIG_OF ++#include ++#endif ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++/* ++ * Alignment of objects allocated by the GPU inside a just-in-time memory ++ * region whose size is given by an end address ++ * ++ * This is the alignment of objects allocated by the GPU, but possibly not ++ * fully written to. When taken into account with ++ * KBASE_GPU_ALLOCATED_OBJECT_MAX_BYTES it gives the maximum number of bytes ++ * that the JIT memory report size can exceed the actual backed memory size. ++ */ ++#define KBASE_GPU_ALLOCATED_OBJECT_ALIGN_BYTES (128u) ++ ++/* ++ * Maximum size of objects allocated by the GPU inside a just-in-time memory ++ * region whose size is given by an end address ++ * ++ * This is the maximum size of objects allocated by the GPU, but possibly not ++ * fully written to. When taken into account with ++ * KBASE_GPU_ALLOCATED_OBJECT_ALIGN_BYTES it gives the maximum number of bytes ++ * that the JIT memory report size can exceed the actual backed memory size. ++ */ ++#define KBASE_GPU_ALLOCATED_OBJECT_MAX_BYTES (512u) ++ ++ ++/* Forward declarations */ ++static void free_partial_locked(struct kbase_context *kctx, ++ struct kbase_mem_pool *pool, struct tagged_addr tp); ++ ++static size_t kbase_get_num_cpu_va_bits(struct kbase_context *kctx) ++{ ++#if defined(CONFIG_ARM64) ++ /* VA_BITS can be as high as 48 bits, but all bits are available for ++ * both user and kernel. ++ */ ++ size_t cpu_va_bits = VA_BITS; ++#elif defined(CONFIG_X86_64) ++ /* x86_64 can access 48 bits of VA, but the 48th is used to denote ++ * kernel (1) vs userspace (0), so the max here is 47. ++ */ ++ size_t cpu_va_bits = 47; ++#elif defined(CONFIG_ARM) || defined(CONFIG_X86_32) ++ size_t cpu_va_bits = sizeof(void *) * BITS_PER_BYTE; ++#else ++#error "Unknown CPU VA width for this architecture" ++#endif ++ ++#ifdef CONFIG_64BIT ++ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) ++ cpu_va_bits = 32; ++#endif ++ ++ return cpu_va_bits; ++} ++ ++/* This function finds out which RB tree the given pfn from the GPU VA belongs ++ * to based on the memory zone the pfn refers to */ ++static struct rb_root *kbase_gpu_va_to_rbtree(struct kbase_context *kctx, ++ u64 gpu_pfn) ++{ ++ struct rb_root *rbtree = NULL; ++ ++ /* The gpu_pfn can only be greater than the starting pfn of the EXEC_VA ++ * zone if this has been initialized. ++ */ ++ if (gpu_pfn >= kctx->exec_va_start) ++ rbtree = &kctx->reg_rbtree_exec; ++ else { ++ u64 same_va_end; ++ ++#ifdef CONFIG_64BIT ++ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) ++#endif /* CONFIG_64BIT */ ++ same_va_end = KBASE_REG_ZONE_CUSTOM_VA_BASE; ++#ifdef CONFIG_64BIT ++ else ++ same_va_end = kctx->same_va_end; ++#endif /* CONFIG_64BIT */ ++ ++ if (gpu_pfn >= same_va_end) ++ rbtree = &kctx->reg_rbtree_custom; ++ else ++ rbtree = &kctx->reg_rbtree_same; ++ } ++ ++ return rbtree; ++} ++ ++/* This function inserts a region into the tree. */ ++static void kbase_region_tracker_insert(struct kbase_va_region *new_reg) ++{ ++ u64 start_pfn = new_reg->start_pfn; ++ struct rb_node **link = NULL; ++ struct rb_node *parent = NULL; ++ struct rb_root *rbtree = NULL; ++ ++ rbtree = new_reg->rbtree; ++ ++ link = &(rbtree->rb_node); ++ /* Find the right place in the tree using tree search */ ++ while (*link) { ++ struct kbase_va_region *old_reg; ++ ++ parent = *link; ++ old_reg = rb_entry(parent, struct kbase_va_region, rblink); ++ ++ /* RBTree requires no duplicate entries. */ ++ KBASE_DEBUG_ASSERT(old_reg->start_pfn != start_pfn); ++ ++ if (old_reg->start_pfn > start_pfn) ++ link = &(*link)->rb_left; ++ else ++ link = &(*link)->rb_right; ++ } ++ ++ /* Put the new node there, and rebalance tree */ ++ rb_link_node(&(new_reg->rblink), parent, link); ++ ++ rb_insert_color(&(new_reg->rblink), rbtree); ++} ++ ++static struct kbase_va_region *find_region_enclosing_range_rbtree( ++ struct rb_root *rbtree, u64 start_pfn, size_t nr_pages) ++{ ++ struct rb_node *rbnode; ++ struct kbase_va_region *reg; ++ u64 end_pfn = start_pfn + nr_pages; ++ ++ rbnode = rbtree->rb_node; ++ ++ while (rbnode) { ++ u64 tmp_start_pfn, tmp_end_pfn; ++ ++ reg = rb_entry(rbnode, struct kbase_va_region, rblink); ++ tmp_start_pfn = reg->start_pfn; ++ tmp_end_pfn = reg->start_pfn + reg->nr_pages; ++ ++ /* If start is lower than this, go left. */ ++ if (start_pfn < tmp_start_pfn) ++ rbnode = rbnode->rb_left; ++ /* If end is higher than this, then go right. */ ++ else if (end_pfn > tmp_end_pfn) ++ rbnode = rbnode->rb_right; ++ else /* Enclosing */ ++ return reg; ++ } ++ ++ return NULL; ++} ++ ++struct kbase_va_region *kbase_find_region_enclosing_address( ++ struct rb_root *rbtree, u64 gpu_addr) ++{ ++ u64 gpu_pfn = gpu_addr >> PAGE_SHIFT; ++ struct rb_node *rbnode; ++ struct kbase_va_region *reg; ++ ++ rbnode = rbtree->rb_node; ++ ++ while (rbnode) { ++ u64 tmp_start_pfn, tmp_end_pfn; ++ ++ reg = rb_entry(rbnode, struct kbase_va_region, rblink); ++ tmp_start_pfn = reg->start_pfn; ++ tmp_end_pfn = reg->start_pfn + reg->nr_pages; ++ ++ /* If start is lower than this, go left. */ ++ if (gpu_pfn < tmp_start_pfn) ++ rbnode = rbnode->rb_left; ++ /* If end is higher than this, then go right. */ ++ else if (gpu_pfn >= tmp_end_pfn) ++ rbnode = rbnode->rb_right; ++ else /* Enclosing */ ++ return reg; ++ } ++ ++ return NULL; ++} ++ ++/* Find region enclosing given address. */ ++struct kbase_va_region *kbase_region_tracker_find_region_enclosing_address( ++ struct kbase_context *kctx, u64 gpu_addr) ++{ ++ u64 gpu_pfn = gpu_addr >> PAGE_SHIFT; ++ struct rb_root *rbtree = NULL; ++ ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ rbtree = kbase_gpu_va_to_rbtree(kctx, gpu_pfn); ++ ++ return kbase_find_region_enclosing_address(rbtree, gpu_addr); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_region_tracker_find_region_enclosing_address); ++ ++struct kbase_va_region *kbase_find_region_base_address( ++ struct rb_root *rbtree, u64 gpu_addr) ++{ ++ u64 gpu_pfn = gpu_addr >> PAGE_SHIFT; ++ struct rb_node *rbnode = NULL; ++ struct kbase_va_region *reg = NULL; ++ ++ rbnode = rbtree->rb_node; ++ ++ while (rbnode) { ++ reg = rb_entry(rbnode, struct kbase_va_region, rblink); ++ if (reg->start_pfn > gpu_pfn) ++ rbnode = rbnode->rb_left; ++ else if (reg->start_pfn < gpu_pfn) ++ rbnode = rbnode->rb_right; ++ else ++ return reg; ++ } ++ ++ return NULL; ++} ++ ++/* Find region with given base address */ ++struct kbase_va_region *kbase_region_tracker_find_region_base_address( ++ struct kbase_context *kctx, u64 gpu_addr) ++{ ++ u64 gpu_pfn = gpu_addr >> PAGE_SHIFT; ++ struct rb_root *rbtree = NULL; ++ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ rbtree = kbase_gpu_va_to_rbtree(kctx, gpu_pfn); ++ ++ return kbase_find_region_base_address(rbtree, gpu_addr); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_region_tracker_find_region_base_address); ++ ++/* Find region meeting given requirements */ ++static struct kbase_va_region *kbase_region_tracker_find_region_meeting_reqs( ++ struct kbase_va_region *reg_reqs, ++ size_t nr_pages, size_t align_offset, size_t align_mask, ++ u64 *out_start_pfn) ++{ ++ struct rb_node *rbnode = NULL; ++ struct kbase_va_region *reg = NULL; ++ struct rb_root *rbtree = NULL; ++ ++ /* Note that this search is a linear search, as we do not have a target ++ address in mind, so does not benefit from the rbtree search */ ++ rbtree = reg_reqs->rbtree; ++ ++ for (rbnode = rb_first(rbtree); rbnode; rbnode = rb_next(rbnode)) { ++ reg = rb_entry(rbnode, struct kbase_va_region, rblink); ++ if ((reg->nr_pages >= nr_pages) && ++ (reg->flags & KBASE_REG_FREE)) { ++ /* Check alignment */ ++ u64 start_pfn = reg->start_pfn; ++ ++ /* When align_offset == align, this sequence is ++ * equivalent to: ++ * (start_pfn + align_mask) & ~(align_mask) ++ * ++ * Otherwise, it aligns to n*align + offset, for the ++ * lowest value n that makes this still >start_pfn */ ++ start_pfn += align_mask; ++ start_pfn -= (start_pfn - align_offset) & (align_mask); ++ ++ if (!(reg_reqs->flags & KBASE_REG_GPU_NX)) { ++ /* Can't end at 4GB boundary */ ++ if (0 == ((start_pfn + nr_pages) & BASE_MEM_PFN_MASK_4GB)) ++ start_pfn += align_offset; ++ ++ /* Can't start at 4GB boundary */ ++ if (0 == (start_pfn & BASE_MEM_PFN_MASK_4GB)) ++ start_pfn += align_offset; ++ ++ if (!((start_pfn + nr_pages) & BASE_MEM_PFN_MASK_4GB) || ++ !(start_pfn & BASE_MEM_PFN_MASK_4GB)) ++ continue; ++ } else if (reg_reqs->flags & ++ KBASE_REG_GPU_VA_SAME_4GB_PAGE) { ++ u64 end_pfn = start_pfn + nr_pages - 1; ++ ++ if ((start_pfn & ~BASE_MEM_PFN_MASK_4GB) != ++ (end_pfn & ~BASE_MEM_PFN_MASK_4GB)) ++ start_pfn = end_pfn & ~BASE_MEM_PFN_MASK_4GB; ++ } ++ ++ if ((start_pfn >= reg->start_pfn) && ++ (start_pfn <= (reg->start_pfn + reg->nr_pages - 1)) && ++ ((start_pfn + nr_pages - 1) <= (reg->start_pfn + reg->nr_pages - 1))) { ++ *out_start_pfn = start_pfn; ++ return reg; ++ } ++ } ++ } ++ ++ return NULL; ++} ++ ++/** ++ * @brief Remove a region object from the global list. ++ * ++ * The region reg is removed, possibly by merging with other free and ++ * compatible adjacent regions. It must be called with the context ++ * region lock held. The associated memory is not released (see ++ * kbase_free_alloced_region). Internal use only. ++ */ ++int kbase_remove_va_region(struct kbase_va_region *reg) ++{ ++ struct rb_node *rbprev; ++ struct kbase_va_region *prev = NULL; ++ struct rb_node *rbnext; ++ struct kbase_va_region *next = NULL; ++ struct rb_root *reg_rbtree = NULL; ++ ++ int merged_front = 0; ++ int merged_back = 0; ++ int err = 0; ++ ++ reg_rbtree = reg->rbtree; ++ ++ /* Try to merge with the previous block first */ ++ rbprev = rb_prev(&(reg->rblink)); ++ if (rbprev) { ++ prev = rb_entry(rbprev, struct kbase_va_region, rblink); ++ if (prev->flags & KBASE_REG_FREE) { ++ /* We're compatible with the previous VMA, ++ * merge with it */ ++ WARN_ON((prev->flags & KBASE_REG_ZONE_MASK) != ++ (reg->flags & KBASE_REG_ZONE_MASK)); ++ prev->nr_pages += reg->nr_pages; ++ rb_erase(&(reg->rblink), reg_rbtree); ++ reg = prev; ++ merged_front = 1; ++ } ++ } ++ ++ /* Try to merge with the next block second */ ++ /* Note we do the lookup here as the tree may have been rebalanced. */ ++ rbnext = rb_next(&(reg->rblink)); ++ if (rbnext) { ++ /* We're compatible with the next VMA, merge with it */ ++ next = rb_entry(rbnext, struct kbase_va_region, rblink); ++ if (next->flags & KBASE_REG_FREE) { ++ WARN_ON((next->flags & KBASE_REG_ZONE_MASK) != ++ (reg->flags & KBASE_REG_ZONE_MASK)); ++ next->start_pfn = reg->start_pfn; ++ next->nr_pages += reg->nr_pages; ++ rb_erase(&(reg->rblink), reg_rbtree); ++ merged_back = 1; ++ if (merged_front) { ++ /* We already merged with prev, free it */ ++ kfree(reg); ++ } ++ } ++ } ++ ++ /* If we failed to merge then we need to add a new block */ ++ if (!(merged_front || merged_back)) { ++ /* ++ * We didn't merge anything. Add a new free ++ * placeholder and remove the original one. ++ */ ++ struct kbase_va_region *free_reg; ++ ++ free_reg = kbase_alloc_free_region(reg_rbtree, ++ reg->start_pfn, reg->nr_pages, ++ reg->flags & KBASE_REG_ZONE_MASK); ++ if (!free_reg) { ++ err = -ENOMEM; ++ goto out; ++ } ++ rb_replace_node(&(reg->rblink), &(free_reg->rblink), reg_rbtree); ++ } ++ ++ out: ++ return err; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_remove_va_region); ++ ++/** ++ * kbase_insert_va_region_nolock - Insert a VA region to the list, ++ * replacing the existing one. ++ * ++ * @new_reg: The new region to insert ++ * @at_reg: The region to replace ++ * @start_pfn: The Page Frame Number to insert at ++ * @nr_pages: The number of pages of the region ++ */ ++static int kbase_insert_va_region_nolock(struct kbase_va_region *new_reg, ++ struct kbase_va_region *at_reg, u64 start_pfn, size_t nr_pages) ++{ ++ struct rb_root *reg_rbtree = NULL; ++ int err = 0; ++ ++ reg_rbtree = at_reg->rbtree; ++ ++ /* Must be a free region */ ++ KBASE_DEBUG_ASSERT((at_reg->flags & KBASE_REG_FREE) != 0); ++ /* start_pfn should be contained within at_reg */ ++ KBASE_DEBUG_ASSERT((start_pfn >= at_reg->start_pfn) && (start_pfn < at_reg->start_pfn + at_reg->nr_pages)); ++ /* at least nr_pages from start_pfn should be contained within at_reg */ ++ KBASE_DEBUG_ASSERT(start_pfn + nr_pages <= at_reg->start_pfn + at_reg->nr_pages); ++ ++ new_reg->start_pfn = start_pfn; ++ new_reg->nr_pages = nr_pages; ++ ++ /* Regions are a whole use, so swap and delete old one. */ ++ if (at_reg->start_pfn == start_pfn && at_reg->nr_pages == nr_pages) { ++ rb_replace_node(&(at_reg->rblink), &(new_reg->rblink), ++ reg_rbtree); ++ kfree(at_reg); ++ } ++ /* New region replaces the start of the old one, so insert before. */ ++ else if (at_reg->start_pfn == start_pfn) { ++ at_reg->start_pfn += nr_pages; ++ KBASE_DEBUG_ASSERT(at_reg->nr_pages >= nr_pages); ++ at_reg->nr_pages -= nr_pages; ++ ++ kbase_region_tracker_insert(new_reg); ++ } ++ /* New region replaces the end of the old one, so insert after. */ ++ else if ((at_reg->start_pfn + at_reg->nr_pages) == (start_pfn + nr_pages)) { ++ at_reg->nr_pages -= nr_pages; ++ ++ kbase_region_tracker_insert(new_reg); ++ } ++ /* New region splits the old one, so insert and create new */ ++ else { ++ struct kbase_va_region *new_front_reg; ++ ++ new_front_reg = kbase_alloc_free_region(reg_rbtree, ++ at_reg->start_pfn, ++ start_pfn - at_reg->start_pfn, ++ at_reg->flags & KBASE_REG_ZONE_MASK); ++ ++ if (new_front_reg) { ++ at_reg->nr_pages -= nr_pages + new_front_reg->nr_pages; ++ at_reg->start_pfn = start_pfn + nr_pages; ++ ++ kbase_region_tracker_insert(new_front_reg); ++ kbase_region_tracker_insert(new_reg); ++ } else { ++ err = -ENOMEM; ++ } ++ } ++ ++ return err; ++} ++ ++/** ++ * kbase_add_va_region - Add a VA region to the region list for a context. ++ * ++ * @kctx: kbase context containing the region ++ * @reg: the region to add ++ * @addr: the address to insert the region at ++ * @nr_pages: the number of pages in the region ++ * @align: the minimum alignment in pages ++ */ ++int kbase_add_va_region(struct kbase_context *kctx, ++ struct kbase_va_region *reg, u64 addr, ++ size_t nr_pages, size_t align) ++{ ++ int err = 0; ++ struct kbase_device *kbdev = kctx->kbdev; ++ int cpu_va_bits = kbase_get_num_cpu_va_bits(kctx); ++ int gpu_pc_bits = ++ kbdev->gpu_props.props.core_props.log2_program_counter_size; ++ ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ KBASE_DEBUG_ASSERT(NULL != reg); ++ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ /* The executable allocation from the SAME_VA zone would already have an ++ * appropriately aligned GPU VA chosen for it. ++ * Also the executable allocation from EXEC_VA zone doesn't need the ++ * special alignment. ++ */ ++ if (!(reg->flags & KBASE_REG_GPU_NX) && !addr && ++ ((reg->flags & KBASE_REG_ZONE_MASK) != KBASE_REG_ZONE_EXEC_VA)) { ++ if (cpu_va_bits > gpu_pc_bits) { ++ align = max(align, (size_t)((1ULL << gpu_pc_bits) ++ >> PAGE_SHIFT)); ++ } ++ } ++ ++ do { ++ err = kbase_add_va_region_rbtree(kbdev, reg, addr, nr_pages, ++ align); ++ if (err != -ENOMEM) ++ break; ++ ++ /* ++ * If the allocation is not from the same zone as JIT ++ * then don't retry, we're out of VA and there is ++ * nothing which can be done about it. ++ */ ++ if ((reg->flags & KBASE_REG_ZONE_MASK) != ++ KBASE_REG_ZONE_CUSTOM_VA) ++ break; ++ } while (kbase_jit_evict(kctx)); ++ ++ return err; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_add_va_region); ++ ++/** ++ * kbase_add_va_region_rbtree - Insert a region into its corresponding rbtree ++ * ++ * Insert a region into the rbtree that was specified when the region was ++ * created. If addr is 0 a free area in the rbtree is used, otherwise the ++ * specified address is used. ++ * ++ * @kbdev: The kbase device ++ * @reg: The region to add ++ * @addr: The address to add the region at, or 0 to map at any available address ++ * @nr_pages: The size of the region in pages ++ * @align: The minimum alignment in pages ++ */ ++int kbase_add_va_region_rbtree(struct kbase_device *kbdev, ++ struct kbase_va_region *reg, ++ u64 addr, size_t nr_pages, size_t align) ++{ ++ struct device *const dev = kbdev->dev; ++ struct rb_root *rbtree = NULL; ++ struct kbase_va_region *tmp; ++ u64 gpu_pfn = addr >> PAGE_SHIFT; ++ int err = 0; ++ ++ rbtree = reg->rbtree; ++ ++ if (!align) ++ align = 1; ++ ++ /* must be a power of 2 */ ++ KBASE_DEBUG_ASSERT(is_power_of_2(align)); ++ KBASE_DEBUG_ASSERT(nr_pages > 0); ++ ++ /* Path 1: Map a specific address. Find the enclosing region, ++ * which *must* be free. ++ */ ++ if (gpu_pfn) { ++ KBASE_DEBUG_ASSERT(!(gpu_pfn & (align - 1))); ++ ++ tmp = find_region_enclosing_range_rbtree(rbtree, gpu_pfn, ++ nr_pages); ++ if (kbase_is_region_invalid(tmp)) { ++ dev_warn(dev, "Enclosing region not found or invalid: 0x%08llx gpu_pfn, %zu nr_pages", gpu_pfn, nr_pages); ++ err = -ENOMEM; ++ goto exit; ++ } else if (!kbase_is_region_free(tmp)) { ++ dev_warn(dev, "!(tmp->flags & KBASE_REG_FREE): tmp->start_pfn=0x%llx tmp->flags=0x%lx tmp->nr_pages=0x%zx gpu_pfn=0x%llx nr_pages=0x%zx\n", ++ tmp->start_pfn, tmp->flags, ++ tmp->nr_pages, gpu_pfn, nr_pages); ++ err = -ENOMEM; ++ goto exit; ++ } ++ ++ err = kbase_insert_va_region_nolock(reg, tmp, gpu_pfn, ++ nr_pages); ++ if (err) { ++ dev_warn(dev, "Failed to insert va region"); ++ err = -ENOMEM; ++ } ++ } else { ++ /* Path 2: Map any free address which meets the requirements. */ ++ u64 start_pfn; ++ size_t align_offset = align; ++ size_t align_mask = align - 1; ++ ++#if !MALI_USE_CSF ++ if ((reg->flags & KBASE_REG_TILER_ALIGN_TOP)) { ++ WARN(align > 1, "%s with align %lx might not be honored for KBASE_REG_TILER_ALIGN_TOP memory", ++ __func__, ++ (unsigned long)align); ++ align_mask = reg->extent - 1; ++ align_offset = reg->extent - reg->initial_commit; ++ } ++#endif /* !MALI_USE_CSF */ ++ ++ tmp = kbase_region_tracker_find_region_meeting_reqs(reg, ++ nr_pages, align_offset, align_mask, ++ &start_pfn); ++ if (tmp) { ++ err = kbase_insert_va_region_nolock(reg, tmp, ++ start_pfn, nr_pages); ++ if (unlikely(err)) { ++ dev_warn(dev, "Failed to insert region: 0x%08llx start_pfn, %zu nr_pages", ++ start_pfn, nr_pages); ++ } ++ } else { ++ dev_dbg(dev, "Failed to find a suitable region: %zu nr_pages, %zu align_offset, %zu align_mask\n", ++ nr_pages, align_offset, align_mask); ++ err = -ENOMEM; ++ } ++ } ++ ++exit: ++ return err; ++} ++ ++/** ++ * @brief Initialize the internal region tracker data structure. ++ */ ++static void kbase_region_tracker_ds_init(struct kbase_context *kctx, ++ struct kbase_va_region *same_va_reg, ++ struct kbase_va_region *custom_va_reg) ++{ ++ kctx->reg_rbtree_same = RB_ROOT; ++ kbase_region_tracker_insert(same_va_reg); ++ ++ /* Although custom_va_reg and exec_va_reg don't always exist, ++ * initialize unconditionally because of the mem_view debugfs ++ * implementation which relies on them being empty. ++ * ++ * The difference between the two is that the EXEC_VA region ++ * is never initialized at this stage. ++ */ ++ kctx->reg_rbtree_custom = RB_ROOT; ++ kctx->reg_rbtree_exec = RB_ROOT; ++ ++ if (custom_va_reg) ++ kbase_region_tracker_insert(custom_va_reg); ++} ++ ++static void kbase_region_tracker_erase_rbtree(struct rb_root *rbtree) ++{ ++ struct rb_node *rbnode; ++ struct kbase_va_region *reg; ++ ++ do { ++ rbnode = rb_first(rbtree); ++ if (rbnode) { ++ rb_erase(rbnode, rbtree); ++ reg = rb_entry(rbnode, struct kbase_va_region, rblink); ++ WARN_ON(reg->va_refcnt != 1); ++ /* Reset the start_pfn - as the rbtree is being ++ * destroyed and we've already erased this region, there ++ * is no further need to attempt to remove it. ++ * This won't affect the cleanup if the region was ++ * being used as a sticky resource as the cleanup ++ * related to sticky resources anyways need to be ++ * performed before the term of region tracker. ++ */ ++ reg->start_pfn = 0; ++ kbase_free_alloced_region(reg); ++ } ++ } while (rbnode); ++} ++ ++void kbase_region_tracker_term(struct kbase_context *kctx) ++{ ++ kbase_gpu_vm_lock(kctx); ++ kbase_region_tracker_erase_rbtree(&kctx->reg_rbtree_same); ++ kbase_region_tracker_erase_rbtree(&kctx->reg_rbtree_custom); ++ kbase_region_tracker_erase_rbtree(&kctx->reg_rbtree_exec); ++#if MALI_USE_CSF ++ WARN_ON(!list_empty(&kctx->csf.event_pages_head)); ++#endif ++ kbase_gpu_vm_unlock(kctx); ++} ++ ++void kbase_region_tracker_term_rbtree(struct rb_root *rbtree) ++{ ++ kbase_region_tracker_erase_rbtree(rbtree); ++} ++ ++static size_t kbase_get_same_va_bits(struct kbase_context *kctx) ++{ ++ return min(kbase_get_num_cpu_va_bits(kctx), ++ (size_t) kctx->kbdev->gpu_props.mmu.va_bits); ++} ++ ++int kbase_region_tracker_init(struct kbase_context *kctx) ++{ ++ struct kbase_va_region *same_va_reg; ++ struct kbase_va_region *custom_va_reg = NULL; ++ size_t same_va_bits = kbase_get_same_va_bits(kctx); ++ u64 custom_va_size = KBASE_REG_ZONE_CUSTOM_VA_SIZE; ++ u64 gpu_va_limit = (1ULL << kctx->kbdev->gpu_props.mmu.va_bits) >> PAGE_SHIFT; ++ u64 same_va_pages; ++ int err; ++ ++ /* Take the lock as kbase_free_alloced_region requires it */ ++ kbase_gpu_vm_lock(kctx); ++ ++ same_va_pages = (1ULL << (same_va_bits - PAGE_SHIFT)) - 1; ++ /* all have SAME_VA */ ++ same_va_reg = kbase_alloc_free_region(&kctx->reg_rbtree_same, 1, ++ same_va_pages, ++ KBASE_REG_ZONE_SAME_VA); ++ ++ if (!same_va_reg) { ++ err = -ENOMEM; ++ goto fail_unlock; ++ } ++ ++#ifdef CONFIG_64BIT ++ /* 32-bit clients have custom VA zones */ ++ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) { ++#endif ++ if (gpu_va_limit <= KBASE_REG_ZONE_CUSTOM_VA_BASE) { ++ err = -EINVAL; ++ goto fail_free_same_va; ++ } ++ /* If the current size of TMEM is out of range of the ++ * virtual address space addressable by the MMU then ++ * we should shrink it to fit ++ */ ++ if ((KBASE_REG_ZONE_CUSTOM_VA_BASE + KBASE_REG_ZONE_CUSTOM_VA_SIZE) >= gpu_va_limit) ++ custom_va_size = gpu_va_limit - KBASE_REG_ZONE_CUSTOM_VA_BASE; ++ ++ custom_va_reg = kbase_alloc_free_region( ++ &kctx->reg_rbtree_custom, ++ KBASE_REG_ZONE_CUSTOM_VA_BASE, ++ custom_va_size, KBASE_REG_ZONE_CUSTOM_VA); ++ ++ if (!custom_va_reg) { ++ err = -ENOMEM; ++ goto fail_free_same_va; ++ } ++#ifdef CONFIG_64BIT ++ } else { ++ custom_va_size = 0; ++ } ++#endif ++ ++ kbase_region_tracker_ds_init(kctx, same_va_reg, custom_va_reg); ++ ++ kctx->same_va_end = same_va_pages + 1; ++ kctx->gpu_va_end = kctx->same_va_end + custom_va_size; ++ kctx->exec_va_start = U64_MAX; ++ kctx->jit_va = false; ++ ++#if MALI_USE_CSF ++ INIT_LIST_HEAD(&kctx->csf.event_pages_head); ++#endif ++ ++ kbase_gpu_vm_unlock(kctx); ++ return 0; ++ ++fail_free_same_va: ++ kbase_free_alloced_region(same_va_reg); ++fail_unlock: ++ kbase_gpu_vm_unlock(kctx); ++ return err; ++} ++ ++#ifdef CONFIG_64BIT ++static int kbase_region_tracker_init_jit_64(struct kbase_context *kctx, ++ u64 jit_va_pages) ++{ ++ struct kbase_va_region *same_va; ++ struct kbase_va_region *custom_va_reg; ++ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ /* First verify that a JIT_VA zone has not been created already. */ ++ if (kctx->jit_va) ++ return -EINVAL; ++ ++ /* ++ * Modify the same VA free region after creation. Be careful to ensure ++ * that allocations haven't been made as they could cause an overlap ++ * to happen with existing same VA allocations and the custom VA zone. ++ */ ++ same_va = kbase_region_tracker_find_region_base_address(kctx, ++ PAGE_SIZE); ++ if (!same_va) ++ return -ENOMEM; ++ ++ if (same_va->nr_pages < jit_va_pages || kctx->same_va_end < jit_va_pages) ++ return -ENOMEM; ++ ++ /* It's safe to adjust the same VA zone now */ ++ same_va->nr_pages -= jit_va_pages; ++ kctx->same_va_end -= jit_va_pages; ++ ++ /* ++ * Create a custom VA zone at the end of the VA for allocations which ++ * JIT can use so it doesn't have to allocate VA from the kernel. ++ */ ++ custom_va_reg = kbase_alloc_free_region(&kctx->reg_rbtree_custom, ++ kctx->same_va_end, ++ jit_va_pages, ++ KBASE_REG_ZONE_CUSTOM_VA); ++ ++ /* ++ * The context will be destroyed if we fail here so no point ++ * reverting the change we made to same_va. ++ */ ++ if (!custom_va_reg) ++ return -ENOMEM; ++ ++ kbase_region_tracker_insert(custom_va_reg); ++ return 0; ++} ++#endif ++ ++int kbase_region_tracker_init_jit(struct kbase_context *kctx, u64 jit_va_pages, ++ int max_allocations, int trim_level, int group_id, ++ u64 phys_pages_limit) ++{ ++ int err = 0; ++ ++ if (trim_level < 0 || trim_level > BASE_JIT_MAX_TRIM_LEVEL) ++ return -EINVAL; ++ ++ if (group_id < 0 || group_id >= MEMORY_GROUP_MANAGER_NR_GROUPS) ++ return -EINVAL; ++ ++ if (phys_pages_limit > jit_va_pages) ++ return -EINVAL; ++ ++#if MALI_JIT_PRESSURE_LIMIT_BASE ++ if (phys_pages_limit != jit_va_pages) ++ kbase_ctx_flag_set(kctx, KCTX_JPL_ENABLED); ++#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ ++ ++ kbase_gpu_vm_lock(kctx); ++ ++#ifdef CONFIG_64BIT ++ if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) ++ err = kbase_region_tracker_init_jit_64(kctx, jit_va_pages); ++#endif ++ /* ++ * Nothing to do for 32-bit clients, JIT uses the existing ++ * custom VA zone. ++ */ ++ ++ if (!err) { ++ kctx->jit_max_allocations = max_allocations; ++ kctx->trim_level = trim_level; ++ kctx->jit_va = true; ++ kctx->jit_group_id = group_id; ++#if MALI_JIT_PRESSURE_LIMIT_BASE ++ kctx->jit_phys_pages_limit = phys_pages_limit; ++ dev_dbg(kctx->kbdev->dev, "phys_pages_limit set to %llu\n", ++ phys_pages_limit); ++#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ ++ } ++ ++ kbase_gpu_vm_unlock(kctx); ++ ++ return err; ++} ++ ++int kbase_region_tracker_init_exec(struct kbase_context *kctx, u64 exec_va_pages) ++{ ++ struct kbase_va_region *shrinking_va_reg; ++ struct kbase_va_region *exec_va_reg; ++ u64 exec_va_start, exec_va_base_addr; ++ int err; ++ ++ /* The EXEC_VA zone shall be created by making space at the end of the ++ * address space. Firstly, verify that the number of EXEC_VA pages ++ * requested by the client is reasonable and then make sure that it is ++ * not greater than the address space itself before calculating the base ++ * address of the new zone. ++ */ ++ if (exec_va_pages == 0 || exec_va_pages > KBASE_REG_ZONE_EXEC_VA_MAX_PAGES) ++ return -EINVAL; ++ ++ kbase_gpu_vm_lock(kctx); ++ ++ /* First verify that a JIT_VA zone has not been created already. */ ++ if (kctx->jit_va) { ++ err = -EPERM; ++ goto exit_unlock; ++ } ++ ++ if (exec_va_pages > kctx->gpu_va_end) { ++ err = -ENOMEM; ++ goto exit_unlock; ++ } ++ ++ exec_va_start = kctx->gpu_va_end - exec_va_pages; ++ exec_va_base_addr = exec_va_start << PAGE_SHIFT; ++ ++ shrinking_va_reg = kbase_region_tracker_find_region_enclosing_address(kctx, ++ exec_va_base_addr); ++ if (!shrinking_va_reg) { ++ err = -ENOMEM; ++ goto exit_unlock; ++ } ++ ++ /* Make sure that the EXEC_VA region is still uninitialized */ ++ if ((shrinking_va_reg->flags & KBASE_REG_ZONE_MASK) == ++ KBASE_REG_ZONE_EXEC_VA) { ++ err = -EPERM; ++ goto exit_unlock; ++ } ++ ++ if (shrinking_va_reg->nr_pages <= exec_va_pages) { ++ err = -ENOMEM; ++ goto exit_unlock; ++ } ++ ++ exec_va_reg = kbase_alloc_free_region(&kctx->reg_rbtree_exec, ++ exec_va_start, ++ exec_va_pages, ++ KBASE_REG_ZONE_EXEC_VA); ++ if (!exec_va_reg) { ++ err = -ENOMEM; ++ goto exit_unlock; ++ } ++ ++ shrinking_va_reg->nr_pages -= exec_va_pages; ++#ifdef CONFIG_64BIT ++ if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) ++ kctx->same_va_end -= exec_va_pages; ++#endif ++ kctx->exec_va_start = exec_va_start; ++ ++ kbase_region_tracker_insert(exec_va_reg); ++ err = 0; ++ ++exit_unlock: ++ kbase_gpu_vm_unlock(kctx); ++ return err; ++} ++ ++#if MALI_USE_CSF ++void kbase_mcu_shared_interface_region_tracker_term(struct kbase_device *kbdev) ++{ ++ kbase_region_tracker_term_rbtree(&kbdev->csf.shared_reg_rbtree); ++} ++ ++int kbase_mcu_shared_interface_region_tracker_init(struct kbase_device *kbdev) ++{ ++ struct kbase_va_region *shared_reg; ++ u64 shared_reg_start_pfn; ++ u64 shared_reg_size; ++ ++ shared_reg_start_pfn = KBASE_REG_ZONE_MCU_SHARED_BASE; ++ shared_reg_size = KBASE_REG_ZONE_MCU_SHARED_SIZE; ++ ++ kbdev->csf.shared_reg_rbtree = RB_ROOT; ++ ++ shared_reg = kbase_alloc_free_region(&kbdev->csf.shared_reg_rbtree, ++ shared_reg_start_pfn, ++ shared_reg_size, ++ KBASE_REG_ZONE_MCU_SHARED); ++ if (!shared_reg) ++ return -ENOMEM; ++ ++ kbase_region_tracker_insert(shared_reg); ++ return 0; ++} ++#endif ++ ++int kbase_mem_init(struct kbase_device *kbdev) ++{ ++ int err = 0; ++ struct kbasep_mem_device *memdev; ++#ifdef CONFIG_OF ++ struct device_node *mgm_node = NULL; ++#endif ++ ++ KBASE_DEBUG_ASSERT(kbdev); ++ ++ memdev = &kbdev->memdev; ++ ++ kbase_mem_pool_group_config_set_max_size(&kbdev->mem_pool_defaults, ++ KBASE_MEM_POOL_MAX_SIZE_KCTX); ++ ++ /* Initialize memory usage */ ++ atomic_set(&memdev->used_pages, 0); ++ ++ spin_lock_init(&kbdev->gpu_mem_usage_lock); ++ kbdev->total_gpu_pages = 0; ++ kbdev->process_root = RB_ROOT; ++ kbdev->dma_buf_root = RB_ROOT; ++ mutex_init(&kbdev->dma_buf_lock); ++ ++#ifdef IR_THRESHOLD ++ atomic_set(&memdev->ir_threshold, IR_THRESHOLD); ++#else ++ atomic_set(&memdev->ir_threshold, DEFAULT_IR_THRESHOLD); ++#endif ++ ++ kbdev->mgm_dev = &kbase_native_mgm_dev; ++ ++#ifdef CONFIG_OF ++ /* Check to see whether or not a platform-specific memory group manager ++ * is configured and available. ++ */ ++ mgm_node = of_parse_phandle(kbdev->dev->of_node, ++ "physical-memory-group-manager", 0); ++ if (!mgm_node) { ++ dev_info(kbdev->dev, ++ "No memory group manager is configured\n"); ++ } else { ++ struct platform_device *const pdev = ++ of_find_device_by_node(mgm_node); ++ ++ if (!pdev) { ++ dev_err(kbdev->dev, ++ "The configured memory group manager was not found\n"); ++ } else { ++ kbdev->mgm_dev = platform_get_drvdata(pdev); ++ if (!kbdev->mgm_dev) { ++ dev_info(kbdev->dev, ++ "Memory group manager is not ready\n"); ++ err = -EPROBE_DEFER; ++ } else if (!try_module_get(kbdev->mgm_dev->owner)) { ++ dev_err(kbdev->dev, ++ "Failed to get memory group manger module\n"); ++ err = -ENODEV; ++ kbdev->mgm_dev = NULL; ++ } else { ++ dev_info(kbdev->dev, ++ "Memory group manager successfully loaded\n"); ++ } ++ } ++ of_node_put(mgm_node); ++ } ++#endif ++ ++ if (likely(!err)) { ++ struct kbase_mem_pool_group_config mem_pool_defaults; ++ ++ kbase_mem_pool_group_config_set_max_size(&mem_pool_defaults, ++ KBASE_MEM_POOL_MAX_SIZE_KBDEV); ++ ++ err = kbase_mem_pool_group_init(&kbdev->mem_pools, kbdev, ++ &mem_pool_defaults, NULL); ++ } ++ ++ return err; ++} ++ ++void kbase_mem_halt(struct kbase_device *kbdev) ++{ ++ CSTD_UNUSED(kbdev); ++} ++ ++void kbase_mem_term(struct kbase_device *kbdev) ++{ ++ struct kbasep_mem_device *memdev; ++ int pages; ++ ++ KBASE_DEBUG_ASSERT(kbdev); ++ ++ memdev = &kbdev->memdev; ++ ++ pages = atomic_read(&memdev->used_pages); ++ if (pages != 0) ++ dev_warn(kbdev->dev, "%s: %d pages in use!\n", __func__, pages); ++ ++ kbase_mem_pool_group_term(&kbdev->mem_pools); ++ ++ WARN_ON(kbdev->total_gpu_pages); ++ WARN_ON(!RB_EMPTY_ROOT(&kbdev->process_root)); ++ WARN_ON(!RB_EMPTY_ROOT(&kbdev->dma_buf_root)); ++ mutex_destroy(&kbdev->dma_buf_lock); ++ ++ if (kbdev->mgm_dev) ++ module_put(kbdev->mgm_dev->owner); ++} ++KBASE_EXPORT_TEST_API(kbase_mem_term); ++ ++/** ++ * @brief Allocate a free region object. ++ * ++ * The allocated object is not part of any list yet, and is flagged as ++ * KBASE_REG_FREE. No mapping is allocated yet. ++ * ++ * zone is KBASE_REG_ZONE_CUSTOM_VA or KBASE_REG_ZONE_SAME_VA. ++ * ++ */ ++struct kbase_va_region *kbase_alloc_free_region(struct rb_root *rbtree, ++ u64 start_pfn, size_t nr_pages, int zone) ++{ ++ struct kbase_va_region *new_reg; ++ ++ KBASE_DEBUG_ASSERT(rbtree != NULL); ++ ++ /* zone argument should only contain zone related region flags */ ++ KBASE_DEBUG_ASSERT((zone & ~KBASE_REG_ZONE_MASK) == 0); ++ KBASE_DEBUG_ASSERT(nr_pages > 0); ++ /* 64-bit address range is the max */ ++ KBASE_DEBUG_ASSERT(start_pfn + nr_pages <= (U64_MAX / PAGE_SIZE)); ++ ++ new_reg = kzalloc(sizeof(*new_reg), GFP_KERNEL); ++ ++ if (!new_reg) ++ return NULL; ++ ++ new_reg->va_refcnt = 1; ++ new_reg->cpu_alloc = NULL; /* no alloc bound yet */ ++ new_reg->gpu_alloc = NULL; /* no alloc bound yet */ ++ new_reg->rbtree = rbtree; ++ new_reg->flags = zone | KBASE_REG_FREE; ++ ++ new_reg->flags |= KBASE_REG_GROWABLE; ++ ++ new_reg->start_pfn = start_pfn; ++ new_reg->nr_pages = nr_pages; ++ ++ INIT_LIST_HEAD(&new_reg->jit_node); ++ INIT_LIST_HEAD(&new_reg->link); ++ ++ return new_reg; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_alloc_free_region); ++ ++static struct kbase_context *kbase_reg_flags_to_kctx( ++ struct kbase_va_region *reg) ++{ ++ struct kbase_context *kctx = NULL; ++ struct rb_root *rbtree = reg->rbtree; ++ ++ switch (reg->flags & KBASE_REG_ZONE_MASK) { ++ case KBASE_REG_ZONE_CUSTOM_VA: ++ kctx = container_of(rbtree, struct kbase_context, ++ reg_rbtree_custom); ++ break; ++ case KBASE_REG_ZONE_SAME_VA: ++ kctx = container_of(rbtree, struct kbase_context, ++ reg_rbtree_same); ++ break; ++ case KBASE_REG_ZONE_EXEC_VA: ++ kctx = container_of(rbtree, struct kbase_context, ++ reg_rbtree_exec); ++ break; ++ default: ++ WARN(1, "Unknown zone in region: flags=0x%lx\n", reg->flags); ++ break; ++ } ++ ++ return kctx; ++} ++ ++/** ++ * @brief Free a region object. ++ * ++ * The described region must be freed of any mapping. ++ * ++ * If the region is not flagged as KBASE_REG_FREE, the region's ++ * alloc object will be released. ++ * It is a bug if no alloc object exists for non-free regions. ++ * ++ */ ++void kbase_free_alloced_region(struct kbase_va_region *reg) ++{ ++#if MALI_USE_CSF ++ if ((reg->flags & KBASE_REG_ZONE_MASK) == ++ KBASE_REG_ZONE_MCU_SHARED) { ++ kfree(reg); ++ return; ++ } ++#endif ++ if (!(reg->flags & KBASE_REG_FREE)) { ++ struct kbase_context *kctx = kbase_reg_flags_to_kctx(reg); ++ ++ if (WARN_ON(!kctx)) ++ return; ++ ++ if (WARN_ON(kbase_is_region_invalid(reg))) ++ return; ++ ++ dev_dbg(kctx->kbdev->dev, "Freeing memory region %p\n", ++ (void *)reg); ++#if MALI_USE_CSF ++ if (reg->flags & KBASE_REG_CSF_EVENT) ++ kbase_unlink_event_mem_page(kctx, reg); ++#endif ++ ++ mutex_lock(&kctx->jit_evict_lock); ++ ++ /* ++ * The physical allocation should have been removed from the ++ * eviction list before this function is called. However, in the ++ * case of abnormal process termination or the app leaking the ++ * memory kbase_mem_free_region is not called so it can still be ++ * on the list at termination time of the region tracker. ++ */ ++ if (!list_empty(®->gpu_alloc->evict_node)) { ++ mutex_unlock(&kctx->jit_evict_lock); ++ ++ /* ++ * Unlink the physical allocation before unmaking it ++ * evictable so that the allocation isn't grown back to ++ * its last backed size as we're going to unmap it ++ * anyway. ++ */ ++ reg->cpu_alloc->reg = NULL; ++ if (reg->cpu_alloc != reg->gpu_alloc) ++ reg->gpu_alloc->reg = NULL; ++ ++ /* ++ * If a region has been made evictable then we must ++ * unmake it before trying to free it. ++ * If the memory hasn't been reclaimed it will be ++ * unmapped and freed below, if it has been reclaimed ++ * then the operations below are no-ops. ++ */ ++ if (reg->flags & KBASE_REG_DONT_NEED) { ++ KBASE_DEBUG_ASSERT(reg->cpu_alloc->type == ++ KBASE_MEM_TYPE_NATIVE); ++ kbase_mem_evictable_unmake(reg->gpu_alloc); ++ } ++ } else { ++ mutex_unlock(&kctx->jit_evict_lock); ++ } ++ ++ /* ++ * Remove the region from the sticky resource metadata ++ * list should it be there. ++ */ ++ kbase_sticky_resource_release_force(kctx, NULL, ++ reg->start_pfn << PAGE_SHIFT); ++ ++ kbase_mem_phy_alloc_put(reg->cpu_alloc); ++ kbase_mem_phy_alloc_put(reg->gpu_alloc); ++ ++ reg->flags |= KBASE_REG_VA_FREED; ++ kbase_va_region_alloc_put(kctx, reg); ++ } else { ++ kfree(reg); ++ } ++} ++ ++KBASE_EXPORT_TEST_API(kbase_free_alloced_region); ++ ++int kbase_gpu_mmap(struct kbase_context *kctx, struct kbase_va_region *reg, u64 addr, size_t nr_pages, size_t align) ++{ ++ int err; ++ size_t i = 0; ++ unsigned long attr; ++ unsigned long mask = ~KBASE_REG_MEMATTR_MASK; ++ unsigned long gwt_mask = ~0; ++ int group_id; ++ struct kbase_mem_phy_alloc *alloc; ++ ++#ifdef CONFIG_MALI_CINSTR_GWT ++ if (kctx->gwt_enabled) ++ gwt_mask = ~KBASE_REG_GPU_WR; ++#endif ++ ++ if ((kctx->kbdev->system_coherency == COHERENCY_ACE) && ++ (reg->flags & KBASE_REG_SHARE_BOTH)) ++ attr = KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_OUTER_WA); ++ else ++ attr = KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_WRITE_ALLOC); ++ ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ KBASE_DEBUG_ASSERT(NULL != reg); ++ ++ err = kbase_add_va_region(kctx, reg, addr, nr_pages, align); ++ if (err) ++ return err; ++ ++ alloc = reg->gpu_alloc; ++ group_id = alloc->group_id; ++ ++ if (reg->gpu_alloc->type == KBASE_MEM_TYPE_ALIAS) { ++ u64 const stride = alloc->imported.alias.stride; ++ ++ KBASE_DEBUG_ASSERT(alloc->imported.alias.aliased); ++ for (i = 0; i < alloc->imported.alias.nents; i++) { ++ if (alloc->imported.alias.aliased[i].alloc) { ++ err = kbase_mmu_insert_pages(kctx->kbdev, ++ &kctx->mmu, ++ reg->start_pfn + (i * stride), ++ alloc->imported.alias.aliased[i].alloc->pages + alloc->imported.alias.aliased[i].offset, ++ alloc->imported.alias.aliased[i].length, ++ reg->flags & gwt_mask, ++ kctx->as_nr, ++ group_id); ++ if (err) ++ goto bad_insert; ++ ++ kbase_mem_phy_alloc_gpu_mapped(alloc->imported.alias.aliased[i].alloc); ++ } else { ++ err = kbase_mmu_insert_single_page(kctx, ++ reg->start_pfn + i * stride, ++ kctx->aliasing_sink_page, ++ alloc->imported.alias.aliased[i].length, ++ (reg->flags & mask & gwt_mask) | attr, ++ group_id); ++ ++ if (err) ++ goto bad_insert; ++ } ++ } ++ } else { ++ err = kbase_mmu_insert_pages(kctx->kbdev, ++ &kctx->mmu, ++ reg->start_pfn, ++ kbase_get_gpu_phy_pages(reg), ++ kbase_reg_current_backed_size(reg), ++ reg->flags & gwt_mask, ++ kctx->as_nr, ++ group_id); ++ if (err) ++ goto bad_insert; ++ kbase_mem_phy_alloc_gpu_mapped(alloc); ++ } ++ ++ if (reg->flags & KBASE_REG_IMPORT_PAD && ++ !WARN_ON(reg->nr_pages < reg->gpu_alloc->nents) && ++ reg->gpu_alloc->type == KBASE_MEM_TYPE_IMPORTED_UMM && ++ reg->gpu_alloc->imported.umm.current_mapping_usage_count) { ++ /* For padded imported dma-buf memory, map the dummy aliasing ++ * page from the end of the dma-buf pages, to the end of the ++ * region using a read only mapping. ++ * ++ * Only map when it's imported dma-buf memory that is currently ++ * mapped. ++ * ++ * Assume reg->gpu_alloc->nents is the number of actual pages ++ * in the dma-buf memory. ++ */ ++ err = kbase_mmu_insert_single_page(kctx, ++ reg->start_pfn + reg->gpu_alloc->nents, ++ kctx->aliasing_sink_page, ++ reg->nr_pages - reg->gpu_alloc->nents, ++ (reg->flags | KBASE_REG_GPU_RD) & ++ ~KBASE_REG_GPU_WR, ++ KBASE_MEM_GROUP_SINK); ++ if (err) ++ goto bad_insert; ++ } ++ ++ return err; ++ ++bad_insert: ++ kbase_mmu_teardown_pages(kctx->kbdev, &kctx->mmu, ++ reg->start_pfn, reg->nr_pages, ++ kctx->as_nr); ++ ++ if (alloc->type == KBASE_MEM_TYPE_ALIAS) { ++ KBASE_DEBUG_ASSERT(alloc->imported.alias.aliased); ++ while (i--) ++ if (alloc->imported.alias.aliased[i].alloc) ++ kbase_mem_phy_alloc_gpu_unmapped(alloc->imported.alias.aliased[i].alloc); ++ } ++ ++ kbase_remove_va_region(reg); ++ ++ return err; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_gpu_mmap); ++ ++static void kbase_jd_user_buf_unmap(struct kbase_context *kctx, ++ struct kbase_mem_phy_alloc *alloc, bool writeable); ++ ++int kbase_gpu_munmap(struct kbase_context *kctx, struct kbase_va_region *reg) ++{ ++ int err = 0; ++ size_t i; ++ ++ if (reg->start_pfn == 0) ++ return 0; ++ ++ if (!reg->gpu_alloc) ++ return -EINVAL; ++ ++ /* Tear down down GPU page tables, depending on memory type. */ ++ switch (reg->gpu_alloc->type) { ++ case KBASE_MEM_TYPE_ALIAS: /* Fall-through */ ++ case KBASE_MEM_TYPE_IMPORTED_UMM: ++ err = kbase_mmu_teardown_pages(kctx->kbdev, &kctx->mmu, ++ reg->start_pfn, reg->nr_pages, kctx->as_nr); ++ break; ++ default: ++ err = kbase_mmu_teardown_pages(kctx->kbdev, &kctx->mmu, ++ reg->start_pfn, kbase_reg_current_backed_size(reg), ++ kctx->as_nr); ++ break; ++ } ++ ++ /* Update tracking, and other cleanup, depending on memory type. */ ++ switch (reg->gpu_alloc->type) { ++ case KBASE_MEM_TYPE_ALIAS: ++ KBASE_DEBUG_ASSERT(reg->gpu_alloc->imported.alias.aliased); ++ for (i = 0; i < reg->gpu_alloc->imported.alias.nents; i++) ++ if (reg->gpu_alloc->imported.alias.aliased[i].alloc) ++ kbase_mem_phy_alloc_gpu_unmapped(reg->gpu_alloc->imported.alias.aliased[i].alloc); ++ break; ++ case KBASE_MEM_TYPE_IMPORTED_USER_BUF: { ++ struct kbase_alloc_import_user_buf *user_buf = ++ ®->gpu_alloc->imported.user_buf; ++ ++ if (user_buf->current_mapping_usage_count & PINNED_ON_IMPORT) { ++ user_buf->current_mapping_usage_count &= ++ ~PINNED_ON_IMPORT; ++ ++ /* The allocation could still have active mappings. */ ++ if (user_buf->current_mapping_usage_count == 0) { ++ kbase_jd_user_buf_unmap(kctx, reg->gpu_alloc, ++ (reg->flags & KBASE_REG_GPU_WR)); ++ } ++ } ++ } ++ /* Fall-through */ ++ default: ++ kbase_mem_phy_alloc_gpu_unmapped(reg->gpu_alloc); ++ break; ++ } ++ ++ return err; ++} ++ ++static struct kbase_cpu_mapping *kbasep_find_enclosing_cpu_mapping( ++ struct kbase_context *kctx, ++ unsigned long uaddr, size_t size, u64 *offset) ++{ ++ struct vm_area_struct *vma; ++ struct kbase_cpu_mapping *map; ++ unsigned long vm_pgoff_in_region; ++ unsigned long vm_off_in_region; ++ unsigned long map_start; ++ size_t map_size; ++ ++ lockdep_assert_held(kbase_mem_get_process_mmap_lock()); ++ ++ if ((uintptr_t) uaddr + size < (uintptr_t) uaddr) /* overflow check */ ++ return NULL; ++ ++ vma = find_vma_intersection(current->mm, uaddr, uaddr+size); ++ ++ if (!vma || vma->vm_start > uaddr) ++ return NULL; ++ if (vma->vm_ops != &kbase_vm_ops) ++ /* Not ours! */ ++ return NULL; ++ ++ map = vma->vm_private_data; ++ ++ if (map->kctx != kctx) ++ /* Not from this context! */ ++ return NULL; ++ ++ vm_pgoff_in_region = vma->vm_pgoff - map->region->start_pfn; ++ vm_off_in_region = vm_pgoff_in_region << PAGE_SHIFT; ++ map_start = vma->vm_start - vm_off_in_region; ++ map_size = map->region->nr_pages << PAGE_SHIFT; ++ ++ if ((uaddr + size) > (map_start + map_size)) ++ /* Not within the CPU mapping */ ++ return NULL; ++ ++ *offset = (uaddr - vma->vm_start) + vm_off_in_region; ++ ++ return map; ++} ++ ++int kbasep_find_enclosing_cpu_mapping_offset( ++ struct kbase_context *kctx, ++ unsigned long uaddr, size_t size, u64 *offset) ++{ ++ struct kbase_cpu_mapping *map; ++ ++ kbase_os_mem_map_lock(kctx); ++ ++ map = kbasep_find_enclosing_cpu_mapping(kctx, uaddr, size, offset); ++ ++ kbase_os_mem_map_unlock(kctx); ++ ++ if (!map) ++ return -EINVAL; ++ ++ return 0; ++} ++ ++KBASE_EXPORT_TEST_API(kbasep_find_enclosing_cpu_mapping_offset); ++ ++int kbasep_find_enclosing_gpu_mapping_start_and_offset(struct kbase_context *kctx, ++ u64 gpu_addr, size_t size, u64 *start, u64 *offset) ++{ ++ struct kbase_va_region *region; ++ ++ kbase_gpu_vm_lock(kctx); ++ ++ region = kbase_region_tracker_find_region_enclosing_address(kctx, gpu_addr); ++ ++ if (!region) { ++ kbase_gpu_vm_unlock(kctx); ++ return -EINVAL; ++ } ++ ++ *start = region->start_pfn << PAGE_SHIFT; ++ ++ *offset = gpu_addr - *start; ++ ++ if (((region->start_pfn + region->nr_pages) << PAGE_SHIFT) < (gpu_addr + size)) { ++ kbase_gpu_vm_unlock(kctx); ++ return -EINVAL; ++ } ++ ++ kbase_gpu_vm_unlock(kctx); ++ ++ return 0; ++} ++ ++KBASE_EXPORT_TEST_API(kbasep_find_enclosing_gpu_mapping_start_and_offset); ++ ++void kbase_sync_single(struct kbase_context *kctx, ++ struct tagged_addr t_cpu_pa, struct tagged_addr t_gpu_pa, ++ off_t offset, size_t size, enum kbase_sync_type sync_fn) ++{ ++ struct page *cpu_page; ++ phys_addr_t cpu_pa = as_phys_addr_t(t_cpu_pa); ++ phys_addr_t gpu_pa = as_phys_addr_t(t_gpu_pa); ++ ++ cpu_page = pfn_to_page(PFN_DOWN(cpu_pa)); ++ ++ if (likely(cpu_pa == gpu_pa)) { ++ dma_addr_t dma_addr; ++ ++ BUG_ON(!cpu_page); ++ BUG_ON(offset + size > PAGE_SIZE); ++ ++ dma_addr = kbase_dma_addr(cpu_page) + offset; ++ if (sync_fn == KBASE_SYNC_TO_CPU) ++ dma_sync_single_for_cpu(kctx->kbdev->dev, dma_addr, ++ size, DMA_BIDIRECTIONAL); ++ else if (sync_fn == KBASE_SYNC_TO_DEVICE) ++ dma_sync_single_for_device(kctx->kbdev->dev, dma_addr, ++ size, DMA_BIDIRECTIONAL); ++ } else { ++ void *src = NULL; ++ void *dst = NULL; ++ struct page *gpu_page; ++ ++ if (WARN(!gpu_pa, "No GPU PA found for infinite cache op")) ++ return; ++ ++ gpu_page = pfn_to_page(PFN_DOWN(gpu_pa)); ++ ++ if (sync_fn == KBASE_SYNC_TO_DEVICE) { ++ src = ((unsigned char *)kmap(cpu_page)) + offset; ++ dst = ((unsigned char *)kmap(gpu_page)) + offset; ++ } else if (sync_fn == KBASE_SYNC_TO_CPU) { ++ dma_sync_single_for_cpu(kctx->kbdev->dev, ++ kbase_dma_addr(gpu_page) + offset, ++ size, DMA_BIDIRECTIONAL); ++ src = ((unsigned char *)kmap(gpu_page)) + offset; ++ dst = ((unsigned char *)kmap(cpu_page)) + offset; ++ } ++ memcpy(dst, src, size); ++ kunmap(gpu_page); ++ kunmap(cpu_page); ++ if (sync_fn == KBASE_SYNC_TO_DEVICE) ++ dma_sync_single_for_device(kctx->kbdev->dev, ++ kbase_dma_addr(gpu_page) + offset, ++ size, DMA_BIDIRECTIONAL); ++ } ++} ++ ++static int kbase_do_syncset(struct kbase_context *kctx, ++ struct basep_syncset *sset, enum kbase_sync_type sync_fn) ++{ ++ int err = 0; ++ struct kbase_va_region *reg; ++ struct kbase_cpu_mapping *map; ++ unsigned long start; ++ size_t size; ++ struct tagged_addr *cpu_pa; ++ struct tagged_addr *gpu_pa; ++ u64 page_off, page_count; ++ u64 i; ++ u64 offset; ++ ++ kbase_os_mem_map_lock(kctx); ++ kbase_gpu_vm_lock(kctx); ++ ++ /* find the region where the virtual address is contained */ ++ reg = kbase_region_tracker_find_region_enclosing_address(kctx, ++ sset->mem_handle.basep.handle); ++ if (kbase_is_region_invalid_or_free(reg)) { ++ dev_warn(kctx->kbdev->dev, "Can't find a valid region at VA 0x%016llX", ++ sset->mem_handle.basep.handle); ++ err = -EINVAL; ++ goto out_unlock; ++ } ++ ++ /* ++ * Handle imported memory before checking for KBASE_REG_CPU_CACHED. The ++ * CPU mapping cacheability is defined by the owner of the imported ++ * memory, and not by kbase, therefore we must assume that any imported ++ * memory may be cached. ++ */ ++ if (kbase_mem_is_imported(reg->gpu_alloc->type)) { ++ err = kbase_mem_do_sync_imported(kctx, reg, sync_fn); ++ goto out_unlock; ++ } ++ ++ if (!(reg->flags & KBASE_REG_CPU_CACHED)) ++ goto out_unlock; ++ ++ start = (uintptr_t)sset->user_addr; ++ size = (size_t)sset->size; ++ ++ map = kbasep_find_enclosing_cpu_mapping(kctx, start, size, &offset); ++ if (!map) { ++ dev_warn(kctx->kbdev->dev, "Can't find CPU mapping 0x%016lX for VA 0x%016llX", ++ start, sset->mem_handle.basep.handle); ++ err = -EINVAL; ++ goto out_unlock; ++ } ++ ++ page_off = offset >> PAGE_SHIFT; ++ offset &= ~PAGE_MASK; ++ page_count = (size + offset + (PAGE_SIZE - 1)) >> PAGE_SHIFT; ++ cpu_pa = kbase_get_cpu_phy_pages(reg); ++ gpu_pa = kbase_get_gpu_phy_pages(reg); ++ ++ if (page_off > reg->nr_pages || ++ page_off + page_count > reg->nr_pages) { ++ /* Sync overflows the region */ ++ err = -EINVAL; ++ goto out_unlock; ++ } ++ ++ /* Sync first page */ ++ if (as_phys_addr_t(cpu_pa[page_off])) { ++ size_t sz = MIN(((size_t) PAGE_SIZE - offset), size); ++ ++ kbase_sync_single(kctx, cpu_pa[page_off], gpu_pa[page_off], ++ offset, sz, sync_fn); ++ } ++ ++ /* Sync middle pages (if any) */ ++ for (i = 1; page_count > 2 && i < page_count - 1; i++) { ++ /* we grow upwards, so bail on first non-present page */ ++ if (!as_phys_addr_t(cpu_pa[page_off + i])) ++ break; ++ ++ kbase_sync_single(kctx, cpu_pa[page_off + i], ++ gpu_pa[page_off + i], 0, PAGE_SIZE, sync_fn); ++ } ++ ++ /* Sync last page (if any) */ ++ if (page_count > 1 && ++ as_phys_addr_t(cpu_pa[page_off + page_count - 1])) { ++ size_t sz = ((start + size - 1) & ~PAGE_MASK) + 1; ++ ++ kbase_sync_single(kctx, cpu_pa[page_off + page_count - 1], ++ gpu_pa[page_off + page_count - 1], 0, sz, ++ sync_fn); ++ } ++ ++out_unlock: ++ kbase_gpu_vm_unlock(kctx); ++ kbase_os_mem_map_unlock(kctx); ++ return err; ++} ++ ++int kbase_sync_now(struct kbase_context *kctx, struct basep_syncset *sset) ++{ ++ int err = -EINVAL; ++ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ KBASE_DEBUG_ASSERT(sset != NULL); ++ ++ if (sset->mem_handle.basep.handle & ~PAGE_MASK) { ++ dev_warn(kctx->kbdev->dev, ++ "mem_handle: passed parameter is invalid"); ++ return -EINVAL; ++ } ++ ++ switch (sset->type) { ++ case BASE_SYNCSET_OP_MSYNC: ++ err = kbase_do_syncset(kctx, sset, KBASE_SYNC_TO_DEVICE); ++ break; ++ ++ case BASE_SYNCSET_OP_CSYNC: ++ err = kbase_do_syncset(kctx, sset, KBASE_SYNC_TO_CPU); ++ break; ++ ++ default: ++ dev_warn(kctx->kbdev->dev, "Unknown msync op %d\n", sset->type); ++ break; ++ } ++ ++ return err; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_sync_now); ++ ++/* vm lock must be held */ ++int kbase_mem_free_region(struct kbase_context *kctx, struct kbase_va_region *reg) ++{ ++ int err; ++ ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ KBASE_DEBUG_ASSERT(NULL != reg); ++ dev_dbg(kctx->kbdev->dev, "%s %p in kctx %p\n", ++ __func__, (void *)reg, (void *)kctx); ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ if (reg->flags & KBASE_REG_NO_USER_FREE) { ++ dev_warn(kctx->kbdev->dev, "Attempt to free GPU memory whose freeing by user space is forbidden!\n"); ++ return -EINVAL; ++ } ++ ++ /* ++ * Unlink the physical allocation before unmaking it evictable so ++ * that the allocation isn't grown back to its last backed size ++ * as we're going to unmap it anyway. ++ */ ++ reg->cpu_alloc->reg = NULL; ++ if (reg->cpu_alloc != reg->gpu_alloc) ++ reg->gpu_alloc->reg = NULL; ++ ++ /* ++ * If a region has been made evictable then we must unmake it ++ * before trying to free it. ++ * If the memory hasn't been reclaimed it will be unmapped and freed ++ * below, if it has been reclaimed then the operations below are no-ops. ++ */ ++ if (reg->flags & KBASE_REG_DONT_NEED) { ++ KBASE_DEBUG_ASSERT(reg->cpu_alloc->type == ++ KBASE_MEM_TYPE_NATIVE); ++ kbase_mem_evictable_unmake(reg->gpu_alloc); ++ } ++ ++ err = kbase_gpu_munmap(kctx, reg); ++ if (err) { ++ dev_warn(kctx->kbdev->dev, "Could not unmap from the GPU...\n"); ++ goto out; ++ } ++ ++ /* This will also free the physical pages */ ++ kbase_free_alloced_region(reg); ++ ++ out: ++ return err; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_mem_free_region); ++ ++/** ++ * @brief Free the region from the GPU and unregister it. ++ * ++ * This function implements the free operation on a memory segment. ++ * It will loudly fail if called with outstanding mappings. ++ */ ++int kbase_mem_free(struct kbase_context *kctx, u64 gpu_addr) ++{ ++ int err = 0; ++ struct kbase_va_region *reg; ++ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ dev_dbg(kctx->kbdev->dev, "%s 0x%llx in kctx %p\n", ++ __func__, gpu_addr, (void *)kctx); ++ ++ if ((gpu_addr & ~PAGE_MASK) && (gpu_addr >= PAGE_SIZE)) { ++ dev_warn(kctx->kbdev->dev, "kbase_mem_free: gpu_addr parameter is invalid"); ++ return -EINVAL; ++ } ++ ++ if (0 == gpu_addr) { ++ dev_warn(kctx->kbdev->dev, "gpu_addr 0 is reserved for the ringbuffer and it's an error to try to free it using kbase_mem_free\n"); ++ return -EINVAL; ++ } ++ kbase_gpu_vm_lock(kctx); ++ ++ if (gpu_addr >= BASE_MEM_COOKIE_BASE && ++ gpu_addr < BASE_MEM_FIRST_FREE_ADDRESS) { ++ int cookie = PFN_DOWN(gpu_addr - BASE_MEM_COOKIE_BASE); ++ ++ reg = kctx->pending_regions[cookie]; ++ if (!reg) { ++ err = -EINVAL; ++ goto out_unlock; ++ } ++ ++ /* ask to unlink the cookie as we'll free it */ ++ ++ kctx->pending_regions[cookie] = NULL; ++ bitmap_set(kctx->cookies, cookie, 1); ++ ++ kbase_free_alloced_region(reg); ++ } else { ++ /* A real GPU va */ ++ /* Validate the region */ ++ reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr); ++ if (kbase_is_region_invalid_or_free(reg)) { ++ dev_warn(kctx->kbdev->dev, "kbase_mem_free called with nonexistent gpu_addr 0x%llX", ++ gpu_addr); ++ err = -EINVAL; ++ goto out_unlock; ++ } ++ ++ if ((reg->flags & KBASE_REG_ZONE_MASK) == KBASE_REG_ZONE_SAME_VA) { ++ /* SAME_VA must be freed through munmap */ ++ dev_warn(kctx->kbdev->dev, "%s called on SAME_VA memory 0x%llX", __func__, ++ gpu_addr); ++ err = -EINVAL; ++ goto out_unlock; ++ } ++ err = kbase_mem_free_region(kctx, reg); ++ } ++ ++ out_unlock: ++ kbase_gpu_vm_unlock(kctx); ++ return err; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_mem_free); ++ ++int kbase_update_region_flags(struct kbase_context *kctx, ++ struct kbase_va_region *reg, unsigned long flags) ++{ ++ KBASE_DEBUG_ASSERT(NULL != reg); ++ KBASE_DEBUG_ASSERT((flags & ~((1ul << BASE_MEM_FLAGS_NR_BITS) - 1)) == 0); ++ ++ reg->flags |= kbase_cache_enabled(flags, reg->nr_pages); ++ /* all memory is now growable */ ++ reg->flags |= KBASE_REG_GROWABLE; ++ ++ if (flags & BASE_MEM_GROW_ON_GPF) ++ reg->flags |= KBASE_REG_PF_GROW; ++ ++ if (flags & BASE_MEM_PROT_CPU_WR) ++ reg->flags |= KBASE_REG_CPU_WR; ++ ++ if (flags & BASE_MEM_PROT_CPU_RD) ++ reg->flags |= KBASE_REG_CPU_RD; ++ ++ if (flags & BASE_MEM_PROT_GPU_WR) ++ reg->flags |= KBASE_REG_GPU_WR; ++ ++ if (flags & BASE_MEM_PROT_GPU_RD) ++ reg->flags |= KBASE_REG_GPU_RD; ++ ++ if (0 == (flags & BASE_MEM_PROT_GPU_EX)) ++ reg->flags |= KBASE_REG_GPU_NX; ++ ++ if (!kbase_device_is_cpu_coherent(kctx->kbdev)) { ++ if (flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED && ++ !(flags & BASE_MEM_UNCACHED_GPU)) ++ return -EINVAL; ++ } else if (flags & (BASE_MEM_COHERENT_SYSTEM | ++ BASE_MEM_COHERENT_SYSTEM_REQUIRED)) { ++ reg->flags |= KBASE_REG_SHARE_BOTH; ++ } ++ ++ if (!(reg->flags & KBASE_REG_SHARE_BOTH) && ++ flags & BASE_MEM_COHERENT_LOCAL) { ++ reg->flags |= KBASE_REG_SHARE_IN; ++ } ++ ++#if !MALI_USE_CSF ++ if (flags & BASE_MEM_TILER_ALIGN_TOP) ++ reg->flags |= KBASE_REG_TILER_ALIGN_TOP; ++#endif /* !MALI_USE_CSF */ ++ ++#if MALI_USE_CSF ++ if (flags & BASE_MEM_CSF_EVENT) { ++ reg->flags |= KBASE_REG_CSF_EVENT; ++ reg->flags |= KBASE_REG_PERMANENT_KERNEL_MAPPING; ++ ++ if (!(reg->flags & KBASE_REG_SHARE_BOTH)) { ++ /* On non coherent platforms need to map as uncached on ++ * both sides. ++ */ ++ reg->flags &= ~KBASE_REG_CPU_CACHED; ++ reg->flags &= ~KBASE_REG_GPU_CACHED; ++ } ++ } ++#endif ++ ++ /* Set up default MEMATTR usage */ ++ if (!(reg->flags & KBASE_REG_GPU_CACHED)) { ++ if (kctx->kbdev->mmu_mode->flags & ++ KBASE_MMU_MODE_HAS_NON_CACHEABLE) { ++ /* Override shareability, and MEMATTR for uncached */ ++ reg->flags &= ~(KBASE_REG_SHARE_IN | KBASE_REG_SHARE_BOTH); ++ reg->flags |= KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_NON_CACHEABLE); ++ } else { ++ dev_warn(kctx->kbdev->dev, ++ "Can't allocate GPU uncached memory due to MMU in Legacy Mode\n"); ++ return -EINVAL; ++ } ++#if MALI_USE_CSF ++ } else if (reg->flags & KBASE_REG_CSF_EVENT) { ++ WARN_ON(!(reg->flags & KBASE_REG_SHARE_BOTH)); ++ ++ reg->flags |= ++ KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_SHARED); ++#endif ++ } else if (kctx->kbdev->system_coherency == COHERENCY_ACE && ++ (reg->flags & KBASE_REG_SHARE_BOTH)) { ++ reg->flags |= ++ KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_DEFAULT_ACE); ++ } else { ++ reg->flags |= ++ KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_DEFAULT); ++ } ++ ++ if (flags & BASEP_MEM_PERMANENT_KERNEL_MAPPING) ++ reg->flags |= KBASE_REG_PERMANENT_KERNEL_MAPPING; ++ ++ if (flags & BASEP_MEM_NO_USER_FREE) ++ reg->flags |= KBASE_REG_NO_USER_FREE; ++ ++ if (flags & BASE_MEM_GPU_VA_SAME_4GB_PAGE) ++ reg->flags |= KBASE_REG_GPU_VA_SAME_4GB_PAGE; ++ ++ return 0; ++} ++ ++int kbase_alloc_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, ++ size_t nr_pages_requested) ++{ ++ int new_page_count __maybe_unused; ++ size_t nr_left = nr_pages_requested; ++ int res; ++ struct kbase_context *kctx; ++ struct kbase_device *kbdev; ++ struct tagged_addr *tp; ++ ++ if (WARN_ON(alloc->type != KBASE_MEM_TYPE_NATIVE) || ++ WARN_ON(alloc->imported.native.kctx == NULL) || ++ WARN_ON(alloc->group_id >= MEMORY_GROUP_MANAGER_NR_GROUPS)) { ++ return -EINVAL; ++ } ++ ++ if (alloc->reg) { ++ if (nr_pages_requested > alloc->reg->nr_pages - alloc->nents) ++ goto invalid_request; ++ } ++ ++ kctx = alloc->imported.native.kctx; ++ kbdev = kctx->kbdev; ++ ++ if (nr_pages_requested == 0) ++ goto done; /*nothing to do*/ ++ ++ new_page_count = atomic_add_return( ++ nr_pages_requested, &kctx->used_pages); ++ atomic_add(nr_pages_requested, ++ &kctx->kbdev->memdev.used_pages); ++ ++ /* Increase mm counters before we allocate pages so that this ++ * allocation is visible to the OOM killer */ ++ kbase_process_page_usage_inc(kctx, nr_pages_requested); ++ ++ tp = alloc->pages + alloc->nents; ++ ++#ifdef CONFIG_MALI_2MB_ALLOC ++ /* Check if we have enough pages requested so we can allocate a large ++ * page (512 * 4KB = 2MB ) ++ */ ++ if (nr_left >= (SZ_2M / SZ_4K)) { ++ int nr_lp = nr_left / (SZ_2M / SZ_4K); ++ ++ res = kbase_mem_pool_alloc_pages( ++ &kctx->mem_pools.large[alloc->group_id], ++ nr_lp * (SZ_2M / SZ_4K), ++ tp, ++ true); ++ ++ if (res > 0) { ++ nr_left -= res; ++ tp += res; ++ } ++ ++ if (nr_left) { ++ struct kbase_sub_alloc *sa, *temp_sa; ++ ++ spin_lock(&kctx->mem_partials_lock); ++ ++ list_for_each_entry_safe(sa, temp_sa, ++ &kctx->mem_partials, link) { ++ int pidx = 0; ++ ++ while (nr_left) { ++ pidx = find_next_zero_bit(sa->sub_pages, ++ SZ_2M / SZ_4K, ++ pidx); ++ bitmap_set(sa->sub_pages, pidx, 1); ++ *tp++ = as_tagged_tag(page_to_phys(sa->page + ++ pidx), ++ FROM_PARTIAL); ++ nr_left--; ++ ++ if (bitmap_full(sa->sub_pages, SZ_2M / SZ_4K)) { ++ /* unlink from partial list when full */ ++ list_del_init(&sa->link); ++ break; ++ } ++ } ++ } ++ spin_unlock(&kctx->mem_partials_lock); ++ } ++ ++ /* only if we actually have a chunk left <512. If more it indicates ++ * that we couldn't allocate a 2MB above, so no point to retry here. ++ */ ++ if (nr_left > 0 && nr_left < (SZ_2M / SZ_4K)) { ++ /* create a new partial and suballocate the rest from it */ ++ struct page *np = NULL; ++ ++ do { ++ int err; ++ ++ np = kbase_mem_pool_alloc( ++ &kctx->mem_pools.large[ ++ alloc->group_id]); ++ if (np) ++ break; ++ ++ err = kbase_mem_pool_grow( ++ &kctx->mem_pools.large[alloc->group_id], ++ 1); ++ if (err) ++ break; ++ } while (1); ++ ++ if (np) { ++ int i; ++ struct kbase_sub_alloc *sa; ++ struct page *p; ++ ++ sa = kmalloc(sizeof(*sa), GFP_KERNEL); ++ if (!sa) { ++ kbase_mem_pool_free( ++ &kctx->mem_pools.large[ ++ alloc->group_id], ++ np, ++ false); ++ goto no_new_partial; ++ } ++ ++ /* store pointers back to the control struct */ ++ np->lru.next = (void *)sa; ++ for (p = np; p < np + SZ_2M / SZ_4K; p++) ++ p->lru.prev = (void *)np; ++ INIT_LIST_HEAD(&sa->link); ++ bitmap_zero(sa->sub_pages, SZ_2M / SZ_4K); ++ sa->page = np; ++ ++ for (i = 0; i < nr_left; i++) ++ *tp++ = as_tagged_tag(page_to_phys(np + i), FROM_PARTIAL); ++ ++ bitmap_set(sa->sub_pages, 0, nr_left); ++ nr_left = 0; ++ ++ /* expose for later use */ ++ spin_lock(&kctx->mem_partials_lock); ++ list_add(&sa->link, &kctx->mem_partials); ++ spin_unlock(&kctx->mem_partials_lock); ++ } ++ } ++ } ++no_new_partial: ++#endif ++ ++ if (nr_left) { ++ res = kbase_mem_pool_alloc_pages( ++ &kctx->mem_pools.small[alloc->group_id], ++ nr_left, tp, false); ++ if (res <= 0) ++ goto alloc_failed; ++ } ++ ++ KBASE_TLSTREAM_AUX_PAGESALLOC( ++ kbdev, ++ kctx->id, ++ (u64)new_page_count); ++ ++ alloc->nents += nr_pages_requested; ++ ++ kbase_trace_gpu_mem_usage_inc(kctx->kbdev, kctx, nr_pages_requested); ++ ++done: ++ return 0; ++ ++alloc_failed: ++ /* rollback needed if got one or more 2MB but failed later */ ++ if (nr_left != nr_pages_requested) { ++ size_t nr_pages_to_free = nr_pages_requested - nr_left; ++ ++ alloc->nents += nr_pages_to_free; ++ ++ kbase_process_page_usage_inc(kctx, nr_pages_to_free); ++ atomic_add(nr_pages_to_free, &kctx->used_pages); ++ atomic_add(nr_pages_to_free, ++ &kctx->kbdev->memdev.used_pages); ++ ++ kbase_free_phy_pages_helper(alloc, nr_pages_to_free); ++ } ++ ++ kbase_process_page_usage_dec(kctx, nr_pages_requested); ++ atomic_sub(nr_pages_requested, &kctx->used_pages); ++ atomic_sub(nr_pages_requested, ++ &kctx->kbdev->memdev.used_pages); ++ ++invalid_request: ++ return -ENOMEM; ++} ++ ++struct tagged_addr *kbase_alloc_phy_pages_helper_locked( ++ struct kbase_mem_phy_alloc *alloc, struct kbase_mem_pool *pool, ++ size_t nr_pages_requested, ++ struct kbase_sub_alloc **prealloc_sa) ++{ ++ int new_page_count __maybe_unused; ++ size_t nr_left = nr_pages_requested; ++ int res; ++ struct kbase_context *kctx; ++ struct kbase_device *kbdev; ++ struct tagged_addr *tp; ++ struct tagged_addr *new_pages = NULL; ++ ++ KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_NATIVE); ++ KBASE_DEBUG_ASSERT(alloc->imported.native.kctx); ++ ++ lockdep_assert_held(&pool->pool_lock); ++ ++#if !defined(CONFIG_MALI_2MB_ALLOC) ++ WARN_ON(pool->order); ++#endif ++ ++ if (alloc->reg) { ++ if (nr_pages_requested > alloc->reg->nr_pages - alloc->nents) ++ goto invalid_request; ++ } ++ ++ kctx = alloc->imported.native.kctx; ++ kbdev = kctx->kbdev; ++ ++ lockdep_assert_held(&kctx->mem_partials_lock); ++ ++ if (nr_pages_requested == 0) ++ goto done; /*nothing to do*/ ++ ++ new_page_count = atomic_add_return( ++ nr_pages_requested, &kctx->used_pages); ++ atomic_add(nr_pages_requested, ++ &kctx->kbdev->memdev.used_pages); ++ ++ /* Increase mm counters before we allocate pages so that this ++ * allocation is visible to the OOM killer ++ */ ++ kbase_process_page_usage_inc(kctx, nr_pages_requested); ++ ++ tp = alloc->pages + alloc->nents; ++ new_pages = tp; ++ ++#ifdef CONFIG_MALI_2MB_ALLOC ++ if (pool->order) { ++ int nr_lp = nr_left / (SZ_2M / SZ_4K); ++ ++ res = kbase_mem_pool_alloc_pages_locked(pool, ++ nr_lp * (SZ_2M / SZ_4K), ++ tp); ++ ++ if (res > 0) { ++ nr_left -= res; ++ tp += res; ++ } ++ ++ if (nr_left) { ++ struct kbase_sub_alloc *sa, *temp_sa; ++ ++ list_for_each_entry_safe(sa, temp_sa, ++ &kctx->mem_partials, link) { ++ int pidx = 0; ++ ++ while (nr_left) { ++ pidx = find_next_zero_bit(sa->sub_pages, ++ SZ_2M / SZ_4K, ++ pidx); ++ bitmap_set(sa->sub_pages, pidx, 1); ++ *tp++ = as_tagged_tag(page_to_phys( ++ sa->page + pidx), ++ FROM_PARTIAL); ++ nr_left--; ++ ++ if (bitmap_full(sa->sub_pages, ++ SZ_2M / SZ_4K)) { ++ /* unlink from partial list when ++ * full ++ */ ++ list_del_init(&sa->link); ++ break; ++ } ++ } ++ } ++ } ++ ++ /* only if we actually have a chunk left <512. If more it ++ * indicates that we couldn't allocate a 2MB above, so no point ++ * to retry here. ++ */ ++ if (nr_left > 0 && nr_left < (SZ_2M / SZ_4K)) { ++ /* create a new partial and suballocate the rest from it ++ */ ++ struct page *np = NULL; ++ ++ np = kbase_mem_pool_alloc_locked(pool); ++ ++ if (np) { ++ int i; ++ struct kbase_sub_alloc *const sa = *prealloc_sa; ++ struct page *p; ++ ++ /* store pointers back to the control struct */ ++ np->lru.next = (void *)sa; ++ for (p = np; p < np + SZ_2M / SZ_4K; p++) ++ p->lru.prev = (void *)np; ++ INIT_LIST_HEAD(&sa->link); ++ bitmap_zero(sa->sub_pages, SZ_2M / SZ_4K); ++ sa->page = np; ++ ++ for (i = 0; i < nr_left; i++) ++ *tp++ = as_tagged_tag( ++ page_to_phys(np + i), ++ FROM_PARTIAL); ++ ++ bitmap_set(sa->sub_pages, 0, nr_left); ++ nr_left = 0; ++ /* Indicate to user that we'll free this memory ++ * later. ++ */ ++ *prealloc_sa = NULL; ++ ++ /* expose for later use */ ++ list_add(&sa->link, &kctx->mem_partials); ++ } ++ } ++ if (nr_left) ++ goto alloc_failed; ++ } else { ++#endif ++ res = kbase_mem_pool_alloc_pages_locked(pool, ++ nr_left, ++ tp); ++ if (res <= 0) ++ goto alloc_failed; ++#ifdef CONFIG_MALI_2MB_ALLOC ++ } ++#endif ++ ++ KBASE_TLSTREAM_AUX_PAGESALLOC( ++ kbdev, ++ kctx->id, ++ (u64)new_page_count); ++ ++ alloc->nents += nr_pages_requested; ++ ++ kbase_trace_gpu_mem_usage_inc(kctx->kbdev, kctx, nr_pages_requested); ++ ++done: ++ return new_pages; ++ ++alloc_failed: ++ /* rollback needed if got one or more 2MB but failed later */ ++ if (nr_left != nr_pages_requested) { ++ size_t nr_pages_to_free = nr_pages_requested - nr_left; ++ ++ struct tagged_addr *start_free = alloc->pages + alloc->nents; ++ ++#ifdef CONFIG_MALI_2MB_ALLOC ++ if (pool->order) { ++ while (nr_pages_to_free) { ++ if (is_huge_head(*start_free)) { ++ kbase_mem_pool_free_pages_locked( ++ pool, 512, ++ start_free, ++ false, /* not dirty */ ++ true); /* return to pool */ ++ nr_pages_to_free -= 512; ++ start_free += 512; ++ } else if (is_partial(*start_free)) { ++ free_partial_locked(kctx, pool, ++ *start_free); ++ nr_pages_to_free--; ++ start_free++; ++ } ++ } ++ } else { ++#endif ++ kbase_mem_pool_free_pages_locked(pool, ++ nr_pages_to_free, ++ start_free, ++ false, /* not dirty */ ++ true); /* return to pool */ ++#ifdef CONFIG_MALI_2MB_ALLOC ++ } ++#endif ++ } ++ ++ kbase_process_page_usage_dec(kctx, nr_pages_requested); ++ atomic_sub(nr_pages_requested, &kctx->used_pages); ++ atomic_sub(nr_pages_requested, &kctx->kbdev->memdev.used_pages); ++ ++invalid_request: ++ return NULL; ++} ++ ++static void free_partial(struct kbase_context *kctx, int group_id, struct ++ tagged_addr tp) ++{ ++ struct page *p, *head_page; ++ struct kbase_sub_alloc *sa; ++ ++ p = as_page(tp); ++ head_page = (struct page *)p->lru.prev; ++ sa = (struct kbase_sub_alloc *)head_page->lru.next; ++ spin_lock(&kctx->mem_partials_lock); ++ clear_bit(p - head_page, sa->sub_pages); ++ if (bitmap_empty(sa->sub_pages, SZ_2M / SZ_4K)) { ++ list_del(&sa->link); ++ kbase_mem_pool_free( ++ &kctx->mem_pools.large[group_id], ++ head_page, ++ true); ++ kfree(sa); ++ } else if (bitmap_weight(sa->sub_pages, SZ_2M / SZ_4K) == ++ SZ_2M / SZ_4K - 1) { ++ /* expose the partial again */ ++ list_add(&sa->link, &kctx->mem_partials); ++ } ++ spin_unlock(&kctx->mem_partials_lock); ++} ++ ++int kbase_free_phy_pages_helper( ++ struct kbase_mem_phy_alloc *alloc, ++ size_t nr_pages_to_free) ++{ ++ struct kbase_context *kctx = alloc->imported.native.kctx; ++ struct kbase_device *kbdev = kctx->kbdev; ++ bool syncback; ++ bool reclaimed = (alloc->evicted != 0); ++ struct tagged_addr *start_free; ++ int new_page_count __maybe_unused; ++ size_t freed = 0; ++ ++ if (WARN_ON(alloc->type != KBASE_MEM_TYPE_NATIVE) || ++ WARN_ON(alloc->imported.native.kctx == NULL) || ++ WARN_ON(alloc->nents < nr_pages_to_free) || ++ WARN_ON(alloc->group_id >= MEMORY_GROUP_MANAGER_NR_GROUPS)) { ++ return -EINVAL; ++ } ++ ++ /* early out if nothing to do */ ++ if (0 == nr_pages_to_free) ++ return 0; ++ ++ start_free = alloc->pages + alloc->nents - nr_pages_to_free; ++ ++ syncback = alloc->properties & KBASE_MEM_PHY_ALLOC_ACCESSED_CACHED; ++ ++ /* pad start_free to a valid start location */ ++ while (nr_pages_to_free && is_huge(*start_free) && ++ !is_huge_head(*start_free)) { ++ nr_pages_to_free--; ++ start_free++; ++ } ++ ++ while (nr_pages_to_free) { ++ if (is_huge_head(*start_free)) { ++ /* This is a 2MB entry, so free all the 512 pages that ++ * it points to ++ */ ++ kbase_mem_pool_free_pages( ++ &kctx->mem_pools.large[alloc->group_id], ++ 512, ++ start_free, ++ syncback, ++ reclaimed); ++ nr_pages_to_free -= 512; ++ start_free += 512; ++ freed += 512; ++ } else if (is_partial(*start_free)) { ++ free_partial(kctx, alloc->group_id, *start_free); ++ nr_pages_to_free--; ++ start_free++; ++ freed++; ++ } else { ++ struct tagged_addr *local_end_free; ++ ++ local_end_free = start_free; ++ while (nr_pages_to_free && ++ !is_huge(*local_end_free) && ++ !is_partial(*local_end_free)) { ++ local_end_free++; ++ nr_pages_to_free--; ++ } ++ kbase_mem_pool_free_pages( ++ &kctx->mem_pools.small[alloc->group_id], ++ local_end_free - start_free, ++ start_free, ++ syncback, ++ reclaimed); ++ freed += local_end_free - start_free; ++ start_free += local_end_free - start_free; ++ } ++ } ++ ++ alloc->nents -= freed; ++ ++ /* ++ * If the allocation was not evicted (i.e. evicted == 0) then ++ * the page accounting needs to be done. ++ */ ++ if (!reclaimed) { ++ kbase_process_page_usage_dec(kctx, freed); ++ new_page_count = atomic_sub_return(freed, ++ &kctx->used_pages); ++ atomic_sub(freed, ++ &kctx->kbdev->memdev.used_pages); ++ ++ KBASE_TLSTREAM_AUX_PAGESALLOC( ++ kbdev, ++ kctx->id, ++ (u64)new_page_count); ++ ++ kbase_trace_gpu_mem_usage_dec(kctx->kbdev, kctx, freed); ++ } ++ ++ return 0; ++} ++ ++static void free_partial_locked(struct kbase_context *kctx, ++ struct kbase_mem_pool *pool, struct tagged_addr tp) ++{ ++ struct page *p, *head_page; ++ struct kbase_sub_alloc *sa; ++ ++ lockdep_assert_held(&pool->pool_lock); ++ lockdep_assert_held(&kctx->mem_partials_lock); ++ ++ p = as_page(tp); ++ head_page = (struct page *)p->lru.prev; ++ sa = (struct kbase_sub_alloc *)head_page->lru.next; ++ clear_bit(p - head_page, sa->sub_pages); ++ if (bitmap_empty(sa->sub_pages, SZ_2M / SZ_4K)) { ++ list_del(&sa->link); ++ kbase_mem_pool_free_locked(pool, head_page, true); ++ kfree(sa); ++ } else if (bitmap_weight(sa->sub_pages, SZ_2M / SZ_4K) == ++ SZ_2M / SZ_4K - 1) { ++ /* expose the partial again */ ++ list_add(&sa->link, &kctx->mem_partials); ++ } ++} ++ ++void kbase_free_phy_pages_helper_locked(struct kbase_mem_phy_alloc *alloc, ++ struct kbase_mem_pool *pool, struct tagged_addr *pages, ++ size_t nr_pages_to_free) ++{ ++ struct kbase_context *kctx = alloc->imported.native.kctx; ++ struct kbase_device *kbdev = kctx->kbdev; ++ bool syncback; ++ bool reclaimed = (alloc->evicted != 0); ++ struct tagged_addr *start_free; ++ size_t freed = 0; ++ ++ KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_NATIVE); ++ KBASE_DEBUG_ASSERT(alloc->imported.native.kctx); ++ KBASE_DEBUG_ASSERT(alloc->nents >= nr_pages_to_free); ++ ++ lockdep_assert_held(&pool->pool_lock); ++ lockdep_assert_held(&kctx->mem_partials_lock); ++ ++ /* early out if nothing to do */ ++ if (!nr_pages_to_free) ++ return; ++ ++ start_free = pages; ++ ++ syncback = alloc->properties & KBASE_MEM_PHY_ALLOC_ACCESSED_CACHED; ++ ++ /* pad start_free to a valid start location */ ++ while (nr_pages_to_free && is_huge(*start_free) && ++ !is_huge_head(*start_free)) { ++ nr_pages_to_free--; ++ start_free++; ++ } ++ ++ while (nr_pages_to_free) { ++ if (is_huge_head(*start_free)) { ++ /* This is a 2MB entry, so free all the 512 pages that ++ * it points to ++ */ ++ WARN_ON(!pool->order); ++ kbase_mem_pool_free_pages_locked(pool, ++ 512, ++ start_free, ++ syncback, ++ reclaimed); ++ nr_pages_to_free -= 512; ++ start_free += 512; ++ freed += 512; ++ } else if (is_partial(*start_free)) { ++ WARN_ON(!pool->order); ++ free_partial_locked(kctx, pool, *start_free); ++ nr_pages_to_free--; ++ start_free++; ++ freed++; ++ } else { ++ struct tagged_addr *local_end_free; ++ ++ WARN_ON(pool->order); ++ local_end_free = start_free; ++ while (nr_pages_to_free && ++ !is_huge(*local_end_free) && ++ !is_partial(*local_end_free)) { ++ local_end_free++; ++ nr_pages_to_free--; ++ } ++ kbase_mem_pool_free_pages_locked(pool, ++ local_end_free - start_free, ++ start_free, ++ syncback, ++ reclaimed); ++ freed += local_end_free - start_free; ++ start_free += local_end_free - start_free; ++ } ++ } ++ ++ alloc->nents -= freed; ++ ++ /* ++ * If the allocation was not evicted (i.e. evicted == 0) then ++ * the page accounting needs to be done. ++ */ ++ if (!reclaimed) { ++ int new_page_count; ++ ++ kbase_process_page_usage_dec(kctx, freed); ++ new_page_count = atomic_sub_return(freed, ++ &kctx->used_pages); ++ atomic_sub(freed, ++ &kctx->kbdev->memdev.used_pages); ++ ++ KBASE_TLSTREAM_AUX_PAGESALLOC( ++ kbdev, ++ kctx->id, ++ (u64)new_page_count); ++ ++ kbase_trace_gpu_mem_usage_dec(kctx->kbdev, kctx, freed); ++ } ++} ++ ++#if MALI_USE_CSF ++/** ++ * kbase_jd_user_buf_unpin_pages - Release the pinned pages of a user buffer. ++ * @alloc: The allocation for the imported user buffer. ++ */ ++static void kbase_jd_user_buf_unpin_pages(struct kbase_mem_phy_alloc *alloc); ++#endif ++ ++void kbase_mem_kref_free(struct kref *kref) ++{ ++ struct kbase_mem_phy_alloc *alloc; ++ ++ alloc = container_of(kref, struct kbase_mem_phy_alloc, kref); ++ ++ switch (alloc->type) { ++ case KBASE_MEM_TYPE_NATIVE: { ++ ++ if (!WARN_ON(!alloc->imported.native.kctx)) { ++ if (alloc->permanent_map) ++ kbase_phy_alloc_mapping_term( ++ alloc->imported.native.kctx, ++ alloc); ++ ++ /* ++ * The physical allocation must have been removed from ++ * the eviction list before trying to free it. ++ */ ++ mutex_lock( ++ &alloc->imported.native.kctx->jit_evict_lock); ++ WARN_ON(!list_empty(&alloc->evict_node)); ++ mutex_unlock( ++ &alloc->imported.native.kctx->jit_evict_lock); ++ ++ kbase_process_page_usage_dec( ++ alloc->imported.native.kctx, ++ alloc->imported.native.nr_struct_pages); ++ } ++ kbase_free_phy_pages_helper(alloc, alloc->nents); ++ break; ++ } ++ case KBASE_MEM_TYPE_ALIAS: { ++ /* just call put on the underlying phy allocs */ ++ size_t i; ++ struct kbase_aliased *aliased; ++ ++ aliased = alloc->imported.alias.aliased; ++ if (aliased) { ++ for (i = 0; i < alloc->imported.alias.nents; i++) ++ if (aliased[i].alloc) ++ kbase_mem_phy_alloc_put(aliased[i].alloc); ++ vfree(aliased); ++ } ++ break; ++ } ++ case KBASE_MEM_TYPE_RAW: ++ /* raw pages, external cleanup */ ++ break; ++ case KBASE_MEM_TYPE_IMPORTED_UMM: ++ if (!IS_ENABLED(CONFIG_MALI_DMA_BUF_MAP_ON_DEMAND)) { ++ WARN_ONCE(alloc->imported.umm.current_mapping_usage_count != 1, ++ "WARNING: expected excatly 1 mapping, got %d", ++ alloc->imported.umm.current_mapping_usage_count); ++ dma_buf_unmap_attachment( ++ alloc->imported.umm.dma_attachment, ++ alloc->imported.umm.sgt, ++ DMA_BIDIRECTIONAL); ++ kbase_remove_dma_buf_usage(alloc->imported.umm.kctx, ++ alloc); ++ } ++ dma_buf_detach(alloc->imported.umm.dma_buf, ++ alloc->imported.umm.dma_attachment); ++ dma_buf_put(alloc->imported.umm.dma_buf); ++ break; ++ case KBASE_MEM_TYPE_IMPORTED_USER_BUF: ++#if MALI_USE_CSF ++ kbase_jd_user_buf_unpin_pages(alloc); ++#endif ++ if (alloc->imported.user_buf.mm) ++ mmdrop(alloc->imported.user_buf.mm); ++ if (alloc->properties & KBASE_MEM_PHY_ALLOC_LARGE) ++ vfree(alloc->imported.user_buf.pages); ++ else ++ kfree(alloc->imported.user_buf.pages); ++ break; ++ default: ++ WARN(1, "Unexecpted free of type %d\n", alloc->type); ++ break; ++ } ++ ++ /* Free based on allocation type */ ++ if (alloc->properties & KBASE_MEM_PHY_ALLOC_LARGE) ++ vfree(alloc); ++ else ++ kfree(alloc); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_mem_kref_free); ++ ++int kbase_alloc_phy_pages(struct kbase_va_region *reg, size_t vsize, size_t size) ++{ ++ KBASE_DEBUG_ASSERT(NULL != reg); ++ KBASE_DEBUG_ASSERT(vsize > 0); ++ ++ /* validate user provided arguments */ ++ if (size > vsize || vsize > reg->nr_pages) ++ goto out_term; ++ ++ /* Prevent vsize*sizeof from wrapping around. ++ * For instance, if vsize is 2**29+1, we'll allocate 1 byte and the alloc won't fail. ++ */ ++ if ((size_t) vsize > ((size_t) -1 / sizeof(*reg->cpu_alloc->pages))) ++ goto out_term; ++ ++ KBASE_DEBUG_ASSERT(0 != vsize); ++ ++ if (kbase_alloc_phy_pages_helper(reg->cpu_alloc, size) != 0) ++ goto out_term; ++ ++ reg->cpu_alloc->reg = reg; ++ if (reg->cpu_alloc != reg->gpu_alloc) { ++ if (kbase_alloc_phy_pages_helper(reg->gpu_alloc, size) != 0) ++ goto out_rollback; ++ reg->gpu_alloc->reg = reg; ++ } ++ ++ return 0; ++ ++out_rollback: ++ kbase_free_phy_pages_helper(reg->cpu_alloc, size); ++out_term: ++ return -1; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_alloc_phy_pages); ++ ++bool kbase_check_alloc_flags(unsigned long flags) ++{ ++ /* Only known input flags should be set. */ ++ if (flags & ~BASE_MEM_FLAGS_INPUT_MASK) ++ return false; ++ ++ /* At least one flag should be set */ ++ if (flags == 0) ++ return false; ++ ++ /* Either the GPU or CPU must be reading from the allocated memory */ ++ if ((flags & (BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_GPU_RD)) == 0) ++ return false; ++ ++ /* Either the GPU or CPU must be writing to the allocated memory */ ++ if ((flags & (BASE_MEM_PROT_CPU_WR | BASE_MEM_PROT_GPU_WR)) == 0) ++ return false; ++ ++ /* GPU executable memory cannot: ++ * - Be written by the GPU ++ * - Be grown on GPU page fault ++ */ ++ if ((flags & BASE_MEM_PROT_GPU_EX) && (flags & ++ (BASE_MEM_PROT_GPU_WR | BASE_MEM_GROW_ON_GPF))) ++ return false; ++ ++#if !MALI_USE_CSF ++ /* GPU executable memory also cannot have the top of its initial ++ * commit aligned to 'extent' ++ */ ++ if ((flags & BASE_MEM_PROT_GPU_EX) && (flags & ++ BASE_MEM_TILER_ALIGN_TOP)) ++ return false; ++#endif /* !MALI_USE_CSF */ ++ ++ /* To have an allocation lie within a 4GB chunk is required only for ++ * TLS memory, which will never be used to contain executable code. ++ */ ++ if ((flags & BASE_MEM_GPU_VA_SAME_4GB_PAGE) && (flags & ++ BASE_MEM_PROT_GPU_EX)) ++ return false; ++ ++#if !MALI_USE_CSF ++ /* TLS memory should also not be used for tiler heap */ ++ if ((flags & BASE_MEM_GPU_VA_SAME_4GB_PAGE) && (flags & ++ BASE_MEM_TILER_ALIGN_TOP)) ++ return false; ++#endif /* !MALI_USE_CSF */ ++ ++ /* GPU should have at least read or write access otherwise there is no ++ reason for allocating. */ ++ if ((flags & (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR)) == 0) ++ return false; ++ ++ /* BASE_MEM_IMPORT_SHARED is only valid for imported memory */ ++ if ((flags & BASE_MEM_IMPORT_SHARED) == BASE_MEM_IMPORT_SHARED) ++ return false; ++ ++ /* BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP is only valid for imported ++ * memory */ ++ if ((flags & BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP) == ++ BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP) ++ return false; ++ ++ /* Should not combine BASE_MEM_COHERENT_LOCAL with ++ * BASE_MEM_COHERENT_SYSTEM */ ++ if ((flags & (BASE_MEM_COHERENT_LOCAL | BASE_MEM_COHERENT_SYSTEM)) == ++ (BASE_MEM_COHERENT_LOCAL | BASE_MEM_COHERENT_SYSTEM)) ++ return false; ++ ++ return true; ++} ++ ++bool kbase_check_import_flags(unsigned long flags) ++{ ++ /* Only known input flags should be set. */ ++ if (flags & ~BASE_MEM_FLAGS_INPUT_MASK) ++ return false; ++ ++ /* At least one flag should be set */ ++ if (flags == 0) ++ return false; ++ ++ /* Imported memory cannot be GPU executable */ ++ if (flags & BASE_MEM_PROT_GPU_EX) ++ return false; ++ ++ /* Imported memory cannot grow on page fault */ ++ if (flags & BASE_MEM_GROW_ON_GPF) ++ return false; ++ ++#if !MALI_USE_CSF ++ /* Imported memory cannot be aligned to the end of its initial commit */ ++ if (flags & BASE_MEM_TILER_ALIGN_TOP) ++ return false; ++#endif /* !MALI_USE_CSF */ ++ ++ /* GPU should have at least read or write access otherwise there is no ++ reason for importing. */ ++ if ((flags & (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR)) == 0) ++ return false; ++ ++ /* Protected memory cannot be read by the CPU */ ++ if ((flags & BASE_MEM_PROTECTED) && (flags & BASE_MEM_PROT_CPU_RD)) ++ return false; ++ ++ return true; ++} ++ ++int kbase_check_alloc_sizes(struct kbase_context *kctx, unsigned long flags, ++ u64 va_pages, u64 commit_pages, u64 large_extent) ++{ ++ struct device *dev = kctx->kbdev->dev; ++ int gpu_pc_bits = kctx->kbdev->gpu_props.props.core_props.log2_program_counter_size; ++ u64 gpu_pc_pages_max = 1ULL << gpu_pc_bits >> PAGE_SHIFT; ++ struct kbase_va_region test_reg; ++ ++ /* kbase_va_region's extent member can be of variable size, so check against that type */ ++ test_reg.extent = large_extent; ++ ++#define KBASE_MSG_PRE "GPU allocation attempted with " ++ ++ if (0 == va_pages) { ++ dev_warn(dev, KBASE_MSG_PRE "0 va_pages!"); ++ return -EINVAL; ++ } ++ ++ if (va_pages > KBASE_MEM_ALLOC_MAX_SIZE) { ++ dev_warn(dev, KBASE_MSG_PRE "va_pages==%lld larger than KBASE_MEM_ALLOC_MAX_SIZE!", ++ (unsigned long long)va_pages); ++ return -ENOMEM; ++ } ++ ++ /* Note: commit_pages is checked against va_pages during ++ * kbase_alloc_phy_pages() */ ++ ++ /* Limit GPU executable allocs to GPU PC size */ ++ if ((flags & BASE_MEM_PROT_GPU_EX) && (va_pages > gpu_pc_pages_max)) { ++ dev_warn(dev, KBASE_MSG_PRE "BASE_MEM_PROT_GPU_EX and va_pages==%lld larger than GPU PC range %lld", ++ (unsigned long long)va_pages, ++ (unsigned long long)gpu_pc_pages_max); ++ ++ return -EINVAL; ++ } ++ ++ if ((flags & BASE_MEM_GROW_ON_GPF) && (test_reg.extent == 0)) { ++ dev_warn(dev, KBASE_MSG_PRE "BASE_MEM_GROW_ON_GPF but extent == 0\n"); ++ return -EINVAL; ++ } ++ ++#if !MALI_USE_CSF ++ if ((flags & BASE_MEM_TILER_ALIGN_TOP) && (test_reg.extent == 0)) { ++ dev_warn(dev, KBASE_MSG_PRE "BASE_MEM_TILER_ALIGN_TOP but extent == 0\n"); ++ return -EINVAL; ++ } ++ ++ if (!(flags & (BASE_MEM_GROW_ON_GPF | BASE_MEM_TILER_ALIGN_TOP)) && ++ test_reg.extent != 0) { ++ dev_warn(dev, KBASE_MSG_PRE "neither BASE_MEM_GROW_ON_GPF nor BASE_MEM_TILER_ALIGN_TOP set but extent != 0\n"); ++ return -EINVAL; ++ } ++#else ++ if (!(flags & BASE_MEM_GROW_ON_GPF) && test_reg.extent != 0) { ++ dev_warn(dev, KBASE_MSG_PRE "BASE_MEM_GROW_ON_GPF not set but extent != 0\n"); ++ return -EINVAL; ++ } ++#endif /* !MALI_USE_CSF */ ++ ++#if !MALI_USE_CSF ++ /* BASE_MEM_TILER_ALIGN_TOP memory has a number of restrictions */ ++ if (flags & BASE_MEM_TILER_ALIGN_TOP) { ++#define KBASE_MSG_PRE_FLAG KBASE_MSG_PRE "BASE_MEM_TILER_ALIGN_TOP and " ++ unsigned long small_extent; ++ ++ if (large_extent > BASE_MEM_TILER_ALIGN_TOP_EXTENT_MAX_PAGES) { ++ dev_warn(dev, KBASE_MSG_PRE_FLAG "extent==%lld pages exceeds limit %lld", ++ (unsigned long long)large_extent, ++ BASE_MEM_TILER_ALIGN_TOP_EXTENT_MAX_PAGES); ++ return -EINVAL; ++ } ++ /* For use with is_power_of_2, which takes unsigned long, so ++ * must ensure e.g. on 32-bit kernel it'll fit in that type */ ++ small_extent = (unsigned long)large_extent; ++ ++ if (!is_power_of_2(small_extent)) { ++ dev_warn(dev, KBASE_MSG_PRE_FLAG "extent==%ld not a non-zero power of 2", ++ small_extent); ++ return -EINVAL; ++ } ++ ++ if (commit_pages > large_extent) { ++ dev_warn(dev, KBASE_MSG_PRE_FLAG "commit_pages==%ld exceeds extent==%ld", ++ (unsigned long)commit_pages, ++ (unsigned long)large_extent); ++ return -EINVAL; ++ } ++#undef KBASE_MSG_PRE_FLAG ++ } ++#endif /* !MALI_USE_CSF */ ++ ++ if ((flags & BASE_MEM_GPU_VA_SAME_4GB_PAGE) && ++ (va_pages > (BASE_MEM_PFN_MASK_4GB + 1))) { ++ dev_warn(dev, KBASE_MSG_PRE "BASE_MEM_GPU_VA_SAME_4GB_PAGE and va_pages==%lld greater than that needed for 4GB space", ++ (unsigned long long)va_pages); ++ return -EINVAL; ++ } ++ ++ return 0; ++#undef KBASE_MSG_PRE ++} ++ ++/** ++ * @brief Acquire the per-context region list lock ++ */ ++void kbase_gpu_vm_lock(struct kbase_context *kctx) ++{ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ mutex_lock(&kctx->reg_lock); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_gpu_vm_lock); ++ ++/** ++ * @brief Release the per-context region list lock ++ */ ++void kbase_gpu_vm_unlock(struct kbase_context *kctx) ++{ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ mutex_unlock(&kctx->reg_lock); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_gpu_vm_unlock); ++ ++#ifdef CONFIG_DEBUG_FS ++struct kbase_jit_debugfs_data { ++ int (*func)(struct kbase_jit_debugfs_data *); ++ struct mutex lock; ++ struct kbase_context *kctx; ++ u64 active_value; ++ u64 pool_value; ++ u64 destroy_value; ++ char buffer[50]; ++}; ++ ++static int kbase_jit_debugfs_common_open(struct inode *inode, ++ struct file *file, int (*func)(struct kbase_jit_debugfs_data *)) ++{ ++ struct kbase_jit_debugfs_data *data; ++ ++ data = kzalloc(sizeof(*data), GFP_KERNEL); ++ if (!data) ++ return -ENOMEM; ++ ++ data->func = func; ++ mutex_init(&data->lock); ++ data->kctx = (struct kbase_context *) inode->i_private; ++ ++ file->private_data = data; ++ ++ return nonseekable_open(inode, file); ++} ++ ++static ssize_t kbase_jit_debugfs_common_read(struct file *file, ++ char __user *buf, size_t len, loff_t *ppos) ++{ ++ struct kbase_jit_debugfs_data *data; ++ size_t size; ++ int ret; ++ ++ data = (struct kbase_jit_debugfs_data *) file->private_data; ++ mutex_lock(&data->lock); ++ ++ if (*ppos) { ++ size = strnlen(data->buffer, sizeof(data->buffer)); ++ } else { ++ if (!data->func) { ++ ret = -EACCES; ++ goto out_unlock; ++ } ++ ++ if (data->func(data)) { ++ ret = -EACCES; ++ goto out_unlock; ++ } ++ ++ size = scnprintf(data->buffer, sizeof(data->buffer), ++ "%llu,%llu,%llu", data->active_value, ++ data->pool_value, data->destroy_value); ++ } ++ ++ ret = simple_read_from_buffer(buf, len, ppos, data->buffer, size); ++ ++out_unlock: ++ mutex_unlock(&data->lock); ++ return ret; ++} ++ ++static int kbase_jit_debugfs_common_release(struct inode *inode, ++ struct file *file) ++{ ++ kfree(file->private_data); ++ return 0; ++} ++ ++#define KBASE_JIT_DEBUGFS_DECLARE(__fops, __func) \ ++static int __fops ## _open(struct inode *inode, struct file *file) \ ++{ \ ++ return kbase_jit_debugfs_common_open(inode, file, __func); \ ++} \ ++static const struct file_operations __fops = { \ ++ .owner = THIS_MODULE, \ ++ .open = __fops ## _open, \ ++ .release = kbase_jit_debugfs_common_release, \ ++ .read = kbase_jit_debugfs_common_read, \ ++ .write = NULL, \ ++ .llseek = generic_file_llseek, \ ++} ++ ++static int kbase_jit_debugfs_count_get(struct kbase_jit_debugfs_data *data) ++{ ++ struct kbase_context *kctx = data->kctx; ++ struct list_head *tmp; ++ ++ mutex_lock(&kctx->jit_evict_lock); ++ list_for_each(tmp, &kctx->jit_active_head) { ++ data->active_value++; ++ } ++ ++ list_for_each(tmp, &kctx->jit_pool_head) { ++ data->pool_value++; ++ } ++ ++ list_for_each(tmp, &kctx->jit_destroy_head) { ++ data->destroy_value++; ++ } ++ mutex_unlock(&kctx->jit_evict_lock); ++ ++ return 0; ++} ++KBASE_JIT_DEBUGFS_DECLARE(kbase_jit_debugfs_count_fops, ++ kbase_jit_debugfs_count_get); ++ ++static int kbase_jit_debugfs_vm_get(struct kbase_jit_debugfs_data *data) ++{ ++ struct kbase_context *kctx = data->kctx; ++ struct kbase_va_region *reg; ++ ++ mutex_lock(&kctx->jit_evict_lock); ++ list_for_each_entry(reg, &kctx->jit_active_head, jit_node) { ++ data->active_value += reg->nr_pages; ++ } ++ ++ list_for_each_entry(reg, &kctx->jit_pool_head, jit_node) { ++ data->pool_value += reg->nr_pages; ++ } ++ ++ list_for_each_entry(reg, &kctx->jit_destroy_head, jit_node) { ++ data->destroy_value += reg->nr_pages; ++ } ++ mutex_unlock(&kctx->jit_evict_lock); ++ ++ return 0; ++} ++KBASE_JIT_DEBUGFS_DECLARE(kbase_jit_debugfs_vm_fops, ++ kbase_jit_debugfs_vm_get); ++ ++static int kbase_jit_debugfs_phys_get(struct kbase_jit_debugfs_data *data) ++{ ++ struct kbase_context *kctx = data->kctx; ++ struct kbase_va_region *reg; ++ ++ mutex_lock(&kctx->jit_evict_lock); ++ list_for_each_entry(reg, &kctx->jit_active_head, jit_node) { ++ data->active_value += reg->gpu_alloc->nents; ++ } ++ ++ list_for_each_entry(reg, &kctx->jit_pool_head, jit_node) { ++ data->pool_value += reg->gpu_alloc->nents; ++ } ++ ++ list_for_each_entry(reg, &kctx->jit_destroy_head, jit_node) { ++ data->destroy_value += reg->gpu_alloc->nents; ++ } ++ mutex_unlock(&kctx->jit_evict_lock); ++ ++ return 0; ++} ++KBASE_JIT_DEBUGFS_DECLARE(kbase_jit_debugfs_phys_fops, ++ kbase_jit_debugfs_phys_get); ++ ++#if MALI_JIT_PRESSURE_LIMIT_BASE ++static int kbase_jit_debugfs_used_get(struct kbase_jit_debugfs_data *data) ++{ ++ struct kbase_context *kctx = data->kctx; ++ struct kbase_va_region *reg; ++ ++#if !MALI_USE_CSF ++ mutex_lock(&kctx->jctx.lock); ++#endif /* !MALI_USE_CSF */ ++ mutex_lock(&kctx->jit_evict_lock); ++ list_for_each_entry(reg, &kctx->jit_active_head, jit_node) { ++ data->active_value += reg->used_pages; ++ } ++ mutex_unlock(&kctx->jit_evict_lock); ++#if !MALI_USE_CSF ++ mutex_unlock(&kctx->jctx.lock); ++#endif /* !MALI_USE_CSF */ ++ ++ return 0; ++} ++ ++KBASE_JIT_DEBUGFS_DECLARE(kbase_jit_debugfs_used_fops, ++ kbase_jit_debugfs_used_get); ++ ++static int kbase_mem_jit_trim_pages_from_region(struct kbase_context *kctx, ++ struct kbase_va_region *reg, size_t pages_needed, ++ size_t *freed, bool shrink); ++ ++static int kbase_jit_debugfs_trim_get(struct kbase_jit_debugfs_data *data) ++{ ++ struct kbase_context *kctx = data->kctx; ++ struct kbase_va_region *reg; ++ ++#if !MALI_USE_CSF ++ mutex_lock(&kctx->jctx.lock); ++#endif /* !MALI_USE_CSF */ ++ kbase_gpu_vm_lock(kctx); ++ mutex_lock(&kctx->jit_evict_lock); ++ list_for_each_entry(reg, &kctx->jit_active_head, jit_node) { ++ int err; ++ size_t freed = 0u; ++ ++ err = kbase_mem_jit_trim_pages_from_region(kctx, reg, ++ SIZE_MAX, &freed, false); ++ ++ if (err) { ++ /* Failed to calculate, try the next region */ ++ continue; ++ } ++ ++ data->active_value += freed; ++ } ++ mutex_unlock(&kctx->jit_evict_lock); ++ kbase_gpu_vm_unlock(kctx); ++#if !MALI_USE_CSF ++ mutex_unlock(&kctx->jctx.lock); ++#endif /* !MALI_USE_CSF */ ++ ++ return 0; ++} ++ ++KBASE_JIT_DEBUGFS_DECLARE(kbase_jit_debugfs_trim_fops, ++ kbase_jit_debugfs_trim_get); ++#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ ++ ++void kbase_jit_debugfs_init(struct kbase_context *kctx) ++{ ++ /* prevent unprivileged use of debug file system ++ * in old kernel version ++ */ ++#if (KERNEL_VERSION(4, 7, 0) <= LINUX_VERSION_CODE) ++ /* only for newer kernel version debug file system is safe */ ++ const mode_t mode = 0444; ++#else ++ const mode_t mode = 0400; ++#endif ++ ++ /* Caller already ensures this, but we keep the pattern for ++ * maintenance safety. ++ */ ++ if (WARN_ON(!kctx) || ++ WARN_ON(IS_ERR_OR_NULL(kctx->kctx_dentry))) ++ return; ++ ++ ++ ++ /* Debugfs entry for getting the number of JIT allocations. */ ++ debugfs_create_file("mem_jit_count", mode, kctx->kctx_dentry, ++ kctx, &kbase_jit_debugfs_count_fops); ++ ++ /* ++ * Debugfs entry for getting the total number of virtual pages ++ * used by JIT allocations. ++ */ ++ debugfs_create_file("mem_jit_vm", mode, kctx->kctx_dentry, ++ kctx, &kbase_jit_debugfs_vm_fops); ++ ++ /* ++ * Debugfs entry for getting the number of physical pages used ++ * by JIT allocations. ++ */ ++ debugfs_create_file("mem_jit_phys", mode, kctx->kctx_dentry, ++ kctx, &kbase_jit_debugfs_phys_fops); ++#if MALI_JIT_PRESSURE_LIMIT_BASE ++ /* ++ * Debugfs entry for getting the number of pages used ++ * by JIT allocations for estimating the physical pressure ++ * limit. ++ */ ++ debugfs_create_file("mem_jit_used", mode, kctx->kctx_dentry, ++ kctx, &kbase_jit_debugfs_used_fops); ++ ++ /* ++ * Debugfs entry for getting the number of pages that could ++ * be trimmed to free space for more JIT allocations. ++ */ ++ debugfs_create_file("mem_jit_trim", mode, kctx->kctx_dentry, ++ kctx, &kbase_jit_debugfs_trim_fops); ++#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ ++} ++#endif /* CONFIG_DEBUG_FS */ ++ ++/** ++ * kbase_jit_destroy_worker - Deferred worker which frees JIT allocations ++ * @work: Work item ++ * ++ * This function does the work of freeing JIT allocations whose physical ++ * backing has been released. ++ */ ++static void kbase_jit_destroy_worker(struct work_struct *work) ++{ ++ struct kbase_context *kctx; ++ struct kbase_va_region *reg; ++ ++ kctx = container_of(work, struct kbase_context, jit_work); ++ do { ++ mutex_lock(&kctx->jit_evict_lock); ++ if (list_empty(&kctx->jit_destroy_head)) { ++ mutex_unlock(&kctx->jit_evict_lock); ++ break; ++ } ++ ++ reg = list_first_entry(&kctx->jit_destroy_head, ++ struct kbase_va_region, jit_node); ++ ++ list_del(®->jit_node); ++ mutex_unlock(&kctx->jit_evict_lock); ++ ++ kbase_gpu_vm_lock(kctx); ++ reg->flags &= ~KBASE_REG_NO_USER_FREE; ++ kbase_mem_free_region(kctx, reg); ++ kbase_gpu_vm_unlock(kctx); ++ } while (1); ++} ++ ++int kbase_jit_init(struct kbase_context *kctx) ++{ ++ mutex_lock(&kctx->jit_evict_lock); ++ INIT_LIST_HEAD(&kctx->jit_active_head); ++ INIT_LIST_HEAD(&kctx->jit_pool_head); ++ INIT_LIST_HEAD(&kctx->jit_destroy_head); ++ INIT_WORK(&kctx->jit_work, kbase_jit_destroy_worker); ++ ++#if MALI_USE_CSF ++ INIT_LIST_HEAD(&kctx->csf.kcpu_queues.jit_cmds_head); ++ INIT_LIST_HEAD(&kctx->csf.kcpu_queues.jit_blocked_queues); ++#else /* !MALI_USE_CSF */ ++ INIT_LIST_HEAD(&kctx->jctx.jit_atoms_head); ++ INIT_LIST_HEAD(&kctx->jctx.jit_pending_alloc); ++#endif /* MALI_USE_CSF */ ++ mutex_unlock(&kctx->jit_evict_lock); ++ ++ kctx->jit_max_allocations = 0; ++ kctx->jit_current_allocations = 0; ++ kctx->trim_level = 0; ++ ++ return 0; ++} ++ ++/* Check if the allocation from JIT pool is of the same size as the new JIT ++ * allocation and also, if BASE_JIT_ALLOC_MEM_TILER_ALIGN_TOP is set, meets ++ * the alignment requirements. ++ */ ++static bool meet_size_and_tiler_align_top_requirements( ++ const struct kbase_va_region *walker, ++ const struct base_jit_alloc_info *info) ++{ ++ bool meet_reqs = true; ++ ++ if (walker->nr_pages != info->va_pages) ++ meet_reqs = false; ++ ++#if !MALI_USE_CSF ++ if (meet_reqs && (info->flags & BASE_JIT_ALLOC_MEM_TILER_ALIGN_TOP)) { ++ size_t align = info->extent; ++ size_t align_mask = align - 1; ++ ++ if ((walker->start_pfn + info->commit_pages) & align_mask) ++ meet_reqs = false; ++ } ++#endif /* !MALI_USE_CSF */ ++ ++ return meet_reqs; ++} ++ ++#if MALI_JIT_PRESSURE_LIMIT_BASE ++/* Function will guarantee *@freed will not exceed @pages_needed ++ */ ++static int kbase_mem_jit_trim_pages_from_region(struct kbase_context *kctx, ++ struct kbase_va_region *reg, size_t pages_needed, ++ size_t *freed, bool shrink) ++{ ++ int err = 0; ++ size_t available_pages = 0u; ++ const size_t old_pages = kbase_reg_current_backed_size(reg); ++ size_t new_pages = old_pages; ++ size_t to_free = 0u; ++ size_t max_allowed_pages = old_pages; ++ ++#if !MALI_USE_CSF ++ lockdep_assert_held(&kctx->jctx.lock); ++#endif /* !MALI_USE_CSF */ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ /* Is this a JIT allocation that has been reported on? */ ++ if (reg->used_pages == reg->nr_pages) ++ goto out; ++ ++ if (!(reg->flags & KBASE_REG_HEAP_INFO_IS_SIZE)) { ++ /* For address based memory usage calculation, the GPU ++ * allocates objects of up to size 's', but aligns every object ++ * to alignment 'a', with a < s. ++ * ++ * It also doesn't have to write to all bytes in an object of ++ * size 's'. ++ * ++ * Hence, we can observe the GPU's address for the end of used ++ * memory being up to (s - a) bytes into the first unallocated ++ * page. ++ * ++ * We allow for this and only warn when it exceeds this bound ++ * (rounded up to page sized units). Note, this is allowed to ++ * exceed reg->nr_pages. ++ */ ++ max_allowed_pages += PFN_UP( ++ KBASE_GPU_ALLOCATED_OBJECT_MAX_BYTES - ++ KBASE_GPU_ALLOCATED_OBJECT_ALIGN_BYTES); ++ } else if (reg->flags & KBASE_REG_TILER_ALIGN_TOP) { ++ /* The GPU could report being ready to write to the next ++ * 'extent' sized chunk, but didn't actually write to it, so we ++ * can report up to 'extent' size pages more than the backed ++ * size. ++ * ++ * Note, this is allowed to exceed reg->nr_pages. ++ */ ++ max_allowed_pages += reg->extent; ++ ++ /* Also note that in these GPUs, the GPU may make a large (>1 ++ * page) initial allocation but not actually write out to all ++ * of it. Hence it might report that a much higher amount of ++ * memory was used than actually was written to. This does not ++ * result in a real warning because on growing this memory we ++ * round up the size of the allocation up to an 'extent' sized ++ * chunk, hence automatically bringing the backed size up to ++ * the reported size. ++ */ ++ } ++ ++ if (old_pages < reg->used_pages) { ++ /* Prevent overflow on available_pages, but only report the ++ * problem if it's in a scenario where used_pages should have ++ * been consistent with the backed size ++ * ++ * Note: In case of a size-based report, this legitimately ++ * happens in common use-cases: we allow for up to this size of ++ * memory being used, but depending on the content it doesn't ++ * have to use all of it. ++ * ++ * Hence, we're much more quiet about that in the size-based ++ * report case - it's not indicating a real problem, it's just ++ * for information ++ */ ++ if (max_allowed_pages < reg->used_pages) { ++ if (!(reg->flags & KBASE_REG_HEAP_INFO_IS_SIZE)) ++ dev_warn(kctx->kbdev->dev, ++ "%s: current backed pages %zu < reported used pages %zu (allowed to be up to %zu) on JIT 0x%llx vapages %zu\n", ++ __func__, ++ old_pages, reg->used_pages, ++ max_allowed_pages, ++ reg->start_pfn << PAGE_SHIFT, ++ reg->nr_pages); ++ else ++ dev_dbg(kctx->kbdev->dev, ++ "%s: no need to trim, current backed pages %zu < reported used pages %zu on size-report for JIT 0x%llx vapages %zu\n", ++ __func__, ++ old_pages, reg->used_pages, ++ reg->start_pfn << PAGE_SHIFT, ++ reg->nr_pages); ++ } ++ /* In any case, no error condition to report here, caller can ++ * try other regions ++ */ ++ ++ goto out; ++ } ++ available_pages = old_pages - reg->used_pages; ++ to_free = min(available_pages, pages_needed); ++ ++ if (shrink) { ++ new_pages -= to_free; ++ ++ err = kbase_mem_shrink(kctx, reg, new_pages); ++ } ++out: ++ trace_mali_jit_trim_from_region(reg, to_free, old_pages, ++ available_pages, new_pages); ++ *freed = to_free; ++ return err; ++} ++ ++ ++/** ++ * kbase_mem_jit_trim_pages - Trim JIT regions until sufficient pages have been ++ * freed ++ * @kctx: Pointer to the kbase context whose active JIT allocations will be ++ * checked. ++ * @pages_needed: The maximum number of pages to trim. ++ * ++ * This functions checks all active JIT allocations in @kctx for unused pages ++ * at the end, and trim the backed memory regions of those allocations down to ++ * the used portion and free the unused pages into the page pool. ++ * ++ * Specifying @pages_needed allows us to stop early when there's enough ++ * physical memory freed to sufficiently bring down the total JIT physical page ++ * usage (e.g. to below the pressure limit) ++ * ++ * Return: Total number of successfully freed pages ++ */ ++static size_t kbase_mem_jit_trim_pages(struct kbase_context *kctx, ++ size_t pages_needed) ++{ ++ struct kbase_va_region *reg, *tmp; ++ size_t total_freed = 0; ++ ++#if !MALI_USE_CSF ++ lockdep_assert_held(&kctx->jctx.lock); ++#endif /* !MALI_USE_CSF */ ++ lockdep_assert_held(&kctx->reg_lock); ++ lockdep_assert_held(&kctx->jit_evict_lock); ++ ++ list_for_each_entry_safe(reg, tmp, &kctx->jit_active_head, jit_node) { ++ int err; ++ size_t freed = 0u; ++ ++ err = kbase_mem_jit_trim_pages_from_region(kctx, reg, ++ pages_needed, &freed, true); ++ ++ if (err) { ++ /* Failed to trim, try the next region */ ++ continue; ++ } ++ ++ total_freed += freed; ++ WARN_ON(freed > pages_needed); ++ pages_needed -= freed; ++ if (!pages_needed) ++ break; ++ } ++ ++ trace_mali_jit_trim(total_freed); ++ ++ return total_freed; ++} ++#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ ++ ++static int kbase_jit_grow(struct kbase_context *kctx, ++ const struct base_jit_alloc_info *info, ++ struct kbase_va_region *reg, ++ struct kbase_sub_alloc **prealloc_sas) ++{ ++ size_t delta; ++ size_t pages_required; ++ size_t old_size; ++ struct kbase_mem_pool *pool; ++ int ret = -ENOMEM; ++ struct tagged_addr *gpu_pages; ++ ++ if (info->commit_pages > reg->nr_pages) { ++ /* Attempted to grow larger than maximum size */ ++ return -EINVAL; ++ } ++ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ /* Make the physical backing no longer reclaimable */ ++ if (!kbase_mem_evictable_unmake(reg->gpu_alloc)) ++ goto update_failed; ++ ++ if (reg->gpu_alloc->nents >= info->commit_pages) ++ goto done; ++ ++ /* Grow the backing */ ++ old_size = reg->gpu_alloc->nents; ++ ++ /* Allocate some more pages */ ++ delta = info->commit_pages - reg->gpu_alloc->nents; ++ pages_required = delta; ++ ++#ifdef CONFIG_MALI_2MB_ALLOC ++ if (pages_required >= (SZ_2M / SZ_4K)) { ++ pool = &kctx->mem_pools.large[kctx->jit_group_id]; ++ /* Round up to number of 2 MB pages required */ ++ pages_required += ((SZ_2M / SZ_4K) - 1); ++ pages_required /= (SZ_2M / SZ_4K); ++ } else { ++#endif ++ pool = &kctx->mem_pools.small[kctx->jit_group_id]; ++#ifdef CONFIG_MALI_2MB_ALLOC ++ } ++#endif ++ ++ if (reg->cpu_alloc != reg->gpu_alloc) ++ pages_required *= 2; ++ ++ spin_lock(&kctx->mem_partials_lock); ++ kbase_mem_pool_lock(pool); ++ ++ /* As we can not allocate memory from the kernel with the vm_lock held, ++ * grow the pool to the required size with the lock dropped. We hold the ++ * pool lock to prevent another thread from allocating from the pool ++ * between the grow and allocation. ++ */ ++ while (kbase_mem_pool_size(pool) < pages_required) { ++ int pool_delta = pages_required - kbase_mem_pool_size(pool); ++ int ret; ++ ++ kbase_mem_pool_unlock(pool); ++ spin_unlock(&kctx->mem_partials_lock); ++ ++ kbase_gpu_vm_unlock(kctx); ++ ret = kbase_mem_pool_grow(pool, pool_delta); ++ kbase_gpu_vm_lock(kctx); ++ ++ if (ret) ++ goto update_failed; ++ ++ spin_lock(&kctx->mem_partials_lock); ++ kbase_mem_pool_lock(pool); ++ } ++ ++ gpu_pages = kbase_alloc_phy_pages_helper_locked(reg->gpu_alloc, pool, ++ delta, &prealloc_sas[0]); ++ if (!gpu_pages) { ++ kbase_mem_pool_unlock(pool); ++ spin_unlock(&kctx->mem_partials_lock); ++ goto update_failed; ++ } ++ ++ if (reg->cpu_alloc != reg->gpu_alloc) { ++ struct tagged_addr *cpu_pages; ++ ++ cpu_pages = kbase_alloc_phy_pages_helper_locked(reg->cpu_alloc, ++ pool, delta, &prealloc_sas[1]); ++ if (!cpu_pages) { ++ kbase_free_phy_pages_helper_locked(reg->gpu_alloc, ++ pool, gpu_pages, delta); ++ kbase_mem_pool_unlock(pool); ++ spin_unlock(&kctx->mem_partials_lock); ++ goto update_failed; ++ } ++ } ++ kbase_mem_pool_unlock(pool); ++ spin_unlock(&kctx->mem_partials_lock); ++ ++ ret = kbase_mem_grow_gpu_mapping(kctx, reg, info->commit_pages, ++ old_size); ++ /* ++ * The grow failed so put the allocation back in the ++ * pool and return failure. ++ */ ++ if (ret) ++ goto update_failed; ++ ++done: ++ ret = 0; ++ ++ /* Update attributes of JIT allocation taken from the pool */ ++ reg->initial_commit = info->commit_pages; ++ reg->extent = info->extent; ++ ++update_failed: ++ return ret; ++} ++ ++static void trace_jit_stats(struct kbase_context *kctx, ++ u32 bin_id, u32 max_allocations) ++{ ++ const u32 alloc_count = ++ kctx->jit_current_allocations_per_bin[bin_id]; ++ struct kbase_device *kbdev = kctx->kbdev; ++ ++ struct kbase_va_region *walker; ++ u32 va_pages = 0; ++ u32 ph_pages = 0; ++ ++ mutex_lock(&kctx->jit_evict_lock); ++ list_for_each_entry(walker, &kctx->jit_active_head, jit_node) { ++ if (walker->jit_bin_id != bin_id) ++ continue; ++ ++ va_pages += walker->nr_pages; ++ ph_pages += walker->gpu_alloc->nents; ++ } ++ mutex_unlock(&kctx->jit_evict_lock); ++ ++ KBASE_TLSTREAM_AUX_JIT_STATS(kbdev, kctx->id, bin_id, ++ max_allocations, alloc_count, va_pages, ph_pages); ++} ++ ++#if MALI_JIT_PRESSURE_LIMIT_BASE ++/** ++ * get_jit_phys_backing() - calculate the physical backing of all JIT ++ * allocations ++ * ++ * @kctx: Pointer to the kbase context whose active JIT allocations will be ++ * checked ++ * ++ * Return: number of pages that are committed by JIT allocations ++ */ ++static size_t get_jit_phys_backing(struct kbase_context *kctx) ++{ ++ struct kbase_va_region *walker; ++ size_t backing = 0; ++ ++ lockdep_assert_held(&kctx->jit_evict_lock); ++ ++ list_for_each_entry(walker, &kctx->jit_active_head, jit_node) { ++ backing += kbase_reg_current_backed_size(walker); ++ } ++ ++ return backing; ++} ++ ++void kbase_jit_trim_necessary_pages(struct kbase_context *kctx, ++ size_t needed_pages) ++{ ++ size_t jit_backing = 0; ++ size_t pages_to_trim = 0; ++ ++#if !MALI_USE_CSF ++ lockdep_assert_held(&kctx->jctx.lock); ++#endif /* !MALI_USE_CSF */ ++ lockdep_assert_held(&kctx->reg_lock); ++ lockdep_assert_held(&kctx->jit_evict_lock); ++ ++ jit_backing = get_jit_phys_backing(kctx); ++ ++ /* It is possible that this is the case - if this is the first ++ * allocation after "ignore_pressure_limit" allocation. ++ */ ++ if (jit_backing > kctx->jit_phys_pages_limit) { ++ pages_to_trim += (jit_backing - kctx->jit_phys_pages_limit) + ++ needed_pages; ++ } else { ++ size_t backed_diff = kctx->jit_phys_pages_limit - jit_backing; ++ ++ if (needed_pages > backed_diff) ++ pages_to_trim += needed_pages - backed_diff; ++ } ++ ++ if (pages_to_trim) { ++ size_t trimmed_pages = ++ kbase_mem_jit_trim_pages(kctx, pages_to_trim); ++ ++ /* This should never happen - we already asserted that ++ * we are not violating JIT pressure limit in earlier ++ * checks, which means that in-flight JIT allocations ++ * must have enough unused pages to satisfy the new ++ * allocation ++ */ ++ WARN_ON(trimmed_pages < pages_to_trim); ++ } ++} ++#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ ++ ++/** ++ * jit_allow_allocate() - check whether basic conditions are satisfied to allow ++ * a new JIT allocation ++ * ++ * @kctx: Pointer to the kbase context ++ * @info: Pointer to JIT allocation information for the new allocation ++ * @ignore_pressure_limit: Flag to indicate whether JIT pressure limit check ++ * should be ignored ++ * ++ * Return: true if allocation can be executed, false otherwise ++ */ ++static bool jit_allow_allocate(struct kbase_context *kctx, ++ const struct base_jit_alloc_info *info, ++ bool ignore_pressure_limit) ++{ ++#if MALI_USE_CSF ++ lockdep_assert_held(&kctx->csf.kcpu_queues.lock); ++#else ++ lockdep_assert_held(&kctx->jctx.lock); ++#endif ++ ++#if MALI_JIT_PRESSURE_LIMIT_BASE ++ if (!ignore_pressure_limit && ++ ((kctx->jit_phys_pages_limit <= kctx->jit_current_phys_pressure) || ++ (info->va_pages > (kctx->jit_phys_pages_limit - kctx->jit_current_phys_pressure)))) { ++ dev_dbg(kctx->kbdev->dev, ++ "Max JIT page allocations limit reached: active pages %llu, max pages %llu\n", ++ kctx->jit_current_phys_pressure + info->va_pages, ++ kctx->jit_phys_pages_limit); ++ return false; ++ } ++#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ ++ ++ if (kctx->jit_current_allocations >= kctx->jit_max_allocations) { ++ /* Too many current allocations */ ++ dev_dbg(kctx->kbdev->dev, ++ "Max JIT allocations limit reached: active allocations %d, max allocations %d\n", ++ kctx->jit_current_allocations, ++ kctx->jit_max_allocations); ++ return false; ++ } ++ ++ if (info->max_allocations > 0 && ++ kctx->jit_current_allocations_per_bin[info->bin_id] >= ++ info->max_allocations) { ++ /* Too many current allocations in this bin */ ++ dev_dbg(kctx->kbdev->dev, ++ "Per bin limit of max JIT allocations reached: bin_id %d, active allocations %d, max allocations %d\n", ++ info->bin_id, ++ kctx->jit_current_allocations_per_bin[info->bin_id], ++ info->max_allocations); ++ return false; ++ } ++ ++ return true; ++} ++ ++static struct kbase_va_region * ++find_reasonable_region(const struct base_jit_alloc_info *info, ++ struct list_head *pool_head, bool ignore_usage_id) ++{ ++ struct kbase_va_region *closest_reg = NULL; ++ struct kbase_va_region *walker; ++ size_t current_diff = SIZE_MAX; ++ ++ list_for_each_entry(walker, pool_head, jit_node) { ++ if ((ignore_usage_id || ++ walker->jit_usage_id == info->usage_id) && ++ walker->jit_bin_id == info->bin_id && ++ meet_size_and_tiler_align_top_requirements(walker, info)) { ++ size_t min_size, max_size, diff; ++ ++ /* ++ * The JIT allocations VA requirements have been met, ++ * it's suitable but other allocations might be a ++ * better fit. ++ */ ++ min_size = min_t(size_t, walker->gpu_alloc->nents, ++ info->commit_pages); ++ max_size = max_t(size_t, walker->gpu_alloc->nents, ++ info->commit_pages); ++ diff = max_size - min_size; ++ ++ if (current_diff > diff) { ++ current_diff = diff; ++ closest_reg = walker; ++ } ++ ++ /* The allocation is an exact match */ ++ if (current_diff == 0) ++ break; ++ } ++ } ++ ++ return closest_reg; ++} ++ ++struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx, ++ const struct base_jit_alloc_info *info, ++ bool ignore_pressure_limit) ++{ ++ struct kbase_va_region *reg = NULL; ++ struct kbase_sub_alloc *prealloc_sas[2] = { NULL, NULL }; ++ int i; ++ ++#if MALI_USE_CSF ++ lockdep_assert_held(&kctx->csf.kcpu_queues.lock); ++#else ++ lockdep_assert_held(&kctx->jctx.lock); ++#endif ++ ++ if (!jit_allow_allocate(kctx, info, ignore_pressure_limit)) ++ return NULL; ++ ++#ifdef CONFIG_MALI_2MB_ALLOC ++ /* Preallocate memory for the sub-allocation structs */ ++ for (i = 0; i != ARRAY_SIZE(prealloc_sas); ++i) { ++ prealloc_sas[i] = kmalloc(sizeof(*prealloc_sas[i]), GFP_KERNEL); ++ if (!prealloc_sas[i]) ++ goto end; ++ } ++#endif ++ ++ kbase_gpu_vm_lock(kctx); ++ mutex_lock(&kctx->jit_evict_lock); ++ ++ /* ++ * Scan the pool for an existing allocation which meets our ++ * requirements and remove it. ++ */ ++ if (info->usage_id != 0) ++ /* First scan for an allocation with the same usage ID */ ++ reg = find_reasonable_region(info, &kctx->jit_pool_head, false); ++ ++ if (!reg) ++ /* No allocation with the same usage ID, or usage IDs not in ++ * use. Search for an allocation we can reuse. ++ */ ++ reg = find_reasonable_region(info, &kctx->jit_pool_head, true); ++ ++ if (reg) { ++#if MALI_JIT_PRESSURE_LIMIT_BASE ++ size_t needed_pages = 0; ++#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ ++ int ret; ++ ++ /* ++ * Remove the found region from the pool and add it to the ++ * active list. ++ */ ++ list_move(®->jit_node, &kctx->jit_active_head); ++ ++ WARN_ON(reg->gpu_alloc->evicted); ++ ++ /* ++ * Remove the allocation from the eviction list as it's no ++ * longer eligible for eviction. This must be done before ++ * dropping the jit_evict_lock ++ */ ++ list_del_init(®->gpu_alloc->evict_node); ++ ++#if MALI_JIT_PRESSURE_LIMIT_BASE ++ if (!ignore_pressure_limit) { ++ if (info->commit_pages > reg->gpu_alloc->nents) ++ needed_pages = info->commit_pages - ++ reg->gpu_alloc->nents; ++ ++ /* Update early the recycled JIT region's estimate of ++ * used_pages to ensure it doesn't get trimmed ++ * undesirably. This is needed as the recycled JIT ++ * region has been added to the active list but the ++ * number of used pages for it would be zero, so it ++ * could get trimmed instead of other allocations only ++ * to be regrown later resulting in a breach of the JIT ++ * physical pressure limit. ++ * Also that trimming would disturb the accounting of ++ * physical pages, i.e. the VM stats, as the number of ++ * backing pages would have changed when the call to ++ * kbase_mem_evictable_unmark_reclaim is made. ++ * ++ * The second call to update pressure at the end of ++ * this function would effectively be a nop. ++ */ ++ kbase_jit_report_update_pressure( ++ kctx, reg, info->va_pages, ++ KBASE_JIT_REPORT_ON_ALLOC_OR_FREE); ++ ++ kbase_jit_request_phys_increase_locked(kctx, ++ needed_pages); ++ } ++#endif ++ mutex_unlock(&kctx->jit_evict_lock); ++ ++ /* kbase_jit_grow() can release & reacquire 'kctx->reg_lock', ++ * so any state protected by that lock might need to be ++ * re-evaluated if more code is added here in future. ++ */ ++ ret = kbase_jit_grow(kctx, info, reg, prealloc_sas); ++ ++#if MALI_JIT_PRESSURE_LIMIT_BASE ++ if (!ignore_pressure_limit) ++ kbase_jit_done_phys_increase(kctx, needed_pages); ++#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ ++ ++ kbase_gpu_vm_unlock(kctx); ++ ++ if (ret < 0) { ++ /* ++ * An update to an allocation from the pool failed, ++ * chances are slim a new allocation would fair any ++ * better so return the allocation to the pool and ++ * return the function with failure. ++ */ ++ dev_dbg(kctx->kbdev->dev, ++ "JIT allocation resize failed: va_pages 0x%llx, commit_pages 0x%llx\n", ++ info->va_pages, info->commit_pages); ++#if MALI_JIT_PRESSURE_LIMIT_BASE ++ /* Undo the early change made to the recycled JIT ++ * region's estimate of used_pages. ++ */ ++ if (!ignore_pressure_limit) { ++ kbase_jit_report_update_pressure( ++ kctx, reg, 0, ++ KBASE_JIT_REPORT_ON_ALLOC_OR_FREE); ++ } ++#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ ++ mutex_lock(&kctx->jit_evict_lock); ++ list_move(®->jit_node, &kctx->jit_pool_head); ++ mutex_unlock(&kctx->jit_evict_lock); ++ reg = NULL; ++ goto end; ++ } ++ } else { ++ /* No suitable JIT allocation was found so create a new one */ ++ u64 flags = BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_GPU_RD | ++ BASE_MEM_PROT_GPU_WR | BASE_MEM_GROW_ON_GPF | ++ BASE_MEM_COHERENT_LOCAL | ++ BASEP_MEM_NO_USER_FREE; ++ u64 gpu_addr; ++ ++#if !MALI_USE_CSF ++ if (info->flags & BASE_JIT_ALLOC_MEM_TILER_ALIGN_TOP) ++ flags |= BASE_MEM_TILER_ALIGN_TOP; ++#endif /* !MALI_USE_CSF */ ++ ++ flags |= base_mem_group_id_set(kctx->jit_group_id); ++#if MALI_JIT_PRESSURE_LIMIT_BASE ++ if (!ignore_pressure_limit) { ++ flags |= BASEP_MEM_PERFORM_JIT_TRIM; ++ /* The corresponding call to 'done_phys_increase' would ++ * be made inside the kbase_mem_alloc(). ++ */ ++ kbase_jit_request_phys_increase_locked( ++ kctx, info->commit_pages); ++ } ++#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ ++ ++ mutex_unlock(&kctx->jit_evict_lock); ++ kbase_gpu_vm_unlock(kctx); ++ ++ reg = kbase_mem_alloc(kctx, info->va_pages, info->commit_pages, ++ info->extent, &flags, &gpu_addr); ++ if (!reg) { ++ /* Most likely not enough GPU virtual space left for ++ * the new JIT allocation. ++ */ ++ dev_dbg(kctx->kbdev->dev, ++ "Failed to allocate JIT memory: va_pages 0x%llx, commit_pages 0x%llx\n", ++ info->va_pages, info->commit_pages); ++ goto end; ++ } ++ ++ if (!ignore_pressure_limit) { ++ /* Due to enforcing of pressure limit, kbase_mem_alloc ++ * was instructed to perform the trimming which in turn ++ * would have ensured that the new JIT allocation is ++ * already in the jit_active_head list, so nothing to ++ * do here. ++ */ ++ WARN_ON(list_empty(®->jit_node)); ++ } else { ++ mutex_lock(&kctx->jit_evict_lock); ++ list_add(®->jit_node, &kctx->jit_active_head); ++ mutex_unlock(&kctx->jit_evict_lock); ++ } ++ } ++ ++ trace_mali_jit_alloc(reg, info->id); ++ ++ kctx->jit_current_allocations++; ++ kctx->jit_current_allocations_per_bin[info->bin_id]++; ++ ++ trace_jit_stats(kctx, info->bin_id, info->max_allocations); ++ ++ reg->jit_usage_id = info->usage_id; ++ reg->jit_bin_id = info->bin_id; ++ reg->flags |= KBASE_REG_ACTIVE_JIT_ALLOC; ++#if MALI_JIT_PRESSURE_LIMIT_BASE ++ if (info->flags & BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE) ++ reg->flags = reg->flags | KBASE_REG_HEAP_INFO_IS_SIZE; ++ reg->heap_info_gpu_addr = info->heap_info_gpu_addr; ++ kbase_jit_report_update_pressure(kctx, reg, info->va_pages, ++ KBASE_JIT_REPORT_ON_ALLOC_OR_FREE); ++#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ ++ ++end: ++ for (i = 0; i != ARRAY_SIZE(prealloc_sas); ++i) ++ kfree(prealloc_sas[i]); ++ ++ return reg; ++} ++ ++void kbase_jit_free(struct kbase_context *kctx, struct kbase_va_region *reg) ++{ ++ u64 old_pages; ++ ++ /* JIT id not immediately available here, so use 0u */ ++ trace_mali_jit_free(reg, 0u); ++ ++ /* Get current size of JIT region */ ++ old_pages = kbase_reg_current_backed_size(reg); ++ if (reg->initial_commit < old_pages) { ++ /* Free trim_level % of region, but don't go below initial ++ * commit size ++ */ ++ u64 new_size = MAX(reg->initial_commit, ++ div_u64(old_pages * (100 - kctx->trim_level), 100)); ++ u64 delta = old_pages - new_size; ++ ++ if (delta) ++ kbase_mem_shrink(kctx, reg, old_pages - delta); ++ } ++ ++#if MALI_JIT_PRESSURE_LIMIT_BASE ++ reg->heap_info_gpu_addr = 0; ++ kbase_jit_report_update_pressure(kctx, reg, 0, ++ KBASE_JIT_REPORT_ON_ALLOC_OR_FREE); ++#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ ++ ++ kctx->jit_current_allocations--; ++ kctx->jit_current_allocations_per_bin[reg->jit_bin_id]--; ++ ++ trace_jit_stats(kctx, reg->jit_bin_id, UINT_MAX); ++ ++ kbase_mem_evictable_mark_reclaim(reg->gpu_alloc); ++ ++ kbase_gpu_vm_lock(kctx); ++ reg->flags |= KBASE_REG_DONT_NEED; ++ reg->flags &= ~KBASE_REG_ACTIVE_JIT_ALLOC; ++ kbase_mem_shrink_cpu_mapping(kctx, reg, 0, reg->gpu_alloc->nents); ++ kbase_gpu_vm_unlock(kctx); ++ ++ /* ++ * Add the allocation to the eviction list and the jit pool, after this ++ * point the shrink can reclaim it, or it may be reused. ++ */ ++ mutex_lock(&kctx->jit_evict_lock); ++ ++ /* This allocation can't already be on a list. */ ++ WARN_ON(!list_empty(®->gpu_alloc->evict_node)); ++ list_add(®->gpu_alloc->evict_node, &kctx->evict_list); ++ ++ list_move(®->jit_node, &kctx->jit_pool_head); ++ ++ mutex_unlock(&kctx->jit_evict_lock); ++} ++ ++void kbase_jit_backing_lost(struct kbase_va_region *reg) ++{ ++ struct kbase_context *kctx = kbase_reg_flags_to_kctx(reg); ++ ++ if (WARN_ON(!kctx)) ++ return; ++ ++ lockdep_assert_held(&kctx->jit_evict_lock); ++ ++ /* ++ * JIT allocations will always be on a list, if the region ++ * is not on a list then it's not a JIT allocation. ++ */ ++ if (list_empty(®->jit_node)) ++ return; ++ ++ /* ++ * Freeing the allocation requires locks we might not be able ++ * to take now, so move the allocation to the free list and kick ++ * the worker which will do the freeing. ++ */ ++ list_move(®->jit_node, &kctx->jit_destroy_head); ++ ++ schedule_work(&kctx->jit_work); ++} ++ ++bool kbase_jit_evict(struct kbase_context *kctx) ++{ ++ struct kbase_va_region *reg = NULL; ++ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ /* Free the oldest allocation from the pool */ ++ mutex_lock(&kctx->jit_evict_lock); ++ if (!list_empty(&kctx->jit_pool_head)) { ++ reg = list_entry(kctx->jit_pool_head.prev, ++ struct kbase_va_region, jit_node); ++ list_del(®->jit_node); ++ list_del_init(®->gpu_alloc->evict_node); ++ } ++ mutex_unlock(&kctx->jit_evict_lock); ++ ++ if (reg) { ++ reg->flags &= ~KBASE_REG_NO_USER_FREE; ++ kbase_mem_free_region(kctx, reg); ++ } ++ ++ return (reg != NULL); ++} ++ ++void kbase_jit_term(struct kbase_context *kctx) ++{ ++ struct kbase_va_region *walker; ++ ++ /* Free all allocations for this context */ ++ ++ kbase_gpu_vm_lock(kctx); ++ mutex_lock(&kctx->jit_evict_lock); ++ /* Free all allocations from the pool */ ++ while (!list_empty(&kctx->jit_pool_head)) { ++ walker = list_first_entry(&kctx->jit_pool_head, ++ struct kbase_va_region, jit_node); ++ list_del(&walker->jit_node); ++ list_del_init(&walker->gpu_alloc->evict_node); ++ mutex_unlock(&kctx->jit_evict_lock); ++ walker->flags &= ~KBASE_REG_NO_USER_FREE; ++ kbase_mem_free_region(kctx, walker); ++ mutex_lock(&kctx->jit_evict_lock); ++ } ++ ++ /* Free all allocations from active list */ ++ while (!list_empty(&kctx->jit_active_head)) { ++ walker = list_first_entry(&kctx->jit_active_head, ++ struct kbase_va_region, jit_node); ++ list_del(&walker->jit_node); ++ list_del_init(&walker->gpu_alloc->evict_node); ++ mutex_unlock(&kctx->jit_evict_lock); ++ walker->flags &= ~KBASE_REG_NO_USER_FREE; ++ kbase_mem_free_region(kctx, walker); ++ mutex_lock(&kctx->jit_evict_lock); ++ } ++#if MALI_JIT_PRESSURE_LIMIT_BASE ++ WARN_ON(kctx->jit_phys_pages_to_be_allocated); ++#endif ++ mutex_unlock(&kctx->jit_evict_lock); ++ kbase_gpu_vm_unlock(kctx); ++ ++ /* ++ * Flush the freeing of allocations whose backing has been freed ++ * (i.e. everything in jit_destroy_head). ++ */ ++ cancel_work_sync(&kctx->jit_work); ++} ++ ++#if MALI_JIT_PRESSURE_LIMIT_BASE ++void kbase_trace_jit_report_gpu_mem_trace_enabled(struct kbase_context *kctx, ++ struct kbase_va_region *reg, unsigned int flags) ++{ ++ /* Offset to the location used for a JIT report within the GPU memory ++ * ++ * This constants only used for this debugging function - not useful ++ * anywhere else in kbase ++ */ ++ const u64 jit_report_gpu_mem_offset = sizeof(u64)*2; ++ ++ u64 addr_start; ++ struct kbase_vmap_struct mapping; ++ u64 *ptr; ++ ++ if (reg->heap_info_gpu_addr == 0ull) ++ goto out; ++ ++ /* Nothing else to trace in the case the memory just contains the ++ * size. Other tracepoints already record the relevant area of memory. ++ */ ++ if (reg->flags & KBASE_REG_HEAP_INFO_IS_SIZE) ++ goto out; ++ ++ addr_start = reg->heap_info_gpu_addr - jit_report_gpu_mem_offset; ++ ++ ptr = kbase_vmap(kctx, addr_start, KBASE_JIT_REPORT_GPU_MEM_SIZE, ++ &mapping); ++ if (!ptr) { ++ dev_warn(kctx->kbdev->dev, ++ "%s: JIT start=0x%llx unable to map memory near end pointer %llx\n", ++ __func__, reg->start_pfn << PAGE_SHIFT, ++ addr_start); ++ goto out; ++ } ++ ++ trace_mali_jit_report_gpu_mem(addr_start, reg->start_pfn << PAGE_SHIFT, ++ ptr, flags); ++ ++ kbase_vunmap(kctx, &mapping); ++out: ++ return; ++} ++#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ ++ ++#if MALI_JIT_PRESSURE_LIMIT_BASE ++void kbase_jit_report_update_pressure(struct kbase_context *kctx, ++ struct kbase_va_region *reg, u64 new_used_pages, ++ unsigned int flags) ++{ ++ u64 diff; ++ ++#if !MALI_USE_CSF ++ lockdep_assert_held(&kctx->jctx.lock); ++#endif /* !MALI_USE_CSF */ ++ ++ trace_mali_jit_report_pressure(reg, new_used_pages, ++ kctx->jit_current_phys_pressure + new_used_pages - ++ reg->used_pages, ++ flags); ++ ++ if (WARN_ON(new_used_pages > reg->nr_pages)) ++ return; ++ ++ if (reg->used_pages > new_used_pages) { ++ /* We reduced the number of used pages */ ++ diff = reg->used_pages - new_used_pages; ++ ++ if (!WARN_ON(diff > kctx->jit_current_phys_pressure)) ++ kctx->jit_current_phys_pressure -= diff; ++ ++ reg->used_pages = new_used_pages; ++ } else { ++ /* We increased the number of used pages */ ++ diff = new_used_pages - reg->used_pages; ++ ++ if (!WARN_ON(diff > U64_MAX - kctx->jit_current_phys_pressure)) ++ kctx->jit_current_phys_pressure += diff; ++ ++ reg->used_pages = new_used_pages; ++ } ++ ++} ++#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ ++ ++bool kbase_has_exec_va_zone(struct kbase_context *kctx) ++{ ++ bool has_exec_va_zone; ++ ++ kbase_gpu_vm_lock(kctx); ++ has_exec_va_zone = (kctx->exec_va_start != U64_MAX); ++ kbase_gpu_vm_unlock(kctx); ++ ++ return has_exec_va_zone; ++} ++ ++#if MALI_USE_CSF ++static void kbase_jd_user_buf_unpin_pages(struct kbase_mem_phy_alloc *alloc) ++{ ++ if (alloc->nents) { ++ struct page **pages = alloc->imported.user_buf.pages; ++ long i; ++ ++ WARN_ON(alloc->nents != alloc->imported.user_buf.nr_pages); ++ ++ for (i = 0; i < alloc->nents; i++) ++ put_page(pages[i]); ++ } ++} ++#endif ++ ++int kbase_jd_user_buf_pin_pages(struct kbase_context *kctx, ++ struct kbase_va_region *reg) ++{ ++ struct kbase_mem_phy_alloc *alloc = reg->gpu_alloc; ++ struct page **pages = alloc->imported.user_buf.pages; ++ unsigned long address = alloc->imported.user_buf.address; ++ struct mm_struct *mm = alloc->imported.user_buf.mm; ++ long pinned_pages; ++ long i; ++ ++ if (WARN_ON(alloc->type != KBASE_MEM_TYPE_IMPORTED_USER_BUF)) ++ return -EINVAL; ++ ++ if (alloc->nents) { ++ if (WARN_ON(alloc->nents != alloc->imported.user_buf.nr_pages)) ++ return -EINVAL; ++ else ++ return 0; ++ } ++ ++ if (WARN_ON(reg->gpu_alloc->imported.user_buf.mm != current->mm)) ++ return -EINVAL; ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) ++ pinned_pages = get_user_pages(NULL, mm, ++ address, ++ alloc->imported.user_buf.nr_pages, ++#if KERNEL_VERSION(4, 4, 168) <= LINUX_VERSION_CODE && \ ++KERNEL_VERSION(4, 5, 0) > LINUX_VERSION_CODE ++ reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0, ++ pages, NULL); ++#else ++ reg->flags & KBASE_REG_GPU_WR, ++ 0, pages, NULL); ++#endif ++#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) ++ pinned_pages = get_user_pages_remote(NULL, mm, ++ address, ++ alloc->imported.user_buf.nr_pages, ++ reg->flags & KBASE_REG_GPU_WR, ++ 0, pages, NULL); ++#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) ++ pinned_pages = get_user_pages_remote(NULL, mm, ++ address, ++ alloc->imported.user_buf.nr_pages, ++ reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0, ++ pages, NULL); ++#elif LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0) ++ pinned_pages = get_user_pages_remote(NULL, mm, ++ address, ++ alloc->imported.user_buf.nr_pages, ++ reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0, ++ pages, NULL, NULL); ++#else ++ pinned_pages = get_user_pages_remote(mm, ++ address, ++ alloc->imported.user_buf.nr_pages, ++ reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0, ++ pages, NULL, NULL); ++#endif ++ ++ if (pinned_pages <= 0) ++ return pinned_pages; ++ ++ if (pinned_pages != alloc->imported.user_buf.nr_pages) { ++ for (i = 0; i < pinned_pages; i++) ++ put_page(pages[i]); ++ return -ENOMEM; ++ } ++ ++ alloc->nents = pinned_pages; ++ ++ return 0; ++} ++ ++static int kbase_jd_user_buf_map(struct kbase_context *kctx, ++ struct kbase_va_region *reg) ++{ ++ long pinned_pages; ++ struct kbase_mem_phy_alloc *alloc; ++ struct page **pages; ++ struct tagged_addr *pa; ++ long i; ++ unsigned long address; ++ struct device *dev; ++ unsigned long offset; ++ unsigned long local_size; ++ unsigned long gwt_mask = ~0; ++ int err = kbase_jd_user_buf_pin_pages(kctx, reg); ++ ++ if (err) ++ return err; ++ ++ alloc = reg->gpu_alloc; ++ pa = kbase_get_gpu_phy_pages(reg); ++ address = alloc->imported.user_buf.address; ++ pinned_pages = alloc->nents; ++ pages = alloc->imported.user_buf.pages; ++ dev = kctx->kbdev->dev; ++ offset = address & ~PAGE_MASK; ++ local_size = alloc->imported.user_buf.size; ++ ++ for (i = 0; i < pinned_pages; i++) { ++ dma_addr_t dma_addr; ++ unsigned long min; ++ ++ min = MIN(PAGE_SIZE - offset, local_size); ++ dma_addr = dma_map_page(dev, pages[i], ++ offset, min, ++ DMA_BIDIRECTIONAL); ++ if (dma_mapping_error(dev, dma_addr)) ++ goto unwind; ++ ++ alloc->imported.user_buf.dma_addrs[i] = dma_addr; ++ pa[i] = as_tagged(page_to_phys(pages[i])); ++ ++ local_size -= min; ++ offset = 0; ++ } ++ ++#ifdef CONFIG_MALI_CINSTR_GWT ++ if (kctx->gwt_enabled) ++ gwt_mask = ~KBASE_REG_GPU_WR; ++#endif ++ ++ err = kbase_mmu_insert_pages(kctx->kbdev, &kctx->mmu, reg->start_pfn, ++ pa, kbase_reg_current_backed_size(reg), ++ reg->flags & gwt_mask, kctx->as_nr, ++ alloc->group_id); ++ if (err == 0) ++ return 0; ++ ++ /* fall down */ ++unwind: ++ alloc->nents = 0; ++ while (i--) { ++ dma_unmap_page(kctx->kbdev->dev, ++ alloc->imported.user_buf.dma_addrs[i], ++ PAGE_SIZE, DMA_BIDIRECTIONAL); ++ } ++ ++ while (++i < pinned_pages) { ++ put_page(pages[i]); ++ pages[i] = NULL; ++ } ++ ++ return err; ++} ++ ++/* This function would also perform the work of unpinning pages on Job Manager ++ * GPUs, which implies that a call to kbase_jd_user_buf_pin_pages() will NOT ++ * have a corresponding call to kbase_jd_user_buf_unpin_pages(). ++ */ ++static void kbase_jd_user_buf_unmap(struct kbase_context *kctx, ++ struct kbase_mem_phy_alloc *alloc, bool writeable) ++{ ++ long i; ++ struct page **pages; ++ unsigned long size = alloc->imported.user_buf.size; ++ ++ KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_IMPORTED_USER_BUF); ++ pages = alloc->imported.user_buf.pages; ++ for (i = 0; i < alloc->imported.user_buf.nr_pages; i++) { ++ unsigned long local_size; ++ dma_addr_t dma_addr = alloc->imported.user_buf.dma_addrs[i]; ++ ++ local_size = MIN(size, PAGE_SIZE - (dma_addr & ~PAGE_MASK)); ++ dma_unmap_page(kctx->kbdev->dev, dma_addr, local_size, ++ DMA_BIDIRECTIONAL); ++ if (writeable) ++ set_page_dirty_lock(pages[i]); ++#if !MALI_USE_CSF ++ put_page(pages[i]); ++ pages[i] = NULL; ++#endif ++ ++ size -= local_size; ++ } ++#if !MALI_USE_CSF ++ alloc->nents = 0; ++#endif ++} ++ ++int kbase_mem_copy_to_pinned_user_pages(struct page **dest_pages, ++ void *src_page, size_t *to_copy, unsigned int nr_pages, ++ unsigned int *target_page_nr, size_t offset) ++{ ++ void *target_page = kmap(dest_pages[*target_page_nr]); ++ size_t chunk = PAGE_SIZE-offset; ++ ++ if (!target_page) { ++ pr_err("%s: kmap failure", __func__); ++ return -ENOMEM; ++ } ++ ++ chunk = min(chunk, *to_copy); ++ ++ memcpy(target_page + offset, src_page, chunk); ++ *to_copy -= chunk; ++ ++ kunmap(dest_pages[*target_page_nr]); ++ ++ *target_page_nr += 1; ++ if (*target_page_nr >= nr_pages || *to_copy == 0) ++ return 0; ++ ++ target_page = kmap(dest_pages[*target_page_nr]); ++ if (!target_page) { ++ pr_err("%s: kmap failure", __func__); ++ return -ENOMEM; ++ } ++ ++ KBASE_DEBUG_ASSERT(target_page); ++ ++ chunk = min(offset, *to_copy); ++ memcpy(target_page, src_page + PAGE_SIZE-offset, chunk); ++ *to_copy -= chunk; ++ ++ kunmap(dest_pages[*target_page_nr]); ++ ++ return 0; ++} ++ ++struct kbase_mem_phy_alloc *kbase_map_external_resource( ++ struct kbase_context *kctx, struct kbase_va_region *reg, ++ struct mm_struct *locked_mm) ++{ ++ int err; ++ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ /* decide what needs to happen for this resource */ ++ switch (reg->gpu_alloc->type) { ++ case KBASE_MEM_TYPE_IMPORTED_USER_BUF: { ++ if ((reg->gpu_alloc->imported.user_buf.mm != locked_mm) && ++ (!reg->gpu_alloc->nents)) ++ goto exit; ++ ++ reg->gpu_alloc->imported.user_buf.current_mapping_usage_count++; ++ if (1 == reg->gpu_alloc->imported.user_buf.current_mapping_usage_count) { ++ err = kbase_jd_user_buf_map(kctx, reg); ++ if (err) { ++ reg->gpu_alloc->imported.user_buf.current_mapping_usage_count--; ++ goto exit; ++ } ++ } ++ } ++ break; ++ case KBASE_MEM_TYPE_IMPORTED_UMM: { ++ err = kbase_mem_umm_map(kctx, reg); ++ if (err) ++ goto exit; ++ break; ++ } ++ default: ++ goto exit; ++ } ++ ++ return kbase_mem_phy_alloc_get(reg->gpu_alloc); ++exit: ++ return NULL; ++} ++ ++void kbase_unmap_external_resource(struct kbase_context *kctx, ++ struct kbase_va_region *reg, struct kbase_mem_phy_alloc *alloc) ++{ ++ switch (alloc->type) { ++ case KBASE_MEM_TYPE_IMPORTED_UMM: { ++ kbase_mem_umm_unmap(kctx, reg, alloc); ++ } ++ break; ++ case KBASE_MEM_TYPE_IMPORTED_USER_BUF: { ++ alloc->imported.user_buf.current_mapping_usage_count--; ++ ++ if (0 == alloc->imported.user_buf.current_mapping_usage_count) { ++ bool writeable = true; ++ ++ if (!kbase_is_region_invalid_or_free(reg) && ++ reg->gpu_alloc == alloc) ++ kbase_mmu_teardown_pages( ++ kctx->kbdev, ++ &kctx->mmu, ++ reg->start_pfn, ++ kbase_reg_current_backed_size(reg), ++ kctx->as_nr); ++ ++ if (reg && ((reg->flags & KBASE_REG_GPU_WR) == 0)) ++ writeable = false; ++ ++ kbase_jd_user_buf_unmap(kctx, alloc, writeable); ++ } ++ } ++ break; ++ default: ++ break; ++ } ++ kbase_mem_phy_alloc_put(alloc); ++} ++ ++struct kbase_ctx_ext_res_meta *kbase_sticky_resource_acquire( ++ struct kbase_context *kctx, u64 gpu_addr) ++{ ++ struct kbase_ctx_ext_res_meta *meta = NULL; ++ struct kbase_ctx_ext_res_meta *walker; ++ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ /* ++ * Walk the per context external resource metadata list for the ++ * metadata which matches the region which is being acquired. ++ */ ++ list_for_each_entry(walker, &kctx->ext_res_meta_head, ext_res_node) { ++ if (walker->gpu_addr == gpu_addr) { ++ meta = walker; ++ meta->ref++; ++ break; ++ } ++ } ++ ++ /* No metadata exists so create one. */ ++ if (!meta) { ++ struct kbase_va_region *reg; ++ ++ /* Find the region */ ++ reg = kbase_region_tracker_find_region_enclosing_address( ++ kctx, gpu_addr); ++ if (kbase_is_region_invalid_or_free(reg)) ++ goto failed; ++ ++ /* Allocate the metadata object */ ++ meta = kzalloc(sizeof(*meta), GFP_KERNEL); ++ if (!meta) ++ goto failed; ++ ++ /* ++ * Fill in the metadata object and acquire a reference ++ * for the physical resource. ++ */ ++ meta->alloc = kbase_map_external_resource(kctx, reg, NULL); ++ meta->ref = 1; ++ ++ if (!meta->alloc) ++ goto fail_map; ++ ++ meta->gpu_addr = reg->start_pfn << PAGE_SHIFT; ++ ++ list_add(&meta->ext_res_node, &kctx->ext_res_meta_head); ++ } ++ ++ return meta; ++ ++fail_map: ++ kfree(meta); ++failed: ++ return NULL; ++} ++ ++static struct kbase_ctx_ext_res_meta * ++find_sticky_resource_meta(struct kbase_context *kctx, u64 gpu_addr) ++{ ++ struct kbase_ctx_ext_res_meta *walker; ++ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ /* ++ * Walk the per context external resource metadata list for the ++ * metadata which matches the region which is being released. ++ */ ++ list_for_each_entry(walker, &kctx->ext_res_meta_head, ext_res_node) ++ if (walker->gpu_addr == gpu_addr) ++ return walker; ++ ++ return NULL; ++} ++ ++static void release_sticky_resource_meta(struct kbase_context *kctx, ++ struct kbase_ctx_ext_res_meta *meta) ++{ ++ struct kbase_va_region *reg; ++ ++ /* Drop the physical memory reference and free the metadata. */ ++ reg = kbase_region_tracker_find_region_enclosing_address( ++ kctx, ++ meta->gpu_addr); ++ ++ kbase_unmap_external_resource(kctx, reg, meta->alloc); ++ list_del(&meta->ext_res_node); ++ kfree(meta); ++} ++ ++bool kbase_sticky_resource_release(struct kbase_context *kctx, ++ struct kbase_ctx_ext_res_meta *meta, u64 gpu_addr) ++{ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ /* Search of the metadata if one isn't provided. */ ++ if (!meta) ++ meta = find_sticky_resource_meta(kctx, gpu_addr); ++ ++ /* No metadata so just return. */ ++ if (!meta) ++ return false; ++ ++ if (--meta->ref != 0) ++ return true; ++ ++ release_sticky_resource_meta(kctx, meta); ++ ++ return true; ++} ++ ++bool kbase_sticky_resource_release_force(struct kbase_context *kctx, ++ struct kbase_ctx_ext_res_meta *meta, u64 gpu_addr) ++{ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ /* Search of the metadata if one isn't provided. */ ++ if (!meta) ++ meta = find_sticky_resource_meta(kctx, gpu_addr); ++ ++ /* No metadata so just return. */ ++ if (!meta) ++ return false; ++ ++ release_sticky_resource_meta(kctx, meta); ++ ++ return true; ++} ++ ++int kbase_sticky_resource_init(struct kbase_context *kctx) ++{ ++ INIT_LIST_HEAD(&kctx->ext_res_meta_head); ++ ++ return 0; ++} ++ ++void kbase_sticky_resource_term(struct kbase_context *kctx) ++{ ++ struct kbase_ctx_ext_res_meta *walker; ++ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ /* ++ * Free any sticky resources which haven't been unmapped. ++ * ++ * Note: ++ * We don't care about refcounts at this point as no future ++ * references to the meta data will be made. ++ * Region termination would find these if we didn't free them ++ * here, but it's more efficient if we do the clean up here. ++ */ ++ while (!list_empty(&kctx->ext_res_meta_head)) { ++ walker = list_first_entry(&kctx->ext_res_meta_head, ++ struct kbase_ctx_ext_res_meta, ext_res_node); ++ ++ kbase_sticky_resource_release_force(kctx, walker, 0); ++ } ++} +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem.h b/drivers/gpu/arm/bifrost/mali_kbase_mem.h +new file mode 100755 +index 000000000000..2238fbfe9e99 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_mem.h +@@ -0,0 +1,1962 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++/** ++ * @file mali_kbase_mem.h ++ * Base kernel memory APIs ++ */ ++ ++#ifndef _KBASE_MEM_H_ ++#define _KBASE_MEM_H_ ++ ++#ifndef _KBASE_H_ ++#error "Don't include this file directly, use mali_kbase.h instead" ++#endif ++ ++#include ++#include "mali_base_kernel.h" ++#include ++#include "mali_kbase_pm.h" ++#include "mali_kbase_defs.h" ++/* Required for kbase_mem_evictable_unmake */ ++#include "mali_kbase_mem_linux.h" ++ ++static inline void kbase_process_page_usage_inc(struct kbase_context *kctx, ++ int pages); ++ ++/* Part of the workaround for uTLB invalid pages is to ensure we grow/shrink tmem by 4 pages at a time */ ++#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_8316 (2) /* round to 4 pages */ ++ ++/* Part of the workaround for PRLAM-9630 requires us to grow/shrink memory by 8 pages. ++The MMU reads in 8 page table entries from memory at a time, if we have more than one page fault within the same 8 pages and ++page tables are updated accordingly, the MMU does not re-read the page table entries from memory for the subsequent page table ++updates and generates duplicate page faults as the page table information used by the MMU is not valid. */ ++#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_9630 (3) /* round to 8 pages */ ++ ++#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2 (0) /* round to 1 page */ ++ ++/* This must always be a power of 2 */ ++#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES (1u << KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2) ++#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_HW_ISSUE_8316 (1u << KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_8316) ++#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_HW_ISSUE_9630 (1u << KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_9630) ++/** ++ * A CPU mapping ++ */ ++struct kbase_cpu_mapping { ++ struct list_head mappings_list; ++ struct kbase_mem_phy_alloc *alloc; ++ struct kbase_context *kctx; ++ struct kbase_va_region *region; ++ int count; ++ int free_on_close; ++}; ++ ++enum kbase_memory_type { ++ KBASE_MEM_TYPE_NATIVE, ++ KBASE_MEM_TYPE_IMPORTED_UMM, ++ KBASE_MEM_TYPE_IMPORTED_USER_BUF, ++ KBASE_MEM_TYPE_ALIAS, ++ KBASE_MEM_TYPE_RAW ++}; ++ ++/* internal structure, mirroring base_mem_aliasing_info, ++ * but with alloc instead of a gpu va (handle) */ ++struct kbase_aliased { ++ struct kbase_mem_phy_alloc *alloc; /* NULL for special, non-NULL for native */ ++ u64 offset; /* in pages */ ++ u64 length; /* in pages */ ++}; ++ ++/** ++ * @brief Physical pages tracking object properties ++ */ ++#define KBASE_MEM_PHY_ALLOC_ACCESSED_CACHED (1u << 0) ++#define KBASE_MEM_PHY_ALLOC_LARGE (1u << 1) ++ ++/* struct kbase_mem_phy_alloc - Physical pages tracking object. ++ * ++ * Set up to track N pages. ++ * N not stored here, the creator holds that info. ++ * This object only tracks how many elements are actually valid (present). ++ * Changing of nents or *pages should only happen if the kbase_mem_phy_alloc ++ * is not shared with another region or client. CPU mappings are OK to ++ * exist when changing, as long as the tracked mappings objects are ++ * updated as part of the change. ++ * ++ * @kref: number of users of this alloc ++ * @gpu_mappings: count number of times mapped on the GPU ++ * @nents: 0..N ++ * @pages: N elements, only 0..nents are valid ++ * @mappings: List of CPU mappings of this physical memory allocation. ++ * @evict_node: Node used to store this allocation on the eviction list ++ * @evicted: Physical backing size when the pages where evicted ++ * @reg: Back reference to the region structure which created this ++ * allocation, or NULL if it has been freed. ++ * @type: type of buffer ++ * @permanent_map: Kernel side mapping of the alloc, shall never be ++ * referred directly. kbase_phy_alloc_mapping_get() & ++ * kbase_phy_alloc_mapping_put() pair should be used ++ * around access to the kernel-side CPU mapping so that ++ * mapping doesn't disappear whilst it is being accessed. ++ * @properties: Bitmask of properties, e.g. KBASE_MEM_PHY_ALLOC_LARGE. ++ * @group_id: A memory group ID to be passed to a platform-specific ++ * memory group manager, if present. ++ * Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). ++ * @imported: member in union valid based on @a type ++ */ ++struct kbase_mem_phy_alloc { ++ struct kref kref; ++ atomic_t gpu_mappings; ++ size_t nents; ++ struct tagged_addr *pages; ++ struct list_head mappings; ++ struct list_head evict_node; ++ size_t evicted; ++ struct kbase_va_region *reg; ++ enum kbase_memory_type type; ++ struct kbase_vmap_struct *permanent_map; ++ u8 properties; ++ u8 group_id; ++ ++ union { ++ struct { ++ struct kbase_context *kctx; ++ struct dma_buf *dma_buf; ++ struct dma_buf_attachment *dma_attachment; ++ unsigned int current_mapping_usage_count; ++ struct sg_table *sgt; ++ bool need_sync; ++ } umm; ++ struct { ++ u64 stride; ++ size_t nents; ++ struct kbase_aliased *aliased; ++ } alias; ++ struct { ++ struct kbase_context *kctx; ++ /* Number of pages in this structure, including *pages. ++ * Used for kernel memory tracking. ++ */ ++ size_t nr_struct_pages; ++ } native; ++ struct kbase_alloc_import_user_buf { ++ unsigned long address; ++ unsigned long size; ++ unsigned long nr_pages; ++ struct page **pages; ++ /* top bit (1<<31) of current_mapping_usage_count ++ * specifies that this import was pinned on import ++ * See PINNED_ON_IMPORT ++ */ ++ u32 current_mapping_usage_count; ++ struct mm_struct *mm; ++ dma_addr_t *dma_addrs; ++ } user_buf; ++ } imported; ++}; ++ ++/* The top bit of kbase_alloc_import_user_buf::current_mapping_usage_count is ++ * used to signify that a buffer was pinned when it was imported. Since the ++ * reference count is limited by the number of atoms that can be submitted at ++ * once there should be no danger of overflowing into this bit. ++ * Stealing the top bit also has the benefit that ++ * current_mapping_usage_count != 0 if and only if the buffer is mapped. ++ */ ++#define PINNED_ON_IMPORT (1<<31) ++ ++/** ++ * enum kbase_jit_report_flags - Flags for just-in-time memory allocation ++ * pressure limit functions ++ * @KBASE_JIT_REPORT_ON_ALLOC_OR_FREE: Notifying about an update happening due ++ * to a just-in-time memory allocation or free ++ * ++ * Used to control flow within pressure limit related functions, or to provide ++ * extra debugging information ++ */ ++enum kbase_jit_report_flags { ++ KBASE_JIT_REPORT_ON_ALLOC_OR_FREE = (1u << 0) ++}; ++ ++static inline void kbase_mem_phy_alloc_gpu_mapped(struct kbase_mem_phy_alloc *alloc) ++{ ++ KBASE_DEBUG_ASSERT(alloc); ++ /* we only track mappings of NATIVE buffers */ ++ if (alloc->type == KBASE_MEM_TYPE_NATIVE) ++ atomic_inc(&alloc->gpu_mappings); ++} ++ ++static inline void kbase_mem_phy_alloc_gpu_unmapped(struct kbase_mem_phy_alloc *alloc) ++{ ++ KBASE_DEBUG_ASSERT(alloc); ++ /* we only track mappings of NATIVE buffers */ ++ if (alloc->type == KBASE_MEM_TYPE_NATIVE) ++ if (0 > atomic_dec_return(&alloc->gpu_mappings)) { ++ pr_err("Mismatched %s:\n", __func__); ++ dump_stack(); ++ } ++} ++ ++/** ++ * kbase_mem_is_imported - Indicate whether a memory type is imported ++ * ++ * @type: the memory type ++ * ++ * Return: true if the memory type is imported, false otherwise ++ */ ++static inline bool kbase_mem_is_imported(enum kbase_memory_type type) ++{ ++ return (type == KBASE_MEM_TYPE_IMPORTED_UMM) || ++ (type == KBASE_MEM_TYPE_IMPORTED_USER_BUF); ++} ++ ++void kbase_mem_kref_free(struct kref *kref); ++ ++int kbase_mem_init(struct kbase_device *kbdev); ++void kbase_mem_halt(struct kbase_device *kbdev); ++void kbase_mem_term(struct kbase_device *kbdev); ++ ++static inline struct kbase_mem_phy_alloc *kbase_mem_phy_alloc_get(struct kbase_mem_phy_alloc *alloc) ++{ ++ kref_get(&alloc->kref); ++ return alloc; ++} ++ ++static inline struct kbase_mem_phy_alloc *kbase_mem_phy_alloc_put(struct kbase_mem_phy_alloc *alloc) ++{ ++ kref_put(&alloc->kref, kbase_mem_kref_free); ++ return NULL; ++} ++ ++/** ++ * A GPU memory region, and attributes for CPU mappings. ++ * ++ * @rblink: Node in a red-black tree of memory regions within the same zone of ++ * the GPU's virtual address space. ++ * @link: Links to neighboring items in a list of growable memory regions ++ * that triggered incremental rendering by growing too much. ++ * @rbtree: Backlink to the red-black tree of memory regions. ++ * @start_pfn: The Page Frame Number in GPU virtual address space. ++ * @nr_pages: The size of the region in pages. ++ * @initial_commit: Initial commit, for aligning the start address and ++ * correctly growing KBASE_REG_TILER_ALIGN_TOP regions. ++ * @threshold_pages: If non-zero and the amount of memory committed to a region ++ * that can grow on page fault exceeds this number of pages ++ * then the driver switches to incremental rendering. ++ * @extent: Number of pages allocated on page fault. ++ * @cpu_alloc: The physical memory we mmap to the CPU when mapping this region. ++ * @gpu_alloc: The physical memory we mmap to the GPU when mapping this region. ++ * @jit_node: Links to neighboring regions in the just-in-time memory pool. ++ * @jit_usage_id: The last just-in-time memory usage ID for this region. ++ * @jit_bin_id: The just-in-time memory bin this region came from. ++ * @va_refcnt: Number of users of this region. Protected by reg_lock. ++ */ ++struct kbase_va_region { ++ struct rb_node rblink; ++ struct list_head link; ++ struct rb_root *rbtree; ++ u64 start_pfn; ++ size_t nr_pages; ++ size_t initial_commit; ++ size_t threshold_pages; ++ ++/* Free region */ ++#define KBASE_REG_FREE (1ul << 0) ++/* CPU write access */ ++#define KBASE_REG_CPU_WR (1ul << 1) ++/* GPU write access */ ++#define KBASE_REG_GPU_WR (1ul << 2) ++/* No eXecute flag */ ++#define KBASE_REG_GPU_NX (1ul << 3) ++/* Is CPU cached? */ ++#define KBASE_REG_CPU_CACHED (1ul << 4) ++/* Is GPU cached? ++ * Some components within the GPU might only be able to access memory that is ++ * GPU cacheable. Refer to the specific GPU implementation for more details. ++ */ ++#define KBASE_REG_GPU_CACHED (1ul << 5) ++ ++#define KBASE_REG_GROWABLE (1ul << 6) ++/* Can grow on pf? */ ++#define KBASE_REG_PF_GROW (1ul << 7) ++ ++/* Allocation doesn't straddle the 4GB boundary in GPU virtual space */ ++#define KBASE_REG_GPU_VA_SAME_4GB_PAGE (1ul << 8) ++ ++/* inner shareable coherency */ ++#define KBASE_REG_SHARE_IN (1ul << 9) ++/* inner & outer shareable coherency */ ++#define KBASE_REG_SHARE_BOTH (1ul << 10) ++ ++/* Space for 4 different zones */ ++#define KBASE_REG_ZONE_MASK (3ul << 11) ++#define KBASE_REG_ZONE(x) (((x) & 3) << 11) ++ ++/* GPU read access */ ++#define KBASE_REG_GPU_RD (1ul<<13) ++/* CPU read access */ ++#define KBASE_REG_CPU_RD (1ul<<14) ++ ++/* Index of chosen MEMATTR for this region (0..7) */ ++#define KBASE_REG_MEMATTR_MASK (7ul << 16) ++#define KBASE_REG_MEMATTR_INDEX(x) (((x) & 7) << 16) ++#define KBASE_REG_MEMATTR_VALUE(x) (((x) & KBASE_REG_MEMATTR_MASK) >> 16) ++ ++#define KBASE_REG_PROTECTED (1ul << 19) ++ ++#define KBASE_REG_DONT_NEED (1ul << 20) ++ ++/* Imported buffer is padded? */ ++#define KBASE_REG_IMPORT_PAD (1ul << 21) ++ ++#if MALI_USE_CSF ++/* CSF event memory */ ++#define KBASE_REG_CSF_EVENT (1ul << 22) ++#else ++/* Bit 22 is reserved. ++ * ++ * Do not remove, use the next unreserved bit for new flags ++ */ ++#define KBASE_REG_RESERVED_BIT_22 (1ul << 22) ++#endif ++ ++#if !MALI_USE_CSF ++/* The top of the initial commit is aligned to extent pages. ++ * Extent must be a power of 2 */ ++#define KBASE_REG_TILER_ALIGN_TOP (1ul << 23) ++#else ++/* Bit 23 is reserved. ++ * ++ * Do not remove, use the next unreserved bit for new flags ++ */ ++#define KBASE_REG_RESERVED_BIT_23 (1ul << 23) ++#endif /* !MALI_USE_CSF */ ++ ++/* Whilst this flag is set the GPU allocation is not supposed to be freed by ++ * user space. The flag will remain set for the lifetime of JIT allocations. ++ */ ++#define KBASE_REG_NO_USER_FREE (1ul << 24) ++ ++/* Memory has permanent kernel side mapping */ ++#define KBASE_REG_PERMANENT_KERNEL_MAPPING (1ul << 25) ++ ++/* GPU VA region has been freed by the userspace, but still remains allocated ++ * due to the reference held by CPU mappings created on the GPU VA region. ++ * ++ * A region with this flag set has had kbase_gpu_munmap() called on it, but can ++ * still be looked-up in the region tracker as a non-free region. Hence must ++ * not create or update any more GPU mappings on such regions because they will ++ * not be unmapped when the region is finally destroyed. ++ * ++ * Since such regions are still present in the region tracker, new allocations ++ * attempted with BASE_MEM_SAME_VA might fail if their address intersects with ++ * a region with this flag set. ++ * ++ * In addition, this flag indicates the gpu_alloc member might no longer valid ++ * e.g. in infinite cache simulation. ++ */ ++#define KBASE_REG_VA_FREED (1ul << 26) ++ ++/* If set, the heap info address points to a u32 holding the used size in bytes; ++ * otherwise it points to a u64 holding the lowest address of unused memory. ++ */ ++#define KBASE_REG_HEAP_INFO_IS_SIZE (1ul << 27) ++ ++/* Allocation is actively used for JIT memory */ ++#define KBASE_REG_ACTIVE_JIT_ALLOC (1ul << 28) ++ ++#define KBASE_REG_ZONE_SAME_VA KBASE_REG_ZONE(0) ++ ++/* only used with 32-bit clients */ ++/* ++ * On a 32bit platform, custom VA should be wired from 4GB ++ * to the VA limit of the GPU. Unfortunately, the Linux mmap() interface ++ * limits us to 2^32 pages (2^44 bytes, see mmap64 man page for reference). ++ * So we put the default limit to the maximum possible on Linux and shrink ++ * it down, if required by the GPU, during initialization. ++ */ ++ ++#define KBASE_REG_ZONE_CUSTOM_VA KBASE_REG_ZONE(1) ++#define KBASE_REG_ZONE_CUSTOM_VA_BASE (0x100000000ULL >> PAGE_SHIFT) ++#define KBASE_REG_ZONE_CUSTOM_VA_SIZE (((1ULL << 44) >> PAGE_SHIFT) - KBASE_REG_ZONE_CUSTOM_VA_BASE) ++/* end 32-bit clients only */ ++ ++/* The starting address and size of the GPU-executable zone are dynamic ++ * and depend on the platform and the number of pages requested by the ++ * user process, with an upper limit of 4 GB. ++ */ ++#define KBASE_REG_ZONE_EXEC_VA KBASE_REG_ZONE(2) ++#define KBASE_REG_ZONE_EXEC_VA_MAX_PAGES ((1ULL << 32) >> PAGE_SHIFT) /* 4 GB */ ++ ++#if MALI_USE_CSF ++#define KBASE_REG_ZONE_MCU_SHARED KBASE_REG_ZONE(3) ++#define KBASE_REG_ZONE_MCU_SHARED_BASE (0x04000000ULL >> PAGE_SHIFT) ++#define KBASE_REG_ZONE_MCU_SHARED_SIZE (((0x08000000ULL) >> PAGE_SHIFT) - \ ++ KBASE_REG_ZONE_MCU_SHARED_BASE) ++#endif ++ ++ unsigned long flags; ++ size_t extent; ++ struct kbase_mem_phy_alloc *cpu_alloc; ++ struct kbase_mem_phy_alloc *gpu_alloc; ++ struct list_head jit_node; ++ u16 jit_usage_id; ++ u8 jit_bin_id; ++#if MALI_JIT_PRESSURE_LIMIT_BASE ++ /* Pointer to an object in GPU memory defining an end of an allocated ++ * region ++ * ++ * The object can be one of: ++ * - u32 value defining the size of the region ++ * - u64 pointer first unused byte in the region ++ * ++ * The interpretation of the object depends on ++ * BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE flag in jit_info_flags - if it is ++ * set, the heap info object should be interpreted as size. ++ */ ++ u64 heap_info_gpu_addr; ++ ++ /* The current estimate of the number of pages used, which in normal ++ * use is either: ++ * - the initial estimate == va_pages ++ * - the actual pages used, as found by a JIT usage report ++ * ++ * Note that since the value is calculated from GPU memory after a JIT ++ * usage report, at any point in time it is allowed to take a random ++ * value that is no greater than va_pages (e.g. it may be greater than ++ * gpu_alloc->nents) ++ */ ++ size_t used_pages; ++#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ ++ ++ int va_refcnt; ++}; ++ ++/* Special marker for failed JIT allocations that still must be marked as ++ * in-use ++ */ ++#define KBASE_RESERVED_REG_JIT_ALLOC ((struct kbase_va_region *)-1) ++ ++static inline bool kbase_is_region_free(struct kbase_va_region *reg) ++{ ++ return (!reg || reg->flags & KBASE_REG_FREE); ++} ++ ++static inline bool kbase_is_region_invalid(struct kbase_va_region *reg) ++{ ++ return (!reg || reg->flags & KBASE_REG_VA_FREED); ++} ++ ++static inline bool kbase_is_region_invalid_or_free(struct kbase_va_region *reg) ++{ ++ /* Possibly not all functions that find regions would be using this ++ * helper, so they need to be checked when maintaining this function. ++ */ ++ return (kbase_is_region_invalid(reg) || kbase_is_region_free(reg)); ++} ++ ++int kbase_remove_va_region(struct kbase_va_region *reg); ++static inline void kbase_region_refcnt_free(struct kbase_va_region *reg) ++{ ++ /* If region was mapped then remove va region*/ ++ if (reg->start_pfn) ++ kbase_remove_va_region(reg); ++ ++ /* To detect use-after-free in debug builds */ ++ KBASE_DEBUG_CODE(reg->flags |= KBASE_REG_FREE); ++ kfree(reg); ++} ++ ++static inline struct kbase_va_region *kbase_va_region_alloc_get( ++ struct kbase_context *kctx, struct kbase_va_region *region) ++{ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ WARN_ON(!region->va_refcnt); ++ ++ /* non-atomic as kctx->reg_lock is held */ ++ dev_dbg(kctx->kbdev->dev, "va_refcnt %d before get %p\n", ++ region->va_refcnt, (void *)region); ++ region->va_refcnt++; ++ ++ return region; ++} ++ ++static inline struct kbase_va_region *kbase_va_region_alloc_put( ++ struct kbase_context *kctx, struct kbase_va_region *region) ++{ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ WARN_ON(region->va_refcnt <= 0); ++ WARN_ON(region->flags & KBASE_REG_FREE); ++ ++ /* non-atomic as kctx->reg_lock is held */ ++ region->va_refcnt--; ++ dev_dbg(kctx->kbdev->dev, "va_refcnt %d after put %p\n", ++ region->va_refcnt, (void *)region); ++ if (!region->va_refcnt) ++ kbase_region_refcnt_free(region); ++ ++ return NULL; ++} ++ ++/* Common functions */ ++static inline struct tagged_addr *kbase_get_cpu_phy_pages( ++ struct kbase_va_region *reg) ++{ ++ KBASE_DEBUG_ASSERT(reg); ++ KBASE_DEBUG_ASSERT(reg->cpu_alloc); ++ KBASE_DEBUG_ASSERT(reg->gpu_alloc); ++ KBASE_DEBUG_ASSERT(reg->cpu_alloc->nents == reg->gpu_alloc->nents); ++ ++ return reg->cpu_alloc->pages; ++} ++ ++static inline struct tagged_addr *kbase_get_gpu_phy_pages( ++ struct kbase_va_region *reg) ++{ ++ KBASE_DEBUG_ASSERT(reg); ++ KBASE_DEBUG_ASSERT(reg->cpu_alloc); ++ KBASE_DEBUG_ASSERT(reg->gpu_alloc); ++ KBASE_DEBUG_ASSERT(reg->cpu_alloc->nents == reg->gpu_alloc->nents); ++ ++ return reg->gpu_alloc->pages; ++} ++ ++static inline size_t kbase_reg_current_backed_size(struct kbase_va_region *reg) ++{ ++ KBASE_DEBUG_ASSERT(reg); ++ /* if no alloc object the backed size naturally is 0 */ ++ if (!reg->cpu_alloc) ++ return 0; ++ ++ KBASE_DEBUG_ASSERT(reg->cpu_alloc); ++ KBASE_DEBUG_ASSERT(reg->gpu_alloc); ++ KBASE_DEBUG_ASSERT(reg->cpu_alloc->nents == reg->gpu_alloc->nents); ++ ++ return reg->cpu_alloc->nents; ++} ++ ++#define KBASE_MEM_PHY_ALLOC_LARGE_THRESHOLD ((size_t)(4*1024)) /* size above which vmalloc is used over kmalloc */ ++ ++static inline struct kbase_mem_phy_alloc *kbase_alloc_create( ++ struct kbase_context *kctx, size_t nr_pages, ++ enum kbase_memory_type type, int group_id) ++{ ++ struct kbase_mem_phy_alloc *alloc; ++ size_t alloc_size = sizeof(*alloc) + sizeof(*alloc->pages) * nr_pages; ++ size_t per_page_size = sizeof(*alloc->pages); ++ ++ /* Imported pages may have page private data already in use */ ++ if (type == KBASE_MEM_TYPE_IMPORTED_USER_BUF) { ++ alloc_size += nr_pages * ++ sizeof(*alloc->imported.user_buf.dma_addrs); ++ per_page_size += sizeof(*alloc->imported.user_buf.dma_addrs); ++ } ++ ++ /* ++ * Prevent nr_pages*per_page_size + sizeof(*alloc) from ++ * wrapping around. ++ */ ++ if (nr_pages > ((((size_t) -1) - sizeof(*alloc)) ++ / per_page_size)) ++ return ERR_PTR(-ENOMEM); ++ ++ /* Allocate based on the size to reduce internal fragmentation of vmem */ ++ if (alloc_size > KBASE_MEM_PHY_ALLOC_LARGE_THRESHOLD) ++ alloc = vzalloc(alloc_size); ++ else ++ alloc = kzalloc(alloc_size, GFP_KERNEL); ++ ++ if (!alloc) ++ return ERR_PTR(-ENOMEM); ++ ++ if (type == KBASE_MEM_TYPE_NATIVE) { ++ alloc->imported.native.nr_struct_pages = ++ (alloc_size + (PAGE_SIZE - 1)) >> PAGE_SHIFT; ++ kbase_process_page_usage_inc(kctx, ++ alloc->imported.native.nr_struct_pages); ++ } ++ ++ /* Store allocation method */ ++ if (alloc_size > KBASE_MEM_PHY_ALLOC_LARGE_THRESHOLD) ++ alloc->properties |= KBASE_MEM_PHY_ALLOC_LARGE; ++ ++ kref_init(&alloc->kref); ++ atomic_set(&alloc->gpu_mappings, 0); ++ alloc->nents = 0; ++ alloc->pages = (void *)(alloc + 1); ++ INIT_LIST_HEAD(&alloc->mappings); ++ alloc->type = type; ++ alloc->group_id = group_id; ++ ++ if (type == KBASE_MEM_TYPE_IMPORTED_USER_BUF) ++ alloc->imported.user_buf.dma_addrs = ++ (void *) (alloc->pages + nr_pages); ++ ++ return alloc; ++} ++ ++static inline int kbase_reg_prepare_native(struct kbase_va_region *reg, ++ struct kbase_context *kctx, int group_id) ++{ ++ KBASE_DEBUG_ASSERT(reg); ++ KBASE_DEBUG_ASSERT(!reg->cpu_alloc); ++ KBASE_DEBUG_ASSERT(!reg->gpu_alloc); ++ KBASE_DEBUG_ASSERT(reg->flags & KBASE_REG_FREE); ++ ++ reg->cpu_alloc = kbase_alloc_create(kctx, reg->nr_pages, ++ KBASE_MEM_TYPE_NATIVE, group_id); ++ if (IS_ERR(reg->cpu_alloc)) ++ return PTR_ERR(reg->cpu_alloc); ++ else if (!reg->cpu_alloc) ++ return -ENOMEM; ++ ++ reg->cpu_alloc->imported.native.kctx = kctx; ++ if (kbase_ctx_flag(kctx, KCTX_INFINITE_CACHE) ++ && (reg->flags & KBASE_REG_CPU_CACHED)) { ++ reg->gpu_alloc = kbase_alloc_create(kctx, reg->nr_pages, ++ KBASE_MEM_TYPE_NATIVE, group_id); ++ if (IS_ERR_OR_NULL(reg->gpu_alloc)) { ++ kbase_mem_phy_alloc_put(reg->cpu_alloc); ++ return -ENOMEM; ++ } ++ reg->gpu_alloc->imported.native.kctx = kctx; ++ } else { ++ reg->gpu_alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc); ++ } ++ ++ mutex_lock(&kctx->jit_evict_lock); ++ INIT_LIST_HEAD(®->cpu_alloc->evict_node); ++ INIT_LIST_HEAD(®->gpu_alloc->evict_node); ++ mutex_unlock(&kctx->jit_evict_lock); ++ ++ reg->flags &= ~KBASE_REG_FREE; ++ ++ return 0; ++} ++ ++/* ++ * Max size for kbdev memory pool (in pages) ++ */ ++#define KBASE_MEM_POOL_MAX_SIZE_KBDEV (SZ_64M >> PAGE_SHIFT) ++ ++/* ++ * Max size for kctx memory pool (in pages) ++ */ ++#define KBASE_MEM_POOL_MAX_SIZE_KCTX (SZ_64M >> PAGE_SHIFT) ++ ++/* ++ * The order required for a 2MB page allocation (2^order * 4KB = 2MB) ++ */ ++#define KBASE_MEM_POOL_2MB_PAGE_TABLE_ORDER 9 ++ ++/* ++ * The order required for a 4KB page allocation ++ */ ++#define KBASE_MEM_POOL_4KB_PAGE_TABLE_ORDER 0 ++ ++/** ++ * kbase_mem_pool_config_set_max_size - Set maximum number of free pages in ++ * initial configuration of a memory pool ++ * ++ * @config: Initial configuration for a physical memory pool ++ * @max_size: Maximum number of free pages that a pool created from ++ * @config can hold ++ */ ++static inline void kbase_mem_pool_config_set_max_size( ++ struct kbase_mem_pool_config *const config, size_t const max_size) ++{ ++ WRITE_ONCE(config->max_size, max_size); ++} ++ ++/** ++ * kbase_mem_pool_config_get_max_size - Get maximum number of free pages from ++ * initial configuration of a memory pool ++ * ++ * @config: Initial configuration for a physical memory pool ++ * ++ * Return: Maximum number of free pages that a pool created from @config ++ * can hold ++ */ ++static inline size_t kbase_mem_pool_config_get_max_size( ++ const struct kbase_mem_pool_config *const config) ++{ ++ return READ_ONCE(config->max_size); ++} ++ ++/** ++ * kbase_mem_pool_init - Create a memory pool for a kbase device ++ * @pool: Memory pool to initialize ++ * @config: Initial configuration for the memory pool ++ * @order: Page order for physical page size (order=0=>4kB, order=9=>2MB) ++ * @group_id: A memory group ID to be passed to a platform-specific ++ * memory group manager, if present. ++ * Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). ++ * @kbdev: Kbase device where memory is used ++ * @next_pool: Pointer to the next pool or NULL. ++ * ++ * Allocations from @pool are in whole pages. Each @pool has a free list where ++ * pages can be quickly allocated from. The free list is initially empty and ++ * filled whenever pages are freed back to the pool. The number of free pages ++ * in the pool will in general not exceed @max_size, but the pool may in ++ * certain corner cases grow above @max_size. ++ * ++ * If @next_pool is not NULL, we will allocate from @next_pool before going to ++ * the memory group manager. Similarly pages can spill over to @next_pool when ++ * @pool is full. Pages are zeroed before they spill over to another pool, to ++ * prevent leaking information between applications. ++ * ++ * A shrinker is registered so that Linux mm can reclaim pages from the pool as ++ * needed. ++ * ++ * Return: 0 on success, negative -errno on error ++ */ ++int kbase_mem_pool_init(struct kbase_mem_pool *pool, ++ const struct kbase_mem_pool_config *config, ++ unsigned int order, ++ int group_id, ++ struct kbase_device *kbdev, ++ struct kbase_mem_pool *next_pool); ++ ++/** ++ * kbase_mem_pool_term - Destroy a memory pool ++ * @pool: Memory pool to destroy ++ * ++ * Pages in the pool will spill over to @next_pool (if available) or freed to ++ * the kernel. ++ */ ++void kbase_mem_pool_term(struct kbase_mem_pool *pool); ++ ++/** ++ * kbase_mem_pool_alloc - Allocate a page from memory pool ++ * @pool: Memory pool to allocate from ++ * ++ * Allocations from the pool are made as follows: ++ * 1. If there are free pages in the pool, allocate a page from @pool. ++ * 2. Otherwise, if @next_pool is not NULL and has free pages, allocate a page ++ * from @next_pool. ++ * 3. Return NULL if no memory in the pool ++ * ++ * Return: Pointer to allocated page, or NULL if allocation failed. ++ * ++ * Note : This function should not be used if the pool lock is held. Use ++ * kbase_mem_pool_alloc_locked() instead. ++ */ ++struct page *kbase_mem_pool_alloc(struct kbase_mem_pool *pool); ++ ++/** ++ * kbase_mem_pool_alloc_locked - Allocate a page from memory pool ++ * @pool: Memory pool to allocate from ++ * ++ * If there are free pages in the pool, this function allocates a page from ++ * @pool. This function does not use @next_pool. ++ * ++ * Return: Pointer to allocated page, or NULL if allocation failed. ++ * ++ * Note : Caller must hold the pool lock. ++ */ ++struct page *kbase_mem_pool_alloc_locked(struct kbase_mem_pool *pool); ++ ++/** ++ * kbase_mem_pool_free - Free a page to memory pool ++ * @pool: Memory pool where page should be freed ++ * @page: Page to free to the pool ++ * @dirty: Whether some of the page may be dirty in the cache. ++ * ++ * Pages are freed to the pool as follows: ++ * 1. If @pool is not full, add @page to @pool. ++ * 2. Otherwise, if @next_pool is not NULL and not full, add @page to ++ * @next_pool. ++ * 3. Finally, free @page to the kernel. ++ * ++ * Note : This function should not be used if the pool lock is held. Use ++ * kbase_mem_pool_free_locked() instead. ++ */ ++void kbase_mem_pool_free(struct kbase_mem_pool *pool, struct page *page, ++ bool dirty); ++ ++/** ++ * kbase_mem_pool_free_locked - Free a page to memory pool ++ * @pool: Memory pool where page should be freed ++ * @p: Page to free to the pool ++ * @dirty: Whether some of the page may be dirty in the cache. ++ * ++ * If @pool is not full, this function adds @page to @pool. Otherwise, @page is ++ * freed to the kernel. This function does not use @next_pool. ++ * ++ * Note : Caller must hold the pool lock. ++ */ ++void kbase_mem_pool_free_locked(struct kbase_mem_pool *pool, struct page *p, ++ bool dirty); ++ ++/** ++ * kbase_mem_pool_alloc_pages - Allocate pages from memory pool ++ * @pool: Memory pool to allocate from ++ * @nr_4k_pages: Number of pages to allocate ++ * @pages: Pointer to array where the physical address of the allocated ++ * pages will be stored. ++ * @partial_allowed: If fewer pages allocated is allowed ++ * ++ * Like kbase_mem_pool_alloc() but optimized for allocating many pages. ++ * ++ * Return: ++ * On success number of pages allocated (could be less than nr_pages if ++ * partial_allowed). ++ * On error an error code. ++ * ++ * Note : This function should not be used if the pool lock is held. Use ++ * kbase_mem_pool_alloc_pages_locked() instead. ++ * ++ * The caller must not hold vm_lock, as this could cause a deadlock if ++ * the kernel OoM killer runs. If the caller must allocate pages while holding ++ * this lock, it should use kbase_mem_pool_alloc_pages_locked() instead. ++ */ ++int kbase_mem_pool_alloc_pages(struct kbase_mem_pool *pool, size_t nr_4k_pages, ++ struct tagged_addr *pages, bool partial_allowed); ++ ++/** ++ * kbase_mem_pool_alloc_pages_locked - Allocate pages from memory pool ++ * @pool: Memory pool to allocate from ++ * @nr_4k_pages: Number of pages to allocate ++ * @pages: Pointer to array where the physical address of the allocated ++ * pages will be stored. ++ * ++ * Like kbase_mem_pool_alloc() but optimized for allocating many pages. This ++ * version does not allocate new pages from the kernel, and therefore will never ++ * trigger the OoM killer. Therefore, it can be run while the vm_lock is held. ++ * ++ * As new pages can not be allocated, the caller must ensure there are ++ * sufficient pages in the pool. Usage of this function should look like : ++ * ++ * kbase_gpu_vm_lock(kctx); ++ * kbase_mem_pool_lock(pool) ++ * while (kbase_mem_pool_size(pool) < pages_required) { ++ * kbase_mem_pool_unlock(pool) ++ * kbase_gpu_vm_unlock(kctx); ++ * kbase_mem_pool_grow(pool) ++ * kbase_gpu_vm_lock(kctx); ++ * kbase_mem_pool_lock(pool) ++ * } ++ * kbase_mem_pool_alloc_pages_locked(pool) ++ * kbase_mem_pool_unlock(pool) ++ * Perform other processing that requires vm_lock... ++ * kbase_gpu_vm_unlock(kctx); ++ * ++ * This ensures that the pool can be grown to the required size and that the ++ * allocation can complete without another thread using the newly grown pages. ++ * ++ * Return: ++ * On success number of pages allocated. ++ * On error an error code. ++ * ++ * Note : Caller must hold the pool lock. ++ */ ++int kbase_mem_pool_alloc_pages_locked(struct kbase_mem_pool *pool, ++ size_t nr_4k_pages, struct tagged_addr *pages); ++ ++/** ++ * kbase_mem_pool_free_pages - Free pages to memory pool ++ * @pool: Memory pool where pages should be freed ++ * @nr_pages: Number of pages to free ++ * @pages: Pointer to array holding the physical addresses of the pages to ++ * free. ++ * @dirty: Whether any pages may be dirty in the cache. ++ * @reclaimed: Whether the pages where reclaimable and thus should bypass ++ * the pool and go straight to the kernel. ++ * ++ * Like kbase_mem_pool_free() but optimized for freeing many pages. ++ */ ++void kbase_mem_pool_free_pages(struct kbase_mem_pool *pool, size_t nr_pages, ++ struct tagged_addr *pages, bool dirty, bool reclaimed); ++ ++/** ++ * kbase_mem_pool_free_pages_locked - Free pages to memory pool ++ * @pool: Memory pool where pages should be freed ++ * @nr_pages: Number of pages to free ++ * @pages: Pointer to array holding the physical addresses of the pages to ++ * free. ++ * @dirty: Whether any pages may be dirty in the cache. ++ * @reclaimed: Whether the pages where reclaimable and thus should bypass ++ * the pool and go straight to the kernel. ++ * ++ * Like kbase_mem_pool_free() but optimized for freeing many pages. ++ */ ++void kbase_mem_pool_free_pages_locked(struct kbase_mem_pool *pool, ++ size_t nr_pages, struct tagged_addr *pages, bool dirty, ++ bool reclaimed); ++ ++/** ++ * kbase_mem_pool_size - Get number of free pages in memory pool ++ * @pool: Memory pool to inspect ++ * ++ * Note: the size of the pool may in certain corner cases exceed @max_size! ++ * ++ * Return: Number of free pages in the pool ++ */ ++static inline size_t kbase_mem_pool_size(struct kbase_mem_pool *pool) ++{ ++ return READ_ONCE(pool->cur_size); ++} ++ ++/** ++ * kbase_mem_pool_max_size - Get maximum number of free pages in memory pool ++ * @pool: Memory pool to inspect ++ * ++ * Return: Maximum number of free pages in the pool ++ */ ++static inline size_t kbase_mem_pool_max_size(struct kbase_mem_pool *pool) ++{ ++ return pool->max_size; ++} ++ ++ ++/** ++ * kbase_mem_pool_set_max_size - Set maximum number of free pages in memory pool ++ * @pool: Memory pool to inspect ++ * @max_size: Maximum number of free pages the pool can hold ++ * ++ * If @max_size is reduced, the pool will be shrunk to adhere to the new limit. ++ * For details see kbase_mem_pool_shrink(). ++ */ ++void kbase_mem_pool_set_max_size(struct kbase_mem_pool *pool, size_t max_size); ++ ++/** ++ * kbase_mem_pool_grow - Grow the pool ++ * @pool: Memory pool to grow ++ * @nr_to_grow: Number of pages to add to the pool ++ * ++ * Adds @nr_to_grow pages to the pool. Note that this may cause the pool to ++ * become larger than the maximum size specified. ++ * ++ * Returns: 0 on success, -ENOMEM if unable to allocate sufficent pages ++ */ ++int kbase_mem_pool_grow(struct kbase_mem_pool *pool, size_t nr_to_grow); ++ ++/** ++ * kbase_mem_pool_trim - Grow or shrink the pool to a new size ++ * @pool: Memory pool to trim ++ * @new_size: New number of pages in the pool ++ * ++ * If @new_size > @cur_size, fill the pool with new pages from the kernel, but ++ * not above the max_size for the pool. ++ * If @new_size < @cur_size, shrink the pool by freeing pages to the kernel. ++ */ ++void kbase_mem_pool_trim(struct kbase_mem_pool *pool, size_t new_size); ++ ++/** ++ * kbase_mem_pool_mark_dying - Mark that this pool is dying ++ * @pool: Memory pool ++ * ++ * This will cause any ongoing allocation operations (eg growing on page fault) ++ * to be terminated. ++ */ ++void kbase_mem_pool_mark_dying(struct kbase_mem_pool *pool); ++ ++/** ++ * kbase_mem_alloc_page - Allocate a new page for a device ++ * @pool: Memory pool to allocate a page from ++ * ++ * Most uses should use kbase_mem_pool_alloc to allocate a page. However that ++ * function can fail in the event the pool is empty. ++ * ++ * Return: A new page or NULL if no memory ++ */ ++struct page *kbase_mem_alloc_page(struct kbase_mem_pool *pool); ++ ++/** ++ * kbase_region_tracker_init - Initialize the region tracker data structure ++ * @kctx: kbase context ++ * ++ * Return: 0 if success, negative error code otherwise. ++ */ ++int kbase_region_tracker_init(struct kbase_context *kctx); ++ ++/** ++ * kbase_region_tracker_init_jit - Initialize the just-in-time memory ++ * allocation region ++ * @kctx: Kbase context. ++ * @jit_va_pages: Size of the JIT region in pages. ++ * @max_allocations: Maximum number of allocations allowed for the JIT region. ++ * Valid range is 0..%BASE_JIT_ALLOC_COUNT. ++ * @trim_level: Trim level for the JIT region. ++ * Valid range is 0..%BASE_JIT_MAX_TRIM_LEVEL. ++ * @group_id: The physical group ID from which to allocate JIT memory. ++ * Valid range is 0..(%MEMORY_GROUP_MANAGER_NR_GROUPS-1). ++ * @phys_pages_limit: Maximum number of physical pages to use to back the JIT ++ * region. Must not exceed @jit_va_pages. ++ * ++ * Return: 0 if success, negative error code otherwise. ++ */ ++int kbase_region_tracker_init_jit(struct kbase_context *kctx, u64 jit_va_pages, ++ int max_allocations, int trim_level, int group_id, ++ u64 phys_pages_limit); ++ ++/** ++ * kbase_region_tracker_init_exec - Initialize the GPU-executable memory region ++ * @kctx: kbase context ++ * @exec_va_pages: Size of the JIT region in pages. ++ * It must not be greater than 4 GB. ++ * ++ * Return: 0 if success, negative error code otherwise. ++ */ ++int kbase_region_tracker_init_exec(struct kbase_context *kctx, u64 exec_va_pages); ++ ++/** ++ * kbase_region_tracker_term - Terminate the JIT region ++ * @kctx: kbase context ++ */ ++void kbase_region_tracker_term(struct kbase_context *kctx); ++ ++/** ++ * kbase_region_tracker_term_rbtree - Free memory for a region tracker ++ * ++ * This will free all the regions within the region tracker ++ * ++ * @rbtree: Region tracker tree root ++ */ ++void kbase_region_tracker_term_rbtree(struct rb_root *rbtree); ++ ++struct kbase_va_region *kbase_region_tracker_find_region_enclosing_address( ++ struct kbase_context *kctx, u64 gpu_addr); ++struct kbase_va_region *kbase_find_region_enclosing_address( ++ struct rb_root *rbtree, u64 gpu_addr); ++ ++/** ++ * @brief Check that a pointer is actually a valid region. ++ * ++ * Must be called with context lock held. ++ */ ++struct kbase_va_region *kbase_region_tracker_find_region_base_address( ++ struct kbase_context *kctx, u64 gpu_addr); ++struct kbase_va_region *kbase_find_region_base_address(struct rb_root *rbtree, ++ u64 gpu_addr); ++ ++struct kbase_va_region *kbase_alloc_free_region(struct rb_root *rbtree, ++ u64 start_pfn, size_t nr_pages, int zone); ++void kbase_free_alloced_region(struct kbase_va_region *reg); ++int kbase_add_va_region(struct kbase_context *kctx, struct kbase_va_region *reg, ++ u64 addr, size_t nr_pages, size_t align); ++int kbase_add_va_region_rbtree(struct kbase_device *kbdev, ++ struct kbase_va_region *reg, u64 addr, size_t nr_pages, ++ size_t align); ++ ++bool kbase_check_alloc_flags(unsigned long flags); ++bool kbase_check_import_flags(unsigned long flags); ++ ++/** ++ * kbase_check_alloc_sizes - check user space sizes parameters for an ++ * allocation ++ * ++ * @kctx: kbase context ++ * @flags: The flags passed from user space ++ * @va_pages: The size of the requested region, in pages. ++ * @commit_pages: Number of pages to commit initially. ++ * @extent: Number of pages to grow by on GPU page fault and/or alignment ++ * (depending on flags) ++ * ++ * Makes checks on the size parameters passed in from user space for a memory ++ * allocation call, with respect to the flags requested. ++ * ++ * Return: 0 if sizes are valid for these flags, negative error code otherwise ++ */ ++int kbase_check_alloc_sizes(struct kbase_context *kctx, unsigned long flags, ++ u64 va_pages, u64 commit_pages, u64 extent); ++ ++/** ++ * kbase_update_region_flags - Convert user space flags to kernel region flags ++ * ++ * @kctx: kbase context ++ * @reg: The region to update the flags on ++ * @flags: The flags passed from user space ++ * ++ * The user space flag BASE_MEM_COHERENT_SYSTEM_REQUIRED will be rejected and ++ * this function will fail if the system does not support system coherency. ++ * ++ * Return: 0 if successful, -EINVAL if the flags are not supported ++ */ ++int kbase_update_region_flags(struct kbase_context *kctx, ++ struct kbase_va_region *reg, unsigned long flags); ++ ++void kbase_gpu_vm_lock(struct kbase_context *kctx); ++void kbase_gpu_vm_unlock(struct kbase_context *kctx); ++ ++int kbase_alloc_phy_pages(struct kbase_va_region *reg, size_t vsize, size_t size); ++ ++/** ++ * @brief Register region and map it on the GPU. ++ * ++ * Call kbase_add_va_region() and map the region on the GPU. ++ */ ++int kbase_gpu_mmap(struct kbase_context *kctx, struct kbase_va_region *reg, u64 addr, size_t nr_pages, size_t align); ++ ++/** ++ * @brief Remove the region from the GPU and unregister it. ++ * ++ * Must be called with context lock held. ++ */ ++int kbase_gpu_munmap(struct kbase_context *kctx, struct kbase_va_region *reg); ++ ++/** ++ * kbase_mmu_update - Configure an address space on the GPU to the specified ++ * MMU tables ++ * ++ * The caller has the following locking conditions: ++ * - It must hold kbase_device->mmu_hw_mutex ++ * - It must hold the hwaccess_lock ++ * ++ * @kbdev: Kbase device structure ++ * @mmut: The set of MMU tables to be configured on the address space ++ * @as_nr: The address space to be configured ++ */ ++void kbase_mmu_update(struct kbase_device *kbdev, struct kbase_mmu_table *mmut, ++ int as_nr); ++ ++/** ++ * kbase_mmu_disable() - Disable the MMU for a previously active kbase context. ++ * @kctx: Kbase context ++ * ++ * Disable and perform the required cache maintenance to remove the all ++ * data from provided kbase context from the GPU caches. ++ * ++ * The caller has the following locking conditions: ++ * - It must hold kbase_device->mmu_hw_mutex ++ * - It must hold the hwaccess_lock ++ */ ++void kbase_mmu_disable(struct kbase_context *kctx); ++ ++/** ++ * kbase_mmu_disable_as() - Set the MMU to unmapped mode for the specified ++ * address space. ++ * @kbdev: Kbase device ++ * @as_nr: The address space number to set to unmapped. ++ * ++ * This function must only be called during reset/power-up and it used to ++ * ensure the registers are in a known state. ++ * ++ * The caller must hold kbdev->mmu_hw_mutex. ++ */ ++void kbase_mmu_disable_as(struct kbase_device *kbdev, int as_nr); ++ ++void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat); ++ ++/** ++ * kbase_mmu_dump() - Dump the MMU tables to a buffer. ++ * ++ * This function allocates a buffer (of @c nr_pages pages) to hold a dump ++ * of the MMU tables and fills it. If the buffer is too small ++ * then the return value will be NULL. ++ * ++ * The GPU vm lock must be held when calling this function. ++ * ++ * The buffer returned should be freed with @ref vfree when it is no longer ++ * required. ++ * ++ * @kctx: The kbase context to dump ++ * @nr_pages: The number of pages to allocate for the buffer. ++ * ++ * Return: The address of the buffer containing the MMU dump or NULL on error ++ * (including if the @c nr_pages is too small) ++ */ ++void *kbase_mmu_dump(struct kbase_context *kctx, int nr_pages); ++ ++/** ++ * kbase_sync_now - Perform cache maintenance on a memory region ++ * ++ * @kctx: The kbase context of the region ++ * @sset: A syncset structure describing the region and direction of the ++ * synchronisation required ++ * ++ * Return: 0 on success or error code ++ */ ++int kbase_sync_now(struct kbase_context *kctx, struct basep_syncset *sset); ++void kbase_sync_single(struct kbase_context *kctx, struct tagged_addr cpu_pa, ++ struct tagged_addr gpu_pa, off_t offset, size_t size, ++ enum kbase_sync_type sync_fn); ++ ++/* OS specific functions */ ++int kbase_mem_free(struct kbase_context *kctx, u64 gpu_addr); ++int kbase_mem_free_region(struct kbase_context *kctx, struct kbase_va_region *reg); ++void kbase_os_mem_map_lock(struct kbase_context *kctx); ++void kbase_os_mem_map_unlock(struct kbase_context *kctx); ++ ++/** ++ * kbasep_os_process_page_usage_update() - Update the memory allocation ++ * counters for the current process. ++ * ++ * OS specific call to updates the current memory allocation counters ++ * for the current process with the supplied delta. ++ * ++ * @kctx: The kbase context ++ * @pages: The desired delta to apply to the memory usage counters. ++ */ ++ ++void kbasep_os_process_page_usage_update(struct kbase_context *kctx, int pages); ++ ++/** ++ * kbase_process_page_usage_inc() - Add to the memory allocation counters for ++ * the current process ++ * ++ * OS specific call to add to the current memory allocation counters for ++ * the current process by the supplied amount. ++ * ++ * @kctx: The kernel base context used for the allocation. ++ * @pages: The desired delta to apply to the memory usage counters. ++ */ ++ ++static inline void kbase_process_page_usage_inc(struct kbase_context *kctx, int pages) ++{ ++ kbasep_os_process_page_usage_update(kctx, pages); ++} ++ ++/** ++ * kbase_process_page_usage_dec() - Subtract from the memory allocation ++ * counters for the current process. ++ * ++ * OS specific call to subtract from the current memory allocation counters ++ * for the current process by the supplied amount. ++ * ++ * @kctx: The kernel base context used for the allocation. ++ * @pages: The desired delta to apply to the memory usage counters. ++ */ ++ ++static inline void kbase_process_page_usage_dec(struct kbase_context *kctx, int pages) ++{ ++ kbasep_os_process_page_usage_update(kctx, 0 - pages); ++} ++ ++/** ++ * kbasep_find_enclosing_cpu_mapping_offset() - Find the offset of the CPU ++ * mapping of a memory allocation containing a given address range ++ * ++ * Searches for a CPU mapping of any part of any region that fully encloses the ++ * CPU virtual address range specified by @uaddr and @size. Returns a failure ++ * indication if only part of the address range lies within a CPU mapping. ++ * ++ * @kctx: The kernel base context used for the allocation. ++ * @uaddr: Start of the CPU virtual address range. ++ * @size: Size of the CPU virtual address range (in bytes). ++ * @offset: The offset from the start of the allocation to the specified CPU ++ * virtual address. ++ * ++ * Return: 0 if offset was obtained successfully. Error code otherwise. ++ */ ++int kbasep_find_enclosing_cpu_mapping_offset( ++ struct kbase_context *kctx, ++ unsigned long uaddr, size_t size, u64 *offset); ++ ++/** ++ * kbasep_find_enclosing_gpu_mapping_start_and_offset() - Find the address of ++ * the start of GPU virtual memory region which encloses @gpu_addr for the ++ * @size length in bytes ++ * ++ * Searches for the memory region in GPU virtual memory space which contains ++ * the region defined by the @gpu_addr and @size, where @gpu_addr is the ++ * beginning and @size the length in bytes of the provided region. If found, ++ * the location of the start address of the GPU virtual memory region is ++ * passed in @start pointer and the location of the offset of the region into ++ * the GPU virtual memory region is passed in @offset pointer. ++ * ++ * @kctx: The kernel base context within which the memory is searched. ++ * @gpu_addr: GPU virtual address for which the region is sought; defines ++ * the beginning of the provided region. ++ * @size: The length (in bytes) of the provided region for which the ++ * GPU virtual memory region is sought. ++ * @start: Pointer to the location where the address of the start of ++ * the found GPU virtual memory region is. ++ * @offset: Pointer to the location where the offset of @gpu_addr into ++ * the found GPU virtual memory region is. ++ */ ++int kbasep_find_enclosing_gpu_mapping_start_and_offset( ++ struct kbase_context *kctx, ++ u64 gpu_addr, size_t size, u64 *start, u64 *offset); ++ ++/** ++ * kbase_alloc_phy_pages_helper - Allocates physical pages. ++ * @alloc: allocation object to add pages to ++ * @nr_pages_requested: number of physical pages to allocate ++ * ++ * Allocates \a nr_pages_requested and updates the alloc object. ++ * ++ * Return: 0 if all pages have been successfully allocated. Error code otherwise ++ * ++ * Note : The caller must not hold vm_lock, as this could cause a deadlock if ++ * the kernel OoM killer runs. If the caller must allocate pages while holding ++ * this lock, it should use kbase_mem_pool_alloc_pages_locked() instead. ++ * ++ * This function cannot be used from interrupt context ++ */ ++int kbase_alloc_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, ++ size_t nr_pages_requested); ++ ++/** ++ * kbase_alloc_phy_pages_helper_locked - Allocates physical pages. ++ * @alloc: allocation object to add pages to ++ * @pool: Memory pool to allocate from ++ * @nr_pages_requested: number of physical pages to allocate ++ * @prealloc_sa: Information about the partial allocation if the amount ++ * of memory requested is not a multiple of 2MB. One ++ * instance of struct kbase_sub_alloc must be allocated by ++ * the caller iff CONFIG_MALI_2MB_ALLOC is enabled. ++ * ++ * Allocates \a nr_pages_requested and updates the alloc object. This function ++ * does not allocate new pages from the kernel, and therefore will never trigger ++ * the OoM killer. Therefore, it can be run while the vm_lock is held. ++ * ++ * As new pages can not be allocated, the caller must ensure there are ++ * sufficient pages in the pool. Usage of this function should look like : ++ * ++ * kbase_gpu_vm_lock(kctx); ++ * kbase_mem_pool_lock(pool) ++ * while (kbase_mem_pool_size(pool) < pages_required) { ++ * kbase_mem_pool_unlock(pool) ++ * kbase_gpu_vm_unlock(kctx); ++ * kbase_mem_pool_grow(pool) ++ * kbase_gpu_vm_lock(kctx); ++ * kbase_mem_pool_lock(pool) ++ * } ++ * kbase_alloc_phy_pages_helper_locked(pool) ++ * kbase_mem_pool_unlock(pool) ++ * Perform other processing that requires vm_lock... ++ * kbase_gpu_vm_unlock(kctx); ++ * ++ * This ensures that the pool can be grown to the required size and that the ++ * allocation can complete without another thread using the newly grown pages. ++ * ++ * If CONFIG_MALI_2MB_ALLOC is defined and the allocation is >= 2MB, then ++ * @pool must be alloc->imported.native.kctx->lp_mem_pool. Otherwise it must be ++ * alloc->imported.native.kctx->mem_pool. ++ * @prealloc_sa is used to manage the non-2MB sub-allocation. It has to be ++ * pre-allocated because we must not sleep (due to the usage of kmalloc()) ++ * whilst holding pool->pool_lock. ++ * @prealloc_sa shall be set to NULL if it has been consumed by this function ++ * to indicate that the caller must not free it. ++ * ++ * Return: Pointer to array of allocated pages. NULL on failure. ++ * ++ * Note : Caller must hold pool->pool_lock ++ */ ++struct tagged_addr *kbase_alloc_phy_pages_helper_locked( ++ struct kbase_mem_phy_alloc *alloc, struct kbase_mem_pool *pool, ++ size_t nr_pages_requested, ++ struct kbase_sub_alloc **prealloc_sa); ++ ++/** ++ * kbase_free_phy_pages_helper() - Free physical pages. ++ * ++ * Frees \a nr_pages and updates the alloc object. ++ * ++ * @alloc: allocation object to free pages from ++ * @nr_pages_to_free: number of physical pages to free ++ * ++ * Return: 0 on success, otherwise a negative error code ++ */ ++int kbase_free_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, size_t nr_pages_to_free); ++ ++/** ++ * kbase_free_phy_pages_helper_locked - Free pages allocated with ++ * kbase_alloc_phy_pages_helper_locked() ++ * @alloc: Allocation object to free pages from ++ * @pool: Memory pool to return freed pages to ++ * @pages: Pages allocated by kbase_alloc_phy_pages_helper_locked() ++ * @nr_pages_to_free: Number of physical pages to free ++ * ++ * This function atomically frees pages allocated with ++ * kbase_alloc_phy_pages_helper_locked(). @pages is the pointer to the page ++ * array that is returned by that function. @pool must be the pool that the ++ * pages were originally allocated from. ++ * ++ * If the mem_pool has been unlocked since the allocation then ++ * kbase_free_phy_pages_helper() should be used instead. ++ */ ++void kbase_free_phy_pages_helper_locked(struct kbase_mem_phy_alloc *alloc, ++ struct kbase_mem_pool *pool, struct tagged_addr *pages, ++ size_t nr_pages_to_free); ++ ++static inline void kbase_set_dma_addr(struct page *p, dma_addr_t dma_addr) ++{ ++ SetPagePrivate(p); ++ if (sizeof(dma_addr_t) > sizeof(p->private)) { ++ /* on 32-bit ARM with LPAE dma_addr_t becomes larger, but the ++ * private field stays the same. So we have to be clever and ++ * use the fact that we only store DMA addresses of whole pages, ++ * so the low bits should be zero */ ++ KBASE_DEBUG_ASSERT(!(dma_addr & (PAGE_SIZE - 1))); ++ set_page_private(p, dma_addr >> PAGE_SHIFT); ++ } else { ++ set_page_private(p, dma_addr); ++ } ++} ++ ++static inline dma_addr_t kbase_dma_addr(struct page *p) ++{ ++ if (sizeof(dma_addr_t) > sizeof(p->private)) ++ return ((dma_addr_t)page_private(p)) << PAGE_SHIFT; ++ ++ return (dma_addr_t)page_private(p); ++} ++ ++static inline void kbase_clear_dma_addr(struct page *p) ++{ ++ ClearPagePrivate(p); ++} ++ ++/** ++ * kbase_flush_mmu_wqs() - Flush MMU workqueues. ++ * @kbdev: Device pointer. ++ * ++ * This function will cause any outstanding page or bus faults to be processed. ++ * It should be called prior to powering off the GPU. ++ */ ++void kbase_flush_mmu_wqs(struct kbase_device *kbdev); ++ ++/** ++ * kbase_sync_single_for_device - update physical memory and give GPU ownership ++ * @kbdev: Device pointer ++ * @handle: DMA address of region ++ * @size: Size of region to sync ++ * @dir: DMA data direction ++ */ ++ ++void kbase_sync_single_for_device(struct kbase_device *kbdev, dma_addr_t handle, ++ size_t size, enum dma_data_direction dir); ++ ++/** ++ * kbase_sync_single_for_cpu - update physical memory and give CPU ownership ++ * @kbdev: Device pointer ++ * @handle: DMA address of region ++ * @size: Size of region to sync ++ * @dir: DMA data direction ++ */ ++ ++void kbase_sync_single_for_cpu(struct kbase_device *kbdev, dma_addr_t handle, ++ size_t size, enum dma_data_direction dir); ++ ++#ifdef CONFIG_DEBUG_FS ++/** ++ * kbase_jit_debugfs_init - Add per context debugfs entry for JIT. ++ * @kctx: kbase context ++ */ ++void kbase_jit_debugfs_init(struct kbase_context *kctx); ++#endif /* CONFIG_DEBUG_FS */ ++ ++/** ++ * kbase_jit_init - Initialize the JIT memory pool management ++ * @kctx: kbase context ++ * ++ * Returns zero on success or negative error number on failure. ++ */ ++int kbase_jit_init(struct kbase_context *kctx); ++ ++/** ++ * kbase_jit_allocate - Allocate JIT memory ++ * @kctx: kbase context ++ * @info: JIT allocation information ++ * @ignore_pressure_limit: Whether the JIT memory pressure limit is ignored ++ * ++ * Return: JIT allocation on success or NULL on failure. ++ */ ++struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx, ++ const struct base_jit_alloc_info *info, ++ bool ignore_pressure_limit); ++ ++/** ++ * kbase_jit_free - Free a JIT allocation ++ * @kctx: kbase context ++ * @reg: JIT allocation ++ * ++ * Frees a JIT allocation and places it into the free pool for later reuse. ++ */ ++void kbase_jit_free(struct kbase_context *kctx, struct kbase_va_region *reg); ++ ++/** ++ * kbase_jit_backing_lost - Inform JIT that an allocation has lost backing ++ * @reg: JIT allocation ++ */ ++void kbase_jit_backing_lost(struct kbase_va_region *reg); ++ ++/** ++ * kbase_jit_evict - Evict a JIT allocation from the pool ++ * @kctx: kbase context ++ * ++ * Evict the least recently used JIT allocation from the pool. This can be ++ * required if normal VA allocations are failing due to VA exhaustion. ++ * ++ * Return: True if a JIT allocation was freed, false otherwise. ++ */ ++bool kbase_jit_evict(struct kbase_context *kctx); ++ ++/** ++ * kbase_jit_term - Terminate the JIT memory pool management ++ * @kctx: kbase context ++ */ ++void kbase_jit_term(struct kbase_context *kctx); ++ ++#if MALI_JIT_PRESSURE_LIMIT_BASE ++/** ++ * kbase_trace_jit_report_gpu_mem_trace_enabled - variant of ++ * kbase_trace_jit_report_gpu_mem() that should only be called once the ++ * corresponding tracepoint is verified to be enabled ++ * @kctx: kbase context ++ * @reg: Just-in-time memory region to trace ++ * @flags: combination of values from enum kbase_jit_report_flags ++ */ ++void kbase_trace_jit_report_gpu_mem_trace_enabled(struct kbase_context *kctx, ++ struct kbase_va_region *reg, unsigned int flags); ++#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ ++ ++/** ++ * kbase_trace_jit_report_gpu_mem - Trace information about the GPU memory used ++ * to make a JIT report ++ * @kctx: kbase context ++ * @reg: Just-in-time memory region to trace ++ * @flags: combination of values from enum kbase_jit_report_flags ++ * ++ * Information is traced using the trace_mali_jit_report_gpu_mem() tracepoint. ++ * ++ * In case that tracepoint is not enabled, this function should have the same ++ * low overheads as a tracepoint itself (i.e. use of 'jump labels' to avoid ++ * conditional branches) ++ * ++ * This can take the reg_lock on @kctx, do not use in places where this lock is ++ * already held. ++ * ++ * Note: this has to be a macro because at this stage the tracepoints have not ++ * been included. Also gives no opportunity for the compiler to mess up ++ * inlining it. ++ */ ++#if MALI_JIT_PRESSURE_LIMIT_BASE ++#define kbase_trace_jit_report_gpu_mem(kctx, reg, flags) \ ++ do { \ ++ if (trace_mali_jit_report_gpu_mem_enabled()) \ ++ kbase_trace_jit_report_gpu_mem_trace_enabled( \ ++ (kctx), (reg), (flags)); \ ++ } while (0) ++#else ++#define kbase_trace_jit_report_gpu_mem(kctx, reg, flags) \ ++ CSTD_NOP(kctx, reg, flags) ++#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ ++ ++#if MALI_JIT_PRESSURE_LIMIT_BASE ++/** ++ * kbase_jit_report_update_pressure - safely update the JIT physical page ++ * pressure and JIT region's estimate of used_pages ++ * @kctx: kbase context, to update the current physical pressure ++ * @reg: Just-in-time memory region to update with @new_used_pages ++ * @new_used_pages: new value of number of pages used in the JIT region ++ * @flags: combination of values from enum kbase_jit_report_flags ++ * ++ * Takes care of: ++ * - correctly updating the pressure given the current reg->used_pages and ++ * new_used_pages ++ * - then updating the %kbase_va_region used_pages member ++ * ++ * Precondition: ++ * - new_used_pages <= reg->nr_pages ++ */ ++void kbase_jit_report_update_pressure(struct kbase_context *kctx, ++ struct kbase_va_region *reg, u64 new_used_pages, ++ unsigned int flags); ++ ++/** ++ * jit_trim_necessary_pages() - calculate and trim the least pages possible to ++ * satisfy a new JIT allocation ++ * ++ * @kctx: Pointer to the kbase context ++ * @needed_pages: Number of JIT physical pages by which trimming is requested. ++ * The actual number of pages trimmed could differ. ++ * ++ * Before allocating a new just-in-time memory region or reusing a previous ++ * one, ensure that the total JIT physical page usage also will not exceed the ++ * pressure limit. ++ * ++ * If there are no reported-on allocations, then we already guarantee this will ++ * be the case - because our current pressure then only comes from the va_pages ++ * of each JIT region, hence JIT physical page usage is guaranteed to be ++ * bounded by this. ++ * ++ * However as soon as JIT allocations become "reported on", the pressure is ++ * lowered to allow new JIT regions to be allocated. It is after such a point ++ * that the total JIT physical page usage could (either now or in the future on ++ * a grow-on-GPU-page-fault) exceed the pressure limit, but only on newly ++ * allocated JIT regions. Hence, trim any "reported on" regions. ++ * ++ * Any pages freed will go into the pool and be allocated from there in ++ * kbase_mem_alloc(). ++ */ ++void kbase_jit_trim_necessary_pages(struct kbase_context *kctx, ++ size_t needed_pages); ++ ++/* ++ * Same as kbase_jit_request_phys_increase(), except that Caller is supposed ++ * to take jit_evict_lock also on @kctx before calling this function. ++ */ ++static inline void ++kbase_jit_request_phys_increase_locked(struct kbase_context *kctx, ++ size_t needed_pages) ++{ ++#if !MALI_USE_CSF ++ lockdep_assert_held(&kctx->jctx.lock); ++#endif /* !MALI_USE_CSF */ ++ lockdep_assert_held(&kctx->reg_lock); ++ lockdep_assert_held(&kctx->jit_evict_lock); ++ ++ kctx->jit_phys_pages_to_be_allocated += needed_pages; ++ ++ kbase_jit_trim_necessary_pages(kctx, ++ kctx->jit_phys_pages_to_be_allocated); ++} ++ ++/** ++ * kbase_jit_request_phys_increase() - Increment the backing pages count and do ++ * the required trimming before allocating pages for a JIT allocation. ++ * ++ * @kctx: Pointer to the kbase context ++ * @needed_pages: Number of pages to be allocated for the JIT allocation. ++ * ++ * This function needs to be called before allocating backing pages for a ++ * just-in-time memory region. The backing pages are currently allocated when, ++ * ++ * - A new JIT region is created. ++ * - An old JIT region is reused from the cached pool. ++ * - GPU page fault occurs for the active JIT region. ++ * - Backing is grown for the JIT region through the commit ioctl. ++ * ++ * This function would ensure that the total JIT physical page usage does not ++ * exceed the pressure limit even when the backing pages get allocated ++ * simultaneously for multiple JIT allocations from different threads. ++ * ++ * There should be a matching call to kbase_jit_done_phys_increase(), after ++ * the pages have been allocated and accounted against the active JIT ++ * allocation. ++ * ++ * Caller is supposed to take reg_lock on @kctx before calling this function. ++ */ ++static inline void kbase_jit_request_phys_increase(struct kbase_context *kctx, ++ size_t needed_pages) ++{ ++#if !MALI_USE_CSF ++ lockdep_assert_held(&kctx->jctx.lock); ++#endif /* !MALI_USE_CSF */ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ mutex_lock(&kctx->jit_evict_lock); ++ kbase_jit_request_phys_increase_locked(kctx, needed_pages); ++ mutex_unlock(&kctx->jit_evict_lock); ++} ++ ++/** ++ * kbase_jit_done_phys_increase() - Decrement the backing pages count after the ++ * allocation of pages for a JIT allocation. ++ * ++ * @kctx: Pointer to the kbase context ++ * @needed_pages: Number of pages that were allocated for the JIT allocation. ++ * ++ * This function should be called after backing pages have been allocated and ++ * accounted against the active JIT allocation. ++ * The call should be made when the following have been satisfied: ++ * when the allocation is on the jit_active_head. ++ * when additional needed_pages have been allocated. ++ * kctx->reg_lock was held during the above and has not yet been unlocked. ++ * Failure to call this function before unlocking the kctx->reg_lock when ++ * either the above have changed may result in over-accounting the memory. ++ * This ensures kbase_jit_trim_necessary_pages() gets a consistent count of ++ * the memory. ++ * ++ * A matching call to kbase_jit_request_phys_increase() should have been made, ++ * before the allocation of backing pages. ++ * ++ * Caller is supposed to take reg_lock on @kctx before calling this function. ++ */ ++static inline void kbase_jit_done_phys_increase(struct kbase_context *kctx, ++ size_t needed_pages) ++{ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ WARN_ON(kctx->jit_phys_pages_to_be_allocated < needed_pages); ++ ++ kctx->jit_phys_pages_to_be_allocated -= needed_pages; ++} ++#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ ++ ++/** ++ * kbase_has_exec_va_zone - EXEC_VA zone predicate ++ * ++ * Determine whether an EXEC_VA zone has been created for the GPU address space ++ * of the given kbase context. ++ * ++ * @kctx: kbase context ++ * ++ * Return: True if the kbase context has an EXEC_VA zone. ++ */ ++bool kbase_has_exec_va_zone(struct kbase_context *kctx); ++ ++/** ++ * kbase_map_external_resource - Map an external resource to the GPU. ++ * @kctx: kbase context. ++ * @reg: The region to map. ++ * @locked_mm: The mm_struct which has been locked for this operation. ++ * ++ * Return: The physical allocation which backs the region on success or NULL ++ * on failure. ++ */ ++struct kbase_mem_phy_alloc *kbase_map_external_resource( ++ struct kbase_context *kctx, struct kbase_va_region *reg, ++ struct mm_struct *locked_mm); ++ ++/** ++ * kbase_unmap_external_resource - Unmap an external resource from the GPU. ++ * @kctx: kbase context. ++ * @reg: The region to unmap or NULL if it has already been released. ++ * @alloc: The physical allocation being unmapped. ++ */ ++void kbase_unmap_external_resource(struct kbase_context *kctx, ++ struct kbase_va_region *reg, struct kbase_mem_phy_alloc *alloc); ++ ++ ++/** ++ * kbase_jd_user_buf_pin_pages - Pin the pages of a user buffer. ++ * @kctx: kbase context. ++ * @reg: The region associated with the imported user buffer. ++ * ++ * To successfully pin the pages for a user buffer the current mm_struct must ++ * be the same as the mm_struct of the user buffer. After successfully pinning ++ * the pages further calls to this function succeed without doing work. ++ * ++ * Return: zero on success or negative number on failure. ++ */ ++int kbase_jd_user_buf_pin_pages(struct kbase_context *kctx, ++ struct kbase_va_region *reg); ++ ++/** ++ * kbase_sticky_resource_init - Initialize sticky resource management. ++ * @kctx: kbase context ++ * ++ * Returns zero on success or negative error number on failure. ++ */ ++int kbase_sticky_resource_init(struct kbase_context *kctx); ++ ++/** ++ * kbase_sticky_resource_acquire - Acquire a reference on a sticky resource. ++ * @kctx: kbase context. ++ * @gpu_addr: The GPU address of the external resource. ++ * ++ * Return: The metadata object which represents the binding between the ++ * external resource and the kbase context on success or NULL on failure. ++ */ ++struct kbase_ctx_ext_res_meta *kbase_sticky_resource_acquire( ++ struct kbase_context *kctx, u64 gpu_addr); ++ ++/** ++ * kbase_sticky_resource_release - Release a reference on a sticky resource. ++ * @kctx: kbase context. ++ * @meta: Binding metadata. ++ * @gpu_addr: GPU address of the external resource. ++ * ++ * If meta is NULL then gpu_addr will be used to scan the metadata list and ++ * find the matching metadata (if any), otherwise the provided meta will be ++ * used and gpu_addr will be ignored. ++ * ++ * Return: True if the release found the metadata and the reference was dropped. ++ */ ++bool kbase_sticky_resource_release(struct kbase_context *kctx, ++ struct kbase_ctx_ext_res_meta *meta, u64 gpu_addr); ++ ++/** ++ * kbase_sticky_resource_release_force - Release a sticky resource. ++ * @kctx: kbase context. ++ * @meta: Binding metadata. ++ * @gpu_addr: GPU address of the external resource. ++ * ++ * If meta is NULL then gpu_addr will be used to scan the metadata list and ++ * find the matching metadata (if any), otherwise the provided meta will be ++ * used and gpu_addr will be ignored. ++ * ++ * Return: True if the release found the metadata and the resource was ++ * released. ++ */ ++bool kbase_sticky_resource_release_force(struct kbase_context *kctx, ++ struct kbase_ctx_ext_res_meta *meta, u64 gpu_addr); ++ ++/** ++ * kbase_sticky_resource_term - Terminate sticky resource management. ++ * @kctx: kbase context ++ */ ++void kbase_sticky_resource_term(struct kbase_context *kctx); ++ ++/** ++ * kbase_mem_pool_lock - Lock a memory pool ++ * @pool: Memory pool to lock ++ */ ++static inline void kbase_mem_pool_lock(struct kbase_mem_pool *pool) ++{ ++ spin_lock(&pool->pool_lock); ++} ++ ++/** ++ * kbase_mem_pool_lock - Release a memory pool ++ * @pool: Memory pool to lock ++ */ ++static inline void kbase_mem_pool_unlock(struct kbase_mem_pool *pool) ++{ ++ spin_unlock(&pool->pool_lock); ++} ++ ++/** ++ * kbase_mem_evictable_mark_reclaim - Mark the pages as reclaimable. ++ * @alloc: The physical allocation ++ */ ++void kbase_mem_evictable_mark_reclaim(struct kbase_mem_phy_alloc *alloc); ++ ++#if MALI_USE_CSF ++/** ++ * kbase_link_event_mem_page - Add the new event memory region to the per ++ * context list of event pages. ++ * @kctx: Pointer to kbase context ++ * @reg: Pointer to the region allocated for event memory. ++ * ++ * The region being linked shouldn't have been marked as free and should ++ * have KBASE_REG_CSF_EVENT flag set for it. ++ */ ++static inline void kbase_link_event_mem_page(struct kbase_context *kctx, ++ struct kbase_va_region *reg) ++{ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ WARN_ON(reg->flags & KBASE_REG_FREE); ++ WARN_ON(!(reg->flags & KBASE_REG_CSF_EVENT)); ++ ++ list_add(®->link, &kctx->csf.event_pages_head); ++} ++ ++/** ++ * kbase_unlink_event_mem_page - Remove the event memory region from the per ++ * context list of event pages. ++ * @kctx: Pointer to kbase context ++ * @reg: Pointer to the region allocated for event memory. ++ * ++ * The region being un-linked shouldn't have been marked as free and should ++ * have KBASE_REG_CSF_EVENT flag set for it. ++ */ ++static inline void kbase_unlink_event_mem_page(struct kbase_context *kctx, ++ struct kbase_va_region *reg) ++{ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ WARN_ON(reg->flags & KBASE_REG_FREE); ++ WARN_ON(!(reg->flags & KBASE_REG_CSF_EVENT)); ++ ++ list_del(®->link); ++} ++ ++/** ++ * kbase_mcu_shared_interface_region_tracker_init - Initialize the rb tree to ++ * manage the shared interface segment of MCU firmware address space. ++ * @kbdev: Pointer to the kbase device ++ * ++ * Returns zero on success or negative error number on failure. ++ */ ++int kbase_mcu_shared_interface_region_tracker_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_mcu_shared_interface_region_tracker_term - Teardown the rb tree ++ * managing the shared interface segment of MCU firmware address space. ++ * @kbdev: Pointer to the kbase device ++ */ ++void kbase_mcu_shared_interface_region_tracker_term(struct kbase_device *kbdev); ++#endif ++ ++/** ++ * kbase_mem_umm_map - Map dma-buf ++ * @kctx: Pointer to the kbase context ++ * @reg: Pointer to the region of the imported dma-buf to map ++ * ++ * Map a dma-buf on the GPU. The mappings are reference counted. ++ * ++ * Returns 0 on success, or a negative error code. ++ */ ++int kbase_mem_umm_map(struct kbase_context *kctx, ++ struct kbase_va_region *reg); ++ ++/** ++ * kbase_mem_umm_unmap - Unmap dma-buf ++ * @kctx: Pointer to the kbase context ++ * @reg: Pointer to the region of the imported dma-buf to unmap ++ * @alloc: Pointer to the alloc to release ++ * ++ * Unmap a dma-buf from the GPU. The mappings are reference counted. ++ * ++ * @reg must be the original region with GPU mapping of @alloc; or NULL. If ++ * @reg is NULL, or doesn't match @alloc, the GPU page table entries matching ++ * @reg will not be updated. ++ * ++ * @alloc must be a valid physical allocation of type ++ * KBASE_MEM_TYPE_IMPORTED_UMM that was previously mapped by ++ * kbase_mem_umm_map(). The dma-buf attachment referenced by @alloc will ++ * release it's mapping reference, and if the refcount reaches 0, also be be ++ * unmapped, regardless of the value of @reg. ++ */ ++void kbase_mem_umm_unmap(struct kbase_context *kctx, ++ struct kbase_va_region *reg, struct kbase_mem_phy_alloc *alloc); ++ ++/** ++ * kbase_mem_do_sync_imported - Sync caches for imported memory ++ * @kctx: Pointer to the kbase context ++ * @reg: Pointer to the region with imported memory to sync ++ * @sync_fn: The type of sync operation to perform ++ * ++ * Sync CPU caches for supported (currently only dma-buf (UMM)) memory. ++ * Attempting to sync unsupported imported memory types will result in an error ++ * code, -EINVAL. ++ * ++ * Return: 0 on success, or a negative error code. ++ */ ++int kbase_mem_do_sync_imported(struct kbase_context *kctx, ++ struct kbase_va_region *reg, enum kbase_sync_type sync_fn); ++ ++/** ++ * kbase_mem_copy_to_pinned_user_pages - Memcpy from source input page to ++ * an unaligned address at a given offset from the start of a target page. ++ * ++ * @dest_pages: Pointer to the array of pages to which the content is ++ * to be copied from the provided @src_page. ++ * @src_page: Pointer to the page which correspond to the source page ++ * from which the copying will take place. ++ * @to_copy: Total number of bytes pending to be copied from ++ * @src_page to @target_page_nr within @dest_pages. ++ * This will get decremented by number of bytes we ++ * managed to copy from source page to target pages. ++ * @nr_pages: Total number of pages present in @dest_pages. ++ * @target_page_nr: Target page number to which @src_page needs to be ++ * copied. This will get incremented by one if ++ * we are successful in copying from source page. ++ * @offset: Offset in bytes into the target pages from which the ++ * copying is to be performed. ++ * ++ * Return: 0 on success, or a negative error code. ++ */ ++int kbase_mem_copy_to_pinned_user_pages(struct page **dest_pages, ++ void *src_page, size_t *to_copy, unsigned int nr_pages, ++ unsigned int *target_page_nr, size_t offset); ++ ++#endif /* _KBASE_MEM_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.c b/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.c +new file mode 100755 +index 000000000000..99b5b852667e +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.c +@@ -0,0 +1,3425 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++/** ++ * @file mali_kbase_mem_linux.c ++ * Base kernel memory APIs, Linux implementation. ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)) && \ ++ (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0)) ++#include ++#endif /* LINUX_VERSION_CODE >= 3.5.0 && < 4.8.0 */ ++#include ++#include ++#include ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#if ((KERNEL_VERSION(5, 3, 0) <= LINUX_VERSION_CODE) || \ ++ (KERNEL_VERSION(5, 0, 0) > LINUX_VERSION_CODE)) ++/* Enable workaround for ion for kernels prior to v5.0.0 and from v5.3.0 ++ * onwards. ++ * ++ * For kernels prior to v4.12, workaround is needed as ion lacks the cache ++ * maintenance in begin_cpu_access and end_cpu_access methods. ++ * ++ * For kernels prior to v4.17.2, workaround is needed to avoid the potentially ++ * disruptive warnings which can come if begin_cpu_access and end_cpu_access ++ * methods are not called in pairs. ++ * Note that some long term maintenance kernel versions (e.g. 4.9.x, 4.14.x) ++ * only require this workaround on their earlier releases. However it is still ++ * safe to use it on such releases, and it simplifies the version check. ++ * ++ * For kernels later than v4.17.2, workaround is needed as ion can potentially ++ * end up calling dma_sync_sg_for_* for a dma-buf importer that hasn't mapped ++ * the attachment. This would result in a kernel panic as ion populates the ++ * dma_address when the attachment is mapped and kernel derives the physical ++ * address for cache maintenance from the dma_address. ++ * With some multi-threaded tests it has been seen that the same dma-buf memory ++ * gets imported twice on Mali DDK side and so the problem of sync happening ++ * with an importer having an unmapped attachment comes at the time of 2nd ++ * import. The same problem can if there is another importer of dma-buf ++ * memory. ++ * ++ * Workaround can be safely disabled for kernels between v5.0.0 and v5.2.2, ++ * as all the above stated issues are not there. ++ * ++ * dma_sync_sg_for_* calls will be made directly as a workaround using the ++ * Kbase's attachment to dma-buf that was previously mapped. ++ */ ++#define KBASE_MEM_ION_SYNC_WORKAROUND ++#endif ++ ++#define IR_THRESHOLD_STEPS (256u) ++ ++#if MALI_USE_CSF ++static int kbase_csf_cpu_mmap_user_reg_page(struct kbase_context *kctx, ++ struct vm_area_struct *vma); ++static int kbase_csf_cpu_mmap_user_io_pages(struct kbase_context *kctx, ++ struct vm_area_struct *vma); ++#endif ++ ++static int kbase_vmap_phy_pages(struct kbase_context *kctx, ++ struct kbase_va_region *reg, u64 offset_bytes, size_t size, ++ struct kbase_vmap_struct *map); ++static void kbase_vunmap_phy_pages(struct kbase_context *kctx, ++ struct kbase_vmap_struct *map); ++ ++static int kbase_tracking_page_setup(struct kbase_context *kctx, struct vm_area_struct *vma); ++ ++static int kbase_mem_shrink_gpu_mapping(struct kbase_context *kctx, ++ struct kbase_va_region *reg, ++ u64 new_pages, u64 old_pages); ++ ++/* Retrieve the associated region pointer if the GPU address corresponds to ++ * one of the event memory pages. The enclosing region, if found, shouldn't ++ * have been marked as free. ++ */ ++static struct kbase_va_region *kbase_find_event_mem_region( ++ struct kbase_context *kctx, u64 gpu_addr) ++{ ++#if MALI_USE_CSF ++ u64 gpu_pfn = gpu_addr >> PAGE_SHIFT; ++ struct kbase_va_region *reg; ++ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ list_for_each_entry(reg, &kctx->csf.event_pages_head, link) { ++ if ((reg->start_pfn <= gpu_pfn) && ++ (gpu_pfn < (reg->start_pfn + reg->nr_pages))) { ++ if (WARN_ON(reg->flags & KBASE_REG_FREE)) ++ return NULL; ++ ++ if (WARN_ON(!(reg->flags & KBASE_REG_CSF_EVENT))) ++ return NULL; ++ ++ return reg; ++ } ++ } ++#endif ++ ++ return NULL; ++} ++ ++/** ++ * kbase_phy_alloc_mapping_init - Initialize the kernel side permanent mapping ++ * of the physical allocation belonging to a ++ * region ++ * @kctx: The kernel base context @reg belongs to. ++ * @reg: The region whose physical allocation is to be mapped ++ * @vsize: The size of the requested region, in pages ++ * @size: The size in pages initially committed to the region ++ * ++ * Return: 0 on success, otherwise an error code indicating failure ++ * ++ * Maps the physical allocation backing a non-free @reg, so it may be ++ * accessed directly from the kernel. This is only supported for physical ++ * allocations of type KBASE_MEM_TYPE_NATIVE, and will fail for other types of ++ * physical allocation. ++ * ++ * The mapping is stored directly in the allocation that backs @reg. The ++ * refcount is not incremented at this point. Instead, use of the mapping should ++ * be surrounded by kbase_phy_alloc_mapping_get() and ++ * kbase_phy_alloc_mapping_put() to ensure it does not disappear whilst the ++ * client is accessing it. ++ * ++ * Both cached and uncached regions are allowed, but any sync operations are the ++ * responsibility of the client using the permanent mapping. ++ * ++ * A number of checks are made to ensure that a region that needs a permanent ++ * mapping can actually be supported: ++ * - The region must be created as fully backed ++ * - The region must not be growable ++ * ++ * This function will fail if those checks are not satisfied. ++ * ++ * On success, the region will also be forced into a certain kind: ++ * - It will no longer be growable ++ */ ++static int kbase_phy_alloc_mapping_init(struct kbase_context *kctx, ++ struct kbase_va_region *reg, size_t vsize, size_t size) ++{ ++ size_t size_bytes = (size << PAGE_SHIFT); ++ struct kbase_vmap_struct *kern_mapping; ++ int err = 0; ++ ++ /* Can only map in regions that are always fully committed ++ * Don't setup the mapping twice ++ * Only support KBASE_MEM_TYPE_NATIVE allocations ++ */ ++ if (vsize != size || reg->cpu_alloc->permanent_map != NULL || ++ reg->cpu_alloc->type != KBASE_MEM_TYPE_NATIVE) ++ return -EINVAL; ++ ++ if (size > (KBASE_PERMANENTLY_MAPPED_MEM_LIMIT_PAGES - ++ atomic_read(&kctx->permanent_mapped_pages))) { ++ dev_warn(kctx->kbdev->dev, "Request for %llu more pages mem needing a permanent mapping would breach limit %lu, currently at %d pages", ++ (u64)size, ++ KBASE_PERMANENTLY_MAPPED_MEM_LIMIT_PAGES, ++ atomic_read(&kctx->permanent_mapped_pages)); ++ return -ENOMEM; ++ } ++ ++ kern_mapping = kzalloc(sizeof(*kern_mapping), GFP_KERNEL); ++ if (!kern_mapping) ++ return -ENOMEM; ++ ++ err = kbase_vmap_phy_pages(kctx, reg, 0u, size_bytes, kern_mapping); ++ if (err < 0) ++ goto vmap_fail; ++ ++ /* No support for growing or shrinking mapped regions */ ++ reg->flags &= ~KBASE_REG_GROWABLE; ++ ++ reg->cpu_alloc->permanent_map = kern_mapping; ++ atomic_add(size, &kctx->permanent_mapped_pages); ++ ++ return 0; ++vmap_fail: ++ kfree(kern_mapping); ++ return err; ++} ++ ++void kbase_phy_alloc_mapping_term(struct kbase_context *kctx, ++ struct kbase_mem_phy_alloc *alloc) ++{ ++ WARN_ON(!alloc->permanent_map); ++ kbase_vunmap_phy_pages(kctx, alloc->permanent_map); ++ kfree(alloc->permanent_map); ++ ++ alloc->permanent_map = NULL; ++ ++ /* Mappings are only done on cpu_alloc, so don't need to worry about ++ * this being reduced a second time if a separate gpu_alloc is ++ * freed ++ */ ++ WARN_ON(alloc->nents > atomic_read(&kctx->permanent_mapped_pages)); ++ atomic_sub(alloc->nents, &kctx->permanent_mapped_pages); ++} ++ ++void *kbase_phy_alloc_mapping_get(struct kbase_context *kctx, ++ u64 gpu_addr, ++ struct kbase_vmap_struct **out_kern_mapping) ++{ ++ struct kbase_va_region *reg; ++ void *kern_mem_ptr = NULL; ++ struct kbase_vmap_struct *kern_mapping; ++ u64 mapping_offset; ++ ++ WARN_ON(!kctx); ++ WARN_ON(!out_kern_mapping); ++ ++ kbase_gpu_vm_lock(kctx); ++ ++ /* First do a quick lookup in the list of event memory regions */ ++ reg = kbase_find_event_mem_region(kctx, gpu_addr); ++ ++ if (!reg) { ++ reg = kbase_region_tracker_find_region_enclosing_address( ++ kctx, gpu_addr); ++ } ++ ++ if (kbase_is_region_invalid_or_free(reg)) ++ goto out_unlock; ++ ++ kern_mapping = reg->cpu_alloc->permanent_map; ++ if (kern_mapping == NULL) ++ goto out_unlock; ++ ++ mapping_offset = gpu_addr - (reg->start_pfn << PAGE_SHIFT); ++ ++ /* Refcount the allocations to prevent them disappearing */ ++ WARN_ON(reg->cpu_alloc != kern_mapping->cpu_alloc); ++ WARN_ON(reg->gpu_alloc != kern_mapping->gpu_alloc); ++ (void)kbase_mem_phy_alloc_get(kern_mapping->cpu_alloc); ++ (void)kbase_mem_phy_alloc_get(kern_mapping->gpu_alloc); ++ ++ kern_mem_ptr = (void *)(uintptr_t)((uintptr_t)kern_mapping->addr + mapping_offset); ++ *out_kern_mapping = kern_mapping; ++out_unlock: ++ kbase_gpu_vm_unlock(kctx); ++ return kern_mem_ptr; ++} ++ ++void kbase_phy_alloc_mapping_put(struct kbase_context *kctx, ++ struct kbase_vmap_struct *kern_mapping) ++{ ++ WARN_ON(!kctx); ++ WARN_ON(!kern_mapping); ++ ++ WARN_ON(kctx != kern_mapping->cpu_alloc->imported.native.kctx); ++ WARN_ON(kern_mapping != kern_mapping->cpu_alloc->permanent_map); ++ ++ kbase_mem_phy_alloc_put(kern_mapping->cpu_alloc); ++ kbase_mem_phy_alloc_put(kern_mapping->gpu_alloc); ++ ++ /* kern_mapping and the gpu/cpu phy allocs backing it must not be used ++ * from now on ++ */ ++} ++ ++struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, ++ u64 va_pages, u64 commit_pages, u64 extent, u64 *flags, ++ u64 *gpu_va) ++{ ++ int zone; ++ struct kbase_va_region *reg; ++ struct rb_root *rbtree; ++ struct device *dev; ++ ++ KBASE_DEBUG_ASSERT(kctx); ++ KBASE_DEBUG_ASSERT(flags); ++ KBASE_DEBUG_ASSERT(gpu_va); ++ ++ dev = kctx->kbdev->dev; ++ dev_dbg(dev, "Allocating %lld va_pages, %lld commit_pages, %lld extent, 0x%llX flags\n", ++ va_pages, commit_pages, extent, *flags); ++ ++#if MALI_USE_CSF ++ *gpu_va = 0; /* return 0 on failure */ ++#else ++ if (!(*flags & BASE_MEM_FLAG_MAP_FIXED)) ++ *gpu_va = 0; /* return 0 on failure */ ++ else ++ dev_err(dev, ++ "Keeping requested GPU VA of 0x%llx\n", ++ (unsigned long long)*gpu_va); ++#endif ++ ++ if (!kbase_check_alloc_flags(*flags)) { ++ dev_warn(dev, ++ "kbase_mem_alloc called with bad flags (%llx)", ++ (unsigned long long)*flags); ++ goto bad_flags; ++ } ++ ++#ifdef CONFIG_DEBUG_FS ++ if (unlikely(kbase_ctx_flag(kctx, KCTX_INFINITE_CACHE))) { ++ /* Mask coherency flags if infinite cache is enabled to prevent ++ * the skipping of syncs from BASE side. ++ */ ++ *flags &= ~(BASE_MEM_COHERENT_SYSTEM_REQUIRED | ++ BASE_MEM_COHERENT_SYSTEM); ++ } ++#endif ++ ++ if ((*flags & BASE_MEM_UNCACHED_GPU) != 0 && ++ (*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0) { ++ /* Remove COHERENT_SYSTEM_REQUIRED flag if uncached GPU mapping is requested */ ++ *flags &= ~BASE_MEM_COHERENT_SYSTEM_REQUIRED; ++ } ++ if ((*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0 && ++ !kbase_device_is_cpu_coherent(kctx->kbdev)) { ++ dev_warn(dev, "kbase_mem_alloc call required coherent mem when unavailable"); ++ goto bad_flags; ++ } ++ if ((*flags & BASE_MEM_COHERENT_SYSTEM) != 0 && ++ !kbase_device_is_cpu_coherent(kctx->kbdev)) { ++ /* Remove COHERENT_SYSTEM flag if coherent mem is unavailable */ ++ *flags &= ~BASE_MEM_COHERENT_SYSTEM; ++ } ++ ++ if (kbase_check_alloc_sizes(kctx, *flags, va_pages, commit_pages, extent)) ++ goto bad_sizes; ++ ++#ifdef CONFIG_MALI_MEMORY_FULLY_BACKED ++ /* Ensure that memory is fully physically-backed. */ ++ if (*flags & BASE_MEM_GROW_ON_GPF) ++ commit_pages = va_pages; ++#endif ++ ++ /* find out which VA zone to use */ ++ if (*flags & BASE_MEM_SAME_VA) { ++ rbtree = &kctx->reg_rbtree_same; ++ zone = KBASE_REG_ZONE_SAME_VA; ++ } else if ((*flags & BASE_MEM_PROT_GPU_EX) && kbase_has_exec_va_zone(kctx)) { ++ rbtree = &kctx->reg_rbtree_exec; ++ zone = KBASE_REG_ZONE_EXEC_VA; ++ } else { ++ rbtree = &kctx->reg_rbtree_custom; ++ zone = KBASE_REG_ZONE_CUSTOM_VA; ++ } ++ ++ reg = kbase_alloc_free_region(rbtree, PFN_DOWN(*gpu_va), ++ va_pages, zone); ++ ++ if (!reg) { ++ dev_err(dev, "Failed to allocate free region"); ++ goto no_region; ++ } ++ ++ if (kbase_update_region_flags(kctx, reg, *flags) != 0) ++ goto invalid_flags; ++ ++ if (kbase_reg_prepare_native(reg, kctx, ++ base_mem_group_id_get(*flags)) != 0) { ++ dev_err(dev, "Failed to prepare region"); ++ goto prepare_failed; ++ } ++ ++ if (unlikely(reg->cpu_alloc != reg->gpu_alloc)) ++ *flags |= BASE_MEM_KERNEL_SYNC; ++ ++ /* make sure base knows if the memory is actually cached or not */ ++ if (reg->flags & KBASE_REG_CPU_CACHED) ++ *flags |= BASE_MEM_CACHED_CPU; ++ else ++ *flags &= ~BASE_MEM_CACHED_CPU; ++ ++ if (*flags & BASE_MEM_GROW_ON_GPF) { ++ unsigned int const ir_threshold = atomic_read( ++ &kctx->kbdev->memdev.ir_threshold); ++ ++ reg->threshold_pages = ((va_pages * ir_threshold) + ++ (IR_THRESHOLD_STEPS / 2)) / IR_THRESHOLD_STEPS; ++ } else ++ reg->threshold_pages = 0; ++ ++ if (*flags & BASE_MEM_GROW_ON_GPF) { ++ /* kbase_check_alloc_sizes() already checks extent is valid for ++ * assigning to reg->extent */ ++ reg->extent = extent; ++#if !MALI_USE_CSF ++ } else if (*flags & BASE_MEM_TILER_ALIGN_TOP) { ++ reg->extent = extent; ++#endif /* !MALI_USE_CSF */ ++ } else { ++ reg->extent = 0; ++ } ++ ++ if (kbase_alloc_phy_pages(reg, va_pages, commit_pages) != 0) { ++ dev_warn(dev, "Failed to allocate %lld pages (va_pages=%lld)", ++ (unsigned long long)commit_pages, ++ (unsigned long long)va_pages); ++ goto no_mem; ++ } ++ reg->initial_commit = commit_pages; ++ ++ kbase_gpu_vm_lock(kctx); ++ ++ if (reg->flags & KBASE_REG_PERMANENT_KERNEL_MAPPING) { ++ /* Permanent kernel mappings must happen as soon as ++ * reg->cpu_alloc->pages is ready. Currently this happens after ++ * kbase_alloc_phy_pages(). If we move that to setup pages ++ * earlier, also move this call too ++ */ ++ int err = kbase_phy_alloc_mapping_init(kctx, reg, va_pages, ++ commit_pages); ++ if (err < 0) { ++ kbase_gpu_vm_unlock(kctx); ++ goto no_kern_mapping; ++ } ++ } ++ ++#if MALI_USE_CSF ++ if (reg->flags & KBASE_REG_CSF_EVENT) { ++ WARN_ON(!(*flags & BASE_MEM_SAME_VA)); ++ ++ kbase_link_event_mem_page(kctx, reg); ++ } ++#endif ++ ++ /* mmap needed to setup VA? */ ++ if (*flags & BASE_MEM_SAME_VA) { ++ unsigned long cookie, cookie_nr; ++ ++ /* Bind to a cookie */ ++ if (bitmap_empty(kctx->cookies, BITS_PER_LONG)) { ++ dev_err(dev, "No cookies available for allocation!"); ++ kbase_gpu_vm_unlock(kctx); ++ goto no_cookie; ++ } ++ /* return a cookie */ ++ cookie_nr = find_first_bit(kctx->cookies, BITS_PER_LONG); ++ bitmap_clear(kctx->cookies, cookie_nr, 1); ++ BUG_ON(kctx->pending_regions[cookie_nr]); ++ kctx->pending_regions[cookie_nr] = reg; ++ ++ /* relocate to correct base */ ++ cookie = cookie_nr + PFN_DOWN(BASE_MEM_COOKIE_BASE); ++ cookie <<= PAGE_SHIFT; ++ ++ *gpu_va = (u64) cookie; ++ } else /* we control the VA */ { ++ if (kbase_gpu_mmap(kctx, reg, *gpu_va, va_pages, 1) != 0) { ++ dev_warn(dev, "Failed to map memory on GPU"); ++ kbase_gpu_vm_unlock(kctx); ++ goto no_mmap; ++ } ++ /* return real GPU VA */ ++ *gpu_va = reg->start_pfn << PAGE_SHIFT; ++ } ++ ++#if MALI_JIT_PRESSURE_LIMIT_BASE ++ if (*flags & BASEP_MEM_PERFORM_JIT_TRIM) { ++ kbase_jit_done_phys_increase(kctx, commit_pages); ++ ++ mutex_lock(&kctx->jit_evict_lock); ++ WARN_ON(!list_empty(®->jit_node)); ++ list_add(®->jit_node, &kctx->jit_active_head); ++ mutex_unlock(&kctx->jit_evict_lock); ++ } ++#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ ++ ++ kbase_gpu_vm_unlock(kctx); ++ return reg; ++ ++no_mmap: ++no_cookie: ++#if MALI_USE_CSF ++ if (reg->flags & KBASE_REG_CSF_EVENT) { ++ kbase_gpu_vm_lock(kctx); ++ kbase_unlink_event_mem_page(kctx, reg); ++ kbase_gpu_vm_unlock(kctx); ++ } ++#endif ++no_kern_mapping: ++no_mem: ++#if MALI_JIT_PRESSURE_LIMIT_BASE ++ if (*flags & BASEP_MEM_PERFORM_JIT_TRIM) { ++ kbase_gpu_vm_lock(kctx); ++ kbase_jit_done_phys_increase(kctx, commit_pages); ++ kbase_gpu_vm_unlock(kctx); ++ } ++#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ ++ kbase_mem_phy_alloc_put(reg->cpu_alloc); ++ kbase_mem_phy_alloc_put(reg->gpu_alloc); ++invalid_flags: ++prepare_failed: ++ kfree(reg); ++no_region: ++bad_sizes: ++bad_flags: ++ return NULL; ++} ++KBASE_EXPORT_TEST_API(kbase_mem_alloc); ++ ++int kbase_mem_query(struct kbase_context *kctx, ++ u64 gpu_addr, u64 query, u64 * const out) ++{ ++ struct kbase_va_region *reg; ++ int ret = -EINVAL; ++ ++ KBASE_DEBUG_ASSERT(kctx); ++ KBASE_DEBUG_ASSERT(out); ++ ++ if (gpu_addr & ~PAGE_MASK) { ++ dev_warn(kctx->kbdev->dev, "mem_query: gpu_addr: passed parameter is invalid"); ++ return -EINVAL; ++ } ++ ++ kbase_gpu_vm_lock(kctx); ++ ++ /* Validate the region */ ++ reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr); ++ if (kbase_is_region_invalid_or_free(reg)) ++ goto out_unlock; ++ ++ switch (query) { ++ case KBASE_MEM_QUERY_COMMIT_SIZE: ++ if (reg->cpu_alloc->type != KBASE_MEM_TYPE_ALIAS) { ++ *out = kbase_reg_current_backed_size(reg); ++ } else { ++ size_t i; ++ struct kbase_aliased *aliased; ++ *out = 0; ++ aliased = reg->cpu_alloc->imported.alias.aliased; ++ for (i = 0; i < reg->cpu_alloc->imported.alias.nents; i++) ++ *out += aliased[i].length; ++ } ++ break; ++ case KBASE_MEM_QUERY_VA_SIZE: ++ *out = reg->nr_pages; ++ break; ++ case KBASE_MEM_QUERY_FLAGS: ++ { ++ *out = 0; ++ if (KBASE_REG_CPU_WR & reg->flags) ++ *out |= BASE_MEM_PROT_CPU_WR; ++ if (KBASE_REG_CPU_RD & reg->flags) ++ *out |= BASE_MEM_PROT_CPU_RD; ++ if (KBASE_REG_CPU_CACHED & reg->flags) ++ *out |= BASE_MEM_CACHED_CPU; ++ if (KBASE_REG_GPU_WR & reg->flags) ++ *out |= BASE_MEM_PROT_GPU_WR; ++ if (KBASE_REG_GPU_RD & reg->flags) ++ *out |= BASE_MEM_PROT_GPU_RD; ++ if (!(KBASE_REG_GPU_NX & reg->flags)) ++ *out |= BASE_MEM_PROT_GPU_EX; ++ if (KBASE_REG_SHARE_BOTH & reg->flags) ++ *out |= BASE_MEM_COHERENT_SYSTEM; ++ if (KBASE_REG_SHARE_IN & reg->flags) ++ *out |= BASE_MEM_COHERENT_LOCAL; ++ if (mali_kbase_supports_mem_grow_on_gpf(kctx->api_version)) { ++ /* Prior to this version, this was known about by ++ * user-side but we did not return them. Returning ++ * it caused certain clients that were not expecting ++ * it to fail, so we omit it as a special-case for ++ * compatibility reasons ++ */ ++ if (KBASE_REG_PF_GROW & reg->flags) ++ *out |= BASE_MEM_GROW_ON_GPF; ++ } ++ if (mali_kbase_supports_mem_protected(kctx->api_version)) { ++ /* Prior to this version, this was known about by ++ * user-side but we did not return them. Returning ++ * it caused certain clients that were not expecting ++ * it to fail, so we omit it as a special-case for ++ * compatibility reasons ++ */ ++ if (KBASE_REG_PROTECTED & reg->flags) ++ *out |= BASE_MEM_PROTECTED; ++ } ++#if !MALI_USE_CSF ++ if (KBASE_REG_TILER_ALIGN_TOP & reg->flags) ++ *out |= BASE_MEM_TILER_ALIGN_TOP; ++#endif /* !MALI_USE_CSF */ ++ if (!(KBASE_REG_GPU_CACHED & reg->flags)) ++ *out |= BASE_MEM_UNCACHED_GPU; ++#if MALI_USE_CSF ++ if (KBASE_REG_CSF_EVENT & reg->flags) ++ *out |= BASE_MEM_CSF_EVENT; ++#endif ++ if (KBASE_REG_GPU_VA_SAME_4GB_PAGE & reg->flags) ++ *out |= BASE_MEM_GPU_VA_SAME_4GB_PAGE; ++ ++ *out |= base_mem_group_id_set(reg->cpu_alloc->group_id); ++ ++ WARN(*out & ~BASE_MEM_FLAGS_QUERYABLE, ++ "BASE_MEM_FLAGS_QUERYABLE needs updating\n"); ++ *out &= BASE_MEM_FLAGS_QUERYABLE; ++ break; ++ } ++ default: ++ *out = 0; ++ goto out_unlock; ++ } ++ ++ ret = 0; ++ ++out_unlock: ++ kbase_gpu_vm_unlock(kctx); ++ return ret; ++} ++ ++/** ++ * kbase_mem_evictable_reclaim_count_objects - Count number of pages in the ++ * Ephemeral memory eviction list. ++ * @s: Shrinker ++ * @sc: Shrinker control ++ * ++ * Return: Number of pages which can be freed. ++ */ ++static ++unsigned long kbase_mem_evictable_reclaim_count_objects(struct shrinker *s, ++ struct shrink_control *sc) ++{ ++ struct kbase_context *kctx; ++ struct kbase_mem_phy_alloc *alloc; ++ unsigned long pages = 0; ++ ++ kctx = container_of(s, struct kbase_context, reclaim); ++ ++ mutex_lock(&kctx->jit_evict_lock); ++ ++ list_for_each_entry(alloc, &kctx->evict_list, evict_node) ++ pages += alloc->nents; ++ ++ mutex_unlock(&kctx->jit_evict_lock); ++ return pages; ++} ++ ++/** ++ * kbase_mem_evictable_reclaim_scan_objects - Scan the Ephemeral memory eviction ++ * list for pages and try to reclaim them. ++ * @s: Shrinker ++ * @sc: Shrinker control ++ * ++ * Return: Number of pages freed (can be less then requested) or -1 if the ++ * shrinker failed to free pages in its pool. ++ * ++ * Note: ++ * This function accesses region structures without taking the region lock, ++ * this is required as the OOM killer can call the shrinker after the region ++ * lock has already been held. ++ * This is safe as we can guarantee that a region on the eviction list will ++ * not be freed (kbase_mem_free_region removes the allocation from the list ++ * before destroying it), or modified by other parts of the driver. ++ * The eviction list itself is guarded by the eviction lock and the MMU updates ++ * are protected by their own lock. ++ */ ++static ++unsigned long kbase_mem_evictable_reclaim_scan_objects(struct shrinker *s, ++ struct shrink_control *sc) ++{ ++ struct kbase_context *kctx; ++ struct kbase_mem_phy_alloc *alloc; ++ struct kbase_mem_phy_alloc *tmp; ++ unsigned long freed = 0; ++ ++ kctx = container_of(s, struct kbase_context, reclaim); ++ mutex_lock(&kctx->jit_evict_lock); ++ ++ list_for_each_entry_safe(alloc, tmp, &kctx->evict_list, evict_node) { ++ int err; ++ ++ err = kbase_mem_shrink_gpu_mapping(kctx, alloc->reg, ++ 0, alloc->nents); ++ if (err != 0) { ++ /* ++ * Failed to remove GPU mapping, tell the shrinker ++ * to stop trying to shrink our slab even though we ++ * have pages in it. ++ */ ++ freed = -1; ++ goto out_unlock; ++ } ++ ++ /* ++ * Update alloc->evicted before freeing the backing so the ++ * helper can determine that it needs to bypass the accounting ++ * and memory pool. ++ */ ++ alloc->evicted = alloc->nents; ++ ++ kbase_free_phy_pages_helper(alloc, alloc->evicted); ++ freed += alloc->evicted; ++ list_del_init(&alloc->evict_node); ++ ++ /* ++ * Inform the JIT allocator this region has lost backing ++ * as it might need to free the allocation. ++ */ ++ kbase_jit_backing_lost(alloc->reg); ++ ++ /* Enough pages have been freed so stop now */ ++ if (freed > sc->nr_to_scan) ++ break; ++ } ++out_unlock: ++ mutex_unlock(&kctx->jit_evict_lock); ++ ++ return freed; ++} ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) ++static int kbase_mem_evictable_reclaim_shrink(struct shrinker *s, ++ struct shrink_control *sc) ++{ ++ if (sc->nr_to_scan == 0) ++ return kbase_mem_evictable_reclaim_count_objects(s, sc); ++ ++ return kbase_mem_evictable_reclaim_scan_objects(s, sc); ++} ++#endif ++ ++int kbase_mem_evictable_init(struct kbase_context *kctx) ++{ ++ INIT_LIST_HEAD(&kctx->evict_list); ++ mutex_init(&kctx->jit_evict_lock); ++ ++ /* Register shrinker */ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) ++ kctx->reclaim.shrink = kbase_mem_evictable_reclaim_shrink; ++#else ++ kctx->reclaim.count_objects = kbase_mem_evictable_reclaim_count_objects; ++ kctx->reclaim.scan_objects = kbase_mem_evictable_reclaim_scan_objects; ++#endif ++ kctx->reclaim.seeks = DEFAULT_SEEKS; ++ /* Kernel versions prior to 3.1 : ++ * struct shrinker does not define batch */ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 1, 0) ++ kctx->reclaim.batch = 0; ++#endif ++ register_shrinker(&kctx->reclaim); ++ return 0; ++} ++ ++void kbase_mem_evictable_deinit(struct kbase_context *kctx) ++{ ++ unregister_shrinker(&kctx->reclaim); ++} ++ ++/** ++ * kbase_mem_evictable_mark_reclaim - Mark the pages as reclaimable. ++ * @alloc: The physical allocation ++ */ ++void kbase_mem_evictable_mark_reclaim(struct kbase_mem_phy_alloc *alloc) ++{ ++ struct kbase_context *kctx = alloc->imported.native.kctx; ++ struct kbase_device *kbdev = kctx->kbdev; ++ int __maybe_unused new_page_count; ++ ++ kbase_process_page_usage_dec(kctx, alloc->nents); ++ new_page_count = atomic_sub_return(alloc->nents, ++ &kctx->used_pages); ++ atomic_sub(alloc->nents, &kctx->kbdev->memdev.used_pages); ++ ++ KBASE_TLSTREAM_AUX_PAGESALLOC( ++ kbdev, ++ kctx->id, ++ (u64)new_page_count); ++ kbase_trace_gpu_mem_usage_dec(kbdev, kctx, alloc->nents); ++} ++ ++/** ++ * kbase_mem_evictable_unmark_reclaim - Mark the pages as no longer reclaimable. ++ * @alloc: The physical allocation ++ */ ++static ++void kbase_mem_evictable_unmark_reclaim(struct kbase_mem_phy_alloc *alloc) ++{ ++ struct kbase_context *kctx = alloc->imported.native.kctx; ++ struct kbase_device *kbdev = kctx->kbdev; ++ int __maybe_unused new_page_count; ++ ++ new_page_count = atomic_add_return(alloc->nents, ++ &kctx->used_pages); ++ atomic_add(alloc->nents, &kctx->kbdev->memdev.used_pages); ++ ++ /* Increase mm counters so that the allocation is accounted for ++ * against the process and thus is visible to the OOM killer, ++ */ ++ kbase_process_page_usage_inc(kctx, alloc->nents); ++ ++ KBASE_TLSTREAM_AUX_PAGESALLOC( ++ kbdev, ++ kctx->id, ++ (u64)new_page_count); ++ kbase_trace_gpu_mem_usage_inc(kbdev, kctx, alloc->nents); ++} ++ ++int kbase_mem_evictable_make(struct kbase_mem_phy_alloc *gpu_alloc) ++{ ++ struct kbase_context *kctx = gpu_alloc->imported.native.kctx; ++ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ kbase_mem_shrink_cpu_mapping(kctx, gpu_alloc->reg, ++ 0, gpu_alloc->nents); ++ ++ mutex_lock(&kctx->jit_evict_lock); ++ /* This allocation can't already be on a list. */ ++ WARN_ON(!list_empty(&gpu_alloc->evict_node)); ++ ++ /* ++ * Add the allocation to the eviction list, after this point the shrink ++ * can reclaim it. ++ */ ++ list_add(&gpu_alloc->evict_node, &kctx->evict_list); ++ mutex_unlock(&kctx->jit_evict_lock); ++ kbase_mem_evictable_mark_reclaim(gpu_alloc); ++ ++ gpu_alloc->reg->flags |= KBASE_REG_DONT_NEED; ++ return 0; ++} ++ ++bool kbase_mem_evictable_unmake(struct kbase_mem_phy_alloc *gpu_alloc) ++{ ++ struct kbase_context *kctx = gpu_alloc->imported.native.kctx; ++ int err = 0; ++ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ mutex_lock(&kctx->jit_evict_lock); ++ /* ++ * First remove the allocation from the eviction list as it's no ++ * longer eligible for eviction. ++ */ ++ list_del_init(&gpu_alloc->evict_node); ++ mutex_unlock(&kctx->jit_evict_lock); ++ ++ if (gpu_alloc->evicted == 0) { ++ /* ++ * The backing is still present, update the VM stats as it's ++ * in use again. ++ */ ++ kbase_mem_evictable_unmark_reclaim(gpu_alloc); ++ } else { ++ /* If the region is still alive ... */ ++ if (gpu_alloc->reg) { ++ /* ... allocate replacement backing ... */ ++ err = kbase_alloc_phy_pages_helper(gpu_alloc, ++ gpu_alloc->evicted); ++ ++ /* ++ * ... and grow the mapping back to its ++ * pre-eviction size. ++ */ ++ if (!err) ++ err = kbase_mem_grow_gpu_mapping(kctx, ++ gpu_alloc->reg, ++ gpu_alloc->evicted, 0); ++ ++ gpu_alloc->evicted = 0; ++ } ++ } ++ ++ /* If the region is still alive remove the DONT_NEED attribute. */ ++ if (gpu_alloc->reg) ++ gpu_alloc->reg->flags &= ~KBASE_REG_DONT_NEED; ++ ++ return (err == 0); ++} ++ ++int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, unsigned int flags, unsigned int mask) ++{ ++ struct kbase_va_region *reg; ++ int ret = -EINVAL; ++ unsigned int real_flags = 0; ++ unsigned int new_flags = 0; ++ bool prev_needed, new_needed; ++ ++ KBASE_DEBUG_ASSERT(kctx); ++ ++ if (!gpu_addr) ++ return -EINVAL; ++ ++ if ((gpu_addr & ~PAGE_MASK) && (gpu_addr >= PAGE_SIZE)) ++ return -EINVAL; ++ ++ /* nuke other bits */ ++ flags &= mask; ++ ++ /* check for only supported flags */ ++ if (flags & ~(BASE_MEM_FLAGS_MODIFIABLE)) ++ goto out; ++ ++ /* mask covers bits we don't support? */ ++ if (mask & ~(BASE_MEM_FLAGS_MODIFIABLE)) ++ goto out; ++ ++ /* convert flags */ ++ if (BASE_MEM_COHERENT_SYSTEM & flags) ++ real_flags |= KBASE_REG_SHARE_BOTH; ++ else if (BASE_MEM_COHERENT_LOCAL & flags) ++ real_flags |= KBASE_REG_SHARE_IN; ++ ++ /* now we can lock down the context, and find the region */ ++ down_write(kbase_mem_get_process_mmap_lock()); ++ kbase_gpu_vm_lock(kctx); ++ ++ /* Validate the region */ ++ reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr); ++ if (kbase_is_region_invalid_or_free(reg)) ++ goto out_unlock; ++ ++ /* Is the region being transitioning between not needed and needed? */ ++ prev_needed = (KBASE_REG_DONT_NEED & reg->flags) == KBASE_REG_DONT_NEED; ++ new_needed = (BASE_MEM_DONT_NEED & flags) == BASE_MEM_DONT_NEED; ++ if (prev_needed != new_needed) { ++ /* Aliased allocations can't be made ephemeral */ ++ if (atomic_read(®->cpu_alloc->gpu_mappings) > 1) ++ goto out_unlock; ++ ++ if (new_needed) { ++ /* Only native allocations can be marked not needed */ ++ if (reg->cpu_alloc->type != KBASE_MEM_TYPE_NATIVE) { ++ ret = -EINVAL; ++ goto out_unlock; ++ } ++ ret = kbase_mem_evictable_make(reg->gpu_alloc); ++ if (ret) ++ goto out_unlock; ++ } else { ++ kbase_mem_evictable_unmake(reg->gpu_alloc); ++ } ++ } ++ ++ /* limit to imported memory */ ++ if (reg->gpu_alloc->type != KBASE_MEM_TYPE_IMPORTED_UMM) ++ goto out_unlock; ++ ++ /* shareability flags are ignored for GPU uncached memory */ ++ if (!(reg->flags & KBASE_REG_GPU_CACHED)) { ++ ret = 0; ++ goto out_unlock; ++ } ++ ++ /* no change? */ ++ if (real_flags == (reg->flags & (KBASE_REG_SHARE_IN | KBASE_REG_SHARE_BOTH))) { ++ ret = 0; ++ goto out_unlock; ++ } ++ ++ new_flags = reg->flags & ~(KBASE_REG_SHARE_IN | KBASE_REG_SHARE_BOTH); ++ new_flags |= real_flags; ++ ++ /* Currently supporting only imported memory */ ++ if (reg->gpu_alloc->type != KBASE_MEM_TYPE_IMPORTED_UMM) { ++ ret = -EINVAL; ++ goto out_unlock; ++ } ++ ++ if (IS_ENABLED(CONFIG_MALI_DMA_BUF_MAP_ON_DEMAND)) { ++ /* Future use will use the new flags, existing mapping ++ * will NOT be updated as memory should not be in use ++ * by the GPU when updating the flags. ++ */ ++ WARN_ON(reg->gpu_alloc->imported.umm.current_mapping_usage_count); ++ ret = 0; ++ } else if (reg->gpu_alloc->imported.umm.current_mapping_usage_count) { ++ /* ++ * When CONFIG_MALI_DMA_BUF_MAP_ON_DEMAND is not enabled the ++ * dma-buf GPU mapping should always be present, check that ++ * this is the case and warn and skip the page table update if ++ * not. ++ * ++ * Then update dma-buf GPU mapping with the new flags. ++ * ++ * Note: The buffer must not be in use on the GPU when ++ * changing flags. If the buffer is in active use on ++ * the GPU, there is a risk that the GPU may trigger a ++ * shareability fault, as it will see the same ++ * addresses from buffer with different shareability ++ * properties. ++ */ ++ dev_dbg(kctx->kbdev->dev, ++ "Updating page tables on mem flag change\n"); ++ ret = kbase_mmu_update_pages(kctx, reg->start_pfn, ++ kbase_get_gpu_phy_pages(reg), ++ kbase_reg_current_backed_size(reg), ++ new_flags, ++ reg->gpu_alloc->group_id); ++ if (ret) ++ dev_warn(kctx->kbdev->dev, ++ "Failed to update GPU page tables on flag change: %d\n", ++ ret); ++ } else ++ WARN_ON(!reg->gpu_alloc->imported.umm.current_mapping_usage_count); ++ ++ /* If everything is good, then set the new flags on the region. */ ++ if (!ret) ++ reg->flags = new_flags; ++ ++out_unlock: ++ kbase_gpu_vm_unlock(kctx); ++ up_write(kbase_mem_get_process_mmap_lock()); ++out: ++ return ret; ++} ++ ++#define KBASE_MEM_IMPORT_HAVE_PAGES (1UL << BASE_MEM_FLAGS_NR_BITS) ++ ++int kbase_mem_do_sync_imported(struct kbase_context *kctx, ++ struct kbase_va_region *reg, enum kbase_sync_type sync_fn) ++{ ++ int ret = -EINVAL; ++ struct dma_buf *dma_buf; ++ enum dma_data_direction dir = DMA_BIDIRECTIONAL; ++ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ /* We assume that the same physical allocation object is used for both ++ * GPU and CPU for imported buffers. ++ */ ++ WARN_ON(reg->cpu_alloc != reg->gpu_alloc); ++ ++ /* Currently only handle dma-bufs */ ++ if (reg->gpu_alloc->type != KBASE_MEM_TYPE_IMPORTED_UMM) ++ return ret; ++ /* ++ * Attempting to sync with CONFIG_MALI_DMA_BUF_MAP_ON_DEMAND ++ * enabled can expose us to a Linux Kernel issue between v4.6 and ++ * v4.19. We will not attempt to support cache syncs on dma-bufs that ++ * are mapped on demand (i.e. not on import), even on pre-4.6, neither ++ * on 4.20 or newer kernels, because this makes it difficult for ++ * userspace to know when they can rely on the cache sync. ++ * Instead, only support syncing when we always map dma-bufs on import, ++ * or if the particular buffer is mapped right now. ++ */ ++ if (IS_ENABLED(CONFIG_MALI_DMA_BUF_MAP_ON_DEMAND) && ++ !reg->gpu_alloc->imported.umm.current_mapping_usage_count) ++ return ret; ++ ++ dma_buf = reg->gpu_alloc->imported.umm.dma_buf; ++ ++ switch (sync_fn) { ++ case KBASE_SYNC_TO_DEVICE: ++ dev_dbg(kctx->kbdev->dev, ++ "Syncing imported buffer at GPU VA %llx to GPU\n", ++ reg->start_pfn); ++#ifdef KBASE_MEM_ION_SYNC_WORKAROUND ++ if (!WARN_ON(!reg->gpu_alloc->imported.umm.dma_attachment)) { ++ struct dma_buf_attachment *attachment = reg->gpu_alloc->imported.umm.dma_attachment; ++ struct sg_table *sgt = reg->gpu_alloc->imported.umm.sgt; ++ ++ dma_sync_sg_for_device(attachment->dev, sgt->sgl, ++ sgt->nents, dir); ++ ret = 0; ++ } ++#else ++ /* Though the below version check could be superfluous depending upon the version condition ++ * used for enabling KBASE_MEM_ION_SYNC_WORKAROUND, we still keep this check here to allow ++ * ease of modification for non-ION systems or systems where ION has been patched. ++ */ ++#if KERNEL_VERSION(4, 6, 0) > LINUX_VERSION_CODE && !defined(CONFIG_CHROMEOS) ++ dma_buf_end_cpu_access(dma_buf, ++ 0, dma_buf->size, ++ dir); ++ ret = 0; ++#else ++ ret = dma_buf_end_cpu_access(dma_buf, ++ dir); ++#endif ++#endif /* KBASE_MEM_ION_SYNC_WORKAROUND */ ++ break; ++ case KBASE_SYNC_TO_CPU: ++ dev_dbg(kctx->kbdev->dev, ++ "Syncing imported buffer at GPU VA %llx to CPU\n", ++ reg->start_pfn); ++#ifdef KBASE_MEM_ION_SYNC_WORKAROUND ++ if (!WARN_ON(!reg->gpu_alloc->imported.umm.dma_attachment)) { ++ struct dma_buf_attachment *attachment = reg->gpu_alloc->imported.umm.dma_attachment; ++ struct sg_table *sgt = reg->gpu_alloc->imported.umm.sgt; ++ ++ dma_sync_sg_for_cpu(attachment->dev, sgt->sgl, ++ sgt->nents, dir); ++ ret = 0; ++ } ++#else ++ ret = dma_buf_begin_cpu_access(dma_buf, ++#if KERNEL_VERSION(4, 6, 0) > LINUX_VERSION_CODE && !defined(CONFIG_CHROMEOS) ++ 0, dma_buf->size, ++#endif ++ dir); ++#endif /* KBASE_MEM_ION_SYNC_WORKAROUND */ ++ break; ++ }; ++ ++ if (unlikely(ret)) ++ dev_warn(kctx->kbdev->dev, ++ "Failed to sync mem region %pK at GPU VA %llx: %d\n", ++ reg, reg->start_pfn, ret); ++ ++ return ret; ++} ++ ++/** ++ * kbase_mem_umm_unmap_attachment - Unmap dma-buf attachment ++ * @kctx: Pointer to kbase context ++ * @alloc: Pointer to allocation with imported dma-buf memory to unmap ++ * ++ * This will unmap a dma-buf. Must be called after the GPU page tables for the ++ * region have been torn down. ++ */ ++static void kbase_mem_umm_unmap_attachment(struct kbase_context *kctx, ++ struct kbase_mem_phy_alloc *alloc) ++{ ++ struct tagged_addr *pa = alloc->pages; ++ ++ dma_buf_unmap_attachment(alloc->imported.umm.dma_attachment, ++ alloc->imported.umm.sgt, DMA_BIDIRECTIONAL); ++ alloc->imported.umm.sgt = NULL; ++ ++ kbase_remove_dma_buf_usage(kctx, alloc); ++ ++ memset(pa, 0xff, sizeof(*pa) * alloc->nents); ++ alloc->nents = 0; ++} ++ ++/* to replace sg_dma_len. */ ++#define MALI_SG_DMA_LEN(sg) ((sg)->length) ++ ++/** ++ * kbase_mem_umm_map_attachment - Prepare attached dma-buf for GPU mapping ++ * @kctx: Pointer to kbase context ++ * @reg: Pointer to region with imported dma-buf memory to map ++ * ++ * Map the dma-buf and prepare the page array with the tagged Mali physical ++ * addresses for GPU mapping. ++ * ++ * Return: 0 on success, or negative error code ++ */ ++static int kbase_mem_umm_map_attachment(struct kbase_context *kctx, ++ struct kbase_va_region *reg) ++{ ++ struct sg_table *sgt; ++ struct scatterlist *s; ++ int i; ++ struct tagged_addr *pa; ++ int err; ++ size_t count = 0; ++ struct kbase_mem_phy_alloc *alloc = reg->gpu_alloc; ++ ++ WARN_ON_ONCE(alloc->type != KBASE_MEM_TYPE_IMPORTED_UMM); ++ WARN_ON_ONCE(alloc->imported.umm.sgt); ++ ++ sgt = dma_buf_map_attachment(alloc->imported.umm.dma_attachment, ++ DMA_BIDIRECTIONAL); ++ if (IS_ERR_OR_NULL(sgt)) ++ return -EINVAL; ++ ++ /* save for later */ ++ alloc->imported.umm.sgt = sgt; ++ ++ pa = kbase_get_gpu_phy_pages(reg); ++ ++ for_each_sg(sgt->sgl, s, sgt->nents, i) { ++ size_t j, pages = PFN_UP(MALI_SG_DMA_LEN(s)); ++ ++ WARN_ONCE(MALI_SG_DMA_LEN(s) & (PAGE_SIZE-1), ++ "MALI_SG_DMA_LEN(s)=%u is not a multiple of PAGE_SIZE\n", ++ MALI_SG_DMA_LEN(s)); ++ ++ WARN_ONCE(sg_dma_address(s) & (PAGE_SIZE-1), ++ "sg_dma_address(s)=%llx is not aligned to PAGE_SIZE\n", ++ (unsigned long long) sg_dma_address(s)); ++ ++ for (j = 0; (j < pages) && (count < reg->nr_pages); j++, count++) ++ *pa++ = as_tagged(sg_dma_address(s) + ++ (j << PAGE_SHIFT)); ++ WARN_ONCE(j < pages, ++ "sg list from dma_buf_map_attachment > dma_buf->size=%zu\n", ++ alloc->imported.umm.dma_buf->size); ++ } ++ ++ if (!(reg->flags & KBASE_REG_IMPORT_PAD) && ++ WARN_ONCE(count < reg->nr_pages, ++ "sg list from dma_buf_map_attachment < dma_buf->size=%zu\n", ++ alloc->imported.umm.dma_buf->size)) { ++ err = -EINVAL; ++ goto err_unmap_attachment; ++ } ++ ++ /* Update nents as we now have pages to map */ ++ alloc->nents = count; ++ kbase_add_dma_buf_usage(kctx, alloc); ++ ++ return 0; ++ ++err_unmap_attachment: ++ kbase_mem_umm_unmap_attachment(kctx, alloc); ++ ++ return err; ++} ++ ++int kbase_mem_umm_map(struct kbase_context *kctx, ++ struct kbase_va_region *reg) ++{ ++ int err; ++ struct kbase_mem_phy_alloc *alloc; ++ unsigned long gwt_mask = ~0; ++ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ alloc = reg->gpu_alloc; ++ ++ alloc->imported.umm.current_mapping_usage_count++; ++ if (alloc->imported.umm.current_mapping_usage_count != 1) { ++ if (IS_ENABLED(CONFIG_MALI_DMA_BUF_LEGACY_COMPAT) || ++ alloc->imported.umm.need_sync) { ++ if (!kbase_is_region_invalid_or_free(reg)) { ++ err = kbase_mem_do_sync_imported(kctx, reg, ++ KBASE_SYNC_TO_DEVICE); ++ WARN_ON_ONCE(err); ++ } ++ } ++ return 0; ++ } ++ ++ err = kbase_mem_umm_map_attachment(kctx, reg); ++ if (err) ++ goto bad_map_attachment; ++ ++#ifdef CONFIG_MALI_CINSTR_GWT ++ if (kctx->gwt_enabled) ++ gwt_mask = ~KBASE_REG_GPU_WR; ++#endif ++ ++ err = kbase_mmu_insert_pages(kctx->kbdev, ++ &kctx->mmu, ++ reg->start_pfn, ++ kbase_get_gpu_phy_pages(reg), ++ kbase_reg_current_backed_size(reg), ++ reg->flags & gwt_mask, ++ kctx->as_nr, ++ alloc->group_id); ++ if (err) ++ goto bad_insert; ++ ++ if (reg->flags & KBASE_REG_IMPORT_PAD && ++ !WARN_ON(reg->nr_pages < alloc->nents)) { ++ /* For padded imported dma-buf memory, map the dummy aliasing ++ * page from the end of the dma-buf pages, to the end of the ++ * region using a read only mapping. ++ * ++ * Assume alloc->nents is the number of actual pages in the ++ * dma-buf memory. ++ */ ++ err = kbase_mmu_insert_single_page(kctx, ++ reg->start_pfn + alloc->nents, ++ kctx->aliasing_sink_page, ++ reg->nr_pages - alloc->nents, ++ (reg->flags | KBASE_REG_GPU_RD) & ++ ~KBASE_REG_GPU_WR, ++ KBASE_MEM_GROUP_SINK); ++ if (err) ++ goto bad_pad_insert; ++ } ++ ++ return 0; ++ ++bad_pad_insert: ++ kbase_mmu_teardown_pages(kctx->kbdev, ++ &kctx->mmu, ++ reg->start_pfn, ++ alloc->nents, ++ kctx->as_nr); ++bad_insert: ++ kbase_mem_umm_unmap_attachment(kctx, alloc); ++bad_map_attachment: ++ alloc->imported.umm.current_mapping_usage_count--; ++ ++ return err; ++} ++ ++void kbase_mem_umm_unmap(struct kbase_context *kctx, ++ struct kbase_va_region *reg, struct kbase_mem_phy_alloc *alloc) ++{ ++ alloc->imported.umm.current_mapping_usage_count--; ++ if (alloc->imported.umm.current_mapping_usage_count) { ++ if (IS_ENABLED(CONFIG_MALI_DMA_BUF_LEGACY_COMPAT) || ++ alloc->imported.umm.need_sync) { ++ if (!kbase_is_region_invalid_or_free(reg)) { ++ int err = kbase_mem_do_sync_imported(kctx, reg, ++ KBASE_SYNC_TO_CPU); ++ WARN_ON_ONCE(err); ++ } ++ } ++ return; ++ } ++ ++ if (!kbase_is_region_invalid_or_free(reg) && reg->gpu_alloc == alloc) { ++ int err; ++ ++ err = kbase_mmu_teardown_pages(kctx->kbdev, ++ &kctx->mmu, ++ reg->start_pfn, ++ reg->nr_pages, ++ kctx->as_nr); ++ WARN_ON(err); ++ } ++ ++ kbase_mem_umm_unmap_attachment(kctx, alloc); ++} ++ ++static int get_umm_memory_group_id(struct kbase_context *kctx, ++ struct dma_buf *dma_buf) ++{ ++ int group_id = BASE_MEM_GROUP_DEFAULT; ++ ++ if (kctx->kbdev->mgm_dev->ops.mgm_get_import_memory_id) { ++ struct memory_group_manager_import_data mgm_import_data; ++ ++ mgm_import_data.type = ++ MEMORY_GROUP_MANAGER_IMPORT_TYPE_DMA_BUF; ++ mgm_import_data.u.dma_buf = dma_buf; ++ ++ group_id = kctx->kbdev->mgm_dev->ops.mgm_get_import_memory_id( ++ kctx->kbdev->mgm_dev, &mgm_import_data); ++ } ++ ++ return group_id; ++} ++ ++/** ++ * kbase_mem_from_umm - Import dma-buf memory into kctx ++ * @kctx: Pointer to kbase context to import memory into ++ * @fd: File descriptor of dma-buf to import ++ * @va_pages: Pointer where virtual size of the region will be output ++ * @flags: Pointer to memory flags ++ * @padding: Number of read only padding pages to be inserted at the end of the ++ * GPU mapping of the dma-buf ++ * ++ * Return: Pointer to new kbase_va_region object of the imported dma-buf, or ++ * NULL on error. ++ * ++ * This function imports a dma-buf into kctx, and created a kbase_va_region ++ * object that wraps the dma-buf. ++ */ ++static struct kbase_va_region *kbase_mem_from_umm(struct kbase_context *kctx, ++ int fd, u64 *va_pages, u64 *flags, u32 padding) ++{ ++ struct kbase_va_region *reg; ++ struct dma_buf *dma_buf; ++ struct dma_buf_attachment *dma_attachment; ++ bool shared_zone = false; ++ bool need_sync = false; ++ int group_id; ++ ++ /* 64-bit address range is the max */ ++ if (*va_pages > (U64_MAX / PAGE_SIZE)) ++ return NULL; ++ ++ dma_buf = dma_buf_get(fd); ++ if (IS_ERR_OR_NULL(dma_buf)) ++ return NULL; ++ ++ dma_attachment = dma_buf_attach(dma_buf, kctx->kbdev->dev); ++ if (IS_ERR_OR_NULL(dma_attachment)) { ++ dma_buf_put(dma_buf); ++ return NULL; ++ } ++ ++ *va_pages = (PAGE_ALIGN(dma_buf->size) >> PAGE_SHIFT) + padding; ++ if (!*va_pages) { ++ dma_buf_detach(dma_buf, dma_attachment); ++ dma_buf_put(dma_buf); ++ return NULL; ++ } ++ ++ /* ignore SAME_VA */ ++ *flags &= ~BASE_MEM_SAME_VA; ++ ++ /* ++ * Force CPU cached flag. ++ * ++ * We can't query the dma-buf exporter to get details about the CPU ++ * cache attributes of CPU mappings, so we have to assume that the ++ * buffer may be cached, and call into the exporter for cache ++ * maintenance, and rely on the exporter to do the right thing when ++ * handling our calls. ++ */ ++ *flags |= BASE_MEM_CACHED_CPU; ++ ++ if (*flags & BASE_MEM_IMPORT_SHARED) ++ shared_zone = true; ++ ++ if (*flags & BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP) ++ need_sync = true; ++ ++#ifdef CONFIG_64BIT ++ if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { ++ /* ++ * 64-bit tasks require us to reserve VA on the CPU that we use ++ * on the GPU. ++ */ ++ shared_zone = true; ++ } ++#endif ++ ++ if (shared_zone) { ++ *flags |= BASE_MEM_NEED_MMAP; ++ reg = kbase_alloc_free_region(&kctx->reg_rbtree_same, ++ 0, *va_pages, KBASE_REG_ZONE_SAME_VA); ++ } else { ++ reg = kbase_alloc_free_region(&kctx->reg_rbtree_custom, ++ 0, *va_pages, KBASE_REG_ZONE_CUSTOM_VA); ++ } ++ ++ if (!reg) { ++ dma_buf_detach(dma_buf, dma_attachment); ++ dma_buf_put(dma_buf); ++ return NULL; ++ } ++ ++ group_id = get_umm_memory_group_id(kctx, dma_buf); ++ ++ reg->gpu_alloc = kbase_alloc_create(kctx, *va_pages, ++ KBASE_MEM_TYPE_IMPORTED_UMM, group_id); ++ if (IS_ERR_OR_NULL(reg->gpu_alloc)) ++ goto no_alloc; ++ ++ reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); ++ ++ if (kbase_update_region_flags(kctx, reg, *flags) != 0) ++ goto error_out; ++ ++ /* No pages to map yet */ ++ reg->gpu_alloc->nents = 0; ++ ++ reg->flags &= ~KBASE_REG_FREE; ++ reg->flags |= KBASE_REG_GPU_NX; /* UMM is always No eXecute */ ++ reg->flags &= ~KBASE_REG_GROWABLE; /* UMM cannot be grown */ ++ ++ if (*flags & BASE_MEM_PROTECTED) ++ reg->flags |= KBASE_REG_PROTECTED; ++ ++ if (padding) ++ reg->flags |= KBASE_REG_IMPORT_PAD; ++ ++ reg->gpu_alloc->type = KBASE_MEM_TYPE_IMPORTED_UMM; ++ reg->gpu_alloc->imported.umm.sgt = NULL; ++ reg->gpu_alloc->imported.umm.dma_buf = dma_buf; ++ reg->gpu_alloc->imported.umm.dma_attachment = dma_attachment; ++ reg->gpu_alloc->imported.umm.current_mapping_usage_count = 0; ++ reg->gpu_alloc->imported.umm.need_sync = need_sync; ++ reg->gpu_alloc->imported.umm.kctx = kctx; ++ reg->extent = 0; ++ ++ if (!IS_ENABLED(CONFIG_MALI_DMA_BUF_MAP_ON_DEMAND)) { ++ int err; ++ ++ reg->gpu_alloc->imported.umm.current_mapping_usage_count = 1; ++ ++ err = kbase_mem_umm_map_attachment(kctx, reg); ++ if (err) { ++ dev_warn(kctx->kbdev->dev, ++ "Failed to map dma-buf %pK on GPU: %d\n", ++ dma_buf, err); ++ goto error_out; ++ } ++ ++ *flags |= KBASE_MEM_IMPORT_HAVE_PAGES; ++ } ++ ++ return reg; ++ ++error_out: ++ kbase_mem_phy_alloc_put(reg->gpu_alloc); ++ kbase_mem_phy_alloc_put(reg->cpu_alloc); ++no_alloc: ++ kfree(reg); ++ ++ return NULL; ++} ++ ++u32 kbase_get_cache_line_alignment(struct kbase_device *kbdev) ++{ ++ u32 cpu_cache_line_size = cache_line_size(); ++ u32 gpu_cache_line_size = ++ (1UL << kbdev->gpu_props.props.l2_props.log2_line_size); ++ ++ return ((cpu_cache_line_size > gpu_cache_line_size) ? ++ cpu_cache_line_size : ++ gpu_cache_line_size); ++} ++ ++static struct kbase_va_region *kbase_mem_from_user_buffer( ++ struct kbase_context *kctx, unsigned long address, ++ unsigned long size, u64 *va_pages, u64 *flags) ++{ ++ long i; ++ struct kbase_va_region *reg; ++ struct rb_root *rbtree; ++ long faulted_pages; ++ int zone = KBASE_REG_ZONE_CUSTOM_VA; ++ bool shared_zone = false; ++ u32 cache_line_alignment = kbase_get_cache_line_alignment(kctx->kbdev); ++ struct kbase_alloc_import_user_buf *user_buf; ++ struct page **pages = NULL; ++ ++ /* Flag supported only for dma-buf imported memory */ ++ if (*flags & BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP) ++ return NULL; ++ ++ if ((address & (cache_line_alignment - 1)) != 0 || ++ (size & (cache_line_alignment - 1)) != 0) { ++ if (*flags & BASE_MEM_UNCACHED_GPU) { ++ dev_warn(kctx->kbdev->dev, ++ "User buffer is not cache line aligned and marked as GPU uncached\n"); ++ goto bad_size; ++ } ++ ++ /* Coherency must be enabled to handle partial cache lines */ ++ if (*flags & (BASE_MEM_COHERENT_SYSTEM | ++ BASE_MEM_COHERENT_SYSTEM_REQUIRED)) { ++ /* Force coherent system required flag, import will ++ * then fail if coherency isn't available ++ */ ++ *flags |= BASE_MEM_COHERENT_SYSTEM_REQUIRED; ++ } else { ++ dev_warn(kctx->kbdev->dev, ++ "User buffer is not cache line aligned and no coherency enabled\n"); ++ goto bad_size; ++ } ++ } ++ ++ *va_pages = (PAGE_ALIGN(address + size) >> PAGE_SHIFT) - ++ PFN_DOWN(address); ++ if (!*va_pages) ++ goto bad_size; ++ ++ if (*va_pages > (UINT64_MAX / PAGE_SIZE)) ++ /* 64-bit address range is the max */ ++ goto bad_size; ++ ++ /* SAME_VA generally not supported with imported memory (no known use cases) */ ++ *flags &= ~BASE_MEM_SAME_VA; ++ ++ if (*flags & BASE_MEM_IMPORT_SHARED) ++ shared_zone = true; ++ ++#ifdef CONFIG_64BIT ++ if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { ++ /* ++ * 64-bit tasks require us to reserve VA on the CPU that we use ++ * on the GPU. ++ */ ++ shared_zone = true; ++ } ++#endif ++ ++ if (shared_zone) { ++ *flags |= BASE_MEM_NEED_MMAP; ++ zone = KBASE_REG_ZONE_SAME_VA; ++ rbtree = &kctx->reg_rbtree_same; ++ } else ++ rbtree = &kctx->reg_rbtree_custom; ++ ++ reg = kbase_alloc_free_region(rbtree, 0, *va_pages, zone); ++ ++ if (!reg) ++ goto no_region; ++ ++ reg->gpu_alloc = kbase_alloc_create( ++ kctx, *va_pages, KBASE_MEM_TYPE_IMPORTED_USER_BUF, ++ BASE_MEM_GROUP_DEFAULT); ++ if (IS_ERR_OR_NULL(reg->gpu_alloc)) ++ goto no_alloc_obj; ++ ++ reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); ++ ++ if (kbase_update_region_flags(kctx, reg, *flags) != 0) ++ goto invalid_flags; ++ ++ reg->flags &= ~KBASE_REG_FREE; ++ reg->flags |= KBASE_REG_GPU_NX; /* User-buffers are always No eXecute */ ++ reg->flags &= ~KBASE_REG_GROWABLE; /* Cannot be grown */ ++ ++ user_buf = ®->gpu_alloc->imported.user_buf; ++ ++ user_buf->size = size; ++ user_buf->address = address; ++ user_buf->nr_pages = *va_pages; ++ user_buf->mm = current->mm; ++#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE ++ atomic_inc(¤t->mm->mm_count); ++#else ++ mmgrab(current->mm); ++#endif ++ if (reg->gpu_alloc->properties & KBASE_MEM_PHY_ALLOC_LARGE) ++ user_buf->pages = vmalloc(*va_pages * sizeof(struct page *)); ++ else ++ user_buf->pages = kmalloc_array(*va_pages, ++ sizeof(struct page *), GFP_KERNEL); ++ ++ if (!user_buf->pages) ++ goto no_page_array; ++ ++ /* If the region is coherent with the CPU then the memory is imported ++ * and mapped onto the GPU immediately. ++ * Otherwise get_user_pages is called as a sanity check, but with ++ * NULL as the pages argument which will fault the pages, but not ++ * pin them. The memory will then be pinned only around the jobs that ++ * specify the region as an external resource. ++ */ ++ if (reg->flags & KBASE_REG_SHARE_BOTH) { ++ pages = user_buf->pages; ++ *flags |= KBASE_MEM_IMPORT_HAVE_PAGES; ++ } ++ ++ down_read(kbase_mem_get_process_mmap_lock()); ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) ++ faulted_pages = get_user_pages(current, current->mm, address, *va_pages, ++#if KERNEL_VERSION(4, 4, 168) <= LINUX_VERSION_CODE && \ ++KERNEL_VERSION(4, 5, 0) > LINUX_VERSION_CODE ++ reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0, ++ pages, NULL); ++#else ++ reg->flags & KBASE_REG_GPU_WR, 0, pages, NULL); ++#endif ++#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) ++ faulted_pages = get_user_pages(address, *va_pages, ++ reg->flags & KBASE_REG_GPU_WR, 0, pages, NULL); ++#else ++ faulted_pages = get_user_pages(address, *va_pages, ++ reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0, ++ pages, NULL); ++#endif ++ ++ up_read(kbase_mem_get_process_mmap_lock()); ++ ++ if (faulted_pages != *va_pages) ++ goto fault_mismatch; ++ ++ reg->gpu_alloc->nents = 0; ++ reg->extent = 0; ++ ++ if (pages) { ++ struct device *dev = kctx->kbdev->dev; ++ unsigned long local_size = user_buf->size; ++ unsigned long offset = user_buf->address & ~PAGE_MASK; ++ struct tagged_addr *pa = kbase_get_gpu_phy_pages(reg); ++ ++ /* Top bit signifies that this was pinned on import */ ++ user_buf->current_mapping_usage_count |= PINNED_ON_IMPORT; ++ ++ for (i = 0; i < faulted_pages; i++) { ++ dma_addr_t dma_addr; ++ unsigned long min; ++ ++ min = MIN(PAGE_SIZE - offset, local_size); ++ dma_addr = dma_map_page(dev, pages[i], ++ offset, min, ++ DMA_BIDIRECTIONAL); ++ if (dma_mapping_error(dev, dma_addr)) ++ goto unwind_dma_map; ++ ++ user_buf->dma_addrs[i] = dma_addr; ++ pa[i] = as_tagged(page_to_phys(pages[i])); ++ ++ local_size -= min; ++ offset = 0; ++ } ++ ++ reg->gpu_alloc->nents = faulted_pages; ++ } ++ ++ return reg; ++ ++unwind_dma_map: ++ while (i--) { ++ dma_unmap_page(kctx->kbdev->dev, ++ user_buf->dma_addrs[i], ++ PAGE_SIZE, DMA_BIDIRECTIONAL); ++ } ++fault_mismatch: ++ if (pages) { ++ for (i = 0; i < faulted_pages; i++) ++ put_page(pages[i]); ++ } ++no_page_array: ++invalid_flags: ++ kbase_mem_phy_alloc_put(reg->cpu_alloc); ++ kbase_mem_phy_alloc_put(reg->gpu_alloc); ++no_alloc_obj: ++ kfree(reg); ++no_region: ++bad_size: ++ return NULL; ++ ++} ++ ++ ++u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, ++ u64 nents, struct base_mem_aliasing_info *ai, ++ u64 *num_pages) ++{ ++ struct kbase_va_region *reg; ++ u64 gpu_va; ++ size_t i; ++ bool coherent; ++ ++ KBASE_DEBUG_ASSERT(kctx); ++ KBASE_DEBUG_ASSERT(flags); ++ KBASE_DEBUG_ASSERT(ai); ++ KBASE_DEBUG_ASSERT(num_pages); ++ ++ /* mask to only allowed flags */ ++ *flags &= (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR | ++ BASE_MEM_COHERENT_SYSTEM | BASE_MEM_COHERENT_LOCAL | ++ BASE_MEM_PROT_CPU_RD | BASE_MEM_COHERENT_SYSTEM_REQUIRED); ++ ++ if (!(*flags & (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR))) { ++ dev_warn(kctx->kbdev->dev, ++ "kbase_mem_alias called with bad flags (%llx)", ++ (unsigned long long)*flags); ++ goto bad_flags; ++ } ++ coherent = (*flags & BASE_MEM_COHERENT_SYSTEM) != 0 || ++ (*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0; ++ ++ if (!stride) ++ goto bad_stride; ++ ++ if (!nents) ++ goto bad_nents; ++ ++ if ((nents * stride) > (U64_MAX / PAGE_SIZE)) ++ /* 64-bit address range is the max */ ++ goto bad_size; ++ ++ /* calculate the number of pages this alias will cover */ ++ *num_pages = nents * stride; ++ ++#ifdef CONFIG_64BIT ++ if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { ++ /* 64-bit tasks must MMAP anyway, but not expose this address to ++ * clients */ ++ *flags |= BASE_MEM_NEED_MMAP; ++ reg = kbase_alloc_free_region(&kctx->reg_rbtree_same, 0, ++ *num_pages, ++ KBASE_REG_ZONE_SAME_VA); ++ } else { ++#else ++ if (1) { ++#endif ++ reg = kbase_alloc_free_region(&kctx->reg_rbtree_custom, ++ 0, *num_pages, ++ KBASE_REG_ZONE_CUSTOM_VA); ++ } ++ ++ if (!reg) ++ goto no_reg; ++ ++ /* zero-sized page array, as we don't need one/can support one */ ++ reg->gpu_alloc = kbase_alloc_create(kctx, 0, KBASE_MEM_TYPE_ALIAS, ++ BASE_MEM_GROUP_DEFAULT); ++ if (IS_ERR_OR_NULL(reg->gpu_alloc)) ++ goto no_alloc_obj; ++ ++ reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); ++ ++ if (kbase_update_region_flags(kctx, reg, *flags) != 0) ++ goto invalid_flags; ++ ++ reg->gpu_alloc->imported.alias.nents = nents; ++ reg->gpu_alloc->imported.alias.stride = stride; ++ reg->gpu_alloc->imported.alias.aliased = vzalloc(sizeof(*reg->gpu_alloc->imported.alias.aliased) * nents); ++ if (!reg->gpu_alloc->imported.alias.aliased) ++ goto no_aliased_array; ++ ++ kbase_gpu_vm_lock(kctx); ++ ++ /* validate and add src handles */ ++ for (i = 0; i < nents; i++) { ++ if (ai[i].handle.basep.handle < BASE_MEM_FIRST_FREE_ADDRESS) { ++ if (ai[i].handle.basep.handle != ++ BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE) ++ goto bad_handle; /* unsupported magic handle */ ++ if (!ai[i].length) ++ goto bad_handle; /* must be > 0 */ ++ if (ai[i].length > stride) ++ goto bad_handle; /* can't be larger than the ++ stride */ ++ reg->gpu_alloc->imported.alias.aliased[i].length = ai[i].length; ++ } else { ++ struct kbase_va_region *aliasing_reg; ++ struct kbase_mem_phy_alloc *alloc; ++ ++ aliasing_reg = kbase_region_tracker_find_region_base_address( ++ kctx, ++ (ai[i].handle.basep.handle >> PAGE_SHIFT) << PAGE_SHIFT); ++ ++ /* validate found region */ ++ if (kbase_is_region_invalid_or_free(aliasing_reg)) ++ goto bad_handle; /* Not found/already free */ ++ if (aliasing_reg->flags & KBASE_REG_DONT_NEED) ++ goto bad_handle; /* Ephemeral region */ ++ if (!(aliasing_reg->flags & KBASE_REG_GPU_CACHED)) ++ goto bad_handle; /* GPU uncached memory */ ++ if (!aliasing_reg->gpu_alloc) ++ goto bad_handle; /* No alloc */ ++ if (aliasing_reg->gpu_alloc->type != KBASE_MEM_TYPE_NATIVE) ++ goto bad_handle; /* Not a native alloc */ ++ if (coherent != ((aliasing_reg->flags & KBASE_REG_SHARE_BOTH) != 0)) ++ goto bad_handle; ++ /* Non-coherent memory cannot alias ++ coherent memory, and vice versa.*/ ++ ++ /* check size against stride */ ++ if (!ai[i].length) ++ goto bad_handle; /* must be > 0 */ ++ if (ai[i].length > stride) ++ goto bad_handle; /* can't be larger than the ++ stride */ ++ ++ alloc = aliasing_reg->gpu_alloc; ++ ++ /* check against the alloc's size */ ++ if (ai[i].offset > alloc->nents) ++ goto bad_handle; /* beyond end */ ++ if (ai[i].offset + ai[i].length > alloc->nents) ++ goto bad_handle; /* beyond end */ ++ ++ reg->gpu_alloc->imported.alias.aliased[i].alloc = kbase_mem_phy_alloc_get(alloc); ++ reg->gpu_alloc->imported.alias.aliased[i].length = ai[i].length; ++ reg->gpu_alloc->imported.alias.aliased[i].offset = ai[i].offset; ++ } ++ } ++ ++#ifdef CONFIG_64BIT ++ if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { ++ /* Bind to a cookie */ ++ if (bitmap_empty(kctx->cookies, BITS_PER_LONG)) { ++ dev_err(kctx->kbdev->dev, "No cookies available for allocation!"); ++ goto no_cookie; ++ } ++ /* return a cookie */ ++ gpu_va = find_first_bit(kctx->cookies, BITS_PER_LONG); ++ bitmap_clear(kctx->cookies, gpu_va, 1); ++ BUG_ON(kctx->pending_regions[gpu_va]); ++ kctx->pending_regions[gpu_va] = reg; ++ ++ /* relocate to correct base */ ++ gpu_va += PFN_DOWN(BASE_MEM_COOKIE_BASE); ++ gpu_va <<= PAGE_SHIFT; ++ } else /* we control the VA */ { ++#else ++ if (1) { ++#endif ++ if (kbase_gpu_mmap(kctx, reg, 0, *num_pages, 1) != 0) { ++ dev_warn(kctx->kbdev->dev, "Failed to map memory on GPU"); ++ goto no_mmap; ++ } ++ /* return real GPU VA */ ++ gpu_va = reg->start_pfn << PAGE_SHIFT; ++ } ++ ++ reg->flags &= ~KBASE_REG_FREE; ++ reg->flags &= ~KBASE_REG_GROWABLE; ++ ++ kbase_gpu_vm_unlock(kctx); ++ ++ return gpu_va; ++ ++#ifdef CONFIG_64BIT ++no_cookie: ++#endif ++no_mmap: ++bad_handle: ++ kbase_gpu_vm_unlock(kctx); ++no_aliased_array: ++invalid_flags: ++ kbase_mem_phy_alloc_put(reg->cpu_alloc); ++ kbase_mem_phy_alloc_put(reg->gpu_alloc); ++no_alloc_obj: ++ kfree(reg); ++no_reg: ++bad_size: ++bad_nents: ++bad_stride: ++bad_flags: ++ return 0; ++} ++ ++int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type, ++ void __user *phandle, u32 padding, u64 *gpu_va, u64 *va_pages, ++ u64 *flags) ++{ ++ struct kbase_va_region *reg; ++ ++ KBASE_DEBUG_ASSERT(kctx); ++ KBASE_DEBUG_ASSERT(gpu_va); ++ KBASE_DEBUG_ASSERT(va_pages); ++ KBASE_DEBUG_ASSERT(flags); ++ ++ if ((!kbase_ctx_flag(kctx, KCTX_COMPAT)) && ++ kbase_ctx_flag(kctx, KCTX_FORCE_SAME_VA)) ++ *flags |= BASE_MEM_SAME_VA; ++ ++ if (!kbase_check_import_flags(*flags)) { ++ dev_warn(kctx->kbdev->dev, ++ "kbase_mem_import called with bad flags (%llx)", ++ (unsigned long long)*flags); ++ goto bad_flags; ++ } ++ ++ if ((*flags & BASE_MEM_UNCACHED_GPU) != 0 && ++ (*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0) { ++ /* Remove COHERENT_SYSTEM_REQUIRED flag if uncached GPU mapping is requested */ ++ *flags &= ~BASE_MEM_COHERENT_SYSTEM_REQUIRED; ++ } ++ if ((*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0 && ++ !kbase_device_is_cpu_coherent(kctx->kbdev)) { ++ dev_warn(kctx->kbdev->dev, ++ "kbase_mem_import call required coherent mem when unavailable"); ++ goto bad_flags; ++ } ++ if ((*flags & BASE_MEM_COHERENT_SYSTEM) != 0 && ++ !kbase_device_is_cpu_coherent(kctx->kbdev)) { ++ /* Remove COHERENT_SYSTEM flag if coherent mem is unavailable */ ++ *flags &= ~BASE_MEM_COHERENT_SYSTEM; ++ } ++ ++ if ((padding != 0) && (type != BASE_MEM_IMPORT_TYPE_UMM)) { ++ dev_warn(kctx->kbdev->dev, ++ "padding is only supported for UMM"); ++ goto bad_flags; ++ } ++ ++ switch (type) { ++ case BASE_MEM_IMPORT_TYPE_UMM: { ++ int fd; ++ ++ if (get_user(fd, (int __user *)phandle)) ++ reg = NULL; ++ else ++ reg = kbase_mem_from_umm(kctx, fd, va_pages, flags, ++ padding); ++ } ++ break; ++ case BASE_MEM_IMPORT_TYPE_USER_BUFFER: { ++ struct base_mem_import_user_buffer user_buffer; ++ void __user *uptr; ++ ++ if (copy_from_user(&user_buffer, phandle, ++ sizeof(user_buffer))) { ++ reg = NULL; ++ } else { ++#ifdef CONFIG_COMPAT ++ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) ++ uptr = compat_ptr(user_buffer.ptr); ++ else ++#endif ++ uptr = u64_to_user_ptr(user_buffer.ptr); ++ ++ reg = kbase_mem_from_user_buffer(kctx, ++ (unsigned long)uptr, user_buffer.length, ++ va_pages, flags); ++ } ++ break; ++ } ++ default: { ++ reg = NULL; ++ break; ++ } ++ } ++ ++ if (!reg) ++ goto no_reg; ++ ++ kbase_gpu_vm_lock(kctx); ++ ++ /* mmap needed to setup VA? */ ++ if (*flags & (BASE_MEM_SAME_VA | BASE_MEM_NEED_MMAP)) { ++ /* Bind to a cookie */ ++ if (bitmap_empty(kctx->cookies, BITS_PER_LONG)) ++ goto no_cookie; ++ /* return a cookie */ ++ *gpu_va = find_first_bit(kctx->cookies, BITS_PER_LONG); ++ bitmap_clear(kctx->cookies, *gpu_va, 1); ++ BUG_ON(kctx->pending_regions[*gpu_va]); ++ kctx->pending_regions[*gpu_va] = reg; ++ ++ /* relocate to correct base */ ++ *gpu_va += PFN_DOWN(BASE_MEM_COOKIE_BASE); ++ *gpu_va <<= PAGE_SHIFT; ++ ++ } else if (*flags & KBASE_MEM_IMPORT_HAVE_PAGES) { ++ /* we control the VA, mmap now to the GPU */ ++ if (kbase_gpu_mmap(kctx, reg, 0, *va_pages, 1) != 0) ++ goto no_gpu_va; ++ /* return real GPU VA */ ++ *gpu_va = reg->start_pfn << PAGE_SHIFT; ++ } else { ++ /* we control the VA, but nothing to mmap yet */ ++ if (kbase_add_va_region(kctx, reg, 0, *va_pages, 1) != 0) ++ goto no_gpu_va; ++ /* return real GPU VA */ ++ *gpu_va = reg->start_pfn << PAGE_SHIFT; ++ } ++ ++ /* clear out private flags */ ++ *flags &= ((1UL << BASE_MEM_FLAGS_NR_BITS) - 1); ++ ++ kbase_gpu_vm_unlock(kctx); ++ ++ return 0; ++ ++no_gpu_va: ++no_cookie: ++ kbase_gpu_vm_unlock(kctx); ++ kbase_mem_phy_alloc_put(reg->cpu_alloc); ++ kbase_mem_phy_alloc_put(reg->gpu_alloc); ++ kfree(reg); ++no_reg: ++bad_flags: ++ *gpu_va = 0; ++ *va_pages = 0; ++ *flags = 0; ++ return -ENOMEM; ++} ++ ++int kbase_mem_grow_gpu_mapping(struct kbase_context *kctx, ++ struct kbase_va_region *reg, ++ u64 new_pages, u64 old_pages) ++{ ++ struct tagged_addr *phy_pages; ++ u64 delta = new_pages - old_pages; ++ int ret = 0; ++ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ /* Map the new pages into the GPU */ ++ phy_pages = kbase_get_gpu_phy_pages(reg); ++ ret = kbase_mmu_insert_pages(kctx->kbdev, &kctx->mmu, ++ reg->start_pfn + old_pages, phy_pages + old_pages, delta, ++ reg->flags, kctx->as_nr, reg->gpu_alloc->group_id); ++ ++ return ret; ++} ++ ++void kbase_mem_shrink_cpu_mapping(struct kbase_context *kctx, ++ struct kbase_va_region *reg, ++ u64 new_pages, u64 old_pages) ++{ ++ u64 gpu_va_start = reg->start_pfn; ++ ++ if (new_pages == old_pages) ++ /* Nothing to do */ ++ return; ++ ++ unmap_mapping_range(kctx->filp->f_inode->i_mapping, ++ (gpu_va_start + new_pages)<kbdev, &kctx->mmu, ++ reg->start_pfn + new_pages, delta, kctx->as_nr); ++ ++ return ret; ++} ++ ++int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages) ++{ ++ u64 old_pages; ++ u64 delta = 0; ++ int res = -EINVAL; ++ struct kbase_va_region *reg; ++ bool read_locked = false; ++ ++ KBASE_DEBUG_ASSERT(kctx); ++ KBASE_DEBUG_ASSERT(gpu_addr != 0); ++ ++ if (gpu_addr & ~PAGE_MASK) { ++ dev_warn(kctx->kbdev->dev, "kbase:mem_commit: gpu_addr: passed parameter is invalid"); ++ return -EINVAL; ++ } ++ ++ down_write(kbase_mem_get_process_mmap_lock()); ++ kbase_gpu_vm_lock(kctx); ++ ++ /* Validate the region */ ++ reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr); ++ if (kbase_is_region_invalid_or_free(reg)) ++ goto out_unlock; ++ ++ KBASE_DEBUG_ASSERT(reg->cpu_alloc); ++ KBASE_DEBUG_ASSERT(reg->gpu_alloc); ++ ++ if (reg->gpu_alloc->type != KBASE_MEM_TYPE_NATIVE) ++ goto out_unlock; ++ ++ if (0 == (reg->flags & KBASE_REG_GROWABLE)) ++ goto out_unlock; ++ ++ if (reg->flags & KBASE_REG_ACTIVE_JIT_ALLOC) ++ goto out_unlock; ++ ++ /* Would overflow the VA region */ ++ if (new_pages > reg->nr_pages) ++ goto out_unlock; ++ ++ /* can't be mapped more than once on the GPU */ ++ if (atomic_read(®->gpu_alloc->gpu_mappings) > 1) ++ goto out_unlock; ++ /* can't grow regions which are ephemeral */ ++ if (reg->flags & KBASE_REG_DONT_NEED) ++ goto out_unlock; ++ ++#ifdef CONFIG_MALI_MEMORY_FULLY_BACKED ++ /* Reject resizing commit size */ ++ if (reg->flags & KBASE_REG_PF_GROW) ++ new_pages = reg->nr_pages; ++#endif ++ ++ if (new_pages == reg->gpu_alloc->nents) { ++ /* no change */ ++ res = 0; ++ goto out_unlock; ++ } ++ ++ old_pages = kbase_reg_current_backed_size(reg); ++ if (new_pages > old_pages) { ++ delta = new_pages - old_pages; ++ ++ /* ++ * No update to the mm so downgrade the writer lock to a read ++ * lock so other readers aren't blocked after this point. ++ */ ++ downgrade_write(kbase_mem_get_process_mmap_lock()); ++ read_locked = true; ++ ++ /* Allocate some more pages */ ++ if (kbase_alloc_phy_pages_helper(reg->cpu_alloc, delta) != 0) { ++ res = -ENOMEM; ++ goto out_unlock; ++ } ++ if (reg->cpu_alloc != reg->gpu_alloc) { ++ if (kbase_alloc_phy_pages_helper( ++ reg->gpu_alloc, delta) != 0) { ++ res = -ENOMEM; ++ kbase_free_phy_pages_helper(reg->cpu_alloc, ++ delta); ++ goto out_unlock; ++ } ++ } ++ ++ /* No update required for CPU mappings, that's done on fault. */ ++ ++ /* Update GPU mapping. */ ++ res = kbase_mem_grow_gpu_mapping(kctx, reg, ++ new_pages, old_pages); ++ ++ /* On error free the new pages */ ++ if (res) { ++ kbase_free_phy_pages_helper(reg->cpu_alloc, delta); ++ if (reg->cpu_alloc != reg->gpu_alloc) ++ kbase_free_phy_pages_helper(reg->gpu_alloc, ++ delta); ++ res = -ENOMEM; ++ goto out_unlock; ++ } ++ } else { ++ res = kbase_mem_shrink(kctx, reg, new_pages); ++ if (res) ++ res = -ENOMEM; ++ } ++ ++out_unlock: ++ kbase_gpu_vm_unlock(kctx); ++ if (read_locked) ++ up_read(kbase_mem_get_process_mmap_lock()); ++ else ++ up_write(kbase_mem_get_process_mmap_lock()); ++ ++ return res; ++} ++ ++int kbase_mem_shrink(struct kbase_context *const kctx, ++ struct kbase_va_region *const reg, u64 const new_pages) ++{ ++ u64 delta, old_pages; ++ int err; ++ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ if (WARN_ON(!kctx)) ++ return -EINVAL; ++ ++ if (WARN_ON(!reg)) ++ return -EINVAL; ++ ++ old_pages = kbase_reg_current_backed_size(reg); ++ if (WARN_ON(old_pages < new_pages)) ++ return -EINVAL; ++ ++ delta = old_pages - new_pages; ++ ++ /* Update the GPU mapping */ ++ err = kbase_mem_shrink_gpu_mapping(kctx, reg, ++ new_pages, old_pages); ++ if (err >= 0) { ++ /* Update all CPU mapping(s) */ ++ kbase_mem_shrink_cpu_mapping(kctx, reg, ++ new_pages, old_pages); ++ ++ kbase_free_phy_pages_helper(reg->cpu_alloc, delta); ++ if (reg->cpu_alloc != reg->gpu_alloc) ++ kbase_free_phy_pages_helper(reg->gpu_alloc, delta); ++ } ++ ++ return err; ++} ++ ++ ++static void kbase_cpu_vm_open(struct vm_area_struct *vma) ++{ ++ struct kbase_cpu_mapping *map = vma->vm_private_data; ++ ++ KBASE_DEBUG_ASSERT(map); ++ KBASE_DEBUG_ASSERT(map->count > 0); ++ /* non-atomic as we're under Linux' mm lock */ ++ map->count++; ++} ++ ++static void kbase_cpu_vm_close(struct vm_area_struct *vma) ++{ ++ struct kbase_cpu_mapping *map = vma->vm_private_data; ++ ++ KBASE_DEBUG_ASSERT(map); ++ KBASE_DEBUG_ASSERT(map->count > 0); ++ ++ /* non-atomic as we're under Linux' mm lock */ ++ if (--map->count) ++ return; ++ ++ KBASE_DEBUG_ASSERT(map->kctx); ++ KBASE_DEBUG_ASSERT(map->alloc); ++ ++ kbase_gpu_vm_lock(map->kctx); ++ ++ if (map->free_on_close) { ++ KBASE_DEBUG_ASSERT((map->region->flags & KBASE_REG_ZONE_MASK) == ++ KBASE_REG_ZONE_SAME_VA); ++ /* Avoid freeing memory on the process death which results in ++ * GPU Page Fault. Memory will be freed in kbase_destroy_context ++ */ ++ if (!(current->flags & PF_EXITING)) ++ kbase_mem_free_region(map->kctx, map->region); ++ } ++ ++ list_del(&map->mappings_list); ++ ++ kbase_va_region_alloc_put(map->kctx, map->region); ++ kbase_gpu_vm_unlock(map->kctx); ++ ++ kbase_mem_phy_alloc_put(map->alloc); ++ kfree(map); ++} ++ ++static struct kbase_aliased *get_aliased_alloc(struct vm_area_struct *vma, ++ struct kbase_va_region *reg, ++ pgoff_t *start_off, ++ size_t nr_pages) ++{ ++ struct kbase_aliased *aliased = ++ reg->cpu_alloc->imported.alias.aliased; ++ ++ if (!reg->cpu_alloc->imported.alias.stride || ++ reg->nr_pages < (*start_off + nr_pages)) { ++ return NULL; ++ } ++ ++ while (*start_off >= reg->cpu_alloc->imported.alias.stride) { ++ aliased++; ++ *start_off -= reg->cpu_alloc->imported.alias.stride; ++ } ++ ++ if (!aliased->alloc) { ++ /* sink page not available for dumping map */ ++ return NULL; ++ } ++ ++ if ((*start_off + nr_pages) > aliased->length) { ++ /* not fully backed by physical pages */ ++ return NULL; ++ } ++ ++ return aliased; ++} ++ ++#if (KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE) ++static vm_fault_t kbase_cpu_vm_fault(struct vm_area_struct *vma, ++ struct vm_fault *vmf) ++{ ++#else ++static vm_fault_t kbase_cpu_vm_fault(struct vm_fault *vmf) ++{ ++ struct vm_area_struct *vma = vmf->vma; ++#endif ++ struct kbase_cpu_mapping *map = vma->vm_private_data; ++ pgoff_t map_start_pgoff; ++ pgoff_t fault_pgoff; ++ size_t i; ++ pgoff_t addr; ++ size_t nents; ++ struct tagged_addr *pages; ++ vm_fault_t ret = VM_FAULT_SIGBUS; ++ struct memory_group_manager_device *mgm_dev; ++ ++ KBASE_DEBUG_ASSERT(map); ++ KBASE_DEBUG_ASSERT(map->count > 0); ++ KBASE_DEBUG_ASSERT(map->kctx); ++ KBASE_DEBUG_ASSERT(map->alloc); ++ ++ map_start_pgoff = vma->vm_pgoff - map->region->start_pfn; ++ ++ kbase_gpu_vm_lock(map->kctx); ++ if (unlikely(map->region->cpu_alloc->type == KBASE_MEM_TYPE_ALIAS)) { ++ struct kbase_aliased *aliased = ++ get_aliased_alloc(vma, map->region, &map_start_pgoff, 1); ++ ++ if (!aliased) ++ goto exit; ++ ++ nents = aliased->length; ++ pages = aliased->alloc->pages + aliased->offset; ++ } else { ++ nents = map->alloc->nents; ++ pages = map->alloc->pages; ++ } ++ ++ fault_pgoff = map_start_pgoff + (vmf->pgoff - vma->vm_pgoff); ++ ++ if (fault_pgoff >= nents) ++ goto exit; ++ ++ /* Fault on access to DONT_NEED regions */ ++ if (map->alloc->reg && (map->alloc->reg->flags & KBASE_REG_DONT_NEED)) ++ goto exit; ++ ++ /* We are inserting all valid pages from the start of CPU mapping and ++ * not from the fault location (the mmap handler was previously doing ++ * the same). ++ */ ++ i = map_start_pgoff; ++ addr = (pgoff_t)(vma->vm_start >> PAGE_SHIFT); ++ mgm_dev = map->kctx->kbdev->mgm_dev; ++ while (i < nents && (addr < vma->vm_end >> PAGE_SHIFT)) { ++ ++ ret = mgm_dev->ops.mgm_vmf_insert_pfn_prot(mgm_dev, ++ map->alloc->group_id, vma, addr << PAGE_SHIFT, ++ PFN_DOWN(as_phys_addr_t(pages[i])), vma->vm_page_prot); ++ ++ if (ret != VM_FAULT_NOPAGE) ++ goto exit; ++ ++ i++; addr++; ++ } ++ ++exit: ++ kbase_gpu_vm_unlock(map->kctx); ++ return ret; ++} ++ ++const struct vm_operations_struct kbase_vm_ops = { ++ .open = kbase_cpu_vm_open, ++ .close = kbase_cpu_vm_close, ++ .fault = kbase_cpu_vm_fault ++}; ++ ++static int kbase_cpu_mmap(struct kbase_context *kctx, ++ struct kbase_va_region *reg, ++ struct vm_area_struct *vma, ++ void *kaddr, ++ size_t nr_pages, ++ unsigned long aligned_offset, ++ int free_on_close) ++{ ++ struct kbase_cpu_mapping *map; ++ int err = 0; ++ ++ map = kzalloc(sizeof(*map), GFP_KERNEL); ++ ++ if (!map) { ++ WARN_ON(1); ++ err = -ENOMEM; ++ goto out; ++ } ++ ++ /* ++ * VM_DONTCOPY - don't make this mapping available in fork'ed processes ++ * VM_DONTEXPAND - disable mremap on this region ++ * VM_IO - disables paging ++ * VM_DONTDUMP - Don't include in core dumps (3.7 only) ++ * VM_MIXEDMAP - Support mixing struct page*s and raw pfns. ++ * This is needed to support using the dedicated and ++ * the OS based memory backends together. ++ */ ++ /* ++ * This will need updating to propagate coherency flags ++ * See MIDBASE-1057 ++ */ ++ ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0)) ++ vma->vm_flags |= VM_DONTCOPY | VM_DONTDUMP | VM_DONTEXPAND | VM_IO; ++#else ++ vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_RESERVED | VM_IO; ++#endif ++ vma->vm_ops = &kbase_vm_ops; ++ vma->vm_private_data = map; ++ ++ if (reg->cpu_alloc->type == KBASE_MEM_TYPE_ALIAS && nr_pages) { ++ pgoff_t rel_pgoff = vma->vm_pgoff - reg->start_pfn + ++ (aligned_offset >> PAGE_SHIFT); ++ struct kbase_aliased *aliased = ++ get_aliased_alloc(vma, reg, &rel_pgoff, nr_pages); ++ ++ if (!aliased) { ++ err = -EINVAL; ++ kfree(map); ++ goto out; ++ } ++ } ++ ++ if (!(reg->flags & KBASE_REG_CPU_CACHED) && ++ (reg->flags & (KBASE_REG_CPU_WR|KBASE_REG_CPU_RD))) { ++ /* We can't map vmalloc'd memory uncached. ++ * Other memory will have been returned from ++ * kbase_mem_pool which would be ++ * suitable for mapping uncached. ++ */ ++ BUG_ON(kaddr); ++ vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); ++ } ++ ++ if (!kaddr) { ++ vma->vm_flags |= VM_PFNMAP; ++ } else { ++ WARN_ON(aligned_offset); ++ /* MIXEDMAP so we can vfree the kaddr early and not track it after map time */ ++ vma->vm_flags |= VM_MIXEDMAP; ++ /* vmalloc remaping is easy... */ ++ err = remap_vmalloc_range(vma, kaddr, 0); ++ WARN_ON(err); ++ } ++ ++ if (err) { ++ kfree(map); ++ goto out; ++ } ++ ++ map->region = kbase_va_region_alloc_get(kctx, reg); ++ map->free_on_close = free_on_close; ++ map->kctx = kctx; ++ map->alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc); ++ map->count = 1; /* start with one ref */ ++ ++ if (reg->flags & KBASE_REG_CPU_CACHED) ++ map->alloc->properties |= KBASE_MEM_PHY_ALLOC_ACCESSED_CACHED; ++ ++ list_add(&map->mappings_list, &map->alloc->mappings); ++ ++ out: ++ return err; ++} ++ ++#ifdef CONFIG_MALI_VECTOR_DUMP ++static void kbase_free_unused_jit_allocations(struct kbase_context *kctx) ++{ ++ /* Free all cached/unused JIT allocations as their contents are not ++ * really needed for the replay. The GPU writes to them would already ++ * have been captured through the GWT mechanism. ++ * This considerably reduces the size of mmu-snapshot-file and it also ++ * helps avoid segmentation fault issue during vector dumping of ++ * complex contents when the unused JIT allocations are accessed to ++ * dump their contents (as they appear in the page tables snapshot) ++ * but they got freed by the shrinker under low memory scenarios ++ * (which do occur with complex contents). ++ */ ++ while (kbase_jit_evict(kctx)) ++ ; ++} ++#endif ++ ++static int kbase_mmu_dump_mmap(struct kbase_context *kctx, ++ struct vm_area_struct *vma, ++ struct kbase_va_region **const reg, ++ void **const kmap_addr) ++{ ++ struct kbase_va_region *new_reg; ++ void *kaddr; ++ u32 nr_pages; ++ size_t size; ++ int err = 0; ++ ++ dev_dbg(kctx->kbdev->dev, "in kbase_mmu_dump_mmap\n"); ++ size = (vma->vm_end - vma->vm_start); ++ nr_pages = size >> PAGE_SHIFT; ++ ++#ifdef CONFIG_MALI_VECTOR_DUMP ++ kbase_free_unused_jit_allocations(kctx); ++#endif ++ ++ kaddr = kbase_mmu_dump(kctx, nr_pages); ++ ++ if (!kaddr) { ++ err = -ENOMEM; ++ goto out; ++ } ++ ++ new_reg = kbase_alloc_free_region(&kctx->reg_rbtree_same, 0, nr_pages, ++ KBASE_REG_ZONE_SAME_VA); ++ if (!new_reg) { ++ err = -ENOMEM; ++ WARN_ON(1); ++ goto out; ++ } ++ ++ new_reg->cpu_alloc = kbase_alloc_create(kctx, 0, KBASE_MEM_TYPE_RAW, ++ BASE_MEM_GROUP_DEFAULT); ++ if (IS_ERR_OR_NULL(new_reg->cpu_alloc)) { ++ err = -ENOMEM; ++ new_reg->cpu_alloc = NULL; ++ WARN_ON(1); ++ goto out_no_alloc; ++ } ++ ++ new_reg->gpu_alloc = kbase_mem_phy_alloc_get(new_reg->cpu_alloc); ++ ++ new_reg->flags &= ~KBASE_REG_FREE; ++ new_reg->flags |= KBASE_REG_CPU_CACHED; ++ if (kbase_add_va_region(kctx, new_reg, vma->vm_start, nr_pages, 1) != 0) { ++ err = -ENOMEM; ++ WARN_ON(1); ++ goto out_va_region; ++ } ++ ++ *kmap_addr = kaddr; ++ *reg = new_reg; ++ ++ dev_dbg(kctx->kbdev->dev, "kbase_mmu_dump_mmap done\n"); ++ return 0; ++ ++out_no_alloc: ++out_va_region: ++ kbase_free_alloced_region(new_reg); ++out: ++ return err; ++} ++ ++ ++void kbase_os_mem_map_lock(struct kbase_context *kctx) ++{ ++ (void)kctx; ++ down_read(kbase_mem_get_process_mmap_lock()); ++} ++ ++void kbase_os_mem_map_unlock(struct kbase_context *kctx) ++{ ++ (void)kctx; ++ up_read(kbase_mem_get_process_mmap_lock()); ++} ++ ++static int kbasep_reg_mmap(struct kbase_context *kctx, ++ struct vm_area_struct *vma, ++ struct kbase_va_region **regm, ++ size_t *nr_pages, size_t *aligned_offset) ++ ++{ ++ int cookie = vma->vm_pgoff - PFN_DOWN(BASE_MEM_COOKIE_BASE); ++ struct kbase_va_region *reg; ++ int err = 0; ++ ++ *aligned_offset = 0; ++ ++ dev_dbg(kctx->kbdev->dev, "in kbasep_reg_mmap\n"); ++ ++ /* SAME_VA stuff, fetch the right region */ ++ reg = kctx->pending_regions[cookie]; ++ if (!reg) { ++ err = -ENOMEM; ++ goto out; ++ } ++ ++ if ((reg->flags & KBASE_REG_GPU_NX) && (reg->nr_pages != *nr_pages)) { ++ /* incorrect mmap size */ ++ /* leave the cookie for a potential later ++ * mapping, or to be reclaimed later when the ++ * context is freed */ ++ err = -ENOMEM; ++ goto out; ++ } ++ ++ if ((vma->vm_flags & VM_READ && !(reg->flags & KBASE_REG_CPU_RD)) || ++ (vma->vm_flags & VM_WRITE && !(reg->flags & KBASE_REG_CPU_WR))) { ++ /* VM flags inconsistent with region flags */ ++ err = -EPERM; ++ dev_err(kctx->kbdev->dev, "%s:%d inconsistent VM flags\n", ++ __FILE__, __LINE__); ++ goto out; ++ } ++ ++ /* adjust down nr_pages to what we have physically */ ++ *nr_pages = kbase_reg_current_backed_size(reg); ++ ++ if (kbase_gpu_mmap(kctx, reg, vma->vm_start + *aligned_offset, ++ reg->nr_pages, 1) != 0) { ++ dev_err(kctx->kbdev->dev, "%s:%d\n", __FILE__, __LINE__); ++ /* Unable to map in GPU space. */ ++ WARN_ON(1); ++ err = -ENOMEM; ++ goto out; ++ } ++ /* no need for the cookie anymore */ ++ kctx->pending_regions[cookie] = NULL; ++ bitmap_set(kctx->cookies, cookie, 1); ++ ++ /* ++ * Overwrite the offset with the region start_pfn, so we effectively ++ * map from offset 0 in the region. However subtract the aligned ++ * offset so that when user space trims the mapping the beginning of ++ * the trimmed VMA has the correct vm_pgoff; ++ */ ++ vma->vm_pgoff = reg->start_pfn - ((*aligned_offset)>>PAGE_SHIFT); ++out: ++ *regm = reg; ++ dev_dbg(kctx->kbdev->dev, "kbasep_reg_mmap done\n"); ++ ++ return err; ++} ++ ++int kbase_context_mmap(struct kbase_context *const kctx, ++ struct vm_area_struct *const vma) ++{ ++ struct kbase_va_region *reg = NULL; ++ void *kaddr = NULL; ++ size_t nr_pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; ++ int err = 0; ++ int free_on_close = 0; ++ struct device *dev = kctx->kbdev->dev; ++ size_t aligned_offset = 0; ++ ++ dev_dbg(dev, "kbase_mmap\n"); ++ ++ if (!(vma->vm_flags & VM_READ)) ++ vma->vm_flags &= ~VM_MAYREAD; ++ if (!(vma->vm_flags & VM_WRITE)) ++ vma->vm_flags &= ~VM_MAYWRITE; ++ ++ if (0 == nr_pages) { ++ err = -EINVAL; ++ goto out; ++ } ++ ++ if (!(vma->vm_flags & VM_SHARED)) { ++ err = -EINVAL; ++ goto out; ++ } ++ ++ kbase_gpu_vm_lock(kctx); ++ ++ if (vma->vm_pgoff == PFN_DOWN(BASE_MEM_MAP_TRACKING_HANDLE)) { ++ /* The non-mapped tracking helper page */ ++ err = kbase_tracking_page_setup(kctx, vma); ++ goto out_unlock; ++ } ++ ++ /* if not the MTP, verify that the MTP has been mapped */ ++ rcu_read_lock(); ++ /* catches both when the special page isn't present or ++ * when we've forked */ ++ if (rcu_dereference(kctx->process_mm) != current->mm) { ++ err = -EINVAL; ++ rcu_read_unlock(); ++ goto out_unlock; ++ } ++ rcu_read_unlock(); ++ ++ switch (vma->vm_pgoff) { ++ case PFN_DOWN(BASEP_MEM_INVALID_HANDLE): ++ case PFN_DOWN(BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE): ++ /* Illegal handle for direct map */ ++ err = -EINVAL; ++ goto out_unlock; ++ case PFN_DOWN(BASE_MEM_MMU_DUMP_HANDLE): ++ /* MMU dump */ ++ err = kbase_mmu_dump_mmap(kctx, vma, ®, &kaddr); ++ if (0 != err) ++ goto out_unlock; ++ /* free the region on munmap */ ++ free_on_close = 1; ++ break; ++#if MALI_USE_CSF ++ case PFN_DOWN(BASEP_MEM_CSF_USER_REG_PAGE_HANDLE): ++ kbase_gpu_vm_unlock(kctx); ++ err = kbase_csf_cpu_mmap_user_reg_page(kctx, vma); ++ goto out; ++ case PFN_DOWN(BASEP_MEM_CSF_USER_IO_PAGES_HANDLE) ... ++ PFN_DOWN(BASE_MEM_COOKIE_BASE) - 1: { ++ kbase_gpu_vm_unlock(kctx); ++ mutex_lock(&kctx->csf.lock); ++ err = kbase_csf_cpu_mmap_user_io_pages(kctx, vma); ++ mutex_unlock(&kctx->csf.lock); ++ goto out; ++ } ++#endif ++ case PFN_DOWN(BASE_MEM_COOKIE_BASE) ... ++ PFN_DOWN(BASE_MEM_FIRST_FREE_ADDRESS) - 1: { ++ err = kbasep_reg_mmap(kctx, vma, ®, &nr_pages, ++ &aligned_offset); ++ if (0 != err) ++ goto out_unlock; ++ /* free the region on munmap */ ++ free_on_close = 1; ++ break; ++ } ++ default: { ++ reg = kbase_region_tracker_find_region_enclosing_address(kctx, ++ (u64)vma->vm_pgoff << PAGE_SHIFT); ++ ++ if (!kbase_is_region_invalid_or_free(reg)) { ++ /* will this mapping overflow the size of the region? */ ++ if (nr_pages > (reg->nr_pages - ++ (vma->vm_pgoff - reg->start_pfn))) { ++ err = -ENOMEM; ++ goto out_unlock; ++ } ++ ++ if ((vma->vm_flags & VM_READ && ++ !(reg->flags & KBASE_REG_CPU_RD)) || ++ (vma->vm_flags & VM_WRITE && ++ !(reg->flags & KBASE_REG_CPU_WR))) { ++ /* VM flags inconsistent with region flags */ ++ err = -EPERM; ++ dev_err(dev, "%s:%d inconsistent VM flags\n", ++ __FILE__, __LINE__); ++ goto out_unlock; ++ } ++ ++ if (KBASE_MEM_TYPE_IMPORTED_UMM == ++ reg->cpu_alloc->type) { ++ if (0 != (vma->vm_pgoff - reg->start_pfn)) { ++ err = -EINVAL; ++ dev_warn(dev, "%s:%d attempt to do a partial map in a dma_buf: non-zero offset to dma_buf mapping!\n", ++ __FILE__, __LINE__); ++ goto out_unlock; ++ } ++ err = dma_buf_mmap( ++ reg->cpu_alloc->imported.umm.dma_buf, ++ vma, vma->vm_pgoff - reg->start_pfn); ++ goto out_unlock; ++ } ++ ++ if (reg->cpu_alloc->type == KBASE_MEM_TYPE_ALIAS) { ++ /* initial params check for aliased dumping map */ ++ if (nr_pages > reg->gpu_alloc->imported.alias.stride || ++ !reg->gpu_alloc->imported.alias.stride || ++ !nr_pages) { ++ err = -EINVAL; ++ dev_warn(dev, "mmap aliased: invalid params!\n"); ++ goto out_unlock; ++ } ++ } ++ else if (reg->cpu_alloc->nents < ++ (vma->vm_pgoff - reg->start_pfn + nr_pages)) { ++ /* limit what we map to the amount currently backed */ ++ if ((vma->vm_pgoff - reg->start_pfn) >= reg->cpu_alloc->nents) ++ nr_pages = 0; ++ else ++ nr_pages = reg->cpu_alloc->nents - (vma->vm_pgoff - reg->start_pfn); ++ } ++ } else { ++ err = -ENOMEM; ++ goto out_unlock; ++ } ++ } /* default */ ++ } /* switch */ ++ ++ err = kbase_cpu_mmap(kctx, reg, vma, kaddr, nr_pages, aligned_offset, ++ free_on_close); ++ ++ if (vma->vm_pgoff == PFN_DOWN(BASE_MEM_MMU_DUMP_HANDLE)) { ++ /* MMU dump - userspace should now have a reference on ++ * the pages, so we can now free the kernel mapping */ ++ vfree(kaddr); ++ } ++ ++out_unlock: ++ kbase_gpu_vm_unlock(kctx); ++out: ++ if (err) ++ dev_err(dev, "mmap failed %d\n", err); ++ ++ return err; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_context_mmap); ++ ++void kbase_sync_mem_regions(struct kbase_context *kctx, ++ struct kbase_vmap_struct *map, enum kbase_sync_type dest) ++{ ++ size_t i; ++ off_t const offset = map->offset_in_page; ++ size_t const page_count = PFN_UP(offset + map->size); ++ ++ /* Sync first page */ ++ size_t sz = MIN(((size_t) PAGE_SIZE - offset), map->size); ++ struct tagged_addr cpu_pa = map->cpu_pages[0]; ++ struct tagged_addr gpu_pa = map->gpu_pages[0]; ++ ++ kbase_sync_single(kctx, cpu_pa, gpu_pa, offset, sz, dest); ++ ++ /* Sync middle pages (if any) */ ++ for (i = 1; page_count > 2 && i < page_count - 1; i++) { ++ cpu_pa = map->cpu_pages[i]; ++ gpu_pa = map->gpu_pages[i]; ++ kbase_sync_single(kctx, cpu_pa, gpu_pa, 0, PAGE_SIZE, dest); ++ } ++ ++ /* Sync last page (if any) */ ++ if (page_count > 1) { ++ cpu_pa = map->cpu_pages[page_count - 1]; ++ gpu_pa = map->gpu_pages[page_count - 1]; ++ sz = ((offset + map->size - 1) & ~PAGE_MASK) + 1; ++ kbase_sync_single(kctx, cpu_pa, gpu_pa, 0, sz, dest); ++ } ++} ++ ++static int kbase_vmap_phy_pages(struct kbase_context *kctx, ++ struct kbase_va_region *reg, u64 offset_bytes, size_t size, ++ struct kbase_vmap_struct *map) ++{ ++ unsigned long page_index; ++ unsigned int offset_in_page = offset_bytes & ~PAGE_MASK; ++ size_t page_count = PFN_UP(offset_in_page + size); ++ struct tagged_addr *page_array; ++ struct page **pages; ++ void *cpu_addr = NULL; ++ pgprot_t prot; ++ size_t i; ++ ++ if (!size || !map || !reg->cpu_alloc || !reg->gpu_alloc) ++ return -EINVAL; ++ ++ /* check if page_count calculation will wrap */ ++ if (size > ((size_t)-1 / PAGE_SIZE)) ++ return -EINVAL; ++ ++ page_index = offset_bytes >> PAGE_SHIFT; ++ ++ /* check if page_index + page_count will wrap */ ++ if (-1UL - page_count < page_index) ++ return -EINVAL; ++ ++ if (page_index + page_count > kbase_reg_current_backed_size(reg)) ++ return -ENOMEM; ++ ++ if (reg->flags & KBASE_REG_DONT_NEED) ++ return -EINVAL; ++ ++ prot = PAGE_KERNEL; ++ if (!(reg->flags & KBASE_REG_CPU_CACHED)) { ++ /* Map uncached */ ++ prot = pgprot_writecombine(prot); ++ } ++ ++ page_array = kbase_get_cpu_phy_pages(reg); ++ if (!page_array) ++ return -ENOMEM; ++ ++ pages = kmalloc_array(page_count, sizeof(struct page *), GFP_KERNEL); ++ if (!pages) ++ return -ENOMEM; ++ ++ for (i = 0; i < page_count; i++) ++ pages[i] = as_page(page_array[page_index + i]); ++ ++ /* Note: enforcing a RO prot_request onto prot is not done, since: ++ * - CPU-arch-specific integration required ++ * - kbase_vmap() requires no access checks to be made/enforced */ ++ ++ cpu_addr = vmap(pages, page_count, VM_MAP, prot); ++ ++ kfree(pages); ++ ++ if (!cpu_addr) ++ return -ENOMEM; ++ ++ map->offset_in_page = offset_in_page; ++ map->cpu_alloc = reg->cpu_alloc; ++ map->cpu_pages = &kbase_get_cpu_phy_pages(reg)[page_index]; ++ map->gpu_alloc = reg->gpu_alloc; ++ map->gpu_pages = &kbase_get_gpu_phy_pages(reg)[page_index]; ++ map->addr = (void *)((uintptr_t)cpu_addr + offset_in_page); ++ map->size = size; ++ map->sync_needed = ((reg->flags & KBASE_REG_CPU_CACHED) != 0) && ++ !kbase_mem_is_imported(map->gpu_alloc->type); ++ ++ if (map->sync_needed) ++ kbase_sync_mem_regions(kctx, map, KBASE_SYNC_TO_CPU); ++ ++ return 0; ++} ++ ++void *kbase_vmap_prot(struct kbase_context *kctx, u64 gpu_addr, size_t size, ++ unsigned long prot_request, struct kbase_vmap_struct *map) ++{ ++ struct kbase_va_region *reg; ++ void *addr = NULL; ++ u64 offset_bytes; ++ struct kbase_mem_phy_alloc *cpu_alloc; ++ struct kbase_mem_phy_alloc *gpu_alloc; ++ int err; ++ ++ kbase_gpu_vm_lock(kctx); ++ ++ reg = kbase_region_tracker_find_region_enclosing_address(kctx, ++ gpu_addr); ++ if (kbase_is_region_invalid_or_free(reg)) ++ goto out_unlock; ++ ++ /* check access permissions can be satisfied ++ * Intended only for checking KBASE_REG_{CPU,GPU}_{RD,WR} ++ */ ++ if ((reg->flags & prot_request) != prot_request) ++ goto out_unlock; ++ ++ offset_bytes = gpu_addr - (reg->start_pfn << PAGE_SHIFT); ++ cpu_alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc); ++ gpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); ++ ++ err = kbase_vmap_phy_pages(kctx, reg, offset_bytes, size, map); ++ if (err < 0) ++ goto fail_vmap_phy_pages; ++ ++ addr = map->addr; ++ ++out_unlock: ++ kbase_gpu_vm_unlock(kctx); ++ return addr; ++ ++fail_vmap_phy_pages: ++ kbase_gpu_vm_unlock(kctx); ++ kbase_mem_phy_alloc_put(cpu_alloc); ++ kbase_mem_phy_alloc_put(gpu_alloc); ++ ++ return NULL; ++} ++ ++void *kbase_vmap(struct kbase_context *kctx, u64 gpu_addr, size_t size, ++ struct kbase_vmap_struct *map) ++{ ++ /* 0 is specified for prot_request to indicate no access checks should ++ * be made. ++ * ++ * As mentioned in kbase_vmap_prot() this means that a kernel-side ++ * CPU-RO mapping is not enforced to allow this to work */ ++ return kbase_vmap_prot(kctx, gpu_addr, size, 0u, map); ++} ++KBASE_EXPORT_TEST_API(kbase_vmap); ++ ++static void kbase_vunmap_phy_pages(struct kbase_context *kctx, ++ struct kbase_vmap_struct *map) ++{ ++ void *addr = (void *)((uintptr_t)map->addr & PAGE_MASK); ++ vunmap(addr); ++ ++ if (map->sync_needed) ++ kbase_sync_mem_regions(kctx, map, KBASE_SYNC_TO_DEVICE); ++ ++ map->offset_in_page = 0; ++ map->cpu_pages = NULL; ++ map->gpu_pages = NULL; ++ map->addr = NULL; ++ map->size = 0; ++ map->sync_needed = false; ++} ++ ++void kbase_vunmap(struct kbase_context *kctx, struct kbase_vmap_struct *map) ++{ ++ kbase_vunmap_phy_pages(kctx, map); ++ map->cpu_alloc = kbase_mem_phy_alloc_put(map->cpu_alloc); ++ map->gpu_alloc = kbase_mem_phy_alloc_put(map->gpu_alloc); ++} ++KBASE_EXPORT_TEST_API(kbase_vunmap); ++ ++static void kbasep_add_mm_counter(struct mm_struct *mm, int member, long value) ++{ ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 19, 0)) ++ /* To avoid the build breakage due to an unexported kernel symbol ++ * 'mm_trace_rss_stat' from later kernels, i.e. from V4.19.0 onwards, ++ * we inline here the equivalent of 'add_mm_counter()' from linux ++ * kernel V5.4.0~8. ++ */ ++ atomic_long_add(value, &mm->rss_stat.count[member]); ++#else ++ add_mm_counter(mm, member, value); ++#endif ++} ++ ++void kbasep_os_process_page_usage_update(struct kbase_context *kctx, int pages) ++{ ++ struct mm_struct *mm; ++ ++ rcu_read_lock(); ++ mm = rcu_dereference(kctx->process_mm); ++ if (mm) { ++ atomic_add(pages, &kctx->nonmapped_pages); ++#ifdef SPLIT_RSS_COUNTING ++ kbasep_add_mm_counter(mm, MM_FILEPAGES, pages); ++#else ++ spin_lock(&mm->page_table_lock); ++ kbasep_add_mm_counter(mm, MM_FILEPAGES, pages); ++ spin_unlock(&mm->page_table_lock); ++#endif ++ } ++ rcu_read_unlock(); ++} ++ ++static void kbasep_os_process_page_usage_drain(struct kbase_context *kctx) ++{ ++ int pages; ++ struct mm_struct *mm; ++ ++ spin_lock(&kctx->mm_update_lock); ++ mm = rcu_dereference_protected(kctx->process_mm, lockdep_is_held(&kctx->mm_update_lock)); ++ if (!mm) { ++ spin_unlock(&kctx->mm_update_lock); ++ return; ++ } ++ ++ rcu_assign_pointer(kctx->process_mm, NULL); ++ spin_unlock(&kctx->mm_update_lock); ++ synchronize_rcu(); ++ ++ pages = atomic_xchg(&kctx->nonmapped_pages, 0); ++#ifdef SPLIT_RSS_COUNTING ++ kbasep_add_mm_counter(mm, MM_FILEPAGES, -pages); ++#else ++ spin_lock(&mm->page_table_lock); ++ kbasep_add_mm_counter(mm, MM_FILEPAGES, -pages); ++ spin_unlock(&mm->page_table_lock); ++#endif ++} ++ ++static void kbase_special_vm_close(struct vm_area_struct *vma) ++{ ++ struct kbase_context *kctx; ++ ++ kctx = vma->vm_private_data; ++ kbasep_os_process_page_usage_drain(kctx); ++} ++ ++static const struct vm_operations_struct kbase_vm_special_ops = { ++ .close = kbase_special_vm_close, ++}; ++ ++static int kbase_tracking_page_setup(struct kbase_context *kctx, struct vm_area_struct *vma) ++{ ++ /* check that this is the only tracking page */ ++ spin_lock(&kctx->mm_update_lock); ++ if (rcu_dereference_protected(kctx->process_mm, lockdep_is_held(&kctx->mm_update_lock))) { ++ spin_unlock(&kctx->mm_update_lock); ++ return -EFAULT; ++ } ++ ++ rcu_assign_pointer(kctx->process_mm, current->mm); ++ ++ spin_unlock(&kctx->mm_update_lock); ++ ++ /* no real access */ ++ vma->vm_flags &= ~(VM_READ | VM_MAYREAD | VM_WRITE | VM_MAYWRITE | VM_EXEC | VM_MAYEXEC); ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0)) ++ vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP | VM_IO; ++#else ++ vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_RESERVED | VM_IO; ++#endif ++ vma->vm_ops = &kbase_vm_special_ops; ++ vma->vm_private_data = kctx; ++ ++ return 0; ++} ++ ++#if MALI_USE_CSF ++static unsigned long get_queue_doorbell_pfn(struct kbase_device *kbdev, ++ struct kbase_queue *queue) ++{ ++ lockdep_assert_held(&kbdev->csf.reg_lock); ++ ++ /* Return the real Hw doorbell page if queue has been ++ * assigned one, otherwise a dummy page. Always return the ++ * dummy page in no mali builds. ++ */ ++ if ((queue->doorbell_nr == KBASEP_USER_DB_NR_INVALID) || ++ IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI)) ++ return PFN_DOWN(as_phys_addr_t(kbdev->csf.dummy_db_page)); ++ ++ return (PFN_DOWN(kbdev->reg_start + CSF_HW_DOORBELL_PAGE_OFFSET + ++ (u64)queue->doorbell_nr * CSF_HW_DOORBELL_PAGE_SIZE)); ++} ++ ++static void kbase_csf_user_io_pages_vm_open(struct vm_area_struct *vma) ++{ ++ WARN(1, "Unexpected attempt to clone private vma\n"); ++ vma->vm_private_data = NULL; ++} ++ ++static void kbase_csf_user_io_pages_vm_close(struct vm_area_struct *vma) ++{ ++ struct kbase_queue *queue = vma->vm_private_data; ++ struct kbase_context *kctx; ++ ++ if (WARN_ON(!queue)) ++ return; ++ ++ kctx = queue->kctx; ++ ++ mutex_lock(&kctx->csf.lock); ++ kbase_csf_queue_unbind(queue); ++ mutex_unlock(&kctx->csf.lock); ++ ++ /* Now as the vma is closed, drop the reference on mali device file */ ++ fput(kctx->filp); ++} ++ ++#if (KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE) ++static vm_fault_t kbase_csf_user_io_pages_vm_fault(struct vm_area_struct *vma, ++ struct vm_fault *vmf) ++{ ++#else ++static vm_fault_t kbase_csf_user_io_pages_vm_fault(struct vm_fault *vmf) ++{ ++ struct vm_area_struct *vma = vmf->vma; ++#endif ++ struct kbase_queue *queue = vma->vm_private_data; ++ unsigned long doorbell_cpu_addr, input_cpu_addr, output_cpu_addr; ++ unsigned long doorbell_page_pfn, input_page_pfn, output_page_pfn; ++ pgprot_t doorbell_pgprot, input_page_pgprot, output_page_pgprot; ++ size_t nr_pages = PFN_DOWN(vma->vm_end - vma->vm_start); ++ vm_fault_t ret; ++ struct kbase_device *kbdev; ++ struct memory_group_manager_device *mgm_dev; ++ ++ /* Few sanity checks up front */ ++ if ((nr_pages != BASEP_QUEUE_NR_MMAP_USER_PAGES) || ++ (vma->vm_pgoff != queue->db_file_offset)) ++ return VM_FAULT_SIGBUS; ++ ++ mutex_lock(&queue->kctx->csf.lock); ++ kbdev = queue->kctx->kbdev; ++ mgm_dev = kbdev->mgm_dev; ++ ++ /* Always map the doorbell page as uncached */ ++ doorbell_pgprot = pgprot_device(vma->vm_page_prot); ++ ++#if ((KERNEL_VERSION(4, 4, 147) >= LINUX_VERSION_CODE) || \ ++ ((KERNEL_VERSION(4, 6, 0) > LINUX_VERSION_CODE) && \ ++ (KERNEL_VERSION(4, 5, 0) <= LINUX_VERSION_CODE))) ++ vma->vm_page_prot = doorbell_pgprot; ++ input_page_pgprot = doorbell_pgprot; ++ output_page_pgprot = doorbell_pgprot; ++#else ++ if (kbdev->system_coherency == COHERENCY_NONE) { ++ input_page_pgprot = pgprot_writecombine(vma->vm_page_prot); ++ output_page_pgprot = pgprot_writecombine(vma->vm_page_prot); ++ } else { ++ input_page_pgprot = vma->vm_page_prot; ++ output_page_pgprot = vma->vm_page_prot; ++ } ++#endif ++ ++ doorbell_cpu_addr = vma->vm_start; ++ ++#if KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE ++ if ((unsigned long)vmf->virtual_address == doorbell_cpu_addr) { ++#else ++ if (vmf->address == doorbell_cpu_addr) { ++#endif ++ mutex_lock(&kbdev->csf.reg_lock); ++ doorbell_page_pfn = get_queue_doorbell_pfn(kbdev, queue); ++ ret = mgm_dev->ops.mgm_vmf_insert_pfn_prot(mgm_dev, ++ KBASE_MEM_GROUP_CSF_IO, vma, doorbell_cpu_addr, ++ doorbell_page_pfn, doorbell_pgprot); ++ mutex_unlock(&kbdev->csf.reg_lock); ++ } else { ++ /* Map the Input page */ ++ input_cpu_addr = doorbell_cpu_addr + PAGE_SIZE; ++ input_page_pfn = PFN_DOWN(as_phys_addr_t(queue->phys[0])); ++ ret = mgm_dev->ops.mgm_vmf_insert_pfn_prot(mgm_dev, ++ KBASE_MEM_GROUP_CSF_IO, vma, input_cpu_addr, ++ input_page_pfn, input_page_pgprot); ++ if (ret != VM_FAULT_NOPAGE) ++ goto exit; ++ ++ /* Map the Output page */ ++ output_cpu_addr = input_cpu_addr + PAGE_SIZE; ++ output_page_pfn = PFN_DOWN(as_phys_addr_t(queue->phys[1])); ++ ret = mgm_dev->ops.mgm_vmf_insert_pfn_prot(mgm_dev, ++ KBASE_MEM_GROUP_CSF_IO, vma, output_cpu_addr, ++ output_page_pfn, output_page_pgprot); ++ } ++ ++exit: ++ mutex_unlock(&queue->kctx->csf.lock); ++ return ret; ++} ++ ++static const struct vm_operations_struct kbase_csf_user_io_pages_vm_ops = { ++ .open = kbase_csf_user_io_pages_vm_open, ++ .close = kbase_csf_user_io_pages_vm_close, ++ .fault = kbase_csf_user_io_pages_vm_fault ++}; ++ ++/* Program the client process's page table entries to map the pair of ++ * input/output pages & Hw doorbell page. The caller should have validated that ++ * vma->vm_pgoff maps to the range of csf cookies. ++ */ ++static int kbase_csf_cpu_mmap_user_io_pages(struct kbase_context *kctx, ++ struct vm_area_struct *vma) ++{ ++ unsigned long cookie = ++ vma->vm_pgoff - PFN_DOWN(BASEP_MEM_CSF_USER_IO_PAGES_HANDLE); ++ size_t nr_pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; ++ struct kbase_queue *queue; ++ int err = 0; ++ ++ lockdep_assert_held(&kctx->csf.lock); ++ ++ queue = kctx->csf.user_pages_info[cookie]; ++ ++ /* Looks like the bind has been aborted */ ++ if (!queue) ++ return -EINVAL; ++ ++ if (WARN_ON(test_bit(cookie, kctx->csf.cookies))) ++ return -EINVAL; ++ ++ /* no need for the cookie anymore */ ++ kctx->csf.user_pages_info[cookie] = NULL; ++ bitmap_set(kctx->csf.cookies, cookie, 1); ++ ++ /* Reset the handle to avoid (re)freeing the cookie (which can ++ * now get re-assigned) on unbind. ++ */ ++ queue->handle = BASEP_MEM_INVALID_HANDLE; ++ ++ if (nr_pages != BASEP_QUEUE_NR_MMAP_USER_PAGES) { ++ err = -EINVAL; ++ goto map_failed; ++ } ++ ++ err = kbase_csf_alloc_command_stream_user_pages(kctx, queue); ++ if (err) ++ goto map_failed; ++ ++#if (KERNEL_VERSION(3, 7, 0) <= LINUX_VERSION_CODE) ++ vma->vm_flags |= VM_DONTCOPY | VM_DONTDUMP | VM_DONTEXPAND | VM_IO; ++#else ++ vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_RESERVED | VM_IO; ++#endif ++ /* TODO use VM_MIXEDMAP, since it is more appropriate as both types of ++ * memory with and without "struct page" backing are being inserted here. ++ * Hw Doorbell pages comes from the device register area so kernel does ++ * not use "struct page" for them. ++ */ ++ vma->vm_flags |= VM_PFNMAP; ++ ++ vma->vm_ops = &kbase_csf_user_io_pages_vm_ops; ++ vma->vm_private_data = queue; ++ ++ /* Make vma point to the special internal file, but don't drop the ++ * reference on mali device file (that would be done later when the ++ * vma is closed). ++ */ ++ vma->vm_file = kctx->kbdev->csf.db_filp; ++ get_file(vma->vm_file); ++ /* Also adjust the vm_pgoff */ ++ vma->vm_pgoff = queue->db_file_offset; ++ ++ return 0; ++ ++map_failed: ++ kbase_csf_queue_unbind(queue); ++ ++ return err; ++} ++ ++static void kbase_csf_user_reg_vm_close(struct vm_area_struct *vma) ++{ ++ struct kbase_context *kctx = vma->vm_private_data; ++ ++ WARN_ON(!kctx->csf.user_reg_vma); ++ ++ kctx->csf.user_reg_vma = NULL; ++} ++ ++#if (KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE) ++static vm_fault_t kbase_csf_user_reg_vm_fault(struct vm_area_struct *vma, ++ struct vm_fault *vmf) ++{ ++#else ++static vm_fault_t kbase_csf_user_reg_vm_fault(struct vm_fault *vmf) ++{ ++ struct vm_area_struct *vma = vmf->vma; ++#endif ++ struct kbase_context *kctx = vma->vm_private_data; ++ struct kbase_device *kbdev = kctx->kbdev; ++ unsigned long pfn = PFN_DOWN(kbdev->reg_start + USER_BASE); ++ size_t nr_pages = PFN_DOWN(vma->vm_end - vma->vm_start); ++ ++ /* Few sanity checks up front */ ++ if (WARN_ON(nr_pages != 1) || ++ WARN_ON(vma != kctx->csf.user_reg_vma) || ++ WARN_ON(vma->vm_pgoff != ++ PFN_DOWN(BASEP_MEM_CSF_USER_REG_PAGE_HANDLE))) ++ return VM_FAULT_SIGBUS; ++ ++ /* TODO: check PM state here and don't map in the actual register page ++ * if GPU is powered down or is about to be powered down. ++ */ ++ ++ return vmf_insert_pfn_prot(vma, vma->vm_start, pfn, vma->vm_page_prot); ++} ++ ++static const struct vm_operations_struct kbase_csf_user_reg_vm_ops = { ++ .close = kbase_csf_user_reg_vm_close, ++ .fault = kbase_csf_user_reg_vm_fault ++}; ++ ++static int kbase_csf_cpu_mmap_user_reg_page(struct kbase_context *kctx, ++ struct vm_area_struct *vma) ++{ ++ size_t nr_pages = PFN_DOWN(vma->vm_end - vma->vm_start); ++ ++ /* Few sanity checks */ ++ if (kctx->csf.user_reg_vma) ++ return -EBUSY; ++ ++ if (nr_pages != 1) ++ return -EINVAL; ++ ++ if (vma->vm_flags & (VM_WRITE | VM_MAYWRITE)) ++ return -EPERM; ++ ++ /* Map uncached */ ++ vma->vm_page_prot = pgprot_device(vma->vm_page_prot); ++ ++ vma->vm_flags |= VM_DONTCOPY | VM_DONTDUMP | VM_DONTEXPAND | VM_IO; ++ ++ /* User register page comes from the device register area so ++ * "struct page" isn't available for it. ++ */ ++ vma->vm_flags |= VM_PFNMAP; ++ ++ kctx->csf.user_reg_vma = vma; ++ ++ vma->vm_ops = &kbase_csf_user_reg_vm_ops; ++ vma->vm_private_data = kctx; ++ ++ return 0; ++} ++ ++#endif /* MALI_USE_CSF */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.h b/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.h +new file mode 100755 +index 000000000000..85e030ab751a +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.h +@@ -0,0 +1,478 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010, 2012-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++/** ++ * @file mali_kbase_mem_linux.h ++ * Base kernel memory APIs, Linux implementation. ++ */ ++ ++#ifndef _KBASE_MEM_LINUX_H_ ++#define _KBASE_MEM_LINUX_H_ ++ ++/** A HWC dump mapping */ ++struct kbase_hwc_dma_mapping { ++ void *cpu_va; ++ dma_addr_t dma_pa; ++ size_t size; ++}; ++ ++/** ++ * kbase_mem_alloc - Create a new allocation for GPU ++ * ++ * @kctx: The kernel context ++ * @va_pages: The number of pages of virtual address space to reserve ++ * @commit_pages: The number of physical pages to allocate upfront ++ * @extent: The number of extra pages to allocate on each GPU fault which ++ * grows the region. ++ * @flags: bitmask of BASE_MEM_* flags to convey special requirements & ++ * properties for the new allocation. ++ * @gpu_va: Start address of the memory region which was allocated from GPU ++ * virtual address space. ++ * ++ * Return: 0 on success or error code ++ */ ++struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, ++ u64 va_pages, u64 commit_pages, u64 extent, u64 *flags, ++ u64 *gpu_va); ++ ++/** ++ * kbase_mem_query - Query properties of a GPU memory region ++ * ++ * @kctx: The kernel context ++ * @gpu_addr: A GPU address contained within the memory region ++ * @query: The type of query, from KBASE_MEM_QUERY_* flags, which could be ++ * regarding the amount of backing physical memory allocated so far ++ * for the region or the size of the region or the flags associated ++ * with the region. ++ * @out: Pointer to the location to store the result of query. ++ * ++ * Return: 0 on success or error code ++ */ ++int kbase_mem_query(struct kbase_context *kctx, u64 gpu_addr, u64 query, ++ u64 *const out); ++ ++/** ++ * kbase_mem_import - Import the external memory for use by the GPU ++ * ++ * @kctx: The kernel context ++ * @type: Type of external memory ++ * @phandle: Handle to the external memory interpreted as per the type. ++ * @padding: Amount of extra VA pages to append to the imported buffer ++ * @gpu_va: GPU address assigned to the imported external memory ++ * @va_pages: Size of the memory region reserved from the GPU address space ++ * @flags: bitmask of BASE_MEM_* flags to convey special requirements & ++ * properties for the new allocation representing the external ++ * memory. ++ * Return: 0 on success or error code ++ */ ++int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type, ++ void __user *phandle, u32 padding, u64 *gpu_va, u64 *va_pages, ++ u64 *flags); ++ ++/** ++ * kbase_mem_alias - Create a new allocation for GPU, aliasing one or more ++ * memory regions ++ * ++ * @kctx: The kernel context ++ * @flags: bitmask of BASE_MEM_* flags. ++ * @stride: Bytes between start of each memory region ++ * @nents: The number of regions to pack together into the alias ++ * @ai: Pointer to the struct containing the memory aliasing info ++ * @num_pages: Number of pages the alias will cover ++ * ++ * Return: 0 on failure or otherwise the GPU VA for the alias ++ */ ++u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, u64 nents, struct base_mem_aliasing_info *ai, u64 *num_pages); ++ ++/** ++ * kbase_mem_flags_change - Change the flags for a memory region ++ * ++ * @kctx: The kernel context ++ * @gpu_addr: A GPU address contained within the memory region to modify. ++ * @flags: The new flags to set ++ * @mask: Mask of the flags, from BASE_MEM_*, to modify. ++ * ++ * Return: 0 on success or error code ++ */ ++int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, unsigned int flags, unsigned int mask); ++ ++/** ++ * kbase_mem_commit - Change the physical backing size of a region ++ * ++ * @kctx: The kernel context ++ * @gpu_addr: Handle to the memory region ++ * @new_pages: Number of physical pages to back the region with ++ * ++ * Return: 0 on success or error code ++ */ ++int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages); ++ ++/** ++ * kbase_mem_shrink - Shrink the physical backing size of a region ++ * ++ * @kctx: The kernel context ++ * @reg: The GPU region ++ * @new_pages: Number of physical pages to back the region with ++ * ++ * Return: 0 on success or error code ++ */ ++int kbase_mem_shrink(struct kbase_context *kctx, ++ struct kbase_va_region *reg, u64 new_pages); ++ ++/** ++ * kbase_context_mmap - Memory map method, gets invoked when mmap system call is ++ * issued on device file /dev/malixx. ++ * @kctx: The kernel context ++ * @vma: Pointer to the struct containing the info where the GPU allocation ++ * will be mapped in virtual address space of CPU. ++ * ++ * Return: 0 on success or error code ++ */ ++int kbase_context_mmap(struct kbase_context *kctx, struct vm_area_struct *vma); ++ ++/** ++ * kbase_mem_evictable_init - Initialize the Ephemeral memory eviction ++ * mechanism. ++ * @kctx: The kbase context to initialize. ++ * ++ * Return: Zero on success or -errno on failure. ++ */ ++int kbase_mem_evictable_init(struct kbase_context *kctx); ++ ++/** ++ * kbase_mem_evictable_deinit - De-initialize the Ephemeral memory eviction ++ * mechanism. ++ * @kctx: The kbase context to de-initialize. ++ */ ++void kbase_mem_evictable_deinit(struct kbase_context *kctx); ++ ++/** ++ * kbase_mem_grow_gpu_mapping - Grow the GPU mapping of an allocation ++ * @kctx: Context the region belongs to ++ * @reg: The GPU region ++ * @new_pages: The number of pages after the grow ++ * @old_pages: The number of pages before the grow ++ * ++ * Return: 0 on success, -errno on error. ++ * ++ * Expand the GPU mapping to encompass the new psychical pages which have ++ * been added to the allocation. ++ * ++ * Note: Caller must be holding the region lock. ++ */ ++int kbase_mem_grow_gpu_mapping(struct kbase_context *kctx, ++ struct kbase_va_region *reg, ++ u64 new_pages, u64 old_pages); ++ ++/** ++ * kbase_mem_evictable_make - Make a physical allocation eligible for eviction ++ * @gpu_alloc: The physical allocation to make evictable ++ * ++ * Return: 0 on success, -errno on error. ++ * ++ * Take the provided region and make all the physical pages within it ++ * reclaimable by the kernel, updating the per-process VM stats as well. ++ * Remove any CPU mappings (as these can't be removed in the shrinker callback ++ * as mmap_sem/mmap_lock might already be taken) but leave the GPU mapping ++ * intact as and until the shrinker reclaims the allocation. ++ * ++ * Note: Must be called with the region lock of the containing context. ++ */ ++int kbase_mem_evictable_make(struct kbase_mem_phy_alloc *gpu_alloc); ++ ++/** ++ * kbase_mem_evictable_unmake - Remove a physical allocations eligibility for ++ * eviction. ++ * @alloc: The physical allocation to remove eviction eligibility from. ++ * ++ * Return: True if the allocation had its backing restored and false if ++ * it hasn't. ++ * ++ * Make the physical pages in the region no longer reclaimable and update the ++ * per-process stats, if the shrinker has already evicted the memory then ++ * re-allocate it if the region is still alive. ++ * ++ * Note: Must be called with the region lock of the containing context. ++ */ ++bool kbase_mem_evictable_unmake(struct kbase_mem_phy_alloc *alloc); ++ ++struct kbase_vmap_struct { ++ off_t offset_in_page; ++ struct kbase_mem_phy_alloc *cpu_alloc; ++ struct kbase_mem_phy_alloc *gpu_alloc; ++ struct tagged_addr *cpu_pages; ++ struct tagged_addr *gpu_pages; ++ void *addr; ++ size_t size; ++ bool sync_needed; ++}; ++ ++ ++/** ++ * kbase_vmap_prot - Map a GPU VA range into the kernel safely, only if the ++ * requested access permissions are supported ++ * @kctx: Context the VA range belongs to ++ * @gpu_addr: Start address of VA range ++ * @size: Size of VA range ++ * @prot_request: Flags indicating how the caller will then access the memory ++ * @map: Structure to be given to kbase_vunmap() on freeing ++ * ++ * Return: Kernel-accessible CPU pointer to the VA range, or NULL on error ++ * ++ * Map a GPU VA Range into the kernel. The VA range must be contained within a ++ * GPU memory region. Appropriate CPU cache-flushing operations are made as ++ * required, dependent on the CPU mapping for the memory region. ++ * ++ * This is safer than using kmap() on the pages directly, ++ * because the pages here are refcounted to prevent freeing (and hence reuse ++ * elsewhere in the system) until an kbase_vunmap() ++ * ++ * The flags in @prot_request should use KBASE_REG_{CPU,GPU}_{RD,WR}, to check ++ * whether the region should allow the intended access, and return an error if ++ * disallowed. This is essential for security of imported memory, particularly ++ * a user buf from SHM mapped into the process as RO. In that case, write ++ * access must be checked if the intention is for kernel to write to the ++ * memory. ++ * ++ * The checks are also there to help catch access errors on memory where ++ * security is not a concern: imported memory that is always RW, and memory ++ * that was allocated and owned by the process attached to @kctx. In this case, ++ * it helps to identify memory that was was mapped with the wrong access type. ++ * ++ * Note: KBASE_REG_GPU_{RD,WR} flags are currently supported for legacy cases ++ * where either the security of memory is solely dependent on those flags, or ++ * when userspace code was expecting only the GPU to access the memory (e.g. HW ++ * workarounds). ++ * ++ * All cache maintenance operations shall be ignored if the ++ * memory region has been imported. ++ * ++ */ ++void *kbase_vmap_prot(struct kbase_context *kctx, u64 gpu_addr, size_t size, ++ unsigned long prot_request, struct kbase_vmap_struct *map); ++ ++/** ++ * kbase_vmap - Map a GPU VA range into the kernel safely ++ * @kctx: Context the VA range belongs to ++ * @gpu_addr: Start address of VA range ++ * @size: Size of VA range ++ * @map: Structure to be given to kbase_vunmap() on freeing ++ * ++ * Return: Kernel-accessible CPU pointer to the VA range, or NULL on error ++ * ++ * Map a GPU VA Range into the kernel. The VA range must be contained within a ++ * GPU memory region. Appropriate CPU cache-flushing operations are made as ++ * required, dependent on the CPU mapping for the memory region. ++ * ++ * This is safer than using kmap() on the pages directly, ++ * because the pages here are refcounted to prevent freeing (and hence reuse ++ * elsewhere in the system) until an kbase_vunmap() ++ * ++ * kbase_vmap_prot() should be used in preference, since kbase_vmap() makes no ++ * checks to ensure the security of e.g. imported user bufs from RO SHM. ++ * ++ * Note: All cache maintenance operations shall be ignored if the memory region ++ * has been imported. ++ */ ++void *kbase_vmap(struct kbase_context *kctx, u64 gpu_addr, size_t size, ++ struct kbase_vmap_struct *map); ++ ++/** ++ * kbase_vunmap - Unmap a GPU VA range from the kernel ++ * @kctx: Context the VA range belongs to ++ * @map: Structure describing the mapping from the corresponding kbase_vmap() ++ * call ++ * ++ * Unmaps a GPU VA range from the kernel, given its @map structure obtained ++ * from kbase_vmap(). Appropriate CPU cache-flushing operations are made as ++ * required, dependent on the CPU mapping for the memory region. ++ * ++ * The reference taken on pages during kbase_vmap() is released. ++ * ++ * Note: All cache maintenance operations shall be ignored if the memory region ++ * has been imported. ++ */ ++void kbase_vunmap(struct kbase_context *kctx, struct kbase_vmap_struct *map); ++ ++extern const struct vm_operations_struct kbase_vm_ops; ++ ++/** ++ * kbase_sync_mem_regions - Perform the cache maintenance for the kernel mode ++ * CPU mapping. ++ * @kctx: Context the CPU mapping belongs to. ++ * @map: Structure describing the CPU mapping, setup previously by the ++ * kbase_vmap() call. ++ * @dest: Indicates the type of maintenance required (i.e. flush or invalidate) ++ * ++ * Note: The caller shall ensure that CPU mapping is not revoked & remains ++ * active whilst the maintenance is in progress. ++ */ ++void kbase_sync_mem_regions(struct kbase_context *kctx, ++ struct kbase_vmap_struct *map, enum kbase_sync_type dest); ++ ++/** ++ * kbase_mem_shrink_cpu_mapping - Shrink the CPU mapping(s) of an allocation ++ * @kctx: Context the region belongs to ++ * @reg: The GPU region ++ * @new_pages: The number of pages after the shrink ++ * @old_pages: The number of pages before the shrink ++ * ++ * Shrink (or completely remove) all CPU mappings which reference the shrunk ++ * part of the allocation. ++ */ ++void kbase_mem_shrink_cpu_mapping(struct kbase_context *kctx, ++ struct kbase_va_region *reg, ++ u64 new_pages, u64 old_pages); ++ ++/** ++ * kbase_phy_alloc_mapping_term - Terminate the kernel side mapping of a ++ * physical allocation ++ * @kctx: The kernel base context associated with the mapping ++ * @alloc: Pointer to the allocation to terminate ++ * ++ * This function will unmap the kernel mapping, and free any structures used to ++ * track it. ++ */ ++void kbase_phy_alloc_mapping_term(struct kbase_context *kctx, ++ struct kbase_mem_phy_alloc *alloc); ++ ++/** ++ * kbase_phy_alloc_mapping_get - Get a kernel-side CPU pointer to the permanent ++ * mapping of a physical allocation ++ * @kctx: The kernel base context @gpu_addr will be looked up in ++ * @gpu_addr: The gpu address to lookup for the kernel-side CPU mapping ++ * @out_kern_mapping: Pointer to storage for a struct kbase_vmap_struct pointer ++ * which will be used for a call to ++ * kbase_phy_alloc_mapping_put() ++ * ++ * Return: Pointer to a kernel-side accessible location that directly ++ * corresponds to @gpu_addr, or NULL on failure ++ * ++ * Looks up @gpu_addr to retrieve the CPU pointer that can be used to access ++ * that location kernel-side. Only certain kinds of memory have a permanent ++ * kernel mapping, refer to the internal functions ++ * kbase_reg_needs_kernel_mapping() and kbase_phy_alloc_mapping_init() for more ++ * information. ++ * ++ * If this function succeeds, a CPU access to the returned pointer will access ++ * the actual location represented by @gpu_addr. That is, the return value does ++ * not require any offset added to it to access the location specified in ++ * @gpu_addr ++ * ++ * The client must take care to either apply any necessary sync operations when ++ * accessing the data, or ensure that the enclosing region was coherent with ++ * the GPU, or uncached in the CPU. ++ * ++ * The refcount on the physical allocations backing the region are taken, so ++ * that they do not disappear whilst the client is accessing it. Once the ++ * client has finished accessing the memory, it must be released with a call to ++ * kbase_phy_alloc_mapping_put() ++ * ++ * Whilst this is expected to execute quickly (the mapping was already setup ++ * when the physical allocation was created), the call is not IRQ-safe due to ++ * the region lookup involved. ++ * ++ * An error code may indicate that: ++ * - a userside process has freed the allocation, and so @gpu_addr is no longer ++ * valid ++ * - the region containing @gpu_addr does not support a permanent kernel mapping ++ */ ++void *kbase_phy_alloc_mapping_get(struct kbase_context *kctx, u64 gpu_addr, ++ struct kbase_vmap_struct **out_kern_mapping); ++ ++/** ++ * kbase_phy_alloc_mapping_put - Put a reference to the kernel-side mapping of a ++ * physical allocation ++ * @kctx: The kernel base context associated with the mapping ++ * @kern_mapping: Pointer to a struct kbase_phy_alloc_mapping pointer obtained ++ * from a call to kbase_phy_alloc_mapping_get() ++ * ++ * Releases the reference to the allocations backing @kern_mapping that was ++ * obtained through a call to kbase_phy_alloc_mapping_get(). This must be used ++ * when the client no longer needs to access the kernel-side CPU pointer. ++ * ++ * If this was the last reference on the underlying physical allocations, they ++ * will go through the normal allocation free steps, which also includes an ++ * unmap of the permanent kernel mapping for those allocations. ++ * ++ * Due to these operations, the function is not IRQ-safe. However it is ++ * expected to execute quickly in the normal case, i.e. when the region holding ++ * the physical allocation is still present. ++ */ ++void kbase_phy_alloc_mapping_put(struct kbase_context *kctx, ++ struct kbase_vmap_struct *kern_mapping); ++ ++/** ++ * kbase_get_cache_line_alignment - Return cache line alignment ++ * ++ * Helper function to return the maximum cache line alignment considering ++ * both CPU and GPU cache sizes. ++ * ++ * Return: CPU and GPU cache line alignment, in bytes. ++ * ++ * @kbdev: Device pointer. ++ */ ++u32 kbase_get_cache_line_alignment(struct kbase_device *kbdev); ++ ++#if (KERNEL_VERSION(4, 20, 0) > LINUX_VERSION_CODE) ++static inline vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma, ++ unsigned long addr, unsigned long pfn, pgprot_t pgprot) ++{ ++ int err; ++ ++#if ((KERNEL_VERSION(4, 4, 147) >= LINUX_VERSION_CODE) || \ ++ ((KERNEL_VERSION(4, 6, 0) > LINUX_VERSION_CODE) && \ ++ (KERNEL_VERSION(4, 5, 0) <= LINUX_VERSION_CODE))) ++ if (pgprot_val(pgprot) != pgprot_val(vma->vm_page_prot)) ++ return VM_FAULT_SIGBUS; ++ ++ err = vm_insert_pfn(vma, addr, pfn); ++#else ++ err = vm_insert_pfn_prot(vma, addr, pfn, pgprot); ++#endif ++ ++ if (unlikely(err == -ENOMEM)) ++ return VM_FAULT_OOM; ++ if (unlikely(err < 0 && err != -EBUSY)) ++ return VM_FAULT_SIGBUS; ++ ++ return VM_FAULT_NOPAGE; ++} ++#endif ++ ++/** ++ * kbase_mem_get_process_mmap_lock - Return the mmap lock for the current process ++ * ++ * Return: the mmap lock for the current process ++ */ ++static inline struct rw_semaphore *kbase_mem_get_process_mmap_lock(void) ++{ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0) ++ return ¤t->mm->mmap_sem; ++#else /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 8, 0) */ ++ return ¤t->mm->mmap_lock; ++#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0) */ ++} ++ ++#endif /* _KBASE_MEM_LINUX_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem_lowlevel.h b/drivers/gpu/arm/bifrost/mali_kbase_mem_lowlevel.h +new file mode 100755 +index 000000000000..70116030f233 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_mem_lowlevel.h +@@ -0,0 +1,166 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2014,2018 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++#ifndef _KBASE_MEM_LOWLEVEL_H ++#define _KBASE_MEM_LOWLEVEL_H ++ ++#ifndef _KBASE_H_ ++#error "Don't include this file directly, use mali_kbase.h instead" ++#endif ++ ++#include ++ ++/** ++ * @brief Flags for kbase_phy_allocator_pages_alloc ++ */ ++#define KBASE_PHY_PAGES_FLAG_DEFAULT (0) /** Default allocation flag */ ++#define KBASE_PHY_PAGES_FLAG_CLEAR (1 << 0) /** Clear the pages after allocation */ ++#define KBASE_PHY_PAGES_FLAG_POISON (1 << 1) /** Fill the memory with a poison value */ ++ ++#define KBASE_PHY_PAGES_SUPPORTED_FLAGS (KBASE_PHY_PAGES_FLAG_DEFAULT|KBASE_PHY_PAGES_FLAG_CLEAR|KBASE_PHY_PAGES_FLAG_POISON) ++ ++#define KBASE_PHY_PAGES_POISON_VALUE 0xFD /** Value to fill the memory with when KBASE_PHY_PAGES_FLAG_POISON is set */ ++ ++enum kbase_sync_type { ++ KBASE_SYNC_TO_CPU, ++ KBASE_SYNC_TO_DEVICE ++}; ++ ++struct tagged_addr { phys_addr_t tagged_addr; }; ++ ++#define HUGE_PAGE (1u << 0) ++#define HUGE_HEAD (1u << 1) ++#define FROM_PARTIAL (1u << 2) ++ ++/* ++ * Note: if macro for converting physical address to page is not defined ++ * in the kernel itself, it is defined hereby. This is to avoid build errors ++ * which are reported during builds for some architectures. ++ */ ++#ifndef phys_to_page ++#define phys_to_page(phys) (pfn_to_page((phys) >> PAGE_SHIFT)) ++#endif ++ ++/** ++ * as_phys_addr_t - Retrieve the physical address from tagged address by ++ * masking the lower order 12 bits. ++ * @t: tagged address to be translated. ++ * ++ * Return: physical address corresponding to tagged address. ++ */ ++static inline phys_addr_t as_phys_addr_t(struct tagged_addr t) ++{ ++ return t.tagged_addr & PAGE_MASK; ++} ++ ++/** ++ * as_page - Retrieve the struct page from a tagged address ++ * @t: tagged address to be translated. ++ * ++ * Return: pointer to struct page corresponding to tagged address. ++ */ ++static inline struct page *as_page(struct tagged_addr t) ++{ ++ return phys_to_page(as_phys_addr_t(t)); ++} ++ ++/** ++ * as_tagged - Convert the physical address to tagged address type though ++ * there is no tag info present, the lower order 12 bits will be 0 ++ * @phys: physical address to be converted to tagged type ++ * ++ * This is used for 4KB physical pages allocated by the Driver or imported pages ++ * and is needed as physical pages tracking object stores the reference for ++ * physical pages using tagged address type in lieu of the type generally used ++ * for physical addresses. ++ * ++ * Return: address of tagged address type. ++ */ ++static inline struct tagged_addr as_tagged(phys_addr_t phys) ++{ ++ struct tagged_addr t; ++ ++ t.tagged_addr = phys & PAGE_MASK; ++ return t; ++} ++ ++/** ++ * as_tagged_tag - Form the tagged address by storing the tag or metadata in the ++ * lower order 12 bits of physial address ++ * @phys: physical address to be converted to tagged address ++ * @tag: tag to be stored along with the physical address. ++ * ++ * The tag info is used while freeing up the pages ++ * ++ * Return: tagged address storing physical address & tag. ++ */ ++static inline struct tagged_addr as_tagged_tag(phys_addr_t phys, int tag) ++{ ++ struct tagged_addr t; ++ ++ t.tagged_addr = (phys & PAGE_MASK) | (tag & ~PAGE_MASK); ++ return t; ++} ++ ++/** ++ * is_huge - Check if the physical page is one of the 512 4KB pages of the ++ * large page which was not split to be used partially ++ * @t: tagged address storing the tag in the lower order bits. ++ * ++ * Return: true if page belongs to large page, or false ++ */ ++static inline bool is_huge(struct tagged_addr t) ++{ ++ return t.tagged_addr & HUGE_PAGE; ++} ++ ++/** ++ * is_huge_head - Check if the physical page is the first 4KB page of the ++ * 512 4KB pages within a large page which was not split ++ * to be used partially ++ * @t: tagged address storing the tag in the lower order bits. ++ * ++ * Return: true if page is the first page of a large page, or false ++ */ ++static inline bool is_huge_head(struct tagged_addr t) ++{ ++ int mask = HUGE_HEAD | HUGE_PAGE; ++ ++ return mask == (t.tagged_addr & mask); ++} ++ ++/** ++ * is_partial - Check if the physical page is one of the 512 pages of the ++ * large page which was split in 4KB pages to be used ++ * partially for allocations >= 2 MB in size. ++ * @t: tagged address storing the tag in the lower order bits. ++ * ++ * Return: true if page was taken from large page used partially, or false ++ */ ++static inline bool is_partial(struct tagged_addr t) ++{ ++ return t.tagged_addr & FROM_PARTIAL; ++} ++ ++#endif /* _KBASE_LOWLEVEL_H */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem_pool.c b/drivers/gpu/arm/bifrost/mali_kbase_mem_pool.c +new file mode 100755 +index 000000000000..0723e32e2003 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_mem_pool.c +@@ -0,0 +1,856 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015-2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#define pool_dbg(pool, format, ...) \ ++ dev_dbg(pool->kbdev->dev, "%s-pool [%zu/%zu]: " format, \ ++ (pool->next_pool) ? "kctx" : "kbdev", \ ++ kbase_mem_pool_size(pool), \ ++ kbase_mem_pool_max_size(pool), \ ++ ##__VA_ARGS__) ++ ++#define NOT_DIRTY false ++#define NOT_RECLAIMED false ++ ++static size_t kbase_mem_pool_capacity(struct kbase_mem_pool *pool) ++{ ++ ssize_t max_size = kbase_mem_pool_max_size(pool); ++ ssize_t cur_size = kbase_mem_pool_size(pool); ++ ++ return max(max_size - cur_size, (ssize_t)0); ++} ++ ++static bool kbase_mem_pool_is_full(struct kbase_mem_pool *pool) ++{ ++ return kbase_mem_pool_size(pool) >= kbase_mem_pool_max_size(pool); ++} ++ ++static bool kbase_mem_pool_is_empty(struct kbase_mem_pool *pool) ++{ ++ return kbase_mem_pool_size(pool) == 0; ++} ++ ++static void kbase_mem_pool_add_locked(struct kbase_mem_pool *pool, ++ struct page *p) ++{ ++ lockdep_assert_held(&pool->pool_lock); ++ ++ list_add(&p->lru, &pool->page_list); ++ pool->cur_size++; ++ ++ pool_dbg(pool, "added page\n"); ++} ++ ++static void kbase_mem_pool_add(struct kbase_mem_pool *pool, struct page *p) ++{ ++ kbase_mem_pool_lock(pool); ++ kbase_mem_pool_add_locked(pool, p); ++ kbase_mem_pool_unlock(pool); ++} ++ ++static void kbase_mem_pool_add_list_locked(struct kbase_mem_pool *pool, ++ struct list_head *page_list, size_t nr_pages) ++{ ++ lockdep_assert_held(&pool->pool_lock); ++ ++ list_splice(page_list, &pool->page_list); ++ pool->cur_size += nr_pages; ++ ++ pool_dbg(pool, "added %zu pages\n", nr_pages); ++} ++ ++static void kbase_mem_pool_add_list(struct kbase_mem_pool *pool, ++ struct list_head *page_list, size_t nr_pages) ++{ ++ kbase_mem_pool_lock(pool); ++ kbase_mem_pool_add_list_locked(pool, page_list, nr_pages); ++ kbase_mem_pool_unlock(pool); ++} ++ ++static struct page *kbase_mem_pool_remove_locked(struct kbase_mem_pool *pool) ++{ ++ struct page *p; ++ ++ lockdep_assert_held(&pool->pool_lock); ++ ++ if (kbase_mem_pool_is_empty(pool)) ++ return NULL; ++ ++ p = list_first_entry(&pool->page_list, struct page, lru); ++ list_del_init(&p->lru); ++ pool->cur_size--; ++ ++ pool_dbg(pool, "removed page\n"); ++ ++ return p; ++} ++ ++static struct page *kbase_mem_pool_remove(struct kbase_mem_pool *pool) ++{ ++ struct page *p; ++ ++ kbase_mem_pool_lock(pool); ++ p = kbase_mem_pool_remove_locked(pool); ++ kbase_mem_pool_unlock(pool); ++ ++ return p; ++} ++ ++static void kbase_mem_pool_sync_page(struct kbase_mem_pool *pool, ++ struct page *p) ++{ ++ struct device *dev = pool->kbdev->dev; ++ dma_sync_single_for_device(dev, kbase_dma_addr(p), ++ (PAGE_SIZE << pool->order), DMA_BIDIRECTIONAL); ++} ++ ++static void kbase_mem_pool_zero_page(struct kbase_mem_pool *pool, ++ struct page *p) ++{ ++ int i; ++ ++ for (i = 0; i < (1U << pool->order); i++) ++ clear_highpage(p+i); ++ ++ kbase_mem_pool_sync_page(pool, p); ++} ++ ++static void kbase_mem_pool_spill(struct kbase_mem_pool *next_pool, ++ struct page *p) ++{ ++ /* Zero page before spilling */ ++ kbase_mem_pool_zero_page(next_pool, p); ++ ++ kbase_mem_pool_add(next_pool, p); ++} ++ ++struct page *kbase_mem_alloc_page(struct kbase_mem_pool *pool) ++{ ++ struct page *p; ++ gfp_t gfp; ++ struct kbase_device *const kbdev = pool->kbdev; ++ struct device *const dev = kbdev->dev; ++ dma_addr_t dma_addr; ++ int i; ++ ++#if defined(CONFIG_ARM) && !defined(CONFIG_HAVE_DMA_ATTRS) && \ ++ LINUX_VERSION_CODE < KERNEL_VERSION(3, 5, 0) ++ /* DMA cache sync fails for HIGHMEM before 3.5 on ARM */ ++ gfp = GFP_USER | __GFP_ZERO; ++#else ++ gfp = GFP_HIGHUSER | __GFP_ZERO; ++#endif ++ ++ /* don't warn on higher order failures */ ++ if (pool->order) ++ gfp |= __GFP_NOWARN; ++ ++ p = kbdev->mgm_dev->ops.mgm_alloc_page(kbdev->mgm_dev, ++ pool->group_id, gfp, pool->order); ++ if (!p) ++ return NULL; ++ ++ dma_addr = dma_map_page(dev, p, 0, (PAGE_SIZE << pool->order), ++ DMA_BIDIRECTIONAL); ++ ++ if (dma_mapping_error(dev, dma_addr)) { ++ kbdev->mgm_dev->ops.mgm_free_page(kbdev->mgm_dev, ++ pool->group_id, p, pool->order); ++ return NULL; ++ } ++ ++ WARN_ON(dma_addr != page_to_phys(p)); ++ for (i = 0; i < (1u << pool->order); i++) ++ kbase_set_dma_addr(p+i, dma_addr + PAGE_SIZE * i); ++ ++ return p; ++} ++ ++static void kbase_mem_pool_free_page(struct kbase_mem_pool *pool, ++ struct page *p) ++{ ++ struct kbase_device *const kbdev = pool->kbdev; ++ struct device *const dev = kbdev->dev; ++ dma_addr_t dma_addr = kbase_dma_addr(p); ++ int i; ++ ++ dma_unmap_page(dev, dma_addr, (PAGE_SIZE << pool->order), ++ DMA_BIDIRECTIONAL); ++ for (i = 0; i < (1u << pool->order); i++) ++ kbase_clear_dma_addr(p+i); ++ ++ kbdev->mgm_dev->ops.mgm_free_page(kbdev->mgm_dev, ++ pool->group_id, p, pool->order); ++ ++ pool_dbg(pool, "freed page to kernel\n"); ++} ++ ++static size_t kbase_mem_pool_shrink_locked(struct kbase_mem_pool *pool, ++ size_t nr_to_shrink) ++{ ++ struct page *p; ++ size_t i; ++ ++ lockdep_assert_held(&pool->pool_lock); ++ ++ for (i = 0; i < nr_to_shrink && !kbase_mem_pool_is_empty(pool); i++) { ++ p = kbase_mem_pool_remove_locked(pool); ++ kbase_mem_pool_free_page(pool, p); ++ } ++ ++ return i; ++} ++ ++static size_t kbase_mem_pool_shrink(struct kbase_mem_pool *pool, ++ size_t nr_to_shrink) ++{ ++ size_t nr_freed; ++ ++ kbase_mem_pool_lock(pool); ++ nr_freed = kbase_mem_pool_shrink_locked(pool, nr_to_shrink); ++ kbase_mem_pool_unlock(pool); ++ ++ return nr_freed; ++} ++ ++int kbase_mem_pool_grow(struct kbase_mem_pool *pool, ++ size_t nr_to_grow) ++{ ++ struct page *p; ++ size_t i; ++ ++ kbase_mem_pool_lock(pool); ++ ++ pool->dont_reclaim = true; ++ for (i = 0; i < nr_to_grow; i++) { ++ if (pool->dying) { ++ pool->dont_reclaim = false; ++ kbase_mem_pool_shrink_locked(pool, nr_to_grow); ++ kbase_mem_pool_unlock(pool); ++ ++ return -ENOMEM; ++ } ++ kbase_mem_pool_unlock(pool); ++ ++ p = kbase_mem_alloc_page(pool); ++ if (!p) { ++ kbase_mem_pool_lock(pool); ++ pool->dont_reclaim = false; ++ kbase_mem_pool_unlock(pool); ++ ++ return -ENOMEM; ++ } ++ ++ kbase_mem_pool_lock(pool); ++ kbase_mem_pool_add_locked(pool, p); ++ } ++ pool->dont_reclaim = false; ++ kbase_mem_pool_unlock(pool); ++ ++ return 0; ++} ++ ++void kbase_mem_pool_trim(struct kbase_mem_pool *pool, size_t new_size) ++{ ++ size_t cur_size; ++ int err = 0; ++ ++ cur_size = kbase_mem_pool_size(pool); ++ ++ if (new_size > pool->max_size) ++ new_size = pool->max_size; ++ ++ if (new_size < cur_size) ++ kbase_mem_pool_shrink(pool, cur_size - new_size); ++ else if (new_size > cur_size) ++ err = kbase_mem_pool_grow(pool, new_size - cur_size); ++ ++ if (err) { ++ size_t grown_size = kbase_mem_pool_size(pool); ++ ++ dev_warn(pool->kbdev->dev, ++ "Mem pool not grown to the required size of %zu bytes, grown for additional %zu bytes instead!\n", ++ (new_size - cur_size), (grown_size - cur_size)); ++ } ++} ++ ++void kbase_mem_pool_set_max_size(struct kbase_mem_pool *pool, size_t max_size) ++{ ++ size_t cur_size; ++ size_t nr_to_shrink; ++ ++ kbase_mem_pool_lock(pool); ++ ++ pool->max_size = max_size; ++ ++ cur_size = kbase_mem_pool_size(pool); ++ if (max_size < cur_size) { ++ nr_to_shrink = cur_size - max_size; ++ kbase_mem_pool_shrink_locked(pool, nr_to_shrink); ++ } ++ ++ kbase_mem_pool_unlock(pool); ++} ++ ++ ++static unsigned long kbase_mem_pool_reclaim_count_objects(struct shrinker *s, ++ struct shrink_control *sc) ++{ ++ struct kbase_mem_pool *pool; ++ size_t pool_size; ++ ++ pool = container_of(s, struct kbase_mem_pool, reclaim); ++ ++ kbase_mem_pool_lock(pool); ++ if (pool->dont_reclaim && !pool->dying) { ++ kbase_mem_pool_unlock(pool); ++ return 0; ++ } ++ pool_size = kbase_mem_pool_size(pool); ++ kbase_mem_pool_unlock(pool); ++ ++ return pool_size; ++} ++ ++static unsigned long kbase_mem_pool_reclaim_scan_objects(struct shrinker *s, ++ struct shrink_control *sc) ++{ ++ struct kbase_mem_pool *pool; ++ unsigned long freed; ++ ++ pool = container_of(s, struct kbase_mem_pool, reclaim); ++ ++ kbase_mem_pool_lock(pool); ++ if (pool->dont_reclaim && !pool->dying) { ++ kbase_mem_pool_unlock(pool); ++ return 0; ++ } ++ ++ pool_dbg(pool, "reclaim scan %ld:\n", sc->nr_to_scan); ++ ++ freed = kbase_mem_pool_shrink_locked(pool, sc->nr_to_scan); ++ ++ kbase_mem_pool_unlock(pool); ++ ++ pool_dbg(pool, "reclaim freed %ld pages\n", freed); ++ ++ return freed; ++} ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) ++static int kbase_mem_pool_reclaim_shrink(struct shrinker *s, ++ struct shrink_control *sc) ++{ ++ if (sc->nr_to_scan == 0) ++ return kbase_mem_pool_reclaim_count_objects(s, sc); ++ ++ return kbase_mem_pool_reclaim_scan_objects(s, sc); ++} ++#endif ++ ++int kbase_mem_pool_init(struct kbase_mem_pool *pool, ++ const struct kbase_mem_pool_config *config, ++ unsigned int order, ++ int group_id, ++ struct kbase_device *kbdev, ++ struct kbase_mem_pool *next_pool) ++{ ++ if (WARN_ON(group_id < 0) || ++ WARN_ON(group_id >= MEMORY_GROUP_MANAGER_NR_GROUPS)) { ++ return -EINVAL; ++ } ++ ++ pool->cur_size = 0; ++ pool->max_size = kbase_mem_pool_config_get_max_size(config); ++ pool->order = order; ++ pool->group_id = group_id; ++ pool->kbdev = kbdev; ++ pool->next_pool = next_pool; ++ pool->dying = false; ++ ++ spin_lock_init(&pool->pool_lock); ++ INIT_LIST_HEAD(&pool->page_list); ++ ++ /* Register shrinker */ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) ++ pool->reclaim.shrink = kbase_mem_pool_reclaim_shrink; ++#else ++ pool->reclaim.count_objects = kbase_mem_pool_reclaim_count_objects; ++ pool->reclaim.scan_objects = kbase_mem_pool_reclaim_scan_objects; ++#endif ++ pool->reclaim.seeks = DEFAULT_SEEKS; ++ /* Kernel versions prior to 3.1 : ++ * struct shrinker does not define batch */ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 1, 0) ++ pool->reclaim.batch = 0; ++#endif ++ register_shrinker(&pool->reclaim); ++ ++ pool_dbg(pool, "initialized\n"); ++ ++ return 0; ++} ++ ++void kbase_mem_pool_mark_dying(struct kbase_mem_pool *pool) ++{ ++ kbase_mem_pool_lock(pool); ++ pool->dying = true; ++ kbase_mem_pool_unlock(pool); ++} ++ ++void kbase_mem_pool_term(struct kbase_mem_pool *pool) ++{ ++ struct kbase_mem_pool *next_pool = pool->next_pool; ++ struct page *p, *tmp; ++ size_t nr_to_spill = 0; ++ LIST_HEAD(spill_list); ++ LIST_HEAD(free_list); ++ int i; ++ ++ pool_dbg(pool, "terminate()\n"); ++ ++ unregister_shrinker(&pool->reclaim); ++ ++ kbase_mem_pool_lock(pool); ++ pool->max_size = 0; ++ ++ if (next_pool && !kbase_mem_pool_is_full(next_pool)) { ++ /* Spill to next pool (may overspill) */ ++ nr_to_spill = kbase_mem_pool_capacity(next_pool); ++ nr_to_spill = min(kbase_mem_pool_size(pool), nr_to_spill); ++ ++ /* Zero pages first without holding the next_pool lock */ ++ for (i = 0; i < nr_to_spill; i++) { ++ p = kbase_mem_pool_remove_locked(pool); ++ list_add(&p->lru, &spill_list); ++ } ++ } ++ ++ while (!kbase_mem_pool_is_empty(pool)) { ++ /* Free remaining pages to kernel */ ++ p = kbase_mem_pool_remove_locked(pool); ++ list_add(&p->lru, &free_list); ++ } ++ ++ kbase_mem_pool_unlock(pool); ++ ++ if (next_pool && nr_to_spill) { ++ list_for_each_entry(p, &spill_list, lru) ++ kbase_mem_pool_zero_page(pool, p); ++ ++ /* Add new page list to next_pool */ ++ kbase_mem_pool_add_list(next_pool, &spill_list, nr_to_spill); ++ ++ pool_dbg(pool, "terminate() spilled %zu pages\n", nr_to_spill); ++ } ++ ++ list_for_each_entry_safe(p, tmp, &free_list, lru) { ++ list_del_init(&p->lru); ++ kbase_mem_pool_free_page(pool, p); ++ } ++ ++ pool_dbg(pool, "terminated\n"); ++} ++ ++struct page *kbase_mem_pool_alloc(struct kbase_mem_pool *pool) ++{ ++ struct page *p; ++ ++ do { ++ pool_dbg(pool, "alloc()\n"); ++ p = kbase_mem_pool_remove(pool); ++ ++ if (p) ++ return p; ++ ++ pool = pool->next_pool; ++ } while (pool); ++ ++ return NULL; ++} ++ ++struct page *kbase_mem_pool_alloc_locked(struct kbase_mem_pool *pool) ++{ ++ struct page *p; ++ ++ lockdep_assert_held(&pool->pool_lock); ++ ++ pool_dbg(pool, "alloc_locked()\n"); ++ p = kbase_mem_pool_remove_locked(pool); ++ ++ if (p) ++ return p; ++ ++ return NULL; ++} ++ ++void kbase_mem_pool_free(struct kbase_mem_pool *pool, struct page *p, ++ bool dirty) ++{ ++ struct kbase_mem_pool *next_pool = pool->next_pool; ++ ++ pool_dbg(pool, "free()\n"); ++ ++ if (!kbase_mem_pool_is_full(pool)) { ++ /* Add to our own pool */ ++ if (dirty) ++ kbase_mem_pool_sync_page(pool, p); ++ ++ kbase_mem_pool_add(pool, p); ++ } else if (next_pool && !kbase_mem_pool_is_full(next_pool)) { ++ /* Spill to next pool */ ++ kbase_mem_pool_spill(next_pool, p); ++ } else { ++ /* Free page */ ++ kbase_mem_pool_free_page(pool, p); ++ } ++} ++ ++void kbase_mem_pool_free_locked(struct kbase_mem_pool *pool, struct page *p, ++ bool dirty) ++{ ++ pool_dbg(pool, "free_locked()\n"); ++ ++ lockdep_assert_held(&pool->pool_lock); ++ ++ if (!kbase_mem_pool_is_full(pool)) { ++ /* Add to our own pool */ ++ if (dirty) ++ kbase_mem_pool_sync_page(pool, p); ++ ++ kbase_mem_pool_add_locked(pool, p); ++ } else { ++ /* Free page */ ++ kbase_mem_pool_free_page(pool, p); ++ } ++} ++ ++int kbase_mem_pool_alloc_pages(struct kbase_mem_pool *pool, size_t nr_4k_pages, ++ struct tagged_addr *pages, bool partial_allowed) ++{ ++ struct page *p; ++ size_t nr_from_pool; ++ size_t i = 0; ++ int err = -ENOMEM; ++ size_t nr_pages_internal; ++ ++ nr_pages_internal = nr_4k_pages / (1u << (pool->order)); ++ ++ if (nr_pages_internal * (1u << pool->order) != nr_4k_pages) ++ return -EINVAL; ++ ++ pool_dbg(pool, "alloc_pages(4k=%zu):\n", nr_4k_pages); ++ pool_dbg(pool, "alloc_pages(internal=%zu):\n", nr_pages_internal); ++ ++ /* Get pages from this pool */ ++ kbase_mem_pool_lock(pool); ++ nr_from_pool = min(nr_pages_internal, kbase_mem_pool_size(pool)); ++ while (nr_from_pool--) { ++ int j; ++ p = kbase_mem_pool_remove_locked(pool); ++ if (pool->order) { ++ pages[i++] = as_tagged_tag(page_to_phys(p), ++ HUGE_HEAD | HUGE_PAGE); ++ for (j = 1; j < (1u << pool->order); j++) ++ pages[i++] = as_tagged_tag(page_to_phys(p) + ++ PAGE_SIZE * j, ++ HUGE_PAGE); ++ } else { ++ pages[i++] = as_tagged(page_to_phys(p)); ++ } ++ } ++ kbase_mem_pool_unlock(pool); ++ ++ if (i != nr_4k_pages && pool->next_pool) { ++ /* Allocate via next pool */ ++ err = kbase_mem_pool_alloc_pages(pool->next_pool, ++ nr_4k_pages - i, pages + i, partial_allowed); ++ ++ if (err < 0) ++ goto err_rollback; ++ ++ i += err; ++ } else { ++ /* Get any remaining pages from kernel */ ++ while (i != nr_4k_pages) { ++ p = kbase_mem_alloc_page(pool); ++ if (!p) { ++ if (partial_allowed) ++ goto done; ++ else ++ goto err_rollback; ++ } ++ ++ if (pool->order) { ++ int j; ++ ++ pages[i++] = as_tagged_tag(page_to_phys(p), ++ HUGE_PAGE | ++ HUGE_HEAD); ++ for (j = 1; j < (1u << pool->order); j++) { ++ phys_addr_t phys; ++ ++ phys = page_to_phys(p) + PAGE_SIZE * j; ++ pages[i++] = as_tagged_tag(phys, ++ HUGE_PAGE); ++ } ++ } else { ++ pages[i++] = as_tagged(page_to_phys(p)); ++ } ++ } ++ } ++ ++done: ++ pool_dbg(pool, "alloc_pages(%zu) done\n", i); ++ return i; ++ ++err_rollback: ++ kbase_mem_pool_free_pages(pool, i, pages, NOT_DIRTY, NOT_RECLAIMED); ++ return err; ++} ++ ++int kbase_mem_pool_alloc_pages_locked(struct kbase_mem_pool *pool, ++ size_t nr_4k_pages, struct tagged_addr *pages) ++{ ++ struct page *p; ++ size_t i; ++ size_t nr_pages_internal; ++ ++ lockdep_assert_held(&pool->pool_lock); ++ ++ nr_pages_internal = nr_4k_pages / (1u << (pool->order)); ++ ++ if (nr_pages_internal * (1u << pool->order) != nr_4k_pages) ++ return -EINVAL; ++ ++ pool_dbg(pool, "alloc_pages_locked(4k=%zu):\n", nr_4k_pages); ++ pool_dbg(pool, "alloc_pages_locked(internal=%zu):\n", ++ nr_pages_internal); ++ ++ if (kbase_mem_pool_size(pool) < nr_pages_internal) { ++ pool_dbg(pool, "Failed alloc\n"); ++ return -ENOMEM; ++ } ++ ++ for (i = 0; i < nr_pages_internal; i++) { ++ int j; ++ ++ p = kbase_mem_pool_remove_locked(pool); ++ if (pool->order) { ++ *pages++ = as_tagged_tag(page_to_phys(p), ++ HUGE_HEAD | HUGE_PAGE); ++ for (j = 1; j < (1u << pool->order); j++) { ++ *pages++ = as_tagged_tag(page_to_phys(p) + ++ PAGE_SIZE * j, ++ HUGE_PAGE); ++ } ++ } else { ++ *pages++ = as_tagged(page_to_phys(p)); ++ } ++ } ++ ++ return nr_4k_pages; ++} ++ ++static void kbase_mem_pool_add_array(struct kbase_mem_pool *pool, ++ size_t nr_pages, struct tagged_addr *pages, ++ bool zero, bool sync) ++{ ++ struct page *p; ++ size_t nr_to_pool = 0; ++ LIST_HEAD(new_page_list); ++ size_t i; ++ ++ if (!nr_pages) ++ return; ++ ++ pool_dbg(pool, "add_array(%zu, zero=%d, sync=%d):\n", ++ nr_pages, zero, sync); ++ ++ /* Zero/sync pages first without holding the pool lock */ ++ for (i = 0; i < nr_pages; i++) { ++ if (unlikely(!as_phys_addr_t(pages[i]))) ++ continue; ++ ++ if (is_huge_head(pages[i]) || !is_huge(pages[i])) { ++ p = as_page(pages[i]); ++ if (zero) ++ kbase_mem_pool_zero_page(pool, p); ++ else if (sync) ++ kbase_mem_pool_sync_page(pool, p); ++ ++ list_add(&p->lru, &new_page_list); ++ nr_to_pool++; ++ } ++ pages[i] = as_tagged(0); ++ } ++ ++ /* Add new page list to pool */ ++ kbase_mem_pool_add_list(pool, &new_page_list, nr_to_pool); ++ ++ pool_dbg(pool, "add_array(%zu) added %zu pages\n", ++ nr_pages, nr_to_pool); ++} ++ ++static void kbase_mem_pool_add_array_locked(struct kbase_mem_pool *pool, ++ size_t nr_pages, struct tagged_addr *pages, ++ bool zero, bool sync) ++{ ++ struct page *p; ++ size_t nr_to_pool = 0; ++ LIST_HEAD(new_page_list); ++ size_t i; ++ ++ lockdep_assert_held(&pool->pool_lock); ++ ++ if (!nr_pages) ++ return; ++ ++ pool_dbg(pool, "add_array_locked(%zu, zero=%d, sync=%d):\n", ++ nr_pages, zero, sync); ++ ++ /* Zero/sync pages first */ ++ for (i = 0; i < nr_pages; i++) { ++ if (unlikely(!as_phys_addr_t(pages[i]))) ++ continue; ++ ++ if (is_huge_head(pages[i]) || !is_huge(pages[i])) { ++ p = as_page(pages[i]); ++ if (zero) ++ kbase_mem_pool_zero_page(pool, p); ++ else if (sync) ++ kbase_mem_pool_sync_page(pool, p); ++ ++ list_add(&p->lru, &new_page_list); ++ nr_to_pool++; ++ } ++ pages[i] = as_tagged(0); ++ } ++ ++ /* Add new page list to pool */ ++ kbase_mem_pool_add_list_locked(pool, &new_page_list, nr_to_pool); ++ ++ pool_dbg(pool, "add_array_locked(%zu) added %zu pages\n", ++ nr_pages, nr_to_pool); ++} ++ ++void kbase_mem_pool_free_pages(struct kbase_mem_pool *pool, size_t nr_pages, ++ struct tagged_addr *pages, bool dirty, bool reclaimed) ++{ ++ struct kbase_mem_pool *next_pool = pool->next_pool; ++ struct page *p; ++ size_t nr_to_pool; ++ LIST_HEAD(to_pool_list); ++ size_t i = 0; ++ ++ pool_dbg(pool, "free_pages(%zu):\n", nr_pages); ++ ++ if (!reclaimed) { ++ /* Add to this pool */ ++ nr_to_pool = kbase_mem_pool_capacity(pool); ++ nr_to_pool = min(nr_pages, nr_to_pool); ++ ++ kbase_mem_pool_add_array(pool, nr_to_pool, pages, false, dirty); ++ ++ i += nr_to_pool; ++ ++ if (i != nr_pages && next_pool) { ++ /* Spill to next pool (may overspill) */ ++ nr_to_pool = kbase_mem_pool_capacity(next_pool); ++ nr_to_pool = min(nr_pages - i, nr_to_pool); ++ ++ kbase_mem_pool_add_array(next_pool, nr_to_pool, ++ pages + i, true, dirty); ++ i += nr_to_pool; ++ } ++ } ++ ++ /* Free any remaining pages to kernel */ ++ for (; i < nr_pages; i++) { ++ if (unlikely(!as_phys_addr_t(pages[i]))) ++ continue; ++ ++ if (is_huge(pages[i]) && !is_huge_head(pages[i])) { ++ pages[i] = as_tagged(0); ++ continue; ++ } ++ ++ p = as_page(pages[i]); ++ ++ kbase_mem_pool_free_page(pool, p); ++ pages[i] = as_tagged(0); ++ } ++ ++ pool_dbg(pool, "free_pages(%zu) done\n", nr_pages); ++} ++ ++ ++void kbase_mem_pool_free_pages_locked(struct kbase_mem_pool *pool, ++ size_t nr_pages, struct tagged_addr *pages, bool dirty, ++ bool reclaimed) ++{ ++ struct page *p; ++ size_t nr_to_pool; ++ LIST_HEAD(to_pool_list); ++ size_t i = 0; ++ ++ lockdep_assert_held(&pool->pool_lock); ++ ++ pool_dbg(pool, "free_pages_locked(%zu):\n", nr_pages); ++ ++ if (!reclaimed) { ++ /* Add to this pool */ ++ nr_to_pool = kbase_mem_pool_capacity(pool); ++ nr_to_pool = min(nr_pages, nr_to_pool); ++ ++ kbase_mem_pool_add_array_locked(pool, nr_pages, pages, false, ++ dirty); ++ ++ i += nr_to_pool; ++ } ++ ++ /* Free any remaining pages to kernel */ ++ for (; i < nr_pages; i++) { ++ if (unlikely(!as_phys_addr_t(pages[i]))) ++ continue; ++ ++ if (is_huge(pages[i]) && !is_huge_head(pages[i])) { ++ pages[i] = as_tagged(0); ++ continue; ++ } ++ ++ p = as_page(pages[i]); ++ ++ kbase_mem_pool_free_page(pool, p); ++ pages[i] = as_tagged(0); ++ } ++ ++ pool_dbg(pool, "free_pages_locked(%zu) done\n", nr_pages); ++} +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem_pool_debugfs.c b/drivers/gpu/arm/bifrost/mali_kbase_mem_pool_debugfs.c +new file mode 100755 +index 000000000000..5879fdf85b1d +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_mem_pool_debugfs.c +@@ -0,0 +1,191 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++#include ++ ++#include "mali_kbase_mem_pool_debugfs.h" ++#include "mali_kbase_debugfs_helper.h" ++ ++void kbase_mem_pool_debugfs_trim(void *const array, size_t const index, ++ size_t const value) ++{ ++ struct kbase_mem_pool *const mem_pools = array; ++ ++ if (WARN_ON(!mem_pools) || ++ WARN_ON(index >= MEMORY_GROUP_MANAGER_NR_GROUPS)) ++ return; ++ ++ kbase_mem_pool_trim(&mem_pools[index], value); ++} ++ ++void kbase_mem_pool_debugfs_set_max_size(void *const array, ++ size_t const index, size_t const value) ++{ ++ struct kbase_mem_pool *const mem_pools = array; ++ ++ if (WARN_ON(!mem_pools) || ++ WARN_ON(index >= MEMORY_GROUP_MANAGER_NR_GROUPS)) ++ return; ++ ++ kbase_mem_pool_set_max_size(&mem_pools[index], value); ++} ++ ++size_t kbase_mem_pool_debugfs_size(void *const array, size_t const index) ++{ ++ struct kbase_mem_pool *const mem_pools = array; ++ ++ if (WARN_ON(!mem_pools) || ++ WARN_ON(index >= MEMORY_GROUP_MANAGER_NR_GROUPS)) ++ return 0; ++ ++ return kbase_mem_pool_size(&mem_pools[index]); ++} ++ ++size_t kbase_mem_pool_debugfs_max_size(void *const array, size_t const index) ++{ ++ struct kbase_mem_pool *const mem_pools = array; ++ ++ if (WARN_ON(!mem_pools) || ++ WARN_ON(index >= MEMORY_GROUP_MANAGER_NR_GROUPS)) ++ return 0; ++ ++ return kbase_mem_pool_max_size(&mem_pools[index]); ++} ++ ++void kbase_mem_pool_config_debugfs_set_max_size(void *const array, ++ size_t const index, size_t const value) ++{ ++ struct kbase_mem_pool_config *const configs = array; ++ ++ if (WARN_ON(!configs) || ++ WARN_ON(index >= MEMORY_GROUP_MANAGER_NR_GROUPS)) ++ return; ++ ++ kbase_mem_pool_config_set_max_size(&configs[index], value); ++} ++ ++size_t kbase_mem_pool_config_debugfs_max_size(void *const array, ++ size_t const index) ++{ ++ struct kbase_mem_pool_config *const configs = array; ++ ++ if (WARN_ON(!configs) || ++ WARN_ON(index >= MEMORY_GROUP_MANAGER_NR_GROUPS)) ++ return 0; ++ ++ return kbase_mem_pool_config_get_max_size(&configs[index]); ++} ++ ++static int kbase_mem_pool_debugfs_size_show(struct seq_file *sfile, void *data) ++{ ++ CSTD_UNUSED(data); ++ return kbase_debugfs_helper_seq_read(sfile, ++ MEMORY_GROUP_MANAGER_NR_GROUPS, kbase_mem_pool_debugfs_size); ++} ++ ++static ssize_t kbase_mem_pool_debugfs_write(struct file *file, ++ const char __user *ubuf, size_t count, loff_t *ppos) ++{ ++ int err; ++ ++ CSTD_UNUSED(ppos); ++ err = kbase_debugfs_helper_seq_write(file, ubuf, count, ++ MEMORY_GROUP_MANAGER_NR_GROUPS, kbase_mem_pool_debugfs_trim); ++ return err ? err : count; ++} ++ ++static int kbase_mem_pool_debugfs_open(struct inode *in, struct file *file) ++{ ++ return single_open(file, kbase_mem_pool_debugfs_size_show, ++ in->i_private); ++} ++ ++static const struct file_operations kbase_mem_pool_debugfs_fops = { ++ .owner = THIS_MODULE, ++ .open = kbase_mem_pool_debugfs_open, ++ .read = seq_read, ++ .write = kbase_mem_pool_debugfs_write, ++ .llseek = seq_lseek, ++ .release = single_release, ++}; ++ ++static int kbase_mem_pool_debugfs_max_size_show(struct seq_file *sfile, ++ void *data) ++{ ++ CSTD_UNUSED(data); ++ return kbase_debugfs_helper_seq_read(sfile, ++ MEMORY_GROUP_MANAGER_NR_GROUPS, ++ kbase_mem_pool_debugfs_max_size); ++} ++ ++static ssize_t kbase_mem_pool_debugfs_max_size_write(struct file *file, ++ const char __user *ubuf, size_t count, loff_t *ppos) ++{ ++ int err; ++ ++ CSTD_UNUSED(ppos); ++ err = kbase_debugfs_helper_seq_write(file, ubuf, count, ++ MEMORY_GROUP_MANAGER_NR_GROUPS, ++ kbase_mem_pool_debugfs_set_max_size); ++ return err ? err : count; ++} ++ ++static int kbase_mem_pool_debugfs_max_size_open(struct inode *in, ++ struct file *file) ++{ ++ return single_open(file, kbase_mem_pool_debugfs_max_size_show, ++ in->i_private); ++} ++ ++static const struct file_operations kbase_mem_pool_debugfs_max_size_fops = { ++ .owner = THIS_MODULE, ++ .open = kbase_mem_pool_debugfs_max_size_open, ++ .read = seq_read, ++ .write = kbase_mem_pool_debugfs_max_size_write, ++ .llseek = seq_lseek, ++ .release = single_release, ++}; ++ ++void kbase_mem_pool_debugfs_init(struct dentry *parent, ++ struct kbase_context *kctx) ++{ ++ /* prevent unprivileged use of debug file in old kernel version */ ++#if (KERNEL_VERSION(4, 7, 0) <= LINUX_VERSION_CODE) ++ /* only for newer kernel version debug file system is safe */ ++ const mode_t mode = 0644; ++#else ++ const mode_t mode = 0600; ++#endif ++ ++ debugfs_create_file("mem_pool_size", mode, parent, ++ &kctx->mem_pools.small, &kbase_mem_pool_debugfs_fops); ++ ++ debugfs_create_file("mem_pool_max_size", mode, parent, ++ &kctx->mem_pools.small, &kbase_mem_pool_debugfs_max_size_fops); ++ ++ debugfs_create_file("lp_mem_pool_size", mode, parent, ++ &kctx->mem_pools.large, &kbase_mem_pool_debugfs_fops); ++ ++ debugfs_create_file("lp_mem_pool_max_size", mode, parent, ++ &kctx->mem_pools.large, &kbase_mem_pool_debugfs_max_size_fops); ++} +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem_pool_debugfs.h b/drivers/gpu/arm/bifrost/mali_kbase_mem_pool_debugfs.h +new file mode 100755 +index 000000000000..2932945b3185 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_mem_pool_debugfs.h +@@ -0,0 +1,123 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_MEM_POOL_DEBUGFS_H_ ++#define _KBASE_MEM_POOL_DEBUGFS_H_ ++ ++#include ++ ++/** ++ * kbase_mem_pool_debugfs_init - add debugfs knobs for @pool ++ * @parent: Parent debugfs dentry ++ * @kctx: The kbase context ++ * ++ * Adds four debugfs files under @parent: ++ * - mem_pool_size: get/set the current sizes of @kctx: mem_pools ++ * - mem_pool_max_size: get/set the max sizes of @kctx: mem_pools ++ * - lp_mem_pool_size: get/set the current sizes of @kctx: lp_mem_pool ++ * - lp_mem_pool_max_size: get/set the max sizes of @kctx:lp_mem_pool ++ */ ++void kbase_mem_pool_debugfs_init(struct dentry *parent, ++ struct kbase_context *kctx); ++ ++/** ++ * kbase_mem_pool_debugfs_trim - Grow or shrink a memory pool to a new size ++ * ++ * @array: Address of the first in an array of physical memory pools. ++ * @index: A memory group ID to be used as an index into the array of memory ++ * pools. Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). ++ * @value: New number of pages in the pool. ++ * ++ * If @value > current size, fill the pool with new pages from the kernel, but ++ * not above the max_size for the pool. ++ * If @value < current size, shrink the pool by freeing pages to the kernel. ++ */ ++void kbase_mem_pool_debugfs_trim(void *array, size_t index, size_t value); ++ ++/** ++ * kbase_mem_pool_debugfs_set_max_size - Set maximum number of free pages in ++ * memory pool ++ * ++ * @array: Address of the first in an array of physical memory pools. ++ * @index: A memory group ID to be used as an index into the array of memory ++ * pools. Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). ++ * @value: Maximum number of free pages the pool can hold. ++ * ++ * If the maximum size is reduced, the pool will be shrunk to adhere to the ++ * new limit. For details see kbase_mem_pool_shrink(). ++ */ ++void kbase_mem_pool_debugfs_set_max_size(void *array, size_t index, ++ size_t value); ++ ++/** ++ * kbase_mem_pool_debugfs_size - Get number of free pages in a memory pool ++ * ++ * @array: Address of the first in an array of physical memory pools. ++ * @index: A memory group ID to be used as an index into the array of memory ++ * pools. Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). ++ * ++ * Note: the size of the pool may in certain corner cases exceed @max_size! ++ * ++ * Return: Number of free pages in the pool ++ */ ++size_t kbase_mem_pool_debugfs_size(void *array, size_t index); ++ ++/** ++ * kbase_mem_pool_debugfs_max_size - Get maximum number of free pages in a ++ * memory pool ++ * ++ * @array: Address of the first in an array of physical memory pools. ++ * @index: A memory group ID to be used as an index into the array of memory ++ * pools. Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). ++ * ++ * Return: Maximum number of free pages in the pool ++ */ ++size_t kbase_mem_pool_debugfs_max_size(void *array, size_t index); ++ ++/** ++ * kbase_mem_pool_config_debugfs_set_max_size - Set maximum number of free pages ++ * in initial configuration of pool ++ * ++ * @array: Array of initial configurations for a set of physical memory pools. ++ * @index: A memory group ID to be used as an index into the array. ++ * Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). ++ * @value : Maximum number of free pages that a memory pool created from the ++ * selected configuration can hold. ++ */ ++void kbase_mem_pool_config_debugfs_set_max_size(void *array, size_t index, ++ size_t value); ++ ++/** ++ * kbase_mem_pool_config_debugfs_max_size - Get maximum number of free pages ++ * from initial configuration of pool ++ * ++ * @array: Array of initial configurations for a set of physical memory pools. ++ * @index: A memory group ID to be used as an index into the array. ++ * Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). ++ * ++ * Return: Maximum number of free pages that a memory pool created from the ++ * selected configuration can hold. ++ */ ++size_t kbase_mem_pool_config_debugfs_max_size(void *array, size_t index); ++ ++#endif /*_KBASE_MEM_POOL_DEBUGFS_H_ */ ++ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem_pool_group.c b/drivers/gpu/arm/bifrost/mali_kbase_mem_pool_group.c +new file mode 100755 +index 000000000000..aa2554805b5b +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_mem_pool_group.c +@@ -0,0 +1,115 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++#include ++#include ++ ++#include ++ ++void kbase_mem_pool_group_config_set_max_size( ++ struct kbase_mem_pool_group_config *const configs, ++ size_t const max_size) ++{ ++ size_t const large_max_size = max_size >> ++ (KBASE_MEM_POOL_2MB_PAGE_TABLE_ORDER - ++ KBASE_MEM_POOL_4KB_PAGE_TABLE_ORDER); ++ int gid; ++ ++ for (gid = 0; gid < MEMORY_GROUP_MANAGER_NR_GROUPS; ++gid) { ++ kbase_mem_pool_config_set_max_size(&configs->small[gid], ++ max_size); ++ ++ kbase_mem_pool_config_set_max_size(&configs->large[gid], ++ large_max_size); ++ } ++} ++ ++int kbase_mem_pool_group_init( ++ struct kbase_mem_pool_group *const mem_pools, ++ struct kbase_device *const kbdev, ++ const struct kbase_mem_pool_group_config *const configs, ++ struct kbase_mem_pool_group *next_pools) ++{ ++ int gid, err = 0; ++ ++ for (gid = 0; gid < MEMORY_GROUP_MANAGER_NR_GROUPS; ++gid) { ++ err = kbase_mem_pool_init(&mem_pools->small[gid], ++ &configs->small[gid], ++ KBASE_MEM_POOL_4KB_PAGE_TABLE_ORDER, ++ gid, ++ kbdev, ++ next_pools ? &next_pools->small[gid] : NULL); ++ ++ if (!err) { ++ err = kbase_mem_pool_init(&mem_pools->large[gid], ++ &configs->large[gid], ++ KBASE_MEM_POOL_2MB_PAGE_TABLE_ORDER, ++ gid, ++ kbdev, ++ next_pools ? &next_pools->large[gid] : NULL); ++ if (err) ++ kbase_mem_pool_term(&mem_pools->small[gid]); ++ } ++ ++ /* Break out of the loop early to avoid incrementing the count ++ * of memory pool pairs successfully initialized. ++ */ ++ if (err) ++ break; ++ } ++ ++ if (err) { ++ /* gid gives the number of memory pool pairs successfully ++ * initialized, which is one greater than the array index of the ++ * last group. ++ */ ++ while (gid-- > 0) { ++ kbase_mem_pool_term(&mem_pools->small[gid]); ++ kbase_mem_pool_term(&mem_pools->large[gid]); ++ } ++ } ++ ++ return err; ++} ++ ++void kbase_mem_pool_group_mark_dying( ++ struct kbase_mem_pool_group *const mem_pools) ++{ ++ int gid; ++ ++ for (gid = 0; gid < MEMORY_GROUP_MANAGER_NR_GROUPS; ++gid) { ++ kbase_mem_pool_mark_dying(&mem_pools->small[gid]); ++ kbase_mem_pool_mark_dying(&mem_pools->large[gid]); ++ } ++} ++ ++void kbase_mem_pool_group_term( ++ struct kbase_mem_pool_group *const mem_pools) ++{ ++ int gid; ++ ++ for (gid = 0; gid < MEMORY_GROUP_MANAGER_NR_GROUPS; ++gid) { ++ kbase_mem_pool_term(&mem_pools->small[gid]); ++ kbase_mem_pool_term(&mem_pools->large[gid]); ++ } ++} +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem_pool_group.h b/drivers/gpu/arm/bifrost/mali_kbase_mem_pool_group.h +new file mode 100755 +index 000000000000..0484f5940ad1 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_mem_pool_group.h +@@ -0,0 +1,92 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_MEM_POOL_GROUP_H_ ++#define _KBASE_MEM_POOL_GROUP_H_ ++ ++#include ++ ++/** ++ * kbase_mem_pool_group_config_init - Set the initial configuration for a ++ * set of memory pools ++ * ++ * This function sets the initial configuration for every memory pool so that ++ * the maximum amount of free memory that each pool can hold is identical. ++ * The equivalent number of 2 MiB pages is calculated automatically for the ++ * purpose of configuring the large page pools. ++ * ++ * @configs: Initial configuration for the set of memory pools ++ * @max_size: Maximum number of free 4 KiB pages each pool can hold ++ */ ++void kbase_mem_pool_group_config_set_max_size( ++ struct kbase_mem_pool_group_config *configs, size_t max_size); ++ ++/** ++ * kbase_mem_pool_group_init - Initialize a set of memory pools ++ * ++ * Initializes a complete set of physical memory pools. Memory pools are used to ++ * allow efficient reallocation of previously-freed physical pages. A pair of ++ * memory pools is initialized for each physical memory group: one for 4 KiB ++ * pages and one for 2 MiB pages. ++ * ++ * If @next_pools is not NULL then a request to allocate memory from an ++ * empty pool in @mem_pools will attempt to allocate from the equivalent pool ++ * in @next_pools before going to the memory group manager. Similarly ++ * pages can spill over to the equivalent pool in @next_pools when a pool ++ * is full in @mem_pools. Pages are zeroed before they spill over to another ++ * pool, to prevent leaking information between applications. ++ * ++ * @mem_pools: Set of memory pools to initialize ++ * @kbdev: Kbase device where memory is used ++ * @configs: Initial configuration for the set of memory pools ++ * @next_pools: Set of memory pools from which to allocate memory if there ++ * is no free memory in one of the @mem_pools ++ * ++ * Return: 0 on success, otherwise a negative error code ++ */ ++int kbase_mem_pool_group_init(struct kbase_mem_pool_group *mem_pools, ++ struct kbase_device *kbdev, ++ const struct kbase_mem_pool_group_config *configs, ++ struct kbase_mem_pool_group *next_pools); ++ ++/** ++ * kbase_mem_pool_group_term - Mark a set of memory pools as dying ++ * ++ * Marks a complete set of physical memory pools previously initialized by ++ * @kbase_mem_pool_group_init as dying. This will cause any ongoing allocation ++ * operations (eg growing on page fault) to be terminated. ++ * ++ * @mem_pools: Set of memory pools to mark ++ */ ++void kbase_mem_pool_group_mark_dying(struct kbase_mem_pool_group *mem_pools); ++ ++/** ++ * kbase_mem_pool_group_term - Terminate a set of memory pools ++ * ++ * Terminates a complete set of physical memory pools previously initialized by ++ * @kbase_mem_pool_group_init. ++ * ++ * @mem_pools: Set of memory pools to terminate ++ */ ++void kbase_mem_pool_group_term(struct kbase_mem_pool_group *mem_pools); ++ ++#endif /* _KBASE_MEM_POOL_GROUP_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem_profile_debugfs.c b/drivers/gpu/arm/bifrost/mali_kbase_mem_profile_debugfs.c +new file mode 100755 +index 000000000000..85723f825054 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_mem_profile_debugfs.c +@@ -0,0 +1,134 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2017, 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++ ++#ifdef CONFIG_DEBUG_FS ++ ++/** Show callback for the @c mem_profile debugfs file. ++ * ++ * This function is called to get the contents of the @c mem_profile debugfs ++ * file. This is a report of current memory usage and distribution in userspace. ++ * ++ * @param sfile The debugfs entry ++ * @param data Data associated with the entry ++ * ++ * @return 0 if it successfully prints data in debugfs entry file, non-zero otherwise ++ */ ++static int kbasep_mem_profile_seq_show(struct seq_file *sfile, void *data) ++{ ++ struct kbase_context *kctx = sfile->private; ++ ++ mutex_lock(&kctx->mem_profile_lock); ++ ++ seq_write(sfile, kctx->mem_profile_data, kctx->mem_profile_size); ++ ++ seq_putc(sfile, '\n'); ++ ++ mutex_unlock(&kctx->mem_profile_lock); ++ ++ return 0; ++} ++ ++/* ++ * File operations related to debugfs entry for mem_profile ++ */ ++static int kbasep_mem_profile_debugfs_open(struct inode *in, struct file *file) ++{ ++ return single_open(file, kbasep_mem_profile_seq_show, in->i_private); ++} ++ ++static const struct file_operations kbasep_mem_profile_debugfs_fops = { ++ .owner = THIS_MODULE, ++ .open = kbasep_mem_profile_debugfs_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = single_release, ++}; ++ ++int kbasep_mem_profile_debugfs_insert(struct kbase_context *kctx, char *data, ++ size_t size) ++{ ++#if (KERNEL_VERSION(4, 7, 0) <= LINUX_VERSION_CODE) ++ const mode_t mode = 0444; ++#else ++ const mode_t mode = 0400; ++#endif ++ int err = 0; ++ ++ mutex_lock(&kctx->mem_profile_lock); ++ ++ dev_dbg(kctx->kbdev->dev, "initialised: %d", ++ kbase_ctx_flag(kctx, KCTX_MEM_PROFILE_INITIALIZED)); ++ ++ if (!kbase_ctx_flag(kctx, KCTX_MEM_PROFILE_INITIALIZED)) { ++ if (IS_ERR_OR_NULL(kctx->kctx_dentry)) { ++ err = -ENOMEM; ++ } else if (!debugfs_create_file("mem_profile", mode, ++ kctx->kctx_dentry, kctx, ++ &kbasep_mem_profile_debugfs_fops)) { ++ err = -EAGAIN; ++ } else { ++ kbase_ctx_flag_set(kctx, ++ KCTX_MEM_PROFILE_INITIALIZED); ++ } ++ } ++ ++ if (kbase_ctx_flag(kctx, KCTX_MEM_PROFILE_INITIALIZED)) { ++ kfree(kctx->mem_profile_data); ++ kctx->mem_profile_data = data; ++ kctx->mem_profile_size = size; ++ } else { ++ kfree(data); ++ } ++ ++ dev_dbg(kctx->kbdev->dev, "returning: %d, initialised: %d", ++ err, kbase_ctx_flag(kctx, KCTX_MEM_PROFILE_INITIALIZED)); ++ ++ mutex_unlock(&kctx->mem_profile_lock); ++ ++ return err; ++} ++ ++void kbasep_mem_profile_debugfs_remove(struct kbase_context *kctx) ++{ ++ mutex_lock(&kctx->mem_profile_lock); ++ ++ dev_dbg(kctx->kbdev->dev, "initialised: %d", ++ kbase_ctx_flag(kctx, KCTX_MEM_PROFILE_INITIALIZED)); ++ ++ kfree(kctx->mem_profile_data); ++ kctx->mem_profile_data = NULL; ++ kctx->mem_profile_size = 0; ++ ++ mutex_unlock(&kctx->mem_profile_lock); ++} ++ ++#else /* CONFIG_DEBUG_FS */ ++ ++int kbasep_mem_profile_debugfs_insert(struct kbase_context *kctx, char *data, ++ size_t size) ++{ ++ kfree(data); ++ return 0; ++} ++#endif /* CONFIG_DEBUG_FS */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem_profile_debugfs.h b/drivers/gpu/arm/bifrost/mali_kbase_mem_profile_debugfs.h +new file mode 100755 +index 000000000000..1462247c3bca +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_mem_profile_debugfs.h +@@ -0,0 +1,64 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++/** ++ * @file mali_kbase_mem_profile_debugfs.h ++ * Header file for mem profiles entries in debugfs ++ * ++ */ ++ ++#ifndef _KBASE_MEM_PROFILE_DEBUGFS_H ++#define _KBASE_MEM_PROFILE_DEBUGFS_H ++ ++#include ++#include ++ ++/** ++ * @brief Remove entry from Mali memory profile debugfs ++ */ ++void kbasep_mem_profile_debugfs_remove(struct kbase_context *kctx); ++ ++/** ++ * @brief Insert @p data to the debugfs file so it can be read by userspace ++ * ++ * The function takes ownership of @p data and frees it later when new data ++ * is inserted. ++ * ++ * If the debugfs entry corresponding to the @p kctx doesn't exist, ++ * an attempt will be made to create it. ++ * ++ * @param kctx The context whose debugfs file @p data should be inserted to ++ * @param data A NULL-terminated string to be inserted to the debugfs file, ++ * without the trailing new line character ++ * @param size The length of the @p data string ++ * @return 0 if @p data inserted correctly ++ * -EAGAIN in case of error ++ * @post @ref mem_profile_initialized will be set to @c true ++ * the first time this function succeeds. ++ */ ++int kbasep_mem_profile_debugfs_insert(struct kbase_context *kctx, char *data, ++ size_t size); ++ ++#endif /*_KBASE_MEM_PROFILE_DEBUGFS_H*/ ++ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem_profile_debugfs_buf_size.h b/drivers/gpu/arm/bifrost/mali_kbase_mem_profile_debugfs_buf_size.h +new file mode 100755 +index 000000000000..d55cc854c415 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_mem_profile_debugfs_buf_size.h +@@ -0,0 +1,39 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014, 2018-2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/** ++ * @file mali_kbase_mem_profile_debugfs_buf_size.h ++ * Header file for the size of the buffer to accumulate the histogram report text in ++ */ ++ ++#ifndef _KBASE_MEM_PROFILE_DEBUGFS_BUF_SIZE_H_ ++#define _KBASE_MEM_PROFILE_DEBUGFS_BUF_SIZE_H_ ++ ++/** ++ * The size of the buffer to accumulate the histogram report text in ++ * @see @ref CCTXP_HIST_BUF_SIZE_MAX_LENGTH_REPORT ++ */ ++#define KBASE_MEM_PROFILE_MAX_BUF_SIZE \ ++ ((size_t) (64 + ((80 + (56 * 64)) * 53) + 56)) ++ ++#endif /*_KBASE_MEM_PROFILE_DEBUGFS_BUF_SIZE_H_*/ ++ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mipe_gen_header.h b/drivers/gpu/arm/bifrost/mali_kbase_mipe_gen_header.h +new file mode 100755 +index 000000000000..72acadfae993 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_mipe_gen_header.h +@@ -0,0 +1,219 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* THIS FILE IS AUTOGENERATED BY mali_trace_generator.py. ++ * DO NOT EDIT. ++ */ ++ ++/* clang-format off */ ++ ++#include "mali_kbase_mipe_proto.h" ++ ++/** ++ * This header generates MIPE tracepoint declaration BLOB at ++ * compile time. ++ * ++ * It is intentional that there is no header guard. ++ * The header could be included multiple times for ++ * different blobs compilation. ++ * ++ * Before including this header MIPE_HEADER_* parameters must be ++ * defined. See documentation below: ++ */ ++ ++/** ++ * The name of the variable where the result BLOB will be stored. ++ */ ++#if !defined(MIPE_HEADER_BLOB_VAR_NAME) ++#error "MIPE_HEADER_BLOB_VAR_NAME must be defined!" ++#endif ++ ++/** ++ * A compiler attribute for the BLOB variable. ++ * ++ * e.g. __attribute__((section("my_section"))) ++ * ++ * Default value is no attribute. ++ */ ++#if !defined(MIPE_HEADER_BLOB_VAR_ATTRIBUTE) ++#define MIPE_HEADER_BLOB_VAR_ATTRIBUTE ++#endif ++ ++/** ++ * MIPE stream id. ++ * ++ * See enum tl_stream_id. ++ */ ++#if !defined(MIPE_HEADER_STREAM_ID) ++#error "MIPE_HEADER_STREAM_ID must be defined!" ++#endif ++ ++/** ++ * MIPE packet class. ++ * ++ * See enum tl_packet_class. ++ */ ++#if !defined(MIPE_HEADER_PKT_CLASS) ++#error "MIPE_HEADER_PKT_CLASS must be defined!" ++#endif ++ ++/** ++ * The list of tracepoints to process. ++ * ++ * It should be defined as follows: ++ * ++ * #define MIPE_HEADER_TRACEPOINT_LIST \ ++ * TRACEPOINT_DESC(FIRST_TRACEPOINT, "Some description", "@II", "first_arg,second_arg") \ ++ * TRACEPOINT_DESC(SECOND_TRACEPOINT, "Some description", "@II", "first_arg,second_arg") \ ++ * etc. ++ * ++ * Where the first argument is tracepoints name, the second ++ * argument is a short tracepoint description, the third argument ++ * argument types (see MIPE documentation), and the fourth argument ++ * is comma separated argument names. ++ */ ++#if !defined(MIPE_HEADER_TRACEPOINT_LIST) ++#error "MIPE_HEADER_TRACEPOINT_LIST must be defined!" ++#endif ++ ++/** ++ * The number of entries in MIPE_HEADER_TRACEPOINT_LIST. ++ */ ++#if !defined(MIPE_HEADER_TRACEPOINT_LIST_SIZE) ++#error "MIPE_HEADER_TRACEPOINT_LIST_SIZE must be defined!" ++#endif ++ ++/** ++ * The list of enums to process. ++ * ++ * It should be defined as follows: ++ * ++ * #define MIPE_HEADER_ENUM_LIST \ ++ * ENUM_DESC(enum_arg_name, enum_value) \ ++ * ENUM_DESC(enum_arg_name, enum_value) \ ++ * etc. ++ * ++ * Where enum_arg_name is the name of a tracepoint argument being used with ++ * this enum. enum_value is a valid C enum value. ++ * ++ * Default value is an empty list. ++ */ ++#if defined(MIPE_HEADER_ENUM_LIST) ++ ++/** ++ * Tracepoint message ID used for enums declaration. ++ */ ++#if !defined(MIPE_HEADER_ENUM_MSG_ID) ++#error "MIPE_HEADER_ENUM_MSG_ID must be defined!" ++#endif ++ ++#else ++#define MIPE_HEADER_ENUM_LIST ++#endif ++ ++/** ++ * The MIPE tracepoint declaration BLOB. ++ */ ++const struct ++{ ++ u32 _mipe_w0; ++ u32 _mipe_w1; ++ u8 _protocol_version; ++ u8 _pointer_size; ++ u32 _tp_count; ++#define TRACEPOINT_DESC(name, desc, arg_types, arg_names) \ ++ struct { \ ++ u32 _name; \ ++ u32 _size_string_name; \ ++ char _string_name[sizeof(#name)]; \ ++ u32 _size_desc; \ ++ char _desc[sizeof(desc)]; \ ++ u32 _size_arg_types; \ ++ char _arg_types[sizeof(arg_types)]; \ ++ u32 _size_arg_names; \ ++ char _arg_names[sizeof(arg_names)]; \ ++ } __attribute__ ((__packed__)) __ ## name; ++ ++#define ENUM_DESC(arg_name, value) \ ++ struct { \ ++ u32 _msg_id; \ ++ u32 _arg_name_len; \ ++ char _arg_name[sizeof(#arg_name)]; \ ++ u32 _value; \ ++ u32 _value_str_len; \ ++ char _value_str[sizeof(#value)]; \ ++ } __attribute__ ((__packed__)) __ ## arg_name ## _ ## value; ++ ++ MIPE_HEADER_TRACEPOINT_LIST ++ MIPE_HEADER_ENUM_LIST ++#undef TRACEPOINT_DESC ++#undef ENUM_DESC ++} __attribute__((packed)) MIPE_HEADER_BLOB_VAR_NAME MIPE_HEADER_BLOB_VAR_ATTRIBUTE = { ++ ._mipe_w0 = MIPE_PACKET_HEADER_W0( ++ TL_PACKET_FAMILY_TL, ++ MIPE_HEADER_PKT_CLASS, ++ TL_PACKET_TYPE_HEADER, ++ MIPE_HEADER_STREAM_ID), ++ ._mipe_w1 = MIPE_PACKET_HEADER_W1( ++ sizeof(MIPE_HEADER_BLOB_VAR_NAME) - PACKET_HEADER_SIZE, ++ 0), ++ ._protocol_version = SWTRACE_VERSION, ++ ._pointer_size = sizeof(void *), ++ ._tp_count = MIPE_HEADER_TRACEPOINT_LIST_SIZE, ++#define TRACEPOINT_DESC(name, desc, arg_types, arg_names) \ ++ .__ ## name = { \ ++ ._name = name, \ ++ ._size_string_name = sizeof(#name), \ ++ ._string_name = #name, \ ++ ._size_desc = sizeof(desc), \ ++ ._desc = desc, \ ++ ._size_arg_types = sizeof(arg_types), \ ++ ._arg_types = arg_types, \ ++ ._size_arg_names = sizeof(arg_names), \ ++ ._arg_names = arg_names \ ++ }, ++#define ENUM_DESC(arg_name, value) \ ++ .__ ## arg_name ## _ ## value = { \ ++ ._msg_id = MIPE_HEADER_ENUM_MSG_ID, \ ++ ._arg_name_len = sizeof(#arg_name), \ ++ ._arg_name = #arg_name, \ ++ ._value = value, \ ++ ._value_str_len = sizeof(#value), \ ++ ._value_str = #value \ ++ }, ++ ++ MIPE_HEADER_TRACEPOINT_LIST ++ MIPE_HEADER_ENUM_LIST ++#undef TRACEPOINT_DESC ++#undef ENUM_DESC ++}; ++ ++#undef MIPE_HEADER_BLOB_VAR_NAME ++#undef MIPE_HEADER_BLOB_VAR_ATTRIBUTE ++#undef MIPE_HEADER_STREAM_ID ++#undef MIPE_HEADER_PKT_CLASS ++#undef MIPE_HEADER_TRACEPOINT_LIST ++#undef MIPE_HEADER_TRACEPOINT_LIST_SIZE ++#undef MIPE_HEADER_ENUM_LIST ++#undef MIPE_HEADER_ENUM_MSG_ID ++ ++/* clang-format on */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mipe_proto.h b/drivers/gpu/arm/bifrost/mali_kbase_mipe_proto.h +new file mode 100755 +index 000000000000..54667cfc6304 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_mipe_proto.h +@@ -0,0 +1,127 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* THIS FILE IS AUTOGENERATED BY mali_trace_generator.py. ++ * DO NOT EDIT. ++ */ ++ ++/* clang-format off */ ++ ++#if !defined(_KBASE_MIPE_PROTO_H) ++#define _KBASE_MIPE_PROTO_H ++ ++#define _BITFIELD_MASK_FIELD(pos, len) \ ++ (((1u << len) - 1) << pos) ++ ++#define _BITFIELD_SET_FIELD(pos, len, value) \ ++ (_BITFIELD_MASK_FIELD(pos, len) & (((u32) value) << pos)) ++ ++#define BITFIELD_SET(field_name, value) \ ++ _BITFIELD_SET_FIELD(field_name ## _POS, field_name ## _LEN, value) ++ ++/* The version of swtrace protocol used in timeline stream. */ ++#define SWTRACE_VERSION 3 ++ ++/* Packet header - first word. ++ * These values must be defined according to MIPE documentation. ++ */ ++#define PACKET_STREAMID_POS 0 ++#define PACKET_STREAMID_LEN 8 ++#define PACKET_RSVD1_POS (PACKET_STREAMID_POS + PACKET_STREAMID_LEN) ++#define PACKET_RSVD1_LEN 8 ++#define PACKET_TYPE_POS (PACKET_RSVD1_POS + PACKET_RSVD1_LEN) ++#define PACKET_TYPE_LEN 3 ++#define PACKET_CLASS_POS (PACKET_TYPE_POS + PACKET_TYPE_LEN) ++#define PACKET_CLASS_LEN 7 ++#define PACKET_FAMILY_POS (PACKET_CLASS_POS + PACKET_CLASS_LEN) ++#define PACKET_FAMILY_LEN 6 ++ ++/* Packet header - second word ++ * These values must be defined according to MIPE documentation. ++ */ ++#define PACKET_LENGTH_POS 0 ++#define PACKET_LENGTH_LEN 24 ++#define PACKET_SEQBIT_POS (PACKET_LENGTH_POS + PACKET_LENGTH_LEN) ++#define PACKET_SEQBIT_LEN 1 ++#define PACKET_RSVD2_POS (PACKET_SEQBIT_POS + PACKET_SEQBIT_LEN) ++#define PACKET_RSVD2_LEN 7 ++ ++/* First word of a MIPE packet */ ++#define MIPE_PACKET_HEADER_W0(pkt_family, pkt_class, pkt_type, stream_id) \ ++ (0 \ ++ | BITFIELD_SET(PACKET_FAMILY, pkt_family) \ ++ | BITFIELD_SET(PACKET_CLASS, pkt_class) \ ++ | BITFIELD_SET(PACKET_TYPE, pkt_type) \ ++ | BITFIELD_SET(PACKET_STREAMID, stream_id)) ++ ++/* Second word of a MIPE packet */ ++#define MIPE_PACKET_HEADER_W1(packet_length, seqbit) \ ++ (0 \ ++ | BITFIELD_SET(PACKET_LENGTH, packet_length) \ ++ | BITFIELD_SET(PACKET_SEQBIT, seqbit)) ++ ++/* The number of bytes reserved for packet header. ++ * These value must be defined according to MIPE documentation. ++ */ ++#define PACKET_HEADER_SIZE 8 /* bytes */ ++ ++/* The number of bytes reserved for packet sequence number. ++ * These value must be defined according to MIPE documentation. ++ */ ++#define PACKET_NUMBER_SIZE 4 /* bytes */ ++ ++/* Timeline packet family ids. ++ * Values are significant! Check MIPE documentation. ++ */ ++enum tl_packet_family { ++ TL_PACKET_FAMILY_CTRL = 0, /* control packets */ ++ TL_PACKET_FAMILY_TL = 1, /* timeline packets */ ++ TL_PACKET_FAMILY_COUNT ++}; ++ ++/* Packet classes used in timeline streams. ++ * Values are significant! Check MIPE documentation. ++ */ ++enum tl_packet_class { ++ TL_PACKET_CLASS_OBJ = 0, /* timeline objects packet */ ++ TL_PACKET_CLASS_AUX = 1, /* auxiliary events packet */ ++}; ++ ++/* Packet types used in timeline streams. ++ * Values are significant! Check MIPE documentation. ++ */ ++enum tl_packet_type { ++ TL_PACKET_TYPE_HEADER = 0, /* stream's header/directory */ ++ TL_PACKET_TYPE_BODY = 1, /* stream's body */ ++ TL_PACKET_TYPE_SUMMARY = 2, /* stream's summary */ ++}; ++ ++/* Stream ID types (timeline family). */ ++enum tl_stream_id { ++ TL_STREAM_ID_USER = 0, /* User-space driver Timeline stream. */ ++ TL_STREAM_ID_KERNEL = 1, /* Kernel-space driver Timeline stream. */ ++ TL_STREAM_ID_CSFFW = 2, /* CSF firmware driver Timeline stream. */ ++}; ++ ++#endif /* _KBASE_MIPE_PROTO_H */ ++ ++/* clang-format on */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_native_mgm.c b/drivers/gpu/arm/bifrost/mali_kbase_native_mgm.c +new file mode 100755 +index 000000000000..38ae46e0ddf1 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_native_mgm.c +@@ -0,0 +1,153 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++#include ++#include ++ ++#include ++#include ++ ++/** ++ * kbase_native_mgm_alloc - Native physical memory allocation method ++ * ++ * @mgm_dev: The memory group manager the request is being made through. ++ * @group_id: A physical memory group ID, which must be valid but is not used. ++ * Its valid range is 0 .. MEMORY_GROUP_MANAGER_NR_GROUPS-1. ++ * @gfp_mask: Bitmask of Get Free Page flags affecting allocator behavior. ++ * @order: Page order for physical page size (order=0 means 4 KiB, ++ * order=9 means 2 MiB). ++ * ++ * Delegates all memory allocation requests to the kernel's alloc_pages ++ * function. ++ * ++ * Return: Pointer to allocated page, or NULL if allocation failed. ++ */ ++static struct page *kbase_native_mgm_alloc( ++ struct memory_group_manager_device *mgm_dev, int group_id, ++ gfp_t gfp_mask, unsigned int order) ++{ ++ /* ++ * Check that the base and the mgm defines, from separate header files, ++ * for the max number of memory groups are compatible. ++ */ ++ BUILD_BUG_ON(BASE_MEM_GROUP_COUNT != MEMORY_GROUP_MANAGER_NR_GROUPS); ++ /* ++ * Check that the mask used for storing the memory group ID is big ++ * enough for the largest possible memory group ID. ++ */ ++ BUILD_BUG_ON((BASEP_CONTEXT_MMU_GROUP_ID_MASK ++ >> BASEP_CONTEXT_MMU_GROUP_ID_SHIFT) ++ < (BASE_MEM_GROUP_COUNT - 1)); ++ ++ CSTD_UNUSED(mgm_dev); ++ CSTD_UNUSED(group_id); ++ ++ return alloc_pages(gfp_mask, order); ++} ++ ++/** ++ * kbase_native_mgm_free - Native physical memory freeing method ++ * ++ * @mgm_dev: The memory group manager the request is being made through. ++ * @group_id: A physical memory group ID, which must be valid but is not used. ++ * Its valid range is 0 .. MEMORY_GROUP_MANAGER_NR_GROUPS-1. ++ * @page: Address of the struct associated with a page of physical ++ * memory that was allocated by calling kbase_native_mgm_alloc ++ * with the same argument values. ++ * @order: Page order for physical page size (order=0 means 4 KiB, ++ * order=9 means 2 MiB). ++ * ++ * Delegates all memory freeing requests to the kernel's __free_pages function. ++ */ ++static void kbase_native_mgm_free(struct memory_group_manager_device *mgm_dev, ++ int group_id, struct page *page, unsigned int order) ++{ ++ CSTD_UNUSED(mgm_dev); ++ CSTD_UNUSED(group_id); ++ ++ __free_pages(page, order); ++} ++ ++/** ++ * kbase_native_mgm_vmf_insert_pfn_prot - Native method to map a page on the CPU ++ * ++ * @mgm_dev: The memory group manager the request is being made through. ++ * @group_id: A physical memory group ID, which must be valid but is not used. ++ * Its valid range is 0 .. MEMORY_GROUP_MANAGER_NR_GROUPS-1. ++ * @vma: The virtual memory area to insert the page into. ++ * @addr: An address contained in @vma to assign to the inserted page. ++ * @pfn: The kernel Page Frame Number to insert at @addr in @vma. ++ * @pgprot: Protection flags for the inserted page. ++ * ++ * Called from a CPU virtual memory page fault handler. Delegates all memory ++ * mapping requests to the kernel's vmf_insert_pfn_prot function. ++ * ++ * Return: Type of fault that occurred or VM_FAULT_NOPAGE if the page table ++ * entry was successfully installed. ++ */ ++static vm_fault_t kbase_native_mgm_vmf_insert_pfn_prot( ++ struct memory_group_manager_device *mgm_dev, int group_id, ++ struct vm_area_struct *vma, unsigned long addr, ++ unsigned long pfn, pgprot_t pgprot) ++{ ++ CSTD_UNUSED(mgm_dev); ++ CSTD_UNUSED(group_id); ++ ++ return vmf_insert_pfn_prot(vma, addr, pfn, pgprot); ++} ++ ++/** ++ * kbase_native_mgm_update_gpu_pte - Native method to modify a GPU page table ++ * entry ++ * ++ * @mgm_dev: The memory group manager the request is being made through. ++ * @group_id: A physical memory group ID, which must be valid but is not used. ++ * Its valid range is 0 .. MEMORY_GROUP_MANAGER_NR_GROUPS-1. ++ * @mmu_level: The level of the MMU page table where the page is getting mapped. ++ * @pte: The prepared page table entry. ++ * ++ * This function simply returns the @pte without modification. ++ * ++ * Return: A GPU page table entry to be stored in a page table. ++ */ ++static u64 ++kbase_native_mgm_update_gpu_pte(struct memory_group_manager_device *mgm_dev, ++ int group_id, int mmu_level, u64 pte) ++{ ++ CSTD_UNUSED(mgm_dev); ++ CSTD_UNUSED(group_id); ++ CSTD_UNUSED(mmu_level); ++ ++ return pte; ++} ++ ++struct memory_group_manager_device kbase_native_mgm_dev = { ++ .ops = { ++ .mgm_alloc_page = kbase_native_mgm_alloc, ++ .mgm_free_page = kbase_native_mgm_free, ++ .mgm_get_import_memory_id = NULL, ++ .mgm_vmf_insert_pfn_prot = kbase_native_mgm_vmf_insert_pfn_prot, ++ .mgm_update_gpu_pte = kbase_native_mgm_update_gpu_pte, ++ }, ++ .data = NULL ++}; +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_native_mgm.h b/drivers/gpu/arm/bifrost/mali_kbase_native_mgm.h +new file mode 100755 +index 000000000000..431b1f4cb5db +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_native_mgm.h +@@ -0,0 +1,39 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_NATIVE_MGM_H_ ++#define _KBASE_NATIVE_MGM_H_ ++ ++#include ++ ++/** ++ * kbase_native_mgm_dev - Native memory group manager device ++ * ++ * An implementation of the memory group manager interface that is intended for ++ * internal use when no platform-specific memory group manager is available. ++ * ++ * It ignores the specified group ID and delegates to the kernel's physical ++ * memory allocation and freeing functions. ++ */ ++extern struct memory_group_manager_device kbase_native_mgm_dev; ++ ++#endif /* _KBASE_NATIVE_MGM_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_platform_fake.c b/drivers/gpu/arm/bifrost/mali_kbase_platform_fake.c +new file mode 100755 +index 000000000000..fbb090e6c21f +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_platform_fake.c +@@ -0,0 +1,124 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++ ++ ++/* ++ * This file is included only for type definitions and functions belonging to ++ * specific platform folders. Do not add dependencies with symbols that are ++ * defined somewhere else. ++ */ ++#include ++ ++#define PLATFORM_CONFIG_RESOURCE_COUNT 4 ++#define PLATFORM_CONFIG_IRQ_RES_COUNT 3 ++ ++static struct platform_device *mali_device; ++ ++#ifndef CONFIG_OF ++/** ++ * @brief Convert data in struct kbase_io_resources struct to Linux-specific resources ++ * ++ * Function converts data in struct kbase_io_resources struct to an array of Linux resource structures. Note that function ++ * assumes that size of linux_resource array is at least PLATFORM_CONFIG_RESOURCE_COUNT. ++ * Resources are put in fixed order: I/O memory region, job IRQ, MMU IRQ, GPU IRQ. ++ * ++ * @param[in] io_resource Input IO resource data ++ * @param[out] linux_resources Pointer to output array of Linux resource structures ++ */ ++static void kbasep_config_parse_io_resources(const struct kbase_io_resources *io_resources, struct resource *const linux_resources) ++{ ++ if (!io_resources || !linux_resources) { ++ pr_err("%s: couldn't find proper resources\n", __func__); ++ return; ++ } ++ ++ memset(linux_resources, 0, PLATFORM_CONFIG_RESOURCE_COUNT * sizeof(struct resource)); ++ ++ linux_resources[0].start = io_resources->io_memory_region.start; ++ linux_resources[0].end = io_resources->io_memory_region.end; ++ linux_resources[0].flags = IORESOURCE_MEM; ++ ++ linux_resources[1].start = io_resources->job_irq_number; ++ linux_resources[1].end = io_resources->job_irq_number; ++ linux_resources[1].flags = IORESOURCE_IRQ | IORESOURCE_IRQ_HIGHLEVEL; ++ ++ linux_resources[2].start = io_resources->mmu_irq_number; ++ linux_resources[2].end = io_resources->mmu_irq_number; ++ linux_resources[2].flags = IORESOURCE_IRQ | IORESOURCE_IRQ_HIGHLEVEL; ++ ++ linux_resources[3].start = io_resources->gpu_irq_number; ++ linux_resources[3].end = io_resources->gpu_irq_number; ++ linux_resources[3].flags = IORESOURCE_IRQ | IORESOURCE_IRQ_HIGHLEVEL; ++} ++#endif /* CONFIG_OF */ ++ ++int kbase_platform_register(void) ++{ ++ struct kbase_platform_config *config; ++#ifndef CONFIG_OF ++ struct resource resources[PLATFORM_CONFIG_RESOURCE_COUNT]; ++#endif ++ int err; ++ ++ config = kbase_get_platform_config(); /* declared in midgard/mali_kbase_config.h but defined in platform folder */ ++ if (config == NULL) { ++ pr_err("%s: couldn't get platform config\n", __func__); ++ return -ENODEV; ++ } ++ ++ mali_device = platform_device_alloc("mali", 0); ++ if (mali_device == NULL) ++ return -ENOMEM; ++ ++#ifndef CONFIG_OF ++ kbasep_config_parse_io_resources(config->io_resources, resources); ++ err = platform_device_add_resources(mali_device, resources, PLATFORM_CONFIG_RESOURCE_COUNT); ++ if (err) { ++ platform_device_put(mali_device); ++ mali_device = NULL; ++ return err; ++ } ++#endif /* CONFIG_OF */ ++ ++ err = platform_device_add(mali_device); ++ if (err) { ++ platform_device_unregister(mali_device); ++ mali_device = NULL; ++ return err; ++ } ++ ++ return 0; ++} ++EXPORT_SYMBOL(kbase_platform_register); ++ ++void kbase_platform_unregister(void) ++{ ++ if (mali_device) ++ platform_device_unregister(mali_device); ++} ++EXPORT_SYMBOL(kbase_platform_unregister); +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_pm.c b/drivers/gpu/arm/bifrost/mali_kbase_pm.c +new file mode 100755 +index 000000000000..630ab1550045 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_pm.c +@@ -0,0 +1,292 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++/** ++ * @file mali_kbase_pm.c ++ * Base kernel power management APIs ++ */ ++ ++#include ++#include ++#include ++#include ++ ++#include ++#include ++ ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++#include ++#endif /* CONFIG_MALI_ARBITER_SUPPORT */ ++ ++#include ++ ++int kbase_pm_powerup(struct kbase_device *kbdev, unsigned int flags) ++{ ++ return kbase_hwaccess_pm_powerup(kbdev, flags); ++} ++ ++void kbase_pm_halt(struct kbase_device *kbdev) ++{ ++ kbase_hwaccess_pm_halt(kbdev); ++} ++ ++void kbase_pm_context_active(struct kbase_device *kbdev) ++{ ++ (void)kbase_pm_context_active_handle_suspend(kbdev, ++ KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE); ++} ++ ++int kbase_pm_context_active_handle_suspend(struct kbase_device *kbdev, ++ enum kbase_pm_suspend_handler suspend_handler) ++{ ++ int c; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ dev_dbg(kbdev->dev, "%s - reason = %d, pid = %d\n", __func__, ++ suspend_handler, current->pid); ++ kbase_pm_lock(kbdev); ++ ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++ if (kbase_arbiter_pm_ctx_active_handle_suspend(kbdev, ++ suspend_handler)) { ++ kbase_pm_unlock(kbdev); ++ return 1; ++ } ++#endif /* CONFIG_MALI_ARBITER_SUPPORT */ ++ ++ if (kbase_pm_is_suspending(kbdev)) { ++ switch (suspend_handler) { ++ case KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE: ++ if (kbdev->pm.active_count != 0) ++ break; ++ /* FALLTHROUGH */ ++ case KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE: ++ kbase_pm_unlock(kbdev); ++ return 1; ++ ++ case KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE: ++ /* FALLTHROUGH */ ++ default: ++ KBASE_DEBUG_ASSERT_MSG(false, "unreachable"); ++ break; ++ } ++ } ++ c = ++kbdev->pm.active_count; ++ KBASE_KTRACE_ADD(kbdev, PM_CONTEXT_ACTIVE, NULL, c); ++ ++ if (c == 1) { ++ /* First context active: Power on the GPU and ++ * any cores requested by the policy ++ */ ++ kbase_hwaccess_pm_gpu_active(kbdev); ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++ kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_REF_EVENT); ++#endif /* CONFIG_MALI_ARBITER_SUPPORT */ ++ kbase_clk_rate_trace_manager_gpu_active(kbdev); ++ } ++ ++ kbase_pm_unlock(kbdev); ++ dev_dbg(kbdev->dev, "%s %d\n", __func__, kbdev->pm.active_count); ++ ++ return 0; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_context_active); ++ ++void kbase_pm_context_idle(struct kbase_device *kbdev) ++{ ++ int c; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ ++ kbase_pm_lock(kbdev); ++ ++ c = --kbdev->pm.active_count; ++ KBASE_KTRACE_ADD(kbdev, PM_CONTEXT_IDLE, NULL, c); ++ ++ KBASE_DEBUG_ASSERT(c >= 0); ++ ++ if (c == 0) { ++ /* Last context has gone idle */ ++ kbase_hwaccess_pm_gpu_idle(kbdev); ++ kbase_clk_rate_trace_manager_gpu_idle(kbdev); ++ ++ /* Wake up anyone waiting for this to become 0 (e.g. suspend). ++ * The waiters must synchronize with us by locking the pm.lock ++ * after waiting. ++ */ ++ wake_up(&kbdev->pm.zero_active_count_wait); ++ } ++ ++ kbase_pm_unlock(kbdev); ++ dev_dbg(kbdev->dev, "%s %d (pid = %d)\n", __func__, ++ kbdev->pm.active_count, current->pid); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_context_idle); ++ ++void kbase_pm_driver_suspend(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev); ++ ++ /* Suspend vinstr. This blocks until the vinstr worker and timer are ++ * no longer running. ++ */ ++ kbase_vinstr_suspend(kbdev->vinstr_ctx); ++ ++ /* Disable GPU hardware counters. ++ * This call will block until counters are disabled. ++ */ ++ kbase_hwcnt_context_disable(kbdev->hwcnt_gpu_ctx); ++ ++ mutex_lock(&kbdev->pm.lock); ++ if (WARN_ON(kbase_pm_is_suspending(kbdev))) { ++ mutex_unlock(&kbdev->pm.lock); ++ return; ++ } ++ kbdev->pm.suspending = true; ++ mutex_unlock(&kbdev->pm.lock); ++ ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++ if (kbdev->arb.arb_if) { ++ int i; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbdev->js_data.runpool_irq.submit_allowed = 0; ++ kbase_disjoint_state_up(kbdev); ++ for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) ++ kbase_job_slot_softstop(kbdev, i, NULL); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ } ++#endif /* CONFIG_MALI_ARBITER_SUPPORT */ ++ ++ /* From now on, the active count will drop towards zero. Sometimes, ++ * it'll go up briefly before going down again. However, once ++ * it reaches zero it will stay there - guaranteeing that we've idled ++ * all pm references ++ */ ++ ++#if !MALI_USE_CSF ++ /* Suspend job scheduler and associated components, so that it releases all ++ * the PM active count references */ ++ kbasep_js_suspend(kbdev); ++#else ++ kbase_csf_scheduler_pm_suspend(kbdev); ++#endif ++ ++ /* Wait for the active count to reach zero. This is not the same as ++ * waiting for a power down, since not all policies power down when this ++ * reaches zero. ++ */ ++ dev_dbg(kbdev->dev, ">wait_event - waiting for active_count == 0 (pid = %d)\n", ++ current->pid); ++ wait_event(kbdev->pm.zero_active_count_wait, ++ kbdev->pm.active_count == 0); ++ dev_dbg(kbdev->dev, ">wait_event - waiting done\n"); ++ ++ /* NOTE: We synchronize with anything that was just finishing a ++ * kbase_pm_context_idle() call by locking the pm.lock below ++ */ ++ kbase_hwaccess_pm_suspend(kbdev); ++ ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++ if (kbdev->arb.arb_if) { ++ mutex_lock(&kbdev->pm.arb_vm_state->vm_state_lock); ++ kbase_arbiter_pm_vm_stopped(kbdev); ++ mutex_unlock(&kbdev->pm.arb_vm_state->vm_state_lock); ++ } ++#endif /* CONFIG_MALI_ARBITER_SUPPORT */ ++} ++ ++void kbase_pm_driver_resume(struct kbase_device *kbdev, bool arb_gpu_start) ++{ ++ unsigned long flags; ++ ++ /* MUST happen before any pm_context_active calls occur */ ++ kbase_hwaccess_pm_resume(kbdev); ++ ++ /* Initial active call, to power on the GPU/cores if needed */ ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++ if (kbase_pm_context_active_handle_suspend(kbdev, ++ (arb_gpu_start ? ++ KBASE_PM_SUSPEND_HANDLER_VM_GPU_GRANTED : ++ KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE))) ++ return; ++#else ++ kbase_pm_context_active(kbdev); ++#endif ++ ++#if !MALI_USE_CSF ++ /* Resume any blocked atoms (which may cause contexts to be scheduled in ++ * and dependent atoms to run) ++ */ ++ kbase_resume_suspended_soft_jobs(kbdev); ++ ++ /* Resume the Job Scheduler and associated components, and start running ++ * atoms ++ */ ++ kbasep_js_resume(kbdev); ++#else ++ kbase_csf_scheduler_pm_resume(kbdev); ++#endif ++ ++ /* Matching idle call, to power off the GPU/cores if we didn't actually ++ * need it and the policy doesn't want it on ++ */ ++ kbase_pm_context_idle(kbdev); ++ ++ /* Re-enable GPU hardware counters */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ /* Resume vinstr */ ++ kbase_vinstr_resume(kbdev->vinstr_ctx); ++} ++ ++void kbase_pm_suspend(struct kbase_device *kbdev) ++{ ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++ if (kbdev->arb.arb_if) ++ kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_OS_SUSPEND_EVENT); ++ else ++ kbase_pm_driver_suspend(kbdev); ++#else ++ kbase_pm_driver_suspend(kbdev); ++#endif /* CONFIG_MALI_ARBITER_SUPPORT */ ++} ++ ++void kbase_pm_resume(struct kbase_device *kbdev) ++{ ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++ if (kbdev->arb.arb_if) ++ kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_OS_RESUME_EVENT); ++ else ++ kbase_pm_driver_resume(kbdev, false); ++#else ++ kbase_pm_driver_resume(kbdev, false); ++#endif /* CONFIG_MALI_ARBITER_SUPPORT */ ++} +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_pm.h b/drivers/gpu/arm/bifrost/mali_kbase_pm.h +new file mode 100755 +index 000000000000..13565186c11f +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_pm.h +@@ -0,0 +1,251 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++/** ++ * @file mali_kbase_pm.h ++ * Power management API definitions ++ */ ++ ++#ifndef _KBASE_PM_H_ ++#define _KBASE_PM_H_ ++ ++#include "mali_kbase_hwaccess_pm.h" ++ ++#define PM_ENABLE_IRQS 0x01 ++#define PM_HW_ISSUES_DETECT 0x02 ++ ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++/* In the case that the GPU was granted by the Arbiter, it will have ++ * already been reset. The following flag ensures it is not reset ++ * twice. ++ */ ++#define PM_NO_RESET 0x04 ++#endif ++ ++/** Initialize the power management framework. ++ * ++ * Must be called before any other power management function ++ * ++ * @param kbdev The kbase device structure for the device ++ * (must be a valid pointer) ++ * ++ * @return 0 if the power management framework was successfully initialized. ++ */ ++int kbase_pm_init(struct kbase_device *kbdev); ++ ++/** Power up GPU after all modules have been initialized and interrupt handlers installed. ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid pointer) ++ * ++ * @param flags Flags to pass on to kbase_pm_init_hw ++ * ++ * @return 0 if powerup was successful. ++ */ ++int kbase_pm_powerup(struct kbase_device *kbdev, unsigned int flags); ++ ++/** ++ * Halt the power management framework. ++ * Should ensure that no new interrupts are generated, ++ * but allow any currently running interrupt handlers to complete successfully. ++ * The GPU is forced off by the time this function returns, regardless of ++ * whether or not the active power policy asks for the GPU to be powered off. ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_halt(struct kbase_device *kbdev); ++ ++/** Terminate the power management framework. ++ * ++ * No power management functions may be called after this ++ * (except @ref kbase_pm_init) ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_term(struct kbase_device *kbdev); ++ ++/** Increment the count of active contexts. ++ * ++ * This function should be called when a context is about to submit a job. ++ * It informs the active power policy that the GPU is going to be in use shortly ++ * and the policy is expected to start turning on the GPU. ++ * ++ * This function will block until the GPU is available. ++ * ++ * This function ASSERTS if a suspend is occuring/has occurred whilst this is ++ * in use. Use kbase_pm_contect_active_unless_suspending() instead. ++ * ++ * @note a Suspend is only visible to Kernel threads; user-space threads in a ++ * syscall cannot witness a suspend, because they are frozen before the suspend ++ * begins. ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_context_active(struct kbase_device *kbdev); ++ ++ ++/** Handler codes for doing kbase_pm_context_active_handle_suspend() */ ++enum kbase_pm_suspend_handler { ++ /** A suspend is not expected/not possible - this is the same as ++ * kbase_pm_context_active() ++ */ ++ KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE, ++ /** If we're suspending, fail and don't increase the active count */ ++ KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE, ++ /** If we're suspending, succeed and allow the active count to increase ++ * if it didn't go from 0->1 (i.e., we didn't re-activate the GPU). ++ * ++ * This should only be used when there is a bounded time on the activation ++ * (e.g. guarantee it's going to be idled very soon after) ++ */ ++ KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE, ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++ /** Special case when Arbiter has notified we can use GPU. ++ * Active count should always start at 0 in this case. ++ */ ++ KBASE_PM_SUSPEND_HANDLER_VM_GPU_GRANTED, ++#endif /* CONFIG_MALI_ARBITER_SUPPORT */ ++}; ++ ++/** Suspend 'safe' variant of kbase_pm_context_active() ++ * ++ * If a suspend is in progress, this allows for various different ways of ++ * handling the suspend. Refer to @ref enum kbase_pm_suspend_handler for details. ++ * ++ * We returns a status code indicating whether we're allowed to keep the GPU ++ * active during the suspend, depending on the handler code. If the status code ++ * indicates a failure, the caller must abort whatever operation it was ++ * attempting, and potentially queue it up for after the OS has resumed. ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid pointer) ++ * @param suspend_handler The handler code for how to handle a suspend that might occur ++ * @return zero Indicates success ++ * @return non-zero Indicates failure due to the system being suspending/suspended. ++ */ ++int kbase_pm_context_active_handle_suspend(struct kbase_device *kbdev, enum kbase_pm_suspend_handler suspend_handler); ++ ++/** Decrement the reference count of active contexts. ++ * ++ * This function should be called when a context becomes idle. ++ * After this call the GPU may be turned off by the power policy so the calling ++ * code should ensure that it does not access the GPU's registers. ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_context_idle(struct kbase_device *kbdev); ++ ++/* NOTE: kbase_pm_is_active() is in mali_kbase.h, because it is an inline ++ * function ++ */ ++ ++/** ++ * Suspend the GPU and prevent any further register accesses to it from Kernel ++ * threads. ++ * ++ * This is called in response to an OS suspend event, and calls into the various ++ * kbase components to complete the suspend. ++ * ++ * @note the mechanisms used here rely on all user-space threads being frozen ++ * by the OS before we suspend. Otherwise, an IOCTL could occur that powers up ++ * the GPU e.g. via atom submission. ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_suspend(struct kbase_device *kbdev); ++ ++/** ++ * Resume the GPU, allow register accesses to it, and resume running atoms on ++ * the GPU. ++ * ++ * This is called in response to an OS resume event, and calls into the various ++ * kbase components to complete the resume. ++ * ++ * Also called when using VM arbiter, when GPU access has been granted. ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_resume(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_vsync_callback - vsync callback ++ * ++ * @buffer_updated: 1 if a new frame was displayed, 0 otherwise ++ * @data: Pointer to the kbase device as returned by kbase_find_device() ++ * ++ * Callback function used to notify the power management code that a vsync has ++ * occurred on the display. ++ */ ++void kbase_pm_vsync_callback(int buffer_updated, void *data); ++ ++/** ++ * kbase_pm_driver_suspend() - Put GPU and driver in suspend state ++ * @param kbdev The kbase device structure for the device ++ * (must be a valid pointer) ++ * ++ * Suspend the GPU and prevent any further register accesses to it from Kernel ++ * threads. ++ * ++ * This is called in response to an OS suspend event, and calls into the various ++ * kbase components to complete the suspend. ++ * ++ * Despite kbase_pm_suspend(), it will ignore to update Arbiter ++ * status if MALI_ARBITER_SUPPORT is enabled. ++ * ++ * @note the mechanisms used here rely on all user-space threads being frozen ++ * by the OS before we suspend. Otherwise, an IOCTL could occur that powers up ++ * the GPU e.g. via atom submission. ++ */ ++void kbase_pm_driver_suspend(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_driver_resume() - Put GPU and driver in resume ++ * @param kbdev The kbase device structure for the device ++ * (must be a valid pointer) ++ * ++ * Resume the GPU, allow register accesses to it, and resume running atoms on ++ * the GPU. ++ * ++ * This is called in response to an OS resume event, and calls into the various ++ * kbase components to complete the resume. ++ * ++ * Also called when using VM arbiter, when GPU access has been granted. ++ * ++ * Despite kbase_pm_resume(), it will ignore to update Arbiter ++ * status if MALI_ARBITER_SUPPORT is enabled. ++ */ ++void kbase_pm_driver_resume(struct kbase_device *kbdev, bool arb_gpu_start); ++ ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++/** ++ * kbase_pm_handle_gpu_lost() - Handle GPU Lost for the VM ++ * @kbdev: Device pointer ++ * ++ * Handles the case that the Arbiter has forced the GPU away from the VM, ++ * so that interrupts will not be received and registers are no longer ++ * accessible because replaced by dummy RAM. ++ * Kill any running tasks and put the driver into a GPU powered-off state. ++ */ ++void kbase_pm_handle_gpu_lost(struct kbase_device *kbdev); ++#endif /* CONFIG_MALI_ARBITER_SUPPORT */ ++ ++#endif /* _KBASE_PM_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_regs_history_debugfs.c b/drivers/gpu/arm/bifrost/mali_kbase_regs_history_debugfs.c +new file mode 100755 +index 000000000000..7b86c58440db +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_regs_history_debugfs.c +@@ -0,0 +1,245 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014, 2016, 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include "mali_kbase.h" ++#include "mali_kbase_regs_history_debugfs.h" ++ ++#if defined(CONFIG_DEBUG_FS) && !defined(CONFIG_MALI_BIFROST_NO_MALI) ++ ++#include ++ ++/** ++ * kbase_io_history_resize - resize the register access history buffer. ++ * ++ * @h: Pointer to a valid register history to resize ++ * @new_size: Number of accesses the buffer could hold ++ * ++ * A successful resize will clear all recent register accesses. ++ * If resizing fails for any reason (e.g., could not allocate memory, invalid ++ * buffer size) then the original buffer will be kept intact. ++ * ++ * @return 0 if the buffer was resized, failure otherwise ++ */ ++static int kbase_io_history_resize(struct kbase_io_history *h, u16 new_size) ++{ ++ struct kbase_io_access *old_buf; ++ struct kbase_io_access *new_buf; ++ unsigned long flags; ++ ++ if (!new_size) ++ goto out_err; /* The new size must not be 0 */ ++ ++ new_buf = vmalloc(new_size * sizeof(*h->buf)); ++ if (!new_buf) ++ goto out_err; ++ ++ spin_lock_irqsave(&h->lock, flags); ++ ++ old_buf = h->buf; ++ ++ /* Note: we won't bother with copying the old data over. The dumping ++ * logic wouldn't work properly as it relies on 'count' both as a ++ * counter and as an index to the buffer which would have changed with ++ * the new array. This is a corner case that we don't need to support. ++ */ ++ h->count = 0; ++ h->size = new_size; ++ h->buf = new_buf; ++ ++ spin_unlock_irqrestore(&h->lock, flags); ++ ++ vfree(old_buf); ++ ++ return 0; ++ ++out_err: ++ return -1; ++} ++ ++int kbase_io_history_init(struct kbase_io_history *h, u16 n) ++{ ++ h->enabled = false; ++ spin_lock_init(&h->lock); ++ h->count = 0; ++ h->size = 0; ++ h->buf = NULL; ++ if (kbase_io_history_resize(h, n)) ++ return -1; ++ ++ return 0; ++} ++ ++void kbase_io_history_term(struct kbase_io_history *h) ++{ ++ vfree(h->buf); ++ h->buf = NULL; ++} ++ ++void kbase_io_history_add(struct kbase_io_history *h, ++ void __iomem const *addr, u32 value, u8 write) ++{ ++ struct kbase_io_access *io; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&h->lock, flags); ++ ++ io = &h->buf[h->count % h->size]; ++ io->addr = (uintptr_t)addr | write; ++ io->value = value; ++ ++h->count; ++ /* If count overflows, move the index by the buffer size so the entire ++ * buffer will still be dumped later ++ */ ++ if (unlikely(!h->count)) ++ h->count = h->size; ++ ++ spin_unlock_irqrestore(&h->lock, flags); ++} ++ ++void kbase_io_history_dump(struct kbase_device *kbdev) ++{ ++ struct kbase_io_history *const h = &kbdev->io_history; ++ u16 i; ++ size_t iters; ++ unsigned long flags; ++ ++ if (!unlikely(h->enabled)) ++ return; ++ ++ spin_lock_irqsave(&h->lock, flags); ++ ++ dev_err(kbdev->dev, "Register IO History:"); ++ iters = (h->size > h->count) ? h->count : h->size; ++ dev_err(kbdev->dev, "Last %zu register accesses of %zu total:\n", iters, ++ h->count); ++ for (i = 0; i < iters; ++i) { ++ struct kbase_io_access *io = ++ &h->buf[(h->count - iters + i) % h->size]; ++ char const access = (io->addr & 1) ? 'w' : 'r'; ++ ++ dev_err(kbdev->dev, "%6i: %c: reg 0x%016lx val %08x\n", i, ++ access, (unsigned long)(io->addr & ~0x1), io->value); ++ } ++ ++ spin_unlock_irqrestore(&h->lock, flags); ++} ++ ++static int regs_history_size_get(void *data, u64 *val) ++{ ++ struct kbase_io_history *const h = data; ++ ++ *val = h->size; ++ ++ return 0; ++} ++ ++static int regs_history_size_set(void *data, u64 val) ++{ ++ struct kbase_io_history *const h = data; ++ ++ return kbase_io_history_resize(h, (u16)val); ++} ++ ++ ++DEFINE_SIMPLE_ATTRIBUTE(regs_history_size_fops, ++ regs_history_size_get, ++ regs_history_size_set, ++ "%llu\n"); ++ ++ ++/** ++ * regs_history_show - show callback for the register access history file. ++ * ++ * @sfile: The debugfs entry ++ * @data: Data associated with the entry ++ * ++ * This function is called to dump all recent accesses to the GPU registers. ++ * ++ * @return 0 if successfully prints data in debugfs entry file, failure ++ * otherwise ++ */ ++static int regs_history_show(struct seq_file *sfile, void *data) ++{ ++ struct kbase_io_history *const h = sfile->private; ++ u16 i; ++ size_t iters; ++ unsigned long flags; ++ ++ if (!h->enabled) { ++ seq_puts(sfile, "The register access history is disabled\n"); ++ goto out; ++ } ++ ++ spin_lock_irqsave(&h->lock, flags); ++ ++ iters = (h->size > h->count) ? h->count : h->size; ++ seq_printf(sfile, "Last %zu register accesses of %zu total:\n", iters, ++ h->count); ++ for (i = 0; i < iters; ++i) { ++ struct kbase_io_access *io = ++ &h->buf[(h->count - iters + i) % h->size]; ++ char const access = (io->addr & 1) ? 'w' : 'r'; ++ ++ seq_printf(sfile, "%6i: %c: reg 0x%016lx val %08x\n", i, access, ++ (unsigned long)(io->addr & ~0x1), io->value); ++ } ++ ++ spin_unlock_irqrestore(&h->lock, flags); ++ ++out: ++ return 0; ++} ++ ++/** ++ * regs_history_open - open operation for regs_history debugfs file ++ * ++ * @in: &struct inode pointer ++ * @file: &struct file pointer ++ * ++ * @return file descriptor ++ */ ++static int regs_history_open(struct inode *in, struct file *file) ++{ ++ return single_open(file, ®s_history_show, in->i_private); ++} ++ ++static const struct file_operations regs_history_fops = { ++ .owner = THIS_MODULE, ++ .open = ®s_history_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = single_release, ++}; ++ ++void kbasep_regs_history_debugfs_init(struct kbase_device *kbdev) ++{ ++ debugfs_create_bool("regs_history_enabled", S_IRUGO | S_IWUSR, ++ kbdev->mali_debugfs_directory, ++ &kbdev->io_history.enabled); ++ debugfs_create_file("regs_history_size", S_IRUGO | S_IWUSR, ++ kbdev->mali_debugfs_directory, ++ &kbdev->io_history, ®s_history_size_fops); ++ debugfs_create_file("regs_history", S_IRUGO, ++ kbdev->mali_debugfs_directory, &kbdev->io_history, ++ ®s_history_fops); ++} ++#endif /* defined(CONFIG_DEBUG_FS) && !defined(CONFIG_MALI_BIFROST_NO_MALI) */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_regs_history_debugfs.h b/drivers/gpu/arm/bifrost/mali_kbase_regs_history_debugfs.h +new file mode 100755 +index 000000000000..200c0c2d8de8 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_regs_history_debugfs.h +@@ -0,0 +1,85 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014, 2016, 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/** ++ * Header file for register access history support via debugfs ++ * ++ * This interface is made available via /sys/kernel/debug/mali#/regs_history*. ++ * ++ * Usage: ++ * - regs_history_enabled: whether recording of register accesses is enabled. ++ * Write 'y' to enable, 'n' to disable. ++ * - regs_history_size: size of the register history buffer, must be > 0 ++ * - regs_history: return the information about last accesses to the registers. ++ */ ++ ++#ifndef _KBASE_REGS_HISTORY_DEBUGFS_H ++#define _KBASE_REGS_HISTORY_DEBUGFS_H ++ ++struct kbase_device; ++ ++#if defined(CONFIG_DEBUG_FS) && !defined(CONFIG_MALI_BIFROST_NO_MALI) ++ ++/** ++ * kbase_io_history_init - initialize data struct for register access history ++ * ++ * @h: The register history to initialize ++ * @n: The number of register accesses that the buffer could hold ++ * ++ * @return 0 if successfully initialized, failure otherwise ++ */ ++int kbase_io_history_init(struct kbase_io_history *h, u16 n); ++ ++/** ++ * kbase_io_history_term - uninit all resources for the register access history ++ * ++ * @h: The register history to terminate ++ */ ++void kbase_io_history_term(struct kbase_io_history *h); ++ ++/** ++ * kbase_io_history_dump - print the register history to the kernel ring buffer ++ * ++ * @kbdev: Pointer to kbase_device containing the register history to dump ++ */ ++void kbase_io_history_dump(struct kbase_device *kbdev); ++ ++/** ++ * kbasep_regs_history_debugfs_init - add debugfs entries for register history ++ * ++ * @kbdev: Pointer to kbase_device containing the register history ++ */ ++void kbasep_regs_history_debugfs_init(struct kbase_device *kbdev); ++ ++#else /* defined(CONFIG_DEBUG_FS) && !defined(CONFIG_MALI_BIFROST_NO_MALI) */ ++ ++#define kbase_io_history_init(...) ((int)0) ++ ++#define kbase_io_history_term CSTD_NOP ++ ++#define kbase_io_history_dump CSTD_NOP ++ ++#define kbasep_regs_history_debugfs_init CSTD_NOP ++ ++#endif /* defined(CONFIG_DEBUG_FS) && !defined(CONFIG_MALI_BIFROST_NO_MALI) */ ++ ++#endif /*_KBASE_REGS_HISTORY_DEBUGFS_H*/ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_reset_gpu.h b/drivers/gpu/arm/bifrost/mali_kbase_reset_gpu.h +new file mode 100755 +index 000000000000..61bbb0b48490 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_reset_gpu.h +@@ -0,0 +1,129 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_RESET_GPU_H_ ++#define _KBASE_RESET_GPU_H_ ++ ++/** ++ * kbase_prepare_to_reset_gpu_locked - Prepare for resetting the GPU. ++ * @kbdev: Device pointer ++ * ++ * Caller is expected to hold the kbdev->hwaccess_lock. ++ * ++ * Return: a boolean which should be interpreted as follows: ++ * - true - Prepared for reset, kbase_reset_gpu should be called. ++ * - false - Another thread is performing a reset, kbase_reset_gpu should ++ * not be called. ++ */ ++bool kbase_prepare_to_reset_gpu_locked(struct kbase_device *kbdev); ++ ++/** ++ * kbase_prepare_to_reset_gpu - Prepare for resetting the GPU. ++ * @kbdev: Device pointer ++ * ++ * Return: a boolean which should be interpreted as follows: ++ * - true - Prepared for reset, kbase_reset_gpu should be called. ++ * - false - Another thread is performing a reset, kbase_reset_gpu should ++ * not be called. ++ */ ++bool kbase_prepare_to_reset_gpu(struct kbase_device *kbdev); ++ ++/** ++ * kbase_reset_gpu - Reset the GPU ++ * @kbdev: Device pointer ++ * ++ * This function should be called after kbase_prepare_to_reset_gpu if it returns ++ * true. It should never be called without a corresponding call to ++ * kbase_prepare_to_reset_gpu (only on Job Manager GPUs). ++ * ++ * After this function is called the caller should call kbase_reset_gpu_wait() ++ * to know when the reset has completed. ++ */ ++void kbase_reset_gpu(struct kbase_device *kbdev); ++ ++/** ++ * kbase_reset_gpu_locked - Reset the GPU ++ * @kbdev: Device pointer ++ * ++ * This function should be called after kbase_prepare_to_reset_gpu_locked if it ++ * returns true. It should never be called without a corresponding call to ++ * kbase_prepare_to_reset_gpu (only on Job Manager GPUs). ++ * Caller is expected to hold the kbdev->hwaccess_lock. ++ * ++ * After this function is called, the caller should call kbase_reset_gpu_wait() ++ * to know when the reset has completed. ++ */ ++void kbase_reset_gpu_locked(struct kbase_device *kbdev); ++ ++/** ++ * kbase_reset_gpu_silent - Reset the GPU silently ++ * @kbdev: Device pointer ++ * ++ * Reset the GPU without trying to cancel jobs (applicable to Job Manager GPUs) ++ * and don't emit messages into the kernel log while doing the reset. ++ * ++ * This function should be used in cases where we are doing a controlled reset ++ * of the GPU as part of normal processing (e.g. exiting protected mode) where ++ * the driver will have ensured the scheduler has been idled and all other ++ * users of the GPU (e.g. instrumentation) have been suspended. ++ * ++ * Return: 0 if the reset was started successfully ++ * -EAGAIN if another reset is currently in progress ++ */ ++int kbase_reset_gpu_silent(struct kbase_device *kbdev); ++ ++/** ++ * kbase_reset_gpu_is_active - Reports if the GPU is being reset ++ * @kbdev: Device pointer ++ * ++ * Return: True if the GPU is in the process of being reset (or if the reset of ++ * GPU failed, not applicable to Job Manager GPUs). ++ */ ++bool kbase_reset_gpu_is_active(struct kbase_device *kbdev); ++ ++/** ++ * kbase_reset_gpu_wait - Wait for a GPU reset to complete ++ * @kbdev: Device pointer ++ * ++ * This function may wait indefinitely. ++ * ++ * Return: 0 if successful or a negative error code on failure. ++ */ ++int kbase_reset_gpu_wait(struct kbase_device *kbdev); ++ ++/** ++ * kbase_reset_gpu_init - Initialize the GPU reset handling mechanism. ++ * ++ * @kbdev: Device pointer ++ * ++ * Return: 0 if successful or a negative error code on failure. ++ */ ++int kbase_reset_gpu_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_reset_gpu_term - Terminate the GPU reset handling mechanism. ++ * ++ * @kbdev: Device pointer ++ */ ++void kbase_reset_gpu_term(struct kbase_device *kbdev); ++ ++#endif +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_smc.c b/drivers/gpu/arm/bifrost/mali_kbase_smc.c +new file mode 100755 +index 000000000000..b5c7b1289846 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_smc.c +@@ -0,0 +1,91 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015, 2018, 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifdef CONFIG_ARM64 ++ ++#include ++#include ++ ++#include ++ ++/* __asmeq is not available on Kernel versions >= 4.20 */ ++#ifndef __asmeq ++/* ++ * This is used to ensure the compiler did actually allocate the register we ++ * asked it for some inline assembly sequences. Apparently we can't trust the ++ * compiler from one version to another so a bit of paranoia won't hurt. This ++ * string is meant to be concatenated with the inline asm string and will ++ * cause compilation to stop on mismatch. (for details, see gcc PR 15089) ++ */ ++#define __asmeq(x, y) ".ifnc " x "," y " ; .err ; .endif\n\t" ++#endif ++ ++static noinline u64 invoke_smc_fid(u64 function_id, ++ u64 arg0, u64 arg1, u64 arg2) ++{ ++ register u64 x0 asm("x0") = function_id; ++ register u64 x1 asm("x1") = arg0; ++ register u64 x2 asm("x2") = arg1; ++ register u64 x3 asm("x3") = arg2; ++ ++ asm volatile( ++ __asmeq("%0", "x0") ++ __asmeq("%1", "x1") ++ __asmeq("%2", "x2") ++ __asmeq("%3", "x3") ++ "smc #0\n" ++ : "+r" (x0) ++ : "r" (x1), "r" (x2), "r" (x3)); ++ ++ return x0; ++} ++ ++u64 kbase_invoke_smc_fid(u32 fid, u64 arg0, u64 arg1, u64 arg2) ++{ ++ /* Is fast call (bit 31 set) */ ++ KBASE_DEBUG_ASSERT(fid & ~SMC_FAST_CALL); ++ /* bits 16-23 must be zero for fast calls */ ++ KBASE_DEBUG_ASSERT((fid & (0xFF << 16)) == 0); ++ ++ return invoke_smc_fid(fid, arg0, arg1, arg2); ++} ++ ++u64 kbase_invoke_smc(u32 oen, u16 function_number, bool smc64, ++ u64 arg0, u64 arg1, u64 arg2) ++{ ++ u32 fid = 0; ++ ++ /* Only the six bits allowed should be used. */ ++ KBASE_DEBUG_ASSERT((oen & ~SMC_OEN_MASK) == 0); ++ ++ fid |= SMC_FAST_CALL; /* Bit 31: Fast call */ ++ if (smc64) ++ fid |= SMC_64; /* Bit 30: 1=SMC64, 0=SMC32 */ ++ fid |= oen; /* Bit 29:24: OEN */ ++ /* Bit 23:16: Must be zero for fast calls */ ++ fid |= (function_number); /* Bit 15:0: function number */ ++ ++ return kbase_invoke_smc_fid(fid, arg0, arg1, arg2); ++} ++ ++#endif /* CONFIG_ARM64 */ ++ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_smc.h b/drivers/gpu/arm/bifrost/mali_kbase_smc.h +new file mode 100755 +index 000000000000..221eb21a8c7f +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_smc.h +@@ -0,0 +1,72 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++#ifndef _KBASE_SMC_H_ ++#define _KBASE_SMC_H_ ++ ++#ifdef CONFIG_ARM64 ++ ++#include ++ ++#define SMC_FAST_CALL (1 << 31) ++#define SMC_64 (1 << 30) ++ ++#define SMC_OEN_OFFSET 24 ++#define SMC_OEN_MASK (0x3F << SMC_OEN_OFFSET) /* 6 bits */ ++#define SMC_OEN_SIP (2 << SMC_OEN_OFFSET) ++#define SMC_OEN_STD (4 << SMC_OEN_OFFSET) ++ ++ ++/** ++ * kbase_invoke_smc_fid - Perform a secure monitor call ++ * @fid: The SMC function to call, see SMC Calling convention. ++ * @arg0: First argument to the SMC. ++ * @arg1: Second argument to the SMC. ++ * @arg2: Third argument to the SMC. ++ * ++ * See SMC Calling Convention for details. ++ * ++ * Return: the return value from the SMC. ++ */ ++u64 kbase_invoke_smc_fid(u32 fid, u64 arg0, u64 arg1, u64 arg2); ++ ++/** ++ * kbase_invoke_smc_fid - Perform a secure monitor call ++ * @oen: Owning Entity number (SIP, STD etc). ++ * @function_number: The function number within the OEN. ++ * @smc64: use SMC64 calling convention instead of SMC32. ++ * @arg0: First argument to the SMC. ++ * @arg1: Second argument to the SMC. ++ * @arg2: Third argument to the SMC. ++ * ++ * See SMC Calling Convention for details. ++ * ++ * Return: the return value from the SMC call. ++ */ ++u64 kbase_invoke_smc(u32 oen, u16 function_number, bool smc64, ++ u64 arg0, u64 arg1, u64 arg2); ++ ++#endif /* CONFIG_ARM64 */ ++ ++#endif /* _KBASE_SMC_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_softjobs.c b/drivers/gpu/arm/bifrost/mali_kbase_softjobs.c +new file mode 100755 +index 000000000000..c164719b3d7b +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_softjobs.c +@@ -0,0 +1,1805 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++#include ++ ++#include ++#include ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++#include ++#endif ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#if !MALI_USE_CSF ++/** ++ * @file mali_kbase_softjobs.c ++ * ++ * This file implements the logic behind software only jobs that are ++ * executed within the driver rather than being handed over to the GPU. ++ */ ++ ++static void kbasep_add_waiting_soft_job(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ unsigned long lflags; ++ ++ spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); ++ list_add_tail(&katom->queue, &kctx->waiting_soft_jobs); ++ spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); ++} ++ ++void kbasep_remove_waiting_soft_job(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ unsigned long lflags; ++ ++ spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); ++ list_del(&katom->queue); ++ spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); ++} ++ ++static void kbasep_add_waiting_with_timeout(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ ++ /* Record the start time of this atom so we could cancel it at ++ * the right time. ++ */ ++ katom->start_timestamp = ktime_get(); ++ ++ /* Add the atom to the waiting list before the timer is ++ * (re)started to make sure that it gets processed. ++ */ ++ kbasep_add_waiting_soft_job(katom); ++ ++ /* Schedule timeout of this atom after a period if it is not active */ ++ if (!timer_pending(&kctx->soft_job_timeout)) { ++ int timeout_ms = atomic_read( ++ &kctx->kbdev->js_data.soft_job_timeout_ms); ++ mod_timer(&kctx->soft_job_timeout, ++ jiffies + msecs_to_jiffies(timeout_ms)); ++ } ++} ++ ++static int kbasep_read_soft_event_status( ++ struct kbase_context *kctx, u64 evt, unsigned char *status) ++{ ++ unsigned char *mapped_evt; ++ struct kbase_vmap_struct map; ++ ++ mapped_evt = kbase_vmap(kctx, evt, sizeof(*mapped_evt), &map); ++ if (!mapped_evt) ++ return -EFAULT; ++ ++ *status = *mapped_evt; ++ ++ kbase_vunmap(kctx, &map); ++ ++ return 0; ++} ++ ++static int kbasep_write_soft_event_status( ++ struct kbase_context *kctx, u64 evt, unsigned char new_status) ++{ ++ unsigned char *mapped_evt; ++ struct kbase_vmap_struct map; ++ ++ if ((new_status != BASE_JD_SOFT_EVENT_SET) && ++ (new_status != BASE_JD_SOFT_EVENT_RESET)) ++ return -EINVAL; ++ ++ mapped_evt = kbase_vmap(kctx, evt, sizeof(*mapped_evt), &map); ++ if (!mapped_evt) ++ return -EFAULT; ++ ++ *mapped_evt = new_status; ++ ++ kbase_vunmap(kctx, &map); ++ ++ return 0; ++} ++ ++static int kbase_dump_cpu_gpu_time(struct kbase_jd_atom *katom) ++{ ++ struct kbase_vmap_struct map; ++ void *user_result; ++ struct timespec64 ts; ++ struct base_dump_cpu_gpu_counters data; ++ u64 system_time; ++ u64 cycle_counter; ++ u64 jc = katom->jc; ++ struct kbase_context *kctx = katom->kctx; ++ int pm_active_err; ++ ++ memset(&data, 0, sizeof(data)); ++ ++ /* Take the PM active reference as late as possible - otherwise, it could ++ * delay suspend until we process the atom (which may be at the end of a ++ * long chain of dependencies */ ++ pm_active_err = kbase_pm_context_active_handle_suspend(kctx->kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE); ++ if (pm_active_err) { ++ struct kbasep_js_device_data *js_devdata = &kctx->kbdev->js_data; ++ ++ /* We're suspended - queue this on the list of suspended jobs ++ * Use dep_item[1], because dep_item[0] was previously in use ++ * for 'waiting_soft_jobs'. ++ */ ++ mutex_lock(&js_devdata->runpool_mutex); ++ list_add_tail(&katom->dep_item[1], &js_devdata->suspended_soft_jobs_list); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ ++ /* Also adding this to the list of waiting soft job */ ++ kbasep_add_waiting_soft_job(katom); ++ ++ return pm_active_err; ++ } ++ ++ kbase_backend_get_gpu_time(kctx->kbdev, &cycle_counter, &system_time, ++ &ts); ++ ++ kbase_pm_context_idle(kctx->kbdev); ++ ++ data.sec = ts.tv_sec; ++ data.usec = ts.tv_nsec / 1000; ++ data.system_time = system_time; ++ data.cycle_counter = cycle_counter; ++ ++ /* Assume this atom will be cancelled until we know otherwise */ ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ ++ /* GPU_WR access is checked on the range for returning the result to ++ * userspace for the following reasons: ++ * - security, this is currently how imported user bufs are checked. ++ * - userspace ddk guaranteed to assume region was mapped as GPU_WR */ ++ user_result = kbase_vmap_prot(kctx, jc, sizeof(data), KBASE_REG_GPU_WR, &map); ++ if (!user_result) ++ return 0; ++ ++ memcpy(user_result, &data, sizeof(data)); ++ ++ kbase_vunmap(kctx, &map); ++ ++ /* Atom was fine - mark it as done */ ++ katom->event_code = BASE_JD_EVENT_DONE; ++ ++ return 0; ++} ++ ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++/* Called by the explicit fence mechanism when a fence wait has completed */ ++void kbase_soft_event_wait_callback(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ ++ mutex_lock(&kctx->jctx.lock); ++ kbasep_remove_waiting_soft_job(katom); ++ kbase_finish_soft_job(katom); ++ if (jd_done_nolock(katom, NULL)) ++ kbase_js_sched_all(kctx->kbdev); ++ mutex_unlock(&kctx->jctx.lock); ++} ++#endif ++ ++static void kbasep_soft_event_complete_job(struct work_struct *work) ++{ ++ struct kbase_jd_atom *katom = container_of(work, struct kbase_jd_atom, ++ work); ++ struct kbase_context *kctx = katom->kctx; ++ int resched; ++ ++ mutex_lock(&kctx->jctx.lock); ++ resched = jd_done_nolock(katom, NULL); ++ mutex_unlock(&kctx->jctx.lock); ++ ++ if (resched) ++ kbase_js_sched_all(kctx->kbdev); ++} ++ ++void kbasep_complete_triggered_soft_events(struct kbase_context *kctx, u64 evt) ++{ ++ int cancel_timer = 1; ++ struct list_head *entry, *tmp; ++ unsigned long lflags; ++ ++ spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); ++ list_for_each_safe(entry, tmp, &kctx->waiting_soft_jobs) { ++ struct kbase_jd_atom *katom = list_entry( ++ entry, struct kbase_jd_atom, queue); ++ ++ switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { ++ case BASE_JD_REQ_SOFT_EVENT_WAIT: ++ if (katom->jc == evt) { ++ list_del(&katom->queue); ++ ++ katom->event_code = BASE_JD_EVENT_DONE; ++ INIT_WORK(&katom->work, ++ kbasep_soft_event_complete_job); ++ queue_work(kctx->jctx.job_done_wq, ++ &katom->work); ++ } else { ++ /* There are still other waiting jobs, we cannot ++ * cancel the timer yet. ++ */ ++ cancel_timer = 0; ++ } ++ break; ++#ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG ++ case BASE_JD_REQ_SOFT_FENCE_WAIT: ++ /* Keep the timer running if fence debug is enabled and ++ * there are waiting fence jobs. ++ */ ++ cancel_timer = 0; ++ break; ++#endif ++ } ++ } ++ ++ if (cancel_timer) ++ del_timer(&kctx->soft_job_timeout); ++ spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); ++} ++ ++#ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG ++static void kbase_fence_debug_check_atom(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ struct device *dev = kctx->kbdev->dev; ++ int i; ++ ++ for (i = 0; i < 2; i++) { ++ struct kbase_jd_atom *dep; ++ ++ list_for_each_entry(dep, &katom->dep_head[i], dep_item[i]) { ++ if (dep->status == KBASE_JD_ATOM_STATE_UNUSED || ++ dep->status == KBASE_JD_ATOM_STATE_COMPLETED) ++ continue; ++ ++ if ((dep->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) ++ == BASE_JD_REQ_SOFT_FENCE_TRIGGER) { ++ /* Found blocked trigger fence. */ ++ struct kbase_sync_fence_info info; ++ ++ if (!kbase_sync_fence_in_info_get(dep, &info)) { ++ dev_warn(dev, ++ "\tVictim trigger atom %d fence [%p] %s: %s\n", ++ kbase_jd_atom_id(kctx, dep), ++ info.fence, ++ info.name, ++ kbase_sync_status_string(info.status)); ++ } ++ } ++ ++ kbase_fence_debug_check_atom(dep); ++ } ++ } ++} ++ ++static void kbase_fence_debug_wait_timeout(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ struct device *dev = katom->kctx->kbdev->dev; ++ int timeout_ms = atomic_read(&kctx->kbdev->js_data.soft_job_timeout_ms); ++ unsigned long lflags; ++ struct kbase_sync_fence_info info; ++ ++ spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); ++ ++ if (kbase_sync_fence_in_info_get(katom, &info)) { ++ /* Fence must have signaled just after timeout. */ ++ spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); ++ return; ++ } ++ ++ dev_warn(dev, "ctx %d_%d: Atom %d still waiting for fence [%p] after %dms\n", ++ kctx->tgid, kctx->id, ++ kbase_jd_atom_id(kctx, katom), ++ info.fence, timeout_ms); ++ dev_warn(dev, "\tGuilty fence [%p] %s: %s\n", ++ info.fence, info.name, ++ kbase_sync_status_string(info.status)); ++ ++ /* Search for blocked trigger atoms */ ++ kbase_fence_debug_check_atom(katom); ++ ++ spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); ++ ++ kbase_sync_fence_in_dump(katom); ++} ++ ++struct kbase_fence_debug_work { ++ struct kbase_jd_atom *katom; ++ struct work_struct work; ++}; ++ ++static void kbase_fence_debug_wait_timeout_worker(struct work_struct *work) ++{ ++ struct kbase_fence_debug_work *w = container_of(work, ++ struct kbase_fence_debug_work, work); ++ struct kbase_jd_atom *katom = w->katom; ++ struct kbase_context *kctx = katom->kctx; ++ ++ mutex_lock(&kctx->jctx.lock); ++ kbase_fence_debug_wait_timeout(katom); ++ mutex_unlock(&kctx->jctx.lock); ++ ++ kfree(w); ++} ++ ++static void kbase_fence_debug_timeout(struct kbase_jd_atom *katom) ++{ ++ struct kbase_fence_debug_work *work; ++ struct kbase_context *kctx = katom->kctx; ++ ++ /* Enqueue fence debug worker. Use job_done_wq to get ++ * debug print ordered with job completion. ++ */ ++ work = kzalloc(sizeof(struct kbase_fence_debug_work), GFP_ATOMIC); ++ /* Ignore allocation failure. */ ++ if (work) { ++ work->katom = katom; ++ INIT_WORK(&work->work, kbase_fence_debug_wait_timeout_worker); ++ queue_work(kctx->jctx.job_done_wq, &work->work); ++ } ++} ++#endif /* CONFIG_MALI_BIFROST_FENCE_DEBUG */ ++ ++void kbasep_soft_job_timeout_worker(struct timer_list *timer) ++{ ++ struct kbase_context *kctx = container_of(timer, struct kbase_context, ++ soft_job_timeout); ++ u32 timeout_ms = (u32)atomic_read( ++ &kctx->kbdev->js_data.soft_job_timeout_ms); ++ ktime_t cur_time = ktime_get(); ++ bool restarting = false; ++ unsigned long lflags; ++ struct list_head *entry, *tmp; ++ ++ spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); ++ list_for_each_safe(entry, tmp, &kctx->waiting_soft_jobs) { ++ struct kbase_jd_atom *katom = list_entry(entry, ++ struct kbase_jd_atom, queue); ++ s64 elapsed_time = ktime_to_ms(ktime_sub(cur_time, ++ katom->start_timestamp)); ++ ++ if (elapsed_time < (s64)timeout_ms) { ++ restarting = true; ++ continue; ++ } ++ ++ switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { ++ case BASE_JD_REQ_SOFT_EVENT_WAIT: ++ /* Take it out of the list to ensure that it ++ * will be cancelled in all cases ++ */ ++ list_del(&katom->queue); ++ ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ INIT_WORK(&katom->work, kbasep_soft_event_complete_job); ++ queue_work(kctx->jctx.job_done_wq, &katom->work); ++ break; ++#ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG ++ case BASE_JD_REQ_SOFT_FENCE_WAIT: ++ kbase_fence_debug_timeout(katom); ++ break; ++#endif ++ } ++ } ++ ++ if (restarting) ++ mod_timer(timer, jiffies + msecs_to_jiffies(timeout_ms)); ++ spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); ++} ++ ++static int kbasep_soft_event_wait(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ unsigned char status; ++ ++ /* The status of this soft-job is stored in jc */ ++ if (kbasep_read_soft_event_status(kctx, katom->jc, &status)) { ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ return 0; ++ } ++ ++ if (status == BASE_JD_SOFT_EVENT_SET) ++ return 0; /* Event already set, nothing to do */ ++ ++ kbasep_add_waiting_with_timeout(katom); ++ ++ return 1; ++} ++ ++static void kbasep_soft_event_update_locked(struct kbase_jd_atom *katom, ++ unsigned char new_status) ++{ ++ /* Complete jobs waiting on the same event */ ++ struct kbase_context *kctx = katom->kctx; ++ ++ if (kbasep_write_soft_event_status(kctx, katom->jc, new_status) != 0) { ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ return; ++ } ++ ++ if (new_status == BASE_JD_SOFT_EVENT_SET) ++ kbasep_complete_triggered_soft_events(kctx, katom->jc); ++} ++ ++/** ++ * kbase_soft_event_update() - Update soft event state ++ * @kctx: Pointer to context ++ * @event: Event to update ++ * @new_status: New status value of event ++ * ++ * Update the event, and wake up any atoms waiting for the event. ++ * ++ * Return: 0 on success, a negative error code on failure. ++ */ ++int kbase_soft_event_update(struct kbase_context *kctx, ++ u64 event, ++ unsigned char new_status) ++{ ++ int err = 0; ++ ++ mutex_lock(&kctx->jctx.lock); ++ ++ if (kbasep_write_soft_event_status(kctx, event, new_status)) { ++ err = -ENOENT; ++ goto out; ++ } ++ ++ if (new_status == BASE_JD_SOFT_EVENT_SET) ++ kbasep_complete_triggered_soft_events(kctx, event); ++ ++out: ++ mutex_unlock(&kctx->jctx.lock); ++ ++ return err; ++} ++ ++static void kbasep_soft_event_cancel_job(struct kbase_jd_atom *katom) ++{ ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ if (jd_done_nolock(katom, NULL)) ++ kbase_js_sched_all(katom->kctx->kbdev); ++} ++ ++static void kbase_debug_copy_finish(struct kbase_jd_atom *katom) ++{ ++ struct kbase_debug_copy_buffer *buffers = katom->softjob_data; ++ unsigned int i; ++ unsigned int nr = katom->nr_extres; ++ ++ if (!buffers) ++ return; ++ ++ kbase_gpu_vm_lock(katom->kctx); ++ for (i = 0; i < nr; i++) { ++ int p; ++ struct kbase_mem_phy_alloc *gpu_alloc = buffers[i].gpu_alloc; ++ ++ if (!buffers[i].pages) ++ break; ++ for (p = 0; p < buffers[i].nr_pages; p++) { ++ struct page *pg = buffers[i].pages[p]; ++ ++ if (pg) ++ put_page(pg); ++ } ++ if (buffers[i].is_vmalloc) ++ vfree(buffers[i].pages); ++ else ++ kfree(buffers[i].pages); ++ if (gpu_alloc) { ++ switch (gpu_alloc->type) { ++ case KBASE_MEM_TYPE_IMPORTED_USER_BUF: ++ { ++ kbase_free_user_buffer(&buffers[i]); ++ break; ++ } ++ default: ++ /* Nothing to be done. */ ++ break; ++ } ++ kbase_mem_phy_alloc_put(gpu_alloc); ++ } ++ } ++ kbase_gpu_vm_unlock(katom->kctx); ++ kfree(buffers); ++ ++ katom->softjob_data = NULL; ++} ++ ++static int kbase_debug_copy_prepare(struct kbase_jd_atom *katom) ++{ ++ struct kbase_debug_copy_buffer *buffers; ++ struct base_jd_debug_copy_buffer *user_buffers = NULL; ++ unsigned int i; ++ unsigned int nr = katom->nr_extres; ++ int ret = 0; ++ void __user *user_structs = (void __user *)(uintptr_t)katom->jc; ++ ++ if (!user_structs) ++ return -EINVAL; ++ ++ buffers = kcalloc(nr, sizeof(*buffers), GFP_KERNEL); ++ if (!buffers) { ++ ret = -ENOMEM; ++ goto out_cleanup; ++ } ++ katom->softjob_data = buffers; ++ ++ user_buffers = kmalloc_array(nr, sizeof(*user_buffers), GFP_KERNEL); ++ ++ if (!user_buffers) { ++ ret = -ENOMEM; ++ goto out_cleanup; ++ } ++ ++ ret = copy_from_user(user_buffers, user_structs, ++ sizeof(*user_buffers)*nr); ++ if (ret) { ++ ret = -EFAULT; ++ goto out_cleanup; ++ } ++ ++ for (i = 0; i < nr; i++) { ++ u64 addr = user_buffers[i].address; ++ u64 page_addr = addr & PAGE_MASK; ++ u64 end_page_addr = addr + user_buffers[i].size - 1; ++ u64 last_page_addr = end_page_addr & PAGE_MASK; ++ int nr_pages = (last_page_addr-page_addr)/PAGE_SIZE+1; ++ int pinned_pages; ++ struct kbase_va_region *reg; ++ struct base_external_resource user_extres; ++ ++ if (!addr) ++ continue; ++ ++ if (last_page_addr < page_addr) { ++ ret = -EINVAL; ++ goto out_cleanup; ++ } ++ ++ buffers[i].nr_pages = nr_pages; ++ buffers[i].offset = addr & ~PAGE_MASK; ++ if (buffers[i].offset >= PAGE_SIZE) { ++ ret = -EINVAL; ++ goto out_cleanup; ++ } ++ buffers[i].size = user_buffers[i].size; ++ ++ if (nr_pages > (KBASE_MEM_PHY_ALLOC_LARGE_THRESHOLD / ++ sizeof(struct page *))) { ++ buffers[i].is_vmalloc = true; ++ buffers[i].pages = vzalloc(nr_pages * ++ sizeof(struct page *)); ++ } else { ++ buffers[i].is_vmalloc = false; ++ buffers[i].pages = kcalloc(nr_pages, ++ sizeof(struct page *), GFP_KERNEL); ++ } ++ ++ if (!buffers[i].pages) { ++ ret = -ENOMEM; ++ goto out_cleanup; ++ } ++ ++ pinned_pages = get_user_pages_fast(page_addr, ++ nr_pages, ++ 1, /* Write */ ++ buffers[i].pages); ++ if (pinned_pages < 0) { ++ /* get_user_pages_fast has failed - page array is not ++ * valid. Don't try to release any pages. ++ */ ++ buffers[i].nr_pages = 0; ++ ++ ret = pinned_pages; ++ goto out_cleanup; ++ } ++ if (pinned_pages != nr_pages) { ++ /* Adjust number of pages, so that we only attempt to ++ * release pages in the array that we know are valid. ++ */ ++ buffers[i].nr_pages = pinned_pages; ++ ++ ret = -EINVAL; ++ goto out_cleanup; ++ } ++ ++ user_extres = user_buffers[i].extres; ++ if (user_extres.ext_resource == 0ULL) { ++ ret = -EINVAL; ++ goto out_cleanup; ++ } ++ ++ kbase_gpu_vm_lock(katom->kctx); ++ reg = kbase_region_tracker_find_region_enclosing_address( ++ katom->kctx, user_extres.ext_resource & ++ ~BASE_EXT_RES_ACCESS_EXCLUSIVE); ++ ++ if (kbase_is_region_invalid_or_free(reg) || ++ reg->gpu_alloc == NULL) { ++ ret = -EINVAL; ++ goto out_unlock; ++ } ++ ++ buffers[i].gpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); ++ buffers[i].nr_extres_pages = reg->nr_pages; ++ ++ if (reg->nr_pages*PAGE_SIZE != buffers[i].size) ++ dev_warn(katom->kctx->kbdev->dev, "Copy buffer is not of same size as the external resource to copy.\n"); ++ ++ switch (reg->gpu_alloc->type) { ++ case KBASE_MEM_TYPE_IMPORTED_USER_BUF: ++ { ++ struct kbase_mem_phy_alloc *alloc = reg->gpu_alloc; ++ unsigned long nr_pages = ++ alloc->imported.user_buf.nr_pages; ++ ++ if (alloc->imported.user_buf.mm != current->mm) { ++ ret = -EINVAL; ++ goto out_unlock; ++ } ++ buffers[i].extres_pages = kcalloc(nr_pages, ++ sizeof(struct page *), GFP_KERNEL); ++ if (!buffers[i].extres_pages) { ++ ret = -ENOMEM; ++ goto out_unlock; ++ } ++ ++ ret = get_user_pages_fast( ++ alloc->imported.user_buf.address, ++ nr_pages, 0, ++ buffers[i].extres_pages); ++ if (ret != nr_pages) { ++ /* Adjust number of pages, so that we only ++ * attempt to release pages in the array that we ++ * know are valid. ++ */ ++ if (ret < 0) ++ buffers[i].nr_extres_pages = 0; ++ else ++ buffers[i].nr_extres_pages = ret; ++ ++ goto out_unlock; ++ } ++ ret = 0; ++ break; ++ } ++ default: ++ /* Nothing to be done. */ ++ break; ++ } ++ kbase_gpu_vm_unlock(katom->kctx); ++ } ++ kfree(user_buffers); ++ ++ return ret; ++ ++out_unlock: ++ kbase_gpu_vm_unlock(katom->kctx); ++ ++out_cleanup: ++ /* Frees allocated memory for kbase_debug_copy_job struct, including ++ * members, and sets jc to 0 */ ++ kbase_debug_copy_finish(katom); ++ kfree(user_buffers); ++ ++ return ret; ++} ++#endif /* !MALI_USE_CSF */ ++ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0) ++static void *dma_buf_kmap_page(struct kbase_mem_phy_alloc *gpu_alloc, ++ unsigned long page_num, struct page **page) ++{ ++ struct sg_table *sgt = gpu_alloc->imported.umm.sgt; ++ struct sg_page_iter sg_iter; ++ unsigned long page_index = 0; ++ ++ if (WARN_ON(gpu_alloc->type != KBASE_MEM_TYPE_IMPORTED_UMM)) ++ return NULL; ++ ++ if (!sgt) ++ return NULL; ++ ++ if (WARN_ON(page_num >= gpu_alloc->nents)) ++ return NULL; ++ ++ for_each_sg_page(sgt->sgl, &sg_iter, sgt->nents, 0) { ++ if (page_index == page_num) { ++ *page = sg_page_iter_page(&sg_iter); ++ ++ return kmap(*page); ++ } ++ page_index++; ++ } ++ ++ return NULL; ++} ++#endif ++ ++int kbase_mem_copy_from_extres(struct kbase_context *kctx, ++ struct kbase_debug_copy_buffer *buf_data) ++{ ++ unsigned int i; ++ unsigned int target_page_nr = 0; ++ struct page **pages = buf_data->pages; ++ u64 offset = buf_data->offset; ++ size_t extres_size = buf_data->nr_extres_pages*PAGE_SIZE; ++ size_t to_copy = min(extres_size, buf_data->size); ++ struct kbase_mem_phy_alloc *gpu_alloc = buf_data->gpu_alloc; ++ int ret = 0; ++ size_t dma_to_copy; ++ ++ KBASE_DEBUG_ASSERT(pages != NULL); ++ ++ kbase_gpu_vm_lock(kctx); ++ if (!gpu_alloc) { ++ ret = -EINVAL; ++ goto out_unlock; ++ } ++ ++ switch (gpu_alloc->type) { ++ case KBASE_MEM_TYPE_IMPORTED_USER_BUF: ++ { ++ for (i = 0; i < buf_data->nr_extres_pages && ++ target_page_nr < buf_data->nr_pages; i++) { ++ struct page *pg = buf_data->extres_pages[i]; ++ void *extres_page = kmap(pg); ++ ++ if (extres_page) { ++ ret = kbase_mem_copy_to_pinned_user_pages( ++ pages, extres_page, &to_copy, ++ buf_data->nr_pages, ++ &target_page_nr, offset); ++ kunmap(pg); ++ if (ret) ++ goto out_unlock; ++ } ++ } ++ } ++ break; ++ case KBASE_MEM_TYPE_IMPORTED_UMM: { ++ struct dma_buf *dma_buf = gpu_alloc->imported.umm.dma_buf; ++ ++ KBASE_DEBUG_ASSERT(dma_buf != NULL); ++ if (dma_buf->size > buf_data->nr_extres_pages * PAGE_SIZE) ++ dev_warn(kctx->kbdev->dev, "External resources buffer size mismatch"); ++ ++ dma_to_copy = min(dma_buf->size, ++ (size_t)(buf_data->nr_extres_pages * PAGE_SIZE)); ++ ret = dma_buf_begin_cpu_access(dma_buf, ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) && !defined(CONFIG_CHROMEOS) ++ 0, dma_to_copy, ++#endif ++ DMA_FROM_DEVICE); ++ if (ret) ++ goto out_unlock; ++ ++ for (i = 0; i < dma_to_copy/PAGE_SIZE && ++ target_page_nr < buf_data->nr_pages; i++) { ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0) ++ struct page *pg; ++ void *extres_page = dma_buf_kmap_page(gpu_alloc, i, &pg); ++#else ++ void *extres_page = dma_buf_kmap(dma_buf, i); ++#endif ++ if (extres_page) { ++ ret = kbase_mem_copy_to_pinned_user_pages( ++ pages, extres_page, &to_copy, ++ buf_data->nr_pages, ++ &target_page_nr, offset); ++ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0) ++ kunmap(pg); ++#else ++ dma_buf_kunmap(dma_buf, i, extres_page); ++#endif ++ if (ret) ++ goto out_unlock; ++ } ++ } ++ dma_buf_end_cpu_access(dma_buf, ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) && !defined(CONFIG_CHROMEOS) ++ 0, dma_to_copy, ++#endif ++ DMA_FROM_DEVICE); ++ break; ++ } ++ default: ++ ret = -EINVAL; ++ } ++out_unlock: ++ kbase_gpu_vm_unlock(kctx); ++ return ret; ++} ++ ++#if !MALI_USE_CSF ++static int kbase_debug_copy(struct kbase_jd_atom *katom) ++{ ++ struct kbase_debug_copy_buffer *buffers = katom->softjob_data; ++ unsigned int i; ++ ++ if (WARN_ON(!buffers)) ++ return -EINVAL; ++ ++ for (i = 0; i < katom->nr_extres; i++) { ++ int res = kbase_mem_copy_from_extres(katom->kctx, &buffers[i]); ++ ++ if (res) ++ return res; ++ } ++ ++ return 0; ++} ++#endif /* !MALI_USE_CSF */ ++ ++#define KBASEP_JIT_ALLOC_GPU_ADDR_ALIGNMENT ((u32)0x7) ++ ++int kbasep_jit_alloc_validate(struct kbase_context *kctx, ++ struct base_jit_alloc_info *info) ++{ ++ int j; ++ /* If the ID is zero, then fail the job */ ++ if (info->id == 0) ++ return -EINVAL; ++ ++ /* Sanity check that the PA fits within the VA */ ++ if (info->va_pages < info->commit_pages) ++ return -EINVAL; ++ ++ /* Ensure the GPU address is correctly aligned */ ++ if ((info->gpu_alloc_addr & KBASEP_JIT_ALLOC_GPU_ADDR_ALIGNMENT) != 0) ++ return -EINVAL; ++ ++ /* Interface version 2 (introduced with kernel driver version 11.5) ++ * onward has padding and a flags member to validate. ++ * ++ * Note: To support earlier versions the extra bytes will have been set ++ * to 0 by the caller. ++ */ ++ ++ /* Check padding is all zeroed */ ++ for (j = 0; j < sizeof(info->padding); j++) { ++ if (info->padding[j] != 0) ++ return -EINVAL; ++ } ++ ++ /* Only valid flags shall be set */ ++ if (info->flags & ~(BASE_JIT_ALLOC_VALID_FLAGS)) ++ return -EINVAL; ++ ++#if !MALI_JIT_PRESSURE_LIMIT_BASE ++ /* If just-in-time memory allocation pressure limit feature is disabled, ++ * heap_info_gpu_addr must be zeroed-out ++ */ ++ if (info->heap_info_gpu_addr) ++ return -EINVAL; ++#endif ++ ++#if !MALI_USE_CSF ++ /* If BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE is set, heap_info_gpu_addr ++ * cannot be 0 ++ */ ++ if ((info->flags & BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE) && ++ !info->heap_info_gpu_addr) ++ return -EINVAL; ++#endif /* !MALI_USE_CSF */ ++ ++ return 0; ++} ++ ++#if !MALI_USE_CSF ++ ++#if (KERNEL_VERSION(3, 18, 63) > LINUX_VERSION_CODE) ++#define offsetofend(TYPE, MEMBER) \ ++ (offsetof(TYPE, MEMBER) + sizeof(((TYPE *)0)->MEMBER)) ++#endif ++ ++/* ++ * Sizes of user data to copy for each just-in-time memory interface version ++ * ++ * In interface version 2 onwards this is the same as the struct size, allowing ++ * copying of arrays of structures from userspace. ++ * ++ * In interface version 1 the structure size was variable, and hence arrays of ++ * structures cannot be supported easily, and were not a feature present in ++ * version 1 anyway. ++ */ ++static const size_t jit_info_copy_size_for_jit_version[] = { ++ /* in jit_version 1, the structure did not have any end padding, hence ++ * it could be a different size on 32 and 64-bit clients. We therefore ++ * do not copy past the last member ++ */ ++ [1] = offsetofend(struct base_jit_alloc_info_10_2, id), ++ [2] = sizeof(struct base_jit_alloc_info_11_5), ++ [3] = sizeof(struct base_jit_alloc_info) ++}; ++ ++static int kbase_jit_allocate_prepare(struct kbase_jd_atom *katom) ++{ ++ __user u8 *data = (__user u8 *)(uintptr_t) katom->jc; ++ struct base_jit_alloc_info *info; ++ struct kbase_context *kctx = katom->kctx; ++ struct kbase_device *kbdev = kctx->kbdev; ++ u32 count; ++ int ret; ++ u32 i; ++ size_t jit_info_user_copy_size; ++ ++ WARN_ON(kctx->jit_version >= ++ ARRAY_SIZE(jit_info_copy_size_for_jit_version)); ++ jit_info_user_copy_size = ++ jit_info_copy_size_for_jit_version[kctx->jit_version]; ++ WARN_ON(jit_info_user_copy_size > sizeof(*info)); ++ ++ /* For backwards compatibility, and to prevent reading more than 1 jit ++ * info struct on jit version 1 ++ */ ++ if (katom->nr_extres == 0 || kctx->jit_version == 1) ++ katom->nr_extres = 1; ++ count = katom->nr_extres; ++ ++ /* Sanity checks */ ++ if (!data || count > kctx->jit_max_allocations || ++ count > ARRAY_SIZE(kctx->jit_alloc)) { ++ ret = -EINVAL; ++ goto fail; ++ } ++ ++ /* Copy the information for safe access and future storage */ ++ info = kmalloc_array(count, sizeof(*info), GFP_KERNEL); ++ if (!info) { ++ ret = -ENOMEM; ++ goto fail; ++ } ++ ++ katom->softjob_data = info; ++ ++ for (i = 0; i < count; i++, info++, data += jit_info_user_copy_size) { ++ if (copy_from_user(info, data, jit_info_user_copy_size) != 0) { ++ ret = -EINVAL; ++ goto free_info; ++ } ++ /* Clear any remaining bytes when user struct is smaller than ++ * kernel struct. For jit version 1, this also clears the ++ * padding bytes ++ */ ++ memset(((u8 *)info) + jit_info_user_copy_size, 0, ++ sizeof(*info) - jit_info_user_copy_size); ++ ++ ret = kbasep_jit_alloc_validate(kctx, info); ++ if (ret) ++ goto free_info; ++ KBASE_TLSTREAM_TL_ATTRIB_ATOM_JITALLOCINFO(kbdev, katom, ++ info->va_pages, info->commit_pages, info->extent, ++ info->id, info->bin_id, info->max_allocations, ++ info->flags, info->usage_id); ++ } ++ ++ katom->jit_blocked = false; ++ ++ lockdep_assert_held(&kctx->jctx.lock); ++ list_add_tail(&katom->jit_node, &kctx->jctx.jit_atoms_head); ++ ++ /* ++ * Note: ++ * The provided info->gpu_alloc_addr isn't validated here as ++ * userland can cache allocations which means that even ++ * though the region is valid it doesn't represent the ++ * same thing it used to. ++ * ++ * Complete validation of va_pages, commit_pages and extent ++ * isn't done here as it will be done during the call to ++ * kbase_mem_alloc. ++ */ ++ return 0; ++ ++free_info: ++ kfree(katom->softjob_data); ++ katom->softjob_data = NULL; ++fail: ++ return ret; ++} ++ ++static u8 *kbase_jit_free_get_ids(struct kbase_jd_atom *katom) ++{ ++ if (WARN_ON((katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) != ++ BASE_JD_REQ_SOFT_JIT_FREE)) ++ return NULL; ++ ++ return (u8 *) katom->softjob_data; ++} ++ ++static void kbase_jit_add_to_pending_alloc_list(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ struct list_head *target_list_head = NULL; ++ struct kbase_jd_atom *entry; ++ ++ list_for_each_entry(entry, &kctx->jctx.jit_pending_alloc, queue) { ++ if (katom->age < entry->age) { ++ target_list_head = &entry->queue; ++ break; ++ } ++ } ++ ++ if (target_list_head == NULL) ++ target_list_head = &kctx->jctx.jit_pending_alloc; ++ ++ list_add_tail(&katom->queue, target_list_head); ++} ++ ++static int kbase_jit_allocate_process(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ struct kbase_device *kbdev = kctx->kbdev; ++ struct base_jit_alloc_info *info; ++ struct kbase_va_region *reg; ++ struct kbase_vmap_struct mapping; ++ u64 *ptr, new_addr; ++ u32 count = katom->nr_extres; ++ u32 i; ++ bool ignore_pressure_limit = false; ++ ++ trace_sysgraph(SGR_SUBMIT, kctx->id, ++ kbase_jd_atom_id(kctx, katom)); ++ ++ if (katom->jit_blocked) { ++ list_del(&katom->queue); ++ katom->jit_blocked = false; ++ } ++ ++ info = katom->softjob_data; ++ if (WARN_ON(!info)) { ++ katom->event_code = BASE_JD_EVENT_JOB_INVALID; ++ return 0; ++ } ++ ++ for (i = 0; i < count; i++, info++) { ++ /* The JIT ID is still in use so fail the allocation */ ++ if (kctx->jit_alloc[info->id]) { ++ katom->event_code = BASE_JD_EVENT_MEM_GROWTH_FAILED; ++ return 0; ++ } ++ } ++ ++#if MALI_JIT_PRESSURE_LIMIT_BASE ++ /** ++ * If this is the only JIT_ALLOC atom in-flight or if JIT pressure limit ++ * is disabled at the context scope, then bypass JIT pressure limit ++ * logic in kbase_jit_allocate(). ++ */ ++ if (!kbase_ctx_flag(kctx, KCTX_JPL_ENABLED) ++ || (kctx->jit_current_allocations == 0)) { ++ ignore_pressure_limit = true; ++ } ++#else ++ ignore_pressure_limit = true; ++#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ ++ ++ for (i = 0, info = katom->softjob_data; i < count; i++, info++) { ++ if (kctx->jit_alloc[info->id]) { ++ /* The JIT ID is duplicated in this atom. Roll back ++ * previous allocations and fail. ++ */ ++ u32 j; ++ ++ info = katom->softjob_data; ++ for (j = 0; j < i; j++, info++) { ++ kbase_jit_free(kctx, kctx->jit_alloc[info->id]); ++ kctx->jit_alloc[info->id] = ++ KBASE_RESERVED_REG_JIT_ALLOC; ++ } ++ ++ katom->event_code = BASE_JD_EVENT_MEM_GROWTH_FAILED; ++ return 0; ++ } ++ ++ /* Create a JIT allocation */ ++ reg = kbase_jit_allocate(kctx, info, ignore_pressure_limit); ++ if (!reg) { ++ struct kbase_jd_atom *jit_atom; ++ bool can_block = false; ++ ++ lockdep_assert_held(&kctx->jctx.lock); ++ ++ list_for_each_entry(jit_atom, &kctx->jctx.jit_atoms_head, jit_node) { ++ if (jit_atom == katom) ++ break; ++ ++ if ((jit_atom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) == ++ BASE_JD_REQ_SOFT_JIT_FREE) { ++ u8 *free_ids = kbase_jit_free_get_ids(jit_atom); ++ ++ if (free_ids && *free_ids && ++ kctx->jit_alloc[*free_ids]) { ++ /* A JIT free which is active and ++ * submitted before this atom ++ */ ++ can_block = true; ++ break; ++ } ++ } ++ } ++ ++ if (!can_block) { ++ /* Mark the failed allocation as well as the ++ * other un-attempted allocations in the set, ++ * so we know they are in use even if the ++ * allocation itself failed. ++ */ ++ for (; i < count; i++, info++) { ++ kctx->jit_alloc[info->id] = ++ KBASE_RESERVED_REG_JIT_ALLOC; ++ } ++ ++ katom->event_code = BASE_JD_EVENT_MEM_GROWTH_FAILED; ++ dev_warn_ratelimited(kbdev->dev, "JIT alloc softjob failed: atom id %d\n", ++ kbase_jd_atom_id(kctx, katom)); ++ return 0; ++ } ++ ++ /* There are pending frees for an active allocation ++ * so we should wait to see whether they free the ++ * memory. Add to the list of atoms for which JIT ++ * allocation is pending. ++ */ ++ kbase_jit_add_to_pending_alloc_list(katom); ++ katom->jit_blocked = true; ++ ++ /* Rollback, the whole set will be re-attempted */ ++ while (i-- > 0) { ++ info--; ++ kbase_jit_free(kctx, kctx->jit_alloc[info->id]); ++ kctx->jit_alloc[info->id] = NULL; ++ } ++ ++ return 1; ++ } ++ ++ /* Bind it to the user provided ID. */ ++ kctx->jit_alloc[info->id] = reg; ++ } ++ ++ for (i = 0, info = katom->softjob_data; i < count; i++, info++) { ++ u64 entry_mmu_flags = 0; ++ /* ++ * Write the address of the JIT allocation to the user provided ++ * GPU allocation. ++ */ ++ ptr = kbase_vmap(kctx, info->gpu_alloc_addr, sizeof(*ptr), ++ &mapping); ++ if (!ptr) { ++ /* ++ * Leave the allocations "live" as the JIT free atom ++ * will be submitted anyway. ++ */ ++ katom->event_code = BASE_JD_EVENT_JOB_INVALID; ++ return 0; ++ } ++ ++ reg = kctx->jit_alloc[info->id]; ++ new_addr = reg->start_pfn << PAGE_SHIFT; ++ *ptr = new_addr; ++ ++#if defined(CONFIG_MALI_VECTOR_DUMP) ++ /* ++ * Retrieve the mmu flags for JIT allocation ++ * only if dumping is enabled ++ */ ++ entry_mmu_flags = kbase_mmu_create_ate(kbdev, ++ (struct tagged_addr){ 0 }, reg->flags, ++ MIDGARD_MMU_BOTTOMLEVEL, kctx->jit_group_id); ++#endif ++ ++ KBASE_TLSTREAM_TL_ATTRIB_ATOM_JIT(kbdev, katom, ++ info->gpu_alloc_addr, new_addr, info->flags, ++ entry_mmu_flags, info->id, info->commit_pages, ++ info->extent, info->va_pages); ++ kbase_vunmap(kctx, &mapping); ++ ++ kbase_trace_jit_report_gpu_mem(kctx, reg, ++ KBASE_JIT_REPORT_ON_ALLOC_OR_FREE); ++ } ++ ++ katom->event_code = BASE_JD_EVENT_DONE; ++ ++ return 0; ++} ++ ++static void kbase_jit_allocate_finish(struct kbase_jd_atom *katom) ++{ ++ struct base_jit_alloc_info *info; ++ ++ lockdep_assert_held(&katom->kctx->jctx.lock); ++ ++ if (WARN_ON(!katom->softjob_data)) ++ return; ++ ++ /* Remove atom from jit_atoms_head list */ ++ list_del(&katom->jit_node); ++ ++ if (katom->jit_blocked) { ++ list_del(&katom->queue); ++ katom->jit_blocked = false; ++ } ++ ++ info = katom->softjob_data; ++ /* Free the info structure */ ++ kfree(info); ++} ++ ++static int kbase_jit_free_prepare(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ struct kbase_device *kbdev = kctx->kbdev; ++ __user void *data = (__user void *)(uintptr_t) katom->jc; ++ u8 *ids; ++ u32 count = MAX(katom->nr_extres, 1); ++ u32 i; ++ int ret; ++ ++ /* Sanity checks */ ++ if (count > ARRAY_SIZE(kctx->jit_alloc)) { ++ ret = -EINVAL; ++ goto fail; ++ } ++ ++ /* Copy the information for safe access and future storage */ ++ ids = kmalloc_array(count, sizeof(*ids), GFP_KERNEL); ++ if (!ids) { ++ ret = -ENOMEM; ++ goto fail; ++ } ++ ++ lockdep_assert_held(&kctx->jctx.lock); ++ katom->softjob_data = ids; ++ ++ /* For backwards compatibility */ ++ if (katom->nr_extres) { ++ /* Fail the job if there is no list of ids */ ++ if (!data) { ++ ret = -EINVAL; ++ goto free_info; ++ } ++ ++ if (copy_from_user(ids, data, sizeof(*ids)*count) != 0) { ++ ret = -EINVAL; ++ goto free_info; ++ } ++ } else { ++ katom->nr_extres = 1; ++ *ids = (u8)katom->jc; ++ } ++ for (i = 0; i < count; i++) ++ KBASE_TLSTREAM_TL_ATTRIB_ATOM_JITFREEINFO(kbdev, katom, ids[i]); ++ ++ list_add_tail(&katom->jit_node, &kctx->jctx.jit_atoms_head); ++ ++ return 0; ++ ++free_info: ++ kfree(katom->softjob_data); ++ katom->softjob_data = NULL; ++fail: ++ return ret; ++} ++ ++static void kbase_jit_free_process(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ u8 *ids = kbase_jit_free_get_ids(katom); ++ u32 count = katom->nr_extres; ++ u32 i; ++ ++ if (ids == NULL) { ++ katom->event_code = BASE_JD_EVENT_JOB_INVALID; ++ return; ++ } ++ ++ for (i = 0; i < count; i++, ids++) { ++ /* ++ * If the ID is zero or it is not in use yet then fail the job. ++ */ ++ if ((*ids == 0) || (kctx->jit_alloc[*ids] == NULL)) { ++ katom->event_code = BASE_JD_EVENT_JOB_INVALID; ++ return; ++ } ++ } ++} ++ ++static void kbasep_jit_finish_worker(struct work_struct *work) ++{ ++ struct kbase_jd_atom *katom = container_of(work, struct kbase_jd_atom, ++ work); ++ struct kbase_context *kctx = katom->kctx; ++ int resched; ++ ++ mutex_lock(&kctx->jctx.lock); ++ kbase_finish_soft_job(katom); ++ resched = jd_done_nolock(katom, NULL); ++ mutex_unlock(&kctx->jctx.lock); ++ ++ if (resched) ++ kbase_js_sched_all(kctx->kbdev); ++} ++ ++void kbase_jit_retry_pending_alloc(struct kbase_context *kctx) ++{ ++ LIST_HEAD(jit_pending_alloc_list); ++ struct list_head *i, *tmp; ++ ++ list_splice_tail_init(&kctx->jctx.jit_pending_alloc, ++ &jit_pending_alloc_list); ++ ++ list_for_each_safe(i, tmp, &jit_pending_alloc_list) { ++ struct kbase_jd_atom *pending_atom = list_entry(i, ++ struct kbase_jd_atom, queue); ++ KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTJOB_START(kctx->kbdev, pending_atom); ++ kbase_kinstr_jm_atom_sw_start(pending_atom); ++ if (kbase_jit_allocate_process(pending_atom) == 0) { ++ /* Atom has completed */ ++ INIT_WORK(&pending_atom->work, ++ kbasep_jit_finish_worker); ++ queue_work(kctx->jctx.job_done_wq, &pending_atom->work); ++ } ++ KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTJOB_END(kctx->kbdev, pending_atom); ++ kbase_kinstr_jm_atom_sw_stop(pending_atom); ++ } ++} ++ ++static void kbase_jit_free_finish(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ u8 *ids; ++ size_t j; ++ ++ lockdep_assert_held(&kctx->jctx.lock); ++ ++ ids = kbase_jit_free_get_ids(katom); ++ if (WARN_ON(ids == NULL)) { ++ return; ++ } ++ ++ /* Remove this atom from the jit_atoms_head list */ ++ list_del(&katom->jit_node); ++ ++ for (j = 0; j != katom->nr_extres; ++j) { ++ if ((ids[j] != 0) && (kctx->jit_alloc[ids[j]] != NULL)) { ++ /* ++ * If the ID is valid but the allocation request failed ++ * still succeed this soft job but don't try and free ++ * the allocation. ++ */ ++ if (kctx->jit_alloc[ids[j]] != ++ KBASE_RESERVED_REG_JIT_ALLOC) { ++ KBASE_TLSTREAM_TL_JIT_USEDPAGES(kctx->kbdev, ++ kctx->jit_alloc[ids[j]]-> ++ gpu_alloc->nents, ids[j]); ++ kbase_jit_free(kctx, kctx->jit_alloc[ids[j]]); ++ } ++ kctx->jit_alloc[ids[j]] = NULL; ++ } ++ } ++ /* Free the list of ids */ ++ kfree(ids); ++ ++ kbase_jit_retry_pending_alloc(kctx); ++} ++ ++static int kbase_ext_res_prepare(struct kbase_jd_atom *katom) ++{ ++ __user struct base_external_resource_list *user_ext_res; ++ struct base_external_resource_list *ext_res; ++ u64 count = 0; ++ size_t copy_size; ++ int ret; ++ ++ user_ext_res = (__user struct base_external_resource_list *) ++ (uintptr_t) katom->jc; ++ ++ /* Fail the job if there is no info structure */ ++ if (!user_ext_res) { ++ ret = -EINVAL; ++ goto fail; ++ } ++ ++ if (copy_from_user(&count, &user_ext_res->count, sizeof(u64)) != 0) { ++ ret = -EINVAL; ++ goto fail; ++ } ++ ++ /* Is the number of external resources in range? */ ++ if (!count || count > BASE_EXT_RES_COUNT_MAX) { ++ ret = -EINVAL; ++ goto fail; ++ } ++ ++ /* Copy the information for safe access and future storage */ ++ copy_size = sizeof(*ext_res); ++ copy_size += sizeof(struct base_external_resource) * (count - 1); ++ ext_res = kzalloc(copy_size, GFP_KERNEL); ++ if (!ext_res) { ++ ret = -ENOMEM; ++ goto fail; ++ } ++ ++ if (copy_from_user(ext_res, user_ext_res, copy_size) != 0) { ++ ret = -EINVAL; ++ goto free_info; ++ } ++ ++ /* ++ * Overwrite the count with the first value incase it was changed ++ * after the fact. ++ */ ++ ext_res->count = count; ++ ++ katom->softjob_data = ext_res; ++ ++ return 0; ++ ++free_info: ++ kfree(ext_res); ++fail: ++ return ret; ++} ++ ++static void kbase_ext_res_process(struct kbase_jd_atom *katom, bool map) ++{ ++ struct base_external_resource_list *ext_res; ++ int i; ++ bool failed = false; ++ ++ ext_res = katom->softjob_data; ++ if (!ext_res) ++ goto failed_jc; ++ ++ kbase_gpu_vm_lock(katom->kctx); ++ ++ for (i = 0; i < ext_res->count; i++) { ++ u64 gpu_addr; ++ ++ gpu_addr = ext_res->ext_res[i].ext_resource & ++ ~BASE_EXT_RES_ACCESS_EXCLUSIVE; ++ if (map) { ++ if (!kbase_sticky_resource_acquire(katom->kctx, ++ gpu_addr)) ++ goto failed_loop; ++ } else ++ if (!kbase_sticky_resource_release_force(katom->kctx, NULL, ++ gpu_addr)) ++ failed = true; ++ } ++ ++ /* ++ * In the case of unmap we continue unmapping other resources in the ++ * case of failure but will always report failure if _any_ unmap ++ * request fails. ++ */ ++ if (failed) ++ katom->event_code = BASE_JD_EVENT_JOB_INVALID; ++ else ++ katom->event_code = BASE_JD_EVENT_DONE; ++ ++ kbase_gpu_vm_unlock(katom->kctx); ++ ++ return; ++ ++failed_loop: ++ while (i > 0) { ++ u64 const gpu_addr = ext_res->ext_res[i - 1].ext_resource & ++ ~BASE_EXT_RES_ACCESS_EXCLUSIVE; ++ ++ kbase_sticky_resource_release_force(katom->kctx, NULL, gpu_addr); ++ ++ --i; ++ } ++ ++ katom->event_code = BASE_JD_EVENT_JOB_INVALID; ++ kbase_gpu_vm_unlock(katom->kctx); ++ ++failed_jc: ++ return; ++} ++ ++static void kbase_ext_res_finish(struct kbase_jd_atom *katom) ++{ ++ struct base_external_resource_list *ext_res; ++ ++ ext_res = katom->softjob_data; ++ /* Free the info structure */ ++ kfree(ext_res); ++} ++ ++int kbase_process_soft_job(struct kbase_jd_atom *katom) ++{ ++ int ret = 0; ++ struct kbase_context *kctx = katom->kctx; ++ struct kbase_device *kbdev = kctx->kbdev; ++ ++ KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTJOB_START(kbdev, katom); ++ kbase_kinstr_jm_atom_sw_start(katom); ++ ++ trace_sysgraph(SGR_SUBMIT, kctx->id, ++ kbase_jd_atom_id(kctx, katom)); ++ ++ switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { ++ case BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME: ++ ret = kbase_dump_cpu_gpu_time(katom); ++ break; ++ ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ case BASE_JD_REQ_SOFT_FENCE_TRIGGER: ++ katom->event_code = kbase_sync_fence_out_trigger(katom, ++ katom->event_code == BASE_JD_EVENT_DONE ? ++ 0 : -EFAULT); ++ break; ++ case BASE_JD_REQ_SOFT_FENCE_WAIT: ++ { ++ ret = kbase_sync_fence_in_wait(katom); ++ ++ if (ret == 1) { ++#ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG ++ kbasep_add_waiting_with_timeout(katom); ++#else ++ kbasep_add_waiting_soft_job(katom); ++#endif ++ } ++ break; ++ } ++#endif ++ case BASE_JD_REQ_SOFT_EVENT_WAIT: ++ ret = kbasep_soft_event_wait(katom); ++ break; ++ case BASE_JD_REQ_SOFT_EVENT_SET: ++ kbasep_soft_event_update_locked(katom, BASE_JD_SOFT_EVENT_SET); ++ break; ++ case BASE_JD_REQ_SOFT_EVENT_RESET: ++ kbasep_soft_event_update_locked(katom, BASE_JD_SOFT_EVENT_RESET); ++ break; ++ case BASE_JD_REQ_SOFT_DEBUG_COPY: ++ { ++ int res = kbase_debug_copy(katom); ++ ++ if (res) ++ katom->event_code = BASE_JD_EVENT_JOB_INVALID; ++ break; ++ } ++ case BASE_JD_REQ_SOFT_JIT_ALLOC: ++ ret = kbase_jit_allocate_process(katom); ++ break; ++ case BASE_JD_REQ_SOFT_JIT_FREE: ++ kbase_jit_free_process(katom); ++ break; ++ case BASE_JD_REQ_SOFT_EXT_RES_MAP: ++ kbase_ext_res_process(katom, true); ++ break; ++ case BASE_JD_REQ_SOFT_EXT_RES_UNMAP: ++ kbase_ext_res_process(katom, false); ++ break; ++ } ++ ++ /* Atom is complete */ ++ KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTJOB_END(kbdev, katom); ++ kbase_kinstr_jm_atom_sw_stop(katom); ++ return ret; ++} ++ ++void kbase_cancel_soft_job(struct kbase_jd_atom *katom) ++{ ++ switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ case BASE_JD_REQ_SOFT_FENCE_WAIT: ++ kbase_sync_fence_in_cancel_wait(katom); ++ break; ++#endif ++ case BASE_JD_REQ_SOFT_EVENT_WAIT: ++ kbasep_soft_event_cancel_job(katom); ++ break; ++ default: ++ /* This soft-job doesn't support cancellation! */ ++ KBASE_DEBUG_ASSERT(0); ++ } ++} ++ ++int kbase_prepare_soft_job(struct kbase_jd_atom *katom) ++{ ++ switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { ++ case BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME: ++ { ++ if (!IS_ALIGNED(katom->jc, cache_line_size())) ++ return -EINVAL; ++ } ++ break; ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ case BASE_JD_REQ_SOFT_FENCE_TRIGGER: ++ { ++ struct base_fence fence; ++ int fd; ++ ++ if (0 != copy_from_user(&fence, (__user void *)(uintptr_t) katom->jc, sizeof(fence))) ++ return -EINVAL; ++ ++ fd = kbase_sync_fence_out_create(katom, ++ fence.basep.stream_fd); ++ if (fd < 0) ++ return -EINVAL; ++ ++ fence.basep.fd = fd; ++ if (0 != copy_to_user((__user void *)(uintptr_t) katom->jc, &fence, sizeof(fence))) { ++ kbase_sync_fence_out_remove(katom); ++ kbase_sync_fence_close_fd(fd); ++ fence.basep.fd = -EINVAL; ++ return -EINVAL; ++ } ++ } ++ break; ++ case BASE_JD_REQ_SOFT_FENCE_WAIT: ++ { ++ struct base_fence fence; ++ int ret; ++ ++ if (0 != copy_from_user(&fence, (__user void *)(uintptr_t) katom->jc, sizeof(fence))) ++ return -EINVAL; ++ ++ /* Get a reference to the fence object */ ++ ret = kbase_sync_fence_in_from_fd(katom, ++ fence.basep.fd); ++ if (ret < 0) ++ return ret; ++ ++#ifdef CONFIG_MALI_BIFROST_DMA_FENCE ++ /* ++ * Set KCTX_NO_IMPLICIT_FENCE in the context the first ++ * time a soft fence wait job is observed. This will ++ * prevent the implicit dma-buf fence to conflict with ++ * the Android native sync fences. ++ */ ++ if (!kbase_ctx_flag(katom->kctx, KCTX_NO_IMPLICIT_SYNC)) ++ kbase_ctx_flag_set(katom->kctx, KCTX_NO_IMPLICIT_SYNC); ++#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ ++ } ++ break; ++#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ ++ case BASE_JD_REQ_SOFT_JIT_ALLOC: ++ return kbase_jit_allocate_prepare(katom); ++ case BASE_JD_REQ_SOFT_JIT_FREE: ++ return kbase_jit_free_prepare(katom); ++ case BASE_JD_REQ_SOFT_EVENT_WAIT: ++ case BASE_JD_REQ_SOFT_EVENT_SET: ++ case BASE_JD_REQ_SOFT_EVENT_RESET: ++ if (katom->jc == 0) ++ return -EINVAL; ++ break; ++ case BASE_JD_REQ_SOFT_DEBUG_COPY: ++ return kbase_debug_copy_prepare(katom); ++ case BASE_JD_REQ_SOFT_EXT_RES_MAP: ++ return kbase_ext_res_prepare(katom); ++ case BASE_JD_REQ_SOFT_EXT_RES_UNMAP: ++ return kbase_ext_res_prepare(katom); ++ default: ++ /* Unsupported soft-job */ ++ return -EINVAL; ++ } ++ return 0; ++} ++ ++void kbase_finish_soft_job(struct kbase_jd_atom *katom) ++{ ++ trace_sysgraph(SGR_COMPLETE, katom->kctx->id, ++ kbase_jd_atom_id(katom->kctx, katom)); ++ ++ switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { ++ case BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME: ++ /* Nothing to do */ ++ break; ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ case BASE_JD_REQ_SOFT_FENCE_TRIGGER: ++ /* If fence has not yet been signaled, do it now */ ++ kbase_sync_fence_out_trigger(katom, katom->event_code == ++ BASE_JD_EVENT_DONE ? 0 : -EFAULT); ++ break; ++ case BASE_JD_REQ_SOFT_FENCE_WAIT: ++ /* Release katom's reference to fence object */ ++ kbase_sync_fence_in_remove(katom); ++ break; ++#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ ++ case BASE_JD_REQ_SOFT_DEBUG_COPY: ++ kbase_debug_copy_finish(katom); ++ break; ++ case BASE_JD_REQ_SOFT_JIT_ALLOC: ++ kbase_jit_allocate_finish(katom); ++ break; ++ case BASE_JD_REQ_SOFT_EXT_RES_MAP: ++ kbase_ext_res_finish(katom); ++ break; ++ case BASE_JD_REQ_SOFT_EXT_RES_UNMAP: ++ kbase_ext_res_finish(katom); ++ break; ++ case BASE_JD_REQ_SOFT_JIT_FREE: ++ kbase_jit_free_finish(katom); ++ break; ++ } ++} ++ ++void kbase_resume_suspended_soft_jobs(struct kbase_device *kbdev) ++{ ++ LIST_HEAD(local_suspended_soft_jobs); ++ struct kbase_jd_atom *tmp_iter; ++ struct kbase_jd_atom *katom_iter; ++ struct kbasep_js_device_data *js_devdata; ++ bool resched = false; ++ ++ KBASE_DEBUG_ASSERT(kbdev); ++ ++ js_devdata = &kbdev->js_data; ++ ++ /* Move out the entire list */ ++ mutex_lock(&js_devdata->runpool_mutex); ++ list_splice_init(&js_devdata->suspended_soft_jobs_list, ++ &local_suspended_soft_jobs); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ ++ /* ++ * Each atom must be detached from the list and ran separately - ++ * it could be re-added to the old list, but this is unlikely ++ */ ++ list_for_each_entry_safe(katom_iter, tmp_iter, ++ &local_suspended_soft_jobs, dep_item[1]) { ++ struct kbase_context *kctx = katom_iter->kctx; ++ ++ mutex_lock(&kctx->jctx.lock); ++ ++ /* Remove from the global list */ ++ list_del(&katom_iter->dep_item[1]); ++ /* Remove from the context's list of waiting soft jobs */ ++ kbasep_remove_waiting_soft_job(katom_iter); ++ ++ if (kbase_process_soft_job(katom_iter) == 0) { ++ kbase_finish_soft_job(katom_iter); ++ resched |= jd_done_nolock(katom_iter, NULL); ++ } ++ mutex_unlock(&kctx->jctx.lock); ++ } ++ ++ if (resched) ++ kbase_js_sched_all(kbdev); ++} ++#endif /* !MALI_USE_CSF */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_strings.c b/drivers/gpu/arm/bifrost/mali_kbase_strings.c +new file mode 100755 +index 000000000000..22caa4a6d814 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_strings.c +@@ -0,0 +1,28 @@ ++ /* ++ * ++ * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++#include "mali_kbase_strings.h" ++ ++#define KBASE_DRV_NAME "mali" ++#define KBASE_TIMELINE_NAME KBASE_DRV_NAME ".timeline" ++ ++const char kbase_drv_name[] = KBASE_DRV_NAME; ++const char kbase_timeline_name[] = KBASE_TIMELINE_NAME; +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_strings.h b/drivers/gpu/arm/bifrost/mali_kbase_strings.h +new file mode 100755 +index 000000000000..d2f1825314fe +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_strings.h +@@ -0,0 +1,24 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++extern const char kbase_drv_name[]; ++extern const char kbase_timeline_name[]; +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_sync.h b/drivers/gpu/arm/bifrost/mali_kbase_sync.h +new file mode 100755 +index 000000000000..4e5ab3ca557a +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_sync.h +@@ -0,0 +1,231 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2016, 2018-2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/** ++ * @file mali_kbase_sync.h ++ * ++ * This file contains our internal "API" for explicit fences. ++ * It hides the implementation details of the actual explicit fence mechanism ++ * used (Android fences or sync file with DMA fences). ++ */ ++ ++#ifndef MALI_KBASE_SYNC_H ++#define MALI_KBASE_SYNC_H ++ ++#include ++#ifdef CONFIG_SYNC ++#include ++#endif ++#ifdef CONFIG_SYNC_FILE ++#include "mali_kbase_fence_defs.h" ++#include ++#endif ++ ++#include "mali_kbase.h" ++ ++/** ++ * struct kbase_sync_fence_info - Information about a fence ++ * @fence: Pointer to fence (type is void*, as underlaying struct can differ) ++ * @name: The name given to this fence when it was created ++ * @status: < 0 means error, 0 means active, 1 means signaled ++ * ++ * Use kbase_sync_fence_in_info_get() or kbase_sync_fence_out_info_get() ++ * to get the information. ++ */ ++struct kbase_sync_fence_info { ++ void *fence; ++ char name[32]; ++ int status; ++}; ++ ++/** ++ * kbase_sync_fence_stream_create() - Create a stream object ++ * @name: Name of stream (only used to ease debugging/visualization) ++ * @out_fd: A file descriptor representing the created stream object ++ * ++ * Can map down to a timeline implementation in some implementations. ++ * Exposed as a file descriptor. ++ * Life-time controlled via the file descriptor: ++ * - dup to add a ref ++ * - close to remove a ref ++ * ++ * return: 0 on success, < 0 on error ++ */ ++int kbase_sync_fence_stream_create(const char *name, int *const out_fd); ++ ++#if !MALI_USE_CSF ++/** ++ * kbase_sync_fence_out_create Create an explicit output fence to specified atom ++ * @katom: Atom to assign the new explicit fence to ++ * @stream_fd: File descriptor for stream object to create fence on ++ * ++ * return: Valid file descriptor to fence or < 0 on error ++ */ ++int kbase_sync_fence_out_create(struct kbase_jd_atom *katom, int stream_fd); ++ ++/** ++ * kbase_sync_fence_in_from_fd() Assigns an existing fence to specified atom ++ * @katom: Atom to assign the existing explicit fence to ++ * @fd: File descriptor to an existing fence ++ * ++ * Assigns an explicit input fence to atom. ++ * This can later be waited for by calling @kbase_sync_fence_in_wait ++ * ++ * return: 0 on success, < 0 on error ++ */ ++int kbase_sync_fence_in_from_fd(struct kbase_jd_atom *katom, int fd); ++#endif /* !MALI_USE_CSF */ ++ ++/** ++ * kbase_sync_fence_validate() - Validate a fd to be a valid fence ++ * @fd: File descriptor to check ++ * ++ * This function is only usable to catch unintentional user errors early, ++ * it does not stop malicious code changing the fd after this function returns. ++ * ++ * return 0: if fd is for a valid fence, < 0 if invalid ++ */ ++int kbase_sync_fence_validate(int fd); ++ ++#if !MALI_USE_CSF ++/** ++ * kbase_sync_fence_out_trigger - Signal explicit output fence attached on katom ++ * @katom: Atom with an explicit fence to signal ++ * @result: < 0 means signal with error, 0 >= indicates success ++ * ++ * Signal output fence attached on katom and remove the fence from the atom. ++ * ++ * return: The "next" event code for atom, typically JOB_CANCELLED or EVENT_DONE ++ */ ++enum base_jd_event_code ++kbase_sync_fence_out_trigger(struct kbase_jd_atom *katom, int result); ++ ++/** ++ * kbase_sync_fence_in_wait() - Wait for explicit input fence to be signaled ++ * @katom: Atom with explicit fence to wait for ++ * ++ * If the fence is already signaled, then 0 is returned, and the caller must ++ * continue processing of the katom. ++ * ++ * If the fence isn't already signaled, then this kbase_sync framework will ++ * take responsibility to continue the processing once the fence is signaled. ++ * ++ * return: 0 if already signaled, otherwise 1 ++ */ ++int kbase_sync_fence_in_wait(struct kbase_jd_atom *katom); ++ ++/** ++ * kbase_sync_fence_in_cancel_wait() - Cancel explicit input fence waits ++ * @katom: Atom to cancel wait for ++ * ++ * This function is fully responsible for continuing processing of this atom ++ * (remove_waiting_soft_job + finish_soft_job + jd_done + js_sched_all) ++ */ ++void kbase_sync_fence_in_cancel_wait(struct kbase_jd_atom *katom); ++ ++/** ++ * kbase_sync_fence_in_remove() - Remove the input fence from the katom ++ * @katom: Atom to remove explicit input fence for ++ * ++ * This will also release the corresponding reference. ++ */ ++void kbase_sync_fence_in_remove(struct kbase_jd_atom *katom); ++ ++/** ++ * kbase_sync_fence_out_remove() - Remove the output fence from the katom ++ * @katom: Atom to remove explicit output fence for ++ * ++ * This will also release the corresponding reference. ++ */ ++void kbase_sync_fence_out_remove(struct kbase_jd_atom *katom); ++#endif /* !MALI_USE_CSF */ ++ ++/** ++ * kbase_sync_fence_close_fd() - Close a file descriptor representing a fence ++ * @fd: File descriptor to close ++ */ ++static inline void kbase_sync_fence_close_fd(int fd) ++{ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 17, 0) ++ ksys_close(fd); ++#else ++ sys_close(fd); ++#endif ++} ++ ++#if !MALI_USE_CSF ++/** ++ * kbase_sync_fence_in_info_get() - Retrieves information about input fence ++ * @katom: Atom to get fence information from ++ * @info: Struct to be filled with fence information ++ * ++ * return: 0 on success, < 0 on error ++ */ ++int kbase_sync_fence_in_info_get(struct kbase_jd_atom *katom, ++ struct kbase_sync_fence_info *info); ++ ++/** ++ * kbase_sync_fence_out_info_get() - Retrieves information about output fence ++ * @katom: Atom to get fence information from ++ * @info: Struct to be filled with fence information ++ * ++ * return: 0 on success, < 0 on error ++ */ ++int kbase_sync_fence_out_info_get(struct kbase_jd_atom *katom, ++ struct kbase_sync_fence_info *info); ++#endif /* !MALI_USE_CSF */ ++ ++#if defined(CONFIG_SYNC_FILE) ++#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) ++void kbase_sync_fence_info_get(struct fence *fence, ++ struct kbase_sync_fence_info *info); ++#else ++void kbase_sync_fence_info_get(struct dma_fence *fence, ++ struct kbase_sync_fence_info *info); ++#endif ++#endif ++ ++/** ++ * kbase_sync_status_string() - Get string matching @status ++ * @status: Value of fence status. ++ * ++ * return: Pointer to string describing @status. ++ */ ++const char *kbase_sync_status_string(int status); ++ ++ ++#if !MALI_USE_CSF ++/* ++ * Internal worker used to continue processing of atom. ++ */ ++void kbase_sync_fence_wait_worker(struct work_struct *data); ++ ++#ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG ++/** ++ * kbase_sync_fence_in_dump() Trigger a debug dump of atoms input fence state ++ * @katom: Atom to trigger fence debug dump for ++ */ ++void kbase_sync_fence_in_dump(struct kbase_jd_atom *katom); ++#endif ++#endif /* !MALI_USE_CSF */ ++ ++#endif /* MALI_KBASE_SYNC_H */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_sync_android.c b/drivers/gpu/arm/bifrost/mali_kbase_sync_android.c +new file mode 100755 +index 000000000000..41f740a7bc8c +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_sync_android.c +@@ -0,0 +1,542 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* ++ * Code for supporting explicit Android fences (CONFIG_SYNC) ++ * Known to be good for kernels 4.5 and earlier. ++ * Replaced with CONFIG_SYNC_FILE for 4.9 and later kernels ++ * (see mali_kbase_sync_file.c) ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include "sync.h" ++#include ++#include ++ ++struct mali_sync_timeline { ++ struct sync_timeline timeline; ++ atomic_t counter; ++ atomic_t signaled; ++}; ++ ++struct mali_sync_pt { ++ struct sync_pt pt; ++ int order; ++ int result; ++}; ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) ++/* For backwards compatibility with kernels before 3.17. After 3.17 ++ * sync_pt_parent is included in the kernel. */ ++static inline struct sync_timeline *sync_pt_parent(struct sync_pt *pt) ++{ ++ return pt->parent; ++} ++#endif ++ ++static struct mali_sync_timeline *to_mali_sync_timeline( ++ struct sync_timeline *timeline) ++{ ++ return container_of(timeline, struct mali_sync_timeline, timeline); ++} ++ ++static struct mali_sync_pt *to_mali_sync_pt(struct sync_pt *pt) ++{ ++ return container_of(pt, struct mali_sync_pt, pt); ++} ++ ++static struct sync_pt *timeline_dup(struct sync_pt *pt) ++{ ++ struct mali_sync_pt *mpt = to_mali_sync_pt(pt); ++ struct mali_sync_pt *new_mpt; ++ struct sync_pt *new_pt = sync_pt_create(sync_pt_parent(pt), ++ sizeof(struct mali_sync_pt)); ++ ++ if (!new_pt) ++ return NULL; ++ ++ new_mpt = to_mali_sync_pt(new_pt); ++ new_mpt->order = mpt->order; ++ new_mpt->result = mpt->result; ++ ++ return new_pt; ++} ++ ++static int timeline_has_signaled(struct sync_pt *pt) ++{ ++ struct mali_sync_pt *mpt = to_mali_sync_pt(pt); ++ struct mali_sync_timeline *mtl = to_mali_sync_timeline( ++ sync_pt_parent(pt)); ++ int result = mpt->result; ++ ++ int diff = atomic_read(&mtl->signaled) - mpt->order; ++ ++ if (diff >= 0) ++ return (result < 0) ? result : 1; ++ ++ return 0; ++} ++ ++static int timeline_compare(struct sync_pt *a, struct sync_pt *b) ++{ ++ struct mali_sync_pt *ma = container_of(a, struct mali_sync_pt, pt); ++ struct mali_sync_pt *mb = container_of(b, struct mali_sync_pt, pt); ++ ++ int diff = ma->order - mb->order; ++ ++ if (diff == 0) ++ return 0; ++ ++ return (diff < 0) ? -1 : 1; ++} ++ ++static void timeline_value_str(struct sync_timeline *timeline, char *str, ++ int size) ++{ ++ struct mali_sync_timeline *mtl = to_mali_sync_timeline(timeline); ++ ++ snprintf(str, size, "%d", atomic_read(&mtl->signaled)); ++} ++ ++static void pt_value_str(struct sync_pt *pt, char *str, int size) ++{ ++ struct mali_sync_pt *mpt = to_mali_sync_pt(pt); ++ ++ snprintf(str, size, "%d(%d)", mpt->order, mpt->result); ++} ++ ++static struct sync_timeline_ops mali_timeline_ops = { ++ .driver_name = "Mali", ++ .dup = timeline_dup, ++ .has_signaled = timeline_has_signaled, ++ .compare = timeline_compare, ++ .timeline_value_str = timeline_value_str, ++ .pt_value_str = pt_value_str, ++}; ++ ++/* Allocates a timeline for Mali ++ * ++ * One timeline should be allocated per API context. ++ */ ++static struct sync_timeline *mali_sync_timeline_alloc(const char *name) ++{ ++ struct sync_timeline *tl; ++ struct mali_sync_timeline *mtl; ++ ++ tl = sync_timeline_create(&mali_timeline_ops, ++ sizeof(struct mali_sync_timeline), name); ++ if (!tl) ++ return NULL; ++ ++ /* Set the counter in our private struct */ ++ mtl = to_mali_sync_timeline(tl); ++ atomic_set(&mtl->counter, 0); ++ atomic_set(&mtl->signaled, 0); ++ ++ return tl; ++} ++ ++static int kbase_stream_close(struct inode *inode, struct file *file) ++{ ++ struct sync_timeline *tl; ++ ++ tl = (struct sync_timeline *)file->private_data; ++ sync_timeline_destroy(tl); ++ return 0; ++} ++ ++static const struct file_operations stream_fops = { ++ .owner = THIS_MODULE, ++ .release = kbase_stream_close, ++}; ++ ++int kbase_sync_fence_stream_create(const char *name, int *const out_fd) ++{ ++ struct sync_timeline *tl; ++ ++ if (!out_fd) ++ return -EINVAL; ++ ++ tl = mali_sync_timeline_alloc(name); ++ if (!tl) ++ return -EINVAL; ++ ++ *out_fd = anon_inode_getfd(name, &stream_fops, tl, O_RDONLY|O_CLOEXEC); ++ ++ if (*out_fd < 0) { ++ sync_timeline_destroy(tl); ++ return -EINVAL; ++ } ++ ++ return 0; ++} ++ ++/* Allocates a sync point within the timeline. ++ * ++ * The timeline must be the one allocated by kbase_sync_timeline_alloc ++ * ++ * Sync points must be triggered in *exactly* the same order as they are ++ * allocated. ++ */ ++static struct sync_pt *kbase_sync_pt_alloc(struct sync_timeline *parent) ++{ ++ struct sync_pt *pt = sync_pt_create(parent, ++ sizeof(struct mali_sync_pt)); ++ struct mali_sync_timeline *mtl = to_mali_sync_timeline(parent); ++ struct mali_sync_pt *mpt; ++ ++ if (!pt) ++ return NULL; ++ ++ mpt = to_mali_sync_pt(pt); ++ mpt->order = atomic_inc_return(&mtl->counter); ++ mpt->result = 0; ++ ++ return pt; ++} ++ ++int kbase_sync_fence_out_create(struct kbase_jd_atom *katom, int tl_fd) ++{ ++ struct sync_timeline *tl; ++ struct sync_pt *pt; ++ struct sync_fence *fence; ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 7, 0) ++ struct files_struct *files; ++ struct fdtable *fdt; ++#endif ++ int fd; ++ struct file *tl_file; ++ ++ tl_file = fget(tl_fd); ++ if (tl_file == NULL) ++ return -EBADF; ++ ++ if (tl_file->f_op != &stream_fops) { ++ fd = -EBADF; ++ goto out; ++ } ++ ++ tl = tl_file->private_data; ++ ++ pt = kbase_sync_pt_alloc(tl); ++ if (!pt) { ++ fd = -EFAULT; ++ goto out; ++ } ++ ++ fence = sync_fence_create("mali_fence", pt); ++ if (!fence) { ++ sync_pt_free(pt); ++ fd = -EFAULT; ++ goto out; ++ } ++ ++ /* from here the fence owns the sync_pt */ ++ ++ /* create a fd representing the fence */ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0) ++ fd = get_unused_fd_flags(O_RDWR | O_CLOEXEC); ++ if (fd < 0) { ++ sync_fence_put(fence); ++ goto out; ++ } ++#else ++ fd = get_unused_fd(); ++ if (fd < 0) { ++ sync_fence_put(fence); ++ goto out; ++ } ++ ++ files = current->files; ++ spin_lock(&files->file_lock); ++ fdt = files_fdtable(files); ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 4, 0) ++ __set_close_on_exec(fd, fdt); ++#else ++ FD_SET(fd, fdt->close_on_exec); ++#endif ++ spin_unlock(&files->file_lock); ++#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0) */ ++ ++ /* bind fence to the new fd */ ++ sync_fence_install(fence, fd); ++ ++ katom->fence = sync_fence_fdget(fd); ++ if (katom->fence == NULL) { ++ /* The only way the fence can be NULL is if userspace closed it ++ * for us, so we don't need to clear it up */ ++ fd = -EINVAL; ++ goto out; ++ } ++ ++out: ++ fput(tl_file); ++ ++ return fd; ++} ++ ++int kbase_sync_fence_in_from_fd(struct kbase_jd_atom *katom, int fd) ++{ ++ katom->fence = sync_fence_fdget(fd); ++ return katom->fence ? 0 : -ENOENT; ++} ++ ++int kbase_sync_fence_validate(int fd) ++{ ++ struct sync_fence *fence; ++ ++ fence = sync_fence_fdget(fd); ++ if (!fence) ++ return -EINVAL; ++ ++ sync_fence_put(fence); ++ return 0; ++} ++ ++/* Returns true if the specified timeline is allocated by Mali */ ++static int kbase_sync_timeline_is_ours(struct sync_timeline *timeline) ++{ ++ return timeline->ops == &mali_timeline_ops; ++} ++ ++/* Signals a particular sync point ++ * ++ * Sync points must be triggered in *exactly* the same order as they are ++ * allocated. ++ * ++ * If they are signaled in the wrong order then a message will be printed in ++ * debug builds and otherwise attempts to signal order sync_pts will be ignored. ++ * ++ * result can be negative to indicate error, any other value is interpreted as ++ * success. ++ */ ++static void kbase_sync_signal_pt(struct sync_pt *pt, int result) ++{ ++ struct mali_sync_pt *mpt = to_mali_sync_pt(pt); ++ struct mali_sync_timeline *mtl = to_mali_sync_timeline( ++ sync_pt_parent(pt)); ++ int signaled; ++ int diff; ++ ++ mpt->result = result; ++ ++ do { ++ signaled = atomic_read(&mtl->signaled); ++ ++ diff = signaled - mpt->order; ++ ++ if (diff > 0) { ++ /* The timeline is already at or ahead of this point. ++ * This should not happen unless userspace has been ++ * signaling fences out of order, so warn but don't ++ * violate the sync_pt API. ++ * The warning is only in debug builds to prevent ++ * a malicious user being able to spam dmesg. ++ */ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++ pr_err("Fences were triggered in a different order to allocation!"); ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++ return; ++ } ++ } while (atomic_cmpxchg(&mtl->signaled, ++ signaled, mpt->order) != signaled); ++} ++ ++enum base_jd_event_code ++kbase_sync_fence_out_trigger(struct kbase_jd_atom *katom, int result) ++{ ++ struct sync_pt *pt; ++ struct sync_timeline *timeline; ++ ++ if (!katom->fence) ++ return BASE_JD_EVENT_JOB_CANCELLED; ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) ++ if (!list_is_singular(&katom->fence->pt_list_head)) { ++#else ++ if (katom->fence->num_fences != 1) { ++#endif ++ /* Not exactly one item in the list - so it didn't (directly) ++ * come from us */ ++ return BASE_JD_EVENT_JOB_CANCELLED; ++ } ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) ++ pt = list_first_entry(&katom->fence->pt_list_head, ++ struct sync_pt, pt_list); ++#else ++ pt = container_of(katom->fence->cbs[0].sync_pt, struct sync_pt, base); ++#endif ++ timeline = sync_pt_parent(pt); ++ ++ if (!kbase_sync_timeline_is_ours(timeline)) { ++ /* Fence has a sync_pt which isn't ours! */ ++ return BASE_JD_EVENT_JOB_CANCELLED; ++ } ++ ++ kbase_sync_signal_pt(pt, result); ++ ++ sync_timeline_signal(timeline); ++ ++ kbase_sync_fence_out_remove(katom); ++ ++ return (result < 0) ? BASE_JD_EVENT_JOB_CANCELLED : BASE_JD_EVENT_DONE; ++} ++ ++static inline int kbase_fence_get_status(struct sync_fence *fence) ++{ ++ if (!fence) ++ return -ENOENT; ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) ++ return fence->status; ++#else ++ return atomic_read(&fence->status); ++#endif ++} ++ ++static void kbase_fence_wait_callback(struct sync_fence *fence, ++ struct sync_fence_waiter *waiter) ++{ ++ struct kbase_jd_atom *katom = container_of(waiter, ++ struct kbase_jd_atom, sync_waiter); ++ struct kbase_context *kctx = katom->kctx; ++ ++ /* Propagate the fence status to the atom. ++ * If negative then cancel this atom and its dependencies. ++ */ ++ if (kbase_fence_get_status(fence) < 0) ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ ++ /* To prevent a potential deadlock we schedule the work onto the ++ * job_done_wq workqueue ++ * ++ * The issue is that we may signal the timeline while holding ++ * kctx->jctx.lock and the callbacks are run synchronously from ++ * sync_timeline_signal. So we simply defer the work. ++ */ ++ ++ INIT_WORK(&katom->work, kbase_sync_fence_wait_worker); ++ queue_work(kctx->jctx.job_done_wq, &katom->work); ++} ++ ++int kbase_sync_fence_in_wait(struct kbase_jd_atom *katom) ++{ ++ int ret; ++ ++ sync_fence_waiter_init(&katom->sync_waiter, kbase_fence_wait_callback); ++ ++ ret = sync_fence_wait_async(katom->fence, &katom->sync_waiter); ++ ++ if (ret == 1) { ++ /* Already signaled */ ++ return 0; ++ } ++ ++ if (ret < 0) { ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ /* We should cause the dependent jobs in the bag to be failed, ++ * to do this we schedule the work queue to complete this job */ ++ INIT_WORK(&katom->work, kbase_sync_fence_wait_worker); ++ queue_work(katom->kctx->jctx.job_done_wq, &katom->work); ++ } ++ ++ return 1; ++} ++ ++void kbase_sync_fence_in_cancel_wait(struct kbase_jd_atom *katom) ++{ ++ if (sync_fence_cancel_async(katom->fence, &katom->sync_waiter) != 0) { ++ /* The wait wasn't cancelled - leave the cleanup for ++ * kbase_fence_wait_callback */ ++ return; ++ } ++ ++ /* Wait was cancelled - zap the atoms */ ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ ++ kbasep_remove_waiting_soft_job(katom); ++ kbase_finish_soft_job(katom); ++ ++ if (jd_done_nolock(katom, NULL)) ++ kbase_js_sched_all(katom->kctx->kbdev); ++} ++ ++void kbase_sync_fence_out_remove(struct kbase_jd_atom *katom) ++{ ++ if (katom->fence) { ++ sync_fence_put(katom->fence); ++ katom->fence = NULL; ++ } ++} ++ ++void kbase_sync_fence_in_remove(struct kbase_jd_atom *katom) ++{ ++ if (katom->fence) { ++ sync_fence_put(katom->fence); ++ katom->fence = NULL; ++ } ++} ++ ++int kbase_sync_fence_in_info_get(struct kbase_jd_atom *katom, ++ struct kbase_sync_fence_info *info) ++{ ++ if (!katom->fence) ++ return -ENOENT; ++ ++ info->fence = katom->fence; ++ info->status = kbase_fence_get_status(katom->fence); ++ strlcpy(info->name, katom->fence->name, sizeof(info->name)); ++ ++ return 0; ++} ++ ++int kbase_sync_fence_out_info_get(struct kbase_jd_atom *katom, ++ struct kbase_sync_fence_info *info) ++{ ++ if (!katom->fence) ++ return -ENOENT; ++ ++ info->fence = katom->fence; ++ info->status = kbase_fence_get_status(katom->fence); ++ strlcpy(info->name, katom->fence->name, sizeof(info->name)); ++ ++ return 0; ++} ++ ++#ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG ++void kbase_sync_fence_in_dump(struct kbase_jd_atom *katom) ++{ ++ /* Dump out the full state of all the Android sync fences. ++ * The function sync_dump() isn't exported to modules, so force ++ * sync_fence_wait() to time out to trigger sync_dump(). ++ */ ++ if (katom->fence) ++ sync_fence_wait(katom->fence, 1); ++} ++#endif +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_sync_common.c b/drivers/gpu/arm/bifrost/mali_kbase_sync_common.c +new file mode 100755 +index 000000000000..866894bd0f94 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_sync_common.c +@@ -0,0 +1,51 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2016, 2018-2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* ++ * @file mali_kbase_sync_common.c ++ * ++ * Common code for our explicit fence functionality ++ */ ++ ++#include ++#include "mali_kbase.h" ++#include "mali_kbase_sync.h" ++ ++#if !MALI_USE_CSF ++void kbase_sync_fence_wait_worker(struct work_struct *data) ++{ ++ struct kbase_jd_atom *katom; ++ ++ katom = container_of(data, struct kbase_jd_atom, work); ++ kbase_soft_event_wait_callback(katom); ++} ++#endif /* !MALI_USE_CSF */ ++ ++const char *kbase_sync_status_string(int status) ++{ ++ if (status == 0) ++ return "active"; ++ else if (status > 0) ++ return "signaled"; ++ else ++ return "error"; ++} +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_sync_file.c b/drivers/gpu/arm/bifrost/mali_kbase_sync_file.c +new file mode 100755 +index 000000000000..271873b9fe29 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_sync_file.c +@@ -0,0 +1,372 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* ++ * Code for supporting explicit Linux fences (CONFIG_SYNC_FILE) ++ * Introduced in kernel 4.9. ++ * Android explicit fences (CONFIG_SYNC) can be used for older kernels ++ * (see mali_kbase_sync_android.c) ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include "mali_kbase_fence_defs.h" ++#include "mali_kbase_sync.h" ++#include "mali_kbase_fence.h" ++#include "mali_kbase.h" ++ ++static const struct file_operations stream_fops = { ++ .owner = THIS_MODULE ++}; ++ ++int kbase_sync_fence_stream_create(const char *name, int *const out_fd) ++{ ++ if (!out_fd) ++ return -EINVAL; ++ ++ *out_fd = anon_inode_getfd(name, &stream_fops, NULL, ++ O_RDONLY | O_CLOEXEC); ++ if (*out_fd < 0) ++ return -EINVAL; ++ ++ return 0; ++} ++ ++#if !MALI_USE_CSF ++int kbase_sync_fence_out_create(struct kbase_jd_atom *katom, int stream_fd) ++{ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ struct fence *fence; ++#else ++ struct dma_fence *fence; ++#endif ++ struct sync_file *sync_file; ++ int fd; ++ ++ fence = kbase_fence_out_new(katom); ++ if (!fence) ++ return -ENOMEM; ++ ++#if (KERNEL_VERSION(4, 9, 67) >= LINUX_VERSION_CODE) ++ /* Take an extra reference to the fence on behalf of the sync_file. ++ * This is only needed on older kernels where sync_file_create() ++ * does not take its own reference. This was changed in v4.9.68, ++ * where sync_file_create() now takes its own reference. ++ */ ++ dma_fence_get(fence); ++#endif ++ ++ /* create a sync_file fd representing the fence */ ++ sync_file = sync_file_create(fence); ++ if (!sync_file) { ++#if (KERNEL_VERSION(4, 9, 67) >= LINUX_VERSION_CODE) ++ dma_fence_put(fence); ++#endif ++ kbase_fence_out_remove(katom); ++ return -ENOMEM; ++ } ++ ++ fd = get_unused_fd_flags(O_CLOEXEC); ++ if (fd < 0) { ++ fput(sync_file->file); ++ kbase_fence_out_remove(katom); ++ return fd; ++ } ++ ++ fd_install(fd, sync_file->file); ++ ++ return fd; ++} ++ ++int kbase_sync_fence_in_from_fd(struct kbase_jd_atom *katom, int fd) ++{ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ struct fence *fence = sync_file_get_fence(fd); ++#else ++ struct dma_fence *fence = sync_file_get_fence(fd); ++#endif ++ ++ if (!fence) ++ return -ENOENT; ++ ++ kbase_fence_fence_in_set(katom, fence); ++ ++ return 0; ++} ++#endif /* !MALI_USE_CSF */ ++ ++int kbase_sync_fence_validate(int fd) ++{ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ struct fence *fence = sync_file_get_fence(fd); ++#else ++ struct dma_fence *fence = sync_file_get_fence(fd); ++#endif ++ ++ if (!fence) ++ return -EINVAL; ++ ++ dma_fence_put(fence); ++ ++ return 0; /* valid */ ++} ++ ++#if !MALI_USE_CSF ++enum base_jd_event_code ++kbase_sync_fence_out_trigger(struct kbase_jd_atom *katom, int result) ++{ ++ int res; ++ ++ if (!kbase_fence_out_is_ours(katom)) { ++ /* Not our fence */ ++ return BASE_JD_EVENT_JOB_CANCELLED; ++ } ++ ++ res = kbase_fence_out_signal(katom, result); ++ if (unlikely(res < 0)) { ++ dev_warn(katom->kctx->kbdev->dev, ++ "fence_signal() failed with %d\n", res); ++ } ++ ++ kbase_sync_fence_out_remove(katom); ++ ++ return (result != 0) ? BASE_JD_EVENT_JOB_CANCELLED : BASE_JD_EVENT_DONE; ++} ++ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++static void kbase_fence_wait_callback(struct fence *fence, ++ struct fence_cb *cb) ++#else ++static void kbase_fence_wait_callback(struct dma_fence *fence, ++ struct dma_fence_cb *cb) ++#endif ++{ ++ struct kbase_fence_cb *kcb = container_of(cb, ++ struct kbase_fence_cb, ++ fence_cb); ++ struct kbase_jd_atom *katom = kcb->katom; ++ struct kbase_context *kctx = katom->kctx; ++ ++ /* Cancel atom if fence is erroneous */ ++#if (KERNEL_VERSION(4, 11, 0) <= LINUX_VERSION_CODE || \ ++ (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE && \ ++ KERNEL_VERSION(4, 9, 68) <= LINUX_VERSION_CODE)) ++ if (dma_fence_is_signaled(kcb->fence) && kcb->fence->error < 0) ++#else ++ if (dma_fence_is_signaled(kcb->fence) && kcb->fence->status < 0) ++#endif ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ ++ if (kbase_fence_dep_count_dec_and_test(katom)) { ++ /* We take responsibility of handling this */ ++ kbase_fence_dep_count_set(katom, -1); ++ ++ /* To prevent a potential deadlock we schedule the work onto the ++ * job_done_wq workqueue ++ * ++ * The issue is that we may signal the timeline while holding ++ * kctx->jctx.lock and the callbacks are run synchronously from ++ * sync_timeline_signal. So we simply defer the work. ++ */ ++ INIT_WORK(&katom->work, kbase_sync_fence_wait_worker); ++ queue_work(kctx->jctx.job_done_wq, &katom->work); ++ } ++} ++ ++int kbase_sync_fence_in_wait(struct kbase_jd_atom *katom) ++{ ++ int err; ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ struct fence *fence; ++#else ++ struct dma_fence *fence; ++#endif ++ ++ fence = kbase_fence_in_get(katom); ++ if (!fence) ++ return 0; /* no input fence to wait for, good to go! */ ++ ++ kbase_fence_dep_count_set(katom, 1); ++ ++ err = kbase_fence_add_callback(katom, fence, kbase_fence_wait_callback); ++ ++ kbase_fence_put(fence); ++ ++ if (likely(!err)) { ++ /* Test if the callbacks are already triggered */ ++ if (kbase_fence_dep_count_dec_and_test(katom)) { ++ kbase_fence_free_callbacks(katom); ++ kbase_fence_dep_count_set(katom, -1); ++ return 0; /* Already signaled, good to go right now */ ++ } ++ ++ /* Callback installed, so we just need to wait for it... */ ++ } else { ++ /* Failure */ ++ kbase_fence_free_callbacks(katom); ++ kbase_fence_dep_count_set(katom, -1); ++ ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ ++ /* We should cause the dependent jobs in the bag to be failed, ++ * to do this we schedule the work queue to complete this job */ ++ ++ INIT_WORK(&katom->work, kbase_sync_fence_wait_worker); ++ queue_work(katom->kctx->jctx.job_done_wq, &katom->work); ++ } ++ ++ return 1; /* completion to be done later by callback/worker */ ++} ++ ++void kbase_sync_fence_in_cancel_wait(struct kbase_jd_atom *katom) ++{ ++ if (!kbase_fence_free_callbacks(katom)) { ++ /* The wait wasn't cancelled - ++ * leave the cleanup for kbase_fence_wait_callback */ ++ return; ++ } ++ ++ /* Take responsibility of completion */ ++ kbase_fence_dep_count_set(katom, -1); ++ ++ /* Wait was cancelled - zap the atoms */ ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ ++ kbasep_remove_waiting_soft_job(katom); ++ kbase_finish_soft_job(katom); ++ ++ if (jd_done_nolock(katom, NULL)) ++ kbase_js_sched_all(katom->kctx->kbdev); ++} ++ ++void kbase_sync_fence_out_remove(struct kbase_jd_atom *katom) ++{ ++ kbase_fence_out_remove(katom); ++} ++ ++void kbase_sync_fence_in_remove(struct kbase_jd_atom *katom) ++{ ++ kbase_fence_free_callbacks(katom); ++ kbase_fence_in_remove(katom); ++} ++#endif /* !MALI_USE_CSF */ ++ ++#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) ++void kbase_sync_fence_info_get(struct fence *fence, ++ struct kbase_sync_fence_info *info) ++#else ++void kbase_sync_fence_info_get(struct dma_fence *fence, ++ struct kbase_sync_fence_info *info) ++#endif ++{ ++ info->fence = fence; ++ ++ /* translate into CONFIG_SYNC status: ++ * < 0 : error ++ * 0 : active ++ * 1 : signaled ++ */ ++ if (dma_fence_is_signaled(fence)) { ++#if (KERNEL_VERSION(4, 11, 0) <= LINUX_VERSION_CODE || \ ++ (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE && \ ++ KERNEL_VERSION(4, 9, 68) <= LINUX_VERSION_CODE)) ++ int status = fence->error; ++#else ++ int status = fence->status; ++#endif ++ if (status < 0) ++ info->status = status; /* signaled with error */ ++ else ++ info->status = 1; /* signaled with success */ ++ } else { ++ info->status = 0; /* still active (unsignaled) */ ++ } ++ ++#if (KERNEL_VERSION(4, 8, 0) > LINUX_VERSION_CODE) ++ scnprintf(info->name, sizeof(info->name), "%u#%u", ++ fence->context, fence->seqno); ++#elif (KERNEL_VERSION(5, 1, 0) > LINUX_VERSION_CODE) ++ scnprintf(info->name, sizeof(info->name), "%llu#%u", ++ fence->context, fence->seqno); ++#else ++ scnprintf(info->name, sizeof(info->name), "%llu#%llu", ++ fence->context, fence->seqno); ++#endif ++} ++ ++#if !MALI_USE_CSF ++int kbase_sync_fence_in_info_get(struct kbase_jd_atom *katom, ++ struct kbase_sync_fence_info *info) ++{ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ struct fence *fence; ++#else ++ struct dma_fence *fence; ++#endif ++ ++ fence = kbase_fence_in_get(katom); ++ if (!fence) ++ return -ENOENT; ++ ++ kbase_sync_fence_info_get(fence, info); ++ ++ kbase_fence_put(fence); ++ ++ return 0; ++} ++ ++int kbase_sync_fence_out_info_get(struct kbase_jd_atom *katom, ++ struct kbase_sync_fence_info *info) ++{ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ struct fence *fence; ++#else ++ struct dma_fence *fence; ++#endif ++ ++ fence = kbase_fence_out_get(katom); ++ if (!fence) ++ return -ENOENT; ++ ++ kbase_sync_fence_info_get(fence, info); ++ ++ kbase_fence_put(fence); ++ ++ return 0; ++} ++ ++ ++#ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG ++void kbase_sync_fence_in_dump(struct kbase_jd_atom *katom) ++{ ++ /* Not implemented */ ++} ++#endif ++#endif /* !MALI_USE_CSF*/ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_trace_gpu_mem.c b/drivers/gpu/arm/bifrost/mali_kbase_trace_gpu_mem.c +new file mode 100755 +index 000000000000..7669895b3c5d +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_trace_gpu_mem.c +@@ -0,0 +1,227 @@ ++/* ++ * ++ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++#include ++#include ++#include ++ ++/** ++ * struct kbase_dma_buf - Object instantiated when a dma-buf imported allocation ++ * is mapped to GPU for the first time within a process. ++ * Another instantiation is done for the case when that ++ * allocation is mapped for the first time to GPU. ++ * ++ * @dma_buf: Reference to dma_buf been imported. ++ * @dma_buf_node: Link node to maintain a rb_tree of kbase_dma_buf. ++ * @import_count: The number of times the dma_buf was imported. ++ */ ++struct kbase_dma_buf { ++ struct dma_buf *dma_buf; ++ struct rb_node dma_buf_node; ++ u32 import_count; ++}; ++ ++/** ++ * kbase_delete_dma_buf_mapping - Delete a dma buffer mapping. ++ * ++ * @kctx: Pointer to kbase context. ++ * @dma_buf: Pointer to a dma buffer mapping. ++ * @tree: Pointer to root of rb_tree containing the dma_buf's mapped. ++ * ++ * when we un-map any dma mapping we need to remove them from rb_tree, ++ * rb_tree is maintained at kbase_device level and kbase_process level ++ * by passing the root of kbase_device or kbase_process we can remove ++ * the node from the tree. ++ */ ++static bool kbase_delete_dma_buf_mapping(struct kbase_context *kctx, ++ struct dma_buf *dma_buf, ++ struct rb_root *tree) ++{ ++ struct kbase_dma_buf *buf_node = NULL; ++ struct rb_node *node = tree->rb_node; ++ bool mapping_removed = false; ++ ++ lockdep_assert_held(&kctx->kbdev->dma_buf_lock); ++ ++ while (node) { ++ buf_node = rb_entry(node, struct kbase_dma_buf, dma_buf_node); ++ ++ if (dma_buf == buf_node->dma_buf) { ++ WARN_ON(!buf_node->import_count); ++ ++ buf_node->import_count--; ++ ++ if (!buf_node->import_count) { ++ rb_erase(&buf_node->dma_buf_node, tree); ++ kfree(buf_node); ++ mapping_removed = true; ++ } ++ ++ break; ++ } ++ ++ if (dma_buf < buf_node->dma_buf) ++ node = node->rb_left; ++ else ++ node = node->rb_right; ++ } ++ ++ WARN_ON(!buf_node); ++ return mapping_removed; ++} ++ ++/** ++ * kbase_capture_dma_buf_mapping - capture a dma buffer mapping. ++ * ++ * @kctx: Pointer to kbase context. ++ * @dma_buf: Pointer to a dma buffer mapping. ++ * @root: Pointer to root of rb_tree containing the dma_buf's. ++ * ++ * We maintain a kbase_device level and kbase_process level rb_tree ++ * of all unique dma_buf's mapped to gpu memory. So when attach any ++ * dma_buf add it the rb_tree's. To add the unique mapping we need ++ * check if the mapping is not a duplicate and then add them. ++ */ ++static bool kbase_capture_dma_buf_mapping(struct kbase_context *kctx, ++ struct dma_buf *dma_buf, ++ struct rb_root *root) ++{ ++ struct kbase_dma_buf *buf_node = NULL; ++ struct rb_node *node = root->rb_node; ++ bool unique_buf_imported = true; ++ ++ lockdep_assert_held(&kctx->kbdev->dma_buf_lock); ++ ++ while (node) { ++ buf_node = rb_entry(node, struct kbase_dma_buf, dma_buf_node); ++ ++ if (dma_buf == buf_node->dma_buf) { ++ unique_buf_imported = false; ++ break; ++ } ++ ++ if (dma_buf < buf_node->dma_buf) ++ node = node->rb_left; ++ else ++ node = node->rb_right; ++ } ++ ++ if (unique_buf_imported) { ++ struct kbase_dma_buf *buf_node = ++ kzalloc(sizeof(*buf_node), GFP_KERNEL); ++ ++ if (buf_node == NULL) { ++ dev_err(kctx->kbdev->dev, "Error allocating memory for kbase_dma_buf\n"); ++ /* Dont account for it if we fail to allocate memory */ ++ unique_buf_imported = false; ++ } else { ++ struct rb_node **new = &(root->rb_node), *parent = NULL; ++ ++ buf_node->dma_buf = dma_buf; ++ buf_node->import_count = 1; ++ while (*new) { ++ struct kbase_dma_buf *node; ++ ++ parent = *new; ++ node = rb_entry(parent, struct kbase_dma_buf, ++ dma_buf_node); ++ if (dma_buf < node->dma_buf) ++ new = &(*new)->rb_left; ++ else ++ new = &(*new)->rb_right; ++ } ++ rb_link_node(&buf_node->dma_buf_node, parent, new); ++ rb_insert_color(&buf_node->dma_buf_node, root); ++ } ++ } else if (!WARN_ON(!buf_node)) { ++ buf_node->import_count++; ++ } ++ ++ return unique_buf_imported; ++} ++ ++void kbase_remove_dma_buf_usage(struct kbase_context *kctx, ++ struct kbase_mem_phy_alloc *alloc) ++{ ++ struct kbase_device *kbdev = kctx->kbdev; ++ bool dev_mapping_removed, prcs_mapping_removed; ++ ++ mutex_lock(&kbdev->dma_buf_lock); ++ ++ dev_mapping_removed = kbase_delete_dma_buf_mapping( ++ kctx, alloc->imported.umm.dma_buf, &kbdev->dma_buf_root); ++ ++ prcs_mapping_removed = kbase_delete_dma_buf_mapping( ++ kctx, alloc->imported.umm.dma_buf, &kctx->kprcs->dma_buf_root); ++ ++ WARN_ON(dev_mapping_removed && !prcs_mapping_removed); ++ ++ spin_lock(&kbdev->gpu_mem_usage_lock); ++ if (dev_mapping_removed) ++ kbdev->total_gpu_pages -= alloc->nents; ++ ++ if (prcs_mapping_removed) ++ kctx->kprcs->total_gpu_pages -= alloc->nents; ++ ++ if (dev_mapping_removed || prcs_mapping_removed) ++ kbase_trace_gpu_mem_usage(kbdev, kctx); ++ spin_unlock(&kbdev->gpu_mem_usage_lock); ++ ++ mutex_unlock(&kbdev->dma_buf_lock); ++} ++ ++void kbase_add_dma_buf_usage(struct kbase_context *kctx, ++ struct kbase_mem_phy_alloc *alloc) ++{ ++ struct kbase_device *kbdev = kctx->kbdev; ++ bool unique_dev_dmabuf, unique_prcs_dmabuf; ++ ++ mutex_lock(&kbdev->dma_buf_lock); ++ ++ /* add dma_buf to device and process. */ ++ unique_dev_dmabuf = kbase_capture_dma_buf_mapping( ++ kctx, alloc->imported.umm.dma_buf, &kbdev->dma_buf_root); ++ ++ unique_prcs_dmabuf = kbase_capture_dma_buf_mapping( ++ kctx, alloc->imported.umm.dma_buf, &kctx->kprcs->dma_buf_root); ++ ++ WARN_ON(unique_dev_dmabuf && !unique_prcs_dmabuf); ++ ++ spin_lock(&kbdev->gpu_mem_usage_lock); ++ if (unique_dev_dmabuf) ++ kbdev->total_gpu_pages += alloc->nents; ++ ++ if (unique_prcs_dmabuf) ++ kctx->kprcs->total_gpu_pages += alloc->nents; ++ ++ if (unique_prcs_dmabuf || unique_dev_dmabuf) ++ kbase_trace_gpu_mem_usage(kbdev, kctx); ++ spin_unlock(&kbdev->gpu_mem_usage_lock); ++ ++ mutex_unlock(&kbdev->dma_buf_lock); ++} ++ ++#if !defined(CONFIG_TRACE_GPU_MEM) && !MALI_CUSTOMER_RELEASE ++#define CREATE_TRACE_POINTS ++#include "mali_gpu_mem_trace.h" ++#endif +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_trace_gpu_mem.h b/drivers/gpu/arm/bifrost/mali_kbase_trace_gpu_mem.h +new file mode 100755 +index 000000000000..7e95956f3132 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_trace_gpu_mem.h +@@ -0,0 +1,103 @@ ++/* ++ * ++ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_TRACE_GPU_MEM_H_ ++#define _KBASE_TRACE_GPU_MEM_H_ ++ ++#ifdef CONFIG_TRACE_GPU_MEM ++#include ++#elif !MALI_CUSTOMER_RELEASE ++#include "mali_gpu_mem_trace.h" ++#endif ++ ++#define DEVICE_TGID ((u32) 0U) ++ ++static void kbase_trace_gpu_mem_usage(struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++ lockdep_assert_held(&kbdev->gpu_mem_usage_lock); ++ ++#if defined(CONFIG_TRACE_GPU_MEM) || !MALI_CUSTOMER_RELEASE ++ trace_gpu_mem_total(kbdev->id, DEVICE_TGID, ++ kbdev->total_gpu_pages << PAGE_SHIFT); ++ ++ if (likely(kctx)) ++ trace_gpu_mem_total(kbdev->id, kctx->kprcs->tgid, ++ kctx->kprcs->total_gpu_pages << PAGE_SHIFT); ++#endif ++} ++ ++static inline void kbase_trace_gpu_mem_usage_dec(struct kbase_device *kbdev, ++ struct kbase_context *kctx, size_t pages) ++{ ++ spin_lock(&kbdev->gpu_mem_usage_lock); ++ ++ if (likely(kctx)) ++ kctx->kprcs->total_gpu_pages -= pages; ++ ++ kbdev->total_gpu_pages -= pages; ++ ++ kbase_trace_gpu_mem_usage(kbdev, kctx); ++ ++ spin_unlock(&kbdev->gpu_mem_usage_lock); ++} ++ ++static inline void kbase_trace_gpu_mem_usage_inc(struct kbase_device *kbdev, ++ struct kbase_context *kctx, size_t pages) ++{ ++ spin_lock(&kbdev->gpu_mem_usage_lock); ++ ++ if (likely(kctx)) ++ kctx->kprcs->total_gpu_pages += pages; ++ ++ kbdev->total_gpu_pages += pages; ++ ++ kbase_trace_gpu_mem_usage(kbdev, kctx); ++ ++ spin_unlock(&kbdev->gpu_mem_usage_lock); ++} ++ ++/** ++ * kbase_remove_dma_buf_usage - Remove a dma-buf entry captured. ++ * ++ * @kctx: Pointer to the kbase context ++ * @alloc: Pointer to the alloc to unmap ++ * ++ * Remove reference to dma buf been unmapped from kbase_device level ++ * rb_tree and Kbase_process level dma buf rb_tree. ++ */ ++void kbase_remove_dma_buf_usage(struct kbase_context *kctx, ++ struct kbase_mem_phy_alloc *alloc); ++ ++/** ++ * kbase_add_dma_buf_usage - Add a dma-buf entry captured. ++ * ++ * @kctx: Pointer to the kbase context ++ * @alloc: Pointer to the alloc to map in ++ * ++ * Add reference to dma buf been mapped to kbase_device level ++ * rb_tree and Kbase_process level dma buf rb_tree. ++ */ ++void kbase_add_dma_buf_usage(struct kbase_context *kctx, ++ struct kbase_mem_phy_alloc *alloc); ++ ++#endif /* _KBASE_TRACE_GPU_MEM_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_utility.h b/drivers/gpu/arm/bifrost/mali_kbase_utility.h +new file mode 100755 +index 000000000000..8d4f044376a9 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_utility.h +@@ -0,0 +1,55 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2013, 2015, 2018 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++#ifndef _KBASE_UTILITY_H ++#define _KBASE_UTILITY_H ++ ++#ifndef _KBASE_H_ ++#error "Don't include this file directly, use mali_kbase.h instead" ++#endif ++ ++static inline void kbase_timer_setup(struct timer_list *timer, ++ void (*callback)(struct timer_list *timer)) ++{ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0) ++ setup_timer(timer, (void (*)(unsigned long)) callback, ++ (unsigned long) timer); ++#else ++ timer_setup(timer, callback, 0); ++#endif ++} ++ ++#ifndef WRITE_ONCE ++ #ifdef ASSIGN_ONCE ++ #define WRITE_ONCE(x, val) ASSIGN_ONCE(val, x) ++ #else ++ #define WRITE_ONCE(x, val) (ACCESS_ONCE(x) = (val)) ++ #endif ++#endif ++ ++#ifndef READ_ONCE ++ #define READ_ONCE(x) ACCESS_ONCE(x) ++#endif ++ ++#endif /* _KBASE_UTILITY_H */ +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_vinstr.c b/drivers/gpu/arm/bifrost/mali_kbase_vinstr.c +new file mode 100755 +index 000000000000..3b0e2d6855ce +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_vinstr.c +@@ -0,0 +1,1083 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include "mali_kbase_vinstr.h" ++#include "mali_kbase_hwcnt_virtualizer.h" ++#include "mali_kbase_hwcnt_types.h" ++#include "mali_kbase_hwcnt_reader.h" ++#include "mali_kbase_hwcnt_gpu.h" ++#include "mali_kbase_ioctl.h" ++#include "mali_malisw.h" ++#include "mali_kbase_debug.h" ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++/* Hwcnt reader API version */ ++#define HWCNT_READER_API 1 ++ ++/* The minimum allowed interval between dumps (equivalent to 10KHz) */ ++#define DUMP_INTERVAL_MIN_NS (100 * NSEC_PER_USEC) ++ ++/* The maximum allowed buffers per client */ ++#define MAX_BUFFER_COUNT 32 ++ ++/** ++ * struct kbase_vinstr_context - IOCTL interface for userspace hardware ++ * counters. ++ * @hvirt: Hardware counter virtualizer used by vinstr. ++ * @metadata: Hardware counter metadata provided by virtualizer. ++ * @lock: Lock protecting all vinstr state. ++ * @suspend_count: Suspend reference count. If non-zero, timer and worker are ++ * prevented from being re-scheduled. ++ * @client_count: Number of vinstr clients. ++ * @clients: List of vinstr clients. ++ * @dump_timer: Timer that enqueues dump_work to a workqueue. ++ * @dump_work: Worker for performing periodic counter dumps. ++ */ ++struct kbase_vinstr_context { ++ struct kbase_hwcnt_virtualizer *hvirt; ++ const struct kbase_hwcnt_metadata *metadata; ++ struct mutex lock; ++ size_t suspend_count; ++ size_t client_count; ++ struct list_head clients; ++ struct hrtimer dump_timer; ++ struct work_struct dump_work; ++}; ++ ++/** ++ * struct kbase_vinstr_client - A vinstr client attached to a vinstr context. ++ * @vctx: Vinstr context client is attached to. ++ * @hvcli: Hardware counter virtualizer client. ++ * @node: Node used to attach this client to list in vinstr ++ * context. ++ * @dump_interval_ns: Interval between periodic dumps. If 0, not a periodic ++ * client. ++ * @next_dump_time_ns: Time in ns when this client's next periodic dump must ++ * occur. If 0, not a periodic client. ++ * @enable_map: Counters enable map. ++ * @tmp_buf: Temporary buffer to use before handing dump to client. ++ * @dump_bufs: Array of dump buffers allocated by this client. ++ * @dump_bufs_meta: Metadata of dump buffers. ++ * @meta_idx: Index of metadata being accessed by userspace. ++ * @read_idx: Index of buffer read by userspace. ++ * @write_idx: Index of buffer being written by dump worker. ++ * @waitq: Client's notification queue. ++ */ ++struct kbase_vinstr_client { ++ struct kbase_vinstr_context *vctx; ++ struct kbase_hwcnt_virtualizer_client *hvcli; ++ struct list_head node; ++ u64 next_dump_time_ns; ++ u32 dump_interval_ns; ++ struct kbase_hwcnt_enable_map enable_map; ++ struct kbase_hwcnt_dump_buffer tmp_buf; ++ struct kbase_hwcnt_dump_buffer_array dump_bufs; ++ struct kbase_hwcnt_reader_metadata *dump_bufs_meta; ++ atomic_t meta_idx; ++ atomic_t read_idx; ++ atomic_t write_idx; ++ wait_queue_head_t waitq; ++}; ++ ++static unsigned int kbasep_vinstr_hwcnt_reader_poll( ++ struct file *filp, ++ poll_table *wait); ++ ++static long kbasep_vinstr_hwcnt_reader_ioctl( ++ struct file *filp, ++ unsigned int cmd, ++ unsigned long arg); ++ ++static int kbasep_vinstr_hwcnt_reader_mmap( ++ struct file *filp, ++ struct vm_area_struct *vma); ++ ++static int kbasep_vinstr_hwcnt_reader_release( ++ struct inode *inode, ++ struct file *filp); ++ ++/* Vinstr client file operations */ ++static const struct file_operations vinstr_client_fops = { ++ .owner = THIS_MODULE, ++ .poll = kbasep_vinstr_hwcnt_reader_poll, ++ .unlocked_ioctl = kbasep_vinstr_hwcnt_reader_ioctl, ++ .compat_ioctl = kbasep_vinstr_hwcnt_reader_ioctl, ++ .mmap = kbasep_vinstr_hwcnt_reader_mmap, ++ .release = kbasep_vinstr_hwcnt_reader_release, ++}; ++ ++/** ++ * kbasep_vinstr_timestamp_ns() - Get the current time in nanoseconds. ++ * ++ * Return: Current time in nanoseconds. ++ */ ++static u64 kbasep_vinstr_timestamp_ns(void) ++{ ++ return ktime_get_raw_ns(); ++} ++ ++/** ++ * kbasep_vinstr_next_dump_time_ns() - Calculate the next periodic dump time. ++ * @cur_ts_ns: Current time in nanoseconds. ++ * @interval: Interval between dumps in nanoseconds. ++ * ++ * Return: 0 if interval is 0 (i.e. a non-periodic client), or the next dump ++ * time that occurs after cur_ts_ns. ++ */ ++static u64 kbasep_vinstr_next_dump_time_ns(u64 cur_ts_ns, u32 interval) ++{ ++ /* Non-periodic client */ ++ if (interval == 0) ++ return 0; ++ ++ /* ++ * Return the next interval after the current time relative to t=0. ++ * This means multiple clients with the same period will synchronise, ++ * regardless of when they were started, allowing the worker to be ++ * scheduled less frequently. ++ */ ++ do_div(cur_ts_ns, interval); ++ return (cur_ts_ns + 1) * interval; ++} ++ ++/** ++ * kbasep_vinstr_client_dump() - Perform a dump for a client. ++ * @vcli: Non-NULL pointer to a vinstr client. ++ * @event_id: Event type that triggered the dump. ++ * ++ * Return: 0 on success, else error code. ++ */ ++static int kbasep_vinstr_client_dump( ++ struct kbase_vinstr_client *vcli, ++ enum base_hwcnt_reader_event event_id) ++{ ++ int errcode; ++ u64 ts_start_ns; ++ u64 ts_end_ns; ++ unsigned int write_idx; ++ unsigned int read_idx; ++ struct kbase_hwcnt_dump_buffer *tmp_buf; ++ struct kbase_hwcnt_dump_buffer *dump_buf; ++ struct kbase_hwcnt_reader_metadata *meta; ++ u8 clk_cnt; ++ ++ WARN_ON(!vcli); ++ lockdep_assert_held(&vcli->vctx->lock); ++ ++ write_idx = atomic_read(&vcli->write_idx); ++ read_idx = atomic_read(&vcli->read_idx); ++ ++ /* Check if there is a place to copy HWC block into. */ ++ if (write_idx - read_idx == vcli->dump_bufs.buf_cnt) ++ return -EBUSY; ++ write_idx %= vcli->dump_bufs.buf_cnt; ++ ++ dump_buf = &vcli->dump_bufs.bufs[write_idx]; ++ meta = &vcli->dump_bufs_meta[write_idx]; ++ tmp_buf = &vcli->tmp_buf; ++ ++ errcode = kbase_hwcnt_virtualizer_client_dump( ++ vcli->hvcli, &ts_start_ns, &ts_end_ns, tmp_buf); ++ if (errcode) ++ return errcode; ++ ++ /* Patch the dump buf headers, to hide the counters that other hwcnt ++ * clients are using. ++ */ ++ kbase_hwcnt_gpu_patch_dump_headers(tmp_buf, &vcli->enable_map); ++ ++ /* Copy the temp buffer to the userspace visible buffer. The strict ++ * variant will explicitly zero any non-enabled counters to ensure ++ * nothing except exactly what the user asked for is made visible. ++ */ ++ kbase_hwcnt_dump_buffer_copy_strict( ++ dump_buf, tmp_buf, &vcli->enable_map); ++ ++ clk_cnt = vcli->vctx->metadata->clk_cnt; ++ ++ meta->timestamp = ts_end_ns; ++ meta->event_id = event_id; ++ meta->buffer_idx = write_idx; ++ meta->cycles.top = (clk_cnt > 0) ? dump_buf->clk_cnt_buf[0] : 0; ++ meta->cycles.shader_cores = ++ (clk_cnt > 1) ? dump_buf->clk_cnt_buf[1] : 0; ++ ++ /* Notify client. Make sure all changes to memory are visible. */ ++ wmb(); ++ atomic_inc(&vcli->write_idx); ++ wake_up_interruptible(&vcli->waitq); ++ return 0; ++} ++ ++/** ++ * kbasep_vinstr_client_clear() - Reset all the client's counters to zero. ++ * @vcli: Non-NULL pointer to a vinstr client. ++ * ++ * Return: 0 on success, else error code. ++ */ ++static int kbasep_vinstr_client_clear(struct kbase_vinstr_client *vcli) ++{ ++ u64 ts_start_ns; ++ u64 ts_end_ns; ++ ++ WARN_ON(!vcli); ++ lockdep_assert_held(&vcli->vctx->lock); ++ ++ /* A virtualizer dump with a NULL buffer will just clear the virtualizer ++ * client's buffer. ++ */ ++ return kbase_hwcnt_virtualizer_client_dump( ++ vcli->hvcli, &ts_start_ns, &ts_end_ns, NULL); ++} ++ ++/** ++ * kbasep_vinstr_reschedule_worker() - Update next dump times for all periodic ++ * vinstr clients, then reschedule the dump ++ * worker appropriately. ++ * @vctx: Non-NULL pointer to the vinstr context. ++ * ++ * If there are no periodic clients, then the dump worker will not be ++ * rescheduled. Else, the dump worker will be rescheduled for the next periodic ++ * client dump. ++ */ ++static void kbasep_vinstr_reschedule_worker(struct kbase_vinstr_context *vctx) ++{ ++ u64 cur_ts_ns; ++ u64 earliest_next_ns = U64_MAX; ++ struct kbase_vinstr_client *pos; ++ ++ WARN_ON(!vctx); ++ lockdep_assert_held(&vctx->lock); ++ ++ cur_ts_ns = kbasep_vinstr_timestamp_ns(); ++ ++ /* ++ * Update each client's next dump time, and find the earliest next ++ * dump time if any of the clients have a non-zero interval. ++ */ ++ list_for_each_entry(pos, &vctx->clients, node) { ++ const u64 cli_next_ns = ++ kbasep_vinstr_next_dump_time_ns( ++ cur_ts_ns, pos->dump_interval_ns); ++ ++ /* Non-zero next dump time implies a periodic client */ ++ if ((cli_next_ns != 0) && (cli_next_ns < earliest_next_ns)) ++ earliest_next_ns = cli_next_ns; ++ ++ pos->next_dump_time_ns = cli_next_ns; ++ } ++ ++ /* Cancel the timer if it is already pending */ ++ hrtimer_cancel(&vctx->dump_timer); ++ ++ /* Start the timer if there are periodic clients and vinstr is not ++ * suspended. ++ */ ++ if ((earliest_next_ns != U64_MAX) && ++ (vctx->suspend_count == 0) && ++ !WARN_ON(earliest_next_ns < cur_ts_ns)) ++ hrtimer_start( ++ &vctx->dump_timer, ++ ns_to_ktime(earliest_next_ns - cur_ts_ns), ++ HRTIMER_MODE_REL); ++} ++ ++/** ++ * kbasep_vinstr_dump_worker()- Dump worker, that dumps all periodic clients ++ * that need to be dumped, then reschedules itself. ++ * @work: Work structure. ++ */ ++static void kbasep_vinstr_dump_worker(struct work_struct *work) ++{ ++ struct kbase_vinstr_context *vctx = ++ container_of(work, struct kbase_vinstr_context, dump_work); ++ struct kbase_vinstr_client *pos; ++ u64 cur_time_ns; ++ ++ mutex_lock(&vctx->lock); ++ ++ cur_time_ns = kbasep_vinstr_timestamp_ns(); ++ ++ /* Dump all periodic clients whose next dump time is before the current ++ * time. ++ */ ++ list_for_each_entry(pos, &vctx->clients, node) { ++ if ((pos->next_dump_time_ns != 0) && ++ (pos->next_dump_time_ns < cur_time_ns)) ++ kbasep_vinstr_client_dump( ++ pos, BASE_HWCNT_READER_EVENT_PERIODIC); ++ } ++ ++ /* Update the next dump times of all periodic clients, then reschedule ++ * this worker at the earliest next dump time. ++ */ ++ kbasep_vinstr_reschedule_worker(vctx); ++ ++ mutex_unlock(&vctx->lock); ++} ++ ++/** ++ * kbasep_vinstr_dump_timer() - Dump timer that schedules the dump worker for ++ * execution as soon as possible. ++ * @timer: Timer structure. ++ */ ++static enum hrtimer_restart kbasep_vinstr_dump_timer(struct hrtimer *timer) ++{ ++ struct kbase_vinstr_context *vctx = ++ container_of(timer, struct kbase_vinstr_context, dump_timer); ++ ++ /* We don't need to check vctx->suspend_count here, as the suspend ++ * function will ensure that any worker enqueued here is immediately ++ * cancelled, and the worker itself won't reschedule this timer if ++ * suspend_count != 0. ++ */ ++#if KERNEL_VERSION(3, 16, 0) > LINUX_VERSION_CODE ++ queue_work(system_wq, &vctx->dump_work); ++#else ++ queue_work(system_highpri_wq, &vctx->dump_work); ++#endif ++ return HRTIMER_NORESTART; ++} ++ ++/** ++ * kbasep_vinstr_client_destroy() - Destroy a vinstr client. ++ * @vcli: vinstr client. Must not be attached to a vinstr context. ++ */ ++static void kbasep_vinstr_client_destroy(struct kbase_vinstr_client *vcli) ++{ ++ if (!vcli) ++ return; ++ ++ kbase_hwcnt_virtualizer_client_destroy(vcli->hvcli); ++ kfree(vcli->dump_bufs_meta); ++ kbase_hwcnt_dump_buffer_array_free(&vcli->dump_bufs); ++ kbase_hwcnt_dump_buffer_free(&vcli->tmp_buf); ++ kbase_hwcnt_enable_map_free(&vcli->enable_map); ++ kfree(vcli); ++} ++ ++/** ++ * kbasep_vinstr_client_create() - Create a vinstr client. Does not attach to ++ * the vinstr context. ++ * @vctx: Non-NULL pointer to vinstr context. ++ * @setup: Non-NULL pointer to hardware counter ioctl setup structure. ++ * setup->buffer_count must not be 0. ++ * @out_vcli: Non-NULL pointer to where created client will be stored on ++ * success. ++ * ++ * Return: 0 on success, else error code. ++ */ ++static int kbasep_vinstr_client_create( ++ struct kbase_vinstr_context *vctx, ++ struct kbase_ioctl_hwcnt_reader_setup *setup, ++ struct kbase_vinstr_client **out_vcli) ++{ ++ int errcode; ++ struct kbase_vinstr_client *vcli; ++ struct kbase_hwcnt_physical_enable_map phys_em; ++ ++ WARN_ON(!vctx); ++ WARN_ON(!setup); ++ WARN_ON(setup->buffer_count == 0); ++ ++ vcli = kzalloc(sizeof(*vcli), GFP_KERNEL); ++ if (!vcli) ++ return -ENOMEM; ++ ++ vcli->vctx = vctx; ++ ++ errcode = kbase_hwcnt_enable_map_alloc( ++ vctx->metadata, &vcli->enable_map); ++ if (errcode) ++ goto error; ++ ++ phys_em.fe_bm = setup->fe_bm; ++ phys_em.shader_bm = setup->shader_bm; ++ phys_em.tiler_bm = setup->tiler_bm; ++ phys_em.mmu_l2_bm = setup->mmu_l2_bm; ++ kbase_hwcnt_gpu_enable_map_from_physical(&vcli->enable_map, &phys_em); ++ ++ errcode = kbase_hwcnt_dump_buffer_alloc(vctx->metadata, &vcli->tmp_buf); ++ if (errcode) ++ goto error; ++ ++ /* Enable all the available clk_enable_map. */ ++ vcli->enable_map.clk_enable_map = (1ull << vctx->metadata->clk_cnt) - 1; ++ ++ errcode = kbase_hwcnt_dump_buffer_array_alloc( ++ vctx->metadata, setup->buffer_count, &vcli->dump_bufs); ++ if (errcode) ++ goto error; ++ ++ errcode = -ENOMEM; ++ vcli->dump_bufs_meta = kmalloc_array( ++ setup->buffer_count, sizeof(*vcli->dump_bufs_meta), GFP_KERNEL); ++ if (!vcli->dump_bufs_meta) ++ goto error; ++ ++ errcode = kbase_hwcnt_virtualizer_client_create( ++ vctx->hvirt, &vcli->enable_map, &vcli->hvcli); ++ if (errcode) ++ goto error; ++ ++ init_waitqueue_head(&vcli->waitq); ++ ++ *out_vcli = vcli; ++ return 0; ++error: ++ kbasep_vinstr_client_destroy(vcli); ++ return errcode; ++} ++ ++int kbase_vinstr_init( ++ struct kbase_hwcnt_virtualizer *hvirt, ++ struct kbase_vinstr_context **out_vctx) ++{ ++ struct kbase_vinstr_context *vctx; ++ const struct kbase_hwcnt_metadata *metadata; ++ ++ if (!hvirt || !out_vctx) ++ return -EINVAL; ++ ++ metadata = kbase_hwcnt_virtualizer_metadata(hvirt); ++ if (!metadata) ++ return -EINVAL; ++ ++ vctx = kzalloc(sizeof(*vctx), GFP_KERNEL); ++ if (!vctx) ++ return -ENOMEM; ++ ++ vctx->hvirt = hvirt; ++ vctx->metadata = metadata; ++ ++ mutex_init(&vctx->lock); ++ INIT_LIST_HEAD(&vctx->clients); ++ hrtimer_init(&vctx->dump_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); ++ vctx->dump_timer.function = kbasep_vinstr_dump_timer; ++ INIT_WORK(&vctx->dump_work, kbasep_vinstr_dump_worker); ++ ++ *out_vctx = vctx; ++ return 0; ++} ++ ++void kbase_vinstr_term(struct kbase_vinstr_context *vctx) ++{ ++ if (!vctx) ++ return; ++ ++ cancel_work_sync(&vctx->dump_work); ++ ++ /* Non-zero client count implies client leak */ ++ if (WARN_ON(vctx->client_count != 0)) { ++ struct kbase_vinstr_client *pos, *n; ++ ++ list_for_each_entry_safe(pos, n, &vctx->clients, node) { ++ list_del(&pos->node); ++ vctx->client_count--; ++ kbasep_vinstr_client_destroy(pos); ++ } ++ } ++ ++ WARN_ON(vctx->client_count != 0); ++ kfree(vctx); ++} ++ ++void kbase_vinstr_suspend(struct kbase_vinstr_context *vctx) ++{ ++ if (WARN_ON(!vctx)) ++ return; ++ ++ mutex_lock(&vctx->lock); ++ ++ if (!WARN_ON(vctx->suspend_count == SIZE_MAX)) ++ vctx->suspend_count++; ++ ++ mutex_unlock(&vctx->lock); ++ ++ /* Always sync cancel the timer and then the worker, regardless of the ++ * new suspend count. ++ * ++ * This ensures concurrent calls to kbase_vinstr_suspend() always block ++ * until vinstr is fully suspended. ++ * ++ * The timer is cancelled before the worker, as the timer ++ * unconditionally re-enqueues the worker, but the worker checks the ++ * suspend_count that we just incremented before rescheduling the timer. ++ * ++ * Therefore if we cancel the worker first, the timer might re-enqueue ++ * the worker before we cancel the timer, but the opposite is not ++ * possible. ++ */ ++ hrtimer_cancel(&vctx->dump_timer); ++ cancel_work_sync(&vctx->dump_work); ++} ++ ++void kbase_vinstr_resume(struct kbase_vinstr_context *vctx) ++{ ++ if (WARN_ON(!vctx)) ++ return; ++ ++ mutex_lock(&vctx->lock); ++ ++ if (!WARN_ON(vctx->suspend_count == 0)) { ++ vctx->suspend_count--; ++ ++ /* Last resume, so re-enqueue the worker if we have any periodic ++ * clients. ++ */ ++ if (vctx->suspend_count == 0) { ++ struct kbase_vinstr_client *pos; ++ bool has_periodic_clients = false; ++ ++ list_for_each_entry(pos, &vctx->clients, node) { ++ if (pos->dump_interval_ns != 0) { ++ has_periodic_clients = true; ++ break; ++ } ++ } ++ ++ if (has_periodic_clients) ++#if KERNEL_VERSION(3, 16, 0) > LINUX_VERSION_CODE ++ queue_work(system_wq, &vctx->dump_work); ++#else ++ queue_work(system_highpri_wq, &vctx->dump_work); ++#endif ++ } ++ } ++ ++ mutex_unlock(&vctx->lock); ++} ++ ++int kbase_vinstr_hwcnt_reader_setup( ++ struct kbase_vinstr_context *vctx, ++ struct kbase_ioctl_hwcnt_reader_setup *setup) ++{ ++ int errcode; ++ int fd; ++ struct kbase_vinstr_client *vcli = NULL; ++ ++ if (!vctx || !setup || ++ (setup->buffer_count == 0) || ++ (setup->buffer_count > MAX_BUFFER_COUNT)) ++ return -EINVAL; ++ ++ errcode = kbasep_vinstr_client_create(vctx, setup, &vcli); ++ if (errcode) ++ goto error; ++ ++ /* Add the new client. No need to reschedule worker, as not periodic */ ++ mutex_lock(&vctx->lock); ++ ++ vctx->client_count++; ++ list_add(&vcli->node, &vctx->clients); ++ ++ mutex_unlock(&vctx->lock); ++ ++ /* Expose to user-space only once the client is fully initialized */ ++ errcode = anon_inode_getfd( ++ "[mali_vinstr_desc]", ++ &vinstr_client_fops, ++ vcli, ++ O_RDONLY | O_CLOEXEC); ++ if (errcode < 0) ++ goto client_installed_error; ++ ++ fd = errcode; ++ ++ return fd; ++ ++client_installed_error: ++ mutex_lock(&vctx->lock); ++ ++ vctx->client_count--; ++ list_del(&vcli->node); ++ ++ mutex_unlock(&vctx->lock); ++error: ++ kbasep_vinstr_client_destroy(vcli); ++ return errcode; ++} ++ ++/** ++ * kbasep_vinstr_hwcnt_reader_buffer_ready() - Check if client has ready ++ * buffers. ++ * @cli: Non-NULL pointer to vinstr client. ++ * ++ * Return: Non-zero if client has at least one dumping buffer filled that was ++ * not notified to user yet. ++ */ ++static int kbasep_vinstr_hwcnt_reader_buffer_ready( ++ struct kbase_vinstr_client *cli) ++{ ++ WARN_ON(!cli); ++ return atomic_read(&cli->write_idx) != atomic_read(&cli->meta_idx); ++} ++ ++/** ++ * kbasep_vinstr_hwcnt_reader_ioctl_dump() - Dump ioctl command. ++ * @cli: Non-NULL pointer to vinstr client. ++ * ++ * Return: 0 on success, else error code. ++ */ ++static long kbasep_vinstr_hwcnt_reader_ioctl_dump( ++ struct kbase_vinstr_client *cli) ++{ ++ int errcode; ++ ++ mutex_lock(&cli->vctx->lock); ++ ++ errcode = kbasep_vinstr_client_dump( ++ cli, BASE_HWCNT_READER_EVENT_MANUAL); ++ ++ mutex_unlock(&cli->vctx->lock); ++ return errcode; ++} ++ ++/** ++ * kbasep_vinstr_hwcnt_reader_ioctl_clear() - Clear ioctl command. ++ * @cli: Non-NULL pointer to vinstr client. ++ * ++ * Return: 0 on success, else error code. ++ */ ++static long kbasep_vinstr_hwcnt_reader_ioctl_clear( ++ struct kbase_vinstr_client *cli) ++{ ++ int errcode; ++ ++ mutex_lock(&cli->vctx->lock); ++ ++ errcode = kbasep_vinstr_client_clear(cli); ++ ++ mutex_unlock(&cli->vctx->lock); ++ return errcode; ++} ++ ++/** ++ * kbasep_vinstr_hwcnt_reader_ioctl_get_buffer() - Get buffer ioctl command. ++ * @cli: Non-NULL pointer to vinstr client. ++ * @buffer: Non-NULL pointer to userspace buffer. ++ * @size: Size of buffer. ++ * ++ * Return: 0 on success, else error code. ++ */ ++static long kbasep_vinstr_hwcnt_reader_ioctl_get_buffer( ++ struct kbase_vinstr_client *cli, ++ void __user *buffer, ++ size_t size) ++{ ++ unsigned int meta_idx = atomic_read(&cli->meta_idx); ++ unsigned int idx = meta_idx % cli->dump_bufs.buf_cnt; ++ ++ struct kbase_hwcnt_reader_metadata *meta = &cli->dump_bufs_meta[idx]; ++ const size_t meta_size = sizeof(struct kbase_hwcnt_reader_metadata); ++ const size_t min_size = min(size, meta_size); ++ ++ /* Metadata sanity check. */ ++ WARN_ON(idx != meta->buffer_idx); ++ ++ /* Check if there is any buffer available. */ ++ if (unlikely(atomic_read(&cli->write_idx) == meta_idx)) ++ return -EAGAIN; ++ ++ /* Check if previously taken buffer was put back. */ ++ if (unlikely(atomic_read(&cli->read_idx) != meta_idx)) ++ return -EBUSY; ++ ++ /* Clear user buffer to zero. */ ++ if (unlikely(meta_size < size && clear_user(buffer, size))) ++ return -EFAULT; ++ ++ /* Copy next available buffer's metadata to user. */ ++ if (unlikely(copy_to_user(buffer, meta, min_size))) ++ return -EFAULT; ++ ++ atomic_inc(&cli->meta_idx); ++ ++ return 0; ++} ++ ++/** ++ * kbasep_vinstr_hwcnt_reader_ioctl_put_buffer() - Put buffer ioctl command. ++ * @cli: Non-NULL pointer to vinstr client. ++ * @buffer: Non-NULL pointer to userspace buffer. ++ * @size: Size of buffer. ++ * ++ * Return: 0 on success, else error code. ++ */ ++static long kbasep_vinstr_hwcnt_reader_ioctl_put_buffer( ++ struct kbase_vinstr_client *cli, ++ void __user *buffer, ++ size_t size) ++{ ++ unsigned int read_idx = atomic_read(&cli->read_idx); ++ unsigned int idx = read_idx % cli->dump_bufs.buf_cnt; ++ ++ struct kbase_hwcnt_reader_metadata *meta; ++ const size_t meta_size = sizeof(struct kbase_hwcnt_reader_metadata); ++ const size_t max_size = max(size, meta_size); ++ int ret = 0; ++ u8 stack_kbuf[64]; ++ u8 *kbuf = NULL; ++ size_t i; ++ ++ /* Check if any buffer was taken. */ ++ if (unlikely(atomic_read(&cli->meta_idx) == read_idx)) ++ return -EPERM; ++ ++ if (likely(max_size <= sizeof(stack_kbuf))) { ++ /* Use stack buffer when the size is small enough. */ ++ if (unlikely(meta_size > size)) ++ memset(stack_kbuf, 0, sizeof(stack_kbuf)); ++ kbuf = stack_kbuf; ++ } else { ++ kbuf = kzalloc(max_size, GFP_KERNEL); ++ if (unlikely(!kbuf)) ++ return -ENOMEM; ++ } ++ ++ /* ++ * Copy user buffer to zero cleared kernel buffer which has enough ++ * space for both user buffer and kernel metadata. ++ */ ++ if (unlikely(copy_from_user(kbuf, buffer, size))) { ++ ret = -EFAULT; ++ goto out; ++ } ++ ++ /* ++ * Make sure any "extra" data passed from userspace is zero. ++ * It's meaningful only in case meta_size < size. ++ */ ++ for (i = meta_size; i < size; i++) { ++ /* Check if user data beyond meta size is zero. */ ++ if (unlikely(kbuf[i] != 0)) { ++ ret = -EINVAL; ++ goto out; ++ } ++ } ++ ++ /* Check if correct buffer is put back. */ ++ meta = (struct kbase_hwcnt_reader_metadata *)kbuf; ++ if (unlikely(idx != meta->buffer_idx)) { ++ ret = -EINVAL; ++ goto out; ++ } ++ ++ atomic_inc(&cli->read_idx); ++out: ++ if (unlikely(kbuf != stack_kbuf)) ++ kfree(kbuf); ++ return ret; ++} ++ ++/** ++ * kbasep_vinstr_hwcnt_reader_ioctl_set_interval() - Set interval ioctl command. ++ * @cli: Non-NULL pointer to vinstr client. ++ * @interval: Periodic dumping interval (disable periodic dumping if 0). ++ * ++ * Return: 0 always. ++ */ ++static long kbasep_vinstr_hwcnt_reader_ioctl_set_interval( ++ struct kbase_vinstr_client *cli, ++ u32 interval) ++{ ++ mutex_lock(&cli->vctx->lock); ++ ++ if ((interval != 0) && (interval < DUMP_INTERVAL_MIN_NS)) ++ interval = DUMP_INTERVAL_MIN_NS; ++ /* Update the interval, and put in a dummy next dump time */ ++ cli->dump_interval_ns = interval; ++ cli->next_dump_time_ns = 0; ++ ++ /* ++ * If it's a periodic client, kick off the worker early to do a proper ++ * timer reschedule. Return value is ignored, as we don't care if the ++ * worker is already queued. ++ */ ++ if ((interval != 0) && (cli->vctx->suspend_count == 0)) ++#if KERNEL_VERSION(3, 16, 0) > LINUX_VERSION_CODE ++ queue_work(system_wq, &cli->vctx->dump_work); ++#else ++ queue_work(system_highpri_wq, &cli->vctx->dump_work); ++#endif ++ ++ mutex_unlock(&cli->vctx->lock); ++ ++ return 0; ++} ++ ++/** ++ * kbasep_vinstr_hwcnt_reader_ioctl_enable_event() - Enable event ioctl command. ++ * @cli: Non-NULL pointer to vinstr client. ++ * @event_id: ID of event to enable. ++ * ++ * Return: 0 always. ++ */ ++static long kbasep_vinstr_hwcnt_reader_ioctl_enable_event( ++ struct kbase_vinstr_client *cli, ++ enum base_hwcnt_reader_event event_id) ++{ ++ /* No-op, as events aren't supported */ ++ return 0; ++} ++ ++/** ++ * kbasep_vinstr_hwcnt_reader_ioctl_disable_event() - Disable event ioctl ++ * command. ++ * @cli: Non-NULL pointer to vinstr client. ++ * @event_id: ID of event to disable. ++ * ++ * Return: 0 always. ++ */ ++static long kbasep_vinstr_hwcnt_reader_ioctl_disable_event( ++ struct kbase_vinstr_client *cli, ++ enum base_hwcnt_reader_event event_id) ++{ ++ /* No-op, as events aren't supported */ ++ return 0; ++} ++ ++/** ++ * kbasep_vinstr_hwcnt_reader_ioctl_get_hwver() - Get HW version ioctl command. ++ * @cli: Non-NULL pointer to vinstr client. ++ * @hwver: Non-NULL pointer to user buffer where HW version will be stored. ++ * ++ * Return: 0 on success, else error code. ++ */ ++static long kbasep_vinstr_hwcnt_reader_ioctl_get_hwver( ++ struct kbase_vinstr_client *cli, ++ u32 __user *hwver) ++{ ++ u32 ver = 5; ++ const enum kbase_hwcnt_gpu_group_type type = ++ kbase_hwcnt_metadata_group_type(cli->vctx->metadata, 0); ++ ++ if (WARN_ON(type != KBASE_HWCNT_GPU_GROUP_TYPE_V5)) ++ return -EINVAL; ++ ++ return put_user(ver, hwver); ++} ++ ++/** ++ * The hwcnt reader's ioctl command - get API version. ++ * @cli: The non-NULL pointer to the client ++ * @arg: Command's argument. ++ * @size: Size of arg. ++ * ++ * @return 0 on success, else error code. ++ */ ++static long kbasep_vinstr_hwcnt_reader_ioctl_get_api_version( ++ struct kbase_vinstr_client *cli, unsigned long arg, size_t size) ++{ ++ long ret = -EINVAL; ++ u8 clk_cnt = cli->vctx->metadata->clk_cnt; ++ ++ if (size == sizeof(u32)) { ++ ret = put_user(HWCNT_READER_API, (u32 __user *)arg); ++ } else if (size == sizeof(struct kbase_hwcnt_reader_api_version)) { ++ struct kbase_hwcnt_reader_api_version api_version = { ++ .version = HWCNT_READER_API, ++ .features = KBASE_HWCNT_READER_API_VERSION_NO_FEATURE, ++ }; ++ ++ if (clk_cnt > 0) ++ api_version.features |= ++ KBASE_HWCNT_READER_API_VERSION_FEATURE_CYCLES_TOP; ++ if (clk_cnt > 1) ++ api_version.features |= ++ KBASE_HWCNT_READER_API_VERSION_FEATURE_CYCLES_SHADER_CORES; ++ ++ ret = copy_to_user( ++ (void __user *)arg, &api_version, sizeof(api_version)); ++ } ++ return ret; ++} ++ ++/** ++ * kbasep_vinstr_hwcnt_reader_ioctl() - hwcnt reader's ioctl. ++ * @filp: Non-NULL pointer to file structure. ++ * @cmd: User command. ++ * @arg: Command's argument. ++ * ++ * Return: 0 on success, else error code. ++ */ ++static long kbasep_vinstr_hwcnt_reader_ioctl( ++ struct file *filp, ++ unsigned int cmd, ++ unsigned long arg) ++{ ++ long rcode; ++ struct kbase_vinstr_client *cli; ++ ++ if (!filp || (_IOC_TYPE(cmd) != KBASE_HWCNT_READER)) ++ return -EINVAL; ++ ++ cli = filp->private_data; ++ if (!cli) ++ return -EINVAL; ++ ++ switch (_IOC_NR(cmd)) { ++ case _IOC_NR(KBASE_HWCNT_READER_GET_API_VERSION): ++ rcode = kbasep_vinstr_hwcnt_reader_ioctl_get_api_version( ++ cli, arg, _IOC_SIZE(cmd)); ++ break; ++ case _IOC_NR(KBASE_HWCNT_READER_GET_HWVER): ++ rcode = kbasep_vinstr_hwcnt_reader_ioctl_get_hwver( ++ cli, (u32 __user *)arg); ++ break; ++ case _IOC_NR(KBASE_HWCNT_READER_GET_BUFFER_SIZE): ++ rcode = put_user( ++ (u32)cli->vctx->metadata->dump_buf_bytes, ++ (u32 __user *)arg); ++ break; ++ case _IOC_NR(KBASE_HWCNT_READER_DUMP): ++ rcode = kbasep_vinstr_hwcnt_reader_ioctl_dump(cli); ++ break; ++ case _IOC_NR(KBASE_HWCNT_READER_CLEAR): ++ rcode = kbasep_vinstr_hwcnt_reader_ioctl_clear(cli); ++ break; ++ case _IOC_NR(KBASE_HWCNT_READER_GET_BUFFER): ++ rcode = kbasep_vinstr_hwcnt_reader_ioctl_get_buffer( ++ cli, (void __user *)arg, _IOC_SIZE(cmd)); ++ break; ++ case _IOC_NR(KBASE_HWCNT_READER_PUT_BUFFER): ++ rcode = kbasep_vinstr_hwcnt_reader_ioctl_put_buffer( ++ cli, (void __user *)arg, _IOC_SIZE(cmd)); ++ break; ++ case _IOC_NR(KBASE_HWCNT_READER_SET_INTERVAL): ++ rcode = kbasep_vinstr_hwcnt_reader_ioctl_set_interval( ++ cli, (u32)arg); ++ break; ++ case _IOC_NR(KBASE_HWCNT_READER_ENABLE_EVENT): ++ rcode = kbasep_vinstr_hwcnt_reader_ioctl_enable_event( ++ cli, (enum base_hwcnt_reader_event)arg); ++ break; ++ case _IOC_NR(KBASE_HWCNT_READER_DISABLE_EVENT): ++ rcode = kbasep_vinstr_hwcnt_reader_ioctl_disable_event( ++ cli, (enum base_hwcnt_reader_event)arg); ++ break; ++ default: ++ pr_warn("Unknown HWCNT ioctl 0x%x nr:%d", cmd, _IOC_NR(cmd)); ++ rcode = -EINVAL; ++ break; ++ } ++ ++ return rcode; ++} ++ ++/** ++ * kbasep_vinstr_hwcnt_reader_poll() - hwcnt reader's poll. ++ * @filp: Non-NULL pointer to file structure. ++ * @wait: Non-NULL pointer to poll table. ++ * ++ * Return: POLLIN if data can be read without blocking, 0 if data can not be ++ * read without blocking, else error code. ++ */ ++static unsigned int kbasep_vinstr_hwcnt_reader_poll( ++ struct file *filp, ++ poll_table *wait) ++{ ++ struct kbase_vinstr_client *cli; ++ ++ if (!filp || !wait) ++ return -EINVAL; ++ ++ cli = filp->private_data; ++ if (!cli) ++ return -EINVAL; ++ ++ poll_wait(filp, &cli->waitq, wait); ++ if (kbasep_vinstr_hwcnt_reader_buffer_ready(cli)) ++ return POLLIN; ++ return 0; ++} ++ ++/** ++ * kbasep_vinstr_hwcnt_reader_mmap() - hwcnt reader's mmap. ++ * @filp: Non-NULL pointer to file structure. ++ * @vma: Non-NULL pointer to vma structure. ++ * ++ * Return: 0 on success, else error code. ++ */ ++static int kbasep_vinstr_hwcnt_reader_mmap( ++ struct file *filp, ++ struct vm_area_struct *vma) ++{ ++ struct kbase_vinstr_client *cli; ++ unsigned long vm_size, size, addr, pfn, offset; ++ ++ if (!filp || !vma) ++ return -EINVAL; ++ ++ cli = filp->private_data; ++ if (!cli) ++ return -EINVAL; ++ ++ vm_size = vma->vm_end - vma->vm_start; ++ size = cli->dump_bufs.buf_cnt * cli->vctx->metadata->dump_buf_bytes; ++ ++ if (vma->vm_pgoff > (size >> PAGE_SHIFT)) ++ return -EINVAL; ++ ++ offset = vma->vm_pgoff << PAGE_SHIFT; ++ if (vm_size > size - offset) ++ return -EINVAL; ++ ++ addr = __pa(cli->dump_bufs.page_addr + offset); ++ pfn = addr >> PAGE_SHIFT; ++ ++ return remap_pfn_range( ++ vma, vma->vm_start, pfn, vm_size, vma->vm_page_prot); ++} ++ ++/** ++ * kbasep_vinstr_hwcnt_reader_release() - hwcnt reader's release. ++ * @inode: Non-NULL pointer to inode structure. ++ * @filp: Non-NULL pointer to file structure. ++ * ++ * Return: 0 always. ++ */ ++static int kbasep_vinstr_hwcnt_reader_release(struct inode *inode, ++ struct file *filp) ++{ ++ struct kbase_vinstr_client *vcli = filp->private_data; ++ ++ mutex_lock(&vcli->vctx->lock); ++ ++ vcli->vctx->client_count--; ++ list_del(&vcli->node); ++ ++ mutex_unlock(&vcli->vctx->lock); ++ ++ kbasep_vinstr_client_destroy(vcli); ++ ++ return 0; ++} +diff --git a/drivers/gpu/arm/bifrost/mali_kbase_vinstr.h b/drivers/gpu/arm/bifrost/mali_kbase_vinstr.h +new file mode 100755 +index 000000000000..81d315f95567 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_kbase_vinstr.h +@@ -0,0 +1,91 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015-2018 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* ++ * Vinstr, used to provide an ioctl for userspace access to periodic hardware ++ * counters. ++ */ ++ ++#ifndef _KBASE_VINSTR_H_ ++#define _KBASE_VINSTR_H_ ++ ++struct kbase_vinstr_context; ++struct kbase_hwcnt_virtualizer; ++struct kbase_ioctl_hwcnt_reader_setup; ++ ++/** ++ * kbase_vinstr_init() - Initialise a vinstr context. ++ * @hvirt: Non-NULL pointer to the hardware counter virtualizer. ++ * @out_vctx: Non-NULL pointer to where the pointer to the created vinstr ++ * context will be stored on success. ++ * ++ * On creation, the suspend count of the context will be 0. ++ * ++ * Return: 0 on success, else error code. ++ */ ++int kbase_vinstr_init( ++ struct kbase_hwcnt_virtualizer *hvirt, ++ struct kbase_vinstr_context **out_vctx); ++ ++/** ++ * kbase_vinstr_term() - Terminate a vinstr context. ++ * @vctx: Pointer to the vinstr context to be terminated. ++ */ ++void kbase_vinstr_term(struct kbase_vinstr_context *vctx); ++ ++/** ++ * kbase_vinstr_suspend() - Increment the suspend count of the context. ++ * @vctx: Non-NULL pointer to the vinstr context to be suspended. ++ * ++ * After this function call returns, it is guaranteed that all timers and ++ * workers in vinstr will be cancelled, and will not be re-triggered until ++ * after the context has been resumed. In effect, this means no new counter ++ * dumps will occur for any existing or subsequently added periodic clients. ++ */ ++void kbase_vinstr_suspend(struct kbase_vinstr_context *vctx); ++ ++/** ++ * kbase_vinstr_resume() - Decrement the suspend count of the context. ++ * @vctx: Non-NULL pointer to the vinstr context to be resumed. ++ * ++ * If a call to this function decrements the suspend count from 1 to 0, then ++ * normal operation of vinstr will be resumed (i.e. counter dumps will once ++ * again be automatically triggered for all periodic clients). ++ * ++ * It is only valid to call this function one time for each prior returned call ++ * to kbase_vinstr_suspend. ++ */ ++void kbase_vinstr_resume(struct kbase_vinstr_context *vctx); ++ ++/** ++ * kbase_vinstr_hwcnt_reader_setup() - Set up a new hardware counter reader ++ * client. ++ * @vinstr_ctx: Non-NULL pointer to the vinstr context. ++ * @setup: Non-NULL pointer to the hwcnt reader configuration. ++ * ++ * Return: file descriptor on success, else a (negative) error code. ++ */ ++int kbase_vinstr_hwcnt_reader_setup( ++ struct kbase_vinstr_context *vinstr_ctx, ++ struct kbase_ioctl_hwcnt_reader_setup *setup); ++ ++#endif /* _KBASE_VINSTR_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_linux_trace.h b/drivers/gpu/arm/bifrost/mali_linux_trace.h +new file mode 100755 +index 000000000000..be812f62c862 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_linux_trace.h +@@ -0,0 +1,552 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2016, 2018-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#undef TRACE_SYSTEM ++#define TRACE_SYSTEM mali ++ ++#if !defined(_TRACE_MALI_H) || defined(TRACE_HEADER_MULTI_READ) ++#define _TRACE_MALI_H ++ ++#include ++ ++#if defined(CONFIG_MALI_BIFROST_GATOR_SUPPORT) ++#define MALI_JOB_SLOTS_EVENT_CHANGED ++ ++/** ++ * mali_job_slots_event - Reports change of job slot status. ++ * @gpu_id: Kbase device id ++ * @event_id: ORed together bitfields representing a type of event, ++ * made with the GATOR_MAKE_EVENT() macro. ++ */ ++TRACE_EVENT(mali_job_slots_event, ++ TP_PROTO(u32 gpu_id, u32 event_id, u32 tgid, u32 pid, ++ u8 job_id), ++ TP_ARGS(gpu_id, event_id, tgid, pid, job_id), ++ TP_STRUCT__entry( ++ __field(u32, gpu_id) ++ __field(u32, event_id) ++ __field(u32, tgid) ++ __field(u32, pid) ++ __field(u8, job_id) ++ ), ++ TP_fast_assign( ++ __entry->gpu_id = gpu_id; ++ __entry->event_id = event_id; ++ __entry->tgid = tgid; ++ __entry->pid = pid; ++ __entry->job_id = job_id; ++ ), ++ TP_printk("gpu=%u event=%u tgid=%u pid=%u job_id=%u", ++ __entry->gpu_id, __entry->event_id, ++ __entry->tgid, __entry->pid, __entry->job_id) ++); ++ ++/** ++ * mali_pm_status - Reports change of power management status. ++ * @gpu_id: Kbase device id ++ * @event_id: Core type (shader, tiler, L2 cache) ++ * @value: 64bits bitmask reporting either power status of ++ * the cores (1-ON, 0-OFF) ++ */ ++TRACE_EVENT(mali_pm_status, ++ TP_PROTO(u32 gpu_id, u32 event_id, u64 value), ++ TP_ARGS(gpu_id, event_id, value), ++ TP_STRUCT__entry( ++ __field(u32, gpu_id) ++ __field(u32, event_id) ++ __field(u64, value) ++ ), ++ TP_fast_assign( ++ __entry->gpu_id = gpu_id; ++ __entry->event_id = event_id; ++ __entry->value = value; ++ ), ++ TP_printk("gpu=%u event %u = %llu", ++ __entry->gpu_id, __entry->event_id, __entry->value) ++); ++ ++/** ++ * mali_page_fault_insert_pages - Reports an MMU page fault ++ * resulting in new pages being mapped. ++ * @gpu_id: Kbase device id ++ * @event_id: MMU address space number ++ * @value: Number of newly allocated pages ++ */ ++TRACE_EVENT(mali_page_fault_insert_pages, ++ TP_PROTO(u32 gpu_id, s32 event_id, u64 value), ++ TP_ARGS(gpu_id, event_id, value), ++ TP_STRUCT__entry( ++ __field(u32, gpu_id) ++ __field(s32, event_id) ++ __field(u64, value) ++ ), ++ TP_fast_assign( ++ __entry->gpu_id = gpu_id; ++ __entry->event_id = event_id; ++ __entry->value = value; ++ ), ++ TP_printk("gpu=%u event %d = %llu", ++ __entry->gpu_id, __entry->event_id, __entry->value) ++); ++ ++/** ++ * mali_total_alloc_pages_change - Reports that the total number of ++ * allocated pages has changed. ++ * @gpu_id: Kbase device id ++ * @event_id: Total number of pages allocated ++ */ ++TRACE_EVENT(mali_total_alloc_pages_change, ++ TP_PROTO(u32 gpu_id, s64 event_id), ++ TP_ARGS(gpu_id, event_id), ++ TP_STRUCT__entry( ++ __field(u32, gpu_id) ++ __field(s64, event_id) ++ ), ++ TP_fast_assign( ++ __entry->gpu_id = gpu_id; ++ __entry->event_id = event_id; ++ ), ++ TP_printk("gpu=%u event=%lld", __entry->gpu_id, __entry->event_id) ++); ++#endif /* CONFIG_MALI_BIFROST_GATOR_SUPPORT */ ++ ++/* ++ * MMU subsystem tracepoints ++ */ ++ ++/* Fault status and exception code helpers ++ * ++ * Must be macros to allow use by user-side tracepoint tools ++ * ++ * bits 0:1 masked off code, and used for the level ++ * ++ * Tracepoint files get included more than once - protect against multiple ++ * definition ++ */ ++#ifndef __TRACE_MALI_MMU_HELPERS ++#define __TRACE_MALI_MMU_HELPERS ++/* Complex macros should be enclosed in parenthesis. ++ * ++ * We need to have those parentheses removed for our arrays of symbolic look-ups ++ * for __print_symbolic() whilst also being able to use them outside trace code ++ */ ++#define _ENSURE_PARENTHESIS(args...) args ++ ++#define KBASE_MMU_FAULT_CODE_EXCEPTION_NAME_PRINT(code) \ ++ (!KBASE_MMU_FAULT_CODE_VALID(code) ? "UNKNOWN,level=" : \ ++ __print_symbolic(((code) & ~3u), \ ++ KBASE_MMU_FAULT_CODE_SYMBOLIC_STRINGS)) ++#define KBASE_MMU_FAULT_CODE_LEVEL(code) \ ++ (((((code) & ~0x3u) == 0xC4) ? 4 : 0) + ((code) & 0x3u)) ++ ++#define KBASE_MMU_FAULT_STATUS_CODE(status) \ ++ ((status) & 0xFFu) ++#define KBASE_MMU_FAULT_STATUS_DECODED_STRING(status) \ ++ (((status) & (1u << 10)) ? "DECODER_FAULT" : "SLAVE_FAULT") ++ ++#define KBASE_MMU_FAULT_STATUS_EXCEPTION_NAME_PRINT(status) \ ++ KBASE_MMU_FAULT_CODE_EXCEPTION_NAME_PRINT( \ ++ KBASE_MMU_FAULT_STATUS_CODE(status)) ++ ++#define KBASE_MMU_FAULT_STATUS_LEVEL(status) \ ++ KBASE_MMU_FAULT_CODE_LEVEL(KBASE_MMU_FAULT_STATUS_CODE(status)) ++ ++#define KBASE_MMU_FAULT_STATUS_ACCESS(status) \ ++ ((status) & AS_FAULTSTATUS_ACCESS_TYPE_MASK) ++#define KBASE_MMU_FAULT_ACCESS_SYMBOLIC_STRINGS _ENSURE_PARENTHESIS(\ ++ {AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC, "ATOMIC" }, \ ++ {AS_FAULTSTATUS_ACCESS_TYPE_EX, "EXECUTE"}, \ ++ {AS_FAULTSTATUS_ACCESS_TYPE_READ, "READ" }, \ ++ {AS_FAULTSTATUS_ACCESS_TYPE_WRITE, "WRITE" }) ++#define KBASE_MMU_FAULT_STATUS_ACCESS_PRINT(status) \ ++ __print_symbolic(KBASE_MMU_FAULT_STATUS_ACCESS(status), \ ++ KBASE_MMU_FAULT_ACCESS_SYMBOLIC_STRINGS) ++ ++#if MALI_USE_CSF ++#define KBASE_MMU_FAULT_CODE_VALID(code) \ ++ ((code >= 0xC0 && code <= 0xEB) && \ ++ (!(code >= 0xC5 && code <= 0xC7)) && \ ++ (!(code >= 0xCC && code <= 0xD8)) && \ ++ (!(code >= 0xDC && code <= 0xDF)) && \ ++ (!(code >= 0xE1 && code <= 0xE3))) ++#define KBASE_MMU_FAULT_CODE_SYMBOLIC_STRINGS _ENSURE_PARENTHESIS(\ ++ {0xC0, "TRANSLATION_FAULT_" }, \ ++ {0xC4, "TRANSLATION_FAULT_" }, \ ++ {0xC8, "PERMISSION_FAULT_" }, \ ++ {0xD0, "TRANSTAB_BUS_FAULT_" }, \ ++ {0xD8, "ACCESS_FLAG_" }, \ ++ {0xE0, "ADDRESS_SIZE_FAULT_IN" }, \ ++ {0xE4, "ADDRESS_SIZE_FAULT_OUT" }, \ ++ {0xE8, "MEMORY_ATTRIBUTES_FAULT_" }) ++#else /* MALI_USE_CSF */ ++#define KBASE_MMU_FAULT_CODE_VALID(code) \ ++ ((code >= 0xC0 && code <= 0xEF) && \ ++ (!(code >= 0xC5 && code <= 0xC6)) && \ ++ (!(code >= 0xCC && code <= 0xCF)) && \ ++ (!(code >= 0xD4 && code <= 0xD7)) && \ ++ (!(code >= 0xDC && code <= 0xDF))) ++#define KBASE_MMU_FAULT_CODE_SYMBOLIC_STRINGS _ENSURE_PARENTHESIS(\ ++ {0xC0, "TRANSLATION_FAULT_" }, \ ++ {0xC4, "TRANSLATION_FAULT(_7==_IDENTITY)_" }, \ ++ {0xC8, "PERMISSION_FAULT_" }, \ ++ {0xD0, "TRANSTAB_BUS_FAULT_" }, \ ++ {0xD8, "ACCESS_FLAG_" }, \ ++ {0xE0, "ADDRESS_SIZE_FAULT_IN" }, \ ++ {0xE4, "ADDRESS_SIZE_FAULT_OUT" }, \ ++ {0xE8, "MEMORY_ATTRIBUTES_FAULT_" }, \ ++ {0xEC, "MEMORY_ATTRIBUTES_NONCACHEABLE_" }) ++#endif /* MALI_USE_CSF */ ++#endif /* __TRACE_MALI_MMU_HELPERS */ ++ ++/* trace_mali_mmu_page_fault_grow ++ * ++ * Tracepoint about a successful grow of a region due to a GPU page fault ++ */ ++TRACE_EVENT(mali_mmu_page_fault_grow, ++ TP_PROTO(struct kbase_va_region *reg, struct kbase_fault *fault, ++ size_t new_pages), ++ TP_ARGS(reg, fault, new_pages), ++ TP_STRUCT__entry( ++ __field(u64, start_addr) ++ __field(u64, fault_addr) ++ __field(u64, fault_extra_addr) ++ __field(size_t, new_pages) ++ __field(u32, status) ++ ), ++ TP_fast_assign( ++ __entry->start_addr = ((u64)reg->start_pfn) << PAGE_SHIFT; ++ __entry->fault_addr = fault->addr; ++ __entry->fault_extra_addr = fault->extra_addr; ++ __entry->new_pages = new_pages; ++ __entry->status = fault->status; ++ ), ++ TP_printk("start=0x%llx fault_addr=0x%llx fault_extra_addr=0x%llx new_pages=%zu raw_fault_status=0x%x decoded_faultstatus=%s exception_type=0x%x,%s%u access_type=0x%x,%s source_id=0x%x", ++ __entry->start_addr, __entry->fault_addr, ++ __entry->fault_extra_addr, __entry->new_pages, ++ __entry->status, ++ KBASE_MMU_FAULT_STATUS_DECODED_STRING(__entry->status), ++ KBASE_MMU_FAULT_STATUS_CODE(__entry->status), ++ KBASE_MMU_FAULT_STATUS_EXCEPTION_NAME_PRINT(__entry->status), ++ KBASE_MMU_FAULT_STATUS_LEVEL(__entry->status), ++ KBASE_MMU_FAULT_STATUS_ACCESS(__entry->status) >> 8, ++ KBASE_MMU_FAULT_STATUS_ACCESS_PRINT(__entry->status), ++ __entry->status >> 16) ++); ++ ++ ++ ++ ++/* ++ * Just-in-time memory allocation subsystem tracepoints ++ */ ++ ++/* Just-in-time memory allocation soft-job template. Override the TP_printk ++ * further if need be. jit_id can be 0. ++ */ ++DECLARE_EVENT_CLASS(mali_jit_softjob_template, ++ TP_PROTO(struct kbase_va_region *reg, u8 jit_id), ++ TP_ARGS(reg, jit_id), ++ TP_STRUCT__entry( ++ __field(u64, start_addr) ++ __field(size_t, nr_pages) ++ __field(size_t, backed_pages) ++ __field(u8, jit_id) ++ ), ++ TP_fast_assign( ++ __entry->start_addr = ((u64)reg->start_pfn) << PAGE_SHIFT; ++ __entry->nr_pages = reg->nr_pages; ++ __entry->backed_pages = kbase_reg_current_backed_size(reg); ++ __entry->jit_id = jit_id; ++ ), ++ TP_printk("jit_id=%u start=0x%llx va_pages=0x%zx backed_size=0x%zx", ++ __entry->jit_id, __entry->start_addr, __entry->nr_pages, ++ __entry->backed_pages) ++); ++ ++/* trace_mali_jit_alloc() ++ * ++ * Tracepoint about a just-in-time memory allocation soft-job successfully ++ * allocating memory ++ */ ++DEFINE_EVENT(mali_jit_softjob_template, mali_jit_alloc, ++ TP_PROTO(struct kbase_va_region *reg, u8 jit_id), ++ TP_ARGS(reg, jit_id)); ++ ++/* trace_mali_jit_free() ++ * ++ * Tracepoint about memory that was allocated just-in-time being freed ++ * (which may happen either on free soft-job, or during rollback error ++ * paths of an allocation soft-job, etc) ++ * ++ * Free doesn't immediately have the just-in-time memory allocation ID so ++ * it's currently suppressed from the output - set jit_id to 0 ++ */ ++DEFINE_EVENT_PRINT(mali_jit_softjob_template, mali_jit_free, ++ TP_PROTO(struct kbase_va_region *reg, u8 jit_id), ++ TP_ARGS(reg, jit_id), ++ TP_printk("start=0x%llx va_pages=0x%zx backed_size=0x%zx", ++ __entry->start_addr, __entry->nr_pages, __entry->backed_pages)); ++ ++#if !MALI_USE_CSF ++#if MALI_JIT_PRESSURE_LIMIT_BASE ++/* trace_mali_jit_report ++ * ++ * Tracepoint about the GPU data structure read to form a just-in-time memory ++ * allocation report, and its calculated physical page usage ++ */ ++TRACE_EVENT(mali_jit_report, ++ TP_PROTO(struct kbase_jd_atom *katom, struct kbase_va_region *reg, ++ unsigned int id_idx, u64 read_val, u64 used_pages), ++ TP_ARGS(katom, reg, id_idx, read_val, used_pages), ++ TP_STRUCT__entry( ++ __field(u64, start_addr) ++ __field(u64, read_val) ++ __field(u64, used_pages) ++ __field(unsigned long, flags) ++ __field(u8, id_idx) ++ __field(u8, jit_id) ++ ), ++ TP_fast_assign( ++ __entry->start_addr = ((u64)reg->start_pfn) << PAGE_SHIFT; ++ __entry->read_val = read_val; ++ __entry->used_pages = used_pages; ++ __entry->flags = reg->flags; ++ __entry->id_idx = id_idx; ++ __entry->jit_id = katom->jit_ids[id_idx]; ++ ), ++ TP_printk("start=0x%llx jit_ids[%u]=%u read_type='%s' read_val=0x%llx used_pages=%llu", ++ __entry->start_addr, __entry->id_idx, __entry->jit_id, ++ __print_symbolic(__entry->flags, ++ { 0, "address"}, ++ { KBASE_REG_TILER_ALIGN_TOP, "address with align" }, ++ { KBASE_REG_HEAP_INFO_IS_SIZE, "size" }, ++ { KBASE_REG_HEAP_INFO_IS_SIZE | ++ KBASE_REG_TILER_ALIGN_TOP, ++ "size with align (invalid)" } ++ ), ++ __entry->read_val, __entry->used_pages) ++); ++#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ ++#endif /* !MALI_USE_CSF */ ++ ++#if (KERNEL_VERSION(4, 1, 0) <= LINUX_VERSION_CODE) ++TRACE_DEFINE_ENUM(KBASE_JIT_REPORT_ON_ALLOC_OR_FREE); ++#endif ++ ++#if MALI_JIT_PRESSURE_LIMIT_BASE ++/* trace_mali_jit_report_pressure ++ * ++ * Tracepoint about change in physical memory pressure, due to the information ++ * about a region changing. Examples include: ++ * - a report on a region that was allocated just-in-time ++ * - just-in-time allocation of a region ++ * - free of a region that was allocated just-in-time ++ */ ++TRACE_EVENT(mali_jit_report_pressure, ++ TP_PROTO(struct kbase_va_region *reg, u64 new_used_pages, ++ u64 new_pressure, unsigned int flags), ++ TP_ARGS(reg, new_used_pages, new_pressure, flags), ++ TP_STRUCT__entry( ++ __field(u64, start_addr) ++ __field(u64, used_pages) ++ __field(u64, new_used_pages) ++ __field(u64, new_pressure) ++ __field(unsigned int, flags) ++ ), ++ TP_fast_assign( ++ __entry->start_addr = ((u64)reg->start_pfn) << PAGE_SHIFT; ++ __entry->used_pages = reg->used_pages; ++ __entry->new_used_pages = new_used_pages; ++ __entry->new_pressure = new_pressure; ++ __entry->flags = flags; ++ ), ++ TP_printk("start=0x%llx old_used_pages=%llu new_used_pages=%llu new_pressure=%llu report_flags=%s", ++ __entry->start_addr, __entry->used_pages, ++ __entry->new_used_pages, __entry->new_pressure, ++ __print_flags(__entry->flags, "|", ++ { KBASE_JIT_REPORT_ON_ALLOC_OR_FREE, ++ "HAPPENED_ON_ALLOC_OR_FREE" })) ++); ++#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ ++ ++#ifndef __TRACE_SYSGRAPH_ENUM ++#define __TRACE_SYSGRAPH_ENUM ++/* Enum of sysgraph message IDs */ ++enum sysgraph_msg { ++ SGR_ARRIVE, ++ SGR_DEP_RES, ++ SGR_SUBMIT, ++ SGR_COMPLETE, ++ SGR_POST, ++ SGR_ACTIVE, ++ SGR_INACTIVE ++}; ++#endif /* __TRACE_SYSGRAPH_ENUM */ ++ ++/* A template for SYSGRAPH events ++ * ++ * Most of the sysgraph events contain only one input argument ++ * which is atom_id therefore they will be using a common template ++ */ ++TRACE_EVENT(sysgraph, ++ TP_PROTO(enum sysgraph_msg message, unsigned int proc_id, ++ unsigned int atom_id), ++ TP_ARGS(message, proc_id, atom_id), ++ TP_STRUCT__entry( ++ __field(unsigned int, proc_id) ++ __field(enum sysgraph_msg, message) ++ __field(unsigned int, atom_id) ++ ), ++ TP_fast_assign( ++ __entry->proc_id = proc_id; ++ __entry->message = message; ++ __entry->atom_id = atom_id; ++ ), ++ TP_printk("msg=%u proc_id=%u, param1=%d\n", __entry->message, ++ __entry->proc_id, __entry->atom_id) ++); ++ ++/* A template for SYSGRAPH GPU events ++ * ++ * Sysgraph events that record start/complete events ++ * on GPU also record a js value in addition to the ++ * atom id. ++ */ ++TRACE_EVENT(sysgraph_gpu, ++ TP_PROTO(enum sysgraph_msg message, unsigned int proc_id, ++ unsigned int atom_id, unsigned int js), ++ TP_ARGS(message, proc_id, atom_id, js), ++ TP_STRUCT__entry( ++ __field(unsigned int, proc_id) ++ __field(enum sysgraph_msg, message) ++ __field(unsigned int, atom_id) ++ __field(unsigned int, js) ++ ), ++ TP_fast_assign( ++ __entry->proc_id = proc_id; ++ __entry->message = message; ++ __entry->atom_id = atom_id; ++ __entry->js = js; ++ ), ++ TP_printk("msg=%u proc_id=%u, param1=%d, param2=%d\n", ++ __entry->message, __entry->proc_id, ++ __entry->atom_id, __entry->js) ++); ++ ++/* Tracepoint files get included more than once - protect against multiple ++ * definition ++ */ ++#undef KBASE_JIT_REPORT_GPU_MEM_SIZE ++ ++/* Size in bytes of the memory surrounding the location used for a just-in-time ++ * memory allocation report ++ */ ++#define KBASE_JIT_REPORT_GPU_MEM_SIZE (4 * sizeof(u64)) ++ ++/* trace_mali_jit_report_gpu_mem ++ * ++ * Tracepoint about the GPU memory nearby the location used for a just-in-time ++ * memory allocation report ++ */ ++TRACE_EVENT(mali_jit_report_gpu_mem, ++ TP_PROTO(u64 base_addr, u64 reg_addr, u64 *gpu_mem, unsigned int flags), ++ TP_ARGS(base_addr, reg_addr, gpu_mem, flags), ++ TP_STRUCT__entry( ++ __field(u64, base_addr) ++ __field(u64, reg_addr) ++ __array(u64, mem_values, ++ KBASE_JIT_REPORT_GPU_MEM_SIZE / sizeof(u64)) ++ __field(unsigned int, flags) ++ ), ++ TP_fast_assign( ++ __entry->base_addr = base_addr; ++ __entry->reg_addr = reg_addr; ++ memcpy(__entry->mem_values, gpu_mem, ++ sizeof(__entry->mem_values)); ++ __entry->flags = flags; ++ ), ++ TP_printk("start=0x%llx read GPU memory base=0x%llx values=%s report_flags=%s", ++ __entry->reg_addr, __entry->base_addr, ++ __print_array(__entry->mem_values, ++ ARRAY_SIZE(__entry->mem_values), sizeof(u64)), ++ __print_flags(__entry->flags, "|", ++ { KBASE_JIT_REPORT_ON_ALLOC_OR_FREE, ++ "HAPPENED_ON_ALLOC_OR_FREE" })) ++); ++ ++/* trace_mali_jit_trim_from_region ++ * ++ * Tracepoint about trimming physical pages from a region ++ */ ++TRACE_EVENT(mali_jit_trim_from_region, ++ TP_PROTO(struct kbase_va_region *reg, size_t freed_pages, ++ size_t old_pages, size_t available_pages, size_t new_pages), ++ TP_ARGS(reg, freed_pages, old_pages, available_pages, new_pages), ++ TP_STRUCT__entry( ++ __field(u64, start_addr) ++ __field(size_t, freed_pages) ++ __field(size_t, old_pages) ++ __field(size_t, available_pages) ++ __field(size_t, new_pages) ++ ), ++ TP_fast_assign( ++ __entry->start_addr = ((u64)reg->start_pfn) << PAGE_SHIFT; ++ __entry->freed_pages = freed_pages; ++ __entry->old_pages = old_pages; ++ __entry->available_pages = available_pages; ++ __entry->new_pages = new_pages; ++ ), ++ TP_printk("start=0x%llx freed_pages=%zu old_pages=%zu available_pages=%zu new_pages=%zu", ++ __entry->start_addr, __entry->freed_pages, __entry->old_pages, ++ __entry->available_pages, __entry->new_pages) ++); ++ ++/* trace_mali_jit_trim ++ * ++ * Tracepoint about total trimmed physical pages ++ */ ++TRACE_EVENT(mali_jit_trim, ++ TP_PROTO(size_t freed_pages), ++ TP_ARGS(freed_pages), ++ TP_STRUCT__entry( ++ __field(size_t, freed_pages) ++ ), ++ TP_fast_assign( ++ __entry->freed_pages = freed_pages; ++ ), ++ TP_printk("freed_pages=%zu", __entry->freed_pages) ++); ++ ++#include "mali_kbase_debug_linux_ktrace.h" ++ ++#endif /* _TRACE_MALI_H */ ++ ++#undef TRACE_INCLUDE_PATH ++/* lwn.net/Articles/383362 suggests this should remain as '.', and instead ++ * extend CFLAGS ++ */ ++#define TRACE_INCLUDE_PATH . ++#undef TRACE_INCLUDE_FILE ++#define TRACE_INCLUDE_FILE mali_linux_trace ++ ++/* This part must be outside protection */ ++#include +diff --git a/drivers/gpu/arm/bifrost/mali_malisw.h b/drivers/gpu/arm/bifrost/mali_malisw.h +new file mode 100755 +index 000000000000..3a4db10bdb3d +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_malisw.h +@@ -0,0 +1,109 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2015, 2018 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/** ++ * Kernel-wide include for common macros and types. ++ */ ++ ++#ifndef _MALISW_H_ ++#define _MALISW_H_ ++ ++#include ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 0) ++#define U8_MAX ((u8)~0U) ++#define S8_MAX ((s8)(U8_MAX>>1)) ++#define S8_MIN ((s8)(-S8_MAX - 1)) ++#define U16_MAX ((u16)~0U) ++#define S16_MAX ((s16)(U16_MAX>>1)) ++#define S16_MIN ((s16)(-S16_MAX - 1)) ++#define U32_MAX ((u32)~0U) ++#define S32_MAX ((s32)(U32_MAX>>1)) ++#define S32_MIN ((s32)(-S32_MAX - 1)) ++#define U64_MAX ((u64)~0ULL) ++#define S64_MAX ((s64)(U64_MAX>>1)) ++#define S64_MIN ((s64)(-S64_MAX - 1)) ++#endif /* LINUX_VERSION_CODE */ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 5, 0) ++#define SIZE_MAX (~(size_t)0) ++#endif /* LINUX_VERSION_CODE */ ++ ++/** ++ * MIN - Return the lesser of two values. ++ * ++ * As a macro it may evaluate its arguments more than once. ++ * Refer to MAX macro for more details ++ */ ++#define MIN(x, y) ((x) < (y) ? (x) : (y)) ++ ++/** ++ * MAX - Return the greater of two values. ++ * ++ * As a macro it may evaluate its arguments more than once. ++ * If called on the same two arguments as MIN it is guaranteed to return ++ * the one that MIN didn't return. This is significant for types where not ++ * all values are comparable e.g. NaNs in floating-point types. But if you want ++ * to retrieve the min and max of two values, consider using a conditional swap ++ * instead. ++ */ ++#define MAX(x, y) ((x) < (y) ? (y) : (x)) ++ ++/** ++ * @hideinitializer ++ * Function-like macro for suppressing unused variable warnings. Where possible ++ * such variables should be removed; this macro is present for cases where we ++ * much support API backwards compatibility. ++ */ ++#define CSTD_UNUSED(x) ((void)(x)) ++ ++/** ++ * @hideinitializer ++ * Function-like macro for use where "no behavior" is desired. This is useful ++ * when compile time macros turn a function-like macro in to a no-op, but ++ * where having no statement is otherwise invalid. ++ */ ++#define CSTD_NOP(...) ((void)#__VA_ARGS__) ++ ++/** ++ * @hideinitializer ++ * Function-like macro for stringizing a single level macro. ++ * @code ++ * #define MY_MACRO 32 ++ * CSTD_STR1( MY_MACRO ) ++ * > "MY_MACRO" ++ * @endcode ++ */ ++#define CSTD_STR1(x) #x ++ ++/** ++ * @hideinitializer ++ * Function-like macro for stringizing a macro's value. This should not be used ++ * if the macro is defined in a way which may have no value; use the ++ * alternative @c CSTD_STR2N macro should be used instead. ++ * @code ++ * #define MY_MACRO 32 ++ * CSTD_STR2( MY_MACRO ) ++ * > "32" ++ * @endcode ++ */ ++#define CSTD_STR2(x) CSTD_STR1(x) ++ ++#endif /* _MALISW_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mali_power_gpu_frequency_trace.c b/drivers/gpu/arm/bifrost/mali_power_gpu_frequency_trace.c +new file mode 100755 +index 000000000000..b6fb5a094fab +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_power_gpu_frequency_trace.c +@@ -0,0 +1,27 @@ ++/* ++ * ++ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* Create the trace point if not configured in kernel */ ++#ifndef CONFIG_TRACE_POWER_GPU_FREQUENCY ++#define CREATE_TRACE_POINTS ++#include "mali_power_gpu_frequency_trace.h" ++#endif +diff --git a/drivers/gpu/arm/bifrost/mali_power_gpu_frequency_trace.h b/drivers/gpu/arm/bifrost/mali_power_gpu_frequency_trace.h +new file mode 100755 +index 000000000000..3b90ae437db9 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_power_gpu_frequency_trace.h +@@ -0,0 +1,69 @@ ++/* ++ * ++ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _TRACE_POWER_GPU_FREQUENCY_MALI ++#define _TRACE_POWER_GPU_FREQUENCY_MALI ++#endif ++ ++#undef TRACE_SYSTEM ++#define TRACE_SYSTEM power ++#undef TRACE_INCLUDE_FILE ++#define TRACE_INCLUDE_FILE mali_power_gpu_frequency_trace ++#undef TRACE_INCLUDE_PATH ++#define TRACE_INCLUDE_PATH . ++ ++#if !defined(_TRACE_POWER_GPU_FREQUENCY_H) || defined(TRACE_HEADER_MULTI_READ) ++#define _TRACE_POWER_GPU_FREQUENCY_H ++ ++#include ++ ++DECLARE_EVENT_CLASS(gpu, ++ ++ TP_PROTO(unsigned int state, unsigned int gpu_id), ++ ++ TP_ARGS(state, gpu_id), ++ ++ TP_STRUCT__entry( ++ __field( u32, state ) ++ __field( u32, gpu_id ) ++ ), ++ ++ TP_fast_assign( ++ __entry->state = state; ++ __entry->gpu_id = gpu_id; ++ ), ++ ++ TP_printk("state=%lu gpu_id=%lu", (unsigned long)__entry->state, ++ (unsigned long)__entry->gpu_id) ++); ++ ++DEFINE_EVENT(gpu, gpu_frequency, ++ ++ TP_PROTO(unsigned int frequency, unsigned int gpu_id), ++ ++ TP_ARGS(frequency, gpu_id) ++); ++ ++#endif /* _TRACE_POWER_GPU_FREQUENCY_H */ ++ ++/* This part must be outside protection */ ++#include +diff --git a/drivers/gpu/arm/bifrost/mali_uk.h b/drivers/gpu/arm/bifrost/mali_uk.h +new file mode 100755 +index 000000000000..701f3909042f +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mali_uk.h +@@ -0,0 +1,84 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010, 2012-2015, 2018 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++/** ++ * @file mali_uk.h ++ * Types and definitions that are common across OSs for both the user ++ * and kernel side of the User-Kernel interface. ++ */ ++ ++#ifndef _UK_H_ ++#define _UK_H_ ++ ++#ifdef __cplusplus ++extern "C" { ++#endif /* __cplusplus */ ++ ++/** ++ * @addtogroup base_api ++ * @{ ++ */ ++ ++/** ++ * @defgroup uk_api User-Kernel Interface API ++ * ++ * The User-Kernel Interface abstracts the communication mechanism between the user and kernel-side code of device ++ * drivers developed as part of the Midgard DDK. Currently that includes the Base driver. ++ * ++ * It exposes an OS independent API to user-side code (UKU) which routes functions calls to an OS-independent ++ * kernel-side API (UKK) via an OS-specific communication mechanism. ++ * ++ * This API is internal to the Midgard DDK and is not exposed to any applications. ++ * ++ * @{ ++ */ ++ ++/** ++ * These are identifiers for kernel-side drivers implementing a UK interface, aka UKK clients. The ++ * UK module maps this to an OS specific device name, e.g. "gpu_base" -> "GPU0:". Specify this ++ * identifier to select a UKK client to the uku_open() function. ++ * ++ * When a new UKK client driver is created a new identifier needs to be added to the uk_client_id ++ * enumeration and the uku_open() implemenation for the various OS ports need to be updated to ++ * provide a mapping of the identifier to the OS specific device name. ++ * ++ */ ++enum uk_client_id { ++ /** ++ * Value used to identify the Base driver UK client. ++ */ ++ UK_CLIENT_MALI_T600_BASE, ++ ++ /** The number of uk clients supported. This must be the last member of the enum */ ++ UK_CLIENT_COUNT ++}; ++ ++/** @} end group uk_api */ ++ ++/** @} *//* end group base_api */ ++ ++#ifdef __cplusplus ++} ++#endif /* __cplusplus */ ++#endif /* _UK_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_csf.c b/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_csf.c +new file mode 100755 +index 000000000000..1d106999228a +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_csf.c +@@ -0,0 +1,532 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/** ++ * Base kernel MMU management specific for CSF GPU. ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include "../mali_kbase_mmu_internal.h" ++ ++void kbase_mmu_get_as_setup(struct kbase_mmu_table *mmut, ++ struct kbase_mmu_setup * const setup) ++{ ++ /* Set up the required caching policies at the correct indices ++ * in the memattr register. ++ */ ++ setup->memattr = ++ (AS_MEMATTR_IMPL_DEF_CACHE_POLICY << ++ (AS_MEMATTR_INDEX_IMPL_DEF_CACHE_POLICY * 8)) | ++ (AS_MEMATTR_FORCE_TO_CACHE_ALL << ++ (AS_MEMATTR_INDEX_FORCE_TO_CACHE_ALL * 8)) | ++ (AS_MEMATTR_WRITE_ALLOC << ++ (AS_MEMATTR_INDEX_WRITE_ALLOC * 8)) | ++ (AS_MEMATTR_AARCH64_OUTER_IMPL_DEF << ++ (AS_MEMATTR_INDEX_OUTER_IMPL_DEF * 8)) | ++ (AS_MEMATTR_AARCH64_OUTER_WA << ++ (AS_MEMATTR_INDEX_OUTER_WA * 8)) | ++ (AS_MEMATTR_AARCH64_NON_CACHEABLE << ++ (AS_MEMATTR_INDEX_NON_CACHEABLE * 8)) | ++ (AS_MEMATTR_AARCH64_SHARED << ++ (AS_MEMATTR_INDEX_SHARED * 8)); ++ ++ setup->transtab = (u64)mmut->pgd & AS_TRANSTAB_BASE_MASK; ++ setup->transcfg = AS_TRANSCFG_ADRMODE_AARCH64_4K; ++} ++ ++/** ++ * submit_work_pagefault() - Submit a work for MMU page fault. ++ * ++ * @kbdev: Kbase device pointer ++ * @as_nr: Faulty address space ++ * @fault: Data relating to the fault ++ * ++ * This function submits a work for reporting the details of MMU fault. ++ */ ++static void submit_work_pagefault(struct kbase_device *kbdev, u32 as_nr, ++ struct kbase_fault *fault) ++{ ++ struct kbase_as *const as = &kbdev->as[as_nr]; ++ ++ as->pf_data = (struct kbase_fault) { ++ .status = fault->status, ++ .addr = fault->addr, ++ }; ++ ++ if (kbase_ctx_sched_as_to_ctx_refcount(kbdev, as_nr)) { ++ WARN_ON(!queue_work(as->pf_wq, &as->work_pagefault)); ++ atomic_inc(&kbdev->faults_pending); ++ } ++} ++ ++void kbase_mmu_report_mcu_as_fault_and_reset(struct kbase_device *kbdev, ++ struct kbase_fault *fault) ++{ ++ /* decode the fault status */ ++ u32 exception_type = fault->status & 0xFF; ++ u32 access_type = (fault->status >> 8) & 0x3; ++ u32 source_id = (fault->status >> 16); ++ int as_no; ++ ++ /* terminal fault, print info about the fault */ ++ dev_err(kbdev->dev, ++ "Unexpected Page fault in firmware address space at VA 0x%016llX\n" ++ "raw fault status: 0x%X\n" ++ "exception type 0x%X: %s\n" ++ "access type 0x%X: %s\n" ++ "source id 0x%X\n", ++ fault->addr, ++ fault->status, ++ exception_type, kbase_gpu_exception_name(exception_type), ++ access_type, kbase_gpu_access_type_name(fault->status), ++ source_id); ++ ++ /* Report MMU fault for all address spaces (except MCU_AS_NR) */ ++ for (as_no = 1; as_no < kbdev->nr_hw_address_spaces; as_no++) ++ if (kbase_ctx_sched_as_to_ctx(kbdev, as_no)) ++ submit_work_pagefault(kbdev, as_no, fault); ++ ++ /* GPU reset is required to recover */ ++ if (kbase_prepare_to_reset_gpu(kbdev)) ++ kbase_reset_gpu(kbdev); ++} ++KBASE_EXPORT_TEST_API(kbase_mmu_report_mcu_as_fault_and_reset); ++ ++void kbase_gpu_report_bus_fault_and_kill(struct kbase_context *kctx, ++ struct kbase_as *as, struct kbase_fault *fault) ++{ ++ struct kbase_device *kbdev = kctx->kbdev; ++ u32 const status = fault->status; ++ int exception_type = (status & GPU_FAULTSTATUS_EXCEPTION_TYPE_MASK) >> ++ GPU_FAULTSTATUS_EXCEPTION_TYPE_SHIFT; ++ int access_type = (status & GPU_FAULTSTATUS_ACCESS_TYPE_MASK) >> ++ GPU_FAULTSTATUS_ACCESS_TYPE_SHIFT; ++ int source_id = (status & GPU_FAULTSTATUS_SOURCE_ID_MASK) >> ++ GPU_FAULTSTATUS_SOURCE_ID_SHIFT; ++ const char *addr_valid = (status & GPU_FAULTSTATUS_ADDR_VALID_FLAG) ? ++ "true" : "false"; ++ int as_no = as->number; ++ unsigned long flags; ++ ++ /* terminal fault, print info about the fault */ ++ dev_err(kbdev->dev, ++ "GPU bus fault in AS%d at VA 0x%016llX\n" ++ "VA_VALID: %s\n" ++ "raw fault status: 0x%X\n" ++ "exception type 0x%X: %s\n" ++ "access type 0x%X: %s\n" ++ "source id 0x%X\n" ++ "pid: %d\n", ++ as_no, fault->addr, ++ addr_valid, ++ status, ++ exception_type, kbase_gpu_exception_name(exception_type), ++ access_type, kbase_gpu_access_type_name(access_type), ++ source_id, ++ kctx->pid); ++ ++ /* AS transaction begin */ ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_mmu_disable(kctx); ++ kbase_ctx_flag_set(kctx, KCTX_AS_DISABLED_ON_FAULT); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ ++ /* Switching to UNMAPPED mode above would have enabled the firmware to ++ * recover from the fault (if the memory access was made by firmware) ++ * and it can then respond to CSG termination requests to be sent now. ++ * All GPU command queue groups associated with the context would be ++ * affected as they use the same GPU address space. ++ */ ++ kbase_csf_ctx_handle_fault(kctx, fault); ++ ++ /* Now clear the GPU fault */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), ++ GPU_COMMAND_CLEAR_FAULT); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++} ++ ++/** ++ * The caller must ensure it's retained the ctx to prevent it from being ++ * scheduled out whilst it's being worked on. ++ */ ++void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, ++ struct kbase_as *as, const char *reason_str, ++ struct kbase_fault *fault) ++{ ++ unsigned long flags; ++ unsigned int exception_type; ++ unsigned int access_type; ++ unsigned int source_id; ++ int as_no; ++ struct kbase_device *kbdev; ++ const u32 status = fault->status; ++ ++ as_no = as->number; ++ kbdev = kctx->kbdev; ++ ++ /* Make sure the context was active */ ++ if (WARN_ON(atomic_read(&kctx->refcount) <= 0)) ++ return; ++ ++ /* decode the fault status */ ++ exception_type = AS_FAULTSTATUS_EXCEPTION_TYPE_GET(status); ++ access_type = AS_FAULTSTATUS_ACCESS_TYPE_GET(status); ++ source_id = AS_FAULTSTATUS_SOURCE_ID_GET(status); ++ ++ /* terminal fault, print info about the fault */ ++ dev_err(kbdev->dev, ++ "Unhandled Page fault in AS%d at VA 0x%016llX\n" ++ "Reason: %s\n" ++ "raw fault status: 0x%X\n" ++ "exception type 0x%X: %s\n" ++ "access type 0x%X: %s\n" ++ "source id 0x%X\n" ++ "pid: %d\n", ++ as_no, fault->addr, ++ reason_str, ++ status, ++ exception_type, kbase_gpu_exception_name(exception_type), ++ access_type, kbase_gpu_access_type_name(status), ++ source_id, ++ kctx->pid); ++ ++ /* AS transaction begin */ ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ ++ /* switch to UNMAPPED mode, ++ * will abort all jobs and stop any hw counter dumping ++ */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_mmu_disable(kctx); ++ kbase_ctx_flag_set(kctx, KCTX_AS_DISABLED_ON_FAULT); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ /* AS transaction end */ ++ ++ /* Switching to UNMAPPED mode above would have enabled the firmware to ++ * recover from the fault (if the memory access was made by firmware) ++ * and it can then respond to CSG termination requests to be sent now. ++ * All GPU command queue groups associated with the context would be ++ * affected as they use the same GPU address space. ++ */ ++ kbase_csf_ctx_handle_fault(kctx, fault); ++ ++ /* Clear down the fault */ ++ kbase_mmu_hw_clear_fault(kbdev, as, ++ KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); ++ kbase_mmu_hw_enable_fault(kbdev, as, ++ KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); ++} ++ ++/** ++ * kbase_mmu_interrupt_process() - Process a bus or page fault. ++ * @kbdev: The kbase_device the fault happened on ++ * @kctx: The kbase_context for the faulting address space if one was ++ * found. ++ * @as: The address space that has the fault ++ * @fault: Data relating to the fault ++ * ++ * This function will process a fault on a specific address space ++ */ ++static void kbase_mmu_interrupt_process(struct kbase_device *kbdev, ++ struct kbase_context *kctx, struct kbase_as *as, ++ struct kbase_fault *fault) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (!kctx) { ++ dev_warn(kbdev->dev, "%s in AS%d at 0x%016llx with no context present! Spurious IRQ or SW Design Error?\n", ++ kbase_as_has_bus_fault(as, fault) ? ++ "Bus error" : "Page fault", ++ as->number, fault->addr); ++ ++ /* Since no ctx was found, the MMU must be disabled. */ ++ WARN_ON(as->current_setup.transtab); ++ ++ if (kbase_as_has_bus_fault(as, fault)) ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), ++ GPU_COMMAND_CLEAR_FAULT); ++ else if (kbase_as_has_page_fault(as, fault)) { ++ kbase_mmu_hw_clear_fault(kbdev, as, ++ KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); ++ kbase_mmu_hw_enable_fault(kbdev, as, ++ KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); ++ } ++ ++ return; ++ } ++ ++ if (kbase_as_has_bus_fault(as, fault)) { ++ /* ++ * We need to switch to UNMAPPED mode - but we do this in a ++ * worker so that we can sleep ++ */ ++ WARN_ON(!queue_work(as->pf_wq, &as->work_busfault)); ++ atomic_inc(&kbdev->faults_pending); ++ } else { ++ WARN_ON(!queue_work(as->pf_wq, &as->work_pagefault)); ++ atomic_inc(&kbdev->faults_pending); ++ } ++} ++ ++int kbase_mmu_bus_fault_interrupt(struct kbase_device *kbdev, ++ u32 status, u32 as_nr) ++{ ++ struct kbase_context *kctx; ++ unsigned long flags; ++ struct kbase_as *as; ++ struct kbase_fault *fault; ++ ++ if (WARN_ON(as_nr == MCU_AS_NR)) ++ return -EINVAL; ++ ++ if (WARN_ON(as_nr >= BASE_MAX_NR_AS)) ++ return -EINVAL; ++ ++ as = &kbdev->as[as_nr]; ++ fault = &as->bf_data; ++ fault->status = status; ++ fault->addr = (u64) kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(GPU_FAULTADDRESS_HI)) << 32; ++ fault->addr |= kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(GPU_FAULTADDRESS_LO)); ++ fault->protected_mode = false; ++ ++ /* report the fault to debugfs */ ++ kbase_as_fault_debugfs_new(kbdev, as_nr); ++ ++ kctx = kbase_ctx_sched_as_to_ctx_refcount(kbdev, as_nr); ++ ++ /* Process the bus fault interrupt for this address space */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_mmu_interrupt_process(kbdev, kctx, as, fault); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ return 0; ++} ++ ++void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat) ++{ ++ const int num_as = 16; ++ const int pf_shift = 0; ++ const unsigned long as_bit_mask = (1UL << num_as) - 1; ++ unsigned long flags; ++ u32 new_mask; ++ u32 tmp; ++ u32 pf_bits = ((irq_stat >> pf_shift) & as_bit_mask); ++ ++ /* remember current mask */ ++ spin_lock_irqsave(&kbdev->mmu_mask_change, flags); ++ new_mask = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK)); ++ /* mask interrupts for now */ ++ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0); ++ spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); ++ ++ while (pf_bits) { ++ struct kbase_context *kctx; ++ int as_no = ffs(pf_bits) - 1; ++ struct kbase_as *as = &kbdev->as[as_no]; ++ struct kbase_fault *fault = &as->pf_data; ++ ++ /* find faulting address */ ++ fault->addr = kbase_reg_read(kbdev, MMU_AS_REG(as_no, ++ AS_FAULTADDRESS_HI)); ++ fault->addr <<= 32; ++ fault->addr |= kbase_reg_read(kbdev, MMU_AS_REG(as_no, ++ AS_FAULTADDRESS_LO)); ++ ++ /* Mark the fault protected or not */ ++ fault->protected_mode = false; ++ ++ /* report the fault to debugfs */ ++ kbase_as_fault_debugfs_new(kbdev, as_no); ++ ++ /* record the fault status */ ++ fault->status = kbase_reg_read(kbdev, MMU_AS_REG(as_no, ++ AS_FAULTSTATUS)); ++ ++ fault->extra_addr = kbase_reg_read(kbdev, ++ MMU_AS_REG(as_no, AS_FAULTEXTRA_HI)); ++ fault->extra_addr <<= 32; ++ fault->extra_addr |= kbase_reg_read(kbdev, ++ MMU_AS_REG(as_no, AS_FAULTEXTRA_LO)); ++ ++ /* Mark page fault as handled */ ++ pf_bits &= ~(1UL << as_no); ++ ++ /* remove the queued PF from the mask */ ++ new_mask &= ~MMU_PAGE_FAULT(as_no); ++ ++ if (as_no == MCU_AS_NR) { ++ kbase_mmu_report_mcu_as_fault_and_reset(kbdev, fault); ++ /* Pointless to handle remaining faults */ ++ break; ++ } ++ ++ /* ++ * Refcount the kctx - it shouldn't disappear anyway, since ++ * Page faults _should_ only occur whilst GPU commands are ++ * executing, and a command causing the Page fault shouldn't ++ * complete until the MMU is updated. ++ * Reference is released at the end of bottom half of page ++ * fault handling. ++ */ ++ kctx = kbase_ctx_sched_as_to_ctx_refcount(kbdev, as_no); ++ ++ /* Process the interrupt for this address space */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_mmu_interrupt_process(kbdev, kctx, as, fault); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ } ++ ++ /* reenable interrupts */ ++ spin_lock_irqsave(&kbdev->mmu_mask_change, flags); ++ tmp = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK)); ++ new_mask |= tmp; ++ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), new_mask); ++ spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); ++} ++ ++int kbase_mmu_switch_to_ir(struct kbase_context *const kctx, ++ struct kbase_va_region *const reg) ++{ ++ /* Can't soft-stop the provoking job */ ++ return -EPERM; ++} ++ ++/** ++ * kbase_mmu_gpu_fault_worker() - Process a GPU fault for the device. ++ * ++ * @data: work_struct passed by queue_work() ++ * ++ * Report a GPU fatal error for all GPU command queue groups that are ++ * using the address space and terminate them. ++ */ ++static void kbase_mmu_gpu_fault_worker(struct work_struct *data) ++{ ++ struct kbase_as *const faulting_as = container_of(data, struct kbase_as, ++ work_gpufault); ++ const u32 as_nr = faulting_as->number; ++ struct kbase_device *const kbdev = container_of(faulting_as, struct ++ kbase_device, as[as_nr]); ++ struct kbase_fault *fault; ++ struct kbase_context *kctx; ++ u32 status; ++ u64 address; ++ u32 as_valid; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ fault = &faulting_as->gf_data; ++ status = fault->status; ++ as_valid = status & GPU_FAULTSTATUS_JASID_VALID_FLAG; ++ address = fault->addr; ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ dev_warn(kbdev->dev, ++ "GPU Fault 0x%08x (%s) in AS%u at 0x%016llx\n" ++ "ASID_VALID: %s, ADDRESS_VALID: %s\n", ++ status, ++ kbase_gpu_exception_name( ++ GPU_FAULTSTATUS_EXCEPTION_TYPE_GET(status)), ++ as_nr, address, ++ as_valid ? "true" : "false", ++ status & GPU_FAULTSTATUS_ADDR_VALID_FLAG ? "true" : "false"); ++ ++ kctx = kbase_ctx_sched_as_to_ctx(kbdev, as_nr); ++ kbase_csf_ctx_handle_fault(kctx, fault); ++ kbase_ctx_sched_release_ctx_lock(kctx); ++ ++ atomic_dec(&kbdev->faults_pending); ++} ++ ++/** ++ * submit_work_gpufault() - Submit a work for GPU fault. ++ * ++ * @kbdev: Kbase device pointer ++ * @status: GPU fault status ++ * @as_nr: Faulty address space ++ * @address: GPU fault address ++ * ++ * This function submits a work for reporting the details of GPU fault. ++ */ ++static void submit_work_gpufault(struct kbase_device *kbdev, u32 status, ++ u32 as_nr, u64 address) ++{ ++ unsigned long flags; ++ struct kbase_as *const as = &kbdev->as[as_nr]; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ as->gf_data = (struct kbase_fault) { ++ .status = status, ++ .addr = address, ++ }; ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ if (kbase_ctx_sched_as_to_ctx_refcount(kbdev, as_nr)) { ++ WARN_ON(!queue_work(as->pf_wq, &as->work_gpufault)); ++ atomic_inc(&kbdev->faults_pending); ++ } ++} ++ ++void kbase_mmu_gpu_fault_interrupt(struct kbase_device *kbdev, u32 status, ++ u32 as_nr, u64 address, bool as_valid) ++{ ++ if (!as_valid || (as_nr == MCU_AS_NR)) { ++ int as; ++ ++ /* Report GPU fault for all contexts (except MCU_AS_NR) in case either ++ * the address space is invalid or it's MCU address space. ++ */ ++ for (as = 1; as < kbdev->nr_hw_address_spaces; as++) ++ submit_work_gpufault(kbdev, status, as, address); ++ } else ++ submit_work_gpufault(kbdev, status, as_nr, address); ++} ++KBASE_EXPORT_TEST_API(kbase_mmu_gpu_fault_interrupt); ++ ++int kbase_mmu_as_init(struct kbase_device *kbdev, int i) ++{ ++ kbdev->as[i].number = i; ++ kbdev->as[i].bf_data.addr = 0ULL; ++ kbdev->as[i].pf_data.addr = 0ULL; ++ kbdev->as[i].gf_data.addr = 0ULL; ++ ++ kbdev->as[i].pf_wq = alloc_workqueue("mali_mmu%d", 0, 1, i); ++ if (!kbdev->as[i].pf_wq) ++ return -ENOMEM; ++ ++ INIT_WORK(&kbdev->as[i].work_pagefault, kbase_mmu_page_fault_worker); ++ INIT_WORK(&kbdev->as[i].work_busfault, kbase_mmu_bus_fault_worker); ++ INIT_WORK(&kbdev->as[i].work_gpufault, kbase_mmu_gpu_fault_worker); ++ ++ return 0; ++} +diff --git a/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_jm.c b/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_jm.c +new file mode 100755 +index 000000000000..b0187a46b733 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_jm.c +@@ -0,0 +1,440 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/** ++ * Base kernel MMU management specific for Job Manager GPU. ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include "../mali_kbase_mmu_internal.h" ++ ++void kbase_mmu_get_as_setup(struct kbase_mmu_table *mmut, ++ struct kbase_mmu_setup * const setup) ++{ ++ /* Set up the required caching policies at the correct indices ++ * in the memattr register. ++ */ ++ setup->memattr = ++ (AS_MEMATTR_IMPL_DEF_CACHE_POLICY << ++ (AS_MEMATTR_INDEX_IMPL_DEF_CACHE_POLICY * 8)) | ++ (AS_MEMATTR_FORCE_TO_CACHE_ALL << ++ (AS_MEMATTR_INDEX_FORCE_TO_CACHE_ALL * 8)) | ++ (AS_MEMATTR_WRITE_ALLOC << ++ (AS_MEMATTR_INDEX_WRITE_ALLOC * 8)) | ++ (AS_MEMATTR_AARCH64_OUTER_IMPL_DEF << ++ (AS_MEMATTR_INDEX_OUTER_IMPL_DEF * 8)) | ++ (AS_MEMATTR_AARCH64_OUTER_WA << ++ (AS_MEMATTR_INDEX_OUTER_WA * 8)) | ++ (AS_MEMATTR_AARCH64_NON_CACHEABLE << ++ (AS_MEMATTR_INDEX_NON_CACHEABLE * 8)); ++ ++ setup->transtab = (u64)mmut->pgd & AS_TRANSTAB_BASE_MASK; ++ setup->transcfg = AS_TRANSCFG_ADRMODE_AARCH64_4K; ++} ++ ++void kbase_gpu_report_bus_fault_and_kill(struct kbase_context *kctx, ++ struct kbase_as *as, struct kbase_fault *fault) ++{ ++ struct kbase_device *const kbdev = kctx->kbdev; ++ u32 const status = fault->status; ++ u32 const exception_type = (status & 0xFF); ++ u32 const exception_data = (status >> 8) & 0xFFFFFF; ++ int const as_no = as->number; ++ unsigned long flags; ++ ++ /* terminal fault, print info about the fault */ ++ dev_err(kbdev->dev, ++ "GPU bus fault in AS%d at VA 0x%016llX\n" ++ "raw fault status: 0x%X\n" ++ "exception type 0x%X: %s\n" ++ "exception data 0x%X\n" ++ "pid: %d\n", ++ as_no, fault->addr, ++ status, ++ exception_type, kbase_gpu_exception_name(exception_type), ++ exception_data, ++ kctx->pid); ++ ++ /* switch to UNMAPPED mode, will abort all jobs and stop any hw counter ++ * dumping AS transaction begin ++ */ ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ ++ /* Set the MMU into unmapped mode */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_mmu_disable(kctx); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ /* AS transaction end */ ++ ++ kbase_mmu_hw_clear_fault(kbdev, as, ++ KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); ++ kbase_mmu_hw_enable_fault(kbdev, as, ++ KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); ++} ++ ++/** ++ * The caller must ensure it's retained the ctx to prevent it from being ++ * scheduled out whilst it's being worked on. ++ */ ++void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, ++ struct kbase_as *as, const char *reason_str, ++ struct kbase_fault *fault) ++{ ++ unsigned long flags; ++ u32 exception_type; ++ u32 access_type; ++ u32 source_id; ++ int as_no; ++ struct kbase_device *kbdev; ++ struct kbasep_js_device_data *js_devdata; ++ ++ as_no = as->number; ++ kbdev = kctx->kbdev; ++ js_devdata = &kbdev->js_data; ++ ++ /* Make sure the context was active */ ++ if (WARN_ON(atomic_read(&kctx->refcount) <= 0)) ++ return; ++ ++ /* decode the fault status */ ++ exception_type = fault->status & 0xFF; ++ access_type = (fault->status >> 8) & 0x3; ++ source_id = (fault->status >> 16); ++ ++ /* terminal fault, print info about the fault */ ++ dev_err(kbdev->dev, ++ "Unhandled Page fault in AS%d at VA 0x%016llX\n" ++ "Reason: %s\n" ++ "raw fault status: 0x%X\n" ++ "exception type 0x%X: %s\n" ++ "access type 0x%X: %s\n" ++ "source id 0x%X\n" ++ "pid: %d\n", ++ as_no, fault->addr, ++ reason_str, ++ fault->status, ++ exception_type, kbase_gpu_exception_name(exception_type), ++ access_type, kbase_gpu_access_type_name(fault->status), ++ source_id, ++ kctx->pid); ++ ++ /* hardware counters dump fault handling */ ++ if ((kbdev->hwcnt.kctx) && (kbdev->hwcnt.kctx->as_nr == as_no) && ++ (kbdev->hwcnt.backend.state == ++ KBASE_INSTR_STATE_DUMPING)) { ++ if ((fault->addr >= kbdev->hwcnt.addr) && ++ (fault->addr < (kbdev->hwcnt.addr + ++ kbdev->hwcnt.addr_bytes))) ++ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_FAULT; ++ } ++ ++ /* Stop the kctx from submitting more jobs and cause it to be scheduled ++ * out/rescheduled - this will occur on releasing the context's refcount ++ */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbasep_js_clear_submit_allowed(js_devdata, kctx); ++ ++ /* Kill any running jobs from the context. Submit is disallowed, so no ++ * more jobs from this context can appear in the job slots from this ++ * point on ++ */ ++ kbase_backend_jm_kill_running_jobs_from_kctx(kctx); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ /* AS transaction begin */ ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ ++ /* switch to UNMAPPED mode, will abort all jobs and stop ++ * any hw counter dumping ++ */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_mmu_disable(kctx); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ ++ /* AS transaction end */ ++ /* Clear down the fault */ ++ kbase_mmu_hw_clear_fault(kbdev, as, ++ KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); ++ kbase_mmu_hw_enable_fault(kbdev, as, ++ KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); ++} ++ ++/** ++ * kbase_mmu_interrupt_process() - Process a bus or page fault. ++ * @kbdev: The kbase_device the fault happened on ++ * @kctx: The kbase_context for the faulting address space if one was ++ * found. ++ * @as: The address space that has the fault ++ * @fault: Data relating to the fault ++ * ++ * This function will process a fault on a specific address space ++ */ ++static void kbase_mmu_interrupt_process(struct kbase_device *kbdev, ++ struct kbase_context *kctx, struct kbase_as *as, ++ struct kbase_fault *fault) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ dev_dbg(kbdev->dev, ++ "Entering %s kctx %p, as %p\n", ++ __func__, (void *)kctx, (void *)as); ++ ++ if (!kctx) { ++ dev_warn(kbdev->dev, "%s in AS%d at 0x%016llx with no context present! Spurious IRQ or SW Design Error?\n", ++ kbase_as_has_bus_fault(as, fault) ? ++ "Bus error" : "Page fault", ++ as->number, fault->addr); ++ ++ /* Since no ctx was found, the MMU must be disabled. */ ++ WARN_ON(as->current_setup.transtab); ++ ++ if (kbase_as_has_bus_fault(as, fault)) { ++ kbase_mmu_hw_clear_fault(kbdev, as, ++ KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); ++ kbase_mmu_hw_enable_fault(kbdev, as, ++ KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); ++ } else if (kbase_as_has_page_fault(as, fault)) { ++ kbase_mmu_hw_clear_fault(kbdev, as, ++ KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); ++ kbase_mmu_hw_enable_fault(kbdev, as, ++ KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); ++ } ++ ++ return; ++ } ++ ++ if (kbase_as_has_bus_fault(as, fault)) { ++ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; ++ ++ /* ++ * hw counters dumping in progress, signal the ++ * other thread that it failed ++ */ ++ if ((kbdev->hwcnt.kctx == kctx) && ++ (kbdev->hwcnt.backend.state == ++ KBASE_INSTR_STATE_DUMPING)) ++ kbdev->hwcnt.backend.state = ++ KBASE_INSTR_STATE_FAULT; ++ ++ /* ++ * Stop the kctx from submitting more jobs and cause it ++ * to be scheduled out/rescheduled when all references ++ * to it are released ++ */ ++ kbasep_js_clear_submit_allowed(js_devdata, kctx); ++ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) ++ dev_warn(kbdev->dev, ++ "Bus error in AS%d at VA=0x%016llx, IPA=0x%016llx\n", ++ as->number, fault->addr, ++ fault->extra_addr); ++ else ++ dev_warn(kbdev->dev, "Bus error in AS%d at 0x%016llx\n", ++ as->number, fault->addr); ++ ++ /* ++ * We need to switch to UNMAPPED mode - but we do this in a ++ * worker so that we can sleep ++ */ ++ WARN_ON(!queue_work(as->pf_wq, &as->work_busfault)); ++ atomic_inc(&kbdev->faults_pending); ++ } else { ++ WARN_ON(!queue_work(as->pf_wq, &as->work_pagefault)); ++ atomic_inc(&kbdev->faults_pending); ++ } ++ ++ dev_dbg(kbdev->dev, ++ "Leaving %s kctx %p, as %p\n", ++ __func__, (void *)kctx, (void *)as); ++} ++ ++static void validate_protected_page_fault(struct kbase_device *kbdev) ++{ ++ /* GPUs which support (native) protected mode shall not report page ++ * fault addresses unless it has protected debug mode and protected ++ * debug mode is turned on ++ */ ++ u32 protected_debug_mode = 0; ++ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE)) { ++ protected_debug_mode = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(GPU_STATUS)) & GPU_DBGEN; ++ } ++ ++ if (!protected_debug_mode) { ++ /* fault_addr should never be reported in protected mode. ++ * However, we just continue by printing an error message ++ */ ++ dev_err(kbdev->dev, "Fault address reported in protected mode\n"); ++ } ++} ++ ++void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat) ++{ ++ const int num_as = 16; ++ const int busfault_shift = MMU_PAGE_FAULT_FLAGS; ++ const int pf_shift = 0; ++ const unsigned long as_bit_mask = (1UL << num_as) - 1; ++ unsigned long flags; ++ u32 new_mask; ++ u32 tmp, bf_bits, pf_bits; ++ ++ dev_dbg(kbdev->dev, "Entering %s irq_stat %u\n", ++ __func__, irq_stat); ++ /* bus faults */ ++ bf_bits = (irq_stat >> busfault_shift) & as_bit_mask; ++ /* page faults (note: Ignore ASes with both pf and bf) */ ++ pf_bits = ((irq_stat >> pf_shift) & as_bit_mask) & ~bf_bits; ++ ++ if (WARN_ON(kbdev == NULL)) ++ return; ++ ++ /* remember current mask */ ++ spin_lock_irqsave(&kbdev->mmu_mask_change, flags); ++ new_mask = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK)); ++ /* mask interrupts for now */ ++ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0); ++ spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); ++ ++ while (bf_bits | pf_bits) { ++ struct kbase_as *as; ++ int as_no; ++ struct kbase_context *kctx; ++ struct kbase_fault *fault; ++ ++ /* ++ * the while logic ensures we have a bit set, no need to check ++ * for not-found here ++ */ ++ as_no = ffs(bf_bits | pf_bits) - 1; ++ as = &kbdev->as[as_no]; ++ ++ /* find the fault type */ ++ if (bf_bits & (1 << as_no)) ++ fault = &as->bf_data; ++ else ++ fault = &as->pf_data; ++ ++ /* ++ * Refcount the kctx ASAP - it shouldn't disappear anyway, since ++ * Bus/Page faults _should_ only occur whilst jobs are running, ++ * and a job causing the Bus/Page fault shouldn't complete until ++ * the MMU is updated ++ */ ++ kctx = kbase_ctx_sched_as_to_ctx_refcount(kbdev, as_no); ++ ++ /* find faulting address */ ++ fault->addr = kbase_reg_read(kbdev, MMU_AS_REG(as_no, ++ AS_FAULTADDRESS_HI)); ++ fault->addr <<= 32; ++ fault->addr |= kbase_reg_read(kbdev, MMU_AS_REG(as_no, ++ AS_FAULTADDRESS_LO)); ++ /* Mark the fault protected or not */ ++ fault->protected_mode = kbdev->protected_mode; ++ ++ if (kbdev->protected_mode && fault->addr) { ++ /* check if address reporting is allowed */ ++ validate_protected_page_fault(kbdev); ++ } ++ ++ /* report the fault to debugfs */ ++ kbase_as_fault_debugfs_new(kbdev, as_no); ++ ++ /* record the fault status */ ++ fault->status = kbase_reg_read(kbdev, MMU_AS_REG(as_no, ++ AS_FAULTSTATUS)); ++ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) { ++ fault->extra_addr = kbase_reg_read(kbdev, ++ MMU_AS_REG(as_no, AS_FAULTEXTRA_HI)); ++ fault->extra_addr <<= 32; ++ fault->extra_addr |= kbase_reg_read(kbdev, ++ MMU_AS_REG(as_no, AS_FAULTEXTRA_LO)); ++ } ++ ++ if (kbase_as_has_bus_fault(as, fault)) { ++ /* Mark bus fault as handled. ++ * Note that a bus fault is processed first in case ++ * where both a bus fault and page fault occur. ++ */ ++ bf_bits &= ~(1UL << as_no); ++ ++ /* remove the queued BF (and PF) from the mask */ ++ new_mask &= ~(MMU_BUS_ERROR(as_no) | ++ MMU_PAGE_FAULT(as_no)); ++ } else { ++ /* Mark page fault as handled */ ++ pf_bits &= ~(1UL << as_no); ++ ++ /* remove the queued PF from the mask */ ++ new_mask &= ~MMU_PAGE_FAULT(as_no); ++ } ++ ++ /* Process the interrupt for this address space */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_mmu_interrupt_process(kbdev, kctx, as, fault); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ } ++ ++ /* reenable interrupts */ ++ spin_lock_irqsave(&kbdev->mmu_mask_change, flags); ++ tmp = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK)); ++ new_mask |= tmp; ++ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), new_mask); ++ spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); ++ ++ dev_dbg(kbdev->dev, "Leaving %s irq_stat %u\n", ++ __func__, irq_stat); ++} ++ ++int kbase_mmu_switch_to_ir(struct kbase_context *const kctx, ++ struct kbase_va_region *const reg) ++{ ++ dev_dbg(kctx->kbdev->dev, ++ "Switching to incremental rendering for region %p\n", ++ (void *)reg); ++ return kbase_job_slot_softstop_start_rp(kctx, reg); ++} ++ ++int kbase_mmu_as_init(struct kbase_device *kbdev, int i) ++{ ++ kbdev->as[i].number = i; ++ kbdev->as[i].bf_data.addr = 0ULL; ++ kbdev->as[i].pf_data.addr = 0ULL; ++ ++ kbdev->as[i].pf_wq = alloc_workqueue("mali_mmu%d", 0, 1, i); ++ if (!kbdev->as[i].pf_wq) ++ return -ENOMEM; ++ ++ INIT_WORK(&kbdev->as[i].work_pagefault, kbase_mmu_page_fault_worker); ++ INIT_WORK(&kbdev->as[i].work_busfault, kbase_mmu_bus_fault_worker); ++ ++ return 0; ++} +diff --git a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu.c b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu.c +new file mode 100755 +index 000000000000..a5cda009426d +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu.c +@@ -0,0 +1,2275 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/** ++ * @file mali_kbase_mmu.c ++ * Base kernel MMU management. ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#define KBASE_MMU_PAGE_ENTRIES 512 ++ ++/** ++ * kbase_mmu_flush_invalidate() - Flush and invalidate the GPU caches. ++ * @kctx: The KBase context. ++ * @vpfn: The virtual page frame number to start the flush on. ++ * @nr: The number of pages to flush. ++ * @sync: Set if the operation should be synchronous or not. ++ * ++ * Issue a cache flush + invalidate to the GPU caches and invalidate the TLBs. ++ * ++ * If sync is not set then transactions still in flight when the flush is issued ++ * may use the old page tables and the data they write will not be written out ++ * to memory, this function returns after the flush has been issued but ++ * before all accesses which might effect the flushed region have completed. ++ * ++ * If sync is set then accesses in the flushed region will be drained ++ * before data is flush and invalidated through L1, L2 and into memory, ++ * after which point this function will return. ++ */ ++static void kbase_mmu_flush_invalidate(struct kbase_context *kctx, ++ u64 vpfn, size_t nr, bool sync); ++ ++/** ++ * kbase_mmu_flush_invalidate_no_ctx() - Flush and invalidate the GPU caches. ++ * @kbdev: Device pointer. ++ * @vpfn: The virtual page frame number to start the flush on. ++ * @nr: The number of pages to flush. ++ * @sync: Set if the operation should be synchronous or not. ++ * @as_nr: GPU address space number for which flush + invalidate is required. ++ * ++ * This is used for MMU tables which do not belong to a user space context. ++ */ ++static void kbase_mmu_flush_invalidate_no_ctx(struct kbase_device *kbdev, ++ u64 vpfn, size_t nr, bool sync, int as_nr); ++ ++/** ++ * kbase_mmu_sync_pgd - sync page directory to memory ++ * @kbdev: Device pointer. ++ * @handle: Address of DMA region. ++ * @size: Size of the region to sync. ++ * ++ * This should be called after each page directory update. ++ */ ++ ++static void kbase_mmu_sync_pgd(struct kbase_device *kbdev, ++ dma_addr_t handle, size_t size) ++{ ++ /* If page table is not coherent then ensure the gpu can read ++ * the pages from memory ++ */ ++ if (kbdev->system_coherency != COHERENCY_ACE) ++ dma_sync_single_for_device(kbdev->dev, handle, size, ++ DMA_TO_DEVICE); ++} ++ ++/* ++ * Definitions: ++ * - PGD: Page Directory. ++ * - PTE: Page Table Entry. A 64bit value pointing to the next ++ * level of translation ++ * - ATE: Address Translation Entry. A 64bit value pointing to ++ * a 4kB physical page. ++ */ ++ ++static int kbase_mmu_update_pages_no_flush(struct kbase_context *kctx, u64 vpfn, ++ struct tagged_addr *phys, size_t nr, ++ unsigned long flags, int group_id); ++ ++/** ++ * reg_grow_calc_extra_pages() - Calculate the number of backed pages to add to ++ * a region on a GPU page fault ++ * ++ * @reg: The region that will be backed with more pages ++ * @fault_rel_pfn: PFN of the fault relative to the start of the region ++ * ++ * This calculates how much to increase the backing of a region by, based on ++ * where a GPU page fault occurred and the flags in the region. ++ * ++ * This can be more than the minimum number of pages that would reach ++ * @fault_rel_pfn, for example to reduce the overall rate of page fault ++ * interrupts on a region, or to ensure that the end address is aligned. ++ * ++ * Return: the number of backed pages to increase by ++ */ ++static size_t reg_grow_calc_extra_pages(struct kbase_device *kbdev, ++ struct kbase_va_region *reg, size_t fault_rel_pfn) ++{ ++ size_t multiple = reg->extent; ++ size_t reg_current_size = kbase_reg_current_backed_size(reg); ++ size_t minimum_extra = fault_rel_pfn - reg_current_size + 1; ++ size_t remainder; ++ ++ if (!multiple) { ++ dev_warn(kbdev->dev, ++ "VA Region 0x%llx extent was 0, allocator needs to set this properly for KBASE_REG_PF_GROW\n", ++ ((unsigned long long)reg->start_pfn) << PAGE_SHIFT); ++ return minimum_extra; ++ } ++ ++ /* Calculate the remainder to subtract from minimum_extra to make it ++ * the desired (rounded down) multiple of the extent. ++ * Depending on reg's flags, the base used for calculating multiples is ++ * different ++ */ ++ ++ /* multiple is based from the current backed size, even if the ++ * current backed size/pfn for end of committed memory are not ++ * themselves aligned to multiple ++ */ ++ remainder = minimum_extra % multiple; ++ ++#if !MALI_USE_CSF ++ if (reg->flags & KBASE_REG_TILER_ALIGN_TOP) { ++ /* multiple is based from the top of the initial commit, which ++ * has been allocated in such a way that (start_pfn + ++ * initial_commit) is already aligned to multiple. Hence the ++ * pfn for the end of committed memory will also be aligned to ++ * multiple ++ */ ++ size_t initial_commit = reg->initial_commit; ++ ++ if (fault_rel_pfn < initial_commit) { ++ /* this case is just to catch in case it's been ++ * recommitted by userspace to be smaller than the ++ * initial commit ++ */ ++ minimum_extra = initial_commit - reg_current_size; ++ remainder = 0; ++ } else { ++ /* same as calculating ++ * (fault_rel_pfn - initial_commit + 1) ++ */ ++ size_t pages_after_initial = minimum_extra + ++ reg_current_size - initial_commit; ++ ++ remainder = pages_after_initial % multiple; ++ } ++ } ++#endif /* !MALI_USE_CSF */ ++ ++ if (remainder == 0) ++ return minimum_extra; ++ ++ return minimum_extra + multiple - remainder; ++} ++ ++#ifdef CONFIG_MALI_CINSTR_GWT ++static void kbase_gpu_mmu_handle_write_faulting_as( ++ struct kbase_device *kbdev, ++ struct kbase_as *faulting_as, ++ u64 start_pfn, size_t nr, u32 op) ++{ ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ ++ kbase_mmu_hw_clear_fault(kbdev, faulting_as, ++ KBASE_MMU_FAULT_TYPE_PAGE); ++ kbase_mmu_hw_do_operation(kbdev, faulting_as, start_pfn, ++ nr, op, 1); ++ ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ ++ kbase_mmu_hw_enable_fault(kbdev, faulting_as, ++ KBASE_MMU_FAULT_TYPE_PAGE); ++} ++ ++static void kbase_gpu_mmu_handle_write_fault(struct kbase_context *kctx, ++ struct kbase_as *faulting_as) ++{ ++ struct kbasep_gwt_list_element *pos; ++ struct kbase_va_region *region; ++ struct kbase_device *kbdev; ++ struct kbase_fault *fault; ++ u64 fault_pfn, pfn_offset; ++ u32 op; ++ int ret; ++ int as_no; ++ ++ as_no = faulting_as->number; ++ kbdev = container_of(faulting_as, struct kbase_device, as[as_no]); ++ fault = &faulting_as->pf_data; ++ fault_pfn = fault->addr >> PAGE_SHIFT; ++ ++ kbase_gpu_vm_lock(kctx); ++ ++ /* Find region and check if it should be writable. */ ++ region = kbase_region_tracker_find_region_enclosing_address(kctx, ++ fault->addr); ++ if (kbase_is_region_invalid_or_free(region)) { ++ kbase_gpu_vm_unlock(kctx); ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Memory is not mapped on the GPU", ++ &faulting_as->pf_data); ++ return; ++ } ++ ++ if (!(region->flags & KBASE_REG_GPU_WR)) { ++ kbase_gpu_vm_unlock(kctx); ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Region does not have write permissions", ++ &faulting_as->pf_data); ++ return; ++ } ++ ++ /* Capture addresses of faulting write location ++ * for job dumping if write tracking is enabled. ++ */ ++ if (kctx->gwt_enabled) { ++ u64 page_addr = fault->addr & PAGE_MASK; ++ bool found = false; ++ /* Check if this write was already handled. */ ++ list_for_each_entry(pos, &kctx->gwt_current_list, link) { ++ if (page_addr == pos->page_addr) { ++ found = true; ++ break; ++ } ++ } ++ ++ if (!found) { ++ pos = kmalloc(sizeof(*pos), GFP_KERNEL); ++ if (pos) { ++ pos->region = region; ++ pos->page_addr = page_addr; ++ pos->num_pages = 1; ++ list_add(&pos->link, &kctx->gwt_current_list); ++ } else { ++ dev_warn(kbdev->dev, "kmalloc failure"); ++ } ++ } ++ } ++ ++ pfn_offset = fault_pfn - region->start_pfn; ++ /* Now make this faulting page writable to GPU. */ ++ ret = kbase_mmu_update_pages_no_flush(kctx, fault_pfn, ++ &kbase_get_gpu_phy_pages(region)[pfn_offset], ++ 1, region->flags, region->gpu_alloc->group_id); ++ ++ /* flush L2 and unlock the VA (resumes the MMU) */ ++ op = AS_COMMAND_FLUSH_PT; ++ ++ kbase_gpu_mmu_handle_write_faulting_as(kbdev, faulting_as, ++ fault_pfn, 1, op); ++ ++ kbase_gpu_vm_unlock(kctx); ++} ++ ++static void kbase_gpu_mmu_handle_permission_fault(struct kbase_context *kctx, ++ struct kbase_as *faulting_as) ++{ ++ struct kbase_fault *fault = &faulting_as->pf_data; ++ ++ switch (AS_FAULTSTATUS_ACCESS_TYPE_GET(fault->status)) { ++ case AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC: ++ case AS_FAULTSTATUS_ACCESS_TYPE_WRITE: ++ kbase_gpu_mmu_handle_write_fault(kctx, faulting_as); ++ break; ++ case AS_FAULTSTATUS_ACCESS_TYPE_EX: ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Execute Permission fault", fault); ++ break; ++ case AS_FAULTSTATUS_ACCESS_TYPE_READ: ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Read Permission fault", fault); ++ break; ++ default: ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Unknown Permission fault", fault); ++ break; ++ } ++} ++#endif ++ ++#define MAX_POOL_LEVEL 2 ++ ++/** ++ * page_fault_try_alloc - Try to allocate memory from a context pool ++ * @kctx: Context pointer ++ * @region: Region to grow ++ * @new_pages: Number of 4 kB pages to allocate ++ * @pages_to_grow: Pointer to variable to store number of outstanding pages on ++ * failure. This can be either 4 kB or 2 MB pages, depending on ++ * the number of pages requested. ++ * @grow_2mb_pool: Pointer to variable to store which pool needs to grow - true ++ * for 2 MB, false for 4 kB. ++ * @prealloc_sas: Pointer to kbase_sub_alloc structures ++ * ++ * This function will try to allocate as many pages as possible from the context ++ * pool, then if required will try to allocate the remaining pages from the ++ * device pool. ++ * ++ * This function will not allocate any new memory beyond that that is already ++ * present in the context or device pools. This is because it is intended to be ++ * called with the vm_lock held, which could cause recursive locking if the ++ * allocation caused the out-of-memory killer to run. ++ * ++ * If 2 MB pages are enabled and new_pages is >= 2 MB then pages_to_grow will be ++ * a count of 2 MB pages, otherwise it will be a count of 4 kB pages. ++ * ++ * Return: true if successful, false on failure ++ */ ++static bool page_fault_try_alloc(struct kbase_context *kctx, ++ struct kbase_va_region *region, size_t new_pages, ++ int *pages_to_grow, bool *grow_2mb_pool, ++ struct kbase_sub_alloc **prealloc_sas) ++{ ++ struct tagged_addr *gpu_pages[MAX_POOL_LEVEL] = {NULL}; ++ struct tagged_addr *cpu_pages[MAX_POOL_LEVEL] = {NULL}; ++ size_t pages_alloced[MAX_POOL_LEVEL] = {0}; ++ struct kbase_mem_pool *pool, *root_pool; ++ int pool_level = 0; ++ bool alloc_failed = false; ++ size_t pages_still_required; ++ ++ if (WARN_ON(region->gpu_alloc->group_id >= ++ MEMORY_GROUP_MANAGER_NR_GROUPS)) { ++ /* Do not try to grow the memory pool */ ++ *pages_to_grow = 0; ++ return false; ++ } ++ ++#ifdef CONFIG_MALI_2MB_ALLOC ++ if (new_pages >= (SZ_2M / SZ_4K)) { ++ root_pool = &kctx->mem_pools.large[region->gpu_alloc->group_id]; ++ *grow_2mb_pool = true; ++ } else { ++#endif ++ root_pool = &kctx->mem_pools.small[region->gpu_alloc->group_id]; ++ *grow_2mb_pool = false; ++#ifdef CONFIG_MALI_2MB_ALLOC ++ } ++#endif ++ ++ if (region->gpu_alloc != region->cpu_alloc) ++ new_pages *= 2; ++ ++ pages_still_required = new_pages; ++ ++ /* Determine how many pages are in the pools before trying to allocate. ++ * Don't attempt to allocate & free if the allocation can't succeed. ++ */ ++ for (pool = root_pool; pool != NULL; pool = pool->next_pool) { ++ size_t pool_size_4k; ++ ++ kbase_mem_pool_lock(pool); ++ ++ pool_size_4k = kbase_mem_pool_size(pool) << pool->order; ++ if (pool_size_4k >= pages_still_required) ++ pages_still_required = 0; ++ else ++ pages_still_required -= pool_size_4k; ++ ++ kbase_mem_pool_unlock(pool); ++ ++ if (!pages_still_required) ++ break; ++ } ++ ++ if (pages_still_required) { ++ /* Insufficient pages in pools. Don't try to allocate - just ++ * request a grow. ++ */ ++ *pages_to_grow = pages_still_required; ++ ++ return false; ++ } ++ ++ /* Since we've dropped the pool locks, the amount of memory in the pools ++ * may change between the above check and the actual allocation. ++ */ ++ pool = root_pool; ++ for (pool_level = 0; pool_level < MAX_POOL_LEVEL; pool_level++) { ++ size_t pool_size_4k; ++ size_t pages_to_alloc_4k; ++ size_t pages_to_alloc_4k_per_alloc; ++ ++ kbase_mem_pool_lock(pool); ++ ++ /* Allocate as much as possible from this pool*/ ++ pool_size_4k = kbase_mem_pool_size(pool) << pool->order; ++ pages_to_alloc_4k = MIN(new_pages, pool_size_4k); ++ if (region->gpu_alloc == region->cpu_alloc) ++ pages_to_alloc_4k_per_alloc = pages_to_alloc_4k; ++ else ++ pages_to_alloc_4k_per_alloc = pages_to_alloc_4k >> 1; ++ ++ pages_alloced[pool_level] = pages_to_alloc_4k; ++ if (pages_to_alloc_4k) { ++ gpu_pages[pool_level] = ++ kbase_alloc_phy_pages_helper_locked( ++ region->gpu_alloc, pool, ++ pages_to_alloc_4k_per_alloc, ++ &prealloc_sas[0]); ++ ++ if (!gpu_pages[pool_level]) { ++ alloc_failed = true; ++ } else if (region->gpu_alloc != region->cpu_alloc) { ++ cpu_pages[pool_level] = ++ kbase_alloc_phy_pages_helper_locked( ++ region->cpu_alloc, pool, ++ pages_to_alloc_4k_per_alloc, ++ &prealloc_sas[1]); ++ ++ if (!cpu_pages[pool_level]) ++ alloc_failed = true; ++ } ++ } ++ ++ kbase_mem_pool_unlock(pool); ++ ++ if (alloc_failed) { ++ WARN_ON(!new_pages); ++ WARN_ON(pages_to_alloc_4k >= new_pages); ++ WARN_ON(pages_to_alloc_4k_per_alloc >= new_pages); ++ break; ++ } ++ ++ new_pages -= pages_to_alloc_4k; ++ ++ if (!new_pages) ++ break; ++ ++ pool = pool->next_pool; ++ if (!pool) ++ break; ++ } ++ ++ if (new_pages) { ++ /* Allocation was unsuccessful */ ++ int max_pool_level = pool_level; ++ ++ pool = root_pool; ++ ++ /* Free memory allocated so far */ ++ for (pool_level = 0; pool_level <= max_pool_level; ++ pool_level++) { ++ kbase_mem_pool_lock(pool); ++ ++ if (region->gpu_alloc != region->cpu_alloc) { ++ if (pages_alloced[pool_level] && ++ cpu_pages[pool_level]) ++ kbase_free_phy_pages_helper_locked( ++ region->cpu_alloc, ++ pool, cpu_pages[pool_level], ++ pages_alloced[pool_level]); ++ } ++ ++ if (pages_alloced[pool_level] && gpu_pages[pool_level]) ++ kbase_free_phy_pages_helper_locked( ++ region->gpu_alloc, ++ pool, gpu_pages[pool_level], ++ pages_alloced[pool_level]); ++ ++ kbase_mem_pool_unlock(pool); ++ ++ pool = pool->next_pool; ++ } ++ ++ /* ++ * If the allocation failed despite there being enough memory in ++ * the pool, then just fail. Otherwise, try to grow the memory ++ * pool. ++ */ ++ if (alloc_failed) ++ *pages_to_grow = 0; ++ else ++ *pages_to_grow = new_pages; ++ ++ return false; ++ } ++ ++ /* Allocation was successful. No pages to grow, return success. */ ++ *pages_to_grow = 0; ++ ++ return true; ++} ++ ++/* Small wrapper function to factor out GPU-dependent context releasing */ ++static void release_ctx(struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++#if MALI_USE_CSF ++ CSTD_UNUSED(kbdev); ++ kbase_ctx_sched_release_ctx_lock(kctx); ++#else /* MALI_USE_CSF */ ++ kbasep_js_runpool_release_ctx(kbdev, kctx); ++#endif /* MALI_USE_CSF */ ++} ++ ++void kbase_mmu_page_fault_worker(struct work_struct *data) ++{ ++ u64 fault_pfn; ++ u32 fault_status; ++ size_t new_pages; ++ size_t fault_rel_pfn; ++ struct kbase_as *faulting_as; ++ int as_no; ++ struct kbase_context *kctx; ++ struct kbase_device *kbdev; ++ struct kbase_va_region *region; ++ struct kbase_fault *fault; ++ int err; ++ bool grown = false; ++ int pages_to_grow; ++ bool grow_2mb_pool; ++ struct kbase_sub_alloc *prealloc_sas[2] = { NULL, NULL }; ++ int i; ++ size_t current_backed_size; ++#if MALI_JIT_PRESSURE_LIMIT_BASE ++ size_t pages_trimmed = 0; ++#endif ++ ++ faulting_as = container_of(data, struct kbase_as, work_pagefault); ++ fault = &faulting_as->pf_data; ++ fault_pfn = fault->addr >> PAGE_SHIFT; ++ as_no = faulting_as->number; ++ ++ kbdev = container_of(faulting_as, struct kbase_device, as[as_no]); ++ dev_dbg(kbdev->dev, ++ "Entering %s %p, fault_pfn %lld, as_no %d\n", ++ __func__, (void *)data, fault_pfn, as_no); ++ ++ /* Grab the context that was already refcounted in kbase_mmu_interrupt() ++ * Therefore, it cannot be scheduled out of this AS until we explicitly ++ * release it ++ */ ++ kctx = kbase_ctx_sched_as_to_ctx(kbdev, as_no); ++ if (!kctx) { ++ atomic_dec(&kbdev->faults_pending); ++ return; ++ } ++ ++ KBASE_DEBUG_ASSERT(kctx->kbdev == kbdev); ++ ++#if MALI_JIT_PRESSURE_LIMIT_BASE ++#if !MALI_USE_CSF ++ mutex_lock(&kctx->jctx.lock); ++#endif ++#endif ++ ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++ /* check if we still have GPU */ ++ if (unlikely(kbase_is_gpu_removed(kbdev))) { ++ dev_dbg(kbdev->dev, ++ "%s: GPU has been removed\n", __func__); ++ goto fault_done; ++ } ++#endif ++ ++ if (unlikely(fault->protected_mode)) { ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Protected mode fault", fault); ++ kbase_mmu_hw_clear_fault(kbdev, faulting_as, ++ KBASE_MMU_FAULT_TYPE_PAGE); ++ ++ goto fault_done; ++ } ++ ++ fault_status = fault->status; ++ switch (fault_status & AS_FAULTSTATUS_EXCEPTION_CODE_MASK) { ++ ++ case AS_FAULTSTATUS_EXCEPTION_CODE_TRANSLATION_FAULT: ++ /* need to check against the region to handle this one */ ++ break; ++ ++ case AS_FAULTSTATUS_EXCEPTION_CODE_PERMISSION_FAULT: ++#ifdef CONFIG_MALI_CINSTR_GWT ++ /* If GWT was ever enabled then we need to handle ++ * write fault pages even if the feature was disabled later. ++ */ ++ if (kctx->gwt_was_enabled) { ++ kbase_gpu_mmu_handle_permission_fault(kctx, ++ faulting_as); ++ goto fault_done; ++ } ++#endif ++ ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Permission failure", fault); ++ goto fault_done; ++ ++ case AS_FAULTSTATUS_EXCEPTION_CODE_TRANSTAB_BUS_FAULT: ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Translation table bus fault", fault); ++ goto fault_done; ++ ++ case AS_FAULTSTATUS_EXCEPTION_CODE_ACCESS_FLAG: ++ /* nothing to do, but we don't expect this fault currently */ ++ dev_warn(kbdev->dev, "Access flag unexpectedly set"); ++ goto fault_done; ++ ++ case AS_FAULTSTATUS_EXCEPTION_CODE_ADDRESS_SIZE_FAULT: ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Address size fault", fault); ++ else ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Unknown fault code", fault); ++ goto fault_done; ++ ++ case AS_FAULTSTATUS_EXCEPTION_CODE_MEMORY_ATTRIBUTES_FAULT: ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Memory attributes fault", fault); ++ else ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Unknown fault code", fault); ++ goto fault_done; ++ ++ default: ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Unknown fault code", fault); ++ goto fault_done; ++ } ++ ++#ifdef CONFIG_MALI_2MB_ALLOC ++ /* Preallocate memory for the sub-allocation structs if necessary */ ++ for (i = 0; i != ARRAY_SIZE(prealloc_sas); ++i) { ++ prealloc_sas[i] = kmalloc(sizeof(*prealloc_sas[i]), GFP_KERNEL); ++ if (!prealloc_sas[i]) { ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Failed pre-allocating memory for sub-allocations' metadata", ++ fault); ++ goto fault_done; ++ } ++ } ++#endif /* CONFIG_MALI_2MB_ALLOC */ ++ ++page_fault_retry: ++ /* so we have a translation fault, ++ * let's see if it is for growable memory ++ */ ++ kbase_gpu_vm_lock(kctx); ++ ++ region = kbase_region_tracker_find_region_enclosing_address(kctx, ++ fault->addr); ++ if (kbase_is_region_invalid_or_free(region)) { ++ kbase_gpu_vm_unlock(kctx); ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Memory is not mapped on the GPU", fault); ++ goto fault_done; ++ } ++ ++ if (region->gpu_alloc->type == KBASE_MEM_TYPE_IMPORTED_UMM) { ++ kbase_gpu_vm_unlock(kctx); ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "DMA-BUF is not mapped on the GPU", fault); ++ goto fault_done; ++ } ++ ++ if (region->gpu_alloc->group_id >= MEMORY_GROUP_MANAGER_NR_GROUPS) { ++ kbase_gpu_vm_unlock(kctx); ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Bad physical memory group ID", fault); ++ goto fault_done; ++ } ++ ++ if ((region->flags & GROWABLE_FLAGS_REQUIRED) ++ != GROWABLE_FLAGS_REQUIRED) { ++ kbase_gpu_vm_unlock(kctx); ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Memory is not growable", fault); ++ goto fault_done; ++ } ++ ++ if ((region->flags & KBASE_REG_DONT_NEED)) { ++ kbase_gpu_vm_unlock(kctx); ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Don't need memory can't be grown", fault); ++ goto fault_done; ++ } ++ ++ /* find the size we need to grow it by ++ * we know the result fit in a size_t due to ++ * kbase_region_tracker_find_region_enclosing_address ++ * validating the fault_address to be within a size_t from the start_pfn ++ */ ++ fault_rel_pfn = fault_pfn - region->start_pfn; ++ ++ current_backed_size = kbase_reg_current_backed_size(region); ++ ++ if (fault_rel_pfn < current_backed_size) { ++ dev_dbg(kbdev->dev, ++ "Page fault @ 0x%llx in allocated region 0x%llx-0x%llx of growable TMEM: Ignoring", ++ fault->addr, region->start_pfn, ++ region->start_pfn + ++ current_backed_size); ++ ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ ++ kbase_mmu_hw_clear_fault(kbdev, faulting_as, ++ KBASE_MMU_FAULT_TYPE_PAGE); ++ /* [1] in case another page fault occurred while we were ++ * handling the (duplicate) page fault we need to ensure we ++ * don't loose the other page fault as result of us clearing ++ * the MMU IRQ. Therefore, after we clear the MMU IRQ we send ++ * an UNLOCK command that will retry any stalled memory ++ * transaction (which should cause the other page fault to be ++ * raised again). ++ */ ++ kbase_mmu_hw_do_operation(kbdev, faulting_as, 0, 0, ++ AS_COMMAND_UNLOCK, 1); ++ ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ ++ kbase_mmu_hw_enable_fault(kbdev, faulting_as, ++ KBASE_MMU_FAULT_TYPE_PAGE); ++ kbase_gpu_vm_unlock(kctx); ++ ++ goto fault_done; ++ } ++ ++ new_pages = reg_grow_calc_extra_pages(kbdev, region, fault_rel_pfn); ++ ++ /* cap to max vsize */ ++ new_pages = min(new_pages, region->nr_pages - current_backed_size); ++ dev_dbg(kctx->kbdev->dev, "Allocate %zu pages on page fault\n", ++ new_pages); ++ ++ if (new_pages == 0) { ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ ++ /* Duplicate of a fault we've already handled, nothing to do */ ++ kbase_mmu_hw_clear_fault(kbdev, faulting_as, ++ KBASE_MMU_FAULT_TYPE_PAGE); ++ /* See comment [1] about UNLOCK usage */ ++ kbase_mmu_hw_do_operation(kbdev, faulting_as, 0, 0, ++ AS_COMMAND_UNLOCK, 1); ++ ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ ++ kbase_mmu_hw_enable_fault(kbdev, faulting_as, ++ KBASE_MMU_FAULT_TYPE_PAGE); ++ kbase_gpu_vm_unlock(kctx); ++ goto fault_done; ++ } ++ ++ pages_to_grow = 0; ++ ++#if MALI_JIT_PRESSURE_LIMIT_BASE ++ if ((region->flags & KBASE_REG_ACTIVE_JIT_ALLOC) && !pages_trimmed) { ++ kbase_jit_request_phys_increase(kctx, new_pages); ++ pages_trimmed = new_pages; ++ } ++#endif ++ ++ spin_lock(&kctx->mem_partials_lock); ++ grown = page_fault_try_alloc(kctx, region, new_pages, &pages_to_grow, ++ &grow_2mb_pool, prealloc_sas); ++ spin_unlock(&kctx->mem_partials_lock); ++ ++ if (grown) { ++ u64 pfn_offset; ++ u32 op; ++ ++ /* alloc success */ ++ WARN_ON(kbase_reg_current_backed_size(region) > ++ region->nr_pages); ++ ++ /* set up the new pages */ ++ pfn_offset = kbase_reg_current_backed_size(region) - new_pages; ++ /* ++ * Note: ++ * Issuing an MMU operation will unlock the MMU and cause the ++ * translation to be replayed. If the page insertion fails then ++ * rather then trying to continue the context should be killed ++ * so the no_flush version of insert_pages is used which allows ++ * us to unlock the MMU as we see fit. ++ */ ++ err = kbase_mmu_insert_pages_no_flush(kbdev, &kctx->mmu, ++ region->start_pfn + pfn_offset, ++ &kbase_get_gpu_phy_pages(region)[pfn_offset], ++ new_pages, region->flags, region->gpu_alloc->group_id); ++ if (err) { ++ kbase_free_phy_pages_helper(region->gpu_alloc, ++ new_pages); ++ if (region->gpu_alloc != region->cpu_alloc) ++ kbase_free_phy_pages_helper(region->cpu_alloc, ++ new_pages); ++ kbase_gpu_vm_unlock(kctx); ++ /* The locked VA region will be unlocked and the cache ++ * invalidated in here ++ */ ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Page table update failure", fault); ++ goto fault_done; ++ } ++ KBASE_TLSTREAM_AUX_PAGEFAULT(kbdev, kctx->id, as_no, ++ (u64)new_pages); ++ trace_mali_mmu_page_fault_grow(region, fault, new_pages); ++ ++#if MALI_INCREMENTAL_RENDERING ++ /* Switch to incremental rendering if we have nearly run out of ++ * memory in a JIT memory allocation. ++ */ ++ if (region->threshold_pages && ++ kbase_reg_current_backed_size(region) > ++ region->threshold_pages) { ++ ++ dev_dbg(kctx->kbdev->dev, ++ "%zu pages exceeded IR threshold %zu\n", ++ new_pages + current_backed_size, ++ region->threshold_pages); ++ ++ if (kbase_mmu_switch_to_ir(kctx, region) >= 0) { ++ dev_dbg(kctx->kbdev->dev, ++ "Get region %p for IR\n", ++ (void *)region); ++ kbase_va_region_alloc_get(kctx, region); ++ } ++ } ++#endif ++ ++ /* AS transaction begin */ ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ ++ /* flush L2 and unlock the VA (resumes the MMU) */ ++ op = AS_COMMAND_FLUSH_PT; ++ ++ /* clear MMU interrupt - this needs to be done after updating ++ * the page tables but before issuing a FLUSH command. The ++ * FLUSH cmd has a side effect that it restarts stalled memory ++ * transactions in other address spaces which may cause ++ * another fault to occur. If we didn't clear the interrupt at ++ * this stage a new IRQ might not be raised when the GPU finds ++ * a MMU IRQ is already pending. ++ */ ++ kbase_mmu_hw_clear_fault(kbdev, faulting_as, ++ KBASE_MMU_FAULT_TYPE_PAGE); ++ ++ kbase_mmu_hw_do_operation(kbdev, faulting_as, ++ fault->addr >> PAGE_SHIFT, ++ new_pages, op, 1); ++ ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ /* AS transaction end */ ++ ++ /* reenable this in the mask */ ++ kbase_mmu_hw_enable_fault(kbdev, faulting_as, ++ KBASE_MMU_FAULT_TYPE_PAGE); ++ ++#ifdef CONFIG_MALI_CINSTR_GWT ++ if (kctx->gwt_enabled) { ++ /* GWT also tracks growable regions. */ ++ struct kbasep_gwt_list_element *pos; ++ ++ pos = kmalloc(sizeof(*pos), GFP_KERNEL); ++ if (pos) { ++ pos->region = region; ++ pos->page_addr = (region->start_pfn + ++ pfn_offset) << ++ PAGE_SHIFT; ++ pos->num_pages = new_pages; ++ list_add(&pos->link, ++ &kctx->gwt_current_list); ++ } else { ++ dev_warn(kbdev->dev, "kmalloc failure"); ++ } ++ } ++#endif ++ ++#if MALI_JIT_PRESSURE_LIMIT_BASE ++ if (pages_trimmed) { ++ kbase_jit_done_phys_increase(kctx, pages_trimmed); ++ pages_trimmed = 0; ++ } ++#endif ++ kbase_gpu_vm_unlock(kctx); ++ } else { ++ int ret = -ENOMEM; ++ ++ kbase_gpu_vm_unlock(kctx); ++ ++ /* If the memory pool was insufficient then grow it and retry. ++ * Otherwise fail the allocation. ++ */ ++ if (pages_to_grow > 0) { ++#ifdef CONFIG_MALI_2MB_ALLOC ++ if (grow_2mb_pool) { ++ /* Round page requirement up to nearest 2 MB */ ++ struct kbase_mem_pool *const lp_mem_pool = ++ &kctx->mem_pools.large[ ++ region->gpu_alloc->group_id]; ++ ++ pages_to_grow = (pages_to_grow + ++ ((1 << lp_mem_pool->order) - 1)) ++ >> lp_mem_pool->order; ++ ++ ret = kbase_mem_pool_grow(lp_mem_pool, ++ pages_to_grow); ++ } else { ++#endif ++ struct kbase_mem_pool *const mem_pool = ++ &kctx->mem_pools.small[ ++ region->gpu_alloc->group_id]; ++ ++ ret = kbase_mem_pool_grow(mem_pool, ++ pages_to_grow); ++#ifdef CONFIG_MALI_2MB_ALLOC ++ } ++#endif ++ } ++ if (ret < 0) { ++ /* failed to extend, handle as a normal PF */ ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Page allocation failure", fault); ++ } else { ++ dev_dbg(kbdev->dev, "Try again after pool_grow\n"); ++ goto page_fault_retry; ++ } ++ } ++ ++fault_done: ++#if MALI_JIT_PRESSURE_LIMIT_BASE ++ if (pages_trimmed) { ++ kbase_gpu_vm_lock(kctx); ++ kbase_jit_done_phys_increase(kctx, pages_trimmed); ++ kbase_gpu_vm_unlock(kctx); ++ } ++#if !MALI_USE_CSF ++ mutex_unlock(&kctx->jctx.lock); ++#endif ++#endif ++ ++ for (i = 0; i != ARRAY_SIZE(prealloc_sas); ++i) ++ kfree(prealloc_sas[i]); ++ ++ /* ++ * By this point, the fault was handled in some way, ++ * so release the ctx refcount ++ */ ++ release_ctx(kbdev, kctx); ++ ++ atomic_dec(&kbdev->faults_pending); ++ dev_dbg(kbdev->dev, "Leaving page_fault_worker %p\n", (void *)data); ++} ++ ++static phys_addr_t kbase_mmu_alloc_pgd(struct kbase_device *kbdev, ++ struct kbase_mmu_table *mmut) ++{ ++ u64 *page; ++ int i; ++ struct page *p; ++ ++ p = kbase_mem_pool_alloc(&kbdev->mem_pools.small[mmut->group_id]); ++ if (!p) ++ return 0; ++ ++ page = kmap(p); ++ if (page == NULL) ++ goto alloc_free; ++ ++ /* If the MMU tables belong to a context then account the memory usage ++ * to that context, otherwise the MMU tables are device wide and are ++ * only accounted to the device. ++ */ ++ if (mmut->kctx) { ++ int new_page_count; ++ ++ new_page_count = atomic_add_return(1, ++ &mmut->kctx->used_pages); ++ KBASE_TLSTREAM_AUX_PAGESALLOC( ++ kbdev, ++ mmut->kctx->id, ++ (u64)new_page_count); ++ kbase_process_page_usage_inc(mmut->kctx, 1); ++ } ++ ++ atomic_add(1, &kbdev->memdev.used_pages); ++ ++ kbase_trace_gpu_mem_usage_inc(kbdev, mmut->kctx, 1); ++ ++ for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++) ++ kbdev->mmu_mode->entry_invalidate(&page[i]); ++ ++ kbase_mmu_sync_pgd(kbdev, kbase_dma_addr(p), PAGE_SIZE); ++ ++ kunmap(p); ++ return page_to_phys(p); ++ ++alloc_free: ++ kbase_mem_pool_free(&kbdev->mem_pools.small[mmut->group_id], p, ++ false); ++ ++ return 0; ++} ++ ++/* Given PGD PFN for level N, return PGD PFN for level N+1, allocating the ++ * new table from the pool if needed and possible ++ */ ++static int mmu_get_next_pgd(struct kbase_device *kbdev, ++ struct kbase_mmu_table *mmut, ++ phys_addr_t *pgd, u64 vpfn, int level) ++{ ++ u64 *page; ++ phys_addr_t target_pgd; ++ struct page *p; ++ ++ KBASE_DEBUG_ASSERT(*pgd); ++ ++ lockdep_assert_held(&mmut->mmu_lock); ++ ++ /* ++ * Architecture spec defines level-0 as being the top-most. ++ * This is a bit unfortunate here, but we keep the same convention. ++ */ ++ vpfn >>= (3 - level) * 9; ++ vpfn &= 0x1FF; ++ ++ p = pfn_to_page(PFN_DOWN(*pgd)); ++ page = kmap(p); ++ if (page == NULL) { ++ dev_warn(kbdev->dev, "%s: kmap failure\n", __func__); ++ return -EINVAL; ++ } ++ ++ target_pgd = kbdev->mmu_mode->pte_to_phy_addr(page[vpfn]); ++ ++ if (!target_pgd) { ++ target_pgd = kbase_mmu_alloc_pgd(kbdev, mmut); ++ if (!target_pgd) { ++ dev_dbg(kbdev->dev, "%s: kbase_mmu_alloc_pgd failure\n", ++ __func__); ++ kunmap(p); ++ return -ENOMEM; ++ } ++ ++ kbdev->mmu_mode->entry_set_pte(&page[vpfn], target_pgd); ++ ++ kbase_mmu_sync_pgd(kbdev, kbase_dma_addr(p), PAGE_SIZE); ++ /* Rely on the caller to update the address space flags. */ ++ } ++ ++ kunmap(p); ++ *pgd = target_pgd; ++ ++ return 0; ++} ++ ++/* ++ * Returns the PGD for the specified level of translation ++ */ ++static int mmu_get_pgd_at_level(struct kbase_device *kbdev, ++ struct kbase_mmu_table *mmut, ++ u64 vpfn, ++ int level, ++ phys_addr_t *out_pgd) ++{ ++ phys_addr_t pgd; ++ int l; ++ ++ lockdep_assert_held(&mmut->mmu_lock); ++ pgd = mmut->pgd; ++ ++ for (l = MIDGARD_MMU_TOPLEVEL; l < level; l++) { ++ int err = mmu_get_next_pgd(kbdev, mmut, &pgd, vpfn, l); ++ /* Handle failure condition */ ++ if (err) { ++ dev_dbg(kbdev->dev, ++ "%s: mmu_get_next_pgd failure at level %d\n", ++ __func__, l); ++ return err; ++ } ++ } ++ ++ *out_pgd = pgd; ++ ++ return 0; ++} ++ ++static int mmu_get_bottom_pgd(struct kbase_device *kbdev, ++ struct kbase_mmu_table *mmut, ++ u64 vpfn, ++ phys_addr_t *out_pgd) ++{ ++ return mmu_get_pgd_at_level(kbdev, mmut, vpfn, MIDGARD_MMU_BOTTOMLEVEL, ++ out_pgd); ++} ++ ++static void mmu_insert_pages_failure_recovery(struct kbase_device *kbdev, ++ struct kbase_mmu_table *mmut, ++ u64 from_vpfn, u64 to_vpfn) ++{ ++ phys_addr_t pgd; ++ u64 vpfn = from_vpfn; ++ struct kbase_mmu_mode const *mmu_mode; ++ ++ /* 64-bit address range is the max */ ++ KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE)); ++ KBASE_DEBUG_ASSERT(from_vpfn <= to_vpfn); ++ ++ lockdep_assert_held(&mmut->mmu_lock); ++ ++ mmu_mode = kbdev->mmu_mode; ++ ++ while (vpfn < to_vpfn) { ++ unsigned int i; ++ unsigned int idx = vpfn & 0x1FF; ++ unsigned int count = KBASE_MMU_PAGE_ENTRIES - idx; ++ unsigned int pcount = 0; ++ unsigned int left = to_vpfn - vpfn; ++ int level; ++ u64 *page; ++ ++ if (count > left) ++ count = left; ++ ++ /* need to check if this is a 2MB page or a 4kB */ ++ pgd = mmut->pgd; ++ ++ for (level = MIDGARD_MMU_TOPLEVEL; ++ level <= MIDGARD_MMU_BOTTOMLEVEL; level++) { ++ idx = (vpfn >> ((3 - level) * 9)) & 0x1FF; ++ page = kmap(phys_to_page(pgd)); ++ if (mmu_mode->ate_is_valid(page[idx], level)) ++ break; /* keep the mapping */ ++ kunmap(phys_to_page(pgd)); ++ pgd = mmu_mode->pte_to_phy_addr(page[idx]); ++ } ++ ++ switch (level) { ++ case MIDGARD_MMU_LEVEL(2): ++ /* remap to single entry to update */ ++ pcount = 1; ++ break; ++ case MIDGARD_MMU_BOTTOMLEVEL: ++ /* page count is the same as the logical count */ ++ pcount = count; ++ break; ++ default: ++ dev_warn(kbdev->dev, "%sNo support for ATEs at level %d\n", ++ __func__, level); ++ goto next; ++ } ++ ++ /* Invalidate the entries we added */ ++ for (i = 0; i < pcount; i++) ++ mmu_mode->entry_invalidate(&page[idx + i]); ++ ++ kbase_mmu_sync_pgd(kbdev, ++ kbase_dma_addr(phys_to_page(pgd)) + 8 * idx, ++ 8 * pcount); ++ kunmap(phys_to_page(pgd)); ++ ++next: ++ vpfn += count; ++ } ++} ++ ++/* ++ * Map the single page 'phys' 'nr' of times, starting at GPU PFN 'vpfn' ++ */ ++int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 vpfn, ++ struct tagged_addr phys, size_t nr, ++ unsigned long flags, int const group_id) ++{ ++ phys_addr_t pgd; ++ u64 *pgd_page; ++ /* In case the insert_single_page only partially completes ++ * we need to be able to recover ++ */ ++ bool recover_required = false; ++ u64 start_vpfn = vpfn; ++ size_t recover_count = 0; ++ size_t remain = nr; ++ int err; ++ struct kbase_device *kbdev; ++ ++ if (WARN_ON(kctx == NULL)) ++ return -EINVAL; ++ ++ /* 64-bit address range is the max */ ++ KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE)); ++ ++ kbdev = kctx->kbdev; ++ ++ /* Early out if there is nothing to do */ ++ if (nr == 0) ++ return 0; ++ ++ mutex_lock(&kctx->mmu.mmu_lock); ++ ++ while (remain) { ++ unsigned int i; ++ unsigned int index = vpfn & 0x1FF; ++ unsigned int count = KBASE_MMU_PAGE_ENTRIES - index; ++ struct page *p; ++ ++ if (count > remain) ++ count = remain; ++ ++ /* ++ * Repeatedly calling mmu_get_bottom_pte() is clearly ++ * suboptimal. We don't have to re-parse the whole tree ++ * each time (just cache the l0-l2 sequence). ++ * On the other hand, it's only a gain when we map more than ++ * 256 pages at once (on average). Do we really care? ++ */ ++ do { ++ err = mmu_get_bottom_pgd(kbdev, &kctx->mmu, ++ vpfn, &pgd); ++ if (err != -ENOMEM) ++ break; ++ /* Fill the memory pool with enough pages for ++ * the page walk to succeed ++ */ ++ mutex_unlock(&kctx->mmu.mmu_lock); ++ err = kbase_mem_pool_grow( ++ &kbdev->mem_pools.small[ ++ kctx->mmu.group_id], ++ MIDGARD_MMU_BOTTOMLEVEL); ++ mutex_lock(&kctx->mmu.mmu_lock); ++ } while (!err); ++ if (err) { ++ dev_warn(kbdev->dev, "kbase_mmu_insert_pages: mmu_get_bottom_pgd failure\n"); ++ if (recover_required) { ++ /* Invalidate the pages we have partially ++ * completed ++ */ ++ mmu_insert_pages_failure_recovery(kbdev, ++ &kctx->mmu, ++ start_vpfn, ++ start_vpfn + recover_count); ++ } ++ goto fail_unlock; ++ } ++ ++ p = pfn_to_page(PFN_DOWN(pgd)); ++ pgd_page = kmap(p); ++ if (!pgd_page) { ++ dev_warn(kbdev->dev, "kbase_mmu_insert_pages: kmap failure\n"); ++ if (recover_required) { ++ /* Invalidate the pages we have partially ++ * completed ++ */ ++ mmu_insert_pages_failure_recovery(kbdev, ++ &kctx->mmu, ++ start_vpfn, ++ start_vpfn + recover_count); ++ } ++ err = -ENOMEM; ++ goto fail_unlock; ++ } ++ ++ for (i = 0; i < count; i++) { ++ unsigned int ofs = index + i; ++ ++ /* Fail if the current page is a valid ATE entry */ ++ KBASE_DEBUG_ASSERT(0 == (pgd_page[ofs] & 1UL)); ++ ++ pgd_page[ofs] = kbase_mmu_create_ate(kbdev, ++ phys, flags, MIDGARD_MMU_BOTTOMLEVEL, group_id); ++ } ++ ++ vpfn += count; ++ remain -= count; ++ ++ kbase_mmu_sync_pgd(kbdev, ++ kbase_dma_addr(p) + (index * sizeof(u64)), ++ count * sizeof(u64)); ++ ++ kunmap(p); ++ /* We have started modifying the page table. ++ * If further pages need inserting and fail we need to undo what ++ * has already taken place ++ */ ++ recover_required = true; ++ recover_count += count; ++ } ++ mutex_unlock(&kctx->mmu.mmu_lock); ++ kbase_mmu_flush_invalidate(kctx, start_vpfn, nr, false); ++ return 0; ++ ++fail_unlock: ++ mutex_unlock(&kctx->mmu.mmu_lock); ++ kbase_mmu_flush_invalidate(kctx, start_vpfn, nr, false); ++ return err; ++} ++ ++static inline void cleanup_empty_pte(struct kbase_device *kbdev, ++ struct kbase_mmu_table *mmut, u64 *pte) ++{ ++ phys_addr_t tmp_pgd; ++ struct page *tmp_p; ++ ++ tmp_pgd = kbdev->mmu_mode->pte_to_phy_addr(*pte); ++ tmp_p = phys_to_page(tmp_pgd); ++ kbase_mem_pool_free(&kbdev->mem_pools.small[mmut->group_id], ++ tmp_p, false); ++ ++ /* If the MMU tables belong to a context then we accounted the memory ++ * usage to that context, so decrement here. ++ */ ++ if (mmut->kctx) { ++ kbase_process_page_usage_dec(mmut->kctx, 1); ++ atomic_sub(1, &mmut->kctx->used_pages); ++ } ++ atomic_sub(1, &kbdev->memdev.used_pages); ++ ++ kbase_trace_gpu_mem_usage_dec(kbdev, mmut->kctx, 1); ++} ++ ++u64 kbase_mmu_create_ate(struct kbase_device *const kbdev, ++ struct tagged_addr const phy, unsigned long const flags, ++ int const level, int const group_id) ++{ ++ u64 entry; ++ ++ kbdev->mmu_mode->entry_set_ate(&entry, phy, flags, level); ++ return kbdev->mgm_dev->ops.mgm_update_gpu_pte(kbdev->mgm_dev, ++ group_id, level, entry); ++} ++ ++int kbase_mmu_insert_pages_no_flush(struct kbase_device *kbdev, ++ struct kbase_mmu_table *mmut, ++ const u64 start_vpfn, ++ struct tagged_addr *phys, size_t nr, ++ unsigned long flags, ++ int const group_id) ++{ ++ phys_addr_t pgd; ++ u64 *pgd_page; ++ u64 insert_vpfn = start_vpfn; ++ size_t remain = nr; ++ int err; ++ struct kbase_mmu_mode const *mmu_mode; ++ ++ /* Note that 0 is a valid start_vpfn */ ++ /* 64-bit address range is the max */ ++ KBASE_DEBUG_ASSERT(start_vpfn <= (U64_MAX / PAGE_SIZE)); ++ ++ mmu_mode = kbdev->mmu_mode; ++ ++ /* Early out if there is nothing to do */ ++ if (nr == 0) ++ return 0; ++ ++ mutex_lock(&mmut->mmu_lock); ++ ++ while (remain) { ++ unsigned int i; ++ unsigned int vindex = insert_vpfn & 0x1FF; ++ unsigned int count = KBASE_MMU_PAGE_ENTRIES - vindex; ++ struct page *p; ++ int cur_level; ++ ++ if (count > remain) ++ count = remain; ++ ++ if (!vindex && is_huge_head(*phys)) ++ cur_level = MIDGARD_MMU_LEVEL(2); ++ else ++ cur_level = MIDGARD_MMU_BOTTOMLEVEL; ++ ++ /* ++ * Repeatedly calling mmu_get_pgd_at_level() is clearly ++ * suboptimal. We don't have to re-parse the whole tree ++ * each time (just cache the l0-l2 sequence). ++ * On the other hand, it's only a gain when we map more than ++ * 256 pages at once (on average). Do we really care? ++ */ ++ do { ++ err = mmu_get_pgd_at_level(kbdev, mmut, insert_vpfn, ++ cur_level, &pgd); ++ if (err != -ENOMEM) ++ break; ++ /* Fill the memory pool with enough pages for ++ * the page walk to succeed ++ */ ++ mutex_unlock(&mmut->mmu_lock); ++ err = kbase_mem_pool_grow( ++ &kbdev->mem_pools.small[mmut->group_id], ++ cur_level); ++ mutex_lock(&mmut->mmu_lock); ++ } while (!err); ++ ++ if (err) { ++ dev_warn(kbdev->dev, ++ "%s: mmu_get_bottom_pgd failure\n", __func__); ++ if (insert_vpfn != start_vpfn) { ++ /* Invalidate the pages we have partially ++ * completed ++ */ ++ mmu_insert_pages_failure_recovery(kbdev, ++ mmut, start_vpfn, insert_vpfn); ++ } ++ goto fail_unlock; ++ } ++ ++ p = pfn_to_page(PFN_DOWN(pgd)); ++ pgd_page = kmap(p); ++ if (!pgd_page) { ++ dev_warn(kbdev->dev, "%s: kmap failure\n", ++ __func__); ++ if (insert_vpfn != start_vpfn) { ++ /* Invalidate the pages we have partially ++ * completed ++ */ ++ mmu_insert_pages_failure_recovery(kbdev, ++ mmut, start_vpfn, insert_vpfn); ++ } ++ err = -ENOMEM; ++ goto fail_unlock; ++ } ++ ++ if (cur_level == MIDGARD_MMU_LEVEL(2)) { ++ int level_index = (insert_vpfn >> 9) & 0x1FF; ++ u64 *target = &pgd_page[level_index]; ++ ++ if (mmu_mode->pte_is_valid(*target, cur_level)) ++ cleanup_empty_pte(kbdev, mmut, target); ++ *target = kbase_mmu_create_ate(kbdev, *phys, flags, ++ cur_level, group_id); ++ } else { ++ for (i = 0; i < count; i++) { ++ unsigned int ofs = vindex + i; ++ u64 *target = &pgd_page[ofs]; ++ ++ /* Warn if the current page is a valid ATE ++ * entry. The page table shouldn't have anything ++ * in the place where we are trying to put a ++ * new entry. Modification to page table entries ++ * should be performed with ++ * kbase_mmu_update_pages() ++ */ ++ WARN_ON((*target & 1UL) != 0); ++ ++ *target = kbase_mmu_create_ate(kbdev, ++ phys[i], flags, cur_level, group_id); ++ } ++ } ++ ++ phys += count; ++ insert_vpfn += count; ++ remain -= count; ++ ++ kbase_mmu_sync_pgd(kbdev, ++ kbase_dma_addr(p) + (vindex * sizeof(u64)), ++ count * sizeof(u64)); ++ ++ kunmap(p); ++ } ++ ++ err = 0; ++ ++fail_unlock: ++ mutex_unlock(&mmut->mmu_lock); ++ return err; ++} ++ ++/* ++ * Map 'nr' pages pointed to by 'phys' at GPU PFN 'vpfn' for GPU address space ++ * number 'as_nr'. ++ */ ++int kbase_mmu_insert_pages(struct kbase_device *kbdev, ++ struct kbase_mmu_table *mmut, u64 vpfn, ++ struct tagged_addr *phys, size_t nr, ++ unsigned long flags, int as_nr, int const group_id) ++{ ++ int err; ++ ++ err = kbase_mmu_insert_pages_no_flush(kbdev, mmut, vpfn, ++ phys, nr, flags, group_id); ++ ++ if (mmut->kctx) ++ kbase_mmu_flush_invalidate(mmut->kctx, vpfn, nr, false); ++ else ++ kbase_mmu_flush_invalidate_no_ctx(kbdev, vpfn, nr, false, ++ as_nr); ++ ++ return err; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_mmu_insert_pages); ++ ++/** ++ * kbase_mmu_flush_invalidate_noretain() - Flush and invalidate the GPU caches ++ * without retaining the kbase context. ++ * @kctx: The KBase context. ++ * @vpfn: The virtual page frame number to start the flush on. ++ * @nr: The number of pages to flush. ++ * @sync: Set if the operation should be synchronous or not. ++ * ++ * As per kbase_mmu_flush_invalidate but doesn't retain the kctx or do any ++ * other locking. ++ */ ++static void kbase_mmu_flush_invalidate_noretain(struct kbase_context *kctx, ++ u64 vpfn, size_t nr, bool sync) ++{ ++ struct kbase_device *kbdev = kctx->kbdev; ++ int err; ++ u32 op; ++ ++ /* Early out if there is nothing to do */ ++ if (nr == 0) ++ return; ++ ++ if (sync) ++ op = AS_COMMAND_FLUSH_MEM; ++ else ++ op = AS_COMMAND_FLUSH_PT; ++ ++ err = kbase_mmu_hw_do_operation(kbdev, ++ &kbdev->as[kctx->as_nr], ++ vpfn, nr, op, 0); ++ if (err) { ++ /* Flush failed to complete, assume the ++ * GPU has hung and perform a reset to recover ++ */ ++ dev_err(kbdev->dev, "Flush for GPU page table update did not complete. Issuing GPU soft-reset to recover\n"); ++ ++ if (kbase_prepare_to_reset_gpu_locked(kbdev)) ++ kbase_reset_gpu_locked(kbdev); ++ } ++} ++ ++/* Perform a flush/invalidate on a particular address space ++ */ ++static void kbase_mmu_flush_invalidate_as(struct kbase_device *kbdev, ++ struct kbase_as *as, ++ u64 vpfn, size_t nr, bool sync) ++{ ++ int err; ++ u32 op; ++ ++ if (kbase_pm_context_active_handle_suspend(kbdev, ++ KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) { ++ /* GPU is off so there's no need to perform flush/invalidate */ ++ return; ++ } ++ ++ /* AS transaction begin */ ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ ++ if (sync) ++ op = AS_COMMAND_FLUSH_MEM; ++ else ++ op = AS_COMMAND_FLUSH_PT; ++ ++ err = kbase_mmu_hw_do_operation(kbdev, ++ as, vpfn, nr, op, 0); ++ ++ if (err) { ++ /* Flush failed to complete, assume the GPU has hung and ++ * perform a reset to recover ++ */ ++ dev_err(kbdev->dev, "Flush for GPU page table update did not complete. Issueing GPU soft-reset to recover\n"); ++ ++ if (kbase_prepare_to_reset_gpu(kbdev)) ++ kbase_reset_gpu(kbdev); ++ } ++ ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ /* AS transaction end */ ++ ++ kbase_pm_context_idle(kbdev); ++} ++ ++static void kbase_mmu_flush_invalidate_no_ctx(struct kbase_device *kbdev, ++ u64 vpfn, size_t nr, bool sync, int as_nr) ++{ ++ /* Skip if there is nothing to do */ ++ if (nr) { ++ kbase_mmu_flush_invalidate_as(kbdev, &kbdev->as[as_nr], vpfn, ++ nr, sync); ++ } ++} ++ ++static void kbase_mmu_flush_invalidate(struct kbase_context *kctx, ++ u64 vpfn, size_t nr, bool sync) ++{ ++ struct kbase_device *kbdev; ++ bool ctx_is_in_runpool; ++ ++ /* Early out if there is nothing to do */ ++ if (nr == 0) ++ return; ++ ++ kbdev = kctx->kbdev; ++#if !MALI_USE_CSF ++ mutex_lock(&kbdev->js_data.queue_mutex); ++#endif /* !MALI_USE_CSF */ ++ ctx_is_in_runpool = kbase_ctx_sched_inc_refcount(kctx); ++#if !MALI_USE_CSF ++ mutex_unlock(&kbdev->js_data.queue_mutex); ++#endif /* !MALI_USE_CSF */ ++ ++ if (ctx_is_in_runpool) { ++ KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); ++ ++ kbase_mmu_flush_invalidate_as(kbdev, &kbdev->as[kctx->as_nr], ++ vpfn, nr, sync); ++ ++ release_ctx(kbdev, kctx); ++ } ++} ++ ++void kbase_mmu_update(struct kbase_device *kbdev, ++ struct kbase_mmu_table *mmut, ++ int as_nr) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ lockdep_assert_held(&kbdev->mmu_hw_mutex); ++ KBASE_DEBUG_ASSERT(as_nr != KBASEP_AS_NR_INVALID); ++ ++ kbdev->mmu_mode->update(kbdev, mmut, as_nr); ++} ++KBASE_EXPORT_TEST_API(kbase_mmu_update); ++ ++void kbase_mmu_disable_as(struct kbase_device *kbdev, int as_nr) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ lockdep_assert_held(&kbdev->mmu_hw_mutex); ++ ++ kbdev->mmu_mode->disable_as(kbdev, as_nr); ++} ++ ++void kbase_mmu_disable(struct kbase_context *kctx) ++{ ++ /* ASSERT that the context has a valid as_nr, which is only the case ++ * when it's scheduled in. ++ * ++ * as_nr won't change because the caller has the hwaccess_lock ++ */ ++ KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); ++ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ ++ /* ++ * The address space is being disabled, drain all knowledge of it out ++ * from the caches as pages and page tables might be freed after this. ++ * ++ * The job scheduler code will already be holding the locks and context ++ * so just do the flush. ++ */ ++ kbase_mmu_flush_invalidate_noretain(kctx, 0, ~0, true); ++ ++ kctx->kbdev->mmu_mode->disable_as(kctx->kbdev, kctx->as_nr); ++} ++KBASE_EXPORT_TEST_API(kbase_mmu_disable); ++ ++/* ++ * We actually only discard the ATE, and not the page table ++ * pages. There is a potential DoS here, as we'll leak memory by ++ * having PTEs that are potentially unused. Will require physical ++ * page accounting, so MMU pages are part of the process allocation. ++ * ++ * IMPORTANT: This uses kbasep_js_runpool_release_ctx() when the context is ++ * currently scheduled into the runpool, and so potentially uses a lot of locks. ++ * These locks must be taken in the correct order with respect to others ++ * already held by the caller. Refer to kbasep_js_runpool_release_ctx() for more ++ * information. ++ */ ++int kbase_mmu_teardown_pages(struct kbase_device *kbdev, ++ struct kbase_mmu_table *mmut, u64 vpfn, size_t nr, int as_nr) ++{ ++ phys_addr_t pgd; ++ u64 start_vpfn = vpfn; ++ size_t requested_nr = nr; ++ struct kbase_mmu_mode const *mmu_mode; ++ int err = -EFAULT; ++ ++ if (nr == 0) { ++ /* early out if nothing to do */ ++ return 0; ++ } ++ ++ mutex_lock(&mmut->mmu_lock); ++ ++ mmu_mode = kbdev->mmu_mode; ++ ++ while (nr) { ++ unsigned int i; ++ unsigned int index = vpfn & 0x1FF; ++ unsigned int count = KBASE_MMU_PAGE_ENTRIES - index; ++ unsigned int pcount; ++ int level; ++ u64 *page; ++ ++ if (count > nr) ++ count = nr; ++ ++ /* need to check if this is a 2MB or a 4kB page */ ++ pgd = mmut->pgd; ++ ++ for (level = MIDGARD_MMU_TOPLEVEL; ++ level <= MIDGARD_MMU_BOTTOMLEVEL; level++) { ++ phys_addr_t next_pgd; ++ ++ index = (vpfn >> ((3 - level) * 9)) & 0x1FF; ++ page = kmap(phys_to_page(pgd)); ++ if (mmu_mode->ate_is_valid(page[index], level)) ++ break; /* keep the mapping */ ++ else if (!mmu_mode->pte_is_valid(page[index], level)) { ++ /* nothing here, advance */ ++ switch (level) { ++ case MIDGARD_MMU_LEVEL(0): ++ count = 134217728; ++ break; ++ case MIDGARD_MMU_LEVEL(1): ++ count = 262144; ++ break; ++ case MIDGARD_MMU_LEVEL(2): ++ count = 512; ++ break; ++ case MIDGARD_MMU_LEVEL(3): ++ count = 1; ++ break; ++ } ++ if (count > nr) ++ count = nr; ++ goto next; ++ } ++ next_pgd = mmu_mode->pte_to_phy_addr(page[index]); ++ kunmap(phys_to_page(pgd)); ++ pgd = next_pgd; ++ } ++ ++ switch (level) { ++ case MIDGARD_MMU_LEVEL(0): ++ case MIDGARD_MMU_LEVEL(1): ++ dev_warn(kbdev->dev, ++ "%s: No support for ATEs at level %d\n", ++ __func__, level); ++ kunmap(phys_to_page(pgd)); ++ goto out; ++ case MIDGARD_MMU_LEVEL(2): ++ /* can only teardown if count >= 512 */ ++ if (count >= 512) { ++ pcount = 1; ++ } else { ++ dev_warn(kbdev->dev, ++ "%s: limiting teardown as it tries to do a partial 2MB teardown, need 512, but have %d to tear down\n", ++ __func__, count); ++ pcount = 0; ++ } ++ break; ++ case MIDGARD_MMU_BOTTOMLEVEL: ++ /* page count is the same as the logical count */ ++ pcount = count; ++ break; ++ default: ++ dev_err(kbdev->dev, ++ "%s: found non-mapped memory, early out\n", ++ __func__); ++ vpfn += count; ++ nr -= count; ++ continue; ++ } ++ ++ /* Invalidate the entries we added */ ++ for (i = 0; i < pcount; i++) ++ mmu_mode->entry_invalidate(&page[index + i]); ++ ++ kbase_mmu_sync_pgd(kbdev, ++ kbase_dma_addr(phys_to_page(pgd)) + ++ 8 * index, 8*pcount); ++ ++next: ++ kunmap(phys_to_page(pgd)); ++ vpfn += count; ++ nr -= count; ++ } ++ err = 0; ++out: ++ mutex_unlock(&mmut->mmu_lock); ++ ++ if (mmut->kctx) ++ kbase_mmu_flush_invalidate(mmut->kctx, start_vpfn, requested_nr, ++ true); ++ else ++ kbase_mmu_flush_invalidate_no_ctx(kbdev, start_vpfn, requested_nr, ++ true, as_nr); ++ ++ return err; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_mmu_teardown_pages); ++ ++/** ++ * kbase_mmu_update_pages_no_flush() - Update page table entries on the GPU ++ * ++ * This will update page table entries that already exist on the GPU based on ++ * the new flags that are passed. It is used as a response to the changes of ++ * the memory attributes ++ * ++ * The caller is responsible for validating the memory attributes ++ * ++ * @kctx: Kbase context ++ * @vpfn: Virtual PFN (Page Frame Number) of the first page to update ++ * @phys: Tagged physical addresses of the physical pages to replace the ++ * current mappings ++ * @nr: Number of pages to update ++ * @flags: Flags ++ * @group_id: The physical memory group in which the page was allocated. ++ * Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). ++ */ ++static int kbase_mmu_update_pages_no_flush(struct kbase_context *kctx, u64 vpfn, ++ struct tagged_addr *phys, size_t nr, ++ unsigned long flags, int const group_id) ++{ ++ phys_addr_t pgd; ++ u64 *pgd_page; ++ int err; ++ struct kbase_device *kbdev; ++ ++ if (WARN_ON(kctx == NULL)) ++ return -EINVAL; ++ ++ KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE)); ++ ++ /* Early out if there is nothing to do */ ++ if (nr == 0) ++ return 0; ++ ++ mutex_lock(&kctx->mmu.mmu_lock); ++ ++ kbdev = kctx->kbdev; ++ ++ while (nr) { ++ unsigned int i; ++ unsigned int index = vpfn & 0x1FF; ++ size_t count = KBASE_MMU_PAGE_ENTRIES - index; ++ struct page *p; ++ ++ if (count > nr) ++ count = nr; ++ ++ do { ++ err = mmu_get_bottom_pgd(kbdev, &kctx->mmu, ++ vpfn, &pgd); ++ if (err != -ENOMEM) ++ break; ++ /* Fill the memory pool with enough pages for ++ * the page walk to succeed ++ */ ++ mutex_unlock(&kctx->mmu.mmu_lock); ++ err = kbase_mem_pool_grow( ++ &kbdev->mem_pools.small[ ++ kctx->mmu.group_id], ++ MIDGARD_MMU_BOTTOMLEVEL); ++ mutex_lock(&kctx->mmu.mmu_lock); ++ } while (!err); ++ if (err) { ++ dev_warn(kbdev->dev, ++ "mmu_get_bottom_pgd failure\n"); ++ goto fail_unlock; ++ } ++ ++ p = pfn_to_page(PFN_DOWN(pgd)); ++ pgd_page = kmap(p); ++ if (!pgd_page) { ++ dev_warn(kbdev->dev, "kmap failure\n"); ++ err = -ENOMEM; ++ goto fail_unlock; ++ } ++ ++ for (i = 0; i < count; i++) ++ pgd_page[index + i] = kbase_mmu_create_ate(kbdev, ++ phys[i], flags, MIDGARD_MMU_BOTTOMLEVEL, ++ group_id); ++ ++ phys += count; ++ vpfn += count; ++ nr -= count; ++ ++ kbase_mmu_sync_pgd(kbdev, ++ kbase_dma_addr(p) + (index * sizeof(u64)), ++ count * sizeof(u64)); ++ ++ kunmap(pfn_to_page(PFN_DOWN(pgd))); ++ } ++ ++ mutex_unlock(&kctx->mmu.mmu_lock); ++ return 0; ++ ++fail_unlock: ++ mutex_unlock(&kctx->mmu.mmu_lock); ++ return err; ++} ++ ++int kbase_mmu_update_pages(struct kbase_context *kctx, u64 vpfn, ++ struct tagged_addr *phys, size_t nr, ++ unsigned long flags, int const group_id) ++{ ++ int err; ++ ++ err = kbase_mmu_update_pages_no_flush(kctx, vpfn, phys, nr, flags, ++ group_id); ++ kbase_mmu_flush_invalidate(kctx, vpfn, nr, true); ++ return err; ++} ++ ++static void mmu_teardown_level(struct kbase_device *kbdev, ++ struct kbase_mmu_table *mmut, phys_addr_t pgd, ++ int level, u64 *pgd_page_buffer) ++{ ++ phys_addr_t target_pgd; ++ struct page *p; ++ u64 *pgd_page; ++ int i; ++ struct kbase_mmu_mode const *mmu_mode; ++ ++ lockdep_assert_held(&mmut->mmu_lock); ++ ++ pgd_page = kmap_atomic(pfn_to_page(PFN_DOWN(pgd))); ++ /* kmap_atomic should NEVER fail. */ ++ if (WARN_ON(pgd_page == NULL)) ++ return; ++ /* Copy the page to our preallocated buffer so that we can minimize ++ * kmap_atomic usage ++ */ ++ memcpy(pgd_page_buffer, pgd_page, PAGE_SIZE); ++ kunmap_atomic(pgd_page); ++ pgd_page = pgd_page_buffer; ++ ++ mmu_mode = kbdev->mmu_mode; ++ ++ for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++) { ++ target_pgd = mmu_mode->pte_to_phy_addr(pgd_page[i]); ++ ++ if (target_pgd) { ++ if (mmu_mode->pte_is_valid(pgd_page[i], level)) { ++ mmu_teardown_level(kbdev, mmut, ++ target_pgd, ++ level + 1, ++ pgd_page_buffer + ++ (PAGE_SIZE / sizeof(u64))); ++ } ++ } ++ } ++ ++ p = pfn_to_page(PFN_DOWN(pgd)); ++ ++ kbase_mem_pool_free(&kbdev->mem_pools.small[mmut->group_id], ++ p, true); ++ ++ atomic_sub(1, &kbdev->memdev.used_pages); ++ ++ /* If MMU tables belong to a context then pages will have been accounted ++ * against it, so we must decrement the usage counts here. ++ */ ++ if (mmut->kctx) { ++ kbase_process_page_usage_dec(mmut->kctx, 1); ++ atomic_sub(1, &mmut->kctx->used_pages); ++ } ++ ++ kbase_trace_gpu_mem_usage_dec(kbdev, mmut->kctx, 1); ++} ++ ++int kbase_mmu_init(struct kbase_device *const kbdev, ++ struct kbase_mmu_table *const mmut, struct kbase_context *const kctx, ++ int const group_id) ++{ ++ if (WARN_ON(group_id >= MEMORY_GROUP_MANAGER_NR_GROUPS) || ++ WARN_ON(group_id < 0)) ++ return -EINVAL; ++ ++ mmut->group_id = group_id; ++ mutex_init(&mmut->mmu_lock); ++ mmut->kctx = kctx; ++ ++ /* Preallocate MMU depth of four pages for mmu_teardown_level to use */ ++ mmut->mmu_teardown_pages = kmalloc(PAGE_SIZE * 4, GFP_KERNEL); ++ ++ if (mmut->mmu_teardown_pages == NULL) ++ return -ENOMEM; ++ ++ mmut->pgd = 0; ++ /* We allocate pages into the kbdev memory pool, then ++ * kbase_mmu_alloc_pgd will allocate out of that pool. This is done to ++ * avoid allocations from the kernel happening with the lock held. ++ */ ++ while (!mmut->pgd) { ++ int err; ++ ++ err = kbase_mem_pool_grow( ++ &kbdev->mem_pools.small[mmut->group_id], ++ MIDGARD_MMU_BOTTOMLEVEL); ++ if (err) { ++ kbase_mmu_term(kbdev, mmut); ++ return -ENOMEM; ++ } ++ ++ mutex_lock(&mmut->mmu_lock); ++ mmut->pgd = kbase_mmu_alloc_pgd(kbdev, mmut); ++ mutex_unlock(&mmut->mmu_lock); ++ } ++ ++ return 0; ++} ++ ++void kbase_mmu_term(struct kbase_device *kbdev, struct kbase_mmu_table *mmut) ++{ ++ if (mmut->pgd) { ++ mutex_lock(&mmut->mmu_lock); ++ mmu_teardown_level(kbdev, mmut, mmut->pgd, MIDGARD_MMU_TOPLEVEL, ++ mmut->mmu_teardown_pages); ++ mutex_unlock(&mmut->mmu_lock); ++ ++ if (mmut->kctx) ++ KBASE_TLSTREAM_AUX_PAGESALLOC(kbdev, mmut->kctx->id, 0); ++ } ++ ++ kfree(mmut->mmu_teardown_pages); ++ mutex_destroy(&mmut->mmu_lock); ++} ++ ++void kbase_mmu_as_term(struct kbase_device *kbdev, int i) ++{ ++ destroy_workqueue(kbdev->as[i].pf_wq); ++} ++ ++static size_t kbasep_mmu_dump_level(struct kbase_context *kctx, phys_addr_t pgd, ++ int level, char ** const buffer, size_t *size_left) ++{ ++ phys_addr_t target_pgd; ++ u64 *pgd_page; ++ int i; ++ size_t size = KBASE_MMU_PAGE_ENTRIES * sizeof(u64) + sizeof(u64); ++ size_t dump_size; ++ struct kbase_device *kbdev; ++ struct kbase_mmu_mode const *mmu_mode; ++ ++ if (WARN_ON(kctx == NULL)) ++ return 0; ++ lockdep_assert_held(&kctx->mmu.mmu_lock); ++ ++ kbdev = kctx->kbdev; ++ mmu_mode = kbdev->mmu_mode; ++ ++ pgd_page = kmap(pfn_to_page(PFN_DOWN(pgd))); ++ if (!pgd_page) { ++ dev_warn(kbdev->dev, "%s: kmap failure\n", __func__); ++ return 0; ++ } ++ ++ if (*size_left >= size) { ++ /* A modified physical address that contains ++ * the page table level ++ */ ++ u64 m_pgd = pgd | level; ++ ++ /* Put the modified physical address in the output buffer */ ++ memcpy(*buffer, &m_pgd, sizeof(m_pgd)); ++ *buffer += sizeof(m_pgd); ++ ++ /* Followed by the page table itself */ ++ memcpy(*buffer, pgd_page, sizeof(u64) * KBASE_MMU_PAGE_ENTRIES); ++ *buffer += sizeof(u64) * KBASE_MMU_PAGE_ENTRIES; ++ ++ *size_left -= size; ++ } ++ ++ if (level < MIDGARD_MMU_BOTTOMLEVEL) { ++ for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++) { ++ if (mmu_mode->pte_is_valid(pgd_page[i], level)) { ++ target_pgd = mmu_mode->pte_to_phy_addr( ++ pgd_page[i]); ++ ++ dump_size = kbasep_mmu_dump_level(kctx, ++ target_pgd, level + 1, ++ buffer, size_left); ++ if (!dump_size) { ++ kunmap(pfn_to_page(PFN_DOWN(pgd))); ++ return 0; ++ } ++ size += dump_size; ++ } ++ } ++ } ++ ++ kunmap(pfn_to_page(PFN_DOWN(pgd))); ++ ++ return size; ++} ++ ++void *kbase_mmu_dump(struct kbase_context *kctx, int nr_pages) ++{ ++ void *kaddr; ++ size_t size_left; ++ ++ KBASE_DEBUG_ASSERT(kctx); ++ ++ if (nr_pages == 0) { ++ /* can't dump in a 0 sized buffer, early out */ ++ return NULL; ++ } ++ ++ size_left = nr_pages * PAGE_SIZE; ++ ++ if (WARN_ON(size_left == 0)) ++ return NULL; ++ kaddr = vmalloc_user(size_left); ++ ++ mutex_lock(&kctx->mmu.mmu_lock); ++ ++ if (kaddr) { ++ u64 end_marker = 0xFFULL; ++ char *buffer; ++ char *mmu_dump_buffer; ++ u64 config[3]; ++ size_t dump_size, size = 0; ++ struct kbase_mmu_setup as_setup; ++ ++ buffer = (char *)kaddr; ++ mmu_dump_buffer = buffer; ++ ++ kctx->kbdev->mmu_mode->get_as_setup(&kctx->mmu, ++ &as_setup); ++ config[0] = as_setup.transtab; ++ config[1] = as_setup.memattr; ++ config[2] = as_setup.transcfg; ++ memcpy(buffer, &config, sizeof(config)); ++ mmu_dump_buffer += sizeof(config); ++ size_left -= sizeof(config); ++ size += sizeof(config); ++ ++ dump_size = kbasep_mmu_dump_level(kctx, ++ kctx->mmu.pgd, ++ MIDGARD_MMU_TOPLEVEL, ++ &mmu_dump_buffer, ++ &size_left); ++ ++ if (!dump_size) ++ goto fail_free; ++ ++ size += dump_size; ++ ++ /* Add on the size for the end marker */ ++ size += sizeof(u64); ++ ++ if (size > (nr_pages * PAGE_SIZE)) { ++ /* The buffer isn't big enough - free the memory and ++ * return failure ++ */ ++ goto fail_free; ++ } ++ ++ /* Add the end marker */ ++ memcpy(mmu_dump_buffer, &end_marker, sizeof(u64)); ++ } ++ ++ mutex_unlock(&kctx->mmu.mmu_lock); ++ return kaddr; ++ ++fail_free: ++ vfree(kaddr); ++ mutex_unlock(&kctx->mmu.mmu_lock); ++ return NULL; ++} ++KBASE_EXPORT_TEST_API(kbase_mmu_dump); ++ ++void kbase_mmu_bus_fault_worker(struct work_struct *data) ++{ ++ struct kbase_as *faulting_as; ++ int as_no; ++ struct kbase_context *kctx; ++ struct kbase_device *kbdev; ++ struct kbase_fault *fault; ++ ++ faulting_as = container_of(data, struct kbase_as, work_busfault); ++ fault = &faulting_as->bf_data; ++ ++ /* Ensure that any pending page fault worker has completed */ ++ flush_work(&faulting_as->work_pagefault); ++ ++ as_no = faulting_as->number; ++ ++ kbdev = container_of(faulting_as, struct kbase_device, as[as_no]); ++ ++ /* Grab the context, already refcounted in kbase_mmu_interrupt() on ++ * flagging of the bus-fault. Therefore, it cannot be scheduled out of ++ * this AS until we explicitly release it ++ */ ++ kctx = kbase_ctx_sched_as_to_ctx(kbdev, as_no); ++ if (!kctx) { ++ atomic_dec(&kbdev->faults_pending); ++ return; ++ } ++ ++#ifdef CONFIG_MALI_ARBITER_SUPPORT ++ /* check if we still have GPU */ ++ if (unlikely(kbase_is_gpu_removed(kbdev))) { ++ dev_dbg(kbdev->dev, ++ "%s: GPU has been removed\n", __func__); ++ release_ctx(kbdev, kctx); ++ atomic_dec(&kbdev->faults_pending); ++ return; ++ } ++#endif ++ ++ if (unlikely(fault->protected_mode)) { ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Permission failure", fault); ++ kbase_mmu_hw_clear_fault(kbdev, faulting_as, ++ KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); ++ release_ctx(kbdev, kctx); ++ atomic_dec(&kbdev->faults_pending); ++ return; ++ ++ } ++ ++ /* NOTE: If GPU already powered off for suspend, ++ * we don't need to switch to unmapped ++ */ ++ if (!kbase_pm_context_active_handle_suspend(kbdev, ++ KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) { ++ kbase_gpu_report_bus_fault_and_kill(kctx, faulting_as, fault); ++ kbase_pm_context_idle(kbdev); ++ } ++ ++ release_ctx(kbdev, kctx); ++ ++ atomic_dec(&kbdev->faults_pending); ++} ++ ++void kbase_flush_mmu_wqs(struct kbase_device *kbdev) ++{ ++ int i; ++ ++ for (i = 0; i < kbdev->nr_hw_address_spaces; i++) { ++ struct kbase_as *as = &kbdev->as[i]; ++ ++ flush_workqueue(as->pf_wq); ++ } ++} +diff --git a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu.h b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu.h +new file mode 100755 +index 000000000000..f2613e881dac +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu.h +@@ -0,0 +1,156 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_MMU_H_ ++#define _KBASE_MMU_H_ ++ ++/** ++ * kbase_mmu_as_init() - Initialising GPU address space object. ++ * ++ * This is called from device probe to initialise an address space object ++ * of the device. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer). ++ * @i: Array index of address space object. ++ * ++ * Return: 0 on success and non-zero value on failure. ++ */ ++int kbase_mmu_as_init(struct kbase_device *kbdev, int i); ++ ++/** ++ * kbase_mmu_as_term() - Terminate address space object. ++ * ++ * This is called upon device termination to destroy ++ * the address space object of the device. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer). ++ * @i: Array index of address space object. ++ */ ++void kbase_mmu_as_term(struct kbase_device *kbdev, int i); ++ ++/** ++ * kbase_mmu_init - Initialise an object representing GPU page tables ++ * ++ * The structure should be terminated using kbase_mmu_term() ++ * ++ * @kbdev: Instance of GPU platform device, allocated from the probe method. ++ * @mmut: GPU page tables to be initialized. ++ * @kctx: Optional kbase context, may be NULL if this set of MMU tables ++ * is not associated with a context. ++ * @group_id: The physical group ID from which to allocate GPU page tables. ++ * Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). ++ * ++ * Return: 0 if successful, otherwise a negative error code. ++ */ ++int kbase_mmu_init(struct kbase_device *kbdev, struct kbase_mmu_table *mmut, ++ struct kbase_context *kctx, int group_id); ++ ++/** ++ * kbase_mmu_interrupt - Process an MMU interrupt. ++ * ++ * Process the MMU interrupt that was reported by the &kbase_device. ++ * ++ * @kbdev: Pointer to the kbase device for which the interrupt happened. ++ * @irq_stat: Value of the MMU_IRQ_STATUS register. ++ */ ++void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat); ++ ++/** ++ * kbase_mmu_term - Terminate an object representing GPU page tables ++ * ++ * This will free any page tables that have been allocated ++ * ++ * @kbdev: Instance of GPU platform device, allocated from the probe method. ++ * @mmut: GPU page tables to be destroyed. ++ */ ++void kbase_mmu_term(struct kbase_device *kbdev, struct kbase_mmu_table *mmut); ++ ++/** ++ * kbase_mmu_create_ate - Create an address translation entry ++ * ++ * @kbdev: Instance of GPU platform device, allocated from the probe method. ++ * @phy: Physical address of the page to be mapped for GPU access. ++ * @flags: Bitmask of attributes of the GPU memory region being mapped. ++ * @level: Page table level for which to build an address translation entry. ++ * @group_id: The physical memory group in which the page was allocated. ++ * Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). ++ * ++ * This function creates an address translation entry to encode the physical ++ * address of a page to be mapped for access by the GPU, along with any extra ++ * attributes required for the GPU memory region. ++ * ++ * Return: An address translation entry, either in LPAE or AArch64 format ++ * (depending on the driver's configuration). ++ */ ++u64 kbase_mmu_create_ate(struct kbase_device *kbdev, ++ struct tagged_addr phy, unsigned long flags, int level, int group_id); ++ ++int kbase_mmu_insert_pages_no_flush(struct kbase_device *kbdev, ++ struct kbase_mmu_table *mmut, ++ const u64 start_vpfn, ++ struct tagged_addr *phys, size_t nr, ++ unsigned long flags, int group_id); ++int kbase_mmu_insert_pages(struct kbase_device *kbdev, ++ struct kbase_mmu_table *mmut, u64 vpfn, ++ struct tagged_addr *phys, size_t nr, ++ unsigned long flags, int as_nr, int group_id); ++int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 vpfn, ++ struct tagged_addr phys, size_t nr, ++ unsigned long flags, int group_id); ++ ++int kbase_mmu_teardown_pages(struct kbase_device *kbdev, ++ struct kbase_mmu_table *mmut, u64 vpfn, ++ size_t nr, int as_nr); ++int kbase_mmu_update_pages(struct kbase_context *kctx, u64 vpfn, ++ struct tagged_addr *phys, size_t nr, ++ unsigned long flags, int const group_id); ++ ++/** ++ * kbase_mmu_bus_fault_interrupt - Process a bus fault interrupt. ++ * ++ * Process the bus fault interrupt that was reported for a particular GPU ++ * address space. ++ * ++ * @kbdev: Pointer to the kbase device for which bus fault was reported. ++ * @status: Value of the GPU_FAULTSTATUS register. ++ * @as_nr: GPU address space for which the bus fault occurred. ++ * ++ * Return: zero if the operation was successful, non-zero otherwise. ++ */ ++int kbase_mmu_bus_fault_interrupt(struct kbase_device *kbdev, u32 status, ++ u32 as_nr); ++ ++/** ++ * kbase_mmu_gpu_fault_interrupt() - Report a GPU fault. ++ * @kbdev: Kbase device pointer ++ * @status: GPU fault status ++ * @as_nr: Faulty address space ++ * @address: GPU fault address ++ * @as_valid: true if address space is valid ++ * ++ * This function builds GPU fault information to submit a work ++ * for reporting the details of the fault. ++ */ ++void kbase_mmu_gpu_fault_interrupt(struct kbase_device *kbdev, u32 status, ++ u32 as_nr, u64 address, bool as_valid); ++ ++#endif /* _KBASE_MMU_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_hw.h b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_hw.h +new file mode 100755 +index 000000000000..e6eef86d7ac0 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_hw.h +@@ -0,0 +1,107 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2015, 2018-2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/** ++ * DOC: Interface file for accessing MMU hardware functionality ++ * ++ * This module provides an abstraction for accessing the functionality provided ++ * by the midgard MMU and thus allows all MMU HW access to be contained within ++ * one common place and allows for different backends (implementations) to ++ * be provided. ++ */ ++ ++#ifndef _KBASE_MMU_HW_H_ ++#define _KBASE_MMU_HW_H_ ++ ++/* Forward declarations */ ++struct kbase_device; ++struct kbase_as; ++struct kbase_context; ++ ++/** ++ * enum kbase_mmu_fault_type - MMU fault type descriptor. ++ */ ++enum kbase_mmu_fault_type { ++ KBASE_MMU_FAULT_TYPE_UNKNOWN = 0, ++ KBASE_MMU_FAULT_TYPE_PAGE, ++ KBASE_MMU_FAULT_TYPE_BUS, ++ KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED, ++ KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED ++}; ++ ++/** ++ * kbase_mmu_hw_configure - Configure an address space for use. ++ * @kbdev: kbase device to configure. ++ * @as: address space to configure. ++ * ++ * Configure the MMU using the address space details setup in the ++ * kbase_context structure. ++ */ ++void kbase_mmu_hw_configure(struct kbase_device *kbdev, ++ struct kbase_as *as); ++ ++/** ++ * kbase_mmu_hw_do_operation - Issue an operation to the MMU. ++ * @kbdev: kbase device to issue the MMU operation on. ++ * @as: address space to issue the MMU operation on. ++ * @vpfn: MMU Virtual Page Frame Number to start the operation on. ++ * @nr: Number of pages to work on. ++ * @type: Operation type (written to ASn_COMMAND). ++ * @handling_irq: Is this operation being called during the handling ++ * of an interrupt? ++ * ++ * Issue an operation (MMU invalidate, MMU flush, etc) on the address space that ++ * is associated with the provided kbase_context over the specified range ++ * ++ * Return: Zero if the operation was successful, non-zero otherwise. ++ */ ++int kbase_mmu_hw_do_operation(struct kbase_device *kbdev, struct kbase_as *as, ++ u64 vpfn, u32 nr, u32 type, ++ unsigned int handling_irq); ++ ++/** ++ * kbase_mmu_hw_clear_fault - Clear a fault that has been previously reported by ++ * the MMU. ++ * @kbdev: kbase device to clear the fault from. ++ * @as: address space to clear the fault from. ++ * @type: The type of fault that needs to be cleared. ++ * ++ * Clear a bus error or page fault that has been reported by the MMU. ++ */ ++void kbase_mmu_hw_clear_fault(struct kbase_device *kbdev, struct kbase_as *as, ++ enum kbase_mmu_fault_type type); ++ ++/** ++ * kbase_mmu_hw_enable_fault - Enable fault that has been previously reported by ++ * the MMU. ++ * @kbdev: kbase device to again enable the fault from. ++ * @as: address space to again enable the fault from. ++ * @type: The type of fault that needs to be enabled again. ++ * ++ * After a page fault or bus error has been reported by the MMU these ++ * will be disabled. After these are handled this function needs to be ++ * called to enable the page fault or bus error fault again. ++ */ ++void kbase_mmu_hw_enable_fault(struct kbase_device *kbdev, struct kbase_as *as, ++ enum kbase_mmu_fault_type type); ++ ++#endif /* _KBASE_MMU_HW_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_hw_direct.c b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_hw_direct.c +new file mode 100755 +index 000000000000..a820ab24ac05 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_hw_direct.c +@@ -0,0 +1,274 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++/** ++ * lock_region() - Generate lockaddr to lock memory region in MMU ++ * @pfn: Starting page frame number of the region to lock ++ * @num_pages: Number of pages to lock. It must be greater than 0. ++ * @lockaddr: Address and size of memory region to lock ++ * ++ * The lockaddr value is a combination of the starting address and ++ * the size of the region that encompasses all the memory pages to lock. ++ * ++ * The size is expressed as a logarithm: it is represented in a way ++ * that is compatible with the HW specification and it also determines ++ * how many of the lowest bits of the address are cleared. ++ * ++ * Return: 0 if success, or an error code on failure. ++ */ ++static int lock_region(u64 pfn, u32 num_pages, u64 *lockaddr) ++{ ++ const u64 lockaddr_base = pfn << PAGE_SHIFT; ++ u64 lockaddr_size_log2, region_frame_number_start, ++ region_frame_number_end; ++ ++ if (num_pages == 0) ++ return -EINVAL; ++ ++ /* The size is expressed as a logarithm and should take into account ++ * the possibility that some pages might spill into the next region. ++ */ ++ lockaddr_size_log2 = fls(num_pages) + PAGE_SHIFT - 1; ++ ++ /* Round up if the number of pages is not a power of 2. */ ++ if (num_pages != ((u32)1 << (lockaddr_size_log2 - PAGE_SHIFT))) ++ lockaddr_size_log2 += 1; ++ ++ /* Round up if some memory pages spill into the next region. */ ++ region_frame_number_start = pfn >> (lockaddr_size_log2 - PAGE_SHIFT); ++ region_frame_number_end = ++ (pfn + num_pages - 1) >> (lockaddr_size_log2 - PAGE_SHIFT); ++ ++ if (region_frame_number_start < region_frame_number_end) ++ lockaddr_size_log2 += 1; ++ ++ /* Represent the size according to the HW specification. */ ++ lockaddr_size_log2 = MAX(lockaddr_size_log2, ++ KBASE_LOCK_REGION_MIN_SIZE_LOG2); ++ ++ if (lockaddr_size_log2 > KBASE_LOCK_REGION_MAX_SIZE_LOG2) ++ return -EINVAL; ++ ++ /* The lowest bits are cleared and then set to size - 1 to represent ++ * the size in a way that is compatible with the HW specification. ++ */ ++ *lockaddr = lockaddr_base & ~((1ull << lockaddr_size_log2) - 1); ++ *lockaddr |= lockaddr_size_log2 - 1; ++ ++ return 0; ++} ++ ++static int wait_ready(struct kbase_device *kbdev, ++ unsigned int as_nr) ++{ ++ unsigned int max_loops = KBASE_AS_INACTIVE_MAX_LOOPS; ++ u32 val = kbase_reg_read(kbdev, MMU_AS_REG(as_nr, AS_STATUS)); ++ ++ /* Wait for the MMU status to indicate there is no active command, in ++ * case one is pending. Do not log remaining register accesses. ++ */ ++ while (--max_loops && (val & AS_STATUS_AS_ACTIVE)) ++ val = kbase_reg_read(kbdev, MMU_AS_REG(as_nr, AS_STATUS)); ++ ++ if (max_loops == 0) { ++ dev_err(kbdev->dev, "AS_ACTIVE bit stuck, might be caused by slow/unstable GPU clock or possible faulty FPGA connector\n"); ++ return -1; ++ } ++ ++ /* If waiting in loop was performed, log last read value. */ ++ if (KBASE_AS_INACTIVE_MAX_LOOPS - 1 > max_loops) ++ kbase_reg_read(kbdev, MMU_AS_REG(as_nr, AS_STATUS)); ++ ++ return 0; ++} ++ ++static int write_cmd(struct kbase_device *kbdev, int as_nr, u32 cmd) ++{ ++ int status; ++ ++ /* write AS_COMMAND when MMU is ready to accept another command */ ++ status = wait_ready(kbdev, as_nr); ++ if (status == 0) ++ kbase_reg_write(kbdev, MMU_AS_REG(as_nr, AS_COMMAND), cmd); ++ ++ return status; ++} ++ ++void kbase_mmu_hw_configure(struct kbase_device *kbdev, struct kbase_as *as) ++{ ++ struct kbase_mmu_setup *current_setup = &as->current_setup; ++ u64 transcfg = 0; ++ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) { ++ transcfg = current_setup->transcfg; ++ ++ /* Set flag AS_TRANSCFG_PTW_MEMATTR_WRITE_BACK ++ * Clear PTW_MEMATTR bits ++ */ ++ transcfg &= ~AS_TRANSCFG_PTW_MEMATTR_MASK; ++ /* Enable correct PTW_MEMATTR bits */ ++ transcfg |= AS_TRANSCFG_PTW_MEMATTR_WRITE_BACK; ++ /* Ensure page-tables reads use read-allocate cache-policy in ++ * the L2 ++ */ ++ transcfg |= AS_TRANSCFG_R_ALLOCATE; ++ ++ if (kbdev->system_coherency == COHERENCY_ACE) { ++ /* Set flag AS_TRANSCFG_PTW_SH_OS (outer shareable) ++ * Clear PTW_SH bits ++ */ ++ transcfg = (transcfg & ~AS_TRANSCFG_PTW_SH_MASK); ++ /* Enable correct PTW_SH bits */ ++ transcfg = (transcfg | AS_TRANSCFG_PTW_SH_OS); ++ } ++ ++ kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSCFG_LO), ++ transcfg); ++ kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSCFG_HI), ++ (transcfg >> 32) & 0xFFFFFFFFUL); ++ } else { ++ if (kbdev->system_coherency == COHERENCY_ACE) ++ current_setup->transtab |= AS_TRANSTAB_LPAE_SHARE_OUTER; ++ } ++ ++ kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSTAB_LO), ++ current_setup->transtab & 0xFFFFFFFFUL); ++ kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSTAB_HI), ++ (current_setup->transtab >> 32) & 0xFFFFFFFFUL); ++ ++ kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_MEMATTR_LO), ++ current_setup->memattr & 0xFFFFFFFFUL); ++ kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_MEMATTR_HI), ++ (current_setup->memattr >> 32) & 0xFFFFFFFFUL); ++ ++ KBASE_TLSTREAM_TL_ATTRIB_AS_CONFIG(kbdev, as, ++ current_setup->transtab, ++ current_setup->memattr, ++ transcfg); ++ ++ write_cmd(kbdev, as->number, AS_COMMAND_UPDATE); ++} ++ ++int kbase_mmu_hw_do_operation(struct kbase_device *kbdev, struct kbase_as *as, ++ u64 vpfn, u32 nr, u32 op, ++ unsigned int handling_irq) ++{ ++ int ret; ++ ++ lockdep_assert_held(&kbdev->mmu_hw_mutex); ++ ++ if (op == AS_COMMAND_UNLOCK) { ++ /* Unlock doesn't require a lock first */ ++ ret = write_cmd(kbdev, as->number, AS_COMMAND_UNLOCK); ++ } else { ++ u64 lock_addr; ++ ++ ret = lock_region(vpfn, nr, &lock_addr); ++ ++ if (!ret) { ++ /* Lock the region that needs to be updated */ ++ kbase_reg_write(kbdev, ++ MMU_AS_REG(as->number, AS_LOCKADDR_LO), ++ lock_addr & 0xFFFFFFFFUL); ++ kbase_reg_write(kbdev, ++ MMU_AS_REG(as->number, AS_LOCKADDR_HI), ++ (lock_addr >> 32) & 0xFFFFFFFFUL); ++ write_cmd(kbdev, as->number, AS_COMMAND_LOCK); ++ ++ /* Run the MMU operation */ ++ write_cmd(kbdev, as->number, op); ++ ++ /* Wait for the flush to complete */ ++ ret = wait_ready(kbdev, as->number); ++ } ++ } ++ ++ return ret; ++} ++ ++void kbase_mmu_hw_clear_fault(struct kbase_device *kbdev, struct kbase_as *as, ++ enum kbase_mmu_fault_type type) ++{ ++ unsigned long flags; ++ u32 pf_bf_mask; ++ ++ spin_lock_irqsave(&kbdev->mmu_mask_change, flags); ++ ++ /* ++ * A reset is in-flight and we're flushing the IRQ + bottom half ++ * so don't update anything as it could race with the reset code. ++ */ ++ if (kbdev->irq_reset_flush) ++ goto unlock; ++ ++ /* Clear the page (and bus fault IRQ as well in case one occurred) */ ++ pf_bf_mask = MMU_PAGE_FAULT(as->number); ++#if !MALI_USE_CSF ++ if (type == KBASE_MMU_FAULT_TYPE_BUS || ++ type == KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED) ++ pf_bf_mask |= MMU_BUS_ERROR(as->number); ++#endif ++ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), pf_bf_mask); ++ ++unlock: ++ spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); ++} ++ ++void kbase_mmu_hw_enable_fault(struct kbase_device *kbdev, struct kbase_as *as, ++ enum kbase_mmu_fault_type type) ++{ ++ unsigned long flags; ++ u32 irq_mask; ++ ++ /* Enable the page fault IRQ ++ * (and bus fault IRQ as well in case one occurred) ++ */ ++ spin_lock_irqsave(&kbdev->mmu_mask_change, flags); ++ ++ /* ++ * A reset is in-flight and we're flushing the IRQ + bottom half ++ * so don't update anything as it could race with the reset code. ++ */ ++ if (kbdev->irq_reset_flush) ++ goto unlock; ++ ++ irq_mask = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK)) | ++ MMU_PAGE_FAULT(as->number); ++ ++#if !MALI_USE_CSF ++ if (type == KBASE_MMU_FAULT_TYPE_BUS || ++ type == KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED) ++ irq_mask |= MMU_BUS_ERROR(as->number); ++#endif ++ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), irq_mask); ++ ++unlock: ++ spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); ++} +diff --git a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_internal.h b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_internal.h +new file mode 100755 +index 000000000000..8ecb14d72327 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_internal.h +@@ -0,0 +1,73 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KBASE_MMU_INTERNAL_H_ ++#define _KBASE_MMU_INTERNAL_H_ ++ ++void kbase_mmu_get_as_setup(struct kbase_mmu_table *mmut, ++ struct kbase_mmu_setup * const setup); ++ ++/** ++ * kbase_mmu_report_mcu_as_fault_and_reset - Report page fault for all ++ * address spaces and reset the GPU. ++ * @kbdev: The kbase_device the fault happened on ++ * @fault: Data relating to the fault ++ */ ++void kbase_mmu_report_mcu_as_fault_and_reset(struct kbase_device *kbdev, ++ struct kbase_fault *fault); ++ ++void kbase_gpu_report_bus_fault_and_kill(struct kbase_context *kctx, ++ struct kbase_as *as, struct kbase_fault *fault); ++ ++void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, ++ struct kbase_as *as, const char *reason_str, ++ struct kbase_fault *fault); ++ ++/** ++ * kbase_mmu_switch_to_ir() - Switch to incremental rendering if possible ++ * @kctx: kbase_context for the faulting address space. ++ * @reg: of a growable GPU memory region in the same context. ++ * Takes ownership of the reference if successful. ++ * ++ * Used to switch to incremental rendering if we have nearly run out of ++ * virtual address space in a growable memory region. ++ * ++ * Return 0 if successful, otherwise a negative error code. ++ */ ++int kbase_mmu_switch_to_ir(struct kbase_context *kctx, ++ struct kbase_va_region *reg); ++ ++/** ++ * kbase_mmu_page_fault_worker() - Process a page fault. ++ * ++ * @data: work_struct passed by queue_work() ++ */ ++void kbase_mmu_page_fault_worker(struct work_struct *data); ++ ++/** ++ * kbase_mmu_bus_fault_worker() - Process a bus fault. ++ * ++ * @data: work_struct passed by queue_work() ++ */ ++void kbase_mmu_bus_fault_worker(struct work_struct *data); ++ ++#endif /* _KBASE_MMU_INTERNAL_H_ */ +diff --git a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_mode_aarch64.c b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_mode_aarch64.c +new file mode 100755 +index 000000000000..02493e9b2621 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_mode_aarch64.c +@@ -0,0 +1,200 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2014, 2016-2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include "mali_kbase.h" ++#include ++#include "mali_kbase_defs.h" ++#include ++#include ++ ++#define ENTRY_TYPE_MASK 3ULL ++/* For valid ATEs bit 1 = ((level == 3) ? 1 : 0). ++ * Valid ATE entries at level 3 are flagged with the value 3. ++ * Valid ATE entries at level 0-2 are flagged with the value 1. ++ */ ++#define ENTRY_IS_ATE_L3 3ULL ++#define ENTRY_IS_ATE_L02 1ULL ++#define ENTRY_IS_INVAL 2ULL ++#define ENTRY_IS_PTE 3ULL ++ ++#define ENTRY_ATTR_BITS (7ULL << 2) /* bits 4:2 */ ++#define ENTRY_ACCESS_RW (1ULL << 6) /* bits 6:7 */ ++#define ENTRY_ACCESS_RO (3ULL << 6) ++#define ENTRY_SHARE_BITS (3ULL << 8) /* bits 9:8 */ ++#define ENTRY_ACCESS_BIT (1ULL << 10) ++#define ENTRY_NX_BIT (1ULL << 54) ++ ++/* Helper Function to perform assignment of page table entries, to ++ * ensure the use of strd, which is required on LPAE systems. ++ */ ++static inline void page_table_entry_set(u64 *pte, u64 phy) ++{ ++#if KERNEL_VERSION(3, 18, 13) <= LINUX_VERSION_CODE ++ WRITE_ONCE(*pte, phy); ++#else ++#ifdef CONFIG_64BIT ++ barrier(); ++ *pte = phy; ++ barrier(); ++#elif defined(CONFIG_ARM) ++ barrier(); ++ asm volatile("ldrd r0, [%1]\n\t" ++ "strd r0, %0\n\t" ++ : "=m" (*pte) ++ : "r" (&phy) ++ : "r0", "r1"); ++ barrier(); ++#else ++#error "64-bit atomic write must be implemented for your architecture" ++#endif ++#endif ++} ++ ++static void mmu_update(struct kbase_device *kbdev, struct kbase_mmu_table *mmut, ++ int as_nr) ++{ ++ struct kbase_as *as; ++ struct kbase_mmu_setup *current_setup; ++ ++ if (WARN_ON(as_nr == KBASEP_AS_NR_INVALID)) ++ return; ++ ++ as = &kbdev->as[as_nr]; ++ current_setup = &as->current_setup; ++ ++ kbase_mmu_get_as_setup(mmut, current_setup); ++ ++ /* Apply the address space setting */ ++ kbase_mmu_hw_configure(kbdev, as); ++} ++ ++static void mmu_disable_as(struct kbase_device *kbdev, int as_nr) ++{ ++ struct kbase_as * const as = &kbdev->as[as_nr]; ++ struct kbase_mmu_setup * const current_setup = &as->current_setup; ++ ++ current_setup->transtab = 0ULL; ++ current_setup->transcfg = AS_TRANSCFG_ADRMODE_UNMAPPED; ++ ++ /* Apply the address space setting */ ++ kbase_mmu_hw_configure(kbdev, as); ++} ++ ++static phys_addr_t pte_to_phy_addr(u64 entry) ++{ ++ if (!(entry & 1)) ++ return 0; ++ ++ return entry & ~0xFFF; ++} ++ ++static int ate_is_valid(u64 ate, int const level) ++{ ++ if (level == MIDGARD_MMU_BOTTOMLEVEL) ++ return ((ate & ENTRY_TYPE_MASK) == ENTRY_IS_ATE_L3); ++ else ++ return ((ate & ENTRY_TYPE_MASK) == ENTRY_IS_ATE_L02); ++} ++ ++static int pte_is_valid(u64 pte, int const level) ++{ ++ /* PTEs cannot exist at the bottom level */ ++ if (level == MIDGARD_MMU_BOTTOMLEVEL) ++ return false; ++ return ((pte & ENTRY_TYPE_MASK) == ENTRY_IS_PTE); ++} ++ ++/* ++ * Map KBASE_REG flags to MMU flags ++ */ ++static u64 get_mmu_flags(unsigned long flags) ++{ ++ u64 mmu_flags; ++ ++ /* store mem_attr index as 4:2 (macro called ensures 3 bits already) */ ++ mmu_flags = KBASE_REG_MEMATTR_VALUE(flags) << 2; ++ ++ /* Set access flags - note that AArch64 stage 1 does not support ++ * write-only access, so we use read/write instead ++ */ ++ if (flags & KBASE_REG_GPU_WR) ++ mmu_flags |= ENTRY_ACCESS_RW; ++ else if (flags & KBASE_REG_GPU_RD) ++ mmu_flags |= ENTRY_ACCESS_RO; ++ ++ /* nx if requested */ ++ mmu_flags |= (flags & KBASE_REG_GPU_NX) ? ENTRY_NX_BIT : 0; ++ ++ if (flags & KBASE_REG_SHARE_BOTH) { ++ /* inner and outer shareable */ ++ mmu_flags |= SHARE_BOTH_BITS; ++ } else if (flags & KBASE_REG_SHARE_IN) { ++ /* inner shareable coherency */ ++ mmu_flags |= SHARE_INNER_BITS; ++ } ++ ++ return mmu_flags; ++} ++ ++static void entry_set_ate(u64 *entry, ++ struct tagged_addr phy, ++ unsigned long flags, ++ int const level) ++{ ++ if (level == MIDGARD_MMU_BOTTOMLEVEL) ++ page_table_entry_set(entry, as_phys_addr_t(phy) | ++ get_mmu_flags(flags) | ++ ENTRY_ACCESS_BIT | ENTRY_IS_ATE_L3); ++ else ++ page_table_entry_set(entry, as_phys_addr_t(phy) | ++ get_mmu_flags(flags) | ++ ENTRY_ACCESS_BIT | ENTRY_IS_ATE_L02); ++} ++ ++static void entry_set_pte(u64 *entry, phys_addr_t phy) ++{ ++ page_table_entry_set(entry, (phy & PAGE_MASK) | ++ ENTRY_ACCESS_BIT | ENTRY_IS_PTE); ++} ++ ++static void entry_invalidate(u64 *entry) ++{ ++ page_table_entry_set(entry, ENTRY_IS_INVAL); ++} ++ ++static struct kbase_mmu_mode const aarch64_mode = { ++ .update = mmu_update, ++ .get_as_setup = kbase_mmu_get_as_setup, ++ .disable_as = mmu_disable_as, ++ .pte_to_phy_addr = pte_to_phy_addr, ++ .ate_is_valid = ate_is_valid, ++ .pte_is_valid = pte_is_valid, ++ .entry_set_ate = entry_set_ate, ++ .entry_set_pte = entry_set_pte, ++ .entry_invalidate = entry_invalidate, ++ .flags = KBASE_MMU_MODE_HAS_NON_CACHEABLE ++}; ++ ++struct kbase_mmu_mode const *kbase_mmu_mode_get_aarch64(void) ++{ ++ return &aarch64_mode; ++} +diff --git a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_mode_lpae.c b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_mode_lpae.c +new file mode 100755 +index 000000000000..91a2d7ac4dcb +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_mode_lpae.c +@@ -0,0 +1,215 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++#include "mali_kbase.h" ++#include ++#include "mali_kbase_defs.h" ++ ++#define ENTRY_TYPE_MASK 3ULL ++#define ENTRY_IS_ATE 1ULL ++#define ENTRY_IS_INVAL 2ULL ++#define ENTRY_IS_PTE 3ULL ++ ++#define ENTRY_ATTR_BITS (7ULL << 2) /* bits 4:2 */ ++#define ENTRY_RD_BIT (1ULL << 6) ++#define ENTRY_WR_BIT (1ULL << 7) ++#define ENTRY_SHARE_BITS (3ULL << 8) /* bits 9:8 */ ++#define ENTRY_ACCESS_BIT (1ULL << 10) ++#define ENTRY_NX_BIT (1ULL << 54) ++ ++#define ENTRY_FLAGS_MASK (ENTRY_ATTR_BITS | ENTRY_RD_BIT | ENTRY_WR_BIT | \ ++ ENTRY_SHARE_BITS | ENTRY_ACCESS_BIT | ENTRY_NX_BIT) ++ ++/* Helper Function to perform assignment of page table entries, to ++ * ensure the use of strd, which is required on LPAE systems. ++ */ ++static inline void page_table_entry_set(u64 *pte, u64 phy) ++{ ++#if KERNEL_VERSION(3, 18, 13) <= LINUX_VERSION_CODE ++ WRITE_ONCE(*pte, phy); ++#else ++#ifdef CONFIG_64BIT ++ barrier(); ++ *pte = phy; ++ barrier(); ++#elif defined(CONFIG_ARM) ++ barrier(); ++ asm volatile("ldrd r0, [%1]\n\t" ++ "strd r0, %0\n\t" ++ : "=m" (*pte) ++ : "r" (&phy) ++ : "r0", "r1"); ++ barrier(); ++#else ++#error "64-bit atomic write must be implemented for your architecture" ++#endif ++#endif ++} ++ ++static void mmu_get_as_setup(struct kbase_mmu_table *mmut, ++ struct kbase_mmu_setup * const setup) ++{ ++ /* Set up the required caching policies at the correct indices ++ * in the memattr register. ++ */ ++ setup->memattr = ++ (AS_MEMATTR_LPAE_IMPL_DEF_CACHE_POLICY << ++ (AS_MEMATTR_INDEX_IMPL_DEF_CACHE_POLICY * 8)) | ++ (AS_MEMATTR_LPAE_FORCE_TO_CACHE_ALL << ++ (AS_MEMATTR_INDEX_FORCE_TO_CACHE_ALL * 8)) | ++ (AS_MEMATTR_LPAE_WRITE_ALLOC << ++ (AS_MEMATTR_INDEX_WRITE_ALLOC * 8)) | ++ (AS_MEMATTR_LPAE_OUTER_IMPL_DEF << ++ (AS_MEMATTR_INDEX_OUTER_IMPL_DEF * 8)) | ++ (AS_MEMATTR_LPAE_OUTER_WA << ++ (AS_MEMATTR_INDEX_OUTER_WA * 8)) | ++ 0; /* The other indices are unused for now */ ++ ++ setup->transtab = ((u64)mmut->pgd & ++ ((0xFFFFFFFFULL << 32) | AS_TRANSTAB_LPAE_ADDR_SPACE_MASK)) | ++ AS_TRANSTAB_LPAE_ADRMODE_TABLE | ++ AS_TRANSTAB_LPAE_READ_INNER; ++ ++ setup->transcfg = 0; ++} ++ ++static void mmu_update(struct kbase_device *kbdev, ++ struct kbase_mmu_table *mmut, ++ int as_nr) ++{ ++ struct kbase_as *as; ++ struct kbase_mmu_setup *current_setup; ++ ++ if (WARN_ON(as_nr == KBASEP_AS_NR_INVALID)) ++ return; ++ ++ as = &kbdev->as[as_nr]; ++ current_setup = &as->current_setup; ++ ++ mmu_get_as_setup(mmut, current_setup); ++ ++ /* Apply the address space setting */ ++ kbase_mmu_hw_configure(kbdev, as); ++} ++ ++static void mmu_disable_as(struct kbase_device *kbdev, int as_nr) ++{ ++ struct kbase_as * const as = &kbdev->as[as_nr]; ++ struct kbase_mmu_setup * const current_setup = &as->current_setup; ++ ++ current_setup->transtab = AS_TRANSTAB_LPAE_ADRMODE_UNMAPPED; ++ ++ /* Apply the address space setting */ ++ kbase_mmu_hw_configure(kbdev, as); ++} ++ ++static phys_addr_t pte_to_phy_addr(u64 entry) ++{ ++ if (!(entry & 1)) ++ return 0; ++ ++ return entry & ~0xFFF; ++} ++ ++static int ate_is_valid(u64 ate, int const level) ++{ ++ return ((ate & ENTRY_TYPE_MASK) == ENTRY_IS_ATE); ++} ++ ++static int pte_is_valid(u64 pte, int const level) ++{ ++ return ((pte & ENTRY_TYPE_MASK) == ENTRY_IS_PTE); ++} ++ ++/* ++ * Map KBASE_REG flags to MMU flags ++ */ ++static u64 get_mmu_flags(unsigned long flags) ++{ ++ u64 mmu_flags; ++ unsigned long memattr_idx; ++ ++ memattr_idx = KBASE_REG_MEMATTR_VALUE(flags); ++ if (WARN(memattr_idx == AS_MEMATTR_INDEX_NON_CACHEABLE, ++ "Legacy Mode MMU cannot honor GPU non-cachable memory, will use default instead\n")) ++ memattr_idx = AS_MEMATTR_INDEX_DEFAULT; ++ /* store mem_attr index as 4:2, noting that: ++ * - macro called above ensures 3 bits already ++ * - all AS_MEMATTR_INDEX_<...> macros only use 3 bits ++ */ ++ mmu_flags = memattr_idx << 2; ++ ++ /* write perm if requested */ ++ mmu_flags |= (flags & KBASE_REG_GPU_WR) ? ENTRY_WR_BIT : 0; ++ /* read perm if requested */ ++ mmu_flags |= (flags & KBASE_REG_GPU_RD) ? ENTRY_RD_BIT : 0; ++ /* nx if requested */ ++ mmu_flags |= (flags & KBASE_REG_GPU_NX) ? ENTRY_NX_BIT : 0; ++ ++ if (flags & KBASE_REG_SHARE_BOTH) { ++ /* inner and outer shareable */ ++ mmu_flags |= SHARE_BOTH_BITS; ++ } else if (flags & KBASE_REG_SHARE_IN) { ++ /* inner shareable coherency */ ++ mmu_flags |= SHARE_INNER_BITS; ++ } ++ ++ return mmu_flags; ++} ++ ++static void entry_set_ate(u64 *entry, ++ struct tagged_addr phy, ++ unsigned long flags, ++ int const level) ++{ ++ page_table_entry_set(entry, as_phys_addr_t(phy) | get_mmu_flags(flags) | ++ ENTRY_IS_ATE); ++} ++ ++static void entry_set_pte(u64 *entry, phys_addr_t phy) ++{ ++ page_table_entry_set(entry, (phy & ~0xFFF) | ENTRY_IS_PTE); ++} ++ ++static void entry_invalidate(u64 *entry) ++{ ++ page_table_entry_set(entry, ENTRY_IS_INVAL); ++} ++ ++static struct kbase_mmu_mode const lpae_mode = { ++ .update = mmu_update, ++ .get_as_setup = mmu_get_as_setup, ++ .disable_as = mmu_disable_as, ++ .pte_to_phy_addr = pte_to_phy_addr, ++ .ate_is_valid = ate_is_valid, ++ .pte_is_valid = pte_is_valid, ++ .entry_set_ate = entry_set_ate, ++ .entry_set_pte = entry_set_pte, ++ .entry_invalidate = entry_invalidate, ++ .flags = 0 ++}; ++ ++struct kbase_mmu_mode const *kbase_mmu_mode_get_lpae(void) ++{ ++ return &lpae_mode; ++} +diff --git a/drivers/gpu/arm/bifrost/platform/Kconfig b/drivers/gpu/arm/bifrost/platform/Kconfig +new file mode 100755 +index 000000000000..ef9fb963ecf5 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/platform/Kconfig +@@ -0,0 +1,30 @@ ++# ++# (C) COPYRIGHT 2012-2013, 2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# GNU General Public License for more details. ++# ++# You should have received a copy of the GNU General Public License ++# along with this program; if not, you can access it online at ++# http://www.gnu.org/licenses/gpl-2.0.html. ++# ++# SPDX-License-Identifier: GPL-2.0 ++# ++# ++ ++ ++ ++# Add your platform specific Kconfig file here ++# ++# "drivers/gpu/arm/midgard/platform/xxx/Kconfig" ++# ++# Where xxx is the platform name is the name set in MALI_PLATFORM_NAME ++# ++ +diff --git a/drivers/gpu/arm/bifrost/platform/devicetree/Kbuild b/drivers/gpu/arm/bifrost/platform/devicetree/Kbuild +new file mode 100755 +index 000000000000..78343c0570d1 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/platform/devicetree/Kbuild +@@ -0,0 +1,25 @@ ++# ++# (C) COPYRIGHT 2012-2017, 2020 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# GNU General Public License for more details. ++# ++# You should have received a copy of the GNU General Public License ++# along with this program; if not, you can access it online at ++# http://www.gnu.org/licenses/gpl-2.0.html. ++# ++# SPDX-License-Identifier: GPL-2.0 ++# ++# ++ ++mali_kbase-y += \ ++ $(MALI_PLATFORM_DIR)/mali_kbase_config_devicetree.o \ ++ $(MALI_PLATFORM_DIR)/mali_kbase_runtime_pm.o \ ++ $(MALI_PLATFORM_DIR)/mali_kbase_clk_rate_trace.o +diff --git a/drivers/gpu/arm/bifrost/platform/devicetree/mali_kbase_clk_rate_trace.c b/drivers/gpu/arm/bifrost/platform/devicetree/mali_kbase_clk_rate_trace.c +new file mode 100755 +index 000000000000..11a8b77dca06 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/platform/devicetree/mali_kbase_clk_rate_trace.c +@@ -0,0 +1,68 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015, 2017-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++#include ++#include ++#include "mali_kbase_config_platform.h" ++ ++static void *enumerate_gpu_clk(struct kbase_device *kbdev, ++ unsigned int index) ++{ ++ if (index >= kbdev->nr_clocks) ++ return NULL; ++ ++ return kbdev->clocks[index]; ++} ++ ++static unsigned long get_gpu_clk_rate(struct kbase_device *kbdev, ++ void *gpu_clk_handle) ++{ ++ return clk_get_rate((struct clk *)gpu_clk_handle); ++} ++ ++static int gpu_clk_notifier_register(struct kbase_device *kbdev, ++ void *gpu_clk_handle, struct notifier_block *nb) ++{ ++ compiletime_assert(offsetof(struct clk_notifier_data, clk) == ++ offsetof(struct kbase_gpu_clk_notifier_data, gpu_clk_handle), ++ "mismatch in the offset of clk member"); ++ ++ compiletime_assert(sizeof(((struct clk_notifier_data *)0)->clk) == ++ sizeof(((struct kbase_gpu_clk_notifier_data *)0)->gpu_clk_handle), ++ "mismatch in the size of clk member"); ++ ++ return clk_notifier_register((struct clk *)gpu_clk_handle, nb); ++} ++ ++static void gpu_clk_notifier_unregister(struct kbase_device *kbdev, ++ void *gpu_clk_handle, struct notifier_block *nb) ++{ ++ clk_notifier_unregister((struct clk *)gpu_clk_handle, nb); ++} ++ ++struct kbase_clk_rate_trace_op_conf clk_rate_trace_ops = { ++ .get_gpu_clk_rate = get_gpu_clk_rate, ++ .enumerate_gpu_clk = enumerate_gpu_clk, ++ .gpu_clk_notifier_register = gpu_clk_notifier_register, ++ .gpu_clk_notifier_unregister = gpu_clk_notifier_unregister, ++}; +diff --git a/drivers/gpu/arm/bifrost/platform/devicetree/mali_kbase_config_devicetree.c b/drivers/gpu/arm/bifrost/platform/devicetree/mali_kbase_config_devicetree.c +new file mode 100755 +index 000000000000..ccefddf882fd +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/platform/devicetree/mali_kbase_config_devicetree.c +@@ -0,0 +1,41 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015, 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++ ++static struct kbase_platform_config dummy_platform_config; ++ ++struct kbase_platform_config *kbase_get_platform_config(void) ++{ ++ return &dummy_platform_config; ++} ++ ++#ifndef CONFIG_OF ++int kbase_platform_register(void) ++{ ++ return 0; ++} ++ ++void kbase_platform_unregister(void) ++{ ++} ++#endif +diff --git a/drivers/gpu/arm/bifrost/platform/devicetree/mali_kbase_config_platform.h b/drivers/gpu/arm/bifrost/platform/devicetree/mali_kbase_config_platform.h +new file mode 100755 +index 000000000000..2137b425c1ab +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/platform/devicetree/mali_kbase_config_platform.h +@@ -0,0 +1,49 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2017, 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/** ++ * Power management configuration ++ * ++ * Attached value: pointer to @ref kbase_pm_callback_conf ++ * Default value: See @ref kbase_pm_callback_conf ++ */ ++#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks) ++ ++/** ++ * Platform specific configuration functions ++ * ++ * Attached value: pointer to @ref kbase_platform_funcs_conf ++ * Default value: See @ref kbase_platform_funcs_conf ++ */ ++#define PLATFORM_FUNCS (NULL) ++ ++#define CLK_RATE_TRACE_OPS (&clk_rate_trace_ops) ++ ++extern struct kbase_pm_callback_conf pm_callbacks; ++extern struct kbase_clk_rate_trace_op_conf clk_rate_trace_ops; ++ ++/** ++ * Autosuspend delay ++ * ++ * The delay time (in milliseconds) to be used for autosuspend ++ */ ++#define AUTO_SUSPEND_DELAY (100) +diff --git a/drivers/gpu/arm/bifrost/platform/devicetree/mali_kbase_runtime_pm.c b/drivers/gpu/arm/bifrost/platform/devicetree/mali_kbase_runtime_pm.c +new file mode 100755 +index 000000000000..8772edb56f73 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/platform/devicetree/mali_kbase_runtime_pm.c +@@ -0,0 +1,185 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015, 2017-2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include "mali_kbase_config_platform.h" ++ ++static void enable_gpu_power_control(struct kbase_device *kbdev) ++{ ++ unsigned int i; ++ ++#if defined(CONFIG_REGULATOR) ++ for (i = 0; i < kbdev->nr_regulators; i++) { ++ if (WARN_ON(kbdev->regulators[i] == NULL)) ++ ; ++ else if (!regulator_is_enabled(kbdev->regulators[i])) ++ WARN_ON(regulator_enable(kbdev->regulators[i])); ++ } ++#endif ++ ++ for (i = 0; i < kbdev->nr_clocks; i++) { ++ if (WARN_ON(kbdev->clocks[i] == NULL)) ++ ; ++ else if (!__clk_is_enabled(kbdev->clocks[i])) ++ WARN_ON(clk_prepare_enable(kbdev->clocks[i])); ++ } ++} ++ ++static void disable_gpu_power_control(struct kbase_device *kbdev) ++{ ++ unsigned int i; ++ ++ for (i = 0; i < kbdev->nr_clocks; i++) { ++ if (WARN_ON(kbdev->clocks[i] == NULL)) ++ ; ++ else if (__clk_is_enabled(kbdev->clocks[i])) { ++ clk_disable_unprepare(kbdev->clocks[i]); ++ WARN_ON(__clk_is_enabled(kbdev->clocks[i])); ++ } ++ ++ } ++ ++#if defined(CONFIG_REGULATOR) ++ for (i = 0; i < kbdev->nr_regulators; i++) { ++ if (WARN_ON(kbdev->regulators[i] == NULL)) ++ ; ++ else if (regulator_is_enabled(kbdev->regulators[i])) ++ WARN_ON(regulator_disable(kbdev->regulators[i])); ++ } ++#endif ++} ++ ++static int pm_callback_power_on(struct kbase_device *kbdev) ++{ ++ int ret = 1; /* Assume GPU has been powered off */ ++ int error; ++ ++ dev_dbg(kbdev->dev, "pm_callback_power_on %p\n", ++ (void *)kbdev->dev->pm_domain); ++ ++ enable_gpu_power_control(kbdev); ++ ++ error = pm_runtime_get_sync(kbdev->dev); ++ if (error == 1) { ++ /* ++ * Let core know that the chip has not been ++ * powered off, so we can save on re-initialization. ++ */ ++ ret = 0; ++ } ++ ++ dev_dbg(kbdev->dev, "pm_runtime_get_sync returned %d\n", error); ++ ++ return ret; ++} ++ ++static void pm_callback_power_off(struct kbase_device *kbdev) ++{ ++ dev_dbg(kbdev->dev, "pm_callback_power_off\n"); ++ ++ pm_runtime_mark_last_busy(kbdev->dev); ++ pm_runtime_put_autosuspend(kbdev->dev); ++ ++#ifndef KBASE_PM_RUNTIME ++ disable_gpu_power_control(kbdev); ++#endif ++} ++ ++#ifdef KBASE_PM_RUNTIME ++static int kbase_device_runtime_init(struct kbase_device *kbdev) ++{ ++ int ret = 0; ++ ++ dev_dbg(kbdev->dev, "kbase_device_runtime_init\n"); ++ ++ pm_runtime_set_autosuspend_delay(kbdev->dev, AUTO_SUSPEND_DELAY); ++ pm_runtime_use_autosuspend(kbdev->dev); ++ ++ pm_runtime_set_active(kbdev->dev); ++ pm_runtime_enable(kbdev->dev); ++ ++ if (!pm_runtime_enabled(kbdev->dev)) { ++ dev_warn(kbdev->dev, "pm_runtime not enabled"); ++ ret = -ENOSYS; ++ } ++ ++ return ret; ++} ++ ++static void kbase_device_runtime_disable(struct kbase_device *kbdev) ++{ ++ dev_dbg(kbdev->dev, "kbase_device_runtime_disable\n"); ++ pm_runtime_disable(kbdev->dev); ++} ++#endif ++ ++static int pm_callback_runtime_on(struct kbase_device *kbdev) ++{ ++ dev_dbg(kbdev->dev, "pm_callback_runtime_on\n"); ++ ++ enable_gpu_power_control(kbdev); ++ return 0; ++} ++ ++static void pm_callback_runtime_off(struct kbase_device *kbdev) ++{ ++ dev_dbg(kbdev->dev, "pm_callback_runtime_off\n"); ++ ++ disable_gpu_power_control(kbdev); ++} ++ ++static void pm_callback_resume(struct kbase_device *kbdev) ++{ ++ int ret = pm_callback_runtime_on(kbdev); ++ ++ WARN_ON(ret); ++} ++ ++static void pm_callback_suspend(struct kbase_device *kbdev) ++{ ++ pm_callback_runtime_off(kbdev); ++} ++ ++struct kbase_pm_callback_conf pm_callbacks = { ++ .power_on_callback = pm_callback_power_on, ++ .power_off_callback = pm_callback_power_off, ++ .power_suspend_callback = pm_callback_suspend, ++ .power_resume_callback = pm_callback_resume, ++#ifdef KBASE_PM_RUNTIME ++ .power_runtime_init_callback = kbase_device_runtime_init, ++ .power_runtime_term_callback = kbase_device_runtime_disable, ++ .power_runtime_on_callback = pm_callback_runtime_on, ++ .power_runtime_off_callback = pm_callback_runtime_off, ++#else /* KBASE_PM_RUNTIME */ ++ .power_runtime_init_callback = NULL, ++ .power_runtime_term_callback = NULL, ++ .power_runtime_on_callback = NULL, ++ .power_runtime_off_callback = NULL, ++#endif /* KBASE_PM_RUNTIME */ ++}; ++ ++ +diff --git a/drivers/gpu/arm/bifrost/platform/rk/Kbuild b/drivers/gpu/arm/bifrost/platform/rk/Kbuild +new file mode 100755 +index 000000000000..7cc6c59d969f +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/platform/rk/Kbuild +@@ -0,0 +1,17 @@ ++# ++# (C) COPYRIGHT 2012-2013 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++bifrost_kbase-y += \ ++ $(MALI_PLATFORM_DIR)/mali_kbase_config_rk.o \ ++ +diff --git a/drivers/gpu/arm/bifrost/platform/rk/custom_log.h b/drivers/gpu/arm/bifrost/platform/rk/custom_log.h +new file mode 100755 +index 000000000000..5de70ee13d25 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/platform/rk/custom_log.h +@@ -0,0 +1,192 @@ ++/* ++ * (C) COPYRIGHT RockChip Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ */ ++ ++#ifndef __CUSTOM_LOG_H__ ++#define __CUSTOM_LOG_H__ ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++/* ----------------------------------------------------------------------------- ++ * Include Files ++ * ----------------------------------------------------------------------------- ++ */ ++#include ++#include ++ ++/* ----------------------------------------------------------------------------- ++ * Macros Definition ++ * ----------------------------------------------------------------------------- ++ */ ++ ++/** 若下列 macro 有被定义, æ‰ ä½¿èƒ½ log 输出. */ ++/* #define ENABLE_DEBUG_LOG */ ++ ++/*----------------------------------------------------------------------------*/ ++ ++#ifdef ENABLE_VERBOSE_LOG ++/** Verbose log. */ ++#define V(fmt, args...) \ ++ pr_debug("V : [File] : %s; [Line] : %d; [Func] : %s(); " fmt \ ++ "\n", \ ++ __FILE__, \ ++ __LINE__, \ ++ __func__, \ ++ ## args) ++#else ++#define V(...) ((void)0) ++#endif ++ ++#ifdef ENABLE_DEBUG_LOG ++/** Debug log. */ ++#define D(fmt, args...) \ ++ pr_info("D : [File] : %s; [Line] : %d; [Func] : %s(); " fmt \ ++ "\n", \ ++ __FILE__, \ ++ __LINE__, \ ++ __func__, \ ++ ## args) ++#else ++#define D(...) ((void)0) ++#endif ++ ++#define I(fmt, args...) \ ++ pr_info("I : [File] : %s; [Line] : %d; [Func] : %s(); " fmt \ ++ "\n", \ ++ __FILE__, \ ++ __LINE__, \ ++ __func__, \ ++ ## args) ++ ++#define W(fmt, args...) \ ++ pr_warn("W : [File] : %s; [Line] : %d; [Func] : %s(); " \ ++ fmt "\n", \ ++ __FILE__, \ ++ __LINE__, \ ++ __func__, \ ++ ## args) ++ ++#define E(fmt, args...) \ ++ pr_err("E : [File] : %s; [Line] : %d; [Func] : %s(); " fmt \ ++ "\n", \ ++ __FILE__, \ ++ __LINE__, \ ++ __func__, \ ++ ## args) ++ ++/*-------------------------------------------------------*/ ++ ++/** 使用 D(), 以å进制的形å¼æ‰“å°å˜é‡ 'var' çš„ value. */ ++#define D_DEC(var) D(#var " = %d.", var) ++ ++#define E_DEC(var) E(#var " = %d.", var) ++ ++/** 使用 D(), 以å六进制的形å¼æ‰“å°å˜é‡ 'var' çš„ value. */ ++#define D_HEX(var) D(#var " = 0x%x.", var) ++ ++#define E_HEX(var) E(#var " = 0x%x.", var) ++ ++/** ++ * 使用 D(), 以å六进制的形å¼, ++ * 打å°æŒ‡é’ˆç±»åž‹å˜é‡ 'ptr' çš„ value. ++ */ ++#define D_PTR(ptr) D(#ptr " = %p.", ptr) ++ ++#define E_PTR(ptr) E(#ptr " = %p.", ptr) ++ ++/** 使用 D(), æ‰“å° char 字串. */ ++#define D_STR(p_str) \ ++do { \ ++ if (!p_str) { \ ++ D(#p_str " = NULL."); \ ++ else \ ++ D(#p_str " = '%s'.", p_str); \ ++} while (0) ++ ++#define E_STR(p_str) \ ++do { \ ++ if (!p_str) \ ++ E(#p_str " = NULL."); \ ++ else \ ++ E(#p_str " = '%s'.", p_str); \ ++} while (0) ++ ++#ifdef ENABLE_DEBUG_LOG ++/** ++ * log 从 'p_start' 地å€å¼€å§‹çš„ 'len' 个字节的数æ®. ++ */ ++#define D_MEM(p_start, len) \ ++do { \ ++ int i = 0; \ ++ char *p = (char *)(p_start); \ ++ D("dump memory from addr of '" #p_start "', from %p, length %d' : ", \ ++ (p_start), \ ++ (len)); \ ++ pr_debug("\t\t"); \ ++ for (i = 0; i < (len); i++) \ ++ pr_debug("0x%02x, ", p[i]); \ ++ pr_debug("\n"); \ ++} while (0) ++#else ++#define D_MEM(...) ((void)0) ++#endif ++ ++/*-------------------------------------------------------*/ ++ ++/** ++ * 在特定æ¡ä»¶ä¸‹, 判定 error å‘生, ++ * å°†å˜é‡ 'ret_var' 设置 'err_code', ++ * log 输出对应的 Error Caution, ++ * 然åŽè·³è½¬ 'label' 指定的代ç å¤„执行. ++ * @param msg ++ * 纯字串形å¼çš„æ示信æ¯. ++ * @param ret_var ++ * 标识函数执行状æ€æˆ–者结果的å˜é‡, ++ * 将被设置具体的 Error Code. ++ * 通常是 'ret' or 'result'. ++ * @param err_code ++ * 表å¾ç‰¹å®š error 的常数标识, ++ * 通常是 å®çš„å½¢æ€. ++ * @param label ++ * 程åºå°†è¦è·³è½¬åˆ°çš„错误处ç†ä»£ç çš„æ ‡å·, ++ * 通常就是 'EXIT'. ++ * @param args... ++ * 对应 'msg_fmt' 实å‚中, ++ * '%s', '%d', ... 等转æ¢è¯´æ˜Žç¬¦çš„具体å¯å˜é•¿å®žå‚. ++ */ ++#define SET_ERROR_AND_JUMP(msg_fmt, ret_var, err_code, label, args...) \ ++do { \ ++ E("To set '" #ret_var "' to %d('" #err_code "'), because : " msg_fmt, \ ++ (err_code), \ ++ ## args); \ ++ (ret_var) = (err_code); \ ++ goto label; \ ++} while (0) ++ ++/* ----------------------------------------------------------------------------- ++ * Types and Structures Definition ++ * ----------------------------------------------------------------------------- ++ */ ++ ++/* ----------------------------------------------------------------------------- ++ * Global Functions' Prototype ++ * ----------------------------------------------------------------------------- ++ */ ++ ++/* ----------------------------------------------------------------------------- ++ * Inline Functions Implementation ++ * ----------------------------------------------------------------------------- ++ */ ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif /* __CUSTOM_LOG_H__ */ +diff --git a/drivers/gpu/arm/bifrost/platform/rk/mali_kbase_config_platform.h b/drivers/gpu/arm/bifrost/platform/rk/mali_kbase_config_platform.h +new file mode 100755 +index 000000000000..07c5b6f8a760 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/platform/rk/mali_kbase_config_platform.h +@@ -0,0 +1,88 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ */ ++ ++/** ++ * @file mali_kbase_config_platform.h ++ * 声明 platform_config_of_rk (platform_rk çš„ platform_config). ++ */ ++ ++/** ++ * Maximum frequency GPU will be clocked at. ++ * Given in kHz. ++ * This must be specified as there is no default value. ++ * ++ * Attached value: number in kHz ++ * Default value: NA ++ */ ++#define GPU_FREQ_KHZ_MAX (5000) ++ ++/** ++ * Minimum frequency GPU will be clocked at. ++ * Given in kHz. ++ * This must be specified as there is no default value. ++ * ++ * Attached value: number in kHz ++ * Default value: NA ++ */ ++#define GPU_FREQ_KHZ_MIN (5000) ++ ++/** ++ * CPU_SPEED_FUNC ++ * - A pointer to a function that calculates the CPU clock ++ * ++ * CPU clock speed of the platform is in MHz ++ * - see kbase_cpu_clk_speed_func for the function prototype. ++ * ++ * Attached value: A kbase_cpu_clk_speed_func. ++ * Default Value: NA ++ */ ++#define CPU_SPEED_FUNC (NULL) ++ ++/** ++ * GPU_SPEED_FUNC ++ * - A pointer to a function that calculates the GPU clock ++ * ++ * GPU clock speed of the platform in MHz ++ * - see kbase_gpu_clk_speed_func for the function prototype. ++ * ++ * Attached value: A kbase_gpu_clk_speed_func. ++ * Default Value: NA ++ */ ++#define GPU_SPEED_FUNC (NULL) ++ ++/** ++ * Power management configuration ++ * ++ * Attached value: ++ * pointer to @ref kbase_pm_callback_conf ++ * Default value: ++ * See @ref kbase_pm_callback_conf ++ */ ++#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks) ++extern struct kbase_pm_callback_conf pm_callbacks; ++ ++/** ++ * Platform specific configuration functions ++ * ++ * Attached value: ++ * pointer to @ref kbase_platform_funcs_conf ++ * Default value: ++ * See @ref kbase_platform_funcs_conf ++ */ ++#define PLATFORM_FUNCS (&platform_funcs) ++extern struct kbase_platform_funcs_conf platform_funcs; ++ ++/** ++ * Secure mode switch ++ * ++ * Attached value: pointer to @ref kbase_secure_ops ++ */ ++#define SECURE_CALLBACKS (NULL) ++ +diff --git a/drivers/gpu/arm/bifrost/platform/rk/mali_kbase_config_rk.c b/drivers/gpu/arm/bifrost/platform/rk/mali_kbase_config_rk.c +new file mode 100755 +index 000000000000..e73ef450d135 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/platform/rk/mali_kbase_config_rk.c +@@ -0,0 +1,459 @@ ++/* ++ * (C) COPYRIGHT RockChip Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ */ ++ ++/* #define ENABLE_DEBUG_LOG */ ++#include "custom_log.h" ++ ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "mali_kbase_rk.h" ++ ++/** ++ * @file mali_kbase_config_rk.c ++ * 对 platform_config_of_rk 的具体实现. ++ * ++ * mali_device_driver 包å«ä¸¤éƒ¨åˆ† : ++ * .DP : platform_dependent_part_in_mdd : ++ * ä¾èµ– platform 部分, ++ * æºç åœ¨ /platform// ++ * 在 mali_device_driver 内部, ++ * 记为 platform_dependent_part, ++ * 也被记为 platform_specific_code. ++ * .DP : common_parts_in_mdd : ++ * arm 实现的通用的部分, ++ * æºç åœ¨ / 下. ++ * 在 mali_device_driver 内部, 记为 common_parts. ++ */ ++ ++/*---------------------------------------------------------------------------*/ ++ ++#ifdef CONFIG_REGULATOR ++static int rk_pm_enable_regulator(struct kbase_device *kbdev); ++static void rk_pm_disable_regulator(struct kbase_device *kbdev); ++#else ++static inline int rk_pm_enable_regulator(struct kbase_device *kbdev) ++{ ++ return 0; ++} ++ ++static inline void rk_pm_disable_regulator(struct kbase_device *kbdev) ++{ ++} ++#endif ++ ++static int rk_pm_enable_clk(struct kbase_device *kbdev); ++ ++static void rk_pm_disable_clk(struct kbase_device *kbdev); ++ ++static int kbase_platform_rk_create_sysfs_files(struct device *dev); ++ ++static void kbase_platform_rk_remove_sysfs_files(struct device *dev); ++ ++/*---------------------------------------------------------------------------*/ ++ ++static void rk_pm_power_off_delay_work(struct work_struct *work) ++{ ++ struct rk_context *platform = ++ container_of(to_delayed_work(work), struct rk_context, work); ++ struct kbase_device *kbdev = platform->kbdev; ++ ++ if (!platform->is_powered) { ++ D("mali_dev is already powered off."); ++ return; ++ } ++ ++ if (pm_runtime_enabled(kbdev->dev)) { ++ D("to put_sync_suspend mali_dev."); ++ pm_runtime_put_sync_suspend(kbdev->dev); ++ } ++ ++ rk_pm_disable_regulator(kbdev); ++ ++ platform->is_powered = false; ++ wake_unlock(&platform->wake_lock); ++} ++ ++static int kbase_platform_rk_init(struct kbase_device *kbdev) ++{ ++ int ret = 0; ++ struct rk_context *platform; ++ ++ platform = kzalloc(sizeof(*platform), GFP_KERNEL); ++ if (!platform) { ++ E("err."); ++ return -ENOMEM; ++ } ++ ++ platform->is_powered = false; ++ platform->kbdev = kbdev; ++ ++ platform->delay_ms = 200; ++ if (of_property_read_u32(kbdev->dev->of_node, "power-off-delay-ms", ++ &platform->delay_ms)) ++ W("power-off-delay-ms not available."); ++ ++ platform->power_off_wq = create_freezable_workqueue("gpu_power_off_wq"); ++ if (!platform->power_off_wq) { ++ E("couldn't create workqueue"); ++ ret = -ENOMEM; ++ goto err_wq; ++ } ++ INIT_DEFERRABLE_WORK(&platform->work, rk_pm_power_off_delay_work); ++ ++ wake_lock_init(&platform->wake_lock, WAKE_LOCK_SUSPEND, "gpu"); ++ ++ platform->utilisation_period = DEFAULT_UTILISATION_PERIOD_IN_MS; ++ ++ ret = kbase_platform_rk_create_sysfs_files(kbdev->dev); ++ if (ret) { ++ E("fail to create sysfs_files. ret = %d.", ret); ++ goto err_sysfs_files; ++ } ++ ++ kbdev->platform_context = (void *)platform; ++ pm_runtime_enable(kbdev->dev); ++ ++ return 0; ++ ++err_sysfs_files: ++ wake_lock_destroy(&platform->wake_lock); ++ destroy_workqueue(platform->power_off_wq); ++err_wq: ++ return ret; ++} ++ ++static void kbase_platform_rk_term(struct kbase_device *kbdev) ++{ ++ struct rk_context *platform = ++ (struct rk_context *)kbdev->platform_context; ++ ++ pm_runtime_disable(kbdev->dev); ++ kbdev->platform_context = NULL; ++ ++ if (platform) { ++ cancel_delayed_work_sync(&platform->work); ++ wake_lock_destroy(&platform->wake_lock); ++ destroy_workqueue(platform->power_off_wq); ++ platform->is_powered = false; ++ platform->kbdev = NULL; ++ kfree(platform); ++ } ++ kbase_platform_rk_remove_sysfs_files(kbdev->dev); ++} ++ ++struct kbase_platform_funcs_conf platform_funcs = { ++ .platform_init_func = &kbase_platform_rk_init, ++ .platform_term_func = &kbase_platform_rk_term, ++}; ++ ++/*---------------------------------------------------------------------------*/ ++ ++static int rk_pm_callback_runtime_on(struct kbase_device *kbdev) ++{ ++ return 0; ++} ++ ++static void rk_pm_callback_runtime_off(struct kbase_device *kbdev) ++{ ++} ++ ++static int rk_pm_callback_power_on(struct kbase_device *kbdev) ++{ ++ int ret = 1; /* Assume GPU has been powered off */ ++ int err = 0; ++ struct rk_context *platform = get_rk_context(kbdev); ++ ++ cancel_delayed_work_sync(&platform->work); ++ ++ err = rk_pm_enable_clk(kbdev); ++ if (err) { ++ E("failed to enable clk: %d", err); ++ return err; ++ } ++ ++ if (platform->is_powered) { ++ D("mali_device is already powered."); ++ return 0; ++ } ++ ++ /* we must enable vdd_gpu before pd_gpu_in_chip. */ ++ err = rk_pm_enable_regulator(kbdev); ++ if (err) { ++ E("fail to enable regulator, err : %d.", err); ++ return err; ++ } ++ ++ /* è‹¥ mali_dev çš„ runtime_pm 是 enabled çš„, 则... */ ++ if (pm_runtime_enabled(kbdev->dev)) { ++ D("to resume mali_dev syncly."); ++ /* 对 pd_in_chip çš„ on æ“作, ++ * 将在 pm_domain çš„ runtime_pm_callbacks 中完æˆ. ++ */ ++ err = pm_runtime_get_sync(kbdev->dev); ++ if (err < 0) { ++ E("failed to runtime resume device: %d.", err); ++ return err; ++ } else if (err == 1) { /* runtime_pm_status is still active */ ++ D("chip has NOT been powered off, no need to re-init."); ++ ret = 0; ++ } ++ } ++ ++ platform->is_powered = true; ++ wake_lock(&platform->wake_lock); ++ ++ return ret; ++} ++ ++static void rk_pm_callback_power_off(struct kbase_device *kbdev) ++{ ++ struct rk_context *platform = get_rk_context(kbdev); ++ ++ rk_pm_disable_clk(kbdev); ++ queue_delayed_work(platform->power_off_wq, &platform->work, ++ msecs_to_jiffies(platform->delay_ms)); ++} ++ ++int rk_kbase_device_runtime_init(struct kbase_device *kbdev) ++{ ++ return 0; ++} ++ ++void rk_kbase_device_runtime_disable(struct kbase_device *kbdev) ++{ ++} ++ ++struct kbase_pm_callback_conf pm_callbacks = { ++ .power_on_callback = rk_pm_callback_power_on, ++ .power_off_callback = rk_pm_callback_power_off, ++#ifdef CONFIG_PM ++ .power_runtime_init_callback = rk_kbase_device_runtime_init, ++ .power_runtime_term_callback = rk_kbase_device_runtime_disable, ++ .power_runtime_on_callback = rk_pm_callback_runtime_on, ++ .power_runtime_off_callback = rk_pm_callback_runtime_off, ++#else /* CONFIG_PM */ ++ .power_runtime_init_callback = NULL, ++ .power_runtime_term_callback = NULL, ++ .power_runtime_on_callback = NULL, ++ .power_runtime_off_callback = NULL, ++#endif /* CONFIG_PM */ ++}; ++ ++int kbase_platform_early_init(void) ++{ ++ /* Nothing needed at this stage */ ++ return 0; ++} ++ ++/*---------------------------------------------------------------------------*/ ++ ++void kbase_platform_rk_shutdown(struct kbase_device *kbdev) ++{ ++ I("to make vdd_gpu enabled for turning off pd_gpu in pm_framework."); ++ rk_pm_enable_regulator(kbdev); ++} ++ ++/*---------------------------------------------------------------------------*/ ++ ++#ifdef CONFIG_REGULATOR ++static int rk_pm_enable_regulator(struct kbase_device *kbdev) ++{ ++ int ret = 0; ++ unsigned int i; ++ ++ for (i = 0; i < kbdev->nr_regulators; i++) { ++ struct regulator *regulator = kbdev->regulators[i]; ++ if (!regulator) { ++ W("no mali regulator control, no need to enable."); ++ goto EXIT; ++ } ++ ++ D("to enable regulator."); ++ ret = regulator_enable(regulator); ++ if (ret) { ++ E("fail to enable regulator, ret : %d.", ret); ++ goto EXIT; ++ } ++ } ++ ++EXIT: ++ return ret; ++} ++ ++static void rk_pm_disable_regulator(struct kbase_device *kbdev) ++{ ++ unsigned int i; ++ ++ for (i = 0; i < kbdev->nr_regulators; i++) { ++ struct regulator *regulator = kbdev->regulators[i]; ++ ++ if (!regulator) { ++ W("no mali regulator control, no need to disable."); ++ return; ++ } ++ ++ D("to disable regulator."); ++ regulator_disable(regulator); ++ } ++} ++#endif ++ ++static int rk_pm_enable_clk(struct kbase_device *kbdev) ++{ ++ int err = 0; ++ unsigned int i; ++ ++ for (i = 0; i < kbdev->nr_clocks; i++) { ++ struct clk *clock = kbdev->clocks[i]; ++ ++ if (!clock) { ++ W("no mali clock control, no need to enable."); ++ } else { ++ D("to enable clk."); ++ err = clk_enable(clock); ++ if (err) ++ E("failed to enable clk: %d.", err); ++ } ++ } ++ ++ return err; ++} ++ ++static void rk_pm_disable_clk(struct kbase_device *kbdev) ++{ ++ unsigned int i; ++ ++ for (i = 0; i < kbdev->nr_clocks; i++) { ++ struct clk *clock = kbdev->clocks[i]; ++ ++ if (!clock) { ++ W("no mali clock control, no need to disable."); ++ } else { ++ D("to disable clk."); ++ clk_disable(clock); ++ } ++ } ++} ++ ++/*---------------------------------------------------------------------------*/ ++ ++static ssize_t utilisation_period_show(struct device *dev, ++ struct device_attribute *attr, ++ char *buf) ++{ ++ struct kbase_device *kbdev = dev_get_drvdata(dev); ++ struct rk_context *platform = get_rk_context(kbdev); ++ ssize_t ret = 0; ++ ++ ret += snprintf(buf, PAGE_SIZE, "%u\n", platform->utilisation_period); ++ ++ return ret; ++} ++ ++static ssize_t utilisation_period_store(struct device *dev, ++ struct device_attribute *attr, ++ const char *buf, ++ size_t count) ++{ ++ struct kbase_device *kbdev = dev_get_drvdata(dev); ++ struct rk_context *platform = get_rk_context(kbdev); ++ int ret = 0; ++ ++ ret = kstrtouint(buf, 0, &platform->utilisation_period); ++ if (ret) { ++ E("invalid input period : %s.", buf); ++ return ret; ++ } ++ D("set utilisation_period to '%d'.", platform->utilisation_period); ++ ++ return count; ++} ++ ++static ssize_t utilisation_show(struct device *dev, ++ struct device_attribute *attr, ++ char *buf) ++{ ++ struct kbase_device *kbdev = dev_get_drvdata(dev); ++ struct rk_context *platform = get_rk_context(kbdev); ++ ssize_t ret = 0; ++ unsigned long period_in_us = platform->utilisation_period * 1000; ++ u32 utilisation; ++ struct kbasep_pm_metrics metrics_when_start; ++ struct kbasep_pm_metrics metrics_diff; /* between start and end. */ ++ u32 total_time = 0; ++ u32 busy_time = 0; ++ ++ /* get current metrics data. */ ++ kbase_pm_get_dvfs_metrics(kbdev, &metrics_when_start, &metrics_diff); ++ /* sleep for 'period_in_us'. */ ++ usleep_range(period_in_us, period_in_us + 100); ++ /* get metrics data between start and end. */ ++ kbase_pm_get_dvfs_metrics(kbdev, &metrics_when_start, &metrics_diff); ++ ++ total_time = metrics_diff.time_busy + metrics_diff.time_idle; ++ busy_time = metrics_diff.time_busy; ++ D("total_time : %u, busy_time : %u.", total_time, busy_time); ++ ++ utilisation = busy_time * 100 / total_time; ++ ret += snprintf(buf, PAGE_SIZE, "%d\n", utilisation); ++ ++ return ret; ++} ++ ++static DEVICE_ATTR_RW(utilisation_period); ++static DEVICE_ATTR_RO(utilisation); ++ ++static int kbase_platform_rk_create_sysfs_files(struct device *dev) ++{ ++ int ret = 0; ++ ++ ret = device_create_file(dev, &dev_attr_utilisation_period); ++ if (ret) { ++ E("fail to create sysfs file 'utilisation_period'."); ++ goto out; ++ } ++ ++ ret = device_create_file(dev, &dev_attr_utilisation); ++ if (ret) { ++ E("fail to create sysfs file 'utilisation'."); ++ goto remove_utilisation_period; ++ } ++ ++ return 0; ++ ++remove_utilisation_period: ++ device_remove_file(dev, &dev_attr_utilisation_period); ++out: ++ return ret; ++} ++ ++static void kbase_platform_rk_remove_sysfs_files(struct device *dev) ++{ ++ device_remove_file(dev, &dev_attr_utilisation_period); ++ device_remove_file(dev, &dev_attr_utilisation); ++} ++ ++int kbase_platform_rk_init_opp_table(struct kbase_device *kbdev) ++{ ++ return rockchip_init_opp_table(kbdev->dev, NULL, ++ "gpu_leakage", "mali"); ++} +diff --git a/drivers/gpu/arm/bifrost/platform/rk/mali_kbase_rk.h b/drivers/gpu/arm/bifrost/platform/rk/mali_kbase_rk.h +new file mode 100755 +index 000000000000..6eab25014d21 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/platform/rk/mali_kbase_rk.h +@@ -0,0 +1,62 @@ ++/* drivers/gpu/t6xx/kbase/src/platform/rk/mali_kbase_platform.h ++ * Rockchip SoC Mali-Midgard platform-dependent codes ++ * ++ * This program is free software; you can redistribute it and/or modify ++ * it under the terms of the GNU General Public License version 2 as ++ * published by the Free Software FoundatIon. ++ */ ++ ++/** ++ * @file mali_kbase_rk.h ++ * ++ * defines work_context type of platform_dependent_part. ++ */ ++ ++#ifndef _MALI_KBASE_RK_H_ ++#define _MALI_KBASE_RK_H_ ++ ++#include ++ ++/*---------------------------------------------------------------------------*/ ++ ++#define DEFAULT_UTILISATION_PERIOD_IN_MS (100) ++ ++/*---------------------------------------------------------------------------*/ ++ ++/* ++ * struct rk_context - work_context of platform_dependent_part_of_rk. ++ */ ++struct rk_context { ++ /* ++ * record the status of common_parts calling 'power_on_callback' ++ * and 'power_off_callback'. ++ */ ++ bool is_powered; ++ ++ struct kbase_device *kbdev; ++ ++ struct workqueue_struct *power_off_wq; ++ /* delayed_work_to_power_off_gpu. */ ++ struct delayed_work work; ++ unsigned int delay_ms; ++ ++ /* ++ * WAKE_LOCK_SUSPEND for ensuring to run ++ * delayed_work_to_power_off_gpu before suspend. ++ */ ++ struct wake_lock wake_lock; ++ ++ /* debug only, the period in ms to count gpu_utilisation. */ ++ unsigned int utilisation_period; ++}; ++ ++/*---------------------------------------------------------------------------*/ ++ ++static inline struct rk_context *get_rk_context( ++ const struct kbase_device *kbdev) ++{ ++ return (struct rk_context *)(kbdev->platform_context); ++} ++ ++#endif /* _MALI_KBASE_RK_H_ */ ++ +diff --git a/drivers/gpu/arm/bifrost/platform/vexpress/Kbuild b/drivers/gpu/arm/bifrost/platform/vexpress/Kbuild +new file mode 100755 +index 000000000000..6780e4c9433b +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/platform/vexpress/Kbuild +@@ -0,0 +1,24 @@ ++# ++# (C) COPYRIGHT 2012-2013, 2016-2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# GNU General Public License for more details. ++# ++# You should have received a copy of the GNU General Public License ++# along with this program; if not, you can access it online at ++# http://www.gnu.org/licenses/gpl-2.0.html. ++# ++# SPDX-License-Identifier: GPL-2.0 ++# ++# ++ ++mali_kbase-y += \ ++ $(MALI_PLATFORM_DIR)/mali_kbase_config_vexpress.o \ ++ mali_kbase_platform_fake.o +diff --git a/drivers/gpu/arm/bifrost/platform/vexpress/mali_kbase_config_platform.h b/drivers/gpu/arm/bifrost/platform/vexpress/mali_kbase_config_platform.h +new file mode 100755 +index 000000000000..fac3cd52182f +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/platform/vexpress/mali_kbase_config_platform.h +@@ -0,0 +1,39 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/** ++ * Power management configuration ++ * ++ * Attached value: pointer to @ref kbase_pm_callback_conf ++ * Default value: See @ref kbase_pm_callback_conf ++ */ ++#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks) ++ ++/** ++ * Platform specific configuration functions ++ * ++ * Attached value: pointer to @ref kbase_platform_funcs_conf ++ * Default value: See @ref kbase_platform_funcs_conf ++ */ ++#define PLATFORM_FUNCS (NULL) ++ ++extern struct kbase_pm_callback_conf pm_callbacks; +diff --git a/drivers/gpu/arm/bifrost/platform/vexpress/mali_kbase_config_vexpress.c b/drivers/gpu/arm/bifrost/platform/vexpress/mali_kbase_config_vexpress.c +new file mode 100755 +index 000000000000..d165ce262814 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/platform/vexpress/mali_kbase_config_vexpress.c +@@ -0,0 +1,69 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++#include ++#include ++#include ++#include ++#include "mali_kbase_config_platform.h" ++ ++#ifndef CONFIG_OF ++static struct kbase_io_resources io_resources = { ++ .job_irq_number = 68, ++ .mmu_irq_number = 69, ++ .gpu_irq_number = 70, ++ .io_memory_region = { ++ .start = 0xFC010000, ++ .end = 0xFC010000 + (4096 * 4) - 1 ++ } ++}; ++#endif /* CONFIG_OF */ ++ ++static int pm_callback_power_on(struct kbase_device *kbdev) ++{ ++ /* Nothing is needed on VExpress, but we may have destroyed GPU state (if the below HARD_RESET code is active) */ ++ return 1; ++} ++ ++static void pm_callback_power_off(struct kbase_device *kbdev) ++{ ++} ++ ++struct kbase_pm_callback_conf pm_callbacks = { ++ .power_on_callback = pm_callback_power_on, ++ .power_off_callback = pm_callback_power_off, ++ .power_suspend_callback = NULL, ++ .power_resume_callback = NULL ++}; ++ ++static struct kbase_platform_config versatile_platform_config = { ++#ifndef CONFIG_OF ++ .io_resources = &io_resources ++#endif ++}; ++ ++struct kbase_platform_config *kbase_get_platform_config(void) ++{ ++ return &versatile_platform_config; ++} +diff --git a/drivers/gpu/arm/bifrost/platform/vexpress_1xv7_a57/Kbuild b/drivers/gpu/arm/bifrost/platform/vexpress_1xv7_a57/Kbuild +new file mode 100755 +index 000000000000..51b408efd48a +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/platform/vexpress_1xv7_a57/Kbuild +@@ -0,0 +1,24 @@ ++# ++# (C) COPYRIGHT 2013-2014, 2016-2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# GNU General Public License for more details. ++# ++# You should have received a copy of the GNU General Public License ++# along with this program; if not, you can access it online at ++# http://www.gnu.org/licenses/gpl-2.0.html. ++# ++# SPDX-License-Identifier: GPL-2.0 ++# ++# ++ ++mali_kbase-y += \ ++ $(MALI_PLATFORM_DIR)/mali_kbase_config_vexpress.o \ ++ mali_kbase_platform_fake.o +diff --git a/drivers/gpu/arm/bifrost/platform/vexpress_1xv7_a57/mali_kbase_config_platform.h b/drivers/gpu/arm/bifrost/platform/vexpress_1xv7_a57/mali_kbase_config_platform.h +new file mode 100755 +index 000000000000..fac3cd52182f +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/platform/vexpress_1xv7_a57/mali_kbase_config_platform.h +@@ -0,0 +1,39 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/** ++ * Power management configuration ++ * ++ * Attached value: pointer to @ref kbase_pm_callback_conf ++ * Default value: See @ref kbase_pm_callback_conf ++ */ ++#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks) ++ ++/** ++ * Platform specific configuration functions ++ * ++ * Attached value: pointer to @ref kbase_platform_funcs_conf ++ * Default value: See @ref kbase_platform_funcs_conf ++ */ ++#define PLATFORM_FUNCS (NULL) ++ ++extern struct kbase_pm_callback_conf pm_callbacks; +diff --git a/drivers/gpu/arm/bifrost/platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c b/drivers/gpu/arm/bifrost/platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c +new file mode 100755 +index 000000000000..efca0a5b3493 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c +@@ -0,0 +1,65 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2014, 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++#include ++#include ++#include ++ ++#ifndef CONFIG_OF ++static struct kbase_io_resources io_resources = { ++ .job_irq_number = 68, ++ .mmu_irq_number = 69, ++ .gpu_irq_number = 70, ++ .io_memory_region = { ++ .start = 0x2f010000, ++ .end = 0x2f010000 + (4096 * 4) - 1} ++}; ++#endif ++ ++static int pm_callback_power_on(struct kbase_device *kbdev) ++{ ++ /* Nothing is needed on VExpress, but we may have destroyed GPU state (if the below HARD_RESET code is active) */ ++ return 1; ++} ++ ++static void pm_callback_power_off(struct kbase_device *kbdev) ++{ ++} ++ ++struct kbase_pm_callback_conf pm_callbacks = { ++ .power_on_callback = pm_callback_power_on, ++ .power_off_callback = pm_callback_power_off, ++ .power_suspend_callback = NULL, ++ .power_resume_callback = NULL ++}; ++ ++static struct kbase_platform_config versatile_platform_config = { ++#ifndef CONFIG_OF ++ .io_resources = &io_resources ++#endif ++}; ++ ++struct kbase_platform_config *kbase_get_platform_config(void) ++{ ++ return &versatile_platform_config; ++} +diff --git a/drivers/gpu/arm/bifrost/platform/vexpress_6xvirtex7_10mhz/Kbuild b/drivers/gpu/arm/bifrost/platform/vexpress_6xvirtex7_10mhz/Kbuild +new file mode 100755 +index 000000000000..e07709c9b1a5 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/platform/vexpress_6xvirtex7_10mhz/Kbuild +@@ -0,0 +1,25 @@ ++# ++# (C) COPYRIGHT 2012-2013, 2016-2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# GNU General Public License for more details. ++# ++# You should have received a copy of the GNU General Public License ++# along with this program; if not, you can access it online at ++# http://www.gnu.org/licenses/gpl-2.0.html. ++# ++# SPDX-License-Identifier: GPL-2.0 ++# ++# ++ ++mali_kbase-y += \ ++ $(MALI_PLATFORM_DIR)/mali_kbase_config_vexpress.o \ ++ $(MALI_PLATFORM_DIR)/mali_kbase_cpu_vexpress.o \ ++ mali_kbase_platform_fake.o +diff --git a/drivers/gpu/arm/bifrost/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_platform.h b/drivers/gpu/arm/bifrost/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_platform.h +new file mode 100755 +index 000000000000..fac3cd52182f +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_platform.h +@@ -0,0 +1,39 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/** ++ * Power management configuration ++ * ++ * Attached value: pointer to @ref kbase_pm_callback_conf ++ * Default value: See @ref kbase_pm_callback_conf ++ */ ++#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks) ++ ++/** ++ * Platform specific configuration functions ++ * ++ * Attached value: pointer to @ref kbase_platform_funcs_conf ++ * Default value: See @ref kbase_platform_funcs_conf ++ */ ++#define PLATFORM_FUNCS (NULL) ++ ++extern struct kbase_pm_callback_conf pm_callbacks; +diff --git a/drivers/gpu/arm/bifrost/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_vexpress.c b/drivers/gpu/arm/bifrost/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_vexpress.c +new file mode 100755 +index 000000000000..b6714b95b776 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_vexpress.c +@@ -0,0 +1,67 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2014, 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++ ++ ++#include ++#include ++#include ++#include ++ ++#ifndef CONFIG_OF ++static struct kbase_io_resources io_resources = { ++ .job_irq_number = 75, ++ .mmu_irq_number = 76, ++ .gpu_irq_number = 77, ++ .io_memory_region = { ++ .start = 0x2F000000, ++ .end = 0x2F000000 + (4096 * 4) - 1} ++}; ++#endif ++ ++static int pm_callback_power_on(struct kbase_device *kbdev) ++{ ++ /* Nothing is needed on VExpress, but we may have destroyed GPU state (if the below HARD_RESET code is active) */ ++ return 1; ++} ++ ++static void pm_callback_power_off(struct kbase_device *kbdev) ++{ ++} ++ ++struct kbase_pm_callback_conf pm_callbacks = { ++ .power_on_callback = pm_callback_power_on, ++ .power_off_callback = pm_callback_power_off, ++ .power_suspend_callback = NULL, ++ .power_resume_callback = NULL ++}; ++ ++static struct kbase_platform_config versatile_platform_config = { ++#ifndef CONFIG_OF ++ .io_resources = &io_resources ++#endif ++}; ++ ++struct kbase_platform_config *kbase_get_platform_config(void) ++{ ++ return &versatile_platform_config; ++} +diff --git a/drivers/gpu/arm/bifrost/protected_mode_switcher.h b/drivers/gpu/arm/bifrost/protected_mode_switcher.h +new file mode 100755 +index 000000000000..8778d812aea0 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/protected_mode_switcher.h +@@ -0,0 +1,69 @@ ++/* ++ * ++ * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _PROTECTED_MODE_SWITCH_H_ ++#define _PROTECTED_MODE_SWITCH_H_ ++ ++struct protected_mode_device; ++ ++/** ++ * struct protected_mode_ops - Callbacks for protected mode switch operations ++ * ++ * @protected_mode_enable: Callback to enable protected mode for device ++ * @protected_mode_disable: Callback to disable protected mode for device ++ */ ++struct protected_mode_ops { ++ /** ++ * protected_mode_enable() - Enable protected mode on device ++ * @dev: The struct device ++ * ++ * Return: 0 on success, non-zero on error ++ */ ++ int (*protected_mode_enable)( ++ struct protected_mode_device *protected_dev); ++ ++ /** ++ * protected_mode_disable() - Disable protected mode on device, and ++ * reset device ++ * @dev: The struct device ++ * ++ * Return: 0 on success, non-zero on error ++ */ ++ int (*protected_mode_disable)( ++ struct protected_mode_device *protected_dev); ++}; ++ ++/** ++ * struct protected_mode_device - Device structure for protected mode devices ++ * ++ * @ops - Callbacks associated with this device ++ * @data - Pointer to device private data ++ * ++ * This structure should be registered with the platform device using ++ * platform_set_drvdata(). ++ */ ++struct protected_mode_device { ++ struct protected_mode_ops ops; ++ void *data; ++}; ++ ++#endif /* _PROTECTED_MODE_SWITCH_H_ */ +diff --git a/drivers/gpu/arm/bifrost/tests/Kbuild b/drivers/gpu/arm/bifrost/tests/Kbuild +new file mode 100755 +index 000000000000..c26bef780781 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/tests/Kbuild +@@ -0,0 +1,24 @@ ++# ++# (C) COPYRIGHT 2017, 2020 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# GNU General Public License for more details. ++# ++# You should have received a copy of the GNU General Public License ++# along with this program; if not, you can access it online at ++# http://www.gnu.org/licenses/gpl-2.0.html. ++# ++# SPDX-License-Identifier: GPL-2.0 ++# ++# ++ ++obj-$(CONFIG_MALI_KUTF) += kutf/ ++obj-$(CONFIG_MALI_IRQ_LATENCY) += mali_kutf_irq_test/ ++obj-$(CONFIG_MALI_CLK_RATE_TRACE_PORTAL) += mali_kutf_clk_rate_trace/kernel/ +diff --git a/drivers/gpu/arm/bifrost/tests/Kconfig b/drivers/gpu/arm/bifrost/tests/Kconfig +new file mode 100755 +index 000000000000..83a4d7764a50 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/tests/Kconfig +@@ -0,0 +1,24 @@ ++# ++# (C) COPYRIGHT 2017, 2020 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# GNU General Public License for more details. ++# ++# You should have received a copy of the GNU General Public License ++# along with this program; if not, you can access it online at ++# http://www.gnu.org/licenses/gpl-2.0.html. ++# ++# SPDX-License-Identifier: GPL-2.0 ++# ++# ++ ++source "drivers/gpu/arm/midgard/tests/kutf/Kconfig" ++source "drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Kconfig" ++source "drivers/gpu/arm/midgard/tests/mali_kutf_clk_rate_trace/kernel/Kconfig" +diff --git a/drivers/gpu/arm/bifrost/tests/Mconfig b/drivers/gpu/arm/bifrost/tests/Mconfig +new file mode 100755 +index 000000000000..bba96b3d9e48 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/tests/Mconfig +@@ -0,0 +1,38 @@ ++# ++# (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++ ++config UNIT_TEST_KERNEL_MODULES ++ bool ++ default y if UNIT_TEST_CODE && BUILD_KERNEL_MODULES ++ default n ++ ++config BUILD_IPA_TESTS ++ bool ++ default y if UNIT_TEST_KERNEL_MODULES && MALI_BIFROST_DEVFREQ ++ default n ++ ++config BUILD_IPA_UNIT_TESTS ++ bool ++ default y if NO_MALI && BUILD_IPA_TESTS ++ default n ++ ++config BUILD_CSF_TESTS ++ bool ++ default y if UNIT_TEST_KERNEL_MODULES && GPU_HAS_CSF ++ default n ++ ++config BUILD_ARBIF_TESTS ++ bool ++ default y if UNIT_TEST_KERNEL_MODULES && MALI_ARBITER_SUPPORT ++ default n ++ +diff --git a/drivers/gpu/arm/bifrost/tests/include/kutf/kutf_helpers.h b/drivers/gpu/arm/bifrost/tests/include/kutf/kutf_helpers.h +new file mode 100755 +index 000000000000..858b9c38b49a +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/tests/include/kutf/kutf_helpers.h +@@ -0,0 +1,85 @@ ++/* ++ * ++ * (C) COPYRIGHT 2017, 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KERNEL_UTF_HELPERS_H_ ++#define _KERNEL_UTF_HELPERS_H_ ++ ++/* kutf_helpers.h ++ * Test helper functions for the kernel UTF test infrastructure. ++ * ++ * These functions provide methods for enqueuing/dequeuing lines of text sent ++ * by user space. They are used to implement the transfer of "userdata" from ++ * user space to kernel. ++ */ ++ ++#include ++ ++/** ++ * kutf_helper_pending_input() - Check any pending lines sent by user space ++ * @context: KUTF context ++ * ++ * Return: true if there are pending lines, otherwise false ++ */ ++bool kutf_helper_pending_input(struct kutf_context *context); ++ ++/** ++ * kutf_helper_input_dequeue() - Dequeue a line sent by user space ++ * @context: KUTF context ++ * @str_size: Pointer to an integer to receive the size of the string ++ * ++ * If no line is available then this function will wait (interruptibly) until ++ * a line is available. ++ * ++ * Return: The line dequeued, ERR_PTR(-EINTR) if interrupted or NULL on end ++ * of data. ++ */ ++char *kutf_helper_input_dequeue(struct kutf_context *context, size_t *str_size); ++ ++/** ++ * kutf_helper_input_enqueue() - Enqueue a line sent by user space ++ * @context: KUTF context ++ * @str: The user space address of the line ++ * @size: The length in bytes of the string ++ * ++ * This function will use copy_from_user to copy the string out of user space. ++ * The string need not be NULL-terminated (@size should not include the NULL ++ * termination). ++ * ++ * As a special case @str==NULL and @size==0 is valid to mark the end of input, ++ * but callers should use kutf_helper_input_enqueue_end_of_data() instead. ++ * ++ * Return: 0 on success, -EFAULT if the line cannot be copied from user space, ++ * -ENOMEM if out of memory. ++ */ ++int kutf_helper_input_enqueue(struct kutf_context *context, ++ const char __user *str, size_t size); ++ ++/** ++ * kutf_helper_input_enqueue_end_of_data() - Signal no more data is to be sent ++ * @context: KUTF context ++ * ++ * After this function has been called, kutf_helper_input_dequeue() will always ++ * return NULL. ++ */ ++void kutf_helper_input_enqueue_end_of_data(struct kutf_context *context); ++ ++#endif /* _KERNEL_UTF_HELPERS_H_ */ +diff --git a/drivers/gpu/arm/bifrost/tests/include/kutf/kutf_helpers_user.h b/drivers/gpu/arm/bifrost/tests/include/kutf/kutf_helpers_user.h +new file mode 100755 +index 000000000000..3b1300e1ce6f +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/tests/include/kutf/kutf_helpers_user.h +@@ -0,0 +1,179 @@ ++/* ++ * ++ * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KERNEL_UTF_HELPERS_USER_H_ ++#define _KERNEL_UTF_HELPERS_USER_H_ ++ ++/* kutf_helpers.h ++ * Test helper functions for the kernel UTF test infrastructure, whose ++ * implementation mirrors that of similar functions for kutf-userside ++ */ ++ ++#include ++#include ++ ++ ++#define KUTF_HELPER_MAX_VAL_NAME_LEN 255 ++ ++enum kutf_helper_valtype { ++ KUTF_HELPER_VALTYPE_INVALID, ++ KUTF_HELPER_VALTYPE_U64, ++ KUTF_HELPER_VALTYPE_STR, ++ ++ KUTF_HELPER_VALTYPE_COUNT /* Must be last */ ++}; ++ ++struct kutf_helper_named_val { ++ enum kutf_helper_valtype type; ++ char *val_name; ++ union { ++ u64 val_u64; ++ char *val_str; ++ } u; ++}; ++ ++/* Extra error values for certain helpers when we want to distinguish between ++ * Linux's own error values too. ++ * ++ * These can only be used on certain functions returning an int type that are ++ * documented as returning one of these potential values, they cannot be used ++ * from functions return a ptr type, since we can't decode it with PTR_ERR ++ * ++ * No negative values are used - Linux error codes should be used instead, and ++ * indicate a problem in accessing the data file itself (are generally ++ * unrecoverable) ++ * ++ * Positive values indicate correct access but invalid parsing (can be ++ * recovered from assuming data in the future is correct) */ ++enum kutf_helper_err { ++ /* No error - must be zero */ ++ KUTF_HELPER_ERR_NONE = 0, ++ /* Named value parsing encountered an invalid name */ ++ KUTF_HELPER_ERR_INVALID_NAME, ++ /* Named value parsing of string or u64 type encountered extra ++ * characters after the value (after the last digit for a u64 type or ++ * after the string end delimiter for string type) */ ++ KUTF_HELPER_ERR_CHARS_AFTER_VAL, ++ /* Named value parsing of string type couldn't find the string end ++ * delimiter. ++ * ++ * This cannot be encountered when the NAME="value" message exceeds the ++ * textbuf's maximum line length, because such messages are not checked ++ * for an end string delimiter */ ++ KUTF_HELPER_ERR_NO_END_DELIMITER, ++ /* Named value didn't parse as any of the known types */ ++ KUTF_HELPER_ERR_INVALID_VALUE, ++}; ++ ++ ++/* Send named NAME=value pair, u64 value ++ * ++ * NAME must match [A-Z0-9_]\+ and can be up to MAX_VAL_NAME_LEN characters long ++ * ++ * Any failure will be logged on the suite's current test fixture ++ * ++ * Returns 0 on success, non-zero on failure ++ */ ++int kutf_helper_send_named_u64(struct kutf_context *context, ++ const char *val_name, u64 val); ++ ++/* Get the maximum length of a string that can be represented as a particular ++ * NAME="value" pair without string-value truncation in the kernel's buffer ++ * ++ * Given val_name and the kernel buffer's size, this can be used to determine ++ * the maximum length of a string that can be sent as val_name="value" pair ++ * without having the string value truncated. Any string longer than this will ++ * be truncated at some point during communication to this size. ++ * ++ * It is assumed that val_name is a valid name for ++ * kutf_helper_send_named_str(), and no checking will be made to ++ * ensure this. ++ * ++ * Returns the maximum string length that can be represented, or a negative ++ * value if the NAME="value" encoding itself wouldn't fit in kern_buf_sz ++ */ ++int kutf_helper_max_str_len_for_kern(const char *val_name, int kern_buf_sz); ++ ++/* Send named NAME="str" pair ++ * ++ * no escaping allowed in str. Any of the following characters will terminate ++ * the string: '"' '\\' '\n' ++ * ++ * NAME must match [A-Z0-9_]\+ and can be up to MAX_VAL_NAME_LEN characters long ++ * ++ * Any failure will be logged on the suite's current test fixture ++ * ++ * Returns 0 on success, non-zero on failure */ ++int kutf_helper_send_named_str(struct kutf_context *context, ++ const char *val_name, const char *val_str); ++ ++/* Receive named NAME=value pair ++ * ++ * This can receive u64 and string values - check named_val->type ++ * ++ * If you are not planning on dynamic handling of the named value's name and ++ * type, then kutf_helper_receive_check_val() is more useful as a ++ * convenience function. ++ * ++ * String members of named_val will come from memory allocated on the fixture's mempool ++ * ++ * Returns 0 on success. Negative value on failure to receive from the 'run' ++ * file, positive value indicates an enum kutf_helper_err value for correct ++ * reception of data but invalid parsing */ ++int kutf_helper_receive_named_val( ++ struct kutf_context *context, ++ struct kutf_helper_named_val *named_val); ++ ++/* Receive and validate NAME=value pair ++ * ++ * As with kutf_helper_receive_named_val, but validate that the ++ * name and type are as expected, as a convenience for a common pattern found ++ * in tests. ++ * ++ * NOTE: this only returns an error value if there was actually a problem ++ * receiving data. ++ * ++ * NOTE: If the underlying data was received correctly, but: ++ * - isn't of the expected name ++ * - isn't the expected type ++ * - isn't correctly parsed for the type ++ * then the following happens: ++ * - failure result is recorded ++ * - named_val->type will be KUTF_HELPER_VALTYPE_INVALID ++ * - named_val->u will contain some default value that should be relatively ++ * harmless for the test, including being writable in the case of string ++ * values ++ * - return value will be 0 to indicate success ++ * ++ * The rationale behind this is that we'd prefer to continue the rest of the ++ * test with failures propagated, rather than hitting a timeout */ ++int kutf_helper_receive_check_val( ++ struct kutf_helper_named_val *named_val, ++ struct kutf_context *context, ++ const char *expect_val_name, ++ enum kutf_helper_valtype expect_val_type); ++ ++/* Output a named value to kmsg */ ++void kutf_helper_output_named_val(struct kutf_helper_named_val *named_val); ++ ++ ++#endif /* _KERNEL_UTF_HELPERS_USER_H_ */ +diff --git a/drivers/gpu/arm/bifrost/tests/include/kutf/kutf_mem.h b/drivers/gpu/arm/bifrost/tests/include/kutf/kutf_mem.h +new file mode 100755 +index 000000000000..988559de1edf +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/tests/include/kutf/kutf_mem.h +@@ -0,0 +1,73 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KERNEL_UTF_MEM_H_ ++#define _KERNEL_UTF_MEM_H_ ++ ++/* kutf_mem.h ++ * Functions for management of memory pools in the kernel. ++ * ++ * This module implements a memory pool allocator, allowing a test ++ * implementation to allocate linked allocations which can then be freed by a ++ * single free which releases all of the resources held by the entire pool. ++ * ++ * Note that it is not possible to free single resources within the pool once ++ * allocated. ++ */ ++ ++#include ++#include ++ ++/** ++ * struct kutf_mempool - the memory pool context management structure ++ * @head: list head on which the allocations in this context are added to ++ * @lock: mutex for concurrent allocation from multiple threads ++ * ++ */ ++struct kutf_mempool { ++ struct list_head head; ++ struct mutex lock; ++}; ++ ++/** ++ * kutf_mempool_init() - Initialize a memory pool. ++ * @pool: Memory pool structure to initialize, provided by the user ++ * ++ * Return: zero on success ++ */ ++int kutf_mempool_init(struct kutf_mempool *pool); ++ ++/** ++ * kutf_mempool_alloc() - Allocate memory from a pool ++ * @pool: Memory pool to allocate from ++ * @size: Size of memory wanted in number of bytes ++ * ++ * Return: Pointer to memory on success, NULL on failure. ++ */ ++void *kutf_mempool_alloc(struct kutf_mempool *pool, size_t size); ++ ++/** ++ * kutf_mempool_destroy() - Destroy a memory pool, freeing all memory within it. ++ * @pool: The memory pool to free ++ */ ++void kutf_mempool_destroy(struct kutf_mempool *pool); ++#endif /* _KERNEL_UTF_MEM_H_ */ +diff --git a/drivers/gpu/arm/bifrost/tests/include/kutf/kutf_resultset.h b/drivers/gpu/arm/bifrost/tests/include/kutf/kutf_resultset.h +new file mode 100755 +index 000000000000..49ebeb4ec546 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/tests/include/kutf/kutf_resultset.h +@@ -0,0 +1,181 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KERNEL_UTF_RESULTSET_H_ ++#define _KERNEL_UTF_RESULTSET_H_ ++ ++/* kutf_resultset.h ++ * Functions and structures for handling test results and result sets. ++ * ++ * This section of the kernel UTF contains structures and functions used for the ++ * management of Results and Result Sets. ++ */ ++ ++/** ++ * enum kutf_result_status - Status values for a single Test error. ++ * @KUTF_RESULT_BENCHMARK: Result is a meta-result containing benchmark ++ * results. ++ * @KUTF_RESULT_SKIP: The test was skipped. ++ * @KUTF_RESULT_UNKNOWN: The test has an unknown result. ++ * @KUTF_RESULT_PASS: The test result passed. ++ * @KUTF_RESULT_DEBUG: The test result passed, but raised a debug ++ * message. ++ * @KUTF_RESULT_INFO: The test result passed, but raised ++ * an informative message. ++ * @KUTF_RESULT_WARN: The test result passed, but raised a warning ++ * message. ++ * @KUTF_RESULT_FAIL: The test result failed with a non-fatal error. ++ * @KUTF_RESULT_FATAL: The test result failed with a fatal error. ++ * @KUTF_RESULT_ABORT: The test result failed due to a non-UTF ++ * assertion failure. ++ * @KUTF_RESULT_USERDATA: User data is ready to be read, ++ * this is not seen outside the kernel ++ * @KUTF_RESULT_USERDATA_WAIT: Waiting for user data to be sent, ++ * this is not seen outside the kernel ++ * @KUTF_RESULT_TEST_FINISHED: The test has finished, no more results will ++ * be produced. This is not seen outside kutf ++ */ ++enum kutf_result_status { ++ KUTF_RESULT_BENCHMARK = -3, ++ KUTF_RESULT_SKIP = -2, ++ KUTF_RESULT_UNKNOWN = -1, ++ ++ KUTF_RESULT_PASS = 0, ++ KUTF_RESULT_DEBUG = 1, ++ KUTF_RESULT_INFO = 2, ++ KUTF_RESULT_WARN = 3, ++ KUTF_RESULT_FAIL = 4, ++ KUTF_RESULT_FATAL = 5, ++ KUTF_RESULT_ABORT = 6, ++ ++ KUTF_RESULT_USERDATA = 7, ++ KUTF_RESULT_USERDATA_WAIT = 8, ++ KUTF_RESULT_TEST_FINISHED = 9 ++}; ++ ++/* The maximum size of a kutf_result_status result when ++ * converted to a string ++ */ ++#define KUTF_ERROR_MAX_NAME_SIZE 21 ++ ++#ifdef __KERNEL__ ++ ++#include ++#include ++ ++struct kutf_context; ++ ++/** ++ * struct kutf_result - Represents a single test result. ++ * @node: Next result in the list of results. ++ * @status: The status summary (pass / warn / fail / etc). ++ * @message: A more verbose status message. ++ */ ++struct kutf_result { ++ struct list_head node; ++ enum kutf_result_status status; ++ const char *message; ++}; ++ ++/** ++ * KUTF_RESULT_SET_WAITING_FOR_INPUT - Test is waiting for user data ++ * ++ * This flag is set within a struct kutf_result_set whenever the test is blocked ++ * waiting for user data. Attempts to dequeue results when this flag is set ++ * will cause a dummy %KUTF_RESULT_USERDATA_WAIT result to be produced. This ++ * is used to output a warning message and end of file. ++ */ ++#define KUTF_RESULT_SET_WAITING_FOR_INPUT 1 ++ ++/** ++ * struct kutf_result_set - Represents a set of results. ++ * @results: List head of a struct kutf_result list for storing the results ++ * @waitq: Wait queue signalled whenever new results are added. ++ * @flags: Flags see %KUTF_RESULT_SET_WAITING_FOR_INPUT ++ */ ++struct kutf_result_set { ++ struct list_head results; ++ wait_queue_head_t waitq; ++ int flags; ++}; ++ ++/** ++ * kutf_create_result_set() - Create a new result set ++ * to which results can be added. ++ * ++ * Return: The created result set. ++ */ ++struct kutf_result_set *kutf_create_result_set(void); ++ ++/** ++ * kutf_add_result() - Add a result to the end of an existing result set. ++ * ++ * @context: The kutf context ++ * @status: The result status to add. ++ * @message: The result message to add. ++ * ++ * Return: 0 if the result is successfully added. -ENOMEM if allocation fails. ++ */ ++int kutf_add_result(struct kutf_context *context, ++ enum kutf_result_status status, const char *message); ++ ++/** ++ * kutf_remove_result() - Remove a result from the head of a result set. ++ * @set: The result set. ++ * ++ * This function will block until there is a result to read. The wait is ++ * interruptible, so this function will return with an ERR_PTR if interrupted. ++ * ++ * Return: result or ERR_PTR if interrupted ++ */ ++struct kutf_result *kutf_remove_result( ++ struct kutf_result_set *set); ++ ++/** ++ * kutf_destroy_result_set() - Free a previously created result set. ++ * ++ * @results: The result set whose resources to free. ++ */ ++void kutf_destroy_result_set(struct kutf_result_set *results); ++ ++/** ++ * kutf_set_waiting_for_input() - The test is waiting for userdata ++ * ++ * @set: The result set to update ++ * ++ * Causes the result set to always have results and return a fake ++ * %KUTF_RESULT_USERDATA_WAIT result. ++ */ ++void kutf_set_waiting_for_input(struct kutf_result_set *set); ++ ++/** ++ * kutf_clear_waiting_for_input() - The test is no longer waiting for userdata ++ * ++ * @set: The result set to update ++ * ++ * Cancels the effect of kutf_set_waiting_for_input() ++ */ ++void kutf_clear_waiting_for_input(struct kutf_result_set *set); ++ ++#endif /* __KERNEL__ */ ++ ++#endif /* _KERNEL_UTF_RESULTSET_H_ */ +diff --git a/drivers/gpu/arm/bifrost/tests/include/kutf/kutf_suite.h b/drivers/gpu/arm/bifrost/tests/include/kutf/kutf_suite.h +new file mode 100755 +index 000000000000..8d75f506f9eb +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/tests/include/kutf/kutf_suite.h +@@ -0,0 +1,569 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KERNEL_UTF_SUITE_H_ ++#define _KERNEL_UTF_SUITE_H_ ++ ++/* kutf_suite.h ++ * Functions for management of test suites. ++ * ++ * This collection of data structures, macros, and functions are used to ++ * create Test Suites, Tests within those Test Suites, and Fixture variants ++ * of each test. ++ */ ++ ++#include ++#include ++#include ++ ++#include ++#include ++ ++/* Arbitrary maximum size to prevent user space allocating too much kernel ++ * memory ++ */ ++#define KUTF_MAX_LINE_LENGTH (1024u) ++ ++/** ++ * Pseudo-flag indicating an absence of any specified test class. Note that ++ * tests should not be annotated with this constant as it is simply a zero ++ * value; tests without a more specific class must be marked with the flag ++ * KUTF_F_TEST_GENERIC. ++ */ ++#define KUTF_F_TEST_NONE ((unsigned int)(0)) ++ ++/** ++ * Class indicating this test is a smoke test. ++ * A given set of smoke tests should be quick to run, enabling rapid turn-around ++ * of "regress-on-commit" test runs. ++ */ ++#define KUTF_F_TEST_SMOKETEST ((unsigned int)(1 << 1)) ++ ++/** ++ * Class indicating this test is a performance test. ++ * These tests typically produce a performance metric, such as "time to run" or ++ * "frames per second", ++ */ ++#define KUTF_F_TEST_PERFORMANCE ((unsigned int)(1 << 2)) ++ ++/** ++ * Class indicating that this test is a deprecated test. ++ * These tests have typically been replaced by an alternative test which is ++ * more efficient, or has better coverage. ++ */ ++#define KUTF_F_TEST_DEPRECATED ((unsigned int)(1 << 3)) ++ ++/** ++ * Class indicating that this test is a known failure. ++ * These tests have typically been run and failed, but marking them as a known ++ * failure means it is easier to triage results. ++ * ++ * It is typically more convenient to triage known failures using the ++ * results database and web UI, as this means there is no need to modify the ++ * test code. ++ */ ++#define KUTF_F_TEST_EXPECTED_FAILURE ((unsigned int)(1 << 4)) ++ ++/** ++ * Class indicating that this test is a generic test, which is not a member of ++ * a more specific test class. Tests which are not created with a specific set ++ * of filter flags by the user are assigned this test class by default. ++ */ ++#define KUTF_F_TEST_GENERIC ((unsigned int)(1 << 5)) ++ ++/** ++ * Class indicating this test is a resource allocation failure test. ++ * A resource allocation failure test will test that an error code is ++ * correctly propagated when an allocation fails. ++ */ ++#define KUTF_F_TEST_RESFAIL ((unsigned int)(1 << 6)) ++ ++/** ++ * Additional flag indicating that this test is an expected failure when ++ * run in resource failure mode. These tests are never run when running ++ * the low resource mode. ++ */ ++#define KUTF_F_TEST_EXPECTED_FAILURE_RF ((unsigned int)(1 << 7)) ++ ++/** ++ * Flag reserved for user-defined filter zero. ++ */ ++#define KUTF_F_TEST_USER_0 ((unsigned int)(1 << 24)) ++ ++/** ++ * Flag reserved for user-defined filter one. ++ */ ++#define KUTF_F_TEST_USER_1 ((unsigned int)(1 << 25)) ++ ++/** ++ * Flag reserved for user-defined filter two. ++ */ ++#define KUTF_F_TEST_USER_2 ((unsigned int)(1 << 26)) ++ ++/** ++ * Flag reserved for user-defined filter three. ++ */ ++#define KUTF_F_TEST_USER_3 ((unsigned int)(1 << 27)) ++ ++/** ++ * Flag reserved for user-defined filter four. ++ */ ++#define KUTF_F_TEST_USER_4 ((unsigned int)(1 << 28)) ++ ++/** ++ * Flag reserved for user-defined filter five. ++ */ ++#define KUTF_F_TEST_USER_5 ((unsigned int)(1 << 29)) ++ ++/** ++ * Flag reserved for user-defined filter six. ++ */ ++#define KUTF_F_TEST_USER_6 ((unsigned int)(1 << 30)) ++ ++/** ++ * Flag reserved for user-defined filter seven. ++ */ ++#define KUTF_F_TEST_USER_7 ((unsigned int)(1 << 31)) ++ ++/** ++ * Pseudo-flag indicating that all test classes should be executed. ++ */ ++#define KUTF_F_TEST_ALL ((unsigned int)(0xFFFFFFFFU)) ++ ++/** ++ * union kutf_callback_data - Union used to store test callback data ++ * @ptr_value: pointer to the location where test callback data ++ * are stored ++ * @u32_value: a number which represents test callback data ++ */ ++union kutf_callback_data { ++ void *ptr_value; ++ u32 u32_value; ++}; ++ ++/** ++ * struct kutf_userdata_line - A line of user data to be returned to the user ++ * @node: struct list_head to link this into a list ++ * @str: The line of user data to return to user space ++ * @size: The number of bytes within @str ++ */ ++struct kutf_userdata_line { ++ struct list_head node; ++ char *str; ++ size_t size; ++}; ++ ++/** ++ * KUTF_USERDATA_WARNING_OUTPUT - Flag specifying that a warning has been output ++ * ++ * If user space reads the "run" file while the test is waiting for user data, ++ * then the framework will output a warning message and set this flag within ++ * struct kutf_userdata. A subsequent read will then simply return an end of ++ * file condition rather than outputting the warning again. The upshot of this ++ * is that simply running 'cat' on a test which requires user data will produce ++ * the warning followed by 'cat' exiting due to EOF - which is much more user ++ * friendly than blocking indefinitely waiting for user data. ++ */ ++#define KUTF_USERDATA_WARNING_OUTPUT 1 ++ ++/** ++ * struct kutf_userdata - Structure holding user data ++ * @flags: See %KUTF_USERDATA_WARNING_OUTPUT ++ * @input_head: List of struct kutf_userdata_line containing user data ++ * to be read by the kernel space test. ++ * @input_waitq: Wait queue signalled when there is new user data to be ++ * read by the kernel space test. ++ */ ++struct kutf_userdata { ++ unsigned long flags; ++ struct list_head input_head; ++ wait_queue_head_t input_waitq; ++}; ++ ++/** ++ * struct kutf_context - Structure representing a kernel test context ++ * @kref: Refcount for number of users of this context ++ * @suite: Convenience pointer to the suite this context ++ * is running ++ * @test_fix: The fixture that is being run in this context ++ * @fixture_pool: The memory pool used for the duration of ++ * the fixture/text context. ++ * @fixture: The user provided fixture structure. ++ * @fixture_index: The index (id) of the current fixture. ++ * @fixture_name: The name of the current fixture (or NULL if unnamed). ++ * @test_data: Any user private data associated with this test ++ * @result_set: All the results logged by this test context ++ * @status: The status of the currently running fixture. ++ * @expected_status: The expected status on exist of the currently ++ * running fixture. ++ * @work: Work item to enqueue onto the work queue to run the test ++ * @userdata: Structure containing the user data for the test to read ++ */ ++struct kutf_context { ++ struct kref kref; ++ struct kutf_suite *suite; ++ struct kutf_test_fixture *test_fix; ++ struct kutf_mempool fixture_pool; ++ void *fixture; ++ unsigned int fixture_index; ++ const char *fixture_name; ++ union kutf_callback_data test_data; ++ struct kutf_result_set *result_set; ++ enum kutf_result_status status; ++ enum kutf_result_status expected_status; ++ ++ struct work_struct work; ++ struct kutf_userdata userdata; ++}; ++ ++/** ++ * struct kutf_suite - Structure representing a kernel test suite ++ * @app: The application this suite belongs to. ++ * @name: The name of this suite. ++ * @suite_data: Any user private data associated with this ++ * suite. ++ * @create_fixture: Function used to create a new fixture instance ++ * @remove_fixture: Function used to destroy a new fixture instance ++ * @fixture_variants: The number of variants (must be at least 1). ++ * @suite_default_flags: Suite global filter flags which are set on ++ * all tests. ++ * @node: List node for suite_list ++ * @dir: The debugfs directory for this suite ++ * @test_list: List head to store all the tests which are ++ * part of this suite ++ */ ++struct kutf_suite { ++ struct kutf_application *app; ++ const char *name; ++ union kutf_callback_data suite_data; ++ void *(*create_fixture)(struct kutf_context *context); ++ void (*remove_fixture)(struct kutf_context *context); ++ unsigned int fixture_variants; ++ unsigned int suite_default_flags; ++ struct list_head node; ++ struct dentry *dir; ++ struct list_head test_list; ++}; ++ ++/* ============================================================================ ++ Application functions ++============================================================================ */ ++ ++/** ++ * kutf_create_application() - Create an in kernel test application. ++ * @name: The name of the test application. ++ * ++ * Return: pointer to the kutf_application on success or NULL ++ * on failure ++ */ ++struct kutf_application *kutf_create_application(const char *name); ++ ++/** ++ * kutf_destroy_application() - Destroy an in kernel test application. ++ * ++ * @app: The test application to destroy. ++ */ ++void kutf_destroy_application(struct kutf_application *app); ++ ++/* ============================================================================ ++ Suite functions ++============================================================================ */ ++ ++/** ++ * kutf_create_suite() - Create a kernel test suite. ++ * @app: The test application to create the suite in. ++ * @name: The name of the suite. ++ * @fixture_count: The number of fixtures to run over the test ++ * functions in this suite ++ * @create_fixture: Callback used to create a fixture. The returned value ++ * is stored in the fixture pointer in the context for ++ * use in the test functions. ++ * @remove_fixture: Callback used to remove a previously created fixture. ++ * ++ * Suite names must be unique. Should two suites with the same name be ++ * registered with the same application then this function will fail, if they ++ * are registered with different applications then the function will not detect ++ * this and the call will succeed. ++ * ++ * Return: pointer to the created kutf_suite on success or NULL ++ * on failure ++ */ ++struct kutf_suite *kutf_create_suite( ++ struct kutf_application *app, ++ const char *name, ++ unsigned int fixture_count, ++ void *(*create_fixture)(struct kutf_context *context), ++ void (*remove_fixture)(struct kutf_context *context)); ++ ++/** ++ * kutf_create_suite_with_filters() - Create a kernel test suite with user ++ * defined default filters. ++ * @app: The test application to create the suite in. ++ * @name: The name of the suite. ++ * @fixture_count: The number of fixtures to run over the test ++ * functions in this suite ++ * @create_fixture: Callback used to create a fixture. The returned value ++ * is stored in the fixture pointer in the context for ++ * use in the test functions. ++ * @remove_fixture: Callback used to remove a previously created fixture. ++ * @filters: Filters to apply to a test if it doesn't provide its own ++ * ++ * Suite names must be unique. Should two suites with the same name be ++ * registered with the same application then this function will fail, if they ++ * are registered with different applications then the function will not detect ++ * this and the call will succeed. ++ * ++ * Return: pointer to the created kutf_suite on success or NULL on failure ++ */ ++struct kutf_suite *kutf_create_suite_with_filters( ++ struct kutf_application *app, ++ const char *name, ++ unsigned int fixture_count, ++ void *(*create_fixture)(struct kutf_context *context), ++ void (*remove_fixture)(struct kutf_context *context), ++ unsigned int filters); ++ ++/** ++ * kutf_create_suite_with_filters_and_data() - Create a kernel test suite with ++ * user defined default filters. ++ * @app: The test application to create the suite in. ++ * @name: The name of the suite. ++ * @fixture_count: The number of fixtures to run over the test ++ * functions in this suite ++ * @create_fixture: Callback used to create a fixture. The returned value ++ * is stored in the fixture pointer in the context for ++ * use in the test functions. ++ * @remove_fixture: Callback used to remove a previously created fixture. ++ * @filters: Filters to apply to a test if it doesn't provide its own ++ * @suite_data: Suite specific callback data, provided during the ++ * running of the test in the kutf_context ++ * ++ * Return: pointer to the created kutf_suite on success or NULL ++ * on failure ++ */ ++struct kutf_suite *kutf_create_suite_with_filters_and_data( ++ struct kutf_application *app, ++ const char *name, ++ unsigned int fixture_count, ++ void *(*create_fixture)(struct kutf_context *context), ++ void (*remove_fixture)(struct kutf_context *context), ++ unsigned int filters, ++ union kutf_callback_data suite_data); ++ ++/** ++ * kutf_add_test() - Add a test to a kernel test suite. ++ * @suite: The suite to add the test to. ++ * @id: The ID of the test. ++ * @name: The name of the test. ++ * @execute: Callback to the test function to run. ++ * ++ * Note: As no filters are provided the test will use the suite filters instead ++ */ ++void kutf_add_test(struct kutf_suite *suite, ++ unsigned int id, ++ const char *name, ++ void (*execute)(struct kutf_context *context)); ++ ++/** ++ * kutf_add_test_with_filters() - Add a test to a kernel test suite with filters ++ * @suite: The suite to add the test to. ++ * @id: The ID of the test. ++ * @name: The name of the test. ++ * @execute: Callback to the test function to run. ++ * @filters: A set of filtering flags, assigning test categories. ++ */ ++void kutf_add_test_with_filters(struct kutf_suite *suite, ++ unsigned int id, ++ const char *name, ++ void (*execute)(struct kutf_context *context), ++ unsigned int filters); ++ ++/** ++ * kutf_add_test_with_filters_and_data() - Add a test to a kernel test suite ++ * with filters. ++ * @suite: The suite to add the test to. ++ * @id: The ID of the test. ++ * @name: The name of the test. ++ * @execute: Callback to the test function to run. ++ * @filters: A set of filtering flags, assigning test categories. ++ * @test_data: Test specific callback data, provided during the ++ * running of the test in the kutf_context ++ */ ++void kutf_add_test_with_filters_and_data( ++ struct kutf_suite *suite, ++ unsigned int id, ++ const char *name, ++ void (*execute)(struct kutf_context *context), ++ unsigned int filters, ++ union kutf_callback_data test_data); ++ ++ ++/* ============================================================================ ++ Test functions ++============================================================================ */ ++/** ++ * kutf_test_log_result_external() - Log a result which has been created ++ * externally into a in a standard form ++ * recognized by the log parser. ++ * @context: The test context the test is running in ++ * @message: The message for this result ++ * @new_status: The result status of this log message ++ */ ++void kutf_test_log_result_external( ++ struct kutf_context *context, ++ const char *message, ++ enum kutf_result_status new_status); ++ ++/** ++ * kutf_test_expect_abort() - Tell the kernel that you expect the current ++ * fixture to produce an abort. ++ * @context: The test context this test is running in. ++ */ ++void kutf_test_expect_abort(struct kutf_context *context); ++ ++/** ++ * kutf_test_expect_fatal() - Tell the kernel that you expect the current ++ * fixture to produce a fatal error. ++ * @context: The test context this test is running in. ++ */ ++void kutf_test_expect_fatal(struct kutf_context *context); ++ ++/** ++ * kutf_test_expect_fail() - Tell the kernel that you expect the current ++ * fixture to fail. ++ * @context: The test context this test is running in. ++ */ ++void kutf_test_expect_fail(struct kutf_context *context); ++ ++/** ++ * kutf_test_expect_warn() - Tell the kernel that you expect the current ++ * fixture to produce a warning. ++ * @context: The test context this test is running in. ++ */ ++void kutf_test_expect_warn(struct kutf_context *context); ++ ++/** ++ * kutf_test_expect_pass() - Tell the kernel that you expect the current ++ * fixture to pass. ++ * @context: The test context this test is running in. ++ */ ++void kutf_test_expect_pass(struct kutf_context *context); ++ ++/** ++ * kutf_test_skip() - Tell the kernel that the test should be skipped. ++ * @context: The test context this test is running in. ++ */ ++void kutf_test_skip(struct kutf_context *context); ++ ++/** ++ * kutf_test_skip_msg() - Tell the kernel that this test has been skipped, ++ * supplying a reason string. ++ * @context: The test context this test is running in. ++ * @message: A message string containing the reason for the skip. ++ * ++ * Note: The message must not be freed during the lifetime of the test run. ++ * This means it should either be a prebaked string, or if a dynamic string ++ * is required it must be created with kutf_dsprintf which will store ++ * the resultant string in a buffer who's lifetime is the same as the test run. ++ */ ++void kutf_test_skip_msg(struct kutf_context *context, const char *message); ++ ++/** ++ * kutf_test_pass() - Tell the kernel that this test has passed. ++ * @context: The test context this test is running in. ++ * @message: A message string containing the reason for the pass. ++ * ++ * Note: The message must not be freed during the lifetime of the test run. ++ * This means it should either be a pre-baked string, or if a dynamic string ++ * is required it must be created with kutf_dsprintf which will store ++ * the resultant string in a buffer who's lifetime is the same as the test run. ++ */ ++void kutf_test_pass(struct kutf_context *context, char const *message); ++ ++/** ++ * kutf_test_debug() - Send a debug message ++ * @context: The test context this test is running in. ++ * @message: A message string containing the debug information. ++ * ++ * Note: The message must not be freed during the lifetime of the test run. ++ * This means it should either be a pre-baked string, or if a dynamic string ++ * is required it must be created with kutf_dsprintf which will store ++ * the resultant string in a buffer who's lifetime is the same as the test run. ++ */ ++void kutf_test_debug(struct kutf_context *context, char const *message); ++ ++/** ++ * kutf_test_info() - Send an information message ++ * @context: The test context this test is running in. ++ * @message: A message string containing the information message. ++ * ++ * Note: The message must not be freed during the lifetime of the test run. ++ * This means it should either be a pre-baked string, or if a dynamic string ++ * is required it must be created with kutf_dsprintf which will store ++ * the resultant string in a buffer who's lifetime is the same as the test run. ++ */ ++void kutf_test_info(struct kutf_context *context, char const *message); ++ ++/** ++ * kutf_test_warn() - Send a warning message ++ * @context: The test context this test is running in. ++ * @message: A message string containing the warning message. ++ * ++ * Note: The message must not be freed during the lifetime of the test run. ++ * This means it should either be a pre-baked string, or if a dynamic string ++ * is required it must be created with kutf_dsprintf which will store ++ * the resultant string in a buffer who's lifetime is the same as the test run. ++ */ ++void kutf_test_warn(struct kutf_context *context, char const *message); ++ ++/** ++ * kutf_test_fail() - Tell the kernel that a test has failed ++ * @context: The test context this test is running in. ++ * @message: A message string containing the failure message. ++ * ++ * Note: The message must not be freed during the lifetime of the test run. ++ * This means it should either be a pre-baked string, or if a dynamic string ++ * is required it must be created with kutf_dsprintf which will store ++ * the resultant string in a buffer who's lifetime is the same as the test run. ++ */ ++void kutf_test_fail(struct kutf_context *context, char const *message); ++ ++/** ++ * kutf_test_fatal() - Tell the kernel that a test has triggered a fatal error ++ * @context: The test context this test is running in. ++ * @message: A message string containing the fatal error message. ++ * ++ * Note: The message must not be freed during the lifetime of the test run. ++ * This means it should either be a pre-baked string, or if a dynamic string ++ * is required it must be created with kutf_dsprintf which will store ++ * the resultant string in a buffer who's lifetime is the same as the test run. ++ */ ++void kutf_test_fatal(struct kutf_context *context, char const *message); ++ ++/** ++ * kutf_test_abort() - Tell the kernel that a test triggered an abort in the test ++ * ++ * @context: The test context this test is running in. ++ */ ++void kutf_test_abort(struct kutf_context *context); ++ ++#endif /* _KERNEL_UTF_SUITE_H_ */ +diff --git a/drivers/gpu/arm/bifrost/tests/include/kutf/kutf_utils.h b/drivers/gpu/arm/bifrost/tests/include/kutf/kutf_utils.h +new file mode 100755 +index 000000000000..25b8285500d7 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/tests/include/kutf/kutf_utils.h +@@ -0,0 +1,60 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KERNEL_UTF_UTILS_H_ ++#define _KERNEL_UTF_UTILS_H_ ++ ++/* kutf_utils.h ++ * Utilities for the kernel UTF test infrastructure. ++ * ++ * This collection of library functions are provided for use by kernel UTF ++ * and users of kernel UTF which don't directly fit within the other ++ * code modules. ++ */ ++ ++#include ++ ++/** ++ * Maximum size of the message strings within kernel UTF, messages longer then ++ * this will be truncated. ++ */ ++#define KUTF_MAX_DSPRINTF_LEN 1024 ++ ++/** ++ * kutf_dsprintf() - dynamic sprintf ++ * @pool: memory pool to allocate from ++ * @fmt: The format string describing the string to document. ++ * @... The parameters to feed in to the format string. ++ * ++ * This function implements sprintf which dynamically allocates memory to store ++ * the string. The library will free the memory containing the string when the ++ * result set is cleared or destroyed. ++ * ++ * Note The returned string may be truncated to fit an internal temporary ++ * buffer, which is KUTF_MAX_DSPRINTF_LEN bytes in length. ++ * ++ * Return: Returns pointer to allocated string, or NULL on error. ++ */ ++const char *kutf_dsprintf(struct kutf_mempool *pool, ++ const char *fmt, ...); ++ ++#endif /* _KERNEL_UTF_UTILS_H_ */ +diff --git a/drivers/gpu/arm/bifrost/tests/kutf/Kbuild b/drivers/gpu/arm/bifrost/tests/kutf/Kbuild +new file mode 100755 +index 000000000000..2531d41ca28d +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/tests/kutf/Kbuild +@@ -0,0 +1,26 @@ ++# ++# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# GNU General Public License for more details. ++# ++# You should have received a copy of the GNU General Public License ++# along with this program; if not, you can access it online at ++# http://www.gnu.org/licenses/gpl-2.0.html. ++# ++# SPDX-License-Identifier: GPL-2.0 ++# ++# ++ ++ccflags-y += -I$(src)/../include ++ ++obj-$(CONFIG_MALI_KUTF) += kutf.o ++ ++kutf-y := kutf_mem.o kutf_resultset.o kutf_suite.o kutf_utils.o kutf_helpers.o kutf_helpers_user.o +diff --git a/drivers/gpu/arm/bifrost/tests/kutf/Kconfig b/drivers/gpu/arm/bifrost/tests/kutf/Kconfig +new file mode 100755 +index 000000000000..0cdb474c06a3 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/tests/kutf/Kconfig +@@ -0,0 +1,28 @@ ++# ++# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# GNU General Public License for more details. ++# ++# You should have received a copy of the GNU General Public License ++# along with this program; if not, you can access it online at ++# http://www.gnu.org/licenses/gpl-2.0.html. ++# ++# SPDX-License-Identifier: GPL-2.0 ++# ++# ++ ++ ++config MALI_KUTF ++ tristate "Mali Kernel Unit Test Framework" ++ default m ++ help ++ Enables MALI testing framework. To compile it as a module, ++ choose M here - this will generate a single module called kutf. +diff --git a/drivers/gpu/arm/bifrost/tests/kutf/Makefile b/drivers/gpu/arm/bifrost/tests/kutf/Makefile +new file mode 100755 +index 000000000000..d848e8774bd0 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/tests/kutf/Makefile +@@ -0,0 +1,35 @@ ++# ++# (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# GNU General Public License for more details. ++# ++# You should have received a copy of the GNU General Public License ++# along with this program; if not, you can access it online at ++# http://www.gnu.org/licenses/gpl-2.0.html. ++# ++# SPDX-License-Identifier: GPL-2.0 ++# ++# ++ ++# linux build system bootstrap for out-of-tree module ++ ++# default to building for the host ++ARCH ?= $(shell uname -m) ++ ++ifeq ($(KDIR),) ++$(error Must specify KDIR to point to the kernel to target)) ++endif ++ ++all: ++ $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) $(SCONS_CONFIGS) EXTRA_CFLAGS=-I$(CURDIR)/../include modules ++ ++clean: ++ $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) clean +diff --git a/drivers/gpu/arm/bifrost/tests/kutf/build.bp b/drivers/gpu/arm/bifrost/tests/kutf/build.bp +new file mode 100755 +index 000000000000..32eab143e669 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/tests/kutf/build.bp +@@ -0,0 +1,36 @@ ++/* ++ * ++ * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++bob_kernel_module { ++ name: "kutf", ++ defaults: [ ++ "kernel_defaults", ++ "kutf_includes", ++ ], ++ srcs: [ ++ "Kbuild", ++ "kutf_helpers.c", ++ "kutf_helpers_user.c", ++ "kutf_mem.c", ++ "kutf_resultset.c", ++ "kutf_suite.c", ++ "kutf_utils.c", ++ ], ++ kbuild_options: ["CONFIG_MALI_KUTF=m"], ++ enabled: false, ++ base_build_kutf: { ++ enabled: true, ++ }, ++} +diff --git a/drivers/gpu/arm/bifrost/tests/kutf/kutf_helpers.c b/drivers/gpu/arm/bifrost/tests/kutf/kutf_helpers.c +new file mode 100755 +index 000000000000..4463b04792f5 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/tests/kutf/kutf_helpers.c +@@ -0,0 +1,131 @@ ++/* ++ * ++ * (C) COPYRIGHT 2017, 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* Kernel UTF test helpers */ ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++static DEFINE_SPINLOCK(kutf_input_lock); ++ ++bool kutf_helper_pending_input(struct kutf_context *context) ++{ ++ bool input_pending; ++ ++ spin_lock(&kutf_input_lock); ++ ++ input_pending = !list_empty(&context->userdata.input_head); ++ ++ spin_unlock(&kutf_input_lock); ++ ++ return input_pending; ++} ++EXPORT_SYMBOL(kutf_helper_pending_input); ++ ++char *kutf_helper_input_dequeue(struct kutf_context *context, size_t *str_size) ++{ ++ struct kutf_userdata_line *line; ++ ++ spin_lock(&kutf_input_lock); ++ ++ while (list_empty(&context->userdata.input_head)) { ++ int err; ++ ++ kutf_set_waiting_for_input(context->result_set); ++ ++ spin_unlock(&kutf_input_lock); ++ ++ err = wait_event_interruptible(context->userdata.input_waitq, ++ kutf_helper_pending_input(context)); ++ ++ if (err) ++ return ERR_PTR(-EINTR); ++ ++ spin_lock(&kutf_input_lock); ++ } ++ ++ line = list_first_entry(&context->userdata.input_head, ++ struct kutf_userdata_line, node); ++ if (line->str) { ++ /* ++ * Unless it is the end-of-input marker, ++ * remove it from the list ++ */ ++ list_del(&line->node); ++ } ++ ++ spin_unlock(&kutf_input_lock); ++ ++ if (str_size) ++ *str_size = line->size; ++ return line->str; ++} ++ ++int kutf_helper_input_enqueue(struct kutf_context *context, ++ const char __user *str, size_t size) ++{ ++ struct kutf_userdata_line *line; ++ ++ line = kutf_mempool_alloc(&context->fixture_pool, ++ sizeof(*line) + size + 1); ++ if (!line) ++ return -ENOMEM; ++ if (str) { ++ unsigned long bytes_not_copied; ++ ++ line->size = size; ++ line->str = (void *)(line + 1); ++ bytes_not_copied = copy_from_user(line->str, str, size); ++ if (bytes_not_copied != 0) ++ return -EFAULT; ++ /* Zero terminate the string */ ++ line->str[size] = '\0'; ++ } else { ++ /* This is used to mark the end of input */ ++ WARN_ON(size); ++ line->size = 0; ++ line->str = NULL; ++ } ++ ++ spin_lock(&kutf_input_lock); ++ ++ list_add_tail(&line->node, &context->userdata.input_head); ++ ++ kutf_clear_waiting_for_input(context->result_set); ++ ++ spin_unlock(&kutf_input_lock); ++ ++ wake_up(&context->userdata.input_waitq); ++ ++ return 0; ++} ++ ++void kutf_helper_input_enqueue_end_of_data(struct kutf_context *context) ++{ ++ kutf_helper_input_enqueue(context, NULL, 0); ++} +diff --git a/drivers/gpu/arm/bifrost/tests/kutf/kutf_helpers_user.c b/drivers/gpu/arm/bifrost/tests/kutf/kutf_helpers_user.c +new file mode 100755 +index 000000000000..108fa82d9b21 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/tests/kutf/kutf_helpers_user.c +@@ -0,0 +1,468 @@ ++/* ++ * ++ * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* Kernel UTF test helpers that mirror those for kutf-userside */ ++#include ++#include ++#include ++ ++#include ++#include ++#include ++ ++const char *valtype_names[] = { ++ "INVALID", ++ "U64", ++ "STR", ++}; ++ ++static const char *get_val_type_name(enum kutf_helper_valtype valtype) ++{ ++ /* enums can be signed or unsigned (implementation dependant), so ++ * enforce it to prevent: ++ * a) "<0 comparison on unsigned type" warning - if we did both upper ++ * and lower bound check ++ * b) incorrect range checking if it was a signed type - if we did ++ * upper bound check only */ ++ unsigned int type_idx = (unsigned int)valtype; ++ ++ if (type_idx >= (unsigned int)KUTF_HELPER_VALTYPE_COUNT) ++ type_idx = (unsigned int)KUTF_HELPER_VALTYPE_INVALID; ++ ++ return valtype_names[type_idx]; ++} ++ ++/* Check up to str_len chars of val_str to see if it's a valid value name: ++ * ++ * - Has between 1 and KUTF_HELPER_MAX_VAL_NAME_LEN characters before the \0 terminator ++ * - And, each char is in the character set [A-Z0-9_] */ ++static int validate_val_name(const char *val_str, int str_len) ++{ ++ int i = 0; ++ ++ for (i = 0; str_len && i <= KUTF_HELPER_MAX_VAL_NAME_LEN && val_str[i] != '\0'; ++i, --str_len) { ++ char val_chr = val_str[i]; ++ ++ if (val_chr >= 'A' && val_chr <= 'Z') ++ continue; ++ if (val_chr >= '0' && val_chr <= '9') ++ continue; ++ if (val_chr == '_') ++ continue; ++ ++ /* Character not in the set [A-Z0-9_] - report error */ ++ return 1; ++ } ++ ++ /* Names of 0 length are not valid */ ++ if (i == 0) ++ return 1; ++ /* Length greater than KUTF_HELPER_MAX_VAL_NAME_LEN not allowed */ ++ if (i > KUTF_HELPER_MAX_VAL_NAME_LEN || (i == KUTF_HELPER_MAX_VAL_NAME_LEN && val_str[i] != '\0')) ++ return 1; ++ ++ return 0; ++} ++ ++/* Find the length of the valid part of the string when it will be in quotes ++ * e.g. "str" ++ * ++ * That is, before any '\\', '\n' or '"' characters. This is so we don't have ++ * to escape the string */ ++static int find_quoted_string_valid_len(const char *str) ++{ ++ char *ptr; ++ const char *check_chars = "\\\n\""; ++ ++ ptr = strpbrk(str, check_chars); ++ if (ptr) ++ return (int)(ptr-str); ++ ++ return (int)strlen(str); ++} ++ ++static int kutf_helper_userdata_enqueue(struct kutf_context *context, ++ const char *str) ++{ ++ char *str_copy; ++ size_t len; ++ int err; ++ ++ len = strlen(str)+1; ++ ++ str_copy = kutf_mempool_alloc(&context->fixture_pool, len); ++ if (!str_copy) ++ return -ENOMEM; ++ ++ strcpy(str_copy, str); ++ ++ err = kutf_add_result(context, KUTF_RESULT_USERDATA, str_copy); ++ ++ return err; ++} ++ ++#define MAX_U64_HEX_LEN 16 ++/* (Name size) + ("=0x" size) + (64-bit hex value size) + (terminator) */ ++#define NAMED_U64_VAL_BUF_SZ (KUTF_HELPER_MAX_VAL_NAME_LEN + 3 + MAX_U64_HEX_LEN + 1) ++ ++int kutf_helper_send_named_u64(struct kutf_context *context, ++ const char *val_name, u64 val) ++{ ++ int ret = 1; ++ char msgbuf[NAMED_U64_VAL_BUF_SZ]; ++ const char *errmsg = NULL; ++ ++ if (validate_val_name(val_name, KUTF_HELPER_MAX_VAL_NAME_LEN + 1)) { ++ errmsg = kutf_dsprintf(&context->fixture_pool, ++ "Failed to send u64 value named '%s': Invalid value name", val_name); ++ goto out_err; ++ } ++ ++ ret = snprintf(msgbuf, NAMED_U64_VAL_BUF_SZ, "%s=0x%llx", val_name, val); ++ if (ret >= NAMED_U64_VAL_BUF_SZ || ret < 0) { ++ errmsg = kutf_dsprintf(&context->fixture_pool, ++ "Failed to send u64 value named '%s': snprintf() problem buffer size==%d ret=%d", ++ val_name, NAMED_U64_VAL_BUF_SZ, ret); ++ goto out_err; ++ } ++ ++ ret = kutf_helper_userdata_enqueue(context, msgbuf); ++ if (ret) { ++ errmsg = kutf_dsprintf(&context->fixture_pool, ++ "Failed to send u64 value named '%s': send returned %d", ++ val_name, ret); ++ goto out_err; ++ } ++ ++ return ret; ++out_err: ++ kutf_test_fail(context, errmsg); ++ return ret; ++} ++EXPORT_SYMBOL(kutf_helper_send_named_u64); ++ ++#define NAMED_VALUE_SEP "=" ++#define NAMED_STR_START_DELIM NAMED_VALUE_SEP "\"" ++#define NAMED_STR_END_DELIM "\"" ++ ++int kutf_helper_max_str_len_for_kern(const char *val_name, ++ int kern_buf_sz) ++{ ++ const int val_name_len = strlen(val_name); ++ const int start_delim_len = strlen(NAMED_STR_START_DELIM); ++ const int end_delim_len = strlen(NAMED_STR_END_DELIM); ++ int max_msg_len = kern_buf_sz; ++ int max_str_len; ++ ++ max_str_len = max_msg_len - val_name_len - start_delim_len - ++ end_delim_len; ++ ++ return max_str_len; ++} ++EXPORT_SYMBOL(kutf_helper_max_str_len_for_kern); ++ ++int kutf_helper_send_named_str(struct kutf_context *context, ++ const char *val_name, ++ const char *val_str) ++{ ++ int val_str_len; ++ int str_buf_sz; ++ char *str_buf = NULL; ++ int ret = 1; ++ char *copy_ptr; ++ int val_name_len; ++ int start_delim_len = strlen(NAMED_STR_START_DELIM); ++ int end_delim_len = strlen(NAMED_STR_END_DELIM); ++ const char *errmsg = NULL; ++ ++ if (validate_val_name(val_name, KUTF_HELPER_MAX_VAL_NAME_LEN + 1)) { ++ errmsg = kutf_dsprintf(&context->fixture_pool, ++ "Failed to send u64 value named '%s': Invalid value name", val_name); ++ goto out_err; ++ } ++ val_name_len = strlen(val_name); ++ ++ val_str_len = find_quoted_string_valid_len(val_str); ++ ++ /* (name length) + ("=\"" length) + (val_str len) + ("\"" length) + terminator */ ++ str_buf_sz = val_name_len + start_delim_len + val_str_len + end_delim_len + 1; ++ ++ /* Using kmalloc() here instead of mempool since we know we need to free ++ * before we return */ ++ str_buf = kmalloc(str_buf_sz, GFP_KERNEL); ++ if (!str_buf) { ++ errmsg = kutf_dsprintf(&context->fixture_pool, ++ "Failed to send str value named '%s': kmalloc failed, str_buf_sz=%d", ++ val_name, str_buf_sz); ++ goto out_err; ++ } ++ copy_ptr = str_buf; ++ ++ /* Manually copy each string component instead of snprintf because ++ * val_str may need to end early, and less error path handling */ ++ ++ /* name */ ++ memcpy(copy_ptr, val_name, val_name_len); ++ copy_ptr += val_name_len; ++ ++ /* str start delimiter */ ++ memcpy(copy_ptr, NAMED_STR_START_DELIM, start_delim_len); ++ copy_ptr += start_delim_len; ++ ++ /* str value */ ++ memcpy(copy_ptr, val_str, val_str_len); ++ copy_ptr += val_str_len; ++ ++ /* str end delimiter */ ++ memcpy(copy_ptr, NAMED_STR_END_DELIM, end_delim_len); ++ copy_ptr += end_delim_len; ++ ++ /* Terminator */ ++ *copy_ptr = '\0'; ++ ++ ret = kutf_helper_userdata_enqueue(context, str_buf); ++ ++ if (ret) { ++ errmsg = kutf_dsprintf(&context->fixture_pool, ++ "Failed to send str value named '%s': send returned %d", ++ val_name, ret); ++ goto out_err; ++ } ++ ++ kfree(str_buf); ++ return ret; ++ ++out_err: ++ kutf_test_fail(context, errmsg); ++ kfree(str_buf); ++ return ret; ++} ++EXPORT_SYMBOL(kutf_helper_send_named_str); ++ ++int kutf_helper_receive_named_val( ++ struct kutf_context *context, ++ struct kutf_helper_named_val *named_val) ++{ ++ size_t recv_sz; ++ char *recv_str; ++ char *search_ptr; ++ char *name_str = NULL; ++ int name_len; ++ int strval_len; ++ enum kutf_helper_valtype type = KUTF_HELPER_VALTYPE_INVALID; ++ char *strval = NULL; ++ u64 u64val = 0; ++ int err = KUTF_HELPER_ERR_INVALID_VALUE; ++ ++ recv_str = kutf_helper_input_dequeue(context, &recv_sz); ++ if (!recv_str) ++ return -EBUSY; ++ else if (IS_ERR(recv_str)) ++ return PTR_ERR(recv_str); ++ ++ /* Find the '=', grab the name and validate it */ ++ search_ptr = strnchr(recv_str, recv_sz, NAMED_VALUE_SEP[0]); ++ if (search_ptr) { ++ name_len = search_ptr - recv_str; ++ if (!validate_val_name(recv_str, name_len)) { ++ /* no need to reallocate - just modify string in place */ ++ name_str = recv_str; ++ name_str[name_len] = '\0'; ++ ++ /* Move until after the '=' */ ++ recv_str += (name_len + 1); ++ recv_sz -= (name_len + 1); ++ } ++ } ++ if (!name_str) { ++ pr_err("Invalid name part for received string '%s'\n", ++ recv_str); ++ return KUTF_HELPER_ERR_INVALID_NAME; ++ } ++ ++ /* detect value type */ ++ if (*recv_str == NAMED_STR_START_DELIM[1]) { ++ /* string delimiter start*/ ++ ++recv_str; ++ --recv_sz; ++ ++ /* Find end of string */ ++ search_ptr = strnchr(recv_str, recv_sz, NAMED_STR_END_DELIM[0]); ++ if (search_ptr) { ++ strval_len = search_ptr - recv_str; ++ /* Validate the string to ensure it contains no quotes */ ++ if (strval_len == find_quoted_string_valid_len(recv_str)) { ++ /* no need to reallocate - just modify string in place */ ++ strval = recv_str; ++ strval[strval_len] = '\0'; ++ ++ /* Move until after the end delimiter */ ++ recv_str += (strval_len + 1); ++ recv_sz -= (strval_len + 1); ++ type = KUTF_HELPER_VALTYPE_STR; ++ } else { ++ pr_err("String value contains invalid characters in rest of received string '%s'\n", recv_str); ++ err = KUTF_HELPER_ERR_CHARS_AFTER_VAL; ++ } ++ } else { ++ pr_err("End of string delimiter not found in rest of received string '%s'\n", recv_str); ++ err = KUTF_HELPER_ERR_NO_END_DELIMITER; ++ } ++ } else { ++ /* possibly a number value - strtoull will parse it */ ++ err = kstrtoull(recv_str, 0, &u64val); ++ /* unlike userspace can't get an end ptr, but if kstrtoull() ++ * reads characters after the number it'll report -EINVAL */ ++ if (!err) { ++ int len_remain = strnlen(recv_str, recv_sz); ++ ++ type = KUTF_HELPER_VALTYPE_U64; ++ recv_str += len_remain; ++ recv_sz -= len_remain; ++ } else { ++ /* special case: not a number, report as such */ ++ pr_err("Rest of received string was not a numeric value or quoted string value: '%s'\n", recv_str); ++ } ++ } ++ ++ if (type == KUTF_HELPER_VALTYPE_INVALID) ++ return err; ++ ++ /* Any remaining characters - error */ ++ if (strnlen(recv_str, recv_sz) != 0) { ++ pr_err("Characters remain after value of type %s: '%s'\n", ++ get_val_type_name(type), recv_str); ++ return KUTF_HELPER_ERR_CHARS_AFTER_VAL; ++ } ++ ++ /* Success - write into the output structure */ ++ switch (type) { ++ case KUTF_HELPER_VALTYPE_U64: ++ named_val->u.val_u64 = u64val; ++ break; ++ case KUTF_HELPER_VALTYPE_STR: ++ named_val->u.val_str = strval; ++ break; ++ default: ++ pr_err("Unreachable, fix kutf_helper_receive_named_val\n"); ++ /* Coding error, report as though 'run' file failed */ ++ return -EINVAL; ++ } ++ ++ named_val->val_name = name_str; ++ named_val->type = type; ++ ++ return KUTF_HELPER_ERR_NONE; ++} ++EXPORT_SYMBOL(kutf_helper_receive_named_val); ++ ++#define DUMMY_MSG "" ++int kutf_helper_receive_check_val( ++ struct kutf_helper_named_val *named_val, ++ struct kutf_context *context, ++ const char *expect_val_name, ++ enum kutf_helper_valtype expect_val_type) ++{ ++ int err; ++ ++ err = kutf_helper_receive_named_val(context, named_val); ++ if (err < 0) { ++ const char *msg = kutf_dsprintf(&context->fixture_pool, ++ "Failed to receive value named '%s'", ++ expect_val_name); ++ kutf_test_fail(context, msg); ++ return err; ++ } else if (err > 0) { ++ const char *msg = kutf_dsprintf(&context->fixture_pool, ++ "Named-value parse error when expecting value named '%s'", ++ expect_val_name); ++ kutf_test_fail(context, msg); ++ goto out_fail_and_fixup; ++ } ++ ++ if (strcmp(named_val->val_name, expect_val_name) != 0) { ++ const char *msg = kutf_dsprintf(&context->fixture_pool, ++ "Expecting to receive value named '%s' but got '%s'", ++ expect_val_name, named_val->val_name); ++ kutf_test_fail(context, msg); ++ goto out_fail_and_fixup; ++ } ++ ++ ++ if (named_val->type != expect_val_type) { ++ const char *msg = kutf_dsprintf(&context->fixture_pool, ++ "Expecting value named '%s' to be of type %s but got %s", ++ expect_val_name, get_val_type_name(expect_val_type), ++ get_val_type_name(named_val->type)); ++ kutf_test_fail(context, msg); ++ goto out_fail_and_fixup; ++ } ++ ++ return err; ++ ++out_fail_and_fixup: ++ /* Produce a valid but incorrect value */ ++ switch (expect_val_type) { ++ case KUTF_HELPER_VALTYPE_U64: ++ named_val->u.val_u64 = 0ull; ++ break; ++ case KUTF_HELPER_VALTYPE_STR: ++ { ++ char *str = kutf_mempool_alloc(&context->fixture_pool, sizeof(DUMMY_MSG)); ++ ++ if (!str) ++ return -1; ++ ++ strcpy(str, DUMMY_MSG); ++ named_val->u.val_str = str; ++ break; ++ } ++ default: ++ break; ++ } ++ ++ /* Indicate that this is invalid */ ++ named_val->type = KUTF_HELPER_VALTYPE_INVALID; ++ ++ /* But at least allow the caller to continue in the test with failures */ ++ return 0; ++} ++EXPORT_SYMBOL(kutf_helper_receive_check_val); ++ ++void kutf_helper_output_named_val(struct kutf_helper_named_val *named_val) ++{ ++ switch (named_val->type) { ++ case KUTF_HELPER_VALTYPE_U64: ++ pr_warn("%s=0x%llx\n", named_val->val_name, named_val->u.val_u64); ++ break; ++ case KUTF_HELPER_VALTYPE_STR: ++ pr_warn("%s=\"%s\"\n", named_val->val_name, named_val->u.val_str); ++ break; ++ case KUTF_HELPER_VALTYPE_INVALID: ++ pr_warn("%s is invalid\n", named_val->val_name); ++ break; ++ default: ++ pr_warn("%s has unknown type %d\n", named_val->val_name, named_val->type); ++ break; ++ } ++} ++EXPORT_SYMBOL(kutf_helper_output_named_val); +diff --git a/drivers/gpu/arm/bifrost/tests/kutf/kutf_mem.c b/drivers/gpu/arm/bifrost/tests/kutf/kutf_mem.c +new file mode 100755 +index 000000000000..fd98beaeb84a +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/tests/kutf/kutf_mem.c +@@ -0,0 +1,108 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* Kernel UTF memory management functions */ ++ ++#include ++#include ++#include ++ ++#include ++ ++ ++/** ++ * struct kutf_alloc_entry - Structure representing an allocation. ++ * @node: List node for use with kutf_mempool. ++ * @data: Data area of the allocation ++ */ ++struct kutf_alloc_entry { ++ struct list_head node; ++ u8 data[0]; ++}; ++ ++int kutf_mempool_init(struct kutf_mempool *pool) ++{ ++ if (!pool) { ++ pr_err("NULL pointer passed to %s\n", __func__); ++ return -1; ++ } ++ ++ INIT_LIST_HEAD(&pool->head); ++ mutex_init(&pool->lock); ++ ++ return 0; ++} ++EXPORT_SYMBOL(kutf_mempool_init); ++ ++void kutf_mempool_destroy(struct kutf_mempool *pool) ++{ ++ struct list_head *remove; ++ struct list_head *tmp; ++ ++ if (!pool) { ++ pr_err("NULL pointer passed to %s\n", __func__); ++ return; ++ } ++ ++ mutex_lock(&pool->lock); ++ list_for_each_safe(remove, tmp, &pool->head) { ++ struct kutf_alloc_entry *remove_alloc; ++ ++ remove_alloc = list_entry(remove, struct kutf_alloc_entry, node); ++ list_del(&remove_alloc->node); ++ kfree(remove_alloc); ++ } ++ mutex_unlock(&pool->lock); ++ ++} ++EXPORT_SYMBOL(kutf_mempool_destroy); ++ ++void *kutf_mempool_alloc(struct kutf_mempool *pool, size_t size) ++{ ++ struct kutf_alloc_entry *ret; ++ ++ if (!pool) { ++ pr_err("NULL pointer passed to %s\n", __func__); ++ goto fail_pool; ++ } ++ ++ mutex_lock(&pool->lock); ++ ++ ret = kmalloc(sizeof(*ret) + size, GFP_KERNEL); ++ if (!ret) { ++ pr_err("Failed to allocate memory\n"); ++ goto fail_alloc; ++ } ++ ++ INIT_LIST_HEAD(&ret->node); ++ list_add(&ret->node, &pool->head); ++ ++ mutex_unlock(&pool->lock); ++ ++ return &ret->data[0]; ++ ++fail_alloc: ++ mutex_unlock(&pool->lock); ++fail_pool: ++ return NULL; ++} ++EXPORT_SYMBOL(kutf_mempool_alloc); +diff --git a/drivers/gpu/arm/bifrost/tests/kutf/kutf_resultset.c b/drivers/gpu/arm/bifrost/tests/kutf/kutf_resultset.c +new file mode 100755 +index 000000000000..94ecfa4421e1 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/tests/kutf/kutf_resultset.c +@@ -0,0 +1,164 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* Kernel UTF result management functions */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#include ++ ++/* Lock to protect all result structures */ ++static DEFINE_SPINLOCK(kutf_result_lock); ++ ++struct kutf_result_set *kutf_create_result_set(void) ++{ ++ struct kutf_result_set *set; ++ ++ set = kmalloc(sizeof(*set), GFP_KERNEL); ++ if (!set) { ++ pr_err("Failed to allocate resultset"); ++ goto fail_alloc; ++ } ++ ++ INIT_LIST_HEAD(&set->results); ++ init_waitqueue_head(&set->waitq); ++ set->flags = 0; ++ ++ return set; ++ ++fail_alloc: ++ return NULL; ++} ++ ++int kutf_add_result(struct kutf_context *context, ++ enum kutf_result_status status, ++ const char *message) ++{ ++ struct kutf_mempool *mempool = &context->fixture_pool; ++ struct kutf_result_set *set = context->result_set; ++ /* Create the new result */ ++ struct kutf_result *new_result; ++ ++ BUG_ON(set == NULL); ++ ++ new_result = kutf_mempool_alloc(mempool, sizeof(*new_result)); ++ if (!new_result) { ++ pr_err("Result allocation failed\n"); ++ return -ENOMEM; ++ } ++ ++ INIT_LIST_HEAD(&new_result->node); ++ new_result->status = status; ++ new_result->message = message; ++ ++ spin_lock(&kutf_result_lock); ++ ++ list_add_tail(&new_result->node, &set->results); ++ ++ spin_unlock(&kutf_result_lock); ++ ++ wake_up(&set->waitq); ++ ++ return 0; ++} ++ ++void kutf_destroy_result_set(struct kutf_result_set *set) ++{ ++ if (!list_empty(&set->results)) ++ pr_err("kutf_destroy_result_set: Unread results from test\n"); ++ ++ kfree(set); ++} ++ ++static bool kutf_has_result(struct kutf_result_set *set) ++{ ++ bool has_result; ++ ++ spin_lock(&kutf_result_lock); ++ if (set->flags & KUTF_RESULT_SET_WAITING_FOR_INPUT) ++ /* Pretend there are results if waiting for input */ ++ has_result = true; ++ else ++ has_result = !list_empty(&set->results); ++ spin_unlock(&kutf_result_lock); ++ ++ return has_result; ++} ++ ++struct kutf_result *kutf_remove_result(struct kutf_result_set *set) ++{ ++ struct kutf_result *result = NULL; ++ int ret; ++ ++ do { ++ ret = wait_event_interruptible(set->waitq, ++ kutf_has_result(set)); ++ ++ if (ret) ++ return ERR_PTR(ret); ++ ++ spin_lock(&kutf_result_lock); ++ ++ if (!list_empty(&set->results)) { ++ result = list_first_entry(&set->results, ++ struct kutf_result, ++ node); ++ list_del(&result->node); ++ } else if (set->flags & KUTF_RESULT_SET_WAITING_FOR_INPUT) { ++ /* Return a fake result */ ++ static struct kutf_result waiting = { ++ .status = KUTF_RESULT_USERDATA_WAIT ++ }; ++ result = &waiting; ++ } ++ /* If result == NULL then there was a race with the event ++ * being removed between the check in kutf_has_result and ++ * the lock being obtained. In this case we retry ++ */ ++ ++ spin_unlock(&kutf_result_lock); ++ } while (result == NULL); ++ ++ return result; ++} ++ ++void kutf_set_waiting_for_input(struct kutf_result_set *set) ++{ ++ spin_lock(&kutf_result_lock); ++ set->flags |= KUTF_RESULT_SET_WAITING_FOR_INPUT; ++ spin_unlock(&kutf_result_lock); ++ ++ wake_up(&set->waitq); ++} ++ ++void kutf_clear_waiting_for_input(struct kutf_result_set *set) ++{ ++ spin_lock(&kutf_result_lock); ++ set->flags &= ~KUTF_RESULT_SET_WAITING_FOR_INPUT; ++ spin_unlock(&kutf_result_lock); ++} +diff --git a/drivers/gpu/arm/bifrost/tests/kutf/kutf_suite.c b/drivers/gpu/arm/bifrost/tests/kutf/kutf_suite.c +new file mode 100755 +index 000000000000..9dc6e2b4bad4 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/tests/kutf/kutf_suite.c +@@ -0,0 +1,1224 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014, 2017-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* Kernel UTF suite, test and fixture management including user to kernel ++ * interaction */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++ ++#include ++#include ++#include ++#include ++ ++/** ++ * struct kutf_application - Structure which represents kutf application ++ * @name: The name of this test application. ++ * @dir: The debugfs directory for this test ++ * @suite_list: List head to store all the suites which are part of this ++ * application ++ */ ++struct kutf_application { ++ const char *name; ++ struct dentry *dir; ++ struct list_head suite_list; ++}; ++ ++/** ++ * struct kutf_test_function - Structure which represents kutf test function ++ * @suite: Back reference to the suite this test function ++ * belongs to ++ * @filters: Filters that apply to this test function ++ * @test_id: Test ID ++ * @execute: Function to run for this test ++ * @test_data: Static data for this test ++ * @node: List node for test_list ++ * @variant_list: List head to store all the variants which can run on ++ * this function ++ * @dir: debugfs directory for this test function ++ */ ++struct kutf_test_function { ++ struct kutf_suite *suite; ++ unsigned int filters; ++ unsigned int test_id; ++ void (*execute)(struct kutf_context *context); ++ union kutf_callback_data test_data; ++ struct list_head node; ++ struct list_head variant_list; ++ struct dentry *dir; ++}; ++ ++/** ++ * struct kutf_test_fixture - Structure which holds information on the kutf ++ * test fixture ++ * @test_func: Test function this fixture belongs to ++ * @fixture_index: Index of this fixture ++ * @node: List node for variant_list ++ * @dir: debugfs directory for this test fixture ++ */ ++struct kutf_test_fixture { ++ struct kutf_test_function *test_func; ++ unsigned int fixture_index; ++ struct list_head node; ++ struct dentry *dir; ++}; ++ ++static struct dentry *base_dir; ++static struct workqueue_struct *kutf_workq; ++ ++/** ++ * struct kutf_convert_table - Structure which keeps test results ++ * @result_name: Status of the test result ++ * @result: Status value for a single test ++ */ ++struct kutf_convert_table { ++ char result_name[50]; ++ enum kutf_result_status result; ++}; ++ ++struct kutf_convert_table kutf_convert[] = { ++#define ADD_UTF_RESULT(_name) \ ++{ \ ++ #_name, \ ++ _name, \ ++}, ++ADD_UTF_RESULT(KUTF_RESULT_BENCHMARK) ++ADD_UTF_RESULT(KUTF_RESULT_SKIP) ++ADD_UTF_RESULT(KUTF_RESULT_UNKNOWN) ++ADD_UTF_RESULT(KUTF_RESULT_PASS) ++ADD_UTF_RESULT(KUTF_RESULT_DEBUG) ++ADD_UTF_RESULT(KUTF_RESULT_INFO) ++ADD_UTF_RESULT(KUTF_RESULT_WARN) ++ADD_UTF_RESULT(KUTF_RESULT_FAIL) ++ADD_UTF_RESULT(KUTF_RESULT_FATAL) ++ADD_UTF_RESULT(KUTF_RESULT_ABORT) ++}; ++ ++#define UTF_CONVERT_SIZE (ARRAY_SIZE(kutf_convert)) ++ ++/** ++ * kutf_create_context() - Create a test context in which a specific fixture ++ * of an application will be run and its results ++ * reported back to the user ++ * @test_fix: Test fixture to be run. ++ * ++ * The context's refcount will be initialized to 1. ++ * ++ * Return: Returns the created test context on success or NULL on failure ++ */ ++static struct kutf_context *kutf_create_context( ++ struct kutf_test_fixture *test_fix); ++ ++/** ++ * kutf_destroy_context() - Destroy a previously created test context, only ++ * once its refcount has become zero ++ * @kref: pointer to kref member within the context ++ * ++ * This should only be used via a kref_put() call on the context's kref member ++ */ ++static void kutf_destroy_context(struct kref *kref); ++ ++/** ++ * kutf_context_get() - increment refcount on a context ++ * @context: the kutf context ++ * ++ * This must be used when the lifetime of the context might exceed that of the ++ * thread creating @context ++ */ ++static void kutf_context_get(struct kutf_context *context); ++ ++/** ++ * kutf_context_put() - decrement refcount on a context, destroying it when it ++ * reached zero ++ * @context: the kutf context ++ * ++ * This must be used only after a corresponding kutf_context_get() call on ++ * @context, and the caller no longer needs access to @context. ++ */ ++static void kutf_context_put(struct kutf_context *context); ++ ++/** ++ * kutf_set_result() - Set the test result against the specified test context ++ * @context: Test context ++ * @status: Result status ++ */ ++static void kutf_set_result(struct kutf_context *context, ++ enum kutf_result_status status); ++ ++/** ++ * kutf_set_expected_result() - Set the expected test result for the specified ++ * test context ++ * @context: Test context ++ * @expected_status: Expected result status ++ */ ++static void kutf_set_expected_result(struct kutf_context *context, ++ enum kutf_result_status expected_status); ++ ++/** ++ * kutf_result_to_string() - Converts a KUTF result into a string ++ * @result_str: Output result string ++ * @result: Result status to convert ++ * ++ * Return: 1 if test result was successfully converted to string, 0 otherwise ++ */ ++static int kutf_result_to_string(char **result_str, ++ enum kutf_result_status result) ++{ ++ int i; ++ int ret = 0; ++ ++ for (i = 0; i < UTF_CONVERT_SIZE; i++) { ++ if (result == kutf_convert[i].result) { ++ *result_str = kutf_convert[i].result_name; ++ ret = 1; ++ } ++ } ++ return ret; ++} ++ ++/** ++ * kutf_debugfs_const_string_read() - Simple debugfs read callback which ++ * returns a constant string ++ * @file: Opened file to read from ++ * @buf: User buffer to write the data into ++ * @len: Amount of data to read ++ * @ppos: Offset into file to read from ++ * ++ * Return: On success, the number of bytes read and offset @ppos advanced by ++ * this number; on error, negative value ++ */ ++static ssize_t kutf_debugfs_const_string_read(struct file *file, ++ char __user *buf, size_t len, loff_t *ppos) ++{ ++ char *str = file->private_data; ++ ++ return simple_read_from_buffer(buf, len, ppos, str, strlen(str)); ++} ++ ++static const struct file_operations kutf_debugfs_const_string_ops = { ++ .owner = THIS_MODULE, ++ .open = simple_open, ++ .read = kutf_debugfs_const_string_read, ++ .llseek = default_llseek, ++}; ++ ++/** ++ * kutf_add_explicit_result() - Check if an explicit result needs to be added ++ * @context: KUTF test context ++ */ ++static void kutf_add_explicit_result(struct kutf_context *context) ++{ ++ switch (context->expected_status) { ++ case KUTF_RESULT_UNKNOWN: ++ break; ++ ++ case KUTF_RESULT_WARN: ++ if (context->status == KUTF_RESULT_WARN) ++ kutf_test_pass(context, ++ "Pass (expected warn occurred)"); ++ else if (context->status != KUTF_RESULT_SKIP) ++ kutf_test_fail(context, ++ "Fail (expected warn missing)"); ++ break; ++ ++ case KUTF_RESULT_FAIL: ++ if (context->status == KUTF_RESULT_FAIL) ++ kutf_test_pass(context, ++ "Pass (expected fail occurred)"); ++ else if (context->status != KUTF_RESULT_SKIP) { ++ /* Force the expected status so the fail gets logged */ ++ context->expected_status = KUTF_RESULT_PASS; ++ kutf_test_fail(context, ++ "Fail (expected fail missing)"); ++ } ++ break; ++ ++ case KUTF_RESULT_FATAL: ++ if (context->status == KUTF_RESULT_FATAL) ++ kutf_test_pass(context, ++ "Pass (expected fatal occurred)"); ++ else if (context->status != KUTF_RESULT_SKIP) ++ kutf_test_fail(context, ++ "Fail (expected fatal missing)"); ++ break; ++ ++ case KUTF_RESULT_ABORT: ++ if (context->status == KUTF_RESULT_ABORT) ++ kutf_test_pass(context, ++ "Pass (expected abort occurred)"); ++ else if (context->status != KUTF_RESULT_SKIP) ++ kutf_test_fail(context, ++ "Fail (expected abort missing)"); ++ break; ++ default: ++ break; ++ } ++} ++ ++static void kutf_run_test(struct work_struct *data) ++{ ++ struct kutf_context *test_context = container_of(data, ++ struct kutf_context, work); ++ struct kutf_suite *suite = test_context->suite; ++ struct kutf_test_function *test_func; ++ ++ test_func = test_context->test_fix->test_func; ++ ++ /* ++ * Call the create fixture function if required before the ++ * fixture is run ++ */ ++ if (suite->create_fixture) ++ test_context->fixture = suite->create_fixture(test_context); ++ ++ /* Only run the test if the fixture was created (if required) */ ++ if ((suite->create_fixture && test_context->fixture) || ++ (!suite->create_fixture)) { ++ /* Run this fixture */ ++ test_func->execute(test_context); ++ ++ if (suite->remove_fixture) ++ suite->remove_fixture(test_context); ++ ++ kutf_add_explicit_result(test_context); ++ } ++ ++ kutf_add_result(test_context, KUTF_RESULT_TEST_FINISHED, NULL); ++ ++ kutf_context_put(test_context); ++} ++ ++/** ++ * kutf_debugfs_run_open() Debugfs open callback for the "run" entry. ++ * @inode: inode of the opened file ++ * @file: Opened file to read from ++ * ++ * This function creates a KUTF context and queues it onto a workqueue to be ++ * run asynchronously. The resulting file descriptor can be used to communicate ++ * userdata to the test and to read back the results of the test execution. ++ * ++ * Return: 0 on success ++ */ ++static int kutf_debugfs_run_open(struct inode *inode, struct file *file) ++{ ++ struct kutf_test_fixture *test_fix = inode->i_private; ++ struct kutf_context *test_context; ++ int err = 0; ++ ++ test_context = kutf_create_context(test_fix); ++ if (!test_context) { ++ err = -ENOMEM; ++ goto finish; ++ } ++ ++ file->private_data = test_context; ++ ++ /* This reference is release by the kutf_run_test */ ++ kutf_context_get(test_context); ++ ++ queue_work(kutf_workq, &test_context->work); ++ ++finish: ++ return err; ++} ++ ++#define USERDATA_WARNING_MESSAGE "WARNING: This test requires userdata\n" ++ ++/** ++ * kutf_debugfs_run_read() - Debugfs read callback for the "run" entry. ++ * @file: Opened file to read from ++ * @buf: User buffer to write the data into ++ * @len: Amount of data to read ++ * @ppos: Offset into file to read from ++ * ++ * This function emits the results of the test, blocking until they are ++ * available. ++ * ++ * If the test involves user data then this will also return user data records ++ * to user space. If the test is waiting for user data then this function will ++ * output a message (to make the likes of 'cat' display it), followed by ++ * returning 0 to mark the end of file. ++ * ++ * Results will be emitted one at a time, once all the results have been read ++ * 0 will be returned to indicate there is no more data. ++ * ++ * Return: Number of bytes read. ++ */ ++static ssize_t kutf_debugfs_run_read(struct file *file, char __user *buf, ++ size_t len, loff_t *ppos) ++{ ++ struct kutf_context *test_context = file->private_data; ++ struct kutf_result *res; ++ unsigned long bytes_not_copied; ++ ssize_t bytes_copied = 0; ++ char *kutf_str_ptr = NULL; ++ size_t kutf_str_len = 0; ++ size_t message_len = 0; ++ char separator = ':'; ++ char terminator = '\n'; ++ ++ res = kutf_remove_result(test_context->result_set); ++ ++ if (IS_ERR(res)) ++ return PTR_ERR(res); ++ ++ /* ++ * Handle 'fake' results - these results are converted to another ++ * form before being returned from the kernel ++ */ ++ switch (res->status) { ++ case KUTF_RESULT_TEST_FINISHED: ++ return 0; ++ case KUTF_RESULT_USERDATA_WAIT: ++ if (test_context->userdata.flags & ++ KUTF_USERDATA_WARNING_OUTPUT) { ++ /* ++ * Warning message already output, ++ * signal end-of-file ++ */ ++ return 0; ++ } ++ ++ message_len = sizeof(USERDATA_WARNING_MESSAGE)-1; ++ if (message_len > len) ++ message_len = len; ++ ++ bytes_not_copied = copy_to_user(buf, ++ USERDATA_WARNING_MESSAGE, ++ message_len); ++ if (bytes_not_copied != 0) ++ return -EFAULT; ++ test_context->userdata.flags |= KUTF_USERDATA_WARNING_OUTPUT; ++ return message_len; ++ case KUTF_RESULT_USERDATA: ++ message_len = strlen(res->message); ++ if (message_len > len-1) { ++ message_len = len-1; ++ pr_warn("User data truncated, read not long enough\n"); ++ } ++ bytes_not_copied = copy_to_user(buf, res->message, ++ message_len); ++ if (bytes_not_copied != 0) { ++ pr_warn("Failed to copy data to user space buffer\n"); ++ return -EFAULT; ++ } ++ /* Finally the terminator */ ++ bytes_not_copied = copy_to_user(&buf[message_len], ++ &terminator, 1); ++ if (bytes_not_copied != 0) { ++ pr_warn("Failed to copy data to user space buffer\n"); ++ return -EFAULT; ++ } ++ return message_len+1; ++ default: ++ /* Fall through - this is a test result */ ++ break; ++ } ++ ++ /* Note: This code assumes a result is read completely */ ++ kutf_result_to_string(&kutf_str_ptr, res->status); ++ if (kutf_str_ptr) ++ kutf_str_len = strlen(kutf_str_ptr); ++ ++ if (res->message) ++ message_len = strlen(res->message); ++ ++ if ((kutf_str_len + 1 + message_len + 1) > len) { ++ pr_err("Not enough space in user buffer for a single result"); ++ return 0; ++ } ++ ++ /* First copy the result string */ ++ if (kutf_str_ptr) { ++ bytes_not_copied = copy_to_user(&buf[0], kutf_str_ptr, ++ kutf_str_len); ++ bytes_copied += kutf_str_len - bytes_not_copied; ++ if (bytes_not_copied) ++ goto exit; ++ } ++ ++ /* Then the separator */ ++ bytes_not_copied = copy_to_user(&buf[bytes_copied], ++ &separator, 1); ++ bytes_copied += 1 - bytes_not_copied; ++ if (bytes_not_copied) ++ goto exit; ++ ++ /* Finally Next copy the result string */ ++ if (res->message) { ++ bytes_not_copied = copy_to_user(&buf[bytes_copied], ++ res->message, message_len); ++ bytes_copied += message_len - bytes_not_copied; ++ if (bytes_not_copied) ++ goto exit; ++ } ++ ++ /* Finally the terminator */ ++ bytes_not_copied = copy_to_user(&buf[bytes_copied], ++ &terminator, 1); ++ bytes_copied += 1 - bytes_not_copied; ++ ++exit: ++ return bytes_copied; ++} ++ ++/** ++ * kutf_debugfs_run_write() Debugfs write callback for the "run" entry. ++ * @file: Opened file to write to ++ * @buf: User buffer to read the data from ++ * @len: Amount of data to write ++ * @ppos: Offset into file to write to ++ * ++ * This function allows user and kernel to exchange extra data necessary for ++ * the test fixture. ++ * ++ * The data is added to the first struct kutf_context running the fixture ++ * ++ * Return: Number of bytes written ++ */ ++static ssize_t kutf_debugfs_run_write(struct file *file, ++ const char __user *buf, size_t len, loff_t *ppos) ++{ ++ int ret = 0; ++ struct kutf_context *test_context = file->private_data; ++ ++ if (len > KUTF_MAX_LINE_LENGTH) ++ return -EINVAL; ++ ++ ret = kutf_helper_input_enqueue(test_context, buf, len); ++ if (ret < 0) ++ return ret; ++ ++ return len; ++} ++ ++/** ++ * kutf_debugfs_run_release() - Debugfs release callback for the "run" entry. ++ * @inode: File entry representation ++ * @file: A specific opening of the file ++ * ++ * Release any resources that were created during the opening of the file ++ * ++ * Note that resources may not be released immediately, that might only happen ++ * later when other users of the kutf_context release their refcount. ++ * ++ * Return: 0 on success ++ */ ++static int kutf_debugfs_run_release(struct inode *inode, struct file *file) ++{ ++ struct kutf_context *test_context = file->private_data; ++ ++ kutf_helper_input_enqueue_end_of_data(test_context); ++ ++ kutf_context_put(test_context); ++ return 0; ++} ++ ++static const struct file_operations kutf_debugfs_run_ops = { ++ .owner = THIS_MODULE, ++ .open = kutf_debugfs_run_open, ++ .read = kutf_debugfs_run_read, ++ .write = kutf_debugfs_run_write, ++ .release = kutf_debugfs_run_release, ++ .llseek = default_llseek, ++}; ++ ++/** ++ * create_fixture_variant() - Creates a fixture variant for the specified ++ * test function and index and the debugfs entries ++ * that represent it. ++ * @test_func: Test function ++ * @fixture_index: Fixture index ++ * ++ * Return: 0 on success, negative value corresponding to error code in failure ++ */ ++static int create_fixture_variant(struct kutf_test_function *test_func, ++ unsigned int fixture_index) ++{ ++ struct kutf_test_fixture *test_fix; ++ char name[11]; /* Enough to print the MAX_UINT32 + the null terminator */ ++ struct dentry *tmp; ++ int err; ++ ++ test_fix = kmalloc(sizeof(*test_fix), GFP_KERNEL); ++ if (!test_fix) { ++ pr_err("Failed to create debugfs directory when adding fixture\n"); ++ err = -ENOMEM; ++ goto fail_alloc; ++ } ++ ++ test_fix->test_func = test_func; ++ test_fix->fixture_index = fixture_index; ++ ++ snprintf(name, sizeof(name), "%d", fixture_index); ++ test_fix->dir = debugfs_create_dir(name, test_func->dir); ++ if (!test_func->dir) { ++ pr_err("Failed to create debugfs directory when adding fixture\n"); ++ /* Might not be the right error, we don't get it passed back to us */ ++ err = -EEXIST; ++ goto fail_dir; ++ } ++ ++ tmp = debugfs_create_file("type", S_IROTH, test_fix->dir, "fixture\n", ++ &kutf_debugfs_const_string_ops); ++ if (!tmp) { ++ pr_err("Failed to create debugfs file \"type\" when adding fixture\n"); ++ /* Might not be the right error, we don't get it passed back to us */ ++ err = -EEXIST; ++ goto fail_file; ++ } ++ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 7, 0) ++ tmp = debugfs_create_file_unsafe( ++#else ++ tmp = debugfs_create_file( ++#endif ++ "run", 0600, test_fix->dir, ++ test_fix, ++ &kutf_debugfs_run_ops); ++ if (!tmp) { ++ pr_err("Failed to create debugfs file \"run\" when adding fixture\n"); ++ /* Might not be the right error, we don't get it passed back to us */ ++ err = -EEXIST; ++ goto fail_file; ++ } ++ ++ list_add(&test_fix->node, &test_func->variant_list); ++ return 0; ++ ++fail_file: ++ debugfs_remove_recursive(test_fix->dir); ++fail_dir: ++ kfree(test_fix); ++fail_alloc: ++ return err; ++} ++ ++/** ++ * kutf_remove_test_variant() - Destroy a previously created fixture variant. ++ * @test_fix: Test fixture ++ */ ++static void kutf_remove_test_variant(struct kutf_test_fixture *test_fix) ++{ ++ debugfs_remove_recursive(test_fix->dir); ++ kfree(test_fix); ++} ++ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0) ++/* Adapting to the upstream debugfs_create_x32() change */ ++static int ktufp_u32_get(void *data, u64 *val) ++{ ++ *val = *(u32 *)data; ++ return 0; ++} ++ ++DEFINE_DEBUGFS_ATTRIBUTE(kutfp_fops_x32_ro, ktufp_u32_get, NULL, "0x%08llx\n"); ++#endif ++ ++void kutf_add_test_with_filters_and_data( ++ struct kutf_suite *suite, ++ unsigned int id, ++ const char *name, ++ void (*execute)(struct kutf_context *context), ++ unsigned int filters, ++ union kutf_callback_data test_data) ++{ ++ struct kutf_test_function *test_func; ++ struct dentry *tmp; ++ unsigned int i; ++ ++ test_func = kmalloc(sizeof(*test_func), GFP_KERNEL); ++ if (!test_func) { ++ pr_err("Failed to allocate memory when adding test %s\n", name); ++ goto fail_alloc; ++ } ++ ++ INIT_LIST_HEAD(&test_func->variant_list); ++ ++ test_func->dir = debugfs_create_dir(name, suite->dir); ++ if (!test_func->dir) { ++ pr_err("Failed to create debugfs directory when adding test %s\n", name); ++ goto fail_dir; ++ } ++ ++ tmp = debugfs_create_file("type", S_IROTH, test_func->dir, "test\n", ++ &kutf_debugfs_const_string_ops); ++ if (!tmp) { ++ pr_err("Failed to create debugfs file \"type\" when adding test %s\n", name); ++ goto fail_file; ++ } ++ ++ test_func->filters = filters; ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0) ++ tmp = debugfs_create_file_unsafe("filters", S_IROTH, test_func->dir, ++ &test_func->filters, &kutfp_fops_x32_ro); ++#else ++ tmp = debugfs_create_x32("filters", S_IROTH, test_func->dir, ++ &test_func->filters); ++#endif ++ if (!tmp) { ++ pr_err("Failed to create debugfs file \"filters\" when adding test %s\n", name); ++ goto fail_file; ++ } ++ ++ test_func->test_id = id; ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0) ++ debugfs_create_u32("test_id", S_IROTH, test_func->dir, ++ &test_func->test_id); ++#else ++ tmp = debugfs_create_u32("test_id", S_IROTH, test_func->dir, ++ &test_func->test_id); ++ if (!tmp) { ++ pr_err("Failed to create debugfs file \"test_id\" when adding test %s\n", name); ++ goto fail_file; ++ } ++#endif ++ ++ for (i = 0; i < suite->fixture_variants; i++) { ++ if (create_fixture_variant(test_func, i)) { ++ pr_err("Failed to create fixture %d when adding test %s\n", i, name); ++ goto fail_file; ++ } ++ } ++ ++ test_func->suite = suite; ++ test_func->execute = execute; ++ test_func->test_data = test_data; ++ ++ list_add(&test_func->node, &suite->test_list); ++ return; ++ ++fail_file: ++ debugfs_remove_recursive(test_func->dir); ++fail_dir: ++ kfree(test_func); ++fail_alloc: ++ return; ++} ++EXPORT_SYMBOL(kutf_add_test_with_filters_and_data); ++ ++void kutf_add_test_with_filters( ++ struct kutf_suite *suite, ++ unsigned int id, ++ const char *name, ++ void (*execute)(struct kutf_context *context), ++ unsigned int filters) ++{ ++ union kutf_callback_data data; ++ ++ data.ptr_value = NULL; ++ ++ kutf_add_test_with_filters_and_data(suite, ++ id, ++ name, ++ execute, ++ suite->suite_default_flags, ++ data); ++} ++EXPORT_SYMBOL(kutf_add_test_with_filters); ++ ++void kutf_add_test(struct kutf_suite *suite, ++ unsigned int id, ++ const char *name, ++ void (*execute)(struct kutf_context *context)) ++{ ++ union kutf_callback_data data; ++ ++ data.ptr_value = NULL; ++ ++ kutf_add_test_with_filters_and_data(suite, ++ id, ++ name, ++ execute, ++ suite->suite_default_flags, ++ data); ++} ++EXPORT_SYMBOL(kutf_add_test); ++ ++/** ++ * kutf_remove_test(): Remove a previously added test function. ++ * @test_func: Test function ++ */ ++static void kutf_remove_test(struct kutf_test_function *test_func) ++{ ++ struct list_head *pos; ++ struct list_head *tmp; ++ ++ list_for_each_safe(pos, tmp, &test_func->variant_list) { ++ struct kutf_test_fixture *test_fix; ++ ++ test_fix = list_entry(pos, struct kutf_test_fixture, node); ++ kutf_remove_test_variant(test_fix); ++ } ++ ++ list_del(&test_func->node); ++ debugfs_remove_recursive(test_func->dir); ++ kfree(test_func); ++} ++ ++struct kutf_suite *kutf_create_suite_with_filters_and_data( ++ struct kutf_application *app, ++ const char *name, ++ unsigned int fixture_count, ++ void *(*create_fixture)(struct kutf_context *context), ++ void (*remove_fixture)(struct kutf_context *context), ++ unsigned int filters, ++ union kutf_callback_data suite_data) ++{ ++ struct kutf_suite *suite; ++ struct dentry *tmp; ++ ++ suite = kmalloc(sizeof(*suite), GFP_KERNEL); ++ if (!suite) { ++ pr_err("Failed to allocate memory when creating suite %s\n", name); ++ goto fail_kmalloc; ++ } ++ ++ suite->dir = debugfs_create_dir(name, app->dir); ++ if (!suite->dir) { ++ pr_err("Failed to create debugfs directory when adding test %s\n", name); ++ goto fail_debugfs; ++ } ++ ++ tmp = debugfs_create_file("type", S_IROTH, suite->dir, "suite\n", ++ &kutf_debugfs_const_string_ops); ++ if (!tmp) { ++ pr_err("Failed to create debugfs file \"type\" when adding test %s\n", name); ++ goto fail_file; ++ } ++ ++ INIT_LIST_HEAD(&suite->test_list); ++ suite->app = app; ++ suite->name = name; ++ suite->fixture_variants = fixture_count; ++ suite->create_fixture = create_fixture; ++ suite->remove_fixture = remove_fixture; ++ suite->suite_default_flags = filters; ++ suite->suite_data = suite_data; ++ ++ list_add(&suite->node, &app->suite_list); ++ ++ return suite; ++ ++fail_file: ++ debugfs_remove_recursive(suite->dir); ++fail_debugfs: ++ kfree(suite); ++fail_kmalloc: ++ return NULL; ++} ++EXPORT_SYMBOL(kutf_create_suite_with_filters_and_data); ++ ++struct kutf_suite *kutf_create_suite_with_filters( ++ struct kutf_application *app, ++ const char *name, ++ unsigned int fixture_count, ++ void *(*create_fixture)(struct kutf_context *context), ++ void (*remove_fixture)(struct kutf_context *context), ++ unsigned int filters) ++{ ++ union kutf_callback_data data; ++ ++ data.ptr_value = NULL; ++ return kutf_create_suite_with_filters_and_data(app, ++ name, ++ fixture_count, ++ create_fixture, ++ remove_fixture, ++ filters, ++ data); ++} ++EXPORT_SYMBOL(kutf_create_suite_with_filters); ++ ++struct kutf_suite *kutf_create_suite( ++ struct kutf_application *app, ++ const char *name, ++ unsigned int fixture_count, ++ void *(*create_fixture)(struct kutf_context *context), ++ void (*remove_fixture)(struct kutf_context *context)) ++{ ++ union kutf_callback_data data; ++ ++ data.ptr_value = NULL; ++ return kutf_create_suite_with_filters_and_data(app, ++ name, ++ fixture_count, ++ create_fixture, ++ remove_fixture, ++ KUTF_F_TEST_GENERIC, ++ data); ++} ++EXPORT_SYMBOL(kutf_create_suite); ++ ++/** ++ * kutf_destroy_suite() - Destroy a previously added test suite. ++ * @suite: Test suite ++ */ ++static void kutf_destroy_suite(struct kutf_suite *suite) ++{ ++ struct list_head *pos; ++ struct list_head *tmp; ++ ++ list_for_each_safe(pos, tmp, &suite->test_list) { ++ struct kutf_test_function *test_func; ++ ++ test_func = list_entry(pos, struct kutf_test_function, node); ++ kutf_remove_test(test_func); ++ } ++ ++ list_del(&suite->node); ++ debugfs_remove_recursive(suite->dir); ++ kfree(suite); ++} ++ ++struct kutf_application *kutf_create_application(const char *name) ++{ ++ struct kutf_application *app; ++ struct dentry *tmp; ++ ++ app = kmalloc(sizeof(*app), GFP_KERNEL); ++ if (!app) { ++ pr_err("Failed to create allocate memory when creating application %s\n", name); ++ goto fail_kmalloc; ++ } ++ ++ app->dir = debugfs_create_dir(name, base_dir); ++ if (!app->dir) { ++ pr_err("Failed to create debugfs direcotry when creating application %s\n", name); ++ goto fail_debugfs; ++ } ++ ++ tmp = debugfs_create_file("type", S_IROTH, app->dir, "application\n", ++ &kutf_debugfs_const_string_ops); ++ if (!tmp) { ++ pr_err("Failed to create debugfs file \"type\" when creating application %s\n", name); ++ goto fail_file; ++ } ++ ++ INIT_LIST_HEAD(&app->suite_list); ++ app->name = name; ++ ++ return app; ++ ++fail_file: ++ debugfs_remove_recursive(app->dir); ++fail_debugfs: ++ kfree(app); ++fail_kmalloc: ++ return NULL; ++} ++EXPORT_SYMBOL(kutf_create_application); ++ ++void kutf_destroy_application(struct kutf_application *app) ++{ ++ struct list_head *pos; ++ struct list_head *tmp; ++ ++ list_for_each_safe(pos, tmp, &app->suite_list) { ++ struct kutf_suite *suite; ++ ++ suite = list_entry(pos, struct kutf_suite, node); ++ kutf_destroy_suite(suite); ++ } ++ ++ debugfs_remove_recursive(app->dir); ++ kfree(app); ++} ++EXPORT_SYMBOL(kutf_destroy_application); ++ ++static struct kutf_context *kutf_create_context( ++ struct kutf_test_fixture *test_fix) ++{ ++ struct kutf_context *new_context; ++ ++ new_context = kmalloc(sizeof(*new_context), GFP_KERNEL); ++ if (!new_context) { ++ pr_err("Failed to allocate test context"); ++ goto fail_alloc; ++ } ++ ++ new_context->result_set = kutf_create_result_set(); ++ if (!new_context->result_set) { ++ pr_err("Failed to create result set"); ++ goto fail_result_set; ++ } ++ ++ new_context->test_fix = test_fix; ++ /* Save the pointer to the suite as the callbacks will require it */ ++ new_context->suite = test_fix->test_func->suite; ++ new_context->status = KUTF_RESULT_UNKNOWN; ++ new_context->expected_status = KUTF_RESULT_UNKNOWN; ++ ++ kutf_mempool_init(&new_context->fixture_pool); ++ new_context->fixture = NULL; ++ new_context->fixture_index = test_fix->fixture_index; ++ new_context->fixture_name = NULL; ++ new_context->test_data = test_fix->test_func->test_data; ++ ++ new_context->userdata.flags = 0; ++ INIT_LIST_HEAD(&new_context->userdata.input_head); ++ init_waitqueue_head(&new_context->userdata.input_waitq); ++ ++ INIT_WORK(&new_context->work, kutf_run_test); ++ ++ kref_init(&new_context->kref); ++ ++ return new_context; ++ ++fail_result_set: ++ kfree(new_context); ++fail_alloc: ++ return NULL; ++} ++ ++static void kutf_destroy_context(struct kref *kref) ++{ ++ struct kutf_context *context; ++ ++ context = container_of(kref, struct kutf_context, kref); ++ kutf_destroy_result_set(context->result_set); ++ kutf_mempool_destroy(&context->fixture_pool); ++ kfree(context); ++} ++ ++static void kutf_context_get(struct kutf_context *context) ++{ ++ kref_get(&context->kref); ++} ++ ++static void kutf_context_put(struct kutf_context *context) ++{ ++ kref_put(&context->kref, kutf_destroy_context); ++} ++ ++ ++static void kutf_set_result(struct kutf_context *context, ++ enum kutf_result_status status) ++{ ++ context->status = status; ++} ++ ++static void kutf_set_expected_result(struct kutf_context *context, ++ enum kutf_result_status expected_status) ++{ ++ context->expected_status = expected_status; ++} ++ ++/** ++ * kutf_test_log_result() - Log a result for the specified test context ++ * @context: Test context ++ * @message: Result string ++ * @new_status: Result status ++ */ ++static void kutf_test_log_result( ++ struct kutf_context *context, ++ const char *message, ++ enum kutf_result_status new_status) ++{ ++ if (context->status < new_status) ++ context->status = new_status; ++ ++ if (context->expected_status != new_status) ++ kutf_add_result(context, new_status, message); ++} ++ ++void kutf_test_log_result_external( ++ struct kutf_context *context, ++ const char *message, ++ enum kutf_result_status new_status) ++{ ++ kutf_test_log_result(context, message, new_status); ++} ++EXPORT_SYMBOL(kutf_test_log_result_external); ++ ++void kutf_test_expect_abort(struct kutf_context *context) ++{ ++ kutf_set_expected_result(context, KUTF_RESULT_ABORT); ++} ++EXPORT_SYMBOL(kutf_test_expect_abort); ++ ++void kutf_test_expect_fatal(struct kutf_context *context) ++{ ++ kutf_set_expected_result(context, KUTF_RESULT_FATAL); ++} ++EXPORT_SYMBOL(kutf_test_expect_fatal); ++ ++void kutf_test_expect_fail(struct kutf_context *context) ++{ ++ kutf_set_expected_result(context, KUTF_RESULT_FAIL); ++} ++EXPORT_SYMBOL(kutf_test_expect_fail); ++ ++void kutf_test_expect_warn(struct kutf_context *context) ++{ ++ kutf_set_expected_result(context, KUTF_RESULT_WARN); ++} ++EXPORT_SYMBOL(kutf_test_expect_warn); ++ ++void kutf_test_expect_pass(struct kutf_context *context) ++{ ++ kutf_set_expected_result(context, KUTF_RESULT_PASS); ++} ++EXPORT_SYMBOL(kutf_test_expect_pass); ++ ++void kutf_test_skip(struct kutf_context *context) ++{ ++ kutf_set_result(context, KUTF_RESULT_SKIP); ++ kutf_set_expected_result(context, KUTF_RESULT_UNKNOWN); ++ ++ kutf_test_log_result(context, "Test skipped", KUTF_RESULT_SKIP); ++} ++EXPORT_SYMBOL(kutf_test_skip); ++ ++void kutf_test_skip_msg(struct kutf_context *context, const char *message) ++{ ++ kutf_set_result(context, KUTF_RESULT_SKIP); ++ kutf_set_expected_result(context, KUTF_RESULT_UNKNOWN); ++ ++ kutf_test_log_result(context, kutf_dsprintf(&context->fixture_pool, ++ "Test skipped: %s", message), KUTF_RESULT_SKIP); ++ kutf_test_log_result(context, "!!!Test skipped!!!", KUTF_RESULT_SKIP); ++} ++EXPORT_SYMBOL(kutf_test_skip_msg); ++ ++void kutf_test_debug(struct kutf_context *context, char const *message) ++{ ++ kutf_test_log_result(context, message, KUTF_RESULT_DEBUG); ++} ++EXPORT_SYMBOL(kutf_test_debug); ++ ++void kutf_test_pass(struct kutf_context *context, char const *message) ++{ ++ static const char explicit_message[] = "(explicit pass)"; ++ ++ if (!message) ++ message = explicit_message; ++ ++ kutf_test_log_result(context, message, KUTF_RESULT_PASS); ++} ++EXPORT_SYMBOL(kutf_test_pass); ++ ++void kutf_test_info(struct kutf_context *context, char const *message) ++{ ++ kutf_test_log_result(context, message, KUTF_RESULT_INFO); ++} ++EXPORT_SYMBOL(kutf_test_info); ++ ++void kutf_test_warn(struct kutf_context *context, char const *message) ++{ ++ kutf_test_log_result(context, message, KUTF_RESULT_WARN); ++} ++EXPORT_SYMBOL(kutf_test_warn); ++ ++void kutf_test_fail(struct kutf_context *context, char const *message) ++{ ++ kutf_test_log_result(context, message, KUTF_RESULT_FAIL); ++} ++EXPORT_SYMBOL(kutf_test_fail); ++ ++void kutf_test_fatal(struct kutf_context *context, char const *message) ++{ ++ kutf_test_log_result(context, message, KUTF_RESULT_FATAL); ++} ++EXPORT_SYMBOL(kutf_test_fatal); ++ ++void kutf_test_abort(struct kutf_context *context) ++{ ++ kutf_test_log_result(context, "", KUTF_RESULT_ABORT); ++} ++EXPORT_SYMBOL(kutf_test_abort); ++ ++#ifdef CONFIG_DEBUG_FS ++ ++/** ++ * init_kutf_core() - Module entry point. ++ * ++ * Create the base entry point in debugfs. ++ */ ++static int __init init_kutf_core(void) ++{ ++ kutf_workq = alloc_workqueue("kutf workq", WQ_UNBOUND, 1); ++ if (!kutf_workq) ++ return -ENOMEM; ++ ++ base_dir = debugfs_create_dir("kutf_tests", NULL); ++ if (!base_dir) { ++ destroy_workqueue(kutf_workq); ++ kutf_workq = NULL; ++ return -ENOMEM; ++ } ++ ++ return 0; ++} ++ ++/** ++ * exit_kutf_core() - Module exit point. ++ * ++ * Remove the base entry point in debugfs. ++ */ ++static void __exit exit_kutf_core(void) ++{ ++ debugfs_remove_recursive(base_dir); ++ ++ if (kutf_workq) ++ destroy_workqueue(kutf_workq); ++} ++ ++#else /* CONFIG_DEBUG_FS */ ++ ++/** ++ * init_kutf_core() - Module entry point. ++ * ++ * Stub for when build against a kernel without debugfs support ++ */ ++static int __init init_kutf_core(void) ++{ ++ pr_debug("KUTF requires a kernel with debug fs support"); ++ ++ return -ENODEV; ++} ++ ++/** ++ * exit_kutf_core() - Module exit point. ++ * ++ * Stub for when build against a kernel without debugfs support ++ */ ++static void __exit exit_kutf_core(void) ++{ ++} ++#endif /* CONFIG_DEBUG_FS */ ++ ++MODULE_LICENSE("GPL"); ++ ++module_init(init_kutf_core); ++module_exit(exit_kutf_core); +diff --git a/drivers/gpu/arm/bifrost/tests/kutf/kutf_utils.c b/drivers/gpu/arm/bifrost/tests/kutf/kutf_utils.c +new file mode 100755 +index 000000000000..7f5ac517fdb4 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/tests/kutf/kutf_utils.c +@@ -0,0 +1,76 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* Kernel UTF utility functions */ ++ ++#include ++#include ++#include ++#include ++ ++#include ++#include ++ ++static char tmp_buffer[KUTF_MAX_DSPRINTF_LEN]; ++ ++DEFINE_MUTEX(buffer_lock); ++ ++const char *kutf_dsprintf(struct kutf_mempool *pool, ++ const char *fmt, ...) ++{ ++ va_list args; ++ int len; ++ int size; ++ void *buffer; ++ ++ mutex_lock(&buffer_lock); ++ va_start(args, fmt); ++ len = vsnprintf(tmp_buffer, sizeof(tmp_buffer), fmt, args); ++ va_end(args); ++ ++ if (len < 0) { ++ pr_err("kutf_dsprintf: Bad format dsprintf format %s\n", fmt); ++ goto fail_format; ++ } ++ ++ if (len >= sizeof(tmp_buffer)) { ++ pr_warn("kutf_dsprintf: Truncated dsprintf message %s\n", fmt); ++ size = sizeof(tmp_buffer); ++ } else { ++ size = len + 1; ++ } ++ ++ buffer = kutf_mempool_alloc(pool, size); ++ if (!buffer) ++ goto fail_alloc; ++ ++ memcpy(buffer, tmp_buffer, size); ++ mutex_unlock(&buffer_lock); ++ ++ return buffer; ++ ++fail_alloc: ++fail_format: ++ mutex_unlock(&buffer_lock); ++ return NULL; ++} ++EXPORT_SYMBOL(kutf_dsprintf); +diff --git a/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/Kbuild b/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/Kbuild +new file mode 100755 +index 000000000000..f5565d30f9cf +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/Kbuild +@@ -0,0 +1,26 @@ ++# ++# (C) COPYRIGHT 2020 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# GNU General Public License for more details. ++# ++# You should have received a copy of the GNU General Public License ++# along with this program; if not, you can access it online at ++# http://www.gnu.org/licenses/gpl-2.0.html. ++# ++# SPDX-License-Identifier: GPL-2.0 ++# ++# ++ ++ccflags-y += -I$(src)/../include -I$(src)/../../../ -I$(src)/../../ -I$(src)/../../backend/gpu -I$(srctree)/drivers/staging/android ++ ++obj-$(CONFIG_MALI_CLK_RATE_TRACE_PORTAL) += mali_kutf_clk_rate_trace_test_portal.o ++ ++mali_kutf_clk_rate_trace_test_portal-y := mali_kutf_clk_rate_trace_test.o +diff --git a/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/Kconfig b/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/Kconfig +new file mode 100755 +index 000000000000..8196e4cc6b37 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/Kconfig +@@ -0,0 +1,30 @@ ++# ++# (C) COPYRIGHT 2020 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# GNU General Public License for more details. ++# ++# You should have received a copy of the GNU General Public License ++# along with this program; if not, you can access it online at ++# http://www.gnu.org/licenses/gpl-2.0.html. ++# ++# SPDX-License-Identifier: GPL-2.0 ++# ++# ++ ++config CONFIG_MALI_CLK_RATE_TRACE_PORTAL ++ tristate "Mali GPU Clock Trace Test portal" ++ depends on MALI_BIFROST && MALI_BIFROST_DEBUG && MALI_KUTF ++ default m ++ help ++ This option will build a test module mali_kutf_clk_rate_trace_test_portal ++ that can test the clocks integration into the platform and exercise some ++ basic trace test in the system. Choosing M here will generate a single ++ module called mali_kutf_clk_rate_trace_test_portal. +diff --git a/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/Makefile b/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/Makefile +new file mode 100755 +index 000000000000..71c78b84830c +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/Makefile +@@ -0,0 +1,57 @@ ++# ++# (C) COPYRIGHT 2020 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# GNU General Public License for more details. ++# ++# You should have received a copy of the GNU General Public License ++# along with this program; if not, you can access it online at ++# http://www.gnu.org/licenses/gpl-2.0.html. ++# ++# SPDX-License-Identifier: GPL-2.0 ++# ++# ++ ++ifneq ($(KERNELRELEASE),) ++ ++ccflags-y := \ ++ -DMALI_UNIT_TEST=$(MALI_UNIT_TEST) \ ++ -DMALI_CUSTOMER_RELEASE=$(MALI_CUSTOMER_RELEASE) \ ++ -I$(src)/../../include \ ++ -I$(src)/../../../../../../../include \ ++ -I$(src)/../../../../ \ ++ -I$(src)/../../../ \ ++ -I$(src)/../../../backend/gpu \ ++ -I$(src)/../../../debug \ ++ -I$(src)/../../../debug/backend \ ++ -I$(src)/ \ ++ -I$(srctree)/drivers/staging/android \ ++ -I$(srctree)/include/linux ++ ++obj-m := mali_kutf_clk_rate_trace_test_portal.o ++mali_kutf_clk_rate_trace_test_portal-y := mali_kutf_clk_rate_trace_test.o ++ ++else ++# linux build system bootstrap for out-of-tree module ++ ++# default to building for the host ++ARCH ?= $(shell uname -m) ++ ++ifeq ($(KDIR),) ++$(error Must specify KDIR to point to the kernel to target)) ++endif ++ ++all: ++ $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) KBUILD_EXTRA_SYMBOLS="$(CURDIR)/../../kutf/Module.symvers $(CURDIR)/../../../Module.symvers" modules ++ ++clean: ++ $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) clean ++ ++endif +diff --git a/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/build.bp b/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/build.bp +new file mode 100755 +index 000000000000..0cc2904db542 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/build.bp +@@ -0,0 +1,34 @@ ++/* ++ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ */ ++ ++bob_kernel_module { ++ name: "mali_kutf_clk_rate_trace_test_portal", ++ defaults: [ ++ "mali_kbase_shared_config_defaults", ++ "kernel_test_includes", ++ ], ++ srcs: [ ++ "../mali_kutf_clk_rate_trace_test.h", ++ "Makefile", ++ "mali_kutf_clk_rate_trace_test.c", ++ ], ++ extra_symbols: [ ++ "mali_kbase", ++ "kutf", ++ ], ++ enabled: false, ++ base_build_kutf: { ++ enabled: true, ++ kbuild_options: ["CONFIG_MALI_CLK_RATE_TRACE_PORTAL=m"], ++ }, ++} +diff --git a/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/mali_kutf_clk_rate_trace_test.c b/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/mali_kutf_clk_rate_trace_test.c +new file mode 100755 +index 000000000000..d74a278bffa7 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/mali_kutf_clk_rate_trace_test.c +@@ -0,0 +1,890 @@ ++/* ++ * ++ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++#include ++ ++#include ++#include ++#include ++#include ++#if (KERNEL_VERSION(4, 11, 0) <= LINUX_VERSION_CODE) ++#include ++#else ++#include ++#endif ++#include "mali_kbase.h" ++#include "mali_kbase_irq_internal.h" ++#include "mali_kbase_pm_internal.h" ++#include "mali_kbase_clk_rate_trace_mgr.h" ++ ++#include ++#include ++#include ++#include ++ ++#include "../mali_kutf_clk_rate_trace_test.h" ++ ++#define MINOR_FOR_FIRST_KBASE_DEV (-1) ++ ++/* KUTF test application pointer for this test */ ++struct kutf_application *kutf_app; ++ ++enum portal_server_state { ++ PORTAL_STATE_NO_CLK, ++ PORTAL_STATE_LIVE, ++ PORTAL_STATE_CLOSING, ++}; ++ ++/** ++ * struct clk_trace_snapshot - Trace info data on a clock. ++ * @previous_rate: Snapshot start point clock rate. ++ * @current_rate: End point clock rate. It becomes the start rate of the ++ * next trace snapshot. ++ * @rate_up_cnt: Count in the snapshot duration when the clock trace ++ * write is a rate of higher value than the last. ++ * @rate_down_cnt: Count in the snapshot duration when the clock trace write ++ * is a rate of lower value than the last. ++ */ ++struct clk_trace_snapshot { ++ unsigned long previous_rate; ++ unsigned long current_rate; ++ u32 rate_up_cnt; ++ u32 rate_down_cnt; ++}; ++ ++/** ++ * struct kutf_clk_rate_trace_fixture_data - Fixture data for the test. ++ * @kbdev: kbase device for the GPU. ++ * @listener: Clock rate change listener structure. ++ * @invoke_notify: When true, invoke notify command is being executed. ++ * @snapshot: Clock trace update snapshot data array. A snapshot ++ * for each clock contains info accumulated beteen two ++ * GET_TRACE_SNAPSHOT requests. ++ * @nclks: Number of clocks visible to the trace portal. ++ * @pm_ctx_cnt: Net count of PM (Power Management) context INC/DEC ++ * PM_CTX_CNT requests made to the portal. On change from ++ * 0 to 1 (INC), or, 1 to 0 (DEC), a PM context action is ++ * triggered. ++ * @total_update_cnt: Total number of received trace write callbacks. ++ * @server_state: Portal server operational state. ++ * @result_msg: Message for the test result. ++ * @test_status: Portal test reslt status. ++ */ ++struct kutf_clk_rate_trace_fixture_data { ++ struct kbase_device *kbdev; ++ struct kbase_clk_rate_listener listener; ++ bool invoke_notify; ++ struct clk_trace_snapshot snapshot[BASE_MAX_NR_CLOCKS_REGULATORS]; ++ unsigned int nclks; ++ unsigned int pm_ctx_cnt; ++ unsigned int total_update_cnt; ++ enum portal_server_state server_state; ++ char const *result_msg; ++ enum kutf_result_status test_status; ++}; ++ ++struct clk_trace_portal_input { ++ struct kutf_helper_named_val cmd_input; ++ enum kbasep_clk_rate_trace_req portal_cmd; ++ int named_val_err; ++}; ++ ++struct kbasep_cmd_name_pair { ++ enum kbasep_clk_rate_trace_req cmd; ++ const char *name; ++}; ++ ++struct kbasep_cmd_name_pair kbasep_portal_cmd_name_map[] = { ++ {PORTAL_CMD_GET_CLK_RATE_MGR, GET_CLK_RATE_MGR}, ++ {PORTAL_CMD_GET_CLK_RATE_TRACE, GET_CLK_RATE_TRACE}, ++ {PORTAL_CMD_GET_TRACE_SNAPSHOT, GET_TRACE_SNAPSHOT}, ++ {PORTAL_CMD_INC_PM_CTX_CNT, INC_PM_CTX_CNT}, ++ {PORTAL_CMD_DEC_PM_CTX_CNT, DEC_PM_CTX_CNT}, ++ {PORTAL_CMD_CLOSE_PORTAL, CLOSE_PORTAL}, ++ {PORTAL_CMD_INVOKE_NOTIFY_42KHZ, INVOKE_NOTIFY_42KHZ}, ++ }; ++ ++/* Global pointer for the kutf_portal_trace_write() to use. When ++ * this pointer is engaged, new requests for create fixture will fail ++ * hence limiting the use of the portal at any time to a singleton. ++ */ ++struct kutf_clk_rate_trace_fixture_data *g_ptr_portal_data; ++ ++#define PORTAL_MSG_LEN (KUTF_MAX_LINE_LENGTH - MAX_REPLY_NAME_LEN) ++static char portal_msg_buf[PORTAL_MSG_LEN]; ++ ++static void kutf_portal_trace_write( ++ struct kbase_clk_rate_listener *listener, ++ u32 index, u32 new_rate) ++{ ++ struct clk_trace_snapshot *snapshot; ++ struct kutf_clk_rate_trace_fixture_data *data = container_of( ++ listener, struct kutf_clk_rate_trace_fixture_data, listener); ++ ++ lockdep_assert_held(&data->kbdev->pm.clk_rtm.lock); ++ ++ if (WARN_ON(g_ptr_portal_data == NULL)) ++ return; ++ if (WARN_ON(index >= g_ptr_portal_data->nclks)) ++ return; ++ ++ /* This callback is triggered by invoke notify command, skipping */ ++ if (data->invoke_notify) ++ return; ++ ++ snapshot = &g_ptr_portal_data->snapshot[index]; ++ if (new_rate > snapshot->current_rate) ++ snapshot->rate_up_cnt++; ++ else ++ snapshot->rate_down_cnt++; ++ snapshot->current_rate = new_rate; ++ g_ptr_portal_data->total_update_cnt++; ++} ++ ++static void kutf_set_pm_ctx_active(struct kutf_context *context) ++{ ++ struct kutf_clk_rate_trace_fixture_data *data = context->fixture; ++ ++ if (WARN_ON(data->pm_ctx_cnt != 1)) ++ return; ++ ++ kbase_pm_context_active(data->kbdev); ++ kbase_pm_wait_for_desired_state(data->kbdev); ++#if !MALI_USE_CSF ++ kbase_pm_request_gpu_cycle_counter(data->kbdev); ++#endif ++} ++ ++static void kutf_set_pm_ctx_idle(struct kutf_context *context) ++{ ++ struct kutf_clk_rate_trace_fixture_data *data = context->fixture; ++ ++ if (WARN_ON(data->pm_ctx_cnt > 0)) ++ return; ++ ++ kbase_pm_context_idle(data->kbdev); ++#if !MALI_USE_CSF ++ kbase_pm_release_gpu_cycle_counter(data->kbdev); ++#endif ++} ++ ++static char const *kutf_clk_trace_do_change_pm_ctx(struct kutf_context *context, ++ struct clk_trace_portal_input *cmd) ++{ ++ struct kutf_clk_rate_trace_fixture_data *data = context->fixture; ++ int seq = cmd->cmd_input.u.val_u64 & 0xFF; ++ const unsigned int cnt = data->pm_ctx_cnt; ++ const enum kbasep_clk_rate_trace_req req = cmd->portal_cmd; ++ char const *errmsg = NULL; ++ ++ WARN_ON(req != PORTAL_CMD_INC_PM_CTX_CNT && ++ req != PORTAL_CMD_DEC_PM_CTX_CNT); ++ ++ if (req == PORTAL_CMD_INC_PM_CTX_CNT && cnt < UINT_MAX) { ++ data->pm_ctx_cnt++; ++ if (data->pm_ctx_cnt == 1) ++ kutf_set_pm_ctx_active(context); ++ } ++ ++ if (req == PORTAL_CMD_DEC_PM_CTX_CNT && cnt > 0) { ++ data->pm_ctx_cnt--; ++ if (data->pm_ctx_cnt == 0) ++ kutf_set_pm_ctx_idle(context); ++ } ++ ++ /* Skip the length check, no chance of overflow for two ints */ ++ snprintf(portal_msg_buf, PORTAL_MSG_LEN, ++ "{SEQ:%d, PM_CTX_CNT:%u}", seq, data->pm_ctx_cnt); ++ ++ if (kutf_helper_send_named_str(context, "ACK", portal_msg_buf)) { ++ pr_warn("Error in sending ack for adjusting pm_ctx_cnt\n"); ++ errmsg = kutf_dsprintf(&context->fixture_pool, ++ "Error in sending ack for adjusting pm_ctx_cnt"); ++ } ++ ++ return errmsg; ++} ++ ++static char const *kutf_clk_trace_do_get_rate(struct kutf_context *context, ++ struct clk_trace_portal_input *cmd) ++{ ++ struct kutf_clk_rate_trace_fixture_data *data = context->fixture; ++ struct kbase_device *kbdev = data->kbdev; ++ int seq = cmd->cmd_input.u.val_u64 & 0xFF; ++ unsigned long rate; ++ bool idle; ++ int ret; ++ int i; ++ char const *errmsg = NULL; ++ ++ WARN_ON((cmd->portal_cmd != PORTAL_CMD_GET_CLK_RATE_MGR) && ++ (cmd->portal_cmd != PORTAL_CMD_GET_CLK_RATE_TRACE)); ++ ++ ret = snprintf(portal_msg_buf, PORTAL_MSG_LEN, ++ "{SEQ:%d, RATE:[", seq); ++ ++ for (i = 0; i < data->nclks; i++) { ++ spin_lock(&kbdev->pm.clk_rtm.lock); ++ if (cmd->portal_cmd == PORTAL_CMD_GET_CLK_RATE_MGR) ++ rate = kbdev->pm.clk_rtm.clks[i]->clock_val; ++ else ++ rate = data->snapshot[i].current_rate; ++ idle = kbdev->pm.clk_rtm.gpu_idle; ++ spin_unlock(&kbdev->pm.clk_rtm.lock); ++ ++ if ((i + 1) == data->nclks) ++ ret += snprintf(portal_msg_buf + ret, ++ PORTAL_MSG_LEN - ret, "0x%lx], GPU_IDLE:%d}", ++ rate, idle); ++ else ++ ret += snprintf(portal_msg_buf + ret, ++ PORTAL_MSG_LEN - ret, "0x%lx, ", rate); ++ ++ if (ret >= PORTAL_MSG_LEN) { ++ pr_warn("Message buf overflow with rate array data\n"); ++ return kutf_dsprintf(&context->fixture_pool, ++ "Message buf overflow with rate array data"); ++ } ++ } ++ ++ if (kutf_helper_send_named_str(context, "ACK", portal_msg_buf)) { ++ pr_warn("Error in sending back rate array\n"); ++ errmsg = kutf_dsprintf(&context->fixture_pool, ++ "Error in sending rate array"); ++ } ++ ++ return errmsg; ++} ++ ++/** ++ * kutf_clk_trace_do_get_snapshot() - Send back the current snapshot ++ * @context: KUTF context ++ * @cmd: The decoded portal input request ++ * ++ * The accumulated clock rate trace information is kept inside as an snapshot ++ * record. A user request of getting the snapshot marks the closure of the ++ * current snapshot record, and the start of the next one. The response ++ * message contains the current snapshot record, with each clock's ++ * data sequentially placed inside (array marker) [ ]. ++ */ ++static char const *kutf_clk_trace_do_get_snapshot(struct kutf_context *context, ++ struct clk_trace_portal_input *cmd) ++{ ++ struct kutf_clk_rate_trace_fixture_data *data = context->fixture; ++ struct clk_trace_snapshot snapshot; ++ int seq = cmd->cmd_input.u.val_u64 & 0xFF; ++ int ret; ++ int i; ++ char const *fmt; ++ char const *errmsg = NULL; ++ ++ WARN_ON(cmd->portal_cmd != PORTAL_CMD_GET_TRACE_SNAPSHOT); ++ ++ ret = snprintf(portal_msg_buf, PORTAL_MSG_LEN, ++ "{SEQ:%d, SNAPSHOT_ARRAY:[", seq); ++ ++ for (i = 0; i < data->nclks; i++) { ++ spin_lock(&data->kbdev->pm.clk_rtm.lock); ++ /* copy out the snapshot of the clock */ ++ snapshot = data->snapshot[i]; ++ /* Set the next snapshot start condition */ ++ data->snapshot[i].previous_rate = snapshot.current_rate; ++ data->snapshot[i].rate_up_cnt = 0; ++ data->snapshot[i].rate_down_cnt = 0; ++ spin_unlock(&data->kbdev->pm.clk_rtm.lock); ++ ++ /* Check i corresponding to the last clock */ ++ if ((i + 1) == data->nclks) ++ fmt = "(0x%lx, 0x%lx, %u, %u)]}"; ++ else ++ fmt = "(0x%lx, 0x%lx, %u, %u), "; ++ ret += snprintf(portal_msg_buf + ret, PORTAL_MSG_LEN - ret, ++ fmt, snapshot.previous_rate, snapshot.current_rate, ++ snapshot.rate_up_cnt, snapshot.rate_down_cnt); ++ if (ret >= PORTAL_MSG_LEN) { ++ pr_warn("Message buf overflow with snapshot data\n"); ++ return kutf_dsprintf(&context->fixture_pool, ++ "Message buf overflow with snapshot data"); ++ } ++ } ++ ++ if (kutf_helper_send_named_str(context, "ACK", portal_msg_buf)) { ++ pr_warn("Error in sending back snapshot array\n"); ++ errmsg = kutf_dsprintf(&context->fixture_pool, ++ "Error in sending snapshot array"); ++ } ++ ++ return errmsg; ++} ++ ++/** ++ * kutf_clk_trace_do_invoke_notify_42k() - Invokes the stored notification callback ++ * @context: KUTF context ++ * @cmd: The decoded portal input request ++ * ++ * Invokes frequency change notification callbacks with a fake ++ * GPU frequency 42 kHz for the top clock domain. ++ */ ++static char const *kutf_clk_trace_do_invoke_notify_42k( ++ struct kutf_context *context, ++ struct clk_trace_portal_input *cmd) ++{ ++ struct kutf_clk_rate_trace_fixture_data *data = context->fixture; ++ int seq = cmd->cmd_input.u.val_u64 & 0xFF; ++ const unsigned long new_rate_hz = 42000; ++ int ret; ++ char const *errmsg = NULL; ++ struct kbase_clk_rate_trace_manager *clk_rtm = &data->kbdev->pm.clk_rtm; ++ ++ WARN_ON(cmd->portal_cmd != PORTAL_CMD_INVOKE_NOTIFY_42KHZ); ++ ++ spin_lock(&clk_rtm->lock); ++ ++ data->invoke_notify = true; ++ kbase_clk_rate_trace_manager_notify_all( ++ clk_rtm, 0, new_rate_hz); ++ data->invoke_notify = false; ++ ++ spin_unlock(&clk_rtm->lock); ++ ++ ret = snprintf(portal_msg_buf, PORTAL_MSG_LEN, ++ "{SEQ:%d, HZ:%lu}", seq, new_rate_hz); ++ ++ if (ret >= PORTAL_MSG_LEN) { ++ pr_warn("Message buf overflow with invoked data\n"); ++ return kutf_dsprintf(&context->fixture_pool, ++ "Message buf overflow with invoked data"); ++ } ++ ++ if (kutf_helper_send_named_str(context, "ACK", portal_msg_buf)) { ++ pr_warn("Error in sending ack for " INVOKE_NOTIFY_42KHZ "request\n"); ++ errmsg = kutf_dsprintf(&context->fixture_pool, ++ "Error in sending ack for " INVOKE_NOTIFY_42KHZ "request"); ++ } ++ ++ return errmsg; ++} ++ ++static char const *kutf_clk_trace_do_close_portal(struct kutf_context *context, ++ struct clk_trace_portal_input *cmd) ++{ ++ struct kutf_clk_rate_trace_fixture_data *data = context->fixture; ++ int seq = cmd->cmd_input.u.val_u64 & 0xFF; ++ char const *errmsg = NULL; ++ ++ WARN_ON(cmd->portal_cmd != PORTAL_CMD_CLOSE_PORTAL); ++ ++ data->server_state = PORTAL_STATE_CLOSING; ++ ++ /* Skip the length check, no chance of overflow for two ints */ ++ snprintf(portal_msg_buf, PORTAL_MSG_LEN, ++ "{SEQ:%d, PM_CTX_CNT:%u}", seq, data->pm_ctx_cnt); ++ ++ if (kutf_helper_send_named_str(context, "ACK", portal_msg_buf)) { ++ pr_warn("Error in sending ack for " CLOSE_PORTAL "reuquest\n"); ++ errmsg = kutf_dsprintf(&context->fixture_pool, ++ "Error in sending ack for " CLOSE_PORTAL "reuquest"); ++ } ++ ++ return errmsg; ++} ++ ++static bool kutf_clk_trace_dequeue_portal_cmd(struct kutf_context *context, ++ struct clk_trace_portal_input *cmd) ++{ ++ int i; ++ int err = kutf_helper_receive_named_val(context, &cmd->cmd_input); ++ ++ cmd->named_val_err = err; ++ if (err == KUTF_HELPER_ERR_NONE && ++ cmd->cmd_input.type == KUTF_HELPER_VALTYPE_U64) { ++ /* All portal request commands are of format (named u64): ++ * CMD_NAME=1234 ++ * where, 1234 is a (variable) sequence number tag. ++ */ ++ for (i = 0; i < PORTAL_TOTAL_CMDS; i++) { ++ if (strcmp(cmd->cmd_input.val_name, ++ kbasep_portal_cmd_name_map[i].name)) ++ continue; ++ ++ cmd->portal_cmd = kbasep_portal_cmd_name_map[i].cmd; ++ return true; ++ } ++ } ++ ++ cmd->portal_cmd = PORTAL_CMD_INVALID; ++ return false; ++} ++ ++static void kutf_clk_trace_flag_result(struct kutf_context *context, ++ enum kutf_result_status result, char const *msg) ++{ ++ struct kutf_clk_rate_trace_fixture_data *data = context->fixture; ++ ++ if (result > data->test_status) { ++ data->test_status = result; ++ if (msg) ++ data->result_msg = msg; ++ if (data->server_state == PORTAL_STATE_LIVE && ++ result > KUTF_RESULT_WARN) { ++ data->server_state = PORTAL_STATE_CLOSING; ++ } ++ } ++} ++ ++static bool kutf_clk_trace_process_portal_cmd(struct kutf_context *context, ++ struct clk_trace_portal_input *cmd) ++{ ++ char const *errmsg = NULL; ++ ++ BUILD_BUG_ON(ARRAY_SIZE(kbasep_portal_cmd_name_map) != ++ PORTAL_TOTAL_CMDS); ++ WARN_ON(cmd->portal_cmd == PORTAL_CMD_INVALID); ++ ++ switch (cmd->portal_cmd) { ++ case PORTAL_CMD_GET_CLK_RATE_MGR: ++ /* Fall through */ ++ case PORTAL_CMD_GET_CLK_RATE_TRACE: ++ errmsg = kutf_clk_trace_do_get_rate(context, cmd); ++ break; ++ case PORTAL_CMD_GET_TRACE_SNAPSHOT: ++ errmsg = kutf_clk_trace_do_get_snapshot(context, cmd); ++ break; ++ case PORTAL_CMD_INC_PM_CTX_CNT: ++ /* Fall through */ ++ case PORTAL_CMD_DEC_PM_CTX_CNT: ++ errmsg = kutf_clk_trace_do_change_pm_ctx(context, cmd); ++ break; ++ case PORTAL_CMD_CLOSE_PORTAL: ++ errmsg = kutf_clk_trace_do_close_portal(context, cmd); ++ break; ++ case PORTAL_CMD_INVOKE_NOTIFY_42KHZ: ++ errmsg = kutf_clk_trace_do_invoke_notify_42k(context, cmd); ++ break; ++ default: ++ pr_warn("Don't know how to handle portal_cmd: %d, abort session.\n", ++ cmd->portal_cmd); ++ errmsg = kutf_dsprintf(&context->fixture_pool, ++ "Don't know how to handle portal_cmd: %d", ++ cmd->portal_cmd); ++ break; ++ } ++ ++ if (errmsg) ++ kutf_clk_trace_flag_result(context, KUTF_RESULT_FAIL, errmsg); ++ ++ return (errmsg == NULL); ++} ++ ++/** ++ * kutf_clk_trace_do_nack_response() - respond a NACK to erroneous input ++ * @context: KUTF context ++ * @cmd: The erroneous input request ++ * ++ * This function deal with an erroneous input request, and respond with ++ * a proper 'NACK' message. ++ */ ++static int kutf_clk_trace_do_nack_response(struct kutf_context *context, ++ struct clk_trace_portal_input *cmd) ++{ ++ int seq; ++ int err; ++ char const *errmsg = NULL; ++ ++ WARN_ON(cmd->portal_cmd != PORTAL_CMD_INVALID); ++ ++ if (cmd->named_val_err == KUTF_HELPER_ERR_NONE && ++ cmd->cmd_input.type == KUTF_HELPER_VALTYPE_U64) { ++ /* Keep seq number as % 256 */ ++ seq = cmd->cmd_input.u.val_u64 & 255; ++ snprintf(portal_msg_buf, PORTAL_MSG_LEN, ++ "{SEQ:%d, MSG: Unknown command '%s'.}", seq, ++ cmd->cmd_input.val_name); ++ err = kutf_helper_send_named_str(context, "NACK", ++ portal_msg_buf); ++ } else ++ err = kutf_helper_send_named_str(context, "NACK", ++ "Wrong portal cmd format (Ref example: CMD_NAME=0X16)"); ++ ++ if (err) { ++ errmsg = kutf_dsprintf(&context->fixture_pool, ++ "Failed to send portal NACK response"); ++ kutf_clk_trace_flag_result(context, KUTF_RESULT_FAIL, errmsg); ++ } ++ ++ return err; ++} ++ ++/** ++ * kutf_clk_trace_barebone_check() - Sanity test on the clock tracing ++ * @context: KUTF context ++ * ++ * This function carries out some basic test on the tracing operation: ++ * 1). GPU idle on test start, trace rate should be 0 (low power state) ++ * 2). Make sure GPU is powered up, the trace rate should match ++ * that from the clcok manager's internal recorded rate ++ * 3). If the GPU active transition occurs following 2), there ++ * must be rate change event from tracing. ++ */ ++void kutf_clk_trace_barebone_check(struct kutf_context *context) ++{ ++ struct kutf_clk_rate_trace_fixture_data *data = context->fixture; ++ struct kbase_device *kbdev = data->kbdev; ++ bool fail = false; ++ bool idle[2] = { false }; ++ char const *msg = NULL; ++ int i; ++ ++ /* Check consistency if gpu happens to be idle */ ++ spin_lock(&kbdev->pm.clk_rtm.lock); ++ idle[0] = kbdev->pm.clk_rtm.gpu_idle; ++ if (kbdev->pm.clk_rtm.gpu_idle) { ++ for (i = 0; i < data->nclks; i++) { ++ if (data->snapshot[i].current_rate) { ++ /* Idle should have a rate 0 */ ++ fail = true; ++ break; ++ } ++ } ++ } ++ spin_unlock(&kbdev->pm.clk_rtm.lock); ++ if (fail) { ++ msg = kutf_dsprintf(&context->fixture_pool, ++ "GPU Idle not yielding 0-rate"); ++ pr_err("Trace did not see idle rate\n"); ++ } else { ++ /* Make local PM active if not done so yet */ ++ if (data->pm_ctx_cnt == 0) { ++ /* Ensure the GPU is powered */ ++ data->pm_ctx_cnt++; ++ kutf_set_pm_ctx_active(context); ++ } ++ /* Checking the rate is consistent */ ++ spin_lock(&kbdev->pm.clk_rtm.lock); ++ idle[1] = kbdev->pm.clk_rtm.gpu_idle; ++ for (i = 0; i < data->nclks; i++) { ++ /* Rate match between the manager and the trace */ ++ if (kbdev->pm.clk_rtm.clks[i]->clock_val != ++ data->snapshot[i].current_rate) { ++ fail = true; ++ break; ++ } ++ } ++ spin_unlock(&kbdev->pm.clk_rtm.lock); ++ ++ if (idle[1]) { ++ msg = kutf_dsprintf(&context->fixture_pool, ++ "GPU still idle after set_pm_ctx_active"); ++ pr_err("GPU still idle after set_pm_ctx_active\n"); ++ } ++ ++ if (!msg && fail) { ++ msg = kutf_dsprintf(&context->fixture_pool, ++ "Trace rate not matching Clk manager's read"); ++ pr_err("Trace rate not matching Clk manager's read\n"); ++ } ++ } ++ ++ if (!msg && idle[0] && !idle[1] && !data->total_update_cnt) { ++ msg = kutf_dsprintf(&context->fixture_pool, ++ "Trace update did not occur"); ++ pr_err("Trace update did not occur\n"); ++ } ++ if (msg) ++ kutf_clk_trace_flag_result(context, KUTF_RESULT_FAIL, msg); ++ else if (!data->total_update_cnt) { ++ msg = kutf_dsprintf(&context->fixture_pool, ++ "No trace update seen during the test!"); ++ kutf_clk_trace_flag_result(context, KUTF_RESULT_WARN, msg); ++ } ++} ++ ++static bool kutf_clk_trace_end_of_stream(struct clk_trace_portal_input *cmd) ++{ ++ return (cmd->named_val_err == -EBUSY); ++} ++ ++void kutf_clk_trace_no_clks_dummy(struct kutf_context *context) ++{ ++ struct clk_trace_portal_input cmd; ++ unsigned long timeout = jiffies + HZ * 2; ++ bool has_cmd; ++ ++ while (time_before(jiffies, timeout)) { ++ if (kutf_helper_pending_input(context)) { ++ has_cmd = kutf_clk_trace_dequeue_portal_cmd(context, ++ &cmd); ++ if (!has_cmd && kutf_clk_trace_end_of_stream(&cmd)) ++ break; ++ ++ kutf_helper_send_named_str(context, "NACK", ++ "Fatal! No clocks visible, aborting"); ++ } ++ msleep(20); ++ } ++ ++ kutf_clk_trace_flag_result(context, KUTF_RESULT_FATAL, ++ "No clocks visble to the portal"); ++} ++ ++/** ++ * mali_kutf_clk_rate_trace_test_portal() - Service portal input ++ * @context: KUTF context ++ * ++ * The test portal operates on input requests. If the input request is one ++ * of the recognized portal commands, it handles it accordingly. Otherwise ++ * a negative response 'NACK' is returned. The portal service terminates ++ * when a 'CLOSE_PORTAL' request is received, or due to an internal error. ++ * Both case would result in the server_state transitioned to CLOSING. ++ * ++ * If the portal is closed on request, a sanity test on the clock rate ++ * trace operation is undertaken via function: ++ * kutf_clk_trace_barebone_check(); ++ */ ++static void mali_kutf_clk_rate_trace_test_portal(struct kutf_context *context) ++{ ++ struct kutf_clk_rate_trace_fixture_data *data = context->fixture; ++ struct clk_trace_portal_input new_cmd; ++ ++ pr_debug("Test portal service start\n"); ++ ++ while (data->server_state == PORTAL_STATE_LIVE) { ++ if (kutf_clk_trace_dequeue_portal_cmd(context, &new_cmd)) ++ kutf_clk_trace_process_portal_cmd(context, &new_cmd); ++ else if (kutf_clk_trace_end_of_stream(&new_cmd)) ++ /* Dequeue on portal input, end of stream */ ++ data->server_state = PORTAL_STATE_CLOSING; ++ else ++ kutf_clk_trace_do_nack_response(context, &new_cmd); ++ } ++ ++ /* Closing, exhausting all the pending inputs with NACKs. */ ++ if (data->server_state == PORTAL_STATE_CLOSING) { ++ while (kutf_helper_pending_input(context) && ++ (kutf_clk_trace_dequeue_portal_cmd(context, &new_cmd) || ++ !kutf_clk_trace_end_of_stream(&new_cmd))) { ++ kutf_helper_send_named_str(context, "NACK", ++ "Portal closing down"); ++ } ++ } ++ ++ /* If no portal error, do a barebone test here irrespective ++ * whatever the portal live session has been testing, which ++ * is entirely driven by the user-side via portal requests. ++ */ ++ if (data->test_status <= KUTF_RESULT_WARN) { ++ if (data->server_state != PORTAL_STATE_NO_CLK) ++ kutf_clk_trace_barebone_check(context); ++ else { ++ /* No clocks case, NACK 2-sec for the fatal situation */ ++ kutf_clk_trace_no_clks_dummy(context); ++ } ++ } ++ ++ /* If we have changed pm_ctx count, drop it back */ ++ if (data->pm_ctx_cnt) { ++ /* Although we count on portal requests, it only has material ++ * impact when from 0 -> 1. So the reverse is a simple one off. ++ */ ++ data->pm_ctx_cnt = 0; ++ kutf_set_pm_ctx_idle(context); ++ } ++ ++ /* Finally log the test result line */ ++ if (data->test_status < KUTF_RESULT_WARN) ++ kutf_test_pass(context, data->result_msg); ++ else if (data->test_status == KUTF_RESULT_WARN) ++ kutf_test_warn(context, data->result_msg); ++ else if (data->test_status == KUTF_RESULT_FATAL) ++ kutf_test_fatal(context, data->result_msg); ++ else ++ kutf_test_fail(context, data->result_msg); ++ ++ pr_debug("Test end\n"); ++} ++ ++/** ++ * mali_kutf_clk_rate_trace_create_fixture() - Creates the fixture data ++ * required for mali_kutf_clk_rate_trace_test_portal. ++ * @context: KUTF context. ++ * ++ * Return: Fixture data created on success or NULL on failure ++ */ ++static void *mali_kutf_clk_rate_trace_create_fixture( ++ struct kutf_context *context) ++{ ++ struct kutf_clk_rate_trace_fixture_data *data; ++ struct kbase_device *kbdev; ++ unsigned long rate; ++ int i; ++ ++ /* Acquire the kbase device */ ++ pr_debug("Finding device\n"); ++ kbdev = kbase_find_device(MINOR_FOR_FIRST_KBASE_DEV); ++ if (kbdev == NULL) { ++ kutf_test_fail(context, "Failed to find kbase device"); ++ return NULL; ++ } ++ ++ pr_debug("Creating fixture\n"); ++ data = kutf_mempool_alloc(&context->fixture_pool, ++ sizeof(struct kutf_clk_rate_trace_fixture_data)); ++ if (!data) ++ return NULL; ++ ++ *data = (const struct kutf_clk_rate_trace_fixture_data) { 0 }; ++ pr_debug("Hooking up the test portal to kbdev clk rate trace\n"); ++ spin_lock(&kbdev->pm.clk_rtm.lock); ++ ++ if (g_ptr_portal_data != NULL) { ++ pr_warn("Test portal is already in use, run aborted\n"); ++ kutf_test_fail(context, "Portal allows single session only"); ++ spin_unlock(&kbdev->pm.clk_rtm.lock); ++ return NULL; ++ } ++ ++ for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) { ++ if (kbdev->pm.clk_rtm.clks[i]) { ++ data->nclks++; ++ if (kbdev->pm.clk_rtm.gpu_idle) ++ rate = 0; ++ else ++ rate = kbdev->pm.clk_rtm.clks[i]->clock_val; ++ data->snapshot[i].previous_rate = rate; ++ data->snapshot[i].current_rate = rate; ++ } ++ } ++ ++ spin_unlock(&kbdev->pm.clk_rtm.lock); ++ ++ if (data->nclks) { ++ /* Subscribe this test server portal */ ++ data->listener.notify = kutf_portal_trace_write; ++ data->invoke_notify = false; ++ ++ kbase_clk_rate_trace_manager_subscribe( ++ &kbdev->pm.clk_rtm, &data->listener); ++ /* Update the kutf_server_portal fixture_data pointer */ ++ g_ptr_portal_data = data; ++ } ++ ++ data->kbdev = kbdev; ++ data->result_msg = NULL; ++ data->test_status = KUTF_RESULT_PASS; ++ ++ if (data->nclks == 0) { ++ data->server_state = PORTAL_STATE_NO_CLK; ++ pr_debug("Kbdev has no clocks for rate trace"); ++ } else ++ data->server_state = PORTAL_STATE_LIVE; ++ ++ pr_debug("Created fixture\n"); ++ ++ return data; ++} ++ ++/** ++ * Destroy fixture data previously created by ++ * mali_kutf_clk_rate_trace_create_fixture. ++ * ++ * @context: KUTF context. ++ */ ++static void mali_kutf_clk_rate_trace_remove_fixture( ++ struct kutf_context *context) ++{ ++ struct kutf_clk_rate_trace_fixture_data *data = context->fixture; ++ struct kbase_device *kbdev = data->kbdev; ++ ++ if (data->nclks) { ++ /* Clean up the portal trace write arrangement */ ++ g_ptr_portal_data = NULL; ++ ++ kbase_clk_rate_trace_manager_unsubscribe( ++ &kbdev->pm.clk_rtm, &data->listener); ++ } ++ pr_debug("Destroying fixture\n"); ++ kbase_release_device(kbdev); ++ pr_debug("Destroyed fixture\n"); ++} ++ ++/** ++ * mali_kutf_clk_rate_trace_test_module_init() - Entry point for test mdoule. ++ */ ++int mali_kutf_clk_rate_trace_test_module_init(void) ++{ ++ struct kutf_suite *suite; ++ unsigned int filters; ++ union kutf_callback_data suite_data = { 0 }; ++ ++ pr_debug("Creating app\n"); ++ ++ g_ptr_portal_data = NULL; ++ kutf_app = kutf_create_application(CLK_RATE_TRACE_APP_NAME); ++ ++ if (!kutf_app) { ++ pr_warn("Creation of app " CLK_RATE_TRACE_APP_NAME ++ " failed!\n"); ++ return -ENOMEM; ++ } ++ ++ pr_debug("Create suite %s\n", CLK_RATE_TRACE_SUITE_NAME); ++ suite = kutf_create_suite_with_filters_and_data( ++ kutf_app, CLK_RATE_TRACE_SUITE_NAME, 1, ++ mali_kutf_clk_rate_trace_create_fixture, ++ mali_kutf_clk_rate_trace_remove_fixture, ++ KUTF_F_TEST_GENERIC, ++ suite_data); ++ ++ if (!suite) { ++ pr_warn("Creation of suite %s failed!\n", ++ CLK_RATE_TRACE_SUITE_NAME); ++ kutf_destroy_application(kutf_app); ++ return -ENOMEM; ++ } ++ ++ filters = suite->suite_default_flags; ++ kutf_add_test_with_filters( ++ suite, 0x0, CLK_RATE_TRACE_PORTAL, ++ mali_kutf_clk_rate_trace_test_portal, ++ filters); ++ ++ pr_debug("Init complete\n"); ++ return 0; ++} ++ ++/** ++ * mali_kutf_clk_rate_trace_test_module_exit() - Module exit point for this ++ * test. ++ */ ++void mali_kutf_clk_rate_trace_test_module_exit(void) ++{ ++ pr_debug("Exit start\n"); ++ kutf_destroy_application(kutf_app); ++ pr_debug("Exit complete\n"); ++} ++ ++ ++module_init(mali_kutf_clk_rate_trace_test_module_init); ++module_exit(mali_kutf_clk_rate_trace_test_module_exit); ++ ++MODULE_LICENSE("GPL"); +diff --git a/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/mali_kutf_clk_rate_trace_test.h b/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/mali_kutf_clk_rate_trace_test.h +new file mode 100755 +index 000000000000..f46afd5086bd +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/mali_kutf_clk_rate_trace_test.h +@@ -0,0 +1,148 @@ ++/* ++ * ++ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#ifndef _KUTF_CLK_RATE_TRACE_TEST_H_ ++#define _KUTF_CLK_RATE_TRACE_TEST_H_ ++ ++#define CLK_RATE_TRACE_APP_NAME "clk_rate_trace" ++#define CLK_RATE_TRACE_SUITE_NAME "rate_trace" ++#define CLK_RATE_TRACE_PORTAL "portal" ++ ++/** ++ * enum kbasep_clk_rate_trace_req - request command to the clock rate trace ++ * service portal. ++ * ++ * @PORTAL_CMD_GET_CLK_RATE_MGR: Request the clock trace manager internal ++ * data record. On a positive acknowledgement ++ * the prevailing clock rates and the GPU idle ++ * condition flag are returned. ++ * @PORTAL_CMD_GET_CLK_RATE_TRACE: Request the clock trace portal to return its ++ * data record. On a positive acknowledgement ++ * the last trace recorded clock rates and the ++ * GPU idle condition flag are returned. ++ * @PORTAL_CMD_GET_TRACE_SNAPSHOT: Request the clock trace portal to return its ++ * current snapshot data record. On a positive ++ * acknowledgement the snapshot array matching ++ * the number of clocks are returned. It also ++ * starts a fresh snapshot inside the clock ++ * trace portal. ++ * @PORTAL_CMD_INC_PM_CTX_CNT: Request the clock trace portal to increase ++ * its internal PM_CTX_COUNT. If this increase ++ * yielded a count of 0 -> 1 change, the portal ++ * will initiate a PM_CTX_ACTIVE call to the ++ * Kbase power management. Futher increase ++ * requests will limit to only affect the ++ * portal internal count value. ++ * @PORTAL_CMD_DEC_PM_CTX_CNT: Request the clock trace portal to decrease ++ * its internal PM_CTX_COUNT. If this decrease ++ * yielded a count of 1 -> 0 change, the portal ++ * will initiate a PM_CTX_IDLE call to the ++ * Kbase power management. ++ * @PORTAL_CMD_CLOSE_PORTAL: Inform the clock trace portal service the ++ * client has completed its session. The portal ++ * will start the close down action. If no ++ * error has occurred during the dynamic ++ * interactive session, an inherent basic test ++ * carrying out some sanity check on the clock ++ * trace is undertaken. ++ * @PORTAL_CMD_INVOKE_NOTIFY_42KHZ: Invokes all clock rate trace manager callbacks ++ * for the top clock domain with a new GPU frequency ++ * set to 42 kHZ. ++ * @PORTAL_CMD_INVALID: Valid commands termination marker. Must be ++ * the highest enumeration value, as it ++ * represents valid command array size. ++ * @PORTAL_TOTAL_CMDS: Alias of PORTAL_CMD_INVALID. ++ */ ++/* PORTAL_CMD_INVALID must be the last one, serving the size */ ++enum kbasep_clk_rate_trace_req { ++ PORTAL_CMD_GET_CLK_RATE_MGR, ++ PORTAL_CMD_GET_CLK_RATE_TRACE, ++ PORTAL_CMD_GET_TRACE_SNAPSHOT, ++ PORTAL_CMD_INC_PM_CTX_CNT, ++ PORTAL_CMD_DEC_PM_CTX_CNT, ++ PORTAL_CMD_CLOSE_PORTAL, ++ PORTAL_CMD_INVOKE_NOTIFY_42KHZ, ++ PORTAL_CMD_INVALID, ++ PORTAL_TOTAL_CMDS = PORTAL_CMD_INVALID, ++}; ++ ++/** ++ * Portal service request command names. The portal request consists of a kutf ++ * named u64-value. For those above enumerated PORTAL_CMD, the names defined ++ * here are used to mark the name and then followed with a sequence number ++ * value. Example (manual script here for illustration): ++ * exec 5<>run # open the portal kutf run as fd-5 ++ * echo GET_CLK_RATE_MGR=1 >&5 # send the cmd and sequence number 1 ++ * head -n 1 <&5 # read back the 1-line server reseponse ++ * ACK="{SEQ:1, RATE:[0x1ad27480], GPU_IDLE:1}" # response string ++ * echo GET_TRACE_SNAPSHOT=1 >&5 # send the cmd and sequence number 1 ++ * head -n 1 <&5 # read back the 1-line server reseponse ++ * ACK="{SEQ:1, SNAPSHOT_ARRAY:[(0x0, 0x1ad27480, 1, 0)]}" ++ * echo CLOSE_PORTAL=1 >&5 # close the portal ++ * cat <&5 # read back all the response lines ++ * ACK="{SEQ:1, PM_CTX_CNT:0}" # response to close command ++ * KUTF_RESULT_PASS:(explicit pass) # internal sanity test passed. ++ * exec 5>&- # close the service portal fd. ++ * ++ * Expected request command return format: ++ * GET_CLK_RATE_MGR: ACK="{SEQ:12, RATE:[1080, 1280], GPU_IDLE:1}" ++ * Note, the above contains 2-clock with rates in [], GPU idle ++ * GET_CLK_RATE_TRACE: ACK="{SEQ:6, RATE:[0x1ad27480], GPU_IDLE:0}" ++ * Note, 1-clock with rate in [], GPU not idle ++ * GET_TRACE_SNAPSHOT: ACK="{SEQ:8, SNAPSHOT_ARRAY:[(0x0, 0x1ad27480, 1, 0)]}" ++ * Note, 1-clock, (start_rate : 0, last_rate : 0x1ad27480, ++ * trace_rate_up_count: 1, trace_rate_down_count : 0) ++ * For the specific sample case here, there is a single rate_trace event ++ * that yielded a rate increase change. No rate drop event recorded in the ++ * reporting snapshot duration. ++ * INC_PM_CTX_CNT: ACK="{SEQ:1, PM_CTX_CNT:1}" ++ * Note, after the increment, M_CTX_CNT is 1. (i.e. 0 -> 1) ++ * DEC_PM_CTX_CNT: ACK="{SEQ:3, PM_CTX_CNT:0}" ++ * Note, after the decrement, PM_CTX_CNT is 0. (i.e. 1 -> 0) ++ * CLOSE_PORTAL: ACK="{SEQ:1, PM_CTX_CNT:1}" ++ * Note, at the close, PM_CTX_CNT is 1. The PM_CTX_CNT will internally be ++ * dropped down to 0 as part of the portal close clean up. ++ */ ++#define GET_CLK_RATE_MGR "GET_CLK_RATE_MGR" ++#define GET_CLK_RATE_TRACE "GET_CLK_RATE_TRACE" ++#define GET_TRACE_SNAPSHOT "GET_TRACE_SNAPSHOT" ++#define INC_PM_CTX_CNT "INC_PM_CTX_CNT" ++#define DEC_PM_CTX_CNT "DEC_PM_CTX_CNT" ++#define CLOSE_PORTAL "CLOSE_PORTAL" ++#define INVOKE_NOTIFY_42KHZ "INVOKE_NOTIFY_42KHZ" ++ ++/** ++ * Portal service response tag names. The response consists of a kutf ++ * named string-value. In case of a 'NACK' (negative acknowledgement), it ++ * can be one of the two formats: ++ * 1. NACK="{SEQ:2, MSG:xyzed}" # NACK on command with sequence tag-2. ++ * Note, the portal has received a valid name and valid sequence number ++ * but can't carry-out the request, reason in the MSG field. ++ * 2. NACK="Failing-message" ++ * Note, unable to parse a valid name or valid sequence number, ++ * or some internal error condition. Reason in the quoted string. ++ */ ++#define ACK "ACK" ++#define NACK "NACK" ++#define MAX_REPLY_NAME_LEN 32 ++ ++#endif /* _KUTF_CLK_RATE_TRACE_TEST_H_ */ +diff --git a/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/Kbuild b/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/Kbuild +new file mode 100755 +index 000000000000..ca8c51273b4c +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/Kbuild +@@ -0,0 +1,26 @@ ++# ++# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# GNU General Public License for more details. ++# ++# You should have received a copy of the GNU General Public License ++# along with this program; if not, you can access it online at ++# http://www.gnu.org/licenses/gpl-2.0.html. ++# ++# SPDX-License-Identifier: GPL-2.0 ++# ++# ++ ++ccflags-y += -I$(src)/../include -I$(src)/../../../ -I$(src)/../../ -I$(src)/../../backend/gpu -I$(srctree)/drivers/staging/android ++ ++obj-$(CONFIG_MALI_IRQ_LATENCY) += mali_kutf_irq_test.o ++ ++mali_kutf_irq_test-y := mali_kutf_irq_test_main.o +diff --git a/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/Kconfig b/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/Kconfig +new file mode 100755 +index 000000000000..78283307713d +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/Kconfig +@@ -0,0 +1,29 @@ ++# ++# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# GNU General Public License for more details. ++# ++# You should have received a copy of the GNU General Public License ++# along with this program; if not, you can access it online at ++# http://www.gnu.org/licenses/gpl-2.0.html. ++# ++# SPDX-License-Identifier: GPL-2.0 ++# ++# ++ ++config MALI_IRQ_LATENCY ++ tristate "Mali GPU IRQ latency measurement" ++ depends on MALI_BIFROST && MALI_BIFROST_DEBUG && MALI_KUTF ++ default m ++ help ++ This option will build a test module mali_kutf_irq_test that ++ can determine the latency of the Mali GPU IRQ on your system. ++ Choosing M here will generate a single module called mali_kutf_irq_test. +diff --git a/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/Makefile b/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/Makefile +new file mode 100755 +index 000000000000..bc4d654a90ca +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/Makefile +@@ -0,0 +1,51 @@ ++# ++# (C) COPYRIGHT 2015, 2017-2018, 2020 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# GNU General Public License for more details. ++# ++# You should have received a copy of the GNU General Public License ++# along with this program; if not, you can access it online at ++# http://www.gnu.org/licenses/gpl-2.0.html. ++# ++# SPDX-License-Identifier: GPL-2.0 ++# ++# ++ ++# linux build system bootstrap for out-of-tree module ++ ++# default to building for the host ++ARCH ?= $(shell uname -m) ++ ++ifeq ($(KDIR),) ++$(error Must specify KDIR to point to the kernel to target)) ++endif ++ ++TEST_CCFLAGS := \ ++ -DMALI_UNIT_TEST=$(MALI_UNIT_TEST) \ ++ -DMALI_CUSTOMER_RELEASE=$(MALI_CUSTOMER_RELEASE) \ ++ -DMALI_USE_CSF=$(MALI_USE_CSF) \ ++ $(SCONS_CFLAGS) \ ++ -I$(CURDIR)/../include \ ++ -I$(CURDIR)/../../../../../../include \ ++ -I$(CURDIR)/../../../ \ ++ -I$(CURDIR)/../../ \ ++ -I$(CURDIR)/../../backend/gpu \ ++ -I$(CURDIR)/../../debug \ ++ -I$(CURDIR)/../../debug/backend \ ++ -I$(CURDIR)/ \ ++ -I$(srctree)/drivers/staging/android \ ++ -I$(srctree)/include/linux ++ ++all: ++ $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) $(SCONS_CONFIGS) EXTRA_CFLAGS="$(TEST_CCFLAGS)" KBUILD_EXTRA_SYMBOLS="$(CURDIR)/../kutf/Module.symvers $(CURDIR)/../../Module.symvers" modules ++ ++clean: ++ $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) clean +diff --git a/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/build.bp b/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/build.bp +new file mode 100755 +index 000000000000..90efdcf9ad9c +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/build.bp +@@ -0,0 +1,35 @@ ++/* ++ * ++ * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++bob_kernel_module { ++ name: "mali_kutf_irq_test", ++ defaults: [ ++ "mali_kbase_shared_config_defaults", ++ "kernel_test_includes", ++ ], ++ srcs: [ ++ "Kbuild", ++ "mali_kutf_irq_test_main.c", ++ ], ++ extra_symbols: [ ++ "mali_kbase", ++ "kutf", ++ ], ++ enabled: false, ++ base_build_kutf: { ++ enabled: true, ++ kbuild_options: ["CONFIG_MALI_IRQ_LATENCY=m"], ++ }, ++} +diff --git a/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c b/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c +new file mode 100755 +index 000000000000..5f27c3a7e9b2 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c +@@ -0,0 +1,278 @@ ++/* ++ * ++ * (C) COPYRIGHT 2016-2018, 2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include ++#include ++#include ++ ++#include "mali_kbase.h" ++#include ++#include ++ ++#include ++#include ++ ++/* ++ * This file contains the code which is used for measuring interrupt latency ++ * of the Mali GPU IRQ. In particular, function mali_kutf_irq_latency() is ++ * used with this purpose and it is called within KUTF framework - a kernel ++ * unit test framework. The measured latency provided by this test should ++ * be representative for the latency of the Mali JOB/MMU IRQs as well. ++ */ ++ ++/* KUTF test application pointer for this test */ ++struct kutf_application *irq_app; ++ ++/** ++ * struct kutf_irq_fixture data - test fixture used by the test functions. ++ * @kbdev: kbase device for the GPU. ++ * ++ */ ++struct kutf_irq_fixture_data { ++ struct kbase_device *kbdev; ++}; ++ ++#define SEC_TO_NANO(s) ((s)*1000000000LL) ++ ++/* ID for the GPU IRQ */ ++#define GPU_IRQ_HANDLER 2 ++ ++#define NR_TEST_IRQS ((u32)1000000) ++ ++/* IRQ for the test to trigger. Currently POWER_CHANGED_SINGLE as it is ++ * otherwise unused in the DDK ++ */ ++#define TEST_IRQ POWER_CHANGED_SINGLE ++ ++#define IRQ_TIMEOUT HZ ++ ++/* Kernel API for setting irq throttle hook callback and irq time in us*/ ++extern int kbase_set_custom_irq_handler(struct kbase_device *kbdev, ++ irq_handler_t custom_handler, ++ int irq_type); ++extern irqreturn_t kbase_gpu_irq_test_handler(int irq, void *data, u32 val); ++ ++static DECLARE_WAIT_QUEUE_HEAD(wait); ++static bool triggered; ++static u64 irq_time; ++ ++static void *kbase_untag(void *ptr) ++{ ++ return (void *)(((uintptr_t) ptr) & ~3); ++} ++ ++/** ++ * kbase_gpu_irq_custom_handler - Custom IRQ throttle handler ++ * @irq: IRQ number ++ * @data: Data associated with this IRQ ++ * ++ * Return: state of the IRQ ++ */ ++static irqreturn_t kbase_gpu_irq_custom_handler(int irq, void *data) ++{ ++ struct kbase_device *kbdev = kbase_untag(data); ++ u32 val = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_STATUS)); ++ irqreturn_t result; ++ u64 tval; ++ bool has_test_irq = val & TEST_IRQ; ++ ++ if (has_test_irq) { ++ tval = ktime_get_real_ns(); ++ /* Clear the test source only here */ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), ++ TEST_IRQ); ++ /* Remove the test IRQ status bit */ ++ val = val ^ TEST_IRQ; ++ } ++ ++ result = kbase_gpu_irq_test_handler(irq, data, val); ++ ++ if (has_test_irq) { ++ irq_time = tval; ++ triggered = true; ++ wake_up(&wait); ++ result = IRQ_HANDLED; ++ } ++ ++ return result; ++} ++ ++/** ++ * mali_kutf_irq_default_create_fixture() - Creates the fixture data required ++ * for all the tests in the irq suite. ++ * @context: KUTF context. ++ * ++ * Return: Fixture data created on success or NULL on failure ++ */ ++static void *mali_kutf_irq_default_create_fixture( ++ struct kutf_context *context) ++{ ++ struct kutf_irq_fixture_data *data; ++ ++ data = kutf_mempool_alloc(&context->fixture_pool, ++ sizeof(struct kutf_irq_fixture_data)); ++ ++ if (!data) ++ goto fail; ++ ++ /* Acquire the kbase device */ ++ data->kbdev = kbase_find_device(-1); ++ if (data->kbdev == NULL) { ++ kutf_test_fail(context, "Failed to find kbase device"); ++ goto fail; ++ } ++ ++ return data; ++ ++fail: ++ return NULL; ++} ++ ++/** ++ * mali_kutf_irq_default_remove_fixture() - Destroy fixture data previously ++ * created by mali_kutf_irq_default_create_fixture. ++ * ++ * @context: KUTF context. ++ */ ++static void mali_kutf_irq_default_remove_fixture( ++ struct kutf_context *context) ++{ ++ struct kutf_irq_fixture_data *data = context->fixture; ++ struct kbase_device *kbdev = data->kbdev; ++ ++ kbase_release_device(kbdev); ++} ++ ++/** ++ * mali_kutf_irq_latency() - measure GPU IRQ latency ++ * @context: kutf context within which to perform the test ++ * ++ * The test triggers IRQs manually, and measures the ++ * time between triggering the IRQ and the IRQ handler being executed. ++ * ++ * This is not a traditional test, in that the pass/fail status has little ++ * meaning (other than indicating that the IRQ handler executed at all). Instead ++ * the results are in the latencies provided with the test result. There is no ++ * meaningful pass/fail result that can be obtained here, instead the latencies ++ * are provided for manual analysis only. ++ */ ++static void mali_kutf_irq_latency(struct kutf_context *context) ++{ ++ struct kutf_irq_fixture_data *data = context->fixture; ++ struct kbase_device *kbdev = data->kbdev; ++ u64 min_time = U64_MAX, max_time = 0, average_time = 0; ++ u32 i; ++ const char *results; ++ ++ /* Force GPU to be powered */ ++ kbase_pm_context_active(kbdev); ++ kbase_pm_wait_for_desired_state(kbdev); ++ ++ kbase_set_custom_irq_handler(kbdev, kbase_gpu_irq_custom_handler, ++ GPU_IRQ_HANDLER); ++ ++ for (i = 1; i <= NR_TEST_IRQS; i++) { ++ u64 start_time = ktime_get_real_ns(); ++ ++ triggered = false; ++ ++ /* Trigger fake IRQ */ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), ++ TEST_IRQ); ++ ++ if (wait_event_timeout(wait, triggered, IRQ_TIMEOUT) == 0) { ++ /* Wait extra time to see if it would come */ ++ wait_event_timeout(wait, triggered, 10 * IRQ_TIMEOUT); ++ break; ++ } ++ ++ if ((irq_time - start_time) < min_time) ++ min_time = irq_time - start_time; ++ if ((irq_time - start_time) > max_time) ++ max_time = irq_time - start_time; ++ average_time += irq_time - start_time; ++ ++ udelay(10); ++ } ++ ++ /* Go back to default handler */ ++ kbase_set_custom_irq_handler(kbdev, NULL, GPU_IRQ_HANDLER); ++ ++ kbase_pm_context_idle(kbdev); ++ ++ if (i > NR_TEST_IRQS) { ++ do_div(average_time, NR_TEST_IRQS); ++ results = kutf_dsprintf(&context->fixture_pool, ++ "Min latency = %lldns, Max latency = %lldns, Average latency = %lldns\n", ++ min_time, max_time, average_time); ++ kutf_test_pass(context, results); ++ } else { ++ results = kutf_dsprintf(&context->fixture_pool, ++ "Timed out for the %u-th IRQ (loop_limit: %u), triggered late: %d\n", ++ i, NR_TEST_IRQS, triggered); ++ kutf_test_fail(context, results); ++ } ++} ++ ++/** ++ * Module entry point for this test. ++ */ ++int mali_kutf_irq_test_main_init(void) ++{ ++ struct kutf_suite *suite; ++ ++ irq_app = kutf_create_application("irq"); ++ ++ if (NULL == irq_app) { ++ pr_warn("Creation of test application failed!\n"); ++ return -ENOMEM; ++ } ++ ++ suite = kutf_create_suite(irq_app, "irq_default", ++ 1, mali_kutf_irq_default_create_fixture, ++ mali_kutf_irq_default_remove_fixture); ++ ++ if (NULL == suite) { ++ pr_warn("Creation of test suite failed!\n"); ++ kutf_destroy_application(irq_app); ++ return -ENOMEM; ++ } ++ ++ kutf_add_test(suite, 0x0, "irq_latency", ++ mali_kutf_irq_latency); ++ return 0; ++} ++ ++/** ++ * Module exit point for this test. ++ */ ++void mali_kutf_irq_test_main_exit(void) ++{ ++ kutf_destroy_application(irq_app); ++} ++ ++module_init(mali_kutf_irq_test_main_init); ++module_exit(mali_kutf_irq_test_main_exit); ++ ++MODULE_LICENSE("GPL"); ++MODULE_AUTHOR("ARM Ltd."); ++MODULE_VERSION("1.0"); +diff --git a/drivers/gpu/arm/bifrost/thirdparty/mali_kbase_mmap.c b/drivers/gpu/arm/bifrost/thirdparty/mali_kbase_mmap.c +new file mode 100755 +index 000000000000..cd90ea0ec285 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/thirdparty/mali_kbase_mmap.c +@@ -0,0 +1,368 @@ ++/* ++ * ++ * (C) COPYRIGHT ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ *//* ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ */ ++ ++#include "linux/mman.h" ++#include "../mali_kbase.h" ++ ++/* mali_kbase_mmap.c ++ * ++ * This file contains Linux specific implementation of ++ * kbase_context_get_unmapped_area() interface. ++ */ ++ ++ ++/** ++ * align_and_check() - Align the specified pointer to the provided alignment and ++ * check that it is still in range. ++ * @gap_end: Highest possible start address for allocation (end of gap in ++ * address space) ++ * @gap_start: Start address of current memory area / gap in address space ++ * @info: vm_unmapped_area_info structure passed to caller, containing ++ * alignment, length and limits for the allocation ++ * @is_shader_code: True if the allocation is for shader code (which has ++ * additional alignment requirements) ++ * @is_same_4gb_page: True if the allocation needs to reside completely within ++ * a 4GB chunk ++ * ++ * Return: true if gap_end is now aligned correctly and is still in range, ++ * false otherwise ++ */ ++static bool align_and_check(unsigned long *gap_end, unsigned long gap_start, ++ struct vm_unmapped_area_info *info, bool is_shader_code, ++ bool is_same_4gb_page) ++{ ++ /* Compute highest gap address at the desired alignment */ ++ (*gap_end) -= info->length; ++ (*gap_end) -= (*gap_end - info->align_offset) & info->align_mask; ++ ++ if (is_shader_code) { ++ /* Check for 4GB boundary */ ++ if (0 == (*gap_end & BASE_MEM_MASK_4GB)) ++ (*gap_end) -= (info->align_offset ? info->align_offset : ++ info->length); ++ if (0 == ((*gap_end + info->length) & BASE_MEM_MASK_4GB)) ++ (*gap_end) -= (info->align_offset ? info->align_offset : ++ info->length); ++ ++ if (!(*gap_end & BASE_MEM_MASK_4GB) || !((*gap_end + ++ info->length) & BASE_MEM_MASK_4GB)) ++ return false; ++ } else if (is_same_4gb_page) { ++ unsigned long start = *gap_end; ++ unsigned long end = *gap_end + info->length; ++ unsigned long mask = ~((unsigned long)U32_MAX); ++ ++ /* Check if 4GB boundary is straddled */ ++ if ((start & mask) != ((end - 1) & mask)) { ++ unsigned long offset = end - (end & mask); ++ /* This is to ensure that alignment doesn't get ++ * disturbed in an attempt to prevent straddling at ++ * 4GB boundary. The GPU VA is aligned to 2MB when the ++ * allocation size is > 2MB and there is enough CPU & ++ * GPU virtual space. ++ */ ++ unsigned long rounded_offset = ++ ALIGN(offset, info->align_mask + 1); ++ ++ start -= rounded_offset; ++ end -= rounded_offset; ++ ++ *gap_end = start; ++ ++ /* The preceding 4GB boundary shall not get straddled, ++ * even after accounting for the alignment, as the ++ * size of allocation is limited to 4GB and the initial ++ * start location was already aligned. ++ */ ++ WARN_ON((start & mask) != ((end - 1) & mask)); ++ } ++ } ++ ++ ++ if ((*gap_end < info->low_limit) || (*gap_end < gap_start)) ++ return false; ++ ++ ++ return true; ++} ++ ++/** ++ * kbase_unmapped_area_topdown() - allocates new areas top-down from ++ * below the stack limit. ++ * @info: Information about the memory area to allocate. ++ * @is_shader_code: Boolean which denotes whether the allocated area is ++ * intended for the use by shader core in which case a ++ * special alignment requirements apply. ++ * @is_same_4gb_page: Boolean which indicates whether the allocated area needs ++ * to reside completely within a 4GB chunk. ++ * ++ * The unmapped_area_topdown() function in the Linux kernel is not exported ++ * using EXPORT_SYMBOL_GPL macro. To allow us to call this function from a ++ * module and also make use of the fact that some of the requirements for ++ * the unmapped area are known in advance, we implemented an extended version ++ * of this function and prefixed it with 'kbase_'. ++ * ++ * The difference in the call parameter list comes from the fact that ++ * kbase_unmapped_area_topdown() is called with additional parameters which ++ * are provided to indicate whether the allocation is for a shader core memory, ++ * which has additional alignment requirements, and whether the allocation can ++ * straddle a 4GB boundary. ++ * ++ * The modification of the original Linux function lies in how the computation ++ * of the highest gap address at the desired alignment is performed once the ++ * gap with desirable properties is found. For this purpose a special function ++ * is introduced (@ref align_and_check()) which beside computing the gap end ++ * at the desired alignment also performs additional alignment checks for the ++ * case when the memory is executable shader core memory, for which it is ++ * ensured that the gap does not end on a 4GB boundary, and for the case when ++ * memory needs to be confined within a 4GB chunk. ++ * ++ * Return: address of the found gap end (high limit) if area is found; ++ * -ENOMEM if search is unsuccessful ++*/ ++ ++static unsigned long kbase_unmapped_area_topdown(struct vm_unmapped_area_info ++ *info, bool is_shader_code, bool is_same_4gb_page) ++{ ++ struct mm_struct *mm = current->mm; ++ struct vm_area_struct *vma; ++ unsigned long length, low_limit, high_limit, gap_start, gap_end; ++ ++ /* Adjust search length to account for worst case alignment overhead */ ++ length = info->length + info->align_mask; ++ if (length < info->length) ++ return -ENOMEM; ++ ++ /* ++ * Adjust search limits by the desired length. ++ * See implementation comment at top of unmapped_area(). ++ */ ++ gap_end = info->high_limit; ++ if (gap_end < length) ++ return -ENOMEM; ++ high_limit = gap_end - length; ++ ++ if (info->low_limit > high_limit) ++ return -ENOMEM; ++ low_limit = info->low_limit + length; ++ ++ /* Check highest gap, which does not precede any rbtree node */ ++ gap_start = mm->highest_vm_end; ++ if (gap_start <= high_limit) { ++ if (align_and_check(&gap_end, gap_start, info, ++ is_shader_code, is_same_4gb_page)) ++ return gap_end; ++ } ++ ++ /* Check if rbtree root looks promising */ ++ if (RB_EMPTY_ROOT(&mm->mm_rb)) ++ return -ENOMEM; ++ vma = rb_entry(mm->mm_rb.rb_node, struct vm_area_struct, vm_rb); ++ if (vma->rb_subtree_gap < length) ++ return -ENOMEM; ++ ++ while (true) { ++ /* Visit right subtree if it looks promising */ ++ gap_start = vma->vm_prev ? vma->vm_prev->vm_end : 0; ++ if (gap_start <= high_limit && vma->vm_rb.rb_right) { ++ struct vm_area_struct *right = ++ rb_entry(vma->vm_rb.rb_right, ++ struct vm_area_struct, vm_rb); ++ if (right->rb_subtree_gap >= length) { ++ vma = right; ++ continue; ++ } ++ } ++ ++check_current: ++ /* Check if current node has a suitable gap */ ++ gap_end = vma->vm_start; ++ if (gap_end < low_limit) ++ return -ENOMEM; ++ if (gap_start <= high_limit && gap_end - gap_start >= length) { ++ /* We found a suitable gap. Clip it with the original ++ * high_limit. */ ++ if (gap_end > info->high_limit) ++ gap_end = info->high_limit; ++ ++ if (align_and_check(&gap_end, gap_start, info, ++ is_shader_code, is_same_4gb_page)) ++ return gap_end; ++ } ++ ++ /* Visit left subtree if it looks promising */ ++ if (vma->vm_rb.rb_left) { ++ struct vm_area_struct *left = ++ rb_entry(vma->vm_rb.rb_left, ++ struct vm_area_struct, vm_rb); ++ if (left->rb_subtree_gap >= length) { ++ vma = left; ++ continue; ++ } ++ } ++ ++ /* Go back up the rbtree to find next candidate node */ ++ while (true) { ++ struct rb_node *prev = &vma->vm_rb; ++ ++ if (!rb_parent(prev)) ++ return -ENOMEM; ++ vma = rb_entry(rb_parent(prev), ++ struct vm_area_struct, vm_rb); ++ if (prev == vma->vm_rb.rb_right) { ++ gap_start = vma->vm_prev ? ++ vma->vm_prev->vm_end : 0; ++ goto check_current; ++ } ++ } ++ } ++ ++ return -ENOMEM; ++} ++ ++ ++/* This function is based on Linux kernel's arch_get_unmapped_area, but ++ * simplified slightly. Modifications come from the fact that some values ++ * about the memory area are known in advance. ++ */ ++unsigned long kbase_context_get_unmapped_area(struct kbase_context *const kctx, ++ const unsigned long addr, const unsigned long len, ++ const unsigned long pgoff, const unsigned long flags) ++{ ++ struct mm_struct *mm = current->mm; ++ struct vm_unmapped_area_info info; ++ unsigned long align_offset = 0; ++ unsigned long align_mask = 0; ++ unsigned long high_limit = mm->mmap_base; ++ unsigned long low_limit = PAGE_SIZE; ++ int cpu_va_bits = BITS_PER_LONG; ++ int gpu_pc_bits = ++ kctx->kbdev->gpu_props.props.core_props.log2_program_counter_size; ++ bool is_shader_code = false; ++ bool is_same_4gb_page = false; ++ unsigned long ret; ++ ++ /* err on fixed address */ ++ if ((flags & MAP_FIXED) || addr) ++ return -EINVAL; ++ ++#ifdef CONFIG_64BIT ++ /* too big? */ ++ if (len > TASK_SIZE - SZ_2M) ++ return -ENOMEM; ++ ++ if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { ++ ++ high_limit = min_t(unsigned long, mm->mmap_base, ++ (kctx->same_va_end << PAGE_SHIFT)); ++ ++ /* If there's enough (> 33 bits) of GPU VA space, align ++ * to 2MB boundaries. ++ */ ++ if (kctx->kbdev->gpu_props.mmu.va_bits > 33) { ++ if (len >= SZ_2M) { ++ align_offset = SZ_2M; ++ align_mask = SZ_2M - 1; ++ } ++ } ++ ++ low_limit = SZ_2M; ++ } else { ++ cpu_va_bits = 32; ++ } ++#endif /* CONFIG_64BIT */ ++ if ((PFN_DOWN(BASE_MEM_COOKIE_BASE) <= pgoff) && ++ (PFN_DOWN(BASE_MEM_FIRST_FREE_ADDRESS) > pgoff)) { ++ int cookie = pgoff - PFN_DOWN(BASE_MEM_COOKIE_BASE); ++ struct kbase_va_region *reg; ++ ++ /* Need to hold gpu vm lock when using reg */ ++ kbase_gpu_vm_lock(kctx); ++ reg = kctx->pending_regions[cookie]; ++ if (!reg) { ++ kbase_gpu_vm_unlock(kctx); ++ return -EINVAL; ++ } ++ if (!(reg->flags & KBASE_REG_GPU_NX)) { ++ if (cpu_va_bits > gpu_pc_bits) { ++ align_offset = 1ULL << gpu_pc_bits; ++ align_mask = align_offset - 1; ++ is_shader_code = true; ++ } ++#if !MALI_USE_CSF ++ } else if (reg->flags & KBASE_REG_TILER_ALIGN_TOP) { ++ unsigned long extent_bytes = ++ (unsigned long)(reg->extent << PAGE_SHIFT); ++ /* kbase_check_alloc_sizes() already satisfies ++ * these checks, but they're here to avoid ++ * maintenance hazards due to the assumptions ++ * involved */ ++ WARN_ON(reg->extent > (ULONG_MAX >> PAGE_SHIFT)); ++ WARN_ON(reg->initial_commit > (ULONG_MAX >> PAGE_SHIFT)); ++ WARN_ON(!is_power_of_2(extent_bytes)); ++ align_mask = extent_bytes - 1; ++ align_offset = ++ extent_bytes - (reg->initial_commit << PAGE_SHIFT); ++#endif /* !MALI_USE_CSF */ ++ } else if (reg->flags & KBASE_REG_GPU_VA_SAME_4GB_PAGE) { ++ is_same_4gb_page = true; ++ } ++ kbase_gpu_vm_unlock(kctx); ++#ifndef CONFIG_64BIT ++ } else { ++ return current->mm->get_unmapped_area( ++ kctx->filp, addr, len, pgoff, flags); ++#endif ++ } ++ ++ info.flags = 0; ++ info.length = len; ++ info.low_limit = low_limit; ++ info.high_limit = high_limit; ++ info.align_offset = align_offset; ++ info.align_mask = align_mask; ++ ++ ret = kbase_unmapped_area_topdown(&info, is_shader_code, ++ is_same_4gb_page); ++ ++ if (IS_ERR_VALUE(ret) && high_limit == mm->mmap_base && ++ high_limit < (kctx->same_va_end << PAGE_SHIFT)) { ++ /* Retry above mmap_base */ ++ info.low_limit = mm->mmap_base; ++ info.high_limit = min_t(u64, TASK_SIZE, ++ (kctx->same_va_end << PAGE_SHIFT)); ++ ++ ret = kbase_unmapped_area_topdown(&info, is_shader_code, ++ is_same_4gb_page); ++ } ++ ++ return ret; ++} +diff --git a/drivers/gpu/arm/bifrost/tl/backend/mali_kbase_timeline_csf.c b/drivers/gpu/arm/bifrost/tl/backend/mali_kbase_timeline_csf.c +new file mode 100755 +index 000000000000..abaa6bb12b9d +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/tl/backend/mali_kbase_timeline_csf.c +@@ -0,0 +1,172 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include "../mali_kbase_tracepoints.h" ++#include "../mali_kbase_timeline.h" ++#include "../mali_kbase_timeline_priv.h" ++ ++#include ++ ++void kbase_create_timeline_objects(struct kbase_device *kbdev) ++{ ++ unsigned int as_nr; ++ unsigned int slot_i; ++ struct kbase_context *kctx; ++ struct kbase_timeline *timeline = kbdev->timeline; ++ struct kbase_tlstream *summary = ++ &kbdev->timeline->streams[TL_STREAM_TYPE_OBJ_SUMMARY]; ++ ++ /* Summarize the Address Space objects. */ ++ for (as_nr = 0; as_nr < kbdev->nr_hw_address_spaces; as_nr++) ++ __kbase_tlstream_tl_new_as(summary, &kbdev->as[as_nr], as_nr); ++ ++ /* Create Legacy GPU object to track in AOM for dumping */ ++ __kbase_tlstream_tl_new_gpu(summary, ++ kbdev, ++ kbdev->gpu_props.props.raw_props.gpu_id, ++ kbdev->gpu_props.num_cores); ++ ++ ++ for (as_nr = 0; as_nr < kbdev->nr_hw_address_spaces; as_nr++) ++ __kbase_tlstream_tl_lifelink_as_gpu(summary, ++ &kbdev->as[as_nr], ++ kbdev); ++ ++ /* Trace the creation of a new kbase device and set its properties. */ ++ __kbase_tlstream_tl_kbase_new_device(summary, ++ kbdev->gpu_props.props.raw_props.gpu_id, ++ kbdev->gpu_props.num_cores, kbdev->csf.global_iface.group_num, ++ kbdev->nr_hw_address_spaces); ++ ++ /* Lock the context list, to ensure no changes to the list are made ++ * while we're summarizing the contexts and their contents. ++ */ ++ mutex_lock(&kbdev->kctx_list_lock); ++ ++ /* Hold the scheduler lock while we emit the current state ++ * We also need to continue holding the lock until after the first body ++ * stream tracepoints are emitted to ensure we don't change the ++ * scheduler until after then ++ */ ++ mutex_lock(&kbdev->csf.scheduler.lock); ++ ++ for (slot_i = 0; slot_i < kbdev->csf.global_iface.group_num; slot_i++) { ++ ++ struct kbase_queue_group *group = ++ kbdev->csf.scheduler.csg_slots[slot_i].resident_group; ++ ++ if (group) ++ __kbase_tlstream_tl_kbase_device_program_csg(summary, ++ kbdev->gpu_props.props.raw_props.gpu_id, ++ group->handle, slot_i); ++ } ++ ++ /* Reset body stream buffers while holding the kctx lock. ++ * As we are holding the lock, we can guarantee that no kctx creation or ++ * deletion tracepoints can be fired from outside of this function by ++ * some other thread. ++ */ ++ kbase_timeline_streams_body_reset(timeline); ++ ++ mutex_unlock(&kbdev->csf.scheduler.lock); ++ ++ /* For each context in the device... */ ++ list_for_each_entry(kctx, &kbdev->kctx_list, kctx_list_link) { ++ size_t i; ++ struct kbase_tlstream *body = ++ &timeline->streams[TL_STREAM_TYPE_OBJ]; ++ ++ /* Lock the context's KCPU queues, to ensure no KCPU-queue ++ * related actions can occur in this context from now on. ++ */ ++ mutex_lock(&kctx->csf.kcpu_queues.lock); ++ ++ /* Acquire the MMU lock, to ensure we don't get a concurrent ++ * address space assignment while summarizing this context's ++ * address space. ++ */ ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ ++ /* Trace the context itself into the body stream, not the ++ * summary stream. ++ * We place this in the body to ensure it is ordered after any ++ * other tracepoints related to the contents of the context that ++ * might have been fired before acquiring all of the per-context ++ * locks. ++ * This ensures that those tracepoints will not actually affect ++ * the object model state, as they reference a context that ++ * hasn't been traced yet. They may, however, cause benign ++ * errors to be emitted. ++ */ ++ __kbase_tlstream_tl_kbase_new_ctx(body, kctx->id, ++ kbdev->gpu_props.props.raw_props.gpu_id); ++ ++ /* Also trace with the legacy AOM tracepoint for dumping */ ++ __kbase_tlstream_tl_new_ctx(body, ++ kctx, ++ kctx->id, ++ (u32)(kctx->tgid)); ++ ++ /* Trace the currently assigned address space */ ++ if (kctx->as_nr != KBASEP_AS_NR_INVALID) ++ __kbase_tlstream_tl_kbase_ctx_assign_as(body, kctx->id, ++ kctx->as_nr); ++ ++ ++ /* Trace all KCPU queues in the context into the body stream. ++ * As we acquired the KCPU lock after resetting the body stream, ++ * it's possible that some KCPU-related events for this context ++ * occurred between that reset and now. ++ * These will cause errors to be emitted when parsing the ++ * timeline, but they will not affect the correctness of the ++ * object model. ++ */ ++ for (i = 0; i < KBASEP_MAX_KCPU_QUEUES; i++) { ++ const struct kbase_kcpu_command_queue *kcpu_queue = ++ kctx->csf.kcpu_queues.array[i]; ++ ++ if (kcpu_queue) ++ __kbase_tlstream_tl_kbase_new_kcpuqueue( ++ body, kcpu_queue, kcpu_queue->kctx->id, ++ kcpu_queue->num_pending_cmds); ++ } ++ ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ mutex_unlock(&kctx->csf.kcpu_queues.lock); ++ ++ /* Now that all per-context locks for this context have been ++ * released, any per-context tracepoints that are fired from ++ * any other threads will go into the body stream after ++ * everything that was just summarised into the body stream in ++ * this iteration of the loop, so will start to correctly update ++ * the object model state. ++ */ ++ }; ++ ++ mutex_unlock(&kbdev->kctx_list_lock); ++ ++ /* Static object are placed into summary packet that needs to be ++ * transmitted first. Flush all streams to make it available to ++ * user space. ++ */ ++ kbase_timeline_streams_flush(timeline); ++} +diff --git a/drivers/gpu/arm/bifrost/tl/backend/mali_kbase_timeline_jm.c b/drivers/gpu/arm/bifrost/tl/backend/mali_kbase_timeline_jm.c +new file mode 100755 +index 000000000000..c368ac7288da +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/tl/backend/mali_kbase_timeline_jm.c +@@ -0,0 +1,97 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include "../mali_kbase_tracepoints.h" ++#include "../mali_kbase_timeline.h" ++#include "../mali_kbase_timeline_priv.h" ++ ++#include ++ ++void kbase_create_timeline_objects(struct kbase_device *kbdev) ++{ ++ unsigned int lpu_id; ++ unsigned int as_nr; ++ struct kbase_context *kctx; ++ struct kbase_timeline *timeline = kbdev->timeline; ++ struct kbase_tlstream *summary = ++ &timeline->streams[TL_STREAM_TYPE_OBJ_SUMMARY]; ++ ++ /* Summarize the LPU objects. */ ++ for (lpu_id = 0; lpu_id < kbdev->gpu_props.num_job_slots; lpu_id++) { ++ u32 *lpu = ++ &kbdev->gpu_props.props.raw_props.js_features[lpu_id]; ++ __kbase_tlstream_tl_new_lpu(summary, lpu, lpu_id, *lpu); ++ } ++ ++ /* Summarize the Address Space objects. */ ++ for (as_nr = 0; as_nr < kbdev->nr_hw_address_spaces; as_nr++) ++ __kbase_tlstream_tl_new_as(summary, &kbdev->as[as_nr], as_nr); ++ ++ /* Create GPU object and make it retain all LPUs and address spaces. */ ++ __kbase_tlstream_tl_new_gpu(summary, ++ kbdev, ++ kbdev->gpu_props.props.raw_props.gpu_id, ++ kbdev->gpu_props.num_cores); ++ ++ for (lpu_id = 0; lpu_id < kbdev->gpu_props.num_job_slots; lpu_id++) { ++ void *lpu = ++ &kbdev->gpu_props.props.raw_props.js_features[lpu_id]; ++ __kbase_tlstream_tl_lifelink_lpu_gpu(summary, lpu, kbdev); ++ } ++ ++ for (as_nr = 0; as_nr < kbdev->nr_hw_address_spaces; as_nr++) ++ __kbase_tlstream_tl_lifelink_as_gpu(summary, ++ &kbdev->as[as_nr], ++ kbdev); ++ ++ /* Lock the context list, to ensure no changes to the list are made ++ * while we're summarizing the contexts and their contents. ++ */ ++ mutex_lock(&kbdev->kctx_list_lock); ++ ++ /* For each context in the device... */ ++ list_for_each_entry(kctx, &kbdev->kctx_list, kctx_list_link) { ++ /* Summarize the context itself */ ++ __kbase_tlstream_tl_new_ctx(summary, ++ kctx, ++ kctx->id, ++ (u32)(kctx->tgid)); ++ }; ++ ++ /* Reset body stream buffers while holding the kctx lock. ++ * This ensures we can't fire both summary and normal tracepoints for ++ * the same objects. ++ * If we weren't holding the lock, it's possible that the summarized ++ * objects could have been created, destroyed, or used after we ++ * constructed the summary stream tracepoints, but before we reset ++ * the body stream, resulting in losing those object event tracepoints. ++ */ ++ kbase_timeline_streams_body_reset(timeline); ++ ++ mutex_unlock(&kbdev->kctx_list_lock); ++ ++ /* Static object are placed into summary packet that needs to be ++ * transmitted first. Flush all streams to make it available to ++ * user space. ++ */ ++ kbase_timeline_streams_flush(timeline); ++} +\ No newline at end of file +diff --git a/drivers/gpu/arm/bifrost/tl/mali_kbase_timeline.c b/drivers/gpu/arm/bifrost/tl/mali_kbase_timeline.c +new file mode 100755 +index 000000000000..8d8834fdcda6 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/tl/mali_kbase_timeline.c +@@ -0,0 +1,308 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include "mali_kbase_timeline.h" ++#include "mali_kbase_timeline_priv.h" ++#include "mali_kbase_tracepoints.h" ++ ++#include ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++ ++/* The period of autoflush checker execution in milliseconds. */ ++#define AUTOFLUSH_INTERVAL 1000 /* ms */ ++ ++/*****************************************************************************/ ++ ++/* These values are used in mali_kbase_tracepoints.h ++ * to retrieve the streams from a kbase_timeline instance. ++ */ ++const size_t __obj_stream_offset = ++ offsetof(struct kbase_timeline, streams) ++ + sizeof(struct kbase_tlstream) * TL_STREAM_TYPE_OBJ; ++ ++const size_t __aux_stream_offset = ++ offsetof(struct kbase_timeline, streams) ++ + sizeof(struct kbase_tlstream) * TL_STREAM_TYPE_AUX; ++ ++/** ++ * kbasep_timeline_autoflush_timer_callback - autoflush timer callback ++ * @timer: Timer list ++ * ++ * Timer is executed periodically to check if any of the stream contains ++ * buffer ready to be submitted to user space. ++ */ ++static void kbasep_timeline_autoflush_timer_callback(struct timer_list *timer) ++{ ++ enum tl_stream_type stype; ++ int rcode; ++ struct kbase_timeline *timeline = ++ container_of(timer, struct kbase_timeline, autoflush_timer); ++ ++ CSTD_UNUSED(timer); ++ ++ for (stype = (enum tl_stream_type)0; stype < TL_STREAM_TYPE_COUNT; ++ stype++) { ++ struct kbase_tlstream *stream = &timeline->streams[stype]; ++ ++ int af_cnt = atomic_read(&stream->autoflush_counter); ++ ++ /* Check if stream contain unflushed data. */ ++ if (af_cnt < 0) ++ continue; ++ ++ /* Check if stream should be flushed now. */ ++ if (af_cnt != atomic_cmpxchg( ++ &stream->autoflush_counter, ++ af_cnt, ++ af_cnt + 1)) ++ continue; ++ if (!af_cnt) ++ continue; ++ ++ /* Autoflush this stream. */ ++ kbase_tlstream_flush_stream(stream); ++ } ++ ++ if (atomic_read(&timeline->autoflush_timer_active)) ++ rcode = mod_timer( ++ &timeline->autoflush_timer, ++ jiffies + msecs_to_jiffies(AUTOFLUSH_INTERVAL)); ++ CSTD_UNUSED(rcode); ++} ++ ++ ++ ++/*****************************************************************************/ ++ ++int kbase_timeline_init(struct kbase_timeline **timeline, ++ atomic_t *timeline_flags) ++{ ++ enum tl_stream_type i; ++ struct kbase_timeline *result; ++#if MALI_USE_CSF ++ struct kbase_tlstream *csffw_stream; ++#endif ++ ++ if (!timeline || !timeline_flags) ++ return -EINVAL; ++ ++ result = kzalloc(sizeof(*result), GFP_KERNEL); ++ if (!result) ++ return -ENOMEM; ++ ++ mutex_init(&result->reader_lock); ++ init_waitqueue_head(&result->event_queue); ++ ++ /* Prepare stream structures. */ ++ for (i = 0; i < TL_STREAM_TYPE_COUNT; i++) ++ kbase_tlstream_init(&result->streams[i], i, ++ &result->event_queue); ++ ++ /* Initialize autoflush timer. */ ++ atomic_set(&result->autoflush_timer_active, 0); ++ kbase_timer_setup(&result->autoflush_timer, ++ kbasep_timeline_autoflush_timer_callback); ++ result->timeline_flags = timeline_flags; ++ ++#if MALI_USE_CSF ++ csffw_stream = &result->streams[TL_STREAM_TYPE_CSFFW]; ++ kbase_csf_tl_reader_init(&result->csf_tl_reader, csffw_stream); ++#endif ++ ++ *timeline = result; ++ return 0; ++} ++ ++void kbase_timeline_term(struct kbase_timeline *timeline) ++{ ++ enum tl_stream_type i; ++ ++ if (!timeline) ++ return; ++ ++#if MALI_USE_CSF ++ kbase_csf_tl_reader_term(&timeline->csf_tl_reader); ++#endif ++ ++ for (i = (enum tl_stream_type)0; i < TL_STREAM_TYPE_COUNT; i++) ++ kbase_tlstream_term(&timeline->streams[i]); ++ ++ kfree(timeline); ++} ++ ++#ifdef CONFIG_MALI_BIFROST_DEVFREQ ++static void kbase_tlstream_current_devfreq_target(struct kbase_device *kbdev) ++{ ++ struct devfreq *devfreq = kbdev->devfreq; ++ ++ /* Devfreq initialization failure isn't a fatal error, so devfreq might ++ * be null. ++ */ ++ if (devfreq) { ++ unsigned long cur_freq = 0; ++ ++ mutex_lock(&devfreq->lock); ++#if KERNEL_VERSION(4, 3, 0) > LINUX_VERSION_CODE ++ cur_freq = kbdev->current_nominal_freq; ++#else ++ cur_freq = devfreq->last_status.current_frequency; ++#endif ++ KBASE_TLSTREAM_AUX_DEVFREQ_TARGET(kbdev, (u64)cur_freq); ++ mutex_unlock(&devfreq->lock); ++ } ++} ++#endif /* CONFIG_MALI_BIFROST_DEVFREQ */ ++ ++int kbase_timeline_io_acquire(struct kbase_device *kbdev, u32 flags) ++{ ++ int ret; ++ u32 timeline_flags = TLSTREAM_ENABLED | flags; ++ struct kbase_timeline *timeline = kbdev->timeline; ++ ++ if (!atomic_cmpxchg(timeline->timeline_flags, 0, timeline_flags)) { ++ int rcode; ++ ++#if MALI_USE_CSF ++ if (flags & BASE_TLSTREAM_ENABLE_CSFFW_TRACEPOINTS) { ++ ret = kbase_csf_tl_reader_start( ++ &timeline->csf_tl_reader, kbdev); ++ if (ret) ++ { ++ atomic_set(timeline->timeline_flags, 0); ++ return ret; ++ } ++ } ++#endif ++ ret = anon_inode_getfd( ++ "[mali_tlstream]", ++ &kbasep_tlstream_fops, ++ timeline, ++ O_RDONLY | O_CLOEXEC); ++ if (ret < 0) { ++ atomic_set(timeline->timeline_flags, 0); ++#if MALI_USE_CSF ++ kbase_csf_tl_reader_stop(&timeline->csf_tl_reader); ++#endif ++ return ret; ++ } ++ ++ /* Reset and initialize header streams. */ ++ kbase_tlstream_reset( ++ &timeline->streams[TL_STREAM_TYPE_OBJ_SUMMARY]); ++ ++ timeline->obj_header_btc = obj_desc_header_size; ++ timeline->aux_header_btc = aux_desc_header_size; ++ ++ /* Start autoflush timer. */ ++ atomic_set(&timeline->autoflush_timer_active, 1); ++ rcode = mod_timer( ++ &timeline->autoflush_timer, ++ jiffies + msecs_to_jiffies(AUTOFLUSH_INTERVAL)); ++ CSTD_UNUSED(rcode); ++ ++#if !MALI_USE_CSF ++ /* If job dumping is enabled, readjust the software event's ++ * timeout as the default value of 3 seconds is often ++ * insufficient. ++ */ ++ if (flags & BASE_TLSTREAM_JOB_DUMPING_ENABLED) { ++ dev_info(kbdev->dev, ++ "Job dumping is enabled, readjusting the software event's timeout\n"); ++ atomic_set(&kbdev->js_data.soft_job_timeout_ms, ++ 1800000); ++ } ++#endif /* !MALI_USE_CSF */ ++ ++ /* Summary stream was cleared during acquire. ++ * Create static timeline objects that will be ++ * read by client. ++ */ ++ kbase_create_timeline_objects(kbdev); ++ ++#ifdef CONFIG_MALI_BIFROST_DEVFREQ ++ /* Devfreq target tracepoints are only fired when the target ++ * changes, so we won't know the current target unless we ++ * send it now. ++ */ ++ kbase_tlstream_current_devfreq_target(kbdev); ++#endif /* CONFIG_MALI_BIFROST_DEVFREQ */ ++ ++ } else { ++ ret = -EBUSY; ++ } ++ ++ return ret; ++} ++ ++void kbase_timeline_streams_flush(struct kbase_timeline *timeline) ++{ ++ enum tl_stream_type stype; ++ ++#if MALI_USE_CSF ++ kbase_csf_tl_reader_flush_buffer(&timeline->csf_tl_reader); ++#endif ++ ++ for (stype = 0; stype < TL_STREAM_TYPE_COUNT; stype++) ++ kbase_tlstream_flush_stream(&timeline->streams[stype]); ++} ++ ++void kbase_timeline_streams_body_reset(struct kbase_timeline *timeline) ++{ ++ kbase_tlstream_reset( ++ &timeline->streams[TL_STREAM_TYPE_OBJ]); ++ kbase_tlstream_reset( ++ &timeline->streams[TL_STREAM_TYPE_AUX]); ++#if MALI_USE_CSF ++ kbase_tlstream_reset( ++ &timeline->streams[TL_STREAM_TYPE_CSFFW]); ++#endif ++} ++ ++#if MALI_UNIT_TEST ++void kbase_timeline_stats(struct kbase_timeline *timeline, ++ u32 *bytes_collected, u32 *bytes_generated) ++{ ++ enum tl_stream_type stype; ++ ++ KBASE_DEBUG_ASSERT(bytes_collected); ++ ++ /* Accumulate bytes generated per stream */ ++ *bytes_generated = 0; ++ for (stype = (enum tl_stream_type)0; stype < TL_STREAM_TYPE_COUNT; ++ stype++) ++ *bytes_generated += atomic_read( ++ &timeline->streams[stype].bytes_generated); ++ ++ *bytes_collected = atomic_read(&timeline->bytes_collected); ++} ++#endif /* MALI_UNIT_TEST */ +diff --git a/drivers/gpu/arm/bifrost/tl/mali_kbase_timeline.h b/drivers/gpu/arm/bifrost/tl/mali_kbase_timeline.h +new file mode 100755 +index 000000000000..cd48411b45cf +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/tl/mali_kbase_timeline.h +@@ -0,0 +1,121 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#if !defined(_KBASE_TIMELINE_H) ++#define _KBASE_TIMELINE_H ++ ++#include ++ ++/*****************************************************************************/ ++ ++struct kbase_timeline; ++ ++/** ++ * kbase_timeline_init - initialize timeline infrastructure in kernel ++ * @timeline: Newly created instance of kbase_timeline will be stored in ++ * this pointer. ++ * @timeline_flags: Timeline status will be written to this variable when a ++ * client is attached/detached. The variable must be valid ++ * while timeline instance is valid. ++ * Return: zero on success, negative number on error ++ */ ++int kbase_timeline_init(struct kbase_timeline **timeline, ++ atomic_t *timeline_flags); ++ ++/** ++ * kbase_timeline_term - terminate timeline infrastructure in kernel ++ * ++ * @timeline: Timeline instance to be terminated. It must be previously created ++ * with kbase_timeline_init(). ++ */ ++void kbase_timeline_term(struct kbase_timeline *timeline); ++ ++/** ++ * kbase_timeline_io_acquire - acquire timeline stream file descriptor ++ * @kbdev: Kbase device ++ * @flags: Timeline stream flags ++ * ++ * This descriptor is meant to be used by userspace timeline to gain access to ++ * kernel timeline stream. This stream is later broadcasted by user space to the ++ * timeline client. ++ * Only one entity can own the descriptor at any given time. Descriptor shall be ++ * closed if unused. If descriptor cannot be obtained (i.e. when it is already ++ * being used) return will be a negative value. ++ * ++ * Return: file descriptor on success, negative number on error ++ */ ++int kbase_timeline_io_acquire(struct kbase_device *kbdev, u32 flags); ++ ++/** ++ * kbase_timeline_streams_flush - flush timeline streams. ++ * @timeline: Timeline instance ++ * ++ * Function will flush pending data in all timeline streams. ++ */ ++void kbase_timeline_streams_flush(struct kbase_timeline *timeline); ++ ++/** ++ * kbase_timeline_streams_body_reset - reset timeline body streams. ++ * ++ * Function will discard pending data in all timeline body streams. ++ * @timeline: Timeline instance ++ */ ++void kbase_timeline_streams_body_reset(struct kbase_timeline *timeline); ++ ++#if MALI_UNIT_TEST ++/** ++ * kbase_timeline_test - start timeline stream data generator ++ * @kbdev: Kernel common context ++ * @tpw_count: Number of trace point writers in each context ++ * @msg_delay: Time delay in milliseconds between trace points written by one ++ * writer ++ * @msg_count: Number of trace points written by one writer ++ * @aux_msg: If non-zero aux messages will be included ++ * ++ * This test starts a requested number of asynchronous writers in both IRQ and ++ * thread context. Each writer will generate required number of test ++ * tracepoints (tracepoints with embedded information about writer that ++ * should be verified by user space reader). Tracepoints will be emitted in ++ * all timeline body streams. If aux_msg is non-zero writer will also ++ * generate not testable tracepoints (tracepoints without information about ++ * writer). These tracepoints are used to check correctness of remaining ++ * timeline message generating functions. Writer will wait requested time ++ * between generating another set of messages. This call blocks until all ++ * writers finish. ++ */ ++void kbase_timeline_test( ++ struct kbase_device *kbdev, ++ unsigned int tpw_count, ++ unsigned int msg_delay, ++ unsigned int msg_count, ++ int aux_msg); ++ ++/** ++ * kbase_timeline_stats - read timeline stream statistics ++ * @timeline: Timeline instance ++ * @bytes_collected: Will hold number of bytes read by the user ++ * @bytes_generated: Will hold number of bytes generated by trace points ++ */ ++void kbase_timeline_stats(struct kbase_timeline *timeline, u32 *bytes_collected, u32 *bytes_generated); ++#endif /* MALI_UNIT_TEST */ ++ ++#endif /* _KBASE_TIMELINE_H */ +diff --git a/drivers/gpu/arm/bifrost/tl/mali_kbase_timeline_io.c b/drivers/gpu/arm/bifrost/tl/mali_kbase_timeline_io.c +new file mode 100755 +index 000000000000..724f5fa23725 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/tl/mali_kbase_timeline_io.c +@@ -0,0 +1,362 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include "mali_kbase_timeline_priv.h" ++#include "mali_kbase_tlstream.h" ++#include "mali_kbase_tracepoints.h" ++ ++#include ++ ++/* The timeline stream file operations functions. */ ++static ssize_t kbasep_timeline_io_read( ++ struct file *filp, ++ char __user *buffer, ++ size_t size, ++ loff_t *f_pos); ++static unsigned int kbasep_timeline_io_poll(struct file *filp, poll_table *wait); ++static int kbasep_timeline_io_release(struct inode *inode, struct file *filp); ++ ++/* The timeline stream file operations structure. */ ++const struct file_operations kbasep_tlstream_fops = { ++ .owner = THIS_MODULE, ++ .release = kbasep_timeline_io_release, ++ .read = kbasep_timeline_io_read, ++ .poll = kbasep_timeline_io_poll, ++}; ++ ++/** ++ * kbasep_timeline_io_packet_pending - check timeline streams for pending packets ++ * @timeline: Timeline instance ++ * @ready_stream: Pointer to variable where stream will be placed ++ * @rb_idx_raw: Pointer to variable where read buffer index will be placed ++ * ++ * Function checks all streams for pending packets. It will stop as soon as ++ * packet ready to be submitted to user space is detected. Variables under ++ * pointers, passed as the parameters to this function will be updated with ++ * values pointing to right stream and buffer. ++ * ++ * Return: non-zero if any of timeline streams has at last one packet ready ++ */ ++static int kbasep_timeline_io_packet_pending( ++ struct kbase_timeline *timeline, ++ struct kbase_tlstream **ready_stream, ++ unsigned int *rb_idx_raw) ++{ ++ enum tl_stream_type i; ++ ++ KBASE_DEBUG_ASSERT(ready_stream); ++ KBASE_DEBUG_ASSERT(rb_idx_raw); ++ ++ for (i = (enum tl_stream_type)0; i < TL_STREAM_TYPE_COUNT; ++i) { ++ struct kbase_tlstream *stream = &timeline->streams[i]; ++ *rb_idx_raw = atomic_read(&stream->rbi); ++ /* Read buffer index may be updated by writer in case of ++ * overflow. Read and write buffer indexes must be ++ * loaded in correct order. ++ */ ++ smp_rmb(); ++ if (atomic_read(&stream->wbi) != *rb_idx_raw) { ++ *ready_stream = stream; ++ return 1; ++ } ++ ++ } ++ ++ return 0; ++} ++ ++/** ++ * kbasep_timeline_has_header_data() - ++ * check timeline headers for pending packets ++ * ++ * @timeline: Timeline instance ++ * ++ * Return: non-zero if any of timeline headers has at last one packet ready. ++ */ ++static int kbasep_timeline_has_header_data( ++ struct kbase_timeline *timeline) ++{ ++ return timeline->obj_header_btc ++ || timeline->aux_header_btc ++#if MALI_USE_CSF ++ || timeline->csf_tl_reader.tl_header.btc ++#endif ++ ; ++} ++ ++/** ++ * copy_stream_header() - copy timeline stream header. ++ * ++ * @buffer: Pointer to the buffer provided by user. ++ * @size: Maximum amount of data that can be stored in the buffer. ++ * @copy_len: Pointer to amount of bytes that has been copied already ++ * within the read system call. ++ * @hdr: Pointer to the stream header. ++ * @hdr_size: Header size. ++ * @hdr_btc: Pointer to the remaining number of bytes to copy. ++ * ++ * Returns: 0 if success, -1 otherwise. ++ */ ++static inline int copy_stream_header( ++ char __user *buffer, size_t size, ssize_t *copy_len, ++ const char *hdr, ++ size_t hdr_size, ++ size_t *hdr_btc) ++{ ++ const size_t offset = hdr_size - *hdr_btc; ++ const size_t copy_size = MIN(size - *copy_len, *hdr_btc); ++ ++ if (!*hdr_btc) ++ return 0; ++ ++ if (WARN_ON(*hdr_btc > hdr_size)) ++ return -1; ++ ++ if (copy_to_user(&buffer[*copy_len], &hdr[offset], copy_size)) ++ return -1; ++ ++ *hdr_btc -= copy_size; ++ *copy_len += copy_size; ++ ++ return 0; ++} ++ ++/** ++ * kbasep_timeline_copy_header - copy timeline headers to the user ++ * @timeline: Timeline instance ++ * @buffer: Pointer to the buffer provided by user ++ * @size: Maximum amount of data that can be stored in the buffer ++ * @copy_len: Pointer to amount of bytes that has been copied already ++ * within the read system call. ++ * ++ * This helper function checks if timeline headers have not been sent ++ * to the user, and if so, sends them. copy_len is respectively ++ * updated. ++ * ++ * Returns: 0 if success, -1 if copy_to_user has failed. ++ */ ++static inline int kbasep_timeline_copy_headers( ++ struct kbase_timeline *timeline, ++ char __user *buffer, ++ size_t size, ++ ssize_t *copy_len) ++{ ++ if (copy_stream_header(buffer, size, copy_len, ++ obj_desc_header, ++ obj_desc_header_size, ++ &timeline->obj_header_btc)) ++ return -1; ++ ++ if (copy_stream_header(buffer, size, copy_len, ++ aux_desc_header, ++ aux_desc_header_size, ++ &timeline->aux_header_btc)) ++ return -1; ++#if MALI_USE_CSF ++ if (copy_stream_header(buffer, size, copy_len, ++ timeline->csf_tl_reader.tl_header.data, ++ timeline->csf_tl_reader.tl_header.size, ++ &timeline->csf_tl_reader.tl_header.btc)) ++ return -1; ++#endif ++ return 0; ++} ++ ++ ++/** ++ * kbasep_timeline_io_read - copy data from streams to buffer provided by user ++ * @filp: Pointer to file structure ++ * @buffer: Pointer to the buffer provided by user ++ * @size: Maximum amount of data that can be stored in the buffer ++ * @f_pos: Pointer to file offset (unused) ++ * ++ * Return: number of bytes stored in the buffer ++ */ ++static ssize_t kbasep_timeline_io_read( ++ struct file *filp, ++ char __user *buffer, ++ size_t size, ++ loff_t *f_pos) ++{ ++ ssize_t copy_len = 0; ++ struct kbase_timeline *timeline; ++ ++ KBASE_DEBUG_ASSERT(filp); ++ KBASE_DEBUG_ASSERT(f_pos); ++ ++ if (WARN_ON(!filp->private_data)) ++ return -EFAULT; ++ ++ timeline = (struct kbase_timeline *) filp->private_data; ++ ++ if (!buffer) ++ return -EINVAL; ++ ++ if ((*f_pos < 0) || (size < PACKET_SIZE)) ++ return -EINVAL; ++ ++ mutex_lock(&timeline->reader_lock); ++ ++ while (copy_len < size) { ++ struct kbase_tlstream *stream = NULL; ++ unsigned int rb_idx_raw = 0; ++ unsigned int wb_idx_raw; ++ unsigned int rb_idx; ++ size_t rb_size; ++ ++ if (kbasep_timeline_copy_headers( ++ timeline, buffer, size, ©_len)) { ++ copy_len = -EFAULT; ++ break; ++ } ++ ++ /* If we already read some packets and there is no ++ * packet pending then return back to user. ++ * If we don't have any data yet, wait for packet to be ++ * submitted. ++ */ ++ if (copy_len > 0) { ++ if (!kbasep_timeline_io_packet_pending( ++ timeline, ++ &stream, ++ &rb_idx_raw)) ++ break; ++ } else { ++ if (wait_event_interruptible( ++ timeline->event_queue, ++ kbasep_timeline_io_packet_pending( ++ timeline, ++ &stream, ++ &rb_idx_raw))) { ++ copy_len = -ERESTARTSYS; ++ break; ++ } ++ } ++ ++ if (WARN_ON(!stream)) { ++ copy_len = -EFAULT; ++ break; ++ } ++ ++ /* Check if this packet fits into the user buffer. ++ * If so copy its content. ++ */ ++ rb_idx = rb_idx_raw % PACKET_COUNT; ++ rb_size = atomic_read(&stream->buffer[rb_idx].size); ++ if (rb_size > size - copy_len) ++ break; ++ if (copy_to_user( ++ &buffer[copy_len], ++ stream->buffer[rb_idx].data, ++ rb_size)) { ++ copy_len = -EFAULT; ++ break; ++ } ++ ++ /* If the distance between read buffer index and write ++ * buffer index became more than PACKET_COUNT, then overflow ++ * happened and we need to ignore the last portion of bytes ++ * that we have just sent to user. ++ */ ++ smp_rmb(); ++ wb_idx_raw = atomic_read(&stream->wbi); ++ ++ if (wb_idx_raw - rb_idx_raw < PACKET_COUNT) { ++ copy_len += rb_size; ++ atomic_inc(&stream->rbi); ++#if MALI_UNIT_TEST ++ atomic_add(rb_size, &timeline->bytes_collected); ++#endif /* MALI_UNIT_TEST */ ++ ++ } else { ++ const unsigned int new_rb_idx_raw = ++ wb_idx_raw - PACKET_COUNT + 1; ++ /* Adjust read buffer index to the next valid buffer */ ++ atomic_set(&stream->rbi, new_rb_idx_raw); ++ } ++ } ++ ++ mutex_unlock(&timeline->reader_lock); ++ ++ return copy_len; ++} ++ ++/** ++ * kbasep_timeline_io_poll - poll timeline stream for packets ++ * @filp: Pointer to file structure ++ * @wait: Pointer to poll table ++ * Return: POLLIN if data can be read without blocking, otherwise zero ++ */ ++static unsigned int kbasep_timeline_io_poll(struct file *filp, poll_table *wait) ++{ ++ struct kbase_tlstream *stream; ++ unsigned int rb_idx; ++ struct kbase_timeline *timeline; ++ ++ KBASE_DEBUG_ASSERT(filp); ++ KBASE_DEBUG_ASSERT(wait); ++ ++ if (WARN_ON(!filp->private_data)) ++ return -EFAULT; ++ ++ timeline = (struct kbase_timeline *) filp->private_data; ++ ++ /* If there are header bytes to copy, read will not block */ ++ if (kbasep_timeline_has_header_data(timeline)) ++ return POLLIN; ++ ++ poll_wait(filp, &timeline->event_queue, wait); ++ if (kbasep_timeline_io_packet_pending(timeline, &stream, &rb_idx)) ++ return POLLIN; ++ return 0; ++} ++ ++/** ++ * kbasep_timeline_io_release - release timeline stream descriptor ++ * @inode: Pointer to inode structure ++ * @filp: Pointer to file structure ++ * ++ * Return always return zero ++ */ ++static int kbasep_timeline_io_release(struct inode *inode, struct file *filp) ++{ ++ struct kbase_timeline *timeline; ++ ++ KBASE_DEBUG_ASSERT(inode); ++ KBASE_DEBUG_ASSERT(filp); ++ KBASE_DEBUG_ASSERT(filp->private_data); ++ ++ CSTD_UNUSED(inode); ++ ++ timeline = (struct kbase_timeline *) filp->private_data; ++ ++#if MALI_USE_CSF ++ kbase_csf_tl_reader_stop(&timeline->csf_tl_reader); ++#endif ++ ++ /* Stop autoflush timer before releasing access to streams. */ ++ atomic_set(&timeline->autoflush_timer_active, 0); ++ del_timer_sync(&timeline->autoflush_timer); ++ ++ atomic_set(timeline->timeline_flags, 0); ++ return 0; ++} +diff --git a/drivers/gpu/arm/bifrost/tl/mali_kbase_timeline_priv.h b/drivers/gpu/arm/bifrost/tl/mali_kbase_timeline_priv.h +new file mode 100755 +index 000000000000..35eec467af90 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/tl/mali_kbase_timeline_priv.h +@@ -0,0 +1,73 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#if !defined(_KBASE_TIMELINE_PRIV_H) ++#define _KBASE_TIMELINE_PRIV_H ++ ++#include ++#include "mali_kbase_tlstream.h" ++ ++#if MALI_USE_CSF ++#include "csf/mali_kbase_csf_tl_reader.h" ++#include "csf/mali_kbase_csf_trace_buffer.h" ++#endif ++ ++#include ++#include ++#include ++ ++/** ++ * struct kbase_timeline - timeline state structure ++ * @streams: The timeline streams generated by kernel ++ * @autoflush_timer: Autoflush timer ++ * @autoflush_timer_active: If non-zero autoflush timer is active ++ * @reader_lock: Reader lock. Only one reader is allowed to ++ * have access to the timeline streams at any given time. ++ * @event_queue: Timeline stream event queue ++ * @bytes_collected: Number of bytes read by user ++ * @timeline_flags: Zero, if timeline is disabled. Timeline stream flags ++ * otherwise. See kbase_timeline_io_acquire(). ++ * @obj_header_btc: Remaining bytes to copy for the object stream header ++ * @aux_header_btc: Remaining bytes to copy for the aux stream header ++ */ ++struct kbase_timeline { ++ struct kbase_tlstream streams[TL_STREAM_TYPE_COUNT]; ++ struct timer_list autoflush_timer; ++ atomic_t autoflush_timer_active; ++ struct mutex reader_lock; ++ wait_queue_head_t event_queue; ++#if MALI_UNIT_TEST ++ atomic_t bytes_collected; ++#endif /* MALI_UNIT_TEST */ ++ atomic_t *timeline_flags; ++ size_t obj_header_btc; ++ size_t aux_header_btc; ++#if MALI_USE_CSF ++ struct kbase_csf_tl_reader csf_tl_reader; ++#endif ++}; ++ ++extern const struct file_operations kbasep_tlstream_fops; ++ ++void kbase_create_timeline_objects(struct kbase_device *kbdev); ++ ++#endif /* _KBASE_TIMELINE_PRIV_H */ +diff --git a/drivers/gpu/arm/bifrost/tl/mali_kbase_tl_serialize.h b/drivers/gpu/arm/bifrost/tl/mali_kbase_tl_serialize.h +new file mode 100755 +index 000000000000..3e378279cf2c +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/tl/mali_kbase_tl_serialize.h +@@ -0,0 +1,125 @@ ++/* ++ * ++ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#if !defined(_KBASE_TL_SERIALIZE_H) ++#define _KBASE_TL_SERIALIZE_H ++ ++#include ++ ++#include ++ ++/* The number of nanoseconds in a second. */ ++#define NSECS_IN_SEC 1000000000ull /* ns */ ++ ++/** ++ * kbasep_serialize_bytes - serialize bytes to the message buffer ++ * ++ * Serialize bytes as is using memcpy() ++ * ++ * @buffer: Message buffer ++ * @pos: Message buffer offset ++ * @bytes: Bytes to serialize ++ * @len: Length of bytes array ++ * ++ * Return: updated position in the buffer ++ */ ++static inline size_t kbasep_serialize_bytes( ++ char *buffer, ++ size_t pos, ++ const void *bytes, ++ size_t len) ++{ ++ KBASE_DEBUG_ASSERT(buffer); ++ KBASE_DEBUG_ASSERT(bytes); ++ ++ memcpy(&buffer[pos], bytes, len); ++ ++ return pos + len; ++} ++ ++/** ++ * kbasep_serialize_string - serialize string to the message buffer ++ * ++ * String is serialized as 4 bytes for string size, ++ * then string content and then null terminator. ++ * ++ * @buffer: Message buffer ++ * @pos: Message buffer offset ++ * @string: String to serialize ++ * @max_write_size: Number of bytes that can be stored in buffer ++ * ++ * Return: updated position in the buffer ++ */ ++static inline size_t kbasep_serialize_string( ++ char *buffer, ++ size_t pos, ++ const char *string, ++ size_t max_write_size) ++{ ++ u32 string_len; ++ ++ KBASE_DEBUG_ASSERT(buffer); ++ KBASE_DEBUG_ASSERT(string); ++ /* Timeline string consists of at least string length and nul ++ * terminator. ++ */ ++ KBASE_DEBUG_ASSERT(max_write_size >= sizeof(string_len) + sizeof(char)); ++ max_write_size -= sizeof(string_len); ++ ++ string_len = strlcpy( ++ &buffer[pos + sizeof(string_len)], ++ string, ++ max_write_size); ++ string_len += sizeof(char); ++ ++ /* Make sure that the source string fit into the buffer. */ ++ KBASE_DEBUG_ASSERT(string_len <= max_write_size); ++ ++ /* Update string length. */ ++ memcpy(&buffer[pos], &string_len, sizeof(string_len)); ++ ++ return pos + sizeof(string_len) + string_len; ++} ++ ++/** ++ * kbasep_serialize_timestamp - serialize timestamp to the message buffer ++ * ++ * Get current timestamp using kbasep_get_timestamp() ++ * and serialize it as 64 bit unsigned integer. ++ * ++ * @buffer: Message buffer ++ * @pos: Message buffer offset ++ * ++ * Return: updated position in the buffer ++ */ ++static inline size_t kbasep_serialize_timestamp(void *buffer, size_t pos) ++{ ++ u64 timestamp; ++ ++ timestamp = ktime_get_raw_ns(); ++ ++ return kbasep_serialize_bytes( ++ buffer, pos, ++ ×tamp, sizeof(timestamp)); ++} ++#endif /* _KBASE_TL_SERIALIZE_H */ ++ +diff --git a/drivers/gpu/arm/bifrost/tl/mali_kbase_tlstream.c b/drivers/gpu/arm/bifrost/tl/mali_kbase_tlstream.c +new file mode 100755 +index 000000000000..f4239cfafb9d +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/tl/mali_kbase_tlstream.c +@@ -0,0 +1,306 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#include "mali_kbase_tlstream.h" ++#include "mali_kbase_tl_serialize.h" ++#include "mali_kbase_mipe_proto.h" ++ ++/** ++ * kbasep_packet_header_setup - setup the packet header ++ * @buffer: pointer to the buffer ++ * @pkt_family: packet's family ++ * @pkt_type: packet's type ++ * @pkt_class: packet's class ++ * @stream_id: stream id ++ * @numbered: non-zero if this stream is numbered ++ * ++ * Function sets up immutable part of packet header in the given buffer. ++ */ ++static void kbasep_packet_header_setup( ++ char *buffer, ++ enum tl_packet_family pkt_family, ++ enum tl_packet_class pkt_class, ++ enum tl_packet_type pkt_type, ++ unsigned int stream_id, ++ int numbered) ++{ ++ u32 words[2] = { ++ MIPE_PACKET_HEADER_W0(pkt_family, pkt_class, pkt_type, stream_id), ++ MIPE_PACKET_HEADER_W1(0, !!numbered), ++ }; ++ memcpy(buffer, words, sizeof(words)); ++} ++ ++/** ++ * kbasep_packet_header_update - update the packet header ++ * @buffer: pointer to the buffer ++ * @data_size: amount of data carried in this packet ++ * @numbered: non-zero if the stream is numbered ++ * ++ * Function updates mutable part of packet header in the given buffer. ++ * Note that value of data_size must not including size of the header. ++ */ ++static void kbasep_packet_header_update( ++ char *buffer, ++ size_t data_size, ++ int numbered) ++{ ++ u32 word0; ++ u32 word1 = MIPE_PACKET_HEADER_W1((u32)data_size, !!numbered); ++ ++ KBASE_DEBUG_ASSERT(buffer); ++ CSTD_UNUSED(word0); ++ ++ memcpy(&buffer[sizeof(word0)], &word1, sizeof(word1)); ++} ++ ++/** ++ * kbasep_packet_number_update - update the packet number ++ * @buffer: pointer to the buffer ++ * @counter: value of packet counter for this packet's stream ++ * ++ * Function updates packet number embedded within the packet placed in the ++ * given buffer. ++ */ ++static void kbasep_packet_number_update(char *buffer, u32 counter) ++{ ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ memcpy(&buffer[PACKET_HEADER_SIZE], &counter, sizeof(counter)); ++} ++ ++void kbase_tlstream_reset(struct kbase_tlstream *stream) ++{ ++ unsigned int i; ++ ++ for (i = 0; i < PACKET_COUNT; i++) { ++ if (stream->numbered) ++ atomic_set( ++ &stream->buffer[i].size, ++ PACKET_HEADER_SIZE + ++ PACKET_NUMBER_SIZE); ++ else ++ atomic_set(&stream->buffer[i].size, PACKET_HEADER_SIZE); ++ } ++ ++ atomic_set(&stream->wbi, 0); ++ atomic_set(&stream->rbi, 0); ++} ++ ++/* Configuration of timeline streams generated by kernel. */ ++static const struct { ++ enum tl_packet_family pkt_family; ++ enum tl_packet_class pkt_class; ++ enum tl_packet_type pkt_type; ++ enum tl_stream_id stream_id; ++} tl_stream_cfg[TL_STREAM_TYPE_COUNT] = { ++ { ++ TL_PACKET_FAMILY_TL, ++ TL_PACKET_CLASS_OBJ, ++ TL_PACKET_TYPE_SUMMARY, ++ TL_STREAM_ID_KERNEL, ++ }, ++ { ++ TL_PACKET_FAMILY_TL, ++ TL_PACKET_CLASS_OBJ, ++ TL_PACKET_TYPE_BODY, ++ TL_STREAM_ID_KERNEL, ++ }, ++ { ++ TL_PACKET_FAMILY_TL, ++ TL_PACKET_CLASS_AUX, ++ TL_PACKET_TYPE_BODY, ++ TL_STREAM_ID_KERNEL, ++ }, ++#if MALI_USE_CSF ++ { ++ TL_PACKET_FAMILY_TL, ++ TL_PACKET_CLASS_OBJ, ++ TL_PACKET_TYPE_BODY, ++ TL_STREAM_ID_CSFFW, ++ }, ++#endif ++}; ++ ++void kbase_tlstream_init( ++ struct kbase_tlstream *stream, ++ enum tl_stream_type stream_type, ++ wait_queue_head_t *ready_read) ++{ ++ unsigned int i; ++ ++ KBASE_DEBUG_ASSERT(stream); ++ KBASE_DEBUG_ASSERT(TL_STREAM_TYPE_COUNT > stream_type); ++ ++ spin_lock_init(&stream->lock); ++ ++ /* All packets carrying tracepoints shall be numbered. */ ++ if (TL_PACKET_TYPE_BODY == tl_stream_cfg[stream_type].pkt_type) ++ stream->numbered = 1; ++ else ++ stream->numbered = 0; ++ ++ for (i = 0; i < PACKET_COUNT; i++) ++ kbasep_packet_header_setup( ++ stream->buffer[i].data, ++ tl_stream_cfg[stream_type].pkt_family, ++ tl_stream_cfg[stream_type].pkt_class, ++ tl_stream_cfg[stream_type].pkt_type, ++ tl_stream_cfg[stream_type].stream_id, ++ stream->numbered); ++ ++#if MALI_UNIT_TEST ++ atomic_set(&stream->bytes_generated, 0); ++#endif ++ stream->ready_read = ready_read; ++ ++ kbase_tlstream_reset(stream); ++} ++ ++void kbase_tlstream_term(struct kbase_tlstream *stream) ++{ ++ KBASE_DEBUG_ASSERT(stream); ++} ++ ++/** ++ * kbase_tlstream_msgbuf_submit - submit packet to user space ++ * @stream: Pointer to the stream structure ++ * @wb_idx_raw: Write buffer index ++ * @wb_size: Length of data stored in the current buffer ++ * ++ * Updates currently written buffer with the packet header. ++ * Then write index is incremented and the buffer is handed to user space. ++ * Parameters of the new buffer are returned using provided arguments. ++ * ++ * Return: length of data in the new buffer ++ * ++ * Warning: the user must update the stream structure with returned value. ++ */ ++static size_t kbasep_tlstream_msgbuf_submit( ++ struct kbase_tlstream *stream, ++ unsigned int wb_idx_raw, ++ unsigned int wb_size) ++{ ++ unsigned int wb_idx = wb_idx_raw % PACKET_COUNT; ++ ++ /* Set stream as flushed. */ ++ atomic_set(&stream->autoflush_counter, -1); ++ ++ kbasep_packet_header_update( ++ stream->buffer[wb_idx].data, ++ wb_size - PACKET_HEADER_SIZE, ++ stream->numbered); ++ ++ if (stream->numbered) ++ kbasep_packet_number_update( ++ stream->buffer[wb_idx].data, ++ wb_idx_raw); ++ ++ /* Increasing write buffer index will expose this packet to the reader. ++ * As stream->lock is not taken on reader side we must make sure memory ++ * is updated correctly before this will happen. */ ++ smp_wmb(); ++ atomic_inc(&stream->wbi); ++ ++ /* Inform user that packets are ready for reading. */ ++ wake_up_interruptible(stream->ready_read); ++ ++ wb_size = PACKET_HEADER_SIZE; ++ if (stream->numbered) ++ wb_size += PACKET_NUMBER_SIZE; ++ ++ return wb_size; ++} ++ ++char *kbase_tlstream_msgbuf_acquire( ++ struct kbase_tlstream *stream, ++ size_t msg_size, ++ unsigned long *flags) __acquires(&stream->lock) ++{ ++ unsigned int wb_idx_raw; ++ unsigned int wb_idx; ++ size_t wb_size; ++ ++ KBASE_DEBUG_ASSERT( ++ PACKET_SIZE - PACKET_HEADER_SIZE - PACKET_NUMBER_SIZE >= ++ msg_size); ++ ++ spin_lock_irqsave(&stream->lock, *flags); ++ ++ wb_idx_raw = atomic_read(&stream->wbi); ++ wb_idx = wb_idx_raw % PACKET_COUNT; ++ wb_size = atomic_read(&stream->buffer[wb_idx].size); ++ ++ /* Select next buffer if data will not fit into current one. */ ++ if (PACKET_SIZE < wb_size + msg_size) { ++ wb_size = kbasep_tlstream_msgbuf_submit( ++ stream, wb_idx_raw, wb_size); ++ wb_idx = (wb_idx_raw + 1) % PACKET_COUNT; ++ } ++ ++ /* Reserve space in selected buffer. */ ++ atomic_set(&stream->buffer[wb_idx].size, wb_size + msg_size); ++ ++#if MALI_UNIT_TEST ++ atomic_add(msg_size, &stream->bytes_generated); ++#endif /* MALI_UNIT_TEST */ ++ ++ return &stream->buffer[wb_idx].data[wb_size]; ++} ++ ++void kbase_tlstream_msgbuf_release( ++ struct kbase_tlstream *stream, ++ unsigned long flags) __releases(&stream->lock) ++{ ++ /* Mark stream as containing unflushed data. */ ++ atomic_set(&stream->autoflush_counter, 0); ++ ++ spin_unlock_irqrestore(&stream->lock, flags); ++} ++ ++void kbase_tlstream_flush_stream( ++ struct kbase_tlstream *stream) ++{ ++ unsigned long flags; ++ unsigned int wb_idx_raw; ++ unsigned int wb_idx; ++ size_t wb_size; ++ size_t min_size = PACKET_HEADER_SIZE; ++ ++ if (stream->numbered) ++ min_size += PACKET_NUMBER_SIZE; ++ ++ spin_lock_irqsave(&stream->lock, flags); ++ ++ wb_idx_raw = atomic_read(&stream->wbi); ++ wb_idx = wb_idx_raw % PACKET_COUNT; ++ wb_size = atomic_read(&stream->buffer[wb_idx].size); ++ ++ if (wb_size > min_size) { ++ wb_size = kbasep_tlstream_msgbuf_submit( ++ stream, wb_idx_raw, wb_size); ++ wb_idx = (wb_idx_raw + 1) % PACKET_COUNT; ++ atomic_set(&stream->buffer[wb_idx].size, wb_size); ++ } ++ spin_unlock_irqrestore(&stream->lock, flags); ++} ++ +diff --git a/drivers/gpu/arm/bifrost/tl/mali_kbase_tlstream.h b/drivers/gpu/arm/bifrost/tl/mali_kbase_tlstream.h +new file mode 100755 +index 000000000000..faf88d676b5d +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/tl/mali_kbase_tlstream.h +@@ -0,0 +1,169 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++#if !defined(_KBASE_TLSTREAM_H) ++#define _KBASE_TLSTREAM_H ++ ++#include ++#include ++#include ++ ++/* The maximum size of a single packet used by timeline. */ ++#define PACKET_SIZE 4096 /* bytes */ ++ ++/* The number of packets used by one timeline stream. */ ++#if defined(CONFIG_MALI_JOB_DUMP) || defined(CONFIG_MALI_VECTOR_DUMP) ++ #define PACKET_COUNT 64 ++#else ++ #define PACKET_COUNT 32 ++#endif ++ ++/* The maximum expected length of string in tracepoint descriptor. */ ++#define STRLEN_MAX 64 /* bytes */ ++ ++/** ++ * struct kbase_tlstream - timeline stream structure ++ * @lock: Message order lock ++ * @buffer: Array of buffers ++ * @wbi: Write buffer index ++ * @rbi: Read buffer index ++ * @numbered: If non-zero stream's packets are sequentially numbered ++ * @autoflush_counter: Counter tracking stream's autoflush state ++ * @ready_read: Pointer to a wait queue, which is signaled when ++ * timeline messages are ready for collection. ++ * @bytes_generated: Number of bytes generated by tracepoint messages ++ * ++ * This structure holds information needed to construct proper packets in the ++ * timeline stream. ++ * ++ * Each message in the sequence must bear a timestamp that is ++ * greater than the previous message in the same stream. For this reason ++ * a lock is held throughout the process of message creation. ++ * ++ * Each stream contains a set of buffers. Each buffer will hold one MIPE ++ * packet. In case there is no free space required to store the incoming ++ * message the oldest buffer is discarded. Each packet in timeline body ++ * stream has a sequence number embedded, this value must increment ++ * monotonically and is used by the packets receiver to discover these ++ * buffer overflows. ++ * ++ * The autoflush counter is set to a negative number when there is no data ++ * pending for flush and it is set to zero on every update of the buffer. The ++ * autoflush timer will increment the counter by one on every expiry. If there ++ * is no activity on the buffer for two consecutive timer expiries, the stream ++ * buffer will be flushed. ++ */ ++struct kbase_tlstream { ++ spinlock_t lock; ++ ++ struct { ++ atomic_t size; /* number of bytes in buffer */ ++ char data[PACKET_SIZE]; /* buffer's data */ ++ } buffer[PACKET_COUNT]; ++ ++ atomic_t wbi; ++ atomic_t rbi; ++ ++ int numbered; ++ atomic_t autoflush_counter; ++ wait_queue_head_t *ready_read; ++#if MALI_UNIT_TEST ++ atomic_t bytes_generated; ++#endif ++}; ++ ++/* Types of streams generated by timeline. */ ++enum tl_stream_type { ++ TL_STREAM_TYPE_FIRST, ++ TL_STREAM_TYPE_OBJ_SUMMARY = TL_STREAM_TYPE_FIRST, ++ TL_STREAM_TYPE_OBJ, ++ TL_STREAM_TYPE_AUX, ++#if MALI_USE_CSF ++ TL_STREAM_TYPE_CSFFW, ++#endif ++ TL_STREAM_TYPE_COUNT ++}; ++ ++/** ++ * kbase_tlstream_init - initialize timeline stream ++ * @stream: Pointer to the stream structure ++ * @stream_type: Stream type ++ * @ready_read: Pointer to a wait queue to signal when ++ * timeline messages are ready for collection. ++ */ ++void kbase_tlstream_init(struct kbase_tlstream *stream, ++ enum tl_stream_type stream_type, ++ wait_queue_head_t *ready_read); ++ ++/** ++ * kbase_tlstream_term - terminate timeline stream ++ * @stream: Pointer to the stream structure ++ */ ++void kbase_tlstream_term(struct kbase_tlstream *stream); ++ ++/** ++ * kbase_tlstream_reset - reset stream ++ * @stream: Pointer to the stream structure ++ * ++ * Function discards all pending messages and resets packet counters. ++ */ ++void kbase_tlstream_reset(struct kbase_tlstream *stream); ++ ++/** ++ * kbase_tlstream_msgbuf_acquire - lock selected stream and reserve a buffer ++ * @stream: Pointer to the stream structure ++ * @msg_size: Message size ++ * @flags: Pointer to store flags passed back on stream release ++ * ++ * Lock the stream and reserve the number of bytes requested ++ * in msg_size for the user. ++ * ++ * Return: pointer to the buffer where a message can be stored ++ * ++ * Warning: The stream must be released with kbase_tlstream_msgbuf_release(). ++ * Only atomic operations are allowed while the stream is locked ++ * (i.e. do not use any operation that may sleep). ++ */ ++char *kbase_tlstream_msgbuf_acquire(struct kbase_tlstream *stream, ++ size_t msg_size, unsigned long *flags) __acquires(&stream->lock); ++ ++/** ++ * kbase_tlstream_msgbuf_release - unlock selected stream ++ * @stream: Pointer to the stream structure ++ * @flags: Value obtained during stream acquire ++ * ++ * Release the stream that has been previously ++ * locked with a call to kbase_tlstream_msgbuf_acquire(). ++ */ ++void kbase_tlstream_msgbuf_release(struct kbase_tlstream *stream, ++ unsigned long flags) __releases(&stream->lock); ++ ++/** ++ * kbase_tlstream_flush_stream - flush stream ++ * @stream: Pointer to the stream structure ++ * ++ * Flush pending data in the timeline stream. ++ */ ++void kbase_tlstream_flush_stream(struct kbase_tlstream *stream); ++ ++#endif /* _KBASE_TLSTREAM_H */ ++ +diff --git a/drivers/gpu/arm/bifrost/tl/mali_kbase_tracepoints.c b/drivers/gpu/arm/bifrost/tl/mali_kbase_tracepoints.c +new file mode 100755 +index 000000000000..de76fa57051e +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/tl/mali_kbase_tracepoints.c +@@ -0,0 +1,2974 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* ++ * THIS FILE IS AUTOGENERATED BY mali_trace_generator.py. ++ * DO NOT EDIT. ++ */ ++ ++#include "mali_kbase_tracepoints.h" ++#include "mali_kbase_tlstream.h" ++#include "mali_kbase_tl_serialize.h" ++ ++/* clang-format off */ ++ ++/* Message ids of trace events that are recorded in the timeline stream. */ ++enum tl_msg_id_obj { ++ KBASE_TL_NEW_CTX, ++ KBASE_TL_NEW_GPU, ++ KBASE_TL_NEW_LPU, ++ KBASE_TL_NEW_ATOM, ++ KBASE_TL_NEW_AS, ++ KBASE_TL_DEL_CTX, ++ KBASE_TL_DEL_ATOM, ++ KBASE_TL_LIFELINK_LPU_GPU, ++ KBASE_TL_LIFELINK_AS_GPU, ++ KBASE_TL_RET_CTX_LPU, ++ KBASE_TL_RET_ATOM_CTX, ++ KBASE_TL_RET_ATOM_LPU, ++ KBASE_TL_NRET_CTX_LPU, ++ KBASE_TL_NRET_ATOM_CTX, ++ KBASE_TL_NRET_ATOM_LPU, ++ KBASE_TL_RET_AS_CTX, ++ KBASE_TL_NRET_AS_CTX, ++ KBASE_TL_RET_ATOM_AS, ++ KBASE_TL_NRET_ATOM_AS, ++ KBASE_TL_ATTRIB_ATOM_CONFIG, ++ KBASE_TL_ATTRIB_ATOM_PRIORITY, ++ KBASE_TL_ATTRIB_ATOM_STATE, ++ KBASE_TL_ATTRIB_ATOM_PRIORITIZED, ++ KBASE_TL_ATTRIB_ATOM_JIT, ++ KBASE_TL_JIT_USEDPAGES, ++ KBASE_TL_ATTRIB_ATOM_JITALLOCINFO, ++ KBASE_TL_ATTRIB_ATOM_JITFREEINFO, ++ KBASE_TL_ATTRIB_AS_CONFIG, ++ KBASE_TL_EVENT_LPU_SOFTSTOP, ++ KBASE_TL_EVENT_ATOM_SOFTSTOP_EX, ++ KBASE_TL_EVENT_ATOM_SOFTSTOP_ISSUE, ++ KBASE_TL_EVENT_ATOM_SOFTJOB_START, ++ KBASE_TL_EVENT_ATOM_SOFTJOB_END, ++ KBASE_TL_ARBITER_GRANTED, ++ KBASE_TL_ARBITER_STARTED, ++ KBASE_TL_ARBITER_STOP_REQUESTED, ++ KBASE_TL_ARBITER_STOPPED, ++ KBASE_JD_GPU_SOFT_RESET, ++ KBASE_TL_KBASE_NEW_DEVICE, ++ KBASE_TL_KBASE_DEVICE_PROGRAM_CSG, ++ KBASE_TL_KBASE_DEVICE_DEPROGRAM_CSG, ++ KBASE_TL_KBASE_NEW_CTX, ++ KBASE_TL_KBASE_DEL_CTX, ++ KBASE_TL_KBASE_CTX_ASSIGN_AS, ++ KBASE_TL_KBASE_CTX_UNASSIGN_AS, ++ KBASE_TL_KBASE_NEW_KCPUQUEUE, ++ KBASE_TL_KBASE_DEL_KCPUQUEUE, ++ KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_SIGNAL, ++ KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_WAIT, ++ KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_WAIT, ++ KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_SET, ++ KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_MAP_IMPORT, ++ KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT, ++ KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT_FORCE, ++ KBASE_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_ALLOC, ++ KBASE_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_ALLOC, ++ KBASE_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_ALLOC, ++ KBASE_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_FREE, ++ KBASE_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_FREE, ++ KBASE_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_FREE, ++ KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_START, ++ KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_END, ++ KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_START, ++ KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_END, ++ KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_START, ++ KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_END, ++ KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_SET, ++ KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_START, ++ KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_END, ++ KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_START, ++ KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_END, ++ KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_START, ++ KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_END, ++ KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_START, ++ KBASE_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_ALLOC_END, ++ KBASE_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_ALLOC_END, ++ KBASE_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_ALLOC_END, ++ KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_START, ++ KBASE_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_FREE_END, ++ KBASE_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_FREE_END, ++ KBASE_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_FREE_END, ++ KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_ERRORBARRIER, ++ KBASE_TL_KBASE_CSFFW_TLSTREAM_OVERFLOW, ++ KBASE_TL_KBASE_CSFFW_RESET, ++ KBASE_OBJ_MSG_COUNT, ++}; ++ ++/* Message ids of trace events that are recorded in the auxiliary stream. */ ++enum tl_msg_id_aux { ++ KBASE_AUX_PM_STATE, ++ KBASE_AUX_PAGEFAULT, ++ KBASE_AUX_PAGESALLOC, ++ KBASE_AUX_DEVFREQ_TARGET, ++ KBASE_AUX_PROTECTED_ENTER_START, ++ KBASE_AUX_PROTECTED_ENTER_END, ++ KBASE_AUX_PROTECTED_LEAVE_START, ++ KBASE_AUX_PROTECTED_LEAVE_END, ++ KBASE_AUX_JIT_STATS, ++ KBASE_AUX_EVENT_JOB_SLOT, ++ KBASE_AUX_MSG_COUNT, ++}; ++ ++#define OBJ_TP_LIST \ ++ TRACEPOINT_DESC(KBASE_TL_NEW_CTX, \ ++ "object ctx is created", \ ++ "@pII", \ ++ "ctx,ctx_nr,tgid") \ ++ TRACEPOINT_DESC(KBASE_TL_NEW_GPU, \ ++ "object gpu is created", \ ++ "@pII", \ ++ "gpu,gpu_id,core_count") \ ++ TRACEPOINT_DESC(KBASE_TL_NEW_LPU, \ ++ "object lpu is created", \ ++ "@pII", \ ++ "lpu,lpu_nr,lpu_fn") \ ++ TRACEPOINT_DESC(KBASE_TL_NEW_ATOM, \ ++ "object atom is created", \ ++ "@pI", \ ++ "atom,atom_nr") \ ++ TRACEPOINT_DESC(KBASE_TL_NEW_AS, \ ++ "address space object is created", \ ++ "@pI", \ ++ "address_space,as_nr") \ ++ TRACEPOINT_DESC(KBASE_TL_DEL_CTX, \ ++ "context is destroyed", \ ++ "@p", \ ++ "ctx") \ ++ TRACEPOINT_DESC(KBASE_TL_DEL_ATOM, \ ++ "atom is destroyed", \ ++ "@p", \ ++ "atom") \ ++ TRACEPOINT_DESC(KBASE_TL_LIFELINK_LPU_GPU, \ ++ "lpu is deleted with gpu", \ ++ "@pp", \ ++ "lpu,gpu") \ ++ TRACEPOINT_DESC(KBASE_TL_LIFELINK_AS_GPU, \ ++ "address space is deleted with gpu", \ ++ "@pp", \ ++ "address_space,gpu") \ ++ TRACEPOINT_DESC(KBASE_TL_RET_CTX_LPU, \ ++ "context is retained by lpu", \ ++ "@pp", \ ++ "ctx,lpu") \ ++ TRACEPOINT_DESC(KBASE_TL_RET_ATOM_CTX, \ ++ "atom is retained by context", \ ++ "@pp", \ ++ "atom,ctx") \ ++ TRACEPOINT_DESC(KBASE_TL_RET_ATOM_LPU, \ ++ "atom is retained by lpu", \ ++ "@pps", \ ++ "atom,lpu,attrib_match_list") \ ++ TRACEPOINT_DESC(KBASE_TL_NRET_CTX_LPU, \ ++ "context is released by lpu", \ ++ "@pp", \ ++ "ctx,lpu") \ ++ TRACEPOINT_DESC(KBASE_TL_NRET_ATOM_CTX, \ ++ "atom is released by context", \ ++ "@pp", \ ++ "atom,ctx") \ ++ TRACEPOINT_DESC(KBASE_TL_NRET_ATOM_LPU, \ ++ "atom is released by lpu", \ ++ "@pp", \ ++ "atom,lpu") \ ++ TRACEPOINT_DESC(KBASE_TL_RET_AS_CTX, \ ++ "address space is retained by context", \ ++ "@pp", \ ++ "address_space,ctx") \ ++ TRACEPOINT_DESC(KBASE_TL_NRET_AS_CTX, \ ++ "address space is released by context", \ ++ "@pp", \ ++ "address_space,ctx") \ ++ TRACEPOINT_DESC(KBASE_TL_RET_ATOM_AS, \ ++ "atom is retained by address space", \ ++ "@pp", \ ++ "atom,address_space") \ ++ TRACEPOINT_DESC(KBASE_TL_NRET_ATOM_AS, \ ++ "atom is released by address space", \ ++ "@pp", \ ++ "atom,address_space") \ ++ TRACEPOINT_DESC(KBASE_TL_ATTRIB_ATOM_CONFIG, \ ++ "atom job slot attributes", \ ++ "@pLLI", \ ++ "atom,descriptor,affinity,config") \ ++ TRACEPOINT_DESC(KBASE_TL_ATTRIB_ATOM_PRIORITY, \ ++ "atom priority", \ ++ "@pI", \ ++ "atom,prio") \ ++ TRACEPOINT_DESC(KBASE_TL_ATTRIB_ATOM_STATE, \ ++ "atom state", \ ++ "@pI", \ ++ "atom,state") \ ++ TRACEPOINT_DESC(KBASE_TL_ATTRIB_ATOM_PRIORITIZED, \ ++ "atom caused priority change", \ ++ "@p", \ ++ "atom") \ ++ TRACEPOINT_DESC(KBASE_TL_ATTRIB_ATOM_JIT, \ ++ "jit done for atom", \ ++ "@pLLILILLL", \ ++ "atom,edit_addr,new_addr,jit_flags,mem_flags,j_id,com_pgs,extent,va_pgs") \ ++ TRACEPOINT_DESC(KBASE_TL_JIT_USEDPAGES, \ ++ "used pages for jit", \ ++ "@LI", \ ++ "used_pages,j_id") \ ++ TRACEPOINT_DESC(KBASE_TL_ATTRIB_ATOM_JITALLOCINFO, \ ++ "Information about JIT allocations", \ ++ "@pLLLIIIII", \ ++ "atom,va_pgs,com_pgs,extent,j_id,bin_id,max_allocs,jit_flags,usg_id") \ ++ TRACEPOINT_DESC(KBASE_TL_ATTRIB_ATOM_JITFREEINFO, \ ++ "Information about JIT frees", \ ++ "@pI", \ ++ "atom,j_id") \ ++ TRACEPOINT_DESC(KBASE_TL_ATTRIB_AS_CONFIG, \ ++ "address space attributes", \ ++ "@pLLL", \ ++ "address_space,transtab,memattr,transcfg") \ ++ TRACEPOINT_DESC(KBASE_TL_EVENT_LPU_SOFTSTOP, \ ++ "softstop event on given lpu", \ ++ "@p", \ ++ "lpu") \ ++ TRACEPOINT_DESC(KBASE_TL_EVENT_ATOM_SOFTSTOP_EX, \ ++ "atom softstopped", \ ++ "@p", \ ++ "atom") \ ++ TRACEPOINT_DESC(KBASE_TL_EVENT_ATOM_SOFTSTOP_ISSUE, \ ++ "atom softstop issued", \ ++ "@p", \ ++ "atom") \ ++ TRACEPOINT_DESC(KBASE_TL_EVENT_ATOM_SOFTJOB_START, \ ++ "atom soft job has started", \ ++ "@p", \ ++ "atom") \ ++ TRACEPOINT_DESC(KBASE_TL_EVENT_ATOM_SOFTJOB_END, \ ++ "atom soft job has completed", \ ++ "@p", \ ++ "atom") \ ++ TRACEPOINT_DESC(KBASE_TL_ARBITER_GRANTED, \ ++ "Arbiter has granted gpu access", \ ++ "@p", \ ++ "gpu") \ ++ TRACEPOINT_DESC(KBASE_TL_ARBITER_STARTED, \ ++ "Driver is running again and able to process jobs", \ ++ "@p", \ ++ "gpu") \ ++ TRACEPOINT_DESC(KBASE_TL_ARBITER_STOP_REQUESTED, \ ++ "Arbiter has requested driver to stop using gpu", \ ++ "@p", \ ++ "gpu") \ ++ TRACEPOINT_DESC(KBASE_TL_ARBITER_STOPPED, \ ++ "Driver has stopped using gpu", \ ++ "@p", \ ++ "gpu") \ ++ TRACEPOINT_DESC(KBASE_JD_GPU_SOFT_RESET, \ ++ "gpu soft reset", \ ++ "@p", \ ++ "gpu") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_NEW_DEVICE, \ ++ "New KBase Device", \ ++ "@IIII", \ ++ "kbase_device_id,kbase_device_gpu_core_count,kbase_device_max_num_csgs,kbase_device_as_count") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_DEVICE_PROGRAM_CSG, \ ++ "CSG is programmed to a slot", \ ++ "@III", \ ++ "kbase_device_id,gpu_cmdq_grp_handle,kbase_device_csg_slot_index") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_DEVICE_DEPROGRAM_CSG, \ ++ "CSG is deprogrammed from a slot", \ ++ "@II", \ ++ "kbase_device_id,kbase_device_csg_slot_index") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_NEW_CTX, \ ++ "New KBase Context", \ ++ "@II", \ ++ "kernel_ctx_id,kbase_device_id") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_DEL_CTX, \ ++ "Delete KBase Context", \ ++ "@I", \ ++ "kernel_ctx_id") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_CTX_ASSIGN_AS, \ ++ "Address Space is assigned to a KBase context", \ ++ "@II", \ ++ "kernel_ctx_id,kbase_device_as_index") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_CTX_UNASSIGN_AS, \ ++ "Address Space is unassigned from a KBase context", \ ++ "@I", \ ++ "kernel_ctx_id") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_NEW_KCPUQUEUE, \ ++ "New KCPU Queue", \ ++ "@pII", \ ++ "kcpu_queue,kernel_ctx_id,kcpuq_num_pending_cmds") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_DEL_KCPUQUEUE, \ ++ "Delete KCPU Queue", \ ++ "@p", \ ++ "kcpu_queue") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_SIGNAL, \ ++ "KCPU Queue enqueues Signal on Fence", \ ++ "@pp", \ ++ "kcpu_queue,fence") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_WAIT, \ ++ "KCPU Queue enqueues Wait on Fence", \ ++ "@pp", \ ++ "kcpu_queue,fence") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_WAIT, \ ++ "KCPU Queue enqueues Wait on Cross Queue Sync Object", \ ++ "@pLI", \ ++ "kcpu_queue,cqs_obj_gpu_addr,cqs_obj_compare_value") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_SET, \ ++ "KCPU Queue enqueues Set on Cross Queue Sync Object", \ ++ "@pL", \ ++ "kcpu_queue,cqs_obj_gpu_addr") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_MAP_IMPORT, \ ++ "KCPU Queue enqueues Map Import", \ ++ "@pL", \ ++ "kcpu_queue,map_import_buf_gpu_addr") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT, \ ++ "KCPU Queue enqueues Unmap Import", \ ++ "@pL", \ ++ "kcpu_queue,map_import_buf_gpu_addr") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT_FORCE, \ ++ "KCPU Queue enqueues Unmap Import ignoring reference count", \ ++ "@pL", \ ++ "kcpu_queue,map_import_buf_gpu_addr") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_ALLOC, \ ++ "Begin array of KCPU Queue enqueues JIT Alloc", \ ++ "@p", \ ++ "kcpu_queue") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_ALLOC, \ ++ "Array item of KCPU Queue enqueues JIT Alloc", \ ++ "@pLLLLIIIII", \ ++ "kcpu_queue,jit_alloc_gpu_alloc_addr_dest,jit_alloc_va_pages,jit_alloc_commit_pages,jit_alloc_extent,jit_alloc_jit_id,jit_alloc_bin_id,jit_alloc_max_allocations,jit_alloc_flags,jit_alloc_usage_id") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_ALLOC, \ ++ "End array of KCPU Queue enqueues JIT Alloc", \ ++ "@p", \ ++ "kcpu_queue") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_FREE, \ ++ "Begin array of KCPU Queue enqueues JIT Free", \ ++ "@p", \ ++ "kcpu_queue") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_FREE, \ ++ "Array item of KCPU Queue enqueues JIT Free", \ ++ "@pI", \ ++ "kcpu_queue,jit_alloc_jit_id") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_FREE, \ ++ "End array of KCPU Queue enqueues JIT Free", \ ++ "@p", \ ++ "kcpu_queue") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_START, \ ++ "KCPU Queue starts a Signal on Fence", \ ++ "@p", \ ++ "kcpu_queue") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_END, \ ++ "KCPU Queue ends a Signal on Fence", \ ++ "@p", \ ++ "kcpu_queue") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_START, \ ++ "KCPU Queue starts a Wait on Fence", \ ++ "@p", \ ++ "kcpu_queue") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_END, \ ++ "KCPU Queue ends a Wait on Fence", \ ++ "@p", \ ++ "kcpu_queue") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_START, \ ++ "KCPU Queue starts a Wait on an array of Cross Queue Sync Objects", \ ++ "@p", \ ++ "kcpu_queue") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_END, \ ++ "KCPU Queue ends a Wait on an array of Cross Queue Sync Objects", \ ++ "@p", \ ++ "kcpu_queue") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_SET, \ ++ "KCPU Queue executes a Set on an array of Cross Queue Sync Objects", \ ++ "@p", \ ++ "kcpu_queue") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_START, \ ++ "KCPU Queue starts a Map Import", \ ++ "@p", \ ++ "kcpu_queue") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_END, \ ++ "KCPU Queue ends a Map Import", \ ++ "@p", \ ++ "kcpu_queue") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_START, \ ++ "KCPU Queue starts an Unmap Import", \ ++ "@p", \ ++ "kcpu_queue") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_END, \ ++ "KCPU Queue ends an Unmap Import", \ ++ "@p", \ ++ "kcpu_queue") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_START, \ ++ "KCPU Queue starts an Unmap Import ignoring reference count", \ ++ "@p", \ ++ "kcpu_queue") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_END, \ ++ "KCPU Queue ends an Unmap Import ignoring reference count", \ ++ "@p", \ ++ "kcpu_queue") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_START, \ ++ "KCPU Queue starts an array of JIT Allocs", \ ++ "@p", \ ++ "kcpu_queue") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_ALLOC_END, \ ++ "Begin array of KCPU Queue ends an array of JIT Allocs", \ ++ "@p", \ ++ "kcpu_queue") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_ALLOC_END, \ ++ "Array item of KCPU Queue ends an array of JIT Allocs", \ ++ "@pLL", \ ++ "kcpu_queue,jit_alloc_gpu_alloc_addr,jit_alloc_mmu_flags") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_ALLOC_END, \ ++ "End array of KCPU Queue ends an array of JIT Allocs", \ ++ "@p", \ ++ "kcpu_queue") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_START, \ ++ "KCPU Queue starts an array of JIT Frees", \ ++ "@p", \ ++ "kcpu_queue") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_FREE_END, \ ++ "Begin array of KCPU Queue ends an array of JIT Frees", \ ++ "@p", \ ++ "kcpu_queue") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_FREE_END, \ ++ "Array item of KCPU Queue ends an array of JIT Frees", \ ++ "@pL", \ ++ "kcpu_queue,jit_free_pages_used") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_FREE_END, \ ++ "End array of KCPU Queue ends an array of JIT Frees", \ ++ "@p", \ ++ "kcpu_queue") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_ERRORBARRIER, \ ++ "KCPU Queue executes an Error Barrier", \ ++ "@p", \ ++ "kcpu_queue") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_CSFFW_TLSTREAM_OVERFLOW, \ ++ "An overflow has happened with the CSFFW Timeline stream", \ ++ "@LL", \ ++ "csffw_timestamp,csffw_cycle") \ ++ TRACEPOINT_DESC(KBASE_TL_KBASE_CSFFW_RESET, \ ++ "A reset has happened with the CSFFW", \ ++ "@L", \ ++ "csffw_cycle") \ ++ ++#define MIPE_HEADER_BLOB_VAR_NAME __obj_desc_header ++#define MIPE_HEADER_STREAM_ID TL_STREAM_ID_KERNEL ++#define MIPE_HEADER_PKT_CLASS TL_PACKET_CLASS_OBJ ++#define MIPE_HEADER_TRACEPOINT_LIST OBJ_TP_LIST ++#define MIPE_HEADER_TRACEPOINT_LIST_SIZE KBASE_OBJ_MSG_COUNT ++ ++#include "mali_kbase_mipe_gen_header.h" ++ ++const char *obj_desc_header = (const char *) &__obj_desc_header; ++const size_t obj_desc_header_size = sizeof(__obj_desc_header); ++ ++#define AUX_TP_LIST \ ++ TRACEPOINT_DESC(KBASE_AUX_PM_STATE, \ ++ "PM state", \ ++ "@IL", \ ++ "core_type,core_state_bitset") \ ++ TRACEPOINT_DESC(KBASE_AUX_PAGEFAULT, \ ++ "Page fault", \ ++ "@IIL", \ ++ "ctx_nr,as_nr,page_cnt_change") \ ++ TRACEPOINT_DESC(KBASE_AUX_PAGESALLOC, \ ++ "Total alloc pages change", \ ++ "@IL", \ ++ "ctx_nr,page_cnt") \ ++ TRACEPOINT_DESC(KBASE_AUX_DEVFREQ_TARGET, \ ++ "New device frequency target", \ ++ "@L", \ ++ "target_freq") \ ++ TRACEPOINT_DESC(KBASE_AUX_PROTECTED_ENTER_START, \ ++ "enter protected mode start", \ ++ "@p", \ ++ "gpu") \ ++ TRACEPOINT_DESC(KBASE_AUX_PROTECTED_ENTER_END, \ ++ "enter protected mode end", \ ++ "@p", \ ++ "gpu") \ ++ TRACEPOINT_DESC(KBASE_AUX_PROTECTED_LEAVE_START, \ ++ "leave protected mode start", \ ++ "@p", \ ++ "gpu") \ ++ TRACEPOINT_DESC(KBASE_AUX_PROTECTED_LEAVE_END, \ ++ "leave protected mode end", \ ++ "@p", \ ++ "gpu") \ ++ TRACEPOINT_DESC(KBASE_AUX_JIT_STATS, \ ++ "per-bin JIT statistics", \ ++ "@IIIIII", \ ++ "ctx_nr,bid,max_allocs,allocs,va_pages,ph_pages") \ ++ TRACEPOINT_DESC(KBASE_AUX_EVENT_JOB_SLOT, \ ++ "event on a given job slot", \ ++ "@pIII", \ ++ "ctx,slot_nr,atom_nr,event") \ ++ ++#define MIPE_HEADER_BLOB_VAR_NAME __aux_desc_header ++#define MIPE_HEADER_STREAM_ID TL_STREAM_ID_KERNEL ++#define MIPE_HEADER_PKT_CLASS TL_PACKET_CLASS_AUX ++#define MIPE_HEADER_TRACEPOINT_LIST AUX_TP_LIST ++#define MIPE_HEADER_TRACEPOINT_LIST_SIZE KBASE_AUX_MSG_COUNT ++ ++#include "mali_kbase_mipe_gen_header.h" ++ ++const char *aux_desc_header = (const char *) &__aux_desc_header; ++const size_t aux_desc_header_size = sizeof(__aux_desc_header); ++ ++void __kbase_tlstream_tl_new_ctx( ++ struct kbase_tlstream *stream, ++ const void *ctx, ++ u32 ctx_nr, ++ u32 tgid) ++{ ++ const u32 msg_id = KBASE_TL_NEW_CTX; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(ctx) ++ + sizeof(ctx_nr) ++ + sizeof(tgid) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &ctx, sizeof(ctx)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &ctx_nr, sizeof(ctx_nr)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &tgid, sizeof(tgid)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_new_gpu( ++ struct kbase_tlstream *stream, ++ const void *gpu, ++ u32 gpu_id, ++ u32 core_count) ++{ ++ const u32 msg_id = KBASE_TL_NEW_GPU; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(gpu) ++ + sizeof(gpu_id) ++ + sizeof(core_count) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &gpu, sizeof(gpu)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &gpu_id, sizeof(gpu_id)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &core_count, sizeof(core_count)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_new_lpu( ++ struct kbase_tlstream *stream, ++ const void *lpu, ++ u32 lpu_nr, ++ u32 lpu_fn) ++{ ++ const u32 msg_id = KBASE_TL_NEW_LPU; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(lpu) ++ + sizeof(lpu_nr) ++ + sizeof(lpu_fn) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &lpu, sizeof(lpu)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &lpu_nr, sizeof(lpu_nr)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &lpu_fn, sizeof(lpu_fn)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_new_atom( ++ struct kbase_tlstream *stream, ++ const void *atom, ++ u32 atom_nr) ++{ ++ const u32 msg_id = KBASE_TL_NEW_ATOM; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(atom) ++ + sizeof(atom_nr) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &atom, sizeof(atom)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &atom_nr, sizeof(atom_nr)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_new_as( ++ struct kbase_tlstream *stream, ++ const void *address_space, ++ u32 as_nr) ++{ ++ const u32 msg_id = KBASE_TL_NEW_AS; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(address_space) ++ + sizeof(as_nr) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &address_space, sizeof(address_space)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &as_nr, sizeof(as_nr)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_del_ctx( ++ struct kbase_tlstream *stream, ++ const void *ctx) ++{ ++ const u32 msg_id = KBASE_TL_DEL_CTX; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(ctx) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &ctx, sizeof(ctx)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_del_atom( ++ struct kbase_tlstream *stream, ++ const void *atom) ++{ ++ const u32 msg_id = KBASE_TL_DEL_ATOM; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(atom) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &atom, sizeof(atom)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_lifelink_lpu_gpu( ++ struct kbase_tlstream *stream, ++ const void *lpu, ++ const void *gpu) ++{ ++ const u32 msg_id = KBASE_TL_LIFELINK_LPU_GPU; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(lpu) ++ + sizeof(gpu) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &lpu, sizeof(lpu)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &gpu, sizeof(gpu)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_lifelink_as_gpu( ++ struct kbase_tlstream *stream, ++ const void *address_space, ++ const void *gpu) ++{ ++ const u32 msg_id = KBASE_TL_LIFELINK_AS_GPU; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(address_space) ++ + sizeof(gpu) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &address_space, sizeof(address_space)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &gpu, sizeof(gpu)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_ret_ctx_lpu( ++ struct kbase_tlstream *stream, ++ const void *ctx, ++ const void *lpu) ++{ ++ const u32 msg_id = KBASE_TL_RET_CTX_LPU; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(ctx) ++ + sizeof(lpu) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &ctx, sizeof(ctx)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &lpu, sizeof(lpu)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_ret_atom_ctx( ++ struct kbase_tlstream *stream, ++ const void *atom, ++ const void *ctx) ++{ ++ const u32 msg_id = KBASE_TL_RET_ATOM_CTX; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(atom) ++ + sizeof(ctx) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &atom, sizeof(atom)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &ctx, sizeof(ctx)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_ret_atom_lpu( ++ struct kbase_tlstream *stream, ++ const void *atom, ++ const void *lpu, ++ const char *attrib_match_list) ++{ ++ const u32 msg_id = KBASE_TL_RET_ATOM_LPU; ++ const size_t s0 = sizeof(u32) + sizeof(char) ++ + strnlen(attrib_match_list, STRLEN_MAX); ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(atom) ++ + sizeof(lpu) ++ + s0 ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &atom, sizeof(atom)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &lpu, sizeof(lpu)); ++ pos = kbasep_serialize_string(buffer, ++ pos, attrib_match_list, s0); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_nret_ctx_lpu( ++ struct kbase_tlstream *stream, ++ const void *ctx, ++ const void *lpu) ++{ ++ const u32 msg_id = KBASE_TL_NRET_CTX_LPU; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(ctx) ++ + sizeof(lpu) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &ctx, sizeof(ctx)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &lpu, sizeof(lpu)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_nret_atom_ctx( ++ struct kbase_tlstream *stream, ++ const void *atom, ++ const void *ctx) ++{ ++ const u32 msg_id = KBASE_TL_NRET_ATOM_CTX; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(atom) ++ + sizeof(ctx) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &atom, sizeof(atom)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &ctx, sizeof(ctx)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_nret_atom_lpu( ++ struct kbase_tlstream *stream, ++ const void *atom, ++ const void *lpu) ++{ ++ const u32 msg_id = KBASE_TL_NRET_ATOM_LPU; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(atom) ++ + sizeof(lpu) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &atom, sizeof(atom)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &lpu, sizeof(lpu)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_ret_as_ctx( ++ struct kbase_tlstream *stream, ++ const void *address_space, ++ const void *ctx) ++{ ++ const u32 msg_id = KBASE_TL_RET_AS_CTX; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(address_space) ++ + sizeof(ctx) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &address_space, sizeof(address_space)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &ctx, sizeof(ctx)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_nret_as_ctx( ++ struct kbase_tlstream *stream, ++ const void *address_space, ++ const void *ctx) ++{ ++ const u32 msg_id = KBASE_TL_NRET_AS_CTX; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(address_space) ++ + sizeof(ctx) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &address_space, sizeof(address_space)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &ctx, sizeof(ctx)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_ret_atom_as( ++ struct kbase_tlstream *stream, ++ const void *atom, ++ const void *address_space) ++{ ++ const u32 msg_id = KBASE_TL_RET_ATOM_AS; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(atom) ++ + sizeof(address_space) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &atom, sizeof(atom)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &address_space, sizeof(address_space)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_nret_atom_as( ++ struct kbase_tlstream *stream, ++ const void *atom, ++ const void *address_space) ++{ ++ const u32 msg_id = KBASE_TL_NRET_ATOM_AS; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(atom) ++ + sizeof(address_space) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &atom, sizeof(atom)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &address_space, sizeof(address_space)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_attrib_atom_config( ++ struct kbase_tlstream *stream, ++ const void *atom, ++ u64 descriptor, ++ u64 affinity, ++ u32 config) ++{ ++ const u32 msg_id = KBASE_TL_ATTRIB_ATOM_CONFIG; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(atom) ++ + sizeof(descriptor) ++ + sizeof(affinity) ++ + sizeof(config) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &atom, sizeof(atom)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &descriptor, sizeof(descriptor)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &affinity, sizeof(affinity)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &config, sizeof(config)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_attrib_atom_priority( ++ struct kbase_tlstream *stream, ++ const void *atom, ++ u32 prio) ++{ ++ const u32 msg_id = KBASE_TL_ATTRIB_ATOM_PRIORITY; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(atom) ++ + sizeof(prio) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &atom, sizeof(atom)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &prio, sizeof(prio)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_attrib_atom_state( ++ struct kbase_tlstream *stream, ++ const void *atom, ++ u32 state) ++{ ++ const u32 msg_id = KBASE_TL_ATTRIB_ATOM_STATE; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(atom) ++ + sizeof(state) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &atom, sizeof(atom)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &state, sizeof(state)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_attrib_atom_prioritized( ++ struct kbase_tlstream *stream, ++ const void *atom) ++{ ++ const u32 msg_id = KBASE_TL_ATTRIB_ATOM_PRIORITIZED; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(atom) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &atom, sizeof(atom)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_attrib_atom_jit( ++ struct kbase_tlstream *stream, ++ const void *atom, ++ u64 edit_addr, ++ u64 new_addr, ++ u32 jit_flags, ++ u64 mem_flags, ++ u32 j_id, ++ u64 com_pgs, ++ u64 extent, ++ u64 va_pgs) ++{ ++ const u32 msg_id = KBASE_TL_ATTRIB_ATOM_JIT; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(atom) ++ + sizeof(edit_addr) ++ + sizeof(new_addr) ++ + sizeof(jit_flags) ++ + sizeof(mem_flags) ++ + sizeof(j_id) ++ + sizeof(com_pgs) ++ + sizeof(extent) ++ + sizeof(va_pgs) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &atom, sizeof(atom)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &edit_addr, sizeof(edit_addr)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &new_addr, sizeof(new_addr)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &jit_flags, sizeof(jit_flags)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &mem_flags, sizeof(mem_flags)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &j_id, sizeof(j_id)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &com_pgs, sizeof(com_pgs)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &extent, sizeof(extent)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &va_pgs, sizeof(va_pgs)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_jit_usedpages( ++ struct kbase_tlstream *stream, ++ u64 used_pages, ++ u32 j_id) ++{ ++ const u32 msg_id = KBASE_TL_JIT_USEDPAGES; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(used_pages) ++ + sizeof(j_id) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &used_pages, sizeof(used_pages)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &j_id, sizeof(j_id)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_attrib_atom_jitallocinfo( ++ struct kbase_tlstream *stream, ++ const void *atom, ++ u64 va_pgs, ++ u64 com_pgs, ++ u64 extent, ++ u32 j_id, ++ u32 bin_id, ++ u32 max_allocs, ++ u32 jit_flags, ++ u32 usg_id) ++{ ++ const u32 msg_id = KBASE_TL_ATTRIB_ATOM_JITALLOCINFO; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(atom) ++ + sizeof(va_pgs) ++ + sizeof(com_pgs) ++ + sizeof(extent) ++ + sizeof(j_id) ++ + sizeof(bin_id) ++ + sizeof(max_allocs) ++ + sizeof(jit_flags) ++ + sizeof(usg_id) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &atom, sizeof(atom)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &va_pgs, sizeof(va_pgs)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &com_pgs, sizeof(com_pgs)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &extent, sizeof(extent)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &j_id, sizeof(j_id)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &bin_id, sizeof(bin_id)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &max_allocs, sizeof(max_allocs)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &jit_flags, sizeof(jit_flags)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &usg_id, sizeof(usg_id)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_attrib_atom_jitfreeinfo( ++ struct kbase_tlstream *stream, ++ const void *atom, ++ u32 j_id) ++{ ++ const u32 msg_id = KBASE_TL_ATTRIB_ATOM_JITFREEINFO; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(atom) ++ + sizeof(j_id) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &atom, sizeof(atom)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &j_id, sizeof(j_id)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_attrib_as_config( ++ struct kbase_tlstream *stream, ++ const void *address_space, ++ u64 transtab, ++ u64 memattr, ++ u64 transcfg) ++{ ++ const u32 msg_id = KBASE_TL_ATTRIB_AS_CONFIG; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(address_space) ++ + sizeof(transtab) ++ + sizeof(memattr) ++ + sizeof(transcfg) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &address_space, sizeof(address_space)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &transtab, sizeof(transtab)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &memattr, sizeof(memattr)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &transcfg, sizeof(transcfg)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_event_lpu_softstop( ++ struct kbase_tlstream *stream, ++ const void *lpu) ++{ ++ const u32 msg_id = KBASE_TL_EVENT_LPU_SOFTSTOP; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(lpu) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &lpu, sizeof(lpu)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_event_atom_softstop_ex( ++ struct kbase_tlstream *stream, ++ const void *atom) ++{ ++ const u32 msg_id = KBASE_TL_EVENT_ATOM_SOFTSTOP_EX; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(atom) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &atom, sizeof(atom)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_event_atom_softstop_issue( ++ struct kbase_tlstream *stream, ++ const void *atom) ++{ ++ const u32 msg_id = KBASE_TL_EVENT_ATOM_SOFTSTOP_ISSUE; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(atom) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &atom, sizeof(atom)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_event_atom_softjob_start( ++ struct kbase_tlstream *stream, ++ const void *atom) ++{ ++ const u32 msg_id = KBASE_TL_EVENT_ATOM_SOFTJOB_START; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(atom) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &atom, sizeof(atom)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_event_atom_softjob_end( ++ struct kbase_tlstream *stream, ++ const void *atom) ++{ ++ const u32 msg_id = KBASE_TL_EVENT_ATOM_SOFTJOB_END; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(atom) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &atom, sizeof(atom)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_arbiter_granted( ++ struct kbase_tlstream *stream, ++ const void *gpu) ++{ ++ const u32 msg_id = KBASE_TL_ARBITER_GRANTED; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(gpu) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &gpu, sizeof(gpu)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_arbiter_started( ++ struct kbase_tlstream *stream, ++ const void *gpu) ++{ ++ const u32 msg_id = KBASE_TL_ARBITER_STARTED; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(gpu) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &gpu, sizeof(gpu)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_arbiter_stop_requested( ++ struct kbase_tlstream *stream, ++ const void *gpu) ++{ ++ const u32 msg_id = KBASE_TL_ARBITER_STOP_REQUESTED; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(gpu) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &gpu, sizeof(gpu)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_arbiter_stopped( ++ struct kbase_tlstream *stream, ++ const void *gpu) ++{ ++ const u32 msg_id = KBASE_TL_ARBITER_STOPPED; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(gpu) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &gpu, sizeof(gpu)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_jd_gpu_soft_reset( ++ struct kbase_tlstream *stream, ++ const void *gpu) ++{ ++ const u32 msg_id = KBASE_JD_GPU_SOFT_RESET; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(gpu) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &gpu, sizeof(gpu)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_aux_pm_state( ++ struct kbase_tlstream *stream, ++ u32 core_type, ++ u64 core_state_bitset) ++{ ++ const u32 msg_id = KBASE_AUX_PM_STATE; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(core_type) ++ + sizeof(core_state_bitset) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &core_type, sizeof(core_type)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &core_state_bitset, sizeof(core_state_bitset)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_aux_pagefault( ++ struct kbase_tlstream *stream, ++ u32 ctx_nr, ++ u32 as_nr, ++ u64 page_cnt_change) ++{ ++ const u32 msg_id = KBASE_AUX_PAGEFAULT; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(ctx_nr) ++ + sizeof(as_nr) ++ + sizeof(page_cnt_change) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &ctx_nr, sizeof(ctx_nr)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &as_nr, sizeof(as_nr)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &page_cnt_change, sizeof(page_cnt_change)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_aux_pagesalloc( ++ struct kbase_tlstream *stream, ++ u32 ctx_nr, ++ u64 page_cnt) ++{ ++ const u32 msg_id = KBASE_AUX_PAGESALLOC; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(ctx_nr) ++ + sizeof(page_cnt) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &ctx_nr, sizeof(ctx_nr)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &page_cnt, sizeof(page_cnt)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_aux_devfreq_target( ++ struct kbase_tlstream *stream, ++ u64 target_freq) ++{ ++ const u32 msg_id = KBASE_AUX_DEVFREQ_TARGET; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(target_freq) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &target_freq, sizeof(target_freq)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_aux_protected_enter_start( ++ struct kbase_tlstream *stream, ++ const void *gpu) ++{ ++ const u32 msg_id = KBASE_AUX_PROTECTED_ENTER_START; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(gpu) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &gpu, sizeof(gpu)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_aux_protected_enter_end( ++ struct kbase_tlstream *stream, ++ const void *gpu) ++{ ++ const u32 msg_id = KBASE_AUX_PROTECTED_ENTER_END; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(gpu) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &gpu, sizeof(gpu)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_aux_protected_leave_start( ++ struct kbase_tlstream *stream, ++ const void *gpu) ++{ ++ const u32 msg_id = KBASE_AUX_PROTECTED_LEAVE_START; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(gpu) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &gpu, sizeof(gpu)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_aux_protected_leave_end( ++ struct kbase_tlstream *stream, ++ const void *gpu) ++{ ++ const u32 msg_id = KBASE_AUX_PROTECTED_LEAVE_END; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(gpu) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &gpu, sizeof(gpu)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_aux_jit_stats( ++ struct kbase_tlstream *stream, ++ u32 ctx_nr, ++ u32 bid, ++ u32 max_allocs, ++ u32 allocs, ++ u32 va_pages, ++ u32 ph_pages) ++{ ++ const u32 msg_id = KBASE_AUX_JIT_STATS; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(ctx_nr) ++ + sizeof(bid) ++ + sizeof(max_allocs) ++ + sizeof(allocs) ++ + sizeof(va_pages) ++ + sizeof(ph_pages) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &ctx_nr, sizeof(ctx_nr)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &bid, sizeof(bid)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &max_allocs, sizeof(max_allocs)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &allocs, sizeof(allocs)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &va_pages, sizeof(va_pages)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &ph_pages, sizeof(ph_pages)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_aux_event_job_slot( ++ struct kbase_tlstream *stream, ++ const void *ctx, ++ u32 slot_nr, ++ u32 atom_nr, ++ u32 event) ++{ ++ const u32 msg_id = KBASE_AUX_EVENT_JOB_SLOT; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(ctx) ++ + sizeof(slot_nr) ++ + sizeof(atom_nr) ++ + sizeof(event) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &ctx, sizeof(ctx)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &slot_nr, sizeof(slot_nr)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &atom_nr, sizeof(atom_nr)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &event, sizeof(event)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_new_device( ++ struct kbase_tlstream *stream, ++ u32 kbase_device_id, ++ u32 kbase_device_gpu_core_count, ++ u32 kbase_device_max_num_csgs, ++ u32 kbase_device_as_count) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_NEW_DEVICE; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(kbase_device_id) ++ + sizeof(kbase_device_gpu_core_count) ++ + sizeof(kbase_device_max_num_csgs) ++ + sizeof(kbase_device_as_count) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kbase_device_id, sizeof(kbase_device_id)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kbase_device_gpu_core_count, sizeof(kbase_device_gpu_core_count)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kbase_device_max_num_csgs, sizeof(kbase_device_max_num_csgs)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kbase_device_as_count, sizeof(kbase_device_as_count)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_device_program_csg( ++ struct kbase_tlstream *stream, ++ u32 kbase_device_id, ++ u32 gpu_cmdq_grp_handle, ++ u32 kbase_device_csg_slot_index) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_DEVICE_PROGRAM_CSG; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(kbase_device_id) ++ + sizeof(gpu_cmdq_grp_handle) ++ + sizeof(kbase_device_csg_slot_index) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kbase_device_id, sizeof(kbase_device_id)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &gpu_cmdq_grp_handle, sizeof(gpu_cmdq_grp_handle)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kbase_device_csg_slot_index, sizeof(kbase_device_csg_slot_index)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_device_deprogram_csg( ++ struct kbase_tlstream *stream, ++ u32 kbase_device_id, ++ u32 kbase_device_csg_slot_index) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_DEVICE_DEPROGRAM_CSG; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(kbase_device_id) ++ + sizeof(kbase_device_csg_slot_index) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kbase_device_id, sizeof(kbase_device_id)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kbase_device_csg_slot_index, sizeof(kbase_device_csg_slot_index)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_new_ctx( ++ struct kbase_tlstream *stream, ++ u32 kernel_ctx_id, ++ u32 kbase_device_id) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_NEW_CTX; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(kernel_ctx_id) ++ + sizeof(kbase_device_id) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kernel_ctx_id, sizeof(kernel_ctx_id)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kbase_device_id, sizeof(kbase_device_id)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_del_ctx( ++ struct kbase_tlstream *stream, ++ u32 kernel_ctx_id) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_DEL_CTX; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(kernel_ctx_id) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kernel_ctx_id, sizeof(kernel_ctx_id)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_ctx_assign_as( ++ struct kbase_tlstream *stream, ++ u32 kernel_ctx_id, ++ u32 kbase_device_as_index) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_CTX_ASSIGN_AS; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(kernel_ctx_id) ++ + sizeof(kbase_device_as_index) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kernel_ctx_id, sizeof(kernel_ctx_id)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kbase_device_as_index, sizeof(kbase_device_as_index)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_ctx_unassign_as( ++ struct kbase_tlstream *stream, ++ u32 kernel_ctx_id) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_CTX_UNASSIGN_AS; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(kernel_ctx_id) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kernel_ctx_id, sizeof(kernel_ctx_id)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_new_kcpuqueue( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue, ++ u32 kernel_ctx_id, ++ u32 kcpuq_num_pending_cmds) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_NEW_KCPUQUEUE; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(kcpu_queue) ++ + sizeof(kernel_ctx_id) ++ + sizeof(kcpuq_num_pending_cmds) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kcpu_queue, sizeof(kcpu_queue)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kernel_ctx_id, sizeof(kernel_ctx_id)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kcpuq_num_pending_cmds, sizeof(kcpuq_num_pending_cmds)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_del_kcpuqueue( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_DEL_KCPUQUEUE; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(kcpu_queue) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kcpu_queue, sizeof(kcpu_queue)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_fence_signal( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue, ++ const void *fence) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_SIGNAL; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(kcpu_queue) ++ + sizeof(fence) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kcpu_queue, sizeof(kcpu_queue)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &fence, sizeof(fence)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_fence_wait( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue, ++ const void *fence) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_WAIT; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(kcpu_queue) ++ + sizeof(fence) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kcpu_queue, sizeof(kcpu_queue)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &fence, sizeof(fence)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_cqs_wait( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue, ++ u64 cqs_obj_gpu_addr, ++ u32 cqs_obj_compare_value) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_WAIT; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(kcpu_queue) ++ + sizeof(cqs_obj_gpu_addr) ++ + sizeof(cqs_obj_compare_value) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kcpu_queue, sizeof(kcpu_queue)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &cqs_obj_gpu_addr, sizeof(cqs_obj_gpu_addr)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &cqs_obj_compare_value, sizeof(cqs_obj_compare_value)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_cqs_set( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue, ++ u64 cqs_obj_gpu_addr) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_SET; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(kcpu_queue) ++ + sizeof(cqs_obj_gpu_addr) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kcpu_queue, sizeof(kcpu_queue)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &cqs_obj_gpu_addr, sizeof(cqs_obj_gpu_addr)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_map_import( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue, ++ u64 map_import_buf_gpu_addr) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_MAP_IMPORT; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(kcpu_queue) ++ + sizeof(map_import_buf_gpu_addr) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kcpu_queue, sizeof(kcpu_queue)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &map_import_buf_gpu_addr, sizeof(map_import_buf_gpu_addr)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_unmap_import( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue, ++ u64 map_import_buf_gpu_addr) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(kcpu_queue) ++ + sizeof(map_import_buf_gpu_addr) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kcpu_queue, sizeof(kcpu_queue)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &map_import_buf_gpu_addr, sizeof(map_import_buf_gpu_addr)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_unmap_import_force( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue, ++ u64 map_import_buf_gpu_addr) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT_FORCE; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(kcpu_queue) ++ + sizeof(map_import_buf_gpu_addr) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kcpu_queue, sizeof(kcpu_queue)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &map_import_buf_gpu_addr, sizeof(map_import_buf_gpu_addr)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_enqueue_jit_alloc( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_ALLOC; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(kcpu_queue) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kcpu_queue, sizeof(kcpu_queue)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_array_item_kcpuqueue_enqueue_jit_alloc( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue, ++ u64 jit_alloc_gpu_alloc_addr_dest, ++ u64 jit_alloc_va_pages, ++ u64 jit_alloc_commit_pages, ++ u64 jit_alloc_extent, ++ u32 jit_alloc_jit_id, ++ u32 jit_alloc_bin_id, ++ u32 jit_alloc_max_allocations, ++ u32 jit_alloc_flags, ++ u32 jit_alloc_usage_id) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_ALLOC; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(kcpu_queue) ++ + sizeof(jit_alloc_gpu_alloc_addr_dest) ++ + sizeof(jit_alloc_va_pages) ++ + sizeof(jit_alloc_commit_pages) ++ + sizeof(jit_alloc_extent) ++ + sizeof(jit_alloc_jit_id) ++ + sizeof(jit_alloc_bin_id) ++ + sizeof(jit_alloc_max_allocations) ++ + sizeof(jit_alloc_flags) ++ + sizeof(jit_alloc_usage_id) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kcpu_queue, sizeof(kcpu_queue)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &jit_alloc_gpu_alloc_addr_dest, sizeof(jit_alloc_gpu_alloc_addr_dest)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &jit_alloc_va_pages, sizeof(jit_alloc_va_pages)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &jit_alloc_commit_pages, sizeof(jit_alloc_commit_pages)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &jit_alloc_extent, sizeof(jit_alloc_extent)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &jit_alloc_jit_id, sizeof(jit_alloc_jit_id)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &jit_alloc_bin_id, sizeof(jit_alloc_bin_id)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &jit_alloc_max_allocations, sizeof(jit_alloc_max_allocations)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &jit_alloc_flags, sizeof(jit_alloc_flags)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &jit_alloc_usage_id, sizeof(jit_alloc_usage_id)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_array_end_kcpuqueue_enqueue_jit_alloc( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_ALLOC; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(kcpu_queue) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kcpu_queue, sizeof(kcpu_queue)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_enqueue_jit_free( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_FREE; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(kcpu_queue) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kcpu_queue, sizeof(kcpu_queue)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_array_item_kcpuqueue_enqueue_jit_free( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue, ++ u32 jit_alloc_jit_id) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_FREE; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(kcpu_queue) ++ + sizeof(jit_alloc_jit_id) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kcpu_queue, sizeof(kcpu_queue)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &jit_alloc_jit_id, sizeof(jit_alloc_jit_id)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_array_end_kcpuqueue_enqueue_jit_free( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_FREE; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(kcpu_queue) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kcpu_queue, sizeof(kcpu_queue)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_signal_start( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_START; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(kcpu_queue) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kcpu_queue, sizeof(kcpu_queue)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_signal_end( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_END; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(kcpu_queue) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kcpu_queue, sizeof(kcpu_queue)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_wait_start( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_START; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(kcpu_queue) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kcpu_queue, sizeof(kcpu_queue)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_wait_end( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_END; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(kcpu_queue) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kcpu_queue, sizeof(kcpu_queue)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_kcpuqueue_execute_cqs_wait_start( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_START; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(kcpu_queue) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kcpu_queue, sizeof(kcpu_queue)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_kcpuqueue_execute_cqs_wait_end( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_END; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(kcpu_queue) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kcpu_queue, sizeof(kcpu_queue)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_kcpuqueue_execute_cqs_set( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_SET; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(kcpu_queue) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kcpu_queue, sizeof(kcpu_queue)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_kcpuqueue_execute_map_import_start( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_START; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(kcpu_queue) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kcpu_queue, sizeof(kcpu_queue)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_kcpuqueue_execute_map_import_end( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_END; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(kcpu_queue) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kcpu_queue, sizeof(kcpu_queue)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_start( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_START; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(kcpu_queue) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kcpu_queue, sizeof(kcpu_queue)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_end( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_END; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(kcpu_queue) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kcpu_queue, sizeof(kcpu_queue)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_force_start( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_START; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(kcpu_queue) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kcpu_queue, sizeof(kcpu_queue)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_force_end( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_END; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(kcpu_queue) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kcpu_queue, sizeof(kcpu_queue)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_kcpuqueue_execute_jit_alloc_start( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_START; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(kcpu_queue) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kcpu_queue, sizeof(kcpu_queue)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_execute_jit_alloc_end( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_ALLOC_END; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(kcpu_queue) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kcpu_queue, sizeof(kcpu_queue)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_array_item_kcpuqueue_execute_jit_alloc_end( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue, ++ u64 jit_alloc_gpu_alloc_addr, ++ u64 jit_alloc_mmu_flags) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_ALLOC_END; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(kcpu_queue) ++ + sizeof(jit_alloc_gpu_alloc_addr) ++ + sizeof(jit_alloc_mmu_flags) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kcpu_queue, sizeof(kcpu_queue)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &jit_alloc_gpu_alloc_addr, sizeof(jit_alloc_gpu_alloc_addr)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &jit_alloc_mmu_flags, sizeof(jit_alloc_mmu_flags)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_array_end_kcpuqueue_execute_jit_alloc_end( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_ALLOC_END; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(kcpu_queue) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kcpu_queue, sizeof(kcpu_queue)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_kcpuqueue_execute_jit_free_start( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_START; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(kcpu_queue) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kcpu_queue, sizeof(kcpu_queue)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_execute_jit_free_end( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_FREE_END; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(kcpu_queue) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kcpu_queue, sizeof(kcpu_queue)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_array_item_kcpuqueue_execute_jit_free_end( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue, ++ u64 jit_free_pages_used) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_FREE_END; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(kcpu_queue) ++ + sizeof(jit_free_pages_used) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kcpu_queue, sizeof(kcpu_queue)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &jit_free_pages_used, sizeof(jit_free_pages_used)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_array_end_kcpuqueue_execute_jit_free_end( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_FREE_END; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(kcpu_queue) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kcpu_queue, sizeof(kcpu_queue)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_kcpuqueue_execute_errorbarrier( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_ERRORBARRIER; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(kcpu_queue) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &kcpu_queue, sizeof(kcpu_queue)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_csffw_tlstream_overflow( ++ struct kbase_tlstream *stream, ++ u64 csffw_timestamp, ++ u64 csffw_cycle) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_CSFFW_TLSTREAM_OVERFLOW; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(csffw_timestamp) ++ + sizeof(csffw_cycle) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &csffw_timestamp, sizeof(csffw_timestamp)); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &csffw_cycle, sizeof(csffw_cycle)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++void __kbase_tlstream_tl_kbase_csffw_reset( ++ struct kbase_tlstream *stream, ++ u64 csffw_cycle) ++{ ++ const u32 msg_id = KBASE_TL_KBASE_CSFFW_RESET; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) ++ + sizeof(csffw_cycle) ++ ; ++ char *buffer; ++ unsigned long acq_flags; ++ size_t pos = 0; ++ ++ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); ++ ++ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_serialize_timestamp(buffer, pos); ++ pos = kbasep_serialize_bytes(buffer, ++ pos, &csffw_cycle, sizeof(csffw_cycle)); ++ ++ kbase_tlstream_msgbuf_release(stream, acq_flags); ++} ++ ++/* clang-format on */ +diff --git a/drivers/gpu/arm/bifrost/tl/mali_kbase_tracepoints.h b/drivers/gpu/arm/bifrost/tl/mali_kbase_tracepoints.h +new file mode 100755 +index 000000000000..5651f0a0fc57 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost/tl/mali_kbase_tracepoints.h +@@ -0,0 +1,2926 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, you can access it online at ++ * http://www.gnu.org/licenses/gpl-2.0.html. ++ * ++ * SPDX-License-Identifier: GPL-2.0 ++ * ++ */ ++ ++/* ++ * THIS FILE IS AUTOGENERATED BY mali_trace_generator.py. ++ * DO NOT EDIT. ++ */ ++ ++#if !defined(_KBASE_TRACEPOINTS_H) ++#define _KBASE_TRACEPOINTS_H ++ ++/* Tracepoints are abstract callbacks notifying that some important ++ * software or hardware event has happened. ++ * ++ * In this particular implementation, it results into a MIPE ++ * timeline event and, in some cases, it also fires an ftrace event ++ * (a.k.a. Gator events, see details below). ++ */ ++ ++#include "mali_kbase.h" ++#include "mali_kbase_gator.h" ++ ++#include ++#include ++ ++/* clang-format off */ ++ ++struct kbase_tlstream; ++ ++extern const size_t __obj_stream_offset; ++extern const size_t __aux_stream_offset; ++ ++/* This macro dispatches a kbase_tlstream from ++ * a kbase_device instance. Only AUX or OBJ ++ * streams can be dispatched. It is aware of ++ * kbase_timeline binary representation and ++ * relies on offset variables: ++ * __obj_stream_offset and __aux_stream_offset. ++ */ ++#define __TL_DISPATCH_STREAM(kbdev, stype) \ ++ ((struct kbase_tlstream *) \ ++ ((u8 *)kbdev->timeline + __ ## stype ## _stream_offset)) ++ ++struct tp_desc; ++ ++/* Descriptors of timeline messages transmitted in object events stream. */ ++extern const char *obj_desc_header; ++extern const size_t obj_desc_header_size; ++/* Descriptors of timeline messages transmitted in auxiliary events stream. */ ++extern const char *aux_desc_header; ++extern const size_t aux_desc_header_size; ++ ++#define TL_ATOM_STATE_IDLE 0 ++#define TL_ATOM_STATE_READY 1 ++#define TL_ATOM_STATE_DONE 2 ++#define TL_ATOM_STATE_POSTED 3 ++ ++#define TL_JS_EVENT_START GATOR_JOB_SLOT_START ++#define TL_JS_EVENT_STOP GATOR_JOB_SLOT_STOP ++#define TL_JS_EVENT_SOFT_STOP GATOR_JOB_SLOT_SOFT_STOPPED ++ ++#define TLSTREAM_ENABLED (1 << 31) ++ ++void __kbase_tlstream_tl_new_ctx( ++ struct kbase_tlstream *stream, ++ const void *ctx, ++ u32 ctx_nr, ++ u32 tgid); ++void __kbase_tlstream_tl_new_gpu( ++ struct kbase_tlstream *stream, ++ const void *gpu, ++ u32 gpu_id, ++ u32 core_count); ++void __kbase_tlstream_tl_new_lpu( ++ struct kbase_tlstream *stream, ++ const void *lpu, ++ u32 lpu_nr, ++ u32 lpu_fn); ++void __kbase_tlstream_tl_new_atom( ++ struct kbase_tlstream *stream, ++ const void *atom, ++ u32 atom_nr); ++void __kbase_tlstream_tl_new_as( ++ struct kbase_tlstream *stream, ++ const void *address_space, ++ u32 as_nr); ++void __kbase_tlstream_tl_del_ctx( ++ struct kbase_tlstream *stream, ++ const void *ctx); ++void __kbase_tlstream_tl_del_atom( ++ struct kbase_tlstream *stream, ++ const void *atom); ++void __kbase_tlstream_tl_lifelink_lpu_gpu( ++ struct kbase_tlstream *stream, ++ const void *lpu, ++ const void *gpu); ++void __kbase_tlstream_tl_lifelink_as_gpu( ++ struct kbase_tlstream *stream, ++ const void *address_space, ++ const void *gpu); ++void __kbase_tlstream_tl_ret_ctx_lpu( ++ struct kbase_tlstream *stream, ++ const void *ctx, ++ const void *lpu); ++void __kbase_tlstream_tl_ret_atom_ctx( ++ struct kbase_tlstream *stream, ++ const void *atom, ++ const void *ctx); ++void __kbase_tlstream_tl_ret_atom_lpu( ++ struct kbase_tlstream *stream, ++ const void *atom, ++ const void *lpu, ++ const char *attrib_match_list); ++void __kbase_tlstream_tl_nret_ctx_lpu( ++ struct kbase_tlstream *stream, ++ const void *ctx, ++ const void *lpu); ++void __kbase_tlstream_tl_nret_atom_ctx( ++ struct kbase_tlstream *stream, ++ const void *atom, ++ const void *ctx); ++void __kbase_tlstream_tl_nret_atom_lpu( ++ struct kbase_tlstream *stream, ++ const void *atom, ++ const void *lpu); ++void __kbase_tlstream_tl_ret_as_ctx( ++ struct kbase_tlstream *stream, ++ const void *address_space, ++ const void *ctx); ++void __kbase_tlstream_tl_nret_as_ctx( ++ struct kbase_tlstream *stream, ++ const void *address_space, ++ const void *ctx); ++void __kbase_tlstream_tl_ret_atom_as( ++ struct kbase_tlstream *stream, ++ const void *atom, ++ const void *address_space); ++void __kbase_tlstream_tl_nret_atom_as( ++ struct kbase_tlstream *stream, ++ const void *atom, ++ const void *address_space); ++void __kbase_tlstream_tl_attrib_atom_config( ++ struct kbase_tlstream *stream, ++ const void *atom, ++ u64 descriptor, ++ u64 affinity, ++ u32 config); ++void __kbase_tlstream_tl_attrib_atom_priority( ++ struct kbase_tlstream *stream, ++ const void *atom, ++ u32 prio); ++void __kbase_tlstream_tl_attrib_atom_state( ++ struct kbase_tlstream *stream, ++ const void *atom, ++ u32 state); ++void __kbase_tlstream_tl_attrib_atom_prioritized( ++ struct kbase_tlstream *stream, ++ const void *atom); ++void __kbase_tlstream_tl_attrib_atom_jit( ++ struct kbase_tlstream *stream, ++ const void *atom, ++ u64 edit_addr, ++ u64 new_addr, ++ u32 jit_flags, ++ u64 mem_flags, ++ u32 j_id, ++ u64 com_pgs, ++ u64 extent, ++ u64 va_pgs); ++void __kbase_tlstream_tl_jit_usedpages( ++ struct kbase_tlstream *stream, ++ u64 used_pages, ++ u32 j_id); ++void __kbase_tlstream_tl_attrib_atom_jitallocinfo( ++ struct kbase_tlstream *stream, ++ const void *atom, ++ u64 va_pgs, ++ u64 com_pgs, ++ u64 extent, ++ u32 j_id, ++ u32 bin_id, ++ u32 max_allocs, ++ u32 jit_flags, ++ u32 usg_id); ++void __kbase_tlstream_tl_attrib_atom_jitfreeinfo( ++ struct kbase_tlstream *stream, ++ const void *atom, ++ u32 j_id); ++void __kbase_tlstream_tl_attrib_as_config( ++ struct kbase_tlstream *stream, ++ const void *address_space, ++ u64 transtab, ++ u64 memattr, ++ u64 transcfg); ++void __kbase_tlstream_tl_event_lpu_softstop( ++ struct kbase_tlstream *stream, ++ const void *lpu); ++void __kbase_tlstream_tl_event_atom_softstop_ex( ++ struct kbase_tlstream *stream, ++ const void *atom); ++void __kbase_tlstream_tl_event_atom_softstop_issue( ++ struct kbase_tlstream *stream, ++ const void *atom); ++void __kbase_tlstream_tl_event_atom_softjob_start( ++ struct kbase_tlstream *stream, ++ const void *atom); ++void __kbase_tlstream_tl_event_atom_softjob_end( ++ struct kbase_tlstream *stream, ++ const void *atom); ++void __kbase_tlstream_tl_arbiter_granted( ++ struct kbase_tlstream *stream, ++ const void *gpu); ++void __kbase_tlstream_tl_arbiter_started( ++ struct kbase_tlstream *stream, ++ const void *gpu); ++void __kbase_tlstream_tl_arbiter_stop_requested( ++ struct kbase_tlstream *stream, ++ const void *gpu); ++void __kbase_tlstream_tl_arbiter_stopped( ++ struct kbase_tlstream *stream, ++ const void *gpu); ++void __kbase_tlstream_jd_gpu_soft_reset( ++ struct kbase_tlstream *stream, ++ const void *gpu); ++void __kbase_tlstream_aux_pm_state( ++ struct kbase_tlstream *stream, ++ u32 core_type, ++ u64 core_state_bitset); ++void __kbase_tlstream_aux_pagefault( ++ struct kbase_tlstream *stream, ++ u32 ctx_nr, ++ u32 as_nr, ++ u64 page_cnt_change); ++void __kbase_tlstream_aux_pagesalloc( ++ struct kbase_tlstream *stream, ++ u32 ctx_nr, ++ u64 page_cnt); ++void __kbase_tlstream_aux_devfreq_target( ++ struct kbase_tlstream *stream, ++ u64 target_freq); ++void __kbase_tlstream_aux_protected_enter_start( ++ struct kbase_tlstream *stream, ++ const void *gpu); ++void __kbase_tlstream_aux_protected_enter_end( ++ struct kbase_tlstream *stream, ++ const void *gpu); ++void __kbase_tlstream_aux_protected_leave_start( ++ struct kbase_tlstream *stream, ++ const void *gpu); ++void __kbase_tlstream_aux_protected_leave_end( ++ struct kbase_tlstream *stream, ++ const void *gpu); ++void __kbase_tlstream_aux_jit_stats( ++ struct kbase_tlstream *stream, ++ u32 ctx_nr, ++ u32 bid, ++ u32 max_allocs, ++ u32 allocs, ++ u32 va_pages, ++ u32 ph_pages); ++void __kbase_tlstream_aux_event_job_slot( ++ struct kbase_tlstream *stream, ++ const void *ctx, ++ u32 slot_nr, ++ u32 atom_nr, ++ u32 event); ++void __kbase_tlstream_tl_kbase_new_device( ++ struct kbase_tlstream *stream, ++ u32 kbase_device_id, ++ u32 kbase_device_gpu_core_count, ++ u32 kbase_device_max_num_csgs, ++ u32 kbase_device_as_count); ++void __kbase_tlstream_tl_kbase_device_program_csg( ++ struct kbase_tlstream *stream, ++ u32 kbase_device_id, ++ u32 gpu_cmdq_grp_handle, ++ u32 kbase_device_csg_slot_index); ++void __kbase_tlstream_tl_kbase_device_deprogram_csg( ++ struct kbase_tlstream *stream, ++ u32 kbase_device_id, ++ u32 kbase_device_csg_slot_index); ++void __kbase_tlstream_tl_kbase_new_ctx( ++ struct kbase_tlstream *stream, ++ u32 kernel_ctx_id, ++ u32 kbase_device_id); ++void __kbase_tlstream_tl_kbase_del_ctx( ++ struct kbase_tlstream *stream, ++ u32 kernel_ctx_id); ++void __kbase_tlstream_tl_kbase_ctx_assign_as( ++ struct kbase_tlstream *stream, ++ u32 kernel_ctx_id, ++ u32 kbase_device_as_index); ++void __kbase_tlstream_tl_kbase_ctx_unassign_as( ++ struct kbase_tlstream *stream, ++ u32 kernel_ctx_id); ++void __kbase_tlstream_tl_kbase_new_kcpuqueue( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue, ++ u32 kernel_ctx_id, ++ u32 kcpuq_num_pending_cmds); ++void __kbase_tlstream_tl_kbase_del_kcpuqueue( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue); ++void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_fence_signal( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue, ++ const void *fence); ++void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_fence_wait( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue, ++ const void *fence); ++void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_cqs_wait( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue, ++ u64 cqs_obj_gpu_addr, ++ u32 cqs_obj_compare_value); ++void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_cqs_set( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue, ++ u64 cqs_obj_gpu_addr); ++void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_map_import( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue, ++ u64 map_import_buf_gpu_addr); ++void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_unmap_import( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue, ++ u64 map_import_buf_gpu_addr); ++void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_unmap_import_force( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue, ++ u64 map_import_buf_gpu_addr); ++void __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_enqueue_jit_alloc( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue); ++void __kbase_tlstream_tl_kbase_array_item_kcpuqueue_enqueue_jit_alloc( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue, ++ u64 jit_alloc_gpu_alloc_addr_dest, ++ u64 jit_alloc_va_pages, ++ u64 jit_alloc_commit_pages, ++ u64 jit_alloc_extent, ++ u32 jit_alloc_jit_id, ++ u32 jit_alloc_bin_id, ++ u32 jit_alloc_max_allocations, ++ u32 jit_alloc_flags, ++ u32 jit_alloc_usage_id); ++void __kbase_tlstream_tl_kbase_array_end_kcpuqueue_enqueue_jit_alloc( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue); ++void __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_enqueue_jit_free( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue); ++void __kbase_tlstream_tl_kbase_array_item_kcpuqueue_enqueue_jit_free( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue, ++ u32 jit_alloc_jit_id); ++void __kbase_tlstream_tl_kbase_array_end_kcpuqueue_enqueue_jit_free( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue); ++void __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_signal_start( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue); ++void __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_signal_end( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue); ++void __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_wait_start( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue); ++void __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_wait_end( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue); ++void __kbase_tlstream_tl_kbase_kcpuqueue_execute_cqs_wait_start( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue); ++void __kbase_tlstream_tl_kbase_kcpuqueue_execute_cqs_wait_end( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue); ++void __kbase_tlstream_tl_kbase_kcpuqueue_execute_cqs_set( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue); ++void __kbase_tlstream_tl_kbase_kcpuqueue_execute_map_import_start( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue); ++void __kbase_tlstream_tl_kbase_kcpuqueue_execute_map_import_end( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue); ++void __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_start( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue); ++void __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_end( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue); ++void __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_force_start( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue); ++void __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_force_end( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue); ++void __kbase_tlstream_tl_kbase_kcpuqueue_execute_jit_alloc_start( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue); ++void __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_execute_jit_alloc_end( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue); ++void __kbase_tlstream_tl_kbase_array_item_kcpuqueue_execute_jit_alloc_end( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue, ++ u64 jit_alloc_gpu_alloc_addr, ++ u64 jit_alloc_mmu_flags); ++void __kbase_tlstream_tl_kbase_array_end_kcpuqueue_execute_jit_alloc_end( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue); ++void __kbase_tlstream_tl_kbase_kcpuqueue_execute_jit_free_start( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue); ++void __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_execute_jit_free_end( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue); ++void __kbase_tlstream_tl_kbase_array_item_kcpuqueue_execute_jit_free_end( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue, ++ u64 jit_free_pages_used); ++void __kbase_tlstream_tl_kbase_array_end_kcpuqueue_execute_jit_free_end( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue); ++void __kbase_tlstream_tl_kbase_kcpuqueue_execute_errorbarrier( ++ struct kbase_tlstream *stream, ++ const void *kcpu_queue); ++void __kbase_tlstream_tl_kbase_csffw_tlstream_overflow( ++ struct kbase_tlstream *stream, ++ u64 csffw_timestamp, ++ u64 csffw_cycle); ++void __kbase_tlstream_tl_kbase_csffw_reset( ++ struct kbase_tlstream *stream, ++ u64 csffw_cycle); ++ ++struct kbase_tlstream; ++ ++/** ++ * KBASE_TLSTREAM_TL_NEW_CTX - ++ * object ctx is created ++ * ++ * @kbdev: Kbase device ++ * @ctx: Name of the context object ++ * @ctx_nr: Kernel context number ++ * @tgid: Thread Group Id ++ */ ++#define KBASE_TLSTREAM_TL_NEW_CTX( \ ++ kbdev, \ ++ ctx, \ ++ ctx_nr, \ ++ tgid \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_tl_new_ctx( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ ctx, ctx_nr, tgid); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_TL_NEW_GPU - ++ * object gpu is created ++ * ++ * @kbdev: Kbase device ++ * @gpu: Name of the GPU object ++ * @gpu_id: Name of the GPU object ++ * @core_count: Number of cores this GPU hosts ++ */ ++#define KBASE_TLSTREAM_TL_NEW_GPU( \ ++ kbdev, \ ++ gpu, \ ++ gpu_id, \ ++ core_count \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_tl_new_gpu( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ gpu, gpu_id, core_count); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_TL_NEW_LPU - ++ * object lpu is created ++ * ++ * @kbdev: Kbase device ++ * @lpu: Name of the Logical Processing Unit object ++ * @lpu_nr: Sequential number assigned to the newly created LPU ++ * @lpu_fn: Property describing functional abilities of this LPU ++ */ ++#define KBASE_TLSTREAM_TL_NEW_LPU( \ ++ kbdev, \ ++ lpu, \ ++ lpu_nr, \ ++ lpu_fn \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_tl_new_lpu( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ lpu, lpu_nr, lpu_fn); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_TL_NEW_ATOM - ++ * object atom is created ++ * ++ * @kbdev: Kbase device ++ * @atom: Atom identifier ++ * @atom_nr: Sequential number of an atom ++ */ ++#define KBASE_TLSTREAM_TL_NEW_ATOM( \ ++ kbdev, \ ++ atom, \ ++ atom_nr \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_tl_new_atom( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ atom, atom_nr); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_TL_NEW_AS - ++ * address space object is created ++ * ++ * @kbdev: Kbase device ++ * @address_space: Name of the address space object ++ * @as_nr: Address space number ++ */ ++#define KBASE_TLSTREAM_TL_NEW_AS( \ ++ kbdev, \ ++ address_space, \ ++ as_nr \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_tl_new_as( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ address_space, as_nr); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_TL_DEL_CTX - ++ * context is destroyed ++ * ++ * @kbdev: Kbase device ++ * @ctx: Name of the context object ++ */ ++#define KBASE_TLSTREAM_TL_DEL_CTX( \ ++ kbdev, \ ++ ctx \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_tl_del_ctx( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ ctx); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_TL_DEL_ATOM - ++ * atom is destroyed ++ * ++ * @kbdev: Kbase device ++ * @atom: Atom identifier ++ */ ++#define KBASE_TLSTREAM_TL_DEL_ATOM( \ ++ kbdev, \ ++ atom \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_tl_del_atom( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ atom); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_TL_LIFELINK_LPU_GPU - ++ * lpu is deleted with gpu ++ * ++ * @kbdev: Kbase device ++ * @lpu: Name of the Logical Processing Unit object ++ * @gpu: Name of the GPU object ++ */ ++#define KBASE_TLSTREAM_TL_LIFELINK_LPU_GPU( \ ++ kbdev, \ ++ lpu, \ ++ gpu \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_tl_lifelink_lpu_gpu( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ lpu, gpu); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_TL_LIFELINK_AS_GPU - ++ * address space is deleted with gpu ++ * ++ * @kbdev: Kbase device ++ * @address_space: Name of the address space object ++ * @gpu: Name of the GPU object ++ */ ++#define KBASE_TLSTREAM_TL_LIFELINK_AS_GPU( \ ++ kbdev, \ ++ address_space, \ ++ gpu \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_tl_lifelink_as_gpu( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ address_space, gpu); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_TL_RET_CTX_LPU - ++ * context is retained by lpu ++ * ++ * @kbdev: Kbase device ++ * @ctx: Name of the context object ++ * @lpu: Name of the Logical Processing Unit object ++ */ ++#define KBASE_TLSTREAM_TL_RET_CTX_LPU( \ ++ kbdev, \ ++ ctx, \ ++ lpu \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_tl_ret_ctx_lpu( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ ctx, lpu); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_TL_RET_ATOM_CTX - ++ * atom is retained by context ++ * ++ * @kbdev: Kbase device ++ * @atom: Atom identifier ++ * @ctx: Name of the context object ++ */ ++#define KBASE_TLSTREAM_TL_RET_ATOM_CTX( \ ++ kbdev, \ ++ atom, \ ++ ctx \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_tl_ret_atom_ctx( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ atom, ctx); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_TL_RET_ATOM_LPU - ++ * atom is retained by lpu ++ * ++ * @kbdev: Kbase device ++ * @atom: Atom identifier ++ * @lpu: Name of the Logical Processing Unit object ++ * @attrib_match_list: List containing match operator attributes ++ */ ++#define KBASE_TLSTREAM_TL_RET_ATOM_LPU( \ ++ kbdev, \ ++ atom, \ ++ lpu, \ ++ attrib_match_list \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_tl_ret_atom_lpu( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ atom, lpu, attrib_match_list); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_TL_NRET_CTX_LPU - ++ * context is released by lpu ++ * ++ * @kbdev: Kbase device ++ * @ctx: Name of the context object ++ * @lpu: Name of the Logical Processing Unit object ++ */ ++#define KBASE_TLSTREAM_TL_NRET_CTX_LPU( \ ++ kbdev, \ ++ ctx, \ ++ lpu \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_tl_nret_ctx_lpu( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ ctx, lpu); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_TL_NRET_ATOM_CTX - ++ * atom is released by context ++ * ++ * @kbdev: Kbase device ++ * @atom: Atom identifier ++ * @ctx: Name of the context object ++ */ ++#define KBASE_TLSTREAM_TL_NRET_ATOM_CTX( \ ++ kbdev, \ ++ atom, \ ++ ctx \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_tl_nret_atom_ctx( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ atom, ctx); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_TL_NRET_ATOM_LPU - ++ * atom is released by lpu ++ * ++ * @kbdev: Kbase device ++ * @atom: Atom identifier ++ * @lpu: Name of the Logical Processing Unit object ++ */ ++#define KBASE_TLSTREAM_TL_NRET_ATOM_LPU( \ ++ kbdev, \ ++ atom, \ ++ lpu \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_tl_nret_atom_lpu( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ atom, lpu); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_TL_RET_AS_CTX - ++ * address space is retained by context ++ * ++ * @kbdev: Kbase device ++ * @address_space: Name of the address space object ++ * @ctx: Name of the context object ++ */ ++#define KBASE_TLSTREAM_TL_RET_AS_CTX( \ ++ kbdev, \ ++ address_space, \ ++ ctx \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_tl_ret_as_ctx( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ address_space, ctx); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_TL_NRET_AS_CTX - ++ * address space is released by context ++ * ++ * @kbdev: Kbase device ++ * @address_space: Name of the address space object ++ * @ctx: Name of the context object ++ */ ++#define KBASE_TLSTREAM_TL_NRET_AS_CTX( \ ++ kbdev, \ ++ address_space, \ ++ ctx \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_tl_nret_as_ctx( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ address_space, ctx); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_TL_RET_ATOM_AS - ++ * atom is retained by address space ++ * ++ * @kbdev: Kbase device ++ * @atom: Atom identifier ++ * @address_space: Name of the address space object ++ */ ++#define KBASE_TLSTREAM_TL_RET_ATOM_AS( \ ++ kbdev, \ ++ atom, \ ++ address_space \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_tl_ret_atom_as( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ atom, address_space); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_TL_NRET_ATOM_AS - ++ * atom is released by address space ++ * ++ * @kbdev: Kbase device ++ * @atom: Atom identifier ++ * @address_space: Name of the address space object ++ */ ++#define KBASE_TLSTREAM_TL_NRET_ATOM_AS( \ ++ kbdev, \ ++ atom, \ ++ address_space \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_tl_nret_atom_as( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ atom, address_space); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_TL_ATTRIB_ATOM_CONFIG - ++ * atom job slot attributes ++ * ++ * @kbdev: Kbase device ++ * @atom: Atom identifier ++ * @descriptor: Job descriptor address ++ * @affinity: Job affinity ++ * @config: Job config ++ */ ++#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_CONFIG( \ ++ kbdev, \ ++ atom, \ ++ descriptor, \ ++ affinity, \ ++ config \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_tl_attrib_atom_config( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ atom, descriptor, affinity, config); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY - ++ * atom priority ++ * ++ * @kbdev: Kbase device ++ * @atom: Atom identifier ++ * @prio: Atom priority ++ */ ++#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY( \ ++ kbdev, \ ++ atom, \ ++ prio \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS) \ ++ __kbase_tlstream_tl_attrib_atom_priority( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ atom, prio); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE - ++ * atom state ++ * ++ * @kbdev: Kbase device ++ * @atom: Atom identifier ++ * @state: Atom state ++ */ ++#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE( \ ++ kbdev, \ ++ atom, \ ++ state \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS) \ ++ __kbase_tlstream_tl_attrib_atom_state( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ atom, state); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITIZED - ++ * atom caused priority change ++ * ++ * @kbdev: Kbase device ++ * @atom: Atom identifier ++ */ ++#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITIZED( \ ++ kbdev, \ ++ atom \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS) \ ++ __kbase_tlstream_tl_attrib_atom_prioritized( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ atom); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_TL_ATTRIB_ATOM_JIT - ++ * jit done for atom ++ * ++ * @kbdev: Kbase device ++ * @atom: Atom identifier ++ * @edit_addr: Address edited by jit ++ * @new_addr: Address placed into the edited location ++ * @jit_flags: Flags specifying the special requirements for ++ * the JIT allocation. ++ * @mem_flags: Flags defining the properties of a memory region ++ * @j_id: Unique ID provided by the caller, this is used ++ * to pair allocation and free requests. ++ * @com_pgs: The minimum number of physical pages which ++ * should back the allocation. ++ * @extent: Granularity of physical pages to grow the ++ * allocation by during a fault. ++ * @va_pgs: The minimum number of virtual pages required ++ */ ++#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_JIT( \ ++ kbdev, \ ++ atom, \ ++ edit_addr, \ ++ new_addr, \ ++ jit_flags, \ ++ mem_flags, \ ++ j_id, \ ++ com_pgs, \ ++ extent, \ ++ va_pgs \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_JOB_DUMPING_ENABLED) \ ++ __kbase_tlstream_tl_attrib_atom_jit( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ atom, edit_addr, new_addr, jit_flags, mem_flags, j_id, com_pgs, extent, va_pgs); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_TL_JIT_USEDPAGES - ++ * used pages for jit ++ * ++ * @kbdev: Kbase device ++ * @used_pages: Number of pages used for jit ++ * @j_id: Unique ID provided by the caller, this is used ++ * to pair allocation and free requests. ++ */ ++#define KBASE_TLSTREAM_TL_JIT_USEDPAGES( \ ++ kbdev, \ ++ used_pages, \ ++ j_id \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_tl_jit_usedpages( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ used_pages, j_id); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_TL_ATTRIB_ATOM_JITALLOCINFO - ++ * Information about JIT allocations ++ * ++ * @kbdev: Kbase device ++ * @atom: Atom identifier ++ * @va_pgs: The minimum number of virtual pages required ++ * @com_pgs: The minimum number of physical pages which ++ * should back the allocation. ++ * @extent: Granularity of physical pages to grow the ++ * allocation by during a fault. ++ * @j_id: Unique ID provided by the caller, this is used ++ * to pair allocation and free requests. ++ * @bin_id: The JIT allocation bin, used in conjunction with ++ * max_allocations to limit the number of each ++ * type of JIT allocation. ++ * @max_allocs: Maximum allocations allowed in this bin. ++ * @jit_flags: Flags specifying the special requirements for ++ * the JIT allocation. ++ * @usg_id: A hint about which allocation should be reused. ++ */ ++#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_JITALLOCINFO( \ ++ kbdev, \ ++ atom, \ ++ va_pgs, \ ++ com_pgs, \ ++ extent, \ ++ j_id, \ ++ bin_id, \ ++ max_allocs, \ ++ jit_flags, \ ++ usg_id \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_tl_attrib_atom_jitallocinfo( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ atom, va_pgs, com_pgs, extent, j_id, bin_id, max_allocs, jit_flags, usg_id); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_TL_ATTRIB_ATOM_JITFREEINFO - ++ * Information about JIT frees ++ * ++ * @kbdev: Kbase device ++ * @atom: Atom identifier ++ * @j_id: Unique ID provided by the caller, this is used ++ * to pair allocation and free requests. ++ */ ++#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_JITFREEINFO( \ ++ kbdev, \ ++ atom, \ ++ j_id \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_tl_attrib_atom_jitfreeinfo( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ atom, j_id); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_TL_ATTRIB_AS_CONFIG - ++ * address space attributes ++ * ++ * @kbdev: Kbase device ++ * @address_space: Name of the address space object ++ * @transtab: Configuration of the TRANSTAB register ++ * @memattr: Configuration of the MEMATTR register ++ * @transcfg: Configuration of the TRANSCFG register (or zero if not present) ++ */ ++#define KBASE_TLSTREAM_TL_ATTRIB_AS_CONFIG( \ ++ kbdev, \ ++ address_space, \ ++ transtab, \ ++ memattr, \ ++ transcfg \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_tl_attrib_as_config( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ address_space, transtab, memattr, transcfg); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_TL_EVENT_LPU_SOFTSTOP - ++ * softstop event on given lpu ++ * ++ * @kbdev: Kbase device ++ * @lpu: Name of the Logical Processing Unit object ++ */ ++#define KBASE_TLSTREAM_TL_EVENT_LPU_SOFTSTOP( \ ++ kbdev, \ ++ lpu \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_tl_event_lpu_softstop( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ lpu); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_EX - ++ * atom softstopped ++ * ++ * @kbdev: Kbase device ++ * @atom: Atom identifier ++ */ ++#define KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_EX( \ ++ kbdev, \ ++ atom \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_tl_event_atom_softstop_ex( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ atom); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_ISSUE - ++ * atom softstop issued ++ * ++ * @kbdev: Kbase device ++ * @atom: Atom identifier ++ */ ++#define KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_ISSUE( \ ++ kbdev, \ ++ atom \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_tl_event_atom_softstop_issue( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ atom); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTJOB_START - ++ * atom soft job has started ++ * ++ * @kbdev: Kbase device ++ * @atom: Atom identifier ++ */ ++#define KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTJOB_START( \ ++ kbdev, \ ++ atom \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_tl_event_atom_softjob_start( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ atom); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTJOB_END - ++ * atom soft job has completed ++ * ++ * @kbdev: Kbase device ++ * @atom: Atom identifier ++ */ ++#define KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTJOB_END( \ ++ kbdev, \ ++ atom \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_tl_event_atom_softjob_end( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ atom); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_TL_ARBITER_GRANTED - ++ * Arbiter has granted gpu access ++ * ++ * @kbdev: Kbase device ++ * @gpu: Name of the GPU object ++ */ ++#define KBASE_TLSTREAM_TL_ARBITER_GRANTED( \ ++ kbdev, \ ++ gpu \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_tl_arbiter_granted( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ gpu); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_TL_ARBITER_STARTED - ++ * Driver is running again and able to process jobs ++ * ++ * @kbdev: Kbase device ++ * @gpu: Name of the GPU object ++ */ ++#define KBASE_TLSTREAM_TL_ARBITER_STARTED( \ ++ kbdev, \ ++ gpu \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_tl_arbiter_started( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ gpu); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_TL_ARBITER_STOP_REQUESTED - ++ * Arbiter has requested driver to stop using gpu ++ * ++ * @kbdev: Kbase device ++ * @gpu: Name of the GPU object ++ */ ++#define KBASE_TLSTREAM_TL_ARBITER_STOP_REQUESTED( \ ++ kbdev, \ ++ gpu \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_tl_arbiter_stop_requested( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ gpu); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_TL_ARBITER_STOPPED - ++ * Driver has stopped using gpu ++ * ++ * @kbdev: Kbase device ++ * @gpu: Name of the GPU object ++ */ ++#define KBASE_TLSTREAM_TL_ARBITER_STOPPED( \ ++ kbdev, \ ++ gpu \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_tl_arbiter_stopped( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ gpu); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_JD_GPU_SOFT_RESET - ++ * gpu soft reset ++ * ++ * @kbdev: Kbase device ++ * @gpu: Name of the GPU object ++ */ ++#define KBASE_TLSTREAM_JD_GPU_SOFT_RESET( \ ++ kbdev, \ ++ gpu \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_jd_gpu_soft_reset( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ gpu); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_AUX_PM_STATE - ++ * PM state ++ * ++ * @kbdev: Kbase device ++ * @core_type: Core type (shader, tiler, l2 cache, l3 cache) ++ * @core_state_bitset: 64bits bitmask reporting power state of the cores ++ * (1-ON, 0-OFF) ++ */ ++#define KBASE_TLSTREAM_AUX_PM_STATE( \ ++ kbdev, \ ++ core_type, \ ++ core_state_bitset \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_aux_pm_state( \ ++ __TL_DISPATCH_STREAM(kbdev, aux), \ ++ core_type, core_state_bitset); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_AUX_PAGEFAULT - ++ * Page fault ++ * ++ * @kbdev: Kbase device ++ * @ctx_nr: Kernel context number ++ * @as_nr: Address space number ++ * @page_cnt_change: Number of pages to be added ++ */ ++#define KBASE_TLSTREAM_AUX_PAGEFAULT( \ ++ kbdev, \ ++ ctx_nr, \ ++ as_nr, \ ++ page_cnt_change \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_aux_pagefault( \ ++ __TL_DISPATCH_STREAM(kbdev, aux), \ ++ ctx_nr, as_nr, page_cnt_change); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_AUX_PAGESALLOC - ++ * Total alloc pages change ++ * ++ * @kbdev: Kbase device ++ * @ctx_nr: Kernel context number ++ * @page_cnt: Number of pages used by the context ++ */ ++#define KBASE_TLSTREAM_AUX_PAGESALLOC( \ ++ kbdev, \ ++ ctx_nr, \ ++ page_cnt \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_aux_pagesalloc( \ ++ __TL_DISPATCH_STREAM(kbdev, aux), \ ++ ctx_nr, page_cnt); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_AUX_DEVFREQ_TARGET - ++ * New device frequency target ++ * ++ * @kbdev: Kbase device ++ * @target_freq: New target frequency ++ */ ++#define KBASE_TLSTREAM_AUX_DEVFREQ_TARGET( \ ++ kbdev, \ ++ target_freq \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_aux_devfreq_target( \ ++ __TL_DISPATCH_STREAM(kbdev, aux), \ ++ target_freq); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_AUX_PROTECTED_ENTER_START - ++ * enter protected mode start ++ * ++ * @kbdev: Kbase device ++ * @gpu: Name of the GPU object ++ */ ++#define KBASE_TLSTREAM_AUX_PROTECTED_ENTER_START( \ ++ kbdev, \ ++ gpu \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS) \ ++ __kbase_tlstream_aux_protected_enter_start( \ ++ __TL_DISPATCH_STREAM(kbdev, aux), \ ++ gpu); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_AUX_PROTECTED_ENTER_END - ++ * enter protected mode end ++ * ++ * @kbdev: Kbase device ++ * @gpu: Name of the GPU object ++ */ ++#define KBASE_TLSTREAM_AUX_PROTECTED_ENTER_END( \ ++ kbdev, \ ++ gpu \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS) \ ++ __kbase_tlstream_aux_protected_enter_end( \ ++ __TL_DISPATCH_STREAM(kbdev, aux), \ ++ gpu); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_START - ++ * leave protected mode start ++ * ++ * @kbdev: Kbase device ++ * @gpu: Name of the GPU object ++ */ ++#define KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_START( \ ++ kbdev, \ ++ gpu \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS) \ ++ __kbase_tlstream_aux_protected_leave_start( \ ++ __TL_DISPATCH_STREAM(kbdev, aux), \ ++ gpu); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_END - ++ * leave protected mode end ++ * ++ * @kbdev: Kbase device ++ * @gpu: Name of the GPU object ++ */ ++#define KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_END( \ ++ kbdev, \ ++ gpu \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS) \ ++ __kbase_tlstream_aux_protected_leave_end( \ ++ __TL_DISPATCH_STREAM(kbdev, aux), \ ++ gpu); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_AUX_JIT_STATS - ++ * per-bin JIT statistics ++ * ++ * @kbdev: Kbase device ++ * @ctx_nr: Kernel context number ++ * @bid: JIT bin id ++ * @max_allocs: Maximum allocations allowed in this bin. ++ * @allocs: Number of active allocations in this bin ++ * @va_pages: Number of virtual pages allocated in this bin ++ * @ph_pages: Number of physical pages allocated in this bin ++ */ ++#define KBASE_TLSTREAM_AUX_JIT_STATS( \ ++ kbdev, \ ++ ctx_nr, \ ++ bid, \ ++ max_allocs, \ ++ allocs, \ ++ va_pages, \ ++ ph_pages \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_aux_jit_stats( \ ++ __TL_DISPATCH_STREAM(kbdev, aux), \ ++ ctx_nr, bid, max_allocs, allocs, va_pages, ph_pages); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_AUX_EVENT_JOB_SLOT - ++ * event on a given job slot ++ * ++ * @kbdev: Kbase device ++ * @ctx: Name of the context object ++ * @slot_nr: Job slot number ++ * @atom_nr: Sequential number of an atom ++ * @event: Event type. One of TL_JS_EVENT values ++ */ ++#define KBASE_TLSTREAM_AUX_EVENT_JOB_SLOT( \ ++ kbdev, \ ++ ctx, \ ++ slot_nr, \ ++ atom_nr, \ ++ event \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_aux_event_job_slot( \ ++ __TL_DISPATCH_STREAM(kbdev, aux), \ ++ ctx, slot_nr, atom_nr, event); \ ++ } while (0) ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_NEW_DEVICE - ++ * New KBase Device ++ * ++ * @kbdev: Kbase device ++ * @kbase_device_id: The id of the physical hardware ++ * @kbase_device_gpu_core_count: The number of gpu cores in the physical hardware ++ * @kbase_device_max_num_csgs: The max number of CSGs the physical hardware supports ++ * @kbase_device_as_count: The number of address spaces the physical hardware has available ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_NEW_DEVICE( \ ++ kbdev, \ ++ kbase_device_id, \ ++ kbase_device_gpu_core_count, \ ++ kbase_device_max_num_csgs, \ ++ kbase_device_as_count \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_new_device( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ kbase_device_id, kbase_device_gpu_core_count, kbase_device_max_num_csgs, kbase_device_as_count); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_NEW_DEVICE( \ ++ kbdev, \ ++ kbase_device_id, \ ++ kbase_device_gpu_core_count, \ ++ kbase_device_max_num_csgs, \ ++ kbase_device_as_count \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_DEVICE_PROGRAM_CSG - ++ * CSG is programmed to a slot ++ * ++ * @kbdev: Kbase device ++ * @kbase_device_id: The id of the physical hardware ++ * @gpu_cmdq_grp_handle: GPU Command Queue Group handle which will match userspace ++ * @kbase_device_csg_slot_index: The index of the slot in the scheduler being programmed ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_DEVICE_PROGRAM_CSG( \ ++ kbdev, \ ++ kbase_device_id, \ ++ gpu_cmdq_grp_handle, \ ++ kbase_device_csg_slot_index \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_device_program_csg( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ kbase_device_id, gpu_cmdq_grp_handle, kbase_device_csg_slot_index); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_DEVICE_PROGRAM_CSG( \ ++ kbdev, \ ++ kbase_device_id, \ ++ gpu_cmdq_grp_handle, \ ++ kbase_device_csg_slot_index \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_DEVICE_DEPROGRAM_CSG - ++ * CSG is deprogrammed from a slot ++ * ++ * @kbdev: Kbase device ++ * @kbase_device_id: The id of the physical hardware ++ * @kbase_device_csg_slot_index: The index of the slot in the scheduler being programmed ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_DEVICE_DEPROGRAM_CSG( \ ++ kbdev, \ ++ kbase_device_id, \ ++ kbase_device_csg_slot_index \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_device_deprogram_csg( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ kbase_device_id, kbase_device_csg_slot_index); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_DEVICE_DEPROGRAM_CSG( \ ++ kbdev, \ ++ kbase_device_id, \ ++ kbase_device_csg_slot_index \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_NEW_CTX - ++ * New KBase Context ++ * ++ * @kbdev: Kbase device ++ * @kernel_ctx_id: Unique ID for the KBase Context ++ * @kbase_device_id: The id of the physical hardware ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_NEW_CTX( \ ++ kbdev, \ ++ kernel_ctx_id, \ ++ kbase_device_id \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_new_ctx( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ kernel_ctx_id, kbase_device_id); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_NEW_CTX( \ ++ kbdev, \ ++ kernel_ctx_id, \ ++ kbase_device_id \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_DEL_CTX - ++ * Delete KBase Context ++ * ++ * @kbdev: Kbase device ++ * @kernel_ctx_id: Unique ID for the KBase Context ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_DEL_CTX( \ ++ kbdev, \ ++ kernel_ctx_id \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_del_ctx( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ kernel_ctx_id); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_DEL_CTX( \ ++ kbdev, \ ++ kernel_ctx_id \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_CTX_ASSIGN_AS - ++ * Address Space is assigned to a KBase context ++ * ++ * @kbdev: Kbase device ++ * @kernel_ctx_id: Unique ID for the KBase Context ++ * @kbase_device_as_index: The index of the device address space being assigned ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_CTX_ASSIGN_AS( \ ++ kbdev, \ ++ kernel_ctx_id, \ ++ kbase_device_as_index \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_ctx_assign_as( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ kernel_ctx_id, kbase_device_as_index); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_CTX_ASSIGN_AS( \ ++ kbdev, \ ++ kernel_ctx_id, \ ++ kbase_device_as_index \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_CTX_UNASSIGN_AS - ++ * Address Space is unassigned from a KBase context ++ * ++ * @kbdev: Kbase device ++ * @kernel_ctx_id: Unique ID for the KBase Context ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_CTX_UNASSIGN_AS( \ ++ kbdev, \ ++ kernel_ctx_id \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_ctx_unassign_as( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ kernel_ctx_id); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_CTX_UNASSIGN_AS( \ ++ kbdev, \ ++ kernel_ctx_id \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_NEW_KCPUQUEUE - ++ * New KCPU Queue ++ * ++ * @kbdev: Kbase device ++ * @kcpu_queue: KCPU queue ++ * @kernel_ctx_id: Unique ID for the KBase Context ++ * @kcpuq_num_pending_cmds: Number of commands already enqueued ++ * in the KCPU queue ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_NEW_KCPUQUEUE( \ ++ kbdev, \ ++ kcpu_queue, \ ++ kernel_ctx_id, \ ++ kcpuq_num_pending_cmds \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_new_kcpuqueue( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ kcpu_queue, kernel_ctx_id, kcpuq_num_pending_cmds); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_NEW_KCPUQUEUE( \ ++ kbdev, \ ++ kcpu_queue, \ ++ kernel_ctx_id, \ ++ kcpuq_num_pending_cmds \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_DEL_KCPUQUEUE - ++ * Delete KCPU Queue ++ * ++ * @kbdev: Kbase device ++ * @kcpu_queue: KCPU queue ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_DEL_KCPUQUEUE( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_del_kcpuqueue( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ kcpu_queue); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_DEL_KCPUQUEUE( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_SIGNAL - ++ * KCPU Queue enqueues Signal on Fence ++ * ++ * @kbdev: Kbase device ++ * @kcpu_queue: KCPU queue ++ * @fence: Fence object handle ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_SIGNAL( \ ++ kbdev, \ ++ kcpu_queue, \ ++ fence \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_fence_signal( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ kcpu_queue, fence); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_SIGNAL( \ ++ kbdev, \ ++ kcpu_queue, \ ++ fence \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_WAIT - ++ * KCPU Queue enqueues Wait on Fence ++ * ++ * @kbdev: Kbase device ++ * @kcpu_queue: KCPU queue ++ * @fence: Fence object handle ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_WAIT( \ ++ kbdev, \ ++ kcpu_queue, \ ++ fence \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_fence_wait( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ kcpu_queue, fence); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_WAIT( \ ++ kbdev, \ ++ kcpu_queue, \ ++ fence \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_WAIT - ++ * KCPU Queue enqueues Wait on Cross Queue Sync Object ++ * ++ * @kbdev: Kbase device ++ * @kcpu_queue: KCPU queue ++ * @cqs_obj_gpu_addr: CQS Object GPU ptr ++ * @cqs_obj_compare_value: Semaphore value that should be exceeded ++ * for the WAIT to pass ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_WAIT( \ ++ kbdev, \ ++ kcpu_queue, \ ++ cqs_obj_gpu_addr, \ ++ cqs_obj_compare_value \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_cqs_wait( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ kcpu_queue, cqs_obj_gpu_addr, cqs_obj_compare_value); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_WAIT( \ ++ kbdev, \ ++ kcpu_queue, \ ++ cqs_obj_gpu_addr, \ ++ cqs_obj_compare_value \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_SET - ++ * KCPU Queue enqueues Set on Cross Queue Sync Object ++ * ++ * @kbdev: Kbase device ++ * @kcpu_queue: KCPU queue ++ * @cqs_obj_gpu_addr: CQS Object GPU ptr ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_SET( \ ++ kbdev, \ ++ kcpu_queue, \ ++ cqs_obj_gpu_addr \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_cqs_set( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ kcpu_queue, cqs_obj_gpu_addr); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_SET( \ ++ kbdev, \ ++ kcpu_queue, \ ++ cqs_obj_gpu_addr \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_MAP_IMPORT - ++ * KCPU Queue enqueues Map Import ++ * ++ * @kbdev: Kbase device ++ * @kcpu_queue: KCPU queue ++ * @map_import_buf_gpu_addr: Map import buffer GPU ptr ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_MAP_IMPORT( \ ++ kbdev, \ ++ kcpu_queue, \ ++ map_import_buf_gpu_addr \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_map_import( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ kcpu_queue, map_import_buf_gpu_addr); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_MAP_IMPORT( \ ++ kbdev, \ ++ kcpu_queue, \ ++ map_import_buf_gpu_addr \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT - ++ * KCPU Queue enqueues Unmap Import ++ * ++ * @kbdev: Kbase device ++ * @kcpu_queue: KCPU queue ++ * @map_import_buf_gpu_addr: Map import buffer GPU ptr ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT( \ ++ kbdev, \ ++ kcpu_queue, \ ++ map_import_buf_gpu_addr \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_unmap_import( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ kcpu_queue, map_import_buf_gpu_addr); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT( \ ++ kbdev, \ ++ kcpu_queue, \ ++ map_import_buf_gpu_addr \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT_FORCE - ++ * KCPU Queue enqueues Unmap Import ignoring reference count ++ * ++ * @kbdev: Kbase device ++ * @kcpu_queue: KCPU queue ++ * @map_import_buf_gpu_addr: Map import buffer GPU ptr ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT_FORCE( \ ++ kbdev, \ ++ kcpu_queue, \ ++ map_import_buf_gpu_addr \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_unmap_import_force( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ kcpu_queue, map_import_buf_gpu_addr); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT_FORCE( \ ++ kbdev, \ ++ kcpu_queue, \ ++ map_import_buf_gpu_addr \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_ALLOC - ++ * Begin array of KCPU Queue enqueues JIT Alloc ++ * ++ * @kbdev: Kbase device ++ * @kcpu_queue: KCPU queue ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_ALLOC( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_enqueue_jit_alloc( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ kcpu_queue); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_ALLOC( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_ALLOC - ++ * Array item of KCPU Queue enqueues JIT Alloc ++ * ++ * @kbdev: Kbase device ++ * @kcpu_queue: KCPU queue ++ * @jit_alloc_gpu_alloc_addr_dest: The GPU virtual address to write ++ * the JIT allocated GPU virtual address to ++ * @jit_alloc_va_pages: The minimum number of virtual pages required ++ * @jit_alloc_commit_pages: The minimum number of physical pages which ++ * should back the allocation ++ * @jit_alloc_extent: Granularity of physical pages to grow the allocation ++ * by during a fault ++ * @jit_alloc_jit_id: Unique ID provided by the caller, this is used ++ * to pair allocation and free requests. Zero is not a valid value ++ * @jit_alloc_bin_id: The JIT allocation bin, used in conjunction with ++ * max_allocations to limit the number of each type of JIT allocation ++ * @jit_alloc_max_allocations: The maximum number of allocations ++ * allowed within the bin specified by bin_id. Should be the same for all ++ * JIT allocations within the same bin. ++ * @jit_alloc_flags: Flags specifying the special requirements for the ++ * JIT allocation ++ * @jit_alloc_usage_id: A hint about which allocation should be ++ * reused. The kernel should attempt to use a previous allocation with the same ++ * usage_id ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_ALLOC( \ ++ kbdev, \ ++ kcpu_queue, \ ++ jit_alloc_gpu_alloc_addr_dest, \ ++ jit_alloc_va_pages, \ ++ jit_alloc_commit_pages, \ ++ jit_alloc_extent, \ ++ jit_alloc_jit_id, \ ++ jit_alloc_bin_id, \ ++ jit_alloc_max_allocations, \ ++ jit_alloc_flags, \ ++ jit_alloc_usage_id \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_array_item_kcpuqueue_enqueue_jit_alloc( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ kcpu_queue, jit_alloc_gpu_alloc_addr_dest, jit_alloc_va_pages, jit_alloc_commit_pages, jit_alloc_extent, jit_alloc_jit_id, jit_alloc_bin_id, jit_alloc_max_allocations, jit_alloc_flags, jit_alloc_usage_id); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_ALLOC( \ ++ kbdev, \ ++ kcpu_queue, \ ++ jit_alloc_gpu_alloc_addr_dest, \ ++ jit_alloc_va_pages, \ ++ jit_alloc_commit_pages, \ ++ jit_alloc_extent, \ ++ jit_alloc_jit_id, \ ++ jit_alloc_bin_id, \ ++ jit_alloc_max_allocations, \ ++ jit_alloc_flags, \ ++ jit_alloc_usage_id \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_ALLOC - ++ * End array of KCPU Queue enqueues JIT Alloc ++ * ++ * @kbdev: Kbase device ++ * @kcpu_queue: KCPU queue ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_ALLOC( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_array_end_kcpuqueue_enqueue_jit_alloc( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ kcpu_queue); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_ALLOC( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_FREE - ++ * Begin array of KCPU Queue enqueues JIT Free ++ * ++ * @kbdev: Kbase device ++ * @kcpu_queue: KCPU queue ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_FREE( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_enqueue_jit_free( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ kcpu_queue); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_FREE( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_FREE - ++ * Array item of KCPU Queue enqueues JIT Free ++ * ++ * @kbdev: Kbase device ++ * @kcpu_queue: KCPU queue ++ * @jit_alloc_jit_id: Unique ID provided by the caller, this is used ++ * to pair allocation and free requests. Zero is not a valid value ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_FREE( \ ++ kbdev, \ ++ kcpu_queue, \ ++ jit_alloc_jit_id \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_array_item_kcpuqueue_enqueue_jit_free( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ kcpu_queue, jit_alloc_jit_id); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_FREE( \ ++ kbdev, \ ++ kcpu_queue, \ ++ jit_alloc_jit_id \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_FREE - ++ * End array of KCPU Queue enqueues JIT Free ++ * ++ * @kbdev: Kbase device ++ * @kcpu_queue: KCPU queue ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_FREE( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_array_end_kcpuqueue_enqueue_jit_free( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ kcpu_queue); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_FREE( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_START - ++ * KCPU Queue starts a Signal on Fence ++ * ++ * @kbdev: Kbase device ++ * @kcpu_queue: KCPU queue ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_START( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_signal_start( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ kcpu_queue); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_START( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_END - ++ * KCPU Queue ends a Signal on Fence ++ * ++ * @kbdev: Kbase device ++ * @kcpu_queue: KCPU queue ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_END( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_signal_end( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ kcpu_queue); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_END( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_START - ++ * KCPU Queue starts a Wait on Fence ++ * ++ * @kbdev: Kbase device ++ * @kcpu_queue: KCPU queue ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_START( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_wait_start( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ kcpu_queue); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_START( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_END - ++ * KCPU Queue ends a Wait on Fence ++ * ++ * @kbdev: Kbase device ++ * @kcpu_queue: KCPU queue ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_END( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_wait_end( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ kcpu_queue); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_END( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_START - ++ * KCPU Queue starts a Wait on an array of Cross Queue Sync Objects ++ * ++ * @kbdev: Kbase device ++ * @kcpu_queue: KCPU queue ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_START( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_kcpuqueue_execute_cqs_wait_start( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ kcpu_queue); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_START( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_END - ++ * KCPU Queue ends a Wait on an array of Cross Queue Sync Objects ++ * ++ * @kbdev: Kbase device ++ * @kcpu_queue: KCPU queue ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_END( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_kcpuqueue_execute_cqs_wait_end( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ kcpu_queue); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_END( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_SET - ++ * KCPU Queue executes a Set on an array of Cross Queue Sync Objects ++ * ++ * @kbdev: Kbase device ++ * @kcpu_queue: KCPU queue ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_SET( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_kcpuqueue_execute_cqs_set( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ kcpu_queue); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_SET( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_START - ++ * KCPU Queue starts a Map Import ++ * ++ * @kbdev: Kbase device ++ * @kcpu_queue: KCPU queue ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_START( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_kcpuqueue_execute_map_import_start( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ kcpu_queue); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_START( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_END - ++ * KCPU Queue ends a Map Import ++ * ++ * @kbdev: Kbase device ++ * @kcpu_queue: KCPU queue ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_END( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_kcpuqueue_execute_map_import_end( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ kcpu_queue); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_END( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_START - ++ * KCPU Queue starts an Unmap Import ++ * ++ * @kbdev: Kbase device ++ * @kcpu_queue: KCPU queue ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_START( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_start( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ kcpu_queue); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_START( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_END - ++ * KCPU Queue ends an Unmap Import ++ * ++ * @kbdev: Kbase device ++ * @kcpu_queue: KCPU queue ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_END( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_end( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ kcpu_queue); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_END( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_START - ++ * KCPU Queue starts an Unmap Import ignoring reference count ++ * ++ * @kbdev: Kbase device ++ * @kcpu_queue: KCPU queue ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_START( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_force_start( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ kcpu_queue); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_START( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_END - ++ * KCPU Queue ends an Unmap Import ignoring reference count ++ * ++ * @kbdev: Kbase device ++ * @kcpu_queue: KCPU queue ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_END( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_force_end( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ kcpu_queue); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_END( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_START - ++ * KCPU Queue starts an array of JIT Allocs ++ * ++ * @kbdev: Kbase device ++ * @kcpu_queue: KCPU queue ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_START( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_kcpuqueue_execute_jit_alloc_start( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ kcpu_queue); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_START( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_ALLOC_END - ++ * Begin array of KCPU Queue ends an array of JIT Allocs ++ * ++ * @kbdev: Kbase device ++ * @kcpu_queue: KCPU queue ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_execute_jit_alloc_end( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ kcpu_queue); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_ALLOC_END - ++ * Array item of KCPU Queue ends an array of JIT Allocs ++ * ++ * @kbdev: Kbase device ++ * @kcpu_queue: KCPU queue ++ * @jit_alloc_gpu_alloc_addr: The JIT allocated GPU virtual address ++ * @jit_alloc_mmu_flags: The MMU flags for the JIT allocation ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( \ ++ kbdev, \ ++ kcpu_queue, \ ++ jit_alloc_gpu_alloc_addr, \ ++ jit_alloc_mmu_flags \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_array_item_kcpuqueue_execute_jit_alloc_end( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ kcpu_queue, jit_alloc_gpu_alloc_addr, jit_alloc_mmu_flags); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( \ ++ kbdev, \ ++ kcpu_queue, \ ++ jit_alloc_gpu_alloc_addr, \ ++ jit_alloc_mmu_flags \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_ALLOC_END - ++ * End array of KCPU Queue ends an array of JIT Allocs ++ * ++ * @kbdev: Kbase device ++ * @kcpu_queue: KCPU queue ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_array_end_kcpuqueue_execute_jit_alloc_end( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ kcpu_queue); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_START - ++ * KCPU Queue starts an array of JIT Frees ++ * ++ * @kbdev: Kbase device ++ * @kcpu_queue: KCPU queue ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_START( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_kcpuqueue_execute_jit_free_start( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ kcpu_queue); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_START( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_FREE_END - ++ * Begin array of KCPU Queue ends an array of JIT Frees ++ * ++ * @kbdev: Kbase device ++ * @kcpu_queue: KCPU queue ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_FREE_END( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_execute_jit_free_end( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ kcpu_queue); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_FREE_END( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_FREE_END - ++ * Array item of KCPU Queue ends an array of JIT Frees ++ * ++ * @kbdev: Kbase device ++ * @kcpu_queue: KCPU queue ++ * @jit_free_pages_used: The actual number of pages used by the JIT ++ * allocation ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_FREE_END( \ ++ kbdev, \ ++ kcpu_queue, \ ++ jit_free_pages_used \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_array_item_kcpuqueue_execute_jit_free_end( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ kcpu_queue, jit_free_pages_used); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_FREE_END( \ ++ kbdev, \ ++ kcpu_queue, \ ++ jit_free_pages_used \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_FREE_END - ++ * End array of KCPU Queue ends an array of JIT Frees ++ * ++ * @kbdev: Kbase device ++ * @kcpu_queue: KCPU queue ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_FREE_END( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_array_end_kcpuqueue_execute_jit_free_end( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ kcpu_queue); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_FREE_END( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_ERRORBARRIER - ++ * KCPU Queue executes an Error Barrier ++ * ++ * @kbdev: Kbase device ++ * @kcpu_queue: KCPU queue ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_ERRORBARRIER( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_kcpuqueue_execute_errorbarrier( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ kcpu_queue); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_ERRORBARRIER( \ ++ kbdev, \ ++ kcpu_queue \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_CSFFW_TLSTREAM_OVERFLOW - ++ * An overflow has happened with the CSFFW Timeline stream ++ * ++ * @kbdev: Kbase device ++ * @csffw_timestamp: Timestamp of a CSFFW event ++ * @csffw_cycle: Cycle number of a CSFFW event ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_CSFFW_TLSTREAM_OVERFLOW( \ ++ kbdev, \ ++ csffw_timestamp, \ ++ csffw_cycle \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSFFW_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_csffw_tlstream_overflow( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ csffw_timestamp, csffw_cycle); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_CSFFW_TLSTREAM_OVERFLOW( \ ++ kbdev, \ ++ csffw_timestamp, \ ++ csffw_cycle \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++/** ++ * KBASE_TLSTREAM_TL_KBASE_CSFFW_RESET - ++ * A reset has happened with the CSFFW ++ * ++ * @kbdev: Kbase device ++ * @csffw_cycle: Cycle number of a CSFFW event ++ */ ++#if MALI_USE_CSF ++#define KBASE_TLSTREAM_TL_KBASE_CSFFW_RESET( \ ++ kbdev, \ ++ csffw_cycle \ ++ ) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_CSFFW_TRACEPOINTS) \ ++ __kbase_tlstream_tl_kbase_csffw_reset( \ ++ __TL_DISPATCH_STREAM(kbdev, obj), \ ++ csffw_cycle); \ ++ } while (0) ++#else ++#define KBASE_TLSTREAM_TL_KBASE_CSFFW_RESET( \ ++ kbdev, \ ++ csffw_cycle \ ++ ) \ ++ do { } while (0) ++#endif /* MALI_USE_CSF */ ++ ++ ++/* Gator tracepoints are hooked into TLSTREAM interface. ++ * When the following tracepoints are called, corresponding ++ * Gator tracepoint will be called as well. ++ */ ++ ++#if defined(CONFIG_MALI_BIFROST_GATOR_SUPPORT) ++/* `event` is one of TL_JS_EVENT values here. ++ * The values of TL_JS_EVENT are guaranteed to match ++ * with corresponding GATOR_JOB_SLOT values. ++ */ ++#undef KBASE_TLSTREAM_AUX_EVENT_JOB_SLOT ++#define KBASE_TLSTREAM_AUX_EVENT_JOB_SLOT(kbdev, \ ++ context, slot_nr, atom_nr, event) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ kbase_trace_mali_job_slots_event(kbdev->id, \ ++ GATOR_MAKE_EVENT(event, slot_nr), \ ++ context, (u8) atom_nr); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_aux_event_job_slot( \ ++ __TL_DISPATCH_STREAM(kbdev, aux), \ ++ context, slot_nr, atom_nr, event); \ ++ } while (0) ++ ++#undef KBASE_TLSTREAM_AUX_PM_STATE ++#define KBASE_TLSTREAM_AUX_PM_STATE(kbdev, core_type, state) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ kbase_trace_mali_pm_status(kbdev->id, \ ++ core_type, state); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_aux_pm_state( \ ++ __TL_DISPATCH_STREAM(kbdev, aux), \ ++ core_type, state); \ ++ } while (0) ++ ++#undef KBASE_TLSTREAM_AUX_PAGEFAULT ++#define KBASE_TLSTREAM_AUX_PAGEFAULT(kbdev, \ ++ ctx_nr, as_nr, page_cnt_change) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ kbase_trace_mali_page_fault_insert_pages(kbdev->id, \ ++ as_nr, \ ++ page_cnt_change); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_aux_pagefault( \ ++ __TL_DISPATCH_STREAM(kbdev, aux), \ ++ ctx_nr, as_nr, page_cnt_change); \ ++ } while (0) ++ ++/* kbase_trace_mali_total_alloc_pages_change is handled differently here. ++ * We stream the total amount of pages allocated for `kbdev` rather ++ * than `page_count`, which is per-context. ++ */ ++#undef KBASE_TLSTREAM_AUX_PAGESALLOC ++#define KBASE_TLSTREAM_AUX_PAGESALLOC(kbdev, ctx_nr, page_cnt) \ ++ do { \ ++ int enabled = atomic_read(&kbdev->timeline_flags); \ ++ u32 global_pages_count = \ ++ atomic_read(&kbdev->memdev.used_pages); \ ++ \ ++ kbase_trace_mali_total_alloc_pages_change(kbdev->id, \ ++ global_pages_count); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_aux_pagesalloc( \ ++ __TL_DISPATCH_STREAM(kbdev, aux), \ ++ ctx_nr, page_cnt); \ ++ } while (0) ++#endif /* CONFIG_MALI_BIFROST_GATOR_SUPPORT */ ++ ++/* clang-format on */ ++#endif +diff --git a/drivers/gpu/arm/bifrost_for_linux/Kbuild b/drivers/gpu/arm/bifrost_for_linux/Kbuild +new file mode 100755 +index 000000000000..531b92126b98 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/Kbuild +@@ -0,0 +1,172 @@ ++# ++# (C) COPYRIGHT 2012-2016, 2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++# Driver version string which is returned to userspace via an ioctl ++MALI_RELEASE_NAME ?= "r8p0-01rel0" ++ ++# Paths required for build ++KBASE_PATH = $(src) ++KBASE_PLATFORM_PATH = $(KBASE_PATH)/platform_dummy ++UMP_PATH = $(src)/../../../base ++ ++ifeq ($(CONFIG_MALI_BIFROST_ERROR_INJECT),y) ++MALI_ERROR_INJECT_ON = 1 ++endif ++ ++# Set up defaults if not defined by build system ++MALI_CUSTOMER_RELEASE ?= 1 ++MALI_UNIT_TEST ?= 0 ++MALI_KERNEL_TEST_API ?= 0 ++MALI_ERROR_INJECT_ON ?= 0 ++MALI_MOCK_TEST ?= 0 ++MALI_COVERAGE ?= 0 ++MALI_INSTRUMENTATION_LEVEL ?= 0 ++CONFIG_MALI_PLATFORM_NAME ?= "devicetree" ++# This workaround is for what seems to be a compiler bug we observed in ++# GCC 4.7 on AOSP 4.3. The bug caused an intermittent failure compiling ++# the "_Pragma" syntax, where an error message is returned: ++# ++# "internal compiler error: unspellable token PRAGMA" ++# ++# This regression has thus far only been seen on the GCC 4.7 compiler bundled ++# with AOSP 4.3.0. So this makefile, intended for in-tree kernel builds ++# which are not known to be used with AOSP, is hardcoded to disable the ++# workaround, i.e. set the define to 0. ++MALI_GCC_WORKAROUND_MIDCOM_4598 ?= 0 ++ ++# Set up our defines, which will be passed to gcc ++DEFINES = \ ++ -DMALI_CUSTOMER_RELEASE=$(MALI_CUSTOMER_RELEASE) \ ++ -DMALI_KERNEL_TEST_API=$(MALI_KERNEL_TEST_API) \ ++ -DMALI_UNIT_TEST=$(MALI_UNIT_TEST) \ ++ -DMALI_ERROR_INJECT_ON=$(MALI_ERROR_INJECT_ON) \ ++ -DMALI_MOCK_TEST=$(MALI_MOCK_TEST) \ ++ -DMALI_COVERAGE=$(MALI_COVERAGE) \ ++ -DMALI_INSTRUMENTATION_LEVEL=$(MALI_INSTRUMENTATION_LEVEL) \ ++ -DMALI_RELEASE_NAME=\"$(MALI_RELEASE_NAME)\" \ ++ -DMALI_GCC_WORKAROUND_MIDCOM_4598=$(MALI_GCC_WORKAROUND_MIDCOM_4598) ++ ++ifeq ($(KBUILD_EXTMOD),) ++# in-tree ++DEFINES +=-DMALI_KBASE_PLATFORM_PATH=../../$(src)/platform/$(CONFIG_MALI_PLATFORM_NAME) ++else ++# out-of-tree ++DEFINES +=-DMALI_KBASE_PLATFORM_PATH=$(src)/platform/$(CONFIG_MALI_PLATFORM_NAME) ++endif ++ ++DEFINES += -I$(srctree)/drivers/staging/android ++ ++# Use our defines when compiling ++ccflags-y += $(DEFINES) -I$(KBASE_PATH) -I$(KBASE_PLATFORM_PATH) -I$(UMP_PATH) -I$(srctree)/include/linux ++subdir-ccflags-y += $(DEFINES) -I$(KBASE_PATH) -I$(KBASE_PLATFORM_PATH) -I$(OSK_PATH) -I$(UMP_PATH) -I$(srctree)/include/linux ++ ++SRC := \ ++ mali_kbase_device.c \ ++ mali_kbase_cache_policy.c \ ++ mali_kbase_mem.c \ ++ mali_kbase_mmu.c \ ++ mali_kbase_ctx_sched.c \ ++ mali_kbase_jd.c \ ++ mali_kbase_jd_debugfs.c \ ++ mali_kbase_jm.c \ ++ mali_kbase_gpuprops.c \ ++ mali_kbase_js.c \ ++ mali_kbase_js_ctx_attr.c \ ++ mali_kbase_event.c \ ++ mali_kbase_context.c \ ++ mali_kbase_pm.c \ ++ mali_kbase_config.c \ ++ mali_kbase_vinstr.c \ ++ mali_kbase_softjobs.c \ ++ mali_kbase_10969_workaround.c \ ++ mali_kbase_hw.c \ ++ mali_kbase_utility.c \ ++ mali_kbase_debug.c \ ++ mali_kbase_trace_timeline.c \ ++ mali_kbase_gpu_memory_debugfs.c \ ++ mali_kbase_mem_linux.c \ ++ mali_kbase_core_linux.c \ ++ mali_kbase_replay.c \ ++ mali_kbase_mem_profile_debugfs.c \ ++ mali_kbase_mmu_mode_lpae.c \ ++ mali_kbase_mmu_mode_aarch64.c \ ++ mali_kbase_disjoint_events.c \ ++ mali_kbase_gator_api.c \ ++ mali_kbase_debug_mem_view.c \ ++ mali_kbase_debug_job_fault.c \ ++ mali_kbase_smc.c \ ++ mali_kbase_mem_pool.c \ ++ mali_kbase_mem_pool_debugfs.c \ ++ mali_kbase_tlstream.c \ ++ mali_kbase_strings.c \ ++ mali_kbase_as_fault_debugfs.c \ ++ mali_kbase_regs_history_debugfs.c ++ ++ ++ ++ ++ifeq ($(MALI_UNIT_TEST),1) ++ SRC += mali_kbase_tlstream_test.c ++endif ++ ++ifeq ($(MALI_CUSTOMER_RELEASE),0) ++ SRC += mali_kbase_regs_dump_debugfs.c ++endif ++ ++ ++ccflags-y += -I$(KBASE_PATH) ++ ++# Tell the Linux build system from which .o file to create the kernel module ++obj-$(CONFIG_MALI_BIFROST) += bifrost_kbase.o ++ ++# Tell the Linux build system to enable building of our .c files ++bifrost_kbase-y := $(SRC:.c=.o) ++ ++# Kconfig passes in the name with quotes for in-tree builds - remove them. ++platform_name := $(shell echo $(CONFIG_MALI_PLATFORM_NAME)) ++MALI_PLATFORM_DIR := platform/$(platform_name) ++ccflags-y += -I$(src)/$(MALI_PLATFORM_DIR) ++include $(src)/$(MALI_PLATFORM_DIR)/Kbuild ++ ++ifeq ($(CONFIG_MALI_BIFROST_DEVFREQ),y) ++ ifeq ($(CONFIG_DEVFREQ_THERMAL),y) ++ include $(src)/ipa/Kbuild ++ endif ++endif ++ ++bifrost_kbase-$(CONFIG_MALI_BIFROST_DMA_FENCE) += \ ++ mali_kbase_dma_fence.o \ ++ mali_kbase_fence.o ++bifrost_kbase-$(CONFIG_SYNC) += \ ++ mali_kbase_sync_android.o \ ++ mali_kbase_sync_common.o ++bifrost_kbase-$(CONFIG_SYNC_FILE) += \ ++ mali_kbase_sync_file.o \ ++ mali_kbase_sync_common.o \ ++ mali_kbase_fence.o ++ ++ifeq ($(MALI_MOCK_TEST),1) ++# Test functionality ++bifrost_kbase-y += tests/internal/src/mock/mali_kbase_pm_driver_mock.o ++endif ++ ++include $(src)/backend/gpu/Kbuild ++bifrost_kbase-y += $(BACKEND:.c=.o) ++ ++ ++ccflags-y += -I$(src)/backend/gpu ++subdir-ccflags-y += -I$(src)/backend/gpu ++ ++# For kutf and mali_kutf_irq_latency_test ++obj-$(CONFIG_MALI_KUTF) += tests/ +diff --git a/drivers/gpu/arm/bifrost_for_linux/Kconfig b/drivers/gpu/arm/bifrost_for_linux/Kconfig +new file mode 100755 +index 000000000000..e78b634031ee +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/Kconfig +@@ -0,0 +1,196 @@ ++# ++# (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++ ++menuconfig MALI_BIFROST ++ tristate "Mali Bifrost series support (Linux only)" ++ default n ++ help ++ Enable this option to build support for a ARM Mali Bifrost GPU. ++ ++ To compile this driver as a module, choose M here: ++ this will generate a single module, called mali_kbase. ++ ++config MALI_BIFROST_GATOR_SUPPORT ++ bool "Streamline support via Gator" ++ depends on MALI_BIFROST ++ default n ++ help ++ Adds diagnostic support for use with the ARM Streamline Performance Analyzer. ++ You will need the Gator device driver already loaded before loading this driver when enabling ++ Streamline debug support. ++ This is a legacy interface required by older versions of Streamline. ++ ++config MALI_BIFROST_DVFS ++ bool "Enable legacy DVFS" ++ depends on MALI_BIFROST && !MALI_BIFROST_DEVFREQ ++ default n ++ help ++ Choose this option to enable legacy DVFS in the Mali Midgard DDK. ++ ++config MALI_BIFROST_ENABLE_TRACE ++ bool "Enable kbase tracing" ++ depends on MALI_BIFROST ++ default n ++ help ++ Enables tracing in kbase. Trace log available through ++ the "mali_trace" debugfs file, when the CONFIG_DEBUG_FS is enabled ++ ++config MALI_BIFROST_DEVFREQ ++ bool "devfreq support for Mali" ++ depends on MALI_BIFROST && PM_DEVFREQ ++ help ++ Support devfreq for Mali. ++ ++ Using the devfreq framework and, by default, the simpleondemand ++ governor, the frequency of Mali will be dynamically selected from the ++ available OPPs. ++ ++config MALI_BIFROST_DMA_FENCE ++ bool "DMA_BUF fence support for Mali" ++ depends on MALI_BIFROST && !KDS ++ default n ++ help ++ Support DMA_BUF fences for Mali. ++ ++ This option should only be enabled if KDS is not present and ++ the Linux Kernel has built in support for DMA_BUF fences. ++ ++config MALI_PLATFORM_NAME ++ depends on MALI_BIFROST ++ string "Platform name" ++ default "devicetree" ++ help ++ Enter the name of the desired platform configuration directory to ++ include in the build. 'platform/$(MALI_PLATFORM_NAME)/Kbuild' must ++ exist. ++ ++# MALI_BIFROST_EXPERT configuration options ++ ++menuconfig MALI_BIFROST_EXPERT ++ depends on MALI_BIFROST ++ bool "Enable Expert Settings" ++ default n ++ help ++ Enabling this option and modifying the default settings may produce a driver with performance or ++ other limitations. ++ ++config MALI_CORESTACK ++ bool "Support controlling power to the GPU core stack" ++ depends on MALI_BIFROST && MALI_BIFROST_EXPERT ++ default n ++ help ++ Enabling this feature on supported GPUs will let the driver powering ++ on/off the GPU core stack independently without involving the Power ++ Domain Controller. This should only be enabled on platforms which ++ integration of the PDC to the Mali GPU is known to be problematic. ++ This feature is currently only supported on t-Six and t-HEx GPUs. ++ ++ If unsure, say N. ++ ++config MALI_BIFROST_PRFCNT_SET_SECONDARY ++ bool "Use secondary set of performance counters" ++ depends on MALI_BIFROST && MALI_BIFROST_EXPERT ++ default n ++ help ++ Select this option to use secondary set of performance counters. Kernel ++ features that depend on an access to the primary set of counters may ++ become unavailable. Enabling this option will prevent power management ++ from working optimally and may cause instrumentation tools to return ++ bogus results. ++ ++ If unsure, say N. ++ ++config MALI_BIFROST_DEBUG ++ bool "Debug build" ++ depends on MALI_BIFROST && MALI_BIFROST_EXPERT ++ default n ++ help ++ Select this option for increased checking and reporting of errors. ++ ++config MALI_BIFROST_FENCE_DEBUG ++ bool "Debug sync fence usage" ++ depends on MALI_BIFROST && MALI_BIFROST_EXPERT && (SYNC || SYNC_FILE) ++ default y if MALI_BIFROST_DEBUG ++ help ++ Select this option to enable additional checking and reporting on the ++ use of sync fences in the Mali driver. ++ ++ This will add a 3s timeout to all sync fence waits in the Mali ++ driver, so that when work for Mali has been waiting on a sync fence ++ for a long time a debug message will be printed, detailing what fence ++ is causing the block, and which dependent Mali atoms are blocked as a ++ result of this. ++ ++ The timeout can be changed at runtime through the js_soft_timeout ++ device attribute, where the timeout is specified in milliseconds. ++ ++config MALI_BIFROST_NO_MALI ++ bool "No Mali" ++ depends on MALI_BIFROST && MALI_BIFROST_EXPERT ++ default n ++ help ++ This can be used to test the driver in a simulated environment ++ whereby the hardware is not physically present. If the hardware is physically ++ present it will not be used. This can be used to test the majority of the ++ driver without needing actual hardware or for software benchmarking. ++ All calls to the simulated hardware will complete immediately as if the hardware ++ completed the task. ++ ++config MALI_BIFROST_ERROR_INJECT ++ bool "Error injection" ++ depends on MALI_BIFROST && MALI_BIFROST_EXPERT && MALI_BIFROST_NO_MALI ++ default n ++ help ++ Enables insertion of errors to test module failure and recovery mechanisms. ++ ++config MALI_BIFROST_TRACE_TIMELINE ++ bool "Timeline tracing" ++ depends on MALI_BIFROST && MALI_BIFROST_EXPERT ++ default n ++ help ++ Enables timeline tracing through the kernel tracepoint system. ++ ++config MALI_BIFROST_SYSTEM_TRACE ++ bool "Enable system event tracing support" ++ depends on MALI_BIFROST && MALI_BIFROST_EXPERT ++ default n ++ help ++ Choose this option to enable system trace events for each ++ kbase event. This is typically used for debugging but has ++ minimal overhead when not in use. Enable only if you know what ++ you are doing. ++ ++config MALI_2MB_ALLOC ++ bool "Attempt to allocate 2MB pages" ++ depends on MALI_BIFROST && MALI_BIFROST_EXPERT ++ default n ++ help ++ Rather than allocating all GPU memory page-by-page, attempt to ++ allocate 2MB pages from the kernel. This reduces TLB pressure and ++ helps to prevent memory fragmentation. ++ ++ If in doubt, say N ++ ++config MALI_PWRSOFT_765 ++ bool "PWRSOFT-765 ticket" ++ default n ++ help ++ PWRSOFT-765 fixes devfreq cooling devices issues. However, they are ++ not merged in mainline kernel yet. So this define helps to guard those ++ parts of the code. ++ ++source "drivers/gpu/arm/bifrost/platform/Kconfig" ++# source "drivers/gpu/arm/bifrost/tests/Kconfig" +diff --git a/drivers/gpu/arm/bifrost_for_linux/Makefile b/drivers/gpu/arm/bifrost_for_linux/Makefile +new file mode 100755 +index 000000000000..26522d566dd0 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/Makefile +@@ -0,0 +1,42 @@ ++# ++# (C) COPYRIGHT 2010-2016, 2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++ ++KDIR ?= /lib/modules/$(shell uname -r)/build ++ ++BUSLOG_PATH_RELATIVE = $(CURDIR)/../../../.. ++UMP_PATH_RELATIVE = $(CURDIR)/../../../base/ump ++KBASE_PATH_RELATIVE = $(CURDIR) ++KDS_PATH_RELATIVE = $(CURDIR)/../../../.. ++EXTRA_SYMBOLS = $(UMP_PATH_RELATIVE)/src/Module.symvers ++ ++ifeq ($(MALI_UNIT_TEST), 1) ++ EXTRA_SYMBOLS += $(KBASE_PATH_RELATIVE)/tests/internal/src/kernel_assert_module/linux/Module.symvers ++endif ++ ++ifeq ($(CONFIG_MALI_FPGA_BUS_LOGGER),y) ++#Add bus logger symbols ++EXTRA_SYMBOLS += $(BUSLOG_PATH_RELATIVE)/drivers/base/bus_logger/Module.symvers ++endif ++ ++# GPL driver supports KDS ++EXTRA_SYMBOLS += $(KDS_PATH_RELATIVE)/drivers/base/kds/Module.symvers ++ ++# we get the symbols from modules using KBUILD_EXTRA_SYMBOLS to prevent warnings about unknown functions ++all: ++ $(MAKE) -C $(KDIR) M=$(CURDIR) EXTRA_CFLAGS="-I$(CURDIR)/../../../../include -I$(CURDIR)/../../../../tests/include $(SCONS_CFLAGS)" $(SCONS_CONFIGS) KBUILD_EXTRA_SYMBOLS="$(EXTRA_SYMBOLS)" modules ++ ++clean: ++ $(MAKE) -C $(KDIR) M=$(CURDIR) clean +diff --git a/drivers/gpu/arm/bifrost_for_linux/Makefile.kbase b/drivers/gpu/arm/bifrost_for_linux/Makefile.kbase +new file mode 100755 +index 000000000000..2bef9c25eaeb +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/Makefile.kbase +@@ -0,0 +1,17 @@ ++# ++# (C) COPYRIGHT 2010 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++EXTRA_CFLAGS += -I$(ROOT) -I$(KBASE_PATH) -I$(OSK_PATH)/src/linux/include -I$(KBASE_PATH)/platform_$(PLATFORM) ++ +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/Kbuild b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/Kbuild +new file mode 100755 +index 000000000000..5eeba1b14710 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/Kbuild +@@ -0,0 +1,60 @@ ++# ++# (C) COPYRIGHT 2014,2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++BACKEND += \ ++ backend/gpu/mali_kbase_cache_policy_backend.c \ ++ backend/gpu/mali_kbase_device_hw.c \ ++ backend/gpu/mali_kbase_gpu.c \ ++ backend/gpu/mali_kbase_gpuprops_backend.c \ ++ backend/gpu/mali_kbase_debug_job_fault_backend.c \ ++ backend/gpu/mali_kbase_irq_linux.c \ ++ backend/gpu/mali_kbase_instr_backend.c \ ++ backend/gpu/mali_kbase_jm_as.c \ ++ backend/gpu/mali_kbase_jm_hw.c \ ++ backend/gpu/mali_kbase_jm_rb.c \ ++ backend/gpu/mali_kbase_js_affinity.c \ ++ backend/gpu/mali_kbase_js_backend.c \ ++ backend/gpu/mali_kbase_mmu_hw_direct.c \ ++ backend/gpu/mali_kbase_pm_backend.c \ ++ backend/gpu/mali_kbase_pm_driver.c \ ++ backend/gpu/mali_kbase_pm_metrics.c \ ++ backend/gpu/mali_kbase_pm_ca.c \ ++ backend/gpu/mali_kbase_pm_ca_fixed.c \ ++ backend/gpu/mali_kbase_pm_always_on.c \ ++ backend/gpu/mali_kbase_pm_coarse_demand.c \ ++ backend/gpu/mali_kbase_pm_demand.c \ ++ backend/gpu/mali_kbase_pm_policy.c \ ++ backend/gpu/mali_kbase_time.c ++ ++ifeq ($(MALI_CUSTOMER_RELEASE),0) ++BACKEND += \ ++ backend/gpu/mali_kbase_pm_ca_random.c \ ++ backend/gpu/mali_kbase_pm_demand_always_powered.c \ ++ backend/gpu/mali_kbase_pm_fast_start.c ++endif ++ ++ifeq ($(CONFIG_MALI_BIFROST_DEVFREQ),y) ++BACKEND += \ ++ backend/gpu/mali_kbase_devfreq.c \ ++ backend/gpu/mali_kbase_pm_ca_devfreq.c ++endif ++ ++ifeq ($(CONFIG_MALI_BIFROST_NO_MALI),y) ++ # Dummy model ++ BACKEND += backend/gpu/mali_kbase_model_dummy.c ++ BACKEND += backend/gpu/mali_kbase_model_linux.c ++ # HW error simulation ++ BACKEND += backend/gpu/mali_kbase_model_error_generator.c ++endif +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_backend_config.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_backend_config.h +new file mode 100755 +index 000000000000..c8ae87eb84a2 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_backend_config.h +@@ -0,0 +1,29 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* ++ * Backend specific configuration ++ */ ++ ++#ifndef _KBASE_BACKEND_CONFIG_H_ ++#define _KBASE_BACKEND_CONFIG_H_ ++ ++/* Enable GPU reset API */ ++#define KBASE_GPU_RESET_EN 1 ++ ++#endif /* _KBASE_BACKEND_CONFIG_H_ */ ++ +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_cache_policy_backend.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_cache_policy_backend.c +new file mode 100755 +index 000000000000..fef9a2cb743e +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_cache_policy_backend.c +@@ -0,0 +1,29 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include "backend/gpu/mali_kbase_cache_policy_backend.h" ++#include ++ ++void kbase_cache_set_coherency_mode(struct kbase_device *kbdev, ++ u32 mode) ++{ ++ kbdev->current_gpu_coherency_mode = mode; ++ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_COHERENCY_REG)) ++ kbase_reg_write(kbdev, COHERENCY_ENABLE, mode, NULL); ++} ++ +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_cache_policy_backend.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_cache_policy_backend.h +new file mode 100755 +index 000000000000..fe9869109a82 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_cache_policy_backend.h +@@ -0,0 +1,34 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++#ifndef _KBASE_CACHE_POLICY_BACKEND_H_ ++#define _KBASE_CACHE_POLICY_BACKEND_H_ ++ ++#include "mali_kbase.h" ++#include "mali_base_kernel.h" ++ ++/** ++ * kbase_cache_set_coherency_mode() - Sets the system coherency mode ++ * in the GPU. ++ * @kbdev: Device pointer ++ * @mode: Coherency mode. COHERENCY_ACE/ACE_LITE ++ */ ++void kbase_cache_set_coherency_mode(struct kbase_device *kbdev, ++ u32 mode); ++ ++#endif /* _KBASE_CACHE_POLICY_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_debug_job_fault_backend.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_debug_job_fault_backend.c +new file mode 100755 +index 000000000000..7851ea6466c7 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_debug_job_fault_backend.c +@@ -0,0 +1,157 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include ++#include ++#include "mali_kbase_debug_job_fault.h" ++ ++#ifdef CONFIG_DEBUG_FS ++ ++/*GPU_CONTROL_REG(r)*/ ++static int gpu_control_reg_snapshot[] = { ++ GPU_ID, ++ SHADER_READY_LO, ++ SHADER_READY_HI, ++ TILER_READY_LO, ++ TILER_READY_HI, ++ L2_READY_LO, ++ L2_READY_HI ++}; ++ ++/* JOB_CONTROL_REG(r) */ ++static int job_control_reg_snapshot[] = { ++ JOB_IRQ_MASK, ++ JOB_IRQ_STATUS ++}; ++ ++/* JOB_SLOT_REG(n,r) */ ++static int job_slot_reg_snapshot[] = { ++ JS_HEAD_LO, ++ JS_HEAD_HI, ++ JS_TAIL_LO, ++ JS_TAIL_HI, ++ JS_AFFINITY_LO, ++ JS_AFFINITY_HI, ++ JS_CONFIG, ++ JS_STATUS, ++ JS_HEAD_NEXT_LO, ++ JS_HEAD_NEXT_HI, ++ JS_AFFINITY_NEXT_LO, ++ JS_AFFINITY_NEXT_HI, ++ JS_CONFIG_NEXT ++}; ++ ++/*MMU_REG(r)*/ ++static int mmu_reg_snapshot[] = { ++ MMU_IRQ_MASK, ++ MMU_IRQ_STATUS ++}; ++ ++/* MMU_AS_REG(n,r) */ ++static int as_reg_snapshot[] = { ++ AS_TRANSTAB_LO, ++ AS_TRANSTAB_HI, ++ AS_MEMATTR_LO, ++ AS_MEMATTR_HI, ++ AS_FAULTSTATUS, ++ AS_FAULTADDRESS_LO, ++ AS_FAULTADDRESS_HI, ++ AS_STATUS ++}; ++ ++bool kbase_debug_job_fault_reg_snapshot_init(struct kbase_context *kctx, ++ int reg_range) ++{ ++ int i, j; ++ int offset = 0; ++ int slot_number; ++ int as_number; ++ ++ if (kctx->reg_dump == NULL) ++ return false; ++ ++ slot_number = kctx->kbdev->gpu_props.num_job_slots; ++ as_number = kctx->kbdev->gpu_props.num_address_spaces; ++ ++ /* get the GPU control registers*/ ++ for (i = 0; i < sizeof(gpu_control_reg_snapshot)/4; i++) { ++ kctx->reg_dump[offset] = ++ GPU_CONTROL_REG(gpu_control_reg_snapshot[i]); ++ offset += 2; ++ } ++ ++ /* get the Job control registers*/ ++ for (i = 0; i < sizeof(job_control_reg_snapshot)/4; i++) { ++ kctx->reg_dump[offset] = ++ JOB_CONTROL_REG(job_control_reg_snapshot[i]); ++ offset += 2; ++ } ++ ++ /* get the Job Slot registers*/ ++ for (j = 0; j < slot_number; j++) { ++ for (i = 0; i < sizeof(job_slot_reg_snapshot)/4; i++) { ++ kctx->reg_dump[offset] = ++ JOB_SLOT_REG(j, job_slot_reg_snapshot[i]); ++ offset += 2; ++ } ++ } ++ ++ /* get the MMU registers*/ ++ for (i = 0; i < sizeof(mmu_reg_snapshot)/4; i++) { ++ kctx->reg_dump[offset] = MMU_REG(mmu_reg_snapshot[i]); ++ offset += 2; ++ } ++ ++ /* get the Address space registers*/ ++ for (j = 0; j < as_number; j++) { ++ for (i = 0; i < sizeof(as_reg_snapshot)/4; i++) { ++ kctx->reg_dump[offset] = ++ MMU_AS_REG(j, as_reg_snapshot[i]); ++ offset += 2; ++ } ++ } ++ ++ WARN_ON(offset >= (reg_range*2/4)); ++ ++ /* set the termination flag*/ ++ kctx->reg_dump[offset] = REGISTER_DUMP_TERMINATION_FLAG; ++ kctx->reg_dump[offset + 1] = REGISTER_DUMP_TERMINATION_FLAG; ++ ++ dev_dbg(kctx->kbdev->dev, "kbase_job_fault_reg_snapshot_init:%d\n", ++ offset); ++ ++ return true; ++} ++ ++bool kbase_job_fault_get_reg_snapshot(struct kbase_context *kctx) ++{ ++ int offset = 0; ++ ++ if (kctx->reg_dump == NULL) ++ return false; ++ ++ while (kctx->reg_dump[offset] != REGISTER_DUMP_TERMINATION_FLAG) { ++ kctx->reg_dump[offset+1] = ++ kbase_reg_read(kctx->kbdev, ++ kctx->reg_dump[offset], NULL); ++ offset += 2; ++ } ++ return true; ++} ++ ++ ++#endif +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_devfreq.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_devfreq.c +new file mode 100755 +index 000000000000..003af36d8fe2 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_devfreq.c +@@ -0,0 +1,495 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include ++#include ++#include ++#include ++ ++#include ++#include ++#include ++#ifdef CONFIG_DEVFREQ_THERMAL ++#include ++#endif ++ ++#include ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0) ++#include ++#else /* Linux >= 3.13 */ ++/* In 3.13 the OPP include header file, types, and functions were all ++ * renamed. Use the old filename for the include, and define the new names to ++ * the old, when an old kernel is detected. ++ */ ++#include ++#define dev_pm_opp opp ++#define dev_pm_opp_get_voltage opp_get_voltage ++#define dev_pm_opp_get_opp_count opp_get_opp_count ++#define dev_pm_opp_find_freq_ceil opp_find_freq_ceil ++#define dev_pm_opp_find_freq_floor opp_find_freq_floor ++#endif /* Linux >= 3.13 */ ++ ++#include ++#include ++#include ++ ++static struct devfreq_simple_ondemand_data ondemand_data; ++ ++static struct monitor_dev_profile mali_mdevp = { ++ .type = MONITOR_TPYE_DEV, ++ .low_temp_adjust = rockchip_monitor_dev_low_temp_adjust, ++ .high_temp_adjust = rockchip_monitor_dev_high_temp_adjust, ++}; ++ ++/** ++ * opp_translate - Translate nominal OPP frequency from devicetree into real ++ * frequency and core mask ++ * @kbdev: Device pointer ++ * @freq: Nominal frequency ++ * @core_mask: Pointer to u64 to store core mask to ++ * ++ * Return: Real target frequency ++ * ++ * This function will only perform translation if an operating-points-v2-mali ++ * table is present in devicetree. If one is not present then it will return an ++ * untranslated frequency and all cores enabled. ++ */ ++static unsigned long opp_translate(struct kbase_device *kbdev, ++ unsigned long freq, u64 *core_mask) ++{ ++ int i; ++ ++ for (i = 0; i < kbdev->num_opps; i++) { ++ if (kbdev->opp_table[i].opp_freq == freq) { ++ *core_mask = kbdev->opp_table[i].core_mask; ++ return kbdev->opp_table[i].real_freq; ++ } ++ } ++ ++ /* Failed to find OPP - return all cores enabled & nominal frequency */ ++ *core_mask = kbdev->gpu_props.props.raw_props.shader_present; ++ ++ return freq; ++} ++ ++static int ++kbase_devfreq_target(struct device *dev, unsigned long *target_freq, u32 flags) ++{ ++ struct kbase_device *kbdev = dev_get_drvdata(dev); ++ struct dev_pm_opp *opp; ++ unsigned long nominal_freq; ++ unsigned long freq = 0; ++ unsigned long voltage; ++ int err; ++ u64 core_mask; ++ ++ freq = *target_freq; ++ ++ opp = devfreq_recommended_opp(dev, &freq, flags); ++ if (IS_ERR(opp)) { ++ dev_err(dev, "Failed to get opp (%ld)\n", PTR_ERR(opp)); ++ return PTR_ERR(opp); ++ } ++ voltage = dev_pm_opp_get_voltage(opp); ++ dev_pm_opp_put(opp); ++ ++ nominal_freq = freq; ++ /* ++ * Only update if there is a change of frequency ++ */ ++ if (kbdev->current_nominal_freq == nominal_freq) { ++ *target_freq = nominal_freq; ++#ifdef CONFIG_REGULATOR ++ if (kbdev->current_voltage == voltage) ++ return 0; ++ err = regulator_set_voltage(kbdev->regulator, voltage, INT_MAX); ++ if (err) { ++ dev_err(dev, "Failed to set voltage (%d)\n", err); ++ return err; ++ } ++ kbdev->current_voltage = voltage; ++#endif ++ return 0; ++ } ++ ++ freq = opp_translate(kbdev, nominal_freq, &core_mask); ++#ifdef CONFIG_REGULATOR ++ if (kbdev->regulator && kbdev->current_voltage != voltage ++ && kbdev->current_freq < freq) { ++ err = regulator_set_voltage(kbdev->regulator, voltage, INT_MAX); ++ if (err) { ++ dev_err(dev, "Failed to increase voltage (%d)\n", err); ++ return err; ++ } ++ } ++#endif ++ ++ err = clk_set_rate(kbdev->clock, freq); ++ if (err) { ++ dev_err(dev, "Failed to set clock %lu (target %lu)\n", ++ freq, *target_freq); ++ return err; ++ } ++ ++#ifdef CONFIG_REGULATOR ++ if (kbdev->regulator && kbdev->current_voltage != voltage ++ && kbdev->current_freq > freq) { ++ err = regulator_set_voltage(kbdev->regulator, voltage, INT_MAX); ++ if (err) { ++ dev_err(dev, "Failed to decrease voltage (%d)\n", err); ++ return err; ++ } ++ } ++#endif ++ ++ if (kbdev->pm.backend.ca_current_policy->id == ++ KBASE_PM_CA_POLICY_ID_DEVFREQ) ++ kbase_devfreq_set_core_mask(kbdev, core_mask); ++ ++ *target_freq = nominal_freq; ++ kbdev->current_voltage = voltage; ++ kbdev->current_nominal_freq = nominal_freq; ++ kbdev->current_freq = freq; ++ kbdev->current_core_mask = core_mask; ++ if (kbdev->devfreq) ++ kbdev->devfreq->last_status.current_frequency = nominal_freq; ++ ++ KBASE_TLSTREAM_AUX_DEVFREQ_TARGET((u64)nominal_freq); ++ ++ kbase_pm_reset_dvfs_utilisation(kbdev); ++ ++ return err; ++} ++ ++static int ++kbase_devfreq_cur_freq(struct device *dev, unsigned long *freq) ++{ ++ struct kbase_device *kbdev = dev_get_drvdata(dev); ++ ++ *freq = kbdev->current_nominal_freq; ++ ++ return 0; ++} ++ ++static int ++kbase_devfreq_status(struct device *dev, struct devfreq_dev_status *stat) ++{ ++ struct kbase_device *kbdev = dev_get_drvdata(dev); ++ ++ kbase_pm_get_dvfs_utilisation(kbdev, ++ &stat->total_time, &stat->busy_time); ++ ++ stat->private_data = NULL; ++ ++ return 0; ++} ++ ++static int kbase_devfreq_init_freq_table(struct kbase_device *kbdev, ++ struct devfreq_dev_profile *dp) ++{ ++ int count; ++ int i = 0; ++ unsigned long freq; ++ struct dev_pm_opp *opp; ++ ++ count = dev_pm_opp_get_opp_count(kbdev->dev); ++ if (count < 0) { ++ return count; ++ } ++ ++ dp->freq_table = kmalloc_array(count, sizeof(dp->freq_table[0]), ++ GFP_KERNEL); ++ if (!dp->freq_table) ++ return -ENOMEM; ++ ++ for (i = 0, freq = ULONG_MAX; i < count; i++, freq--) { ++ opp = dev_pm_opp_find_freq_floor(kbdev->dev, &freq); ++ if (IS_ERR(opp)) ++ break; ++ dev_pm_opp_put(opp); ++ ++ dp->freq_table[i] = freq; ++ } ++ ++ if (count != i) ++ dev_warn(kbdev->dev, "Unable to enumerate all OPPs (%d!=%d\n", ++ count, i); ++ ++ dp->max_state = i; ++ ++ return 0; ++} ++ ++static void kbase_devfreq_term_freq_table(struct kbase_device *kbdev) ++{ ++ struct devfreq_dev_profile *dp = kbdev->devfreq->profile; ++ ++ kfree(dp->freq_table); ++} ++ ++static void kbase_devfreq_exit(struct device *dev) ++{ ++ struct kbase_device *kbdev = dev_get_drvdata(dev); ++ ++ kbase_devfreq_term_freq_table(kbdev); ++} ++ ++static int kbase_devfreq_init_core_mask_table(struct kbase_device *kbdev) ++{ ++ struct device_node *opp_node = of_parse_phandle(kbdev->dev->of_node, ++ "operating-points-v2", 0); ++ struct device_node *node; ++ int i = 0; ++ int count; ++ ++ if (!opp_node) ++ return 0; ++ if (!of_device_is_compatible(opp_node, "operating-points-v2-mali")) ++ return 0; ++ ++ count = dev_pm_opp_get_opp_count(kbdev->dev); ++ kbdev->opp_table = kmalloc_array(count, ++ sizeof(struct kbase_devfreq_opp), GFP_KERNEL); ++ if (!kbdev->opp_table) ++ return -ENOMEM; ++ ++ for_each_available_child_of_node(opp_node, node) { ++ u64 core_mask; ++ u64 opp_freq, real_freq; ++ const void *core_count_p; ++ ++ if (of_property_read_u64(node, "opp-hz", &opp_freq)) { ++ dev_warn(kbdev->dev, "OPP is missing required opp-hz property\n"); ++ continue; ++ } ++ if (of_property_read_u64(node, "opp-hz-real", &real_freq)) ++ real_freq = opp_freq; ++ if (of_property_read_u64(node, "opp-core-mask", &core_mask)) ++ core_mask = ++ kbdev->gpu_props.props.raw_props.shader_present; ++ core_count_p = of_get_property(node, "opp-core-count", NULL); ++ if (core_count_p) { ++ u64 remaining_core_mask = ++ kbdev->gpu_props.props.raw_props.shader_present; ++ int core_count = be32_to_cpup(core_count_p); ++ ++ core_mask = 0; ++ ++ for (; core_count > 0; core_count--) { ++ int core = ffs(remaining_core_mask); ++ ++ if (!core) { ++ dev_err(kbdev->dev, "OPP has more cores than GPU\n"); ++ return -ENODEV; ++ } ++ ++ core_mask |= (1ull << (core-1)); ++ remaining_core_mask &= ~(1ull << (core-1)); ++ } ++ } ++ ++ if (!core_mask) { ++ dev_err(kbdev->dev, "OPP has invalid core mask of 0\n"); ++ return -ENODEV; ++ } ++ ++ kbdev->opp_table[i].opp_freq = opp_freq; ++ kbdev->opp_table[i].real_freq = real_freq; ++ kbdev->opp_table[i].core_mask = core_mask; ++ ++ dev_info(kbdev->dev, "OPP %d : opp_freq=%llu real_freq=%llu core_mask=%llx\n", ++ i, opp_freq, real_freq, core_mask); ++ ++ i++; ++ } ++ ++ kbdev->num_opps = i; ++ ++ return 0; ++} ++ ++static unsigned long kbase_devfreq_get_static_power(struct devfreq *devfreq, ++ unsigned long voltage) ++{ ++ struct device *dev = devfreq->dev.parent; ++ struct kbase_device *kbdev = dev_get_drvdata(dev); ++ ++ return rockchip_ipa_get_static_power(kbdev->model_data, voltage); ++} ++ ++static struct devfreq_cooling_power kbase_cooling_power = { ++ .get_static_power = &kbase_devfreq_get_static_power, ++}; ++ ++int kbase_devfreq_init(struct kbase_device *kbdev) ++{ ++ struct device_node *np = kbdev->dev->of_node; ++ struct devfreq_dev_profile *dp; ++ struct dev_pm_opp *opp; ++ unsigned long opp_rate; ++ int err; ++ ++ if (!kbdev->clock) { ++ dev_err(kbdev->dev, "Clock not available for devfreq\n"); ++ return -ENODEV; ++ } ++ ++ kbdev->current_freq = clk_get_rate(kbdev->clock); ++ kbdev->current_nominal_freq = kbdev->current_freq; ++ ++ dp = &kbdev->devfreq_profile; ++ ++ dp->initial_freq = kbdev->current_freq; ++ dp->polling_ms = 100; ++ dp->target = kbase_devfreq_target; ++ dp->get_dev_status = kbase_devfreq_status; ++ dp->get_cur_freq = kbase_devfreq_cur_freq; ++ dp->exit = kbase_devfreq_exit; ++ ++ if (kbase_devfreq_init_freq_table(kbdev, dp)) ++ return -EFAULT; ++ ++ err = kbase_devfreq_init_core_mask_table(kbdev); ++ if (err) ++ return err; ++ of_property_read_u32(np, "upthreshold", ++ &ondemand_data.upthreshold); ++ of_property_read_u32(np, "downdifferential", ++ &ondemand_data.downdifferential); ++ kbdev->devfreq = devfreq_add_device(kbdev->dev, dp, ++ "simple_ondemand", &ondemand_data); ++ if (IS_ERR(kbdev->devfreq)) { ++ kbase_devfreq_term_freq_table(kbdev); ++ return PTR_ERR(kbdev->devfreq); ++ } ++ ++ /* devfreq_add_device only copies a few of kbdev->dev's fields, so ++ * set drvdata explicitly so IPA models can access kbdev. */ ++ dev_set_drvdata(&kbdev->devfreq->dev, kbdev); ++ ++ err = devfreq_register_opp_notifier(kbdev->dev, kbdev->devfreq); ++ if (err) { ++ dev_err(kbdev->dev, ++ "Failed to register OPP notifier (%d)\n", err); ++ goto opp_notifier_failed; ++ } ++ ++ opp_rate = kbdev->current_freq; ++ opp = devfreq_recommended_opp(kbdev->dev, &opp_rate, 0); ++ if (!IS_ERR(opp)) ++ dev_pm_opp_put(opp); ++ kbdev->devfreq->last_status.current_frequency = opp_rate; ++ ++ mali_mdevp.data = kbdev->devfreq; ++ kbdev->mdev_info = rockchip_system_monitor_register(kbdev->dev, ++ &mali_mdevp); ++ if (IS_ERR(kbdev->mdev_info)) { ++ dev_dbg(kbdev->dev, "without system monitor\n"); ++ kbdev->mdev_info = NULL; ++ } ++#ifdef CONFIG_DEVFREQ_THERMAL ++ if (of_find_compatible_node(kbdev->dev->of_node, NULL, ++ "simple-power-model")) { ++ of_property_read_u32(kbdev->dev->of_node, ++ "dynamic-power-coefficient", ++ (u32 *)&kbase_dcp->dyn_power_coeff); ++ kbdev->model_data = rockchip_ipa_power_model_init(kbdev->dev, ++ "gpu_leakage"); ++ if (IS_ERR_OR_NULL(kbdev->model_data)) { ++ kbdev->model_data = NULL; ++ dev_err(kbdev->dev, "failed to initialize power model\n"); ++ } else if (kbdev->model_data->dynamic_coefficient) { ++ kbase_dcp->dyn_power_coeff = ++ kbdev->model_data->dynamic_coefficient; ++ } ++ if (!kbase_dcp->dyn_power_coeff) { ++ dev_err(kbdev->dev, "failed to get dynamic-coefficient\n"); ++ err = -EINVAL; ++ goto cooling_failed; ++ } ++ ++ kbdev->devfreq_cooling = ++ of_devfreq_cooling_register_power(kbdev->dev->of_node, ++ kbdev->devfreq, ++ kbase_dcp); ++ if (IS_ERR(kbdev->devfreq_cooling)) { ++ dev_err(kbdev->dev, "failed to register cooling device\n"); ++ err = PTR_ERR(kbdev->devfreq_cooling); ++ goto cooling_failed; ++ } ++ } else { ++ err = kbase_ipa_init(kbdev); ++ if (err) { ++ dev_err(kbdev->dev, "IPA initialization failed\n"); ++ goto cooling_failed; ++ } ++ ++ kbdev->devfreq_cooling = of_devfreq_cooling_register_power( ++ kbdev->dev->of_node, ++ kbdev->devfreq, ++ &kbase_ipa_power_model_ops); ++ if (IS_ERR(kbdev->devfreq_cooling)) { ++ err = PTR_ERR(kbdev->devfreq_cooling); ++ dev_err(kbdev->dev, ++ "Failed to register cooling device (%d)\n", ++ err); ++ goto cooling_failed; ++ } ++ } ++#endif ++ ++ return 0; ++ ++#ifdef CONFIG_DEVFREQ_THERMAL ++cooling_failed: ++ devfreq_unregister_opp_notifier(kbdev->dev, kbdev->devfreq); ++#endif /* CONFIG_DEVFREQ_THERMAL */ ++opp_notifier_failed: ++ if (devfreq_remove_device(kbdev->devfreq)) ++ dev_err(kbdev->dev, "Failed to terminate devfreq (%d)\n", err); ++ else ++ kbdev->devfreq = NULL; ++ ++ return err; ++} ++ ++void kbase_devfreq_term(struct kbase_device *kbdev) ++{ ++ int err; ++ ++ dev_dbg(kbdev->dev, "Term Mali devfreq\n"); ++ ++ rockchip_system_monitor_unregister(kbdev->mdev_info); ++#ifdef CONFIG_DEVFREQ_THERMAL ++ if (kbdev->devfreq_cooling) ++ devfreq_cooling_unregister(kbdev->devfreq_cooling); ++ ++ if (!kbdev->model_data) ++ kbase_ipa_term(kbdev); ++ kfree(kbdev->model_data); ++#endif ++ ++ devfreq_unregister_opp_notifier(kbdev->dev, kbdev->devfreq); ++ ++ err = devfreq_remove_device(kbdev->devfreq); ++ if (err) ++ dev_err(kbdev->dev, "Failed to terminate devfreq (%d)\n", err); ++ else ++ kbdev->devfreq = NULL; ++ ++ kfree(kbdev->opp_table); ++} +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_devfreq.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_devfreq.h +new file mode 100755 +index 000000000000..7bcc350f3006 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_devfreq.h +@@ -0,0 +1,25 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _BASE_DEVFREQ_H_ ++#define _BASE_DEVFREQ_H_ ++ ++int kbase_devfreq_init(struct kbase_device *kbdev); ++void kbase_devfreq_term(struct kbase_device *kbdev); ++int kbase_platform_rk_init_opp_table(struct kbase_device *kbdev); ++ ++#endif /* _BASE_DEVFREQ_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_device_hw.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_device_hw.c +new file mode 100755 +index 000000000000..17f253308ffc +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_device_hw.c +@@ -0,0 +1,255 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++/* ++ * ++ */ ++#include ++#include ++#include ++ ++#include ++ ++#if !defined(CONFIG_MALI_BIFROST_NO_MALI) ++ ++ ++#ifdef CONFIG_DEBUG_FS ++ ++ ++int kbase_io_history_resize(struct kbase_io_history *h, u16 new_size) ++{ ++ struct kbase_io_access *old_buf; ++ struct kbase_io_access *new_buf; ++ unsigned long flags; ++ ++ if (!new_size) ++ goto out_err; /* The new size must not be 0 */ ++ ++ new_buf = vmalloc(new_size * sizeof(*h->buf)); ++ if (!new_buf) ++ goto out_err; ++ ++ spin_lock_irqsave(&h->lock, flags); ++ ++ old_buf = h->buf; ++ ++ /* Note: we won't bother with copying the old data over. The dumping ++ * logic wouldn't work properly as it relies on 'count' both as a ++ * counter and as an index to the buffer which would have changed with ++ * the new array. This is a corner case that we don't need to support. ++ */ ++ h->count = 0; ++ h->size = new_size; ++ h->buf = new_buf; ++ ++ spin_unlock_irqrestore(&h->lock, flags); ++ ++ vfree(old_buf); ++ ++ return 0; ++ ++out_err: ++ return -1; ++} ++ ++ ++int kbase_io_history_init(struct kbase_io_history *h, u16 n) ++{ ++ h->enabled = false; ++ spin_lock_init(&h->lock); ++ h->count = 0; ++ h->size = 0; ++ h->buf = NULL; ++ if (kbase_io_history_resize(h, n)) ++ return -1; ++ ++ return 0; ++} ++ ++ ++void kbase_io_history_term(struct kbase_io_history *h) ++{ ++ vfree(h->buf); ++ h->buf = NULL; ++} ++ ++ ++/* kbase_io_history_add - add new entry to the register access history ++ * ++ * @h: Pointer to the history data structure ++ * @addr: Register address ++ * @value: The value that is either read from or written to the register ++ * @write: 1 if it's a register write, 0 if it's a read ++ */ ++static void kbase_io_history_add(struct kbase_io_history *h, ++ void __iomem const *addr, u32 value, u8 write) ++{ ++ struct kbase_io_access *io; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&h->lock, flags); ++ ++ io = &h->buf[h->count % h->size]; ++ io->addr = (uintptr_t)addr | write; ++ io->value = value; ++ ++h->count; ++ /* If count overflows, move the index by the buffer size so the entire ++ * buffer will still be dumped later */ ++ if (unlikely(!h->count)) ++ h->count = h->size; ++ ++ spin_unlock_irqrestore(&h->lock, flags); ++} ++ ++ ++void kbase_io_history_dump(struct kbase_device *kbdev) ++{ ++ struct kbase_io_history *const h = &kbdev->io_history; ++ u16 i; ++ size_t iters; ++ unsigned long flags; ++ ++ if (!unlikely(h->enabled)) ++ return; ++ ++ spin_lock_irqsave(&h->lock, flags); ++ ++ dev_err(kbdev->dev, "Register IO History:"); ++ iters = (h->size > h->count) ? h->count : h->size; ++ dev_err(kbdev->dev, "Last %zu register accesses of %zu total:\n", iters, ++ h->count); ++ for (i = 0; i < iters; ++i) { ++ struct kbase_io_access *io = ++ &h->buf[(h->count - iters + i) % h->size]; ++ char const access = (io->addr & 1) ? 'w' : 'r'; ++ ++ dev_err(kbdev->dev, "%6i: %c: reg 0x%p val %08x\n", i, access, ++ (void *)(io->addr & ~0x1), io->value); ++ } ++ ++ spin_unlock_irqrestore(&h->lock, flags); ++} ++ ++ ++#endif /* CONFIG_DEBUG_FS */ ++ ++ ++void kbase_reg_write(struct kbase_device *kbdev, u16 offset, u32 value, ++ struct kbase_context *kctx) ++{ ++ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered); ++ KBASE_DEBUG_ASSERT(kctx == NULL || kctx->as_nr != KBASEP_AS_NR_INVALID); ++ KBASE_DEBUG_ASSERT(kbdev->dev != NULL); ++ ++ writel(value, kbdev->reg + offset); ++ ++#ifdef CONFIG_DEBUG_FS ++ if (unlikely(kbdev->io_history.enabled)) ++ kbase_io_history_add(&kbdev->io_history, kbdev->reg + offset, ++ value, 1); ++#endif /* CONFIG_DEBUG_FS */ ++ dev_dbg(kbdev->dev, "w: reg %04x val %08x", offset, value); ++ ++ if (kctx && kctx->jctx.tb) ++ kbase_device_trace_register_access(kctx, REG_WRITE, offset, ++ value); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_reg_write); ++ ++u32 kbase_reg_read(struct kbase_device *kbdev, u16 offset, ++ struct kbase_context *kctx) ++{ ++ u32 val; ++ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered); ++ KBASE_DEBUG_ASSERT(kctx == NULL || kctx->as_nr != KBASEP_AS_NR_INVALID); ++ KBASE_DEBUG_ASSERT(kbdev->dev != NULL); ++ ++ val = readl(kbdev->reg + offset); ++ ++#ifdef CONFIG_DEBUG_FS ++ if (unlikely(kbdev->io_history.enabled)) ++ kbase_io_history_add(&kbdev->io_history, kbdev->reg + offset, ++ val, 0); ++#endif /* CONFIG_DEBUG_FS */ ++ dev_dbg(kbdev->dev, "r: reg %04x val %08x", offset, val); ++ ++ if (kctx && kctx->jctx.tb) ++ kbase_device_trace_register_access(kctx, REG_READ, offset, val); ++ return val; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_reg_read); ++#endif /* !defined(CONFIG_MALI_BIFROST_NO_MALI) */ ++ ++/** ++ * kbase_report_gpu_fault - Report a GPU fault. ++ * @kbdev: Kbase device pointer ++ * @multiple: Zero if only GPU_FAULT was raised, non-zero if MULTIPLE_GPU_FAULTS ++ * was also set ++ * ++ * This function is called from the interrupt handler when a GPU fault occurs. ++ * It reports the details of the fault using dev_warn(). ++ */ ++static void kbase_report_gpu_fault(struct kbase_device *kbdev, int multiple) ++{ ++ u32 status; ++ u64 address; ++ ++ status = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_FAULTSTATUS), NULL); ++ address = (u64) kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(GPU_FAULTADDRESS_HI), NULL) << 32; ++ address |= kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(GPU_FAULTADDRESS_LO), NULL); ++ ++ dev_warn(kbdev->dev, "GPU Fault 0x%08x (%s) at 0x%016llx", ++ status & 0xFF, ++ kbase_exception_name(kbdev, status), ++ address); ++ if (multiple) ++ dev_warn(kbdev->dev, "There were multiple GPU faults - some have not been reported\n"); ++} ++ ++void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val) ++{ ++ KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ, NULL, NULL, 0u, val); ++ if (val & GPU_FAULT) ++ kbase_report_gpu_fault(kbdev, val & MULTIPLE_GPU_FAULTS); ++ ++ if (val & RESET_COMPLETED) ++ kbase_pm_reset_done(kbdev); ++ ++ if (val & PRFCNT_SAMPLE_COMPLETED) ++ kbase_instr_hwcnt_sample_done(kbdev); ++ ++ if (val & CLEAN_CACHES_COMPLETED) ++ kbase_clean_caches_done(kbdev); ++ ++ KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, NULL, 0u, val); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), val, NULL); ++ ++ /* kbase_pm_check_transitions must be called after the IRQ has been ++ * cleared. This is because it might trigger further power transitions ++ * and we don't want to miss the interrupt raised to notify us that ++ * these further transitions have finished. ++ */ ++ if (val & POWER_CHANGED_ALL) ++ kbase_pm_power_changed(kbdev); ++ ++ KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ_DONE, NULL, NULL, 0u, val); ++} +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_device_internal.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_device_internal.h +new file mode 100755 +index 000000000000..5b20445932fb +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_device_internal.h +@@ -0,0 +1,67 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * Backend-specific HW access device APIs ++ */ ++ ++#ifndef _KBASE_DEVICE_INTERNAL_H_ ++#define _KBASE_DEVICE_INTERNAL_H_ ++ ++/** ++ * kbase_reg_write - write to GPU register ++ * @kbdev: Kbase device pointer ++ * @offset: Offset of register ++ * @value: Value to write ++ * @kctx: Kbase context pointer. May be NULL ++ * ++ * Caller must ensure the GPU is powered (@kbdev->pm.gpu_powered != false). If ++ * @kctx is not NULL then the caller must ensure it is scheduled (@kctx->as_nr ++ * != KBASEP_AS_NR_INVALID). ++ */ ++void kbase_reg_write(struct kbase_device *kbdev, u16 offset, u32 value, ++ struct kbase_context *kctx); ++ ++/** ++ * kbase_reg_read - read from GPU register ++ * @kbdev: Kbase device pointer ++ * @offset: Offset of register ++ * @kctx: Kbase context pointer. May be NULL ++ * ++ * Caller must ensure the GPU is powered (@kbdev->pm.gpu_powered != false). If ++ * @kctx is not NULL then the caller must ensure it is scheduled (@kctx->as_nr ++ * != KBASEP_AS_NR_INVALID). ++ * ++ * Return: Value in desired register ++ */ ++u32 kbase_reg_read(struct kbase_device *kbdev, u16 offset, ++ struct kbase_context *kctx); ++ ++ ++/** ++ * kbase_gpu_interrupt - GPU interrupt handler ++ * @kbdev: Kbase device pointer ++ * @val: The value of the GPU IRQ status register which triggered the call ++ * ++ * This function is called from the interrupt handler when a GPU irq is to be ++ * handled. ++ */ ++void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val); ++ ++#endif /* _KBASE_DEVICE_INTERNAL_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_gpu.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_gpu.c +new file mode 100755 +index 000000000000..a7c3a77d8ac8 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_gpu.c +@@ -0,0 +1,123 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++/* ++ * Register-based HW access backend APIs ++ */ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++int kbase_backend_early_init(struct kbase_device *kbdev) ++{ ++ int err; ++ ++ err = kbasep_platform_device_init(kbdev); ++ if (err) ++ return err; ++ ++ /* Ensure we can access the GPU registers */ ++ kbase_pm_register_access_enable(kbdev); ++ ++ /* Find out GPU properties based on the GPU feature registers */ ++ kbase_gpuprops_set(kbdev); ++ ++ /* We're done accessing the GPU registers for now. */ ++ kbase_pm_register_access_disable(kbdev); ++ ++ err = kbase_install_interrupts(kbdev); ++ if (err) ++ goto fail_interrupts; ++ ++ err = kbase_hwaccess_pm_init(kbdev); ++ if (err) ++ goto fail_pm; ++ ++ return 0; ++ ++fail_pm: ++ kbase_release_interrupts(kbdev); ++fail_interrupts: ++ kbasep_platform_device_term(kbdev); ++ ++ return err; ++} ++ ++void kbase_backend_early_term(struct kbase_device *kbdev) ++{ ++ kbase_hwaccess_pm_term(kbdev); ++ kbase_release_interrupts(kbdev); ++ kbasep_platform_device_term(kbdev); ++} ++ ++int kbase_backend_late_init(struct kbase_device *kbdev) ++{ ++ int err; ++ ++ err = kbase_hwaccess_pm_powerup(kbdev, PM_HW_ISSUES_DETECT); ++ if (err) ++ return err; ++ ++ err = kbase_backend_timer_init(kbdev); ++ if (err) ++ goto fail_timer; ++ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++#ifndef CONFIG_MALI_BIFROST_NO_MALI ++ if (kbasep_common_test_interrupt_handlers(kbdev) != 0) { ++ dev_err(kbdev->dev, "Interrupt assigment check failed.\n"); ++ err = -EINVAL; ++ goto fail_interrupt_test; ++ } ++#endif /* !CONFIG_MALI_BIFROST_NO_MALI */ ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++ ++ err = kbase_job_slot_init(kbdev); ++ if (err) ++ goto fail_job_slot; ++ ++ init_waitqueue_head(&kbdev->hwaccess.backend.reset_wait); ++ ++ return 0; ++ ++fail_job_slot: ++ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++#ifndef CONFIG_MALI_BIFROST_NO_MALI ++fail_interrupt_test: ++#endif /* !CONFIG_MALI_BIFROST_NO_MALI */ ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++ ++ kbase_backend_timer_term(kbdev); ++fail_timer: ++ kbase_hwaccess_pm_halt(kbdev); ++ ++ return err; ++} ++ ++void kbase_backend_late_term(struct kbase_device *kbdev) ++{ ++ kbase_job_slot_halt(kbdev); ++ kbase_job_slot_term(kbdev); ++ kbase_backend_timer_term(kbdev); ++ kbase_hwaccess_pm_halt(kbdev); ++} ++ +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_gpuprops_backend.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_gpuprops_backend.c +new file mode 100755 +index 000000000000..b395325b556b +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_gpuprops_backend.c +@@ -0,0 +1,110 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * Base kernel property query backend APIs ++ */ ++ ++#include ++#include ++#include ++#include ++ ++void kbase_backend_gpuprops_get(struct kbase_device *kbdev, ++ struct kbase_gpuprops_regdump *regdump) ++{ ++ int i; ++ ++ /* Fill regdump with the content of the relevant registers */ ++ regdump->gpu_id = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_ID), NULL); ++ ++ regdump->l2_features = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(L2_FEATURES), NULL); ++ regdump->suspend_size = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(SUSPEND_SIZE), NULL); ++ regdump->tiler_features = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(TILER_FEATURES), NULL); ++ regdump->mem_features = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(MEM_FEATURES), NULL); ++ regdump->mmu_features = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(MMU_FEATURES), NULL); ++ regdump->as_present = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(AS_PRESENT), NULL); ++ regdump->js_present = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(JS_PRESENT), NULL); ++ ++ for (i = 0; i < GPU_MAX_JOB_SLOTS; i++) ++ regdump->js_features[i] = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(JS_FEATURES_REG(i)), NULL); ++ ++ for (i = 0; i < BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS; i++) ++ regdump->texture_features[i] = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(TEXTURE_FEATURES_REG(i)), NULL); ++ ++ regdump->thread_max_threads = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(THREAD_MAX_THREADS), NULL); ++ regdump->thread_max_workgroup_size = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(THREAD_MAX_WORKGROUP_SIZE), ++ NULL); ++ regdump->thread_max_barrier_size = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(THREAD_MAX_BARRIER_SIZE), NULL); ++ regdump->thread_features = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(THREAD_FEATURES), NULL); ++ ++ regdump->shader_present_lo = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(SHADER_PRESENT_LO), NULL); ++ regdump->shader_present_hi = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(SHADER_PRESENT_HI), NULL); ++ ++ regdump->tiler_present_lo = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(TILER_PRESENT_LO), NULL); ++ regdump->tiler_present_hi = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(TILER_PRESENT_HI), NULL); ++ ++ regdump->l2_present_lo = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(L2_PRESENT_LO), NULL); ++ regdump->l2_present_hi = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(L2_PRESENT_HI), NULL); ++ ++ regdump->stack_present_lo = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(STACK_PRESENT_LO), NULL); ++ regdump->stack_present_hi = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(STACK_PRESENT_HI), NULL); ++} ++ ++void kbase_backend_gpuprops_get_features(struct kbase_device *kbdev, ++ struct kbase_gpuprops_regdump *regdump) ++{ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_COHERENCY_REG)) { ++ /* Ensure we can access the GPU registers */ ++ kbase_pm_register_access_enable(kbdev); ++ ++ regdump->coherency_features = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(COHERENCY_FEATURES), NULL); ++ ++ /* We're done accessing the GPU registers for now. */ ++ kbase_pm_register_access_disable(kbdev); ++ } else { ++ /* Pre COHERENCY_FEATURES we only supported ACE_LITE */ ++ regdump->coherency_features = ++ COHERENCY_FEATURE_BIT(COHERENCY_NONE) | ++ COHERENCY_FEATURE_BIT(COHERENCY_ACE_LITE); ++ } ++} ++ +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_instr_backend.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_instr_backend.c +new file mode 100755 +index 000000000000..8084d054cc5b +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_instr_backend.c +@@ -0,0 +1,492 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * GPU backend instrumentation APIs. ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++/** ++ * kbasep_instr_hwcnt_cacheclean - Issue Cache Clean & Invalidate command to ++ * hardware ++ * ++ * @kbdev: Kbase device ++ */ ++static void kbasep_instr_hwcnt_cacheclean(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ unsigned long pm_flags; ++ u32 irq_mask; ++ ++ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); ++ KBASE_DEBUG_ASSERT(kbdev->hwcnt.backend.state == ++ KBASE_INSTR_STATE_REQUEST_CLEAN); ++ ++ /* Enable interrupt */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, pm_flags); ++ irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), ++ irq_mask | CLEAN_CACHES_COMPLETED, NULL); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); ++ ++ /* clean&invalidate the caches so we're sure the mmu tables for the dump ++ * buffer is valid */ ++ KBASE_TRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, NULL, 0u, 0); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), ++ GPU_COMMAND_CLEAN_INV_CACHES, NULL); ++ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_CLEANING; ++ ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++} ++ ++int kbase_instr_hwcnt_enable_internal(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ struct kbase_uk_hwcnt_setup *setup) ++{ ++ unsigned long flags, pm_flags; ++ int err = -EINVAL; ++ u32 irq_mask; ++ int ret; ++ u64 shader_cores_needed; ++ u32 prfcnt_config; ++ ++ shader_cores_needed = kbase_pm_get_present_cores(kbdev, ++ KBASE_PM_CORE_SHADER); ++ ++ /* alignment failure */ ++ if ((setup->dump_buffer == 0ULL) || (setup->dump_buffer & (2048 - 1))) ++ goto out_err; ++ ++ /* Override core availability policy to ensure all cores are available ++ */ ++ kbase_pm_ca_instr_enable(kbdev); ++ ++ /* Request the cores early on synchronously - we'll release them on any ++ * errors (e.g. instrumentation already active) */ ++ kbase_pm_request_cores_sync(kbdev, true, shader_cores_needed); ++ ++ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); ++ ++ if (kbdev->hwcnt.backend.state != KBASE_INSTR_STATE_DISABLED) { ++ /* Instrumentation is already enabled */ ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++ goto out_unrequest_cores; ++ } ++ ++ /* Enable interrupt */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, pm_flags); ++ irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), irq_mask | ++ PRFCNT_SAMPLE_COMPLETED, NULL); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); ++ ++ /* In use, this context is the owner */ ++ kbdev->hwcnt.kctx = kctx; ++ /* Remember the dump address so we can reprogram it later */ ++ kbdev->hwcnt.addr = setup->dump_buffer; ++ ++ /* Request the clean */ ++ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_REQUEST_CLEAN; ++ kbdev->hwcnt.backend.triggered = 0; ++ /* Clean&invalidate the caches so we're sure the mmu tables for the dump ++ * buffer is valid */ ++ ret = queue_work(kbdev->hwcnt.backend.cache_clean_wq, ++ &kbdev->hwcnt.backend.cache_clean_work); ++ KBASE_DEBUG_ASSERT(ret); ++ ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++ ++ /* Wait for cacheclean to complete */ ++ wait_event(kbdev->hwcnt.backend.wait, ++ kbdev->hwcnt.backend.triggered != 0); ++ ++ KBASE_DEBUG_ASSERT(kbdev->hwcnt.backend.state == ++ KBASE_INSTR_STATE_IDLE); ++ ++ kbase_pm_request_l2_caches(kbdev); ++ ++ /* Configure */ ++ prfcnt_config = kctx->as_nr << PRFCNT_CONFIG_AS_SHIFT; ++#ifdef CONFIG_MALI_BIFROST_PRFCNT_SET_SECONDARY ++ { ++ u32 gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; ++ u32 product_id = (gpu_id & GPU_ID_VERSION_PRODUCT_ID) ++ >> GPU_ID_VERSION_PRODUCT_ID_SHIFT; ++ int arch_v6 = GPU_ID_IS_NEW_FORMAT(product_id); ++ ++ if (arch_v6) ++ prfcnt_config |= 1 << PRFCNT_CONFIG_SETSELECT_SHIFT; ++ } ++#endif ++ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), ++ prfcnt_config | PRFCNT_CONFIG_MODE_OFF, kctx); ++ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_LO), ++ setup->dump_buffer & 0xFFFFFFFF, kctx); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_HI), ++ setup->dump_buffer >> 32, kctx); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_JM_EN), ++ setup->jm_bm, kctx); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_SHADER_EN), ++ setup->shader_bm, kctx); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_MMU_L2_EN), ++ setup->mmu_l2_bm, kctx); ++ /* Due to PRLAM-8186 we need to disable the Tiler before we enable the ++ * HW counter dump. */ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8186)) ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), 0, ++ kctx); ++ else ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), ++ setup->tiler_bm, kctx); ++ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), ++ prfcnt_config | PRFCNT_CONFIG_MODE_MANUAL, kctx); ++ ++ /* If HW has PRLAM-8186 we can now re-enable the tiler HW counters dump ++ */ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8186)) ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), ++ setup->tiler_bm, kctx); ++ ++ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); ++ ++ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; ++ kbdev->hwcnt.backend.triggered = 1; ++ wake_up(&kbdev->hwcnt.backend.wait); ++ ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++ ++ err = 0; ++ ++ dev_dbg(kbdev->dev, "HW counters dumping set-up for context %p", kctx); ++ return err; ++ out_unrequest_cores: ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_pm_unrequest_cores(kbdev, true, shader_cores_needed); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ out_err: ++ return err; ++} ++ ++int kbase_instr_hwcnt_disable_internal(struct kbase_context *kctx) ++{ ++ unsigned long flags, pm_flags; ++ int err = -EINVAL; ++ u32 irq_mask; ++ struct kbase_device *kbdev = kctx->kbdev; ++ ++ while (1) { ++ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); ++ ++ if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_DISABLED) { ++ /* Instrumentation is not enabled */ ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++ goto out; ++ } ++ ++ if (kbdev->hwcnt.kctx != kctx) { ++ /* Instrumentation has been setup for another context */ ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++ goto out; ++ } ++ ++ if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_IDLE) ++ break; ++ ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++ ++ /* Ongoing dump/setup - wait for its completion */ ++ wait_event(kbdev->hwcnt.backend.wait, ++ kbdev->hwcnt.backend.triggered != 0); ++ } ++ ++ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DISABLED; ++ kbdev->hwcnt.backend.triggered = 0; ++ ++ /* Disable interrupt */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, pm_flags); ++ irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), ++ irq_mask & ~PRFCNT_SAMPLE_COMPLETED, NULL); ++ ++ /* Disable the counters */ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), 0, kctx); ++ ++ kbdev->hwcnt.kctx = NULL; ++ kbdev->hwcnt.addr = 0ULL; ++ ++ kbase_pm_ca_instr_disable(kbdev); ++ ++ kbase_pm_unrequest_cores(kbdev, true, ++ kbase_pm_get_present_cores(kbdev, KBASE_PM_CORE_SHADER)); ++ ++ kbase_pm_release_l2_caches(kbdev); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++ ++ dev_dbg(kbdev->dev, "HW counters dumping disabled for context %p", ++ kctx); ++ ++ err = 0; ++ ++ out: ++ return err; ++} ++ ++int kbase_instr_hwcnt_request_dump(struct kbase_context *kctx) ++{ ++ unsigned long flags; ++ int err = -EINVAL; ++ struct kbase_device *kbdev = kctx->kbdev; ++ ++ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); ++ ++ if (kbdev->hwcnt.kctx != kctx) { ++ /* The instrumentation has been setup for another context */ ++ goto unlock; ++ } ++ ++ if (kbdev->hwcnt.backend.state != KBASE_INSTR_STATE_IDLE) { ++ /* HW counters are disabled or another dump is ongoing, or we're ++ * resetting */ ++ goto unlock; ++ } ++ ++ kbdev->hwcnt.backend.triggered = 0; ++ ++ /* Mark that we're dumping - the PF handler can signal that we faulted ++ */ ++ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DUMPING; ++ ++ /* Reconfigure the dump address */ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_LO), ++ kbdev->hwcnt.addr & 0xFFFFFFFF, NULL); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_HI), ++ kbdev->hwcnt.addr >> 32, NULL); ++ ++ /* Start dumping */ ++ KBASE_TRACE_ADD(kbdev, CORE_GPU_PRFCNT_SAMPLE, NULL, NULL, ++ kbdev->hwcnt.addr, 0); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), ++ GPU_COMMAND_PRFCNT_SAMPLE, kctx); ++ ++ dev_dbg(kbdev->dev, "HW counters dumping done for context %p", kctx); ++ ++ err = 0; ++ ++ unlock: ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++ return err; ++} ++KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_request_dump); ++ ++bool kbase_instr_hwcnt_dump_complete(struct kbase_context *kctx, ++ bool * const success) ++{ ++ unsigned long flags; ++ bool complete = false; ++ struct kbase_device *kbdev = kctx->kbdev; ++ ++ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); ++ ++ if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_IDLE) { ++ *success = true; ++ complete = true; ++ } else if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_FAULT) { ++ *success = false; ++ complete = true; ++ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; ++ } ++ ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++ ++ return complete; ++} ++KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_dump_complete); ++ ++void kbasep_cache_clean_worker(struct work_struct *data) ++{ ++ struct kbase_device *kbdev; ++ unsigned long flags; ++ ++ kbdev = container_of(data, struct kbase_device, ++ hwcnt.backend.cache_clean_work); ++ ++ mutex_lock(&kbdev->cacheclean_lock); ++ kbasep_instr_hwcnt_cacheclean(kbdev); ++ ++ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); ++ /* Wait for our condition, and any reset to complete */ ++ while (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_CLEANING) { ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++ wait_event(kbdev->hwcnt.backend.cache_clean_wait, ++ kbdev->hwcnt.backend.state != ++ KBASE_INSTR_STATE_CLEANING); ++ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); ++ } ++ KBASE_DEBUG_ASSERT(kbdev->hwcnt.backend.state == ++ KBASE_INSTR_STATE_CLEANED); ++ ++ /* All finished and idle */ ++ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; ++ kbdev->hwcnt.backend.triggered = 1; ++ wake_up(&kbdev->hwcnt.backend.wait); ++ ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++ mutex_unlock(&kbdev->cacheclean_lock); ++} ++ ++void kbase_instr_hwcnt_sample_done(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); ++ ++ if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_FAULT) { ++ kbdev->hwcnt.backend.triggered = 1; ++ wake_up(&kbdev->hwcnt.backend.wait); ++ } else if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_DUMPING) { ++ int ret; ++ /* Always clean and invalidate the cache after a successful dump ++ */ ++ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_REQUEST_CLEAN; ++ ret = queue_work(kbdev->hwcnt.backend.cache_clean_wq, ++ &kbdev->hwcnt.backend.cache_clean_work); ++ KBASE_DEBUG_ASSERT(ret); ++ } ++ ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++} ++ ++void kbase_clean_caches_done(struct kbase_device *kbdev) ++{ ++ u32 irq_mask; ++ ++ if (kbdev->hwcnt.backend.state != KBASE_INSTR_STATE_DISABLED) { ++ unsigned long flags; ++ unsigned long pm_flags; ++ ++ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); ++ /* Disable interrupt */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, pm_flags); ++ irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), ++ NULL); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), ++ irq_mask & ~CLEAN_CACHES_COMPLETED, NULL); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); ++ ++ /* Wakeup... */ ++ if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_CLEANING) { ++ /* Only wake if we weren't resetting */ ++ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_CLEANED; ++ wake_up(&kbdev->hwcnt.backend.cache_clean_wait); ++ } ++ ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++ } ++} ++ ++int kbase_instr_hwcnt_wait_for_dump(struct kbase_context *kctx) ++{ ++ struct kbase_device *kbdev = kctx->kbdev; ++ unsigned long flags; ++ int err; ++ ++ /* Wait for dump & cacheclean to complete */ ++ wait_event(kbdev->hwcnt.backend.wait, ++ kbdev->hwcnt.backend.triggered != 0); ++ ++ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); ++ ++ if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_FAULT) { ++ err = -EINVAL; ++ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; ++ } else { ++ /* Dump done */ ++ KBASE_DEBUG_ASSERT(kbdev->hwcnt.backend.state == ++ KBASE_INSTR_STATE_IDLE); ++ err = 0; ++ } ++ ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++ ++ return err; ++} ++ ++int kbase_instr_hwcnt_clear(struct kbase_context *kctx) ++{ ++ unsigned long flags; ++ int err = -EINVAL; ++ struct kbase_device *kbdev = kctx->kbdev; ++ ++ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); ++ ++ /* Check it's the context previously set up and we're not already ++ * dumping */ ++ if (kbdev->hwcnt.kctx != kctx || kbdev->hwcnt.backend.state != ++ KBASE_INSTR_STATE_IDLE) ++ goto out; ++ ++ /* Clear the counters */ ++ KBASE_TRACE_ADD(kbdev, CORE_GPU_PRFCNT_CLEAR, NULL, NULL, 0u, 0); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), ++ GPU_COMMAND_PRFCNT_CLEAR, kctx); ++ ++ err = 0; ++ ++out: ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++ return err; ++} ++KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_clear); ++ ++int kbase_instr_backend_init(struct kbase_device *kbdev) ++{ ++ int ret = 0; ++ ++ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DISABLED; ++ ++ init_waitqueue_head(&kbdev->hwcnt.backend.wait); ++ init_waitqueue_head(&kbdev->hwcnt.backend.cache_clean_wait); ++ INIT_WORK(&kbdev->hwcnt.backend.cache_clean_work, ++ kbasep_cache_clean_worker); ++ kbdev->hwcnt.backend.triggered = 0; ++ ++ kbdev->hwcnt.backend.cache_clean_wq = ++ alloc_workqueue("Mali cache cleaning workqueue", 0, 1); ++ if (NULL == kbdev->hwcnt.backend.cache_clean_wq) ++ ret = -EINVAL; ++ ++ return ret; ++} ++ ++void kbase_instr_backend_term(struct kbase_device *kbdev) ++{ ++ destroy_workqueue(kbdev->hwcnt.backend.cache_clean_wq); ++} ++ +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_instr_defs.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_instr_defs.h +new file mode 100755 +index 000000000000..4794672da8f0 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_instr_defs.h +@@ -0,0 +1,58 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014, 2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* ++ * Backend-specific instrumentation definitions ++ */ ++ ++#ifndef _KBASE_INSTR_DEFS_H_ ++#define _KBASE_INSTR_DEFS_H_ ++ ++/* ++ * Instrumentation State Machine States ++ */ ++enum kbase_instr_state { ++ /* State where instrumentation is not active */ ++ KBASE_INSTR_STATE_DISABLED = 0, ++ /* State machine is active and ready for a command. */ ++ KBASE_INSTR_STATE_IDLE, ++ /* Hardware is currently dumping a frame. */ ++ KBASE_INSTR_STATE_DUMPING, ++ /* We've requested a clean to occur on a workqueue */ ++ KBASE_INSTR_STATE_REQUEST_CLEAN, ++ /* Hardware is currently cleaning and invalidating caches. */ ++ KBASE_INSTR_STATE_CLEANING, ++ /* Cache clean completed, and either a) a dump is complete, or ++ * b) instrumentation can now be setup. */ ++ KBASE_INSTR_STATE_CLEANED, ++ /* An error has occured during DUMPING (page fault). */ ++ KBASE_INSTR_STATE_FAULT ++}; ++ ++/* Structure used for instrumentation and HW counters dumping */ ++struct kbase_instr_backend { ++ wait_queue_head_t wait; ++ int triggered; ++ ++ enum kbase_instr_state state; ++ wait_queue_head_t cache_clean_wait; ++ struct workqueue_struct *cache_clean_wq; ++ struct work_struct cache_clean_work; ++}; ++ ++#endif /* _KBASE_INSTR_DEFS_H_ */ ++ +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_instr_internal.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_instr_internal.h +new file mode 100755 +index 000000000000..e96aeae786e1 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_instr_internal.h +@@ -0,0 +1,45 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * Backend-specific HW access instrumentation APIs ++ */ ++ ++#ifndef _KBASE_INSTR_INTERNAL_H_ ++#define _KBASE_INSTR_INTERNAL_H_ ++ ++/** ++ * kbasep_cache_clean_worker() - Workqueue for handling cache cleaning ++ * @data: a &struct work_struct ++ */ ++void kbasep_cache_clean_worker(struct work_struct *data); ++ ++/** ++ * kbase_clean_caches_done() - Cache clean interrupt received ++ * @kbdev: Kbase device ++ */ ++void kbase_clean_caches_done(struct kbase_device *kbdev); ++ ++/** ++ * kbase_instr_hwcnt_sample_done() - Dump complete interrupt received ++ * @kbdev: Kbase device ++ */ ++void kbase_instr_hwcnt_sample_done(struct kbase_device *kbdev); ++ ++#endif /* _KBASE_INSTR_INTERNAL_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_irq_internal.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_irq_internal.h +new file mode 100755 +index 000000000000..8781561e73d0 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_irq_internal.h +@@ -0,0 +1,39 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* ++ * Backend specific IRQ APIs ++ */ ++ ++#ifndef _KBASE_IRQ_INTERNAL_H_ ++#define _KBASE_IRQ_INTERNAL_H_ ++ ++int kbase_install_interrupts(struct kbase_device *kbdev); ++ ++void kbase_release_interrupts(struct kbase_device *kbdev); ++ ++/** ++ * kbase_synchronize_irqs - Ensure that all IRQ handlers have completed ++ * execution ++ * @kbdev: The kbase device ++ */ ++void kbase_synchronize_irqs(struct kbase_device *kbdev); ++ ++int kbasep_common_test_interrupt_handlers( ++ struct kbase_device * const kbdev); ++ ++#endif /* _KBASE_IRQ_INTERNAL_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_irq_linux.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_irq_linux.c +new file mode 100755 +index 000000000000..d0666c86cf59 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_irq_linux.c +@@ -0,0 +1,469 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include ++#include ++#include ++ ++#include ++ ++#if !defined(CONFIG_MALI_BIFROST_NO_MALI) ++ ++/* GPU IRQ Tags */ ++#define JOB_IRQ_TAG 0 ++#define MMU_IRQ_TAG 1 ++#define GPU_IRQ_TAG 2 ++ ++static void *kbase_tag(void *ptr, u32 tag) ++{ ++ return (void *)(((uintptr_t) ptr) | tag); ++} ++ ++static void *kbase_untag(void *ptr) ++{ ++ return (void *)(((uintptr_t) ptr) & ~3); ++} ++ ++static irqreturn_t kbase_job_irq_handler(int irq, void *data) ++{ ++ unsigned long flags; ++ struct kbase_device *kbdev = kbase_untag(data); ++ u32 val; ++ ++ spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags); ++ ++ if (!kbdev->pm.backend.gpu_powered) { ++ /* GPU is turned off - IRQ is not for us */ ++ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, ++ flags); ++ return IRQ_NONE; ++ } ++ ++ val = kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_STATUS), NULL); ++ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++ if (!kbdev->pm.backend.driver_ready_for_irqs) ++ dev_warn(kbdev->dev, "%s: irq %d irqstatus 0x%x before driver is ready\n", ++ __func__, irq, val); ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags); ++ ++ if (!val) ++ return IRQ_NONE; ++ ++ dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val); ++ ++ kbase_job_done(kbdev, val); ++ ++ return IRQ_HANDLED; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_job_irq_handler); ++ ++static irqreturn_t kbase_mmu_irq_handler(int irq, void *data) ++{ ++ unsigned long flags; ++ struct kbase_device *kbdev = kbase_untag(data); ++ u32 val; ++ ++ spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags); ++ ++ if (!kbdev->pm.backend.gpu_powered) { ++ /* GPU is turned off - IRQ is not for us */ ++ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, ++ flags); ++ return IRQ_NONE; ++ } ++ ++ atomic_inc(&kbdev->faults_pending); ++ ++ val = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_STATUS), NULL); ++ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++ if (!kbdev->pm.backend.driver_ready_for_irqs) ++ dev_warn(kbdev->dev, "%s: irq %d irqstatus 0x%x before driver is ready\n", ++ __func__, irq, val); ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags); ++ ++ if (!val) { ++ atomic_dec(&kbdev->faults_pending); ++ return IRQ_NONE; ++ } ++ ++ dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val); ++ ++ kbase_mmu_interrupt(kbdev, val); ++ ++ atomic_dec(&kbdev->faults_pending); ++ ++ return IRQ_HANDLED; ++} ++ ++static irqreturn_t kbase_gpu_irq_handler(int irq, void *data) ++{ ++ unsigned long flags; ++ struct kbase_device *kbdev = kbase_untag(data); ++ u32 val; ++ ++ spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags); ++ ++ if (!kbdev->pm.backend.gpu_powered) { ++ /* GPU is turned off - IRQ is not for us */ ++ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, ++ flags); ++ return IRQ_NONE; ++ } ++ ++ val = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_STATUS), NULL); ++ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++ if (!kbdev->pm.backend.driver_ready_for_irqs) ++ dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x before driver is ready\n", ++ __func__, irq, val); ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags); ++ ++ if (!val) ++ return IRQ_NONE; ++ ++ dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val); ++ ++ kbase_gpu_interrupt(kbdev, val); ++ ++ return IRQ_HANDLED; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_gpu_irq_handler); ++ ++static irq_handler_t kbase_handler_table[] = { ++ [JOB_IRQ_TAG] = kbase_job_irq_handler, ++ [MMU_IRQ_TAG] = kbase_mmu_irq_handler, ++ [GPU_IRQ_TAG] = kbase_gpu_irq_handler, ++}; ++ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++#define JOB_IRQ_HANDLER JOB_IRQ_TAG ++#define MMU_IRQ_HANDLER MMU_IRQ_TAG ++#define GPU_IRQ_HANDLER GPU_IRQ_TAG ++ ++/** ++ * kbase_set_custom_irq_handler - Set a custom IRQ handler ++ * @kbdev: Device for which the handler is to be registered ++ * @custom_handler: Handler to be registered ++ * @irq_type: Interrupt type ++ * ++ * Registers given interrupt handler for requested interrupt type ++ * In the case where irq handler is not specified, the default handler shall be ++ * registered ++ * ++ * Return: 0 case success, error code otherwise ++ */ ++int kbase_set_custom_irq_handler(struct kbase_device *kbdev, ++ irq_handler_t custom_handler, ++ int irq_type) ++{ ++ int result = 0; ++ irq_handler_t requested_irq_handler = NULL; ++ ++ KBASE_DEBUG_ASSERT((JOB_IRQ_HANDLER <= irq_type) && ++ (GPU_IRQ_HANDLER >= irq_type)); ++ ++ /* Release previous handler */ ++ if (kbdev->irqs[irq_type].irq) ++ free_irq(kbdev->irqs[irq_type].irq, kbase_tag(kbdev, irq_type)); ++ ++ requested_irq_handler = (NULL != custom_handler) ? custom_handler : ++ kbase_handler_table[irq_type]; ++ ++ if (0 != request_irq(kbdev->irqs[irq_type].irq, ++ requested_irq_handler, ++ kbdev->irqs[irq_type].flags | IRQF_SHARED, ++ dev_name(kbdev->dev), kbase_tag(kbdev, irq_type))) { ++ result = -EINVAL; ++ dev_err(kbdev->dev, "Can't request interrupt %d (index %d)\n", ++ kbdev->irqs[irq_type].irq, irq_type); ++#ifdef CONFIG_SPARSE_IRQ ++ dev_err(kbdev->dev, "You have CONFIG_SPARSE_IRQ support enabled - is the interrupt number correct for this configuration?\n"); ++#endif /* CONFIG_SPARSE_IRQ */ ++ } ++ ++ return result; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_set_custom_irq_handler); ++ ++/* test correct interrupt assigment and reception by cpu */ ++struct kbasep_irq_test { ++ struct hrtimer timer; ++ wait_queue_head_t wait; ++ int triggered; ++ u32 timeout; ++}; ++ ++static struct kbasep_irq_test kbasep_irq_test_data; ++ ++#define IRQ_TEST_TIMEOUT 500 ++ ++static irqreturn_t kbase_job_irq_test_handler(int irq, void *data) ++{ ++ unsigned long flags; ++ struct kbase_device *kbdev = kbase_untag(data); ++ u32 val; ++ ++ spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags); ++ ++ if (!kbdev->pm.backend.gpu_powered) { ++ /* GPU is turned off - IRQ is not for us */ ++ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, ++ flags); ++ return IRQ_NONE; ++ } ++ ++ val = kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_STATUS), NULL); ++ ++ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags); ++ ++ if (!val) ++ return IRQ_NONE; ++ ++ dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val); ++ ++ kbasep_irq_test_data.triggered = 1; ++ wake_up(&kbasep_irq_test_data.wait); ++ ++ kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), val, NULL); ++ ++ return IRQ_HANDLED; ++} ++ ++static irqreturn_t kbase_mmu_irq_test_handler(int irq, void *data) ++{ ++ unsigned long flags; ++ struct kbase_device *kbdev = kbase_untag(data); ++ u32 val; ++ ++ spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags); ++ ++ if (!kbdev->pm.backend.gpu_powered) { ++ /* GPU is turned off - IRQ is not for us */ ++ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, ++ flags); ++ return IRQ_NONE; ++ } ++ ++ val = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_STATUS), NULL); ++ ++ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags); ++ ++ if (!val) ++ return IRQ_NONE; ++ ++ dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val); ++ ++ kbasep_irq_test_data.triggered = 1; ++ wake_up(&kbasep_irq_test_data.wait); ++ ++ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), val, NULL); ++ ++ return IRQ_HANDLED; ++} ++ ++static enum hrtimer_restart kbasep_test_interrupt_timeout(struct hrtimer *timer) ++{ ++ struct kbasep_irq_test *test_data = container_of(timer, ++ struct kbasep_irq_test, timer); ++ ++ test_data->timeout = 1; ++ test_data->triggered = 1; ++ wake_up(&test_data->wait); ++ return HRTIMER_NORESTART; ++} ++ ++static int kbasep_common_test_interrupt( ++ struct kbase_device * const kbdev, u32 tag) ++{ ++ int err = 0; ++ irq_handler_t test_handler; ++ ++ u32 old_mask_val; ++ u16 mask_offset; ++ u16 rawstat_offset; ++ ++ switch (tag) { ++ case JOB_IRQ_TAG: ++ test_handler = kbase_job_irq_test_handler; ++ rawstat_offset = JOB_CONTROL_REG(JOB_IRQ_RAWSTAT); ++ mask_offset = JOB_CONTROL_REG(JOB_IRQ_MASK); ++ break; ++ case MMU_IRQ_TAG: ++ test_handler = kbase_mmu_irq_test_handler; ++ rawstat_offset = MMU_REG(MMU_IRQ_RAWSTAT); ++ mask_offset = MMU_REG(MMU_IRQ_MASK); ++ break; ++ case GPU_IRQ_TAG: ++ /* already tested by pm_driver - bail out */ ++ default: ++ return 0; ++ } ++ ++ /* store old mask */ ++ old_mask_val = kbase_reg_read(kbdev, mask_offset, NULL); ++ /* mask interrupts */ ++ kbase_reg_write(kbdev, mask_offset, 0x0, NULL); ++ ++ if (kbdev->irqs[tag].irq) { ++ /* release original handler and install test handler */ ++ if (kbase_set_custom_irq_handler(kbdev, test_handler, tag) != 0) { ++ err = -EINVAL; ++ } else { ++ kbasep_irq_test_data.timeout = 0; ++ hrtimer_init(&kbasep_irq_test_data.timer, ++ CLOCK_MONOTONIC, HRTIMER_MODE_REL); ++ kbasep_irq_test_data.timer.function = ++ kbasep_test_interrupt_timeout; ++ ++ /* trigger interrupt */ ++ kbase_reg_write(kbdev, mask_offset, 0x1, NULL); ++ kbase_reg_write(kbdev, rawstat_offset, 0x1, NULL); ++ ++ hrtimer_start(&kbasep_irq_test_data.timer, ++ HR_TIMER_DELAY_MSEC(IRQ_TEST_TIMEOUT), ++ HRTIMER_MODE_REL); ++ ++ wait_event(kbasep_irq_test_data.wait, ++ kbasep_irq_test_data.triggered != 0); ++ ++ if (kbasep_irq_test_data.timeout != 0) { ++ dev_err(kbdev->dev, "Interrupt %d (index %d) didn't reach CPU.\n", ++ kbdev->irqs[tag].irq, tag); ++ err = -EINVAL; ++ } else { ++ dev_dbg(kbdev->dev, "Interrupt %d (index %d) reached CPU.\n", ++ kbdev->irqs[tag].irq, tag); ++ } ++ ++ hrtimer_cancel(&kbasep_irq_test_data.timer); ++ kbasep_irq_test_data.triggered = 0; ++ ++ /* mask interrupts */ ++ kbase_reg_write(kbdev, mask_offset, 0x0, NULL); ++ ++ /* release test handler */ ++ free_irq(kbdev->irqs[tag].irq, kbase_tag(kbdev, tag)); ++ } ++ ++ /* restore original interrupt */ ++ if (request_irq(kbdev->irqs[tag].irq, kbase_handler_table[tag], ++ kbdev->irqs[tag].flags | IRQF_SHARED, ++ dev_name(kbdev->dev), kbase_tag(kbdev, tag))) { ++ dev_err(kbdev->dev, "Can't restore original interrupt %d (index %d)\n", ++ kbdev->irqs[tag].irq, tag); ++ err = -EINVAL; ++ } ++ } ++ /* restore old mask */ ++ kbase_reg_write(kbdev, mask_offset, old_mask_val, NULL); ++ ++ return err; ++} ++ ++int kbasep_common_test_interrupt_handlers( ++ struct kbase_device * const kbdev) ++{ ++ int err; ++ ++ init_waitqueue_head(&kbasep_irq_test_data.wait); ++ kbasep_irq_test_data.triggered = 0; ++ ++ /* A suspend won't happen during startup/insmod */ ++ kbase_pm_context_active(kbdev); ++ ++ err = kbasep_common_test_interrupt(kbdev, JOB_IRQ_TAG); ++ if (err) { ++ dev_err(kbdev->dev, "Interrupt JOB_IRQ didn't reach CPU. Check interrupt assignments.\n"); ++ goto out; ++ } ++ ++ err = kbasep_common_test_interrupt(kbdev, MMU_IRQ_TAG); ++ if (err) { ++ dev_err(kbdev->dev, "Interrupt MMU_IRQ didn't reach CPU. Check interrupt assignments.\n"); ++ goto out; ++ } ++ ++ dev_dbg(kbdev->dev, "Interrupts are correctly assigned.\n"); ++ ++ out: ++ kbase_pm_context_idle(kbdev); ++ ++ return err; ++} ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++ ++int kbase_install_interrupts(struct kbase_device *kbdev) ++{ ++ u32 nr = ARRAY_SIZE(kbase_handler_table); ++ int err; ++ u32 i; ++ ++ for (i = 0; i < nr; i++) { ++ err = request_irq(kbdev->irqs[i].irq, kbase_handler_table[i], ++ kbdev->irqs[i].flags | IRQF_SHARED, ++ dev_name(kbdev->dev), ++ kbase_tag(kbdev, i)); ++ if (err) { ++ dev_err(kbdev->dev, "Can't request interrupt %d (index %d)\n", ++ kbdev->irqs[i].irq, i); ++#ifdef CONFIG_SPARSE_IRQ ++ dev_err(kbdev->dev, "You have CONFIG_SPARSE_IRQ support enabled - is the interrupt number correct for this configuration?\n"); ++#endif /* CONFIG_SPARSE_IRQ */ ++ goto release; ++ } ++ } ++ ++ return 0; ++ ++ release: ++ while (i-- > 0) ++ free_irq(kbdev->irqs[i].irq, kbase_tag(kbdev, i)); ++ ++ return err; ++} ++ ++void kbase_release_interrupts(struct kbase_device *kbdev) ++{ ++ u32 nr = ARRAY_SIZE(kbase_handler_table); ++ u32 i; ++ ++ for (i = 0; i < nr; i++) { ++ if (kbdev->irqs[i].irq) ++ free_irq(kbdev->irqs[i].irq, kbase_tag(kbdev, i)); ++ } ++} ++ ++void kbase_synchronize_irqs(struct kbase_device *kbdev) ++{ ++ u32 nr = ARRAY_SIZE(kbase_handler_table); ++ u32 i; ++ ++ for (i = 0; i < nr; i++) { ++ if (kbdev->irqs[i].irq) ++ synchronize_irq(kbdev->irqs[i].irq); ++ } ++} ++ ++#endif /* !defined(CONFIG_MALI_BIFROST_NO_MALI) */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_jm_as.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_jm_as.c +new file mode 100755 +index 000000000000..c660c80341f4 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_jm_as.c +@@ -0,0 +1,235 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++/* ++ * Register backend context / address space management ++ */ ++ ++#include ++#include ++#include ++ ++/** ++ * assign_and_activate_kctx_addr_space - Assign an AS to a context ++ * @kbdev: Kbase device ++ * @kctx: Kbase context ++ * @current_as: Address Space to assign ++ * ++ * Assign an Address Space (AS) to a context, and add the context to the Policy. ++ * ++ * This includes ++ * setting up the global runpool_irq structure and the context on the AS, ++ * Activating the MMU on the AS, ++ * Allowing jobs to be submitted on the AS. ++ * ++ * Context: ++ * kbasep_js_kctx_info.jsctx_mutex held, ++ * kbasep_js_device_data.runpool_mutex held, ++ * AS transaction mutex held, ++ * Runpool IRQ lock held ++ */ ++static void assign_and_activate_kctx_addr_space(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ struct kbase_as *current_as) ++{ ++ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; ++ ++ lockdep_assert_held(&kctx->jctx.sched_info.ctx.jsctx_mutex); ++ lockdep_assert_held(&js_devdata->runpool_mutex); ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ /* Attribute handling */ ++ kbasep_js_ctx_attr_runpool_retain_ctx(kbdev, kctx); ++ ++ /* Allow it to run jobs */ ++ kbasep_js_set_submit_allowed(js_devdata, kctx); ++ ++ kbase_js_runpool_inc_context_count(kbdev, kctx); ++} ++ ++bool kbase_backend_use_ctx_sched(struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++ int i; ++ ++ if (kbdev->hwaccess.active_kctx == kctx) { ++ /* Context is already active */ ++ return true; ++ } ++ ++ for (i = 0; i < kbdev->nr_hw_address_spaces; i++) { ++ if (kbdev->as_to_kctx[i] == kctx) { ++ /* Context already has ASID - mark as active */ ++ return true; ++ } ++ } ++ ++ /* Context does not have address space assigned */ ++ return false; ++} ++ ++void kbase_backend_release_ctx_irq(struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++ int as_nr = kctx->as_nr; ++ ++ if (as_nr == KBASEP_AS_NR_INVALID) { ++ WARN(1, "Attempting to release context without ASID\n"); ++ return; ++ } ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (atomic_read(&kctx->refcount) != 1) { ++ WARN(1, "Attempting to release active ASID\n"); ++ return; ++ } ++ ++ kbasep_js_clear_submit_allowed(&kbdev->js_data, kctx); ++ ++ kbase_ctx_sched_release_ctx(kctx); ++ kbase_js_runpool_dec_context_count(kbdev, kctx); ++} ++ ++void kbase_backend_release_ctx_noirq(struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++} ++ ++int kbase_backend_find_and_release_free_address_space( ++ struct kbase_device *kbdev, struct kbase_context *kctx) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ unsigned long flags; ++ int i; ++ ++ js_devdata = &kbdev->js_data; ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); ++ mutex_lock(&js_devdata->runpool_mutex); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ for (i = 0; i < kbdev->nr_hw_address_spaces; i++) { ++ struct kbasep_js_kctx_info *as_js_kctx_info; ++ struct kbase_context *as_kctx; ++ ++ as_kctx = kbdev->as_to_kctx[i]; ++ as_js_kctx_info = &as_kctx->jctx.sched_info; ++ ++ /* Don't release privileged or active contexts, or contexts with ++ * jobs running. ++ * Note that a context will have at least 1 reference (which ++ * was previously taken by kbasep_js_schedule_ctx()) until ++ * descheduled. ++ */ ++ if (as_kctx && !kbase_ctx_flag(as_kctx, KCTX_PRIVILEGED) && ++ atomic_read(&as_kctx->refcount) == 1) { ++ if (!kbasep_js_runpool_retain_ctx_nolock(kbdev, ++ as_kctx)) { ++ WARN(1, "Failed to retain active context\n"); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, ++ flags); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ ++ return KBASEP_AS_NR_INVALID; ++ } ++ ++ kbasep_js_clear_submit_allowed(js_devdata, as_kctx); ++ ++ /* Drop and retake locks to take the jsctx_mutex on the ++ * context we're about to release without violating lock ++ * ordering ++ */ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ ++ ++ /* Release context from address space */ ++ mutex_lock(&as_js_kctx_info->ctx.jsctx_mutex); ++ mutex_lock(&js_devdata->runpool_mutex); ++ ++ kbasep_js_runpool_release_ctx_nolock(kbdev, as_kctx); ++ ++ if (!kbase_ctx_flag(as_kctx, KCTX_SCHEDULED)) { ++ kbasep_js_runpool_requeue_or_kill_ctx(kbdev, ++ as_kctx, ++ true); ++ ++ mutex_unlock(&js_devdata->runpool_mutex); ++ mutex_unlock(&as_js_kctx_info->ctx.jsctx_mutex); ++ ++ return i; ++ } ++ ++ /* Context was retained while locks were dropped, ++ * continue looking for free AS */ ++ ++ mutex_unlock(&js_devdata->runpool_mutex); ++ mutex_unlock(&as_js_kctx_info->ctx.jsctx_mutex); ++ ++ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); ++ mutex_lock(&js_devdata->runpool_mutex); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ } ++ } ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ mutex_unlock(&js_devdata->runpool_mutex); ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ ++ return KBASEP_AS_NR_INVALID; ++} ++ ++bool kbase_backend_use_ctx(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ int as_nr) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ struct kbase_as *new_address_space = NULL; ++ ++ js_devdata = &kbdev->js_data; ++ ++ if (kbdev->hwaccess.active_kctx == kctx) { ++ WARN(1, "Context is already scheduled in\n"); ++ return false; ++ } ++ ++ new_address_space = &kbdev->as[as_nr]; ++ ++ lockdep_assert_held(&js_devdata->runpool_mutex); ++ lockdep_assert_held(&kbdev->mmu_hw_mutex); ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ assign_and_activate_kctx_addr_space(kbdev, kctx, new_address_space); ++ ++ if (kbase_ctx_flag(kctx, KCTX_PRIVILEGED)) { ++ /* We need to retain it to keep the corresponding address space ++ */ ++ kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx); ++ } ++ ++ return true; ++} ++ +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_jm_defs.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_jm_defs.h +new file mode 100755 +index 000000000000..08a7400e66d5 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_jm_defs.h +@@ -0,0 +1,123 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++/* ++ * Register-based HW access backend specific definitions ++ */ ++ ++#ifndef _KBASE_HWACCESS_GPU_DEFS_H_ ++#define _KBASE_HWACCESS_GPU_DEFS_H_ ++ ++/* SLOT_RB_SIZE must be < 256 */ ++#define SLOT_RB_SIZE 2 ++#define SLOT_RB_MASK (SLOT_RB_SIZE - 1) ++ ++/** ++ * struct rb_entry - Ringbuffer entry ++ * @katom: Atom associated with this entry ++ */ ++struct rb_entry { ++ struct kbase_jd_atom *katom; ++}; ++ ++/** ++ * struct slot_rb - Slot ringbuffer ++ * @entries: Ringbuffer entries ++ * @last_context: The last context to submit a job on this slot ++ * @read_idx: Current read index of buffer ++ * @write_idx: Current write index of buffer ++ * @job_chain_flag: Flag used to implement jobchain disambiguation ++ */ ++struct slot_rb { ++ struct rb_entry entries[SLOT_RB_SIZE]; ++ ++ struct kbase_context *last_context; ++ ++ u8 read_idx; ++ u8 write_idx; ++ ++ u8 job_chain_flag; ++}; ++ ++/** ++ * struct kbase_backend_data - GPU backend specific data for HW access layer ++ * @slot_rb: Slot ringbuffers ++ * @rmu_workaround_flag: When PRLAM-8987 is present, this flag determines ++ * whether slots 0/1 or slot 2 are currently being ++ * pulled from ++ * @scheduling_timer: The timer tick used for rescheduling jobs ++ * @timer_running: Is the timer running? The runpool_mutex must be ++ * held whilst modifying this. ++ * @suspend_timer: Is the timer suspended? Set when a suspend ++ * occurs and cleared on resume. The runpool_mutex ++ * must be held whilst modifying this. ++ * @reset_gpu: Set to a KBASE_RESET_xxx value (see comments) ++ * @reset_workq: Work queue for performing the reset ++ * @reset_work: Work item for performing the reset ++ * @reset_wait: Wait event signalled when the reset is complete ++ * @reset_timer: Timeout for soft-stops before the reset ++ * @timeouts_updated: Have timeout values just been updated? ++ * ++ * The hwaccess_lock (a spinlock) must be held when accessing this structure ++ */ ++struct kbase_backend_data { ++ struct slot_rb slot_rb[BASE_JM_MAX_NR_SLOTS]; ++ ++ bool rmu_workaround_flag; ++ ++ struct hrtimer scheduling_timer; ++ ++ bool timer_running; ++ bool suspend_timer; ++ ++ atomic_t reset_gpu; ++ ++/* The GPU reset isn't pending */ ++#define KBASE_RESET_GPU_NOT_PENDING 0 ++/* kbase_prepare_to_reset_gpu has been called */ ++#define KBASE_RESET_GPU_PREPARED 1 ++/* kbase_reset_gpu has been called - the reset will now definitely happen ++ * within the timeout period */ ++#define KBASE_RESET_GPU_COMMITTED 2 ++/* The GPU reset process is currently occuring (timeout has expired or ++ * kbasep_try_reset_gpu_early was called) */ ++#define KBASE_RESET_GPU_HAPPENING 3 ++/* Reset the GPU silently, used when resetting the GPU as part of normal ++ * behavior (e.g. when exiting protected mode). */ ++#define KBASE_RESET_GPU_SILENT 4 ++ struct workqueue_struct *reset_workq; ++ struct work_struct reset_work; ++ wait_queue_head_t reset_wait; ++ struct hrtimer reset_timer; ++ ++ bool timeouts_updated; ++}; ++ ++/** ++ * struct kbase_jd_atom_backend - GPU backend specific katom data ++ */ ++struct kbase_jd_atom_backend { ++}; ++ ++/** ++ * struct kbase_context_backend - GPU backend specific context data ++ */ ++struct kbase_context_backend { ++}; ++ ++#endif /* _KBASE_HWACCESS_GPU_DEFS_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_jm_hw.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_jm_hw.c +new file mode 100755 +index 000000000000..cbca5eac82f1 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_jm_hw.c +@@ -0,0 +1,1512 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* ++ * Base kernel job manager APIs ++ */ ++ ++#include ++#include ++#include ++#if defined(CONFIG_MALI_BIFROST_GATOR_SUPPORT) ++#include ++#endif ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#define beenthere(kctx, f, a...) \ ++ dev_dbg(kctx->kbdev->dev, "%s:" f, __func__, ##a) ++ ++#if KBASE_GPU_RESET_EN ++static void kbasep_try_reset_gpu_early(struct kbase_device *kbdev); ++static void kbasep_reset_timeout_worker(struct work_struct *data); ++static enum hrtimer_restart kbasep_reset_timer_callback(struct hrtimer *timer); ++#endif /* KBASE_GPU_RESET_EN */ ++ ++static inline int kbasep_jm_is_js_free(struct kbase_device *kbdev, int js, ++ struct kbase_context *kctx) ++{ ++ return !kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT), kctx); ++} ++ ++void kbase_job_hw_submit(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom, ++ int js) ++{ ++ struct kbase_context *kctx; ++ u32 cfg; ++ u64 jc_head = katom->jc; ++ ++ KBASE_DEBUG_ASSERT(kbdev); ++ KBASE_DEBUG_ASSERT(katom); ++ ++ kctx = katom->kctx; ++ ++ /* Command register must be available */ ++ KBASE_DEBUG_ASSERT(kbasep_jm_is_js_free(kbdev, js, kctx)); ++ /* Affinity is not violating */ ++ kbase_js_debug_log_current_affinities(kbdev); ++ KBASE_DEBUG_ASSERT(!kbase_js_affinity_would_violate(kbdev, js, ++ katom->affinity)); ++ ++ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_LO), ++ jc_head & 0xFFFFFFFF, kctx); ++ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_HI), ++ jc_head >> 32, kctx); ++ ++ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_AFFINITY_NEXT_LO), ++ katom->affinity & 0xFFFFFFFF, kctx); ++ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_AFFINITY_NEXT_HI), ++ katom->affinity >> 32, kctx); ++ ++ /* start MMU, medium priority, cache clean/flush on end, clean/flush on ++ * start */ ++ cfg = kctx->as_nr; ++ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_FLUSH_REDUCTION) && ++ !(kbdev->serialize_jobs & KBASE_SERIALIZE_RESET)) ++ cfg |= JS_CONFIG_ENABLE_FLUSH_REDUCTION; ++ ++ if (0 != (katom->core_req & BASE_JD_REQ_SKIP_CACHE_START)) ++ cfg |= JS_CONFIG_START_FLUSH_NO_ACTION; ++ else ++ cfg |= JS_CONFIG_START_FLUSH_CLEAN_INVALIDATE; ++ ++ if (0 != (katom->core_req & BASE_JD_REQ_SKIP_CACHE_END) && ++ !(kbdev->serialize_jobs & KBASE_SERIALIZE_RESET)) ++ cfg |= JS_CONFIG_END_FLUSH_NO_ACTION; ++ else ++ cfg |= JS_CONFIG_END_FLUSH_CLEAN_INVALIDATE; ++ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10649)) ++ cfg |= JS_CONFIG_START_MMU; ++ ++ cfg |= JS_CONFIG_THREAD_PRI(8); ++ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_MODE) && ++ (katom->atom_flags & KBASE_KATOM_FLAG_PROTECTED)) ++ cfg |= JS_CONFIG_DISABLE_DESCRIPTOR_WR_BK; ++ ++ if (kbase_hw_has_feature(kbdev, ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) { ++ if (!kbdev->hwaccess.backend.slot_rb[js].job_chain_flag) { ++ cfg |= JS_CONFIG_JOB_CHAIN_FLAG; ++ katom->atom_flags |= KBASE_KATOM_FLAGS_JOBCHAIN; ++ kbdev->hwaccess.backend.slot_rb[js].job_chain_flag = ++ true; ++ } else { ++ katom->atom_flags &= ~KBASE_KATOM_FLAGS_JOBCHAIN; ++ kbdev->hwaccess.backend.slot_rb[js].job_chain_flag = ++ false; ++ } ++ } ++ ++ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_CONFIG_NEXT), cfg, kctx); ++ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_FLUSH_REDUCTION)) ++ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_FLUSH_ID_NEXT), ++ katom->flush_id, kctx); ++ ++ /* Write an approximate start timestamp. ++ * It's approximate because there might be a job in the HEAD register. ++ */ ++ katom->start_timestamp = ktime_get(); ++ ++ /* GO ! */ ++ dev_dbg(kbdev->dev, "JS: Submitting atom %p from ctx %p to js[%d] with head=0x%llx, affinity=0x%llx", ++ katom, kctx, js, jc_head, katom->affinity); ++ ++ KBASE_TRACE_ADD_SLOT_INFO(kbdev, JM_SUBMIT, kctx, katom, jc_head, js, ++ (u32) katom->affinity); ++ ++#if defined(CONFIG_MALI_BIFROST_GATOR_SUPPORT) ++ kbase_trace_mali_job_slots_event( ++ GATOR_MAKE_EVENT(GATOR_JOB_SLOT_START, js), ++ kctx, kbase_jd_atom_id(kctx, katom)); ++#endif ++ KBASE_TLSTREAM_TL_ATTRIB_ATOM_CONFIG(katom, jc_head, ++ katom->affinity, cfg); ++ KBASE_TLSTREAM_TL_RET_CTX_LPU( ++ kctx, ++ &kbdev->gpu_props.props.raw_props.js_features[ ++ katom->slot_nr]); ++ KBASE_TLSTREAM_TL_RET_ATOM_AS(katom, &kbdev->as[kctx->as_nr]); ++ KBASE_TLSTREAM_TL_RET_ATOM_LPU( ++ katom, ++ &kbdev->gpu_props.props.raw_props.js_features[js], ++ "ctx_nr,atom_nr"); ++#ifdef CONFIG_GPU_TRACEPOINTS ++ if (!kbase_backend_nr_atoms_submitted(kbdev, js)) { ++ /* If this is the only job on the slot, trace it as starting */ ++ char js_string[16]; ++ ++ trace_gpu_sched_switch( ++ kbasep_make_job_slot_string(js, js_string, ++ sizeof(js_string)), ++ ktime_to_ns(katom->start_timestamp), ++ (u32)katom->kctx->id, 0, katom->work_id); ++ kbdev->hwaccess.backend.slot_rb[js].last_context = katom->kctx; ++ } ++#endif ++ kbase_timeline_job_slot_submit(kbdev, kctx, katom, js); ++ ++ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT), ++ JS_COMMAND_START, katom->kctx); ++} ++ ++/** ++ * kbasep_job_slot_update_head_start_timestamp - Update timestamp ++ * @kbdev: kbase device ++ * @js: job slot ++ * @end_timestamp: timestamp ++ * ++ * Update the start_timestamp of the job currently in the HEAD, based on the ++ * fact that we got an IRQ for the previous set of completed jobs. ++ * ++ * The estimate also takes into account the time the job was submitted, to ++ * work out the best estimate (which might still result in an over-estimate to ++ * the calculated time spent) ++ */ ++static void kbasep_job_slot_update_head_start_timestamp( ++ struct kbase_device *kbdev, ++ int js, ++ ktime_t end_timestamp) ++{ ++ if (kbase_backend_nr_atoms_on_slot(kbdev, js) > 0) { ++ struct kbase_jd_atom *katom; ++ ktime_t timestamp_diff; ++ /* The atom in the HEAD */ ++ katom = kbase_gpu_inspect(kbdev, js, 0); ++ ++ KBASE_DEBUG_ASSERT(katom != NULL); ++ ++ timestamp_diff = ktime_sub(end_timestamp, ++ katom->start_timestamp); ++ if (ktime_to_ns(timestamp_diff) >= 0) { ++ /* Only update the timestamp if it's a better estimate ++ * than what's currently stored. This is because our ++ * estimate that accounts for the throttle time may be ++ * too much of an overestimate */ ++ katom->start_timestamp = end_timestamp; ++ } ++ } ++} ++ ++/** ++ * kbasep_trace_tl_event_lpu_softstop - Call event_lpu_softstop timeline ++ * tracepoint ++ * @kbdev: kbase device ++ * @js: job slot ++ * ++ * Make a tracepoint call to the instrumentation module informing that ++ * softstop happened on given lpu (job slot). ++ */ ++static void kbasep_trace_tl_event_lpu_softstop(struct kbase_device *kbdev, ++ int js) ++{ ++ KBASE_TLSTREAM_TL_EVENT_LPU_SOFTSTOP( ++ &kbdev->gpu_props.props.raw_props.js_features[js]); ++} ++ ++void kbase_job_done(struct kbase_device *kbdev, u32 done) ++{ ++ unsigned long flags; ++ int i; ++ u32 count = 0; ++ ktime_t end_timestamp = ktime_get(); ++ struct kbasep_js_device_data *js_devdata; ++ ++ KBASE_DEBUG_ASSERT(kbdev); ++ js_devdata = &kbdev->js_data; ++ ++ KBASE_TRACE_ADD(kbdev, JM_IRQ, NULL, NULL, 0, done); ++ ++ memset(&kbdev->slot_submit_count_irq[0], 0, ++ sizeof(kbdev->slot_submit_count_irq)); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ while (done) { ++ u32 failed = done >> 16; ++ ++ /* treat failed slots as finished slots */ ++ u32 finished = (done & 0xFFFF) | failed; ++ ++ /* Note: This is inherently unfair, as we always check ++ * for lower numbered interrupts before the higher ++ * numbered ones.*/ ++ i = ffs(finished) - 1; ++ KBASE_DEBUG_ASSERT(i >= 0); ++ ++ do { ++ int nr_done; ++ u32 active; ++ u32 completion_code = BASE_JD_EVENT_DONE;/* assume OK */ ++ u64 job_tail = 0; ++ ++ if (failed & (1u << i)) { ++ /* read out the job slot status code if the job ++ * slot reported failure */ ++ completion_code = kbase_reg_read(kbdev, ++ JOB_SLOT_REG(i, JS_STATUS), NULL); ++ ++ switch (completion_code) { ++ case BASE_JD_EVENT_STOPPED: ++#if defined(CONFIG_MALI_BIFROST_GATOR_SUPPORT) ++ kbase_trace_mali_job_slots_event( ++ GATOR_MAKE_EVENT( ++ GATOR_JOB_SLOT_SOFT_STOPPED, i), ++ NULL, 0); ++#endif ++ ++ kbasep_trace_tl_event_lpu_softstop( ++ kbdev, i); ++ ++ /* Soft-stopped job - read the value of ++ * JS_TAIL so that the job chain can ++ * be resumed */ ++ job_tail = (u64)kbase_reg_read(kbdev, ++ JOB_SLOT_REG(i, JS_TAIL_LO), ++ NULL) | ++ ((u64)kbase_reg_read(kbdev, ++ JOB_SLOT_REG(i, JS_TAIL_HI), ++ NULL) << 32); ++ break; ++ case BASE_JD_EVENT_NOT_STARTED: ++ /* PRLAM-10673 can cause a TERMINATED ++ * job to come back as NOT_STARTED, but ++ * the error interrupt helps us detect ++ * it */ ++ completion_code = ++ BASE_JD_EVENT_TERMINATED; ++ /* fall through */ ++ default: ++ dev_warn(kbdev->dev, "error detected from slot %d, job status 0x%08x (%s)", ++ i, completion_code, ++ kbase_exception_name ++ (kbdev, ++ completion_code)); ++ } ++ ++ kbase_gpu_irq_evict(kbdev, i); ++ } ++ ++ kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), ++ done & ((1 << i) | (1 << (i + 16))), ++ NULL); ++ active = kbase_reg_read(kbdev, ++ JOB_CONTROL_REG(JOB_IRQ_JS_STATE), ++ NULL); ++ ++ if (((active >> i) & 1) == 0 && ++ (((done >> (i + 16)) & 1) == 0)) { ++ /* There is a potential race we must work ++ * around: ++ * ++ * 1. A job slot has a job in both current and ++ * next registers ++ * 2. The job in current completes ++ * successfully, the IRQ handler reads ++ * RAWSTAT and calls this function with the ++ * relevant bit set in "done" ++ * 3. The job in the next registers becomes the ++ * current job on the GPU ++ * 4. Sometime before the JOB_IRQ_CLEAR line ++ * above the job on the GPU _fails_ ++ * 5. The IRQ_CLEAR clears the done bit but not ++ * the failed bit. This atomically sets ++ * JOB_IRQ_JS_STATE. However since both jobs ++ * have now completed the relevant bits for ++ * the slot are set to 0. ++ * ++ * If we now did nothing then we'd incorrectly ++ * assume that _both_ jobs had completed ++ * successfully (since we haven't yet observed ++ * the fail bit being set in RAWSTAT). ++ * ++ * So at this point if there are no active jobs ++ * left we check to see if RAWSTAT has a failure ++ * bit set for the job slot. If it does we know ++ * that there has been a new failure that we ++ * didn't previously know about, so we make sure ++ * that we record this in active (but we wait ++ * for the next loop to deal with it). ++ * ++ * If we were handling a job failure (i.e. done ++ * has the relevant high bit set) then we know ++ * that the value read back from ++ * JOB_IRQ_JS_STATE is the correct number of ++ * remaining jobs because the failed job will ++ * have prevented any futher jobs from starting ++ * execution. ++ */ ++ u32 rawstat = kbase_reg_read(kbdev, ++ JOB_CONTROL_REG(JOB_IRQ_RAWSTAT), NULL); ++ ++ if ((rawstat >> (i + 16)) & 1) { ++ /* There is a failed job that we've ++ * missed - add it back to active */ ++ active |= (1u << i); ++ } ++ } ++ ++ dev_dbg(kbdev->dev, "Job ended with status 0x%08X\n", ++ completion_code); ++ ++ nr_done = kbase_backend_nr_atoms_submitted(kbdev, i); ++ nr_done -= (active >> i) & 1; ++ nr_done -= (active >> (i + 16)) & 1; ++ ++ if (nr_done <= 0) { ++ dev_warn(kbdev->dev, "Spurious interrupt on slot %d", ++ i); ++ ++ goto spurious; ++ } ++ ++ count += nr_done; ++ ++ while (nr_done) { ++ if (nr_done == 1) { ++ kbase_gpu_complete_hw(kbdev, i, ++ completion_code, ++ job_tail, ++ &end_timestamp); ++ kbase_jm_try_kick_all(kbdev); ++ } else { ++ /* More than one job has completed. ++ * Since this is not the last job being ++ * reported this time it must have ++ * passed. This is because the hardware ++ * will not allow further jobs in a job ++ * slot to complete until the failed job ++ * is cleared from the IRQ status. ++ */ ++ kbase_gpu_complete_hw(kbdev, i, ++ BASE_JD_EVENT_DONE, ++ 0, ++ &end_timestamp); ++ } ++ nr_done--; ++ } ++ spurious: ++ done = kbase_reg_read(kbdev, ++ JOB_CONTROL_REG(JOB_IRQ_RAWSTAT), NULL); ++ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10883)) { ++ /* Workaround for missing interrupt caused by ++ * PRLAM-10883 */ ++ if (((active >> i) & 1) && (0 == ++ kbase_reg_read(kbdev, ++ JOB_SLOT_REG(i, ++ JS_STATUS), NULL))) { ++ /* Force job slot to be processed again ++ */ ++ done |= (1u << i); ++ } ++ } ++ ++ failed = done >> 16; ++ finished = (done & 0xFFFF) | failed; ++ if (done) ++ end_timestamp = ktime_get(); ++ } while (finished & (1 << i)); ++ ++ kbasep_job_slot_update_head_start_timestamp(kbdev, i, ++ end_timestamp); ++ } ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++#if KBASE_GPU_RESET_EN ++ if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) == ++ KBASE_RESET_GPU_COMMITTED) { ++ /* If we're trying to reset the GPU then we might be able to do ++ * it early (without waiting for a timeout) because some jobs ++ * have completed ++ */ ++ kbasep_try_reset_gpu_early(kbdev); ++ } ++#endif /* KBASE_GPU_RESET_EN */ ++ KBASE_TRACE_ADD(kbdev, JM_IRQ_END, NULL, NULL, 0, count); ++} ++KBASE_EXPORT_TEST_API(kbase_job_done); ++ ++static bool kbasep_soft_stop_allowed(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom) ++{ ++ bool soft_stops_allowed = true; ++ ++ if (kbase_jd_katom_is_protected(katom)) { ++ soft_stops_allowed = false; ++ } else if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408)) { ++ if ((katom->core_req & BASE_JD_REQ_T) != 0) ++ soft_stops_allowed = false; ++ } ++ return soft_stops_allowed; ++} ++ ++static bool kbasep_hard_stop_allowed(struct kbase_device *kbdev, ++ base_jd_core_req core_reqs) ++{ ++ bool hard_stops_allowed = true; ++ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8394)) { ++ if ((core_reqs & BASE_JD_REQ_T) != 0) ++ hard_stops_allowed = false; ++ } ++ return hard_stops_allowed; ++} ++ ++void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev, ++ int js, ++ u32 action, ++ base_jd_core_req core_reqs, ++ struct kbase_jd_atom *target_katom) ++{ ++ struct kbase_context *kctx = target_katom->kctx; ++#if KBASE_TRACE_ENABLE ++ u32 status_reg_before; ++ u64 job_in_head_before; ++ u32 status_reg_after; ++ ++ KBASE_DEBUG_ASSERT(!(action & (~JS_COMMAND_MASK))); ++ ++ /* Check the head pointer */ ++ job_in_head_before = ((u64) kbase_reg_read(kbdev, ++ JOB_SLOT_REG(js, JS_HEAD_LO), NULL)) ++ | (((u64) kbase_reg_read(kbdev, ++ JOB_SLOT_REG(js, JS_HEAD_HI), NULL)) ++ << 32); ++ status_reg_before = kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_STATUS), ++ NULL); ++#endif ++ ++ if (action == JS_COMMAND_SOFT_STOP) { ++ bool soft_stop_allowed = kbasep_soft_stop_allowed(kbdev, ++ target_katom); ++ ++ if (!soft_stop_allowed) { ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++ dev_dbg(kbdev->dev, ++ "Attempt made to soft-stop a job that cannot be soft-stopped. core_reqs = 0x%X", ++ (unsigned int)core_reqs); ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++ return; ++ } ++ ++ /* We are about to issue a soft stop, so mark the atom as having ++ * been soft stopped */ ++ target_katom->atom_flags |= KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED; ++ ++ /* Mark the point where we issue the soft-stop command */ ++ KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_ISSUE(target_katom); ++ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316)) { ++ int i; ++ ++ for (i = 0; ++ i < kbase_backend_nr_atoms_submitted(kbdev, js); ++ i++) { ++ struct kbase_jd_atom *katom; ++ ++ katom = kbase_gpu_inspect(kbdev, js, i); ++ ++ KBASE_DEBUG_ASSERT(katom); ++ ++ /* For HW_ISSUE_8316, only 'bad' jobs attacking ++ * the system can cause this issue: normally, ++ * all memory should be allocated in multiples ++ * of 4 pages, and growable memory should be ++ * changed size in multiples of 4 pages. ++ * ++ * Whilst such 'bad' jobs can be cleared by a ++ * GPU reset, the locking up of a uTLB entry ++ * caused by the bad job could also stall other ++ * ASs, meaning that other ASs' jobs don't ++ * complete in the 'grace' period before the ++ * reset. We don't want to lose other ASs' jobs ++ * when they would normally complete fine, so we ++ * must 'poke' the MMU regularly to help other ++ * ASs complete */ ++ kbase_as_poking_timer_retain_atom( ++ kbdev, katom->kctx, katom); ++ } ++ } ++ ++ if (kbase_hw_has_feature( ++ kbdev, ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) { ++ action = (target_katom->atom_flags & ++ KBASE_KATOM_FLAGS_JOBCHAIN) ? ++ JS_COMMAND_SOFT_STOP_1 : ++ JS_COMMAND_SOFT_STOP_0; ++ } ++ } else if (action == JS_COMMAND_HARD_STOP) { ++ bool hard_stop_allowed = kbasep_hard_stop_allowed(kbdev, ++ core_reqs); ++ ++ if (!hard_stop_allowed) { ++ /* Jobs can be hard-stopped for the following reasons: ++ * * CFS decides the job has been running too long (and ++ * soft-stop has not occurred). In this case the GPU ++ * will be reset by CFS if the job remains on the ++ * GPU. ++ * ++ * * The context is destroyed, kbase_jd_zap_context ++ * will attempt to hard-stop the job. However it also ++ * has a watchdog which will cause the GPU to be ++ * reset if the job remains on the GPU. ++ * ++ * * An (unhandled) MMU fault occurred. As long as ++ * BASE_HW_ISSUE_8245 is defined then the GPU will be ++ * reset. ++ * ++ * All three cases result in the GPU being reset if the ++ * hard-stop fails, so it is safe to just return and ++ * ignore the hard-stop request. ++ */ ++ dev_warn(kbdev->dev, ++ "Attempt made to hard-stop a job that cannot be hard-stopped. core_reqs = 0x%X", ++ (unsigned int)core_reqs); ++ return; ++ } ++ target_katom->atom_flags |= KBASE_KATOM_FLAG_BEEN_HARD_STOPPED; ++ ++ if (kbase_hw_has_feature( ++ kbdev, ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) { ++ action = (target_katom->atom_flags & ++ KBASE_KATOM_FLAGS_JOBCHAIN) ? ++ JS_COMMAND_HARD_STOP_1 : ++ JS_COMMAND_HARD_STOP_0; ++ } ++ } ++ ++ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND), action, kctx); ++ ++#if KBASE_TRACE_ENABLE ++ status_reg_after = kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_STATUS), ++ NULL); ++ if (status_reg_after == BASE_JD_EVENT_ACTIVE) { ++ struct kbase_jd_atom *head; ++ struct kbase_context *head_kctx; ++ ++ head = kbase_gpu_inspect(kbdev, js, 0); ++ head_kctx = head->kctx; ++ ++ if (status_reg_before == BASE_JD_EVENT_ACTIVE) ++ KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, head_kctx, ++ head, job_in_head_before, js); ++ else ++ KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, ++ 0, js); ++ ++ switch (action) { ++ case JS_COMMAND_SOFT_STOP: ++ KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP, head_kctx, ++ head, head->jc, js); ++ break; ++ case JS_COMMAND_SOFT_STOP_0: ++ KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_0, head_kctx, ++ head, head->jc, js); ++ break; ++ case JS_COMMAND_SOFT_STOP_1: ++ KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_1, head_kctx, ++ head, head->jc, js); ++ break; ++ case JS_COMMAND_HARD_STOP: ++ KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP, head_kctx, ++ head, head->jc, js); ++ break; ++ case JS_COMMAND_HARD_STOP_0: ++ KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_0, head_kctx, ++ head, head->jc, js); ++ break; ++ case JS_COMMAND_HARD_STOP_1: ++ KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_1, head_kctx, ++ head, head->jc, js); ++ break; ++ default: ++ BUG(); ++ break; ++ } ++ } else { ++ if (status_reg_before == BASE_JD_EVENT_ACTIVE) ++ KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, ++ job_in_head_before, js); ++ else ++ KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, ++ 0, js); ++ ++ switch (action) { ++ case JS_COMMAND_SOFT_STOP: ++ KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP, NULL, NULL, 0, ++ js); ++ break; ++ case JS_COMMAND_SOFT_STOP_0: ++ KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_0, NULL, NULL, ++ 0, js); ++ break; ++ case JS_COMMAND_SOFT_STOP_1: ++ KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_1, NULL, NULL, ++ 0, js); ++ break; ++ case JS_COMMAND_HARD_STOP: ++ KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP, NULL, NULL, 0, ++ js); ++ break; ++ case JS_COMMAND_HARD_STOP_0: ++ KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_0, NULL, NULL, ++ 0, js); ++ break; ++ case JS_COMMAND_HARD_STOP_1: ++ KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_1, NULL, NULL, ++ 0, js); ++ break; ++ default: ++ BUG(); ++ break; ++ } ++ } ++#endif ++} ++ ++void kbase_backend_jm_kill_jobs_from_kctx(struct kbase_context *kctx) ++{ ++ unsigned long flags; ++ struct kbase_device *kbdev; ++ int i; ++ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ kbdev = kctx->kbdev; ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ /* Cancel any remaining running jobs for this kctx */ ++ mutex_lock(&kctx->jctx.lock); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ /* Invalidate all jobs in context, to prevent re-submitting */ ++ for (i = 0; i < BASE_JD_ATOM_COUNT; i++) { ++ if (!work_pending(&kctx->jctx.atoms[i].work)) ++ kctx->jctx.atoms[i].event_code = ++ BASE_JD_EVENT_JOB_CANCELLED; ++ } ++ ++ for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) ++ kbase_job_slot_hardstop(kctx, i, NULL); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&kctx->jctx.lock); ++} ++ ++void kbase_job_slot_ctx_priority_check_locked(struct kbase_context *kctx, ++ struct kbase_jd_atom *target_katom) ++{ ++ struct kbase_device *kbdev; ++ int js = target_katom->slot_nr; ++ int priority = target_katom->sched_priority; ++ int i; ++ bool stop_sent = false; ++ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ kbdev = kctx->kbdev; ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ for (i = 0; i < kbase_backend_nr_atoms_on_slot(kbdev, js); i++) { ++ struct kbase_jd_atom *katom; ++ ++ katom = kbase_gpu_inspect(kbdev, js, i); ++ if (!katom) ++ continue; ++ ++ if (katom->kctx != kctx) ++ continue; ++ ++ if (katom->sched_priority > priority) { ++ if (!stop_sent) ++ KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY_CHANGE( ++ target_katom); ++ ++ kbase_job_slot_softstop(kbdev, js, katom); ++ stop_sent = true; ++ } ++ } ++} ++ ++struct zap_reset_data { ++ /* The stages are: ++ * 1. The timer has never been called ++ * 2. The zap has timed out, all slots are soft-stopped - the GPU reset ++ * will happen. The GPU has been reset when ++ * kbdev->hwaccess.backend.reset_waitq is signalled ++ * ++ * (-1 - The timer has been cancelled) ++ */ ++ int stage; ++ struct kbase_device *kbdev; ++ struct hrtimer timer; ++ spinlock_t lock; /* protects updates to stage member */ ++}; ++ ++static enum hrtimer_restart zap_timeout_callback(struct hrtimer *timer) ++{ ++ struct zap_reset_data *reset_data = container_of(timer, ++ struct zap_reset_data, timer); ++ struct kbase_device *kbdev = reset_data->kbdev; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&reset_data->lock, flags); ++ ++ if (reset_data->stage == -1) ++ goto out; ++ ++#if KBASE_GPU_RESET_EN ++ if (kbase_prepare_to_reset_gpu(kbdev)) { ++ dev_err(kbdev->dev, "Issueing GPU soft-reset because jobs failed to be killed (within %d ms) as part of context termination (e.g. process exit)\n", ++ ZAP_TIMEOUT); ++ kbase_reset_gpu(kbdev); ++ } ++#endif /* KBASE_GPU_RESET_EN */ ++ reset_data->stage = 2; ++ ++ out: ++ spin_unlock_irqrestore(&reset_data->lock, flags); ++ ++ return HRTIMER_NORESTART; ++} ++ ++void kbase_jm_wait_for_zero_jobs(struct kbase_context *kctx) ++{ ++ struct kbase_device *kbdev = kctx->kbdev; ++ struct zap_reset_data reset_data; ++ unsigned long flags; ++ ++ hrtimer_init_on_stack(&reset_data.timer, CLOCK_MONOTONIC, ++ HRTIMER_MODE_REL); ++ reset_data.timer.function = zap_timeout_callback; ++ ++ spin_lock_init(&reset_data.lock); ++ ++ reset_data.kbdev = kbdev; ++ reset_data.stage = 1; ++ ++ hrtimer_start(&reset_data.timer, HR_TIMER_DELAY_MSEC(ZAP_TIMEOUT), ++ HRTIMER_MODE_REL); ++ ++ /* Wait for all jobs to finish, and for the context to be not-scheduled ++ * (due to kbase_job_zap_context(), we also guarentee it's not in the JS ++ * policy queue either */ ++ wait_event(kctx->jctx.zero_jobs_wait, kctx->jctx.job_nr == 0); ++ wait_event(kctx->jctx.sched_info.ctx.is_scheduled_wait, ++ !kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ ++ spin_lock_irqsave(&reset_data.lock, flags); ++ if (reset_data.stage == 1) { ++ /* The timer hasn't run yet - so cancel it */ ++ reset_data.stage = -1; ++ } ++ spin_unlock_irqrestore(&reset_data.lock, flags); ++ ++ hrtimer_cancel(&reset_data.timer); ++ ++ if (reset_data.stage == 2) { ++ /* The reset has already started. ++ * Wait for the reset to complete ++ */ ++ wait_event(kbdev->hwaccess.backend.reset_wait, ++ atomic_read(&kbdev->hwaccess.backend.reset_gpu) ++ == KBASE_RESET_GPU_NOT_PENDING); ++ } ++ destroy_hrtimer_on_stack(&reset_data.timer); ++ ++ dev_dbg(kbdev->dev, "Zap: Finished Context %p", kctx); ++ ++ /* Ensure that the signallers of the waitqs have finished */ ++ mutex_lock(&kctx->jctx.lock); ++ mutex_lock(&kctx->jctx.sched_info.ctx.jsctx_mutex); ++ mutex_unlock(&kctx->jctx.sched_info.ctx.jsctx_mutex); ++ mutex_unlock(&kctx->jctx.lock); ++} ++ ++u32 kbase_backend_get_current_flush_id(struct kbase_device *kbdev) ++{ ++ u32 flush_id = 0; ++ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_FLUSH_REDUCTION)) { ++ mutex_lock(&kbdev->pm.lock); ++ if (kbdev->pm.backend.gpu_powered) ++ flush_id = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(LATEST_FLUSH), NULL); ++ mutex_unlock(&kbdev->pm.lock); ++ } ++ ++ return flush_id; ++} ++ ++int kbase_job_slot_init(struct kbase_device *kbdev) ++{ ++#if KBASE_GPU_RESET_EN ++ kbdev->hwaccess.backend.reset_workq = alloc_workqueue( ++ "Mali reset workqueue", 0, 1); ++ if (NULL == kbdev->hwaccess.backend.reset_workq) ++ return -EINVAL; ++ ++ KBASE_DEBUG_ASSERT(0 == ++ object_is_on_stack(&kbdev->hwaccess.backend.reset_work)); ++ INIT_WORK(&kbdev->hwaccess.backend.reset_work, ++ kbasep_reset_timeout_worker); ++ ++ hrtimer_init(&kbdev->hwaccess.backend.reset_timer, CLOCK_MONOTONIC, ++ HRTIMER_MODE_REL); ++ kbdev->hwaccess.backend.reset_timer.function = ++ kbasep_reset_timer_callback; ++#endif ++ ++ return 0; ++} ++KBASE_EXPORT_TEST_API(kbase_job_slot_init); ++ ++void kbase_job_slot_halt(struct kbase_device *kbdev) ++{ ++ CSTD_UNUSED(kbdev); ++} ++ ++void kbase_job_slot_term(struct kbase_device *kbdev) ++{ ++#if KBASE_GPU_RESET_EN ++ destroy_workqueue(kbdev->hwaccess.backend.reset_workq); ++#endif ++} ++KBASE_EXPORT_TEST_API(kbase_job_slot_term); ++ ++#if KBASE_GPU_RESET_EN ++/** ++ * kbasep_check_for_afbc_on_slot() - Check whether AFBC is in use on this slot ++ * @kbdev: kbase device pointer ++ * @kctx: context to check against ++ * @js: slot to check ++ * @target_katom: An atom to check, or NULL if all atoms from @kctx on ++ * slot @js should be checked ++ * ++ * This checks are based upon parameters that would normally be passed to ++ * kbase_job_slot_hardstop(). ++ * ++ * In the event of @target_katom being NULL, this will check the last jobs that ++ * are likely to be running on the slot to see if a) they belong to kctx, and ++ * so would be stopped, and b) whether they have AFBC ++ * ++ * In that case, It's guaranteed that a job currently executing on the HW with ++ * AFBC will be detected. However, this is a conservative check because it also ++ * detects jobs that have just completed too. ++ * ++ * Return: true when hard-stop _might_ stop an afbc atom, else false. ++ */ ++static bool kbasep_check_for_afbc_on_slot(struct kbase_device *kbdev, ++ struct kbase_context *kctx, int js, ++ struct kbase_jd_atom *target_katom) ++{ ++ bool ret = false; ++ int i; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ /* When we have an atom the decision can be made straight away. */ ++ if (target_katom) ++ return !!(target_katom->core_req & BASE_JD_REQ_FS_AFBC); ++ ++ /* Otherwise, we must chweck the hardware to see if it has atoms from ++ * this context with AFBC. */ ++ for (i = 0; i < kbase_backend_nr_atoms_on_slot(kbdev, js); i++) { ++ struct kbase_jd_atom *katom; ++ ++ katom = kbase_gpu_inspect(kbdev, js, i); ++ if (!katom) ++ continue; ++ ++ /* Ignore atoms from other contexts, they won't be stopped when ++ * we use this for checking if we should hard-stop them */ ++ if (katom->kctx != kctx) ++ continue; ++ ++ /* An atom on this slot and this context: check for AFBC */ ++ if (katom->core_req & BASE_JD_REQ_FS_AFBC) { ++ ret = true; ++ break; ++ } ++ } ++ ++ return ret; ++} ++#endif /* KBASE_GPU_RESET_EN */ ++ ++/** ++ * kbase_job_slot_softstop_swflags - Soft-stop a job with flags ++ * @kbdev: The kbase device ++ * @js: The job slot to soft-stop ++ * @target_katom: The job that should be soft-stopped (or NULL for any job) ++ * @sw_flags: Flags to pass in about the soft-stop ++ * ++ * Context: ++ * The job slot lock must be held when calling this function. ++ * The job slot must not already be in the process of being soft-stopped. ++ * ++ * Soft-stop the specified job slot, with extra information about the stop ++ * ++ * Where possible any job in the next register is evicted before the soft-stop. ++ */ ++void kbase_job_slot_softstop_swflags(struct kbase_device *kbdev, int js, ++ struct kbase_jd_atom *target_katom, u32 sw_flags) ++{ ++ KBASE_DEBUG_ASSERT(!(sw_flags & JS_COMMAND_MASK)); ++ kbase_backend_soft_hard_stop_slot(kbdev, NULL, js, target_katom, ++ JS_COMMAND_SOFT_STOP | sw_flags); ++} ++ ++/** ++ * kbase_job_slot_softstop - Soft-stop the specified job slot ++ * @kbdev: The kbase device ++ * @js: The job slot to soft-stop ++ * @target_katom: The job that should be soft-stopped (or NULL for any job) ++ * Context: ++ * The job slot lock must be held when calling this function. ++ * The job slot must not already be in the process of being soft-stopped. ++ * ++ * Where possible any job in the next register is evicted before the soft-stop. ++ */ ++void kbase_job_slot_softstop(struct kbase_device *kbdev, int js, ++ struct kbase_jd_atom *target_katom) ++{ ++ kbase_job_slot_softstop_swflags(kbdev, js, target_katom, 0u); ++} ++ ++/** ++ * kbase_job_slot_hardstop - Hard-stop the specified job slot ++ * @kctx: The kbase context that contains the job(s) that should ++ * be hard-stopped ++ * @js: The job slot to hard-stop ++ * @target_katom: The job that should be hard-stopped (or NULL for all ++ * jobs from the context) ++ * Context: ++ * The job slot lock must be held when calling this function. ++ */ ++void kbase_job_slot_hardstop(struct kbase_context *kctx, int js, ++ struct kbase_jd_atom *target_katom) ++{ ++ struct kbase_device *kbdev = kctx->kbdev; ++ bool stopped; ++#if KBASE_GPU_RESET_EN ++ /* We make the check for AFBC before evicting/stopping atoms. Note ++ * that no other thread can modify the slots whilst we have the ++ * hwaccess_lock. */ ++ int needs_workaround_for_afbc = ++ kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_T76X_3542) ++ && kbasep_check_for_afbc_on_slot(kbdev, kctx, js, ++ target_katom); ++#endif ++ ++ stopped = kbase_backend_soft_hard_stop_slot(kbdev, kctx, js, ++ target_katom, ++ JS_COMMAND_HARD_STOP); ++#if KBASE_GPU_RESET_EN ++ if (stopped && (kbase_hw_has_issue(kctx->kbdev, BASE_HW_ISSUE_8401) || ++ kbase_hw_has_issue(kctx->kbdev, BASE_HW_ISSUE_9510) || ++ needs_workaround_for_afbc)) { ++ /* MIDBASE-2916 if a fragment job with AFBC encoding is ++ * hardstopped, ensure to do a soft reset also in order to ++ * clear the GPU status. ++ * Workaround for HW issue 8401 has an issue,so after ++ * hard-stopping just reset the GPU. This will ensure that the ++ * jobs leave the GPU.*/ ++ if (kbase_prepare_to_reset_gpu_locked(kbdev)) { ++ dev_err(kbdev->dev, "Issueing GPU soft-reset after hard stopping due to hardware issue"); ++ kbase_reset_gpu_locked(kbdev); ++ } ++ } ++#endif ++} ++ ++/** ++ * kbase_job_check_enter_disjoint - potentiall enter disjoint mode ++ * @kbdev: kbase device ++ * @action: the event which has occurred ++ * @core_reqs: core requirements of the atom ++ * @target_katom: the atom which is being affected ++ * ++ * For a certain soft/hard-stop action, work out whether to enter disjoint ++ * state. ++ * ++ * This does not register multiple disjoint events if the atom has already ++ * started a disjoint period ++ * ++ * @core_reqs can be supplied as 0 if the atom had not started on the hardware ++ * (and so a 'real' soft/hard-stop was not required, but it still interrupted ++ * flow, perhaps on another context) ++ * ++ * kbase_job_check_leave_disjoint() should be used to end the disjoint ++ * state when the soft/hard-stop action is complete ++ */ ++void kbase_job_check_enter_disjoint(struct kbase_device *kbdev, u32 action, ++ base_jd_core_req core_reqs, struct kbase_jd_atom *target_katom) ++{ ++ u32 hw_action = action & JS_COMMAND_MASK; ++ ++ /* For hard-stop, don't enter if hard-stop not allowed */ ++ if (hw_action == JS_COMMAND_HARD_STOP && ++ !kbasep_hard_stop_allowed(kbdev, core_reqs)) ++ return; ++ ++ /* For soft-stop, don't enter if soft-stop not allowed, or isn't ++ * causing disjoint */ ++ if (hw_action == JS_COMMAND_SOFT_STOP && ++ !(kbasep_soft_stop_allowed(kbdev, target_katom) && ++ (action & JS_COMMAND_SW_CAUSES_DISJOINT))) ++ return; ++ ++ /* Nothing to do if already logged disjoint state on this atom */ ++ if (target_katom->atom_flags & KBASE_KATOM_FLAG_IN_DISJOINT) ++ return; ++ ++ target_katom->atom_flags |= KBASE_KATOM_FLAG_IN_DISJOINT; ++ kbase_disjoint_state_up(kbdev); ++} ++ ++/** ++ * kbase_job_check_enter_disjoint - potentially leave disjoint state ++ * @kbdev: kbase device ++ * @target_katom: atom which is finishing ++ * ++ * Work out whether to leave disjoint state when finishing an atom that was ++ * originated by kbase_job_check_enter_disjoint(). ++ */ ++void kbase_job_check_leave_disjoint(struct kbase_device *kbdev, ++ struct kbase_jd_atom *target_katom) ++{ ++ if (target_katom->atom_flags & KBASE_KATOM_FLAG_IN_DISJOINT) { ++ target_katom->atom_flags &= ~KBASE_KATOM_FLAG_IN_DISJOINT; ++ kbase_disjoint_state_down(kbdev); ++ } ++} ++ ++ ++#if KBASE_GPU_RESET_EN ++static void kbase_debug_dump_registers(struct kbase_device *kbdev) ++{ ++ int i; ++ ++ kbase_io_history_dump(kbdev); ++ ++ dev_err(kbdev->dev, "Register state:"); ++ dev_err(kbdev->dev, " GPU_IRQ_RAWSTAT=0x%08x GPU_STATUS=0x%08x", ++ kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), NULL), ++ kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_STATUS), NULL)); ++ dev_err(kbdev->dev, " JOB_IRQ_RAWSTAT=0x%08x JOB_IRQ_JS_STATE=0x%08x", ++ kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_RAWSTAT), NULL), ++ kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_JS_STATE), NULL)); ++ for (i = 0; i < 3; i++) { ++ dev_err(kbdev->dev, " JS%d_STATUS=0x%08x JS%d_HEAD_LO=0x%08x", ++ i, kbase_reg_read(kbdev, JOB_SLOT_REG(i, JS_STATUS), ++ NULL), ++ i, kbase_reg_read(kbdev, JOB_SLOT_REG(i, JS_HEAD_LO), ++ NULL)); ++ } ++ dev_err(kbdev->dev, " MMU_IRQ_RAWSTAT=0x%08x GPU_FAULTSTATUS=0x%08x", ++ kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_RAWSTAT), NULL), ++ kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_FAULTSTATUS), NULL)); ++ dev_err(kbdev->dev, " GPU_IRQ_MASK=0x%08x JOB_IRQ_MASK=0x%08x MMU_IRQ_MASK=0x%08x", ++ kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL), ++ kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), NULL), ++ kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK), NULL)); ++ dev_err(kbdev->dev, " PWR_OVERRIDE0=0x%08x PWR_OVERRIDE1=0x%08x", ++ kbase_reg_read(kbdev, GPU_CONTROL_REG(PWR_OVERRIDE0), NULL), ++ kbase_reg_read(kbdev, GPU_CONTROL_REG(PWR_OVERRIDE1), NULL)); ++ dev_err(kbdev->dev, " SHADER_CONFIG=0x%08x L2_MMU_CONFIG=0x%08x", ++ kbase_reg_read(kbdev, GPU_CONTROL_REG(SHADER_CONFIG), NULL), ++ kbase_reg_read(kbdev, GPU_CONTROL_REG(L2_MMU_CONFIG), NULL)); ++ dev_err(kbdev->dev, " TILER_CONFIG=0x%08x JM_CONFIG=0x%08x", ++ kbase_reg_read(kbdev, GPU_CONTROL_REG(TILER_CONFIG), NULL), ++ kbase_reg_read(kbdev, GPU_CONTROL_REG(JM_CONFIG), NULL)); ++} ++ ++static void kbasep_reset_timeout_worker(struct work_struct *data) ++{ ++ unsigned long flags; ++ struct kbase_device *kbdev; ++ ktime_t end_timestamp = ktime_get(); ++ struct kbasep_js_device_data *js_devdata; ++ bool try_schedule = false; ++ bool silent = false; ++ u32 max_loops = KBASE_CLEAN_CACHE_MAX_LOOPS; ++ ++ KBASE_DEBUG_ASSERT(data); ++ ++ kbdev = container_of(data, struct kbase_device, ++ hwaccess.backend.reset_work); ++ ++ KBASE_DEBUG_ASSERT(kbdev); ++ js_devdata = &kbdev->js_data; ++ ++ if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) == ++ KBASE_RESET_GPU_SILENT) ++ silent = true; ++ ++ KBASE_TRACE_ADD(kbdev, JM_BEGIN_RESET_WORKER, NULL, NULL, 0u, 0); ++ ++ /* Suspend vinstr. ++ * This call will block until vinstr is suspended. */ ++ kbase_vinstr_suspend(kbdev->vinstr_ctx); ++ ++ /* Make sure the timer has completed - this cannot be done from ++ * interrupt context, so this cannot be done within ++ * kbasep_try_reset_gpu_early. */ ++ hrtimer_cancel(&kbdev->hwaccess.backend.reset_timer); ++ ++ if (kbase_pm_context_active_handle_suspend(kbdev, ++ KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) { ++ /* This would re-activate the GPU. Since it's already idle, ++ * there's no need to reset it */ ++ atomic_set(&kbdev->hwaccess.backend.reset_gpu, ++ KBASE_RESET_GPU_NOT_PENDING); ++ kbase_disjoint_state_down(kbdev); ++ wake_up(&kbdev->hwaccess.backend.reset_wait); ++ kbase_vinstr_resume(kbdev->vinstr_ctx); ++ return; ++ } ++ ++ KBASE_DEBUG_ASSERT(kbdev->irq_reset_flush == false); ++ ++ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); ++ spin_lock(&kbdev->hwaccess_lock); ++ spin_lock(&kbdev->mmu_mask_change); ++ /* We're about to flush out the IRQs and their bottom half's */ ++ kbdev->irq_reset_flush = true; ++ ++ /* Disable IRQ to avoid IRQ handlers to kick in after releasing the ++ * spinlock; this also clears any outstanding interrupts */ ++ kbase_pm_disable_interrupts_nolock(kbdev); ++ ++ spin_unlock(&kbdev->mmu_mask_change); ++ spin_unlock(&kbdev->hwaccess_lock); ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++ ++ /* Ensure that any IRQ handlers have finished ++ * Must be done without any locks IRQ handlers will take */ ++ kbase_synchronize_irqs(kbdev); ++ ++ /* Flush out any in-flight work items */ ++ kbase_flush_mmu_wqs(kbdev); ++ ++ /* The flush has completed so reset the active indicator */ ++ kbdev->irq_reset_flush = false; ++ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TMIX_8463)) { ++ /* Ensure that L2 is not transitioning when we send the reset ++ * command */ ++ while (--max_loops && kbase_pm_get_trans_cores(kbdev, ++ KBASE_PM_CORE_L2)) ++ ; ++ ++ WARN(!max_loops, "L2 power transition timed out while trying to reset\n"); ++ } ++ ++ mutex_lock(&kbdev->pm.lock); ++ /* We hold the pm lock, so there ought to be a current policy */ ++ KBASE_DEBUG_ASSERT(kbdev->pm.backend.pm_current_policy); ++ ++ /* All slot have been soft-stopped and we've waited ++ * SOFT_STOP_RESET_TIMEOUT for the slots to clear, at this point we ++ * assume that anything that is still left on the GPU is stuck there and ++ * we'll kill it when we reset the GPU */ ++ ++ if (!silent) ++ dev_err(kbdev->dev, "Resetting GPU (allowing up to %d ms)", ++ RESET_TIMEOUT); ++ ++ /* Output the state of some interesting registers to help in the ++ * debugging of GPU resets */ ++ if (!silent) ++ kbase_debug_dump_registers(kbdev); ++ ++ /* Complete any jobs that were still on the GPU */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbdev->protected_mode = false; ++ kbase_backend_reset(kbdev, &end_timestamp); ++ kbase_pm_metrics_update(kbdev, NULL); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ /* Reset the GPU */ ++ kbase_pm_init_hw(kbdev, 0); ++ ++ mutex_unlock(&kbdev->pm.lock); ++ ++ mutex_lock(&js_devdata->runpool_mutex); ++ ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_ctx_sched_restore_all_as(kbdev); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ ++ kbase_pm_enable_interrupts(kbdev); ++ ++ atomic_set(&kbdev->hwaccess.backend.reset_gpu, ++ KBASE_RESET_GPU_NOT_PENDING); ++ ++ kbase_disjoint_state_down(kbdev); ++ ++ wake_up(&kbdev->hwaccess.backend.reset_wait); ++ if (!silent) ++ dev_err(kbdev->dev, "Reset complete"); ++ ++ if (js_devdata->nr_contexts_pullable > 0 && !kbdev->poweroff_pending) ++ try_schedule = true; ++ ++ mutex_unlock(&js_devdata->runpool_mutex); ++ ++ mutex_lock(&kbdev->pm.lock); ++ ++ /* Find out what cores are required now */ ++ kbase_pm_update_cores_state(kbdev); ++ ++ /* Synchronously request and wait for those cores, because if ++ * instrumentation is enabled it would need them immediately. */ ++ kbase_pm_check_transitions_sync(kbdev); ++ ++ mutex_unlock(&kbdev->pm.lock); ++ ++ /* Try submitting some jobs to restart processing */ ++ if (try_schedule) { ++ KBASE_TRACE_ADD(kbdev, JM_SUBMIT_AFTER_RESET, NULL, NULL, 0u, ++ 0); ++ kbase_js_sched_all(kbdev); ++ } ++ ++ /* Process any pending slot updates */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_backend_slot_update(kbdev); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ kbase_pm_context_idle(kbdev); ++ ++ /* Release vinstr */ ++ kbase_vinstr_resume(kbdev->vinstr_ctx); ++ ++ KBASE_TRACE_ADD(kbdev, JM_END_RESET_WORKER, NULL, NULL, 0u, 0); ++} ++ ++static enum hrtimer_restart kbasep_reset_timer_callback(struct hrtimer *timer) ++{ ++ struct kbase_device *kbdev = container_of(timer, struct kbase_device, ++ hwaccess.backend.reset_timer); ++ ++ KBASE_DEBUG_ASSERT(kbdev); ++ ++ /* Reset still pending? */ ++ if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu, ++ KBASE_RESET_GPU_COMMITTED, KBASE_RESET_GPU_HAPPENING) == ++ KBASE_RESET_GPU_COMMITTED) ++ queue_work(kbdev->hwaccess.backend.reset_workq, ++ &kbdev->hwaccess.backend.reset_work); ++ ++ return HRTIMER_NORESTART; ++} ++ ++/* ++ * If all jobs are evicted from the GPU then we can reset the GPU ++ * immediately instead of waiting for the timeout to elapse ++ */ ++ ++static void kbasep_try_reset_gpu_early_locked(struct kbase_device *kbdev) ++{ ++ int i; ++ int pending_jobs = 0; ++ ++ KBASE_DEBUG_ASSERT(kbdev); ++ ++ /* Count the number of jobs */ ++ for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) ++ pending_jobs += kbase_backend_nr_atoms_submitted(kbdev, i); ++ ++ if (pending_jobs > 0) { ++ /* There are still jobs on the GPU - wait */ ++ return; ++ } ++ ++ /* To prevent getting incorrect registers when dumping failed job, ++ * skip early reset. ++ */ ++ if (kbdev->job_fault_debug != false) ++ return; ++ ++ /* Check that the reset has been committed to (i.e. kbase_reset_gpu has ++ * been called), and that no other thread beat this thread to starting ++ * the reset */ ++ if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu, ++ KBASE_RESET_GPU_COMMITTED, KBASE_RESET_GPU_HAPPENING) != ++ KBASE_RESET_GPU_COMMITTED) { ++ /* Reset has already occurred */ ++ return; ++ } ++ ++ queue_work(kbdev->hwaccess.backend.reset_workq, ++ &kbdev->hwaccess.backend.reset_work); ++} ++ ++static void kbasep_try_reset_gpu_early(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbasep_try_reset_gpu_early_locked(kbdev); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++} ++ ++/** ++ * kbase_prepare_to_reset_gpu_locked - Prepare for resetting the GPU ++ * @kbdev: kbase device ++ * ++ * This function just soft-stops all the slots to ensure that as many jobs as ++ * possible are saved. ++ * ++ * Return: ++ * The function returns a boolean which should be interpreted as follows: ++ * true - Prepared for reset, kbase_reset_gpu_locked should be called. ++ * false - Another thread is performing a reset, kbase_reset_gpu should ++ * not be called. ++ */ ++bool kbase_prepare_to_reset_gpu_locked(struct kbase_device *kbdev) ++{ ++ int i; ++ ++ KBASE_DEBUG_ASSERT(kbdev); ++ ++ if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu, ++ KBASE_RESET_GPU_NOT_PENDING, ++ KBASE_RESET_GPU_PREPARED) != ++ KBASE_RESET_GPU_NOT_PENDING) { ++ /* Some other thread is already resetting the GPU */ ++ return false; ++ } ++ ++ kbase_disjoint_state_up(kbdev); ++ ++ for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) ++ kbase_job_slot_softstop(kbdev, i, NULL); ++ ++ return true; ++} ++ ++bool kbase_prepare_to_reset_gpu(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ bool ret; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ret = kbase_prepare_to_reset_gpu_locked(kbdev); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ return ret; ++} ++KBASE_EXPORT_TEST_API(kbase_prepare_to_reset_gpu); ++ ++/* ++ * This function should be called after kbase_prepare_to_reset_gpu if it ++ * returns true. It should never be called without a corresponding call to ++ * kbase_prepare_to_reset_gpu. ++ * ++ * After this function is called (or not called if kbase_prepare_to_reset_gpu ++ * returned false), the caller should wait for ++ * kbdev->hwaccess.backend.reset_waitq to be signalled to know when the reset ++ * has completed. ++ */ ++void kbase_reset_gpu(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev); ++ ++ /* Note this is an assert/atomic_set because it is a software issue for ++ * a race to be occuring here */ ++ KBASE_DEBUG_ASSERT(atomic_read(&kbdev->hwaccess.backend.reset_gpu) == ++ KBASE_RESET_GPU_PREPARED); ++ atomic_set(&kbdev->hwaccess.backend.reset_gpu, ++ KBASE_RESET_GPU_COMMITTED); ++ ++ dev_err(kbdev->dev, "Preparing to soft-reset GPU: Waiting (upto %d ms) for all jobs to complete soft-stop\n", ++ kbdev->reset_timeout_ms); ++ ++ hrtimer_start(&kbdev->hwaccess.backend.reset_timer, ++ HR_TIMER_DELAY_MSEC(kbdev->reset_timeout_ms), ++ HRTIMER_MODE_REL); ++ ++ /* Try resetting early */ ++ kbasep_try_reset_gpu_early(kbdev); ++} ++KBASE_EXPORT_TEST_API(kbase_reset_gpu); ++ ++void kbase_reset_gpu_locked(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev); ++ ++ /* Note this is an assert/atomic_set because it is a software issue for ++ * a race to be occuring here */ ++ KBASE_DEBUG_ASSERT(atomic_read(&kbdev->hwaccess.backend.reset_gpu) == ++ KBASE_RESET_GPU_PREPARED); ++ atomic_set(&kbdev->hwaccess.backend.reset_gpu, ++ KBASE_RESET_GPU_COMMITTED); ++ ++ dev_err(kbdev->dev, "Preparing to soft-reset GPU: Waiting (upto %d ms) for all jobs to complete soft-stop\n", ++ kbdev->reset_timeout_ms); ++ hrtimer_start(&kbdev->hwaccess.backend.reset_timer, ++ HR_TIMER_DELAY_MSEC(kbdev->reset_timeout_ms), ++ HRTIMER_MODE_REL); ++ ++ /* Try resetting early */ ++ kbasep_try_reset_gpu_early_locked(kbdev); ++} ++ ++void kbase_reset_gpu_silent(struct kbase_device *kbdev) ++{ ++ if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu, ++ KBASE_RESET_GPU_NOT_PENDING, ++ KBASE_RESET_GPU_SILENT) != ++ KBASE_RESET_GPU_NOT_PENDING) { ++ /* Some other thread is already resetting the GPU */ ++ return; ++ } ++ ++ kbase_disjoint_state_up(kbdev); ++ ++ queue_work(kbdev->hwaccess.backend.reset_workq, ++ &kbdev->hwaccess.backend.reset_work); ++} ++ ++bool kbase_reset_gpu_active(struct kbase_device *kbdev) ++{ ++ if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) == ++ KBASE_RESET_GPU_NOT_PENDING) ++ return false; ++ ++ return true; ++} ++#endif /* KBASE_GPU_RESET_EN */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_jm_internal.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_jm_internal.h +new file mode 100755 +index 000000000000..1f382b3c1af4 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_jm_internal.h +@@ -0,0 +1,164 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * Job Manager backend-specific low-level APIs. ++ */ ++ ++#ifndef _KBASE_JM_HWACCESS_H_ ++#define _KBASE_JM_HWACCESS_H_ ++ ++#include ++#include ++#include ++ ++#include ++ ++/** ++ * kbase_job_submit_nolock() - Submit a job to a certain job-slot ++ * @kbdev: Device pointer ++ * @katom: Atom to submit ++ * @js: Job slot to submit on ++ * ++ * The caller must check kbasep_jm_is_submit_slots_free() != false before ++ * calling this. ++ * ++ * The following locking conditions are made on the caller: ++ * - it must hold the hwaccess_lock ++ */ ++void kbase_job_submit_nolock(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom, int js); ++ ++/** ++ * kbase_job_done_slot() - Complete the head job on a particular job-slot ++ * @kbdev: Device pointer ++ * @s: Job slot ++ * @completion_code: Completion code of job reported by GPU ++ * @job_tail: Job tail address reported by GPU ++ * @end_timestamp: Timestamp of job completion ++ */ ++void kbase_job_done_slot(struct kbase_device *kbdev, int s, u32 completion_code, ++ u64 job_tail, ktime_t *end_timestamp); ++ ++#ifdef CONFIG_GPU_TRACEPOINTS ++static inline char *kbasep_make_job_slot_string(int js, char *js_string, ++ size_t js_size) ++{ ++ snprintf(js_string, js_size, "job_slot_%i", js); ++ return js_string; ++} ++#endif ++ ++/** ++ * kbase_job_hw_submit() - Submit a job to the GPU ++ * @kbdev: Device pointer ++ * @katom: Atom to submit ++ * @js: Job slot to submit on ++ * ++ * The caller must check kbasep_jm_is_submit_slots_free() != false before ++ * calling this. ++ * ++ * The following locking conditions are made on the caller: ++ * - it must hold the hwaccess_lock ++ */ ++void kbase_job_hw_submit(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom, ++ int js); ++ ++/** ++ * kbasep_job_slot_soft_or_hard_stop_do_action() - Perform a soft or hard stop ++ * on the specified atom ++ * @kbdev: Device pointer ++ * @js: Job slot to stop on ++ * @action: The action to perform, either JSn_COMMAND_HARD_STOP or ++ * JSn_COMMAND_SOFT_STOP ++ * @core_reqs: Core requirements of atom to stop ++ * @target_katom: Atom to stop ++ * ++ * The following locking conditions are made on the caller: ++ * - it must hold the hwaccess_lock ++ */ ++void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev, ++ int js, ++ u32 action, ++ base_jd_core_req core_reqs, ++ struct kbase_jd_atom *target_katom); ++ ++/** ++ * kbase_backend_soft_hard_stop_slot() - Soft or hard stop jobs on a given job ++ * slot belonging to a given context. ++ * @kbdev: Device pointer ++ * @kctx: Context pointer. May be NULL ++ * @katom: Specific atom to stop. May be NULL ++ * @js: Job slot to hard stop ++ * @action: The action to perform, either JSn_COMMAND_HARD_STOP or ++ * JSn_COMMAND_SOFT_STOP ++ * ++ * If no context is provided then all jobs on the slot will be soft or hard ++ * stopped. ++ * ++ * If a katom is provided then only that specific atom will be stopped. In this ++ * case the kctx parameter is ignored. ++ * ++ * Jobs that are on the slot but are not yet on the GPU will be unpulled and ++ * returned to the job scheduler. ++ * ++ * Return: true if an atom was stopped, false otherwise ++ */ ++bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ int js, ++ struct kbase_jd_atom *katom, ++ u32 action); ++ ++/** ++ * kbase_job_slot_init - Initialise job slot framework ++ * @kbdev: Device pointer ++ * ++ * Called on driver initialisation ++ * ++ * Return: 0 on success ++ */ ++int kbase_job_slot_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_job_slot_halt - Halt the job slot framework ++ * @kbdev: Device pointer ++ * ++ * Should prevent any further job slot processing ++ */ ++void kbase_job_slot_halt(struct kbase_device *kbdev); ++ ++/** ++ * kbase_job_slot_term - Terminate job slot framework ++ * @kbdev: Device pointer ++ * ++ * Called on driver termination ++ */ ++void kbase_job_slot_term(struct kbase_device *kbdev); ++ ++/** ++ * kbase_gpu_cacheclean - Cause a GPU cache clean & flush ++ * @kbdev: Device pointer ++ * ++ * Caller must not be in IRQ context ++ */ ++void kbase_gpu_cacheclean(struct kbase_device *kbdev); ++ ++#endif /* _KBASE_JM_HWACCESS_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_jm_rb.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_jm_rb.c +new file mode 100755 +index 000000000000..a41e7b5b7afb +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_jm_rb.c +@@ -0,0 +1,1947 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++/* ++ * Register-based HW access backend specific APIs ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++/* Return whether the specified ringbuffer is empty. HW access lock must be ++ * held */ ++#define SLOT_RB_EMPTY(rb) (rb->write_idx == rb->read_idx) ++/* Return number of atoms currently in the specified ringbuffer. HW access lock ++ * must be held */ ++#define SLOT_RB_ENTRIES(rb) (int)(s8)(rb->write_idx - rb->read_idx) ++ ++static void kbase_gpu_release_atom(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom, ++ ktime_t *end_timestamp); ++ ++/** ++ * kbase_gpu_enqueue_atom - Enqueue an atom in the HW access ringbuffer ++ * @kbdev: Device pointer ++ * @katom: Atom to enqueue ++ * ++ * Context: Caller must hold the HW access lock ++ */ ++static void kbase_gpu_enqueue_atom(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom) ++{ ++ struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[katom->slot_nr]; ++ ++ WARN_ON(SLOT_RB_ENTRIES(rb) >= SLOT_RB_SIZE); ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ rb->entries[rb->write_idx & SLOT_RB_MASK].katom = katom; ++ rb->write_idx++; ++ ++ katom->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_BLOCKED; ++} ++ ++/** ++ * kbase_gpu_dequeue_atom - Remove an atom from the HW access ringbuffer, once ++ * it has been completed ++ * @kbdev: Device pointer ++ * @js: Job slot to remove atom from ++ * @end_timestamp: Pointer to timestamp of atom completion. May be NULL, in ++ * which case current time will be used. ++ * ++ * Context: Caller must hold the HW access lock ++ * ++ * Return: Atom removed from ringbuffer ++ */ ++static struct kbase_jd_atom *kbase_gpu_dequeue_atom(struct kbase_device *kbdev, ++ int js, ++ ktime_t *end_timestamp) ++{ ++ struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js]; ++ struct kbase_jd_atom *katom; ++ ++ if (SLOT_RB_EMPTY(rb)) { ++ WARN(1, "GPU ringbuffer unexpectedly empty\n"); ++ return NULL; ++ } ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ katom = rb->entries[rb->read_idx & SLOT_RB_MASK].katom; ++ ++ kbase_gpu_release_atom(kbdev, katom, end_timestamp); ++ ++ rb->read_idx++; ++ ++ katom->gpu_rb_state = KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB; ++ ++ kbase_js_debug_log_current_affinities(kbdev); ++ ++ return katom; ++} ++ ++struct kbase_jd_atom *kbase_gpu_inspect(struct kbase_device *kbdev, int js, ++ int idx) ++{ ++ struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js]; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if ((SLOT_RB_ENTRIES(rb) - 1) < idx) ++ return NULL; /* idx out of range */ ++ ++ return rb->entries[(rb->read_idx + idx) & SLOT_RB_MASK].katom; ++} ++ ++struct kbase_jd_atom *kbase_backend_inspect_head(struct kbase_device *kbdev, ++ int js) ++{ ++ return kbase_gpu_inspect(kbdev, js, 0); ++} ++ ++struct kbase_jd_atom *kbase_backend_inspect_tail(struct kbase_device *kbdev, ++ int js) ++{ ++ struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js]; ++ ++ if (SLOT_RB_EMPTY(rb)) ++ return NULL; ++ ++ return rb->entries[(rb->write_idx - 1) & SLOT_RB_MASK].katom; ++} ++ ++/** ++ * kbase_gpu_atoms_submitted - Inspect whether a slot has any atoms currently ++ * on the GPU ++ * @kbdev: Device pointer ++ * @js: Job slot to inspect ++ * ++ * Return: true if there are atoms on the GPU for slot js, ++ * false otherwise ++ */ ++static bool kbase_gpu_atoms_submitted(struct kbase_device *kbdev, int js) ++{ ++ int i; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ for (i = 0; i < SLOT_RB_SIZE; i++) { ++ struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i); ++ ++ if (!katom) ++ return false; ++ if (katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED || ++ katom->gpu_rb_state == KBASE_ATOM_GPU_RB_READY) ++ return true; ++ } ++ ++ return false; ++} ++ ++/** ++ * kbase_gpu_atoms_submitted_any() - Inspect whether there are any atoms ++ * currently on the GPU ++ * @kbdev: Device pointer ++ * ++ * Return: true if there are any atoms on the GPU, false otherwise ++ */ ++static bool kbase_gpu_atoms_submitted_any(struct kbase_device *kbdev) ++{ ++ int js; ++ int i; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { ++ for (i = 0; i < SLOT_RB_SIZE; i++) { ++ struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i); ++ ++ if (katom && katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED) ++ return true; ++ } ++ } ++ return false; ++} ++ ++int kbase_backend_nr_atoms_submitted(struct kbase_device *kbdev, int js) ++{ ++ int nr = 0; ++ int i; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ for (i = 0; i < SLOT_RB_SIZE; i++) { ++ struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i); ++ ++ if (katom && (katom->gpu_rb_state == ++ KBASE_ATOM_GPU_RB_SUBMITTED)) ++ nr++; ++ } ++ ++ return nr; ++} ++ ++int kbase_backend_nr_atoms_on_slot(struct kbase_device *kbdev, int js) ++{ ++ int nr = 0; ++ int i; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ for (i = 0; i < SLOT_RB_SIZE; i++) { ++ if (kbase_gpu_inspect(kbdev, js, i)) ++ nr++; ++ } ++ ++ return nr; ++} ++ ++static int kbase_gpu_nr_atoms_on_slot_min(struct kbase_device *kbdev, int js, ++ enum kbase_atom_gpu_rb_state min_rb_state) ++{ ++ int nr = 0; ++ int i; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ for (i = 0; i < SLOT_RB_SIZE; i++) { ++ struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i); ++ ++ if (katom && (katom->gpu_rb_state >= min_rb_state)) ++ nr++; ++ } ++ ++ return nr; ++} ++ ++/** ++ * check_secure_atom - Check if the given atom is in the given secure state and ++ * has a ringbuffer state of at least ++ * KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION ++ * @katom: Atom pointer ++ * @secure: Desired secure state ++ * ++ * Return: true if atom is in the given state, false otherwise ++ */ ++static bool check_secure_atom(struct kbase_jd_atom *katom, bool secure) ++{ ++ if (katom->gpu_rb_state >= ++ KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION && ++ ((kbase_jd_katom_is_protected(katom) && secure) || ++ (!kbase_jd_katom_is_protected(katom) && !secure))) ++ return true; ++ ++ return false; ++} ++ ++/** ++ * kbase_gpu_check_secure_atoms - Check if there are any atoms in the given ++ * secure state in the ringbuffers of at least ++ * state ++ * KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE ++ * @kbdev: Device pointer ++ * @secure: Desired secure state ++ * ++ * Return: true if any atoms are in the given state, false otherwise ++ */ ++static bool kbase_gpu_check_secure_atoms(struct kbase_device *kbdev, ++ bool secure) ++{ ++ int js, i; ++ ++ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { ++ for (i = 0; i < SLOT_RB_SIZE; i++) { ++ struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, ++ js, i); ++ ++ if (katom) { ++ if (check_secure_atom(katom, secure)) ++ return true; ++ } ++ } ++ } ++ ++ return false; ++} ++ ++int kbase_backend_slot_free(struct kbase_device *kbdev, int js) ++{ ++ if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) != ++ KBASE_RESET_GPU_NOT_PENDING) { ++ /* The GPU is being reset - so prevent submission */ ++ return 0; ++ } ++ ++ return SLOT_RB_SIZE - kbase_backend_nr_atoms_on_slot(kbdev, js); ++} ++ ++ ++static void kbasep_js_job_check_deref_cores(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom); ++ ++static bool kbasep_js_job_check_ref_cores(struct kbase_device *kbdev, ++ int js, ++ struct kbase_jd_atom *katom) ++{ ++ /* The most recently checked affinity. Having this at this scope allows ++ * us to guarantee that we've checked the affinity in this function ++ * call. ++ */ ++ u64 recently_chosen_affinity = 0; ++ bool chosen_affinity = false; ++ bool retry; ++ ++ do { ++ retry = false; ++ ++ /* NOTE: The following uses a number of FALLTHROUGHs to optimize ++ * the calls to this function. Ending of the function is ++ * indicated by BREAK OUT */ ++ switch (katom->coreref_state) { ++ /* State when job is first attempted to be run */ ++ case KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED: ++ KBASE_DEBUG_ASSERT(katom->affinity == 0); ++ ++ /* Compute affinity */ ++ if (false == kbase_js_choose_affinity( ++ &recently_chosen_affinity, kbdev, katom, ++ js)) { ++ /* No cores are currently available */ ++ /* *** BREAK OUT: No state transition *** */ ++ break; ++ } ++ ++ chosen_affinity = true; ++ ++ /* Request the cores */ ++ kbase_pm_request_cores(kbdev, ++ katom->core_req & BASE_JD_REQ_T, ++ recently_chosen_affinity); ++ ++ katom->affinity = recently_chosen_affinity; ++ ++ /* Proceed to next state */ ++ katom->coreref_state = ++ KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES; ++ ++ /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ ++ ++ case KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES: ++ { ++ enum kbase_pm_cores_ready cores_ready; ++ ++ KBASE_DEBUG_ASSERT(katom->affinity != 0 || ++ (katom->core_req & BASE_JD_REQ_T)); ++ ++ cores_ready = kbase_pm_register_inuse_cores( ++ kbdev, ++ katom->core_req & BASE_JD_REQ_T, ++ katom->affinity); ++ if (cores_ready == KBASE_NEW_AFFINITY) { ++ /* Affinity no longer valid - return to ++ * previous state */ ++ kbasep_js_job_check_deref_cores(kbdev, ++ katom); ++ KBASE_TRACE_ADD_SLOT_INFO(kbdev, ++ JS_CORE_REF_REGISTER_INUSE_FAILED, ++ katom->kctx, katom, ++ katom->jc, js, ++ (u32) katom->affinity); ++ /* *** BREAK OUT: Return to previous ++ * state, retry *** */ ++ retry = true; ++ break; ++ } ++ if (cores_ready == KBASE_CORES_NOT_READY) { ++ /* Stay in this state and return, to ++ * retry at this state later */ ++ KBASE_TRACE_ADD_SLOT_INFO(kbdev, ++ JS_CORE_REF_REGISTER_INUSE_FAILED, ++ katom->kctx, katom, ++ katom->jc, js, ++ (u32) katom->affinity); ++ /* *** BREAK OUT: No state transition ++ * *** */ ++ break; ++ } ++ /* Proceed to next state */ ++ katom->coreref_state = ++ KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY; ++ } ++ ++ /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ ++ ++ case KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY: ++ KBASE_DEBUG_ASSERT(katom->affinity != 0 || ++ (katom->core_req & BASE_JD_REQ_T)); ++ ++ /* Optimize out choosing the affinity twice in the same ++ * function call */ ++ if (chosen_affinity == false) { ++ /* See if the affinity changed since a previous ++ * call. */ ++ if (false == kbase_js_choose_affinity( ++ &recently_chosen_affinity, ++ kbdev, katom, js)) { ++ /* No cores are currently available */ ++ kbasep_js_job_check_deref_cores(kbdev, ++ katom); ++ KBASE_TRACE_ADD_SLOT_INFO(kbdev, ++ JS_CORE_REF_REQUEST_ON_RECHECK_FAILED, ++ katom->kctx, katom, ++ katom->jc, js, ++ (u32) recently_chosen_affinity); ++ /* *** BREAK OUT: Transition to lower ++ * state *** */ ++ break; ++ } ++ chosen_affinity = true; ++ } ++ ++ /* Now see if this requires a different set of cores */ ++ if (recently_chosen_affinity != katom->affinity) { ++ enum kbase_pm_cores_ready cores_ready; ++ ++ kbase_pm_request_cores(kbdev, ++ katom->core_req & BASE_JD_REQ_T, ++ recently_chosen_affinity); ++ ++ /* Register new cores whilst we still hold the ++ * old ones, to minimize power transitions */ ++ cores_ready = ++ kbase_pm_register_inuse_cores(kbdev, ++ katom->core_req & BASE_JD_REQ_T, ++ recently_chosen_affinity); ++ kbasep_js_job_check_deref_cores(kbdev, katom); ++ ++ /* Fixup the state that was reduced by ++ * deref_cores: */ ++ katom->coreref_state = ++ KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY; ++ katom->affinity = recently_chosen_affinity; ++ if (cores_ready == KBASE_NEW_AFFINITY) { ++ /* Affinity no longer valid - return to ++ * previous state */ ++ katom->coreref_state = ++ KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES; ++ ++ kbasep_js_job_check_deref_cores(kbdev, ++ katom); ++ ++ KBASE_TRACE_ADD_SLOT_INFO(kbdev, ++ JS_CORE_REF_REGISTER_INUSE_FAILED, ++ katom->kctx, katom, ++ katom->jc, js, ++ (u32) katom->affinity); ++ /* *** BREAK OUT: Return to previous ++ * state, retry *** */ ++ retry = true; ++ break; ++ } ++ /* Now might be waiting for powerup again, with ++ * a new affinity */ ++ if (cores_ready == KBASE_CORES_NOT_READY) { ++ /* Return to previous state */ ++ katom->coreref_state = ++ KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES; ++ KBASE_TRACE_ADD_SLOT_INFO(kbdev, ++ JS_CORE_REF_REGISTER_ON_RECHECK_FAILED, ++ katom->kctx, katom, ++ katom->jc, js, ++ (u32) katom->affinity); ++ /* *** BREAK OUT: Transition to lower ++ * state *** */ ++ break; ++ } ++ } ++ /* Proceed to next state */ ++ katom->coreref_state = ++ KBASE_ATOM_COREREF_STATE_CHECK_AFFINITY_VIOLATIONS; ++ ++ /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ ++ case KBASE_ATOM_COREREF_STATE_CHECK_AFFINITY_VIOLATIONS: ++ KBASE_DEBUG_ASSERT(katom->affinity != 0 || ++ (katom->core_req & BASE_JD_REQ_T)); ++ KBASE_DEBUG_ASSERT(katom->affinity == ++ recently_chosen_affinity); ++ ++ /* Note: this is where the caller must've taken the ++ * hwaccess_lock */ ++ ++ /* Check for affinity violations - if there are any, ++ * then we just ask the caller to requeue and try again ++ * later */ ++ if (kbase_js_affinity_would_violate(kbdev, js, ++ katom->affinity) != false) { ++ /* Return to previous state */ ++ katom->coreref_state = ++ KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY; ++ /* *** BREAK OUT: Transition to lower state *** ++ */ ++ KBASE_TRACE_ADD_SLOT_INFO(kbdev, ++ JS_CORE_REF_AFFINITY_WOULD_VIOLATE, ++ katom->kctx, katom, katom->jc, js, ++ (u32) katom->affinity); ++ break; ++ } ++ ++ /* No affinity violations would result, so the cores are ++ * ready */ ++ katom->coreref_state = KBASE_ATOM_COREREF_STATE_READY; ++ /* *** BREAK OUT: Cores Ready *** */ ++ break; ++ ++ default: ++ KBASE_DEBUG_ASSERT_MSG(false, ++ "Unhandled kbase_atom_coreref_state %d", ++ katom->coreref_state); ++ break; ++ } ++ } while (retry != false); ++ ++ return (katom->coreref_state == KBASE_ATOM_COREREF_STATE_READY); ++} ++ ++static void kbasep_js_job_check_deref_cores(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(katom != NULL); ++ ++ switch (katom->coreref_state) { ++ case KBASE_ATOM_COREREF_STATE_READY: ++ /* State where atom was submitted to the HW - just proceed to ++ * power-down */ ++ KBASE_DEBUG_ASSERT(katom->affinity != 0 || ++ (katom->core_req & BASE_JD_REQ_T)); ++ ++ /* *** FALLTHROUGH *** */ ++ ++ case KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY: ++ /* State where cores were registered */ ++ KBASE_DEBUG_ASSERT(katom->affinity != 0 || ++ (katom->core_req & BASE_JD_REQ_T)); ++ kbase_pm_release_cores(kbdev, katom->core_req & BASE_JD_REQ_T, ++ katom->affinity); ++ ++ break; ++ ++ case KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES: ++ /* State where cores were requested, but not registered */ ++ KBASE_DEBUG_ASSERT(katom->affinity != 0 || ++ (katom->core_req & BASE_JD_REQ_T)); ++ kbase_pm_unrequest_cores(kbdev, katom->core_req & BASE_JD_REQ_T, ++ katom->affinity); ++ break; ++ ++ case KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED: ++ /* Initial state - nothing required */ ++ KBASE_DEBUG_ASSERT(katom->affinity == 0); ++ break; ++ ++ default: ++ KBASE_DEBUG_ASSERT_MSG(false, ++ "Unhandled coreref_state: %d", ++ katom->coreref_state); ++ break; ++ } ++ ++ katom->affinity = 0; ++ katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED; ++} ++ ++static void kbasep_js_job_check_deref_cores_nokatom(struct kbase_device *kbdev, ++ base_jd_core_req core_req, u64 affinity, ++ enum kbase_atom_coreref_state coreref_state) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ switch (coreref_state) { ++ case KBASE_ATOM_COREREF_STATE_READY: ++ /* State where atom was submitted to the HW - just proceed to ++ * power-down */ ++ KBASE_DEBUG_ASSERT(affinity != 0 || ++ (core_req & BASE_JD_REQ_T)); ++ ++ /* *** FALLTHROUGH *** */ ++ ++ case KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY: ++ /* State where cores were registered */ ++ KBASE_DEBUG_ASSERT(affinity != 0 || ++ (core_req & BASE_JD_REQ_T)); ++ kbase_pm_release_cores(kbdev, core_req & BASE_JD_REQ_T, ++ affinity); ++ ++ break; ++ ++ case KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES: ++ /* State where cores were requested, but not registered */ ++ KBASE_DEBUG_ASSERT(affinity != 0 || ++ (core_req & BASE_JD_REQ_T)); ++ kbase_pm_unrequest_cores(kbdev, core_req & BASE_JD_REQ_T, ++ affinity); ++ break; ++ ++ case KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED: ++ /* Initial state - nothing required */ ++ KBASE_DEBUG_ASSERT(affinity == 0); ++ break; ++ ++ default: ++ KBASE_DEBUG_ASSERT_MSG(false, ++ "Unhandled coreref_state: %d", ++ coreref_state); ++ break; ++ } ++} ++ ++static void kbase_gpu_release_atom(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom, ++ ktime_t *end_timestamp) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ ++ switch (katom->gpu_rb_state) { ++ case KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB: ++ /* Should be impossible */ ++ WARN(1, "Attempting to release atom not in ringbuffer\n"); ++ break; ++ ++ case KBASE_ATOM_GPU_RB_SUBMITTED: ++ /* Inform power management at start/finish of atom so it can ++ * update its GPU utilisation metrics. Mark atom as not ++ * submitted beforehand. */ ++ katom->gpu_rb_state = KBASE_ATOM_GPU_RB_READY; ++ kbase_pm_metrics_update(kbdev, end_timestamp); ++ ++ if (katom->core_req & BASE_JD_REQ_PERMON) ++ kbase_pm_release_gpu_cycle_counter_nolock(kbdev); ++ /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ ++ ++ KBASE_TLSTREAM_TL_NRET_ATOM_LPU(katom, ++ &kbdev->gpu_props.props.raw_props.js_features ++ [katom->slot_nr]); ++ KBASE_TLSTREAM_TL_NRET_ATOM_AS(katom, &kbdev->as[kctx->as_nr]); ++ KBASE_TLSTREAM_TL_NRET_CTX_LPU(kctx, ++ &kbdev->gpu_props.props.raw_props.js_features ++ [katom->slot_nr]); ++ ++ case KBASE_ATOM_GPU_RB_READY: ++ /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ ++ ++ case KBASE_ATOM_GPU_RB_WAITING_AFFINITY: ++ kbase_js_affinity_release_slot_cores(kbdev, katom->slot_nr, ++ katom->affinity); ++ /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ ++ ++ case KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE: ++ break; ++ ++ case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION: ++ if (katom->protected_state.enter != ++ KBASE_ATOM_ENTER_PROTECTED_CHECK || ++ katom->protected_state.exit != ++ KBASE_ATOM_EXIT_PROTECTED_CHECK) ++ kbdev->protected_mode_transition = false; ++ ++ if (kbase_jd_katom_is_protected(katom) && ++ (katom->protected_state.enter == ++ KBASE_ATOM_ENTER_PROTECTED_IDLE_L2)) { ++ kbase_vinstr_resume(kbdev->vinstr_ctx); ++ ++ /* Go back to configured model for IPA */ ++ kbase_ipa_model_use_configured_locked(kbdev); ++ } ++ ++ ++ /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ ++ ++ case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV: ++ /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ ++ ++ case KBASE_ATOM_GPU_RB_WAITING_BLOCKED: ++ /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ ++ ++ case KBASE_ATOM_GPU_RB_RETURN_TO_JS: ++ break; ++ } ++ ++ katom->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_BLOCKED; ++ katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK; ++} ++ ++static void kbase_gpu_mark_atom_for_return(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom) ++{ ++ kbase_gpu_release_atom(kbdev, katom, NULL); ++ katom->gpu_rb_state = KBASE_ATOM_GPU_RB_RETURN_TO_JS; ++} ++ ++static inline bool kbase_gpu_rmu_workaround(struct kbase_device *kbdev, int js) ++{ ++ struct kbase_backend_data *backend = &kbdev->hwaccess.backend; ++ bool slot_busy[3]; ++ ++ if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) ++ return true; ++ slot_busy[0] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 0, ++ KBASE_ATOM_GPU_RB_WAITING_AFFINITY); ++ slot_busy[1] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 1, ++ KBASE_ATOM_GPU_RB_WAITING_AFFINITY); ++ slot_busy[2] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 2, ++ KBASE_ATOM_GPU_RB_WAITING_AFFINITY); ++ ++ if ((js == 2 && !(slot_busy[0] || slot_busy[1])) || ++ (js != 2 && !slot_busy[2])) ++ return true; ++ ++ /* Don't submit slot 2 atom while GPU has jobs on slots 0/1 */ ++ if (js == 2 && (kbase_gpu_atoms_submitted(kbdev, 0) || ++ kbase_gpu_atoms_submitted(kbdev, 1) || ++ backend->rmu_workaround_flag)) ++ return false; ++ ++ /* Don't submit slot 0/1 atom while GPU has jobs on slot 2 */ ++ if (js != 2 && (kbase_gpu_atoms_submitted(kbdev, 2) || ++ !backend->rmu_workaround_flag)) ++ return false; ++ ++ backend->rmu_workaround_flag = !backend->rmu_workaround_flag; ++ ++ return true; ++} ++ ++/** ++ * other_slots_busy - Determine if any job slots other than @js are currently ++ * running atoms ++ * @kbdev: Device pointer ++ * @js: Job slot ++ * ++ * Return: true if any slots other than @js are busy, false otherwise ++ */ ++static inline bool other_slots_busy(struct kbase_device *kbdev, int js) ++{ ++ int slot; ++ ++ for (slot = 0; slot < kbdev->gpu_props.num_job_slots; slot++) { ++ if (slot == js) ++ continue; ++ ++ if (kbase_gpu_nr_atoms_on_slot_min(kbdev, slot, ++ KBASE_ATOM_GPU_RB_SUBMITTED)) ++ return true; ++ } ++ ++ return false; ++} ++ ++static inline bool kbase_gpu_in_protected_mode(struct kbase_device *kbdev) ++{ ++ return kbdev->protected_mode; ++} ++ ++static int kbase_gpu_protected_mode_enter(struct kbase_device *kbdev) ++{ ++ int err = -EINVAL; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ WARN_ONCE(!kbdev->protected_ops, ++ "Cannot enter protected mode: protected callbacks not specified.\n"); ++ ++ /* ++ * When entering into protected mode, we must ensure that the ++ * GPU is not operating in coherent mode as well. This is to ++ * ensure that no protected memory can be leaked. ++ */ ++ if (kbdev->system_coherency == COHERENCY_ACE) ++ kbase_cache_set_coherency_mode(kbdev, COHERENCY_ACE_LITE); ++ ++ if (kbdev->protected_ops) { ++ /* Switch GPU to protected mode */ ++ err = kbdev->protected_ops->protected_mode_enable( ++ kbdev->protected_dev); ++ ++ if (err) ++ dev_warn(kbdev->dev, "Failed to enable protected mode: %d\n", ++ err); ++ else ++ kbdev->protected_mode = true; ++ } ++ ++ return err; ++} ++ ++static int kbase_gpu_protected_mode_reset(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ WARN_ONCE(!kbdev->protected_ops, ++ "Cannot exit protected mode: protected callbacks not specified.\n"); ++ ++ if (!kbdev->protected_ops) ++ return -EINVAL; ++ ++ /* The protected mode disable callback will be called as part of reset ++ */ ++ kbase_reset_gpu_silent(kbdev); ++ ++ return 0; ++} ++ ++static int kbase_jm_enter_protected_mode(struct kbase_device *kbdev, ++ struct kbase_jd_atom **katom, int idx, int js) ++{ ++ int err = 0; ++ ++ switch (katom[idx]->protected_state.enter) { ++ case KBASE_ATOM_ENTER_PROTECTED_CHECK: ++ KBASE_TLSTREAM_AUX_PROTECTED_ENTER_START(kbdev); ++ /* The checks in KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV ++ * should ensure that we are not already transitiong, and that ++ * there are no atoms currently on the GPU. */ ++ WARN_ON(kbdev->protected_mode_transition); ++ WARN_ON(kbase_gpu_atoms_submitted_any(kbdev)); ++ ++ kbdev->protected_mode_transition = true; ++ katom[idx]->protected_state.enter = ++ KBASE_ATOM_ENTER_PROTECTED_VINSTR; ++ ++ /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ ++ ++ case KBASE_ATOM_ENTER_PROTECTED_VINSTR: ++ if (kbase_vinstr_try_suspend(kbdev->vinstr_ctx) < 0) { ++ /* ++ * We can't switch now because ++ * the vinstr core state switch ++ * is not done yet. ++ */ ++ return -EAGAIN; ++ } ++ ++ /* Use generic model for IPA in protected mode */ ++ kbase_ipa_model_use_fallback_locked(kbdev); ++ ++ /* Once reaching this point GPU must be ++ * switched to protected mode or vinstr ++ * re-enabled. */ ++ ++ /* ++ * Not in correct mode, begin protected mode switch. ++ * Entering protected mode requires us to power down the L2, ++ * and drop out of fully coherent mode. ++ */ ++ katom[idx]->protected_state.enter = ++ KBASE_ATOM_ENTER_PROTECTED_IDLE_L2; ++ ++ kbase_pm_update_cores_state_nolock(kbdev); ++ ++ /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ ++ ++ case KBASE_ATOM_ENTER_PROTECTED_IDLE_L2: ++ /* Avoid unnecessary waiting on non-ACE platforms. */ ++ if (kbdev->current_gpu_coherency_mode == COHERENCY_ACE) { ++ if (kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_L2) || ++ kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_L2)) { ++ /* ++ * The L2 is still powered, wait for all the users to ++ * finish with it before doing the actual reset. ++ */ ++ return -EAGAIN; ++ } ++ } ++ ++ katom[idx]->protected_state.enter = ++ KBASE_ATOM_ENTER_PROTECTED_FINISHED; ++ ++ /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ ++ ++ case KBASE_ATOM_ENTER_PROTECTED_FINISHED: ++ ++ /* No jobs running, so we can switch GPU mode right now. */ ++ err = kbase_gpu_protected_mode_enter(kbdev); ++ ++ /* ++ * Regardless of result, we are no longer transitioning ++ * the GPU. ++ */ ++ kbdev->protected_mode_transition = false; ++ KBASE_TLSTREAM_AUX_PROTECTED_ENTER_END(kbdev); ++ if (err) { ++ /* ++ * Failed to switch into protected mode, resume ++ * vinstr core and fail atom. ++ */ ++ kbase_vinstr_resume(kbdev->vinstr_ctx); ++ katom[idx]->event_code = BASE_JD_EVENT_JOB_INVALID; ++ kbase_gpu_mark_atom_for_return(kbdev, katom[idx]); ++ /* Only return if head atom or previous atom ++ * already removed - as atoms must be returned ++ * in order. */ ++ if (idx == 0 || katom[0]->gpu_rb_state == ++ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { ++ kbase_gpu_dequeue_atom(kbdev, js, NULL); ++ kbase_jm_return_atom_to_js(kbdev, katom[idx]); ++ } ++ ++ /* Go back to configured model for IPA */ ++ kbase_ipa_model_use_configured_locked(kbdev); ++ ++ return -EINVAL; ++ } ++ ++ /* Protected mode sanity checks. */ ++ KBASE_DEBUG_ASSERT_MSG( ++ kbase_jd_katom_is_protected(katom[idx]) == ++ kbase_gpu_in_protected_mode(kbdev), ++ "Protected mode of atom (%d) doesn't match protected mode of GPU (%d)", ++ kbase_jd_katom_is_protected(katom[idx]), ++ kbase_gpu_in_protected_mode(kbdev)); ++ katom[idx]->gpu_rb_state = ++ KBASE_ATOM_GPU_RB_READY; ++ } ++ ++ return 0; ++} ++ ++static int kbase_jm_exit_protected_mode(struct kbase_device *kbdev, ++ struct kbase_jd_atom **katom, int idx, int js) ++{ ++ int err = 0; ++ ++ ++ switch (katom[idx]->protected_state.exit) { ++ case KBASE_ATOM_EXIT_PROTECTED_CHECK: ++ KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_START(kbdev); ++ /* The checks in KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV ++ * should ensure that we are not already transitiong, and that ++ * there are no atoms currently on the GPU. */ ++ WARN_ON(kbdev->protected_mode_transition); ++ WARN_ON(kbase_gpu_atoms_submitted_any(kbdev)); ++ ++ /* ++ * Exiting protected mode requires a reset, but first the L2 ++ * needs to be powered down to ensure it's not active when the ++ * reset is issued. ++ */ ++ katom[idx]->protected_state.exit = ++ KBASE_ATOM_EXIT_PROTECTED_IDLE_L2; ++ ++ kbdev->protected_mode_transition = true; ++ kbase_pm_update_cores_state_nolock(kbdev); ++ ++ /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ ++ case KBASE_ATOM_EXIT_PROTECTED_IDLE_L2: ++ if (kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_L2) || ++ kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_L2)) { ++ /* ++ * The L2 is still powered, wait for all the users to ++ * finish with it before doing the actual reset. ++ */ ++ return -EAGAIN; ++ } ++ katom[idx]->protected_state.exit = ++ KBASE_ATOM_EXIT_PROTECTED_RESET; ++ ++ /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ ++ ++ case KBASE_ATOM_EXIT_PROTECTED_RESET: ++ /* Issue the reset to the GPU */ ++ err = kbase_gpu_protected_mode_reset(kbdev); ++ ++ if (err) { ++ kbdev->protected_mode_transition = false; ++ ++ /* Failed to exit protected mode, fail atom */ ++ katom[idx]->event_code = BASE_JD_EVENT_JOB_INVALID; ++ kbase_gpu_mark_atom_for_return(kbdev, katom[idx]); ++ /* Only return if head atom or previous atom ++ * already removed - as atoms must be returned ++ * in order */ ++ if (idx == 0 || katom[0]->gpu_rb_state == ++ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { ++ kbase_gpu_dequeue_atom(kbdev, js, NULL); ++ kbase_jm_return_atom_to_js(kbdev, katom[idx]); ++ } ++ ++ kbase_vinstr_resume(kbdev->vinstr_ctx); ++ ++ /* Use generic model for IPA in protected mode */ ++ kbase_ipa_model_use_fallback_locked(kbdev); ++ ++ return -EINVAL; ++ } ++ ++ katom[idx]->protected_state.exit = ++ KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT; ++ ++ /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ ++ ++ case KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT: ++ /* A GPU reset is issued when exiting protected mode. Once the ++ * reset is done all atoms' state will also be reset. For this ++ * reason, if the atom is still in this state we can safely ++ * say that the reset has not completed i.e., we have not ++ * finished exiting protected mode yet. ++ */ ++ return -EAGAIN; ++ } ++ ++ return 0; ++} ++ ++void kbase_backend_slot_update(struct kbase_device *kbdev) ++{ ++ int js; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { ++ struct kbase_jd_atom *katom[2]; ++ int idx; ++ ++ katom[0] = kbase_gpu_inspect(kbdev, js, 0); ++ katom[1] = kbase_gpu_inspect(kbdev, js, 1); ++ WARN_ON(katom[1] && !katom[0]); ++ ++ for (idx = 0; idx < SLOT_RB_SIZE; idx++) { ++ bool cores_ready; ++ int ret; ++ ++ if (!katom[idx]) ++ continue; ++ ++ switch (katom[idx]->gpu_rb_state) { ++ case KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB: ++ /* Should be impossible */ ++ WARN(1, "Attempting to update atom not in ringbuffer\n"); ++ break; ++ ++ case KBASE_ATOM_GPU_RB_WAITING_BLOCKED: ++ if (katom[idx]->atom_flags & ++ KBASE_KATOM_FLAG_X_DEP_BLOCKED) ++ break; ++ ++ katom[idx]->gpu_rb_state = ++ KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV; ++ ++ /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ ++ ++ case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV: ++ if (kbase_gpu_check_secure_atoms(kbdev, ++ !kbase_jd_katom_is_protected( ++ katom[idx]))) ++ break; ++ ++ if ((idx == 1) && (kbase_jd_katom_is_protected( ++ katom[0]) != ++ kbase_jd_katom_is_protected( ++ katom[1]))) ++ break; ++ ++ if (kbdev->protected_mode_transition) ++ break; ++ ++ katom[idx]->gpu_rb_state = ++ KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION; ++ ++ /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ ++ ++ case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION: ++ ++ /* ++ * Exiting protected mode must be done before ++ * the references on the cores are taken as ++ * a power down the L2 is required which ++ * can't happen after the references for this ++ * atom are taken. ++ */ ++ ++ if (!kbase_gpu_in_protected_mode(kbdev) && ++ kbase_jd_katom_is_protected(katom[idx])) { ++ /* Atom needs to transition into protected mode. */ ++ ret = kbase_jm_enter_protected_mode(kbdev, ++ katom, idx, js); ++ if (ret) ++ break; ++ } else if (kbase_gpu_in_protected_mode(kbdev) && ++ !kbase_jd_katom_is_protected(katom[idx])) { ++ /* Atom needs to transition out of protected mode. */ ++ ret = kbase_jm_exit_protected_mode(kbdev, ++ katom, idx, js); ++ if (ret) ++ break; ++ } ++ katom[idx]->protected_state.exit = ++ KBASE_ATOM_EXIT_PROTECTED_CHECK; ++ ++ /* Atom needs no protected mode transition. */ ++ ++ katom[idx]->gpu_rb_state = ++ KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE; ++ ++ /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ ++ ++ case KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE: ++ if (katom[idx]->will_fail_event_code) { ++ kbase_gpu_mark_atom_for_return(kbdev, ++ katom[idx]); ++ /* Set EVENT_DONE so this atom will be ++ completed, not unpulled. */ ++ katom[idx]->event_code = ++ BASE_JD_EVENT_DONE; ++ /* Only return if head atom or previous ++ * atom already removed - as atoms must ++ * be returned in order. */ ++ if (idx == 0 || katom[0]->gpu_rb_state == ++ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { ++ kbase_gpu_dequeue_atom(kbdev, js, NULL); ++ kbase_jm_return_atom_to_js(kbdev, katom[idx]); ++ } ++ break; ++ } ++ ++ cores_ready = ++ kbasep_js_job_check_ref_cores(kbdev, js, ++ katom[idx]); ++ ++ if (katom[idx]->event_code == ++ BASE_JD_EVENT_PM_EVENT) { ++ katom[idx]->gpu_rb_state = ++ KBASE_ATOM_GPU_RB_RETURN_TO_JS; ++ break; ++ } ++ ++ if (!cores_ready) ++ break; ++ ++ kbase_js_affinity_retain_slot_cores(kbdev, js, ++ katom[idx]->affinity); ++ katom[idx]->gpu_rb_state = ++ KBASE_ATOM_GPU_RB_WAITING_AFFINITY; ++ ++ /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ ++ ++ case KBASE_ATOM_GPU_RB_WAITING_AFFINITY: ++ if (!kbase_gpu_rmu_workaround(kbdev, js)) ++ break; ++ ++ katom[idx]->gpu_rb_state = ++ KBASE_ATOM_GPU_RB_READY; ++ ++ /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ ++ ++ case KBASE_ATOM_GPU_RB_READY: ++ ++ if (idx == 1) { ++ /* Only submit if head atom or previous ++ * atom already submitted */ ++ if ((katom[0]->gpu_rb_state != ++ KBASE_ATOM_GPU_RB_SUBMITTED && ++ katom[0]->gpu_rb_state != ++ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB)) ++ break; ++ ++ /* If intra-slot serialization in use ++ * then don't submit atom to NEXT slot ++ */ ++ if (kbdev->serialize_jobs & ++ KBASE_SERIALIZE_INTRA_SLOT) ++ break; ++ } ++ ++ /* If inter-slot serialization in use then don't ++ * submit atom if any other slots are in use */ ++ if ((kbdev->serialize_jobs & ++ KBASE_SERIALIZE_INTER_SLOT) && ++ other_slots_busy(kbdev, js)) ++ break; ++ ++ if ((kbdev->serialize_jobs & ++ KBASE_SERIALIZE_RESET) && ++ kbase_reset_gpu_active(kbdev)) ++ break; ++ ++ /* Check if this job needs the cycle counter ++ * enabled before submission */ ++ if (katom[idx]->core_req & BASE_JD_REQ_PERMON) ++ kbase_pm_request_gpu_cycle_counter_l2_is_on( ++ kbdev); ++ ++ kbase_job_hw_submit(kbdev, katom[idx], js); ++ katom[idx]->gpu_rb_state = ++ KBASE_ATOM_GPU_RB_SUBMITTED; ++ ++ /* Inform power management at start/finish of ++ * atom so it can update its GPU utilisation ++ * metrics. */ ++ kbase_pm_metrics_update(kbdev, ++ &katom[idx]->start_timestamp); ++ ++ /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ ++ ++ case KBASE_ATOM_GPU_RB_SUBMITTED: ++ /* Atom submitted to HW, nothing else to do */ ++ break; ++ ++ case KBASE_ATOM_GPU_RB_RETURN_TO_JS: ++ /* Only return if head atom or previous atom ++ * already removed - as atoms must be returned ++ * in order */ ++ if (idx == 0 || katom[0]->gpu_rb_state == ++ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { ++ kbase_gpu_dequeue_atom(kbdev, js, NULL); ++ kbase_jm_return_atom_to_js(kbdev, ++ katom[idx]); ++ } ++ break; ++ } ++ } ++ } ++ ++ /* Warn if PRLAM-8987 affinity restrictions are violated */ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) ++ WARN_ON((kbase_gpu_atoms_submitted(kbdev, 0) || ++ kbase_gpu_atoms_submitted(kbdev, 1)) && ++ kbase_gpu_atoms_submitted(kbdev, 2)); ++} ++ ++ ++void kbase_backend_run_atom(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ kbase_gpu_enqueue_atom(kbdev, katom); ++ kbase_backend_slot_update(kbdev); ++} ++ ++#define HAS_DEP(katom) (katom->pre_dep || katom->atom_flags & \ ++ (KBASE_KATOM_FLAG_X_DEP_BLOCKED | KBASE_KATOM_FLAG_FAIL_BLOCKER)) ++ ++bool kbase_gpu_irq_evict(struct kbase_device *kbdev, int js) ++{ ++ struct kbase_jd_atom *katom; ++ struct kbase_jd_atom *next_katom; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ katom = kbase_gpu_inspect(kbdev, js, 0); ++ next_katom = kbase_gpu_inspect(kbdev, js, 1); ++ ++ if (next_katom && katom->kctx == next_katom->kctx && ++ next_katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED && ++ HAS_DEP(next_katom) && ++ (kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_LO), NULL) ++ != 0 || ++ kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_HI), NULL) ++ != 0)) { ++ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT), ++ JS_COMMAND_NOP, NULL); ++ next_katom->gpu_rb_state = KBASE_ATOM_GPU_RB_READY; ++ ++ KBASE_TLSTREAM_TL_NRET_ATOM_LPU(katom, ++ &kbdev->gpu_props.props.raw_props.js_features ++ [katom->slot_nr]); ++ KBASE_TLSTREAM_TL_NRET_ATOM_AS(katom, &kbdev->as ++ [katom->kctx->as_nr]); ++ KBASE_TLSTREAM_TL_NRET_CTX_LPU(katom->kctx, ++ &kbdev->gpu_props.props.raw_props.js_features ++ [katom->slot_nr]); ++ ++ return true; ++ } ++ ++ return false; ++} ++ ++void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js, ++ u32 completion_code, ++ u64 job_tail, ++ ktime_t *end_timestamp) ++{ ++ struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, 0); ++ struct kbase_context *kctx = katom->kctx; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ /* ++ * When a hard-stop is followed close after a soft-stop, the completion ++ * code may be set to STOPPED, even though the job is terminated ++ */ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TMIX_8438)) { ++ if (completion_code == BASE_JD_EVENT_STOPPED && ++ (katom->atom_flags & ++ KBASE_KATOM_FLAG_BEEN_HARD_STOPPED)) { ++ completion_code = BASE_JD_EVENT_TERMINATED; ++ } ++ } ++ ++ if ((kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_6787) || (katom->core_req & ++ BASE_JD_REQ_SKIP_CACHE_END)) && ++ completion_code != BASE_JD_EVENT_DONE && ++ !(completion_code & BASE_JD_SW_EVENT)) { ++ /* When a job chain fails, on a T60x or when ++ * BASE_JD_REQ_SKIP_CACHE_END is set, the GPU cache is not ++ * flushed. To prevent future evictions causing possible memory ++ * corruption we need to flush the cache manually before any ++ * affected memory gets reused. */ ++ katom->need_cache_flush_cores_retained = katom->affinity; ++ kbase_pm_request_cores(kbdev, false, katom->affinity); ++ } else if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10676)) { ++ if (kbdev->gpu_props.num_core_groups > 1 && ++ !(katom->affinity & ++ kbdev->gpu_props.props.coherency_info.group[0].core_mask ++ ) && ++ (katom->affinity & ++ kbdev->gpu_props.props.coherency_info.group[1].core_mask ++ )) { ++ dev_info(kbdev->dev, "JD: Flushing cache due to PRLAM-10676\n"); ++ katom->need_cache_flush_cores_retained = ++ katom->affinity; ++ kbase_pm_request_cores(kbdev, false, ++ katom->affinity); ++ } ++ } ++ ++ katom = kbase_gpu_dequeue_atom(kbdev, js, end_timestamp); ++ kbase_timeline_job_slot_done(kbdev, katom->kctx, katom, js, 0); ++ ++ if (completion_code == BASE_JD_EVENT_STOPPED) { ++ struct kbase_jd_atom *next_katom = kbase_gpu_inspect(kbdev, js, ++ 0); ++ ++ /* ++ * Dequeue next atom from ringbuffers on same slot if required. ++ * This atom will already have been removed from the NEXT ++ * registers by kbase_gpu_soft_hard_stop_slot(), to ensure that ++ * the atoms on this slot are returned in the correct order. ++ */ ++ if (next_katom && katom->kctx == next_katom->kctx && ++ next_katom->sched_priority == ++ katom->sched_priority) { ++ kbase_gpu_dequeue_atom(kbdev, js, end_timestamp); ++ kbase_jm_return_atom_to_js(kbdev, next_katom); ++ } ++ } else if (completion_code != BASE_JD_EVENT_DONE) { ++ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; ++ int i; ++ ++#if KBASE_TRACE_DUMP_ON_JOB_SLOT_ERROR != 0 ++ KBASE_TRACE_DUMP(kbdev); ++#endif ++ kbasep_js_clear_submit_allowed(js_devdata, katom->kctx); ++ ++ /* ++ * Remove all atoms on the same context from ringbuffers. This ++ * will not remove atoms that are already on the GPU, as these ++ * are guaranteed not to have fail dependencies on the failed ++ * atom. ++ */ ++ for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) { ++ struct kbase_jd_atom *katom_idx0 = ++ kbase_gpu_inspect(kbdev, i, 0); ++ struct kbase_jd_atom *katom_idx1 = ++ kbase_gpu_inspect(kbdev, i, 1); ++ ++ if (katom_idx0 && katom_idx0->kctx == katom->kctx && ++ HAS_DEP(katom_idx0) && ++ katom_idx0->gpu_rb_state != ++ KBASE_ATOM_GPU_RB_SUBMITTED) { ++ /* Dequeue katom_idx0 from ringbuffer */ ++ kbase_gpu_dequeue_atom(kbdev, i, end_timestamp); ++ ++ if (katom_idx1 && ++ katom_idx1->kctx == katom->kctx ++ && HAS_DEP(katom_idx1) && ++ katom_idx0->gpu_rb_state != ++ KBASE_ATOM_GPU_RB_SUBMITTED) { ++ /* Dequeue katom_idx1 from ringbuffer */ ++ kbase_gpu_dequeue_atom(kbdev, i, ++ end_timestamp); ++ ++ katom_idx1->event_code = ++ BASE_JD_EVENT_STOPPED; ++ kbase_jm_return_atom_to_js(kbdev, ++ katom_idx1); ++ } ++ katom_idx0->event_code = BASE_JD_EVENT_STOPPED; ++ kbase_jm_return_atom_to_js(kbdev, katom_idx0); ++ ++ } else if (katom_idx1 && ++ katom_idx1->kctx == katom->kctx && ++ HAS_DEP(katom_idx1) && ++ katom_idx1->gpu_rb_state != ++ KBASE_ATOM_GPU_RB_SUBMITTED) { ++ /* Can not dequeue this atom yet - will be ++ * dequeued when atom at idx0 completes */ ++ katom_idx1->event_code = BASE_JD_EVENT_STOPPED; ++ kbase_gpu_mark_atom_for_return(kbdev, ++ katom_idx1); ++ } ++ } ++ } ++ ++ KBASE_TRACE_ADD_SLOT_INFO(kbdev, JM_JOB_DONE, kctx, katom, katom->jc, ++ js, completion_code); ++ ++ if (job_tail != 0 && job_tail != katom->jc) { ++ bool was_updated = (job_tail != katom->jc); ++ ++ /* Some of the job has been executed, so we update the job chain ++ * address to where we should resume from */ ++ katom->jc = job_tail; ++ if (was_updated) ++ KBASE_TRACE_ADD_SLOT(kbdev, JM_UPDATE_HEAD, katom->kctx, ++ katom, job_tail, js); ++ } ++ ++ /* Only update the event code for jobs that weren't cancelled */ ++ if (katom->event_code != BASE_JD_EVENT_JOB_CANCELLED) ++ katom->event_code = (base_jd_event_code)completion_code; ++ ++ kbase_device_trace_register_access(kctx, REG_WRITE, ++ JOB_CONTROL_REG(JOB_IRQ_CLEAR), ++ 1 << js); ++ ++ /* Complete the job, and start new ones ++ * ++ * Also defer remaining work onto the workqueue: ++ * - Re-queue Soft-stopped jobs ++ * - For any other jobs, queue the job back into the dependency system ++ * - Schedule out the parent context if necessary, and schedule a new ++ * one in. ++ */ ++#ifdef CONFIG_GPU_TRACEPOINTS ++ { ++ /* The atom in the HEAD */ ++ struct kbase_jd_atom *next_katom = kbase_gpu_inspect(kbdev, js, ++ 0); ++ ++ if (next_katom && next_katom->gpu_rb_state == ++ KBASE_ATOM_GPU_RB_SUBMITTED) { ++ char js_string[16]; ++ ++ trace_gpu_sched_switch(kbasep_make_job_slot_string(js, ++ js_string, ++ sizeof(js_string)), ++ ktime_to_ns(*end_timestamp), ++ (u32)next_katom->kctx->id, 0, ++ next_katom->work_id); ++ kbdev->hwaccess.backend.slot_rb[js].last_context = ++ next_katom->kctx; ++ } else { ++ char js_string[16]; ++ ++ trace_gpu_sched_switch(kbasep_make_job_slot_string(js, ++ js_string, ++ sizeof(js_string)), ++ ktime_to_ns(ktime_get()), 0, 0, ++ 0); ++ kbdev->hwaccess.backend.slot_rb[js].last_context = 0; ++ } ++ } ++#endif ++ ++ if (kbdev->serialize_jobs & KBASE_SERIALIZE_RESET) ++ kbase_reset_gpu_silent(kbdev); ++ ++ if (completion_code == BASE_JD_EVENT_STOPPED) ++ katom = kbase_jm_return_atom_to_js(kbdev, katom); ++ else ++ katom = kbase_jm_complete(kbdev, katom, end_timestamp); ++ ++ if (katom) { ++ /* Cross-slot dependency has now become runnable. Try to submit ++ * it. */ ++ ++ /* Check if there are lower priority jobs to soft stop */ ++ kbase_job_slot_ctx_priority_check_locked(kctx, katom); ++ ++ kbase_jm_try_kick(kbdev, 1 << katom->slot_nr); ++ } ++ ++ /* Job completion may have unblocked other atoms. Try to update all job ++ * slots */ ++ kbase_backend_slot_update(kbdev); ++} ++ ++void kbase_backend_reset(struct kbase_device *kbdev, ktime_t *end_timestamp) ++{ ++ int js; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ /* Reset should always take the GPU out of protected mode */ ++ WARN_ON(kbase_gpu_in_protected_mode(kbdev)); ++ ++ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { ++ int atom_idx = 0; ++ int idx; ++ ++ for (idx = 0; idx < SLOT_RB_SIZE; idx++) { ++ struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, ++ js, atom_idx); ++ bool keep_in_jm_rb = false; ++ ++ if (!katom) ++ break; ++ if (katom->protected_state.exit == ++ KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT) ++ { ++ KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_END(kbdev); ++ ++ kbase_vinstr_resume(kbdev->vinstr_ctx); ++ ++ /* protected mode sanity checks */ ++ KBASE_DEBUG_ASSERT_MSG( ++ kbase_jd_katom_is_protected(katom) == kbase_gpu_in_protected_mode(kbdev), ++ "Protected mode of atom (%d) doesn't match protected mode of GPU (%d)", ++ kbase_jd_katom_is_protected(katom), kbase_gpu_in_protected_mode(kbdev)); ++ KBASE_DEBUG_ASSERT_MSG( ++ (kbase_jd_katom_is_protected(katom) && js == 0) || ++ !kbase_jd_katom_is_protected(katom), ++ "Protected atom on JS%d not supported", js); ++ } ++ if (katom->gpu_rb_state < KBASE_ATOM_GPU_RB_SUBMITTED) ++ keep_in_jm_rb = true; ++ ++ kbase_gpu_release_atom(kbdev, katom, NULL); ++ ++ /* ++ * If the atom wasn't on HW when the reset was issued ++ * then leave it in the RB and next time we're kicked ++ * it will be processed again from the starting state. ++ */ ++ if (keep_in_jm_rb) { ++ kbasep_js_job_check_deref_cores(kbdev, katom); ++ katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED; ++ katom->affinity = 0; ++ katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK; ++ /* As the atom was not removed, increment the ++ * index so that we read the correct atom in the ++ * next iteration. */ ++ atom_idx++; ++ continue; ++ } ++ ++ /* ++ * The atom was on the HW when the reset was issued ++ * all we can do is fail the atom. ++ */ ++ kbase_gpu_dequeue_atom(kbdev, js, NULL); ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ kbase_jm_complete(kbdev, katom, end_timestamp); ++ } ++ } ++ ++ kbdev->protected_mode_transition = false; ++} ++ ++static inline void kbase_gpu_stop_atom(struct kbase_device *kbdev, ++ int js, ++ struct kbase_jd_atom *katom, ++ u32 action) ++{ ++ u32 hw_action = action & JS_COMMAND_MASK; ++ ++ kbase_job_check_enter_disjoint(kbdev, action, katom->core_req, katom); ++ kbasep_job_slot_soft_or_hard_stop_do_action(kbdev, js, hw_action, ++ katom->core_req, katom); ++ katom->kctx->blocked_js[js][katom->sched_priority] = true; ++} ++ ++static inline void kbase_gpu_remove_atom(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom, ++ u32 action, ++ bool disjoint) ++{ ++ katom->event_code = BASE_JD_EVENT_REMOVED_FROM_NEXT; ++ kbase_gpu_mark_atom_for_return(kbdev, katom); ++ katom->kctx->blocked_js[katom->slot_nr][katom->sched_priority] = true; ++ ++ if (disjoint) ++ kbase_job_check_enter_disjoint(kbdev, action, katom->core_req, ++ katom); ++} ++ ++static int should_stop_x_dep_slot(struct kbase_jd_atom *katom) ++{ ++ if (katom->x_post_dep) { ++ struct kbase_jd_atom *dep_atom = katom->x_post_dep; ++ ++ if (dep_atom->gpu_rb_state != ++ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB && ++ dep_atom->gpu_rb_state != ++ KBASE_ATOM_GPU_RB_RETURN_TO_JS) ++ return dep_atom->slot_nr; ++ } ++ return -1; ++} ++ ++static void kbase_job_evicted(struct kbase_jd_atom *katom) ++{ ++ kbase_timeline_job_slot_done(katom->kctx->kbdev, katom->kctx, katom, ++ katom->slot_nr, KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT); ++} ++ ++bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ int js, ++ struct kbase_jd_atom *katom, ++ u32 action) ++{ ++ struct kbase_jd_atom *katom_idx0; ++ struct kbase_jd_atom *katom_idx1; ++ ++ bool katom_idx0_valid, katom_idx1_valid; ++ ++ bool ret = false; ++ ++ int stop_x_dep_idx0 = -1, stop_x_dep_idx1 = -1; ++ int prio_idx0 = 0, prio_idx1 = 0; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ katom_idx0 = kbase_gpu_inspect(kbdev, js, 0); ++ katom_idx1 = kbase_gpu_inspect(kbdev, js, 1); ++ ++ if (katom_idx0) ++ prio_idx0 = katom_idx0->sched_priority; ++ if (katom_idx1) ++ prio_idx1 = katom_idx1->sched_priority; ++ ++ if (katom) { ++ katom_idx0_valid = (katom_idx0 == katom); ++ /* If idx0 is to be removed and idx1 is on the same context, ++ * then idx1 must also be removed otherwise the atoms might be ++ * returned out of order */ ++ if (katom_idx1) ++ katom_idx1_valid = (katom_idx1 == katom) || ++ (katom_idx0_valid && ++ (katom_idx0->kctx == ++ katom_idx1->kctx)); ++ else ++ katom_idx1_valid = false; ++ } else { ++ katom_idx0_valid = (katom_idx0 && ++ (!kctx || katom_idx0->kctx == kctx)); ++ katom_idx1_valid = (katom_idx1 && ++ (!kctx || katom_idx1->kctx == kctx) && ++ prio_idx0 == prio_idx1); ++ } ++ ++ if (katom_idx0_valid) ++ stop_x_dep_idx0 = should_stop_x_dep_slot(katom_idx0); ++ if (katom_idx1_valid) ++ stop_x_dep_idx1 = should_stop_x_dep_slot(katom_idx1); ++ ++ if (katom_idx0_valid) { ++ if (katom_idx0->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) { ++ /* Simple case - just dequeue and return */ ++ kbase_gpu_dequeue_atom(kbdev, js, NULL); ++ if (katom_idx1_valid) { ++ kbase_gpu_dequeue_atom(kbdev, js, NULL); ++ katom_idx1->event_code = ++ BASE_JD_EVENT_REMOVED_FROM_NEXT; ++ kbase_jm_return_atom_to_js(kbdev, katom_idx1); ++ katom_idx1->kctx->blocked_js[js][prio_idx1] = ++ true; ++ } ++ ++ katom_idx0->event_code = ++ BASE_JD_EVENT_REMOVED_FROM_NEXT; ++ kbase_jm_return_atom_to_js(kbdev, katom_idx0); ++ katom_idx0->kctx->blocked_js[js][prio_idx0] = true; ++ } else { ++ /* katom_idx0 is on GPU */ ++ if (katom_idx1 && katom_idx1->gpu_rb_state == ++ KBASE_ATOM_GPU_RB_SUBMITTED) { ++ /* katom_idx0 and katom_idx1 are on GPU */ ++ ++ if (kbase_reg_read(kbdev, JOB_SLOT_REG(js, ++ JS_COMMAND_NEXT), NULL) == 0) { ++ /* idx0 has already completed - stop ++ * idx1 if needed*/ ++ if (katom_idx1_valid) { ++ kbase_gpu_stop_atom(kbdev, js, ++ katom_idx1, ++ action); ++ ret = true; ++ } ++ } else { ++ /* idx1 is in NEXT registers - attempt ++ * to remove */ ++ kbase_reg_write(kbdev, ++ JOB_SLOT_REG(js, ++ JS_COMMAND_NEXT), ++ JS_COMMAND_NOP, NULL); ++ ++ if (kbase_reg_read(kbdev, ++ JOB_SLOT_REG(js, ++ JS_HEAD_NEXT_LO), NULL) ++ != 0 || ++ kbase_reg_read(kbdev, ++ JOB_SLOT_REG(js, ++ JS_HEAD_NEXT_HI), NULL) ++ != 0) { ++ /* idx1 removed successfully, ++ * will be handled in IRQ */ ++ kbase_job_evicted(katom_idx1); ++ kbase_gpu_remove_atom(kbdev, ++ katom_idx1, ++ action, true); ++ stop_x_dep_idx1 = ++ should_stop_x_dep_slot(katom_idx1); ++ ++ /* stop idx0 if still on GPU */ ++ kbase_gpu_stop_atom(kbdev, js, ++ katom_idx0, ++ action); ++ ret = true; ++ } else if (katom_idx1_valid) { ++ /* idx0 has already completed, ++ * stop idx1 if needed */ ++ kbase_gpu_stop_atom(kbdev, js, ++ katom_idx1, ++ action); ++ ret = true; ++ } ++ } ++ } else if (katom_idx1_valid) { ++ /* idx1 not on GPU but must be dequeued*/ ++ ++ /* idx1 will be handled in IRQ */ ++ kbase_gpu_remove_atom(kbdev, katom_idx1, action, ++ false); ++ /* stop idx0 */ ++ /* This will be repeated for anything removed ++ * from the next registers, since their normal ++ * flow was also interrupted, and this function ++ * might not enter disjoint state e.g. if we ++ * don't actually do a hard stop on the head ++ * atom */ ++ kbase_gpu_stop_atom(kbdev, js, katom_idx0, ++ action); ++ ret = true; ++ } else { ++ /* no atom in idx1 */ ++ /* just stop idx0 */ ++ kbase_gpu_stop_atom(kbdev, js, katom_idx0, ++ action); ++ ret = true; ++ } ++ } ++ } else if (katom_idx1_valid) { ++ if (katom_idx1->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) { ++ /* Mark for return */ ++ /* idx1 will be returned once idx0 completes */ ++ kbase_gpu_remove_atom(kbdev, katom_idx1, action, ++ false); ++ } else { ++ /* idx1 is on GPU */ ++ if (kbase_reg_read(kbdev, JOB_SLOT_REG(js, ++ JS_COMMAND_NEXT), NULL) == 0) { ++ /* idx0 has already completed - stop idx1 */ ++ kbase_gpu_stop_atom(kbdev, js, katom_idx1, ++ action); ++ ret = true; ++ } else { ++ /* idx1 is in NEXT registers - attempt to ++ * remove */ ++ kbase_reg_write(kbdev, JOB_SLOT_REG(js, ++ JS_COMMAND_NEXT), ++ JS_COMMAND_NOP, NULL); ++ ++ if (kbase_reg_read(kbdev, JOB_SLOT_REG(js, ++ JS_HEAD_NEXT_LO), NULL) != 0 || ++ kbase_reg_read(kbdev, JOB_SLOT_REG(js, ++ JS_HEAD_NEXT_HI), NULL) != 0) { ++ /* idx1 removed successfully, will be ++ * handled in IRQ once idx0 completes */ ++ kbase_job_evicted(katom_idx1); ++ kbase_gpu_remove_atom(kbdev, katom_idx1, ++ action, ++ false); ++ } else { ++ /* idx0 has already completed - stop ++ * idx1 */ ++ kbase_gpu_stop_atom(kbdev, js, ++ katom_idx1, ++ action); ++ ret = true; ++ } ++ } ++ } ++ } ++ ++ ++ if (stop_x_dep_idx0 != -1) ++ kbase_backend_soft_hard_stop_slot(kbdev, kctx, stop_x_dep_idx0, ++ NULL, action); ++ ++ if (stop_x_dep_idx1 != -1) ++ kbase_backend_soft_hard_stop_slot(kbdev, kctx, stop_x_dep_idx1, ++ NULL, action); ++ ++ return ret; ++} ++ ++void kbase_gpu_cacheclean(struct kbase_device *kbdev) ++{ ++ /* Limit the number of loops to avoid a hang if the interrupt is missed ++ */ ++ u32 max_loops = KBASE_CLEAN_CACHE_MAX_LOOPS; ++ ++ mutex_lock(&kbdev->cacheclean_lock); ++ ++ /* use GPU_COMMAND completion solution */ ++ /* clean & invalidate the caches */ ++ KBASE_TRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, NULL, 0u, 0); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), ++ GPU_COMMAND_CLEAN_INV_CACHES, NULL); ++ ++ /* wait for cache flush to complete before continuing */ ++ while (--max_loops && ++ (kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), NULL) & ++ CLEAN_CACHES_COMPLETED) == 0) ++ ; ++ ++ /* clear the CLEAN_CACHES_COMPLETED irq */ ++ KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, NULL, 0u, ++ CLEAN_CACHES_COMPLETED); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), ++ CLEAN_CACHES_COMPLETED, NULL); ++ KBASE_DEBUG_ASSERT_MSG(kbdev->hwcnt.backend.state != ++ KBASE_INSTR_STATE_CLEANING, ++ "Instrumentation code was cleaning caches, but Job Management code cleared their IRQ - Instrumentation code will now hang."); ++ ++ mutex_unlock(&kbdev->cacheclean_lock); ++} ++ ++void kbase_backend_cacheclean(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom) ++{ ++ if (katom->need_cache_flush_cores_retained) { ++ unsigned long flags; ++ ++ kbase_gpu_cacheclean(kbdev); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_pm_unrequest_cores(kbdev, false, ++ katom->need_cache_flush_cores_retained); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ katom->need_cache_flush_cores_retained = 0; ++ } ++} ++ ++void kbase_backend_complete_wq(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom) ++{ ++ /* ++ * If cache flush required due to HW workaround then perform the flush ++ * now ++ */ ++ kbase_backend_cacheclean(kbdev, katom); ++ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10969) && ++ (katom->core_req & BASE_JD_REQ_FS) && ++ katom->event_code == BASE_JD_EVENT_TILE_RANGE_FAULT && ++ (katom->atom_flags & KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED) && ++ !(katom->atom_flags & KBASE_KATOM_FLAGS_RERUN)) { ++ dev_dbg(kbdev->dev, "Soft-stopped fragment shader job got a TILE_RANGE_FAULT. Possible HW issue, trying SW workaround\n"); ++ if (kbasep_10969_workaround_clamp_coordinates(katom)) { ++ /* The job had a TILE_RANGE_FAULT after was soft-stopped ++ * Due to an HW issue we try to execute the job again. ++ */ ++ dev_dbg(kbdev->dev, ++ "Clamping has been executed, try to rerun the job\n" ++ ); ++ katom->event_code = BASE_JD_EVENT_STOPPED; ++ katom->atom_flags |= KBASE_KATOM_FLAGS_RERUN; ++ } ++ } ++ ++ /* Clear the coreref_state now - while check_deref_cores() may not have ++ * been called yet, the caller will have taken a copy of this field. If ++ * this is not done, then if the atom is re-scheduled (following a soft ++ * stop) then the core reference would not be retaken. */ ++ katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED; ++ katom->affinity = 0; ++} ++ ++void kbase_backend_complete_wq_post_sched(struct kbase_device *kbdev, ++ base_jd_core_req core_req, u64 affinity, ++ enum kbase_atom_coreref_state coreref_state) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbasep_js_job_check_deref_cores_nokatom(kbdev, core_req, affinity, ++ coreref_state); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ if (!kbdev->pm.active_count) { ++ mutex_lock(&kbdev->js_data.runpool_mutex); ++ mutex_lock(&kbdev->pm.lock); ++ kbase_pm_update_active(kbdev); ++ mutex_unlock(&kbdev->pm.lock); ++ mutex_unlock(&kbdev->js_data.runpool_mutex); ++ } ++} ++ ++void kbase_gpu_dump_slots(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ int js; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ dev_info(kbdev->dev, "kbase_gpu_dump_slots:\n"); ++ ++ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { ++ int idx; ++ ++ for (idx = 0; idx < SLOT_RB_SIZE; idx++) { ++ struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, ++ js, ++ idx); ++ ++ if (katom) ++ dev_info(kbdev->dev, ++ " js%d idx%d : katom=%p gpu_rb_state=%d\n", ++ js, idx, katom, katom->gpu_rb_state); ++ else ++ dev_info(kbdev->dev, " js%d idx%d : empty\n", ++ js, idx); ++ } ++ } ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++} ++ ++ ++ +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_jm_rb.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_jm_rb.h +new file mode 100755 +index 000000000000..1e0e05ad3ea4 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_jm_rb.h +@@ -0,0 +1,76 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++/* ++ * Register-based HW access backend specific APIs ++ */ ++ ++#ifndef _KBASE_HWACCESS_GPU_H_ ++#define _KBASE_HWACCESS_GPU_H_ ++ ++#include ++ ++/** ++ * kbase_gpu_irq_evict - Evict an atom from a NEXT slot ++ * ++ * @kbdev: Device pointer ++ * @js: Job slot to evict from ++ * ++ * Evict the atom in the NEXT slot for the specified job slot. This function is ++ * called from the job complete IRQ handler when the previous job has failed. ++ * ++ * Return: true if job evicted from NEXT registers, false otherwise ++ */ ++bool kbase_gpu_irq_evict(struct kbase_device *kbdev, int js); ++ ++/** ++ * kbase_gpu_complete_hw - Complete an atom on job slot js ++ * ++ * @kbdev: Device pointer ++ * @js: Job slot that has completed ++ * @completion_code: Event code from job that has completed ++ * @job_tail: The tail address from the hardware if the job has partially ++ * completed ++ * @end_timestamp: Time of completion ++ */ ++void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js, ++ u32 completion_code, ++ u64 job_tail, ++ ktime_t *end_timestamp); ++ ++/** ++ * kbase_gpu_inspect - Inspect the contents of the HW access ringbuffer ++ * ++ * @kbdev: Device pointer ++ * @js: Job slot to inspect ++ * @idx: Index into ringbuffer. 0 is the job currently running on ++ * the slot, 1 is the job waiting, all other values are invalid. ++ * Return: The atom at that position in the ringbuffer ++ * or NULL if no atom present ++ */ ++struct kbase_jd_atom *kbase_gpu_inspect(struct kbase_device *kbdev, int js, ++ int idx); ++ ++/** ++ * kbase_gpu_dump_slots - Print the contents of the slot ringbuffers ++ * ++ * @kbdev: Device pointer ++ */ ++void kbase_gpu_dump_slots(struct kbase_device *kbdev); ++ ++#endif /* _KBASE_HWACCESS_GPU_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_js_affinity.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_js_affinity.c +new file mode 100755 +index 000000000000..54d8ddd80097 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_js_affinity.c +@@ -0,0 +1,303 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * Base kernel affinity manager APIs ++ */ ++ ++#include ++#include "mali_kbase_js_affinity.h" ++#include "mali_kbase_hw.h" ++ ++#include ++ ++ ++bool kbase_js_can_run_job_on_slot_no_lock(struct kbase_device *kbdev, ++ int js) ++{ ++ /* ++ * Here are the reasons for using job slot 2: ++ * - BASE_HW_ISSUE_8987 (which is entirely used for that purpose) ++ * - In absence of the above, then: ++ * - Atoms with BASE_JD_REQ_COHERENT_GROUP ++ * - But, only when there aren't contexts with ++ * KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES, because the atoms that run on ++ * all cores on slot 1 could be blocked by those using a coherent group ++ * on slot 2 ++ * - And, only when you actually have 2 or more coregroups - if you ++ * only have 1 coregroup, then having jobs for slot 2 implies they'd ++ * also be for slot 1, meaning you'll get interference from them. Jobs ++ * able to run on slot 2 could also block jobs that can only run on ++ * slot 1 (tiler jobs) ++ */ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) ++ return true; ++ ++ if (js != 2) ++ return true; ++ ++ /* Only deal with js==2 now: */ ++ if (kbdev->gpu_props.num_core_groups > 1) { ++ /* Only use slot 2 in the 2+ coregroup case */ ++ if (kbasep_js_ctx_attr_is_attr_on_runpool(kbdev, ++ KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES) == ++ false) { ++ /* ...But only when we *don't* have atoms that run on ++ * all cores */ ++ ++ /* No specific check for BASE_JD_REQ_COHERENT_GROUP ++ * atoms - the policy will sort that out */ ++ return true; ++ } ++ } ++ ++ /* Above checks failed mean we shouldn't use slot 2 */ ++ return false; ++} ++ ++/* ++ * As long as it has been decided to have a deeper modification of ++ * what job scheduler, power manager and affinity manager will ++ * implement, this function is just an intermediate step that ++ * assumes: ++ * - all working cores will be powered on when this is called. ++ * - largest current configuration is 2 core groups. ++ * - It has been decided not to have hardcoded values so the low ++ * and high cores in a core split will be evently distributed. ++ * - Odd combinations of core requirements have been filtered out ++ * and do not get to this function (e.g. CS+T+NSS is not ++ * supported here). ++ * - This function is frequently called and can be optimized, ++ * (see notes in loops), but as the functionallity will likely ++ * be modified, optimization has not been addressed. ++*/ ++bool kbase_js_choose_affinity(u64 * const affinity, ++ struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom, int js) ++{ ++ base_jd_core_req core_req = katom->core_req; ++ unsigned int num_core_groups = kbdev->gpu_props.num_core_groups; ++ u64 core_availability_mask; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ core_availability_mask = kbase_pm_ca_get_core_mask(kbdev); ++ ++ /* ++ * If no cores are currently available (core availability policy is ++ * transitioning) then fail. ++ */ ++ if (0 == core_availability_mask) { ++ *affinity = 0; ++ return false; ++ } ++ ++ KBASE_DEBUG_ASSERT(js >= 0); ++ ++ if ((core_req & (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T)) == ++ BASE_JD_REQ_T) { ++ /* If the hardware supports XAFFINITY then we'll only enable ++ * the tiler (which is the default so this is a no-op), ++ * otherwise enable shader core 0. */ ++ if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_XAFFINITY)) ++ *affinity = 1; ++ else ++ *affinity = 0; ++ ++ return true; ++ } ++ ++ if (1 == kbdev->gpu_props.num_cores) { ++ /* trivial case only one core, nothing to do */ ++ *affinity = core_availability_mask & ++ kbdev->pm.debug_core_mask[js]; ++ } else { ++ if ((core_req & (BASE_JD_REQ_COHERENT_GROUP | ++ BASE_JD_REQ_SPECIFIC_COHERENT_GROUP))) { ++ if (js == 0 || num_core_groups == 1) { ++ /* js[0] and single-core-group systems just get ++ * the first core group */ ++ *affinity = ++ kbdev->gpu_props.props.coherency_info.group[0].core_mask ++ & core_availability_mask & ++ kbdev->pm.debug_core_mask[js]; ++ } else { ++ /* js[1], js[2] use core groups 0, 1 for ++ * dual-core-group systems */ ++ u32 core_group_idx = ((u32) js) - 1; ++ ++ KBASE_DEBUG_ASSERT(core_group_idx < ++ num_core_groups); ++ *affinity = ++ kbdev->gpu_props.props.coherency_info.group[core_group_idx].core_mask ++ & core_availability_mask & ++ kbdev->pm.debug_core_mask[js]; ++ ++ /* If the job is specifically targeting core ++ * group 1 and the core availability policy is ++ * keeping that core group off, then fail */ ++ if (*affinity == 0 && core_group_idx == 1 && ++ kbdev->pm.backend.cg1_disabled ++ == true) ++ katom->event_code = ++ BASE_JD_EVENT_PM_EVENT; ++ } ++ } else { ++ /* All cores are available when no core split is ++ * required */ ++ *affinity = core_availability_mask & ++ kbdev->pm.debug_core_mask[js]; ++ } ++ } ++ ++ /* ++ * If no cores are currently available in the desired core group(s) ++ * (core availability policy is transitioning) then fail. ++ */ ++ if (*affinity == 0) ++ return false; ++ ++ /* Enable core 0 if tiler required for hardware without XAFFINITY ++ * support (notes above) */ ++ if (core_req & BASE_JD_REQ_T) { ++ if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_XAFFINITY)) ++ *affinity = *affinity | 1; ++ } ++ ++ return true; ++} ++ ++static inline bool kbase_js_affinity_is_violating( ++ struct kbase_device *kbdev, ++ u64 *affinities) ++{ ++ /* This implementation checks whether the two slots involved in Generic ++ * thread creation have intersecting affinity. This is due to micro- ++ * architectural issues where a job in slot A targetting cores used by ++ * slot B could prevent the job in slot B from making progress until the ++ * job in slot A has completed. ++ */ ++ u64 affinity_set_left; ++ u64 affinity_set_right; ++ u64 intersection; ++ ++ KBASE_DEBUG_ASSERT(affinities != NULL); ++ ++ affinity_set_left = affinities[1]; ++ ++ affinity_set_right = affinities[2]; ++ ++ /* A violation occurs when any bit in the left_set is also in the ++ * right_set */ ++ intersection = affinity_set_left & affinity_set_right; ++ ++ return (bool) (intersection != (u64) 0u); ++} ++ ++bool kbase_js_affinity_would_violate(struct kbase_device *kbdev, int js, ++ u64 affinity) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ u64 new_affinities[BASE_JM_MAX_NR_SLOTS]; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(js < BASE_JM_MAX_NR_SLOTS); ++ js_devdata = &kbdev->js_data; ++ ++ memcpy(new_affinities, js_devdata->runpool_irq.slot_affinities, ++ sizeof(js_devdata->runpool_irq.slot_affinities)); ++ ++ new_affinities[js] |= affinity; ++ ++ return kbase_js_affinity_is_violating(kbdev, new_affinities); ++} ++ ++void kbase_js_affinity_retain_slot_cores(struct kbase_device *kbdev, int js, ++ u64 affinity) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ u64 cores; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(js < BASE_JM_MAX_NR_SLOTS); ++ js_devdata = &kbdev->js_data; ++ ++ KBASE_DEBUG_ASSERT(kbase_js_affinity_would_violate(kbdev, js, affinity) ++ == false); ++ ++ cores = affinity; ++ while (cores) { ++ int bitnum = fls64(cores) - 1; ++ u64 bit = 1ULL << bitnum; ++ s8 cnt; ++ ++ cnt = ++ ++(js_devdata->runpool_irq.slot_affinity_refcount[js][bitnum]); ++ ++ if (cnt == 1) ++ js_devdata->runpool_irq.slot_affinities[js] |= bit; ++ ++ cores &= ~bit; ++ } ++} ++ ++void kbase_js_affinity_release_slot_cores(struct kbase_device *kbdev, int js, ++ u64 affinity) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ u64 cores; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(js < BASE_JM_MAX_NR_SLOTS); ++ js_devdata = &kbdev->js_data; ++ ++ cores = affinity; ++ while (cores) { ++ int bitnum = fls64(cores) - 1; ++ u64 bit = 1ULL << bitnum; ++ s8 cnt; ++ ++ KBASE_DEBUG_ASSERT( ++ js_devdata->runpool_irq.slot_affinity_refcount[js][bitnum] > 0); ++ ++ cnt = ++ --(js_devdata->runpool_irq.slot_affinity_refcount[js][bitnum]); ++ ++ if (0 == cnt) ++ js_devdata->runpool_irq.slot_affinities[js] &= ~bit; ++ ++ cores &= ~bit; ++ } ++} ++ ++#if KBASE_TRACE_ENABLE ++void kbase_js_debug_log_current_affinities(struct kbase_device *kbdev) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ int slot_nr; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ js_devdata = &kbdev->js_data; ++ ++ for (slot_nr = 0; slot_nr < 3; ++slot_nr) ++ KBASE_TRACE_ADD_SLOT_INFO(kbdev, JS_AFFINITY_CURRENT, NULL, ++ NULL, 0u, slot_nr, ++ (u32) js_devdata->runpool_irq.slot_affinities[slot_nr]); ++} ++#endif /* KBASE_TRACE_ENABLE */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_js_affinity.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_js_affinity.h +new file mode 100755 +index 000000000000..35d9781ae092 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_js_affinity.h +@@ -0,0 +1,129 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * Affinity Manager internal APIs. ++ */ ++ ++#ifndef _KBASE_JS_AFFINITY_H_ ++#define _KBASE_JS_AFFINITY_H_ ++ ++/** ++ * kbase_js_can_run_job_on_slot_no_lock - Decide whether it is possible to ++ * submit a job to a particular job slot in the current status ++ * ++ * @kbdev: The kbase device structure of the device ++ * @js: Job slot number to check for allowance ++ * ++ * Will check if submitting to the given job slot is allowed in the current ++ * status. For example using job slot 2 while in soft-stoppable state and only ++ * having 1 coregroup is not allowed by the policy. This function should be ++ * called prior to submitting a job to a slot to make sure policy rules are not ++ * violated. ++ * ++ * The following locking conditions are made on the caller ++ * - it must hold hwaccess_lock ++ */ ++bool kbase_js_can_run_job_on_slot_no_lock(struct kbase_device *kbdev, int js); ++ ++/** ++ * kbase_js_choose_affinity - Compute affinity for a given job. ++ * ++ * @affinity: Affinity bitmap computed ++ * @kbdev: The kbase device structure of the device ++ * @katom: Job chain of which affinity is going to be found ++ * @js: Slot the job chain is being submitted ++ * ++ * Currently assumes an all-on/all-off power management policy. ++ * Also assumes there is at least one core with tiler available. ++ * ++ * Returns true if a valid affinity was chosen, false if ++ * no cores were available. ++ */ ++bool kbase_js_choose_affinity(u64 * const affinity, ++ struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom, ++ int js); ++ ++/** ++ * kbase_js_affinity_would_violate - Determine whether a proposed affinity on ++ * job slot @js would cause a violation of affinity restrictions. ++ * ++ * @kbdev: Kbase device structure ++ * @js: The job slot to test ++ * @affinity: The affinity mask to test ++ * ++ * The following locks must be held by the caller ++ * - hwaccess_lock ++ * ++ * Return: true if the affinity would violate the restrictions ++ */ ++bool kbase_js_affinity_would_violate(struct kbase_device *kbdev, int js, ++ u64 affinity); ++ ++/** ++ * kbase_js_affinity_retain_slot_cores - Affinity tracking: retain cores used by ++ * a slot ++ * ++ * @kbdev: Kbase device structure ++ * @js: The job slot retaining the cores ++ * @affinity: The cores to retain ++ * ++ * The following locks must be held by the caller ++ * - hwaccess_lock ++ */ ++void kbase_js_affinity_retain_slot_cores(struct kbase_device *kbdev, int js, ++ u64 affinity); ++ ++/** ++ * kbase_js_affinity_release_slot_cores - Affinity tracking: release cores used ++ * by a slot ++ * ++ * @kbdev: Kbase device structure ++ * @js: Job slot ++ * @affinity: Bit mask of core to be released ++ * ++ * Cores must be released as soon as a job is dequeued from a slot's 'submit ++ * slots', and before another job is submitted to those slots. Otherwise, the ++ * refcount could exceed the maximum number submittable to a slot, ++ * %BASE_JM_SUBMIT_SLOTS. ++ * ++ * The following locks must be held by the caller ++ * - hwaccess_lock ++ */ ++void kbase_js_affinity_release_slot_cores(struct kbase_device *kbdev, int js, ++ u64 affinity); ++ ++/** ++ * kbase_js_debug_log_current_affinities - log the current affinities ++ * ++ * @kbdev: Kbase device structure ++ * ++ * Output to the Trace log the current tracked affinities on all slots ++ */ ++#if KBASE_TRACE_ENABLE ++void kbase_js_debug_log_current_affinities(struct kbase_device *kbdev); ++#else /* KBASE_TRACE_ENABLE */ ++static inline void ++kbase_js_debug_log_current_affinities(struct kbase_device *kbdev) ++{ ++} ++#endif /* KBASE_TRACE_ENABLE */ ++ ++#endif /* _KBASE_JS_AFFINITY_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_js_backend.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_js_backend.c +new file mode 100755 +index 000000000000..d392fa2a85d9 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_js_backend.c +@@ -0,0 +1,356 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++/* ++ * Register-based HW access backend specific job scheduler APIs ++ */ ++ ++#include ++#include ++#include ++#include ++ ++/* ++ * Define for when dumping is enabled. ++ * This should not be based on the instrumentation level as whether dumping is ++ * enabled for a particular level is down to the integrator. However this is ++ * being used for now as otherwise the cinstr headers would be needed. ++ */ ++#define CINSTR_DUMPING_ENABLED (2 == MALI_INSTRUMENTATION_LEVEL) ++ ++/* ++ * Hold the runpool_mutex for this ++ */ ++static inline bool timer_callback_should_run(struct kbase_device *kbdev) ++{ ++ struct kbase_backend_data *backend = &kbdev->hwaccess.backend; ++ s8 nr_running_ctxs; ++ ++ lockdep_assert_held(&kbdev->js_data.runpool_mutex); ++ ++ /* Timer must stop if we are suspending */ ++ if (backend->suspend_timer) ++ return false; ++ ++ /* nr_contexts_pullable is updated with the runpool_mutex. However, the ++ * locking in the caller gives us a barrier that ensures ++ * nr_contexts_pullable is up-to-date for reading */ ++ nr_running_ctxs = atomic_read(&kbdev->js_data.nr_contexts_runnable); ++ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++ if (kbdev->js_data.softstop_always) { ++ /* Debug support for allowing soft-stop on a single context */ ++ return true; ++ } ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_9435)) { ++ /* Timeouts would have to be 4x longer (due to micro- ++ * architectural design) to support OpenCL conformance tests, so ++ * only run the timer when there's: ++ * - 2 or more CL contexts ++ * - 1 or more GLES contexts ++ * ++ * NOTE: We will treat a context that has both Compute and Non- ++ * Compute jobs will be treated as an OpenCL context (hence, we ++ * don't check KBASEP_JS_CTX_ATTR_NON_COMPUTE). ++ */ ++ { ++ s8 nr_compute_ctxs = ++ kbasep_js_ctx_attr_count_on_runpool(kbdev, ++ KBASEP_JS_CTX_ATTR_COMPUTE); ++ s8 nr_noncompute_ctxs = nr_running_ctxs - ++ nr_compute_ctxs; ++ ++ return (bool) (nr_compute_ctxs >= 2 || ++ nr_noncompute_ctxs > 0); ++ } ++ } else { ++ /* Run the timer callback whenever you have at least 1 context ++ */ ++ return (bool) (nr_running_ctxs > 0); ++ } ++} ++ ++static enum hrtimer_restart timer_callback(struct hrtimer *timer) ++{ ++ unsigned long flags; ++ struct kbase_device *kbdev; ++ struct kbasep_js_device_data *js_devdata; ++ struct kbase_backend_data *backend; ++ int s; ++ bool reset_needed = false; ++ ++ KBASE_DEBUG_ASSERT(timer != NULL); ++ ++ backend = container_of(timer, struct kbase_backend_data, ++ scheduling_timer); ++ kbdev = container_of(backend, struct kbase_device, hwaccess.backend); ++ js_devdata = &kbdev->js_data; ++ ++ /* Loop through the slots */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ for (s = 0; s < kbdev->gpu_props.num_job_slots; s++) { ++ struct kbase_jd_atom *atom = NULL; ++ ++ if (kbase_backend_nr_atoms_on_slot(kbdev, s) > 0) { ++ atom = kbase_gpu_inspect(kbdev, s, 0); ++ KBASE_DEBUG_ASSERT(atom != NULL); ++ } ++ ++ if (atom != NULL) { ++ /* The current version of the model doesn't support ++ * Soft-Stop */ ++ if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_5736)) { ++ u32 ticks = atom->ticks++; ++ ++#if !CINSTR_DUMPING_ENABLED ++ u32 soft_stop_ticks, hard_stop_ticks, ++ gpu_reset_ticks; ++ if (atom->core_req & BASE_JD_REQ_ONLY_COMPUTE) { ++ soft_stop_ticks = ++ js_devdata->soft_stop_ticks_cl; ++ hard_stop_ticks = ++ js_devdata->hard_stop_ticks_cl; ++ gpu_reset_ticks = ++ js_devdata->gpu_reset_ticks_cl; ++ } else { ++ soft_stop_ticks = ++ js_devdata->soft_stop_ticks; ++ hard_stop_ticks = ++ js_devdata->hard_stop_ticks_ss; ++ gpu_reset_ticks = ++ js_devdata->gpu_reset_ticks_ss; ++ } ++ ++ /* If timeouts have been changed then ensure ++ * that atom tick count is not greater than the ++ * new soft_stop timeout. This ensures that ++ * atoms do not miss any of the timeouts due to ++ * races between this worker and the thread ++ * changing the timeouts. */ ++ if (backend->timeouts_updated && ++ ticks > soft_stop_ticks) ++ ticks = atom->ticks = soft_stop_ticks; ++ ++ /* Job is Soft-Stoppable */ ++ if (ticks == soft_stop_ticks) { ++ int disjoint_threshold = ++ KBASE_DISJOINT_STATE_INTERLEAVED_CONTEXT_COUNT_THRESHOLD; ++ u32 softstop_flags = 0u; ++ /* Job has been scheduled for at least ++ * js_devdata->soft_stop_ticks ticks. ++ * Soft stop the slot so we can run ++ * other jobs. ++ */ ++ dev_dbg(kbdev->dev, "Soft-stop"); ++#if !KBASE_DISABLE_SCHEDULING_SOFT_STOPS ++ /* nr_user_contexts_running is updated ++ * with the runpool_mutex, but we can't ++ * take that here. ++ * ++ * However, if it's about to be ++ * increased then the new context can't ++ * run any jobs until they take the ++ * hwaccess_lock, so it's OK to observe ++ * the older value. ++ * ++ * Similarly, if it's about to be ++ * decreased, the last job from another ++ * context has already finished, so it's ++ * not too bad that we observe the older ++ * value and register a disjoint event ++ * when we try soft-stopping */ ++ if (js_devdata->nr_user_contexts_running ++ >= disjoint_threshold) ++ softstop_flags |= ++ JS_COMMAND_SW_CAUSES_DISJOINT; ++ ++ kbase_job_slot_softstop_swflags(kbdev, ++ s, atom, softstop_flags); ++#endif ++ } else if (ticks == hard_stop_ticks) { ++ /* Job has been scheduled for at least ++ * js_devdata->hard_stop_ticks_ss ticks. ++ * It should have been soft-stopped by ++ * now. Hard stop the slot. ++ */ ++#if !KBASE_DISABLE_SCHEDULING_HARD_STOPS ++ int ms = ++ js_devdata->scheduling_period_ns ++ / 1000000u; ++ dev_warn(kbdev->dev, "JS: Job Hard-Stopped (took more than %lu ticks at %lu ms/tick)", ++ (unsigned long)ticks, ++ (unsigned long)ms); ++ kbase_job_slot_hardstop(atom->kctx, s, ++ atom); ++#endif ++ } else if (ticks == gpu_reset_ticks) { ++ /* Job has been scheduled for at least ++ * js_devdata->gpu_reset_ticks_ss ticks. ++ * It should have left the GPU by now. ++ * Signal that the GPU needs to be ++ * reset. ++ */ ++ reset_needed = true; ++ } ++#else /* !CINSTR_DUMPING_ENABLED */ ++ /* NOTE: During CINSTR_DUMPING_ENABLED, we use ++ * the alternate timeouts, which makes the hard- ++ * stop and GPU reset timeout much longer. We ++ * also ensure that we don't soft-stop at all. ++ */ ++ if (ticks == js_devdata->soft_stop_ticks) { ++ /* Job has been scheduled for at least ++ * js_devdata->soft_stop_ticks. We do ++ * not soft-stop during ++ * CINSTR_DUMPING_ENABLED, however. ++ */ ++ dev_dbg(kbdev->dev, "Soft-stop"); ++ } else if (ticks == ++ js_devdata->hard_stop_ticks_dumping) { ++ /* Job has been scheduled for at least ++ * js_devdata->hard_stop_ticks_dumping ++ * ticks. Hard stop the slot. ++ */ ++#if !KBASE_DISABLE_SCHEDULING_HARD_STOPS ++ int ms = ++ js_devdata->scheduling_period_ns ++ / 1000000u; ++ dev_warn(kbdev->dev, "JS: Job Hard-Stopped (took more than %lu ticks at %lu ms/tick)", ++ (unsigned long)ticks, ++ (unsigned long)ms); ++ kbase_job_slot_hardstop(atom->kctx, s, ++ atom); ++#endif ++ } else if (ticks == ++ js_devdata->gpu_reset_ticks_dumping) { ++ /* Job has been scheduled for at least ++ * js_devdata->gpu_reset_ticks_dumping ++ * ticks. It should have left the GPU by ++ * now. Signal that the GPU needs to be ++ * reset. ++ */ ++ reset_needed = true; ++ } ++#endif /* !CINSTR_DUMPING_ENABLED */ ++ } ++ } ++ } ++#if KBASE_GPU_RESET_EN ++ if (reset_needed) { ++ dev_err(kbdev->dev, "JS: Job has been on the GPU for too long (JS_RESET_TICKS_SS/DUMPING timeout hit). Issueing GPU soft-reset to resolve."); ++ ++ if (kbase_prepare_to_reset_gpu_locked(kbdev)) ++ kbase_reset_gpu_locked(kbdev); ++ } ++#endif /* KBASE_GPU_RESET_EN */ ++ /* the timer is re-issued if there is contexts in the run-pool */ ++ ++ if (backend->timer_running) ++ hrtimer_start(&backend->scheduling_timer, ++ HR_TIMER_DELAY_NSEC(js_devdata->scheduling_period_ns), ++ HRTIMER_MODE_REL); ++ ++ backend->timeouts_updated = false; ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ return HRTIMER_NORESTART; ++} ++ ++void kbase_backend_ctx_count_changed(struct kbase_device *kbdev) ++{ ++ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; ++ struct kbase_backend_data *backend = &kbdev->hwaccess.backend; ++ unsigned long flags; ++ ++ lockdep_assert_held(&js_devdata->runpool_mutex); ++ ++ if (!timer_callback_should_run(kbdev)) { ++ /* Take spinlock to force synchronisation with timer */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ backend->timer_running = false; ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ /* From now on, return value of timer_callback_should_run() will ++ * also cause the timer to not requeue itself. Its return value ++ * cannot change, because it depends on variables updated with ++ * the runpool_mutex held, which the caller of this must also ++ * hold */ ++ hrtimer_cancel(&backend->scheduling_timer); ++ } ++ ++ if (timer_callback_should_run(kbdev) && !backend->timer_running) { ++ /* Take spinlock to force synchronisation with timer */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ backend->timer_running = true; ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ hrtimer_start(&backend->scheduling_timer, ++ HR_TIMER_DELAY_NSEC(js_devdata->scheduling_period_ns), ++ HRTIMER_MODE_REL); ++ ++ KBASE_TRACE_ADD(kbdev, JS_POLICY_TIMER_START, NULL, NULL, 0u, ++ 0u); ++ } ++} ++ ++int kbase_backend_timer_init(struct kbase_device *kbdev) ++{ ++ struct kbase_backend_data *backend = &kbdev->hwaccess.backend; ++ ++ hrtimer_init(&backend->scheduling_timer, CLOCK_MONOTONIC, ++ HRTIMER_MODE_REL); ++ backend->scheduling_timer.function = timer_callback; ++ ++ backend->timer_running = false; ++ ++ return 0; ++} ++ ++void kbase_backend_timer_term(struct kbase_device *kbdev) ++{ ++ struct kbase_backend_data *backend = &kbdev->hwaccess.backend; ++ ++ hrtimer_cancel(&backend->scheduling_timer); ++} ++ ++void kbase_backend_timer_suspend(struct kbase_device *kbdev) ++{ ++ struct kbase_backend_data *backend = &kbdev->hwaccess.backend; ++ ++ backend->suspend_timer = true; ++ ++ kbase_backend_ctx_count_changed(kbdev); ++} ++ ++void kbase_backend_timer_resume(struct kbase_device *kbdev) ++{ ++ struct kbase_backend_data *backend = &kbdev->hwaccess.backend; ++ ++ backend->suspend_timer = false; ++ ++ kbase_backend_ctx_count_changed(kbdev); ++} ++ ++void kbase_backend_timeouts_changed(struct kbase_device *kbdev) ++{ ++ struct kbase_backend_data *backend = &kbdev->hwaccess.backend; ++ ++ backend->timeouts_updated = true; ++} ++ +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_js_internal.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_js_internal.h +new file mode 100755 +index 000000000000..3f53779c6747 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_js_internal.h +@@ -0,0 +1,69 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++/* ++ * Register-based HW access backend specific job scheduler APIs ++ */ ++ ++#ifndef _KBASE_JS_BACKEND_H_ ++#define _KBASE_JS_BACKEND_H_ ++ ++/** ++ * kbase_backend_timer_init() - Initialise the JS scheduling timer ++ * @kbdev: Device pointer ++ * ++ * This function should be called at driver initialisation ++ * ++ * Return: 0 on success ++ */ ++int kbase_backend_timer_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_backend_timer_term() - Terminate the JS scheduling timer ++ * @kbdev: Device pointer ++ * ++ * This function should be called at driver termination ++ */ ++void kbase_backend_timer_term(struct kbase_device *kbdev); ++ ++/** ++ * kbase_backend_timer_suspend - Suspend is happening, stop the JS scheduling ++ * timer ++ * @kbdev: Device pointer ++ * ++ * This function should be called on suspend, after the active count has reached ++ * zero. This is required as the timer may have been started on job submission ++ * to the job scheduler, but before jobs are submitted to the GPU. ++ * ++ * Caller must hold runpool_mutex. ++ */ ++void kbase_backend_timer_suspend(struct kbase_device *kbdev); ++ ++/** ++ * kbase_backend_timer_resume - Resume is happening, re-evaluate the JS ++ * scheduling timer ++ * @kbdev: Device pointer ++ * ++ * This function should be called on resume. Note that is is not guaranteed to ++ * re-start the timer, only evalute whether it should be re-started. ++ * ++ * Caller must hold runpool_mutex. ++ */ ++void kbase_backend_timer_resume(struct kbase_device *kbdev); ++ ++#endif /* _KBASE_JS_BACKEND_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_mmu_hw_direct.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_mmu_hw_direct.c +new file mode 100755 +index 000000000000..aa1817c8bca9 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_mmu_hw_direct.c +@@ -0,0 +1,401 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++static inline u64 lock_region(struct kbase_device *kbdev, u64 pfn, ++ u32 num_pages) ++{ ++ u64 region; ++ ++ /* can't lock a zero sized range */ ++ KBASE_DEBUG_ASSERT(num_pages); ++ ++ region = pfn << PAGE_SHIFT; ++ /* ++ * fls returns (given the ASSERT above): ++ * 1 .. 32 ++ * ++ * 10 + fls(num_pages) ++ * results in the range (11 .. 42) ++ */ ++ ++ /* gracefully handle num_pages being zero */ ++ if (0 == num_pages) { ++ region |= 11; ++ } else { ++ u8 region_width; ++ ++ region_width = 10 + fls(num_pages); ++ if (num_pages != (1ul << (region_width - 11))) { ++ /* not pow2, so must go up to the next pow2 */ ++ region_width += 1; ++ } ++ KBASE_DEBUG_ASSERT(region_width <= KBASE_LOCK_REGION_MAX_SIZE); ++ KBASE_DEBUG_ASSERT(region_width >= KBASE_LOCK_REGION_MIN_SIZE); ++ region |= region_width; ++ } ++ ++ return region; ++} ++ ++static int wait_ready(struct kbase_device *kbdev, ++ unsigned int as_nr, struct kbase_context *kctx) ++{ ++ unsigned int max_loops = KBASE_AS_INACTIVE_MAX_LOOPS; ++ u32 val = kbase_reg_read(kbdev, MMU_AS_REG(as_nr, AS_STATUS), kctx); ++ ++ /* Wait for the MMU status to indicate there is no active command, in ++ * case one is pending. Do not log remaining register accesses. */ ++ while (--max_loops && (val & AS_STATUS_AS_ACTIVE)) ++ val = kbase_reg_read(kbdev, MMU_AS_REG(as_nr, AS_STATUS), NULL); ++ ++ if (max_loops == 0) { ++ dev_err(kbdev->dev, "AS_ACTIVE bit stuck\n"); ++ return -1; ++ } ++ ++ /* If waiting in loop was performed, log last read value. */ ++ if (KBASE_AS_INACTIVE_MAX_LOOPS - 1 > max_loops) ++ kbase_reg_read(kbdev, MMU_AS_REG(as_nr, AS_STATUS), kctx); ++ ++ return 0; ++} ++ ++static int write_cmd(struct kbase_device *kbdev, int as_nr, u32 cmd, ++ struct kbase_context *kctx) ++{ ++ int status; ++ ++ /* write AS_COMMAND when MMU is ready to accept another command */ ++ status = wait_ready(kbdev, as_nr, kctx); ++ if (status == 0) ++ kbase_reg_write(kbdev, MMU_AS_REG(as_nr, AS_COMMAND), cmd, ++ kctx); ++ ++ return status; ++} ++ ++static void validate_protected_page_fault(struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++ /* GPUs which support (native) protected mode shall not report page ++ * fault addresses unless it has protected debug mode and protected ++ * debug mode is turned on */ ++ u32 protected_debug_mode = 0; ++ ++ if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_MODE)) ++ return; ++ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE)) { ++ protected_debug_mode = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(GPU_STATUS), ++ kctx) & GPU_DBGEN; ++ } ++ ++ if (!protected_debug_mode) { ++ /* fault_addr should never be reported in protected mode. ++ * However, we just continue by printing an error message */ ++ dev_err(kbdev->dev, "Fault address reported in protected mode\n"); ++ } ++} ++ ++void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat) ++{ ++ const int num_as = 16; ++ const int busfault_shift = MMU_PAGE_FAULT_FLAGS; ++ const int pf_shift = 0; ++ const unsigned long as_bit_mask = (1UL << num_as) - 1; ++ unsigned long flags; ++ u32 new_mask; ++ u32 tmp; ++ ++ /* bus faults */ ++ u32 bf_bits = (irq_stat >> busfault_shift) & as_bit_mask; ++ /* page faults (note: Ignore ASes with both pf and bf) */ ++ u32 pf_bits = ((irq_stat >> pf_shift) & as_bit_mask) & ~bf_bits; ++ ++ KBASE_DEBUG_ASSERT(NULL != kbdev); ++ ++ /* remember current mask */ ++ spin_lock_irqsave(&kbdev->mmu_mask_change, flags); ++ new_mask = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK), NULL); ++ /* mask interrupts for now */ ++ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0, NULL); ++ spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); ++ ++ while (bf_bits | pf_bits) { ++ struct kbase_as *as; ++ int as_no; ++ struct kbase_context *kctx; ++ ++ /* ++ * the while logic ensures we have a bit set, no need to check ++ * for not-found here ++ */ ++ as_no = ffs(bf_bits | pf_bits) - 1; ++ as = &kbdev->as[as_no]; ++ ++ /* ++ * Refcount the kctx ASAP - it shouldn't disappear anyway, since ++ * Bus/Page faults _should_ only occur whilst jobs are running, ++ * and a job causing the Bus/Page fault shouldn't complete until ++ * the MMU is updated ++ */ ++ kctx = kbasep_js_runpool_lookup_ctx(kbdev, as_no); ++ ++ ++ /* find faulting address */ ++ as->fault_addr = kbase_reg_read(kbdev, ++ MMU_AS_REG(as_no, ++ AS_FAULTADDRESS_HI), ++ kctx); ++ as->fault_addr <<= 32; ++ as->fault_addr |= kbase_reg_read(kbdev, ++ MMU_AS_REG(as_no, ++ AS_FAULTADDRESS_LO), ++ kctx); ++ ++ /* Mark the fault protected or not */ ++ as->protected_mode = kbdev->protected_mode; ++ ++ if (kbdev->protected_mode && as->fault_addr) ++ { ++ /* check if address reporting is allowed */ ++ validate_protected_page_fault(kbdev, kctx); ++ } ++ ++ /* report the fault to debugfs */ ++ kbase_as_fault_debugfs_new(kbdev, as_no); ++ ++ /* record the fault status */ ++ as->fault_status = kbase_reg_read(kbdev, ++ MMU_AS_REG(as_no, ++ AS_FAULTSTATUS), ++ kctx); ++ ++ /* find the fault type */ ++ as->fault_type = (bf_bits & (1 << as_no)) ? ++ KBASE_MMU_FAULT_TYPE_BUS : ++ KBASE_MMU_FAULT_TYPE_PAGE; ++ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) { ++ as->fault_extra_addr = kbase_reg_read(kbdev, ++ MMU_AS_REG(as_no, AS_FAULTEXTRA_HI), ++ kctx); ++ as->fault_extra_addr <<= 32; ++ as->fault_extra_addr |= kbase_reg_read(kbdev, ++ MMU_AS_REG(as_no, AS_FAULTEXTRA_LO), ++ kctx); ++ } ++ ++ if (kbase_as_has_bus_fault(as)) { ++ /* Mark bus fault as handled. ++ * Note that a bus fault is processed first in case ++ * where both a bus fault and page fault occur. ++ */ ++ bf_bits &= ~(1UL << as_no); ++ ++ /* remove the queued BF (and PF) from the mask */ ++ new_mask &= ~(MMU_BUS_ERROR(as_no) | ++ MMU_PAGE_FAULT(as_no)); ++ } else { ++ /* Mark page fault as handled */ ++ pf_bits &= ~(1UL << as_no); ++ ++ /* remove the queued PF from the mask */ ++ new_mask &= ~MMU_PAGE_FAULT(as_no); ++ } ++ ++ /* Process the interrupt for this address space */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_mmu_interrupt_process(kbdev, kctx, as); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ } ++ ++ /* reenable interrupts */ ++ spin_lock_irqsave(&kbdev->mmu_mask_change, flags); ++ tmp = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK), NULL); ++ new_mask |= tmp; ++ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), new_mask, NULL); ++ spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); ++} ++ ++void kbase_mmu_hw_configure(struct kbase_device *kbdev, struct kbase_as *as, ++ struct kbase_context *kctx) ++{ ++ struct kbase_mmu_setup *current_setup = &as->current_setup; ++ u32 transcfg = 0; ++ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) { ++ transcfg = current_setup->transcfg & 0xFFFFFFFFUL; ++ ++ /* Set flag AS_TRANSCFG_PTW_MEMATTR_WRITE_BACK */ ++ /* Clear PTW_MEMATTR bits */ ++ transcfg &= ~AS_TRANSCFG_PTW_MEMATTR_MASK; ++ /* Enable correct PTW_MEMATTR bits */ ++ transcfg |= AS_TRANSCFG_PTW_MEMATTR_WRITE_BACK; ++ ++ if (kbdev->system_coherency == COHERENCY_ACE) { ++ /* Set flag AS_TRANSCFG_PTW_SH_OS (outer shareable) */ ++ /* Clear PTW_SH bits */ ++ transcfg = (transcfg & ~AS_TRANSCFG_PTW_SH_MASK); ++ /* Enable correct PTW_SH bits */ ++ transcfg = (transcfg | AS_TRANSCFG_PTW_SH_OS); ++ } ++ ++ kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSCFG_LO), ++ transcfg, kctx); ++ kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSCFG_HI), ++ (current_setup->transcfg >> 32) & 0xFFFFFFFFUL, ++ kctx); ++ } else { ++ if (kbdev->system_coherency == COHERENCY_ACE) ++ current_setup->transtab |= AS_TRANSTAB_LPAE_SHARE_OUTER; ++ } ++ ++ kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSTAB_LO), ++ current_setup->transtab & 0xFFFFFFFFUL, kctx); ++ kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSTAB_HI), ++ (current_setup->transtab >> 32) & 0xFFFFFFFFUL, kctx); ++ ++ kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_MEMATTR_LO), ++ current_setup->memattr & 0xFFFFFFFFUL, kctx); ++ kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_MEMATTR_HI), ++ (current_setup->memattr >> 32) & 0xFFFFFFFFUL, kctx); ++ ++ KBASE_TLSTREAM_TL_ATTRIB_AS_CONFIG(as, ++ current_setup->transtab, ++ current_setup->memattr, ++ transcfg); ++ ++ write_cmd(kbdev, as->number, AS_COMMAND_UPDATE, kctx); ++} ++ ++int kbase_mmu_hw_do_operation(struct kbase_device *kbdev, struct kbase_as *as, ++ struct kbase_context *kctx, u64 vpfn, u32 nr, u32 op, ++ unsigned int handling_irq) ++{ ++ int ret; ++ ++ lockdep_assert_held(&kbdev->mmu_hw_mutex); ++ ++ if (op == AS_COMMAND_UNLOCK) { ++ /* Unlock doesn't require a lock first */ ++ ret = write_cmd(kbdev, as->number, AS_COMMAND_UNLOCK, kctx); ++ } else { ++ u64 lock_addr = lock_region(kbdev, vpfn, nr); ++ ++ /* Lock the region that needs to be updated */ ++ kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_LOCKADDR_LO), ++ lock_addr & 0xFFFFFFFFUL, kctx); ++ kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_LOCKADDR_HI), ++ (lock_addr >> 32) & 0xFFFFFFFFUL, kctx); ++ write_cmd(kbdev, as->number, AS_COMMAND_LOCK, kctx); ++ ++ /* Run the MMU operation */ ++ write_cmd(kbdev, as->number, op, kctx); ++ ++ /* Wait for the flush to complete */ ++ ret = wait_ready(kbdev, as->number, kctx); ++ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_9630)) { ++ /* Issue an UNLOCK command to ensure that valid page ++ tables are re-read by the GPU after an update. ++ Note that, the FLUSH command should perform all the ++ actions necessary, however the bus logs show that if ++ multiple page faults occur within an 8 page region ++ the MMU does not always re-read the updated page ++ table entries for later faults or is only partially ++ read, it subsequently raises the page fault IRQ for ++ the same addresses, the unlock ensures that the MMU ++ cache is flushed, so updates can be re-read. As the ++ region is now unlocked we need to issue 2 UNLOCK ++ commands in order to flush the MMU/uTLB, ++ see PRLAM-8812. ++ */ ++ write_cmd(kbdev, as->number, AS_COMMAND_UNLOCK, kctx); ++ write_cmd(kbdev, as->number, AS_COMMAND_UNLOCK, kctx); ++ } ++ } ++ ++ return ret; ++} ++ ++void kbase_mmu_hw_clear_fault(struct kbase_device *kbdev, struct kbase_as *as, ++ struct kbase_context *kctx, enum kbase_mmu_fault_type type) ++{ ++ unsigned long flags; ++ u32 pf_bf_mask; ++ ++ spin_lock_irqsave(&kbdev->mmu_mask_change, flags); ++ ++ /* ++ * A reset is in-flight and we're flushing the IRQ + bottom half ++ * so don't update anything as it could race with the reset code. ++ */ ++ if (kbdev->irq_reset_flush) ++ goto unlock; ++ ++ /* Clear the page (and bus fault IRQ as well in case one occurred) */ ++ pf_bf_mask = MMU_PAGE_FAULT(as->number); ++ if (type == KBASE_MMU_FAULT_TYPE_BUS || ++ type == KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED) ++ pf_bf_mask |= MMU_BUS_ERROR(as->number); ++ ++ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), pf_bf_mask, kctx); ++ ++unlock: ++ spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); ++} ++ ++void kbase_mmu_hw_enable_fault(struct kbase_device *kbdev, struct kbase_as *as, ++ struct kbase_context *kctx, enum kbase_mmu_fault_type type) ++{ ++ unsigned long flags; ++ u32 irq_mask; ++ ++ /* Enable the page fault IRQ (and bus fault IRQ as well in case one ++ * occurred) */ ++ spin_lock_irqsave(&kbdev->mmu_mask_change, flags); ++ ++ /* ++ * A reset is in-flight and we're flushing the IRQ + bottom half ++ * so don't update anything as it could race with the reset code. ++ */ ++ if (kbdev->irq_reset_flush) ++ goto unlock; ++ ++ irq_mask = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK), kctx) | ++ MMU_PAGE_FAULT(as->number); ++ ++ if (type == KBASE_MMU_FAULT_TYPE_BUS || ++ type == KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED) ++ irq_mask |= MMU_BUS_ERROR(as->number); ++ ++ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), irq_mask, kctx); ++ ++unlock: ++ spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); ++} +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_mmu_hw_direct.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_mmu_hw_direct.h +new file mode 100755 +index 000000000000..c02253c6acc3 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_mmu_hw_direct.h +@@ -0,0 +1,42 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* ++ * Interface file for the direct implementation for MMU hardware access ++ * ++ * Direct MMU hardware interface ++ * ++ * This module provides the interface(s) that are required by the direct ++ * register access implementation of the MMU hardware interface ++ */ ++ ++#ifndef _MALI_KBASE_MMU_HW_DIRECT_H_ ++#define _MALI_KBASE_MMU_HW_DIRECT_H_ ++ ++#include ++ ++/** ++ * kbase_mmu_interrupt - Process an MMU interrupt. ++ * ++ * Process the MMU interrupt that was reported by the &kbase_device. ++ * ++ * @kbdev: kbase context to clear the fault from. ++ * @irq_stat: Value of the MMU_IRQ_STATUS register ++ */ ++void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat); ++ ++#endif /* _MALI_KBASE_MMU_HW_DIRECT_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_always_on.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_always_on.c +new file mode 100755 +index 000000000000..0614348e935a +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_always_on.c +@@ -0,0 +1,63 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * "Always on" power management policy ++ */ ++ ++#include ++#include ++ ++static u64 always_on_get_core_mask(struct kbase_device *kbdev) ++{ ++ return kbdev->gpu_props.props.raw_props.shader_present; ++} ++ ++static bool always_on_get_core_active(struct kbase_device *kbdev) ++{ ++ return true; ++} ++ ++static void always_on_init(struct kbase_device *kbdev) ++{ ++ CSTD_UNUSED(kbdev); ++} ++ ++static void always_on_term(struct kbase_device *kbdev) ++{ ++ CSTD_UNUSED(kbdev); ++} ++ ++/* ++ * The struct kbase_pm_policy structure for the demand power policy. ++ * ++ * This is the static structure that defines the demand power policy's callback ++ * and name. ++ */ ++const struct kbase_pm_policy kbase_pm_always_on_policy_ops = { ++ "always_on", /* name */ ++ always_on_init, /* init */ ++ always_on_term, /* term */ ++ always_on_get_core_mask, /* get_core_mask */ ++ always_on_get_core_active, /* get_core_active */ ++ 0u, /* flags */ ++ KBASE_PM_POLICY_ID_ALWAYS_ON, /* id */ ++}; ++ ++KBASE_EXPORT_TEST_API(kbase_pm_always_on_policy_ops); +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_always_on.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_always_on.h +new file mode 100755 +index 000000000000..f9d244b01bc2 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_always_on.h +@@ -0,0 +1,77 @@ ++ ++/* ++ * ++ * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * "Always on" power management policy ++ */ ++ ++#ifndef MALI_KBASE_PM_ALWAYS_ON_H ++#define MALI_KBASE_PM_ALWAYS_ON_H ++ ++/** ++ * DOC: ++ * The "Always on" power management policy has the following ++ * characteristics: ++ * ++ * - When KBase indicates that the GPU will be powered up, but we don't yet ++ * know which Job Chains are to be run: ++ * All Shader Cores are powered up, regardless of whether or not they will ++ * be needed later. ++ * ++ * - When KBase indicates that a set of Shader Cores are needed to submit the ++ * currently queued Job Chains: ++ * All Shader Cores are kept powered, regardless of whether or not they will ++ * be needed ++ * ++ * - When KBase indicates that the GPU need not be powered: ++ * The Shader Cores are kept powered, regardless of whether or not they will ++ * be needed. The GPU itself is also kept powered, even though it is not ++ * needed. ++ * ++ * This policy is automatically overridden during system suspend: the desired ++ * core state is ignored, and the cores are forced off regardless of what the ++ * policy requests. After resuming from suspend, new changes to the desired ++ * core state made by the policy are honored. ++ * ++ * Note: ++ * ++ * - KBase indicates the GPU will be powered up when it has a User Process that ++ * has just started to submit Job Chains. ++ * ++ * - KBase indicates the GPU need not be powered when all the Job Chains from ++ * User Processes have finished, and it is waiting for a User Process to ++ * submit some more Job Chains. ++ */ ++ ++/** ++ * struct kbasep_pm_policy_always_on - Private struct for policy instance data ++ * @dummy: unused dummy variable ++ * ++ * This contains data that is private to the particular power policy that is ++ * active. ++ */ ++struct kbasep_pm_policy_always_on { ++ int dummy; ++}; ++ ++extern const struct kbase_pm_policy kbase_pm_always_on_policy_ops; ++ ++#endif /* MALI_KBASE_PM_ALWAYS_ON_H */ ++ +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_backend.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_backend.c +new file mode 100755 +index 000000000000..cd8932650ed5 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_backend.c +@@ -0,0 +1,478 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++/* ++ * GPU backend implementation of base kernel power management APIs ++ */ ++ ++#include ++#include ++#include ++ ++#include ++#include ++#include ++#include ++#include ++ ++static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data); ++ ++void kbase_pm_register_access_enable(struct kbase_device *kbdev) ++{ ++ struct kbase_pm_callback_conf *callbacks; ++ ++ callbacks = (struct kbase_pm_callback_conf *)POWER_MANAGEMENT_CALLBACKS; ++ ++ if (callbacks) ++ callbacks->power_on_callback(kbdev); ++ ++ kbdev->pm.backend.gpu_powered = true; ++} ++ ++void kbase_pm_register_access_disable(struct kbase_device *kbdev) ++{ ++ struct kbase_pm_callback_conf *callbacks; ++ ++ callbacks = (struct kbase_pm_callback_conf *)POWER_MANAGEMENT_CALLBACKS; ++ ++ if (callbacks) ++ callbacks->power_off_callback(kbdev); ++ ++ kbdev->pm.backend.gpu_powered = false; ++} ++ ++int kbase_hwaccess_pm_init(struct kbase_device *kbdev) ++{ ++ int ret = 0; ++ struct kbase_pm_callback_conf *callbacks; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ mutex_init(&kbdev->pm.lock); ++ ++ kbdev->pm.backend.gpu_poweroff_wait_wq = alloc_workqueue("kbase_pm_poweroff_wait", ++ WQ_HIGHPRI | WQ_UNBOUND, 1); ++ if (!kbdev->pm.backend.gpu_poweroff_wait_wq) ++ return -ENOMEM; ++ ++ INIT_WORK(&kbdev->pm.backend.gpu_poweroff_wait_work, ++ kbase_pm_gpu_poweroff_wait_wq); ++ ++ kbdev->pm.backend.gpu_powered = false; ++ kbdev->pm.suspending = false; ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++ kbdev->pm.backend.driver_ready_for_irqs = false; ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++ kbdev->pm.backend.gpu_in_desired_state = true; ++ init_waitqueue_head(&kbdev->pm.backend.gpu_in_desired_state_wait); ++ ++ callbacks = (struct kbase_pm_callback_conf *)POWER_MANAGEMENT_CALLBACKS; ++ if (callbacks) { ++ kbdev->pm.backend.callback_power_on = ++ callbacks->power_on_callback; ++ kbdev->pm.backend.callback_power_off = ++ callbacks->power_off_callback; ++ kbdev->pm.backend.callback_power_suspend = ++ callbacks->power_suspend_callback; ++ kbdev->pm.backend.callback_power_resume = ++ callbacks->power_resume_callback; ++ kbdev->pm.callback_power_runtime_init = ++ callbacks->power_runtime_init_callback; ++ kbdev->pm.callback_power_runtime_term = ++ callbacks->power_runtime_term_callback; ++ kbdev->pm.backend.callback_power_runtime_on = ++ callbacks->power_runtime_on_callback; ++ kbdev->pm.backend.callback_power_runtime_off = ++ callbacks->power_runtime_off_callback; ++ kbdev->pm.backend.callback_power_runtime_idle = ++ callbacks->power_runtime_idle_callback; ++ } else { ++ kbdev->pm.backend.callback_power_on = NULL; ++ kbdev->pm.backend.callback_power_off = NULL; ++ kbdev->pm.backend.callback_power_suspend = NULL; ++ kbdev->pm.backend.callback_power_resume = NULL; ++ kbdev->pm.callback_power_runtime_init = NULL; ++ kbdev->pm.callback_power_runtime_term = NULL; ++ kbdev->pm.backend.callback_power_runtime_on = NULL; ++ kbdev->pm.backend.callback_power_runtime_off = NULL; ++ kbdev->pm.backend.callback_power_runtime_idle = NULL; ++ } ++ ++ /* Initialise the metrics subsystem */ ++ ret = kbasep_pm_metrics_init(kbdev); ++ if (ret) ++ return ret; ++ ++ init_waitqueue_head(&kbdev->pm.backend.l2_powered_wait); ++ kbdev->pm.backend.l2_powered = 0; ++ ++ init_waitqueue_head(&kbdev->pm.backend.reset_done_wait); ++ kbdev->pm.backend.reset_done = false; ++ ++ init_waitqueue_head(&kbdev->pm.zero_active_count_wait); ++ kbdev->pm.active_count = 0; ++ ++ spin_lock_init(&kbdev->pm.backend.gpu_cycle_counter_requests_lock); ++ spin_lock_init(&kbdev->pm.backend.gpu_powered_lock); ++ ++ init_waitqueue_head(&kbdev->pm.backend.poweroff_wait); ++ ++ if (kbase_pm_ca_init(kbdev) != 0) ++ goto workq_fail; ++ ++ if (kbase_pm_policy_init(kbdev) != 0) ++ goto pm_policy_fail; ++ ++ return 0; ++ ++pm_policy_fail: ++ kbase_pm_ca_term(kbdev); ++workq_fail: ++ kbasep_pm_metrics_term(kbdev); ++ return -EINVAL; ++} ++ ++void kbase_pm_do_poweron(struct kbase_device *kbdev, bool is_resume) ++{ ++ lockdep_assert_held(&kbdev->pm.lock); ++ ++ /* Turn clocks and interrupts on - no-op if we haven't done a previous ++ * kbase_pm_clock_off() */ ++ kbase_pm_clock_on(kbdev, is_resume); ++ ++ /* Update core status as required by the policy */ ++ KBASE_TIMELINE_PM_CHECKTRANS(kbdev, ++ SW_FLOW_PM_CHECKTRANS_PM_DO_POWERON_START); ++ kbase_pm_update_cores_state(kbdev); ++ KBASE_TIMELINE_PM_CHECKTRANS(kbdev, ++ SW_FLOW_PM_CHECKTRANS_PM_DO_POWERON_END); ++ ++ /* NOTE: We don't wait to reach the desired state, since running atoms ++ * will wait for that state to be reached anyway */ ++} ++ ++static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data) ++{ ++ struct kbase_device *kbdev = container_of(data, struct kbase_device, ++ pm.backend.gpu_poweroff_wait_work); ++ struct kbase_pm_device_data *pm = &kbdev->pm; ++ struct kbase_pm_backend_data *backend = &pm->backend; ++ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; ++ unsigned long flags; ++ ++#if !PLATFORM_POWER_DOWN_ONLY ++ /* Wait for power transitions to complete. We do this with no locks held ++ * so that we don't deadlock with any pending workqueues */ ++ KBASE_TIMELINE_PM_CHECKTRANS(kbdev, ++ SW_FLOW_PM_CHECKTRANS_PM_DO_POWEROFF_START); ++ kbase_pm_check_transitions_sync(kbdev); ++ KBASE_TIMELINE_PM_CHECKTRANS(kbdev, ++ SW_FLOW_PM_CHECKTRANS_PM_DO_POWEROFF_END); ++#endif /* !PLATFORM_POWER_DOWN_ONLY */ ++ ++ mutex_lock(&js_devdata->runpool_mutex); ++ mutex_lock(&kbdev->pm.lock); ++ ++#if PLATFORM_POWER_DOWN_ONLY ++ if (kbdev->pm.backend.gpu_powered) { ++ if (kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_L2)) { ++ /* If L2 cache is powered then we must flush it before ++ * we power off the GPU. Normally this would have been ++ * handled when the L2 was powered off. */ ++ kbase_gpu_cacheclean(kbdev); ++ } ++ } ++#endif /* PLATFORM_POWER_DOWN_ONLY */ ++ ++ if (!backend->poweron_required) { ++#if !PLATFORM_POWER_DOWN_ONLY ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ WARN_ON(kbdev->l2_available_bitmap || ++ kbdev->shader_available_bitmap || ++ kbdev->tiler_available_bitmap); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++#endif /* !PLATFORM_POWER_DOWN_ONLY */ ++ ++ /* Consume any change-state events */ ++ kbase_timeline_pm_check_handle_event(kbdev, ++ KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED); ++ ++ /* Disable interrupts and turn the clock off */ ++ if (!kbase_pm_clock_off(kbdev, backend->poweroff_is_suspend)) { ++ /* ++ * Page/bus faults are pending, must drop locks to ++ * process. Interrupts are disabled so no more faults ++ * should be generated at this point. ++ */ ++ mutex_unlock(&kbdev->pm.lock); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ kbase_flush_mmu_wqs(kbdev); ++ mutex_lock(&js_devdata->runpool_mutex); ++ mutex_lock(&kbdev->pm.lock); ++ ++ /* Turn off clock now that fault have been handled. We ++ * dropped locks so poweron_required may have changed - ++ * power back on if this is the case.*/ ++ if (backend->poweron_required) ++ kbase_pm_clock_on(kbdev, false); ++ else ++ WARN_ON(!kbase_pm_clock_off(kbdev, ++ backend->poweroff_is_suspend)); ++ } ++ } ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ backend->poweroff_wait_in_progress = false; ++ if (backend->poweron_required) { ++ backend->poweron_required = false; ++ kbase_pm_update_cores_state_nolock(kbdev); ++ kbase_backend_slot_update(kbdev); ++ } ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ mutex_unlock(&kbdev->pm.lock); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ ++ wake_up(&kbdev->pm.backend.poweroff_wait); ++} ++ ++void kbase_pm_do_poweroff(struct kbase_device *kbdev, bool is_suspend) ++{ ++ unsigned long flags; ++ ++ lockdep_assert_held(&kbdev->pm.lock); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ if (!kbdev->pm.backend.poweroff_wait_in_progress) { ++ /* Force all cores off */ ++ kbdev->pm.backend.desired_shader_state = 0; ++ kbdev->pm.backend.desired_tiler_state = 0; ++ ++ /* Force all cores to be unavailable, in the situation where ++ * transitions are in progress for some cores but not others, ++ * and kbase_pm_check_transitions_nolock can not immediately ++ * power off the cores */ ++ kbdev->shader_available_bitmap = 0; ++ kbdev->tiler_available_bitmap = 0; ++ kbdev->l2_available_bitmap = 0; ++ ++ kbdev->pm.backend.poweroff_wait_in_progress = true; ++ kbdev->pm.backend.poweroff_is_suspend = is_suspend; ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ /*Kick off wq here. Callers will have to wait*/ ++ queue_work(kbdev->pm.backend.gpu_poweroff_wait_wq, ++ &kbdev->pm.backend.gpu_poweroff_wait_work); ++ } else { ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ } ++} ++ ++static bool is_poweroff_in_progress(struct kbase_device *kbdev) ++{ ++ bool ret; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ret = (kbdev->pm.backend.poweroff_wait_in_progress == false); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ return ret; ++} ++ ++void kbase_pm_wait_for_poweroff_complete(struct kbase_device *kbdev) ++{ ++ wait_event_killable(kbdev->pm.backend.poweroff_wait, ++ is_poweroff_in_progress(kbdev)); ++} ++ ++int kbase_hwaccess_pm_powerup(struct kbase_device *kbdev, ++ unsigned int flags) ++{ ++ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; ++ unsigned long irq_flags; ++ int ret; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ mutex_lock(&js_devdata->runpool_mutex); ++ mutex_lock(&kbdev->pm.lock); ++ ++ /* A suspend won't happen during startup/insmod */ ++ KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev)); ++ ++ /* Power up the GPU, don't enable IRQs as we are not ready to receive ++ * them. */ ++ ret = kbase_pm_init_hw(kbdev, flags); ++ if (ret) { ++ mutex_unlock(&kbdev->pm.lock); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ return ret; ++ } ++ ++ kbasep_pm_init_core_use_bitmaps(kbdev); ++ ++ kbdev->pm.debug_core_mask_all = kbdev->pm.debug_core_mask[0] = ++ kbdev->pm.debug_core_mask[1] = ++ kbdev->pm.debug_core_mask[2] = ++ kbdev->gpu_props.props.raw_props.shader_present; ++ ++ /* Pretend the GPU is active to prevent a power policy turning the GPU ++ * cores off */ ++ kbdev->pm.active_count = 1; ++ ++ spin_lock_irqsave(&kbdev->pm.backend.gpu_cycle_counter_requests_lock, ++ irq_flags); ++ /* Ensure cycle counter is off */ ++ kbdev->pm.backend.gpu_cycle_counter_requests = 0; ++ spin_unlock_irqrestore( ++ &kbdev->pm.backend.gpu_cycle_counter_requests_lock, ++ irq_flags); ++ ++ /* We are ready to receive IRQ's now as power policy is set up, so ++ * enable them now. */ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++ spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, irq_flags); ++ kbdev->pm.backend.driver_ready_for_irqs = true; ++ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, irq_flags); ++#endif ++ kbase_pm_enable_interrupts(kbdev); ++ ++ /* Turn on the GPU and any cores needed by the policy */ ++ kbase_pm_do_poweron(kbdev, false); ++ mutex_unlock(&kbdev->pm.lock); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ ++ /* Idle the GPU and/or cores, if the policy wants it to */ ++ kbase_pm_context_idle(kbdev); ++ ++ return 0; ++} ++ ++void kbase_hwaccess_pm_halt(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ mutex_lock(&kbdev->pm.lock); ++ kbase_pm_cancel_deferred_poweroff(kbdev); ++ kbase_pm_do_poweroff(kbdev, false); ++ mutex_unlock(&kbdev->pm.lock); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_hwaccess_pm_halt); ++ ++void kbase_hwaccess_pm_term(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kbdev->pm.active_count == 0); ++ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests == 0); ++ ++ /* Free any resources the policy allocated */ ++ kbase_pm_policy_term(kbdev); ++ kbase_pm_ca_term(kbdev); ++ ++ /* Shut down the metrics subsystem */ ++ kbasep_pm_metrics_term(kbdev); ++ ++ destroy_workqueue(kbdev->pm.backend.gpu_poweroff_wait_wq); ++} ++ ++void kbase_pm_power_changed(struct kbase_device *kbdev) ++{ ++ bool cores_are_available; ++ unsigned long flags; ++ ++ KBASE_TIMELINE_PM_CHECKTRANS(kbdev, ++ SW_FLOW_PM_CHECKTRANS_GPU_INTERRUPT_START); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ cores_are_available = kbase_pm_check_transitions_nolock(kbdev); ++ KBASE_TIMELINE_PM_CHECKTRANS(kbdev, ++ SW_FLOW_PM_CHECKTRANS_GPU_INTERRUPT_END); ++ ++ if (cores_are_available) { ++ /* Log timelining information that a change in state has ++ * completed */ ++ kbase_timeline_pm_handle_event(kbdev, ++ KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED); ++ ++ kbase_backend_slot_update(kbdev); ++ } ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++} ++ ++void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev, ++ u64 new_core_mask_js0, u64 new_core_mask_js1, ++ u64 new_core_mask_js2) ++{ ++ kbdev->pm.debug_core_mask[0] = new_core_mask_js0; ++ kbdev->pm.debug_core_mask[1] = new_core_mask_js1; ++ kbdev->pm.debug_core_mask[2] = new_core_mask_js2; ++ kbdev->pm.debug_core_mask_all = new_core_mask_js0 | new_core_mask_js1 | ++ new_core_mask_js2; ++ ++ kbase_pm_update_cores_state_nolock(kbdev); ++} ++ ++void kbase_hwaccess_pm_gpu_active(struct kbase_device *kbdev) ++{ ++ kbase_pm_update_active(kbdev); ++} ++ ++void kbase_hwaccess_pm_gpu_idle(struct kbase_device *kbdev) ++{ ++ kbase_pm_update_active(kbdev); ++} ++ ++void kbase_hwaccess_pm_suspend(struct kbase_device *kbdev) ++{ ++ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; ++ ++ /* Force power off the GPU and all cores (regardless of policy), only ++ * after the PM active count reaches zero (otherwise, we risk turning it ++ * off prematurely) */ ++ mutex_lock(&js_devdata->runpool_mutex); ++ mutex_lock(&kbdev->pm.lock); ++ ++ kbase_pm_cancel_deferred_poweroff(kbdev); ++ kbase_pm_do_poweroff(kbdev, true); ++ ++ kbase_backend_timer_suspend(kbdev); ++ ++ mutex_unlock(&kbdev->pm.lock); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ ++ kbase_pm_wait_for_poweroff_complete(kbdev); ++} ++ ++void kbase_hwaccess_pm_resume(struct kbase_device *kbdev) ++{ ++ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; ++ ++ mutex_lock(&js_devdata->runpool_mutex); ++ mutex_lock(&kbdev->pm.lock); ++ ++ kbdev->pm.suspending = false; ++ kbase_pm_do_poweron(kbdev, true); ++ ++ kbase_backend_timer_resume(kbdev); ++ ++ mutex_unlock(&kbdev->pm.lock); ++ mutex_unlock(&js_devdata->runpool_mutex); ++} +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_ca.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_ca.c +new file mode 100755 +index 000000000000..c17db8be8877 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_ca.c +@@ -0,0 +1,182 @@ ++/* ++ * ++ * (C) COPYRIGHT 2013-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* ++ * Base kernel core availability APIs ++ */ ++ ++#include ++#include ++#include ++ ++static const struct kbase_pm_ca_policy *const policy_list[] = { ++ &kbase_pm_ca_fixed_policy_ops, ++#ifdef CONFIG_MALI_BIFROST_DEVFREQ ++ &kbase_pm_ca_devfreq_policy_ops, ++#endif ++#if !MALI_CUSTOMER_RELEASE ++ &kbase_pm_ca_random_policy_ops ++#endif ++}; ++ ++/** ++ * POLICY_COUNT - The number of policies available in the system. ++ * ++ * This is derived from the number of functions listed in policy_list. ++ */ ++#define POLICY_COUNT (sizeof(policy_list)/sizeof(*policy_list)) ++ ++int kbase_pm_ca_init(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ kbdev->pm.backend.ca_current_policy = policy_list[0]; ++ ++ kbdev->pm.backend.ca_current_policy->init(kbdev); ++ ++ return 0; ++} ++ ++void kbase_pm_ca_term(struct kbase_device *kbdev) ++{ ++ kbdev->pm.backend.ca_current_policy->term(kbdev); ++} ++ ++int kbase_pm_ca_list_policies(const struct kbase_pm_ca_policy * const **list) ++{ ++ if (!list) ++ return POLICY_COUNT; ++ ++ *list = policy_list; ++ ++ return POLICY_COUNT; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_ca_list_policies); ++ ++const struct kbase_pm_ca_policy ++*kbase_pm_ca_get_policy(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ return kbdev->pm.backend.ca_current_policy; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_ca_get_policy); ++ ++void kbase_pm_ca_set_policy(struct kbase_device *kbdev, ++ const struct kbase_pm_ca_policy *new_policy) ++{ ++ const struct kbase_pm_ca_policy *old_policy; ++ unsigned long flags; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(new_policy != NULL); ++ ++ KBASE_TRACE_ADD(kbdev, PM_CA_SET_POLICY, NULL, NULL, 0u, ++ new_policy->id); ++ ++ /* During a policy change we pretend the GPU is active */ ++ /* A suspend won't happen here, because we're in a syscall from a ++ * userspace thread */ ++ kbase_pm_context_active(kbdev); ++ ++ mutex_lock(&kbdev->pm.lock); ++ ++ /* Remove the policy to prevent IRQ handlers from working on it */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ old_policy = kbdev->pm.backend.ca_current_policy; ++ kbdev->pm.backend.ca_current_policy = NULL; ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ if (old_policy->term) ++ old_policy->term(kbdev); ++ ++ if (new_policy->init) ++ new_policy->init(kbdev); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbdev->pm.backend.ca_current_policy = new_policy; ++ ++ /* If any core power state changes were previously attempted, but ++ * couldn't be made because the policy was changing (current_policy was ++ * NULL), then re-try them here. */ ++ kbase_pm_update_cores_state_nolock(kbdev); ++ ++ kbdev->pm.backend.ca_current_policy->update_core_status(kbdev, ++ kbdev->shader_ready_bitmap, ++ kbdev->shader_transitioning_bitmap); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ mutex_unlock(&kbdev->pm.lock); ++ ++ /* Now the policy change is finished, we release our fake context active ++ * reference */ ++ kbase_pm_context_idle(kbdev); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_ca_set_policy); ++ ++u64 kbase_pm_ca_get_core_mask(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ /* All cores must be enabled when instrumentation is in use */ ++ if (kbdev->pm.backend.instr_enabled) ++ return kbdev->gpu_props.props.raw_props.shader_present & ++ kbdev->pm.debug_core_mask_all; ++ ++ if (kbdev->pm.backend.ca_current_policy == NULL) ++ return kbdev->gpu_props.props.raw_props.shader_present & ++ kbdev->pm.debug_core_mask_all; ++ ++ return kbdev->pm.backend.ca_current_policy->get_core_mask(kbdev) & ++ kbdev->pm.debug_core_mask_all; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_ca_get_core_mask); ++ ++void kbase_pm_ca_update_core_status(struct kbase_device *kbdev, u64 cores_ready, ++ u64 cores_transitioning) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (kbdev->pm.backend.ca_current_policy != NULL) ++ kbdev->pm.backend.ca_current_policy->update_core_status(kbdev, ++ cores_ready, ++ cores_transitioning); ++} ++ ++void kbase_pm_ca_instr_enable(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbdev->pm.backend.instr_enabled = true; ++ ++ kbase_pm_update_cores_state_nolock(kbdev); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++} ++ ++void kbase_pm_ca_instr_disable(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ kbdev->pm.backend.instr_enabled = false; ++ ++ kbase_pm_update_cores_state_nolock(kbdev); ++} +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_ca.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_ca.h +new file mode 100755 +index 000000000000..ee9e751f2d79 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_ca.h +@@ -0,0 +1,92 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* ++ * Base kernel core availability APIs ++ */ ++ ++#ifndef _KBASE_PM_CA_H_ ++#define _KBASE_PM_CA_H_ ++ ++/** ++ * kbase_pm_ca_init - Initialize core availability framework ++ * ++ * Must be called before calling any other core availability function ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * Return: 0 if the core availability framework was successfully initialized, ++ * -errno otherwise ++ */ ++int kbase_pm_ca_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_ca_term - Terminate core availability framework ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_ca_term(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_ca_get_core_mask - Get currently available shaders core mask ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * Returns a mask of the currently available shader cores. ++ * Calls into the core availability policy ++ * ++ * Return: The bit mask of available cores ++ */ ++u64 kbase_pm_ca_get_core_mask(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_ca_update_core_status - Update core status ++ * ++ * @kbdev: The kbase device structure for the device (must be ++ * a valid pointer) ++ * @cores_ready: The bit mask of cores ready for job submission ++ * @cores_transitioning: The bit mask of cores that are transitioning power ++ * state ++ * ++ * Update core availability policy with current core power status ++ * ++ * Calls into the core availability policy ++ */ ++void kbase_pm_ca_update_core_status(struct kbase_device *kbdev, u64 cores_ready, ++ u64 cores_transitioning); ++ ++/** ++ * kbase_pm_ca_instr_enable - Enable override for instrumentation ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * This overrides the output of the core availability policy, ensuring that all ++ * cores are available ++ */ ++void kbase_pm_ca_instr_enable(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_ca_instr_disable - Disable override for instrumentation ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * This disables any previously enabled override, and resumes normal policy ++ * functionality ++ */ ++void kbase_pm_ca_instr_disable(struct kbase_device *kbdev); ++ ++#endif /* _KBASE_PM_CA_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_ca_devfreq.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_ca_devfreq.c +new file mode 100755 +index 000000000000..66bf660cffb6 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_ca_devfreq.c +@@ -0,0 +1,129 @@ ++/* ++ * ++ * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* ++ * A core availability policy implementing core mask selection from devfreq OPPs ++ * ++ */ ++ ++#include ++#include ++#include ++#include ++ ++void kbase_devfreq_set_core_mask(struct kbase_device *kbdev, u64 core_mask) ++{ ++ struct kbasep_pm_ca_policy_devfreq *data = ++ &kbdev->pm.backend.ca_policy_data.devfreq; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ data->cores_desired = core_mask; ++ ++ /* Disable any cores that are now unwanted */ ++ data->cores_enabled &= data->cores_desired; ++ ++ kbdev->pm.backend.ca_in_transition = true; ++ ++ /* If there are no cores to be powered off then power on desired cores ++ */ ++ if (!(data->cores_used & ~data->cores_desired)) { ++ data->cores_enabled = data->cores_desired; ++ kbdev->pm.backend.ca_in_transition = false; ++ } ++ ++ kbase_pm_update_cores_state_nolock(kbdev); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ dev_dbg(kbdev->dev, "Devfreq policy : new core mask=%llX %llX\n", ++ data->cores_desired, data->cores_enabled); ++} ++ ++static void devfreq_init(struct kbase_device *kbdev) ++{ ++ struct kbasep_pm_ca_policy_devfreq *data = ++ &kbdev->pm.backend.ca_policy_data.devfreq; ++ ++ if (kbdev->current_core_mask) { ++ data->cores_enabled = kbdev->current_core_mask; ++ data->cores_desired = kbdev->current_core_mask; ++ } else { ++ data->cores_enabled = ++ kbdev->gpu_props.props.raw_props.shader_present; ++ data->cores_desired = ++ kbdev->gpu_props.props.raw_props.shader_present; ++ } ++ data->cores_used = 0; ++ kbdev->pm.backend.ca_in_transition = false; ++} ++ ++static void devfreq_term(struct kbase_device *kbdev) ++{ ++} ++ ++static u64 devfreq_get_core_mask(struct kbase_device *kbdev) ++{ ++ return kbdev->pm.backend.ca_policy_data.devfreq.cores_enabled; ++} ++ ++static void devfreq_update_core_status(struct kbase_device *kbdev, ++ u64 cores_ready, ++ u64 cores_transitioning) ++{ ++ struct kbasep_pm_ca_policy_devfreq *data = ++ &kbdev->pm.backend.ca_policy_data.devfreq; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ data->cores_used = cores_ready | cores_transitioning; ++ ++ /* If in desired state then clear transition flag */ ++ if (data->cores_enabled == data->cores_desired) ++ kbdev->pm.backend.ca_in_transition = false; ++ ++ /* If all undesired cores are now off then power on desired cores. ++ * The direct comparison against cores_enabled limits potential ++ * recursion to one level */ ++ if (!(data->cores_used & ~data->cores_desired) && ++ data->cores_enabled != data->cores_desired) { ++ data->cores_enabled = data->cores_desired; ++ ++ kbase_pm_update_cores_state_nolock(kbdev); ++ ++ kbdev->pm.backend.ca_in_transition = false; ++ } ++} ++ ++/* ++ * The struct kbase_pm_ca_policy structure for the devfreq core availability ++ * policy. ++ * ++ * This is the static structure that defines the devfreq core availability power ++ * policy's callback and name. ++ */ ++const struct kbase_pm_ca_policy kbase_pm_ca_devfreq_policy_ops = { ++ "devfreq", /* name */ ++ devfreq_init, /* init */ ++ devfreq_term, /* term */ ++ devfreq_get_core_mask, /* get_core_mask */ ++ devfreq_update_core_status, /* update_core_status */ ++ 0u, /* flags */ ++ KBASE_PM_CA_POLICY_ID_DEVFREQ, /* id */ ++}; ++ +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_ca_devfreq.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_ca_devfreq.h +new file mode 100755 +index 000000000000..7ab3cd4d8460 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_ca_devfreq.h +@@ -0,0 +1,55 @@ ++/* ++ * ++ * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* ++ * A core availability policy for use with devfreq, where core masks are ++ * associated with OPPs. ++ */ ++ ++#ifndef MALI_KBASE_PM_CA_DEVFREQ_H ++#define MALI_KBASE_PM_CA_DEVFREQ_H ++ ++/** ++ * struct kbasep_pm_ca_policy_devfreq - Private structure for devfreq ca policy ++ * ++ * This contains data that is private to the devfreq core availability ++ * policy. ++ * ++ * @cores_desired: Cores that the policy wants to be available ++ * @cores_enabled: Cores that the policy is currently returning as available ++ * @cores_used: Cores currently powered or transitioning ++ */ ++struct kbasep_pm_ca_policy_devfreq { ++ u64 cores_desired; ++ u64 cores_enabled; ++ u64 cores_used; ++}; ++ ++extern const struct kbase_pm_ca_policy kbase_pm_ca_devfreq_policy_ops; ++ ++/** ++ * kbase_devfreq_set_core_mask - Set core mask for policy to use ++ * @kbdev: Device pointer ++ * @core_mask: New core mask ++ * ++ * The new core mask will have immediate effect if the GPU is powered, or will ++ * take effect when it is next powered on. ++ */ ++void kbase_devfreq_set_core_mask(struct kbase_device *kbdev, u64 core_mask); ++ ++#endif /* MALI_KBASE_PM_CA_DEVFREQ_H */ ++ +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_ca_fixed.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_ca_fixed.c +new file mode 100755 +index 000000000000..864612d31f9b +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_ca_fixed.c +@@ -0,0 +1,65 @@ ++/* ++ * ++ * (C) COPYRIGHT 2013-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* ++ * A power policy implementing fixed core availability ++ */ ++ ++#include ++#include ++ ++static void fixed_init(struct kbase_device *kbdev) ++{ ++ kbdev->pm.backend.ca_in_transition = false; ++} ++ ++static void fixed_term(struct kbase_device *kbdev) ++{ ++ CSTD_UNUSED(kbdev); ++} ++ ++static u64 fixed_get_core_mask(struct kbase_device *kbdev) ++{ ++ return kbdev->gpu_props.props.raw_props.shader_present; ++} ++ ++static void fixed_update_core_status(struct kbase_device *kbdev, ++ u64 cores_ready, ++ u64 cores_transitioning) ++{ ++ CSTD_UNUSED(kbdev); ++ CSTD_UNUSED(cores_ready); ++ CSTD_UNUSED(cores_transitioning); ++} ++ ++/* ++ * The struct kbase_pm_policy structure for the fixed power policy. ++ * ++ * This is the static structure that defines the fixed power policy's callback ++ * and name. ++ */ ++const struct kbase_pm_ca_policy kbase_pm_ca_fixed_policy_ops = { ++ "fixed", /* name */ ++ fixed_init, /* init */ ++ fixed_term, /* term */ ++ fixed_get_core_mask, /* get_core_mask */ ++ fixed_update_core_status, /* update_core_status */ ++ 0u, /* flags */ ++ KBASE_PM_CA_POLICY_ID_FIXED, /* id */ ++}; ++ ++KBASE_EXPORT_TEST_API(kbase_pm_ca_fixed_policy_ops); +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_ca_fixed.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_ca_fixed.h +new file mode 100755 +index 000000000000..a763155cb703 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_ca_fixed.h +@@ -0,0 +1,40 @@ ++/* ++ * ++ * (C) COPYRIGHT 2013-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* ++ * A power policy implementing fixed core availability ++ */ ++ ++#ifndef MALI_KBASE_PM_CA_FIXED_H ++#define MALI_KBASE_PM_CA_FIXED_H ++ ++/** ++ * struct kbasep_pm_ca_policy_fixed - Private structure for policy instance data ++ * ++ * @dummy: Dummy member - no state is needed ++ * ++ * This contains data that is private to the particular power policy that is ++ * active. ++ */ ++struct kbasep_pm_ca_policy_fixed { ++ int dummy; ++}; ++ ++extern const struct kbase_pm_ca_policy kbase_pm_ca_fixed_policy_ops; ++ ++#endif /* MALI_KBASE_PM_CA_FIXED_H */ ++ +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_coarse_demand.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_coarse_demand.c +new file mode 100755 +index 000000000000..f891fa225a89 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_coarse_demand.c +@@ -0,0 +1,70 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * "Coarse Demand" power management policy ++ */ ++ ++#include ++#include ++ ++static u64 coarse_demand_get_core_mask(struct kbase_device *kbdev) ++{ ++ if (kbdev->pm.active_count == 0) ++ return 0; ++ ++ return kbdev->gpu_props.props.raw_props.shader_present; ++} ++ ++static bool coarse_demand_get_core_active(struct kbase_device *kbdev) ++{ ++ if (0 == kbdev->pm.active_count && !(kbdev->shader_needed_bitmap | ++ kbdev->shader_inuse_bitmap) && !kbdev->tiler_needed_cnt ++ && !kbdev->tiler_inuse_cnt) ++ return false; ++ ++ return true; ++} ++ ++static void coarse_demand_init(struct kbase_device *kbdev) ++{ ++ CSTD_UNUSED(kbdev); ++} ++ ++static void coarse_demand_term(struct kbase_device *kbdev) ++{ ++ CSTD_UNUSED(kbdev); ++} ++ ++/* The struct kbase_pm_policy structure for the demand power policy. ++ * ++ * This is the static structure that defines the demand power policy's callback ++ * and name. ++ */ ++const struct kbase_pm_policy kbase_pm_coarse_demand_policy_ops = { ++ "coarse_demand", /* name */ ++ coarse_demand_init, /* init */ ++ coarse_demand_term, /* term */ ++ coarse_demand_get_core_mask, /* get_core_mask */ ++ coarse_demand_get_core_active, /* get_core_active */ ++ 0u, /* flags */ ++ KBASE_PM_POLICY_ID_COARSE_DEMAND, /* id */ ++}; ++ ++KBASE_EXPORT_TEST_API(kbase_pm_coarse_demand_policy_ops); +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_coarse_demand.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_coarse_demand.h +new file mode 100755 +index 000000000000..749d305eee9a +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_coarse_demand.h +@@ -0,0 +1,64 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * "Coarse Demand" power management policy ++ */ ++ ++#ifndef MALI_KBASE_PM_COARSE_DEMAND_H ++#define MALI_KBASE_PM_COARSE_DEMAND_H ++ ++/** ++ * DOC: ++ * The "Coarse" demand power management policy has the following ++ * characteristics: ++ * - When KBase indicates that the GPU will be powered up, but we don't yet ++ * know which Job Chains are to be run: ++ * - All Shader Cores are powered up, regardless of whether or not they will ++ * be needed later. ++ * - When KBase indicates that a set of Shader Cores are needed to submit the ++ * currently queued Job Chains: ++ * - All Shader Cores are kept powered, regardless of whether or not they will ++ * be needed ++ * - When KBase indicates that the GPU need not be powered: ++ * - The Shader Cores are powered off, and the GPU itself is powered off too. ++ * ++ * @note: ++ * - KBase indicates the GPU will be powered up when it has a User Process that ++ * has just started to submit Job Chains. ++ * - KBase indicates the GPU need not be powered when all the Job Chains from ++ * User Processes have finished, and it is waiting for a User Process to ++ * submit some more Job Chains. ++ */ ++ ++/** ++ * struct kbasep_pm_policy_coarse_demand - Private structure for coarse demand ++ * policy ++ * ++ * This contains data that is private to the coarse demand power policy. ++ * ++ * @dummy: Dummy member - no state needed ++ */ ++struct kbasep_pm_policy_coarse_demand { ++ int dummy; ++}; ++ ++extern const struct kbase_pm_policy kbase_pm_coarse_demand_policy_ops; ++ ++#endif /* MALI_KBASE_PM_COARSE_DEMAND_H */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_defs.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_defs.h +new file mode 100755 +index 000000000000..564fbda1116a +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_defs.h +@@ -0,0 +1,519 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* ++ * Backend-specific Power Manager definitions ++ */ ++ ++#ifndef _KBASE_PM_HWACCESS_DEFS_H_ ++#define _KBASE_PM_HWACCESS_DEFS_H_ ++ ++#include "mali_kbase_pm_ca_fixed.h" ++#include "mali_kbase_pm_ca_devfreq.h" ++#if !MALI_CUSTOMER_RELEASE ++#include "mali_kbase_pm_ca_random.h" ++#endif ++ ++#include "mali_kbase_pm_always_on.h" ++#include "mali_kbase_pm_coarse_demand.h" ++#include "mali_kbase_pm_demand.h" ++#if !MALI_CUSTOMER_RELEASE ++#include "mali_kbase_pm_demand_always_powered.h" ++#include "mali_kbase_pm_fast_start.h" ++#endif ++ ++/* Forward definition - see mali_kbase.h */ ++struct kbase_device; ++struct kbase_jd_atom; ++ ++/** ++ * enum kbase_pm_core_type - The types of core in a GPU. ++ * ++ * These enumerated values are used in calls to ++ * - kbase_pm_get_present_cores() ++ * - kbase_pm_get_active_cores() ++ * - kbase_pm_get_trans_cores() ++ * - kbase_pm_get_ready_cores(). ++ * ++ * They specify which type of core should be acted on. These values are set in ++ * a manner that allows core_type_to_reg() function to be simpler and more ++ * efficient. ++ * ++ * @KBASE_PM_CORE_L2: The L2 cache ++ * @KBASE_PM_CORE_SHADER: Shader cores ++ * @KBASE_PM_CORE_TILER: Tiler cores ++ * @KBASE_PM_CORE_STACK: Core stacks ++ */ ++enum kbase_pm_core_type { ++ KBASE_PM_CORE_L2 = L2_PRESENT_LO, ++ KBASE_PM_CORE_SHADER = SHADER_PRESENT_LO, ++ KBASE_PM_CORE_TILER = TILER_PRESENT_LO, ++ KBASE_PM_CORE_STACK = STACK_PRESENT_LO ++}; ++ ++/** ++ * struct kbasep_pm_metrics_data - Metrics data collected for use by the power ++ * management framework. ++ * ++ * @time_period_start: time at which busy/idle measurements started ++ * @time_busy: number of ns the GPU was busy executing jobs since the ++ * @time_period_start timestamp. ++ * @time_idle: number of ns since time_period_start the GPU was not executing ++ * jobs since the @time_period_start timestamp. ++ * @prev_busy: busy time in ns of previous time period. ++ * Updated when metrics are reset. ++ * @prev_idle: idle time in ns of previous time period ++ * Updated when metrics are reset. ++ * @gpu_active: true when the GPU is executing jobs. false when ++ * not. Updated when the job scheduler informs us a job in submitted ++ * or removed from a GPU slot. ++ * @busy_cl: number of ns the GPU was busy executing CL jobs. Note that ++ * if two CL jobs were active for 400ns, this value would be updated ++ * with 800. ++ * @busy_gl: number of ns the GPU was busy executing GL jobs. Note that ++ * if two GL jobs were active for 400ns, this value would be updated ++ * with 800. ++ * @active_cl_ctx: number of CL jobs active on the GPU. Array is per-device. ++ * @active_gl_ctx: number of GL jobs active on the GPU. Array is per-slot. As ++ * GL jobs never run on slot 2 this slot is not recorded. ++ * @lock: spinlock protecting the kbasep_pm_metrics_data structure ++ * @timer: timer to regularly make DVFS decisions based on the power ++ * management metrics. ++ * @timer_active: boolean indicating @timer is running ++ * @platform_data: pointer to data controlled by platform specific code ++ * @kbdev: pointer to kbase device for which metrics are collected ++ * ++ */ ++struct kbasep_pm_metrics_data { ++ ktime_t time_period_start; ++ u32 time_busy; ++ u32 time_idle; ++ u32 prev_busy; ++ u32 prev_idle; ++ bool gpu_active; ++ u32 busy_cl[2]; ++ u32 busy_gl; ++ u32 active_cl_ctx[2]; ++ u32 active_gl_ctx[2]; /* GL jobs can only run on 2 of the 3 job slots */ ++ spinlock_t lock; ++ ++#ifdef CONFIG_MALI_BIFROST_DVFS ++ struct hrtimer timer; ++ bool timer_active; ++#endif ++ ++ void *platform_data; ++ struct kbase_device *kbdev; ++}; ++ ++union kbase_pm_policy_data { ++ struct kbasep_pm_policy_always_on always_on; ++ struct kbasep_pm_policy_coarse_demand coarse_demand; ++ struct kbasep_pm_policy_demand demand; ++#if !MALI_CUSTOMER_RELEASE ++ struct kbasep_pm_policy_demand_always_powered demand_always_powered; ++ struct kbasep_pm_policy_fast_start fast_start; ++#endif ++}; ++ ++union kbase_pm_ca_policy_data { ++ struct kbasep_pm_ca_policy_fixed fixed; ++ struct kbasep_pm_ca_policy_devfreq devfreq; ++#if !MALI_CUSTOMER_RELEASE ++ struct kbasep_pm_ca_policy_random random; ++#endif ++}; ++ ++/** ++ * struct kbase_pm_backend_data - Data stored per device for power management. ++ * ++ * This structure contains data for the power management framework. There is one ++ * instance of this structure per device in the system. ++ * ++ * @ca_current_policy: The policy that is currently actively controlling core ++ * availability. ++ * @pm_current_policy: The policy that is currently actively controlling the ++ * power state. ++ * @ca_policy_data: Private data for current CA policy ++ * @pm_policy_data: Private data for current PM policy ++ * @ca_in_transition: Flag indicating when core availability policy is ++ * transitioning cores. The core availability policy must ++ * set this when a change in core availability is occurring. ++ * power_change_lock must be held when accessing this. ++ * @reset_done: Flag when a reset is complete ++ * @reset_done_wait: Wait queue to wait for changes to @reset_done ++ * @l2_powered_wait: Wait queue for whether the l2 cache has been powered as ++ * requested ++ * @l2_powered: State indicating whether all the l2 caches are powered. ++ * Non-zero indicates they're *all* powered ++ * Zero indicates that some (or all) are not powered ++ * @gpu_cycle_counter_requests: The reference count of active gpu cycle counter ++ * users ++ * @gpu_cycle_counter_requests_lock: Lock to protect @gpu_cycle_counter_requests ++ * @desired_shader_state: A bit mask identifying the shader cores that the ++ * power policy would like to be on. The current state ++ * of the cores may be different, but there should be ++ * transitions in progress that will eventually achieve ++ * this state (assuming that the policy doesn't change ++ * its mind in the mean time). ++ * @powering_on_shader_state: A bit mask indicating which shader cores are ++ * currently in a power-on transition ++ * @desired_tiler_state: A bit mask identifying the tiler cores that the power ++ * policy would like to be on. See @desired_shader_state ++ * @powering_on_tiler_state: A bit mask indicating which tiler core are ++ * currently in a power-on transition ++ * @powering_on_l2_state: A bit mask indicating which l2-caches are currently ++ * in a power-on transition ++ * @powering_on_stack_state: A bit mask indicating which core stacks are ++ * currently in a power-on transition ++ * @gpu_in_desired_state: This flag is set if the GPU is powered as requested ++ * by the desired_xxx_state variables ++ * @gpu_in_desired_state_wait: Wait queue set when @gpu_in_desired_state != 0 ++ * @gpu_powered: Set to true when the GPU is powered and register ++ * accesses are possible, false otherwise ++ * @instr_enabled: Set to true when instrumentation is enabled, ++ * false otherwise ++ * @cg1_disabled: Set if the policy wants to keep the second core group ++ * powered off ++ * @driver_ready_for_irqs: Debug state indicating whether sufficient ++ * initialization of the driver has occurred to handle ++ * IRQs ++ * @gpu_powered_lock: Spinlock that must be held when writing @gpu_powered or ++ * accessing @driver_ready_for_irqs ++ * @metrics: Structure to hold metrics for the GPU ++ * @gpu_poweroff_pending: number of poweroff timer ticks until the GPU is ++ * powered off ++ * @shader_poweroff_pending_time: number of poweroff timer ticks until shaders ++ * and/or timers are powered off ++ * @gpu_poweroff_timer: Timer for powering off GPU ++ * @gpu_poweroff_wq: Workqueue to power off GPU on when timer fires ++ * @gpu_poweroff_work: Workitem used on @gpu_poweroff_wq ++ * @shader_poweroff_pending: Bit mask of shaders to be powered off on next ++ * timer callback ++ * @tiler_poweroff_pending: Bit mask of tilers to be powered off on next timer ++ * callback ++ * @poweroff_timer_needed: true if the poweroff timer is currently required, ++ * false otherwise ++ * @poweroff_timer_running: true if the poweroff timer is currently running, ++ * false otherwise ++ * power_change_lock should be held when accessing, ++ * unless there is no way the timer can be running (eg ++ * hrtimer_cancel() was called immediately before) ++ * @poweroff_wait_in_progress: true if a wait for GPU power off is in progress. ++ * hwaccess_lock must be held when accessing ++ * @poweron_required: true if a GPU power on is required. Should only be set ++ * when poweroff_wait_in_progress is true, and therefore the ++ * GPU can not immediately be powered on. pm.lock must be ++ * held when accessing ++ * @poweroff_is_suspend: true if the GPU is being powered off due to a suspend ++ * request. pm.lock must be held when accessing ++ * @gpu_poweroff_wait_wq: workqueue for waiting for GPU to power off ++ * @gpu_poweroff_wait_work: work item for use with @gpu_poweroff_wait_wq ++ * @poweroff_wait: waitqueue for waiting for @gpu_poweroff_wait_work to complete ++ * @callback_power_on: Callback when the GPU needs to be turned on. See ++ * &struct kbase_pm_callback_conf ++ * @callback_power_off: Callback when the GPU may be turned off. See ++ * &struct kbase_pm_callback_conf ++ * @callback_power_suspend: Callback when a suspend occurs and the GPU needs to ++ * be turned off. See &struct kbase_pm_callback_conf ++ * @callback_power_resume: Callback when a resume occurs and the GPU needs to ++ * be turned on. See &struct kbase_pm_callback_conf ++ * @callback_power_runtime_on: Callback when the GPU needs to be turned on. See ++ * &struct kbase_pm_callback_conf ++ * @callback_power_runtime_off: Callback when the GPU may be turned off. See ++ * &struct kbase_pm_callback_conf ++ * @callback_power_runtime_idle: Optional callback when the GPU may be idle. See ++ * &struct kbase_pm_callback_conf ++ * ++ * Note: ++ * During an IRQ, @ca_current_policy or @pm_current_policy can be NULL when the ++ * policy is being changed with kbase_pm_ca_set_policy() or ++ * kbase_pm_set_policy(). The change is protected under ++ * kbase_device.pm.power_change_lock. Direct access to this ++ * from IRQ context must therefore check for NULL. If NULL, then ++ * kbase_pm_ca_set_policy() or kbase_pm_set_policy() will re-issue the policy ++ * functions that would have been done under IRQ. ++ */ ++struct kbase_pm_backend_data { ++ const struct kbase_pm_ca_policy *ca_current_policy; ++ const struct kbase_pm_policy *pm_current_policy; ++ union kbase_pm_ca_policy_data ca_policy_data; ++ union kbase_pm_policy_data pm_policy_data; ++ bool ca_in_transition; ++ bool reset_done; ++ wait_queue_head_t reset_done_wait; ++ wait_queue_head_t l2_powered_wait; ++ int l2_powered; ++ int gpu_cycle_counter_requests; ++ spinlock_t gpu_cycle_counter_requests_lock; ++ ++ u64 desired_shader_state; ++ u64 powering_on_shader_state; ++ u64 desired_tiler_state; ++ u64 powering_on_tiler_state; ++ u64 powering_on_l2_state; ++#ifdef CONFIG_MALI_CORESTACK ++ u64 powering_on_stack_state; ++#endif /* CONFIG_MALI_CORESTACK */ ++ ++ bool gpu_in_desired_state; ++ wait_queue_head_t gpu_in_desired_state_wait; ++ ++ bool gpu_powered; ++ ++ bool instr_enabled; ++ ++ bool cg1_disabled; ++ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++ bool driver_ready_for_irqs; ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++ ++ spinlock_t gpu_powered_lock; ++ ++ ++ struct kbasep_pm_metrics_data metrics; ++ ++ int gpu_poweroff_pending; ++ int shader_poweroff_pending_time; ++ ++ struct hrtimer gpu_poweroff_timer; ++ struct workqueue_struct *gpu_poweroff_wq; ++ struct work_struct gpu_poweroff_work; ++ ++ u64 shader_poweroff_pending; ++ u64 tiler_poweroff_pending; ++ ++ bool poweroff_timer_needed; ++ bool poweroff_timer_running; ++ ++ bool poweroff_wait_in_progress; ++ bool poweron_required; ++ bool poweroff_is_suspend; ++ ++ struct workqueue_struct *gpu_poweroff_wait_wq; ++ struct work_struct gpu_poweroff_wait_work; ++ ++ wait_queue_head_t poweroff_wait; ++ ++ int (*callback_power_on)(struct kbase_device *kbdev); ++ void (*callback_power_off)(struct kbase_device *kbdev); ++ void (*callback_power_suspend)(struct kbase_device *kbdev); ++ void (*callback_power_resume)(struct kbase_device *kbdev); ++ int (*callback_power_runtime_on)(struct kbase_device *kbdev); ++ void (*callback_power_runtime_off)(struct kbase_device *kbdev); ++ int (*callback_power_runtime_idle)(struct kbase_device *kbdev); ++}; ++ ++ ++/* List of policy IDs */ ++enum kbase_pm_policy_id { ++ KBASE_PM_POLICY_ID_DEMAND = 1, ++ KBASE_PM_POLICY_ID_ALWAYS_ON, ++ KBASE_PM_POLICY_ID_COARSE_DEMAND, ++#if !MALI_CUSTOMER_RELEASE ++ KBASE_PM_POLICY_ID_DEMAND_ALWAYS_POWERED, ++ KBASE_PM_POLICY_ID_FAST_START ++#endif ++}; ++ ++typedef u32 kbase_pm_policy_flags; ++ ++/** ++ * struct kbase_pm_policy - Power policy structure. ++ * ++ * Each power policy exposes a (static) instance of this structure which ++ * contains function pointers to the policy's methods. ++ * ++ * @name: The name of this policy ++ * @init: Function called when the policy is selected ++ * @term: Function called when the policy is unselected ++ * @get_core_mask: Function called to get the current shader core mask ++ * @get_core_active: Function called to get the current overall GPU power ++ * state ++ * @flags: Field indicating flags for this policy ++ * @id: Field indicating an ID for this policy. This is not ++ * necessarily the same as its index in the list returned ++ * by kbase_pm_list_policies(). ++ * It is used purely for debugging. ++ */ ++struct kbase_pm_policy { ++ char *name; ++ ++ /** ++ * Function called when the policy is selected ++ * ++ * This should initialize the kbdev->pm.pm_policy_data structure. It ++ * should not attempt to make any changes to hardware state. ++ * ++ * It is undefined what state the cores are in when the function is ++ * called. ++ * ++ * @kbdev: The kbase device structure for the device (must be a ++ * valid pointer) ++ */ ++ void (*init)(struct kbase_device *kbdev); ++ ++ /** ++ * Function called when the policy is unselected. ++ * ++ * @kbdev: The kbase device structure for the device (must be a ++ * valid pointer) ++ */ ++ void (*term)(struct kbase_device *kbdev); ++ ++ /** ++ * Function called to get the current shader core mask ++ * ++ * The returned mask should meet or exceed (kbdev->shader_needed_bitmap ++ * | kbdev->shader_inuse_bitmap). ++ * ++ * @kbdev: The kbase device structure for the device (must be a ++ * valid pointer) ++ * ++ * Return: The mask of shader cores to be powered ++ */ ++ u64 (*get_core_mask)(struct kbase_device *kbdev); ++ ++ /** ++ * Function called to get the current overall GPU power state ++ * ++ * This function should consider the state of kbdev->pm.active_count. If ++ * this count is greater than 0 then there is at least one active ++ * context on the device and the GPU should be powered. If it is equal ++ * to 0 then there are no active contexts and the GPU could be powered ++ * off if desired. ++ * ++ * @kbdev: The kbase device structure for the device (must be a ++ * valid pointer) ++ * ++ * Return: true if the GPU should be powered, false otherwise ++ */ ++ bool (*get_core_active)(struct kbase_device *kbdev); ++ ++ kbase_pm_policy_flags flags; ++ enum kbase_pm_policy_id id; ++}; ++ ++ ++enum kbase_pm_ca_policy_id { ++ KBASE_PM_CA_POLICY_ID_FIXED = 1, ++ KBASE_PM_CA_POLICY_ID_DEVFREQ, ++ KBASE_PM_CA_POLICY_ID_RANDOM ++}; ++ ++typedef u32 kbase_pm_ca_policy_flags; ++ ++/** ++ * Maximum length of a CA policy names ++ */ ++#define KBASE_PM_CA_MAX_POLICY_NAME_LEN 15 ++ ++/** ++ * struct kbase_pm_ca_policy - Core availability policy structure. ++ * ++ * Each core availability policy exposes a (static) instance of this structure ++ * which contains function pointers to the policy's methods. ++ * ++ * @name: The name of this policy ++ * @init: Function called when the policy is selected ++ * @term: Function called when the policy is unselected ++ * @get_core_mask: Function called to get the current shader core ++ * availability mask ++ * @update_core_status: Function called to update the current core status ++ * @flags: Field indicating flags for this policy ++ * @id: Field indicating an ID for this policy. This is not ++ * necessarily the same as its index in the list returned ++ * by kbase_pm_list_policies(). ++ * It is used purely for debugging. ++ */ ++struct kbase_pm_ca_policy { ++ char name[KBASE_PM_CA_MAX_POLICY_NAME_LEN + 1]; ++ ++ /** ++ * Function called when the policy is selected ++ * ++ * This should initialize the kbdev->pm.ca_policy_data structure. It ++ * should not attempt to make any changes to hardware state. ++ * ++ * It is undefined what state the cores are in when the function is ++ * called. ++ * ++ * @kbdev The kbase device structure for the device (must be a ++ * valid pointer) ++ */ ++ void (*init)(struct kbase_device *kbdev); ++ ++ /** ++ * Function called when the policy is unselected. ++ * ++ * @kbdev The kbase device structure for the device (must be a ++ * valid pointer) ++ */ ++ void (*term)(struct kbase_device *kbdev); ++ ++ /** ++ * Function called to get the current shader core availability mask ++ * ++ * When a change in core availability is occurring, the policy must set ++ * kbdev->pm.ca_in_transition to true. This is to indicate that ++ * reporting changes in power state cannot be optimized out, even if ++ * kbdev->pm.desired_shader_state remains unchanged. This must be done ++ * by any functions internal to the Core Availability Policy that change ++ * the return value of kbase_pm_ca_policy::get_core_mask. ++ * ++ * @kbdev The kbase device structure for the device (must be a ++ * valid pointer) ++ * ++ * Return: The current core availability mask ++ */ ++ u64 (*get_core_mask)(struct kbase_device *kbdev); ++ ++ /** ++ * Function called to update the current core status ++ * ++ * If none of the cores in core group 0 are ready or transitioning, then ++ * the policy must ensure that the next call to get_core_mask does not ++ * return 0 for all cores in core group 0. It is an error to disable ++ * core group 0 through the core availability policy. ++ * ++ * When a change in core availability has finished, the policy must set ++ * kbdev->pm.ca_in_transition to false. This is to indicate that ++ * changes in power state can once again be optimized out when ++ * kbdev->pm.desired_shader_state is unchanged. ++ * ++ * @kbdev: The kbase device structure for the device ++ * (must be a valid pointer) ++ * @cores_ready: The mask of cores currently powered and ++ * ready to run jobs ++ * @cores_transitioning: The mask of cores currently transitioning ++ * power state ++ */ ++ void (*update_core_status)(struct kbase_device *kbdev, u64 cores_ready, ++ u64 cores_transitioning); ++ ++ kbase_pm_ca_policy_flags flags; ++ ++ /** ++ * Field indicating an ID for this policy. This is not necessarily the ++ * same as its index in the list returned by kbase_pm_list_policies(). ++ * It is used purely for debugging. ++ */ ++ enum kbase_pm_ca_policy_id id; ++}; ++ ++#endif /* _KBASE_PM_HWACCESS_DEFS_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_demand.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_demand.c +new file mode 100755 +index 000000000000..81322fd0dd17 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_demand.c +@@ -0,0 +1,73 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * A simple demand based power management policy ++ */ ++ ++#include ++#include ++ ++static u64 demand_get_core_mask(struct kbase_device *kbdev) ++{ ++ u64 desired = kbdev->shader_needed_bitmap | kbdev->shader_inuse_bitmap; ++ ++ if (0 == kbdev->pm.active_count) ++ return 0; ++ ++ return desired; ++} ++ ++static bool demand_get_core_active(struct kbase_device *kbdev) ++{ ++ if (0 == kbdev->pm.active_count && !(kbdev->shader_needed_bitmap | ++ kbdev->shader_inuse_bitmap) && !kbdev->tiler_needed_cnt ++ && !kbdev->tiler_inuse_cnt) ++ return false; ++ ++ return true; ++} ++ ++static void demand_init(struct kbase_device *kbdev) ++{ ++ CSTD_UNUSED(kbdev); ++} ++ ++static void demand_term(struct kbase_device *kbdev) ++{ ++ CSTD_UNUSED(kbdev); ++} ++ ++/* ++ * The struct kbase_pm_policy structure for the demand power policy. ++ * ++ * This is the static structure that defines the demand power policy's callback ++ * and name. ++ */ ++const struct kbase_pm_policy kbase_pm_demand_policy_ops = { ++ "demand", /* name */ ++ demand_init, /* init */ ++ demand_term, /* term */ ++ demand_get_core_mask, /* get_core_mask */ ++ demand_get_core_active, /* get_core_active */ ++ 0u, /* flags */ ++ KBASE_PM_POLICY_ID_DEMAND, /* id */ ++}; ++ ++KBASE_EXPORT_TEST_API(kbase_pm_demand_policy_ops); +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_demand.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_demand.h +new file mode 100755 +index 000000000000..c0c84b6e9189 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_demand.h +@@ -0,0 +1,64 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * A simple demand based power management policy ++ */ ++ ++#ifndef MALI_KBASE_PM_DEMAND_H ++#define MALI_KBASE_PM_DEMAND_H ++ ++/** ++ * DOC: Demand power management policy ++ * ++ * The demand power management policy has the following characteristics: ++ * - When KBase indicates that the GPU will be powered up, but we don't yet ++ * know which Job Chains are to be run: ++ * - The Shader Cores are not powered up ++ * ++ * - When KBase indicates that a set of Shader Cores are needed to submit the ++ * currently queued Job Chains: ++ * - Only those Shader Cores are powered up ++ * ++ * - When KBase indicates that the GPU need not be powered: ++ * - The Shader Cores are powered off, and the GPU itself is powered off too. ++ * ++ * Note: ++ * - KBase indicates the GPU will be powered up when it has a User Process that ++ * has just started to submit Job Chains. ++ * ++ * - KBase indicates the GPU need not be powered when all the Job Chains from ++ * User Processes have finished, and it is waiting for a User Process to ++ * submit some more Job Chains. ++ */ ++ ++/** ++ * struct kbasep_pm_policy_demand - Private structure for policy instance data ++ * ++ * @dummy: No state is needed, a dummy variable ++ * ++ * This contains data that is private to the demand power policy. ++ */ ++struct kbasep_pm_policy_demand { ++ int dummy; ++}; ++ ++extern const struct kbase_pm_policy kbase_pm_demand_policy_ops; ++ ++#endif /* MALI_KBASE_PM_DEMAND_H */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_driver.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_driver.c +new file mode 100755 +index 000000000000..707f71a79a77 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_driver.c +@@ -0,0 +1,1672 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * Base kernel Power Management hardware control ++ */ ++ ++#include ++#include ++#include ++#if defined(CONFIG_MALI_BIFROST_GATOR_SUPPORT) ++#include ++#endif ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++ ++#if MALI_MOCK_TEST ++#define MOCKABLE(function) function##_original ++#else ++#define MOCKABLE(function) function ++#endif /* MALI_MOCK_TEST */ ++ ++/** ++ * enum kbasep_pm_action - Actions that can be performed on a core. ++ * ++ * This enumeration is private to the file. Its values are set to allow ++ * core_type_to_reg() function, which decodes this enumeration, to be simpler ++ * and more efficient. ++ * ++ * @ACTION_PRESENT: The cores that are present ++ * @ACTION_READY: The cores that are ready ++ * @ACTION_PWRON: Power on the cores specified ++ * @ACTION_PWROFF: Power off the cores specified ++ * @ACTION_PWRTRANS: The cores that are transitioning ++ * @ACTION_PWRACTIVE: The cores that are active ++ */ ++enum kbasep_pm_action { ++ ACTION_PRESENT = 0, ++ ACTION_READY = (SHADER_READY_LO - SHADER_PRESENT_LO), ++ ACTION_PWRON = (SHADER_PWRON_LO - SHADER_PRESENT_LO), ++ ACTION_PWROFF = (SHADER_PWROFF_LO - SHADER_PRESENT_LO), ++ ACTION_PWRTRANS = (SHADER_PWRTRANS_LO - SHADER_PRESENT_LO), ++ ACTION_PWRACTIVE = (SHADER_PWRACTIVE_LO - SHADER_PRESENT_LO) ++}; ++ ++static u64 kbase_pm_get_state( ++ struct kbase_device *kbdev, ++ enum kbase_pm_core_type core_type, ++ enum kbasep_pm_action action); ++ ++/** ++ * core_type_to_reg - Decode a core type and action to a register. ++ * ++ * Given a core type (defined by kbase_pm_core_type) and an action (defined ++ * by kbasep_pm_action) this function will return the register offset that ++ * will perform the action on the core type. The register returned is the _LO ++ * register and an offset must be applied to use the _HI register. ++ * ++ * @core_type: The type of core ++ * @action: The type of action ++ * ++ * Return: The register offset of the _LO register that performs an action of ++ * type @action on a core of type @core_type. ++ */ ++static u32 core_type_to_reg(enum kbase_pm_core_type core_type, ++ enum kbasep_pm_action action) ++{ ++#ifdef CONFIG_MALI_CORESTACK ++ if (core_type == KBASE_PM_CORE_STACK) { ++ switch (action) { ++ case ACTION_PRESENT: ++ return STACK_PRESENT_LO; ++ case ACTION_READY: ++ return STACK_READY_LO; ++ case ACTION_PWRON: ++ return STACK_PWRON_LO; ++ case ACTION_PWROFF: ++ return STACK_PWROFF_LO; ++ case ACTION_PWRTRANS: ++ return STACK_PWRTRANS_LO; ++ default: ++ BUG(); ++ } ++ } ++#endif /* CONFIG_MALI_CORESTACK */ ++ ++ return (u32)core_type + (u32)action; ++} ++ ++#ifdef CONFIG_ARM64 ++static void mali_cci_flush_l2(struct kbase_device *kbdev) ++{ ++ const u32 mask = CLEAN_CACHES_COMPLETED | RESET_COMPLETED; ++ u32 loops = KBASE_CLEAN_CACHE_MAX_LOOPS; ++ u32 raw; ++ ++ /* ++ * Note that we don't take the cache flush mutex here since ++ * we expect to be the last user of the L2, all other L2 users ++ * would have dropped their references, to initiate L2 power ++ * down, L2 power down being the only valid place for this ++ * to be called from. ++ */ ++ ++ kbase_reg_write(kbdev, ++ GPU_CONTROL_REG(GPU_COMMAND), ++ GPU_COMMAND_CLEAN_INV_CACHES, ++ NULL); ++ ++ raw = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), ++ NULL); ++ ++ /* Wait for cache flush to complete before continuing, exit on ++ * gpu resets or loop expiry. */ ++ while (((raw & mask) == 0) && --loops) { ++ raw = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), ++ NULL); ++ } ++} ++#endif ++ ++/** ++ * kbase_pm_invoke - Invokes an action on a core set ++ * ++ * This function performs the action given by @action on a set of cores of a ++ * type given by @core_type. It is a static function used by ++ * kbase_pm_transition_core_type() ++ * ++ * @kbdev: The kbase device structure of the device ++ * @core_type: The type of core that the action should be performed on ++ * @cores: A bit mask of cores to perform the action on (low 32 bits) ++ * @action: The action to perform on the cores ++ */ ++static void kbase_pm_invoke(struct kbase_device *kbdev, ++ enum kbase_pm_core_type core_type, ++ u64 cores, ++ enum kbasep_pm_action action) ++{ ++ u32 reg; ++ u32 lo = cores & 0xFFFFFFFF; ++ u32 hi = (cores >> 32) & 0xFFFFFFFF; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ reg = core_type_to_reg(core_type, action); ++ ++ KBASE_DEBUG_ASSERT(reg); ++#if defined(CONFIG_MALI_BIFROST_GATOR_SUPPORT) ++ if (cores) { ++ if (action == ACTION_PWRON) ++ kbase_trace_mali_pm_power_on(core_type, cores); ++ else if (action == ACTION_PWROFF) ++ kbase_trace_mali_pm_power_off(core_type, cores); ++ } ++#endif ++ ++ if (cores) { ++ u64 state = kbase_pm_get_state(kbdev, core_type, ACTION_READY); ++ ++ if (action == ACTION_PWRON) ++ state |= cores; ++ else if (action == ACTION_PWROFF) ++ state &= ~cores; ++ KBASE_TLSTREAM_AUX_PM_STATE(core_type, state); ++ } ++ ++ /* Tracing */ ++ if (cores) { ++ if (action == ACTION_PWRON) ++ switch (core_type) { ++ case KBASE_PM_CORE_SHADER: ++ KBASE_TRACE_ADD(kbdev, PM_PWRON, NULL, NULL, 0u, ++ lo); ++ break; ++ case KBASE_PM_CORE_TILER: ++ KBASE_TRACE_ADD(kbdev, PM_PWRON_TILER, NULL, ++ NULL, 0u, lo); ++ break; ++ case KBASE_PM_CORE_L2: ++ KBASE_TRACE_ADD(kbdev, PM_PWRON_L2, NULL, NULL, ++ 0u, lo); ++ break; ++ default: ++ break; ++ } ++ else if (action == ACTION_PWROFF) ++ switch (core_type) { ++ case KBASE_PM_CORE_SHADER: ++ KBASE_TRACE_ADD(kbdev, PM_PWROFF, NULL, NULL, ++ 0u, lo); ++ break; ++ case KBASE_PM_CORE_TILER: ++ KBASE_TRACE_ADD(kbdev, PM_PWROFF_TILER, NULL, ++ NULL, 0u, lo); ++ break; ++ case KBASE_PM_CORE_L2: ++ KBASE_TRACE_ADD(kbdev, PM_PWROFF_L2, NULL, NULL, ++ 0u, lo); ++ /* disable snoops before L2 is turned off */ ++ kbase_pm_cache_snoop_disable(kbdev); ++ break; ++ default: ++ break; ++ } ++ } ++ ++ if (lo != 0) ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(reg), lo, NULL); ++ ++ if (hi != 0) ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(reg + 4), hi, NULL); ++} ++ ++/** ++ * kbase_pm_get_state - Get information about a core set ++ * ++ * This function gets information (chosen by @action) about a set of cores of ++ * a type given by @core_type. It is a static function used by ++ * kbase_pm_get_active_cores(), kbase_pm_get_trans_cores() and ++ * kbase_pm_get_ready_cores(). ++ * ++ * @kbdev: The kbase device structure of the device ++ * @core_type: The type of core that the should be queried ++ * @action: The property of the cores to query ++ * ++ * Return: A bit mask specifying the state of the cores ++ */ ++static u64 kbase_pm_get_state(struct kbase_device *kbdev, ++ enum kbase_pm_core_type core_type, ++ enum kbasep_pm_action action) ++{ ++ u32 reg; ++ u32 lo, hi; ++ ++ reg = core_type_to_reg(core_type, action); ++ ++ KBASE_DEBUG_ASSERT(reg); ++ ++ lo = kbase_reg_read(kbdev, GPU_CONTROL_REG(reg), NULL); ++ hi = kbase_reg_read(kbdev, GPU_CONTROL_REG(reg + 4), NULL); ++ ++ return (((u64) hi) << 32) | ((u64) lo); ++} ++ ++void kbasep_pm_init_core_use_bitmaps(struct kbase_device *kbdev) ++{ ++ kbdev->shader_inuse_bitmap = 0; ++ kbdev->shader_needed_bitmap = 0; ++ kbdev->shader_available_bitmap = 0; ++ kbdev->tiler_available_bitmap = 0; ++ kbdev->l2_users_count = 0; ++ kbdev->l2_available_bitmap = 0; ++ kbdev->tiler_needed_cnt = 0; ++ kbdev->tiler_inuse_cnt = 0; ++ ++ memset(kbdev->shader_needed_cnt, 0, sizeof(kbdev->shader_needed_cnt)); ++} ++ ++/** ++ * kbase_pm_get_present_cores - Get the cores that are present ++ * ++ * @kbdev: Kbase device ++ * @type: The type of cores to query ++ * ++ * Return: Bitmask of the cores that are present ++ */ ++u64 kbase_pm_get_present_cores(struct kbase_device *kbdev, ++ enum kbase_pm_core_type type) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ switch (type) { ++ case KBASE_PM_CORE_L2: ++ return kbdev->gpu_props.props.raw_props.l2_present; ++ case KBASE_PM_CORE_SHADER: ++ return kbdev->gpu_props.props.raw_props.shader_present; ++ case KBASE_PM_CORE_TILER: ++ return kbdev->gpu_props.props.raw_props.tiler_present; ++#ifdef CONFIG_MALI_CORESTACK ++ case KBASE_PM_CORE_STACK: ++ return kbdev->gpu_props.props.raw_props.stack_present; ++#endif /* CONFIG_MALI_CORESTACK */ ++ default: ++ break; ++ } ++ KBASE_DEBUG_ASSERT(0); ++ ++ return 0; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_get_present_cores); ++ ++/** ++ * kbase_pm_get_active_cores - Get the cores that are "active" ++ * (busy processing work) ++ * ++ * @kbdev: Kbase device ++ * @type: The type of cores to query ++ * ++ * Return: Bitmask of cores that are active ++ */ ++u64 kbase_pm_get_active_cores(struct kbase_device *kbdev, ++ enum kbase_pm_core_type type) ++{ ++ return kbase_pm_get_state(kbdev, type, ACTION_PWRACTIVE); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_get_active_cores); ++ ++/** ++ * kbase_pm_get_trans_cores - Get the cores that are transitioning between ++ * power states ++ * ++ * @kbdev: Kbase device ++ * @type: The type of cores to query ++ * ++ * Return: Bitmask of cores that are transitioning ++ */ ++u64 kbase_pm_get_trans_cores(struct kbase_device *kbdev, ++ enum kbase_pm_core_type type) ++{ ++ return kbase_pm_get_state(kbdev, type, ACTION_PWRTRANS); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_get_trans_cores); ++ ++/** ++ * kbase_pm_get_ready_cores - Get the cores that are powered on ++ * ++ * @kbdev: Kbase device ++ * @type: The type of cores to query ++ * ++ * Return: Bitmask of cores that are ready (powered on) ++ */ ++u64 kbase_pm_get_ready_cores(struct kbase_device *kbdev, ++ enum kbase_pm_core_type type) ++{ ++ u64 result; ++ ++ result = kbase_pm_get_state(kbdev, type, ACTION_READY); ++ ++ switch (type) { ++ case KBASE_PM_CORE_SHADER: ++ KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED, NULL, NULL, 0u, ++ (u32) result); ++ break; ++ case KBASE_PM_CORE_TILER: ++ KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED_TILER, NULL, NULL, 0u, ++ (u32) result); ++ break; ++ case KBASE_PM_CORE_L2: ++ KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED_L2, NULL, NULL, 0u, ++ (u32) result); ++ break; ++ default: ++ break; ++ } ++ ++ return result; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_get_ready_cores); ++ ++/** ++ * kbase_pm_transition_core_type - Perform power transitions for a particular ++ * core type. ++ * ++ * This function will perform any available power transitions to make the actual ++ * hardware state closer to the desired state. If a core is currently ++ * transitioning then changes to the power state of that call cannot be made ++ * until the transition has finished. Cores which are not present in the ++ * hardware are ignored if they are specified in the desired_state bitmask, ++ * however the return value will always be 0 in this case. ++ * ++ * @kbdev: The kbase device ++ * @type: The core type to perform transitions for ++ * @desired_state: A bit mask of the desired state of the cores ++ * @in_use: A bit mask of the cores that are currently running ++ * jobs. These cores have to be kept powered up because ++ * there are jobs running (or about to run) on them. ++ * @available: Receives a bit mask of the cores that the job ++ * scheduler can use to submit jobs to. May be NULL if ++ * this is not needed. ++ * @powering_on: Bit mask to update with cores that are ++ * transitioning to a power-on state. ++ * ++ * Return: true if the desired state has been reached, false otherwise ++ */ ++static bool kbase_pm_transition_core_type(struct kbase_device *kbdev, ++ enum kbase_pm_core_type type, ++ u64 desired_state, ++ u64 in_use, ++ u64 * const available, ++ u64 *powering_on) ++{ ++ u64 present; ++ u64 ready; ++ u64 trans; ++ u64 powerup; ++ u64 powerdown; ++ u64 powering_on_trans; ++ u64 desired_state_in_use; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ /* Get current state */ ++ present = kbase_pm_get_present_cores(kbdev, type); ++ trans = kbase_pm_get_trans_cores(kbdev, type); ++ ready = kbase_pm_get_ready_cores(kbdev, type); ++ /* mask off ready from trans in case transitions finished between the ++ * register reads */ ++ trans &= ~ready; ++ ++ if (trans) /* Do not progress if any cores are transitioning */ ++ return false; ++ ++ powering_on_trans = trans & *powering_on; ++ *powering_on = powering_on_trans; ++ ++ if (available != NULL) ++ *available = (ready | powering_on_trans) & desired_state; ++ ++ /* Update desired state to include the in-use cores. These have to be ++ * kept powered up because there are jobs running or about to run on ++ * these cores ++ */ ++ desired_state_in_use = desired_state | in_use; ++ ++ /* Update state of whether l2 caches are powered */ ++ if (type == KBASE_PM_CORE_L2) { ++ if ((ready == present) && (desired_state_in_use == ready) && ++ (trans == 0)) { ++ /* All are ready, none will be turned off, and none are ++ * transitioning */ ++ kbdev->pm.backend.l2_powered = 1; ++ /* ++ * Ensure snoops are enabled after L2 is powered up, ++ * note that kbase keeps track of the snoop state, so ++ * safe to repeatedly call. ++ */ ++ kbase_pm_cache_snoop_enable(kbdev); ++ if (kbdev->l2_users_count > 0) { ++ /* Notify any registered l2 cache users ++ * (optimized out when no users waiting) */ ++ wake_up(&kbdev->pm.backend.l2_powered_wait); ++ } ++ } else ++ kbdev->pm.backend.l2_powered = 0; ++ } ++ ++ if (desired_state == ready && (trans == 0)) ++ return true; ++ ++ /* Restrict the cores to those that are actually present */ ++ powerup = desired_state_in_use & present; ++ powerdown = (~desired_state_in_use) & present; ++ ++ /* Restrict to cores that are not already in the desired state */ ++ powerup &= ~ready; ++ powerdown &= ready; ++ ++ /* Don't transition any cores that are already transitioning, except for ++ * Mali cores that support the following case: ++ * ++ * If the SHADER_PWRON or TILER_PWRON registers are written to turn on ++ * a core that is currently transitioning to power off, then this is ++ * remembered and the shader core is automatically powered up again once ++ * the original transition completes. Once the automatic power on is ++ * complete any job scheduled on the shader core should start. ++ */ ++ powerdown &= ~trans; ++ ++ if (kbase_hw_has_feature(kbdev, ++ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS)) ++ if (KBASE_PM_CORE_SHADER == type || KBASE_PM_CORE_TILER == type) ++ trans = powering_on_trans; /* for exception cases, only ++ * mask off cores in power on ++ * transitions */ ++ ++ powerup &= ~trans; ++ ++ /* Perform transitions if any */ ++ kbase_pm_invoke(kbdev, type, powerup, ACTION_PWRON); ++#if !PLATFORM_POWER_DOWN_ONLY ++ kbase_pm_invoke(kbdev, type, powerdown, ACTION_PWROFF); ++#endif ++ ++ /* Recalculate cores transitioning on, and re-evaluate our state */ ++ powering_on_trans |= powerup; ++ *powering_on = powering_on_trans; ++ if (available != NULL) ++ *available = (ready | powering_on_trans) & desired_state; ++ ++ return false; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_transition_core_type); ++ ++/** ++ * get_desired_cache_status - Determine which caches should be on for a ++ * particular core state ++ * ++ * This function takes a bit mask of the present caches and the cores (or ++ * caches) that are attached to the caches that will be powered. It then ++ * computes which caches should be turned on to allow the cores requested to be ++ * powered up. ++ * ++ * @present: The bit mask of present caches ++ * @cores_powered: A bit mask of cores (or L2 caches) that are desired to ++ * be powered ++ * @tilers_powered: The bit mask of tilers that are desired to be powered ++ * ++ * Return: A bit mask of the caches that should be turned on ++ */ ++static u64 get_desired_cache_status(u64 present, u64 cores_powered, ++ u64 tilers_powered) ++{ ++ u64 desired = 0; ++ ++ while (present) { ++ /* Find out which is the highest set bit */ ++ u64 bit = fls64(present) - 1; ++ u64 bit_mask = 1ull << bit; ++ /* Create a mask which has all bits from 'bit' upwards set */ ++ ++ u64 mask = ~(bit_mask - 1); ++ ++ /* If there are any cores powered at this bit or above (that ++ * haven't previously been processed) then we need this core on ++ */ ++ if (cores_powered & mask) ++ desired |= bit_mask; ++ ++ /* Remove bits from cores_powered and present */ ++ cores_powered &= ~mask; ++ present &= ~bit_mask; ++ } ++ ++ /* Power up the required L2(s) for the tiler */ ++ if (tilers_powered) ++ desired |= 1; ++ ++ return desired; ++} ++ ++KBASE_EXPORT_TEST_API(get_desired_cache_status); ++ ++#ifdef CONFIG_MALI_CORESTACK ++u64 kbase_pm_core_stack_mask(u64 cores) ++{ ++ u64 stack_mask = 0; ++ size_t const MAX_CORE_ID = 31; ++ size_t const NUM_CORES_PER_STACK = 4; ++ size_t i; ++ ++ for (i = 0; i <= MAX_CORE_ID; ++i) { ++ if (test_bit(i, (unsigned long *)&cores)) { ++ /* Every core which ID >= 16 is filled to stacks 4-7 ++ * instead of 0-3 */ ++ size_t const stack_num = (i > 16) ? ++ (i % NUM_CORES_PER_STACK) + 4 : ++ (i % NUM_CORES_PER_STACK); ++ set_bit(stack_num, (unsigned long *)&stack_mask); ++ } ++ } ++ ++ return stack_mask; ++} ++#endif /* CONFIG_MALI_CORESTACK */ ++ ++bool ++MOCKABLE(kbase_pm_check_transitions_nolock) (struct kbase_device *kbdev) ++{ ++ bool cores_are_available = false; ++ bool in_desired_state = true; ++ u64 desired_l2_state; ++#ifdef CONFIG_MALI_CORESTACK ++ u64 desired_stack_state; ++ u64 stacks_powered; ++#endif /* CONFIG_MALI_CORESTACK */ ++ u64 cores_powered; ++ u64 tilers_powered; ++ u64 tiler_available_bitmap; ++ u64 tiler_transitioning_bitmap; ++ u64 shader_available_bitmap; ++ u64 shader_ready_bitmap; ++ u64 shader_transitioning_bitmap; ++ u64 l2_available_bitmap; ++ u64 prev_l2_available_bitmap; ++ u64 l2_inuse_bitmap; ++ ++ KBASE_DEBUG_ASSERT(NULL != kbdev); ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ spin_lock(&kbdev->pm.backend.gpu_powered_lock); ++ if (kbdev->pm.backend.gpu_powered == false) { ++ spin_unlock(&kbdev->pm.backend.gpu_powered_lock); ++ if (kbdev->pm.backend.desired_shader_state == 0 && ++ kbdev->pm.backend.desired_tiler_state == 0) ++ return true; ++ return false; ++ } ++ ++ /* Trace that a change-state is being requested, and that it took ++ * (effectively) no time to start it. This is useful for counting how ++ * many state changes occurred, in a way that's backwards-compatible ++ * with processing the trace data */ ++ kbase_timeline_pm_send_event(kbdev, ++ KBASE_TIMELINE_PM_EVENT_CHANGE_GPU_STATE); ++ kbase_timeline_pm_handle_event(kbdev, ++ KBASE_TIMELINE_PM_EVENT_CHANGE_GPU_STATE); ++ ++ /* If any cores are already powered then, we must keep the caches on */ ++ shader_transitioning_bitmap = kbase_pm_get_trans_cores(kbdev, ++ KBASE_PM_CORE_SHADER); ++ cores_powered = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER); ++ cores_powered |= kbdev->pm.backend.desired_shader_state; ++ ++#ifdef CONFIG_MALI_CORESTACK ++ /* Work out which core stacks want to be powered */ ++ desired_stack_state = kbase_pm_core_stack_mask(cores_powered); ++ stacks_powered = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_STACK) | ++ desired_stack_state; ++#endif /* CONFIG_MALI_CORESTACK */ ++ ++ /* Work out which tilers want to be powered */ ++ tiler_transitioning_bitmap = kbase_pm_get_trans_cores(kbdev, ++ KBASE_PM_CORE_TILER); ++ tilers_powered = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_TILER); ++ tilers_powered |= kbdev->pm.backend.desired_tiler_state; ++ ++ /* If there are l2 cache users registered, keep all l2s powered even if ++ * all other cores are off. */ ++ if (kbdev->l2_users_count > 0) ++ cores_powered |= kbdev->gpu_props.props.raw_props.l2_present; ++ ++ desired_l2_state = get_desired_cache_status( ++ kbdev->gpu_props.props.raw_props.l2_present, ++ cores_powered, tilers_powered); ++ ++ l2_inuse_bitmap = get_desired_cache_status( ++ kbdev->gpu_props.props.raw_props.l2_present, ++ cores_powered | shader_transitioning_bitmap, ++ tilers_powered | tiler_transitioning_bitmap); ++ ++#ifdef CONFIG_MALI_CORESTACK ++ if (stacks_powered) ++ desired_l2_state |= 1; ++#endif /* CONFIG_MALI_CORESTACK */ ++ ++ /* If any l2 cache is on, then enable l2 #0, for use by job manager */ ++ if (0 != desired_l2_state) ++ desired_l2_state |= 1; ++ ++ prev_l2_available_bitmap = kbdev->l2_available_bitmap; ++ in_desired_state &= kbase_pm_transition_core_type(kbdev, ++ KBASE_PM_CORE_L2, desired_l2_state, l2_inuse_bitmap, ++ &l2_available_bitmap, ++ &kbdev->pm.backend.powering_on_l2_state); ++ ++ if (kbdev->l2_available_bitmap != l2_available_bitmap) ++ KBASE_TIMELINE_POWER_L2(kbdev, l2_available_bitmap); ++ ++ kbdev->l2_available_bitmap = l2_available_bitmap; ++ ++ ++#ifdef CONFIG_MALI_CORESTACK ++ if (in_desired_state) { ++ in_desired_state &= kbase_pm_transition_core_type(kbdev, ++ KBASE_PM_CORE_STACK, desired_stack_state, 0, ++ &kbdev->stack_available_bitmap, ++ &kbdev->pm.backend.powering_on_stack_state); ++ } ++#endif /* CONFIG_MALI_CORESTACK */ ++ ++ if (in_desired_state) { ++ in_desired_state &= kbase_pm_transition_core_type(kbdev, ++ KBASE_PM_CORE_TILER, ++ kbdev->pm.backend.desired_tiler_state, ++ 0, &tiler_available_bitmap, ++ &kbdev->pm.backend.powering_on_tiler_state); ++ in_desired_state &= kbase_pm_transition_core_type(kbdev, ++ KBASE_PM_CORE_SHADER, ++ kbdev->pm.backend.desired_shader_state, ++ kbdev->shader_inuse_bitmap, ++ &shader_available_bitmap, ++ &kbdev->pm.backend.powering_on_shader_state); ++ ++ if (kbdev->shader_available_bitmap != shader_available_bitmap) { ++ KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, ++ NULL, 0u, ++ (u32) shader_available_bitmap); ++ KBASE_TIMELINE_POWER_SHADER(kbdev, ++ shader_available_bitmap); ++ } ++ ++ kbdev->shader_available_bitmap = shader_available_bitmap; ++ ++ if (kbdev->tiler_available_bitmap != tiler_available_bitmap) { ++ KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER, ++ NULL, NULL, 0u, ++ (u32) tiler_available_bitmap); ++ KBASE_TIMELINE_POWER_TILER(kbdev, ++ tiler_available_bitmap); ++ } ++ ++ kbdev->tiler_available_bitmap = tiler_available_bitmap; ++ ++ } else if ((l2_available_bitmap & ++ kbdev->gpu_props.props.raw_props.tiler_present) != ++ kbdev->gpu_props.props.raw_props.tiler_present) { ++ tiler_available_bitmap = 0; ++ ++ if (kbdev->tiler_available_bitmap != tiler_available_bitmap) ++ KBASE_TIMELINE_POWER_TILER(kbdev, ++ tiler_available_bitmap); ++ ++ kbdev->tiler_available_bitmap = tiler_available_bitmap; ++ } ++ ++ /* State updated for slow-path waiters */ ++ kbdev->pm.backend.gpu_in_desired_state = in_desired_state; ++ ++ shader_ready_bitmap = kbase_pm_get_ready_cores(kbdev, ++ KBASE_PM_CORE_SHADER); ++ shader_transitioning_bitmap = kbase_pm_get_trans_cores(kbdev, ++ KBASE_PM_CORE_SHADER); ++ ++ /* Determine whether the cores are now available (even if the set of ++ * available cores is empty). Note that they can be available even if ++ * we've not finished transitioning to the desired state */ ++ if ((kbdev->shader_available_bitmap & ++ kbdev->pm.backend.desired_shader_state) ++ == kbdev->pm.backend.desired_shader_state && ++ (kbdev->tiler_available_bitmap & ++ kbdev->pm.backend.desired_tiler_state) ++ == kbdev->pm.backend.desired_tiler_state) { ++ cores_are_available = true; ++ ++ KBASE_TRACE_ADD(kbdev, PM_CORES_AVAILABLE, NULL, NULL, 0u, ++ (u32)(kbdev->shader_available_bitmap & ++ kbdev->pm.backend.desired_shader_state)); ++ KBASE_TRACE_ADD(kbdev, PM_CORES_AVAILABLE_TILER, NULL, NULL, 0u, ++ (u32)(kbdev->tiler_available_bitmap & ++ kbdev->pm.backend.desired_tiler_state)); ++ ++ /* Log timelining information about handling events that power ++ * up cores, to match up either with immediate submission either ++ * because cores already available, or from PM IRQ */ ++ if (!in_desired_state) ++ kbase_timeline_pm_send_event(kbdev, ++ KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED); ++ } ++ ++ if (in_desired_state) { ++ KBASE_DEBUG_ASSERT(cores_are_available); ++ ++#if defined(CONFIG_MALI_BIFROST_GATOR_SUPPORT) ++ kbase_trace_mali_pm_status(KBASE_PM_CORE_L2, ++ kbase_pm_get_ready_cores(kbdev, ++ KBASE_PM_CORE_L2)); ++ kbase_trace_mali_pm_status(KBASE_PM_CORE_SHADER, ++ kbase_pm_get_ready_cores(kbdev, ++ KBASE_PM_CORE_SHADER)); ++ kbase_trace_mali_pm_status(KBASE_PM_CORE_TILER, ++ kbase_pm_get_ready_cores(kbdev, ++ KBASE_PM_CORE_TILER)); ++#ifdef CONFIG_MALI_CORESTACK ++ kbase_trace_mali_pm_status(KBASE_PM_CORE_STACK, ++ kbase_pm_get_ready_cores(kbdev, ++ KBASE_PM_CORE_STACK)); ++#endif /* CONFIG_MALI_CORESTACK */ ++#endif ++ ++ KBASE_TLSTREAM_AUX_PM_STATE( ++ KBASE_PM_CORE_L2, ++ kbase_pm_get_ready_cores( ++ kbdev, KBASE_PM_CORE_L2)); ++ KBASE_TLSTREAM_AUX_PM_STATE( ++ KBASE_PM_CORE_SHADER, ++ kbase_pm_get_ready_cores( ++ kbdev, KBASE_PM_CORE_SHADER)); ++ KBASE_TLSTREAM_AUX_PM_STATE( ++ KBASE_PM_CORE_TILER, ++ kbase_pm_get_ready_cores( ++ kbdev, ++ KBASE_PM_CORE_TILER)); ++#ifdef CONFIG_MALI_CORESTACK ++ KBASE_TLSTREAM_AUX_PM_STATE( ++ KBASE_PM_CORE_STACK, ++ kbase_pm_get_ready_cores( ++ kbdev, ++ KBASE_PM_CORE_STACK)); ++#endif /* CONFIG_MALI_CORESTACK */ ++ ++ KBASE_TRACE_ADD(kbdev, PM_DESIRED_REACHED, NULL, NULL, ++ kbdev->pm.backend.gpu_in_desired_state, ++ (u32)kbdev->pm.backend.desired_shader_state); ++ KBASE_TRACE_ADD(kbdev, PM_DESIRED_REACHED_TILER, NULL, NULL, 0u, ++ (u32)kbdev->pm.backend.desired_tiler_state); ++ ++ /* Log timelining information for synchronous waiters */ ++ kbase_timeline_pm_send_event(kbdev, ++ KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED); ++ /* Wake slow-path waiters. Job scheduler does not use this. */ ++ KBASE_TRACE_ADD(kbdev, PM_WAKE_WAITERS, NULL, NULL, 0u, 0); ++ ++ wake_up(&kbdev->pm.backend.gpu_in_desired_state_wait); ++ } ++ ++ spin_unlock(&kbdev->pm.backend.gpu_powered_lock); ++ ++ /* kbase_pm_ca_update_core_status can cause one-level recursion into ++ * this function, so it must only be called once all changes to kbdev ++ * have been committed, and after the gpu_powered_lock has been ++ * dropped. */ ++ if (kbdev->shader_ready_bitmap != shader_ready_bitmap || ++ kbdev->shader_transitioning_bitmap != shader_transitioning_bitmap) { ++ kbdev->shader_ready_bitmap = shader_ready_bitmap; ++ kbdev->shader_transitioning_bitmap = ++ shader_transitioning_bitmap; ++ ++ kbase_pm_ca_update_core_status(kbdev, shader_ready_bitmap, ++ shader_transitioning_bitmap); ++ } ++ ++ /* The core availability policy is not allowed to keep core group 0 ++ * turned off (unless it was changing the l2 power state) */ ++ if (!((shader_ready_bitmap | shader_transitioning_bitmap) & ++ kbdev->gpu_props.props.coherency_info.group[0].core_mask) && ++ (prev_l2_available_bitmap == desired_l2_state) && ++ !(kbase_pm_ca_get_core_mask(kbdev) & ++ kbdev->gpu_props.props.coherency_info.group[0].core_mask)) ++ BUG(); ++ ++ /* The core availability policy is allowed to keep core group 1 off, ++ * but all jobs specifically targeting CG1 must fail */ ++ if (!((shader_ready_bitmap | shader_transitioning_bitmap) & ++ kbdev->gpu_props.props.coherency_info.group[1].core_mask) && ++ !(kbase_pm_ca_get_core_mask(kbdev) & ++ kbdev->gpu_props.props.coherency_info.group[1].core_mask)) ++ kbdev->pm.backend.cg1_disabled = true; ++ else ++ kbdev->pm.backend.cg1_disabled = false; ++ ++ return cores_are_available; ++} ++KBASE_EXPORT_TEST_API(kbase_pm_check_transitions_nolock); ++ ++/* Timeout for kbase_pm_check_transitions_sync when wait_event_killable has ++ * aborted due to a fatal signal. If the time spent waiting has exceeded this ++ * threshold then there is most likely a hardware issue. */ ++#define PM_TIMEOUT (5*HZ) /* 5s */ ++ ++void kbase_pm_check_transitions_sync(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ unsigned long timeout; ++ bool cores_are_available; ++ int ret; ++ ++ /* Force the transition to be checked and reported - the cores may be ++ * 'available' (for job submission) but not fully powered up. */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ cores_are_available = kbase_pm_check_transitions_nolock(kbdev); ++ ++ /* Don't need 'cores_are_available', because we don't return anything */ ++ CSTD_UNUSED(cores_are_available); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ timeout = jiffies + PM_TIMEOUT; ++ ++ /* Wait for cores */ ++ ret = wait_event_killable(kbdev->pm.backend.gpu_in_desired_state_wait, ++ kbdev->pm.backend.gpu_in_desired_state); ++ ++ if (ret < 0 && time_after(jiffies, timeout)) { ++ dev_err(kbdev->dev, "Power transition timed out unexpectedly\n"); ++ dev_err(kbdev->dev, "Desired state :\n"); ++ dev_err(kbdev->dev, "\tShader=%016llx\n", ++ kbdev->pm.backend.desired_shader_state); ++ dev_err(kbdev->dev, "\tTiler =%016llx\n", ++ kbdev->pm.backend.desired_tiler_state); ++ dev_err(kbdev->dev, "Current state :\n"); ++ dev_err(kbdev->dev, "\tShader=%08x%08x\n", ++ kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(SHADER_READY_HI), NULL), ++ kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(SHADER_READY_LO), ++ NULL)); ++ dev_err(kbdev->dev, "\tTiler =%08x%08x\n", ++ kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(TILER_READY_HI), NULL), ++ kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(TILER_READY_LO), NULL)); ++ dev_err(kbdev->dev, "\tL2 =%08x%08x\n", ++ kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(L2_READY_HI), NULL), ++ kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(L2_READY_LO), NULL)); ++ dev_err(kbdev->dev, "Cores transitioning :\n"); ++ dev_err(kbdev->dev, "\tShader=%08x%08x\n", ++ kbase_reg_read(kbdev, GPU_CONTROL_REG( ++ SHADER_PWRTRANS_HI), NULL), ++ kbase_reg_read(kbdev, GPU_CONTROL_REG( ++ SHADER_PWRTRANS_LO), NULL)); ++ dev_err(kbdev->dev, "\tTiler =%08x%08x\n", ++ kbase_reg_read(kbdev, GPU_CONTROL_REG( ++ TILER_PWRTRANS_HI), NULL), ++ kbase_reg_read(kbdev, GPU_CONTROL_REG( ++ TILER_PWRTRANS_LO), NULL)); ++ dev_err(kbdev->dev, "\tL2 =%08x%08x\n", ++ kbase_reg_read(kbdev, GPU_CONTROL_REG( ++ L2_PWRTRANS_HI), NULL), ++ kbase_reg_read(kbdev, GPU_CONTROL_REG( ++ L2_PWRTRANS_LO), NULL)); ++#if KBASE_GPU_RESET_EN ++ dev_err(kbdev->dev, "Sending reset to GPU - all running jobs will be lost\n"); ++ if (kbase_prepare_to_reset_gpu(kbdev)) ++ kbase_reset_gpu(kbdev); ++#endif /* KBASE_GPU_RESET_EN */ ++ } else { ++ /* Log timelining information that a change in state has ++ * completed */ ++ kbase_timeline_pm_handle_event(kbdev, ++ KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED); ++ } ++} ++KBASE_EXPORT_TEST_API(kbase_pm_check_transitions_sync); ++ ++void kbase_pm_enable_interrupts(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ ++ KBASE_DEBUG_ASSERT(NULL != kbdev); ++ /* ++ * Clear all interrupts, ++ * and unmask them all. ++ */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), GPU_IRQ_REG_ALL, ++ NULL); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), GPU_IRQ_REG_ALL, ++ NULL); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), 0xFFFFFFFF, ++ NULL); ++ kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), 0xFFFFFFFF, NULL); ++ ++ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), 0xFFFFFFFF, NULL); ++ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0xFFFFFFFF, NULL); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_enable_interrupts); ++ ++void kbase_pm_disable_interrupts_nolock(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(NULL != kbdev); ++ /* ++ * Mask all interrupts, ++ * and clear them all. ++ */ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), 0, NULL); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), GPU_IRQ_REG_ALL, ++ NULL); ++ kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), 0, NULL); ++ kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), 0xFFFFFFFF, ++ NULL); ++ ++ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0, NULL); ++ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), 0xFFFFFFFF, NULL); ++} ++ ++void kbase_pm_disable_interrupts(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_pm_disable_interrupts_nolock(kbdev); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_disable_interrupts); ++ ++ ++/* ++ * pmu layout: ++ * 0x0000: PMU TAG (RO) (0xCAFECAFE) ++ * 0x0004: PMU VERSION ID (RO) (0x00000000) ++ * 0x0008: CLOCK ENABLE (RW) (31:1 SBZ, 0 CLOCK STATE) ++ */ ++void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume) ++{ ++ bool reset_required = is_resume; ++ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; ++ unsigned long flags; ++ ++ KBASE_DEBUG_ASSERT(NULL != kbdev); ++ lockdep_assert_held(&js_devdata->runpool_mutex); ++ lockdep_assert_held(&kbdev->pm.lock); ++ ++ if (kbdev->pm.backend.gpu_powered) { ++ /* Already turned on */ ++ if (kbdev->poweroff_pending) ++ kbase_pm_enable_interrupts(kbdev); ++ kbdev->poweroff_pending = false; ++ KBASE_DEBUG_ASSERT(!is_resume); ++ return; ++ } ++ ++ kbdev->poweroff_pending = false; ++ ++ KBASE_TRACE_ADD(kbdev, PM_GPU_ON, NULL, NULL, 0u, 0u); ++ ++ if (is_resume && kbdev->pm.backend.callback_power_resume) { ++ kbdev->pm.backend.callback_power_resume(kbdev); ++ return; ++ } else if (kbdev->pm.backend.callback_power_on) { ++ kbdev->pm.backend.callback_power_on(kbdev); ++ /* If your platform properly keeps the GPU state you may use the ++ * return value of the callback_power_on function to ++ * conditionally reset the GPU on power up. Currently we are ++ * conservative and always reset the GPU. */ ++ reset_required = true; ++ } ++ ++ spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags); ++ kbdev->pm.backend.gpu_powered = true; ++ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags); ++ ++ if (reset_required) { ++ /* GPU state was lost, reset GPU to ensure it is in a ++ * consistent state */ ++ kbase_pm_init_hw(kbdev, PM_ENABLE_IRQS); ++ } ++ ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_ctx_sched_restore_all_as(kbdev); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ ++ /* Lastly, enable the interrupts */ ++ kbase_pm_enable_interrupts(kbdev); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_clock_on); ++ ++bool kbase_pm_clock_off(struct kbase_device *kbdev, bool is_suspend) ++{ ++ unsigned long flags; ++ ++ KBASE_DEBUG_ASSERT(NULL != kbdev); ++ lockdep_assert_held(&kbdev->pm.lock); ++ ++ /* ASSERT that the cores should now be unavailable. No lock needed. */ ++ KBASE_DEBUG_ASSERT(kbdev->shader_available_bitmap == 0u); ++ ++ kbdev->poweroff_pending = true; ++ ++ if (!kbdev->pm.backend.gpu_powered) { ++ /* Already turned off */ ++ if (is_suspend && kbdev->pm.backend.callback_power_suspend) ++ kbdev->pm.backend.callback_power_suspend(kbdev); ++ return true; ++ } ++ ++ KBASE_TRACE_ADD(kbdev, PM_GPU_OFF, NULL, NULL, 0u, 0u); ++ ++ /* Disable interrupts. This also clears any outstanding interrupts */ ++ kbase_pm_disable_interrupts(kbdev); ++ /* Ensure that any IRQ handlers have finished */ ++ kbase_synchronize_irqs(kbdev); ++ ++ spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags); ++ ++ if (atomic_read(&kbdev->faults_pending)) { ++ /* Page/bus faults are still being processed. The GPU can not ++ * be powered off until they have completed */ ++ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, ++ flags); ++ return false; ++ } ++ ++ kbase_pm_cache_snoop_disable(kbdev); ++ ++ /* The GPU power may be turned off from this point */ ++ kbdev->pm.backend.gpu_powered = false; ++ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags); ++ ++ if (is_suspend && kbdev->pm.backend.callback_power_suspend) ++ kbdev->pm.backend.callback_power_suspend(kbdev); ++ else if (kbdev->pm.backend.callback_power_off) ++ kbdev->pm.backend.callback_power_off(kbdev); ++ return true; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_clock_off); ++ ++struct kbasep_reset_timeout_data { ++ struct hrtimer timer; ++ bool timed_out; ++ struct kbase_device *kbdev; ++}; ++ ++void kbase_pm_reset_done(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ kbdev->pm.backend.reset_done = true; ++ wake_up(&kbdev->pm.backend.reset_done_wait); ++} ++ ++/** ++ * kbase_pm_wait_for_reset - Wait for a reset to happen ++ * ++ * Wait for the %RESET_COMPLETED IRQ to occur, then reset the waiting state. ++ * ++ * @kbdev: Kbase device ++ */ ++static void kbase_pm_wait_for_reset(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->pm.lock); ++ ++ wait_event(kbdev->pm.backend.reset_done_wait, ++ (kbdev->pm.backend.reset_done)); ++ kbdev->pm.backend.reset_done = false; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_reset_done); ++ ++static enum hrtimer_restart kbasep_reset_timeout(struct hrtimer *timer) ++{ ++ struct kbasep_reset_timeout_data *rtdata = ++ container_of(timer, struct kbasep_reset_timeout_data, timer); ++ ++ rtdata->timed_out = 1; ++ ++ /* Set the wait queue to wake up kbase_pm_init_hw even though the reset ++ * hasn't completed */ ++ kbase_pm_reset_done(rtdata->kbdev); ++ ++ return HRTIMER_NORESTART; ++} ++ ++static void kbase_pm_hw_issues_detect(struct kbase_device *kbdev) ++{ ++ struct device_node *np = kbdev->dev->of_node; ++ u32 jm_values[4]; ++ const u32 gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; ++ const u32 prod_id = (gpu_id & GPU_ID_VERSION_PRODUCT_ID) >> ++ GPU_ID_VERSION_PRODUCT_ID_SHIFT; ++ const u32 major = (gpu_id & GPU_ID_VERSION_MAJOR) >> ++ GPU_ID_VERSION_MAJOR_SHIFT; ++ ++ kbdev->hw_quirks_sc = 0; ++ ++ /* Needed due to MIDBASE-1494: LS_PAUSEBUFFER_DISABLE. See PRLAM-8443. ++ * and needed due to MIDGLES-3539. See PRLAM-11035 */ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8443) || ++ kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_11035)) ++ kbdev->hw_quirks_sc |= SC_LS_PAUSEBUFFER_DISABLE; ++ ++ /* Needed due to MIDBASE-2054: SDC_DISABLE_OQ_DISCARD. See PRLAM-10327. ++ */ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10327)) ++ kbdev->hw_quirks_sc |= SC_SDC_DISABLE_OQ_DISCARD; ++ ++#ifdef CONFIG_MALI_BIFROST_PRFCNT_SET_SECONDARY ++ /* Enable alternative hardware counter selection if configured. */ ++ if (!GPU_ID_IS_NEW_FORMAT(prod_id)) ++ kbdev->hw_quirks_sc |= SC_ALT_COUNTERS; ++#endif ++ ++ /* Needed due to MIDBASE-2795. ENABLE_TEXGRD_FLAGS. See PRLAM-10797. */ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10797)) ++ kbdev->hw_quirks_sc |= SC_ENABLE_TEXGRD_FLAGS; ++ ++ if (!kbase_hw_has_issue(kbdev, GPUCORE_1619)) { ++ if (prod_id < 0x750 || prod_id == 0x6956) /* T60x, T62x, T72x */ ++ kbdev->hw_quirks_sc |= SC_LS_ATTR_CHECK_DISABLE; ++ else if (prod_id >= 0x750 && prod_id <= 0x880) /* T76x, T8xx */ ++ kbdev->hw_quirks_sc |= SC_LS_ALLOW_ATTR_TYPES; ++ } ++ ++ if (!kbdev->hw_quirks_sc) ++ kbdev->hw_quirks_sc = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(SHADER_CONFIG), NULL); ++ ++ kbdev->hw_quirks_tiler = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(TILER_CONFIG), NULL); ++ ++ /* Set tiler clock gate override if required */ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_T76X_3953)) ++ kbdev->hw_quirks_tiler |= TC_CLOCK_GATE_OVERRIDE; ++ ++ /* Limit the GPU bus bandwidth if the platform needs this. */ ++ kbdev->hw_quirks_mmu = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(L2_MMU_CONFIG), NULL); ++ ++ /* Limit read ID width for AXI */ ++ kbdev->hw_quirks_mmu &= ~(L2_MMU_CONFIG_LIMIT_EXTERNAL_READS); ++ kbdev->hw_quirks_mmu |= (DEFAULT_ARID_LIMIT & 0x3) << ++ L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT; ++ ++ /* Limit write ID width for AXI */ ++ kbdev->hw_quirks_mmu &= ~(L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES); ++ kbdev->hw_quirks_mmu |= (DEFAULT_AWID_LIMIT & 0x3) << ++ L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT; ++ ++ if (kbdev->system_coherency == COHERENCY_ACE) { ++ /* Allow memory configuration disparity to be ignored, we ++ * optimize the use of shared memory and thus we expect ++ * some disparity in the memory configuration */ ++ kbdev->hw_quirks_mmu |= L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY; ++ } ++ ++ kbdev->hw_quirks_jm = 0; ++ /* Only for T86x/T88x-based products after r2p0 */ ++ if (prod_id >= 0x860 && prod_id <= 0x880 && major >= 2) { ++ ++ if (of_property_read_u32_array(np, ++ "jm_config", ++ &jm_values[0], ++ ARRAY_SIZE(jm_values))) { ++ /* Entry not in device tree, use defaults */ ++ jm_values[0] = 0; ++ jm_values[1] = 0; ++ jm_values[2] = 0; ++ jm_values[3] = JM_MAX_JOB_THROTTLE_LIMIT; ++ } ++ ++ /* Limit throttle limit to 6 bits*/ ++ if (jm_values[3] > JM_MAX_JOB_THROTTLE_LIMIT) { ++ dev_dbg(kbdev->dev, "JOB_THROTTLE_LIMIT supplied in device tree is too large. Limiting to MAX (63)."); ++ jm_values[3] = JM_MAX_JOB_THROTTLE_LIMIT; ++ } ++ ++ /* Aggregate to one integer. */ ++ kbdev->hw_quirks_jm |= (jm_values[0] ? ++ JM_TIMESTAMP_OVERRIDE : 0); ++ kbdev->hw_quirks_jm |= (jm_values[1] ? ++ JM_CLOCK_GATE_OVERRIDE : 0); ++ kbdev->hw_quirks_jm |= (jm_values[2] ? ++ JM_JOB_THROTTLE_ENABLE : 0); ++ kbdev->hw_quirks_jm |= (jm_values[3] << ++ JM_JOB_THROTTLE_LIMIT_SHIFT); ++ ++ } else if (GPU_ID_IS_NEW_FORMAT(prod_id) && ++ (GPU_ID2_MODEL_MATCH_VALUE(prod_id) == ++ GPU_ID2_PRODUCT_TMIX)) { ++ /* Only for tMIx */ ++ u32 coherency_features; ++ ++ coherency_features = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(COHERENCY_FEATURES), NULL); ++ ++ /* (COHERENCY_ACE_LITE | COHERENCY_ACE) was incorrectly ++ * documented for tMIx so force correct value here. ++ */ ++ if (coherency_features == ++ COHERENCY_FEATURE_BIT(COHERENCY_ACE)) { ++ kbdev->hw_quirks_jm |= ++ (COHERENCY_ACE_LITE | COHERENCY_ACE) << ++ JM_FORCE_COHERENCY_FEATURES_SHIFT; ++ } ++ } ++ ++ ++ if (!kbdev->hw_quirks_jm) ++ kbdev->hw_quirks_jm = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(JM_CONFIG), NULL); ++ ++#ifdef CONFIG_MALI_CORESTACK ++#define MANUAL_POWER_CONTROL ((u32)(1 << 8)) ++ kbdev->hw_quirks_jm |= MANUAL_POWER_CONTROL; ++#endif /* CONFIG_MALI_CORESTACK */ ++} ++ ++static void kbase_pm_hw_issues_apply(struct kbase_device *kbdev) ++{ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(SHADER_CONFIG), ++ kbdev->hw_quirks_sc, NULL); ++ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(TILER_CONFIG), ++ kbdev->hw_quirks_tiler, NULL); ++ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(L2_MMU_CONFIG), ++ kbdev->hw_quirks_mmu, NULL); ++ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(JM_CONFIG), ++ kbdev->hw_quirks_jm, NULL); ++ ++} ++ ++void kbase_pm_cache_snoop_enable(struct kbase_device *kbdev) ++{ ++ if ((kbdev->current_gpu_coherency_mode == COHERENCY_ACE) && ++ !kbdev->cci_snoop_enabled) { ++#ifdef CONFIG_ARM64 ++ if (kbdev->snoop_enable_smc != 0) ++ kbase_invoke_smc_fid(kbdev->snoop_enable_smc, 0, 0, 0); ++#endif /* CONFIG_ARM64 */ ++ dev_dbg(kbdev->dev, "MALI - CCI Snoops - Enabled\n"); ++ kbdev->cci_snoop_enabled = true; ++ } ++} ++ ++void kbase_pm_cache_snoop_disable(struct kbase_device *kbdev) ++{ ++ if (kbdev->cci_snoop_enabled) { ++#ifdef CONFIG_ARM64 ++ if (kbdev->snoop_disable_smc != 0) { ++ mali_cci_flush_l2(kbdev); ++ kbase_invoke_smc_fid(kbdev->snoop_disable_smc, 0, 0, 0); ++ } ++#endif /* CONFIG_ARM64 */ ++ dev_dbg(kbdev->dev, "MALI - CCI Snoops Disabled\n"); ++ kbdev->cci_snoop_enabled = false; ++ } ++} ++ ++static int kbase_pm_do_reset(struct kbase_device *kbdev) ++{ ++ struct kbasep_reset_timeout_data rtdata; ++ ++ KBASE_TRACE_ADD(kbdev, CORE_GPU_SOFT_RESET, NULL, NULL, 0u, 0); ++ ++ KBASE_TLSTREAM_JD_GPU_SOFT_RESET(kbdev); ++ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), ++ GPU_COMMAND_SOFT_RESET, NULL); ++ ++ /* Unmask the reset complete interrupt only */ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), RESET_COMPLETED, ++ NULL); ++ ++ /* Initialize a structure for tracking the status of the reset */ ++ rtdata.kbdev = kbdev; ++ rtdata.timed_out = 0; ++ ++ /* Create a timer to use as a timeout on the reset */ ++ hrtimer_init_on_stack(&rtdata.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); ++ rtdata.timer.function = kbasep_reset_timeout; ++ ++ hrtimer_start(&rtdata.timer, HR_TIMER_DELAY_MSEC(RESET_TIMEOUT), ++ HRTIMER_MODE_REL); ++ ++ /* Wait for the RESET_COMPLETED interrupt to be raised */ ++ kbase_pm_wait_for_reset(kbdev); ++ ++ if (rtdata.timed_out == 0) { ++ /* GPU has been reset */ ++ hrtimer_cancel(&rtdata.timer); ++ destroy_hrtimer_on_stack(&rtdata.timer); ++ return 0; ++ } ++ ++ /* No interrupt has been received - check if the RAWSTAT register says ++ * the reset has completed */ ++ if (kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), NULL) & ++ RESET_COMPLETED) { ++ /* The interrupt is set in the RAWSTAT; this suggests that the ++ * interrupts are not getting to the CPU */ ++ dev_err(kbdev->dev, "Reset interrupt didn't reach CPU. Check interrupt assignments.\n"); ++ /* If interrupts aren't working we can't continue. */ ++ destroy_hrtimer_on_stack(&rtdata.timer); ++ return -EINVAL; ++ } ++ ++ /* The GPU doesn't seem to be responding to the reset so try a hard ++ * reset */ ++ dev_err(kbdev->dev, "Failed to soft-reset GPU (timed out after %d ms), now attempting a hard reset\n", ++ RESET_TIMEOUT); ++ KBASE_TRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, NULL, 0u, 0); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), ++ GPU_COMMAND_HARD_RESET, NULL); ++ ++ /* Restart the timer to wait for the hard reset to complete */ ++ rtdata.timed_out = 0; ++ ++ hrtimer_start(&rtdata.timer, HR_TIMER_DELAY_MSEC(RESET_TIMEOUT), ++ HRTIMER_MODE_REL); ++ ++ /* Wait for the RESET_COMPLETED interrupt to be raised */ ++ kbase_pm_wait_for_reset(kbdev); ++ ++ if (rtdata.timed_out == 0) { ++ /* GPU has been reset */ ++ hrtimer_cancel(&rtdata.timer); ++ destroy_hrtimer_on_stack(&rtdata.timer); ++ return 0; ++ } ++ ++ destroy_hrtimer_on_stack(&rtdata.timer); ++ ++ dev_err(kbdev->dev, "Failed to hard-reset the GPU (timed out after %d ms)\n", ++ RESET_TIMEOUT); ++ ++ return -EINVAL; ++} ++ ++static int kbasep_protected_mode_enable(struct protected_mode_device *pdev) ++{ ++ struct kbase_device *kbdev = pdev->data; ++ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), ++ GPU_COMMAND_SET_PROTECTED_MODE, NULL); ++ return 0; ++} ++ ++static int kbasep_protected_mode_disable(struct protected_mode_device *pdev) ++{ ++ struct kbase_device *kbdev = pdev->data; ++ ++ lockdep_assert_held(&kbdev->pm.lock); ++ ++ return kbase_pm_do_reset(kbdev); ++} ++ ++struct protected_mode_ops kbase_native_protected_ops = { ++ .protected_mode_enable = kbasep_protected_mode_enable, ++ .protected_mode_disable = kbasep_protected_mode_disable ++}; ++ ++int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags) ++{ ++ unsigned long irq_flags; ++ int err; ++ bool resume_vinstr = false; ++ ++ KBASE_DEBUG_ASSERT(NULL != kbdev); ++ lockdep_assert_held(&kbdev->pm.lock); ++ ++ /* Ensure the clock is on before attempting to access the hardware */ ++ if (!kbdev->pm.backend.gpu_powered) { ++ if (kbdev->pm.backend.callback_power_on) ++ kbdev->pm.backend.callback_power_on(kbdev); ++ ++ spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, ++ irq_flags); ++ kbdev->pm.backend.gpu_powered = true; ++ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, ++ irq_flags); ++ } ++ ++ /* Ensure interrupts are off to begin with, this also clears any ++ * outstanding interrupts */ ++ kbase_pm_disable_interrupts(kbdev); ++ /* Ensure cache snoops are disabled before reset. */ ++ kbase_pm_cache_snoop_disable(kbdev); ++ /* Prepare for the soft-reset */ ++ kbdev->pm.backend.reset_done = false; ++ ++ /* The cores should be made unavailable due to the reset */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); ++ if (kbdev->shader_available_bitmap != 0u) ++ KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, ++ NULL, 0u, (u32)0u); ++ if (kbdev->tiler_available_bitmap != 0u) ++ KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER, ++ NULL, NULL, 0u, (u32)0u); ++ kbdev->shader_available_bitmap = 0u; ++ kbdev->tiler_available_bitmap = 0u; ++ kbdev->l2_available_bitmap = 0u; ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); ++ ++ /* Soft reset the GPU */ ++ if (kbdev->protected_mode_support) ++ err = kbdev->protected_ops->protected_mode_disable( ++ kbdev->protected_dev); ++ else ++ err = kbase_pm_do_reset(kbdev); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); ++ if (kbdev->protected_mode) ++ resume_vinstr = true; ++ kbdev->protected_mode = false; ++ kbase_ipa_model_use_configured_locked(kbdev); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); ++ ++ if (err) ++ goto exit; ++ ++ if (flags & PM_HW_ISSUES_DETECT) ++ kbase_pm_hw_issues_detect(kbdev); ++ ++ kbase_pm_hw_issues_apply(kbdev); ++ kbase_cache_set_coherency_mode(kbdev, kbdev->system_coherency); ++ ++ /* Sanity check protected mode was left after reset */ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_MODE)) { ++ u32 gpu_status = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(GPU_STATUS), NULL); ++ ++ WARN_ON(gpu_status & GPU_STATUS_PROTECTED_MODE_ACTIVE); ++ } ++ ++ /* If cycle counter was in use re-enable it, enable_irqs will only be ++ * false when called from kbase_pm_powerup */ ++ if (kbdev->pm.backend.gpu_cycle_counter_requests && ++ (flags & PM_ENABLE_IRQS)) { ++ /* enable interrupts as the L2 may have to be powered on */ ++ kbase_pm_enable_interrupts(kbdev); ++ kbase_pm_request_l2_caches(kbdev); ++ ++ /* Re-enable the counters if we need to */ ++ spin_lock_irqsave( ++ &kbdev->pm.backend.gpu_cycle_counter_requests_lock, ++ irq_flags); ++ if (kbdev->pm.backend.gpu_cycle_counter_requests) ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), ++ GPU_COMMAND_CYCLE_COUNT_START, NULL); ++ spin_unlock_irqrestore( ++ &kbdev->pm.backend.gpu_cycle_counter_requests_lock, ++ irq_flags); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); ++ kbase_pm_release_l2_caches(kbdev); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); ++ ++ kbase_pm_disable_interrupts(kbdev); ++ } ++ ++ if (flags & PM_ENABLE_IRQS) ++ kbase_pm_enable_interrupts(kbdev); ++ ++exit: ++ /* If GPU is leaving protected mode resume vinstr operation. */ ++ if (kbdev->vinstr_ctx && resume_vinstr) ++ kbase_vinstr_resume(kbdev->vinstr_ctx); ++ ++ return err; ++} ++ ++/** ++ * kbase_pm_request_gpu_cycle_counter_do_request - Request cycle counters ++ * ++ * Increase the count of cycle counter users and turn the cycle counters on if ++ * they were previously off ++ * ++ * This function is designed to be called by ++ * kbase_pm_request_gpu_cycle_counter() or ++ * kbase_pm_request_gpu_cycle_counter_l2_is_on() only ++ * ++ * When this function is called the l2 cache must be on and the l2 cache users ++ * count must have been incremented by a call to ( ++ * kbase_pm_request_l2_caches() or kbase_pm_request_l2_caches_l2_on() ) ++ * ++ * @kbdev: The kbase device structure of the device ++ */ ++static void ++kbase_pm_request_gpu_cycle_counter_do_request(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->pm.backend.gpu_cycle_counter_requests_lock, ++ flags); ++ ++ ++kbdev->pm.backend.gpu_cycle_counter_requests; ++ ++ if (1 == kbdev->pm.backend.gpu_cycle_counter_requests) ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), ++ GPU_COMMAND_CYCLE_COUNT_START, NULL); ++ ++ spin_unlock_irqrestore( ++ &kbdev->pm.backend.gpu_cycle_counter_requests_lock, ++ flags); ++} ++ ++void kbase_pm_request_gpu_cycle_counter(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered); ++ ++ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests < ++ INT_MAX); ++ ++ kbase_pm_request_l2_caches(kbdev); ++ ++ kbase_pm_request_gpu_cycle_counter_do_request(kbdev); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_request_gpu_cycle_counter); ++ ++void kbase_pm_request_gpu_cycle_counter_l2_is_on(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered); ++ ++ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests < ++ INT_MAX); ++ ++ kbase_pm_request_l2_caches_l2_is_on(kbdev); ++ ++ kbase_pm_request_gpu_cycle_counter_do_request(kbdev); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_request_gpu_cycle_counter_l2_is_on); ++ ++void kbase_pm_release_gpu_cycle_counter_nolock(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ spin_lock_irqsave(&kbdev->pm.backend.gpu_cycle_counter_requests_lock, ++ flags); ++ ++ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests > 0); ++ ++ --kbdev->pm.backend.gpu_cycle_counter_requests; ++ ++ if (0 == kbdev->pm.backend.gpu_cycle_counter_requests) ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), ++ GPU_COMMAND_CYCLE_COUNT_STOP, NULL); ++ ++ spin_unlock_irqrestore( ++ &kbdev->pm.backend.gpu_cycle_counter_requests_lock, ++ flags); ++ ++ kbase_pm_release_l2_caches(kbdev); ++} ++ ++void kbase_pm_release_gpu_cycle_counter(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ kbase_pm_release_gpu_cycle_counter_nolock(kbdev); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_release_gpu_cycle_counter); +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_internal.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_internal.h +new file mode 100755 +index 000000000000..9fbe094541c5 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_internal.h +@@ -0,0 +1,548 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * Power management API definitions used internally by GPU backend ++ */ ++ ++#ifndef _KBASE_BACKEND_PM_INTERNAL_H_ ++#define _KBASE_BACKEND_PM_INTERNAL_H_ ++ ++#include ++ ++#include "mali_kbase_pm_ca.h" ++#include "mali_kbase_pm_policy.h" ++ ++ ++/** ++ * kbase_pm_dev_idle - The GPU is idle. ++ * ++ * The OS may choose to turn off idle devices ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_dev_idle(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_dev_activate - The GPU is active. ++ * ++ * The OS should avoid opportunistically turning off the GPU while it is active ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_dev_activate(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_get_present_cores - Get details of the cores that are present in ++ * the device. ++ * ++ * This function can be called by the active power policy to return a bitmask of ++ * the cores (of a specified type) present in the GPU device and also a count of ++ * the number of cores. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid ++ * pointer) ++ * @type: The type of core (see the enum kbase_pm_core_type enumeration) ++ * ++ * Return: The bit mask of cores present ++ */ ++u64 kbase_pm_get_present_cores(struct kbase_device *kbdev, ++ enum kbase_pm_core_type type); ++ ++/** ++ * kbase_pm_get_active_cores - Get details of the cores that are currently ++ * active in the device. ++ * ++ * This function can be called by the active power policy to return a bitmask of ++ * the cores (of a specified type) that are actively processing work (i.e. ++ * turned on *and* busy). ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * @type: The type of core (see the enum kbase_pm_core_type enumeration) ++ * ++ * Return: The bit mask of active cores ++ */ ++u64 kbase_pm_get_active_cores(struct kbase_device *kbdev, ++ enum kbase_pm_core_type type); ++ ++/** ++ * kbase_pm_get_trans_cores - Get details of the cores that are currently ++ * transitioning between power states. ++ * ++ * This function can be called by the active power policy to return a bitmask of ++ * the cores (of a specified type) that are currently transitioning between ++ * power states. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * @type: The type of core (see the enum kbase_pm_core_type enumeration) ++ * ++ * Return: The bit mask of transitioning cores ++ */ ++u64 kbase_pm_get_trans_cores(struct kbase_device *kbdev, ++ enum kbase_pm_core_type type); ++ ++/** ++ * kbase_pm_get_ready_cores - Get details of the cores that are currently ++ * powered and ready for jobs. ++ * ++ * This function can be called by the active power policy to return a bitmask of ++ * the cores (of a specified type) that are powered and ready for jobs (they may ++ * or may not be currently executing jobs). ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * @type: The type of core (see the enum kbase_pm_core_type enumeration) ++ * ++ * Return: The bit mask of ready cores ++ */ ++u64 kbase_pm_get_ready_cores(struct kbase_device *kbdev, ++ enum kbase_pm_core_type type); ++ ++/** ++ * kbase_pm_clock_on - Turn the clock for the device on, and enable device ++ * interrupts. ++ * ++ * This function can be used by a power policy to turn the clock for the GPU on. ++ * It should be modified during integration to perform the necessary actions to ++ * ensure that the GPU is fully powered and clocked. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid ++ * pointer) ++ * @is_resume: true if clock on due to resume after suspend, false otherwise ++ */ ++void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume); ++ ++/** ++ * kbase_pm_clock_off - Disable device interrupts, and turn the clock for the ++ * device off. ++ * ++ * This function can be used by a power policy to turn the clock for the GPU ++ * off. It should be modified during integration to perform the necessary ++ * actions to turn the clock off (if this is possible in the integration). ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid ++ * pointer) ++ * @is_suspend: true if clock off due to suspend, false otherwise ++ * ++ * Return: true if clock was turned off, or ++ * false if clock can not be turned off due to pending page/bus fault ++ * workers. Caller must flush MMU workqueues and retry ++ */ ++bool kbase_pm_clock_off(struct kbase_device *kbdev, bool is_suspend); ++ ++/** ++ * kbase_pm_enable_interrupts - Enable interrupts on the device. ++ * ++ * Interrupts are also enabled after a call to kbase_pm_clock_on(). ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_enable_interrupts(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_disable_interrupts - Disable interrupts on the device. ++ * ++ * This prevents delivery of Power Management interrupts to the CPU so that ++ * kbase_pm_check_transitions_nolock() will not be called from the IRQ handler ++ * until kbase_pm_enable_interrupts() or kbase_pm_clock_on() is called. ++ * ++ * Interrupts are also disabled after a call to kbase_pm_clock_off(). ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_disable_interrupts(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_disable_interrupts_nolock - Version of kbase_pm_disable_interrupts() ++ * that does not take the hwaccess_lock ++ * ++ * Caller must hold the hwaccess_lock. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_disable_interrupts_nolock(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_init_hw - Initialize the hardware. ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * @flags: Flags specifying the type of PM init ++ * ++ * This function checks the GPU ID register to ensure that the GPU is supported ++ * by the driver and performs a reset on the device so that it is in a known ++ * state before the device is used. ++ * ++ * Return: 0 if the device is supported and successfully reset. ++ */ ++int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags); ++ ++/** ++ * kbase_pm_reset_done - The GPU has been reset successfully. ++ * ++ * This function must be called by the GPU interrupt handler when the ++ * RESET_COMPLETED bit is set. It signals to the power management initialization ++ * code that the GPU has been successfully reset. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_reset_done(struct kbase_device *kbdev); ++ ++ ++/** ++ * kbase_pm_check_transitions_nolock - Check if there are any power transitions ++ * to make, and if so start them. ++ * ++ * This function will check the desired_xx_state members of ++ * struct kbase_pm_device_data and the actual status of the hardware to see if ++ * any power transitions can be made at this time to make the hardware state ++ * closer to the state desired by the power policy. ++ * ++ * The return value can be used to check whether all the desired cores are ++ * available, and so whether it's worth submitting a job (e.g. from a Power ++ * Management IRQ). ++ * ++ * Note that this still returns true when desired_xx_state has no ++ * cores. That is: of the no cores desired, none were *un*available. In ++ * this case, the caller may still need to try submitting jobs. This is because ++ * the Core Availability Policy might have taken us to an intermediate state ++ * where no cores are powered, before powering on more cores (e.g. for core ++ * rotation) ++ * ++ * The caller must hold kbase_device.pm.power_change_lock ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * Return: non-zero when all desired cores are available. That is, ++ * it's worthwhile for the caller to submit a job. ++ * false otherwise ++ */ ++bool kbase_pm_check_transitions_nolock(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_check_transitions_sync - Synchronous and locking variant of ++ * kbase_pm_check_transitions_nolock() ++ * ++ * On returning, the desired state at the time of the call will have been met. ++ * ++ * There is nothing to stop the core being switched off by calls to ++ * kbase_pm_release_cores() or kbase_pm_unrequest_cores(). Therefore, the ++ * caller must have already made a call to ++ * kbase_pm_request_cores()/kbase_pm_request_cores_sync() previously. ++ * ++ * The usual use-case for this is to ensure cores are 'READY' after performing ++ * a GPU Reset. ++ * ++ * Unlike kbase_pm_check_transitions_nolock(), the caller must not hold ++ * kbase_device.pm.power_change_lock, because this function will take that ++ * lock itself. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_check_transitions_sync(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_update_cores_state_nolock - Variant of kbase_pm_update_cores_state() ++ * where the caller must hold ++ * kbase_device.pm.power_change_lock ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_update_cores_state_nolock(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_update_cores_state - Update the desired state of shader cores from ++ * the Power Policy, and begin any power ++ * transitions. ++ * ++ * This function will update the desired_xx_state members of ++ * struct kbase_pm_device_data by calling into the current Power Policy. It will ++ * then begin power transitions to make the hardware acheive the desired shader ++ * core state. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_update_cores_state(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_cancel_deferred_poweroff - Cancel any pending requests to power off ++ * the GPU and/or shader cores. ++ * ++ * This should be called by any functions which directly power off the GPU. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_cancel_deferred_poweroff(struct kbase_device *kbdev); ++ ++/** ++ * kbasep_pm_init_core_use_bitmaps - Initialise data tracking the required ++ * and used cores. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbasep_pm_init_core_use_bitmaps(struct kbase_device *kbdev); ++ ++/** ++ * kbasep_pm_metrics_init - Initialize the metrics gathering framework. ++ * ++ * This must be called before other metric gathering APIs are called. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * Return: 0 on success, error code on error ++ */ ++int kbasep_pm_metrics_init(struct kbase_device *kbdev); ++ ++/** ++ * kbasep_pm_metrics_term - Terminate the metrics gathering framework. ++ * ++ * This must be called when metric gathering is no longer required. It is an ++ * error to call any metrics gathering function (other than ++ * kbasep_pm_metrics_init()) after calling this function. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbasep_pm_metrics_term(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_report_vsync - Function to be called by the frame buffer driver to ++ * update the vsync metric. ++ * ++ * This function should be called by the frame buffer driver to update whether ++ * the system is hitting the vsync target or not. buffer_updated should be true ++ * if the vsync corresponded with a new frame being displayed, otherwise it ++ * should be false. This function does not need to be called every vsync, but ++ * only when the value of @buffer_updated differs from a previous call. ++ * ++ * @kbdev: The kbase device structure for the device (must be a ++ * valid pointer) ++ * @buffer_updated: True if the buffer has been updated on this VSync, ++ * false otherwise ++ */ ++void kbase_pm_report_vsync(struct kbase_device *kbdev, int buffer_updated); ++ ++/** ++ * kbase_pm_get_dvfs_action - Determine whether the DVFS system should change ++ * the clock speed of the GPU. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * This function should be called regularly by the DVFS system to check whether ++ * the clock speed of the GPU needs updating. ++ */ ++void kbase_pm_get_dvfs_action(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_request_gpu_cycle_counter - Mark that the GPU cycle counter is ++ * needed ++ * ++ * If the caller is the first caller then the GPU cycle counters will be enabled ++ * along with the l2 cache ++ * ++ * The GPU must be powered when calling this function (i.e. ++ * kbase_pm_context_active() must have been called). ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_request_gpu_cycle_counter(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_request_gpu_cycle_counter_l2_is_on - Mark GPU cycle counter is ++ * needed (l2 cache already on) ++ * ++ * This is a version of the above function ++ * (kbase_pm_request_gpu_cycle_counter()) suitable for being called when the ++ * l2 cache is known to be on and assured to be on until the subsequent call of ++ * kbase_pm_release_gpu_cycle_counter() such as when a job is submitted. It does ++ * not sleep and can be called from atomic functions. ++ * ++ * The GPU must be powered when calling this function (i.e. ++ * kbase_pm_context_active() must have been called) and the l2 cache must be ++ * powered on. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_request_gpu_cycle_counter_l2_is_on(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_release_gpu_cycle_counter - Mark that the GPU cycle counter is no ++ * longer in use ++ * ++ * If the caller is the last caller then the GPU cycle counters will be ++ * disabled. A request must have been made before a call to this. ++ * ++ * Caller must not hold the hwaccess_lock, as it will be taken in this function. ++ * If the caller is already holding this lock then ++ * kbase_pm_release_gpu_cycle_counter_nolock() must be used instead. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_release_gpu_cycle_counter(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_release_gpu_cycle_counter_nolock - Version of kbase_pm_release_gpu_cycle_counter() ++ * that does not take hwaccess_lock ++ * ++ * Caller must hold the hwaccess_lock. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_release_gpu_cycle_counter_nolock(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_wait_for_poweroff_complete - Wait for the poweroff workqueue to ++ * complete ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_wait_for_poweroff_complete(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_register_access_enable - Enable access to GPU registers ++ * ++ * Enables access to the GPU registers before power management has powered up ++ * the GPU with kbase_pm_powerup(). ++ * ++ * Access to registers should be done using kbase_os_reg_read()/write() at this ++ * stage, not kbase_reg_read()/write(). ++ * ++ * This results in the power management callbacks provided in the driver ++ * configuration to get called to turn on power and/or clocks to the GPU. See ++ * kbase_pm_callback_conf. ++ * ++ * This should only be used before power management is powered up with ++ * kbase_pm_powerup() ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_register_access_enable(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_register_access_disable - Disable early register access ++ * ++ * Disables access to the GPU registers enabled earlier by a call to ++ * kbase_pm_register_access_enable(). ++ * ++ * This results in the power management callbacks provided in the driver ++ * configuration to get called to turn off power and/or clocks to the GPU. See ++ * kbase_pm_callback_conf ++ * ++ * This should only be used before power management is powered up with ++ * kbase_pm_powerup() ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_register_access_disable(struct kbase_device *kbdev); ++ ++/* NOTE: kbase_pm_is_suspending is in mali_kbase.h, because it is an inline ++ * function */ ++ ++/** ++ * kbase_pm_metrics_is_active - Check if the power management metrics ++ * collection is active. ++ * ++ * Note that this returns if the power management metrics collection was ++ * active at the time of calling, it is possible that after the call the metrics ++ * collection enable may have changed state. ++ * ++ * The caller must handle the consequence that the state may have changed. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * Return: true if metrics collection was active else false. ++ */ ++bool kbase_pm_metrics_is_active(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_do_poweron - Power on the GPU, and any cores that are requested. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid ++ * pointer) ++ * @is_resume: true if power on due to resume after suspend, ++ * false otherwise ++ */ ++void kbase_pm_do_poweron(struct kbase_device *kbdev, bool is_resume); ++ ++/** ++ * kbase_pm_do_poweroff - Power off the GPU, and any cores that have been ++ * requested. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid ++ * pointer) ++ * @is_suspend: true if power off due to suspend, ++ * false otherwise ++ */ ++void kbase_pm_do_poweroff(struct kbase_device *kbdev, bool is_suspend); ++ ++#if defined(CONFIG_MALI_BIFROST_DEVFREQ) || defined(CONFIG_MALI_BIFROST_DVFS) ++void kbase_pm_get_dvfs_utilisation(struct kbase_device *kbdev, ++ unsigned long *total, unsigned long *busy); ++void kbase_pm_reset_dvfs_utilisation(struct kbase_device *kbdev); ++#endif /* defined(CONFIG_MALI_BIFROST_DEVFREQ) || defined(CONFIG_MALI_BIFROST_DVFS) */ ++ ++#ifdef CONFIG_MALI_BIFROST_DVFS ++ ++/** ++ * kbase_platform_dvfs_event - Report utilisation to DVFS code ++ * ++ * Function provided by platform specific code when DVFS is enabled to allow ++ * the power management metrics system to report utilisation. ++ * ++ * @kbdev: The kbase device structure for the device (must be a ++ * valid pointer) ++ * @utilisation: The current calculated utilisation by the metrics system. ++ * @util_gl_share: The current calculated gl share of utilisation. ++ * @util_cl_share: The current calculated cl share of utilisation per core ++ * group. ++ * Return: Returns 0 on failure and non zero on success. ++ */ ++ ++int kbase_platform_dvfs_event(struct kbase_device *kbdev, u32 utilisation, ++ u32 util_gl_share, u32 util_cl_share[2]); ++#endif ++ ++void kbase_pm_power_changed(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_metrics_update - Inform the metrics system that an atom is either ++ * about to be run or has just completed. ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * @now: Pointer to the timestamp of the change, or NULL to use current time ++ * ++ * Caller must hold hwaccess_lock ++ */ ++void kbase_pm_metrics_update(struct kbase_device *kbdev, ++ ktime_t *now); ++ ++/** ++ * kbase_pm_cache_snoop_enable - Allow CPU snoops on the GPU ++ * If the GPU does not have coherency this is a no-op ++ * @kbdev: Device pointer ++ * ++ * This function should be called after L2 power up. ++ */ ++ ++void kbase_pm_cache_snoop_enable(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_cache_snoop_disable - Prevent CPU snoops on the GPU ++ * If the GPU does not have coherency this is a no-op ++ * @kbdev: Device pointer ++ * ++ * This function should be called before L2 power off. ++ */ ++void kbase_pm_cache_snoop_disable(struct kbase_device *kbdev); ++ ++#endif /* _KBASE_BACKEND_PM_INTERNAL_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_metrics.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_metrics.c +new file mode 100755 +index 000000000000..ba13bcd8b291 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_metrics.c +@@ -0,0 +1,401 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * Metrics for power management ++ */ ++ ++#include ++#include ++#include ++#include ++ ++/* When VSync is being hit aim for utilisation between 70-90% */ ++#define KBASE_PM_VSYNC_MIN_UTILISATION 70 ++#define KBASE_PM_VSYNC_MAX_UTILISATION 90 ++/* Otherwise aim for 10-40% */ ++#define KBASE_PM_NO_VSYNC_MIN_UTILISATION 10 ++#define KBASE_PM_NO_VSYNC_MAX_UTILISATION 40 ++ ++/* Shift used for kbasep_pm_metrics_data.time_busy/idle - units of (1 << 8) ns ++ * This gives a maximum period between samples of 2^(32+8)/100 ns = slightly ++ * under 11s. Exceeding this will cause overflow */ ++#define KBASE_PM_TIME_SHIFT 8 ++ ++/* Maximum time between sampling of utilization data, without resetting the ++ * counters. */ ++#define MALI_UTILIZATION_MAX_PERIOD 100000 /* ns = 100ms */ ++ ++#ifdef CONFIG_MALI_BIFROST_DVFS ++static enum hrtimer_restart dvfs_callback(struct hrtimer *timer) ++{ ++ unsigned long flags; ++ struct kbasep_pm_metrics_data *metrics; ++ ++ KBASE_DEBUG_ASSERT(timer != NULL); ++ ++ metrics = container_of(timer, struct kbasep_pm_metrics_data, timer); ++ kbase_pm_get_dvfs_action(metrics->kbdev); ++ ++ spin_lock_irqsave(&metrics->lock, flags); ++ ++ if (metrics->timer_active) ++ hrtimer_start(timer, ++ HR_TIMER_DELAY_MSEC(metrics->kbdev->pm.dvfs_period), ++ HRTIMER_MODE_REL); ++ ++ spin_unlock_irqrestore(&metrics->lock, flags); ++ ++ return HRTIMER_NORESTART; ++} ++#endif /* CONFIG_MALI_BIFROST_DVFS */ ++ ++int kbasep_pm_metrics_init(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ kbdev->pm.backend.metrics.kbdev = kbdev; ++ ++ kbdev->pm.backend.metrics.time_period_start = ktime_get(); ++ kbdev->pm.backend.metrics.time_busy = 0; ++ kbdev->pm.backend.metrics.time_idle = 0; ++ kbdev->pm.backend.metrics.prev_busy = 0; ++ kbdev->pm.backend.metrics.prev_idle = 0; ++ kbdev->pm.backend.metrics.gpu_active = false; ++ kbdev->pm.backend.metrics.active_cl_ctx[0] = 0; ++ kbdev->pm.backend.metrics.active_cl_ctx[1] = 0; ++ kbdev->pm.backend.metrics.active_gl_ctx[0] = 0; ++ kbdev->pm.backend.metrics.active_gl_ctx[1] = 0; ++ kbdev->pm.backend.metrics.busy_cl[0] = 0; ++ kbdev->pm.backend.metrics.busy_cl[1] = 0; ++ kbdev->pm.backend.metrics.busy_gl = 0; ++ ++ spin_lock_init(&kbdev->pm.backend.metrics.lock); ++ ++#ifdef CONFIG_MALI_BIFROST_DVFS ++ kbdev->pm.backend.metrics.timer_active = true; ++ hrtimer_init(&kbdev->pm.backend.metrics.timer, CLOCK_MONOTONIC, ++ HRTIMER_MODE_REL); ++ kbdev->pm.backend.metrics.timer.function = dvfs_callback; ++ ++ hrtimer_start(&kbdev->pm.backend.metrics.timer, ++ HR_TIMER_DELAY_MSEC(kbdev->pm.dvfs_period), ++ HRTIMER_MODE_REL); ++#endif /* CONFIG_MALI_BIFROST_DVFS */ ++ ++ return 0; ++} ++ ++KBASE_EXPORT_TEST_API(kbasep_pm_metrics_init); ++ ++void kbasep_pm_metrics_term(struct kbase_device *kbdev) ++{ ++#ifdef CONFIG_MALI_BIFROST_DVFS ++ unsigned long flags; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); ++ kbdev->pm.backend.metrics.timer_active = false; ++ spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); ++ ++ hrtimer_cancel(&kbdev->pm.backend.metrics.timer); ++#endif /* CONFIG_MALI_BIFROST_DVFS */ ++} ++ ++KBASE_EXPORT_TEST_API(kbasep_pm_metrics_term); ++ ++/* caller needs to hold kbdev->pm.backend.metrics.lock before calling this ++ * function ++ */ ++static void kbase_pm_get_dvfs_utilisation_calc(struct kbase_device *kbdev, ++ ktime_t now) ++{ ++ ktime_t diff; ++ ++ lockdep_assert_held(&kbdev->pm.backend.metrics.lock); ++ ++ diff = ktime_sub(now, kbdev->pm.backend.metrics.time_period_start); ++ if (ktime_to_ns(diff) < 0) ++ return; ++ ++ if (kbdev->pm.backend.metrics.gpu_active) { ++ u32 ns_time = (u32) (ktime_to_ns(diff) >> KBASE_PM_TIME_SHIFT); ++ ++ kbdev->pm.backend.metrics.time_busy += ns_time; ++ if (kbdev->pm.backend.metrics.active_cl_ctx[0]) ++ kbdev->pm.backend.metrics.busy_cl[0] += ns_time; ++ if (kbdev->pm.backend.metrics.active_cl_ctx[1]) ++ kbdev->pm.backend.metrics.busy_cl[1] += ns_time; ++ if (kbdev->pm.backend.metrics.active_gl_ctx[0]) ++ kbdev->pm.backend.metrics.busy_gl += ns_time; ++ if (kbdev->pm.backend.metrics.active_gl_ctx[1]) ++ kbdev->pm.backend.metrics.busy_gl += ns_time; ++ } else { ++ kbdev->pm.backend.metrics.time_idle += (u32) (ktime_to_ns(diff) ++ >> KBASE_PM_TIME_SHIFT); ++ } ++ ++ kbdev->pm.backend.metrics.time_period_start = now; ++} ++ ++#if defined(CONFIG_MALI_BIFROST_DEVFREQ) || defined(CONFIG_MALI_BIFROST_DVFS) ++/* Caller needs to hold kbdev->pm.backend.metrics.lock before calling this ++ * function. ++ */ ++static void kbase_pm_reset_dvfs_utilisation_unlocked(struct kbase_device *kbdev, ++ ktime_t now) ++{ ++ /* Store previous value */ ++ kbdev->pm.backend.metrics.prev_idle = ++ kbdev->pm.backend.metrics.time_idle; ++ kbdev->pm.backend.metrics.prev_busy = ++ kbdev->pm.backend.metrics.time_busy; ++ ++ /* Reset current values */ ++ kbdev->pm.backend.metrics.time_period_start = now; ++ kbdev->pm.backend.metrics.time_idle = 0; ++ kbdev->pm.backend.metrics.time_busy = 0; ++ kbdev->pm.backend.metrics.busy_cl[0] = 0; ++ kbdev->pm.backend.metrics.busy_cl[1] = 0; ++ kbdev->pm.backend.metrics.busy_gl = 0; ++} ++ ++void kbase_pm_reset_dvfs_utilisation(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); ++ kbase_pm_reset_dvfs_utilisation_unlocked(kbdev, ktime_get()); ++ spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); ++} ++ ++void kbase_pm_get_dvfs_utilisation(struct kbase_device *kbdev, ++ unsigned long *total_out, unsigned long *busy_out) ++{ ++ ktime_t now = ktime_get(); ++ unsigned long flags, busy, total; ++ ++ spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); ++ kbase_pm_get_dvfs_utilisation_calc(kbdev, now); ++ ++ busy = kbdev->pm.backend.metrics.time_busy; ++ total = busy + kbdev->pm.backend.metrics.time_idle; ++ ++ /* Reset stats if older than MALI_UTILIZATION_MAX_PERIOD (default ++ * 100ms) */ ++ if (total >= MALI_UTILIZATION_MAX_PERIOD) { ++ kbase_pm_reset_dvfs_utilisation_unlocked(kbdev, now); ++ } else if (total < (MALI_UTILIZATION_MAX_PERIOD / 2)) { ++ total += kbdev->pm.backend.metrics.prev_idle + ++ kbdev->pm.backend.metrics.prev_busy; ++ busy += kbdev->pm.backend.metrics.prev_busy; ++ } ++ ++ *total_out = total; ++ *busy_out = busy; ++ spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); ++} ++#endif ++ ++#ifdef CONFIG_MALI_BIFROST_DVFS ++ ++/* caller needs to hold kbdev->pm.backend.metrics.lock before calling this ++ * function ++ */ ++int kbase_pm_get_dvfs_utilisation_old(struct kbase_device *kbdev, ++ int *util_gl_share, ++ int util_cl_share[2], ++ ktime_t now) ++{ ++ int utilisation; ++ int busy; ++ ++ kbase_pm_get_dvfs_utilisation_calc(kbdev, now); ++ ++ if (kbdev->pm.backend.metrics.time_idle + ++ kbdev->pm.backend.metrics.time_busy == 0) { ++ /* No data - so we return NOP */ ++ utilisation = -1; ++ if (util_gl_share) ++ *util_gl_share = -1; ++ if (util_cl_share) { ++ util_cl_share[0] = -1; ++ util_cl_share[1] = -1; ++ } ++ goto out; ++ } ++ ++ utilisation = (100 * kbdev->pm.backend.metrics.time_busy) / ++ (kbdev->pm.backend.metrics.time_idle + ++ kbdev->pm.backend.metrics.time_busy); ++ ++ busy = kbdev->pm.backend.metrics.busy_gl + ++ kbdev->pm.backend.metrics.busy_cl[0] + ++ kbdev->pm.backend.metrics.busy_cl[1]; ++ ++ if (busy != 0) { ++ if (util_gl_share) ++ *util_gl_share = ++ (100 * kbdev->pm.backend.metrics.busy_gl) / ++ busy; ++ if (util_cl_share) { ++ util_cl_share[0] = ++ (100 * kbdev->pm.backend.metrics.busy_cl[0]) / ++ busy; ++ util_cl_share[1] = ++ (100 * kbdev->pm.backend.metrics.busy_cl[1]) / ++ busy; ++ } ++ } else { ++ if (util_gl_share) ++ *util_gl_share = -1; ++ if (util_cl_share) { ++ util_cl_share[0] = -1; ++ util_cl_share[1] = -1; ++ } ++ } ++ ++out: ++ return utilisation; ++} ++ ++void kbase_pm_get_dvfs_action(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ int utilisation, util_gl_share; ++ int util_cl_share[2]; ++ ktime_t now; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); ++ ++ now = ktime_get(); ++ ++ utilisation = kbase_pm_get_dvfs_utilisation_old(kbdev, &util_gl_share, ++ util_cl_share, now); ++ ++ if (utilisation < 0 || util_gl_share < 0 || util_cl_share[0] < 0 || ++ util_cl_share[1] < 0) { ++ utilisation = 0; ++ util_gl_share = 0; ++ util_cl_share[0] = 0; ++ util_cl_share[1] = 0; ++ goto out; ++ } ++ ++out: ++#ifdef CONFIG_MALI_BIFROST_DVFS ++ kbase_platform_dvfs_event(kbdev, utilisation, util_gl_share, ++ util_cl_share); ++#endif /*CONFIG_MALI_BIFROST_DVFS */ ++ ++ kbase_pm_reset_dvfs_utilisation_unlocked(kbdev, now); ++ ++ spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); ++} ++ ++bool kbase_pm_metrics_is_active(struct kbase_device *kbdev) ++{ ++ bool isactive; ++ unsigned long flags; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); ++ isactive = kbdev->pm.backend.metrics.timer_active; ++ spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); ++ ++ return isactive; ++} ++KBASE_EXPORT_TEST_API(kbase_pm_metrics_is_active); ++ ++#endif /* CONFIG_MALI_BIFROST_DVFS */ ++ ++/** ++ * kbase_pm_metrics_active_calc - Update PM active counts based on currently ++ * running atoms ++ * @kbdev: Device pointer ++ * ++ * The caller must hold kbdev->pm.backend.metrics.lock ++ */ ++static void kbase_pm_metrics_active_calc(struct kbase_device *kbdev) ++{ ++ int js; ++ ++ lockdep_assert_held(&kbdev->pm.backend.metrics.lock); ++ ++ kbdev->pm.backend.metrics.active_gl_ctx[0] = 0; ++ kbdev->pm.backend.metrics.active_gl_ctx[1] = 0; ++ kbdev->pm.backend.metrics.active_cl_ctx[0] = 0; ++ kbdev->pm.backend.metrics.active_cl_ctx[1] = 0; ++ kbdev->pm.backend.metrics.gpu_active = false; ++ ++ for (js = 0; js < BASE_JM_MAX_NR_SLOTS; js++) { ++ struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, 0); ++ ++ /* Head atom may have just completed, so if it isn't running ++ * then try the next atom */ ++ if (katom && katom->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) ++ katom = kbase_gpu_inspect(kbdev, js, 1); ++ ++ if (katom && katom->gpu_rb_state == ++ KBASE_ATOM_GPU_RB_SUBMITTED) { ++ if (katom->core_req & BASE_JD_REQ_ONLY_COMPUTE) { ++ int device_nr = (katom->core_req & ++ BASE_JD_REQ_SPECIFIC_COHERENT_GROUP) ++ ? katom->device_nr : 0; ++ if (!WARN_ON(device_nr >= 2)) ++ kbdev->pm.backend.metrics. ++ active_cl_ctx[device_nr] = 1; ++ } else { ++ /* Slot 2 should not be running non-compute ++ * atoms */ ++ if (!WARN_ON(js >= 2)) ++ kbdev->pm.backend.metrics. ++ active_gl_ctx[js] = 1; ++ } ++ kbdev->pm.backend.metrics.gpu_active = true; ++ } ++ } ++} ++ ++/* called when job is submitted to or removed from a GPU slot */ ++void kbase_pm_metrics_update(struct kbase_device *kbdev, ktime_t *timestamp) ++{ ++ unsigned long flags; ++ ktime_t now; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); ++ ++ if (!timestamp) { ++ now = ktime_get(); ++ timestamp = &now; ++ } ++ ++ /* Track how long CL and/or GL jobs have been busy for */ ++ kbase_pm_get_dvfs_utilisation_calc(kbdev, *timestamp); ++ ++ kbase_pm_metrics_active_calc(kbdev); ++ ++ spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); ++} +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_policy.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_policy.c +new file mode 100755 +index 000000000000..b98c68d9a42a +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_policy.c +@@ -0,0 +1,973 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* ++ * Power policy API implementations ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++ ++static const struct kbase_pm_policy *const policy_list[] = { ++#ifdef CONFIG_MALI_BIFROST_NO_MALI ++ &kbase_pm_always_on_policy_ops, ++ &kbase_pm_demand_policy_ops, ++ &kbase_pm_coarse_demand_policy_ops, ++#if !MALI_CUSTOMER_RELEASE ++ &kbase_pm_demand_always_powered_policy_ops, ++ &kbase_pm_fast_start_policy_ops, ++#endif ++#else /* CONFIG_MALI_BIFROST_NO_MALI */ ++#if !PLATFORM_POWER_DOWN_ONLY ++ &kbase_pm_demand_policy_ops, ++#endif /* !PLATFORM_POWER_DOWN_ONLY */ ++ &kbase_pm_coarse_demand_policy_ops, ++ &kbase_pm_always_on_policy_ops, ++#if !MALI_CUSTOMER_RELEASE ++#if !PLATFORM_POWER_DOWN_ONLY ++ &kbase_pm_demand_always_powered_policy_ops, ++ &kbase_pm_fast_start_policy_ops, ++#endif /* !PLATFORM_POWER_DOWN_ONLY */ ++#endif ++#endif /* CONFIG_MALI_BIFROST_NO_MALI */ ++}; ++ ++/* The number of policies available in the system. ++ * This is derived from the number of functions listed in policy_get_functions. ++ */ ++#define POLICY_COUNT (sizeof(policy_list)/sizeof(*policy_list)) ++ ++ ++/* Function IDs for looking up Timeline Trace codes in ++ * kbase_pm_change_state_trace_code */ ++enum kbase_pm_func_id { ++ KBASE_PM_FUNC_ID_REQUEST_CORES_START, ++ KBASE_PM_FUNC_ID_REQUEST_CORES_END, ++ KBASE_PM_FUNC_ID_RELEASE_CORES_START, ++ KBASE_PM_FUNC_ID_RELEASE_CORES_END, ++ /* Note: kbase_pm_unrequest_cores() is on the slow path, and we neither ++ * expect to hit it nor tend to hit it very much anyway. We can detect ++ * whether we need more instrumentation by a difference between ++ * PM_CHECKTRANS events and PM_SEND/HANDLE_EVENT. */ ++ ++ /* Must be the last */ ++ KBASE_PM_FUNC_ID_COUNT ++}; ++ ++ ++/* State changes during request/unrequest/release-ing cores */ ++enum { ++ KBASE_PM_CHANGE_STATE_SHADER = (1u << 0), ++ KBASE_PM_CHANGE_STATE_TILER = (1u << 1), ++ ++ /* These two must be last */ ++ KBASE_PM_CHANGE_STATE_MASK = (KBASE_PM_CHANGE_STATE_TILER | ++ KBASE_PM_CHANGE_STATE_SHADER), ++ KBASE_PM_CHANGE_STATE_COUNT = KBASE_PM_CHANGE_STATE_MASK + 1 ++}; ++typedef u32 kbase_pm_change_state; ++ ++ ++#ifdef CONFIG_MALI_BIFROST_TRACE_TIMELINE ++/* Timeline Trace code lookups for each function */ ++static u32 kbase_pm_change_state_trace_code[KBASE_PM_FUNC_ID_COUNT] ++ [KBASE_PM_CHANGE_STATE_COUNT] = { ++ /* kbase_pm_request_cores */ ++ [KBASE_PM_FUNC_ID_REQUEST_CORES_START][0] = 0, ++ [KBASE_PM_FUNC_ID_REQUEST_CORES_START][KBASE_PM_CHANGE_STATE_SHADER] = ++ SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_START, ++ [KBASE_PM_FUNC_ID_REQUEST_CORES_START][KBASE_PM_CHANGE_STATE_TILER] = ++ SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_TILER_START, ++ [KBASE_PM_FUNC_ID_REQUEST_CORES_START][KBASE_PM_CHANGE_STATE_SHADER | ++ KBASE_PM_CHANGE_STATE_TILER] = ++ SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_TILER_START, ++ ++ [KBASE_PM_FUNC_ID_REQUEST_CORES_END][0] = 0, ++ [KBASE_PM_FUNC_ID_REQUEST_CORES_END][KBASE_PM_CHANGE_STATE_SHADER] = ++ SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_END, ++ [KBASE_PM_FUNC_ID_REQUEST_CORES_END][KBASE_PM_CHANGE_STATE_TILER] = ++ SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_TILER_END, ++ [KBASE_PM_FUNC_ID_REQUEST_CORES_END][KBASE_PM_CHANGE_STATE_SHADER | ++ KBASE_PM_CHANGE_STATE_TILER] = ++ SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_TILER_END, ++ ++ /* kbase_pm_release_cores */ ++ [KBASE_PM_FUNC_ID_RELEASE_CORES_START][0] = 0, ++ [KBASE_PM_FUNC_ID_RELEASE_CORES_START][KBASE_PM_CHANGE_STATE_SHADER] = ++ SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_START, ++ [KBASE_PM_FUNC_ID_RELEASE_CORES_START][KBASE_PM_CHANGE_STATE_TILER] = ++ SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_TILER_START, ++ [KBASE_PM_FUNC_ID_RELEASE_CORES_START][KBASE_PM_CHANGE_STATE_SHADER | ++ KBASE_PM_CHANGE_STATE_TILER] = ++ SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_TILER_START, ++ ++ [KBASE_PM_FUNC_ID_RELEASE_CORES_END][0] = 0, ++ [KBASE_PM_FUNC_ID_RELEASE_CORES_END][KBASE_PM_CHANGE_STATE_SHADER] = ++ SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_END, ++ [KBASE_PM_FUNC_ID_RELEASE_CORES_END][KBASE_PM_CHANGE_STATE_TILER] = ++ SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_TILER_END, ++ [KBASE_PM_FUNC_ID_RELEASE_CORES_END][KBASE_PM_CHANGE_STATE_SHADER | ++ KBASE_PM_CHANGE_STATE_TILER] = ++ SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_TILER_END ++}; ++ ++static inline void kbase_timeline_pm_cores_func(struct kbase_device *kbdev, ++ enum kbase_pm_func_id func_id, ++ kbase_pm_change_state state) ++{ ++ int trace_code; ++ ++ KBASE_DEBUG_ASSERT(func_id >= 0 && func_id < KBASE_PM_FUNC_ID_COUNT); ++ KBASE_DEBUG_ASSERT(state != 0 && (state & KBASE_PM_CHANGE_STATE_MASK) == ++ state); ++ ++ trace_code = kbase_pm_change_state_trace_code[func_id][state]; ++ KBASE_TIMELINE_PM_CHECKTRANS(kbdev, trace_code); ++} ++ ++#else /* CONFIG_MALI_BIFROST_TRACE_TIMELINE */ ++static inline void kbase_timeline_pm_cores_func(struct kbase_device *kbdev, ++ enum kbase_pm_func_id func_id, kbase_pm_change_state state) ++{ ++} ++ ++#endif /* CONFIG_MALI_BIFROST_TRACE_TIMELINE */ ++ ++/** ++ * kbasep_pm_do_poweroff_cores - Process a poweroff request and power down any ++ * requested shader cores ++ * @kbdev: Device pointer ++ */ ++static void kbasep_pm_do_poweroff_cores(struct kbase_device *kbdev) ++{ ++ u64 prev_shader_state = kbdev->pm.backend.desired_shader_state; ++ u64 prev_tiler_state = kbdev->pm.backend.desired_tiler_state; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ kbdev->pm.backend.desired_shader_state &= ++ ~kbdev->pm.backend.shader_poweroff_pending; ++ kbdev->pm.backend.desired_tiler_state &= ++ ~kbdev->pm.backend.tiler_poweroff_pending; ++ ++ kbdev->pm.backend.shader_poweroff_pending = 0; ++ kbdev->pm.backend.tiler_poweroff_pending = 0; ++ ++ if (prev_shader_state != kbdev->pm.backend.desired_shader_state || ++ prev_tiler_state != ++ kbdev->pm.backend.desired_tiler_state || ++ kbdev->pm.backend.ca_in_transition) { ++ bool cores_are_available; ++ ++ KBASE_TIMELINE_PM_CHECKTRANS(kbdev, ++ SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_DEFERRED_START); ++ cores_are_available = kbase_pm_check_transitions_nolock(kbdev); ++ KBASE_TIMELINE_PM_CHECKTRANS(kbdev, ++ SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_DEFERRED_END); ++ ++ /* Don't need 'cores_are_available', ++ * because we don't return anything */ ++ CSTD_UNUSED(cores_are_available); ++ } ++} ++ ++static enum hrtimer_restart ++kbasep_pm_do_gpu_poweroff_callback(struct hrtimer *timer) ++{ ++ struct kbase_device *kbdev; ++ unsigned long flags; ++ ++ kbdev = container_of(timer, struct kbase_device, ++ pm.backend.gpu_poweroff_timer); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ /* It is safe for this call to do nothing if the work item is already ++ * queued. The worker function will read the must up-to-date state of ++ * kbdev->pm.backend.gpu_poweroff_pending under lock. ++ * ++ * If a state change occurs while the worker function is processing, ++ * this call will succeed as a work item can be requeued once it has ++ * started processing. ++ */ ++ if (kbdev->pm.backend.gpu_poweroff_pending) ++ queue_work(kbdev->pm.backend.gpu_poweroff_wq, ++ &kbdev->pm.backend.gpu_poweroff_work); ++ ++ if (kbdev->pm.backend.shader_poweroff_pending || ++ kbdev->pm.backend.tiler_poweroff_pending) { ++ kbdev->pm.backend.shader_poweroff_pending_time--; ++ ++ KBASE_DEBUG_ASSERT( ++ kbdev->pm.backend.shader_poweroff_pending_time ++ >= 0); ++ ++ if (!kbdev->pm.backend.shader_poweroff_pending_time) ++ kbasep_pm_do_poweroff_cores(kbdev); ++ } ++ ++ if (kbdev->pm.backend.poweroff_timer_needed) { ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ hrtimer_add_expires(timer, kbdev->pm.gpu_poweroff_time); ++ ++ return HRTIMER_RESTART; ++ } ++ ++ kbdev->pm.backend.poweroff_timer_running = false; ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ return HRTIMER_NORESTART; ++} ++ ++static void kbasep_pm_do_gpu_poweroff_wq(struct work_struct *data) ++{ ++ unsigned long flags; ++ struct kbase_device *kbdev; ++ bool do_poweroff = false; ++ ++ kbdev = container_of(data, struct kbase_device, ++ pm.backend.gpu_poweroff_work); ++ ++ mutex_lock(&kbdev->pm.lock); ++ ++ if (kbdev->pm.backend.gpu_poweroff_pending == 0) { ++ mutex_unlock(&kbdev->pm.lock); ++ return; ++ } ++ ++ kbdev->pm.backend.gpu_poweroff_pending--; ++ ++ if (kbdev->pm.backend.gpu_poweroff_pending > 0) { ++ mutex_unlock(&kbdev->pm.lock); ++ return; ++ } ++ ++ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_poweroff_pending == 0); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ /* Only power off the GPU if a request is still pending */ ++ if (!kbdev->pm.backend.pm_current_policy->get_core_active(kbdev)) ++ do_poweroff = true; ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ if (do_poweroff) { ++ kbdev->pm.backend.poweroff_timer_needed = false; ++ hrtimer_cancel(&kbdev->pm.backend.gpu_poweroff_timer); ++ kbdev->pm.backend.poweroff_timer_running = false; ++ ++ /* Power off the GPU */ ++ kbase_pm_do_poweroff(kbdev, false); ++ } ++ ++ mutex_unlock(&kbdev->pm.lock); ++} ++ ++int kbase_pm_policy_init(struct kbase_device *kbdev) ++{ ++ struct workqueue_struct *wq; ++ ++ wq = alloc_workqueue("kbase_pm_do_poweroff", ++ WQ_HIGHPRI | WQ_UNBOUND, 1); ++ if (!wq) ++ return -ENOMEM; ++ ++ kbdev->pm.backend.gpu_poweroff_wq = wq; ++ INIT_WORK(&kbdev->pm.backend.gpu_poweroff_work, ++ kbasep_pm_do_gpu_poweroff_wq); ++ hrtimer_init(&kbdev->pm.backend.gpu_poweroff_timer, ++ CLOCK_MONOTONIC, HRTIMER_MODE_REL); ++ kbdev->pm.backend.gpu_poweroff_timer.function = ++ kbasep_pm_do_gpu_poweroff_callback; ++ kbdev->pm.backend.pm_current_policy = policy_list[0]; ++ kbdev->pm.backend.pm_current_policy->init(kbdev); ++ kbdev->pm.gpu_poweroff_time = ++ HR_TIMER_DELAY_NSEC(DEFAULT_PM_GPU_POWEROFF_TICK_NS); ++ kbdev->pm.poweroff_shader_ticks = DEFAULT_PM_POWEROFF_TICK_SHADER; ++ kbdev->pm.poweroff_gpu_ticks = DEFAULT_PM_POWEROFF_TICK_GPU; ++ ++ return 0; ++} ++ ++void kbase_pm_policy_term(struct kbase_device *kbdev) ++{ ++ kbdev->pm.backend.pm_current_policy->term(kbdev); ++ destroy_workqueue(kbdev->pm.backend.gpu_poweroff_wq); ++} ++ ++void kbase_pm_cancel_deferred_poweroff(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ ++ lockdep_assert_held(&kbdev->pm.lock); ++ ++ kbdev->pm.backend.poweroff_timer_needed = false; ++ hrtimer_cancel(&kbdev->pm.backend.gpu_poweroff_timer); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbdev->pm.backend.poweroff_timer_running = false; ++ ++ /* If wq is already running but is held off by pm.lock, make sure it has ++ * no effect */ ++ kbdev->pm.backend.gpu_poweroff_pending = 0; ++ ++ kbdev->pm.backend.shader_poweroff_pending = 0; ++ kbdev->pm.backend.tiler_poweroff_pending = 0; ++ kbdev->pm.backend.shader_poweroff_pending_time = 0; ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++} ++ ++void kbase_pm_update_active(struct kbase_device *kbdev) ++{ ++ struct kbase_pm_device_data *pm = &kbdev->pm; ++ struct kbase_pm_backend_data *backend = &pm->backend; ++ unsigned long flags; ++ bool active; ++ ++ lockdep_assert_held(&pm->lock); ++ ++ /* pm_current_policy will never be NULL while pm.lock is held */ ++ KBASE_DEBUG_ASSERT(backend->pm_current_policy); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ active = backend->pm_current_policy->get_core_active(kbdev); ++ ++ if (active) { ++ if (backend->gpu_poweroff_pending) { ++ /* Cancel any pending power off request */ ++ backend->gpu_poweroff_pending = 0; ++ ++ /* If a request was pending then the GPU was still ++ * powered, so no need to continue */ ++ if (!kbdev->poweroff_pending) { ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, ++ flags); ++ return; ++ } ++ } ++ ++ if (!backend->poweroff_timer_running && !backend->gpu_powered && ++ (pm->poweroff_gpu_ticks || ++ pm->poweroff_shader_ticks)) { ++ backend->poweroff_timer_needed = true; ++ backend->poweroff_timer_running = true; ++ hrtimer_start(&backend->gpu_poweroff_timer, ++ pm->gpu_poweroff_time, ++ HRTIMER_MODE_REL); ++ } ++ ++ /* Power on the GPU and any cores requested by the policy */ ++ if (pm->backend.poweroff_wait_in_progress) { ++ pm->backend.poweron_required = true; ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ } else { ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ kbase_pm_do_poweron(kbdev, false); ++ } ++ } else { ++ /* It is an error for the power policy to power off the GPU ++ * when there are contexts active */ ++ KBASE_DEBUG_ASSERT(pm->active_count == 0); ++ ++ if (backend->shader_poweroff_pending || ++ backend->tiler_poweroff_pending) { ++ backend->shader_poweroff_pending = 0; ++ backend->tiler_poweroff_pending = 0; ++ backend->shader_poweroff_pending_time = 0; ++ } ++ ++ /* Request power off */ ++ if (pm->backend.gpu_powered) { ++ if (pm->poweroff_gpu_ticks) { ++ backend->gpu_poweroff_pending = ++ pm->poweroff_gpu_ticks; ++ backend->poweroff_timer_needed = true; ++ if (!backend->poweroff_timer_running) { ++ /* Start timer if not running (eg if ++ * power policy has been changed from ++ * always_on to something else). This ++ * will ensure the GPU is actually ++ * powered off */ ++ backend->poweroff_timer_running ++ = true; ++ hrtimer_start( ++ &backend->gpu_poweroff_timer, ++ pm->gpu_poweroff_time, ++ HRTIMER_MODE_REL); ++ } ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, ++ flags); ++ } else { ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, ++ flags); ++ ++ /* Power off the GPU immediately */ ++ kbase_pm_do_poweroff(kbdev, false); ++ } ++ } else { ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ } ++ } ++} ++ ++void kbase_pm_update_cores_state_nolock(struct kbase_device *kbdev) ++{ ++ u64 desired_bitmap; ++ u64 desired_tiler_bitmap; ++ bool cores_are_available; ++ bool do_poweroff = false; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (kbdev->pm.backend.pm_current_policy == NULL) ++ return; ++ if (kbdev->pm.backend.poweroff_wait_in_progress) ++ return; ++ ++ if (kbdev->protected_mode_transition && !kbdev->shader_needed_bitmap && ++ !kbdev->shader_inuse_bitmap && !kbdev->tiler_needed_cnt ++ && !kbdev->tiler_inuse_cnt) { ++ /* We are trying to change in/out of protected mode - force all ++ * cores off so that the L2 powers down */ ++ desired_bitmap = 0; ++ desired_tiler_bitmap = 0; ++ } else { ++ desired_bitmap = ++ kbdev->pm.backend.pm_current_policy->get_core_mask(kbdev); ++ desired_bitmap &= kbase_pm_ca_get_core_mask(kbdev); ++ ++ if (kbdev->tiler_needed_cnt > 0 || kbdev->tiler_inuse_cnt > 0) ++ desired_tiler_bitmap = 1; ++ else ++ desired_tiler_bitmap = 0; ++ ++ if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_XAFFINITY)) { ++ /* Unless XAFFINITY is supported, enable core 0 if tiler ++ * required, regardless of core availability */ ++ if (kbdev->tiler_needed_cnt > 0 || ++ kbdev->tiler_inuse_cnt > 0) ++ desired_bitmap |= 1; ++ } ++ } ++ ++ if (kbdev->pm.backend.desired_shader_state != desired_bitmap) ++ KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_DESIRED, NULL, NULL, 0u, ++ (u32)desired_bitmap); ++ /* Are any cores being powered on? */ ++ if (~kbdev->pm.backend.desired_shader_state & desired_bitmap || ++ ~kbdev->pm.backend.desired_tiler_state & desired_tiler_bitmap || ++ kbdev->pm.backend.ca_in_transition) { ++ /* Check if we are powering off any cores before updating shader ++ * state */ ++ if (kbdev->pm.backend.desired_shader_state & ~desired_bitmap || ++ kbdev->pm.backend.desired_tiler_state & ++ ~desired_tiler_bitmap) { ++ /* Start timer to power off cores */ ++ kbdev->pm.backend.shader_poweroff_pending |= ++ (kbdev->pm.backend.desired_shader_state & ++ ~desired_bitmap); ++ kbdev->pm.backend.tiler_poweroff_pending |= ++ (kbdev->pm.backend.desired_tiler_state & ++ ~desired_tiler_bitmap); ++ ++ if (kbdev->pm.poweroff_shader_ticks && ++ !kbdev->protected_mode_transition) ++ kbdev->pm.backend.shader_poweroff_pending_time = ++ kbdev->pm.poweroff_shader_ticks; ++ else ++ do_poweroff = true; ++ } ++ ++ kbdev->pm.backend.desired_shader_state = desired_bitmap; ++ kbdev->pm.backend.desired_tiler_state = desired_tiler_bitmap; ++ ++ /* If any cores are being powered on, transition immediately */ ++ cores_are_available = kbase_pm_check_transitions_nolock(kbdev); ++ } else if (kbdev->pm.backend.desired_shader_state & ~desired_bitmap || ++ kbdev->pm.backend.desired_tiler_state & ++ ~desired_tiler_bitmap) { ++ /* Start timer to power off cores */ ++ kbdev->pm.backend.shader_poweroff_pending |= ++ (kbdev->pm.backend.desired_shader_state & ++ ~desired_bitmap); ++ kbdev->pm.backend.tiler_poweroff_pending |= ++ (kbdev->pm.backend.desired_tiler_state & ++ ~desired_tiler_bitmap); ++ if (kbdev->pm.poweroff_shader_ticks && ++ !kbdev->protected_mode_transition) ++ kbdev->pm.backend.shader_poweroff_pending_time = ++ kbdev->pm.poweroff_shader_ticks; ++ else ++ kbasep_pm_do_poweroff_cores(kbdev); ++ } else if (kbdev->pm.active_count == 0 && desired_bitmap != 0 && ++ desired_tiler_bitmap != 0 && ++ kbdev->pm.backend.poweroff_timer_needed) { ++ /* If power policy is keeping cores on despite there being no ++ * active contexts then disable poweroff timer as it isn't ++ * required. ++ * Only reset poweroff_timer_needed if we're not in the middle ++ * of the power off callback */ ++ kbdev->pm.backend.poweroff_timer_needed = false; ++ } ++ ++ /* Ensure timer does not power off wanted cores and make sure to power ++ * off unwanted cores */ ++ if (kbdev->pm.backend.shader_poweroff_pending || ++ kbdev->pm.backend.tiler_poweroff_pending) { ++ kbdev->pm.backend.shader_poweroff_pending &= ++ ~(kbdev->pm.backend.desired_shader_state & ++ desired_bitmap); ++ kbdev->pm.backend.tiler_poweroff_pending &= ++ ~(kbdev->pm.backend.desired_tiler_state & ++ desired_tiler_bitmap); ++ ++ if (!kbdev->pm.backend.shader_poweroff_pending && ++ !kbdev->pm.backend.tiler_poweroff_pending) ++ kbdev->pm.backend.shader_poweroff_pending_time = 0; ++ } ++ ++ /* Shader poweroff is deferred to the end of the function, to eliminate ++ * issues caused by the core availability policy recursing into this ++ * function */ ++ if (do_poweroff) ++ kbasep_pm_do_poweroff_cores(kbdev); ++ ++ /* Don't need 'cores_are_available', because we don't return anything */ ++ CSTD_UNUSED(cores_are_available); ++} ++ ++void kbase_pm_update_cores_state(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ kbase_pm_update_cores_state_nolock(kbdev); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++} ++ ++int kbase_pm_list_policies(const struct kbase_pm_policy * const **list) ++{ ++ if (!list) ++ return POLICY_COUNT; ++ ++ *list = policy_list; ++ ++ return POLICY_COUNT; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_list_policies); ++ ++const struct kbase_pm_policy *kbase_pm_get_policy(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ return kbdev->pm.backend.pm_current_policy; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_get_policy); ++ ++void kbase_pm_set_policy(struct kbase_device *kbdev, ++ const struct kbase_pm_policy *new_policy) ++{ ++ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; ++ const struct kbase_pm_policy *old_policy; ++ unsigned long flags; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(new_policy != NULL); ++ ++ KBASE_TRACE_ADD(kbdev, PM_SET_POLICY, NULL, NULL, 0u, new_policy->id); ++ ++ /* During a policy change we pretend the GPU is active */ ++ /* A suspend won't happen here, because we're in a syscall from a ++ * userspace thread */ ++ kbase_pm_context_active(kbdev); ++ ++ mutex_lock(&js_devdata->runpool_mutex); ++ mutex_lock(&kbdev->pm.lock); ++ ++ /* Remove the policy to prevent IRQ handlers from working on it */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ old_policy = kbdev->pm.backend.pm_current_policy; ++ kbdev->pm.backend.pm_current_policy = NULL; ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ KBASE_TRACE_ADD(kbdev, PM_CURRENT_POLICY_TERM, NULL, NULL, 0u, ++ old_policy->id); ++ if (old_policy->term) ++ old_policy->term(kbdev); ++ ++ KBASE_TRACE_ADD(kbdev, PM_CURRENT_POLICY_INIT, NULL, NULL, 0u, ++ new_policy->id); ++ if (new_policy->init) ++ new_policy->init(kbdev); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbdev->pm.backend.pm_current_policy = new_policy; ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ /* If any core power state changes were previously attempted, but ++ * couldn't be made because the policy was changing (current_policy was ++ * NULL), then re-try them here. */ ++ kbase_pm_update_active(kbdev); ++ kbase_pm_update_cores_state(kbdev); ++ ++ mutex_unlock(&kbdev->pm.lock); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ ++ /* Now the policy change is finished, we release our fake context active ++ * reference */ ++ kbase_pm_context_idle(kbdev); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_set_policy); ++ ++/* Check whether a state change has finished, and trace it as completed */ ++static void ++kbase_pm_trace_check_and_finish_state_change(struct kbase_device *kbdev) ++{ ++ if ((kbdev->shader_available_bitmap & ++ kbdev->pm.backend.desired_shader_state) ++ == kbdev->pm.backend.desired_shader_state && ++ (kbdev->tiler_available_bitmap & ++ kbdev->pm.backend.desired_tiler_state) ++ == kbdev->pm.backend.desired_tiler_state) ++ kbase_timeline_pm_check_handle_event(kbdev, ++ KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED); ++} ++ ++void kbase_pm_request_cores(struct kbase_device *kbdev, ++ bool tiler_required, u64 shader_cores) ++{ ++ u64 cores; ++ ++ kbase_pm_change_state change_gpu_state = 0u; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ cores = shader_cores; ++ while (cores) { ++ int bitnum = fls64(cores) - 1; ++ u64 bit = 1ULL << bitnum; ++ ++ /* It should be almost impossible for this to overflow. It would ++ * require 2^32 atoms to request a particular core, which would ++ * require 2^24 contexts to submit. This would require an amount ++ * of memory that is impossible on a 32-bit system and extremely ++ * unlikely on a 64-bit system. */ ++ int cnt = ++kbdev->shader_needed_cnt[bitnum]; ++ ++ if (1 == cnt) { ++ kbdev->shader_needed_bitmap |= bit; ++ change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER; ++ } ++ ++ cores &= ~bit; ++ } ++ ++ if (tiler_required) { ++ int cnt = ++kbdev->tiler_needed_cnt; ++ ++ if (1 == cnt) ++ change_gpu_state |= KBASE_PM_CHANGE_STATE_TILER; ++ ++ KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt != 0); ++ } ++ ++ if (change_gpu_state) { ++ KBASE_TRACE_ADD(kbdev, PM_REQUEST_CHANGE_SHADER_NEEDED, NULL, ++ NULL, 0u, (u32) kbdev->shader_needed_bitmap); ++ ++ kbase_timeline_pm_cores_func(kbdev, ++ KBASE_PM_FUNC_ID_REQUEST_CORES_START, ++ change_gpu_state); ++ kbase_pm_update_cores_state_nolock(kbdev); ++ kbase_timeline_pm_cores_func(kbdev, ++ KBASE_PM_FUNC_ID_REQUEST_CORES_END, ++ change_gpu_state); ++ } ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_request_cores); ++ ++void kbase_pm_unrequest_cores(struct kbase_device *kbdev, ++ bool tiler_required, u64 shader_cores) ++{ ++ kbase_pm_change_state change_gpu_state = 0u; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ while (shader_cores) { ++ int bitnum = fls64(shader_cores) - 1; ++ u64 bit = 1ULL << bitnum; ++ int cnt; ++ ++ KBASE_DEBUG_ASSERT(kbdev->shader_needed_cnt[bitnum] > 0); ++ ++ cnt = --kbdev->shader_needed_cnt[bitnum]; ++ ++ if (0 == cnt) { ++ kbdev->shader_needed_bitmap &= ~bit; ++ ++ change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER; ++ } ++ ++ shader_cores &= ~bit; ++ } ++ ++ if (tiler_required) { ++ int cnt; ++ ++ KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt > 0); ++ ++ cnt = --kbdev->tiler_needed_cnt; ++ ++ if (0 == cnt) ++ change_gpu_state |= KBASE_PM_CHANGE_STATE_TILER; ++ } ++ ++ if (change_gpu_state) { ++ KBASE_TRACE_ADD(kbdev, PM_UNREQUEST_CHANGE_SHADER_NEEDED, NULL, ++ NULL, 0u, (u32) kbdev->shader_needed_bitmap); ++ ++ kbase_pm_update_cores_state_nolock(kbdev); ++ ++ /* Trace that any state change effectively completes immediately ++ * - no-one will wait on the state change */ ++ kbase_pm_trace_check_and_finish_state_change(kbdev); ++ } ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_unrequest_cores); ++ ++enum kbase_pm_cores_ready ++kbase_pm_register_inuse_cores(struct kbase_device *kbdev, ++ bool tiler_required, u64 shader_cores) ++{ ++ u64 prev_shader_needed; /* Just for tracing */ ++ u64 prev_shader_inuse; /* Just for tracing */ ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ prev_shader_needed = kbdev->shader_needed_bitmap; ++ prev_shader_inuse = kbdev->shader_inuse_bitmap; ++ ++ /* If desired_shader_state does not contain the requested cores, then ++ * power management is not attempting to powering those cores (most ++ * likely due to core availability policy) and a new job affinity must ++ * be chosen */ ++ if ((kbdev->pm.backend.desired_shader_state & shader_cores) != ++ shader_cores) { ++ return (kbdev->pm.backend.poweroff_wait_in_progress || ++ kbdev->pm.backend.pm_current_policy == NULL) ? ++ KBASE_CORES_NOT_READY : KBASE_NEW_AFFINITY; ++ } ++ ++ if ((kbdev->shader_available_bitmap & shader_cores) != shader_cores || ++ (tiler_required && !kbdev->tiler_available_bitmap)) { ++ /* Trace ongoing core transition */ ++ kbase_timeline_pm_l2_transition_start(kbdev); ++ return KBASE_CORES_NOT_READY; ++ } ++ ++ /* If we started to trace a state change, then trace it has being ++ * finished by now, at the very latest */ ++ kbase_pm_trace_check_and_finish_state_change(kbdev); ++ /* Trace core transition done */ ++ kbase_timeline_pm_l2_transition_done(kbdev); ++ ++ while (shader_cores) { ++ int bitnum = fls64(shader_cores) - 1; ++ u64 bit = 1ULL << bitnum; ++ int cnt; ++ ++ KBASE_DEBUG_ASSERT(kbdev->shader_needed_cnt[bitnum] > 0); ++ ++ cnt = --kbdev->shader_needed_cnt[bitnum]; ++ ++ if (0 == cnt) ++ kbdev->shader_needed_bitmap &= ~bit; ++ ++ /* shader_inuse_cnt should not overflow because there can only ++ * be a very limited number of jobs on the h/w at one time */ ++ ++ kbdev->shader_inuse_cnt[bitnum]++; ++ kbdev->shader_inuse_bitmap |= bit; ++ ++ shader_cores &= ~bit; ++ } ++ ++ if (tiler_required) { ++ KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt > 0); ++ ++ --kbdev->tiler_needed_cnt; ++ ++ kbdev->tiler_inuse_cnt++; ++ ++ KBASE_DEBUG_ASSERT(kbdev->tiler_inuse_cnt != 0); ++ } ++ ++ if (prev_shader_needed != kbdev->shader_needed_bitmap) ++ KBASE_TRACE_ADD(kbdev, PM_REGISTER_CHANGE_SHADER_NEEDED, NULL, ++ NULL, 0u, (u32) kbdev->shader_needed_bitmap); ++ ++ if (prev_shader_inuse != kbdev->shader_inuse_bitmap) ++ KBASE_TRACE_ADD(kbdev, PM_REGISTER_CHANGE_SHADER_INUSE, NULL, ++ NULL, 0u, (u32) kbdev->shader_inuse_bitmap); ++ ++ return KBASE_CORES_READY; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_register_inuse_cores); ++ ++void kbase_pm_release_cores(struct kbase_device *kbdev, ++ bool tiler_required, u64 shader_cores) ++{ ++ kbase_pm_change_state change_gpu_state = 0u; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ while (shader_cores) { ++ int bitnum = fls64(shader_cores) - 1; ++ u64 bit = 1ULL << bitnum; ++ int cnt; ++ ++ KBASE_DEBUG_ASSERT(kbdev->shader_inuse_cnt[bitnum] > 0); ++ ++ cnt = --kbdev->shader_inuse_cnt[bitnum]; ++ ++ if (0 == cnt) { ++ kbdev->shader_inuse_bitmap &= ~bit; ++ change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER; ++ } ++ ++ shader_cores &= ~bit; ++ } ++ ++ if (tiler_required) { ++ int cnt; ++ ++ KBASE_DEBUG_ASSERT(kbdev->tiler_inuse_cnt > 0); ++ ++ cnt = --kbdev->tiler_inuse_cnt; ++ ++ if (0 == cnt) ++ change_gpu_state |= KBASE_PM_CHANGE_STATE_TILER; ++ } ++ ++ if (change_gpu_state) { ++ KBASE_TRACE_ADD(kbdev, PM_RELEASE_CHANGE_SHADER_INUSE, NULL, ++ NULL, 0u, (u32) kbdev->shader_inuse_bitmap); ++ ++ kbase_timeline_pm_cores_func(kbdev, ++ KBASE_PM_FUNC_ID_RELEASE_CORES_START, ++ change_gpu_state); ++ kbase_pm_update_cores_state_nolock(kbdev); ++ kbase_timeline_pm_cores_func(kbdev, ++ KBASE_PM_FUNC_ID_RELEASE_CORES_END, ++ change_gpu_state); ++ ++ /* Trace that any state change completed immediately */ ++ kbase_pm_trace_check_and_finish_state_change(kbdev); ++ } ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_release_cores); ++ ++void kbase_pm_request_cores_sync(struct kbase_device *kbdev, ++ bool tiler_required, ++ u64 shader_cores) ++{ ++ unsigned long flags; ++ ++ kbase_pm_wait_for_poweroff_complete(kbdev); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_pm_request_cores(kbdev, tiler_required, shader_cores); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ kbase_pm_check_transitions_sync(kbdev); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_request_cores_sync); ++ ++void kbase_pm_request_l2_caches(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ u32 prior_l2_users_count; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ prior_l2_users_count = kbdev->l2_users_count++; ++ ++ KBASE_DEBUG_ASSERT(kbdev->l2_users_count != 0); ++ ++ /* if the GPU is reset while the l2 is on, l2 will be off but ++ * prior_l2_users_count will be > 0. l2_available_bitmap will have been ++ * set to 0 though by kbase_pm_init_hw */ ++ if (!prior_l2_users_count || !kbdev->l2_available_bitmap) ++ kbase_pm_check_transitions_nolock(kbdev); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ wait_event(kbdev->pm.backend.l2_powered_wait, ++ kbdev->pm.backend.l2_powered == 1); ++ ++ /* Trace that any state change completed immediately */ ++ kbase_pm_trace_check_and_finish_state_change(kbdev); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_request_l2_caches); ++ ++void kbase_pm_request_l2_caches_l2_is_on(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ kbdev->l2_users_count++; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_request_l2_caches_l2_is_on); ++ ++void kbase_pm_release_l2_caches(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ KBASE_DEBUG_ASSERT(kbdev->l2_users_count > 0); ++ ++ --kbdev->l2_users_count; ++ ++ if (!kbdev->l2_users_count) { ++ kbase_pm_check_transitions_nolock(kbdev); ++ /* Trace that any state change completed immediately */ ++ kbase_pm_trace_check_and_finish_state_change(kbdev); ++ } ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_release_l2_caches); +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_policy.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_policy.h +new file mode 100755 +index 000000000000..611a90e66e65 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_policy.h +@@ -0,0 +1,227 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* ++ * Power policy API definitions ++ */ ++ ++#ifndef _KBASE_PM_POLICY_H_ ++#define _KBASE_PM_POLICY_H_ ++ ++/** ++ * kbase_pm_policy_init - Initialize power policy framework ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * Must be called before calling any other policy function ++ * ++ * Return: 0 if the power policy framework was successfully ++ * initialized, -errno otherwise. ++ */ ++int kbase_pm_policy_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_policy_term - Terminate power policy framework ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_policy_term(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_update_active - Update the active power state of the GPU ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * Calls into the current power policy ++ */ ++void kbase_pm_update_active(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_update_cores - Update the desired core state of the GPU ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * Calls into the current power policy ++ */ ++void kbase_pm_update_cores(struct kbase_device *kbdev); ++ ++ ++enum kbase_pm_cores_ready { ++ KBASE_CORES_NOT_READY = 0, ++ KBASE_NEW_AFFINITY = 1, ++ KBASE_CORES_READY = 2 ++}; ++ ++ ++/** ++ * kbase_pm_request_cores_sync - Synchronous variant of kbase_pm_request_cores() ++ * ++ * @kbdev: The kbase device structure for the device ++ * @tiler_required: true if the tiler is required, false otherwise ++ * @shader_cores: A bitmask of shader cores which are necessary for the job ++ * ++ * When this function returns, the @shader_cores will be in the READY state. ++ * ++ * This is safe variant of kbase_pm_check_transitions_sync(): it handles the ++ * work of ensuring the requested cores will remain powered until a matching ++ * call to kbase_pm_unrequest_cores()/kbase_pm_release_cores() (as appropriate) ++ * is made. ++ */ ++void kbase_pm_request_cores_sync(struct kbase_device *kbdev, ++ bool tiler_required, u64 shader_cores); ++ ++/** ++ * kbase_pm_request_cores - Mark one or more cores as being required ++ * for jobs to be submitted ++ * ++ * @kbdev: The kbase device structure for the device ++ * @tiler_required: true if the tiler is required, false otherwise ++ * @shader_cores: A bitmask of shader cores which are necessary for the job ++ * ++ * This function is called by the job scheduler to mark one or more cores as ++ * being required to submit jobs that are ready to run. ++ * ++ * The cores requested are reference counted and a subsequent call to ++ * kbase_pm_register_inuse_cores() or kbase_pm_unrequest_cores() should be ++ * made to dereference the cores as being 'needed'. ++ * ++ * The active power policy will meet or exceed the requirements of the ++ * requested cores in the system. Any core transitions needed will be begun ++ * immediately, but they might not complete/the cores might not be available ++ * until a Power Management IRQ. ++ * ++ * Return: 0 if the cores were successfully requested, or -errno otherwise. ++ */ ++void kbase_pm_request_cores(struct kbase_device *kbdev, ++ bool tiler_required, u64 shader_cores); ++ ++/** ++ * kbase_pm_unrequest_cores - Unmark one or more cores as being required for ++ * jobs to be submitted. ++ * ++ * @kbdev: The kbase device structure for the device ++ * @tiler_required: true if the tiler is required, false otherwise ++ * @shader_cores: A bitmask of shader cores (as given to ++ * kbase_pm_request_cores() ) ++ * ++ * This function undoes the effect of kbase_pm_request_cores(). It should be ++ * used when a job is not going to be submitted to the hardware (e.g. the job is ++ * cancelled before it is enqueued). ++ * ++ * The active power policy will meet or exceed the requirements of the ++ * requested cores in the system. Any core transitions needed will be begun ++ * immediately, but they might not complete until a Power Management IRQ. ++ * ++ * The policy may use this as an indication that it can power down cores. ++ */ ++void kbase_pm_unrequest_cores(struct kbase_device *kbdev, ++ bool tiler_required, u64 shader_cores); ++ ++/** ++ * kbase_pm_register_inuse_cores - Register a set of cores as in use by a job ++ * ++ * @kbdev: The kbase device structure for the device ++ * @tiler_required: true if the tiler is required, false otherwise ++ * @shader_cores: A bitmask of shader cores (as given to ++ * kbase_pm_request_cores() ) ++ * ++ * This function should be called after kbase_pm_request_cores() when the job ++ * is about to be submitted to the hardware. It will check that the necessary ++ * cores are available and if so update the 'needed' and 'inuse' bitmasks to ++ * reflect that the job is now committed to being run. ++ * ++ * If the necessary cores are not currently available then the function will ++ * return %KBASE_CORES_NOT_READY and have no effect. ++ * ++ * Return: %KBASE_CORES_NOT_READY if the cores are not immediately ready, ++ * ++ * %KBASE_NEW_AFFINITY if the affinity requested is not allowed, ++ * ++ * %KBASE_CORES_READY if the cores requested are already available ++ */ ++enum kbase_pm_cores_ready kbase_pm_register_inuse_cores( ++ struct kbase_device *kbdev, ++ bool tiler_required, ++ u64 shader_cores); ++ ++/** ++ * kbase_pm_release_cores - Release cores after a job has run ++ * ++ * @kbdev: The kbase device structure for the device ++ * @tiler_required: true if the tiler is required, false otherwise ++ * @shader_cores: A bitmask of shader cores (as given to ++ * kbase_pm_register_inuse_cores() ) ++ * ++ * This function should be called when a job has finished running on the ++ * hardware. A call to kbase_pm_register_inuse_cores() must have previously ++ * occurred. The reference counts of the specified cores will be decremented ++ * which may cause the bitmask of 'inuse' cores to be reduced. The power policy ++ * may then turn off any cores which are no longer 'inuse'. ++ */ ++void kbase_pm_release_cores(struct kbase_device *kbdev, ++ bool tiler_required, u64 shader_cores); ++ ++/** ++ * kbase_pm_request_l2_caches - Request l2 caches ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * Request the use of l2 caches for all core groups, power up, wait and prevent ++ * the power manager from powering down the l2 caches. ++ * ++ * This tells the power management that the caches should be powered up, and ++ * they should remain powered, irrespective of the usage of shader cores. This ++ * does not return until the l2 caches are powered up. ++ * ++ * The caller must call kbase_pm_release_l2_caches() when they are finished ++ * to allow normal power management of the l2 caches to resume. ++ * ++ * This should only be used when power management is active. ++ */ ++void kbase_pm_request_l2_caches(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_request_l2_caches_l2_is_on - Request l2 caches but don't power on ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * Increment the count of l2 users but do not attempt to power on the l2 ++ * ++ * It is the callers responsibility to ensure that the l2 is already powered up ++ * and to eventually call kbase_pm_release_l2_caches() ++ */ ++void kbase_pm_request_l2_caches_l2_is_on(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_request_l2_caches - Release l2 caches ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * Release the use of l2 caches for all core groups and allow the power manager ++ * to power them down when necessary. ++ * ++ * This tells the power management that the caches can be powered down if ++ * necessary, with respect to the usage of shader cores. ++ * ++ * The caller must have called kbase_pm_request_l2_caches() prior to a call ++ * to this. ++ * ++ * This should only be used when power management is active. ++ */ ++void kbase_pm_release_l2_caches(struct kbase_device *kbdev); ++ ++#endif /* _KBASE_PM_POLICY_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_time.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_time.c +new file mode 100755 +index 000000000000..0068e1091f4c +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_time.c +@@ -0,0 +1,103 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include ++#include ++#include ++#include ++ ++void kbase_backend_get_gpu_time(struct kbase_device *kbdev, u64 *cycle_counter, ++ u64 *system_time, struct timespec *ts) ++{ ++ u32 hi1, hi2; ++ ++ kbase_pm_request_gpu_cycle_counter(kbdev); ++ ++ /* Read hi, lo, hi to ensure that overflow from lo to hi is handled ++ * correctly */ ++ do { ++ hi1 = kbase_reg_read(kbdev, GPU_CONTROL_REG(CYCLE_COUNT_HI), ++ NULL); ++ *cycle_counter = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(CYCLE_COUNT_LO), NULL); ++ hi2 = kbase_reg_read(kbdev, GPU_CONTROL_REG(CYCLE_COUNT_HI), ++ NULL); ++ *cycle_counter |= (((u64) hi1) << 32); ++ } while (hi1 != hi2); ++ ++ /* Read hi, lo, hi to ensure that overflow from lo to hi is handled ++ * correctly */ ++ do { ++ hi1 = kbase_reg_read(kbdev, GPU_CONTROL_REG(TIMESTAMP_HI), ++ NULL); ++ *system_time = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(TIMESTAMP_LO), NULL); ++ hi2 = kbase_reg_read(kbdev, GPU_CONTROL_REG(TIMESTAMP_HI), ++ NULL); ++ *system_time |= (((u64) hi1) << 32); ++ } while (hi1 != hi2); ++ ++ /* Record the CPU's idea of current time */ ++ getrawmonotonic(ts); ++ ++ kbase_pm_release_gpu_cycle_counter(kbdev); ++} ++ ++/** ++ * kbase_wait_write_flush - Wait for GPU write flush ++ * @kctx: Context pointer ++ * ++ * Wait 1000 GPU clock cycles. This delay is known to give the GPU time to flush ++ * its write buffer. ++ * ++ * Only in use for BASE_HW_ISSUE_6367 ++ * ++ * Note : If GPU resets occur then the counters are reset to zero, the delay may ++ * not be as expected. ++ */ ++#ifndef CONFIG_MALI_BIFROST_NO_MALI ++void kbase_wait_write_flush(struct kbase_context *kctx) ++{ ++ u32 base_count = 0; ++ ++ /* ++ * The caller must be holding onto the kctx or the call is from ++ * userspace. ++ */ ++ kbase_pm_context_active(kctx->kbdev); ++ kbase_pm_request_gpu_cycle_counter(kctx->kbdev); ++ ++ while (true) { ++ u32 new_count; ++ ++ new_count = kbase_reg_read(kctx->kbdev, ++ GPU_CONTROL_REG(CYCLE_COUNT_LO), NULL); ++ /* First time around, just store the count. */ ++ if (base_count == 0) { ++ base_count = new_count; ++ continue; ++ } ++ ++ /* No need to handle wrapping, unsigned maths works for this. */ ++ if ((new_count - base_count) > 1000) ++ break; ++ } ++ ++ kbase_pm_release_gpu_cycle_counter(kctx->kbdev); ++ kbase_pm_context_idle(kctx->kbdev); ++} ++#endif /* CONFIG_MALI_BIFROST_NO_MALI */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_time.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_time.h +new file mode 100755 +index 000000000000..0559b2f7097d +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_time.h +@@ -0,0 +1,52 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KBASE_BACKEND_TIME_H_ ++#define _KBASE_BACKEND_TIME_H_ ++ ++/** ++ * kbase_backend_get_gpu_time() - Get current GPU time ++ * @kbdev: Device pointer ++ * @cycle_counter: Pointer to u64 to store cycle counter in ++ * @system_time: Pointer to u64 to store system time in ++ * @ts: Pointer to struct timespec to store current monotonic ++ * time in ++ */ ++void kbase_backend_get_gpu_time(struct kbase_device *kbdev, u64 *cycle_counter, ++ u64 *system_time, struct timespec *ts); ++ ++/** ++ * kbase_wait_write_flush() - Wait for GPU write flush ++ * @kctx: Context pointer ++ * ++ * Wait 1000 GPU clock cycles. This delay is known to give the GPU time to flush ++ * its write buffer. ++ * ++ * If GPU resets occur then the counters are reset to zero, the delay may not be ++ * as expected. ++ * ++ * This function is only in use for BASE_HW_ISSUE_6367 ++ */ ++#ifdef CONFIG_MALI_BIFROST_NO_MALI ++static inline void kbase_wait_write_flush(struct kbase_context *kctx) ++{ ++} ++#else ++void kbase_wait_write_flush(struct kbase_context *kctx); ++#endif ++ ++#endif /* _KBASE_BACKEND_TIME_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/docs/Doxyfile b/drivers/gpu/arm/bifrost_for_linux/docs/Doxyfile +new file mode 100755 +index 000000000000..35ff2f1ce4a0 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/docs/Doxyfile +@@ -0,0 +1,126 @@ ++# ++# (C) COPYRIGHT 2011-2013, 2015 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++ ++############################################################################## ++ ++# This file contains per-module Doxygen configuration. Please do not add ++# extra settings to this file without consulting all stakeholders, as they ++# may cause override project-wide settings. ++# ++# Additionally, when defining aliases, macros, sections etc, use the module ++# name as a prefix e.g. gles_my_alias. ++ ++############################################################################## ++ ++@INCLUDE = ../../bldsys/Doxyfile_common ++ ++# The INPUT tag can be used to specify the files and/or directories that contain ++# documented source files. You may enter file names like "myfile.cpp" or ++# directories like "/usr/src/myproject". Separate the files or directories ++# with spaces. ++ ++INPUT += ../../kernel/drivers/gpu/arm/midgard/ ++ ++############################################################################## ++# Everything below here is optional, and in most cases not required ++############################################################################## ++ ++# This tag can be used to specify a number of aliases that acts ++# as commands in the documentation. An alias has the form "name=value". ++# For example adding "sideeffect=\par Side Effects:\n" will allow you to ++# put the command \sideeffect (or @sideeffect) in the documentation, which ++# will result in a user-defined paragraph with heading "Side Effects:". ++# You can put \n's in the value part of an alias to insert newlines. ++ ++ALIASES += ++ ++# The ENABLED_SECTIONS tag can be used to enable conditional ++# documentation sections, marked by \if sectionname ... \endif. ++ ++ENABLED_SECTIONS += ++ ++# If the value of the INPUT tag contains directories, you can use the ++# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp ++# and *.h) to filter out the source-files in the directories. If left ++# blank the following patterns are tested: ++# *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx ++# *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.py *.f90 ++ ++FILE_PATTERNS += ++ ++# The EXCLUDE tag can be used to specify files and/or directories that should ++# excluded from the INPUT source files. This way you can easily exclude a ++# subdirectory from a directory tree whose root is specified with the INPUT tag. ++EXCLUDE += ../../kernel/drivers/gpu/arm/midgard/platform ../../kernel/drivers/gpu/arm/midgard/platform_dummy ../../kernel/drivers/gpu/arm/midgard/scripts ../../kernel/drivers/gpu/arm/midgard/tests ../../kernel/drivers/gpu/arm/midgard/Makefile ../../kernel/drivers/gpu/arm/midgard/Makefile.kbase ../../kernel/drivers/gpu/arm/midgard/Kbuild ../../kernel/drivers/gpu/arm/midgard/Kconfig ../../kernel/drivers/gpu/arm/midgard/sconscript ../../kernel/drivers/gpu/arm/midgard/docs ../../kernel/drivers/gpu/arm/midgard/pm_test_script.sh ../../kernel/drivers/gpu/arm/midgard/mali_uk.h ../../kernel/drivers/gpu/arm/midgard/Makefile ++ ++ ++# If the value of the INPUT tag contains directories, you can use the ++# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude ++# certain files from those directories. Note that the wildcards are matched ++# against the file with absolute path, so to exclude all test directories ++# for example use the pattern */test/* ++ ++EXCLUDE_PATTERNS += ++ ++# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names ++# (namespaces, classes, functions, etc.) that should be excluded from the ++# output. The symbol name can be a fully qualified name, a word, or if the ++# wildcard * is used, a substring. Examples: ANamespace, AClass, ++# AClass::ANamespace, ANamespace::*Test ++ ++EXCLUDE_SYMBOLS += ++ ++# The EXAMPLE_PATH tag can be used to specify one or more files or ++# directories that contain example code fragments that are included (see ++# the \include command). ++ ++EXAMPLE_PATH += ++ ++# The IMAGE_PATH tag can be used to specify one or more files or ++# directories that contain image that are included in the documentation (see ++# the \image command). ++ ++IMAGE_PATH += ++ ++# The INCLUDE_PATH tag can be used to specify one or more directories that ++# contain include files that are not input files but should be processed by ++# the preprocessor. ++ ++INCLUDE_PATH += ++ ++# The PREDEFINED tag can be used to specify one or more macro names that ++# are defined before the preprocessor is started (similar to the -D option of ++# gcc). The argument of the tag is a list of macros of the form: name ++# or name=definition (no spaces). If the definition and the = are ++# omitted =1 is assumed. To prevent a macro definition from being ++# undefined via #undef or recursively expanded use the := operator ++# instead of the = operator. ++ ++PREDEFINED += ++ ++# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then ++# this tag can be used to specify a list of macro names that should be expanded. ++# The macro definition that is found in the sources will be used. ++# Use the PREDEFINED tag if you want to use a different macro definition. ++ ++EXPAND_AS_DEFINED += ++ ++# The DOTFILE_DIRS tag can be used to specify one or more directories that ++# contain dot files that are included in the documentation (see the ++# \dotfile command). ++ ++DOTFILE_DIRS += ../../kernel/drivers/gpu/arm/midgard/docs ++ +diff --git a/drivers/gpu/arm/bifrost_for_linux/docs/policy_operation_diagram.dot b/drivers/gpu/arm/bifrost_for_linux/docs/policy_operation_diagram.dot +new file mode 100755 +index 000000000000..7ae05c2f8ded +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/docs/policy_operation_diagram.dot +@@ -0,0 +1,112 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++digraph policy_objects_diagram { ++ rankdir=LR; ++ size="12,8"; ++ compound=true; ++ ++ node [ shape = box ]; ++ ++ subgraph cluster_policy_queues { ++ low_queue [ shape=record label = "LowP | {ctx_lo | ... | ctx_i | ... | ctx_hi}" ]; ++ queues_middle_sep [ label="" shape=plaintext width=0 height=0 ]; ++ ++ rt_queue [ shape=record label = "RT | {ctx_lo | ... | ctx_j | ... | ctx_hi}" ]; ++ ++ label = "Policy's Queue(s)"; ++ } ++ ++ call_enqueue [ shape=plaintext label="enqueue_ctx()" ]; ++ ++ { ++ rank=same; ++ ordering=out; ++ call_dequeue [ shape=plaintext label="dequeue_head_ctx()\n+ runpool_add_ctx()" ]; ++ call_ctxfinish [ shape=plaintext label="runpool_remove_ctx()" ]; ++ ++ call_ctxdone [ shape=plaintext label="don't requeue;\n/* ctx has no more jobs */" ]; ++ } ++ ++ subgraph cluster_runpool { ++ ++ as0 [ width=2 height = 0.25 label="AS0: Job_1, ..., Job_n" ]; ++ as1 [ width=2 height = 0.25 label="AS1: Job_1, ..., Job_m" ]; ++ as2 [ width=2 height = 0.25 label="AS2: Job_1, ..., Job_p" ]; ++ as3 [ width=2 height = 0.25 label="AS3: Job_1, ..., Job_q" ]; ++ ++ label = "Policy's Run Pool"; ++ } ++ ++ { ++ rank=same; ++ call_jdequeue [ shape=plaintext label="dequeue_job()" ]; ++ sstop_dotfixup [ shape=plaintext label="" width=0 height=0 ]; ++ } ++ ++ { ++ rank=same; ++ ordering=out; ++ sstop [ shape=ellipse label="SS-Timer expires" ] ++ jobslots [ shape=record label="Jobslots: | <0>js[0] | <1>js[1] | <2>js[2]" ]; ++ ++ irq [ label="IRQ" shape=ellipse ]; ++ ++ job_finish [ shape=plaintext label="don't requeue;\n/* job done */" ]; ++ } ++ ++ hstop [ shape=ellipse label="HS-Timer expires" ] ++ ++ /* ++ * Edges ++ */ ++ ++ call_enqueue -> queues_middle_sep [ lhead=cluster_policy_queues ]; ++ ++ low_queue:qr -> call_dequeue:w; ++ rt_queue:qr -> call_dequeue:w; ++ ++ call_dequeue -> as1 [lhead=cluster_runpool]; ++ ++ as1->call_jdequeue [ltail=cluster_runpool]; ++ call_jdequeue->jobslots:0; ++ call_jdequeue->sstop_dotfixup [ arrowhead=none]; ++ sstop_dotfixup->sstop [label="Spawn SS-Timer"]; ++ sstop->jobslots [label="SoftStop"]; ++ sstop->hstop [label="Spawn HS-Timer"]; ++ hstop->jobslots:ne [label="HardStop"]; ++ ++ ++ as3->call_ctxfinish:ne [ ltail=cluster_runpool ]; ++ call_ctxfinish:sw->rt_queue:qm [ lhead=cluster_policy_queues label="enqueue_ctx()\n/* ctx still has jobs */" ]; ++ ++ call_ctxfinish->call_ctxdone [constraint=false]; ++ ++ call_ctxdone->call_enqueue [weight=0.1 labeldistance=20.0 labelangle=0.0 taillabel="Job submitted to the ctx" style=dotted constraint=false]; ++ ++ ++ { ++ jobslots->irq [constraint=false]; ++ ++ irq->job_finish [constraint=false]; ++ } ++ ++ irq->as2 [lhead=cluster_runpool label="requeue_job()\n/* timeslice expired */" ]; ++ ++} +diff --git a/drivers/gpu/arm/bifrost_for_linux/docs/policy_overview.dot b/drivers/gpu/arm/bifrost_for_linux/docs/policy_overview.dot +new file mode 100755 +index 000000000000..159b993b7d61 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/docs/policy_overview.dot +@@ -0,0 +1,63 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++digraph policy_objects_diagram { ++ rankdir=LR ++ size="6,6" ++ compound=true; ++ ++ node [ shape = box ]; ++ ++ call_enqueue [ shape=plaintext label="enqueue ctx" ]; ++ ++ ++ policy_queue [ label="Policy's Queue" ]; ++ ++ { ++ rank=same; ++ runpool [ label="Policy's Run Pool" ]; ++ ++ ctx_finish [ label="ctx finished" ]; ++ } ++ ++ { ++ rank=same; ++ jobslots [ shape=record label="Jobslots: | <0>js[0] | <1>js[1] | <2>js[2]" ]; ++ ++ job_finish [ label="Job finished" ]; ++ } ++ ++ ++ ++ /* ++ * Edges ++ */ ++ ++ call_enqueue -> policy_queue; ++ ++ policy_queue->runpool [label="dequeue ctx" weight=0.1]; ++ runpool->policy_queue [label="requeue ctx" weight=0.1]; ++ ++ runpool->ctx_finish [ style=dotted ]; ++ ++ runpool->jobslots [label="dequeue job" weight=0.1]; ++ jobslots->runpool [label="requeue job" weight=0.1]; ++ ++ jobslots->job_finish [ style=dotted ]; ++} +diff --git a/drivers/gpu/arm/bifrost_for_linux/ipa/Kbuild b/drivers/gpu/arm/bifrost_for_linux/ipa/Kbuild +new file mode 100755 +index 000000000000..0776428fce4f +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/ipa/Kbuild +@@ -0,0 +1,27 @@ ++# ++# (C) COPYRIGHT 2016-2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++bifrost_kbase-y += \ ++ ipa/mali_kbase_ipa_simple.o \ ++ ipa/mali_kbase_ipa.o ++ ++bifrost_kbase-$(CONFIG_DEBUG_FS) += ipa/mali_kbase_ipa_debugfs.o ++ ++ifneq ($(wildcard $(srctree)/$(src)/ipa/mali_kbase_ipa_vinstr_g71.c),) ++ bifrost_kbase-y += \ ++ ipa/mali_kbase_ipa_vinstr_g71.o \ ++ ipa/mali_kbase_ipa_vinstr_common.o ++ ++endif +diff --git a/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa.c b/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa.c +new file mode 100755 +index 000000000000..d6332b55e970 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa.c +@@ -0,0 +1,580 @@ ++/* ++ * ++ * (C) COPYRIGHT 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++#include ++#include ++#include ++#include "mali_kbase.h" ++#include "mali_kbase_ipa.h" ++#include "mali_kbase_ipa_debugfs.h" ++#include "mali_kbase_ipa_simple.h" ++ ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0)) ++#include ++#else ++#include ++#define dev_pm_opp_find_freq_exact opp_find_freq_exact ++#define dev_pm_opp_get_voltage opp_get_voltage ++#define dev_pm_opp opp ++#endif ++ ++#define KBASE_IPA_FALLBACK_MODEL_NAME "mali-simple-power-model" ++#define KBASE_IPA_G71_MODEL_NAME "mali-g71-power-model" ++ ++static struct kbase_ipa_model_ops *kbase_ipa_all_model_ops[] = { ++ &kbase_simple_ipa_model_ops, ++ &kbase_g71_ipa_model_ops ++}; ++ ++int kbase_ipa_model_recalculate(struct kbase_ipa_model *model) ++{ ++ int err = 0; ++ ++ lockdep_assert_held(&model->kbdev->ipa.lock); ++ ++ if (model->ops->recalculate) { ++ err = model->ops->recalculate(model); ++ if (err) { ++ dev_err(model->kbdev->dev, ++ "recalculation of power model %s returned error %d\n", ++ model->ops->name, err); ++ } ++ } ++ ++ return err; ++} ++ ++static struct kbase_ipa_model_ops *kbase_ipa_model_ops_find(struct kbase_device *kbdev, ++ const char *name) ++{ ++ int i; ++ ++ for (i = 0; i < ARRAY_SIZE(kbase_ipa_all_model_ops); ++i) { ++ struct kbase_ipa_model_ops *ops = kbase_ipa_all_model_ops[i]; ++ ++ if (!strcmp(ops->name, name)) ++ return ops; ++ } ++ ++ dev_err(kbdev->dev, "power model \'%s\' not found\n", name); ++ ++ return NULL; ++} ++ ++void kbase_ipa_model_use_fallback_locked(struct kbase_device *kbdev) ++{ ++ atomic_set(&kbdev->ipa_use_configured_model, false); ++} ++ ++void kbase_ipa_model_use_configured_locked(struct kbase_device *kbdev) ++{ ++ atomic_set(&kbdev->ipa_use_configured_model, true); ++} ++ ++const char *kbase_ipa_model_name_from_id(u32 gpu_id) ++{ ++ const u32 prod_id = (gpu_id & GPU_ID_VERSION_PRODUCT_ID) >> ++ GPU_ID_VERSION_PRODUCT_ID_SHIFT; ++ ++ if (GPU_ID_IS_NEW_FORMAT(prod_id)) { ++ switch (GPU_ID2_MODEL_MATCH_VALUE(prod_id)) { ++ case GPU_ID2_PRODUCT_TMIX: ++ return KBASE_IPA_G71_MODEL_NAME; ++ default: ++ return KBASE_IPA_FALLBACK_MODEL_NAME; ++ } ++ } ++ ++ return KBASE_IPA_FALLBACK_MODEL_NAME; ++} ++ ++static struct device_node *get_model_dt_node(struct kbase_ipa_model *model) ++{ ++ struct device_node *model_dt_node; ++ char compat_string[64]; ++ ++ snprintf(compat_string, sizeof(compat_string), "arm,%s", ++ model->ops->name); ++ ++ model_dt_node = of_find_compatible_node(model->kbdev->dev->of_node, ++ NULL, compat_string); ++ if (!model_dt_node && !model->missing_dt_node_warning) { ++ dev_warn(model->kbdev->dev, ++ "Couldn't find power_model DT node matching \'%s\'\n", ++ compat_string); ++ model->missing_dt_node_warning = true; ++ } ++ ++ return model_dt_node; ++} ++ ++int kbase_ipa_model_add_param_s32(struct kbase_ipa_model *model, ++ const char *name, s32 *addr, ++ size_t num_elems, bool dt_required) ++{ ++ int err, i; ++ struct device_node *model_dt_node = get_model_dt_node(model); ++ char *origin; ++ ++ err = of_property_read_u32_array(model_dt_node, name, addr, num_elems); ++ ++ if (err && dt_required) { ++ memset(addr, 0, sizeof(s32) * num_elems); ++ dev_warn(model->kbdev->dev, ++ "Error %d, no DT entry: %s.%s = %zu*[0]\n", ++ err, model->ops->name, name, num_elems); ++ origin = "zero"; ++ } else if (err && !dt_required) { ++ origin = "default"; ++ } else /* !err */ { ++ origin = "DT"; ++ } ++ ++ /* Create a unique debugfs entry for each element */ ++ for (i = 0; i < num_elems; ++i) { ++ char elem_name[32]; ++ ++ if (num_elems == 1) ++ snprintf(elem_name, sizeof(elem_name), "%s", name); ++ else ++ snprintf(elem_name, sizeof(elem_name), "%s.%d", ++ name, i); ++ ++ dev_dbg(model->kbdev->dev, "%s.%s = %d (%s)\n", ++ model->ops->name, elem_name, addr[i], origin); ++ ++ err = kbase_ipa_model_param_add(model, elem_name, ++ &addr[i], sizeof(s32), ++ PARAM_TYPE_S32); ++ if (err) ++ goto exit; ++ } ++exit: ++ return err; ++} ++ ++int kbase_ipa_model_add_param_string(struct kbase_ipa_model *model, ++ const char *name, char *addr, ++ size_t size, bool dt_required) ++{ ++ int err; ++ struct device_node *model_dt_node = get_model_dt_node(model); ++ const char *string_prop_value; ++ char *origin; ++ ++ err = of_property_read_string(model_dt_node, name, ++ &string_prop_value); ++ if (err && dt_required) { ++ strncpy(addr, "", size - 1); ++ dev_warn(model->kbdev->dev, ++ "Error %d, no DT entry: %s.%s = \'%s\'\n", ++ err, model->ops->name, name, addr); ++ err = 0; ++ origin = "zero"; ++ } else if (err && !dt_required) { ++ origin = "default"; ++ } else /* !err */ { ++ strncpy(addr, string_prop_value, size - 1); ++ origin = "DT"; ++ } ++ ++ addr[size - 1] = '\0'; ++ ++ dev_dbg(model->kbdev->dev, "%s.%s = \'%s\' (%s)\n", ++ model->ops->name, name, string_prop_value, origin); ++ ++ err = kbase_ipa_model_param_add(model, name, addr, size, ++ PARAM_TYPE_STRING); ++ ++ return err; ++} ++ ++void kbase_ipa_term_model(struct kbase_ipa_model *model) ++{ ++ if (!model) ++ return; ++ ++ lockdep_assert_held(&model->kbdev->ipa.lock); ++ ++ if (model->ops->term) ++ model->ops->term(model); ++ ++ kbase_ipa_model_param_free_all(model); ++ ++ kfree(model); ++} ++KBASE_EXPORT_TEST_API(kbase_ipa_term_model); ++ ++struct kbase_ipa_model *kbase_ipa_init_model(struct kbase_device *kbdev, ++ struct kbase_ipa_model_ops *ops) ++{ ++ struct kbase_ipa_model *model; ++ int err; ++ ++ lockdep_assert_held(&kbdev->ipa.lock); ++ ++ if (!ops || !ops->name) ++ return NULL; ++ ++ model = kzalloc(sizeof(struct kbase_ipa_model), GFP_KERNEL); ++ if (!model) ++ return NULL; ++ ++ model->kbdev = kbdev; ++ model->ops = ops; ++ INIT_LIST_HEAD(&model->params); ++ ++ err = model->ops->init(model); ++ if (err) { ++ dev_err(kbdev->dev, ++ "init of power model \'%s\' returned error %d\n", ++ ops->name, err); ++ kfree(model); ++ return NULL; ++ } ++ ++ err = kbase_ipa_model_recalculate(model); ++ if (err) { ++ kbase_ipa_term_model(model); ++ return NULL; ++ } ++ ++ return model; ++} ++KBASE_EXPORT_TEST_API(kbase_ipa_init_model); ++ ++static void kbase_ipa_term_locked(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->ipa.lock); ++ ++ /* Clean up the models */ ++ if (kbdev->ipa.configured_model != kbdev->ipa.fallback_model) ++ kbase_ipa_term_model(kbdev->ipa.configured_model); ++ kbase_ipa_term_model(kbdev->ipa.fallback_model); ++ ++ kbdev->ipa.configured_model = NULL; ++ kbdev->ipa.fallback_model = NULL; ++} ++ ++int kbase_ipa_init(struct kbase_device *kbdev) ++{ ++ ++ const char *model_name; ++ struct kbase_ipa_model_ops *ops; ++ struct kbase_ipa_model *default_model = NULL; ++ int err; ++ ++ mutex_init(&kbdev->ipa.lock); ++ /* ++ * Lock during init to avoid warnings from lockdep_assert_held (there ++ * shouldn't be any concurrent access yet). ++ */ ++ mutex_lock(&kbdev->ipa.lock); ++ ++ /* The simple IPA model must *always* be present.*/ ++ ops = kbase_ipa_model_ops_find(kbdev, KBASE_IPA_FALLBACK_MODEL_NAME); ++ ++ if (!ops->do_utilization_scaling_in_framework) { ++ dev_err(kbdev->dev, ++ "Fallback IPA model %s should not account for utilization\n", ++ ops->name); ++ err = -EINVAL; ++ goto end; ++ } ++ ++ default_model = kbase_ipa_init_model(kbdev, ops); ++ if (!default_model) { ++ err = -EINVAL; ++ goto end; ++ } ++ ++ kbdev->ipa.fallback_model = default_model; ++ err = of_property_read_string(kbdev->dev->of_node, ++ "ipa-model", ++ &model_name); ++ if (err) { ++ /* Attempt to load a match from GPU-ID */ ++ u32 gpu_id; ++ ++ gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; ++ model_name = kbase_ipa_model_name_from_id(gpu_id); ++ dev_dbg(kbdev->dev, ++ "Inferring model from GPU ID 0x%x: \'%s\'\n", ++ gpu_id, model_name); ++ err = 0; ++ } else { ++ dev_dbg(kbdev->dev, ++ "Using ipa-model parameter from DT: \'%s\'\n", ++ model_name); ++ } ++ ++ if (strcmp(KBASE_IPA_FALLBACK_MODEL_NAME, model_name) != 0) { ++ ops = kbase_ipa_model_ops_find(kbdev, model_name); ++ kbdev->ipa.configured_model = kbase_ipa_init_model(kbdev, ops); ++ if (!kbdev->ipa.configured_model) { ++ err = -EINVAL; ++ goto end; ++ } ++ } else { ++ kbdev->ipa.configured_model = default_model; ++ } ++ ++ kbase_ipa_model_use_configured_locked(kbdev); ++ ++end: ++ if (err) ++ kbase_ipa_term_locked(kbdev); ++ else ++ dev_info(kbdev->dev, ++ "Using configured power model %s, and fallback %s\n", ++ kbdev->ipa.configured_model->ops->name, ++ kbdev->ipa.fallback_model->ops->name); ++ ++ mutex_unlock(&kbdev->ipa.lock); ++ return err; ++} ++KBASE_EXPORT_TEST_API(kbase_ipa_init); ++ ++void kbase_ipa_term(struct kbase_device *kbdev) ++{ ++ mutex_lock(&kbdev->ipa.lock); ++ kbase_ipa_term_locked(kbdev); ++ mutex_unlock(&kbdev->ipa.lock); ++} ++KBASE_EXPORT_TEST_API(kbase_ipa_term); ++ ++/** ++ * kbase_scale_dynamic_power() - Scale a dynamic power coefficient to an OPP ++ * @c: Dynamic model coefficient, in pW/(Hz V^2). Should be in range ++ * 0 < c < 2^26 to prevent overflow. ++ * @freq: Frequency, in Hz. Range: 2^23 < freq < 2^30 (~8MHz to ~1GHz) ++ * @voltage: Voltage, in mV. Range: 2^9 < voltage < 2^13 (~0.5V to ~8V) ++ * ++ * Keep a record of the approximate range of each value at every stage of the ++ * calculation, to ensure we don't overflow. This makes heavy use of the ++ * approximations 1000 = 2^10 and 1000000 = 2^20, but does the actual ++ * calculations in decimal for increased accuracy. ++ * ++ * Return: Power consumption, in mW. Range: 0 < p < 2^13 (0W to ~8W) ++ */ ++static u32 kbase_scale_dynamic_power(const u32 c, const u32 freq, ++ const u32 voltage) ++{ ++ /* Range: 2^8 < v2 < 2^16 m(V^2) */ ++ const u32 v2 = (voltage * voltage) / 1000; ++ ++ /* Range: 2^3 < f_MHz < 2^10 MHz */ ++ const u32 f_MHz = freq / 1000000; ++ ++ /* Range: 2^11 < v2f_big < 2^26 kHz V^2 */ ++ const u32 v2f_big = v2 * f_MHz; ++ ++ /* Range: 2^1 < v2f < 2^16 MHz V^2 */ ++ const u32 v2f = v2f_big / 1000; ++ ++ /* Range (working backwards from next line): 0 < v2fc < 2^23 uW. ++ * Must be < 2^42 to avoid overflowing the return value. */ ++ const u64 v2fc = (u64) c * (u64) v2f; ++ ++ /* Range: 0 < v2fc / 1000 < 2^13 mW */ ++ return v2fc / 1000; ++} ++ ++/** ++ * kbase_scale_static_power() - Scale a static power coefficient to an OPP ++ * @c: Static model coefficient, in uW/V^3. Should be in range ++ * 0 < c < 2^32 to prevent overflow. ++ * @voltage: Voltage, in mV. Range: 2^9 < voltage < 2^13 (~0.5V to ~8V) ++ * ++ * Return: Power consumption, in mW. Range: 0 < p < 2^13 (0W to ~8W) ++ */ ++u32 kbase_scale_static_power(const u32 c, const u32 voltage) ++{ ++ /* Range: 2^8 < v2 < 2^16 m(V^2) */ ++ const u32 v2 = (voltage * voltage) / 1000; ++ ++ /* Range: 2^17 < v3_big < 2^29 m(V^2) mV */ ++ const u32 v3_big = v2 * voltage; ++ ++ /* Range: 2^7 < v3 < 2^19 m(V^3) */ ++ const u32 v3 = v3_big / 1000; ++ ++ /* ++ * Range (working backwards from next line): 0 < v3c_big < 2^33 nW. ++ * The result should be < 2^52 to avoid overflowing the return value. ++ */ ++ const u64 v3c_big = (u64) c * (u64) v3; ++ ++ /* Range: 0 < v3c_big / 1000000 < 2^13 mW */ ++ return v3c_big / 1000000; ++} ++ ++static struct kbase_ipa_model *get_current_model(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->ipa.lock); ++ ++ if (atomic_read(&kbdev->ipa_use_configured_model)) ++ return kbdev->ipa.configured_model; ++ else ++ return kbdev->ipa.fallback_model; ++} ++ ++static u32 get_static_power_locked(struct kbase_device *kbdev, ++ struct kbase_ipa_model *model, ++ unsigned long voltage) ++{ ++ u32 power = 0; ++ int err; ++ u32 power_coeff; ++ ++ lockdep_assert_held(&model->kbdev->ipa.lock); ++ ++ if (!model->ops->get_static_coeff) ++ model = kbdev->ipa.fallback_model; ++ ++ if (model->ops->get_static_coeff) { ++ err = model->ops->get_static_coeff(model, &power_coeff); ++ if (!err) ++ power = kbase_scale_static_power(power_coeff, ++ (u32) voltage); ++ } ++ ++ return power; ++} ++ ++#ifdef CONFIG_MALI_PWRSOFT_765 ++static unsigned long kbase_get_static_power(struct devfreq *df, ++ unsigned long voltage) ++#else ++static unsigned long kbase_get_static_power(unsigned long voltage) ++#endif ++{ ++ struct kbase_ipa_model *model; ++ u32 power = 0; ++#ifdef CONFIG_MALI_PWRSOFT_765 ++ struct kbase_device *kbdev = dev_get_drvdata(&df->dev); ++#else ++ struct kbase_device *kbdev = kbase_find_device(-1); ++#endif ++ ++ mutex_lock(&kbdev->ipa.lock); ++ ++ model = get_current_model(kbdev); ++ power = get_static_power_locked(kbdev, model, voltage); ++ ++ mutex_unlock(&kbdev->ipa.lock); ++ ++#ifndef CONFIG_MALI_PWRSOFT_765 ++ kbase_release_device(kbdev); ++#endif ++ ++ return power; ++} ++ ++#ifdef CONFIG_MALI_PWRSOFT_765 ++static unsigned long kbase_get_dynamic_power(struct devfreq *df, ++ unsigned long freq, ++ unsigned long voltage) ++#else ++static unsigned long kbase_get_dynamic_power(unsigned long freq, ++ unsigned long voltage) ++#endif ++{ ++ struct kbase_ipa_model *model; ++ u32 power_coeff = 0, power = 0; ++ int err = 0; ++#ifdef CONFIG_MALI_PWRSOFT_765 ++ struct kbase_device *kbdev = dev_get_drvdata(&df->dev); ++#else ++ struct kbase_device *kbdev = kbase_find_device(-1); ++#endif ++ ++ mutex_lock(&kbdev->ipa.lock); ++ ++ model = kbdev->ipa.fallback_model; ++ ++ err = model->ops->get_dynamic_coeff(model, &power_coeff, freq); ++ ++ if (!err) ++ power = kbase_scale_dynamic_power(power_coeff, freq, voltage); ++ else ++ dev_err_ratelimited(kbdev->dev, ++ "Model %s returned error code %d\n", ++ model->ops->name, err); ++ ++ mutex_unlock(&kbdev->ipa.lock); ++ ++#ifndef CONFIG_MALI_PWRSOFT_765 ++ kbase_release_device(kbdev); ++#endif ++ ++ return power; ++} ++ ++int kbase_get_real_power(struct devfreq *df, u32 *power, ++ unsigned long freq, ++ unsigned long voltage) ++{ ++ struct kbase_ipa_model *model; ++ u32 power_coeff = 0; ++ int err = 0; ++ struct kbase_device *kbdev = dev_get_drvdata(&df->dev); ++ ++ mutex_lock(&kbdev->ipa.lock); ++ ++ model = get_current_model(kbdev); ++ ++ err = model->ops->get_dynamic_coeff(model, &power_coeff, freq); ++ ++ /* If we switch to protected model between get_current_model() and ++ * get_dynamic_coeff(), counter reading could fail. If that happens ++ * (unlikely, but possible), revert to the fallback model. */ ++ if (err && model != kbdev->ipa.fallback_model) { ++ model = kbdev->ipa.fallback_model; ++ err = model->ops->get_dynamic_coeff(model, &power_coeff, freq); ++ } ++ ++ if (err) ++ goto exit_unlock; ++ ++ *power = kbase_scale_dynamic_power(power_coeff, freq, voltage); ++ ++ if (model->ops->do_utilization_scaling_in_framework) { ++ struct devfreq_dev_status *status = &df->last_status; ++ unsigned long total_time = max(status->total_time, 1ul); ++ u64 busy_time = min(status->busy_time, total_time); ++ ++ *power = ((u64) *power * (u64) busy_time) / total_time; ++ } ++ ++ *power += get_static_power_locked(kbdev, model, voltage); ++ ++exit_unlock: ++ mutex_unlock(&kbdev->ipa.lock); ++ ++ return err; ++} ++KBASE_EXPORT_TEST_API(kbase_get_real_power); ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) ++struct devfreq_cooling_ops kbase_ipa_power_model_ops = { ++#else ++struct devfreq_cooling_power kbase_ipa_power_model_ops = { ++#endif ++ .get_static_power = &kbase_get_static_power, ++ .get_dynamic_power = &kbase_get_dynamic_power, ++}; ++KBASE_EXPORT_TEST_API(kbase_ipa_power_model_ops); +diff --git a/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa.h b/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa.h +new file mode 100755 +index 000000000000..67478fe911ea +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa.h +@@ -0,0 +1,165 @@ ++/* ++ * ++ * (C) COPYRIGHT 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KBASE_IPA_H_ ++#define _KBASE_IPA_H_ ++ ++#if defined(CONFIG_MALI_BIFROST_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL) ++ ++struct devfreq; ++ ++struct kbase_ipa_model { ++ struct list_head link; ++ struct kbase_device *kbdev; ++ void *model_data; ++ struct kbase_ipa_model_ops *ops; ++ struct list_head params; ++ bool missing_dt_node_warning; ++}; ++ ++/** ++ * kbase_ipa_model_add_param_s32 - Add an integer model parameter ++ * @model: pointer to IPA model ++ * @name: name of corresponding debugfs entry ++ * @addr: address where the value is stored ++ * @num_elems: number of elements (1 if not an array) ++ * @dt_required: if false, a corresponding devicetree entry is not required, ++ * and the current value will be used. If true, a warning is ++ * output and the data is zeroed ++ * ++ * Return: 0 on success, or an error code ++ */ ++int kbase_ipa_model_add_param_s32(struct kbase_ipa_model *model, ++ const char *name, s32 *addr, ++ size_t num_elems, bool dt_required); ++ ++/** ++ * kbase_ipa_model_add_param_string - Add a string model parameter ++ * @model: pointer to IPA model ++ * @name: name of corresponding debugfs entry ++ * @addr: address where the value is stored ++ * @size: size, in bytes, of the value storage (so the maximum string ++ * length is size - 1) ++ * @dt_required: if false, a corresponding devicetree entry is not required, ++ * and the current value will be used. If true, a warning is ++ * output and the data is zeroed ++ * ++ * Return: 0 on success, or an error code ++ */ ++int kbase_ipa_model_add_param_string(struct kbase_ipa_model *model, ++ const char *name, char *addr, ++ size_t size, bool dt_required); ++ ++struct kbase_ipa_model_ops { ++ char *name; ++ /* The init, recalculate and term ops on the default model are always ++ * called. However, all the other models are only invoked if the model ++ * is selected in the device tree. Otherwise they are never ++ * initialized. Additional resources can be acquired by models in ++ * init(), however they must be terminated in the term(). ++ */ ++ int (*init)(struct kbase_ipa_model *model); ++ /* Called immediately after init(), or when a parameter is changed, so ++ * that any coefficients derived from model parameters can be ++ * recalculated. */ ++ int (*recalculate)(struct kbase_ipa_model *model); ++ void (*term)(struct kbase_ipa_model *model); ++ /* ++ * get_dynamic_coeff() - calculate dynamic power coefficient ++ * @model: pointer to model ++ * @coeffp: pointer to return value location ++ * @current_freq: frequency the GPU has been running at for the ++ * previous sampling period. ++ * ++ * Calculate a dynamic power coefficient, with units pW/(Hz V^2), which ++ * is then scaled by the IPA framework according to the current OPP's ++ * frequency and voltage. ++ * ++ * Return: 0 on success, or an error code. ++ */ ++ int (*get_dynamic_coeff)(struct kbase_ipa_model *model, u32 *coeffp, ++ u32 current_freq); ++ /* ++ * get_static_coeff() - calculate static power coefficient ++ * @model: pointer to model ++ * @coeffp: pointer to return value location ++ * ++ * Calculate a static power coefficient, with units uW/(V^3), which is ++ * scaled by the IPA framework according to the current OPP's voltage. ++ * ++ * Return: 0 on success, or an error code. ++ */ ++ int (*get_static_coeff)(struct kbase_ipa_model *model, u32 *coeffp); ++ /* If false, the model's get_dynamic_coeff() method accounts for how ++ * long the GPU was active over the sample period. If true, the ++ * framework will scale the calculated power according to the ++ * utilization stats recorded by devfreq in get_real_power(). */ ++ bool do_utilization_scaling_in_framework; ++}; ++ ++/* Models can be registered only in the platform's platform_init_func call */ ++int kbase_ipa_model_ops_register(struct kbase_device *kbdev, ++ struct kbase_ipa_model_ops *new_model_ops); ++struct kbase_ipa_model *kbase_ipa_get_model(struct kbase_device *kbdev, ++ const char *name); ++ ++int kbase_ipa_init(struct kbase_device *kbdev); ++void kbase_ipa_term(struct kbase_device *kbdev); ++void kbase_ipa_model_use_fallback_locked(struct kbase_device *kbdev); ++void kbase_ipa_model_use_configured_locked(struct kbase_device *kbdev); ++int kbase_ipa_model_recalculate(struct kbase_ipa_model *model); ++struct kbase_ipa_model *kbase_ipa_init_model(struct kbase_device *kbdev, ++ struct kbase_ipa_model_ops *ops); ++void kbase_ipa_term_model(struct kbase_ipa_model *model); ++ ++extern struct kbase_ipa_model_ops kbase_g71_ipa_model_ops; ++ ++#if MALI_UNIT_TEST ++/** ++ * kbase_get_real_power() - get the real power consumption of the GPU ++ * @df: dynamic voltage and frequency scaling information for the GPU. ++ * @power: where to store the power consumption, in mW. ++ * @freq: a frequency, in HZ. ++ * @voltage: a voltage, in mV. ++ * ++ * This function is only exposed for use by unit tests. The returned value ++ * incorporates both static and dynamic power consumption. ++ * ++ * Return: 0 on success, or an error code. ++ */ ++int kbase_get_real_power(struct devfreq *df, u32 *power, ++ unsigned long freq, ++ unsigned long voltage); ++#endif /* MALI_UNIT_TEST */ ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) ++extern struct devfreq_cooling_ops kbase_ipa_power_model_ops; ++#else ++extern struct devfreq_cooling_power kbase_ipa_power_model_ops; ++#endif ++ ++#else /* !(defined(CONFIG_MALI_BIFROST_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL)) */ ++ ++static inline void kbase_ipa_model_use_fallback_locked(struct kbase_device *kbdev) ++{ } ++ ++static inline void kbase_ipa_model_use_configured_locked(struct kbase_device *kbdev) ++{ } ++ ++#endif /* (defined(CONFIG_MALI_BIFROST_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL)) */ ++ ++#endif +diff --git a/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_debugfs.c b/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_debugfs.c +new file mode 100755 +index 000000000000..eafc14009ddc +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_debugfs.c +@@ -0,0 +1,219 @@ ++/* ++ * ++ * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include ++#include ++#include ++ ++#include "mali_kbase.h" ++#include "mali_kbase_ipa.h" ++#include "mali_kbase_ipa_debugfs.h" ++ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0)) ++#define DEFINE_DEBUGFS_ATTRIBUTE DEFINE_SIMPLE_ATTRIBUTE ++#endif ++ ++struct kbase_ipa_model_param { ++ char *name; ++ union { ++ void *voidp; ++ s32 *s32p; ++ char *str; ++ } addr; ++ size_t size; ++ enum kbase_ipa_model_param_type type; ++ struct kbase_ipa_model *model; ++ struct list_head link; ++}; ++ ++static int param_int_get(void *data, u64 *val) ++{ ++ struct kbase_ipa_model_param *param = data; ++ ++ mutex_lock(¶m->model->kbdev->ipa.lock); ++ *(s64 *) val = *param->addr.s32p; ++ mutex_unlock(¶m->model->kbdev->ipa.lock); ++ ++ return 0; ++} ++ ++static int param_int_set(void *data, u64 val) ++{ ++ struct kbase_ipa_model_param *param = data; ++ struct kbase_ipa_model *model = param->model; ++ s64 sval = (s64) val; ++ int err = 0; ++ ++ if (sval < S32_MIN || sval > S32_MAX) ++ return -ERANGE; ++ ++ mutex_lock(¶m->model->kbdev->ipa.lock); ++ *param->addr.s32p = val; ++ err = kbase_ipa_model_recalculate(model); ++ mutex_unlock(¶m->model->kbdev->ipa.lock); ++ ++ return err; ++} ++ ++DEFINE_DEBUGFS_ATTRIBUTE(fops_s32, param_int_get, param_int_set, "%lld\n"); ++ ++static ssize_t param_string_get(struct file *file, char __user *user_buf, ++ size_t count, loff_t *ppos) ++{ ++ struct kbase_ipa_model_param *param = file->private_data; ++ ssize_t ret; ++ size_t len; ++ ++ mutex_lock(¶m->model->kbdev->ipa.lock); ++ len = strnlen(param->addr.str, param->size - 1) + 1; ++ ret = simple_read_from_buffer(user_buf, count, ppos, ++ param->addr.str, len); ++ mutex_unlock(¶m->model->kbdev->ipa.lock); ++ ++ return ret; ++} ++ ++static ssize_t param_string_set(struct file *file, const char __user *user_buf, ++ size_t count, loff_t *ppos) ++{ ++ struct kbase_ipa_model_param *param = file->private_data; ++ struct kbase_ipa_model *model = param->model; ++ ssize_t ret = count; ++ size_t buf_size; ++ int err; ++ ++ mutex_lock(&model->kbdev->ipa.lock); ++ ++ if (count > param->size) { ++ ret = -EINVAL; ++ goto end; ++ } ++ ++ buf_size = min(param->size - 1, count); ++ if (copy_from_user(param->addr.str, user_buf, buf_size)) { ++ ret = -EFAULT; ++ goto end; ++ } ++ ++ param->addr.str[buf_size] = '\0'; ++ ++ err = kbase_ipa_model_recalculate(model); ++ if (err < 0) ++ ret = err; ++ ++end: ++ mutex_unlock(&model->kbdev->ipa.lock); ++ ++ return ret; ++} ++ ++static const struct file_operations fops_string = { ++ .read = param_string_get, ++ .write = param_string_set, ++ .open = simple_open, ++ .llseek = default_llseek, ++}; ++ ++int kbase_ipa_model_param_add(struct kbase_ipa_model *model, const char *name, ++ void *addr, size_t size, ++ enum kbase_ipa_model_param_type type) ++{ ++ struct kbase_ipa_model_param *param; ++ ++ param = kzalloc(sizeof(*param), GFP_KERNEL); ++ ++ if (!param) ++ return -ENOMEM; ++ ++ /* 'name' is stack-allocated for array elements, so copy it into ++ * heap-allocated storage */ ++ param->name = kstrdup(name, GFP_KERNEL); ++ param->addr.voidp = addr; ++ param->size = size; ++ param->type = type; ++ param->model = model; ++ ++ list_add(¶m->link, &model->params); ++ ++ return 0; ++} ++ ++void kbase_ipa_model_param_free_all(struct kbase_ipa_model *model) ++{ ++ struct kbase_ipa_model_param *param_p, *param_n; ++ ++ list_for_each_entry_safe(param_p, param_n, &model->params, link) { ++ list_del(¶m_p->link); ++ kfree(param_p->name); ++ kfree(param_p); ++ } ++} ++ ++static void kbase_ipa_model_debugfs_init(struct kbase_ipa_model *model) ++{ ++ struct list_head *it; ++ struct dentry *dir; ++ ++ lockdep_assert_held(&model->kbdev->ipa.lock); ++ ++ dir = debugfs_create_dir(model->ops->name, ++ model->kbdev->mali_debugfs_directory); ++ ++ if (!dir) { ++ dev_err(model->kbdev->dev, ++ "Couldn't create mali debugfs %s directory", ++ model->ops->name); ++ return; ++ } ++ ++ list_for_each(it, &model->params) { ++ struct kbase_ipa_model_param *param = ++ list_entry(it, ++ struct kbase_ipa_model_param, ++ link); ++ const struct file_operations *fops = NULL; ++ ++ switch (param->type) { ++ case PARAM_TYPE_S32: ++ fops = &fops_s32; ++ break; ++ case PARAM_TYPE_STRING: ++ fops = &fops_string; ++ break; ++ } ++ ++ if (unlikely(!fops)) { ++ dev_err(model->kbdev->dev, ++ "Type not set for %s parameter %s\n", ++ model->ops->name, param->name); ++ } else { ++ debugfs_create_file(param->name, S_IRUGO | S_IWUSR, ++ dir, param, fops); ++ } ++ } ++} ++ ++void kbase_ipa_debugfs_init(struct kbase_device *kbdev) ++{ ++ mutex_lock(&kbdev->ipa.lock); ++ ++ if (kbdev->ipa.configured_model != kbdev->ipa.fallback_model) ++ kbase_ipa_model_debugfs_init(kbdev->ipa.configured_model); ++ kbase_ipa_model_debugfs_init(kbdev->ipa.fallback_model); ++ ++ mutex_unlock(&kbdev->ipa.lock); ++} +diff --git a/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_debugfs.h b/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_debugfs.h +new file mode 100755 +index 000000000000..ec06e2096f94 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_debugfs.h +@@ -0,0 +1,49 @@ ++/* ++ * ++ * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KBASE_IPA_DEBUGFS_H_ ++#define _KBASE_IPA_DEBUGFS_H_ ++ ++enum kbase_ipa_model_param_type { ++ PARAM_TYPE_S32 = 1, ++ PARAM_TYPE_STRING, ++}; ++ ++#ifdef CONFIG_DEBUG_FS ++ ++void kbase_ipa_debugfs_init(struct kbase_device *kbdev); ++int kbase_ipa_model_param_add(struct kbase_ipa_model *model, const char *name, ++ void *addr, size_t size, ++ enum kbase_ipa_model_param_type type); ++void kbase_ipa_model_param_free_all(struct kbase_ipa_model *model); ++ ++#else /* CONFIG_DEBUG_FS */ ++ ++static inline int kbase_ipa_model_param_add(struct kbase_ipa_model *model, ++ const char *name, void *addr, ++ size_t size, ++ enum kbase_ipa_model_param_type type) ++{ ++ return 0; ++} ++ ++static inline void kbase_ipa_model_param_free_all(struct kbase_ipa_model *model) ++{ } ++ ++#endif /* CONFIG_DEBUG_FS */ ++ ++#endif /* _KBASE_IPA_DEBUGFS_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_simple.c b/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_simple.c +new file mode 100755 +index 000000000000..b35cea451765 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_simple.c +@@ -0,0 +1,327 @@ ++/* ++ * ++ * (C) COPYRIGHT 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include ++#include ++#ifdef CONFIG_DEVFREQ_THERMAL ++#include ++#endif ++#include ++#include ++#include ++ ++#include "mali_kbase.h" ++#include "mali_kbase_defs.h" ++#include "mali_kbase_ipa_simple.h" ++#include "mali_kbase_ipa_debugfs.h" ++ ++#if MALI_UNIT_TEST ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) ++static unsigned long dummy_temp; ++ ++static int kbase_simple_power_model_get_dummy_temp( ++ struct thermal_zone_device *tz, ++ unsigned long *temp) ++{ ++ *temp = READ_ONCE(dummy_temp); ++ return 0; ++} ++ ++#else ++static int dummy_temp; ++ ++static int kbase_simple_power_model_get_dummy_temp( ++ struct thermal_zone_device *tz, ++ int *dummy_temp) ++{ ++ *temp = READ_ONCE(dummy_temp); ++ return 0; ++} ++#endif ++ ++/* Intercept calls to the kernel function using a macro */ ++#ifdef thermal_zone_get_temp ++#undef thermal_zone_get_temp ++#endif ++#define thermal_zone_get_temp(tz, temp) \ ++ kbase_simple_power_model_get_dummy_temp(tz, temp) ++ ++void kbase_simple_power_model_set_dummy_temp(int temp) ++{ ++ WRITE_ONCE(dummy_temp, temp); ++} ++KBASE_EXPORT_TEST_API(kbase_simple_power_model_set_dummy_temp); ++ ++#endif /* MALI_UNIT_TEST */ ++ ++/* ++ * This model is primarily designed for the Juno platform. It may not be ++ * suitable for other platforms. The additional resources in this model ++ * should preferably be minimal, as this model is rarely used when a dynamic ++ * model is available. ++ */ ++ ++/** ++ * struct kbase_ipa_model_simple_data - IPA context per device ++ * @dynamic_coefficient: dynamic coefficient of the model ++ * @static_coefficient: static coefficient of the model ++ * @ts: Thermal scaling coefficients of the model ++ * @tz_name: Thermal zone name ++ * @gpu_tz: thermal zone device ++ * @poll_temperature_thread: Handle for temperature polling thread ++ * @current_temperature: Most recent value of polled temperature ++ * @temperature_poll_interval_ms: How often temperature should be checked, in ms ++ */ ++ ++struct kbase_ipa_model_simple_data { ++ u32 dynamic_coefficient; ++ u32 static_coefficient; ++ s32 ts[4]; ++ char tz_name[16]; ++ struct thermal_zone_device *gpu_tz; ++ struct task_struct *poll_temperature_thread; ++ int current_temperature; ++ int temperature_poll_interval_ms; ++}; ++#define FALLBACK_STATIC_TEMPERATURE 55000 ++ ++/** ++ * calculate_temp_scaling_factor() - Calculate temperature scaling coefficient ++ * @ts: Signed coefficients, in order t^0 to t^3, with units Deg^-N ++ * @t: Temperature, in mDeg C. Range: -2^17 < t < 2^17 ++ * ++ * Scale the temperature according to a cubic polynomial whose coefficients are ++ * provided in the device tree. The result is used to scale the static power ++ * coefficient, where 1000000 means no change. ++ * ++ * Return: Temperature scaling factor. Range 0 <= ret <= 10,000,000. ++ */ ++static u32 calculate_temp_scaling_factor(s32 ts[4], s64 t) ++{ ++ /* Range: -2^24 < t2 < 2^24 m(Deg^2) */ ++ const s64 t2 = (t * t) / 1000; ++ ++ /* Range: -2^31 < t3 < 2^31 m(Deg^3) */ ++ const s64 t3 = (t * t2) / 1000; ++ ++ /* ++ * Sum the parts. t^[1-3] are in m(Deg^N), but the coefficients are in ++ * Deg^-N, so we need to multiply the last coefficient by 1000. ++ * Range: -2^63 < res_big < 2^63 ++ */ ++ const s64 res_big = ts[3] * t3 /* +/- 2^62 */ ++ + ts[2] * t2 /* +/- 2^55 */ ++ + ts[1] * t /* +/- 2^48 */ ++ + ts[0] * 1000; /* +/- 2^41 */ ++ ++ /* Range: -2^60 < res_unclamped < 2^60 */ ++ s64 res_unclamped = res_big / 1000; ++ ++ /* Clamp to range of 0x to 10x the static power */ ++ return clamp(res_unclamped, (s64) 0, (s64) 10000000); ++} ++ ++/* We can't call thermal_zone_get_temp() directly in model_static_coeff(), ++ * because we don't know if tz->lock is held in the same thread. So poll it in ++ * a separate thread to get around this. */ ++static int poll_temperature(void *data) ++{ ++ struct kbase_ipa_model_simple_data *model_data = ++ (struct kbase_ipa_model_simple_data *) data; ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) ++ unsigned long temp; ++#else ++ int temp; ++#endif ++ ++ set_freezable(); ++ ++ while (!kthread_should_stop()) { ++ struct thermal_zone_device *tz = READ_ONCE(model_data->gpu_tz); ++ ++ if (tz) { ++ int ret; ++ ++ ret = thermal_zone_get_temp(tz, &temp); ++ if (ret) { ++ pr_warn_ratelimited("Error reading temperature for gpu thermal zone: %d\n", ++ ret); ++ temp = FALLBACK_STATIC_TEMPERATURE; ++ } ++ } else { ++ temp = FALLBACK_STATIC_TEMPERATURE; ++ } ++ ++ WRITE_ONCE(model_data->current_temperature, temp); ++ ++ msleep_interruptible(READ_ONCE(model_data->temperature_poll_interval_ms)); ++ ++ try_to_freeze(); ++ } ++ ++ return 0; ++} ++ ++static int model_static_coeff(struct kbase_ipa_model *model, u32 *coeffp) ++{ ++ u32 temp_scaling_factor; ++ struct kbase_ipa_model_simple_data *model_data = ++ (struct kbase_ipa_model_simple_data *) model->model_data; ++ u64 coeff_big; ++ int temp; ++ ++ temp = READ_ONCE(model_data->current_temperature); ++ ++ /* Range: 0 <= temp_scaling_factor < 2^24 */ ++ temp_scaling_factor = calculate_temp_scaling_factor(model_data->ts, ++ temp); ++ ++ /* ++ * Range: 0 <= coeff_big < 2^52 to avoid overflowing *coeffp. This ++ * means static_coefficient must be in range ++ * 0 <= static_coefficient < 2^28. ++ */ ++ coeff_big = (u64) model_data->static_coefficient * (u64) temp_scaling_factor; ++ *coeffp = coeff_big / 1000000; ++ ++ return 0; ++} ++ ++static int model_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp, ++ u32 current_freq) ++{ ++ struct kbase_ipa_model_simple_data *model_data = ++ (struct kbase_ipa_model_simple_data *) model->model_data; ++ ++ *coeffp = model_data->dynamic_coefficient; ++ ++ return 0; ++} ++ ++static int add_params(struct kbase_ipa_model *model) ++{ ++ int err = 0; ++ struct kbase_ipa_model_simple_data *model_data = ++ (struct kbase_ipa_model_simple_data *)model->model_data; ++ ++ err = kbase_ipa_model_add_param_s32(model, "static-coefficient", ++ &model_data->static_coefficient, ++ 1, true); ++ if (err) ++ goto end; ++ ++ err = kbase_ipa_model_add_param_s32(model, "dynamic-coefficient", ++ &model_data->dynamic_coefficient, ++ 1, true); ++ if (err) ++ goto end; ++ ++ err = kbase_ipa_model_add_param_s32(model, "ts", ++ model_data->ts, 4, true); ++ if (err) ++ goto end; ++ ++ err = kbase_ipa_model_add_param_string(model, "thermal-zone", ++ model_data->tz_name, ++ sizeof(model_data->tz_name), true); ++ if (err) ++ goto end; ++ ++ model_data->temperature_poll_interval_ms = 200; ++ err = kbase_ipa_model_add_param_s32(model, "temp-poll-interval-ms", ++ &model_data->temperature_poll_interval_ms, ++ 1, false); ++ ++end: ++ return err; ++} ++ ++static int kbase_simple_power_model_init(struct kbase_ipa_model *model) ++{ ++ int err; ++ struct kbase_ipa_model_simple_data *model_data; ++ ++ model_data = kzalloc(sizeof(struct kbase_ipa_model_simple_data), ++ GFP_KERNEL); ++ if (!model_data) ++ return -ENOMEM; ++ ++ model->model_data = (void *) model_data; ++ ++ model_data->current_temperature = FALLBACK_STATIC_TEMPERATURE; ++ model_data->poll_temperature_thread = kthread_run(poll_temperature, ++ (void *) model_data, ++ "mali-simple-power-model-temp-poll"); ++ if (IS_ERR(model_data->poll_temperature_thread)) { ++ kfree(model_data); ++ return PTR_ERR(model_data->poll_temperature_thread); ++ } ++ ++ err = add_params(model); ++ if (err) { ++ kbase_ipa_model_param_free_all(model); ++ kthread_stop(model_data->poll_temperature_thread); ++ kfree(model_data); ++ } ++ ++ return err; ++} ++ ++static int kbase_simple_power_model_recalculate(struct kbase_ipa_model *model) ++{ ++ struct kbase_ipa_model_simple_data *model_data = ++ (struct kbase_ipa_model_simple_data *)model->model_data; ++ struct thermal_zone_device *tz; ++ ++ if (!strnlen(model_data->tz_name, sizeof(model_data->tz_name))) { ++ tz = NULL; ++ } else { ++ tz = thermal_zone_get_zone_by_name(model_data->tz_name); ++ ++ if (IS_ERR_OR_NULL(tz)) { ++ pr_warn_ratelimited("Error %ld getting thermal zone \'%s\', not yet ready?\n", ++ PTR_ERR(tz), model_data->tz_name); ++ tz = NULL; ++ return -EPROBE_DEFER; ++ } ++ } ++ ++ WRITE_ONCE(model_data->gpu_tz, tz); ++ ++ return 0; ++} ++ ++static void kbase_simple_power_model_term(struct kbase_ipa_model *model) ++{ ++ struct kbase_ipa_model_simple_data *model_data = ++ (struct kbase_ipa_model_simple_data *)model->model_data; ++ ++ kthread_stop(model_data->poll_temperature_thread); ++ ++ kfree(model_data); ++} ++ ++struct kbase_ipa_model_ops kbase_simple_ipa_model_ops = { ++ .name = "mali-simple-power-model", ++ .init = &kbase_simple_power_model_init, ++ .recalculate = &kbase_simple_power_model_recalculate, ++ .term = &kbase_simple_power_model_term, ++ .get_dynamic_coeff = &model_dynamic_coeff, ++ .get_static_coeff = &model_static_coeff, ++ .do_utilization_scaling_in_framework = true, ++}; +diff --git a/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_simple.h b/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_simple.h +new file mode 100755 +index 000000000000..23cd55f5867d +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_simple.h +@@ -0,0 +1,40 @@ ++/* ++ * ++ * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KBASE_IPA_SIMPLE_H_ ++#define _KBASE_IPA_SIMPLE_H_ ++ ++#if defined(CONFIG_MALI_BIFROST_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL) ++ ++extern struct kbase_ipa_model_ops kbase_simple_ipa_model_ops; ++ ++#if MALI_UNIT_TEST ++/** ++ * kbase_simple_power_model_set_dummy_temp() - set a dummy temperature value ++ * @temp: Temperature of the thermal zone, in millidegrees celsius. ++ * ++ * This is only intended for use in unit tests, to ensure that the temperature ++ * values used by the simple power model are predictable. Deterministic ++ * behavior is necessary to allow validation of the static power values ++ * computed by this model. ++ */ ++void kbase_simple_power_model_set_dummy_temp(int temp); ++#endif /* MALI_UNIT_TEST */ ++ ++#endif /* (defined(CONFIG_MALI_BIFROST_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL)) */ ++ ++#endif /* _KBASE_IPA_SIMPLE_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_vinstr_common.c b/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_vinstr_common.c +new file mode 100755 +index 000000000000..b3d480030c2b +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_vinstr_common.c +@@ -0,0 +1,217 @@ ++/* ++ * ++ * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include "mali_kbase_ipa_vinstr_common.h" ++ ++#if MALI_UNIT_TEST ++static ktime_t dummy_time; ++ ++/* Intercept calls to the kernel function using a macro */ ++#ifdef ktime_get ++#undef ktime_get ++#endif ++#define ktime_get() (READ_ONCE(dummy_time)) ++ ++void kbase_ipa_set_dummy_time(ktime_t t) ++{ ++ WRITE_ONCE(dummy_time, t); ++} ++KBASE_EXPORT_TEST_API(kbase_ipa_set_dummy_time); ++ ++#endif /* MALI_UNIT_TEST */ ++ ++/** ++ * read_hwcnt() - read a counter value ++ * @model_data: pointer to model data ++ * @offset: offset, in bytes, into vinstr buffer ++ * ++ * Return: A 32-bit counter value. Range: 0 < value < 2^27 (worst case would be ++ * incrementing every cycle over a ~100ms sample period at a high frequency, ++ * e.g. 1 GHz: 2^30 * 0.1seconds ~= 2^27. ++ */ ++static inline u32 kbase_ipa_read_hwcnt( ++ struct kbase_ipa_model_vinstr_data *model_data, ++ u32 offset) ++{ ++ u8 *p = model_data->vinstr_buffer; ++ ++ return *(u32 *)&p[offset]; ++} ++ ++static inline s64 kbase_ipa_add_saturate(s64 a, s64 b) ++{ ++ if (S64_MAX - a < b) ++ return S64_MAX; ++ return a + b; ++} ++ ++s64 kbase_ipa_sum_all_shader_cores( ++ struct kbase_ipa_model_vinstr_data *model_data, ++ s32 coeff, u32 counter) ++{ ++ struct kbase_device *kbdev = model_data->kbdev; ++ u64 core_mask; ++ u32 base = 0; ++ s64 ret = 0; ++ ++ core_mask = kbdev->gpu_props.props.coherency_info.group[0].core_mask; ++ while (core_mask != 0ull) { ++ if ((core_mask & 1ull) != 0ull) { ++ /* 0 < counter_value < 2^27 */ ++ u32 counter_value = kbase_ipa_read_hwcnt(model_data, ++ base + counter); ++ ++ /* 0 < ret < 2^27 * max_num_cores = 2^32 */ ++ ret = kbase_ipa_add_saturate(ret, counter_value); ++ } ++ base += KBASE_IPA_NR_BYTES_PER_BLOCK; ++ core_mask >>= 1; ++ } ++ ++ /* Range: -2^54 < ret < 2^54 */ ++ ret *= coeff; ++ ++ return ret / 1000000; ++} ++ ++s64 kbase_ipa_single_counter( ++ struct kbase_ipa_model_vinstr_data *model_data, ++ s32 coeff, u32 counter) ++{ ++ /* Range: 0 < counter_value < 2^27 */ ++ const u32 counter_value = kbase_ipa_read_hwcnt(model_data, counter); ++ ++ /* Range: -2^49 < ret < 2^49 */ ++ const s64 multiplied = (s64) counter_value * (s64) coeff; ++ ++ /* Range: -2^29 < return < 2^29 */ ++ return multiplied / 1000000; ++} ++ ++int kbase_ipa_attach_vinstr(struct kbase_ipa_model_vinstr_data *model_data) ++{ ++ struct kbase_device *kbdev = model_data->kbdev; ++ struct kbase_uk_hwcnt_reader_setup setup; ++ size_t dump_size; ++ ++ dump_size = kbase_vinstr_dump_size(kbdev); ++ model_data->vinstr_buffer = kzalloc(dump_size, GFP_KERNEL); ++ if (!model_data->vinstr_buffer) { ++ dev_err(kbdev->dev, "Failed to allocate IPA dump buffer"); ++ return -1; ++ } ++ ++ setup.jm_bm = ~0u; ++ setup.shader_bm = ~0u; ++ setup.tiler_bm = ~0u; ++ setup.mmu_l2_bm = ~0u; ++ model_data->vinstr_cli = kbase_vinstr_hwcnt_kernel_setup(kbdev->vinstr_ctx, ++ &setup, model_data->vinstr_buffer); ++ if (!model_data->vinstr_cli) { ++ dev_err(kbdev->dev, "Failed to register IPA with vinstr core"); ++ kfree(model_data->vinstr_buffer); ++ model_data->vinstr_buffer = NULL; ++ return -1; ++ } ++ ++ model_data->last_sample_read_time = ktime_get(); ++ kbase_vinstr_hwc_clear(model_data->vinstr_cli); ++ ++ return 0; ++} ++ ++void kbase_ipa_detach_vinstr(struct kbase_ipa_model_vinstr_data *model_data) ++{ ++ if (model_data->vinstr_cli) ++ kbase_vinstr_detach_client(model_data->vinstr_cli); ++ model_data->vinstr_cli = NULL; ++ kfree(model_data->vinstr_buffer); ++ model_data->vinstr_buffer = NULL; ++} ++ ++int kbase_ipa_vinstr_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp, ++ u32 current_freq) ++{ ++ struct kbase_ipa_model_vinstr_data *model_data = ++ (struct kbase_ipa_model_vinstr_data *)model->model_data; ++ s64 energy = 0; ++ size_t i; ++ ktime_t now = ktime_get(); ++ ktime_t time_since_last_sample = ++ ktime_sub(now, model_data->last_sample_read_time); ++ /* Range: 2^0 < time_since_last_sample_ms < 2^10 (1-1000ms) */ ++ s64 time_since_last_sample_ms = ktime_to_ms(time_since_last_sample); ++ u64 coeff = 0; ++ u64 num_cycles; ++ int err = 0; ++ ++ err = kbase_vinstr_hwc_dump(model_data->vinstr_cli, ++ BASE_HWCNT_READER_EVENT_MANUAL); ++ if (err) ++ goto err0; ++ ++ model_data->last_sample_read_time = now; ++ ++ /* Range of 'energy' is +/- 2^34 * number of IPA groups, so around ++ * -2^38 < energy < 2^38 */ ++ for (i = 0; i < model_data->groups_def_num; i++) { ++ const struct kbase_ipa_group *group = &model_data->groups_def[i]; ++ s32 coeff, group_energy; ++ ++ coeff = model_data->group_values[i]; ++ group_energy = group->op(model_data, coeff, group->counter); ++ ++ energy = kbase_ipa_add_saturate(energy, group_energy); ++ } ++ ++ /* Range: 0 <= coeff < 2^38 */ ++ if (energy > 0) ++ coeff = energy; ++ ++ /* Scale by user-specified factor and divide by 1000. But actually ++ * cancel the division out, because we want the num_cycles in KHz and ++ * don't want to lose precision. */ ++ ++ /* Range: 0 < coeff < 2^53 */ ++ coeff = coeff * model_data->scaling_factor; ++ ++ if (time_since_last_sample_ms == 0) { ++ time_since_last_sample_ms = 1; ++ } else if (time_since_last_sample_ms < 0) { ++ err = -ERANGE; ++ goto err0; ++ } ++ ++ /* Range: 2^20 < num_cycles < 2^40 mCycles */ ++ num_cycles = (u64) current_freq * (u64) time_since_last_sample_ms; ++ /* Range: 2^10 < num_cycles < 2^30 Cycles */ ++ num_cycles /= 1000000; ++ ++ /* num_cycles should never be 0 in _normal_ usage (because we expect ++ * frequencies on the order of MHz and >10ms polling intervals), but ++ * protect against divide-by-zero anyway. */ ++ if (num_cycles == 0) ++ num_cycles = 1; ++ ++ /* Range: 0 < coeff < 2^43 */ ++ coeff = div_u64(coeff, num_cycles); ++ ++err0: ++ /* Clamp to a sensible range - 2^16 gives about 14W at 400MHz/750mV */ ++ *coeffp = clamp(coeff, (u64) 0, (u64) 1 << 16); ++ return err; ++} +diff --git a/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_vinstr_common.h b/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_vinstr_common.h +new file mode 100755 +index 000000000000..25b36c8e3089 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_vinstr_common.h +@@ -0,0 +1,161 @@ ++/* ++ * ++ * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KBASE_IPA_VINSTR_COMMON_H_ ++#define _KBASE_IPA_VINSTR_COMMON_H_ ++ ++#include "mali_kbase.h" ++ ++/* Maximum length for the name of an IPA group. */ ++#define KBASE_IPA_MAX_GROUP_NAME_LEN 15 ++ ++/* Maximum number of IPA groups for an IPA model. */ ++#define KBASE_IPA_MAX_GROUP_DEF_NUM 16 ++ ++/* Number of bytes per hardware counter in a vinstr_buffer. */ ++#define KBASE_IPA_NR_BYTES_PER_CNT 4 ++ ++/* Number of hardware counters per block in a vinstr_buffer. */ ++#define KBASE_IPA_NR_CNT_PER_BLOCK 64 ++ ++/* Number of bytes per block in a vinstr_buffer. */ ++#define KBASE_IPA_NR_BYTES_PER_BLOCK \ ++ (KBASE_IPA_NR_CNT_PER_BLOCK * KBASE_IPA_NR_BYTES_PER_CNT) ++ ++ ++ ++/** ++ * struct kbase_ipa_model_vinstr_data - IPA context per device ++ * @kbdev: pointer to kbase device ++ * @groups_def: Array of IPA groups. ++ * @groups_def_num: Number of elements in the array of IPA groups. ++ * @vinstr_cli: vinstr client handle ++ * @vinstr_buffer: buffer to dump hardware counters onto ++ * @last_sample_read_time: timestamp of last vinstr buffer read ++ * @scaling_factor: user-specified power scaling factor. This is ++ * interpreted as a fraction where the denominator is ++ * 1000. Range approx 0.0-32.0: ++ * 0 < scaling_factor < 2^15 ++ */ ++struct kbase_ipa_model_vinstr_data { ++ struct kbase_device *kbdev; ++ s32 group_values[KBASE_IPA_MAX_GROUP_DEF_NUM]; ++ const struct kbase_ipa_group *groups_def; ++ size_t groups_def_num; ++ struct kbase_vinstr_client *vinstr_cli; ++ void *vinstr_buffer; ++ ktime_t last_sample_read_time; ++ s32 scaling_factor; ++}; ++ ++/** ++ * struct ipa_group - represents a single IPA group ++ * @name: name of the IPA group ++ * @default_value: default value of coefficient for IPA group. ++ * Coefficients are interpreted as fractions where the ++ * denominator is 1000000. ++ * @op: which operation to be performed on the counter values ++ * @counter: counter used to calculate energy for IPA group ++ */ ++struct kbase_ipa_group { ++ char name[KBASE_IPA_MAX_GROUP_NAME_LEN + 1]; ++ s32 default_value; ++ s64 (*op)(struct kbase_ipa_model_vinstr_data *, s32, u32); ++ u32 counter; ++}; ++ ++/* ++ * sum_all_shader_cores() - sum a counter over all cores ++ * @model_data pointer to model data ++ * @coeff model coefficient. Unity is ~2^20, so range approx ++ * +/- 4.0: -2^22 < coeff < 2^22 ++ ++ * Calculate energy estimation based on hardware counter `counter' ++ * across all shader cores. ++ * ++ * Return: Sum of counter values. Range: -2^34 < ret < 2^34 ++ */ ++s64 kbase_ipa_sum_all_shader_cores( ++ struct kbase_ipa_model_vinstr_data *model_data, ++ s32 coeff, u32 counter); ++ ++/* ++ * sum_single_counter() - sum a single counter ++ * @model_data pointer to model data ++ * @coeff model coefficient. Unity is ~2^20, so range approx ++ * +/- 4.0: -2^22 < coeff < 2^22 ++ ++ * Calculate energy estimation based on hardware counter `counter'. ++ * ++ * Return: Counter value. Range: -2^34 < ret < 2^34 ++ */ ++s64 kbase_ipa_single_counter( ++ struct kbase_ipa_model_vinstr_data *model_data, ++ s32 coeff, u32 counter); ++ ++/* ++ * attach_vinstr() - attach a vinstr_buffer to an IPA model. ++ * @model_data pointer to model data ++ * ++ * Attach a vinstr_buffer to an IPA model. The vinstr_buffer ++ * allows access to the hardware counters used to calculate ++ * energy consumption. ++ * ++ * Return: 0 on success, or an error code. ++ */ ++int kbase_ipa_attach_vinstr(struct kbase_ipa_model_vinstr_data *model_data); ++ ++/* ++ * detach_vinstr() - detach a vinstr_buffer from an IPA model. ++ * @model_data pointer to model data ++ * ++ * Detach a vinstr_buffer from an IPA model. ++ */ ++void kbase_ipa_detach_vinstr(struct kbase_ipa_model_vinstr_data *model_data); ++ ++/** ++ * kbase_ipa_vinstr_dynamic_coeff() - calculate dynamic power based on HW counters ++ * @model: pointer to instantiated model ++ * @coeffp: pointer to location where calculated power, in ++ * pW/(Hz V^2), is stored. ++ * @current_freq: frequency the GPU has been running at over the sample ++ * period. In Hz. Range: 10 MHz < 1GHz, ++ * 2^20 < current_freq < 2^30 ++ * ++ * This is a GPU-agnostic implementation of the get_dynamic_coeff() ++ * function of an IPA model. It relies on the model being populated ++ * with GPU-specific attributes at initialization time. ++ * ++ * Return: 0 on success, or an error code. ++ */ ++int kbase_ipa_vinstr_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp, ++ u32 current_freq); ++ ++#if MALI_UNIT_TEST ++/** ++ * kbase_ipa_set_dummy_time() - set a dummy monotonic time value ++ * @t: a monotonic time value ++ * ++ * This is only intended for use in unit tests, to ensure that the kernel time ++ * values used by a power model are predictable. Deterministic behavior is ++ * necessary to allow validation of the dynamic power values computed by the ++ * model. ++ */ ++void kbase_ipa_set_dummy_time(ktime_t t); ++#endif /* MALI_UNIT_TEST */ ++ ++#endif /* _KBASE_IPA_VINSTR_COMMON_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_vinstr_g71.c b/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_vinstr_g71.c +new file mode 100755 +index 000000000000..81f6fddbd79b +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_vinstr_g71.c +@@ -0,0 +1,136 @@ ++/* ++ * ++ * (C) COPYRIGHT 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++#include ++ ++#include "mali_kbase_ipa_vinstr_common.h" ++#include "mali_kbase.h" ++#include "mali_kbase_ipa_debugfs.h" ++ ++ ++#define JM_BASE (0 * KBASE_IPA_NR_BYTES_PER_BLOCK) ++#define TILER_BASE (1 * KBASE_IPA_NR_BYTES_PER_BLOCK) ++#define MMU_BASE (2 * KBASE_IPA_NR_BYTES_PER_BLOCK) ++#define SC0_BASE (3 * KBASE_IPA_NR_BYTES_PER_BLOCK) ++ ++#define GPU_ACTIVE (JM_BASE + KBASE_IPA_NR_BYTES_PER_CNT * 6) ++#define TILER_ACTIVE (TILER_BASE + KBASE_IPA_NR_BYTES_PER_CNT * 45) ++#define L2_ANY_LOOKUP (MMU_BASE + KBASE_IPA_NR_BYTES_PER_CNT * 25) ++#define FRAG_ACTIVE (SC0_BASE + KBASE_IPA_NR_BYTES_PER_CNT * 4) ++#define EXEC_CORE_ACTIVE (SC0_BASE + KBASE_IPA_NR_BYTES_PER_CNT * 26) ++#define EXEC_INSTR_COUNT (SC0_BASE + KBASE_IPA_NR_BYTES_PER_CNT * 28) ++#define TEX_COORD_ISSUE (SC0_BASE + KBASE_IPA_NR_BYTES_PER_CNT * 40) ++#define VARY_SLOT_32 (SC0_BASE + KBASE_IPA_NR_BYTES_PER_CNT * 50) ++#define VARY_SLOT_16 (SC0_BASE + KBASE_IPA_NR_BYTES_PER_CNT * 51) ++#define BEATS_RD_LSC (SC0_BASE + KBASE_IPA_NR_BYTES_PER_CNT * 56) ++#define BEATS_WR_LSC (SC0_BASE + KBASE_IPA_NR_BYTES_PER_CNT * 61) ++#define BEATS_WR_TIB (SC0_BASE + KBASE_IPA_NR_BYTES_PER_CNT * 62) ++ ++static const struct kbase_ipa_group ipa_groups_def[] = { ++ { ++ .name = "l2_access", ++ .default_value = 526300, ++ .op = kbase_ipa_single_counter, ++ .counter = L2_ANY_LOOKUP, ++ }, ++ { ++ .name = "exec_instr_count", ++ .default_value = 301100, ++ .op = kbase_ipa_sum_all_shader_cores, ++ .counter = EXEC_INSTR_COUNT, ++ }, ++ { ++ .name = "tex_issue", ++ .default_value = 197400, ++ .op = kbase_ipa_sum_all_shader_cores, ++ .counter = TEX_COORD_ISSUE, ++ }, ++ { ++ .name = "tile_wb", ++ .default_value = -156400, ++ .op = kbase_ipa_sum_all_shader_cores, ++ .counter = BEATS_WR_TIB, ++ }, ++ { ++ .name = "gpu_active", ++ .default_value = 115800, ++ .op = kbase_ipa_single_counter, ++ .counter = GPU_ACTIVE, ++ }, ++}; ++ ++static int kbase_g71_power_model_init(struct kbase_ipa_model *model) ++{ ++ int i, err = 0; ++ struct kbase_ipa_model_vinstr_data *model_data; ++ ++ model_data = kzalloc(sizeof(*model_data), GFP_KERNEL); ++ if (!model_data) ++ return -ENOMEM; ++ ++ model_data->kbdev = model->kbdev; ++ model_data->groups_def = ipa_groups_def; ++ BUILD_BUG_ON(ARRAY_SIZE(ipa_groups_def) > KBASE_IPA_MAX_GROUP_DEF_NUM); ++ model_data->groups_def_num = ARRAY_SIZE(ipa_groups_def); ++ ++ model->model_data = (void *) model_data; ++ ++ for (i = 0; i < ARRAY_SIZE(ipa_groups_def); ++i) { ++ const struct kbase_ipa_group *group = &ipa_groups_def[i]; ++ ++ model_data->group_values[i] = group->default_value; ++ err = kbase_ipa_model_add_param_s32(model, group->name, ++ &model_data->group_values[i], ++ 1, false); ++ if (err) ++ goto exit; ++ } ++ ++ model_data->scaling_factor = 15000; ++ err = kbase_ipa_model_add_param_s32(model, "scale", ++ &model_data->scaling_factor, ++ 1, false); ++ if (err) ++ goto exit; ++ ++ err = kbase_ipa_attach_vinstr(model_data); ++ ++exit: ++ if (err) { ++ kbase_ipa_model_param_free_all(model); ++ kfree(model_data); ++ } ++ return err; ++} ++ ++static void kbase_g71_power_model_term(struct kbase_ipa_model *model) ++{ ++ struct kbase_ipa_model_vinstr_data *model_data = ++ (struct kbase_ipa_model_vinstr_data *)model->model_data; ++ ++ kbase_ipa_detach_vinstr(model_data); ++ kfree(model_data); ++} ++ ++ ++struct kbase_ipa_model_ops kbase_g71_ipa_model_ops = { ++ .name = "mali-g71-power-model", ++ .init = kbase_g71_power_model_init, ++ .term = kbase_g71_power_model_term, ++ .get_dynamic_coeff = kbase_ipa_vinstr_dynamic_coeff, ++ .do_utilization_scaling_in_framework = false, ++}; ++KBASE_EXPORT_TEST_API(kbase_g71_ipa_model_ops); +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_base_hwconfig_features.h b/drivers/gpu/arm/bifrost_for_linux/mali_base_hwconfig_features.h +new file mode 100755 +index 000000000000..219586d4d2da +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_base_hwconfig_features.h +@@ -0,0 +1,282 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* AUTOMATICALLY GENERATED FILE. If you want to amend the issues/features, ++ * please update base/tools/hwconfig_generator/hwc_{issues,features}.py ++ * For more information see base/tools/hwconfig_generator/README ++ */ ++ ++#ifndef _BASE_HWCONFIG_FEATURES_H_ ++#define _BASE_HWCONFIG_FEATURES_H_ ++ ++enum base_hw_feature { ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, ++ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, ++ BASE_HW_FEATURE_33BIT_VA, ++ BASE_HW_FEATURE_XAFFINITY, ++ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, ++ BASE_HW_FEATURE_MRT, ++ BASE_HW_FEATURE_BRNDOUT_CC, ++ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, ++ BASE_HW_FEATURE_LD_ST_TILEBUFFER, ++ BASE_HW_FEATURE_MSAA_16X, ++ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, ++ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, ++ BASE_HW_FEATURE_OPTIMIZED_COVERAGE_MASK, ++ BASE_HW_FEATURE_T7XX_PAIRING_RULES, ++ BASE_HW_FEATURE_LD_ST_LEA_TEX, ++ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, ++ BASE_HW_FEATURE_WORKGROUP_ROUND_MULTIPLE_OF_4, ++ BASE_HW_FEATURE_IMAGES_IN_FRAGMENT_SHADERS, ++ BASE_HW_FEATURE_TEST4_DATUM_MODE, ++ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, ++ BASE_HW_FEATURE_BRNDOUT_KILL, ++ BASE_HW_FEATURE_WARPING, ++ BASE_HW_FEATURE_V4, ++ BASE_HW_FEATURE_FLUSH_REDUCTION, ++ BASE_HW_FEATURE_PROTECTED_MODE, ++ BASE_HW_FEATURE_COHERENCY_REG, ++ BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, ++ BASE_HW_FEATURE_AARCH64_MMU, ++ BASE_HW_FEATURE_TLS_HASHING, ++ BASE_HW_FEATURE_END ++}; ++ ++static const enum base_hw_feature base_hw_features_generic[] = { ++ BASE_HW_FEATURE_END ++}; ++ ++static const enum base_hw_feature base_hw_features_t60x[] = { ++ BASE_HW_FEATURE_LD_ST_LEA_TEX, ++ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, ++ BASE_HW_FEATURE_V4, ++ BASE_HW_FEATURE_END ++}; ++ ++static const enum base_hw_feature base_hw_features_t62x[] = { ++ BASE_HW_FEATURE_LD_ST_LEA_TEX, ++ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, ++ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, ++ BASE_HW_FEATURE_V4, ++ BASE_HW_FEATURE_END ++}; ++ ++static const enum base_hw_feature base_hw_features_t72x[] = { ++ BASE_HW_FEATURE_33BIT_VA, ++ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, ++ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, ++ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, ++ BASE_HW_FEATURE_OPTIMIZED_COVERAGE_MASK, ++ BASE_HW_FEATURE_T7XX_PAIRING_RULES, ++ BASE_HW_FEATURE_WORKGROUP_ROUND_MULTIPLE_OF_4, ++ BASE_HW_FEATURE_WARPING, ++ BASE_HW_FEATURE_V4, ++ BASE_HW_FEATURE_END ++}; ++ ++static const enum base_hw_feature base_hw_features_t76x[] = { ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, ++ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, ++ BASE_HW_FEATURE_XAFFINITY, ++ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, ++ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, ++ BASE_HW_FEATURE_BRNDOUT_CC, ++ BASE_HW_FEATURE_LD_ST_LEA_TEX, ++ BASE_HW_FEATURE_LD_ST_TILEBUFFER, ++ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, ++ BASE_HW_FEATURE_MRT, ++ BASE_HW_FEATURE_MSAA_16X, ++ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, ++ BASE_HW_FEATURE_T7XX_PAIRING_RULES, ++ BASE_HW_FEATURE_TEST4_DATUM_MODE, ++ BASE_HW_FEATURE_END ++}; ++ ++static const enum base_hw_feature base_hw_features_tFxx[] = { ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, ++ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, ++ BASE_HW_FEATURE_XAFFINITY, ++ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, ++ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, ++ BASE_HW_FEATURE_BRNDOUT_CC, ++ BASE_HW_FEATURE_BRNDOUT_KILL, ++ BASE_HW_FEATURE_LD_ST_LEA_TEX, ++ BASE_HW_FEATURE_LD_ST_TILEBUFFER, ++ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, ++ BASE_HW_FEATURE_MRT, ++ BASE_HW_FEATURE_MSAA_16X, ++ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, ++ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, ++ BASE_HW_FEATURE_T7XX_PAIRING_RULES, ++ BASE_HW_FEATURE_TEST4_DATUM_MODE, ++ BASE_HW_FEATURE_END ++}; ++ ++static const enum base_hw_feature base_hw_features_t83x[] = { ++ BASE_HW_FEATURE_33BIT_VA, ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, ++ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, ++ BASE_HW_FEATURE_XAFFINITY, ++ BASE_HW_FEATURE_WARPING, ++ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, ++ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, ++ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, ++ BASE_HW_FEATURE_BRNDOUT_CC, ++ BASE_HW_FEATURE_BRNDOUT_KILL, ++ BASE_HW_FEATURE_LD_ST_LEA_TEX, ++ BASE_HW_FEATURE_LD_ST_TILEBUFFER, ++ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, ++ BASE_HW_FEATURE_MRT, ++ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, ++ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, ++ BASE_HW_FEATURE_T7XX_PAIRING_RULES, ++ BASE_HW_FEATURE_TEST4_DATUM_MODE, ++ BASE_HW_FEATURE_END ++}; ++ ++static const enum base_hw_feature base_hw_features_t82x[] = { ++ BASE_HW_FEATURE_33BIT_VA, ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, ++ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, ++ BASE_HW_FEATURE_XAFFINITY, ++ BASE_HW_FEATURE_WARPING, ++ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, ++ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, ++ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, ++ BASE_HW_FEATURE_BRNDOUT_CC, ++ BASE_HW_FEATURE_BRNDOUT_KILL, ++ BASE_HW_FEATURE_LD_ST_LEA_TEX, ++ BASE_HW_FEATURE_LD_ST_TILEBUFFER, ++ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, ++ BASE_HW_FEATURE_MRT, ++ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, ++ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, ++ BASE_HW_FEATURE_T7XX_PAIRING_RULES, ++ BASE_HW_FEATURE_TEST4_DATUM_MODE, ++ BASE_HW_FEATURE_END ++}; ++ ++static const enum base_hw_feature base_hw_features_tMIx[] = { ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, ++ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, ++ BASE_HW_FEATURE_XAFFINITY, ++ BASE_HW_FEATURE_WARPING, ++ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, ++ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, ++ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, ++ BASE_HW_FEATURE_BRNDOUT_CC, ++ BASE_HW_FEATURE_BRNDOUT_KILL, ++ BASE_HW_FEATURE_LD_ST_LEA_TEX, ++ BASE_HW_FEATURE_LD_ST_TILEBUFFER, ++ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, ++ BASE_HW_FEATURE_MRT, ++ BASE_HW_FEATURE_MSAA_16X, ++ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, ++ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, ++ BASE_HW_FEATURE_T7XX_PAIRING_RULES, ++ BASE_HW_FEATURE_TEST4_DATUM_MODE, ++ BASE_HW_FEATURE_FLUSH_REDUCTION, ++ BASE_HW_FEATURE_PROTECTED_MODE, ++ BASE_HW_FEATURE_COHERENCY_REG, ++ BASE_HW_FEATURE_END ++}; ++ ++static const enum base_hw_feature base_hw_features_tHEx[] = { ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, ++ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, ++ BASE_HW_FEATURE_XAFFINITY, ++ BASE_HW_FEATURE_WARPING, ++ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, ++ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, ++ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, ++ BASE_HW_FEATURE_BRNDOUT_CC, ++ BASE_HW_FEATURE_BRNDOUT_KILL, ++ BASE_HW_FEATURE_LD_ST_LEA_TEX, ++ BASE_HW_FEATURE_LD_ST_TILEBUFFER, ++ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, ++ BASE_HW_FEATURE_MRT, ++ BASE_HW_FEATURE_MSAA_16X, ++ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, ++ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, ++ BASE_HW_FEATURE_T7XX_PAIRING_RULES, ++ BASE_HW_FEATURE_TEST4_DATUM_MODE, ++ BASE_HW_FEATURE_FLUSH_REDUCTION, ++ BASE_HW_FEATURE_PROTECTED_MODE, ++ BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, ++ BASE_HW_FEATURE_COHERENCY_REG, ++ BASE_HW_FEATURE_END ++}; ++ ++static const enum base_hw_feature base_hw_features_tSIx[] = { ++ BASE_HW_FEATURE_33BIT_VA, ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, ++ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, ++ BASE_HW_FEATURE_XAFFINITY, ++ BASE_HW_FEATURE_WARPING, ++ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, ++ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, ++ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, ++ BASE_HW_FEATURE_BRNDOUT_CC, ++ BASE_HW_FEATURE_BRNDOUT_KILL, ++ BASE_HW_FEATURE_LD_ST_LEA_TEX, ++ BASE_HW_FEATURE_LD_ST_TILEBUFFER, ++ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, ++ BASE_HW_FEATURE_MRT, ++ BASE_HW_FEATURE_MSAA_16X, ++ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, ++ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, ++ BASE_HW_FEATURE_T7XX_PAIRING_RULES, ++ BASE_HW_FEATURE_TEST4_DATUM_MODE, ++ BASE_HW_FEATURE_FLUSH_REDUCTION, ++ BASE_HW_FEATURE_PROTECTED_MODE, ++ BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, ++ BASE_HW_FEATURE_COHERENCY_REG, ++ BASE_HW_FEATURE_END ++}; ++ ++static const enum base_hw_feature base_hw_features_tDVx[] = { ++ BASE_HW_FEATURE_33BIT_VA, ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, ++ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, ++ BASE_HW_FEATURE_XAFFINITY, ++ BASE_HW_FEATURE_WARPING, ++ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, ++ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, ++ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, ++ BASE_HW_FEATURE_BRNDOUT_CC, ++ BASE_HW_FEATURE_BRNDOUT_KILL, ++ BASE_HW_FEATURE_LD_ST_LEA_TEX, ++ BASE_HW_FEATURE_LD_ST_TILEBUFFER, ++ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, ++ BASE_HW_FEATURE_MRT, ++ BASE_HW_FEATURE_MSAA_16X, ++ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, ++ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, ++ BASE_HW_FEATURE_T7XX_PAIRING_RULES, ++ BASE_HW_FEATURE_TEST4_DATUM_MODE, ++ BASE_HW_FEATURE_FLUSH_REDUCTION, ++ BASE_HW_FEATURE_PROTECTED_MODE, ++ BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, ++ BASE_HW_FEATURE_COHERENCY_REG, ++ BASE_HW_FEATURE_END ++}; ++ ++ ++ ++ ++ ++#endif /* _BASE_HWCONFIG_FEATURES_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_base_hwconfig_issues.h b/drivers/gpu/arm/bifrost_for_linux/mali_base_hwconfig_issues.h +new file mode 100755 +index 000000000000..1c5ee496ac85 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_base_hwconfig_issues.h +@@ -0,0 +1,1126 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* AUTOMATICALLY GENERATED FILE. If you want to amend the issues/features, ++ * please update base/tools/hwconfig_generator/hwc_{issues,features}.py ++ * For more information see base/tools/hwconfig_generator/README ++ */ ++ ++#ifndef _BASE_HWCONFIG_ISSUES_H_ ++#define _BASE_HWCONFIG_ISSUES_H_ ++ ++enum base_hw_issue { ++ BASE_HW_ISSUE_5736, ++ BASE_HW_ISSUE_6367, ++ BASE_HW_ISSUE_6398, ++ BASE_HW_ISSUE_6402, ++ BASE_HW_ISSUE_6787, ++ BASE_HW_ISSUE_7027, ++ BASE_HW_ISSUE_7144, ++ BASE_HW_ISSUE_7304, ++ BASE_HW_ISSUE_8073, ++ BASE_HW_ISSUE_8186, ++ BASE_HW_ISSUE_8215, ++ BASE_HW_ISSUE_8245, ++ BASE_HW_ISSUE_8250, ++ BASE_HW_ISSUE_8260, ++ BASE_HW_ISSUE_8280, ++ BASE_HW_ISSUE_8316, ++ BASE_HW_ISSUE_8381, ++ BASE_HW_ISSUE_8394, ++ BASE_HW_ISSUE_8401, ++ BASE_HW_ISSUE_8408, ++ BASE_HW_ISSUE_8443, ++ BASE_HW_ISSUE_8456, ++ BASE_HW_ISSUE_8564, ++ BASE_HW_ISSUE_8634, ++ BASE_HW_ISSUE_8778, ++ BASE_HW_ISSUE_8791, ++ BASE_HW_ISSUE_8833, ++ BASE_HW_ISSUE_8879, ++ BASE_HW_ISSUE_8896, ++ BASE_HW_ISSUE_8975, ++ BASE_HW_ISSUE_8986, ++ BASE_HW_ISSUE_8987, ++ BASE_HW_ISSUE_9010, ++ BASE_HW_ISSUE_9418, ++ BASE_HW_ISSUE_9423, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_9510, ++ BASE_HW_ISSUE_9566, ++ BASE_HW_ISSUE_9630, ++ BASE_HW_ISSUE_10127, ++ BASE_HW_ISSUE_10327, ++ BASE_HW_ISSUE_10410, ++ BASE_HW_ISSUE_10471, ++ BASE_HW_ISSUE_10472, ++ BASE_HW_ISSUE_10487, ++ BASE_HW_ISSUE_10607, ++ BASE_HW_ISSUE_10632, ++ BASE_HW_ISSUE_10649, ++ BASE_HW_ISSUE_10676, ++ BASE_HW_ISSUE_10682, ++ BASE_HW_ISSUE_10684, ++ BASE_HW_ISSUE_10797, ++ BASE_HW_ISSUE_10817, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10931, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_10959, ++ BASE_HW_ISSUE_10969, ++ BASE_HW_ISSUE_10984, ++ BASE_HW_ISSUE_10995, ++ BASE_HW_ISSUE_11012, ++ BASE_HW_ISSUE_11020, ++ BASE_HW_ISSUE_11024, ++ BASE_HW_ISSUE_11035, ++ BASE_HW_ISSUE_11042, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T720_1386, ++ BASE_HW_ISSUE_T76X_26, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3542, ++ BASE_HW_ISSUE_T76X_3556, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3953, ++ BASE_HW_ISSUE_T76X_3960, ++ BASE_HW_ISSUE_T76X_3964, ++ BASE_HW_ISSUE_T76X_3966, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_7940, ++ BASE_HW_ISSUE_TMIX_8042, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TMIX_8138, ++ BASE_HW_ISSUE_TMIX_8206, ++ BASE_HW_ISSUE_TMIX_8343, ++ BASE_HW_ISSUE_TMIX_8463, ++ BASE_HW_ISSUE_TMIX_8456, ++ GPUCORE_1619, ++ BASE_HW_ISSUE_TSIX_1116, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_generic[] = { ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t60x_r0p0_15dev0[] = { ++ BASE_HW_ISSUE_6367, ++ BASE_HW_ISSUE_6398, ++ BASE_HW_ISSUE_6402, ++ BASE_HW_ISSUE_6787, ++ BASE_HW_ISSUE_7027, ++ BASE_HW_ISSUE_7144, ++ BASE_HW_ISSUE_7304, ++ BASE_HW_ISSUE_8073, ++ BASE_HW_ISSUE_8186, ++ BASE_HW_ISSUE_8215, ++ BASE_HW_ISSUE_8245, ++ BASE_HW_ISSUE_8250, ++ BASE_HW_ISSUE_8260, ++ BASE_HW_ISSUE_8280, ++ BASE_HW_ISSUE_8316, ++ BASE_HW_ISSUE_8381, ++ BASE_HW_ISSUE_8394, ++ BASE_HW_ISSUE_8401, ++ BASE_HW_ISSUE_8408, ++ BASE_HW_ISSUE_8443, ++ BASE_HW_ISSUE_8456, ++ BASE_HW_ISSUE_8564, ++ BASE_HW_ISSUE_8634, ++ BASE_HW_ISSUE_8778, ++ BASE_HW_ISSUE_8791, ++ BASE_HW_ISSUE_8833, ++ BASE_HW_ISSUE_8896, ++ BASE_HW_ISSUE_8975, ++ BASE_HW_ISSUE_8986, ++ BASE_HW_ISSUE_8987, ++ BASE_HW_ISSUE_9010, ++ BASE_HW_ISSUE_9418, ++ BASE_HW_ISSUE_9423, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_9510, ++ BASE_HW_ISSUE_9566, ++ BASE_HW_ISSUE_9630, ++ BASE_HW_ISSUE_10410, ++ BASE_HW_ISSUE_10471, ++ BASE_HW_ISSUE_10472, ++ BASE_HW_ISSUE_10487, ++ BASE_HW_ISSUE_10607, ++ BASE_HW_ISSUE_10632, ++ BASE_HW_ISSUE_10649, ++ BASE_HW_ISSUE_10676, ++ BASE_HW_ISSUE_10682, ++ BASE_HW_ISSUE_10684, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10931, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_10969, ++ BASE_HW_ISSUE_10984, ++ BASE_HW_ISSUE_10995, ++ BASE_HW_ISSUE_11012, ++ BASE_HW_ISSUE_11020, ++ BASE_HW_ISSUE_11035, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_3964, ++ GPUCORE_1619, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t60x_r0p0_eac[] = { ++ BASE_HW_ISSUE_6367, ++ BASE_HW_ISSUE_6402, ++ BASE_HW_ISSUE_6787, ++ BASE_HW_ISSUE_7027, ++ BASE_HW_ISSUE_7304, ++ BASE_HW_ISSUE_8408, ++ BASE_HW_ISSUE_8564, ++ BASE_HW_ISSUE_8778, ++ BASE_HW_ISSUE_8975, ++ BASE_HW_ISSUE_9010, ++ BASE_HW_ISSUE_9418, ++ BASE_HW_ISSUE_9423, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_9510, ++ BASE_HW_ISSUE_10410, ++ BASE_HW_ISSUE_10471, ++ BASE_HW_ISSUE_10472, ++ BASE_HW_ISSUE_10487, ++ BASE_HW_ISSUE_10607, ++ BASE_HW_ISSUE_10632, ++ BASE_HW_ISSUE_10649, ++ BASE_HW_ISSUE_10676, ++ BASE_HW_ISSUE_10682, ++ BASE_HW_ISSUE_10684, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10931, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_10969, ++ BASE_HW_ISSUE_11012, ++ BASE_HW_ISSUE_11020, ++ BASE_HW_ISSUE_11035, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_3964, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t60x_r0p1[] = { ++ BASE_HW_ISSUE_6367, ++ BASE_HW_ISSUE_6402, ++ BASE_HW_ISSUE_6787, ++ BASE_HW_ISSUE_7027, ++ BASE_HW_ISSUE_7304, ++ BASE_HW_ISSUE_8408, ++ BASE_HW_ISSUE_8564, ++ BASE_HW_ISSUE_8778, ++ BASE_HW_ISSUE_8975, ++ BASE_HW_ISSUE_9010, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_9510, ++ BASE_HW_ISSUE_10410, ++ BASE_HW_ISSUE_10471, ++ BASE_HW_ISSUE_10472, ++ BASE_HW_ISSUE_10487, ++ BASE_HW_ISSUE_10607, ++ BASE_HW_ISSUE_10632, ++ BASE_HW_ISSUE_10649, ++ BASE_HW_ISSUE_10676, ++ BASE_HW_ISSUE_10682, ++ BASE_HW_ISSUE_10684, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10931, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_11012, ++ BASE_HW_ISSUE_11020, ++ BASE_HW_ISSUE_11035, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3964, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t62x_r0p1[] = { ++ BASE_HW_ISSUE_6402, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10127, ++ BASE_HW_ISSUE_10327, ++ BASE_HW_ISSUE_10410, ++ BASE_HW_ISSUE_10471, ++ BASE_HW_ISSUE_10472, ++ BASE_HW_ISSUE_10487, ++ BASE_HW_ISSUE_10607, ++ BASE_HW_ISSUE_10632, ++ BASE_HW_ISSUE_10649, ++ BASE_HW_ISSUE_10676, ++ BASE_HW_ISSUE_10682, ++ BASE_HW_ISSUE_10684, ++ BASE_HW_ISSUE_10817, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10931, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_10959, ++ BASE_HW_ISSUE_11012, ++ BASE_HW_ISSUE_11020, ++ BASE_HW_ISSUE_11024, ++ BASE_HW_ISSUE_11035, ++ BASE_HW_ISSUE_11042, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t62x_r1p0[] = { ++ BASE_HW_ISSUE_6402, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10471, ++ BASE_HW_ISSUE_10472, ++ BASE_HW_ISSUE_10649, ++ BASE_HW_ISSUE_10684, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10931, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_10959, ++ BASE_HW_ISSUE_11012, ++ BASE_HW_ISSUE_11020, ++ BASE_HW_ISSUE_11024, ++ BASE_HW_ISSUE_11042, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3964, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t62x_r1p1[] = { ++ BASE_HW_ISSUE_6402, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10471, ++ BASE_HW_ISSUE_10472, ++ BASE_HW_ISSUE_10649, ++ BASE_HW_ISSUE_10684, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10931, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_10959, ++ BASE_HW_ISSUE_11012, ++ BASE_HW_ISSUE_11042, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t76x_r0p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_11020, ++ BASE_HW_ISSUE_11024, ++ BASE_HW_ISSUE_11042, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T76X_26, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3542, ++ BASE_HW_ISSUE_T76X_3556, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3953, ++ BASE_HW_ISSUE_T76X_3960, ++ BASE_HW_ISSUE_T76X_3964, ++ BASE_HW_ISSUE_T76X_3966, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t76x_r0p1[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_11020, ++ BASE_HW_ISSUE_11024, ++ BASE_HW_ISSUE_11042, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T76X_26, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3542, ++ BASE_HW_ISSUE_T76X_3556, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3953, ++ BASE_HW_ISSUE_T76X_3960, ++ BASE_HW_ISSUE_T76X_3964, ++ BASE_HW_ISSUE_T76X_3966, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t76x_r0p1_50rel0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_11042, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T76X_26, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3542, ++ BASE_HW_ISSUE_T76X_3556, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3953, ++ BASE_HW_ISSUE_T76X_3960, ++ BASE_HW_ISSUE_T76X_3964, ++ BASE_HW_ISSUE_T76X_3966, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t76x_r0p2[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_11020, ++ BASE_HW_ISSUE_11024, ++ BASE_HW_ISSUE_11042, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T76X_26, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3542, ++ BASE_HW_ISSUE_T76X_3556, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3953, ++ BASE_HW_ISSUE_T76X_3960, ++ BASE_HW_ISSUE_T76X_3964, ++ BASE_HW_ISSUE_T76X_3966, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t76x_r0p3[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_11042, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T76X_26, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3542, ++ BASE_HW_ISSUE_T76X_3556, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3953, ++ BASE_HW_ISSUE_T76X_3960, ++ BASE_HW_ISSUE_T76X_3964, ++ BASE_HW_ISSUE_T76X_3966, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t76x_r1p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_11042, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3953, ++ BASE_HW_ISSUE_T76X_3960, ++ BASE_HW_ISSUE_T76X_3964, ++ BASE_HW_ISSUE_T76X_3966, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t72x_r0p0[] = { ++ BASE_HW_ISSUE_6402, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10471, ++ BASE_HW_ISSUE_10649, ++ BASE_HW_ISSUE_10684, ++ BASE_HW_ISSUE_10797, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_11042, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3964, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t72x_r1p0[] = { ++ BASE_HW_ISSUE_6402, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10471, ++ BASE_HW_ISSUE_10649, ++ BASE_HW_ISSUE_10684, ++ BASE_HW_ISSUE_10797, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_11042, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T720_1386, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3964, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t72x_r1p1[] = { ++ BASE_HW_ISSUE_6402, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10471, ++ BASE_HW_ISSUE_10649, ++ BASE_HW_ISSUE_10684, ++ BASE_HW_ISSUE_10797, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_11042, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T720_1386, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3964, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_model_t72x[] = { ++ BASE_HW_ISSUE_5736, ++ BASE_HW_ISSUE_6402, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10471, ++ BASE_HW_ISSUE_10649, ++ BASE_HW_ISSUE_10797, ++ BASE_HW_ISSUE_11042, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3964, ++ GPUCORE_1619, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_model_t76x[] = { ++ BASE_HW_ISSUE_5736, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_11020, ++ BASE_HW_ISSUE_11024, ++ BASE_HW_ISSUE_11042, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3964, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ GPUCORE_1619, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_model_t60x[] = { ++ BASE_HW_ISSUE_5736, ++ BASE_HW_ISSUE_6402, ++ BASE_HW_ISSUE_8778, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10472, ++ BASE_HW_ISSUE_10649, ++ BASE_HW_ISSUE_10931, ++ BASE_HW_ISSUE_11012, ++ BASE_HW_ISSUE_11020, ++ BASE_HW_ISSUE_11024, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3964, ++ GPUCORE_1619, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_model_t62x[] = { ++ BASE_HW_ISSUE_5736, ++ BASE_HW_ISSUE_6402, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10472, ++ BASE_HW_ISSUE_10649, ++ BASE_HW_ISSUE_10931, ++ BASE_HW_ISSUE_11012, ++ BASE_HW_ISSUE_11020, ++ BASE_HW_ISSUE_11024, ++ BASE_HW_ISSUE_11042, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3964, ++ GPUCORE_1619, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tFRx_r0p1[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3953, ++ BASE_HW_ISSUE_T76X_3960, ++ BASE_HW_ISSUE_T76X_3964, ++ BASE_HW_ISSUE_T76X_3966, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tFRx_r0p2[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3953, ++ BASE_HW_ISSUE_T76X_3964, ++ BASE_HW_ISSUE_T76X_3966, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tFRx_r1p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3953, ++ BASE_HW_ISSUE_T76X_3966, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tFRx_r2p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3953, ++ BASE_HW_ISSUE_T76X_3966, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_model_tFRx[] = { ++ BASE_HW_ISSUE_5736, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3964, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ GPUCORE_1619, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t86x_r0p2[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3953, ++ BASE_HW_ISSUE_T76X_3964, ++ BASE_HW_ISSUE_T76X_3966, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t86x_r1p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3953, ++ BASE_HW_ISSUE_T76X_3966, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t86x_r2p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3953, ++ BASE_HW_ISSUE_T76X_3966, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_model_t86x[] = { ++ BASE_HW_ISSUE_5736, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ GPUCORE_1619, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t83x_r0p1[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T720_1386, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3953, ++ BASE_HW_ISSUE_T76X_3960, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t83x_r1p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T720_1386, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3953, ++ BASE_HW_ISSUE_T76X_3960, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_model_t83x[] = { ++ BASE_HW_ISSUE_5736, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3964, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ GPUCORE_1619, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t82x_r0p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T720_1386, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3953, ++ BASE_HW_ISSUE_T76X_3960, ++ BASE_HW_ISSUE_T76X_3964, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t82x_r0p1[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T720_1386, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3953, ++ BASE_HW_ISSUE_T76X_3960, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t82x_r1p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T720_1386, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3953, ++ BASE_HW_ISSUE_T76X_3960, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_model_t82x[] = { ++ BASE_HW_ISSUE_5736, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ GPUCORE_1619, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tMIx_r0p0_05dev0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10682, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T76X_3953, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8042, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TMIX_8138, ++ BASE_HW_ISSUE_TMIX_8206, ++ BASE_HW_ISSUE_TMIX_8343, ++ BASE_HW_ISSUE_TMIX_8463, ++ BASE_HW_ISSUE_TMIX_8456, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tMIx_r0p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10682, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_7940, ++ BASE_HW_ISSUE_TMIX_8042, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TMIX_8138, ++ BASE_HW_ISSUE_TMIX_8206, ++ BASE_HW_ISSUE_TMIX_8343, ++ BASE_HW_ISSUE_TMIX_8463, ++ BASE_HW_ISSUE_TMIX_8456, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tMIx_r0p1[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10682, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_7940, ++ BASE_HW_ISSUE_TMIX_8042, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TMIX_8138, ++ BASE_HW_ISSUE_TMIX_8206, ++ BASE_HW_ISSUE_TMIX_8343, ++ BASE_HW_ISSUE_TMIX_8463, ++ BASE_HW_ISSUE_TMIX_8456, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_model_tMIx[] = { ++ BASE_HW_ISSUE_5736, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_7940, ++ BASE_HW_ISSUE_TMIX_8042, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TMIX_8138, ++ BASE_HW_ISSUE_TMIX_8206, ++ BASE_HW_ISSUE_TMIX_8343, ++ BASE_HW_ISSUE_TMIX_8456, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tHEx_r0p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10682, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8042, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tHEx_r0p1[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10682, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8042, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tHEx_r0p2[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10682, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8042, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tHEx_r0p3[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10682, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8042, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_model_tHEx[] = { ++ BASE_HW_ISSUE_5736, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8042, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tSIx_r0p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TSIX_1116, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tSIx_r0p1[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TSIX_1116, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tSIx_r1p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TSIX_1116, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tSIx_r1p1[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TSIX_1116, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_model_tSIx[] = { ++ BASE_HW_ISSUE_5736, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TSIX_1116, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tDVx_r0p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TSIX_1116, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_model_tDVx[] = { ++ BASE_HW_ISSUE_5736, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TSIX_1116, ++ BASE_HW_ISSUE_END ++}; ++ ++ ++ ++ ++ ++ ++ ++ ++ ++#endif /* _BASE_HWCONFIG_ISSUES_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_base_kernel.h b/drivers/gpu/arm/bifrost_for_linux/mali_base_kernel.h +new file mode 100755 +index 000000000000..6f5c68e288cd +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_base_kernel.h +@@ -0,0 +1,1822 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * Base structures shared with the kernel. ++ */ ++ ++#ifndef _BASE_KERNEL_H_ ++#define _BASE_KERNEL_H_ ++ ++/* Support UK10_2 IOCTLS */ ++#define BASE_LEGACY_UK10_2_SUPPORT 1 ++ ++/* Support UK10_4 IOCTLS */ ++#define BASE_LEGACY_UK10_4_SUPPORT 1 ++ ++typedef struct base_mem_handle { ++ struct { ++ u64 handle; ++ } basep; ++} base_mem_handle; ++ ++#include "mali_base_mem_priv.h" ++#include "mali_kbase_profiling_gator_api.h" ++#include "mali_midg_coherency.h" ++#include "mali_kbase_gpu_id.h" ++ ++/* ++ * Dependency stuff, keep it private for now. May want to expose it if ++ * we decide to make the number of semaphores a configurable ++ * option. ++ */ ++#define BASE_JD_ATOM_COUNT 256 ++ ++/* Set/reset values for a software event */ ++#define BASE_JD_SOFT_EVENT_SET ((unsigned char)1) ++#define BASE_JD_SOFT_EVENT_RESET ((unsigned char)0) ++ ++#define BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS 3 ++ ++#define BASE_MAX_COHERENT_GROUPS 16 ++ ++#if defined CDBG_ASSERT ++#define LOCAL_ASSERT CDBG_ASSERT ++#elif defined KBASE_DEBUG_ASSERT ++#define LOCAL_ASSERT KBASE_DEBUG_ASSERT ++#else ++#error assert macro not defined! ++#endif ++ ++#if defined PAGE_MASK ++#define LOCAL_PAGE_LSB ~PAGE_MASK ++#else ++#include ++ ++#if defined OSU_CONFIG_CPU_PAGE_SIZE_LOG2 ++#define LOCAL_PAGE_LSB ((1ul << OSU_CONFIG_CPU_PAGE_SIZE_LOG2) - 1) ++#else ++#error Failed to find page size ++#endif ++#endif ++ ++/** ++ * @addtogroup base_user_api User-side Base APIs ++ * @{ ++ */ ++ ++/** ++ * @addtogroup base_user_api_memory User-side Base Memory APIs ++ * @{ ++ */ ++ ++/** ++ * typedef base_mem_alloc_flags - Memory allocation, access/hint flags. ++ * ++ * A combination of MEM_PROT/MEM_HINT flags must be passed to each allocator ++ * in order to determine the best cache policy. Some combinations are ++ * of course invalid (e.g. MEM_PROT_CPU_WR | MEM_HINT_CPU_RD), ++ * which defines a write-only region on the CPU side, which is ++ * heavily read by the CPU... ++ * Other flags are only meaningful to a particular allocator. ++ * More flags can be added to this list, as long as they don't clash ++ * (see BASE_MEM_FLAGS_NR_BITS for the number of the first free bit). ++ */ ++typedef u32 base_mem_alloc_flags; ++ ++/* Memory allocation, access/hint flags. ++ * ++ * See base_mem_alloc_flags. ++ */ ++ ++/* IN */ ++/* Read access CPU side ++ */ ++#define BASE_MEM_PROT_CPU_RD ((base_mem_alloc_flags)1 << 0) ++ ++/* Write access CPU side ++ */ ++#define BASE_MEM_PROT_CPU_WR ((base_mem_alloc_flags)1 << 1) ++ ++/* Read access GPU side ++ */ ++#define BASE_MEM_PROT_GPU_RD ((base_mem_alloc_flags)1 << 2) ++ ++/* Write access GPU side ++ */ ++#define BASE_MEM_PROT_GPU_WR ((base_mem_alloc_flags)1 << 3) ++ ++/* Execute allowed on the GPU side ++ */ ++#define BASE_MEM_PROT_GPU_EX ((base_mem_alloc_flags)1 << 4) ++ ++ /* BASE_MEM_HINT flags have been removed, but their values are reserved ++ * for backwards compatibility with older user-space drivers. The values ++ * can be re-used once support for r5p0 user-space drivers is removed, ++ * presumably in r7p0. ++ * ++ * RESERVED: (1U << 5) ++ * RESERVED: (1U << 6) ++ * RESERVED: (1U << 7) ++ * RESERVED: (1U << 8) ++ */ ++ ++/* Grow backing store on GPU Page Fault ++ */ ++#define BASE_MEM_GROW_ON_GPF ((base_mem_alloc_flags)1 << 9) ++ ++/* Page coherence Outer shareable, if available ++ */ ++#define BASE_MEM_COHERENT_SYSTEM ((base_mem_alloc_flags)1 << 10) ++ ++/* Page coherence Inner shareable ++ */ ++#define BASE_MEM_COHERENT_LOCAL ((base_mem_alloc_flags)1 << 11) ++ ++/* Should be cached on the CPU ++ */ ++#define BASE_MEM_CACHED_CPU ((base_mem_alloc_flags)1 << 12) ++ ++/* IN/OUT */ ++/* Must have same VA on both the GPU and the CPU ++ */ ++#define BASE_MEM_SAME_VA ((base_mem_alloc_flags)1 << 13) ++ ++/* OUT */ ++/* Must call mmap to acquire a GPU address for the alloc ++ */ ++#define BASE_MEM_NEED_MMAP ((base_mem_alloc_flags)1 << 14) ++ ++/* IN */ ++/* Page coherence Outer shareable, required. ++ */ ++#define BASE_MEM_COHERENT_SYSTEM_REQUIRED ((base_mem_alloc_flags)1 << 15) ++ ++/* Secure memory ++ */ ++#define BASE_MEM_SECURE ((base_mem_alloc_flags)1 << 16) ++ ++/* Not needed physical memory ++ */ ++#define BASE_MEM_DONT_NEED ((base_mem_alloc_flags)1 << 17) ++ ++/* Must use shared CPU/GPU zone (SAME_VA zone) but doesn't require the ++ * addresses to be the same ++ */ ++#define BASE_MEM_IMPORT_SHARED ((base_mem_alloc_flags)1 << 18) ++ ++/** ++ * Bit 19 is reserved. ++ * ++ * Do not remove, use the next unreserved bit for new flags ++ **/ ++#define BASE_MEM_RESERVED_BIT_19 ((base_mem_alloc_flags)1 << 19) ++ ++/* Number of bits used as flags for base memory management ++ * ++ * Must be kept in sync with the base_mem_alloc_flags flags ++ */ ++#define BASE_MEM_FLAGS_NR_BITS 20 ++ ++/* A mask for all output bits, excluding IN/OUT bits. ++ */ ++#define BASE_MEM_FLAGS_OUTPUT_MASK BASE_MEM_NEED_MMAP ++ ++/* A mask for all input bits, including IN/OUT bits. ++ */ ++#define BASE_MEM_FLAGS_INPUT_MASK \ ++ (((1 << BASE_MEM_FLAGS_NR_BITS) - 1) & ~BASE_MEM_FLAGS_OUTPUT_MASK) ++ ++/* A mask for all the flags which are modifiable via the base_mem_set_flags ++ * interface. ++ */ ++#define BASE_MEM_FLAGS_MODIFIABLE \ ++ (BASE_MEM_DONT_NEED | BASE_MEM_COHERENT_SYSTEM | \ ++ BASE_MEM_COHERENT_LOCAL) ++ ++/** ++ * enum base_mem_import_type - Memory types supported by @a base_mem_import ++ * ++ * @BASE_MEM_IMPORT_TYPE_INVALID: Invalid type ++ * @BASE_MEM_IMPORT_TYPE_UMP: UMP import. Handle type is ump_secure_id. ++ * @BASE_MEM_IMPORT_TYPE_UMM: UMM import. Handle type is a file descriptor (int) ++ * @BASE_MEM_IMPORT_TYPE_USER_BUFFER: User buffer import. Handle is a ++ * base_mem_import_user_buffer ++ * ++ * Each type defines what the supported handle type is. ++ * ++ * If any new type is added here ARM must be contacted ++ * to allocate a numeric value for it. ++ * Do not just add a new type without synchronizing with ARM ++ * as future releases from ARM might include other new types ++ * which could clash with your custom types. ++ */ ++typedef enum base_mem_import_type { ++ BASE_MEM_IMPORT_TYPE_INVALID = 0, ++ BASE_MEM_IMPORT_TYPE_UMP = 1, ++ BASE_MEM_IMPORT_TYPE_UMM = 2, ++ BASE_MEM_IMPORT_TYPE_USER_BUFFER = 3 ++} base_mem_import_type; ++ ++/** ++ * struct base_mem_import_user_buffer - Handle of an imported user buffer ++ * ++ * @ptr: address of imported user buffer ++ * @length: length of imported user buffer in bytes ++ * ++ * This structure is used to represent a handle of an imported user buffer. ++ */ ++ ++struct base_mem_import_user_buffer { ++ u64 ptr; ++ u64 length; ++}; ++ ++/** ++ * @brief Invalid memory handle. ++ * ++ * Return value from functions returning @ref base_mem_handle on error. ++ * ++ * @warning @ref base_mem_handle_new_invalid must be used instead of this macro ++ * in C++ code or other situations where compound literals cannot be used. ++ */ ++#define BASE_MEM_INVALID_HANDLE ((base_mem_handle) { {BASEP_MEM_INVALID_HANDLE} }) ++ ++/** ++ * @brief Special write-alloc memory handle. ++ * ++ * A special handle is used to represent a region where a special page is mapped ++ * with a write-alloc cache setup, typically used when the write result of the ++ * GPU isn't needed, but the GPU must write anyway. ++ * ++ * @warning @ref base_mem_handle_new_write_alloc must be used instead of this macro ++ * in C++ code or other situations where compound literals cannot be used. ++ */ ++#define BASE_MEM_WRITE_ALLOC_PAGES_HANDLE ((base_mem_handle) { {BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE} }) ++ ++#define BASEP_MEM_INVALID_HANDLE (0ull << 12) ++#define BASE_MEM_MMU_DUMP_HANDLE (1ull << 12) ++#define BASE_MEM_TRACE_BUFFER_HANDLE (2ull << 12) ++#define BASE_MEM_MAP_TRACKING_HANDLE (3ull << 12) ++#define BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE (4ull << 12) ++/* reserved handles ..-64< for future special handles */ ++#define BASE_MEM_COOKIE_BASE (64ul << 12) ++#define BASE_MEM_FIRST_FREE_ADDRESS ((BITS_PER_LONG << 12) + \ ++ BASE_MEM_COOKIE_BASE) ++ ++/* Mask to detect 4GB boundary alignment */ ++#define BASE_MEM_MASK_4GB 0xfffff000UL ++ ++ ++/* Bit mask of cookies used for for memory allocation setup */ ++#define KBASE_COOKIE_MASK ~1UL /* bit 0 is reserved */ ++ ++ ++/** ++ * @brief Result codes of changing the size of the backing store allocated to a tmem region ++ */ ++typedef enum base_backing_threshold_status { ++ BASE_BACKING_THRESHOLD_OK = 0, /**< Resize successful */ ++ BASE_BACKING_THRESHOLD_ERROR_OOM = -2, /**< Increase failed due to an out-of-memory condition */ ++ BASE_BACKING_THRESHOLD_ERROR_INVALID_ARGUMENTS = -4 /**< Invalid arguments (not tmem, illegal size request, etc.) */ ++} base_backing_threshold_status; ++ ++/** ++ * @addtogroup base_user_api_memory_defered User-side Base Defered Memory Coherency APIs ++ * @{ ++ */ ++ ++/** ++ * @brief a basic memory operation (sync-set). ++ * ++ * The content of this structure is private, and should only be used ++ * by the accessors. ++ */ ++typedef struct base_syncset { ++ struct basep_syncset basep_sset; ++} base_syncset; ++ ++/** @} end group base_user_api_memory_defered */ ++ ++/** ++ * Handle to represent imported memory object. ++ * Simple opague handle to imported memory, can't be used ++ * with anything but base_external_resource_init to bind to an atom. ++ */ ++typedef struct base_import_handle { ++ struct { ++ u64 handle; ++ } basep; ++} base_import_handle; ++ ++/** @} end group base_user_api_memory */ ++ ++/** ++ * @addtogroup base_user_api_job_dispatch User-side Base Job Dispatcher APIs ++ * @{ ++ */ ++ ++typedef int platform_fence_type; ++#define INVALID_PLATFORM_FENCE ((platform_fence_type)-1) ++ ++/** ++ * Base stream handle. ++ * ++ * References an underlying base stream object. ++ */ ++typedef struct base_stream { ++ struct { ++ int fd; ++ } basep; ++} base_stream; ++ ++/** ++ * Base fence handle. ++ * ++ * References an underlying base fence object. ++ */ ++typedef struct base_fence { ++ struct { ++ int fd; ++ int stream_fd; ++ } basep; ++} base_fence; ++ ++/** ++ * @brief Per-job data ++ * ++ * This structure is used to store per-job data, and is completely unused ++ * by the Base driver. It can be used to store things such as callback ++ * function pointer, data to handle job completion. It is guaranteed to be ++ * untouched by the Base driver. ++ */ ++typedef struct base_jd_udata { ++ u64 blob[2]; /**< per-job data array */ ++} base_jd_udata; ++ ++/** ++ * @brief Memory aliasing info ++ * ++ * Describes a memory handle to be aliased. ++ * A subset of the handle can be chosen for aliasing, given an offset and a ++ * length. ++ * A special handle BASE_MEM_WRITE_ALLOC_PAGES_HANDLE is used to represent a ++ * region where a special page is mapped with a write-alloc cache setup, ++ * typically used when the write result of the GPU isn't needed, but the GPU ++ * must write anyway. ++ * ++ * Offset and length are specified in pages. ++ * Offset must be within the size of the handle. ++ * Offset+length must not overrun the size of the handle. ++ * ++ * @handle Handle to alias, can be BASE_MEM_WRITE_ALLOC_PAGES_HANDLE ++ * @offset Offset within the handle to start aliasing from, in pages. ++ * Not used with BASE_MEM_WRITE_ALLOC_PAGES_HANDLE. ++ * @length Length to alias, in pages. For BASE_MEM_WRITE_ALLOC_PAGES_HANDLE ++ * specifies the number of times the special page is needed. ++ */ ++struct base_mem_aliasing_info { ++ base_mem_handle handle; ++ u64 offset; ++ u64 length; ++}; ++ ++/** ++ * struct base_jit_alloc_info - Structure which describes a JIT allocation ++ * request. ++ * @gpu_alloc_addr: The GPU virtual address to write the JIT ++ * allocated GPU virtual address to. ++ * @va_pages: The minimum number of virtual pages required. ++ * @commit_pages: The minimum number of physical pages which ++ * should back the allocation. ++ * @extent: Granularity of physical pages to grow the ++ * allocation by during a fault. ++ * @id: Unique ID provided by the caller, this is used ++ * to pair allocation and free requests. ++ * Zero is not a valid value. ++ */ ++struct base_jit_alloc_info { ++ u64 gpu_alloc_addr; ++ u64 va_pages; ++ u64 commit_pages; ++ u64 extent; ++ u8 id; ++}; ++ ++/** ++ * @brief Job dependency type. ++ * ++ * A flags field will be inserted into the atom structure to specify whether a dependency is a data or ++ * ordering dependency (by putting it before/after 'core_req' in the structure it should be possible to add without ++ * changing the structure size). ++ * When the flag is set for a particular dependency to signal that it is an ordering only dependency then ++ * errors will not be propagated. ++ */ ++typedef u8 base_jd_dep_type; ++ ++ ++#define BASE_JD_DEP_TYPE_INVALID (0) /**< Invalid dependency */ ++#define BASE_JD_DEP_TYPE_DATA (1U << 0) /**< Data dependency */ ++#define BASE_JD_DEP_TYPE_ORDER (1U << 1) /**< Order dependency */ ++ ++/** ++ * @brief Job chain hardware requirements. ++ * ++ * A job chain must specify what GPU features it needs to allow the ++ * driver to schedule the job correctly. By not specifying the ++ * correct settings can/will cause an early job termination. Multiple ++ * values can be ORed together to specify multiple requirements. ++ * Special case is ::BASE_JD_REQ_DEP, which is used to express complex ++ * dependencies, and that doesn't execute anything on the hardware. ++ */ ++typedef u32 base_jd_core_req; ++ ++/* Requirements that come from the HW */ ++ ++/** ++ * No requirement, dependency only ++ */ ++#define BASE_JD_REQ_DEP ((base_jd_core_req)0) ++ ++/** ++ * Requires fragment shaders ++ */ ++#define BASE_JD_REQ_FS ((base_jd_core_req)1 << 0) ++ ++/** ++ * Requires compute shaders ++ * This covers any of the following Midgard Job types: ++ * - Vertex Shader Job ++ * - Geometry Shader Job ++ * - An actual Compute Shader Job ++ * ++ * Compare this with @ref BASE_JD_REQ_ONLY_COMPUTE, which specifies that the ++ * job is specifically just the "Compute Shader" job type, and not the "Vertex ++ * Shader" nor the "Geometry Shader" job type. ++ */ ++#define BASE_JD_REQ_CS ((base_jd_core_req)1 << 1) ++#define BASE_JD_REQ_T ((base_jd_core_req)1 << 2) /**< Requires tiling */ ++#define BASE_JD_REQ_CF ((base_jd_core_req)1 << 3) /**< Requires cache flushes */ ++#define BASE_JD_REQ_V ((base_jd_core_req)1 << 4) /**< Requires value writeback */ ++ ++/* SW-only requirements - the HW does not expose these as part of the job slot capabilities */ ++ ++/* Requires fragment job with AFBC encoding */ ++#define BASE_JD_REQ_FS_AFBC ((base_jd_core_req)1 << 13) ++ ++/** ++ * SW-only requirement: coalesce completion events. ++ * If this bit is set then completion of this atom will not cause an event to ++ * be sent to userspace, whether successful or not; completion events will be ++ * deferred until an atom completes which does not have this bit set. ++ * ++ * This bit may not be used in combination with BASE_JD_REQ_EXTERNAL_RESOURCES. ++ */ ++#define BASE_JD_REQ_EVENT_COALESCE ((base_jd_core_req)1 << 5) ++ ++/** ++ * SW Only requirement: the job chain requires a coherent core group. We don't ++ * mind which coherent core group is used. ++ */ ++#define BASE_JD_REQ_COHERENT_GROUP ((base_jd_core_req)1 << 6) ++ ++/** ++ * SW Only requirement: The performance counters should be enabled only when ++ * they are needed, to reduce power consumption. ++ */ ++ ++#define BASE_JD_REQ_PERMON ((base_jd_core_req)1 << 7) ++ ++/** ++ * SW Only requirement: External resources are referenced by this atom. ++ * When external resources are referenced no syncsets can be bundled with the atom ++ * but should instead be part of a NULL jobs inserted into the dependency tree. ++ * The first pre_dep object must be configured for the external resouces to use, ++ * the second pre_dep object can be used to create other dependencies. ++ * ++ * This bit may not be used in combination with BASE_JD_REQ_EVENT_COALESCE. ++ */ ++#define BASE_JD_REQ_EXTERNAL_RESOURCES ((base_jd_core_req)1 << 8) ++ ++/** ++ * SW Only requirement: Software defined job. Jobs with this bit set will not be submitted ++ * to the hardware but will cause some action to happen within the driver ++ */ ++#define BASE_JD_REQ_SOFT_JOB ((base_jd_core_req)1 << 9) ++ ++#define BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME (BASE_JD_REQ_SOFT_JOB | 0x1) ++#define BASE_JD_REQ_SOFT_FENCE_TRIGGER (BASE_JD_REQ_SOFT_JOB | 0x2) ++#define BASE_JD_REQ_SOFT_FENCE_WAIT (BASE_JD_REQ_SOFT_JOB | 0x3) ++ ++/** ++ * SW Only requirement : Replay job. ++ * ++ * If the preceding job fails, the replay job will cause the jobs specified in ++ * the list of base_jd_replay_payload pointed to by the jc pointer to be ++ * replayed. ++ * ++ * A replay job will only cause jobs to be replayed up to BASEP_JD_REPLAY_LIMIT ++ * times. If a job fails more than BASEP_JD_REPLAY_LIMIT times then the replay ++ * job is failed, as well as any following dependencies. ++ * ++ * The replayed jobs will require a number of atom IDs. If there are not enough ++ * free atom IDs then the replay job will fail. ++ * ++ * If the preceding job does not fail, then the replay job is returned as ++ * completed. ++ * ++ * The replayed jobs will never be returned to userspace. The preceding failed ++ * job will be returned to userspace as failed; the status of this job should ++ * be ignored. Completion should be determined by the status of the replay soft ++ * job. ++ * ++ * In order for the jobs to be replayed, the job headers will have to be ++ * modified. The Status field will be reset to NOT_STARTED. If the Job Type ++ * field indicates a Vertex Shader Job then it will be changed to Null Job. ++ * ++ * The replayed jobs have the following assumptions : ++ * ++ * - No external resources. Any required external resources will be held by the ++ * replay atom. ++ * - Pre-dependencies are created based on job order. ++ * - Atom numbers are automatically assigned. ++ * - device_nr is set to 0. This is not relevant as ++ * BASE_JD_REQ_SPECIFIC_COHERENT_GROUP should not be set. ++ * - Priority is inherited from the replay job. ++ */ ++#define BASE_JD_REQ_SOFT_REPLAY (BASE_JD_REQ_SOFT_JOB | 0x4) ++/** ++ * SW only requirement: event wait/trigger job. ++ * ++ * - BASE_JD_REQ_SOFT_EVENT_WAIT: this job will block until the event is set. ++ * - BASE_JD_REQ_SOFT_EVENT_SET: this job sets the event, thus unblocks the ++ * other waiting jobs. It completes immediately. ++ * - BASE_JD_REQ_SOFT_EVENT_RESET: this job resets the event, making it ++ * possible for other jobs to wait upon. It completes immediately. ++ */ ++#define BASE_JD_REQ_SOFT_EVENT_WAIT (BASE_JD_REQ_SOFT_JOB | 0x5) ++#define BASE_JD_REQ_SOFT_EVENT_SET (BASE_JD_REQ_SOFT_JOB | 0x6) ++#define BASE_JD_REQ_SOFT_EVENT_RESET (BASE_JD_REQ_SOFT_JOB | 0x7) ++ ++#define BASE_JD_REQ_SOFT_DEBUG_COPY (BASE_JD_REQ_SOFT_JOB | 0x8) ++ ++/** ++ * SW only requirement: Just In Time allocation ++ * ++ * This job requests a JIT allocation based on the request in the ++ * @base_jit_alloc_info structure which is passed via the jc element of ++ * the atom. ++ * ++ * It should be noted that the id entry in @base_jit_alloc_info must not ++ * be reused until it has been released via @BASE_JD_REQ_SOFT_JIT_FREE. ++ * ++ * Should this soft job fail it is expected that a @BASE_JD_REQ_SOFT_JIT_FREE ++ * soft job to free the JIT allocation is still made. ++ * ++ * The job will complete immediately. ++ */ ++#define BASE_JD_REQ_SOFT_JIT_ALLOC (BASE_JD_REQ_SOFT_JOB | 0x9) ++/** ++ * SW only requirement: Just In Time free ++ * ++ * This job requests a JIT allocation created by @BASE_JD_REQ_SOFT_JIT_ALLOC ++ * to be freed. The ID of the JIT allocation is passed via the jc element of ++ * the atom. ++ * ++ * The job will complete immediately. ++ */ ++#define BASE_JD_REQ_SOFT_JIT_FREE (BASE_JD_REQ_SOFT_JOB | 0xa) ++ ++/** ++ * SW only requirement: Map external resource ++ * ++ * This job requests external resource(s) are mapped once the dependencies ++ * of the job have been satisfied. The list of external resources are ++ * passed via the jc element of the atom which is a pointer to a ++ * @base_external_resource_list. ++ */ ++#define BASE_JD_REQ_SOFT_EXT_RES_MAP (BASE_JD_REQ_SOFT_JOB | 0xb) ++/** ++ * SW only requirement: Unmap external resource ++ * ++ * This job requests external resource(s) are unmapped once the dependencies ++ * of the job has been satisfied. The list of external resources are ++ * passed via the jc element of the atom which is a pointer to a ++ * @base_external_resource_list. ++ */ ++#define BASE_JD_REQ_SOFT_EXT_RES_UNMAP (BASE_JD_REQ_SOFT_JOB | 0xc) ++ ++/** ++ * HW Requirement: Requires Compute shaders (but not Vertex or Geometry Shaders) ++ * ++ * This indicates that the Job Chain contains Midgard Jobs of the 'Compute Shaders' type. ++ * ++ * In contrast to @ref BASE_JD_REQ_CS, this does \b not indicate that the Job ++ * Chain contains 'Geometry Shader' or 'Vertex Shader' jobs. ++ */ ++#define BASE_JD_REQ_ONLY_COMPUTE ((base_jd_core_req)1 << 10) ++ ++/** ++ * HW Requirement: Use the base_jd_atom::device_nr field to specify a ++ * particular core group ++ * ++ * If both @ref BASE_JD_REQ_COHERENT_GROUP and this flag are set, this flag takes priority ++ * ++ * This is only guaranteed to work for @ref BASE_JD_REQ_ONLY_COMPUTE atoms. ++ * ++ * If the core availability policy is keeping the required core group turned off, then ++ * the job will fail with a @ref BASE_JD_EVENT_PM_EVENT error code. ++ */ ++#define BASE_JD_REQ_SPECIFIC_COHERENT_GROUP ((base_jd_core_req)1 << 11) ++ ++/** ++ * SW Flag: If this bit is set then the successful completion of this atom ++ * will not cause an event to be sent to userspace ++ */ ++#define BASE_JD_REQ_EVENT_ONLY_ON_FAILURE ((base_jd_core_req)1 << 12) ++ ++/** ++ * SW Flag: If this bit is set then completion of this atom will not cause an ++ * event to be sent to userspace, whether successful or not. ++ */ ++#define BASEP_JD_REQ_EVENT_NEVER ((base_jd_core_req)1 << 14) ++ ++/** ++ * SW Flag: Skip GPU cache clean and invalidation before starting a GPU job. ++ * ++ * If this bit is set then the GPU's cache will not be cleaned and invalidated ++ * until a GPU job starts which does not have this bit set or a job completes ++ * which does not have the @ref BASE_JD_REQ_SKIP_CACHE_END bit set. Do not use if ++ * the CPU may have written to memory addressed by the job since the last job ++ * without this bit set was submitted. ++ */ ++#define BASE_JD_REQ_SKIP_CACHE_START ((base_jd_core_req)1 << 15) ++ ++/** ++ * SW Flag: Skip GPU cache clean and invalidation after a GPU job completes. ++ * ++ * If this bit is set then the GPU's cache will not be cleaned and invalidated ++ * until a GPU job completes which does not have this bit set or a job starts ++ * which does not have the @ref BASE_JD_REQ_SKIP_CACHE_START bti set. Do not use if ++ * the CPU may read from or partially overwrite memory addressed by the job ++ * before the next job without this bit set completes. ++ */ ++#define BASE_JD_REQ_SKIP_CACHE_END ((base_jd_core_req)1 << 16) ++ ++/** ++ * These requirement bits are currently unused in base_jd_core_req ++ */ ++#define BASEP_JD_REQ_RESERVED \ ++ (~(BASE_JD_REQ_ATOM_TYPE | BASE_JD_REQ_EXTERNAL_RESOURCES | \ ++ BASE_JD_REQ_EVENT_ONLY_ON_FAILURE | BASEP_JD_REQ_EVENT_NEVER | \ ++ BASE_JD_REQ_EVENT_COALESCE | \ ++ BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_SPECIFIC_COHERENT_GROUP | \ ++ BASE_JD_REQ_FS_AFBC | BASE_JD_REQ_PERMON | \ ++ BASE_JD_REQ_SKIP_CACHE_START | BASE_JD_REQ_SKIP_CACHE_END)) ++ ++/** ++ * Mask of all bits in base_jd_core_req that control the type of the atom. ++ * ++ * This allows dependency only atoms to have flags set ++ */ ++#define BASE_JD_REQ_ATOM_TYPE \ ++ (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T | BASE_JD_REQ_CF | \ ++ BASE_JD_REQ_V | BASE_JD_REQ_SOFT_JOB | BASE_JD_REQ_ONLY_COMPUTE) ++ ++/** ++ * Mask of all bits in base_jd_core_req that control the type of a soft job. ++ */ ++#define BASE_JD_REQ_SOFT_JOB_TYPE (BASE_JD_REQ_SOFT_JOB | 0x1f) ++ ++/* ++ * Returns non-zero value if core requirements passed define a soft job or ++ * a dependency only job. ++ */ ++#define BASE_JD_REQ_SOFT_JOB_OR_DEP(core_req) \ ++ ((core_req & BASE_JD_REQ_SOFT_JOB) || \ ++ (core_req & BASE_JD_REQ_ATOM_TYPE) == BASE_JD_REQ_DEP) ++ ++/** ++ * @brief States to model state machine processed by kbasep_js_job_check_ref_cores(), which ++ * handles retaining cores for power management and affinity management. ++ * ++ * The state @ref KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY prevents an attack ++ * where lots of atoms could be submitted before powerup, and each has an ++ * affinity chosen that causes other atoms to have an affinity ++ * violation. Whilst the affinity was not causing violations at the time it ++ * was chosen, it could cause violations thereafter. For example, 1000 jobs ++ * could have had their affinity chosen during the powerup time, so any of ++ * those 1000 jobs could cause an affinity violation later on. ++ * ++ * The attack would otherwise occur because other atoms/contexts have to wait for: ++ * -# the currently running atoms (which are causing the violation) to ++ * finish ++ * -# and, the atoms that had their affinity chosen during powerup to ++ * finish. These are run preferentially because they don't cause a ++ * violation, but instead continue to cause the violation in others. ++ * -# or, the attacker is scheduled out (which might not happen for just 2 ++ * contexts) ++ * ++ * By re-choosing the affinity (which is designed to avoid violations at the ++ * time it's chosen), we break condition (2) of the wait, which minimizes the ++ * problem to just waiting for current jobs to finish (which can be bounded if ++ * the Job Scheduling Policy has a timer). ++ */ ++enum kbase_atom_coreref_state { ++ /** Starting state: No affinity chosen, and cores must be requested. kbase_jd_atom::affinity==0 */ ++ KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED, ++ /** Cores requested, but waiting for them to be powered. Requested cores given by kbase_jd_atom::affinity */ ++ KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES, ++ /** Cores given by kbase_jd_atom::affinity are powered, but affinity might be out-of-date, so must recheck */ ++ KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY, ++ /** Cores given by kbase_jd_atom::affinity are powered, and affinity is up-to-date, but must check for violations */ ++ KBASE_ATOM_COREREF_STATE_CHECK_AFFINITY_VIOLATIONS, ++ /** Cores are powered, kbase_jd_atom::affinity up-to-date, no affinity violations: atom can be submitted to HW */ ++ KBASE_ATOM_COREREF_STATE_READY ++}; ++ ++/* ++ * Base Atom priority ++ * ++ * Only certain priority levels are actually implemented, as specified by the ++ * BASE_JD_PRIO_<...> definitions below. It is undefined to use a priority ++ * level that is not one of those defined below. ++ * ++ * Priority levels only affect scheduling between atoms of the same type within ++ * a base context, and only after the atoms have had dependencies resolved. ++ * Fragment atoms does not affect non-frament atoms with lower priorities, and ++ * the other way around. For example, a low priority atom that has had its ++ * dependencies resolved might run before a higher priority atom that has not ++ * had its dependencies resolved. ++ * ++ * The scheduling between base contexts/processes and between atoms from ++ * different base contexts/processes is unaffected by atom priority. ++ * ++ * The atoms are scheduled as follows with respect to their priorities: ++ * - Let atoms 'X' and 'Y' be for the same job slot who have dependencies ++ * resolved, and atom 'X' has a higher priority than atom 'Y' ++ * - If atom 'Y' is currently running on the HW, then it is interrupted to ++ * allow atom 'X' to run soon after ++ * - If instead neither atom 'Y' nor atom 'X' are running, then when choosing ++ * the next atom to run, atom 'X' will always be chosen instead of atom 'Y' ++ * - Any two atoms that have the same priority could run in any order with ++ * respect to each other. That is, there is no ordering constraint between ++ * atoms of the same priority. ++ */ ++typedef u8 base_jd_prio; ++ ++/* Medium atom priority. This is a priority higher than BASE_JD_PRIO_LOW */ ++#define BASE_JD_PRIO_MEDIUM ((base_jd_prio)0) ++/* High atom priority. This is a priority higher than BASE_JD_PRIO_MEDIUM and ++ * BASE_JD_PRIO_LOW */ ++#define BASE_JD_PRIO_HIGH ((base_jd_prio)1) ++/* Low atom priority. */ ++#define BASE_JD_PRIO_LOW ((base_jd_prio)2) ++ ++/* Count of the number of priority levels. This itself is not a valid ++ * base_jd_prio setting */ ++#define BASE_JD_NR_PRIO_LEVELS 3 ++ ++enum kbase_jd_atom_state { ++ /** Atom is not used */ ++ KBASE_JD_ATOM_STATE_UNUSED, ++ /** Atom is queued in JD */ ++ KBASE_JD_ATOM_STATE_QUEUED, ++ /** Atom has been given to JS (is runnable/running) */ ++ KBASE_JD_ATOM_STATE_IN_JS, ++ /** Atom has been completed, but not yet handed back to job dispatcher ++ * for dependency resolution */ ++ KBASE_JD_ATOM_STATE_HW_COMPLETED, ++ /** Atom has been completed, but not yet handed back to userspace */ ++ KBASE_JD_ATOM_STATE_COMPLETED ++}; ++ ++typedef u8 base_atom_id; /**< Type big enough to store an atom number in */ ++ ++struct base_dependency { ++ base_atom_id atom_id; /**< An atom number */ ++ base_jd_dep_type dependency_type; /**< Dependency type */ ++}; ++ ++/* This structure has changed since UK 10.2 for which base_jd_core_req was a u16 value. ++ * In order to keep the size of the structure same, padding field has been adjusted ++ * accordingly and core_req field of a u32 type (to which UK 10.3 base_jd_core_req defines) ++ * is added at the end of the structure. Place in the structure previously occupied by u16 core_req ++ * is kept but renamed to compat_core_req and as such it can be used in ioctl call for job submission ++ * as long as UK 10.2 legacy is supported. Once when this support ends, this field can be left ++ * for possible future use. */ ++typedef struct base_jd_atom_v2 { ++ u64 jc; /**< job-chain GPU address */ ++ struct base_jd_udata udata; /**< user data */ ++ u64 extres_list; /**< list of external resources */ ++ u16 nr_extres; /**< nr of external resources */ ++ u16 compat_core_req; /**< core requirements which correspond to the legacy support for UK 10.2 */ ++ struct base_dependency pre_dep[2]; /**< pre-dependencies, one need to use SETTER function to assign this field, ++ this is done in order to reduce possibility of improper assigment of a dependency field */ ++ base_atom_id atom_number; /**< unique number to identify the atom */ ++ base_jd_prio prio; /**< Atom priority. Refer to @ref base_jd_prio for more details */ ++ u8 device_nr; /**< coregroup when BASE_JD_REQ_SPECIFIC_COHERENT_GROUP specified */ ++ u8 padding[1]; ++ base_jd_core_req core_req; /**< core requirements */ ++} base_jd_atom_v2; ++ ++typedef enum base_external_resource_access { ++ BASE_EXT_RES_ACCESS_SHARED, ++ BASE_EXT_RES_ACCESS_EXCLUSIVE ++} base_external_resource_access; ++ ++typedef struct base_external_resource { ++ u64 ext_resource; ++} base_external_resource; ++ ++ ++/** ++ * The maximum number of external resources which can be mapped/unmapped ++ * in a single request. ++ */ ++#define BASE_EXT_RES_COUNT_MAX 10 ++ ++/** ++ * struct base_external_resource_list - Structure which describes a list of ++ * external resources. ++ * @count: The number of resources. ++ * @ext_res: Array of external resources which is ++ * sized at allocation time. ++ */ ++struct base_external_resource_list { ++ u64 count; ++ struct base_external_resource ext_res[1]; ++}; ++ ++struct base_jd_debug_copy_buffer { ++ u64 address; ++ u64 size; ++ struct base_external_resource extres; ++}; ++ ++/** ++ * @brief Setter for a dependency structure ++ * ++ * @param[in] dep The kbase jd atom dependency to be initialized. ++ * @param id The atom_id to be assigned. ++ * @param dep_type The dep_type to be assigned. ++ * ++ */ ++static inline void base_jd_atom_dep_set(struct base_dependency *dep, ++ base_atom_id id, base_jd_dep_type dep_type) ++{ ++ LOCAL_ASSERT(dep != NULL); ++ ++ /* ++ * make sure we don't set not allowed combinations ++ * of atom_id/dependency_type. ++ */ ++ LOCAL_ASSERT((id == 0 && dep_type == BASE_JD_DEP_TYPE_INVALID) || ++ (id > 0 && dep_type != BASE_JD_DEP_TYPE_INVALID)); ++ ++ dep->atom_id = id; ++ dep->dependency_type = dep_type; ++} ++ ++/** ++ * @brief Make a copy of a dependency structure ++ * ++ * @param[in,out] dep The kbase jd atom dependency to be written. ++ * @param[in] from The dependency to make a copy from. ++ * ++ */ ++static inline void base_jd_atom_dep_copy(struct base_dependency *dep, ++ const struct base_dependency *from) ++{ ++ LOCAL_ASSERT(dep != NULL); ++ ++ base_jd_atom_dep_set(dep, from->atom_id, from->dependency_type); ++} ++ ++/** ++ * @brief Soft-atom fence trigger setup. ++ * ++ * Sets up an atom to be a SW-only atom signaling a fence ++ * when it reaches the run state. ++ * ++ * Using the existing base dependency system the fence can ++ * be set to trigger when a GPU job has finished. ++ * ++ * The base fence object must not be terminated until the atom ++ * has been submitted to @ref base_jd_submit and @ref base_jd_submit ++ * has returned. ++ * ++ * @a fence must be a valid fence set up with @a base_fence_init. ++ * Calling this function with a uninitialized fence results in undefined behavior. ++ * ++ * @param[out] atom A pre-allocated atom to configure as a fence trigger SW atom ++ * @param[in] fence The base fence object to trigger. ++ * ++ * @pre @p fence must reference a @ref base_fence successfully initialized by ++ * calling @ref base_fence_init. ++ * @pre @p fence was @e not initialized by calling @ref base_fence_import, nor ++ * is it associated with a fence-trigger job that was already submitted ++ * by calling @ref base_jd_submit. ++ * @post @p atom can be submitted by calling @ref base_jd_submit. ++ */ ++static inline void base_jd_fence_trigger_setup_v2(struct base_jd_atom_v2 *atom, struct base_fence *fence) ++{ ++ LOCAL_ASSERT(atom); ++ LOCAL_ASSERT(fence); ++ LOCAL_ASSERT(fence->basep.fd == INVALID_PLATFORM_FENCE); ++ LOCAL_ASSERT(fence->basep.stream_fd >= 0); ++ atom->jc = (uintptr_t) fence; ++ atom->core_req = BASE_JD_REQ_SOFT_FENCE_TRIGGER; ++} ++ ++/** ++ * @brief Soft-atom fence wait setup. ++ * ++ * Sets up an atom to be a SW-only atom waiting on a fence. ++ * When the fence becomes triggered the atom becomes runnable ++ * and completes immediately. ++ * ++ * Using the existing base dependency system the fence can ++ * be set to block a GPU job until it has been triggered. ++ * ++ * The base fence object must not be terminated until the atom ++ * has been submitted to @ref base_jd_submit and ++ * @ref base_jd_submit has returned. ++ * ++ * @param[out] atom A pre-allocated atom to configure as a fence wait SW atom ++ * @param[in] fence The base fence object to wait on ++ * ++ * @pre @p fence must reference a @ref base_fence successfully initialized by ++ * calling @ref base_fence_import, or it must be associated with a ++ * fence-trigger job that was already submitted by calling ++ * @ref base_jd_submit. ++ * @post @p atom can be submitted by calling @ref base_jd_submit. ++ */ ++static inline void base_jd_fence_wait_setup_v2(struct base_jd_atom_v2 *atom, struct base_fence *fence) ++{ ++ LOCAL_ASSERT(atom); ++ LOCAL_ASSERT(fence); ++ LOCAL_ASSERT(fence->basep.fd >= 0); ++ atom->jc = (uintptr_t) fence; ++ atom->core_req = BASE_JD_REQ_SOFT_FENCE_WAIT; ++} ++ ++/** ++ * @brief External resource info initialization. ++ * ++ * Sets up an external resource object to reference ++ * a memory allocation and the type of access requested. ++ * ++ * @param[in] res The resource object to initialize ++ * @param handle The handle to the imported memory object, must be ++ * obtained by calling @ref base_mem_as_import_handle(). ++ * @param access The type of access requested ++ */ ++static inline void base_external_resource_init(struct base_external_resource *res, struct base_import_handle handle, base_external_resource_access access) ++{ ++ u64 address; ++ ++ address = handle.basep.handle; ++ ++ LOCAL_ASSERT(res != NULL); ++ LOCAL_ASSERT(0 == (address & LOCAL_PAGE_LSB)); ++ LOCAL_ASSERT(access == BASE_EXT_RES_ACCESS_SHARED || access == BASE_EXT_RES_ACCESS_EXCLUSIVE); ++ ++ res->ext_resource = address | (access & LOCAL_PAGE_LSB); ++} ++ ++/** ++ * @brief Job chain event code bits ++ * Defines the bits used to create ::base_jd_event_code ++ */ ++enum { ++ BASE_JD_SW_EVENT_KERNEL = (1u << 15), /**< Kernel side event */ ++ BASE_JD_SW_EVENT = (1u << 14), /**< SW defined event */ ++ BASE_JD_SW_EVENT_SUCCESS = (1u << 13), /**< Event idicates success (SW events only) */ ++ BASE_JD_SW_EVENT_JOB = (0u << 11), /**< Job related event */ ++ BASE_JD_SW_EVENT_BAG = (1u << 11), /**< Bag related event */ ++ BASE_JD_SW_EVENT_INFO = (2u << 11), /**< Misc/info event */ ++ BASE_JD_SW_EVENT_RESERVED = (3u << 11), /**< Reserved event type */ ++ BASE_JD_SW_EVENT_TYPE_MASK = (3u << 11) /**< Mask to extract the type from an event code */ ++}; ++ ++/** ++ * @brief Job chain event codes ++ * ++ * HW and low-level SW events are represented by event codes. ++ * The status of jobs which succeeded are also represented by ++ * an event code (see ::BASE_JD_EVENT_DONE). ++ * Events are usually reported as part of a ::base_jd_event. ++ * ++ * The event codes are encoded in the following way: ++ * @li 10:0 - subtype ++ * @li 12:11 - type ++ * @li 13 - SW success (only valid if the SW bit is set) ++ * @li 14 - SW event (HW event if not set) ++ * @li 15 - Kernel event (should never be seen in userspace) ++ * ++ * Events are split up into ranges as follows: ++ * - BASE_JD_EVENT_RANGE_\_START ++ * - BASE_JD_EVENT_RANGE_\_END ++ * ++ * \a code is in \'s range when: ++ * - BASE_JD_EVENT_RANGE_\_START <= code < BASE_JD_EVENT_RANGE_\_END ++ * ++ * Ranges can be asserted for adjacency by testing that the END of the previous ++ * is equal to the START of the next. This is useful for optimizing some tests ++ * for range. ++ * ++ * A limitation is that the last member of this enum must explicitly be handled ++ * (with an assert-unreachable statement) in switch statements that use ++ * variables of this type. Otherwise, the compiler warns that we have not ++ * handled that enum value. ++ */ ++typedef enum base_jd_event_code { ++ /* HW defined exceptions */ ++ ++ /** Start of HW Non-fault status codes ++ * ++ * @note Obscurely, BASE_JD_EVENT_TERMINATED indicates a real fault, ++ * because the job was hard-stopped ++ */ ++ BASE_JD_EVENT_RANGE_HW_NONFAULT_START = 0, ++ ++ /* non-fatal exceptions */ ++ BASE_JD_EVENT_NOT_STARTED = 0x00, /**< Can't be seen by userspace, treated as 'previous job done' */ ++ BASE_JD_EVENT_DONE = 0x01, ++ BASE_JD_EVENT_STOPPED = 0x03, /**< Can't be seen by userspace, becomes TERMINATED, DONE or JOB_CANCELLED */ ++ BASE_JD_EVENT_TERMINATED = 0x04, /**< This is actually a fault status code - the job was hard stopped */ ++ BASE_JD_EVENT_ACTIVE = 0x08, /**< Can't be seen by userspace, jobs only returned on complete/fail/cancel */ ++ ++ /** End of HW Non-fault status codes ++ * ++ * @note Obscurely, BASE_JD_EVENT_TERMINATED indicates a real fault, ++ * because the job was hard-stopped ++ */ ++ BASE_JD_EVENT_RANGE_HW_NONFAULT_END = 0x40, ++ ++ /** Start of HW fault and SW Error status codes */ ++ BASE_JD_EVENT_RANGE_HW_FAULT_OR_SW_ERROR_START = 0x40, ++ ++ /* job exceptions */ ++ BASE_JD_EVENT_JOB_CONFIG_FAULT = 0x40, ++ BASE_JD_EVENT_JOB_POWER_FAULT = 0x41, ++ BASE_JD_EVENT_JOB_READ_FAULT = 0x42, ++ BASE_JD_EVENT_JOB_WRITE_FAULT = 0x43, ++ BASE_JD_EVENT_JOB_AFFINITY_FAULT = 0x44, ++ BASE_JD_EVENT_JOB_BUS_FAULT = 0x48, ++ BASE_JD_EVENT_INSTR_INVALID_PC = 0x50, ++ BASE_JD_EVENT_INSTR_INVALID_ENC = 0x51, ++ BASE_JD_EVENT_INSTR_TYPE_MISMATCH = 0x52, ++ BASE_JD_EVENT_INSTR_OPERAND_FAULT = 0x53, ++ BASE_JD_EVENT_INSTR_TLS_FAULT = 0x54, ++ BASE_JD_EVENT_INSTR_BARRIER_FAULT = 0x55, ++ BASE_JD_EVENT_INSTR_ALIGN_FAULT = 0x56, ++ BASE_JD_EVENT_DATA_INVALID_FAULT = 0x58, ++ BASE_JD_EVENT_TILE_RANGE_FAULT = 0x59, ++ BASE_JD_EVENT_STATE_FAULT = 0x5A, ++ BASE_JD_EVENT_OUT_OF_MEMORY = 0x60, ++ BASE_JD_EVENT_UNKNOWN = 0x7F, ++ ++ /* GPU exceptions */ ++ BASE_JD_EVENT_DELAYED_BUS_FAULT = 0x80, ++ BASE_JD_EVENT_SHAREABILITY_FAULT = 0x88, ++ ++ /* MMU exceptions */ ++ BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL1 = 0xC1, ++ BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL2 = 0xC2, ++ BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL3 = 0xC3, ++ BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL4 = 0xC4, ++ BASE_JD_EVENT_PERMISSION_FAULT = 0xC8, ++ BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL1 = 0xD1, ++ BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL2 = 0xD2, ++ BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL3 = 0xD3, ++ BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL4 = 0xD4, ++ BASE_JD_EVENT_ACCESS_FLAG = 0xD8, ++ ++ /* SW defined exceptions */ ++ BASE_JD_EVENT_MEM_GROWTH_FAILED = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x000, ++ BASE_JD_EVENT_TIMED_OUT = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x001, ++ BASE_JD_EVENT_JOB_CANCELLED = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x002, ++ BASE_JD_EVENT_JOB_INVALID = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x003, ++ BASE_JD_EVENT_PM_EVENT = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x004, ++ BASE_JD_EVENT_FORCE_REPLAY = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x005, ++ ++ BASE_JD_EVENT_BAG_INVALID = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_BAG | 0x003, ++ ++ /** End of HW fault and SW Error status codes */ ++ BASE_JD_EVENT_RANGE_HW_FAULT_OR_SW_ERROR_END = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_RESERVED | 0x3FF, ++ ++ /** Start of SW Success status codes */ ++ BASE_JD_EVENT_RANGE_SW_SUCCESS_START = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_SUCCESS | 0x000, ++ ++ BASE_JD_EVENT_PROGRESS_REPORT = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_SUCCESS | BASE_JD_SW_EVENT_JOB | 0x000, ++ BASE_JD_EVENT_BAG_DONE = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_SUCCESS | BASE_JD_SW_EVENT_BAG | 0x000, ++ BASE_JD_EVENT_DRV_TERMINATED = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_SUCCESS | BASE_JD_SW_EVENT_INFO | 0x000, ++ ++ /** End of SW Success status codes */ ++ BASE_JD_EVENT_RANGE_SW_SUCCESS_END = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_SUCCESS | BASE_JD_SW_EVENT_RESERVED | 0x3FF, ++ ++ /** Start of Kernel-only status codes. Such codes are never returned to user-space */ ++ BASE_JD_EVENT_RANGE_KERNEL_ONLY_START = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_KERNEL | 0x000, ++ BASE_JD_EVENT_REMOVED_FROM_NEXT = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_KERNEL | BASE_JD_SW_EVENT_JOB | 0x000, ++ ++ /** End of Kernel-only status codes. */ ++ BASE_JD_EVENT_RANGE_KERNEL_ONLY_END = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_KERNEL | BASE_JD_SW_EVENT_RESERVED | 0x3FF ++} base_jd_event_code; ++ ++/** ++ * @brief Event reporting structure ++ * ++ * This structure is used by the kernel driver to report information ++ * about GPU events. The can either be HW-specific events or low-level ++ * SW events, such as job-chain completion. ++ * ++ * The event code contains an event type field which can be extracted ++ * by ANDing with ::BASE_JD_SW_EVENT_TYPE_MASK. ++ * ++ * Based on the event type base_jd_event::data holds: ++ * @li ::BASE_JD_SW_EVENT_JOB : the offset in the ring-buffer for the completed ++ * job-chain ++ * @li ::BASE_JD_SW_EVENT_BAG : The address of the ::base_jd_bag that has ++ * been completed (ie all contained job-chains have been completed). ++ * @li ::BASE_JD_SW_EVENT_INFO : base_jd_event::data not used ++ */ ++typedef struct base_jd_event_v2 { ++ base_jd_event_code event_code; /**< event code */ ++ base_atom_id atom_number; /**< the atom number that has completed */ ++ struct base_jd_udata udata; /**< user data */ ++} base_jd_event_v2; ++ ++/** ++ * @brief Structure for BASE_JD_REQ_SOFT_DUMP_CPU_GPU_COUNTERS jobs. ++ * ++ * This structure is stored into the memory pointed to by the @c jc field ++ * of @ref base_jd_atom. ++ * ++ * It must not occupy the same CPU cache line(s) as any neighboring data. ++ * This is to avoid cases where access to pages containing the structure ++ * is shared between cached and un-cached memory regions, which would ++ * cause memory corruption. ++ */ ++ ++typedef struct base_dump_cpu_gpu_counters { ++ u64 system_time; ++ u64 cycle_counter; ++ u64 sec; ++ u32 usec; ++ u8 padding[36]; ++} base_dump_cpu_gpu_counters; ++ ++/** @} end group base_user_api_job_dispatch */ ++ ++#define GPU_MAX_JOB_SLOTS 16 ++ ++/** ++ * @page page_base_user_api_gpuprops User-side Base GPU Property Query API ++ * ++ * The User-side Base GPU Property Query API encapsulates two ++ * sub-modules: ++ * ++ * - @ref base_user_api_gpuprops_dyn "Dynamic GPU Properties" ++ * - @ref base_plat_config_gpuprops "Base Platform Config GPU Properties" ++ * ++ * There is a related third module outside of Base, which is owned by the MIDG ++ * module: ++ * - @ref gpu_props_static "Midgard Compile-time GPU Properties" ++ * ++ * Base only deals with properties that vary between different Midgard ++ * implementations - the Dynamic GPU properties and the Platform Config ++ * properties. ++ * ++ * For properties that are constant for the Midgard Architecture, refer to the ++ * MIDG module. However, we will discuss their relevance here just to ++ * provide background information. ++ * ++ * @section sec_base_user_api_gpuprops_about About the GPU Properties in Base and MIDG modules ++ * ++ * The compile-time properties (Platform Config, Midgard Compile-time ++ * properties) are exposed as pre-processor macros. ++ * ++ * Complementing the compile-time properties are the Dynamic GPU ++ * Properties, which act as a conduit for the Midgard Configuration ++ * Discovery. ++ * ++ * In general, the dynamic properties are present to verify that the platform ++ * has been configured correctly with the right set of Platform Config ++ * Compile-time Properties. ++ * ++ * As a consistent guide across the entire DDK, the choice for dynamic or ++ * compile-time should consider the following, in order: ++ * -# Can the code be written so that it doesn't need to know the ++ * implementation limits at all? ++ * -# If you need the limits, get the information from the Dynamic Property ++ * lookup. This should be done once as you fetch the context, and then cached ++ * as part of the context data structure, so it's cheap to access. ++ * -# If there's a clear and arguable inefficiency in using Dynamic Properties, ++ * then use a Compile-Time Property (Platform Config, or Midgard Compile-time ++ * property). Examples of where this might be sensible follow: ++ * - Part of a critical inner-loop ++ * - Frequent re-use throughout the driver, causing significant extra load ++ * instructions or control flow that would be worthwhile optimizing out. ++ * ++ * We cannot provide an exhaustive set of examples, neither can we provide a ++ * rule for every possible situation. Use common sense, and think about: what ++ * the rest of the driver will be doing; how the compiler might represent the ++ * value if it is a compile-time constant; whether an OEM shipping multiple ++ * devices would benefit much more from a single DDK binary, instead of ++ * insignificant micro-optimizations. ++ * ++ * @section sec_base_user_api_gpuprops_dyn Dynamic GPU Properties ++ * ++ * Dynamic GPU properties are presented in two sets: ++ * -# the commonly used properties in @ref base_gpu_props, which have been ++ * unpacked from GPU register bitfields. ++ * -# The full set of raw, unprocessed properties in @ref gpu_raw_gpu_props ++ * (also a member of @ref base_gpu_props). All of these are presented in ++ * the packed form, as presented by the GPU registers themselves. ++ * ++ * @usecase The raw properties in @ref gpu_raw_gpu_props are necessary to ++ * allow a user of the Mali Tools (e.g. PAT) to determine "Why is this device ++ * behaving differently?". In this case, all information about the ++ * configuration is potentially useful, but it does not need to be processed ++ * by the driver. Instead, the raw registers can be processed by the Mali ++ * Tools software on the host PC. ++ * ++ * The properties returned extend the Midgard Configuration Discovery ++ * registers. For example, GPU clock speed is not specified in the Midgard ++ * Architecture, but is necessary for OpenCL's clGetDeviceInfo() function. ++ * ++ * The GPU properties are obtained by a call to ++ * _mali_base_get_gpu_props(). This simply returns a pointer to a const ++ * base_gpu_props structure. It is constant for the life of a base ++ * context. Multiple calls to _mali_base_get_gpu_props() to a base context ++ * return the same pointer to a constant structure. This avoids cache pollution ++ * of the common data. ++ * ++ * This pointer must not be freed, because it does not point to the start of a ++ * region allocated by the memory allocator; instead, just close the @ref ++ * base_context. ++ * ++ * ++ * @section sec_base_user_api_gpuprops_config Platform Config Compile-time Properties ++ * ++ * The Platform Config File sets up gpu properties that are specific to a ++ * certain platform. Properties that are 'Implementation Defined' in the ++ * Midgard Architecture spec are placed here. ++ * ++ * @note Reference configurations are provided for Midgard Implementations, such as ++ * the Mali-T600 family. The customer need not repeat this information, and can select one of ++ * these reference configurations. For example, VA_BITS, PA_BITS and the ++ * maximum number of samples per pixel might vary between Midgard Implementations, but ++ * \b not for platforms using the Mali-T604. This information is placed in ++ * the reference configuration files. ++ * ++ * The System Integrator creates the following structure: ++ * - platform_XYZ ++ * - platform_XYZ/plat ++ * - platform_XYZ/plat/plat_config.h ++ * ++ * They then edit plat_config.h, using the example plat_config.h files as a ++ * guide. ++ * ++ * At the very least, the customer must set @ref CONFIG_GPU_CORE_TYPE, and will ++ * receive a helpful \#error message if they do not do this correctly. This ++ * selects the Reference Configuration for the Midgard Implementation. The rationale ++ * behind this decision (against asking the customer to write \#include ++ * in their plat_config.h) is as follows: ++ * - This mechanism 'looks' like a regular config file (such as Linux's ++ * .config) ++ * - It is difficult to get wrong in a way that will produce strange build ++ * errors: ++ * - They need not know where the mali_t600.h, other_midg_gpu.h etc. files are stored - and ++ * so they won't accidentally pick another file with 'mali_t600' in its name ++ * - When the build doesn't work, the System Integrator may think the DDK is ++ * doesn't work, and attempt to fix it themselves: ++ * - For the @ref CONFIG_GPU_CORE_TYPE mechanism, the only way to get past the ++ * error is to set @ref CONFIG_GPU_CORE_TYPE, and this is what the \#error tells ++ * you. ++ * - For a \#include mechanism, checks must still be made elsewhere, which the ++ * System Integrator may try working around by setting \#defines (such as ++ * VA_BITS) themselves in their plat_config.h. In the worst case, they may ++ * set the prevention-mechanism \#define of ++ * "A_CORRECT_MIDGARD_CORE_WAS_CHOSEN". ++ * - In this case, they would believe they are on the right track, because ++ * the build progresses with their fix, but with errors elsewhere. ++ * ++ * However, there is nothing to prevent the customer using \#include to organize ++ * their own configurations files hierarchically. ++ * ++ * The mechanism for the header file processing is as follows: ++ * ++ * @dot ++ digraph plat_config_mechanism { ++ rankdir=BT ++ size="6,6" ++ ++ "mali_base.h"; ++ "gpu/mali_gpu.h"; ++ ++ node [ shape=box ]; ++ { ++ rank = same; ordering = out; ++ ++ "gpu/mali_gpu_props.h"; ++ "base/midg_gpus/mali_t600.h"; ++ "base/midg_gpus/other_midg_gpu.h"; ++ } ++ { rank = same; "plat/plat_config.h"; } ++ { ++ rank = same; ++ "gpu/mali_gpu.h" [ shape=box ]; ++ gpu_chooser [ label="" style="invisible" width=0 height=0 fixedsize=true ]; ++ select_gpu [ label="Mali-T600 | Other\n(select_gpu.h)" shape=polygon,sides=4,distortion=0.25 width=3.3 height=0.99 fixedsize=true ] ; ++ } ++ node [ shape=box ]; ++ { rank = same; "plat/plat_config.h"; } ++ { rank = same; "mali_base.h"; } ++ ++ "mali_base.h" -> "gpu/mali_gpu.h" -> "gpu/mali_gpu_props.h"; ++ "mali_base.h" -> "plat/plat_config.h" ; ++ "mali_base.h" -> select_gpu ; ++ ++ "plat/plat_config.h" -> gpu_chooser [style="dotted,bold" dir=none weight=4] ; ++ gpu_chooser -> select_gpu [style="dotted,bold"] ; ++ ++ select_gpu -> "base/midg_gpus/mali_t600.h" ; ++ select_gpu -> "base/midg_gpus/other_midg_gpu.h" ; ++ } ++ @enddot ++ * ++ * ++ * @section sec_base_user_api_gpuprops_kernel Kernel Operation ++ * ++ * During Base Context Create time, user-side makes a single kernel call: ++ * - A call to fill user memory with GPU information structures ++ * ++ * The kernel-side will fill the provided the entire processed @ref base_gpu_props ++ * structure, because this information is required in both ++ * user and kernel side; it does not make sense to decode it twice. ++ * ++ * Coherency groups must be derived from the bitmasks, but this can be done ++ * kernel side, and just once at kernel startup: Coherency groups must already ++ * be known kernel-side, to support chains that specify a 'Only Coherent Group' ++ * SW requirement, or 'Only Coherent Group with Tiler' SW requirement. ++ * ++ * @section sec_base_user_api_gpuprops_cocalc Coherency Group calculation ++ * Creation of the coherent group data is done at device-driver startup, and so ++ * is one-time. This will most likely involve a loop with CLZ, shifting, and ++ * bit clearing on the L2_PRESENT mask, depending on whether the ++ * system is L2 Coherent. The number of shader cores is done by a ++ * population count, since faulty cores may be disabled during production, ++ * producing a non-contiguous mask. ++ * ++ * The memory requirements for this algorithm can be determined either by a u64 ++ * population count on the L2_PRESENT mask (a LUT helper already is ++ * required for the above), or simple assumption that there can be no more than ++ * 16 coherent groups, since core groups are typically 4 cores. ++ */ ++ ++/** ++ * @addtogroup base_user_api_gpuprops User-side Base GPU Property Query APIs ++ * @{ ++ */ ++ ++/** ++ * @addtogroup base_user_api_gpuprops_dyn Dynamic HW Properties ++ * @{ ++ */ ++ ++#define BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS 3 ++ ++#define BASE_MAX_COHERENT_GROUPS 16 ++ ++struct mali_base_gpu_core_props { ++ /** ++ * Product specific value. ++ */ ++ u32 product_id; ++ ++ /** ++ * Status of the GPU release. ++ * No defined values, but starts at 0 and increases by one for each ++ * release status (alpha, beta, EAC, etc.). ++ * 4 bit values (0-15). ++ */ ++ u16 version_status; ++ ++ /** ++ * Minor release number of the GPU. "P" part of an "RnPn" release number. ++ * 8 bit values (0-255). ++ */ ++ u16 minor_revision; ++ ++ /** ++ * Major release number of the GPU. "R" part of an "RnPn" release number. ++ * 4 bit values (0-15). ++ */ ++ u16 major_revision; ++ ++ u16 padding; ++ ++ /** ++ * This property is deprecated since it has not contained the real current ++ * value of GPU clock speed. It is kept here only for backwards compatibility. ++ * For the new ioctl interface, it is ignored and is treated as a padding ++ * to keep the structure of the same size and retain the placement of its ++ * members. ++ */ ++ u32 gpu_speed_mhz; ++ ++ /** ++ * @usecase GPU clock max/min speed is required for computing best/worst case ++ * in tasks as job scheduling ant irq_throttling. (It is not specified in the ++ * Midgard Architecture). ++ * Also, GPU clock max speed is used for OpenCL's clGetDeviceInfo() function. ++ */ ++ u32 gpu_freq_khz_max; ++ u32 gpu_freq_khz_min; ++ ++ /** ++ * Size of the shader program counter, in bits. ++ */ ++ u32 log2_program_counter_size; ++ ++ /** ++ * TEXTURE_FEATURES_x registers, as exposed by the GPU. This is a ++ * bitpattern where a set bit indicates that the format is supported. ++ * ++ * Before using a texture format, it is recommended that the corresponding ++ * bit be checked. ++ */ ++ u32 texture_features[BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS]; ++ ++ /** ++ * Theoretical maximum memory available to the GPU. It is unlikely that a ++ * client will be able to allocate all of this memory for their own ++ * purposes, but this at least provides an upper bound on the memory ++ * available to the GPU. ++ * ++ * This is required for OpenCL's clGetDeviceInfo() call when ++ * CL_DEVICE_GLOBAL_MEM_SIZE is requested, for OpenCL GPU devices. The ++ * client will not be expecting to allocate anywhere near this value. ++ */ ++ u64 gpu_available_memory_size; ++}; ++ ++/** ++ * ++ * More information is possible - but associativity and bus width are not ++ * required by upper-level apis. ++ */ ++struct mali_base_gpu_l2_cache_props { ++ u8 log2_line_size; ++ u8 log2_cache_size; ++ u8 num_l2_slices; /* Number of L2C slices. 1 or higher */ ++ u8 padding[5]; ++}; ++ ++struct mali_base_gpu_tiler_props { ++ u32 bin_size_bytes; /* Max is 4*2^15 */ ++ u32 max_active_levels; /* Max is 2^15 */ ++}; ++ ++/** ++ * GPU threading system details. ++ */ ++struct mali_base_gpu_thread_props { ++ u32 max_threads; /* Max. number of threads per core */ ++ u32 max_workgroup_size; /* Max. number of threads per workgroup */ ++ u32 max_barrier_size; /* Max. number of threads that can synchronize on a simple barrier */ ++ u16 max_registers; /* Total size [1..65535] of the register file available per core. */ ++ u8 max_task_queue; /* Max. tasks [1..255] which may be sent to a core before it becomes blocked. */ ++ u8 max_thread_group_split; /* Max. allowed value [1..15] of the Thread Group Split field. */ ++ u8 impl_tech; /* 0 = Not specified, 1 = Silicon, 2 = FPGA, 3 = SW Model/Emulation */ ++ u8 padding[7]; ++}; ++ ++/** ++ * @brief descriptor for a coherent group ++ * ++ * \c core_mask exposes all cores in that coherent group, and \c num_cores ++ * provides a cached population-count for that mask. ++ * ++ * @note Whilst all cores are exposed in the mask, not all may be available to ++ * the application, depending on the Kernel Power policy. ++ * ++ * @note if u64s must be 8-byte aligned, then this structure has 32-bits of wastage. ++ */ ++struct mali_base_gpu_coherent_group { ++ u64 core_mask; /**< Core restriction mask required for the group */ ++ u16 num_cores; /**< Number of cores in the group */ ++ u16 padding[3]; ++}; ++ ++/** ++ * @brief Coherency group information ++ * ++ * Note that the sizes of the members could be reduced. However, the \c group ++ * member might be 8-byte aligned to ensure the u64 core_mask is 8-byte ++ * aligned, thus leading to wastage if the other members sizes were reduced. ++ * ++ * The groups are sorted by core mask. The core masks are non-repeating and do ++ * not intersect. ++ */ ++struct mali_base_gpu_coherent_group_info { ++ u32 num_groups; ++ ++ /** ++ * Number of core groups (coherent or not) in the GPU. Equivalent to the number of L2 Caches. ++ * ++ * The GPU Counter dumping writes 2048 bytes per core group, regardless of ++ * whether the core groups are coherent or not. Hence this member is needed ++ * to calculate how much memory is required for dumping. ++ * ++ * @note Do not use it to work out how many valid elements are in the ++ * group[] member. Use num_groups instead. ++ */ ++ u32 num_core_groups; ++ ++ /** ++ * Coherency features of the memory, accessed by @ref gpu_mem_features ++ * methods ++ */ ++ u32 coherency; ++ ++ u32 padding; ++ ++ /** ++ * Descriptors of coherent groups ++ */ ++ struct mali_base_gpu_coherent_group group[BASE_MAX_COHERENT_GROUPS]; ++}; ++ ++/** ++ * A complete description of the GPU's Hardware Configuration Discovery ++ * registers. ++ * ++ * The information is presented inefficiently for access. For frequent access, ++ * the values should be better expressed in an unpacked form in the ++ * base_gpu_props structure. ++ * ++ * @usecase The raw properties in @ref gpu_raw_gpu_props are necessary to ++ * allow a user of the Mali Tools (e.g. PAT) to determine "Why is this device ++ * behaving differently?". In this case, all information about the ++ * configuration is potentially useful, but it does not need to be processed ++ * by the driver. Instead, the raw registers can be processed by the Mali ++ * Tools software on the host PC. ++ * ++ */ ++struct gpu_raw_gpu_props { ++ u64 shader_present; ++ u64 tiler_present; ++ u64 l2_present; ++ u64 stack_present; ++ ++ u32 l2_features; ++ u32 suspend_size; /* API 8.2+ */ ++ u32 mem_features; ++ u32 mmu_features; ++ ++ u32 as_present; ++ ++ u32 js_present; ++ u32 js_features[GPU_MAX_JOB_SLOTS]; ++ u32 tiler_features; ++ u32 texture_features[3]; ++ ++ u32 gpu_id; ++ ++ u32 thread_max_threads; ++ u32 thread_max_workgroup_size; ++ u32 thread_max_barrier_size; ++ u32 thread_features; ++ ++ /* ++ * Note: This is the _selected_ coherency mode rather than the ++ * available modes as exposed in the coherency_features register. ++ */ ++ u32 coherency_mode; ++}; ++ ++/** ++ * Return structure for _mali_base_get_gpu_props(). ++ * ++ * NOTE: the raw_props member in this data structure contains the register ++ * values from which the value of the other members are derived. The derived ++ * members exist to allow for efficient access and/or shielding the details ++ * of the layout of the registers. ++ * ++ */ ++typedef struct mali_base_gpu_props { ++ struct mali_base_gpu_core_props core_props; ++ struct mali_base_gpu_l2_cache_props l2_props; ++ u64 unused_1; /* keep for backwards compatibility */ ++ struct mali_base_gpu_tiler_props tiler_props; ++ struct mali_base_gpu_thread_props thread_props; ++ ++ /** This member is large, likely to be 128 bytes */ ++ struct gpu_raw_gpu_props raw_props; ++ ++ /** This must be last member of the structure */ ++ struct mali_base_gpu_coherent_group_info coherency_info; ++} base_gpu_props; ++ ++/** @} end group base_user_api_gpuprops_dyn */ ++ ++/** @} end group base_user_api_gpuprops */ ++ ++/** ++ * @addtogroup base_user_api_core User-side Base core APIs ++ * @{ ++ */ ++ ++/** ++ * \enum base_context_create_flags ++ * ++ * Flags to pass to ::base_context_init. ++ * Flags can be ORed together to enable multiple things. ++ * ++ * These share the same space as BASEP_CONTEXT_FLAG_*, and so must ++ * not collide with them. ++ */ ++enum base_context_create_flags { ++ /** No flags set */ ++ BASE_CONTEXT_CREATE_FLAG_NONE = 0, ++ ++ /** Base context is embedded in a cctx object (flag used for CINSTR software counter macros) */ ++ BASE_CONTEXT_CCTX_EMBEDDED = (1u << 0), ++ ++ /** Base context is a 'System Monitor' context for Hardware counters. ++ * ++ * One important side effect of this is that job submission is disabled. */ ++ BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED = (1u << 1) ++}; ++ ++/** ++ * Bitpattern describing the ::base_context_create_flags that can be passed to base_context_init() ++ */ ++#define BASE_CONTEXT_CREATE_ALLOWED_FLAGS \ ++ (((u32)BASE_CONTEXT_CCTX_EMBEDDED) | \ ++ ((u32)BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED)) ++ ++/** ++ * Bitpattern describing the ::base_context_create_flags that can be passed to the kernel ++ */ ++#define BASE_CONTEXT_CREATE_KERNEL_FLAGS \ ++ ((u32)BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED) ++ ++/* ++ * Private flags used on the base context ++ * ++ * These start at bit 31, and run down to zero. ++ * ++ * They share the same space as @ref base_context_create_flags, and so must ++ * not collide with them. ++ */ ++/** Private flag tracking whether job descriptor dumping is disabled */ ++#define BASEP_CONTEXT_FLAG_JOB_DUMP_DISABLED ((u32)(1 << 31)) ++ ++/** @} end group base_user_api_core */ ++ ++/** @} end group base_user_api */ ++ ++/** ++ * @addtogroup base_plat_config_gpuprops Base Platform Config GPU Properties ++ * @{ ++ * ++ * C Pre-processor macros are exposed here to do with Platform ++ * Config. ++ * ++ * These include: ++ * - GPU Properties that are constant on a particular Midgard Family ++ * Implementation e.g. Maximum samples per pixel on Mali-T600. ++ * - General platform config for the GPU, such as the GPU major and minor ++ * revison. ++ */ ++ ++/** @} end group base_plat_config_gpuprops */ ++ ++/** ++ * @addtogroup base_api Base APIs ++ * @{ ++ */ ++ ++/** ++ * @brief The payload for a replay job. This must be in GPU memory. ++ */ ++typedef struct base_jd_replay_payload { ++ /** ++ * Pointer to the first entry in the base_jd_replay_jc list. These ++ * will be replayed in @b reverse order (so that extra ones can be added ++ * to the head in future soft jobs without affecting this soft job) ++ */ ++ u64 tiler_jc_list; ++ ++ /** ++ * Pointer to the fragment job chain. ++ */ ++ u64 fragment_jc; ++ ++ /** ++ * Pointer to the tiler heap free FBD field to be modified. ++ */ ++ u64 tiler_heap_free; ++ ++ /** ++ * Hierarchy mask for the replayed fragment jobs. May be zero. ++ */ ++ u16 fragment_hierarchy_mask; ++ ++ /** ++ * Hierarchy mask for the replayed tiler jobs. May be zero. ++ */ ++ u16 tiler_hierarchy_mask; ++ ++ /** ++ * Default weight to be used for hierarchy levels not in the original ++ * mask. ++ */ ++ u32 hierarchy_default_weight; ++ ++ /** ++ * Core requirements for the tiler job chain ++ */ ++ base_jd_core_req tiler_core_req; ++ ++ /** ++ * Core requirements for the fragment job chain ++ */ ++ base_jd_core_req fragment_core_req; ++} base_jd_replay_payload; ++ ++#ifdef BASE_LEGACY_UK10_2_SUPPORT ++typedef struct base_jd_replay_payload_uk10_2 { ++ u64 tiler_jc_list; ++ u64 fragment_jc; ++ u64 tiler_heap_free; ++ u16 fragment_hierarchy_mask; ++ u16 tiler_hierarchy_mask; ++ u32 hierarchy_default_weight; ++ u16 tiler_core_req; ++ u16 fragment_core_req; ++ u8 padding[4]; ++} base_jd_replay_payload_uk10_2; ++#endif /* BASE_LEGACY_UK10_2_SUPPORT */ ++ ++/** ++ * @brief An entry in the linked list of job chains to be replayed. This must ++ * be in GPU memory. ++ */ ++typedef struct base_jd_replay_jc { ++ /** ++ * Pointer to next entry in the list. A setting of NULL indicates the ++ * end of the list. ++ */ ++ u64 next; ++ ++ /** ++ * Pointer to the job chain. ++ */ ++ u64 jc; ++ ++} base_jd_replay_jc; ++ ++/* Maximum number of jobs allowed in a fragment chain in the payload of a ++ * replay job */ ++#define BASE_JD_REPLAY_F_CHAIN_JOB_LIMIT 256 ++ ++/** @} end group base_api */ ++ ++typedef struct base_profiling_controls { ++ u32 profiling_controls[FBDUMP_CONTROL_MAX]; ++} base_profiling_controls; ++ ++/* Enable additional tracepoints for latency measurements (TL_ATOM_READY, ++ * TL_ATOM_DONE, TL_ATOM_PRIO_CHANGE, TL_ATOM_EVENT_POST) */ ++#define BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS (1 << 0) ++ ++/* Indicate that job dumping is enabled. This could affect certain timers ++ * to account for the performance impact. */ ++#define BASE_TLSTREAM_JOB_DUMPING_ENABLED (1 << 1) ++ ++#define BASE_TLSTREAM_FLAGS_MASK (BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS | \ ++ BASE_TLSTREAM_JOB_DUMPING_ENABLED) ++ ++#endif /* _BASE_KERNEL_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_base_mem_priv.h b/drivers/gpu/arm/bifrost_for_linux/mali_base_mem_priv.h +new file mode 100755 +index 000000000000..4a98a72cc37a +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_base_mem_priv.h +@@ -0,0 +1,52 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2014 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#ifndef _BASE_MEM_PRIV_H_ ++#define _BASE_MEM_PRIV_H_ ++ ++#define BASE_SYNCSET_OP_MSYNC (1U << 0) ++#define BASE_SYNCSET_OP_CSYNC (1U << 1) ++ ++/* ++ * This structure describe a basic memory coherency operation. ++ * It can either be: ++ * @li a sync from CPU to Memory: ++ * - type = ::BASE_SYNCSET_OP_MSYNC ++ * - mem_handle = a handle to the memory object on which the operation ++ * is taking place ++ * - user_addr = the address of the range to be synced ++ * - size = the amount of data to be synced, in bytes ++ * - offset is ignored. ++ * @li a sync from Memory to CPU: ++ * - type = ::BASE_SYNCSET_OP_CSYNC ++ * - mem_handle = a handle to the memory object on which the operation ++ * is taking place ++ * - user_addr = the address of the range to be synced ++ * - size = the amount of data to be synced, in bytes. ++ * - offset is ignored. ++ */ ++struct basep_syncset { ++ base_mem_handle mem_handle; ++ u64 user_addr; ++ u64 size; ++ u8 type; ++ u8 padding[7]; ++}; ++ ++#endif +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_base_vendor_specific_func.h b/drivers/gpu/arm/bifrost_for_linux/mali_base_vendor_specific_func.h +new file mode 100755 +index 000000000000..be454a216a39 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_base_vendor_specific_func.h +@@ -0,0 +1,24 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010, 2012-2013, 2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++#ifndef _BASE_VENDOR_SPEC_FUNC_H_ ++#define _BASE_VENDOR_SPEC_FUNC_H_ ++ ++int kbase_get_vendor_specific_cpu_clock_speed(u32 * const); ++ ++#endif /*_BASE_VENDOR_SPEC_FUNC_H_*/ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase.h +new file mode 100755 +index 000000000000..1fe936ea6012 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase.h +@@ -0,0 +1,616 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#ifndef _KBASE_H_ ++#define _KBASE_H_ ++ ++#include ++ ++#include ++ ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)) ++#include ++#endif ++#include ++#include ++#include ++#include ++#include ++ ++#include "mali_base_kernel.h" ++#include ++#include ++ ++/* ++ * Include mali_kbase_defs.h first as this provides types needed by other local ++ * header files. ++ */ ++#include "mali_kbase_defs.h" ++ ++#include "mali_kbase_context.h" ++#include "mali_kbase_strings.h" ++#include "mali_kbase_mem_lowlevel.h" ++#include "mali_kbase_trace_timeline.h" ++#include "mali_kbase_js.h" ++#include "mali_kbase_mem.h" ++#include "mali_kbase_utility.h" ++#include "mali_kbase_gpu_memory_debugfs.h" ++#include "mali_kbase_mem_profile_debugfs.h" ++#include "mali_kbase_debug_job_fault.h" ++#include "mali_kbase_jd_debugfs.h" ++#include "mali_kbase_gpuprops.h" ++#include "mali_kbase_jm.h" ++#include "mali_kbase_vinstr.h" ++ ++#include "ipa/mali_kbase_ipa.h" ++ ++#ifdef CONFIG_GPU_TRACEPOINTS ++#include ++#endif ++ ++#ifndef u64_to_user_ptr ++/* Introduced in Linux v4.6 */ ++#define u64_to_user_ptr(x) ((void __user *)(uintptr_t)x) ++#endif ++ ++/* ++ * Kernel-side Base (KBase) APIs ++ */ ++ ++struct kbase_device *kbase_device_alloc(void); ++/* ++* note: configuration attributes member of kbdev needs to have ++* been setup before calling kbase_device_init ++*/ ++ ++/* ++* API to acquire device list semaphore and return pointer ++* to the device list head ++*/ ++const struct list_head *kbase_dev_list_get(void); ++/* API to release the device list semaphore */ ++void kbase_dev_list_put(const struct list_head *dev_list); ++ ++int kbase_device_init(struct kbase_device * const kbdev); ++void kbase_device_term(struct kbase_device *kbdev); ++void kbase_device_free(struct kbase_device *kbdev); ++int kbase_device_has_feature(struct kbase_device *kbdev, u32 feature); ++ ++/* Needed for gator integration and for reporting vsync information */ ++struct kbase_device *kbase_find_device(int minor); ++void kbase_release_device(struct kbase_device *kbdev); ++ ++void kbase_set_profiling_control(struct kbase_device *kbdev, u32 control, u32 value); ++ ++struct kbase_context * ++kbase_create_context(struct kbase_device *kbdev, bool is_compat); ++void kbase_destroy_context(struct kbase_context *kctx); ++ ++int kbase_jd_init(struct kbase_context *kctx); ++void kbase_jd_exit(struct kbase_context *kctx); ++ ++/** ++ * kbase_jd_submit - Submit atoms to the job dispatcher ++ * ++ * @kctx: The kbase context to submit to ++ * @user_addr: The address in user space of the struct base_jd_atom_v2 array ++ * @nr_atoms: The number of atoms in the array ++ * @stride: sizeof(struct base_jd_atom_v2) ++ * @uk6_atom: true if the atoms are legacy atoms (struct base_jd_atom_v2_uk6) ++ * ++ * Return: 0 on success or error code ++ */ ++int kbase_jd_submit(struct kbase_context *kctx, ++ void __user *user_addr, u32 nr_atoms, u32 stride, ++ bool uk6_atom); ++ ++/** ++ * kbase_jd_done_worker - Handle a job completion ++ * @data: a &struct work_struct ++ * ++ * This function requeues the job from the runpool (if it was soft-stopped or ++ * removed from NEXT registers). ++ * ++ * Removes it from the system if it finished/failed/was cancelled. ++ * ++ * Resolves dependencies to add dependent jobs to the context, potentially ++ * starting them if necessary (which may add more references to the context) ++ * ++ * Releases the reference to the context from the no-longer-running job. ++ * ++ * Handles retrying submission outside of IRQ context if it failed from within ++ * IRQ context. ++ */ ++void kbase_jd_done_worker(struct work_struct *data); ++ ++void kbase_jd_done(struct kbase_jd_atom *katom, int slot_nr, ktime_t *end_timestamp, ++ kbasep_js_atom_done_code done_code); ++void kbase_jd_cancel(struct kbase_device *kbdev, struct kbase_jd_atom *katom); ++void kbase_jd_zap_context(struct kbase_context *kctx); ++bool jd_done_nolock(struct kbase_jd_atom *katom, ++ struct list_head *completed_jobs_ctx); ++void kbase_jd_free_external_resources(struct kbase_jd_atom *katom); ++bool jd_submit_atom(struct kbase_context *kctx, ++ const struct base_jd_atom_v2 *user_atom, ++ struct kbase_jd_atom *katom); ++void kbase_jd_dep_clear_locked(struct kbase_jd_atom *katom); ++ ++void kbase_job_done(struct kbase_device *kbdev, u32 done); ++ ++/** ++ * kbase_job_slot_ctx_priority_check_locked(): - Check for lower priority atoms ++ * and soft stop them ++ * @kctx: Pointer to context to check. ++ * @katom: Pointer to priority atom. ++ * ++ * Atoms from @kctx on the same job slot as @katom, which have lower priority ++ * than @katom will be soft stopped and put back in the queue, so that atoms ++ * with higher priority can run. ++ * ++ * The hwaccess_lock must be held when calling this function. ++ */ ++void kbase_job_slot_ctx_priority_check_locked(struct kbase_context *kctx, ++ struct kbase_jd_atom *katom); ++ ++void kbase_job_slot_softstop(struct kbase_device *kbdev, int js, ++ struct kbase_jd_atom *target_katom); ++void kbase_job_slot_softstop_swflags(struct kbase_device *kbdev, int js, ++ struct kbase_jd_atom *target_katom, u32 sw_flags); ++void kbase_job_slot_hardstop(struct kbase_context *kctx, int js, ++ struct kbase_jd_atom *target_katom); ++void kbase_job_check_enter_disjoint(struct kbase_device *kbdev, u32 action, ++ base_jd_core_req core_reqs, struct kbase_jd_atom *target_katom); ++void kbase_job_check_leave_disjoint(struct kbase_device *kbdev, ++ struct kbase_jd_atom *target_katom); ++ ++void kbase_event_post(struct kbase_context *ctx, struct kbase_jd_atom *event); ++int kbase_event_dequeue(struct kbase_context *ctx, struct base_jd_event_v2 *uevent); ++int kbase_event_pending(struct kbase_context *ctx); ++int kbase_event_init(struct kbase_context *kctx); ++void kbase_event_close(struct kbase_context *kctx); ++void kbase_event_cleanup(struct kbase_context *kctx); ++void kbase_event_wakeup(struct kbase_context *kctx); ++ ++int kbase_process_soft_job(struct kbase_jd_atom *katom); ++int kbase_prepare_soft_job(struct kbase_jd_atom *katom); ++void kbase_finish_soft_job(struct kbase_jd_atom *katom); ++void kbase_cancel_soft_job(struct kbase_jd_atom *katom); ++void kbase_resume_suspended_soft_jobs(struct kbase_device *kbdev); ++void kbasep_remove_waiting_soft_job(struct kbase_jd_atom *katom); ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++void kbase_soft_event_wait_callback(struct kbase_jd_atom *katom); ++#endif ++int kbase_soft_event_update(struct kbase_context *kctx, ++ u64 event, ++ unsigned char new_status); ++ ++bool kbase_replay_process(struct kbase_jd_atom *katom); ++ ++void kbasep_soft_job_timeout_worker(struct timer_list *t); ++void kbasep_complete_triggered_soft_events(struct kbase_context *kctx, u64 evt); ++ ++/* api used internally for register access. Contains validation and tracing */ ++void kbase_device_trace_register_access(struct kbase_context *kctx, enum kbase_reg_access_type type, u16 reg_offset, u32 reg_value); ++int kbase_device_trace_buffer_install( ++ struct kbase_context *kctx, u32 *tb, size_t size); ++void kbase_device_trace_buffer_uninstall(struct kbase_context *kctx); ++ ++/* api to be ported per OS, only need to do the raw register access */ ++void kbase_os_reg_write(struct kbase_device *kbdev, u16 offset, u32 value); ++u32 kbase_os_reg_read(struct kbase_device *kbdev, u16 offset); ++ ++void kbasep_as_do_poke(struct work_struct *work); ++ ++/** Returns the name associated with a Mali exception code ++ * ++ * This function is called from the interrupt handler when a GPU fault occurs. ++ * It reports the details of the fault using KBASE_DEBUG_PRINT_WARN. ++ * ++ * @param[in] kbdev The kbase device that the GPU fault occurred from. ++ * @param[in] exception_code exception code ++ * @return name associated with the exception code ++ */ ++const char *kbase_exception_name(struct kbase_device *kbdev, ++ u32 exception_code); ++ ++/** ++ * Check whether a system suspend is in progress, or has already been suspended ++ * ++ * The caller should ensure that either kbdev->pm.active_count_lock is held, or ++ * a dmb was executed recently (to ensure the value is most ++ * up-to-date). However, without a lock the value could change afterwards. ++ * ++ * @return false if a suspend is not in progress ++ * @return !=false otherwise ++ */ ++static inline bool kbase_pm_is_suspending(struct kbase_device *kbdev) ++{ ++ return kbdev->pm.suspending; ++} ++ ++/** ++ * Return the atom's ID, as was originally supplied by userspace in ++ * base_jd_atom_v2::atom_number ++ */ ++static inline int kbase_jd_atom_id(struct kbase_context *kctx, struct kbase_jd_atom *katom) ++{ ++ int result; ++ ++ KBASE_DEBUG_ASSERT(kctx); ++ KBASE_DEBUG_ASSERT(katom); ++ KBASE_DEBUG_ASSERT(katom->kctx == kctx); ++ ++ result = katom - &kctx->jctx.atoms[0]; ++ KBASE_DEBUG_ASSERT(result >= 0 && result <= BASE_JD_ATOM_COUNT); ++ return result; ++} ++ ++/** ++ * kbase_jd_atom_from_id - Return the atom structure for the given atom ID ++ * @kctx: Context pointer ++ * @id: ID of atom to retrieve ++ * ++ * Return: Pointer to struct kbase_jd_atom associated with the supplied ID ++ */ ++static inline struct kbase_jd_atom *kbase_jd_atom_from_id( ++ struct kbase_context *kctx, int id) ++{ ++ return &kctx->jctx.atoms[id]; ++} ++ ++/** ++ * Initialize the disjoint state ++ * ++ * The disjoint event count and state are both set to zero. ++ * ++ * Disjoint functions usage: ++ * ++ * The disjoint event count should be incremented whenever a disjoint event occurs. ++ * ++ * There are several cases which are regarded as disjoint behavior. Rather than just increment ++ * the counter during disjoint events we also increment the counter when jobs may be affected ++ * by what the GPU is currently doing. To facilitate this we have the concept of disjoint state. ++ * ++ * Disjoint state is entered during GPU reset and for the entire time that an atom is replaying ++ * (as part of the replay workaround). Increasing the disjoint state also increases the count of ++ * disjoint events. ++ * ++ * The disjoint state is then used to increase the count of disjoint events during job submission ++ * and job completion. Any atom submitted or completed while the disjoint state is greater than ++ * zero is regarded as a disjoint event. ++ * ++ * The disjoint event counter is also incremented immediately whenever a job is soft stopped ++ * and during context creation. ++ * ++ * @param kbdev The kbase device ++ */ ++void kbase_disjoint_init(struct kbase_device *kbdev); ++ ++/** ++ * Increase the count of disjoint events ++ * called when a disjoint event has happened ++ * ++ * @param kbdev The kbase device ++ */ ++void kbase_disjoint_event(struct kbase_device *kbdev); ++ ++/** ++ * Increase the count of disjoint events only if the GPU is in a disjoint state ++ * ++ * This should be called when something happens which could be disjoint if the GPU ++ * is in a disjoint state. The state refcount keeps track of this. ++ * ++ * @param kbdev The kbase device ++ */ ++void kbase_disjoint_event_potential(struct kbase_device *kbdev); ++ ++/** ++ * Returns the count of disjoint events ++ * ++ * @param kbdev The kbase device ++ * @return the count of disjoint events ++ */ ++u32 kbase_disjoint_event_get(struct kbase_device *kbdev); ++ ++/** ++ * Increment the refcount state indicating that the GPU is in a disjoint state. ++ * ++ * Also Increment the disjoint event count (calls @ref kbase_disjoint_event) ++ * eventually after the disjoint state has completed @ref kbase_disjoint_state_down ++ * should be called ++ * ++ * @param kbdev The kbase device ++ */ ++void kbase_disjoint_state_up(struct kbase_device *kbdev); ++ ++/** ++ * Decrement the refcount state ++ * ++ * Also Increment the disjoint event count (calls @ref kbase_disjoint_event) ++ * ++ * Called after @ref kbase_disjoint_state_up once the disjoint state is over ++ * ++ * @param kbdev The kbase device ++ */ ++void kbase_disjoint_state_down(struct kbase_device *kbdev); ++ ++/** ++ * If a job is soft stopped and the number of contexts is >= this value ++ * it is reported as a disjoint event ++ */ ++#define KBASE_DISJOINT_STATE_INTERLEAVED_CONTEXT_COUNT_THRESHOLD 2 ++ ++#if !defined(UINT64_MAX) ++ #define UINT64_MAX ((uint64_t)0xFFFFFFFFFFFFFFFFULL) ++#endif ++ ++#if KBASE_TRACE_ENABLE ++void kbasep_trace_debugfs_init(struct kbase_device *kbdev); ++ ++#ifndef CONFIG_MALI_BIFROST_SYSTEM_TRACE ++/** Add trace values about a job-slot ++ * ++ * @note Any functions called through this macro will still be evaluated in ++ * Release builds (CONFIG_MALI_BIFROST_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any ++ * functions called to get the parameters supplied to this macro must: ++ * - be static or static inline ++ * - must just return 0 and have no other statements present in the body. ++ */ ++#define KBASE_TRACE_ADD_SLOT(kbdev, code, ctx, katom, gpu_addr, jobslot) \ ++ kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \ ++ KBASE_TRACE_FLAG_JOBSLOT, 0, jobslot, 0) ++ ++/** Add trace values about a job-slot, with info ++ * ++ * @note Any functions called through this macro will still be evaluated in ++ * Release builds (CONFIG_MALI_BIFROST_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any ++ * functions called to get the parameters supplied to this macro must: ++ * - be static or static inline ++ * - must just return 0 and have no other statements present in the body. ++ */ ++#define KBASE_TRACE_ADD_SLOT_INFO(kbdev, code, ctx, katom, gpu_addr, jobslot, info_val) \ ++ kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \ ++ KBASE_TRACE_FLAG_JOBSLOT, 0, jobslot, info_val) ++ ++/** Add trace values about a ctx refcount ++ * ++ * @note Any functions called through this macro will still be evaluated in ++ * Release builds (CONFIG_MALI_BIFROST_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any ++ * functions called to get the parameters supplied to this macro must: ++ * - be static or static inline ++ * - must just return 0 and have no other statements present in the body. ++ */ ++#define KBASE_TRACE_ADD_REFCOUNT(kbdev, code, ctx, katom, gpu_addr, refcount) \ ++ kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \ ++ KBASE_TRACE_FLAG_REFCOUNT, refcount, 0, 0) ++/** Add trace values about a ctx refcount, and info ++ * ++ * @note Any functions called through this macro will still be evaluated in ++ * Release builds (CONFIG_MALI_BIFROST_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any ++ * functions called to get the parameters supplied to this macro must: ++ * - be static or static inline ++ * - must just return 0 and have no other statements present in the body. ++ */ ++#define KBASE_TRACE_ADD_REFCOUNT_INFO(kbdev, code, ctx, katom, gpu_addr, refcount, info_val) \ ++ kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \ ++ KBASE_TRACE_FLAG_REFCOUNT, refcount, 0, info_val) ++ ++/** Add trace values (no slot or refcount) ++ * ++ * @note Any functions called through this macro will still be evaluated in ++ * Release builds (CONFIG_MALI_BIFROST_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any ++ * functions called to get the parameters supplied to this macro must: ++ * - be static or static inline ++ * - must just return 0 and have no other statements present in the body. ++ */ ++#define KBASE_TRACE_ADD(kbdev, code, ctx, katom, gpu_addr, info_val) \ ++ kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \ ++ 0, 0, 0, info_val) ++ ++/** Clear the trace */ ++#define KBASE_TRACE_CLEAR(kbdev) \ ++ kbasep_trace_clear(kbdev) ++ ++/** Dump the slot trace */ ++#define KBASE_TRACE_DUMP(kbdev) \ ++ kbasep_trace_dump(kbdev) ++ ++/** PRIVATE - do not use directly. Use KBASE_TRACE_ADD() instead */ ++void kbasep_trace_add(struct kbase_device *kbdev, enum kbase_trace_code code, void *ctx, struct kbase_jd_atom *katom, u64 gpu_addr, u8 flags, int refcount, int jobslot, unsigned long info_val); ++/** PRIVATE - do not use directly. Use KBASE_TRACE_CLEAR() instead */ ++void kbasep_trace_clear(struct kbase_device *kbdev); ++#else /* #ifndef CONFIG_MALI_BIFROST_SYSTEM_TRACE */ ++/* Dispatch kbase trace events as system trace events */ ++#include ++#define KBASE_TRACE_ADD_SLOT(kbdev, code, ctx, katom, gpu_addr, jobslot)\ ++ trace_mali_##code(jobslot, 0) ++ ++#define KBASE_TRACE_ADD_SLOT_INFO(kbdev, code, ctx, katom, gpu_addr, jobslot, info_val)\ ++ trace_mali_##code(jobslot, info_val) ++ ++#define KBASE_TRACE_ADD_REFCOUNT(kbdev, code, ctx, katom, gpu_addr, refcount)\ ++ trace_mali_##code(refcount, 0) ++ ++#define KBASE_TRACE_ADD_REFCOUNT_INFO(kbdev, code, ctx, katom, gpu_addr, refcount, info_val)\ ++ trace_mali_##code(refcount, info_val) ++ ++#define KBASE_TRACE_ADD(kbdev, code, ctx, katom, gpu_addr, info_val)\ ++ trace_mali_##code(gpu_addr, info_val) ++ ++#define KBASE_TRACE_CLEAR(kbdev)\ ++ do {\ ++ CSTD_UNUSED(kbdev);\ ++ CSTD_NOP(0);\ ++ } while (0) ++#define KBASE_TRACE_DUMP(kbdev)\ ++ do {\ ++ CSTD_UNUSED(kbdev);\ ++ CSTD_NOP(0);\ ++ } while (0) ++ ++#endif /* #ifndef CONFIG_MALI_BIFROST_SYSTEM_TRACE */ ++#else ++#define KBASE_TRACE_ADD_SLOT(kbdev, code, ctx, katom, gpu_addr, jobslot)\ ++ do {\ ++ CSTD_UNUSED(kbdev);\ ++ CSTD_NOP(code);\ ++ CSTD_UNUSED(ctx);\ ++ CSTD_UNUSED(katom);\ ++ CSTD_UNUSED(gpu_addr);\ ++ CSTD_UNUSED(jobslot);\ ++ } while (0) ++ ++#define KBASE_TRACE_ADD_SLOT_INFO(kbdev, code, ctx, katom, gpu_addr, jobslot, info_val)\ ++ do {\ ++ CSTD_UNUSED(kbdev);\ ++ CSTD_NOP(code);\ ++ CSTD_UNUSED(ctx);\ ++ CSTD_UNUSED(katom);\ ++ CSTD_UNUSED(gpu_addr);\ ++ CSTD_UNUSED(jobslot);\ ++ CSTD_UNUSED(info_val);\ ++ CSTD_NOP(0);\ ++ } while (0) ++ ++#define KBASE_TRACE_ADD_REFCOUNT(kbdev, code, ctx, katom, gpu_addr, refcount)\ ++ do {\ ++ CSTD_UNUSED(kbdev);\ ++ CSTD_NOP(code);\ ++ CSTD_UNUSED(ctx);\ ++ CSTD_UNUSED(katom);\ ++ CSTD_UNUSED(gpu_addr);\ ++ CSTD_UNUSED(refcount);\ ++ CSTD_NOP(0);\ ++ } while (0) ++ ++#define KBASE_TRACE_ADD_REFCOUNT_INFO(kbdev, code, ctx, katom, gpu_addr, refcount, info_val)\ ++ do {\ ++ CSTD_UNUSED(kbdev);\ ++ CSTD_NOP(code);\ ++ CSTD_UNUSED(ctx);\ ++ CSTD_UNUSED(katom);\ ++ CSTD_UNUSED(gpu_addr);\ ++ CSTD_UNUSED(info_val);\ ++ CSTD_NOP(0);\ ++ } while (0) ++ ++#define KBASE_TRACE_ADD(kbdev, code, subcode, ctx, katom, val)\ ++ do {\ ++ CSTD_UNUSED(kbdev);\ ++ CSTD_NOP(code);\ ++ CSTD_UNUSED(subcode);\ ++ CSTD_UNUSED(ctx);\ ++ CSTD_UNUSED(katom);\ ++ CSTD_UNUSED(val);\ ++ CSTD_NOP(0);\ ++ } while (0) ++ ++#define KBASE_TRACE_CLEAR(kbdev)\ ++ do {\ ++ CSTD_UNUSED(kbdev);\ ++ CSTD_NOP(0);\ ++ } while (0) ++#define KBASE_TRACE_DUMP(kbdev)\ ++ do {\ ++ CSTD_UNUSED(kbdev);\ ++ CSTD_NOP(0);\ ++ } while (0) ++#endif /* KBASE_TRACE_ENABLE */ ++/** PRIVATE - do not use directly. Use KBASE_TRACE_DUMP() instead */ ++void kbasep_trace_dump(struct kbase_device *kbdev); ++ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++/** ++ * kbase_set_driver_inactive - Force driver to go inactive ++ * @kbdev: Device pointer ++ * @inactive: true if driver should go inactive, false otherwise ++ * ++ * Forcing the driver inactive will cause all future IOCTLs to wait until the ++ * driver is made active again. This is intended solely for the use of tests ++ * which require that no jobs are running while the test executes. ++ */ ++void kbase_set_driver_inactive(struct kbase_device *kbdev, bool inactive); ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++ ++ ++#if defined(CONFIG_DEBUG_FS) && !defined(CONFIG_MALI_BIFROST_NO_MALI) ++ ++/* kbase_io_history_init - initialize data struct for register access history ++ * ++ * @kbdev The register history to initialize ++ * @n The number of register accesses that the buffer could hold ++ * ++ * @return 0 if successfully initialized, failure otherwise ++ */ ++int kbase_io_history_init(struct kbase_io_history *h, u16 n); ++ ++/* kbase_io_history_term - uninit all resources for the register access history ++ * ++ * @h The register history to terminate ++ */ ++void kbase_io_history_term(struct kbase_io_history *h); ++ ++/* kbase_io_history_dump - print the register history to the kernel ring buffer ++ * ++ * @kbdev Pointer to kbase_device containing the register history to dump ++ */ ++void kbase_io_history_dump(struct kbase_device *kbdev); ++ ++/** ++ * kbase_io_history_resize - resize the register access history buffer. ++ * ++ * @h: Pointer to a valid register history to resize ++ * @new_size: Number of accesses the buffer could hold ++ * ++ * A successful resize will clear all recent register accesses. ++ * If resizing fails for any reason (e.g., could not allocate memory, invalid ++ * buffer size) then the original buffer will be kept intact. ++ * ++ * @return 0 if the buffer was resized, failure otherwise ++ */ ++int kbase_io_history_resize(struct kbase_io_history *h, u16 new_size); ++ ++#else /* CONFIG_DEBUG_FS */ ++ ++#define kbase_io_history_init(...) ((int)0) ++ ++#define kbase_io_history_term CSTD_NOP ++ ++#define kbase_io_history_dump CSTD_NOP ++ ++#define kbase_io_history_resize CSTD_NOP ++ ++#endif /* CONFIG_DEBUG_FS */ ++ ++ ++#endif ++ ++ ++ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_10969_workaround.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_10969_workaround.c +new file mode 100755 +index 000000000000..6b3559d93351 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_10969_workaround.c +@@ -0,0 +1,210 @@ ++/* ++ * ++ * (C) COPYRIGHT 2013-2015,2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++#include ++#include ++#include ++ ++/* This function is used to solve an HW issue with single iterator GPUs. ++ * If a fragment job is soft-stopped on the edge of its bounding box, can happen that the ++ * restart index is out of bounds and the rerun causes a tile range fault. If this happens ++ * we try to clamp the restart index to a correct value and rerun the job. ++ */ ++/* Mask of X and Y coordinates for the coordinates words in the descriptors*/ ++#define X_COORDINATE_MASK 0x00000FFF ++#define Y_COORDINATE_MASK 0x0FFF0000 ++/* Max number of words needed from the fragment shader job descriptor */ ++#define JOB_HEADER_SIZE_IN_WORDS 10 ++#define JOB_HEADER_SIZE (JOB_HEADER_SIZE_IN_WORDS*sizeof(u32)) ++ ++/* Word 0: Status Word */ ++#define JOB_DESC_STATUS_WORD 0 ++/* Word 1: Restart Index */ ++#define JOB_DESC_RESTART_INDEX_WORD 1 ++/* Word 2: Fault address low word */ ++#define JOB_DESC_FAULT_ADDR_LOW_WORD 2 ++/* Word 8: Minimum Tile Coordinates */ ++#define FRAG_JOB_DESC_MIN_TILE_COORD_WORD 8 ++/* Word 9: Maximum Tile Coordinates */ ++#define FRAG_JOB_DESC_MAX_TILE_COORD_WORD 9 ++ ++int kbasep_10969_workaround_clamp_coordinates(struct kbase_jd_atom *katom) ++{ ++ struct device *dev = katom->kctx->kbdev->dev; ++ u32 clamped = 0; ++ struct kbase_va_region *region; ++ struct tagged_addr *page_array; ++ u64 page_index; ++ u32 offset = katom->jc & (~PAGE_MASK); ++ u32 *page_1 = NULL; ++ u32 *page_2 = NULL; ++ u32 job_header[JOB_HEADER_SIZE_IN_WORDS]; ++ void *dst = job_header; ++ u32 minX, minY, maxX, maxY; ++ u32 restartX, restartY; ++ struct page *p; ++ u32 copy_size; ++ ++ dev_warn(dev, "Called TILE_RANGE_FAULT workaround clamping function.\n"); ++ if (!(katom->core_req & BASE_JD_REQ_FS)) ++ return 0; ++ ++ kbase_gpu_vm_lock(katom->kctx); ++ region = kbase_region_tracker_find_region_enclosing_address(katom->kctx, ++ katom->jc); ++ if (!region || (region->flags & KBASE_REG_FREE)) ++ goto out_unlock; ++ ++ page_array = kbase_get_cpu_phy_pages(region); ++ if (!page_array) ++ goto out_unlock; ++ ++ page_index = (katom->jc >> PAGE_SHIFT) - region->start_pfn; ++ ++ p = phys_to_page(as_phys_addr_t(page_array[page_index])); ++ ++ /* we need the first 10 words of the fragment shader job descriptor. ++ * We need to check that the offset + 10 words is less that the page ++ * size otherwise we need to load the next page. ++ * page_size_overflow will be equal to 0 in case the whole descriptor ++ * is within the page > 0 otherwise. ++ */ ++ copy_size = MIN(PAGE_SIZE - offset, JOB_HEADER_SIZE); ++ ++ page_1 = kmap_atomic(p); ++ ++ /* page_1 is a u32 pointer, offset is expressed in bytes */ ++ page_1 += offset>>2; ++ ++ kbase_sync_single_for_cpu(katom->kctx->kbdev, ++ kbase_dma_addr(p) + offset, ++ copy_size, DMA_BIDIRECTIONAL); ++ ++ memcpy(dst, page_1, copy_size); ++ ++ /* The data needed overflows page the dimension, ++ * need to map the subsequent page */ ++ if (copy_size < JOB_HEADER_SIZE) { ++ p = phys_to_page(as_phys_addr_t(page_array[page_index + 1])); ++ page_2 = kmap_atomic(p); ++ ++ kbase_sync_single_for_cpu(katom->kctx->kbdev, ++ kbase_dma_addr(p), ++ JOB_HEADER_SIZE - copy_size, DMA_BIDIRECTIONAL); ++ ++ memcpy(dst + copy_size, page_2, JOB_HEADER_SIZE - copy_size); ++ } ++ ++ /* We managed to correctly map one or two pages (in case of overflow) */ ++ /* Get Bounding Box data and restart index from fault address low word */ ++ minX = job_header[FRAG_JOB_DESC_MIN_TILE_COORD_WORD] & X_COORDINATE_MASK; ++ minY = job_header[FRAG_JOB_DESC_MIN_TILE_COORD_WORD] & Y_COORDINATE_MASK; ++ maxX = job_header[FRAG_JOB_DESC_MAX_TILE_COORD_WORD] & X_COORDINATE_MASK; ++ maxY = job_header[FRAG_JOB_DESC_MAX_TILE_COORD_WORD] & Y_COORDINATE_MASK; ++ restartX = job_header[JOB_DESC_FAULT_ADDR_LOW_WORD] & X_COORDINATE_MASK; ++ restartY = job_header[JOB_DESC_FAULT_ADDR_LOW_WORD] & Y_COORDINATE_MASK; ++ ++ dev_warn(dev, "Before Clamping:\n" ++ "Jobstatus: %08x\n" ++ "restartIdx: %08x\n" ++ "Fault_addr_low: %08x\n" ++ "minCoordsX: %08x minCoordsY: %08x\n" ++ "maxCoordsX: %08x maxCoordsY: %08x\n", ++ job_header[JOB_DESC_STATUS_WORD], ++ job_header[JOB_DESC_RESTART_INDEX_WORD], ++ job_header[JOB_DESC_FAULT_ADDR_LOW_WORD], ++ minX, minY, ++ maxX, maxY); ++ ++ /* Set the restart index to the one which generated the fault*/ ++ job_header[JOB_DESC_RESTART_INDEX_WORD] = ++ job_header[JOB_DESC_FAULT_ADDR_LOW_WORD]; ++ ++ if (restartX < minX) { ++ job_header[JOB_DESC_RESTART_INDEX_WORD] = (minX) | restartY; ++ dev_warn(dev, ++ "Clamping restart X index to minimum. %08x clamped to %08x\n", ++ restartX, minX); ++ clamped = 1; ++ } ++ if (restartY < minY) { ++ job_header[JOB_DESC_RESTART_INDEX_WORD] = (minY) | restartX; ++ dev_warn(dev, ++ "Clamping restart Y index to minimum. %08x clamped to %08x\n", ++ restartY, minY); ++ clamped = 1; ++ } ++ if (restartX > maxX) { ++ job_header[JOB_DESC_RESTART_INDEX_WORD] = (maxX) | restartY; ++ dev_warn(dev, ++ "Clamping restart X index to maximum. %08x clamped to %08x\n", ++ restartX, maxX); ++ clamped = 1; ++ } ++ if (restartY > maxY) { ++ job_header[JOB_DESC_RESTART_INDEX_WORD] = (maxY) | restartX; ++ dev_warn(dev, ++ "Clamping restart Y index to maximum. %08x clamped to %08x\n", ++ restartY, maxY); ++ clamped = 1; ++ } ++ ++ if (clamped) { ++ /* Reset the fault address low word ++ * and set the job status to STOPPED */ ++ job_header[JOB_DESC_FAULT_ADDR_LOW_WORD] = 0x0; ++ job_header[JOB_DESC_STATUS_WORD] = BASE_JD_EVENT_STOPPED; ++ dev_warn(dev, "After Clamping:\n" ++ "Jobstatus: %08x\n" ++ "restartIdx: %08x\n" ++ "Fault_addr_low: %08x\n" ++ "minCoordsX: %08x minCoordsY: %08x\n" ++ "maxCoordsX: %08x maxCoordsY: %08x\n", ++ job_header[JOB_DESC_STATUS_WORD], ++ job_header[JOB_DESC_RESTART_INDEX_WORD], ++ job_header[JOB_DESC_FAULT_ADDR_LOW_WORD], ++ minX, minY, ++ maxX, maxY); ++ ++ /* Flush CPU cache to update memory for future GPU reads*/ ++ memcpy(page_1, dst, copy_size); ++ p = phys_to_page(as_phys_addr_t(page_array[page_index])); ++ ++ kbase_sync_single_for_device(katom->kctx->kbdev, ++ kbase_dma_addr(p) + offset, ++ copy_size, DMA_TO_DEVICE); ++ ++ if (copy_size < JOB_HEADER_SIZE) { ++ memcpy(page_2, dst + copy_size, ++ JOB_HEADER_SIZE - copy_size); ++ p = phys_to_page(as_phys_addr_t(page_array[page_index + ++ 1])); ++ ++ kbase_sync_single_for_device(katom->kctx->kbdev, ++ kbase_dma_addr(p), ++ JOB_HEADER_SIZE - copy_size, ++ DMA_TO_DEVICE); ++ } ++ } ++ if (copy_size < JOB_HEADER_SIZE) ++ kunmap_atomic(page_2); ++ ++ kunmap_atomic(page_1); ++ ++out_unlock: ++ kbase_gpu_vm_unlock(katom->kctx); ++ return clamped; ++} +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_10969_workaround.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_10969_workaround.h +new file mode 100755 +index 000000000000..099a29861672 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_10969_workaround.h +@@ -0,0 +1,23 @@ ++/* ++ * ++ * (C) COPYRIGHT 2013-2014 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KBASE_10969_WORKAROUND_ ++#define _KBASE_10969_WORKAROUND_ ++ ++int kbasep_10969_workaround_clamp_coordinates(struct kbase_jd_atom *katom); ++ ++#endif /* _KBASE_10969_WORKAROUND_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_as_fault_debugfs.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_as_fault_debugfs.c +new file mode 100755 +index 000000000000..cc729d416858 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_as_fault_debugfs.c +@@ -0,0 +1,102 @@ ++/* ++ * ++ * (C) COPYRIGHT 2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include ++ ++#include ++#include ++ ++#ifdef CONFIG_DEBUG_FS ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++ ++static int kbase_as_fault_read(struct seq_file *sfile, void *data) ++{ ++ uintptr_t as_no = (uintptr_t) sfile->private; ++ ++ struct list_head *entry; ++ const struct list_head *kbdev_list; ++ struct kbase_device *kbdev = NULL; ++ ++ kbdev_list = kbase_dev_list_get(); ++ ++ list_for_each(entry, kbdev_list) { ++ kbdev = list_entry(entry, struct kbase_device, entry); ++ ++ if(kbdev->debugfs_as_read_bitmap & (1ULL << as_no)) { ++ ++ /* don't show this one again until another fault occors */ ++ kbdev->debugfs_as_read_bitmap &= ~(1ULL << as_no); ++ ++ /* output the last page fault addr */ ++ seq_printf(sfile, "%llu\n", (u64) kbdev->as[as_no].fault_addr); ++ } ++ ++ } ++ ++ kbase_dev_list_put(kbdev_list); ++ ++ return 0; ++} ++ ++static int kbase_as_fault_debugfs_open(struct inode *in, struct file *file) ++{ ++ return single_open(file, kbase_as_fault_read , in->i_private); ++} ++ ++static const struct file_operations as_fault_fops = { ++ .open = kbase_as_fault_debugfs_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = single_release, ++}; ++ ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++#endif /* CONFIG_DEBUG_FS */ ++ ++/* ++ * Initialize debugfs entry for each address space ++ */ ++void kbase_as_fault_debugfs_init(struct kbase_device *kbdev) ++{ ++#ifdef CONFIG_DEBUG_FS ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++ uint i; ++ char as_name[64]; ++ struct dentry *debugfs_directory; ++ ++ kbdev->debugfs_as_read_bitmap = 0ULL; ++ ++ KBASE_DEBUG_ASSERT(kbdev->nr_hw_address_spaces); ++ KBASE_DEBUG_ASSERT(sizeof(kbdev->as[0].fault_addr) == sizeof(u64)); ++ ++ debugfs_directory = debugfs_create_dir("address_spaces", ++ kbdev->mali_debugfs_directory); ++ ++ if(debugfs_directory) { ++ for(i = 0; i < kbdev->nr_hw_address_spaces; i++) { ++ snprintf(as_name, ARRAY_SIZE(as_name), "as%u", i); ++ debugfs_create_file(as_name, S_IRUGO, ++ debugfs_directory, (void*) ((uintptr_t) i), &as_fault_fops); ++ } ++ } ++ else ++ dev_warn(kbdev->dev, "unable to create address_spaces debugfs directory"); ++ ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++#endif /* CONFIG_DEBUG_FS */ ++ return; ++} +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_as_fault_debugfs.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_as_fault_debugfs.h +new file mode 100755 +index 000000000000..66387e1c3f6a +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_as_fault_debugfs.h +@@ -0,0 +1,45 @@ ++/* ++ * ++ * (C) COPYRIGHT 2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KBASE_AS_FAULT_DEBUG_FS_H ++#define _KBASE_AS_FAULT_DEBUG_FS_H ++ ++/** ++ * kbase_as_fault_debugfs_init() - Add debugfs files for reporting page faults ++ * ++ * @kbdev: Pointer to kbase_device ++ */ ++void kbase_as_fault_debugfs_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_as_fault_debugfs_new() - make the last fault available on debugfs ++ * ++ * @kbdev: Pointer to kbase_device ++ * @as_no: The address space the fault occurred on ++ */ ++static inline void ++kbase_as_fault_debugfs_new(struct kbase_device *kbdev, int as_no) ++{ ++#ifdef CONFIG_DEBUG_FS ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++ kbdev->debugfs_as_read_bitmap |= (1ULL << as_no); ++#endif /* CONFIG_DEBUG_FS */ ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++ return; ++} ++ ++#endif /*_KBASE_AS_FAULT_DEBUG_FS_H*/ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_cache_policy.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_cache_policy.c +new file mode 100755 +index 000000000000..1d11de67aa80 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_cache_policy.c +@@ -0,0 +1,54 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * Cache Policy API. ++ */ ++ ++#include "mali_kbase_cache_policy.h" ++ ++/* ++ * The output flags should be a combination of the following values: ++ * KBASE_REG_CPU_CACHED: CPU cache should be enabled. ++ */ ++u32 kbase_cache_enabled(u32 flags, u32 nr_pages) ++{ ++ u32 cache_flags = 0; ++ ++ CSTD_UNUSED(nr_pages); ++ ++ if (flags & BASE_MEM_CACHED_CPU) ++ cache_flags |= KBASE_REG_CPU_CACHED; ++ ++ return cache_flags; ++} ++ ++ ++void kbase_sync_single_for_device(struct kbase_device *kbdev, dma_addr_t handle, ++ size_t size, enum dma_data_direction dir) ++{ ++ dma_sync_single_for_device(kbdev->dev, handle, size, dir); ++} ++ ++ ++void kbase_sync_single_for_cpu(struct kbase_device *kbdev, dma_addr_t handle, ++ size_t size, enum dma_data_direction dir) ++{ ++ dma_sync_single_for_cpu(kbdev->dev, handle, size, dir); ++} +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_cache_policy.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_cache_policy.h +new file mode 100755 +index 000000000000..0c18bdb357b0 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_cache_policy.h +@@ -0,0 +1,45 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2013, 2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * Cache Policy API. ++ */ ++ ++#ifndef _KBASE_CACHE_POLICY_H_ ++#define _KBASE_CACHE_POLICY_H_ ++ ++#include "mali_kbase.h" ++#include "mali_base_kernel.h" ++ ++/** ++ * kbase_cache_enabled - Choose the cache policy for a specific region ++ * @flags: flags describing attributes of the region ++ * @nr_pages: total number of pages (backed or not) for the region ++ * ++ * Tells whether the CPU and GPU caches should be enabled or not for a specific ++ * region. ++ * This function can be modified to customize the cache policy depending on the ++ * flags and size of the region. ++ * ++ * Return: a combination of %KBASE_REG_CPU_CACHED and %KBASE_REG_GPU_CACHED ++ * depending on the cache policy ++ */ ++u32 kbase_cache_enabled(u32 flags, u32 nr_pages); ++ ++#endif /* _KBASE_CACHE_POLICY_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_config.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_config.c +new file mode 100755 +index 000000000000..fb615ae02ead +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_config.c +@@ -0,0 +1,51 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#include ++#include ++#include ++ ++int kbasep_platform_device_init(struct kbase_device *kbdev) ++{ ++ struct kbase_platform_funcs_conf *platform_funcs_p; ++ ++ platform_funcs_p = (struct kbase_platform_funcs_conf *)PLATFORM_FUNCS; ++ if (platform_funcs_p && platform_funcs_p->platform_init_func) ++ return platform_funcs_p->platform_init_func(kbdev); ++ ++ return 0; ++} ++ ++void kbasep_platform_device_term(struct kbase_device *kbdev) ++{ ++ struct kbase_platform_funcs_conf *platform_funcs_p; ++ ++ platform_funcs_p = (struct kbase_platform_funcs_conf *)PLATFORM_FUNCS; ++ if (platform_funcs_p && platform_funcs_p->platform_term_func) ++ platform_funcs_p->platform_term_func(kbdev); ++} ++ ++int kbase_cpuprops_get_default_clock_speed(u32 * const clock_speed) ++{ ++ KBASE_DEBUG_ASSERT(NULL != clock_speed); ++ ++ *clock_speed = 100; ++ return 0; ++} ++ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_config.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_config.h +new file mode 100755 +index 000000000000..212e3b14d96c +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_config.h +@@ -0,0 +1,343 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/** ++ * @file mali_kbase_config.h ++ * Configuration API and Attributes for KBase ++ */ ++ ++#ifndef _KBASE_CONFIG_H_ ++#define _KBASE_CONFIG_H_ ++ ++#include ++ ++#include ++#include ++ ++/** ++ * @addtogroup base_api ++ * @{ ++ */ ++ ++/** ++ * @addtogroup base_kbase_api ++ * @{ ++ */ ++ ++/** ++ * @addtogroup kbase_config Configuration API and Attributes ++ * @{ ++ */ ++ ++#include ++ ++/* Forward declaration of struct kbase_device */ ++struct kbase_device; ++ ++/** ++ * kbase_platform_funcs_conf - Specifies platform init/term function pointers ++ * ++ * Specifies the functions pointers for platform specific initialization and ++ * termination. By default no functions are required. No additional platform ++ * specific control is necessary. ++ */ ++struct kbase_platform_funcs_conf { ++ /** ++ * platform_init_func - platform specific init function pointer ++ * @kbdev - kbase_device pointer ++ * ++ * Returns 0 on success, negative error code otherwise. ++ * ++ * Function pointer for platform specific initialization or NULL if no ++ * initialization function is required. At the point this the GPU is ++ * not active and its power and clocks are in unknown (platform specific ++ * state) as kbase doesn't yet have control of power and clocks. ++ * ++ * The platform specific private pointer kbase_device::platform_context ++ * can be accessed (and possibly initialized) in here. ++ */ ++ int (*platform_init_func)(struct kbase_device *kbdev); ++ /** ++ * platform_term_func - platform specific termination function pointer ++ * @kbdev - kbase_device pointer ++ * ++ * Function pointer for platform specific termination or NULL if no ++ * termination function is required. At the point this the GPU will be ++ * idle but still powered and clocked. ++ * ++ * The platform specific private pointer kbase_device::platform_context ++ * can be accessed (and possibly terminated) in here. ++ */ ++ void (*platform_term_func)(struct kbase_device *kbdev); ++}; ++ ++/* ++ * @brief Specifies the callbacks for power management ++ * ++ * By default no callbacks will be made and the GPU must not be powered off. ++ */ ++struct kbase_pm_callback_conf { ++ /** Callback for when the GPU is idle and the power to it can be switched off. ++ * ++ * The system integrator can decide whether to either do nothing, just switch off ++ * the clocks to the GPU, or to completely power down the GPU. ++ * The platform specific private pointer kbase_device::platform_context can be accessed and modified in here. It is the ++ * platform \em callbacks responsibility to initialize and terminate this pointer if used (see @ref kbase_platform_funcs_conf). ++ */ ++ void (*power_off_callback)(struct kbase_device *kbdev); ++ ++ /** Callback for when the GPU is about to become active and power must be supplied. ++ * ++ * This function must not return until the GPU is powered and clocked sufficiently for register access to ++ * succeed. The return value specifies whether the GPU was powered down since the call to power_off_callback. ++ * If the GPU state has been lost then this function must return 1, otherwise it should return 0. ++ * The platform specific private pointer kbase_device::platform_context can be accessed and modified in here. It is the ++ * platform \em callbacks responsibility to initialize and terminate this pointer if used (see @ref kbase_platform_funcs_conf). ++ * ++ * The return value of the first call to this function is ignored. ++ * ++ * @return 1 if the GPU state may have been lost, 0 otherwise. ++ */ ++ int (*power_on_callback)(struct kbase_device *kbdev); ++ ++ /** Callback for when the system is requesting a suspend and GPU power ++ * must be switched off. ++ * ++ * Note that if this callback is present, then this may be called ++ * without a preceding call to power_off_callback. Therefore this ++ * callback must be able to take any action that might otherwise happen ++ * in power_off_callback. ++ * ++ * The platform specific private pointer kbase_device::platform_context ++ * can be accessed and modified in here. It is the platform \em ++ * callbacks responsibility to initialize and terminate this pointer if ++ * used (see @ref kbase_platform_funcs_conf). ++ */ ++ void (*power_suspend_callback)(struct kbase_device *kbdev); ++ ++ /** Callback for when the system is resuming from a suspend and GPU ++ * power must be switched on. ++ * ++ * Note that if this callback is present, then this may be called ++ * without a following call to power_on_callback. Therefore this ++ * callback must be able to take any action that might otherwise happen ++ * in power_on_callback. ++ * ++ * The platform specific private pointer kbase_device::platform_context ++ * can be accessed and modified in here. It is the platform \em ++ * callbacks responsibility to initialize and terminate this pointer if ++ * used (see @ref kbase_platform_funcs_conf). ++ */ ++ void (*power_resume_callback)(struct kbase_device *kbdev); ++ ++ /** Callback for handling runtime power management initialization. ++ * ++ * The runtime power management callbacks @ref power_runtime_off_callback and @ref power_runtime_on_callback ++ * will become active from calls made to the OS from within this function. ++ * The runtime calls can be triggered by calls from @ref power_off_callback and @ref power_on_callback. ++ * Note: for linux the kernel must have CONFIG_PM_RUNTIME enabled to use this feature. ++ * ++ * @return 0 on success, else int error code. ++ */ ++ int (*power_runtime_init_callback)(struct kbase_device *kbdev); ++ ++ /** Callback for handling runtime power management termination. ++ * ++ * The runtime power management callbacks @ref power_runtime_off_callback and @ref power_runtime_on_callback ++ * should no longer be called by the OS on completion of this function. ++ * Note: for linux the kernel must have CONFIG_PM_RUNTIME enabled to use this feature. ++ */ ++ void (*power_runtime_term_callback)(struct kbase_device *kbdev); ++ ++ /** Callback for runtime power-off power management callback ++ * ++ * For linux this callback will be called by the kernel runtime_suspend callback. ++ * Note: for linux the kernel must have CONFIG_PM_RUNTIME enabled to use this feature. ++ * ++ * @return 0 on success, else OS error code. ++ */ ++ void (*power_runtime_off_callback)(struct kbase_device *kbdev); ++ ++ /** Callback for runtime power-on power management callback ++ * ++ * For linux this callback will be called by the kernel runtime_resume callback. ++ * Note: for linux the kernel must have CONFIG_PM_RUNTIME enabled to use this feature. ++ */ ++ int (*power_runtime_on_callback)(struct kbase_device *kbdev); ++ ++ /* ++ * Optional callback for checking if GPU can be suspended when idle ++ * ++ * This callback will be called by the runtime power management core ++ * when the reference count goes to 0 to provide notification that the ++ * GPU now seems idle. ++ * ++ * If this callback finds that the GPU can't be powered off, or handles ++ * suspend by powering off directly or queueing up a power off, a ++ * non-zero value must be returned to prevent the runtime PM core from ++ * also triggering a suspend. ++ * ++ * Returning 0 will cause the runtime PM core to conduct a regular ++ * autosuspend. ++ * ++ * This callback is optional and if not provided regular autosuspend ++ * will be triggered. ++ * ++ * Note: The Linux kernel must have CONFIG_PM_RUNTIME enabled to use ++ * this feature. ++ * ++ * Return 0 if GPU can be suspended, positive value if it can not be ++ * suspeneded by runtime PM, else OS error code ++ */ ++ int (*power_runtime_idle_callback)(struct kbase_device *kbdev); ++}; ++ ++/** ++ * kbase_cpuprops_get_default_clock_speed - default for CPU_SPEED_FUNC ++ * @clock_speed - see kbase_cpu_clk_speed_func for details on the parameters ++ * ++ * Returns 0 on success, negative error code otherwise. ++ * ++ * Default implementation of CPU_SPEED_FUNC. This function sets clock_speed ++ * to 100, so will be an underestimate for any real system. ++ */ ++int kbase_cpuprops_get_default_clock_speed(u32 * const clock_speed); ++ ++/** ++ * kbase_cpu_clk_speed_func - Type of the function pointer for CPU_SPEED_FUNC ++ * @param clock_speed - pointer to store the current CPU clock speed in MHz ++ * ++ * Returns 0 on success, otherwise negative error code. ++ * ++ * This is mainly used to implement OpenCL's clGetDeviceInfo(). ++ */ ++typedef int (*kbase_cpu_clk_speed_func) (u32 *clock_speed); ++ ++/** ++ * kbase_gpu_clk_speed_func - Type of the function pointer for GPU_SPEED_FUNC ++ * @param clock_speed - pointer to store the current GPU clock speed in MHz ++ * ++ * Returns 0 on success, otherwise negative error code. ++ * When an error is returned the caller assumes maximum GPU speed stored in ++ * gpu_freq_khz_max. ++ * ++ * If the system timer is not available then this function is required ++ * for the OpenCL queue profiling to return correct timing information. ++ * ++ */ ++typedef int (*kbase_gpu_clk_speed_func) (u32 *clock_speed); ++ ++#ifdef CONFIG_OF ++struct kbase_platform_config { ++}; ++#else ++ ++/* ++ * @brief Specifies start and end of I/O memory region. ++ */ ++struct kbase_io_memory_region { ++ u64 start; ++ u64 end; ++}; ++ ++/* ++ * @brief Specifies I/O related resources like IRQs and memory region for I/O operations. ++ */ ++struct kbase_io_resources { ++ u32 job_irq_number; ++ u32 mmu_irq_number; ++ u32 gpu_irq_number; ++ struct kbase_io_memory_region io_memory_region; ++}; ++ ++struct kbase_platform_config { ++ const struct kbase_io_resources *io_resources; ++}; ++ ++#endif /* CONFIG_OF */ ++ ++/** ++ * @brief Gets the pointer to platform config. ++ * ++ * @return Pointer to the platform config ++ */ ++struct kbase_platform_config *kbase_get_platform_config(void); ++ ++/** ++ * kbasep_platform_device_init: - Platform specific call to initialize hardware ++ * @kbdev: kbase device pointer ++ * ++ * Function calls a platform defined routine if specified in the configuration ++ * attributes. The routine can initialize any hardware and context state that ++ * is required for the GPU block to function. ++ * ++ * Return: 0 if no errors have been found in the config. ++ * Negative error code otherwise. ++ */ ++int kbasep_platform_device_init(struct kbase_device *kbdev); ++ ++/** ++ * kbasep_platform_device_term - Platform specific call to terminate hardware ++ * @kbdev: Kbase device pointer ++ * ++ * Function calls a platform defined routine if specified in the configuration ++ * attributes. The routine can destroy any platform specific context state and ++ * shut down any hardware functionality that are outside of the Power Management ++ * callbacks. ++ * ++ */ ++void kbasep_platform_device_term(struct kbase_device *kbdev); ++ ++ ++/** ++ * kbase_platform_early_init - Early initialisation of the platform code ++ * ++ * This function will be called when the module is loaded to perform any ++ * early initialisation required by the platform code. Such as reading ++ * platform specific device tree entries for the GPU. ++ * ++ * Return: 0 for success, any other fail causes module initialisation to fail ++ */ ++int kbase_platform_early_init(void); ++ ++#ifndef CONFIG_OF ++/** ++ * kbase_platform_register - Register a platform device for the GPU ++ * ++ * This can be used to register a platform device on systems where device tree ++ * is not enabled and the platform initialisation code in the kernel doesn't ++ * create the GPU device. Where possible device tree should be used instead. ++ * ++ * Return: 0 for success, any other fail causes module initialisation to fail ++ */ ++int kbase_platform_register(void); ++ ++/** ++ * kbase_platform_unregister - Unregister a fake platform device ++ * ++ * Unregister the platform device created with kbase_platform_register() ++ */ ++void kbase_platform_unregister(void); ++#endif ++ ++ /** @} *//* end group kbase_config */ ++ /** @} *//* end group base_kbase_api */ ++ /** @} *//* end group base_api */ ++ ++#endif /* _KBASE_CONFIG_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_config_defaults.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_config_defaults.h +new file mode 100755 +index 000000000000..69079e7d9680 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_config_defaults.h +@@ -0,0 +1,226 @@ ++/* ++ * ++ * (C) COPYRIGHT 2013-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/** ++ * @file mali_kbase_config_defaults.h ++ * ++ * Default values for configuration settings ++ * ++ */ ++ ++#ifndef _KBASE_CONFIG_DEFAULTS_H_ ++#define _KBASE_CONFIG_DEFAULTS_H_ ++ ++/* Include mandatory definitions per platform */ ++#include ++ ++/** ++* Boolean indicating whether the driver is configured to be secure at ++* a potential loss of performance. ++* ++* This currently affects only r0p0-15dev0 HW and earlier. ++* ++* On r0p0-15dev0 HW and earlier, there are tradeoffs between security and ++* performance: ++* ++* - When this is set to true, the driver remains fully secure, ++* but potentially loses performance compared with setting this to ++* false. ++* - When set to false, the driver is open to certain security ++* attacks. ++* ++* From r0p0-00rel0 and onwards, there is no security loss by setting ++* this to false, and no performance loss by setting it to ++* true. ++*/ ++#define DEFAULT_SECURE_BUT_LOSS_OF_PERFORMANCE false ++ ++enum { ++ /** ++ * Use unrestricted Address ID width on the AXI bus. ++ */ ++ KBASE_AID_32 = 0x0, ++ ++ /** ++ * Restrict GPU to a half of maximum Address ID count. ++ * This will reduce performance, but reduce bus load due to GPU. ++ */ ++ KBASE_AID_16 = 0x3, ++ ++ /** ++ * Restrict GPU to a quarter of maximum Address ID count. ++ * This will reduce performance, but reduce bus load due to GPU. ++ */ ++ KBASE_AID_8 = 0x2, ++ ++ /** ++ * Restrict GPU to an eighth of maximum Address ID count. ++ * This will reduce performance, but reduce bus load due to GPU. ++ */ ++ KBASE_AID_4 = 0x1 ++}; ++ ++/** ++ * Default setting for read Address ID limiting on AXI bus. ++ * ++ * Attached value: u32 register value ++ * KBASE_AID_32 - use the full 32 IDs (5 ID bits) ++ * KBASE_AID_16 - use 16 IDs (4 ID bits) ++ * KBASE_AID_8 - use 8 IDs (3 ID bits) ++ * KBASE_AID_4 - use 4 IDs (2 ID bits) ++ * Default value: KBASE_AID_32 (no limit). Note hardware implementation ++ * may limit to a lower value. ++ */ ++#define DEFAULT_ARID_LIMIT KBASE_AID_32 ++ ++/** ++ * Default setting for write Address ID limiting on AXI. ++ * ++ * Attached value: u32 register value ++ * KBASE_AID_32 - use the full 32 IDs (5 ID bits) ++ * KBASE_AID_16 - use 16 IDs (4 ID bits) ++ * KBASE_AID_8 - use 8 IDs (3 ID bits) ++ * KBASE_AID_4 - use 4 IDs (2 ID bits) ++ * Default value: KBASE_AID_32 (no limit). Note hardware implementation ++ * may limit to a lower value. ++ */ ++#define DEFAULT_AWID_LIMIT KBASE_AID_32 ++ ++/** ++ * Default UMP device mapping. A UMP_DEVICE__SHIFT value which ++ * defines which UMP device this GPU should be mapped to. ++ */ ++#define DEFAULT_UMP_GPU_DEVICE_SHIFT UMP_DEVICE_Z_SHIFT ++ ++/* ++ * Default period for DVFS sampling ++ */ ++#define DEFAULT_PM_DVFS_PERIOD 100 /* 100ms */ ++ ++/* ++ * Power Management poweroff tick granuality. This is in nanoseconds to ++ * allow HR timer support. ++ * ++ * On each scheduling tick, the power manager core may decide to: ++ * -# Power off one or more shader cores ++ * -# Power off the entire GPU ++ */ ++#define DEFAULT_PM_GPU_POWEROFF_TICK_NS (400000) /* 400us */ ++ ++/* ++ * Power Manager number of ticks before shader cores are powered off ++ */ ++#define DEFAULT_PM_POWEROFF_TICK_SHADER (2) /* 400-800us */ ++ ++/* ++ * Power Manager number of ticks before GPU is powered off ++ */ ++#define DEFAULT_PM_POWEROFF_TICK_GPU (2) /* 400-800us */ ++ ++/* ++ * Default scheduling tick granuality ++ */ ++#define DEFAULT_JS_SCHEDULING_PERIOD_NS (100000000u) /* 100ms */ ++ ++/* ++ * Default minimum number of scheduling ticks before jobs are soft-stopped. ++ * ++ * This defines the time-slice for a job (which may be different from that of a ++ * context) ++ */ ++#define DEFAULT_JS_SOFT_STOP_TICKS (1) /* 100ms-200ms */ ++ ++/* ++ * Default minimum number of scheduling ticks before CL jobs are soft-stopped. ++ */ ++#define DEFAULT_JS_SOFT_STOP_TICKS_CL (1) /* 100ms-200ms */ ++ ++/* ++ * Default minimum number of scheduling ticks before jobs are hard-stopped ++ */ ++#define DEFAULT_JS_HARD_STOP_TICKS_SS (50) /* 5s */ ++#define DEFAULT_JS_HARD_STOP_TICKS_SS_8408 (300) /* 30s */ ++ ++/* ++ * Default minimum number of scheduling ticks before CL jobs are hard-stopped. ++ */ ++#define DEFAULT_JS_HARD_STOP_TICKS_CL (50) /* 5s */ ++ ++/* ++ * Default minimum number of scheduling ticks before jobs are hard-stopped ++ * during dumping ++ */ ++#define DEFAULT_JS_HARD_STOP_TICKS_DUMPING (15000) /* 1500s */ ++ ++/* ++ * Default timeout for some software jobs, after which the software event wait ++ * jobs will be cancelled. ++ */ ++#define DEFAULT_JS_SOFT_JOB_TIMEOUT (3000) /* 3s */ ++ ++/* ++ * Default minimum number of scheduling ticks before the GPU is reset to clear a ++ * "stuck" job ++ */ ++#define DEFAULT_JS_RESET_TICKS_SS (55) /* 5.5s */ ++#define DEFAULT_JS_RESET_TICKS_SS_8408 (450) /* 45s */ ++ ++/* ++ * Default minimum number of scheduling ticks before the GPU is reset to clear a ++ * "stuck" CL job. ++ */ ++#define DEFAULT_JS_RESET_TICKS_CL (55) /* 5.5s */ ++ ++/* ++ * Default minimum number of scheduling ticks before the GPU is reset to clear a ++ * "stuck" job during dumping. ++ */ ++#define DEFAULT_JS_RESET_TICKS_DUMPING (15020) /* 1502s */ ++ ++/* ++ * Default number of milliseconds given for other jobs on the GPU to be ++ * soft-stopped when the GPU needs to be reset. ++ */ ++#define DEFAULT_RESET_TIMEOUT_MS (3000) /* 3s */ ++ ++/* ++ * Default timeslice that a context is scheduled in for, in nanoseconds. ++ * ++ * When a context has used up this amount of time across its jobs, it is ++ * scheduled out to let another run. ++ * ++ * @note the resolution is nanoseconds (ns) here, because that's the format ++ * often used by the OS. ++ */ ++#define DEFAULT_JS_CTX_TIMESLICE_NS (50000000) /* 50ms */ ++ ++/* ++ * Perform GPU power down using only platform specific code, skipping DDK power ++ * management. ++ * ++ * If this is non-zero then kbase will avoid powering down shader cores, the ++ * tiler, and the L2 cache, instead just powering down the entire GPU through ++ * platform specific code. This may be required for certain platform ++ * integrations. ++ * ++ * Note that as this prevents kbase from powering down shader cores, this limits ++ * the available power policies to coarse_demand and always_on. ++ */ ++#define PLATFORM_POWER_DOWN_ONLY (0) ++ ++#endif /* _KBASE_CONFIG_DEFAULTS_H_ */ ++ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_context.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_context.c +new file mode 100755 +index 000000000000..7a09aa26128e +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_context.c +@@ -0,0 +1,362 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * Base kernel context APIs ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++ ++/** ++ * kbase_create_context() - Create a kernel base context. ++ * @kbdev: Kbase device ++ * @is_compat: Force creation of a 32-bit context ++ * ++ * Allocate and init a kernel base context. ++ * ++ * Return: new kbase context ++ */ ++struct kbase_context * ++kbase_create_context(struct kbase_device *kbdev, bool is_compat) ++{ ++ struct kbase_context *kctx; ++ int err; ++ struct page *p; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ /* zero-inited as lot of code assume it's zero'ed out on create */ ++ kctx = vzalloc(sizeof(*kctx)); ++ ++ if (!kctx) ++ goto out; ++ ++ /* creating a context is considered a disjoint event */ ++ kbase_disjoint_event(kbdev); ++ ++ kctx->kbdev = kbdev; ++ kctx->as_nr = KBASEP_AS_NR_INVALID; ++ atomic_set(&kctx->refcount, 0); ++ if (is_compat) ++ kbase_ctx_flag_set(kctx, KCTX_COMPAT); ++#ifdef CONFIG_MALI_BIFROST_TRACE_TIMELINE ++ kctx->timeline.owner_tgid = task_tgid_nr(current); ++#endif ++ atomic_set(&kctx->setup_complete, 0); ++ atomic_set(&kctx->setup_in_progress, 0); ++ spin_lock_init(&kctx->mm_update_lock); ++ kctx->process_mm = NULL; ++ atomic_set(&kctx->nonmapped_pages, 0); ++ kctx->slots_pullable = 0; ++ kctx->tgid = current->tgid; ++ kctx->pid = current->pid; ++ ++ err = kbase_mem_pool_init(&kctx->mem_pool, ++ kbdev->mem_pool_max_size_default, ++ KBASE_MEM_POOL_4KB_PAGE_TABLE_ORDER, ++ kctx->kbdev, ++ &kbdev->mem_pool); ++ if (err) ++ goto free_kctx; ++ ++ err = kbase_mem_pool_init(&kctx->lp_mem_pool, ++ (kbdev->mem_pool_max_size_default >> 9), ++ KBASE_MEM_POOL_2MB_PAGE_TABLE_ORDER, ++ kctx->kbdev, ++ &kbdev->lp_mem_pool); ++ if (err) ++ goto free_mem_pool; ++ ++ err = kbase_mem_evictable_init(kctx); ++ if (err) ++ goto free_both_pools; ++ ++ atomic_set(&kctx->used_pages, 0); ++ ++ err = kbase_jd_init(kctx); ++ if (err) ++ goto deinit_evictable; ++ ++ err = kbasep_js_kctx_init(kctx); ++ if (err) ++ goto free_jd; /* safe to call kbasep_js_kctx_term in this case */ ++ ++ err = kbase_event_init(kctx); ++ if (err) ++ goto free_jd; ++ ++ atomic_set(&kctx->drain_pending, 0); ++ ++ mutex_init(&kctx->reg_lock); ++ ++ mutex_init(&kctx->mem_partials_lock); ++ INIT_LIST_HEAD(&kctx->mem_partials); ++ ++ INIT_LIST_HEAD(&kctx->waiting_soft_jobs); ++ spin_lock_init(&kctx->waiting_soft_jobs_lock); ++#ifdef CONFIG_KDS ++ INIT_LIST_HEAD(&kctx->waiting_kds_resource); ++#endif ++ err = kbase_dma_fence_init(kctx); ++ if (err) ++ goto free_event; ++ ++ err = kbase_mmu_init(kctx); ++ if (err) ++ goto term_dma_fence; ++ ++ do { ++ err = kbase_mem_pool_grow(&kctx->mem_pool, ++ MIDGARD_MMU_BOTTOMLEVEL); ++ if (err) ++ goto pgd_no_mem; ++ ++ mutex_lock(&kctx->mmu_lock); ++ kctx->pgd = kbase_mmu_alloc_pgd(kctx); ++ mutex_unlock(&kctx->mmu_lock); ++ } while (!kctx->pgd); ++ ++ p = kbase_mem_alloc_page(&kctx->mem_pool); ++ if (!p) ++ goto no_sink_page; ++ kctx->aliasing_sink_page = as_tagged(page_to_phys(p)); ++ ++ init_waitqueue_head(&kctx->event_queue); ++ ++ kctx->cookies = KBASE_COOKIE_MASK; ++ ++ /* Make sure page 0 is not used... */ ++ err = kbase_region_tracker_init(kctx); ++ if (err) ++ goto no_region_tracker; ++ ++ err = kbase_sticky_resource_init(kctx); ++ if (err) ++ goto no_sticky; ++ ++ err = kbase_jit_init(kctx); ++ if (err) ++ goto no_jit; ++#ifdef CONFIG_GPU_TRACEPOINTS ++ atomic_set(&kctx->jctx.work_id, 0); ++#endif ++#ifdef CONFIG_MALI_BIFROST_TRACE_TIMELINE ++ atomic_set(&kctx->timeline.jd_atoms_in_flight, 0); ++#endif ++ ++ kctx->id = atomic_add_return(1, &(kbdev->ctx_num)) - 1; ++ ++ mutex_init(&kctx->vinstr_cli_lock); ++ ++ timer_setup(&kctx->soft_job_timeout, ++ kbasep_soft_job_timeout_worker, ++ 0); ++ ++ return kctx; ++ ++no_jit: ++ kbase_gpu_vm_lock(kctx); ++ kbase_sticky_resource_term(kctx); ++ kbase_gpu_vm_unlock(kctx); ++no_sticky: ++ kbase_region_tracker_term(kctx); ++no_region_tracker: ++ kbase_mem_pool_free(&kctx->mem_pool, p, false); ++no_sink_page: ++ /* VM lock needed for the call to kbase_mmu_free_pgd */ ++ kbase_gpu_vm_lock(kctx); ++ kbase_mmu_free_pgd(kctx); ++ kbase_gpu_vm_unlock(kctx); ++pgd_no_mem: ++ kbase_mmu_term(kctx); ++term_dma_fence: ++ kbase_dma_fence_term(kctx); ++free_event: ++ kbase_event_cleanup(kctx); ++free_jd: ++ /* Safe to call this one even when didn't initialize (assuming kctx was sufficiently zeroed) */ ++ kbasep_js_kctx_term(kctx); ++ kbase_jd_exit(kctx); ++deinit_evictable: ++ kbase_mem_evictable_deinit(kctx); ++free_both_pools: ++ kbase_mem_pool_term(&kctx->lp_mem_pool); ++free_mem_pool: ++ kbase_mem_pool_term(&kctx->mem_pool); ++free_kctx: ++ vfree(kctx); ++out: ++ return NULL; ++} ++KBASE_EXPORT_SYMBOL(kbase_create_context); ++ ++static void kbase_reg_pending_dtor(struct kbase_va_region *reg) ++{ ++ dev_dbg(reg->kctx->kbdev->dev, "Freeing pending unmapped region\n"); ++ kbase_mem_phy_alloc_put(reg->cpu_alloc); ++ kbase_mem_phy_alloc_put(reg->gpu_alloc); ++ kfree(reg); ++} ++ ++/** ++ * kbase_destroy_context - Destroy a kernel base context. ++ * @kctx: Context to destroy ++ * ++ * Calls kbase_destroy_os_context() to free OS specific structures. ++ * Will release all outstanding regions. ++ */ ++void kbase_destroy_context(struct kbase_context *kctx) ++{ ++ struct kbase_device *kbdev; ++ int pages; ++ unsigned long pending_regions_to_clean; ++ unsigned long flags; ++ struct page *p; ++ ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ ++ kbdev = kctx->kbdev; ++ KBASE_DEBUG_ASSERT(NULL != kbdev); ++ ++ KBASE_TRACE_ADD(kbdev, CORE_CTX_DESTROY, kctx, NULL, 0u, 0u); ++ ++ /* Ensure the core is powered up for the destroy process */ ++ /* A suspend won't happen here, because we're in a syscall from a userspace ++ * thread. */ ++ kbase_pm_context_active(kbdev); ++ ++ kbase_jd_zap_context(kctx); ++ ++#ifdef CONFIG_DEBUG_FS ++ /* Removing the rest of the debugfs entries here as we want to keep the ++ * atom debugfs interface alive until all atoms have completed. This ++ * is useful for debugging hung contexts. */ ++ debugfs_remove_recursive(kctx->kctx_dentry); ++#endif ++ ++ kbase_event_cleanup(kctx); ++ ++ /* ++ * JIT must be terminated before the code below as it must be called ++ * without the region lock being held. ++ * The code above ensures no new JIT allocations can be made by ++ * by the time we get to this point of context tear down. ++ */ ++ kbase_jit_term(kctx); ++ ++ kbase_gpu_vm_lock(kctx); ++ ++ kbase_sticky_resource_term(kctx); ++ ++ /* MMU is disabled as part of scheduling out the context */ ++ kbase_mmu_free_pgd(kctx); ++ ++ /* drop the aliasing sink page now that it can't be mapped anymore */ ++ p = phys_to_page(as_phys_addr_t(kctx->aliasing_sink_page)); ++ kbase_mem_pool_free(&kctx->mem_pool, p, false); ++ ++ /* free pending region setups */ ++ pending_regions_to_clean = (~kctx->cookies) & KBASE_COOKIE_MASK; ++ while (pending_regions_to_clean) { ++ unsigned int cookie = __ffs(pending_regions_to_clean); ++ ++ BUG_ON(!kctx->pending_regions[cookie]); ++ ++ kbase_reg_pending_dtor(kctx->pending_regions[cookie]); ++ ++ kctx->pending_regions[cookie] = NULL; ++ pending_regions_to_clean &= ~(1UL << cookie); ++ } ++ ++ kbase_region_tracker_term(kctx); ++ kbase_gpu_vm_unlock(kctx); ++ ++ /* Safe to call this one even when didn't initialize (assuming kctx was sufficiently zeroed) */ ++ kbasep_js_kctx_term(kctx); ++ ++ kbase_jd_exit(kctx); ++ ++ kbase_pm_context_idle(kbdev); ++ ++ kbase_dma_fence_term(kctx); ++ ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, flags); ++ kbase_ctx_sched_remove_ctx(kctx); ++ spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, flags); ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ ++ kbase_mmu_term(kctx); ++ ++ pages = atomic_read(&kctx->used_pages); ++ if (pages != 0) ++ dev_warn(kbdev->dev, "%s: %d pages in use!\n", __func__, pages); ++ ++ kbase_mem_evictable_deinit(kctx); ++ kbase_mem_pool_term(&kctx->mem_pool); ++ kbase_mem_pool_term(&kctx->lp_mem_pool); ++ WARN_ON(atomic_read(&kctx->nonmapped_pages) != 0); ++ ++ vfree(kctx); ++} ++KBASE_EXPORT_SYMBOL(kbase_destroy_context); ++ ++/** ++ * kbase_context_set_create_flags - Set creation flags on a context ++ * @kctx: Kbase context ++ * @flags: Flags to set ++ * ++ * Return: 0 on success ++ */ ++int kbase_context_set_create_flags(struct kbase_context *kctx, u32 flags) ++{ ++ int err = 0; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ unsigned long irq_flags; ++ ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ /* Validate flags */ ++ if (flags != (flags & BASE_CONTEXT_CREATE_KERNEL_FLAGS)) { ++ err = -EINVAL; ++ goto out; ++ } ++ ++ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); ++ spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, irq_flags); ++ ++ /* Translate the flags */ ++ if ((flags & BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED) == 0) ++ kbase_ctx_flag_clear(kctx, KCTX_SUBMIT_DISABLED); ++ ++ /* Latch the initial attributes into the Job Scheduler */ ++ kbasep_js_ctx_attr_set_initial_attrs(kctx->kbdev, kctx); ++ ++ spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, irq_flags); ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ out: ++ return err; ++} ++KBASE_EXPORT_SYMBOL(kbase_context_set_create_flags); +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_context.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_context.h +new file mode 100755 +index 000000000000..a3f5bb0ce0da +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_context.h +@@ -0,0 +1,90 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KBASE_CONTEXT_H_ ++#define _KBASE_CONTEXT_H_ ++ ++#include ++ ++ ++int kbase_context_set_create_flags(struct kbase_context *kctx, u32 flags); ++ ++/** ++ * kbase_ctx_flag - Check if @flag is set on @kctx ++ * @kctx: Pointer to kbase context to check ++ * @flag: Flag to check ++ * ++ * Return: true if @flag is set on @kctx, false if not. ++ */ ++static inline bool kbase_ctx_flag(struct kbase_context *kctx, ++ enum kbase_context_flags flag) ++{ ++ return atomic_read(&kctx->flags) & flag; ++} ++ ++/** ++ * kbase_ctx_flag_clear - Clear @flag on @kctx ++ * @kctx: Pointer to kbase context ++ * @flag: Flag to clear ++ * ++ * Clear the @flag on @kctx. This is done atomically, so other flags being ++ * cleared or set at the same time will be safe. ++ * ++ * Some flags have locking requirements, check the documentation for the ++ * respective flags. ++ */ ++static inline void kbase_ctx_flag_clear(struct kbase_context *kctx, ++ enum kbase_context_flags flag) ++{ ++#if KERNEL_VERSION(4, 3, 0) > LINUX_VERSION_CODE ++ /* ++ * Earlier kernel versions doesn't have atomic_andnot() or ++ * atomic_and(). atomic_clear_mask() was only available on some ++ * architectures and removed on arm in v3.13 on arm and arm64. ++ * ++ * Use a compare-exchange loop to clear the flag on pre 4.3 kernels, ++ * when atomic_andnot() becomes available. ++ */ ++ int old, new; ++ ++ do { ++ old = atomic_read(&kctx->flags); ++ new = old & ~flag; ++ ++ } while (atomic_cmpxchg(&kctx->flags, old, new) != old); ++#else ++ atomic_andnot(flag, &kctx->flags); ++#endif ++} ++ ++/** ++ * kbase_ctx_flag_set - Set @flag on @kctx ++ * @kctx: Pointer to kbase context ++ * @flag: Flag to clear ++ * ++ * Set the @flag on @kctx. This is done atomically, so other flags being ++ * cleared or set at the same time will be safe. ++ * ++ * Some flags have locking requirements, check the documentation for the ++ * respective flags. ++ */ ++static inline void kbase_ctx_flag_set(struct kbase_context *kctx, ++ enum kbase_context_flags flag) ++{ ++ atomic_or(flag, &kctx->flags); ++} ++#endif /* _KBASE_CONTEXT_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_core_linux.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_core_linux.c +new file mode 100755 +index 000000000000..347fee2643bb +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_core_linux.c +@@ -0,0 +1,4971 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#ifdef CONFIG_MALI_BIFROST_DEVFREQ ++#include ++#include ++#ifdef CONFIG_DEVFREQ_THERMAL ++#include ++#endif /* CONFIG_DEVFREQ_THERMAL */ ++#endif /* CONFIG_MALI_BIFROST_DEVFREQ */ ++#ifdef CONFIG_MALI_BIFROST_NO_MALI ++#include "mali_kbase_model_linux.h" ++#endif /* CONFIG_MALI_BIFROST_NO_MALI */ ++#include "mali_kbase_mem_profile_debugfs_buf_size.h" ++#include "mali_kbase_debug_mem_view.h" ++#include "mali_kbase_mem.h" ++#include "mali_kbase_mem_pool_debugfs.h" ++#if !MALI_CUSTOMER_RELEASE ++#include "mali_kbase_regs_dump_debugfs.h" ++#endif /* !MALI_CUSTOMER_RELEASE */ ++#include "mali_kbase_regs_history_debugfs.h" ++#include ++#include ++#include ++#include ++#include "mali_kbase_ioctl.h" ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include /* is_compat_task */ ++#include ++#include ++#include ++#include ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++#include ++#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ ++#include ++#include ++ ++#include ++ ++ ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0)) ++#include ++#else ++#include ++#endif ++ ++#include ++ ++#include ++ ++/* GPU IRQ Tags */ ++#define JOB_IRQ_TAG 0 ++#define MMU_IRQ_TAG 1 ++#define GPU_IRQ_TAG 2 ++ ++#if MALI_UNIT_TEST ++static struct kbase_exported_test_data shared_kernel_test_data; ++EXPORT_SYMBOL(shared_kernel_test_data); ++#endif /* MALI_UNIT_TEST */ ++ ++static int kbase_dev_nr; ++ ++static DEFINE_MUTEX(kbase_dev_list_lock); ++static LIST_HEAD(kbase_dev_list); ++ ++#define KERNEL_SIDE_DDK_VERSION_STRING "K:" MALI_RELEASE_NAME "(GPL)" ++static inline void __compile_time_asserts(void) ++{ ++ CSTD_COMPILE_TIME_ASSERT(sizeof(KERNEL_SIDE_DDK_VERSION_STRING) <= KBASE_GET_VERSION_BUFFER_SIZE); ++} ++ ++static int kbase_api_handshake(struct kbase_context *kctx, ++ struct kbase_ioctl_version_check *version) ++{ ++ switch (version->major) { ++ case BASE_UK_VERSION_MAJOR: ++ /* set minor to be the lowest common */ ++ version->minor = min_t(int, BASE_UK_VERSION_MINOR, ++ (int)version->minor); ++ break; ++ default: ++ /* We return our actual version regardless if it ++ * matches the version returned by userspace - ++ * userspace can bail if it can't handle this ++ * version */ ++ version->major = BASE_UK_VERSION_MAJOR; ++ version->minor = BASE_UK_VERSION_MINOR; ++ break; ++ } ++ ++ /* save the proposed version number for later use */ ++ kctx->api_version = KBASE_API_VERSION(version->major, version->minor); ++ ++ return 0; ++} ++ ++/** ++ * enum mali_error - Mali error codes shared with userspace ++ * ++ * This is subset of those common Mali errors that can be returned to userspace. ++ * Values of matching user and kernel space enumerators MUST be the same. ++ * MALI_ERROR_NONE is guaranteed to be 0. ++ * ++ * @MALI_ERROR_NONE: Success ++ * @MALI_ERROR_OUT_OF_GPU_MEMORY: Not used in the kernel driver ++ * @MALI_ERROR_OUT_OF_MEMORY: Memory allocation failure ++ * @MALI_ERROR_FUNCTION_FAILED: Generic error code ++ */ ++enum mali_error { ++ MALI_ERROR_NONE = 0, ++ MALI_ERROR_OUT_OF_GPU_MEMORY, ++ MALI_ERROR_OUT_OF_MEMORY, ++ MALI_ERROR_FUNCTION_FAILED, ++}; ++ ++enum { ++ inited_mem = (1u << 0), ++ inited_js = (1u << 1), ++ inited_pm_runtime_init = (1u << 2), ++#ifdef CONFIG_MALI_BIFROST_DEVFREQ ++ inited_devfreq = (1u << 3), ++#endif /* CONFIG_MALI_BIFROST_DEVFREQ */ ++ inited_tlstream = (1u << 4), ++ inited_backend_early = (1u << 5), ++ inited_backend_late = (1u << 6), ++ inited_device = (1u << 7), ++ inited_vinstr = (1u << 8), ++ ++ inited_job_fault = (1u << 10), ++ inited_sysfs_group = (1u << 11), ++ inited_misc_register = (1u << 12), ++ inited_get_device = (1u << 13), ++ inited_dev_list = (1u << 14), ++ inited_debugfs = (1u << 15), ++ inited_gpu_device = (1u << 16), ++ inited_registers_map = (1u << 17), ++ inited_io_history = (1u << 18), ++ inited_power_control = (1u << 19), ++ inited_buslogger = (1u << 20), ++ inited_protected = (1u << 21), ++ inited_ctx_sched = (1u << 22) ++}; ++ ++ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++#define INACTIVE_WAIT_MS (5000) ++ ++void kbase_set_driver_inactive(struct kbase_device *kbdev, bool inactive) ++{ ++ kbdev->driver_inactive = inactive; ++ wake_up(&kbdev->driver_inactive_wait); ++ ++ /* Wait for any running IOCTLs to complete */ ++ if (inactive) ++ msleep(INACTIVE_WAIT_MS); ++} ++KBASE_EXPORT_TEST_API(kbase_set_driver_inactive); ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++ ++/** ++ * kbase_legacy_dispatch - UKK dispatch function ++ * ++ * This is the dispatch function for the legacy UKK ioctl interface. No new ++ * ioctls should be added to this function, see kbase_ioctl instead. ++ * ++ * @kctx: The kernel context structure ++ * @args: Pointer to the data structure passed from/to user space ++ * @args_size: Size of the data structure ++ */ ++static int kbase_legacy_dispatch(struct kbase_context *kctx, ++ void * const args, u32 args_size) ++{ ++ struct kbase_device *kbdev; ++ union uk_header *ukh = args; ++ u32 id; ++ int ret = 0; ++ ++ KBASE_DEBUG_ASSERT(ukh != NULL); ++ ++ kbdev = kctx->kbdev; ++ id = ukh->id; ++ ukh->ret = MALI_ERROR_NONE; /* Be optimistic */ ++ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++ wait_event(kbdev->driver_inactive_wait, ++ kbdev->driver_inactive == false); ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++ ++ if (UKP_FUNC_ID_CHECK_VERSION == id) { ++ struct uku_version_check_args *version_check; ++ struct kbase_ioctl_version_check version; ++ ++ if (args_size != sizeof(struct uku_version_check_args)) { ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ return 0; ++ } ++ version_check = (struct uku_version_check_args *)args; ++ version.minor = version_check->minor; ++ version.major = version_check->major; ++ ++ kbase_api_handshake(kctx, &version); ++ ++ version_check->minor = version.minor; ++ version_check->major = version.major; ++ ukh->ret = MALI_ERROR_NONE; ++ return 0; ++ } ++ ++ /* block calls until version handshake */ ++ if (kctx->api_version == 0) ++ return -EINVAL; ++ ++ if (!atomic_read(&kctx->setup_complete)) { ++ struct kbase_uk_set_flags *kbase_set_flags; ++ ++ /* setup pending, try to signal that we'll do the setup, ++ * if setup was already in progress, err this call ++ */ ++ if (atomic_cmpxchg(&kctx->setup_in_progress, 0, 1) != 0) ++ return -EINVAL; ++ ++ /* if unexpected call, will stay stuck in setup mode ++ * (is it the only call we accept?) ++ */ ++ if (id != KBASE_FUNC_SET_FLAGS) ++ return -EINVAL; ++ ++ kbase_set_flags = (struct kbase_uk_set_flags *)args; ++ ++ /* if not matching the expected call, stay in setup mode */ ++ if (sizeof(*kbase_set_flags) != args_size) ++ goto bad_size; ++ ++ /* if bad flags, will stay stuck in setup mode */ ++ if (kbase_context_set_create_flags(kctx, ++ kbase_set_flags->create_flags) != 0) ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ ++ atomic_set(&kctx->setup_complete, 1); ++ return 0; ++ } ++ ++ /* setup complete, perform normal operation */ ++ switch (id) { ++ case KBASE_FUNC_MEM_JIT_INIT: ++ { ++ struct kbase_uk_mem_jit_init *jit_init = args; ++ ++ if (sizeof(*jit_init) != args_size) ++ goto bad_size; ++ ++ if (kbase_region_tracker_init_jit(kctx, ++ jit_init->va_pages)) ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ break; ++ } ++ case KBASE_FUNC_MEM_ALLOC: ++ { ++ struct kbase_uk_mem_alloc *mem = args; ++ struct kbase_va_region *reg; ++ ++ if (sizeof(*mem) != args_size) ++ goto bad_size; ++ ++#if defined(CONFIG_64BIT) ++ if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { ++ /* force SAME_VA if a 64-bit client */ ++ mem->flags |= BASE_MEM_SAME_VA; ++ } ++#endif ++ ++ reg = kbase_mem_alloc(kctx, mem->va_pages, ++ mem->commit_pages, mem->extent, ++ &mem->flags, &mem->gpu_va); ++ mem->va_alignment = 0; ++ ++ if (!reg) ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ break; ++ } ++ case KBASE_FUNC_MEM_IMPORT: { ++ struct kbase_uk_mem_import *mem_import = args; ++ void __user *phandle; ++ ++ if (sizeof(*mem_import) != args_size) ++ goto bad_size; ++#ifdef CONFIG_COMPAT ++ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) ++ phandle = compat_ptr(mem_import->phandle); ++ else ++#endif ++ phandle = u64_to_user_ptr(mem_import->phandle); ++ ++ if (mem_import->type == BASE_MEM_IMPORT_TYPE_INVALID) { ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ break; ++ } ++ ++ if (kbase_mem_import(kctx, ++ (enum base_mem_import_type) ++ mem_import->type, ++ phandle, ++ 0, ++ &mem_import->gpu_va, ++ &mem_import->va_pages, ++ &mem_import->flags)) { ++ mem_import->type = BASE_MEM_IMPORT_TYPE_INVALID; ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ } ++ break; ++ } ++ case KBASE_FUNC_MEM_ALIAS: { ++ struct kbase_uk_mem_alias *alias = args; ++ struct base_mem_aliasing_info __user *user_ai; ++ struct base_mem_aliasing_info *ai; ++ ++ if (sizeof(*alias) != args_size) ++ goto bad_size; ++ ++ if (alias->nents > 2048) { ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ break; ++ } ++ if (!alias->nents) { ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ break; ++ } ++ ++#ifdef CONFIG_COMPAT ++ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) ++ user_ai = compat_ptr(alias->ai); ++ else ++#endif ++ user_ai = u64_to_user_ptr(alias->ai); ++ ++ ai = vmalloc(sizeof(*ai) * alias->nents); ++ ++ if (!ai) { ++ ukh->ret = MALI_ERROR_OUT_OF_MEMORY; ++ break; ++ } ++ ++ if (copy_from_user(ai, user_ai, ++ sizeof(*ai) * alias->nents)) { ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ goto copy_failed; ++ } ++ ++ alias->gpu_va = kbase_mem_alias(kctx, &alias->flags, ++ alias->stride, ++ alias->nents, ai, ++ &alias->va_pages); ++ if (!alias->gpu_va) { ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ goto no_alias; ++ } ++no_alias: ++copy_failed: ++ vfree(ai); ++ break; ++ } ++ case KBASE_FUNC_MEM_COMMIT: ++ { ++ struct kbase_uk_mem_commit *commit = args; ++ int ret; ++ ++ if (sizeof(*commit) != args_size) ++ goto bad_size; ++ ++ ret = kbase_mem_commit(kctx, commit->gpu_addr, ++ commit->pages); ++ ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ commit->result_subcode = ++ BASE_BACKING_THRESHOLD_ERROR_INVALID_ARGUMENTS; ++ ++ if (ret == 0) { ++ ukh->ret = MALI_ERROR_NONE; ++ commit->result_subcode = ++ BASE_BACKING_THRESHOLD_OK; ++ } else if (ret == -ENOMEM) { ++ commit->result_subcode = ++ BASE_BACKING_THRESHOLD_ERROR_OOM; ++ } ++ ++ break; ++ } ++ ++ case KBASE_FUNC_MEM_QUERY: ++ { ++ struct kbase_uk_mem_query *query = args; ++ ++ if (sizeof(*query) != args_size) ++ goto bad_size; ++ ++ if (kbase_mem_query(kctx, query->gpu_addr, ++ query->query, &query->value) != 0) ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ else ++ ukh->ret = MALI_ERROR_NONE; ++ break; ++ } ++ break; ++ ++ case KBASE_FUNC_MEM_FLAGS_CHANGE: ++ { ++ struct kbase_uk_mem_flags_change *fc = args; ++ ++ if (sizeof(*fc) != args_size) ++ goto bad_size; ++ ++ if (kbase_mem_flags_change(kctx, fc->gpu_va, ++ fc->flags, fc->mask) != 0) ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ ++ break; ++ } ++ case KBASE_FUNC_MEM_FREE: ++ { ++ struct kbase_uk_mem_free *mem = args; ++ ++ if (sizeof(*mem) != args_size) ++ goto bad_size; ++ ++ if (kbase_mem_free(kctx, mem->gpu_addr) != 0) ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ break; ++ } ++ ++ case KBASE_FUNC_JOB_SUBMIT: ++ { ++ struct kbase_uk_job_submit *job = args; ++ char __user *user_buf; ++ ++ if (sizeof(*job) != args_size) ++ goto bad_size; ++ ++#ifdef CONFIG_COMPAT ++ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) ++ user_buf = compat_ptr(job->addr); ++ else ++#endif ++ user_buf = u64_to_user_ptr(job->addr); ++ ++ if (kbase_jd_submit(kctx, user_buf, ++ job->nr_atoms, ++ job->stride, ++ false) != 0) ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ break; ++ } ++ ++ case KBASE_FUNC_SYNC: ++ { ++ struct kbase_uk_sync_now *sn = args; ++ ++ if (sizeof(*sn) != args_size) ++ goto bad_size; ++ ++ if (kbase_sync_now(kctx, &sn->sset.basep_sset) != 0) ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ break; ++ } ++ ++ case KBASE_FUNC_DISJOINT_QUERY: ++ { ++ struct kbase_uk_disjoint_query *dquery = args; ++ ++ if (sizeof(*dquery) != args_size) ++ goto bad_size; ++ ++ /* Get the disjointness counter value. */ ++ dquery->counter = kbase_disjoint_event_get(kctx->kbdev); ++ break; ++ } ++ ++ case KBASE_FUNC_POST_TERM: ++ { ++ kbase_event_close(kctx); ++ break; ++ } ++ ++ case KBASE_FUNC_HWCNT_SETUP: ++ { ++ struct kbase_uk_hwcnt_setup *setup = args; ++ ++ if (sizeof(*setup) != args_size) ++ goto bad_size; ++ ++ mutex_lock(&kctx->vinstr_cli_lock); ++ if (kbase_vinstr_legacy_hwc_setup(kbdev->vinstr_ctx, ++ &kctx->vinstr_cli, setup) != 0) ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ mutex_unlock(&kctx->vinstr_cli_lock); ++ break; ++ } ++ ++ case KBASE_FUNC_HWCNT_DUMP: ++ { ++ /* args ignored */ ++ mutex_lock(&kctx->vinstr_cli_lock); ++ if (kbase_vinstr_hwc_dump(kctx->vinstr_cli, ++ BASE_HWCNT_READER_EVENT_MANUAL) != 0) ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ mutex_unlock(&kctx->vinstr_cli_lock); ++ break; ++ } ++ ++ case KBASE_FUNC_HWCNT_CLEAR: ++ { ++ /* args ignored */ ++ mutex_lock(&kctx->vinstr_cli_lock); ++ if (kbase_vinstr_hwc_clear(kctx->vinstr_cli) != 0) ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ mutex_unlock(&kctx->vinstr_cli_lock); ++ break; ++ } ++ ++ case KBASE_FUNC_HWCNT_READER_SETUP: ++ { ++ struct kbase_uk_hwcnt_reader_setup *setup = args; ++ ++ if (sizeof(*setup) != args_size) ++ goto bad_size; ++ ++ mutex_lock(&kctx->vinstr_cli_lock); ++ if (kbase_vinstr_hwcnt_reader_setup(kbdev->vinstr_ctx, ++ setup) != 0) ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ mutex_unlock(&kctx->vinstr_cli_lock); ++ break; ++ } ++ ++ case KBASE_FUNC_GPU_PROPS_REG_DUMP: ++ { ++ struct kbase_uk_gpuprops *setup = args; ++ ++ if (sizeof(*setup) != args_size) ++ goto bad_size; ++ ++ if (kbase_gpuprops_uk_get_props(kctx, setup) != 0) ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ break; ++ } ++ case KBASE_FUNC_FIND_CPU_OFFSET: ++ { ++ struct kbase_uk_find_cpu_offset *find = args; ++ ++ if (sizeof(*find) != args_size) ++ goto bad_size; ++ ++ if (find->gpu_addr & ~PAGE_MASK) { ++ dev_warn(kbdev->dev, "kbase_legacy_dispatch case KBASE_FUNC_FIND_CPU_OFFSET: find->gpu_addr: passed parameter is invalid"); ++ goto out_bad; ++ } ++ ++ if (find->size > SIZE_MAX || find->cpu_addr > ULONG_MAX) { ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ } else { ++ int err; ++ ++ err = kbasep_find_enclosing_cpu_mapping_offset( ++ kctx, ++ find->cpu_addr, ++ find->size, ++ &find->offset); ++ ++ if (err) ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ } ++ break; ++ } ++ case KBASE_FUNC_GET_VERSION: ++ { ++ struct kbase_uk_get_ddk_version *get_version = (struct kbase_uk_get_ddk_version *)args; ++ ++ if (sizeof(*get_version) != args_size) ++ goto bad_size; ++ ++ /* version buffer size check is made in compile time assert */ ++ memcpy(get_version->version_buffer, KERNEL_SIDE_DDK_VERSION_STRING, sizeof(KERNEL_SIDE_DDK_VERSION_STRING)); ++ get_version->version_string_size = sizeof(KERNEL_SIDE_DDK_VERSION_STRING); ++ break; ++ } ++ ++ case KBASE_FUNC_STREAM_CREATE: ++ { ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ struct kbase_uk_stream_create *screate = (struct kbase_uk_stream_create *)args; ++ ++ if (sizeof(*screate) != args_size) ++ goto bad_size; ++ ++ if (strnlen(screate->name, sizeof(screate->name)) >= sizeof(screate->name)) { ++ /* not NULL terminated */ ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ break; ++ } ++ ++ if (kbase_sync_fence_stream_create(screate->name, ++ &screate->fd) != 0) ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ else ++ ukh->ret = MALI_ERROR_NONE; ++#else /* CONFIG_SYNC || CONFIG_SYNC_FILE */ ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ ++ break; ++ } ++ case KBASE_FUNC_FENCE_VALIDATE: ++ { ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ struct kbase_uk_fence_validate *fence_validate = (struct kbase_uk_fence_validate *)args; ++ ++ if (sizeof(*fence_validate) != args_size) ++ goto bad_size; ++ ++ if (kbase_sync_fence_validate(fence_validate->fd) != 0) ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ else ++ ukh->ret = MALI_ERROR_NONE; ++#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ ++ break; ++ } ++ ++ case KBASE_FUNC_SET_TEST_DATA: ++ { ++#if MALI_UNIT_TEST ++ struct kbase_uk_set_test_data *set_data = args; ++ ++ shared_kernel_test_data = set_data->test_data; ++ shared_kernel_test_data.kctx = (uintptr_t)kctx; ++ shared_kernel_test_data.mm = (uintptr_t)current->mm; ++ ukh->ret = MALI_ERROR_NONE; ++#endif /* MALI_UNIT_TEST */ ++ break; ++ } ++ ++ case KBASE_FUNC_INJECT_ERROR: ++ { ++#ifdef CONFIG_MALI_BIFROST_ERROR_INJECT ++ unsigned long flags; ++ struct kbase_error_params params = ((struct kbase_uk_error_params *)args)->params; ++ ++ /*mutex lock */ ++ spin_lock_irqsave(&kbdev->reg_op_lock, flags); ++ if (job_atom_inject_error(¶ms) != 0) ++ ukh->ret = MALI_ERROR_OUT_OF_MEMORY; ++ else ++ ukh->ret = MALI_ERROR_NONE; ++ spin_unlock_irqrestore(&kbdev->reg_op_lock, flags); ++ /*mutex unlock */ ++#endif /* CONFIG_MALI_BIFROST_ERROR_INJECT */ ++ break; ++ } ++ ++ case KBASE_FUNC_MODEL_CONTROL: ++ { ++#ifdef CONFIG_MALI_BIFROST_NO_MALI ++ unsigned long flags; ++ struct kbase_model_control_params params = ++ ((struct kbase_uk_model_control_params *)args)->params; ++ ++ /*mutex lock */ ++ spin_lock_irqsave(&kbdev->reg_op_lock, flags); ++ if (gpu_model_control(kbdev->model, ¶ms) != 0) ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ else ++ ukh->ret = MALI_ERROR_NONE; ++ spin_unlock_irqrestore(&kbdev->reg_op_lock, flags); ++ /*mutex unlock */ ++#endif /* CONFIG_MALI_BIFROST_NO_MALI */ ++ break; ++ } ++ ++ case KBASE_FUNC_GET_PROFILING_CONTROLS: ++ { ++ struct kbase_uk_profiling_controls *controls = ++ (struct kbase_uk_profiling_controls *)args; ++ u32 i; ++ ++ if (sizeof(*controls) != args_size) ++ goto bad_size; ++ ++ for (i = FBDUMP_CONTROL_MIN; i < FBDUMP_CONTROL_MAX; i++) ++ controls->profiling_controls[i] = ++ kbdev->kbase_profiling_controls[i]; ++ ++ break; ++ } ++ ++ /* used only for testing purposes; these controls are to be set by gator through gator API */ ++ case KBASE_FUNC_SET_PROFILING_CONTROLS: ++ { ++ struct kbase_uk_profiling_controls *controls = ++ (struct kbase_uk_profiling_controls *)args; ++ u32 i; ++ ++ if (sizeof(*controls) != args_size) ++ goto bad_size; ++ ++ for (i = FBDUMP_CONTROL_MIN; i < FBDUMP_CONTROL_MAX; i++) ++ _mali_profiling_control(i, controls->profiling_controls[i]); ++ ++ break; ++ } ++ ++ case KBASE_FUNC_DEBUGFS_MEM_PROFILE_ADD: ++ { ++ struct kbase_uk_debugfs_mem_profile_add *add_data = ++ (struct kbase_uk_debugfs_mem_profile_add *)args; ++ char *buf; ++ char __user *user_buf; ++ ++ if (sizeof(*add_data) != args_size) ++ goto bad_size; ++ ++ if (add_data->len > KBASE_MEM_PROFILE_MAX_BUF_SIZE) { ++ dev_err(kbdev->dev, "buffer too big\n"); ++ goto out_bad; ++ } ++ ++#ifdef CONFIG_COMPAT ++ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) ++ user_buf = compat_ptr(add_data->buf); ++ else ++#endif ++ user_buf = u64_to_user_ptr(add_data->buf); ++ ++ buf = kmalloc(add_data->len, GFP_KERNEL); ++ if (ZERO_OR_NULL_PTR(buf)) ++ goto out_bad; ++ ++ if (0 != copy_from_user(buf, user_buf, add_data->len)) { ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ kfree(buf); ++ goto out_bad; ++ } ++ ++ if (kbasep_mem_profile_debugfs_insert(kctx, buf, ++ add_data->len)) { ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ goto out_bad; ++ } ++ ++ break; ++ } ++ ++#ifdef CONFIG_MALI_BIFROST_NO_MALI ++ case KBASE_FUNC_SET_PRFCNT_VALUES: ++ { ++ ++ struct kbase_uk_prfcnt_values *params = ++ ((struct kbase_uk_prfcnt_values *)args); ++ gpu_model_set_dummy_prfcnt_sample(params->data, ++ params->size); ++ ++ break; ++ } ++#endif /* CONFIG_MALI_BIFROST_NO_MALI */ ++#ifdef BASE_LEGACY_UK10_4_SUPPORT ++ case KBASE_FUNC_TLSTREAM_ACQUIRE_V10_4: ++ { ++ struct kbase_uk_tlstream_acquire_v10_4 *tlstream_acquire ++ = args; ++ int ret; ++ ++ if (sizeof(*tlstream_acquire) != args_size) ++ goto bad_size; ++ ++ ret = kbase_tlstream_acquire( ++ kctx, 0); ++ if (ret < 0) ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ else ++ tlstream_acquire->fd = ret; ++ break; ++ } ++#endif /* BASE_LEGACY_UK10_4_SUPPORT */ ++ case KBASE_FUNC_TLSTREAM_ACQUIRE: ++ { ++ struct kbase_uk_tlstream_acquire *tlstream_acquire = ++ args; ++ int ret; ++ ++ if (sizeof(*tlstream_acquire) != args_size) ++ goto bad_size; ++ ++ if (tlstream_acquire->flags & ~BASE_TLSTREAM_FLAGS_MASK) ++ goto out_bad; ++ ++ ret = kbase_tlstream_acquire( ++ kctx, tlstream_acquire->flags); ++ if (ret < 0) ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ else ++ tlstream_acquire->fd = ret; ++ break; ++ } ++ case KBASE_FUNC_TLSTREAM_FLUSH: ++ { ++ struct kbase_uk_tlstream_flush *tlstream_flush = ++ args; ++ ++ if (sizeof(*tlstream_flush) != args_size) ++ goto bad_size; ++ ++ kbase_tlstream_flush_streams(); ++ break; ++ } ++#if MALI_UNIT_TEST ++ case KBASE_FUNC_TLSTREAM_TEST: ++ { ++ struct kbase_uk_tlstream_test *tlstream_test = args; ++ ++ if (sizeof(*tlstream_test) != args_size) ++ goto bad_size; ++ ++ kbase_tlstream_test( ++ tlstream_test->tpw_count, ++ tlstream_test->msg_delay, ++ tlstream_test->msg_count, ++ tlstream_test->aux_msg); ++ break; ++ } ++ case KBASE_FUNC_TLSTREAM_STATS: ++ { ++ struct kbase_uk_tlstream_stats *tlstream_stats = args; ++ ++ if (sizeof(*tlstream_stats) != args_size) ++ goto bad_size; ++ ++ kbase_tlstream_stats( ++ &tlstream_stats->bytes_collected, ++ &tlstream_stats->bytes_generated); ++ break; ++ } ++#endif /* MALI_UNIT_TEST */ ++ ++ case KBASE_FUNC_GET_CONTEXT_ID: ++ { ++ struct kbase_uk_context_id *info = args; ++ ++ info->id = kctx->id; ++ break; ++ } ++ ++ case KBASE_FUNC_SOFT_EVENT_UPDATE: ++ { ++ struct kbase_uk_soft_event_update *update = args; ++ ++ if (sizeof(*update) != args_size) ++ goto bad_size; ++ ++ if (((update->new_status != BASE_JD_SOFT_EVENT_SET) && ++ (update->new_status != BASE_JD_SOFT_EVENT_RESET)) || ++ (update->flags != 0)) ++ goto out_bad; ++ ++ if (kbase_soft_event_update(kctx, update->evt, ++ update->new_status)) ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ ++ break; ++ } ++ ++ default: ++ dev_err(kbdev->dev, "unknown ioctl %u\n", id); ++ goto out_bad; ++ } ++ ++ return ret; ++ ++ bad_size: ++ dev_err(kbdev->dev, "Wrong syscall size (%d) for %08x\n", args_size, id); ++ out_bad: ++ return -EINVAL; ++} ++ ++static struct kbase_device *to_kbase_device(struct device *dev) ++{ ++ return dev_get_drvdata(dev); ++} ++ ++static int assign_irqs(struct platform_device *pdev) ++{ ++ struct kbase_device *kbdev = to_kbase_device(&pdev->dev); ++ int i; ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ /* 3 IRQ resources */ ++ for (i = 0; i < 3; i++) { ++ struct resource *irq_res; ++ int irqtag; ++ ++ irq_res = platform_get_resource(pdev, IORESOURCE_IRQ, i); ++ if (!irq_res) { ++ dev_err(kbdev->dev, "No IRQ resource at index %d\n", i); ++ return -ENOENT; ++ } ++ ++#ifdef CONFIG_OF ++ if (!strncmp(irq_res->name, "JOB", 4)) { ++ irqtag = JOB_IRQ_TAG; ++ } else if (!strncmp(irq_res->name, "MMU", 4)) { ++ irqtag = MMU_IRQ_TAG; ++ } else if (!strncmp(irq_res->name, "GPU", 4)) { ++ irqtag = GPU_IRQ_TAG; ++ } else { ++ dev_err(&pdev->dev, "Invalid irq res name: '%s'\n", ++ irq_res->name); ++ return -EINVAL; ++ } ++#else ++ irqtag = i; ++#endif /* CONFIG_OF */ ++ kbdev->irqs[irqtag].irq = irq_res->start; ++ kbdev->irqs[irqtag].flags = irq_res->flags & IRQF_TRIGGER_MASK; ++ } ++ ++ return 0; ++} ++ ++/* ++ * API to acquire device list mutex and ++ * return pointer to the device list head ++ */ ++const struct list_head *kbase_dev_list_get(void) ++{ ++ mutex_lock(&kbase_dev_list_lock); ++ return &kbase_dev_list; ++} ++KBASE_EXPORT_TEST_API(kbase_dev_list_get); ++ ++/* API to release the device list mutex */ ++void kbase_dev_list_put(const struct list_head *dev_list) ++{ ++ mutex_unlock(&kbase_dev_list_lock); ++} ++KBASE_EXPORT_TEST_API(kbase_dev_list_put); ++ ++/* Find a particular kbase device (as specified by minor number), or find the "first" device if -1 is specified */ ++struct kbase_device *kbase_find_device(int minor) ++{ ++ struct kbase_device *kbdev = NULL; ++ struct list_head *entry; ++ const struct list_head *dev_list = kbase_dev_list_get(); ++ ++ list_for_each(entry, dev_list) { ++ struct kbase_device *tmp; ++ ++ tmp = list_entry(entry, struct kbase_device, entry); ++ if (tmp->mdev.minor == minor || minor == -1) { ++ kbdev = tmp; ++ get_device(kbdev->dev); ++ break; ++ } ++ } ++ kbase_dev_list_put(dev_list); ++ ++ return kbdev; ++} ++EXPORT_SYMBOL(kbase_find_device); ++ ++void kbase_release_device(struct kbase_device *kbdev) ++{ ++ put_device(kbdev->dev); ++} ++EXPORT_SYMBOL(kbase_release_device); ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) && \ ++ !(LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 28) && \ ++ LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0)) ++/* ++ * Older versions, before v4.6, of the kernel doesn't have ++ * kstrtobool_from_user(), except longterm 4.4.y which had it added in 4.4.28 ++ */ ++static int kstrtobool_from_user(const char __user *s, size_t count, bool *res) ++{ ++ char buf[32]; ++ ++ count = min(sizeof(buf), count); ++ ++ if (copy_from_user(buf, s, count)) ++ return -EFAULT; ++ buf[count] = '\0'; ++ ++ return strtobool(buf, res); ++} ++#endif ++ ++static ssize_t write_ctx_infinite_cache(struct file *f, const char __user *ubuf, size_t size, loff_t *off) ++{ ++ struct kbase_context *kctx = f->private_data; ++ int err; ++ bool value; ++ ++ err = kstrtobool_from_user(ubuf, size, &value); ++ if (err) ++ return err; ++ ++ if (value) ++ kbase_ctx_flag_set(kctx, KCTX_INFINITE_CACHE); ++ else ++ kbase_ctx_flag_clear(kctx, KCTX_INFINITE_CACHE); ++ ++ return size; ++} ++ ++static ssize_t read_ctx_infinite_cache(struct file *f, char __user *ubuf, size_t size, loff_t *off) ++{ ++ struct kbase_context *kctx = f->private_data; ++ char buf[32]; ++ int count; ++ bool value; ++ ++ value = kbase_ctx_flag(kctx, KCTX_INFINITE_CACHE); ++ ++ count = scnprintf(buf, sizeof(buf), "%s\n", value ? "Y" : "N"); ++ ++ return simple_read_from_buffer(ubuf, size, off, buf, count); ++} ++ ++static const struct file_operations kbase_infinite_cache_fops = { ++ .open = simple_open, ++ .write = write_ctx_infinite_cache, ++ .read = read_ctx_infinite_cache, ++}; ++ ++static int kbase_open(struct inode *inode, struct file *filp) ++{ ++ struct kbase_device *kbdev = NULL; ++ struct kbase_context *kctx; ++ int ret = 0; ++#ifdef CONFIG_DEBUG_FS ++ char kctx_name[64]; ++#endif ++ ++ kbdev = kbase_find_device(iminor(inode)); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ kctx = kbase_create_context(kbdev, is_compat_task()); ++ if (!kctx) { ++ ret = -ENOMEM; ++ goto out; ++ } ++ ++ init_waitqueue_head(&kctx->event_queue); ++ filp->private_data = kctx; ++ kctx->filp = filp; ++ ++ if (kbdev->infinite_cache_active_default) ++ kbase_ctx_flag_set(kctx, KCTX_INFINITE_CACHE); ++ ++#ifdef CONFIG_DEBUG_FS ++ snprintf(kctx_name, 64, "%d_%d", kctx->tgid, kctx->id); ++ ++ kctx->kctx_dentry = debugfs_create_dir(kctx_name, ++ kbdev->debugfs_ctx_directory); ++ ++ if (IS_ERR_OR_NULL(kctx->kctx_dentry)) { ++ ret = -ENOMEM; ++ goto out; ++ } ++ ++ debugfs_create_file("infinite_cache", 0644, kctx->kctx_dentry, ++ kctx, &kbase_infinite_cache_fops); ++ ++ mutex_init(&kctx->mem_profile_lock); ++ ++ kbasep_jd_debugfs_ctx_init(kctx); ++ kbase_debug_mem_view_init(filp); ++ ++ kbase_debug_job_fault_context_init(kctx); ++ ++ kbase_mem_pool_debugfs_init(kctx->kctx_dentry, &kctx->mem_pool, &kctx->lp_mem_pool); ++ ++ kbase_jit_debugfs_init(kctx); ++#endif /* CONFIG_DEBUG_FS */ ++ ++ dev_dbg(kbdev->dev, "created base context\n"); ++ ++ { ++ struct kbasep_kctx_list_element *element; ++ ++ element = kzalloc(sizeof(*element), GFP_KERNEL); ++ if (element) { ++ mutex_lock(&kbdev->kctx_list_lock); ++ element->kctx = kctx; ++ list_add(&element->link, &kbdev->kctx_list); ++ KBASE_TLSTREAM_TL_NEW_CTX( ++ element->kctx, ++ element->kctx->id, ++ (u32)(element->kctx->tgid)); ++ mutex_unlock(&kbdev->kctx_list_lock); ++ } else { ++ /* we don't treat this as a fail - just warn about it */ ++ dev_warn(kbdev->dev, "couldn't add kctx to kctx_list\n"); ++ } ++ } ++ return 0; ++ ++ out: ++ kbase_release_device(kbdev); ++ return ret; ++} ++ ++static int kbase_release(struct inode *inode, struct file *filp) ++{ ++ struct kbase_context *kctx = filp->private_data; ++ struct kbase_device *kbdev = kctx->kbdev; ++ struct kbasep_kctx_list_element *element, *tmp; ++ bool found_element = false; ++ ++ KBASE_TLSTREAM_TL_DEL_CTX(kctx); ++ ++#ifdef CONFIG_DEBUG_FS ++ kbasep_mem_profile_debugfs_remove(kctx); ++ kbase_debug_job_fault_context_term(kctx); ++#endif ++ ++ mutex_lock(&kbdev->kctx_list_lock); ++ list_for_each_entry_safe(element, tmp, &kbdev->kctx_list, link) { ++ if (element->kctx == kctx) { ++ list_del(&element->link); ++ kfree(element); ++ found_element = true; ++ } ++ } ++ mutex_unlock(&kbdev->kctx_list_lock); ++ if (!found_element) ++ dev_warn(kbdev->dev, "kctx not in kctx_list\n"); ++ ++ filp->private_data = NULL; ++ ++ mutex_lock(&kctx->vinstr_cli_lock); ++ /* If this client was performing hwcnt dumping and did not explicitly ++ * detach itself, remove it from the vinstr core now */ ++ if (kctx->vinstr_cli) { ++ struct kbase_uk_hwcnt_setup setup; ++ ++ setup.dump_buffer = 0llu; ++ kbase_vinstr_legacy_hwc_setup( ++ kbdev->vinstr_ctx, &kctx->vinstr_cli, &setup); ++ } ++ mutex_unlock(&kctx->vinstr_cli_lock); ++ ++ kbase_destroy_context(kctx); ++ ++ dev_dbg(kbdev->dev, "deleted base context\n"); ++ kbase_release_device(kbdev); ++ return 0; ++} ++ ++#define CALL_MAX_SIZE 536 ++ ++static long kbase_legacy_ioctl(struct file *filp, unsigned int cmd, ++ unsigned long arg) ++{ ++ u64 msg[(CALL_MAX_SIZE + 7) >> 3] = { 0xdeadbeefdeadbeefull }; /* alignment fixup */ ++ u32 size = _IOC_SIZE(cmd); ++ struct kbase_context *kctx = filp->private_data; ++ ++ if (size > CALL_MAX_SIZE) ++ return -ENOTTY; ++ ++ if (0 != copy_from_user(&msg, (void __user *)arg, size)) { ++ dev_err(kctx->kbdev->dev, "failed to copy ioctl argument into kernel space\n"); ++ return -EFAULT; ++ } ++ ++ if (kbase_legacy_dispatch(kctx, &msg, size) != 0) ++ return -EFAULT; ++ ++ if (0 != copy_to_user((void __user *)arg, &msg, size)) { ++ dev_err(kctx->kbdev->dev, "failed to copy results of UK call back to user space\n"); ++ return -EFAULT; ++ } ++ return 0; ++} ++ ++static int kbase_api_set_flags(struct kbase_context *kctx, ++ struct kbase_ioctl_set_flags *flags) ++{ ++ int err; ++ ++ /* setup pending, try to signal that we'll do the setup, ++ * if setup was already in progress, err this call ++ */ ++ if (atomic_cmpxchg(&kctx->setup_in_progress, 0, 1) != 0) ++ return -EINVAL; ++ ++ err = kbase_context_set_create_flags(kctx, flags->create_flags); ++ /* if bad flags, will stay stuck in setup mode */ ++ if (err) ++ return err; ++ ++ atomic_set(&kctx->setup_complete, 1); ++ return 0; ++} ++ ++static int kbase_api_job_submit(struct kbase_context *kctx, ++ struct kbase_ioctl_job_submit *submit) ++{ ++ return kbase_jd_submit(kctx, u64_to_user_ptr(submit->addr), ++ submit->nr_atoms, ++ submit->stride, false); ++} ++ ++static int kbase_api_get_gpuprops(struct kbase_context *kctx, ++ struct kbase_ioctl_get_gpuprops *get_props) ++{ ++ struct kbase_gpu_props *kprops = &kctx->kbdev->gpu_props; ++ int err; ++ ++ if (get_props->flags != 0) { ++ dev_err(kctx->kbdev->dev, "Unsupported flags to get_gpuprops"); ++ return -EINVAL; ++ } ++ ++ if (get_props->size == 0) ++ return kprops->prop_buffer_size; ++ if (get_props->size < kprops->prop_buffer_size) ++ return -EINVAL; ++ ++ err = copy_to_user(u64_to_user_ptr(get_props->buffer), ++ kprops->prop_buffer, ++ kprops->prop_buffer_size); ++ if (err) ++ return -EFAULT; ++ return kprops->prop_buffer_size; ++} ++ ++static int kbase_api_post_term(struct kbase_context *kctx) ++{ ++ kbase_event_close(kctx); ++ return 0; ++} ++ ++static int kbase_api_mem_alloc(struct kbase_context *kctx, ++ union kbase_ioctl_mem_alloc *alloc) ++{ ++ struct kbase_va_region *reg; ++ u64 flags = alloc->in.flags; ++ u64 gpu_va; ++ ++#if defined(CONFIG_64BIT) ++ if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { ++ /* force SAME_VA if a 64-bit client */ ++ flags |= BASE_MEM_SAME_VA; ++ } ++#endif ++ ++ reg = kbase_mem_alloc(kctx, alloc->in.va_pages, ++ alloc->in.commit_pages, ++ alloc->in.extent, ++ &flags, &gpu_va); ++ ++ if (!reg) ++ return -ENOMEM; ++ ++ alloc->out.flags = flags; ++ alloc->out.gpu_va = gpu_va; ++ ++ return 0; ++} ++ ++static int kbase_api_mem_query(struct kbase_context *kctx, ++ union kbase_ioctl_mem_query *query) ++{ ++ return kbase_mem_query(kctx, query->in.gpu_addr, ++ query->in.query, &query->out.value); ++} ++ ++static int kbase_api_mem_free(struct kbase_context *kctx, ++ struct kbase_ioctl_mem_free *free) ++{ ++ return kbase_mem_free(kctx, free->gpu_addr); ++} ++ ++static int kbase_api_hwcnt_reader_setup(struct kbase_context *kctx, ++ struct kbase_ioctl_hwcnt_reader_setup *setup) ++{ ++ int ret; ++ struct kbase_uk_hwcnt_reader_setup args = { ++ .buffer_count = setup->buffer_count, ++ .jm_bm = setup->jm_bm, ++ .shader_bm = setup->shader_bm, ++ .tiler_bm = setup->tiler_bm, ++ .mmu_l2_bm = setup->mmu_l2_bm ++ }; ++ ++ mutex_lock(&kctx->vinstr_cli_lock); ++ ret = kbase_vinstr_hwcnt_reader_setup(kctx->kbdev->vinstr_ctx, &args); ++ mutex_unlock(&kctx->vinstr_cli_lock); ++ ++ if (ret) ++ return ret; ++ return args.fd; ++} ++ ++static int kbase_api_hwcnt_enable(struct kbase_context *kctx, ++ struct kbase_ioctl_hwcnt_enable *enable) ++{ ++ int ret; ++ struct kbase_uk_hwcnt_setup args = { ++ .dump_buffer = enable->dump_buffer, ++ .jm_bm = enable->jm_bm, ++ .shader_bm = enable->shader_bm, ++ .tiler_bm = enable->tiler_bm, ++ .mmu_l2_bm = enable->mmu_l2_bm ++ }; ++ ++ mutex_lock(&kctx->vinstr_cli_lock); ++ ret = kbase_vinstr_legacy_hwc_setup(kctx->kbdev->vinstr_ctx, ++ &kctx->vinstr_cli, &args); ++ mutex_unlock(&kctx->vinstr_cli_lock); ++ ++ return ret; ++} ++ ++static int kbase_api_hwcnt_dump(struct kbase_context *kctx) ++{ ++ int ret; ++ ++ mutex_lock(&kctx->vinstr_cli_lock); ++ ret = kbase_vinstr_hwc_dump(kctx->vinstr_cli, ++ BASE_HWCNT_READER_EVENT_MANUAL); ++ mutex_unlock(&kctx->vinstr_cli_lock); ++ ++ return ret; ++} ++ ++static int kbase_api_hwcnt_clear(struct kbase_context *kctx) ++{ ++ int ret; ++ ++ mutex_lock(&kctx->vinstr_cli_lock); ++ ret = kbase_vinstr_hwc_clear(kctx->vinstr_cli); ++ mutex_unlock(&kctx->vinstr_cli_lock); ++ ++ return ret; ++} ++ ++static int kbase_api_disjoint_query(struct kbase_context *kctx, ++ struct kbase_ioctl_disjoint_query *query) ++{ ++ query->counter = kbase_disjoint_event_get(kctx->kbdev); ++ ++ return 0; ++} ++ ++static int kbase_api_get_ddk_version(struct kbase_context *kctx, ++ struct kbase_ioctl_get_ddk_version *version) ++{ ++ int ret; ++ int len = sizeof(KERNEL_SIDE_DDK_VERSION_STRING); ++ ++ if (version->version_buffer == 0) ++ return len; ++ ++ if (version->size < len) ++ return -EOVERFLOW; ++ ++ ret = copy_to_user(u64_to_user_ptr(version->version_buffer), ++ KERNEL_SIDE_DDK_VERSION_STRING, ++ sizeof(KERNEL_SIDE_DDK_VERSION_STRING)); ++ ++ if (ret) ++ return -EFAULT; ++ ++ return len; ++} ++ ++static int kbase_api_mem_jit_init(struct kbase_context *kctx, ++ struct kbase_ioctl_mem_jit_init *jit_init) ++{ ++ return kbase_region_tracker_init_jit(kctx, jit_init->va_pages); ++} ++ ++static int kbase_api_mem_sync(struct kbase_context *kctx, ++ struct kbase_ioctl_mem_sync *sync) ++{ ++ struct basep_syncset sset = { ++ .mem_handle.basep.handle = sync->handle, ++ .user_addr = sync->user_addr, ++ .size = sync->size, ++ .type = sync->type ++ }; ++ ++ return kbase_sync_now(kctx, &sset); ++} ++ ++static int kbase_api_mem_find_cpu_offset(struct kbase_context *kctx, ++ union kbase_ioctl_mem_find_cpu_offset *find) ++{ ++ return kbasep_find_enclosing_cpu_mapping_offset( ++ kctx, ++ find->in.cpu_addr, ++ find->in.size, ++ &find->out.offset); ++} ++ ++static int kbase_api_get_context_id(struct kbase_context *kctx, ++ struct kbase_ioctl_get_context_id *info) ++{ ++ info->id = kctx->id; ++ ++ return 0; ++} ++ ++static int kbase_api_tlstream_acquire(struct kbase_context *kctx, ++ struct kbase_ioctl_tlstream_acquire *acquire) ++{ ++ return kbase_tlstream_acquire(kctx, acquire->flags); ++} ++ ++static int kbase_api_tlstream_flush(struct kbase_context *kctx) ++{ ++ kbase_tlstream_flush_streams(); ++ ++ return 0; ++} ++ ++static int kbase_api_mem_commit(struct kbase_context *kctx, ++ struct kbase_ioctl_mem_commit *commit) ++{ ++ return kbase_mem_commit(kctx, commit->gpu_addr, commit->pages); ++} ++ ++static int kbase_api_mem_alias(struct kbase_context *kctx, ++ union kbase_ioctl_mem_alias *alias) ++{ ++ struct base_mem_aliasing_info *ai; ++ u64 flags; ++ int err; ++ ++ if (alias->in.nents == 0 || alias->in.nents > 2048) ++ return -EINVAL; ++ ++ ai = vmalloc(sizeof(*ai) * alias->in.nents); ++ if (!ai) ++ return -ENOMEM; ++ ++ err = copy_from_user(ai, ++ u64_to_user_ptr(alias->in.aliasing_info), ++ sizeof(*ai) * alias->in.nents); ++ if (err) { ++ vfree(ai); ++ return -EFAULT; ++ } ++ ++ flags = alias->in.flags; ++ ++ alias->out.gpu_va = kbase_mem_alias(kctx, &flags, ++ alias->in.stride, alias->in.nents, ++ ai, &alias->out.va_pages); ++ ++ alias->out.flags = flags; ++ ++ vfree(ai); ++ ++ if (alias->out.gpu_va == 0) ++ return -ENOMEM; ++ ++ return 0; ++} ++ ++static int kbase_api_mem_import(struct kbase_context *kctx, ++ union kbase_ioctl_mem_import *import) ++{ ++ int ret; ++ u64 flags = import->in.flags; ++ ++ ret = kbase_mem_import(kctx, ++ import->in.type, ++ u64_to_user_ptr(import->in.phandle), ++ import->in.padding, ++ &import->out.gpu_va, ++ &import->out.va_pages, ++ &flags); ++ ++ import->out.flags = flags; ++ ++ return ret; ++} ++ ++static int kbase_api_mem_flags_change(struct kbase_context *kctx, ++ struct kbase_ioctl_mem_flags_change *change) ++{ ++ return kbase_mem_flags_change(kctx, change->gpu_va, ++ change->flags, change->mask); ++} ++ ++static int kbase_api_stream_create(struct kbase_context *kctx, ++ struct kbase_ioctl_stream_create *stream) ++{ ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ int fd, ret; ++ ++ /* Name must be NULL-terminated and padded with NULLs, so check last ++ * character is NULL ++ */ ++ if (stream->name[sizeof(stream->name)-1] != 0) ++ return -EINVAL; ++ ++ ret = kbase_sync_fence_stream_create(stream->name, &fd); ++ ++ if (ret) ++ return ret; ++ return fd; ++#else ++ return -ENOENT; ++#endif ++} ++ ++static int kbase_api_fence_validate(struct kbase_context *kctx, ++ struct kbase_ioctl_fence_validate *validate) ++{ ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ return kbase_sync_fence_validate(validate->fd); ++#else ++ return -ENOENT; ++#endif ++} ++ ++static int kbase_api_get_profiling_controls(struct kbase_context *kctx, ++ struct kbase_ioctl_get_profiling_controls *controls) ++{ ++ int ret; ++ ++ if (controls->count > (FBDUMP_CONTROL_MAX - FBDUMP_CONTROL_MIN)) ++ return -EINVAL; ++ ++ ret = copy_to_user(u64_to_user_ptr(controls->buffer), ++ &kctx->kbdev->kbase_profiling_controls[ ++ FBDUMP_CONTROL_MIN], ++ controls->count * sizeof(u32)); ++ ++ if (ret) ++ return -EFAULT; ++ return 0; ++} ++ ++static int kbase_api_mem_profile_add(struct kbase_context *kctx, ++ struct kbase_ioctl_mem_profile_add *data) ++{ ++ char *buf; ++ int err; ++ ++ if (data->len > KBASE_MEM_PROFILE_MAX_BUF_SIZE) { ++ dev_err(kctx->kbdev->dev, "mem_profile_add: buffer too big\n"); ++ return -EINVAL; ++ } ++ ++ buf = kmalloc(data->len, GFP_KERNEL); ++ if (ZERO_OR_NULL_PTR(buf)) ++ return -ENOMEM; ++ ++ err = copy_from_user(buf, u64_to_user_ptr(data->buffer), ++ data->len); ++ if (err) { ++ kfree(buf); ++ return -EFAULT; ++ } ++ ++ return kbasep_mem_profile_debugfs_insert(kctx, buf, data->len); ++} ++ ++static int kbase_api_soft_event_update(struct kbase_context *kctx, ++ struct kbase_ioctl_soft_event_update *update) ++{ ++ if (update->flags != 0) ++ return -EINVAL; ++ ++ return kbase_soft_event_update(kctx, update->event, update->new_status); ++} ++ ++#if MALI_UNIT_TEST ++static int kbase_api_tlstream_test(struct kbase_context *kctx, ++ struct kbase_ioctl_tlstream_test *test) ++{ ++ kbase_tlstream_test( ++ test->tpw_count, ++ test->msg_delay, ++ test->msg_count, ++ test->aux_msg); ++ ++ return 0; ++} ++ ++static int kbase_api_tlstream_stats(struct kbase_context *kctx, ++ struct kbase_ioctl_tlstream_stats *stats) ++{ ++ kbase_tlstream_stats( ++ &stats->bytes_collected, ++ &stats->bytes_generated); ++ ++ return 0; ++} ++#endif /* MALI_UNIT_TEST */ ++ ++#define KBASE_HANDLE_IOCTL(cmd, function) \ ++ case cmd: \ ++ do { \ ++ BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_NONE); \ ++ return function(kctx); \ ++ } while (0) ++ ++#define KBASE_HANDLE_IOCTL_IN(cmd, function, type) \ ++ case cmd: \ ++ do { \ ++ type param; \ ++ int err; \ ++ BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_WRITE); \ ++ BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ ++ err = copy_from_user(¶m, uarg, sizeof(param)); \ ++ if (err) \ ++ return -EFAULT; \ ++ return function(kctx, ¶m); \ ++ } while (0) ++ ++#define KBASE_HANDLE_IOCTL_OUT(cmd, function, type) \ ++ case cmd: \ ++ do { \ ++ type param; \ ++ int ret, err; \ ++ BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_READ); \ ++ BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ ++ ret = function(kctx, ¶m); \ ++ err = copy_to_user(uarg, ¶m, sizeof(param)); \ ++ if (err) \ ++ return -EFAULT; \ ++ return ret; \ ++ } while (0) ++ ++#define KBASE_HANDLE_IOCTL_INOUT(cmd, function, type) \ ++ case cmd: \ ++ do { \ ++ type param; \ ++ int ret, err; \ ++ BUILD_BUG_ON(_IOC_DIR(cmd) != (_IOC_WRITE|_IOC_READ)); \ ++ BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ ++ err = copy_from_user(¶m, uarg, sizeof(param)); \ ++ if (err) \ ++ return -EFAULT; \ ++ ret = function(kctx, ¶m); \ ++ err = copy_to_user(uarg, ¶m, sizeof(param)); \ ++ if (err) \ ++ return -EFAULT; \ ++ return ret; \ ++ } while (0) ++ ++static long kbase_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) ++{ ++ struct kbase_context *kctx = filp->private_data; ++ struct kbase_device *kbdev = kctx->kbdev; ++ void __user *uarg = (void __user *)arg; ++ ++ /* The UK ioctl values overflow the cmd field causing the type to be ++ * incremented ++ */ ++ if (_IOC_TYPE(cmd) == LINUX_UK_BASE_MAGIC+2) ++ return kbase_legacy_ioctl(filp, cmd, arg); ++ ++ /* The UK version check IOCTL doesn't overflow the cmd field, so is ++ * handled separately here ++ */ ++ if (cmd == _IOC(_IOC_READ|_IOC_WRITE, LINUX_UK_BASE_MAGIC, ++ UKP_FUNC_ID_CHECK_VERSION, ++ sizeof(struct uku_version_check_args))) ++ return kbase_legacy_ioctl(filp, cmd, arg); ++ ++ /* Only these ioctls are available until setup is complete */ ++ switch (cmd) { ++ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_VERSION_CHECK, ++ kbase_api_handshake, ++ struct kbase_ioctl_version_check); ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_SET_FLAGS, ++ kbase_api_set_flags, ++ struct kbase_ioctl_set_flags); ++ } ++ ++ /* Block call until version handshake and setup is complete */ ++ if (kctx->api_version == 0 || !atomic_read(&kctx->setup_complete)) ++ return -EINVAL; ++ ++ /* Normal ioctls */ ++ switch (cmd) { ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_JOB_SUBMIT, ++ kbase_api_job_submit, ++ struct kbase_ioctl_job_submit); ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_GET_GPUPROPS, ++ kbase_api_get_gpuprops, ++ struct kbase_ioctl_get_gpuprops); ++ KBASE_HANDLE_IOCTL(KBASE_IOCTL_POST_TERM, ++ kbase_api_post_term); ++ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_ALLOC, ++ kbase_api_mem_alloc, ++ union kbase_ioctl_mem_alloc); ++ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_QUERY, ++ kbase_api_mem_query, ++ union kbase_ioctl_mem_query); ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_FREE, ++ kbase_api_mem_free, ++ struct kbase_ioctl_mem_free); ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_HWCNT_READER_SETUP, ++ kbase_api_hwcnt_reader_setup, ++ struct kbase_ioctl_hwcnt_reader_setup); ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_HWCNT_ENABLE, ++ kbase_api_hwcnt_enable, ++ struct kbase_ioctl_hwcnt_enable); ++ KBASE_HANDLE_IOCTL(KBASE_IOCTL_HWCNT_DUMP, ++ kbase_api_hwcnt_dump); ++ KBASE_HANDLE_IOCTL(KBASE_IOCTL_HWCNT_CLEAR, ++ kbase_api_hwcnt_clear); ++ KBASE_HANDLE_IOCTL_OUT(KBASE_IOCTL_DISJOINT_QUERY, ++ kbase_api_disjoint_query, ++ struct kbase_ioctl_disjoint_query); ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_GET_DDK_VERSION, ++ kbase_api_get_ddk_version, ++ struct kbase_ioctl_get_ddk_version); ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_JIT_INIT, ++ kbase_api_mem_jit_init, ++ struct kbase_ioctl_mem_jit_init); ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_SYNC, ++ kbase_api_mem_sync, ++ struct kbase_ioctl_mem_sync); ++ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_FIND_CPU_OFFSET, ++ kbase_api_mem_find_cpu_offset, ++ union kbase_ioctl_mem_find_cpu_offset); ++ KBASE_HANDLE_IOCTL_OUT(KBASE_IOCTL_GET_CONTEXT_ID, ++ kbase_api_get_context_id, ++ struct kbase_ioctl_get_context_id); ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_TLSTREAM_ACQUIRE, ++ kbase_api_tlstream_acquire, ++ struct kbase_ioctl_tlstream_acquire); ++ KBASE_HANDLE_IOCTL(KBASE_IOCTL_TLSTREAM_FLUSH, ++ kbase_api_tlstream_flush); ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_COMMIT, ++ kbase_api_mem_commit, ++ struct kbase_ioctl_mem_commit); ++ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_ALIAS, ++ kbase_api_mem_alias, ++ union kbase_ioctl_mem_alias); ++ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_IMPORT, ++ kbase_api_mem_import, ++ union kbase_ioctl_mem_import); ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_FLAGS_CHANGE, ++ kbase_api_mem_flags_change, ++ struct kbase_ioctl_mem_flags_change); ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_STREAM_CREATE, ++ kbase_api_stream_create, ++ struct kbase_ioctl_stream_create); ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_FENCE_VALIDATE, ++ kbase_api_fence_validate, ++ struct kbase_ioctl_fence_validate); ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_GET_PROFILING_CONTROLS, ++ kbase_api_get_profiling_controls, ++ struct kbase_ioctl_get_profiling_controls); ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_PROFILE_ADD, ++ kbase_api_mem_profile_add, ++ struct kbase_ioctl_mem_profile_add); ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_SOFT_EVENT_UPDATE, ++ kbase_api_soft_event_update, ++ struct kbase_ioctl_soft_event_update); ++ ++#if MALI_UNIT_TEST ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_TLSTREAM_TEST, ++ kbase_api_tlstream_test, ++ struct kbase_ioctl_tlstream_test); ++ KBASE_HANDLE_IOCTL_OUT(KBASE_IOCTL_TLSTREAM_STATS, ++ kbase_api_tlstream_stats, ++ struct kbase_ioctl_tlstream_stats); ++#endif ++ } ++ ++ dev_warn(kbdev->dev, "Unknown ioctl 0x%x nr:%d", cmd, _IOC_NR(cmd)); ++ ++ return -ENOIOCTLCMD; ++} ++ ++static ssize_t kbase_read(struct file *filp, char __user *buf, size_t count, loff_t *f_pos) ++{ ++ struct kbase_context *kctx = filp->private_data; ++ struct base_jd_event_v2 uevent; ++ int out_count = 0; ++ ++ if (count < sizeof(uevent)) ++ return -ENOBUFS; ++ ++ do { ++ while (kbase_event_dequeue(kctx, &uevent)) { ++ if (out_count > 0) ++ goto out; ++ ++ if (filp->f_flags & O_NONBLOCK) ++ return -EAGAIN; ++ ++ if (wait_event_interruptible(kctx->event_queue, ++ kbase_event_pending(kctx)) != 0) ++ return -ERESTARTSYS; ++ } ++ if (uevent.event_code == BASE_JD_EVENT_DRV_TERMINATED) { ++ if (out_count == 0) ++ return -EPIPE; ++ goto out; ++ } ++ ++ if (copy_to_user(buf, &uevent, sizeof(uevent)) != 0) ++ return -EFAULT; ++ ++ buf += sizeof(uevent); ++ out_count++; ++ count -= sizeof(uevent); ++ } while (count >= sizeof(uevent)); ++ ++ out: ++ return out_count * sizeof(uevent); ++} ++ ++static unsigned int kbase_poll(struct file *filp, poll_table *wait) ++{ ++ struct kbase_context *kctx = filp->private_data; ++ ++ poll_wait(filp, &kctx->event_queue, wait); ++ if (kbase_event_pending(kctx)) ++ return POLLIN | POLLRDNORM; ++ ++ return 0; ++} ++ ++void kbase_event_wakeup(struct kbase_context *kctx) ++{ ++ KBASE_DEBUG_ASSERT(kctx); ++ ++ wake_up_interruptible(&kctx->event_queue); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_event_wakeup); ++ ++static int kbase_check_flags(int flags) ++{ ++ /* Enforce that the driver keeps the O_CLOEXEC flag so that execve() always ++ * closes the file descriptor in a child process. ++ */ ++ if (0 == (flags & O_CLOEXEC)) ++ return -EINVAL; ++ ++ return 0; ++} ++ ++ ++/** ++ * align_and_check - Align the specified pointer to the provided alignment and ++ * check that it is still in range. ++ * @gap_end: Highest possible start address for allocation (end of gap in ++ * address space) ++ * @gap_start: Start address of current memory area / gap in address space ++ * @info: vm_unmapped_area_info structure passed to caller, containing ++ * alignment, length and limits for the allocation ++ * @is_shader_code: True if the allocation is for shader code (which has ++ * additional alignment requirements) ++ * ++ * Return: true if gap_end is now aligned correctly and is still in range, ++ * false otherwise ++ */ ++static bool align_and_check(unsigned long *gap_end, unsigned long gap_start, ++ struct vm_unmapped_area_info *info, bool is_shader_code) ++{ ++ /* Compute highest gap address at the desired alignment */ ++ (*gap_end) -= info->length; ++ (*gap_end) -= (*gap_end - info->align_offset) & info->align_mask; ++ ++ if (is_shader_code) { ++ /* Check for 4GB boundary */ ++ if (0 == (*gap_end & BASE_MEM_MASK_4GB)) ++ (*gap_end) -= (info->align_offset ? info->align_offset : ++ info->length); ++ if (0 == ((*gap_end + info->length) & BASE_MEM_MASK_4GB)) ++ (*gap_end) -= (info->align_offset ? info->align_offset : ++ info->length); ++ ++ if (!(*gap_end & BASE_MEM_MASK_4GB) || !((*gap_end + ++ info->length) & BASE_MEM_MASK_4GB)) ++ return false; ++ } ++ ++ ++ if ((*gap_end < info->low_limit) || (*gap_end < gap_start)) ++ return false; ++ ++ ++ return true; ++} ++ ++/* The following function is taken from the kernel and just ++ * renamed. As it's not exported to modules we must copy-paste it here. ++ */ ++ ++static unsigned long kbase_unmapped_area_topdown(struct vm_unmapped_area_info ++ *info, bool is_shader_code) ++{ ++ struct mm_struct *mm = current->mm; ++ struct vm_area_struct *vma; ++ unsigned long length, low_limit, high_limit, gap_start, gap_end; ++ ++ /* Adjust search length to account for worst case alignment overhead */ ++ length = info->length + info->align_mask; ++ if (length < info->length) ++ return -ENOMEM; ++ ++ /* ++ * Adjust search limits by the desired length. ++ * See implementation comment at top of unmapped_area(). ++ */ ++ gap_end = info->high_limit; ++ if (gap_end < length) ++ return -ENOMEM; ++ high_limit = gap_end - length; ++ ++ if (info->low_limit > high_limit) ++ return -ENOMEM; ++ low_limit = info->low_limit + length; ++ ++ /* Check highest gap, which does not precede any rbtree node */ ++ gap_start = mm->highest_vm_end; ++ if (gap_start <= high_limit) { ++ if (align_and_check(&gap_end, gap_start, info, is_shader_code)) ++ return gap_end; ++ } ++ ++ /* Check if rbtree root looks promising */ ++ if (RB_EMPTY_ROOT(&mm->mm_rb)) ++ return -ENOMEM; ++ vma = rb_entry(mm->mm_rb.rb_node, struct vm_area_struct, vm_rb); ++ if (vma->rb_subtree_gap < length) ++ return -ENOMEM; ++ ++ while (true) { ++ /* Visit right subtree if it looks promising */ ++ gap_start = vma->vm_prev ? vma->vm_prev->vm_end : 0; ++ if (gap_start <= high_limit && vma->vm_rb.rb_right) { ++ struct vm_area_struct *right = ++ rb_entry(vma->vm_rb.rb_right, ++ struct vm_area_struct, vm_rb); ++ if (right->rb_subtree_gap >= length) { ++ vma = right; ++ continue; ++ } ++ } ++ ++check_current: ++ /* Check if current node has a suitable gap */ ++ gap_end = vma->vm_start; ++ if (gap_end < low_limit) ++ return -ENOMEM; ++ if (gap_start <= high_limit && gap_end - gap_start >= length) { ++ /* We found a suitable gap. Clip it with the original ++ * high_limit. */ ++ if (gap_end > info->high_limit) ++ gap_end = info->high_limit; ++ ++ if (align_and_check(&gap_end, gap_start, info, ++ is_shader_code)) ++ return gap_end; ++ } ++ ++ /* Visit left subtree if it looks promising */ ++ if (vma->vm_rb.rb_left) { ++ struct vm_area_struct *left = ++ rb_entry(vma->vm_rb.rb_left, ++ struct vm_area_struct, vm_rb); ++ if (left->rb_subtree_gap >= length) { ++ vma = left; ++ continue; ++ } ++ } ++ ++ /* Go back up the rbtree to find next candidate node */ ++ while (true) { ++ struct rb_node *prev = &vma->vm_rb; ++ if (!rb_parent(prev)) ++ return -ENOMEM; ++ vma = rb_entry(rb_parent(prev), ++ struct vm_area_struct, vm_rb); ++ if (prev == vma->vm_rb.rb_right) { ++ gap_start = vma->vm_prev ? ++ vma->vm_prev->vm_end : 0; ++ goto check_current; ++ } ++ } ++ } ++ ++ return -ENOMEM; ++} ++ ++static unsigned long kbase_get_unmapped_area(struct file *filp, ++ const unsigned long addr, const unsigned long len, ++ const unsigned long pgoff, const unsigned long flags) ++{ ++ /* based on get_unmapped_area, but simplified slightly due to that some ++ * values are known in advance */ ++ struct kbase_context *kctx = filp->private_data; ++ struct mm_struct *mm = current->mm; ++ struct vm_unmapped_area_info info; ++ unsigned long align_offset = 0; ++ unsigned long align_mask = 0; ++ unsigned long high_limit = mm->mmap_base; ++ unsigned long low_limit = PAGE_SIZE; ++ int cpu_va_bits = BITS_PER_LONG; ++ int gpu_pc_bits = ++ kctx->kbdev->gpu_props.props.core_props.log2_program_counter_size; ++ bool is_shader_code = false; ++ unsigned long ret; ++ ++ /* err on fixed address */ ++ if ((flags & MAP_FIXED) || addr) ++ return -EINVAL; ++ ++#ifdef CONFIG_64BIT ++ /* too big? */ ++ if (len > TASK_SIZE - SZ_2M) ++ return -ENOMEM; ++ ++ if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { ++ ++ if (kbase_hw_has_feature(kctx->kbdev, ++ BASE_HW_FEATURE_33BIT_VA)) { ++ high_limit = kctx->same_va_end << PAGE_SHIFT; ++ } else { ++ high_limit = min_t(unsigned long, mm->mmap_base, ++ (kctx->same_va_end << PAGE_SHIFT)); ++ if (len >= SZ_2M) { ++ align_offset = SZ_2M; ++ align_mask = SZ_2M - 1; ++ } ++ } ++ ++ low_limit = SZ_2M; ++ } else { ++ cpu_va_bits = 32; ++ } ++#endif /* CONFIG_64BIT */ ++ if ((PFN_DOWN(BASE_MEM_COOKIE_BASE) <= pgoff) && ++ (PFN_DOWN(BASE_MEM_FIRST_FREE_ADDRESS) > pgoff)) { ++ int cookie = pgoff - PFN_DOWN(BASE_MEM_COOKIE_BASE); ++ ++ if (!kctx->pending_regions[cookie]) ++ return -EINVAL; ++ ++ if (!(kctx->pending_regions[cookie]->flags & ++ KBASE_REG_GPU_NX)) { ++ if (cpu_va_bits > gpu_pc_bits) { ++ align_offset = 1ULL << gpu_pc_bits; ++ align_mask = align_offset - 1; ++ is_shader_code = true; ++ } ++ } ++#ifndef CONFIG_64BIT ++ } else { ++ return current->mm->get_unmapped_area(filp, addr, len, pgoff, ++ flags); ++#endif ++ } ++ ++ info.flags = 0; ++ info.length = len; ++ info.low_limit = low_limit; ++ info.high_limit = high_limit; ++ info.align_offset = align_offset; ++ info.align_mask = align_mask; ++ ++ ret = kbase_unmapped_area_topdown(&info, is_shader_code); ++ ++ if (IS_ERR_VALUE(ret) && high_limit == mm->mmap_base && ++ high_limit < (kctx->same_va_end << PAGE_SHIFT)) { ++ /* Retry above mmap_base */ ++ info.low_limit = mm->mmap_base; ++ info.high_limit = min_t(u64, TASK_SIZE, ++ (kctx->same_va_end << PAGE_SHIFT)); ++ ++ ret = kbase_unmapped_area_topdown(&info, is_shader_code); ++ } ++ ++ return ret; ++} ++ ++static const struct file_operations kbase_fops = { ++ .owner = THIS_MODULE, ++ .open = kbase_open, ++ .release = kbase_release, ++ .read = kbase_read, ++ .poll = kbase_poll, ++ .unlocked_ioctl = kbase_ioctl, ++ .compat_ioctl = kbase_ioctl, ++ .mmap = kbase_mmap, ++ .check_flags = kbase_check_flags, ++ .get_unmapped_area = kbase_get_unmapped_area, ++}; ++ ++#ifndef CONFIG_MALI_BIFROST_NO_MALI ++void kbase_os_reg_write(struct kbase_device *kbdev, u16 offset, u32 value) ++{ ++ writel(value, kbdev->reg + offset); ++} ++ ++u32 kbase_os_reg_read(struct kbase_device *kbdev, u16 offset) ++{ ++ return readl(kbdev->reg + offset); ++} ++#endif /* !CONFIG_MALI_BIFROST_NO_MALI */ ++ ++/** ++ * show_policy - Show callback for the power_policy sysfs file. ++ * ++ * This function is called to get the contents of the power_policy sysfs ++ * file. This is a list of the available policies with the currently active one ++ * surrounded by square brackets. ++ * ++ * @dev: The device this sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The output buffer for the sysfs file contents ++ * ++ * Return: The number of bytes output to @buf. ++ */ ++static ssize_t show_policy(struct device *dev, struct device_attribute *attr, char *const buf) ++{ ++ struct kbase_device *kbdev; ++ const struct kbase_pm_policy *current_policy; ++ const struct kbase_pm_policy *const *policy_list; ++ int policy_count; ++ int i; ++ ssize_t ret = 0; ++ ++ kbdev = to_kbase_device(dev); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ current_policy = kbase_pm_get_policy(kbdev); ++ ++ policy_count = kbase_pm_list_policies(&policy_list); ++ ++ for (i = 0; i < policy_count && ret < PAGE_SIZE; i++) { ++ if (policy_list[i] == current_policy) ++ ret += scnprintf(buf + ret, PAGE_SIZE - ret, "[%s] ", policy_list[i]->name); ++ else ++ ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s ", policy_list[i]->name); ++ } ++ ++ if (ret < PAGE_SIZE - 1) { ++ ret += scnprintf(buf + ret, PAGE_SIZE - ret, "\n"); ++ } else { ++ buf[PAGE_SIZE - 2] = '\n'; ++ buf[PAGE_SIZE - 1] = '\0'; ++ ret = PAGE_SIZE - 1; ++ } ++ ++ return ret; ++} ++ ++/** ++ * set_policy - Store callback for the power_policy sysfs file. ++ * ++ * This function is called when the power_policy sysfs file is written to. ++ * It matches the requested policy against the available policies and if a ++ * matching policy is found calls kbase_pm_set_policy() to change the ++ * policy. ++ * ++ * @dev: The device with sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The value written to the sysfs file ++ * @count: The number of bytes written to the sysfs file ++ * ++ * Return: @count if the function succeeded. An error code on failure. ++ */ ++static ssize_t set_policy(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) ++{ ++ struct kbase_device *kbdev; ++ const struct kbase_pm_policy *new_policy = NULL; ++ const struct kbase_pm_policy *const *policy_list; ++ int policy_count; ++ int i; ++ ++ kbdev = to_kbase_device(dev); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ policy_count = kbase_pm_list_policies(&policy_list); ++ ++ for (i = 0; i < policy_count; i++) { ++ if (sysfs_streq(policy_list[i]->name, buf)) { ++ new_policy = policy_list[i]; ++ break; ++ } ++ } ++ ++ if (!new_policy) { ++ dev_err(dev, "power_policy: policy not found\n"); ++ return -EINVAL; ++ } ++ ++ kbase_pm_set_policy(kbdev, new_policy); ++ ++ return count; ++} ++ ++/* ++ * The sysfs file power_policy. ++ * ++ * This is used for obtaining information about the available policies, ++ * determining which policy is currently active, and changing the active ++ * policy. ++ */ ++static DEVICE_ATTR(power_policy, S_IRUGO | S_IWUSR, show_policy, set_policy); ++ ++/** ++ * show_ca_policy - Show callback for the core_availability_policy sysfs file. ++ * ++ * This function is called to get the contents of the core_availability_policy ++ * sysfs file. This is a list of the available policies with the currently ++ * active one surrounded by square brackets. ++ * ++ * @dev: The device this sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The output buffer for the sysfs file contents ++ * ++ * Return: The number of bytes output to @buf. ++ */ ++static ssize_t show_ca_policy(struct device *dev, struct device_attribute *attr, char * const buf) ++{ ++ struct kbase_device *kbdev; ++ const struct kbase_pm_ca_policy *current_policy; ++ const struct kbase_pm_ca_policy *const *policy_list; ++ int policy_count; ++ int i; ++ ssize_t ret = 0; ++ ++ kbdev = to_kbase_device(dev); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ current_policy = kbase_pm_ca_get_policy(kbdev); ++ ++ policy_count = kbase_pm_ca_list_policies(&policy_list); ++ ++ for (i = 0; i < policy_count && ret < PAGE_SIZE; i++) { ++ if (policy_list[i] == current_policy) ++ ret += scnprintf(buf + ret, PAGE_SIZE - ret, "[%s] ", policy_list[i]->name); ++ else ++ ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s ", policy_list[i]->name); ++ } ++ ++ if (ret < PAGE_SIZE - 1) { ++ ret += scnprintf(buf + ret, PAGE_SIZE - ret, "\n"); ++ } else { ++ buf[PAGE_SIZE - 2] = '\n'; ++ buf[PAGE_SIZE - 1] = '\0'; ++ ret = PAGE_SIZE - 1; ++ } ++ ++ return ret; ++} ++ ++/** ++ * set_ca_policy - Store callback for the core_availability_policy sysfs file. ++ * ++ * This function is called when the core_availability_policy sysfs file is ++ * written to. It matches the requested policy against the available policies ++ * and if a matching policy is found calls kbase_pm_set_policy() to change ++ * the policy. ++ * ++ * @dev: The device with sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The value written to the sysfs file ++ * @count: The number of bytes written to the sysfs file ++ * ++ * Return: @count if the function succeeded. An error code on failure. ++ */ ++static ssize_t set_ca_policy(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) ++{ ++ struct kbase_device *kbdev; ++ const struct kbase_pm_ca_policy *new_policy = NULL; ++ const struct kbase_pm_ca_policy *const *policy_list; ++ int policy_count; ++ int i; ++ ++ kbdev = to_kbase_device(dev); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ policy_count = kbase_pm_ca_list_policies(&policy_list); ++ ++ for (i = 0; i < policy_count; i++) { ++ if (sysfs_streq(policy_list[i]->name, buf)) { ++ new_policy = policy_list[i]; ++ break; ++ } ++ } ++ ++ if (!new_policy) { ++ dev_err(dev, "core_availability_policy: policy not found\n"); ++ return -EINVAL; ++ } ++ ++ kbase_pm_ca_set_policy(kbdev, new_policy); ++ ++ return count; ++} ++ ++/* ++ * The sysfs file core_availability_policy ++ * ++ * This is used for obtaining information about the available policies, ++ * determining which policy is currently active, and changing the active ++ * policy. ++ */ ++static DEVICE_ATTR(core_availability_policy, S_IRUGO | S_IWUSR, show_ca_policy, set_ca_policy); ++ ++/* ++ * show_core_mask - Show callback for the core_mask sysfs file. ++ * ++ * This function is called to get the contents of the core_mask sysfs file. ++ * ++ * @dev: The device this sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The output buffer for the sysfs file contents ++ * ++ * Return: The number of bytes output to @buf. ++ */ ++static ssize_t show_core_mask(struct device *dev, struct device_attribute *attr, char * const buf) ++{ ++ struct kbase_device *kbdev; ++ ssize_t ret = 0; ++ ++ kbdev = to_kbase_device(dev); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ ret += scnprintf(buf + ret, PAGE_SIZE - ret, ++ "Current core mask (JS0) : 0x%llX\n", ++ kbdev->pm.debug_core_mask[0]); ++ ret += scnprintf(buf + ret, PAGE_SIZE - ret, ++ "Current core mask (JS1) : 0x%llX\n", ++ kbdev->pm.debug_core_mask[1]); ++ ret += scnprintf(buf + ret, PAGE_SIZE - ret, ++ "Current core mask (JS2) : 0x%llX\n", ++ kbdev->pm.debug_core_mask[2]); ++ ret += scnprintf(buf + ret, PAGE_SIZE - ret, ++ "Available core mask : 0x%llX\n", ++ kbdev->gpu_props.props.raw_props.shader_present); ++ ++ return ret; ++} ++ ++/** ++ * set_core_mask - Store callback for the core_mask sysfs file. ++ * ++ * This function is called when the core_mask sysfs file is written to. ++ * ++ * @dev: The device with sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The value written to the sysfs file ++ * @count: The number of bytes written to the sysfs file ++ * ++ * Return: @count if the function succeeded. An error code on failure. ++ */ ++static ssize_t set_core_mask(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) ++{ ++ struct kbase_device *kbdev; ++ u64 new_core_mask[3]; ++ int items; ++ ++ kbdev = to_kbase_device(dev); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ items = sscanf(buf, "%llx %llx %llx", ++ &new_core_mask[0], &new_core_mask[1], ++ &new_core_mask[2]); ++ ++ if (items == 1) ++ new_core_mask[1] = new_core_mask[2] = new_core_mask[0]; ++ ++ if (items == 1 || items == 3) { ++ u64 shader_present = ++ kbdev->gpu_props.props.raw_props.shader_present; ++ u64 group0_core_mask = ++ kbdev->gpu_props.props.coherency_info.group[0]. ++ core_mask; ++ ++ if ((new_core_mask[0] & shader_present) != new_core_mask[0] || ++ !(new_core_mask[0] & group0_core_mask) || ++ (new_core_mask[1] & shader_present) != ++ new_core_mask[1] || ++ !(new_core_mask[1] & group0_core_mask) || ++ (new_core_mask[2] & shader_present) != ++ new_core_mask[2] || ++ !(new_core_mask[2] & group0_core_mask)) { ++ dev_err(dev, "power_policy: invalid core specification\n"); ++ return -EINVAL; ++ } ++ ++ if (kbdev->pm.debug_core_mask[0] != new_core_mask[0] || ++ kbdev->pm.debug_core_mask[1] != ++ new_core_mask[1] || ++ kbdev->pm.debug_core_mask[2] != ++ new_core_mask[2]) { ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ kbase_pm_set_debug_core_mask(kbdev, new_core_mask[0], ++ new_core_mask[1], new_core_mask[2]); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ } ++ ++ return count; ++ } ++ ++ dev_err(kbdev->dev, "Couldn't process set_core_mask write operation.\n" ++ "Use format \n" ++ "or \n"); ++ return -EINVAL; ++} ++ ++/* ++ * The sysfs file core_mask. ++ * ++ * This is used to restrict shader core availability for debugging purposes. ++ * Reading it will show the current core mask and the mask of cores available. ++ * Writing to it will set the current core mask. ++ */ ++static DEVICE_ATTR(core_mask, S_IRUGO | S_IWUSR, show_core_mask, set_core_mask); ++ ++/** ++ * set_soft_job_timeout - Store callback for the soft_job_timeout sysfs ++ * file. ++ * ++ * @dev: The device this sysfs file is for. ++ * @attr: The attributes of the sysfs file. ++ * @buf: The value written to the sysfs file. ++ * @count: The number of bytes written to the sysfs file. ++ * ++ * This allows setting the timeout for software jobs. Waiting soft event wait ++ * jobs will be cancelled after this period expires, while soft fence wait jobs ++ * will print debug information if the fence debug feature is enabled. ++ * ++ * This is expressed in milliseconds. ++ * ++ * Return: count if the function succeeded. An error code on failure. ++ */ ++static ssize_t set_soft_job_timeout(struct device *dev, ++ struct device_attribute *attr, ++ const char *buf, size_t count) ++{ ++ struct kbase_device *kbdev; ++ int soft_job_timeout_ms; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ if ((kstrtoint(buf, 0, &soft_job_timeout_ms) != 0) || ++ (soft_job_timeout_ms <= 0)) ++ return -EINVAL; ++ ++ atomic_set(&kbdev->js_data.soft_job_timeout_ms, ++ soft_job_timeout_ms); ++ ++ return count; ++} ++ ++/** ++ * show_soft_job_timeout - Show callback for the soft_job_timeout sysfs ++ * file. ++ * ++ * This will return the timeout for the software jobs. ++ * ++ * @dev: The device this sysfs file is for. ++ * @attr: The attributes of the sysfs file. ++ * @buf: The output buffer for the sysfs file contents. ++ * ++ * Return: The number of bytes output to buf. ++ */ ++static ssize_t show_soft_job_timeout(struct device *dev, ++ struct device_attribute *attr, ++ char * const buf) ++{ ++ struct kbase_device *kbdev; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ return scnprintf(buf, PAGE_SIZE, "%i\n", ++ atomic_read(&kbdev->js_data.soft_job_timeout_ms)); ++} ++ ++static DEVICE_ATTR(soft_job_timeout, S_IRUGO | S_IWUSR, ++ show_soft_job_timeout, set_soft_job_timeout); ++ ++static u32 timeout_ms_to_ticks(struct kbase_device *kbdev, long timeout_ms, ++ int default_ticks, u32 old_ticks) ++{ ++ if (timeout_ms > 0) { ++ u64 ticks = timeout_ms * 1000000ULL; ++ do_div(ticks, kbdev->js_data.scheduling_period_ns); ++ if (!ticks) ++ return 1; ++ return ticks; ++ } else if (timeout_ms < 0) { ++ return default_ticks; ++ } else { ++ return old_ticks; ++ } ++} ++ ++/** ++ * set_js_timeouts - Store callback for the js_timeouts sysfs file. ++ * ++ * This function is called to get the contents of the js_timeouts sysfs ++ * file. This file contains five values separated by whitespace. The values ++ * are basically the same as %JS_SOFT_STOP_TICKS, %JS_HARD_STOP_TICKS_SS, ++ * %JS_HARD_STOP_TICKS_DUMPING, %JS_RESET_TICKS_SS, %JS_RESET_TICKS_DUMPING ++ * configuration values (in that order), with the difference that the js_timeout ++ * values are expressed in MILLISECONDS. ++ * ++ * The js_timeouts sysfile file allows the current values in ++ * use by the job scheduler to get override. Note that a value needs to ++ * be other than 0 for it to override the current job scheduler value. ++ * ++ * @dev: The device with sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The value written to the sysfs file ++ * @count: The number of bytes written to the sysfs file ++ * ++ * Return: @count if the function succeeded. An error code on failure. ++ */ ++static ssize_t set_js_timeouts(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) ++{ ++ struct kbase_device *kbdev; ++ int items; ++ long js_soft_stop_ms; ++ long js_soft_stop_ms_cl; ++ long js_hard_stop_ms_ss; ++ long js_hard_stop_ms_cl; ++ long js_hard_stop_ms_dumping; ++ long js_reset_ms_ss; ++ long js_reset_ms_cl; ++ long js_reset_ms_dumping; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ items = sscanf(buf, "%ld %ld %ld %ld %ld %ld %ld %ld", ++ &js_soft_stop_ms, &js_soft_stop_ms_cl, ++ &js_hard_stop_ms_ss, &js_hard_stop_ms_cl, ++ &js_hard_stop_ms_dumping, &js_reset_ms_ss, ++ &js_reset_ms_cl, &js_reset_ms_dumping); ++ ++ if (items == 8) { ++ struct kbasep_js_device_data *js_data = &kbdev->js_data; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++#define UPDATE_TIMEOUT(ticks_name, ms_name, default) do {\ ++ js_data->ticks_name = timeout_ms_to_ticks(kbdev, ms_name, \ ++ default, js_data->ticks_name); \ ++ dev_dbg(kbdev->dev, "Overriding " #ticks_name \ ++ " with %lu ticks (%lu ms)\n", \ ++ (unsigned long)js_data->ticks_name, \ ++ ms_name); \ ++ } while (0) ++ ++ UPDATE_TIMEOUT(soft_stop_ticks, js_soft_stop_ms, ++ DEFAULT_JS_SOFT_STOP_TICKS); ++ UPDATE_TIMEOUT(soft_stop_ticks_cl, js_soft_stop_ms_cl, ++ DEFAULT_JS_SOFT_STOP_TICKS_CL); ++ UPDATE_TIMEOUT(hard_stop_ticks_ss, js_hard_stop_ms_ss, ++ kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408) ? ++ DEFAULT_JS_HARD_STOP_TICKS_SS_8408 : ++ DEFAULT_JS_HARD_STOP_TICKS_SS); ++ UPDATE_TIMEOUT(hard_stop_ticks_cl, js_hard_stop_ms_cl, ++ DEFAULT_JS_HARD_STOP_TICKS_CL); ++ UPDATE_TIMEOUT(hard_stop_ticks_dumping, ++ js_hard_stop_ms_dumping, ++ DEFAULT_JS_HARD_STOP_TICKS_DUMPING); ++ UPDATE_TIMEOUT(gpu_reset_ticks_ss, js_reset_ms_ss, ++ kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408) ? ++ DEFAULT_JS_RESET_TICKS_SS_8408 : ++ DEFAULT_JS_RESET_TICKS_SS); ++ UPDATE_TIMEOUT(gpu_reset_ticks_cl, js_reset_ms_cl, ++ DEFAULT_JS_RESET_TICKS_CL); ++ UPDATE_TIMEOUT(gpu_reset_ticks_dumping, js_reset_ms_dumping, ++ DEFAULT_JS_RESET_TICKS_DUMPING); ++ ++ kbase_js_set_timeouts(kbdev); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ return count; ++ } ++ ++ dev_err(kbdev->dev, "Couldn't process js_timeouts write operation.\n" ++ "Use format \n" ++ "Write 0 for no change, -1 to restore default timeout\n"); ++ return -EINVAL; ++} ++ ++static unsigned long get_js_timeout_in_ms( ++ u32 scheduling_period_ns, ++ u32 ticks) ++{ ++ u64 ms = (u64)ticks * scheduling_period_ns; ++ ++ do_div(ms, 1000000UL); ++ return ms; ++} ++ ++/** ++ * show_js_timeouts - Show callback for the js_timeouts sysfs file. ++ * ++ * This function is called to get the contents of the js_timeouts sysfs ++ * file. It returns the last set values written to the js_timeouts sysfs file. ++ * If the file didn't get written yet, the values will be current setting in ++ * use. ++ * @dev: The device this sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The output buffer for the sysfs file contents ++ * ++ * Return: The number of bytes output to @buf. ++ */ ++static ssize_t show_js_timeouts(struct device *dev, struct device_attribute *attr, char * const buf) ++{ ++ struct kbase_device *kbdev; ++ ssize_t ret; ++ unsigned long js_soft_stop_ms; ++ unsigned long js_soft_stop_ms_cl; ++ unsigned long js_hard_stop_ms_ss; ++ unsigned long js_hard_stop_ms_cl; ++ unsigned long js_hard_stop_ms_dumping; ++ unsigned long js_reset_ms_ss; ++ unsigned long js_reset_ms_cl; ++ unsigned long js_reset_ms_dumping; ++ u32 scheduling_period_ns; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ scheduling_period_ns = kbdev->js_data.scheduling_period_ns; ++ ++#define GET_TIMEOUT(name) get_js_timeout_in_ms(\ ++ scheduling_period_ns, \ ++ kbdev->js_data.name) ++ ++ js_soft_stop_ms = GET_TIMEOUT(soft_stop_ticks); ++ js_soft_stop_ms_cl = GET_TIMEOUT(soft_stop_ticks_cl); ++ js_hard_stop_ms_ss = GET_TIMEOUT(hard_stop_ticks_ss); ++ js_hard_stop_ms_cl = GET_TIMEOUT(hard_stop_ticks_cl); ++ js_hard_stop_ms_dumping = GET_TIMEOUT(hard_stop_ticks_dumping); ++ js_reset_ms_ss = GET_TIMEOUT(gpu_reset_ticks_ss); ++ js_reset_ms_cl = GET_TIMEOUT(gpu_reset_ticks_cl); ++ js_reset_ms_dumping = GET_TIMEOUT(gpu_reset_ticks_dumping); ++ ++#undef GET_TIMEOUT ++ ++ ret = scnprintf(buf, PAGE_SIZE, "%lu %lu %lu %lu %lu %lu %lu %lu\n", ++ js_soft_stop_ms, js_soft_stop_ms_cl, ++ js_hard_stop_ms_ss, js_hard_stop_ms_cl, ++ js_hard_stop_ms_dumping, js_reset_ms_ss, ++ js_reset_ms_cl, js_reset_ms_dumping); ++ ++ if (ret >= PAGE_SIZE) { ++ buf[PAGE_SIZE - 2] = '\n'; ++ buf[PAGE_SIZE - 1] = '\0'; ++ ret = PAGE_SIZE - 1; ++ } ++ ++ return ret; ++} ++ ++/* ++ * The sysfs file js_timeouts. ++ * ++ * This is used to override the current job scheduler values for ++ * JS_STOP_STOP_TICKS_SS ++ * JS_STOP_STOP_TICKS_CL ++ * JS_HARD_STOP_TICKS_SS ++ * JS_HARD_STOP_TICKS_CL ++ * JS_HARD_STOP_TICKS_DUMPING ++ * JS_RESET_TICKS_SS ++ * JS_RESET_TICKS_CL ++ * JS_RESET_TICKS_DUMPING. ++ */ ++static DEVICE_ATTR(js_timeouts, S_IRUGO | S_IWUSR, show_js_timeouts, set_js_timeouts); ++ ++static u32 get_new_js_timeout( ++ u32 old_period, ++ u32 old_ticks, ++ u32 new_scheduling_period_ns) ++{ ++ u64 ticks = (u64)old_period * (u64)old_ticks; ++ do_div(ticks, new_scheduling_period_ns); ++ return ticks?ticks:1; ++} ++ ++/** ++ * set_js_scheduling_period - Store callback for the js_scheduling_period sysfs ++ * file ++ * @dev: The device the sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The value written to the sysfs file ++ * @count: The number of bytes written to the sysfs file ++ * ++ * This function is called when the js_scheduling_period sysfs file is written ++ * to. It checks the data written, and if valid updates the js_scheduling_period ++ * value ++ * ++ * Return: @count if the function succeeded. An error code on failure. ++ */ ++static ssize_t set_js_scheduling_period(struct device *dev, ++ struct device_attribute *attr, const char *buf, size_t count) ++{ ++ struct kbase_device *kbdev; ++ int ret; ++ unsigned int js_scheduling_period; ++ u32 new_scheduling_period_ns; ++ u32 old_period; ++ struct kbasep_js_device_data *js_data; ++ unsigned long flags; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ js_data = &kbdev->js_data; ++ ++ ret = kstrtouint(buf, 0, &js_scheduling_period); ++ if (ret || !js_scheduling_period) { ++ dev_err(kbdev->dev, "Couldn't process js_scheduling_period write operation.\n" ++ "Use format \n"); ++ return -EINVAL; ++ } ++ ++ new_scheduling_period_ns = js_scheduling_period * 1000000; ++ ++ /* Update scheduling timeouts */ ++ mutex_lock(&js_data->runpool_mutex); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ /* If no contexts have been scheduled since js_timeouts was last written ++ * to, the new timeouts might not have been latched yet. So check if an ++ * update is pending and use the new values if necessary. */ ++ ++ /* Use previous 'new' scheduling period as a base if present. */ ++ old_period = js_data->scheduling_period_ns; ++ ++#define SET_TIMEOUT(name) \ ++ (js_data->name = get_new_js_timeout(\ ++ old_period, \ ++ kbdev->js_data.name, \ ++ new_scheduling_period_ns)) ++ ++ SET_TIMEOUT(soft_stop_ticks); ++ SET_TIMEOUT(soft_stop_ticks_cl); ++ SET_TIMEOUT(hard_stop_ticks_ss); ++ SET_TIMEOUT(hard_stop_ticks_cl); ++ SET_TIMEOUT(hard_stop_ticks_dumping); ++ SET_TIMEOUT(gpu_reset_ticks_ss); ++ SET_TIMEOUT(gpu_reset_ticks_cl); ++ SET_TIMEOUT(gpu_reset_ticks_dumping); ++ ++#undef SET_TIMEOUT ++ ++ js_data->scheduling_period_ns = new_scheduling_period_ns; ++ ++ kbase_js_set_timeouts(kbdev); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&js_data->runpool_mutex); ++ ++ dev_dbg(kbdev->dev, "JS scheduling period: %dms\n", ++ js_scheduling_period); ++ ++ return count; ++} ++ ++/** ++ * show_js_scheduling_period - Show callback for the js_scheduling_period sysfs ++ * entry. ++ * @dev: The device this sysfs file is for. ++ * @attr: The attributes of the sysfs file. ++ * @buf: The output buffer to receive the GPU information. ++ * ++ * This function is called to get the current period used for the JS scheduling ++ * period. ++ * ++ * Return: The number of bytes output to @buf. ++ */ ++static ssize_t show_js_scheduling_period(struct device *dev, ++ struct device_attribute *attr, char * const buf) ++{ ++ struct kbase_device *kbdev; ++ u32 period; ++ ssize_t ret; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ period = kbdev->js_data.scheduling_period_ns; ++ ++ ret = scnprintf(buf, PAGE_SIZE, "%d\n", ++ period / 1000000); ++ ++ return ret; ++} ++ ++static DEVICE_ATTR(js_scheduling_period, S_IRUGO | S_IWUSR, ++ show_js_scheduling_period, set_js_scheduling_period); ++ ++#if !MALI_CUSTOMER_RELEASE ++/** ++ * set_force_replay - Store callback for the force_replay sysfs file. ++ * ++ * @dev: The device with sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The value written to the sysfs file ++ * @count: The number of bytes written to the sysfs file ++ * ++ * Return: @count if the function succeeded. An error code on failure. ++ */ ++static ssize_t set_force_replay(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) ++{ ++ struct kbase_device *kbdev; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ if (!strncmp("limit=", buf, MIN(6, count))) { ++ int force_replay_limit; ++ int items = sscanf(buf, "limit=%u", &force_replay_limit); ++ ++ if (items == 1) { ++ kbdev->force_replay_random = false; ++ kbdev->force_replay_limit = force_replay_limit; ++ kbdev->force_replay_count = 0; ++ ++ return count; ++ } ++ } else if (!strncmp("random_limit", buf, MIN(12, count))) { ++ kbdev->force_replay_random = true; ++ kbdev->force_replay_count = 0; ++ ++ return count; ++ } else if (!strncmp("norandom_limit", buf, MIN(14, count))) { ++ kbdev->force_replay_random = false; ++ kbdev->force_replay_limit = KBASEP_FORCE_REPLAY_DISABLED; ++ kbdev->force_replay_count = 0; ++ ++ return count; ++ } else if (!strncmp("core_req=", buf, MIN(9, count))) { ++ unsigned int core_req; ++ int items = sscanf(buf, "core_req=%x", &core_req); ++ ++ if (items == 1) { ++ kbdev->force_replay_core_req = (base_jd_core_req)core_req; ++ ++ return count; ++ } ++ } ++ dev_err(kbdev->dev, "Couldn't process force_replay write operation.\nPossible settings: limit=, random_limit, norandom_limit, core_req=\n"); ++ return -EINVAL; ++} ++ ++/** ++ * show_force_replay - Show callback for the force_replay sysfs file. ++ * ++ * This function is called to get the contents of the force_replay sysfs ++ * file. It returns the last set value written to the force_replay sysfs file. ++ * If the file didn't get written yet, the values will be 0. ++ * ++ * @dev: The device this sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The output buffer for the sysfs file contents ++ * ++ * Return: The number of bytes output to @buf. ++ */ ++static ssize_t show_force_replay(struct device *dev, ++ struct device_attribute *attr, char * const buf) ++{ ++ struct kbase_device *kbdev; ++ ssize_t ret; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ if (kbdev->force_replay_random) ++ ret = scnprintf(buf, PAGE_SIZE, ++ "limit=0\nrandom_limit\ncore_req=%x\n", ++ kbdev->force_replay_core_req); ++ else ++ ret = scnprintf(buf, PAGE_SIZE, ++ "limit=%u\nnorandom_limit\ncore_req=%x\n", ++ kbdev->force_replay_limit, ++ kbdev->force_replay_core_req); ++ ++ if (ret >= PAGE_SIZE) { ++ buf[PAGE_SIZE - 2] = '\n'; ++ buf[PAGE_SIZE - 1] = '\0'; ++ ret = PAGE_SIZE - 1; ++ } ++ ++ return ret; ++} ++ ++/* ++ * The sysfs file force_replay. ++ */ ++static DEVICE_ATTR(force_replay, S_IRUGO | S_IWUSR, show_force_replay, ++ set_force_replay); ++#endif /* !MALI_CUSTOMER_RELEASE */ ++ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++static ssize_t set_js_softstop_always(struct device *dev, ++ struct device_attribute *attr, const char *buf, size_t count) ++{ ++ struct kbase_device *kbdev; ++ int ret; ++ int softstop_always; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ ret = kstrtoint(buf, 0, &softstop_always); ++ if (ret || ((softstop_always != 0) && (softstop_always != 1))) { ++ dev_err(kbdev->dev, "Couldn't process js_softstop_always write operation.\n" ++ "Use format \n"); ++ return -EINVAL; ++ } ++ ++ kbdev->js_data.softstop_always = (bool) softstop_always; ++ dev_dbg(kbdev->dev, "Support for softstop on a single context: %s\n", ++ (kbdev->js_data.softstop_always) ? ++ "Enabled" : "Disabled"); ++ return count; ++} ++ ++static ssize_t show_js_softstop_always(struct device *dev, ++ struct device_attribute *attr, char * const buf) ++{ ++ struct kbase_device *kbdev; ++ ssize_t ret; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ ret = scnprintf(buf, PAGE_SIZE, "%d\n", kbdev->js_data.softstop_always); ++ ++ if (ret >= PAGE_SIZE) { ++ buf[PAGE_SIZE - 2] = '\n'; ++ buf[PAGE_SIZE - 1] = '\0'; ++ ret = PAGE_SIZE - 1; ++ } ++ ++ return ret; ++} ++ ++/* ++ * By default, soft-stops are disabled when only a single context is present. ++ * The ability to enable soft-stop when only a single context is present can be ++ * used for debug and unit-testing purposes. ++ * (see CL t6xx_stress_1 unit-test as an example whereby this feature is used.) ++ */ ++static DEVICE_ATTR(js_softstop_always, S_IRUGO | S_IWUSR, show_js_softstop_always, set_js_softstop_always); ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++typedef void (kbasep_debug_command_func) (struct kbase_device *); ++ ++enum kbasep_debug_command_code { ++ KBASEP_DEBUG_COMMAND_DUMPTRACE, ++ ++ /* This must be the last enum */ ++ KBASEP_DEBUG_COMMAND_COUNT ++}; ++ ++struct kbasep_debug_command { ++ char *str; ++ kbasep_debug_command_func *func; ++}; ++ ++/* Debug commands supported by the driver */ ++static const struct kbasep_debug_command debug_commands[] = { ++ { ++ .str = "dumptrace", ++ .func = &kbasep_trace_dump, ++ } ++}; ++ ++/** ++ * show_debug - Show callback for the debug_command sysfs file. ++ * ++ * This function is called to get the contents of the debug_command sysfs ++ * file. This is a list of the available debug commands, separated by newlines. ++ * ++ * @dev: The device this sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The output buffer for the sysfs file contents ++ * ++ * Return: The number of bytes output to @buf. ++ */ ++static ssize_t show_debug(struct device *dev, struct device_attribute *attr, char * const buf) ++{ ++ struct kbase_device *kbdev; ++ int i; ++ ssize_t ret = 0; ++ ++ kbdev = to_kbase_device(dev); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ for (i = 0; i < KBASEP_DEBUG_COMMAND_COUNT && ret < PAGE_SIZE; i++) ++ ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s\n", debug_commands[i].str); ++ ++ if (ret >= PAGE_SIZE) { ++ buf[PAGE_SIZE - 2] = '\n'; ++ buf[PAGE_SIZE - 1] = '\0'; ++ ret = PAGE_SIZE - 1; ++ } ++ ++ return ret; ++} ++ ++/** ++ * issue_debug - Store callback for the debug_command sysfs file. ++ * ++ * This function is called when the debug_command sysfs file is written to. ++ * It matches the requested command against the available commands, and if ++ * a matching command is found calls the associated function from ++ * @debug_commands to issue the command. ++ * ++ * @dev: The device with sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The value written to the sysfs file ++ * @count: The number of bytes written to the sysfs file ++ * ++ * Return: @count if the function succeeded. An error code on failure. ++ */ ++static ssize_t issue_debug(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) ++{ ++ struct kbase_device *kbdev; ++ int i; ++ ++ kbdev = to_kbase_device(dev); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ for (i = 0; i < KBASEP_DEBUG_COMMAND_COUNT; i++) { ++ if (sysfs_streq(debug_commands[i].str, buf)) { ++ debug_commands[i].func(kbdev); ++ return count; ++ } ++ } ++ ++ /* Debug Command not found */ ++ dev_err(dev, "debug_command: command not known\n"); ++ return -EINVAL; ++} ++ ++/* The sysfs file debug_command. ++ * ++ * This is used to issue general debug commands to the device driver. ++ * Reading it will produce a list of debug commands, separated by newlines. ++ * Writing to it with one of those commands will issue said command. ++ */ ++static DEVICE_ATTR(debug_command, S_IRUGO | S_IWUSR, show_debug, issue_debug); ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++ ++/** ++ * kbase_show_gpuinfo - Show callback for the gpuinfo sysfs entry. ++ * @dev: The device this sysfs file is for. ++ * @attr: The attributes of the sysfs file. ++ * @buf: The output buffer to receive the GPU information. ++ * ++ * This function is called to get a description of the present Mali ++ * GPU via the gpuinfo sysfs entry. This includes the GPU family, the ++ * number of cores, the hardware version and the raw product id. For ++ * example ++ * ++ * Mali-T60x MP4 r0p0 0x6956 ++ * ++ * Return: The number of bytes output to @buf. ++ */ ++static ssize_t kbase_show_gpuinfo(struct device *dev, ++ struct device_attribute *attr, char *buf) ++{ ++ static const struct gpu_product_id_name { ++ unsigned id; ++ char *name; ++ } gpu_product_id_names[] = { ++ { .id = GPU_ID_PI_T60X, .name = "Mali-T60x" }, ++ { .id = GPU_ID_PI_T62X, .name = "Mali-T62x" }, ++ { .id = GPU_ID_PI_T72X, .name = "Mali-T72x" }, ++ { .id = GPU_ID_PI_T76X, .name = "Mali-T76x" }, ++ { .id = GPU_ID_PI_T82X, .name = "Mali-T82x" }, ++ { .id = GPU_ID_PI_T83X, .name = "Mali-T83x" }, ++ { .id = GPU_ID_PI_T86X, .name = "Mali-T86x" }, ++ { .id = GPU_ID_PI_TFRX, .name = "Mali-T88x" }, ++ { .id = GPU_ID2_PRODUCT_TMIX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, ++ .name = "Mali-G71" }, ++ { .id = GPU_ID2_PRODUCT_THEX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, ++ .name = "Mali-G72" }, ++ { .id = GPU_ID2_PRODUCT_TSIX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, ++ .name = "Mali-G51" }, ++ { .id = GPU_ID2_PRODUCT_TDVX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, ++ .name = "Mali-G31" }, ++ }; ++ const char *product_name = "(Unknown Mali GPU)"; ++ struct kbase_device *kbdev; ++ u32 gpu_id; ++ unsigned product_id, product_id_mask; ++ unsigned i; ++ bool is_new_format; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; ++ product_id = gpu_id >> GPU_ID_VERSION_PRODUCT_ID_SHIFT; ++ is_new_format = GPU_ID_IS_NEW_FORMAT(product_id); ++ product_id_mask = ++ (is_new_format ? ++ GPU_ID2_PRODUCT_MODEL : ++ GPU_ID_VERSION_PRODUCT_ID) >> ++ GPU_ID_VERSION_PRODUCT_ID_SHIFT; ++ ++ for (i = 0; i < ARRAY_SIZE(gpu_product_id_names); ++i) { ++ const struct gpu_product_id_name *p = &gpu_product_id_names[i]; ++ ++ if ((GPU_ID_IS_NEW_FORMAT(p->id) == is_new_format) && ++ (p->id & product_id_mask) == ++ (product_id & product_id_mask)) { ++ product_name = p->name; ++ break; ++ } ++ } ++ ++ return scnprintf(buf, PAGE_SIZE, "%s %d cores 2EE r%dp%d 0x%04X\n", ++ product_name, kbdev->gpu_props.num_cores, ++ (gpu_id & GPU_ID_VERSION_MAJOR) >> GPU_ID_VERSION_MAJOR_SHIFT, ++ (gpu_id & GPU_ID_VERSION_MINOR) >> GPU_ID_VERSION_MINOR_SHIFT, ++ product_id); ++} ++static DEVICE_ATTR(gpuinfo, S_IRUGO, kbase_show_gpuinfo, NULL); ++ ++/** ++ * set_dvfs_period - Store callback for the dvfs_period sysfs file. ++ * @dev: The device with sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The value written to the sysfs file ++ * @count: The number of bytes written to the sysfs file ++ * ++ * This function is called when the dvfs_period sysfs file is written to. It ++ * checks the data written, and if valid updates the DVFS period variable, ++ * ++ * Return: @count if the function succeeded. An error code on failure. ++ */ ++static ssize_t set_dvfs_period(struct device *dev, ++ struct device_attribute *attr, const char *buf, size_t count) ++{ ++ struct kbase_device *kbdev; ++ int ret; ++ int dvfs_period; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ ret = kstrtoint(buf, 0, &dvfs_period); ++ if (ret || dvfs_period <= 0) { ++ dev_err(kbdev->dev, "Couldn't process dvfs_period write operation.\n" ++ "Use format \n"); ++ return -EINVAL; ++ } ++ ++ kbdev->pm.dvfs_period = dvfs_period; ++ dev_dbg(kbdev->dev, "DVFS period: %dms\n", dvfs_period); ++ ++ return count; ++} ++ ++/** ++ * show_dvfs_period - Show callback for the dvfs_period sysfs entry. ++ * @dev: The device this sysfs file is for. ++ * @attr: The attributes of the sysfs file. ++ * @buf: The output buffer to receive the GPU information. ++ * ++ * This function is called to get the current period used for the DVFS sample ++ * timer. ++ * ++ * Return: The number of bytes output to @buf. ++ */ ++static ssize_t show_dvfs_period(struct device *dev, ++ struct device_attribute *attr, char * const buf) ++{ ++ struct kbase_device *kbdev; ++ ssize_t ret; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ ret = scnprintf(buf, PAGE_SIZE, "%d\n", kbdev->pm.dvfs_period); ++ ++ return ret; ++} ++ ++static DEVICE_ATTR(dvfs_period, S_IRUGO | S_IWUSR, show_dvfs_period, ++ set_dvfs_period); ++ ++/** ++ * set_pm_poweroff - Store callback for the pm_poweroff sysfs file. ++ * @dev: The device with sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The value written to the sysfs file ++ * @count: The number of bytes written to the sysfs file ++ * ++ * This function is called when the pm_poweroff sysfs file is written to. ++ * ++ * This file contains three values separated by whitespace. The values ++ * are gpu_poweroff_time (the period of the poweroff timer, in ns), ++ * poweroff_shader_ticks (the number of poweroff timer ticks before an idle ++ * shader is powered off), and poweroff_gpu_ticks (the number of poweroff timer ++ * ticks before the GPU is powered off), in that order. ++ * ++ * Return: @count if the function succeeded. An error code on failure. ++ */ ++static ssize_t set_pm_poweroff(struct device *dev, ++ struct device_attribute *attr, const char *buf, size_t count) ++{ ++ struct kbase_device *kbdev; ++ int items; ++ s64 gpu_poweroff_time; ++ int poweroff_shader_ticks, poweroff_gpu_ticks; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ items = sscanf(buf, "%llu %u %u", &gpu_poweroff_time, ++ &poweroff_shader_ticks, ++ &poweroff_gpu_ticks); ++ if (items != 3) { ++ dev_err(kbdev->dev, "Couldn't process pm_poweroff write operation.\n" ++ "Use format \n"); ++ return -EINVAL; ++ } ++ ++ kbdev->pm.gpu_poweroff_time = HR_TIMER_DELAY_NSEC(gpu_poweroff_time); ++ kbdev->pm.poweroff_shader_ticks = poweroff_shader_ticks; ++ kbdev->pm.poweroff_gpu_ticks = poweroff_gpu_ticks; ++ ++ return count; ++} ++ ++/** ++ * show_pm_poweroff - Show callback for the pm_poweroff sysfs entry. ++ * @dev: The device this sysfs file is for. ++ * @attr: The attributes of the sysfs file. ++ * @buf: The output buffer to receive the GPU information. ++ * ++ * This function is called to get the current period used for the DVFS sample ++ * timer. ++ * ++ * Return: The number of bytes output to @buf. ++ */ ++static ssize_t show_pm_poweroff(struct device *dev, ++ struct device_attribute *attr, char * const buf) ++{ ++ struct kbase_device *kbdev; ++ ssize_t ret; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ ret = scnprintf(buf, PAGE_SIZE, "%llu %u %u\n", ++ ktime_to_ns(kbdev->pm.gpu_poweroff_time), ++ kbdev->pm.poweroff_shader_ticks, ++ kbdev->pm.poweroff_gpu_ticks); ++ ++ return ret; ++} ++ ++static DEVICE_ATTR(pm_poweroff, S_IRUGO | S_IWUSR, show_pm_poweroff, ++ set_pm_poweroff); ++ ++/** ++ * set_reset_timeout - Store callback for the reset_timeout sysfs file. ++ * @dev: The device with sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The value written to the sysfs file ++ * @count: The number of bytes written to the sysfs file ++ * ++ * This function is called when the reset_timeout sysfs file is written to. It ++ * checks the data written, and if valid updates the reset timeout. ++ * ++ * Return: @count if the function succeeded. An error code on failure. ++ */ ++static ssize_t set_reset_timeout(struct device *dev, ++ struct device_attribute *attr, const char *buf, size_t count) ++{ ++ struct kbase_device *kbdev; ++ int ret; ++ int reset_timeout; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ ret = kstrtoint(buf, 0, &reset_timeout); ++ if (ret || reset_timeout <= 0) { ++ dev_err(kbdev->dev, "Couldn't process reset_timeout write operation.\n" ++ "Use format \n"); ++ return -EINVAL; ++ } ++ ++ kbdev->reset_timeout_ms = reset_timeout; ++ dev_dbg(kbdev->dev, "Reset timeout: %dms\n", reset_timeout); ++ ++ return count; ++} ++ ++/** ++ * show_reset_timeout - Show callback for the reset_timeout sysfs entry. ++ * @dev: The device this sysfs file is for. ++ * @attr: The attributes of the sysfs file. ++ * @buf: The output buffer to receive the GPU information. ++ * ++ * This function is called to get the current reset timeout. ++ * ++ * Return: The number of bytes output to @buf. ++ */ ++static ssize_t show_reset_timeout(struct device *dev, ++ struct device_attribute *attr, char * const buf) ++{ ++ struct kbase_device *kbdev; ++ ssize_t ret; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ ret = scnprintf(buf, PAGE_SIZE, "%d\n", kbdev->reset_timeout_ms); ++ ++ return ret; ++} ++ ++static DEVICE_ATTR(reset_timeout, S_IRUGO | S_IWUSR, show_reset_timeout, ++ set_reset_timeout); ++ ++ ++ ++static ssize_t show_mem_pool_size(struct device *dev, ++ struct device_attribute *attr, char * const buf) ++{ ++ struct kbase_device *kbdev; ++ ssize_t ret; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ ret = scnprintf(buf, PAGE_SIZE, "%zu\n", ++ kbase_mem_pool_size(&kbdev->mem_pool)); ++ ++ return ret; ++} ++ ++static ssize_t set_mem_pool_size(struct device *dev, ++ struct device_attribute *attr, const char *buf, size_t count) ++{ ++ struct kbase_device *kbdev; ++ size_t new_size; ++ int err; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ err = kstrtoul(buf, 0, (unsigned long *)&new_size); ++ if (err) ++ return err; ++ ++ kbase_mem_pool_trim(&kbdev->mem_pool, new_size); ++ ++ return count; ++} ++ ++static DEVICE_ATTR(mem_pool_size, S_IRUGO | S_IWUSR, show_mem_pool_size, ++ set_mem_pool_size); ++ ++static ssize_t show_mem_pool_max_size(struct device *dev, ++ struct device_attribute *attr, char * const buf) ++{ ++ struct kbase_device *kbdev; ++ ssize_t ret; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ ret = scnprintf(buf, PAGE_SIZE, "%zu\n", ++ kbase_mem_pool_max_size(&kbdev->mem_pool)); ++ ++ return ret; ++} ++ ++static ssize_t set_mem_pool_max_size(struct device *dev, ++ struct device_attribute *attr, const char *buf, size_t count) ++{ ++ struct kbase_device *kbdev; ++ size_t new_max_size; ++ int err; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ err = kstrtoul(buf, 0, (unsigned long *)&new_max_size); ++ if (err) ++ return -EINVAL; ++ ++ kbase_mem_pool_set_max_size(&kbdev->mem_pool, new_max_size); ++ ++ return count; ++} ++ ++static DEVICE_ATTR(mem_pool_max_size, S_IRUGO | S_IWUSR, show_mem_pool_max_size, ++ set_mem_pool_max_size); ++ ++/** ++ * show_lp_mem_pool_size - Show size of the large memory pages pool. ++ * @dev: The device this sysfs file is for. ++ * @attr: The attributes of the sysfs file. ++ * @buf: The output buffer to receive the pool size. ++ * ++ * This function is called to get the number of large memory pages which currently populate the kbdev pool. ++ * ++ * Return: The number of bytes output to @buf. ++ */ ++static ssize_t show_lp_mem_pool_size(struct device *dev, ++ struct device_attribute *attr, char * const buf) ++{ ++ struct kbase_device *kbdev; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ return scnprintf(buf, PAGE_SIZE, "%zu\n", kbase_mem_pool_size(&kbdev->lp_mem_pool)); ++} ++ ++/** ++ * set_lp_mem_pool_size - Set size of the large memory pages pool. ++ * @dev: The device this sysfs file is for. ++ * @attr: The attributes of the sysfs file. ++ * @buf: The value written to the sysfs file. ++ * @count: The number of bytes written to the sysfs file. ++ * ++ * This function is called to set the number of large memory pages which should populate the kbdev pool. ++ * This may cause existing pages to be removed from the pool, or new pages to be created and then added to the pool. ++ * ++ * Return: @count if the function succeeded. An error code on failure. ++ */ ++static ssize_t set_lp_mem_pool_size(struct device *dev, ++ struct device_attribute *attr, const char *buf, size_t count) ++{ ++ struct kbase_device *kbdev; ++ unsigned long new_size; ++ int err; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ err = kstrtoul(buf, 0, &new_size); ++ if (err) ++ return err; ++ ++ kbase_mem_pool_trim(&kbdev->lp_mem_pool, new_size); ++ ++ return count; ++} ++ ++static DEVICE_ATTR(lp_mem_pool_size, S_IRUGO | S_IWUSR, show_lp_mem_pool_size, ++ set_lp_mem_pool_size); ++ ++/** ++ * show_lp_mem_pool_max_size - Show maximum size of the large memory pages pool. ++ * @dev: The device this sysfs file is for. ++ * @attr: The attributes of the sysfs file. ++ * @buf: The output buffer to receive the pool size. ++ * ++ * This function is called to get the maximum number of large memory pages that the kbdev pool can possibly contain. ++ * ++ * Return: The number of bytes output to @buf. ++ */ ++static ssize_t show_lp_mem_pool_max_size(struct device *dev, ++ struct device_attribute *attr, char * const buf) ++{ ++ struct kbase_device *kbdev; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ return scnprintf(buf, PAGE_SIZE, "%zu\n", kbase_mem_pool_max_size(&kbdev->lp_mem_pool)); ++} ++ ++/** ++ * set_lp_mem_pool_max_size - Set maximum size of the large memory pages pool. ++ * @dev: The device this sysfs file is for. ++ * @attr: The attributes of the sysfs file. ++ * @buf: The value written to the sysfs file. ++ * @count: The number of bytes written to the sysfs file. ++ * ++ * This function is called to set the maximum number of large memory pages that the kbdev pool can possibly contain. ++ * ++ * Return: @count if the function succeeded. An error code on failure. ++ */ ++static ssize_t set_lp_mem_pool_max_size(struct device *dev, ++ struct device_attribute *attr, const char *buf, size_t count) ++{ ++ struct kbase_device *kbdev; ++ unsigned long new_max_size; ++ int err; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ err = kstrtoul(buf, 0, &new_max_size); ++ if (err) ++ return -EINVAL; ++ ++ kbase_mem_pool_set_max_size(&kbdev->lp_mem_pool, new_max_size); ++ ++ return count; ++} ++ ++static DEVICE_ATTR(lp_mem_pool_max_size, S_IRUGO | S_IWUSR, show_lp_mem_pool_max_size, ++ set_lp_mem_pool_max_size); ++ ++#ifdef CONFIG_DEBUG_FS ++ ++/* Number of entries in serialize_jobs_settings[] */ ++#define NR_SERIALIZE_JOBS_SETTINGS 5 ++/* Maximum string length in serialize_jobs_settings[].name */ ++#define MAX_SERIALIZE_JOBS_NAME_LEN 16 ++ ++static struct ++{ ++ char *name; ++ u8 setting; ++} serialize_jobs_settings[NR_SERIALIZE_JOBS_SETTINGS] = { ++ {"none", 0}, ++ {"intra-slot", KBASE_SERIALIZE_INTRA_SLOT}, ++ {"inter-slot", KBASE_SERIALIZE_INTER_SLOT}, ++ {"full", KBASE_SERIALIZE_INTRA_SLOT | KBASE_SERIALIZE_INTER_SLOT}, ++ {"full-reset", KBASE_SERIALIZE_INTRA_SLOT | KBASE_SERIALIZE_INTER_SLOT | ++ KBASE_SERIALIZE_RESET} ++}; ++ ++/** ++ * kbasep_serialize_jobs_seq_show - Show callback for the serialize_jobs debugfs ++ * file ++ * @sfile: seq_file pointer ++ * @data: Private callback data ++ * ++ * This function is called to get the contents of the serialize_jobs debugfs ++ * file. This is a list of the available settings with the currently active one ++ * surrounded by square brackets. ++ * ++ * Return: 0 on success, or an error code on error ++ */ ++static int kbasep_serialize_jobs_seq_show(struct seq_file *sfile, void *data) ++{ ++ struct kbase_device *kbdev = sfile->private; ++ int i; ++ ++ CSTD_UNUSED(data); ++ ++ for (i = 0; i < NR_SERIALIZE_JOBS_SETTINGS; i++) { ++ if (kbdev->serialize_jobs == serialize_jobs_settings[i].setting) ++ seq_printf(sfile, "[%s] ", ++ serialize_jobs_settings[i].name); ++ else ++ seq_printf(sfile, "%s ", ++ serialize_jobs_settings[i].name); ++ } ++ ++ seq_puts(sfile, "\n"); ++ ++ return 0; ++} ++ ++/** ++ * kbasep_serialize_jobs_debugfs_write - Store callback for the serialize_jobs ++ * debugfs file. ++ * @file: File pointer ++ * @ubuf: User buffer containing data to store ++ * @count: Number of bytes in user buffer ++ * @ppos: File position ++ * ++ * This function is called when the serialize_jobs debugfs file is written to. ++ * It matches the requested setting against the available settings and if a ++ * matching setting is found updates kbdev->serialize_jobs. ++ * ++ * Return: @count if the function succeeded. An error code on failure. ++ */ ++static ssize_t kbasep_serialize_jobs_debugfs_write(struct file *file, ++ const char __user *ubuf, size_t count, loff_t *ppos) ++{ ++ struct seq_file *s = file->private_data; ++ struct kbase_device *kbdev = s->private; ++ char buf[MAX_SERIALIZE_JOBS_NAME_LEN]; ++ int i; ++ bool valid = false; ++ ++ CSTD_UNUSED(ppos); ++ ++ count = min_t(size_t, sizeof(buf) - 1, count); ++ if (copy_from_user(buf, ubuf, count)) ++ return -EFAULT; ++ ++ buf[count] = 0; ++ ++ for (i = 0; i < NR_SERIALIZE_JOBS_SETTINGS; i++) { ++ if (sysfs_streq(serialize_jobs_settings[i].name, buf)) { ++ kbdev->serialize_jobs = ++ serialize_jobs_settings[i].setting; ++ valid = true; ++ break; ++ } ++ } ++ ++ if (!valid) { ++ dev_err(kbdev->dev, "serialize_jobs: invalid setting\n"); ++ return -EINVAL; ++ } ++ ++ return count; ++} ++ ++/** ++ * kbasep_serialize_jobs_debugfs_open - Open callback for the serialize_jobs ++ * debugfs file ++ * @in: inode pointer ++ * @file: file pointer ++ * ++ * Return: Zero on success, error code on failure ++ */ ++static int kbasep_serialize_jobs_debugfs_open(struct inode *in, ++ struct file *file) ++{ ++ return single_open(file, kbasep_serialize_jobs_seq_show, in->i_private); ++} ++ ++static const struct file_operations kbasep_serialize_jobs_debugfs_fops = { ++ .open = kbasep_serialize_jobs_debugfs_open, ++ .read = seq_read, ++ .write = kbasep_serialize_jobs_debugfs_write, ++ .llseek = seq_lseek, ++ .release = single_release, ++}; ++ ++#endif /* CONFIG_DEBUG_FS */ ++ ++static int kbasep_protected_mode_init(struct kbase_device *kbdev) ++{ ++#ifdef CONFIG_OF ++ struct device_node *protected_node; ++ struct platform_device *pdev; ++ struct protected_mode_device *protected_dev; ++#endif ++ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_MODE)) { ++ /* Use native protected ops */ ++ kbdev->protected_dev = kzalloc(sizeof(*kbdev->protected_dev), ++ GFP_KERNEL); ++ if (!kbdev->protected_dev) ++ return -ENOMEM; ++ kbdev->protected_dev->data = kbdev; ++ kbdev->protected_ops = &kbase_native_protected_ops; ++ kbdev->protected_mode_support = true; ++ return 0; ++ } ++ ++ kbdev->protected_mode_support = false; ++ ++#ifdef CONFIG_OF ++ protected_node = of_parse_phandle(kbdev->dev->of_node, ++ "protected-mode-switcher", 0); ++ ++ if (!protected_node) ++ protected_node = of_parse_phandle(kbdev->dev->of_node, ++ "secure-mode-switcher", 0); ++ ++ if (!protected_node) { ++ /* If protected_node cannot be looked up then we assume ++ * protected mode is not supported on this platform. */ ++ dev_info(kbdev->dev, "Protected mode not available\n"); ++ return 0; ++ } ++ ++ pdev = of_find_device_by_node(protected_node); ++ if (!pdev) ++ return -EINVAL; ++ ++ protected_dev = platform_get_drvdata(pdev); ++ if (!protected_dev) ++ return -EPROBE_DEFER; ++ ++ kbdev->protected_ops = &protected_dev->ops; ++ kbdev->protected_dev = protected_dev; ++ ++ if (kbdev->protected_ops) { ++ int err; ++ ++ /* Make sure protected mode is disabled on startup */ ++ mutex_lock(&kbdev->pm.lock); ++ err = kbdev->protected_ops->protected_mode_disable( ++ kbdev->protected_dev); ++ mutex_unlock(&kbdev->pm.lock); ++ ++ /* protected_mode_disable() returns -EINVAL if not supported */ ++ kbdev->protected_mode_support = (err != -EINVAL); ++ } ++#endif ++ return 0; ++} ++ ++static void kbasep_protected_mode_term(struct kbase_device *kbdev) ++{ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_MODE)) ++ kfree(kbdev->protected_dev); ++} ++ ++#ifdef CONFIG_MALI_BIFROST_NO_MALI ++static int kbase_common_reg_map(struct kbase_device *kbdev) ++{ ++ return 0; ++} ++static void kbase_common_reg_unmap(struct kbase_device * const kbdev) ++{ ++} ++#else /* CONFIG_MALI_BIFROST_NO_MALI */ ++static int kbase_common_reg_map(struct kbase_device *kbdev) ++{ ++ int err = 0; ++ ++ if (!request_mem_region(kbdev->reg_start, kbdev->reg_size, dev_name(kbdev->dev))) { ++ dev_err(kbdev->dev, "Register window unavailable\n"); ++ err = -EIO; ++ goto out_region; ++ } ++ ++ kbdev->reg = ioremap(kbdev->reg_start, kbdev->reg_size); ++ if (!kbdev->reg) { ++ dev_err(kbdev->dev, "Can't remap register window\n"); ++ err = -EINVAL; ++ goto out_ioremap; ++ } ++ ++ return err; ++ ++ out_ioremap: ++ release_mem_region(kbdev->reg_start, kbdev->reg_size); ++ out_region: ++ return err; ++} ++ ++static void kbase_common_reg_unmap(struct kbase_device * const kbdev) ++{ ++ if (kbdev->reg) { ++ iounmap(kbdev->reg); ++ release_mem_region(kbdev->reg_start, kbdev->reg_size); ++ kbdev->reg = NULL; ++ kbdev->reg_start = 0; ++ kbdev->reg_size = 0; ++ } ++} ++#endif /* CONFIG_MALI_BIFROST_NO_MALI */ ++ ++static int registers_map(struct kbase_device * const kbdev) ++{ ++ ++ /* the first memory resource is the physical address of the GPU ++ * registers */ ++ struct platform_device *pdev = to_platform_device(kbdev->dev); ++ struct resource *reg_res; ++ int err; ++ ++ reg_res = platform_get_resource(pdev, IORESOURCE_MEM, 0); ++ if (!reg_res) { ++ dev_err(kbdev->dev, "Invalid register resource\n"); ++ return -ENOENT; ++ } ++ ++ kbdev->reg_start = reg_res->start; ++ kbdev->reg_size = resource_size(reg_res); ++ ++ err = kbase_common_reg_map(kbdev); ++ if (err) { ++ dev_err(kbdev->dev, "Failed to map registers\n"); ++ return err; ++ } ++ ++ return 0; ++} ++ ++static void registers_unmap(struct kbase_device *kbdev) ++{ ++ kbase_common_reg_unmap(kbdev); ++} ++ ++static int power_control_init(struct platform_device *pdev) ++{ ++ struct kbase_device *kbdev = to_kbase_device(&pdev->dev); ++ int err = 0; ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0)) && defined(CONFIG_OF) \ ++ && defined(CONFIG_REGULATOR) ++ kbdev->regulator = regulator_get_optional(kbdev->dev, "mali"); ++ if (IS_ERR_OR_NULL(kbdev->regulator)) { ++ err = PTR_ERR(kbdev->regulator); ++ kbdev->regulator = NULL; ++ if (err == -EPROBE_DEFER) { ++ dev_err(&pdev->dev, "Failed to get regulator\n"); ++ return err; ++ } ++ dev_info(kbdev->dev, ++ "Continuing without Mali regulator control\n"); ++ /* Allow probe to continue without regulator */ ++ } ++#endif /* LINUX_VERSION_CODE >= 3, 12, 0 */ ++ ++ kbdev->clock = clk_get(kbdev->dev, "clk_mali"); ++ if (IS_ERR_OR_NULL(kbdev->clock)) { ++ err = PTR_ERR(kbdev->clock); ++ kbdev->clock = NULL; ++ if (err == -EPROBE_DEFER) { ++ dev_err(&pdev->dev, "Failed to get clock\n"); ++ goto fail; ++ } ++ dev_info(kbdev->dev, "Continuing without Mali clock control\n"); ++ /* Allow probe to continue without clock. */ ++ } else { ++ err = clk_prepare(kbdev->clock); ++ if (err) { ++ dev_err(kbdev->dev, ++ "Failed to prepare and enable clock (%d)\n", ++ err); ++ goto fail; ++ } ++ } ++ ++ err = kbase_platform_rk_init_opp_table(kbdev); ++ if (err) ++ dev_err(kbdev->dev, "Failed to init_opp_table (%d)\n", err); ++ ++ return 0; ++ ++fail: ++ ++if (kbdev->clock != NULL) { ++ clk_put(kbdev->clock); ++ kbdev->clock = NULL; ++} ++ ++#ifdef CONFIG_REGULATOR ++ if (NULL != kbdev->regulator) { ++ regulator_put(kbdev->regulator); ++ kbdev->regulator = NULL; ++ } ++#endif ++ ++ return err; ++} ++ ++static void power_control_term(struct kbase_device *kbdev) ++{ ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0)) || \ ++ defined(LSK_OPPV2_BACKPORT) ++ dev_pm_opp_of_remove_table(kbdev->dev); ++#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0)) ++ of_free_opp_table(kbdev->dev); ++#endif ++ ++ if (kbdev->clock) { ++ clk_unprepare(kbdev->clock); ++ clk_put(kbdev->clock); ++ kbdev->clock = NULL; ++ } ++ ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0)) && defined(CONFIG_OF) \ ++ && defined(CONFIG_REGULATOR) ++ if (kbdev->regulator) { ++ regulator_put(kbdev->regulator); ++ kbdev->regulator = NULL; ++ } ++#endif /* LINUX_VERSION_CODE >= 3, 12, 0 */ ++} ++ ++#ifdef CONFIG_DEBUG_FS ++ ++#if KBASE_GPU_RESET_EN ++#include ++ ++static void trigger_quirks_reload(struct kbase_device *kbdev) ++{ ++ kbase_pm_context_active(kbdev); ++ if (kbase_prepare_to_reset_gpu(kbdev)) ++ kbase_reset_gpu(kbdev); ++ kbase_pm_context_idle(kbdev); ++} ++ ++#define MAKE_QUIRK_ACCESSORS(type) \ ++static int type##_quirks_set(void *data, u64 val) \ ++{ \ ++ struct kbase_device *kbdev; \ ++ kbdev = (struct kbase_device *)data; \ ++ kbdev->hw_quirks_##type = (u32)val; \ ++ trigger_quirks_reload(kbdev); \ ++ return 0;\ ++} \ ++\ ++static int type##_quirks_get(void *data, u64 *val) \ ++{ \ ++ struct kbase_device *kbdev;\ ++ kbdev = (struct kbase_device *)data;\ ++ *val = kbdev->hw_quirks_##type;\ ++ return 0;\ ++} \ ++DEFINE_SIMPLE_ATTRIBUTE(fops_##type##_quirks, type##_quirks_get,\ ++ type##_quirks_set, "%llu\n") ++ ++MAKE_QUIRK_ACCESSORS(sc); ++MAKE_QUIRK_ACCESSORS(tiler); ++MAKE_QUIRK_ACCESSORS(mmu); ++MAKE_QUIRK_ACCESSORS(jm); ++ ++#endif /* KBASE_GPU_RESET_EN */ ++ ++/** ++ * debugfs_protected_debug_mode_read - "protected_debug_mode" debugfs read ++ * @file: File object to read is for ++ * @buf: User buffer to populate with data ++ * @len: Length of user buffer ++ * @ppos: Offset within file object ++ * ++ * Retrieves the current status of protected debug mode ++ * (0 = disabled, 1 = enabled) ++ * ++ * Return: Number of bytes added to user buffer ++ */ ++static ssize_t debugfs_protected_debug_mode_read(struct file *file, ++ char __user *buf, size_t len, loff_t *ppos) ++{ ++ struct kbase_device *kbdev = (struct kbase_device *)file->private_data; ++ u32 gpu_status; ++ ssize_t ret_val; ++ ++ kbase_pm_context_active(kbdev); ++ gpu_status = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_STATUS), NULL); ++ kbase_pm_context_idle(kbdev); ++ ++ if (gpu_status & GPU_DBGEN) ++ ret_val = simple_read_from_buffer(buf, len, ppos, "1\n", 2); ++ else ++ ret_val = simple_read_from_buffer(buf, len, ppos, "0\n", 2); ++ ++ return ret_val; ++} ++ ++/* ++ * struct fops_protected_debug_mode - "protected_debug_mode" debugfs fops ++ * ++ * Contains the file operations for the "protected_debug_mode" debugfs file ++ */ ++static const struct file_operations fops_protected_debug_mode = { ++ .open = simple_open, ++ .read = debugfs_protected_debug_mode_read, ++ .llseek = default_llseek, ++}; ++ ++static int kbase_device_debugfs_init(struct kbase_device *kbdev) ++{ ++ struct dentry *debugfs_ctx_defaults_directory; ++ int err; ++ ++ kbdev->mali_debugfs_directory = debugfs_create_dir(kbdev->devname, ++ NULL); ++ if (!kbdev->mali_debugfs_directory) { ++ dev_err(kbdev->dev, "Couldn't create mali debugfs directory\n"); ++ err = -ENOMEM; ++ goto out; ++ } ++ ++ kbdev->debugfs_ctx_directory = debugfs_create_dir("ctx", ++ kbdev->mali_debugfs_directory); ++ if (!kbdev->debugfs_ctx_directory) { ++ dev_err(kbdev->dev, "Couldn't create mali debugfs ctx directory\n"); ++ err = -ENOMEM; ++ goto out; ++ } ++ ++ debugfs_ctx_defaults_directory = debugfs_create_dir("defaults", ++ kbdev->debugfs_ctx_directory); ++ if (!debugfs_ctx_defaults_directory) { ++ dev_err(kbdev->dev, "Couldn't create mali debugfs ctx defaults directory\n"); ++ err = -ENOMEM; ++ goto out; ++ } ++ ++#if !MALI_CUSTOMER_RELEASE ++ kbasep_regs_dump_debugfs_init(kbdev); ++#endif /* !MALI_CUSTOMER_RELEASE */ ++ kbasep_regs_history_debugfs_init(kbdev); ++ ++ kbase_debug_job_fault_debugfs_init(kbdev); ++ kbasep_gpu_memory_debugfs_init(kbdev); ++ kbase_as_fault_debugfs_init(kbdev); ++#if KBASE_GPU_RESET_EN ++ /* fops_* variables created by invocations of macro ++ * MAKE_QUIRK_ACCESSORS() above. */ ++ debugfs_create_file("quirks_sc", 0644, ++ kbdev->mali_debugfs_directory, kbdev, ++ &fops_sc_quirks); ++ debugfs_create_file("quirks_tiler", 0644, ++ kbdev->mali_debugfs_directory, kbdev, ++ &fops_tiler_quirks); ++ debugfs_create_file("quirks_mmu", 0644, ++ kbdev->mali_debugfs_directory, kbdev, ++ &fops_mmu_quirks); ++ debugfs_create_file("quirks_jm", 0644, ++ kbdev->mali_debugfs_directory, kbdev, ++ &fops_jm_quirks); ++#endif /* KBASE_GPU_RESET_EN */ ++ ++ debugfs_create_bool("infinite_cache", 0644, ++ debugfs_ctx_defaults_directory, ++ &kbdev->infinite_cache_active_default); ++ ++ debugfs_create_size_t("mem_pool_max_size", 0644, ++ debugfs_ctx_defaults_directory, ++ &kbdev->mem_pool_max_size_default); ++ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE)) { ++ debugfs_create_file("protected_debug_mode", S_IRUGO, ++ kbdev->mali_debugfs_directory, kbdev, ++ &fops_protected_debug_mode); ++ } ++ ++#if KBASE_TRACE_ENABLE ++ kbasep_trace_debugfs_init(kbdev); ++#endif /* KBASE_TRACE_ENABLE */ ++ ++#ifdef CONFIG_MALI_BIFROST_TRACE_TIMELINE ++ kbasep_trace_timeline_debugfs_init(kbdev); ++#endif /* CONFIG_MALI_BIFROST_TRACE_TIMELINE */ ++ ++#ifdef CONFIG_MALI_BIFROST_DEVFREQ ++#ifdef CONFIG_DEVFREQ_THERMAL ++ if ((kbdev->inited_subsys & inited_devfreq) && !kbdev->model_data) ++ kbase_ipa_debugfs_init(kbdev); ++#endif /* CONFIG_DEVFREQ_THERMAL */ ++#endif /* CONFIG_MALI_BIFROST_DEVFREQ */ ++ ++#ifdef CONFIG_DEBUG_FS ++ debugfs_create_file("serialize_jobs", S_IRUGO | S_IWUSR, ++ kbdev->mali_debugfs_directory, kbdev, ++ &kbasep_serialize_jobs_debugfs_fops); ++#endif /* CONFIG_DEBUG_FS */ ++ ++ return 0; ++ ++out: ++ debugfs_remove_recursive(kbdev->mali_debugfs_directory); ++ return err; ++} ++ ++static void kbase_device_debugfs_term(struct kbase_device *kbdev) ++{ ++ debugfs_remove_recursive(kbdev->mali_debugfs_directory); ++} ++ ++#else /* CONFIG_DEBUG_FS */ ++static inline int kbase_device_debugfs_init(struct kbase_device *kbdev) ++{ ++ return 0; ++} ++ ++static inline void kbase_device_debugfs_term(struct kbase_device *kbdev) { } ++#endif /* CONFIG_DEBUG_FS */ ++ ++static void kbase_device_coherency_init(struct kbase_device *kbdev, ++ unsigned prod_id) ++{ ++#ifdef CONFIG_OF ++ u32 supported_coherency_bitmap = ++ kbdev->gpu_props.props.raw_props.coherency_mode; ++ const void *coherency_override_dts; ++ u32 override_coherency; ++ ++ /* Only for tMIx : ++ * (COHERENCY_ACE_LITE | COHERENCY_ACE) was incorrectly ++ * documented for tMIx so force correct value here. ++ */ ++ if (GPU_ID_IS_NEW_FORMAT(prod_id) && ++ (GPU_ID2_MODEL_MATCH_VALUE(prod_id) == ++ GPU_ID2_PRODUCT_TMIX)) ++ if (supported_coherency_bitmap == ++ COHERENCY_FEATURE_BIT(COHERENCY_ACE)) ++ supported_coherency_bitmap |= ++ COHERENCY_FEATURE_BIT(COHERENCY_ACE_LITE); ++ ++#endif /* CONFIG_OF */ ++ ++ kbdev->system_coherency = COHERENCY_NONE; ++ ++ /* device tree may override the coherency */ ++#ifdef CONFIG_OF ++ coherency_override_dts = of_get_property(kbdev->dev->of_node, ++ "system-coherency", ++ NULL); ++ if (coherency_override_dts) { ++ ++ override_coherency = be32_to_cpup(coherency_override_dts); ++ ++ if ((override_coherency <= COHERENCY_NONE) && ++ (supported_coherency_bitmap & ++ COHERENCY_FEATURE_BIT(override_coherency))) { ++ ++ kbdev->system_coherency = override_coherency; ++ ++ dev_info(kbdev->dev, ++ "Using coherency mode %u set from dtb", ++ override_coherency); ++ } else ++ dev_warn(kbdev->dev, ++ "Ignoring unsupported coherency mode %u set from dtb", ++ override_coherency); ++ } ++ ++#endif /* CONFIG_OF */ ++ ++ kbdev->gpu_props.props.raw_props.coherency_mode = ++ kbdev->system_coherency; ++} ++ ++#ifdef CONFIG_MALI_FPGA_BUS_LOGGER ++ ++/* Callback used by the kbase bus logger client, to initiate a GPU reset ++ * when the bus log is restarted. GPU reset is used as reference point ++ * in HW bus log analyses. ++ */ ++static void kbase_logging_started_cb(void *data) ++{ ++ struct kbase_device *kbdev = (struct kbase_device *)data; ++ ++ if (kbase_prepare_to_reset_gpu(kbdev)) ++ kbase_reset_gpu(kbdev); ++ dev_info(kbdev->dev, "KBASE - Bus logger restarted\n"); ++} ++#endif ++ ++static struct attribute *kbase_attrs[] = { ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++ &dev_attr_debug_command.attr, ++ &dev_attr_js_softstop_always.attr, ++#endif ++#if !MALI_CUSTOMER_RELEASE ++ &dev_attr_force_replay.attr, ++#endif ++ &dev_attr_js_timeouts.attr, ++ &dev_attr_soft_job_timeout.attr, ++ &dev_attr_gpuinfo.attr, ++ &dev_attr_dvfs_period.attr, ++ &dev_attr_pm_poweroff.attr, ++ &dev_attr_reset_timeout.attr, ++ &dev_attr_js_scheduling_period.attr, ++ &dev_attr_power_policy.attr, ++ &dev_attr_core_availability_policy.attr, ++ &dev_attr_core_mask.attr, ++ &dev_attr_mem_pool_size.attr, ++ &dev_attr_mem_pool_max_size.attr, ++ &dev_attr_lp_mem_pool_size.attr, ++ &dev_attr_lp_mem_pool_max_size.attr, ++ NULL ++}; ++ ++static const struct attribute_group kbase_attr_group = { ++ .attrs = kbase_attrs, ++}; ++ ++static int kbase_platform_device_remove(struct platform_device *pdev) ++{ ++ struct kbase_device *kbdev = to_kbase_device(&pdev->dev); ++ const struct list_head *dev_list; ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ kfree(kbdev->gpu_props.prop_buffer); ++ ++#ifdef CONFIG_MALI_FPGA_BUS_LOGGER ++ if (kbdev->inited_subsys & inited_buslogger) { ++ bl_core_client_unregister(kbdev->buslogger); ++ kbdev->inited_subsys &= ~inited_buslogger; ++ } ++#endif ++ ++ ++ if (kbdev->inited_subsys & inited_dev_list) { ++ dev_list = kbase_dev_list_get(); ++ list_del(&kbdev->entry); ++ kbase_dev_list_put(dev_list); ++ kbdev->inited_subsys &= ~inited_dev_list; ++ } ++ ++ if (kbdev->inited_subsys & inited_misc_register) { ++ misc_deregister(&kbdev->mdev); ++ kbdev->inited_subsys &= ~inited_misc_register; ++ } ++ ++ if (kbdev->inited_subsys & inited_sysfs_group) { ++ sysfs_remove_group(&kbdev->dev->kobj, &kbase_attr_group); ++ kbdev->inited_subsys &= ~inited_sysfs_group; ++ } ++ ++ if (kbdev->inited_subsys & inited_get_device) { ++ put_device(kbdev->dev); ++ kbdev->inited_subsys &= ~inited_get_device; ++ } ++ ++ if (kbdev->inited_subsys & inited_debugfs) { ++ kbase_device_debugfs_term(kbdev); ++ kbdev->inited_subsys &= ~inited_debugfs; ++ } ++ ++ if (kbdev->inited_subsys & inited_job_fault) { ++ kbase_debug_job_fault_dev_term(kbdev); ++ kbdev->inited_subsys &= ~inited_job_fault; ++ } ++ if (kbdev->inited_subsys & inited_vinstr) { ++ kbase_vinstr_term(kbdev->vinstr_ctx); ++ kbdev->inited_subsys &= ~inited_vinstr; ++ } ++ ++#ifdef CONFIG_MALI_BIFROST_DEVFREQ ++ if (kbdev->inited_subsys & inited_devfreq) { ++ kbase_devfreq_term(kbdev); ++ kbdev->inited_subsys &= ~inited_devfreq; ++ } ++#endif ++ ++ if (kbdev->inited_subsys & inited_backend_late) { ++ kbase_backend_late_term(kbdev); ++ kbdev->inited_subsys &= ~inited_backend_late; ++ } ++ ++ if (kbdev->inited_subsys & inited_tlstream) { ++ kbase_tlstream_term(); ++ kbdev->inited_subsys &= ~inited_tlstream; ++ } ++ ++ /* Bring job and mem sys to a halt before we continue termination */ ++ ++ if (kbdev->inited_subsys & inited_js) ++ kbasep_js_devdata_halt(kbdev); ++ ++ if (kbdev->inited_subsys & inited_mem) ++ kbase_mem_halt(kbdev); ++ ++ if (kbdev->inited_subsys & inited_protected) { ++ kbasep_protected_mode_term(kbdev); ++ kbdev->inited_subsys &= ~inited_protected; ++ } ++ ++ if (kbdev->inited_subsys & inited_js) { ++ kbasep_js_devdata_term(kbdev); ++ kbdev->inited_subsys &= ~inited_js; ++ } ++ ++ if (kbdev->inited_subsys & inited_mem) { ++ kbase_mem_term(kbdev); ++ kbdev->inited_subsys &= ~inited_mem; ++ } ++ ++ if (kbdev->inited_subsys & inited_pm_runtime_init) { ++ kbdev->pm.callback_power_runtime_term(kbdev); ++ kbdev->inited_subsys &= ~inited_pm_runtime_init; ++ } ++ ++ if (kbdev->inited_subsys & inited_ctx_sched) { ++ kbase_ctx_sched_term(kbdev); ++ kbdev->inited_subsys &= ~inited_ctx_sched; ++ } ++ ++ if (kbdev->inited_subsys & inited_device) { ++ kbase_device_term(kbdev); ++ kbdev->inited_subsys &= ~inited_device; ++ } ++ ++ if (kbdev->inited_subsys & inited_backend_early) { ++ kbase_backend_early_term(kbdev); ++ kbdev->inited_subsys &= ~inited_backend_early; ++ } ++ ++ if (kbdev->inited_subsys & inited_io_history) { ++ kbase_io_history_term(&kbdev->io_history); ++ kbdev->inited_subsys &= ~inited_io_history; ++ } ++ ++ if (kbdev->inited_subsys & inited_power_control) { ++ power_control_term(kbdev); ++ kbdev->inited_subsys &= ~inited_power_control; ++ } ++ ++ if (kbdev->inited_subsys & inited_registers_map) { ++ registers_unmap(kbdev); ++ kbdev->inited_subsys &= ~inited_registers_map; ++ } ++ ++#ifdef CONFIG_MALI_BIFROST_NO_MALI ++ if (kbdev->inited_subsys & inited_gpu_device) { ++ gpu_device_destroy(kbdev); ++ kbdev->inited_subsys &= ~inited_gpu_device; ++ } ++#endif /* CONFIG_MALI_BIFROST_NO_MALI */ ++ ++ if (kbdev->inited_subsys != 0) ++ dev_err(kbdev->dev, "Missing sub system termination\n"); ++ ++ kbase_device_free(kbdev); ++ ++ return 0; ++} ++ ++ ++/* Number of register accesses for the buffer that we allocate during ++ * initialization time. The buffer size can be changed later via debugfs. */ ++#define KBASEP_DEFAULT_REGISTER_HISTORY_SIZE ((u16)512) ++ ++static int kbase_platform_device_probe(struct platform_device *pdev) ++{ ++ struct kbase_device *kbdev; ++ struct mali_base_gpu_core_props *core_props; ++ u32 gpu_id; ++ unsigned prod_id; ++ const struct list_head *dev_list; ++ int err = 0; ++ ++#ifdef CONFIG_OF ++ err = kbase_platform_early_init(); ++ if (err) { ++ dev_err(&pdev->dev, "Early platform initialization failed\n"); ++ kbase_platform_device_remove(pdev); ++ return err; ++ } ++#endif ++ kbdev = kbase_device_alloc(); ++ if (!kbdev) { ++ dev_err(&pdev->dev, "Allocate device failed\n"); ++ kbase_platform_device_remove(pdev); ++ return -ENOMEM; ++ } ++ ++ kbdev->dev = &pdev->dev; ++ dev_set_drvdata(kbdev->dev, kbdev); ++ ++#ifdef CONFIG_MALI_BIFROST_NO_MALI ++ err = gpu_device_create(kbdev); ++ if (err) { ++ dev_err(&pdev->dev, "Dummy model initialization failed\n"); ++ kbase_platform_device_remove(pdev); ++ return err; ++ } ++ kbdev->inited_subsys |= inited_gpu_device; ++#endif /* CONFIG_MALI_BIFROST_NO_MALI */ ++ ++ err = assign_irqs(pdev); ++ if (err) { ++ dev_err(&pdev->dev, "IRQ search failed\n"); ++ kbase_platform_device_remove(pdev); ++ return err; ++ } ++ ++ err = registers_map(kbdev); ++ if (err) { ++ dev_err(&pdev->dev, "Register map failed\n"); ++ kbase_platform_device_remove(pdev); ++ return err; ++ } ++ kbdev->inited_subsys |= inited_registers_map; ++ ++ err = power_control_init(pdev); ++ if (err) { ++ dev_err(&pdev->dev, "Power control initialization failed\n"); ++ kbase_platform_device_remove(pdev); ++ return err; ++ } ++ kbdev->inited_subsys |= inited_power_control; ++ ++ err = kbase_io_history_init(&kbdev->io_history, ++ KBASEP_DEFAULT_REGISTER_HISTORY_SIZE); ++ if (err) { ++ dev_err(&pdev->dev, "Register access history initialization failed\n"); ++ kbase_platform_device_remove(pdev); ++ return -ENOMEM; ++ } ++ kbdev->inited_subsys |= inited_io_history; ++ ++ err = kbase_backend_early_init(kbdev); ++ if (err) { ++ dev_err(kbdev->dev, "Early backend initialization failed\n"); ++ kbase_platform_device_remove(pdev); ++ return err; ++ } ++ kbdev->inited_subsys |= inited_backend_early; ++ ++ scnprintf(kbdev->devname, DEVNAME_SIZE, "%s%d", kbase_drv_name, ++ kbase_dev_nr); ++ ++ kbase_disjoint_init(kbdev); ++ ++ /* obtain min/max configured gpu frequencies */ ++ core_props = &(kbdev->gpu_props.props.core_props); ++ core_props->gpu_freq_khz_min = GPU_FREQ_KHZ_MIN; ++ core_props->gpu_freq_khz_max = GPU_FREQ_KHZ_MAX; ++ ++ err = kbase_device_init(kbdev); ++ if (err) { ++ dev_err(kbdev->dev, "Device initialization failed (%d)\n", err); ++ kbase_platform_device_remove(pdev); ++ return err; ++ } ++ kbdev->inited_subsys |= inited_device; ++ ++ err = kbase_ctx_sched_init(kbdev); ++ if (err) { ++ dev_err(kbdev->dev, "Context scheduler initialization failed (%d)\n", ++ err); ++ kbase_platform_device_remove(pdev); ++ return err; ++ } ++ kbdev->inited_subsys |= inited_ctx_sched; ++ ++ if (kbdev->pm.callback_power_runtime_init) { ++ err = kbdev->pm.callback_power_runtime_init(kbdev); ++ if (err) { ++ dev_err(kbdev->dev, ++ "Runtime PM initialization failed\n"); ++ kbase_platform_device_remove(pdev); ++ return err; ++ } ++ kbdev->inited_subsys |= inited_pm_runtime_init; ++ } ++ ++ err = kbase_mem_init(kbdev); ++ if (err) { ++ dev_err(kbdev->dev, "Memory subsystem initialization failed\n"); ++ kbase_platform_device_remove(pdev); ++ return err; ++ } ++ kbdev->inited_subsys |= inited_mem; ++ ++ gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; ++ gpu_id &= GPU_ID_VERSION_PRODUCT_ID; ++ prod_id = gpu_id >> GPU_ID_VERSION_PRODUCT_ID_SHIFT; ++ ++ kbase_device_coherency_init(kbdev, prod_id); ++ ++ err = kbasep_protected_mode_init(kbdev); ++ if (err) { ++ dev_err(kbdev->dev, "Protected mode subsystem initialization failed\n"); ++ kbase_platform_device_remove(pdev); ++ return err; ++ } ++ kbdev->inited_subsys |= inited_protected; ++ ++ dev_list = kbase_dev_list_get(); ++ list_add(&kbdev->entry, &kbase_dev_list); ++ kbase_dev_list_put(dev_list); ++ kbdev->inited_subsys |= inited_dev_list; ++ ++ err = kbasep_js_devdata_init(kbdev); ++ if (err) { ++ dev_err(kbdev->dev, "Job JS devdata initialization failed\n"); ++ kbase_platform_device_remove(pdev); ++ return err; ++ } ++ kbdev->inited_subsys |= inited_js; ++ ++ err = kbase_tlstream_init(); ++ if (err) { ++ dev_err(kbdev->dev, "Timeline stream initialization failed\n"); ++ kbase_platform_device_remove(pdev); ++ return err; ++ } ++ kbdev->inited_subsys |= inited_tlstream; ++ ++ err = kbase_backend_late_init(kbdev); ++ if (err) { ++ dev_err(kbdev->dev, "Late backend initialization failed\n"); ++ kbase_platform_device_remove(pdev); ++ return err; ++ } ++ kbdev->inited_subsys |= inited_backend_late; ++ ++ /* Initialize the kctx list. This is used by vinstr. */ ++ mutex_init(&kbdev->kctx_list_lock); ++ INIT_LIST_HEAD(&kbdev->kctx_list); ++ ++ kbdev->vinstr_ctx = kbase_vinstr_init(kbdev); ++ if (!kbdev->vinstr_ctx) { ++ dev_err(kbdev->dev, ++ "Virtual instrumentation initialization failed\n"); ++ kbase_platform_device_remove(pdev); ++ return -EINVAL; ++ } ++ kbdev->inited_subsys |= inited_vinstr; ++ ++#ifdef CONFIG_MALI_BIFROST_DEVFREQ ++ /* Devfreq uses vinstr, so must be initialized after it. */ ++ err = kbase_devfreq_init(kbdev); ++ if (!err) ++ kbdev->inited_subsys |= inited_devfreq; ++ else ++ dev_err(kbdev->dev, "Continuing without devfreq\n"); ++#endif /* CONFIG_MALI_BIFROST_DEVFREQ */ ++ ++ err = kbase_debug_job_fault_dev_init(kbdev); ++ if (err) { ++ dev_err(kbdev->dev, "Job fault debug initialization failed\n"); ++ kbase_platform_device_remove(pdev); ++ return err; ++ } ++ kbdev->inited_subsys |= inited_job_fault; ++ ++ err = kbase_device_debugfs_init(kbdev); ++ if (err) { ++ dev_err(kbdev->dev, "DebugFS initialization failed"); ++ kbase_platform_device_remove(pdev); ++ return err; ++ } ++ kbdev->inited_subsys |= inited_debugfs; ++ ++ kbdev->mdev.minor = MISC_DYNAMIC_MINOR; ++ kbdev->mdev.name = kbdev->devname; ++ kbdev->mdev.fops = &kbase_fops; ++ kbdev->mdev.parent = get_device(kbdev->dev); ++ kbdev->inited_subsys |= inited_get_device; ++ ++ /* This needs to happen before registering the device with misc_register(), ++ * otherwise it causes a race condition between registering the device and a ++ * uevent event being generated for userspace, causing udev rules to run ++ * which might expect certain sysfs attributes present. As a result of the ++ * race condition we avoid, some Mali sysfs entries may have appeared to ++ * udev to not exist. ++ ++ * For more information, see ++ * https://www.kernel.org/doc/Documentation/driver-model/device.txt, the ++ * paragraph that starts with "Word of warning", currently the second-last ++ * paragraph. ++ */ ++ err = sysfs_create_group(&kbdev->dev->kobj, &kbase_attr_group); ++ if (err) { ++ dev_err(&pdev->dev, "SysFS group creation failed\n"); ++ kbase_platform_device_remove(pdev); ++ return err; ++ } ++ kbdev->inited_subsys |= inited_sysfs_group; ++ ++ err = misc_register(&kbdev->mdev); ++ if (err) { ++ dev_err(kbdev->dev, "Misc device registration failed for %s\n", ++ kbdev->devname); ++ kbase_platform_device_remove(pdev); ++ return err; ++ } ++ kbdev->inited_subsys |= inited_misc_register; ++ ++ ++#ifdef CONFIG_MALI_FPGA_BUS_LOGGER ++ err = bl_core_client_register(kbdev->devname, ++ kbase_logging_started_cb, ++ kbdev, &kbdev->buslogger, ++ THIS_MODULE, NULL); ++ if (err == 0) { ++ kbdev->inited_subsys |= inited_buslogger; ++ bl_core_set_threshold(kbdev->buslogger, 1024*1024*1024); ++ } else { ++ dev_warn(kbdev->dev, "Bus log client registration failed\n"); ++ err = 0; ++ } ++#endif ++ ++ err = kbase_gpuprops_populate_user_buffer(kbdev); ++ if (err) { ++ dev_err(&pdev->dev, "GPU property population failed"); ++ kbase_platform_device_remove(pdev); ++ return err; ++ } ++ ++ dev_info(kbdev->dev, ++ "Probed as %s\n", dev_name(kbdev->mdev.this_device)); ++ ++ kbase_dev_nr++; ++ ++ return err; ++} ++ ++#undef KBASEP_DEFAULT_REGISTER_HISTORY_SIZE ++ ++/** ++ * kbase_device_suspend - Suspend callback from the OS. ++ * ++ * This is called by Linux when the device should suspend. ++ * ++ * @dev: The device to suspend ++ * ++ * Return: A standard Linux error code ++ */ ++static int kbase_device_suspend(struct device *dev) ++{ ++ struct kbase_device *kbdev = to_kbase_device(dev); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++#if defined(CONFIG_MALI_BIFROST_DEVFREQ) && \ ++ (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) ++ if (kbdev->inited_subsys & inited_devfreq) ++ devfreq_suspend_device(kbdev->devfreq); ++#endif ++ ++ kbase_pm_suspend(kbdev); ++ return 0; ++} ++ ++/** ++ * kbase_device_resume - Resume callback from the OS. ++ * ++ * This is called by Linux when the device should resume from suspension. ++ * ++ * @dev: The device to resume ++ * ++ * Return: A standard Linux error code ++ */ ++static int kbase_device_resume(struct device *dev) ++{ ++ struct kbase_device *kbdev = to_kbase_device(dev); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ kbase_pm_resume(kbdev); ++ ++#if defined(CONFIG_MALI_BIFROST_DEVFREQ) && \ ++ (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) ++ if (kbdev->inited_subsys & inited_devfreq) ++ devfreq_resume_device(kbdev->devfreq); ++#endif ++ return 0; ++} ++ ++/** ++ * kbase_device_runtime_suspend - Runtime suspend callback from the OS. ++ * ++ * This is called by Linux when the device should prepare for a condition in ++ * which it will not be able to communicate with the CPU(s) and RAM due to ++ * power management. ++ * ++ * @dev: The device to suspend ++ * ++ * Return: A standard Linux error code ++ */ ++#ifdef KBASE_PM_RUNTIME ++static int kbase_device_runtime_suspend(struct device *dev) ++{ ++ struct kbase_device *kbdev = to_kbase_device(dev); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++#if defined(CONFIG_MALI_BIFROST_DEVFREQ) && \ ++ (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) ++ if (kbdev->inited_subsys & inited_devfreq) ++ devfreq_suspend_device(kbdev->devfreq); ++#endif ++ ++ if (kbdev->pm.backend.callback_power_runtime_off) { ++ kbdev->pm.backend.callback_power_runtime_off(kbdev); ++ dev_dbg(dev, "runtime suspend\n"); ++ } ++ return 0; ++} ++#endif /* KBASE_PM_RUNTIME */ ++ ++/** ++ * kbase_device_runtime_resume - Runtime resume callback from the OS. ++ * ++ * This is called by Linux when the device should go into a fully active state. ++ * ++ * @dev: The device to suspend ++ * ++ * Return: A standard Linux error code ++ */ ++ ++#ifdef KBASE_PM_RUNTIME ++static int kbase_device_runtime_resume(struct device *dev) ++{ ++ int ret = 0; ++ struct kbase_device *kbdev = to_kbase_device(dev); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ if (kbdev->pm.backend.callback_power_runtime_on) { ++ ret = kbdev->pm.backend.callback_power_runtime_on(kbdev); ++ dev_dbg(dev, "runtime resume\n"); ++ } ++ ++#if defined(CONFIG_MALI_BIFROST_DEVFREQ) && \ ++ (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) ++ if (kbdev->inited_subsys & inited_devfreq) ++ devfreq_resume_device(kbdev->devfreq); ++#endif ++ ++ return ret; ++} ++#endif /* KBASE_PM_RUNTIME */ ++ ++ ++#ifdef KBASE_PM_RUNTIME ++/** ++ * kbase_device_runtime_idle - Runtime idle callback from the OS. ++ * @dev: The device to suspend ++ * ++ * This is called by Linux when the device appears to be inactive and it might ++ * be placed into a low power state. ++ * ++ * Return: 0 if device can be suspended, non-zero to avoid runtime autosuspend, ++ * otherwise a standard Linux error code ++ */ ++static int kbase_device_runtime_idle(struct device *dev) ++{ ++ struct kbase_device *kbdev = to_kbase_device(dev); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ /* Use platform specific implementation if it exists. */ ++ if (kbdev->pm.backend.callback_power_runtime_idle) ++ return kbdev->pm.backend.callback_power_runtime_idle(kbdev); ++ ++ return 0; ++} ++#endif /* KBASE_PM_RUNTIME */ ++ ++/* The power management operations for the platform driver. ++ */ ++static const struct dev_pm_ops kbase_pm_ops = { ++ .suspend = kbase_device_suspend, ++ .resume = kbase_device_resume, ++#ifdef KBASE_PM_RUNTIME ++ .runtime_suspend = kbase_device_runtime_suspend, ++ .runtime_resume = kbase_device_runtime_resume, ++ .runtime_idle = kbase_device_runtime_idle, ++#endif /* KBASE_PM_RUNTIME */ ++}; ++ ++#ifdef CONFIG_OF ++static const struct of_device_id kbase_dt_ids[] = { ++ { .compatible = "arm,malit6xx" }, ++ { .compatible = "arm,mali-midgard" }, ++ { /* sentinel */ } ++}; ++MODULE_DEVICE_TABLE(of, kbase_dt_ids); ++#endif ++ ++static struct platform_driver kbase_platform_driver = { ++ .probe = kbase_platform_device_probe, ++ .remove = kbase_platform_device_remove, ++ .driver = { ++ .name = kbase_drv_name, ++ .owner = THIS_MODULE, ++ .pm = &kbase_pm_ops, ++ .of_match_table = of_match_ptr(kbase_dt_ids), ++ }, ++}; ++ ++/* ++ * The driver will not provide a shortcut to create the Mali platform device ++ * anymore when using Device Tree. ++ */ ++#ifdef CONFIG_OF ++module_platform_driver(kbase_platform_driver); ++#else ++ ++static int __init kbase_driver_init(void) ++{ ++ int ret; ++ ++ ret = kbase_platform_early_init(); ++ if (ret) ++ return ret; ++ ++ ret = kbase_platform_register(); ++ if (ret) ++ return ret; ++ ++ ret = platform_driver_register(&kbase_platform_driver); ++ ++ if (ret) ++ kbase_platform_unregister(); ++ ++ return ret; ++} ++ ++static void __exit kbase_driver_exit(void) ++{ ++ platform_driver_unregister(&kbase_platform_driver); ++ kbase_platform_unregister(); ++} ++ ++module_init(kbase_driver_init); ++module_exit(kbase_driver_exit); ++ ++#endif /* CONFIG_OF */ ++ ++MODULE_LICENSE("GPL"); ++MODULE_VERSION(MALI_RELEASE_NAME " (UK version " \ ++ __stringify(BASE_UK_VERSION_MAJOR) "." \ ++ __stringify(BASE_UK_VERSION_MINOR) ")"); ++ ++#if defined(CONFIG_MALI_BIFROST_GATOR_SUPPORT) || defined(CONFIG_MALI_BIFROST_SYSTEM_TRACE) ++#define CREATE_TRACE_POINTS ++#endif ++ ++#ifdef CONFIG_MALI_BIFROST_GATOR_SUPPORT ++/* Create the trace points (otherwise we just get code to call a tracepoint) */ ++#include "mali_linux_trace.h" ++ ++EXPORT_TRACEPOINT_SYMBOL_GPL(mali_job_slots_event); ++EXPORT_TRACEPOINT_SYMBOL_GPL(mali_pm_status); ++EXPORT_TRACEPOINT_SYMBOL_GPL(mali_pm_power_on); ++EXPORT_TRACEPOINT_SYMBOL_GPL(mali_pm_power_off); ++EXPORT_TRACEPOINT_SYMBOL_GPL(mali_page_fault_insert_pages); ++EXPORT_TRACEPOINT_SYMBOL_GPL(mali_mmu_as_in_use); ++EXPORT_TRACEPOINT_SYMBOL_GPL(mali_mmu_as_released); ++EXPORT_TRACEPOINT_SYMBOL_GPL(mali_total_alloc_pages_change); ++ ++void kbase_trace_mali_pm_status(u32 event, u64 value) ++{ ++ trace_mali_pm_status(event, value); ++} ++ ++void kbase_trace_mali_pm_power_off(u32 event, u64 value) ++{ ++ trace_mali_pm_power_off(event, value); ++} ++ ++void kbase_trace_mali_pm_power_on(u32 event, u64 value) ++{ ++ trace_mali_pm_power_on(event, value); ++} ++ ++void kbase_trace_mali_job_slots_event(u32 event, const struct kbase_context *kctx, u8 atom_id) ++{ ++ trace_mali_job_slots_event(event, (kctx != NULL ? kctx->tgid : 0), (kctx != NULL ? kctx->pid : 0), atom_id); ++} ++ ++void kbase_trace_mali_page_fault_insert_pages(int event, u32 value) ++{ ++ trace_mali_page_fault_insert_pages(event, value); ++} ++ ++void kbase_trace_mali_mmu_as_in_use(int event) ++{ ++ trace_mali_mmu_as_in_use(event); ++} ++ ++void kbase_trace_mali_mmu_as_released(int event) ++{ ++ trace_mali_mmu_as_released(event); ++} ++ ++void kbase_trace_mali_total_alloc_pages_change(long long int event) ++{ ++ trace_mali_total_alloc_pages_change(event); ++} ++#endif /* CONFIG_MALI_BIFROST_GATOR_SUPPORT */ ++#ifdef CONFIG_MALI_BIFROST_SYSTEM_TRACE ++#include "mali_linux_kbase_trace.h" ++#endif +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_ctx_sched.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_ctx_sched.c +new file mode 100755 +index 000000000000..e2f7baabad43 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_ctx_sched.c +@@ -0,0 +1,203 @@ ++/* ++ * ++ * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include ++#include ++ ++#include "mali_kbase_ctx_sched.h" ++ ++int kbase_ctx_sched_init(struct kbase_device *kbdev) ++{ ++ int as_present = (1U << kbdev->nr_hw_address_spaces) - 1; ++ ++ /* These two must be recalculated if nr_hw_address_spaces changes ++ * (e.g. for HW workarounds) */ ++ kbdev->nr_user_address_spaces = kbdev->nr_hw_address_spaces; ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) { ++ bool use_workaround; ++ ++ use_workaround = DEFAULT_SECURE_BUT_LOSS_OF_PERFORMANCE; ++ if (use_workaround) { ++ dev_dbg(kbdev->dev, "GPU has HW ISSUE 8987, and driver configured for security workaround: 1 address space only"); ++ kbdev->nr_user_address_spaces = 1; ++ } ++ } ++ ++ kbdev->as_free = as_present; /* All ASs initially free */ ++ ++ memset(kbdev->as_to_kctx, 0, sizeof(kbdev->as_to_kctx)); ++ ++ return 0; ++} ++ ++void kbase_ctx_sched_term(struct kbase_device *kbdev) ++{ ++ s8 i; ++ ++ /* Sanity checks */ ++ for (i = 0; i != kbdev->nr_hw_address_spaces; ++i) { ++ WARN_ON(kbdev->as_to_kctx[i] != NULL); ++ WARN_ON(!(kbdev->as_free & (1u << i))); ++ } ++} ++ ++/* kbasep_ctx_sched_find_as_for_ctx - Find a free address space ++ * ++ * @kbdev: The context for which to find a free address space ++ * ++ * Return: A valid AS if successful, otherwise KBASEP_AS_NR_INVALID ++ * ++ * This function returns an address space available for use. It would prefer ++ * returning an AS that has been previously assigned to the context to ++ * avoid having to reprogram the MMU. ++ */ ++static int kbasep_ctx_sched_find_as_for_ctx(struct kbase_context *kctx) ++{ ++ struct kbase_device *const kbdev = kctx->kbdev; ++ int free_as; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ /* First check if the previously assigned AS is available */ ++ if ((kctx->as_nr != KBASEP_AS_NR_INVALID) && ++ (kbdev->as_free & (1u << kctx->as_nr))) ++ return kctx->as_nr; ++ ++ /* The previously assigned AS was taken, we'll be returning any free ++ * AS at this point. ++ */ ++ free_as = ffs(kbdev->as_free) - 1; ++ if (free_as >= 0 && free_as < kbdev->nr_hw_address_spaces) ++ return free_as; ++ ++ return KBASEP_AS_NR_INVALID; ++} ++ ++int kbase_ctx_sched_retain_ctx(struct kbase_context *kctx) ++{ ++ struct kbase_device *const kbdev = kctx->kbdev; ++ ++ lockdep_assert_held(&kbdev->mmu_hw_mutex); ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ WARN_ON(!kbdev->pm.backend.gpu_powered); ++ ++ if (atomic_inc_return(&kctx->refcount) == 1) { ++ int const free_as = kbasep_ctx_sched_find_as_for_ctx(kctx); ++ ++ if (free_as != KBASEP_AS_NR_INVALID) { ++ kbdev->as_free &= ~(1u << free_as); ++ /* Only program the MMU if the context has not been ++ * assigned the same address space before. ++ */ ++ if (free_as != kctx->as_nr) { ++ struct kbase_context *const prev_kctx = ++ kbdev->as_to_kctx[free_as]; ++ ++ if (prev_kctx) { ++ WARN_ON(atomic_read(&prev_kctx->refcount) != 0); ++ kbase_mmu_disable(prev_kctx); ++ prev_kctx->as_nr = KBASEP_AS_NR_INVALID; ++ } ++ ++ kctx->as_nr = free_as; ++ kbdev->as_to_kctx[free_as] = kctx; ++ kbase_mmu_update(kctx); ++ } ++ } else { ++ atomic_dec(&kctx->refcount); ++ ++ /* Failed to find an available address space, we must ++ * be returning an error at this point. ++ */ ++ WARN_ON(kctx->as_nr != KBASEP_AS_NR_INVALID); ++ } ++ } ++ ++ return kctx->as_nr; ++} ++ ++void kbase_ctx_sched_retain_ctx_refcount(struct kbase_context *kctx) ++{ ++ struct kbase_device *const kbdev = kctx->kbdev; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ WARN_ON(atomic_read(&kctx->refcount) == 0); ++ WARN_ON(kctx->as_nr == KBASEP_AS_NR_INVALID); ++ WARN_ON(kbdev->as_to_kctx[kctx->as_nr] != kctx); ++ ++ atomic_inc(&kctx->refcount); ++} ++ ++void kbase_ctx_sched_release_ctx(struct kbase_context *kctx) ++{ ++ struct kbase_device *const kbdev = kctx->kbdev; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (atomic_dec_return(&kctx->refcount) == 0) ++ kbdev->as_free |= (1u << kctx->as_nr); ++} ++ ++void kbase_ctx_sched_remove_ctx(struct kbase_context *kctx) ++{ ++ struct kbase_device *const kbdev = kctx->kbdev; ++ ++ lockdep_assert_held(&kbdev->mmu_hw_mutex); ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ WARN_ON(atomic_read(&kctx->refcount) != 0); ++ ++ if (kctx->as_nr != KBASEP_AS_NR_INVALID) { ++ if (kbdev->pm.backend.gpu_powered) ++ kbase_mmu_disable(kctx); ++ ++ kbdev->as_to_kctx[kctx->as_nr] = NULL; ++ kctx->as_nr = KBASEP_AS_NR_INVALID; ++ } ++} ++ ++void kbase_ctx_sched_restore_all_as(struct kbase_device *kbdev) ++{ ++ s8 i; ++ ++ lockdep_assert_held(&kbdev->mmu_hw_mutex); ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ WARN_ON(!kbdev->pm.backend.gpu_powered); ++ ++ for (i = 0; i != kbdev->nr_hw_address_spaces; ++i) { ++ struct kbase_context *kctx; ++ ++ kctx = kbdev->as_to_kctx[i]; ++ if (kctx) { ++ if (atomic_read(&kctx->refcount)) { ++ WARN_ON(kctx->as_nr != i); ++ ++ kbase_mmu_update(kctx); ++ } else { ++ /* This context might have been assigned an ++ * AS before, clear it. ++ */ ++ kbdev->as_to_kctx[kctx->as_nr] = NULL; ++ kctx->as_nr = KBASEP_AS_NR_INVALID; ++ } ++ } else { ++ kbase_mmu_disable_as(kbdev, i); ++ } ++ } ++} +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_ctx_sched.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_ctx_sched.h +new file mode 100755 +index 000000000000..2330d48c8e51 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_ctx_sched.h +@@ -0,0 +1,131 @@ ++/* ++ * ++ * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KBASE_CTX_SCHED_H_ ++#define _KBASE_CTX_SCHED_H_ ++ ++#include ++ ++/* The Context Scheduler manages address space assignment and reference ++ * counting to kbase_context. The interface has been designed to minimise ++ * interactions between the Job Scheduler and Power Management/MMU to support ++ * the existing Job Scheduler interface. ++ * ++ * The initial implementation of the Context Scheduler does not schedule ++ * contexts. Instead it relies on the Job Scheduler to make decisions of ++ * when to schedule/evict contexts if address spaces are starved. In the ++ * future, once an interface between the CS and JS have been devised to ++ * provide enough information about how each context is consuming GPU resources, ++ * those decisions can be made in the CS itself, thereby reducing duplicated ++ * code. ++ */ ++ ++/* base_ctx_sched_init - Initialise the context scheduler ++ * ++ * @kbdev: The device for which the context scheduler needs to be ++ * initialised ++ * ++ * Return: 0 for success, otherwise failure ++ * ++ * This must be called during device initilisation. The number of hardware ++ * address spaces must already be established before calling this function. ++ */ ++int kbase_ctx_sched_init(struct kbase_device *kbdev); ++ ++/* base_ctx_sched_term - Terminate the context scheduler ++ * ++ * @kbdev: The device for which the context scheduler needs to be ++ * terminated ++ * ++ * This must be called during device termination after all contexts have been ++ * destroyed. ++ */ ++void kbase_ctx_sched_term(struct kbase_device *kbdev); ++ ++/* kbase_ctx_sched_retain_ctx - Retain a reference to the @ref kbase_context ++ * ++ * @kctx: The context to which to retain a reference ++ * ++ * Return: The address space that the context has been assigned to or ++ * KBASEP_AS_NR_INVALID if no address space was available. ++ * ++ * This function should be called whenever an address space should be assigned ++ * to a context and programmed onto the MMU. It should typically be called ++ * when jobs are ready to be submitted to the GPU. ++ * ++ * It can be called as many times as necessary. The address space will be ++ * assigned to the context for as long as there is a reference to said context. ++ * ++ * The kbase_device::mmu_hw_mutex and kbase_device::hwaccess_lock locks must be ++ * held whilst calling this function. ++ */ ++int kbase_ctx_sched_retain_ctx(struct kbase_context *kctx); ++ ++/* kbase_ctx_sched_retain_ctx_refcount ++ * ++ * @kctx: The context to which to retain a reference ++ * ++ * This function only retains a reference to the context. It must be called ++ * only when the context already has a reference. ++ * ++ * This is typically called inside an atomic session where we know the context ++ * is already scheduled in but want to take an extra reference to ensure that ++ * it doesn't get descheduled. ++ * ++ * The kbase_device::hwaccess_lock must be held whilst calling this function ++ */ ++void kbase_ctx_sched_retain_ctx_refcount(struct kbase_context *kctx); ++ ++/* kbase_ctx_sched_release_ctx - Release a reference to the @ref kbase_context ++ * ++ * @kctx: The context from which to release a reference ++ * ++ * This function should be called whenever an address space could be unassigned ++ * from a context. When there are no more references to said context, the ++ * address space previously assigned to this context shall be reassigned to ++ * other contexts as needed. ++ * ++ * The kbase_device::hwaccess_lock must be held whilst calling this function ++ */ ++void kbase_ctx_sched_release_ctx(struct kbase_context *kctx); ++ ++/* kbase_ctx_sched_remove_ctx - Unassign previously assigned address space ++ * ++ * @kctx: The context to be removed ++ * ++ * This function should be called when a context is being destroyed. The ++ * context must no longer have any reference. If it has been assigned an ++ * address space before then the AS will be unprogrammed. ++ * ++ * The kbase_device::mmu_hw_mutex and kbase_device::hwaccess_lock locks must be ++ * held whilst calling this function. ++ */ ++void kbase_ctx_sched_remove_ctx(struct kbase_context *kctx); ++ ++/* kbase_ctx_sched_restore_all_as - Reprogram all address spaces ++ * ++ * @kbdev: The device for which address spaces to be reprogrammed ++ * ++ * This function shall reprogram all address spaces previously assigned to ++ * contexts. It can be used after the GPU is reset. ++ * ++ * The kbase_device::mmu_hw_mutex and kbase_device::hwaccess_lock locks must be ++ * held whilst calling this function. ++ */ ++void kbase_ctx_sched_restore_all_as(struct kbase_device *kbdev); ++ ++#endif /* _KBASE_CTX_SCHED_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_debug.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_debug.c +new file mode 100755 +index 000000000000..fb57ac2e31ad +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_debug.c +@@ -0,0 +1,39 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2014 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#include ++ ++static struct kbasep_debug_assert_cb kbasep_debug_assert_registered_cb = { ++ NULL, ++ NULL ++}; ++ ++void kbase_debug_assert_register_hook(kbase_debug_assert_hook *func, void *param) ++{ ++ kbasep_debug_assert_registered_cb.func = func; ++ kbasep_debug_assert_registered_cb.param = param; ++} ++ ++void kbasep_debug_assert_call_hook(void) ++{ ++ if (kbasep_debug_assert_registered_cb.func != NULL) ++ kbasep_debug_assert_registered_cb.func(kbasep_debug_assert_registered_cb.param); ++} ++KBASE_EXPORT_SYMBOL(kbasep_debug_assert_call_hook); ++ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_debug.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_debug.h +new file mode 100755 +index 000000000000..31b754c5507b +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_debug.h +@@ -0,0 +1,164 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2015, 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#ifndef _KBASE_DEBUG_H ++#define _KBASE_DEBUG_H ++ ++#include ++ ++/** @brief If equals to 0, a trace containing the file, line, and function will be displayed before each message. */ ++#define KBASE_DEBUG_SKIP_TRACE 0 ++ ++/** @brief If different from 0, the trace will only contain the file and line. */ ++#define KBASE_DEBUG_SKIP_FUNCTION_NAME 0 ++ ++/** @brief Disable the asserts tests if set to 1. Default is to disable the asserts in release. */ ++#ifndef KBASE_DEBUG_DISABLE_ASSERTS ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++#define KBASE_DEBUG_DISABLE_ASSERTS 0 ++#else ++#define KBASE_DEBUG_DISABLE_ASSERTS 1 ++#endif ++#endif /* KBASE_DEBUG_DISABLE_ASSERTS */ ++ ++/** Function type that is called on an KBASE_DEBUG_ASSERT() or KBASE_DEBUG_ASSERT_MSG() */ ++typedef void (kbase_debug_assert_hook) (void *); ++ ++struct kbasep_debug_assert_cb { ++ kbase_debug_assert_hook *func; ++ void *param; ++}; ++ ++/** ++ * @def KBASEP_DEBUG_PRINT_TRACE ++ * @brief Private macro containing the format of the trace to display before every message ++ * @sa KBASE_DEBUG_SKIP_TRACE, KBASE_DEBUG_SKIP_FUNCTION_NAME ++ */ ++#if !KBASE_DEBUG_SKIP_TRACE ++#define KBASEP_DEBUG_PRINT_TRACE \ ++ "In file: " __FILE__ " line: " CSTD_STR2(__LINE__) ++#if !KBASE_DEBUG_SKIP_FUNCTION_NAME ++#define KBASEP_DEBUG_PRINT_FUNCTION __func__ ++#else ++#define KBASEP_DEBUG_PRINT_FUNCTION "" ++#endif ++#else ++#define KBASEP_DEBUG_PRINT_TRACE "" ++#endif ++ ++/** ++ * @def KBASEP_DEBUG_ASSERT_OUT(trace, function, ...) ++ * @brief (Private) system printing function associated to the @ref KBASE_DEBUG_ASSERT_MSG event. ++ * @param trace location in the code from where the message is printed ++ * @param function function from where the message is printed ++ * @param ... Format string followed by format arguments. ++ * @note function parameter cannot be concatenated with other strings ++ */ ++/* Select the correct system output function*/ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++#define KBASEP_DEBUG_ASSERT_OUT(trace, function, ...)\ ++ do { \ ++ pr_err("Mali: %s function:%s ", trace, function);\ ++ pr_err(__VA_ARGS__);\ ++ pr_err("\n");\ ++ } while (false) ++#else ++#define KBASEP_DEBUG_ASSERT_OUT(trace, function, ...) CSTD_NOP() ++#endif ++ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++#define KBASE_CALL_ASSERT_HOOK() kbasep_debug_assert_call_hook() ++#else ++#define KBASE_CALL_ASSERT_HOOK() CSTD_NOP() ++#endif ++ ++/** ++ * @def KBASE_DEBUG_ASSERT(expr) ++ * @brief Calls @ref KBASE_PRINT_ASSERT and prints the expression @a expr if @a expr is false ++ * ++ * @note This macro does nothing if the flag @ref KBASE_DEBUG_DISABLE_ASSERTS is set to 1 ++ * ++ * @param expr Boolean expression ++ */ ++#define KBASE_DEBUG_ASSERT(expr) \ ++ KBASE_DEBUG_ASSERT_MSG(expr, #expr) ++ ++#if KBASE_DEBUG_DISABLE_ASSERTS ++#define KBASE_DEBUG_ASSERT_MSG(expr, ...) CSTD_NOP() ++#else ++ /** ++ * @def KBASE_DEBUG_ASSERT_MSG(expr, ...) ++ * @brief Calls @ref KBASEP_DEBUG_ASSERT_OUT and prints the given message if @a expr is false ++ * ++ * @note This macro does nothing if the flag @ref KBASE_DEBUG_DISABLE_ASSERTS is set to 1 ++ * ++ * @param expr Boolean expression ++ * @param ... Message to display when @a expr is false, as a format string followed by format arguments. ++ */ ++#define KBASE_DEBUG_ASSERT_MSG(expr, ...) \ ++ do { \ ++ if (!(expr)) { \ ++ KBASEP_DEBUG_ASSERT_OUT(KBASEP_DEBUG_PRINT_TRACE, KBASEP_DEBUG_PRINT_FUNCTION, __VA_ARGS__);\ ++ KBASE_CALL_ASSERT_HOOK();\ ++ BUG();\ ++ } \ ++ } while (false) ++#endif /* KBASE_DEBUG_DISABLE_ASSERTS */ ++ ++/** ++ * @def KBASE_DEBUG_CODE( X ) ++ * @brief Executes the code inside the macro only in debug mode ++ * ++ * @param X Code to compile only in debug mode. ++ */ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++#define KBASE_DEBUG_CODE(X) X ++#else ++#define KBASE_DEBUG_CODE(X) CSTD_NOP() ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++ ++/** @} */ ++ ++/** ++ * @brief Register a function to call on ASSERT ++ * ++ * Such functions will \b only be called during Debug mode, and for debugging ++ * features \b only. Do not rely on them to be called in general use. ++ * ++ * To disable the hook, supply NULL to \a func. ++ * ++ * @note This function is not thread-safe, and should only be used to ++ * register/deregister once in the module's lifetime. ++ * ++ * @param[in] func the function to call when an assert is triggered. ++ * @param[in] param the parameter to pass to \a func when calling it ++ */ ++void kbase_debug_assert_register_hook(kbase_debug_assert_hook *func, void *param); ++ ++/** ++ * @brief Call a debug assert hook previously registered with kbase_debug_assert_register_hook() ++ * ++ * @note This function is not thread-safe with respect to multiple threads ++ * registering functions and parameters with ++ * kbase_debug_assert_register_hook(). Otherwise, thread safety is the ++ * responsibility of the registered hook. ++ */ ++void kbasep_debug_assert_call_hook(void); ++ ++#endif /* _KBASE_DEBUG_H */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_debug_job_fault.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_debug_job_fault.c +new file mode 100755 +index 000000000000..f29430ddf8f9 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_debug_job_fault.c +@@ -0,0 +1,499 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include ++#include ++#include ++ ++#ifdef CONFIG_DEBUG_FS ++ ++static bool kbase_is_job_fault_event_pending(struct kbase_device *kbdev) ++{ ++ struct list_head *event_list = &kbdev->job_fault_event_list; ++ unsigned long flags; ++ bool ret; ++ ++ spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); ++ ret = !list_empty(event_list); ++ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); ++ ++ return ret; ++} ++ ++static bool kbase_ctx_has_no_event_pending(struct kbase_context *kctx) ++{ ++ struct kbase_device *kbdev = kctx->kbdev; ++ struct list_head *event_list = &kctx->kbdev->job_fault_event_list; ++ struct base_job_fault_event *event; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); ++ if (list_empty(event_list)) { ++ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); ++ return true; ++ } ++ list_for_each_entry(event, event_list, head) { ++ if (event->katom->kctx == kctx) { ++ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, ++ flags); ++ return false; ++ } ++ } ++ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); ++ return true; ++} ++ ++/* wait until the fault happen and copy the event */ ++static int kbase_job_fault_event_wait(struct kbase_device *kbdev, ++ struct base_job_fault_event *event) ++{ ++ struct list_head *event_list = &kbdev->job_fault_event_list; ++ struct base_job_fault_event *event_in; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); ++ if (list_empty(event_list)) { ++ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); ++ if (wait_event_interruptible(kbdev->job_fault_wq, ++ kbase_is_job_fault_event_pending(kbdev))) ++ return -ERESTARTSYS; ++ spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); ++ } ++ ++ event_in = list_entry(event_list->next, ++ struct base_job_fault_event, head); ++ event->event_code = event_in->event_code; ++ event->katom = event_in->katom; ++ ++ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); ++ ++ return 0; ++ ++} ++ ++/* remove the event from the queue */ ++static struct base_job_fault_event *kbase_job_fault_event_dequeue( ++ struct kbase_device *kbdev, struct list_head *event_list) ++{ ++ struct base_job_fault_event *event; ++ ++ event = list_entry(event_list->next, ++ struct base_job_fault_event, head); ++ list_del(event_list->next); ++ ++ return event; ++ ++} ++ ++/* Remove all the following atoms after the failed atom in the same context ++ * Call the postponed bottom half of job done. ++ * Then, this context could be rescheduled. ++ */ ++static void kbase_job_fault_resume_event_cleanup(struct kbase_context *kctx) ++{ ++ struct list_head *event_list = &kctx->job_fault_resume_event_list; ++ ++ while (!list_empty(event_list)) { ++ struct base_job_fault_event *event; ++ ++ event = kbase_job_fault_event_dequeue(kctx->kbdev, ++ &kctx->job_fault_resume_event_list); ++ kbase_jd_done_worker(&event->katom->work); ++ } ++ ++} ++ ++/* Remove all the failed atoms that belong to different contexts ++ * Resume all the contexts that were suspend due to failed job ++ */ ++static void kbase_job_fault_event_cleanup(struct kbase_device *kbdev) ++{ ++ struct list_head *event_list = &kbdev->job_fault_event_list; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); ++ while (!list_empty(event_list)) { ++ kbase_job_fault_event_dequeue(kbdev, event_list); ++ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); ++ wake_up(&kbdev->job_fault_resume_wq); ++ spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); ++ } ++ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); ++} ++ ++static void kbase_job_fault_resume_worker(struct work_struct *data) ++{ ++ struct base_job_fault_event *event = container_of(data, ++ struct base_job_fault_event, job_fault_work); ++ struct kbase_context *kctx; ++ struct kbase_jd_atom *katom; ++ ++ katom = event->katom; ++ kctx = katom->kctx; ++ ++ dev_info(kctx->kbdev->dev, "Job dumping wait\n"); ++ ++ /* When it was waked up, it need to check if queue is empty or the ++ * failed atom belongs to different context. If yes, wake up. Both ++ * of them mean the failed job has been dumped. Please note, it ++ * should never happen that the job_fault_event_list has the two ++ * atoms belong to the same context. ++ */ ++ wait_event(kctx->kbdev->job_fault_resume_wq, ++ kbase_ctx_has_no_event_pending(kctx)); ++ ++ atomic_set(&kctx->job_fault_count, 0); ++ kbase_jd_done_worker(&katom->work); ++ ++ /* In case the following atoms were scheduled during failed job dump ++ * the job_done_worker was held. We need to rerun it after the dump ++ * was finished ++ */ ++ kbase_job_fault_resume_event_cleanup(kctx); ++ ++ dev_info(kctx->kbdev->dev, "Job dumping finish, resume scheduler\n"); ++} ++ ++static struct base_job_fault_event *kbase_job_fault_event_queue( ++ struct list_head *event_list, ++ struct kbase_jd_atom *atom, ++ u32 completion_code) ++{ ++ struct base_job_fault_event *event; ++ ++ event = &atom->fault_event; ++ ++ event->katom = atom; ++ event->event_code = completion_code; ++ ++ list_add_tail(&event->head, event_list); ++ ++ return event; ++ ++} ++ ++static void kbase_job_fault_event_post(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom, u32 completion_code) ++{ ++ struct base_job_fault_event *event; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); ++ event = kbase_job_fault_event_queue(&kbdev->job_fault_event_list, ++ katom, completion_code); ++ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); ++ ++ wake_up_interruptible(&kbdev->job_fault_wq); ++ ++ INIT_WORK(&event->job_fault_work, kbase_job_fault_resume_worker); ++ queue_work(kbdev->job_fault_resume_workq, &event->job_fault_work); ++ ++ dev_info(katom->kctx->kbdev->dev, "Job fault happen, start dump: %d_%d", ++ katom->kctx->tgid, katom->kctx->id); ++ ++} ++ ++/* ++ * This function will process the job fault ++ * Get the register copy ++ * Send the failed job dump event ++ * Create a Wait queue to wait until the job dump finish ++ */ ++ ++bool kbase_debug_job_fault_process(struct kbase_jd_atom *katom, ++ u32 completion_code) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ ++ /* Check if dumping is in the process ++ * only one atom of each context can be dumped at the same time ++ * If the atom belongs to different context, it can be dumped ++ */ ++ if (atomic_read(&kctx->job_fault_count) > 0) { ++ kbase_job_fault_event_queue( ++ &kctx->job_fault_resume_event_list, ++ katom, completion_code); ++ dev_info(kctx->kbdev->dev, "queue:%d\n", ++ kbase_jd_atom_id(kctx, katom)); ++ return true; ++ } ++ ++ if (kctx->kbdev->job_fault_debug == true) { ++ ++ if (completion_code != BASE_JD_EVENT_DONE) { ++ ++ if (kbase_job_fault_get_reg_snapshot(kctx) == false) { ++ dev_warn(kctx->kbdev->dev, "get reg dump failed\n"); ++ return false; ++ } ++ ++ kbase_job_fault_event_post(kctx->kbdev, katom, ++ completion_code); ++ atomic_inc(&kctx->job_fault_count); ++ dev_info(kctx->kbdev->dev, "post:%d\n", ++ kbase_jd_atom_id(kctx, katom)); ++ return true; ++ ++ } ++ } ++ return false; ++ ++} ++ ++static int debug_job_fault_show(struct seq_file *m, void *v) ++{ ++ struct kbase_device *kbdev = m->private; ++ struct base_job_fault_event *event = (struct base_job_fault_event *)v; ++ struct kbase_context *kctx = event->katom->kctx; ++ int i; ++ ++ dev_info(kbdev->dev, "debug job fault seq show:%d_%d, %d", ++ kctx->tgid, kctx->id, event->reg_offset); ++ ++ if (kctx->reg_dump == NULL) { ++ dev_warn(kbdev->dev, "reg dump is NULL"); ++ return -1; ++ } ++ ++ if (kctx->reg_dump[event->reg_offset] == ++ REGISTER_DUMP_TERMINATION_FLAG) { ++ /* Return the error here to stop the read. And the ++ * following next() will not be called. The stop can ++ * get the real event resource and release it ++ */ ++ return -1; ++ } ++ ++ if (event->reg_offset == 0) ++ seq_printf(m, "%d_%d\n", kctx->tgid, kctx->id); ++ ++ for (i = 0; i < 50; i++) { ++ if (kctx->reg_dump[event->reg_offset] == ++ REGISTER_DUMP_TERMINATION_FLAG) { ++ break; ++ } ++ seq_printf(m, "%08x: %08x\n", ++ kctx->reg_dump[event->reg_offset], ++ kctx->reg_dump[1+event->reg_offset]); ++ event->reg_offset += 2; ++ ++ } ++ ++ ++ return 0; ++} ++static void *debug_job_fault_next(struct seq_file *m, void *v, loff_t *pos) ++{ ++ struct kbase_device *kbdev = m->private; ++ struct base_job_fault_event *event = (struct base_job_fault_event *)v; ++ ++ dev_info(kbdev->dev, "debug job fault seq next:%d, %d", ++ event->reg_offset, (int)*pos); ++ ++ return event; ++} ++ ++static void *debug_job_fault_start(struct seq_file *m, loff_t *pos) ++{ ++ struct kbase_device *kbdev = m->private; ++ struct base_job_fault_event *event; ++ ++ dev_info(kbdev->dev, "fault job seq start:%d", (int)*pos); ++ ++ /* The condition is trick here. It needs make sure the ++ * fault hasn't happened and the dumping hasn't been started, ++ * or the dumping has finished ++ */ ++ if (*pos == 0) { ++ event = kmalloc(sizeof(*event), GFP_KERNEL); ++ if (!event) ++ return NULL; ++ event->reg_offset = 0; ++ if (kbase_job_fault_event_wait(kbdev, event)) { ++ kfree(event); ++ return NULL; ++ } ++ ++ /* The cache flush workaround is called in bottom half of ++ * job done but we delayed it. Now we should clean cache ++ * earlier. Then the GPU memory dump should be correct. ++ */ ++ kbase_backend_cacheclean(kbdev, event->katom); ++ } else ++ return NULL; ++ ++ return event; ++} ++ ++static void debug_job_fault_stop(struct seq_file *m, void *v) ++{ ++ struct kbase_device *kbdev = m->private; ++ ++ /* here we wake up the kbase_jd_done_worker after stop, it needs ++ * get the memory dump before the register dump in debug daemon, ++ * otherwise, the memory dump may be incorrect. ++ */ ++ ++ if (v != NULL) { ++ kfree(v); ++ dev_info(kbdev->dev, "debug job fault seq stop stage 1"); ++ ++ } else { ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); ++ if (!list_empty(&kbdev->job_fault_event_list)) { ++ kbase_job_fault_event_dequeue(kbdev, ++ &kbdev->job_fault_event_list); ++ wake_up(&kbdev->job_fault_resume_wq); ++ } ++ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); ++ dev_info(kbdev->dev, "debug job fault seq stop stage 2"); ++ } ++ ++} ++ ++static const struct seq_operations ops = { ++ .start = debug_job_fault_start, ++ .next = debug_job_fault_next, ++ .stop = debug_job_fault_stop, ++ .show = debug_job_fault_show, ++}; ++ ++static int debug_job_fault_open(struct inode *in, struct file *file) ++{ ++ struct kbase_device *kbdev = in->i_private; ++ ++ seq_open(file, &ops); ++ ++ ((struct seq_file *)file->private_data)->private = kbdev; ++ dev_info(kbdev->dev, "debug job fault seq open"); ++ ++ kbdev->job_fault_debug = true; ++ ++ return 0; ++ ++} ++ ++static int debug_job_fault_release(struct inode *in, struct file *file) ++{ ++ struct kbase_device *kbdev = in->i_private; ++ ++ seq_release(in, file); ++ ++ kbdev->job_fault_debug = false; ++ ++ /* Clean the unprocessed job fault. After that, all the suspended ++ * contexts could be rescheduled. ++ */ ++ kbase_job_fault_event_cleanup(kbdev); ++ ++ dev_info(kbdev->dev, "debug job fault seq close"); ++ ++ return 0; ++} ++ ++static const struct file_operations kbasep_debug_job_fault_fops = { ++ .open = debug_job_fault_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = debug_job_fault_release, ++}; ++ ++/* ++ * Initialize debugfs entry for job fault dump ++ */ ++void kbase_debug_job_fault_debugfs_init(struct kbase_device *kbdev) ++{ ++ debugfs_create_file("job_fault", S_IRUGO, ++ kbdev->mali_debugfs_directory, kbdev, ++ &kbasep_debug_job_fault_fops); ++} ++ ++ ++int kbase_debug_job_fault_dev_init(struct kbase_device *kbdev) ++{ ++ ++ INIT_LIST_HEAD(&kbdev->job_fault_event_list); ++ ++ init_waitqueue_head(&(kbdev->job_fault_wq)); ++ init_waitqueue_head(&(kbdev->job_fault_resume_wq)); ++ spin_lock_init(&kbdev->job_fault_event_lock); ++ ++ kbdev->job_fault_resume_workq = alloc_workqueue( ++ "kbase_job_fault_resume_work_queue", WQ_MEM_RECLAIM, 1); ++ if (!kbdev->job_fault_resume_workq) ++ return -ENOMEM; ++ ++ kbdev->job_fault_debug = false; ++ ++ return 0; ++} ++ ++/* ++ * Release the relevant resource per device ++ */ ++void kbase_debug_job_fault_dev_term(struct kbase_device *kbdev) ++{ ++ destroy_workqueue(kbdev->job_fault_resume_workq); ++} ++ ++ ++/* ++ * Initialize the relevant data structure per context ++ */ ++void kbase_debug_job_fault_context_init(struct kbase_context *kctx) ++{ ++ ++ /* We need allocate double size register range ++ * Because this memory will keep the register address and value ++ */ ++ kctx->reg_dump = vmalloc(0x4000 * 2); ++ if (kctx->reg_dump == NULL) ++ return; ++ ++ if (kbase_debug_job_fault_reg_snapshot_init(kctx, 0x4000) == false) { ++ vfree(kctx->reg_dump); ++ kctx->reg_dump = NULL; ++ } ++ INIT_LIST_HEAD(&kctx->job_fault_resume_event_list); ++ atomic_set(&kctx->job_fault_count, 0); ++ ++} ++ ++/* ++ * release the relevant resource per context ++ */ ++void kbase_debug_job_fault_context_term(struct kbase_context *kctx) ++{ ++ vfree(kctx->reg_dump); ++} ++ ++#else /* CONFIG_DEBUG_FS */ ++ ++int kbase_debug_job_fault_dev_init(struct kbase_device *kbdev) ++{ ++ kbdev->job_fault_debug = false; ++ ++ return 0; ++} ++ ++void kbase_debug_job_fault_dev_term(struct kbase_device *kbdev) ++{ ++} ++ ++#endif /* CONFIG_DEBUG_FS */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_debug_job_fault.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_debug_job_fault.h +new file mode 100755 +index 000000000000..a2bf8983c37c +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_debug_job_fault.h +@@ -0,0 +1,96 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KBASE_DEBUG_JOB_FAULT_H ++#define _KBASE_DEBUG_JOB_FAULT_H ++ ++#include ++#include ++ ++#define REGISTER_DUMP_TERMINATION_FLAG 0xFFFFFFFF ++ ++/** ++ * kbase_debug_job_fault_dev_init - Create the fault event wait queue ++ * per device and initialize the required lists. ++ * @kbdev: Device pointer ++ * ++ * Return: Zero on success or a negative error code. ++ */ ++int kbase_debug_job_fault_dev_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_debug_job_fault_debugfs_init - Initialize job fault debug sysfs ++ * @kbdev: Device pointer ++ */ ++void kbase_debug_job_fault_debugfs_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_debug_job_fault_dev_term - Clean up resources created in ++ * kbase_debug_job_fault_dev_init. ++ * @kbdev: Device pointer ++ */ ++void kbase_debug_job_fault_dev_term(struct kbase_device *kbdev); ++ ++/** ++ * kbase_debug_job_fault_context_init - Initialize the relevant ++ * data structure per context ++ * @kctx: KBase context pointer ++ */ ++void kbase_debug_job_fault_context_init(struct kbase_context *kctx); ++ ++/** ++ * kbase_debug_job_fault_context_term - Release the relevant ++ * resource per context ++ * @kctx: KBase context pointer ++ */ ++void kbase_debug_job_fault_context_term(struct kbase_context *kctx); ++ ++/** ++ * kbase_debug_job_fault_process - Process the failed job. ++ * It will send a event and wake up the job fault waiting queue ++ * Then create a work queue to wait for job dump finish ++ * This function should be called in the interrupt handler and before ++ * jd_done that make sure the jd_done_worker will be delayed until the ++ * job dump finish ++ * @katom: The failed atom pointer ++ * @completion_code: the job status ++ * @return true if dump is going on ++ */ ++bool kbase_debug_job_fault_process(struct kbase_jd_atom *katom, ++ u32 completion_code); ++ ++ ++/** ++ * kbase_debug_job_fault_reg_snapshot_init - Set the interested registers ++ * address during the job fault process, the relevant registers will ++ * be saved when a job fault happen ++ * @kctx: KBase context pointer ++ * @reg_range: Maximum register address space ++ * @return true if initializing successfully ++ */ ++bool kbase_debug_job_fault_reg_snapshot_init(struct kbase_context *kctx, ++ int reg_range); ++ ++/** ++ * kbase_job_fault_get_reg_snapshot - Read the interested registers for ++ * failed job dump ++ * @kctx: KBase context pointer ++ * @return true if getting registers successfully ++ */ ++bool kbase_job_fault_get_reg_snapshot(struct kbase_context *kctx); ++ ++#endif /*_KBASE_DEBUG_JOB_FAULT_H*/ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_debug_mem_view.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_debug_mem_view.c +new file mode 100755 +index 000000000000..aa271566e917 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_debug_mem_view.c +@@ -0,0 +1,306 @@ ++/* ++ * ++ * (C) COPYRIGHT 2013-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* ++ * Debugfs interface to dump the memory visible to the GPU ++ */ ++ ++#include "mali_kbase_debug_mem_view.h" ++#include "mali_kbase.h" ++ ++#include ++#include ++ ++#ifdef CONFIG_DEBUG_FS ++ ++struct debug_mem_mapping { ++ struct list_head node; ++ ++ struct kbase_mem_phy_alloc *alloc; ++ unsigned long flags; ++ ++ u64 start_pfn; ++ size_t nr_pages; ++}; ++ ++struct debug_mem_data { ++ struct list_head mapping_list; ++ struct kbase_context *kctx; ++}; ++ ++struct debug_mem_seq_off { ++ struct list_head *lh; ++ size_t offset; ++}; ++ ++static void *debug_mem_start(struct seq_file *m, loff_t *_pos) ++{ ++ struct debug_mem_data *mem_data = m->private; ++ struct debug_mem_seq_off *data; ++ struct debug_mem_mapping *map; ++ loff_t pos = *_pos; ++ ++ list_for_each_entry(map, &mem_data->mapping_list, node) { ++ if (pos >= map->nr_pages) { ++ pos -= map->nr_pages; ++ } else { ++ data = kmalloc(sizeof(*data), GFP_KERNEL); ++ if (!data) ++ return NULL; ++ data->lh = &map->node; ++ data->offset = pos; ++ return data; ++ } ++ } ++ ++ /* Beyond the end */ ++ return NULL; ++} ++ ++static void debug_mem_stop(struct seq_file *m, void *v) ++{ ++ kfree(v); ++} ++ ++static void *debug_mem_next(struct seq_file *m, void *v, loff_t *pos) ++{ ++ struct debug_mem_data *mem_data = m->private; ++ struct debug_mem_seq_off *data = v; ++ struct debug_mem_mapping *map; ++ ++ map = list_entry(data->lh, struct debug_mem_mapping, node); ++ ++ if (data->offset < map->nr_pages - 1) { ++ data->offset++; ++ ++*pos; ++ return data; ++ } ++ ++ if (list_is_last(data->lh, &mem_data->mapping_list)) { ++ kfree(data); ++ return NULL; ++ } ++ ++ data->lh = data->lh->next; ++ data->offset = 0; ++ ++*pos; ++ ++ return data; ++} ++ ++static int debug_mem_show(struct seq_file *m, void *v) ++{ ++ struct debug_mem_data *mem_data = m->private; ++ struct debug_mem_seq_off *data = v; ++ struct debug_mem_mapping *map; ++ int i, j; ++ struct page *page; ++ uint32_t *mapping; ++ pgprot_t prot = PAGE_KERNEL; ++ ++ map = list_entry(data->lh, struct debug_mem_mapping, node); ++ ++ kbase_gpu_vm_lock(mem_data->kctx); ++ ++ if (data->offset >= map->alloc->nents) { ++ seq_printf(m, "%016llx: Unbacked page\n\n", (map->start_pfn + ++ data->offset) << PAGE_SHIFT); ++ goto out; ++ } ++ ++ if (!(map->flags & KBASE_REG_CPU_CACHED)) ++ prot = pgprot_writecombine(prot); ++ ++ page = phys_to_page(as_phys_addr_t(map->alloc->pages[data->offset])); ++ mapping = vmap(&page, 1, VM_MAP, prot); ++ if (!mapping) ++ goto out; ++ ++ for (i = 0; i < PAGE_SIZE; i += 4*sizeof(*mapping)) { ++ seq_printf(m, "%016llx:", i + ((map->start_pfn + ++ data->offset) << PAGE_SHIFT)); ++ ++ for (j = 0; j < 4*sizeof(*mapping); j += sizeof(*mapping)) ++ seq_printf(m, " %08x", mapping[(i+j)/sizeof(*mapping)]); ++ seq_putc(m, '\n'); ++ } ++ ++ vunmap(mapping); ++ ++ seq_putc(m, '\n'); ++ ++out: ++ kbase_gpu_vm_unlock(mem_data->kctx); ++ return 0; ++} ++ ++static const struct seq_operations ops = { ++ .start = debug_mem_start, ++ .next = debug_mem_next, ++ .stop = debug_mem_stop, ++ .show = debug_mem_show, ++}; ++ ++static int debug_mem_zone_open(struct rb_root *rbtree, ++ struct debug_mem_data *mem_data) ++{ ++ int ret = 0; ++ struct rb_node *p; ++ struct kbase_va_region *reg; ++ struct debug_mem_mapping *mapping; ++ ++ for (p = rb_first(rbtree); p; p = rb_next(p)) { ++ reg = rb_entry(p, struct kbase_va_region, rblink); ++ ++ if (reg->gpu_alloc == NULL) ++ /* Empty region - ignore */ ++ continue; ++ ++ mapping = kmalloc(sizeof(*mapping), GFP_KERNEL); ++ if (!mapping) { ++ ret = -ENOMEM; ++ goto out; ++ } ++ ++ mapping->alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); ++ mapping->start_pfn = reg->start_pfn; ++ mapping->nr_pages = reg->nr_pages; ++ mapping->flags = reg->flags; ++ list_add_tail(&mapping->node, &mem_data->mapping_list); ++ } ++ ++out: ++ return ret; ++} ++ ++static int debug_mem_open(struct inode *i, struct file *file) ++{ ++ struct file *kctx_file = i->i_private; ++ struct kbase_context *kctx = kctx_file->private_data; ++ struct debug_mem_data *mem_data; ++ int ret; ++ ++ ret = seq_open(file, &ops); ++ if (ret) ++ return ret; ++ ++ mem_data = kmalloc(sizeof(*mem_data), GFP_KERNEL); ++ if (!mem_data) { ++ ret = -ENOMEM; ++ goto out; ++ } ++ ++ mem_data->kctx = kctx; ++ ++ INIT_LIST_HEAD(&mem_data->mapping_list); ++ ++ get_file(kctx_file); ++ ++ kbase_gpu_vm_lock(kctx); ++ ++ ret = debug_mem_zone_open(&kctx->reg_rbtree_same, mem_data); ++ if (0 != ret) { ++ kbase_gpu_vm_unlock(kctx); ++ goto out; ++ } ++ ++ ret = debug_mem_zone_open(&kctx->reg_rbtree_exec, mem_data); ++ if (0 != ret) { ++ kbase_gpu_vm_unlock(kctx); ++ goto out; ++ } ++ ++ ret = debug_mem_zone_open(&kctx->reg_rbtree_custom, mem_data); ++ if (0 != ret) { ++ kbase_gpu_vm_unlock(kctx); ++ goto out; ++ } ++ ++ kbase_gpu_vm_unlock(kctx); ++ ++ ((struct seq_file *)file->private_data)->private = mem_data; ++ ++ return 0; ++ ++out: ++ if (mem_data) { ++ while (!list_empty(&mem_data->mapping_list)) { ++ struct debug_mem_mapping *mapping; ++ ++ mapping = list_first_entry(&mem_data->mapping_list, ++ struct debug_mem_mapping, node); ++ kbase_mem_phy_alloc_put(mapping->alloc); ++ list_del(&mapping->node); ++ kfree(mapping); ++ } ++ fput(kctx_file); ++ kfree(mem_data); ++ } ++ seq_release(i, file); ++ return ret; ++} ++ ++static int debug_mem_release(struct inode *inode, struct file *file) ++{ ++ struct file *kctx_file = inode->i_private; ++ struct seq_file *sfile = file->private_data; ++ struct debug_mem_data *mem_data = sfile->private; ++ struct debug_mem_mapping *mapping; ++ ++ seq_release(inode, file); ++ ++ while (!list_empty(&mem_data->mapping_list)) { ++ mapping = list_first_entry(&mem_data->mapping_list, ++ struct debug_mem_mapping, node); ++ kbase_mem_phy_alloc_put(mapping->alloc); ++ list_del(&mapping->node); ++ kfree(mapping); ++ } ++ ++ kfree(mem_data); ++ ++ fput(kctx_file); ++ ++ return 0; ++} ++ ++static const struct file_operations kbase_debug_mem_view_fops = { ++ .open = debug_mem_open, ++ .release = debug_mem_release, ++ .read = seq_read, ++ .llseek = seq_lseek ++}; ++ ++/** ++ * kbase_debug_mem_view_init - Initialise the mem_view sysfs file ++ * @kctx_file: The /dev/mali0 file instance for the context ++ * ++ * This function creates a "mem_view" file which can be used to get a view of ++ * the context's memory as the GPU sees it (i.e. using the GPU's page tables). ++ * ++ * The file is cleaned up by a call to debugfs_remove_recursive() deleting the ++ * parent directory. ++ */ ++void kbase_debug_mem_view_init(struct file *kctx_file) ++{ ++ struct kbase_context *kctx = kctx_file->private_data; ++ ++ debugfs_create_file("mem_view", S_IRUGO, kctx->kctx_dentry, kctx_file, ++ &kbase_debug_mem_view_fops); ++} ++ ++#endif +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_debug_mem_view.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_debug_mem_view.h +new file mode 100755 +index 000000000000..20ab51a776c6 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_debug_mem_view.h +@@ -0,0 +1,25 @@ ++/* ++ * ++ * (C) COPYRIGHT 2013-2014 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KBASE_DEBUG_MEM_VIEW_H ++#define _KBASE_DEBUG_MEM_VIEW_H ++ ++#include ++ ++void kbase_debug_mem_view_init(struct file *kctx_file); ++ ++#endif +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_defs.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_defs.h +new file mode 100755 +index 000000000000..73721f5da139 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_defs.h +@@ -0,0 +1,1641 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/** ++ * @file mali_kbase_defs.h ++ * ++ * Defintions (types, defines, etcs) common to Kbase. They are placed here to ++ * allow the hierarchy of header files to work. ++ */ ++ ++#ifndef _KBASE_DEFS_H_ ++#define _KBASE_DEFS_H_ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#include ++#include ++#include ++#include ++ ++#ifdef CONFIG_MALI_FPGA_BUS_LOGGER ++#include ++#endif ++ ++ ++#ifdef CONFIG_KDS ++#include ++#endif /* CONFIG_KDS */ ++ ++#if defined(CONFIG_SYNC) ++#include ++#else ++#include "mali_kbase_fence_defs.h" ++#endif ++ ++#ifdef CONFIG_DEBUG_FS ++#include ++#endif /* CONFIG_DEBUG_FS */ ++ ++#ifdef CONFIG_MALI_BIFROST_DEVFREQ ++#include ++#endif /* CONFIG_MALI_BIFROST_DEVFREQ */ ++ ++#include ++#include ++ ++#if defined(CONFIG_PM_RUNTIME) || \ ++ (defined(CONFIG_PM) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0)) ++#define KBASE_PM_RUNTIME 1 ++#endif ++ ++/** Enable SW tracing when set */ ++#ifdef CONFIG_MALI_BIFROST_ENABLE_TRACE ++#define KBASE_TRACE_ENABLE 1 ++#endif ++ ++#ifndef KBASE_TRACE_ENABLE ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++#define KBASE_TRACE_ENABLE 1 ++#else ++#define KBASE_TRACE_ENABLE 0 ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++#endif /* KBASE_TRACE_ENABLE */ ++ ++/** Dump Job slot trace on error (only active if KBASE_TRACE_ENABLE != 0) */ ++#define KBASE_TRACE_DUMP_ON_JOB_SLOT_ERROR 1 ++ ++/** ++ * Number of milliseconds before resetting the GPU when a job cannot be "zapped" from the hardware. ++ * Note that the time is actually ZAP_TIMEOUT+SOFT_STOP_RESET_TIMEOUT between the context zap starting and the GPU ++ * actually being reset to give other contexts time for their jobs to be soft-stopped and removed from the hardware ++ * before resetting. ++ */ ++#define ZAP_TIMEOUT 1000 ++ ++/** Number of milliseconds before we time out on a GPU soft/hard reset */ ++#define RESET_TIMEOUT 500 ++ ++/** ++ * Prevent soft-stops from occuring in scheduling situations ++ * ++ * This is not due to HW issues, but when scheduling is desired to be more predictable. ++ * ++ * Therefore, soft stop may still be disabled due to HW issues. ++ * ++ * @note Soft stop will still be used for non-scheduling purposes e.g. when terminating a context. ++ * ++ * @note if not in use, define this value to 0 instead of \#undef'ing it ++ */ ++#define KBASE_DISABLE_SCHEDULING_SOFT_STOPS 0 ++ ++/** ++ * Prevent hard-stops from occuring in scheduling situations ++ * ++ * This is not due to HW issues, but when scheduling is desired to be more predictable. ++ * ++ * @note Hard stop will still be used for non-scheduling purposes e.g. when terminating a context. ++ * ++ * @note if not in use, define this value to 0 instead of \#undef'ing it ++ */ ++#define KBASE_DISABLE_SCHEDULING_HARD_STOPS 0 ++ ++/** ++ * The maximum number of Job Slots to support in the Hardware. ++ * ++ * You can optimize this down if your target devices will only ever support a ++ * small number of job slots. ++ */ ++#define BASE_JM_MAX_NR_SLOTS 3 ++ ++/** ++ * The maximum number of Address Spaces to support in the Hardware. ++ * ++ * You can optimize this down if your target devices will only ever support a ++ * small number of Address Spaces ++ */ ++#define BASE_MAX_NR_AS 16 ++ ++/* mmu */ ++#define MIDGARD_MMU_VA_BITS 48 ++ ++#define MIDGARD_MMU_LEVEL(x) (x) ++ ++#if MIDGARD_MMU_VA_BITS > 39 ++#define MIDGARD_MMU_TOPLEVEL MIDGARD_MMU_LEVEL(0) ++#else ++#define MIDGARD_MMU_TOPLEVEL MIDGARD_MMU_LEVEL(1) ++#endif ++ ++#define MIDGARD_MMU_BOTTOMLEVEL MIDGARD_MMU_LEVEL(3) ++ ++#define GROWABLE_FLAGS_REQUIRED (KBASE_REG_PF_GROW | KBASE_REG_GPU_WR) ++ ++/** setting in kbase_context::as_nr that indicates it's invalid */ ++#define KBASEP_AS_NR_INVALID (-1) ++ ++#define KBASE_LOCK_REGION_MAX_SIZE (63) ++#define KBASE_LOCK_REGION_MIN_SIZE (11) ++ ++#define KBASE_TRACE_SIZE_LOG2 8 /* 256 entries */ ++#define KBASE_TRACE_SIZE (1 << KBASE_TRACE_SIZE_LOG2) ++#define KBASE_TRACE_MASK ((1 << KBASE_TRACE_SIZE_LOG2)-1) ++ ++#include "mali_kbase_js_defs.h" ++#include "mali_kbase_hwaccess_defs.h" ++ ++#define KBASEP_FORCE_REPLAY_DISABLED 0 ++ ++/* Maximum force replay limit when randomization is enabled */ ++#define KBASEP_FORCE_REPLAY_RANDOM_LIMIT 16 ++ ++/** Atom has been previously soft-stoppped */ ++#define KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED (1<<1) ++/** Atom has been previously retried to execute */ ++#define KBASE_KATOM_FLAGS_RERUN (1<<2) ++#define KBASE_KATOM_FLAGS_JOBCHAIN (1<<3) ++/** Atom has been previously hard-stopped. */ ++#define KBASE_KATOM_FLAG_BEEN_HARD_STOPPED (1<<4) ++/** Atom has caused us to enter disjoint state */ ++#define KBASE_KATOM_FLAG_IN_DISJOINT (1<<5) ++/* Atom blocked on cross-slot dependency */ ++#define KBASE_KATOM_FLAG_X_DEP_BLOCKED (1<<7) ++/* Atom has fail dependency on cross-slot dependency */ ++#define KBASE_KATOM_FLAG_FAIL_BLOCKER (1<<8) ++/* Atom is currently in the list of atoms blocked on cross-slot dependencies */ ++#define KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST (1<<9) ++/* Atom is currently holding a context reference */ ++#define KBASE_KATOM_FLAG_HOLDING_CTX_REF (1<<10) ++/* Atom requires GPU to be in protected mode */ ++#define KBASE_KATOM_FLAG_PROTECTED (1<<11) ++/* Atom has been stored in runnable_tree */ ++#define KBASE_KATOM_FLAG_JSCTX_IN_TREE (1<<12) ++ ++/* SW related flags about types of JS_COMMAND action ++ * NOTE: These must be masked off by JS_COMMAND_MASK */ ++ ++/** This command causes a disjoint event */ ++#define JS_COMMAND_SW_CAUSES_DISJOINT 0x100 ++ ++/** Bitmask of all SW related flags */ ++#define JS_COMMAND_SW_BITS (JS_COMMAND_SW_CAUSES_DISJOINT) ++ ++#if (JS_COMMAND_SW_BITS & JS_COMMAND_MASK) ++#error JS_COMMAND_SW_BITS not masked off by JS_COMMAND_MASK. Must update JS_COMMAND_SW_<..> bitmasks ++#endif ++ ++/** Soft-stop command that causes a Disjoint event. This of course isn't ++ * entirely masked off by JS_COMMAND_MASK */ ++#define JS_COMMAND_SOFT_STOP_WITH_SW_DISJOINT \ ++ (JS_COMMAND_SW_CAUSES_DISJOINT | JS_COMMAND_SOFT_STOP) ++ ++#define KBASEP_ATOM_ID_INVALID BASE_JD_ATOM_COUNT ++ ++/* Serialize atoms within a slot (ie only one atom per job slot) */ ++#define KBASE_SERIALIZE_INTRA_SLOT (1 << 0) ++/* Serialize atoms between slots (ie only one job slot running at any time) */ ++#define KBASE_SERIALIZE_INTER_SLOT (1 << 1) ++/* Reset the GPU after each atom completion */ ++#define KBASE_SERIALIZE_RESET (1 << 2) ++ ++/* Forward declarations */ ++struct kbase_context; ++struct kbase_device; ++struct kbase_as; ++struct kbase_mmu_setup; ++ ++#ifdef CONFIG_DEBUG_FS ++struct base_job_fault_event { ++ ++ u32 event_code; ++ struct kbase_jd_atom *katom; ++ struct work_struct job_fault_work; ++ struct list_head head; ++ int reg_offset; ++}; ++ ++#endif ++ ++struct kbase_jd_atom_dependency { ++ struct kbase_jd_atom *atom; ++ u8 dep_type; ++}; ++ ++/** ++ * struct kbase_io_access - holds information about 1 register access ++ * ++ * @addr: first bit indicates r/w (r=0, w=1) ++ * @value: value written or read ++ */ ++struct kbase_io_access { ++ uintptr_t addr; ++ u32 value; ++}; ++ ++/** ++ * struct kbase_io_history - keeps track of all recent register accesses ++ * ++ * @enabled: true if register accesses are recorded, false otherwise ++ * @lock: spinlock protecting kbase_io_access array ++ * @count: number of registers read/written ++ * @size: number of elements in kbase_io_access array ++ * @buf: array of kbase_io_access ++ */ ++struct kbase_io_history { ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0)) ++ bool enabled; ++#else ++ u32 enabled; ++#endif ++ ++ spinlock_t lock; ++ size_t count; ++ u16 size; ++ struct kbase_io_access *buf; ++}; ++ ++/** ++ * @brief The function retrieves a read-only reference to the atom field from ++ * the kbase_jd_atom_dependency structure ++ * ++ * @param[in] dep kbase jd atom dependency. ++ * ++ * @return readonly reference to dependent ATOM. ++ */ ++static inline const struct kbase_jd_atom * kbase_jd_katom_dep_atom(const struct kbase_jd_atom_dependency *dep) ++{ ++ LOCAL_ASSERT(dep != NULL); ++ ++ return (const struct kbase_jd_atom *)(dep->atom); ++} ++ ++/** ++ * @brief The function retrieves a read-only reference to the dependency type field from ++ * the kbase_jd_atom_dependency structure ++ * ++ * @param[in] dep kbase jd atom dependency. ++ * ++ * @return A dependency type value. ++ */ ++static inline u8 kbase_jd_katom_dep_type(const struct kbase_jd_atom_dependency *dep) ++{ ++ LOCAL_ASSERT(dep != NULL); ++ ++ return dep->dep_type; ++} ++ ++/** ++ * @brief Setter macro for dep_atom array entry in kbase_jd_atom ++ * ++ * @param[in] dep The kbase jd atom dependency. ++ * @param[in] a The ATOM to be set as a dependency. ++ * @param type The ATOM dependency type to be set. ++ * ++ */ ++static inline void kbase_jd_katom_dep_set(const struct kbase_jd_atom_dependency *const_dep, ++ struct kbase_jd_atom *a, u8 type) ++{ ++ struct kbase_jd_atom_dependency *dep; ++ ++ LOCAL_ASSERT(const_dep != NULL); ++ ++ dep = (struct kbase_jd_atom_dependency *)const_dep; ++ ++ dep->atom = a; ++ dep->dep_type = type; ++} ++ ++/** ++ * @brief Setter macro for dep_atom array entry in kbase_jd_atom ++ * ++ * @param[in] dep The kbase jd atom dependency to be cleared. ++ * ++ */ ++static inline void kbase_jd_katom_dep_clear(const struct kbase_jd_atom_dependency *const_dep) ++{ ++ struct kbase_jd_atom_dependency *dep; ++ ++ LOCAL_ASSERT(const_dep != NULL); ++ ++ dep = (struct kbase_jd_atom_dependency *)const_dep; ++ ++ dep->atom = NULL; ++ dep->dep_type = BASE_JD_DEP_TYPE_INVALID; ++} ++ ++enum kbase_atom_gpu_rb_state { ++ /* Atom is not currently present in slot ringbuffer */ ++ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB, ++ /* Atom is in slot ringbuffer but is blocked on a previous atom */ ++ KBASE_ATOM_GPU_RB_WAITING_BLOCKED, ++ /* Atom is in slot ringbuffer but is waiting for a previous protected ++ * mode transition to complete */ ++ KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV, ++ /* Atom is in slot ringbuffer but is waiting for proected mode ++ * transition */ ++ KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION, ++ /* Atom is in slot ringbuffer but is waiting for cores to become ++ * available */ ++ KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE, ++ /* Atom is in slot ringbuffer but is blocked on affinity */ ++ KBASE_ATOM_GPU_RB_WAITING_AFFINITY, ++ /* Atom is in slot ringbuffer and ready to run */ ++ KBASE_ATOM_GPU_RB_READY, ++ /* Atom is in slot ringbuffer and has been submitted to the GPU */ ++ KBASE_ATOM_GPU_RB_SUBMITTED, ++ /* Atom must be returned to JS as soon as it reaches the head of the ++ * ringbuffer due to a previous failure */ ++ KBASE_ATOM_GPU_RB_RETURN_TO_JS = -1 ++}; ++ ++enum kbase_atom_enter_protected_state { ++ /* ++ * Starting state: ++ * Check if a transition into protected mode is required. ++ * ++ * NOTE: The integer value of this must ++ * match KBASE_ATOM_EXIT_PROTECTED_CHECK. ++ */ ++ KBASE_ATOM_ENTER_PROTECTED_CHECK = 0, ++ /* Wait for vinstr to suspend. */ ++ KBASE_ATOM_ENTER_PROTECTED_VINSTR, ++ /* Wait for the L2 to become idle in preparation for ++ * the coherency change. */ ++ KBASE_ATOM_ENTER_PROTECTED_IDLE_L2, ++ /* End state; ++ * Prepare coherency change. */ ++ KBASE_ATOM_ENTER_PROTECTED_FINISHED, ++}; ++ ++enum kbase_atom_exit_protected_state { ++ /* ++ * Starting state: ++ * Check if a transition out of protected mode is required. ++ * ++ * NOTE: The integer value of this must ++ * match KBASE_ATOM_ENTER_PROTECTED_CHECK. ++ */ ++ KBASE_ATOM_EXIT_PROTECTED_CHECK = 0, ++ /* Wait for the L2 to become idle in preparation ++ * for the reset. */ ++ KBASE_ATOM_EXIT_PROTECTED_IDLE_L2, ++ /* Issue the protected reset. */ ++ KBASE_ATOM_EXIT_PROTECTED_RESET, ++ /* End state; ++ * Wait for the reset to complete. */ ++ KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT, ++}; ++ ++struct kbase_ext_res { ++ u64 gpu_address; ++ struct kbase_mem_phy_alloc *alloc; ++}; ++ ++struct kbase_jd_atom { ++ struct work_struct work; ++ ktime_t start_timestamp; ++ ++ struct base_jd_udata udata; ++ struct kbase_context *kctx; ++ ++ struct list_head dep_head[2]; ++ struct list_head dep_item[2]; ++ const struct kbase_jd_atom_dependency dep[2]; ++ /* List head used during job dispatch job_done processing - as ++ * dependencies may not be entirely resolved at this point, we need to ++ * use a separate list head. */ ++ struct list_head jd_item; ++ /* true if atom's jd_item is currently on a list. Prevents atom being ++ * processed twice. */ ++ bool in_jd_list; ++ ++ u16 nr_extres; ++ struct kbase_ext_res *extres; ++ ++ u32 device_nr; ++ u64 affinity; ++ u64 jc; ++ enum kbase_atom_coreref_state coreref_state; ++#ifdef CONFIG_KDS ++ struct list_head node; ++ struct kds_resource_set *kds_rset; ++ bool kds_dep_satisfied; ++#endif /* CONFIG_KDS */ ++#if defined(CONFIG_SYNC) ++ /* Stores either an input or output fence, depending on soft-job type */ ++ struct sync_fence *fence; ++ struct sync_fence_waiter sync_waiter; ++#endif /* CONFIG_SYNC */ ++#if defined(CONFIG_MALI_BIFROST_DMA_FENCE) || defined(CONFIG_SYNC_FILE) ++ struct { ++ /* Use the functions/API defined in mali_kbase_fence.h to ++ * when working with this sub struct */ ++#if defined(CONFIG_SYNC_FILE) ++ /* Input fence */ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ struct fence *fence_in; ++#else ++ struct dma_fence *fence_in; ++#endif ++#endif ++ /* This points to the dma-buf output fence for this atom. If ++ * this is NULL then there is no fence for this atom and the ++ * following fields related to dma_fence may have invalid data. ++ * ++ * The context and seqno fields contain the details for this ++ * fence. ++ * ++ * This fence is signaled when the katom is completed, ++ * regardless of the event_code of the katom (signal also on ++ * failure). ++ */ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ struct fence *fence; ++#else ++ struct dma_fence *fence; ++#endif ++ /* The dma-buf fence context number for this atom. A unique ++ * context number is allocated to each katom in the context on ++ * context creation. ++ */ ++ unsigned int context; ++ /* The dma-buf fence sequence number for this atom. This is ++ * increased every time this katom uses dma-buf fence. ++ */ ++ atomic_t seqno; ++ /* This contains a list of all callbacks set up to wait on ++ * other fences. This atom must be held back from JS until all ++ * these callbacks have been called and dep_count have reached ++ * 0. The initial value of dep_count must be equal to the ++ * number of callbacks on this list. ++ * ++ * This list is protected by jctx.lock. Callbacks are added to ++ * this list when the atom is built and the wait are set up. ++ * All the callbacks then stay on the list until all callbacks ++ * have been called and the atom is queued, or cancelled, and ++ * then all callbacks are taken off the list and freed. ++ */ ++ struct list_head callbacks; ++ /* Atomic counter of number of outstandind dma-buf fence ++ * dependencies for this atom. When dep_count reaches 0 the ++ * atom may be queued. ++ * ++ * The special value "-1" may only be set after the count ++ * reaches 0, while holding jctx.lock. This indicates that the ++ * atom has been handled, either queued in JS or cancelled. ++ * ++ * If anyone but the dma-fence worker sets this to -1 they must ++ * ensure that any potentially queued worker must have ++ * completed before allowing the atom to be marked as unused. ++ * This can be done by flushing the fence work queue: ++ * kctx->dma_fence.wq. ++ */ ++ atomic_t dep_count; ++ } dma_fence; ++#endif /* CONFIG_MALI_BIFROST_DMA_FENCE || CONFIG_SYNC_FILE*/ ++ ++ /* Note: refer to kbasep_js_atom_retained_state, which will take a copy of some of the following members */ ++ enum base_jd_event_code event_code; ++ base_jd_core_req core_req; /**< core requirements */ ++ /** Job Slot to retry submitting to if submission from IRQ handler failed ++ * ++ * NOTE: see if this can be unified into the another member e.g. the event */ ++ int retry_submit_on_slot; ++ ++ u32 ticks; ++ /* JS atom priority with respect to other atoms on its kctx. */ ++ int sched_priority; ++ ++ int poking; /* BASE_HW_ISSUE_8316 */ ++ ++ wait_queue_head_t completed; ++ enum kbase_jd_atom_state status; ++#ifdef CONFIG_GPU_TRACEPOINTS ++ int work_id; ++#endif ++ /* Assigned after atom is completed. Used to check whether PRLAM-10676 workaround should be applied */ ++ int slot_nr; ++ ++ u32 atom_flags; ++ ++ /* Number of times this atom has been retried. Used by replay soft job. ++ */ ++ int retry_count; ++ ++ enum kbase_atom_gpu_rb_state gpu_rb_state; ++ ++ u64 need_cache_flush_cores_retained; ++ ++ atomic_t blocked; ++ ++ /* Pointer to atom that this atom has same-slot dependency on */ ++ struct kbase_jd_atom *pre_dep; ++ /* Pointer to atom that has same-slot dependency on this atom */ ++ struct kbase_jd_atom *post_dep; ++ ++ /* Pointer to atom that this atom has cross-slot dependency on */ ++ struct kbase_jd_atom *x_pre_dep; ++ /* Pointer to atom that has cross-slot dependency on this atom */ ++ struct kbase_jd_atom *x_post_dep; ++ ++ /* The GPU's flush count recorded at the time of submission, used for ++ * the cache flush optimisation */ ++ u32 flush_id; ++ ++ struct kbase_jd_atom_backend backend; ++#ifdef CONFIG_DEBUG_FS ++ struct base_job_fault_event fault_event; ++#endif ++ ++ /* List head used for three different purposes: ++ * 1. Overflow list for JS ring buffers. If an atom is ready to run, ++ * but there is no room in the JS ring buffer, then the atom is put ++ * on the ring buffer's overflow list using this list node. ++ * 2. List of waiting soft jobs. ++ */ ++ struct list_head queue; ++ ++ /* Used to keep track of all JIT free/alloc jobs in submission order ++ */ ++ struct list_head jit_node; ++ bool jit_blocked; ++ ++ /* If non-zero, this indicates that the atom will fail with the set ++ * event_code when the atom is processed. */ ++ enum base_jd_event_code will_fail_event_code; ++ ++ /* Atoms will only ever be transitioning into, or out of ++ * protected mode so we do not need two separate fields. ++ */ ++ union { ++ enum kbase_atom_enter_protected_state enter; ++ enum kbase_atom_exit_protected_state exit; ++ } protected_state; ++ ++ struct rb_node runnable_tree_node; ++ ++ /* 'Age' of atom relative to other atoms in the context. */ ++ u32 age; ++}; ++ ++static inline bool kbase_jd_katom_is_protected(const struct kbase_jd_atom *katom) ++{ ++ return (bool)(katom->atom_flags & KBASE_KATOM_FLAG_PROTECTED); ++} ++ ++/* ++ * Theory of operations: ++ * ++ * Atom objects are statically allocated within the context structure. ++ * ++ * Each atom is the head of two lists, one for the "left" set of dependencies, one for the "right" set. ++ */ ++ ++#define KBASE_JD_DEP_QUEUE_SIZE 256 ++ ++struct kbase_jd_context { ++ struct mutex lock; ++ struct kbasep_js_kctx_info sched_info; ++ struct kbase_jd_atom atoms[BASE_JD_ATOM_COUNT]; ++ ++ /** Tracks all job-dispatch jobs. This includes those not tracked by ++ * the scheduler: 'not ready to run' and 'dependency-only' jobs. */ ++ u32 job_nr; ++ ++ /** Waitq that reflects whether there are no jobs (including SW-only ++ * dependency jobs). This is set when no jobs are present on the ctx, ++ * and clear when there are jobs. ++ * ++ * @note: Job Dispatcher knows about more jobs than the Job Scheduler: ++ * the Job Scheduler is unaware of jobs that are blocked on dependencies, ++ * and SW-only dependency jobs. ++ * ++ * This waitq can be waited upon to find out when the context jobs are all ++ * done/cancelled (including those that might've been blocked on ++ * dependencies) - and so, whether it can be terminated. However, it should ++ * only be terminated once it is not present in the run-pool (see ++ * kbasep_js_kctx_info::ctx::is_scheduled). ++ * ++ * Since the waitq is only set under kbase_jd_context::lock, ++ * the waiter should also briefly obtain and drop kbase_jd_context::lock to ++ * guarentee that the setter has completed its work on the kbase_context ++ * ++ * This must be updated atomically with: ++ * - kbase_jd_context::job_nr */ ++ wait_queue_head_t zero_jobs_wait; ++ ++ /** Job Done workqueue. */ ++ struct workqueue_struct *job_done_wq; ++ ++ spinlock_t tb_lock; ++ u32 *tb; ++ size_t tb_wrap_offset; ++ ++#ifdef CONFIG_KDS ++ struct kds_callback kds_cb; ++#endif /* CONFIG_KDS */ ++#ifdef CONFIG_GPU_TRACEPOINTS ++ atomic_t work_id; ++#endif ++}; ++ ++struct kbase_device_info { ++ u32 features; ++}; ++ ++/** Poking state for BASE_HW_ISSUE_8316 */ ++enum { ++ KBASE_AS_POKE_STATE_IN_FLIGHT = 1<<0, ++ KBASE_AS_POKE_STATE_KILLING_POKE = 1<<1 ++}; ++ ++/** Poking state for BASE_HW_ISSUE_8316 */ ++typedef u32 kbase_as_poke_state; ++ ++struct kbase_mmu_setup { ++ u64 transtab; ++ u64 memattr; ++ u64 transcfg; ++}; ++ ++/** ++ * Important: Our code makes assumptions that a struct kbase_as structure is always at ++ * kbase_device->as[number]. This is used to recover the containing ++ * struct kbase_device from a struct kbase_as structure. ++ * ++ * Therefore, struct kbase_as structures must not be allocated anywhere else. ++ */ ++struct kbase_as { ++ int number; ++ ++ struct workqueue_struct *pf_wq; ++ struct work_struct work_pagefault; ++ struct work_struct work_busfault; ++ enum kbase_mmu_fault_type fault_type; ++ bool protected_mode; ++ u32 fault_status; ++ u64 fault_addr; ++ u64 fault_extra_addr; ++ ++ struct kbase_mmu_setup current_setup; ++ ++ /* BASE_HW_ISSUE_8316 */ ++ struct workqueue_struct *poke_wq; ++ struct work_struct poke_work; ++ /** Protected by hwaccess_lock */ ++ int poke_refcount; ++ /** Protected by hwaccess_lock */ ++ kbase_as_poke_state poke_state; ++ struct hrtimer poke_timer; ++}; ++ ++static inline int kbase_as_has_bus_fault(struct kbase_as *as) ++{ ++ return as->fault_type == KBASE_MMU_FAULT_TYPE_BUS; ++} ++ ++static inline int kbase_as_has_page_fault(struct kbase_as *as) ++{ ++ return as->fault_type == KBASE_MMU_FAULT_TYPE_PAGE; ++} ++ ++struct kbasep_mem_device { ++ atomic_t used_pages; /* Tracks usage of OS shared memory. Updated ++ when OS memory is allocated/freed. */ ++ ++}; ++ ++#define KBASE_TRACE_CODE(X) KBASE_TRACE_CODE_ ## X ++ ++enum kbase_trace_code { ++ /* IMPORTANT: USE OF SPECIAL #INCLUDE OF NON-STANDARD HEADER FILE ++ * THIS MUST BE USED AT THE START OF THE ENUM */ ++#define KBASE_TRACE_CODE_MAKE_CODE(X) KBASE_TRACE_CODE(X) ++#include "mali_kbase_trace_defs.h" ++#undef KBASE_TRACE_CODE_MAKE_CODE ++ /* Comma on its own, to extend the list */ ++ , ++ /* Must be the last in the enum */ ++ KBASE_TRACE_CODE_COUNT ++}; ++ ++#define KBASE_TRACE_FLAG_REFCOUNT (((u8)1) << 0) ++#define KBASE_TRACE_FLAG_JOBSLOT (((u8)1) << 1) ++ ++struct kbase_trace { ++ struct timespec timestamp; ++ u32 thread_id; ++ u32 cpu; ++ void *ctx; ++ bool katom; ++ int atom_number; ++ u64 atom_udata[2]; ++ u64 gpu_addr; ++ unsigned long info_val; ++ u8 code; ++ u8 jobslot; ++ u8 refcount; ++ u8 flags; ++}; ++ ++/** Event IDs for the power management framework. ++ * ++ * Any of these events might be missed, so they should not be relied upon to ++ * find the precise state of the GPU at a particular time in the ++ * trace. Overall, we should get a high percentage of these events for ++ * statisical purposes, and so a few missing should not be a problem */ ++enum kbase_timeline_pm_event { ++ /* helper for tests */ ++ KBASEP_TIMELINE_PM_EVENT_FIRST, ++ ++ /** Event reserved for backwards compatibility with 'init' events */ ++ KBASE_TIMELINE_PM_EVENT_RESERVED_0 = KBASEP_TIMELINE_PM_EVENT_FIRST, ++ ++ /** The power state of the device has changed. ++ * ++ * Specifically, the device has reached a desired or available state. ++ */ ++ KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED, ++ ++ /** The GPU is becoming active. ++ * ++ * This event is sent when the first context is about to use the GPU. ++ */ ++ KBASE_TIMELINE_PM_EVENT_GPU_ACTIVE, ++ ++ /** The GPU is becoming idle. ++ * ++ * This event is sent when the last context has finished using the GPU. ++ */ ++ KBASE_TIMELINE_PM_EVENT_GPU_IDLE, ++ ++ /** Event reserved for backwards compatibility with 'policy_change' ++ * events */ ++ KBASE_TIMELINE_PM_EVENT_RESERVED_4, ++ ++ /** Event reserved for backwards compatibility with 'system_suspend' ++ * events */ ++ KBASE_TIMELINE_PM_EVENT_RESERVED_5, ++ ++ /** Event reserved for backwards compatibility with 'system_resume' ++ * events */ ++ KBASE_TIMELINE_PM_EVENT_RESERVED_6, ++ ++ /** The job scheduler is requesting to power up/down cores. ++ * ++ * This event is sent when: ++ * - powered down cores are needed to complete a job ++ * - powered up cores are not needed anymore ++ */ ++ KBASE_TIMELINE_PM_EVENT_CHANGE_GPU_STATE, ++ ++ KBASEP_TIMELINE_PM_EVENT_LAST = KBASE_TIMELINE_PM_EVENT_CHANGE_GPU_STATE, ++}; ++ ++#ifdef CONFIG_MALI_BIFROST_TRACE_TIMELINE ++struct kbase_trace_kctx_timeline { ++ atomic_t jd_atoms_in_flight; ++ u32 owner_tgid; ++}; ++ ++struct kbase_trace_kbdev_timeline { ++ /* Note: strictly speaking, not needed, because it's in sync with ++ * kbase_device::jm_slots[]::submitted_nr ++ * ++ * But it's kept as an example of how to add global timeline tracking ++ * information ++ * ++ * The caller must hold hwaccess_lock when accessing this */ ++ u8 slot_atoms_submitted[BASE_JM_MAX_NR_SLOTS]; ++ ++ /* Last UID for each PM event */ ++ atomic_t pm_event_uid[KBASEP_TIMELINE_PM_EVENT_LAST+1]; ++ /* Counter for generating PM event UIDs */ ++ atomic_t pm_event_uid_counter; ++ /* ++ * L2 transition state - true indicates that the transition is ongoing ++ * Expected to be protected by hwaccess_lock */ ++ bool l2_transitioning; ++}; ++#endif /* CONFIG_MALI_BIFROST_TRACE_TIMELINE */ ++ ++ ++struct kbasep_kctx_list_element { ++ struct list_head link; ++ struct kbase_context *kctx; ++}; ++ ++/** ++ * Data stored per device for power management. ++ * ++ * This structure contains data for the power management framework. There is one ++ * instance of this structure per device in the system. ++ */ ++struct kbase_pm_device_data { ++ /** ++ * The lock protecting Power Management structures accessed outside of ++ * IRQ. ++ * ++ * This lock must also be held whenever the GPU is being powered on or ++ * off. ++ */ ++ struct mutex lock; ++ ++ /** The reference count of active contexts on this device. */ ++ int active_count; ++ /** Flag indicating suspending/suspended */ ++ bool suspending; ++ /* Wait queue set when active_count == 0 */ ++ wait_queue_head_t zero_active_count_wait; ++ ++ /** ++ * Bit masks identifying the available shader cores that are specified ++ * via sysfs. One mask per job slot. ++ */ ++ u64 debug_core_mask[BASE_JM_MAX_NR_SLOTS]; ++ u64 debug_core_mask_all; ++ ++ /** ++ * Callback for initializing the runtime power management. ++ * ++ * @param kbdev The kbase device ++ * ++ * @return 0 on success, else error code ++ */ ++ int (*callback_power_runtime_init)(struct kbase_device *kbdev); ++ ++ /** ++ * Callback for terminating the runtime power management. ++ * ++ * @param kbdev The kbase device ++ */ ++ void (*callback_power_runtime_term)(struct kbase_device *kbdev); ++ ++ /* Time in milliseconds between each dvfs sample */ ++ u32 dvfs_period; ++ ++ /* Period of GPU poweroff timer */ ++ ktime_t gpu_poweroff_time; ++ ++ /* Number of ticks of GPU poweroff timer before shader is powered off */ ++ int poweroff_shader_ticks; ++ ++ /* Number of ticks of GPU poweroff timer before GPU is powered off */ ++ int poweroff_gpu_ticks; ++ ++ struct kbase_pm_backend_data backend; ++}; ++ ++/** ++ * struct kbase_mem_pool - Page based memory pool for kctx/kbdev ++ * @kbdev: Kbase device where memory is used ++ * @cur_size: Number of free pages currently in the pool (may exceed @max_size ++ * in some corner cases) ++ * @max_size: Maximum number of free pages in the pool ++ * @order: order = 0 refers to a pool of 4 KB pages ++ * order = 9 refers to a pool of 2 MB pages (2^9 * 4KB = 2 MB) ++ * @pool_lock: Lock protecting the pool - must be held when modifying @cur_size ++ * and @page_list ++ * @page_list: List of free pages in the pool ++ * @reclaim: Shrinker for kernel reclaim of free pages ++ * @next_pool: Pointer to next pool where pages can be allocated when this pool ++ * is empty. Pages will spill over to the next pool when this pool ++ * is full. Can be NULL if there is no next pool. ++ */ ++struct kbase_mem_pool { ++ struct kbase_device *kbdev; ++ size_t cur_size; ++ size_t max_size; ++ size_t order; ++ spinlock_t pool_lock; ++ struct list_head page_list; ++ struct shrinker reclaim; ++ ++ struct kbase_mem_pool *next_pool; ++}; ++ ++/** ++ * struct kbase_devfreq_opp - Lookup table for converting between nominal OPP ++ * frequency, and real frequency and core mask ++ * @opp_freq: Nominal OPP frequency ++ * @real_freq: Real GPU frequency ++ * @core_mask: Shader core mask ++ */ ++struct kbase_devfreq_opp { ++ u64 opp_freq; ++ u64 real_freq; ++ u64 core_mask; ++}; ++ ++struct kbase_mmu_mode { ++ void (*update)(struct kbase_context *kctx); ++ void (*get_as_setup)(struct kbase_context *kctx, ++ struct kbase_mmu_setup * const setup); ++ void (*disable_as)(struct kbase_device *kbdev, int as_nr); ++ phys_addr_t (*pte_to_phy_addr)(u64 entry); ++ int (*ate_is_valid)(u64 ate, unsigned int level); ++ int (*pte_is_valid)(u64 pte, unsigned int level); ++ void (*entry_set_ate)(u64 *entry, struct tagged_addr phy, ++ unsigned long flags, unsigned int level); ++ void (*entry_set_pte)(u64 *entry, phys_addr_t phy); ++ void (*entry_invalidate)(u64 *entry); ++}; ++ ++struct kbase_mmu_mode const *kbase_mmu_mode_get_lpae(void); ++struct kbase_mmu_mode const *kbase_mmu_mode_get_aarch64(void); ++ ++ ++#define DEVNAME_SIZE 16 ++ ++struct kbase_device { ++ s8 slot_submit_count_irq[BASE_JM_MAX_NR_SLOTS]; ++ ++ u32 hw_quirks_sc; ++ u32 hw_quirks_tiler; ++ u32 hw_quirks_mmu; ++ u32 hw_quirks_jm; ++ ++ struct list_head entry; ++ struct device *dev; ++ struct miscdevice mdev; ++ u64 reg_start; ++ size_t reg_size; ++ void __iomem *reg; ++ ++ struct { ++ int irq; ++ int flags; ++ } irqs[3]; ++ ++ struct clk *clock; ++#ifdef CONFIG_REGULATOR ++ struct regulator *regulator; ++#endif ++ char devname[DEVNAME_SIZE]; ++ ++#ifdef CONFIG_MALI_BIFROST_NO_MALI ++ void *model; ++ struct kmem_cache *irq_slab; ++ struct workqueue_struct *irq_workq; ++ atomic_t serving_job_irq; ++ atomic_t serving_gpu_irq; ++ atomic_t serving_mmu_irq; ++ spinlock_t reg_op_lock; ++#endif /* CONFIG_MALI_BIFROST_NO_MALI */ ++ ++ struct kbase_pm_device_data pm; ++ struct kbasep_js_device_data js_data; ++ struct kbase_mem_pool mem_pool; ++ struct kbase_mem_pool lp_mem_pool; ++ struct kbasep_mem_device memdev; ++ struct kbase_mmu_mode const *mmu_mode; ++ ++ struct kbase_as as[BASE_MAX_NR_AS]; ++ /* The below variables (as_free and as_to_kctx) are managed by the ++ * Context Scheduler. The kbasep_js_device_data::runpool_irq::lock must ++ * be held whilst accessing these. ++ */ ++ u16 as_free; /* Bitpattern of free Address Spaces */ ++ /* Mapping from active Address Spaces to kbase_context */ ++ struct kbase_context *as_to_kctx[BASE_MAX_NR_AS]; ++ ++ ++ spinlock_t mmu_mask_change; ++ ++ struct kbase_gpu_props gpu_props; ++ ++ /** List of SW workarounds for HW issues */ ++ unsigned long hw_issues_mask[(BASE_HW_ISSUE_END + BITS_PER_LONG - 1) / BITS_PER_LONG]; ++ /** List of features available */ ++ unsigned long hw_features_mask[(BASE_HW_FEATURE_END + BITS_PER_LONG - 1) / BITS_PER_LONG]; ++ ++ /* Bitmaps of cores that are currently in use (running jobs). ++ * These should be kept up to date by the job scheduler. ++ * ++ * pm.power_change_lock should be held when accessing these members. ++ * ++ * kbase_pm_check_transitions_nolock() should be called when bits are ++ * cleared to update the power management system and allow transitions to ++ * occur. */ ++ u64 shader_inuse_bitmap; ++ ++ /* Refcount for cores in use */ ++ u32 shader_inuse_cnt[64]; ++ ++ /* Bitmaps of cores the JS needs for jobs ready to run */ ++ u64 shader_needed_bitmap; ++ ++ /* Refcount for cores needed */ ++ u32 shader_needed_cnt[64]; ++ ++ u32 tiler_inuse_cnt; ++ ++ u32 tiler_needed_cnt; ++ ++ /* struct for keeping track of the disjoint information ++ * ++ * The state is > 0 if the GPU is in a disjoint state. Otherwise 0 ++ * The count is the number of disjoint events that have occurred on the GPU ++ */ ++ struct { ++ atomic_t count; ++ atomic_t state; ++ } disjoint_event; ++ ++ /* Refcount for tracking users of the l2 cache, e.g. when using hardware counter instrumentation. */ ++ u32 l2_users_count; ++ ++ /* Bitmaps of cores that are currently available (powered up and the power policy is happy for jobs to be ++ * submitted to these cores. These are updated by the power management code. The job scheduler should avoid ++ * submitting new jobs to any cores that are not marked as available. ++ * ++ * pm.power_change_lock should be held when accessing these members. ++ */ ++ u64 shader_available_bitmap; ++ u64 tiler_available_bitmap; ++ u64 l2_available_bitmap; ++ u64 stack_available_bitmap; ++ ++ u64 shader_ready_bitmap; ++ u64 shader_transitioning_bitmap; ++ ++ s8 nr_hw_address_spaces; /**< Number of address spaces in the GPU (constant after driver initialisation) */ ++ s8 nr_user_address_spaces; /**< Number of address spaces available to user contexts */ ++ ++ /* Structure used for instrumentation and HW counters dumping */ ++ struct kbase_hwcnt { ++ /* The lock should be used when accessing any of the following members */ ++ spinlock_t lock; ++ ++ struct kbase_context *kctx; ++ u64 addr; ++ ++ struct kbase_instr_backend backend; ++ } hwcnt; ++ ++ struct kbase_vinstr_context *vinstr_ctx; ++ ++#if KBASE_TRACE_ENABLE ++ spinlock_t trace_lock; ++ u16 trace_first_out; ++ u16 trace_next_in; ++ struct kbase_trace *trace_rbuf; ++#endif ++ ++ u32 reset_timeout_ms; ++ ++ struct mutex cacheclean_lock; ++ ++ /* Platform specific private data to be accessed by mali_kbase_config_xxx.c only */ ++ void *platform_context; ++ ++ /* List of kbase_contexts created */ ++ struct list_head kctx_list; ++ struct mutex kctx_list_lock; ++ ++#ifdef CONFIG_MALI_BIFROST_DEVFREQ ++ struct devfreq_dev_profile devfreq_profile; ++ struct devfreq *devfreq; ++ unsigned long current_freq; ++ unsigned long current_nominal_freq; ++ unsigned long current_voltage; ++ u64 current_core_mask; ++ struct kbase_devfreq_opp *opp_table; ++ int num_opps; ++ struct monitor_dev_info *mdev_info; ++ struct ipa_power_model_data *model_data; ++#ifdef CONFIG_DEVFREQ_THERMAL ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) ++ struct devfreq_cooling_device *devfreq_cooling; ++#else ++ struct thermal_cooling_device *devfreq_cooling; ++#endif ++ /* Current IPA model - true for configured model, false for fallback */ ++ atomic_t ipa_use_configured_model; ++ struct { ++ /* Access to this struct must be with ipa.lock held */ ++ struct mutex lock; ++ struct kbase_ipa_model *configured_model; ++ struct kbase_ipa_model *fallback_model; ++ } ipa; ++#endif /* CONFIG_DEVFREQ_THERMAL */ ++#endif /* CONFIG_MALI_BIFROST_DEVFREQ */ ++ ++ ++#ifdef CONFIG_MALI_BIFROST_TRACE_TIMELINE ++ struct kbase_trace_kbdev_timeline timeline; ++#endif ++ ++ /* ++ * Control for enabling job dump on failure, set when control debugfs ++ * is opened. ++ */ ++ bool job_fault_debug; ++ ++#ifdef CONFIG_DEBUG_FS ++ /* directory for debugfs entries */ ++ struct dentry *mali_debugfs_directory; ++ /* Root directory for per context entry */ ++ struct dentry *debugfs_ctx_directory; ++ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++ /* bit for each as, set if there is new data to report */ ++ u64 debugfs_as_read_bitmap; ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++ ++ /* failed job dump, used for separate debug process */ ++ wait_queue_head_t job_fault_wq; ++ wait_queue_head_t job_fault_resume_wq; ++ struct workqueue_struct *job_fault_resume_workq; ++ struct list_head job_fault_event_list; ++ spinlock_t job_fault_event_lock; ++ struct kbase_context *kctx_fault; ++ ++#if !MALI_CUSTOMER_RELEASE ++ /* Per-device data for register dumping interface */ ++ struct { ++ u16 reg_offset; /* Offset of a GPU_CONTROL register to be ++ dumped upon request */ ++ } regs_dump_debugfs_data; ++#endif /* !MALI_CUSTOMER_RELEASE */ ++#endif /* CONFIG_DEBUG_FS */ ++ ++ /* fbdump profiling controls set by gator */ ++ u32 kbase_profiling_controls[FBDUMP_CONTROL_MAX]; ++ ++ ++#if MALI_CUSTOMER_RELEASE == 0 ++ /* Number of jobs that are run before a job is forced to fail and ++ * replay. May be KBASEP_FORCE_REPLAY_DISABLED, to disable forced ++ * failures. */ ++ int force_replay_limit; ++ /* Count of jobs between forced failures. Incremented on each job. A ++ * job is forced to fail once this is greater than or equal to ++ * force_replay_limit. */ ++ int force_replay_count; ++ /* Core requirement for jobs to be failed and replayed. May be zero. */ ++ base_jd_core_req force_replay_core_req; ++ /* true if force_replay_limit should be randomized. The random ++ * value will be in the range of 1 - KBASEP_FORCE_REPLAY_RANDOM_LIMIT. ++ */ ++ bool force_replay_random; ++#endif ++ ++ /* Total number of created contexts */ ++ atomic_t ctx_num; ++ ++#ifdef CONFIG_DEBUG_FS ++ /* Holds the most recent register accesses */ ++ struct kbase_io_history io_history; ++#endif /* CONFIG_DEBUG_FS */ ++ ++ struct kbase_hwaccess_data hwaccess; ++ ++ /* Count of page/bus faults waiting for workqueues to process */ ++ atomic_t faults_pending; ++ ++ /* true if GPU is powered off or power off operation is in progress */ ++ bool poweroff_pending; ++ ++ ++ /* defaults for new context created for this device */ ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0)) ++ bool infinite_cache_active_default; ++#else ++ u32 infinite_cache_active_default; ++#endif ++ size_t mem_pool_max_size_default; ++ ++ /* current gpu coherency mode */ ++ u32 current_gpu_coherency_mode; ++ /* system coherency mode */ ++ u32 system_coherency; ++ /* Flag to track when cci snoops have been enabled on the interface */ ++ bool cci_snoop_enabled; ++ ++ /* SMC function IDs to call into Trusted firmware to enable/disable ++ * cache snooping. Value of 0 indicates that they are not used ++ */ ++ u32 snoop_enable_smc; ++ u32 snoop_disable_smc; ++ ++ /* Protected mode operations */ ++ struct protected_mode_ops *protected_ops; ++ ++ /* Protected device attached to this kbase device */ ++ struct protected_mode_device *protected_dev; ++ ++ /* ++ * true when GPU is put into protected mode ++ */ ++ bool protected_mode; ++ ++ /* ++ * true when GPU is transitioning into or out of protected mode ++ */ ++ bool protected_mode_transition; ++ ++ /* ++ * true if protected mode is supported ++ */ ++ bool protected_mode_support; ++ ++ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++ wait_queue_head_t driver_inactive_wait; ++ bool driver_inactive; ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++ ++#ifdef CONFIG_MALI_FPGA_BUS_LOGGER ++ /* ++ * Bus logger integration. ++ */ ++ struct bus_logger_client *buslogger; ++#endif ++ /* Boolean indicating if an IRQ flush during reset is in progress. */ ++ bool irq_reset_flush; ++ ++ /* list of inited sub systems. Used during terminate/error recovery */ ++ u32 inited_subsys; ++ ++ spinlock_t hwaccess_lock; ++ ++ /* Protects access to MMU operations */ ++ struct mutex mmu_hw_mutex; ++ ++ /* Current serialization mode. See KBASE_SERIALIZE_* for details */ ++ u8 serialize_jobs; ++}; ++ ++/** ++ * struct jsctx_queue - JS context atom queue ++ * @runnable_tree: Root of RB-tree containing currently runnable atoms on this ++ * job slot. ++ * @x_dep_head: Head item of the linked list of atoms blocked on cross-slot ++ * dependencies. Atoms on this list will be moved to the ++ * runnable_tree when the blocking atom completes. ++ * ++ * hwaccess_lock must be held when accessing this structure. ++ */ ++struct jsctx_queue { ++ struct rb_root runnable_tree; ++ struct list_head x_dep_head; ++}; ++ ++ ++#define KBASE_API_VERSION(major, minor) ((((major) & 0xFFF) << 20) | \ ++ (((minor) & 0xFFF) << 8) | \ ++ ((0 & 0xFF) << 0)) ++ ++/** ++ * enum kbase_context_flags - Flags for kbase contexts ++ * ++ * @KCTX_COMPAT: Set when the context process is a compat process, 32-bit ++ * process on a 64-bit kernel. ++ * ++ * @KCTX_RUNNABLE_REF: Set when context is counted in ++ * kbdev->js_data.nr_contexts_runnable. Must hold queue_mutex when accessing. ++ * ++ * @KCTX_ACTIVE: Set when the context is active. ++ * ++ * @KCTX_PULLED: Set when last kick() caused atoms to be pulled from this ++ * context. ++ * ++ * @KCTX_MEM_PROFILE_INITIALIZED: Set when the context's memory profile has been ++ * initialized. ++ * ++ * @KCTX_INFINITE_CACHE: Set when infinite cache is to be enabled for new ++ * allocations. Existing allocations will not change. ++ * ++ * @KCTX_SUBMIT_DISABLED: Set to prevent context from submitting any jobs. ++ * ++ * @KCTX_PRIVILEGED:Set if the context uses an address space and should be kept ++ * scheduled in. ++ * ++ * @KCTX_SCHEDULED: Set when the context is scheduled on the Run Pool. ++ * This is only ever updated whilst the jsctx_mutex is held. ++ * ++ * @KCTX_DYING: Set when the context process is in the process of being evicted. ++ * ++ * @KCTX_NO_IMPLICIT_SYNC: Set when explicit Android fences are in use on this ++ * context, to disable use of implicit dma-buf fences. This is used to avoid ++ * potential synchronization deadlocks. ++ * ++ * All members need to be separate bits. This enum is intended for use in a ++ * bitmask where multiple values get OR-ed together. ++ */ ++enum kbase_context_flags { ++ KCTX_COMPAT = 1U << 0, ++ KCTX_RUNNABLE_REF = 1U << 1, ++ KCTX_ACTIVE = 1U << 2, ++ KCTX_PULLED = 1U << 3, ++ KCTX_MEM_PROFILE_INITIALIZED = 1U << 4, ++ KCTX_INFINITE_CACHE = 1U << 5, ++ KCTX_SUBMIT_DISABLED = 1U << 6, ++ KCTX_PRIVILEGED = 1U << 7, ++ KCTX_SCHEDULED = 1U << 8, ++ KCTX_DYING = 1U << 9, ++ KCTX_NO_IMPLICIT_SYNC = 1U << 10, ++}; ++ ++struct kbase_sub_alloc { ++ struct list_head link; ++ struct page *page; ++ DECLARE_BITMAP(sub_pages, SZ_2M / SZ_4K); ++}; ++ ++struct kbase_context { ++ struct file *filp; ++ struct kbase_device *kbdev; ++ u32 id; /* System wide unique id */ ++ unsigned long api_version; ++ phys_addr_t pgd; ++ struct list_head event_list; ++ struct list_head event_coalesce_list; ++ struct mutex event_mutex; ++ atomic_t event_closed; ++ struct workqueue_struct *event_workq; ++ atomic_t event_count; ++ int event_coalesce_count; ++ ++ atomic_t flags; ++ ++ atomic_t setup_complete; ++ atomic_t setup_in_progress; ++ ++ u64 *mmu_teardown_pages; ++ ++ struct tagged_addr aliasing_sink_page; ++ ++ struct mutex mem_partials_lock; ++ struct list_head mem_partials; ++ ++ struct mutex mmu_lock; ++ struct mutex reg_lock; /* To be converted to a rwlock? */ ++ struct rb_root reg_rbtree_same; /* RB tree of GPU (live) regions, ++ * SAME_VA zone */ ++ struct rb_root reg_rbtree_exec; /* RB tree of GPU (live) regions, ++ * EXEC zone */ ++ struct rb_root reg_rbtree_custom; /* RB tree of GPU (live) regions, ++ * CUSTOM_VA zone */ ++ ++ unsigned long cookies; ++ struct kbase_va_region *pending_regions[BITS_PER_LONG]; ++ ++ wait_queue_head_t event_queue; ++ pid_t tgid; ++ pid_t pid; ++ ++ struct kbase_jd_context jctx; ++ atomic_t used_pages; ++ atomic_t nonmapped_pages; ++ ++ struct kbase_mem_pool mem_pool; ++ struct kbase_mem_pool lp_mem_pool; ++ ++ struct shrinker reclaim; ++ struct list_head evict_list; ++ ++ struct list_head waiting_soft_jobs; ++ spinlock_t waiting_soft_jobs_lock; ++#ifdef CONFIG_KDS ++ struct list_head waiting_kds_resource; ++#endif ++#ifdef CONFIG_MALI_BIFROST_DMA_FENCE ++ struct { ++ struct list_head waiting_resource; ++ struct workqueue_struct *wq; ++ } dma_fence; ++#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ ++ /** This is effectively part of the Run Pool, because it only has a valid ++ * setting (!=KBASEP_AS_NR_INVALID) whilst the context is scheduled in ++ * ++ * The hwaccess_lock must be held whilst accessing this. ++ * ++ * If the context relating to this as_nr is required, you must use ++ * kbasep_js_runpool_retain_ctx() to ensure that the context doesn't disappear ++ * whilst you're using it. Alternatively, just hold the hwaccess_lock ++ * to ensure the context doesn't disappear (but this has restrictions on what other locks ++ * you can take whilst doing this) */ ++ int as_nr; ++ ++ /* Keeps track of the number of users of this context. A user can be a ++ * job that is available for execution, instrumentation needing to 'pin' ++ * a context for counter collection, etc. If the refcount reaches 0 then ++ * this context is considered inactive and the previously programmed ++ * AS might be cleared at any point. ++ */ ++ atomic_t refcount; ++ ++ /* NOTE: ++ * ++ * Flags are in jctx.sched_info.ctx.flags ++ * Mutable flags *must* be accessed under jctx.sched_info.ctx.jsctx_mutex ++ * ++ * All other flags must be added there */ ++ spinlock_t mm_update_lock; ++ struct mm_struct *process_mm; ++ /* End of the SAME_VA zone */ ++ u64 same_va_end; ++ ++#ifdef CONFIG_MALI_BIFROST_TRACE_TIMELINE ++ struct kbase_trace_kctx_timeline timeline; ++#endif ++#ifdef CONFIG_DEBUG_FS ++ /* Content of mem_profile file */ ++ char *mem_profile_data; ++ /* Size of @c mem_profile_data */ ++ size_t mem_profile_size; ++ /* Mutex guarding memory profile state */ ++ struct mutex mem_profile_lock; ++ /* Memory profile directory under debugfs */ ++ struct dentry *kctx_dentry; ++ ++ /* for job fault debug */ ++ unsigned int *reg_dump; ++ atomic_t job_fault_count; ++ /* This list will keep the following atoms during the dump ++ * in the same context ++ */ ++ struct list_head job_fault_resume_event_list; ++ ++#endif /* CONFIG_DEBUG_FS */ ++ ++ struct jsctx_queue jsctx_queue ++ [KBASE_JS_ATOM_SCHED_PRIO_COUNT][BASE_JM_MAX_NR_SLOTS]; ++ ++ /* Number of atoms currently pulled from this context */ ++ atomic_t atoms_pulled; ++ /* Number of atoms currently pulled from this context, per slot */ ++ atomic_t atoms_pulled_slot[BASE_JM_MAX_NR_SLOTS]; ++ /* Number of atoms currently pulled from this context, per slot and ++ * priority. Hold hwaccess_lock when accessing */ ++ int atoms_pulled_slot_pri[BASE_JM_MAX_NR_SLOTS][ ++ KBASE_JS_ATOM_SCHED_PRIO_COUNT]; ++ ++ /* true if slot is blocked on the given priority. This will be set on a ++ * soft-stop */ ++ bool blocked_js[BASE_JM_MAX_NR_SLOTS][KBASE_JS_ATOM_SCHED_PRIO_COUNT]; ++ ++ /* Bitmask of slots that can be pulled from */ ++ u32 slots_pullable; ++ ++ /* Backend specific data */ ++ struct kbase_context_backend backend; ++ ++ /* Work structure used for deferred ASID assignment */ ++ struct work_struct work; ++ ++ /* Only one userspace vinstr client per kbase context */ ++ struct kbase_vinstr_client *vinstr_cli; ++ struct mutex vinstr_cli_lock; ++ ++ /* List of completed jobs waiting for events to be posted */ ++ struct list_head completed_jobs; ++ /* Number of work items currently pending on job_done_wq */ ++ atomic_t work_count; ++ ++ /* Waiting soft-jobs will fail when this timer expires */ ++ struct timer_list soft_job_timeout; ++ ++ /* JIT allocation management */ ++ struct kbase_va_region *jit_alloc[256]; ++ struct list_head jit_active_head; ++ struct list_head jit_pool_head; ++ struct list_head jit_destroy_head; ++ struct mutex jit_evict_lock; ++ struct work_struct jit_work; ++ ++ /* A list of the JIT soft-jobs in submission order ++ * (protected by kbase_jd_context.lock) ++ */ ++ struct list_head jit_atoms_head; ++ /* A list of pending JIT alloc soft-jobs (using the 'queue' list_head) ++ * (protected by kbase_jd_context.lock) ++ */ ++ struct list_head jit_pending_alloc; ++ ++ /* External sticky resource management */ ++ struct list_head ext_res_meta_head; ++ ++ /* Used to record that a drain was requested from atomic context */ ++ atomic_t drain_pending; ++ ++ /* Current age count, used to determine age for newly submitted atoms */ ++ u32 age_count; ++}; ++ ++/** ++ * struct kbase_ctx_ext_res_meta - Structure which binds an external resource ++ * to a @kbase_context. ++ * @ext_res_node: List head for adding the metadata to a ++ * @kbase_context. ++ * @alloc: The physical memory allocation structure ++ * which is mapped. ++ * @gpu_addr: The GPU virtual address the resource is ++ * mapped to. ++ * ++ * External resources can be mapped into multiple contexts as well as the same ++ * context multiple times. ++ * As kbase_va_region itself isn't refcounted we can't attach our extra ++ * information to it as it could be removed under our feet leaving external ++ * resources pinned. ++ * This metadata structure binds a single external resource to a single ++ * context, ensuring that per context mapping is tracked separately so it can ++ * be overridden when needed and abuses by the application (freeing the resource ++ * multiple times) don't effect the refcount of the physical allocation. ++ */ ++struct kbase_ctx_ext_res_meta { ++ struct list_head ext_res_node; ++ struct kbase_mem_phy_alloc *alloc; ++ u64 gpu_addr; ++}; ++ ++enum kbase_reg_access_type { ++ REG_READ, ++ REG_WRITE ++}; ++ ++enum kbase_share_attr_bits { ++ /* (1ULL << 8) bit is reserved */ ++ SHARE_BOTH_BITS = (2ULL << 8), /* inner and outer shareable coherency */ ++ SHARE_INNER_BITS = (3ULL << 8) /* inner shareable coherency */ ++}; ++ ++/** ++ * kbase_device_is_cpu_coherent - Returns if the device is CPU coherent. ++ * @kbdev: kbase device ++ * ++ * Return: true if the device access are coherent, false if not. ++ */ ++static inline bool kbase_device_is_cpu_coherent(struct kbase_device *kbdev) ++{ ++ if ((kbdev->system_coherency == COHERENCY_ACE_LITE) || ++ (kbdev->system_coherency == COHERENCY_ACE)) ++ return true; ++ ++ return false; ++} ++ ++/* Conversion helpers for setting up high resolution timers */ ++#define HR_TIMER_DELAY_MSEC(x) (ns_to_ktime(((u64)(x))*1000000U)) ++#define HR_TIMER_DELAY_NSEC(x) (ns_to_ktime(x)) ++ ++/* Maximum number of loops polling the GPU for a cache flush before we assume it must have completed */ ++#define KBASE_CLEAN_CACHE_MAX_LOOPS 100000 ++/* Maximum number of loops polling the GPU for an AS command to complete before we assume the GPU has hung */ ++#define KBASE_AS_INACTIVE_MAX_LOOPS 100000 ++ ++/* Maximum number of times a job can be replayed */ ++#define BASEP_JD_REPLAY_LIMIT 15 ++ ++/* JobDescriptorHeader - taken from the architecture specifications, the layout ++ * is currently identical for all GPU archs. */ ++struct job_descriptor_header { ++ u32 exception_status; ++ u32 first_incomplete_task; ++ u64 fault_pointer; ++ u8 job_descriptor_size : 1; ++ u8 job_type : 7; ++ u8 job_barrier : 1; ++ u8 _reserved_01 : 1; ++ u8 _reserved_1 : 1; ++ u8 _reserved_02 : 1; ++ u8 _reserved_03 : 1; ++ u8 _reserved_2 : 1; ++ u8 _reserved_04 : 1; ++ u8 _reserved_05 : 1; ++ u16 job_index; ++ u16 job_dependency_index_1; ++ u16 job_dependency_index_2; ++ union { ++ u64 _64; ++ u32 _32; ++ } next_job; ++}; ++ ++#endif /* _KBASE_DEFS_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_device.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_device.c +new file mode 100755 +index 000000000000..2d11f11f3be0 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_device.c +@@ -0,0 +1,674 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * Base kernel device APIs ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#include ++#include ++#include ++#include ++ ++#include ++ ++/* NOTE: Magic - 0x45435254 (TRCE in ASCII). ++ * Supports tracing feature provided in the base module. ++ * Please keep it in sync with the value of base module. ++ */ ++#define TRACE_BUFFER_HEADER_SPECIAL 0x45435254 ++ ++#if KBASE_TRACE_ENABLE ++static const char *kbasep_trace_code_string[] = { ++ /* IMPORTANT: USE OF SPECIAL #INCLUDE OF NON-STANDARD HEADER FILE ++ * THIS MUST BE USED AT THE START OF THE ARRAY */ ++#define KBASE_TRACE_CODE_MAKE_CODE(X) # X ++#include "mali_kbase_trace_defs.h" ++#undef KBASE_TRACE_CODE_MAKE_CODE ++}; ++#endif ++ ++#define DEBUG_MESSAGE_SIZE 256 ++ ++static int kbasep_trace_init(struct kbase_device *kbdev); ++static void kbasep_trace_term(struct kbase_device *kbdev); ++static void kbasep_trace_hook_wrapper(void *param); ++ ++struct kbase_device *kbase_device_alloc(void) ++{ ++ return kzalloc(sizeof(struct kbase_device), GFP_KERNEL); ++} ++ ++static int kbase_device_as_init(struct kbase_device *kbdev, int i) ++{ ++ const char format[] = "mali_mmu%d"; ++ char name[sizeof(format)]; ++ const char poke_format[] = "mali_mmu%d_poker"; ++ char poke_name[sizeof(poke_format)]; ++ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316)) ++ snprintf(poke_name, sizeof(poke_name), poke_format, i); ++ ++ snprintf(name, sizeof(name), format, i); ++ ++ kbdev->as[i].number = i; ++ kbdev->as[i].fault_addr = 0ULL; ++ ++ kbdev->as[i].pf_wq = alloc_workqueue(name, 0, 1); ++ if (!kbdev->as[i].pf_wq) ++ return -EINVAL; ++ ++ INIT_WORK(&kbdev->as[i].work_pagefault, page_fault_worker); ++ INIT_WORK(&kbdev->as[i].work_busfault, bus_fault_worker); ++ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316)) { ++ struct hrtimer *poke_timer = &kbdev->as[i].poke_timer; ++ struct work_struct *poke_work = &kbdev->as[i].poke_work; ++ ++ kbdev->as[i].poke_wq = alloc_workqueue(poke_name, 0, 1); ++ if (!kbdev->as[i].poke_wq) { ++ destroy_workqueue(kbdev->as[i].pf_wq); ++ return -EINVAL; ++ } ++ KBASE_DEBUG_ASSERT(!object_is_on_stack(poke_work)); ++ INIT_WORK(poke_work, kbasep_as_do_poke); ++ ++ hrtimer_init(poke_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); ++ ++ poke_timer->function = kbasep_as_poke_timer_callback; ++ ++ kbdev->as[i].poke_refcount = 0; ++ kbdev->as[i].poke_state = 0u; ++ } ++ ++ return 0; ++} ++ ++static void kbase_device_as_term(struct kbase_device *kbdev, int i) ++{ ++ destroy_workqueue(kbdev->as[i].pf_wq); ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316)) ++ destroy_workqueue(kbdev->as[i].poke_wq); ++} ++ ++static int kbase_device_all_as_init(struct kbase_device *kbdev) ++{ ++ int i, err; ++ ++ for (i = 0; i < kbdev->nr_hw_address_spaces; i++) { ++ err = kbase_device_as_init(kbdev, i); ++ if (err) ++ goto free_workqs; ++ } ++ ++ return 0; ++ ++free_workqs: ++ for (; i > 0; i--) ++ kbase_device_as_term(kbdev, i); ++ ++ return err; ++} ++ ++static void kbase_device_all_as_term(struct kbase_device *kbdev) ++{ ++ int i; ++ ++ for (i = 0; i < kbdev->nr_hw_address_spaces; i++) ++ kbase_device_as_term(kbdev, i); ++} ++ ++int kbase_device_init(struct kbase_device * const kbdev) ++{ ++ int i, err; ++#ifdef CONFIG_ARM64 ++ struct device_node *np = NULL; ++#endif /* CONFIG_ARM64 */ ++ ++ spin_lock_init(&kbdev->mmu_mask_change); ++ mutex_init(&kbdev->mmu_hw_mutex); ++#ifdef CONFIG_ARM64 ++ kbdev->cci_snoop_enabled = false; ++ np = kbdev->dev->of_node; ++ if (np != NULL) { ++ if (of_property_read_u32(np, "snoop_enable_smc", ++ &kbdev->snoop_enable_smc)) ++ kbdev->snoop_enable_smc = 0; ++ if (of_property_read_u32(np, "snoop_disable_smc", ++ &kbdev->snoop_disable_smc)) ++ kbdev->snoop_disable_smc = 0; ++ /* Either both or none of the calls should be provided. */ ++ if (!((kbdev->snoop_disable_smc == 0 ++ && kbdev->snoop_enable_smc == 0) ++ || (kbdev->snoop_disable_smc != 0 ++ && kbdev->snoop_enable_smc != 0))) { ++ WARN_ON(1); ++ err = -EINVAL; ++ goto fail; ++ } ++ } ++#endif /* CONFIG_ARM64 */ ++ /* Get the list of workarounds for issues on the current HW ++ * (identified by the GPU_ID register) ++ */ ++ err = kbase_hw_set_issues_mask(kbdev); ++ if (err) ++ goto fail; ++ ++ /* Set the list of features available on the current HW ++ * (identified by the GPU_ID register) ++ */ ++ kbase_hw_set_features_mask(kbdev); ++ ++ kbase_gpuprops_set_features(kbdev); ++ ++ /* On Linux 4.0+, dma coherency is determined from device tree */ ++#if defined(CONFIG_ARM64) && LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0) ++ set_dma_ops(kbdev->dev, &noncoherent_swiotlb_dma_ops); ++#endif ++ ++ /* Workaround a pre-3.13 Linux issue, where dma_mask is NULL when our ++ * device structure was created by device-tree ++ */ ++ if (!kbdev->dev->dma_mask) ++ kbdev->dev->dma_mask = &kbdev->dev->coherent_dma_mask; ++ ++ err = dma_set_mask(kbdev->dev, ++ DMA_BIT_MASK(kbdev->gpu_props.mmu.pa_bits)); ++ if (err) ++ goto dma_set_mask_failed; ++ ++ err = dma_set_coherent_mask(kbdev->dev, ++ DMA_BIT_MASK(kbdev->gpu_props.mmu.pa_bits)); ++ if (err) ++ goto dma_set_mask_failed; ++ ++ kbdev->nr_hw_address_spaces = kbdev->gpu_props.num_address_spaces; ++ ++ err = kbase_device_all_as_init(kbdev); ++ if (err) ++ goto as_init_failed; ++ ++ spin_lock_init(&kbdev->hwcnt.lock); ++ ++ err = kbasep_trace_init(kbdev); ++ if (err) ++ goto term_as; ++ ++ mutex_init(&kbdev->cacheclean_lock); ++ ++#ifdef CONFIG_MALI_BIFROST_TRACE_TIMELINE ++ for (i = 0; i < BASE_JM_MAX_NR_SLOTS; ++i) ++ kbdev->timeline.slot_atoms_submitted[i] = 0; ++ ++ for (i = 0; i <= KBASEP_TIMELINE_PM_EVENT_LAST; ++i) ++ atomic_set(&kbdev->timeline.pm_event_uid[i], 0); ++#endif /* CONFIG_MALI_BIFROST_TRACE_TIMELINE */ ++ ++ /* fbdump profiling controls set to 0 - fbdump not enabled until changed by gator */ ++ for (i = 0; i < FBDUMP_CONTROL_MAX; i++) ++ kbdev->kbase_profiling_controls[i] = 0; ++ ++ kbase_debug_assert_register_hook(&kbasep_trace_hook_wrapper, kbdev); ++ ++ atomic_set(&kbdev->ctx_num, 0); ++ ++ err = kbase_instr_backend_init(kbdev); ++ if (err) ++ goto term_trace; ++ ++ kbdev->pm.dvfs_period = DEFAULT_PM_DVFS_PERIOD; ++ ++ kbdev->reset_timeout_ms = DEFAULT_RESET_TIMEOUT_MS; ++ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) ++ kbdev->mmu_mode = kbase_mmu_mode_get_aarch64(); ++ else ++ kbdev->mmu_mode = kbase_mmu_mode_get_lpae(); ++ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++ init_waitqueue_head(&kbdev->driver_inactive_wait); ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++ ++ return 0; ++term_trace: ++ kbasep_trace_term(kbdev); ++term_as: ++ kbase_device_all_as_term(kbdev); ++as_init_failed: ++dma_set_mask_failed: ++fail: ++ return err; ++} ++ ++void kbase_device_term(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev); ++ ++#if KBASE_TRACE_ENABLE ++ kbase_debug_assert_register_hook(NULL, NULL); ++#endif ++ ++ kbase_instr_backend_term(kbdev); ++ ++ kbasep_trace_term(kbdev); ++ ++ kbase_device_all_as_term(kbdev); ++} ++ ++void kbase_device_free(struct kbase_device *kbdev) ++{ ++ kfree(kbdev); ++} ++ ++int kbase_device_trace_buffer_install( ++ struct kbase_context *kctx, u32 *tb, size_t size) ++{ ++ unsigned long flags; ++ ++ KBASE_DEBUG_ASSERT(kctx); ++ KBASE_DEBUG_ASSERT(tb); ++ ++ /* Interface uses 16-bit value to track last accessed entry. Each entry ++ * is composed of two 32-bit words. ++ * This limits the size that can be handled without an overflow. */ ++ if (0xFFFF * (2 * sizeof(u32)) < size) ++ return -EINVAL; ++ ++ /* set up the header */ ++ /* magic number in the first 4 bytes */ ++ tb[0] = TRACE_BUFFER_HEADER_SPECIAL; ++ /* Store (write offset = 0, wrap counter = 0, transaction active = no) ++ * write offset 0 means never written. ++ * Offsets 1 to (wrap_offset - 1) used to store values when trace started ++ */ ++ tb[1] = 0; ++ ++ /* install trace buffer */ ++ spin_lock_irqsave(&kctx->jctx.tb_lock, flags); ++ kctx->jctx.tb_wrap_offset = size / 8; ++ kctx->jctx.tb = tb; ++ spin_unlock_irqrestore(&kctx->jctx.tb_lock, flags); ++ ++ return 0; ++} ++ ++void kbase_device_trace_buffer_uninstall(struct kbase_context *kctx) ++{ ++ unsigned long flags; ++ ++ KBASE_DEBUG_ASSERT(kctx); ++ spin_lock_irqsave(&kctx->jctx.tb_lock, flags); ++ kctx->jctx.tb = NULL; ++ kctx->jctx.tb_wrap_offset = 0; ++ spin_unlock_irqrestore(&kctx->jctx.tb_lock, flags); ++} ++ ++void kbase_device_trace_register_access(struct kbase_context *kctx, enum kbase_reg_access_type type, u16 reg_offset, u32 reg_value) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kctx->jctx.tb_lock, flags); ++ if (kctx->jctx.tb) { ++ u16 wrap_count; ++ u16 write_offset; ++ u32 *tb = kctx->jctx.tb; ++ u32 header_word; ++ ++ header_word = tb[1]; ++ KBASE_DEBUG_ASSERT(0 == (header_word & 0x1)); ++ ++ wrap_count = (header_word >> 1) & 0x7FFF; ++ write_offset = (header_word >> 16) & 0xFFFF; ++ ++ /* mark as transaction in progress */ ++ tb[1] |= 0x1; ++ mb(); ++ ++ /* calculate new offset */ ++ write_offset++; ++ if (write_offset == kctx->jctx.tb_wrap_offset) { ++ /* wrap */ ++ write_offset = 1; ++ wrap_count++; ++ wrap_count &= 0x7FFF; /* 15bit wrap counter */ ++ } ++ ++ /* store the trace entry at the selected offset */ ++ tb[write_offset * 2 + 0] = (reg_offset & ~0x3) | ((type == REG_WRITE) ? 0x1 : 0x0); ++ tb[write_offset * 2 + 1] = reg_value; ++ mb(); ++ ++ /* new header word */ ++ header_word = (write_offset << 16) | (wrap_count << 1) | 0x0; /* transaction complete */ ++ tb[1] = header_word; ++ } ++ spin_unlock_irqrestore(&kctx->jctx.tb_lock, flags); ++} ++ ++/* ++ * Device trace functions ++ */ ++#if KBASE_TRACE_ENABLE ++ ++static int kbasep_trace_init(struct kbase_device *kbdev) ++{ ++ struct kbase_trace *rbuf; ++ ++ rbuf = kmalloc_array(KBASE_TRACE_SIZE, sizeof(*rbuf), GFP_KERNEL); ++ ++ if (!rbuf) ++ return -EINVAL; ++ ++ kbdev->trace_rbuf = rbuf; ++ spin_lock_init(&kbdev->trace_lock); ++ return 0; ++} ++ ++static void kbasep_trace_term(struct kbase_device *kbdev) ++{ ++ kfree(kbdev->trace_rbuf); ++} ++ ++static void kbasep_trace_format_msg(struct kbase_trace *trace_msg, char *buffer, int len) ++{ ++ s32 written = 0; ++ ++ /* Initial part of message */ ++ written += MAX(snprintf(buffer + written, MAX(len - written, 0), "%d.%.6d,%d,%d,%s,%p,", (int)trace_msg->timestamp.tv_sec, (int)(trace_msg->timestamp.tv_nsec / 1000), trace_msg->thread_id, trace_msg->cpu, kbasep_trace_code_string[trace_msg->code], trace_msg->ctx), 0); ++ ++ if (trace_msg->katom) ++ written += MAX(snprintf(buffer + written, MAX(len - written, 0), "atom %d (ud: 0x%llx 0x%llx)", trace_msg->atom_number, trace_msg->atom_udata[0], trace_msg->atom_udata[1]), 0); ++ ++ written += MAX(snprintf(buffer + written, MAX(len - written, 0), ",%.8llx,", trace_msg->gpu_addr), 0); ++ ++ /* NOTE: Could add function callbacks to handle different message types */ ++ /* Jobslot present */ ++ if (trace_msg->flags & KBASE_TRACE_FLAG_JOBSLOT) ++ written += MAX(snprintf(buffer + written, MAX(len - written, 0), "%d", trace_msg->jobslot), 0); ++ ++ written += MAX(snprintf(buffer + written, MAX(len - written, 0), ","), 0); ++ ++ /* Refcount present */ ++ if (trace_msg->flags & KBASE_TRACE_FLAG_REFCOUNT) ++ written += MAX(snprintf(buffer + written, MAX(len - written, 0), "%d", trace_msg->refcount), 0); ++ ++ written += MAX(snprintf(buffer + written, MAX(len - written, 0), ","), 0); ++ ++ /* Rest of message */ ++ written += MAX(snprintf(buffer + written, MAX(len - written, 0), "0x%.8lx", trace_msg->info_val), 0); ++} ++ ++static void kbasep_trace_dump_msg(struct kbase_device *kbdev, struct kbase_trace *trace_msg) ++{ ++ char buffer[DEBUG_MESSAGE_SIZE]; ++ ++ kbasep_trace_format_msg(trace_msg, buffer, DEBUG_MESSAGE_SIZE); ++ dev_dbg(kbdev->dev, "%s", buffer); ++} ++ ++void kbasep_trace_add(struct kbase_device *kbdev, enum kbase_trace_code code, void *ctx, struct kbase_jd_atom *katom, u64 gpu_addr, u8 flags, int refcount, int jobslot, unsigned long info_val) ++{ ++ unsigned long irqflags; ++ struct kbase_trace *trace_msg; ++ ++ spin_lock_irqsave(&kbdev->trace_lock, irqflags); ++ ++ trace_msg = &kbdev->trace_rbuf[kbdev->trace_next_in]; ++ ++ /* Fill the message */ ++ trace_msg->thread_id = task_pid_nr(current); ++ trace_msg->cpu = task_cpu(current); ++ ++ getnstimeofday(&trace_msg->timestamp); ++ ++ trace_msg->code = code; ++ trace_msg->ctx = ctx; ++ ++ if (NULL == katom) { ++ trace_msg->katom = false; ++ } else { ++ trace_msg->katom = true; ++ trace_msg->atom_number = kbase_jd_atom_id(katom->kctx, katom); ++ trace_msg->atom_udata[0] = katom->udata.blob[0]; ++ trace_msg->atom_udata[1] = katom->udata.blob[1]; ++ } ++ ++ trace_msg->gpu_addr = gpu_addr; ++ trace_msg->jobslot = jobslot; ++ trace_msg->refcount = MIN((unsigned int)refcount, 0xFF); ++ trace_msg->info_val = info_val; ++ trace_msg->flags = flags; ++ ++ /* Update the ringbuffer indices */ ++ kbdev->trace_next_in = (kbdev->trace_next_in + 1) & KBASE_TRACE_MASK; ++ if (kbdev->trace_next_in == kbdev->trace_first_out) ++ kbdev->trace_first_out = (kbdev->trace_first_out + 1) & KBASE_TRACE_MASK; ++ ++ /* Done */ ++ ++ spin_unlock_irqrestore(&kbdev->trace_lock, irqflags); ++} ++ ++void kbasep_trace_clear(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->trace_lock, flags); ++ kbdev->trace_first_out = kbdev->trace_next_in; ++ spin_unlock_irqrestore(&kbdev->trace_lock, flags); ++} ++ ++void kbasep_trace_dump(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ u32 start; ++ u32 end; ++ ++ dev_dbg(kbdev->dev, "Dumping trace:\nsecs,nthread,cpu,code,ctx,katom,gpu_addr,jobslot,refcount,info_val"); ++ spin_lock_irqsave(&kbdev->trace_lock, flags); ++ start = kbdev->trace_first_out; ++ end = kbdev->trace_next_in; ++ ++ while (start != end) { ++ struct kbase_trace *trace_msg = &kbdev->trace_rbuf[start]; ++ ++ kbasep_trace_dump_msg(kbdev, trace_msg); ++ ++ start = (start + 1) & KBASE_TRACE_MASK; ++ } ++ dev_dbg(kbdev->dev, "TRACE_END"); ++ ++ spin_unlock_irqrestore(&kbdev->trace_lock, flags); ++ ++ KBASE_TRACE_CLEAR(kbdev); ++} ++ ++static void kbasep_trace_hook_wrapper(void *param) ++{ ++ struct kbase_device *kbdev = (struct kbase_device *)param; ++ ++ kbasep_trace_dump(kbdev); ++} ++ ++#ifdef CONFIG_DEBUG_FS ++struct trace_seq_state { ++ struct kbase_trace trace_buf[KBASE_TRACE_SIZE]; ++ u32 start; ++ u32 end; ++}; ++ ++static void *kbasep_trace_seq_start(struct seq_file *s, loff_t *pos) ++{ ++ struct trace_seq_state *state = s->private; ++ int i; ++ ++ if (*pos > KBASE_TRACE_SIZE) ++ return NULL; ++ i = state->start + *pos; ++ if ((state->end >= state->start && i >= state->end) || ++ i >= state->end + KBASE_TRACE_SIZE) ++ return NULL; ++ ++ i &= KBASE_TRACE_MASK; ++ ++ return &state->trace_buf[i]; ++} ++ ++static void kbasep_trace_seq_stop(struct seq_file *s, void *data) ++{ ++} ++ ++static void *kbasep_trace_seq_next(struct seq_file *s, void *data, loff_t *pos) ++{ ++ struct trace_seq_state *state = s->private; ++ int i; ++ ++ (*pos)++; ++ ++ i = (state->start + *pos) & KBASE_TRACE_MASK; ++ if (i == state->end) ++ return NULL; ++ ++ return &state->trace_buf[i]; ++} ++ ++static int kbasep_trace_seq_show(struct seq_file *s, void *data) ++{ ++ struct kbase_trace *trace_msg = data; ++ char buffer[DEBUG_MESSAGE_SIZE]; ++ ++ kbasep_trace_format_msg(trace_msg, buffer, DEBUG_MESSAGE_SIZE); ++ seq_printf(s, "%s\n", buffer); ++ return 0; ++} ++ ++static const struct seq_operations kbasep_trace_seq_ops = { ++ .start = kbasep_trace_seq_start, ++ .next = kbasep_trace_seq_next, ++ .stop = kbasep_trace_seq_stop, ++ .show = kbasep_trace_seq_show, ++}; ++ ++static int kbasep_trace_debugfs_open(struct inode *inode, struct file *file) ++{ ++ struct kbase_device *kbdev = inode->i_private; ++ unsigned long flags; ++ ++ struct trace_seq_state *state; ++ ++ state = __seq_open_private(file, &kbasep_trace_seq_ops, sizeof(*state)); ++ if (!state) ++ return -ENOMEM; ++ ++ spin_lock_irqsave(&kbdev->trace_lock, flags); ++ state->start = kbdev->trace_first_out; ++ state->end = kbdev->trace_next_in; ++ memcpy(state->trace_buf, kbdev->trace_rbuf, sizeof(state->trace_buf)); ++ spin_unlock_irqrestore(&kbdev->trace_lock, flags); ++ ++ return 0; ++} ++ ++static const struct file_operations kbasep_trace_debugfs_fops = { ++ .open = kbasep_trace_debugfs_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = seq_release_private, ++}; ++ ++void kbasep_trace_debugfs_init(struct kbase_device *kbdev) ++{ ++ debugfs_create_file("mali_trace", S_IRUGO, ++ kbdev->mali_debugfs_directory, kbdev, ++ &kbasep_trace_debugfs_fops); ++} ++ ++#else ++void kbasep_trace_debugfs_init(struct kbase_device *kbdev) ++{ ++} ++#endif /* CONFIG_DEBUG_FS */ ++ ++#else /* KBASE_TRACE_ENABLE */ ++static int kbasep_trace_init(struct kbase_device *kbdev) ++{ ++ CSTD_UNUSED(kbdev); ++ return 0; ++} ++ ++static void kbasep_trace_term(struct kbase_device *kbdev) ++{ ++ CSTD_UNUSED(kbdev); ++} ++ ++static void kbasep_trace_hook_wrapper(void *param) ++{ ++ CSTD_UNUSED(param); ++} ++ ++void kbasep_trace_dump(struct kbase_device *kbdev) ++{ ++ CSTD_UNUSED(kbdev); ++} ++#endif /* KBASE_TRACE_ENABLE */ ++ ++void kbase_set_profiling_control(struct kbase_device *kbdev, u32 control, u32 value) ++{ ++ switch (control) { ++ case FBDUMP_CONTROL_ENABLE: ++ /* fall through */ ++ case FBDUMP_CONTROL_RATE: ++ /* fall through */ ++ case SW_COUNTER_ENABLE: ++ /* fall through */ ++ case FBDUMP_CONTROL_RESIZE_FACTOR: ++ kbdev->kbase_profiling_controls[control] = value; ++ break; ++ default: ++ dev_err(kbdev->dev, "Profiling control %d not found\n", control); ++ break; ++ } ++} ++ ++/* ++ * Called by gator to control the production of ++ * profiling information at runtime ++ * */ ++ ++void _mali_profiling_control(u32 action, u32 value) ++{ ++ struct kbase_device *kbdev = NULL; ++ ++ /* find the first i.e. call with -1 */ ++ kbdev = kbase_find_device(-1); ++ ++ if (NULL != kbdev) ++ kbase_set_profiling_control(kbdev, action, value); ++} ++KBASE_EXPORT_SYMBOL(_mali_profiling_control); ++ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_disjoint_events.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_disjoint_events.c +new file mode 100755 +index 000000000000..f70bcccf4050 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_disjoint_events.c +@@ -0,0 +1,76 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* ++ * Base kernel disjoint events helper functions ++ */ ++ ++#include ++ ++void kbase_disjoint_init(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ atomic_set(&kbdev->disjoint_event.count, 0); ++ atomic_set(&kbdev->disjoint_event.state, 0); ++} ++ ++/* increment the disjoint event count */ ++void kbase_disjoint_event(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ atomic_inc(&kbdev->disjoint_event.count); ++} ++ ++/* increment the state and the event counter */ ++void kbase_disjoint_state_up(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ atomic_inc(&kbdev->disjoint_event.state); ++ ++ kbase_disjoint_event(kbdev); ++} ++ ++/* decrement the state */ ++void kbase_disjoint_state_down(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(atomic_read(&kbdev->disjoint_event.state) > 0); ++ ++ kbase_disjoint_event(kbdev); ++ ++ atomic_dec(&kbdev->disjoint_event.state); ++} ++ ++/* increments the count only if the state is > 0 */ ++void kbase_disjoint_event_potential(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ if (atomic_read(&kbdev->disjoint_event.state)) ++ kbase_disjoint_event(kbdev); ++} ++ ++u32 kbase_disjoint_event_get(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ return atomic_read(&kbdev->disjoint_event.count); ++} ++KBASE_EXPORT_TEST_API(kbase_disjoint_event_get); +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_dma_fence.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_dma_fence.c +new file mode 100755 +index 000000000000..8a571266534b +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_dma_fence.c +@@ -0,0 +1,449 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++/* Include mali_kbase_dma_fence.h before checking for CONFIG_MALI_BIFROST_DMA_FENCE as ++ * it will be set there. ++ */ ++#include "mali_kbase_dma_fence.h" ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++ ++static void ++kbase_dma_fence_work(struct work_struct *pwork); ++ ++static void ++kbase_dma_fence_waiters_add(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ ++ list_add_tail(&katom->queue, &kctx->dma_fence.waiting_resource); ++} ++ ++static void ++kbase_dma_fence_waiters_remove(struct kbase_jd_atom *katom) ++{ ++ list_del(&katom->queue); ++} ++ ++static int ++kbase_dma_fence_lock_reservations(struct kbase_dma_fence_resv_info *info, ++ struct ww_acquire_ctx *ctx) ++{ ++ struct reservation_object *content_res = NULL; ++ unsigned int content_res_idx = 0; ++ unsigned int r; ++ int err = 0; ++ ++ ww_acquire_init(ctx, &reservation_ww_class); ++ ++retry: ++ for (r = 0; r < info->dma_fence_resv_count; r++) { ++ if (info->resv_objs[r] == content_res) { ++ content_res = NULL; ++ continue; ++ } ++ ++ err = ww_mutex_lock(&info->resv_objs[r]->lock, ctx); ++ if (err) ++ goto error; ++ } ++ ++ ww_acquire_done(ctx); ++ return err; ++ ++error: ++ content_res_idx = r; ++ ++ /* Unlock the locked one ones */ ++ while (r--) ++ ww_mutex_unlock(&info->resv_objs[r]->lock); ++ ++ if (content_res) ++ ww_mutex_unlock(&content_res->lock); ++ ++ /* If we deadlock try with lock_slow and retry */ ++ if (err == -EDEADLK) { ++ content_res = info->resv_objs[content_res_idx]; ++ ww_mutex_lock_slow(&content_res->lock, ctx); ++ goto retry; ++ } ++ ++ /* If we are here the function failed */ ++ ww_acquire_fini(ctx); ++ return err; ++} ++ ++static void ++kbase_dma_fence_unlock_reservations(struct kbase_dma_fence_resv_info *info, ++ struct ww_acquire_ctx *ctx) ++{ ++ unsigned int r; ++ ++ for (r = 0; r < info->dma_fence_resv_count; r++) ++ ww_mutex_unlock(&info->resv_objs[r]->lock); ++ ww_acquire_fini(ctx); ++} ++ ++/** ++ * kbase_dma_fence_queue_work() - Queue work to handle @katom ++ * @katom: Pointer to atom for which to queue work ++ * ++ * Queue kbase_dma_fence_work() for @katom to clean up the fence callbacks and ++ * submit the atom. ++ */ ++static void ++kbase_dma_fence_queue_work(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ bool ret; ++ ++ INIT_WORK(&katom->work, kbase_dma_fence_work); ++ ret = queue_work(kctx->dma_fence.wq, &katom->work); ++ /* Warn if work was already queued, that should not happen. */ ++ WARN_ON(!ret); ++} ++ ++/** ++ * kbase_dma_fence_cancel_atom() - Cancels waiting on an atom ++ * @katom: Katom to cancel ++ * ++ * Locking: katom->dma_fence.callbacks list assumes jctx.lock is held. ++ */ ++static void ++kbase_dma_fence_cancel_atom(struct kbase_jd_atom *katom) ++{ ++ lockdep_assert_held(&katom->kctx->jctx.lock); ++ ++ /* Cancel callbacks and clean up. */ ++ kbase_fence_free_callbacks(katom); ++ ++ /* Mark the atom as handled in case all fences signaled just before ++ * canceling the callbacks and the worker was queued. ++ */ ++ kbase_fence_dep_count_set(katom, -1); ++ ++ /* Prevent job_done_nolock from being called twice on an atom when ++ * there is a race between job completion and cancellation. ++ */ ++ ++ if (katom->status == KBASE_JD_ATOM_STATE_QUEUED) { ++ /* Wait was cancelled - zap the atom */ ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ if (jd_done_nolock(katom, NULL)) ++ kbase_js_sched_all(katom->kctx->kbdev); ++ } ++} ++ ++/** ++ * kbase_dma_fence_work() - Worker thread called when a fence is signaled ++ * @pwork: work_struct containing a pointer to a katom ++ * ++ * This function will clean and mark all dependencies as satisfied ++ */ ++static void ++kbase_dma_fence_work(struct work_struct *pwork) ++{ ++ struct kbase_jd_atom *katom; ++ struct kbase_jd_context *ctx; ++ ++ katom = container_of(pwork, struct kbase_jd_atom, work); ++ ctx = &katom->kctx->jctx; ++ ++ mutex_lock(&ctx->lock); ++ if (kbase_fence_dep_count_read(katom) != 0) ++ goto out; ++ ++ kbase_fence_dep_count_set(katom, -1); ++ ++ /* Remove atom from list of dma-fence waiting atoms. */ ++ kbase_dma_fence_waiters_remove(katom); ++ /* Cleanup callbacks. */ ++ kbase_fence_free_callbacks(katom); ++ /* ++ * Queue atom on GPU, unless it has already completed due to a failing ++ * dependency. Run jd_done_nolock() on the katom if it is completed. ++ */ ++ if (unlikely(katom->status == KBASE_JD_ATOM_STATE_COMPLETED)) ++ jd_done_nolock(katom, NULL); ++ else ++ kbase_jd_dep_clear_locked(katom); ++ ++out: ++ mutex_unlock(&ctx->lock); ++} ++ ++static void ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++kbase_dma_fence_cb(struct fence *fence, struct fence_cb *cb) ++#else ++kbase_dma_fence_cb(struct dma_fence *fence, struct dma_fence_cb *cb) ++#endif ++{ ++ struct kbase_fence_cb *kcb = container_of(cb, ++ struct kbase_fence_cb, ++ fence_cb); ++ struct kbase_jd_atom *katom = kcb->katom; ++ ++ /* If the atom is zapped dep_count will be forced to a negative number ++ * preventing this callback from ever scheduling work. Which in turn ++ * would reschedule the atom. ++ */ ++ ++ if (kbase_fence_dep_count_dec_and_test(katom)) ++ kbase_dma_fence_queue_work(katom); ++} ++ ++static int ++kbase_dma_fence_add_reservation_callback(struct kbase_jd_atom *katom, ++ struct reservation_object *resv, ++ bool exclusive) ++{ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ struct fence *excl_fence = NULL; ++ struct fence **shared_fences = NULL; ++#else ++ struct dma_fence *excl_fence = NULL; ++ struct dma_fence **shared_fences = NULL; ++#endif ++ unsigned int shared_count = 0; ++ int err, i; ++ ++ err = reservation_object_get_fences_rcu(resv, ++ &excl_fence, ++ &shared_count, ++ &shared_fences); ++ if (err) ++ return err; ++ ++ if (excl_fence) { ++ err = kbase_fence_add_callback(katom, ++ excl_fence, ++ kbase_dma_fence_cb); ++ ++ /* Release our reference, taken by reservation_object_get_fences_rcu(), ++ * to the fence. We have set up our callback (if that was possible), ++ * and it's the fence's owner is responsible for singling the fence ++ * before allowing it to disappear. ++ */ ++ dma_fence_put(excl_fence); ++ ++ if (err) ++ goto out; ++ } ++ ++ if (exclusive) { ++ for (i = 0; i < shared_count; i++) { ++ err = kbase_fence_add_callback(katom, ++ shared_fences[i], ++ kbase_dma_fence_cb); ++ if (err) ++ goto out; ++ } ++ } ++ ++ /* Release all our references to the shared fences, taken by ++ * reservation_object_get_fences_rcu(). We have set up our callback (if ++ * that was possible), and it's the fence's owner is responsible for ++ * signaling the fence before allowing it to disappear. ++ */ ++out: ++ for (i = 0; i < shared_count; i++) ++ dma_fence_put(shared_fences[i]); ++ kfree(shared_fences); ++ ++ if (err) { ++ /* ++ * On error, cancel and clean up all callbacks that was set up ++ * before the error. ++ */ ++ kbase_fence_free_callbacks(katom); ++ } ++ ++ return err; ++} ++ ++void kbase_dma_fence_add_reservation(struct reservation_object *resv, ++ struct kbase_dma_fence_resv_info *info, ++ bool exclusive) ++{ ++ unsigned int i; ++ ++ for (i = 0; i < info->dma_fence_resv_count; i++) { ++ /* Duplicate resource, ignore */ ++ if (info->resv_objs[i] == resv) ++ return; ++ } ++ ++ info->resv_objs[info->dma_fence_resv_count] = resv; ++ if (exclusive) ++ set_bit(info->dma_fence_resv_count, ++ info->dma_fence_excl_bitmap); ++ (info->dma_fence_resv_count)++; ++} ++ ++int kbase_dma_fence_wait(struct kbase_jd_atom *katom, ++ struct kbase_dma_fence_resv_info *info) ++{ ++ int err, i; ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ struct fence *fence; ++#else ++ struct dma_fence *fence; ++#endif ++ struct ww_acquire_ctx ww_ctx; ++ ++ lockdep_assert_held(&katom->kctx->jctx.lock); ++ ++ fence = kbase_fence_out_new(katom); ++ if (!fence) { ++ err = -ENOMEM; ++ dev_err(katom->kctx->kbdev->dev, ++ "Error %d creating fence.\n", err); ++ return err; ++ } ++ ++ kbase_fence_dep_count_set(katom, 1); ++ ++ err = kbase_dma_fence_lock_reservations(info, &ww_ctx); ++ if (err) { ++ dev_err(katom->kctx->kbdev->dev, ++ "Error %d locking reservations.\n", err); ++ kbase_fence_dep_count_set(katom, -1); ++ kbase_fence_out_remove(katom); ++ return err; ++ } ++ ++ for (i = 0; i < info->dma_fence_resv_count; i++) { ++ struct reservation_object *obj = info->resv_objs[i]; ++ ++ if (!test_bit(i, info->dma_fence_excl_bitmap)) { ++ err = reservation_object_reserve_shared(obj); ++ if (err) { ++ dev_err(katom->kctx->kbdev->dev, ++ "Error %d reserving space for shared fence.\n", err); ++ goto end; ++ } ++ ++ err = kbase_dma_fence_add_reservation_callback(katom, obj, false); ++ if (err) { ++ dev_err(katom->kctx->kbdev->dev, ++ "Error %d adding reservation to callback.\n", err); ++ goto end; ++ } ++ ++ reservation_object_add_shared_fence(obj, fence); ++ } else { ++ err = kbase_dma_fence_add_reservation_callback(katom, obj, true); ++ if (err) { ++ dev_err(katom->kctx->kbdev->dev, ++ "Error %d adding reservation to callback.\n", err); ++ goto end; ++ } ++ ++ reservation_object_add_excl_fence(obj, fence); ++ } ++ } ++ ++end: ++ kbase_dma_fence_unlock_reservations(info, &ww_ctx); ++ ++ if (likely(!err)) { ++ /* Test if the callbacks are already triggered */ ++ if (kbase_fence_dep_count_dec_and_test(katom)) { ++ kbase_fence_dep_count_set(katom, -1); ++ kbase_fence_free_callbacks(katom); ++ } else { ++ /* Add katom to the list of dma-buf fence waiting atoms ++ * only if it is still waiting. ++ */ ++ kbase_dma_fence_waiters_add(katom); ++ } ++ } else { ++ /* There was an error, cancel callbacks, set dep_count to -1 to ++ * indicate that the atom has been handled (the caller will ++ * kill it for us), signal the fence, free callbacks and the ++ * fence. ++ */ ++ kbase_fence_free_callbacks(katom); ++ kbase_fence_dep_count_set(katom, -1); ++ kbase_dma_fence_signal(katom); ++ } ++ ++ return err; ++} ++ ++void kbase_dma_fence_cancel_all_atoms(struct kbase_context *kctx) ++{ ++ struct list_head *list = &kctx->dma_fence.waiting_resource; ++ ++ while (!list_empty(list)) { ++ struct kbase_jd_atom *katom; ++ ++ katom = list_first_entry(list, struct kbase_jd_atom, queue); ++ kbase_dma_fence_waiters_remove(katom); ++ kbase_dma_fence_cancel_atom(katom); ++ } ++} ++ ++void kbase_dma_fence_cancel_callbacks(struct kbase_jd_atom *katom) ++{ ++ /* Cancel callbacks and clean up. */ ++ if (kbase_fence_free_callbacks(katom)) ++ kbase_dma_fence_queue_work(katom); ++} ++ ++void kbase_dma_fence_signal(struct kbase_jd_atom *katom) ++{ ++ if (!katom->dma_fence.fence) ++ return; ++ ++ /* Signal the atom's fence. */ ++ dma_fence_signal(katom->dma_fence.fence); ++ ++ kbase_fence_out_remove(katom); ++ ++ kbase_fence_free_callbacks(katom); ++} ++ ++void kbase_dma_fence_term(struct kbase_context *kctx) ++{ ++ destroy_workqueue(kctx->dma_fence.wq); ++ kctx->dma_fence.wq = NULL; ++} ++ ++int kbase_dma_fence_init(struct kbase_context *kctx) ++{ ++ INIT_LIST_HEAD(&kctx->dma_fence.waiting_resource); ++ ++ kctx->dma_fence.wq = alloc_workqueue("mali-fence-%d", ++ WQ_UNBOUND, 1, kctx->pid); ++ if (!kctx->dma_fence.wq) ++ return -ENOMEM; ++ ++ return 0; ++} +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_dma_fence.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_dma_fence.h +new file mode 100755 +index 000000000000..b02ea9774c4f +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_dma_fence.h +@@ -0,0 +1,131 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KBASE_DMA_FENCE_H_ ++#define _KBASE_DMA_FENCE_H_ ++ ++#ifdef CONFIG_MALI_BIFROST_DMA_FENCE ++ ++#include ++#include ++#include ++ ++ ++/* Forward declaration from mali_kbase_defs.h */ ++struct kbase_jd_atom; ++struct kbase_context; ++ ++/** ++ * struct kbase_dma_fence_resv_info - Structure with list of reservation objects ++ * @resv_objs: Array of reservation objects to attach the ++ * new fence to. ++ * @dma_fence_resv_count: Number of reservation objects in the array. ++ * @dma_fence_excl_bitmap: Specifies which resv_obj are exclusive. ++ * ++ * This is used by some functions to pass around a collection of data about ++ * reservation objects. ++ */ ++struct kbase_dma_fence_resv_info { ++ struct reservation_object **resv_objs; ++ unsigned int dma_fence_resv_count; ++ unsigned long *dma_fence_excl_bitmap; ++}; ++ ++/** ++ * kbase_dma_fence_add_reservation() - Adds a resv to the array of resv_objs ++ * @resv: Reservation object to add to the array. ++ * @info: Pointer to struct with current reservation info ++ * @exclusive: Boolean indicating if exclusive access is needed ++ * ++ * The function adds a new reservation_object to an existing array of ++ * reservation_objects. At the same time keeps track of which objects require ++ * exclusive access in dma_fence_excl_bitmap. ++ */ ++void kbase_dma_fence_add_reservation(struct reservation_object *resv, ++ struct kbase_dma_fence_resv_info *info, ++ bool exclusive); ++ ++/** ++ * kbase_dma_fence_wait() - Creates a new fence and attaches it to the resv_objs ++ * @katom: Katom with the external dependency. ++ * @info: Pointer to struct with current reservation info ++ * ++ * Return: An error code or 0 if succeeds ++ */ ++int kbase_dma_fence_wait(struct kbase_jd_atom *katom, ++ struct kbase_dma_fence_resv_info *info); ++ ++/** ++ * kbase_dma_fence_cancel_ctx() - Cancel all dma-fences blocked atoms on kctx ++ * @kctx: Pointer to kbase context ++ * ++ * This function will cancel and clean up all katoms on @kctx that is waiting ++ * on dma-buf fences. ++ * ++ * Locking: jctx.lock needs to be held when calling this function. ++ */ ++void kbase_dma_fence_cancel_all_atoms(struct kbase_context *kctx); ++ ++/** ++ * kbase_dma_fence_cancel_callbacks() - Cancel only callbacks on katom ++ * @katom: Pointer to katom whose callbacks are to be canceled ++ * ++ * This function cancels all dma-buf fence callbacks on @katom, but does not ++ * cancel the katom itself. ++ * ++ * The caller is responsible for ensuring that jd_done_nolock is called on ++ * @katom. ++ * ++ * Locking: jctx.lock must be held when calling this function. ++ */ ++void kbase_dma_fence_cancel_callbacks(struct kbase_jd_atom *katom); ++ ++/** ++ * kbase_dma_fence_signal() - Signal katom's fence and clean up after wait ++ * @katom: Pointer to katom to signal and clean up ++ * ++ * This function will signal the @katom's fence, if it has one, and clean up ++ * the callback data from the katom's wait on earlier fences. ++ * ++ * Locking: jctx.lock must be held while calling this function. ++ */ ++void kbase_dma_fence_signal(struct kbase_jd_atom *katom); ++ ++/** ++ * kbase_dma_fence_term() - Terminate Mali dma-fence context ++ * @kctx: kbase context to terminate ++ */ ++void kbase_dma_fence_term(struct kbase_context *kctx); ++ ++/** ++ * kbase_dma_fence_init() - Initialize Mali dma-fence context ++ * @kctx: kbase context to initialize ++ */ ++int kbase_dma_fence_init(struct kbase_context *kctx); ++ ++ ++#else /* CONFIG_MALI_BIFROST_DMA_FENCE */ ++/* Dummy functions for when dma-buf fence isn't enabled. */ ++ ++static inline int kbase_dma_fence_init(struct kbase_context *kctx) ++{ ++ return 0; ++} ++ ++static inline void kbase_dma_fence_term(struct kbase_context *kctx) {} ++#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ ++#endif +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_event.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_event.c +new file mode 100755 +index 000000000000..188148645f37 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_event.c +@@ -0,0 +1,259 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#include ++#include ++#include ++ ++static struct base_jd_udata kbase_event_process(struct kbase_context *kctx, struct kbase_jd_atom *katom) ++{ ++ struct base_jd_udata data; ++ ++ lockdep_assert_held(&kctx->jctx.lock); ++ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ KBASE_DEBUG_ASSERT(katom != NULL); ++ KBASE_DEBUG_ASSERT(katom->status == KBASE_JD_ATOM_STATE_COMPLETED); ++ ++ data = katom->udata; ++ ++ KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, atomic_sub_return(1, &kctx->timeline.jd_atoms_in_flight)); ++ ++ KBASE_TLSTREAM_TL_NRET_ATOM_CTX(katom, kctx); ++ KBASE_TLSTREAM_TL_DEL_ATOM(katom); ++ ++ katom->status = KBASE_JD_ATOM_STATE_UNUSED; ++ ++ wake_up(&katom->completed); ++ ++ return data; ++} ++ ++int kbase_event_pending(struct kbase_context *ctx) ++{ ++ KBASE_DEBUG_ASSERT(ctx); ++ ++ return (atomic_read(&ctx->event_count) != 0) || ++ (atomic_read(&ctx->event_closed) != 0); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_event_pending); ++ ++int kbase_event_dequeue(struct kbase_context *ctx, struct base_jd_event_v2 *uevent) ++{ ++ struct kbase_jd_atom *atom; ++ ++ KBASE_DEBUG_ASSERT(ctx); ++ ++ mutex_lock(&ctx->event_mutex); ++ ++ if (list_empty(&ctx->event_list)) { ++ if (!atomic_read(&ctx->event_closed)) { ++ mutex_unlock(&ctx->event_mutex); ++ return -1; ++ } ++ ++ /* generate the BASE_JD_EVENT_DRV_TERMINATED message on the fly */ ++ mutex_unlock(&ctx->event_mutex); ++ uevent->event_code = BASE_JD_EVENT_DRV_TERMINATED; ++ memset(&uevent->udata, 0, sizeof(uevent->udata)); ++ dev_dbg(ctx->kbdev->dev, ++ "event system closed, returning BASE_JD_EVENT_DRV_TERMINATED(0x%X)\n", ++ BASE_JD_EVENT_DRV_TERMINATED); ++ return 0; ++ } ++ ++ /* normal event processing */ ++ atomic_dec(&ctx->event_count); ++ atom = list_entry(ctx->event_list.next, struct kbase_jd_atom, dep_item[0]); ++ list_del(ctx->event_list.next); ++ ++ mutex_unlock(&ctx->event_mutex); ++ ++ dev_dbg(ctx->kbdev->dev, "event dequeuing %p\n", (void *)atom); ++ uevent->event_code = atom->event_code; ++ uevent->atom_number = (atom - ctx->jctx.atoms); ++ ++ if (atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) ++ kbase_jd_free_external_resources(atom); ++ ++ mutex_lock(&ctx->jctx.lock); ++ uevent->udata = kbase_event_process(ctx, atom); ++ mutex_unlock(&ctx->jctx.lock); ++ ++ return 0; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_event_dequeue); ++ ++/** ++ * kbase_event_process_noreport_worker - Worker for processing atoms that do not ++ * return an event but do have external ++ * resources ++ * @data: Work structure ++ */ ++static void kbase_event_process_noreport_worker(struct work_struct *data) ++{ ++ struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, ++ work); ++ struct kbase_context *kctx = katom->kctx; ++ ++ if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) ++ kbase_jd_free_external_resources(katom); ++ ++ mutex_lock(&kctx->jctx.lock); ++ kbase_event_process(kctx, katom); ++ mutex_unlock(&kctx->jctx.lock); ++} ++ ++/** ++ * kbase_event_process_noreport - Process atoms that do not return an event ++ * @kctx: Context pointer ++ * @katom: Atom to be processed ++ * ++ * Atoms that do not have external resources will be processed immediately. ++ * Atoms that do have external resources will be processed on a workqueue, in ++ * order to avoid locking issues. ++ */ ++static void kbase_event_process_noreport(struct kbase_context *kctx, ++ struct kbase_jd_atom *katom) ++{ ++ if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) { ++ INIT_WORK(&katom->work, kbase_event_process_noreport_worker); ++ queue_work(kctx->event_workq, &katom->work); ++ } else { ++ kbase_event_process(kctx, katom); ++ } ++} ++ ++/** ++ * kbase_event_coalesce - Move pending events to the main event list ++ * @kctx: Context pointer ++ * ++ * kctx->event_list and kctx->event_coalesce_count must be protected ++ * by a lock unless this is the last thread using them ++ * (and we're about to terminate the lock). ++ * ++ * Return: The number of pending events moved to the main event list ++ */ ++static int kbase_event_coalesce(struct kbase_context *kctx) ++{ ++ const int event_count = kctx->event_coalesce_count; ++ ++ /* Join the list of pending events onto the tail of the main list ++ and reset it */ ++ list_splice_tail_init(&kctx->event_coalesce_list, &kctx->event_list); ++ kctx->event_coalesce_count = 0; ++ ++ /* Return the number of events moved */ ++ return event_count; ++} ++ ++void kbase_event_post(struct kbase_context *ctx, struct kbase_jd_atom *atom) ++{ ++ if (atom->core_req & BASE_JD_REQ_EVENT_ONLY_ON_FAILURE) { ++ if (atom->event_code == BASE_JD_EVENT_DONE) { ++ /* Don't report the event */ ++ kbase_event_process_noreport(ctx, atom); ++ return; ++ } ++ } ++ ++ if (atom->core_req & BASEP_JD_REQ_EVENT_NEVER) { ++ /* Don't report the event */ ++ kbase_event_process_noreport(ctx, atom); ++ return; ++ } ++ KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(atom, TL_ATOM_STATE_POSTED); ++ if (atom->core_req & BASE_JD_REQ_EVENT_COALESCE) { ++ /* Don't report the event until other event(s) have completed */ ++ mutex_lock(&ctx->event_mutex); ++ list_add_tail(&atom->dep_item[0], &ctx->event_coalesce_list); ++ ++ctx->event_coalesce_count; ++ mutex_unlock(&ctx->event_mutex); ++ } else { ++ /* Report the event and any pending events now */ ++ int event_count = 1; ++ ++ mutex_lock(&ctx->event_mutex); ++ event_count += kbase_event_coalesce(ctx); ++ list_add_tail(&atom->dep_item[0], &ctx->event_list); ++ atomic_add(event_count, &ctx->event_count); ++ mutex_unlock(&ctx->event_mutex); ++ ++ kbase_event_wakeup(ctx); ++ } ++} ++KBASE_EXPORT_TEST_API(kbase_event_post); ++ ++void kbase_event_close(struct kbase_context *kctx) ++{ ++ mutex_lock(&kctx->event_mutex); ++ atomic_set(&kctx->event_closed, true); ++ mutex_unlock(&kctx->event_mutex); ++ kbase_event_wakeup(kctx); ++} ++ ++int kbase_event_init(struct kbase_context *kctx) ++{ ++ KBASE_DEBUG_ASSERT(kctx); ++ ++ INIT_LIST_HEAD(&kctx->event_list); ++ INIT_LIST_HEAD(&kctx->event_coalesce_list); ++ mutex_init(&kctx->event_mutex); ++ atomic_set(&kctx->event_count, 0); ++ kctx->event_coalesce_count = 0; ++ atomic_set(&kctx->event_closed, false); ++ kctx->event_workq = alloc_workqueue("kbase_event", WQ_MEM_RECLAIM, 1); ++ ++ if (NULL == kctx->event_workq) ++ return -EINVAL; ++ ++ return 0; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_event_init); ++ ++void kbase_event_cleanup(struct kbase_context *kctx) ++{ ++ int event_count; ++ ++ KBASE_DEBUG_ASSERT(kctx); ++ KBASE_DEBUG_ASSERT(kctx->event_workq); ++ ++ flush_workqueue(kctx->event_workq); ++ destroy_workqueue(kctx->event_workq); ++ ++ /* We use kbase_event_dequeue to remove the remaining events as that ++ * deals with all the cleanup needed for the atoms. ++ * ++ * Note: use of kctx->event_list without a lock is safe because this must be the last ++ * thread using it (because we're about to terminate the lock) ++ */ ++ event_count = kbase_event_coalesce(kctx); ++ atomic_add(event_count, &kctx->event_count); ++ ++ while (!list_empty(&kctx->event_list)) { ++ struct base_jd_event_v2 event; ++ ++ kbase_event_dequeue(kctx, &event); ++ } ++} ++ ++KBASE_EXPORT_TEST_API(kbase_event_cleanup); +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_fence.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_fence.c +new file mode 100755 +index 000000000000..fcb373372596 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_fence.c +@@ -0,0 +1,196 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++/* Spin lock protecting all Mali fences as fence->lock. */ ++static DEFINE_SPINLOCK(kbase_fence_lock); ++ ++static const char * ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++kbase_fence_get_driver_name(struct fence *fence) ++#else ++kbase_fence_get_driver_name(struct dma_fence *fence) ++#endif ++{ ++ return kbase_drv_name; ++} ++ ++static const char * ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++kbase_fence_get_timeline_name(struct fence *fence) ++#else ++kbase_fence_get_timeline_name(struct dma_fence *fence) ++#endif ++{ ++ return kbase_timeline_name; ++} ++ ++static bool ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++kbase_fence_enable_signaling(struct fence *fence) ++#else ++kbase_fence_enable_signaling(struct dma_fence *fence) ++#endif ++{ ++ return true; ++} ++ ++static void ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++kbase_fence_fence_value_str(struct fence *fence, char *str, int size) ++#else ++kbase_fence_fence_value_str(struct dma_fence *fence, char *str, int size) ++#endif ++{ ++ snprintf(str, size, "%u", fence->seqno); ++} ++ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++const struct fence_ops kbase_fence_ops = { ++ .wait = fence_default_wait, ++#else ++const struct dma_fence_ops kbase_fence_ops = { ++ .wait = dma_fence_default_wait, ++#endif ++ .get_driver_name = kbase_fence_get_driver_name, ++ .get_timeline_name = kbase_fence_get_timeline_name, ++ .enable_signaling = kbase_fence_enable_signaling, ++ .fence_value_str = kbase_fence_fence_value_str ++}; ++ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++struct fence * ++kbase_fence_out_new(struct kbase_jd_atom *katom) ++#else ++struct dma_fence * ++kbase_fence_out_new(struct kbase_jd_atom *katom) ++#endif ++{ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ struct fence *fence; ++#else ++ struct dma_fence *fence; ++#endif ++ ++ WARN_ON(katom->dma_fence.fence); ++ ++ fence = kzalloc(sizeof(*fence), GFP_KERNEL); ++ if (!fence) ++ return NULL; ++ ++ dma_fence_init(fence, ++ &kbase_fence_ops, ++ &kbase_fence_lock, ++ katom->dma_fence.context, ++ atomic_inc_return(&katom->dma_fence.seqno)); ++ ++ katom->dma_fence.fence = fence; ++ ++ return fence; ++} ++ ++bool ++kbase_fence_free_callbacks(struct kbase_jd_atom *katom) ++{ ++ struct kbase_fence_cb *cb, *tmp; ++ bool res = false; ++ ++ lockdep_assert_held(&katom->kctx->jctx.lock); ++ ++ /* Clean up and free callbacks. */ ++ list_for_each_entry_safe(cb, tmp, &katom->dma_fence.callbacks, node) { ++ bool ret; ++ ++ /* Cancel callbacks that hasn't been called yet. */ ++ ret = dma_fence_remove_callback(cb->fence, &cb->fence_cb); ++ if (ret) { ++ int ret; ++ ++ /* Fence had not signaled, clean up after ++ * canceling. ++ */ ++ ret = atomic_dec_return(&katom->dma_fence.dep_count); ++ ++ if (unlikely(ret == 0)) ++ res = true; ++ } ++ ++ /* ++ * Release the reference taken in ++ * kbase_fence_add_callback(). ++ */ ++ dma_fence_put(cb->fence); ++ list_del(&cb->node); ++ kfree(cb); ++ } ++ ++ return res; ++} ++ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++int ++kbase_fence_add_callback(struct kbase_jd_atom *katom, ++ struct fence *fence, ++ fence_func_t callback) ++#else ++int ++kbase_fence_add_callback(struct kbase_jd_atom *katom, ++ struct dma_fence *fence, ++ dma_fence_func_t callback) ++#endif ++{ ++ int err = 0; ++ struct kbase_fence_cb *kbase_fence_cb; ++ ++ if (!fence) ++ return -EINVAL; ++ ++ kbase_fence_cb = kmalloc(sizeof(*kbase_fence_cb), GFP_KERNEL); ++ if (!kbase_fence_cb) ++ return -ENOMEM; ++ ++ kbase_fence_cb->fence = fence; ++ kbase_fence_cb->katom = katom; ++ INIT_LIST_HEAD(&kbase_fence_cb->node); ++ ++ err = dma_fence_add_callback(fence, &kbase_fence_cb->fence_cb, ++ callback); ++ if (err == -ENOENT) { ++ /* Fence signaled, clear the error and return */ ++ err = 0; ++ kfree(kbase_fence_cb); ++ } else if (err) { ++ kfree(kbase_fence_cb); ++ } else { ++ /* ++ * Get reference to fence that will be kept until callback gets ++ * cleaned up in kbase_fence_free_callbacks(). ++ */ ++ dma_fence_get(fence); ++ atomic_inc(&katom->dma_fence.dep_count); ++ /* Add callback to katom's list of callbacks */ ++ list_add(&kbase_fence_cb->node, &katom->dma_fence.callbacks); ++ } ++ ++ return err; ++} +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_fence.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_fence.h +new file mode 100755 +index 000000000000..9f59d30a1e2e +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_fence.h +@@ -0,0 +1,270 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KBASE_FENCE_H_ ++#define _KBASE_FENCE_H_ ++ ++/* ++ * mali_kbase_fence.[hc] has common fence code used by both ++ * - CONFIG_MALI_BIFROST_DMA_FENCE - implicit DMA fences ++ * - CONFIG_SYNC_FILE - explicit fences beginning with 4.9 kernel ++ */ ++ ++#if defined(CONFIG_MALI_BIFROST_DMA_FENCE) || defined(CONFIG_SYNC_FILE) ++ ++#include ++#include "mali_kbase_fence_defs.h" ++#include "mali_kbase.h" ++ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++extern const struct fence_ops kbase_fence_ops; ++#else ++extern const struct dma_fence_ops kbase_fence_ops; ++#endif ++ ++/** ++* struct kbase_fence_cb - Mali dma-fence callback data struct ++* @fence_cb: Callback function ++* @katom: Pointer to katom that is waiting on this callback ++* @fence: Pointer to the fence object on which this callback is waiting ++* @node: List head for linking this callback to the katom ++*/ ++struct kbase_fence_cb { ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ struct fence_cb fence_cb; ++ struct fence *fence; ++#else ++ struct dma_fence_cb fence_cb; ++ struct dma_fence *fence; ++#endif ++ struct kbase_jd_atom *katom; ++ struct list_head node; ++}; ++ ++/** ++ * kbase_fence_out_new() - Creates a new output fence and puts it on the atom ++ * @katom: Atom to create an output fence for ++ * ++ * return: A new fence object on success, NULL on failure. ++ */ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++struct fence *kbase_fence_out_new(struct kbase_jd_atom *katom); ++#else ++struct dma_fence *kbase_fence_out_new(struct kbase_jd_atom *katom); ++#endif ++ ++#if defined(CONFIG_SYNC_FILE) ++/** ++ * kbase_fence_fence_in_set() - Assign input fence to atom ++ * @katom: Atom to assign input fence to ++ * @fence: Input fence to assign to atom ++ * ++ * This function will take ownership of one fence reference! ++ */ ++#define kbase_fence_fence_in_set(katom, fence) \ ++ do { \ ++ WARN_ON((katom)->dma_fence.fence_in); \ ++ (katom)->dma_fence.fence_in = fence; \ ++ } while (0) ++#endif ++ ++/** ++ * kbase_fence_out_remove() - Removes the output fence from atom ++ * @katom: Atom to remove output fence for ++ * ++ * This will also release the reference to this fence which the atom keeps ++ */ ++static inline void kbase_fence_out_remove(struct kbase_jd_atom *katom) ++{ ++ if (katom->dma_fence.fence) { ++ dma_fence_put(katom->dma_fence.fence); ++ katom->dma_fence.fence = NULL; ++ } ++} ++ ++#if defined(CONFIG_SYNC_FILE) ++/** ++ * kbase_fence_out_remove() - Removes the input fence from atom ++ * @katom: Atom to remove input fence for ++ * ++ * This will also release the reference to this fence which the atom keeps ++ */ ++static inline void kbase_fence_in_remove(struct kbase_jd_atom *katom) ++{ ++ if (katom->dma_fence.fence_in) { ++ dma_fence_put(katom->dma_fence.fence_in); ++ katom->dma_fence.fence_in = NULL; ++ } ++} ++#endif ++ ++/** ++ * kbase_fence_out_is_ours() - Check if atom has a valid fence created by us ++ * @katom: Atom to check output fence for ++ * ++ * Return: true if fence exists and is valid, otherwise false ++ */ ++static inline bool kbase_fence_out_is_ours(struct kbase_jd_atom *katom) ++{ ++ return katom->dma_fence.fence && ++ katom->dma_fence.fence->ops == &kbase_fence_ops; ++} ++ ++/** ++ * kbase_fence_out_signal() - Signal output fence of atom ++ * @katom: Atom to signal output fence for ++ * @status: Status to signal with (0 for success, < 0 for error) ++ * ++ * Return: 0 on success, < 0 on error ++ */ ++static inline int kbase_fence_out_signal(struct kbase_jd_atom *katom, ++ int status) ++{ ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)) ++ katom->dma_fence.fence->error = status; ++#else ++ katom->dma_fence.fence->status = status; ++#endif ++ return dma_fence_signal(katom->dma_fence.fence); ++} ++ ++/** ++ * kbase_fence_add_callback() - Add callback on @fence to block @katom ++ * @katom: Pointer to katom that will be blocked by @fence ++ * @fence: Pointer to fence on which to set up the callback ++ * @callback: Pointer to function to be called when fence is signaled ++ * ++ * Caller needs to hold a reference to @fence when calling this function, and ++ * the caller is responsible for releasing that reference. An additional ++ * reference to @fence will be taken when the callback was successfully set up ++ * and @fence needs to be kept valid until the callback has been called and ++ * cleanup have been done. ++ * ++ * Return: 0 on success: fence was either already signaled, or callback was ++ * set up. Negative error code is returned on error. ++ */ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++int kbase_fence_add_callback(struct kbase_jd_atom *katom, ++ struct fence *fence, ++ fence_func_t callback); ++#else ++int kbase_fence_add_callback(struct kbase_jd_atom *katom, ++ struct dma_fence *fence, ++ dma_fence_func_t callback); ++#endif ++ ++/** ++ * kbase_fence_dep_count_set() - Set dep_count value on atom to specified value ++ * @katom: Atom to set dep_count for ++ * @val: value to set dep_count to ++ * ++ * The dep_count is available to the users of this module so that they can ++ * synchronize completion of the wait with cancellation and adding of more ++ * callbacks. For instance, a user could do the following: ++ * ++ * dep_count set to 1 ++ * callback #1 added, dep_count is increased to 2 ++ * callback #1 happens, dep_count decremented to 1 ++ * since dep_count > 0, no completion is done ++ * callback #2 is added, dep_count is increased to 2 ++ * dep_count decremented to 1 ++ * callback #2 happens, dep_count decremented to 0 ++ * since dep_count now is zero, completion executes ++ * ++ * The dep_count can also be used to make sure that the completion only ++ * executes once. This is typically done by setting dep_count to -1 for the ++ * thread that takes on this responsibility. ++ */ ++static inline void ++kbase_fence_dep_count_set(struct kbase_jd_atom *katom, int val) ++{ ++ atomic_set(&katom->dma_fence.dep_count, val); ++} ++ ++/** ++ * kbase_fence_dep_count_dec_and_test() - Decrements dep_count ++ * @katom: Atom to decrement dep_count for ++ * ++ * See @kbase_fence_dep_count_set for general description about dep_count ++ * ++ * Return: true if value was decremented to zero, otherwise false ++ */ ++static inline bool ++kbase_fence_dep_count_dec_and_test(struct kbase_jd_atom *katom) ++{ ++ return atomic_dec_and_test(&katom->dma_fence.dep_count); ++} ++ ++/** ++ * kbase_fence_dep_count_read() - Returns the current dep_count value ++ * @katom: Pointer to katom ++ * ++ * See @kbase_fence_dep_count_set for general description about dep_count ++ * ++ * Return: The current dep_count value ++ */ ++static inline int kbase_fence_dep_count_read(struct kbase_jd_atom *katom) ++{ ++ return atomic_read(&katom->dma_fence.dep_count); ++} ++ ++/** ++ * kbase_fence_free_callbacks() - Free dma-fence callbacks on a katom ++ * @katom: Pointer to katom ++ * ++ * This function will free all fence callbacks on the katom's list of ++ * callbacks. Callbacks that have not yet been called, because their fence ++ * hasn't yet signaled, will first be removed from the fence. ++ * ++ * Locking: katom->dma_fence.callbacks list assumes jctx.lock is held. ++ * ++ * Return: true if dep_count reached 0, otherwise false. ++ */ ++bool kbase_fence_free_callbacks(struct kbase_jd_atom *katom); ++ ++#if defined(CONFIG_SYNC_FILE) ++/** ++ * kbase_fence_in_get() - Retrieve input fence for atom. ++ * @katom: Atom to get input fence from ++ * ++ * A ref will be taken for the fence, so use @kbase_fence_put() to release it ++ * ++ * Return: The fence, or NULL if there is no input fence for atom ++ */ ++#define kbase_fence_in_get(katom) dma_fence_get((katom)->dma_fence.fence_in) ++#endif ++ ++/** ++ * kbase_fence_out_get() - Retrieve output fence for atom. ++ * @katom: Atom to get output fence from ++ * ++ * A ref will be taken for the fence, so use @kbase_fence_put() to release it ++ * ++ * Return: The fence, or NULL if there is no output fence for atom ++ */ ++#define kbase_fence_out_get(katom) dma_fence_get((katom)->dma_fence.fence) ++ ++/** ++ * kbase_fence_put() - Releases a reference to a fence ++ * @fence: Fence to release reference for. ++ */ ++#define kbase_fence_put(fence) dma_fence_put(fence) ++ ++ ++#endif /* CONFIG_MALI_BIFROST_DMA_FENCE || defined(CONFIG_SYNC_FILE */ ++ ++#endif /* _KBASE_FENCE_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_fence_defs.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_fence_defs.h +new file mode 100755 +index 000000000000..d2d7c436918c +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_fence_defs.h +@@ -0,0 +1,51 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KBASE_FENCE_DEFS_H_ ++#define _KBASE_FENCE_DEFS_H_ ++ ++/* ++ * There was a big rename in the 4.10 kernel (fence* -> dma_fence*) ++ * This file hides the compatibility issues with this for the rest the driver ++ */ ++ ++#if defined(CONFIG_MALI_BIFROST_DMA_FENCE) || defined(CONFIG_SYNC_FILE) ++ ++#include ++ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ ++#include ++ ++#define dma_fence_context_alloc(a) fence_context_alloc(a) ++#define dma_fence_init(a, b, c, d, e) fence_init(a, b, c, d, e) ++#define dma_fence_get(a) fence_get(a) ++#define dma_fence_put(a) fence_put(a) ++#define dma_fence_signal(a) fence_signal(a) ++#define dma_fence_is_signaled(a) fence_is_signaled(a) ++#define dma_fence_add_callback(a, b, c) fence_add_callback(a, b, c) ++#define dma_fence_remove_callback(a, b) fence_remove_callback(a, b) ++ ++#else ++ ++#include ++ ++#endif /* < 4.10.0 */ ++ ++#endif /* CONFIG_MALI_BIFROST_DMA_FENCE || CONFIG_SYNC_FILE */ ++ ++#endif /* _KBASE_FENCE_DEFS_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator.h +new file mode 100755 +index 000000000000..87697b15d986 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator.h +@@ -0,0 +1,45 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* NB taken from gator */ ++/* ++ * List of possible actions to be controlled by DS-5 Streamline. ++ * The following numbers are used by gator to control the frame buffer dumping ++ * and s/w counter reporting. We cannot use the enums in mali_uk_types.h because ++ * they are unknown inside gator. ++ */ ++#ifndef _KBASE_GATOR_H_ ++#define _KBASE_GATOR_H_ ++ ++#ifdef CONFIG_MALI_BIFROST_GATOR_SUPPORT ++#define GATOR_MAKE_EVENT(type, number) (((type) << 24) | ((number) << 16)) ++#define GATOR_JOB_SLOT_START 1 ++#define GATOR_JOB_SLOT_STOP 2 ++#define GATOR_JOB_SLOT_SOFT_STOPPED 3 ++ ++void kbase_trace_mali_job_slots_event(u32 event, const struct kbase_context *kctx, u8 atom_id); ++void kbase_trace_mali_pm_status(u32 event, u64 value); ++void kbase_trace_mali_pm_power_off(u32 event, u64 value); ++void kbase_trace_mali_pm_power_on(u32 event, u64 value); ++void kbase_trace_mali_page_fault_insert_pages(int event, u32 value); ++void kbase_trace_mali_mmu_as_in_use(int event); ++void kbase_trace_mali_mmu_as_released(int event); ++void kbase_trace_mali_total_alloc_pages_change(long long int event); ++ ++#endif /* CONFIG_MALI_BIFROST_GATOR_SUPPORT */ ++ ++#endif /* _KBASE_GATOR_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator_api.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator_api.c +new file mode 100755 +index 000000000000..860e10159fb3 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator_api.c +@@ -0,0 +1,334 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include "mali_kbase.h" ++#include "mali_kbase_hw.h" ++#include "mali_kbase_mem_linux.h" ++#include "mali_kbase_gator_api.h" ++#include "mali_kbase_gator_hwcnt_names.h" ++ ++#define MALI_MAX_CORES_PER_GROUP 4 ++#define MALI_MAX_NUM_BLOCKS_PER_GROUP 8 ++#define MALI_COUNTERS_PER_BLOCK 64 ++#define MALI_BYTES_PER_COUNTER 4 ++ ++struct kbase_gator_hwcnt_handles { ++ struct kbase_device *kbdev; ++ struct kbase_vinstr_client *vinstr_cli; ++ void *vinstr_buffer; ++ struct work_struct dump_work; ++ int dump_complete; ++ spinlock_t dump_lock; ++}; ++ ++static void dump_worker(struct work_struct *work); ++ ++const char * const *kbase_gator_hwcnt_init_names(uint32_t *total_counters) ++{ ++ const char * const *hardware_counters; ++ struct kbase_device *kbdev; ++ uint32_t product_id; ++ uint32_t count; ++ ++ if (!total_counters) ++ return NULL; ++ ++ /* Get the first device - it doesn't matter in this case */ ++ kbdev = kbase_find_device(-1); ++ if (!kbdev) ++ return NULL; ++ ++ product_id = kbdev->gpu_props.props.core_props.product_id; ++ ++ if (GPU_ID_IS_NEW_FORMAT(product_id)) { ++ switch (GPU_ID2_MODEL_MATCH_VALUE(product_id)) { ++ case GPU_ID2_PRODUCT_TMIX: ++ hardware_counters = hardware_counters_mali_tMIx; ++ count = ARRAY_SIZE(hardware_counters_mali_tMIx); ++ break; ++ case GPU_ID2_PRODUCT_THEX: ++ hardware_counters = hardware_counters_mali_tHEx; ++ count = ARRAY_SIZE(hardware_counters_mali_tHEx); ++ break; ++ case GPU_ID2_PRODUCT_TSIX: ++ hardware_counters = hardware_counters_mali_tSIx; ++ count = ARRAY_SIZE(hardware_counters_mali_tSIx); ++ break; ++ default: ++ hardware_counters = NULL; ++ count = 0; ++ dev_err(kbdev->dev, "Unrecognized product ID: %u\n", ++ product_id); ++ break; ++ } ++ } else { ++ switch (product_id) { ++ /* If we are using a Mali-T60x device */ ++ case GPU_ID_PI_T60X: ++ hardware_counters = hardware_counters_mali_t60x; ++ count = ARRAY_SIZE(hardware_counters_mali_t60x); ++ break; ++ /* If we are using a Mali-T62x device */ ++ case GPU_ID_PI_T62X: ++ hardware_counters = hardware_counters_mali_t62x; ++ count = ARRAY_SIZE(hardware_counters_mali_t62x); ++ break; ++ /* If we are using a Mali-T72x device */ ++ case GPU_ID_PI_T72X: ++ hardware_counters = hardware_counters_mali_t72x; ++ count = ARRAY_SIZE(hardware_counters_mali_t72x); ++ break; ++ /* If we are using a Mali-T76x device */ ++ case GPU_ID_PI_T76X: ++ hardware_counters = hardware_counters_mali_t76x; ++ count = ARRAY_SIZE(hardware_counters_mali_t76x); ++ break; ++ /* If we are using a Mali-T82x device */ ++ case GPU_ID_PI_T82X: ++ hardware_counters = hardware_counters_mali_t82x; ++ count = ARRAY_SIZE(hardware_counters_mali_t82x); ++ break; ++ /* If we are using a Mali-T83x device */ ++ case GPU_ID_PI_T83X: ++ hardware_counters = hardware_counters_mali_t83x; ++ count = ARRAY_SIZE(hardware_counters_mali_t83x); ++ break; ++ /* If we are using a Mali-T86x device */ ++ case GPU_ID_PI_T86X: ++ hardware_counters = hardware_counters_mali_t86x; ++ count = ARRAY_SIZE(hardware_counters_mali_t86x); ++ break; ++ /* If we are using a Mali-T88x device */ ++ case GPU_ID_PI_TFRX: ++ hardware_counters = hardware_counters_mali_t88x; ++ count = ARRAY_SIZE(hardware_counters_mali_t88x); ++ break; ++ default: ++ hardware_counters = NULL; ++ count = 0; ++ dev_err(kbdev->dev, "Unrecognized product ID: %u\n", ++ product_id); ++ break; ++ } ++ } ++ ++ /* Release the kbdev reference. */ ++ kbase_release_device(kbdev); ++ ++ *total_counters = count; ++ ++ /* If we return a string array take a reference on the module (or fail). */ ++ if (hardware_counters && !try_module_get(THIS_MODULE)) ++ return NULL; ++ ++ return hardware_counters; ++} ++KBASE_EXPORT_SYMBOL(kbase_gator_hwcnt_init_names); ++ ++void kbase_gator_hwcnt_term_names(void) ++{ ++ /* Release the module reference. */ ++ module_put(THIS_MODULE); ++} ++KBASE_EXPORT_SYMBOL(kbase_gator_hwcnt_term_names); ++ ++struct kbase_gator_hwcnt_handles *kbase_gator_hwcnt_init(struct kbase_gator_hwcnt_info *in_out_info) ++{ ++ struct kbase_gator_hwcnt_handles *hand; ++ struct kbase_uk_hwcnt_reader_setup setup; ++ uint32_t dump_size = 0, i = 0; ++ ++ if (!in_out_info) ++ return NULL; ++ ++ hand = kzalloc(sizeof(*hand), GFP_KERNEL); ++ if (!hand) ++ return NULL; ++ ++ INIT_WORK(&hand->dump_work, dump_worker); ++ spin_lock_init(&hand->dump_lock); ++ ++ /* Get the first device */ ++ hand->kbdev = kbase_find_device(-1); ++ if (!hand->kbdev) ++ goto free_hand; ++ ++ dump_size = kbase_vinstr_dump_size(hand->kbdev); ++ hand->vinstr_buffer = kzalloc(dump_size, GFP_KERNEL); ++ if (!hand->vinstr_buffer) ++ goto release_device; ++ in_out_info->kernel_dump_buffer = hand->vinstr_buffer; ++ ++ in_out_info->nr_cores = hand->kbdev->gpu_props.num_cores; ++ in_out_info->nr_core_groups = hand->kbdev->gpu_props.num_core_groups; ++ in_out_info->gpu_id = hand->kbdev->gpu_props.props.core_props.product_id; ++ ++ /* If we are using a v4 device (Mali-T6xx or Mali-T72x) */ ++ if (kbase_hw_has_feature(hand->kbdev, BASE_HW_FEATURE_V4)) { ++ uint32_t cg, j; ++ uint64_t core_mask; ++ ++ /* There are 8 hardware counters blocks per core group */ ++ in_out_info->hwc_layout = kmalloc(sizeof(enum hwc_type) * ++ MALI_MAX_NUM_BLOCKS_PER_GROUP * ++ in_out_info->nr_core_groups, GFP_KERNEL); ++ ++ if (!in_out_info->hwc_layout) ++ goto free_vinstr_buffer; ++ ++ dump_size = in_out_info->nr_core_groups * ++ MALI_MAX_NUM_BLOCKS_PER_GROUP * ++ MALI_COUNTERS_PER_BLOCK * ++ MALI_BYTES_PER_COUNTER; ++ ++ for (cg = 0; cg < in_out_info->nr_core_groups; cg++) { ++ core_mask = hand->kbdev->gpu_props.props.coherency_info.group[cg].core_mask; ++ ++ for (j = 0; j < MALI_MAX_CORES_PER_GROUP; j++) { ++ if (core_mask & (1u << j)) ++ in_out_info->hwc_layout[i++] = SHADER_BLOCK; ++ else ++ in_out_info->hwc_layout[i++] = RESERVED_BLOCK; ++ } ++ ++ in_out_info->hwc_layout[i++] = TILER_BLOCK; ++ in_out_info->hwc_layout[i++] = MMU_L2_BLOCK; ++ ++ in_out_info->hwc_layout[i++] = RESERVED_BLOCK; ++ ++ if (0 == cg) ++ in_out_info->hwc_layout[i++] = JM_BLOCK; ++ else ++ in_out_info->hwc_layout[i++] = RESERVED_BLOCK; ++ } ++ /* If we are using any other device */ ++ } else { ++ uint32_t nr_l2, nr_sc_bits, j; ++ uint64_t core_mask; ++ ++ nr_l2 = hand->kbdev->gpu_props.props.l2_props.num_l2_slices; ++ ++ core_mask = hand->kbdev->gpu_props.props.coherency_info.group[0].core_mask; ++ ++ nr_sc_bits = fls64(core_mask); ++ ++ /* The job manager and tiler sets of counters ++ * are always present */ ++ in_out_info->hwc_layout = kmalloc(sizeof(enum hwc_type) * (2 + nr_sc_bits + nr_l2), GFP_KERNEL); ++ ++ if (!in_out_info->hwc_layout) ++ goto free_vinstr_buffer; ++ ++ dump_size = (2 + nr_sc_bits + nr_l2) * MALI_COUNTERS_PER_BLOCK * MALI_BYTES_PER_COUNTER; ++ ++ in_out_info->hwc_layout[i++] = JM_BLOCK; ++ in_out_info->hwc_layout[i++] = TILER_BLOCK; ++ ++ for (j = 0; j < nr_l2; j++) ++ in_out_info->hwc_layout[i++] = MMU_L2_BLOCK; ++ ++ while (core_mask != 0ull) { ++ if ((core_mask & 1ull) != 0ull) ++ in_out_info->hwc_layout[i++] = SHADER_BLOCK; ++ else ++ in_out_info->hwc_layout[i++] = RESERVED_BLOCK; ++ core_mask >>= 1; ++ } ++ } ++ ++ in_out_info->nr_hwc_blocks = i; ++ in_out_info->size = dump_size; ++ ++ setup.jm_bm = in_out_info->bitmask[0]; ++ setup.tiler_bm = in_out_info->bitmask[1]; ++ setup.shader_bm = in_out_info->bitmask[2]; ++ setup.mmu_l2_bm = in_out_info->bitmask[3]; ++ hand->vinstr_cli = kbase_vinstr_hwcnt_kernel_setup(hand->kbdev->vinstr_ctx, ++ &setup, hand->vinstr_buffer); ++ if (!hand->vinstr_cli) { ++ dev_err(hand->kbdev->dev, "Failed to register gator with vinstr core"); ++ goto free_layout; ++ } ++ ++ return hand; ++ ++free_layout: ++ kfree(in_out_info->hwc_layout); ++ ++free_vinstr_buffer: ++ kfree(hand->vinstr_buffer); ++ ++release_device: ++ kbase_release_device(hand->kbdev); ++ ++free_hand: ++ kfree(hand); ++ return NULL; ++} ++KBASE_EXPORT_SYMBOL(kbase_gator_hwcnt_init); ++ ++void kbase_gator_hwcnt_term(struct kbase_gator_hwcnt_info *in_out_info, struct kbase_gator_hwcnt_handles *opaque_handles) ++{ ++ if (in_out_info) ++ kfree(in_out_info->hwc_layout); ++ ++ if (opaque_handles) { ++ cancel_work_sync(&opaque_handles->dump_work); ++ kbase_vinstr_detach_client(opaque_handles->vinstr_cli); ++ kfree(opaque_handles->vinstr_buffer); ++ kbase_release_device(opaque_handles->kbdev); ++ kfree(opaque_handles); ++ } ++} ++KBASE_EXPORT_SYMBOL(kbase_gator_hwcnt_term); ++ ++static void dump_worker(struct work_struct *work) ++{ ++ struct kbase_gator_hwcnt_handles *hand; ++ ++ hand = container_of(work, struct kbase_gator_hwcnt_handles, dump_work); ++ if (!kbase_vinstr_hwc_dump(hand->vinstr_cli, ++ BASE_HWCNT_READER_EVENT_MANUAL)) { ++ spin_lock_bh(&hand->dump_lock); ++ hand->dump_complete = 1; ++ spin_unlock_bh(&hand->dump_lock); ++ } else { ++ schedule_work(&hand->dump_work); ++ } ++} ++ ++uint32_t kbase_gator_instr_hwcnt_dump_complete( ++ struct kbase_gator_hwcnt_handles *opaque_handles, ++ uint32_t * const success) ++{ ++ ++ if (opaque_handles && success) { ++ *success = opaque_handles->dump_complete; ++ opaque_handles->dump_complete = 0; ++ return *success; ++ } ++ return 0; ++} ++KBASE_EXPORT_SYMBOL(kbase_gator_instr_hwcnt_dump_complete); ++ ++uint32_t kbase_gator_instr_hwcnt_dump_irq(struct kbase_gator_hwcnt_handles *opaque_handles) ++{ ++ if (opaque_handles) ++ schedule_work(&opaque_handles->dump_work); ++ return 0; ++} ++KBASE_EXPORT_SYMBOL(kbase_gator_instr_hwcnt_dump_irq); +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator_api.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator_api.h +new file mode 100755 +index 000000000000..ef9ac0f7b633 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator_api.h +@@ -0,0 +1,219 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KBASE_GATOR_API_H_ ++#define _KBASE_GATOR_API_H_ ++ ++/** ++ * @brief This file describes the API used by Gator to fetch hardware counters. ++ */ ++ ++/* This define is used by the gator kernel module compile to select which DDK ++ * API calling convention to use. If not defined (legacy DDK) gator assumes ++ * version 1. The version to DDK release mapping is: ++ * Version 1 API: DDK versions r1px, r2px ++ * Version 2 API: DDK versions r3px, r4px ++ * Version 3 API: DDK version r5p0 and newer ++ * ++ * API Usage ++ * ========= ++ * ++ * 1] Call kbase_gator_hwcnt_init_names() to return the list of short counter ++ * names for the GPU present in this device. ++ * ++ * 2] Create a kbase_gator_hwcnt_info structure and set the counter enables for ++ * the counters you want enabled. The enables can all be set for simplicity in ++ * most use cases, but disabling some will let you minimize bandwidth impact. ++ * ++ * 3] Call kbase_gator_hwcnt_init() using the above structure, to create a ++ * counter context. On successful return the DDK will have populated the ++ * structure with a variety of useful information. ++ * ++ * 4] Call kbase_gator_hwcnt_dump_irq() to queue a non-blocking request for a ++ * counter dump. If this returns a non-zero value the request has been queued, ++ * otherwise the driver has been unable to do so (typically because of another ++ * user of the instrumentation exists concurrently). ++ * ++ * 5] Call kbase_gator_hwcnt_dump_complete() to test whether the previously ++ * requested dump has been succesful. If this returns non-zero the counter dump ++ * has resolved, but the value of *success must also be tested as the dump ++ * may have not been successful. If it returns zero the counter dump was ++ * abandoned due to the device being busy (typically because of another ++ * user of the instrumentation exists concurrently). ++ * ++ * 6] Process the counters stored in the buffer pointed to by ... ++ * ++ * kbase_gator_hwcnt_info->kernel_dump_buffer ++ * ++ * In pseudo code you can find all of the counters via this approach: ++ * ++ * ++ * hwcnt_info # pointer to kbase_gator_hwcnt_info structure ++ * hwcnt_name # pointer to name list ++ * ++ * u32 * hwcnt_data = (u32*)hwcnt_info->kernel_dump_buffer ++ * ++ * # Iterate over each 64-counter block in this GPU configuration ++ * for( i = 0; i < hwcnt_info->nr_hwc_blocks; i++) { ++ * hwc_type type = hwcnt_info->hwc_layout[i]; ++ * ++ * # Skip reserved type blocks - they contain no counters at all ++ * if( type == RESERVED_BLOCK ) { ++ * continue; ++ * } ++ * ++ * size_t name_offset = type * 64; ++ * size_t data_offset = i * 64; ++ * ++ * # Iterate over the names of the counters in this block type ++ * for( j = 0; j < 64; j++) { ++ * const char * name = hwcnt_name[name_offset+j]; ++ * ++ * # Skip empty name strings - there is no counter here ++ * if( name[0] == '\0' ) { ++ * continue; ++ * } ++ * ++ * u32 data = hwcnt_data[data_offset+j]; ++ * ++ * printk( "COUNTER: %s DATA: %u\n", name, data ); ++ * } ++ * } ++ * ++ * ++ * Note that in most implementations you typically want to either SUM or ++ * AVERAGE multiple instances of the same counter if, for example, you have ++ * multiple shader cores or multiple L2 caches. The most sensible view for ++ * analysis is to AVERAGE shader core counters, but SUM L2 cache and MMU ++ * counters. ++ * ++ * 7] Goto 4, repeating until you want to stop collecting counters. ++ * ++ * 8] Release the dump resources by calling kbase_gator_hwcnt_term(). ++ * ++ * 9] Release the name table resources by calling ++ * kbase_gator_hwcnt_term_names(). This function must only be called if ++ * init_names() returned a non-NULL value. ++ **/ ++ ++#define MALI_DDK_GATOR_API_VERSION 3 ++ ++enum hwc_type { ++ JM_BLOCK = 0, ++ TILER_BLOCK, ++ SHADER_BLOCK, ++ MMU_L2_BLOCK, ++ RESERVED_BLOCK ++}; ++ ++struct kbase_gator_hwcnt_info { ++ /* Passed from Gator to kbase */ ++ ++ /* the bitmask of enabled hardware counters for each counter block */ ++ uint16_t bitmask[4]; ++ ++ /* Passed from kbase to Gator */ ++ ++ /* ptr to counter dump memory */ ++ void *kernel_dump_buffer; ++ ++ /* size of counter dump memory */ ++ uint32_t size; ++ ++ /* the ID of the Mali device */ ++ uint32_t gpu_id; ++ ++ /* the number of shader cores in the GPU */ ++ uint32_t nr_cores; ++ ++ /* the number of core groups */ ++ uint32_t nr_core_groups; ++ ++ /* the memory layout of the performance counters */ ++ enum hwc_type *hwc_layout; ++ ++ /* the total number of hardware couter blocks */ ++ uint32_t nr_hwc_blocks; ++}; ++ ++/** ++ * @brief Opaque block of Mali data which Gator needs to return to the API later. ++ */ ++struct kbase_gator_hwcnt_handles; ++ ++/** ++ * @brief Initialize the resources Gator needs for performance profiling. ++ * ++ * @param in_out_info A pointer to a structure containing the enabled counters passed from Gator and all the Mali ++ * specific information that will be returned to Gator. On entry Gator must have populated the ++ * 'bitmask' field with the counters it wishes to enable for each class of counter block. ++ * Each entry in the array corresponds to a single counter class based on the "hwc_type" ++ * enumeration, and each bit corresponds to an enable for 4 sequential counters (LSB enables ++ * the first 4 counters in the block, and so on). See the GPU counter array as returned by ++ * kbase_gator_hwcnt_get_names() for the index values of each counter for the curernt GPU. ++ * ++ * @return Pointer to an opaque handle block on success, NULL on error. ++ */ ++extern struct kbase_gator_hwcnt_handles *kbase_gator_hwcnt_init(struct kbase_gator_hwcnt_info *in_out_info); ++ ++/** ++ * @brief Free all resources once Gator has finished using performance counters. ++ * ++ * @param in_out_info A pointer to a structure containing the enabled counters passed from Gator and all the ++ * Mali specific information that will be returned to Gator. ++ * @param opaque_handles A wrapper structure for kbase structures. ++ */ ++extern void kbase_gator_hwcnt_term(struct kbase_gator_hwcnt_info *in_out_info, struct kbase_gator_hwcnt_handles *opaque_handles); ++ ++/** ++ * @brief Poll whether a counter dump is successful. ++ * ++ * @param opaque_handles A wrapper structure for kbase structures. ++ * @param[out] success Non-zero on success, zero on failure. ++ * ++ * @return Zero if the dump is still pending, non-zero if the dump has completed. Note that a ++ * completed dump may not have dumped succesfully, so the caller must test for both ++ * a completed and successful dump before processing counters. ++ */ ++extern uint32_t kbase_gator_instr_hwcnt_dump_complete(struct kbase_gator_hwcnt_handles *opaque_handles, uint32_t * const success); ++ ++/** ++ * @brief Request the generation of a new counter dump. ++ * ++ * @param opaque_handles A wrapper structure for kbase structures. ++ * ++ * @return Zero if the hardware device is busy and cannot handle the request, non-zero otherwise. ++ */ ++extern uint32_t kbase_gator_instr_hwcnt_dump_irq(struct kbase_gator_hwcnt_handles *opaque_handles); ++ ++/** ++ * @brief This function is used to fetch the names table based on the Mali device in use. ++ * ++ * @param[out] total_counters The total number of counters short names in the Mali devices' list. ++ * ++ * @return Pointer to an array of strings of length *total_counters. ++ */ ++extern const char * const *kbase_gator_hwcnt_init_names(uint32_t *total_counters); ++ ++/** ++ * @brief This function is used to terminate the use of the names table. ++ * ++ * This function must only be called if the initial call to kbase_gator_hwcnt_init_names returned a non-NULL value. ++ */ ++extern void kbase_gator_hwcnt_term_names(void); ++ ++#endif +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator_hwcnt_names.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator_hwcnt_names.h +new file mode 100755 +index 000000000000..24103e292453 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator_hwcnt_names.h +@@ -0,0 +1,2167 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KBASE_GATOR_HWCNT_NAMES_H_ ++#define _KBASE_GATOR_HWCNT_NAMES_H_ ++ ++/* ++ * "Short names" for hardware counters used by Streamline. Counters names are ++ * stored in accordance with their memory layout in the binary counter block ++ * emitted by the Mali GPU. Each "master" in the GPU emits a fixed-size block ++ * of 64 counters, and each GPU implements the same set of "masters" although ++ * the counters each master exposes within its block of 64 may vary. ++ * ++ * Counters which are an empty string are simply "holes" in the counter memory ++ * where no counter exists. ++ */ ++ ++static const char * const hardware_counters_mali_t60x[] = { ++ /* Job Manager */ ++ "", ++ "", ++ "", ++ "", ++ "T60x_MESSAGES_SENT", ++ "T60x_MESSAGES_RECEIVED", ++ "T60x_GPU_ACTIVE", ++ "T60x_IRQ_ACTIVE", ++ "T60x_JS0_JOBS", ++ "T60x_JS0_TASKS", ++ "T60x_JS0_ACTIVE", ++ "", ++ "T60x_JS0_WAIT_READ", ++ "T60x_JS0_WAIT_ISSUE", ++ "T60x_JS0_WAIT_DEPEND", ++ "T60x_JS0_WAIT_FINISH", ++ "T60x_JS1_JOBS", ++ "T60x_JS1_TASKS", ++ "T60x_JS1_ACTIVE", ++ "", ++ "T60x_JS1_WAIT_READ", ++ "T60x_JS1_WAIT_ISSUE", ++ "T60x_JS1_WAIT_DEPEND", ++ "T60x_JS1_WAIT_FINISH", ++ "T60x_JS2_JOBS", ++ "T60x_JS2_TASKS", ++ "T60x_JS2_ACTIVE", ++ "", ++ "T60x_JS2_WAIT_READ", ++ "T60x_JS2_WAIT_ISSUE", ++ "T60x_JS2_WAIT_DEPEND", ++ "T60x_JS2_WAIT_FINISH", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ ++ /*Tiler */ ++ "", ++ "", ++ "", ++ "T60x_TI_JOBS_PROCESSED", ++ "T60x_TI_TRIANGLES", ++ "T60x_TI_QUADS", ++ "T60x_TI_POLYGONS", ++ "T60x_TI_POINTS", ++ "T60x_TI_LINES", ++ "T60x_TI_VCACHE_HIT", ++ "T60x_TI_VCACHE_MISS", ++ "T60x_TI_FRONT_FACING", ++ "T60x_TI_BACK_FACING", ++ "T60x_TI_PRIM_VISIBLE", ++ "T60x_TI_PRIM_CULLED", ++ "T60x_TI_PRIM_CLIPPED", ++ "T60x_TI_LEVEL0", ++ "T60x_TI_LEVEL1", ++ "T60x_TI_LEVEL2", ++ "T60x_TI_LEVEL3", ++ "T60x_TI_LEVEL4", ++ "T60x_TI_LEVEL5", ++ "T60x_TI_LEVEL6", ++ "T60x_TI_LEVEL7", ++ "T60x_TI_COMMAND_1", ++ "T60x_TI_COMMAND_2", ++ "T60x_TI_COMMAND_3", ++ "T60x_TI_COMMAND_4", ++ "T60x_TI_COMMAND_4_7", ++ "T60x_TI_COMMAND_8_15", ++ "T60x_TI_COMMAND_16_63", ++ "T60x_TI_COMMAND_64", ++ "T60x_TI_COMPRESS_IN", ++ "T60x_TI_COMPRESS_OUT", ++ "T60x_TI_COMPRESS_FLUSH", ++ "T60x_TI_TIMESTAMPS", ++ "T60x_TI_PCACHE_HIT", ++ "T60x_TI_PCACHE_MISS", ++ "T60x_TI_PCACHE_LINE", ++ "T60x_TI_PCACHE_STALL", ++ "T60x_TI_WRBUF_HIT", ++ "T60x_TI_WRBUF_MISS", ++ "T60x_TI_WRBUF_LINE", ++ "T60x_TI_WRBUF_PARTIAL", ++ "T60x_TI_WRBUF_STALL", ++ "T60x_TI_ACTIVE", ++ "T60x_TI_LOADING_DESC", ++ "T60x_TI_INDEX_WAIT", ++ "T60x_TI_INDEX_RANGE_WAIT", ++ "T60x_TI_VERTEX_WAIT", ++ "T60x_TI_PCACHE_WAIT", ++ "T60x_TI_WRBUF_WAIT", ++ "T60x_TI_BUS_READ", ++ "T60x_TI_BUS_WRITE", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "T60x_TI_UTLB_STALL", ++ "T60x_TI_UTLB_REPLAY_MISS", ++ "T60x_TI_UTLB_REPLAY_FULL", ++ "T60x_TI_UTLB_NEW_MISS", ++ "T60x_TI_UTLB_HIT", ++ ++ /* Shader Core */ ++ "", ++ "", ++ "", ++ "", ++ "T60x_FRAG_ACTIVE", ++ "T60x_FRAG_PRIMITIVES", ++ "T60x_FRAG_PRIMITIVES_DROPPED", ++ "T60x_FRAG_CYCLES_DESC", ++ "T60x_FRAG_CYCLES_PLR", ++ "T60x_FRAG_CYCLES_VERT", ++ "T60x_FRAG_CYCLES_TRISETUP", ++ "T60x_FRAG_CYCLES_RAST", ++ "T60x_FRAG_THREADS", ++ "T60x_FRAG_DUMMY_THREADS", ++ "T60x_FRAG_QUADS_RAST", ++ "T60x_FRAG_QUADS_EZS_TEST", ++ "T60x_FRAG_QUADS_EZS_KILLED", ++ "T60x_FRAG_THREADS_LZS_TEST", ++ "T60x_FRAG_THREADS_LZS_KILLED", ++ "T60x_FRAG_CYCLES_NO_TILE", ++ "T60x_FRAG_NUM_TILES", ++ "T60x_FRAG_TRANS_ELIM", ++ "T60x_COMPUTE_ACTIVE", ++ "T60x_COMPUTE_TASKS", ++ "T60x_COMPUTE_THREADS", ++ "T60x_COMPUTE_CYCLES_DESC", ++ "T60x_TRIPIPE_ACTIVE", ++ "T60x_ARITH_WORDS", ++ "T60x_ARITH_CYCLES_REG", ++ "T60x_ARITH_CYCLES_L0", ++ "T60x_ARITH_FRAG_DEPEND", ++ "T60x_LS_WORDS", ++ "T60x_LS_ISSUES", ++ "T60x_LS_RESTARTS", ++ "T60x_LS_REISSUES_MISS", ++ "T60x_LS_REISSUES_VD", ++ "T60x_LS_REISSUE_ATTRIB_MISS", ++ "T60x_LS_NO_WB", ++ "T60x_TEX_WORDS", ++ "T60x_TEX_BUBBLES", ++ "T60x_TEX_WORDS_L0", ++ "T60x_TEX_WORDS_DESC", ++ "T60x_TEX_ISSUES", ++ "T60x_TEX_RECIRC_FMISS", ++ "T60x_TEX_RECIRC_DESC", ++ "T60x_TEX_RECIRC_MULTI", ++ "T60x_TEX_RECIRC_PMISS", ++ "T60x_TEX_RECIRC_CONF", ++ "T60x_LSC_READ_HITS", ++ "T60x_LSC_READ_MISSES", ++ "T60x_LSC_WRITE_HITS", ++ "T60x_LSC_WRITE_MISSES", ++ "T60x_LSC_ATOMIC_HITS", ++ "T60x_LSC_ATOMIC_MISSES", ++ "T60x_LSC_LINE_FETCHES", ++ "T60x_LSC_DIRTY_LINE", ++ "T60x_LSC_SNOOPS", ++ "T60x_AXI_TLB_STALL", ++ "T60x_AXI_TLB_MISS", ++ "T60x_AXI_TLB_TRANSACTION", ++ "T60x_LS_TLB_MISS", ++ "T60x_LS_TLB_HIT", ++ "T60x_AXI_BEATS_READ", ++ "T60x_AXI_BEATS_WRITTEN", ++ ++ /*L2 and MMU */ ++ "", ++ "", ++ "", ++ "", ++ "T60x_MMU_HIT", ++ "T60x_MMU_NEW_MISS", ++ "T60x_MMU_REPLAY_FULL", ++ "T60x_MMU_REPLAY_MISS", ++ "T60x_MMU_TABLE_WALK", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "T60x_UTLB_HIT", ++ "T60x_UTLB_NEW_MISS", ++ "T60x_UTLB_REPLAY_FULL", ++ "T60x_UTLB_REPLAY_MISS", ++ "T60x_UTLB_STALL", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "T60x_L2_EXT_WRITE_BEATS", ++ "T60x_L2_EXT_READ_BEATS", ++ "T60x_L2_ANY_LOOKUP", ++ "T60x_L2_READ_LOOKUP", ++ "T60x_L2_SREAD_LOOKUP", ++ "T60x_L2_READ_REPLAY", ++ "T60x_L2_READ_SNOOP", ++ "T60x_L2_READ_HIT", ++ "T60x_L2_CLEAN_MISS", ++ "T60x_L2_WRITE_LOOKUP", ++ "T60x_L2_SWRITE_LOOKUP", ++ "T60x_L2_WRITE_REPLAY", ++ "T60x_L2_WRITE_SNOOP", ++ "T60x_L2_WRITE_HIT", ++ "T60x_L2_EXT_READ_FULL", ++ "T60x_L2_EXT_READ_HALF", ++ "T60x_L2_EXT_WRITE_FULL", ++ "T60x_L2_EXT_WRITE_HALF", ++ "T60x_L2_EXT_READ", ++ "T60x_L2_EXT_READ_LINE", ++ "T60x_L2_EXT_WRITE", ++ "T60x_L2_EXT_WRITE_LINE", ++ "T60x_L2_EXT_WRITE_SMALL", ++ "T60x_L2_EXT_BARRIER", ++ "T60x_L2_EXT_AR_STALL", ++ "T60x_L2_EXT_R_BUF_FULL", ++ "T60x_L2_EXT_RD_BUF_FULL", ++ "T60x_L2_EXT_R_RAW", ++ "T60x_L2_EXT_W_STALL", ++ "T60x_L2_EXT_W_BUF_FULL", ++ "T60x_L2_EXT_R_W_HAZARD", ++ "T60x_L2_TAG_HAZARD", ++ "T60x_L2_SNOOP_FULL", ++ "T60x_L2_REPLAY_FULL" ++}; ++static const char * const hardware_counters_mali_t62x[] = { ++ /* Job Manager */ ++ "", ++ "", ++ "", ++ "", ++ "T62x_MESSAGES_SENT", ++ "T62x_MESSAGES_RECEIVED", ++ "T62x_GPU_ACTIVE", ++ "T62x_IRQ_ACTIVE", ++ "T62x_JS0_JOBS", ++ "T62x_JS0_TASKS", ++ "T62x_JS0_ACTIVE", ++ "", ++ "T62x_JS0_WAIT_READ", ++ "T62x_JS0_WAIT_ISSUE", ++ "T62x_JS0_WAIT_DEPEND", ++ "T62x_JS0_WAIT_FINISH", ++ "T62x_JS1_JOBS", ++ "T62x_JS1_TASKS", ++ "T62x_JS1_ACTIVE", ++ "", ++ "T62x_JS1_WAIT_READ", ++ "T62x_JS1_WAIT_ISSUE", ++ "T62x_JS1_WAIT_DEPEND", ++ "T62x_JS1_WAIT_FINISH", ++ "T62x_JS2_JOBS", ++ "T62x_JS2_TASKS", ++ "T62x_JS2_ACTIVE", ++ "", ++ "T62x_JS2_WAIT_READ", ++ "T62x_JS2_WAIT_ISSUE", ++ "T62x_JS2_WAIT_DEPEND", ++ "T62x_JS2_WAIT_FINISH", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ ++ /*Tiler */ ++ "", ++ "", ++ "", ++ "T62x_TI_JOBS_PROCESSED", ++ "T62x_TI_TRIANGLES", ++ "T62x_TI_QUADS", ++ "T62x_TI_POLYGONS", ++ "T62x_TI_POINTS", ++ "T62x_TI_LINES", ++ "T62x_TI_VCACHE_HIT", ++ "T62x_TI_VCACHE_MISS", ++ "T62x_TI_FRONT_FACING", ++ "T62x_TI_BACK_FACING", ++ "T62x_TI_PRIM_VISIBLE", ++ "T62x_TI_PRIM_CULLED", ++ "T62x_TI_PRIM_CLIPPED", ++ "T62x_TI_LEVEL0", ++ "T62x_TI_LEVEL1", ++ "T62x_TI_LEVEL2", ++ "T62x_TI_LEVEL3", ++ "T62x_TI_LEVEL4", ++ "T62x_TI_LEVEL5", ++ "T62x_TI_LEVEL6", ++ "T62x_TI_LEVEL7", ++ "T62x_TI_COMMAND_1", ++ "T62x_TI_COMMAND_2", ++ "T62x_TI_COMMAND_3", ++ "T62x_TI_COMMAND_4", ++ "T62x_TI_COMMAND_5_7", ++ "T62x_TI_COMMAND_8_15", ++ "T62x_TI_COMMAND_16_63", ++ "T62x_TI_COMMAND_64", ++ "T62x_TI_COMPRESS_IN", ++ "T62x_TI_COMPRESS_OUT", ++ "T62x_TI_COMPRESS_FLUSH", ++ "T62x_TI_TIMESTAMPS", ++ "T62x_TI_PCACHE_HIT", ++ "T62x_TI_PCACHE_MISS", ++ "T62x_TI_PCACHE_LINE", ++ "T62x_TI_PCACHE_STALL", ++ "T62x_TI_WRBUF_HIT", ++ "T62x_TI_WRBUF_MISS", ++ "T62x_TI_WRBUF_LINE", ++ "T62x_TI_WRBUF_PARTIAL", ++ "T62x_TI_WRBUF_STALL", ++ "T62x_TI_ACTIVE", ++ "T62x_TI_LOADING_DESC", ++ "T62x_TI_INDEX_WAIT", ++ "T62x_TI_INDEX_RANGE_WAIT", ++ "T62x_TI_VERTEX_WAIT", ++ "T62x_TI_PCACHE_WAIT", ++ "T62x_TI_WRBUF_WAIT", ++ "T62x_TI_BUS_READ", ++ "T62x_TI_BUS_WRITE", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "T62x_TI_UTLB_STALL", ++ "T62x_TI_UTLB_REPLAY_MISS", ++ "T62x_TI_UTLB_REPLAY_FULL", ++ "T62x_TI_UTLB_NEW_MISS", ++ "T62x_TI_UTLB_HIT", ++ ++ /* Shader Core */ ++ "", ++ "", ++ "", ++ "T62x_SHADER_CORE_ACTIVE", ++ "T62x_FRAG_ACTIVE", ++ "T62x_FRAG_PRIMITIVES", ++ "T62x_FRAG_PRIMITIVES_DROPPED", ++ "T62x_FRAG_CYCLES_DESC", ++ "T62x_FRAG_CYCLES_FPKQ_ACTIVE", ++ "T62x_FRAG_CYCLES_VERT", ++ "T62x_FRAG_CYCLES_TRISETUP", ++ "T62x_FRAG_CYCLES_EZS_ACTIVE", ++ "T62x_FRAG_THREADS", ++ "T62x_FRAG_DUMMY_THREADS", ++ "T62x_FRAG_QUADS_RAST", ++ "T62x_FRAG_QUADS_EZS_TEST", ++ "T62x_FRAG_QUADS_EZS_KILLED", ++ "T62x_FRAG_THREADS_LZS_TEST", ++ "T62x_FRAG_THREADS_LZS_KILLED", ++ "T62x_FRAG_CYCLES_NO_TILE", ++ "T62x_FRAG_NUM_TILES", ++ "T62x_FRAG_TRANS_ELIM", ++ "T62x_COMPUTE_ACTIVE", ++ "T62x_COMPUTE_TASKS", ++ "T62x_COMPUTE_THREADS", ++ "T62x_COMPUTE_CYCLES_DESC", ++ "T62x_TRIPIPE_ACTIVE", ++ "T62x_ARITH_WORDS", ++ "T62x_ARITH_CYCLES_REG", ++ "T62x_ARITH_CYCLES_L0", ++ "T62x_ARITH_FRAG_DEPEND", ++ "T62x_LS_WORDS", ++ "T62x_LS_ISSUES", ++ "T62x_LS_RESTARTS", ++ "T62x_LS_REISSUES_MISS", ++ "T62x_LS_REISSUES_VD", ++ "T62x_LS_REISSUE_ATTRIB_MISS", ++ "T62x_LS_NO_WB", ++ "T62x_TEX_WORDS", ++ "T62x_TEX_BUBBLES", ++ "T62x_TEX_WORDS_L0", ++ "T62x_TEX_WORDS_DESC", ++ "T62x_TEX_ISSUES", ++ "T62x_TEX_RECIRC_FMISS", ++ "T62x_TEX_RECIRC_DESC", ++ "T62x_TEX_RECIRC_MULTI", ++ "T62x_TEX_RECIRC_PMISS", ++ "T62x_TEX_RECIRC_CONF", ++ "T62x_LSC_READ_HITS", ++ "T62x_LSC_READ_MISSES", ++ "T62x_LSC_WRITE_HITS", ++ "T62x_LSC_WRITE_MISSES", ++ "T62x_LSC_ATOMIC_HITS", ++ "T62x_LSC_ATOMIC_MISSES", ++ "T62x_LSC_LINE_FETCHES", ++ "T62x_LSC_DIRTY_LINE", ++ "T62x_LSC_SNOOPS", ++ "T62x_AXI_TLB_STALL", ++ "T62x_AXI_TLB_MISS", ++ "T62x_AXI_TLB_TRANSACTION", ++ "T62x_LS_TLB_MISS", ++ "T62x_LS_TLB_HIT", ++ "T62x_AXI_BEATS_READ", ++ "T62x_AXI_BEATS_WRITTEN", ++ ++ /*L2 and MMU */ ++ "", ++ "", ++ "", ++ "", ++ "T62x_MMU_HIT", ++ "T62x_MMU_NEW_MISS", ++ "T62x_MMU_REPLAY_FULL", ++ "T62x_MMU_REPLAY_MISS", ++ "T62x_MMU_TABLE_WALK", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "T62x_UTLB_HIT", ++ "T62x_UTLB_NEW_MISS", ++ "T62x_UTLB_REPLAY_FULL", ++ "T62x_UTLB_REPLAY_MISS", ++ "T62x_UTLB_STALL", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "T62x_L2_EXT_WRITE_BEATS", ++ "T62x_L2_EXT_READ_BEATS", ++ "T62x_L2_ANY_LOOKUP", ++ "T62x_L2_READ_LOOKUP", ++ "T62x_L2_SREAD_LOOKUP", ++ "T62x_L2_READ_REPLAY", ++ "T62x_L2_READ_SNOOP", ++ "T62x_L2_READ_HIT", ++ "T62x_L2_CLEAN_MISS", ++ "T62x_L2_WRITE_LOOKUP", ++ "T62x_L2_SWRITE_LOOKUP", ++ "T62x_L2_WRITE_REPLAY", ++ "T62x_L2_WRITE_SNOOP", ++ "T62x_L2_WRITE_HIT", ++ "T62x_L2_EXT_READ_FULL", ++ "T62x_L2_EXT_READ_HALF", ++ "T62x_L2_EXT_WRITE_FULL", ++ "T62x_L2_EXT_WRITE_HALF", ++ "T62x_L2_EXT_READ", ++ "T62x_L2_EXT_READ_LINE", ++ "T62x_L2_EXT_WRITE", ++ "T62x_L2_EXT_WRITE_LINE", ++ "T62x_L2_EXT_WRITE_SMALL", ++ "T62x_L2_EXT_BARRIER", ++ "T62x_L2_EXT_AR_STALL", ++ "T62x_L2_EXT_R_BUF_FULL", ++ "T62x_L2_EXT_RD_BUF_FULL", ++ "T62x_L2_EXT_R_RAW", ++ "T62x_L2_EXT_W_STALL", ++ "T62x_L2_EXT_W_BUF_FULL", ++ "T62x_L2_EXT_R_W_HAZARD", ++ "T62x_L2_TAG_HAZARD", ++ "T62x_L2_SNOOP_FULL", ++ "T62x_L2_REPLAY_FULL" ++}; ++ ++static const char * const hardware_counters_mali_t72x[] = { ++ /* Job Manager */ ++ "", ++ "", ++ "", ++ "", ++ "T72x_GPU_ACTIVE", ++ "T72x_IRQ_ACTIVE", ++ "T72x_JS0_JOBS", ++ "T72x_JS0_TASKS", ++ "T72x_JS0_ACTIVE", ++ "T72x_JS1_JOBS", ++ "T72x_JS1_TASKS", ++ "T72x_JS1_ACTIVE", ++ "T72x_JS2_JOBS", ++ "T72x_JS2_TASKS", ++ "T72x_JS2_ACTIVE", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ ++ /*Tiler */ ++ "", ++ "", ++ "", ++ "T72x_TI_JOBS_PROCESSED", ++ "T72x_TI_TRIANGLES", ++ "T72x_TI_QUADS", ++ "T72x_TI_POLYGONS", ++ "T72x_TI_POINTS", ++ "T72x_TI_LINES", ++ "T72x_TI_FRONT_FACING", ++ "T72x_TI_BACK_FACING", ++ "T72x_TI_PRIM_VISIBLE", ++ "T72x_TI_PRIM_CULLED", ++ "T72x_TI_PRIM_CLIPPED", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "T72x_TI_ACTIVE", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ ++ /* Shader Core */ ++ "", ++ "", ++ "", ++ "", ++ "T72x_FRAG_ACTIVE", ++ "T72x_FRAG_PRIMITIVES", ++ "T72x_FRAG_PRIMITIVES_DROPPED", ++ "T72x_FRAG_THREADS", ++ "T72x_FRAG_DUMMY_THREADS", ++ "T72x_FRAG_QUADS_RAST", ++ "T72x_FRAG_QUADS_EZS_TEST", ++ "T72x_FRAG_QUADS_EZS_KILLED", ++ "T72x_FRAG_THREADS_LZS_TEST", ++ "T72x_FRAG_THREADS_LZS_KILLED", ++ "T72x_FRAG_CYCLES_NO_TILE", ++ "T72x_FRAG_NUM_TILES", ++ "T72x_FRAG_TRANS_ELIM", ++ "T72x_COMPUTE_ACTIVE", ++ "T72x_COMPUTE_TASKS", ++ "T72x_COMPUTE_THREADS", ++ "T72x_TRIPIPE_ACTIVE", ++ "T72x_ARITH_WORDS", ++ "T72x_ARITH_CYCLES_REG", ++ "T72x_LS_WORDS", ++ "T72x_LS_ISSUES", ++ "T72x_LS_RESTARTS", ++ "T72x_LS_REISSUES_MISS", ++ "T72x_TEX_WORDS", ++ "T72x_TEX_BUBBLES", ++ "T72x_TEX_ISSUES", ++ "T72x_LSC_READ_HITS", ++ "T72x_LSC_READ_MISSES", ++ "T72x_LSC_WRITE_HITS", ++ "T72x_LSC_WRITE_MISSES", ++ "T72x_LSC_ATOMIC_HITS", ++ "T72x_LSC_ATOMIC_MISSES", ++ "T72x_LSC_LINE_FETCHES", ++ "T72x_LSC_DIRTY_LINE", ++ "T72x_LSC_SNOOPS", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ ++ /*L2 and MMU */ ++ "", ++ "", ++ "", ++ "", ++ "T72x_L2_EXT_WRITE_BEAT", ++ "T72x_L2_EXT_READ_BEAT", ++ "T72x_L2_READ_SNOOP", ++ "T72x_L2_READ_HIT", ++ "T72x_L2_WRITE_SNOOP", ++ "T72x_L2_WRITE_HIT", ++ "T72x_L2_EXT_WRITE_SMALL", ++ "T72x_L2_EXT_BARRIER", ++ "T72x_L2_EXT_AR_STALL", ++ "T72x_L2_EXT_W_STALL", ++ "T72x_L2_SNOOP_FULL", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "" ++}; ++ ++static const char * const hardware_counters_mali_t76x[] = { ++ /* Job Manager */ ++ "", ++ "", ++ "", ++ "", ++ "T76x_MESSAGES_SENT", ++ "T76x_MESSAGES_RECEIVED", ++ "T76x_GPU_ACTIVE", ++ "T76x_IRQ_ACTIVE", ++ "T76x_JS0_JOBS", ++ "T76x_JS0_TASKS", ++ "T76x_JS0_ACTIVE", ++ "", ++ "T76x_JS0_WAIT_READ", ++ "T76x_JS0_WAIT_ISSUE", ++ "T76x_JS0_WAIT_DEPEND", ++ "T76x_JS0_WAIT_FINISH", ++ "T76x_JS1_JOBS", ++ "T76x_JS1_TASKS", ++ "T76x_JS1_ACTIVE", ++ "", ++ "T76x_JS1_WAIT_READ", ++ "T76x_JS1_WAIT_ISSUE", ++ "T76x_JS1_WAIT_DEPEND", ++ "T76x_JS1_WAIT_FINISH", ++ "T76x_JS2_JOBS", ++ "T76x_JS2_TASKS", ++ "T76x_JS2_ACTIVE", ++ "", ++ "T76x_JS2_WAIT_READ", ++ "T76x_JS2_WAIT_ISSUE", ++ "T76x_JS2_WAIT_DEPEND", ++ "T76x_JS2_WAIT_FINISH", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ ++ /*Tiler */ ++ "", ++ "", ++ "", ++ "T76x_TI_JOBS_PROCESSED", ++ "T76x_TI_TRIANGLES", ++ "T76x_TI_QUADS", ++ "T76x_TI_POLYGONS", ++ "T76x_TI_POINTS", ++ "T76x_TI_LINES", ++ "T76x_TI_VCACHE_HIT", ++ "T76x_TI_VCACHE_MISS", ++ "T76x_TI_FRONT_FACING", ++ "T76x_TI_BACK_FACING", ++ "T76x_TI_PRIM_VISIBLE", ++ "T76x_TI_PRIM_CULLED", ++ "T76x_TI_PRIM_CLIPPED", ++ "T76x_TI_LEVEL0", ++ "T76x_TI_LEVEL1", ++ "T76x_TI_LEVEL2", ++ "T76x_TI_LEVEL3", ++ "T76x_TI_LEVEL4", ++ "T76x_TI_LEVEL5", ++ "T76x_TI_LEVEL6", ++ "T76x_TI_LEVEL7", ++ "T76x_TI_COMMAND_1", ++ "T76x_TI_COMMAND_2", ++ "T76x_TI_COMMAND_3", ++ "T76x_TI_COMMAND_4", ++ "T76x_TI_COMMAND_5_7", ++ "T76x_TI_COMMAND_8_15", ++ "T76x_TI_COMMAND_16_63", ++ "T76x_TI_COMMAND_64", ++ "T76x_TI_COMPRESS_IN", ++ "T76x_TI_COMPRESS_OUT", ++ "T76x_TI_COMPRESS_FLUSH", ++ "T76x_TI_TIMESTAMPS", ++ "T76x_TI_PCACHE_HIT", ++ "T76x_TI_PCACHE_MISS", ++ "T76x_TI_PCACHE_LINE", ++ "T76x_TI_PCACHE_STALL", ++ "T76x_TI_WRBUF_HIT", ++ "T76x_TI_WRBUF_MISS", ++ "T76x_TI_WRBUF_LINE", ++ "T76x_TI_WRBUF_PARTIAL", ++ "T76x_TI_WRBUF_STALL", ++ "T76x_TI_ACTIVE", ++ "T76x_TI_LOADING_DESC", ++ "T76x_TI_INDEX_WAIT", ++ "T76x_TI_INDEX_RANGE_WAIT", ++ "T76x_TI_VERTEX_WAIT", ++ "T76x_TI_PCACHE_WAIT", ++ "T76x_TI_WRBUF_WAIT", ++ "T76x_TI_BUS_READ", ++ "T76x_TI_BUS_WRITE", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "T76x_TI_UTLB_HIT", ++ "T76x_TI_UTLB_NEW_MISS", ++ "T76x_TI_UTLB_REPLAY_FULL", ++ "T76x_TI_UTLB_REPLAY_MISS", ++ "T76x_TI_UTLB_STALL", ++ ++ /* Shader Core */ ++ "", ++ "", ++ "", ++ "", ++ "T76x_FRAG_ACTIVE", ++ "T76x_FRAG_PRIMITIVES", ++ "T76x_FRAG_PRIMITIVES_DROPPED", ++ "T76x_FRAG_CYCLES_DESC", ++ "T76x_FRAG_CYCLES_FPKQ_ACTIVE", ++ "T76x_FRAG_CYCLES_VERT", ++ "T76x_FRAG_CYCLES_TRISETUP", ++ "T76x_FRAG_CYCLES_EZS_ACTIVE", ++ "T76x_FRAG_THREADS", ++ "T76x_FRAG_DUMMY_THREADS", ++ "T76x_FRAG_QUADS_RAST", ++ "T76x_FRAG_QUADS_EZS_TEST", ++ "T76x_FRAG_QUADS_EZS_KILLED", ++ "T76x_FRAG_THREADS_LZS_TEST", ++ "T76x_FRAG_THREADS_LZS_KILLED", ++ "T76x_FRAG_CYCLES_NO_TILE", ++ "T76x_FRAG_NUM_TILES", ++ "T76x_FRAG_TRANS_ELIM", ++ "T76x_COMPUTE_ACTIVE", ++ "T76x_COMPUTE_TASKS", ++ "T76x_COMPUTE_THREADS", ++ "T76x_COMPUTE_CYCLES_DESC", ++ "T76x_TRIPIPE_ACTIVE", ++ "T76x_ARITH_WORDS", ++ "T76x_ARITH_CYCLES_REG", ++ "T76x_ARITH_CYCLES_L0", ++ "T76x_ARITH_FRAG_DEPEND", ++ "T76x_LS_WORDS", ++ "T76x_LS_ISSUES", ++ "T76x_LS_REISSUE_ATTR", ++ "T76x_LS_REISSUES_VARY", ++ "T76x_LS_VARY_RV_MISS", ++ "T76x_LS_VARY_RV_HIT", ++ "T76x_LS_NO_UNPARK", ++ "T76x_TEX_WORDS", ++ "T76x_TEX_BUBBLES", ++ "T76x_TEX_WORDS_L0", ++ "T76x_TEX_WORDS_DESC", ++ "T76x_TEX_ISSUES", ++ "T76x_TEX_RECIRC_FMISS", ++ "T76x_TEX_RECIRC_DESC", ++ "T76x_TEX_RECIRC_MULTI", ++ "T76x_TEX_RECIRC_PMISS", ++ "T76x_TEX_RECIRC_CONF", ++ "T76x_LSC_READ_HITS", ++ "T76x_LSC_READ_OP", ++ "T76x_LSC_WRITE_HITS", ++ "T76x_LSC_WRITE_OP", ++ "T76x_LSC_ATOMIC_HITS", ++ "T76x_LSC_ATOMIC_OP", ++ "T76x_LSC_LINE_FETCHES", ++ "T76x_LSC_DIRTY_LINE", ++ "T76x_LSC_SNOOPS", ++ "T76x_AXI_TLB_STALL", ++ "T76x_AXI_TLB_MISS", ++ "T76x_AXI_TLB_TRANSACTION", ++ "T76x_LS_TLB_MISS", ++ "T76x_LS_TLB_HIT", ++ "T76x_AXI_BEATS_READ", ++ "T76x_AXI_BEATS_WRITTEN", ++ ++ /*L2 and MMU */ ++ "", ++ "", ++ "", ++ "", ++ "T76x_MMU_HIT", ++ "T76x_MMU_NEW_MISS", ++ "T76x_MMU_REPLAY_FULL", ++ "T76x_MMU_REPLAY_MISS", ++ "T76x_MMU_TABLE_WALK", ++ "T76x_MMU_REQUESTS", ++ "", ++ "", ++ "T76x_UTLB_HIT", ++ "T76x_UTLB_NEW_MISS", ++ "T76x_UTLB_REPLAY_FULL", ++ "T76x_UTLB_REPLAY_MISS", ++ "T76x_UTLB_STALL", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "T76x_L2_EXT_WRITE_BEATS", ++ "T76x_L2_EXT_READ_BEATS", ++ "T76x_L2_ANY_LOOKUP", ++ "T76x_L2_READ_LOOKUP", ++ "T76x_L2_SREAD_LOOKUP", ++ "T76x_L2_READ_REPLAY", ++ "T76x_L2_READ_SNOOP", ++ "T76x_L2_READ_HIT", ++ "T76x_L2_CLEAN_MISS", ++ "T76x_L2_WRITE_LOOKUP", ++ "T76x_L2_SWRITE_LOOKUP", ++ "T76x_L2_WRITE_REPLAY", ++ "T76x_L2_WRITE_SNOOP", ++ "T76x_L2_WRITE_HIT", ++ "T76x_L2_EXT_READ_FULL", ++ "", ++ "T76x_L2_EXT_WRITE_FULL", ++ "T76x_L2_EXT_R_W_HAZARD", ++ "T76x_L2_EXT_READ", ++ "T76x_L2_EXT_READ_LINE", ++ "T76x_L2_EXT_WRITE", ++ "T76x_L2_EXT_WRITE_LINE", ++ "T76x_L2_EXT_WRITE_SMALL", ++ "T76x_L2_EXT_BARRIER", ++ "T76x_L2_EXT_AR_STALL", ++ "T76x_L2_EXT_R_BUF_FULL", ++ "T76x_L2_EXT_RD_BUF_FULL", ++ "T76x_L2_EXT_R_RAW", ++ "T76x_L2_EXT_W_STALL", ++ "T76x_L2_EXT_W_BUF_FULL", ++ "T76x_L2_EXT_R_BUF_FULL", ++ "T76x_L2_TAG_HAZARD", ++ "T76x_L2_SNOOP_FULL", ++ "T76x_L2_REPLAY_FULL" ++}; ++ ++static const char * const hardware_counters_mali_t82x[] = { ++ /* Job Manager */ ++ "", ++ "", ++ "", ++ "", ++ "T82x_MESSAGES_SENT", ++ "T82x_MESSAGES_RECEIVED", ++ "T82x_GPU_ACTIVE", ++ "T82x_IRQ_ACTIVE", ++ "T82x_JS0_JOBS", ++ "T82x_JS0_TASKS", ++ "T82x_JS0_ACTIVE", ++ "", ++ "T82x_JS0_WAIT_READ", ++ "T82x_JS0_WAIT_ISSUE", ++ "T82x_JS0_WAIT_DEPEND", ++ "T82x_JS0_WAIT_FINISH", ++ "T82x_JS1_JOBS", ++ "T82x_JS1_TASKS", ++ "T82x_JS1_ACTIVE", ++ "", ++ "T82x_JS1_WAIT_READ", ++ "T82x_JS1_WAIT_ISSUE", ++ "T82x_JS1_WAIT_DEPEND", ++ "T82x_JS1_WAIT_FINISH", ++ "T82x_JS2_JOBS", ++ "T82x_JS2_TASKS", ++ "T82x_JS2_ACTIVE", ++ "", ++ "T82x_JS2_WAIT_READ", ++ "T82x_JS2_WAIT_ISSUE", ++ "T82x_JS2_WAIT_DEPEND", ++ "T82x_JS2_WAIT_FINISH", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ ++ /*Tiler */ ++ "", ++ "", ++ "", ++ "T82x_TI_JOBS_PROCESSED", ++ "T82x_TI_TRIANGLES", ++ "T82x_TI_QUADS", ++ "T82x_TI_POLYGONS", ++ "T82x_TI_POINTS", ++ "T82x_TI_LINES", ++ "T82x_TI_FRONT_FACING", ++ "T82x_TI_BACK_FACING", ++ "T82x_TI_PRIM_VISIBLE", ++ "T82x_TI_PRIM_CULLED", ++ "T82x_TI_PRIM_CLIPPED", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "T82x_TI_ACTIVE", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ ++ /* Shader Core */ ++ "", ++ "", ++ "", ++ "", ++ "T82x_FRAG_ACTIVE", ++ "T82x_FRAG_PRIMITIVES", ++ "T82x_FRAG_PRIMITIVES_DROPPED", ++ "T82x_FRAG_CYCLES_DESC", ++ "T82x_FRAG_CYCLES_FPKQ_ACTIVE", ++ "T82x_FRAG_CYCLES_VERT", ++ "T82x_FRAG_CYCLES_TRISETUP", ++ "T82x_FRAG_CYCLES_EZS_ACTIVE", ++ "T82x_FRAG_THREADS", ++ "T82x_FRAG_DUMMY_THREADS", ++ "T82x_FRAG_QUADS_RAST", ++ "T82x_FRAG_QUADS_EZS_TEST", ++ "T82x_FRAG_QUADS_EZS_KILLED", ++ "T82x_FRAG_THREADS_LZS_TEST", ++ "T82x_FRAG_THREADS_LZS_KILLED", ++ "T82x_FRAG_CYCLES_NO_TILE", ++ "T82x_FRAG_NUM_TILES", ++ "T82x_FRAG_TRANS_ELIM", ++ "T82x_COMPUTE_ACTIVE", ++ "T82x_COMPUTE_TASKS", ++ "T82x_COMPUTE_THREADS", ++ "T82x_COMPUTE_CYCLES_DESC", ++ "T82x_TRIPIPE_ACTIVE", ++ "T82x_ARITH_WORDS", ++ "T82x_ARITH_CYCLES_REG", ++ "T82x_ARITH_CYCLES_L0", ++ "T82x_ARITH_FRAG_DEPEND", ++ "T82x_LS_WORDS", ++ "T82x_LS_ISSUES", ++ "T82x_LS_REISSUE_ATTR", ++ "T82x_LS_REISSUES_VARY", ++ "T82x_LS_VARY_RV_MISS", ++ "T82x_LS_VARY_RV_HIT", ++ "T82x_LS_NO_UNPARK", ++ "T82x_TEX_WORDS", ++ "T82x_TEX_BUBBLES", ++ "T82x_TEX_WORDS_L0", ++ "T82x_TEX_WORDS_DESC", ++ "T82x_TEX_ISSUES", ++ "T82x_TEX_RECIRC_FMISS", ++ "T82x_TEX_RECIRC_DESC", ++ "T82x_TEX_RECIRC_MULTI", ++ "T82x_TEX_RECIRC_PMISS", ++ "T82x_TEX_RECIRC_CONF", ++ "T82x_LSC_READ_HITS", ++ "T82x_LSC_READ_OP", ++ "T82x_LSC_WRITE_HITS", ++ "T82x_LSC_WRITE_OP", ++ "T82x_LSC_ATOMIC_HITS", ++ "T82x_LSC_ATOMIC_OP", ++ "T82x_LSC_LINE_FETCHES", ++ "T82x_LSC_DIRTY_LINE", ++ "T82x_LSC_SNOOPS", ++ "T82x_AXI_TLB_STALL", ++ "T82x_AXI_TLB_MISS", ++ "T82x_AXI_TLB_TRANSACTION", ++ "T82x_LS_TLB_MISS", ++ "T82x_LS_TLB_HIT", ++ "T82x_AXI_BEATS_READ", ++ "T82x_AXI_BEATS_WRITTEN", ++ ++ /*L2 and MMU */ ++ "", ++ "", ++ "", ++ "", ++ "T82x_MMU_HIT", ++ "T82x_MMU_NEW_MISS", ++ "T82x_MMU_REPLAY_FULL", ++ "T82x_MMU_REPLAY_MISS", ++ "T82x_MMU_TABLE_WALK", ++ "T82x_MMU_REQUESTS", ++ "", ++ "", ++ "T82x_UTLB_HIT", ++ "T82x_UTLB_NEW_MISS", ++ "T82x_UTLB_REPLAY_FULL", ++ "T82x_UTLB_REPLAY_MISS", ++ "T82x_UTLB_STALL", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "T82x_L2_EXT_WRITE_BEATS", ++ "T82x_L2_EXT_READ_BEATS", ++ "T82x_L2_ANY_LOOKUP", ++ "T82x_L2_READ_LOOKUP", ++ "T82x_L2_SREAD_LOOKUP", ++ "T82x_L2_READ_REPLAY", ++ "T82x_L2_READ_SNOOP", ++ "T82x_L2_READ_HIT", ++ "T82x_L2_CLEAN_MISS", ++ "T82x_L2_WRITE_LOOKUP", ++ "T82x_L2_SWRITE_LOOKUP", ++ "T82x_L2_WRITE_REPLAY", ++ "T82x_L2_WRITE_SNOOP", ++ "T82x_L2_WRITE_HIT", ++ "T82x_L2_EXT_READ_FULL", ++ "", ++ "T82x_L2_EXT_WRITE_FULL", ++ "T82x_L2_EXT_R_W_HAZARD", ++ "T82x_L2_EXT_READ", ++ "T82x_L2_EXT_READ_LINE", ++ "T82x_L2_EXT_WRITE", ++ "T82x_L2_EXT_WRITE_LINE", ++ "T82x_L2_EXT_WRITE_SMALL", ++ "T82x_L2_EXT_BARRIER", ++ "T82x_L2_EXT_AR_STALL", ++ "T82x_L2_EXT_R_BUF_FULL", ++ "T82x_L2_EXT_RD_BUF_FULL", ++ "T82x_L2_EXT_R_RAW", ++ "T82x_L2_EXT_W_STALL", ++ "T82x_L2_EXT_W_BUF_FULL", ++ "T82x_L2_EXT_R_BUF_FULL", ++ "T82x_L2_TAG_HAZARD", ++ "T82x_L2_SNOOP_FULL", ++ "T82x_L2_REPLAY_FULL" ++}; ++ ++static const char * const hardware_counters_mali_t83x[] = { ++ /* Job Manager */ ++ "", ++ "", ++ "", ++ "", ++ "T83x_MESSAGES_SENT", ++ "T83x_MESSAGES_RECEIVED", ++ "T83x_GPU_ACTIVE", ++ "T83x_IRQ_ACTIVE", ++ "T83x_JS0_JOBS", ++ "T83x_JS0_TASKS", ++ "T83x_JS0_ACTIVE", ++ "", ++ "T83x_JS0_WAIT_READ", ++ "T83x_JS0_WAIT_ISSUE", ++ "T83x_JS0_WAIT_DEPEND", ++ "T83x_JS0_WAIT_FINISH", ++ "T83x_JS1_JOBS", ++ "T83x_JS1_TASKS", ++ "T83x_JS1_ACTIVE", ++ "", ++ "T83x_JS1_WAIT_READ", ++ "T83x_JS1_WAIT_ISSUE", ++ "T83x_JS1_WAIT_DEPEND", ++ "T83x_JS1_WAIT_FINISH", ++ "T83x_JS2_JOBS", ++ "T83x_JS2_TASKS", ++ "T83x_JS2_ACTIVE", ++ "", ++ "T83x_JS2_WAIT_READ", ++ "T83x_JS2_WAIT_ISSUE", ++ "T83x_JS2_WAIT_DEPEND", ++ "T83x_JS2_WAIT_FINISH", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ ++ /*Tiler */ ++ "", ++ "", ++ "", ++ "T83x_TI_JOBS_PROCESSED", ++ "T83x_TI_TRIANGLES", ++ "T83x_TI_QUADS", ++ "T83x_TI_POLYGONS", ++ "T83x_TI_POINTS", ++ "T83x_TI_LINES", ++ "T83x_TI_FRONT_FACING", ++ "T83x_TI_BACK_FACING", ++ "T83x_TI_PRIM_VISIBLE", ++ "T83x_TI_PRIM_CULLED", ++ "T83x_TI_PRIM_CLIPPED", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "T83x_TI_ACTIVE", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ ++ /* Shader Core */ ++ "", ++ "", ++ "", ++ "", ++ "T83x_FRAG_ACTIVE", ++ "T83x_FRAG_PRIMITIVES", ++ "T83x_FRAG_PRIMITIVES_DROPPED", ++ "T83x_FRAG_CYCLES_DESC", ++ "T83x_FRAG_CYCLES_FPKQ_ACTIVE", ++ "T83x_FRAG_CYCLES_VERT", ++ "T83x_FRAG_CYCLES_TRISETUP", ++ "T83x_FRAG_CYCLES_EZS_ACTIVE", ++ "T83x_FRAG_THREADS", ++ "T83x_FRAG_DUMMY_THREADS", ++ "T83x_FRAG_QUADS_RAST", ++ "T83x_FRAG_QUADS_EZS_TEST", ++ "T83x_FRAG_QUADS_EZS_KILLED", ++ "T83x_FRAG_THREADS_LZS_TEST", ++ "T83x_FRAG_THREADS_LZS_KILLED", ++ "T83x_FRAG_CYCLES_NO_TILE", ++ "T83x_FRAG_NUM_TILES", ++ "T83x_FRAG_TRANS_ELIM", ++ "T83x_COMPUTE_ACTIVE", ++ "T83x_COMPUTE_TASKS", ++ "T83x_COMPUTE_THREADS", ++ "T83x_COMPUTE_CYCLES_DESC", ++ "T83x_TRIPIPE_ACTIVE", ++ "T83x_ARITH_WORDS", ++ "T83x_ARITH_CYCLES_REG", ++ "T83x_ARITH_CYCLES_L0", ++ "T83x_ARITH_FRAG_DEPEND", ++ "T83x_LS_WORDS", ++ "T83x_LS_ISSUES", ++ "T83x_LS_REISSUE_ATTR", ++ "T83x_LS_REISSUES_VARY", ++ "T83x_LS_VARY_RV_MISS", ++ "T83x_LS_VARY_RV_HIT", ++ "T83x_LS_NO_UNPARK", ++ "T83x_TEX_WORDS", ++ "T83x_TEX_BUBBLES", ++ "T83x_TEX_WORDS_L0", ++ "T83x_TEX_WORDS_DESC", ++ "T83x_TEX_ISSUES", ++ "T83x_TEX_RECIRC_FMISS", ++ "T83x_TEX_RECIRC_DESC", ++ "T83x_TEX_RECIRC_MULTI", ++ "T83x_TEX_RECIRC_PMISS", ++ "T83x_TEX_RECIRC_CONF", ++ "T83x_LSC_READ_HITS", ++ "T83x_LSC_READ_OP", ++ "T83x_LSC_WRITE_HITS", ++ "T83x_LSC_WRITE_OP", ++ "T83x_LSC_ATOMIC_HITS", ++ "T83x_LSC_ATOMIC_OP", ++ "T83x_LSC_LINE_FETCHES", ++ "T83x_LSC_DIRTY_LINE", ++ "T83x_LSC_SNOOPS", ++ "T83x_AXI_TLB_STALL", ++ "T83x_AXI_TLB_MISS", ++ "T83x_AXI_TLB_TRANSACTION", ++ "T83x_LS_TLB_MISS", ++ "T83x_LS_TLB_HIT", ++ "T83x_AXI_BEATS_READ", ++ "T83x_AXI_BEATS_WRITTEN", ++ ++ /*L2 and MMU */ ++ "", ++ "", ++ "", ++ "", ++ "T83x_MMU_HIT", ++ "T83x_MMU_NEW_MISS", ++ "T83x_MMU_REPLAY_FULL", ++ "T83x_MMU_REPLAY_MISS", ++ "T83x_MMU_TABLE_WALK", ++ "T83x_MMU_REQUESTS", ++ "", ++ "", ++ "T83x_UTLB_HIT", ++ "T83x_UTLB_NEW_MISS", ++ "T83x_UTLB_REPLAY_FULL", ++ "T83x_UTLB_REPLAY_MISS", ++ "T83x_UTLB_STALL", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "T83x_L2_EXT_WRITE_BEATS", ++ "T83x_L2_EXT_READ_BEATS", ++ "T83x_L2_ANY_LOOKUP", ++ "T83x_L2_READ_LOOKUP", ++ "T83x_L2_SREAD_LOOKUP", ++ "T83x_L2_READ_REPLAY", ++ "T83x_L2_READ_SNOOP", ++ "T83x_L2_READ_HIT", ++ "T83x_L2_CLEAN_MISS", ++ "T83x_L2_WRITE_LOOKUP", ++ "T83x_L2_SWRITE_LOOKUP", ++ "T83x_L2_WRITE_REPLAY", ++ "T83x_L2_WRITE_SNOOP", ++ "T83x_L2_WRITE_HIT", ++ "T83x_L2_EXT_READ_FULL", ++ "", ++ "T83x_L2_EXT_WRITE_FULL", ++ "T83x_L2_EXT_R_W_HAZARD", ++ "T83x_L2_EXT_READ", ++ "T83x_L2_EXT_READ_LINE", ++ "T83x_L2_EXT_WRITE", ++ "T83x_L2_EXT_WRITE_LINE", ++ "T83x_L2_EXT_WRITE_SMALL", ++ "T83x_L2_EXT_BARRIER", ++ "T83x_L2_EXT_AR_STALL", ++ "T83x_L2_EXT_R_BUF_FULL", ++ "T83x_L2_EXT_RD_BUF_FULL", ++ "T83x_L2_EXT_R_RAW", ++ "T83x_L2_EXT_W_STALL", ++ "T83x_L2_EXT_W_BUF_FULL", ++ "T83x_L2_EXT_R_BUF_FULL", ++ "T83x_L2_TAG_HAZARD", ++ "T83x_L2_SNOOP_FULL", ++ "T83x_L2_REPLAY_FULL" ++}; ++ ++static const char * const hardware_counters_mali_t86x[] = { ++ /* Job Manager */ ++ "", ++ "", ++ "", ++ "", ++ "T86x_MESSAGES_SENT", ++ "T86x_MESSAGES_RECEIVED", ++ "T86x_GPU_ACTIVE", ++ "T86x_IRQ_ACTIVE", ++ "T86x_JS0_JOBS", ++ "T86x_JS0_TASKS", ++ "T86x_JS0_ACTIVE", ++ "", ++ "T86x_JS0_WAIT_READ", ++ "T86x_JS0_WAIT_ISSUE", ++ "T86x_JS0_WAIT_DEPEND", ++ "T86x_JS0_WAIT_FINISH", ++ "T86x_JS1_JOBS", ++ "T86x_JS1_TASKS", ++ "T86x_JS1_ACTIVE", ++ "", ++ "T86x_JS1_WAIT_READ", ++ "T86x_JS1_WAIT_ISSUE", ++ "T86x_JS1_WAIT_DEPEND", ++ "T86x_JS1_WAIT_FINISH", ++ "T86x_JS2_JOBS", ++ "T86x_JS2_TASKS", ++ "T86x_JS2_ACTIVE", ++ "", ++ "T86x_JS2_WAIT_READ", ++ "T86x_JS2_WAIT_ISSUE", ++ "T86x_JS2_WAIT_DEPEND", ++ "T86x_JS2_WAIT_FINISH", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ ++ /*Tiler */ ++ "", ++ "", ++ "", ++ "T86x_TI_JOBS_PROCESSED", ++ "T86x_TI_TRIANGLES", ++ "T86x_TI_QUADS", ++ "T86x_TI_POLYGONS", ++ "T86x_TI_POINTS", ++ "T86x_TI_LINES", ++ "T86x_TI_VCACHE_HIT", ++ "T86x_TI_VCACHE_MISS", ++ "T86x_TI_FRONT_FACING", ++ "T86x_TI_BACK_FACING", ++ "T86x_TI_PRIM_VISIBLE", ++ "T86x_TI_PRIM_CULLED", ++ "T86x_TI_PRIM_CLIPPED", ++ "T86x_TI_LEVEL0", ++ "T86x_TI_LEVEL1", ++ "T86x_TI_LEVEL2", ++ "T86x_TI_LEVEL3", ++ "T86x_TI_LEVEL4", ++ "T86x_TI_LEVEL5", ++ "T86x_TI_LEVEL6", ++ "T86x_TI_LEVEL7", ++ "T86x_TI_COMMAND_1", ++ "T86x_TI_COMMAND_2", ++ "T86x_TI_COMMAND_3", ++ "T86x_TI_COMMAND_4", ++ "T86x_TI_COMMAND_5_7", ++ "T86x_TI_COMMAND_8_15", ++ "T86x_TI_COMMAND_16_63", ++ "T86x_TI_COMMAND_64", ++ "T86x_TI_COMPRESS_IN", ++ "T86x_TI_COMPRESS_OUT", ++ "T86x_TI_COMPRESS_FLUSH", ++ "T86x_TI_TIMESTAMPS", ++ "T86x_TI_PCACHE_HIT", ++ "T86x_TI_PCACHE_MISS", ++ "T86x_TI_PCACHE_LINE", ++ "T86x_TI_PCACHE_STALL", ++ "T86x_TI_WRBUF_HIT", ++ "T86x_TI_WRBUF_MISS", ++ "T86x_TI_WRBUF_LINE", ++ "T86x_TI_WRBUF_PARTIAL", ++ "T86x_TI_WRBUF_STALL", ++ "T86x_TI_ACTIVE", ++ "T86x_TI_LOADING_DESC", ++ "T86x_TI_INDEX_WAIT", ++ "T86x_TI_INDEX_RANGE_WAIT", ++ "T86x_TI_VERTEX_WAIT", ++ "T86x_TI_PCACHE_WAIT", ++ "T86x_TI_WRBUF_WAIT", ++ "T86x_TI_BUS_READ", ++ "T86x_TI_BUS_WRITE", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "T86x_TI_UTLB_HIT", ++ "T86x_TI_UTLB_NEW_MISS", ++ "T86x_TI_UTLB_REPLAY_FULL", ++ "T86x_TI_UTLB_REPLAY_MISS", ++ "T86x_TI_UTLB_STALL", ++ ++ /* Shader Core */ ++ "", ++ "", ++ "", ++ "", ++ "T86x_FRAG_ACTIVE", ++ "T86x_FRAG_PRIMITIVES", ++ "T86x_FRAG_PRIMITIVES_DROPPED", ++ "T86x_FRAG_CYCLES_DESC", ++ "T86x_FRAG_CYCLES_FPKQ_ACTIVE", ++ "T86x_FRAG_CYCLES_VERT", ++ "T86x_FRAG_CYCLES_TRISETUP", ++ "T86x_FRAG_CYCLES_EZS_ACTIVE", ++ "T86x_FRAG_THREADS", ++ "T86x_FRAG_DUMMY_THREADS", ++ "T86x_FRAG_QUADS_RAST", ++ "T86x_FRAG_QUADS_EZS_TEST", ++ "T86x_FRAG_QUADS_EZS_KILLED", ++ "T86x_FRAG_THREADS_LZS_TEST", ++ "T86x_FRAG_THREADS_LZS_KILLED", ++ "T86x_FRAG_CYCLES_NO_TILE", ++ "T86x_FRAG_NUM_TILES", ++ "T86x_FRAG_TRANS_ELIM", ++ "T86x_COMPUTE_ACTIVE", ++ "T86x_COMPUTE_TASKS", ++ "T86x_COMPUTE_THREADS", ++ "T86x_COMPUTE_CYCLES_DESC", ++ "T86x_TRIPIPE_ACTIVE", ++ "T86x_ARITH_WORDS", ++ "T86x_ARITH_CYCLES_REG", ++ "T86x_ARITH_CYCLES_L0", ++ "T86x_ARITH_FRAG_DEPEND", ++ "T86x_LS_WORDS", ++ "T86x_LS_ISSUES", ++ "T86x_LS_REISSUE_ATTR", ++ "T86x_LS_REISSUES_VARY", ++ "T86x_LS_VARY_RV_MISS", ++ "T86x_LS_VARY_RV_HIT", ++ "T86x_LS_NO_UNPARK", ++ "T86x_TEX_WORDS", ++ "T86x_TEX_BUBBLES", ++ "T86x_TEX_WORDS_L0", ++ "T86x_TEX_WORDS_DESC", ++ "T86x_TEX_ISSUES", ++ "T86x_TEX_RECIRC_FMISS", ++ "T86x_TEX_RECIRC_DESC", ++ "T86x_TEX_RECIRC_MULTI", ++ "T86x_TEX_RECIRC_PMISS", ++ "T86x_TEX_RECIRC_CONF", ++ "T86x_LSC_READ_HITS", ++ "T86x_LSC_READ_OP", ++ "T86x_LSC_WRITE_HITS", ++ "T86x_LSC_WRITE_OP", ++ "T86x_LSC_ATOMIC_HITS", ++ "T86x_LSC_ATOMIC_OP", ++ "T86x_LSC_LINE_FETCHES", ++ "T86x_LSC_DIRTY_LINE", ++ "T86x_LSC_SNOOPS", ++ "T86x_AXI_TLB_STALL", ++ "T86x_AXI_TLB_MISS", ++ "T86x_AXI_TLB_TRANSACTION", ++ "T86x_LS_TLB_MISS", ++ "T86x_LS_TLB_HIT", ++ "T86x_AXI_BEATS_READ", ++ "T86x_AXI_BEATS_WRITTEN", ++ ++ /*L2 and MMU */ ++ "", ++ "", ++ "", ++ "", ++ "T86x_MMU_HIT", ++ "T86x_MMU_NEW_MISS", ++ "T86x_MMU_REPLAY_FULL", ++ "T86x_MMU_REPLAY_MISS", ++ "T86x_MMU_TABLE_WALK", ++ "T86x_MMU_REQUESTS", ++ "", ++ "", ++ "T86x_UTLB_HIT", ++ "T86x_UTLB_NEW_MISS", ++ "T86x_UTLB_REPLAY_FULL", ++ "T86x_UTLB_REPLAY_MISS", ++ "T86x_UTLB_STALL", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "T86x_L2_EXT_WRITE_BEATS", ++ "T86x_L2_EXT_READ_BEATS", ++ "T86x_L2_ANY_LOOKUP", ++ "T86x_L2_READ_LOOKUP", ++ "T86x_L2_SREAD_LOOKUP", ++ "T86x_L2_READ_REPLAY", ++ "T86x_L2_READ_SNOOP", ++ "T86x_L2_READ_HIT", ++ "T86x_L2_CLEAN_MISS", ++ "T86x_L2_WRITE_LOOKUP", ++ "T86x_L2_SWRITE_LOOKUP", ++ "T86x_L2_WRITE_REPLAY", ++ "T86x_L2_WRITE_SNOOP", ++ "T86x_L2_WRITE_HIT", ++ "T86x_L2_EXT_READ_FULL", ++ "", ++ "T86x_L2_EXT_WRITE_FULL", ++ "T86x_L2_EXT_R_W_HAZARD", ++ "T86x_L2_EXT_READ", ++ "T86x_L2_EXT_READ_LINE", ++ "T86x_L2_EXT_WRITE", ++ "T86x_L2_EXT_WRITE_LINE", ++ "T86x_L2_EXT_WRITE_SMALL", ++ "T86x_L2_EXT_BARRIER", ++ "T86x_L2_EXT_AR_STALL", ++ "T86x_L2_EXT_R_BUF_FULL", ++ "T86x_L2_EXT_RD_BUF_FULL", ++ "T86x_L2_EXT_R_RAW", ++ "T86x_L2_EXT_W_STALL", ++ "T86x_L2_EXT_W_BUF_FULL", ++ "T86x_L2_EXT_R_BUF_FULL", ++ "T86x_L2_TAG_HAZARD", ++ "T86x_L2_SNOOP_FULL", ++ "T86x_L2_REPLAY_FULL" ++}; ++ ++static const char * const hardware_counters_mali_t88x[] = { ++ /* Job Manager */ ++ "", ++ "", ++ "", ++ "", ++ "T88x_MESSAGES_SENT", ++ "T88x_MESSAGES_RECEIVED", ++ "T88x_GPU_ACTIVE", ++ "T88x_IRQ_ACTIVE", ++ "T88x_JS0_JOBS", ++ "T88x_JS0_TASKS", ++ "T88x_JS0_ACTIVE", ++ "", ++ "T88x_JS0_WAIT_READ", ++ "T88x_JS0_WAIT_ISSUE", ++ "T88x_JS0_WAIT_DEPEND", ++ "T88x_JS0_WAIT_FINISH", ++ "T88x_JS1_JOBS", ++ "T88x_JS1_TASKS", ++ "T88x_JS1_ACTIVE", ++ "", ++ "T88x_JS1_WAIT_READ", ++ "T88x_JS1_WAIT_ISSUE", ++ "T88x_JS1_WAIT_DEPEND", ++ "T88x_JS1_WAIT_FINISH", ++ "T88x_JS2_JOBS", ++ "T88x_JS2_TASKS", ++ "T88x_JS2_ACTIVE", ++ "", ++ "T88x_JS2_WAIT_READ", ++ "T88x_JS2_WAIT_ISSUE", ++ "T88x_JS2_WAIT_DEPEND", ++ "T88x_JS2_WAIT_FINISH", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ ++ /*Tiler */ ++ "", ++ "", ++ "", ++ "T88x_TI_JOBS_PROCESSED", ++ "T88x_TI_TRIANGLES", ++ "T88x_TI_QUADS", ++ "T88x_TI_POLYGONS", ++ "T88x_TI_POINTS", ++ "T88x_TI_LINES", ++ "T88x_TI_VCACHE_HIT", ++ "T88x_TI_VCACHE_MISS", ++ "T88x_TI_FRONT_FACING", ++ "T88x_TI_BACK_FACING", ++ "T88x_TI_PRIM_VISIBLE", ++ "T88x_TI_PRIM_CULLED", ++ "T88x_TI_PRIM_CLIPPED", ++ "T88x_TI_LEVEL0", ++ "T88x_TI_LEVEL1", ++ "T88x_TI_LEVEL2", ++ "T88x_TI_LEVEL3", ++ "T88x_TI_LEVEL4", ++ "T88x_TI_LEVEL5", ++ "T88x_TI_LEVEL6", ++ "T88x_TI_LEVEL7", ++ "T88x_TI_COMMAND_1", ++ "T88x_TI_COMMAND_2", ++ "T88x_TI_COMMAND_3", ++ "T88x_TI_COMMAND_4", ++ "T88x_TI_COMMAND_5_7", ++ "T88x_TI_COMMAND_8_15", ++ "T88x_TI_COMMAND_16_63", ++ "T88x_TI_COMMAND_64", ++ "T88x_TI_COMPRESS_IN", ++ "T88x_TI_COMPRESS_OUT", ++ "T88x_TI_COMPRESS_FLUSH", ++ "T88x_TI_TIMESTAMPS", ++ "T88x_TI_PCACHE_HIT", ++ "T88x_TI_PCACHE_MISS", ++ "T88x_TI_PCACHE_LINE", ++ "T88x_TI_PCACHE_STALL", ++ "T88x_TI_WRBUF_HIT", ++ "T88x_TI_WRBUF_MISS", ++ "T88x_TI_WRBUF_LINE", ++ "T88x_TI_WRBUF_PARTIAL", ++ "T88x_TI_WRBUF_STALL", ++ "T88x_TI_ACTIVE", ++ "T88x_TI_LOADING_DESC", ++ "T88x_TI_INDEX_WAIT", ++ "T88x_TI_INDEX_RANGE_WAIT", ++ "T88x_TI_VERTEX_WAIT", ++ "T88x_TI_PCACHE_WAIT", ++ "T88x_TI_WRBUF_WAIT", ++ "T88x_TI_BUS_READ", ++ "T88x_TI_BUS_WRITE", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "T88x_TI_UTLB_HIT", ++ "T88x_TI_UTLB_NEW_MISS", ++ "T88x_TI_UTLB_REPLAY_FULL", ++ "T88x_TI_UTLB_REPLAY_MISS", ++ "T88x_TI_UTLB_STALL", ++ ++ /* Shader Core */ ++ "", ++ "", ++ "", ++ "", ++ "T88x_FRAG_ACTIVE", ++ "T88x_FRAG_PRIMITIVES", ++ "T88x_FRAG_PRIMITIVES_DROPPED", ++ "T88x_FRAG_CYCLES_DESC", ++ "T88x_FRAG_CYCLES_FPKQ_ACTIVE", ++ "T88x_FRAG_CYCLES_VERT", ++ "T88x_FRAG_CYCLES_TRISETUP", ++ "T88x_FRAG_CYCLES_EZS_ACTIVE", ++ "T88x_FRAG_THREADS", ++ "T88x_FRAG_DUMMY_THREADS", ++ "T88x_FRAG_QUADS_RAST", ++ "T88x_FRAG_QUADS_EZS_TEST", ++ "T88x_FRAG_QUADS_EZS_KILLED", ++ "T88x_FRAG_THREADS_LZS_TEST", ++ "T88x_FRAG_THREADS_LZS_KILLED", ++ "T88x_FRAG_CYCLES_NO_TILE", ++ "T88x_FRAG_NUM_TILES", ++ "T88x_FRAG_TRANS_ELIM", ++ "T88x_COMPUTE_ACTIVE", ++ "T88x_COMPUTE_TASKS", ++ "T88x_COMPUTE_THREADS", ++ "T88x_COMPUTE_CYCLES_DESC", ++ "T88x_TRIPIPE_ACTIVE", ++ "T88x_ARITH_WORDS", ++ "T88x_ARITH_CYCLES_REG", ++ "T88x_ARITH_CYCLES_L0", ++ "T88x_ARITH_FRAG_DEPEND", ++ "T88x_LS_WORDS", ++ "T88x_LS_ISSUES", ++ "T88x_LS_REISSUE_ATTR", ++ "T88x_LS_REISSUES_VARY", ++ "T88x_LS_VARY_RV_MISS", ++ "T88x_LS_VARY_RV_HIT", ++ "T88x_LS_NO_UNPARK", ++ "T88x_TEX_WORDS", ++ "T88x_TEX_BUBBLES", ++ "T88x_TEX_WORDS_L0", ++ "T88x_TEX_WORDS_DESC", ++ "T88x_TEX_ISSUES", ++ "T88x_TEX_RECIRC_FMISS", ++ "T88x_TEX_RECIRC_DESC", ++ "T88x_TEX_RECIRC_MULTI", ++ "T88x_TEX_RECIRC_PMISS", ++ "T88x_TEX_RECIRC_CONF", ++ "T88x_LSC_READ_HITS", ++ "T88x_LSC_READ_OP", ++ "T88x_LSC_WRITE_HITS", ++ "T88x_LSC_WRITE_OP", ++ "T88x_LSC_ATOMIC_HITS", ++ "T88x_LSC_ATOMIC_OP", ++ "T88x_LSC_LINE_FETCHES", ++ "T88x_LSC_DIRTY_LINE", ++ "T88x_LSC_SNOOPS", ++ "T88x_AXI_TLB_STALL", ++ "T88x_AXI_TLB_MISS", ++ "T88x_AXI_TLB_TRANSACTION", ++ "T88x_LS_TLB_MISS", ++ "T88x_LS_TLB_HIT", ++ "T88x_AXI_BEATS_READ", ++ "T88x_AXI_BEATS_WRITTEN", ++ ++ /*L2 and MMU */ ++ "", ++ "", ++ "", ++ "", ++ "T88x_MMU_HIT", ++ "T88x_MMU_NEW_MISS", ++ "T88x_MMU_REPLAY_FULL", ++ "T88x_MMU_REPLAY_MISS", ++ "T88x_MMU_TABLE_WALK", ++ "T88x_MMU_REQUESTS", ++ "", ++ "", ++ "T88x_UTLB_HIT", ++ "T88x_UTLB_NEW_MISS", ++ "T88x_UTLB_REPLAY_FULL", ++ "T88x_UTLB_REPLAY_MISS", ++ "T88x_UTLB_STALL", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "T88x_L2_EXT_WRITE_BEATS", ++ "T88x_L2_EXT_READ_BEATS", ++ "T88x_L2_ANY_LOOKUP", ++ "T88x_L2_READ_LOOKUP", ++ "T88x_L2_SREAD_LOOKUP", ++ "T88x_L2_READ_REPLAY", ++ "T88x_L2_READ_SNOOP", ++ "T88x_L2_READ_HIT", ++ "T88x_L2_CLEAN_MISS", ++ "T88x_L2_WRITE_LOOKUP", ++ "T88x_L2_SWRITE_LOOKUP", ++ "T88x_L2_WRITE_REPLAY", ++ "T88x_L2_WRITE_SNOOP", ++ "T88x_L2_WRITE_HIT", ++ "T88x_L2_EXT_READ_FULL", ++ "", ++ "T88x_L2_EXT_WRITE_FULL", ++ "T88x_L2_EXT_R_W_HAZARD", ++ "T88x_L2_EXT_READ", ++ "T88x_L2_EXT_READ_LINE", ++ "T88x_L2_EXT_WRITE", ++ "T88x_L2_EXT_WRITE_LINE", ++ "T88x_L2_EXT_WRITE_SMALL", ++ "T88x_L2_EXT_BARRIER", ++ "T88x_L2_EXT_AR_STALL", ++ "T88x_L2_EXT_R_BUF_FULL", ++ "T88x_L2_EXT_RD_BUF_FULL", ++ "T88x_L2_EXT_R_RAW", ++ "T88x_L2_EXT_W_STALL", ++ "T88x_L2_EXT_W_BUF_FULL", ++ "T88x_L2_EXT_R_BUF_FULL", ++ "T88x_L2_TAG_HAZARD", ++ "T88x_L2_SNOOP_FULL", ++ "T88x_L2_REPLAY_FULL" ++}; ++ ++#include "mali_kbase_gator_hwcnt_names_tmix.h" ++ ++#include "mali_kbase_gator_hwcnt_names_thex.h" ++ ++#include "mali_kbase_gator_hwcnt_names_tsix.h" ++ ++ ++ ++#endif +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator_hwcnt_names_thex.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator_hwcnt_names_thex.h +new file mode 100755 +index 000000000000..15fd4efdc6ca +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator_hwcnt_names_thex.h +@@ -0,0 +1,291 @@ ++/* ++ * ++ * (C) COPYRIGHT 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* ++ * This header was autogenerated, it should not be edited. ++ */ ++ ++#ifndef _KBASE_GATOR_HWCNT_NAMES_THEX_H_ ++#define _KBASE_GATOR_HWCNT_NAMES_THEX_H_ ++ ++static const char * const hardware_counters_mali_tHEx[] = { ++ /* Performance counters for the Job Manager */ ++ "", ++ "", ++ "", ++ "", ++ "THEx_MESSAGES_SENT", ++ "THEx_MESSAGES_RECEIVED", ++ "THEx_GPU_ACTIVE", ++ "THEx_IRQ_ACTIVE", ++ "THEx_JS0_JOBS", ++ "THEx_JS0_TASKS", ++ "THEx_JS0_ACTIVE", ++ "", ++ "THEx_JS0_WAIT_READ", ++ "THEx_JS0_WAIT_ISSUE", ++ "THEx_JS0_WAIT_DEPEND", ++ "THEx_JS0_WAIT_FINISH", ++ "THEx_JS1_JOBS", ++ "THEx_JS1_TASKS", ++ "THEx_JS1_ACTIVE", ++ "", ++ "THEx_JS1_WAIT_READ", ++ "THEx_JS1_WAIT_ISSUE", ++ "THEx_JS1_WAIT_DEPEND", ++ "THEx_JS1_WAIT_FINISH", ++ "THEx_JS2_JOBS", ++ "THEx_JS2_TASKS", ++ "THEx_JS2_ACTIVE", ++ "", ++ "THEx_JS2_WAIT_READ", ++ "THEx_JS2_WAIT_ISSUE", ++ "THEx_JS2_WAIT_DEPEND", ++ "THEx_JS2_WAIT_FINISH", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ ++ /* Performance counters for the Tiler */ ++ "", ++ "", ++ "", ++ "", ++ "THEx_TILER_ACTIVE", ++ "THEx_JOBS_PROCESSED", ++ "THEx_TRIANGLES", ++ "THEx_LINES", ++ "THEx_POINTS", ++ "THEx_FRONT_FACING", ++ "THEx_BACK_FACING", ++ "THEx_PRIM_VISIBLE", ++ "THEx_PRIM_CULLED", ++ "THEx_PRIM_CLIPPED", ++ "THEx_PRIM_SAT_CULLED", ++ "THEx_BIN_ALLOC_INIT", ++ "THEx_BIN_ALLOC_OVERFLOW", ++ "THEx_BUS_READ", ++ "", ++ "THEx_BUS_WRITE", ++ "THEx_LOADING_DESC", ++ "THEx_IDVS_POS_SHAD_REQ", ++ "THEx_IDVS_POS_SHAD_WAIT", ++ "THEx_IDVS_POS_SHAD_STALL", ++ "THEx_IDVS_POS_FIFO_FULL", ++ "THEx_PREFETCH_STALL", ++ "THEx_VCACHE_HIT", ++ "THEx_VCACHE_MISS", ++ "THEx_VCACHE_LINE_WAIT", ++ "THEx_VFETCH_POS_READ_WAIT", ++ "THEx_VFETCH_VERTEX_WAIT", ++ "THEx_VFETCH_STALL", ++ "THEx_PRIMASSY_STALL", ++ "THEx_BBOX_GEN_STALL", ++ "THEx_IDVS_VBU_HIT", ++ "THEx_IDVS_VBU_MISS", ++ "THEx_IDVS_VBU_LINE_DEALLOCATE", ++ "THEx_IDVS_VAR_SHAD_REQ", ++ "THEx_IDVS_VAR_SHAD_STALL", ++ "THEx_BINNER_STALL", ++ "THEx_ITER_STALL", ++ "THEx_COMPRESS_MISS", ++ "THEx_COMPRESS_STALL", ++ "THEx_PCACHE_HIT", ++ "THEx_PCACHE_MISS", ++ "THEx_PCACHE_MISS_STALL", ++ "THEx_PCACHE_EVICT_STALL", ++ "THEx_PMGR_PTR_WR_STALL", ++ "THEx_PMGR_PTR_RD_STALL", ++ "THEx_PMGR_CMD_WR_STALL", ++ "THEx_WRBUF_ACTIVE", ++ "THEx_WRBUF_HIT", ++ "THEx_WRBUF_MISS", ++ "THEx_WRBUF_NO_FREE_LINE_STALL", ++ "THEx_WRBUF_NO_AXI_ID_STALL", ++ "THEx_WRBUF_AXI_STALL", ++ "", ++ "", ++ "", ++ "THEx_UTLB_TRANS", ++ "THEx_UTLB_TRANS_HIT", ++ "THEx_UTLB_TRANS_STALL", ++ "THEx_UTLB_TRANS_MISS_DELAY", ++ "THEx_UTLB_MMU_REQ", ++ ++ /* Performance counters for the Shader Core */ ++ "", ++ "", ++ "", ++ "", ++ "THEx_FRAG_ACTIVE", ++ "THEx_FRAG_PRIMITIVES", ++ "THEx_FRAG_PRIM_RAST", ++ "THEx_FRAG_FPK_ACTIVE", ++ "THEx_FRAG_STARVING", ++ "THEx_FRAG_WARPS", ++ "THEx_FRAG_PARTIAL_WARPS", ++ "THEx_FRAG_QUADS_RAST", ++ "THEx_FRAG_QUADS_EZS_TEST", ++ "THEx_FRAG_QUADS_EZS_UPDATE", ++ "THEx_FRAG_QUADS_EZS_KILL", ++ "THEx_FRAG_LZS_TEST", ++ "THEx_FRAG_LZS_KILL", ++ "", ++ "THEx_FRAG_PTILES", ++ "THEx_FRAG_TRANS_ELIM", ++ "THEx_QUAD_FPK_KILLER", ++ "", ++ "THEx_COMPUTE_ACTIVE", ++ "THEx_COMPUTE_TASKS", ++ "THEx_COMPUTE_WARPS", ++ "THEx_COMPUTE_STARVING", ++ "THEx_EXEC_CORE_ACTIVE", ++ "THEx_EXEC_ACTIVE", ++ "THEx_EXEC_INSTR_COUNT", ++ "THEx_EXEC_INSTR_DIVERGED", ++ "THEx_EXEC_INSTR_STARVING", ++ "THEx_ARITH_INSTR_SINGLE_FMA", ++ "THEx_ARITH_INSTR_DOUBLE", ++ "THEx_ARITH_INSTR_MSG", ++ "THEx_ARITH_INSTR_MSG_ONLY", ++ "THEx_TEX_INSTR", ++ "THEx_TEX_INSTR_MIPMAP", ++ "THEx_TEX_INSTR_COMPRESSED", ++ "THEx_TEX_INSTR_3D", ++ "THEx_TEX_INSTR_TRILINEAR", ++ "THEx_TEX_COORD_ISSUE", ++ "THEx_TEX_COORD_STALL", ++ "THEx_TEX_STARVE_CACHE", ++ "THEx_TEX_STARVE_FILTER", ++ "THEx_LS_MEM_READ_FULL", ++ "THEx_LS_MEM_READ_SHORT", ++ "THEx_LS_MEM_WRITE_FULL", ++ "THEx_LS_MEM_WRITE_SHORT", ++ "THEx_LS_MEM_ATOMIC", ++ "THEx_VARY_INSTR", ++ "THEx_VARY_SLOT_32", ++ "THEx_VARY_SLOT_16", ++ "THEx_ATTR_INSTR", ++ "THEx_ARITH_INSTR_FP_MUL", ++ "THEx_BEATS_RD_FTC", ++ "THEx_BEATS_RD_FTC_EXT", ++ "THEx_BEATS_RD_LSC", ++ "THEx_BEATS_RD_LSC_EXT", ++ "THEx_BEATS_RD_TEX", ++ "THEx_BEATS_RD_TEX_EXT", ++ "THEx_BEATS_RD_OTHER", ++ "THEx_BEATS_WR_LSC", ++ "THEx_BEATS_WR_TIB", ++ "", ++ ++ /* Performance counters for the Memory System */ ++ "", ++ "", ++ "", ++ "", ++ "THEx_MMU_REQUESTS", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "THEx_L2_RD_MSG_IN", ++ "THEx_L2_RD_MSG_IN_STALL", ++ "THEx_L2_WR_MSG_IN", ++ "THEx_L2_WR_MSG_IN_STALL", ++ "THEx_L2_SNP_MSG_IN", ++ "THEx_L2_SNP_MSG_IN_STALL", ++ "THEx_L2_RD_MSG_OUT", ++ "THEx_L2_RD_MSG_OUT_STALL", ++ "THEx_L2_WR_MSG_OUT", ++ "THEx_L2_ANY_LOOKUP", ++ "THEx_L2_READ_LOOKUP", ++ "THEx_L2_WRITE_LOOKUP", ++ "THEx_L2_EXT_SNOOP_LOOKUP", ++ "THEx_L2_EXT_READ", ++ "THEx_L2_EXT_READ_NOSNP", ++ "THEx_L2_EXT_READ_UNIQUE", ++ "THEx_L2_EXT_READ_BEATS", ++ "THEx_L2_EXT_AR_STALL", ++ "THEx_L2_EXT_AR_CNT_Q1", ++ "THEx_L2_EXT_AR_CNT_Q2", ++ "THEx_L2_EXT_AR_CNT_Q3", ++ "THEx_L2_EXT_RRESP_0_127", ++ "THEx_L2_EXT_RRESP_128_191", ++ "THEx_L2_EXT_RRESP_192_255", ++ "THEx_L2_EXT_RRESP_256_319", ++ "THEx_L2_EXT_RRESP_320_383", ++ "THEx_L2_EXT_WRITE", ++ "THEx_L2_EXT_WRITE_NOSNP_FULL", ++ "THEx_L2_EXT_WRITE_NOSNP_PTL", ++ "THEx_L2_EXT_WRITE_SNP_FULL", ++ "THEx_L2_EXT_WRITE_SNP_PTL", ++ "THEx_L2_EXT_WRITE_BEATS", ++ "THEx_L2_EXT_W_STALL", ++ "THEx_L2_EXT_AW_CNT_Q1", ++ "THEx_L2_EXT_AW_CNT_Q2", ++ "THEx_L2_EXT_AW_CNT_Q3", ++ "THEx_L2_EXT_SNOOP", ++ "THEx_L2_EXT_SNOOP_STALL", ++ "THEx_L2_EXT_SNOOP_RESP_CLEAN", ++ "THEx_L2_EXT_SNOOP_RESP_DATA", ++ "THEx_L2_EXT_SNOOP_INTERNAL", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++}; ++ ++#endif /* _KBASE_GATOR_HWCNT_NAMES_THEX_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator_hwcnt_names_tmix.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator_hwcnt_names_tmix.h +new file mode 100755 +index 000000000000..8a215f723570 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator_hwcnt_names_tmix.h +@@ -0,0 +1,291 @@ ++/* ++ * ++ * (C) COPYRIGHT 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* ++ * This header was autogenerated, it should not be edited. ++ */ ++ ++#ifndef _KBASE_GATOR_HWCNT_NAMES_TMIX_H_ ++#define _KBASE_GATOR_HWCNT_NAMES_TMIX_H_ ++ ++static const char * const hardware_counters_mali_tMIx[] = { ++ /* Performance counters for the Job Manager */ ++ "", ++ "", ++ "", ++ "", ++ "TMIx_MESSAGES_SENT", ++ "TMIx_MESSAGES_RECEIVED", ++ "TMIx_GPU_ACTIVE", ++ "TMIx_IRQ_ACTIVE", ++ "TMIx_JS0_JOBS", ++ "TMIx_JS0_TASKS", ++ "TMIx_JS0_ACTIVE", ++ "", ++ "TMIx_JS0_WAIT_READ", ++ "TMIx_JS0_WAIT_ISSUE", ++ "TMIx_JS0_WAIT_DEPEND", ++ "TMIx_JS0_WAIT_FINISH", ++ "TMIx_JS1_JOBS", ++ "TMIx_JS1_TASKS", ++ "TMIx_JS1_ACTIVE", ++ "", ++ "TMIx_JS1_WAIT_READ", ++ "TMIx_JS1_WAIT_ISSUE", ++ "TMIx_JS1_WAIT_DEPEND", ++ "TMIx_JS1_WAIT_FINISH", ++ "TMIx_JS2_JOBS", ++ "TMIx_JS2_TASKS", ++ "TMIx_JS2_ACTIVE", ++ "", ++ "TMIx_JS2_WAIT_READ", ++ "TMIx_JS2_WAIT_ISSUE", ++ "TMIx_JS2_WAIT_DEPEND", ++ "TMIx_JS2_WAIT_FINISH", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ ++ /* Performance counters for the Tiler */ ++ "", ++ "", ++ "", ++ "", ++ "TMIx_TILER_ACTIVE", ++ "TMIx_JOBS_PROCESSED", ++ "TMIx_TRIANGLES", ++ "TMIx_LINES", ++ "TMIx_POINTS", ++ "TMIx_FRONT_FACING", ++ "TMIx_BACK_FACING", ++ "TMIx_PRIM_VISIBLE", ++ "TMIx_PRIM_CULLED", ++ "TMIx_PRIM_CLIPPED", ++ "TMIx_PRIM_SAT_CULLED", ++ "TMIx_BIN_ALLOC_INIT", ++ "TMIx_BIN_ALLOC_OVERFLOW", ++ "TMIx_BUS_READ", ++ "", ++ "TMIx_BUS_WRITE", ++ "TMIx_LOADING_DESC", ++ "TMIx_IDVS_POS_SHAD_REQ", ++ "TMIx_IDVS_POS_SHAD_WAIT", ++ "TMIx_IDVS_POS_SHAD_STALL", ++ "TMIx_IDVS_POS_FIFO_FULL", ++ "TMIx_PREFETCH_STALL", ++ "TMIx_VCACHE_HIT", ++ "TMIx_VCACHE_MISS", ++ "TMIx_VCACHE_LINE_WAIT", ++ "TMIx_VFETCH_POS_READ_WAIT", ++ "TMIx_VFETCH_VERTEX_WAIT", ++ "TMIx_VFETCH_STALL", ++ "TMIx_PRIMASSY_STALL", ++ "TMIx_BBOX_GEN_STALL", ++ "TMIx_IDVS_VBU_HIT", ++ "TMIx_IDVS_VBU_MISS", ++ "TMIx_IDVS_VBU_LINE_DEALLOCATE", ++ "TMIx_IDVS_VAR_SHAD_REQ", ++ "TMIx_IDVS_VAR_SHAD_STALL", ++ "TMIx_BINNER_STALL", ++ "TMIx_ITER_STALL", ++ "TMIx_COMPRESS_MISS", ++ "TMIx_COMPRESS_STALL", ++ "TMIx_PCACHE_HIT", ++ "TMIx_PCACHE_MISS", ++ "TMIx_PCACHE_MISS_STALL", ++ "TMIx_PCACHE_EVICT_STALL", ++ "TMIx_PMGR_PTR_WR_STALL", ++ "TMIx_PMGR_PTR_RD_STALL", ++ "TMIx_PMGR_CMD_WR_STALL", ++ "TMIx_WRBUF_ACTIVE", ++ "TMIx_WRBUF_HIT", ++ "TMIx_WRBUF_MISS", ++ "TMIx_WRBUF_NO_FREE_LINE_STALL", ++ "TMIx_WRBUF_NO_AXI_ID_STALL", ++ "TMIx_WRBUF_AXI_STALL", ++ "", ++ "", ++ "", ++ "TMIx_UTLB_TRANS", ++ "TMIx_UTLB_TRANS_HIT", ++ "TMIx_UTLB_TRANS_STALL", ++ "TMIx_UTLB_TRANS_MISS_DELAY", ++ "TMIx_UTLB_MMU_REQ", ++ ++ /* Performance counters for the Shader Core */ ++ "", ++ "", ++ "", ++ "", ++ "TMIx_FRAG_ACTIVE", ++ "TMIx_FRAG_PRIMITIVES", ++ "TMIx_FRAG_PRIM_RAST", ++ "TMIx_FRAG_FPK_ACTIVE", ++ "TMIx_FRAG_STARVING", ++ "TMIx_FRAG_WARPS", ++ "TMIx_FRAG_PARTIAL_WARPS", ++ "TMIx_FRAG_QUADS_RAST", ++ "TMIx_FRAG_QUADS_EZS_TEST", ++ "TMIx_FRAG_QUADS_EZS_UPDATE", ++ "TMIx_FRAG_QUADS_EZS_KILL", ++ "TMIx_FRAG_LZS_TEST", ++ "TMIx_FRAG_LZS_KILL", ++ "", ++ "TMIx_FRAG_PTILES", ++ "TMIx_FRAG_TRANS_ELIM", ++ "TMIx_QUAD_FPK_KILLER", ++ "", ++ "TMIx_COMPUTE_ACTIVE", ++ "TMIx_COMPUTE_TASKS", ++ "TMIx_COMPUTE_WARPS", ++ "TMIx_COMPUTE_STARVING", ++ "TMIx_EXEC_CORE_ACTIVE", ++ "TMIx_EXEC_ACTIVE", ++ "TMIx_EXEC_INSTR_COUNT", ++ "TMIx_EXEC_INSTR_DIVERGED", ++ "TMIx_EXEC_INSTR_STARVING", ++ "TMIx_ARITH_INSTR_SINGLE_FMA", ++ "TMIx_ARITH_INSTR_DOUBLE", ++ "TMIx_ARITH_INSTR_MSG", ++ "TMIx_ARITH_INSTR_MSG_ONLY", ++ "TMIx_TEX_INSTR", ++ "TMIx_TEX_INSTR_MIPMAP", ++ "TMIx_TEX_INSTR_COMPRESSED", ++ "TMIx_TEX_INSTR_3D", ++ "TMIx_TEX_INSTR_TRILINEAR", ++ "TMIx_TEX_COORD_ISSUE", ++ "TMIx_TEX_COORD_STALL", ++ "TMIx_TEX_STARVE_CACHE", ++ "TMIx_TEX_STARVE_FILTER", ++ "TMIx_LS_MEM_READ_FULL", ++ "TMIx_LS_MEM_READ_SHORT", ++ "TMIx_LS_MEM_WRITE_FULL", ++ "TMIx_LS_MEM_WRITE_SHORT", ++ "TMIx_LS_MEM_ATOMIC", ++ "TMIx_VARY_INSTR", ++ "TMIx_VARY_SLOT_32", ++ "TMIx_VARY_SLOT_16", ++ "TMIx_ATTR_INSTR", ++ "TMIx_ARITH_INSTR_FP_MUL", ++ "TMIx_BEATS_RD_FTC", ++ "TMIx_BEATS_RD_FTC_EXT", ++ "TMIx_BEATS_RD_LSC", ++ "TMIx_BEATS_RD_LSC_EXT", ++ "TMIx_BEATS_RD_TEX", ++ "TMIx_BEATS_RD_TEX_EXT", ++ "TMIx_BEATS_RD_OTHER", ++ "TMIx_BEATS_WR_LSC", ++ "TMIx_BEATS_WR_TIB", ++ "", ++ ++ /* Performance counters for the Memory System */ ++ "", ++ "", ++ "", ++ "", ++ "TMIx_MMU_REQUESTS", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "TMIx_L2_RD_MSG_IN", ++ "TMIx_L2_RD_MSG_IN_STALL", ++ "TMIx_L2_WR_MSG_IN", ++ "TMIx_L2_WR_MSG_IN_STALL", ++ "TMIx_L2_SNP_MSG_IN", ++ "TMIx_L2_SNP_MSG_IN_STALL", ++ "TMIx_L2_RD_MSG_OUT", ++ "TMIx_L2_RD_MSG_OUT_STALL", ++ "TMIx_L2_WR_MSG_OUT", ++ "TMIx_L2_ANY_LOOKUP", ++ "TMIx_L2_READ_LOOKUP", ++ "TMIx_L2_WRITE_LOOKUP", ++ "TMIx_L2_EXT_SNOOP_LOOKUP", ++ "TMIx_L2_EXT_READ", ++ "TMIx_L2_EXT_READ_NOSNP", ++ "TMIx_L2_EXT_READ_UNIQUE", ++ "TMIx_L2_EXT_READ_BEATS", ++ "TMIx_L2_EXT_AR_STALL", ++ "TMIx_L2_EXT_AR_CNT_Q1", ++ "TMIx_L2_EXT_AR_CNT_Q2", ++ "TMIx_L2_EXT_AR_CNT_Q3", ++ "TMIx_L2_EXT_RRESP_0_127", ++ "TMIx_L2_EXT_RRESP_128_191", ++ "TMIx_L2_EXT_RRESP_192_255", ++ "TMIx_L2_EXT_RRESP_256_319", ++ "TMIx_L2_EXT_RRESP_320_383", ++ "TMIx_L2_EXT_WRITE", ++ "TMIx_L2_EXT_WRITE_NOSNP_FULL", ++ "TMIx_L2_EXT_WRITE_NOSNP_PTL", ++ "TMIx_L2_EXT_WRITE_SNP_FULL", ++ "TMIx_L2_EXT_WRITE_SNP_PTL", ++ "TMIx_L2_EXT_WRITE_BEATS", ++ "TMIx_L2_EXT_W_STALL", ++ "TMIx_L2_EXT_AW_CNT_Q1", ++ "TMIx_L2_EXT_AW_CNT_Q2", ++ "TMIx_L2_EXT_AW_CNT_Q3", ++ "TMIx_L2_EXT_SNOOP", ++ "TMIx_L2_EXT_SNOOP_STALL", ++ "TMIx_L2_EXT_SNOOP_RESP_CLEAN", ++ "TMIx_L2_EXT_SNOOP_RESP_DATA", ++ "TMIx_L2_EXT_SNOOP_INTERNAL", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++}; ++ ++#endif /* _KBASE_GATOR_HWCNT_NAMES_TMIX_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator_hwcnt_names_tsix.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator_hwcnt_names_tsix.h +new file mode 100755 +index 000000000000..fb6a1437a1f6 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator_hwcnt_names_tsix.h +@@ -0,0 +1,291 @@ ++/* ++ * ++ * (C) COPYRIGHT 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* ++ * This header was autogenerated, it should not be edited. ++ */ ++ ++#ifndef _KBASE_GATOR_HWCNT_NAMES_TSIX_H_ ++#define _KBASE_GATOR_HWCNT_NAMES_TSIX_H_ ++ ++static const char * const hardware_counters_mali_tSIx[] = { ++ /* Performance counters for the Job Manager */ ++ "", ++ "", ++ "", ++ "", ++ "TSIx_MESSAGES_SENT", ++ "TSIx_MESSAGES_RECEIVED", ++ "TSIx_GPU_ACTIVE", ++ "TSIx_IRQ_ACTIVE", ++ "TSIx_JS0_JOBS", ++ "TSIx_JS0_TASKS", ++ "TSIx_JS0_ACTIVE", ++ "", ++ "TSIx_JS0_WAIT_READ", ++ "TSIx_JS0_WAIT_ISSUE", ++ "TSIx_JS0_WAIT_DEPEND", ++ "TSIx_JS0_WAIT_FINISH", ++ "TSIx_JS1_JOBS", ++ "TSIx_JS1_TASKS", ++ "TSIx_JS1_ACTIVE", ++ "", ++ "TSIx_JS1_WAIT_READ", ++ "TSIx_JS1_WAIT_ISSUE", ++ "TSIx_JS1_WAIT_DEPEND", ++ "TSIx_JS1_WAIT_FINISH", ++ "TSIx_JS2_JOBS", ++ "TSIx_JS2_TASKS", ++ "TSIx_JS2_ACTIVE", ++ "", ++ "TSIx_JS2_WAIT_READ", ++ "TSIx_JS2_WAIT_ISSUE", ++ "TSIx_JS2_WAIT_DEPEND", ++ "TSIx_JS2_WAIT_FINISH", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ ++ /* Performance counters for the Tiler */ ++ "", ++ "", ++ "", ++ "", ++ "TSIx_TILER_ACTIVE", ++ "TSIx_JOBS_PROCESSED", ++ "TSIx_TRIANGLES", ++ "TSIx_LINES", ++ "TSIx_POINTS", ++ "TSIx_FRONT_FACING", ++ "TSIx_BACK_FACING", ++ "TSIx_PRIM_VISIBLE", ++ "TSIx_PRIM_CULLED", ++ "TSIx_PRIM_CLIPPED", ++ "TSIx_PRIM_SAT_CULLED", ++ "TSIx_BIN_ALLOC_INIT", ++ "TSIx_BIN_ALLOC_OVERFLOW", ++ "TSIx_BUS_READ", ++ "", ++ "TSIx_BUS_WRITE", ++ "TSIx_LOADING_DESC", ++ "TSIx_IDVS_POS_SHAD_REQ", ++ "TSIx_IDVS_POS_SHAD_WAIT", ++ "TSIx_IDVS_POS_SHAD_STALL", ++ "TSIx_IDVS_POS_FIFO_FULL", ++ "TSIx_PREFETCH_STALL", ++ "TSIx_VCACHE_HIT", ++ "TSIx_VCACHE_MISS", ++ "TSIx_VCACHE_LINE_WAIT", ++ "TSIx_VFETCH_POS_READ_WAIT", ++ "TSIx_VFETCH_VERTEX_WAIT", ++ "TSIx_VFETCH_STALL", ++ "TSIx_PRIMASSY_STALL", ++ "TSIx_BBOX_GEN_STALL", ++ "TSIx_IDVS_VBU_HIT", ++ "TSIx_IDVS_VBU_MISS", ++ "TSIx_IDVS_VBU_LINE_DEALLOCATE", ++ "TSIx_IDVS_VAR_SHAD_REQ", ++ "TSIx_IDVS_VAR_SHAD_STALL", ++ "TSIx_BINNER_STALL", ++ "TSIx_ITER_STALL", ++ "TSIx_COMPRESS_MISS", ++ "TSIx_COMPRESS_STALL", ++ "TSIx_PCACHE_HIT", ++ "TSIx_PCACHE_MISS", ++ "TSIx_PCACHE_MISS_STALL", ++ "TSIx_PCACHE_EVICT_STALL", ++ "TSIx_PMGR_PTR_WR_STALL", ++ "TSIx_PMGR_PTR_RD_STALL", ++ "TSIx_PMGR_CMD_WR_STALL", ++ "TSIx_WRBUF_ACTIVE", ++ "TSIx_WRBUF_HIT", ++ "TSIx_WRBUF_MISS", ++ "TSIx_WRBUF_NO_FREE_LINE_STALL", ++ "TSIx_WRBUF_NO_AXI_ID_STALL", ++ "TSIx_WRBUF_AXI_STALL", ++ "", ++ "", ++ "", ++ "TSIx_UTLB_TRANS", ++ "TSIx_UTLB_TRANS_HIT", ++ "TSIx_UTLB_TRANS_STALL", ++ "TSIx_UTLB_TRANS_MISS_DELAY", ++ "TSIx_UTLB_MMU_REQ", ++ ++ /* Performance counters for the Shader Core */ ++ "", ++ "", ++ "", ++ "", ++ "TSIx_FRAG_ACTIVE", ++ "TSIx_FRAG_PRIMITIVES", ++ "TSIx_FRAG_PRIM_RAST", ++ "TSIx_FRAG_FPK_ACTIVE", ++ "TSIx_FRAG_STARVING", ++ "TSIx_FRAG_WARPS", ++ "TSIx_FRAG_PARTIAL_WARPS", ++ "TSIx_FRAG_QUADS_RAST", ++ "TSIx_FRAG_QUADS_EZS_TEST", ++ "TSIx_FRAG_QUADS_EZS_UPDATE", ++ "TSIx_FRAG_QUADS_EZS_KILL", ++ "TSIx_FRAG_LZS_TEST", ++ "TSIx_FRAG_LZS_KILL", ++ "", ++ "TSIx_FRAG_PTILES", ++ "TSIx_FRAG_TRANS_ELIM", ++ "TSIx_QUAD_FPK_KILLER", ++ "", ++ "TSIx_COMPUTE_ACTIVE", ++ "TSIx_COMPUTE_TASKS", ++ "TSIx_COMPUTE_WARPS", ++ "TSIx_COMPUTE_STARVING", ++ "TSIx_EXEC_CORE_ACTIVE", ++ "TSIx_EXEC_ACTIVE", ++ "TSIx_EXEC_INSTR_COUNT", ++ "TSIx_EXEC_INSTR_DIVERGED", ++ "TSIx_EXEC_INSTR_STARVING", ++ "TSIx_ARITH_INSTR_SINGLE_FMA", ++ "TSIx_ARITH_INSTR_DOUBLE", ++ "TSIx_ARITH_INSTR_MSG", ++ "TSIx_ARITH_INSTR_MSG_ONLY", ++ "TSIx_TEX_MSGI_NUM_QUADS", ++ "TSIx_TEX_DFCH_NUM_PASSES", ++ "TSIx_TEX_DFCH_NUM_PASSES_MISS", ++ "TSIx_TEX_DFCH_NUM_PASSES_MIP_MAP", ++ "TSIx_TEX_TIDX_NUM_SPLIT_MIP_MAP", ++ "TSIx_TEX_TFCH_NUM_LINES_FETCHED", ++ "TSIx_TEX_TFCH_NUM_LINES_FETCHED_BLOCK", ++ "TSIx_TEX_TFCH_NUM_OPERATIONS", ++ "TSIx_TEX_FILT_NUM_OPERATIONS", ++ "TSIx_LS_MEM_READ_FULL", ++ "TSIx_LS_MEM_READ_SHORT", ++ "TSIx_LS_MEM_WRITE_FULL", ++ "TSIx_LS_MEM_WRITE_SHORT", ++ "TSIx_LS_MEM_ATOMIC", ++ "TSIx_VARY_INSTR", ++ "TSIx_VARY_SLOT_32", ++ "TSIx_VARY_SLOT_16", ++ "TSIx_ATTR_INSTR", ++ "TSIx_ARITH_INSTR_FP_MUL", ++ "TSIx_BEATS_RD_FTC", ++ "TSIx_BEATS_RD_FTC_EXT", ++ "TSIx_BEATS_RD_LSC", ++ "TSIx_BEATS_RD_LSC_EXT", ++ "TSIx_BEATS_RD_TEX", ++ "TSIx_BEATS_RD_TEX_EXT", ++ "TSIx_BEATS_RD_OTHER", ++ "TSIx_BEATS_WR_LSC", ++ "TSIx_BEATS_WR_TIB", ++ "", ++ ++ /* Performance counters for the Memory System */ ++ "", ++ "", ++ "", ++ "", ++ "TSIx_MMU_REQUESTS", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "TSIx_L2_RD_MSG_IN", ++ "TSIx_L2_RD_MSG_IN_STALL", ++ "TSIx_L2_WR_MSG_IN", ++ "TSIx_L2_WR_MSG_IN_STALL", ++ "TSIx_L2_SNP_MSG_IN", ++ "TSIx_L2_SNP_MSG_IN_STALL", ++ "TSIx_L2_RD_MSG_OUT", ++ "TSIx_L2_RD_MSG_OUT_STALL", ++ "TSIx_L2_WR_MSG_OUT", ++ "TSIx_L2_ANY_LOOKUP", ++ "TSIx_L2_READ_LOOKUP", ++ "TSIx_L2_WRITE_LOOKUP", ++ "TSIx_L2_EXT_SNOOP_LOOKUP", ++ "TSIx_L2_EXT_READ", ++ "TSIx_L2_EXT_READ_NOSNP", ++ "TSIx_L2_EXT_READ_UNIQUE", ++ "TSIx_L2_EXT_READ_BEATS", ++ "TSIx_L2_EXT_AR_STALL", ++ "TSIx_L2_EXT_AR_CNT_Q1", ++ "TSIx_L2_EXT_AR_CNT_Q2", ++ "TSIx_L2_EXT_AR_CNT_Q3", ++ "TSIx_L2_EXT_RRESP_0_127", ++ "TSIx_L2_EXT_RRESP_128_191", ++ "TSIx_L2_EXT_RRESP_192_255", ++ "TSIx_L2_EXT_RRESP_256_319", ++ "TSIx_L2_EXT_RRESP_320_383", ++ "TSIx_L2_EXT_WRITE", ++ "TSIx_L2_EXT_WRITE_NOSNP_FULL", ++ "TSIx_L2_EXT_WRITE_NOSNP_PTL", ++ "TSIx_L2_EXT_WRITE_SNP_FULL", ++ "TSIx_L2_EXT_WRITE_SNP_PTL", ++ "TSIx_L2_EXT_WRITE_BEATS", ++ "TSIx_L2_EXT_W_STALL", ++ "TSIx_L2_EXT_AW_CNT_Q1", ++ "TSIx_L2_EXT_AW_CNT_Q2", ++ "TSIx_L2_EXT_AW_CNT_Q3", ++ "TSIx_L2_EXT_SNOOP", ++ "TSIx_L2_EXT_SNOOP_STALL", ++ "TSIx_L2_EXT_SNOOP_RESP_CLEAN", ++ "TSIx_L2_EXT_SNOOP_RESP_DATA", ++ "TSIx_L2_EXT_SNOOP_INTERNAL", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++}; ++ ++#endif /* _KBASE_GATOR_HWCNT_NAMES_TSIX_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gpu_id.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gpu_id.h +new file mode 100755 +index 000000000000..2d368dfaf644 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gpu_id.h +@@ -0,0 +1,118 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++#ifndef _KBASE_GPU_ID_H_ ++#define _KBASE_GPU_ID_H_ ++ ++/* GPU_ID register */ ++#define GPU_ID_VERSION_STATUS_SHIFT 0 ++#define GPU_ID_VERSION_MINOR_SHIFT 4 ++#define GPU_ID_VERSION_MAJOR_SHIFT 12 ++#define GPU_ID_VERSION_PRODUCT_ID_SHIFT 16 ++#define GPU_ID_VERSION_STATUS (0xF << GPU_ID_VERSION_STATUS_SHIFT) ++#define GPU_ID_VERSION_MINOR (0xFF << GPU_ID_VERSION_MINOR_SHIFT) ++#define GPU_ID_VERSION_MAJOR (0xF << GPU_ID_VERSION_MAJOR_SHIFT) ++#define GPU_ID_VERSION_PRODUCT_ID (0xFFFF << GPU_ID_VERSION_PRODUCT_ID_SHIFT) ++ ++/* Values for GPU_ID_VERSION_PRODUCT_ID bitfield */ ++#define GPU_ID_PI_T60X 0x6956 ++#define GPU_ID_PI_T62X 0x0620 ++#define GPU_ID_PI_T76X 0x0750 ++#define GPU_ID_PI_T72X 0x0720 ++#define GPU_ID_PI_TFRX 0x0880 ++#define GPU_ID_PI_T86X 0x0860 ++#define GPU_ID_PI_T82X 0x0820 ++#define GPU_ID_PI_T83X 0x0830 ++ ++/* New GPU ID format when PRODUCT_ID is >= 0x1000 (and not 0x6956) */ ++#define GPU_ID_PI_NEW_FORMAT_START 0x1000 ++#define GPU_ID_IS_NEW_FORMAT(product_id) ((product_id) != GPU_ID_PI_T60X && \ ++ (product_id) >= \ ++ GPU_ID_PI_NEW_FORMAT_START) ++ ++#define GPU_ID2_VERSION_STATUS_SHIFT 0 ++#define GPU_ID2_VERSION_MINOR_SHIFT 4 ++#define GPU_ID2_VERSION_MAJOR_SHIFT 12 ++#define GPU_ID2_PRODUCT_MAJOR_SHIFT 16 ++#define GPU_ID2_ARCH_REV_SHIFT 20 ++#define GPU_ID2_ARCH_MINOR_SHIFT 24 ++#define GPU_ID2_ARCH_MAJOR_SHIFT 28 ++#define GPU_ID2_VERSION_STATUS (0xF << GPU_ID2_VERSION_STATUS_SHIFT) ++#define GPU_ID2_VERSION_MINOR (0xFF << GPU_ID2_VERSION_MINOR_SHIFT) ++#define GPU_ID2_VERSION_MAJOR (0xF << GPU_ID2_VERSION_MAJOR_SHIFT) ++#define GPU_ID2_PRODUCT_MAJOR (0xF << GPU_ID2_PRODUCT_MAJOR_SHIFT) ++#define GPU_ID2_ARCH_REV (0xF << GPU_ID2_ARCH_REV_SHIFT) ++#define GPU_ID2_ARCH_MINOR (0xF << GPU_ID2_ARCH_MINOR_SHIFT) ++#define GPU_ID2_ARCH_MAJOR (0xF << GPU_ID2_ARCH_MAJOR_SHIFT) ++#define GPU_ID2_PRODUCT_MODEL (GPU_ID2_ARCH_MAJOR | GPU_ID2_PRODUCT_MAJOR) ++#define GPU_ID2_VERSION (GPU_ID2_VERSION_MAJOR | \ ++ GPU_ID2_VERSION_MINOR | \ ++ GPU_ID2_VERSION_STATUS) ++ ++/* Helper macro to create a partial GPU_ID (new format) that defines ++ a product ignoring its version. */ ++#define GPU_ID2_PRODUCT_MAKE(arch_major, arch_minor, arch_rev, product_major) \ ++ (((arch_major) << GPU_ID2_ARCH_MAJOR_SHIFT) | \ ++ ((arch_minor) << GPU_ID2_ARCH_MINOR_SHIFT) | \ ++ ((arch_rev) << GPU_ID2_ARCH_REV_SHIFT) | \ ++ ((product_major) << GPU_ID2_PRODUCT_MAJOR_SHIFT)) ++ ++/* Helper macro to create a partial GPU_ID (new format) that specifies the ++ revision (major, minor, status) of a product */ ++#define GPU_ID2_VERSION_MAKE(version_major, version_minor, version_status) \ ++ (((version_major) << GPU_ID2_VERSION_MAJOR_SHIFT) | \ ++ ((version_minor) << GPU_ID2_VERSION_MINOR_SHIFT) | \ ++ ((version_status) << GPU_ID2_VERSION_STATUS_SHIFT)) ++ ++/* Helper macro to create a complete GPU_ID (new format) */ ++#define GPU_ID2_MAKE(arch_major, arch_minor, arch_rev, product_major, \ ++ version_major, version_minor, version_status) \ ++ (GPU_ID2_PRODUCT_MAKE(arch_major, arch_minor, arch_rev, \ ++ product_major) | \ ++ GPU_ID2_VERSION_MAKE(version_major, version_minor, \ ++ version_status)) ++ ++/* Helper macro to create a partial GPU_ID (new format) that identifies ++ a particular GPU model by its arch_major and product_major. */ ++#define GPU_ID2_MODEL_MAKE(arch_major, product_major) \ ++ (((arch_major) << GPU_ID2_ARCH_MAJOR_SHIFT) | \ ++ ((product_major) << GPU_ID2_PRODUCT_MAJOR_SHIFT)) ++ ++/* Strip off the non-relevant bits from a product_id value and make it suitable ++ for comparison against the GPU_ID2_PRODUCT_xxx values which identify a GPU ++ model. */ ++#define GPU_ID2_MODEL_MATCH_VALUE(product_id) \ ++ (((product_id) << GPU_ID2_PRODUCT_MAJOR_SHIFT) & \ ++ GPU_ID2_PRODUCT_MODEL) ++ ++#define GPU_ID2_PRODUCT_TMIX GPU_ID2_MODEL_MAKE(6u, 0) ++#define GPU_ID2_PRODUCT_THEX GPU_ID2_MODEL_MAKE(6u, 1) ++#define GPU_ID2_PRODUCT_TSIX GPU_ID2_MODEL_MAKE(7u, 0) ++#define GPU_ID2_PRODUCT_TDVX GPU_ID2_MODEL_MAKE(7u, 3) ++ ++/* Values for GPU_ID_VERSION_STATUS field for PRODUCT_ID GPU_ID_PI_T60X */ ++#define GPU_ID_S_15DEV0 0x1 ++#define GPU_ID_S_EAC 0x2 ++ ++/* Helper macro to create a GPU_ID assuming valid values for id, major, ++ minor, status */ ++#define GPU_ID_MAKE(id, major, minor, status) \ ++ (((id) << GPU_ID_VERSION_PRODUCT_ID_SHIFT) | \ ++ ((major) << GPU_ID_VERSION_MAJOR_SHIFT) | \ ++ ((minor) << GPU_ID_VERSION_MINOR_SHIFT) | \ ++ ((status) << GPU_ID_VERSION_STATUS_SHIFT)) ++ ++#endif /* _KBASE_GPU_ID_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gpu_memory_debugfs.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gpu_memory_debugfs.c +new file mode 100755 +index 000000000000..6df0a1cb1264 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gpu_memory_debugfs.c +@@ -0,0 +1,97 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include ++ ++#ifdef CONFIG_DEBUG_FS ++/** Show callback for the @c gpu_memory debugfs file. ++ * ++ * This function is called to get the contents of the @c gpu_memory debugfs ++ * file. This is a report of current gpu memory usage. ++ * ++ * @param sfile The debugfs entry ++ * @param data Data associated with the entry ++ * ++ * @return 0 if successfully prints data in debugfs entry file ++ * -1 if it encountered an error ++ */ ++ ++static int kbasep_gpu_memory_seq_show(struct seq_file *sfile, void *data) ++{ ++ struct list_head *entry; ++ const struct list_head *kbdev_list; ++ ++ kbdev_list = kbase_dev_list_get(); ++ list_for_each(entry, kbdev_list) { ++ struct kbase_device *kbdev = NULL; ++ struct kbasep_kctx_list_element *element; ++ ++ kbdev = list_entry(entry, struct kbase_device, entry); ++ /* output the total memory usage and cap for this device */ ++ seq_printf(sfile, "%-16s %10u\n", ++ kbdev->devname, ++ atomic_read(&(kbdev->memdev.used_pages))); ++ mutex_lock(&kbdev->kctx_list_lock); ++ list_for_each_entry(element, &kbdev->kctx_list, link) { ++ /* output the memory usage and cap for each kctx ++ * opened on this device */ ++ seq_printf(sfile, " %s-0x%p %10u\n", ++ "kctx", ++ element->kctx, ++ atomic_read(&(element->kctx->used_pages))); ++ } ++ mutex_unlock(&kbdev->kctx_list_lock); ++ } ++ kbase_dev_list_put(kbdev_list); ++ return 0; ++} ++ ++/* ++ * File operations related to debugfs entry for gpu_memory ++ */ ++static int kbasep_gpu_memory_debugfs_open(struct inode *in, struct file *file) ++{ ++ return single_open(file, kbasep_gpu_memory_seq_show , NULL); ++} ++ ++static const struct file_operations kbasep_gpu_memory_debugfs_fops = { ++ .open = kbasep_gpu_memory_debugfs_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = single_release, ++}; ++ ++/* ++ * Initialize debugfs entry for gpu_memory ++ */ ++void kbasep_gpu_memory_debugfs_init(struct kbase_device *kbdev) ++{ ++ debugfs_create_file("gpu_memory", S_IRUGO, ++ kbdev->mali_debugfs_directory, NULL, ++ &kbasep_gpu_memory_debugfs_fops); ++ return; ++} ++ ++#else ++/* ++ * Stub functions for when debugfs is disabled ++ */ ++void kbasep_gpu_memory_debugfs_init(struct kbase_device *kbdev) ++{ ++ return; ++} ++#endif +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gpu_memory_debugfs.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gpu_memory_debugfs.h +new file mode 100755 +index 000000000000..7045693eb910 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gpu_memory_debugfs.h +@@ -0,0 +1,37 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2014, 2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/** ++ * @file mali_kbase_gpu_memory_debugfs.h ++ * Header file for gpu_memory entry in debugfs ++ * ++ */ ++ ++#ifndef _KBASE_GPU_MEMORY_DEBUGFS_H ++#define _KBASE_GPU_MEMORY_DEBUGFS_H ++ ++#include ++#include ++ ++/** ++ * @brief Initialize gpu_memory debugfs entry ++ */ ++void kbasep_gpu_memory_debugfs_init(struct kbase_device *kbdev); ++ ++#endif /*_KBASE_GPU_MEMORY_DEBUGFS_H*/ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gpuprops.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gpuprops.c +new file mode 100755 +index 000000000000..4130810f1038 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gpuprops.c +@@ -0,0 +1,514 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * Base kernel property query APIs ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include "mali_kbase_ioctl.h" ++#include ++ ++/** ++ * KBASE_UBFX32 - Extracts bits from a 32-bit bitfield. ++ * @value: The value from which to extract bits. ++ * @offset: The first bit to extract (0 being the LSB). ++ * @size: The number of bits to extract. ++ * ++ * Context: @offset + @size <= 32. ++ * ++ * Return: Bits [@offset, @offset + @size) from @value. ++ */ ++/* from mali_cdsb.h */ ++#define KBASE_UBFX32(value, offset, size) \ ++ (((u32)(value) >> (u32)(offset)) & (u32)((1ULL << (u32)(size)) - 1)) ++ ++int kbase_gpuprops_uk_get_props(struct kbase_context *kctx, struct kbase_uk_gpuprops * const kbase_props) ++{ ++ kbase_gpu_clk_speed_func get_gpu_speed_mhz; ++ u32 gpu_speed_mhz; ++ int rc = 1; ++ ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ KBASE_DEBUG_ASSERT(NULL != kbase_props); ++ ++ /* Current GPU speed is requested from the system integrator via the GPU_SPEED_FUNC function. ++ * If that function fails, or the function is not provided by the system integrator, we report the maximum ++ * GPU speed as specified by GPU_FREQ_KHZ_MAX. ++ */ ++ get_gpu_speed_mhz = (kbase_gpu_clk_speed_func) GPU_SPEED_FUNC; ++ if (get_gpu_speed_mhz != NULL) { ++ rc = get_gpu_speed_mhz(&gpu_speed_mhz); ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++ /* Issue a warning message when the reported GPU speed falls outside the min/max range */ ++ if (rc == 0) { ++ u32 gpu_speed_khz = gpu_speed_mhz * 1000; ++ ++ if (gpu_speed_khz < kctx->kbdev->gpu_props.props.core_props.gpu_freq_khz_min || ++ gpu_speed_khz > kctx->kbdev->gpu_props.props.core_props.gpu_freq_khz_max) ++ dev_warn(kctx->kbdev->dev, "GPU Speed is outside of min/max range (got %lu Khz, min %lu Khz, max %lu Khz)\n", ++ (unsigned long)gpu_speed_khz, ++ (unsigned long)kctx->kbdev->gpu_props.props.core_props.gpu_freq_khz_min, ++ (unsigned long)kctx->kbdev->gpu_props.props.core_props.gpu_freq_khz_max); ++ } ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++ } ++ if (kctx->kbdev->clock) { ++ gpu_speed_mhz = clk_get_rate(kctx->kbdev->clock) / 1000000; ++ rc = 0; ++ } ++ if (rc != 0) ++ gpu_speed_mhz = kctx->kbdev->gpu_props.props.core_props.gpu_freq_khz_max / 1000; ++ ++ kctx->kbdev->gpu_props.props.core_props.gpu_speed_mhz = gpu_speed_mhz; ++ ++ memcpy(&kbase_props->props, &kctx->kbdev->gpu_props.props, sizeof(kbase_props->props)); ++ ++ /* Before API 8.2 they expect L3 cache info here, which was always 0 */ ++ if (kctx->api_version < KBASE_API_VERSION(8, 2)) ++ kbase_props->props.raw_props.suspend_size = 0; ++ ++ return 0; ++} ++ ++static void kbase_gpuprops_construct_coherent_groups(base_gpu_props * const props) ++{ ++ struct mali_base_gpu_coherent_group *current_group; ++ u64 group_present; ++ u64 group_mask; ++ u64 first_set, first_set_prev; ++ u32 num_groups = 0; ++ ++ KBASE_DEBUG_ASSERT(NULL != props); ++ ++ props->coherency_info.coherency = props->raw_props.mem_features; ++ props->coherency_info.num_core_groups = hweight64(props->raw_props.l2_present); ++ ++ if (props->coherency_info.coherency & GROUPS_L2_COHERENT) { ++ /* Group is l2 coherent */ ++ group_present = props->raw_props.l2_present; ++ } else { ++ /* Group is l1 coherent */ ++ group_present = props->raw_props.shader_present; ++ } ++ ++ /* ++ * The coherent group mask can be computed from the l2 present ++ * register. ++ * ++ * For the coherent group n: ++ * group_mask[n] = (first_set[n] - 1) & ~(first_set[n-1] - 1) ++ * where first_set is group_present with only its nth set-bit kept ++ * (i.e. the position from where a new group starts). ++ * ++ * For instance if the groups are l2 coherent and l2_present=0x0..01111: ++ * The first mask is: ++ * group_mask[1] = (first_set[1] - 1) & ~(first_set[0] - 1) ++ * = (0x0..010 - 1) & ~(0x0..01 - 1) ++ * = 0x0..00f ++ * The second mask is: ++ * group_mask[2] = (first_set[2] - 1) & ~(first_set[1] - 1) ++ * = (0x0..100 - 1) & ~(0x0..010 - 1) ++ * = 0x0..0f0 ++ * And so on until all the bits from group_present have been cleared ++ * (i.e. there is no group left). ++ */ ++ ++ current_group = props->coherency_info.group; ++ first_set = group_present & ~(group_present - 1); ++ ++ while (group_present != 0 && num_groups < BASE_MAX_COHERENT_GROUPS) { ++ group_present -= first_set; /* Clear the current group bit */ ++ first_set_prev = first_set; ++ ++ first_set = group_present & ~(group_present - 1); ++ group_mask = (first_set - 1) & ~(first_set_prev - 1); ++ ++ /* Populate the coherent_group structure for each group */ ++ current_group->core_mask = group_mask & props->raw_props.shader_present; ++ current_group->num_cores = hweight64(current_group->core_mask); ++ ++ num_groups++; ++ current_group++; ++ } ++ ++ if (group_present != 0) ++ pr_warn("Too many coherent groups (keeping only %d groups).\n", BASE_MAX_COHERENT_GROUPS); ++ ++ props->coherency_info.num_groups = num_groups; ++} ++ ++/** ++ * kbase_gpuprops_get_props - Get the GPU configuration ++ * @gpu_props: The &base_gpu_props structure ++ * @kbdev: The &struct kbase_device structure for the device ++ * ++ * Fill the &base_gpu_props structure with values from the GPU configuration ++ * registers. Only the raw properties are filled in this function ++ */ ++static void kbase_gpuprops_get_props(base_gpu_props * const gpu_props, struct kbase_device *kbdev) ++{ ++ struct kbase_gpuprops_regdump regdump; ++ int i; ++ ++ KBASE_DEBUG_ASSERT(NULL != kbdev); ++ KBASE_DEBUG_ASSERT(NULL != gpu_props); ++ ++ /* Dump relevant registers */ ++ kbase_backend_gpuprops_get(kbdev, ®dump); ++ ++ gpu_props->raw_props.gpu_id = regdump.gpu_id; ++ gpu_props->raw_props.tiler_features = regdump.tiler_features; ++ gpu_props->raw_props.mem_features = regdump.mem_features; ++ gpu_props->raw_props.mmu_features = regdump.mmu_features; ++ gpu_props->raw_props.l2_features = regdump.l2_features; ++ gpu_props->raw_props.suspend_size = regdump.suspend_size; ++ ++ gpu_props->raw_props.as_present = regdump.as_present; ++ gpu_props->raw_props.js_present = regdump.js_present; ++ gpu_props->raw_props.shader_present = ++ ((u64) regdump.shader_present_hi << 32) + ++ regdump.shader_present_lo; ++ gpu_props->raw_props.tiler_present = ++ ((u64) regdump.tiler_present_hi << 32) + ++ regdump.tiler_present_lo; ++ gpu_props->raw_props.l2_present = ++ ((u64) regdump.l2_present_hi << 32) + ++ regdump.l2_present_lo; ++#ifdef CONFIG_MALI_CORESTACK ++ gpu_props->raw_props.stack_present = ++ ((u64) regdump.stack_present_hi << 32) + ++ regdump.stack_present_lo; ++#else /* CONFIG_MALI_CORESTACK */ ++ gpu_props->raw_props.stack_present = 0; ++#endif /* CONFIG_MALI_CORESTACK */ ++ ++ for (i = 0; i < GPU_MAX_JOB_SLOTS; i++) ++ gpu_props->raw_props.js_features[i] = regdump.js_features[i]; ++ ++ for (i = 0; i < BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS; i++) ++ gpu_props->raw_props.texture_features[i] = regdump.texture_features[i]; ++ ++ gpu_props->raw_props.thread_max_barrier_size = regdump.thread_max_barrier_size; ++ gpu_props->raw_props.thread_max_threads = regdump.thread_max_threads; ++ gpu_props->raw_props.thread_max_workgroup_size = regdump.thread_max_workgroup_size; ++ gpu_props->raw_props.thread_features = regdump.thread_features; ++} ++ ++void kbase_gpuprops_update_core_props_gpu_id(base_gpu_props * const gpu_props) ++{ ++ gpu_props->core_props.version_status = ++ KBASE_UBFX32(gpu_props->raw_props.gpu_id, 0U, 4); ++ gpu_props->core_props.minor_revision = ++ KBASE_UBFX32(gpu_props->raw_props.gpu_id, 4U, 8); ++ gpu_props->core_props.major_revision = ++ KBASE_UBFX32(gpu_props->raw_props.gpu_id, 12U, 4); ++ gpu_props->core_props.product_id = ++ KBASE_UBFX32(gpu_props->raw_props.gpu_id, 16U, 16); ++} ++ ++/** ++ * kbase_gpuprops_calculate_props - Calculate the derived properties ++ * @gpu_props: The &base_gpu_props structure ++ * @kbdev: The &struct kbase_device structure for the device ++ * ++ * Fill the &base_gpu_props structure with values derived from the GPU ++ * configuration registers ++ */ ++static void kbase_gpuprops_calculate_props(base_gpu_props * const gpu_props, struct kbase_device *kbdev) ++{ ++ int i; ++ ++ /* Populate the base_gpu_props structure */ ++ kbase_gpuprops_update_core_props_gpu_id(gpu_props); ++ gpu_props->core_props.log2_program_counter_size = KBASE_GPU_PC_SIZE_LOG2; ++ gpu_props->core_props.gpu_available_memory_size = totalram_pages << PAGE_SHIFT; ++ ++ for (i = 0; i < BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS; i++) ++ gpu_props->core_props.texture_features[i] = gpu_props->raw_props.texture_features[i]; ++ ++ gpu_props->l2_props.log2_line_size = KBASE_UBFX32(gpu_props->raw_props.l2_features, 0U, 8); ++ gpu_props->l2_props.log2_cache_size = KBASE_UBFX32(gpu_props->raw_props.l2_features, 16U, 8); ++ ++ /* Field with number of l2 slices is added to MEM_FEATURES register ++ * since t76x. Below code assumes that for older GPU reserved bits will ++ * be read as zero. */ ++ gpu_props->l2_props.num_l2_slices = ++ KBASE_UBFX32(gpu_props->raw_props.mem_features, 8U, 4) + 1; ++ ++ gpu_props->tiler_props.bin_size_bytes = 1 << KBASE_UBFX32(gpu_props->raw_props.tiler_features, 0U, 6); ++ gpu_props->tiler_props.max_active_levels = KBASE_UBFX32(gpu_props->raw_props.tiler_features, 8U, 4); ++ ++ if (gpu_props->raw_props.thread_max_threads == 0) ++ gpu_props->thread_props.max_threads = THREAD_MT_DEFAULT; ++ else ++ gpu_props->thread_props.max_threads = gpu_props->raw_props.thread_max_threads; ++ ++ if (gpu_props->raw_props.thread_max_workgroup_size == 0) ++ gpu_props->thread_props.max_workgroup_size = THREAD_MWS_DEFAULT; ++ else ++ gpu_props->thread_props.max_workgroup_size = gpu_props->raw_props.thread_max_workgroup_size; ++ ++ if (gpu_props->raw_props.thread_max_barrier_size == 0) ++ gpu_props->thread_props.max_barrier_size = THREAD_MBS_DEFAULT; ++ else ++ gpu_props->thread_props.max_barrier_size = gpu_props->raw_props.thread_max_barrier_size; ++ ++ gpu_props->thread_props.max_registers = KBASE_UBFX32(gpu_props->raw_props.thread_features, 0U, 16); ++ gpu_props->thread_props.max_task_queue = KBASE_UBFX32(gpu_props->raw_props.thread_features, 16U, 8); ++ gpu_props->thread_props.max_thread_group_split = KBASE_UBFX32(gpu_props->raw_props.thread_features, 24U, 6); ++ gpu_props->thread_props.impl_tech = KBASE_UBFX32(gpu_props->raw_props.thread_features, 30U, 2); ++ ++ /* If values are not specified, then use defaults */ ++ if (gpu_props->thread_props.max_registers == 0) { ++ gpu_props->thread_props.max_registers = THREAD_MR_DEFAULT; ++ gpu_props->thread_props.max_task_queue = THREAD_MTQ_DEFAULT; ++ gpu_props->thread_props.max_thread_group_split = THREAD_MTGS_DEFAULT; ++ } ++ /* Initialize the coherent_group structure for each group */ ++ kbase_gpuprops_construct_coherent_groups(gpu_props); ++} ++ ++void kbase_gpuprops_set(struct kbase_device *kbdev) ++{ ++ struct kbase_gpu_props *gpu_props; ++ struct gpu_raw_gpu_props *raw; ++ ++ KBASE_DEBUG_ASSERT(NULL != kbdev); ++ gpu_props = &kbdev->gpu_props; ++ raw = &gpu_props->props.raw_props; ++ ++ /* Initialize the base_gpu_props structure from the hardware */ ++ kbase_gpuprops_get_props(&gpu_props->props, kbdev); ++ ++ /* Populate the derived properties */ ++ kbase_gpuprops_calculate_props(&gpu_props->props, kbdev); ++ ++ /* Populate kbase-only fields */ ++ gpu_props->l2_props.associativity = KBASE_UBFX32(raw->l2_features, 8U, 8); ++ gpu_props->l2_props.external_bus_width = KBASE_UBFX32(raw->l2_features, 24U, 8); ++ ++ gpu_props->mem.core_group = KBASE_UBFX32(raw->mem_features, 0U, 1); ++ ++ gpu_props->mmu.va_bits = KBASE_UBFX32(raw->mmu_features, 0U, 8); ++ gpu_props->mmu.pa_bits = KBASE_UBFX32(raw->mmu_features, 8U, 8); ++ ++ gpu_props->num_cores = hweight64(raw->shader_present); ++ gpu_props->num_core_groups = hweight64(raw->l2_present); ++ gpu_props->num_address_spaces = hweight32(raw->as_present); ++ gpu_props->num_job_slots = hweight32(raw->js_present); ++} ++ ++void kbase_gpuprops_set_features(struct kbase_device *kbdev) ++{ ++ base_gpu_props *gpu_props; ++ struct kbase_gpuprops_regdump regdump; ++ ++ gpu_props = &kbdev->gpu_props.props; ++ ++ /* Dump relevant registers */ ++ kbase_backend_gpuprops_get_features(kbdev, ®dump); ++ ++ /* ++ * Copy the raw value from the register, later this will get turned ++ * into the selected coherency mode. ++ * Additionally, add non-coherent mode, as this is always supported. ++ */ ++ gpu_props->raw_props.coherency_mode = regdump.coherency_features | ++ COHERENCY_FEATURE_BIT(COHERENCY_NONE); ++} ++ ++static struct { ++ u32 type; ++ size_t offset; ++ int size; ++} gpu_property_mapping[] = { ++#define PROP(name, member) \ ++ {KBASE_GPUPROP_ ## name, offsetof(struct mali_base_gpu_props, member), \ ++ sizeof(((struct mali_base_gpu_props *)0)->member)} ++ PROP(PRODUCT_ID, core_props.product_id), ++ PROP(VERSION_STATUS, core_props.version_status), ++ PROP(MINOR_REVISION, core_props.minor_revision), ++ PROP(MAJOR_REVISION, core_props.major_revision), ++ PROP(GPU_SPEED_MHZ, core_props.gpu_speed_mhz), ++ PROP(GPU_FREQ_KHZ_MAX, core_props.gpu_freq_khz_max), ++ PROP(GPU_FREQ_KHZ_MIN, core_props.gpu_freq_khz_min), ++ PROP(LOG2_PROGRAM_COUNTER_SIZE, core_props.log2_program_counter_size), ++ PROP(TEXTURE_FEATURES_0, core_props.texture_features[0]), ++ PROP(TEXTURE_FEATURES_1, core_props.texture_features[1]), ++ PROP(TEXTURE_FEATURES_2, core_props.texture_features[2]), ++ PROP(GPU_AVAILABLE_MEMORY_SIZE, core_props.gpu_available_memory_size), ++ ++ PROP(L2_LOG2_LINE_SIZE, l2_props.log2_line_size), ++ PROP(L2_LOG2_CACHE_SIZE, l2_props.log2_cache_size), ++ PROP(L2_NUM_L2_SLICES, l2_props.num_l2_slices), ++ ++ PROP(TILER_BIN_SIZE_BYTES, tiler_props.bin_size_bytes), ++ PROP(TILER_MAX_ACTIVE_LEVELS, tiler_props.max_active_levels), ++ ++ PROP(MAX_THREADS, thread_props.max_threads), ++ PROP(MAX_WORKGROUP_SIZE, thread_props.max_workgroup_size), ++ PROP(MAX_BARRIER_SIZE, thread_props.max_barrier_size), ++ PROP(MAX_REGISTERS, thread_props.max_registers), ++ PROP(MAX_TASK_QUEUE, thread_props.max_task_queue), ++ PROP(MAX_THREAD_GROUP_SPLIT, thread_props.max_thread_group_split), ++ PROP(IMPL_TECH, thread_props.impl_tech), ++ ++ PROP(RAW_SHADER_PRESENT, raw_props.shader_present), ++ PROP(RAW_TILER_PRESENT, raw_props.tiler_present), ++ PROP(RAW_L2_PRESENT, raw_props.l2_present), ++ PROP(RAW_STACK_PRESENT, raw_props.stack_present), ++ PROP(RAW_L2_FEATURES, raw_props.l2_features), ++ PROP(RAW_SUSPEND_SIZE, raw_props.suspend_size), ++ PROP(RAW_MEM_FEATURES, raw_props.mem_features), ++ PROP(RAW_MMU_FEATURES, raw_props.mmu_features), ++ PROP(RAW_AS_PRESENT, raw_props.as_present), ++ PROP(RAW_JS_PRESENT, raw_props.js_present), ++ PROP(RAW_JS_FEATURES_0, raw_props.js_features[0]), ++ PROP(RAW_JS_FEATURES_1, raw_props.js_features[1]), ++ PROP(RAW_JS_FEATURES_2, raw_props.js_features[2]), ++ PROP(RAW_JS_FEATURES_3, raw_props.js_features[3]), ++ PROP(RAW_JS_FEATURES_4, raw_props.js_features[4]), ++ PROP(RAW_JS_FEATURES_5, raw_props.js_features[5]), ++ PROP(RAW_JS_FEATURES_6, raw_props.js_features[6]), ++ PROP(RAW_JS_FEATURES_7, raw_props.js_features[7]), ++ PROP(RAW_JS_FEATURES_8, raw_props.js_features[8]), ++ PROP(RAW_JS_FEATURES_9, raw_props.js_features[9]), ++ PROP(RAW_JS_FEATURES_10, raw_props.js_features[10]), ++ PROP(RAW_JS_FEATURES_11, raw_props.js_features[11]), ++ PROP(RAW_JS_FEATURES_12, raw_props.js_features[12]), ++ PROP(RAW_JS_FEATURES_13, raw_props.js_features[13]), ++ PROP(RAW_JS_FEATURES_14, raw_props.js_features[14]), ++ PROP(RAW_JS_FEATURES_15, raw_props.js_features[15]), ++ PROP(RAW_TILER_FEATURES, raw_props.tiler_features), ++ PROP(RAW_TEXTURE_FEATURES_0, raw_props.texture_features[0]), ++ PROP(RAW_TEXTURE_FEATURES_1, raw_props.texture_features[1]), ++ PROP(RAW_TEXTURE_FEATURES_2, raw_props.texture_features[2]), ++ PROP(RAW_GPU_ID, raw_props.gpu_id), ++ PROP(RAW_THREAD_MAX_THREADS, raw_props.thread_max_threads), ++ PROP(RAW_THREAD_MAX_WORKGROUP_SIZE, ++ raw_props.thread_max_workgroup_size), ++ PROP(RAW_THREAD_MAX_BARRIER_SIZE, raw_props.thread_max_barrier_size), ++ PROP(RAW_THREAD_FEATURES, raw_props.thread_features), ++ PROP(RAW_COHERENCY_MODE, raw_props.coherency_mode), ++ ++ PROP(COHERENCY_NUM_GROUPS, coherency_info.num_groups), ++ PROP(COHERENCY_NUM_CORE_GROUPS, coherency_info.num_core_groups), ++ PROP(COHERENCY_COHERENCY, coherency_info.coherency), ++ PROP(COHERENCY_GROUP_0, coherency_info.group[0].core_mask), ++ PROP(COHERENCY_GROUP_1, coherency_info.group[1].core_mask), ++ PROP(COHERENCY_GROUP_2, coherency_info.group[2].core_mask), ++ PROP(COHERENCY_GROUP_3, coherency_info.group[3].core_mask), ++ PROP(COHERENCY_GROUP_4, coherency_info.group[4].core_mask), ++ PROP(COHERENCY_GROUP_5, coherency_info.group[5].core_mask), ++ PROP(COHERENCY_GROUP_6, coherency_info.group[6].core_mask), ++ PROP(COHERENCY_GROUP_7, coherency_info.group[7].core_mask), ++ PROP(COHERENCY_GROUP_8, coherency_info.group[8].core_mask), ++ PROP(COHERENCY_GROUP_9, coherency_info.group[9].core_mask), ++ PROP(COHERENCY_GROUP_10, coherency_info.group[10].core_mask), ++ PROP(COHERENCY_GROUP_11, coherency_info.group[11].core_mask), ++ PROP(COHERENCY_GROUP_12, coherency_info.group[12].core_mask), ++ PROP(COHERENCY_GROUP_13, coherency_info.group[13].core_mask), ++ PROP(COHERENCY_GROUP_14, coherency_info.group[14].core_mask), ++ PROP(COHERENCY_GROUP_15, coherency_info.group[15].core_mask), ++ ++#undef PROP ++}; ++ ++int kbase_gpuprops_populate_user_buffer(struct kbase_device *kbdev) ++{ ++ struct kbase_gpu_props *kprops = &kbdev->gpu_props; ++ struct mali_base_gpu_props *props = &kprops->props; ++ u32 count = ARRAY_SIZE(gpu_property_mapping); ++ u32 i; ++ u32 size = 0; ++ u8 *p; ++ ++ for (i = 0; i < count; i++) { ++ /* 4 bytes for the ID, and the size of the property */ ++ size += 4 + gpu_property_mapping[i].size; ++ } ++ ++ kprops->prop_buffer_size = size; ++ kprops->prop_buffer = kmalloc(size, GFP_KERNEL); ++ ++ if (!kprops->prop_buffer) { ++ kprops->prop_buffer_size = 0; ++ return -ENOMEM; ++ } ++ ++ p = kprops->prop_buffer; ++ ++#define WRITE_U8(v) (*p++ = (v) & 0xFF) ++#define WRITE_U16(v) do { WRITE_U8(v); WRITE_U8((v) >> 8); } while (0) ++#define WRITE_U32(v) do { WRITE_U16(v); WRITE_U16((v) >> 16); } while (0) ++#define WRITE_U64(v) do { WRITE_U32(v); WRITE_U32((v) >> 32); } while (0) ++ ++ for (i = 0; i < count; i++) { ++ u32 type = gpu_property_mapping[i].type; ++ u8 type_size; ++ void *field = ((u8 *)props) + gpu_property_mapping[i].offset; ++ ++ switch (gpu_property_mapping[i].size) { ++ case 1: ++ type_size = KBASE_GPUPROP_VALUE_SIZE_U8; ++ break; ++ case 2: ++ type_size = KBASE_GPUPROP_VALUE_SIZE_U16; ++ break; ++ case 4: ++ type_size = KBASE_GPUPROP_VALUE_SIZE_U32; ++ break; ++ case 8: ++ type_size = KBASE_GPUPROP_VALUE_SIZE_U64; ++ break; ++ default: ++ dev_err(kbdev->dev, ++ "Invalid gpu_property_mapping type=%d size=%d", ++ type, gpu_property_mapping[i].size); ++ return -EINVAL; ++ } ++ ++ WRITE_U32((type<<2) | type_size); ++ ++ switch (type_size) { ++ case KBASE_GPUPROP_VALUE_SIZE_U8: ++ WRITE_U8(*((u8 *)field)); ++ break; ++ case KBASE_GPUPROP_VALUE_SIZE_U16: ++ WRITE_U16(*((u16 *)field)); ++ break; ++ case KBASE_GPUPROP_VALUE_SIZE_U32: ++ WRITE_U32(*((u32 *)field)); ++ break; ++ case KBASE_GPUPROP_VALUE_SIZE_U64: ++ WRITE_U64(*((u64 *)field)); ++ break; ++ default: /* Cannot be reached */ ++ WARN_ON(1); ++ return -EINVAL; ++ } ++ } ++ ++ return 0; ++} +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gpuprops.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gpuprops.h +new file mode 100755 +index 000000000000..57b3eaf9cd53 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gpuprops.h +@@ -0,0 +1,84 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2015,2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/** ++ * @file mali_kbase_gpuprops.h ++ * Base kernel property query APIs ++ */ ++ ++#ifndef _KBASE_GPUPROPS_H_ ++#define _KBASE_GPUPROPS_H_ ++ ++#include "mali_kbase_gpuprops_types.h" ++ ++/* Forward definition - see mali_kbase.h */ ++struct kbase_device; ++ ++/** ++ * @brief Set up Kbase GPU properties. ++ * ++ * Set up Kbase GPU properties with information from the GPU registers ++ * ++ * @param kbdev The struct kbase_device structure for the device ++ */ ++void kbase_gpuprops_set(struct kbase_device *kbdev); ++ ++/** ++ * kbase_gpuprops_set_features - Set up Kbase GPU properties ++ * @kbdev: Device pointer ++ * ++ * This function sets up GPU properties that are dependent on the hardware ++ * features bitmask. This function must be preceeded by a call to ++ * kbase_hw_set_features_mask(). ++ */ ++void kbase_gpuprops_set_features(struct kbase_device *kbdev); ++ ++/** ++ * @brief Provide GPU properties to userside through UKU call. ++ * ++ * Fill the struct kbase_uk_gpuprops with values from GPU configuration registers. ++ * ++ * @param kctx The struct kbase_context structure ++ * @param kbase_props A copy of the struct kbase_uk_gpuprops structure from userspace ++ * ++ * @return 0 on success. Any other value indicates failure. ++ */ ++int kbase_gpuprops_uk_get_props(struct kbase_context *kctx, struct kbase_uk_gpuprops * const kbase_props); ++ ++/** ++ * kbase_gpuprops_populate_user_buffer - Populate the GPU properties buffer ++ * @kbdev: The kbase device ++ * ++ * Fills kbdev->gpu_props->prop_buffer with the GPU properties for user ++ * space to read. ++ */ ++int kbase_gpuprops_populate_user_buffer(struct kbase_device *kbdev); ++ ++/** ++ * kbase_gpuprops_update_core_props_gpu_id - break down gpu id value ++ * @gpu_props: the &base_gpu_props structure ++ * ++ * Break down gpu_id value stored in base_gpu_props::raw_props.gpu_id into ++ * separate fields (version_status, minor_revision, major_revision, product_id) ++ * stored in base_gpu_props::core_props. ++ */ ++void kbase_gpuprops_update_core_props_gpu_id(base_gpu_props * const gpu_props); ++ ++ ++#endif /* _KBASE_GPUPROPS_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gpuprops_types.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gpuprops_types.h +new file mode 100755 +index 000000000000..10794fc27318 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gpuprops_types.h +@@ -0,0 +1,92 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/** ++ * @file mali_kbase_gpuprops_types.h ++ * Base kernel property query APIs ++ */ ++ ++#ifndef _KBASE_GPUPROPS_TYPES_H_ ++#define _KBASE_GPUPROPS_TYPES_H_ ++ ++#include "mali_base_kernel.h" ++ ++#define KBASE_GPU_SPEED_MHZ 123 ++#define KBASE_GPU_PC_SIZE_LOG2 24U ++ ++struct kbase_gpuprops_regdump { ++ u32 gpu_id; ++ u32 l2_features; ++ u32 suspend_size; /* API 8.2+ */ ++ u32 tiler_features; ++ u32 mem_features; ++ u32 mmu_features; ++ u32 as_present; ++ u32 js_present; ++ u32 thread_max_threads; ++ u32 thread_max_workgroup_size; ++ u32 thread_max_barrier_size; ++ u32 thread_features; ++ u32 texture_features[BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS]; ++ u32 js_features[GPU_MAX_JOB_SLOTS]; ++ u32 shader_present_lo; ++ u32 shader_present_hi; ++ u32 tiler_present_lo; ++ u32 tiler_present_hi; ++ u32 l2_present_lo; ++ u32 l2_present_hi; ++ u32 stack_present_lo; ++ u32 stack_present_hi; ++ u32 coherency_features; ++}; ++ ++struct kbase_gpu_cache_props { ++ u8 associativity; ++ u8 external_bus_width; ++}; ++ ++struct kbase_gpu_mem_props { ++ u8 core_group; ++}; ++ ++struct kbase_gpu_mmu_props { ++ u8 va_bits; ++ u8 pa_bits; ++}; ++ ++struct kbase_gpu_props { ++ /* kernel-only properties */ ++ u8 num_cores; ++ u8 num_core_groups; ++ u8 num_address_spaces; ++ u8 num_job_slots; ++ ++ struct kbase_gpu_cache_props l2_props; ++ ++ struct kbase_gpu_mem_props mem; ++ struct kbase_gpu_mmu_props mmu; ++ ++ /* Properties shared with userspace */ ++ base_gpu_props props; ++ ++ u32 prop_buffer_size; ++ void *prop_buffer; ++}; ++ ++#endif /* _KBASE_GPUPROPS_TYPES_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hw.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hw.c +new file mode 100755 +index 000000000000..eb8368ccee5e +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hw.c +@@ -0,0 +1,446 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * Run-time work-arounds helpers ++ */ ++ ++#include ++#include ++#include ++#include "mali_kbase.h" ++#include "mali_kbase_hw.h" ++ ++void kbase_hw_set_features_mask(struct kbase_device *kbdev) ++{ ++ const enum base_hw_feature *features; ++ u32 gpu_id; ++ u32 product_id; ++ ++ gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; ++ product_id = gpu_id & GPU_ID_VERSION_PRODUCT_ID; ++ product_id >>= GPU_ID_VERSION_PRODUCT_ID_SHIFT; ++ ++ if (GPU_ID_IS_NEW_FORMAT(product_id)) { ++ switch (gpu_id & GPU_ID2_PRODUCT_MODEL) { ++ case GPU_ID2_PRODUCT_TMIX: ++ features = base_hw_features_tMIx; ++ break; ++ case GPU_ID2_PRODUCT_THEX: ++ features = base_hw_features_tHEx; ++ break; ++ case GPU_ID2_PRODUCT_TSIX: ++ features = base_hw_features_tSIx; ++ break; ++ case GPU_ID2_PRODUCT_TDVX: ++ features = base_hw_features_tDVx; ++ break; ++ default: ++ features = base_hw_features_generic; ++ break; ++ } ++ } else { ++ switch (product_id) { ++ case GPU_ID_PI_TFRX: ++ /* FALLTHROUGH */ ++ case GPU_ID_PI_T86X: ++ features = base_hw_features_tFxx; ++ break; ++ case GPU_ID_PI_T83X: ++ features = base_hw_features_t83x; ++ break; ++ case GPU_ID_PI_T82X: ++ features = base_hw_features_t82x; ++ break; ++ case GPU_ID_PI_T76X: ++ features = base_hw_features_t76x; ++ break; ++ case GPU_ID_PI_T72X: ++ features = base_hw_features_t72x; ++ break; ++ case GPU_ID_PI_T62X: ++ features = base_hw_features_t62x; ++ break; ++ case GPU_ID_PI_T60X: ++ features = base_hw_features_t60x; ++ break; ++ default: ++ features = base_hw_features_generic; ++ break; ++ } ++ } ++ ++ for (; *features != BASE_HW_FEATURE_END; features++) ++ set_bit(*features, &kbdev->hw_features_mask[0]); ++} ++ ++/** ++ * kbase_hw_get_issues_for_new_id - Get the hardware issues for a new GPU ID ++ * @kbdev: Device pointer ++ * ++ * Return: pointer to an array of hardware issues, terminated by ++ * BASE_HW_ISSUE_END. ++ * ++ * This function can only be used on new-format GPU IDs, i.e. those for which ++ * GPU_ID_IS_NEW_FORMAT evaluates as true. The GPU ID is read from the @kbdev. ++ * ++ * In debugging versions of the driver, unknown versions of a known GPU will ++ * be treated as the most recent known version not later than the actual ++ * version. In such circumstances, the GPU ID in @kbdev will also be replaced ++ * with the most recent known version. ++ * ++ * Note: The GPU configuration must have been read by kbase_gpuprops_get_props() ++ * before calling this function. ++ */ ++static const enum base_hw_issue *kbase_hw_get_issues_for_new_id( ++ struct kbase_device *kbdev) ++{ ++ const enum base_hw_issue *issues = NULL; ++ ++ struct base_hw_product { ++ u32 product_model; ++ struct { ++ u32 version; ++ const enum base_hw_issue *issues; ++ } map[7]; ++ }; ++ ++ static const struct base_hw_product base_hw_products[] = { ++ {GPU_ID2_PRODUCT_TMIX, ++ {{GPU_ID2_VERSION_MAKE(0, 0, 1), ++ base_hw_issues_tMIx_r0p0_05dev0}, ++ {GPU_ID2_VERSION_MAKE(0, 0, 2), base_hw_issues_tMIx_r0p0}, ++ {GPU_ID2_VERSION_MAKE(0, 1, 0), base_hw_issues_tMIx_r0p1}, ++ {U32_MAX /* sentinel value */, NULL} } }, ++ ++ {GPU_ID2_PRODUCT_THEX, ++ {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tHEx_r0p0}, ++ {GPU_ID2_VERSION_MAKE(0, 0, 1), base_hw_issues_tHEx_r0p0}, ++ {GPU_ID2_VERSION_MAKE(0, 1, 0), base_hw_issues_tHEx_r0p1}, ++ {GPU_ID2_VERSION_MAKE(0, 1, 1), base_hw_issues_tHEx_r0p1}, ++ {GPU_ID2_VERSION_MAKE(0, 2, 0), base_hw_issues_tHEx_r0p2}, ++ {GPU_ID2_VERSION_MAKE(0, 3, 0), base_hw_issues_tHEx_r0p3}, ++ {U32_MAX, NULL} } }, ++ ++ {GPU_ID2_PRODUCT_TSIX, ++ {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tSIx_r0p0}, ++ {GPU_ID2_VERSION_MAKE(0, 0, 1), base_hw_issues_tSIx_r0p0}, ++ {GPU_ID2_VERSION_MAKE(0, 1, 0), base_hw_issues_tSIx_r0p1}, ++ {GPU_ID2_VERSION_MAKE(1, 0, 0), base_hw_issues_tSIx_r1p0}, ++ {GPU_ID2_VERSION_MAKE(1, 1, 0), base_hw_issues_tSIx_r1p1}, ++ {U32_MAX, NULL} } }, ++ ++ {GPU_ID2_PRODUCT_TDVX, ++ {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tDVx_r0p0}, ++ {U32_MAX, NULL} } }, ++ ++ ++ ++ ++ }; ++ ++ u32 gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; ++ const u32 product_model = gpu_id & GPU_ID2_PRODUCT_MODEL; ++ const struct base_hw_product *product = NULL; ++ size_t p; ++ ++ /* Stop when we reach the end of the products array. */ ++ for (p = 0; p < ARRAY_SIZE(base_hw_products); ++p) { ++ if (product_model == base_hw_products[p].product_model) { ++ product = &base_hw_products[p]; ++ break; ++ } ++ } ++ ++ if (product != NULL) { ++ /* Found a matching product. */ ++ const u32 version = gpu_id & GPU_ID2_VERSION; ++#if !MALI_CUSTOMER_RELEASE ++ u32 fallback_version = 0; ++ const enum base_hw_issue *fallback_issues = NULL; ++#endif ++ size_t v; ++ ++ /* Stop when we reach the end of the map. */ ++ for (v = 0; product->map[v].version != U32_MAX; ++v) { ++ ++ if (version == product->map[v].version) { ++ /* Exact match so stop. */ ++ issues = product->map[v].issues; ++ break; ++ } ++ ++#if !MALI_CUSTOMER_RELEASE ++ /* Check whether this is a candidate for most recent ++ known version not later than the actual ++ version. */ ++ if ((version > product->map[v].version) && ++ (product->map[v].version >= fallback_version)) { ++ fallback_version = product->map[v].version; ++ fallback_issues = product->map[v].issues; ++ } ++#endif ++ } ++ ++#if !MALI_CUSTOMER_RELEASE ++ if ((issues == NULL) && (fallback_issues != NULL)) { ++ /* Fall back to the issue set of the most recent known ++ version not later than the actual version. */ ++ issues = fallback_issues; ++ ++ dev_info(kbdev->dev, ++ "r%dp%d status %d is unknown; treating as r%dp%d status %d", ++ (gpu_id & GPU_ID2_VERSION_MAJOR) >> ++ GPU_ID2_VERSION_MAJOR_SHIFT, ++ (gpu_id & GPU_ID2_VERSION_MINOR) >> ++ GPU_ID2_VERSION_MINOR_SHIFT, ++ (gpu_id & GPU_ID2_VERSION_STATUS) >> ++ GPU_ID2_VERSION_STATUS_SHIFT, ++ (fallback_version & GPU_ID2_VERSION_MAJOR) >> ++ GPU_ID2_VERSION_MAJOR_SHIFT, ++ (fallback_version & GPU_ID2_VERSION_MINOR) >> ++ GPU_ID2_VERSION_MINOR_SHIFT, ++ (fallback_version & GPU_ID2_VERSION_STATUS) >> ++ GPU_ID2_VERSION_STATUS_SHIFT); ++ ++ gpu_id &= ~GPU_ID2_VERSION; ++ gpu_id |= fallback_version; ++ kbdev->gpu_props.props.raw_props.gpu_id = gpu_id; ++ ++ kbase_gpuprops_update_core_props_gpu_id( ++ &kbdev->gpu_props.props); ++ } ++#endif ++ } ++ return issues; ++} ++ ++int kbase_hw_set_issues_mask(struct kbase_device *kbdev) ++{ ++ const enum base_hw_issue *issues; ++ u32 gpu_id; ++ u32 product_id; ++ u32 impl_tech; ++ ++ gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; ++ product_id = gpu_id & GPU_ID_VERSION_PRODUCT_ID; ++ product_id >>= GPU_ID_VERSION_PRODUCT_ID_SHIFT; ++ impl_tech = kbdev->gpu_props.props.thread_props.impl_tech; ++ ++ if (impl_tech != IMPLEMENTATION_MODEL) { ++ if (GPU_ID_IS_NEW_FORMAT(product_id)) { ++ issues = kbase_hw_get_issues_for_new_id(kbdev); ++ if (issues == NULL) { ++ dev_err(kbdev->dev, ++ "Unknown GPU ID %x", gpu_id); ++ return -EINVAL; ++ } ++ ++#if !MALI_CUSTOMER_RELEASE ++ /* The GPU ID might have been replaced with the last ++ known version of the same GPU. */ ++ gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; ++#endif ++ ++ } else { ++ switch (gpu_id) { ++ case GPU_ID_MAKE(GPU_ID_PI_T60X, 0, 0, GPU_ID_S_15DEV0): ++ issues = base_hw_issues_t60x_r0p0_15dev0; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_T60X, 0, 0, GPU_ID_S_EAC): ++ issues = base_hw_issues_t60x_r0p0_eac; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_T60X, 0, 1, 0): ++ issues = base_hw_issues_t60x_r0p1; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_T62X, 0, 1, 0): ++ issues = base_hw_issues_t62x_r0p1; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_T62X, 1, 0, 0): ++ case GPU_ID_MAKE(GPU_ID_PI_T62X, 1, 0, 1): ++ issues = base_hw_issues_t62x_r1p0; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_T62X, 1, 1, 0): ++ issues = base_hw_issues_t62x_r1p1; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_T76X, 0, 0, 1): ++ issues = base_hw_issues_t76x_r0p0; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_T76X, 0, 1, 1): ++ issues = base_hw_issues_t76x_r0p1; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_T76X, 0, 1, 9): ++ issues = base_hw_issues_t76x_r0p1_50rel0; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_T76X, 0, 2, 1): ++ issues = base_hw_issues_t76x_r0p2; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_T76X, 0, 3, 1): ++ issues = base_hw_issues_t76x_r0p3; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_T76X, 1, 0, 0): ++ issues = base_hw_issues_t76x_r1p0; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_T72X, 0, 0, 0): ++ case GPU_ID_MAKE(GPU_ID_PI_T72X, 0, 0, 1): ++ case GPU_ID_MAKE(GPU_ID_PI_T72X, 0, 0, 2): ++ issues = base_hw_issues_t72x_r0p0; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_T72X, 1, 0, 0): ++ issues = base_hw_issues_t72x_r1p0; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_T72X, 1, 1, 0): ++ issues = base_hw_issues_t72x_r1p1; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_TFRX, 0, 1, 2): ++ issues = base_hw_issues_tFRx_r0p1; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_TFRX, 0, 2, 0): ++ issues = base_hw_issues_tFRx_r0p2; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_TFRX, 1, 0, 0): ++ case GPU_ID_MAKE(GPU_ID_PI_TFRX, 1, 0, 8): ++ issues = base_hw_issues_tFRx_r1p0; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_TFRX, 2, 0, 0): ++ issues = base_hw_issues_tFRx_r2p0; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_T86X, 0, 2, 0): ++ issues = base_hw_issues_t86x_r0p2; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_T86X, 1, 0, 0): ++ case GPU_ID_MAKE(GPU_ID_PI_T86X, 1, 0, 8): ++ issues = base_hw_issues_t86x_r1p0; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_T86X, 2, 0, 0): ++ issues = base_hw_issues_t86x_r2p0; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_T83X, 0, 1, 0): ++ issues = base_hw_issues_t83x_r0p1; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_T83X, 1, 0, 0): ++ case GPU_ID_MAKE(GPU_ID_PI_T83X, 1, 0, 8): ++ issues = base_hw_issues_t83x_r1p0; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_T82X, 0, 0, 0): ++ issues = base_hw_issues_t82x_r0p0; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_T82X, 0, 1, 0): ++ issues = base_hw_issues_t82x_r0p1; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_T82X, 1, 0, 0): ++ case GPU_ID_MAKE(GPU_ID_PI_T82X, 1, 0, 8): ++ issues = base_hw_issues_t82x_r1p0; ++ break; ++ default: ++ dev_err(kbdev->dev, ++ "Unknown GPU ID %x", gpu_id); ++ return -EINVAL; ++ } ++ } ++ } else { ++ /* Software model */ ++ if (GPU_ID_IS_NEW_FORMAT(product_id)) { ++ switch (gpu_id & GPU_ID2_PRODUCT_MODEL) { ++ case GPU_ID2_PRODUCT_TMIX: ++ issues = base_hw_issues_model_tMIx; ++ break; ++ case GPU_ID2_PRODUCT_THEX: ++ issues = base_hw_issues_model_tHEx; ++ break; ++ case GPU_ID2_PRODUCT_TSIX: ++ issues = base_hw_issues_model_tSIx; ++ break; ++ case GPU_ID2_PRODUCT_TDVX: ++ issues = base_hw_issues_model_tDVx; ++ break; ++ default: ++ dev_err(kbdev->dev, ++ "Unknown GPU ID %x", gpu_id); ++ return -EINVAL; ++ } ++ } else { ++ switch (product_id) { ++ case GPU_ID_PI_T60X: ++ issues = base_hw_issues_model_t60x; ++ break; ++ case GPU_ID_PI_T62X: ++ issues = base_hw_issues_model_t62x; ++ break; ++ case GPU_ID_PI_T72X: ++ issues = base_hw_issues_model_t72x; ++ break; ++ case GPU_ID_PI_T76X: ++ issues = base_hw_issues_model_t76x; ++ break; ++ case GPU_ID_PI_TFRX: ++ issues = base_hw_issues_model_tFRx; ++ break; ++ case GPU_ID_PI_T86X: ++ issues = base_hw_issues_model_t86x; ++ break; ++ case GPU_ID_PI_T83X: ++ issues = base_hw_issues_model_t83x; ++ break; ++ case GPU_ID_PI_T82X: ++ issues = base_hw_issues_model_t82x; ++ break; ++ default: ++ dev_err(kbdev->dev, "Unknown GPU ID %x", ++ gpu_id); ++ return -EINVAL; ++ } ++ } ++ } ++ ++ if (GPU_ID_IS_NEW_FORMAT(product_id)) { ++ dev_info(kbdev->dev, ++ "GPU identified as 0x%x arch %d.%d.%d r%dp%d status %d", ++ (gpu_id & GPU_ID2_PRODUCT_MAJOR) >> ++ GPU_ID2_PRODUCT_MAJOR_SHIFT, ++ (gpu_id & GPU_ID2_ARCH_MAJOR) >> ++ GPU_ID2_ARCH_MAJOR_SHIFT, ++ (gpu_id & GPU_ID2_ARCH_MINOR) >> ++ GPU_ID2_ARCH_MINOR_SHIFT, ++ (gpu_id & GPU_ID2_ARCH_REV) >> ++ GPU_ID2_ARCH_REV_SHIFT, ++ (gpu_id & GPU_ID2_VERSION_MAJOR) >> ++ GPU_ID2_VERSION_MAJOR_SHIFT, ++ (gpu_id & GPU_ID2_VERSION_MINOR) >> ++ GPU_ID2_VERSION_MINOR_SHIFT, ++ (gpu_id & GPU_ID2_VERSION_STATUS) >> ++ GPU_ID2_VERSION_STATUS_SHIFT); ++ } else { ++ dev_info(kbdev->dev, ++ "GPU identified as 0x%04x r%dp%d status %d", ++ (gpu_id & GPU_ID_VERSION_PRODUCT_ID) >> ++ GPU_ID_VERSION_PRODUCT_ID_SHIFT, ++ (gpu_id & GPU_ID_VERSION_MAJOR) >> ++ GPU_ID_VERSION_MAJOR_SHIFT, ++ (gpu_id & GPU_ID_VERSION_MINOR) >> ++ GPU_ID_VERSION_MINOR_SHIFT, ++ (gpu_id & GPU_ID_VERSION_STATUS) >> ++ GPU_ID_VERSION_STATUS_SHIFT); ++ } ++ ++ for (; *issues != BASE_HW_ISSUE_END; issues++) ++ set_bit(*issues, &kbdev->hw_issues_mask[0]); ++ ++ return 0; ++} +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hw.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hw.h +new file mode 100755 +index 000000000000..754250ce968d +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hw.h +@@ -0,0 +1,65 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/** ++ * @file ++ * Run-time work-arounds helpers ++ */ ++ ++#ifndef _KBASE_HW_H_ ++#define _KBASE_HW_H_ ++ ++#include "mali_kbase_defs.h" ++ ++/** ++ * @brief Tell whether a work-around should be enabled ++ */ ++#define kbase_hw_has_issue(kbdev, issue)\ ++ test_bit(issue, &(kbdev)->hw_issues_mask[0]) ++ ++/** ++ * @brief Tell whether a feature is supported ++ */ ++#define kbase_hw_has_feature(kbdev, feature)\ ++ test_bit(feature, &(kbdev)->hw_features_mask[0]) ++ ++/** ++ * kbase_hw_set_issues_mask - Set the hardware issues mask based on the GPU ID ++ * @kbdev: Device pointer ++ * ++ * Return: 0 if the GPU ID was recognized, otherwise -EINVAL. ++ * ++ * The GPU ID is read from the @kbdev. ++ * ++ * In debugging versions of the driver, unknown versions of a known GPU with a ++ * new-format ID will be treated as the most recent known version not later ++ * than the actual version. In such circumstances, the GPU ID in @kbdev will ++ * also be replaced with the most recent known version. ++ * ++ * Note: The GPU configuration must have been read by ++ * kbase_gpuprops_get_props() before calling this function. ++ */ ++int kbase_hw_set_issues_mask(struct kbase_device *kbdev); ++ ++/** ++ * @brief Set the features mask depending on the GPU ID ++ */ ++void kbase_hw_set_features_mask(struct kbase_device *kbdev); ++ ++#endif /* _KBASE_HW_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_backend.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_backend.h +new file mode 100755 +index 000000000000..b09be99e6b4e +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_backend.h +@@ -0,0 +1,54 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++/* ++ * HW access backend common APIs ++ */ ++ ++#ifndef _KBASE_HWACCESS_BACKEND_H_ ++#define _KBASE_HWACCESS_BACKEND_H_ ++ ++/** ++ * kbase_backend_early_init - Perform any backend-specific initialization. ++ * @kbdev: Device pointer ++ * ++ * Return: 0 on success, or an error code on failure. ++ */ ++int kbase_backend_early_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_backend_late_init - Perform any backend-specific initialization. ++ * @kbdev: Device pointer ++ * ++ * Return: 0 on success, or an error code on failure. ++ */ ++int kbase_backend_late_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_backend_early_term - Perform any backend-specific termination. ++ * @kbdev: Device pointer ++ */ ++void kbase_backend_early_term(struct kbase_device *kbdev); ++ ++/** ++ * kbase_backend_late_term - Perform any backend-specific termination. ++ * @kbdev: Device pointer ++ */ ++void kbase_backend_late_term(struct kbase_device *kbdev); ++ ++#endif /* _KBASE_HWACCESS_BACKEND_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_defs.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_defs.h +new file mode 100755 +index 000000000000..0acf297192fd +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_defs.h +@@ -0,0 +1,36 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014, 2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++/** ++ * @file mali_kbase_hwaccess_gpu_defs.h ++ * HW access common definitions ++ */ ++ ++#ifndef _KBASE_HWACCESS_DEFS_H_ ++#define _KBASE_HWACCESS_DEFS_H_ ++ ++#include ++ ++/* The hwaccess_lock (a spinlock) must be held when accessing this structure */ ++struct kbase_hwaccess_data { ++ struct kbase_context *active_kctx; ++ ++ struct kbase_backend_data backend; ++}; ++ ++#endif /* _KBASE_HWACCESS_DEFS_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_gpuprops.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_gpuprops.h +new file mode 100755 +index 000000000000..cf8a8131c22e +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_gpuprops.h +@@ -0,0 +1,47 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++/** ++ * Base kernel property query backend APIs ++ */ ++ ++#ifndef _KBASE_HWACCESS_GPUPROPS_H_ ++#define _KBASE_HWACCESS_GPUPROPS_H_ ++ ++/** ++ * kbase_backend_gpuprops_get() - Fill @regdump with GPU properties read from ++ * GPU ++ * @kbdev: Device pointer ++ * @regdump: Pointer to struct kbase_gpuprops_regdump structure ++ */ ++void kbase_backend_gpuprops_get(struct kbase_device *kbdev, ++ struct kbase_gpuprops_regdump *regdump); ++ ++/** ++ * kbase_backend_gpuprops_get - Fill @regdump with GPU properties read from GPU ++ * @kbdev: Device pointer ++ * @regdump: Pointer to struct kbase_gpuprops_regdump structure ++ * ++ * This function reads GPU properties that are dependent on the hardware ++ * features bitmask ++ */ ++void kbase_backend_gpuprops_get_features(struct kbase_device *kbdev, ++ struct kbase_gpuprops_regdump *regdump); ++ ++ ++#endif /* _KBASE_HWACCESS_GPUPROPS_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_instr.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_instr.h +new file mode 100755 +index 000000000000..5de2b7535bb4 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_instr.h +@@ -0,0 +1,116 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * HW Access instrumentation common APIs ++ */ ++ ++#ifndef _KBASE_HWACCESS_INSTR_H_ ++#define _KBASE_HWACCESS_INSTR_H_ ++ ++#include ++ ++/** ++ * kbase_instr_hwcnt_enable_internal - Enable HW counters collection ++ * @kbdev: Kbase device ++ * @kctx: Kbase context ++ * @setup: HW counter setup parameters ++ * ++ * Context: might sleep, waiting for reset to complete ++ * ++ * Return: 0 on success ++ */ ++int kbase_instr_hwcnt_enable_internal(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ struct kbase_uk_hwcnt_setup *setup); ++ ++/** ++ * kbase_instr_hwcnt_disable_internal - Disable HW counters collection ++ * @kctx: Kbase context ++ * ++ * Context: might sleep, waiting for an ongoing dump to complete ++ * ++ * Return: 0 on success ++ */ ++int kbase_instr_hwcnt_disable_internal(struct kbase_context *kctx); ++ ++/** ++ * kbase_instr_hwcnt_request_dump() - Request HW counter dump from GPU ++ * @kctx: Kbase context ++ * ++ * Caller must either wait for kbase_instr_hwcnt_dump_complete() to return true, ++ * of call kbase_instr_hwcnt_wait_for_dump(). ++ * ++ * Return: 0 on success ++ */ ++int kbase_instr_hwcnt_request_dump(struct kbase_context *kctx); ++ ++/** ++ * kbase_instr_hwcnt_wait_for_dump() - Wait until pending HW counter dump has ++ * completed. ++ * @kctx: Kbase context ++ * ++ * Context: will sleep, waiting for dump to complete ++ * ++ * Return: 0 on success ++ */ ++int kbase_instr_hwcnt_wait_for_dump(struct kbase_context *kctx); ++ ++/** ++ * kbase_instr_hwcnt_dump_complete - Tell whether the HW counters dump has ++ * completed ++ * @kctx: Kbase context ++ * @success: Set to true if successful ++ * ++ * Context: does not sleep. ++ * ++ * Return: true if the dump is complete ++ */ ++bool kbase_instr_hwcnt_dump_complete(struct kbase_context *kctx, ++ bool * const success); ++ ++/** ++ * kbase_instr_hwcnt_clear() - Clear HW counters ++ * @kctx: Kbase context ++ * ++ * Context: might sleep, waiting for reset to complete ++ * ++ * Return: 0 on success ++ */ ++int kbase_instr_hwcnt_clear(struct kbase_context *kctx); ++ ++/** ++ * kbase_instr_backend_init() - Initialise the instrumentation backend ++ * @kbdev: Kbase device ++ * ++ * This function should be called during driver initialization. ++ * ++ * Return: 0 on success ++ */ ++int kbase_instr_backend_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_instr_backend_init() - Terminate the instrumentation backend ++ * @kbdev: Kbase device ++ * ++ * This function should be called during driver termination. ++ */ ++void kbase_instr_backend_term(struct kbase_device *kbdev); ++ ++#endif /* _KBASE_HWACCESS_INSTR_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_jm.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_jm.h +new file mode 100755 +index 000000000000..750fda2cd81d +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_jm.h +@@ -0,0 +1,381 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++/* ++ * HW access job manager common APIs ++ */ ++ ++#ifndef _KBASE_HWACCESS_JM_H_ ++#define _KBASE_HWACCESS_JM_H_ ++ ++/** ++ * kbase_backend_run_atom() - Run an atom on the GPU ++ * @kbdev: Device pointer ++ * @atom: Atom to run ++ * ++ * Caller must hold the HW access lock ++ */ ++void kbase_backend_run_atom(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom); ++ ++/** ++ * kbase_backend_slot_update - Update state based on slot ringbuffers ++ * ++ * @kbdev: Device pointer ++ * ++ * Inspect the jobs in the slot ringbuffers and update state. ++ * ++ * This will cause jobs to be submitted to hardware if they are unblocked ++ */ ++void kbase_backend_slot_update(struct kbase_device *kbdev); ++ ++/** ++ * kbase_backend_find_and_release_free_address_space() - Release a free AS ++ * @kbdev: Device pointer ++ * @kctx: Context pointer ++ * ++ * This function can evict an idle context from the runpool, freeing up the ++ * address space it was using. ++ * ++ * The address space is marked as in use. The caller must either assign a ++ * context using kbase_gpu_use_ctx(), or release it using ++ * kbase_ctx_sched_release() ++ * ++ * Return: Number of free address space, or KBASEP_AS_NR_INVALID if none ++ * available ++ */ ++int kbase_backend_find_and_release_free_address_space( ++ struct kbase_device *kbdev, struct kbase_context *kctx); ++ ++/** ++ * kbase_backend_use_ctx() - Activate a currently unscheduled context, using the ++ * provided address space. ++ * @kbdev: Device pointer ++ * @kctx: Context pointer. May be NULL ++ * @as_nr: Free address space to use ++ * ++ * kbase_gpu_next_job() will pull atoms from the active context. ++ * ++ * Return: true if successful, false if ASID not assigned. ++ */ ++bool kbase_backend_use_ctx(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ int as_nr); ++ ++/** ++ * kbase_backend_use_ctx_sched() - Activate a context. ++ * @kbdev: Device pointer ++ * @kctx: Context pointer ++ * ++ * kbase_gpu_next_job() will pull atoms from the active context. ++ * ++ * The context must already be scheduled and assigned to an address space. If ++ * the context is not scheduled, then kbase_gpu_use_ctx() should be used ++ * instead. ++ * ++ * Caller must hold hwaccess_lock ++ * ++ * Return: true if context is now active, false otherwise (ie if context does ++ * not have an address space assigned) ++ */ ++bool kbase_backend_use_ctx_sched(struct kbase_device *kbdev, ++ struct kbase_context *kctx); ++ ++/** ++ * kbase_backend_release_ctx_irq - Release a context from the GPU. This will ++ * de-assign the assigned address space. ++ * @kbdev: Device pointer ++ * @kctx: Context pointer ++ * ++ * Caller must hold kbase_device->mmu_hw_mutex and hwaccess_lock ++ */ ++void kbase_backend_release_ctx_irq(struct kbase_device *kbdev, ++ struct kbase_context *kctx); ++ ++/** ++ * kbase_backend_release_ctx_noirq - Release a context from the GPU. This will ++ * de-assign the assigned address space. ++ * @kbdev: Device pointer ++ * @kctx: Context pointer ++ * ++ * Caller must hold kbase_device->mmu_hw_mutex ++ * ++ * This function must perform any operations that could not be performed in IRQ ++ * context by kbase_backend_release_ctx_irq(). ++ */ ++void kbase_backend_release_ctx_noirq(struct kbase_device *kbdev, ++ struct kbase_context *kctx); ++ ++/** ++ * kbase_backend_cacheclean - Perform a cache clean if the given atom requires ++ * one ++ * @kbdev: Device pointer ++ * @katom: Pointer to the failed atom ++ * ++ * On some GPUs, the GPU cache must be cleaned following a failed atom. This ++ * function performs a clean if it is required by @katom. ++ */ ++void kbase_backend_cacheclean(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom); ++ ++ ++/** ++ * kbase_backend_complete_wq() - Perform backend-specific actions required on ++ * completing an atom. ++ * @kbdev: Device pointer ++ * @katom: Pointer to the atom to complete ++ * ++ * This function should only be called from kbase_jd_done_worker() or ++ * js_return_worker(). ++ * ++ * Return: true if atom has completed, false if atom should be re-submitted ++ */ ++void kbase_backend_complete_wq(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom); ++ ++/** ++ * kbase_backend_complete_wq_post_sched - Perform backend-specific actions ++ * required on completing an atom, after ++ * any scheduling has taken place. ++ * @kbdev: Device pointer ++ * @core_req: Core requirements of atom ++ * @affinity: Affinity of atom ++ * @coreref_state: Coreref state of atom ++ * ++ * This function should only be called from kbase_jd_done_worker() or ++ * js_return_worker(). ++ */ ++void kbase_backend_complete_wq_post_sched(struct kbase_device *kbdev, ++ base_jd_core_req core_req, u64 affinity, ++ enum kbase_atom_coreref_state coreref_state); ++ ++/** ++ * kbase_backend_reset() - The GPU is being reset. Cancel all jobs on the GPU ++ * and remove any others from the ringbuffers. ++ * @kbdev: Device pointer ++ * @end_timestamp: Timestamp of reset ++ */ ++void kbase_backend_reset(struct kbase_device *kbdev, ktime_t *end_timestamp); ++ ++/** ++ * kbase_backend_inspect_head() - Return the atom currently at the head of slot ++ * @js ++ * @kbdev: Device pointer ++ * @js: Job slot to inspect ++ * ++ * Return : Atom currently at the head of slot @js, or NULL ++ */ ++struct kbase_jd_atom *kbase_backend_inspect_head(struct kbase_device *kbdev, ++ int js); ++ ++/** ++ * kbase_backend_inspect_tail - Return the atom currently at the tail of slot ++ * @js ++ * @kbdev: Device pointer ++ * @js: Job slot to inspect ++ * ++ * Return : Atom currently at the head of slot @js, or NULL ++ */ ++struct kbase_jd_atom *kbase_backend_inspect_tail(struct kbase_device *kbdev, ++ int js); ++ ++/** ++ * kbase_backend_nr_atoms_on_slot() - Return the number of atoms currently on a ++ * slot. ++ * @kbdev: Device pointer ++ * @js: Job slot to inspect ++ * ++ * Return : Number of atoms currently on slot ++ */ ++int kbase_backend_nr_atoms_on_slot(struct kbase_device *kbdev, int js); ++ ++/** ++ * kbase_backend_nr_atoms_submitted() - Return the number of atoms on a slot ++ * that are currently on the GPU. ++ * @kbdev: Device pointer ++ * @js: Job slot to inspect ++ * ++ * Return : Number of atoms currently on slot @js that are currently on the GPU. ++ */ ++int kbase_backend_nr_atoms_submitted(struct kbase_device *kbdev, int js); ++ ++/** ++ * kbase_backend_ctx_count_changed() - Number of contexts ready to submit jobs ++ * has changed. ++ * @kbdev: Device pointer ++ * ++ * Perform any required backend-specific actions (eg starting/stopping ++ * scheduling timers). ++ */ ++void kbase_backend_ctx_count_changed(struct kbase_device *kbdev); ++ ++/** ++ * kbase_backend_timeouts_changed() - Job Scheduler timeouts have changed. ++ * @kbdev: Device pointer ++ * ++ * Perform any required backend-specific actions (eg updating timeouts of ++ * currently running atoms). ++ */ ++void kbase_backend_timeouts_changed(struct kbase_device *kbdev); ++ ++/** ++ * kbase_backend_slot_free() - Return the number of jobs that can be currently ++ * submitted to slot @js. ++ * @kbdev: Device pointer ++ * @js: Job slot to inspect ++ * ++ * Return : Number of jobs that can be submitted. ++ */ ++int kbase_backend_slot_free(struct kbase_device *kbdev, int js); ++ ++/** ++ * kbase_job_check_enter_disjoint - potentially leave disjoint state ++ * @kbdev: kbase device ++ * @target_katom: atom which is finishing ++ * ++ * Work out whether to leave disjoint state when finishing an atom that was ++ * originated by kbase_job_check_enter_disjoint(). ++ */ ++void kbase_job_check_leave_disjoint(struct kbase_device *kbdev, ++ struct kbase_jd_atom *target_katom); ++ ++/** ++ * kbase_backend_jm_kill_jobs_from_kctx - Kill all jobs that are currently ++ * running from a context ++ * @kctx: Context pointer ++ * ++ * This is used in response to a page fault to remove all jobs from the faulting ++ * context from the hardware. ++ */ ++void kbase_backend_jm_kill_jobs_from_kctx(struct kbase_context *kctx); ++ ++/** ++ * kbase_jm_wait_for_zero_jobs - Wait for context to have zero jobs running, and ++ * to be descheduled. ++ * @kctx: Context pointer ++ * ++ * This should be called following kbase_js_zap_context(), to ensure the context ++ * can be safely destroyed. ++ */ ++void kbase_jm_wait_for_zero_jobs(struct kbase_context *kctx); ++ ++/** ++ * kbase_backend_get_current_flush_id - Return the current flush ID ++ * ++ * @kbdev: Device pointer ++ * ++ * Return: the current flush ID to be recorded for each job chain ++ */ ++u32 kbase_backend_get_current_flush_id(struct kbase_device *kbdev); ++ ++#if KBASE_GPU_RESET_EN ++/** ++ * kbase_prepare_to_reset_gpu - Prepare for resetting the GPU. ++ * @kbdev: Device pointer ++ * ++ * This function just soft-stops all the slots to ensure that as many jobs as ++ * possible are saved. ++ * ++ * Return: a boolean which should be interpreted as follows: ++ * - true - Prepared for reset, kbase_reset_gpu should be called. ++ * - false - Another thread is performing a reset, kbase_reset_gpu should ++ * not be called. ++ */ ++bool kbase_prepare_to_reset_gpu(struct kbase_device *kbdev); ++ ++/** ++ * kbase_reset_gpu - Reset the GPU ++ * @kbdev: Device pointer ++ * ++ * This function should be called after kbase_prepare_to_reset_gpu if it returns ++ * true. It should never be called without a corresponding call to ++ * kbase_prepare_to_reset_gpu. ++ * ++ * After this function is called (or not called if kbase_prepare_to_reset_gpu ++ * returned false), the caller should wait for kbdev->reset_waitq to be ++ * signalled to know when the reset has completed. ++ */ ++void kbase_reset_gpu(struct kbase_device *kbdev); ++ ++/** ++ * kbase_prepare_to_reset_gpu_locked - Prepare for resetting the GPU. ++ * @kbdev: Device pointer ++ * ++ * This function just soft-stops all the slots to ensure that as many jobs as ++ * possible are saved. ++ * ++ * Return: a boolean which should be interpreted as follows: ++ * - true - Prepared for reset, kbase_reset_gpu should be called. ++ * - false - Another thread is performing a reset, kbase_reset_gpu should ++ * not be called. ++ */ ++bool kbase_prepare_to_reset_gpu_locked(struct kbase_device *kbdev); ++ ++/** ++ * kbase_reset_gpu_locked - Reset the GPU ++ * @kbdev: Device pointer ++ * ++ * This function should be called after kbase_prepare_to_reset_gpu if it ++ * returns true. It should never be called without a corresponding call to ++ * kbase_prepare_to_reset_gpu. ++ * ++ * After this function is called (or not called if kbase_prepare_to_reset_gpu ++ * returned false), the caller should wait for kbdev->reset_waitq to be ++ * signalled to know when the reset has completed. ++ */ ++void kbase_reset_gpu_locked(struct kbase_device *kbdev); ++ ++/** ++ * kbase_reset_gpu_silent - Reset the GPU silently ++ * @kbdev: Device pointer ++ * ++ * Reset the GPU without trying to cancel jobs and don't emit messages into ++ * the kernel log while doing the reset. ++ * ++ * This function should be used in cases where we are doing a controlled reset ++ * of the GPU as part of normal processing (e.g. exiting protected mode) where ++ * the driver will have ensured the scheduler has been idled and all other ++ * users of the GPU (e.g. instrumentation) have been suspended. ++ */ ++void kbase_reset_gpu_silent(struct kbase_device *kbdev); ++ ++/** ++ * kbase_reset_gpu_active - Reports if the GPU is being reset ++ * @kbdev: Device pointer ++ * ++ * Return: True if the GPU is in the process of being reset. ++ */ ++bool kbase_reset_gpu_active(struct kbase_device *kbdev); ++#endif ++ ++/** ++ * kbase_job_slot_hardstop - Hard-stop the specified job slot ++ * @kctx: The kbase context that contains the job(s) that should ++ * be hard-stopped ++ * @js: The job slot to hard-stop ++ * @target_katom: The job that should be hard-stopped (or NULL for all ++ * jobs from the context) ++ * Context: ++ * The job slot lock must be held when calling this function. ++ */ ++void kbase_job_slot_hardstop(struct kbase_context *kctx, int js, ++ struct kbase_jd_atom *target_katom); ++ ++extern struct protected_mode_ops kbase_native_protected_ops; ++ ++#endif /* _KBASE_HWACCESS_JM_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_pm.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_pm.h +new file mode 100755 +index 000000000000..71c7d495c40a +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_pm.h +@@ -0,0 +1,209 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++/** ++ * @file mali_kbase_hwaccess_pm.h ++ * HW access power manager common APIs ++ */ ++ ++#ifndef _KBASE_HWACCESS_PM_H_ ++#define _KBASE_HWACCESS_PM_H_ ++ ++#include ++#include ++ ++#include ++ ++/* Forward definition - see mali_kbase.h */ ++struct kbase_device; ++ ++/* Functions common to all HW access backends */ ++ ++/** ++ * Initialize the power management framework. ++ * ++ * Must be called before any other power management function ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid ++ * pointer) ++ * ++ * @return 0 if the power management framework was successfully ++ * initialized. ++ */ ++int kbase_hwaccess_pm_init(struct kbase_device *kbdev); ++ ++/** ++ * Terminate the power management framework. ++ * ++ * No power management functions may be called after this (except ++ * @ref kbase_pm_init) ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid ++ * pointer) ++ */ ++void kbase_hwaccess_pm_term(struct kbase_device *kbdev); ++ ++/** ++ * kbase_hwaccess_pm_powerup - Power up the GPU. ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * @flags: Flags to pass on to kbase_pm_init_hw ++ * ++ * Power up GPU after all modules have been initialized and interrupt handlers ++ * installed. ++ * ++ * Return: 0 if powerup was successful. ++ */ ++int kbase_hwaccess_pm_powerup(struct kbase_device *kbdev, ++ unsigned int flags); ++ ++/** ++ * Halt the power management framework. ++ * ++ * Should ensure that no new interrupts are generated, but allow any currently ++ * running interrupt handlers to complete successfully. The GPU is forced off by ++ * the time this function returns, regardless of whether or not the active power ++ * policy asks for the GPU to be powered off. ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid ++ * pointer) ++ */ ++void kbase_hwaccess_pm_halt(struct kbase_device *kbdev); ++ ++/** ++ * Perform any backend-specific actions to suspend the GPU ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid ++ * pointer) ++ */ ++void kbase_hwaccess_pm_suspend(struct kbase_device *kbdev); ++ ++/** ++ * Perform any backend-specific actions to resume the GPU from a suspend ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid ++ * pointer) ++ */ ++void kbase_hwaccess_pm_resume(struct kbase_device *kbdev); ++ ++/** ++ * Perform any required actions for activating the GPU. Called when the first ++ * context goes active. ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid ++ * pointer) ++ */ ++void kbase_hwaccess_pm_gpu_active(struct kbase_device *kbdev); ++ ++/** ++ * Perform any required actions for idling the GPU. Called when the last ++ * context goes idle. ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid ++ * pointer) ++ */ ++void kbase_hwaccess_pm_gpu_idle(struct kbase_device *kbdev); ++ ++ ++/** ++ * Set the debug core mask. ++ * ++ * This determines which cores the power manager is allowed to use. ++ * ++ * @param kbdev The kbase device structure for the device (must be a ++ * valid pointer) ++ * @param new_core_mask_js0 The core mask to use for job slot 0 ++ * @param new_core_mask_js0 The core mask to use for job slot 1 ++ * @param new_core_mask_js0 The core mask to use for job slot 2 ++ */ ++void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev, ++ u64 new_core_mask_js0, u64 new_core_mask_js1, ++ u64 new_core_mask_js2); ++ ++ ++/** ++ * Get the current policy. ++ * ++ * Returns the policy that is currently active. ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid ++ * pointer) ++ * ++ * @return The current policy ++ */ ++const struct kbase_pm_ca_policy ++*kbase_pm_ca_get_policy(struct kbase_device *kbdev); ++ ++/** ++ * Change the policy to the one specified. ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid ++ * pointer) ++ * @param policy The policy to change to (valid pointer returned from ++ * @ref kbase_pm_ca_list_policies) ++ */ ++void kbase_pm_ca_set_policy(struct kbase_device *kbdev, ++ const struct kbase_pm_ca_policy *policy); ++ ++/** ++ * Retrieve a static list of the available policies. ++ * ++ * @param[out] policies An array pointer to take the list of policies. This may ++ * be NULL. The contents of this array must not be ++ * modified. ++ * ++ * @return The number of policies ++ */ ++int ++kbase_pm_ca_list_policies(const struct kbase_pm_ca_policy * const **policies); ++ ++ ++/** ++ * Get the current policy. ++ * ++ * Returns the policy that is currently active. ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid ++ * pointer) ++ * ++ * @return The current policy ++ */ ++const struct kbase_pm_policy *kbase_pm_get_policy(struct kbase_device *kbdev); ++ ++/** ++ * Change the policy to the one specified. ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid ++ * pointer) ++ * @param policy The policy to change to (valid pointer returned from ++ * @ref kbase_pm_list_policies) ++ */ ++void kbase_pm_set_policy(struct kbase_device *kbdev, ++ const struct kbase_pm_policy *policy); ++ ++/** ++ * Retrieve a static list of the available policies. ++ * ++ * @param[out] policies An array pointer to take the list of policies. This may ++ * be NULL. The contents of this array must not be ++ * modified. ++ * ++ * @return The number of policies ++ */ ++int kbase_pm_list_policies(const struct kbase_pm_policy * const **policies); ++ ++#endif /* _KBASE_HWACCESS_PM_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_time.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_time.h +new file mode 100755 +index 000000000000..b9fe8e669c63 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_time.h +@@ -0,0 +1,53 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++/** ++ * ++ */ ++ ++#ifndef _KBASE_BACKEND_TIME_H_ ++#define _KBASE_BACKEND_TIME_H_ ++ ++/** ++ * kbase_backend_get_gpu_time() - Get current GPU time ++ * @kbdev: Device pointer ++ * @cycle_counter: Pointer to u64 to store cycle counter in ++ * @system_time: Pointer to u64 to store system time in ++ * @ts: Pointer to struct timespec to store current monotonic ++ * time in ++ */ ++void kbase_backend_get_gpu_time(struct kbase_device *kbdev, u64 *cycle_counter, ++ u64 *system_time, struct timespec *ts); ++ ++/** ++ * kbase_wait_write_flush() - Wait for GPU write flush ++ * @kctx: Context pointer ++ * ++ * Wait 1000 GPU clock cycles. This delay is known to give the GPU time to flush ++ * its write buffer. ++ * ++ * If GPU resets occur then the counters are reset to zero, the delay may not be ++ * as expected. ++ * ++ * This function is only in use for BASE_HW_ISSUE_6367 ++ */ ++#ifndef CONFIG_MALI_BIFROST_NO_MALI ++void kbase_wait_write_flush(struct kbase_context *kctx); ++#endif ++ ++#endif /* _KBASE_BACKEND_TIME_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwcnt_reader.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwcnt_reader.h +new file mode 100755 +index 000000000000..cf7bf1b35dc5 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwcnt_reader.h +@@ -0,0 +1,66 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KBASE_HWCNT_READER_H_ ++#define _KBASE_HWCNT_READER_H_ ++ ++/* The ids of ioctl commands. */ ++#define KBASE_HWCNT_READER 0xBE ++#define KBASE_HWCNT_READER_GET_HWVER _IOR(KBASE_HWCNT_READER, 0x00, u32) ++#define KBASE_HWCNT_READER_GET_BUFFER_SIZE _IOR(KBASE_HWCNT_READER, 0x01, u32) ++#define KBASE_HWCNT_READER_DUMP _IOW(KBASE_HWCNT_READER, 0x10, u32) ++#define KBASE_HWCNT_READER_CLEAR _IOW(KBASE_HWCNT_READER, 0x11, u32) ++#define KBASE_HWCNT_READER_GET_BUFFER _IOR(KBASE_HWCNT_READER, 0x20,\ ++ struct kbase_hwcnt_reader_metadata) ++#define KBASE_HWCNT_READER_PUT_BUFFER _IOW(KBASE_HWCNT_READER, 0x21,\ ++ struct kbase_hwcnt_reader_metadata) ++#define KBASE_HWCNT_READER_SET_INTERVAL _IOW(KBASE_HWCNT_READER, 0x30, u32) ++#define KBASE_HWCNT_READER_ENABLE_EVENT _IOW(KBASE_HWCNT_READER, 0x40, u32) ++#define KBASE_HWCNT_READER_DISABLE_EVENT _IOW(KBASE_HWCNT_READER, 0x41, u32) ++#define KBASE_HWCNT_READER_GET_API_VERSION _IOW(KBASE_HWCNT_READER, 0xFF, u32) ++ ++/** ++ * struct kbase_hwcnt_reader_metadata - hwcnt reader sample buffer metadata ++ * @timestamp: time when sample was collected ++ * @event_id: id of an event that triggered sample collection ++ * @buffer_idx: position in sampling area where sample buffer was stored ++ */ ++struct kbase_hwcnt_reader_metadata { ++ u64 timestamp; ++ u32 event_id; ++ u32 buffer_idx; ++}; ++ ++/** ++ * enum base_hwcnt_reader_event - hwcnt dumping events ++ * @BASE_HWCNT_READER_EVENT_MANUAL: manual request for dump ++ * @BASE_HWCNT_READER_EVENT_PERIODIC: periodic dump ++ * @BASE_HWCNT_READER_EVENT_PREJOB: prejob dump request ++ * @BASE_HWCNT_READER_EVENT_POSTJOB: postjob dump request ++ * @BASE_HWCNT_READER_EVENT_COUNT: number of supported events ++ */ ++enum base_hwcnt_reader_event { ++ BASE_HWCNT_READER_EVENT_MANUAL, ++ BASE_HWCNT_READER_EVENT_PERIODIC, ++ BASE_HWCNT_READER_EVENT_PREJOB, ++ BASE_HWCNT_READER_EVENT_POSTJOB, ++ ++ BASE_HWCNT_READER_EVENT_COUNT ++}; ++ ++#endif /* _KBASE_HWCNT_READER_H_ */ ++ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_ioctl.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_ioctl.h +new file mode 100755 +index 000000000000..e7c1daee470b +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_ioctl.h +@@ -0,0 +1,658 @@ ++/* ++ * ++ * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KBASE_IOCTL_H_ ++#define _KBASE_IOCTL_H_ ++ ++#ifdef __cpluscplus ++extern "C" { ++#endif ++ ++#include ++ ++#define KBASE_IOCTL_TYPE 0x80 ++ ++#ifdef ANDROID ++/* Android's definition of ioctl is incorrect, specifying the type argument as ++ * 'int'. This creates a warning when using _IOWR (as the top bit is set). Work ++ * round this by redefining _IOC to include a case to 'int'. ++ */ ++#undef _IOC ++#define _IOC(dir, type, nr, size) \ ++ ((int)(((dir) << _IOC_DIRSHIFT) | ((type) << _IOC_TYPESHIFT) | \ ++ ((nr) << _IOC_NRSHIFT) | ((size) << _IOC_SIZESHIFT))) ++#endif ++ ++/** ++ * struct kbase_ioctl_version_check - Check version compatibility with kernel ++ * ++ * @major: Major version number ++ * @minor: Minor version number ++ */ ++struct kbase_ioctl_version_check { ++ __u16 major; ++ __u16 minor; ++}; ++ ++#define KBASE_IOCTL_VERSION_CHECK \ ++ _IOWR(KBASE_IOCTL_TYPE, 0, struct kbase_ioctl_version_check) ++ ++/** ++ * struct kbase_ioctl_set_flags - Set kernel context creation flags ++ * ++ * @create_flags: Flags - see base_context_create_flags ++ */ ++struct kbase_ioctl_set_flags { ++ __u32 create_flags; ++}; ++ ++#define KBASE_IOCTL_SET_FLAGS \ ++ _IOW(KBASE_IOCTL_TYPE, 1, struct kbase_ioctl_set_flags) ++ ++/** ++ * struct kbase_ioctl_job_submit - Submit jobs/atoms to the kernel ++ * ++ * @addr: Memory address of an array of struct base_jd_atom_v2 ++ * @nr_atoms: Number of entries in the array ++ * @stride: sizeof(struct base_jd_atom_v2) ++ */ ++struct kbase_ioctl_job_submit { ++ __u64 addr; ++ __u32 nr_atoms; ++ __u32 stride; ++}; ++ ++#define KBASE_IOCTL_JOB_SUBMIT \ ++ _IOW(KBASE_IOCTL_TYPE, 2, struct kbase_ioctl_job_submit) ++ ++/** ++ * struct kbase_ioctl_get_gpuprops - Read GPU properties from the kernel ++ * ++ * @buffer: Pointer to the buffer to store properties into ++ * @size: Size of the buffer ++ * @flags: Flags - must be zero for now ++ * ++ * The ioctl will return the number of bytes stored into @buffer or an error ++ * on failure (e.g. @size is too small). If @size is specified as 0 then no ++ * data will be written but the return value will be the number of bytes needed ++ * for all the properties. ++ * ++ * @flags may be used in the future to request a different format for the ++ * buffer. With @flags == 0 the following format is used. ++ * ++ * The buffer will be filled with pairs of values, a u32 key identifying the ++ * property followed by the value. The size of the value is identified using ++ * the bottom bits of the key. The value then immediately followed the key and ++ * is tightly packed (there is no padding). All keys and values are ++ * little-endian. ++ * ++ * 00 = u8 ++ * 01 = u16 ++ * 10 = u32 ++ * 11 = u64 ++ */ ++struct kbase_ioctl_get_gpuprops { ++ __u64 buffer; ++ __u32 size; ++ __u32 flags; ++}; ++ ++#define KBASE_IOCTL_GET_GPUPROPS \ ++ _IOW(KBASE_IOCTL_TYPE, 3, struct kbase_ioctl_get_gpuprops) ++ ++#define KBASE_IOCTL_POST_TERM \ ++ _IO(KBASE_IOCTL_TYPE, 4) ++ ++/** ++ * union kbase_ioctl_mem_alloc - Allocate memory on the GPU ++ * ++ * @va_pages: The number of pages of virtual address space to reserve ++ * @commit_pages: The number of physical pages to allocate ++ * @extent: The number of extra pages to allocate on each GPU fault which grows ++ * the region ++ * @flags: Flags ++ * @gpu_va: The GPU virtual address which is allocated ++ * ++ * @in: Input parameters ++ * @out: Output parameters ++ */ ++union kbase_ioctl_mem_alloc { ++ struct { ++ __u64 va_pages; ++ __u64 commit_pages; ++ __u64 extent; ++ __u64 flags; ++ } in; ++ struct { ++ __u64 flags; ++ __u64 gpu_va; ++ } out; ++}; ++ ++#define KBASE_IOCTL_MEM_ALLOC \ ++ _IOWR(KBASE_IOCTL_TYPE, 5, union kbase_ioctl_mem_alloc) ++ ++/** ++ * struct kbase_ioctl_mem_query - Query properties of a GPU memory region ++ * @gpu_addr: A GPU address contained within the region ++ * @query: The type of query ++ * @value: The result of the query ++ * ++ * Use a %KBASE_MEM_QUERY_xxx flag as input for @query. ++ * ++ * @in: Input parameters ++ * @out: Output parameters ++ */ ++union kbase_ioctl_mem_query { ++ struct { ++ __u64 gpu_addr; ++ __u64 query; ++ } in; ++ struct { ++ __u64 value; ++ } out; ++}; ++ ++#define KBASE_IOCTL_MEM_QUERY \ ++ _IOWR(KBASE_IOCTL_TYPE, 6, union kbase_ioctl_mem_query) ++ ++#define KBASE_MEM_QUERY_COMMIT_SIZE 1 ++#define KBASE_MEM_QUERY_VA_SIZE 2 ++#define KBASE_MEM_QUERY_FLAGS 3 ++ ++/** ++ * struct kbase_ioctl_mem_free - Free a memory region ++ * @gpu_addr: Handle to the region to free ++ */ ++struct kbase_ioctl_mem_free { ++ __u64 gpu_addr; ++}; ++ ++#define KBASE_IOCTL_MEM_FREE \ ++ _IOW(KBASE_IOCTL_TYPE, 7, struct kbase_ioctl_mem_free) ++ ++/** ++ * struct kbase_ioctl_hwcnt_reader_setup - Setup HWC dumper/reader ++ * @buffer_count: requested number of dumping buffers ++ * @jm_bm: counters selection bitmask (JM) ++ * @shader_bm: counters selection bitmask (Shader) ++ * @tiler_bm: counters selection bitmask (Tiler) ++ * @mmu_l2_bm: counters selection bitmask (MMU_L2) ++ * ++ * A fd is returned from the ioctl if successful, or a negative value on error ++ */ ++struct kbase_ioctl_hwcnt_reader_setup { ++ __u32 buffer_count; ++ __u32 jm_bm; ++ __u32 shader_bm; ++ __u32 tiler_bm; ++ __u32 mmu_l2_bm; ++}; ++ ++#define KBASE_IOCTL_HWCNT_READER_SETUP \ ++ _IOW(KBASE_IOCTL_TYPE, 8, struct kbase_ioctl_hwcnt_reader_setup) ++ ++/** ++ * struct kbase_ioctl_hwcnt_enable - Enable hardware counter collection ++ * @dump_buffer: GPU address to write counters to ++ * @jm_bm: counters selection bitmask (JM) ++ * @shader_bm: counters selection bitmask (Shader) ++ * @tiler_bm: counters selection bitmask (Tiler) ++ * @mmu_l2_bm: counters selection bitmask (MMU_L2) ++ */ ++struct kbase_ioctl_hwcnt_enable { ++ __u64 dump_buffer; ++ __u32 jm_bm; ++ __u32 shader_bm; ++ __u32 tiler_bm; ++ __u32 mmu_l2_bm; ++}; ++ ++#define KBASE_IOCTL_HWCNT_ENABLE \ ++ _IOW(KBASE_IOCTL_TYPE, 9, struct kbase_ioctl_hwcnt_enable) ++ ++#define KBASE_IOCTL_HWCNT_DUMP \ ++ _IO(KBASE_IOCTL_TYPE, 10) ++ ++#define KBASE_IOCTL_HWCNT_CLEAR \ ++ _IO(KBASE_IOCTL_TYPE, 11) ++ ++/** ++ * struct kbase_ioctl_disjoint_query - Query the disjoint counter ++ * @counter: A counter of disjoint events in the kernel ++ */ ++struct kbase_ioctl_disjoint_query { ++ __u32 counter; ++}; ++ ++#define KBASE_IOCTL_DISJOINT_QUERY \ ++ _IOR(KBASE_IOCTL_TYPE, 12, struct kbase_ioctl_disjoint_query) ++ ++/** ++ * struct kbase_ioctl_get_ddk_version - Query the kernel version ++ * @version_buffer: Buffer to receive the kernel version string ++ * @size: Size of the buffer ++ * ++ * The ioctl will return the number of bytes written into version_buffer ++ * (which includes a NULL byte) or a negative error code ++ */ ++struct kbase_ioctl_get_ddk_version { ++ __u64 version_buffer; ++ __u32 size; ++}; ++ ++#define KBASE_IOCTL_GET_DDK_VERSION \ ++ _IOW(KBASE_IOCTL_TYPE, 13, struct kbase_ioctl_get_ddk_version) ++ ++/** ++ * struct kbase_ioctl_mem_jit_init - Initialise the JIT memory allocator ++ * ++ * @va_pages: Number of VA pages to reserve for JIT ++ * ++ * Note that depending on the VA size of the application and GPU, the value ++ * specified in @va_pages may be ignored. ++ */ ++struct kbase_ioctl_mem_jit_init { ++ __u64 va_pages; ++}; ++ ++#define KBASE_IOCTL_MEM_JIT_INIT \ ++ _IOW(KBASE_IOCTL_TYPE, 14, struct kbase_ioctl_mem_jit_init) ++ ++/** ++ * struct kbase_ioctl_mem_sync - Perform cache maintenance on memory ++ * ++ * @handle: GPU memory handle (GPU VA) ++ * @user_addr: The address where it is mapped in user space ++ * @size: The number of bytes to synchronise ++ * @type: The direction to synchronise: 0 is sync to memory (clean), ++ * 1 is sync from memory (invalidate). Use the BASE_SYNCSET_OP_xxx constants. ++ * @padding: Padding to round up to a multiple of 8 bytes, must be zero ++ */ ++struct kbase_ioctl_mem_sync { ++ __u64 handle; ++ __u64 user_addr; ++ __u64 size; ++ __u8 type; ++ __u8 padding[7]; ++}; ++ ++#define KBASE_IOCTL_MEM_SYNC \ ++ _IOW(KBASE_IOCTL_TYPE, 15, struct kbase_ioctl_mem_sync) ++ ++/** ++ * union kbase_ioctl_mem_find_cpu_offset - Find the offset of a CPU pointer ++ * ++ * @gpu_addr: The GPU address of the memory region ++ * @cpu_addr: The CPU address to locate ++ * @size: A size in bytes to validate is contained within the region ++ * @offset: The offset from the start of the memory region to @cpu_addr ++ * ++ * @in: Input parameters ++ * @out: Output parameters ++ */ ++union kbase_ioctl_mem_find_cpu_offset { ++ struct { ++ __u64 gpu_addr; ++ __u64 cpu_addr; ++ __u64 size; ++ } in; ++ struct { ++ __u64 offset; ++ } out; ++}; ++ ++#define KBASE_IOCTL_MEM_FIND_CPU_OFFSET \ ++ _IOWR(KBASE_IOCTL_TYPE, 16, union kbase_ioctl_mem_find_cpu_offset) ++ ++/** ++ * struct kbase_ioctl_get_context_id - Get the kernel context ID ++ * ++ * @id: The kernel context ID ++ */ ++struct kbase_ioctl_get_context_id { ++ __u32 id; ++}; ++ ++#define KBASE_IOCTL_GET_CONTEXT_ID \ ++ _IOR(KBASE_IOCTL_TYPE, 17, struct kbase_ioctl_get_context_id) ++ ++/** ++ * struct kbase_ioctl_tlstream_acquire - Acquire a tlstream fd ++ * ++ * @flags: Flags ++ * ++ * The ioctl returns a file descriptor when successful ++ */ ++struct kbase_ioctl_tlstream_acquire { ++ __u32 flags; ++}; ++ ++#define KBASE_IOCTL_TLSTREAM_ACQUIRE \ ++ _IOW(KBASE_IOCTL_TYPE, 18, struct kbase_ioctl_tlstream_acquire) ++ ++#define KBASE_IOCTL_TLSTREAM_FLUSH \ ++ _IO(KBASE_IOCTL_TYPE, 19) ++ ++/** ++ * struct kbase_ioctl_mem_commit - Change the amount of memory backing a region ++ * ++ * @gpu_addr: The memory region to modify ++ * @pages: The number of physical pages that should be present ++ * ++ * The ioctl may return on the following error codes or 0 for success: ++ * -ENOMEM: Out of memory ++ * -EINVAL: Invalid arguments ++ */ ++struct kbase_ioctl_mem_commit { ++ __u64 gpu_addr; ++ __u64 pages; ++}; ++ ++#define KBASE_IOCTL_MEM_COMMIT \ ++ _IOW(KBASE_IOCTL_TYPE, 20, struct kbase_ioctl_mem_commit) ++ ++/** ++ * union kbase_ioctl_mem_alias - Create an alias of memory regions ++ * @flags: Flags, see BASE_MEM_xxx ++ * @stride: Bytes between start of each memory region ++ * @nents: The number of regions to pack together into the alias ++ * @aliasing_info: Pointer to an array of struct base_mem_aliasing_info ++ * @gpu_va: Address of the new alias ++ * @va_pages: Size of the new alias ++ * ++ * @in: Input parameters ++ * @out: Output parameters ++ */ ++union kbase_ioctl_mem_alias { ++ struct { ++ __u64 flags; ++ __u64 stride; ++ __u64 nents; ++ __u64 aliasing_info; ++ } in; ++ struct { ++ __u64 flags; ++ __u64 gpu_va; ++ __u64 va_pages; ++ } out; ++}; ++ ++#define KBASE_IOCTL_MEM_ALIAS \ ++ _IOWR(KBASE_IOCTL_TYPE, 21, union kbase_ioctl_mem_alias) ++ ++/** ++ * union kbase_ioctl_mem_import - Import memory for use by the GPU ++ * @flags: Flags, see BASE_MEM_xxx ++ * @phandle: Handle to the external memory ++ * @type: Type of external memory, see base_mem_import_type ++ * @padding: Amount of extra VA pages to append to the imported buffer ++ * @gpu_va: Address of the new alias ++ * @va_pages: Size of the new alias ++ * ++ * @in: Input parameters ++ * @out: Output parameters ++ */ ++union kbase_ioctl_mem_import { ++ struct { ++ __u64 flags; ++ __u64 phandle; ++ __u32 type; ++ __u32 padding; ++ } in; ++ struct { ++ __u64 flags; ++ __u64 gpu_va; ++ __u64 va_pages; ++ } out; ++}; ++ ++#define KBASE_IOCTL_MEM_IMPORT \ ++ _IOWR(KBASE_IOCTL_TYPE, 22, union kbase_ioctl_mem_import) ++ ++/** ++ * struct kbase_ioctl_mem_flags_change - Change the flags for a memory region ++ * @gpu_va: The GPU region to modify ++ * @flags: The new flags to set ++ * @mask: Mask of the flags to modify ++ */ ++struct kbase_ioctl_mem_flags_change { ++ __u64 gpu_va; ++ __u64 flags; ++ __u64 mask; ++}; ++ ++#define KBASE_IOCTL_MEM_FLAGS_CHANGE \ ++ _IOW(KBASE_IOCTL_TYPE, 23, struct kbase_ioctl_mem_flags_change) ++ ++/** ++ * struct kbase_ioctl_stream_create - Create a synchronisation stream ++ * @name: A name to identify this stream. Must be NULL-terminated. ++ * ++ * Note that this is also called a "timeline", but is named stream to avoid ++ * confusion with other uses of the word. ++ * ++ * Unused bytes in @name (after the first NULL byte) must be also be NULL bytes. ++ * ++ * The ioctl returns a file descriptor. ++ */ ++struct kbase_ioctl_stream_create { ++ char name[32]; ++}; ++ ++#define KBASE_IOCTL_STREAM_CREATE \ ++ _IOW(KBASE_IOCTL_TYPE, 24, struct kbase_ioctl_stream_create) ++ ++/** ++ * struct kbase_ioctl_fence_validate - Validate a fd refers to a fence ++ * @fd: The file descriptor to validate ++ */ ++struct kbase_ioctl_fence_validate { ++ int fd; ++}; ++ ++#define KBASE_IOCTL_FENCE_VALIDATE \ ++ _IOW(KBASE_IOCTL_TYPE, 25, struct kbase_ioctl_fence_validate) ++ ++/** ++ * struct kbase_ioctl_get_profiling_controls - Get the profiling controls ++ * @count: The size of @buffer in u32 words ++ * @buffer: The buffer to receive the profiling controls ++ */ ++struct kbase_ioctl_get_profiling_controls { ++ __u64 buffer; ++ __u32 count; ++}; ++ ++#define KBASE_IOCTL_GET_PROFILING_CONTROLS \ ++ _IOW(KBASE_IOCTL_TYPE, 26, struct kbase_ioctl_get_profiling_controls) ++ ++/** ++ * struct kbase_ioctl_mem_profile_add - Provide profiling information to kernel ++ * @buffer: Pointer to the information ++ * @len: Length ++ * @padding: Padding ++ * ++ * The data provided is accessible through a debugfs file ++ */ ++struct kbase_ioctl_mem_profile_add { ++ __u64 buffer; ++ __u32 len; ++ __u32 padding; ++}; ++ ++#define KBASE_IOCTL_MEM_PROFILE_ADD \ ++ _IOW(KBASE_IOCTL_TYPE, 27, struct kbase_ioctl_mem_profile_add) ++ ++/** ++ * struct kbase_ioctl_soft_event_update - Update the status of a soft-event ++ * @event: GPU address of the event which has been updated ++ * @new_status: The new status to set ++ * @flags: Flags for future expansion ++ */ ++struct kbase_ioctl_soft_event_update { ++ __u64 event; ++ __u32 new_status; ++ __u32 flags; ++}; ++ ++#define KBASE_IOCTL_SOFT_EVENT_UPDATE \ ++ _IOW(KBASE_IOCTL_TYPE, 28, struct kbase_ioctl_soft_event_update) ++ ++/* IOCTLs 29-32 are reserved */ ++ ++/*************** ++ * test ioctls * ++ ***************/ ++#if MALI_UNIT_TEST ++/* These ioctls are purely for test purposes and are not used in the production ++ * driver, they therefore may change without notice ++ */ ++ ++#define KBASE_IOCTL_TEST_TYPE (KBASE_IOCTL_TYPE + 1) ++ ++/** ++ * struct kbase_ioctl_tlstream_test - Start a timeline stream test ++ * ++ * @tpw_count: number of trace point writers in each context ++ * @msg_delay: time delay between tracepoints from one writer in milliseconds ++ * @msg_count: number of trace points written by one writer ++ * @aux_msg: if non-zero aux messages will be included ++ */ ++struct kbase_ioctl_tlstream_test { ++ __u32 tpw_count; ++ __u32 msg_delay; ++ __u32 msg_count; ++ __u32 aux_msg; ++}; ++ ++#define KBASE_IOCTL_TLSTREAM_TEST \ ++ _IOW(KBASE_IOCTL_TEST_TYPE, 1, struct kbase_ioctl_tlstream_test) ++ ++/** ++ * struct kbase_ioctl_tlstream_stats - Read tlstream stats for test purposes ++ * @bytes_collected: number of bytes read by user ++ * @bytes_generated: number of bytes generated by tracepoints ++ */ ++struct kbase_ioctl_tlstream_stats { ++ __u32 bytes_collected; ++ __u32 bytes_generated; ++}; ++ ++#define KBASE_IOCTL_TLSTREAM_STATS \ ++ _IOR(KBASE_IOCTL_TEST_TYPE, 2, struct kbase_ioctl_tlstream_stats) ++ ++#endif ++ ++/********************************** ++ * Definitions for GPU properties * ++ **********************************/ ++#define KBASE_GPUPROP_VALUE_SIZE_U8 (0x0) ++#define KBASE_GPUPROP_VALUE_SIZE_U16 (0x1) ++#define KBASE_GPUPROP_VALUE_SIZE_U32 (0x2) ++#define KBASE_GPUPROP_VALUE_SIZE_U64 (0x3) ++ ++#define KBASE_GPUPROP_PRODUCT_ID 1 ++#define KBASE_GPUPROP_VERSION_STATUS 2 ++#define KBASE_GPUPROP_MINOR_REVISION 3 ++#define KBASE_GPUPROP_MAJOR_REVISION 4 ++#define KBASE_GPUPROP_GPU_SPEED_MHZ 5 ++#define KBASE_GPUPROP_GPU_FREQ_KHZ_MAX 6 ++#define KBASE_GPUPROP_GPU_FREQ_KHZ_MIN 7 ++#define KBASE_GPUPROP_LOG2_PROGRAM_COUNTER_SIZE 8 ++#define KBASE_GPUPROP_TEXTURE_FEATURES_0 9 ++#define KBASE_GPUPROP_TEXTURE_FEATURES_1 10 ++#define KBASE_GPUPROP_TEXTURE_FEATURES_2 11 ++#define KBASE_GPUPROP_GPU_AVAILABLE_MEMORY_SIZE 12 ++ ++#define KBASE_GPUPROP_L2_LOG2_LINE_SIZE 13 ++#define KBASE_GPUPROP_L2_LOG2_CACHE_SIZE 14 ++#define KBASE_GPUPROP_L2_NUM_L2_SLICES 15 ++ ++#define KBASE_GPUPROP_TILER_BIN_SIZE_BYTES 16 ++#define KBASE_GPUPROP_TILER_MAX_ACTIVE_LEVELS 17 ++ ++#define KBASE_GPUPROP_MAX_THREADS 18 ++#define KBASE_GPUPROP_MAX_WORKGROUP_SIZE 19 ++#define KBASE_GPUPROP_MAX_BARRIER_SIZE 20 ++#define KBASE_GPUPROP_MAX_REGISTERS 21 ++#define KBASE_GPUPROP_MAX_TASK_QUEUE 22 ++#define KBASE_GPUPROP_MAX_THREAD_GROUP_SPLIT 23 ++#define KBASE_GPUPROP_IMPL_TECH 24 ++ ++#define KBASE_GPUPROP_RAW_SHADER_PRESENT 25 ++#define KBASE_GPUPROP_RAW_TILER_PRESENT 26 ++#define KBASE_GPUPROP_RAW_L2_PRESENT 27 ++#define KBASE_GPUPROP_RAW_STACK_PRESENT 28 ++#define KBASE_GPUPROP_RAW_L2_FEATURES 29 ++#define KBASE_GPUPROP_RAW_SUSPEND_SIZE 30 ++#define KBASE_GPUPROP_RAW_MEM_FEATURES 31 ++#define KBASE_GPUPROP_RAW_MMU_FEATURES 32 ++#define KBASE_GPUPROP_RAW_AS_PRESENT 33 ++#define KBASE_GPUPROP_RAW_JS_PRESENT 34 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_0 35 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_1 36 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_2 37 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_3 38 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_4 39 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_5 40 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_6 41 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_7 42 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_8 43 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_9 44 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_10 45 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_11 46 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_12 47 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_13 48 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_14 49 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_15 50 ++#define KBASE_GPUPROP_RAW_TILER_FEATURES 51 ++#define KBASE_GPUPROP_RAW_TEXTURE_FEATURES_0 52 ++#define KBASE_GPUPROP_RAW_TEXTURE_FEATURES_1 53 ++#define KBASE_GPUPROP_RAW_TEXTURE_FEATURES_2 54 ++#define KBASE_GPUPROP_RAW_GPU_ID 55 ++#define KBASE_GPUPROP_RAW_THREAD_MAX_THREADS 56 ++#define KBASE_GPUPROP_RAW_THREAD_MAX_WORKGROUP_SIZE 57 ++#define KBASE_GPUPROP_RAW_THREAD_MAX_BARRIER_SIZE 58 ++#define KBASE_GPUPROP_RAW_THREAD_FEATURES 59 ++#define KBASE_GPUPROP_RAW_COHERENCY_MODE 60 ++ ++#define KBASE_GPUPROP_COHERENCY_NUM_GROUPS 61 ++#define KBASE_GPUPROP_COHERENCY_NUM_CORE_GROUPS 62 ++#define KBASE_GPUPROP_COHERENCY_COHERENCY 63 ++#define KBASE_GPUPROP_COHERENCY_GROUP_0 64 ++#define KBASE_GPUPROP_COHERENCY_GROUP_1 65 ++#define KBASE_GPUPROP_COHERENCY_GROUP_2 66 ++#define KBASE_GPUPROP_COHERENCY_GROUP_3 67 ++#define KBASE_GPUPROP_COHERENCY_GROUP_4 68 ++#define KBASE_GPUPROP_COHERENCY_GROUP_5 69 ++#define KBASE_GPUPROP_COHERENCY_GROUP_6 70 ++#define KBASE_GPUPROP_COHERENCY_GROUP_7 71 ++#define KBASE_GPUPROP_COHERENCY_GROUP_8 72 ++#define KBASE_GPUPROP_COHERENCY_GROUP_9 73 ++#define KBASE_GPUPROP_COHERENCY_GROUP_10 74 ++#define KBASE_GPUPROP_COHERENCY_GROUP_11 75 ++#define KBASE_GPUPROP_COHERENCY_GROUP_12 76 ++#define KBASE_GPUPROP_COHERENCY_GROUP_13 77 ++#define KBASE_GPUPROP_COHERENCY_GROUP_14 78 ++#define KBASE_GPUPROP_COHERENCY_GROUP_15 79 ++ ++#ifdef __cpluscplus ++} ++#endif ++ ++#endif +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_jd.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_jd.c +new file mode 100755 +index 000000000000..144ebfcdfc59 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_jd.c +@@ -0,0 +1,1847 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#if defined(CONFIG_DMA_SHARED_BUFFER) ++#include ++#endif /* defined(CONFIG_DMA_SHARED_BUFFER) */ ++#ifdef CONFIG_COMPAT ++#include ++#endif ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#include ++#include ++ ++#include "mali_kbase_dma_fence.h" ++ ++#define beenthere(kctx, f, a...) dev_dbg(kctx->kbdev->dev, "%s:" f, __func__, ##a) ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 8, 0) ++/* random32 was renamed to prandom_u32 in 3.8 */ ++#define prandom_u32 random32 ++#endif ++ ++/* Return whether katom will run on the GPU or not. Currently only soft jobs and ++ * dependency-only atoms do not run on the GPU */ ++#define IS_GPU_ATOM(katom) (!((katom->core_req & BASE_JD_REQ_SOFT_JOB) || \ ++ ((katom->core_req & BASE_JD_REQ_ATOM_TYPE) == \ ++ BASE_JD_REQ_DEP))) ++/* ++ * This is the kernel side of the API. Only entry points are: ++ * - kbase_jd_submit(): Called from userspace to submit a single bag ++ * - kbase_jd_done(): Called from interrupt context to track the ++ * completion of a job. ++ * Callouts: ++ * - to the job manager (enqueue a job) ++ * - to the event subsystem (signals the completion/failure of bag/job-chains). ++ */ ++ ++static void __user * ++get_compat_pointer(struct kbase_context *kctx, const u64 p) ++{ ++#ifdef CONFIG_COMPAT ++ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) ++ return compat_ptr(p); ++#endif ++ return u64_to_user_ptr(p); ++} ++ ++/* Runs an atom, either by handing to the JS or by immediately running it in the case of soft-jobs ++ * ++ * Returns whether the JS needs a reschedule. ++ * ++ * Note that the caller must also check the atom status and ++ * if it is KBASE_JD_ATOM_STATE_COMPLETED must call jd_done_nolock ++ */ ++static int jd_run_atom(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ ++ KBASE_DEBUG_ASSERT(katom->status != KBASE_JD_ATOM_STATE_UNUSED); ++ ++ if ((katom->core_req & BASE_JD_REQ_ATOM_TYPE) == BASE_JD_REQ_DEP) { ++ /* Dependency only atom */ ++ katom->status = KBASE_JD_ATOM_STATE_COMPLETED; ++ return 0; ++ } else if (katom->core_req & BASE_JD_REQ_SOFT_JOB) { ++ /* Soft-job */ ++ if (katom->will_fail_event_code) { ++ katom->status = KBASE_JD_ATOM_STATE_COMPLETED; ++ return 0; ++ } ++ if ((katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) ++ == BASE_JD_REQ_SOFT_REPLAY) { ++ if (!kbase_replay_process(katom)) ++ katom->status = KBASE_JD_ATOM_STATE_COMPLETED; ++ } else if (kbase_process_soft_job(katom) == 0) { ++ kbase_finish_soft_job(katom); ++ katom->status = KBASE_JD_ATOM_STATE_COMPLETED; ++ } ++ return 0; ++ } ++ ++ katom->status = KBASE_JD_ATOM_STATE_IN_JS; ++ /* Queue an action about whether we should try scheduling a context */ ++ return kbasep_js_add_job(kctx, katom); ++} ++ ++#if defined(CONFIG_KDS) || defined(CONFIG_MALI_BIFROST_DMA_FENCE) ++void kbase_jd_dep_clear_locked(struct kbase_jd_atom *katom) ++{ ++ struct kbase_device *kbdev; ++ ++ KBASE_DEBUG_ASSERT(katom); ++ kbdev = katom->kctx->kbdev; ++ KBASE_DEBUG_ASSERT(kbdev); ++ ++ /* Check whether the atom's other dependencies were already met. If ++ * katom is a GPU atom then the job scheduler may be able to represent ++ * the dependencies, hence we may attempt to submit it before they are ++ * met. Other atoms must have had both dependencies resolved. ++ */ ++ if (IS_GPU_ATOM(katom) || ++ (!kbase_jd_katom_dep_atom(&katom->dep[0]) && ++ !kbase_jd_katom_dep_atom(&katom->dep[1]))) { ++ /* katom dep complete, attempt to run it */ ++ bool resched = false; ++ ++ resched = jd_run_atom(katom); ++ ++ if (katom->status == KBASE_JD_ATOM_STATE_COMPLETED) { ++ /* The atom has already finished */ ++ resched |= jd_done_nolock(katom, NULL); ++ } ++ ++ if (resched) ++ kbase_js_sched_all(kbdev); ++ } ++} ++#endif ++ ++#ifdef CONFIG_KDS ++ ++/* Add the katom to the kds waiting list. ++ * Atoms must be added to the waiting list after a successful call to kds_async_waitall. ++ * The caller must hold the kbase_jd_context.lock */ ++ ++static void kbase_jd_kds_waiters_add(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx; ++ ++ KBASE_DEBUG_ASSERT(katom); ++ ++ kctx = katom->kctx; ++ ++ list_add_tail(&katom->node, &kctx->waiting_kds_resource); ++} ++ ++/* Remove the katom from the kds waiting list. ++ * Atoms must be removed from the waiting list before a call to kds_resource_set_release_sync. ++ * The supplied katom must first have been added to the list with a call to kbase_jd_kds_waiters_add. ++ * The caller must hold the kbase_jd_context.lock */ ++ ++static void kbase_jd_kds_waiters_remove(struct kbase_jd_atom *katom) ++{ ++ KBASE_DEBUG_ASSERT(katom); ++ list_del(&katom->node); ++} ++ ++static void kds_dep_clear(void *callback_parameter, void *callback_extra_parameter) ++{ ++ struct kbase_jd_atom *katom; ++ struct kbase_jd_context *ctx; ++ ++ katom = (struct kbase_jd_atom *)callback_parameter; ++ KBASE_DEBUG_ASSERT(katom); ++ ++ ctx = &katom->kctx->jctx; ++ ++ /* If KDS resource has already been satisfied (e.g. due to zapping) ++ * do nothing. ++ */ ++ mutex_lock(&ctx->lock); ++ if (!katom->kds_dep_satisfied) { ++ katom->kds_dep_satisfied = true; ++ kbase_jd_dep_clear_locked(katom); ++ } ++ mutex_unlock(&ctx->lock); ++} ++ ++static void kbase_cancel_kds_wait_job(struct kbase_jd_atom *katom) ++{ ++ KBASE_DEBUG_ASSERT(katom); ++ ++ /* Prevent job_done_nolock from being called twice on an atom when ++ * there is a race between job completion and cancellation */ ++ ++ if (katom->status == KBASE_JD_ATOM_STATE_QUEUED) { ++ /* Wait was cancelled - zap the atom */ ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ if (jd_done_nolock(katom, NULL)) ++ kbase_js_sched_all(katom->kctx->kbdev); ++ } ++} ++#endif /* CONFIG_KDS */ ++ ++void kbase_jd_free_external_resources(struct kbase_jd_atom *katom) ++{ ++#ifdef CONFIG_KDS ++ if (katom->kds_rset) { ++ struct kbase_jd_context *jctx = &katom->kctx->jctx; ++ ++ /* ++ * As the atom is no longer waiting, remove it from ++ * the waiting list. ++ */ ++ ++ mutex_lock(&jctx->lock); ++ kbase_jd_kds_waiters_remove(katom); ++ mutex_unlock(&jctx->lock); ++ ++ /* Release the kds resource or cancel if zapping */ ++ kds_resource_set_release_sync(&katom->kds_rset); ++ } ++#endif /* CONFIG_KDS */ ++ ++#ifdef CONFIG_MALI_BIFROST_DMA_FENCE ++ /* Flush dma-fence workqueue to ensure that any callbacks that may have ++ * been queued are done before continuing. ++ * Any successfully completed atom would have had all it's callbacks ++ * completed before the atom was run, so only flush for failed atoms. ++ */ ++ if (katom->event_code != BASE_JD_EVENT_DONE) ++ flush_workqueue(katom->kctx->dma_fence.wq); ++#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ ++} ++ ++static void kbase_jd_post_external_resources(struct kbase_jd_atom *katom) ++{ ++ KBASE_DEBUG_ASSERT(katom); ++ KBASE_DEBUG_ASSERT(katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES); ++ ++#ifdef CONFIG_KDS ++ /* Prevent the KDS resource from triggering the atom in case of zapping */ ++ if (katom->kds_rset) ++ katom->kds_dep_satisfied = true; ++#endif /* CONFIG_KDS */ ++ ++#ifdef CONFIG_MALI_BIFROST_DMA_FENCE ++ kbase_dma_fence_signal(katom); ++#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ ++ ++ kbase_gpu_vm_lock(katom->kctx); ++ /* only roll back if extres is non-NULL */ ++ if (katom->extres) { ++ u32 res_no; ++ ++ res_no = katom->nr_extres; ++ while (res_no-- > 0) { ++ struct kbase_mem_phy_alloc *alloc = katom->extres[res_no].alloc; ++ struct kbase_va_region *reg; ++ ++ reg = kbase_region_tracker_find_region_base_address( ++ katom->kctx, ++ katom->extres[res_no].gpu_address); ++ kbase_unmap_external_resource(katom->kctx, reg, alloc); ++ } ++ kfree(katom->extres); ++ katom->extres = NULL; ++ } ++ kbase_gpu_vm_unlock(katom->kctx); ++} ++ ++/* ++ * Set up external resources needed by this job. ++ * ++ * jctx.lock must be held when this is called. ++ */ ++ ++static int kbase_jd_pre_external_resources(struct kbase_jd_atom *katom, const struct base_jd_atom_v2 *user_atom) ++{ ++ int err_ret_val = -EINVAL; ++ u32 res_no; ++#ifdef CONFIG_KDS ++ u32 kds_res_count = 0; ++ struct kds_resource **kds_resources = NULL; ++ unsigned long *kds_access_bitmap = NULL; ++#endif /* CONFIG_KDS */ ++#ifdef CONFIG_MALI_BIFROST_DMA_FENCE ++ struct kbase_dma_fence_resv_info info = { ++ .dma_fence_resv_count = 0, ++ }; ++#ifdef CONFIG_SYNC ++ /* ++ * When both dma-buf fence and Android native sync is enabled, we ++ * disable dma-buf fence for contexts that are using Android native ++ * fences. ++ */ ++ const bool implicit_sync = !kbase_ctx_flag(katom->kctx, ++ KCTX_NO_IMPLICIT_SYNC); ++#else /* CONFIG_SYNC */ ++ const bool implicit_sync = true; ++#endif /* CONFIG_SYNC */ ++#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ ++ struct base_external_resource *input_extres; ++ ++ KBASE_DEBUG_ASSERT(katom); ++ KBASE_DEBUG_ASSERT(katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES); ++ ++ /* no resources encoded, early out */ ++ if (!katom->nr_extres) ++ return -EINVAL; ++ ++ katom->extres = kmalloc_array(katom->nr_extres, sizeof(*katom->extres), GFP_KERNEL); ++ if (NULL == katom->extres) { ++ err_ret_val = -ENOMEM; ++ goto early_err_out; ++ } ++ ++ /* copy user buffer to the end of our real buffer. ++ * Make sure the struct sizes haven't changed in a way ++ * we don't support */ ++ BUILD_BUG_ON(sizeof(*input_extres) > sizeof(*katom->extres)); ++ input_extres = (struct base_external_resource *) ++ (((unsigned char *)katom->extres) + ++ (sizeof(*katom->extres) - sizeof(*input_extres)) * ++ katom->nr_extres); ++ ++ if (copy_from_user(input_extres, ++ get_compat_pointer(katom->kctx, user_atom->extres_list), ++ sizeof(*input_extres) * katom->nr_extres) != 0) { ++ err_ret_val = -EINVAL; ++ goto early_err_out; ++ } ++#ifdef CONFIG_KDS ++ /* assume we have to wait for all */ ++ KBASE_DEBUG_ASSERT(0 != katom->nr_extres); ++ kds_resources = kmalloc_array(katom->nr_extres, sizeof(struct kds_resource *), GFP_KERNEL); ++ ++ if (!kds_resources) { ++ err_ret_val = -ENOMEM; ++ goto early_err_out; ++ } ++ ++ KBASE_DEBUG_ASSERT(0 != katom->nr_extres); ++ kds_access_bitmap = kcalloc(BITS_TO_LONGS(katom->nr_extres), ++ sizeof(unsigned long), ++ GFP_KERNEL); ++ if (!kds_access_bitmap) { ++ err_ret_val = -ENOMEM; ++ goto early_err_out; ++ } ++#endif /* CONFIG_KDS */ ++ ++#ifdef CONFIG_MALI_BIFROST_DMA_FENCE ++ if (implicit_sync) { ++ info.resv_objs = kmalloc_array(katom->nr_extres, ++ sizeof(struct reservation_object *), ++ GFP_KERNEL); ++ if (!info.resv_objs) { ++ err_ret_val = -ENOMEM; ++ goto early_err_out; ++ } ++ ++ info.dma_fence_excl_bitmap = ++ kcalloc(BITS_TO_LONGS(katom->nr_extres), ++ sizeof(unsigned long), GFP_KERNEL); ++ if (!info.dma_fence_excl_bitmap) { ++ err_ret_val = -ENOMEM; ++ goto early_err_out; ++ } ++ } ++#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ ++ ++ /* Take the processes mmap lock */ ++ down_read(¤t->mm->mmap_sem); ++ ++ /* need to keep the GPU VM locked while we set up UMM buffers */ ++ kbase_gpu_vm_lock(katom->kctx); ++ for (res_no = 0; res_no < katom->nr_extres; res_no++) { ++ struct base_external_resource *res; ++ struct kbase_va_region *reg; ++ struct kbase_mem_phy_alloc *alloc; ++ bool exclusive; ++ ++ res = &input_extres[res_no]; ++ exclusive = (res->ext_resource & BASE_EXT_RES_ACCESS_EXCLUSIVE) ++ ? true : false; ++ reg = kbase_region_tracker_find_region_enclosing_address( ++ katom->kctx, ++ res->ext_resource & ~BASE_EXT_RES_ACCESS_EXCLUSIVE); ++ /* did we find a matching region object? */ ++ if (NULL == reg || (reg->flags & KBASE_REG_FREE)) { ++ /* roll back */ ++ goto failed_loop; ++ } ++ ++ if (!(katom->core_req & BASE_JD_REQ_SOFT_JOB) && ++ (reg->flags & KBASE_REG_SECURE)) { ++ katom->atom_flags |= KBASE_KATOM_FLAG_PROTECTED; ++ } ++ ++ alloc = kbase_map_external_resource(katom->kctx, reg, ++ current->mm ++#ifdef CONFIG_KDS ++ , &kds_res_count, kds_resources, ++ kds_access_bitmap, exclusive ++#endif ++ ); ++ if (!alloc) { ++ err_ret_val = -EINVAL; ++ goto failed_loop; ++ } ++ ++#ifdef CONFIG_MALI_BIFROST_DMA_FENCE ++ if (implicit_sync && ++ reg->gpu_alloc->type == KBASE_MEM_TYPE_IMPORTED_UMM) { ++ struct reservation_object *resv; ++ ++ resv = reg->gpu_alloc->imported.umm.dma_buf->resv; ++ if (resv) ++ kbase_dma_fence_add_reservation(resv, &info, ++ exclusive); ++ } ++#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ ++ ++ /* finish with updating out array with the data we found */ ++ /* NOTE: It is important that this is the last thing we do (or ++ * at least not before the first write) as we overwrite elements ++ * as we loop and could be overwriting ourself, so no writes ++ * until the last read for an element. ++ * */ ++ katom->extres[res_no].gpu_address = reg->start_pfn << PAGE_SHIFT; /* save the start_pfn (as an address, not pfn) to use fast lookup later */ ++ katom->extres[res_no].alloc = alloc; ++ } ++ /* successfully parsed the extres array */ ++ /* drop the vm lock before we call into kds */ ++ kbase_gpu_vm_unlock(katom->kctx); ++ ++ /* Release the processes mmap lock */ ++ up_read(¤t->mm->mmap_sem); ++ ++#ifdef CONFIG_KDS ++ if (kds_res_count) { ++ int wait_failed; ++ ++ /* We have resources to wait for with kds */ ++ katom->kds_dep_satisfied = false; ++ ++ wait_failed = kds_async_waitall(&katom->kds_rset, ++ &katom->kctx->jctx.kds_cb, katom, NULL, ++ kds_res_count, kds_access_bitmap, ++ kds_resources); ++ ++ if (wait_failed) ++ goto failed_kds_setup; ++ else ++ kbase_jd_kds_waiters_add(katom); ++ } else { ++ /* Nothing to wait for, so kds dep met */ ++ katom->kds_dep_satisfied = true; ++ } ++ kfree(kds_resources); ++ kfree(kds_access_bitmap); ++#endif /* CONFIG_KDS */ ++ ++#ifdef CONFIG_MALI_BIFROST_DMA_FENCE ++ if (implicit_sync) { ++ if (info.dma_fence_resv_count) { ++ int ret; ++ ++ ret = kbase_dma_fence_wait(katom, &info); ++ if (ret < 0) ++ goto failed_dma_fence_setup; ++ } ++ ++ kfree(info.resv_objs); ++ kfree(info.dma_fence_excl_bitmap); ++ } ++#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ ++ ++ /* all done OK */ ++ return 0; ++ ++/* error handling section */ ++ ++#ifdef CONFIG_MALI_BIFROST_DMA_FENCE ++failed_dma_fence_setup: ++#ifdef CONFIG_KDS ++ /* If we are here, dma_fence setup failed but KDS didn't. ++ * Revert KDS setup if any. ++ */ ++ if (kds_res_count) { ++ mutex_unlock(&katom->kctx->jctx.lock); ++ kds_resource_set_release_sync(&katom->kds_rset); ++ mutex_lock(&katom->kctx->jctx.lock); ++ ++ kbase_jd_kds_waiters_remove(katom); ++ katom->kds_dep_satisfied = true; ++ } ++#endif /* CONFIG_KDS */ ++#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ ++#ifdef CONFIG_KDS ++failed_kds_setup: ++#endif ++#if defined(CONFIG_KDS) || defined(CONFIG_MALI_BIFROST_DMA_FENCE) ++ /* Lock the processes mmap lock */ ++ down_read(¤t->mm->mmap_sem); ++ ++ /* lock before we unmap */ ++ kbase_gpu_vm_lock(katom->kctx); ++#endif ++ ++ failed_loop: ++ /* undo the loop work */ ++ while (res_no-- > 0) { ++ struct kbase_mem_phy_alloc *alloc = katom->extres[res_no].alloc; ++ ++ kbase_unmap_external_resource(katom->kctx, NULL, alloc); ++ } ++ kbase_gpu_vm_unlock(katom->kctx); ++ ++ /* Release the processes mmap lock */ ++ up_read(¤t->mm->mmap_sem); ++ ++ early_err_out: ++ kfree(katom->extres); ++ katom->extres = NULL; ++#ifdef CONFIG_KDS ++ kfree(kds_resources); ++ kfree(kds_access_bitmap); ++#endif /* CONFIG_KDS */ ++#ifdef CONFIG_MALI_BIFROST_DMA_FENCE ++ if (implicit_sync) { ++ kfree(info.resv_objs); ++ kfree(info.dma_fence_excl_bitmap); ++ } ++#endif ++ return err_ret_val; ++} ++ ++static inline void jd_resolve_dep(struct list_head *out_list, ++ struct kbase_jd_atom *katom, ++ u8 d, bool ctx_is_dying) ++{ ++ u8 other_d = !d; ++ ++ while (!list_empty(&katom->dep_head[d])) { ++ struct kbase_jd_atom *dep_atom; ++ struct kbase_jd_atom *other_dep_atom; ++ u8 dep_type; ++ ++ dep_atom = list_entry(katom->dep_head[d].next, ++ struct kbase_jd_atom, dep_item[d]); ++ list_del(katom->dep_head[d].next); ++ ++ dep_type = kbase_jd_katom_dep_type(&dep_atom->dep[d]); ++ kbase_jd_katom_dep_clear(&dep_atom->dep[d]); ++ ++ if (katom->event_code != BASE_JD_EVENT_DONE && ++ (dep_type != BASE_JD_DEP_TYPE_ORDER)) { ++#ifdef CONFIG_KDS ++ if (!dep_atom->kds_dep_satisfied) { ++ /* Just set kds_dep_satisfied to true. If the callback happens after this then it will early out and ++ * do nothing. If the callback doesn't happen then kbase_jd_post_external_resources will clean up ++ */ ++ dep_atom->kds_dep_satisfied = true; ++ } ++#endif ++ ++#ifdef CONFIG_MALI_BIFROST_DMA_FENCE ++ kbase_dma_fence_cancel_callbacks(dep_atom); ++#endif ++ ++ dep_atom->event_code = katom->event_code; ++ KBASE_DEBUG_ASSERT(dep_atom->status != ++ KBASE_JD_ATOM_STATE_UNUSED); ++ ++ if ((dep_atom->core_req & BASE_JD_REQ_SOFT_REPLAY) ++ != BASE_JD_REQ_SOFT_REPLAY) { ++ dep_atom->will_fail_event_code = ++ dep_atom->event_code; ++ } else { ++ dep_atom->status = ++ KBASE_JD_ATOM_STATE_COMPLETED; ++ } ++ } ++ other_dep_atom = (struct kbase_jd_atom *) ++ kbase_jd_katom_dep_atom(&dep_atom->dep[other_d]); ++ ++ if (!dep_atom->in_jd_list && (!other_dep_atom || ++ (IS_GPU_ATOM(dep_atom) && !ctx_is_dying && ++ !dep_atom->will_fail_event_code && ++ !other_dep_atom->will_fail_event_code))) { ++ bool dep_satisfied = true; ++#ifdef CONFIG_MALI_BIFROST_DMA_FENCE ++ int dep_count; ++ ++ dep_count = kbase_fence_dep_count_read(dep_atom); ++ if (likely(dep_count == -1)) { ++ dep_satisfied = true; ++ } else { ++ /* ++ * There are either still active callbacks, or ++ * all fences for this @dep_atom has signaled, ++ * but the worker that will queue the atom has ++ * not yet run. ++ * ++ * Wait for the fences to signal and the fence ++ * worker to run and handle @dep_atom. If ++ * @dep_atom was completed due to error on ++ * @katom, then the fence worker will pick up ++ * the complete status and error code set on ++ * @dep_atom above. ++ */ ++ dep_satisfied = false; ++ } ++#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ ++ ++#ifdef CONFIG_KDS ++ dep_satisfied = dep_satisfied && dep_atom->kds_dep_satisfied; ++#endif ++ ++ if (dep_satisfied) { ++ dep_atom->in_jd_list = true; ++ list_add_tail(&dep_atom->jd_item, out_list); ++ } ++ } ++ } ++} ++ ++KBASE_EXPORT_TEST_API(jd_resolve_dep); ++ ++#if MALI_CUSTOMER_RELEASE == 0 ++static void jd_force_failure(struct kbase_device *kbdev, struct kbase_jd_atom *katom) ++{ ++ kbdev->force_replay_count++; ++ ++ if (kbdev->force_replay_count >= kbdev->force_replay_limit) { ++ kbdev->force_replay_count = 0; ++ katom->event_code = BASE_JD_EVENT_FORCE_REPLAY; ++ ++ if (kbdev->force_replay_random) ++ kbdev->force_replay_limit = ++ (prandom_u32() % KBASEP_FORCE_REPLAY_RANDOM_LIMIT) + 1; ++ ++ dev_info(kbdev->dev, "force_replay : promoting to error\n"); ++ } ++} ++ ++/** Test to see if atom should be forced to fail. ++ * ++ * This function will check if an atom has a replay job as a dependent. If so ++ * then it will be considered for forced failure. */ ++static void jd_check_force_failure(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ struct kbase_device *kbdev = kctx->kbdev; ++ int i; ++ ++ if ((kbdev->force_replay_limit == KBASEP_FORCE_REPLAY_DISABLED) || ++ (katom->core_req & BASEP_JD_REQ_EVENT_NEVER)) ++ return; ++ ++ for (i = 1; i < BASE_JD_ATOM_COUNT; i++) { ++ if (kbase_jd_katom_dep_atom(&kctx->jctx.atoms[i].dep[0]) == katom || ++ kbase_jd_katom_dep_atom(&kctx->jctx.atoms[i].dep[1]) == katom) { ++ struct kbase_jd_atom *dep_atom = &kctx->jctx.atoms[i]; ++ ++ if ((dep_atom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) == ++ BASE_JD_REQ_SOFT_REPLAY && ++ (dep_atom->core_req & kbdev->force_replay_core_req) ++ == kbdev->force_replay_core_req) { ++ jd_force_failure(kbdev, katom); ++ return; ++ } ++ } ++ } ++} ++#endif ++ ++/** ++ * is_dep_valid - Validate that a dependency is valid for early dependency ++ * submission ++ * @katom: Dependency atom to validate ++ * ++ * A dependency is valid if any of the following are true : ++ * - It does not exist (a non-existent dependency does not block submission) ++ * - It is in the job scheduler ++ * - It has completed, does not have a failure event code, and has not been ++ * marked to fail in the future ++ * ++ * Return: true if valid, false otherwise ++ */ ++static bool is_dep_valid(struct kbase_jd_atom *katom) ++{ ++ /* If there's no dependency then this is 'valid' from the perspective of ++ * early dependency submission */ ++ if (!katom) ++ return true; ++ ++ /* Dependency must have reached the job scheduler */ ++ if (katom->status < KBASE_JD_ATOM_STATE_IN_JS) ++ return false; ++ ++ /* If dependency has completed and has failed or will fail then it is ++ * not valid */ ++ if (katom->status >= KBASE_JD_ATOM_STATE_HW_COMPLETED && ++ (katom->event_code != BASE_JD_EVENT_DONE || ++ katom->will_fail_event_code)) ++ return false; ++ ++ return true; ++} ++ ++static void jd_try_submitting_deps(struct list_head *out_list, ++ struct kbase_jd_atom *node) ++{ ++ int i; ++ ++ for (i = 0; i < 2; i++) { ++ struct list_head *pos; ++ ++ list_for_each(pos, &node->dep_head[i]) { ++ struct kbase_jd_atom *dep_atom = list_entry(pos, ++ struct kbase_jd_atom, dep_item[i]); ++ ++ if (IS_GPU_ATOM(dep_atom) && !dep_atom->in_jd_list) { ++ /*Check if atom deps look sane*/ ++ bool dep0_valid = is_dep_valid( ++ dep_atom->dep[0].atom); ++ bool dep1_valid = is_dep_valid( ++ dep_atom->dep[1].atom); ++ bool dep_satisfied = true; ++#ifdef CONFIG_MALI_BIFROST_DMA_FENCE ++ int dep_count; ++ ++ dep_count = kbase_fence_dep_count_read( ++ dep_atom); ++ if (likely(dep_count == -1)) { ++ dep_satisfied = true; ++ } else { ++ /* ++ * There are either still active callbacks, or ++ * all fences for this @dep_atom has signaled, ++ * but the worker that will queue the atom has ++ * not yet run. ++ * ++ * Wait for the fences to signal and the fence ++ * worker to run and handle @dep_atom. If ++ * @dep_atom was completed due to error on ++ * @katom, then the fence worker will pick up ++ * the complete status and error code set on ++ * @dep_atom above. ++ */ ++ dep_satisfied = false; ++ } ++#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ ++#ifdef CONFIG_KDS ++ dep_satisfied = dep_satisfied && ++ dep_atom->kds_dep_satisfied; ++#endif ++ ++ if (dep0_valid && dep1_valid && dep_satisfied) { ++ dep_atom->in_jd_list = true; ++ list_add(&dep_atom->jd_item, out_list); ++ } ++ } ++ } ++ } ++} ++ ++/* ++ * Perform the necessary handling of an atom that has finished running ++ * on the GPU. ++ * ++ * Note that if this is a soft-job that has had kbase_prepare_soft_job called on it then the caller ++ * is responsible for calling kbase_finish_soft_job *before* calling this function. ++ * ++ * The caller must hold the kbase_jd_context.lock. ++ */ ++bool jd_done_nolock(struct kbase_jd_atom *katom, ++ struct list_head *completed_jobs_ctx) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ struct kbase_device *kbdev = kctx->kbdev; ++ struct list_head completed_jobs; ++ struct list_head runnable_jobs; ++ bool need_to_try_schedule_context = false; ++ int i; ++ ++ INIT_LIST_HEAD(&completed_jobs); ++ INIT_LIST_HEAD(&runnable_jobs); ++ ++ KBASE_DEBUG_ASSERT(katom->status != KBASE_JD_ATOM_STATE_UNUSED); ++ ++#if MALI_CUSTOMER_RELEASE == 0 ++ jd_check_force_failure(katom); ++#endif ++ ++ /* This is needed in case an atom is failed due to being invalid, this ++ * can happen *before* the jobs that the atom depends on have completed */ ++ for (i = 0; i < 2; i++) { ++ if (kbase_jd_katom_dep_atom(&katom->dep[i])) { ++ list_del(&katom->dep_item[i]); ++ kbase_jd_katom_dep_clear(&katom->dep[i]); ++ } ++ } ++ ++ /* With PRLAM-10817 or PRLAM-10959 the last tile of a fragment job being soft-stopped can fail with ++ * BASE_JD_EVENT_TILE_RANGE_FAULT. ++ * ++ * So here if the fragment job failed with TILE_RANGE_FAULT and it has been soft-stopped, then we promote the ++ * error code to BASE_JD_EVENT_DONE ++ */ ++ ++ if ((kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10817) || kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10959)) && ++ katom->event_code == BASE_JD_EVENT_TILE_RANGE_FAULT) { ++ if ((katom->core_req & BASE_JD_REQ_FS) && (katom->atom_flags & KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED)) { ++ /* Promote the failure to job done */ ++ katom->event_code = BASE_JD_EVENT_DONE; ++ katom->atom_flags = katom->atom_flags & (~KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED); ++ } ++ } ++ ++ katom->status = KBASE_JD_ATOM_STATE_COMPLETED; ++ list_add_tail(&katom->jd_item, &completed_jobs); ++ ++ while (!list_empty(&completed_jobs)) { ++ katom = list_entry(completed_jobs.prev, struct kbase_jd_atom, jd_item); ++ list_del(completed_jobs.prev); ++ KBASE_DEBUG_ASSERT(katom->status == KBASE_JD_ATOM_STATE_COMPLETED); ++ ++ for (i = 0; i < 2; i++) ++ jd_resolve_dep(&runnable_jobs, katom, i, ++ kbase_ctx_flag(kctx, KCTX_DYING)); ++ ++ if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) ++ kbase_jd_post_external_resources(katom); ++ ++ while (!list_empty(&runnable_jobs)) { ++ struct kbase_jd_atom *node; ++ ++ node = list_entry(runnable_jobs.next, ++ struct kbase_jd_atom, jd_item); ++ list_del(runnable_jobs.next); ++ node->in_jd_list = false; ++ ++ KBASE_DEBUG_ASSERT(node->status != KBASE_JD_ATOM_STATE_UNUSED); ++ ++ if (node->status != KBASE_JD_ATOM_STATE_COMPLETED && ++ !kbase_ctx_flag(kctx, KCTX_DYING)) { ++ need_to_try_schedule_context |= jd_run_atom(node); ++ } else { ++ node->event_code = katom->event_code; ++ ++ if ((node->core_req & ++ BASE_JD_REQ_SOFT_JOB_TYPE) == ++ BASE_JD_REQ_SOFT_REPLAY) { ++ if (kbase_replay_process(node)) ++ /* Don't complete this atom */ ++ continue; ++ } else if (node->core_req & ++ BASE_JD_REQ_SOFT_JOB) { ++ /* If this is a fence wait soft job ++ * then remove it from the list of sync ++ * waiters. ++ */ ++ if (BASE_JD_REQ_SOFT_FENCE_WAIT == node->core_req) ++ kbasep_remove_waiting_soft_job(node); ++ ++ kbase_finish_soft_job(node); ++ } ++ node->status = KBASE_JD_ATOM_STATE_COMPLETED; ++ } ++ ++ if (node->status == KBASE_JD_ATOM_STATE_COMPLETED) { ++ list_add_tail(&node->jd_item, &completed_jobs); ++ } else if (node->status == KBASE_JD_ATOM_STATE_IN_JS && ++ !node->will_fail_event_code) { ++ /* Node successfully submitted, try submitting ++ * dependencies as they may now be representable ++ * in JS */ ++ jd_try_submitting_deps(&runnable_jobs, node); ++ } ++ } ++ ++ /* Register a completed job as a disjoint event when the GPU ++ * is in a disjoint state (ie. being reset or replaying jobs). ++ */ ++ kbase_disjoint_event_potential(kctx->kbdev); ++ if (completed_jobs_ctx) ++ list_add_tail(&katom->jd_item, completed_jobs_ctx); ++ else ++ kbase_event_post(kctx, katom); ++ ++ /* Decrement and check the TOTAL number of jobs. This includes ++ * those not tracked by the scheduler: 'not ready to run' and ++ * 'dependency-only' jobs. */ ++ if (--kctx->jctx.job_nr == 0) ++ wake_up(&kctx->jctx.zero_jobs_wait); /* All events are safely queued now, and we can signal any waiter ++ * that we've got no more jobs (so we can be safely terminated) */ ++ } ++ ++ return need_to_try_schedule_context; ++} ++ ++KBASE_EXPORT_TEST_API(jd_done_nolock); ++ ++#ifdef CONFIG_GPU_TRACEPOINTS ++enum { ++ CORE_REQ_DEP_ONLY, ++ CORE_REQ_SOFT, ++ CORE_REQ_COMPUTE, ++ CORE_REQ_FRAGMENT, ++ CORE_REQ_VERTEX, ++ CORE_REQ_TILER, ++ CORE_REQ_FRAGMENT_VERTEX, ++ CORE_REQ_FRAGMENT_VERTEX_TILER, ++ CORE_REQ_FRAGMENT_TILER, ++ CORE_REQ_VERTEX_TILER, ++ CORE_REQ_UNKNOWN ++}; ++static const char * const core_req_strings[] = { ++ "Dependency Only Job", ++ "Soft Job", ++ "Compute Shader Job", ++ "Fragment Shader Job", ++ "Vertex/Geometry Shader Job", ++ "Tiler Job", ++ "Fragment Shader + Vertex/Geometry Shader Job", ++ "Fragment Shader + Vertex/Geometry Shader Job + Tiler Job", ++ "Fragment Shader + Tiler Job", ++ "Vertex/Geometry Shader Job + Tiler Job", ++ "Unknown Job" ++}; ++static const char *kbasep_map_core_reqs_to_string(base_jd_core_req core_req) ++{ ++ if (core_req & BASE_JD_REQ_SOFT_JOB) ++ return core_req_strings[CORE_REQ_SOFT]; ++ if (core_req & BASE_JD_REQ_ONLY_COMPUTE) ++ return core_req_strings[CORE_REQ_COMPUTE]; ++ switch (core_req & (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T)) { ++ case BASE_JD_REQ_DEP: ++ return core_req_strings[CORE_REQ_DEP_ONLY]; ++ case BASE_JD_REQ_FS: ++ return core_req_strings[CORE_REQ_FRAGMENT]; ++ case BASE_JD_REQ_CS: ++ return core_req_strings[CORE_REQ_VERTEX]; ++ case BASE_JD_REQ_T: ++ return core_req_strings[CORE_REQ_TILER]; ++ case (BASE_JD_REQ_FS | BASE_JD_REQ_CS): ++ return core_req_strings[CORE_REQ_FRAGMENT_VERTEX]; ++ case (BASE_JD_REQ_FS | BASE_JD_REQ_T): ++ return core_req_strings[CORE_REQ_FRAGMENT_TILER]; ++ case (BASE_JD_REQ_CS | BASE_JD_REQ_T): ++ return core_req_strings[CORE_REQ_VERTEX_TILER]; ++ case (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T): ++ return core_req_strings[CORE_REQ_FRAGMENT_VERTEX_TILER]; ++ } ++ return core_req_strings[CORE_REQ_UNKNOWN]; ++} ++#endif ++ ++bool jd_submit_atom(struct kbase_context *kctx, const struct base_jd_atom_v2 *user_atom, struct kbase_jd_atom *katom) ++{ ++ struct kbase_jd_context *jctx = &kctx->jctx; ++ int queued = 0; ++ int i; ++ int sched_prio; ++ bool ret; ++ bool will_fail = false; ++ ++ /* Update the TOTAL number of jobs. This includes those not tracked by ++ * the scheduler: 'not ready to run' and 'dependency-only' jobs. */ ++ jctx->job_nr++; ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) ++ katom->start_timestamp.tv64 = 0; ++#else ++ katom->start_timestamp = 0; ++#endif ++ katom->udata = user_atom->udata; ++ katom->kctx = kctx; ++ katom->nr_extres = user_atom->nr_extres; ++ katom->extres = NULL; ++ katom->device_nr = user_atom->device_nr; ++ katom->affinity = 0; ++ katom->jc = user_atom->jc; ++ katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED; ++ katom->core_req = user_atom->core_req; ++ katom->atom_flags = 0; ++ katom->retry_count = 0; ++ katom->need_cache_flush_cores_retained = 0; ++ katom->pre_dep = NULL; ++ katom->post_dep = NULL; ++ katom->x_pre_dep = NULL; ++ katom->x_post_dep = NULL; ++ katom->will_fail_event_code = BASE_JD_EVENT_NOT_STARTED; ++ ++ /* Implicitly sets katom->protected_state.enter as well. */ ++ katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK; ++ ++ katom->age = kctx->age_count++; ++ ++ INIT_LIST_HEAD(&katom->jd_item); ++#ifdef CONFIG_KDS ++ /* Start by assuming that the KDS dependencies are satisfied, ++ * kbase_jd_pre_external_resources will correct this if there are dependencies */ ++ katom->kds_dep_satisfied = true; ++ katom->kds_rset = NULL; ++#endif /* CONFIG_KDS */ ++#ifdef CONFIG_MALI_BIFROST_DMA_FENCE ++ kbase_fence_dep_count_set(katom, -1); ++#endif ++ ++ /* Don't do anything if there is a mess up with dependencies. ++ This is done in a separate cycle to check both the dependencies at ones, otherwise ++ it will be extra complexity to deal with 1st dependency ( just added to the list ) ++ if only the 2nd one has invalid config. ++ */ ++ for (i = 0; i < 2; i++) { ++ int dep_atom_number = user_atom->pre_dep[i].atom_id; ++ base_jd_dep_type dep_atom_type = user_atom->pre_dep[i].dependency_type; ++ ++ if (dep_atom_number) { ++ if (dep_atom_type != BASE_JD_DEP_TYPE_ORDER && ++ dep_atom_type != BASE_JD_DEP_TYPE_DATA) { ++ katom->event_code = BASE_JD_EVENT_JOB_CONFIG_FAULT; ++ katom->status = KBASE_JD_ATOM_STATE_COMPLETED; ++ ++ /* Wrong dependency setup. Atom will be sent ++ * back to user space. Do not record any ++ * dependencies. */ ++ KBASE_TLSTREAM_TL_NEW_ATOM( ++ katom, ++ kbase_jd_atom_id(kctx, katom)); ++ KBASE_TLSTREAM_TL_RET_ATOM_CTX( ++ katom, kctx); ++ KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(katom, ++ TL_ATOM_STATE_IDLE); ++ ++ ret = jd_done_nolock(katom, NULL); ++ goto out; ++ } ++ } ++ } ++ ++ /* Add dependencies */ ++ for (i = 0; i < 2; i++) { ++ int dep_atom_number = user_atom->pre_dep[i].atom_id; ++ base_jd_dep_type dep_atom_type; ++ struct kbase_jd_atom *dep_atom = &jctx->atoms[dep_atom_number]; ++ ++ dep_atom_type = user_atom->pre_dep[i].dependency_type; ++ kbase_jd_katom_dep_clear(&katom->dep[i]); ++ ++ if (!dep_atom_number) ++ continue; ++ ++ if (dep_atom->status == KBASE_JD_ATOM_STATE_UNUSED || ++ dep_atom->status == KBASE_JD_ATOM_STATE_COMPLETED) { ++ ++ if (dep_atom->event_code == BASE_JD_EVENT_DONE) ++ continue; ++ /* don't stop this atom if it has an order dependency ++ * only to the failed one, try to submit it through ++ * the normal path ++ */ ++ if (dep_atom_type == BASE_JD_DEP_TYPE_ORDER && ++ dep_atom->event_code > BASE_JD_EVENT_ACTIVE) { ++ continue; ++ } ++ ++ /* Atom has completed, propagate the error code if any */ ++ katom->event_code = dep_atom->event_code; ++ katom->status = KBASE_JD_ATOM_STATE_QUEUED; ++ ++ /* This atom is going through soft replay or ++ * will be sent back to user space. Do not record any ++ * dependencies. */ ++ KBASE_TLSTREAM_TL_NEW_ATOM( ++ katom, ++ kbase_jd_atom_id(kctx, katom)); ++ KBASE_TLSTREAM_TL_RET_ATOM_CTX(katom, kctx); ++ KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(katom, ++ TL_ATOM_STATE_IDLE); ++ ++ if ((katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) ++ == BASE_JD_REQ_SOFT_REPLAY) { ++ if (kbase_replay_process(katom)) { ++ ret = false; ++ goto out; ++ } ++ } ++ will_fail = true; ++ ++ } else { ++ /* Atom is in progress, add this atom to the list */ ++ list_add_tail(&katom->dep_item[i], &dep_atom->dep_head[i]); ++ kbase_jd_katom_dep_set(&katom->dep[i], dep_atom, dep_atom_type); ++ queued = 1; ++ } ++ } ++ ++ if (will_fail) { ++ if (!queued) { ++ ret = jd_done_nolock(katom, NULL); ++ ++ goto out; ++ } else { ++ katom->will_fail_event_code = katom->event_code; ++ ret = false; ++ ++ goto out; ++ } ++ } else { ++ /* These must occur after the above loop to ensure that an atom ++ * that depends on a previous atom with the same number behaves ++ * as expected */ ++ katom->event_code = BASE_JD_EVENT_DONE; ++ katom->status = KBASE_JD_ATOM_STATE_QUEUED; ++ } ++ ++ /* For invalid priority, be most lenient and choose the default */ ++ sched_prio = kbasep_js_atom_prio_to_sched_prio(user_atom->prio); ++ if (sched_prio == KBASE_JS_ATOM_SCHED_PRIO_INVALID) ++ sched_prio = KBASE_JS_ATOM_SCHED_PRIO_DEFAULT; ++ katom->sched_priority = sched_prio; ++ ++ /* Create a new atom recording all dependencies it was set up with. */ ++ KBASE_TLSTREAM_TL_NEW_ATOM( ++ katom, ++ kbase_jd_atom_id(kctx, katom)); ++ KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(katom, TL_ATOM_STATE_IDLE); ++ KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY(katom, katom->sched_priority); ++ KBASE_TLSTREAM_TL_RET_ATOM_CTX(katom, kctx); ++ for (i = 0; i < 2; i++) ++ if (BASE_JD_DEP_TYPE_INVALID != kbase_jd_katom_dep_type( ++ &katom->dep[i])) { ++ KBASE_TLSTREAM_TL_DEP_ATOM_ATOM( ++ (void *)kbase_jd_katom_dep_atom( ++ &katom->dep[i]), ++ (void *)katom); ++ } else if (BASE_JD_DEP_TYPE_INVALID != ++ user_atom->pre_dep[i].dependency_type) { ++ /* Resolved dependency. */ ++ int dep_atom_number = ++ user_atom->pre_dep[i].atom_id; ++ struct kbase_jd_atom *dep_atom = ++ &jctx->atoms[dep_atom_number]; ++ ++ KBASE_TLSTREAM_TL_RDEP_ATOM_ATOM( ++ (void *)dep_atom, ++ (void *)katom); ++ } ++ ++ /* Reject atoms with job chain = NULL, as these cause issues with soft-stop */ ++ if (!katom->jc && (katom->core_req & BASE_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_DEP) { ++ dev_warn(kctx->kbdev->dev, "Rejecting atom with jc = NULL"); ++ katom->event_code = BASE_JD_EVENT_JOB_INVALID; ++ ret = jd_done_nolock(katom, NULL); ++ goto out; ++ } ++ ++ /* Reject atoms with an invalid device_nr */ ++ if ((katom->core_req & BASE_JD_REQ_SPECIFIC_COHERENT_GROUP) && ++ (katom->device_nr >= kctx->kbdev->gpu_props.num_core_groups)) { ++ dev_warn(kctx->kbdev->dev, ++ "Rejecting atom with invalid device_nr %d", ++ katom->device_nr); ++ katom->event_code = BASE_JD_EVENT_JOB_INVALID; ++ ret = jd_done_nolock(katom, NULL); ++ goto out; ++ } ++ ++ /* Reject atoms with invalid core requirements */ ++ if ((katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) && ++ (katom->core_req & BASE_JD_REQ_EVENT_COALESCE)) { ++ dev_warn(kctx->kbdev->dev, ++ "Rejecting atom with invalid core requirements"); ++ katom->event_code = BASE_JD_EVENT_JOB_INVALID; ++ katom->core_req &= ~BASE_JD_REQ_EVENT_COALESCE; ++ ret = jd_done_nolock(katom, NULL); ++ goto out; ++ } ++ ++ if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) { ++ /* handle what we need to do to access the external resources */ ++ if (kbase_jd_pre_external_resources(katom, user_atom) != 0) { ++ /* setup failed (no access, bad resource, unknown resource types, etc.) */ ++ katom->event_code = BASE_JD_EVENT_JOB_INVALID; ++ ret = jd_done_nolock(katom, NULL); ++ goto out; ++ } ++ } ++ ++ /* Validate the atom. Function will return error if the atom is ++ * malformed. ++ * ++ * Soft-jobs never enter the job scheduler but have their own initialize method. ++ * ++ * If either fail then we immediately complete the atom with an error. ++ */ ++ if ((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0) { ++ if (!kbase_js_is_atom_valid(kctx->kbdev, katom)) { ++ katom->event_code = BASE_JD_EVENT_JOB_INVALID; ++ ret = jd_done_nolock(katom, NULL); ++ goto out; ++ } ++ } else { ++ /* Soft-job */ ++ if (kbase_prepare_soft_job(katom) != 0) { ++ katom->event_code = BASE_JD_EVENT_JOB_INVALID; ++ ret = jd_done_nolock(katom, NULL); ++ goto out; ++ } ++ } ++ ++#ifdef CONFIG_GPU_TRACEPOINTS ++ katom->work_id = atomic_inc_return(&jctx->work_id); ++ trace_gpu_job_enqueue(kctx->id, katom->work_id, ++ kbasep_map_core_reqs_to_string(katom->core_req)); ++#endif ++ ++ if (queued && !IS_GPU_ATOM(katom)) { ++ ret = false; ++ goto out; ++ } ++#ifdef CONFIG_KDS ++ if (!katom->kds_dep_satisfied) { ++ /* Queue atom due to KDS dependency */ ++ ret = false; ++ goto out; ++ } ++#endif /* CONFIG_KDS */ ++ ++ ++#ifdef CONFIG_MALI_BIFROST_DMA_FENCE ++ if (kbase_fence_dep_count_read(katom) != -1) { ++ ret = false; ++ goto out; ++ } ++#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ ++ ++ if ((katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) ++ == BASE_JD_REQ_SOFT_REPLAY) { ++ if (kbase_replay_process(katom)) ++ ret = false; ++ else ++ ret = jd_done_nolock(katom, NULL); ++ ++ goto out; ++ } else if (katom->core_req & BASE_JD_REQ_SOFT_JOB) { ++ if (kbase_process_soft_job(katom) == 0) { ++ kbase_finish_soft_job(katom); ++ ret = jd_done_nolock(katom, NULL); ++ goto out; ++ } ++ ++ ret = false; ++ } else if ((katom->core_req & BASE_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_DEP) { ++ katom->status = KBASE_JD_ATOM_STATE_IN_JS; ++ ret = kbasep_js_add_job(kctx, katom); ++ /* If job was cancelled then resolve immediately */ ++ if (katom->event_code == BASE_JD_EVENT_JOB_CANCELLED) ++ ret = jd_done_nolock(katom, NULL); ++ } else { ++ /* This is a pure dependency. Resolve it immediately */ ++ ret = jd_done_nolock(katom, NULL); ++ } ++ ++ out: ++ return ret; ++} ++ ++int kbase_jd_submit(struct kbase_context *kctx, ++ void __user *user_addr, u32 nr_atoms, u32 stride, ++ bool uk6_atom) ++{ ++ struct kbase_jd_context *jctx = &kctx->jctx; ++ int err = 0; ++ int i; ++ bool need_to_try_schedule_context = false; ++ struct kbase_device *kbdev; ++ u32 latest_flush; ++ ++ /* ++ * kbase_jd_submit isn't expected to fail and so all errors with the ++ * jobs are reported by immediately failing them (through event system) ++ */ ++ kbdev = kctx->kbdev; ++ ++ beenthere(kctx, "%s", "Enter"); ++ ++ if (kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) { ++ dev_err(kbdev->dev, "Attempt to submit to a context that has SUBMIT_DISABLED set on it"); ++ return -EINVAL; ++ } ++ ++ if (stride != sizeof(base_jd_atom_v2)) { ++ dev_err(kbdev->dev, "Stride passed to job_submit doesn't match kernel"); ++ return -EINVAL; ++ } ++ ++ KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, atomic_add_return(nr_atoms, ++ &kctx->timeline.jd_atoms_in_flight)); ++ ++ /* All atoms submitted in this call have the same flush ID */ ++ latest_flush = kbase_backend_get_current_flush_id(kbdev); ++ ++ for (i = 0; i < nr_atoms; i++) { ++ struct base_jd_atom_v2 user_atom; ++ struct kbase_jd_atom *katom; ++ ++ if (copy_from_user(&user_atom, user_addr, ++ sizeof(user_atom)) != 0) { ++ err = -EINVAL; ++ KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, ++ atomic_sub_return(nr_atoms - i, ++ &kctx->timeline.jd_atoms_in_flight)); ++ break; ++ } ++ ++#ifdef BASE_LEGACY_UK10_2_SUPPORT ++ if (KBASE_API_VERSION(10, 3) > kctx->api_version) ++ user_atom.core_req = (u32)(user_atom.compat_core_req ++ & 0x7fff); ++#endif /* BASE_LEGACY_UK10_2_SUPPORT */ ++ ++ user_addr = (void __user *)((uintptr_t) user_addr + stride); ++ ++ mutex_lock(&jctx->lock); ++#ifndef compiletime_assert ++#define compiletime_assert_defined ++#define compiletime_assert(x, msg) do { switch (0) { case 0: case (x):; } } \ ++while (false) ++#endif ++ compiletime_assert((1 << (8*sizeof(user_atom.atom_number))) == ++ BASE_JD_ATOM_COUNT, ++ "BASE_JD_ATOM_COUNT and base_atom_id type out of sync"); ++ compiletime_assert(sizeof(user_atom.pre_dep[0].atom_id) == ++ sizeof(user_atom.atom_number), ++ "BASE_JD_ATOM_COUNT and base_atom_id type out of sync"); ++#ifdef compiletime_assert_defined ++#undef compiletime_assert ++#undef compiletime_assert_defined ++#endif ++ katom = &jctx->atoms[user_atom.atom_number]; ++ ++ /* Record the flush ID for the cache flush optimisation */ ++ katom->flush_id = latest_flush; ++ ++ while (katom->status != KBASE_JD_ATOM_STATE_UNUSED) { ++ /* Atom number is already in use, wait for the atom to ++ * complete ++ */ ++ mutex_unlock(&jctx->lock); ++ ++ /* This thread will wait for the atom to complete. Due ++ * to thread scheduling we are not sure that the other ++ * thread that owns the atom will also schedule the ++ * context, so we force the scheduler to be active and ++ * hence eventually schedule this context at some point ++ * later. ++ */ ++ kbase_js_sched_all(kbdev); ++ ++ if (wait_event_killable(katom->completed, ++ katom->status == ++ KBASE_JD_ATOM_STATE_UNUSED) != 0) { ++ /* We're being killed so the result code ++ * doesn't really matter ++ */ ++ return 0; ++ } ++ mutex_lock(&jctx->lock); ++ } ++ ++ need_to_try_schedule_context |= ++ jd_submit_atom(kctx, &user_atom, katom); ++ ++ /* Register a completed job as a disjoint event when the GPU is in a disjoint state ++ * (ie. being reset or replaying jobs). ++ */ ++ kbase_disjoint_event_potential(kbdev); ++ ++ mutex_unlock(&jctx->lock); ++ } ++ ++ if (need_to_try_schedule_context) ++ kbase_js_sched_all(kbdev); ++ ++ return err; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_jd_submit); ++ ++void kbase_jd_done_worker(struct work_struct *data) ++{ ++ struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, work); ++ struct kbase_jd_context *jctx; ++ struct kbase_context *kctx; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ struct kbase_device *kbdev; ++ struct kbasep_js_device_data *js_devdata; ++ u64 cache_jc = katom->jc; ++ struct kbasep_js_atom_retained_state katom_retained_state; ++ bool context_idle; ++ base_jd_core_req core_req = katom->core_req; ++ u64 affinity = katom->affinity; ++ enum kbase_atom_coreref_state coreref_state = katom->coreref_state; ++ ++ /* Soft jobs should never reach this function */ ++ KBASE_DEBUG_ASSERT((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0); ++ ++ kctx = katom->kctx; ++ jctx = &kctx->jctx; ++ kbdev = kctx->kbdev; ++ js_kctx_info = &kctx->jctx.sched_info; ++ js_devdata = &kbdev->js_data; ++ ++ KBASE_TRACE_ADD(kbdev, JD_DONE_WORKER, kctx, katom, katom->jc, 0); ++ ++ kbase_backend_complete_wq(kbdev, katom); ++ ++ /* ++ * Begin transaction on JD context and JS context ++ */ ++ mutex_lock(&jctx->lock); ++ KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(katom, TL_ATOM_STATE_DONE); ++ mutex_lock(&js_devdata->queue_mutex); ++ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); ++ ++ /* This worker only gets called on contexts that are scheduled *in*. This is ++ * because it only happens in response to an IRQ from a job that was ++ * running. ++ */ ++ KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ ++ if (katom->event_code == BASE_JD_EVENT_STOPPED) { ++ /* Atom has been promoted to stopped */ ++ unsigned long flags; ++ ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ mutex_unlock(&js_devdata->queue_mutex); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ katom->status = KBASE_JD_ATOM_STATE_IN_JS; ++ kbase_js_unpull(kctx, katom); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&jctx->lock); ++ ++ return; ++ } ++ ++ if (katom->event_code != BASE_JD_EVENT_DONE) ++ dev_err(kbdev->dev, ++ "t6xx: GPU fault 0x%02lx from job slot %d\n", ++ (unsigned long)katom->event_code, ++ katom->slot_nr); ++ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316)) ++ kbase_as_poking_timer_release_atom(kbdev, kctx, katom); ++ ++ /* Retain state before the katom disappears */ ++ kbasep_js_atom_retained_state_copy(&katom_retained_state, katom); ++ ++ context_idle = kbase_js_complete_atom_wq(kctx, katom); ++ ++ KBASE_DEBUG_ASSERT(kbasep_js_has_atom_finished(&katom_retained_state)); ++ ++ kbasep_js_remove_job(kbdev, kctx, katom); ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ mutex_unlock(&js_devdata->queue_mutex); ++ katom->atom_flags &= ~KBASE_KATOM_FLAG_HOLDING_CTX_REF; ++ /* jd_done_nolock() requires the jsctx_mutex lock to be dropped */ ++ jd_done_nolock(katom, &kctx->completed_jobs); ++ ++ /* katom may have been freed now, do not use! */ ++ ++ if (context_idle) { ++ unsigned long flags; ++ ++ context_idle = false; ++ mutex_lock(&js_devdata->queue_mutex); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ /* If kbase_sched() has scheduled this context back in then ++ * KCTX_ACTIVE will have been set after we marked it as ++ * inactive, and another pm reference will have been taken, so ++ * drop our reference. But do not call kbase_jm_idle_ctx(), as ++ * the context is active and fast-starting is allowed. ++ * ++ * If an atom has been fast-started then kctx->atoms_pulled will ++ * be non-zero but KCTX_ACTIVE will still be false (as the ++ * previous pm reference has been inherited). Do NOT drop our ++ * reference, as it has been re-used, and leave the context as ++ * active. ++ * ++ * If no new atoms have been started then KCTX_ACTIVE will still ++ * be false and atoms_pulled will be zero, so drop the reference ++ * and call kbase_jm_idle_ctx(). ++ * ++ * As the checks are done under both the queue_mutex and ++ * hwaccess_lock is should be impossible for this to race ++ * with the scheduler code. ++ */ ++ if (kbase_ctx_flag(kctx, KCTX_ACTIVE) || ++ !atomic_read(&kctx->atoms_pulled)) { ++ /* Calling kbase_jm_idle_ctx() here will ensure that ++ * atoms are not fast-started when we drop the ++ * hwaccess_lock. This is not performed if ++ * KCTX_ACTIVE is set as in that case another pm ++ * reference has been taken and a fast-start would be ++ * valid. ++ */ ++ if (!kbase_ctx_flag(kctx, KCTX_ACTIVE)) ++ kbase_jm_idle_ctx(kbdev, kctx); ++ context_idle = true; ++ } else { ++ kbase_ctx_flag_set(kctx, KCTX_ACTIVE); ++ } ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&js_devdata->queue_mutex); ++ } ++ ++ /* ++ * Transaction complete ++ */ ++ mutex_unlock(&jctx->lock); ++ ++ /* Job is now no longer running, so can now safely release the context ++ * reference, and handle any actions that were logged against the atom's retained state */ ++ ++ kbasep_js_runpool_release_ctx_and_katom_retained_state(kbdev, kctx, &katom_retained_state); ++ ++ kbase_js_sched_all(kbdev); ++ ++ if (!atomic_dec_return(&kctx->work_count)) { ++ /* If worker now idle then post all events that jd_done_nolock() ++ * has queued */ ++ mutex_lock(&jctx->lock); ++ while (!list_empty(&kctx->completed_jobs)) { ++ struct kbase_jd_atom *atom = list_entry( ++ kctx->completed_jobs.next, ++ struct kbase_jd_atom, jd_item); ++ list_del(kctx->completed_jobs.next); ++ ++ kbase_event_post(kctx, atom); ++ } ++ mutex_unlock(&jctx->lock); ++ } ++ ++ kbase_backend_complete_wq_post_sched(kbdev, core_req, affinity, ++ coreref_state); ++ ++ if (context_idle) ++ kbase_pm_context_idle(kbdev); ++ ++ KBASE_TRACE_ADD(kbdev, JD_DONE_WORKER_END, kctx, NULL, cache_jc, 0); ++} ++ ++/** ++ * jd_cancel_worker - Work queue job cancel function. ++ * @data: a &struct work_struct ++ * ++ * Only called as part of 'Zapping' a context (which occurs on termination). ++ * Operates serially with the kbase_jd_done_worker() on the work queue. ++ * ++ * This can only be called on contexts that aren't scheduled. ++ * ++ * We don't need to release most of the resources that would occur on ++ * kbase_jd_done() or kbase_jd_done_worker(), because the atoms here must not be ++ * running (by virtue of only being called on contexts that aren't ++ * scheduled). ++ */ ++static void jd_cancel_worker(struct work_struct *data) ++{ ++ struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, work); ++ struct kbase_jd_context *jctx; ++ struct kbase_context *kctx; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ bool need_to_try_schedule_context; ++ bool attr_state_changed; ++ struct kbase_device *kbdev; ++ ++ /* Soft jobs should never reach this function */ ++ KBASE_DEBUG_ASSERT((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0); ++ ++ kctx = katom->kctx; ++ kbdev = kctx->kbdev; ++ jctx = &kctx->jctx; ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ KBASE_TRACE_ADD(kbdev, JD_CANCEL_WORKER, kctx, katom, katom->jc, 0); ++ ++ /* This only gets called on contexts that are scheduled out. Hence, we must ++ * make sure we don't de-ref the number of running jobs (there aren't ++ * any), nor must we try to schedule out the context (it's already ++ * scheduled out). ++ */ ++ KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ ++ /* Scheduler: Remove the job from the system */ ++ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); ++ attr_state_changed = kbasep_js_remove_cancelled_job(kbdev, kctx, katom); ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ ++ mutex_lock(&jctx->lock); ++ ++ need_to_try_schedule_context = jd_done_nolock(katom, NULL); ++ /* Because we're zapping, we're not adding any more jobs to this ctx, so no need to ++ * schedule the context. There's also no need for the jsctx_mutex to have been taken ++ * around this too. */ ++ KBASE_DEBUG_ASSERT(!need_to_try_schedule_context); ++ ++ /* katom may have been freed now, do not use! */ ++ mutex_unlock(&jctx->lock); ++ ++ if (attr_state_changed) ++ kbase_js_sched_all(kbdev); ++} ++ ++/** ++ * kbase_jd_done - Complete a job that has been removed from the Hardware ++ * @katom: atom which has been completed ++ * @slot_nr: slot the atom was on ++ * @end_timestamp: completion time ++ * @done_code: completion code ++ * ++ * This must be used whenever a job has been removed from the Hardware, e.g.: ++ * An IRQ indicates that the job finished (for both error and 'done' codes), or ++ * the job was evicted from the JS_HEAD_NEXT registers during a Soft/Hard stop. ++ * ++ * Some work is carried out immediately, and the rest is deferred onto a ++ * workqueue ++ * ++ * Context: ++ * This can be called safely from atomic context. ++ * The caller must hold kbdev->hwaccess_lock ++ */ ++void kbase_jd_done(struct kbase_jd_atom *katom, int slot_nr, ++ ktime_t *end_timestamp, kbasep_js_atom_done_code done_code) ++{ ++ struct kbase_context *kctx; ++ struct kbase_device *kbdev; ++ ++ KBASE_DEBUG_ASSERT(katom); ++ kctx = katom->kctx; ++ KBASE_DEBUG_ASSERT(kctx); ++ kbdev = kctx->kbdev; ++ KBASE_DEBUG_ASSERT(kbdev); ++ ++ if (done_code & KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT) ++ katom->event_code = BASE_JD_EVENT_REMOVED_FROM_NEXT; ++ ++ KBASE_TRACE_ADD(kbdev, JD_DONE, kctx, katom, katom->jc, 0); ++ ++ kbase_job_check_leave_disjoint(kbdev, katom); ++ ++ katom->slot_nr = slot_nr; ++ ++ atomic_inc(&kctx->work_count); ++ ++#ifdef CONFIG_DEBUG_FS ++ /* a failed job happened and is waiting for dumping*/ ++ if (!katom->will_fail_event_code && ++ kbase_debug_job_fault_process(katom, katom->event_code)) ++ return; ++#endif ++ ++ WARN_ON(work_pending(&katom->work)); ++ KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&katom->work)); ++ INIT_WORK(&katom->work, kbase_jd_done_worker); ++ queue_work(kctx->jctx.job_done_wq, &katom->work); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_jd_done); ++ ++void kbase_jd_cancel(struct kbase_device *kbdev, struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx; ++ ++ KBASE_DEBUG_ASSERT(NULL != kbdev); ++ KBASE_DEBUG_ASSERT(NULL != katom); ++ kctx = katom->kctx; ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ ++ KBASE_TRACE_ADD(kbdev, JD_CANCEL, kctx, katom, katom->jc, 0); ++ ++ /* This should only be done from a context that is not scheduled */ ++ KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ ++ WARN_ON(work_pending(&katom->work)); ++ ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ ++ KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&katom->work)); ++ INIT_WORK(&katom->work, jd_cancel_worker); ++ queue_work(kctx->jctx.job_done_wq, &katom->work); ++} ++ ++ ++void kbase_jd_zap_context(struct kbase_context *kctx) ++{ ++ struct kbase_jd_atom *katom; ++ struct list_head *entry, *tmp; ++ struct kbase_device *kbdev; ++ ++ KBASE_DEBUG_ASSERT(kctx); ++ ++ kbdev = kctx->kbdev; ++ ++ KBASE_TRACE_ADD(kbdev, JD_ZAP_CONTEXT, kctx, NULL, 0u, 0u); ++ ++ kbase_js_zap_context(kctx); ++ ++ mutex_lock(&kctx->jctx.lock); ++ ++ /* ++ * While holding the struct kbase_jd_context lock clean up jobs which are known to kbase but are ++ * queued outside the job scheduler. ++ */ ++ ++ del_timer_sync(&kctx->soft_job_timeout); ++ list_for_each_safe(entry, tmp, &kctx->waiting_soft_jobs) { ++ katom = list_entry(entry, struct kbase_jd_atom, queue); ++ kbase_cancel_soft_job(katom); ++ } ++ ++ ++#ifdef CONFIG_KDS ++ ++ /* For each job waiting on a kds resource, cancel the wait and force the job to ++ * complete early, this is done so that we don't leave jobs outstanding waiting ++ * on kds resources which may never be released when contexts are zapped, resulting ++ * in a hang. ++ * ++ * Note that we can safely iterate over the list as the struct kbase_jd_context lock is held, ++ * this prevents items being removed when calling job_done_nolock in kbase_cancel_kds_wait_job. ++ */ ++ ++ list_for_each(entry, &kctx->waiting_kds_resource) { ++ katom = list_entry(entry, struct kbase_jd_atom, node); ++ ++ kbase_cancel_kds_wait_job(katom); ++ } ++#endif ++ ++#ifdef CONFIG_MALI_BIFROST_DMA_FENCE ++ kbase_dma_fence_cancel_all_atoms(kctx); ++#endif ++ ++ mutex_unlock(&kctx->jctx.lock); ++ ++#ifdef CONFIG_MALI_BIFROST_DMA_FENCE ++ /* Flush dma-fence workqueue to ensure that any callbacks that may have ++ * been queued are done before continuing. ++ */ ++ flush_workqueue(kctx->dma_fence.wq); ++#endif ++ ++ kbase_jm_wait_for_zero_jobs(kctx); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_jd_zap_context); ++ ++int kbase_jd_init(struct kbase_context *kctx) ++{ ++ int i; ++ int mali_err = 0; ++#ifdef CONFIG_KDS ++ int err; ++#endif /* CONFIG_KDS */ ++ ++ KBASE_DEBUG_ASSERT(kctx); ++ ++ kctx->jctx.job_done_wq = alloc_workqueue("mali_jd", ++ WQ_HIGHPRI | WQ_UNBOUND, 1); ++ if (NULL == kctx->jctx.job_done_wq) { ++ mali_err = -ENOMEM; ++ goto out1; ++ } ++ ++ for (i = 0; i < BASE_JD_ATOM_COUNT; i++) { ++ init_waitqueue_head(&kctx->jctx.atoms[i].completed); ++ ++ INIT_LIST_HEAD(&kctx->jctx.atoms[i].dep_head[0]); ++ INIT_LIST_HEAD(&kctx->jctx.atoms[i].dep_head[1]); ++ ++ /* Catch userspace attempting to use an atom which doesn't exist as a pre-dependency */ ++ kctx->jctx.atoms[i].event_code = BASE_JD_EVENT_JOB_INVALID; ++ kctx->jctx.atoms[i].status = KBASE_JD_ATOM_STATE_UNUSED; ++ ++#if defined(CONFIG_MALI_BIFROST_DMA_FENCE) || defined(CONFIG_SYNC_FILE) ++ kctx->jctx.atoms[i].dma_fence.context = ++ dma_fence_context_alloc(1); ++ atomic_set(&kctx->jctx.atoms[i].dma_fence.seqno, 0); ++ INIT_LIST_HEAD(&kctx->jctx.atoms[i].dma_fence.callbacks); ++#endif ++ } ++ ++ mutex_init(&kctx->jctx.lock); ++ ++ init_waitqueue_head(&kctx->jctx.zero_jobs_wait); ++ ++ spin_lock_init(&kctx->jctx.tb_lock); ++ ++#ifdef CONFIG_KDS ++ err = kds_callback_init(&kctx->jctx.kds_cb, 0, kds_dep_clear); ++ if (0 != err) { ++ mali_err = -EINVAL; ++ goto out2; ++ } ++#endif /* CONFIG_KDS */ ++ ++ kctx->jctx.job_nr = 0; ++ INIT_LIST_HEAD(&kctx->completed_jobs); ++ atomic_set(&kctx->work_count, 0); ++ ++ return 0; ++ ++#ifdef CONFIG_KDS ++ out2: ++ destroy_workqueue(kctx->jctx.job_done_wq); ++#endif /* CONFIG_KDS */ ++ out1: ++ return mali_err; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_jd_init); ++ ++void kbase_jd_exit(struct kbase_context *kctx) ++{ ++ KBASE_DEBUG_ASSERT(kctx); ++ ++#ifdef CONFIG_KDS ++ kds_callback_term(&kctx->jctx.kds_cb); ++#endif /* CONFIG_KDS */ ++ /* Work queue is emptied by this */ ++ destroy_workqueue(kctx->jctx.job_done_wq); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_jd_exit); +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_jd_debugfs.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_jd_debugfs.c +new file mode 100755 +index 000000000000..fed4ad5816ab +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_jd_debugfs.c +@@ -0,0 +1,235 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifdef CONFIG_DEBUG_FS ++ ++#include ++#include ++#include ++#include ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++#include ++#endif ++ ++struct kbase_jd_debugfs_depinfo { ++ u8 id; ++ char type; ++}; ++ ++static void kbase_jd_debugfs_fence_info(struct kbase_jd_atom *atom, ++ struct seq_file *sfile) ++{ ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ struct kbase_sync_fence_info info; ++ int res; ++ ++ switch (atom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { ++ case BASE_JD_REQ_SOFT_FENCE_TRIGGER: ++ res = kbase_sync_fence_out_info_get(atom, &info); ++ if (0 == res) { ++ seq_printf(sfile, "Sa([%p]%d) ", ++ info.fence, info.status); ++ break; ++ } ++ case BASE_JD_REQ_SOFT_FENCE_WAIT: ++ res = kbase_sync_fence_in_info_get(atom, &info); ++ if (0 == res) { ++ seq_printf(sfile, "Wa([%p]%d) ", ++ info.fence, info.status); ++ break; ++ } ++ default: ++ break; ++ } ++#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ ++ ++#ifdef CONFIG_MALI_BIFROST_DMA_FENCE ++ if (atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) { ++ struct kbase_fence_cb *cb; ++ ++ if (atom->dma_fence.fence) { ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ struct fence *fence = atom->dma_fence.fence; ++#else ++ struct dma_fence *fence = atom->dma_fence.fence; ++#endif ++ ++ seq_printf(sfile, ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0)) ++ "Sd(%u#%u: %s) ", ++#else ++ "Sd(%llu#%u: %s) ", ++#endif ++ fence->context, ++ fence->seqno, ++ dma_fence_is_signaled(fence) ? ++ "signaled" : "active"); ++ } ++ ++ list_for_each_entry(cb, &atom->dma_fence.callbacks, ++ node) { ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ struct fence *fence = cb->fence; ++#else ++ struct dma_fence *fence = cb->fence; ++#endif ++ ++ seq_printf(sfile, ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0)) ++ "Wd(%u#%u: %s) ", ++#else ++ "Wd(%llu#%u: %s) ", ++#endif ++ fence->context, ++ fence->seqno, ++ dma_fence_is_signaled(fence) ? ++ "signaled" : "active"); ++ } ++ } ++#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ ++ ++} ++ ++static void kbasep_jd_debugfs_atom_deps( ++ struct kbase_jd_debugfs_depinfo *deps, ++ struct kbase_jd_atom *atom) ++{ ++ struct kbase_context *kctx = atom->kctx; ++ int i; ++ ++ for (i = 0; i < 2; i++) { ++ deps[i].id = (unsigned)(atom->dep[i].atom ? ++ kbase_jd_atom_id(kctx, atom->dep[i].atom) : 0); ++ ++ switch (atom->dep[i].dep_type) { ++ case BASE_JD_DEP_TYPE_INVALID: ++ deps[i].type = ' '; ++ break; ++ case BASE_JD_DEP_TYPE_DATA: ++ deps[i].type = 'D'; ++ break; ++ case BASE_JD_DEP_TYPE_ORDER: ++ deps[i].type = '>'; ++ break; ++ default: ++ deps[i].type = '?'; ++ break; ++ } ++ } ++} ++/** ++ * kbasep_jd_debugfs_atoms_show - Show callback for the JD atoms debugfs file. ++ * @sfile: The debugfs entry ++ * @data: Data associated with the entry ++ * ++ * This function is called to get the contents of the JD atoms debugfs file. ++ * This is a report of all atoms managed by kbase_jd_context.atoms ++ * ++ * Return: 0 if successfully prints data in debugfs entry file, failure ++ * otherwise ++ */ ++static int kbasep_jd_debugfs_atoms_show(struct seq_file *sfile, void *data) ++{ ++ struct kbase_context *kctx = sfile->private; ++ struct kbase_jd_atom *atoms; ++ unsigned long irq_flags; ++ int i; ++ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ ++ /* Print version */ ++ seq_printf(sfile, "v%u\n", MALI_JD_DEBUGFS_VERSION); ++ ++ /* Print U/K API version */ ++ seq_printf(sfile, "ukv%u.%u\n", BASE_UK_VERSION_MAJOR, ++ BASE_UK_VERSION_MINOR); ++ ++ /* Print table heading */ ++ seq_puts(sfile, " ID, Core req, St, CR, Predeps, Start time, Additional info...\n"); ++ ++ atoms = kctx->jctx.atoms; ++ /* General atom states */ ++ mutex_lock(&kctx->jctx.lock); ++ /* JS-related states */ ++ spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, irq_flags); ++ for (i = 0; i != BASE_JD_ATOM_COUNT; ++i) { ++ struct kbase_jd_atom *atom = &atoms[i]; ++ s64 start_timestamp = 0; ++ struct kbase_jd_debugfs_depinfo deps[2]; ++ ++ if (atom->status == KBASE_JD_ATOM_STATE_UNUSED) ++ continue; ++ ++ /* start_timestamp is cleared as soon as the atom leaves UNUSED state ++ * and set before a job is submitted to the h/w, a non-zero value means ++ * it is valid */ ++ if (ktime_to_ns(atom->start_timestamp)) ++ start_timestamp = ktime_to_ns( ++ ktime_sub(ktime_get(), atom->start_timestamp)); ++ ++ kbasep_jd_debugfs_atom_deps(deps, atom); ++ ++ seq_printf(sfile, ++ "%3u, %8x, %2u, %2u, %c%3u %c%3u, %20lld, ", ++ i, atom->core_req, atom->status, ++ atom->coreref_state, ++ deps[0].type, deps[0].id, ++ deps[1].type, deps[1].id, ++ start_timestamp); ++ ++ ++ kbase_jd_debugfs_fence_info(atom, sfile); ++ ++ seq_puts(sfile, "\n"); ++ } ++ spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, irq_flags); ++ mutex_unlock(&kctx->jctx.lock); ++ ++ return 0; ++} ++ ++ ++/** ++ * kbasep_jd_debugfs_atoms_open - open operation for atom debugfs file ++ * @in: &struct inode pointer ++ * @file: &struct file pointer ++ * ++ * Return: file descriptor ++ */ ++static int kbasep_jd_debugfs_atoms_open(struct inode *in, struct file *file) ++{ ++ return single_open(file, kbasep_jd_debugfs_atoms_show, in->i_private); ++} ++ ++static const struct file_operations kbasep_jd_debugfs_atoms_fops = { ++ .open = kbasep_jd_debugfs_atoms_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = single_release, ++}; ++ ++void kbasep_jd_debugfs_ctx_init(struct kbase_context *kctx) ++{ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ ++ /* Expose all atoms */ ++ debugfs_create_file("atoms", S_IRUGO, kctx->kctx_dentry, kctx, ++ &kbasep_jd_debugfs_atoms_fops); ++ ++} ++ ++#endif /* CONFIG_DEBUG_FS */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_jd_debugfs.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_jd_debugfs.h +new file mode 100755 +index 000000000000..fae32919b22f +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_jd_debugfs.h +@@ -0,0 +1,40 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/** ++ * @file mali_kbase_jd_debugfs.h ++ * Header file for job dispatcher-related entries in debugfs ++ */ ++ ++#ifndef _KBASE_JD_DEBUGFS_H ++#define _KBASE_JD_DEBUGFS_H ++ ++#include ++ ++#define MALI_JD_DEBUGFS_VERSION 2 ++ ++/* Forward declarations */ ++struct kbase_context; ++ ++/** ++ * kbasep_jd_debugfs_ctx_init() - Add debugfs entries for JD system ++ * ++ * @kctx Pointer to kbase_context ++ */ ++void kbasep_jd_debugfs_ctx_init(struct kbase_context *kctx); ++ ++#endif /*_KBASE_JD_DEBUGFS_H*/ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_jm.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_jm.c +new file mode 100755 +index 000000000000..0c5c6a6f78cb +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_jm.c +@@ -0,0 +1,131 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++/* ++ * HW access job manager common APIs ++ */ ++ ++#include ++#include "mali_kbase_hwaccess_jm.h" ++#include "mali_kbase_jm.h" ++ ++/** ++ * kbase_jm_next_job() - Attempt to run the next @nr_jobs_to_submit jobs on slot ++ * @js on the active context. ++ * @kbdev: Device pointer ++ * @js: Job slot to run on ++ * @nr_jobs_to_submit: Number of jobs to attempt to submit ++ * ++ * Return: true if slot can still be submitted on, false if slot is now full. ++ */ ++static bool kbase_jm_next_job(struct kbase_device *kbdev, int js, ++ int nr_jobs_to_submit) ++{ ++ struct kbase_context *kctx; ++ int i; ++ ++ kctx = kbdev->hwaccess.active_kctx; ++ ++ if (!kctx) ++ return true; ++ ++ for (i = 0; i < nr_jobs_to_submit; i++) { ++ struct kbase_jd_atom *katom = kbase_js_pull(kctx, js); ++ ++ if (!katom) ++ return true; /* Context has no jobs on this slot */ ++ ++ kbase_backend_run_atom(kbdev, katom); ++ } ++ ++ return false; /* Slot ringbuffer should now be full */ ++} ++ ++u32 kbase_jm_kick(struct kbase_device *kbdev, u32 js_mask) ++{ ++ u32 ret_mask = 0; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ while (js_mask) { ++ int js = ffs(js_mask) - 1; ++ int nr_jobs_to_submit = kbase_backend_slot_free(kbdev, js); ++ ++ if (kbase_jm_next_job(kbdev, js, nr_jobs_to_submit)) ++ ret_mask |= (1 << js); ++ ++ js_mask &= ~(1 << js); ++ } ++ ++ return ret_mask; ++} ++ ++void kbase_jm_try_kick(struct kbase_device *kbdev, u32 js_mask) ++{ ++ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (!down_trylock(&js_devdata->schedule_sem)) { ++ kbase_jm_kick(kbdev, js_mask); ++ up(&js_devdata->schedule_sem); ++ } ++} ++ ++void kbase_jm_try_kick_all(struct kbase_device *kbdev) ++{ ++ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (!down_trylock(&js_devdata->schedule_sem)) { ++ kbase_jm_kick_all(kbdev); ++ up(&js_devdata->schedule_sem); ++ } ++} ++ ++void kbase_jm_idle_ctx(struct kbase_device *kbdev, struct kbase_context *kctx) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (kbdev->hwaccess.active_kctx == kctx) ++ kbdev->hwaccess.active_kctx = NULL; ++} ++ ++struct kbase_jd_atom *kbase_jm_return_atom_to_js(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (katom->event_code != BASE_JD_EVENT_STOPPED && ++ katom->event_code != BASE_JD_EVENT_REMOVED_FROM_NEXT) { ++ return kbase_js_complete_atom(katom, NULL); ++ } else { ++ kbase_js_unpull(katom->kctx, katom); ++ return NULL; ++ } ++} ++ ++struct kbase_jd_atom *kbase_jm_complete(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom, ktime_t *end_timestamp) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ return kbase_js_complete_atom(katom, end_timestamp); ++} ++ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_jm.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_jm.h +new file mode 100755 +index 000000000000..a74ee24c8058 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_jm.h +@@ -0,0 +1,110 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014, 2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++/* ++ * Job manager common APIs ++ */ ++ ++#ifndef _KBASE_JM_H_ ++#define _KBASE_JM_H_ ++ ++/** ++ * kbase_jm_kick() - Indicate that there are jobs ready to run. ++ * @kbdev: Device pointer ++ * @js_mask: Mask of the job slots that can be pulled from. ++ * ++ * Caller must hold the hwaccess_lock and schedule_sem semaphore ++ * ++ * Return: Mask of the job slots that can still be submitted to. ++ */ ++u32 kbase_jm_kick(struct kbase_device *kbdev, u32 js_mask); ++ ++/** ++ * kbase_jm_kick_all() - Indicate that there are jobs ready to run on all job ++ * slots. ++ * @kbdev: Device pointer ++ * ++ * Caller must hold the hwaccess_lock and schedule_sem semaphore ++ * ++ * Return: Mask of the job slots that can still be submitted to. ++ */ ++static inline u32 kbase_jm_kick_all(struct kbase_device *kbdev) ++{ ++ return kbase_jm_kick(kbdev, (1 << kbdev->gpu_props.num_job_slots) - 1); ++} ++ ++/** ++ * kbase_jm_try_kick - Attempt to call kbase_jm_kick ++ * @kbdev: Device pointer ++ * @js_mask: Mask of the job slots that can be pulled from ++ * Context: Caller must hold hwaccess_lock ++ * ++ * If schedule_sem can be immediately obtained then this function will call ++ * kbase_jm_kick() otherwise it will do nothing. ++ */ ++void kbase_jm_try_kick(struct kbase_device *kbdev, u32 js_mask); ++ ++/** ++ * kbase_jm_try_kick_all() - Attempt to call kbase_jm_kick_all ++ * @kbdev: Device pointer ++ * Context: Caller must hold hwaccess_lock ++ * ++ * If schedule_sem can be immediately obtained then this function will call ++ * kbase_jm_kick_all() otherwise it will do nothing. ++ */ ++void kbase_jm_try_kick_all(struct kbase_device *kbdev); ++ ++/** ++ * kbase_jm_idle_ctx() - Mark a context as idle. ++ * @kbdev: Device pointer ++ * @kctx: Context to mark as idle ++ * ++ * No more atoms will be pulled from this context until it is marked as active ++ * by kbase_js_use_ctx(). ++ * ++ * The context should have no atoms currently pulled from it ++ * (kctx->atoms_pulled == 0). ++ * ++ * Caller must hold the hwaccess_lock ++ */ ++void kbase_jm_idle_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); ++ ++/** ++ * kbase_jm_return_atom_to_js() - Return an atom to the job scheduler that has ++ * been soft-stopped or will fail due to a ++ * dependency ++ * @kbdev: Device pointer ++ * @katom: Atom that has been stopped or will be failed ++ * ++ * Return: Atom that has now been unblocked and can now be run, or NULL if none ++ */ ++struct kbase_jd_atom *kbase_jm_return_atom_to_js(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom); ++ ++/** ++ * kbase_jm_complete() - Complete an atom ++ * @kbdev: Device pointer ++ * @katom: Atom that has completed ++ * @end_timestamp: Timestamp of atom completion ++ * ++ * Return: Atom that has now been unblocked and can now be run, or NULL if none ++ */ ++struct kbase_jd_atom *kbase_jm_complete(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom, ktime_t *end_timestamp); ++ ++#endif /* _KBASE_JM_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_js.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_js.c +new file mode 100755 +index 000000000000..677e438aedfa +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_js.c +@@ -0,0 +1,2798 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * Job Scheduler Implementation ++ */ ++#include ++#include ++#if defined(CONFIG_MALI_BIFROST_GATOR_SUPPORT) ++#include ++#endif ++#include ++#include ++#include ++ ++#include ++#include ++ ++#include "mali_kbase_jm.h" ++#include "mali_kbase_hwaccess_jm.h" ++ ++/* ++ * Private types ++ */ ++ ++/* Bitpattern indicating the result of releasing a context */ ++enum { ++ /* The context was descheduled - caller should try scheduling in a new ++ * one to keep the runpool full */ ++ KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED = (1u << 0), ++ /* Ctx attributes were changed - caller should try scheduling all ++ * contexts */ ++ KBASEP_JS_RELEASE_RESULT_SCHED_ALL = (1u << 1) ++}; ++ ++typedef u32 kbasep_js_release_result; ++ ++const int kbasep_js_atom_priority_to_relative[BASE_JD_NR_PRIO_LEVELS] = { ++ KBASE_JS_ATOM_SCHED_PRIO_MED, /* BASE_JD_PRIO_MEDIUM */ ++ KBASE_JS_ATOM_SCHED_PRIO_HIGH, /* BASE_JD_PRIO_HIGH */ ++ KBASE_JS_ATOM_SCHED_PRIO_LOW /* BASE_JD_PRIO_LOW */ ++}; ++ ++const base_jd_prio ++kbasep_js_relative_priority_to_atom[KBASE_JS_ATOM_SCHED_PRIO_COUNT] = { ++ BASE_JD_PRIO_HIGH, /* KBASE_JS_ATOM_SCHED_PRIO_HIGH */ ++ BASE_JD_PRIO_MEDIUM, /* KBASE_JS_ATOM_SCHED_PRIO_MED */ ++ BASE_JD_PRIO_LOW /* KBASE_JS_ATOM_SCHED_PRIO_LOW */ ++}; ++ ++ ++/* ++ * Private function prototypes ++ */ ++static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal( ++ struct kbase_device *kbdev, struct kbase_context *kctx, ++ struct kbasep_js_atom_retained_state *katom_retained_state); ++ ++static int kbase_js_get_slot(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom); ++ ++static void kbase_js_foreach_ctx_job(struct kbase_context *kctx, ++ kbasep_js_ctx_job_cb callback); ++ ++/* Helper for trace subcodes */ ++#if KBASE_TRACE_ENABLE ++static int kbasep_js_trace_get_refcnt(struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++ return atomic_read(&kctx->refcount); ++} ++#else /* KBASE_TRACE_ENABLE */ ++static int kbasep_js_trace_get_refcnt(struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++ CSTD_UNUSED(kbdev); ++ CSTD_UNUSED(kctx); ++ return 0; ++} ++#endif /* KBASE_TRACE_ENABLE */ ++ ++/* ++ * Private functions ++ */ ++ ++/** ++ * core_reqs_from_jsn_features - Convert JSn_FEATURES to core requirements ++ * @features: JSn_FEATURE register value ++ * ++ * Given a JSn_FEATURE register value returns the core requirements that match ++ * ++ * Return: Core requirement bit mask ++ */ ++static base_jd_core_req core_reqs_from_jsn_features(u16 features) ++{ ++ base_jd_core_req core_req = 0u; ++ ++ if ((features & JS_FEATURE_SET_VALUE_JOB) != 0) ++ core_req |= BASE_JD_REQ_V; ++ ++ if ((features & JS_FEATURE_CACHE_FLUSH_JOB) != 0) ++ core_req |= BASE_JD_REQ_CF; ++ ++ if ((features & JS_FEATURE_COMPUTE_JOB) != 0) ++ core_req |= BASE_JD_REQ_CS; ++ ++ if ((features & JS_FEATURE_TILER_JOB) != 0) ++ core_req |= BASE_JD_REQ_T; ++ ++ if ((features & JS_FEATURE_FRAGMENT_JOB) != 0) ++ core_req |= BASE_JD_REQ_FS; ++ ++ return core_req; ++} ++ ++static void kbase_js_sync_timers(struct kbase_device *kbdev) ++{ ++ mutex_lock(&kbdev->js_data.runpool_mutex); ++ kbase_backend_ctx_count_changed(kbdev); ++ mutex_unlock(&kbdev->js_data.runpool_mutex); ++} ++ ++/* Hold the mmu_hw_mutex and hwaccess_lock for this */ ++bool kbasep_js_runpool_retain_ctx_nolock(struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ bool result = false; ++ int as_nr; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ js_devdata = &kbdev->js_data; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ as_nr = kctx->as_nr; ++ if (atomic_read(&kctx->refcount) > 0) { ++ KBASE_DEBUG_ASSERT(as_nr >= 0); ++ ++ kbase_ctx_sched_retain_ctx_refcount(kctx); ++ KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_RETAIN_CTX_NOLOCK, kctx, ++ NULL, 0u, atomic_read(&kctx->refcount)); ++ result = true; ++ } ++ ++ return result; ++} ++ ++/** ++ * jsctx_rb_none_to_pull_prio(): - Check if there are no pullable atoms ++ * @kctx: Pointer to kbase context with ring buffer. ++ * @js: Job slot id to check. ++ * @prio: Priority to check. ++ * ++ * Return true if there are no atoms to pull. There may be running atoms in the ++ * ring buffer even if there are no atoms to pull. It is also possible for the ++ * ring buffer to be full (with running atoms) when this functions returns ++ * true. ++ * ++ * Return: true if there are no atoms to pull, false otherwise. ++ */ ++static inline bool ++jsctx_rb_none_to_pull_prio(struct kbase_context *kctx, int js, int prio) ++{ ++ struct jsctx_queue *rb = &kctx->jsctx_queue[prio][js]; ++ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ ++ return RB_EMPTY_ROOT(&rb->runnable_tree); ++} ++ ++/** ++ * jsctx_rb_none_to_pull(): - Check if all priority ring buffers have no ++ * pullable atoms ++ * @kctx: Pointer to kbase context with ring buffer. ++ * @js: Job slot id to check. ++ * ++ * Caller must hold hwaccess_lock ++ * ++ * Return: true if the ring buffers for all priorities have no pullable atoms, ++ * false otherwise. ++ */ ++static inline bool ++jsctx_rb_none_to_pull(struct kbase_context *kctx, int js) ++{ ++ int prio; ++ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ ++ for (prio = 0; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) { ++ if (!jsctx_rb_none_to_pull_prio(kctx, js, prio)) ++ return false; ++ } ++ ++ return true; ++} ++ ++/** ++ * jsctx_queue_foreach_prio(): - Execute callback for each entry in the queue. ++ * @kctx: Pointer to kbase context with the queue. ++ * @js: Job slot id to iterate. ++ * @prio: Priority id to iterate. ++ * @callback: Function pointer to callback. ++ * ++ * Iterate over a queue and invoke @callback for each entry in the queue, and ++ * remove the entry from the queue. ++ * ++ * If entries are added to the queue while this is running those entries may, or ++ * may not be covered. To ensure that all entries in the buffer have been ++ * enumerated when this function returns jsctx->lock must be held when calling ++ * this function. ++ * ++ * The HW access lock must always be held when calling this function. ++ */ ++static void ++jsctx_queue_foreach_prio(struct kbase_context *kctx, int js, int prio, ++ kbasep_js_ctx_job_cb callback) ++{ ++ struct jsctx_queue *queue = &kctx->jsctx_queue[prio][js]; ++ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ ++ while (!RB_EMPTY_ROOT(&queue->runnable_tree)) { ++ struct rb_node *node = rb_first(&queue->runnable_tree); ++ struct kbase_jd_atom *entry = rb_entry(node, ++ struct kbase_jd_atom, runnable_tree_node); ++ ++ rb_erase(node, &queue->runnable_tree); ++ callback(kctx->kbdev, entry); ++ } ++ ++ while (!list_empty(&queue->x_dep_head)) { ++ struct kbase_jd_atom *entry = list_entry(queue->x_dep_head.next, ++ struct kbase_jd_atom, queue); ++ ++ list_del(queue->x_dep_head.next); ++ ++ callback(kctx->kbdev, entry); ++ } ++} ++ ++/** ++ * jsctx_queue_foreach(): - Execute callback for each entry in every queue ++ * @kctx: Pointer to kbase context with queue. ++ * @js: Job slot id to iterate. ++ * @callback: Function pointer to callback. ++ * ++ * Iterate over all the different priorities, and for each call ++ * jsctx_queue_foreach_prio() to iterate over the queue and invoke @callback ++ * for each entry, and remove the entry from the queue. ++ */ ++static inline void ++jsctx_queue_foreach(struct kbase_context *kctx, int js, ++ kbasep_js_ctx_job_cb callback) ++{ ++ int prio; ++ ++ for (prio = 0; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) ++ jsctx_queue_foreach_prio(kctx, js, prio, callback); ++} ++ ++/** ++ * jsctx_rb_peek_prio(): - Check buffer and get next atom ++ * @kctx: Pointer to kbase context with ring buffer. ++ * @js: Job slot id to check. ++ * @prio: Priority id to check. ++ * ++ * Check the ring buffer for the specified @js and @prio and return a pointer to ++ * the next atom, unless the ring buffer is empty. ++ * ++ * Return: Pointer to next atom in buffer, or NULL if there is no atom. ++ */ ++static inline struct kbase_jd_atom * ++jsctx_rb_peek_prio(struct kbase_context *kctx, int js, int prio) ++{ ++ struct jsctx_queue *rb = &kctx->jsctx_queue[prio][js]; ++ struct rb_node *node; ++ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ ++ node = rb_first(&rb->runnable_tree); ++ if (!node) ++ return NULL; ++ ++ return rb_entry(node, struct kbase_jd_atom, runnable_tree_node); ++} ++ ++/** ++ * jsctx_rb_peek(): - Check all priority buffers and get next atom ++ * @kctx: Pointer to kbase context with ring buffer. ++ * @js: Job slot id to check. ++ * ++ * Check the ring buffers for all priorities, starting from ++ * KBASE_JS_ATOM_SCHED_PRIO_HIGH, for the specified @js and @prio and return a ++ * pointer to the next atom, unless all the priority's ring buffers are empty. ++ * ++ * Caller must hold the hwaccess_lock. ++ * ++ * Return: Pointer to next atom in buffer, or NULL if there is no atom. ++ */ ++static inline struct kbase_jd_atom * ++jsctx_rb_peek(struct kbase_context *kctx, int js) ++{ ++ int prio; ++ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ ++ for (prio = 0; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) { ++ struct kbase_jd_atom *katom; ++ ++ katom = jsctx_rb_peek_prio(kctx, js, prio); ++ if (katom) ++ return katom; ++ } ++ ++ return NULL; ++} ++ ++/** ++ * jsctx_rb_pull(): - Mark atom in list as running ++ * @kctx: Pointer to kbase context with ring buffer. ++ * @katom: Pointer to katom to pull. ++ * ++ * Mark an atom previously obtained from jsctx_rb_peek() as running. ++ * ++ * @katom must currently be at the head of the ring buffer. ++ */ ++static inline void ++jsctx_rb_pull(struct kbase_context *kctx, struct kbase_jd_atom *katom) ++{ ++ int prio = katom->sched_priority; ++ int js = katom->slot_nr; ++ struct jsctx_queue *rb = &kctx->jsctx_queue[prio][js]; ++ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ ++ /* Atoms must be pulled in the correct order. */ ++ WARN_ON(katom != jsctx_rb_peek_prio(kctx, js, prio)); ++ ++ rb_erase(&katom->runnable_tree_node, &rb->runnable_tree); ++} ++ ++#define LESS_THAN_WRAP(a, b) ((s32)(a - b) < 0) ++ ++static void ++jsctx_tree_add(struct kbase_context *kctx, struct kbase_jd_atom *katom) ++{ ++ int prio = katom->sched_priority; ++ int js = katom->slot_nr; ++ struct jsctx_queue *queue = &kctx->jsctx_queue[prio][js]; ++ struct rb_node **new = &(queue->runnable_tree.rb_node), *parent = NULL; ++ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ ++ while (*new) { ++ struct kbase_jd_atom *entry = container_of(*new, ++ struct kbase_jd_atom, runnable_tree_node); ++ ++ parent = *new; ++ if (LESS_THAN_WRAP(katom->age, entry->age)) ++ new = &((*new)->rb_left); ++ else ++ new = &((*new)->rb_right); ++ } ++ ++ /* Add new node and rebalance tree. */ ++ rb_link_node(&katom->runnable_tree_node, parent, new); ++ rb_insert_color(&katom->runnable_tree_node, &queue->runnable_tree); ++} ++ ++/** ++ * jsctx_rb_unpull(): - Undo marking of atom in list as running ++ * @kctx: Pointer to kbase context with ring buffer. ++ * @katom: Pointer to katom to unpull. ++ * ++ * Undo jsctx_rb_pull() and put @katom back in the queue. ++ * ++ * jsctx_rb_unpull() must be called on atoms in the same order the atoms were ++ * pulled. ++ */ ++static inline void ++jsctx_rb_unpull(struct kbase_context *kctx, struct kbase_jd_atom *katom) ++{ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ ++ jsctx_tree_add(kctx, katom); ++} ++ ++static bool kbase_js_ctx_pullable(struct kbase_context *kctx, ++ int js, ++ bool is_scheduled); ++static bool kbase_js_ctx_list_add_pullable_nolock(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ int js); ++static bool kbase_js_ctx_list_add_unpullable_nolock(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ int js); ++ ++/* ++ * Functions private to KBase ('Protected' functions) ++ */ ++int kbasep_js_devdata_init(struct kbase_device * const kbdev) ++{ ++ struct kbasep_js_device_data *jsdd; ++ int i; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ jsdd = &kbdev->js_data; ++ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++ /* Soft-stop will be disabled on a single context by default unless ++ * softstop_always is set */ ++ jsdd->softstop_always = false; ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++ jsdd->nr_all_contexts_running = 0; ++ jsdd->nr_user_contexts_running = 0; ++ jsdd->nr_contexts_pullable = 0; ++ atomic_set(&jsdd->nr_contexts_runnable, 0); ++ /* No ctx allowed to submit */ ++ jsdd->runpool_irq.submit_allowed = 0u; ++ memset(jsdd->runpool_irq.ctx_attr_ref_count, 0, ++ sizeof(jsdd->runpool_irq.ctx_attr_ref_count)); ++ memset(jsdd->runpool_irq.slot_affinities, 0, ++ sizeof(jsdd->runpool_irq.slot_affinities)); ++ memset(jsdd->runpool_irq.slot_affinity_refcount, 0, ++ sizeof(jsdd->runpool_irq.slot_affinity_refcount)); ++ INIT_LIST_HEAD(&jsdd->suspended_soft_jobs_list); ++ ++ /* Config attributes */ ++ jsdd->scheduling_period_ns = DEFAULT_JS_SCHEDULING_PERIOD_NS; ++ jsdd->soft_stop_ticks = DEFAULT_JS_SOFT_STOP_TICKS; ++ jsdd->soft_stop_ticks_cl = DEFAULT_JS_SOFT_STOP_TICKS_CL; ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408)) ++ jsdd->hard_stop_ticks_ss = DEFAULT_JS_HARD_STOP_TICKS_SS_8408; ++ else ++ jsdd->hard_stop_ticks_ss = DEFAULT_JS_HARD_STOP_TICKS_SS; ++ jsdd->hard_stop_ticks_cl = DEFAULT_JS_HARD_STOP_TICKS_CL; ++ jsdd->hard_stop_ticks_dumping = DEFAULT_JS_HARD_STOP_TICKS_DUMPING; ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408)) ++ jsdd->gpu_reset_ticks_ss = DEFAULT_JS_RESET_TICKS_SS_8408; ++ else ++ jsdd->gpu_reset_ticks_ss = DEFAULT_JS_RESET_TICKS_SS; ++ jsdd->gpu_reset_ticks_cl = DEFAULT_JS_RESET_TICKS_CL; ++ jsdd->gpu_reset_ticks_dumping = DEFAULT_JS_RESET_TICKS_DUMPING; ++ jsdd->ctx_timeslice_ns = DEFAULT_JS_CTX_TIMESLICE_NS; ++ atomic_set(&jsdd->soft_job_timeout_ms, DEFAULT_JS_SOFT_JOB_TIMEOUT); ++ ++ dev_dbg(kbdev->dev, "JS Config Attribs: "); ++ dev_dbg(kbdev->dev, "\tscheduling_period_ns:%u", ++ jsdd->scheduling_period_ns); ++ dev_dbg(kbdev->dev, "\tsoft_stop_ticks:%u", ++ jsdd->soft_stop_ticks); ++ dev_dbg(kbdev->dev, "\tsoft_stop_ticks_cl:%u", ++ jsdd->soft_stop_ticks_cl); ++ dev_dbg(kbdev->dev, "\thard_stop_ticks_ss:%u", ++ jsdd->hard_stop_ticks_ss); ++ dev_dbg(kbdev->dev, "\thard_stop_ticks_cl:%u", ++ jsdd->hard_stop_ticks_cl); ++ dev_dbg(kbdev->dev, "\thard_stop_ticks_dumping:%u", ++ jsdd->hard_stop_ticks_dumping); ++ dev_dbg(kbdev->dev, "\tgpu_reset_ticks_ss:%u", ++ jsdd->gpu_reset_ticks_ss); ++ dev_dbg(kbdev->dev, "\tgpu_reset_ticks_cl:%u", ++ jsdd->gpu_reset_ticks_cl); ++ dev_dbg(kbdev->dev, "\tgpu_reset_ticks_dumping:%u", ++ jsdd->gpu_reset_ticks_dumping); ++ dev_dbg(kbdev->dev, "\tctx_timeslice_ns:%u", ++ jsdd->ctx_timeslice_ns); ++ dev_dbg(kbdev->dev, "\tsoft_job_timeout:%i", ++ atomic_read(&jsdd->soft_job_timeout_ms)); ++ ++ if (!(jsdd->soft_stop_ticks < jsdd->hard_stop_ticks_ss && ++ jsdd->hard_stop_ticks_ss < jsdd->gpu_reset_ticks_ss && ++ jsdd->soft_stop_ticks < jsdd->hard_stop_ticks_dumping && ++ jsdd->hard_stop_ticks_dumping < ++ jsdd->gpu_reset_ticks_dumping)) { ++ dev_err(kbdev->dev, "Job scheduler timeouts invalid; soft/hard/reset tick counts should be in increasing order\n"); ++ return -EINVAL; ++ } ++ ++#if KBASE_DISABLE_SCHEDULING_SOFT_STOPS ++ dev_dbg(kbdev->dev, "Job Scheduling Soft-stops disabled, ignoring value for soft_stop_ticks==%u at %uns per tick. Other soft-stops may still occur.", ++ jsdd->soft_stop_ticks, ++ jsdd->scheduling_period_ns); ++#endif ++#if KBASE_DISABLE_SCHEDULING_HARD_STOPS ++ dev_dbg(kbdev->dev, "Job Scheduling Hard-stops disabled, ignoring values for hard_stop_ticks_ss==%d and hard_stop_ticks_dumping==%u at %uns per tick. Other hard-stops may still occur.", ++ jsdd->hard_stop_ticks_ss, ++ jsdd->hard_stop_ticks_dumping, ++ jsdd->scheduling_period_ns); ++#endif ++#if KBASE_DISABLE_SCHEDULING_SOFT_STOPS && KBASE_DISABLE_SCHEDULING_HARD_STOPS ++ dev_dbg(kbdev->dev, "Note: The JS tick timer (if coded) will still be run, but do nothing."); ++#endif ++ ++ for (i = 0; i < kbdev->gpu_props.num_job_slots; ++i) ++ jsdd->js_reqs[i] = core_reqs_from_jsn_features( ++ kbdev->gpu_props.props.raw_props.js_features[i]); ++ ++ /* On error, we could continue on: providing none of the below resources ++ * rely on the ones above */ ++ ++ mutex_init(&jsdd->runpool_mutex); ++ mutex_init(&jsdd->queue_mutex); ++ spin_lock_init(&kbdev->hwaccess_lock); ++ sema_init(&jsdd->schedule_sem, 1); ++ ++ for (i = 0; i < kbdev->gpu_props.num_job_slots; ++i) { ++ INIT_LIST_HEAD(&jsdd->ctx_list_pullable[i]); ++ INIT_LIST_HEAD(&jsdd->ctx_list_unpullable[i]); ++ } ++ ++ return 0; ++} ++ ++void kbasep_js_devdata_halt(struct kbase_device *kbdev) ++{ ++ CSTD_UNUSED(kbdev); ++} ++ ++void kbasep_js_devdata_term(struct kbase_device *kbdev) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ s8 zero_ctx_attr_ref_count[KBASEP_JS_CTX_ATTR_COUNT] = { 0, }; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ js_devdata = &kbdev->js_data; ++ ++ /* The caller must de-register all contexts before calling this ++ */ ++ KBASE_DEBUG_ASSERT(js_devdata->nr_all_contexts_running == 0); ++ KBASE_DEBUG_ASSERT(memcmp( ++ js_devdata->runpool_irq.ctx_attr_ref_count, ++ zero_ctx_attr_ref_count, ++ sizeof(zero_ctx_attr_ref_count)) == 0); ++ CSTD_UNUSED(zero_ctx_attr_ref_count); ++} ++ ++int kbasep_js_kctx_init(struct kbase_context * const kctx) ++{ ++ struct kbase_device *kbdev; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ int i, j; ++ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ ++ kbdev = kctx->kbdev; ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ for (i = 0; i < BASE_JM_MAX_NR_SLOTS; ++i) ++ INIT_LIST_HEAD(&kctx->jctx.sched_info.ctx.ctx_list_entry[i]); ++ ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ js_kctx_info->ctx.nr_jobs = 0; ++ kbase_ctx_flag_clear(kctx, KCTX_SCHEDULED); ++ kbase_ctx_flag_clear(kctx, KCTX_DYING); ++ memset(js_kctx_info->ctx.ctx_attr_ref_count, 0, ++ sizeof(js_kctx_info->ctx.ctx_attr_ref_count)); ++ ++ /* Initially, the context is disabled from submission until the create ++ * flags are set */ ++ kbase_ctx_flag_set(kctx, KCTX_SUBMIT_DISABLED); ++ ++ /* On error, we could continue on: providing none of the below resources ++ * rely on the ones above */ ++ mutex_init(&js_kctx_info->ctx.jsctx_mutex); ++ ++ init_waitqueue_head(&js_kctx_info->ctx.is_scheduled_wait); ++ ++ for (i = 0; i < KBASE_JS_ATOM_SCHED_PRIO_COUNT; i++) { ++ for (j = 0; j < BASE_JM_MAX_NR_SLOTS; j++) { ++ INIT_LIST_HEAD(&kctx->jsctx_queue[i][j].x_dep_head); ++ kctx->jsctx_queue[i][j].runnable_tree = RB_ROOT; ++ } ++ } ++ ++ return 0; ++} ++ ++void kbasep_js_kctx_term(struct kbase_context *kctx) ++{ ++ struct kbase_device *kbdev; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ int js; ++ bool update_ctx_count = false; ++ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ ++ kbdev = kctx->kbdev; ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ /* The caller must de-register all jobs before calling this */ ++ KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ KBASE_DEBUG_ASSERT(js_kctx_info->ctx.nr_jobs == 0); ++ ++ mutex_lock(&kbdev->js_data.queue_mutex); ++ mutex_lock(&kctx->jctx.sched_info.ctx.jsctx_mutex); ++ ++ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) ++ list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); ++ ++ if (kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)) { ++ WARN_ON(atomic_read(&kbdev->js_data.nr_contexts_runnable) <= 0); ++ atomic_dec(&kbdev->js_data.nr_contexts_runnable); ++ update_ctx_count = true; ++ kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); ++ } ++ ++ mutex_unlock(&kctx->jctx.sched_info.ctx.jsctx_mutex); ++ mutex_unlock(&kbdev->js_data.queue_mutex); ++ ++ if (update_ctx_count) { ++ mutex_lock(&kbdev->js_data.runpool_mutex); ++ kbase_backend_ctx_count_changed(kbdev); ++ mutex_unlock(&kbdev->js_data.runpool_mutex); ++ } ++} ++ ++/** ++ * kbase_js_ctx_list_add_pullable_nolock - Variant of ++ * kbase_jd_ctx_list_add_pullable() ++ * where the caller must hold ++ * hwaccess_lock ++ * @kbdev: Device pointer ++ * @kctx: Context to add to queue ++ * @js: Job slot to use ++ * ++ * Caller must hold hwaccess_lock ++ * ++ * Return: true if caller should call kbase_backend_ctx_count_changed() ++ */ ++static bool kbase_js_ctx_list_add_pullable_nolock(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ int js) ++{ ++ bool ret = false; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (!list_empty(&kctx->jctx.sched_info.ctx.ctx_list_entry[js])) ++ list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); ++ ++ list_add_tail(&kctx->jctx.sched_info.ctx.ctx_list_entry[js], ++ &kbdev->js_data.ctx_list_pullable[js]); ++ ++ if (!kctx->slots_pullable) { ++ kbdev->js_data.nr_contexts_pullable++; ++ ret = true; ++ if (!atomic_read(&kctx->atoms_pulled)) { ++ WARN_ON(kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); ++ kbase_ctx_flag_set(kctx, KCTX_RUNNABLE_REF); ++ atomic_inc(&kbdev->js_data.nr_contexts_runnable); ++ } ++ } ++ kctx->slots_pullable |= (1 << js); ++ ++ return ret; ++} ++ ++/** ++ * kbase_js_ctx_list_add_pullable_head_nolock - Variant of ++ * kbase_js_ctx_list_add_pullable_head() ++ * where the caller must hold ++ * hwaccess_lock ++ * @kbdev: Device pointer ++ * @kctx: Context to add to queue ++ * @js: Job slot to use ++ * ++ * Caller must hold hwaccess_lock ++ * ++ * Return: true if caller should call kbase_backend_ctx_count_changed() ++ */ ++static bool kbase_js_ctx_list_add_pullable_head_nolock( ++ struct kbase_device *kbdev, struct kbase_context *kctx, int js) ++{ ++ bool ret = false; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (!list_empty(&kctx->jctx.sched_info.ctx.ctx_list_entry[js])) ++ list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); ++ ++ list_add(&kctx->jctx.sched_info.ctx.ctx_list_entry[js], ++ &kbdev->js_data.ctx_list_pullable[js]); ++ ++ if (!kctx->slots_pullable) { ++ kbdev->js_data.nr_contexts_pullable++; ++ ret = true; ++ if (!atomic_read(&kctx->atoms_pulled)) { ++ WARN_ON(kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); ++ kbase_ctx_flag_set(kctx, KCTX_RUNNABLE_REF); ++ atomic_inc(&kbdev->js_data.nr_contexts_runnable); ++ } ++ } ++ kctx->slots_pullable |= (1 << js); ++ ++ return ret; ++} ++ ++/** ++ * kbase_js_ctx_list_add_pullable_head - Add context to the head of the ++ * per-slot pullable context queue ++ * @kbdev: Device pointer ++ * @kctx: Context to add to queue ++ * @js: Job slot to use ++ * ++ * If the context is on either the pullable or unpullable queues, then it is ++ * removed before being added to the head. ++ * ++ * This function should be used when a context has been scheduled, but no jobs ++ * can currently be pulled from it. ++ * ++ * Return: true if caller should call kbase_backend_ctx_count_changed() ++ */ ++static bool kbase_js_ctx_list_add_pullable_head(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ int js) ++{ ++ bool ret; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ret = kbase_js_ctx_list_add_pullable_head_nolock(kbdev, kctx, js); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ return ret; ++} ++ ++/** ++ * kbase_js_ctx_list_add_unpullable_nolock - Add context to the tail of the ++ * per-slot unpullable context queue ++ * @kbdev: Device pointer ++ * @kctx: Context to add to queue ++ * @js: Job slot to use ++ * ++ * The context must already be on the per-slot pullable queue. It will be ++ * removed from the pullable queue before being added to the unpullable queue. ++ * ++ * This function should be used when a context has been pulled from, and there ++ * are no jobs remaining on the specified slot. ++ * ++ * Caller must hold hwaccess_lock ++ * ++ * Return: true if caller should call kbase_backend_ctx_count_changed() ++ */ ++static bool kbase_js_ctx_list_add_unpullable_nolock(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ int js) ++{ ++ bool ret = false; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ list_move_tail(&kctx->jctx.sched_info.ctx.ctx_list_entry[js], ++ &kbdev->js_data.ctx_list_unpullable[js]); ++ ++ if (kctx->slots_pullable == (1 << js)) { ++ kbdev->js_data.nr_contexts_pullable--; ++ ret = true; ++ if (!atomic_read(&kctx->atoms_pulled)) { ++ WARN_ON(!kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); ++ kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); ++ atomic_dec(&kbdev->js_data.nr_contexts_runnable); ++ } ++ } ++ kctx->slots_pullable &= ~(1 << js); ++ ++ return ret; ++} ++ ++/** ++ * kbase_js_ctx_list_remove_nolock - Remove context from the per-slot pullable ++ * or unpullable context queues ++ * @kbdev: Device pointer ++ * @kctx: Context to remove from queue ++ * @js: Job slot to use ++ * ++ * The context must already be on one of the queues. ++ * ++ * This function should be used when a context has no jobs on the GPU, and no ++ * jobs remaining for the specified slot. ++ * ++ * Caller must hold hwaccess_lock ++ * ++ * Return: true if caller should call kbase_backend_ctx_count_changed() ++ */ ++static bool kbase_js_ctx_list_remove_nolock(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ int js) ++{ ++ bool ret = false; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ WARN_ON(list_empty(&kctx->jctx.sched_info.ctx.ctx_list_entry[js])); ++ ++ list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); ++ ++ if (kctx->slots_pullable == (1 << js)) { ++ kbdev->js_data.nr_contexts_pullable--; ++ ret = true; ++ if (!atomic_read(&kctx->atoms_pulled)) { ++ WARN_ON(!kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); ++ kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); ++ atomic_dec(&kbdev->js_data.nr_contexts_runnable); ++ } ++ } ++ kctx->slots_pullable &= ~(1 << js); ++ ++ return ret; ++} ++ ++/** ++ * kbase_js_ctx_list_pop_head_nolock - Variant of kbase_js_ctx_list_pop_head() ++ * where the caller must hold ++ * hwaccess_lock ++ * @kbdev: Device pointer ++ * @js: Job slot to use ++ * ++ * Caller must hold hwaccess_lock ++ * ++ * Return: Context to use for specified slot. ++ * NULL if no contexts present for specified slot ++ */ ++static struct kbase_context *kbase_js_ctx_list_pop_head_nolock( ++ struct kbase_device *kbdev, ++ int js) ++{ ++ struct kbase_context *kctx; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (list_empty(&kbdev->js_data.ctx_list_pullable[js])) ++ return NULL; ++ ++ kctx = list_entry(kbdev->js_data.ctx_list_pullable[js].next, ++ struct kbase_context, ++ jctx.sched_info.ctx.ctx_list_entry[js]); ++ ++ list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); ++ ++ return kctx; ++} ++ ++/** ++ * kbase_js_ctx_list_pop_head - Pop the head context off the per-slot pullable ++ * queue. ++ * @kbdev: Device pointer ++ * @js: Job slot to use ++ * ++ * Return: Context to use for specified slot. ++ * NULL if no contexts present for specified slot ++ */ ++static struct kbase_context *kbase_js_ctx_list_pop_head( ++ struct kbase_device *kbdev, int js) ++{ ++ struct kbase_context *kctx; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kctx = kbase_js_ctx_list_pop_head_nolock(kbdev, js); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ return kctx; ++} ++ ++/** ++ * kbase_js_ctx_pullable - Return if a context can be pulled from on the ++ * specified slot ++ * @kctx: Context pointer ++ * @js: Job slot to use ++ * @is_scheduled: true if the context is currently scheduled ++ * ++ * Caller must hold hwaccess_lock ++ * ++ * Return: true if context can be pulled from on specified slot ++ * false otherwise ++ */ ++static bool kbase_js_ctx_pullable(struct kbase_context *kctx, int js, ++ bool is_scheduled) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ struct kbase_jd_atom *katom; ++ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ ++ js_devdata = &kctx->kbdev->js_data; ++ ++ if (is_scheduled) { ++ if (!kbasep_js_is_submit_allowed(js_devdata, kctx)) ++ return false; ++ } ++ katom = jsctx_rb_peek(kctx, js); ++ if (!katom) ++ return false; /* No pullable atoms */ ++ if (kctx->blocked_js[js][katom->sched_priority]) ++ return false; ++ if (atomic_read(&katom->blocked)) ++ return false; /* next atom blocked */ ++ if (katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED) { ++ if (katom->x_pre_dep->gpu_rb_state == ++ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB || ++ katom->x_pre_dep->will_fail_event_code) ++ return false; ++ if ((katom->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER) && ++ kbase_backend_nr_atoms_on_slot(kctx->kbdev, js)) ++ return false; ++ } ++ ++ return true; ++} ++ ++static bool kbase_js_dep_validate(struct kbase_context *kctx, ++ struct kbase_jd_atom *katom) ++{ ++ struct kbase_device *kbdev = kctx->kbdev; ++ bool ret = true; ++ bool has_dep = false, has_x_dep = false; ++ int js = kbase_js_get_slot(kbdev, katom); ++ int prio = katom->sched_priority; ++ int i; ++ ++ for (i = 0; i < 2; i++) { ++ struct kbase_jd_atom *dep_atom = katom->dep[i].atom; ++ ++ if (dep_atom) { ++ int dep_js = kbase_js_get_slot(kbdev, dep_atom); ++ int dep_prio = dep_atom->sched_priority; ++ ++ /* Dependent atom must already have been submitted */ ++ if (!(dep_atom->atom_flags & ++ KBASE_KATOM_FLAG_JSCTX_IN_TREE)) { ++ ret = false; ++ break; ++ } ++ ++ /* Dependencies with different priorities can't ++ be represented in the ringbuffer */ ++ if (prio != dep_prio) { ++ ret = false; ++ break; ++ } ++ ++ if (js == dep_js) { ++ /* Only one same-slot dependency can be ++ * represented in the ringbuffer */ ++ if (has_dep) { ++ ret = false; ++ break; ++ } ++ /* Each dependee atom can only have one ++ * same-slot dependency */ ++ if (dep_atom->post_dep) { ++ ret = false; ++ break; ++ } ++ has_dep = true; ++ } else { ++ /* Only one cross-slot dependency can be ++ * represented in the ringbuffer */ ++ if (has_x_dep) { ++ ret = false; ++ break; ++ } ++ /* Each dependee atom can only have one ++ * cross-slot dependency */ ++ if (dep_atom->x_post_dep) { ++ ret = false; ++ break; ++ } ++ /* The dependee atom can not already be in the ++ * HW access ringbuffer */ ++ if (dep_atom->gpu_rb_state != ++ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { ++ ret = false; ++ break; ++ } ++ /* The dependee atom can not already have ++ * completed */ ++ if (dep_atom->status != ++ KBASE_JD_ATOM_STATE_IN_JS) { ++ ret = false; ++ break; ++ } ++ /* Cross-slot dependencies must not violate ++ * PRLAM-8987 affinity restrictions */ ++ if (kbase_hw_has_issue(kbdev, ++ BASE_HW_ISSUE_8987) && ++ (js == 2 || dep_js == 2)) { ++ ret = false; ++ break; ++ } ++ has_x_dep = true; ++ } ++ ++ /* Dependency can be represented in ringbuffers */ ++ } ++ } ++ ++ /* If dependencies can be represented by ringbuffer then clear them from ++ * atom structure */ ++ if (ret) { ++ for (i = 0; i < 2; i++) { ++ struct kbase_jd_atom *dep_atom = katom->dep[i].atom; ++ ++ if (dep_atom) { ++ int dep_js = kbase_js_get_slot(kbdev, dep_atom); ++ ++ if ((js != dep_js) && ++ (dep_atom->status != ++ KBASE_JD_ATOM_STATE_COMPLETED) ++ && (dep_atom->status != ++ KBASE_JD_ATOM_STATE_HW_COMPLETED) ++ && (dep_atom->status != ++ KBASE_JD_ATOM_STATE_UNUSED)) { ++ ++ katom->atom_flags |= ++ KBASE_KATOM_FLAG_X_DEP_BLOCKED; ++ katom->x_pre_dep = dep_atom; ++ dep_atom->x_post_dep = katom; ++ if (kbase_jd_katom_dep_type( ++ &katom->dep[i]) == ++ BASE_JD_DEP_TYPE_DATA) ++ katom->atom_flags |= ++ KBASE_KATOM_FLAG_FAIL_BLOCKER; ++ } ++ if ((kbase_jd_katom_dep_type(&katom->dep[i]) ++ == BASE_JD_DEP_TYPE_DATA) && ++ (js == dep_js)) { ++ katom->pre_dep = dep_atom; ++ dep_atom->post_dep = katom; ++ } ++ ++ list_del(&katom->dep_item[i]); ++ kbase_jd_katom_dep_clear(&katom->dep[i]); ++ } ++ } ++ } ++ ++ return ret; ++} ++ ++bool kbasep_js_add_job(struct kbase_context *kctx, ++ struct kbase_jd_atom *atom) ++{ ++ unsigned long flags; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ struct kbase_device *kbdev; ++ struct kbasep_js_device_data *js_devdata; ++ ++ bool enqueue_required = false; ++ bool timer_sync = false; ++ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ KBASE_DEBUG_ASSERT(atom != NULL); ++ lockdep_assert_held(&kctx->jctx.lock); ++ ++ kbdev = kctx->kbdev; ++ js_devdata = &kbdev->js_data; ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ mutex_lock(&js_devdata->queue_mutex); ++ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); ++ ++ /* ++ * Begin Runpool transaction ++ */ ++ mutex_lock(&js_devdata->runpool_mutex); ++ ++ /* Refcount ctx.nr_jobs */ ++ KBASE_DEBUG_ASSERT(js_kctx_info->ctx.nr_jobs < U32_MAX); ++ ++(js_kctx_info->ctx.nr_jobs); ++ ++ /* Setup any scheduling information */ ++ kbasep_js_clear_job_retry_submit(atom); ++ ++ /* Lock for state available during IRQ */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ if (!kbase_js_dep_validate(kctx, atom)) { ++ /* Dependencies could not be represented */ ++ --(js_kctx_info->ctx.nr_jobs); ++ ++ /* Setting atom status back to queued as it still has unresolved ++ * dependencies */ ++ atom->status = KBASE_JD_ATOM_STATE_QUEUED; ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ ++ goto out_unlock; ++ } ++ ++ KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(atom, TL_ATOM_STATE_READY); ++ KBASE_TIMELINE_ATOM_READY(kctx, kbase_jd_atom_id(kctx, atom)); ++ ++ enqueue_required = kbase_js_dep_resolved_submit(kctx, atom); ++ ++ KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_ADD_JOB, kctx, atom, atom->jc, ++ kbasep_js_trace_get_refcnt(kbdev, kctx)); ++ ++ /* Context Attribute Refcounting */ ++ kbasep_js_ctx_attr_ctx_retain_atom(kbdev, kctx, atom); ++ ++ if (enqueue_required) { ++ if (kbase_js_ctx_pullable(kctx, atom->slot_nr, false)) ++ timer_sync = kbase_js_ctx_list_add_pullable_nolock( ++ kbdev, kctx, atom->slot_nr); ++ else ++ timer_sync = kbase_js_ctx_list_add_unpullable_nolock( ++ kbdev, kctx, atom->slot_nr); ++ } ++ /* If this context is active and the atom is the first on its slot, ++ * kick the job manager to attempt to fast-start the atom */ ++ if (enqueue_required && kctx == kbdev->hwaccess.active_kctx) ++ kbase_jm_try_kick(kbdev, 1 << atom->slot_nr); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ if (timer_sync) ++ kbase_backend_ctx_count_changed(kbdev); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ /* End runpool transaction */ ++ ++ if (!kbase_ctx_flag(kctx, KCTX_SCHEDULED)) { ++ if (kbase_ctx_flag(kctx, KCTX_DYING)) { ++ /* A job got added while/after kbase_job_zap_context() ++ * was called on a non-scheduled context (e.g. KDS ++ * dependency resolved). Kill that job by killing the ++ * context. */ ++ kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, ++ false); ++ } else if (js_kctx_info->ctx.nr_jobs == 1) { ++ /* Handle Refcount going from 0 to 1: schedule the ++ * context on the Queue */ ++ KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ dev_dbg(kbdev->dev, "JS: Enqueue Context %p", kctx); ++ ++ /* Queue was updated - caller must try to ++ * schedule the head context */ ++ WARN_ON(!enqueue_required); ++ } ++ } ++out_unlock: ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ ++ mutex_unlock(&js_devdata->queue_mutex); ++ ++ return enqueue_required; ++} ++ ++void kbasep_js_remove_job(struct kbase_device *kbdev, ++ struct kbase_context *kctx, struct kbase_jd_atom *atom) ++{ ++ struct kbasep_js_kctx_info *js_kctx_info; ++ struct kbasep_js_device_data *js_devdata; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ KBASE_DEBUG_ASSERT(atom != NULL); ++ ++ js_devdata = &kbdev->js_data; ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_REMOVE_JOB, kctx, atom, atom->jc, ++ kbasep_js_trace_get_refcnt(kbdev, kctx)); ++ ++ /* De-refcount ctx.nr_jobs */ ++ KBASE_DEBUG_ASSERT(js_kctx_info->ctx.nr_jobs > 0); ++ --(js_kctx_info->ctx.nr_jobs); ++} ++ ++bool kbasep_js_remove_cancelled_job(struct kbase_device *kbdev, ++ struct kbase_context *kctx, struct kbase_jd_atom *katom) ++{ ++ unsigned long flags; ++ struct kbasep_js_atom_retained_state katom_retained_state; ++ struct kbasep_js_device_data *js_devdata; ++ bool attr_state_changed; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ KBASE_DEBUG_ASSERT(katom != NULL); ++ ++ js_devdata = &kbdev->js_data; ++ ++ kbasep_js_atom_retained_state_copy(&katom_retained_state, katom); ++ kbasep_js_remove_job(kbdev, kctx, katom); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ /* The atom has 'finished' (will not be re-run), so no need to call ++ * kbasep_js_has_atom_finished(). ++ * ++ * This is because it returns false for soft-stopped atoms, but we ++ * want to override that, because we're cancelling an atom regardless of ++ * whether it was soft-stopped or not */ ++ attr_state_changed = kbasep_js_ctx_attr_ctx_release_atom(kbdev, kctx, ++ &katom_retained_state); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ return attr_state_changed; ++} ++ ++bool kbasep_js_runpool_retain_ctx(struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++ unsigned long flags; ++ struct kbasep_js_device_data *js_devdata; ++ bool result; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ js_devdata = &kbdev->js_data; ++ ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ result = kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ ++ return result; ++} ++ ++struct kbase_context *kbasep_js_runpool_lookup_ctx(struct kbase_device *kbdev, ++ int as_nr) ++{ ++ unsigned long flags; ++ struct kbase_context *found_kctx = NULL; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(0 <= as_nr && as_nr < BASE_MAX_NR_AS); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ found_kctx = kbdev->as_to_kctx[as_nr]; ++ ++ if (found_kctx != NULL) ++ kbase_ctx_sched_retain_ctx_refcount(found_kctx); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ return found_kctx; ++} ++ ++/** ++ * kbasep_js_release_result - Try running more jobs after releasing a context ++ * and/or atom ++ * ++ * @kbdev: The kbase_device to operate on ++ * @kctx: The kbase_context to operate on ++ * @katom_retained_state: Retained state from the atom ++ * @runpool_ctx_attr_change: True if the runpool context attributes have changed ++ * ++ * This collates a set of actions that must happen whilst hwaccess_lock is held. ++ * ++ * This includes running more jobs when: ++ * - The previously released kctx caused a ctx attribute change, ++ * - The released atom caused a ctx attribute change, ++ * - Slots were previously blocked due to affinity restrictions, ++ * - Submission during IRQ handling failed. ++ * ++ * Return: %KBASEP_JS_RELEASE_RESULT_SCHED_ALL if context attributes were ++ * changed. The caller should try scheduling all contexts ++ */ ++static kbasep_js_release_result kbasep_js_run_jobs_after_ctx_and_atom_release( ++ struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ struct kbasep_js_atom_retained_state *katom_retained_state, ++ bool runpool_ctx_attr_change) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ kbasep_js_release_result result = 0; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ KBASE_DEBUG_ASSERT(katom_retained_state != NULL); ++ js_devdata = &kbdev->js_data; ++ ++ lockdep_assert_held(&kctx->jctx.sched_info.ctx.jsctx_mutex); ++ lockdep_assert_held(&js_devdata->runpool_mutex); ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (js_devdata->nr_user_contexts_running != 0) { ++ bool retry_submit = false; ++ int retry_jobslot = 0; ++ ++ if (katom_retained_state) ++ retry_submit = kbasep_js_get_atom_retry_submit_slot( ++ katom_retained_state, &retry_jobslot); ++ ++ if (runpool_ctx_attr_change || retry_submit) { ++ /* A change in runpool ctx attributes might mean we can ++ * run more jobs than before */ ++ result = KBASEP_JS_RELEASE_RESULT_SCHED_ALL; ++ ++ KBASE_TRACE_ADD_SLOT(kbdev, JD_DONE_TRY_RUN_NEXT_JOB, ++ kctx, NULL, 0u, retry_jobslot); ++ } ++ } ++ return result; ++} ++ ++/* ++ * Internal function to release the reference on a ctx and an atom's "retained ++ * state", only taking the runpool and as transaction mutexes ++ * ++ * This also starts more jobs running in the case of an ctx-attribute state ++ * change ++ * ++ * This does none of the followup actions for scheduling: ++ * - It does not schedule in a new context ++ * - It does not requeue or handle dying contexts ++ * ++ * For those tasks, just call kbasep_js_runpool_release_ctx() instead ++ * ++ * Requires: ++ * - Context is scheduled in, and kctx->as_nr matches kctx_as_nr ++ * - Context has a non-zero refcount ++ * - Caller holds js_kctx_info->ctx.jsctx_mutex ++ * - Caller holds js_devdata->runpool_mutex ++ */ ++static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal( ++ struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ struct kbasep_js_atom_retained_state *katom_retained_state) ++{ ++ unsigned long flags; ++ struct kbasep_js_device_data *js_devdata; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ ++ kbasep_js_release_result release_result = 0u; ++ bool runpool_ctx_attr_change = false; ++ int kctx_as_nr; ++ int new_ref_count; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ js_kctx_info = &kctx->jctx.sched_info; ++ js_devdata = &kbdev->js_data; ++ ++ /* Ensure context really is scheduled in */ ++ KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ ++ kctx_as_nr = kctx->as_nr; ++ KBASE_DEBUG_ASSERT(kctx_as_nr != KBASEP_AS_NR_INVALID); ++ KBASE_DEBUG_ASSERT(atomic_read(&kctx->refcount) > 0); ++ ++ /* ++ * Transaction begins on AS and runpool_irq ++ * ++ * Assert about out calling contract ++ */ ++ mutex_lock(&kbdev->pm.lock); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ KBASE_DEBUG_ASSERT(kctx_as_nr == kctx->as_nr); ++ KBASE_DEBUG_ASSERT(atomic_read(&kctx->refcount) > 0); ++ ++ /* Update refcount */ ++ kbase_ctx_sched_release_ctx(kctx); ++ new_ref_count = atomic_read(&kctx->refcount); ++ ++ /* Release the atom if it finished (i.e. wasn't soft-stopped) */ ++ if (kbasep_js_has_atom_finished(katom_retained_state)) ++ runpool_ctx_attr_change |= kbasep_js_ctx_attr_ctx_release_atom( ++ kbdev, kctx, katom_retained_state); ++ ++ KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_RELEASE_CTX, kctx, NULL, 0u, ++ new_ref_count); ++ ++ if (new_ref_count == 2 && kbase_ctx_flag(kctx, KCTX_PRIVILEGED) && ++ !kbase_pm_is_suspending(kbdev)) { ++ /* Context is kept scheduled into an address space even when ++ * there are no jobs, in this case we have to handle the ++ * situation where all jobs have been evicted from the GPU and ++ * submission is disabled. ++ * ++ * At this point we re-enable submission to allow further jobs ++ * to be executed ++ */ ++ kbasep_js_set_submit_allowed(js_devdata, kctx); ++ } ++ ++ /* Make a set of checks to see if the context should be scheduled out. ++ * Note that there'll always be at least 1 reference to the context ++ * which was previously acquired by kbasep_js_schedule_ctx(). */ ++ if (new_ref_count == 1 && ++ (!kbasep_js_is_submit_allowed(js_devdata, kctx) || ++ kbdev->pm.suspending)) { ++ int num_slots = kbdev->gpu_props.num_job_slots; ++ int slot; ++ ++ /* Last reference, and we've been told to remove this context ++ * from the Run Pool */ ++ dev_dbg(kbdev->dev, "JS: RunPool Remove Context %p because refcount=%d, jobs=%d, allowed=%d", ++ kctx, new_ref_count, js_kctx_info->ctx.nr_jobs, ++ kbasep_js_is_submit_allowed(js_devdata, kctx)); ++ ++#if defined(CONFIG_MALI_BIFROST_GATOR_SUPPORT) ++ kbase_trace_mali_mmu_as_released(kctx->as_nr); ++#endif ++ KBASE_TLSTREAM_TL_NRET_AS_CTX(&kbdev->as[kctx->as_nr], kctx); ++ ++ kbase_backend_release_ctx_irq(kbdev, kctx); ++ ++ if (kbdev->hwaccess.active_kctx == kctx) ++ kbdev->hwaccess.active_kctx = NULL; ++ ++ /* Ctx Attribute handling ++ * ++ * Releasing atoms attributes must either happen before this, or ++ * after the KCTX_SHEDULED flag is changed, otherwise we ++ * double-decount the attributes ++ */ ++ runpool_ctx_attr_change |= ++ kbasep_js_ctx_attr_runpool_release_ctx(kbdev, kctx); ++ ++ /* Releasing the context and katom retained state can allow ++ * more jobs to run */ ++ release_result |= ++ kbasep_js_run_jobs_after_ctx_and_atom_release(kbdev, ++ kctx, katom_retained_state, ++ runpool_ctx_attr_change); ++ ++ /* ++ * Transaction ends on AS and runpool_irq: ++ * ++ * By this point, the AS-related data is now clear and ready ++ * for re-use. ++ * ++ * Since releases only occur once for each previous successful ++ * retain, and no more retains are allowed on this context, no ++ * other thread will be operating in this ++ * code whilst we are ++ */ ++ ++ /* Recalculate pullable status for all slots */ ++ for (slot = 0; slot < num_slots; slot++) { ++ if (kbase_js_ctx_pullable(kctx, slot, false)) ++ kbase_js_ctx_list_add_pullable_nolock(kbdev, ++ kctx, slot); ++ } ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ kbase_backend_release_ctx_noirq(kbdev, kctx); ++ ++ mutex_unlock(&kbdev->pm.lock); ++ ++ /* Note: Don't reuse kctx_as_nr now */ ++ ++ /* Synchronize with any timers */ ++ kbase_backend_ctx_count_changed(kbdev); ++ ++ /* update book-keeping info */ ++ kbase_ctx_flag_clear(kctx, KCTX_SCHEDULED); ++ /* Signal any waiter that the context is not scheduled, so is ++ * safe for termination - once the jsctx_mutex is also dropped, ++ * and jobs have finished. */ ++ wake_up(&js_kctx_info->ctx.is_scheduled_wait); ++ ++ /* Queue an action to occur after we've dropped the lock */ ++ release_result |= KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED | ++ KBASEP_JS_RELEASE_RESULT_SCHED_ALL; ++ } else { ++ kbasep_js_run_jobs_after_ctx_and_atom_release(kbdev, kctx, ++ katom_retained_state, runpool_ctx_attr_change); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&kbdev->pm.lock); ++ } ++ ++ return release_result; ++} ++ ++void kbasep_js_runpool_release_ctx_nolock(struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++ struct kbasep_js_atom_retained_state katom_retained_state; ++ ++ /* Setup a dummy katom_retained_state */ ++ kbasep_js_atom_retained_state_init_invalid(&katom_retained_state); ++ ++ kbasep_js_runpool_release_ctx_internal(kbdev, kctx, ++ &katom_retained_state); ++} ++ ++void kbasep_js_runpool_requeue_or_kill_ctx(struct kbase_device *kbdev, ++ struct kbase_context *kctx, bool has_pm_ref) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ ++ /* This is called if and only if you've you've detached the context from ++ * the Runpool Queue, and not added it back to the Runpool ++ */ ++ KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ ++ if (kbase_ctx_flag(kctx, KCTX_DYING)) { ++ /* Dying: don't requeue, but kill all jobs on the context. This ++ * happens asynchronously */ ++ dev_dbg(kbdev->dev, ++ "JS: ** Killing Context %p on RunPool Remove **", kctx); ++ kbase_js_foreach_ctx_job(kctx, &kbase_jd_cancel); ++ } ++} ++ ++void kbasep_js_runpool_release_ctx_and_katom_retained_state( ++ struct kbase_device *kbdev, struct kbase_context *kctx, ++ struct kbasep_js_atom_retained_state *katom_retained_state) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ kbasep_js_release_result release_result; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ js_kctx_info = &kctx->jctx.sched_info; ++ js_devdata = &kbdev->js_data; ++ ++ mutex_lock(&js_devdata->queue_mutex); ++ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); ++ mutex_lock(&js_devdata->runpool_mutex); ++ ++ release_result = kbasep_js_runpool_release_ctx_internal(kbdev, kctx, ++ katom_retained_state); ++ ++ /* Drop the runpool mutex to allow requeing kctx */ ++ mutex_unlock(&js_devdata->runpool_mutex); ++ ++ if ((release_result & KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED) != 0u) ++ kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, true); ++ ++ /* Drop the jsctx_mutex to allow scheduling in a new context */ ++ ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ mutex_unlock(&js_devdata->queue_mutex); ++ ++ if (release_result & KBASEP_JS_RELEASE_RESULT_SCHED_ALL) ++ kbase_js_sched_all(kbdev); ++} ++ ++void kbasep_js_runpool_release_ctx(struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++ struct kbasep_js_atom_retained_state katom_retained_state; ++ ++ kbasep_js_atom_retained_state_init_invalid(&katom_retained_state); ++ ++ kbasep_js_runpool_release_ctx_and_katom_retained_state(kbdev, kctx, ++ &katom_retained_state); ++} ++ ++/* Variant of kbasep_js_runpool_release_ctx() that doesn't call into ++ * kbase_js_sched_all() */ ++static void kbasep_js_runpool_release_ctx_no_schedule( ++ struct kbase_device *kbdev, struct kbase_context *kctx) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ kbasep_js_release_result release_result; ++ struct kbasep_js_atom_retained_state katom_retained_state_struct; ++ struct kbasep_js_atom_retained_state *katom_retained_state = ++ &katom_retained_state_struct; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ js_kctx_info = &kctx->jctx.sched_info; ++ js_devdata = &kbdev->js_data; ++ kbasep_js_atom_retained_state_init_invalid(katom_retained_state); ++ ++ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); ++ mutex_lock(&js_devdata->runpool_mutex); ++ ++ release_result = kbasep_js_runpool_release_ctx_internal(kbdev, kctx, ++ katom_retained_state); ++ ++ /* Drop the runpool mutex to allow requeing kctx */ ++ mutex_unlock(&js_devdata->runpool_mutex); ++ if ((release_result & KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED) != 0u) ++ kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, true); ++ ++ /* Drop the jsctx_mutex to allow scheduling in a new context */ ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ ++ /* NOTE: could return release_result if the caller would like to know ++ * whether it should schedule a new context, but currently no callers do ++ */ ++} ++ ++void kbase_js_set_timeouts(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ kbase_backend_timeouts_changed(kbdev); ++} ++ ++static bool kbasep_js_schedule_ctx(struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ unsigned long flags; ++ bool kctx_suspended = false; ++ int as_nr; ++ ++ js_devdata = &kbdev->js_data; ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ /* Pick available address space for this context */ ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ as_nr = kbase_ctx_sched_retain_ctx(kctx); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ if (as_nr == KBASEP_AS_NR_INVALID) { ++ as_nr = kbase_backend_find_and_release_free_address_space( ++ kbdev, kctx); ++ if (as_nr != KBASEP_AS_NR_INVALID) { ++ /* Attempt to retain the context again, this should ++ * succeed */ ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ as_nr = kbase_ctx_sched_retain_ctx(kctx); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ ++ WARN_ON(as_nr == KBASEP_AS_NR_INVALID); ++ } ++ } ++ if (as_nr == KBASEP_AS_NR_INVALID) ++ return false; /* No address spaces currently available */ ++ ++ /* ++ * Atomic transaction on the Context and Run Pool begins ++ */ ++ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); ++ mutex_lock(&js_devdata->runpool_mutex); ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ /* Check to see if context is dying due to kbase_job_zap_context() */ ++ if (kbase_ctx_flag(kctx, KCTX_DYING)) { ++ /* Roll back the transaction so far and return */ ++ kbase_ctx_sched_release_ctx(kctx); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ ++ return false; ++ } ++ ++ KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_TRY_SCHEDULE_HEAD_CTX, kctx, NULL, ++ 0u, ++ kbasep_js_trace_get_refcnt(kbdev, kctx)); ++ ++ kbase_ctx_flag_set(kctx, KCTX_SCHEDULED); ++ ++ /* Assign context to previously chosen address space */ ++ if (!kbase_backend_use_ctx(kbdev, kctx, as_nr)) { ++ /* Roll back the transaction so far and return */ ++ kbase_ctx_sched_release_ctx(kctx); ++ kbase_ctx_flag_clear(kctx, KCTX_SCHEDULED); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ ++ return false; ++ } ++ ++ kbdev->hwaccess.active_kctx = kctx; ++ ++#if defined(CONFIG_MALI_BIFROST_GATOR_SUPPORT) ++ kbase_trace_mali_mmu_as_in_use(kctx->as_nr); ++#endif ++ KBASE_TLSTREAM_TL_RET_AS_CTX(&kbdev->as[kctx->as_nr], kctx); ++ ++ /* Cause any future waiter-on-termination to wait until the context is ++ * descheduled */ ++ wake_up(&js_kctx_info->ctx.is_scheduled_wait); ++ ++ /* Re-check for suspending: a suspend could've occurred, and all the ++ * contexts could've been removed from the runpool before we took this ++ * lock. In this case, we don't want to allow this context to run jobs, ++ * we just want it out immediately. ++ * ++ * The DMB required to read the suspend flag was issued recently as part ++ * of the hwaccess_lock locking. If a suspend occurs *after* that lock ++ * was taken (i.e. this condition doesn't execute), then the ++ * kbasep_js_suspend() code will cleanup this context instead (by virtue ++ * of it being called strictly after the suspend flag is set, and will ++ * wait for this lock to drop) */ ++ if (kbase_pm_is_suspending(kbdev)) { ++ /* Cause it to leave at some later point */ ++ bool retained; ++ ++ retained = kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx); ++ KBASE_DEBUG_ASSERT(retained); ++ ++ kbasep_js_clear_submit_allowed(js_devdata, kctx); ++ kctx_suspended = true; ++ } ++ ++ /* Transaction complete */ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ ++ /* Synchronize with any timers */ ++ kbase_backend_ctx_count_changed(kbdev); ++ ++ mutex_unlock(&js_devdata->runpool_mutex); ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ /* Note: after this point, the context could potentially get scheduled ++ * out immediately */ ++ ++ if (kctx_suspended) { ++ /* Finishing forcing out the context due to a suspend. Use a ++ * variant of kbasep_js_runpool_release_ctx() that doesn't ++ * schedule a new context, to prevent a risk of recursion back ++ * into this function */ ++ kbasep_js_runpool_release_ctx_no_schedule(kbdev, kctx); ++ return false; ++ } ++ return true; ++} ++ ++static bool kbase_js_use_ctx(struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ if (kbase_ctx_flag(kctx, KCTX_SCHEDULED) && ++ kbase_backend_use_ctx_sched(kbdev, kctx)) { ++ /* Context already has ASID - mark as active */ ++ kbdev->hwaccess.active_kctx = kctx; ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ return true; /* Context already scheduled */ ++ } ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ return kbasep_js_schedule_ctx(kbdev, kctx); ++} ++ ++void kbasep_js_schedule_privileged_ctx(struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++ struct kbasep_js_kctx_info *js_kctx_info; ++ struct kbasep_js_device_data *js_devdata; ++ bool is_scheduled; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ ++ js_devdata = &kbdev->js_data; ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ /* This must never be attempted whilst suspending - i.e. it should only ++ * happen in response to a syscall from a user-space thread */ ++ BUG_ON(kbase_pm_is_suspending(kbdev)); ++ ++ mutex_lock(&js_devdata->queue_mutex); ++ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); ++ ++ /* Mark the context as privileged */ ++ kbase_ctx_flag_set(kctx, KCTX_PRIVILEGED); ++ ++ is_scheduled = kbase_ctx_flag(kctx, KCTX_SCHEDULED); ++ if (!is_scheduled) { ++ /* Add the context to the pullable list */ ++ if (kbase_js_ctx_list_add_pullable_head(kbdev, kctx, 0)) ++ kbase_js_sync_timers(kbdev); ++ ++ /* Fast-starting requires the jsctx_mutex to be dropped, ++ * because it works on multiple ctxs */ ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ mutex_unlock(&js_devdata->queue_mutex); ++ ++ /* Try to schedule the context in */ ++ kbase_js_sched_all(kbdev); ++ ++ /* Wait for the context to be scheduled in */ ++ wait_event(kctx->jctx.sched_info.ctx.is_scheduled_wait, ++ kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ } else { ++ /* Already scheduled in - We need to retain it to keep the ++ * corresponding address space */ ++ kbasep_js_runpool_retain_ctx(kbdev, kctx); ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ mutex_unlock(&js_devdata->queue_mutex); ++ } ++} ++KBASE_EXPORT_TEST_API(kbasep_js_schedule_privileged_ctx); ++ ++void kbasep_js_release_privileged_ctx(struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++ struct kbasep_js_kctx_info *js_kctx_info; ++ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ /* We don't need to use the address space anymore */ ++ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); ++ kbase_ctx_flag_clear(kctx, KCTX_PRIVILEGED); ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ ++ /* Release the context - it will be scheduled out */ ++ kbasep_js_runpool_release_ctx(kbdev, kctx); ++ ++ kbase_js_sched_all(kbdev); ++} ++KBASE_EXPORT_TEST_API(kbasep_js_release_privileged_ctx); ++ ++void kbasep_js_suspend(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ struct kbasep_js_device_data *js_devdata; ++ int i; ++ u16 retained = 0u; ++ int nr_privileged_ctx = 0; ++ ++ KBASE_DEBUG_ASSERT(kbdev); ++ KBASE_DEBUG_ASSERT(kbase_pm_is_suspending(kbdev)); ++ js_devdata = &kbdev->js_data; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ /* Prevent all contexts from submitting */ ++ js_devdata->runpool_irq.submit_allowed = 0; ++ ++ /* Retain each of the contexts, so we can cause it to leave even if it ++ * had no refcount to begin with */ ++ for (i = BASE_MAX_NR_AS - 1; i >= 0; --i) { ++ struct kbase_context *kctx = kbdev->as_to_kctx[i]; ++ ++ retained = retained << 1; ++ ++ if (kctx) { ++ kbase_ctx_sched_retain_ctx_refcount(kctx); ++ retained |= 1u; ++ /* We can only cope with up to 1 privileged context - ++ * the instrumented context. It'll be suspended by ++ * disabling instrumentation */ ++ if (kbase_ctx_flag(kctx, KCTX_PRIVILEGED)) { ++ ++nr_privileged_ctx; ++ WARN_ON(nr_privileged_ctx != 1); ++ } ++ } ++ } ++ CSTD_UNUSED(nr_privileged_ctx); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ /* De-ref the previous retain to ensure each context gets pulled out ++ * sometime later. */ ++ for (i = 0; ++ i < BASE_MAX_NR_AS; ++ ++i, retained = retained >> 1) { ++ struct kbase_context *kctx = kbdev->as_to_kctx[i]; ++ ++ if (retained & 1u) ++ kbasep_js_runpool_release_ctx(kbdev, kctx); ++ } ++ ++ /* Caller must wait for all Power Manager active references to be ++ * dropped */ ++} ++ ++void kbasep_js_resume(struct kbase_device *kbdev) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ int js; ++ ++ KBASE_DEBUG_ASSERT(kbdev); ++ js_devdata = &kbdev->js_data; ++ KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev)); ++ ++ mutex_lock(&js_devdata->queue_mutex); ++ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { ++ struct kbase_context *kctx, *n; ++ ++ list_for_each_entry_safe(kctx, n, ++ &kbdev->js_data.ctx_list_unpullable[js], ++ jctx.sched_info.ctx.ctx_list_entry[js]) { ++ struct kbasep_js_kctx_info *js_kctx_info; ++ unsigned long flags; ++ bool timer_sync = false; ++ ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); ++ mutex_lock(&js_devdata->runpool_mutex); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ if (!kbase_ctx_flag(kctx, KCTX_SCHEDULED) && ++ kbase_js_ctx_pullable(kctx, js, false)) ++ timer_sync = ++ kbase_js_ctx_list_add_pullable_nolock( ++ kbdev, kctx, js); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ if (timer_sync) ++ kbase_backend_ctx_count_changed(kbdev); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ } ++ } ++ mutex_unlock(&js_devdata->queue_mutex); ++ ++ /* Restart atom processing */ ++ kbase_js_sched_all(kbdev); ++ ++ /* JS Resume complete */ ++} ++ ++bool kbase_js_is_atom_valid(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom) ++{ ++ if ((katom->core_req & BASE_JD_REQ_FS) && ++ (katom->core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE | ++ BASE_JD_REQ_T))) ++ return false; ++ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987) && ++ (katom->core_req & BASE_JD_REQ_ONLY_COMPUTE) && ++ (katom->core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_T))) ++ return false; ++ ++ return true; ++} ++ ++static int kbase_js_get_slot(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom) ++{ ++ if (katom->core_req & BASE_JD_REQ_FS) ++ return 0; ++ ++ if (katom->core_req & BASE_JD_REQ_ONLY_COMPUTE) { ++ if (katom->device_nr == 1 && ++ kbdev->gpu_props.num_core_groups == 2) ++ return 2; ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) ++ return 2; ++ } ++ ++ return 1; ++} ++ ++bool kbase_js_dep_resolved_submit(struct kbase_context *kctx, ++ struct kbase_jd_atom *katom) ++{ ++ bool enqueue_required; ++ ++ katom->slot_nr = kbase_js_get_slot(kctx->kbdev, katom); ++ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ lockdep_assert_held(&kctx->jctx.lock); ++ ++ /* If slot will transition from unpullable to pullable then add to ++ * pullable list */ ++ if (jsctx_rb_none_to_pull(kctx, katom->slot_nr)) { ++ enqueue_required = true; ++ } else { ++ enqueue_required = false; ++ } ++ if ((katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED) || ++ (katom->pre_dep && (katom->pre_dep->atom_flags & ++ KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST))) { ++ int prio = katom->sched_priority; ++ int js = katom->slot_nr; ++ struct jsctx_queue *queue = &kctx->jsctx_queue[prio][js]; ++ ++ list_add_tail(&katom->queue, &queue->x_dep_head); ++ katom->atom_flags |= KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST; ++ enqueue_required = false; ++ } else { ++ /* Check if there are lower priority jobs to soft stop */ ++ kbase_job_slot_ctx_priority_check_locked(kctx, katom); ++ ++ /* Add atom to ring buffer. */ ++ jsctx_tree_add(kctx, katom); ++ katom->atom_flags |= KBASE_KATOM_FLAG_JSCTX_IN_TREE; ++ } ++ ++ return enqueue_required; ++} ++ ++/** ++ * kbase_js_move_to_tree - Move atom (and any dependent atoms) to the ++ * runnable_tree, ready for execution ++ * @katom: Atom to submit ++ * ++ * It is assumed that @katom does not have KBASE_KATOM_FLAG_X_DEP_BLOCKED set, ++ * but is still present in the x_dep list. If @katom has a same-slot dependent ++ * atom then that atom (and any dependents) will also be moved. ++ */ ++static void kbase_js_move_to_tree(struct kbase_jd_atom *katom) ++{ ++ lockdep_assert_held(&katom->kctx->kbdev->hwaccess_lock); ++ ++ while (katom) { ++ WARN_ON(!(katom->atom_flags & ++ KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST)); ++ ++ if (!(katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED)) { ++ list_del(&katom->queue); ++ katom->atom_flags &= ++ ~KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST; ++ jsctx_tree_add(katom->kctx, katom); ++ katom->atom_flags |= KBASE_KATOM_FLAG_JSCTX_IN_TREE; ++ } else { ++ break; ++ } ++ ++ katom = katom->post_dep; ++ } ++} ++ ++ ++/** ++ * kbase_js_evict_deps - Evict dependencies of a failed atom. ++ * @kctx: Context pointer ++ * @katom: Pointer to the atom that has failed. ++ * @js: The job slot the katom was run on. ++ * @prio: Priority of the katom. ++ * ++ * Remove all post dependencies of an atom from the context ringbuffers. ++ * ++ * The original atom's event_code will be propogated to all dependent atoms. ++ * ++ * Context: Caller must hold the HW access lock ++ */ ++static void kbase_js_evict_deps(struct kbase_context *kctx, ++ struct kbase_jd_atom *katom, int js, int prio) ++{ ++ struct kbase_jd_atom *x_dep = katom->x_post_dep; ++ struct kbase_jd_atom *next_katom = katom->post_dep; ++ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ ++ if (next_katom) { ++ KBASE_DEBUG_ASSERT(next_katom->status != ++ KBASE_JD_ATOM_STATE_HW_COMPLETED); ++ next_katom->will_fail_event_code = katom->event_code; ++ ++ } ++ ++ /* Has cross slot depenency. */ ++ if (x_dep && (x_dep->atom_flags & (KBASE_KATOM_FLAG_JSCTX_IN_TREE | ++ KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST))) { ++ /* Remove dependency.*/ ++ x_dep->atom_flags &= ~KBASE_KATOM_FLAG_X_DEP_BLOCKED; ++ ++ /* Fail if it had a data dependency. */ ++ if (x_dep->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER) { ++ x_dep->will_fail_event_code = katom->event_code; ++ } ++ if (x_dep->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST) ++ kbase_js_move_to_tree(x_dep); ++ } ++} ++ ++struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, int js) ++{ ++ struct kbase_jd_atom *katom; ++ struct kbasep_js_device_data *js_devdata; ++ struct kbase_device *kbdev; ++ int pulled; ++ ++ KBASE_DEBUG_ASSERT(kctx); ++ ++ kbdev = kctx->kbdev; ++ ++ js_devdata = &kbdev->js_data; ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (!kbasep_js_is_submit_allowed(js_devdata, kctx)) ++ return NULL; ++ if (kbase_pm_is_suspending(kbdev)) ++ return NULL; ++ ++ katom = jsctx_rb_peek(kctx, js); ++ if (!katom) ++ return NULL; ++ if (kctx->blocked_js[js][katom->sched_priority]) ++ return NULL; ++ if (atomic_read(&katom->blocked)) ++ return NULL; ++ ++ /* Due to ordering restrictions when unpulling atoms on failure, we do ++ * not allow multiple runs of fail-dep atoms from the same context to be ++ * present on the same slot */ ++ if (katom->pre_dep && atomic_read(&kctx->atoms_pulled_slot[js])) { ++ struct kbase_jd_atom *prev_atom = ++ kbase_backend_inspect_tail(kbdev, js); ++ ++ if (prev_atom && prev_atom->kctx != kctx) ++ return NULL; ++ } ++ ++ if (katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED) { ++ if (katom->x_pre_dep->gpu_rb_state == ++ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB || ++ katom->x_pre_dep->will_fail_event_code) ++ return NULL; ++ if ((katom->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER) && ++ kbase_backend_nr_atoms_on_slot(kbdev, js)) ++ return NULL; ++ } ++ ++ kbase_ctx_flag_set(kctx, KCTX_PULLED); ++ ++ pulled = atomic_inc_return(&kctx->atoms_pulled); ++ if (pulled == 1 && !kctx->slots_pullable) { ++ WARN_ON(kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); ++ kbase_ctx_flag_set(kctx, KCTX_RUNNABLE_REF); ++ atomic_inc(&kbdev->js_data.nr_contexts_runnable); ++ } ++ atomic_inc(&kctx->atoms_pulled_slot[katom->slot_nr]); ++ kctx->atoms_pulled_slot_pri[katom->slot_nr][katom->sched_priority]++; ++ jsctx_rb_pull(kctx, katom); ++ ++ kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx); ++ ++ katom->atom_flags |= KBASE_KATOM_FLAG_HOLDING_CTX_REF; ++ ++ katom->ticks = 0; ++ ++ return katom; ++} ++ ++ ++static void js_return_worker(struct work_struct *data) ++{ ++ struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, ++ work); ++ struct kbase_context *kctx = katom->kctx; ++ struct kbase_device *kbdev = kctx->kbdev; ++ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; ++ struct kbasep_js_kctx_info *js_kctx_info = &kctx->jctx.sched_info; ++ struct kbasep_js_atom_retained_state retained_state; ++ int js = katom->slot_nr; ++ int prio = katom->sched_priority; ++ bool timer_sync = false; ++ bool context_idle = false; ++ unsigned long flags; ++ base_jd_core_req core_req = katom->core_req; ++ u64 affinity = katom->affinity; ++ enum kbase_atom_coreref_state coreref_state = katom->coreref_state; ++ ++ KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_EX(katom); ++ ++ kbase_backend_complete_wq(kbdev, katom); ++ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316)) ++ kbase_as_poking_timer_release_atom(kbdev, kctx, katom); ++ ++ kbasep_js_atom_retained_state_copy(&retained_state, katom); ++ ++ mutex_lock(&js_devdata->queue_mutex); ++ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); ++ ++ atomic_dec(&kctx->atoms_pulled); ++ atomic_dec(&kctx->atoms_pulled_slot[js]); ++ ++ atomic_dec(&katom->blocked); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ kctx->atoms_pulled_slot_pri[js][katom->sched_priority]--; ++ ++ if (!atomic_read(&kctx->atoms_pulled_slot[js]) && ++ jsctx_rb_none_to_pull(kctx, js)) ++ timer_sync |= kbase_js_ctx_list_remove_nolock(kbdev, kctx, js); ++ ++ /* If this slot has been blocked due to soft-stopped atoms, and all ++ * atoms have now been processed, then unblock the slot */ ++ if (!kctx->atoms_pulled_slot_pri[js][prio] && ++ kctx->blocked_js[js][prio]) { ++ kctx->blocked_js[js][prio] = false; ++ ++ /* Only mark the slot as pullable if the context is not idle - ++ * that case is handled below */ ++ if (atomic_read(&kctx->atoms_pulled) && ++ kbase_js_ctx_pullable(kctx, js, true)) ++ timer_sync |= kbase_js_ctx_list_add_pullable_nolock( ++ kbdev, kctx, js); ++ } ++ ++ if (!atomic_read(&kctx->atoms_pulled)) { ++ if (!kctx->slots_pullable) { ++ WARN_ON(!kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); ++ kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); ++ atomic_dec(&kbdev->js_data.nr_contexts_runnable); ++ timer_sync = true; ++ } ++ ++ if (kctx->as_nr != KBASEP_AS_NR_INVALID && ++ !kbase_ctx_flag(kctx, KCTX_DYING)) { ++ int num_slots = kbdev->gpu_props.num_job_slots; ++ int slot; ++ ++ if (!kbasep_js_is_submit_allowed(js_devdata, kctx)) ++ kbasep_js_set_submit_allowed(js_devdata, kctx); ++ ++ for (slot = 0; slot < num_slots; slot++) { ++ if (kbase_js_ctx_pullable(kctx, slot, true)) ++ timer_sync |= ++ kbase_js_ctx_list_add_pullable_nolock( ++ kbdev, kctx, slot); ++ } ++ } ++ ++ kbase_jm_idle_ctx(kbdev, kctx); ++ ++ context_idle = true; ++ } ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ if (context_idle) { ++ WARN_ON(!kbase_ctx_flag(kctx, KCTX_ACTIVE)); ++ kbase_ctx_flag_clear(kctx, KCTX_ACTIVE); ++ kbase_pm_context_idle(kbdev); ++ } ++ ++ if (timer_sync) ++ kbase_js_sync_timers(kbdev); ++ ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ mutex_unlock(&js_devdata->queue_mutex); ++ ++ katom->atom_flags &= ~KBASE_KATOM_FLAG_HOLDING_CTX_REF; ++ kbasep_js_runpool_release_ctx_and_katom_retained_state(kbdev, kctx, ++ &retained_state); ++ ++ kbase_js_sched_all(kbdev); ++ ++ kbase_backend_complete_wq_post_sched(kbdev, core_req, affinity, ++ coreref_state); ++} ++ ++void kbase_js_unpull(struct kbase_context *kctx, struct kbase_jd_atom *katom) ++{ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ ++ jsctx_rb_unpull(kctx, katom); ++ ++ WARN_ON(work_pending(&katom->work)); ++ ++ /* Block re-submission until workqueue has run */ ++ atomic_inc(&katom->blocked); ++ ++ kbase_job_check_leave_disjoint(kctx->kbdev, katom); ++ ++ KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&katom->work)); ++ INIT_WORK(&katom->work, js_return_worker); ++ queue_work(kctx->jctx.job_done_wq, &katom->work); ++} ++ ++bool kbase_js_complete_atom_wq(struct kbase_context *kctx, ++ struct kbase_jd_atom *katom) ++{ ++ struct kbasep_js_kctx_info *js_kctx_info; ++ struct kbasep_js_device_data *js_devdata; ++ struct kbase_device *kbdev; ++ unsigned long flags; ++ bool timer_sync = false; ++ int atom_slot; ++ bool context_idle = false; ++ int prio = katom->sched_priority; ++ ++ kbdev = kctx->kbdev; ++ atom_slot = katom->slot_nr; ++ ++ js_kctx_info = &kctx->jctx.sched_info; ++ js_devdata = &kbdev->js_data; ++ ++ lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); ++ ++ mutex_lock(&js_devdata->runpool_mutex); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ if (katom->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_TREE) { ++ context_idle = !atomic_dec_return(&kctx->atoms_pulled); ++ atomic_dec(&kctx->atoms_pulled_slot[atom_slot]); ++ kctx->atoms_pulled_slot_pri[atom_slot][prio]--; ++ ++ if (!atomic_read(&kctx->atoms_pulled) && ++ !kctx->slots_pullable) { ++ WARN_ON(!kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); ++ kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); ++ atomic_dec(&kbdev->js_data.nr_contexts_runnable); ++ timer_sync = true; ++ } ++ ++ /* If this slot has been blocked due to soft-stopped atoms, and ++ * all atoms have now been processed, then unblock the slot */ ++ if (!kctx->atoms_pulled_slot_pri[atom_slot][prio] ++ && kctx->blocked_js[atom_slot][prio]) { ++ kctx->blocked_js[atom_slot][prio] = false; ++ if (kbase_js_ctx_pullable(kctx, atom_slot, true)) ++ timer_sync |= ++ kbase_js_ctx_list_add_pullable_nolock( ++ kbdev, kctx, atom_slot); ++ } ++ } ++ WARN_ON(!(katom->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_TREE)); ++ ++ if (!atomic_read(&kctx->atoms_pulled_slot[atom_slot]) && ++ jsctx_rb_none_to_pull(kctx, atom_slot)) { ++ if (!list_empty( ++ &kctx->jctx.sched_info.ctx.ctx_list_entry[atom_slot])) ++ timer_sync |= kbase_js_ctx_list_remove_nolock( ++ kctx->kbdev, kctx, atom_slot); ++ } ++ ++ /* ++ * If submission is disabled on this context (most likely due to an ++ * atom failure) and there are now no atoms left in the system then ++ * re-enable submission so that context can be scheduled again. ++ */ ++ if (!kbasep_js_is_submit_allowed(js_devdata, kctx) && ++ !atomic_read(&kctx->atoms_pulled) && ++ !kbase_ctx_flag(kctx, KCTX_DYING)) { ++ int js; ++ ++ kbasep_js_set_submit_allowed(js_devdata, kctx); ++ ++ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { ++ if (kbase_js_ctx_pullable(kctx, js, true)) ++ timer_sync |= ++ kbase_js_ctx_list_add_pullable_nolock( ++ kbdev, kctx, js); ++ } ++ } else if (katom->x_post_dep && ++ kbasep_js_is_submit_allowed(js_devdata, kctx)) { ++ int js; ++ ++ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { ++ if (kbase_js_ctx_pullable(kctx, js, true)) ++ timer_sync |= ++ kbase_js_ctx_list_add_pullable_nolock( ++ kbdev, kctx, js); ++ } ++ } ++ ++ /* Mark context as inactive. The pm reference will be dropped later in ++ * jd_done_worker(). ++ */ ++ if (context_idle) ++ kbase_ctx_flag_clear(kctx, KCTX_ACTIVE); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ if (timer_sync) ++ kbase_backend_ctx_count_changed(kbdev); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ ++ return context_idle; ++} ++ ++struct kbase_jd_atom *kbase_js_complete_atom(struct kbase_jd_atom *katom, ++ ktime_t *end_timestamp) ++{ ++ struct kbase_device *kbdev; ++ struct kbase_context *kctx = katom->kctx; ++ struct kbase_jd_atom *x_dep = katom->x_post_dep; ++ ++ kbdev = kctx->kbdev; ++ ++ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ ++ if (katom->will_fail_event_code) ++ katom->event_code = katom->will_fail_event_code; ++ ++ katom->status = KBASE_JD_ATOM_STATE_HW_COMPLETED; ++ ++ if (katom->event_code != BASE_JD_EVENT_DONE) { ++ kbase_js_evict_deps(kctx, katom, katom->slot_nr, ++ katom->sched_priority); ++ } ++ ++#if defined(CONFIG_MALI_BIFROST_GATOR_SUPPORT) ++ kbase_trace_mali_job_slots_event(GATOR_MAKE_EVENT(GATOR_JOB_SLOT_STOP, ++ katom->slot_nr), NULL, 0); ++#endif ++ ++ kbase_jd_done(katom, katom->slot_nr, end_timestamp, 0); ++ ++ /* Unblock cross dependency if present */ ++ if (x_dep && (katom->event_code == BASE_JD_EVENT_DONE || ++ !(x_dep->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER)) && ++ (x_dep->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED)) { ++ bool was_pullable = kbase_js_ctx_pullable(kctx, x_dep->slot_nr, ++ false); ++ x_dep->atom_flags &= ~KBASE_KATOM_FLAG_X_DEP_BLOCKED; ++ kbase_js_move_to_tree(x_dep); ++ if (!was_pullable && kbase_js_ctx_pullable(kctx, x_dep->slot_nr, ++ false)) ++ kbase_js_ctx_list_add_pullable_nolock(kbdev, kctx, ++ x_dep->slot_nr); ++ ++ if (x_dep->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_TREE) ++ return x_dep; ++ } ++ ++ return NULL; ++} ++ ++void kbase_js_sched(struct kbase_device *kbdev, int js_mask) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ struct kbase_context *last_active; ++ bool timer_sync = false; ++ bool ctx_waiting = false; ++ ++ js_devdata = &kbdev->js_data; ++ ++ down(&js_devdata->schedule_sem); ++ mutex_lock(&js_devdata->queue_mutex); ++ ++ last_active = kbdev->hwaccess.active_kctx; ++ ++ while (js_mask) { ++ int js; ++ ++ js = ffs(js_mask) - 1; ++ ++ while (1) { ++ struct kbase_context *kctx; ++ unsigned long flags; ++ bool context_idle = false; ++ ++ kctx = kbase_js_ctx_list_pop_head(kbdev, js); ++ ++ if (!kctx) { ++ js_mask &= ~(1 << js); ++ break; /* No contexts on pullable list */ ++ } ++ ++ if (!kbase_ctx_flag(kctx, KCTX_ACTIVE)) { ++ context_idle = true; ++ ++ if (kbase_pm_context_active_handle_suspend( ++ kbdev, ++ KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE)) { ++ /* Suspend pending - return context to ++ * queue and stop scheduling */ ++ mutex_lock( ++ &kctx->jctx.sched_info.ctx.jsctx_mutex); ++ if (kbase_js_ctx_list_add_pullable_head( ++ kctx->kbdev, kctx, js)) ++ kbase_js_sync_timers(kbdev); ++ mutex_unlock( ++ &kctx->jctx.sched_info.ctx.jsctx_mutex); ++ mutex_unlock(&js_devdata->queue_mutex); ++ up(&js_devdata->schedule_sem); ++ return; ++ } ++ kbase_ctx_flag_set(kctx, KCTX_ACTIVE); ++ } ++ ++ if (!kbase_js_use_ctx(kbdev, kctx)) { ++ mutex_lock( ++ &kctx->jctx.sched_info.ctx.jsctx_mutex); ++ /* Context can not be used at this time */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ if (kbase_js_ctx_pullable(kctx, js, false) ++ || kbase_ctx_flag(kctx, KCTX_PRIVILEGED)) ++ timer_sync |= ++ kbase_js_ctx_list_add_pullable_head_nolock( ++ kctx->kbdev, kctx, js); ++ else ++ timer_sync |= ++ kbase_js_ctx_list_add_unpullable_nolock( ++ kctx->kbdev, kctx, js); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, ++ flags); ++ mutex_unlock( ++ &kctx->jctx.sched_info.ctx.jsctx_mutex); ++ if (context_idle) { ++ WARN_ON(!kbase_ctx_flag(kctx, KCTX_ACTIVE)); ++ kbase_ctx_flag_clear(kctx, KCTX_ACTIVE); ++ kbase_pm_context_idle(kbdev); ++ } ++ ++ /* No more jobs can be submitted on this slot */ ++ js_mask &= ~(1 << js); ++ break; ++ } ++ mutex_lock(&kctx->jctx.sched_info.ctx.jsctx_mutex); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ kbase_ctx_flag_clear(kctx, KCTX_PULLED); ++ ++ if (!kbase_jm_kick(kbdev, 1 << js)) ++ /* No more jobs can be submitted on this slot */ ++ js_mask &= ~(1 << js); ++ ++ if (!kbase_ctx_flag(kctx, KCTX_PULLED)) { ++ bool pullable = kbase_js_ctx_pullable(kctx, js, ++ true); ++ ++ /* Failed to pull jobs - push to head of list. ++ * Unless this context is already 'active', in ++ * which case it's effectively already scheduled ++ * so push it to the back of the list. */ ++ if (pullable && kctx == last_active) ++ timer_sync |= ++ kbase_js_ctx_list_add_pullable_nolock( ++ kctx->kbdev, ++ kctx, js); ++ else if (pullable) ++ timer_sync |= ++ kbase_js_ctx_list_add_pullable_head_nolock( ++ kctx->kbdev, ++ kctx, js); ++ else ++ timer_sync |= ++ kbase_js_ctx_list_add_unpullable_nolock( ++ kctx->kbdev, ++ kctx, js); ++ ++ /* If this context is not the active context, ++ * but the active context is pullable on this ++ * slot, then we need to remove the active ++ * marker to prevent it from submitting atoms in ++ * the IRQ handler, which would prevent this ++ * context from making progress. */ ++ if (last_active && kctx != last_active && ++ kbase_js_ctx_pullable( ++ last_active, js, true)) ++ ctx_waiting = true; ++ ++ if (context_idle) { ++ kbase_jm_idle_ctx(kbdev, kctx); ++ spin_unlock_irqrestore( ++ &kbdev->hwaccess_lock, ++ flags); ++ WARN_ON(!kbase_ctx_flag(kctx, KCTX_ACTIVE)); ++ kbase_ctx_flag_clear(kctx, KCTX_ACTIVE); ++ kbase_pm_context_idle(kbdev); ++ } else { ++ spin_unlock_irqrestore( ++ &kbdev->hwaccess_lock, ++ flags); ++ } ++ mutex_unlock( ++ &kctx->jctx.sched_info.ctx.jsctx_mutex); ++ ++ js_mask &= ~(1 << js); ++ break; /* Could not run atoms on this slot */ ++ } ++ ++ /* Push to back of list */ ++ if (kbase_js_ctx_pullable(kctx, js, true)) ++ timer_sync |= ++ kbase_js_ctx_list_add_pullable_nolock( ++ kctx->kbdev, kctx, js); ++ else ++ timer_sync |= ++ kbase_js_ctx_list_add_unpullable_nolock( ++ kctx->kbdev, kctx, js); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&kctx->jctx.sched_info.ctx.jsctx_mutex); ++ } ++ } ++ ++ if (timer_sync) ++ kbase_js_sync_timers(kbdev); ++ ++ if (kbdev->hwaccess.active_kctx == last_active && ctx_waiting) ++ kbdev->hwaccess.active_kctx = NULL; ++ ++ mutex_unlock(&js_devdata->queue_mutex); ++ up(&js_devdata->schedule_sem); ++} ++ ++void kbase_js_zap_context(struct kbase_context *kctx) ++{ ++ struct kbase_device *kbdev = kctx->kbdev; ++ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; ++ struct kbasep_js_kctx_info *js_kctx_info = &kctx->jctx.sched_info; ++ int js; ++ ++ /* ++ * Critical assumption: No more submission is possible outside of the ++ * workqueue. This is because the OS *must* prevent U/K calls (IOCTLs) ++ * whilst the struct kbase_context is terminating. ++ */ ++ ++ /* First, atomically do the following: ++ * - mark the context as dying ++ * - try to evict it from the queue */ ++ mutex_lock(&kctx->jctx.lock); ++ mutex_lock(&js_devdata->queue_mutex); ++ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); ++ kbase_ctx_flag_set(kctx, KCTX_DYING); ++ ++ dev_dbg(kbdev->dev, "Zap: Try Evict Ctx %p", kctx); ++ ++ /* ++ * At this point we know: ++ * - If eviction succeeded, it was in the queue, but now no ++ * longer is ++ * - We must cancel the jobs here. No Power Manager active reference to ++ * release. ++ * - This happens asynchronously - kbase_jd_zap_context() will wait for ++ * those jobs to be killed. ++ * - If eviction failed, then it wasn't in the queue. It is one ++ * of the following: ++ * - a. it didn't have any jobs, and so is not in the Queue or ++ * the Run Pool (not scheduled) ++ * - Hence, no more work required to cancel jobs. No Power Manager ++ * active reference to release. ++ * - b. it was in the middle of a scheduling transaction (and thus must ++ * have at least 1 job). This can happen from a syscall or a ++ * kernel thread. We still hold the jsctx_mutex, and so the thread ++ * must be waiting inside kbasep_js_try_schedule_head_ctx(), ++ * before checking whether the runpool is full. That thread will ++ * continue after we drop the mutex, and will notice the context ++ * is dying. It will rollback the transaction, killing all jobs at ++ * the same time. kbase_jd_zap_context() will wait for those jobs ++ * to be killed. ++ * - Hence, no more work required to cancel jobs, or to release the ++ * Power Manager active reference. ++ * - c. it is scheduled, and may or may not be running jobs ++ * - We must cause it to leave the runpool by stopping it from ++ * submitting any more jobs. When it finally does leave, ++ * kbasep_js_runpool_requeue_or_kill_ctx() will kill all remaining jobs ++ * (because it is dying), release the Power Manager active reference, ++ * and will not requeue the context in the queue. ++ * kbase_jd_zap_context() will wait for those jobs to be killed. ++ * - Hence, work required just to make it leave the runpool. Cancelling ++ * jobs and releasing the Power manager active reference will be ++ * handled when it leaves the runpool. ++ */ ++ if (!kbase_ctx_flag(kctx, KCTX_SCHEDULED)) { ++ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { ++ if (!list_empty( ++ &kctx->jctx.sched_info.ctx.ctx_list_entry[js])) ++ list_del_init( ++ &kctx->jctx.sched_info.ctx.ctx_list_entry[js]); ++ } ++ ++ /* The following events require us to kill off remaining jobs ++ * and update PM book-keeping: ++ * - we evicted it correctly (it must have jobs to be in the ++ * Queue) ++ * ++ * These events need no action, but take this path anyway: ++ * - Case a: it didn't have any jobs, and was never in the Queue ++ * - Case b: scheduling transaction will be partially rolled- ++ * back (this already cancels the jobs) ++ */ ++ ++ KBASE_TRACE_ADD(kbdev, JM_ZAP_NON_SCHEDULED, kctx, NULL, 0u, ++ kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ ++ dev_dbg(kbdev->dev, "Zap: Ctx %p scheduled=0", kctx); ++ ++ /* Only cancel jobs when we evicted from the ++ * queue. No Power Manager active reference was held. ++ * ++ * Having is_dying set ensures that this kills, and ++ * doesn't requeue */ ++ kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, false); ++ ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ mutex_unlock(&js_devdata->queue_mutex); ++ mutex_unlock(&kctx->jctx.lock); ++ } else { ++ unsigned long flags; ++ bool was_retained; ++ ++ /* Case c: didn't evict, but it is scheduled - it's in the Run ++ * Pool */ ++ KBASE_TRACE_ADD(kbdev, JM_ZAP_SCHEDULED, kctx, NULL, 0u, ++ kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ dev_dbg(kbdev->dev, "Zap: Ctx %p is in RunPool", kctx); ++ ++ /* Disable the ctx from submitting any more jobs */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ kbasep_js_clear_submit_allowed(js_devdata, kctx); ++ ++ /* Retain and (later) release the context whilst it is is now ++ * disallowed from submitting jobs - ensures that someone ++ * somewhere will be removing the context later on */ ++ was_retained = kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx); ++ ++ /* Since it's scheduled and we have the jsctx_mutex, it must be ++ * retained successfully */ ++ KBASE_DEBUG_ASSERT(was_retained); ++ ++ dev_dbg(kbdev->dev, "Zap: Ctx %p Kill Any Running jobs", kctx); ++ ++ /* Cancel any remaining running jobs for this kctx - if any. ++ * Submit is disallowed which takes effect immediately, so no ++ * more new jobs will appear after we do this. */ ++ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) ++ kbase_job_slot_hardstop(kctx, js, NULL); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ mutex_unlock(&js_devdata->queue_mutex); ++ mutex_unlock(&kctx->jctx.lock); ++ ++ dev_dbg(kbdev->dev, "Zap: Ctx %p Release (may or may not schedule out immediately)", ++ kctx); ++ ++ kbasep_js_runpool_release_ctx(kbdev, kctx); ++ } ++ ++ KBASE_TRACE_ADD(kbdev, JM_ZAP_DONE, kctx, NULL, 0u, 0u); ++ ++ /* After this, you must wait on both the ++ * kbase_jd_context::zero_jobs_wait and the ++ * kbasep_js_kctx_info::ctx::is_scheduled_waitq - to wait for the jobs ++ * to be destroyed, and the context to be de-scheduled (if it was on the ++ * runpool). ++ * ++ * kbase_jd_zap_context() will do this. */ ++} ++ ++static inline int trace_get_refcnt(struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++ return atomic_read(&kctx->refcount); ++} ++ ++/** ++ * kbase_js_foreach_ctx_job(): - Call a function on all jobs in context ++ * @kctx: Pointer to context. ++ * @callback: Pointer to function to call for each job. ++ * ++ * Call a function on all jobs belonging to a non-queued, non-running ++ * context, and detach the jobs from the context as it goes. ++ * ++ * Due to the locks that might be held at the time of the call, the callback ++ * may need to defer work on a workqueue to complete its actions (e.g. when ++ * cancelling jobs) ++ * ++ * Atoms will be removed from the queue, so this must only be called when ++ * cancelling jobs (which occurs as part of context destruction). ++ * ++ * The locking conditions on the caller are as follows: ++ * - it will be holding kbasep_js_kctx_info::ctx::jsctx_mutex. ++ */ ++static void kbase_js_foreach_ctx_job(struct kbase_context *kctx, ++ kbasep_js_ctx_job_cb callback) ++{ ++ struct kbase_device *kbdev; ++ unsigned long flags; ++ u32 js; ++ ++ kbdev = kctx->kbdev; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_POLICY_FOREACH_CTX_JOBS, kctx, NULL, ++ 0u, trace_get_refcnt(kbdev, kctx)); ++ ++ /* Invoke callback on jobs on each slot in turn */ ++ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) ++ jsctx_queue_foreach(kctx, js, callback); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++} +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_js.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_js.h +new file mode 100755 +index 000000000000..ddada8e468a1 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_js.h +@@ -0,0 +1,925 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/** ++ * @file mali_kbase_js.h ++ * Job Scheduler APIs. ++ */ ++ ++#ifndef _KBASE_JS_H_ ++#define _KBASE_JS_H_ ++ ++#include "mali_kbase_js_defs.h" ++#include "mali_kbase_context.h" ++#include "mali_kbase_defs.h" ++#include "mali_kbase_debug.h" ++ ++#include "mali_kbase_js_ctx_attr.h" ++ ++/** ++ * @addtogroup base_api ++ * @{ ++ */ ++ ++/** ++ * @addtogroup base_kbase_api ++ * @{ ++ */ ++ ++/** ++ * @addtogroup kbase_js Job Scheduler Internal APIs ++ * @{ ++ * ++ * These APIs are Internal to KBase. ++ */ ++ ++/** ++ * @brief Initialize the Job Scheduler ++ * ++ * The struct kbasep_js_device_data sub-structure of \a kbdev must be zero ++ * initialized before passing to the kbasep_js_devdata_init() function. This is ++ * to give efficient error path code. ++ */ ++int kbasep_js_devdata_init(struct kbase_device * const kbdev); ++ ++/** ++ * @brief Halt the Job Scheduler. ++ * ++ * It is safe to call this on \a kbdev even if it the kbasep_js_device_data ++ * sub-structure was never initialized/failed initialization, to give efficient ++ * error-path code. ++ * ++ * For this to work, the struct kbasep_js_device_data sub-structure of \a kbdev must ++ * be zero initialized before passing to the kbasep_js_devdata_init() ++ * function. This is to give efficient error path code. ++ * ++ * It is a Programming Error to call this whilst there are still kbase_context ++ * structures registered with this scheduler. ++ * ++ */ ++void kbasep_js_devdata_halt(struct kbase_device *kbdev); ++ ++/** ++ * @brief Terminate the Job Scheduler ++ * ++ * It is safe to call this on \a kbdev even if it the kbasep_js_device_data ++ * sub-structure was never initialized/failed initialization, to give efficient ++ * error-path code. ++ * ++ * For this to work, the struct kbasep_js_device_data sub-structure of \a kbdev must ++ * be zero initialized before passing to the kbasep_js_devdata_init() ++ * function. This is to give efficient error path code. ++ * ++ * It is a Programming Error to call this whilst there are still kbase_context ++ * structures registered with this scheduler. ++ */ ++void kbasep_js_devdata_term(struct kbase_device *kbdev); ++ ++/** ++ * @brief Initialize the Scheduling Component of a struct kbase_context on the Job Scheduler. ++ * ++ * This effectively registers a struct kbase_context with a Job Scheduler. ++ * ++ * It does not register any jobs owned by the struct kbase_context with the scheduler. ++ * Those must be separately registered by kbasep_js_add_job(). ++ * ++ * The struct kbase_context must be zero intitialized before passing to the ++ * kbase_js_init() function. This is to give efficient error path code. ++ */ ++int kbasep_js_kctx_init(struct kbase_context * const kctx); ++ ++/** ++ * @brief Terminate the Scheduling Component of a struct kbase_context on the Job Scheduler ++ * ++ * This effectively de-registers a struct kbase_context from its Job Scheduler ++ * ++ * It is safe to call this on a struct kbase_context that has never had or failed ++ * initialization of its jctx.sched_info member, to give efficient error-path ++ * code. ++ * ++ * For this to work, the struct kbase_context must be zero intitialized before passing ++ * to the kbase_js_init() function. ++ * ++ * It is a Programming Error to call this whilst there are still jobs ++ * registered with this context. ++ */ ++void kbasep_js_kctx_term(struct kbase_context *kctx); ++ ++/** ++ * @brief Add a job chain to the Job Scheduler, and take necessary actions to ++ * schedule the context/run the job. ++ * ++ * This atomically does the following: ++ * - Update the numbers of jobs information ++ * - Add the job to the run pool if necessary (part of init_job) ++ * ++ * Once this is done, then an appropriate action is taken: ++ * - If the ctx is scheduled, it attempts to start the next job (which might be ++ * this added job) ++ * - Otherwise, and if this is the first job on the context, it enqueues it on ++ * the Policy Queue ++ * ++ * The Policy's Queue can be updated by this in the following ways: ++ * - In the above case that this is the first job on the context ++ * - If the context is high priority and the context is not scheduled, then it ++ * could cause the Policy to schedule out a low-priority context, allowing ++ * this context to be scheduled in. ++ * ++ * If the context is already scheduled on the RunPool, then adding a job to it ++ * is guarenteed not to update the Policy Queue. And so, the caller is ++ * guarenteed to not need to try scheduling a context from the Run Pool - it ++ * can safely assert that the result is false. ++ * ++ * It is a programming error to have more than U32_MAX jobs in flight at a time. ++ * ++ * The following locking conditions are made on the caller: ++ * - it must \em not hold kbasep_js_kctx_info::ctx::jsctx_mutex. ++ * - it must \em not hold hwaccess_lock (as this will be obtained internally) ++ * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this will be ++ * obtained internally) ++ * - it must \em not hold kbasep_jd_device_data::queue_mutex (again, it's used internally). ++ * ++ * @return true indicates that the Policy Queue was updated, and so the ++ * caller will need to try scheduling a context onto the Run Pool. ++ * @return false indicates that no updates were made to the Policy Queue, ++ * so no further action is required from the caller. This is \b always returned ++ * when the context is currently scheduled. ++ */ ++bool kbasep_js_add_job(struct kbase_context *kctx, struct kbase_jd_atom *atom); ++ ++/** ++ * @brief Remove a job chain from the Job Scheduler, except for its 'retained state'. ++ * ++ * Completely removing a job requires several calls: ++ * - kbasep_js_copy_atom_retained_state(), to capture the 'retained state' of ++ * the atom ++ * - kbasep_js_remove_job(), to partially remove the atom from the Job Scheduler ++ * - kbasep_js_runpool_release_ctx_and_katom_retained_state(), to release the ++ * remaining state held as part of the job having been run. ++ * ++ * In the common case of atoms completing normally, this set of actions is more optimal for spinlock purposes than having kbasep_js_remove_job() handle all of the actions. ++ * ++ * In the case of cancelling atoms, it is easier to call kbasep_js_remove_cancelled_job(), which handles all the necessary actions. ++ * ++ * It is a programming error to call this when: ++ * - \a atom is not a job belonging to kctx. ++ * - \a atom has already been removed from the Job Scheduler. ++ * - \a atom is still in the runpool ++ * ++ * Do not use this for removing jobs being killed by kbase_jd_cancel() - use ++ * kbasep_js_remove_cancelled_job() instead. ++ * ++ * The following locking conditions are made on the caller: ++ * - it must hold kbasep_js_kctx_info::ctx::jsctx_mutex. ++ * ++ */ ++void kbasep_js_remove_job(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *atom); ++ ++/** ++ * @brief Completely remove a job chain from the Job Scheduler, in the case ++ * where the job chain was cancelled. ++ * ++ * This is a variant of kbasep_js_remove_job() that takes care of removing all ++ * of the retained state too. This is generally useful for cancelled atoms, ++ * which need not be handled in an optimal way. ++ * ++ * It is a programming error to call this when: ++ * - \a atom is not a job belonging to kctx. ++ * - \a atom has already been removed from the Job Scheduler. ++ * - \a atom is still in the runpool: ++ * - it is not being killed with kbasep_jd_cancel() ++ * ++ * The following locking conditions are made on the caller: ++ * - it must hold kbasep_js_kctx_info::ctx::jsctx_mutex. ++ * - it must \em not hold the hwaccess_lock, (as this will be obtained ++ * internally) ++ * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this could be ++ * obtained internally) ++ * ++ * @return true indicates that ctx attributes have changed and the caller ++ * should call kbase_js_sched_all() to try to run more jobs ++ * @return false otherwise ++ */ ++bool kbasep_js_remove_cancelled_job(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ struct kbase_jd_atom *katom); ++ ++/** ++ * @brief Refcount a context as being busy, preventing it from being scheduled ++ * out. ++ * ++ * @note This function can safely be called from IRQ context. ++ * ++ * The following locking conditions are made on the caller: ++ * - it must \em not hold mmu_hw_mutex and hwaccess_lock, because they will be ++ * used internally. ++ * ++ * @return value != false if the retain succeeded, and the context will not be scheduled out. ++ * @return false if the retain failed (because the context is being/has been scheduled out). ++ */ ++bool kbasep_js_runpool_retain_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); ++ ++/** ++ * @brief Refcount a context as being busy, preventing it from being scheduled ++ * out. ++ * ++ * @note This function can safely be called from IRQ context. ++ * ++ * The following locks must be held by the caller: ++ * - mmu_hw_mutex, hwaccess_lock ++ * ++ * @return value != false if the retain succeeded, and the context will not be scheduled out. ++ * @return false if the retain failed (because the context is being/has been scheduled out). ++ */ ++bool kbasep_js_runpool_retain_ctx_nolock(struct kbase_device *kbdev, struct kbase_context *kctx); ++ ++/** ++ * @brief Lookup a context in the Run Pool based upon its current address space ++ * and ensure that is stays scheduled in. ++ * ++ * The context is refcounted as being busy to prevent it from scheduling ++ * out. It must be released with kbasep_js_runpool_release_ctx() when it is no ++ * longer required to stay scheduled in. ++ * ++ * @note This function can safely be called from IRQ context. ++ * ++ * The following locking conditions are made on the caller: ++ * - it must \em not hold the hwaccess_lock, because it will be used internally. ++ * If the hwaccess_lock is already held, then the caller should use ++ * kbasep_js_runpool_lookup_ctx_nolock() instead. ++ * ++ * @return a valid struct kbase_context on success, which has been refcounted as being busy. ++ * @return NULL on failure, indicating that no context was found in \a as_nr ++ */ ++struct kbase_context *kbasep_js_runpool_lookup_ctx(struct kbase_device *kbdev, int as_nr); ++ ++/** ++ * @brief Handling the requeuing/killing of a context that was evicted from the ++ * policy queue or runpool. ++ * ++ * This should be used whenever handing off a context that has been evicted ++ * from the policy queue or the runpool: ++ * - If the context is not dying and has jobs, it gets re-added to the policy ++ * queue ++ * - Otherwise, it is not added ++ * ++ * In addition, if the context is dying the jobs are killed asynchronously. ++ * ++ * In all cases, the Power Manager active reference is released ++ * (kbase_pm_context_idle()) whenever the has_pm_ref parameter is true. \a ++ * has_pm_ref must be set to false whenever the context was not previously in ++ * the runpool and does not hold a Power Manager active refcount. Note that ++ * contexts in a rollback of kbasep_js_try_schedule_head_ctx() might have an ++ * active refcount even though they weren't in the runpool. ++ * ++ * The following locking conditions are made on the caller: ++ * - it must hold kbasep_js_kctx_info::ctx::jsctx_mutex. ++ * - it must \em not hold kbasep_jd_device_data::queue_mutex (as this will be ++ * obtained internally) ++ */ ++void kbasep_js_runpool_requeue_or_kill_ctx(struct kbase_device *kbdev, struct kbase_context *kctx, bool has_pm_ref); ++ ++/** ++ * @brief Release a refcount of a context being busy, allowing it to be ++ * scheduled out. ++ * ++ * When the refcount reaches zero and the context \em might be scheduled out ++ * (depending on whether the Scheudling Policy has deemed it so, or if it has run ++ * out of jobs). ++ * ++ * If the context does get scheduled out, then The following actions will be ++ * taken as part of deschduling a context: ++ * - For the context being descheduled: ++ * - If the context is in the processing of dying (all the jobs are being ++ * removed from it), then descheduling also kills off any jobs remaining in the ++ * context. ++ * - If the context is not dying, and any jobs remain after descheduling the ++ * context then it is re-enqueued to the Policy's Queue. ++ * - Otherwise, the context is still known to the scheduler, but remains absent ++ * from the Policy Queue until a job is next added to it. ++ * - In all descheduling cases, the Power Manager active reference (obtained ++ * during kbasep_js_try_schedule_head_ctx()) is released (kbase_pm_context_idle()). ++ * ++ * Whilst the context is being descheduled, this also handles actions that ++ * cause more atoms to be run: ++ * - Attempt submitting atoms when the Context Attributes on the Runpool have ++ * changed. This is because the context being scheduled out could mean that ++ * there are more opportunities to run atoms. ++ * - Attempt submitting to a slot that was previously blocked due to affinity ++ * restrictions. This is usually only necessary when releasing a context ++ * happens as part of completing a previous job, but is harmless nonetheless. ++ * - Attempt scheduling in a new context (if one is available), and if necessary, ++ * running a job from that new context. ++ * ++ * Unlike retaining a context in the runpool, this function \b cannot be called ++ * from IRQ context. ++ * ++ * It is a programming error to call this on a \a kctx that is not currently ++ * scheduled, or that already has a zero refcount. ++ * ++ * The following locking conditions are made on the caller: ++ * - it must \em not hold the hwaccess_lock, because it will be used internally. ++ * - it must \em not hold kbasep_js_kctx_info::ctx::jsctx_mutex. ++ * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this will be ++ * obtained internally) ++ * - it must \em not hold the kbase_device::mmu_hw_mutex (as this will be ++ * obtained internally) ++ * - it must \em not hold kbasep_jd_device_data::queue_mutex (as this will be ++ * obtained internally) ++ * ++ */ ++void kbasep_js_runpool_release_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); ++ ++/** ++ * @brief Variant of kbasep_js_runpool_release_ctx() that handles additional ++ * actions from completing an atom. ++ * ++ * This is usually called as part of completing an atom and releasing the ++ * refcount on the context held by the atom. ++ * ++ * Therefore, the extra actions carried out are part of handling actions queued ++ * on a completed atom, namely: ++ * - Releasing the atom's context attributes ++ * - Retrying the submission on a particular slot, because we couldn't submit ++ * on that slot from an IRQ handler. ++ * ++ * The locking conditions of this function are the same as those for ++ * kbasep_js_runpool_release_ctx() ++ */ ++void kbasep_js_runpool_release_ctx_and_katom_retained_state(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbasep_js_atom_retained_state *katom_retained_state); ++ ++/** ++ * @brief Variant of kbase_js_runpool_release_ctx() that assumes that ++ * kbasep_js_device_data::runpool_mutex and ++ * kbasep_js_kctx_info::ctx::jsctx_mutex are held by the caller, and does not ++ * attempt to schedule new contexts. ++ */ ++void kbasep_js_runpool_release_ctx_nolock(struct kbase_device *kbdev, ++ struct kbase_context *kctx); ++ ++/** ++ * @brief Schedule in a privileged context ++ * ++ * This schedules a context in regardless of the context priority. ++ * If the runpool is full, a context will be forced out of the runpool and the function will wait ++ * for the new context to be scheduled in. ++ * The context will be kept scheduled in (and the corresponding address space reserved) until ++ * kbasep_js_release_privileged_ctx is called). ++ * ++ * The following locking conditions are made on the caller: ++ * - it must \em not hold the hwaccess_lock, because it will be used internally. ++ * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this will be ++ * obtained internally) ++ * - it must \em not hold the kbase_device::mmu_hw_mutex (as this will be ++ * obtained internally) ++ * - it must \em not hold kbasep_jd_device_data::queue_mutex (again, it's used internally). ++ * - it must \em not hold kbasep_js_kctx_info::ctx::jsctx_mutex, because it will ++ * be used internally. ++ * ++ */ ++void kbasep_js_schedule_privileged_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); ++ ++/** ++ * @brief Release a privileged context, allowing it to be scheduled out. ++ * ++ * See kbasep_js_runpool_release_ctx for potential side effects. ++ * ++ * The following locking conditions are made on the caller: ++ * - it must \em not hold the hwaccess_lock, because it will be used internally. ++ * - it must \em not hold kbasep_js_kctx_info::ctx::jsctx_mutex. ++ * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this will be ++ * obtained internally) ++ * - it must \em not hold the kbase_device::mmu_hw_mutex (as this will be ++ * obtained internally) ++ * ++ */ ++void kbasep_js_release_privileged_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); ++ ++/** ++ * @brief Try to submit the next job on each slot ++ * ++ * The following locks may be used: ++ * - kbasep_js_device_data::runpool_mutex ++ * - hwaccess_lock ++ */ ++void kbase_js_try_run_jobs(struct kbase_device *kbdev); ++ ++/** ++ * @brief Suspend the job scheduler during a Power Management Suspend event. ++ * ++ * Causes all contexts to be removed from the runpool, and prevents any ++ * contexts from (re)entering the runpool. ++ * ++ * This does not handle suspending the one privileged context: the caller must ++ * instead do this by by suspending the GPU HW Counter Instrumentation. ++ * ++ * This will eventually cause all Power Management active references held by ++ * contexts on the runpool to be released, without running any more atoms. ++ * ++ * The caller must then wait for all Power Mangement active refcount to become ++ * zero before completing the suspend. ++ * ++ * The emptying mechanism may take some time to complete, since it can wait for ++ * jobs to complete naturally instead of forcing them to end quickly. However, ++ * this is bounded by the Job Scheduler's Job Timeouts. Hence, this ++ * function is guaranteed to complete in a finite time. ++ */ ++void kbasep_js_suspend(struct kbase_device *kbdev); ++ ++/** ++ * @brief Resume the Job Scheduler after a Power Management Resume event. ++ * ++ * This restores the actions from kbasep_js_suspend(): ++ * - Schedules contexts back into the runpool ++ * - Resumes running atoms on the GPU ++ */ ++void kbasep_js_resume(struct kbase_device *kbdev); ++ ++/** ++ * @brief Submit an atom to the job scheduler. ++ * ++ * The atom is enqueued on the context's ringbuffer. The caller must have ++ * ensured that all dependencies can be represented in the ringbuffer. ++ * ++ * Caller must hold jctx->lock ++ * ++ * @param[in] kctx Context pointer ++ * @param[in] atom Pointer to the atom to submit ++ * ++ * @return Whether the context requires to be enqueued. */ ++bool kbase_js_dep_resolved_submit(struct kbase_context *kctx, ++ struct kbase_jd_atom *katom); ++ ++/** ++ * jsctx_ll_flush_to_rb() - Pushes atoms from the linked list to ringbuffer. ++ * @kctx: Context Pointer ++ * @prio: Priority (specifies the queue together with js). ++ * @js: Job slot (specifies the queue together with prio). ++ * ++ * Pushes all possible atoms from the linked list to the ringbuffer. ++ * Number of atoms are limited to free space in the ringbuffer and ++ * number of available atoms in the linked list. ++ * ++ */ ++void jsctx_ll_flush_to_rb(struct kbase_context *kctx, int prio, int js); ++/** ++ * @brief Pull an atom from a context in the job scheduler for execution. ++ * ++ * The atom will not be removed from the ringbuffer at this stage. ++ * ++ * The HW access lock must be held when calling this function. ++ * ++ * @param[in] kctx Context to pull from ++ * @param[in] js Job slot to pull from ++ * @return Pointer to an atom, or NULL if there are no atoms for this ++ * slot that can be currently run. ++ */ ++struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, int js); ++ ++/** ++ * @brief Return an atom to the job scheduler ringbuffer. ++ * ++ * An atom is 'unpulled' if execution is stopped but intended to be returned to ++ * later. The most common reason for this is that the atom has been ++ * soft-stopped. ++ * ++ * Note that if multiple atoms are to be 'unpulled', they must be returned in ++ * the reverse order to which they were originally pulled. It is a programming ++ * error to return atoms in any other order. ++ * ++ * The HW access lock must be held when calling this function. ++ * ++ * @param[in] kctx Context pointer ++ * @param[in] atom Pointer to the atom to unpull ++ */ ++void kbase_js_unpull(struct kbase_context *kctx, struct kbase_jd_atom *katom); ++ ++/** ++ * @brief Complete an atom from jd_done_worker(), removing it from the job ++ * scheduler ringbuffer. ++ * ++ * If the atom failed then all dependee atoms marked for failure propagation ++ * will also fail. ++ * ++ * @param[in] kctx Context pointer ++ * @param[in] katom Pointer to the atom to complete ++ * @return true if the context is now idle (no jobs pulled) ++ * false otherwise ++ */ ++bool kbase_js_complete_atom_wq(struct kbase_context *kctx, ++ struct kbase_jd_atom *katom); ++ ++/** ++ * @brief Complete an atom. ++ * ++ * Most of the work required to complete an atom will be performed by ++ * jd_done_worker(). ++ * ++ * The HW access lock must be held when calling this function. ++ * ++ * @param[in] katom Pointer to the atom to complete ++ * @param[in] end_timestamp The time that the atom completed (may be NULL) ++ * ++ * Return: Atom that has now been unblocked and can now be run, or NULL if none ++ */ ++struct kbase_jd_atom *kbase_js_complete_atom(struct kbase_jd_atom *katom, ++ ktime_t *end_timestamp); ++ ++/** ++ * @brief Submit atoms from all available contexts. ++ * ++ * This will attempt to submit as many jobs as possible to the provided job ++ * slots. It will exit when either all job slots are full, or all contexts have ++ * been used. ++ * ++ * @param[in] kbdev Device pointer ++ * @param[in] js_mask Mask of job slots to submit to ++ */ ++void kbase_js_sched(struct kbase_device *kbdev, int js_mask); ++ ++/** ++ * kbase_jd_zap_context - Attempt to deschedule a context that is being ++ * destroyed ++ * @kctx: Context pointer ++ * ++ * This will attempt to remove a context from any internal job scheduler queues ++ * and perform any other actions to ensure a context will not be submitted ++ * from. ++ * ++ * If the context is currently scheduled, then the caller must wait for all ++ * pending jobs to complete before taking any further action. ++ */ ++void kbase_js_zap_context(struct kbase_context *kctx); ++ ++/** ++ * @brief Validate an atom ++ * ++ * This will determine whether the atom can be scheduled onto the GPU. Atoms ++ * with invalid combinations of core requirements will be rejected. ++ * ++ * @param[in] kbdev Device pointer ++ * @param[in] katom Atom to validate ++ * @return true if atom is valid ++ * false otherwise ++ */ ++bool kbase_js_is_atom_valid(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom); ++ ++/** ++ * kbase_js_set_timeouts - update all JS timeouts with user specified data ++ * @kbdev: Device pointer ++ * ++ * Timeouts are specified through the 'js_timeouts' sysfs file. If a timeout is ++ * set to a positive number then that becomes the new value used, if a timeout ++ * is negative then the default is set. ++ */ ++void kbase_js_set_timeouts(struct kbase_device *kbdev); ++ ++/* ++ * Helpers follow ++ */ ++ ++/** ++ * @brief Check that a context is allowed to submit jobs on this policy ++ * ++ * The purpose of this abstraction is to hide the underlying data size, and wrap up ++ * the long repeated line of code. ++ * ++ * As with any bool, never test the return value with true. ++ * ++ * The caller must hold hwaccess_lock. ++ */ ++static inline bool kbasep_js_is_submit_allowed(struct kbasep_js_device_data *js_devdata, struct kbase_context *kctx) ++{ ++ u16 test_bit; ++ ++ /* Ensure context really is scheduled in */ ++ KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); ++ KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ ++ test_bit = (u16) (1u << kctx->as_nr); ++ ++ return (bool) (js_devdata->runpool_irq.submit_allowed & test_bit); ++} ++ ++/** ++ * @brief Allow a context to submit jobs on this policy ++ * ++ * The purpose of this abstraction is to hide the underlying data size, and wrap up ++ * the long repeated line of code. ++ * ++ * The caller must hold hwaccess_lock. ++ */ ++static inline void kbasep_js_set_submit_allowed(struct kbasep_js_device_data *js_devdata, struct kbase_context *kctx) ++{ ++ u16 set_bit; ++ ++ /* Ensure context really is scheduled in */ ++ KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); ++ KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ ++ set_bit = (u16) (1u << kctx->as_nr); ++ ++ dev_dbg(kctx->kbdev->dev, "JS: Setting Submit Allowed on %p (as=%d)", kctx, kctx->as_nr); ++ ++ js_devdata->runpool_irq.submit_allowed |= set_bit; ++} ++ ++/** ++ * @brief Prevent a context from submitting more jobs on this policy ++ * ++ * The purpose of this abstraction is to hide the underlying data size, and wrap up ++ * the long repeated line of code. ++ * ++ * The caller must hold hwaccess_lock. ++ */ ++static inline void kbasep_js_clear_submit_allowed(struct kbasep_js_device_data *js_devdata, struct kbase_context *kctx) ++{ ++ u16 clear_bit; ++ u16 clear_mask; ++ ++ /* Ensure context really is scheduled in */ ++ KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); ++ KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ ++ clear_bit = (u16) (1u << kctx->as_nr); ++ clear_mask = ~clear_bit; ++ ++ dev_dbg(kctx->kbdev->dev, "JS: Clearing Submit Allowed on %p (as=%d)", kctx, kctx->as_nr); ++ ++ js_devdata->runpool_irq.submit_allowed &= clear_mask; ++} ++ ++/** ++ * @brief Manage the 'retry_submit_on_slot' part of a kbase_jd_atom ++ */ ++static inline void kbasep_js_clear_job_retry_submit(struct kbase_jd_atom *atom) ++{ ++ atom->retry_submit_on_slot = KBASEP_JS_RETRY_SUBMIT_SLOT_INVALID; ++} ++ ++/** ++ * Mark a slot as requiring resubmission by carrying that information on a ++ * completing atom. ++ * ++ * @note This can ASSERT in debug builds if the submit slot has been set to ++ * something other than the current value for @a js. This is because you might ++ * be unintentionally stopping more jobs being submitted on the old submit ++ * slot, and that might cause a scheduling-hang. ++ * ++ * @note If you can guarantee that the atoms for the original slot will be ++ * submitted on some other slot, then call kbasep_js_clear_job_retry_submit() ++ * first to silence the ASSERT. ++ */ ++static inline void kbasep_js_set_job_retry_submit_slot(struct kbase_jd_atom *atom, int js) ++{ ++ KBASE_DEBUG_ASSERT(0 <= js && js <= BASE_JM_MAX_NR_SLOTS); ++ KBASE_DEBUG_ASSERT((atom->retry_submit_on_slot == ++ KBASEP_JS_RETRY_SUBMIT_SLOT_INVALID) ++ || (atom->retry_submit_on_slot == js)); ++ ++ atom->retry_submit_on_slot = js; ++} ++ ++/** ++ * Create an initial 'invalid' atom retained state, that requires no ++ * atom-related work to be done on releasing with ++ * kbasep_js_runpool_release_ctx_and_katom_retained_state() ++ */ ++static inline void kbasep_js_atom_retained_state_init_invalid(struct kbasep_js_atom_retained_state *retained_state) ++{ ++ retained_state->event_code = BASE_JD_EVENT_NOT_STARTED; ++ retained_state->core_req = KBASEP_JS_ATOM_RETAINED_STATE_CORE_REQ_INVALID; ++ retained_state->retry_submit_on_slot = KBASEP_JS_RETRY_SUBMIT_SLOT_INVALID; ++} ++ ++/** ++ * Copy atom state that can be made available after jd_done_nolock() is called ++ * on that atom. ++ */ ++static inline void kbasep_js_atom_retained_state_copy(struct kbasep_js_atom_retained_state *retained_state, const struct kbase_jd_atom *katom) ++{ ++ retained_state->event_code = katom->event_code; ++ retained_state->core_req = katom->core_req; ++ retained_state->retry_submit_on_slot = katom->retry_submit_on_slot; ++ retained_state->sched_priority = katom->sched_priority; ++ retained_state->device_nr = katom->device_nr; ++} ++ ++/** ++ * @brief Determine whether an atom has finished (given its retained state), ++ * and so should be given back to userspace/removed from the system. ++ * ++ * Reasons for an atom not finishing include: ++ * - Being soft-stopped (and so, the atom should be resubmitted sometime later) ++ * ++ * @param[in] katom_retained_state the retained state of the atom to check ++ * @return false if the atom has not finished ++ * @return !=false if the atom has finished ++ */ ++static inline bool kbasep_js_has_atom_finished(const struct kbasep_js_atom_retained_state *katom_retained_state) ++{ ++ return (bool) (katom_retained_state->event_code != BASE_JD_EVENT_STOPPED && katom_retained_state->event_code != BASE_JD_EVENT_REMOVED_FROM_NEXT); ++} ++ ++/** ++ * @brief Determine whether a struct kbasep_js_atom_retained_state is valid ++ * ++ * An invalid struct kbasep_js_atom_retained_state is allowed, and indicates that the ++ * code should just ignore it. ++ * ++ * @param[in] katom_retained_state the atom's retained state to check ++ * @return false if the retained state is invalid, and can be ignored ++ * @return !=false if the retained state is valid ++ */ ++static inline bool kbasep_js_atom_retained_state_is_valid(const struct kbasep_js_atom_retained_state *katom_retained_state) ++{ ++ return (bool) (katom_retained_state->core_req != KBASEP_JS_ATOM_RETAINED_STATE_CORE_REQ_INVALID); ++} ++ ++static inline bool kbasep_js_get_atom_retry_submit_slot(const struct kbasep_js_atom_retained_state *katom_retained_state, int *res) ++{ ++ int js = katom_retained_state->retry_submit_on_slot; ++ ++ *res = js; ++ return (bool) (js >= 0); ++} ++ ++/** ++ * @brief Variant of kbasep_js_runpool_lookup_ctx() that can be used when the ++ * context is guaranteed to be already previously retained. ++ * ++ * It is a programming error to supply the \a as_nr of a context that has not ++ * been previously retained/has a busy refcount of zero. The only exception is ++ * when there is no ctx in \a as_nr (NULL returned). ++ * ++ * The following locking conditions are made on the caller: ++ * - it must \em not hold the hwaccess_lock, because it will be used internally. ++ * ++ * @return a valid struct kbase_context on success, with a refcount that is guaranteed ++ * to be non-zero and unmodified by this function. ++ * @return NULL on failure, indicating that no context was found in \a as_nr ++ */ ++static inline struct kbase_context *kbasep_js_runpool_lookup_ctx_noretain(struct kbase_device *kbdev, int as_nr) ++{ ++ struct kbase_context *found_kctx; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(0 <= as_nr && as_nr < BASE_MAX_NR_AS); ++ ++ found_kctx = kbdev->as_to_kctx[as_nr]; ++ KBASE_DEBUG_ASSERT(found_kctx == NULL || ++ atomic_read(&found_kctx->refcount) > 0); ++ ++ return found_kctx; ++} ++ ++/* ++ * The following locking conditions are made on the caller: ++ * - The caller must hold the kbasep_js_kctx_info::ctx::jsctx_mutex. ++ * - The caller must hold the kbasep_js_device_data::runpool_mutex ++ */ ++static inline void kbase_js_runpool_inc_context_count( ++ struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ ++ js_devdata = &kbdev->js_data; ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); ++ lockdep_assert_held(&js_devdata->runpool_mutex); ++ ++ /* Track total contexts */ ++ KBASE_DEBUG_ASSERT(js_devdata->nr_all_contexts_running < S8_MAX); ++ ++(js_devdata->nr_all_contexts_running); ++ ++ if (!kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) { ++ /* Track contexts that can submit jobs */ ++ KBASE_DEBUG_ASSERT(js_devdata->nr_user_contexts_running < ++ S8_MAX); ++ ++(js_devdata->nr_user_contexts_running); ++ } ++} ++ ++/* ++ * The following locking conditions are made on the caller: ++ * - The caller must hold the kbasep_js_kctx_info::ctx::jsctx_mutex. ++ * - The caller must hold the kbasep_js_device_data::runpool_mutex ++ */ ++static inline void kbase_js_runpool_dec_context_count( ++ struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ ++ js_devdata = &kbdev->js_data; ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); ++ lockdep_assert_held(&js_devdata->runpool_mutex); ++ ++ /* Track total contexts */ ++ --(js_devdata->nr_all_contexts_running); ++ KBASE_DEBUG_ASSERT(js_devdata->nr_all_contexts_running >= 0); ++ ++ if (!kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) { ++ /* Track contexts that can submit jobs */ ++ --(js_devdata->nr_user_contexts_running); ++ KBASE_DEBUG_ASSERT(js_devdata->nr_user_contexts_running >= 0); ++ } ++} ++ ++ ++/** ++ * @brief Submit atoms from all available contexts to all job slots. ++ * ++ * This will attempt to submit as many jobs as possible. It will exit when ++ * either all job slots are full, or all contexts have been used. ++ * ++ * @param[in] kbdev Device pointer ++ */ ++static inline void kbase_js_sched_all(struct kbase_device *kbdev) ++{ ++ kbase_js_sched(kbdev, (1 << kbdev->gpu_props.num_job_slots) - 1); ++} ++ ++extern const int ++kbasep_js_atom_priority_to_relative[BASE_JD_NR_PRIO_LEVELS]; ++ ++extern const base_jd_prio ++kbasep_js_relative_priority_to_atom[KBASE_JS_ATOM_SCHED_PRIO_COUNT]; ++ ++/** ++ * kbasep_js_atom_prio_to_sched_prio(): - Convert atom priority (base_jd_prio) ++ * to relative ordering ++ * @atom_prio: Priority ID to translate. ++ * ++ * Atom priority values for @ref base_jd_prio cannot be compared directly to ++ * find out which are higher or lower. ++ * ++ * This function will convert base_jd_prio values for successively lower ++ * priorities into a monotonically increasing sequence. That is, the lower the ++ * base_jd_prio priority, the higher the value produced by this function. This ++ * is in accordance with how the rest of the kernel treates priority. ++ * ++ * The mapping is 1:1 and the size of the valid input range is the same as the ++ * size of the valid output range, i.e. ++ * KBASE_JS_ATOM_SCHED_PRIO_COUNT == BASE_JD_NR_PRIO_LEVELS ++ * ++ * Note This must be kept in sync with BASE_JD_PRIO_<...> definitions ++ * ++ * Return: On success: a value in the inclusive range ++ * 0..KBASE_JS_ATOM_SCHED_PRIO_COUNT-1. On failure: ++ * KBASE_JS_ATOM_SCHED_PRIO_INVALID ++ */ ++static inline int kbasep_js_atom_prio_to_sched_prio(base_jd_prio atom_prio) ++{ ++ if (atom_prio >= BASE_JD_NR_PRIO_LEVELS) ++ return KBASE_JS_ATOM_SCHED_PRIO_INVALID; ++ ++ return kbasep_js_atom_priority_to_relative[atom_prio]; ++} ++ ++static inline base_jd_prio kbasep_js_sched_prio_to_atom_prio(int sched_prio) ++{ ++ unsigned int prio_idx; ++ ++ KBASE_DEBUG_ASSERT(0 <= sched_prio ++ && sched_prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT); ++ ++ prio_idx = (unsigned int)sched_prio; ++ ++ return kbasep_js_relative_priority_to_atom[prio_idx]; ++} ++ ++ /** @} *//* end group kbase_js */ ++ /** @} *//* end group base_kbase_api */ ++ /** @} *//* end group base_api */ ++ ++#endif /* _KBASE_JS_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_js_ctx_attr.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_js_ctx_attr.c +new file mode 100755 +index 000000000000..321506ada835 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_js_ctx_attr.c +@@ -0,0 +1,301 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++#include ++#include ++ ++/* ++ * Private functions follow ++ */ ++ ++/** ++ * @brief Check whether a ctx has a certain attribute, and if so, retain that ++ * attribute on the runpool. ++ * ++ * Requires: ++ * - jsctx mutex ++ * - runpool_irq spinlock ++ * - ctx is scheduled on the runpool ++ * ++ * @return true indicates a change in ctx attributes state of the runpool. ++ * In this state, the scheduler might be able to submit more jobs than ++ * previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock() ++ * or similar is called sometime later. ++ * @return false indicates no change in ctx attributes state of the runpool. ++ */ ++static bool kbasep_js_ctx_attr_runpool_retain_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ bool runpool_state_changed = false; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); ++ js_devdata = &kbdev->js_data; ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ ++ if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, attribute) != false) { ++ KBASE_DEBUG_ASSERT(js_devdata->runpool_irq.ctx_attr_ref_count[attribute] < S8_MAX); ++ ++(js_devdata->runpool_irq.ctx_attr_ref_count[attribute]); ++ ++ if (js_devdata->runpool_irq.ctx_attr_ref_count[attribute] == 1) { ++ /* First refcount indicates a state change */ ++ runpool_state_changed = true; ++ KBASE_TRACE_ADD(kbdev, JS_CTX_ATTR_NOW_ON_RUNPOOL, kctx, NULL, 0u, attribute); ++ } ++ } ++ ++ return runpool_state_changed; ++} ++ ++/** ++ * @brief Check whether a ctx has a certain attribute, and if so, release that ++ * attribute on the runpool. ++ * ++ * Requires: ++ * - jsctx mutex ++ * - runpool_irq spinlock ++ * - ctx is scheduled on the runpool ++ * ++ * @return true indicates a change in ctx attributes state of the runpool. ++ * In this state, the scheduler might be able to submit more jobs than ++ * previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock() ++ * or similar is called sometime later. ++ * @return false indicates no change in ctx attributes state of the runpool. ++ */ ++static bool kbasep_js_ctx_attr_runpool_release_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ bool runpool_state_changed = false; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); ++ js_devdata = &kbdev->js_data; ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ ++ if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, attribute) != false) { ++ KBASE_DEBUG_ASSERT(js_devdata->runpool_irq.ctx_attr_ref_count[attribute] > 0); ++ --(js_devdata->runpool_irq.ctx_attr_ref_count[attribute]); ++ ++ if (js_devdata->runpool_irq.ctx_attr_ref_count[attribute] == 0) { ++ /* Last de-refcount indicates a state change */ ++ runpool_state_changed = true; ++ KBASE_TRACE_ADD(kbdev, JS_CTX_ATTR_NOW_OFF_RUNPOOL, kctx, NULL, 0u, attribute); ++ } ++ } ++ ++ return runpool_state_changed; ++} ++ ++/** ++ * @brief Retain a certain attribute on a ctx, also retaining it on the runpool ++ * if the context is scheduled. ++ * ++ * Requires: ++ * - jsctx mutex ++ * - If the context is scheduled, then runpool_irq spinlock must also be held ++ * ++ * @return true indicates a change in ctx attributes state of the runpool. ++ * This may allow the scheduler to submit more jobs than previously. ++ * @return false indicates no change in ctx attributes state of the runpool. ++ */ ++static bool kbasep_js_ctx_attr_ctx_retain_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute) ++{ ++ struct kbasep_js_kctx_info *js_kctx_info; ++ bool runpool_state_changed = false; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); ++ KBASE_DEBUG_ASSERT(js_kctx_info->ctx.ctx_attr_ref_count[attribute] < U32_MAX); ++ ++ ++(js_kctx_info->ctx.ctx_attr_ref_count[attribute]); ++ ++ if (kbase_ctx_flag(kctx, KCTX_SCHEDULED) && js_kctx_info->ctx.ctx_attr_ref_count[attribute] == 1) { ++ /* Only ref-count the attribute on the runpool for the first time this contexts sees this attribute */ ++ KBASE_TRACE_ADD(kbdev, JS_CTX_ATTR_NOW_ON_CTX, kctx, NULL, 0u, attribute); ++ runpool_state_changed = kbasep_js_ctx_attr_runpool_retain_attr(kbdev, kctx, attribute); ++ } ++ ++ return runpool_state_changed; ++} ++ ++/* ++ * @brief Release a certain attribute on a ctx, also releasing it from the runpool ++ * if the context is scheduled. ++ * ++ * Requires: ++ * - jsctx mutex ++ * - If the context is scheduled, then runpool_irq spinlock must also be held ++ * ++ * @return true indicates a change in ctx attributes state of the runpool. ++ * This may allow the scheduler to submit more jobs than previously. ++ * @return false indicates no change in ctx attributes state of the runpool. ++ */ ++static bool kbasep_js_ctx_attr_ctx_release_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute) ++{ ++ struct kbasep_js_kctx_info *js_kctx_info; ++ bool runpool_state_changed = false; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); ++ KBASE_DEBUG_ASSERT(js_kctx_info->ctx.ctx_attr_ref_count[attribute] > 0); ++ ++ if (kbase_ctx_flag(kctx, KCTX_SCHEDULED) && js_kctx_info->ctx.ctx_attr_ref_count[attribute] == 1) { ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ /* Only de-ref-count the attribute on the runpool when this is the last ctx-reference to it */ ++ runpool_state_changed = kbasep_js_ctx_attr_runpool_release_attr(kbdev, kctx, attribute); ++ KBASE_TRACE_ADD(kbdev, JS_CTX_ATTR_NOW_OFF_CTX, kctx, NULL, 0u, attribute); ++ } ++ ++ /* De-ref must happen afterwards, because kbasep_js_ctx_attr_runpool_release() needs to check it too */ ++ --(js_kctx_info->ctx.ctx_attr_ref_count[attribute]); ++ ++ return runpool_state_changed; ++} ++ ++/* ++ * More commonly used public functions ++ */ ++ ++void kbasep_js_ctx_attr_set_initial_attrs(struct kbase_device *kbdev, struct kbase_context *kctx) ++{ ++ bool runpool_state_changed = false; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ ++ if (kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) { ++ /* This context never submits, so don't track any scheduling attributes */ ++ return; ++ } ++ ++ /* Transfer attributes held in the context flags for contexts that have submit enabled */ ++ ++ /* ... More attributes can be added here ... */ ++ ++ /* The context should not have been scheduled yet, so ASSERT if this caused ++ * runpool state changes (note that other threads *can't* affect the value ++ * of runpool_state_changed, due to how it's calculated) */ ++ KBASE_DEBUG_ASSERT(runpool_state_changed == false); ++ CSTD_UNUSED(runpool_state_changed); ++} ++ ++void kbasep_js_ctx_attr_runpool_retain_ctx(struct kbase_device *kbdev, struct kbase_context *kctx) ++{ ++ bool runpool_state_changed; ++ int i; ++ ++ /* Retain any existing attributes */ ++ for (i = 0; i < KBASEP_JS_CTX_ATTR_COUNT; ++i) { ++ if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, (enum kbasep_js_ctx_attr) i) != false) { ++ /* The context is being scheduled in, so update the runpool with the new attributes */ ++ runpool_state_changed = kbasep_js_ctx_attr_runpool_retain_attr(kbdev, kctx, (enum kbasep_js_ctx_attr) i); ++ ++ /* We don't need to know about state changed, because retaining a ++ * context occurs on scheduling it, and that itself will also try ++ * to run new atoms */ ++ CSTD_UNUSED(runpool_state_changed); ++ } ++ } ++} ++ ++bool kbasep_js_ctx_attr_runpool_release_ctx(struct kbase_device *kbdev, struct kbase_context *kctx) ++{ ++ bool runpool_state_changed = false; ++ int i; ++ ++ /* Release any existing attributes */ ++ for (i = 0; i < KBASEP_JS_CTX_ATTR_COUNT; ++i) { ++ if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, (enum kbasep_js_ctx_attr) i) != false) { ++ /* The context is being scheduled out, so update the runpool on the removed attributes */ ++ runpool_state_changed |= kbasep_js_ctx_attr_runpool_release_attr(kbdev, kctx, (enum kbasep_js_ctx_attr) i); ++ } ++ } ++ ++ return runpool_state_changed; ++} ++ ++void kbasep_js_ctx_attr_ctx_retain_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom) ++{ ++ bool runpool_state_changed = false; ++ base_jd_core_req core_req; ++ ++ KBASE_DEBUG_ASSERT(katom); ++ core_req = katom->core_req; ++ ++ if (core_req & BASE_JD_REQ_ONLY_COMPUTE) ++ runpool_state_changed |= kbasep_js_ctx_attr_ctx_retain_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE); ++ else ++ runpool_state_changed |= kbasep_js_ctx_attr_ctx_retain_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_NON_COMPUTE); ++ ++ if ((core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE | BASE_JD_REQ_T)) != 0 && (core_req & (BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_SPECIFIC_COHERENT_GROUP)) == 0) { ++ /* Atom that can run on slot1 or slot2, and can use all cores */ ++ runpool_state_changed |= kbasep_js_ctx_attr_ctx_retain_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES); ++ } ++ ++ /* We don't need to know about state changed, because retaining an ++ * atom occurs on adding it, and that itself will also try to run ++ * new atoms */ ++ CSTD_UNUSED(runpool_state_changed); ++} ++ ++bool kbasep_js_ctx_attr_ctx_release_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbasep_js_atom_retained_state *katom_retained_state) ++{ ++ bool runpool_state_changed = false; ++ base_jd_core_req core_req; ++ ++ KBASE_DEBUG_ASSERT(katom_retained_state); ++ core_req = katom_retained_state->core_req; ++ ++ /* No-op for invalid atoms */ ++ if (kbasep_js_atom_retained_state_is_valid(katom_retained_state) == false) ++ return false; ++ ++ if (core_req & BASE_JD_REQ_ONLY_COMPUTE) ++ runpool_state_changed |= kbasep_js_ctx_attr_ctx_release_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE); ++ else ++ runpool_state_changed |= kbasep_js_ctx_attr_ctx_release_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_NON_COMPUTE); ++ ++ if ((core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE | BASE_JD_REQ_T)) != 0 && (core_req & (BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_SPECIFIC_COHERENT_GROUP)) == 0) { ++ /* Atom that can run on slot1 or slot2, and can use all cores */ ++ runpool_state_changed |= kbasep_js_ctx_attr_ctx_release_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES); ++ } ++ ++ return runpool_state_changed; ++} +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_js_ctx_attr.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_js_ctx_attr.h +new file mode 100755 +index 000000000000..ce9183326a57 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_js_ctx_attr.h +@@ -0,0 +1,158 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/** ++ * @file mali_kbase_js_ctx_attr.h ++ * Job Scheduler Context Attribute APIs ++ */ ++ ++#ifndef _KBASE_JS_CTX_ATTR_H_ ++#define _KBASE_JS_CTX_ATTR_H_ ++ ++/** ++ * @addtogroup base_api ++ * @{ ++ */ ++ ++/** ++ * @addtogroup base_kbase_api ++ * @{ ++ */ ++ ++/** ++ * @addtogroup kbase_js ++ * @{ ++ */ ++ ++/** ++ * Set the initial attributes of a context (when context create flags are set) ++ * ++ * Requires: ++ * - Hold the jsctx_mutex ++ */ ++void kbasep_js_ctx_attr_set_initial_attrs(struct kbase_device *kbdev, struct kbase_context *kctx); ++ ++/** ++ * Retain all attributes of a context ++ * ++ * This occurs on scheduling in the context on the runpool (but after ++ * is_scheduled is set) ++ * ++ * Requires: ++ * - jsctx mutex ++ * - runpool_irq spinlock ++ * - ctx->is_scheduled is true ++ */ ++void kbasep_js_ctx_attr_runpool_retain_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); ++ ++/** ++ * Release all attributes of a context ++ * ++ * This occurs on scheduling out the context from the runpool (but before ++ * is_scheduled is cleared) ++ * ++ * Requires: ++ * - jsctx mutex ++ * - runpool_irq spinlock ++ * - ctx->is_scheduled is true ++ * ++ * @return true indicates a change in ctx attributes state of the runpool. ++ * In this state, the scheduler might be able to submit more jobs than ++ * previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock() ++ * or similar is called sometime later. ++ * @return false indicates no change in ctx attributes state of the runpool. ++ */ ++bool kbasep_js_ctx_attr_runpool_release_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); ++ ++/** ++ * Retain all attributes of an atom ++ * ++ * This occurs on adding an atom to a context ++ * ++ * Requires: ++ * - jsctx mutex ++ * - If the context is scheduled, then runpool_irq spinlock must also be held ++ */ ++void kbasep_js_ctx_attr_ctx_retain_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom); ++ ++/** ++ * Release all attributes of an atom, given its retained state. ++ * ++ * This occurs after (permanently) removing an atom from a context ++ * ++ * Requires: ++ * - jsctx mutex ++ * - If the context is scheduled, then runpool_irq spinlock must also be held ++ * ++ * This is a no-op when \a katom_retained_state is invalid. ++ * ++ * @return true indicates a change in ctx attributes state of the runpool. ++ * In this state, the scheduler might be able to submit more jobs than ++ * previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock() ++ * or similar is called sometime later. ++ * @return false indicates no change in ctx attributes state of the runpool. ++ */ ++bool kbasep_js_ctx_attr_ctx_release_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbasep_js_atom_retained_state *katom_retained_state); ++ ++/** ++ * Requires: ++ * - runpool_irq spinlock ++ */ ++static inline s8 kbasep_js_ctx_attr_count_on_runpool(struct kbase_device *kbdev, enum kbasep_js_ctx_attr attribute) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); ++ js_devdata = &kbdev->js_data; ++ ++ return js_devdata->runpool_irq.ctx_attr_ref_count[attribute]; ++} ++ ++/** ++ * Requires: ++ * - runpool_irq spinlock ++ */ ++static inline bool kbasep_js_ctx_attr_is_attr_on_runpool(struct kbase_device *kbdev, enum kbasep_js_ctx_attr attribute) ++{ ++ /* In general, attributes are 'on' when they have a non-zero refcount (note: the refcount will never be < 0) */ ++ return (bool) kbasep_js_ctx_attr_count_on_runpool(kbdev, attribute); ++} ++ ++/** ++ * Requires: ++ * - jsctx mutex ++ */ ++static inline bool kbasep_js_ctx_attr_is_attr_on_ctx(struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute) ++{ ++ struct kbasep_js_kctx_info *js_kctx_info; ++ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ /* In general, attributes are 'on' when they have a refcount (which should never be < 0) */ ++ return (bool) (js_kctx_info->ctx.ctx_attr_ref_count[attribute]); ++} ++ ++ /** @} *//* end group kbase_js */ ++ /** @} *//* end group base_kbase_api */ ++ /** @} *//* end group base_api */ ++ ++#endif /* _KBASE_JS_DEFS_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_js_defs.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_js_defs.h +new file mode 100755 +index 000000000000..0b4890d6b50e +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_js_defs.h +@@ -0,0 +1,386 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/** ++ * @file mali_kbase_js.h ++ * Job Scheduler Type Definitions ++ */ ++ ++#ifndef _KBASE_JS_DEFS_H_ ++#define _KBASE_JS_DEFS_H_ ++ ++/** ++ * @addtogroup base_api ++ * @{ ++ */ ++ ++/** ++ * @addtogroup base_kbase_api ++ * @{ ++ */ ++ ++/** ++ * @addtogroup kbase_js ++ * @{ ++ */ ++/* Forward decls */ ++struct kbase_device; ++struct kbase_jd_atom; ++ ++ ++typedef u32 kbase_context_flags; ++ ++struct kbasep_atom_req { ++ base_jd_core_req core_req; ++ kbase_context_flags ctx_req; ++ u32 device_nr; ++}; ++ ++/** Callback function run on all of a context's jobs registered with the Job ++ * Scheduler */ ++typedef void (*kbasep_js_ctx_job_cb)(struct kbase_device *kbdev, struct kbase_jd_atom *katom); ++ ++/** ++ * @brief Maximum number of jobs that can be submitted to a job slot whilst ++ * inside the IRQ handler. ++ * ++ * This is important because GPU NULL jobs can complete whilst the IRQ handler ++ * is running. Otherwise, it potentially allows an unlimited number of GPU NULL ++ * jobs to be submitted inside the IRQ handler, which increases IRQ latency. ++ */ ++#define KBASE_JS_MAX_JOB_SUBMIT_PER_SLOT_PER_IRQ 2 ++ ++/** ++ * @brief Context attributes ++ * ++ * Each context attribute can be thought of as a boolean value that caches some ++ * state information about either the runpool, or the context: ++ * - In the case of the runpool, it is a cache of "Do any contexts owned by ++ * the runpool have attribute X?" ++ * - In the case of a context, it is a cache of "Do any atoms owned by the ++ * context have attribute X?" ++ * ++ * The boolean value of the context attributes often affect scheduling ++ * decisions, such as affinities to use and job slots to use. ++ * ++ * To accomodate changes of state in the context, each attribute is refcounted ++ * in the context, and in the runpool for all running contexts. Specifically: ++ * - The runpool holds a refcount of how many contexts in the runpool have this ++ * attribute. ++ * - The context holds a refcount of how many atoms have this attribute. ++ */ ++enum kbasep_js_ctx_attr { ++ /** Attribute indicating a context that contains Compute jobs. That is, ++ * the context has jobs of type @ref BASE_JD_REQ_ONLY_COMPUTE ++ * ++ * @note A context can be both 'Compute' and 'Non Compute' if it contains ++ * both types of jobs. ++ */ ++ KBASEP_JS_CTX_ATTR_COMPUTE, ++ ++ /** Attribute indicating a context that contains Non-Compute jobs. That is, ++ * the context has some jobs that are \b not of type @ref ++ * BASE_JD_REQ_ONLY_COMPUTE. ++ * ++ * @note A context can be both 'Compute' and 'Non Compute' if it contains ++ * both types of jobs. ++ */ ++ KBASEP_JS_CTX_ATTR_NON_COMPUTE, ++ ++ /** Attribute indicating that a context contains compute-job atoms that ++ * aren't restricted to a coherent group, and can run on all cores. ++ * ++ * Specifically, this is when the atom's \a core_req satisfy: ++ * - (\a core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE | BASE_JD_REQ_T) // uses slot 1 or slot 2 ++ * - && !(\a core_req & BASE_JD_REQ_COHERENT_GROUP) // not restricted to coherent groups ++ * ++ * Such atoms could be blocked from running if one of the coherent groups ++ * is being used by another job slot, so tracking this context attribute ++ * allows us to prevent such situations. ++ * ++ * @note This doesn't take into account the 1-coregroup case, where all ++ * compute atoms would effectively be able to run on 'all cores', but ++ * contexts will still not always get marked with this attribute. Instead, ++ * it is the caller's responsibility to take into account the number of ++ * coregroups when interpreting this attribute. ++ * ++ * @note Whilst Tiler atoms are normally combined with ++ * BASE_JD_REQ_COHERENT_GROUP, it is possible to send such atoms without ++ * BASE_JD_REQ_COHERENT_GROUP set. This is an unlikely case, but it's easy ++ * enough to handle anyway. ++ */ ++ KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES, ++ ++ /** Must be the last in the enum */ ++ KBASEP_JS_CTX_ATTR_COUNT ++}; ++ ++enum { ++ /** Bit indicating that new atom should be started because this atom completed */ ++ KBASE_JS_ATOM_DONE_START_NEW_ATOMS = (1u << 0), ++ /** Bit indicating that the atom was evicted from the JS_NEXT registers */ ++ KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT = (1u << 1) ++}; ++ ++/** Combination of KBASE_JS_ATOM_DONE_<...> bits */ ++typedef u32 kbasep_js_atom_done_code; ++ ++/** ++ * @brief KBase Device Data Job Scheduler sub-structure ++ * ++ * This encapsulates the current context of the Job Scheduler on a particular ++ * device. This context is global to the device, and is not tied to any ++ * particular struct kbase_context running on the device. ++ * ++ * nr_contexts_running and as_free are optimized for packing together (by making ++ * them smaller types than u32). The operations on them should rarely involve ++ * masking. The use of signed types for arithmetic indicates to the compiler that ++ * the value will not rollover (which would be undefined behavior), and so under ++ * the Total License model, it is free to make optimizations based on that (i.e. ++ * to remove masking). ++ */ ++struct kbasep_js_device_data { ++ /* Sub-structure to collect together Job Scheduling data used in IRQ ++ * context. The hwaccess_lock must be held when accessing. */ ++ struct runpool_irq { ++ /** Bitvector indicating whether a currently scheduled context is allowed to submit jobs. ++ * When bit 'N' is set in this, it indicates whether the context bound to address space ++ * 'N' is allowed to submit jobs. ++ */ ++ u16 submit_allowed; ++ ++ /** Context Attributes: ++ * Each is large enough to hold a refcount of the number of contexts ++ * that can fit into the runpool. This is currently BASE_MAX_NR_AS ++ * ++ * Note that when BASE_MAX_NR_AS==16 we need 5 bits (not 4) to store ++ * the refcount. Hence, it's not worthwhile reducing this to ++ * bit-manipulation on u32s to save space (where in contrast, 4 bit ++ * sub-fields would be easy to do and would save space). ++ * ++ * Whilst this must not become negative, the sign bit is used for: ++ * - error detection in debug builds ++ * - Optimization: it is undefined for a signed int to overflow, and so ++ * the compiler can optimize for that never happening (thus, no masking ++ * is required on updating the variable) */ ++ s8 ctx_attr_ref_count[KBASEP_JS_CTX_ATTR_COUNT]; ++ ++ /* ++ * Affinity management and tracking ++ */ ++ /** Bitvector to aid affinity checking. Element 'n' bit 'i' indicates ++ * that slot 'n' is using core i (i.e. slot_affinity_refcount[n][i] > 0) */ ++ u64 slot_affinities[BASE_JM_MAX_NR_SLOTS]; ++ /** Refcount for each core owned by each slot. Used to generate the ++ * slot_affinities array of bitvectors ++ * ++ * The value of the refcount will not exceed BASE_JM_SUBMIT_SLOTS, ++ * because it is refcounted only when a job is definitely about to be ++ * submitted to a slot, and is de-refcounted immediately after a job ++ * finishes */ ++ s8 slot_affinity_refcount[BASE_JM_MAX_NR_SLOTS][64]; ++ } runpool_irq; ++ ++ /** ++ * Run Pool mutex, for managing contexts within the runpool. ++ * Unless otherwise specified, you must hold this lock whilst accessing any ++ * members that follow ++ * ++ * In addition, this is used to access: ++ * - the kbasep_js_kctx_info::runpool substructure ++ */ ++ struct mutex runpool_mutex; ++ ++ /** ++ * Queue Lock, used to access the Policy's queue of contexts independently ++ * of the Run Pool. ++ * ++ * Of course, you don't need the Run Pool lock to access this. ++ */ ++ struct mutex queue_mutex; ++ ++ /** ++ * Scheduling semaphore. This must be held when calling ++ * kbase_jm_kick() ++ */ ++ struct semaphore schedule_sem; ++ ++ /** ++ * List of contexts that can currently be pulled from ++ */ ++ struct list_head ctx_list_pullable[BASE_JM_MAX_NR_SLOTS]; ++ /** ++ * List of contexts that can not currently be pulled from, but have ++ * jobs currently running. ++ */ ++ struct list_head ctx_list_unpullable[BASE_JM_MAX_NR_SLOTS]; ++ ++ /** Number of currently scheduled user contexts (excluding ones that are not submitting jobs) */ ++ s8 nr_user_contexts_running; ++ /** Number of currently scheduled contexts (including ones that are not submitting jobs) */ ++ s8 nr_all_contexts_running; ++ ++ /** Core Requirements to match up with base_js_atom's core_req memeber ++ * @note This is a write-once member, and so no locking is required to read */ ++ base_jd_core_req js_reqs[BASE_JM_MAX_NR_SLOTS]; ++ ++ u32 scheduling_period_ns; /*< Value for JS_SCHEDULING_PERIOD_NS */ ++ u32 soft_stop_ticks; /*< Value for JS_SOFT_STOP_TICKS */ ++ u32 soft_stop_ticks_cl; /*< Value for JS_SOFT_STOP_TICKS_CL */ ++ u32 hard_stop_ticks_ss; /*< Value for JS_HARD_STOP_TICKS_SS */ ++ u32 hard_stop_ticks_cl; /*< Value for JS_HARD_STOP_TICKS_CL */ ++ u32 hard_stop_ticks_dumping; /*< Value for JS_HARD_STOP_TICKS_DUMPING */ ++ u32 gpu_reset_ticks_ss; /*< Value for JS_RESET_TICKS_SS */ ++ u32 gpu_reset_ticks_cl; /*< Value for JS_RESET_TICKS_CL */ ++ u32 gpu_reset_ticks_dumping; /*< Value for JS_RESET_TICKS_DUMPING */ ++ u32 ctx_timeslice_ns; /**< Value for JS_CTX_TIMESLICE_NS */ ++ ++ /**< Value for JS_SOFT_JOB_TIMEOUT */ ++ atomic_t soft_job_timeout_ms; ++ ++ /** List of suspended soft jobs */ ++ struct list_head suspended_soft_jobs_list; ++ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++ /* Support soft-stop on a single context */ ++ bool softstop_always; ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++ ++ /** The initalized-flag is placed at the end, to avoid cache-pollution (we should ++ * only be using this during init/term paths). ++ * @note This is a write-once member, and so no locking is required to read */ ++ int init_status; ++ ++ /* Number of contexts that can currently be pulled from */ ++ u32 nr_contexts_pullable; ++ ++ /* Number of contexts that can either be pulled from or are currently ++ * running */ ++ atomic_t nr_contexts_runnable; ++}; ++ ++/** ++ * @brief KBase Context Job Scheduling information structure ++ * ++ * This is a substructure in the struct kbase_context that encapsulates all the ++ * scheduling information. ++ */ ++struct kbasep_js_kctx_info { ++ ++ /** ++ * Job Scheduler Context information sub-structure. These members are ++ * accessed regardless of whether the context is: ++ * - In the Policy's Run Pool ++ * - In the Policy's Queue ++ * - Not queued nor in the Run Pool. ++ * ++ * You must obtain the jsctx_mutex before accessing any other members of ++ * this substructure. ++ * ++ * You may not access any of these members from IRQ context. ++ */ ++ struct kbase_jsctx { ++ struct mutex jsctx_mutex; /**< Job Scheduler Context lock */ ++ ++ /** Number of jobs ready to run - does \em not include the jobs waiting in ++ * the dispatcher, and dependency-only jobs. See kbase_jd_context::job_nr ++ * for such jobs*/ ++ u32 nr_jobs; ++ ++ /** Context Attributes: ++ * Each is large enough to hold a refcount of the number of atoms on ++ * the context. **/ ++ u32 ctx_attr_ref_count[KBASEP_JS_CTX_ATTR_COUNT]; ++ ++ /** ++ * Wait queue to wait for KCTX_SHEDULED flag state changes. ++ * */ ++ wait_queue_head_t is_scheduled_wait; ++ ++ /** Link implementing JS queues. Context can be present on one ++ * list per job slot ++ */ ++ struct list_head ctx_list_entry[BASE_JM_MAX_NR_SLOTS]; ++ } ctx; ++ ++ /* The initalized-flag is placed at the end, to avoid cache-pollution (we should ++ * only be using this during init/term paths) */ ++ int init_status; ++}; ++ ++/** Subset of atom state that can be available after jd_done_nolock() is called ++ * on that atom. A copy must be taken via kbasep_js_atom_retained_state_copy(), ++ * because the original atom could disappear. */ ++struct kbasep_js_atom_retained_state { ++ /** Event code - to determine whether the atom has finished */ ++ enum base_jd_event_code event_code; ++ /** core requirements */ ++ base_jd_core_req core_req; ++ /* priority */ ++ int sched_priority; ++ /** Job Slot to retry submitting to if submission from IRQ handler failed */ ++ int retry_submit_on_slot; ++ /* Core group atom was executed on */ ++ u32 device_nr; ++ ++}; ++ ++/** ++ * Value signifying 'no retry on a slot required' for: ++ * - kbase_js_atom_retained_state::retry_submit_on_slot ++ * - kbase_jd_atom::retry_submit_on_slot ++ */ ++#define KBASEP_JS_RETRY_SUBMIT_SLOT_INVALID (-1) ++ ++/** ++ * base_jd_core_req value signifying 'invalid' for a kbase_jd_atom_retained_state. ++ * ++ * @see kbase_atom_retained_state_is_valid() ++ */ ++#define KBASEP_JS_ATOM_RETAINED_STATE_CORE_REQ_INVALID BASE_JD_REQ_DEP ++ ++/** ++ * @brief The JS timer resolution, in microseconds ++ * ++ * Any non-zero difference in time will be at least this size. ++ */ ++#define KBASEP_JS_TICK_RESOLUTION_US 1 ++ ++/* ++ * Internal atom priority defines for kbase_jd_atom::sched_prio ++ */ ++enum { ++ KBASE_JS_ATOM_SCHED_PRIO_HIGH = 0, ++ KBASE_JS_ATOM_SCHED_PRIO_MED, ++ KBASE_JS_ATOM_SCHED_PRIO_LOW, ++ KBASE_JS_ATOM_SCHED_PRIO_COUNT, ++}; ++ ++/* Invalid priority for kbase_jd_atom::sched_prio */ ++#define KBASE_JS_ATOM_SCHED_PRIO_INVALID -1 ++ ++/* Default priority in the case of contexts with no atoms, or being lenient ++ * about invalid priorities from userspace */ ++#define KBASE_JS_ATOM_SCHED_PRIO_DEFAULT KBASE_JS_ATOM_SCHED_PRIO_MED ++ ++ /** @} *//* end group kbase_js */ ++ /** @} *//* end group base_kbase_api */ ++ /** @} *//* end group base_api */ ++ ++#endif /* _KBASE_JS_DEFS_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_linux.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_linux.h +new file mode 100755 +index 000000000000..6d1e61fd41e0 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_linux.h +@@ -0,0 +1,43 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2014 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/** ++ * @file mali_kbase_linux.h ++ * Base kernel APIs, Linux implementation. ++ */ ++ ++#ifndef _KBASE_LINUX_H_ ++#define _KBASE_LINUX_H_ ++ ++/* All things that are needed for the Linux port. */ ++#include ++#include ++#include ++#include ++#include ++ ++#if (defined(MALI_KERNEL_TEST_API) && (1 == MALI_KERNEL_TEST_API)) ++ #define KBASE_EXPORT_TEST_API(func) EXPORT_SYMBOL(func) ++#else ++ #define KBASE_EXPORT_TEST_API(func) ++#endif ++ ++#define KBASE_EXPORT_SYMBOL(func) EXPORT_SYMBOL(func) ++ ++#endif /* _KBASE_LINUX_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem.c +new file mode 100755 +index 000000000000..a105b15d641c +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem.c +@@ -0,0 +1,2875 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/** ++ * @file mali_kbase_mem.c ++ * Base kernel memory APIs ++ */ ++#ifdef CONFIG_DMA_SHARED_BUFFER ++#include ++#endif /* CONFIG_DMA_SHARED_BUFFER */ ++#ifdef CONFIG_UMP ++#include ++#endif /* CONFIG_UMP */ ++#include ++#include ++#include ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++/* This function finds out which RB tree the given GPU VA region belongs to ++ * based on the region zone */ ++static struct rb_root *kbase_reg_flags_to_rbtree(struct kbase_context *kctx, ++ struct kbase_va_region *reg) ++{ ++ struct rb_root *rbtree = NULL; ++ ++ switch (reg->flags & KBASE_REG_ZONE_MASK) { ++ case KBASE_REG_ZONE_CUSTOM_VA: ++ rbtree = &kctx->reg_rbtree_custom; ++ break; ++ case KBASE_REG_ZONE_EXEC: ++ rbtree = &kctx->reg_rbtree_exec; ++ break; ++ case KBASE_REG_ZONE_SAME_VA: ++ rbtree = &kctx->reg_rbtree_same; ++ /* fall through */ ++ default: ++ rbtree = &kctx->reg_rbtree_same; ++ break; ++ } ++ ++ return rbtree; ++} ++ ++/* This function finds out which RB tree the given pfn from the GPU VA belongs ++ * to based on the memory zone the pfn refers to */ ++static struct rb_root *kbase_gpu_va_to_rbtree(struct kbase_context *kctx, ++ u64 gpu_pfn) ++{ ++ struct rb_root *rbtree = NULL; ++ ++#ifdef CONFIG_64BIT ++ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) { ++#endif /* CONFIG_64BIT */ ++ if (gpu_pfn >= KBASE_REG_ZONE_CUSTOM_VA_BASE) ++ rbtree = &kctx->reg_rbtree_custom; ++ else if (gpu_pfn >= KBASE_REG_ZONE_EXEC_BASE) ++ rbtree = &kctx->reg_rbtree_exec; ++ else ++ rbtree = &kctx->reg_rbtree_same; ++#ifdef CONFIG_64BIT ++ } else { ++ if (gpu_pfn >= kctx->same_va_end) ++ rbtree = &kctx->reg_rbtree_custom; ++ else ++ rbtree = &kctx->reg_rbtree_same; ++ } ++#endif /* CONFIG_64BIT */ ++ ++ return rbtree; ++} ++ ++/* This function inserts a region into the tree. */ ++static void kbase_region_tracker_insert(struct kbase_context *kctx, ++ struct kbase_va_region *new_reg) ++{ ++ u64 start_pfn = new_reg->start_pfn; ++ struct rb_node **link = NULL; ++ struct rb_node *parent = NULL; ++ struct rb_root *rbtree = NULL; ++ ++ rbtree = kbase_reg_flags_to_rbtree(kctx, new_reg); ++ ++ link = &(rbtree->rb_node); ++ /* Find the right place in the tree using tree search */ ++ while (*link) { ++ struct kbase_va_region *old_reg; ++ ++ parent = *link; ++ old_reg = rb_entry(parent, struct kbase_va_region, rblink); ++ ++ /* RBTree requires no duplicate entries. */ ++ KBASE_DEBUG_ASSERT(old_reg->start_pfn != start_pfn); ++ ++ if (old_reg->start_pfn > start_pfn) ++ link = &(*link)->rb_left; ++ else ++ link = &(*link)->rb_right; ++ } ++ ++ /* Put the new node there, and rebalance tree */ ++ rb_link_node(&(new_reg->rblink), parent, link); ++ ++ rb_insert_color(&(new_reg->rblink), rbtree); ++} ++ ++/* Find allocated region enclosing free range. */ ++static struct kbase_va_region *kbase_region_tracker_find_region_enclosing_range_free( ++ struct kbase_context *kctx, u64 start_pfn, size_t nr_pages) ++{ ++ struct rb_node *rbnode = NULL; ++ struct kbase_va_region *reg = NULL; ++ struct rb_root *rbtree = NULL; ++ ++ u64 end_pfn = start_pfn + nr_pages; ++ ++ rbtree = kbase_gpu_va_to_rbtree(kctx, start_pfn); ++ ++ rbnode = rbtree->rb_node; ++ ++ while (rbnode) { ++ u64 tmp_start_pfn, tmp_end_pfn; ++ ++ reg = rb_entry(rbnode, struct kbase_va_region, rblink); ++ tmp_start_pfn = reg->start_pfn; ++ tmp_end_pfn = reg->start_pfn + reg->nr_pages; ++ ++ /* If start is lower than this, go left. */ ++ if (start_pfn < tmp_start_pfn) ++ rbnode = rbnode->rb_left; ++ /* If end is higher than this, then go right. */ ++ else if (end_pfn > tmp_end_pfn) ++ rbnode = rbnode->rb_right; ++ else /* Enclosing */ ++ return reg; ++ } ++ ++ return NULL; ++} ++ ++/* Find region enclosing given address. */ ++struct kbase_va_region *kbase_region_tracker_find_region_enclosing_address(struct kbase_context *kctx, u64 gpu_addr) ++{ ++ struct rb_node *rbnode; ++ struct kbase_va_region *reg; ++ u64 gpu_pfn = gpu_addr >> PAGE_SHIFT; ++ struct rb_root *rbtree = NULL; ++ ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ rbtree = kbase_gpu_va_to_rbtree(kctx, gpu_pfn); ++ ++ rbnode = rbtree->rb_node; ++ ++ while (rbnode) { ++ u64 tmp_start_pfn, tmp_end_pfn; ++ ++ reg = rb_entry(rbnode, struct kbase_va_region, rblink); ++ tmp_start_pfn = reg->start_pfn; ++ tmp_end_pfn = reg->start_pfn + reg->nr_pages; ++ ++ /* If start is lower than this, go left. */ ++ if (gpu_pfn < tmp_start_pfn) ++ rbnode = rbnode->rb_left; ++ /* If end is higher than this, then go right. */ ++ else if (gpu_pfn >= tmp_end_pfn) ++ rbnode = rbnode->rb_right; ++ else /* Enclosing */ ++ return reg; ++ } ++ ++ return NULL; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_region_tracker_find_region_enclosing_address); ++ ++/* Find region with given base address */ ++struct kbase_va_region *kbase_region_tracker_find_region_base_address(struct kbase_context *kctx, u64 gpu_addr) ++{ ++ u64 gpu_pfn = gpu_addr >> PAGE_SHIFT; ++ struct rb_node *rbnode = NULL; ++ struct kbase_va_region *reg = NULL; ++ struct rb_root *rbtree = NULL; ++ ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ rbtree = kbase_gpu_va_to_rbtree(kctx, gpu_pfn); ++ ++ rbnode = rbtree->rb_node; ++ ++ while (rbnode) { ++ reg = rb_entry(rbnode, struct kbase_va_region, rblink); ++ if (reg->start_pfn > gpu_pfn) ++ rbnode = rbnode->rb_left; ++ else if (reg->start_pfn < gpu_pfn) ++ rbnode = rbnode->rb_right; ++ else ++ return reg; ++ ++ } ++ ++ return NULL; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_region_tracker_find_region_base_address); ++ ++/* Find region meeting given requirements */ ++static struct kbase_va_region *kbase_region_tracker_find_region_meeting_reqs(struct kbase_context *kctx, struct kbase_va_region *reg_reqs, size_t nr_pages, size_t align) ++{ ++ struct rb_node *rbnode = NULL; ++ struct kbase_va_region *reg = NULL; ++ struct rb_root *rbtree = NULL; ++ ++ /* Note that this search is a linear search, as we do not have a target ++ address in mind, so does not benefit from the rbtree search */ ++ ++ rbtree = kbase_reg_flags_to_rbtree(kctx, reg_reqs); ++ ++ rbnode = rb_first(rbtree); ++ ++ while (rbnode) { ++ reg = rb_entry(rbnode, struct kbase_va_region, rblink); ++ if ((reg->nr_pages >= nr_pages) && ++ (reg->flags & KBASE_REG_FREE)) { ++ /* Check alignment */ ++ u64 start_pfn = (reg->start_pfn + align - 1) & ~(align - 1); ++ ++ if ((start_pfn >= reg->start_pfn) && ++ (start_pfn <= (reg->start_pfn + reg->nr_pages - 1)) && ++ ((start_pfn + nr_pages - 1) <= (reg->start_pfn + reg->nr_pages - 1))) ++ return reg; ++ } ++ rbnode = rb_next(rbnode); ++ } ++ ++ return NULL; ++} ++ ++/** ++ * @brief Remove a region object from the global list. ++ * ++ * The region reg is removed, possibly by merging with other free and ++ * compatible adjacent regions. It must be called with the context ++ * region lock held. The associated memory is not released (see ++ * kbase_free_alloced_region). Internal use only. ++ */ ++static int kbase_remove_va_region(struct kbase_context *kctx, struct kbase_va_region *reg) ++{ ++ struct rb_node *rbprev; ++ struct kbase_va_region *prev = NULL; ++ struct rb_node *rbnext; ++ struct kbase_va_region *next = NULL; ++ struct rb_root *reg_rbtree = NULL; ++ ++ int merged_front = 0; ++ int merged_back = 0; ++ int err = 0; ++ ++ reg_rbtree = kbase_reg_flags_to_rbtree(kctx, reg); ++ ++ /* Try to merge with the previous block first */ ++ rbprev = rb_prev(&(reg->rblink)); ++ if (rbprev) { ++ prev = rb_entry(rbprev, struct kbase_va_region, rblink); ++ if (prev->flags & KBASE_REG_FREE) { ++ /* We're compatible with the previous VMA, ++ * merge with it */ ++ WARN_ON((prev->flags & KBASE_REG_ZONE_MASK) != ++ (reg->flags & KBASE_REG_ZONE_MASK)); ++ prev->nr_pages += reg->nr_pages; ++ rb_erase(&(reg->rblink), reg_rbtree); ++ reg = prev; ++ merged_front = 1; ++ } ++ } ++ ++ /* Try to merge with the next block second */ ++ /* Note we do the lookup here as the tree may have been rebalanced. */ ++ rbnext = rb_next(&(reg->rblink)); ++ if (rbnext) { ++ /* We're compatible with the next VMA, merge with it */ ++ next = rb_entry(rbnext, struct kbase_va_region, rblink); ++ if (next->flags & KBASE_REG_FREE) { ++ WARN_ON((next->flags & KBASE_REG_ZONE_MASK) != ++ (reg->flags & KBASE_REG_ZONE_MASK)); ++ next->start_pfn = reg->start_pfn; ++ next->nr_pages += reg->nr_pages; ++ rb_erase(&(reg->rblink), reg_rbtree); ++ merged_back = 1; ++ if (merged_front) { ++ /* We already merged with prev, free it */ ++ kbase_free_alloced_region(reg); ++ } ++ } ++ } ++ ++ /* If we failed to merge then we need to add a new block */ ++ if (!(merged_front || merged_back)) { ++ /* ++ * We didn't merge anything. Add a new free ++ * placeholder and remove the original one. ++ */ ++ struct kbase_va_region *free_reg; ++ ++ free_reg = kbase_alloc_free_region(kctx, reg->start_pfn, reg->nr_pages, reg->flags & KBASE_REG_ZONE_MASK); ++ if (!free_reg) { ++ err = -ENOMEM; ++ goto out; ++ } ++ rb_replace_node(&(reg->rblink), &(free_reg->rblink), reg_rbtree); ++ } ++ ++ out: ++ return err; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_remove_va_region); ++ ++/** ++ * @brief Insert a VA region to the list, replacing the current at_reg. ++ */ ++static int kbase_insert_va_region_nolock(struct kbase_context *kctx, struct kbase_va_region *new_reg, struct kbase_va_region *at_reg, u64 start_pfn, size_t nr_pages) ++{ ++ struct rb_root *reg_rbtree = NULL; ++ int err = 0; ++ ++ reg_rbtree = kbase_reg_flags_to_rbtree(kctx, at_reg); ++ ++ /* Must be a free region */ ++ KBASE_DEBUG_ASSERT((at_reg->flags & KBASE_REG_FREE) != 0); ++ /* start_pfn should be contained within at_reg */ ++ KBASE_DEBUG_ASSERT((start_pfn >= at_reg->start_pfn) && (start_pfn < at_reg->start_pfn + at_reg->nr_pages)); ++ /* at least nr_pages from start_pfn should be contained within at_reg */ ++ KBASE_DEBUG_ASSERT(start_pfn + nr_pages <= at_reg->start_pfn + at_reg->nr_pages); ++ ++ new_reg->start_pfn = start_pfn; ++ new_reg->nr_pages = nr_pages; ++ ++ /* Regions are a whole use, so swap and delete old one. */ ++ if (at_reg->start_pfn == start_pfn && at_reg->nr_pages == nr_pages) { ++ rb_replace_node(&(at_reg->rblink), &(new_reg->rblink), ++ reg_rbtree); ++ kbase_free_alloced_region(at_reg); ++ } ++ /* New region replaces the start of the old one, so insert before. */ ++ else if (at_reg->start_pfn == start_pfn) { ++ at_reg->start_pfn += nr_pages; ++ KBASE_DEBUG_ASSERT(at_reg->nr_pages >= nr_pages); ++ at_reg->nr_pages -= nr_pages; ++ ++ kbase_region_tracker_insert(kctx, new_reg); ++ } ++ /* New region replaces the end of the old one, so insert after. */ ++ else if ((at_reg->start_pfn + at_reg->nr_pages) == (start_pfn + nr_pages)) { ++ at_reg->nr_pages -= nr_pages; ++ ++ kbase_region_tracker_insert(kctx, new_reg); ++ } ++ /* New region splits the old one, so insert and create new */ ++ else { ++ struct kbase_va_region *new_front_reg; ++ ++ new_front_reg = kbase_alloc_free_region(kctx, ++ at_reg->start_pfn, ++ start_pfn - at_reg->start_pfn, ++ at_reg->flags & KBASE_REG_ZONE_MASK); ++ ++ if (new_front_reg) { ++ at_reg->nr_pages -= nr_pages + new_front_reg->nr_pages; ++ at_reg->start_pfn = start_pfn + nr_pages; ++ ++ kbase_region_tracker_insert(kctx, new_front_reg); ++ kbase_region_tracker_insert(kctx, new_reg); ++ } else { ++ err = -ENOMEM; ++ } ++ } ++ ++ return err; ++} ++ ++/** ++ * @brief Add a VA region to the list. ++ */ ++int kbase_add_va_region(struct kbase_context *kctx, ++ struct kbase_va_region *reg, u64 addr, ++ size_t nr_pages, size_t align) ++{ ++ struct kbase_va_region *tmp; ++ u64 gpu_pfn = addr >> PAGE_SHIFT; ++ int err = 0; ++ ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ KBASE_DEBUG_ASSERT(NULL != reg); ++ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ if (!align) ++ align = 1; ++ ++ /* must be a power of 2 */ ++ KBASE_DEBUG_ASSERT((align & (align - 1)) == 0); ++ KBASE_DEBUG_ASSERT(nr_pages > 0); ++ ++ /* Path 1: Map a specific address. Find the enclosing region, which *must* be free. */ ++ if (gpu_pfn) { ++ struct device *dev = kctx->kbdev->dev; ++ ++ KBASE_DEBUG_ASSERT(!(gpu_pfn & (align - 1))); ++ ++ tmp = kbase_region_tracker_find_region_enclosing_range_free(kctx, gpu_pfn, nr_pages); ++ if (!tmp) { ++ dev_warn(dev, "Enclosing region not found: 0x%08llx gpu_pfn, %zu nr_pages", gpu_pfn, nr_pages); ++ err = -ENOMEM; ++ goto exit; ++ } ++ if (!(tmp->flags & KBASE_REG_FREE)) { ++ dev_warn(dev, "Zone mismatch: %lu != %lu", tmp->flags & KBASE_REG_ZONE_MASK, reg->flags & KBASE_REG_ZONE_MASK); ++ dev_warn(dev, "!(tmp->flags & KBASE_REG_FREE): tmp->start_pfn=0x%llx tmp->flags=0x%lx tmp->nr_pages=0x%zx gpu_pfn=0x%llx nr_pages=0x%zx\n", tmp->start_pfn, tmp->flags, tmp->nr_pages, gpu_pfn, nr_pages); ++ dev_warn(dev, "in function %s (%p, %p, 0x%llx, 0x%zx, 0x%zx)\n", __func__, kctx, reg, addr, nr_pages, align); ++ err = -ENOMEM; ++ goto exit; ++ } ++ ++ err = kbase_insert_va_region_nolock(kctx, reg, tmp, gpu_pfn, nr_pages); ++ if (err) { ++ dev_warn(dev, "Failed to insert va region"); ++ err = -ENOMEM; ++ goto exit; ++ } ++ ++ goto exit; ++ } ++ ++ /* Path 2: Map any free address which meets the requirements. */ ++ { ++ u64 start_pfn; ++ ++ /* ++ * Depending on the zone the allocation request is for ++ * we might need to retry it. ++ */ ++ do { ++ tmp = kbase_region_tracker_find_region_meeting_reqs( ++ kctx, reg, nr_pages, align); ++ if (tmp) { ++ start_pfn = (tmp->start_pfn + align - 1) & ++ ~(align - 1); ++ err = kbase_insert_va_region_nolock(kctx, reg, ++ tmp, start_pfn, nr_pages); ++ break; ++ } ++ ++ /* ++ * If the allocation is not from the same zone as JIT ++ * then don't retry, we're out of VA and there is ++ * nothing which can be done about it. ++ */ ++ if ((reg->flags & KBASE_REG_ZONE_MASK) != ++ KBASE_REG_ZONE_CUSTOM_VA) ++ break; ++ } while (kbase_jit_evict(kctx)); ++ ++ if (!tmp) ++ err = -ENOMEM; ++ } ++ ++ exit: ++ return err; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_add_va_region); ++ ++/** ++ * @brief Initialize the internal region tracker data structure. ++ */ ++static void kbase_region_tracker_ds_init(struct kbase_context *kctx, ++ struct kbase_va_region *same_va_reg, ++ struct kbase_va_region *exec_reg, ++ struct kbase_va_region *custom_va_reg) ++{ ++ kctx->reg_rbtree_same = RB_ROOT; ++ kbase_region_tracker_insert(kctx, same_va_reg); ++ ++ /* Although exec and custom_va_reg don't always exist, ++ * initialize unconditionally because of the mem_view debugfs ++ * implementation which relies on these being empty */ ++ kctx->reg_rbtree_exec = RB_ROOT; ++ kctx->reg_rbtree_custom = RB_ROOT; ++ ++ if (exec_reg) ++ kbase_region_tracker_insert(kctx, exec_reg); ++ if (custom_va_reg) ++ kbase_region_tracker_insert(kctx, custom_va_reg); ++} ++ ++static void kbase_region_tracker_erase_rbtree(struct rb_root *rbtree) ++{ ++ struct rb_node *rbnode; ++ struct kbase_va_region *reg; ++ ++ do { ++ rbnode = rb_first(rbtree); ++ if (rbnode) { ++ rb_erase(rbnode, rbtree); ++ reg = rb_entry(rbnode, struct kbase_va_region, rblink); ++ kbase_free_alloced_region(reg); ++ } ++ } while (rbnode); ++} ++ ++void kbase_region_tracker_term(struct kbase_context *kctx) ++{ ++ kbase_region_tracker_erase_rbtree(&kctx->reg_rbtree_same); ++ kbase_region_tracker_erase_rbtree(&kctx->reg_rbtree_exec); ++ kbase_region_tracker_erase_rbtree(&kctx->reg_rbtree_custom); ++} ++ ++/** ++ * Initialize the region tracker data structure. ++ */ ++int kbase_region_tracker_init(struct kbase_context *kctx) ++{ ++ struct kbase_va_region *same_va_reg; ++ struct kbase_va_region *exec_reg = NULL; ++ struct kbase_va_region *custom_va_reg = NULL; ++ size_t same_va_bits = sizeof(void *) * BITS_PER_BYTE; ++ u64 custom_va_size = KBASE_REG_ZONE_CUSTOM_VA_SIZE; ++ u64 gpu_va_limit = (1ULL << kctx->kbdev->gpu_props.mmu.va_bits) >> PAGE_SHIFT; ++ u64 same_va_pages; ++ int err; ++ ++ /* Take the lock as kbase_free_alloced_region requires it */ ++ kbase_gpu_vm_lock(kctx); ++ ++#if defined(CONFIG_ARM64) ++ same_va_bits = VA_BITS; ++#elif defined(CONFIG_X86_64) ++ same_va_bits = 47; ++#elif defined(CONFIG_64BIT) ++#error Unsupported 64-bit architecture ++#endif ++ ++#ifdef CONFIG_64BIT ++ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) ++ same_va_bits = 32; ++ else if (kbase_hw_has_feature(kctx->kbdev, BASE_HW_FEATURE_33BIT_VA)) ++ same_va_bits = 33; ++#endif ++ ++ if (kctx->kbdev->gpu_props.mmu.va_bits < same_va_bits) { ++ err = -EINVAL; ++ goto fail_unlock; ++ } ++ ++ same_va_pages = (1ULL << (same_va_bits - PAGE_SHIFT)) - 1; ++ /* all have SAME_VA */ ++ same_va_reg = kbase_alloc_free_region(kctx, 1, ++ same_va_pages, ++ KBASE_REG_ZONE_SAME_VA); ++ ++ if (!same_va_reg) { ++ err = -ENOMEM; ++ goto fail_unlock; ++ } ++ ++#ifdef CONFIG_64BIT ++ /* 32-bit clients have exec and custom VA zones */ ++ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) { ++#endif ++ if (gpu_va_limit <= KBASE_REG_ZONE_CUSTOM_VA_BASE) { ++ err = -EINVAL; ++ goto fail_free_same_va; ++ } ++ /* If the current size of TMEM is out of range of the ++ * virtual address space addressable by the MMU then ++ * we should shrink it to fit ++ */ ++ if ((KBASE_REG_ZONE_CUSTOM_VA_BASE + KBASE_REG_ZONE_CUSTOM_VA_SIZE) >= gpu_va_limit) ++ custom_va_size = gpu_va_limit - KBASE_REG_ZONE_CUSTOM_VA_BASE; ++ ++ exec_reg = kbase_alloc_free_region(kctx, ++ KBASE_REG_ZONE_EXEC_BASE, ++ KBASE_REG_ZONE_EXEC_SIZE, ++ KBASE_REG_ZONE_EXEC); ++ ++ if (!exec_reg) { ++ err = -ENOMEM; ++ goto fail_free_same_va; ++ } ++ ++ custom_va_reg = kbase_alloc_free_region(kctx, ++ KBASE_REG_ZONE_CUSTOM_VA_BASE, ++ custom_va_size, KBASE_REG_ZONE_CUSTOM_VA); ++ ++ if (!custom_va_reg) { ++ err = -ENOMEM; ++ goto fail_free_exec; ++ } ++#ifdef CONFIG_64BIT ++ } ++#endif ++ ++ kbase_region_tracker_ds_init(kctx, same_va_reg, exec_reg, custom_va_reg); ++ ++ kctx->same_va_end = same_va_pages + 1; ++ ++ kbase_gpu_vm_unlock(kctx); ++ return 0; ++ ++fail_free_exec: ++ kbase_free_alloced_region(exec_reg); ++fail_free_same_va: ++ kbase_free_alloced_region(same_va_reg); ++fail_unlock: ++ kbase_gpu_vm_unlock(kctx); ++ return err; ++} ++ ++int kbase_region_tracker_init_jit(struct kbase_context *kctx, u64 jit_va_pages) ++{ ++#ifdef CONFIG_64BIT ++ struct kbase_va_region *same_va; ++ struct kbase_va_region *custom_va_reg; ++ u64 same_va_bits; ++ u64 total_va_size; ++ int err; ++ ++ /* ++ * Nothing to do for 32-bit clients, JIT uses the existing ++ * custom VA zone. ++ */ ++ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) ++ return 0; ++ ++#if defined(CONFIG_ARM64) ++ same_va_bits = VA_BITS; ++#elif defined(CONFIG_X86_64) ++ same_va_bits = 47; ++#elif defined(CONFIG_64BIT) ++#error Unsupported 64-bit architecture ++#endif ++ ++ if (kbase_hw_has_feature(kctx->kbdev, BASE_HW_FEATURE_33BIT_VA)) ++ same_va_bits = 33; ++ ++ total_va_size = (1ULL << (same_va_bits - PAGE_SHIFT)) - 1; ++ ++ kbase_gpu_vm_lock(kctx); ++ ++ /* ++ * Modify the same VA free region after creation. Be careful to ensure ++ * that allocations haven't been made as they could cause an overlap ++ * to happen with existing same VA allocations and the custom VA zone. ++ */ ++ same_va = kbase_region_tracker_find_region_base_address(kctx, ++ PAGE_SIZE); ++ if (!same_va) { ++ err = -ENOMEM; ++ goto fail_unlock; ++ } ++ ++ /* The region flag or region size has changed since creation so bail. */ ++ if ((!(same_va->flags & KBASE_REG_FREE)) || ++ (same_va->nr_pages != total_va_size)) { ++ err = -ENOMEM; ++ goto fail_unlock; ++ } ++ ++ if (same_va->nr_pages < jit_va_pages || ++ kctx->same_va_end < jit_va_pages) { ++ err = -ENOMEM; ++ goto fail_unlock; ++ } ++ ++ /* It's safe to adjust the same VA zone now */ ++ same_va->nr_pages -= jit_va_pages; ++ kctx->same_va_end -= jit_va_pages; ++ ++ /* ++ * Create a custom VA zone at the end of the VA for allocations which ++ * JIT can use so it doesn't have to allocate VA from the kernel. ++ */ ++ custom_va_reg = kbase_alloc_free_region(kctx, ++ kctx->same_va_end, ++ jit_va_pages, ++ KBASE_REG_ZONE_CUSTOM_VA); ++ ++ if (!custom_va_reg) { ++ /* ++ * The context will be destroyed if we fail here so no point ++ * reverting the change we made to same_va. ++ */ ++ err = -ENOMEM; ++ goto fail_unlock; ++ } ++ ++ kbase_region_tracker_insert(kctx, custom_va_reg); ++ ++ kbase_gpu_vm_unlock(kctx); ++ return 0; ++ ++fail_unlock: ++ kbase_gpu_vm_unlock(kctx); ++ return err; ++#else ++ return 0; ++#endif ++} ++ ++int kbase_mem_init(struct kbase_device *kbdev) ++{ ++ struct kbasep_mem_device *memdev; ++ int ret; ++ ++ KBASE_DEBUG_ASSERT(kbdev); ++ ++ memdev = &kbdev->memdev; ++ kbdev->mem_pool_max_size_default = KBASE_MEM_POOL_MAX_SIZE_KCTX; ++ ++ /* Initialize memory usage */ ++ atomic_set(&memdev->used_pages, 0); ++ ++ ret = kbase_mem_pool_init(&kbdev->mem_pool, ++ KBASE_MEM_POOL_MAX_SIZE_KBDEV, ++ KBASE_MEM_POOL_4KB_PAGE_TABLE_ORDER, ++ kbdev, ++ NULL); ++ if (ret) ++ return ret; ++ ++ ret = kbase_mem_pool_init(&kbdev->lp_mem_pool, ++ (KBASE_MEM_POOL_MAX_SIZE_KBDEV >> 9), ++ KBASE_MEM_POOL_2MB_PAGE_TABLE_ORDER, ++ kbdev, ++ NULL); ++ if (ret) ++ kbase_mem_pool_term(&kbdev->mem_pool); ++ ++ return ret; ++} ++ ++void kbase_mem_halt(struct kbase_device *kbdev) ++{ ++ CSTD_UNUSED(kbdev); ++} ++ ++void kbase_mem_term(struct kbase_device *kbdev) ++{ ++ struct kbasep_mem_device *memdev; ++ int pages; ++ ++ KBASE_DEBUG_ASSERT(kbdev); ++ ++ memdev = &kbdev->memdev; ++ ++ pages = atomic_read(&memdev->used_pages); ++ if (pages != 0) ++ dev_warn(kbdev->dev, "%s: %d pages in use!\n", __func__, pages); ++ ++ kbase_mem_pool_term(&kbdev->mem_pool); ++ kbase_mem_pool_term(&kbdev->lp_mem_pool); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_mem_term); ++ ++ ++ ++ ++/** ++ * @brief Allocate a free region object. ++ * ++ * The allocated object is not part of any list yet, and is flagged as ++ * KBASE_REG_FREE. No mapping is allocated yet. ++ * ++ * zone is KBASE_REG_ZONE_CUSTOM_VA, KBASE_REG_ZONE_SAME_VA, or KBASE_REG_ZONE_EXEC ++ * ++ */ ++struct kbase_va_region *kbase_alloc_free_region(struct kbase_context *kctx, u64 start_pfn, size_t nr_pages, int zone) ++{ ++ struct kbase_va_region *new_reg; ++ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ ++ /* zone argument should only contain zone related region flags */ ++ KBASE_DEBUG_ASSERT((zone & ~KBASE_REG_ZONE_MASK) == 0); ++ KBASE_DEBUG_ASSERT(nr_pages > 0); ++ /* 64-bit address range is the max */ ++ KBASE_DEBUG_ASSERT(start_pfn + nr_pages <= (U64_MAX / PAGE_SIZE)); ++ ++ new_reg = kzalloc(sizeof(*new_reg), GFP_KERNEL); ++ ++ if (!new_reg) ++ return NULL; ++ ++ new_reg->cpu_alloc = NULL; /* no alloc bound yet */ ++ new_reg->gpu_alloc = NULL; /* no alloc bound yet */ ++ new_reg->kctx = kctx; ++ new_reg->flags = zone | KBASE_REG_FREE; ++ ++ new_reg->flags |= KBASE_REG_GROWABLE; ++ ++ new_reg->start_pfn = start_pfn; ++ new_reg->nr_pages = nr_pages; ++ ++ return new_reg; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_alloc_free_region); ++ ++/** ++ * @brief Free a region object. ++ * ++ * The described region must be freed of any mapping. ++ * ++ * If the region is not flagged as KBASE_REG_FREE, the region's ++ * alloc object will be released. ++ * It is a bug if no alloc object exists for non-free regions. ++ * ++ */ ++void kbase_free_alloced_region(struct kbase_va_region *reg) ++{ ++ if (!(reg->flags & KBASE_REG_FREE)) { ++ /* ++ * The physical allocation should have been removed from the ++ * eviction list before this function is called. However, in the ++ * case of abnormal process termination or the app leaking the ++ * memory kbase_mem_free_region is not called so it can still be ++ * on the list at termination time of the region tracker. ++ */ ++ if (!list_empty(®->gpu_alloc->evict_node)) { ++ /* ++ * Unlink the physical allocation before unmaking it ++ * evictable so that the allocation isn't grown back to ++ * its last backed size as we're going to unmap it ++ * anyway. ++ */ ++ reg->cpu_alloc->reg = NULL; ++ if (reg->cpu_alloc != reg->gpu_alloc) ++ reg->gpu_alloc->reg = NULL; ++ ++ /* ++ * If a region has been made evictable then we must ++ * unmake it before trying to free it. ++ * If the memory hasn't been reclaimed it will be ++ * unmapped and freed below, if it has been reclaimed ++ * then the operations below are no-ops. ++ */ ++ if (reg->flags & KBASE_REG_DONT_NEED) { ++ KBASE_DEBUG_ASSERT(reg->cpu_alloc->type == ++ KBASE_MEM_TYPE_NATIVE); ++ kbase_mem_evictable_unmake(reg->gpu_alloc); ++ } ++ } ++ ++ /* ++ * Remove the region from the sticky resource metadata ++ * list should it be there. ++ */ ++ kbase_sticky_resource_release(reg->kctx, NULL, ++ reg->start_pfn << PAGE_SHIFT); ++ ++ kbase_mem_phy_alloc_put(reg->cpu_alloc); ++ kbase_mem_phy_alloc_put(reg->gpu_alloc); ++ /* To detect use-after-free in debug builds */ ++ KBASE_DEBUG_CODE(reg->flags |= KBASE_REG_FREE); ++ } ++ kfree(reg); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_free_alloced_region); ++ ++int kbase_gpu_mmap(struct kbase_context *kctx, struct kbase_va_region *reg, u64 addr, size_t nr_pages, size_t align) ++{ ++ int err; ++ size_t i = 0; ++ unsigned long attr; ++ unsigned long mask = ~KBASE_REG_MEMATTR_MASK; ++ ++ if ((kctx->kbdev->system_coherency == COHERENCY_ACE) && ++ (reg->flags & KBASE_REG_SHARE_BOTH)) ++ attr = KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_OUTER_WA); ++ else ++ attr = KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_WRITE_ALLOC); ++ ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ KBASE_DEBUG_ASSERT(NULL != reg); ++ ++ err = kbase_add_va_region(kctx, reg, addr, nr_pages, align); ++ if (err) ++ return err; ++ ++ if (reg->gpu_alloc->type == KBASE_MEM_TYPE_ALIAS) { ++ u64 stride; ++ struct kbase_mem_phy_alloc *alloc; ++ ++ alloc = reg->gpu_alloc; ++ stride = alloc->imported.alias.stride; ++ KBASE_DEBUG_ASSERT(alloc->imported.alias.aliased); ++ for (i = 0; i < alloc->imported.alias.nents; i++) { ++ if (alloc->imported.alias.aliased[i].alloc) { ++ err = kbase_mmu_insert_pages(kctx, ++ reg->start_pfn + (i * stride), ++ alloc->imported.alias.aliased[i].alloc->pages + alloc->imported.alias.aliased[i].offset, ++ alloc->imported.alias.aliased[i].length, ++ reg->flags); ++ if (err) ++ goto bad_insert; ++ ++ kbase_mem_phy_alloc_gpu_mapped(alloc->imported.alias.aliased[i].alloc); ++ } else { ++ err = kbase_mmu_insert_single_page(kctx, ++ reg->start_pfn + i * stride, ++ kctx->aliasing_sink_page, ++ alloc->imported.alias.aliased[i].length, ++ (reg->flags & mask) | attr); ++ ++ if (err) ++ goto bad_insert; ++ } ++ } ++ } else { ++ err = kbase_mmu_insert_pages(kctx, reg->start_pfn, ++ kbase_get_gpu_phy_pages(reg), ++ kbase_reg_current_backed_size(reg), ++ reg->flags); ++ if (err) ++ goto bad_insert; ++ kbase_mem_phy_alloc_gpu_mapped(reg->gpu_alloc); ++ } ++ ++ return err; ++ ++bad_insert: ++ if (reg->gpu_alloc->type == KBASE_MEM_TYPE_ALIAS) { ++ u64 stride; ++ ++ stride = reg->gpu_alloc->imported.alias.stride; ++ KBASE_DEBUG_ASSERT(reg->gpu_alloc->imported.alias.aliased); ++ while (i--) ++ if (reg->gpu_alloc->imported.alias.aliased[i].alloc) { ++ kbase_mmu_teardown_pages(kctx, reg->start_pfn + (i * stride), reg->gpu_alloc->imported.alias.aliased[i].length); ++ kbase_mem_phy_alloc_gpu_unmapped(reg->gpu_alloc->imported.alias.aliased[i].alloc); ++ } ++ } ++ ++ kbase_remove_va_region(kctx, reg); ++ ++ return err; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_gpu_mmap); ++ ++static void kbase_jd_user_buf_unmap(struct kbase_context *kctx, ++ struct kbase_mem_phy_alloc *alloc, bool writeable); ++ ++int kbase_gpu_munmap(struct kbase_context *kctx, struct kbase_va_region *reg) ++{ ++ int err; ++ ++ if (reg->start_pfn == 0) ++ return 0; ++ ++ if (reg->gpu_alloc && reg->gpu_alloc->type == KBASE_MEM_TYPE_ALIAS) { ++ size_t i; ++ ++ err = kbase_mmu_teardown_pages(kctx, reg->start_pfn, reg->nr_pages); ++ KBASE_DEBUG_ASSERT(reg->gpu_alloc->imported.alias.aliased); ++ for (i = 0; i < reg->gpu_alloc->imported.alias.nents; i++) ++ if (reg->gpu_alloc->imported.alias.aliased[i].alloc) ++ kbase_mem_phy_alloc_gpu_unmapped(reg->gpu_alloc->imported.alias.aliased[i].alloc); ++ } else { ++ err = kbase_mmu_teardown_pages(kctx, reg->start_pfn, kbase_reg_current_backed_size(reg)); ++ kbase_mem_phy_alloc_gpu_unmapped(reg->gpu_alloc); ++ } ++ ++ if (reg->gpu_alloc && reg->gpu_alloc->type == ++ KBASE_MEM_TYPE_IMPORTED_USER_BUF) { ++ struct kbase_alloc_import_user_buf *user_buf = ++ ®->gpu_alloc->imported.user_buf; ++ ++ if (user_buf->current_mapping_usage_count & PINNED_ON_IMPORT) { ++ user_buf->current_mapping_usage_count &= ++ ~PINNED_ON_IMPORT; ++ ++ kbase_jd_user_buf_unmap(kctx, reg->gpu_alloc, ++ (reg->flags & KBASE_REG_GPU_WR)); ++ } ++ } ++ ++ if (err) ++ return err; ++ ++ err = kbase_remove_va_region(kctx, reg); ++ return err; ++} ++ ++static struct kbase_cpu_mapping *kbasep_find_enclosing_cpu_mapping( ++ struct kbase_context *kctx, ++ unsigned long uaddr, size_t size, u64 *offset) ++{ ++ struct vm_area_struct *vma; ++ struct kbase_cpu_mapping *map; ++ unsigned long vm_pgoff_in_region; ++ unsigned long vm_off_in_region; ++ unsigned long map_start; ++ size_t map_size; ++ ++ lockdep_assert_held(¤t->mm->mmap_sem); ++ ++ if ((uintptr_t) uaddr + size < (uintptr_t) uaddr) /* overflow check */ ++ return NULL; ++ ++ vma = find_vma_intersection(current->mm, uaddr, uaddr+size); ++ ++ if (!vma || vma->vm_start > uaddr) ++ return NULL; ++ if (vma->vm_ops != &kbase_vm_ops) ++ /* Not ours! */ ++ return NULL; ++ ++ map = vma->vm_private_data; ++ ++ if (map->kctx != kctx) ++ /* Not from this context! */ ++ return NULL; ++ ++ vm_pgoff_in_region = vma->vm_pgoff - map->region->start_pfn; ++ vm_off_in_region = vm_pgoff_in_region << PAGE_SHIFT; ++ map_start = vma->vm_start - vm_off_in_region; ++ map_size = map->region->nr_pages << PAGE_SHIFT; ++ ++ if ((uaddr + size) > (map_start + map_size)) ++ /* Not within the CPU mapping */ ++ return NULL; ++ ++ *offset = (uaddr - vma->vm_start) + vm_off_in_region; ++ ++ return map; ++} ++ ++int kbasep_find_enclosing_cpu_mapping_offset( ++ struct kbase_context *kctx, ++ unsigned long uaddr, size_t size, u64 *offset) ++{ ++ struct kbase_cpu_mapping *map; ++ ++ kbase_os_mem_map_lock(kctx); ++ ++ map = kbasep_find_enclosing_cpu_mapping(kctx, uaddr, size, offset); ++ ++ kbase_os_mem_map_unlock(kctx); ++ ++ if (!map) ++ return -EINVAL; ++ ++ return 0; ++} ++ ++KBASE_EXPORT_TEST_API(kbasep_find_enclosing_cpu_mapping_offset); ++ ++void kbase_sync_single(struct kbase_context *kctx, ++ struct tagged_addr t_cpu_pa, struct tagged_addr t_gpu_pa, ++ off_t offset, size_t size, enum kbase_sync_type sync_fn) ++{ ++ struct page *cpu_page; ++ phys_addr_t cpu_pa = as_phys_addr_t(t_cpu_pa); ++ phys_addr_t gpu_pa = as_phys_addr_t(t_gpu_pa); ++ ++ cpu_page = pfn_to_page(PFN_DOWN(cpu_pa)); ++ ++ if (likely(cpu_pa == gpu_pa)) { ++ dma_addr_t dma_addr; ++ ++ BUG_ON(!cpu_page); ++ BUG_ON(offset + size > PAGE_SIZE); ++ ++ dma_addr = kbase_dma_addr(cpu_page) + offset; ++ if (sync_fn == KBASE_SYNC_TO_CPU) ++ dma_sync_single_for_cpu(kctx->kbdev->dev, dma_addr, ++ size, DMA_BIDIRECTIONAL); ++ else if (sync_fn == KBASE_SYNC_TO_DEVICE) ++ dma_sync_single_for_device(kctx->kbdev->dev, dma_addr, ++ size, DMA_BIDIRECTIONAL); ++ } else { ++ void *src = NULL; ++ void *dst = NULL; ++ struct page *gpu_page; ++ ++ if (WARN(!gpu_pa, "No GPU PA found for infinite cache op")) ++ return; ++ ++ gpu_page = pfn_to_page(PFN_DOWN(gpu_pa)); ++ ++ if (sync_fn == KBASE_SYNC_TO_DEVICE) { ++ src = ((unsigned char *)kmap(cpu_page)) + offset; ++ dst = ((unsigned char *)kmap(gpu_page)) + offset; ++ } else if (sync_fn == KBASE_SYNC_TO_CPU) { ++ dma_sync_single_for_cpu(kctx->kbdev->dev, ++ kbase_dma_addr(gpu_page) + offset, ++ size, DMA_BIDIRECTIONAL); ++ src = ((unsigned char *)kmap(gpu_page)) + offset; ++ dst = ((unsigned char *)kmap(cpu_page)) + offset; ++ } ++ memcpy(dst, src, size); ++ kunmap(gpu_page); ++ kunmap(cpu_page); ++ if (sync_fn == KBASE_SYNC_TO_DEVICE) ++ dma_sync_single_for_device(kctx->kbdev->dev, ++ kbase_dma_addr(gpu_page) + offset, ++ size, DMA_BIDIRECTIONAL); ++ } ++} ++ ++static int kbase_do_syncset(struct kbase_context *kctx, ++ struct basep_syncset *sset, enum kbase_sync_type sync_fn) ++{ ++ int err = 0; ++ struct kbase_va_region *reg; ++ struct kbase_cpu_mapping *map; ++ unsigned long start; ++ size_t size; ++ struct tagged_addr *cpu_pa; ++ struct tagged_addr *gpu_pa; ++ u64 page_off, page_count; ++ u64 i; ++ u64 offset; ++ ++ kbase_os_mem_map_lock(kctx); ++ kbase_gpu_vm_lock(kctx); ++ ++ /* find the region where the virtual address is contained */ ++ reg = kbase_region_tracker_find_region_enclosing_address(kctx, ++ sset->mem_handle.basep.handle); ++ if (!reg) { ++ dev_warn(kctx->kbdev->dev, "Can't find region at VA 0x%016llX", ++ sset->mem_handle.basep.handle); ++ err = -EINVAL; ++ goto out_unlock; ++ } ++ ++ if (!(reg->flags & KBASE_REG_CPU_CACHED) || ++ kbase_mem_is_imported(reg->gpu_alloc->type)) ++ goto out_unlock; ++ ++ start = (uintptr_t)sset->user_addr; ++ size = (size_t)sset->size; ++ ++ map = kbasep_find_enclosing_cpu_mapping(kctx, start, size, &offset); ++ if (!map) { ++ dev_warn(kctx->kbdev->dev, "Can't find CPU mapping 0x%016lX for VA 0x%016llX", ++ start, sset->mem_handle.basep.handle); ++ err = -EINVAL; ++ goto out_unlock; ++ } ++ ++ page_off = offset >> PAGE_SHIFT; ++ offset &= ~PAGE_MASK; ++ page_count = (size + offset + (PAGE_SIZE - 1)) >> PAGE_SHIFT; ++ cpu_pa = kbase_get_cpu_phy_pages(reg); ++ gpu_pa = kbase_get_gpu_phy_pages(reg); ++ ++ if (page_off > reg->nr_pages || ++ page_off + page_count > reg->nr_pages) { ++ /* Sync overflows the region */ ++ err = -EINVAL; ++ goto out_unlock; ++ } ++ ++ /* Sync first page */ ++ if (as_phys_addr_t(cpu_pa[page_off])) { ++ size_t sz = MIN(((size_t) PAGE_SIZE - offset), size); ++ ++ kbase_sync_single(kctx, cpu_pa[page_off], gpu_pa[page_off], ++ offset, sz, sync_fn); ++ } ++ ++ /* Sync middle pages (if any) */ ++ for (i = 1; page_count > 2 && i < page_count - 1; i++) { ++ /* we grow upwards, so bail on first non-present page */ ++ if (!as_phys_addr_t(cpu_pa[page_off + i])) ++ break; ++ ++ kbase_sync_single(kctx, cpu_pa[page_off + i], ++ gpu_pa[page_off + i], 0, PAGE_SIZE, sync_fn); ++ } ++ ++ /* Sync last page (if any) */ ++ if (page_count > 1 && ++ as_phys_addr_t(cpu_pa[page_off + page_count - 1])) { ++ size_t sz = ((start + size - 1) & ~PAGE_MASK) + 1; ++ ++ kbase_sync_single(kctx, cpu_pa[page_off + page_count - 1], ++ gpu_pa[page_off + page_count - 1], 0, sz, ++ sync_fn); ++ } ++ ++out_unlock: ++ kbase_gpu_vm_unlock(kctx); ++ kbase_os_mem_map_unlock(kctx); ++ return err; ++} ++ ++int kbase_sync_now(struct kbase_context *kctx, struct basep_syncset *sset) ++{ ++ int err = -EINVAL; ++ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ KBASE_DEBUG_ASSERT(sset != NULL); ++ ++ if (sset->mem_handle.basep.handle & ~PAGE_MASK) { ++ dev_warn(kctx->kbdev->dev, ++ "mem_handle: passed parameter is invalid"); ++ return -EINVAL; ++ } ++ ++ switch (sset->type) { ++ case BASE_SYNCSET_OP_MSYNC: ++ err = kbase_do_syncset(kctx, sset, KBASE_SYNC_TO_DEVICE); ++ break; ++ ++ case BASE_SYNCSET_OP_CSYNC: ++ err = kbase_do_syncset(kctx, sset, KBASE_SYNC_TO_CPU); ++ break; ++ ++ default: ++ dev_warn(kctx->kbdev->dev, "Unknown msync op %d\n", sset->type); ++ break; ++ } ++ ++ return err; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_sync_now); ++ ++/* vm lock must be held */ ++int kbase_mem_free_region(struct kbase_context *kctx, struct kbase_va_region *reg) ++{ ++ int err; ++ ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ KBASE_DEBUG_ASSERT(NULL != reg); ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ /* ++ * Unlink the physical allocation before unmaking it evictable so ++ * that the allocation isn't grown back to its last backed size ++ * as we're going to unmap it anyway. ++ */ ++ reg->cpu_alloc->reg = NULL; ++ if (reg->cpu_alloc != reg->gpu_alloc) ++ reg->gpu_alloc->reg = NULL; ++ ++ /* ++ * If a region has been made evictable then we must unmake it ++ * before trying to free it. ++ * If the memory hasn't been reclaimed it will be unmapped and freed ++ * below, if it has been reclaimed then the operations below are no-ops. ++ */ ++ if (reg->flags & KBASE_REG_DONT_NEED) { ++ KBASE_DEBUG_ASSERT(reg->cpu_alloc->type == ++ KBASE_MEM_TYPE_NATIVE); ++ kbase_mem_evictable_unmake(reg->gpu_alloc); ++ } ++ ++ err = kbase_gpu_munmap(kctx, reg); ++ if (err) { ++ dev_warn(reg->kctx->kbdev->dev, "Could not unmap from the GPU...\n"); ++ goto out; ++ } ++ ++ /* This will also free the physical pages */ ++ kbase_free_alloced_region(reg); ++ ++ out: ++ return err; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_mem_free_region); ++ ++/** ++ * @brief Free the region from the GPU and unregister it. ++ * ++ * This function implements the free operation on a memory segment. ++ * It will loudly fail if called with outstanding mappings. ++ */ ++int kbase_mem_free(struct kbase_context *kctx, u64 gpu_addr) ++{ ++ int err = 0; ++ struct kbase_va_region *reg; ++ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ ++ if ((gpu_addr & ~PAGE_MASK) && (gpu_addr >= PAGE_SIZE)) { ++ dev_warn(kctx->kbdev->dev, "kbase_mem_free: gpu_addr parameter is invalid"); ++ return -EINVAL; ++ } ++ ++ if (0 == gpu_addr) { ++ dev_warn(kctx->kbdev->dev, "gpu_addr 0 is reserved for the ringbuffer and it's an error to try to free it using kbase_mem_free\n"); ++ return -EINVAL; ++ } ++ kbase_gpu_vm_lock(kctx); ++ ++ if (gpu_addr >= BASE_MEM_COOKIE_BASE && ++ gpu_addr < BASE_MEM_FIRST_FREE_ADDRESS) { ++ int cookie = PFN_DOWN(gpu_addr - BASE_MEM_COOKIE_BASE); ++ ++ reg = kctx->pending_regions[cookie]; ++ if (!reg) { ++ err = -EINVAL; ++ goto out_unlock; ++ } ++ ++ /* ask to unlink the cookie as we'll free it */ ++ ++ kctx->pending_regions[cookie] = NULL; ++ kctx->cookies |= (1UL << cookie); ++ ++ kbase_free_alloced_region(reg); ++ } else { ++ /* A real GPU va */ ++ /* Validate the region */ ++ reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr); ++ if (!reg || (reg->flags & KBASE_REG_FREE)) { ++ dev_warn(kctx->kbdev->dev, "kbase_mem_free called with nonexistent gpu_addr 0x%llX", ++ gpu_addr); ++ err = -EINVAL; ++ goto out_unlock; ++ } ++ ++ if ((reg->flags & KBASE_REG_ZONE_MASK) == KBASE_REG_ZONE_SAME_VA) { ++ /* SAME_VA must be freed through munmap */ ++ dev_warn(kctx->kbdev->dev, "%s called on SAME_VA memory 0x%llX", __func__, ++ gpu_addr); ++ err = -EINVAL; ++ goto out_unlock; ++ } ++ err = kbase_mem_free_region(kctx, reg); ++ } ++ ++ out_unlock: ++ kbase_gpu_vm_unlock(kctx); ++ return err; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_mem_free); ++ ++int kbase_update_region_flags(struct kbase_context *kctx, ++ struct kbase_va_region *reg, unsigned long flags) ++{ ++ KBASE_DEBUG_ASSERT(NULL != reg); ++ KBASE_DEBUG_ASSERT((flags & ~((1ul << BASE_MEM_FLAGS_NR_BITS) - 1)) == 0); ++ ++ reg->flags |= kbase_cache_enabled(flags, reg->nr_pages); ++ /* all memory is now growable */ ++ reg->flags |= KBASE_REG_GROWABLE; ++ ++ if (flags & BASE_MEM_GROW_ON_GPF) ++ reg->flags |= KBASE_REG_PF_GROW; ++ ++ if (flags & BASE_MEM_PROT_CPU_WR) ++ reg->flags |= KBASE_REG_CPU_WR; ++ ++ if (flags & BASE_MEM_PROT_CPU_RD) ++ reg->flags |= KBASE_REG_CPU_RD; ++ ++ if (flags & BASE_MEM_PROT_GPU_WR) ++ reg->flags |= KBASE_REG_GPU_WR; ++ ++ if (flags & BASE_MEM_PROT_GPU_RD) ++ reg->flags |= KBASE_REG_GPU_RD; ++ ++ if (0 == (flags & BASE_MEM_PROT_GPU_EX)) ++ reg->flags |= KBASE_REG_GPU_NX; ++ ++ if (!kbase_device_is_cpu_coherent(kctx->kbdev)) { ++ if (flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) ++ return -EINVAL; ++ } else if (flags & (BASE_MEM_COHERENT_SYSTEM | ++ BASE_MEM_COHERENT_SYSTEM_REQUIRED)) { ++ reg->flags |= KBASE_REG_SHARE_BOTH; ++ } ++ ++ if (!(reg->flags & KBASE_REG_SHARE_BOTH) && ++ flags & BASE_MEM_COHERENT_LOCAL) { ++ reg->flags |= KBASE_REG_SHARE_IN; ++ } ++ ++ /* Set up default MEMATTR usage */ ++ if (kctx->kbdev->system_coherency == COHERENCY_ACE && ++ (reg->flags & KBASE_REG_SHARE_BOTH)) { ++ reg->flags |= ++ KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_DEFAULT_ACE); ++ } else { ++ reg->flags |= ++ KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_DEFAULT); ++ } ++ ++ return 0; ++} ++ ++int kbase_alloc_phy_pages_helper( ++ struct kbase_mem_phy_alloc *alloc, ++ size_t nr_pages_requested) ++{ ++ int new_page_count __maybe_unused; ++ size_t old_page_count = alloc->nents; ++ size_t nr_left = nr_pages_requested; ++ int res; ++ struct kbase_context *kctx; ++ struct tagged_addr *tp; ++ ++ KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_NATIVE); ++ KBASE_DEBUG_ASSERT(alloc->imported.kctx); ++ ++ kctx = alloc->imported.kctx; ++ ++ if (nr_pages_requested == 0) ++ goto done; /*nothing to do*/ ++ ++ new_page_count = kbase_atomic_add_pages( ++ nr_pages_requested, &kctx->used_pages); ++ kbase_atomic_add_pages(nr_pages_requested, ++ &kctx->kbdev->memdev.used_pages); ++ ++ /* Increase mm counters before we allocate pages so that this ++ * allocation is visible to the OOM killer */ ++ kbase_process_page_usage_inc(kctx, nr_pages_requested); ++ ++ tp = alloc->pages + old_page_count; ++ ++#ifdef CONFIG_MALI_2MB_ALLOC ++ /* Check if we have enough pages requested so we can allocate a large ++ * page (512 * 4KB = 2MB ) ++ */ ++ if (nr_left >= (SZ_2M / SZ_4K)) { ++ int nr_lp = nr_left / (SZ_2M / SZ_4K); ++ ++ res = kbase_mem_pool_alloc_pages(&kctx->lp_mem_pool, ++ nr_lp * (SZ_2M / SZ_4K), ++ tp, ++ true); ++ ++ if (res > 0) { ++ nr_left -= res; ++ tp += res; ++ } ++ ++ if (nr_left) { ++ struct kbase_sub_alloc *sa, *temp_sa; ++ ++ mutex_lock(&kctx->mem_partials_lock); ++ ++ list_for_each_entry_safe(sa, temp_sa, ++ &kctx->mem_partials, link) { ++ int pidx = 0; ++ ++ while (nr_left) { ++ pidx = find_next_zero_bit(sa->sub_pages, ++ SZ_2M / SZ_4K, ++ pidx); ++ bitmap_set(sa->sub_pages, pidx, 1); ++ *tp++ = as_tagged_tag(page_to_phys(sa->page + ++ pidx), ++ FROM_PARTIAL); ++ nr_left--; ++ ++ if (bitmap_full(sa->sub_pages, SZ_2M / SZ_4K)) { ++ /* unlink from partial list when full */ ++ list_del_init(&sa->link); ++ break; ++ } ++ } ++ } ++ mutex_unlock(&kctx->mem_partials_lock); ++ } ++ ++ /* only if we actually have a chunk left <512. If more it indicates ++ * that we couldn't allocate a 2MB above, so no point to retry here. ++ */ ++ if (nr_left > 0 && nr_left < (SZ_2M / SZ_4K)) { ++ /* create a new partial and suballocate the rest from it */ ++ struct page *np = NULL; ++ ++ do { ++ int err; ++ ++ np = kbase_mem_pool_alloc(&kctx->lp_mem_pool); ++ if (np) ++ break; ++ err = kbase_mem_pool_grow(&kctx->lp_mem_pool, 1); ++ if (err) ++ break; ++ } while (1); ++ ++ if (np) { ++ int i; ++ struct kbase_sub_alloc *sa; ++ struct page *p; ++ ++ sa = kmalloc(sizeof(*sa), GFP_KERNEL); ++ if (!sa) { ++ kbase_mem_pool_free(&kctx->lp_mem_pool, np, false); ++ goto no_new_partial; ++ } ++ ++ /* store pointers back to the control struct */ ++ np->lru.next = (void *)sa; ++ for (p = np; p < np + SZ_2M / SZ_4K; p++) ++ p->lru.prev = (void *)np; ++ INIT_LIST_HEAD(&sa->link); ++ bitmap_zero(sa->sub_pages, SZ_2M / SZ_4K); ++ sa->page = np; ++ ++ for (i = 0; i < nr_left; i++) ++ *tp++ = as_tagged_tag(page_to_phys(np + i), FROM_PARTIAL); ++ ++ bitmap_set(sa->sub_pages, 0, nr_left); ++ nr_left = 0; ++ ++ /* expose for later use */ ++ mutex_lock(&kctx->mem_partials_lock); ++ list_add(&sa->link, &kctx->mem_partials); ++ mutex_unlock(&kctx->mem_partials_lock); ++ } ++ } ++ } ++no_new_partial: ++#endif ++ ++ if (nr_left) { ++ res = kbase_mem_pool_alloc_pages(&kctx->mem_pool, ++ nr_left, ++ tp, ++ false); ++ if (res <= 0) ++ goto alloc_failed; ++ } ++ ++ /* ++ * Request a zone cache update, this scans only the new pages an ++ * appends their information to the zone cache. if the update ++ * fails then clear the cache so we fall-back to doing things ++ * page by page. ++ */ ++ if (kbase_zone_cache_update(alloc, old_page_count) != 0) ++ kbase_zone_cache_clear(alloc); ++ ++ KBASE_TLSTREAM_AUX_PAGESALLOC( ++ kctx->id, ++ (u64)new_page_count); ++ ++ alloc->nents += nr_pages_requested; ++done: ++ return 0; ++ ++alloc_failed: ++ /* rollback needed if got one or more 2MB but failed later */ ++ if (nr_left != nr_pages_requested) ++ kbase_mem_pool_free_pages(&kctx->lp_mem_pool, ++ nr_pages_requested - nr_left, ++ alloc->pages + old_page_count, ++ false, ++ false); ++ ++ kbase_process_page_usage_dec(kctx, nr_pages_requested); ++ kbase_atomic_sub_pages(nr_pages_requested, &kctx->used_pages); ++ kbase_atomic_sub_pages(nr_pages_requested, ++ &kctx->kbdev->memdev.used_pages); ++ ++ return -ENOMEM; ++} ++ ++static void free_partial(struct kbase_context *kctx, struct tagged_addr tp) ++{ ++ struct page *p, *head_page; ++ struct kbase_sub_alloc *sa; ++ ++ p = phys_to_page(as_phys_addr_t(tp)); ++ head_page = (struct page *)p->lru.prev; ++ sa = (struct kbase_sub_alloc *)head_page->lru.next; ++ mutex_lock(&kctx->mem_partials_lock); ++ clear_bit(p - head_page, sa->sub_pages); ++ if (bitmap_empty(sa->sub_pages, SZ_2M / SZ_4K)) { ++ list_del(&sa->link); ++ kbase_mem_pool_free(&kctx->lp_mem_pool, head_page, true); ++ kfree(sa); ++ } else if (bitmap_weight(sa->sub_pages, SZ_2M / SZ_4K) == ++ SZ_2M / SZ_4K - 1) { ++ /* expose the partial again */ ++ list_add(&sa->link, &kctx->mem_partials); ++ } ++ mutex_unlock(&kctx->mem_partials_lock); ++} ++ ++int kbase_free_phy_pages_helper( ++ struct kbase_mem_phy_alloc *alloc, ++ size_t nr_pages_to_free) ++{ ++ struct kbase_context *kctx = alloc->imported.kctx; ++ bool syncback; ++ bool reclaimed = (alloc->evicted != 0); ++ struct tagged_addr *start_free; ++ int new_page_count __maybe_unused; ++ size_t freed = 0; ++ ++ KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_NATIVE); ++ KBASE_DEBUG_ASSERT(alloc->imported.kctx); ++ KBASE_DEBUG_ASSERT(alloc->nents >= nr_pages_to_free); ++ ++ /* early out if nothing to do */ ++ if (0 == nr_pages_to_free) ++ return 0; ++ ++ start_free = alloc->pages + alloc->nents - nr_pages_to_free; ++ ++ syncback = alloc->properties & KBASE_MEM_PHY_ALLOC_ACCESSED_CACHED; ++ ++ /* pad start_free to a valid start location */ ++ while (nr_pages_to_free && is_huge(*start_free) && ++ !is_huge_head(*start_free)) { ++ nr_pages_to_free--; ++ start_free++; ++ } ++ ++ /* ++ * Clear the zone cache, we don't expect JIT allocations to be ++ * shrunk in parts so there is no point trying to optimize for that ++ * by scanning for the changes caused by freeing this memory and ++ * updating the existing cache entries. ++ */ ++ kbase_zone_cache_clear(alloc); ++ ++ ++ while (nr_pages_to_free) { ++ if (is_huge_head(*start_free)) { ++ /* This is a 2MB entry, so free all the 512 pages that ++ * it points to ++ */ ++ kbase_mem_pool_free_pages(&kctx->lp_mem_pool, ++ 512, ++ start_free, ++ syncback, ++ reclaimed); ++ nr_pages_to_free -= 512; ++ start_free += 512; ++ freed += 512; ++ } else if (is_partial(*start_free)) { ++ free_partial(kctx, *start_free); ++ nr_pages_to_free--; ++ start_free++; ++ freed++; ++ } else { ++ struct tagged_addr *local_end_free; ++ ++ local_end_free = start_free; ++ while (nr_pages_to_free && ++ !is_huge(*local_end_free) && ++ !is_partial(*local_end_free)) { ++ local_end_free++; ++ nr_pages_to_free--; ++ } ++ kbase_mem_pool_free_pages(&kctx->mem_pool, ++ local_end_free - start_free, ++ start_free, ++ syncback, ++ reclaimed); ++ freed += local_end_free - start_free; ++ start_free += local_end_free - start_free; ++ } ++ } ++ ++ alloc->nents -= freed; ++ ++ /* ++ * If the allocation was not evicted (i.e. evicted == 0) then ++ * the page accounting needs to be done. ++ */ ++ if (!reclaimed) { ++ kbase_process_page_usage_dec(kctx, freed); ++ new_page_count = kbase_atomic_sub_pages(freed, ++ &kctx->used_pages); ++ kbase_atomic_sub_pages(freed, ++ &kctx->kbdev->memdev.used_pages); ++ ++ KBASE_TLSTREAM_AUX_PAGESALLOC( ++ kctx->id, ++ (u64)new_page_count); ++ } ++ ++ return 0; ++} ++ ++void kbase_mem_kref_free(struct kref *kref) ++{ ++ struct kbase_mem_phy_alloc *alloc; ++ ++ alloc = container_of(kref, struct kbase_mem_phy_alloc, kref); ++ ++ switch (alloc->type) { ++ case KBASE_MEM_TYPE_NATIVE: { ++ WARN_ON(!alloc->imported.kctx); ++ /* ++ * The physical allocation must have been removed from the ++ * eviction list before trying to free it. ++ */ ++ WARN_ON(!list_empty(&alloc->evict_node)); ++ kbase_free_phy_pages_helper(alloc, alloc->nents); ++ break; ++ } ++ case KBASE_MEM_TYPE_ALIAS: { ++ /* just call put on the underlying phy allocs */ ++ size_t i; ++ struct kbase_aliased *aliased; ++ ++ aliased = alloc->imported.alias.aliased; ++ if (aliased) { ++ for (i = 0; i < alloc->imported.alias.nents; i++) ++ if (aliased[i].alloc) ++ kbase_mem_phy_alloc_put(aliased[i].alloc); ++ vfree(aliased); ++ } ++ break; ++ } ++ case KBASE_MEM_TYPE_RAW: ++ /* raw pages, external cleanup */ ++ break; ++ #ifdef CONFIG_UMP ++ case KBASE_MEM_TYPE_IMPORTED_UMP: ++ ump_dd_release(alloc->imported.ump_handle); ++ break; ++#endif ++#ifdef CONFIG_DMA_SHARED_BUFFER ++ case KBASE_MEM_TYPE_IMPORTED_UMM: ++ dma_buf_detach(alloc->imported.umm.dma_buf, ++ alloc->imported.umm.dma_attachment); ++ dma_buf_put(alloc->imported.umm.dma_buf); ++ break; ++#endif ++ case KBASE_MEM_TYPE_IMPORTED_USER_BUF: ++ if (alloc->imported.user_buf.mm) ++ mmdrop(alloc->imported.user_buf.mm); ++ kfree(alloc->imported.user_buf.pages); ++ break; ++ case KBASE_MEM_TYPE_TB:{ ++ void *tb; ++ ++ tb = alloc->imported.kctx->jctx.tb; ++ kbase_device_trace_buffer_uninstall(alloc->imported.kctx); ++ vfree(tb); ++ break; ++ } ++ default: ++ WARN(1, "Unexecpted free of type %d\n", alloc->type); ++ break; ++ } ++ ++ /* Free based on allocation type */ ++ if (alloc->properties & KBASE_MEM_PHY_ALLOC_LARGE) ++ vfree(alloc); ++ else ++ kfree(alloc); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_mem_kref_free); ++ ++int kbase_alloc_phy_pages(struct kbase_va_region *reg, size_t vsize, size_t size) ++{ ++ KBASE_DEBUG_ASSERT(NULL != reg); ++ KBASE_DEBUG_ASSERT(vsize > 0); ++ ++ /* validate user provided arguments */ ++ if (size > vsize || vsize > reg->nr_pages) ++ goto out_term; ++ ++ /* Prevent vsize*sizeof from wrapping around. ++ * For instance, if vsize is 2**29+1, we'll allocate 1 byte and the alloc won't fail. ++ */ ++ if ((size_t) vsize > ((size_t) -1 / sizeof(*reg->cpu_alloc->pages))) ++ goto out_term; ++ ++ KBASE_DEBUG_ASSERT(0 != vsize); ++ ++ if (kbase_alloc_phy_pages_helper(reg->cpu_alloc, size) != 0) ++ goto out_term; ++ ++ reg->cpu_alloc->reg = reg; ++ if (reg->cpu_alloc != reg->gpu_alloc) { ++ if (kbase_alloc_phy_pages_helper(reg->gpu_alloc, size) != 0) ++ goto out_rollback; ++ reg->gpu_alloc->reg = reg; ++ } ++ ++ return 0; ++ ++out_rollback: ++ kbase_free_phy_pages_helper(reg->cpu_alloc, size); ++out_term: ++ return -1; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_alloc_phy_pages); ++ ++bool kbase_check_alloc_flags(unsigned long flags) ++{ ++ /* Only known input flags should be set. */ ++ if (flags & ~BASE_MEM_FLAGS_INPUT_MASK) ++ return false; ++ ++ /* At least one flag should be set */ ++ if (flags == 0) ++ return false; ++ ++ /* Either the GPU or CPU must be reading from the allocated memory */ ++ if ((flags & (BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_GPU_RD)) == 0) ++ return false; ++ ++ /* Either the GPU or CPU must be writing to the allocated memory */ ++ if ((flags & (BASE_MEM_PROT_CPU_WR | BASE_MEM_PROT_GPU_WR)) == 0) ++ return false; ++ ++ /* GPU cannot be writing to GPU executable memory and cannot grow the memory on page fault. */ ++ if ((flags & BASE_MEM_PROT_GPU_EX) && (flags & (BASE_MEM_PROT_GPU_WR | BASE_MEM_GROW_ON_GPF))) ++ return false; ++ ++ /* GPU should have at least read or write access otherwise there is no ++ reason for allocating. */ ++ if ((flags & (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR)) == 0) ++ return false; ++ ++ /* BASE_MEM_IMPORT_SHARED is only valid for imported memory */ ++ if ((flags & BASE_MEM_IMPORT_SHARED) == BASE_MEM_IMPORT_SHARED) ++ return false; ++ ++ return true; ++} ++ ++bool kbase_check_import_flags(unsigned long flags) ++{ ++ /* Only known input flags should be set. */ ++ if (flags & ~BASE_MEM_FLAGS_INPUT_MASK) ++ return false; ++ ++ /* At least one flag should be set */ ++ if (flags == 0) ++ return false; ++ ++ /* Imported memory cannot be GPU executable */ ++ if (flags & BASE_MEM_PROT_GPU_EX) ++ return false; ++ ++ /* Imported memory cannot grow on page fault */ ++ if (flags & BASE_MEM_GROW_ON_GPF) ++ return false; ++ ++ /* GPU should have at least read or write access otherwise there is no ++ reason for importing. */ ++ if ((flags & (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR)) == 0) ++ return false; ++ ++ /* Secure memory cannot be read by the CPU */ ++ if ((flags & BASE_MEM_SECURE) && (flags & BASE_MEM_PROT_CPU_RD)) ++ return false; ++ ++ return true; ++} ++ ++/** ++ * @brief Acquire the per-context region list lock ++ */ ++void kbase_gpu_vm_lock(struct kbase_context *kctx) ++{ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ mutex_lock(&kctx->reg_lock); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_gpu_vm_lock); ++ ++/** ++ * @brief Release the per-context region list lock ++ */ ++void kbase_gpu_vm_unlock(struct kbase_context *kctx) ++{ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ mutex_unlock(&kctx->reg_lock); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_gpu_vm_unlock); ++ ++#ifdef CONFIG_DEBUG_FS ++struct kbase_jit_debugfs_data { ++ int (*func)(struct kbase_jit_debugfs_data *); ++ struct mutex lock; ++ struct kbase_context *kctx; ++ u64 active_value; ++ u64 pool_value; ++ u64 destroy_value; ++ char buffer[50]; ++}; ++ ++static int kbase_jit_debugfs_common_open(struct inode *inode, ++ struct file *file, int (*func)(struct kbase_jit_debugfs_data *)) ++{ ++ struct kbase_jit_debugfs_data *data; ++ ++ data = kzalloc(sizeof(*data), GFP_KERNEL); ++ if (!data) ++ return -ENOMEM; ++ ++ data->func = func; ++ mutex_init(&data->lock); ++ data->kctx = (struct kbase_context *) inode->i_private; ++ ++ file->private_data = data; ++ ++ return nonseekable_open(inode, file); ++} ++ ++static ssize_t kbase_jit_debugfs_common_read(struct file *file, ++ char __user *buf, size_t len, loff_t *ppos) ++{ ++ struct kbase_jit_debugfs_data *data; ++ size_t size; ++ int ret; ++ ++ data = (struct kbase_jit_debugfs_data *) file->private_data; ++ mutex_lock(&data->lock); ++ ++ if (*ppos) { ++ size = strnlen(data->buffer, sizeof(data->buffer)); ++ } else { ++ if (!data->func) { ++ ret = -EACCES; ++ goto out_unlock; ++ } ++ ++ if (data->func(data)) { ++ ret = -EACCES; ++ goto out_unlock; ++ } ++ ++ size = scnprintf(data->buffer, sizeof(data->buffer), ++ "%llu,%llu,%llu", data->active_value, ++ data->pool_value, data->destroy_value); ++ } ++ ++ ret = simple_read_from_buffer(buf, len, ppos, data->buffer, size); ++ ++out_unlock: ++ mutex_unlock(&data->lock); ++ return ret; ++} ++ ++static int kbase_jit_debugfs_common_release(struct inode *inode, ++ struct file *file) ++{ ++ kfree(file->private_data); ++ return 0; ++} ++ ++#define KBASE_JIT_DEBUGFS_DECLARE(__fops, __func) \ ++static int __fops ## _open(struct inode *inode, struct file *file) \ ++{ \ ++ return kbase_jit_debugfs_common_open(inode, file, __func); \ ++} \ ++static const struct file_operations __fops = { \ ++ .owner = THIS_MODULE, \ ++ .open = __fops ## _open, \ ++ .release = kbase_jit_debugfs_common_release, \ ++ .read = kbase_jit_debugfs_common_read, \ ++ .write = NULL, \ ++ .llseek = generic_file_llseek, \ ++} ++ ++static int kbase_jit_debugfs_count_get(struct kbase_jit_debugfs_data *data) ++{ ++ struct kbase_context *kctx = data->kctx; ++ struct list_head *tmp; ++ ++ mutex_lock(&kctx->jit_evict_lock); ++ list_for_each(tmp, &kctx->jit_active_head) { ++ data->active_value++; ++ } ++ ++ list_for_each(tmp, &kctx->jit_pool_head) { ++ data->pool_value++; ++ } ++ ++ list_for_each(tmp, &kctx->jit_destroy_head) { ++ data->destroy_value++; ++ } ++ mutex_unlock(&kctx->jit_evict_lock); ++ ++ return 0; ++} ++KBASE_JIT_DEBUGFS_DECLARE(kbase_jit_debugfs_count_fops, ++ kbase_jit_debugfs_count_get); ++ ++static int kbase_jit_debugfs_vm_get(struct kbase_jit_debugfs_data *data) ++{ ++ struct kbase_context *kctx = data->kctx; ++ struct kbase_va_region *reg; ++ ++ mutex_lock(&kctx->jit_evict_lock); ++ list_for_each_entry(reg, &kctx->jit_active_head, jit_node) { ++ data->active_value += reg->nr_pages; ++ } ++ ++ list_for_each_entry(reg, &kctx->jit_pool_head, jit_node) { ++ data->pool_value += reg->nr_pages; ++ } ++ ++ list_for_each_entry(reg, &kctx->jit_destroy_head, jit_node) { ++ data->destroy_value += reg->nr_pages; ++ } ++ mutex_unlock(&kctx->jit_evict_lock); ++ ++ return 0; ++} ++KBASE_JIT_DEBUGFS_DECLARE(kbase_jit_debugfs_vm_fops, ++ kbase_jit_debugfs_vm_get); ++ ++static int kbase_jit_debugfs_phys_get(struct kbase_jit_debugfs_data *data) ++{ ++ struct kbase_context *kctx = data->kctx; ++ struct kbase_va_region *reg; ++ ++ mutex_lock(&kctx->jit_evict_lock); ++ list_for_each_entry(reg, &kctx->jit_active_head, jit_node) { ++ data->active_value += reg->gpu_alloc->nents; ++ } ++ ++ list_for_each_entry(reg, &kctx->jit_pool_head, jit_node) { ++ data->pool_value += reg->gpu_alloc->nents; ++ } ++ ++ list_for_each_entry(reg, &kctx->jit_destroy_head, jit_node) { ++ data->destroy_value += reg->gpu_alloc->nents; ++ } ++ mutex_unlock(&kctx->jit_evict_lock); ++ ++ return 0; ++} ++KBASE_JIT_DEBUGFS_DECLARE(kbase_jit_debugfs_phys_fops, ++ kbase_jit_debugfs_phys_get); ++ ++void kbase_jit_debugfs_init(struct kbase_context *kctx) ++{ ++ /* Debugfs entry for getting the number of JIT allocations. */ ++ debugfs_create_file("mem_jit_count", S_IRUGO, kctx->kctx_dentry, ++ kctx, &kbase_jit_debugfs_count_fops); ++ ++ /* ++ * Debugfs entry for getting the total number of virtual pages ++ * used by JIT allocations. ++ */ ++ debugfs_create_file("mem_jit_vm", S_IRUGO, kctx->kctx_dentry, ++ kctx, &kbase_jit_debugfs_vm_fops); ++ ++ /* ++ * Debugfs entry for getting the number of physical pages used ++ * by JIT allocations. ++ */ ++ debugfs_create_file("mem_jit_phys", S_IRUGO, kctx->kctx_dentry, ++ kctx, &kbase_jit_debugfs_phys_fops); ++} ++#endif /* CONFIG_DEBUG_FS */ ++ ++/** ++ * kbase_jit_destroy_worker - Deferred worker which frees JIT allocations ++ * @work: Work item ++ * ++ * This function does the work of freeing JIT allocations whose physical ++ * backing has been released. ++ */ ++static void kbase_jit_destroy_worker(struct work_struct *work) ++{ ++ struct kbase_context *kctx; ++ struct kbase_va_region *reg; ++ ++ kctx = container_of(work, struct kbase_context, jit_work); ++ do { ++ mutex_lock(&kctx->jit_evict_lock); ++ if (list_empty(&kctx->jit_destroy_head)) { ++ mutex_unlock(&kctx->jit_evict_lock); ++ break; ++ } ++ ++ reg = list_first_entry(&kctx->jit_destroy_head, ++ struct kbase_va_region, jit_node); ++ ++ list_del(®->jit_node); ++ mutex_unlock(&kctx->jit_evict_lock); ++ ++ kbase_gpu_vm_lock(kctx); ++ kbase_mem_free_region(kctx, reg); ++ kbase_gpu_vm_unlock(kctx); ++ } while (1); ++} ++ ++int kbase_jit_init(struct kbase_context *kctx) ++{ ++ INIT_LIST_HEAD(&kctx->jit_active_head); ++ INIT_LIST_HEAD(&kctx->jit_pool_head); ++ INIT_LIST_HEAD(&kctx->jit_destroy_head); ++ INIT_WORK(&kctx->jit_work, kbase_jit_destroy_worker); ++ ++ INIT_LIST_HEAD(&kctx->jit_pending_alloc); ++ INIT_LIST_HEAD(&kctx->jit_atoms_head); ++ ++ return 0; ++} ++ ++struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx, ++ struct base_jit_alloc_info *info) ++{ ++ struct kbase_va_region *reg = NULL; ++ struct kbase_va_region *walker; ++ struct kbase_va_region *temp; ++ size_t current_diff = SIZE_MAX; ++ ++ int ret; ++ ++ mutex_lock(&kctx->jit_evict_lock); ++ /* ++ * Scan the pool for an existing allocation which meets our ++ * requirements and remove it. ++ */ ++ list_for_each_entry_safe(walker, temp, &kctx->jit_pool_head, jit_node) { ++ ++ if (walker->nr_pages >= info->va_pages) { ++ size_t min_size, max_size, diff; ++ ++ /* ++ * The JIT allocations VA requirements have been ++ * meet, it's suitable but other allocations ++ * might be a better fit. ++ */ ++ min_size = min_t(size_t, walker->gpu_alloc->nents, ++ info->commit_pages); ++ max_size = max_t(size_t, walker->gpu_alloc->nents, ++ info->commit_pages); ++ diff = max_size - min_size; ++ ++ if (current_diff > diff) { ++ current_diff = diff; ++ reg = walker; ++ } ++ ++ /* The allocation is an exact match, stop looking */ ++ if (current_diff == 0) ++ break; ++ } ++ } ++ ++ if (reg) { ++ /* ++ * Remove the found region from the pool and add it to the ++ * active list. ++ */ ++ list_move(®->jit_node, &kctx->jit_active_head); ++ ++ /* ++ * Remove the allocation from the eviction list as it's no ++ * longer eligible for eviction. This must be done before ++ * dropping the jit_evict_lock ++ */ ++ list_del_init(®->gpu_alloc->evict_node); ++ mutex_unlock(&kctx->jit_evict_lock); ++ ++ kbase_gpu_vm_lock(kctx); ++ ++ /* Make the physical backing no longer reclaimable */ ++ if (!kbase_mem_evictable_unmake(reg->gpu_alloc)) ++ goto update_failed; ++ ++ /* Grow the backing if required */ ++ if (reg->gpu_alloc->nents < info->commit_pages) { ++ size_t delta; ++ size_t old_size = reg->gpu_alloc->nents; ++ ++ /* Allocate some more pages */ ++ delta = info->commit_pages - reg->gpu_alloc->nents; ++ if (kbase_alloc_phy_pages_helper(reg->gpu_alloc, delta) ++ != 0) ++ goto update_failed; ++ ++ if (reg->cpu_alloc != reg->gpu_alloc) { ++ if (kbase_alloc_phy_pages_helper( ++ reg->cpu_alloc, delta) != 0) { ++ kbase_free_phy_pages_helper( ++ reg->gpu_alloc, delta); ++ goto update_failed; ++ } ++ } ++ ++ ret = kbase_mem_grow_gpu_mapping(kctx, reg, ++ info->commit_pages, old_size); ++ /* ++ * The grow failed so put the allocation back in the ++ * pool and return failure. ++ */ ++ if (ret) ++ goto update_failed; ++ } ++ kbase_gpu_vm_unlock(kctx); ++ } else { ++ /* No suitable JIT allocation was found so create a new one */ ++ u64 flags = BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_GPU_RD | ++ BASE_MEM_PROT_GPU_WR | BASE_MEM_GROW_ON_GPF | ++ BASE_MEM_COHERENT_LOCAL; ++ u64 gpu_addr; ++ ++ mutex_unlock(&kctx->jit_evict_lock); ++ ++ reg = kbase_mem_alloc(kctx, info->va_pages, info->commit_pages, ++ info->extent, &flags, &gpu_addr); ++ if (!reg) ++ goto out_unlocked; ++ ++ mutex_lock(&kctx->jit_evict_lock); ++ list_add(®->jit_node, &kctx->jit_active_head); ++ mutex_unlock(&kctx->jit_evict_lock); ++ } ++ ++ return reg; ++ ++update_failed: ++ /* ++ * An update to an allocation from the pool failed, chances ++ * are slim a new allocation would fair any better so return ++ * the allocation to the pool and return the function with failure. ++ */ ++ kbase_gpu_vm_unlock(kctx); ++ mutex_lock(&kctx->jit_evict_lock); ++ list_move(®->jit_node, &kctx->jit_pool_head); ++ mutex_unlock(&kctx->jit_evict_lock); ++out_unlocked: ++ return NULL; ++} ++ ++void kbase_jit_free(struct kbase_context *kctx, struct kbase_va_region *reg) ++{ ++ /* The physical backing of memory in the pool is always reclaimable */ ++ kbase_gpu_vm_lock(kctx); ++ kbase_mem_evictable_make(reg->gpu_alloc); ++ kbase_gpu_vm_unlock(kctx); ++ ++ mutex_lock(&kctx->jit_evict_lock); ++ list_move(®->jit_node, &kctx->jit_pool_head); ++ mutex_unlock(&kctx->jit_evict_lock); ++} ++ ++void kbase_jit_backing_lost(struct kbase_va_region *reg) ++{ ++ struct kbase_context *kctx = reg->kctx; ++ ++ lockdep_assert_held(&kctx->jit_evict_lock); ++ ++ /* ++ * JIT allocations will always be on a list, if the region ++ * is not on a list then it's not a JIT allocation. ++ */ ++ if (list_empty(®->jit_node)) ++ return; ++ ++ /* ++ * Freeing the allocation requires locks we might not be able ++ * to take now, so move the allocation to the free list and kick ++ * the worker which will do the freeing. ++ */ ++ list_move(®->jit_node, &kctx->jit_destroy_head); ++ ++ schedule_work(&kctx->jit_work); ++} ++ ++bool kbase_jit_evict(struct kbase_context *kctx) ++{ ++ struct kbase_va_region *reg = NULL; ++ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ /* Free the oldest allocation from the pool */ ++ mutex_lock(&kctx->jit_evict_lock); ++ if (!list_empty(&kctx->jit_pool_head)) { ++ reg = list_entry(kctx->jit_pool_head.prev, ++ struct kbase_va_region, jit_node); ++ list_del(®->jit_node); ++ } ++ mutex_unlock(&kctx->jit_evict_lock); ++ ++ if (reg) ++ kbase_mem_free_region(kctx, reg); ++ ++ return (reg != NULL); ++} ++ ++void kbase_jit_term(struct kbase_context *kctx) ++{ ++ struct kbase_va_region *walker; ++ ++ /* Free all allocations for this context */ ++ ++ /* ++ * Flush the freeing of allocations whose backing has been freed ++ * (i.e. everything in jit_destroy_head). ++ */ ++ cancel_work_sync(&kctx->jit_work); ++ ++ kbase_gpu_vm_lock(kctx); ++ mutex_lock(&kctx->jit_evict_lock); ++ /* Free all allocations from the pool */ ++ while (!list_empty(&kctx->jit_pool_head)) { ++ walker = list_first_entry(&kctx->jit_pool_head, ++ struct kbase_va_region, jit_node); ++ list_del(&walker->jit_node); ++ mutex_unlock(&kctx->jit_evict_lock); ++ kbase_mem_free_region(kctx, walker); ++ mutex_lock(&kctx->jit_evict_lock); ++ } ++ ++ /* Free all allocations from active list */ ++ while (!list_empty(&kctx->jit_active_head)) { ++ walker = list_first_entry(&kctx->jit_active_head, ++ struct kbase_va_region, jit_node); ++ list_del(&walker->jit_node); ++ mutex_unlock(&kctx->jit_evict_lock); ++ kbase_mem_free_region(kctx, walker); ++ mutex_lock(&kctx->jit_evict_lock); ++ } ++ mutex_unlock(&kctx->jit_evict_lock); ++ kbase_gpu_vm_unlock(kctx); ++} ++ ++static int kbase_jd_user_buf_map(struct kbase_context *kctx, ++ struct kbase_va_region *reg) ++{ ++ long pinned_pages; ++ struct kbase_mem_phy_alloc *alloc; ++ struct page **pages; ++ struct tagged_addr *pa; ++ long i; ++ int err = -ENOMEM; ++ unsigned long address; ++ struct mm_struct *mm; ++ struct device *dev; ++ unsigned long offset; ++ unsigned long local_size; ++ ++ alloc = reg->gpu_alloc; ++ pa = kbase_get_gpu_phy_pages(reg); ++ address = alloc->imported.user_buf.address; ++ mm = alloc->imported.user_buf.mm; ++ ++ KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_IMPORTED_USER_BUF); ++ ++ pages = alloc->imported.user_buf.pages; ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) ++ pinned_pages = get_user_pages(NULL, mm, ++ address, ++ alloc->imported.user_buf.nr_pages, ++ reg->flags & KBASE_REG_GPU_WR, ++ 0, pages, NULL); ++#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) ++ pinned_pages = get_user_pages_remote(NULL, mm, ++ address, ++ alloc->imported.user_buf.nr_pages, ++ reg->flags & KBASE_REG_GPU_WR, ++ 0, pages, NULL); ++#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) ++ pinned_pages = get_user_pages_remote(NULL, mm, ++ address, ++ alloc->imported.user_buf.nr_pages, ++ reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0, ++ pages, NULL); ++#else ++ pinned_pages = get_user_pages_remote(NULL, mm, ++ address, ++ alloc->imported.user_buf.nr_pages, ++ reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0, ++ pages, NULL, NULL); ++#endif ++ ++ if (pinned_pages <= 0) ++ return pinned_pages; ++ ++ if (pinned_pages != alloc->imported.user_buf.nr_pages) { ++ for (i = 0; i < pinned_pages; i++) ++ put_page(pages[i]); ++ return -ENOMEM; ++ } ++ ++ dev = kctx->kbdev->dev; ++ offset = address & ~PAGE_MASK; ++ local_size = alloc->imported.user_buf.size; ++ ++ for (i = 0; i < pinned_pages; i++) { ++ dma_addr_t dma_addr; ++ unsigned long min; ++ ++ min = MIN(PAGE_SIZE - offset, local_size); ++ dma_addr = dma_map_page(dev, pages[i], ++ offset, min, ++ DMA_BIDIRECTIONAL); ++ if (dma_mapping_error(dev, dma_addr)) ++ goto unwind; ++ ++ alloc->imported.user_buf.dma_addrs[i] = dma_addr; ++ pa[i] = as_tagged(page_to_phys(pages[i])); ++ ++ local_size -= min; ++ offset = 0; ++ } ++ ++ alloc->nents = pinned_pages; ++ ++ err = kbase_mmu_insert_pages(kctx, reg->start_pfn, pa, ++ kbase_reg_current_backed_size(reg), ++ reg->flags); ++ if (err == 0) ++ return 0; ++ ++ alloc->nents = 0; ++ /* fall down */ ++unwind: ++ while (i--) { ++ dma_unmap_page(kctx->kbdev->dev, ++ alloc->imported.user_buf.dma_addrs[i], ++ PAGE_SIZE, DMA_BIDIRECTIONAL); ++ } ++ ++ while (++i < pinned_pages) { ++ put_page(pages[i]); ++ pages[i] = NULL; ++ } ++ ++ return err; ++} ++ ++static void kbase_jd_user_buf_unmap(struct kbase_context *kctx, ++ struct kbase_mem_phy_alloc *alloc, bool writeable) ++{ ++ long i; ++ struct page **pages; ++ unsigned long size = alloc->imported.user_buf.size; ++ ++ KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_IMPORTED_USER_BUF); ++ pages = alloc->imported.user_buf.pages; ++ for (i = 0; i < alloc->imported.user_buf.nr_pages; i++) { ++ unsigned long local_size; ++ dma_addr_t dma_addr = alloc->imported.user_buf.dma_addrs[i]; ++ ++ local_size = MIN(size, PAGE_SIZE - (dma_addr & ~PAGE_MASK)); ++ dma_unmap_page(kctx->kbdev->dev, dma_addr, local_size, ++ DMA_BIDIRECTIONAL); ++ if (writeable) ++ set_page_dirty_lock(pages[i]); ++ put_page(pages[i]); ++ pages[i] = NULL; ++ ++ size -= local_size; ++ } ++ alloc->nents = 0; ++} ++ ++/* to replace sg_dma_len. */ ++#define MALI_SG_DMA_LEN(sg) ((sg)->length) ++ ++#ifdef CONFIG_DMA_SHARED_BUFFER ++static int kbase_jd_umm_map(struct kbase_context *kctx, ++ struct kbase_va_region *reg) ++{ ++ struct sg_table *sgt; ++ struct scatterlist *s; ++ int i; ++ struct tagged_addr *pa; ++ int err; ++ size_t count = 0; ++ struct kbase_mem_phy_alloc *alloc; ++ ++ alloc = reg->gpu_alloc; ++ ++ KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_IMPORTED_UMM); ++ KBASE_DEBUG_ASSERT(NULL == alloc->imported.umm.sgt); ++ sgt = dma_buf_map_attachment(alloc->imported.umm.dma_attachment, ++ DMA_BIDIRECTIONAL); ++ ++ if (IS_ERR_OR_NULL(sgt)) ++ return -EINVAL; ++ ++ /* save for later */ ++ alloc->imported.umm.sgt = sgt; ++ ++ pa = kbase_get_gpu_phy_pages(reg); ++ KBASE_DEBUG_ASSERT(pa); ++ ++ for_each_sg(sgt->sgl, s, sgt->nents, i) { ++ int j; ++ size_t pages = PFN_UP(MALI_SG_DMA_LEN(s)); ++ ++ WARN_ONCE(MALI_SG_DMA_LEN(s) & (PAGE_SIZE-1), ++ "MALI_SG_DMA_LEN(s)=%u is not a multiple of PAGE_SIZE\n", ++ MALI_SG_DMA_LEN(s)); ++ ++ WARN_ONCE(sg_dma_address(s) & (PAGE_SIZE-1), ++ "sg_dma_address(s)=%llx is not aligned to PAGE_SIZE\n", ++ (unsigned long long) sg_dma_address(s)); ++ ++ for (j = 0; (j < pages) && (count < reg->nr_pages); j++, ++ count++) ++ *pa++ = as_tagged(sg_dma_address(s) + ++ (j << PAGE_SHIFT)); ++ WARN_ONCE(j < pages, ++ "sg list from dma_buf_map_attachment > dma_buf->size=%zu\n", ++ alloc->imported.umm.dma_buf->size); ++ } ++ ++ if (!(reg->flags & KBASE_REG_IMPORT_PAD) && ++ WARN_ONCE(count < reg->nr_pages, ++ "sg list from dma_buf_map_attachment < dma_buf->size=%zu\n", ++ alloc->imported.umm.dma_buf->size)) { ++ err = -EINVAL; ++ goto err_unmap_attachment; ++ } ++ ++ /* Update nents as we now have pages to map */ ++ alloc->nents = reg->nr_pages; ++ ++ err = kbase_mmu_insert_pages(kctx, reg->start_pfn, ++ kbase_get_gpu_phy_pages(reg), ++ count, ++ reg->flags | KBASE_REG_GPU_WR | KBASE_REG_GPU_RD); ++ if (err) ++ goto err_unmap_attachment; ++ ++ if (reg->flags & KBASE_REG_IMPORT_PAD) { ++ err = kbase_mmu_insert_single_page(kctx, ++ reg->start_pfn + count, ++ kctx->aliasing_sink_page, ++ reg->nr_pages - count, ++ (reg->flags | KBASE_REG_GPU_RD) & ++ ~KBASE_REG_GPU_WR); ++ if (err) ++ goto err_teardown_orig_pages; ++ } ++ ++ return 0; ++ ++err_teardown_orig_pages: ++ kbase_mmu_teardown_pages(kctx, reg->start_pfn, count); ++err_unmap_attachment: ++ dma_buf_unmap_attachment(alloc->imported.umm.dma_attachment, ++ alloc->imported.umm.sgt, DMA_BIDIRECTIONAL); ++ alloc->imported.umm.sgt = NULL; ++ ++ return err; ++} ++ ++static void kbase_jd_umm_unmap(struct kbase_context *kctx, ++ struct kbase_mem_phy_alloc *alloc) ++{ ++ KBASE_DEBUG_ASSERT(kctx); ++ KBASE_DEBUG_ASSERT(alloc); ++ KBASE_DEBUG_ASSERT(alloc->imported.umm.dma_attachment); ++ KBASE_DEBUG_ASSERT(alloc->imported.umm.sgt); ++ dma_buf_unmap_attachment(alloc->imported.umm.dma_attachment, ++ alloc->imported.umm.sgt, DMA_BIDIRECTIONAL); ++ alloc->imported.umm.sgt = NULL; ++ alloc->nents = 0; ++} ++#endif /* CONFIG_DMA_SHARED_BUFFER */ ++ ++#if (defined(CONFIG_KDS) && defined(CONFIG_UMP)) \ ++ || defined(CONFIG_DMA_SHARED_BUFFER_USES_KDS) ++static void add_kds_resource(struct kds_resource *kds_res, ++ struct kds_resource **kds_resources, u32 *kds_res_count, ++ unsigned long *kds_access_bitmap, bool exclusive) ++{ ++ u32 i; ++ ++ for (i = 0; i < *kds_res_count; i++) { ++ /* Duplicate resource, ignore */ ++ if (kds_resources[i] == kds_res) ++ return; ++ } ++ ++ kds_resources[*kds_res_count] = kds_res; ++ if (exclusive) ++ set_bit(*kds_res_count, kds_access_bitmap); ++ (*kds_res_count)++; ++} ++#endif ++ ++struct kbase_mem_phy_alloc *kbase_map_external_resource( ++ struct kbase_context *kctx, struct kbase_va_region *reg, ++ struct mm_struct *locked_mm ++#ifdef CONFIG_KDS ++ , u32 *kds_res_count, struct kds_resource **kds_resources, ++ unsigned long *kds_access_bitmap, bool exclusive ++#endif ++ ) ++{ ++ int err; ++ ++ /* decide what needs to happen for this resource */ ++ switch (reg->gpu_alloc->type) { ++ case KBASE_MEM_TYPE_IMPORTED_USER_BUF: { ++ if (reg->gpu_alloc->imported.user_buf.mm != locked_mm) ++ goto exit; ++ ++ reg->gpu_alloc->imported.user_buf.current_mapping_usage_count++; ++ if (1 == reg->gpu_alloc->imported.user_buf.current_mapping_usage_count) { ++ err = kbase_jd_user_buf_map(kctx, reg); ++ if (err) { ++ reg->gpu_alloc->imported.user_buf.current_mapping_usage_count--; ++ goto exit; ++ } ++ } ++ } ++ break; ++ case KBASE_MEM_TYPE_IMPORTED_UMP: { ++#if defined(CONFIG_KDS) && defined(CONFIG_UMP) ++ if (kds_res_count) { ++ struct kds_resource *kds_res; ++ ++ kds_res = ump_dd_kds_resource_get( ++ reg->gpu_alloc->imported.ump_handle); ++ if (kds_res) ++ add_kds_resource(kds_res, kds_resources, ++ kds_res_count, ++ kds_access_bitmap, exclusive); ++ } ++#endif /*defined(CONFIG_KDS) && defined(CONFIG_UMP) */ ++ break; ++ } ++#ifdef CONFIG_DMA_SHARED_BUFFER ++ case KBASE_MEM_TYPE_IMPORTED_UMM: { ++#ifdef CONFIG_DMA_SHARED_BUFFER_USES_KDS ++ if (kds_res_count) { ++ struct kds_resource *kds_res; ++ ++ kds_res = get_dma_buf_kds_resource( ++ reg->gpu_alloc->imported.umm.dma_buf); ++ if (kds_res) ++ add_kds_resource(kds_res, kds_resources, ++ kds_res_count, ++ kds_access_bitmap, exclusive); ++ } ++#endif ++ reg->gpu_alloc->imported.umm.current_mapping_usage_count++; ++ if (1 == reg->gpu_alloc->imported.umm.current_mapping_usage_count) { ++ err = kbase_jd_umm_map(kctx, reg); ++ if (err) { ++ reg->gpu_alloc->imported.umm.current_mapping_usage_count--; ++ goto exit; ++ } ++ } ++ break; ++ } ++#endif ++ default: ++ goto exit; ++ } ++ ++ return kbase_mem_phy_alloc_get(reg->gpu_alloc); ++exit: ++ return NULL; ++} ++ ++void kbase_unmap_external_resource(struct kbase_context *kctx, ++ struct kbase_va_region *reg, struct kbase_mem_phy_alloc *alloc) ++{ ++ switch (alloc->type) { ++#ifdef CONFIG_DMA_SHARED_BUFFER ++ case KBASE_MEM_TYPE_IMPORTED_UMM: { ++ alloc->imported.umm.current_mapping_usage_count--; ++ ++ if (0 == alloc->imported.umm.current_mapping_usage_count) { ++ if (reg && reg->gpu_alloc == alloc) { ++ int err; ++ ++ err = kbase_mmu_teardown_pages( ++ kctx, ++ reg->start_pfn, ++ alloc->nents); ++ WARN_ON(err); ++ } ++ ++ kbase_jd_umm_unmap(kctx, alloc); ++ } ++ } ++ break; ++#endif /* CONFIG_DMA_SHARED_BUFFER */ ++ case KBASE_MEM_TYPE_IMPORTED_USER_BUF: { ++ alloc->imported.user_buf.current_mapping_usage_count--; ++ ++ if (0 == alloc->imported.user_buf.current_mapping_usage_count) { ++ bool writeable = true; ++ ++ if (reg && reg->gpu_alloc == alloc) ++ kbase_mmu_teardown_pages( ++ kctx, ++ reg->start_pfn, ++ kbase_reg_current_backed_size(reg)); ++ ++ if (reg && ((reg->flags & KBASE_REG_GPU_WR) == 0)) ++ writeable = false; ++ ++ kbase_jd_user_buf_unmap(kctx, alloc, writeable); ++ } ++ } ++ break; ++ default: ++ break; ++ } ++ kbase_mem_phy_alloc_put(alloc); ++} ++ ++struct kbase_ctx_ext_res_meta *kbase_sticky_resource_acquire( ++ struct kbase_context *kctx, u64 gpu_addr) ++{ ++ struct kbase_ctx_ext_res_meta *meta = NULL; ++ struct kbase_ctx_ext_res_meta *walker; ++ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ /* ++ * Walk the per context external resource metadata list for the ++ * metadata which matches the region which is being acquired. ++ */ ++ list_for_each_entry(walker, &kctx->ext_res_meta_head, ext_res_node) { ++ if (walker->gpu_addr == gpu_addr) { ++ meta = walker; ++ break; ++ } ++ } ++ ++ /* No metadata exists so create one. */ ++ if (!meta) { ++ struct kbase_va_region *reg; ++ ++ /* Find the region */ ++ reg = kbase_region_tracker_find_region_enclosing_address( ++ kctx, gpu_addr); ++ if (NULL == reg || (reg->flags & KBASE_REG_FREE)) ++ goto failed; ++ ++ /* Allocate the metadata object */ ++ meta = kzalloc(sizeof(*meta), GFP_KERNEL); ++ if (!meta) ++ goto failed; ++ ++ /* ++ * Fill in the metadata object and acquire a reference ++ * for the physical resource. ++ */ ++ meta->alloc = kbase_map_external_resource(kctx, reg, NULL ++#ifdef CONFIG_KDS ++ , NULL, NULL, ++ NULL, false ++#endif ++ ); ++ ++ if (!meta->alloc) ++ goto fail_map; ++ ++ meta->gpu_addr = reg->start_pfn << PAGE_SHIFT; ++ ++ list_add(&meta->ext_res_node, &kctx->ext_res_meta_head); ++ } ++ ++ return meta; ++ ++fail_map: ++ kfree(meta); ++failed: ++ return NULL; ++} ++ ++bool kbase_sticky_resource_release(struct kbase_context *kctx, ++ struct kbase_ctx_ext_res_meta *meta, u64 gpu_addr) ++{ ++ struct kbase_ctx_ext_res_meta *walker; ++ struct kbase_va_region *reg; ++ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ /* Search of the metadata if one isn't provided. */ ++ if (!meta) { ++ /* ++ * Walk the per context external resource metadata list for the ++ * metadata which matches the region which is being released. ++ */ ++ list_for_each_entry(walker, &kctx->ext_res_meta_head, ++ ext_res_node) { ++ if (walker->gpu_addr == gpu_addr) { ++ meta = walker; ++ break; ++ } ++ } ++ } ++ ++ /* No metadata so just return. */ ++ if (!meta) ++ return false; ++ ++ /* Drop the physical memory reference and free the metadata. */ ++ reg = kbase_region_tracker_find_region_enclosing_address( ++ kctx, ++ meta->gpu_addr); ++ ++ kbase_unmap_external_resource(kctx, reg, meta->alloc); ++ list_del(&meta->ext_res_node); ++ kfree(meta); ++ ++ return true; ++} ++ ++int kbase_sticky_resource_init(struct kbase_context *kctx) ++{ ++ INIT_LIST_HEAD(&kctx->ext_res_meta_head); ++ ++ return 0; ++} ++ ++void kbase_sticky_resource_term(struct kbase_context *kctx) ++{ ++ struct kbase_ctx_ext_res_meta *walker; ++ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ /* ++ * Free any sticky resources which haven't been unmapped. ++ * ++ * Note: ++ * We don't care about refcounts at this point as no future ++ * references to the meta data will be made. ++ * Region termination would find these if we didn't free them ++ * here, but it's more efficient if we do the clean up here. ++ */ ++ while (!list_empty(&kctx->ext_res_meta_head)) { ++ walker = list_first_entry(&kctx->ext_res_meta_head, ++ struct kbase_ctx_ext_res_meta, ext_res_node); ++ ++ kbase_sticky_resource_release(kctx, walker, 0); ++ } ++} +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem.h +new file mode 100755 +index 000000000000..f2fd75e2018b +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem.h +@@ -0,0 +1,1142 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/** ++ * @file mali_kbase_mem.h ++ * Base kernel memory APIs ++ */ ++ ++#ifndef _KBASE_MEM_H_ ++#define _KBASE_MEM_H_ ++ ++#ifndef _KBASE_H_ ++#error "Don't include this file directly, use mali_kbase.h instead" ++#endif ++ ++#include ++#ifdef CONFIG_KDS ++#include ++#endif /* CONFIG_KDS */ ++#ifdef CONFIG_UMP ++#include ++#endif /* CONFIG_UMP */ ++#include "mali_base_kernel.h" ++#include ++#include "mali_kbase_pm.h" ++#include "mali_kbase_defs.h" ++#if defined(CONFIG_MALI_BIFROST_GATOR_SUPPORT) ++#include "mali_kbase_gator.h" ++#endif ++/* Required for kbase_mem_evictable_unmake */ ++#include "mali_kbase_mem_linux.h" ++ ++/* Part of the workaround for uTLB invalid pages is to ensure we grow/shrink tmem by 4 pages at a time */ ++#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_8316 (2) /* round to 4 pages */ ++ ++/* Part of the workaround for PRLAM-9630 requires us to grow/shrink memory by 8 pages. ++The MMU reads in 8 page table entries from memory at a time, if we have more than one page fault within the same 8 pages and ++page tables are updated accordingly, the MMU does not re-read the page table entries from memory for the subsequent page table ++updates and generates duplicate page faults as the page table information used by the MMU is not valid. */ ++#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_9630 (3) /* round to 8 pages */ ++ ++#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2 (0) /* round to 1 page */ ++ ++/* This must always be a power of 2 */ ++#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES (1u << KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2) ++#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_HW_ISSUE_8316 (1u << KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_8316) ++#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_HW_ISSUE_9630 (1u << KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_9630) ++/** ++ * A CPU mapping ++ */ ++struct kbase_cpu_mapping { ++ struct list_head mappings_list; ++ struct kbase_mem_phy_alloc *alloc; ++ struct kbase_context *kctx; ++ struct kbase_va_region *region; ++ int count; ++ int free_on_close; ++}; ++ ++enum kbase_memory_type { ++ KBASE_MEM_TYPE_NATIVE, ++ KBASE_MEM_TYPE_IMPORTED_UMP, ++ KBASE_MEM_TYPE_IMPORTED_UMM, ++ KBASE_MEM_TYPE_IMPORTED_USER_BUF, ++ KBASE_MEM_TYPE_ALIAS, ++ KBASE_MEM_TYPE_TB, ++ KBASE_MEM_TYPE_RAW ++}; ++ ++/* internal structure, mirroring base_mem_aliasing_info, ++ * but with alloc instead of a gpu va (handle) */ ++struct kbase_aliased { ++ struct kbase_mem_phy_alloc *alloc; /* NULL for special, non-NULL for native */ ++ u64 offset; /* in pages */ ++ u64 length; /* in pages */ ++}; ++ ++/** ++ * @brief Physical pages tracking object properties ++ */ ++#define KBASE_MEM_PHY_ALLOC_ACCESSED_CACHED (1ul << 0) ++#define KBASE_MEM_PHY_ALLOC_LARGE (1ul << 1) ++ ++/* physical pages tracking object. ++ * Set up to track N pages. ++ * N not stored here, the creator holds that info. ++ * This object only tracks how many elements are actually valid (present). ++ * Changing of nents or *pages should only happen if the kbase_mem_phy_alloc is not ++ * shared with another region or client. CPU mappings are OK to exist when changing, as ++ * long as the tracked mappings objects are updated as part of the change. ++ */ ++struct kbase_mem_phy_alloc { ++ struct kref kref; /* number of users of this alloc */ ++ atomic_t gpu_mappings; ++ size_t nents; /* 0..N */ ++ struct tagged_addr *pages; /* N elements, only 0..nents are valid */ ++ ++ /* kbase_cpu_mappings */ ++ struct list_head mappings; ++ ++ /* Node used to store this allocation on the eviction list */ ++ struct list_head evict_node; ++ /* Physical backing size when the pages where evicted */ ++ size_t evicted; ++ /* ++ * Back reference to the region structure which created this ++ * allocation, or NULL if it has been freed. ++ */ ++ struct kbase_va_region *reg; ++ ++ /* type of buffer */ ++ enum kbase_memory_type type; ++ ++ unsigned long properties; ++ ++ struct list_head zone_cache; ++ ++ /* member in union valid based on @a type */ ++ union { ++#ifdef CONFIG_UMP ++ ump_dd_handle ump_handle; ++#endif /* CONFIG_UMP */ ++#if defined(CONFIG_DMA_SHARED_BUFFER) ++ struct { ++ struct dma_buf *dma_buf; ++ struct dma_buf_attachment *dma_attachment; ++ unsigned int current_mapping_usage_count; ++ struct sg_table *sgt; ++ } umm; ++#endif /* defined(CONFIG_DMA_SHARED_BUFFER) */ ++ struct { ++ u64 stride; ++ size_t nents; ++ struct kbase_aliased *aliased; ++ } alias; ++ /* Used by type = (KBASE_MEM_TYPE_NATIVE, KBASE_MEM_TYPE_TB) */ ++ struct kbase_context *kctx; ++ struct kbase_alloc_import_user_buf { ++ unsigned long address; ++ unsigned long size; ++ unsigned long nr_pages; ++ struct page **pages; ++ /* top bit (1<<31) of current_mapping_usage_count ++ * specifies that this import was pinned on import ++ * See PINNED_ON_IMPORT ++ */ ++ u32 current_mapping_usage_count; ++ struct mm_struct *mm; ++ dma_addr_t *dma_addrs; ++ } user_buf; ++ } imported; ++}; ++ ++/* The top bit of kbase_alloc_import_user_buf::current_mapping_usage_count is ++ * used to signify that a buffer was pinned when it was imported. Since the ++ * reference count is limited by the number of atoms that can be submitted at ++ * once there should be no danger of overflowing into this bit. ++ * Stealing the top bit also has the benefit that ++ * current_mapping_usage_count != 0 if and only if the buffer is mapped. ++ */ ++#define PINNED_ON_IMPORT (1<<31) ++ ++static inline void kbase_mem_phy_alloc_gpu_mapped(struct kbase_mem_phy_alloc *alloc) ++{ ++ KBASE_DEBUG_ASSERT(alloc); ++ /* we only track mappings of NATIVE buffers */ ++ if (alloc->type == KBASE_MEM_TYPE_NATIVE) ++ atomic_inc(&alloc->gpu_mappings); ++} ++ ++static inline void kbase_mem_phy_alloc_gpu_unmapped(struct kbase_mem_phy_alloc *alloc) ++{ ++ KBASE_DEBUG_ASSERT(alloc); ++ /* we only track mappings of NATIVE buffers */ ++ if (alloc->type == KBASE_MEM_TYPE_NATIVE) ++ if (0 > atomic_dec_return(&alloc->gpu_mappings)) { ++ pr_err("Mismatched %s:\n", __func__); ++ dump_stack(); ++ } ++} ++ ++/** ++ * kbase_mem_is_imported - Indicate whether a memory type is imported ++ * ++ * @type: the memory type ++ * ++ * Return: true if the memory type is imported, false otherwise ++ */ ++static inline bool kbase_mem_is_imported(enum kbase_memory_type type) ++{ ++ return (type == KBASE_MEM_TYPE_IMPORTED_UMP) || ++ (type == KBASE_MEM_TYPE_IMPORTED_UMM) || ++ (type == KBASE_MEM_TYPE_IMPORTED_USER_BUF); ++} ++ ++void kbase_mem_kref_free(struct kref *kref); ++ ++int kbase_mem_init(struct kbase_device *kbdev); ++void kbase_mem_halt(struct kbase_device *kbdev); ++void kbase_mem_term(struct kbase_device *kbdev); ++ ++static inline struct kbase_mem_phy_alloc *kbase_mem_phy_alloc_get(struct kbase_mem_phy_alloc *alloc) ++{ ++ kref_get(&alloc->kref); ++ return alloc; ++} ++ ++static inline struct kbase_mem_phy_alloc *kbase_mem_phy_alloc_put(struct kbase_mem_phy_alloc *alloc) ++{ ++ kref_put(&alloc->kref, kbase_mem_kref_free); ++ return NULL; ++} ++ ++/** ++ * A GPU memory region, and attributes for CPU mappings. ++ */ ++struct kbase_va_region { ++ struct rb_node rblink; ++ struct list_head link; ++ ++ struct kbase_context *kctx; /* Backlink to base context */ ++ ++ u64 start_pfn; /* The PFN in GPU space */ ++ size_t nr_pages; ++ ++/* Free region */ ++#define KBASE_REG_FREE (1ul << 0) ++/* CPU write access */ ++#define KBASE_REG_CPU_WR (1ul << 1) ++/* GPU write access */ ++#define KBASE_REG_GPU_WR (1ul << 2) ++/* No eXecute flag */ ++#define KBASE_REG_GPU_NX (1ul << 3) ++/* Is CPU cached? */ ++#define KBASE_REG_CPU_CACHED (1ul << 4) ++/* Is GPU cached? */ ++#define KBASE_REG_GPU_CACHED (1ul << 5) ++ ++#define KBASE_REG_GROWABLE (1ul << 6) ++/* Can grow on pf? */ ++#define KBASE_REG_PF_GROW (1ul << 7) ++ ++/* Bit 8 is unused */ ++ ++/* inner shareable coherency */ ++#define KBASE_REG_SHARE_IN (1ul << 9) ++/* inner & outer shareable coherency */ ++#define KBASE_REG_SHARE_BOTH (1ul << 10) ++ ++/* Space for 4 different zones */ ++#define KBASE_REG_ZONE_MASK (3ul << 11) ++#define KBASE_REG_ZONE(x) (((x) & 3) << 11) ++ ++/* GPU read access */ ++#define KBASE_REG_GPU_RD (1ul<<13) ++/* CPU read access */ ++#define KBASE_REG_CPU_RD (1ul<<14) ++ ++/* Index of chosen MEMATTR for this region (0..7) */ ++#define KBASE_REG_MEMATTR_MASK (7ul << 16) ++#define KBASE_REG_MEMATTR_INDEX(x) (((x) & 7) << 16) ++#define KBASE_REG_MEMATTR_VALUE(x) (((x) & KBASE_REG_MEMATTR_MASK) >> 16) ++ ++#define KBASE_REG_SECURE (1ul << 19) ++ ++#define KBASE_REG_DONT_NEED (1ul << 20) ++ ++/* Imported buffer is padded? */ ++#define KBASE_REG_IMPORT_PAD (1ul << 21) ++ ++/* Bit 22 is reserved. ++ * ++ * Do not remove, use the next unreserved bit for new flags */ ++#define KBASE_REG_RESERVED_BIT_22 (1ul << 22) ++ ++#define KBASE_REG_ZONE_SAME_VA KBASE_REG_ZONE(0) ++ ++/* only used with 32-bit clients */ ++/* ++ * On a 32bit platform, custom VA should be wired from (4GB + shader region) ++ * to the VA limit of the GPU. Unfortunately, the Linux mmap() interface ++ * limits us to 2^32 pages (2^44 bytes, see mmap64 man page for reference). ++ * So we put the default limit to the maximum possible on Linux and shrink ++ * it down, if required by the GPU, during initialization. ++ */ ++ ++/* ++ * Dedicated 16MB region for shader code: ++ * VA range 0x101000000-0x102000000 ++ */ ++#define KBASE_REG_ZONE_EXEC KBASE_REG_ZONE(1) ++#define KBASE_REG_ZONE_EXEC_BASE (0x101000000ULL >> PAGE_SHIFT) ++#define KBASE_REG_ZONE_EXEC_SIZE ((16ULL * 1024 * 1024) >> PAGE_SHIFT) ++ ++#define KBASE_REG_ZONE_CUSTOM_VA KBASE_REG_ZONE(2) ++#define KBASE_REG_ZONE_CUSTOM_VA_BASE (KBASE_REG_ZONE_EXEC_BASE + KBASE_REG_ZONE_EXEC_SIZE) /* Starting after KBASE_REG_ZONE_EXEC */ ++#define KBASE_REG_ZONE_CUSTOM_VA_SIZE (((1ULL << 44) >> PAGE_SHIFT) - KBASE_REG_ZONE_CUSTOM_VA_BASE) ++/* end 32-bit clients only */ ++ ++ unsigned long flags; ++ ++ size_t extent; /* nr of pages alloc'd on PF */ ++ ++ struct kbase_mem_phy_alloc *cpu_alloc; /* the one alloc object we mmap to the CPU when mapping this region */ ++ struct kbase_mem_phy_alloc *gpu_alloc; /* the one alloc object we mmap to the GPU when mapping this region */ ++ ++ /* non-NULL if this memory object is a kds_resource */ ++ struct kds_resource *kds_res; ++ ++ /* List head used to store the region in the JIT allocation pool */ ++ struct list_head jit_node; ++}; ++ ++/* Common functions */ ++static inline struct tagged_addr *kbase_get_cpu_phy_pages( ++ struct kbase_va_region *reg) ++{ ++ KBASE_DEBUG_ASSERT(reg); ++ KBASE_DEBUG_ASSERT(reg->cpu_alloc); ++ KBASE_DEBUG_ASSERT(reg->gpu_alloc); ++ KBASE_DEBUG_ASSERT(reg->cpu_alloc->nents == reg->gpu_alloc->nents); ++ ++ return reg->cpu_alloc->pages; ++} ++ ++static inline struct tagged_addr *kbase_get_gpu_phy_pages( ++ struct kbase_va_region *reg) ++{ ++ KBASE_DEBUG_ASSERT(reg); ++ KBASE_DEBUG_ASSERT(reg->cpu_alloc); ++ KBASE_DEBUG_ASSERT(reg->gpu_alloc); ++ KBASE_DEBUG_ASSERT(reg->cpu_alloc->nents == reg->gpu_alloc->nents); ++ ++ return reg->gpu_alloc->pages; ++} ++ ++static inline size_t kbase_reg_current_backed_size(struct kbase_va_region *reg) ++{ ++ KBASE_DEBUG_ASSERT(reg); ++ /* if no alloc object the backed size naturally is 0 */ ++ if (!reg->cpu_alloc) ++ return 0; ++ ++ KBASE_DEBUG_ASSERT(reg->cpu_alloc); ++ KBASE_DEBUG_ASSERT(reg->gpu_alloc); ++ KBASE_DEBUG_ASSERT(reg->cpu_alloc->nents == reg->gpu_alloc->nents); ++ ++ return reg->cpu_alloc->nents; ++} ++ ++#define KBASE_MEM_PHY_ALLOC_LARGE_THRESHOLD ((size_t)(4*1024)) /* size above which vmalloc is used over kmalloc */ ++ ++static inline struct kbase_mem_phy_alloc *kbase_alloc_create(size_t nr_pages, enum kbase_memory_type type) ++{ ++ struct kbase_mem_phy_alloc *alloc; ++ size_t alloc_size = sizeof(*alloc) + sizeof(*alloc->pages) * nr_pages; ++ size_t per_page_size = sizeof(*alloc->pages); ++ ++ /* Imported pages may have page private data already in use */ ++ if (type == KBASE_MEM_TYPE_IMPORTED_USER_BUF) { ++ alloc_size += nr_pages * ++ sizeof(*alloc->imported.user_buf.dma_addrs); ++ per_page_size += sizeof(*alloc->imported.user_buf.dma_addrs); ++ } ++ ++ /* ++ * Prevent nr_pages*per_page_size + sizeof(*alloc) from ++ * wrapping around. ++ */ ++ if (nr_pages > ((((size_t) -1) - sizeof(*alloc)) ++ / per_page_size)) ++ return ERR_PTR(-ENOMEM); ++ ++ /* Allocate based on the size to reduce internal fragmentation of vmem */ ++ if (alloc_size > KBASE_MEM_PHY_ALLOC_LARGE_THRESHOLD) ++ alloc = vzalloc(alloc_size); ++ else ++ alloc = kzalloc(alloc_size, GFP_KERNEL); ++ ++ if (!alloc) ++ return ERR_PTR(-ENOMEM); ++ ++ /* Store allocation method */ ++ if (alloc_size > KBASE_MEM_PHY_ALLOC_LARGE_THRESHOLD) ++ alloc->properties |= KBASE_MEM_PHY_ALLOC_LARGE; ++ ++ kref_init(&alloc->kref); ++ atomic_set(&alloc->gpu_mappings, 0); ++ alloc->nents = 0; ++ alloc->pages = (void *)(alloc + 1); ++ INIT_LIST_HEAD(&alloc->mappings); ++ alloc->type = type; ++ INIT_LIST_HEAD(&alloc->zone_cache); ++ ++ if (type == KBASE_MEM_TYPE_IMPORTED_USER_BUF) ++ alloc->imported.user_buf.dma_addrs = ++ (void *) (alloc->pages + nr_pages); ++ ++ return alloc; ++} ++ ++static inline int kbase_reg_prepare_native(struct kbase_va_region *reg, ++ struct kbase_context *kctx) ++{ ++ KBASE_DEBUG_ASSERT(reg); ++ KBASE_DEBUG_ASSERT(!reg->cpu_alloc); ++ KBASE_DEBUG_ASSERT(!reg->gpu_alloc); ++ KBASE_DEBUG_ASSERT(reg->flags & KBASE_REG_FREE); ++ ++ reg->cpu_alloc = kbase_alloc_create(reg->nr_pages, ++ KBASE_MEM_TYPE_NATIVE); ++ if (IS_ERR(reg->cpu_alloc)) ++ return PTR_ERR(reg->cpu_alloc); ++ else if (!reg->cpu_alloc) ++ return -ENOMEM; ++ reg->cpu_alloc->imported.kctx = kctx; ++ INIT_LIST_HEAD(®->cpu_alloc->evict_node); ++ if (kbase_ctx_flag(kctx, KCTX_INFINITE_CACHE) ++ && (reg->flags & KBASE_REG_CPU_CACHED)) { ++ reg->gpu_alloc = kbase_alloc_create(reg->nr_pages, ++ KBASE_MEM_TYPE_NATIVE); ++ reg->gpu_alloc->imported.kctx = kctx; ++ INIT_LIST_HEAD(®->gpu_alloc->evict_node); ++ } else { ++ reg->gpu_alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc); ++ } ++ ++ INIT_LIST_HEAD(®->jit_node); ++ reg->flags &= ~KBASE_REG_FREE; ++ return 0; ++} ++ ++static inline int kbase_atomic_add_pages(int num_pages, atomic_t *used_pages) ++{ ++ int new_val = atomic_add_return(num_pages, used_pages); ++#if defined(CONFIG_MALI_BIFROST_GATOR_SUPPORT) ++ kbase_trace_mali_total_alloc_pages_change((long long int)new_val); ++#endif ++ return new_val; ++} ++ ++static inline int kbase_atomic_sub_pages(int num_pages, atomic_t *used_pages) ++{ ++ int new_val = atomic_sub_return(num_pages, used_pages); ++#if defined(CONFIG_MALI_BIFROST_GATOR_SUPPORT) ++ kbase_trace_mali_total_alloc_pages_change((long long int)new_val); ++#endif ++ return new_val; ++} ++ ++/* ++ * Max size for kbdev memory pool (in pages) ++ */ ++#define KBASE_MEM_POOL_MAX_SIZE_KBDEV (SZ_64M >> PAGE_SHIFT) ++ ++/* ++ * Max size for kctx memory pool (in pages) ++ */ ++#define KBASE_MEM_POOL_MAX_SIZE_KCTX (SZ_64M >> PAGE_SHIFT) ++ ++/* ++ * The order required for a 2MB page allocation (2^order * 4KB = 2MB) ++ */ ++#define KBASE_MEM_POOL_2MB_PAGE_TABLE_ORDER 9 ++ ++/* ++ * The order required for a 4KB page allocation ++ */ ++#define KBASE_MEM_POOL_4KB_PAGE_TABLE_ORDER 0 ++ ++/** ++ * kbase_mem_pool_init - Create a memory pool for a kbase device ++ * @pool: Memory pool to initialize ++ * @max_size: Maximum number of free pages the pool can hold ++ * @order: Page order for physical page size (order=0=>4kB, order=9=>2MB) ++ * @kbdev: Kbase device where memory is used ++ * @next_pool: Pointer to the next pool or NULL. ++ * ++ * Allocations from @pool are in whole pages. Each @pool has a free list where ++ * pages can be quickly allocated from. The free list is initially empty and ++ * filled whenever pages are freed back to the pool. The number of free pages ++ * in the pool will in general not exceed @max_size, but the pool may in ++ * certain corner cases grow above @max_size. ++ * ++ * If @next_pool is not NULL, we will allocate from @next_pool before going to ++ * the kernel allocator. Similarily pages can spill over to @next_pool when ++ * @pool is full. Pages are zeroed before they spill over to another pool, to ++ * prevent leaking information between applications. ++ * ++ * A shrinker is registered so that Linux mm can reclaim pages from the pool as ++ * needed. ++ * ++ * Return: 0 on success, negative -errno on error ++ */ ++int kbase_mem_pool_init(struct kbase_mem_pool *pool, ++ size_t max_size, ++ size_t order, ++ struct kbase_device *kbdev, ++ struct kbase_mem_pool *next_pool); ++ ++/** ++ * kbase_mem_pool_term - Destroy a memory pool ++ * @pool: Memory pool to destroy ++ * ++ * Pages in the pool will spill over to @next_pool (if available) or freed to ++ * the kernel. ++ */ ++void kbase_mem_pool_term(struct kbase_mem_pool *pool); ++ ++/** ++ * kbase_mem_pool_alloc - Allocate a page from memory pool ++ * @pool: Memory pool to allocate from ++ * ++ * Allocations from the pool are made as follows: ++ * 1. If there are free pages in the pool, allocate a page from @pool. ++ * 2. Otherwise, if @next_pool is not NULL and has free pages, allocate a page ++ * from @next_pool. ++ * 3. Return NULL if no memory in the pool ++ * ++ * Return: Pointer to allocated page, or NULL if allocation failed. ++ */ ++struct page *kbase_mem_pool_alloc(struct kbase_mem_pool *pool); ++ ++/** ++ * kbase_mem_pool_free - Free a page to memory pool ++ * @pool: Memory pool where page should be freed ++ * @page: Page to free to the pool ++ * @dirty: Whether some of the page may be dirty in the cache. ++ * ++ * Pages are freed to the pool as follows: ++ * 1. If @pool is not full, add @page to @pool. ++ * 2. Otherwise, if @next_pool is not NULL and not full, add @page to ++ * @next_pool. ++ * 3. Finally, free @page to the kernel. ++ */ ++void kbase_mem_pool_free(struct kbase_mem_pool *pool, struct page *page, ++ bool dirty); ++ ++/** ++ * kbase_mem_pool_alloc_pages - Allocate pages from memory pool ++ * @pool: Memory pool to allocate from ++ * @nr_pages: Number of pages to allocate ++ * @pages: Pointer to array where the physical address of the allocated ++ * pages will be stored. ++ * @partial_allowed: If fewer pages allocated is allowed ++ * ++ * Like kbase_mem_pool_alloc() but optimized for allocating many pages. ++ * ++ * Return: ++ * On success number of pages allocated (could be less than nr_pages if ++ * partial_allowed). ++ * On error an error code. ++ */ ++int kbase_mem_pool_alloc_pages(struct kbase_mem_pool *pool, size_t nr_pages, ++ struct tagged_addr *pages, bool partial_allowed); ++ ++/** ++ * kbase_mem_pool_free_pages - Free pages to memory pool ++ * @pool: Memory pool where pages should be freed ++ * @nr_pages: Number of pages to free ++ * @pages: Pointer to array holding the physical addresses of the pages to ++ * free. ++ * @dirty: Whether any pages may be dirty in the cache. ++ * @reclaimed: Whether the pages where reclaimable and thus should bypass ++ * the pool and go straight to the kernel. ++ * ++ * Like kbase_mem_pool_free() but optimized for freeing many pages. ++ */ ++void kbase_mem_pool_free_pages(struct kbase_mem_pool *pool, size_t nr_pages, ++ struct tagged_addr *pages, bool dirty, bool reclaimed); ++ ++/** ++ * kbase_mem_pool_size - Get number of free pages in memory pool ++ * @pool: Memory pool to inspect ++ * ++ * Note: the size of the pool may in certain corner cases exceed @max_size! ++ * ++ * Return: Number of free pages in the pool ++ */ ++static inline size_t kbase_mem_pool_size(struct kbase_mem_pool *pool) ++{ ++ return READ_ONCE(pool->cur_size); ++} ++ ++/** ++ * kbase_mem_pool_max_size - Get maximum number of free pages in memory pool ++ * @pool: Memory pool to inspect ++ * ++ * Return: Maximum number of free pages in the pool ++ */ ++static inline size_t kbase_mem_pool_max_size(struct kbase_mem_pool *pool) ++{ ++ return pool->max_size; ++} ++ ++ ++/** ++ * kbase_mem_pool_set_max_size - Set maximum number of free pages in memory pool ++ * @pool: Memory pool to inspect ++ * @max_size: Maximum number of free pages the pool can hold ++ * ++ * If @max_size is reduced, the pool will be shrunk to adhere to the new limit. ++ * For details see kbase_mem_pool_shrink(). ++ */ ++void kbase_mem_pool_set_max_size(struct kbase_mem_pool *pool, size_t max_size); ++ ++/** ++ * kbase_mem_pool_grow - Grow the pool ++ * @pool: Memory pool to grow ++ * @nr_to_grow: Number of pages to add to the pool ++ * ++ * Adds @nr_to_grow pages to the pool. Note that this may cause the pool to ++ * become larger than the maximum size specified. ++ * ++ * Returns: 0 on success, -ENOMEM if unable to allocate sufficent pages ++ */ ++int kbase_mem_pool_grow(struct kbase_mem_pool *pool, size_t nr_to_grow); ++ ++/** ++ * kbase_mem_pool_trim - Grow or shrink the pool to a new size ++ * @pool: Memory pool to trim ++ * @new_size: New number of pages in the pool ++ * ++ * If @new_size > @cur_size, fill the pool with new pages from the kernel, but ++ * not above the max_size for the pool. ++ * If @new_size < @cur_size, shrink the pool by freeing pages to the kernel. ++ */ ++void kbase_mem_pool_trim(struct kbase_mem_pool *pool, size_t new_size); ++ ++/** ++ * kbase_mem_alloc_page - Allocate a new page for a device ++ * @pool: Memory pool to allocate a page from ++ * ++ * Most uses should use kbase_mem_pool_alloc to allocate a page. However that ++ * function can fail in the event the pool is empty. ++ * ++ * Return: A new page or NULL if no memory ++ */ ++struct page *kbase_mem_alloc_page(struct kbase_mem_pool *pool); ++ ++int kbase_region_tracker_init(struct kbase_context *kctx); ++int kbase_region_tracker_init_jit(struct kbase_context *kctx, u64 jit_va_pages); ++void kbase_region_tracker_term(struct kbase_context *kctx); ++ ++struct kbase_va_region *kbase_region_tracker_find_region_enclosing_address(struct kbase_context *kctx, u64 gpu_addr); ++ ++/** ++ * @brief Check that a pointer is actually a valid region. ++ * ++ * Must be called with context lock held. ++ */ ++struct kbase_va_region *kbase_region_tracker_find_region_base_address(struct kbase_context *kctx, u64 gpu_addr); ++ ++struct kbase_va_region *kbase_alloc_free_region(struct kbase_context *kctx, u64 start_pfn, size_t nr_pages, int zone); ++void kbase_free_alloced_region(struct kbase_va_region *reg); ++int kbase_add_va_region(struct kbase_context *kctx, struct kbase_va_region *reg, u64 addr, size_t nr_pages, size_t align); ++ ++bool kbase_check_alloc_flags(unsigned long flags); ++bool kbase_check_import_flags(unsigned long flags); ++ ++/** ++ * kbase_update_region_flags - Convert user space flags to kernel region flags ++ * ++ * @kctx: kbase context ++ * @reg: The region to update the flags on ++ * @flags: The flags passed from user space ++ * ++ * The user space flag BASE_MEM_COHERENT_SYSTEM_REQUIRED will be rejected and ++ * this function will fail if the system does not support system coherency. ++ * ++ * Return: 0 if successful, -EINVAL if the flags are not supported ++ */ ++int kbase_update_region_flags(struct kbase_context *kctx, ++ struct kbase_va_region *reg, unsigned long flags); ++ ++void kbase_gpu_vm_lock(struct kbase_context *kctx); ++void kbase_gpu_vm_unlock(struct kbase_context *kctx); ++ ++int kbase_alloc_phy_pages(struct kbase_va_region *reg, size_t vsize, size_t size); ++ ++int kbase_mmu_init(struct kbase_context *kctx); ++void kbase_mmu_term(struct kbase_context *kctx); ++ ++phys_addr_t kbase_mmu_alloc_pgd(struct kbase_context *kctx); ++void kbase_mmu_free_pgd(struct kbase_context *kctx); ++int kbase_mmu_insert_pages_no_flush(struct kbase_context *kctx, u64 vpfn, ++ struct tagged_addr *phys, size_t nr, ++ unsigned long flags); ++int kbase_mmu_insert_pages(struct kbase_context *kctx, u64 vpfn, ++ struct tagged_addr *phys, size_t nr, ++ unsigned long flags); ++int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 vpfn, ++ struct tagged_addr phys, size_t nr, ++ unsigned long flags); ++ ++int kbase_mmu_teardown_pages(struct kbase_context *kctx, u64 vpfn, size_t nr); ++int kbase_mmu_update_pages(struct kbase_context *kctx, u64 vpfn, ++ struct tagged_addr *phys, size_t nr, ++ unsigned long flags); ++ ++/** ++ * @brief Register region and map it on the GPU. ++ * ++ * Call kbase_add_va_region() and map the region on the GPU. ++ */ ++int kbase_gpu_mmap(struct kbase_context *kctx, struct kbase_va_region *reg, u64 addr, size_t nr_pages, size_t align); ++ ++/** ++ * @brief Remove the region from the GPU and unregister it. ++ * ++ * Must be called with context lock held. ++ */ ++int kbase_gpu_munmap(struct kbase_context *kctx, struct kbase_va_region *reg); ++ ++/** ++ * The caller has the following locking conditions: ++ * - It must hold kbase_device->mmu_hw_mutex ++ * - It must hold the hwaccess_lock ++ */ ++void kbase_mmu_update(struct kbase_context *kctx); ++ ++/** ++ * kbase_mmu_disable() - Disable the MMU for a previously active kbase context. ++ * @kctx: Kbase context ++ * ++ * Disable and perform the required cache maintenance to remove the all ++ * data from provided kbase context from the GPU caches. ++ * ++ * The caller has the following locking conditions: ++ * - It must hold kbase_device->mmu_hw_mutex ++ * - It must hold the hwaccess_lock ++ */ ++void kbase_mmu_disable(struct kbase_context *kctx); ++ ++/** ++ * kbase_mmu_disable_as() - Set the MMU to unmapped mode for the specified ++ * address space. ++ * @kbdev: Kbase device ++ * @as_nr: The address space number to set to unmapped. ++ * ++ * This function must only be called during reset/power-up and it used to ++ * ensure the registers are in a known state. ++ * ++ * The caller must hold kbdev->mmu_hw_mutex. ++ */ ++void kbase_mmu_disable_as(struct kbase_device *kbdev, int as_nr); ++ ++void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat); ++ ++/** Dump the MMU tables to a buffer ++ * ++ * This function allocates a buffer (of @c nr_pages pages) to hold a dump of the MMU tables and fills it. If the ++ * buffer is too small then the return value will be NULL. ++ * ++ * The GPU vm lock must be held when calling this function. ++ * ++ * The buffer returned should be freed with @ref vfree when it is no longer required. ++ * ++ * @param[in] kctx The kbase context to dump ++ * @param[in] nr_pages The number of pages to allocate for the buffer. ++ * ++ * @return The address of the buffer containing the MMU dump or NULL on error (including if the @c nr_pages is too ++ * small) ++ */ ++void *kbase_mmu_dump(struct kbase_context *kctx, int nr_pages); ++ ++/** ++ * kbase_sync_now - Perform cache maintenance on a memory region ++ * ++ * @kctx: The kbase context of the region ++ * @sset: A syncset structure describing the region and direction of the ++ * synchronisation required ++ * ++ * Return: 0 on success or error code ++ */ ++int kbase_sync_now(struct kbase_context *kctx, struct basep_syncset *sset); ++void kbase_sync_single(struct kbase_context *kctx, struct tagged_addr cpu_pa, ++ struct tagged_addr gpu_pa, off_t offset, size_t size, ++ enum kbase_sync_type sync_fn); ++void kbase_pre_job_sync(struct kbase_context *kctx, struct base_syncset *syncsets, size_t nr); ++void kbase_post_job_sync(struct kbase_context *kctx, struct base_syncset *syncsets, size_t nr); ++ ++/* OS specific functions */ ++int kbase_mem_free(struct kbase_context *kctx, u64 gpu_addr); ++int kbase_mem_free_region(struct kbase_context *kctx, struct kbase_va_region *reg); ++void kbase_os_mem_map_lock(struct kbase_context *kctx); ++void kbase_os_mem_map_unlock(struct kbase_context *kctx); ++ ++/** ++ * @brief Update the memory allocation counters for the current process ++ * ++ * OS specific call to updates the current memory allocation counters for the current process with ++ * the supplied delta. ++ * ++ * @param[in] kctx The kbase context ++ * @param[in] pages The desired delta to apply to the memory usage counters. ++ */ ++ ++void kbasep_os_process_page_usage_update(struct kbase_context *kctx, int pages); ++ ++/** ++ * @brief Add to the memory allocation counters for the current process ++ * ++ * OS specific call to add to the current memory allocation counters for the current process by ++ * the supplied amount. ++ * ++ * @param[in] kctx The kernel base context used for the allocation. ++ * @param[in] pages The desired delta to apply to the memory usage counters. ++ */ ++ ++static inline void kbase_process_page_usage_inc(struct kbase_context *kctx, int pages) ++{ ++ kbasep_os_process_page_usage_update(kctx, pages); ++} ++ ++/** ++ * @brief Subtract from the memory allocation counters for the current process ++ * ++ * OS specific call to subtract from the current memory allocation counters for the current process by ++ * the supplied amount. ++ * ++ * @param[in] kctx The kernel base context used for the allocation. ++ * @param[in] pages The desired delta to apply to the memory usage counters. ++ */ ++ ++static inline void kbase_process_page_usage_dec(struct kbase_context *kctx, int pages) ++{ ++ kbasep_os_process_page_usage_update(kctx, 0 - pages); ++} ++ ++/** ++ * kbasep_find_enclosing_cpu_mapping_offset() - Find the offset of the CPU ++ * mapping of a memory allocation containing a given address range ++ * ++ * Searches for a CPU mapping of any part of any region that fully encloses the ++ * CPU virtual address range specified by @uaddr and @size. Returns a failure ++ * indication if only part of the address range lies within a CPU mapping. ++ * ++ * @kctx: The kernel base context used for the allocation. ++ * @uaddr: Start of the CPU virtual address range. ++ * @size: Size of the CPU virtual address range (in bytes). ++ * @offset: The offset from the start of the allocation to the specified CPU ++ * virtual address. ++ * ++ * Return: 0 if offset was obtained successfully. Error code otherwise. ++ */ ++int kbasep_find_enclosing_cpu_mapping_offset( ++ struct kbase_context *kctx, ++ unsigned long uaddr, size_t size, u64 *offset); ++ ++enum hrtimer_restart kbasep_as_poke_timer_callback(struct hrtimer *timer); ++void kbase_as_poking_timer_retain_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom); ++void kbase_as_poking_timer_release_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom); ++ ++/** ++* @brief Allocates physical pages. ++* ++* Allocates \a nr_pages_requested and updates the alloc object. ++* ++* @param[in] alloc allocation object to add pages to ++* @param[in] nr_pages_requested number of physical pages to allocate ++* ++* @return 0 if all pages have been successfully allocated. Error code otherwise ++*/ ++int kbase_alloc_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, size_t nr_pages_requested); ++ ++/** ++* @brief Free physical pages. ++* ++* Frees \a nr_pages and updates the alloc object. ++* ++* @param[in] alloc allocation object to free pages from ++* @param[in] nr_pages_to_free number of physical pages to free ++*/ ++int kbase_free_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, size_t nr_pages_to_free); ++ ++static inline void kbase_set_dma_addr(struct page *p, dma_addr_t dma_addr) ++{ ++ SetPagePrivate(p); ++ if (sizeof(dma_addr_t) > sizeof(p->private)) { ++ /* on 32-bit ARM with LPAE dma_addr_t becomes larger, but the ++ * private field stays the same. So we have to be clever and ++ * use the fact that we only store DMA addresses of whole pages, ++ * so the low bits should be zero */ ++ KBASE_DEBUG_ASSERT(!(dma_addr & (PAGE_SIZE - 1))); ++ set_page_private(p, dma_addr >> PAGE_SHIFT); ++ } else { ++ set_page_private(p, dma_addr); ++ } ++} ++ ++static inline dma_addr_t kbase_dma_addr(struct page *p) ++{ ++ if (sizeof(dma_addr_t) > sizeof(p->private)) ++ return ((dma_addr_t)page_private(p)) << PAGE_SHIFT; ++ ++ return (dma_addr_t)page_private(p); ++} ++ ++static inline void kbase_clear_dma_addr(struct page *p) ++{ ++ ClearPagePrivate(p); ++} ++ ++/** ++* @brief Process a bus or page fault. ++* ++* This function will process a fault on a specific address space ++* ++* @param[in] kbdev The @ref kbase_device the fault happened on ++* @param[in] kctx The @ref kbase_context for the faulting address space if ++* one was found. ++* @param[in] as The address space that has the fault ++*/ ++void kbase_mmu_interrupt_process(struct kbase_device *kbdev, ++ struct kbase_context *kctx, struct kbase_as *as); ++ ++/** ++ * @brief Process a page fault. ++ * ++ * @param[in] data work_struct passed by queue_work() ++ */ ++void page_fault_worker(struct work_struct *data); ++ ++/** ++ * @brief Process a bus fault. ++ * ++ * @param[in] data work_struct passed by queue_work() ++ */ ++void bus_fault_worker(struct work_struct *data); ++ ++/** ++ * @brief Flush MMU workqueues. ++ * ++ * This function will cause any outstanding page or bus faults to be processed. ++ * It should be called prior to powering off the GPU. ++ * ++ * @param[in] kbdev Device pointer ++ */ ++void kbase_flush_mmu_wqs(struct kbase_device *kbdev); ++ ++/** ++ * kbase_sync_single_for_device - update physical memory and give GPU ownership ++ * @kbdev: Device pointer ++ * @handle: DMA address of region ++ * @size: Size of region to sync ++ * @dir: DMA data direction ++ */ ++ ++void kbase_sync_single_for_device(struct kbase_device *kbdev, dma_addr_t handle, ++ size_t size, enum dma_data_direction dir); ++ ++/** ++ * kbase_sync_single_for_cpu - update physical memory and give CPU ownership ++ * @kbdev: Device pointer ++ * @handle: DMA address of region ++ * @size: Size of region to sync ++ * @dir: DMA data direction ++ */ ++ ++void kbase_sync_single_for_cpu(struct kbase_device *kbdev, dma_addr_t handle, ++ size_t size, enum dma_data_direction dir); ++ ++#ifdef CONFIG_DEBUG_FS ++/** ++ * kbase_jit_debugfs_init - Add per context debugfs entry for JIT. ++ * @kctx: kbase context ++ */ ++void kbase_jit_debugfs_init(struct kbase_context *kctx); ++#endif /* CONFIG_DEBUG_FS */ ++ ++/** ++ * kbase_jit_init - Initialize the JIT memory pool management ++ * @kctx: kbase context ++ * ++ * Returns zero on success or negative error number on failure. ++ */ ++int kbase_jit_init(struct kbase_context *kctx); ++ ++/** ++ * kbase_jit_allocate - Allocate JIT memory ++ * @kctx: kbase context ++ * @info: JIT allocation information ++ * ++ * Return: JIT allocation on success or NULL on failure. ++ */ ++struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx, ++ struct base_jit_alloc_info *info); ++ ++/** ++ * kbase_jit_free - Free a JIT allocation ++ * @kctx: kbase context ++ * @reg: JIT allocation ++ * ++ * Frees a JIT allocation and places it into the free pool for later reuse. ++ */ ++void kbase_jit_free(struct kbase_context *kctx, struct kbase_va_region *reg); ++ ++/** ++ * kbase_jit_backing_lost - Inform JIT that an allocation has lost backing ++ * @reg: JIT allocation ++ */ ++void kbase_jit_backing_lost(struct kbase_va_region *reg); ++ ++/** ++ * kbase_jit_evict - Evict a JIT allocation from the pool ++ * @kctx: kbase context ++ * ++ * Evict the least recently used JIT allocation from the pool. This can be ++ * required if normal VA allocations are failing due to VA exhaustion. ++ * ++ * Return: True if a JIT allocation was freed, false otherwise. ++ */ ++bool kbase_jit_evict(struct kbase_context *kctx); ++ ++/** ++ * kbase_jit_term - Terminate the JIT memory pool management ++ * @kctx: kbase context ++ */ ++void kbase_jit_term(struct kbase_context *kctx); ++ ++/** ++ * kbase_map_external_resource - Map an external resource to the GPU. ++ * @kctx: kbase context. ++ * @reg: The region to map. ++ * @locked_mm: The mm_struct which has been locked for this operation. ++ * @kds_res_count: The number of KDS resources. ++ * @kds_resources: Array of KDS resources. ++ * @kds_access_bitmap: Access bitmap for KDS. ++ * @exclusive: If the KDS resource requires exclusive access. ++ * ++ * Return: The physical allocation which backs the region on success or NULL ++ * on failure. ++ */ ++struct kbase_mem_phy_alloc *kbase_map_external_resource( ++ struct kbase_context *kctx, struct kbase_va_region *reg, ++ struct mm_struct *locked_mm ++#ifdef CONFIG_KDS ++ , u32 *kds_res_count, struct kds_resource **kds_resources, ++ unsigned long *kds_access_bitmap, bool exclusive ++#endif ++ ); ++ ++/** ++ * kbase_unmap_external_resource - Unmap an external resource from the GPU. ++ * @kctx: kbase context. ++ * @reg: The region to unmap or NULL if it has already been released. ++ * @alloc: The physical allocation being unmapped. ++ */ ++void kbase_unmap_external_resource(struct kbase_context *kctx, ++ struct kbase_va_region *reg, struct kbase_mem_phy_alloc *alloc); ++ ++/** ++ * kbase_sticky_resource_init - Initialize sticky resource management. ++ * @kctx: kbase context ++ * ++ * Returns zero on success or negative error number on failure. ++ */ ++int kbase_sticky_resource_init(struct kbase_context *kctx); ++ ++/** ++ * kbase_sticky_resource_acquire - Acquire a reference on a sticky resource. ++ * @kctx: kbase context. ++ * @gpu_addr: The GPU address of the external resource. ++ * ++ * Return: The metadata object which represents the binding between the ++ * external resource and the kbase context on success or NULL on failure. ++ */ ++struct kbase_ctx_ext_res_meta *kbase_sticky_resource_acquire( ++ struct kbase_context *kctx, u64 gpu_addr); ++ ++/** ++ * kbase_sticky_resource_release - Release a reference on a sticky resource. ++ * @kctx: kbase context. ++ * @meta: Binding metadata. ++ * @gpu_addr: GPU address of the external resource. ++ * ++ * If meta is NULL then gpu_addr will be used to scan the metadata list and ++ * find the matching metadata (if any), otherwise the provided meta will be ++ * used and gpu_addr will be ignored. ++ * ++ * Return: True if the release found the metadata and the reference was dropped. ++ */ ++bool kbase_sticky_resource_release(struct kbase_context *kctx, ++ struct kbase_ctx_ext_res_meta *meta, u64 gpu_addr); ++ ++/** ++ * kbase_sticky_resource_term - Terminate sticky resource management. ++ * @kctx: kbase context ++ */ ++void kbase_sticky_resource_term(struct kbase_context *kctx); ++ ++/** ++ * kbase_zone_cache_update - Update the memory zone cache after new pages have ++ * been added. ++ * @alloc: The physical memory allocation to build the cache for. ++ * @start_offset: Offset to where the new pages start. ++ * ++ * Updates an existing memory zone cache, updating the counters for the ++ * various zones. ++ * If the memory allocation doesn't already have a zone cache assume that ++ * one isn't created and thus don't do anything. ++ * ++ * Return: Zero cache was updated, negative error code on error. ++ */ ++int kbase_zone_cache_update(struct kbase_mem_phy_alloc *alloc, ++ size_t start_offset); ++ ++/** ++ * kbase_zone_cache_build - Build the memory zone cache. ++ * @alloc: The physical memory allocation to build the cache for. ++ * ++ * Create a new zone cache for the provided physical memory allocation if ++ * one doesn't already exist, if one does exist then just return. ++ * ++ * Return: Zero if the zone cache was created, negative error code on error. ++ */ ++int kbase_zone_cache_build(struct kbase_mem_phy_alloc *alloc); ++ ++/** ++ * kbase_zone_cache_clear - Clear the memory zone cache. ++ * @alloc: The physical memory allocation to clear the cache on. ++ */ ++void kbase_zone_cache_clear(struct kbase_mem_phy_alloc *alloc); ++ ++#endif /* _KBASE_MEM_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_linux.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_linux.c +new file mode 100755 +index 000000000000..842444c9b0bd +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_linux.c +@@ -0,0 +1,2678 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/** ++ * @file mali_kbase_mem_linux.c ++ * Base kernel memory APIs, Linux implementation. ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)) && \ ++ (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0)) ++#include ++#endif /* LINUX_VERSION_CODE >= 3.5.0 && < 4.8.0 */ ++#ifdef CONFIG_DMA_SHARED_BUFFER ++#include ++#endif /* defined(CONFIG_DMA_SHARED_BUFFER) */ ++#include ++#include ++ ++#include ++#include ++#include ++#include ++ ++static int kbase_tracking_page_setup(struct kbase_context *kctx, struct vm_area_struct *vma); ++ ++/** ++ * kbase_mem_shrink_cpu_mapping - Shrink the CPU mapping(s) of an allocation ++ * @kctx: Context the region belongs to ++ * @reg: The GPU region ++ * @new_pages: The number of pages after the shrink ++ * @old_pages: The number of pages before the shrink ++ * ++ * Shrink (or completely remove) all CPU mappings which reference the shrunk ++ * part of the allocation. ++ * ++ * Note: Caller must be holding the processes mmap_sem lock. ++ */ ++static void kbase_mem_shrink_cpu_mapping(struct kbase_context *kctx, ++ struct kbase_va_region *reg, ++ u64 new_pages, u64 old_pages); ++ ++/** ++ * kbase_mem_shrink_gpu_mapping - Shrink the GPU mapping of an allocation ++ * @kctx: Context the region belongs to ++ * @reg: The GPU region or NULL if there isn't one ++ * @new_pages: The number of pages after the shrink ++ * @old_pages: The number of pages before the shrink ++ * ++ * Return: 0 on success, negative -errno on error ++ * ++ * Unmap the shrunk pages from the GPU mapping. Note that the size of the region ++ * itself is unmodified as we still need to reserve the VA, only the page tables ++ * will be modified by this function. ++ */ ++static int kbase_mem_shrink_gpu_mapping(struct kbase_context *kctx, ++ struct kbase_va_region *reg, ++ u64 new_pages, u64 old_pages); ++ ++struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, ++ u64 va_pages, u64 commit_pages, u64 extent, u64 *flags, ++ u64 *gpu_va) ++{ ++ int zone; ++ int gpu_pc_bits; ++ struct kbase_va_region *reg; ++ struct device *dev; ++ ++ KBASE_DEBUG_ASSERT(kctx); ++ KBASE_DEBUG_ASSERT(flags); ++ KBASE_DEBUG_ASSERT(gpu_va); ++ ++ dev = kctx->kbdev->dev; ++ *gpu_va = 0; /* return 0 on failure */ ++ ++ gpu_pc_bits = kctx->kbdev->gpu_props.props.core_props.log2_program_counter_size; ++ ++ if (0 == va_pages) { ++ dev_warn(dev, "kbase_mem_alloc called with 0 va_pages!"); ++ goto bad_size; ++ } ++ ++ if (va_pages > (U64_MAX / PAGE_SIZE)) ++ /* 64-bit address range is the max */ ++ goto bad_size; ++ ++ if (!kbase_check_alloc_flags(*flags)) { ++ dev_warn(dev, ++ "kbase_mem_alloc called with bad flags (%llx)", ++ (unsigned long long)*flags); ++ goto bad_flags; ++ } ++ ++ if ((*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0 && ++ !kbase_device_is_cpu_coherent(kctx->kbdev)) { ++ dev_warn(dev, "kbase_mem_alloc call required coherent mem when unavailable"); ++ goto bad_flags; ++ } ++ if ((*flags & BASE_MEM_COHERENT_SYSTEM) != 0 && ++ !kbase_device_is_cpu_coherent(kctx->kbdev)) { ++ /* Remove COHERENT_SYSTEM flag if coherent mem is unavailable */ ++ *flags &= ~BASE_MEM_COHERENT_SYSTEM; ++ } ++ ++ /* Limit GPU executable allocs to GPU PC size */ ++ if ((*flags & BASE_MEM_PROT_GPU_EX) && ++ (va_pages > (1ULL << gpu_pc_bits >> PAGE_SHIFT))) ++ goto bad_ex_size; ++ ++ /* find out which VA zone to use */ ++ if (*flags & BASE_MEM_SAME_VA) ++ zone = KBASE_REG_ZONE_SAME_VA; ++ else if (*flags & BASE_MEM_PROT_GPU_EX) ++ zone = KBASE_REG_ZONE_EXEC; ++ else ++ zone = KBASE_REG_ZONE_CUSTOM_VA; ++ ++ reg = kbase_alloc_free_region(kctx, 0, va_pages, zone); ++ if (!reg) { ++ dev_err(dev, "Failed to allocate free region"); ++ goto no_region; ++ } ++ ++ if (kbase_update_region_flags(kctx, reg, *flags) != 0) ++ goto invalid_flags; ++ ++ if (kbase_reg_prepare_native(reg, kctx) != 0) { ++ dev_err(dev, "Failed to prepare region"); ++ goto prepare_failed; ++ } ++ ++ if (*flags & BASE_MEM_GROW_ON_GPF) ++ reg->extent = extent; ++ else ++ reg->extent = 0; ++ ++ if (kbase_alloc_phy_pages(reg, va_pages, commit_pages) != 0) { ++ dev_warn(dev, "Failed to allocate %lld pages (va_pages=%lld)", ++ (unsigned long long)commit_pages, ++ (unsigned long long)va_pages); ++ goto no_mem; ++ } ++ ++ kbase_gpu_vm_lock(kctx); ++ ++ /* mmap needed to setup VA? */ ++ if (*flags & BASE_MEM_SAME_VA) { ++ unsigned long prot = PROT_NONE; ++ unsigned long va_size = va_pages << PAGE_SHIFT; ++ unsigned long va_map = va_size; ++ unsigned long cookie, cookie_nr; ++ unsigned long cpu_addr; ++ ++ /* Bind to a cookie */ ++ if (!kctx->cookies) { ++ dev_err(dev, "No cookies available for allocation!"); ++ kbase_gpu_vm_unlock(kctx); ++ goto no_cookie; ++ } ++ /* return a cookie */ ++ cookie_nr = __ffs(kctx->cookies); ++ kctx->cookies &= ~(1UL << cookie_nr); ++ BUG_ON(kctx->pending_regions[cookie_nr]); ++ kctx->pending_regions[cookie_nr] = reg; ++ ++ kbase_gpu_vm_unlock(kctx); ++ ++ /* relocate to correct base */ ++ cookie = cookie_nr + PFN_DOWN(BASE_MEM_COOKIE_BASE); ++ cookie <<= PAGE_SHIFT; ++ ++ /* ++ * 10.1-10.4 UKU userland relies on the kernel to call mmap. ++ * For all other versions we can just return the cookie ++ */ ++ if (kctx->api_version < KBASE_API_VERSION(10, 1) || ++ kctx->api_version > KBASE_API_VERSION(10, 4)) { ++ *gpu_va = (u64) cookie; ++ return reg; ++ } ++ if (*flags & BASE_MEM_PROT_CPU_RD) ++ prot |= PROT_READ; ++ if (*flags & BASE_MEM_PROT_CPU_WR) ++ prot |= PROT_WRITE; ++ ++ cpu_addr = vm_mmap(kctx->filp, 0, va_map, prot, ++ MAP_SHARED, cookie); ++ ++ if (IS_ERR_VALUE(cpu_addr)) { ++ kbase_gpu_vm_lock(kctx); ++ kctx->pending_regions[cookie_nr] = NULL; ++ kctx->cookies |= (1UL << cookie_nr); ++ kbase_gpu_vm_unlock(kctx); ++ goto no_mmap; ++ } ++ ++ *gpu_va = (u64) cpu_addr; ++ } else /* we control the VA */ { ++ if (kbase_gpu_mmap(kctx, reg, 0, va_pages, 1) != 0) { ++ dev_warn(dev, "Failed to map memory on GPU"); ++ kbase_gpu_vm_unlock(kctx); ++ goto no_mmap; ++ } ++ /* return real GPU VA */ ++ *gpu_va = reg->start_pfn << PAGE_SHIFT; ++ ++ kbase_gpu_vm_unlock(kctx); ++ } ++ ++ return reg; ++ ++no_mmap: ++no_cookie: ++no_mem: ++ kbase_mem_phy_alloc_put(reg->cpu_alloc); ++ kbase_mem_phy_alloc_put(reg->gpu_alloc); ++invalid_flags: ++prepare_failed: ++ kfree(reg); ++no_region: ++bad_ex_size: ++bad_flags: ++bad_size: ++ return NULL; ++} ++KBASE_EXPORT_TEST_API(kbase_mem_alloc); ++ ++int kbase_mem_query(struct kbase_context *kctx, u64 gpu_addr, int query, u64 * const out) ++{ ++ struct kbase_va_region *reg; ++ int ret = -EINVAL; ++ ++ KBASE_DEBUG_ASSERT(kctx); ++ KBASE_DEBUG_ASSERT(out); ++ ++ if (gpu_addr & ~PAGE_MASK) { ++ dev_warn(kctx->kbdev->dev, "mem_query: gpu_addr: passed parameter is invalid"); ++ return -EINVAL; ++ } ++ ++ kbase_gpu_vm_lock(kctx); ++ ++ /* Validate the region */ ++ reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr); ++ if (!reg || (reg->flags & KBASE_REG_FREE)) ++ goto out_unlock; ++ ++ switch (query) { ++ case KBASE_MEM_QUERY_COMMIT_SIZE: ++ if (reg->cpu_alloc->type != KBASE_MEM_TYPE_ALIAS) { ++ *out = kbase_reg_current_backed_size(reg); ++ } else { ++ size_t i; ++ struct kbase_aliased *aliased; ++ *out = 0; ++ aliased = reg->cpu_alloc->imported.alias.aliased; ++ for (i = 0; i < reg->cpu_alloc->imported.alias.nents; i++) ++ *out += aliased[i].length; ++ } ++ break; ++ case KBASE_MEM_QUERY_VA_SIZE: ++ *out = reg->nr_pages; ++ break; ++ case KBASE_MEM_QUERY_FLAGS: ++ { ++ *out = 0; ++ if (KBASE_REG_CPU_WR & reg->flags) ++ *out |= BASE_MEM_PROT_CPU_WR; ++ if (KBASE_REG_CPU_RD & reg->flags) ++ *out |= BASE_MEM_PROT_CPU_RD; ++ if (KBASE_REG_CPU_CACHED & reg->flags) ++ *out |= BASE_MEM_CACHED_CPU; ++ if (KBASE_REG_GPU_WR & reg->flags) ++ *out |= BASE_MEM_PROT_GPU_WR; ++ if (KBASE_REG_GPU_RD & reg->flags) ++ *out |= BASE_MEM_PROT_GPU_RD; ++ if (!(KBASE_REG_GPU_NX & reg->flags)) ++ *out |= BASE_MEM_PROT_GPU_EX; ++ if (KBASE_REG_SHARE_BOTH & reg->flags) ++ *out |= BASE_MEM_COHERENT_SYSTEM; ++ if (KBASE_REG_SHARE_IN & reg->flags) ++ *out |= BASE_MEM_COHERENT_LOCAL; ++ break; ++ } ++ default: ++ *out = 0; ++ goto out_unlock; ++ } ++ ++ ret = 0; ++ ++out_unlock: ++ kbase_gpu_vm_unlock(kctx); ++ return ret; ++} ++ ++/** ++ * kbase_mem_evictable_reclaim_count_objects - Count number of pages in the ++ * Ephemeral memory eviction list. ++ * @s: Shrinker ++ * @sc: Shrinker control ++ * ++ * Return: Number of pages which can be freed. ++ */ ++static ++unsigned long kbase_mem_evictable_reclaim_count_objects(struct shrinker *s, ++ struct shrink_control *sc) ++{ ++ struct kbase_context *kctx; ++ struct kbase_mem_phy_alloc *alloc; ++ unsigned long pages = 0; ++ ++ kctx = container_of(s, struct kbase_context, reclaim); ++ ++ mutex_lock(&kctx->jit_evict_lock); ++ ++ list_for_each_entry(alloc, &kctx->evict_list, evict_node) ++ pages += alloc->nents; ++ ++ mutex_unlock(&kctx->jit_evict_lock); ++ return pages; ++} ++ ++/** ++ * kbase_mem_evictable_reclaim_scan_objects - Scan the Ephemeral memory eviction ++ * list for pages and try to reclaim them. ++ * @s: Shrinker ++ * @sc: Shrinker control ++ * ++ * Return: Number of pages freed (can be less then requested) or -1 if the ++ * shrinker failed to free pages in its pool. ++ * ++ * Note: ++ * This function accesses region structures without taking the region lock, ++ * this is required as the OOM killer can call the shrinker after the region ++ * lock has already been held. ++ * This is safe as we can guarantee that a region on the eviction list will ++ * not be freed (kbase_mem_free_region removes the allocation from the list ++ * before destroying it), or modified by other parts of the driver. ++ * The eviction list itself is guarded by the eviction lock and the MMU updates ++ * are protected by their own lock. ++ */ ++static ++unsigned long kbase_mem_evictable_reclaim_scan_objects(struct shrinker *s, ++ struct shrink_control *sc) ++{ ++ struct kbase_context *kctx; ++ struct kbase_mem_phy_alloc *alloc; ++ struct kbase_mem_phy_alloc *tmp; ++ unsigned long freed = 0; ++ ++ kctx = container_of(s, struct kbase_context, reclaim); ++ mutex_lock(&kctx->jit_evict_lock); ++ ++ list_for_each_entry_safe(alloc, tmp, &kctx->evict_list, evict_node) { ++ int err; ++ ++ err = kbase_mem_shrink_gpu_mapping(kctx, alloc->reg, ++ 0, alloc->nents); ++ if (err != 0) { ++ /* ++ * Failed to remove GPU mapping, tell the shrinker ++ * to stop trying to shrink our slab even though we ++ * have pages in it. ++ */ ++ freed = -1; ++ goto out_unlock; ++ } ++ ++ /* ++ * Update alloc->evicted before freeing the backing so the ++ * helper can determine that it needs to bypass the accounting ++ * and memory pool. ++ */ ++ alloc->evicted = alloc->nents; ++ ++ kbase_free_phy_pages_helper(alloc, alloc->evicted); ++ freed += alloc->evicted; ++ list_del_init(&alloc->evict_node); ++ ++ /* ++ * Inform the JIT allocator this region has lost backing ++ * as it might need to free the allocation. ++ */ ++ kbase_jit_backing_lost(alloc->reg); ++ ++ /* Enough pages have been freed so stop now */ ++ if (freed > sc->nr_to_scan) ++ break; ++ } ++out_unlock: ++ mutex_unlock(&kctx->jit_evict_lock); ++ ++ return freed; ++} ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) ++static int kbase_mem_evictable_reclaim_shrink(struct shrinker *s, ++ struct shrink_control *sc) ++{ ++ if (sc->nr_to_scan == 0) ++ return kbase_mem_evictable_reclaim_count_objects(s, sc); ++ ++ return kbase_mem_evictable_reclaim_scan_objects(s, sc); ++} ++#endif ++ ++int kbase_mem_evictable_init(struct kbase_context *kctx) ++{ ++ INIT_LIST_HEAD(&kctx->evict_list); ++ mutex_init(&kctx->jit_evict_lock); ++ ++ /* Register shrinker */ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) ++ kctx->reclaim.shrink = kbase_mem_evictable_reclaim_shrink; ++#else ++ kctx->reclaim.count_objects = kbase_mem_evictable_reclaim_count_objects; ++ kctx->reclaim.scan_objects = kbase_mem_evictable_reclaim_scan_objects; ++#endif ++ kctx->reclaim.seeks = DEFAULT_SEEKS; ++ /* Kernel versions prior to 3.1 : ++ * struct shrinker does not define batch */ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 1, 0) ++ kctx->reclaim.batch = 0; ++#endif ++ register_shrinker(&kctx->reclaim); ++ return 0; ++} ++ ++void kbase_mem_evictable_deinit(struct kbase_context *kctx) ++{ ++ unregister_shrinker(&kctx->reclaim); ++} ++ ++struct kbase_mem_zone_cache_entry { ++ /* List head used to link the cache entry to the memory allocation. */ ++ struct list_head zone_node; ++ /* The zone the cacheline is for. */ ++ struct zone *zone; ++ /* The number of pages in the allocation which belong to this zone. */ ++ u64 count; ++}; ++ ++static bool kbase_zone_cache_builder(struct kbase_mem_phy_alloc *alloc, ++ size_t start_offset) ++{ ++ struct kbase_mem_zone_cache_entry *cache = NULL; ++ size_t i; ++ int ret = 0; ++ ++ for (i = start_offset; i < alloc->nents; i++) { ++ struct page *p = phys_to_page(as_phys_addr_t(alloc->pages[i])); ++ struct zone *zone = page_zone(p); ++ bool create = true; ++ ++ if (cache && (cache->zone == zone)) { ++ /* ++ * Fast path check as most of the time adjacent ++ * pages come from the same zone. ++ */ ++ create = false; ++ } else { ++ /* ++ * Slow path check, walk all the cache entries to see ++ * if we already know about this zone. ++ */ ++ list_for_each_entry(cache, &alloc->zone_cache, zone_node) { ++ if (cache->zone == zone) { ++ create = false; ++ break; ++ } ++ } ++ } ++ ++ /* This zone wasn't found in the cache, create an entry for it */ ++ if (create) { ++ cache = kmalloc(sizeof(*cache), GFP_KERNEL); ++ if (!cache) { ++ ret = -ENOMEM; ++ goto bail; ++ } ++ cache->zone = zone; ++ cache->count = 0; ++ list_add(&cache->zone_node, &alloc->zone_cache); ++ } ++ ++ cache->count++; ++ } ++ return 0; ++ ++bail: ++ return ret; ++} ++ ++int kbase_zone_cache_update(struct kbase_mem_phy_alloc *alloc, ++ size_t start_offset) ++{ ++ /* ++ * Bail if the zone cache is empty, only update the cache if it ++ * existed in the first place. ++ */ ++ if (list_empty(&alloc->zone_cache)) ++ return 0; ++ ++ return kbase_zone_cache_builder(alloc, start_offset); ++} ++ ++int kbase_zone_cache_build(struct kbase_mem_phy_alloc *alloc) ++{ ++ /* Bail if the zone cache already exists */ ++ if (!list_empty(&alloc->zone_cache)) ++ return 0; ++ ++ return kbase_zone_cache_builder(alloc, 0); ++} ++ ++void kbase_zone_cache_clear(struct kbase_mem_phy_alloc *alloc) ++{ ++ struct kbase_mem_zone_cache_entry *walker; ++ ++ while(!list_empty(&alloc->zone_cache)){ ++ walker = list_first_entry(&alloc->zone_cache, ++ struct kbase_mem_zone_cache_entry, zone_node); ++ list_del(&walker->zone_node); ++ kfree(walker); ++ } ++} ++ ++/** ++ * kbase_mem_evictable_mark_reclaim - Mark the pages as reclaimable. ++ * @alloc: The physical allocation ++ */ ++static void kbase_mem_evictable_mark_reclaim(struct kbase_mem_phy_alloc *alloc) ++{ ++ struct kbase_context *kctx = alloc->imported.kctx; ++ struct kbase_mem_zone_cache_entry *zone_cache; ++ int __maybe_unused new_page_count; ++ int err; ++ ++ /* Attempt to build a zone cache of tracking */ ++ err = kbase_zone_cache_build(alloc); ++ if (err == 0) { ++ /* Bulk update all the zones */ ++ list_for_each_entry(zone_cache, &alloc->zone_cache, zone_node) { ++ zone_page_state_add(zone_cache->count, ++ zone_cache->zone, NR_SLAB_RECLAIMABLE); ++ } ++ } else { ++ /* Fall-back to page by page updates */ ++ int i; ++ ++ for (i = 0; i < alloc->nents; i++) { ++ struct page *p; ++ struct zone *zone; ++ ++ p = phys_to_page(as_phys_addr_t(alloc->pages[i])); ++ zone = page_zone(p); ++ ++ zone_page_state_add(1, zone, NR_SLAB_RECLAIMABLE); ++ } ++ } ++ ++ kbase_process_page_usage_dec(kctx, alloc->nents); ++ new_page_count = kbase_atomic_sub_pages(alloc->nents, ++ &kctx->used_pages); ++ kbase_atomic_sub_pages(alloc->nents, &kctx->kbdev->memdev.used_pages); ++ ++ KBASE_TLSTREAM_AUX_PAGESALLOC( ++ kctx->id, ++ (u64)new_page_count); ++} ++ ++/** ++ * kbase_mem_evictable_unmark_reclaim - Mark the pages as no longer reclaimable. ++ * @alloc: The physical allocation ++ */ ++static ++void kbase_mem_evictable_unmark_reclaim(struct kbase_mem_phy_alloc *alloc) ++{ ++ struct kbase_context *kctx = alloc->imported.kctx; ++ struct kbase_mem_zone_cache_entry *zone_cache; ++ int __maybe_unused new_page_count; ++ int err; ++ ++ new_page_count = kbase_atomic_add_pages(alloc->nents, ++ &kctx->used_pages); ++ kbase_atomic_add_pages(alloc->nents, &kctx->kbdev->memdev.used_pages); ++ ++ /* Increase mm counters so that the allocation is accounted for ++ * against the process and thus is visible to the OOM killer, ++ * then remove it from the reclaimable accounting. */ ++ kbase_process_page_usage_inc(kctx, alloc->nents); ++ ++ /* Attempt to build a zone cache of tracking */ ++ err = kbase_zone_cache_build(alloc); ++ if (err == 0) { ++ /* Bulk update all the zones */ ++ list_for_each_entry(zone_cache, &alloc->zone_cache, zone_node) { ++ zone_page_state_add(-zone_cache->count, ++ zone_cache->zone, NR_SLAB_RECLAIMABLE); ++ } ++ } else { ++ /* Fall-back to page by page updates */ ++ int i; ++ ++ for (i = 0; i < alloc->nents; i++) { ++ struct page *p; ++ struct zone *zone; ++ ++ p = phys_to_page(as_phys_addr_t(alloc->pages[i])); ++ zone = page_zone(p); ++ zone_page_state_add(-1, zone, NR_SLAB_RECLAIMABLE); ++ } ++ } ++ ++ KBASE_TLSTREAM_AUX_PAGESALLOC( ++ kctx->id, ++ (u64)new_page_count); ++} ++ ++int kbase_mem_evictable_make(struct kbase_mem_phy_alloc *gpu_alloc) ++{ ++ struct kbase_context *kctx = gpu_alloc->imported.kctx; ++ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ /* This alloction can't already be on a list. */ ++ WARN_ON(!list_empty(&gpu_alloc->evict_node)); ++ ++ kbase_mem_shrink_cpu_mapping(kctx, gpu_alloc->reg, ++ 0, gpu_alloc->nents); ++ ++ /* ++ * Add the allocation to the eviction list, after this point the shrink ++ * can reclaim it. ++ */ ++ mutex_lock(&kctx->jit_evict_lock); ++ list_add(&gpu_alloc->evict_node, &kctx->evict_list); ++ mutex_unlock(&kctx->jit_evict_lock); ++ kbase_mem_evictable_mark_reclaim(gpu_alloc); ++ ++ gpu_alloc->reg->flags |= KBASE_REG_DONT_NEED; ++ return 0; ++} ++ ++bool kbase_mem_evictable_unmake(struct kbase_mem_phy_alloc *gpu_alloc) ++{ ++ struct kbase_context *kctx = gpu_alloc->imported.kctx; ++ int err = 0; ++ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ /* ++ * First remove the allocation from the eviction list as it's no ++ * longer eligible for eviction. ++ */ ++ list_del_init(&gpu_alloc->evict_node); ++ ++ if (gpu_alloc->evicted == 0) { ++ /* ++ * The backing is still present, update the VM stats as it's ++ * in use again. ++ */ ++ kbase_mem_evictable_unmark_reclaim(gpu_alloc); ++ } else { ++ /* If the region is still alive ... */ ++ if (gpu_alloc->reg) { ++ /* ... allocate replacement backing ... */ ++ err = kbase_alloc_phy_pages_helper(gpu_alloc, ++ gpu_alloc->evicted); ++ ++ /* ++ * ... and grow the mapping back to its ++ * pre-eviction size. ++ */ ++ if (!err) ++ err = kbase_mem_grow_gpu_mapping(kctx, ++ gpu_alloc->reg, ++ gpu_alloc->evicted, 0); ++ ++ gpu_alloc->evicted = 0; ++ } ++ } ++ ++ /* If the region is still alive remove the DONT_NEED attribute. */ ++ if (gpu_alloc->reg) ++ gpu_alloc->reg->flags &= ~KBASE_REG_DONT_NEED; ++ ++ return (err == 0); ++} ++ ++int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, unsigned int flags, unsigned int mask) ++{ ++ struct kbase_va_region *reg; ++ int ret = -EINVAL; ++ unsigned int real_flags = 0; ++ unsigned int prev_flags = 0; ++ bool prev_needed, new_needed; ++ ++ KBASE_DEBUG_ASSERT(kctx); ++ ++ if (!gpu_addr) ++ return -EINVAL; ++ ++ if ((gpu_addr & ~PAGE_MASK) && (gpu_addr >= PAGE_SIZE)) ++ return -EINVAL; ++ ++ /* nuke other bits */ ++ flags &= mask; ++ ++ /* check for only supported flags */ ++ if (flags & ~(BASE_MEM_FLAGS_MODIFIABLE)) ++ goto out; ++ ++ /* mask covers bits we don't support? */ ++ if (mask & ~(BASE_MEM_FLAGS_MODIFIABLE)) ++ goto out; ++ ++ /* convert flags */ ++ if (BASE_MEM_COHERENT_SYSTEM & flags) ++ real_flags |= KBASE_REG_SHARE_BOTH; ++ else if (BASE_MEM_COHERENT_LOCAL & flags) ++ real_flags |= KBASE_REG_SHARE_IN; ++ ++ /* now we can lock down the context, and find the region */ ++ down_write(¤t->mm->mmap_sem); ++ kbase_gpu_vm_lock(kctx); ++ ++ /* Validate the region */ ++ reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr); ++ if (!reg || (reg->flags & KBASE_REG_FREE)) ++ goto out_unlock; ++ ++ /* Is the region being transitioning between not needed and needed? */ ++ prev_needed = (KBASE_REG_DONT_NEED & reg->flags) == KBASE_REG_DONT_NEED; ++ new_needed = (BASE_MEM_DONT_NEED & flags) == BASE_MEM_DONT_NEED; ++ if (prev_needed != new_needed) { ++ /* Aliased allocations can't be made ephemeral */ ++ if (atomic_read(®->cpu_alloc->gpu_mappings) > 1) ++ goto out_unlock; ++ ++ if (new_needed) { ++ /* Only native allocations can be marked not needed */ ++ if (reg->cpu_alloc->type != KBASE_MEM_TYPE_NATIVE) { ++ ret = -EINVAL; ++ goto out_unlock; ++ } ++ ret = kbase_mem_evictable_make(reg->gpu_alloc); ++ if (ret) ++ goto out_unlock; ++ } else { ++ kbase_mem_evictable_unmake(reg->gpu_alloc); ++ } ++ } ++ ++ /* limit to imported memory */ ++ if ((reg->gpu_alloc->type != KBASE_MEM_TYPE_IMPORTED_UMP) && ++ (reg->gpu_alloc->type != KBASE_MEM_TYPE_IMPORTED_UMM)) ++ goto out_unlock; ++ ++ /* no change? */ ++ if (real_flags == (reg->flags & (KBASE_REG_SHARE_IN | KBASE_REG_SHARE_BOTH))) { ++ ret = 0; ++ goto out_unlock; ++ } ++ ++ /* save for roll back */ ++ prev_flags = reg->flags; ++ reg->flags &= ~(KBASE_REG_SHARE_IN | KBASE_REG_SHARE_BOTH); ++ reg->flags |= real_flags; ++ ++ /* Currently supporting only imported memory */ ++ switch (reg->gpu_alloc->type) { ++#ifdef CONFIG_UMP ++ case KBASE_MEM_TYPE_IMPORTED_UMP: ++ ret = kbase_mmu_update_pages(kctx, reg->start_pfn, ++ kbase_get_gpu_phy_pages(reg), ++ reg->gpu_alloc->nents, reg->flags); ++ break; ++#endif ++#ifdef CONFIG_DMA_SHARED_BUFFER ++ case KBASE_MEM_TYPE_IMPORTED_UMM: ++ /* Future use will use the new flags, existing mapping will NOT be updated ++ * as memory should not be in use by the GPU when updating the flags. ++ */ ++ ret = 0; ++ WARN_ON(reg->gpu_alloc->imported.umm.current_mapping_usage_count); ++ break; ++#endif ++ default: ++ break; ++ } ++ ++ /* roll back on error, i.e. not UMP */ ++ if (ret) ++ reg->flags = prev_flags; ++ ++out_unlock: ++ kbase_gpu_vm_unlock(kctx); ++ up_write(¤t->mm->mmap_sem); ++out: ++ return ret; ++} ++ ++#define KBASE_MEM_IMPORT_HAVE_PAGES (1UL << BASE_MEM_FLAGS_NR_BITS) ++ ++#ifdef CONFIG_UMP ++static struct kbase_va_region *kbase_mem_from_ump(struct kbase_context *kctx, ump_secure_id id, u64 *va_pages, u64 *flags) ++{ ++ struct kbase_va_region *reg; ++ ump_dd_handle umph; ++ u64 block_count; ++ const ump_dd_physical_block_64 *block_array; ++ u64 i, j; ++ int page = 0; ++ ump_alloc_flags ump_flags; ++ ump_alloc_flags cpu_flags; ++ ump_alloc_flags gpu_flags; ++ ++ if (*flags & BASE_MEM_SECURE) ++ goto bad_flags; ++ ++ umph = ump_dd_from_secure_id(id); ++ if (UMP_DD_INVALID_MEMORY_HANDLE == umph) ++ goto bad_id; ++ ++ ump_flags = ump_dd_allocation_flags_get(umph); ++ cpu_flags = (ump_flags >> UMP_DEVICE_CPU_SHIFT) & UMP_DEVICE_MASK; ++ gpu_flags = (ump_flags >> DEFAULT_UMP_GPU_DEVICE_SHIFT) & ++ UMP_DEVICE_MASK; ++ ++ *va_pages = ump_dd_size_get_64(umph); ++ *va_pages >>= PAGE_SHIFT; ++ ++ if (!*va_pages) ++ goto bad_size; ++ ++ if (*va_pages > (U64_MAX / PAGE_SIZE)) ++ /* 64-bit address range is the max */ ++ goto bad_size; ++ ++ if (*flags & BASE_MEM_SAME_VA) ++ reg = kbase_alloc_free_region(kctx, 0, *va_pages, KBASE_REG_ZONE_SAME_VA); ++ else ++ reg = kbase_alloc_free_region(kctx, 0, *va_pages, KBASE_REG_ZONE_CUSTOM_VA); ++ ++ if (!reg) ++ goto no_region; ++ ++ /* we've got pages to map now, and support SAME_VA */ ++ *flags |= KBASE_MEM_IMPORT_HAVE_PAGES; ++ ++ reg->gpu_alloc = kbase_alloc_create(*va_pages, KBASE_MEM_TYPE_IMPORTED_UMP); ++ if (IS_ERR_OR_NULL(reg->gpu_alloc)) ++ goto no_alloc_obj; ++ ++ reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); ++ ++ reg->gpu_alloc->imported.ump_handle = umph; ++ ++ reg->flags &= ~KBASE_REG_FREE; ++ reg->flags |= KBASE_REG_GPU_NX; /* UMP is always No eXecute */ ++ reg->flags &= ~KBASE_REG_GROWABLE; /* UMP cannot be grown */ ++ ++ /* Override import flags based on UMP flags */ ++ *flags &= ~(BASE_MEM_CACHED_CPU); ++ *flags &= ~(BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_CPU_WR); ++ *flags &= ~(BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR); ++ ++ if ((cpu_flags & (UMP_HINT_DEVICE_RD | UMP_HINT_DEVICE_WR)) == ++ (UMP_HINT_DEVICE_RD | UMP_HINT_DEVICE_WR)) { ++ reg->flags |= KBASE_REG_CPU_CACHED; ++ *flags |= BASE_MEM_CACHED_CPU; ++ } ++ ++ if (cpu_flags & UMP_PROT_CPU_WR) { ++ reg->flags |= KBASE_REG_CPU_WR; ++ *flags |= BASE_MEM_PROT_CPU_WR; ++ } ++ ++ if (cpu_flags & UMP_PROT_CPU_RD) { ++ reg->flags |= KBASE_REG_CPU_RD; ++ *flags |= BASE_MEM_PROT_CPU_RD; ++ } ++ ++ if ((gpu_flags & (UMP_HINT_DEVICE_RD | UMP_HINT_DEVICE_WR)) == ++ (UMP_HINT_DEVICE_RD | UMP_HINT_DEVICE_WR)) ++ reg->flags |= KBASE_REG_GPU_CACHED; ++ ++ if (gpu_flags & UMP_PROT_DEVICE_WR) { ++ reg->flags |= KBASE_REG_GPU_WR; ++ *flags |= BASE_MEM_PROT_GPU_WR; ++ } ++ ++ if (gpu_flags & UMP_PROT_DEVICE_RD) { ++ reg->flags |= KBASE_REG_GPU_RD; ++ *flags |= BASE_MEM_PROT_GPU_RD; ++ } ++ ++ /* ump phys block query */ ++ ump_dd_phys_blocks_get_64(umph, &block_count, &block_array); ++ ++ for (i = 0; i < block_count; i++) { ++ for (j = 0; j < (block_array[i].size >> PAGE_SHIFT); j++) { ++ struct tagged_addr tagged; ++ ++ tagged = as_tagged(block_array[i].addr + ++ (j << PAGE_SHIFT)); ++ reg->gpu_alloc->pages[page] = tagged; ++ page++; ++ } ++ } ++ reg->gpu_alloc->nents = *va_pages; ++ reg->extent = 0; ++ ++ return reg; ++ ++no_alloc_obj: ++ kfree(reg); ++no_region: ++bad_size: ++ ump_dd_release(umph); ++bad_id: ++bad_flags: ++ return NULL; ++} ++#endif /* CONFIG_UMP */ ++ ++#ifdef CONFIG_DMA_SHARED_BUFFER ++static struct kbase_va_region *kbase_mem_from_umm(struct kbase_context *kctx, ++ int fd, u64 *va_pages, u64 *flags, u32 padding) ++{ ++ struct kbase_va_region *reg; ++ struct dma_buf *dma_buf; ++ struct dma_buf_attachment *dma_attachment; ++ bool shared_zone = false; ++ ++ dma_buf = dma_buf_get(fd); ++ if (IS_ERR_OR_NULL(dma_buf)) ++ goto no_buf; ++ ++ dma_attachment = dma_buf_attach(dma_buf, kctx->kbdev->dev); ++ if (!dma_attachment) ++ goto no_attachment; ++ ++ *va_pages = (PAGE_ALIGN(dma_buf->size) >> PAGE_SHIFT) + padding; ++ if (!*va_pages) ++ goto bad_size; ++ ++ if (*va_pages > (U64_MAX / PAGE_SIZE)) ++ /* 64-bit address range is the max */ ++ goto bad_size; ++ ++ /* ignore SAME_VA */ ++ *flags &= ~BASE_MEM_SAME_VA; ++ ++ if (*flags & BASE_MEM_IMPORT_SHARED) ++ shared_zone = true; ++ ++#ifdef CONFIG_64BIT ++ if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { ++ /* ++ * 64-bit tasks require us to reserve VA on the CPU that we use ++ * on the GPU. ++ */ ++ shared_zone = true; ++ } ++#endif ++ ++ if (shared_zone) { ++ *flags |= BASE_MEM_NEED_MMAP; ++ reg = kbase_alloc_free_region(kctx, 0, *va_pages, KBASE_REG_ZONE_SAME_VA); ++ } else { ++ reg = kbase_alloc_free_region(kctx, 0, *va_pages, KBASE_REG_ZONE_CUSTOM_VA); ++ } ++ ++ if (!reg) ++ goto no_region; ++ ++ reg->gpu_alloc = kbase_alloc_create(*va_pages, KBASE_MEM_TYPE_IMPORTED_UMM); ++ if (IS_ERR_OR_NULL(reg->gpu_alloc)) ++ goto no_alloc_obj; ++ ++ reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); ++ ++ /* No pages to map yet */ ++ reg->gpu_alloc->nents = 0; ++ ++ if (kbase_update_region_flags(kctx, reg, *flags) != 0) ++ goto invalid_flags; ++ ++ reg->flags &= ~KBASE_REG_FREE; ++ reg->flags |= KBASE_REG_GPU_NX; /* UMM is always No eXecute */ ++ reg->flags &= ~KBASE_REG_GROWABLE; /* UMM cannot be grown */ ++ reg->flags |= KBASE_REG_GPU_CACHED; ++ ++ if (*flags & BASE_MEM_SECURE) ++ reg->flags |= KBASE_REG_SECURE; ++ ++ if (padding) ++ reg->flags |= KBASE_REG_IMPORT_PAD; ++ ++ reg->gpu_alloc->type = KBASE_MEM_TYPE_IMPORTED_UMM; ++ reg->gpu_alloc->imported.umm.sgt = NULL; ++ reg->gpu_alloc->imported.umm.dma_buf = dma_buf; ++ reg->gpu_alloc->imported.umm.dma_attachment = dma_attachment; ++ reg->gpu_alloc->imported.umm.current_mapping_usage_count = 0; ++ reg->extent = 0; ++ ++ return reg; ++ ++invalid_flags: ++ kbase_mem_phy_alloc_put(reg->gpu_alloc); ++no_alloc_obj: ++ kfree(reg); ++no_region: ++bad_size: ++ dma_buf_detach(dma_buf, dma_attachment); ++no_attachment: ++ dma_buf_put(dma_buf); ++no_buf: ++ return NULL; ++} ++#endif /* CONFIG_DMA_SHARED_BUFFER */ ++ ++static u32 kbase_get_cache_line_alignment(struct kbase_context *kctx) ++{ ++ u32 cpu_cache_line_size = cache_line_size(); ++ u32 gpu_cache_line_size = ++ (1UL << kctx->kbdev->gpu_props.props.l2_props.log2_line_size); ++ ++ return ((cpu_cache_line_size > gpu_cache_line_size) ? ++ cpu_cache_line_size : ++ gpu_cache_line_size); ++} ++ ++static struct kbase_va_region *kbase_mem_from_user_buffer( ++ struct kbase_context *kctx, unsigned long address, ++ unsigned long size, u64 *va_pages, u64 *flags) ++{ ++ long i; ++ struct kbase_va_region *reg; ++ long faulted_pages; ++ int zone = KBASE_REG_ZONE_CUSTOM_VA; ++ bool shared_zone = false; ++ u32 cache_line_alignment = kbase_get_cache_line_alignment(kctx); ++ struct kbase_alloc_import_user_buf *user_buf; ++ struct page **pages = NULL; ++ ++ if ((address & (cache_line_alignment - 1)) != 0 || ++ (size & (cache_line_alignment - 1)) != 0) { ++ /* Coherency must be enabled to handle partial cache lines */ ++ if (*flags & (BASE_MEM_COHERENT_SYSTEM | ++ BASE_MEM_COHERENT_SYSTEM_REQUIRED)) { ++ /* Force coherent system required flag, import will ++ * then fail if coherency isn't available ++ */ ++ *flags |= BASE_MEM_COHERENT_SYSTEM_REQUIRED; ++ } else { ++ dev_warn(kctx->kbdev->dev, ++ "User buffer is not cache line aligned and no coherency enabled\n"); ++ goto bad_size; ++ } ++ } ++ ++ *va_pages = (PAGE_ALIGN(address + size) >> PAGE_SHIFT) - ++ PFN_DOWN(address); ++ if (!*va_pages) ++ goto bad_size; ++ ++ if (*va_pages > (UINT64_MAX / PAGE_SIZE)) ++ /* 64-bit address range is the max */ ++ goto bad_size; ++ ++ /* SAME_VA generally not supported with imported memory (no known use cases) */ ++ *flags &= ~BASE_MEM_SAME_VA; ++ ++ if (*flags & BASE_MEM_IMPORT_SHARED) ++ shared_zone = true; ++ ++#ifdef CONFIG_64BIT ++ if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { ++ /* ++ * 64-bit tasks require us to reserve VA on the CPU that we use ++ * on the GPU. ++ */ ++ shared_zone = true; ++ } ++#endif ++ ++ if (shared_zone) { ++ *flags |= BASE_MEM_NEED_MMAP; ++ zone = KBASE_REG_ZONE_SAME_VA; ++ } ++ ++ reg = kbase_alloc_free_region(kctx, 0, *va_pages, zone); ++ ++ if (!reg) ++ goto no_region; ++ ++ reg->gpu_alloc = kbase_alloc_create(*va_pages, ++ KBASE_MEM_TYPE_IMPORTED_USER_BUF); ++ if (IS_ERR_OR_NULL(reg->gpu_alloc)) ++ goto no_alloc_obj; ++ ++ reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); ++ ++ if (kbase_update_region_flags(kctx, reg, *flags) != 0) ++ goto invalid_flags; ++ ++ reg->flags &= ~KBASE_REG_FREE; ++ reg->flags |= KBASE_REG_GPU_NX; /* User-buffers are always No eXecute */ ++ reg->flags &= ~KBASE_REG_GROWABLE; /* Cannot be grown */ ++ ++ user_buf = ®->gpu_alloc->imported.user_buf; ++ ++ user_buf->size = size; ++ user_buf->address = address; ++ user_buf->nr_pages = *va_pages; ++ user_buf->mm = current->mm; ++ user_buf->pages = kmalloc_array(*va_pages, sizeof(struct page *), ++ GFP_KERNEL); ++ ++ if (!user_buf->pages) ++ goto no_page_array; ++ ++ /* If the region is coherent with the CPU then the memory is imported ++ * and mapped onto the GPU immediately. ++ * Otherwise get_user_pages is called as a sanity check, but with ++ * NULL as the pages argument which will fault the pages, but not ++ * pin them. The memory will then be pinned only around the jobs that ++ * specify the region as an external resource. ++ */ ++ if (reg->flags & KBASE_REG_SHARE_BOTH) { ++ pages = user_buf->pages; ++ *flags |= KBASE_MEM_IMPORT_HAVE_PAGES; ++ } ++ ++ down_read(¤t->mm->mmap_sem); ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) ++ faulted_pages = get_user_pages(current, current->mm, address, *va_pages, ++ reg->flags & KBASE_REG_GPU_WR, 0, pages, NULL); ++#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) ++ faulted_pages = get_user_pages(address, *va_pages, ++ reg->flags & KBASE_REG_GPU_WR, 0, pages, NULL); ++#else ++ faulted_pages = get_user_pages(address, *va_pages, ++ reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0, ++ pages, NULL); ++#endif ++ ++ up_read(¤t->mm->mmap_sem); ++ ++ if (faulted_pages != *va_pages) ++ goto fault_mismatch; ++ ++ atomic_inc(¤t->mm->mm_count); ++ ++ reg->gpu_alloc->nents = 0; ++ reg->extent = 0; ++ ++ if (pages) { ++ struct device *dev = kctx->kbdev->dev; ++ unsigned long local_size = user_buf->size; ++ unsigned long offset = user_buf->address & ~PAGE_MASK; ++ struct tagged_addr *pa = kbase_get_gpu_phy_pages(reg); ++ ++ /* Top bit signifies that this was pinned on import */ ++ user_buf->current_mapping_usage_count |= PINNED_ON_IMPORT; ++ ++ for (i = 0; i < faulted_pages; i++) { ++ dma_addr_t dma_addr; ++ unsigned long min; ++ ++ min = MIN(PAGE_SIZE - offset, local_size); ++ dma_addr = dma_map_page(dev, pages[i], ++ offset, min, ++ DMA_BIDIRECTIONAL); ++ if (dma_mapping_error(dev, dma_addr)) ++ goto unwind_dma_map; ++ ++ user_buf->dma_addrs[i] = dma_addr; ++ pa[i] = as_tagged(page_to_phys(pages[i])); ++ ++ local_size -= min; ++ offset = 0; ++ } ++ ++ reg->gpu_alloc->nents = faulted_pages; ++ } ++ ++ return reg; ++ ++unwind_dma_map: ++ while (i--) { ++ dma_unmap_page(kctx->kbdev->dev, ++ user_buf->dma_addrs[i], ++ PAGE_SIZE, DMA_BIDIRECTIONAL); ++ } ++fault_mismatch: ++ if (pages) { ++ for (i = 0; i < faulted_pages; i++) ++ put_page(pages[i]); ++ } ++ kfree(user_buf->pages); ++no_page_array: ++invalid_flags: ++ kbase_mem_phy_alloc_put(reg->cpu_alloc); ++ kbase_mem_phy_alloc_put(reg->gpu_alloc); ++no_alloc_obj: ++ kfree(reg); ++no_region: ++bad_size: ++ return NULL; ++ ++} ++ ++ ++u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, ++ u64 nents, struct base_mem_aliasing_info *ai, ++ u64 *num_pages) ++{ ++ struct kbase_va_region *reg; ++ u64 gpu_va; ++ size_t i; ++ bool coherent; ++ ++ KBASE_DEBUG_ASSERT(kctx); ++ KBASE_DEBUG_ASSERT(flags); ++ KBASE_DEBUG_ASSERT(ai); ++ KBASE_DEBUG_ASSERT(num_pages); ++ ++ /* mask to only allowed flags */ ++ *flags &= (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR | ++ BASE_MEM_COHERENT_SYSTEM | BASE_MEM_COHERENT_LOCAL | ++ BASE_MEM_COHERENT_SYSTEM_REQUIRED); ++ ++ if (!(*flags & (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR))) { ++ dev_warn(kctx->kbdev->dev, ++ "kbase_mem_alias called with bad flags (%llx)", ++ (unsigned long long)*flags); ++ goto bad_flags; ++ } ++ coherent = (*flags & BASE_MEM_COHERENT_SYSTEM) != 0 || ++ (*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0; ++ ++ if (!stride) ++ goto bad_stride; ++ ++ if (!nents) ++ goto bad_nents; ++ ++ if ((nents * stride) > (U64_MAX / PAGE_SIZE)) ++ /* 64-bit address range is the max */ ++ goto bad_size; ++ ++ /* calculate the number of pages this alias will cover */ ++ *num_pages = nents * stride; ++ ++#ifdef CONFIG_64BIT ++ if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { ++ /* 64-bit tasks must MMAP anyway, but not expose this address to ++ * clients */ ++ *flags |= BASE_MEM_NEED_MMAP; ++ reg = kbase_alloc_free_region(kctx, 0, *num_pages, ++ KBASE_REG_ZONE_SAME_VA); ++ } else { ++#else ++ if (1) { ++#endif ++ reg = kbase_alloc_free_region(kctx, 0, *num_pages, ++ KBASE_REG_ZONE_CUSTOM_VA); ++ } ++ ++ if (!reg) ++ goto no_reg; ++ ++ /* zero-sized page array, as we don't need one/can support one */ ++ reg->gpu_alloc = kbase_alloc_create(0, KBASE_MEM_TYPE_ALIAS); ++ if (IS_ERR_OR_NULL(reg->gpu_alloc)) ++ goto no_alloc_obj; ++ ++ reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); ++ ++ if (kbase_update_region_flags(kctx, reg, *flags) != 0) ++ goto invalid_flags; ++ ++ reg->gpu_alloc->imported.alias.nents = nents; ++ reg->gpu_alloc->imported.alias.stride = stride; ++ reg->gpu_alloc->imported.alias.aliased = vzalloc(sizeof(*reg->gpu_alloc->imported.alias.aliased) * nents); ++ if (!reg->gpu_alloc->imported.alias.aliased) ++ goto no_aliased_array; ++ ++ kbase_gpu_vm_lock(kctx); ++ ++ /* validate and add src handles */ ++ for (i = 0; i < nents; i++) { ++ if (ai[i].handle.basep.handle < BASE_MEM_FIRST_FREE_ADDRESS) { ++ if (ai[i].handle.basep.handle != ++ BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE) ++ goto bad_handle; /* unsupported magic handle */ ++ if (!ai[i].length) ++ goto bad_handle; /* must be > 0 */ ++ if (ai[i].length > stride) ++ goto bad_handle; /* can't be larger than the ++ stride */ ++ reg->gpu_alloc->imported.alias.aliased[i].length = ai[i].length; ++ } else { ++ struct kbase_va_region *aliasing_reg; ++ struct kbase_mem_phy_alloc *alloc; ++ ++ aliasing_reg = kbase_region_tracker_find_region_base_address( ++ kctx, ++ (ai[i].handle.basep.handle >> PAGE_SHIFT) << PAGE_SHIFT); ++ ++ /* validate found region */ ++ if (!aliasing_reg) ++ goto bad_handle; /* Not found */ ++ if (aliasing_reg->flags & KBASE_REG_FREE) ++ goto bad_handle; /* Free region */ ++ if (aliasing_reg->flags & KBASE_REG_DONT_NEED) ++ goto bad_handle; /* Ephemeral region */ ++ if (!aliasing_reg->gpu_alloc) ++ goto bad_handle; /* No alloc */ ++ if (aliasing_reg->gpu_alloc->type != KBASE_MEM_TYPE_NATIVE) ++ goto bad_handle; /* Not a native alloc */ ++ if (coherent != ((aliasing_reg->flags & KBASE_REG_SHARE_BOTH) != 0)) ++ goto bad_handle; ++ /* Non-coherent memory cannot alias ++ coherent memory, and vice versa.*/ ++ ++ /* check size against stride */ ++ if (!ai[i].length) ++ goto bad_handle; /* must be > 0 */ ++ if (ai[i].length > stride) ++ goto bad_handle; /* can't be larger than the ++ stride */ ++ ++ alloc = aliasing_reg->gpu_alloc; ++ ++ /* check against the alloc's size */ ++ if (ai[i].offset > alloc->nents) ++ goto bad_handle; /* beyond end */ ++ if (ai[i].offset + ai[i].length > alloc->nents) ++ goto bad_handle; /* beyond end */ ++ ++ reg->gpu_alloc->imported.alias.aliased[i].alloc = kbase_mem_phy_alloc_get(alloc); ++ reg->gpu_alloc->imported.alias.aliased[i].length = ai[i].length; ++ reg->gpu_alloc->imported.alias.aliased[i].offset = ai[i].offset; ++ } ++ } ++ ++#ifdef CONFIG_64BIT ++ if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { ++ /* Bind to a cookie */ ++ if (!kctx->cookies) { ++ dev_err(kctx->kbdev->dev, "No cookies available for allocation!"); ++ goto no_cookie; ++ } ++ /* return a cookie */ ++ gpu_va = __ffs(kctx->cookies); ++ kctx->cookies &= ~(1UL << gpu_va); ++ BUG_ON(kctx->pending_regions[gpu_va]); ++ kctx->pending_regions[gpu_va] = reg; ++ ++ /* relocate to correct base */ ++ gpu_va += PFN_DOWN(BASE_MEM_COOKIE_BASE); ++ gpu_va <<= PAGE_SHIFT; ++ } else /* we control the VA */ { ++#else ++ if (1) { ++#endif ++ if (kbase_gpu_mmap(kctx, reg, 0, *num_pages, 1) != 0) { ++ dev_warn(kctx->kbdev->dev, "Failed to map memory on GPU"); ++ goto no_mmap; ++ } ++ /* return real GPU VA */ ++ gpu_va = reg->start_pfn << PAGE_SHIFT; ++ } ++ ++ reg->flags &= ~KBASE_REG_FREE; ++ reg->flags &= ~KBASE_REG_GROWABLE; ++ ++ kbase_gpu_vm_unlock(kctx); ++ ++ return gpu_va; ++ ++#ifdef CONFIG_64BIT ++no_cookie: ++#endif ++no_mmap: ++bad_handle: ++ kbase_gpu_vm_unlock(kctx); ++no_aliased_array: ++invalid_flags: ++ kbase_mem_phy_alloc_put(reg->cpu_alloc); ++ kbase_mem_phy_alloc_put(reg->gpu_alloc); ++no_alloc_obj: ++ kfree(reg); ++no_reg: ++bad_size: ++bad_nents: ++bad_stride: ++bad_flags: ++ return 0; ++} ++ ++int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type, ++ void __user *phandle, u32 padding, u64 *gpu_va, u64 *va_pages, ++ u64 *flags) ++{ ++ struct kbase_va_region *reg; ++ ++ KBASE_DEBUG_ASSERT(kctx); ++ KBASE_DEBUG_ASSERT(gpu_va); ++ KBASE_DEBUG_ASSERT(va_pages); ++ KBASE_DEBUG_ASSERT(flags); ++ ++#ifdef CONFIG_64BIT ++ if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) ++ *flags |= BASE_MEM_SAME_VA; ++#endif ++ ++ if (!kbase_check_import_flags(*flags)) { ++ dev_warn(kctx->kbdev->dev, ++ "kbase_mem_import called with bad flags (%llx)", ++ (unsigned long long)*flags); ++ goto bad_flags; ++ } ++ ++ if ((*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0 && ++ !kbase_device_is_cpu_coherent(kctx->kbdev)) { ++ dev_warn(kctx->kbdev->dev, ++ "kbase_mem_import call required coherent mem when unavailable"); ++ goto bad_flags; ++ } ++ if ((*flags & BASE_MEM_COHERENT_SYSTEM) != 0 && ++ !kbase_device_is_cpu_coherent(kctx->kbdev)) { ++ /* Remove COHERENT_SYSTEM flag if coherent mem is unavailable */ ++ *flags &= ~BASE_MEM_COHERENT_SYSTEM; ++ } ++ ++ if ((padding != 0) && (type != BASE_MEM_IMPORT_TYPE_UMM)) { ++ dev_warn(kctx->kbdev->dev, ++ "padding is only supported for UMM"); ++ goto bad_flags; ++ } ++ ++ switch (type) { ++#ifdef CONFIG_UMP ++ case BASE_MEM_IMPORT_TYPE_UMP: { ++ ump_secure_id id; ++ ++ if (get_user(id, (ump_secure_id __user *)phandle)) ++ reg = NULL; ++ else ++ reg = kbase_mem_from_ump(kctx, id, va_pages, flags); ++ } ++ break; ++#endif /* CONFIG_UMP */ ++#ifdef CONFIG_DMA_SHARED_BUFFER ++ case BASE_MEM_IMPORT_TYPE_UMM: { ++ int fd; ++ ++ if (get_user(fd, (int __user *)phandle)) ++ reg = NULL; ++ else ++ reg = kbase_mem_from_umm(kctx, fd, va_pages, flags, ++ padding); ++ } ++ break; ++#endif /* CONFIG_DMA_SHARED_BUFFER */ ++ case BASE_MEM_IMPORT_TYPE_USER_BUFFER: { ++ struct base_mem_import_user_buffer user_buffer; ++ void __user *uptr; ++ ++ if (copy_from_user(&user_buffer, phandle, ++ sizeof(user_buffer))) { ++ reg = NULL; ++ } else { ++#ifdef CONFIG_COMPAT ++ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) ++ uptr = compat_ptr(user_buffer.ptr); ++ else ++#endif ++ uptr = u64_to_user_ptr(user_buffer.ptr); ++ ++ reg = kbase_mem_from_user_buffer(kctx, ++ (unsigned long)uptr, user_buffer.length, ++ va_pages, flags); ++ } ++ break; ++ } ++ default: { ++ reg = NULL; ++ break; ++ } ++ } ++ ++ if (!reg) ++ goto no_reg; ++ ++ kbase_gpu_vm_lock(kctx); ++ ++ /* mmap needed to setup VA? */ ++ if (*flags & (BASE_MEM_SAME_VA | BASE_MEM_NEED_MMAP)) { ++ /* Bind to a cookie */ ++ if (!kctx->cookies) ++ goto no_cookie; ++ /* return a cookie */ ++ *gpu_va = __ffs(kctx->cookies); ++ kctx->cookies &= ~(1UL << *gpu_va); ++ BUG_ON(kctx->pending_regions[*gpu_va]); ++ kctx->pending_regions[*gpu_va] = reg; ++ ++ /* relocate to correct base */ ++ *gpu_va += PFN_DOWN(BASE_MEM_COOKIE_BASE); ++ *gpu_va <<= PAGE_SHIFT; ++ ++ } else if (*flags & KBASE_MEM_IMPORT_HAVE_PAGES) { ++ /* we control the VA, mmap now to the GPU */ ++ if (kbase_gpu_mmap(kctx, reg, 0, *va_pages, 1) != 0) ++ goto no_gpu_va; ++ /* return real GPU VA */ ++ *gpu_va = reg->start_pfn << PAGE_SHIFT; ++ } else { ++ /* we control the VA, but nothing to mmap yet */ ++ if (kbase_add_va_region(kctx, reg, 0, *va_pages, 1) != 0) ++ goto no_gpu_va; ++ /* return real GPU VA */ ++ *gpu_va = reg->start_pfn << PAGE_SHIFT; ++ } ++ ++ /* clear out private flags */ ++ *flags &= ((1UL << BASE_MEM_FLAGS_NR_BITS) - 1); ++ ++ kbase_gpu_vm_unlock(kctx); ++ ++ return 0; ++ ++no_gpu_va: ++no_cookie: ++ kbase_gpu_vm_unlock(kctx); ++ kbase_mem_phy_alloc_put(reg->cpu_alloc); ++ kbase_mem_phy_alloc_put(reg->gpu_alloc); ++ kfree(reg); ++no_reg: ++bad_flags: ++ *gpu_va = 0; ++ *va_pages = 0; ++ *flags = 0; ++ return -ENOMEM; ++} ++ ++int kbase_mem_grow_gpu_mapping(struct kbase_context *kctx, ++ struct kbase_va_region *reg, ++ u64 new_pages, u64 old_pages) ++{ ++ struct tagged_addr *phy_pages; ++ u64 delta = new_pages - old_pages; ++ int ret = 0; ++ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ /* Map the new pages into the GPU */ ++ phy_pages = kbase_get_gpu_phy_pages(reg); ++ ret = kbase_mmu_insert_pages(kctx, reg->start_pfn + old_pages, ++ phy_pages + old_pages, delta, reg->flags); ++ ++ return ret; ++} ++ ++static void kbase_mem_shrink_cpu_mapping(struct kbase_context *kctx, ++ struct kbase_va_region *reg, ++ u64 new_pages, u64 old_pages) ++{ ++ u64 gpu_va_start = reg->start_pfn; ++ ++ if (new_pages == old_pages) ++ /* Nothing to do */ ++ return; ++ ++ unmap_mapping_range(kctx->filp->f_inode->i_mapping, ++ (gpu_va_start + new_pages)<start_pfn + new_pages, delta); ++ ++ return ret; ++} ++ ++int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages) ++{ ++ u64 old_pages; ++ u64 delta; ++ int res = -EINVAL; ++ struct kbase_va_region *reg; ++ bool read_locked = false; ++ ++ KBASE_DEBUG_ASSERT(kctx); ++ KBASE_DEBUG_ASSERT(gpu_addr != 0); ++ ++ if (gpu_addr & ~PAGE_MASK) { ++ dev_warn(kctx->kbdev->dev, "kbase:mem_commit: gpu_addr: passed parameter is invalid"); ++ return -EINVAL; ++ } ++ ++ down_write(¤t->mm->mmap_sem); ++ kbase_gpu_vm_lock(kctx); ++ ++ /* Validate the region */ ++ reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr); ++ if (!reg || (reg->flags & KBASE_REG_FREE)) ++ goto out_unlock; ++ ++ KBASE_DEBUG_ASSERT(reg->cpu_alloc); ++ KBASE_DEBUG_ASSERT(reg->gpu_alloc); ++ ++ if (reg->gpu_alloc->type != KBASE_MEM_TYPE_NATIVE) ++ goto out_unlock; ++ ++ if (0 == (reg->flags & KBASE_REG_GROWABLE)) ++ goto out_unlock; ++ ++ /* Would overflow the VA region */ ++ if (new_pages > reg->nr_pages) ++ goto out_unlock; ++ ++ /* can't be mapped more than once on the GPU */ ++ if (atomic_read(®->gpu_alloc->gpu_mappings) > 1) ++ goto out_unlock; ++ /* can't grow regions which are ephemeral */ ++ if (reg->flags & KBASE_REG_DONT_NEED) ++ goto out_unlock; ++ ++ if (new_pages == reg->gpu_alloc->nents) { ++ /* no change */ ++ res = 0; ++ goto out_unlock; ++ } ++ ++ old_pages = kbase_reg_current_backed_size(reg); ++ if (new_pages > old_pages) { ++ delta = new_pages - old_pages; ++ ++ /* ++ * No update to the mm so downgrade the writer lock to a read ++ * lock so other readers aren't blocked after this point. ++ */ ++ downgrade_write(¤t->mm->mmap_sem); ++ read_locked = true; ++ ++ /* Allocate some more pages */ ++ if (kbase_alloc_phy_pages_helper(reg->cpu_alloc, delta) != 0) { ++ res = -ENOMEM; ++ goto out_unlock; ++ } ++ if (reg->cpu_alloc != reg->gpu_alloc) { ++ if (kbase_alloc_phy_pages_helper( ++ reg->gpu_alloc, delta) != 0) { ++ res = -ENOMEM; ++ kbase_free_phy_pages_helper(reg->cpu_alloc, ++ delta); ++ goto out_unlock; ++ } ++ } ++ ++ /* No update required for CPU mappings, that's done on fault. */ ++ ++ /* Update GPU mapping. */ ++ res = kbase_mem_grow_gpu_mapping(kctx, reg, ++ new_pages, old_pages); ++ ++ /* On error free the new pages */ ++ if (res) { ++ kbase_free_phy_pages_helper(reg->cpu_alloc, delta); ++ if (reg->cpu_alloc != reg->gpu_alloc) ++ kbase_free_phy_pages_helper(reg->gpu_alloc, ++ delta); ++ res = -ENOMEM; ++ goto out_unlock; ++ } ++ } else { ++ delta = old_pages - new_pages; ++ ++ /* Update all CPU mapping(s) */ ++ kbase_mem_shrink_cpu_mapping(kctx, reg, ++ new_pages, old_pages); ++ ++ /* Update the GPU mapping */ ++ res = kbase_mem_shrink_gpu_mapping(kctx, reg, ++ new_pages, old_pages); ++ if (res) { ++ res = -ENOMEM; ++ goto out_unlock; ++ } ++ ++ kbase_free_phy_pages_helper(reg->cpu_alloc, delta); ++ if (reg->cpu_alloc != reg->gpu_alloc) ++ kbase_free_phy_pages_helper(reg->gpu_alloc, delta); ++ } ++ ++out_unlock: ++ kbase_gpu_vm_unlock(kctx); ++ if (read_locked) ++ up_read(¤t->mm->mmap_sem); ++ else ++ up_write(¤t->mm->mmap_sem); ++ ++ return res; ++} ++ ++static void kbase_cpu_vm_open(struct vm_area_struct *vma) ++{ ++ struct kbase_cpu_mapping *map = vma->vm_private_data; ++ ++ KBASE_DEBUG_ASSERT(map); ++ KBASE_DEBUG_ASSERT(map->count > 0); ++ /* non-atomic as we're under Linux' mm lock */ ++ map->count++; ++} ++ ++static void kbase_cpu_vm_close(struct vm_area_struct *vma) ++{ ++ struct kbase_cpu_mapping *map = vma->vm_private_data; ++ ++ KBASE_DEBUG_ASSERT(map); ++ KBASE_DEBUG_ASSERT(map->count > 0); ++ ++ /* non-atomic as we're under Linux' mm lock */ ++ if (--map->count) ++ return; ++ ++ KBASE_DEBUG_ASSERT(map->kctx); ++ KBASE_DEBUG_ASSERT(map->alloc); ++ ++ kbase_gpu_vm_lock(map->kctx); ++ ++ if (map->free_on_close) { ++ KBASE_DEBUG_ASSERT((map->region->flags & KBASE_REG_ZONE_MASK) == ++ KBASE_REG_ZONE_SAME_VA); ++ /* Avoid freeing memory on the process death which results in ++ * GPU Page Fault. Memory will be freed in kbase_destroy_context ++ */ ++ if (!(current->flags & PF_EXITING)) ++ kbase_mem_free_region(map->kctx, map->region); ++ } ++ ++ list_del(&map->mappings_list); ++ ++ kbase_gpu_vm_unlock(map->kctx); ++ ++ kbase_mem_phy_alloc_put(map->alloc); ++ kfree(map); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_cpu_vm_close); ++ ++ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0)) ++static int kbase_cpu_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) ++{ ++#else ++static int kbase_cpu_vm_fault(struct vm_fault *vmf) ++{ ++ struct vm_area_struct *vma = vmf->vma; ++#endif ++ struct kbase_cpu_mapping *map = vma->vm_private_data; ++ pgoff_t rel_pgoff; ++ size_t i; ++ pgoff_t addr; ++ ++ KBASE_DEBUG_ASSERT(map); ++ KBASE_DEBUG_ASSERT(map->count > 0); ++ KBASE_DEBUG_ASSERT(map->kctx); ++ KBASE_DEBUG_ASSERT(map->alloc); ++ ++ rel_pgoff = vmf->pgoff - map->region->start_pfn; ++ ++ kbase_gpu_vm_lock(map->kctx); ++ if (rel_pgoff >= map->alloc->nents) ++ goto locked_bad_fault; ++ ++ /* Fault on access to DONT_NEED regions */ ++ if (map->alloc->reg && (map->alloc->reg->flags & KBASE_REG_DONT_NEED)) ++ goto locked_bad_fault; ++ ++ /* insert all valid pages from the fault location */ ++ i = rel_pgoff; ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ addr = (pgoff_t)((uintptr_t)vmf->virtual_address >> PAGE_SHIFT); ++#else ++ addr = (pgoff_t)(vmf->address >> PAGE_SHIFT); ++#endif ++ while (i < map->alloc->nents && (addr < vma->vm_end >> PAGE_SHIFT)) { ++ int ret = vm_insert_pfn(vma, addr << PAGE_SHIFT, ++ PFN_DOWN(as_phys_addr_t(map->alloc->pages[i]))); ++ if (ret < 0 && ret != -EBUSY) ++ goto locked_bad_fault; ++ ++ i++; addr++; ++ } ++ ++ kbase_gpu_vm_unlock(map->kctx); ++ /* we resolved it, nothing for VM to do */ ++ return VM_FAULT_NOPAGE; ++ ++locked_bad_fault: ++ kbase_gpu_vm_unlock(map->kctx); ++ return VM_FAULT_SIGBUS; ++} ++ ++const struct vm_operations_struct kbase_vm_ops = { ++ .open = kbase_cpu_vm_open, ++ .close = kbase_cpu_vm_close, ++ .fault = kbase_cpu_vm_fault ++}; ++ ++static int kbase_cpu_mmap(struct kbase_va_region *reg, struct vm_area_struct *vma, void *kaddr, size_t nr_pages, unsigned long aligned_offset, int free_on_close) ++{ ++ struct kbase_cpu_mapping *map; ++ struct tagged_addr *page_array; ++ int err = 0; ++ int i; ++ ++ map = kzalloc(sizeof(*map), GFP_KERNEL); ++ ++ if (!map) { ++ WARN_ON(1); ++ err = -ENOMEM; ++ goto out; ++ } ++ ++ /* ++ * VM_DONTCOPY - don't make this mapping available in fork'ed processes ++ * VM_DONTEXPAND - disable mremap on this region ++ * VM_IO - disables paging ++ * VM_DONTDUMP - Don't include in core dumps (3.7 only) ++ * VM_MIXEDMAP - Support mixing struct page*s and raw pfns. ++ * This is needed to support using the dedicated and ++ * the OS based memory backends together. ++ */ ++ /* ++ * This will need updating to propagate coherency flags ++ * See MIDBASE-1057 ++ */ ++ ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0)) ++ vma->vm_flags |= VM_DONTCOPY | VM_DONTDUMP | VM_DONTEXPAND | VM_IO; ++#else ++ vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_RESERVED | VM_IO; ++#endif ++ vma->vm_ops = &kbase_vm_ops; ++ vma->vm_private_data = map; ++ ++ page_array = kbase_get_cpu_phy_pages(reg); ++ ++ if (!(reg->flags & KBASE_REG_CPU_CACHED) && ++ (reg->flags & (KBASE_REG_CPU_WR|KBASE_REG_CPU_RD))) { ++ /* We can't map vmalloc'd memory uncached. ++ * Other memory will have been returned from ++ * kbase_mem_pool which would be ++ * suitable for mapping uncached. ++ */ ++ BUG_ON(kaddr); ++ vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); ++ } ++ ++ if (!kaddr) { ++ unsigned long addr = vma->vm_start + aligned_offset; ++ u64 start_off = vma->vm_pgoff - reg->start_pfn + ++ (aligned_offset>>PAGE_SHIFT); ++ ++ vma->vm_flags |= VM_PFNMAP; ++ for (i = 0; i < nr_pages; i++) { ++ phys_addr_t phys; ++ ++ phys = as_phys_addr_t(page_array[i + start_off]); ++ err = vm_insert_pfn(vma, addr, PFN_DOWN(phys)); ++ if (WARN_ON(err)) ++ break; ++ ++ addr += PAGE_SIZE; ++ } ++ } else { ++ WARN_ON(aligned_offset); ++ /* MIXEDMAP so we can vfree the kaddr early and not track it after map time */ ++ vma->vm_flags |= VM_MIXEDMAP; ++ /* vmalloc remaping is easy... */ ++ err = remap_vmalloc_range(vma, kaddr, 0); ++ WARN_ON(err); ++ } ++ ++ if (err) { ++ kfree(map); ++ goto out; ++ } ++ ++ map->region = reg; ++ map->free_on_close = free_on_close; ++ map->kctx = reg->kctx; ++ map->alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc); ++ map->count = 1; /* start with one ref */ ++ ++ if (reg->flags & KBASE_REG_CPU_CACHED) ++ map->alloc->properties |= KBASE_MEM_PHY_ALLOC_ACCESSED_CACHED; ++ ++ list_add(&map->mappings_list, &map->alloc->mappings); ++ ++ out: ++ return err; ++} ++ ++static int kbase_trace_buffer_mmap(struct kbase_context *kctx, struct vm_area_struct *vma, struct kbase_va_region **const reg, void **const kaddr) ++{ ++ struct kbase_va_region *new_reg; ++ u32 nr_pages; ++ size_t size; ++ int err = 0; ++ u32 *tb; ++ int owns_tb = 1; ++ ++ dev_dbg(kctx->kbdev->dev, "in %s\n", __func__); ++ size = (vma->vm_end - vma->vm_start); ++ nr_pages = size >> PAGE_SHIFT; ++ ++ if (!kctx->jctx.tb) { ++ KBASE_DEBUG_ASSERT(0 != size); ++ tb = vmalloc_user(size); ++ ++ if (NULL == tb) { ++ err = -ENOMEM; ++ goto out; ++ } ++ ++ err = kbase_device_trace_buffer_install(kctx, tb, size); ++ if (err) { ++ vfree(tb); ++ goto out; ++ } ++ } else { ++ err = -EINVAL; ++ goto out; ++ } ++ ++ *kaddr = kctx->jctx.tb; ++ ++ new_reg = kbase_alloc_free_region(kctx, 0, nr_pages, KBASE_REG_ZONE_SAME_VA); ++ if (!new_reg) { ++ err = -ENOMEM; ++ WARN_ON(1); ++ goto out_no_region; ++ } ++ ++ new_reg->cpu_alloc = kbase_alloc_create(0, KBASE_MEM_TYPE_TB); ++ if (IS_ERR_OR_NULL(new_reg->cpu_alloc)) { ++ err = -ENOMEM; ++ new_reg->cpu_alloc = NULL; ++ WARN_ON(1); ++ goto out_no_alloc; ++ } ++ ++ new_reg->gpu_alloc = kbase_mem_phy_alloc_get(new_reg->cpu_alloc); ++ ++ new_reg->cpu_alloc->imported.kctx = kctx; ++ new_reg->flags &= ~KBASE_REG_FREE; ++ new_reg->flags |= KBASE_REG_CPU_CACHED; ++ ++ /* alloc now owns the tb */ ++ owns_tb = 0; ++ ++ if (kbase_add_va_region(kctx, new_reg, vma->vm_start, nr_pages, 1) != 0) { ++ err = -ENOMEM; ++ WARN_ON(1); ++ goto out_no_va_region; ++ } ++ ++ *reg = new_reg; ++ ++ /* map read only, noexec */ ++ vma->vm_flags &= ~(VM_WRITE | VM_MAYWRITE | VM_EXEC | VM_MAYEXEC); ++ /* the rest of the flags is added by the cpu_mmap handler */ ++ ++ dev_dbg(kctx->kbdev->dev, "%s done\n", __func__); ++ return 0; ++ ++out_no_va_region: ++out_no_alloc: ++ kbase_free_alloced_region(new_reg); ++out_no_region: ++ if (owns_tb) { ++ kbase_device_trace_buffer_uninstall(kctx); ++ vfree(tb); ++ } ++out: ++ return err; ++} ++ ++static int kbase_mmu_dump_mmap(struct kbase_context *kctx, struct vm_area_struct *vma, struct kbase_va_region **const reg, void **const kmap_addr) ++{ ++ struct kbase_va_region *new_reg; ++ void *kaddr; ++ u32 nr_pages; ++ size_t size; ++ int err = 0; ++ ++ dev_dbg(kctx->kbdev->dev, "in kbase_mmu_dump_mmap\n"); ++ size = (vma->vm_end - vma->vm_start); ++ nr_pages = size >> PAGE_SHIFT; ++ ++ kaddr = kbase_mmu_dump(kctx, nr_pages); ++ ++ if (!kaddr) { ++ err = -ENOMEM; ++ goto out; ++ } ++ ++ new_reg = kbase_alloc_free_region(kctx, 0, nr_pages, KBASE_REG_ZONE_SAME_VA); ++ if (!new_reg) { ++ err = -ENOMEM; ++ WARN_ON(1); ++ goto out; ++ } ++ ++ new_reg->cpu_alloc = kbase_alloc_create(0, KBASE_MEM_TYPE_RAW); ++ if (IS_ERR_OR_NULL(new_reg->cpu_alloc)) { ++ err = -ENOMEM; ++ new_reg->cpu_alloc = NULL; ++ WARN_ON(1); ++ goto out_no_alloc; ++ } ++ ++ new_reg->gpu_alloc = kbase_mem_phy_alloc_get(new_reg->cpu_alloc); ++ ++ new_reg->flags &= ~KBASE_REG_FREE; ++ new_reg->flags |= KBASE_REG_CPU_CACHED; ++ if (kbase_add_va_region(kctx, new_reg, vma->vm_start, nr_pages, 1) != 0) { ++ err = -ENOMEM; ++ WARN_ON(1); ++ goto out_va_region; ++ } ++ ++ *kmap_addr = kaddr; ++ *reg = new_reg; ++ ++ dev_dbg(kctx->kbdev->dev, "kbase_mmu_dump_mmap done\n"); ++ return 0; ++ ++out_no_alloc: ++out_va_region: ++ kbase_free_alloced_region(new_reg); ++out: ++ return err; ++} ++ ++ ++void kbase_os_mem_map_lock(struct kbase_context *kctx) ++{ ++ struct mm_struct *mm = current->mm; ++ (void)kctx; ++ down_read(&mm->mmap_sem); ++} ++ ++void kbase_os_mem_map_unlock(struct kbase_context *kctx) ++{ ++ struct mm_struct *mm = current->mm; ++ (void)kctx; ++ up_read(&mm->mmap_sem); ++} ++ ++static int kbasep_reg_mmap(struct kbase_context *kctx, ++ struct vm_area_struct *vma, ++ struct kbase_va_region **regm, ++ size_t *nr_pages, size_t *aligned_offset) ++ ++{ ++ int cookie = vma->vm_pgoff - PFN_DOWN(BASE_MEM_COOKIE_BASE); ++ struct kbase_va_region *reg; ++ int err = 0; ++ ++ *aligned_offset = 0; ++ ++ dev_dbg(kctx->kbdev->dev, "in kbasep_reg_mmap\n"); ++ ++ /* SAME_VA stuff, fetch the right region */ ++ reg = kctx->pending_regions[cookie]; ++ if (!reg) { ++ err = -ENOMEM; ++ goto out; ++ } ++ ++ if ((reg->flags & KBASE_REG_GPU_NX) && (reg->nr_pages != *nr_pages)) { ++ /* incorrect mmap size */ ++ /* leave the cookie for a potential later ++ * mapping, or to be reclaimed later when the ++ * context is freed */ ++ err = -ENOMEM; ++ goto out; ++ } ++ ++ if ((vma->vm_flags & VM_READ && !(reg->flags & KBASE_REG_CPU_RD)) || ++ (vma->vm_flags & VM_WRITE && !(reg->flags & KBASE_REG_CPU_WR))) { ++ /* VM flags inconsistent with region flags */ ++ err = -EPERM; ++ dev_err(kctx->kbdev->dev, "%s:%d inconsistent VM flags\n", ++ __FILE__, __LINE__); ++ goto out; ++ } ++ ++ /* adjust down nr_pages to what we have physically */ ++ *nr_pages = kbase_reg_current_backed_size(reg); ++ ++ if (kbase_gpu_mmap(kctx, reg, vma->vm_start + *aligned_offset, ++ reg->nr_pages, 1) != 0) { ++ dev_err(kctx->kbdev->dev, "%s:%d\n", __FILE__, __LINE__); ++ /* Unable to map in GPU space. */ ++ WARN_ON(1); ++ err = -ENOMEM; ++ goto out; ++ } ++ /* no need for the cookie anymore */ ++ kctx->pending_regions[cookie] = NULL; ++ kctx->cookies |= (1UL << cookie); ++ ++ /* ++ * Overwrite the offset with the region start_pfn, so we effectively ++ * map from offset 0 in the region. However subtract the aligned ++ * offset so that when user space trims the mapping the beginning of ++ * the trimmed VMA has the correct vm_pgoff; ++ */ ++ vma->vm_pgoff = reg->start_pfn - ((*aligned_offset)>>PAGE_SHIFT); ++out: ++ *regm = reg; ++ dev_dbg(kctx->kbdev->dev, "kbasep_reg_mmap done\n"); ++ ++ return err; ++} ++ ++int kbase_mmap(struct file *file, struct vm_area_struct *vma) ++{ ++ struct kbase_context *kctx = file->private_data; ++ struct kbase_va_region *reg = NULL; ++ void *kaddr = NULL; ++ size_t nr_pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; ++ int err = 0; ++ int free_on_close = 0; ++ struct device *dev = kctx->kbdev->dev; ++ size_t aligned_offset = 0; ++ ++ dev_dbg(dev, "kbase_mmap\n"); ++ ++ /* strip away corresponding VM_MAY% flags to the VM_% flags requested */ ++ vma->vm_flags &= ~((vma->vm_flags & (VM_READ | VM_WRITE)) << 4); ++ ++ if (0 == nr_pages) { ++ err = -EINVAL; ++ goto out; ++ } ++ ++ if (!(vma->vm_flags & VM_SHARED)) { ++ err = -EINVAL; ++ goto out; ++ } ++ ++ kbase_gpu_vm_lock(kctx); ++ ++ if (vma->vm_pgoff == PFN_DOWN(BASE_MEM_MAP_TRACKING_HANDLE)) { ++ /* The non-mapped tracking helper page */ ++ err = kbase_tracking_page_setup(kctx, vma); ++ goto out_unlock; ++ } ++ ++ /* if not the MTP, verify that the MTP has been mapped */ ++ rcu_read_lock(); ++ /* catches both when the special page isn't present or ++ * when we've forked */ ++ if (rcu_dereference(kctx->process_mm) != current->mm) { ++ err = -EINVAL; ++ rcu_read_unlock(); ++ goto out_unlock; ++ } ++ rcu_read_unlock(); ++ ++ switch (vma->vm_pgoff) { ++ case PFN_DOWN(BASEP_MEM_INVALID_HANDLE): ++ case PFN_DOWN(BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE): ++ /* Illegal handle for direct map */ ++ err = -EINVAL; ++ goto out_unlock; ++ case PFN_DOWN(BASE_MEM_TRACE_BUFFER_HANDLE): ++ err = kbase_trace_buffer_mmap(kctx, vma, ®, &kaddr); ++ if (0 != err) ++ goto out_unlock; ++ dev_dbg(dev, "kbase_trace_buffer_mmap ok\n"); ++ /* free the region on munmap */ ++ free_on_close = 1; ++ break; ++ case PFN_DOWN(BASE_MEM_MMU_DUMP_HANDLE): ++ /* MMU dump */ ++ err = kbase_mmu_dump_mmap(kctx, vma, ®, &kaddr); ++ if (0 != err) ++ goto out_unlock; ++ /* free the region on munmap */ ++ free_on_close = 1; ++ break; ++ case PFN_DOWN(BASE_MEM_COOKIE_BASE) ... ++ PFN_DOWN(BASE_MEM_FIRST_FREE_ADDRESS) - 1: { ++ err = kbasep_reg_mmap(kctx, vma, ®, &nr_pages, ++ &aligned_offset); ++ if (0 != err) ++ goto out_unlock; ++ /* free the region on munmap */ ++ free_on_close = 1; ++ break; ++ } ++ default: { ++ reg = kbase_region_tracker_find_region_enclosing_address(kctx, ++ (u64)vma->vm_pgoff << PAGE_SHIFT); ++ ++ if (reg && !(reg->flags & KBASE_REG_FREE)) { ++ /* will this mapping overflow the size of the region? */ ++ if (nr_pages > (reg->nr_pages - ++ (vma->vm_pgoff - reg->start_pfn))) { ++ err = -ENOMEM; ++ goto out_unlock; ++ } ++ ++ if ((vma->vm_flags & VM_READ && ++ !(reg->flags & KBASE_REG_CPU_RD)) || ++ (vma->vm_flags & VM_WRITE && ++ !(reg->flags & KBASE_REG_CPU_WR))) { ++ /* VM flags inconsistent with region flags */ ++ err = -EPERM; ++ dev_err(dev, "%s:%d inconsistent VM flags\n", ++ __FILE__, __LINE__); ++ goto out_unlock; ++ } ++ ++#ifdef CONFIG_DMA_SHARED_BUFFER ++ if (KBASE_MEM_TYPE_IMPORTED_UMM == ++ reg->cpu_alloc->type) { ++ err = dma_buf_mmap( ++ reg->cpu_alloc->imported.umm.dma_buf, ++ vma, vma->vm_pgoff - reg->start_pfn); ++ goto out_unlock; ++ } ++#endif /* CONFIG_DMA_SHARED_BUFFER */ ++ ++ /* limit what we map to the amount currently backed */ ++ if (reg->cpu_alloc->nents < (vma->vm_pgoff - reg->start_pfn + nr_pages)) { ++ if ((vma->vm_pgoff - reg->start_pfn) >= reg->cpu_alloc->nents) ++ nr_pages = 0; ++ else ++ nr_pages = reg->cpu_alloc->nents - (vma->vm_pgoff - reg->start_pfn); ++ } ++ } else { ++ err = -ENOMEM; ++ goto out_unlock; ++ } ++ } /* default */ ++ } /* switch */ ++ ++ err = kbase_cpu_mmap(reg, vma, kaddr, nr_pages, aligned_offset, free_on_close); ++ ++ if (vma->vm_pgoff == PFN_DOWN(BASE_MEM_MMU_DUMP_HANDLE)) { ++ /* MMU dump - userspace should now have a reference on ++ * the pages, so we can now free the kernel mapping */ ++ vfree(kaddr); ++ } ++ ++out_unlock: ++ kbase_gpu_vm_unlock(kctx); ++out: ++ if (err) ++ dev_err(dev, "mmap failed %d\n", err); ++ ++ return err; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_mmap); ++ ++static void kbasep_sync_mem_regions(struct kbase_context *kctx, ++ struct kbase_vmap_struct *map, enum kbase_sync_type dest) ++{ ++ size_t i; ++ off_t const offset = (uintptr_t)map->gpu_addr & ~PAGE_MASK; ++ size_t const page_count = PFN_UP(offset + map->size); ++ ++ /* Sync first page */ ++ size_t sz = MIN(((size_t) PAGE_SIZE - offset), map->size); ++ struct tagged_addr cpu_pa = map->cpu_pages[0]; ++ struct tagged_addr gpu_pa = map->gpu_pages[0]; ++ ++ kbase_sync_single(kctx, cpu_pa, gpu_pa, offset, sz, dest); ++ ++ /* Sync middle pages (if any) */ ++ for (i = 1; page_count > 2 && i < page_count - 1; i++) { ++ cpu_pa = map->cpu_pages[i]; ++ gpu_pa = map->gpu_pages[i]; ++ kbase_sync_single(kctx, cpu_pa, gpu_pa, 0, PAGE_SIZE, dest); ++ } ++ ++ /* Sync last page (if any) */ ++ if (page_count > 1) { ++ cpu_pa = map->cpu_pages[page_count - 1]; ++ gpu_pa = map->gpu_pages[page_count - 1]; ++ sz = ((offset + map->size - 1) & ~PAGE_MASK) + 1; ++ kbase_sync_single(kctx, cpu_pa, gpu_pa, 0, sz, dest); ++ } ++} ++ ++void *kbase_vmap_prot(struct kbase_context *kctx, u64 gpu_addr, size_t size, ++ unsigned long prot_request, struct kbase_vmap_struct *map) ++{ ++ struct kbase_va_region *reg; ++ unsigned long page_index; ++ unsigned int offset = gpu_addr & ~PAGE_MASK; ++ size_t page_count = PFN_UP(offset + size); ++ struct tagged_addr *page_array; ++ struct page **pages; ++ void *cpu_addr = NULL; ++ pgprot_t prot; ++ size_t i; ++ ++ if (!size || !map) ++ return NULL; ++ ++ /* check if page_count calculation will wrap */ ++ if (size > ((size_t)-1 / PAGE_SIZE)) ++ return NULL; ++ ++ kbase_gpu_vm_lock(kctx); ++ ++ reg = kbase_region_tracker_find_region_enclosing_address(kctx, gpu_addr); ++ if (!reg || (reg->flags & KBASE_REG_FREE)) ++ goto out_unlock; ++ ++ page_index = (gpu_addr >> PAGE_SHIFT) - reg->start_pfn; ++ ++ /* check if page_index + page_count will wrap */ ++ if (-1UL - page_count < page_index) ++ goto out_unlock; ++ ++ if (page_index + page_count > kbase_reg_current_backed_size(reg)) ++ goto out_unlock; ++ ++ if (reg->flags & KBASE_REG_DONT_NEED) ++ goto out_unlock; ++ ++ /* check access permissions can be satisfied ++ * Intended only for checking KBASE_REG_{CPU,GPU}_{RD,WR} */ ++ if ((reg->flags & prot_request) != prot_request) ++ goto out_unlock; ++ ++ page_array = kbase_get_cpu_phy_pages(reg); ++ if (!page_array) ++ goto out_unlock; ++ ++ pages = kmalloc_array(page_count, sizeof(struct page *), GFP_KERNEL); ++ if (!pages) ++ goto out_unlock; ++ ++ for (i = 0; i < page_count; i++) ++ pages[i] = phys_to_page(as_phys_addr_t(page_array[page_index + ++ i])); ++ ++ prot = PAGE_KERNEL; ++ if (!(reg->flags & KBASE_REG_CPU_CACHED)) { ++ /* Map uncached */ ++ prot = pgprot_writecombine(prot); ++ } ++ /* Note: enforcing a RO prot_request onto prot is not done, since: ++ * - CPU-arch-specific integration required ++ * - kbase_vmap() requires no access checks to be made/enforced */ ++ ++ cpu_addr = vmap(pages, page_count, VM_MAP, prot); ++ ++ kfree(pages); ++ ++ if (!cpu_addr) ++ goto out_unlock; ++ ++ map->gpu_addr = gpu_addr; ++ map->cpu_alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc); ++ map->cpu_pages = &kbase_get_cpu_phy_pages(reg)[page_index]; ++ map->gpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); ++ map->gpu_pages = &kbase_get_gpu_phy_pages(reg)[page_index]; ++ map->addr = (void *)((uintptr_t)cpu_addr + offset); ++ map->size = size; ++ map->sync_needed = ((reg->flags & KBASE_REG_CPU_CACHED) != 0) && ++ !kbase_mem_is_imported(map->gpu_alloc->type); ++ ++ if (map->sync_needed) ++ kbasep_sync_mem_regions(kctx, map, KBASE_SYNC_TO_CPU); ++ kbase_gpu_vm_unlock(kctx); ++ ++ return map->addr; ++ ++out_unlock: ++ kbase_gpu_vm_unlock(kctx); ++ return NULL; ++} ++ ++void *kbase_vmap(struct kbase_context *kctx, u64 gpu_addr, size_t size, ++ struct kbase_vmap_struct *map) ++{ ++ /* 0 is specified for prot_request to indicate no access checks should ++ * be made. ++ * ++ * As mentioned in kbase_vmap_prot() this means that a kernel-side ++ * CPU-RO mapping is not enforced to allow this to work */ ++ return kbase_vmap_prot(kctx, gpu_addr, size, 0u, map); ++} ++KBASE_EXPORT_TEST_API(kbase_vmap); ++ ++void kbase_vunmap(struct kbase_context *kctx, struct kbase_vmap_struct *map) ++{ ++ void *addr = (void *)((uintptr_t)map->addr & PAGE_MASK); ++ vunmap(addr); ++ ++ if (map->sync_needed) ++ kbasep_sync_mem_regions(kctx, map, KBASE_SYNC_TO_DEVICE); ++ map->gpu_addr = 0; ++ map->cpu_alloc = kbase_mem_phy_alloc_put(map->cpu_alloc); ++ map->gpu_alloc = kbase_mem_phy_alloc_put(map->gpu_alloc); ++ map->cpu_pages = NULL; ++ map->gpu_pages = NULL; ++ map->addr = NULL; ++ map->size = 0; ++ map->sync_needed = false; ++} ++KBASE_EXPORT_TEST_API(kbase_vunmap); ++ ++void kbasep_os_process_page_usage_update(struct kbase_context *kctx, int pages) ++{ ++ struct mm_struct *mm; ++ ++ rcu_read_lock(); ++ mm = rcu_dereference(kctx->process_mm); ++ if (mm) { ++ atomic_add(pages, &kctx->nonmapped_pages); ++#ifdef SPLIT_RSS_COUNTING ++ add_mm_counter(mm, MM_FILEPAGES, pages); ++#else ++ spin_lock(&mm->page_table_lock); ++ add_mm_counter(mm, MM_FILEPAGES, pages); ++ spin_unlock(&mm->page_table_lock); ++#endif ++ } ++ rcu_read_unlock(); ++} ++ ++static void kbasep_os_process_page_usage_drain(struct kbase_context *kctx) ++{ ++ int pages; ++ struct mm_struct *mm; ++ ++ spin_lock(&kctx->mm_update_lock); ++ mm = rcu_dereference_protected(kctx->process_mm, lockdep_is_held(&kctx->mm_update_lock)); ++ if (!mm) { ++ spin_unlock(&kctx->mm_update_lock); ++ return; ++ } ++ ++ rcu_assign_pointer(kctx->process_mm, NULL); ++ spin_unlock(&kctx->mm_update_lock); ++ synchronize_rcu(); ++ ++ pages = atomic_xchg(&kctx->nonmapped_pages, 0); ++#ifdef SPLIT_RSS_COUNTING ++ add_mm_counter(mm, MM_FILEPAGES, -pages); ++#else ++ spin_lock(&mm->page_table_lock); ++ add_mm_counter(mm, MM_FILEPAGES, -pages); ++ spin_unlock(&mm->page_table_lock); ++#endif ++} ++ ++static void kbase_special_vm_close(struct vm_area_struct *vma) ++{ ++ struct kbase_context *kctx; ++ ++ kctx = vma->vm_private_data; ++ kbasep_os_process_page_usage_drain(kctx); ++} ++ ++static const struct vm_operations_struct kbase_vm_special_ops = { ++ .close = kbase_special_vm_close, ++}; ++ ++static int kbase_tracking_page_setup(struct kbase_context *kctx, struct vm_area_struct *vma) ++{ ++ /* check that this is the only tracking page */ ++ spin_lock(&kctx->mm_update_lock); ++ if (rcu_dereference_protected(kctx->process_mm, lockdep_is_held(&kctx->mm_update_lock))) { ++ spin_unlock(&kctx->mm_update_lock); ++ return -EFAULT; ++ } ++ ++ rcu_assign_pointer(kctx->process_mm, current->mm); ++ ++ spin_unlock(&kctx->mm_update_lock); ++ ++ /* no real access */ ++ vma->vm_flags &= ~(VM_READ | VM_MAYREAD | VM_WRITE | VM_MAYWRITE | VM_EXEC | VM_MAYEXEC); ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0)) ++ vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP | VM_IO; ++#else ++ vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_RESERVED | VM_IO; ++#endif ++ vma->vm_ops = &kbase_vm_special_ops; ++ vma->vm_private_data = kctx; ++ ++ return 0; ++} ++void *kbase_va_alloc(struct kbase_context *kctx, u32 size, struct kbase_hwc_dma_mapping *handle) ++{ ++ int res; ++ void *va; ++ dma_addr_t dma_pa; ++ struct kbase_va_region *reg; ++ struct tagged_addr *page_array; ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0)) ++ unsigned long attrs = DMA_ATTR_WRITE_COMBINE; ++#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)) ++ DEFINE_DMA_ATTRS(attrs); ++#endif ++ ++ u32 pages = ((size - 1) >> PAGE_SHIFT) + 1; ++ u32 flags = BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_CPU_WR | ++ BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR; ++ u32 i; ++ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ KBASE_DEBUG_ASSERT(0 != size); ++ KBASE_DEBUG_ASSERT(0 != pages); ++ ++ if (size == 0) ++ goto err; ++ ++ /* All the alloc calls return zeroed memory */ ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0)) ++ va = dma_alloc_attrs(kctx->kbdev->dev, size, &dma_pa, GFP_KERNEL, ++ attrs); ++#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)) ++ dma_set_attr(DMA_ATTR_WRITE_COMBINE, &attrs); ++ va = dma_alloc_attrs(kctx->kbdev->dev, size, &dma_pa, GFP_KERNEL, ++ &attrs); ++#else ++ va = dma_alloc_writecombine(kctx->kbdev->dev, size, &dma_pa, GFP_KERNEL); ++#endif ++ if (!va) ++ goto err; ++ ++ /* Store the state so we can free it later. */ ++ handle->cpu_va = va; ++ handle->dma_pa = dma_pa; ++ handle->size = size; ++ ++ ++ reg = kbase_alloc_free_region(kctx, 0, pages, KBASE_REG_ZONE_SAME_VA); ++ if (!reg) ++ goto no_reg; ++ ++ reg->flags &= ~KBASE_REG_FREE; ++ if (kbase_update_region_flags(kctx, reg, flags) != 0) ++ goto invalid_flags; ++ ++ reg->cpu_alloc = kbase_alloc_create(pages, KBASE_MEM_TYPE_RAW); ++ if (IS_ERR_OR_NULL(reg->cpu_alloc)) ++ goto no_alloc; ++ ++ reg->gpu_alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc); ++ ++ page_array = kbase_get_cpu_phy_pages(reg); ++ ++ for (i = 0; i < pages; i++) ++ page_array[i] = as_tagged(dma_pa + (i << PAGE_SHIFT)); ++ ++ reg->cpu_alloc->nents = pages; ++ ++ kbase_gpu_vm_lock(kctx); ++ res = kbase_gpu_mmap(kctx, reg, (uintptr_t) va, pages, 1); ++ kbase_gpu_vm_unlock(kctx); ++ if (res) ++ goto no_mmap; ++ ++ return va; ++ ++no_mmap: ++ kbase_mem_phy_alloc_put(reg->cpu_alloc); ++ kbase_mem_phy_alloc_put(reg->gpu_alloc); ++no_alloc: ++invalid_flags: ++ kfree(reg); ++no_reg: ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0)) ++ dma_free_attrs(kctx->kbdev->dev, size, va, dma_pa, attrs); ++#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)) ++ dma_free_attrs(kctx->kbdev->dev, size, va, dma_pa, &attrs); ++#else ++ dma_free_writecombine(kctx->kbdev->dev, size, va, dma_pa); ++#endif ++err: ++ return NULL; ++} ++KBASE_EXPORT_SYMBOL(kbase_va_alloc); ++ ++void kbase_va_free(struct kbase_context *kctx, struct kbase_hwc_dma_mapping *handle) ++{ ++ struct kbase_va_region *reg; ++ int err; ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)) && \ ++ (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0)) ++ DEFINE_DMA_ATTRS(attrs); ++#endif ++ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ KBASE_DEBUG_ASSERT(handle->cpu_va != NULL); ++ ++ kbase_gpu_vm_lock(kctx); ++ reg = kbase_region_tracker_find_region_base_address(kctx, (uintptr_t)handle->cpu_va); ++ KBASE_DEBUG_ASSERT(reg); ++ err = kbase_gpu_munmap(kctx, reg); ++ kbase_gpu_vm_unlock(kctx); ++ KBASE_DEBUG_ASSERT(!err); ++ ++ kbase_mem_phy_alloc_put(reg->cpu_alloc); ++ kbase_mem_phy_alloc_put(reg->gpu_alloc); ++ kfree(reg); ++ ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0)) ++ dma_free_attrs(kctx->kbdev->dev, handle->size, ++ handle->cpu_va, handle->dma_pa, DMA_ATTR_WRITE_COMBINE); ++#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)) ++ dma_set_attr(DMA_ATTR_WRITE_COMBINE, &attrs); ++ dma_free_attrs(kctx->kbdev->dev, handle->size, ++ handle->cpu_va, handle->dma_pa, &attrs); ++#else ++ dma_free_writecombine(kctx->kbdev->dev, handle->size, ++ handle->cpu_va, handle->dma_pa); ++#endif ++} ++KBASE_EXPORT_SYMBOL(kbase_va_free); ++ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_linux.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_linux.h +new file mode 100755 +index 000000000000..db35f62a7431 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_linux.h +@@ -0,0 +1,240 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010, 2012-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/** ++ * @file mali_kbase_mem_linux.h ++ * Base kernel memory APIs, Linux implementation. ++ */ ++ ++#ifndef _KBASE_MEM_LINUX_H_ ++#define _KBASE_MEM_LINUX_H_ ++ ++/** A HWC dump mapping */ ++struct kbase_hwc_dma_mapping { ++ void *cpu_va; ++ dma_addr_t dma_pa; ++ size_t size; ++}; ++ ++struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, ++ u64 va_pages, u64 commit_pages, u64 extent, u64 *flags, ++ u64 *gpu_va); ++int kbase_mem_query(struct kbase_context *kctx, u64 gpu_addr, int query, u64 *const pages); ++int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type, ++ void __user *phandle, u32 padding, u64 *gpu_va, u64 *va_pages, ++ u64 *flags); ++u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, u64 nents, struct base_mem_aliasing_info *ai, u64 *num_pages); ++int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, unsigned int flags, unsigned int mask); ++ ++/** ++ * kbase_mem_commit - Change the physical backing size of a region ++ * ++ * @kctx: The kernel context ++ * @gpu_addr: Handle to the memory region ++ * @new_pages: Number of physical pages to back the region with ++ * ++ * Return: 0 on success or error code ++ */ ++int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages); ++ ++int kbase_mmap(struct file *file, struct vm_area_struct *vma); ++ ++/** ++ * kbase_mem_evictable_init - Initialize the Ephemeral memory the eviction ++ * mechanism. ++ * @kctx: The kbase context to initialize. ++ * ++ * Return: Zero on success or -errno on failure. ++ */ ++int kbase_mem_evictable_init(struct kbase_context *kctx); ++ ++/** ++ * kbase_mem_evictable_deinit - De-initialize the Ephemeral memory eviction ++ * mechanism. ++ * @kctx: The kbase context to de-initialize. ++ */ ++void kbase_mem_evictable_deinit(struct kbase_context *kctx); ++ ++/** ++ * kbase_mem_grow_gpu_mapping - Grow the GPU mapping of an allocation ++ * @kctx: Context the region belongs to ++ * @reg: The GPU region ++ * @new_pages: The number of pages after the grow ++ * @old_pages: The number of pages before the grow ++ * ++ * Return: 0 on success, -errno on error. ++ * ++ * Expand the GPU mapping to encompass the new psychical pages which have ++ * been added to the allocation. ++ * ++ * Note: Caller must be holding the region lock. ++ */ ++int kbase_mem_grow_gpu_mapping(struct kbase_context *kctx, ++ struct kbase_va_region *reg, ++ u64 new_pages, u64 old_pages); ++ ++/** ++ * kbase_mem_evictable_make - Make a physical allocation eligible for eviction ++ * @gpu_alloc: The physical allocation to make evictable ++ * ++ * Return: 0 on success, -errno on error. ++ * ++ * Take the provided region and make all the physical pages within it ++ * reclaimable by the kernel, updating the per-process VM stats as well. ++ * Remove any CPU mappings (as these can't be removed in the shrinker callback ++ * as mmap_sem might already be taken) but leave the GPU mapping intact as ++ * and until the shrinker reclaims the allocation. ++ * ++ * Note: Must be called with the region lock of the containing context. ++ */ ++int kbase_mem_evictable_make(struct kbase_mem_phy_alloc *gpu_alloc); ++ ++/** ++ * kbase_mem_evictable_unmake - Remove a physical allocations eligibility for ++ * eviction. ++ * @alloc: The physical allocation to remove eviction eligibility from. ++ * ++ * Return: True if the allocation had its backing restored and false if ++ * it hasn't. ++ * ++ * Make the physical pages in the region no longer reclaimable and update the ++ * per-process stats, if the shrinker has already evicted the memory then ++ * re-allocate it if the region is still alive. ++ * ++ * Note: Must be called with the region lock of the containing context. ++ */ ++bool kbase_mem_evictable_unmake(struct kbase_mem_phy_alloc *alloc); ++ ++struct kbase_vmap_struct { ++ u64 gpu_addr; ++ struct kbase_mem_phy_alloc *cpu_alloc; ++ struct kbase_mem_phy_alloc *gpu_alloc; ++ struct tagged_addr *cpu_pages; ++ struct tagged_addr *gpu_pages; ++ void *addr; ++ size_t size; ++ bool sync_needed; ++}; ++ ++ ++/** ++ * kbase_vmap_prot - Map a GPU VA range into the kernel safely, only if the ++ * requested access permissions are supported ++ * @kctx: Context the VA range belongs to ++ * @gpu_addr: Start address of VA range ++ * @size: Size of VA range ++ * @prot_request: Flags indicating how the caller will then access the memory ++ * @map: Structure to be given to kbase_vunmap() on freeing ++ * ++ * Return: Kernel-accessible CPU pointer to the VA range, or NULL on error ++ * ++ * Map a GPU VA Range into the kernel. The VA range must be contained within a ++ * GPU memory region. Appropriate CPU cache-flushing operations are made as ++ * required, dependent on the CPU mapping for the memory region. ++ * ++ * This is safer than using kmap() on the pages directly, ++ * because the pages here are refcounted to prevent freeing (and hence reuse ++ * elsewhere in the system) until an kbase_vunmap() ++ * ++ * The flags in @prot_request should use KBASE_REG_{CPU,GPU}_{RD,WR}, to check ++ * whether the region should allow the intended access, and return an error if ++ * disallowed. This is essential for security of imported memory, particularly ++ * a user buf from SHM mapped into the process as RO. In that case, write ++ * access must be checked if the intention is for kernel to write to the ++ * memory. ++ * ++ * The checks are also there to help catch access errors on memory where ++ * security is not a concern: imported memory that is always RW, and memory ++ * that was allocated and owned by the process attached to @kctx. In this case, ++ * it helps to identify memory that was was mapped with the wrong access type. ++ * ++ * Note: KBASE_REG_GPU_{RD,WR} flags are currently supported for legacy cases ++ * where either the security of memory is solely dependent on those flags, or ++ * when userspace code was expecting only the GPU to access the memory (e.g. HW ++ * workarounds). ++ * ++ * All cache maintenance operations shall be ignored if the ++ * memory region has been imported. ++ * ++ */ ++void *kbase_vmap_prot(struct kbase_context *kctx, u64 gpu_addr, size_t size, ++ unsigned long prot_request, struct kbase_vmap_struct *map); ++ ++/** ++ * kbase_vmap - Map a GPU VA range into the kernel safely ++ * @kctx: Context the VA range belongs to ++ * @gpu_addr: Start address of VA range ++ * @size: Size of VA range ++ * @map: Structure to be given to kbase_vunmap() on freeing ++ * ++ * Return: Kernel-accessible CPU pointer to the VA range, or NULL on error ++ * ++ * Map a GPU VA Range into the kernel. The VA range must be contained within a ++ * GPU memory region. Appropriate CPU cache-flushing operations are made as ++ * required, dependent on the CPU mapping for the memory region. ++ * ++ * This is safer than using kmap() on the pages directly, ++ * because the pages here are refcounted to prevent freeing (and hence reuse ++ * elsewhere in the system) until an kbase_vunmap() ++ * ++ * kbase_vmap_prot() should be used in preference, since kbase_vmap() makes no ++ * checks to ensure the security of e.g. imported user bufs from RO SHM. ++ * ++ * Note: All cache maintenance operations shall be ignored if the memory region ++ * has been imported. ++ */ ++void *kbase_vmap(struct kbase_context *kctx, u64 gpu_addr, size_t size, ++ struct kbase_vmap_struct *map); ++ ++/** ++ * kbase_vunmap - Unmap a GPU VA range from the kernel ++ * @kctx: Context the VA range belongs to ++ * @map: Structure describing the mapping from the corresponding kbase_vmap() ++ * call ++ * ++ * Unmaps a GPU VA range from the kernel, given its @map structure obtained ++ * from kbase_vmap(). Appropriate CPU cache-flushing operations are made as ++ * required, dependent on the CPU mapping for the memory region. ++ * ++ * The reference taken on pages during kbase_vmap() is released. ++ * ++ * Note: All cache maintenance operations shall be ignored if the memory region ++ * has been imported. ++ */ ++void kbase_vunmap(struct kbase_context *kctx, struct kbase_vmap_struct *map); ++ ++/** @brief Allocate memory from kernel space and map it onto the GPU ++ * ++ * @param kctx The context used for the allocation/mapping ++ * @param size The size of the allocation in bytes ++ * @param handle An opaque structure used to contain the state needed to free the memory ++ * @return the VA for kernel space and GPU MMU ++ */ ++void *kbase_va_alloc(struct kbase_context *kctx, u32 size, struct kbase_hwc_dma_mapping *handle); ++ ++/** @brief Free/unmap memory allocated by kbase_va_alloc ++ * ++ * @param kctx The context used for the allocation/mapping ++ * @param handle An opaque structure returned by the kbase_va_alloc function. ++ */ ++void kbase_va_free(struct kbase_context *kctx, struct kbase_hwc_dma_mapping *handle); ++ ++extern const struct vm_operations_struct kbase_vm_ops; ++ ++#endif /* _KBASE_MEM_LINUX_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_lowlevel.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_lowlevel.h +new file mode 100755 +index 000000000000..f4e88491327e +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_lowlevel.h +@@ -0,0 +1,89 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2014,2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#ifndef _KBASE_MEM_LOWLEVEL_H ++#define _KBASE_MEM_LOWLEVEL_H ++ ++#ifndef _KBASE_H_ ++#error "Don't include this file directly, use mali_kbase.h instead" ++#endif ++ ++#include ++ ++/** ++ * @brief Flags for kbase_phy_allocator_pages_alloc ++ */ ++#define KBASE_PHY_PAGES_FLAG_DEFAULT (0) /** Default allocation flag */ ++#define KBASE_PHY_PAGES_FLAG_CLEAR (1 << 0) /** Clear the pages after allocation */ ++#define KBASE_PHY_PAGES_FLAG_POISON (1 << 1) /** Fill the memory with a poison value */ ++ ++#define KBASE_PHY_PAGES_SUPPORTED_FLAGS (KBASE_PHY_PAGES_FLAG_DEFAULT|KBASE_PHY_PAGES_FLAG_CLEAR|KBASE_PHY_PAGES_FLAG_POISON) ++ ++#define KBASE_PHY_PAGES_POISON_VALUE 0xFD /** Value to fill the memory with when KBASE_PHY_PAGES_FLAG_POISON is set */ ++ ++enum kbase_sync_type { ++ KBASE_SYNC_TO_CPU, ++ KBASE_SYNC_TO_DEVICE ++}; ++ ++struct tagged_addr { phys_addr_t tagged_addr; }; ++ ++#define HUGE_PAGE (1u << 0) ++#define HUGE_HEAD (1u << 1) ++#define FROM_PARTIAL (1u << 2) ++ ++static inline phys_addr_t as_phys_addr_t(struct tagged_addr t) ++{ ++ return t.tagged_addr & PAGE_MASK; ++} ++ ++static inline struct tagged_addr as_tagged(phys_addr_t phys) ++{ ++ struct tagged_addr t; ++ ++ t.tagged_addr = phys & PAGE_MASK; ++ return t; ++} ++ ++static inline struct tagged_addr as_tagged_tag(phys_addr_t phys, int tag) ++{ ++ struct tagged_addr t; ++ ++ t.tagged_addr = (phys & PAGE_MASK) | (tag & ~PAGE_MASK); ++ return t; ++} ++ ++static inline bool is_huge(struct tagged_addr t) ++{ ++ return t.tagged_addr & HUGE_PAGE; ++} ++ ++static inline bool is_huge_head(struct tagged_addr t) ++{ ++ int mask = HUGE_HEAD | HUGE_PAGE; ++ ++ return mask == (t.tagged_addr & mask); ++} ++ ++static inline bool is_partial(struct tagged_addr t) ++{ ++ return t.tagged_addr & FROM_PARTIAL; ++} ++ ++#endif /* _KBASE_LOWLEVEL_H */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_pool.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_pool.c +new file mode 100755 +index 000000000000..696730ac5b2b +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_pool.c +@@ -0,0 +1,651 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#define pool_dbg(pool, format, ...) \ ++ dev_dbg(pool->kbdev->dev, "%s-pool [%zu/%zu]: " format, \ ++ (pool->next_pool) ? "kctx" : "kbdev", \ ++ kbase_mem_pool_size(pool), \ ++ kbase_mem_pool_max_size(pool), \ ++ ##__VA_ARGS__) ++ ++#define NOT_DIRTY false ++#define NOT_RECLAIMED false ++ ++static inline void kbase_mem_pool_lock(struct kbase_mem_pool *pool) ++{ ++ spin_lock(&pool->pool_lock); ++} ++ ++static inline void kbase_mem_pool_unlock(struct kbase_mem_pool *pool) ++{ ++ spin_unlock(&pool->pool_lock); ++} ++ ++static size_t kbase_mem_pool_capacity(struct kbase_mem_pool *pool) ++{ ++ ssize_t max_size = kbase_mem_pool_max_size(pool); ++ ssize_t cur_size = kbase_mem_pool_size(pool); ++ ++ return max(max_size - cur_size, (ssize_t)0); ++} ++ ++static bool kbase_mem_pool_is_full(struct kbase_mem_pool *pool) ++{ ++ return kbase_mem_pool_size(pool) >= kbase_mem_pool_max_size(pool); ++} ++ ++static bool kbase_mem_pool_is_empty(struct kbase_mem_pool *pool) ++{ ++ return kbase_mem_pool_size(pool) == 0; ++} ++ ++static void kbase_mem_pool_add_locked(struct kbase_mem_pool *pool, ++ struct page *p) ++{ ++ lockdep_assert_held(&pool->pool_lock); ++ ++ list_add(&p->lru, &pool->page_list); ++ pool->cur_size++; ++ ++ zone_page_state_add(1, page_zone(p), NR_SLAB_RECLAIMABLE); ++ ++ pool_dbg(pool, "added page\n"); ++} ++ ++static void kbase_mem_pool_add(struct kbase_mem_pool *pool, struct page *p) ++{ ++ kbase_mem_pool_lock(pool); ++ kbase_mem_pool_add_locked(pool, p); ++ kbase_mem_pool_unlock(pool); ++} ++ ++static void kbase_mem_pool_add_list_locked(struct kbase_mem_pool *pool, ++ struct list_head *page_list, size_t nr_pages) ++{ ++ struct page *p; ++ ++ lockdep_assert_held(&pool->pool_lock); ++ ++ list_for_each_entry(p, page_list, lru) { ++ zone_page_state_add(1, page_zone(p), NR_SLAB_RECLAIMABLE); ++ } ++ ++ list_splice(page_list, &pool->page_list); ++ pool->cur_size += nr_pages; ++ ++ pool_dbg(pool, "added %zu pages\n", nr_pages); ++} ++ ++static void kbase_mem_pool_add_list(struct kbase_mem_pool *pool, ++ struct list_head *page_list, size_t nr_pages) ++{ ++ kbase_mem_pool_lock(pool); ++ kbase_mem_pool_add_list_locked(pool, page_list, nr_pages); ++ kbase_mem_pool_unlock(pool); ++} ++ ++static struct page *kbase_mem_pool_remove_locked(struct kbase_mem_pool *pool) ++{ ++ struct page *p; ++ ++ lockdep_assert_held(&pool->pool_lock); ++ ++ if (kbase_mem_pool_is_empty(pool)) ++ return NULL; ++ ++ p = list_first_entry(&pool->page_list, struct page, lru); ++ list_del_init(&p->lru); ++ pool->cur_size--; ++ ++ zone_page_state_add(-1, page_zone(p), NR_SLAB_RECLAIMABLE); ++ ++ pool_dbg(pool, "removed page\n"); ++ ++ return p; ++} ++ ++static struct page *kbase_mem_pool_remove(struct kbase_mem_pool *pool) ++{ ++ struct page *p; ++ ++ kbase_mem_pool_lock(pool); ++ p = kbase_mem_pool_remove_locked(pool); ++ kbase_mem_pool_unlock(pool); ++ ++ return p; ++} ++ ++static void kbase_mem_pool_sync_page(struct kbase_mem_pool *pool, ++ struct page *p) ++{ ++ struct device *dev = pool->kbdev->dev; ++ dma_sync_single_for_device(dev, kbase_dma_addr(p), ++ (PAGE_SIZE << pool->order), DMA_BIDIRECTIONAL); ++} ++ ++static void kbase_mem_pool_zero_page(struct kbase_mem_pool *pool, ++ struct page *p) ++{ ++ int i; ++ ++ for (i = 0; i < (1U << pool->order); i++) ++ clear_highpage(p+i); ++ ++ kbase_mem_pool_sync_page(pool, p); ++} ++ ++static void kbase_mem_pool_spill(struct kbase_mem_pool *next_pool, ++ struct page *p) ++{ ++ /* Zero page before spilling */ ++ kbase_mem_pool_zero_page(next_pool, p); ++ ++ kbase_mem_pool_add(next_pool, p); ++} ++ ++struct page *kbase_mem_alloc_page(struct kbase_mem_pool *pool) ++{ ++ struct page *p; ++ gfp_t gfp; ++ struct device *dev = pool->kbdev->dev; ++ dma_addr_t dma_addr; ++ int i; ++ ++#if defined(CONFIG_ARM) && !defined(CONFIG_HAVE_DMA_ATTRS) && \ ++ LINUX_VERSION_CODE < KERNEL_VERSION(3, 5, 0) ++ /* DMA cache sync fails for HIGHMEM before 3.5 on ARM */ ++ gfp = GFP_USER | __GFP_ZERO; ++#else ++ gfp = GFP_HIGHUSER | __GFP_ZERO; ++#endif ++ ++ if (current->flags & PF_KTHREAD) { ++ /* Don't trigger OOM killer from kernel threads, e.g. when ++ * growing memory on GPU page fault */ ++ gfp |= __GFP_NORETRY; ++ } ++ ++ /* don't warn on higer order failures */ ++ if (pool->order) ++ gfp |= __GFP_NOWARN; ++ ++ p = alloc_pages(gfp, pool->order); ++ if (!p) ++ return NULL; ++ ++ dma_addr = dma_map_page(dev, p, 0, (PAGE_SIZE << pool->order), ++ DMA_BIDIRECTIONAL); ++ if (dma_mapping_error(dev, dma_addr)) { ++ __free_pages(p, pool->order); ++ return NULL; ++ } ++ ++ WARN_ON(dma_addr != page_to_phys(p)); ++ for (i = 0; i < (1u << pool->order); i++) ++ kbase_set_dma_addr(p+i, dma_addr + PAGE_SIZE * i); ++ ++ return p; ++} ++ ++static void kbase_mem_pool_free_page(struct kbase_mem_pool *pool, ++ struct page *p) ++{ ++ struct device *dev = pool->kbdev->dev; ++ dma_addr_t dma_addr = kbase_dma_addr(p); ++ int i; ++ ++ dma_unmap_page(dev, dma_addr, (PAGE_SIZE << pool->order), ++ DMA_BIDIRECTIONAL); ++ for (i = 0; i < (1u << pool->order); i++) ++ kbase_clear_dma_addr(p+i); ++ __free_pages(p, pool->order); ++ ++ pool_dbg(pool, "freed page to kernel\n"); ++} ++ ++static size_t kbase_mem_pool_shrink_locked(struct kbase_mem_pool *pool, ++ size_t nr_to_shrink) ++{ ++ struct page *p; ++ size_t i; ++ ++ lockdep_assert_held(&pool->pool_lock); ++ ++ for (i = 0; i < nr_to_shrink && !kbase_mem_pool_is_empty(pool); i++) { ++ p = kbase_mem_pool_remove_locked(pool); ++ kbase_mem_pool_free_page(pool, p); ++ } ++ ++ return i; ++} ++ ++static size_t kbase_mem_pool_shrink(struct kbase_mem_pool *pool, ++ size_t nr_to_shrink) ++{ ++ size_t nr_freed; ++ ++ kbase_mem_pool_lock(pool); ++ nr_freed = kbase_mem_pool_shrink_locked(pool, nr_to_shrink); ++ kbase_mem_pool_unlock(pool); ++ ++ return nr_freed; ++} ++ ++int kbase_mem_pool_grow(struct kbase_mem_pool *pool, ++ size_t nr_to_grow) ++{ ++ struct page *p; ++ size_t i; ++ ++ for (i = 0; i < nr_to_grow; i++) { ++ p = kbase_mem_alloc_page(pool); ++ if (!p) ++ return -ENOMEM; ++ kbase_mem_pool_add(pool, p); ++ } ++ ++ return 0; ++} ++ ++void kbase_mem_pool_trim(struct kbase_mem_pool *pool, size_t new_size) ++{ ++ size_t cur_size; ++ int err = 0; ++ ++ cur_size = kbase_mem_pool_size(pool); ++ ++ if (new_size > pool->max_size) ++ new_size = pool->max_size; ++ ++ if (new_size < cur_size) ++ kbase_mem_pool_shrink(pool, cur_size - new_size); ++ else if (new_size > cur_size) ++ err = kbase_mem_pool_grow(pool, new_size - cur_size); ++ ++ if (err) { ++ size_t grown_size = kbase_mem_pool_size(pool); ++ ++ dev_warn(pool->kbdev->dev, ++ "Mem pool not grown to the required size of %zu bytes, grown for additional %zu bytes instead!\n", ++ (new_size - cur_size), (grown_size - cur_size)); ++ } ++} ++ ++void kbase_mem_pool_set_max_size(struct kbase_mem_pool *pool, size_t max_size) ++{ ++ size_t cur_size; ++ size_t nr_to_shrink; ++ ++ kbase_mem_pool_lock(pool); ++ ++ pool->max_size = max_size; ++ ++ cur_size = kbase_mem_pool_size(pool); ++ if (max_size < cur_size) { ++ nr_to_shrink = cur_size - max_size; ++ kbase_mem_pool_shrink_locked(pool, nr_to_shrink); ++ } ++ ++ kbase_mem_pool_unlock(pool); ++} ++ ++ ++static unsigned long kbase_mem_pool_reclaim_count_objects(struct shrinker *s, ++ struct shrink_control *sc) ++{ ++ struct kbase_mem_pool *pool; ++ ++ pool = container_of(s, struct kbase_mem_pool, reclaim); ++ pool_dbg(pool, "reclaim count: %zu\n", kbase_mem_pool_size(pool)); ++ return kbase_mem_pool_size(pool); ++} ++ ++static unsigned long kbase_mem_pool_reclaim_scan_objects(struct shrinker *s, ++ struct shrink_control *sc) ++{ ++ struct kbase_mem_pool *pool; ++ unsigned long freed; ++ ++ pool = container_of(s, struct kbase_mem_pool, reclaim); ++ ++ pool_dbg(pool, "reclaim scan %ld:\n", sc->nr_to_scan); ++ ++ freed = kbase_mem_pool_shrink(pool, sc->nr_to_scan); ++ ++ pool_dbg(pool, "reclaim freed %ld pages\n", freed); ++ ++ return freed; ++} ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) ++static int kbase_mem_pool_reclaim_shrink(struct shrinker *s, ++ struct shrink_control *sc) ++{ ++ if (sc->nr_to_scan == 0) ++ return kbase_mem_pool_reclaim_count_objects(s, sc); ++ ++ return kbase_mem_pool_reclaim_scan_objects(s, sc); ++} ++#endif ++ ++int kbase_mem_pool_init(struct kbase_mem_pool *pool, ++ size_t max_size, ++ size_t order, ++ struct kbase_device *kbdev, ++ struct kbase_mem_pool *next_pool) ++{ ++ pool->cur_size = 0; ++ pool->max_size = max_size; ++ pool->order = order; ++ pool->kbdev = kbdev; ++ pool->next_pool = next_pool; ++ ++ spin_lock_init(&pool->pool_lock); ++ INIT_LIST_HEAD(&pool->page_list); ++ ++ /* Register shrinker */ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) ++ pool->reclaim.shrink = kbase_mem_pool_reclaim_shrink; ++#else ++ pool->reclaim.count_objects = kbase_mem_pool_reclaim_count_objects; ++ pool->reclaim.scan_objects = kbase_mem_pool_reclaim_scan_objects; ++#endif ++ pool->reclaim.seeks = DEFAULT_SEEKS; ++ /* Kernel versions prior to 3.1 : ++ * struct shrinker does not define batch */ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 1, 0) ++ pool->reclaim.batch = 0; ++#endif ++ register_shrinker(&pool->reclaim); ++ ++ pool_dbg(pool, "initialized\n"); ++ ++ return 0; ++} ++ ++void kbase_mem_pool_term(struct kbase_mem_pool *pool) ++{ ++ struct kbase_mem_pool *next_pool = pool->next_pool; ++ struct page *p; ++ size_t nr_to_spill = 0; ++ LIST_HEAD(spill_list); ++ int i; ++ ++ pool_dbg(pool, "terminate()\n"); ++ ++ unregister_shrinker(&pool->reclaim); ++ ++ kbase_mem_pool_lock(pool); ++ pool->max_size = 0; ++ ++ if (next_pool && !kbase_mem_pool_is_full(next_pool)) { ++ /* Spill to next pool (may overspill) */ ++ nr_to_spill = kbase_mem_pool_capacity(next_pool); ++ nr_to_spill = min(kbase_mem_pool_size(pool), nr_to_spill); ++ ++ /* Zero pages first without holding the next_pool lock */ ++ for (i = 0; i < nr_to_spill; i++) { ++ p = kbase_mem_pool_remove_locked(pool); ++ kbase_mem_pool_zero_page(pool, p); ++ list_add(&p->lru, &spill_list); ++ } ++ } ++ ++ while (!kbase_mem_pool_is_empty(pool)) { ++ /* Free remaining pages to kernel */ ++ p = kbase_mem_pool_remove_locked(pool); ++ kbase_mem_pool_free_page(pool, p); ++ } ++ ++ kbase_mem_pool_unlock(pool); ++ ++ if (next_pool && nr_to_spill) { ++ /* Add new page list to next_pool */ ++ kbase_mem_pool_add_list(next_pool, &spill_list, nr_to_spill); ++ ++ pool_dbg(pool, "terminate() spilled %zu pages\n", nr_to_spill); ++ } ++ ++ pool_dbg(pool, "terminated\n"); ++} ++ ++struct page *kbase_mem_pool_alloc(struct kbase_mem_pool *pool) ++{ ++ struct page *p; ++ ++ do { ++ pool_dbg(pool, "alloc()\n"); ++ p = kbase_mem_pool_remove(pool); ++ ++ if (p) ++ return p; ++ ++ pool = pool->next_pool; ++ } while (pool); ++ ++ return NULL; ++} ++ ++void kbase_mem_pool_free(struct kbase_mem_pool *pool, struct page *p, ++ bool dirty) ++{ ++ struct kbase_mem_pool *next_pool = pool->next_pool; ++ ++ pool_dbg(pool, "free()\n"); ++ ++ if (!kbase_mem_pool_is_full(pool)) { ++ /* Add to our own pool */ ++ if (dirty) ++ kbase_mem_pool_sync_page(pool, p); ++ ++ kbase_mem_pool_add(pool, p); ++ } else if (next_pool && !kbase_mem_pool_is_full(next_pool)) { ++ /* Spill to next pool */ ++ kbase_mem_pool_spill(next_pool, p); ++ } else { ++ /* Free page */ ++ kbase_mem_pool_free_page(pool, p); ++ } ++} ++ ++int kbase_mem_pool_alloc_pages(struct kbase_mem_pool *pool, size_t nr_4k_pages, ++ struct tagged_addr *pages, bool partial_allowed) ++{ ++ struct page *p; ++ size_t nr_from_pool; ++ size_t i = 0; ++ int err = -ENOMEM; ++ size_t nr_pages_internal; ++ ++ nr_pages_internal = nr_4k_pages / (1u << (pool->order)); ++ ++ if (nr_pages_internal * (1u << pool->order) != nr_4k_pages) ++ return -EINVAL; ++ ++ pool_dbg(pool, "alloc_pages(4k=%zu):\n", nr_4k_pages); ++ pool_dbg(pool, "alloc_pages(internal=%zu):\n", nr_pages_internal); ++ ++ /* Get pages from this pool */ ++ kbase_mem_pool_lock(pool); ++ nr_from_pool = min(nr_pages_internal, kbase_mem_pool_size(pool)); ++ while (nr_from_pool--) { ++ int j; ++ p = kbase_mem_pool_remove_locked(pool); ++ if (pool->order) { ++ pages[i++] = as_tagged_tag(page_to_phys(p), ++ HUGE_HEAD | HUGE_PAGE); ++ for (j = 1; j < (1u << pool->order); j++) ++ pages[i++] = as_tagged_tag(page_to_phys(p) + ++ PAGE_SIZE * j, ++ HUGE_PAGE); ++ } else { ++ pages[i++] = as_tagged(page_to_phys(p)); ++ } ++ } ++ kbase_mem_pool_unlock(pool); ++ ++ if (i != nr_4k_pages && pool->next_pool) { ++ /* Allocate via next pool */ ++ err = kbase_mem_pool_alloc_pages(pool->next_pool, ++ nr_4k_pages - i, pages + i, partial_allowed); ++ ++ if (err < 0) ++ goto err_rollback; ++ ++ i += err; ++ } else { ++ /* Get any remaining pages from kernel */ ++ while (i != nr_4k_pages) { ++ p = kbase_mem_alloc_page(pool); ++ if (!p) { ++ if (partial_allowed) ++ goto done; ++ else ++ goto err_rollback; ++ } ++ ++ if (pool->order) { ++ int j; ++ ++ pages[i++] = as_tagged_tag(page_to_phys(p), ++ HUGE_PAGE | ++ HUGE_HEAD); ++ for (j = 1; j < (1u << pool->order); j++) { ++ phys_addr_t phys; ++ ++ phys = page_to_phys(p) + PAGE_SIZE * j; ++ pages[i++] = as_tagged_tag(phys, ++ HUGE_PAGE); ++ } ++ } else { ++ pages[i++] = as_tagged(page_to_phys(p)); ++ } ++ } ++ } ++ ++done: ++ pool_dbg(pool, "alloc_pages(%zu) done\n", i); ++ ++ return i; ++ ++err_rollback: ++ kbase_mem_pool_free_pages(pool, i, pages, NOT_DIRTY, NOT_RECLAIMED); ++ return err; ++} ++ ++static void kbase_mem_pool_add_array(struct kbase_mem_pool *pool, ++ size_t nr_pages, struct tagged_addr *pages, ++ bool zero, bool sync) ++{ ++ struct page *p; ++ size_t nr_to_pool = 0; ++ LIST_HEAD(new_page_list); ++ size_t i; ++ ++ if (!nr_pages) ++ return; ++ ++ pool_dbg(pool, "add_array(%zu, zero=%d, sync=%d):\n", ++ nr_pages, zero, sync); ++ ++ /* Zero/sync pages first without holding the pool lock */ ++ for (i = 0; i < nr_pages; i++) { ++ if (unlikely(!as_phys_addr_t(pages[i]))) ++ continue; ++ ++ if (is_huge_head(pages[i]) || !is_huge(pages[i])) { ++ p = phys_to_page(as_phys_addr_t(pages[i])); ++ if (zero) ++ kbase_mem_pool_zero_page(pool, p); ++ else if (sync) ++ kbase_mem_pool_sync_page(pool, p); ++ ++ list_add(&p->lru, &new_page_list); ++ nr_to_pool++; ++ } ++ pages[i] = as_tagged(0); ++ } ++ ++ /* Add new page list to pool */ ++ kbase_mem_pool_add_list(pool, &new_page_list, nr_to_pool); ++ ++ pool_dbg(pool, "add_array(%zu) added %zu pages\n", ++ nr_pages, nr_to_pool); ++} ++ ++void kbase_mem_pool_free_pages(struct kbase_mem_pool *pool, size_t nr_pages, ++ struct tagged_addr *pages, bool dirty, bool reclaimed) ++{ ++ struct kbase_mem_pool *next_pool = pool->next_pool; ++ struct page *p; ++ size_t nr_to_pool; ++ LIST_HEAD(to_pool_list); ++ size_t i = 0; ++ ++ pool_dbg(pool, "free_pages(%zu):\n", nr_pages); ++ ++ if (!reclaimed) { ++ /* Add to this pool */ ++ nr_to_pool = kbase_mem_pool_capacity(pool); ++ nr_to_pool = min(nr_pages, nr_to_pool); ++ ++ kbase_mem_pool_add_array(pool, nr_to_pool, pages, false, dirty); ++ ++ i += nr_to_pool; ++ ++ if (i != nr_pages && next_pool) { ++ /* Spill to next pool (may overspill) */ ++ nr_to_pool = kbase_mem_pool_capacity(next_pool); ++ nr_to_pool = min(nr_pages - i, nr_to_pool); ++ ++ kbase_mem_pool_add_array(next_pool, nr_to_pool, ++ pages + i, true, dirty); ++ i += nr_to_pool; ++ } ++ } ++ ++ /* Free any remaining pages to kernel */ ++ for (; i < nr_pages; i++) { ++ if (unlikely(!as_phys_addr_t(pages[i]))) ++ continue; ++ ++ if (is_huge(pages[i]) && !is_huge_head(pages[i])) { ++ pages[i] = as_tagged(0); ++ continue; ++ } ++ ++ p = phys_to_page(as_phys_addr_t(pages[i])); ++ ++ if (reclaimed) ++ zone_page_state_add(-1, page_zone(p), ++ NR_SLAB_RECLAIMABLE); ++ ++ kbase_mem_pool_free_page(pool, p); ++ pages[i] = as_tagged(0); ++ } ++ ++ pool_dbg(pool, "free_pages(%zu) done\n", nr_pages); ++} +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_pool_debugfs.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_pool_debugfs.c +new file mode 100755 +index 000000000000..319cf2568aba +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_pool_debugfs.c +@@ -0,0 +1,88 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2015, 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include ++#include ++ ++#include ++ ++#ifdef CONFIG_DEBUG_FS ++ ++static int kbase_mem_pool_debugfs_size_get(void *data, u64 *val) ++{ ++ struct kbase_mem_pool *pool = (struct kbase_mem_pool *)data; ++ ++ *val = kbase_mem_pool_size(pool); ++ ++ return 0; ++} ++ ++static int kbase_mem_pool_debugfs_size_set(void *data, u64 val) ++{ ++ struct kbase_mem_pool *pool = (struct kbase_mem_pool *)data; ++ ++ kbase_mem_pool_trim(pool, val); ++ ++ return 0; ++} ++ ++DEFINE_SIMPLE_ATTRIBUTE(kbase_mem_pool_debugfs_size_fops, ++ kbase_mem_pool_debugfs_size_get, ++ kbase_mem_pool_debugfs_size_set, ++ "%llu\n"); ++ ++static int kbase_mem_pool_debugfs_max_size_get(void *data, u64 *val) ++{ ++ struct kbase_mem_pool *pool = (struct kbase_mem_pool *)data; ++ ++ *val = kbase_mem_pool_max_size(pool); ++ ++ return 0; ++} ++ ++static int kbase_mem_pool_debugfs_max_size_set(void *data, u64 val) ++{ ++ struct kbase_mem_pool *pool = (struct kbase_mem_pool *)data; ++ ++ kbase_mem_pool_set_max_size(pool, val); ++ ++ return 0; ++} ++ ++DEFINE_SIMPLE_ATTRIBUTE(kbase_mem_pool_debugfs_max_size_fops, ++ kbase_mem_pool_debugfs_max_size_get, ++ kbase_mem_pool_debugfs_max_size_set, ++ "%llu\n"); ++ ++void kbase_mem_pool_debugfs_init(struct dentry *parent, ++ struct kbase_mem_pool *pool, ++ struct kbase_mem_pool *lp_pool) ++{ ++ debugfs_create_file("mem_pool_size", S_IRUGO | S_IWUSR, parent, ++ pool, &kbase_mem_pool_debugfs_size_fops); ++ ++ debugfs_create_file("mem_pool_max_size", S_IRUGO | S_IWUSR, parent, ++ pool, &kbase_mem_pool_debugfs_max_size_fops); ++ ++ debugfs_create_file("lp_mem_pool_size", S_IRUGO | S_IWUSR, parent, ++ lp_pool, &kbase_mem_pool_debugfs_size_fops); ++ ++ debugfs_create_file("lp_mem_pool_max_size", S_IRUGO | S_IWUSR, parent, ++ lp_pool, &kbase_mem_pool_debugfs_max_size_fops); ++} ++ ++#endif /* CONFIG_DEBUG_FS */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_pool_debugfs.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_pool_debugfs.h +new file mode 100755 +index 000000000000..496eaf3f1e1a +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_pool_debugfs.h +@@ -0,0 +1,40 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2015, 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KBASE_MEM_POOL_DEBUGFS_H ++#define _KBASE_MEM_POOL_DEBUGFS_H ++ ++#include ++ ++/** ++ * kbase_mem_pool_debugfs_init - add debugfs knobs for @pool ++ * @parent: Parent debugfs dentry ++ * @pool: Memory pool of small pages to control ++ * @lp_pool: Memory pool of large pages to control ++ * ++ * Adds four debugfs files under @parent: ++ * - mem_pool_size: get/set the current size of @pool ++ * - mem_pool_max_size: get/set the max size of @pool ++ * - lp_mem_pool_size: get/set the current size of @lp_pool ++ * - lp_mem_pool_max_size: get/set the max size of @lp_pool ++ */ ++void kbase_mem_pool_debugfs_init(struct dentry *parent, ++ struct kbase_mem_pool *pool, ++ struct kbase_mem_pool *lp_pool); ++ ++#endif /*_KBASE_MEM_POOL_DEBUGFS_H*/ ++ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_profile_debugfs.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_profile_debugfs.c +new file mode 100755 +index 000000000000..d58fd8d62fde +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_profile_debugfs.c +@@ -0,0 +1,121 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include ++ ++#ifdef CONFIG_DEBUG_FS ++ ++/** Show callback for the @c mem_profile debugfs file. ++ * ++ * This function is called to get the contents of the @c mem_profile debugfs ++ * file. This is a report of current memory usage and distribution in userspace. ++ * ++ * @param sfile The debugfs entry ++ * @param data Data associated with the entry ++ * ++ * @return 0 if it successfully prints data in debugfs entry file, non-zero otherwise ++ */ ++static int kbasep_mem_profile_seq_show(struct seq_file *sfile, void *data) ++{ ++ struct kbase_context *kctx = sfile->private; ++ ++ mutex_lock(&kctx->mem_profile_lock); ++ ++ seq_write(sfile, kctx->mem_profile_data, kctx->mem_profile_size); ++ ++ seq_putc(sfile, '\n'); ++ ++ mutex_unlock(&kctx->mem_profile_lock); ++ ++ return 0; ++} ++ ++/* ++ * File operations related to debugfs entry for mem_profile ++ */ ++static int kbasep_mem_profile_debugfs_open(struct inode *in, struct file *file) ++{ ++ return single_open(file, kbasep_mem_profile_seq_show, in->i_private); ++} ++ ++static const struct file_operations kbasep_mem_profile_debugfs_fops = { ++ .open = kbasep_mem_profile_debugfs_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = single_release, ++}; ++ ++int kbasep_mem_profile_debugfs_insert(struct kbase_context *kctx, char *data, ++ size_t size) ++{ ++ int err = 0; ++ ++ mutex_lock(&kctx->mem_profile_lock); ++ ++ dev_dbg(kctx->kbdev->dev, "initialised: %d", ++ kbase_ctx_flag(kctx, KCTX_MEM_PROFILE_INITIALIZED)); ++ ++ if (!kbase_ctx_flag(kctx, KCTX_MEM_PROFILE_INITIALIZED)) { ++ if (!debugfs_create_file("mem_profile", S_IRUGO, ++ kctx->kctx_dentry, kctx, ++ &kbasep_mem_profile_debugfs_fops)) { ++ err = -EAGAIN; ++ } else { ++ kbase_ctx_flag_set(kctx, ++ KCTX_MEM_PROFILE_INITIALIZED); ++ } ++ } ++ ++ if (kbase_ctx_flag(kctx, KCTX_MEM_PROFILE_INITIALIZED)) { ++ kfree(kctx->mem_profile_data); ++ kctx->mem_profile_data = data; ++ kctx->mem_profile_size = size; ++ } else { ++ kfree(data); ++ } ++ ++ dev_dbg(kctx->kbdev->dev, "returning: %d, initialised: %d", ++ err, kbase_ctx_flag(kctx, KCTX_MEM_PROFILE_INITIALIZED)); ++ ++ mutex_unlock(&kctx->mem_profile_lock); ++ ++ return err; ++} ++ ++void kbasep_mem_profile_debugfs_remove(struct kbase_context *kctx) ++{ ++ mutex_lock(&kctx->mem_profile_lock); ++ ++ dev_dbg(kctx->kbdev->dev, "initialised: %d", ++ kbase_ctx_flag(kctx, KCTX_MEM_PROFILE_INITIALIZED)); ++ ++ kfree(kctx->mem_profile_data); ++ kctx->mem_profile_data = NULL; ++ kctx->mem_profile_size = 0; ++ ++ mutex_unlock(&kctx->mem_profile_lock); ++} ++ ++#else /* CONFIG_DEBUG_FS */ ++ ++int kbasep_mem_profile_debugfs_insert(struct kbase_context *kctx, char *data, ++ size_t size) ++{ ++ kfree(data); ++ return 0; ++} ++#endif /* CONFIG_DEBUG_FS */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_profile_debugfs.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_profile_debugfs.h +new file mode 100755 +index 000000000000..a1dc2e0b165b +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_profile_debugfs.h +@@ -0,0 +1,59 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/** ++ * @file mali_kbase_mem_profile_debugfs.h ++ * Header file for mem profiles entries in debugfs ++ * ++ */ ++ ++#ifndef _KBASE_MEM_PROFILE_DEBUGFS_H ++#define _KBASE_MEM_PROFILE_DEBUGFS_H ++ ++#include ++#include ++ ++/** ++ * @brief Remove entry from Mali memory profile debugfs ++ */ ++void kbasep_mem_profile_debugfs_remove(struct kbase_context *kctx); ++ ++/** ++ * @brief Insert @p data to the debugfs file so it can be read by userspace ++ * ++ * The function takes ownership of @p data and frees it later when new data ++ * is inserted. ++ * ++ * If the debugfs entry corresponding to the @p kctx doesn't exist, ++ * an attempt will be made to create it. ++ * ++ * @param kctx The context whose debugfs file @p data should be inserted to ++ * @param data A NULL-terminated string to be inserted to the debugfs file, ++ * without the trailing new line character ++ * @param size The length of the @p data string ++ * @return 0 if @p data inserted correctly ++ * -EAGAIN in case of error ++ * @post @ref mem_profile_initialized will be set to @c true ++ * the first time this function succeeds. ++ */ ++int kbasep_mem_profile_debugfs_insert(struct kbase_context *kctx, char *data, ++ size_t size); ++ ++#endif /*_KBASE_MEM_PROFILE_DEBUGFS_H*/ ++ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_profile_debugfs_buf_size.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_profile_debugfs_buf_size.h +new file mode 100755 +index 000000000000..82f0702974c2 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_profile_debugfs_buf_size.h +@@ -0,0 +1,33 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/** ++ * @file mali_kbase_mem_profile_debugfs_buf_size.h ++ * Header file for the size of the buffer to accumulate the histogram report text in ++ */ ++ ++#ifndef _KBASE_MEM_PROFILE_DEBUGFS_BUF_SIZE_H_ ++#define _KBASE_MEM_PROFILE_DEBUGFS_BUF_SIZE_H_ ++ ++/** ++ * The size of the buffer to accumulate the histogram report text in ++ * @see @ref CCTXP_HIST_BUF_SIZE_MAX_LENGTH_REPORT ++ */ ++#define KBASE_MEM_PROFILE_MAX_BUF_SIZE ((size_t) (64 + ((80 + (56 * 64)) * 15) + 56)) ++ ++#endif /*_KBASE_MEM_PROFILE_DEBUGFS_BUF_SIZE_H_*/ ++ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mmu.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mmu.c +new file mode 100755 +index 000000000000..c63269aed53c +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mmu.c +@@ -0,0 +1,2138 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/** ++ * @file mali_kbase_mmu.c ++ * Base kernel MMU management. ++ */ ++ ++/* #define DEBUG 1 */ ++#include ++#include ++#include ++#include ++#if defined(CONFIG_MALI_BIFROST_GATOR_SUPPORT) ++#include ++#endif ++#include ++#include ++#include ++ ++#define beenthere(kctx, f, a...) dev_dbg(kctx->kbdev->dev, "%s:" f, __func__, ##a) ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#define KBASE_MMU_PAGE_ENTRIES 512 ++ ++/** ++ * kbase_mmu_flush_invalidate() - Flush and invalidate the GPU caches. ++ * @kctx: The KBase context. ++ * @vpfn: The virtual page frame number to start the flush on. ++ * @nr: The number of pages to flush. ++ * @sync: Set if the operation should be synchronous or not. ++ * ++ * Issue a cache flush + invalidate to the GPU caches and invalidate the TLBs. ++ * ++ * If sync is not set then transactions still in flight when the flush is issued ++ * may use the old page tables and the data they write will not be written out ++ * to memory, this function returns after the flush has been issued but ++ * before all accesses which might effect the flushed region have completed. ++ * ++ * If sync is set then accesses in the flushed region will be drained ++ * before data is flush and invalidated through L1, L2 and into memory, ++ * after which point this function will return. ++ */ ++static void kbase_mmu_flush_invalidate(struct kbase_context *kctx, ++ u64 vpfn, size_t nr, bool sync); ++ ++/** ++ * kbase_mmu_sync_pgd - sync page directory to memory ++ * @kbdev: Device pointer. ++ * @handle: Address of DMA region. ++ * @size: Size of the region to sync. ++ * ++ * This should be called after each page directory update. ++ */ ++ ++static void kbase_mmu_sync_pgd(struct kbase_device *kbdev, ++ dma_addr_t handle, size_t size) ++{ ++ /* If page table is not coherent then ensure the gpu can read ++ * the pages from memory ++ */ ++ if (kbdev->system_coherency != COHERENCY_ACE) ++ dma_sync_single_for_device(kbdev->dev, handle, size, ++ DMA_TO_DEVICE); ++} ++ ++/* ++ * Definitions: ++ * - PGD: Page Directory. ++ * - PTE: Page Table Entry. A 64bit value pointing to the next ++ * level of translation ++ * - ATE: Address Transation Entry. A 64bit value pointing to ++ * a 4kB physical page. ++ */ ++ ++static void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, ++ struct kbase_as *as, const char *reason_str); ++ ++ ++static size_t make_multiple(size_t minimum, size_t multiple) ++{ ++ size_t remainder = minimum % multiple; ++ ++ if (remainder == 0) ++ return minimum; ++ ++ return minimum + multiple - remainder; ++} ++ ++void page_fault_worker(struct work_struct *data) ++{ ++ u64 fault_pfn; ++ u32 fault_status; ++ size_t new_pages; ++ size_t fault_rel_pfn; ++ struct kbase_as *faulting_as; ++ int as_no; ++ struct kbase_context *kctx; ++ struct kbase_device *kbdev; ++ struct kbase_va_region *region; ++ int err; ++ bool grown = false; ++ ++ faulting_as = container_of(data, struct kbase_as, work_pagefault); ++ fault_pfn = faulting_as->fault_addr >> PAGE_SHIFT; ++ as_no = faulting_as->number; ++ ++ kbdev = container_of(faulting_as, struct kbase_device, as[as_no]); ++ ++ /* Grab the context that was already refcounted in kbase_mmu_interrupt(). ++ * Therefore, it cannot be scheduled out of this AS until we explicitly release it ++ */ ++ kctx = kbasep_js_runpool_lookup_ctx_noretain(kbdev, as_no); ++ if (WARN_ON(!kctx)) { ++ atomic_dec(&kbdev->faults_pending); ++ return; ++ } ++ ++ KBASE_DEBUG_ASSERT(kctx->kbdev == kbdev); ++ ++ if (unlikely(faulting_as->protected_mode)) ++ { ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Protected mode fault"); ++ kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx, ++ KBASE_MMU_FAULT_TYPE_PAGE); ++ ++ goto fault_done; ++ } ++ ++ fault_status = faulting_as->fault_status; ++ switch (fault_status & AS_FAULTSTATUS_EXCEPTION_CODE_MASK) { ++ ++ case AS_FAULTSTATUS_EXCEPTION_CODE_TRANSLATION_FAULT: ++ /* need to check against the region to handle this one */ ++ break; ++ ++ case AS_FAULTSTATUS_EXCEPTION_CODE_PERMISSION_FAULT: ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Permission failure"); ++ goto fault_done; ++ ++ case AS_FAULTSTATUS_EXCEPTION_CODE_TRANSTAB_BUS_FAULT: ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Translation table bus fault"); ++ goto fault_done; ++ ++ case AS_FAULTSTATUS_EXCEPTION_CODE_ACCESS_FLAG: ++ /* nothing to do, but we don't expect this fault currently */ ++ dev_warn(kbdev->dev, "Access flag unexpectedly set"); ++ goto fault_done; ++ ++ case AS_FAULTSTATUS_EXCEPTION_CODE_ADDRESS_SIZE_FAULT: ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Address size fault"); ++ else ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Unknown fault code"); ++ goto fault_done; ++ ++ case AS_FAULTSTATUS_EXCEPTION_CODE_MEMORY_ATTRIBUTES_FAULT: ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Memory attributes fault"); ++ else ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Unknown fault code"); ++ goto fault_done; ++ ++ default: ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Unknown fault code"); ++ goto fault_done; ++ } ++ ++ /* so we have a translation fault, let's see if it is for growable ++ * memory */ ++ kbase_gpu_vm_lock(kctx); ++ ++ region = kbase_region_tracker_find_region_enclosing_address(kctx, ++ faulting_as->fault_addr); ++ if (!region || region->flags & KBASE_REG_FREE) { ++ kbase_gpu_vm_unlock(kctx); ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Memory is not mapped on the GPU"); ++ goto fault_done; ++ } ++ ++ if (region->gpu_alloc->type == KBASE_MEM_TYPE_IMPORTED_UMM) { ++ kbase_gpu_vm_unlock(kctx); ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "DMA-BUF is not mapped on the GPU"); ++ goto fault_done; ++ } ++ ++ if ((region->flags & GROWABLE_FLAGS_REQUIRED) ++ != GROWABLE_FLAGS_REQUIRED) { ++ kbase_gpu_vm_unlock(kctx); ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Memory is not growable"); ++ goto fault_done; ++ } ++ ++ if ((region->flags & KBASE_REG_DONT_NEED)) { ++ kbase_gpu_vm_unlock(kctx); ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Don't need memory can't be grown"); ++ goto fault_done; ++ } ++ ++ /* find the size we need to grow it by */ ++ /* we know the result fit in a size_t due to kbase_region_tracker_find_region_enclosing_address ++ * validating the fault_adress to be within a size_t from the start_pfn */ ++ fault_rel_pfn = fault_pfn - region->start_pfn; ++ ++ if (fault_rel_pfn < kbase_reg_current_backed_size(region)) { ++ dev_dbg(kbdev->dev, "Page fault @ 0x%llx in allocated region 0x%llx-0x%llx of growable TMEM: Ignoring", ++ faulting_as->fault_addr, region->start_pfn, ++ region->start_pfn + ++ kbase_reg_current_backed_size(region)); ++ ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ ++ kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx, ++ KBASE_MMU_FAULT_TYPE_PAGE); ++ /* [1] in case another page fault occurred while we were ++ * handling the (duplicate) page fault we need to ensure we ++ * don't loose the other page fault as result of us clearing ++ * the MMU IRQ. Therefore, after we clear the MMU IRQ we send ++ * an UNLOCK command that will retry any stalled memory ++ * transaction (which should cause the other page fault to be ++ * raised again). ++ */ ++ kbase_mmu_hw_do_operation(kbdev, faulting_as, NULL, 0, 0, ++ AS_COMMAND_UNLOCK, 1); ++ ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ ++ kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx, ++ KBASE_MMU_FAULT_TYPE_PAGE); ++ kbase_gpu_vm_unlock(kctx); ++ ++ goto fault_done; ++ } ++ ++ new_pages = make_multiple(fault_rel_pfn - ++ kbase_reg_current_backed_size(region) + 1, ++ region->extent); ++ ++ /* cap to max vsize */ ++ if (new_pages + kbase_reg_current_backed_size(region) > ++ region->nr_pages) ++ new_pages = region->nr_pages - ++ kbase_reg_current_backed_size(region); ++ ++ if (0 == new_pages) { ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ ++ /* Duplicate of a fault we've already handled, nothing to do */ ++ kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx, ++ KBASE_MMU_FAULT_TYPE_PAGE); ++ /* See comment [1] about UNLOCK usage */ ++ kbase_mmu_hw_do_operation(kbdev, faulting_as, NULL, 0, 0, ++ AS_COMMAND_UNLOCK, 1); ++ ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ ++ kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx, ++ KBASE_MMU_FAULT_TYPE_PAGE); ++ kbase_gpu_vm_unlock(kctx); ++ goto fault_done; ++ } ++ ++ if (kbase_alloc_phy_pages_helper(region->gpu_alloc, new_pages) == 0) { ++ if (region->gpu_alloc != region->cpu_alloc) { ++ if (kbase_alloc_phy_pages_helper( ++ region->cpu_alloc, new_pages) == 0) { ++ grown = true; ++ } else { ++ kbase_free_phy_pages_helper(region->gpu_alloc, ++ new_pages); ++ } ++ } else { ++ grown = true; ++ } ++ } ++ ++ ++ if (grown) { ++ u64 pfn_offset; ++ u32 op; ++ ++ /* alloc success */ ++ KBASE_DEBUG_ASSERT(kbase_reg_current_backed_size(region) <= region->nr_pages); ++ ++ /* set up the new pages */ ++ pfn_offset = kbase_reg_current_backed_size(region) - new_pages; ++ /* ++ * Note: ++ * Issuing an MMU operation will unlock the MMU and cause the ++ * translation to be replayed. If the page insertion fails then ++ * rather then trying to continue the context should be killed ++ * so the no_flush version of insert_pages is used which allows ++ * us to unlock the MMU as we see fit. ++ */ ++ err = kbase_mmu_insert_pages_no_flush(kctx, ++ region->start_pfn + pfn_offset, ++ &kbase_get_gpu_phy_pages(region)[pfn_offset], ++ new_pages, region->flags); ++ if (err) { ++ kbase_free_phy_pages_helper(region->gpu_alloc, new_pages); ++ if (region->gpu_alloc != region->cpu_alloc) ++ kbase_free_phy_pages_helper(region->cpu_alloc, ++ new_pages); ++ kbase_gpu_vm_unlock(kctx); ++ /* The locked VA region will be unlocked and the cache invalidated in here */ ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Page table update failure"); ++ goto fault_done; ++ } ++#if defined(CONFIG_MALI_BIFROST_GATOR_SUPPORT) ++ kbase_trace_mali_page_fault_insert_pages(as_no, new_pages); ++#endif ++ KBASE_TLSTREAM_AUX_PAGEFAULT(kctx->id, (u64)new_pages); ++ ++ /* AS transaction begin */ ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ ++ /* flush L2 and unlock the VA (resumes the MMU) */ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_6367)) ++ op = AS_COMMAND_FLUSH; ++ else ++ op = AS_COMMAND_FLUSH_PT; ++ ++ /* clear MMU interrupt - this needs to be done after updating ++ * the page tables but before issuing a FLUSH command. The ++ * FLUSH cmd has a side effect that it restarts stalled memory ++ * transactions in other address spaces which may cause ++ * another fault to occur. If we didn't clear the interrupt at ++ * this stage a new IRQ might not be raised when the GPU finds ++ * a MMU IRQ is already pending. ++ */ ++ kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx, ++ KBASE_MMU_FAULT_TYPE_PAGE); ++ ++ kbase_mmu_hw_do_operation(kbdev, faulting_as, kctx, ++ faulting_as->fault_addr >> PAGE_SHIFT, ++ new_pages, ++ op, 1); ++ ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ /* AS transaction end */ ++ ++ /* reenable this in the mask */ ++ kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx, ++ KBASE_MMU_FAULT_TYPE_PAGE); ++ kbase_gpu_vm_unlock(kctx); ++ } else { ++ /* failed to extend, handle as a normal PF */ ++ kbase_gpu_vm_unlock(kctx); ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Page allocation failure"); ++ } ++ ++fault_done: ++ /* ++ * By this point, the fault was handled in some way, ++ * so release the ctx refcount ++ */ ++ kbasep_js_runpool_release_ctx(kbdev, kctx); ++ ++ atomic_dec(&kbdev->faults_pending); ++} ++ ++phys_addr_t kbase_mmu_alloc_pgd(struct kbase_context *kctx) ++{ ++ u64 *page; ++ int i; ++ struct page *p; ++ int new_page_count __maybe_unused; ++ ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ new_page_count = kbase_atomic_add_pages(1, &kctx->used_pages); ++ kbase_atomic_add_pages(1, &kctx->kbdev->memdev.used_pages); ++ ++ p = kbase_mem_pool_alloc(&kctx->mem_pool); ++ if (!p) ++ goto sub_pages; ++ ++ KBASE_TLSTREAM_AUX_PAGESALLOC( ++ kctx->id, ++ (u64)new_page_count); ++ ++ page = kmap(p); ++ if (NULL == page) ++ goto alloc_free; ++ ++ kbase_process_page_usage_inc(kctx, 1); ++ ++ for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++) ++ kctx->kbdev->mmu_mode->entry_invalidate(&page[i]); ++ ++ kbase_mmu_sync_pgd(kctx->kbdev, kbase_dma_addr(p), PAGE_SIZE); ++ ++ kunmap(p); ++ return page_to_phys(p); ++ ++alloc_free: ++ kbase_mem_pool_free(&kctx->mem_pool, p, false); ++sub_pages: ++ kbase_atomic_sub_pages(1, &kctx->used_pages); ++ kbase_atomic_sub_pages(1, &kctx->kbdev->memdev.used_pages); ++ ++ return 0; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_mmu_alloc_pgd); ++ ++/* Given PGD PFN for level N, return PGD PFN for level N+1, allocating the ++ * new table from the pool if needed and possible ++ */ ++static int mmu_get_next_pgd(struct kbase_context *kctx, ++ phys_addr_t *pgd, u64 vpfn, int level) ++{ ++ u64 *page; ++ phys_addr_t target_pgd; ++ struct page *p; ++ ++ KBASE_DEBUG_ASSERT(*pgd); ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ ++ lockdep_assert_held(&kctx->mmu_lock); ++ ++ /* ++ * Architecture spec defines level-0 as being the top-most. ++ * This is a bit unfortunate here, but we keep the same convention. ++ */ ++ vpfn >>= (3 - level) * 9; ++ vpfn &= 0x1FF; ++ ++ p = pfn_to_page(PFN_DOWN(*pgd)); ++ page = kmap(p); ++ if (NULL == page) { ++ dev_warn(kctx->kbdev->dev, "mmu_get_next_pgd: kmap failure\n"); ++ return -EINVAL; ++ } ++ ++ target_pgd = kctx->kbdev->mmu_mode->pte_to_phy_addr(page[vpfn]); ++ ++ if (!target_pgd) { ++ target_pgd = kbase_mmu_alloc_pgd(kctx); ++ if (!target_pgd) { ++ dev_dbg(kctx->kbdev->dev, "mmu_get_next_pgd: kbase_mmu_alloc_pgd failure\n"); ++ kunmap(p); ++ return -ENOMEM; ++ } ++ ++ kctx->kbdev->mmu_mode->entry_set_pte(&page[vpfn], target_pgd); ++ ++ kbase_mmu_sync_pgd(kctx->kbdev, kbase_dma_addr(p), PAGE_SIZE); ++ /* Rely on the caller to update the address space flags. */ ++ } ++ ++ kunmap(p); ++ *pgd = target_pgd; ++ ++ return 0; ++} ++ ++/* ++ * Returns the PGD for the specified level of translation ++ */ ++static int mmu_get_pgd_at_level(struct kbase_context *kctx, ++ u64 vpfn, ++ unsigned int level, ++ phys_addr_t *out_pgd) ++{ ++ phys_addr_t pgd; ++ int l; ++ ++ lockdep_assert_held(&kctx->mmu_lock); ++ pgd = kctx->pgd; ++ ++ for (l = MIDGARD_MMU_TOPLEVEL; l < level; l++) { ++ int err = mmu_get_next_pgd(kctx, &pgd, vpfn, l); ++ /* Handle failure condition */ ++ if (err) { ++ dev_dbg(kctx->kbdev->dev, ++ "%s: mmu_get_next_pgd failure at level %d\n", ++ __func__, l); ++ return err; ++ } ++ } ++ ++ *out_pgd = pgd; ++ ++ return 0; ++} ++ ++#define mmu_get_bottom_pgd(kctx, vpfn, out_pgd) \ ++ mmu_get_pgd_at_level((kctx), (vpfn), MIDGARD_MMU_BOTTOMLEVEL, (out_pgd)) ++ ++ ++static void mmu_insert_pages_failure_recovery(struct kbase_context *kctx, ++ u64 from_vpfn, u64 to_vpfn) ++{ ++ phys_addr_t pgd; ++ u64 vpfn = from_vpfn; ++ struct kbase_mmu_mode const *mmu_mode; ++ ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ KBASE_DEBUG_ASSERT(0 != vpfn); ++ /* 64-bit address range is the max */ ++ KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE)); ++ KBASE_DEBUG_ASSERT(from_vpfn <= to_vpfn); ++ ++ lockdep_assert_held(&kctx->mmu_lock); ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ mmu_mode = kctx->kbdev->mmu_mode; ++ ++ while (vpfn < to_vpfn) { ++ unsigned int i; ++ unsigned int idx = vpfn & 0x1FF; ++ unsigned int count = KBASE_MMU_PAGE_ENTRIES - idx; ++ unsigned int pcount = 0; ++ unsigned int left = to_vpfn - vpfn; ++ unsigned int level; ++ u64 *page; ++ ++ if (count > left) ++ count = left; ++ ++ /* need to check if this is a 2MB page or a 4kB */ ++ pgd = kctx->pgd; ++ ++ for (level = MIDGARD_MMU_TOPLEVEL; ++ level <= MIDGARD_MMU_BOTTOMLEVEL; level++) { ++ idx = (vpfn >> ((3 - level) * 9)) & 0x1FF; ++ page = kmap(phys_to_page(pgd)); ++ if (mmu_mode->ate_is_valid(page[idx], level)) ++ break; /* keep the mapping */ ++ kunmap(phys_to_page(pgd)); ++ pgd = mmu_mode->pte_to_phy_addr(page[idx]); ++ } ++ ++ switch (level) { ++ case MIDGARD_MMU_LEVEL(2): ++ /* remap to single entry to update */ ++ pcount = 1; ++ break; ++ case MIDGARD_MMU_BOTTOMLEVEL: ++ /* page count is the same as the logical count */ ++ pcount = count; ++ break; ++ default: ++ dev_warn(kctx->kbdev->dev, "%sNo support for ATEs at level %d\n", ++ __func__, level); ++ goto next; ++ } ++ ++ /* Invalidate the entries we added */ ++ for (i = 0; i < pcount; i++) ++ mmu_mode->entry_invalidate(&page[idx + i]); ++ ++ kbase_mmu_sync_pgd(kctx->kbdev, ++ kbase_dma_addr(phys_to_page(pgd)) + 8 * idx, ++ 8 * pcount); ++ kunmap(phys_to_page(pgd)); ++ ++next: ++ vpfn += count; ++ } ++} ++ ++/* ++ * Map the single page 'phys' 'nr' of times, starting at GPU PFN 'vpfn' ++ */ ++int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 vpfn, ++ struct tagged_addr phys, size_t nr, ++ unsigned long flags) ++{ ++ phys_addr_t pgd; ++ u64 *pgd_page; ++ /* In case the insert_single_page only partially completes we need to be ++ * able to recover */ ++ bool recover_required = false; ++ u64 recover_vpfn = vpfn; ++ size_t recover_count = 0; ++ size_t remain = nr; ++ int err; ++ struct kbase_mmu_mode const *mmu_mode; ++ ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ KBASE_DEBUG_ASSERT(0 != vpfn); ++ /* 64-bit address range is the max */ ++ KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE)); ++ ++ mmu_mode = kctx->kbdev->mmu_mode; ++ ++ /* Early out if there is nothing to do */ ++ if (nr == 0) ++ return 0; ++ ++ mutex_lock(&kctx->mmu_lock); ++ ++ while (remain) { ++ unsigned int i; ++ unsigned int index = vpfn & 0x1FF; ++ unsigned int count = KBASE_MMU_PAGE_ENTRIES - index; ++ struct page *p; ++ ++ if (count > remain) ++ count = remain; ++ ++ /* ++ * Repeatedly calling mmu_get_bottom_pte() is clearly ++ * suboptimal. We don't have to re-parse the whole tree ++ * each time (just cache the l0-l2 sequence). ++ * On the other hand, it's only a gain when we map more than ++ * 256 pages at once (on average). Do we really care? ++ */ ++ do { ++ err = mmu_get_bottom_pgd(kctx, vpfn, &pgd); ++ if (err != -ENOMEM) ++ break; ++ /* Fill the memory pool with enough pages for ++ * the page walk to succeed ++ */ ++ mutex_unlock(&kctx->mmu_lock); ++ err = kbase_mem_pool_grow(&kctx->mem_pool, ++ MIDGARD_MMU_BOTTOMLEVEL); ++ mutex_lock(&kctx->mmu_lock); ++ } while (!err); ++ if (err) { ++ dev_warn(kctx->kbdev->dev, "kbase_mmu_insert_pages: mmu_get_bottom_pgd failure\n"); ++ if (recover_required) { ++ /* Invalidate the pages we have partially ++ * completed */ ++ mmu_insert_pages_failure_recovery(kctx, ++ recover_vpfn, ++ recover_vpfn + ++ recover_count ++ ); ++ } ++ goto fail_unlock; ++ } ++ ++ p = pfn_to_page(PFN_DOWN(pgd)); ++ pgd_page = kmap(p); ++ if (!pgd_page) { ++ dev_warn(kctx->kbdev->dev, "kbase_mmu_insert_pages: kmap failure\n"); ++ if (recover_required) { ++ /* Invalidate the pages we have partially ++ * completed */ ++ mmu_insert_pages_failure_recovery(kctx, ++ recover_vpfn, ++ recover_vpfn + ++ recover_count ++ ); ++ } ++ err = -ENOMEM; ++ goto fail_unlock; ++ } ++ ++ for (i = 0; i < count; i++) { ++ unsigned int ofs = index + i; ++ ++ /* Fail if the current page is a valid ATE entry */ ++ KBASE_DEBUG_ASSERT(0 == (pgd_page[ofs] & 1UL)); ++ ++ mmu_mode->entry_set_ate(&pgd_page[ofs], ++ phys, flags, ++ MIDGARD_MMU_BOTTOMLEVEL); ++ } ++ ++ vpfn += count; ++ remain -= count; ++ ++ kbase_mmu_sync_pgd(kctx->kbdev, ++ kbase_dma_addr(p) + (index * sizeof(u64)), ++ count * sizeof(u64)); ++ ++ kunmap(p); ++ /* We have started modifying the page table. ++ * If further pages need inserting and fail we need to undo what ++ * has already taken place */ ++ recover_required = true; ++ recover_count += count; ++ } ++ mutex_unlock(&kctx->mmu_lock); ++ kbase_mmu_flush_invalidate(kctx, vpfn, nr, false); ++ return 0; ++ ++fail_unlock: ++ mutex_unlock(&kctx->mmu_lock); ++ kbase_mmu_flush_invalidate(kctx, vpfn, nr, false); ++ return err; ++} ++ ++static inline void cleanup_empty_pte(struct kbase_context *kctx, u64 *pte) ++{ ++ phys_addr_t tmp_pgd; ++ struct page *tmp_p; ++ ++ tmp_pgd = kctx->kbdev->mmu_mode->pte_to_phy_addr(*pte); ++ tmp_p = phys_to_page(tmp_pgd); ++ kbase_mem_pool_free(&kctx->mem_pool, tmp_p, false); ++ kbase_process_page_usage_dec(kctx, 1); ++ kbase_atomic_sub_pages(1, &kctx->used_pages); ++ kbase_atomic_sub_pages(1, &kctx->kbdev->memdev.used_pages); ++} ++ ++int kbase_mmu_insert_pages_no_flush(struct kbase_context *kctx, ++ const u64 start_vpfn, ++ struct tagged_addr *phys, size_t nr, ++ unsigned long flags) ++{ ++ phys_addr_t pgd; ++ u64 *pgd_page; ++ u64 insert_vpfn = start_vpfn; ++ size_t remain = nr; ++ int err; ++ struct kbase_mmu_mode const *mmu_mode; ++ ++ KBASE_DEBUG_ASSERT(kctx); ++ KBASE_DEBUG_ASSERT(start_vpfn); ++ /* 64-bit address range is the max */ ++ KBASE_DEBUG_ASSERT(start_vpfn <= (U64_MAX / PAGE_SIZE)); ++ ++ mmu_mode = kctx->kbdev->mmu_mode; ++ ++ /* Early out if there is nothing to do */ ++ if (nr == 0) ++ return 0; ++ ++ mutex_lock(&kctx->mmu_lock); ++ ++ while (remain) { ++ unsigned int i; ++ unsigned int vindex = insert_vpfn & 0x1FF; ++ unsigned int count = KBASE_MMU_PAGE_ENTRIES - vindex; ++ struct page *p; ++ unsigned int cur_level; ++ ++ if (count > remain) ++ count = remain; ++ ++ if (!vindex && is_huge_head(*phys)) ++ cur_level = MIDGARD_MMU_LEVEL(2); ++ else ++ cur_level = MIDGARD_MMU_BOTTOMLEVEL; ++ ++ /* ++ * Repeatedly calling mmu_get_pgd_at_level() is clearly ++ * suboptimal. We don't have to re-parse the whole tree ++ * each time (just cache the l0-l2 sequence). ++ * On the other hand, it's only a gain when we map more than ++ * 256 pages at once (on average). Do we really care? ++ */ ++ do { ++ err = mmu_get_pgd_at_level(kctx, insert_vpfn, cur_level, ++ &pgd); ++ if (err != -ENOMEM) ++ break; ++ /* Fill the memory pool with enough pages for ++ * the page walk to succeed ++ */ ++ mutex_unlock(&kctx->mmu_lock); ++ err = kbase_mem_pool_grow(&kctx->mem_pool, ++ cur_level); ++ mutex_lock(&kctx->mmu_lock); ++ } while (!err); ++ ++ if (err) { ++ dev_warn(kctx->kbdev->dev, ++ "%s: mmu_get_bottom_pgd failure\n", __func__); ++ if (insert_vpfn != start_vpfn) { ++ /* Invalidate the pages we have partially ++ * completed */ ++ mmu_insert_pages_failure_recovery(kctx, ++ start_vpfn, ++ insert_vpfn); ++ } ++ goto fail_unlock; ++ } ++ ++ p = pfn_to_page(PFN_DOWN(pgd)); ++ pgd_page = kmap(p); ++ if (!pgd_page) { ++ dev_warn(kctx->kbdev->dev, "%s: kmap failure\n", ++ __func__); ++ if (insert_vpfn != start_vpfn) { ++ /* Invalidate the pages we have partially ++ * completed */ ++ mmu_insert_pages_failure_recovery(kctx, ++ start_vpfn, ++ insert_vpfn); ++ } ++ err = -ENOMEM; ++ goto fail_unlock; ++ } ++ ++ if (cur_level == MIDGARD_MMU_LEVEL(2)) { ++ unsigned int level_index = (insert_vpfn >> 9) & 0x1FF; ++ u64 *target = &pgd_page[level_index]; ++ ++ if (mmu_mode->pte_is_valid(*target, cur_level)) ++ cleanup_empty_pte(kctx, target); ++ mmu_mode->entry_set_ate(target, *phys, flags, ++ cur_level); ++ } else { ++ for (i = 0; i < count; i++) { ++ unsigned int ofs = vindex + i; ++ u64 *target = &pgd_page[ofs]; ++ ++ /* Fail if the current page is a valid ATE entry ++ */ ++ KBASE_DEBUG_ASSERT(0 == (*target & 1UL)); ++ ++ kctx->kbdev->mmu_mode->entry_set_ate(target, ++ phys[i], flags, cur_level); ++ } ++ } ++ ++ phys += count; ++ insert_vpfn += count; ++ remain -= count; ++ ++ kbase_mmu_sync_pgd(kctx->kbdev, ++ kbase_dma_addr(p) + (vindex * sizeof(u64)), ++ count * sizeof(u64)); ++ ++ kunmap(p); ++ } ++ ++ mutex_unlock(&kctx->mmu_lock); ++ return 0; ++ ++fail_unlock: ++ mutex_unlock(&kctx->mmu_lock); ++ return err; ++} ++ ++/* ++ * Map 'nr' pages pointed to by 'phys' at GPU PFN 'vpfn' ++ */ ++int kbase_mmu_insert_pages(struct kbase_context *kctx, u64 vpfn, ++ struct tagged_addr *phys, size_t nr, ++ unsigned long flags) ++{ ++ int err; ++ ++ err = kbase_mmu_insert_pages_no_flush(kctx, vpfn, phys, nr, flags); ++ kbase_mmu_flush_invalidate(kctx, vpfn, nr, false); ++ return err; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_mmu_insert_pages); ++ ++/** ++ * kbase_mmu_flush_invalidate_noretain() - Flush and invalidate the GPU caches ++ * without retaining the kbase context. ++ * @kctx: The KBase context. ++ * @vpfn: The virtual page frame number to start the flush on. ++ * @nr: The number of pages to flush. ++ * @sync: Set if the operation should be synchronous or not. ++ * ++ * As per kbase_mmu_flush_invalidate but doesn't retain the kctx or do any ++ * other locking. ++ */ ++static void kbase_mmu_flush_invalidate_noretain(struct kbase_context *kctx, ++ u64 vpfn, size_t nr, bool sync) ++{ ++ struct kbase_device *kbdev = kctx->kbdev; ++ int err; ++ u32 op; ++ ++ /* Early out if there is nothing to do */ ++ if (nr == 0) ++ return; ++ ++ if (sync) ++ op = AS_COMMAND_FLUSH_MEM; ++ else ++ op = AS_COMMAND_FLUSH_PT; ++ ++ err = kbase_mmu_hw_do_operation(kbdev, ++ &kbdev->as[kctx->as_nr], ++ kctx, vpfn, nr, op, 0); ++#if KBASE_GPU_RESET_EN ++ if (err) { ++ /* Flush failed to complete, assume the ++ * GPU has hung and perform a reset to ++ * recover */ ++ dev_err(kbdev->dev, "Flush for GPU page table update did not complete. Issuing GPU soft-reset to recover\n"); ++ ++ if (kbase_prepare_to_reset_gpu_locked(kbdev)) ++ kbase_reset_gpu_locked(kbdev); ++ } ++#endif /* KBASE_GPU_RESET_EN */ ++ ++#ifndef CONFIG_MALI_BIFROST_NO_MALI ++ /* ++ * As this function could be called in interrupt context the sync ++ * request can't block. Instead log the request and the next flush ++ * request will pick it up. ++ */ ++ if ((!err) && sync && ++ kbase_hw_has_issue(kctx->kbdev, BASE_HW_ISSUE_6367)) ++ atomic_set(&kctx->drain_pending, 1); ++#endif /* !CONFIG_MALI_BIFROST_NO_MALI */ ++} ++ ++static void kbase_mmu_flush_invalidate(struct kbase_context *kctx, ++ u64 vpfn, size_t nr, bool sync) ++{ ++ struct kbase_device *kbdev; ++ bool ctx_is_in_runpool; ++#ifndef CONFIG_MALI_BIFROST_NO_MALI ++ bool drain_pending = false; ++ ++ if (atomic_xchg(&kctx->drain_pending, 0)) ++ drain_pending = true; ++#endif /* !CONFIG_MALI_BIFROST_NO_MALI */ ++ ++ /* Early out if there is nothing to do */ ++ if (nr == 0) ++ return; ++ ++ kbdev = kctx->kbdev; ++ mutex_lock(&kbdev->js_data.queue_mutex); ++ ctx_is_in_runpool = kbasep_js_runpool_retain_ctx(kbdev, kctx); ++ mutex_unlock(&kbdev->js_data.queue_mutex); ++ ++ if (ctx_is_in_runpool) { ++ KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); ++ ++ if (!kbase_pm_context_active_handle_suspend(kbdev, ++ KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) { ++ int err; ++ u32 op; ++ ++ /* AS transaction begin */ ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ ++ if (sync) ++ op = AS_COMMAND_FLUSH_MEM; ++ else ++ op = AS_COMMAND_FLUSH_PT; ++ ++ err = kbase_mmu_hw_do_operation(kbdev, ++ &kbdev->as[kctx->as_nr], ++ kctx, vpfn, nr, op, 0); ++ ++#if KBASE_GPU_RESET_EN ++ if (err) { ++ /* Flush failed to complete, assume the ++ * GPU has hung and perform a reset to ++ * recover */ ++ dev_err(kbdev->dev, "Flush for GPU page table update did not complete. Issueing GPU soft-reset to recover\n"); ++ ++ if (kbase_prepare_to_reset_gpu(kbdev)) ++ kbase_reset_gpu(kbdev); ++ } ++#endif /* KBASE_GPU_RESET_EN */ ++ ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ /* AS transaction end */ ++ ++#ifndef CONFIG_MALI_BIFROST_NO_MALI ++ /* ++ * The transaction lock must be dropped before here ++ * as kbase_wait_write_flush could take it if ++ * the GPU was powered down (static analysis doesn't ++ * know this can't happen). ++ */ ++ drain_pending |= (!err) && sync && ++ kbase_hw_has_issue(kctx->kbdev, ++ BASE_HW_ISSUE_6367); ++ if (drain_pending) { ++ /* Wait for GPU to flush write buffer */ ++ kbase_wait_write_flush(kctx); ++ } ++#endif /* !CONFIG_MALI_BIFROST_NO_MALI */ ++ ++ kbase_pm_context_idle(kbdev); ++ } ++ kbasep_js_runpool_release_ctx(kbdev, kctx); ++ } ++} ++ ++void kbase_mmu_update(struct kbase_context *kctx) ++{ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ lockdep_assert_held(&kctx->kbdev->mmu_hw_mutex); ++ /* ASSERT that the context has a valid as_nr, which is only the case ++ * when it's scheduled in. ++ * ++ * as_nr won't change because the caller has the hwaccess_lock */ ++ KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); ++ ++ kctx->kbdev->mmu_mode->update(kctx); ++} ++KBASE_EXPORT_TEST_API(kbase_mmu_update); ++ ++void kbase_mmu_disable_as(struct kbase_device *kbdev, int as_nr) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ lockdep_assert_held(&kbdev->mmu_hw_mutex); ++ ++ kbdev->mmu_mode->disable_as(kbdev, as_nr); ++} ++ ++void kbase_mmu_disable(struct kbase_context *kctx) ++{ ++ /* ASSERT that the context has a valid as_nr, which is only the case ++ * when it's scheduled in. ++ * ++ * as_nr won't change because the caller has the hwaccess_lock */ ++ KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); ++ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ ++ /* ++ * The address space is being disabled, drain all knowledge of it out ++ * from the caches as pages and page tables might be freed after this. ++ * ++ * The job scheduler code will already be holding the locks and context ++ * so just do the flush. ++ */ ++ kbase_mmu_flush_invalidate_noretain(kctx, 0, ~0, true); ++ ++ kctx->kbdev->mmu_mode->disable_as(kctx->kbdev, kctx->as_nr); ++} ++KBASE_EXPORT_TEST_API(kbase_mmu_disable); ++ ++/* ++ * We actually only discard the ATE, and not the page table ++ * pages. There is a potential DoS here, as we'll leak memory by ++ * having PTEs that are potentially unused. Will require physical ++ * page accounting, so MMU pages are part of the process allocation. ++ * ++ * IMPORTANT: This uses kbasep_js_runpool_release_ctx() when the context is ++ * currently scheduled into the runpool, and so potentially uses a lot of locks. ++ * These locks must be taken in the correct order with respect to others ++ * already held by the caller. Refer to kbasep_js_runpool_release_ctx() for more ++ * information. ++ */ ++int kbase_mmu_teardown_pages(struct kbase_context *kctx, u64 vpfn, size_t nr) ++{ ++ phys_addr_t pgd; ++ size_t requested_nr = nr; ++ struct kbase_mmu_mode const *mmu_mode; ++ int err = -EFAULT; ++ ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ beenthere(kctx, "kctx %p vpfn %lx nr %zd", (void *)kctx, (unsigned long)vpfn, nr); ++ ++ if (0 == nr) { ++ /* early out if nothing to do */ ++ return 0; ++ } ++ ++ mutex_lock(&kctx->mmu_lock); ++ ++ mmu_mode = kctx->kbdev->mmu_mode; ++ ++ while (nr) { ++ unsigned int i; ++ unsigned int index = vpfn & 0x1FF; ++ unsigned int count = KBASE_MMU_PAGE_ENTRIES - index; ++ unsigned int pcount; ++ unsigned int level; ++ u64 *page; ++ ++ if (count > nr) ++ count = nr; ++ ++ /* need to check if this is a 2MB or a 4kB page */ ++ pgd = kctx->pgd; ++ ++ for (level = MIDGARD_MMU_TOPLEVEL; ++ level <= MIDGARD_MMU_BOTTOMLEVEL; level++) { ++ phys_addr_t next_pgd; ++ ++ index = (vpfn >> ((3 - level) * 9)) & 0x1FF; ++ page = kmap(phys_to_page(pgd)); ++ if (mmu_mode->ate_is_valid(page[index], level)) ++ break; /* keep the mapping */ ++ else if (!mmu_mode->pte_is_valid(page[index], level)) { ++ /* nothing here, advance */ ++ switch (level) { ++ case MIDGARD_MMU_LEVEL(0): ++ count = 134217728; ++ break; ++ case MIDGARD_MMU_LEVEL(1): ++ count = 262144; ++ break; ++ case MIDGARD_MMU_LEVEL(2): ++ count = 512; ++ break; ++ case MIDGARD_MMU_LEVEL(3): ++ count = 1; ++ break; ++ } ++ if (count > nr) ++ count = nr; ++ goto next; ++ } ++ next_pgd = mmu_mode->pte_to_phy_addr(page[index]); ++ kunmap(phys_to_page(pgd)); ++ pgd = next_pgd; ++ } ++ ++ switch (level) { ++ case MIDGARD_MMU_LEVEL(0): ++ case MIDGARD_MMU_LEVEL(1): ++ dev_warn(kctx->kbdev->dev, ++ "%s: No support for ATEs at level %d\n", ++ __func__, level); ++ kunmap(phys_to_page(pgd)); ++ goto out; ++ case MIDGARD_MMU_LEVEL(2): ++ /* can only teardown if count >= 512 */ ++ if (count >= 512) { ++ pcount = 1; ++ } else { ++ dev_warn(kctx->kbdev->dev, ++ "%s: limiting teardown as it tries to do a partial 2MB teardown, need 512, but have %d to tear down\n", ++ __func__, count); ++ pcount = 0; ++ } ++ break; ++ case MIDGARD_MMU_BOTTOMLEVEL: ++ /* page count is the same as the logical count */ ++ pcount = count; ++ break; ++ default: ++ dev_err(kctx->kbdev->dev, ++ "%s: found non-mapped memory, early out\n", ++ __func__); ++ vpfn += count; ++ nr -= count; ++ continue; ++ } ++ ++ /* Invalidate the entries we added */ ++ for (i = 0; i < pcount; i++) ++ mmu_mode->entry_invalidate(&page[index + i]); ++ ++ kbase_mmu_sync_pgd(kctx->kbdev, ++ kbase_dma_addr(phys_to_page(pgd)) + ++ 8 * index, 8*pcount); ++ ++next: ++ kunmap(phys_to_page(pgd)); ++ vpfn += count; ++ nr -= count; ++ } ++ err = 0; ++out: ++ mutex_unlock(&kctx->mmu_lock); ++ kbase_mmu_flush_invalidate(kctx, vpfn, requested_nr, true); ++ return err; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_mmu_teardown_pages); ++ ++/** ++ * Update the entries for specified number of pages pointed to by 'phys' at GPU PFN 'vpfn'. ++ * This call is being triggered as a response to the changes of the mem attributes ++ * ++ * @pre : The caller is responsible for validating the memory attributes ++ * ++ * IMPORTANT: This uses kbasep_js_runpool_release_ctx() when the context is ++ * currently scheduled into the runpool, and so potentially uses a lot of locks. ++ * These locks must be taken in the correct order with respect to others ++ * already held by the caller. Refer to kbasep_js_runpool_release_ctx() for more ++ * information. ++ */ ++int kbase_mmu_update_pages(struct kbase_context *kctx, u64 vpfn, ++ struct tagged_addr *phys, size_t nr, ++ unsigned long flags) ++{ ++ phys_addr_t pgd; ++ u64 *pgd_page; ++ size_t requested_nr = nr; ++ struct kbase_mmu_mode const *mmu_mode; ++ int err; ++ ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ KBASE_DEBUG_ASSERT(0 != vpfn); ++ KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE)); ++ ++ /* Early out if there is nothing to do */ ++ if (nr == 0) ++ return 0; ++ ++ mutex_lock(&kctx->mmu_lock); ++ ++ mmu_mode = kctx->kbdev->mmu_mode; ++ ++ dev_warn(kctx->kbdev->dev, "kbase_mmu_update_pages(): updating page share flags on GPU PFN 0x%llx from phys %p, %zu pages", ++ vpfn, phys, nr); ++ ++ while (nr) { ++ unsigned int i; ++ unsigned int index = vpfn & 0x1FF; ++ size_t count = KBASE_MMU_PAGE_ENTRIES - index; ++ struct page *p; ++ ++ if (count > nr) ++ count = nr; ++ ++ do { ++ err = mmu_get_bottom_pgd(kctx, vpfn, &pgd); ++ if (err != -ENOMEM) ++ break; ++ /* Fill the memory pool with enough pages for ++ * the page walk to succeed ++ */ ++ mutex_unlock(&kctx->mmu_lock); ++ err = kbase_mem_pool_grow(&kctx->mem_pool, ++ MIDGARD_MMU_BOTTOMLEVEL); ++ mutex_lock(&kctx->mmu_lock); ++ } while (!err); ++ if (err) { ++ dev_warn(kctx->kbdev->dev, ++ "mmu_get_bottom_pgd failure\n"); ++ goto fail_unlock; ++ } ++ ++ p = pfn_to_page(PFN_DOWN(pgd)); ++ pgd_page = kmap(p); ++ if (!pgd_page) { ++ dev_warn(kctx->kbdev->dev, "kmap failure\n"); ++ err = -ENOMEM; ++ goto fail_unlock; ++ } ++ ++ for (i = 0; i < count; i++) ++ mmu_mode->entry_set_ate(&pgd_page[index + i], phys[i], ++ flags, MIDGARD_MMU_BOTTOMLEVEL); ++ ++ phys += count; ++ vpfn += count; ++ nr -= count; ++ ++ kbase_mmu_sync_pgd(kctx->kbdev, ++ kbase_dma_addr(p) + (index * sizeof(u64)), ++ count * sizeof(u64)); ++ ++ kunmap(pfn_to_page(PFN_DOWN(pgd))); ++ } ++ ++ mutex_unlock(&kctx->mmu_lock); ++ kbase_mmu_flush_invalidate(kctx, vpfn, requested_nr, true); ++ return 0; ++ ++fail_unlock: ++ mutex_unlock(&kctx->mmu_lock); ++ kbase_mmu_flush_invalidate(kctx, vpfn, requested_nr, true); ++ return err; ++} ++ ++static void mmu_teardown_level(struct kbase_context *kctx, phys_addr_t pgd, ++ int level, u64 *pgd_page_buffer) ++{ ++ phys_addr_t target_pgd; ++ struct page *p; ++ u64 *pgd_page; ++ int i; ++ struct kbase_mmu_mode const *mmu_mode; ++ ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ lockdep_assert_held(&kctx->mmu_lock); ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ pgd_page = kmap_atomic(pfn_to_page(PFN_DOWN(pgd))); ++ /* kmap_atomic should NEVER fail. */ ++ KBASE_DEBUG_ASSERT(NULL != pgd_page); ++ /* Copy the page to our preallocated buffer so that we can minimize ++ * kmap_atomic usage */ ++ memcpy(pgd_page_buffer, pgd_page, PAGE_SIZE); ++ kunmap_atomic(pgd_page); ++ pgd_page = pgd_page_buffer; ++ ++ mmu_mode = kctx->kbdev->mmu_mode; ++ ++ for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++) { ++ target_pgd = mmu_mode->pte_to_phy_addr(pgd_page[i]); ++ ++ if (target_pgd) { ++ if (mmu_mode->pte_is_valid(pgd_page[i], level)) { ++ mmu_teardown_level(kctx, ++ target_pgd, ++ level + 1, ++ pgd_page_buffer + ++ (PAGE_SIZE / sizeof(u64))); ++ } ++ } ++ } ++ ++ p = pfn_to_page(PFN_DOWN(pgd)); ++ kbase_mem_pool_free(&kctx->mem_pool, p, true); ++ kbase_process_page_usage_dec(kctx, 1); ++ kbase_atomic_sub_pages(1, &kctx->used_pages); ++ kbase_atomic_sub_pages(1, &kctx->kbdev->memdev.used_pages); ++} ++ ++int kbase_mmu_init(struct kbase_context *kctx) ++{ ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ KBASE_DEBUG_ASSERT(NULL == kctx->mmu_teardown_pages); ++ ++ mutex_init(&kctx->mmu_lock); ++ ++ /* Preallocate MMU depth of four pages for mmu_teardown_level to use */ ++ kctx->mmu_teardown_pages = kmalloc(PAGE_SIZE * 4, GFP_KERNEL); ++ ++ if (NULL == kctx->mmu_teardown_pages) ++ return -ENOMEM; ++ ++ return 0; ++} ++ ++void kbase_mmu_term(struct kbase_context *kctx) ++{ ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ KBASE_DEBUG_ASSERT(NULL != kctx->mmu_teardown_pages); ++ ++ kfree(kctx->mmu_teardown_pages); ++ kctx->mmu_teardown_pages = NULL; ++} ++ ++void kbase_mmu_free_pgd(struct kbase_context *kctx) ++{ ++ int new_page_count = 0; ++ ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ KBASE_DEBUG_ASSERT(NULL != kctx->mmu_teardown_pages); ++ ++ mutex_lock(&kctx->mmu_lock); ++ mmu_teardown_level(kctx, kctx->pgd, MIDGARD_MMU_TOPLEVEL, ++ kctx->mmu_teardown_pages); ++ mutex_unlock(&kctx->mmu_lock); ++ ++ KBASE_TLSTREAM_AUX_PAGESALLOC( ++ kctx->id, ++ (u64)new_page_count); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_mmu_free_pgd); ++ ++static size_t kbasep_mmu_dump_level(struct kbase_context *kctx, phys_addr_t pgd, int level, char ** const buffer, size_t *size_left) ++{ ++ phys_addr_t target_pgd; ++ u64 *pgd_page; ++ int i; ++ size_t size = KBASE_MMU_PAGE_ENTRIES * sizeof(u64) + sizeof(u64); ++ size_t dump_size; ++ struct kbase_mmu_mode const *mmu_mode; ++ ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ lockdep_assert_held(&kctx->mmu_lock); ++ ++ mmu_mode = kctx->kbdev->mmu_mode; ++ ++ pgd_page = kmap(pfn_to_page(PFN_DOWN(pgd))); ++ if (!pgd_page) { ++ dev_warn(kctx->kbdev->dev, "kbasep_mmu_dump_level: kmap failure\n"); ++ return 0; ++ } ++ ++ if (*size_left >= size) { ++ /* A modified physical address that contains the page table level */ ++ u64 m_pgd = pgd | level; ++ ++ /* Put the modified physical address in the output buffer */ ++ memcpy(*buffer, &m_pgd, sizeof(m_pgd)); ++ *buffer += sizeof(m_pgd); ++ ++ /* Followed by the page table itself */ ++ memcpy(*buffer, pgd_page, sizeof(u64) * KBASE_MMU_PAGE_ENTRIES); ++ *buffer += sizeof(u64) * KBASE_MMU_PAGE_ENTRIES; ++ ++ *size_left -= size; ++ } ++ ++ if (level < MIDGARD_MMU_BOTTOMLEVEL) { ++ for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++) { ++ if (mmu_mode->pte_is_valid(pgd_page[i], level)) { ++ target_pgd = mmu_mode->pte_to_phy_addr( ++ pgd_page[i]); ++ ++ dump_size = kbasep_mmu_dump_level(kctx, ++ target_pgd, level + 1, ++ buffer, size_left); ++ if (!dump_size) { ++ kunmap(pfn_to_page(PFN_DOWN(pgd))); ++ return 0; ++ } ++ size += dump_size; ++ } ++ } ++ } ++ ++ kunmap(pfn_to_page(PFN_DOWN(pgd))); ++ ++ return size; ++} ++ ++void *kbase_mmu_dump(struct kbase_context *kctx, int nr_pages) ++{ ++ void *kaddr; ++ size_t size_left; ++ ++ KBASE_DEBUG_ASSERT(kctx); ++ ++ if (0 == nr_pages) { ++ /* can't dump in a 0 sized buffer, early out */ ++ return NULL; ++ } ++ ++ size_left = nr_pages * PAGE_SIZE; ++ ++ KBASE_DEBUG_ASSERT(0 != size_left); ++ kaddr = vmalloc_user(size_left); ++ ++ mutex_lock(&kctx->mmu_lock); ++ ++ if (kaddr) { ++ u64 end_marker = 0xFFULL; ++ char *buffer; ++ char *mmu_dump_buffer; ++ u64 config[3]; ++ size_t dump_size, size = 0; ++ ++ buffer = (char *)kaddr; ++ mmu_dump_buffer = buffer; ++ ++ if (kctx->api_version >= KBASE_API_VERSION(8, 4)) { ++ struct kbase_mmu_setup as_setup; ++ ++ kctx->kbdev->mmu_mode->get_as_setup(kctx, &as_setup); ++ config[0] = as_setup.transtab; ++ config[1] = as_setup.memattr; ++ config[2] = as_setup.transcfg; ++ memcpy(buffer, &config, sizeof(config)); ++ mmu_dump_buffer += sizeof(config); ++ size_left -= sizeof(config); ++ size += sizeof(config); ++ } ++ ++ dump_size = kbasep_mmu_dump_level(kctx, ++ kctx->pgd, ++ MIDGARD_MMU_TOPLEVEL, ++ &mmu_dump_buffer, ++ &size_left); ++ ++ if (!dump_size) ++ goto fail_free; ++ ++ size += dump_size; ++ ++ /* Add on the size for the end marker */ ++ size += sizeof(u64); ++ ++ if (size > (nr_pages * PAGE_SIZE)) { ++ /* The buffer isn't big enough - free the memory and return failure */ ++ goto fail_free; ++ } ++ ++ /* Add the end marker */ ++ memcpy(mmu_dump_buffer, &end_marker, sizeof(u64)); ++ } ++ ++ mutex_unlock(&kctx->mmu_lock); ++ return kaddr; ++ ++fail_free: ++ vfree(kaddr); ++ mutex_unlock(&kctx->mmu_lock); ++ return NULL; ++} ++KBASE_EXPORT_TEST_API(kbase_mmu_dump); ++ ++void bus_fault_worker(struct work_struct *data) ++{ ++ struct kbase_as *faulting_as; ++ int as_no; ++ struct kbase_context *kctx; ++ struct kbase_device *kbdev; ++#if KBASE_GPU_RESET_EN ++ bool reset_status = false; ++#endif /* KBASE_GPU_RESET_EN */ ++ ++ faulting_as = container_of(data, struct kbase_as, work_busfault); ++ ++ as_no = faulting_as->number; ++ ++ kbdev = container_of(faulting_as, struct kbase_device, as[as_no]); ++ ++ /* Grab the context that was already refcounted in kbase_mmu_interrupt(). ++ * Therefore, it cannot be scheduled out of this AS until we explicitly release it ++ */ ++ kctx = kbasep_js_runpool_lookup_ctx_noretain(kbdev, as_no); ++ if (WARN_ON(!kctx)) { ++ atomic_dec(&kbdev->faults_pending); ++ return; ++ } ++ ++ if (unlikely(faulting_as->protected_mode)) ++ { ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Permission failure"); ++ kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx, ++ KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); ++ kbasep_js_runpool_release_ctx(kbdev, kctx); ++ atomic_dec(&kbdev->faults_pending); ++ return; ++ ++ } ++ ++#if KBASE_GPU_RESET_EN ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245)) { ++ /* Due to H/W issue 8245 we need to reset the GPU after using UNMAPPED mode. ++ * We start the reset before switching to UNMAPPED to ensure that unrelated jobs ++ * are evicted from the GPU before the switch. ++ */ ++ dev_err(kbdev->dev, "GPU bus error occurred. For this GPU version we now soft-reset as part of bus error recovery\n"); ++ reset_status = kbase_prepare_to_reset_gpu(kbdev); ++ } ++#endif /* KBASE_GPU_RESET_EN */ ++ /* NOTE: If GPU already powered off for suspend, we don't need to switch to unmapped */ ++ if (!kbase_pm_context_active_handle_suspend(kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) { ++ unsigned long flags; ++ ++ /* switch to UNMAPPED mode, will abort all jobs and stop any hw counter dumping */ ++ /* AS transaction begin */ ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ ++ /* Set the MMU into unmapped mode */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_mmu_disable(kctx); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ /* AS transaction end */ ++ ++ kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx, ++ KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); ++ kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx, ++ KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); ++ ++ kbase_pm_context_idle(kbdev); ++ } ++ ++#if KBASE_GPU_RESET_EN ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245) && reset_status) ++ kbase_reset_gpu(kbdev); ++#endif /* KBASE_GPU_RESET_EN */ ++ ++ kbasep_js_runpool_release_ctx(kbdev, kctx); ++ ++ atomic_dec(&kbdev->faults_pending); ++} ++ ++const char *kbase_exception_name(struct kbase_device *kbdev, u32 exception_code) ++{ ++ const char *e; ++ ++ switch (exception_code) { ++ /* Non-Fault Status code */ ++ case 0x00: ++ e = "NOT_STARTED/IDLE/OK"; ++ break; ++ case 0x01: ++ e = "DONE"; ++ break; ++ case 0x02: ++ e = "INTERRUPTED"; ++ break; ++ case 0x03: ++ e = "STOPPED"; ++ break; ++ case 0x04: ++ e = "TERMINATED"; ++ break; ++ case 0x08: ++ e = "ACTIVE"; ++ break; ++ /* Job exceptions */ ++ case 0x40: ++ e = "JOB_CONFIG_FAULT"; ++ break; ++ case 0x41: ++ e = "JOB_POWER_FAULT"; ++ break; ++ case 0x42: ++ e = "JOB_READ_FAULT"; ++ break; ++ case 0x43: ++ e = "JOB_WRITE_FAULT"; ++ break; ++ case 0x44: ++ e = "JOB_AFFINITY_FAULT"; ++ break; ++ case 0x48: ++ e = "JOB_BUS_FAULT"; ++ break; ++ case 0x50: ++ e = "INSTR_INVALID_PC"; ++ break; ++ case 0x51: ++ e = "INSTR_INVALID_ENC"; ++ break; ++ case 0x52: ++ e = "INSTR_TYPE_MISMATCH"; ++ break; ++ case 0x53: ++ e = "INSTR_OPERAND_FAULT"; ++ break; ++ case 0x54: ++ e = "INSTR_TLS_FAULT"; ++ break; ++ case 0x55: ++ e = "INSTR_BARRIER_FAULT"; ++ break; ++ case 0x56: ++ e = "INSTR_ALIGN_FAULT"; ++ break; ++ case 0x58: ++ e = "DATA_INVALID_FAULT"; ++ break; ++ case 0x59: ++ e = "TILE_RANGE_FAULT"; ++ break; ++ case 0x5A: ++ e = "ADDR_RANGE_FAULT"; ++ break; ++ case 0x60: ++ e = "OUT_OF_MEMORY"; ++ break; ++ /* GPU exceptions */ ++ case 0x80: ++ e = "DELAYED_BUS_FAULT"; ++ break; ++ case 0x88: ++ e = "SHAREABILITY_FAULT"; ++ break; ++ /* MMU exceptions */ ++ case 0xC0: ++ case 0xC1: ++ case 0xC2: ++ case 0xC3: ++ case 0xC4: ++ case 0xC5: ++ case 0xC6: ++ case 0xC7: ++ e = "TRANSLATION_FAULT"; ++ break; ++ case 0xC8: ++ e = "PERMISSION_FAULT"; ++ break; ++ case 0xC9: ++ case 0xCA: ++ case 0xCB: ++ case 0xCC: ++ case 0xCD: ++ case 0xCE: ++ case 0xCF: ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) ++ e = "PERMISSION_FAULT"; ++ else ++ e = "UNKNOWN"; ++ break; ++ case 0xD0: ++ case 0xD1: ++ case 0xD2: ++ case 0xD3: ++ case 0xD4: ++ case 0xD5: ++ case 0xD6: ++ case 0xD7: ++ e = "TRANSTAB_BUS_FAULT"; ++ break; ++ case 0xD8: ++ e = "ACCESS_FLAG"; ++ break; ++ case 0xD9: ++ case 0xDA: ++ case 0xDB: ++ case 0xDC: ++ case 0xDD: ++ case 0xDE: ++ case 0xDF: ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) ++ e = "ACCESS_FLAG"; ++ else ++ e = "UNKNOWN"; ++ break; ++ case 0xE0: ++ case 0xE1: ++ case 0xE2: ++ case 0xE3: ++ case 0xE4: ++ case 0xE5: ++ case 0xE6: ++ case 0xE7: ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) ++ e = "ADDRESS_SIZE_FAULT"; ++ else ++ e = "UNKNOWN"; ++ break; ++ case 0xE8: ++ case 0xE9: ++ case 0xEA: ++ case 0xEB: ++ case 0xEC: ++ case 0xED: ++ case 0xEE: ++ case 0xEF: ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) ++ e = "MEMORY_ATTRIBUTES_FAULT"; ++ else ++ e = "UNKNOWN"; ++ break; ++ default: ++ e = "UNKNOWN"; ++ break; ++ }; ++ ++ return e; ++} ++ ++static const char *access_type_name(struct kbase_device *kbdev, ++ u32 fault_status) ++{ ++ switch (fault_status & AS_FAULTSTATUS_ACCESS_TYPE_MASK) { ++ case AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC: ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) ++ return "ATOMIC"; ++ else ++ return "UNKNOWN"; ++ case AS_FAULTSTATUS_ACCESS_TYPE_READ: ++ return "READ"; ++ case AS_FAULTSTATUS_ACCESS_TYPE_WRITE: ++ return "WRITE"; ++ case AS_FAULTSTATUS_ACCESS_TYPE_EX: ++ return "EXECUTE"; ++ default: ++ WARN_ON(1); ++ return NULL; ++ } ++} ++ ++/** ++ * The caller must ensure it's retained the ctx to prevent it from being scheduled out whilst it's being worked on. ++ */ ++static void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, ++ struct kbase_as *as, const char *reason_str) ++{ ++ unsigned long flags; ++ int exception_type; ++ int access_type; ++ int source_id; ++ int as_no; ++ struct kbase_device *kbdev; ++ struct kbasep_js_device_data *js_devdata; ++ ++#if KBASE_GPU_RESET_EN ++ bool reset_status = false; ++#endif ++ ++ as_no = as->number; ++ kbdev = kctx->kbdev; ++ js_devdata = &kbdev->js_data; ++ ++ /* ASSERT that the context won't leave the runpool */ ++ KBASE_DEBUG_ASSERT(atomic_read(&kctx->refcount) > 0); ++ ++ /* decode the fault status */ ++ exception_type = as->fault_status & 0xFF; ++ access_type = (as->fault_status >> 8) & 0x3; ++ source_id = (as->fault_status >> 16); ++ ++ /* terminal fault, print info about the fault */ ++ dev_err(kbdev->dev, ++ "Unhandled Page fault in AS%d at VA 0x%016llX\n" ++ "Reason: %s\n" ++ "raw fault status: 0x%X\n" ++ "decoded fault status: %s\n" ++ "exception type 0x%X: %s\n" ++ "access type 0x%X: %s\n" ++ "source id 0x%X\n" ++ "pid: %d\n", ++ as_no, as->fault_addr, ++ reason_str, ++ as->fault_status, ++ (as->fault_status & (1 << 10) ? "DECODER FAULT" : "SLAVE FAULT"), ++ exception_type, kbase_exception_name(kbdev, exception_type), ++ access_type, access_type_name(kbdev, as->fault_status), ++ source_id, ++ kctx->pid); ++ ++ /* hardware counters dump fault handling */ ++ if ((kbdev->hwcnt.kctx) && (kbdev->hwcnt.kctx->as_nr == as_no) && ++ (kbdev->hwcnt.backend.state == ++ KBASE_INSTR_STATE_DUMPING)) { ++ unsigned int num_core_groups = kbdev->gpu_props.num_core_groups; ++ ++ if ((as->fault_addr >= kbdev->hwcnt.addr) && ++ (as->fault_addr < (kbdev->hwcnt.addr + ++ (num_core_groups * 2048)))) ++ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_FAULT; ++ } ++ ++ /* Stop the kctx from submitting more jobs and cause it to be scheduled ++ * out/rescheduled - this will occur on releasing the context's refcount */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbasep_js_clear_submit_allowed(js_devdata, kctx); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ /* Kill any running jobs from the context. Submit is disallowed, so no more jobs from this ++ * context can appear in the job slots from this point on */ ++ kbase_backend_jm_kill_jobs_from_kctx(kctx); ++ /* AS transaction begin */ ++ mutex_lock(&kbdev->mmu_hw_mutex); ++#if KBASE_GPU_RESET_EN ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245)) { ++ /* Due to H/W issue 8245 we need to reset the GPU after using UNMAPPED mode. ++ * We start the reset before switching to UNMAPPED to ensure that unrelated jobs ++ * are evicted from the GPU before the switch. ++ */ ++ dev_err(kbdev->dev, "Unhandled page fault. For this GPU version we now soft-reset the GPU as part of page fault recovery."); ++ reset_status = kbase_prepare_to_reset_gpu(kbdev); ++ } ++#endif /* KBASE_GPU_RESET_EN */ ++ /* switch to UNMAPPED mode, will abort all jobs and stop any hw counter dumping */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_mmu_disable(kctx); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ /* AS transaction end */ ++ /* Clear down the fault */ ++ kbase_mmu_hw_clear_fault(kbdev, as, kctx, ++ KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); ++ kbase_mmu_hw_enable_fault(kbdev, as, kctx, ++ KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); ++ ++#if KBASE_GPU_RESET_EN ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245) && reset_status) ++ kbase_reset_gpu(kbdev); ++#endif /* KBASE_GPU_RESET_EN */ ++} ++ ++void kbasep_as_do_poke(struct work_struct *work) ++{ ++ struct kbase_as *as; ++ struct kbase_device *kbdev; ++ struct kbase_context *kctx; ++ unsigned long flags; ++ ++ KBASE_DEBUG_ASSERT(work); ++ as = container_of(work, struct kbase_as, poke_work); ++ kbdev = container_of(as, struct kbase_device, as[as->number]); ++ KBASE_DEBUG_ASSERT(as->poke_state & KBASE_AS_POKE_STATE_IN_FLIGHT); ++ ++ /* GPU power will already be active by virtue of the caller holding a JS ++ * reference on the address space, and will not release it until this worker ++ * has finished */ ++ ++ /* Further to the comment above, we know that while this function is running ++ * the AS will not be released as before the atom is released this workqueue ++ * is flushed (in kbase_as_poking_timer_release_atom) ++ */ ++ kctx = kbasep_js_runpool_lookup_ctx_noretain(kbdev, as->number); ++ ++ /* AS transaction begin */ ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ /* Force a uTLB invalidate */ ++ kbase_mmu_hw_do_operation(kbdev, as, kctx, 0, 0, ++ AS_COMMAND_UNLOCK, 0); ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ /* AS transaction end */ ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ if (as->poke_refcount && ++ !(as->poke_state & KBASE_AS_POKE_STATE_KILLING_POKE)) { ++ /* Only queue up the timer if we need it, and we're not trying to kill it */ ++ hrtimer_start(&as->poke_timer, HR_TIMER_DELAY_MSEC(5), HRTIMER_MODE_REL); ++ } ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++} ++ ++enum hrtimer_restart kbasep_as_poke_timer_callback(struct hrtimer *timer) ++{ ++ struct kbase_as *as; ++ int queue_work_ret; ++ ++ KBASE_DEBUG_ASSERT(NULL != timer); ++ as = container_of(timer, struct kbase_as, poke_timer); ++ KBASE_DEBUG_ASSERT(as->poke_state & KBASE_AS_POKE_STATE_IN_FLIGHT); ++ ++ queue_work_ret = queue_work(as->poke_wq, &as->poke_work); ++ KBASE_DEBUG_ASSERT(queue_work_ret); ++ return HRTIMER_NORESTART; ++} ++ ++/** ++ * Retain the poking timer on an atom's context (if the atom hasn't already ++ * done so), and start the timer (if it's not already started). ++ * ++ * This must only be called on a context that's scheduled in, and an atom ++ * that's running on the GPU. ++ * ++ * The caller must hold hwaccess_lock ++ * ++ * This can be called safely from atomic context ++ */ ++void kbase_as_poking_timer_retain_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom) ++{ ++ struct kbase_as *as; ++ ++ KBASE_DEBUG_ASSERT(kbdev); ++ KBASE_DEBUG_ASSERT(kctx); ++ KBASE_DEBUG_ASSERT(katom); ++ KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (katom->poking) ++ return; ++ ++ katom->poking = 1; ++ ++ /* It's safe to work on the as/as_nr without an explicit reference, ++ * because the caller holds the hwaccess_lock, and the atom itself ++ * was also running and had already taken a reference */ ++ as = &kbdev->as[kctx->as_nr]; ++ ++ if (++(as->poke_refcount) == 1) { ++ /* First refcount for poke needed: check if not already in flight */ ++ if (!as->poke_state) { ++ /* need to start poking */ ++ as->poke_state |= KBASE_AS_POKE_STATE_IN_FLIGHT; ++ queue_work(as->poke_wq, &as->poke_work); ++ } ++ } ++} ++ ++/** ++ * If an atom holds a poking timer, release it and wait for it to finish ++ * ++ * This must only be called on a context that's scheduled in, and an atom ++ * that still has a JS reference on the context ++ * ++ * This must \b not be called from atomic context, since it can sleep. ++ */ ++void kbase_as_poking_timer_release_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom) ++{ ++ struct kbase_as *as; ++ unsigned long flags; ++ ++ KBASE_DEBUG_ASSERT(kbdev); ++ KBASE_DEBUG_ASSERT(kctx); ++ KBASE_DEBUG_ASSERT(katom); ++ KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); ++ ++ if (!katom->poking) ++ return; ++ ++ as = &kbdev->as[kctx->as_nr]; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ KBASE_DEBUG_ASSERT(as->poke_refcount > 0); ++ KBASE_DEBUG_ASSERT(as->poke_state & KBASE_AS_POKE_STATE_IN_FLIGHT); ++ ++ if (--(as->poke_refcount) == 0) { ++ as->poke_state |= KBASE_AS_POKE_STATE_KILLING_POKE; ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ hrtimer_cancel(&as->poke_timer); ++ flush_workqueue(as->poke_wq); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ /* Re-check whether it's still needed */ ++ if (as->poke_refcount) { ++ int queue_work_ret; ++ /* Poking still needed: ++ * - Another retain will not be starting the timer or queueing work, ++ * because it's still marked as in-flight ++ * - The hrtimer has finished, and has not started a new timer or ++ * queued work because it's been marked as killing ++ * ++ * So whatever happens now, just queue the work again */ ++ as->poke_state &= ~((kbase_as_poke_state)KBASE_AS_POKE_STATE_KILLING_POKE); ++ queue_work_ret = queue_work(as->poke_wq, &as->poke_work); ++ KBASE_DEBUG_ASSERT(queue_work_ret); ++ } else { ++ /* It isn't - so mark it as not in flight, and not killing */ ++ as->poke_state = 0u; ++ ++ /* The poke associated with the atom has now finished. If this is ++ * also the last atom on the context, then we can guarentee no more ++ * pokes (and thus no more poking register accesses) will occur on ++ * the context until new atoms are run */ ++ } ++ } ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ katom->poking = 0; ++} ++ ++void kbase_mmu_interrupt_process(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_as *as) ++{ ++ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (!kctx) { ++ dev_warn(kbdev->dev, "%s in AS%d at 0x%016llx with no context present! Suprious IRQ or SW Design Error?\n", ++ kbase_as_has_bus_fault(as) ? "Bus error" : "Page fault", ++ as->number, as->fault_addr); ++ ++ /* Since no ctx was found, the MMU must be disabled. */ ++ WARN_ON(as->current_setup.transtab); ++ ++ if (kbase_as_has_bus_fault(as)) { ++ kbase_mmu_hw_clear_fault(kbdev, as, kctx, ++ KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); ++ kbase_mmu_hw_enable_fault(kbdev, as, kctx, ++ KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); ++ } else if (kbase_as_has_page_fault(as)) { ++ kbase_mmu_hw_clear_fault(kbdev, as, kctx, ++ KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); ++ kbase_mmu_hw_enable_fault(kbdev, as, kctx, ++ KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); ++ } ++ ++#if KBASE_GPU_RESET_EN ++ if (kbase_as_has_bus_fault(as) && ++ kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245)) { ++ bool reset_status; ++ /* ++ * Reset the GPU, like in bus_fault_worker, in case an ++ * earlier error hasn't been properly cleared by this ++ * point. ++ */ ++ dev_err(kbdev->dev, "GPU bus error occurred. For this GPU version we now soft-reset as part of bus error recovery\n"); ++ reset_status = kbase_prepare_to_reset_gpu_locked(kbdev); ++ if (reset_status) ++ kbase_reset_gpu_locked(kbdev); ++ } ++#endif /* KBASE_GPU_RESET_EN */ ++ ++ return; ++ } ++ ++ if (kbase_as_has_bus_fault(as)) { ++ /* ++ * hw counters dumping in progress, signal the ++ * other thread that it failed ++ */ ++ if ((kbdev->hwcnt.kctx == kctx) && ++ (kbdev->hwcnt.backend.state == ++ KBASE_INSTR_STATE_DUMPING)) ++ kbdev->hwcnt.backend.state = ++ KBASE_INSTR_STATE_FAULT; ++ ++ /* ++ * Stop the kctx from submitting more jobs and cause it ++ * to be scheduled out/rescheduled when all references ++ * to it are released ++ */ ++ kbasep_js_clear_submit_allowed(js_devdata, kctx); ++ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) ++ dev_warn(kbdev->dev, ++ "Bus error in AS%d at VA=0x%016llx, IPA=0x%016llx\n", ++ as->number, as->fault_addr, ++ as->fault_extra_addr); ++ else ++ dev_warn(kbdev->dev, "Bus error in AS%d at 0x%016llx\n", ++ as->number, as->fault_addr); ++ ++ /* ++ * We need to switch to UNMAPPED mode - but we do this in a ++ * worker so that we can sleep ++ */ ++ KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&as->work_busfault)); ++ WARN_ON(work_pending(&as->work_busfault)); ++ queue_work(as->pf_wq, &as->work_busfault); ++ atomic_inc(&kbdev->faults_pending); ++ } else { ++ KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&as->work_pagefault)); ++ WARN_ON(work_pending(&as->work_pagefault)); ++ queue_work(as->pf_wq, &as->work_pagefault); ++ atomic_inc(&kbdev->faults_pending); ++ } ++} ++ ++void kbase_flush_mmu_wqs(struct kbase_device *kbdev) ++{ ++ int i; ++ ++ for (i = 0; i < kbdev->nr_hw_address_spaces; i++) { ++ struct kbase_as *as = &kbdev->as[i]; ++ ++ flush_workqueue(as->pf_wq); ++ } ++} +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mmu_hw.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mmu_hw.h +new file mode 100755 +index 000000000000..986e959e9a0c +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mmu_hw.h +@@ -0,0 +1,123 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/** ++ * @file ++ * Interface file for accessing MMU hardware functionality ++ */ ++ ++/** ++ * @page mali_kbase_mmu_hw_page MMU hardware interface ++ * ++ * @section mali_kbase_mmu_hw_intro_sec Introduction ++ * This module provides an abstraction for accessing the functionality provided ++ * by the midgard MMU and thus allows all MMU HW access to be contained within ++ * one common place and allows for different backends (implementations) to ++ * be provided. ++ */ ++ ++#ifndef _MALI_KBASE_MMU_HW_H_ ++#define _MALI_KBASE_MMU_HW_H_ ++ ++/* Forward declarations */ ++struct kbase_device; ++struct kbase_as; ++struct kbase_context; ++ ++/** ++ * @addtogroup base_kbase_api ++ * @{ ++ */ ++ ++/** ++ * @addtogroup mali_kbase_mmu_hw MMU access APIs ++ * @{ ++ */ ++ ++/** @brief MMU fault type descriptor. ++ */ ++enum kbase_mmu_fault_type { ++ KBASE_MMU_FAULT_TYPE_UNKNOWN = 0, ++ KBASE_MMU_FAULT_TYPE_PAGE, ++ KBASE_MMU_FAULT_TYPE_BUS, ++ KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED, ++ KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED ++}; ++ ++/** @brief Configure an address space for use. ++ * ++ * Configure the MMU using the address space details setup in the ++ * @ref kbase_context structure. ++ * ++ * @param[in] kbdev kbase device to configure. ++ * @param[in] as address space to configure. ++ * @param[in] kctx kbase context to configure. ++ */ ++void kbase_mmu_hw_configure(struct kbase_device *kbdev, ++ struct kbase_as *as, struct kbase_context *kctx); ++ ++/** @brief Issue an operation to the MMU. ++ * ++ * Issue an operation (MMU invalidate, MMU flush, etc) on the address space that ++ * is associated with the provided @ref kbase_context over the specified range ++ * ++ * @param[in] kbdev kbase device to issue the MMU operation on. ++ * @param[in] as address space to issue the MMU operation on. ++ * @param[in] kctx kbase context to issue the MMU operation on. ++ * @param[in] vpfn MMU Virtual Page Frame Number to start the ++ * operation on. ++ * @param[in] nr Number of pages to work on. ++ * @param[in] type Operation type (written to ASn_COMMAND). ++ * @param[in] handling_irq Is this operation being called during the handling ++ * of an interrupt? ++ * ++ * @return Zero if the operation was successful, non-zero otherwise. ++ */ ++int kbase_mmu_hw_do_operation(struct kbase_device *kbdev, struct kbase_as *as, ++ struct kbase_context *kctx, u64 vpfn, u32 nr, u32 type, ++ unsigned int handling_irq); ++ ++/** @brief Clear a fault that has been previously reported by the MMU. ++ * ++ * Clear a bus error or page fault that has been reported by the MMU. ++ * ++ * @param[in] kbdev kbase device to clear the fault from. ++ * @param[in] as address space to clear the fault from. ++ * @param[in] kctx kbase context to clear the fault from or NULL. ++ * @param[in] type The type of fault that needs to be cleared. ++ */ ++void kbase_mmu_hw_clear_fault(struct kbase_device *kbdev, struct kbase_as *as, ++ struct kbase_context *kctx, enum kbase_mmu_fault_type type); ++ ++/** @brief Enable fault that has been previously reported by the MMU. ++ * ++ * After a page fault or bus error has been reported by the MMU these ++ * will be disabled. After these are handled this function needs to be ++ * called to enable the page fault or bus error fault again. ++ * ++ * @param[in] kbdev kbase device to again enable the fault from. ++ * @param[in] as address space to again enable the fault from. ++ * @param[in] kctx kbase context to again enable the fault from. ++ * @param[in] type The type of fault that needs to be enabled again. ++ */ ++void kbase_mmu_hw_enable_fault(struct kbase_device *kbdev, struct kbase_as *as, ++ struct kbase_context *kctx, enum kbase_mmu_fault_type type); ++ ++/** @} *//* end group mali_kbase_mmu_hw */ ++/** @} *//* end group base_kbase_api */ ++ ++#endif /* _MALI_KBASE_MMU_HW_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mmu_mode_aarch64.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mmu_mode_aarch64.c +new file mode 100755 +index 000000000000..0fb717b67af9 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mmu_mode_aarch64.c +@@ -0,0 +1,214 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#include "mali_kbase.h" ++#include "mali_midg_regmap.h" ++#include "mali_kbase_defs.h" ++ ++#define ENTRY_TYPE_MASK 3ULL ++/* For valid ATEs bit 1 = ((level == 3) ? 1 : 0). ++ * Valid ATE entries at level 3 are flagged with the value 3. ++ * Valid ATE entries at level 0-2 are flagged with the value 1. ++ */ ++#define ENTRY_IS_ATE_L3 3ULL ++#define ENTRY_IS_ATE_L02 1ULL ++#define ENTRY_IS_INVAL 2ULL ++#define ENTRY_IS_PTE 3ULL ++ ++#define ENTRY_ATTR_BITS (7ULL << 2) /* bits 4:2 */ ++#define ENTRY_ACCESS_RW (1ULL << 6) /* bits 6:7 */ ++#define ENTRY_ACCESS_RO (3ULL << 6) ++#define ENTRY_SHARE_BITS (3ULL << 8) /* bits 9:8 */ ++#define ENTRY_ACCESS_BIT (1ULL << 10) ++#define ENTRY_NX_BIT (1ULL << 54) ++ ++/* Helper Function to perform assignment of page table entries, to ++ * ensure the use of strd, which is required on LPAE systems. ++ */ ++static inline void page_table_entry_set(u64 *pte, u64 phy) ++{ ++#ifdef CONFIG_64BIT ++ *pte = phy; ++#elif defined(CONFIG_ARM) ++ /* ++ * In order to prevent the compiler keeping cached copies of ++ * memory, we have to explicitly say that we have updated memory. ++ * ++ * Note: We could manually move the data ourselves into R0 and ++ * R1 by specifying register variables that are explicitly ++ * given registers assignments, the down side of this is that ++ * we have to assume cpu endianness. To avoid this we can use ++ * the ldrd to read the data from memory into R0 and R1 which ++ * will respect the cpu endianness, we then use strd to make ++ * the 64 bit assignment to the page table entry. ++ */ ++ asm volatile("ldrd r0, r1, [%[ptemp]]\n\t" ++ "strd r0, r1, [%[pte]]\n\t" ++ : "=m" (*pte) ++ : [ptemp] "r" (&phy), [pte] "r" (pte), "m" (phy) ++ : "r0", "r1"); ++#else ++#error "64-bit atomic write must be implemented for your architecture" ++#endif ++} ++ ++static void mmu_get_as_setup(struct kbase_context *kctx, ++ struct kbase_mmu_setup * const setup) ++{ ++ /* Set up the required caching policies at the correct indices ++ * in the memattr register. ++ */ ++ setup->memattr = ++ (AS_MEMATTR_IMPL_DEF_CACHE_POLICY << ++ (AS_MEMATTR_INDEX_IMPL_DEF_CACHE_POLICY * 8)) | ++ (AS_MEMATTR_FORCE_TO_CACHE_ALL << ++ (AS_MEMATTR_INDEX_FORCE_TO_CACHE_ALL * 8)) | ++ (AS_MEMATTR_WRITE_ALLOC << ++ (AS_MEMATTR_INDEX_WRITE_ALLOC * 8)) | ++ (AS_MEMATTR_AARCH64_OUTER_IMPL_DEF << ++ (AS_MEMATTR_INDEX_OUTER_IMPL_DEF * 8)) | ++ (AS_MEMATTR_AARCH64_OUTER_WA << ++ (AS_MEMATTR_INDEX_OUTER_WA * 8)); ++ ++ setup->transtab = (u64)kctx->pgd & AS_TRANSTAB_BASE_MASK; ++ setup->transcfg = AS_TRANSCFG_ADRMODE_AARCH64_4K; ++} ++ ++static void mmu_update(struct kbase_context *kctx) ++{ ++ struct kbase_device * const kbdev = kctx->kbdev; ++ struct kbase_as * const as = &kbdev->as[kctx->as_nr]; ++ struct kbase_mmu_setup * const current_setup = &as->current_setup; ++ ++ mmu_get_as_setup(kctx, current_setup); ++ ++ /* Apply the address space setting */ ++ kbase_mmu_hw_configure(kbdev, as, kctx); ++} ++ ++static void mmu_disable_as(struct kbase_device *kbdev, int as_nr) ++{ ++ struct kbase_as * const as = &kbdev->as[as_nr]; ++ struct kbase_mmu_setup * const current_setup = &as->current_setup; ++ ++ current_setup->transtab = 0ULL; ++ current_setup->transcfg = AS_TRANSCFG_ADRMODE_UNMAPPED; ++ ++ /* Apply the address space setting */ ++ kbase_mmu_hw_configure(kbdev, as, NULL); ++} ++ ++static phys_addr_t pte_to_phy_addr(u64 entry) ++{ ++ if (!(entry & 1)) ++ return 0; ++ ++ return entry & ~0xFFF; ++} ++ ++static int ate_is_valid(u64 ate, unsigned int level) ++{ ++ if (level == MIDGARD_MMU_BOTTOMLEVEL) ++ return ((ate & ENTRY_TYPE_MASK) == ENTRY_IS_ATE_L3); ++ else ++ return ((ate & ENTRY_TYPE_MASK) == ENTRY_IS_ATE_L02); ++} ++ ++static int pte_is_valid(u64 pte, unsigned int level) ++{ ++ /* PTEs cannot exist at the bottom level */ ++ if (level == MIDGARD_MMU_BOTTOMLEVEL) ++ return false; ++ return ((pte & ENTRY_TYPE_MASK) == ENTRY_IS_PTE); ++} ++ ++/* ++ * Map KBASE_REG flags to MMU flags ++ */ ++static u64 get_mmu_flags(unsigned long flags) ++{ ++ u64 mmu_flags; ++ ++ /* store mem_attr index as 4:2 (macro called ensures 3 bits already) */ ++ mmu_flags = KBASE_REG_MEMATTR_VALUE(flags) << 2; ++ ++ /* Set access flags - note that AArch64 stage 1 does not support ++ * write-only access, so we use read/write instead ++ */ ++ if (flags & KBASE_REG_GPU_WR) ++ mmu_flags |= ENTRY_ACCESS_RW; ++ else if (flags & KBASE_REG_GPU_RD) ++ mmu_flags |= ENTRY_ACCESS_RO; ++ ++ /* nx if requested */ ++ mmu_flags |= (flags & KBASE_REG_GPU_NX) ? ENTRY_NX_BIT : 0; ++ ++ if (flags & KBASE_REG_SHARE_BOTH) { ++ /* inner and outer shareable */ ++ mmu_flags |= SHARE_BOTH_BITS; ++ } else if (flags & KBASE_REG_SHARE_IN) { ++ /* inner shareable coherency */ ++ mmu_flags |= SHARE_INNER_BITS; ++ } ++ ++ return mmu_flags; ++} ++ ++static void entry_set_ate(u64 *entry, ++ struct tagged_addr phy, ++ unsigned long flags, ++ unsigned int level) ++{ ++ if (level == MIDGARD_MMU_BOTTOMLEVEL) ++ page_table_entry_set(entry, as_phys_addr_t(phy) | ++ get_mmu_flags(flags) | ++ ENTRY_ACCESS_BIT | ENTRY_IS_ATE_L3); ++ else ++ page_table_entry_set(entry, as_phys_addr_t(phy) | ++ get_mmu_flags(flags) | ++ ENTRY_ACCESS_BIT | ENTRY_IS_ATE_L02); ++} ++ ++static void entry_set_pte(u64 *entry, phys_addr_t phy) ++{ ++ page_table_entry_set(entry, (phy & PAGE_MASK) | ++ ENTRY_ACCESS_BIT | ENTRY_IS_PTE); ++} ++ ++static void entry_invalidate(u64 *entry) ++{ ++ page_table_entry_set(entry, ENTRY_IS_INVAL); ++} ++ ++static struct kbase_mmu_mode const aarch64_mode = { ++ .update = mmu_update, ++ .get_as_setup = mmu_get_as_setup, ++ .disable_as = mmu_disable_as, ++ .pte_to_phy_addr = pte_to_phy_addr, ++ .ate_is_valid = ate_is_valid, ++ .pte_is_valid = pte_is_valid, ++ .entry_set_ate = entry_set_ate, ++ .entry_set_pte = entry_set_pte, ++ .entry_invalidate = entry_invalidate ++}; ++ ++struct kbase_mmu_mode const *kbase_mmu_mode_get_aarch64(void) ++{ ++ return &aarch64_mode; ++} +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mmu_mode_lpae.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mmu_mode_lpae.c +new file mode 100755 +index 000000000000..f080fdc0be88 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mmu_mode_lpae.c +@@ -0,0 +1,199 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#include "mali_kbase.h" ++#include "mali_midg_regmap.h" ++#include "mali_kbase_defs.h" ++ ++#define ENTRY_TYPE_MASK 3ULL ++#define ENTRY_IS_ATE 1ULL ++#define ENTRY_IS_INVAL 2ULL ++#define ENTRY_IS_PTE 3ULL ++ ++#define ENTRY_ATTR_BITS (7ULL << 2) /* bits 4:2 */ ++#define ENTRY_RD_BIT (1ULL << 6) ++#define ENTRY_WR_BIT (1ULL << 7) ++#define ENTRY_SHARE_BITS (3ULL << 8) /* bits 9:8 */ ++#define ENTRY_ACCESS_BIT (1ULL << 10) ++#define ENTRY_NX_BIT (1ULL << 54) ++ ++#define ENTRY_FLAGS_MASK (ENTRY_ATTR_BITS | ENTRY_RD_BIT | ENTRY_WR_BIT | \ ++ ENTRY_SHARE_BITS | ENTRY_ACCESS_BIT | ENTRY_NX_BIT) ++ ++/* Helper Function to perform assignment of page table entries, to ++ * ensure the use of strd, which is required on LPAE systems. ++ */ ++static inline void page_table_entry_set(u64 *pte, u64 phy) ++{ ++#ifdef CONFIG_64BIT ++ *pte = phy; ++#elif defined(CONFIG_ARM) ++ /* ++ * In order to prevent the compiler keeping cached copies of ++ * memory, we have to explicitly say that we have updated ++ * memory. ++ * ++ * Note: We could manually move the data ourselves into R0 and ++ * R1 by specifying register variables that are explicitly ++ * given registers assignments, the down side of this is that ++ * we have to assume cpu endianness. To avoid this we can use ++ * the ldrd to read the data from memory into R0 and R1 which ++ * will respect the cpu endianness, we then use strd to make ++ * the 64 bit assignment to the page table entry. ++ */ ++ asm volatile("ldrd r0, r1, [%[ptemp]]\n\t" ++ "strd r0, r1, [%[pte]]\n\t" ++ : "=m" (*pte) ++ : [ptemp] "r" (&phy), [pte] "r" (pte), "m" (phy) ++ : "r0", "r1"); ++#else ++#error "64-bit atomic write must be implemented for your architecture" ++#endif ++} ++ ++static void mmu_get_as_setup(struct kbase_context *kctx, ++ struct kbase_mmu_setup * const setup) ++{ ++ /* Set up the required caching policies at the correct indices ++ * in the memattr register. */ ++ setup->memattr = ++ (AS_MEMATTR_LPAE_IMPL_DEF_CACHE_POLICY << ++ (AS_MEMATTR_INDEX_IMPL_DEF_CACHE_POLICY * 8)) | ++ (AS_MEMATTR_LPAE_FORCE_TO_CACHE_ALL << ++ (AS_MEMATTR_INDEX_FORCE_TO_CACHE_ALL * 8)) | ++ (AS_MEMATTR_LPAE_WRITE_ALLOC << ++ (AS_MEMATTR_INDEX_WRITE_ALLOC * 8)) | ++ (AS_MEMATTR_LPAE_OUTER_IMPL_DEF << ++ (AS_MEMATTR_INDEX_OUTER_IMPL_DEF * 8)) | ++ (AS_MEMATTR_LPAE_OUTER_WA << ++ (AS_MEMATTR_INDEX_OUTER_WA * 8)) | ++ 0; /* The other indices are unused for now */ ++ ++ setup->transtab = ((u64)kctx->pgd & ++ ((0xFFFFFFFFULL << 32) | AS_TRANSTAB_LPAE_ADDR_SPACE_MASK)) | ++ AS_TRANSTAB_LPAE_ADRMODE_TABLE | ++ AS_TRANSTAB_LPAE_READ_INNER; ++ ++ setup->transcfg = 0; ++} ++ ++static void mmu_update(struct kbase_context *kctx) ++{ ++ struct kbase_device * const kbdev = kctx->kbdev; ++ struct kbase_as * const as = &kbdev->as[kctx->as_nr]; ++ struct kbase_mmu_setup * const current_setup = &as->current_setup; ++ ++ mmu_get_as_setup(kctx, current_setup); ++ ++ /* Apply the address space setting */ ++ kbase_mmu_hw_configure(kbdev, as, kctx); ++} ++ ++static void mmu_disable_as(struct kbase_device *kbdev, int as_nr) ++{ ++ struct kbase_as * const as = &kbdev->as[as_nr]; ++ struct kbase_mmu_setup * const current_setup = &as->current_setup; ++ ++ current_setup->transtab = AS_TRANSTAB_LPAE_ADRMODE_UNMAPPED; ++ ++ /* Apply the address space setting */ ++ kbase_mmu_hw_configure(kbdev, as, NULL); ++} ++ ++static phys_addr_t pte_to_phy_addr(u64 entry) ++{ ++ if (!(entry & 1)) ++ return 0; ++ ++ return entry & ~0xFFF; ++} ++ ++static int ate_is_valid(u64 ate, unsigned int level) ++{ ++ return ((ate & ENTRY_TYPE_MASK) == ENTRY_IS_ATE); ++} ++ ++static int pte_is_valid(u64 pte, unsigned int level) ++{ ++ return ((pte & ENTRY_TYPE_MASK) == ENTRY_IS_PTE); ++} ++ ++/* ++ * Map KBASE_REG flags to MMU flags ++ */ ++static u64 get_mmu_flags(unsigned long flags) ++{ ++ u64 mmu_flags; ++ ++ /* store mem_attr index as 4:2 (macro called ensures 3 bits already) */ ++ mmu_flags = KBASE_REG_MEMATTR_VALUE(flags) << 2; ++ ++ /* write perm if requested */ ++ mmu_flags |= (flags & KBASE_REG_GPU_WR) ? ENTRY_WR_BIT : 0; ++ /* read perm if requested */ ++ mmu_flags |= (flags & KBASE_REG_GPU_RD) ? ENTRY_RD_BIT : 0; ++ /* nx if requested */ ++ mmu_flags |= (flags & KBASE_REG_GPU_NX) ? ENTRY_NX_BIT : 0; ++ ++ if (flags & KBASE_REG_SHARE_BOTH) { ++ /* inner and outer shareable */ ++ mmu_flags |= SHARE_BOTH_BITS; ++ } else if (flags & KBASE_REG_SHARE_IN) { ++ /* inner shareable coherency */ ++ mmu_flags |= SHARE_INNER_BITS; ++ } ++ ++ return mmu_flags; ++} ++ ++static void entry_set_ate(u64 *entry, ++ struct tagged_addr phy, ++ unsigned long flags, ++ unsigned int level) ++{ ++ page_table_entry_set(entry, as_phys_addr_t(phy) | get_mmu_flags(flags) | ++ ENTRY_IS_ATE); ++} ++ ++static void entry_set_pte(u64 *entry, phys_addr_t phy) ++{ ++ page_table_entry_set(entry, (phy & ~0xFFF) | ENTRY_IS_PTE); ++} ++ ++static void entry_invalidate(u64 *entry) ++{ ++ page_table_entry_set(entry, ENTRY_IS_INVAL); ++} ++ ++static struct kbase_mmu_mode const lpae_mode = { ++ .update = mmu_update, ++ .get_as_setup = mmu_get_as_setup, ++ .disable_as = mmu_disable_as, ++ .pte_to_phy_addr = pte_to_phy_addr, ++ .ate_is_valid = ate_is_valid, ++ .pte_is_valid = pte_is_valid, ++ .entry_set_ate = entry_set_ate, ++ .entry_set_pte = entry_set_pte, ++ .entry_invalidate = entry_invalidate ++}; ++ ++struct kbase_mmu_mode const *kbase_mmu_mode_get_lpae(void) ++{ ++ return &lpae_mode; ++} +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_platform_fake.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_platform_fake.c +new file mode 100755 +index 000000000000..0152b35f711b +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_platform_fake.c +@@ -0,0 +1,119 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include ++#include ++#include ++#include ++#include ++ ++ ++/* ++ * This file is included only for type definitions and functions belonging to ++ * specific platform folders. Do not add dependencies with symbols that are ++ * defined somewhere else. ++ */ ++#include ++ ++#define PLATFORM_CONFIG_RESOURCE_COUNT 4 ++#define PLATFORM_CONFIG_IRQ_RES_COUNT 3 ++ ++static struct platform_device *mali_device; ++ ++#ifndef CONFIG_OF ++/** ++ * @brief Convert data in struct kbase_io_resources struct to Linux-specific resources ++ * ++ * Function converts data in struct kbase_io_resources struct to an array of Linux resource structures. Note that function ++ * assumes that size of linux_resource array is at least PLATFORM_CONFIG_RESOURCE_COUNT. ++ * Resources are put in fixed order: I/O memory region, job IRQ, MMU IRQ, GPU IRQ. ++ * ++ * @param[in] io_resource Input IO resource data ++ * @param[out] linux_resources Pointer to output array of Linux resource structures ++ */ ++static void kbasep_config_parse_io_resources(const struct kbase_io_resources *io_resources, struct resource *const linux_resources) ++{ ++ if (!io_resources || !linux_resources) { ++ pr_err("%s: couldn't find proper resources\n", __func__); ++ return; ++ } ++ ++ memset(linux_resources, 0, PLATFORM_CONFIG_RESOURCE_COUNT * sizeof(struct resource)); ++ ++ linux_resources[0].start = io_resources->io_memory_region.start; ++ linux_resources[0].end = io_resources->io_memory_region.end; ++ linux_resources[0].flags = IORESOURCE_MEM; ++ ++ linux_resources[1].start = io_resources->job_irq_number; ++ linux_resources[1].end = io_resources->job_irq_number; ++ linux_resources[1].flags = IORESOURCE_IRQ | IORESOURCE_IRQ_HIGHLEVEL; ++ ++ linux_resources[2].start = io_resources->mmu_irq_number; ++ linux_resources[2].end = io_resources->mmu_irq_number; ++ linux_resources[2].flags = IORESOURCE_IRQ | IORESOURCE_IRQ_HIGHLEVEL; ++ ++ linux_resources[3].start = io_resources->gpu_irq_number; ++ linux_resources[3].end = io_resources->gpu_irq_number; ++ linux_resources[3].flags = IORESOURCE_IRQ | IORESOURCE_IRQ_HIGHLEVEL; ++} ++#endif /* CONFIG_OF */ ++ ++int kbase_platform_register(void) ++{ ++ struct kbase_platform_config *config; ++#ifndef CONFIG_OF ++ struct resource resources[PLATFORM_CONFIG_RESOURCE_COUNT]; ++#endif ++ int err; ++ ++ config = kbase_get_platform_config(); /* declared in midgard/mali_kbase_config.h but defined in platform folder */ ++ if (config == NULL) { ++ pr_err("%s: couldn't get platform config\n", __func__); ++ return -ENODEV; ++ } ++ ++ mali_device = platform_device_alloc("mali", 0); ++ if (mali_device == NULL) ++ return -ENOMEM; ++ ++#ifndef CONFIG_OF ++ kbasep_config_parse_io_resources(config->io_resources, resources); ++ err = platform_device_add_resources(mali_device, resources, PLATFORM_CONFIG_RESOURCE_COUNT); ++ if (err) { ++ platform_device_put(mali_device); ++ mali_device = NULL; ++ return err; ++ } ++#endif /* CONFIG_OF */ ++ ++ err = platform_device_add(mali_device); ++ if (err) { ++ platform_device_unregister(mali_device); ++ mali_device = NULL; ++ return err; ++ } ++ ++ return 0; ++} ++EXPORT_SYMBOL(kbase_platform_register); ++ ++void kbase_platform_unregister(void) ++{ ++ if (mali_device) ++ platform_device_unregister(mali_device); ++} ++EXPORT_SYMBOL(kbase_platform_unregister); +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_pm.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_pm.c +new file mode 100755 +index 000000000000..97d543464c28 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_pm.c +@@ -0,0 +1,205 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/** ++ * @file mali_kbase_pm.c ++ * Base kernel power management APIs ++ */ ++ ++#include ++#include ++#include ++ ++#include ++ ++int kbase_pm_powerup(struct kbase_device *kbdev, unsigned int flags) ++{ ++ return kbase_hwaccess_pm_powerup(kbdev, flags); ++} ++ ++void kbase_pm_halt(struct kbase_device *kbdev) ++{ ++ kbase_hwaccess_pm_halt(kbdev); ++} ++ ++void kbase_pm_context_active(struct kbase_device *kbdev) ++{ ++ (void)kbase_pm_context_active_handle_suspend(kbdev, KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE); ++} ++ ++int kbase_pm_context_active_handle_suspend(struct kbase_device *kbdev, enum kbase_pm_suspend_handler suspend_handler) ++{ ++ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; ++ int c; ++ int old_count; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ /* Trace timeline information about how long it took to handle the decision ++ * to powerup. Sometimes the event might be missed due to reading the count ++ * outside of mutex, but this is necessary to get the trace timing ++ * correct. */ ++ old_count = kbdev->pm.active_count; ++ if (old_count == 0) ++ kbase_timeline_pm_send_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_ACTIVE); ++ ++ mutex_lock(&js_devdata->runpool_mutex); ++ mutex_lock(&kbdev->pm.lock); ++ if (kbase_pm_is_suspending(kbdev)) { ++ switch (suspend_handler) { ++ case KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE: ++ if (kbdev->pm.active_count != 0) ++ break; ++ /* FALLTHROUGH */ ++ case KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE: ++ mutex_unlock(&kbdev->pm.lock); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ if (old_count == 0) ++ kbase_timeline_pm_handle_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_ACTIVE); ++ return 1; ++ ++ case KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE: ++ /* FALLTHROUGH */ ++ default: ++ KBASE_DEBUG_ASSERT_MSG(false, "unreachable"); ++ break; ++ } ++ } ++ c = ++kbdev->pm.active_count; ++ KBASE_TIMELINE_CONTEXT_ACTIVE(kbdev, c); ++ KBASE_TRACE_ADD_REFCOUNT(kbdev, PM_CONTEXT_ACTIVE, NULL, NULL, 0u, c); ++ ++ /* Trace the event being handled */ ++ if (old_count == 0) ++ kbase_timeline_pm_handle_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_ACTIVE); ++ ++ if (c == 1) ++ /* First context active: Power on the GPU and any cores requested by ++ * the policy */ ++ kbase_hwaccess_pm_gpu_active(kbdev); ++ ++ mutex_unlock(&kbdev->pm.lock); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ ++ return 0; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_context_active); ++ ++void kbase_pm_context_idle(struct kbase_device *kbdev) ++{ ++ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; ++ int c; ++ int old_count; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ /* Trace timeline information about how long it took to handle the decision ++ * to powerdown. Sometimes the event might be missed due to reading the ++ * count outside of mutex, but this is necessary to get the trace timing ++ * correct. */ ++ old_count = kbdev->pm.active_count; ++ if (old_count == 0) ++ kbase_timeline_pm_send_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_IDLE); ++ ++ mutex_lock(&js_devdata->runpool_mutex); ++ mutex_lock(&kbdev->pm.lock); ++ ++ c = --kbdev->pm.active_count; ++ KBASE_TIMELINE_CONTEXT_ACTIVE(kbdev, c); ++ KBASE_TRACE_ADD_REFCOUNT(kbdev, PM_CONTEXT_IDLE, NULL, NULL, 0u, c); ++ ++ KBASE_DEBUG_ASSERT(c >= 0); ++ ++ /* Trace the event being handled */ ++ if (old_count == 0) ++ kbase_timeline_pm_handle_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_IDLE); ++ ++ if (c == 0) { ++ /* Last context has gone idle */ ++ kbase_hwaccess_pm_gpu_idle(kbdev); ++ ++ /* Wake up anyone waiting for this to become 0 (e.g. suspend). The ++ * waiters must synchronize with us by locking the pm.lock after ++ * waiting */ ++ wake_up(&kbdev->pm.zero_active_count_wait); ++ } ++ ++ mutex_unlock(&kbdev->pm.lock); ++ mutex_unlock(&js_devdata->runpool_mutex); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_context_idle); ++ ++void kbase_pm_suspend(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev); ++ ++ /* Suspend vinstr. ++ * This call will block until vinstr is suspended. */ ++ kbase_vinstr_suspend(kbdev->vinstr_ctx); ++ ++ mutex_lock(&kbdev->pm.lock); ++ KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev)); ++ kbdev->pm.suspending = true; ++ mutex_unlock(&kbdev->pm.lock); ++ ++ /* From now on, the active count will drop towards zero. Sometimes, it'll ++ * go up briefly before going down again. However, once it reaches zero it ++ * will stay there - guaranteeing that we've idled all pm references */ ++ ++ /* Suspend job scheduler and associated components, so that it releases all ++ * the PM active count references */ ++ kbasep_js_suspend(kbdev); ++ ++ /* Wait for the active count to reach zero. This is not the same as ++ * waiting for a power down, since not all policies power down when this ++ * reaches zero. */ ++ wait_event(kbdev->pm.zero_active_count_wait, kbdev->pm.active_count == 0); ++ ++ /* NOTE: We synchronize with anything that was just finishing a ++ * kbase_pm_context_idle() call by locking the pm.lock below */ ++ ++ kbase_hwaccess_pm_suspend(kbdev); ++} ++ ++void kbase_pm_resume(struct kbase_device *kbdev) ++{ ++ /* MUST happen before any pm_context_active calls occur */ ++ kbase_hwaccess_pm_resume(kbdev); ++ ++ /* Initial active call, to power on the GPU/cores if needed */ ++ kbase_pm_context_active(kbdev); ++ ++ /* Resume any blocked atoms (which may cause contexts to be scheduled in ++ * and dependent atoms to run) */ ++ kbase_resume_suspended_soft_jobs(kbdev); ++ ++ /* Resume the Job Scheduler and associated components, and start running ++ * atoms */ ++ kbasep_js_resume(kbdev); ++ ++ /* Matching idle call, to power off the GPU/cores if we didn't actually ++ * need it and the policy doesn't want it on */ ++ kbase_pm_context_idle(kbdev); ++ ++ /* Resume vinstr operation */ ++ kbase_vinstr_resume(kbdev->vinstr_ctx); ++} ++ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_pm.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_pm.h +new file mode 100755 +index 000000000000..37fa2479df74 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_pm.h +@@ -0,0 +1,171 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/** ++ * @file mali_kbase_pm.h ++ * Power management API definitions ++ */ ++ ++#ifndef _KBASE_PM_H_ ++#define _KBASE_PM_H_ ++ ++#include "mali_kbase_hwaccess_pm.h" ++ ++#define PM_ENABLE_IRQS 0x01 ++#define PM_HW_ISSUES_DETECT 0x02 ++ ++ ++/** Initialize the power management framework. ++ * ++ * Must be called before any other power management function ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid pointer) ++ * ++ * @return 0 if the power management framework was successfully initialized. ++ */ ++int kbase_pm_init(struct kbase_device *kbdev); ++ ++/** Power up GPU after all modules have been initialized and interrupt handlers installed. ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid pointer) ++ * ++ * @param flags Flags to pass on to kbase_pm_init_hw ++ * ++ * @return 0 if powerup was successful. ++ */ ++int kbase_pm_powerup(struct kbase_device *kbdev, unsigned int flags); ++ ++/** ++ * Halt the power management framework. ++ * Should ensure that no new interrupts are generated, ++ * but allow any currently running interrupt handlers to complete successfully. ++ * The GPU is forced off by the time this function returns, regardless of ++ * whether or not the active power policy asks for the GPU to be powered off. ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_halt(struct kbase_device *kbdev); ++ ++/** Terminate the power management framework. ++ * ++ * No power management functions may be called after this ++ * (except @ref kbase_pm_init) ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_term(struct kbase_device *kbdev); ++ ++/** Increment the count of active contexts. ++ * ++ * This function should be called when a context is about to submit a job. It informs the active power policy that the ++ * GPU is going to be in use shortly and the policy is expected to start turning on the GPU. ++ * ++ * This function will block until the GPU is available. ++ * ++ * This function ASSERTS if a suspend is occuring/has occurred whilst this is ++ * in use. Use kbase_pm_contect_active_unless_suspending() instead. ++ * ++ * @note a Suspend is only visible to Kernel threads; user-space threads in a ++ * syscall cannot witness a suspend, because they are frozen before the suspend ++ * begins. ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_context_active(struct kbase_device *kbdev); ++ ++ ++/** Handler codes for doing kbase_pm_context_active_handle_suspend() */ ++enum kbase_pm_suspend_handler { ++ /** A suspend is not expected/not possible - this is the same as ++ * kbase_pm_context_active() */ ++ KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE, ++ /** If we're suspending, fail and don't increase the active count */ ++ KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE, ++ /** If we're suspending, succeed and allow the active count to increase iff ++ * it didn't go from 0->1 (i.e., we didn't re-activate the GPU). ++ * ++ * This should only be used when there is a bounded time on the activation ++ * (e.g. guarantee it's going to be idled very soon after) */ ++ KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE ++}; ++ ++/** Suspend 'safe' variant of kbase_pm_context_active() ++ * ++ * If a suspend is in progress, this allows for various different ways of ++ * handling the suspend. Refer to @ref enum kbase_pm_suspend_handler for details. ++ * ++ * We returns a status code indicating whether we're allowed to keep the GPU ++ * active during the suspend, depending on the handler code. If the status code ++ * indicates a failure, the caller must abort whatever operation it was ++ * attempting, and potentially queue it up for after the OS has resumed. ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid pointer) ++ * @param suspend_handler The handler code for how to handle a suspend that might occur ++ * @return zero Indicates success ++ * @return non-zero Indicates failure due to the system being suspending/suspended. ++ */ ++int kbase_pm_context_active_handle_suspend(struct kbase_device *kbdev, enum kbase_pm_suspend_handler suspend_handler); ++ ++/** Decrement the reference count of active contexts. ++ * ++ * This function should be called when a context becomes idle. After this call the GPU may be turned off by the power ++ * policy so the calling code should ensure that it does not access the GPU's registers. ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_context_idle(struct kbase_device *kbdev); ++ ++/** ++ * Suspend the GPU and prevent any further register accesses to it from Kernel ++ * threads. ++ * ++ * This is called in response to an OS suspend event, and calls into the various ++ * kbase components to complete the suspend. ++ * ++ * @note the mechanisms used here rely on all user-space threads being frozen ++ * by the OS before we suspend. Otherwise, an IOCTL could occur that powers up ++ * the GPU e.g. via atom submission. ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_suspend(struct kbase_device *kbdev); ++ ++/** ++ * Resume the GPU, allow register accesses to it, and resume running atoms on ++ * the GPU. ++ * ++ * This is called in response to an OS resume event, and calls into the various ++ * kbase components to complete the resume. ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_resume(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_vsync_callback - vsync callback ++ * ++ * @buffer_updated: 1 if a new frame was displayed, 0 otherwise ++ * @data: Pointer to the kbase device as returned by kbase_find_device() ++ * ++ * Callback function used to notify the power management code that a vsync has ++ * occurred on the display. ++ */ ++void kbase_pm_vsync_callback(int buffer_updated, void *data); ++ ++#endif /* _KBASE_PM_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_profiling_gator_api.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_profiling_gator_api.h +new file mode 100755 +index 000000000000..7fb674eded37 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_profiling_gator_api.h +@@ -0,0 +1,40 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010, 2013 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/** ++ * @file mali_kbase_profiling_gator_api.h ++ * Model interface ++ */ ++ ++#ifndef _KBASE_PROFILING_GATOR_API_H_ ++#define _KBASE_PROFILING_GATOR_API_H_ ++ ++/* ++ * List of possible actions to be controlled by Streamline. ++ * The following numbers are used by gator to control ++ * the frame buffer dumping and s/w counter reporting. ++ */ ++#define FBDUMP_CONTROL_ENABLE (1) ++#define FBDUMP_CONTROL_RATE (2) ++#define SW_COUNTER_ENABLE (3) ++#define FBDUMP_CONTROL_RESIZE_FACTOR (4) ++#define FBDUMP_CONTROL_MAX (5) ++#define FBDUMP_CONTROL_MIN FBDUMP_CONTROL_ENABLE ++ ++void _mali_profiling_control(u32 action, u32 value); ++ ++#endif /* _KBASE_PROFILING_GATOR_API */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_regs_history_debugfs.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_regs_history_debugfs.c +new file mode 100755 +index 000000000000..9e73f9f4999e +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_regs_history_debugfs.c +@@ -0,0 +1,130 @@ ++/* ++ * ++ * (C) COPYRIGHT 2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include "mali_kbase.h" ++ ++#include "mali_kbase_regs_history_debugfs.h" ++ ++#if defined(CONFIG_DEBUG_FS) && !defined(CONFIG_MALI_BIFROST_NO_MALI) ++ ++#include ++ ++ ++static int regs_history_size_get(void *data, u64 *val) ++{ ++ struct kbase_io_history *const h = data; ++ ++ *val = h->size; ++ ++ return 0; ++} ++ ++static int regs_history_size_set(void *data, u64 val) ++{ ++ struct kbase_io_history *const h = data; ++ ++ return kbase_io_history_resize(h, (u16)val); ++} ++ ++ ++DEFINE_SIMPLE_ATTRIBUTE(regs_history_size_fops, ++ regs_history_size_get, ++ regs_history_size_set, ++ "%llu\n"); ++ ++ ++/** ++ * regs_history_show - show callback for the register access history file. ++ * ++ * @sfile: The debugfs entry ++ * @data: Data associated with the entry ++ * ++ * This function is called to dump all recent accesses to the GPU registers. ++ * ++ * @return 0 if successfully prints data in debugfs entry file, failure ++ * otherwise ++ */ ++static int regs_history_show(struct seq_file *sfile, void *data) ++{ ++ struct kbase_io_history *const h = sfile->private; ++ u16 i; ++ size_t iters; ++ unsigned long flags; ++ ++ if (!h->enabled) { ++ seq_puts(sfile, "The register access history is disabled\n"); ++ goto out; ++ } ++ ++ spin_lock_irqsave(&h->lock, flags); ++ ++ iters = (h->size > h->count) ? h->count : h->size; ++ seq_printf(sfile, "Last %zu register accesses of %zu total:\n", iters, ++ h->count); ++ for (i = 0; i < iters; ++i) { ++ struct kbase_io_access *io = ++ &h->buf[(h->count - iters + i) % h->size]; ++ char const access = (io->addr & 1) ? 'w' : 'r'; ++ ++ seq_printf(sfile, "%6i: %c: reg 0x%p val %08x\n", i, access, ++ (void *)(io->addr & ~0x1), io->value); ++ } ++ ++ spin_unlock_irqrestore(&h->lock, flags); ++ ++out: ++ return 0; ++} ++ ++ ++/** ++ * regs_history_open - open operation for regs_history debugfs file ++ * ++ * @in: &struct inode pointer ++ * @file: &struct file pointer ++ * ++ * @return file descriptor ++ */ ++static int regs_history_open(struct inode *in, struct file *file) ++{ ++ return single_open(file, ®s_history_show, in->i_private); ++} ++ ++ ++static const struct file_operations regs_history_fops = { ++ .open = ®s_history_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = single_release, ++}; ++ ++ ++void kbasep_regs_history_debugfs_init(struct kbase_device *kbdev) ++{ ++ debugfs_create_bool("regs_history_enabled", S_IRUGO | S_IWUSR, ++ kbdev->mali_debugfs_directory, ++ &kbdev->io_history.enabled); ++ debugfs_create_file("regs_history_size", S_IRUGO | S_IWUSR, ++ kbdev->mali_debugfs_directory, ++ &kbdev->io_history, ®s_history_size_fops); ++ debugfs_create_file("regs_history", S_IRUGO, ++ kbdev->mali_debugfs_directory, &kbdev->io_history, ++ ®s_history_fops); ++} ++ ++ ++#endif /* CONFIG_DEBUG_FS */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_regs_history_debugfs.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_regs_history_debugfs.h +new file mode 100755 +index 000000000000..fbb36b3f22e4 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_regs_history_debugfs.h +@@ -0,0 +1,50 @@ ++/* ++ * ++ * (C) COPYRIGHT 2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/** ++ * Header file for register access history support via debugfs ++ * ++ * This interface is made available via /sys/kernel/debug/mali#/regs_history*. ++ * ++ * Usage: ++ * - regs_history_enabled: whether recording of register accesses is enabled. ++ * Write 'y' to enable, 'n' to disable. ++ * - regs_history_size: size of the register history buffer, must be > 0 ++ * - regs_history: return the information about last accesses to the registers. ++ */ ++ ++#ifndef _KBASE_REGS_HISTORY_DEBUGFS_H ++#define _KBASE_REGS_HISTORY_DEBUGFS_H ++ ++struct kbase_device; ++ ++#if defined(CONFIG_DEBUG_FS) && !defined(CONFIG_MALI_BIFROST_NO_MALI) ++ ++/** ++ * kbasep_regs_history_debugfs_init - add debugfs entries for register history ++ * ++ * @kbdev: Pointer to kbase_device containing the register history ++ */ ++void kbasep_regs_history_debugfs_init(struct kbase_device *kbdev); ++ ++#else /* CONFIG_DEBUG_FS */ ++ ++#define kbasep_regs_history_debugfs_init CSTD_NOP ++ ++#endif /* CONFIG_DEBUG_FS */ ++ ++#endif /*_KBASE_REGS_HISTORY_DEBUGFS_H*/ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_replay.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_replay.c +new file mode 100755 +index 000000000000..9f4dc372770d +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_replay.c +@@ -0,0 +1,1166 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/** ++ * @file mali_kbase_replay.c ++ * Replay soft job handlers ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++ ++#define JOB_NOT_STARTED 0 ++#define JOB_TYPE_NULL (1) ++#define JOB_TYPE_VERTEX (5) ++#define JOB_TYPE_TILER (7) ++#define JOB_TYPE_FUSED (8) ++#define JOB_TYPE_FRAGMENT (9) ++ ++#define JOB_HEADER_32_FBD_OFFSET (31*4) ++#define JOB_HEADER_64_FBD_OFFSET (44*4) ++ ++#define FBD_POINTER_MASK (~0x3f) ++ ++#define SFBD_TILER_OFFSET (48*4) ++ ++#define MFBD_TILER_OFFSET (14*4) ++ ++#define FBD_HIERARCHY_WEIGHTS 8 ++#define FBD_HIERARCHY_MASK_MASK 0x1fff ++ ++#define FBD_TYPE 1 ++ ++#define HIERARCHY_WEIGHTS 13 ++ ++#define JOB_HEADER_ID_MAX 0xffff ++ ++#define JOB_SOURCE_ID(status) (((status) >> 16) & 0xFFFF) ++#define JOB_POLYGON_LIST (0x03) ++ ++struct fragment_job { ++ struct job_descriptor_header header; ++ ++ u32 x[2]; ++ union { ++ u64 _64; ++ u32 _32; ++ } fragment_fbd; ++}; ++ ++static void dump_job_head(struct kbase_context *kctx, char *head_str, ++ struct job_descriptor_header *job) ++{ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++ dev_dbg(kctx->kbdev->dev, "%s\n", head_str); ++ dev_dbg(kctx->kbdev->dev, ++ "addr = %p\n" ++ "exception_status = %x (Source ID: 0x%x Access: 0x%x Exception: 0x%x)\n" ++ "first_incomplete_task = %x\n" ++ "fault_pointer = %llx\n" ++ "job_descriptor_size = %x\n" ++ "job_type = %x\n" ++ "job_barrier = %x\n" ++ "_reserved_01 = %x\n" ++ "_reserved_02 = %x\n" ++ "_reserved_03 = %x\n" ++ "_reserved_04/05 = %x,%x\n" ++ "job_index = %x\n" ++ "dependencies = %x,%x\n", ++ job, job->exception_status, ++ JOB_SOURCE_ID(job->exception_status), ++ (job->exception_status >> 8) & 0x3, ++ job->exception_status & 0xFF, ++ job->first_incomplete_task, ++ job->fault_pointer, job->job_descriptor_size, ++ job->job_type, job->job_barrier, job->_reserved_01, ++ job->_reserved_02, job->_reserved_03, ++ job->_reserved_04, job->_reserved_05, ++ job->job_index, ++ job->job_dependency_index_1, ++ job->job_dependency_index_2); ++ ++ if (job->job_descriptor_size) ++ dev_dbg(kctx->kbdev->dev, "next = %llx\n", ++ job->next_job._64); ++ else ++ dev_dbg(kctx->kbdev->dev, "next = %x\n", ++ job->next_job._32); ++#endif ++} ++ ++static int kbasep_replay_reset_sfbd(struct kbase_context *kctx, ++ u64 fbd_address, u64 tiler_heap_free, ++ u16 hierarchy_mask, u32 default_weight) ++{ ++ struct { ++ u32 padding_1[1]; ++ u32 flags; ++ u64 padding_2[2]; ++ u64 heap_free_address; ++ u32 padding[8]; ++ u32 weights[FBD_HIERARCHY_WEIGHTS]; ++ } *fbd_tiler; ++ struct kbase_vmap_struct map; ++ ++ dev_dbg(kctx->kbdev->dev, "fbd_address: %llx\n", fbd_address); ++ ++ fbd_tiler = kbase_vmap(kctx, fbd_address + SFBD_TILER_OFFSET, ++ sizeof(*fbd_tiler), &map); ++ if (!fbd_tiler) { ++ dev_err(kctx->kbdev->dev, "kbasep_replay_reset_fbd: failed to map fbd\n"); ++ return -EINVAL; ++ } ++ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++ dev_dbg(kctx->kbdev->dev, ++ "FBD tiler:\n" ++ "flags = %x\n" ++ "heap_free_address = %llx\n", ++ fbd_tiler->flags, fbd_tiler->heap_free_address); ++#endif ++ if (hierarchy_mask) { ++ u32 weights[HIERARCHY_WEIGHTS]; ++ u16 old_hierarchy_mask = fbd_tiler->flags & ++ FBD_HIERARCHY_MASK_MASK; ++ int i, j = 0; ++ ++ for (i = 0; i < HIERARCHY_WEIGHTS; i++) { ++ if (old_hierarchy_mask & (1 << i)) { ++ KBASE_DEBUG_ASSERT(j < FBD_HIERARCHY_WEIGHTS); ++ weights[i] = fbd_tiler->weights[j++]; ++ } else { ++ weights[i] = default_weight; ++ } ++ } ++ ++ ++ dev_dbg(kctx->kbdev->dev, "Old hierarchy mask=%x New hierarchy mask=%x\n", ++ old_hierarchy_mask, hierarchy_mask); ++ ++ for (i = 0; i < HIERARCHY_WEIGHTS; i++) ++ dev_dbg(kctx->kbdev->dev, " Hierarchy weight %02d: %08x\n", ++ i, weights[i]); ++ ++ j = 0; ++ ++ for (i = 0; i < HIERARCHY_WEIGHTS; i++) { ++ if (hierarchy_mask & (1 << i)) { ++ KBASE_DEBUG_ASSERT(j < FBD_HIERARCHY_WEIGHTS); ++ ++ dev_dbg(kctx->kbdev->dev, " Writing hierarchy level %02d (%08x) to %d\n", ++ i, weights[i], j); ++ ++ fbd_tiler->weights[j++] = weights[i]; ++ } ++ } ++ ++ for (; j < FBD_HIERARCHY_WEIGHTS; j++) ++ fbd_tiler->weights[j] = 0; ++ ++ fbd_tiler->flags = hierarchy_mask | (1 << 16); ++ } ++ ++ fbd_tiler->heap_free_address = tiler_heap_free; ++ ++ dev_dbg(kctx->kbdev->dev, "heap_free_address=%llx flags=%x\n", ++ fbd_tiler->heap_free_address, fbd_tiler->flags); ++ ++ kbase_vunmap(kctx, &map); ++ ++ return 0; ++} ++ ++static int kbasep_replay_reset_mfbd(struct kbase_context *kctx, ++ u64 fbd_address, u64 tiler_heap_free, ++ u16 hierarchy_mask, u32 default_weight) ++{ ++ struct kbase_vmap_struct map; ++ struct { ++ u32 padding_0; ++ u32 flags; ++ u64 padding_1[2]; ++ u64 heap_free_address; ++ u64 padding_2; ++ u32 weights[FBD_HIERARCHY_WEIGHTS]; ++ } *fbd_tiler; ++ ++ dev_dbg(kctx->kbdev->dev, "fbd_address: %llx\n", fbd_address); ++ ++ fbd_tiler = kbase_vmap(kctx, fbd_address + MFBD_TILER_OFFSET, ++ sizeof(*fbd_tiler), &map); ++ if (!fbd_tiler) { ++ dev_err(kctx->kbdev->dev, ++ "kbasep_replay_reset_fbd: failed to map fbd\n"); ++ return -EINVAL; ++ } ++ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++ dev_dbg(kctx->kbdev->dev, "FBD tiler:\n" ++ "flags = %x\n" ++ "heap_free_address = %llx\n", ++ fbd_tiler->flags, ++ fbd_tiler->heap_free_address); ++#endif ++ if (hierarchy_mask) { ++ u32 weights[HIERARCHY_WEIGHTS]; ++ u16 old_hierarchy_mask = (fbd_tiler->flags) & ++ FBD_HIERARCHY_MASK_MASK; ++ int i, j = 0; ++ ++ for (i = 0; i < HIERARCHY_WEIGHTS; i++) { ++ if (old_hierarchy_mask & (1 << i)) { ++ KBASE_DEBUG_ASSERT(j < FBD_HIERARCHY_WEIGHTS); ++ weights[i] = fbd_tiler->weights[j++]; ++ } else { ++ weights[i] = default_weight; ++ } ++ } ++ ++ ++ dev_dbg(kctx->kbdev->dev, "Old hierarchy mask=%x New hierarchy mask=%x\n", ++ old_hierarchy_mask, hierarchy_mask); ++ ++ for (i = 0; i < HIERARCHY_WEIGHTS; i++) ++ dev_dbg(kctx->kbdev->dev, " Hierarchy weight %02d: %08x\n", ++ i, weights[i]); ++ ++ j = 0; ++ ++ for (i = 0; i < HIERARCHY_WEIGHTS; i++) { ++ if (hierarchy_mask & (1 << i)) { ++ KBASE_DEBUG_ASSERT(j < FBD_HIERARCHY_WEIGHTS); ++ ++ dev_dbg(kctx->kbdev->dev, ++ " Writing hierarchy level %02d (%08x) to %d\n", ++ i, weights[i], j); ++ ++ fbd_tiler->weights[j++] = weights[i]; ++ } ++ } ++ ++ for (; j < FBD_HIERARCHY_WEIGHTS; j++) ++ fbd_tiler->weights[j] = 0; ++ ++ fbd_tiler->flags = hierarchy_mask | (1 << 16); ++ } ++ ++ fbd_tiler->heap_free_address = tiler_heap_free; ++ ++ kbase_vunmap(kctx, &map); ++ ++ return 0; ++} ++ ++/** ++ * @brief Reset the status of an FBD pointed to by a tiler job ++ * ++ * This performs two functions : ++ * - Set the hierarchy mask ++ * - Reset the tiler free heap address ++ * ++ * @param[in] kctx Context pointer ++ * @param[in] job_header Address of job header to reset. ++ * @param[in] tiler_heap_free The value to reset Tiler Heap Free to ++ * @param[in] hierarchy_mask The hierarchy mask to use ++ * @param[in] default_weight Default hierarchy weight to write when no other ++ * weight is given in the FBD ++ * @param[in] job_64 true if this job is using 64-bit ++ * descriptors ++ * ++ * @return 0 on success, error code on failure ++ */ ++static int kbasep_replay_reset_tiler_job(struct kbase_context *kctx, ++ u64 job_header, u64 tiler_heap_free, ++ u16 hierarchy_mask, u32 default_weight, bool job_64) ++{ ++ struct kbase_vmap_struct map; ++ u64 fbd_address; ++ ++ if (job_64) { ++ u64 *job_ext; ++ ++ job_ext = kbase_vmap(kctx, ++ job_header + JOB_HEADER_64_FBD_OFFSET, ++ sizeof(*job_ext), &map); ++ ++ if (!job_ext) { ++ dev_err(kctx->kbdev->dev, "kbasep_replay_reset_tiler_job: failed to map jc\n"); ++ return -EINVAL; ++ } ++ ++ fbd_address = *job_ext; ++ ++ kbase_vunmap(kctx, &map); ++ } else { ++ u32 *job_ext; ++ ++ job_ext = kbase_vmap(kctx, ++ job_header + JOB_HEADER_32_FBD_OFFSET, ++ sizeof(*job_ext), &map); ++ ++ if (!job_ext) { ++ dev_err(kctx->kbdev->dev, "kbasep_replay_reset_tiler_job: failed to map jc\n"); ++ return -EINVAL; ++ } ++ ++ fbd_address = *job_ext; ++ ++ kbase_vunmap(kctx, &map); ++ } ++ ++ if (fbd_address & FBD_TYPE) { ++ return kbasep_replay_reset_mfbd(kctx, ++ fbd_address & FBD_POINTER_MASK, ++ tiler_heap_free, ++ hierarchy_mask, ++ default_weight); ++ } else { ++ return kbasep_replay_reset_sfbd(kctx, ++ fbd_address & FBD_POINTER_MASK, ++ tiler_heap_free, ++ hierarchy_mask, ++ default_weight); ++ } ++} ++ ++/** ++ * @brief Reset the status of a job ++ * ++ * This performs the following functions : ++ * ++ * - Reset the Job Status field of each job to NOT_STARTED. ++ * - Set the Job Type field of any Vertex Jobs to Null Job. ++ * - For any jobs using an FBD, set the Tiler Heap Free field to the value of ++ * the tiler_heap_free parameter, and set the hierarchy level mask to the ++ * hier_mask parameter. ++ * - Offset HW dependencies by the hw_job_id_offset parameter ++ * - Set the Perform Job Barrier flag if this job is the first in the chain ++ * - Read the address of the next job header ++ * ++ * @param[in] kctx Context pointer ++ * @param[in,out] job_header Address of job header to reset. Set to address ++ * of next job header on exit. ++ * @param[in] prev_jc Previous job chain to link to, if this job is ++ * the last in the chain. ++ * @param[in] hw_job_id_offset Offset for HW job IDs ++ * @param[in] tiler_heap_free The value to reset Tiler Heap Free to ++ * @param[in] hierarchy_mask The hierarchy mask to use ++ * @param[in] default_weight Default hierarchy weight to write when no other ++ * weight is given in the FBD ++ * @param[in] first_in_chain true if this job is the first in the chain ++ * @param[in] fragment_chain true if this job is in the fragment chain ++ * ++ * @return 0 on success, error code on failure ++ */ ++static int kbasep_replay_reset_job(struct kbase_context *kctx, ++ u64 *job_header, u64 prev_jc, ++ u64 tiler_heap_free, u16 hierarchy_mask, ++ u32 default_weight, u16 hw_job_id_offset, ++ bool first_in_chain, bool fragment_chain) ++{ ++ struct fragment_job *frag_job; ++ struct job_descriptor_header *job; ++ u64 new_job_header; ++ struct kbase_vmap_struct map; ++ ++ frag_job = kbase_vmap(kctx, *job_header, sizeof(*frag_job), &map); ++ if (!frag_job) { ++ dev_err(kctx->kbdev->dev, ++ "kbasep_replay_parse_jc: failed to map jc\n"); ++ return -EINVAL; ++ } ++ job = &frag_job->header; ++ ++ dump_job_head(kctx, "Job header:", job); ++ ++ if (job->exception_status == JOB_NOT_STARTED && !fragment_chain) { ++ dev_err(kctx->kbdev->dev, "Job already not started\n"); ++ goto out_unmap; ++ } ++ job->exception_status = JOB_NOT_STARTED; ++ ++ if (job->job_type == JOB_TYPE_VERTEX) ++ job->job_type = JOB_TYPE_NULL; ++ ++ if (job->job_type == JOB_TYPE_FUSED) { ++ dev_err(kctx->kbdev->dev, "Fused jobs can not be replayed\n"); ++ goto out_unmap; ++ } ++ ++ if (first_in_chain) ++ job->job_barrier = 1; ++ ++ if ((job->job_dependency_index_1 + hw_job_id_offset) > ++ JOB_HEADER_ID_MAX || ++ (job->job_dependency_index_2 + hw_job_id_offset) > ++ JOB_HEADER_ID_MAX || ++ (job->job_index + hw_job_id_offset) > JOB_HEADER_ID_MAX) { ++ dev_err(kctx->kbdev->dev, ++ "Job indicies/dependencies out of valid range\n"); ++ goto out_unmap; ++ } ++ ++ if (job->job_dependency_index_1) ++ job->job_dependency_index_1 += hw_job_id_offset; ++ if (job->job_dependency_index_2) ++ job->job_dependency_index_2 += hw_job_id_offset; ++ ++ job->job_index += hw_job_id_offset; ++ ++ if (job->job_descriptor_size) { ++ new_job_header = job->next_job._64; ++ if (!job->next_job._64) ++ job->next_job._64 = prev_jc; ++ } else { ++ new_job_header = job->next_job._32; ++ if (!job->next_job._32) ++ job->next_job._32 = prev_jc; ++ } ++ dump_job_head(kctx, "Updated to:", job); ++ ++ if (job->job_type == JOB_TYPE_TILER) { ++ bool job_64 = job->job_descriptor_size != 0; ++ ++ if (kbasep_replay_reset_tiler_job(kctx, *job_header, ++ tiler_heap_free, hierarchy_mask, ++ default_weight, job_64) != 0) ++ goto out_unmap; ++ ++ } else if (job->job_type == JOB_TYPE_FRAGMENT) { ++ u64 fbd_address; ++ ++ if (job->job_descriptor_size) ++ fbd_address = frag_job->fragment_fbd._64; ++ else ++ fbd_address = (u64)frag_job->fragment_fbd._32; ++ ++ if (fbd_address & FBD_TYPE) { ++ if (kbasep_replay_reset_mfbd(kctx, ++ fbd_address & FBD_POINTER_MASK, ++ tiler_heap_free, ++ hierarchy_mask, ++ default_weight) != 0) ++ goto out_unmap; ++ } else { ++ if (kbasep_replay_reset_sfbd(kctx, ++ fbd_address & FBD_POINTER_MASK, ++ tiler_heap_free, ++ hierarchy_mask, ++ default_weight) != 0) ++ goto out_unmap; ++ } ++ } ++ ++ kbase_vunmap(kctx, &map); ++ ++ *job_header = new_job_header; ++ ++ return 0; ++ ++out_unmap: ++ kbase_vunmap(kctx, &map); ++ return -EINVAL; ++} ++ ++/** ++ * @brief Find the highest job ID in a job chain ++ * ++ * @param[in] kctx Context pointer ++ * @param[in] jc Job chain start address ++ * @param[out] hw_job_id Highest job ID in chain ++ * ++ * @return 0 on success, error code on failure ++ */ ++static int kbasep_replay_find_hw_job_id(struct kbase_context *kctx, ++ u64 jc, u16 *hw_job_id) ++{ ++ while (jc) { ++ struct job_descriptor_header *job; ++ struct kbase_vmap_struct map; ++ ++ dev_dbg(kctx->kbdev->dev, ++ "kbasep_replay_find_hw_job_id: parsing jc=%llx\n", jc); ++ ++ job = kbase_vmap(kctx, jc, sizeof(*job), &map); ++ if (!job) { ++ dev_err(kctx->kbdev->dev, "failed to map jc\n"); ++ ++ return -EINVAL; ++ } ++ ++ if (job->job_index > *hw_job_id) ++ *hw_job_id = job->job_index; ++ ++ if (job->job_descriptor_size) ++ jc = job->next_job._64; ++ else ++ jc = job->next_job._32; ++ ++ kbase_vunmap(kctx, &map); ++ } ++ ++ return 0; ++} ++ ++/** ++ * @brief Reset the status of a number of jobs ++ * ++ * This function walks the provided job chain, and calls ++ * kbasep_replay_reset_job for each job. It also links the job chain to the ++ * provided previous job chain. ++ * ++ * The function will fail if any of the jobs passed already have status of ++ * NOT_STARTED. ++ * ++ * @param[in] kctx Context pointer ++ * @param[in] jc Job chain to be processed ++ * @param[in] prev_jc Job chain to be added to. May be NULL ++ * @param[in] tiler_heap_free The value to reset Tiler Heap Free to ++ * @param[in] hierarchy_mask The hierarchy mask to use ++ * @param[in] default_weight Default hierarchy weight to write when no other ++ * weight is given in the FBD ++ * @param[in] hw_job_id_offset Offset for HW job IDs ++ * @param[in] fragment_chain true if this chain is the fragment chain ++ * ++ * @return 0 on success, error code otherwise ++ */ ++static int kbasep_replay_parse_jc(struct kbase_context *kctx, ++ u64 jc, u64 prev_jc, ++ u64 tiler_heap_free, u16 hierarchy_mask, ++ u32 default_weight, u16 hw_job_id_offset, ++ bool fragment_chain) ++{ ++ bool first_in_chain = true; ++ int nr_jobs = 0; ++ ++ dev_dbg(kctx->kbdev->dev, "kbasep_replay_parse_jc: jc=%llx hw_job_id=%x\n", ++ jc, hw_job_id_offset); ++ ++ while (jc) { ++ dev_dbg(kctx->kbdev->dev, "kbasep_replay_parse_jc: parsing jc=%llx\n", jc); ++ ++ if (kbasep_replay_reset_job(kctx, &jc, prev_jc, ++ tiler_heap_free, hierarchy_mask, ++ default_weight, hw_job_id_offset, ++ first_in_chain, fragment_chain) != 0) ++ return -EINVAL; ++ ++ first_in_chain = false; ++ ++ nr_jobs++; ++ if (fragment_chain && ++ nr_jobs >= BASE_JD_REPLAY_F_CHAIN_JOB_LIMIT) { ++ dev_err(kctx->kbdev->dev, ++ "Exceeded maximum number of jobs in fragment chain\n"); ++ return -EINVAL; ++ } ++ } ++ ++ return 0; ++} ++ ++/** ++ * @brief Reset the status of a replay job, and set up dependencies ++ * ++ * This performs the actions to allow the replay job to be re-run following ++ * completion of the passed dependency. ++ * ++ * @param[in] katom The atom to be reset ++ * @param[in] dep_atom The dependency to be attached to the atom ++ */ ++static void kbasep_replay_reset_softjob(struct kbase_jd_atom *katom, ++ struct kbase_jd_atom *dep_atom) ++{ ++ katom->status = KBASE_JD_ATOM_STATE_QUEUED; ++ kbase_jd_katom_dep_set(&katom->dep[0], dep_atom, BASE_JD_DEP_TYPE_DATA); ++ list_add_tail(&katom->dep_item[0], &dep_atom->dep_head[0]); ++} ++ ++/** ++ * @brief Allocate an unused katom ++ * ++ * This will search the provided context for an unused katom, and will mark it ++ * as KBASE_JD_ATOM_STATE_QUEUED. ++ * ++ * If no atoms are available then the function will fail. ++ * ++ * @param[in] kctx Context pointer ++ * @return An atom ID, or -1 on failure ++ */ ++static int kbasep_allocate_katom(struct kbase_context *kctx) ++{ ++ struct kbase_jd_context *jctx = &kctx->jctx; ++ int i; ++ ++ for (i = BASE_JD_ATOM_COUNT-1; i > 0; i--) { ++ if (jctx->atoms[i].status == KBASE_JD_ATOM_STATE_UNUSED) { ++ jctx->atoms[i].status = KBASE_JD_ATOM_STATE_QUEUED; ++ dev_dbg(kctx->kbdev->dev, ++ "kbasep_allocate_katom: Allocated atom %d\n", ++ i); ++ return i; ++ } ++ } ++ ++ return -1; ++} ++ ++/** ++ * @brief Release a katom ++ * ++ * This will mark the provided atom as available, and remove any dependencies. ++ * ++ * For use on error path. ++ * ++ * @param[in] kctx Context pointer ++ * @param[in] atom_id ID of atom to release ++ */ ++static void kbasep_release_katom(struct kbase_context *kctx, int atom_id) ++{ ++ struct kbase_jd_context *jctx = &kctx->jctx; ++ ++ dev_dbg(kctx->kbdev->dev, "kbasep_release_katom: Released atom %d\n", ++ atom_id); ++ ++ while (!list_empty(&jctx->atoms[atom_id].dep_head[0])) ++ list_del(jctx->atoms[atom_id].dep_head[0].next); ++ ++ while (!list_empty(&jctx->atoms[atom_id].dep_head[1])) ++ list_del(jctx->atoms[atom_id].dep_head[1].next); ++ ++ jctx->atoms[atom_id].status = KBASE_JD_ATOM_STATE_UNUSED; ++} ++ ++static void kbasep_replay_create_atom(struct kbase_context *kctx, ++ struct base_jd_atom_v2 *atom, ++ int atom_nr, ++ base_jd_prio prio) ++{ ++ atom->nr_extres = 0; ++ atom->extres_list = 0; ++ atom->device_nr = 0; ++ atom->prio = prio; ++ atom->atom_number = atom_nr; ++ ++ base_jd_atom_dep_set(&atom->pre_dep[0], 0 , BASE_JD_DEP_TYPE_INVALID); ++ base_jd_atom_dep_set(&atom->pre_dep[1], 0 , BASE_JD_DEP_TYPE_INVALID); ++ ++ atom->udata.blob[0] = 0; ++ atom->udata.blob[1] = 0; ++} ++ ++/** ++ * @brief Create two atoms for the purpose of replaying jobs ++ * ++ * Two atoms are allocated and created. The jc pointer is not set at this ++ * stage. The second atom has a dependency on the first. The remaining fields ++ * are set up as follows : ++ * ++ * - No external resources. Any required external resources will be held by the ++ * replay atom. ++ * - device_nr is set to 0. This is not relevant as ++ * BASE_JD_REQ_SPECIFIC_COHERENT_GROUP should not be set. ++ * - Priority is inherited from the replay job. ++ * ++ * @param[out] t_atom Atom to use for tiler jobs ++ * @param[out] f_atom Atom to use for fragment jobs ++ * @param[in] prio Priority of new atom (inherited from replay soft ++ * job) ++ * @return 0 on success, error code on failure ++ */ ++static int kbasep_replay_create_atoms(struct kbase_context *kctx, ++ struct base_jd_atom_v2 *t_atom, ++ struct base_jd_atom_v2 *f_atom, ++ base_jd_prio prio) ++{ ++ int t_atom_nr, f_atom_nr; ++ ++ t_atom_nr = kbasep_allocate_katom(kctx); ++ if (t_atom_nr < 0) { ++ dev_err(kctx->kbdev->dev, "Failed to allocate katom\n"); ++ return -EINVAL; ++ } ++ ++ f_atom_nr = kbasep_allocate_katom(kctx); ++ if (f_atom_nr < 0) { ++ dev_err(kctx->kbdev->dev, "Failed to allocate katom\n"); ++ kbasep_release_katom(kctx, t_atom_nr); ++ return -EINVAL; ++ } ++ ++ kbasep_replay_create_atom(kctx, t_atom, t_atom_nr, prio); ++ kbasep_replay_create_atom(kctx, f_atom, f_atom_nr, prio); ++ ++ base_jd_atom_dep_set(&f_atom->pre_dep[0], t_atom_nr , BASE_JD_DEP_TYPE_DATA); ++ ++ return 0; ++} ++ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++static void payload_dump(struct kbase_context *kctx, base_jd_replay_payload *payload) ++{ ++ u64 next; ++ ++ dev_dbg(kctx->kbdev->dev, "Tiler jc list :\n"); ++ next = payload->tiler_jc_list; ++ ++ while (next) { ++ struct kbase_vmap_struct map; ++ base_jd_replay_jc *jc_struct; ++ ++ jc_struct = kbase_vmap(kctx, next, sizeof(*jc_struct), &map); ++ ++ if (!jc_struct) ++ return; ++ ++ dev_dbg(kctx->kbdev->dev, "* jc_struct=%p jc=%llx next=%llx\n", ++ jc_struct, jc_struct->jc, jc_struct->next); ++ ++ next = jc_struct->next; ++ ++ kbase_vunmap(kctx, &map); ++ } ++} ++#endif ++ ++/** ++ * @brief Parse a base_jd_replay_payload provided by userspace ++ * ++ * This will read the payload from userspace, and parse the job chains. ++ * ++ * @param[in] kctx Context pointer ++ * @param[in] replay_atom Replay soft job atom ++ * @param[in] t_atom Atom to use for tiler jobs ++ * @param[in] f_atom Atom to use for fragment jobs ++ * @return 0 on success, error code on failure ++ */ ++static int kbasep_replay_parse_payload(struct kbase_context *kctx, ++ struct kbase_jd_atom *replay_atom, ++ struct base_jd_atom_v2 *t_atom, ++ struct base_jd_atom_v2 *f_atom) ++{ ++ base_jd_replay_payload *payload = NULL; ++ u64 next; ++ u64 prev_jc = 0; ++ u16 hw_job_id_offset = 0; ++ int ret = -EINVAL; ++ struct kbase_vmap_struct map; ++ ++ dev_dbg(kctx->kbdev->dev, "kbasep_replay_parse_payload: replay_atom->jc = %llx sizeof(payload) = %zu\n", ++ replay_atom->jc, sizeof(payload)); ++ ++ payload = kbase_vmap(kctx, replay_atom->jc, sizeof(*payload), &map); ++ if (!payload) { ++ dev_err(kctx->kbdev->dev, "kbasep_replay_parse_payload: failed to map payload into kernel space\n"); ++ return -EINVAL; ++ } ++ ++#ifdef BASE_LEGACY_UK10_2_SUPPORT ++ if (KBASE_API_VERSION(10, 3) > replay_atom->kctx->api_version) { ++ base_jd_replay_payload_uk10_2 *payload_uk10_2; ++ u16 tiler_core_req; ++ u16 fragment_core_req; ++ ++ payload_uk10_2 = (base_jd_replay_payload_uk10_2 *) payload; ++ memcpy(&tiler_core_req, &payload_uk10_2->tiler_core_req, ++ sizeof(tiler_core_req)); ++ memcpy(&fragment_core_req, &payload_uk10_2->fragment_core_req, ++ sizeof(fragment_core_req)); ++ payload->tiler_core_req = (u32)(tiler_core_req & 0x7fff); ++ payload->fragment_core_req = (u32)(fragment_core_req & 0x7fff); ++ } ++#endif /* BASE_LEGACY_UK10_2_SUPPORT */ ++ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++ dev_dbg(kctx->kbdev->dev, "kbasep_replay_parse_payload: payload=%p\n", payload); ++ dev_dbg(kctx->kbdev->dev, "Payload structure:\n" ++ "tiler_jc_list = %llx\n" ++ "fragment_jc = %llx\n" ++ "tiler_heap_free = %llx\n" ++ "fragment_hierarchy_mask = %x\n" ++ "tiler_hierarchy_mask = %x\n" ++ "hierarchy_default_weight = %x\n" ++ "tiler_core_req = %x\n" ++ "fragment_core_req = %x\n", ++ payload->tiler_jc_list, ++ payload->fragment_jc, ++ payload->tiler_heap_free, ++ payload->fragment_hierarchy_mask, ++ payload->tiler_hierarchy_mask, ++ payload->hierarchy_default_weight, ++ payload->tiler_core_req, ++ payload->fragment_core_req); ++ payload_dump(kctx, payload); ++#endif ++ t_atom->core_req = payload->tiler_core_req | BASEP_JD_REQ_EVENT_NEVER; ++ f_atom->core_req = payload->fragment_core_req | BASEP_JD_REQ_EVENT_NEVER; ++ ++ /* Sanity check core requirements*/ ++ if ((t_atom->core_req & BASE_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_T || ++ (f_atom->core_req & BASE_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_FS || ++ t_atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES || ++ f_atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) { ++ ++ int t_atom_type = t_atom->core_req & BASE_JD_REQ_ATOM_TYPE & ~BASE_JD_REQ_COHERENT_GROUP; ++ int f_atom_type = f_atom->core_req & BASE_JD_REQ_ATOM_TYPE & ~BASE_JD_REQ_COHERENT_GROUP & ~BASE_JD_REQ_FS_AFBC; ++ int t_has_ex_res = t_atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES; ++ int f_has_ex_res = f_atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES; ++ ++ if (t_atom_type != BASE_JD_REQ_T) { ++ dev_err(kctx->kbdev->dev, "Invalid core requirement: Tiler atom not a tiler job. Was: 0x%x\n Expected: 0x%x", ++ t_atom_type, BASE_JD_REQ_T); ++ } ++ if (f_atom_type != BASE_JD_REQ_FS) { ++ dev_err(kctx->kbdev->dev, "Invalid core requirement: Fragment shader atom not a fragment shader. Was 0x%x Expected: 0x%x\n", ++ f_atom_type, BASE_JD_REQ_FS); ++ } ++ if (t_has_ex_res) { ++ dev_err(kctx->kbdev->dev, "Invalid core requirement: Tiler atom has external resources.\n"); ++ } ++ if (f_has_ex_res) { ++ dev_err(kctx->kbdev->dev, "Invalid core requirement: Fragment shader atom has external resources.\n"); ++ } ++ ++ goto out; ++ } ++ ++ /* Process tiler job chains */ ++ next = payload->tiler_jc_list; ++ if (!next) { ++ dev_err(kctx->kbdev->dev, "Invalid tiler JC list\n"); ++ goto out; ++ } ++ ++ while (next) { ++ base_jd_replay_jc *jc_struct; ++ struct kbase_vmap_struct jc_map; ++ u64 jc; ++ ++ jc_struct = kbase_vmap(kctx, next, sizeof(*jc_struct), &jc_map); ++ ++ if (!jc_struct) { ++ dev_err(kctx->kbdev->dev, "Failed to map jc struct\n"); ++ goto out; ++ } ++ ++ jc = jc_struct->jc; ++ next = jc_struct->next; ++ if (next) ++ jc_struct->jc = 0; ++ ++ kbase_vunmap(kctx, &jc_map); ++ ++ if (jc) { ++ u16 max_hw_job_id = 0; ++ ++ if (kbasep_replay_find_hw_job_id(kctx, jc, ++ &max_hw_job_id) != 0) ++ goto out; ++ ++ if (kbasep_replay_parse_jc(kctx, jc, prev_jc, ++ payload->tiler_heap_free, ++ payload->tiler_hierarchy_mask, ++ payload->hierarchy_default_weight, ++ hw_job_id_offset, false) != 0) { ++ goto out; ++ } ++ ++ hw_job_id_offset += max_hw_job_id; ++ ++ prev_jc = jc; ++ } ++ } ++ t_atom->jc = prev_jc; ++ ++ /* Process fragment job chain */ ++ f_atom->jc = payload->fragment_jc; ++ if (kbasep_replay_parse_jc(kctx, payload->fragment_jc, 0, ++ payload->tiler_heap_free, ++ payload->fragment_hierarchy_mask, ++ payload->hierarchy_default_weight, 0, ++ true) != 0) { ++ goto out; ++ } ++ ++ if (!t_atom->jc || !f_atom->jc) { ++ dev_err(kctx->kbdev->dev, "Invalid payload\n"); ++ goto out; ++ } ++ ++ dev_dbg(kctx->kbdev->dev, "t_atom->jc=%llx f_atom->jc=%llx\n", ++ t_atom->jc, f_atom->jc); ++ ret = 0; ++ ++out: ++ kbase_vunmap(kctx, &map); ++ ++ return ret; ++} ++ ++static void kbase_replay_process_worker(struct work_struct *data) ++{ ++ struct kbase_jd_atom *katom; ++ struct kbase_context *kctx; ++ struct kbase_jd_context *jctx; ++ bool need_to_try_schedule_context = false; ++ ++ struct base_jd_atom_v2 t_atom, f_atom; ++ struct kbase_jd_atom *t_katom, *f_katom; ++ base_jd_prio atom_prio; ++ ++ katom = container_of(data, struct kbase_jd_atom, work); ++ kctx = katom->kctx; ++ jctx = &kctx->jctx; ++ ++ mutex_lock(&jctx->lock); ++ ++ atom_prio = kbasep_js_sched_prio_to_atom_prio(katom->sched_priority); ++ ++ if (kbasep_replay_create_atoms( ++ kctx, &t_atom, &f_atom, atom_prio) != 0) { ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ goto out; ++ } ++ ++ t_katom = &jctx->atoms[t_atom.atom_number]; ++ f_katom = &jctx->atoms[f_atom.atom_number]; ++ ++ if (kbasep_replay_parse_payload(kctx, katom, &t_atom, &f_atom) != 0) { ++ kbasep_release_katom(kctx, t_atom.atom_number); ++ kbasep_release_katom(kctx, f_atom.atom_number); ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ goto out; ++ } ++ ++ kbasep_replay_reset_softjob(katom, f_katom); ++ ++ need_to_try_schedule_context |= jd_submit_atom(kctx, &t_atom, t_katom); ++ if (t_katom->event_code == BASE_JD_EVENT_JOB_INVALID) { ++ dev_err(kctx->kbdev->dev, "Replay failed to submit atom\n"); ++ kbasep_release_katom(kctx, f_atom.atom_number); ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ goto out; ++ } ++ need_to_try_schedule_context |= jd_submit_atom(kctx, &f_atom, f_katom); ++ if (f_katom->event_code == BASE_JD_EVENT_JOB_INVALID) { ++ dev_err(kctx->kbdev->dev, "Replay failed to submit atom\n"); ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ goto out; ++ } ++ ++ katom->event_code = BASE_JD_EVENT_DONE; ++ ++out: ++ if (katom->event_code != BASE_JD_EVENT_DONE) { ++ kbase_disjoint_state_down(kctx->kbdev); ++ ++ need_to_try_schedule_context |= jd_done_nolock(katom, NULL); ++ } ++ ++ if (need_to_try_schedule_context) ++ kbase_js_sched_all(kctx->kbdev); ++ ++ mutex_unlock(&jctx->lock); ++} ++ ++/** ++ * @brief Check job replay fault ++ * ++ * This will read the job payload, checks fault type and source, then decides ++ * whether replay is required. ++ * ++ * @param[in] katom The atom to be processed ++ * @return true (success) if replay required or false on failure. ++ */ ++static bool kbase_replay_fault_check(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ struct device *dev = kctx->kbdev->dev; ++ base_jd_replay_payload *payload; ++ u64 job_header; ++ u64 job_loop_detect; ++ struct job_descriptor_header *job; ++ struct kbase_vmap_struct job_map; ++ struct kbase_vmap_struct map; ++ bool err = false; ++ ++ /* Replay job if fault is of type BASE_JD_EVENT_JOB_WRITE_FAULT or ++ * if force_replay is enabled. ++ */ ++ if (BASE_JD_EVENT_TERMINATED == katom->event_code) { ++ return false; ++ } else if (BASE_JD_EVENT_JOB_WRITE_FAULT == katom->event_code) { ++ return true; ++ } else if (BASE_JD_EVENT_FORCE_REPLAY == katom->event_code) { ++ katom->event_code = BASE_JD_EVENT_DATA_INVALID_FAULT; ++ return true; ++ } else if (BASE_JD_EVENT_DATA_INVALID_FAULT != katom->event_code) { ++ /* No replay for faults of type other than ++ * BASE_JD_EVENT_DATA_INVALID_FAULT. ++ */ ++ return false; ++ } ++ ++ /* Job fault is BASE_JD_EVENT_DATA_INVALID_FAULT, now scan fragment jc ++ * to find out whether the source of exception is POLYGON_LIST. Replay ++ * is required if the source of fault is POLYGON_LIST. ++ */ ++ payload = kbase_vmap(kctx, katom->jc, sizeof(*payload), &map); ++ if (!payload) { ++ dev_err(dev, "kbase_replay_fault_check: failed to map payload.\n"); ++ return false; ++ } ++ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++ dev_dbg(dev, "kbase_replay_fault_check: payload=%p\n", payload); ++ dev_dbg(dev, "\nPayload structure:\n" ++ "fragment_jc = 0x%llx\n" ++ "fragment_hierarchy_mask = 0x%x\n" ++ "fragment_core_req = 0x%x\n", ++ payload->fragment_jc, ++ payload->fragment_hierarchy_mask, ++ payload->fragment_core_req); ++#endif ++ /* Process fragment job chain */ ++ job_header = (u64) payload->fragment_jc; ++ job_loop_detect = job_header; ++ while (job_header) { ++ job = kbase_vmap(kctx, job_header, sizeof(*job), &job_map); ++ if (!job) { ++ dev_err(dev, "failed to map jc\n"); ++ /* unmap payload*/ ++ kbase_vunmap(kctx, &map); ++ return false; ++ } ++ ++ ++ dump_job_head(kctx, "\njob_head structure:\n", job); ++ ++ /* Replay only when the polygon list reader caused the ++ * DATA_INVALID_FAULT */ ++ if ((BASE_JD_EVENT_DATA_INVALID_FAULT == katom->event_code) && ++ (JOB_POLYGON_LIST == JOB_SOURCE_ID(job->exception_status))) { ++ err = true; ++ kbase_vunmap(kctx, &job_map); ++ break; ++ } ++ ++ /* Move on to next fragment job in the list */ ++ if (job->job_descriptor_size) ++ job_header = job->next_job._64; ++ else ++ job_header = job->next_job._32; ++ ++ kbase_vunmap(kctx, &job_map); ++ ++ /* Job chain loop detected */ ++ if (job_header == job_loop_detect) ++ break; ++ } ++ ++ /* unmap payload*/ ++ kbase_vunmap(kctx, &map); ++ ++ return err; ++} ++ ++ ++/** ++ * @brief Process a replay job ++ * ++ * Called from kbase_process_soft_job. ++ * ++ * On exit, if the job has completed, katom->event_code will have been updated. ++ * If the job has not completed, and is replaying jobs, then the atom status ++ * will have been reset to KBASE_JD_ATOM_STATE_QUEUED. ++ * ++ * @param[in] katom The atom to be processed ++ * @return false if the atom has completed ++ * true if the atom is replaying jobs ++ */ ++bool kbase_replay_process(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ struct kbase_device *kbdev = kctx->kbdev; ++ ++ /* Don't replay this atom if these issues are not present in the ++ * hardware */ ++ if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_11020) && ++ !kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_11024)) { ++ dev_dbg(kbdev->dev, "Hardware does not need replay workaround"); ++ ++ /* Signal failure to userspace */ ++ katom->event_code = BASE_JD_EVENT_JOB_INVALID; ++ ++ return false; ++ } ++ ++ if (katom->event_code == BASE_JD_EVENT_DONE) { ++ dev_dbg(kbdev->dev, "Previous job succeeded - not replaying\n"); ++ ++ if (katom->retry_count) ++ kbase_disjoint_state_down(kbdev); ++ ++ return false; ++ } ++ ++ if (kbase_ctx_flag(kctx, KCTX_DYING)) { ++ dev_dbg(kbdev->dev, "Not replaying; context is dying\n"); ++ ++ if (katom->retry_count) ++ kbase_disjoint_state_down(kbdev); ++ ++ return false; ++ } ++ ++ /* Check job exception type and source before replaying. */ ++ if (!kbase_replay_fault_check(katom)) { ++ dev_dbg(kbdev->dev, ++ "Replay cancelled on event %x\n", katom->event_code); ++ /* katom->event_code is already set to the failure code of the ++ * previous job. ++ */ ++ return false; ++ } ++ ++ dev_warn(kbdev->dev, "Replaying jobs retry=%d\n", ++ katom->retry_count); ++ ++ katom->retry_count++; ++ ++ if (katom->retry_count > BASEP_JD_REPLAY_LIMIT) { ++ dev_err(kbdev->dev, "Replay exceeded limit - failing jobs\n"); ++ ++ kbase_disjoint_state_down(kbdev); ++ ++ /* katom->event_code is already set to the failure code of the ++ previous job */ ++ return false; ++ } ++ ++ /* only enter the disjoint state once for the whole time while the replay is ongoing */ ++ if (katom->retry_count == 1) ++ kbase_disjoint_state_up(kbdev); ++ ++ INIT_WORK(&katom->work, kbase_replay_process_worker); ++ queue_work(kctx->event_workq, &katom->work); ++ ++ return true; ++} +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_smc.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_smc.c +new file mode 100755 +index 000000000000..43175c85988f +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_smc.c +@@ -0,0 +1,74 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifdef CONFIG_ARM64 ++ ++#include ++#include ++ ++#include ++ ++static noinline u64 invoke_smc_fid(u64 function_id, ++ u64 arg0, u64 arg1, u64 arg2) ++{ ++ register u64 x0 asm("x0") = function_id; ++ register u64 x1 asm("x1") = arg0; ++ register u64 x2 asm("x2") = arg1; ++ register u64 x3 asm("x3") = arg2; ++ ++ asm volatile( ++ __asmeq("%0", "x0") ++ __asmeq("%1", "x1") ++ __asmeq("%2", "x2") ++ __asmeq("%3", "x3") ++ "smc #0\n" ++ : "+r" (x0) ++ : "r" (x1), "r" (x2), "r" (x3)); ++ ++ return x0; ++} ++ ++u64 kbase_invoke_smc_fid(u32 fid, u64 arg0, u64 arg1, u64 arg2) ++{ ++ /* Is fast call (bit 31 set) */ ++ KBASE_DEBUG_ASSERT(fid & ~SMC_FAST_CALL); ++ /* bits 16-23 must be zero for fast calls */ ++ KBASE_DEBUG_ASSERT((fid & (0xFF << 16)) == 0); ++ ++ return invoke_smc_fid(fid, arg0, arg1, arg2); ++} ++ ++u64 kbase_invoke_smc(u32 oen, u16 function_number, bool smc64, ++ u64 arg0, u64 arg1, u64 arg2) ++{ ++ u32 fid = 0; ++ ++ /* Only the six bits allowed should be used. */ ++ KBASE_DEBUG_ASSERT((oen & ~SMC_OEN_MASK) == 0); ++ ++ fid |= SMC_FAST_CALL; /* Bit 31: Fast call */ ++ if (smc64) ++ fid |= SMC_64; /* Bit 30: 1=SMC64, 0=SMC32 */ ++ fid |= oen; /* Bit 29:24: OEN */ ++ /* Bit 23:16: Must be zero for fast calls */ ++ fid |= (function_number); /* Bit 15:0: function number */ ++ ++ return kbase_invoke_smc_fid(fid, arg0, arg1, arg2); ++} ++ ++#endif /* CONFIG_ARM64 */ ++ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_smc.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_smc.h +new file mode 100755 +index 000000000000..9bff3d2e8b4d +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_smc.h +@@ -0,0 +1,67 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#ifndef _KBASE_SMC_H_ ++#define _KBASE_SMC_H_ ++ ++#ifdef CONFIG_ARM64 ++ ++#include ++ ++#define SMC_FAST_CALL (1 << 31) ++#define SMC_64 (1 << 30) ++ ++#define SMC_OEN_OFFSET 24 ++#define SMC_OEN_MASK (0x3F << SMC_OEN_OFFSET) /* 6 bits */ ++#define SMC_OEN_SIP (2 << SMC_OEN_OFFSET) ++#define SMC_OEN_STD (4 << SMC_OEN_OFFSET) ++ ++ ++/** ++ * kbase_invoke_smc_fid - Perform a secure monitor call ++ * @fid: The SMC function to call, see SMC Calling convention. ++ * @arg0: First argument to the SMC. ++ * @arg1: Second argument to the SMC. ++ * @arg2: Third argument to the SMC. ++ * ++ * See SMC Calling Convention for details. ++ * ++ * Return: the return value from the SMC. ++ */ ++u64 kbase_invoke_smc_fid(u32 fid, u64 arg0, u64 arg1, u64 arg2); ++ ++/** ++ * kbase_invoke_smc_fid - Perform a secure monitor call ++ * @oen: Owning Entity number (SIP, STD etc). ++ * @function_number: The function number within the OEN. ++ * @smc64: use SMC64 calling convention instead of SMC32. ++ * @arg0: First argument to the SMC. ++ * @arg1: Second argument to the SMC. ++ * @arg2: Third argument to the SMC. ++ * ++ * See SMC Calling Convention for details. ++ * ++ * Return: the return value from the SMC call. ++ */ ++u64 kbase_invoke_smc(u32 oen, u16 function_number, bool smc64, ++ u64 arg0, u64 arg1, u64 arg2); ++ ++#endif /* CONFIG_ARM64 */ ++ ++#endif /* _KBASE_SMC_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_softjobs.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_softjobs.c +new file mode 100755 +index 000000000000..c24b94e0d6eb +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_softjobs.c +@@ -0,0 +1,1513 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#include ++ ++#if defined(CONFIG_DMA_SHARED_BUFFER) ++#include ++#include ++#endif /* defined(CONFIG_DMA_SHARED_BUFFER) */ ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++#include ++#endif ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++/** ++ * @file mali_kbase_softjobs.c ++ * ++ * This file implements the logic behind software only jobs that are ++ * executed within the driver rather than being handed over to the GPU. ++ */ ++ ++static void kbasep_add_waiting_soft_job(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ unsigned long lflags; ++ ++ spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); ++ list_add_tail(&katom->queue, &kctx->waiting_soft_jobs); ++ spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); ++} ++ ++void kbasep_remove_waiting_soft_job(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ unsigned long lflags; ++ ++ spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); ++ list_del(&katom->queue); ++ spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); ++} ++ ++static void kbasep_add_waiting_with_timeout(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ ++ /* Record the start time of this atom so we could cancel it at ++ * the right time. ++ */ ++ katom->start_timestamp = ktime_get(); ++ ++ /* Add the atom to the waiting list before the timer is ++ * (re)started to make sure that it gets processed. ++ */ ++ kbasep_add_waiting_soft_job(katom); ++ ++ /* Schedule timeout of this atom after a period if it is not active */ ++ if (!timer_pending(&kctx->soft_job_timeout)) { ++ int timeout_ms = atomic_read( ++ &kctx->kbdev->js_data.soft_job_timeout_ms); ++ mod_timer(&kctx->soft_job_timeout, ++ jiffies + msecs_to_jiffies(timeout_ms)); ++ } ++} ++ ++static int kbasep_read_soft_event_status( ++ struct kbase_context *kctx, u64 evt, unsigned char *status) ++{ ++ unsigned char *mapped_evt; ++ struct kbase_vmap_struct map; ++ ++ mapped_evt = kbase_vmap(kctx, evt, sizeof(*mapped_evt), &map); ++ if (!mapped_evt) ++ return -EFAULT; ++ ++ *status = *mapped_evt; ++ ++ kbase_vunmap(kctx, &map); ++ ++ return 0; ++} ++ ++static int kbasep_write_soft_event_status( ++ struct kbase_context *kctx, u64 evt, unsigned char new_status) ++{ ++ unsigned char *mapped_evt; ++ struct kbase_vmap_struct map; ++ ++ if ((new_status != BASE_JD_SOFT_EVENT_SET) && ++ (new_status != BASE_JD_SOFT_EVENT_RESET)) ++ return -EINVAL; ++ ++ mapped_evt = kbase_vmap(kctx, evt, sizeof(*mapped_evt), &map); ++ if (!mapped_evt) ++ return -EFAULT; ++ ++ *mapped_evt = new_status; ++ ++ kbase_vunmap(kctx, &map); ++ ++ return 0; ++} ++ ++static int kbase_dump_cpu_gpu_time(struct kbase_jd_atom *katom) ++{ ++ struct kbase_vmap_struct map; ++ void *user_result; ++ struct timespec ts; ++ struct base_dump_cpu_gpu_counters data; ++ u64 system_time; ++ u64 cycle_counter; ++ u64 jc = katom->jc; ++ struct kbase_context *kctx = katom->kctx; ++ int pm_active_err; ++ ++ memset(&data, 0, sizeof(data)); ++ ++ /* Take the PM active reference as late as possible - otherwise, it could ++ * delay suspend until we process the atom (which may be at the end of a ++ * long chain of dependencies */ ++ pm_active_err = kbase_pm_context_active_handle_suspend(kctx->kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE); ++ if (pm_active_err) { ++ struct kbasep_js_device_data *js_devdata = &kctx->kbdev->js_data; ++ ++ /* We're suspended - queue this on the list of suspended jobs ++ * Use dep_item[1], because dep_item[0] was previously in use ++ * for 'waiting_soft_jobs'. ++ */ ++ mutex_lock(&js_devdata->runpool_mutex); ++ list_add_tail(&katom->dep_item[1], &js_devdata->suspended_soft_jobs_list); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ ++ /* Also adding this to the list of waiting soft job */ ++ kbasep_add_waiting_soft_job(katom); ++ ++ return pm_active_err; ++ } ++ ++ kbase_backend_get_gpu_time(kctx->kbdev, &cycle_counter, &system_time, ++ &ts); ++ ++ kbase_pm_context_idle(kctx->kbdev); ++ ++ data.sec = ts.tv_sec; ++ data.usec = ts.tv_nsec / 1000; ++ data.system_time = system_time; ++ data.cycle_counter = cycle_counter; ++ ++ /* Assume this atom will be cancelled until we know otherwise */ ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ ++ /* GPU_WR access is checked on the range for returning the result to ++ * userspace for the following reasons: ++ * - security, this is currently how imported user bufs are checked. ++ * - userspace ddk guaranteed to assume region was mapped as GPU_WR */ ++ user_result = kbase_vmap_prot(kctx, jc, sizeof(data), KBASE_REG_GPU_WR, &map); ++ if (!user_result) ++ return 0; ++ ++ memcpy(user_result, &data, sizeof(data)); ++ ++ kbase_vunmap(kctx, &map); ++ ++ /* Atom was fine - mark it as done */ ++ katom->event_code = BASE_JD_EVENT_DONE; ++ ++ return 0; ++} ++ ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++/* Called by the explicit fence mechanism when a fence wait has completed */ ++void kbase_soft_event_wait_callback(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ ++ mutex_lock(&kctx->jctx.lock); ++ kbasep_remove_waiting_soft_job(katom); ++ kbase_finish_soft_job(katom); ++ if (jd_done_nolock(katom, NULL)) ++ kbase_js_sched_all(kctx->kbdev); ++ mutex_unlock(&kctx->jctx.lock); ++} ++#endif ++ ++static void kbasep_soft_event_complete_job(struct work_struct *work) ++{ ++ struct kbase_jd_atom *katom = container_of(work, struct kbase_jd_atom, ++ work); ++ struct kbase_context *kctx = katom->kctx; ++ int resched; ++ ++ mutex_lock(&kctx->jctx.lock); ++ resched = jd_done_nolock(katom, NULL); ++ mutex_unlock(&kctx->jctx.lock); ++ ++ if (resched) ++ kbase_js_sched_all(kctx->kbdev); ++} ++ ++void kbasep_complete_triggered_soft_events(struct kbase_context *kctx, u64 evt) ++{ ++ int cancel_timer = 1; ++ struct list_head *entry, *tmp; ++ unsigned long lflags; ++ ++ spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); ++ list_for_each_safe(entry, tmp, &kctx->waiting_soft_jobs) { ++ struct kbase_jd_atom *katom = list_entry( ++ entry, struct kbase_jd_atom, queue); ++ ++ switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { ++ case BASE_JD_REQ_SOFT_EVENT_WAIT: ++ if (katom->jc == evt) { ++ list_del(&katom->queue); ++ ++ katom->event_code = BASE_JD_EVENT_DONE; ++ INIT_WORK(&katom->work, ++ kbasep_soft_event_complete_job); ++ queue_work(kctx->jctx.job_done_wq, ++ &katom->work); ++ } else { ++ /* There are still other waiting jobs, we cannot ++ * cancel the timer yet. ++ */ ++ cancel_timer = 0; ++ } ++ break; ++#ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG ++ case BASE_JD_REQ_SOFT_FENCE_WAIT: ++ /* Keep the timer running if fence debug is enabled and ++ * there are waiting fence jobs. ++ */ ++ cancel_timer = 0; ++ break; ++#endif ++ } ++ } ++ ++ if (cancel_timer) ++ del_timer(&kctx->soft_job_timeout); ++ spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); ++} ++ ++#ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG ++static void kbase_fence_debug_check_atom(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ struct device *dev = kctx->kbdev->dev; ++ int i; ++ ++ for (i = 0; i < 2; i++) { ++ struct kbase_jd_atom *dep; ++ ++ list_for_each_entry(dep, &katom->dep_head[i], dep_item[i]) { ++ if (dep->status == KBASE_JD_ATOM_STATE_UNUSED || ++ dep->status == KBASE_JD_ATOM_STATE_COMPLETED) ++ continue; ++ ++ if ((dep->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) ++ == BASE_JD_REQ_SOFT_FENCE_TRIGGER) { ++ /* Found blocked trigger fence. */ ++ struct kbase_sync_fence_info info; ++ ++ if (!kbase_sync_fence_in_info_get(dep, &info)) { ++ dev_warn(dev, ++ "\tVictim trigger atom %d fence [%p] %s: %s\n", ++ kbase_jd_atom_id(kctx, dep), ++ info.fence, ++ info.name, ++ kbase_sync_status_string(info.status)); ++ } ++ } ++ ++ kbase_fence_debug_check_atom(dep); ++ } ++ } ++} ++ ++static void kbase_fence_debug_wait_timeout(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ struct device *dev = katom->kctx->kbdev->dev; ++ int timeout_ms = atomic_read(&kctx->kbdev->js_data.soft_job_timeout_ms); ++ unsigned long lflags; ++ struct kbase_sync_fence_info info; ++ ++ spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); ++ ++ if (kbase_sync_fence_in_info_get(katom, &info)) { ++ /* Fence must have signaled just after timeout. */ ++ spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); ++ return; ++ } ++ ++ dev_warn(dev, "ctx %d_%d: Atom %d still waiting for fence [%p] after %dms\n", ++ kctx->tgid, kctx->id, ++ kbase_jd_atom_id(kctx, katom), ++ info.fence, timeout_ms); ++ dev_warn(dev, "\tGuilty fence [%p] %s: %s\n", ++ info.fence, info.name, ++ kbase_sync_status_string(info.status)); ++ ++ /* Search for blocked trigger atoms */ ++ kbase_fence_debug_check_atom(katom); ++ ++ spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); ++ ++ kbase_sync_fence_in_dump(katom); ++} ++ ++struct kbase_fence_debug_work { ++ struct kbase_jd_atom *katom; ++ struct work_struct work; ++}; ++ ++static void kbase_fence_debug_wait_timeout_worker(struct work_struct *work) ++{ ++ struct kbase_fence_debug_work *w = container_of(work, ++ struct kbase_fence_debug_work, work); ++ struct kbase_jd_atom *katom = w->katom; ++ struct kbase_context *kctx = katom->kctx; ++ ++ mutex_lock(&kctx->jctx.lock); ++ kbase_fence_debug_wait_timeout(katom); ++ mutex_unlock(&kctx->jctx.lock); ++ ++ kfree(w); ++} ++ ++static void kbase_fence_debug_timeout(struct kbase_jd_atom *katom) ++{ ++ struct kbase_fence_debug_work *work; ++ struct kbase_context *kctx = katom->kctx; ++ ++ /* Enqueue fence debug worker. Use job_done_wq to get ++ * debug print ordered with job completion. ++ */ ++ work = kzalloc(sizeof(struct kbase_fence_debug_work), GFP_ATOMIC); ++ /* Ignore allocation failure. */ ++ if (work) { ++ work->katom = katom; ++ INIT_WORK(&work->work, kbase_fence_debug_wait_timeout_worker); ++ queue_work(kctx->jctx.job_done_wq, &work->work); ++ } ++} ++#endif /* CONFIG_MALI_BIFROST_FENCE_DEBUG */ ++ ++void kbasep_soft_job_timeout_worker(struct timer_list *t) ++{ ++ struct kbase_context *kctx = from_timer(kctx, t, soft_job_timeout); ++ u32 timeout_ms = (u32)atomic_read( ++ &kctx->kbdev->js_data.soft_job_timeout_ms); ++ struct timer_list *timer = &kctx->soft_job_timeout; ++ ktime_t cur_time = ktime_get(); ++ bool restarting = false; ++ unsigned long lflags; ++ struct list_head *entry, *tmp; ++ ++ spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); ++ list_for_each_safe(entry, tmp, &kctx->waiting_soft_jobs) { ++ struct kbase_jd_atom *katom = list_entry(entry, ++ struct kbase_jd_atom, queue); ++ s64 elapsed_time = ktime_to_ms(ktime_sub(cur_time, ++ katom->start_timestamp)); ++ ++ if (elapsed_time < (s64)timeout_ms) { ++ restarting = true; ++ continue; ++ } ++ ++ switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { ++ case BASE_JD_REQ_SOFT_EVENT_WAIT: ++ /* Take it out of the list to ensure that it ++ * will be cancelled in all cases ++ */ ++ list_del(&katom->queue); ++ ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ INIT_WORK(&katom->work, kbasep_soft_event_complete_job); ++ queue_work(kctx->jctx.job_done_wq, &katom->work); ++ break; ++#ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG ++ case BASE_JD_REQ_SOFT_FENCE_WAIT: ++ kbase_fence_debug_timeout(katom); ++ break; ++#endif ++ } ++ } ++ ++ if (restarting) ++ mod_timer(timer, jiffies + msecs_to_jiffies(timeout_ms)); ++ spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); ++} ++ ++static int kbasep_soft_event_wait(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ unsigned char status; ++ ++ /* The status of this soft-job is stored in jc */ ++ if (kbasep_read_soft_event_status(kctx, katom->jc, &status)) { ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ return 0; ++ } ++ ++ if (status == BASE_JD_SOFT_EVENT_SET) ++ return 0; /* Event already set, nothing to do */ ++ ++ kbasep_add_waiting_with_timeout(katom); ++ ++ return 1; ++} ++ ++static void kbasep_soft_event_update_locked(struct kbase_jd_atom *katom, ++ unsigned char new_status) ++{ ++ /* Complete jobs waiting on the same event */ ++ struct kbase_context *kctx = katom->kctx; ++ ++ if (kbasep_write_soft_event_status(kctx, katom->jc, new_status) != 0) { ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ return; ++ } ++ ++ if (new_status == BASE_JD_SOFT_EVENT_SET) ++ kbasep_complete_triggered_soft_events(kctx, katom->jc); ++} ++ ++/** ++ * kbase_soft_event_update() - Update soft event state ++ * @kctx: Pointer to context ++ * @event: Event to update ++ * @new_status: New status value of event ++ * ++ * Update the event, and wake up any atoms waiting for the event. ++ * ++ * Return: 0 on success, a negative error code on failure. ++ */ ++int kbase_soft_event_update(struct kbase_context *kctx, ++ u64 event, ++ unsigned char new_status) ++{ ++ int err = 0; ++ ++ mutex_lock(&kctx->jctx.lock); ++ ++ if (kbasep_write_soft_event_status(kctx, event, new_status)) { ++ err = -ENOENT; ++ goto out; ++ } ++ ++ if (new_status == BASE_JD_SOFT_EVENT_SET) ++ kbasep_complete_triggered_soft_events(kctx, event); ++ ++out: ++ mutex_unlock(&kctx->jctx.lock); ++ ++ return err; ++} ++ ++static void kbasep_soft_event_cancel_job(struct kbase_jd_atom *katom) ++{ ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ if (jd_done_nolock(katom, NULL)) ++ kbase_js_sched_all(katom->kctx->kbdev); ++} ++ ++struct kbase_debug_copy_buffer { ++ size_t size; ++ struct page **pages; ++ int nr_pages; ++ size_t offset; ++ struct kbase_mem_phy_alloc *gpu_alloc; ++ ++ struct page **extres_pages; ++ int nr_extres_pages; ++}; ++ ++static inline void free_user_buffer(struct kbase_debug_copy_buffer *buffer) ++{ ++ struct page **pages = buffer->extres_pages; ++ int nr_pages = buffer->nr_extres_pages; ++ ++ if (pages) { ++ int i; ++ ++ for (i = 0; i < nr_pages; i++) { ++ struct page *pg = pages[i]; ++ ++ if (pg) ++ put_page(pg); ++ } ++ kfree(pages); ++ } ++} ++ ++static void kbase_debug_copy_finish(struct kbase_jd_atom *katom) ++{ ++ struct kbase_debug_copy_buffer *buffers = ++ (struct kbase_debug_copy_buffer *)(uintptr_t)katom->jc; ++ unsigned int i; ++ unsigned int nr = katom->nr_extres; ++ ++ if (!buffers) ++ return; ++ ++ kbase_gpu_vm_lock(katom->kctx); ++ for (i = 0; i < nr; i++) { ++ int p; ++ struct kbase_mem_phy_alloc *gpu_alloc = buffers[i].gpu_alloc; ++ ++ if (!buffers[i].pages) ++ break; ++ for (p = 0; p < buffers[i].nr_pages; p++) { ++ struct page *pg = buffers[i].pages[p]; ++ ++ if (pg) ++ put_page(pg); ++ } ++ kfree(buffers[i].pages); ++ if (gpu_alloc) { ++ switch (gpu_alloc->type) { ++ case KBASE_MEM_TYPE_IMPORTED_USER_BUF: ++ { ++ free_user_buffer(&buffers[i]); ++ break; ++ } ++ default: ++ /* Nothing to be done. */ ++ break; ++ } ++ kbase_mem_phy_alloc_put(gpu_alloc); ++ } ++ } ++ kbase_gpu_vm_unlock(katom->kctx); ++ kfree(buffers); ++ ++ katom->jc = 0; ++} ++ ++static int kbase_debug_copy_prepare(struct kbase_jd_atom *katom) ++{ ++ struct kbase_debug_copy_buffer *buffers; ++ struct base_jd_debug_copy_buffer *user_buffers = NULL; ++ unsigned int i; ++ unsigned int nr = katom->nr_extres; ++ int ret = 0; ++ void __user *user_structs = (void __user *)(uintptr_t)katom->jc; ++ ++ if (!user_structs) ++ return -EINVAL; ++ ++ buffers = kcalloc(nr, sizeof(*buffers), GFP_KERNEL); ++ if (!buffers) { ++ ret = -ENOMEM; ++ katom->jc = 0; ++ goto out_cleanup; ++ } ++ katom->jc = (u64)(uintptr_t)buffers; ++ ++ user_buffers = kmalloc_array(nr, sizeof(*user_buffers), GFP_KERNEL); ++ ++ if (!user_buffers) { ++ ret = -ENOMEM; ++ goto out_cleanup; ++ } ++ ++ ret = copy_from_user(user_buffers, user_structs, ++ sizeof(*user_buffers)*nr); ++ if (ret) { ++ ret = -EFAULT; ++ goto out_cleanup; ++ } ++ ++ for (i = 0; i < nr; i++) { ++ u64 addr = user_buffers[i].address; ++ u64 page_addr = addr & PAGE_MASK; ++ u64 end_page_addr = addr + user_buffers[i].size - 1; ++ u64 last_page_addr = end_page_addr & PAGE_MASK; ++ int nr_pages = (last_page_addr-page_addr)/PAGE_SIZE+1; ++ int pinned_pages; ++ struct kbase_va_region *reg; ++ struct base_external_resource user_extres; ++ ++ if (!addr) ++ continue; ++ ++ buffers[i].nr_pages = nr_pages; ++ buffers[i].offset = addr & ~PAGE_MASK; ++ if (buffers[i].offset >= PAGE_SIZE) { ++ ret = -EINVAL; ++ goto out_cleanup; ++ } ++ buffers[i].size = user_buffers[i].size; ++ ++ buffers[i].pages = kcalloc(nr_pages, sizeof(struct page *), ++ GFP_KERNEL); ++ if (!buffers[i].pages) { ++ ret = -ENOMEM; ++ goto out_cleanup; ++ } ++ ++ pinned_pages = get_user_pages_fast(page_addr, ++ nr_pages, ++ 1, /* Write */ ++ buffers[i].pages); ++ if (pinned_pages < 0) { ++ ret = pinned_pages; ++ goto out_cleanup; ++ } ++ if (pinned_pages != nr_pages) { ++ ret = -EINVAL; ++ goto out_cleanup; ++ } ++ ++ user_extres = user_buffers[i].extres; ++ if (user_extres.ext_resource == 0ULL) { ++ ret = -EINVAL; ++ goto out_cleanup; ++ } ++ ++ kbase_gpu_vm_lock(katom->kctx); ++ reg = kbase_region_tracker_find_region_enclosing_address( ++ katom->kctx, user_extres.ext_resource & ++ ~BASE_EXT_RES_ACCESS_EXCLUSIVE); ++ ++ if (NULL == reg || NULL == reg->gpu_alloc || ++ (reg->flags & KBASE_REG_FREE)) { ++ ret = -EINVAL; ++ goto out_unlock; ++ } ++ ++ buffers[i].gpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); ++ buffers[i].nr_extres_pages = reg->nr_pages; ++ ++ if (reg->nr_pages*PAGE_SIZE != buffers[i].size) ++ dev_warn(katom->kctx->kbdev->dev, "Copy buffer is not of same size as the external resource to copy.\n"); ++ ++ switch (reg->gpu_alloc->type) { ++ case KBASE_MEM_TYPE_IMPORTED_USER_BUF: ++ { ++ struct kbase_mem_phy_alloc *alloc = reg->gpu_alloc; ++ unsigned long nr_pages = ++ alloc->imported.user_buf.nr_pages; ++ ++ if (alloc->imported.user_buf.mm != current->mm) { ++ ret = -EINVAL; ++ goto out_unlock; ++ } ++ buffers[i].extres_pages = kcalloc(nr_pages, ++ sizeof(struct page *), GFP_KERNEL); ++ if (!buffers[i].extres_pages) { ++ ret = -ENOMEM; ++ goto out_unlock; ++ } ++ ++ ret = get_user_pages_fast( ++ alloc->imported.user_buf.address, ++ nr_pages, 0, ++ buffers[i].extres_pages); ++ if (ret != nr_pages) ++ goto out_unlock; ++ ret = 0; ++ break; ++ } ++ case KBASE_MEM_TYPE_IMPORTED_UMP: ++ { ++ dev_warn(katom->kctx->kbdev->dev, ++ "UMP is not supported for debug_copy jobs\n"); ++ ret = -EINVAL; ++ goto out_unlock; ++ } ++ default: ++ /* Nothing to be done. */ ++ break; ++ } ++ kbase_gpu_vm_unlock(katom->kctx); ++ } ++ kfree(user_buffers); ++ ++ return ret; ++ ++out_unlock: ++ kbase_gpu_vm_unlock(katom->kctx); ++ ++out_cleanup: ++ /* Frees allocated memory for kbase_debug_copy_job struct, including ++ * members, and sets jc to 0 */ ++ kbase_debug_copy_finish(katom); ++ kfree(user_buffers); ++ ++ return ret; ++} ++ ++static void kbase_mem_copy_from_extres_page(struct kbase_context *kctx, ++ void *extres_page, struct page **pages, unsigned int nr_pages, ++ unsigned int *target_page_nr, size_t offset, size_t *to_copy) ++{ ++ void *target_page = kmap(pages[*target_page_nr]); ++ size_t chunk = PAGE_SIZE-offset; ++ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ if (!target_page) { ++ *target_page_nr += 1; ++ dev_warn(kctx->kbdev->dev, "kmap failed in debug_copy job."); ++ return; ++ } ++ ++ chunk = min(chunk, *to_copy); ++ ++ memcpy(target_page + offset, extres_page, chunk); ++ *to_copy -= chunk; ++ ++ kunmap(pages[*target_page_nr]); ++ ++ *target_page_nr += 1; ++ if (*target_page_nr >= nr_pages) ++ return; ++ ++ target_page = kmap(pages[*target_page_nr]); ++ if (!target_page) { ++ *target_page_nr += 1; ++ dev_warn(kctx->kbdev->dev, "kmap failed in debug_copy job."); ++ return; ++ } ++ ++ KBASE_DEBUG_ASSERT(target_page); ++ ++ chunk = min(offset, *to_copy); ++ memcpy(target_page, extres_page + PAGE_SIZE-offset, chunk); ++ *to_copy -= chunk; ++ ++ kunmap(pages[*target_page_nr]); ++} ++ ++static int kbase_mem_copy_from_extres(struct kbase_context *kctx, ++ struct kbase_debug_copy_buffer *buf_data) ++{ ++ unsigned int i; ++ unsigned int target_page_nr = 0; ++ struct page **pages = buf_data->pages; ++ u64 offset = buf_data->offset; ++ size_t extres_size = buf_data->nr_extres_pages*PAGE_SIZE; ++ size_t to_copy = min(extres_size, buf_data->size); ++ size_t dma_to_copy; ++ struct kbase_mem_phy_alloc *gpu_alloc = buf_data->gpu_alloc; ++ int ret = 0; ++ ++ KBASE_DEBUG_ASSERT(pages != NULL); ++ ++ kbase_gpu_vm_lock(kctx); ++ if (!gpu_alloc) { ++ ret = -EINVAL; ++ goto out_unlock; ++ } ++ ++ switch (gpu_alloc->type) { ++ case KBASE_MEM_TYPE_IMPORTED_USER_BUF: ++ { ++ for (i = 0; i < buf_data->nr_extres_pages; i++) { ++ struct page *pg = buf_data->extres_pages[i]; ++ void *extres_page = kmap(pg); ++ ++ if (extres_page) ++ kbase_mem_copy_from_extres_page(kctx, ++ extres_page, pages, ++ buf_data->nr_pages, ++ &target_page_nr, ++ offset, &to_copy); ++ ++ kunmap(pg); ++ if (target_page_nr >= buf_data->nr_pages) ++ break; ++ } ++ break; ++ } ++ break; ++#ifdef CONFIG_DMA_SHARED_BUFFER ++ case KBASE_MEM_TYPE_IMPORTED_UMM: { ++ struct dma_buf *dma_buf = gpu_alloc->imported.umm.dma_buf; ++ ++ KBASE_DEBUG_ASSERT(dma_buf != NULL); ++ if (dma_buf->size > buf_data->nr_extres_pages * PAGE_SIZE) ++ dev_warn(kctx->kbdev->dev, "External resources buffer size mismatch"); ++ ++ dma_to_copy = min(dma_buf->size, ++ (size_t)(buf_data->nr_extres_pages * PAGE_SIZE)); ++ ret = dma_buf_begin_cpu_access(dma_buf, ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) && !defined(CONFIG_CHROMEOS) ++ 0, dma_to_copy, ++#endif ++ DMA_FROM_DEVICE); ++ if (ret) ++ goto out_unlock; ++ ++ for (i = 0; i < dma_to_copy/PAGE_SIZE; i++) { ++ ++ void *extres_page = dma_buf_kmap(dma_buf, i); ++ ++ if (extres_page) ++ kbase_mem_copy_from_extres_page(kctx, ++ extres_page, pages, ++ buf_data->nr_pages, ++ &target_page_nr, ++ offset, &to_copy); ++ ++ dma_buf_kunmap(dma_buf, i, extres_page); ++ if (target_page_nr >= buf_data->nr_pages) ++ break; ++ } ++ dma_buf_end_cpu_access(dma_buf, ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) && !defined(CONFIG_CHROMEOS) ++ 0, dma_to_copy, ++#endif ++ DMA_FROM_DEVICE); ++ break; ++ } ++#endif ++ default: ++ ret = -EINVAL; ++ } ++out_unlock: ++ kbase_gpu_vm_unlock(kctx); ++ return ret; ++ ++} ++ ++static int kbase_debug_copy(struct kbase_jd_atom *katom) ++{ ++ struct kbase_debug_copy_buffer *buffers = ++ (struct kbase_debug_copy_buffer *)(uintptr_t)katom->jc; ++ unsigned int i; ++ ++ for (i = 0; i < katom->nr_extres; i++) { ++ int res = kbase_mem_copy_from_extres(katom->kctx, &buffers[i]); ++ ++ if (res) ++ return res; ++ } ++ ++ return 0; ++} ++ ++static int kbase_jit_allocate_prepare(struct kbase_jd_atom *katom) ++{ ++ __user void *data = (__user void *)(uintptr_t) katom->jc; ++ struct base_jit_alloc_info *info; ++ struct kbase_context *kctx = katom->kctx; ++ int ret; ++ ++ /* Fail the job if there is no info structure */ ++ if (!data) { ++ ret = -EINVAL; ++ goto fail; ++ } ++ ++ /* Copy the information for safe access and future storage */ ++ info = kzalloc(sizeof(*info), GFP_KERNEL); ++ if (!info) { ++ ret = -ENOMEM; ++ goto fail; ++ } ++ ++ if (copy_from_user(info, data, sizeof(*info)) != 0) { ++ ret = -EINVAL; ++ goto free_info; ++ } ++ ++ /* If the ID is zero then fail the job */ ++ if (info->id == 0) { ++ ret = -EINVAL; ++ goto free_info; ++ } ++ ++ /* Sanity check that the PA fits within the VA */ ++ if (info->va_pages < info->commit_pages) { ++ ret = -EINVAL; ++ goto free_info; ++ } ++ ++ /* Ensure the GPU address is correctly aligned */ ++ if ((info->gpu_alloc_addr & 0x7) != 0) { ++ ret = -EINVAL; ++ goto free_info; ++ } ++ ++ /* Replace the user pointer with our kernel allocated info structure */ ++ katom->jc = (u64)(uintptr_t) info; ++ katom->jit_blocked = false; ++ ++ lockdep_assert_held(&kctx->jctx.lock); ++ list_add_tail(&katom->jit_node, &kctx->jit_atoms_head); ++ ++ /* ++ * Note: ++ * The provided info->gpu_alloc_addr isn't validated here as ++ * userland can cache allocations which means that even ++ * though the region is valid it doesn't represent the ++ * same thing it used to. ++ * ++ * Complete validation of va_pages, commit_pages and extent ++ * isn't done here as it will be done during the call to ++ * kbase_mem_alloc. ++ */ ++ return 0; ++ ++free_info: ++ kfree(info); ++fail: ++ katom->jc = 0; ++ return ret; ++} ++ ++static u8 kbase_jit_free_get_id(struct kbase_jd_atom *katom) ++{ ++ if (WARN_ON(katom->core_req != BASE_JD_REQ_SOFT_JIT_FREE)) ++ return 0; ++ ++ return (u8) katom->jc; ++} ++ ++static int kbase_jit_allocate_process(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ struct base_jit_alloc_info *info; ++ struct kbase_va_region *reg; ++ struct kbase_vmap_struct mapping; ++ u64 *ptr, new_addr; ++ ++ if (katom->jit_blocked) { ++ list_del(&katom->queue); ++ katom->jit_blocked = false; ++ } ++ ++ info = (struct base_jit_alloc_info *) (uintptr_t) katom->jc; ++ ++ /* The JIT ID is still in use so fail the allocation */ ++ if (kctx->jit_alloc[info->id]) { ++ katom->event_code = BASE_JD_EVENT_MEM_GROWTH_FAILED; ++ return 0; ++ } ++ ++ /* Create a JIT allocation */ ++ reg = kbase_jit_allocate(kctx, info); ++ if (!reg) { ++ struct kbase_jd_atom *jit_atom; ++ bool can_block = false; ++ ++ lockdep_assert_held(&kctx->jctx.lock); ++ ++ jit_atom = list_first_entry(&kctx->jit_atoms_head, ++ struct kbase_jd_atom, jit_node); ++ ++ list_for_each_entry(jit_atom, &kctx->jit_atoms_head, jit_node) { ++ if (jit_atom == katom) ++ break; ++ if (jit_atom->core_req == BASE_JD_REQ_SOFT_JIT_FREE) { ++ u8 free_id = kbase_jit_free_get_id(jit_atom); ++ ++ if (free_id && kctx->jit_alloc[free_id]) { ++ /* A JIT free which is active and ++ * submitted before this atom ++ */ ++ can_block = true; ++ break; ++ } ++ } ++ } ++ ++ if (!can_block) { ++ /* Mark the allocation so we know it's in use even if ++ * the allocation itself fails. ++ */ ++ kctx->jit_alloc[info->id] = ++ (struct kbase_va_region *) -1; ++ ++ katom->event_code = BASE_JD_EVENT_MEM_GROWTH_FAILED; ++ return 0; ++ } ++ ++ /* There are pending frees for an active allocation ++ * so we should wait to see whether they free the memory. ++ * Add to the beginning of the list to ensure that the atom is ++ * processed only once in kbase_jit_free_finish ++ */ ++ list_add(&katom->queue, &kctx->jit_pending_alloc); ++ katom->jit_blocked = true; ++ ++ return 1; ++ } ++ ++ /* ++ * Write the address of the JIT allocation to the user provided ++ * GPU allocation. ++ */ ++ ptr = kbase_vmap(kctx, info->gpu_alloc_addr, sizeof(*ptr), ++ &mapping); ++ if (!ptr) { ++ /* ++ * Leave the allocation "live" as the JIT free jit will be ++ * submitted anyway. ++ */ ++ katom->event_code = BASE_JD_EVENT_JOB_INVALID; ++ return 0; ++ } ++ ++ new_addr = reg->start_pfn << PAGE_SHIFT; ++ *ptr = new_addr; ++ KBASE_TLSTREAM_TL_ATTRIB_ATOM_JIT( ++ katom, info->gpu_alloc_addr, new_addr); ++ kbase_vunmap(kctx, &mapping); ++ ++ katom->event_code = BASE_JD_EVENT_DONE; ++ ++ /* ++ * Bind it to the user provided ID. Do this last so we can check for ++ * the JIT free racing this JIT alloc job. ++ */ ++ kctx->jit_alloc[info->id] = reg; ++ ++ return 0; ++} ++ ++static void kbase_jit_allocate_finish(struct kbase_jd_atom *katom) ++{ ++ struct base_jit_alloc_info *info; ++ ++ lockdep_assert_held(&katom->kctx->jctx.lock); ++ ++ /* Remove atom from jit_atoms_head list */ ++ list_del(&katom->jit_node); ++ ++ if (katom->jit_blocked) { ++ list_del(&katom->queue); ++ katom->jit_blocked = false; ++ } ++ ++ info = (struct base_jit_alloc_info *) (uintptr_t) katom->jc; ++ /* Free the info structure */ ++ kfree(info); ++} ++ ++static int kbase_jit_free_prepare(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ ++ lockdep_assert_held(&kctx->jctx.lock); ++ list_add_tail(&katom->jit_node, &kctx->jit_atoms_head); ++ ++ return 0; ++} ++ ++static void kbase_jit_free_process(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ u8 id = kbase_jit_free_get_id(katom); ++ ++ /* ++ * If the ID is zero or it is not in use yet then fail the job. ++ */ ++ if ((id == 0) || (kctx->jit_alloc[id] == NULL)) { ++ katom->event_code = BASE_JD_EVENT_JOB_INVALID; ++ return; ++ } ++ ++ /* ++ * If the ID is valid but the allocation request failed still succeed ++ * this soft job but don't try and free the allocation. ++ */ ++ if (kctx->jit_alloc[id] != (struct kbase_va_region *) -1) ++ kbase_jit_free(kctx, kctx->jit_alloc[id]); ++ ++ kctx->jit_alloc[id] = NULL; ++} ++ ++static void kbasep_jit_free_finish_worker(struct work_struct *work) ++{ ++ struct kbase_jd_atom *katom = container_of(work, struct kbase_jd_atom, ++ work); ++ struct kbase_context *kctx = katom->kctx; ++ int resched; ++ ++ mutex_lock(&kctx->jctx.lock); ++ kbase_finish_soft_job(katom); ++ resched = jd_done_nolock(katom, NULL); ++ mutex_unlock(&kctx->jctx.lock); ++ ++ if (resched) ++ kbase_js_sched_all(kctx->kbdev); ++} ++ ++static void kbase_jit_free_finish(struct kbase_jd_atom *katom) ++{ ++ struct list_head *i, *tmp; ++ struct kbase_context *kctx = katom->kctx; ++ ++ lockdep_assert_held(&kctx->jctx.lock); ++ /* Remove this atom from the kctx->jit_atoms_head list */ ++ list_del(&katom->jit_node); ++ ++ list_for_each_safe(i, tmp, &kctx->jit_pending_alloc) { ++ struct kbase_jd_atom *pending_atom = list_entry(i, ++ struct kbase_jd_atom, queue); ++ if (kbase_jit_allocate_process(pending_atom) == 0) { ++ /* Atom has completed */ ++ INIT_WORK(&pending_atom->work, ++ kbasep_jit_free_finish_worker); ++ queue_work(kctx->jctx.job_done_wq, &pending_atom->work); ++ } ++ } ++} ++ ++static int kbase_ext_res_prepare(struct kbase_jd_atom *katom) ++{ ++ __user struct base_external_resource_list *user_ext_res; ++ struct base_external_resource_list *ext_res; ++ u64 count = 0; ++ size_t copy_size; ++ int ret; ++ ++ user_ext_res = (__user struct base_external_resource_list *) ++ (uintptr_t) katom->jc; ++ ++ /* Fail the job if there is no info structure */ ++ if (!user_ext_res) { ++ ret = -EINVAL; ++ goto fail; ++ } ++ ++ if (copy_from_user(&count, &user_ext_res->count, sizeof(u64)) != 0) { ++ ret = -EINVAL; ++ goto fail; ++ } ++ ++ /* Is the number of external resources in range? */ ++ if (!count || count > BASE_EXT_RES_COUNT_MAX) { ++ ret = -EINVAL; ++ goto fail; ++ } ++ ++ /* Copy the information for safe access and future storage */ ++ copy_size = sizeof(*ext_res); ++ copy_size += sizeof(struct base_external_resource) * (count - 1); ++ ext_res = kzalloc(copy_size, GFP_KERNEL); ++ if (!ext_res) { ++ ret = -ENOMEM; ++ goto fail; ++ } ++ ++ if (copy_from_user(ext_res, user_ext_res, copy_size) != 0) { ++ ret = -EINVAL; ++ goto free_info; ++ } ++ ++ /* ++ * Overwrite the count with the first value incase it was changed ++ * after the fact. ++ */ ++ ext_res->count = count; ++ ++ /* ++ * Replace the user pointer with our kernel allocated ++ * ext_res structure. ++ */ ++ katom->jc = (u64)(uintptr_t) ext_res; ++ ++ return 0; ++ ++free_info: ++ kfree(ext_res); ++fail: ++ return ret; ++} ++ ++static void kbase_ext_res_process(struct kbase_jd_atom *katom, bool map) ++{ ++ struct base_external_resource_list *ext_res; ++ int i; ++ bool failed = false; ++ ++ ext_res = (struct base_external_resource_list *) (uintptr_t) katom->jc; ++ if (!ext_res) ++ goto failed_jc; ++ ++ kbase_gpu_vm_lock(katom->kctx); ++ ++ for (i = 0; i < ext_res->count; i++) { ++ u64 gpu_addr; ++ ++ gpu_addr = ext_res->ext_res[i].ext_resource & ++ ~BASE_EXT_RES_ACCESS_EXCLUSIVE; ++ if (map) { ++ if (!kbase_sticky_resource_acquire(katom->kctx, ++ gpu_addr)) ++ goto failed_loop; ++ } else ++ if (!kbase_sticky_resource_release(katom->kctx, NULL, ++ gpu_addr)) ++ failed = true; ++ } ++ ++ /* ++ * In the case of unmap we continue unmapping other resources in the ++ * case of failure but will always report failure if _any_ unmap ++ * request fails. ++ */ ++ if (failed) ++ katom->event_code = BASE_JD_EVENT_JOB_INVALID; ++ else ++ katom->event_code = BASE_JD_EVENT_DONE; ++ ++ kbase_gpu_vm_unlock(katom->kctx); ++ ++ return; ++ ++failed_loop: ++ while (--i > 0) { ++ u64 gpu_addr; ++ ++ gpu_addr = ext_res->ext_res[i].ext_resource & ++ ~BASE_EXT_RES_ACCESS_EXCLUSIVE; ++ ++ kbase_sticky_resource_release(katom->kctx, NULL, gpu_addr); ++ } ++ ++ katom->event_code = BASE_JD_EVENT_JOB_INVALID; ++ kbase_gpu_vm_unlock(katom->kctx); ++ ++failed_jc: ++ return; ++} ++ ++static void kbase_ext_res_finish(struct kbase_jd_atom *katom) ++{ ++ struct base_external_resource_list *ext_res; ++ ++ ext_res = (struct base_external_resource_list *) (uintptr_t) katom->jc; ++ /* Free the info structure */ ++ kfree(ext_res); ++} ++ ++int kbase_process_soft_job(struct kbase_jd_atom *katom) ++{ ++ switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { ++ case BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME: ++ return kbase_dump_cpu_gpu_time(katom); ++ ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ case BASE_JD_REQ_SOFT_FENCE_TRIGGER: ++ katom->event_code = kbase_sync_fence_out_trigger(katom, ++ katom->event_code == BASE_JD_EVENT_DONE ? ++ 0 : -EFAULT); ++ break; ++ case BASE_JD_REQ_SOFT_FENCE_WAIT: ++ { ++ int ret = kbase_sync_fence_in_wait(katom); ++ ++ if (ret == 1) { ++#ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG ++ kbasep_add_waiting_with_timeout(katom); ++#else ++ kbasep_add_waiting_soft_job(katom); ++#endif ++ } ++ return ret; ++ } ++#endif ++ ++ case BASE_JD_REQ_SOFT_REPLAY: ++ return kbase_replay_process(katom); ++ case BASE_JD_REQ_SOFT_EVENT_WAIT: ++ return kbasep_soft_event_wait(katom); ++ case BASE_JD_REQ_SOFT_EVENT_SET: ++ kbasep_soft_event_update_locked(katom, BASE_JD_SOFT_EVENT_SET); ++ break; ++ case BASE_JD_REQ_SOFT_EVENT_RESET: ++ kbasep_soft_event_update_locked(katom, BASE_JD_SOFT_EVENT_RESET); ++ break; ++ case BASE_JD_REQ_SOFT_DEBUG_COPY: ++ { ++ int res = kbase_debug_copy(katom); ++ ++ if (res) ++ katom->event_code = BASE_JD_EVENT_JOB_INVALID; ++ break; ++ } ++ case BASE_JD_REQ_SOFT_JIT_ALLOC: ++ return kbase_jit_allocate_process(katom); ++ case BASE_JD_REQ_SOFT_JIT_FREE: ++ kbase_jit_free_process(katom); ++ break; ++ case BASE_JD_REQ_SOFT_EXT_RES_MAP: ++ kbase_ext_res_process(katom, true); ++ break; ++ case BASE_JD_REQ_SOFT_EXT_RES_UNMAP: ++ kbase_ext_res_process(katom, false); ++ break; ++ } ++ ++ /* Atom is complete */ ++ return 0; ++} ++ ++void kbase_cancel_soft_job(struct kbase_jd_atom *katom) ++{ ++ switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ case BASE_JD_REQ_SOFT_FENCE_WAIT: ++ kbase_sync_fence_in_cancel_wait(katom); ++ break; ++#endif ++ case BASE_JD_REQ_SOFT_EVENT_WAIT: ++ kbasep_soft_event_cancel_job(katom); ++ break; ++ default: ++ /* This soft-job doesn't support cancellation! */ ++ KBASE_DEBUG_ASSERT(0); ++ } ++} ++ ++int kbase_prepare_soft_job(struct kbase_jd_atom *katom) ++{ ++ switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { ++ case BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME: ++ { ++ if (!IS_ALIGNED(katom->jc, cache_line_size())) ++ return -EINVAL; ++ } ++ break; ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ case BASE_JD_REQ_SOFT_FENCE_TRIGGER: ++ { ++ struct base_fence fence; ++ int fd; ++ ++ if (0 != copy_from_user(&fence, (__user void *)(uintptr_t) katom->jc, sizeof(fence))) ++ return -EINVAL; ++ ++ fd = kbase_sync_fence_out_create(katom, ++ fence.basep.stream_fd); ++ if (fd < 0) ++ return -EINVAL; ++ ++ fence.basep.fd = fd; ++ if (0 != copy_to_user((__user void *)(uintptr_t) katom->jc, &fence, sizeof(fence))) { ++ kbase_sync_fence_out_remove(katom); ++ kbase_sync_fence_close_fd(fd); ++ fence.basep.fd = -EINVAL; ++ return -EINVAL; ++ } ++ } ++ break; ++ case BASE_JD_REQ_SOFT_FENCE_WAIT: ++ { ++ struct base_fence fence; ++ int ret; ++ ++ if (0 != copy_from_user(&fence, (__user void *)(uintptr_t) katom->jc, sizeof(fence))) ++ return -EINVAL; ++ ++ /* Get a reference to the fence object */ ++ ret = kbase_sync_fence_in_from_fd(katom, ++ fence.basep.fd); ++ if (ret < 0) ++ return ret; ++ ++#ifdef CONFIG_MALI_BIFROST_DMA_FENCE ++ /* ++ * Set KCTX_NO_IMPLICIT_FENCE in the context the first ++ * time a soft fence wait job is observed. This will ++ * prevent the implicit dma-buf fence to conflict with ++ * the Android native sync fences. ++ */ ++ if (!kbase_ctx_flag(katom->kctx, KCTX_NO_IMPLICIT_SYNC)) ++ kbase_ctx_flag_set(katom->kctx, KCTX_NO_IMPLICIT_SYNC); ++#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ ++ } ++ break; ++#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ ++ case BASE_JD_REQ_SOFT_JIT_ALLOC: ++ return kbase_jit_allocate_prepare(katom); ++ case BASE_JD_REQ_SOFT_REPLAY: ++ break; ++ case BASE_JD_REQ_SOFT_JIT_FREE: ++ return kbase_jit_free_prepare(katom); ++ case BASE_JD_REQ_SOFT_EVENT_WAIT: ++ case BASE_JD_REQ_SOFT_EVENT_SET: ++ case BASE_JD_REQ_SOFT_EVENT_RESET: ++ if (katom->jc == 0) ++ return -EINVAL; ++ break; ++ case BASE_JD_REQ_SOFT_DEBUG_COPY: ++ return kbase_debug_copy_prepare(katom); ++ case BASE_JD_REQ_SOFT_EXT_RES_MAP: ++ return kbase_ext_res_prepare(katom); ++ case BASE_JD_REQ_SOFT_EXT_RES_UNMAP: ++ return kbase_ext_res_prepare(katom); ++ default: ++ /* Unsupported soft-job */ ++ return -EINVAL; ++ } ++ return 0; ++} ++ ++void kbase_finish_soft_job(struct kbase_jd_atom *katom) ++{ ++ switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { ++ case BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME: ++ /* Nothing to do */ ++ break; ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ case BASE_JD_REQ_SOFT_FENCE_TRIGGER: ++ /* If fence has not yet been signaled, do it now */ ++ kbase_sync_fence_out_trigger(katom, katom->event_code == ++ BASE_JD_EVENT_DONE ? 0 : -EFAULT); ++ break; ++ case BASE_JD_REQ_SOFT_FENCE_WAIT: ++ /* Release katom's reference to fence object */ ++ kbase_sync_fence_in_remove(katom); ++ break; ++#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ ++ case BASE_JD_REQ_SOFT_DEBUG_COPY: ++ kbase_debug_copy_finish(katom); ++ break; ++ case BASE_JD_REQ_SOFT_JIT_ALLOC: ++ kbase_jit_allocate_finish(katom); ++ break; ++ case BASE_JD_REQ_SOFT_EXT_RES_MAP: ++ kbase_ext_res_finish(katom); ++ break; ++ case BASE_JD_REQ_SOFT_EXT_RES_UNMAP: ++ kbase_ext_res_finish(katom); ++ break; ++ case BASE_JD_REQ_SOFT_JIT_FREE: ++ kbase_jit_free_finish(katom); ++ break; ++ } ++} ++ ++void kbase_resume_suspended_soft_jobs(struct kbase_device *kbdev) ++{ ++ LIST_HEAD(local_suspended_soft_jobs); ++ struct kbase_jd_atom *tmp_iter; ++ struct kbase_jd_atom *katom_iter; ++ struct kbasep_js_device_data *js_devdata; ++ bool resched = false; ++ ++ KBASE_DEBUG_ASSERT(kbdev); ++ ++ js_devdata = &kbdev->js_data; ++ ++ /* Move out the entire list */ ++ mutex_lock(&js_devdata->runpool_mutex); ++ list_splice_init(&js_devdata->suspended_soft_jobs_list, ++ &local_suspended_soft_jobs); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ ++ /* ++ * Each atom must be detached from the list and ran separately - ++ * it could be re-added to the old list, but this is unlikely ++ */ ++ list_for_each_entry_safe(katom_iter, tmp_iter, ++ &local_suspended_soft_jobs, dep_item[1]) { ++ struct kbase_context *kctx = katom_iter->kctx; ++ ++ mutex_lock(&kctx->jctx.lock); ++ ++ /* Remove from the global list */ ++ list_del(&katom_iter->dep_item[1]); ++ /* Remove from the context's list of waiting soft jobs */ ++ kbasep_remove_waiting_soft_job(katom_iter); ++ ++ if (kbase_process_soft_job(katom_iter) == 0) { ++ kbase_finish_soft_job(katom_iter); ++ resched |= jd_done_nolock(katom_iter, NULL); ++ } else { ++ KBASE_DEBUG_ASSERT((katom_iter->core_req & ++ BASE_JD_REQ_SOFT_JOB_TYPE) ++ != BASE_JD_REQ_SOFT_REPLAY); ++ } ++ ++ mutex_unlock(&kctx->jctx.lock); ++ } ++ ++ if (resched) ++ kbase_js_sched_all(kbdev); ++} +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_strings.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_strings.c +new file mode 100755 +index 000000000000..c98762cec244 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_strings.c +@@ -0,0 +1,23 @@ ++ /* ++ * ++ * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++#include "mali_kbase_strings.h" ++ ++#define KBASE_DRV_NAME "mali" ++#define KBASE_TIMELINE_NAME KBASE_DRV_NAME ".timeline" ++ ++const char kbase_drv_name[] = KBASE_DRV_NAME; ++const char kbase_timeline_name[] = KBASE_TIMELINE_NAME; +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_strings.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_strings.h +new file mode 100755 +index 000000000000..41b8fdbec6a4 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_strings.h +@@ -0,0 +1,19 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++extern const char kbase_drv_name[]; ++extern const char kbase_timeline_name[]; +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_sync.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_sync.h +new file mode 100755 +index 000000000000..54159262314a +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_sync.h +@@ -0,0 +1,203 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/** ++ * @file mali_kbase_sync.h ++ * ++ * This file contains our internal "API" for explicit fences. ++ * It hides the implementation details of the actual explicit fence mechanism ++ * used (Android fences or sync file with DMA fences). ++ */ ++ ++#ifndef MALI_KBASE_SYNC_H ++#define MALI_KBASE_SYNC_H ++ ++#include ++#ifdef CONFIG_SYNC ++#include ++#endif ++#ifdef CONFIG_SYNC_FILE ++#include "mali_kbase_fence_defs.h" ++#include ++#endif ++ ++#include "mali_kbase.h" ++ ++/** ++ * struct kbase_sync_fence_info - Information about a fence ++ * @fence: Pointer to fence (type is void*, as underlaying struct can differ) ++ * @name: The name given to this fence when it was created ++ * @status: < 0 means error, 0 means active, 1 means signaled ++ * ++ * Use kbase_sync_fence_in_info_get() or kbase_sync_fence_out_info_get() ++ * to get the information. ++ */ ++struct kbase_sync_fence_info { ++ void *fence; ++ char name[32]; ++ int status; ++}; ++ ++/** ++ * kbase_sync_fence_stream_create() - Create a stream object ++ * @name: Name of stream (only used to ease debugging/visualization) ++ * @out_fd: A file descriptor representing the created stream object ++ * ++ * Can map down to a timeline implementation in some implementations. ++ * Exposed as a file descriptor. ++ * Life-time controlled via the file descriptor: ++ * - dup to add a ref ++ * - close to remove a ref ++ * ++ * return: 0 on success, < 0 on error ++ */ ++int kbase_sync_fence_stream_create(const char *name, int *const out_fd); ++ ++/** ++ * kbase_sync_fence_out_create Create an explicit output fence to specified atom ++ * @katom: Atom to assign the new explicit fence to ++ * @stream_fd: File descriptor for stream object to create fence on ++ * ++ * return: Valid file descriptor to fence or < 0 on error ++ */ ++int kbase_sync_fence_out_create(struct kbase_jd_atom *katom, int stream_fd); ++ ++/** ++ * kbase_sync_fence_in_from_fd() Assigns an existing fence to specified atom ++ * @katom: Atom to assign the existing explicit fence to ++ * @fd: File descriptor to an existing fence ++ * ++ * Assigns an explicit input fence to atom. ++ * This can later be waited for by calling @kbase_sync_fence_in_wait ++ * ++ * return: 0 on success, < 0 on error ++ */ ++int kbase_sync_fence_in_from_fd(struct kbase_jd_atom *katom, int fd); ++ ++/** ++ * kbase_sync_fence_validate() - Validate a fd to be a valid fence ++ * @fd: File descriptor to check ++ * ++ * This function is only usable to catch unintentional user errors early, ++ * it does not stop malicious code changing the fd after this function returns. ++ * ++ * return 0: if fd is for a valid fence, < 0 if invalid ++ */ ++int kbase_sync_fence_validate(int fd); ++ ++/** ++ * kbase_sync_fence_out_trigger - Signal explicit output fence attached on katom ++ * @katom: Atom with an explicit fence to signal ++ * @result: < 0 means signal with error, 0 >= indicates success ++ * ++ * Signal output fence attached on katom and remove the fence from the atom. ++ * ++ * return: The "next" event code for atom, typically JOB_CANCELLED or EVENT_DONE ++ */ ++enum base_jd_event_code ++kbase_sync_fence_out_trigger(struct kbase_jd_atom *katom, int result); ++ ++/** ++ * kbase_sync_fence_in_wait() - Wait for explicit input fence to be signaled ++ * @katom: Atom with explicit fence to wait for ++ * ++ * If the fence is already signaled, then 0 is returned, and the caller must ++ * continue processing of the katom. ++ * ++ * If the fence isn't already signaled, then this kbase_sync framework will ++ * take responsibility to continue the processing once the fence is signaled. ++ * ++ * return: 0 if already signaled, otherwise 1 ++ */ ++int kbase_sync_fence_in_wait(struct kbase_jd_atom *katom); ++ ++/** ++ * kbase_sync_fence_in_cancel_wait() - Cancel explicit input fence waits ++ * @katom: Atom to cancel wait for ++ * ++ * This function is fully responsible for continuing processing of this atom ++ * (remove_waiting_soft_job + finish_soft_job + jd_done + js_sched_all) ++ */ ++void kbase_sync_fence_in_cancel_wait(struct kbase_jd_atom *katom); ++ ++/** ++ * kbase_sync_fence_in_remove() - Remove the input fence from the katom ++ * @katom: Atom to remove explicit input fence for ++ * ++ * This will also release the corresponding reference. ++ */ ++void kbase_sync_fence_in_remove(struct kbase_jd_atom *katom); ++ ++/** ++ * kbase_sync_fence_out_remove() - Remove the output fence from the katom ++ * @katom: Atom to remove explicit output fence for ++ * ++ * This will also release the corresponding reference. ++ */ ++void kbase_sync_fence_out_remove(struct kbase_jd_atom *katom); ++ ++/** ++ * kbase_sync_fence_close_fd() - Close a file descriptor representing a fence ++ * @fd: File descriptor to close ++ */ ++static inline void kbase_sync_fence_close_fd(int fd) ++{ ++ ksys_close(fd); ++} ++ ++/** ++ * kbase_sync_fence_in_info_get() - Retrieves information about input fence ++ * @katom: Atom to get fence information from ++ * @info: Struct to be filled with fence information ++ * ++ * return: 0 on success, < 0 on error ++ */ ++int kbase_sync_fence_in_info_get(struct kbase_jd_atom *katom, ++ struct kbase_sync_fence_info *info); ++ ++/** ++ * kbase_sync_fence_out_info_get() - Retrieves information about output fence ++ * @katom: Atom to get fence information from ++ * @info: Struct to be filled with fence information ++ * ++ * return: 0 on success, < 0 on error ++ */ ++int kbase_sync_fence_out_info_get(struct kbase_jd_atom *katom, ++ struct kbase_sync_fence_info *info); ++ ++/** ++ * kbase_sync_status_string() - Get string matching @status ++ * @status: Value of fence status. ++ * ++ * return: Pointer to string describing @status. ++ */ ++const char *kbase_sync_status_string(int status); ++ ++/* ++ * Internal worker used to continue processing of atom. ++ */ ++void kbase_sync_fence_wait_worker(struct work_struct *data); ++ ++#ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG ++/** ++ * kbase_sync_fence_in_dump() Trigger a debug dump of atoms input fence state ++ * @katom: Atom to trigger fence debug dump for ++ */ ++void kbase_sync_fence_in_dump(struct kbase_jd_atom *katom); ++#endif ++ ++#endif /* MALI_KBASE_SYNC_H */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_sync_android.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_sync_android.c +new file mode 100755 +index 000000000000..e4528e2b9f25 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_sync_android.c +@@ -0,0 +1,537 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* ++ * Code for supporting explicit Android fences (CONFIG_SYNC) ++ * Known to be good for kernels 4.5 and earlier. ++ * Replaced with CONFIG_SYNC_FILE for 4.9 and later kernels ++ * (see mali_kbase_sync_file.c) ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include "sync.h" ++#include ++#include ++ ++struct mali_sync_timeline { ++ struct sync_timeline timeline; ++ atomic_t counter; ++ atomic_t signaled; ++}; ++ ++struct mali_sync_pt { ++ struct sync_pt pt; ++ int order; ++ int result; ++}; ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) ++/* For backwards compatibility with kernels before 3.17. After 3.17 ++ * sync_pt_parent is included in the kernel. */ ++static inline struct sync_timeline *sync_pt_parent(struct sync_pt *pt) ++{ ++ return pt->parent; ++} ++#endif ++ ++static struct mali_sync_timeline *to_mali_sync_timeline( ++ struct sync_timeline *timeline) ++{ ++ return container_of(timeline, struct mali_sync_timeline, timeline); ++} ++ ++static struct mali_sync_pt *to_mali_sync_pt(struct sync_pt *pt) ++{ ++ return container_of(pt, struct mali_sync_pt, pt); ++} ++ ++static struct sync_pt *timeline_dup(struct sync_pt *pt) ++{ ++ struct mali_sync_pt *mpt = to_mali_sync_pt(pt); ++ struct mali_sync_pt *new_mpt; ++ struct sync_pt *new_pt = sync_pt_create(sync_pt_parent(pt), ++ sizeof(struct mali_sync_pt)); ++ ++ if (!new_pt) ++ return NULL; ++ ++ new_mpt = to_mali_sync_pt(new_pt); ++ new_mpt->order = mpt->order; ++ new_mpt->result = mpt->result; ++ ++ return new_pt; ++} ++ ++static int timeline_has_signaled(struct sync_pt *pt) ++{ ++ struct mali_sync_pt *mpt = to_mali_sync_pt(pt); ++ struct mali_sync_timeline *mtl = to_mali_sync_timeline( ++ sync_pt_parent(pt)); ++ int result = mpt->result; ++ ++ int diff = atomic_read(&mtl->signaled) - mpt->order; ++ ++ if (diff >= 0) ++ return (result < 0) ? result : 1; ++ ++ return 0; ++} ++ ++static int timeline_compare(struct sync_pt *a, struct sync_pt *b) ++{ ++ struct mali_sync_pt *ma = container_of(a, struct mali_sync_pt, pt); ++ struct mali_sync_pt *mb = container_of(b, struct mali_sync_pt, pt); ++ ++ int diff = ma->order - mb->order; ++ ++ if (diff == 0) ++ return 0; ++ ++ return (diff < 0) ? -1 : 1; ++} ++ ++static void timeline_value_str(struct sync_timeline *timeline, char *str, ++ int size) ++{ ++ struct mali_sync_timeline *mtl = to_mali_sync_timeline(timeline); ++ ++ snprintf(str, size, "%d", atomic_read(&mtl->signaled)); ++} ++ ++static void pt_value_str(struct sync_pt *pt, char *str, int size) ++{ ++ struct mali_sync_pt *mpt = to_mali_sync_pt(pt); ++ ++ snprintf(str, size, "%d(%d)", mpt->order, mpt->result); ++} ++ ++static struct sync_timeline_ops mali_timeline_ops = { ++ .driver_name = "Mali", ++ .dup = timeline_dup, ++ .has_signaled = timeline_has_signaled, ++ .compare = timeline_compare, ++ .timeline_value_str = timeline_value_str, ++ .pt_value_str = pt_value_str, ++}; ++ ++/* Allocates a timeline for Mali ++ * ++ * One timeline should be allocated per API context. ++ */ ++static struct sync_timeline *mali_sync_timeline_alloc(const char *name) ++{ ++ struct sync_timeline *tl; ++ struct mali_sync_timeline *mtl; ++ ++ tl = sync_timeline_create(&mali_timeline_ops, ++ sizeof(struct mali_sync_timeline), name); ++ if (!tl) ++ return NULL; ++ ++ /* Set the counter in our private struct */ ++ mtl = to_mali_sync_timeline(tl); ++ atomic_set(&mtl->counter, 0); ++ atomic_set(&mtl->signaled, 0); ++ ++ return tl; ++} ++ ++static int kbase_stream_close(struct inode *inode, struct file *file) ++{ ++ struct sync_timeline *tl; ++ ++ tl = (struct sync_timeline *)file->private_data; ++ sync_timeline_destroy(tl); ++ return 0; ++} ++ ++static const struct file_operations stream_fops = { ++ .owner = THIS_MODULE, ++ .release = kbase_stream_close, ++}; ++ ++int kbase_sync_fence_stream_create(const char *name, int *const out_fd) ++{ ++ struct sync_timeline *tl; ++ ++ if (!out_fd) ++ return -EINVAL; ++ ++ tl = mali_sync_timeline_alloc(name); ++ if (!tl) ++ return -EINVAL; ++ ++ *out_fd = anon_inode_getfd(name, &stream_fops, tl, O_RDONLY|O_CLOEXEC); ++ ++ if (*out_fd < 0) { ++ sync_timeline_destroy(tl); ++ return -EINVAL; ++ } ++ ++ return 0; ++} ++ ++/* Allocates a sync point within the timeline. ++ * ++ * The timeline must be the one allocated by kbase_sync_timeline_alloc ++ * ++ * Sync points must be triggered in *exactly* the same order as they are ++ * allocated. ++ */ ++static struct sync_pt *kbase_sync_pt_alloc(struct sync_timeline *parent) ++{ ++ struct sync_pt *pt = sync_pt_create(parent, ++ sizeof(struct mali_sync_pt)); ++ struct mali_sync_timeline *mtl = to_mali_sync_timeline(parent); ++ struct mali_sync_pt *mpt; ++ ++ if (!pt) ++ return NULL; ++ ++ mpt = to_mali_sync_pt(pt); ++ mpt->order = atomic_inc_return(&mtl->counter); ++ mpt->result = 0; ++ ++ return pt; ++} ++ ++int kbase_sync_fence_out_create(struct kbase_jd_atom *katom, int tl_fd) ++{ ++ struct sync_timeline *tl; ++ struct sync_pt *pt; ++ struct sync_fence *fence; ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 7, 0) ++ struct files_struct *files; ++ struct fdtable *fdt; ++#endif ++ int fd; ++ struct file *tl_file; ++ ++ tl_file = fget(tl_fd); ++ if (tl_file == NULL) ++ return -EBADF; ++ ++ if (tl_file->f_op != &stream_fops) { ++ fd = -EBADF; ++ goto out; ++ } ++ ++ tl = tl_file->private_data; ++ ++ pt = kbase_sync_pt_alloc(tl); ++ if (!pt) { ++ fd = -EFAULT; ++ goto out; ++ } ++ ++ fence = sync_fence_create("mali_fence", pt); ++ if (!fence) { ++ sync_pt_free(pt); ++ fd = -EFAULT; ++ goto out; ++ } ++ ++ /* from here the fence owns the sync_pt */ ++ ++ /* create a fd representing the fence */ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0) ++ fd = get_unused_fd_flags(O_RDWR | O_CLOEXEC); ++ if (fd < 0) { ++ sync_fence_put(fence); ++ goto out; ++ } ++#else ++ fd = get_unused_fd(); ++ if (fd < 0) { ++ sync_fence_put(fence); ++ goto out; ++ } ++ ++ files = current->files; ++ spin_lock(&files->file_lock); ++ fdt = files_fdtable(files); ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 4, 0) ++ __set_close_on_exec(fd, fdt); ++#else ++ FD_SET(fd, fdt->close_on_exec); ++#endif ++ spin_unlock(&files->file_lock); ++#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0) */ ++ ++ /* bind fence to the new fd */ ++ sync_fence_install(fence, fd); ++ ++ katom->fence = sync_fence_fdget(fd); ++ if (katom->fence == NULL) { ++ /* The only way the fence can be NULL is if userspace closed it ++ * for us, so we don't need to clear it up */ ++ fd = -EINVAL; ++ goto out; ++ } ++ ++out: ++ fput(tl_file); ++ ++ return fd; ++} ++ ++int kbase_sync_fence_in_from_fd(struct kbase_jd_atom *katom, int fd) ++{ ++ katom->fence = sync_fence_fdget(fd); ++ return katom->fence ? 0 : -ENOENT; ++} ++ ++int kbase_sync_fence_validate(int fd) ++{ ++ struct sync_fence *fence; ++ ++ fence = sync_fence_fdget(fd); ++ if (!fence) ++ return -EINVAL; ++ ++ sync_fence_put(fence); ++ return 0; ++} ++ ++/* Returns true if the specified timeline is allocated by Mali */ ++static int kbase_sync_timeline_is_ours(struct sync_timeline *timeline) ++{ ++ return timeline->ops == &mali_timeline_ops; ++} ++ ++/* Signals a particular sync point ++ * ++ * Sync points must be triggered in *exactly* the same order as they are ++ * allocated. ++ * ++ * If they are signaled in the wrong order then a message will be printed in ++ * debug builds and otherwise attempts to signal order sync_pts will be ignored. ++ * ++ * result can be negative to indicate error, any other value is interpreted as ++ * success. ++ */ ++static void kbase_sync_signal_pt(struct sync_pt *pt, int result) ++{ ++ struct mali_sync_pt *mpt = to_mali_sync_pt(pt); ++ struct mali_sync_timeline *mtl = to_mali_sync_timeline( ++ sync_pt_parent(pt)); ++ int signaled; ++ int diff; ++ ++ mpt->result = result; ++ ++ do { ++ signaled = atomic_read(&mtl->signaled); ++ ++ diff = signaled - mpt->order; ++ ++ if (diff > 0) { ++ /* The timeline is already at or ahead of this point. ++ * This should not happen unless userspace has been ++ * signaling fences out of order, so warn but don't ++ * violate the sync_pt API. ++ * The warning is only in debug builds to prevent ++ * a malicious user being able to spam dmesg. ++ */ ++#ifdef CONFIG_MALI_BIFROST_DEBUG ++ pr_err("Fences were triggered in a different order to allocation!"); ++#endif /* CONFIG_MALI_BIFROST_DEBUG */ ++ return; ++ } ++ } while (atomic_cmpxchg(&mtl->signaled, ++ signaled, mpt->order) != signaled); ++} ++ ++enum base_jd_event_code ++kbase_sync_fence_out_trigger(struct kbase_jd_atom *katom, int result) ++{ ++ struct sync_pt *pt; ++ struct sync_timeline *timeline; ++ ++ if (!katom->fence) ++ return BASE_JD_EVENT_JOB_CANCELLED; ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) ++ if (!list_is_singular(&katom->fence->pt_list_head)) { ++#else ++ if (katom->fence->num_fences != 1) { ++#endif ++ /* Not exactly one item in the list - so it didn't (directly) ++ * come from us */ ++ return BASE_JD_EVENT_JOB_CANCELLED; ++ } ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) ++ pt = list_first_entry(&katom->fence->pt_list_head, ++ struct sync_pt, pt_list); ++#else ++ pt = container_of(katom->fence->cbs[0].sync_pt, struct sync_pt, base); ++#endif ++ timeline = sync_pt_parent(pt); ++ ++ if (!kbase_sync_timeline_is_ours(timeline)) { ++ /* Fence has a sync_pt which isn't ours! */ ++ return BASE_JD_EVENT_JOB_CANCELLED; ++ } ++ ++ kbase_sync_signal_pt(pt, result); ++ ++ sync_timeline_signal(timeline); ++ ++ kbase_sync_fence_out_remove(katom); ++ ++ return (result < 0) ? BASE_JD_EVENT_JOB_CANCELLED : BASE_JD_EVENT_DONE; ++} ++ ++static inline int kbase_fence_get_status(struct sync_fence *fence) ++{ ++ if (!fence) ++ return -ENOENT; ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) ++ return fence->status; ++#else ++ return atomic_read(&fence->status); ++#endif ++} ++ ++static void kbase_fence_wait_callback(struct sync_fence *fence, ++ struct sync_fence_waiter *waiter) ++{ ++ struct kbase_jd_atom *katom = container_of(waiter, ++ struct kbase_jd_atom, sync_waiter); ++ struct kbase_context *kctx = katom->kctx; ++ ++ /* Propagate the fence status to the atom. ++ * If negative then cancel this atom and its dependencies. ++ */ ++ if (kbase_fence_get_status(fence) < 0) ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ ++ /* To prevent a potential deadlock we schedule the work onto the ++ * job_done_wq workqueue ++ * ++ * The issue is that we may signal the timeline while holding ++ * kctx->jctx.lock and the callbacks are run synchronously from ++ * sync_timeline_signal. So we simply defer the work. ++ */ ++ ++ INIT_WORK(&katom->work, kbase_sync_fence_wait_worker); ++ queue_work(kctx->jctx.job_done_wq, &katom->work); ++} ++ ++int kbase_sync_fence_in_wait(struct kbase_jd_atom *katom) ++{ ++ int ret; ++ ++ sync_fence_waiter_init(&katom->sync_waiter, kbase_fence_wait_callback); ++ ++ ret = sync_fence_wait_async(katom->fence, &katom->sync_waiter); ++ ++ if (ret == 1) { ++ /* Already signaled */ ++ return 0; ++ } ++ ++ if (ret < 0) { ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ /* We should cause the dependent jobs in the bag to be failed, ++ * to do this we schedule the work queue to complete this job */ ++ INIT_WORK(&katom->work, kbase_sync_fence_wait_worker); ++ queue_work(katom->kctx->jctx.job_done_wq, &katom->work); ++ } ++ ++ return 1; ++} ++ ++void kbase_sync_fence_in_cancel_wait(struct kbase_jd_atom *katom) ++{ ++ if (sync_fence_cancel_async(katom->fence, &katom->sync_waiter) != 0) { ++ /* The wait wasn't cancelled - leave the cleanup for ++ * kbase_fence_wait_callback */ ++ return; ++ } ++ ++ /* Wait was cancelled - zap the atoms */ ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ ++ kbasep_remove_waiting_soft_job(katom); ++ kbase_finish_soft_job(katom); ++ ++ if (jd_done_nolock(katom, NULL)) ++ kbase_js_sched_all(katom->kctx->kbdev); ++} ++ ++void kbase_sync_fence_out_remove(struct kbase_jd_atom *katom) ++{ ++ if (katom->fence) { ++ sync_fence_put(katom->fence); ++ katom->fence = NULL; ++ } ++} ++ ++void kbase_sync_fence_in_remove(struct kbase_jd_atom *katom) ++{ ++ if (katom->fence) { ++ sync_fence_put(katom->fence); ++ katom->fence = NULL; ++ } ++} ++ ++int kbase_sync_fence_in_info_get(struct kbase_jd_atom *katom, ++ struct kbase_sync_fence_info *info) ++{ ++ if (!katom->fence) ++ return -ENOENT; ++ ++ info->fence = katom->fence; ++ info->status = kbase_fence_get_status(katom->fence); ++ strlcpy(info->name, katom->fence->name, sizeof(info->name)); ++ ++ return 0; ++} ++ ++int kbase_sync_fence_out_info_get(struct kbase_jd_atom *katom, ++ struct kbase_sync_fence_info *info) ++{ ++ if (!katom->fence) ++ return -ENOENT; ++ ++ info->fence = katom->fence; ++ info->status = kbase_fence_get_status(katom->fence); ++ strlcpy(info->name, katom->fence->name, sizeof(info->name)); ++ ++ return 0; ++} ++ ++#ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG ++void kbase_sync_fence_in_dump(struct kbase_jd_atom *katom) ++{ ++ /* Dump out the full state of all the Android sync fences. ++ * The function sync_dump() isn't exported to modules, so force ++ * sync_fence_wait() to time out to trigger sync_dump(). ++ */ ++ if (katom->fence) ++ sync_fence_wait(katom->fence, 1); ++} ++#endif +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_sync_common.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_sync_common.c +new file mode 100755 +index 000000000000..457def296684 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_sync_common.c +@@ -0,0 +1,43 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* ++ * @file mali_kbase_sync_common.c ++ * ++ * Common code for our explicit fence functionality ++ */ ++ ++#include ++#include "mali_kbase.h" ++ ++void kbase_sync_fence_wait_worker(struct work_struct *data) ++{ ++ struct kbase_jd_atom *katom; ++ ++ katom = container_of(data, struct kbase_jd_atom, work); ++ kbase_soft_event_wait_callback(katom); ++} ++ ++const char *kbase_sync_status_string(int status) ++{ ++ if (status == 0) ++ return "signaled"; ++ else if (status > 0) ++ return "active"; ++ else ++ return "error"; ++} +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_sync_file.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_sync_file.c +new file mode 100755 +index 000000000000..509c0666f10f +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_sync_file.c +@@ -0,0 +1,348 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* ++ * Code for supporting explicit Linux fences (CONFIG_SYNC_FILE) ++ * Introduced in kernel 4.9. ++ * Android explicit fences (CONFIG_SYNC) can be used for older kernels ++ * (see mali_kbase_sync_android.c) ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include "mali_kbase_fence_defs.h" ++#include "mali_kbase_sync.h" ++#include "mali_kbase_fence.h" ++#include "mali_kbase.h" ++ ++static const struct file_operations stream_fops = { ++ .owner = THIS_MODULE ++}; ++ ++int kbase_sync_fence_stream_create(const char *name, int *const out_fd) ++{ ++ if (!out_fd) ++ return -EINVAL; ++ ++ *out_fd = anon_inode_getfd(name, &stream_fops, NULL, ++ O_RDONLY | O_CLOEXEC); ++ if (*out_fd < 0) ++ return -EINVAL; ++ ++ return 0; ++} ++ ++int kbase_sync_fence_out_create(struct kbase_jd_atom *katom, int stream_fd) ++{ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ struct fence *fence; ++#else ++ struct dma_fence *fence; ++#endif ++ struct sync_file *sync_file; ++ int fd; ++ ++ fence = kbase_fence_out_new(katom); ++ if (!fence) ++ return -ENOMEM; ++ ++ /* Take an extra reference to the fence on behalf of the katom. ++ * This is needed because sync_file_create() will take ownership of ++ * one of these refs */ ++ dma_fence_get(fence); ++ ++ /* create a sync_file fd representing the fence */ ++ sync_file = sync_file_create(fence); ++ if (!sync_file) { ++ dma_fence_put(fence); ++ kbase_fence_out_remove(katom); ++ return -ENOMEM; ++ } ++ ++ fd = get_unused_fd_flags(O_CLOEXEC); ++ if (fd < 0) { ++ fput(sync_file->file); ++ kbase_fence_out_remove(katom); ++ return fd; ++ } ++ ++ fd_install(fd, sync_file->file); ++ ++ return fd; ++} ++ ++int kbase_sync_fence_in_from_fd(struct kbase_jd_atom *katom, int fd) ++{ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ struct fence *fence = sync_file_get_fence(fd); ++#else ++ struct dma_fence *fence = sync_file_get_fence(fd); ++#endif ++ ++ if (!fence) ++ return -ENOENT; ++ ++ kbase_fence_fence_in_set(katom, fence); ++ ++ return 0; ++} ++ ++int kbase_sync_fence_validate(int fd) ++{ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ struct fence *fence = sync_file_get_fence(fd); ++#else ++ struct dma_fence *fence = sync_file_get_fence(fd); ++#endif ++ ++ if (!fence) ++ return -EINVAL; ++ ++ dma_fence_put(fence); ++ ++ return 0; /* valid */ ++} ++ ++enum base_jd_event_code ++kbase_sync_fence_out_trigger(struct kbase_jd_atom *katom, int result) ++{ ++ int res; ++ ++ if (!kbase_fence_out_is_ours(katom)) { ++ /* Not our fence */ ++ return BASE_JD_EVENT_JOB_CANCELLED; ++ } ++ ++ res = kbase_fence_out_signal(katom, result); ++ if (unlikely(res < 0)) { ++ dev_warn(katom->kctx->kbdev->dev, ++ "fence_signal() failed with %d\n", res); ++ } ++ ++ kbase_sync_fence_out_remove(katom); ++ ++ return (result != 0) ? BASE_JD_EVENT_JOB_CANCELLED : BASE_JD_EVENT_DONE; ++} ++ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++static void kbase_fence_wait_callback(struct fence *fence, ++ struct fence_cb *cb) ++#else ++static void kbase_fence_wait_callback(struct dma_fence *fence, ++ struct dma_fence_cb *cb) ++#endif ++{ ++ struct kbase_fence_cb *kcb = container_of(cb, ++ struct kbase_fence_cb, ++ fence_cb); ++ struct kbase_jd_atom *katom = kcb->katom; ++ struct kbase_context *kctx = katom->kctx; ++ ++ /* Cancel atom if fence is erroneous */ ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)) ++ if (dma_fence_is_signaled(kcb->fence) && kcb->fence->error) ++#else ++ if (dma_fence_is_signaled(kcb->fence) && kcb->fence->status < 0) ++#endif ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ ++ if (kbase_fence_dep_count_dec_and_test(katom)) { ++ /* We take responsibility of handling this */ ++ kbase_fence_dep_count_set(katom, -1); ++ ++ /* To prevent a potential deadlock we schedule the work onto the ++ * job_done_wq workqueue ++ * ++ * The issue is that we may signal the timeline while holding ++ * kctx->jctx.lock and the callbacks are run synchronously from ++ * sync_timeline_signal. So we simply defer the work. ++ */ ++ INIT_WORK(&katom->work, kbase_sync_fence_wait_worker); ++ queue_work(kctx->jctx.job_done_wq, &katom->work); ++ } ++} ++ ++int kbase_sync_fence_in_wait(struct kbase_jd_atom *katom) ++{ ++ int err; ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ struct fence *fence; ++#else ++ struct dma_fence *fence; ++#endif ++ ++ fence = kbase_fence_in_get(katom); ++ if (!fence) ++ return 0; /* no input fence to wait for, good to go! */ ++ ++ kbase_fence_dep_count_set(katom, 1); ++ ++ err = kbase_fence_add_callback(katom, fence, kbase_fence_wait_callback); ++ ++ kbase_fence_put(fence); ++ ++ if (likely(!err)) { ++ /* Test if the callbacks are already triggered */ ++ if (kbase_fence_dep_count_dec_and_test(katom)) { ++ kbase_fence_free_callbacks(katom); ++ kbase_fence_dep_count_set(katom, -1); ++ return 0; /* Already signaled, good to go right now */ ++ } ++ ++ /* Callback installed, so we just need to wait for it... */ ++ } else { ++ /* Failure */ ++ kbase_fence_free_callbacks(katom); ++ kbase_fence_dep_count_set(katom, -1); ++ ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ ++ /* We should cause the dependent jobs in the bag to be failed, ++ * to do this we schedule the work queue to complete this job */ ++ ++ INIT_WORK(&katom->work, kbase_sync_fence_wait_worker); ++ queue_work(katom->kctx->jctx.job_done_wq, &katom->work); ++ } ++ ++ return 1; /* completion to be done later by callback/worker */ ++} ++ ++void kbase_sync_fence_in_cancel_wait(struct kbase_jd_atom *katom) ++{ ++ if (!kbase_fence_free_callbacks(katom)) { ++ /* The wait wasn't cancelled - ++ * leave the cleanup for kbase_fence_wait_callback */ ++ return; ++ } ++ ++ /* Take responsibility of completion */ ++ kbase_fence_dep_count_set(katom, -1); ++ ++ /* Wait was cancelled - zap the atoms */ ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ ++ kbasep_remove_waiting_soft_job(katom); ++ kbase_finish_soft_job(katom); ++ ++ if (jd_done_nolock(katom, NULL)) ++ kbase_js_sched_all(katom->kctx->kbdev); ++} ++ ++void kbase_sync_fence_out_remove(struct kbase_jd_atom *katom) ++{ ++ kbase_fence_out_remove(katom); ++} ++ ++void kbase_sync_fence_in_remove(struct kbase_jd_atom *katom) ++{ ++ kbase_fence_free_callbacks(katom); ++ kbase_fence_in_remove(katom); ++} ++ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++static void kbase_sync_fence_info_get(struct fence *fence, ++ struct kbase_sync_fence_info *info) ++#else ++static void kbase_sync_fence_info_get(struct dma_fence *fence, ++ struct kbase_sync_fence_info *info) ++#endif ++{ ++ info->fence = fence; ++ ++ /* translate into CONFIG_SYNC status: ++ * < 0 : error ++ * 0 : active ++ * 1 : signaled ++ */ ++ if (dma_fence_is_signaled(fence)) { ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)) ++ int status = fence->error; ++#else ++ int status = fence->status; ++#endif ++ if (status < 0) ++ info->status = status; /* signaled with error */ ++ else ++ info->status = 1; /* signaled with success */ ++ } else { ++ info->status = 0; /* still active (unsignaled) */ ++ } ++ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0)) ++ scnprintf(info->name, sizeof(info->name), "%u#%u", ++ fence->context, fence->seqno); ++#else ++ scnprintf(info->name, sizeof(info->name), "%llu#%u", ++ fence->context, fence->seqno); ++#endif ++} ++ ++int kbase_sync_fence_in_info_get(struct kbase_jd_atom *katom, ++ struct kbase_sync_fence_info *info) ++{ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ struct fence *fence; ++#else ++ struct dma_fence *fence; ++#endif ++ ++ fence = kbase_fence_in_get(katom); ++ if (!fence) ++ return -ENOENT; ++ ++ kbase_sync_fence_info_get(fence, info); ++ ++ kbase_fence_put(fence); ++ ++ return 0; ++} ++ ++int kbase_sync_fence_out_info_get(struct kbase_jd_atom *katom, ++ struct kbase_sync_fence_info *info) ++{ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ struct fence *fence; ++#else ++ struct dma_fence *fence; ++#endif ++ ++ fence = kbase_fence_out_get(katom); ++ if (!fence) ++ return -ENOENT; ++ ++ kbase_sync_fence_info_get(fence, info); ++ ++ kbase_fence_put(fence); ++ ++ return 0; ++} ++ ++ ++#ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG ++void kbase_sync_fence_in_dump(struct kbase_jd_atom *katom) ++{ ++ /* Not implemented */ ++} ++#endif +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_tlstream.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_tlstream.c +new file mode 100755 +index 000000000000..485565ebfe80 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_tlstream.c +@@ -0,0 +1,2569 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#include ++#include ++ ++/*****************************************************************************/ ++ ++/* The version of swtrace protocol used in timeline stream. */ ++#define SWTRACE_VERSION 3 ++ ++/* The maximum expected length of string in tracepoint descriptor. */ ++#define STRLEN_MAX 64 /* bytes */ ++ ++/* The number of nanoseconds in a second. */ ++#define NSECS_IN_SEC 1000000000ull /* ns */ ++ ++/* The period of autoflush checker execution in milliseconds. */ ++#define AUTOFLUSH_INTERVAL 1000 /* ms */ ++ ++/* The maximum size of a single packet used by timeline. */ ++#define PACKET_SIZE 4096 /* bytes */ ++ ++/* The number of packets used by one timeline stream. */ ++#define PACKET_COUNT 16 ++ ++/* The number of bytes reserved for packet header. ++ * These value must be defined according to MIPE documentation. */ ++#define PACKET_HEADER_SIZE 8 /* bytes */ ++ ++/* The number of bytes reserved for packet sequence number. ++ * These value must be defined according to MIPE documentation. */ ++#define PACKET_NUMBER_SIZE 4 /* bytes */ ++ ++/* Packet header - first word. ++ * These values must be defined according to MIPE documentation. */ ++#define PACKET_STREAMID_POS 0 ++#define PACKET_STREAMID_LEN 8 ++#define PACKET_RSVD1_POS (PACKET_STREAMID_POS + PACKET_STREAMID_LEN) ++#define PACKET_RSVD1_LEN 8 ++#define PACKET_TYPE_POS (PACKET_RSVD1_POS + PACKET_RSVD1_LEN) ++#define PACKET_TYPE_LEN 3 ++#define PACKET_CLASS_POS (PACKET_TYPE_POS + PACKET_TYPE_LEN) ++#define PACKET_CLASS_LEN 7 ++#define PACKET_FAMILY_POS (PACKET_CLASS_POS + PACKET_CLASS_LEN) ++#define PACKET_FAMILY_LEN 6 ++ ++/* Packet header - second word ++ * These values must be defined according to MIPE documentation. */ ++#define PACKET_LENGTH_POS 0 ++#define PACKET_LENGTH_LEN 24 ++#define PACKET_SEQBIT_POS (PACKET_LENGTH_POS + PACKET_LENGTH_LEN) ++#define PACKET_SEQBIT_LEN 1 ++#define PACKET_RSVD2_POS (PACKET_SEQBIT_POS + PACKET_SEQBIT_LEN) ++#define PACKET_RSVD2_LEN 7 ++ ++/* Types of streams generated by timeline. ++ * Order is significant! Header streams must precede respective body streams. */ ++enum tl_stream_type { ++ TL_STREAM_TYPE_OBJ_HEADER, ++ TL_STREAM_TYPE_OBJ_SUMMARY, ++ TL_STREAM_TYPE_OBJ, ++ TL_STREAM_TYPE_AUX_HEADER, ++ TL_STREAM_TYPE_AUX, ++ ++ TL_STREAM_TYPE_COUNT ++}; ++ ++/* Timeline packet family ids. ++ * Values are significant! Check MIPE documentation. */ ++enum tl_packet_family { ++ TL_PACKET_FAMILY_CTRL = 0, /* control packets */ ++ TL_PACKET_FAMILY_TL = 1, /* timeline packets */ ++ ++ TL_PACKET_FAMILY_COUNT ++}; ++ ++/* Packet classes used in timeline streams. ++ * Values are significant! Check MIPE documentation. */ ++enum tl_packet_class { ++ TL_PACKET_CLASS_OBJ = 0, /* timeline objects packet */ ++ TL_PACKET_CLASS_AUX = 1, /* auxiliary events packet */ ++}; ++ ++/* Packet types used in timeline streams. ++ * Values are significant! Check MIPE documentation. */ ++enum tl_packet_type { ++ TL_PACKET_TYPE_HEADER = 0, /* stream's header/directory */ ++ TL_PACKET_TYPE_BODY = 1, /* stream's body */ ++ TL_PACKET_TYPE_SUMMARY = 2, /* stream's summary */ ++}; ++ ++/* Message ids of trace events that are recorded in the timeline stream. */ ++enum tl_msg_id_obj { ++ /* Timeline object events. */ ++ KBASE_TL_NEW_CTX, ++ KBASE_TL_NEW_GPU, ++ KBASE_TL_NEW_LPU, ++ KBASE_TL_NEW_ATOM, ++ KBASE_TL_NEW_AS, ++ KBASE_TL_DEL_CTX, ++ KBASE_TL_DEL_ATOM, ++ KBASE_TL_LIFELINK_LPU_GPU, ++ KBASE_TL_LIFELINK_AS_GPU, ++ KBASE_TL_RET_CTX_LPU, ++ KBASE_TL_RET_ATOM_CTX, ++ KBASE_TL_RET_ATOM_LPU, ++ KBASE_TL_NRET_CTX_LPU, ++ KBASE_TL_NRET_ATOM_CTX, ++ KBASE_TL_NRET_ATOM_LPU, ++ KBASE_TL_RET_AS_CTX, ++ KBASE_TL_NRET_AS_CTX, ++ KBASE_TL_RET_ATOM_AS, ++ KBASE_TL_NRET_ATOM_AS, ++ KBASE_TL_DEP_ATOM_ATOM, ++ KBASE_TL_NDEP_ATOM_ATOM, ++ KBASE_TL_RDEP_ATOM_ATOM, ++ KBASE_TL_ATTRIB_ATOM_CONFIG, ++ KBASE_TL_ATTRIB_ATOM_PRIORITY, ++ KBASE_TL_ATTRIB_ATOM_STATE, ++ KBASE_TL_ATTRIB_ATOM_PRIORITY_CHANGE, ++ KBASE_TL_ATTRIB_ATOM_JIT, ++ KBASE_TL_ATTRIB_AS_CONFIG, ++ KBASE_TL_EVENT_LPU_SOFTSTOP, ++ KBASE_TL_EVENT_ATOM_SOFTSTOP_EX, ++ KBASE_TL_EVENT_ATOM_SOFTSTOP_ISSUE, ++ ++ /* Job dump specific events. */ ++ KBASE_JD_GPU_SOFT_RESET ++}; ++ ++/* Message ids of trace events that are recorded in the auxiliary stream. */ ++enum tl_msg_id_aux { ++ KBASE_AUX_PM_STATE, ++ KBASE_AUX_PAGEFAULT, ++ KBASE_AUX_PAGESALLOC, ++ KBASE_AUX_DEVFREQ_TARGET, ++ KBASE_AUX_PROTECTED_ENTER_START, ++ KBASE_AUX_PROTECTED_ENTER_END, ++ KBASE_AUX_PROTECTED_LEAVE_START, ++ KBASE_AUX_PROTECTED_LEAVE_END ++}; ++ ++/*****************************************************************************/ ++ ++/** ++ * struct tl_stream - timeline stream structure ++ * @lock: message order lock ++ * @buffer: array of buffers ++ * @wbi: write buffer index ++ * @rbi: read buffer index ++ * @numbered: if non-zero stream's packets are sequentially numbered ++ * @autoflush_counter: counter tracking stream's autoflush state ++ * ++ * This structure holds information needed to construct proper packets in the ++ * timeline stream. Each message in sequence must bear timestamp that is greater ++ * to one in previous message in the same stream. For this reason lock is held ++ * throughout the process of message creation. Each stream contains set of ++ * buffers. Each buffer will hold one MIPE packet. In case there is no free ++ * space required to store incoming message the oldest buffer is discarded. ++ * Each packet in timeline body stream has sequence number embedded (this value ++ * must increment monotonically and is used by packets receiver to discover ++ * buffer overflows. ++ * Autoflush counter is set to negative number when there is no data pending ++ * for flush and it is set to zero on every update of the buffer. Autoflush ++ * timer will increment the counter by one on every expiry. In case there will ++ * be no activity on the buffer during two consecutive timer expiries, stream ++ * buffer will be flushed. ++ */ ++struct tl_stream { ++ spinlock_t lock; ++ ++ struct { ++ atomic_t size; /* number of bytes in buffer */ ++ char data[PACKET_SIZE]; /* buffer's data */ ++ } buffer[PACKET_COUNT]; ++ ++ atomic_t wbi; ++ atomic_t rbi; ++ ++ int numbered; ++ atomic_t autoflush_counter; ++}; ++ ++/** ++ * struct tp_desc - tracepoint message descriptor structure ++ * @id: tracepoint ID identifying message in stream ++ * @id_str: human readable version of tracepoint ID ++ * @name: tracepoint description ++ * @arg_types: tracepoint's arguments types declaration ++ * @arg_names: comma separated list of tracepoint's arguments names ++ */ ++struct tp_desc { ++ u32 id; ++ const char *id_str; ++ const char *name; ++ const char *arg_types; ++ const char *arg_names; ++}; ++ ++/*****************************************************************************/ ++ ++/* Configuration of timeline streams generated by kernel. ++ * Kernel emit only streams containing either timeline object events or ++ * auxiliary events. All streams have stream id value of 1 (as opposed to user ++ * space streams that have value of 0). */ ++static const struct { ++ enum tl_packet_family pkt_family; ++ enum tl_packet_class pkt_class; ++ enum tl_packet_type pkt_type; ++ unsigned int stream_id; ++} tl_stream_cfg[TL_STREAM_TYPE_COUNT] = { ++ {TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_OBJ, TL_PACKET_TYPE_HEADER, 1}, ++ {TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_OBJ, TL_PACKET_TYPE_SUMMARY, 1}, ++ {TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_OBJ, TL_PACKET_TYPE_BODY, 1}, ++ {TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_AUX, TL_PACKET_TYPE_HEADER, 1}, ++ {TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_AUX, TL_PACKET_TYPE_BODY, 1} ++}; ++ ++/* The timeline streams generated by kernel. */ ++static struct tl_stream *tl_stream[TL_STREAM_TYPE_COUNT]; ++ ++/* Autoflush timer. */ ++static struct timer_list autoflush_timer; ++ ++/* If non-zero autoflush timer is active. */ ++static atomic_t autoflush_timer_active; ++ ++/* Reader lock. Only one reader is allowed to have access to the timeline ++ * streams at any given time. */ ++static DEFINE_MUTEX(tl_reader_lock); ++ ++/* Timeline stream event queue. */ ++static DECLARE_WAIT_QUEUE_HEAD(tl_event_queue); ++ ++/* The timeline stream file operations functions. */ ++static ssize_t kbasep_tlstream_read( ++ struct file *filp, ++ char __user *buffer, ++ size_t size, ++ loff_t *f_pos); ++static unsigned int kbasep_tlstream_poll(struct file *filp, poll_table *wait); ++static int kbasep_tlstream_release(struct inode *inode, struct file *filp); ++ ++/* The timeline stream file operations structure. */ ++static const struct file_operations kbasep_tlstream_fops = { ++ .release = kbasep_tlstream_release, ++ .read = kbasep_tlstream_read, ++ .poll = kbasep_tlstream_poll, ++}; ++ ++/* Descriptors of timeline messages transmitted in object events stream. */ ++static const struct tp_desc tp_desc_obj[] = { ++ { ++ KBASE_TL_NEW_CTX, ++ __stringify(KBASE_TL_NEW_CTX), ++ "object ctx is created", ++ "@pII", ++ "ctx,ctx_nr,tgid" ++ }, ++ { ++ KBASE_TL_NEW_GPU, ++ __stringify(KBASE_TL_NEW_GPU), ++ "object gpu is created", ++ "@pII", ++ "gpu,gpu_id,core_count" ++ }, ++ { ++ KBASE_TL_NEW_LPU, ++ __stringify(KBASE_TL_NEW_LPU), ++ "object lpu is created", ++ "@pII", ++ "lpu,lpu_nr,lpu_fn" ++ }, ++ { ++ KBASE_TL_NEW_ATOM, ++ __stringify(KBASE_TL_NEW_ATOM), ++ "object atom is created", ++ "@pI", ++ "atom,atom_nr" ++ }, ++ { ++ KBASE_TL_NEW_AS, ++ __stringify(KBASE_TL_NEW_AS), ++ "address space object is created", ++ "@pI", ++ "address_space,as_nr" ++ }, ++ { ++ KBASE_TL_DEL_CTX, ++ __stringify(KBASE_TL_DEL_CTX), ++ "context is destroyed", ++ "@p", ++ "ctx" ++ }, ++ { ++ KBASE_TL_DEL_ATOM, ++ __stringify(KBASE_TL_DEL_ATOM), ++ "atom is destroyed", ++ "@p", ++ "atom" ++ }, ++ { ++ KBASE_TL_LIFELINK_LPU_GPU, ++ __stringify(KBASE_TL_LIFELINK_LPU_GPU), ++ "lpu is deleted with gpu", ++ "@pp", ++ "lpu,gpu" ++ }, ++ { ++ KBASE_TL_LIFELINK_AS_GPU, ++ __stringify(KBASE_TL_LIFELINK_AS_GPU), ++ "address space is deleted with gpu", ++ "@pp", ++ "address_space,gpu" ++ }, ++ { ++ KBASE_TL_RET_CTX_LPU, ++ __stringify(KBASE_TL_RET_CTX_LPU), ++ "context is retained by lpu", ++ "@pp", ++ "ctx,lpu" ++ }, ++ { ++ KBASE_TL_RET_ATOM_CTX, ++ __stringify(KBASE_TL_RET_ATOM_CTX), ++ "atom is retained by context", ++ "@pp", ++ "atom,ctx" ++ }, ++ { ++ KBASE_TL_RET_ATOM_LPU, ++ __stringify(KBASE_TL_RET_ATOM_LPU), ++ "atom is retained by lpu", ++ "@pps", ++ "atom,lpu,attrib_match_list" ++ }, ++ { ++ KBASE_TL_NRET_CTX_LPU, ++ __stringify(KBASE_TL_NRET_CTX_LPU), ++ "context is released by lpu", ++ "@pp", ++ "ctx,lpu" ++ }, ++ { ++ KBASE_TL_NRET_ATOM_CTX, ++ __stringify(KBASE_TL_NRET_ATOM_CTX), ++ "atom is released by context", ++ "@pp", ++ "atom,ctx" ++ }, ++ { ++ KBASE_TL_NRET_ATOM_LPU, ++ __stringify(KBASE_TL_NRET_ATOM_LPU), ++ "atom is released by lpu", ++ "@pp", ++ "atom,lpu" ++ }, ++ { ++ KBASE_TL_RET_AS_CTX, ++ __stringify(KBASE_TL_RET_AS_CTX), ++ "address space is retained by context", ++ "@pp", ++ "address_space,ctx" ++ }, ++ { ++ KBASE_TL_NRET_AS_CTX, ++ __stringify(KBASE_TL_NRET_AS_CTX), ++ "address space is released by context", ++ "@pp", ++ "address_space,ctx" ++ }, ++ { ++ KBASE_TL_RET_ATOM_AS, ++ __stringify(KBASE_TL_RET_ATOM_AS), ++ "atom is retained by address space", ++ "@pp", ++ "atom,address_space" ++ }, ++ { ++ KBASE_TL_NRET_ATOM_AS, ++ __stringify(KBASE_TL_NRET_ATOM_AS), ++ "atom is released by address space", ++ "@pp", ++ "atom,address_space" ++ }, ++ { ++ KBASE_TL_DEP_ATOM_ATOM, ++ __stringify(KBASE_TL_DEP_ATOM_ATOM), ++ "atom2 depends on atom1", ++ "@pp", ++ "atom1,atom2" ++ }, ++ { ++ KBASE_TL_NDEP_ATOM_ATOM, ++ __stringify(KBASE_TL_NDEP_ATOM_ATOM), ++ "atom2 no longer depends on atom1", ++ "@pp", ++ "atom1,atom2" ++ }, ++ { ++ KBASE_TL_RDEP_ATOM_ATOM, ++ __stringify(KBASE_TL_RDEP_ATOM_ATOM), ++ "resolved dependecy of atom2 depending on atom1", ++ "@pp", ++ "atom1,atom2" ++ }, ++ { ++ KBASE_TL_ATTRIB_ATOM_CONFIG, ++ __stringify(KBASE_TL_ATTRIB_ATOM_CONFIG), ++ "atom job slot attributes", ++ "@pLLI", ++ "atom,descriptor,affinity,config" ++ }, ++ { ++ KBASE_TL_ATTRIB_ATOM_PRIORITY, ++ __stringify(KBASE_TL_ATTRIB_ATOM_PRIORITY), ++ "atom priority", ++ "@pI", ++ "atom,prio" ++ }, ++ { ++ KBASE_TL_ATTRIB_ATOM_STATE, ++ __stringify(KBASE_TL_ATTRIB_ATOM_STATE), ++ "atom state", ++ "@pI", ++ "atom,state" ++ }, ++ { ++ KBASE_TL_ATTRIB_ATOM_PRIORITY_CHANGE, ++ __stringify(KBASE_TL_ATTRIB_ATOM_PRIORITY_CHANGE), ++ "atom caused priority change", ++ "@p", ++ "atom" ++ }, ++ { ++ KBASE_TL_ATTRIB_ATOM_JIT, ++ __stringify(KBASE_TL_ATTRIB_ATOM_JIT), ++ "jit done for atom", ++ "@pLL", ++ "atom,edit_addr,new_addr" ++ }, ++ { ++ KBASE_TL_ATTRIB_AS_CONFIG, ++ __stringify(KBASE_TL_ATTRIB_AS_CONFIG), ++ "address space attributes", ++ "@pLLL", ++ "address_space,transtab,memattr,transcfg" ++ }, ++ { ++ KBASE_TL_EVENT_LPU_SOFTSTOP, ++ __stringify(KBASE_TL_EVENT_LPU_SOFTSTOP), ++ "softstop event on given lpu", ++ "@p", ++ "lpu" ++ }, ++ { ++ KBASE_TL_EVENT_ATOM_SOFTSTOP_EX, ++ __stringify(KBASE_TL_EVENT_ATOM_SOFTSTOP_EX), ++ "atom softstopped", ++ "@p", ++ "atom" ++ }, ++ { ++ KBASE_TL_EVENT_ATOM_SOFTSTOP_ISSUE, ++ __stringify(KBASE_TL_EVENT_SOFTSTOP_ISSUE), ++ "atom softstop issued", ++ "@p", ++ "atom" ++ }, ++ { ++ KBASE_JD_GPU_SOFT_RESET, ++ __stringify(KBASE_JD_GPU_SOFT_RESET), ++ "gpu soft reset", ++ "@p", ++ "gpu" ++ }, ++}; ++ ++/* Descriptors of timeline messages transmitted in auxiliary events stream. */ ++static const struct tp_desc tp_desc_aux[] = { ++ { ++ KBASE_AUX_PM_STATE, ++ __stringify(KBASE_AUX_PM_STATE), ++ "PM state", ++ "@IL", ++ "core_type,core_state_bitset" ++ }, ++ { ++ KBASE_AUX_PAGEFAULT, ++ __stringify(KBASE_AUX_PAGEFAULT), ++ "Page fault", ++ "@IL", ++ "ctx_nr,page_cnt_change" ++ }, ++ { ++ KBASE_AUX_PAGESALLOC, ++ __stringify(KBASE_AUX_PAGESALLOC), ++ "Total alloc pages change", ++ "@IL", ++ "ctx_nr,page_cnt" ++ }, ++ { ++ KBASE_AUX_DEVFREQ_TARGET, ++ __stringify(KBASE_AUX_DEVFREQ_TARGET), ++ "New device frequency target", ++ "@L", ++ "target_freq" ++ }, ++ { ++ KBASE_AUX_PROTECTED_ENTER_START, ++ __stringify(KBASE_AUX_PROTECTED_ENTER_START), ++ "enter protected mode start", ++ "@p", ++ "gpu" ++ }, ++ { ++ KBASE_AUX_PROTECTED_ENTER_END, ++ __stringify(KBASE_AUX_PROTECTED_ENTER_END), ++ "enter protected mode end", ++ "@p", ++ "gpu" ++ }, ++ { ++ KBASE_AUX_PROTECTED_LEAVE_START, ++ __stringify(KBASE_AUX_PROTECTED_LEAVE_START), ++ "leave protected mode start", ++ "@p", ++ "gpu" ++ }, ++ { ++ KBASE_AUX_PROTECTED_LEAVE_END, ++ __stringify(KBASE_AUX_PROTECTED_LEAVE_END), ++ "leave protected mode end", ++ "@p", ++ "gpu" ++ } ++}; ++ ++#if MALI_UNIT_TEST ++/* Number of bytes read by user. */ ++static atomic_t tlstream_bytes_collected = {0}; ++ ++/* Number of bytes generated by tracepoint messages. */ ++static atomic_t tlstream_bytes_generated = {0}; ++#endif /* MALI_UNIT_TEST */ ++ ++/*****************************************************************************/ ++ ++/* Indicator of whether the timeline stream file descriptor is used. */ ++atomic_t kbase_tlstream_enabled = {0}; ++ ++/*****************************************************************************/ ++ ++/** ++ * kbasep_tlstream_get_timestamp - return timestamp ++ * ++ * Function returns timestamp value based on raw monotonic timer. Value will ++ * wrap around zero in case of overflow. ++ * Return: timestamp value ++ */ ++static u64 kbasep_tlstream_get_timestamp(void) ++{ ++ struct timespec ts; ++ u64 timestamp; ++ ++ getrawmonotonic(&ts); ++ timestamp = (u64)ts.tv_sec * NSECS_IN_SEC + ts.tv_nsec; ++ return timestamp; ++} ++ ++/** ++ * kbasep_tlstream_write_bytes - write data to message buffer ++ * @buffer: buffer where data will be written ++ * @pos: position in the buffer where to place data ++ * @bytes: pointer to buffer holding data ++ * @len: length of data to be written ++ * ++ * Return: updated position in the buffer ++ */ ++static size_t kbasep_tlstream_write_bytes( ++ char *buffer, ++ size_t pos, ++ const void *bytes, ++ size_t len) ++{ ++ KBASE_DEBUG_ASSERT(buffer); ++ KBASE_DEBUG_ASSERT(bytes); ++ ++ memcpy(&buffer[pos], bytes, len); ++ ++ return pos + len; ++} ++ ++/** ++ * kbasep_tlstream_write_string - write string to message buffer ++ * @buffer: buffer where data will be written ++ * @pos: position in the buffer where to place data ++ * @string: pointer to buffer holding the source string ++ * @max_write_size: number of bytes that can be stored in buffer ++ * ++ * Return: updated position in the buffer ++ */ ++static size_t kbasep_tlstream_write_string( ++ char *buffer, ++ size_t pos, ++ const char *string, ++ size_t max_write_size) ++{ ++ u32 string_len; ++ ++ KBASE_DEBUG_ASSERT(buffer); ++ KBASE_DEBUG_ASSERT(string); ++ /* Timeline string consists of at least string length and nul ++ * terminator. */ ++ KBASE_DEBUG_ASSERT(max_write_size >= sizeof(string_len) + sizeof(char)); ++ max_write_size -= sizeof(string_len); ++ ++ string_len = strlcpy( ++ &buffer[pos + sizeof(string_len)], ++ string, ++ max_write_size); ++ string_len += sizeof(char); ++ ++ /* Make sure that the source string fit into the buffer. */ ++ KBASE_DEBUG_ASSERT(string_len <= max_write_size); ++ ++ /* Update string length. */ ++ memcpy(&buffer[pos], &string_len, sizeof(string_len)); ++ ++ return pos + sizeof(string_len) + string_len; ++} ++ ++/** ++ * kbasep_tlstream_write_timestamp - write timestamp to message buffer ++ * @buffer: buffer where data will be written ++ * @pos: position in the buffer where to place data ++ * ++ * Return: updated position in the buffer ++ */ ++static size_t kbasep_tlstream_write_timestamp(void *buffer, size_t pos) ++{ ++ u64 timestamp = kbasep_tlstream_get_timestamp(); ++ ++ return kbasep_tlstream_write_bytes( ++ buffer, pos, ++ ×tamp, sizeof(timestamp)); ++} ++ ++/** ++ * kbasep_tlstream_put_bits - put bits in a word ++ * @word: pointer to the words being modified ++ * @value: value that shall be written to given position ++ * @bitpos: position where value shall be written (in bits) ++ * @bitlen: length of value (in bits) ++ */ ++static void kbasep_tlstream_put_bits( ++ u32 *word, ++ u32 value, ++ unsigned int bitpos, ++ unsigned int bitlen) ++{ ++ const u32 mask = ((1 << bitlen) - 1) << bitpos; ++ ++ KBASE_DEBUG_ASSERT(word); ++ KBASE_DEBUG_ASSERT((0 != bitlen) && (32 >= bitlen)); ++ KBASE_DEBUG_ASSERT((bitpos + bitlen) <= 32); ++ ++ *word &= ~mask; ++ *word |= ((value << bitpos) & mask); ++} ++ ++/** ++ * kbasep_tlstream_packet_header_setup - setup the packet header ++ * @buffer: pointer to the buffer ++ * @pkt_family: packet's family ++ * @pkt_type: packet's type ++ * @pkt_class: packet's class ++ * @stream_id: stream id ++ * @numbered: non-zero if this stream is numbered ++ * ++ * Function sets up immutable part of packet header in the given buffer. ++ */ ++static void kbasep_tlstream_packet_header_setup( ++ char *buffer, ++ enum tl_packet_family pkt_family, ++ enum tl_packet_class pkt_class, ++ enum tl_packet_type pkt_type, ++ unsigned int stream_id, ++ int numbered) ++{ ++ u32 word0 = 0; ++ u32 word1 = 0; ++ ++ KBASE_DEBUG_ASSERT(buffer); ++ KBASE_DEBUG_ASSERT(pkt_family == TL_PACKET_FAMILY_TL); ++ KBASE_DEBUG_ASSERT( ++ (pkt_type == TL_PACKET_TYPE_HEADER) || ++ (pkt_type == TL_PACKET_TYPE_SUMMARY) || ++ (pkt_type == TL_PACKET_TYPE_BODY)); ++ KBASE_DEBUG_ASSERT( ++ (pkt_class == TL_PACKET_CLASS_OBJ) || ++ (pkt_class == TL_PACKET_CLASS_AUX)); ++ ++ kbasep_tlstream_put_bits( ++ &word0, pkt_family, ++ PACKET_FAMILY_POS, PACKET_FAMILY_LEN); ++ kbasep_tlstream_put_bits( ++ &word0, pkt_class, ++ PACKET_CLASS_POS, PACKET_CLASS_LEN); ++ kbasep_tlstream_put_bits( ++ &word0, pkt_type, ++ PACKET_TYPE_POS, PACKET_TYPE_LEN); ++ kbasep_tlstream_put_bits( ++ &word0, stream_id, ++ PACKET_STREAMID_POS, PACKET_STREAMID_LEN); ++ ++ if (numbered) ++ kbasep_tlstream_put_bits( ++ &word1, 1, ++ PACKET_SEQBIT_POS, PACKET_SEQBIT_LEN); ++ ++ memcpy(&buffer[0], &word0, sizeof(word0)); ++ memcpy(&buffer[sizeof(word0)], &word1, sizeof(word1)); ++} ++ ++/** ++ * kbasep_tlstream_packet_header_update - update the packet header ++ * @buffer: pointer to the buffer ++ * @data_size: amount of data carried in this packet ++ * ++ * Function updates mutable part of packet header in the given buffer. ++ * Note that value of data_size must not including size of the header. ++ */ ++static void kbasep_tlstream_packet_header_update( ++ char *buffer, ++ size_t data_size) ++{ ++ u32 word0; ++ u32 word1; ++ ++ KBASE_DEBUG_ASSERT(buffer); ++ CSTD_UNUSED(word0); ++ ++ memcpy(&word1, &buffer[sizeof(word0)], sizeof(word1)); ++ ++ kbasep_tlstream_put_bits( ++ &word1, data_size, ++ PACKET_LENGTH_POS, PACKET_LENGTH_LEN); ++ ++ memcpy(&buffer[sizeof(word0)], &word1, sizeof(word1)); ++} ++ ++/** ++ * kbasep_tlstream_packet_number_update - update the packet number ++ * @buffer: pointer to the buffer ++ * @counter: value of packet counter for this packet's stream ++ * ++ * Function updates packet number embedded within the packet placed in the ++ * given buffer. ++ */ ++static void kbasep_tlstream_packet_number_update(char *buffer, u32 counter) ++{ ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ memcpy(&buffer[PACKET_HEADER_SIZE], &counter, sizeof(counter)); ++} ++ ++/** ++ * kbasep_timeline_stream_reset - reset stream ++ * @stream: pointer to the stream structure ++ * ++ * Function discards all pending messages and resets packet counters. ++ */ ++static void kbasep_timeline_stream_reset(struct tl_stream *stream) ++{ ++ unsigned int i; ++ ++ for (i = 0; i < PACKET_COUNT; i++) { ++ if (stream->numbered) ++ atomic_set( ++ &stream->buffer[i].size, ++ PACKET_HEADER_SIZE + ++ PACKET_NUMBER_SIZE); ++ else ++ atomic_set(&stream->buffer[i].size, PACKET_HEADER_SIZE); ++ } ++ ++ atomic_set(&stream->wbi, 0); ++ atomic_set(&stream->rbi, 0); ++} ++ ++/** ++ * kbasep_timeline_stream_init - initialize timeline stream ++ * @stream: pointer to the stream structure ++ * @stream_type: stream type ++ */ ++static void kbasep_timeline_stream_init( ++ struct tl_stream *stream, ++ enum tl_stream_type stream_type) ++{ ++ unsigned int i; ++ ++ KBASE_DEBUG_ASSERT(stream); ++ KBASE_DEBUG_ASSERT(TL_STREAM_TYPE_COUNT > stream_type); ++ ++ spin_lock_init(&stream->lock); ++ ++ /* All packets carrying tracepoints shall be numbered. */ ++ if (TL_PACKET_TYPE_BODY == tl_stream_cfg[stream_type].pkt_type) ++ stream->numbered = 1; ++ else ++ stream->numbered = 0; ++ ++ for (i = 0; i < PACKET_COUNT; i++) ++ kbasep_tlstream_packet_header_setup( ++ stream->buffer[i].data, ++ tl_stream_cfg[stream_type].pkt_family, ++ tl_stream_cfg[stream_type].pkt_class, ++ tl_stream_cfg[stream_type].pkt_type, ++ tl_stream_cfg[stream_type].stream_id, ++ stream->numbered); ++ ++ kbasep_timeline_stream_reset(tl_stream[stream_type]); ++} ++ ++/** ++ * kbasep_timeline_stream_term - terminate timeline stream ++ * @stream: pointer to the stream structure ++ */ ++static void kbasep_timeline_stream_term(struct tl_stream *stream) ++{ ++ KBASE_DEBUG_ASSERT(stream); ++} ++ ++/** ++ * kbasep_tlstream_msgbuf_submit - submit packet to the user space ++ * @stream: pointer to the stream structure ++ * @wb_idx_raw: write buffer index ++ * @wb_size: length of data stored in current buffer ++ * ++ * Function updates currently written buffer with packet header. Then write ++ * index is incremented and buffer is handled to user space. Parameters ++ * of new buffer are returned using provided arguments. ++ * ++ * Return: length of data in new buffer ++ * ++ * Warning: User must update the stream structure with returned value. ++ */ ++static size_t kbasep_tlstream_msgbuf_submit( ++ struct tl_stream *stream, ++ unsigned int wb_idx_raw, ++ unsigned int wb_size) ++{ ++ unsigned int rb_idx_raw = atomic_read(&stream->rbi); ++ unsigned int wb_idx = wb_idx_raw % PACKET_COUNT; ++ ++ /* Set stream as flushed. */ ++ atomic_set(&stream->autoflush_counter, -1); ++ ++ kbasep_tlstream_packet_header_update( ++ stream->buffer[wb_idx].data, ++ wb_size - PACKET_HEADER_SIZE); ++ ++ if (stream->numbered) ++ kbasep_tlstream_packet_number_update( ++ stream->buffer[wb_idx].data, ++ wb_idx_raw); ++ ++ /* Increasing write buffer index will expose this packet to the reader. ++ * As stream->lock is not taken on reader side we must make sure memory ++ * is updated correctly before this will happen. */ ++ smp_wmb(); ++ wb_idx_raw++; ++ atomic_set(&stream->wbi, wb_idx_raw); ++ ++ /* Inform user that packets are ready for reading. */ ++ wake_up_interruptible(&tl_event_queue); ++ ++ /* Detect and mark overflow in this stream. */ ++ if (PACKET_COUNT == wb_idx_raw - rb_idx_raw) { ++ /* Reader side depends on this increment to correctly handle ++ * overflows. The value shall be updated only if it was not ++ * modified by the reader. The data holding buffer will not be ++ * updated before stream->lock is released, however size of the ++ * buffer will. Make sure this increment is globally visible ++ * before information about selected write buffer size. */ ++ atomic_cmpxchg(&stream->rbi, rb_idx_raw, rb_idx_raw + 1); ++ } ++ ++ wb_size = PACKET_HEADER_SIZE; ++ if (stream->numbered) ++ wb_size += PACKET_NUMBER_SIZE; ++ ++ return wb_size; ++} ++ ++/** ++ * kbasep_tlstream_msgbuf_acquire - lock selected stream and reserves buffer ++ * @stream_type: type of the stream that shall be locked ++ * @msg_size: message size ++ * @flags: pointer to store flags passed back on stream release ++ * ++ * Function will lock the stream and reserve the number of bytes requested ++ * in msg_size for the user. ++ * ++ * Return: pointer to the buffer where message can be stored ++ * ++ * Warning: Stream must be released with kbasep_tlstream_msgbuf_release(). ++ * Only atomic operations are allowed while stream is locked ++ * (i.e. do not use any operation that may sleep). ++ */ ++static char *kbasep_tlstream_msgbuf_acquire( ++ enum tl_stream_type stream_type, ++ size_t msg_size, ++ unsigned long *flags) __acquires(&stream->lock) ++{ ++ struct tl_stream *stream; ++ unsigned int wb_idx_raw; ++ unsigned int wb_idx; ++ size_t wb_size; ++ ++ KBASE_DEBUG_ASSERT(TL_STREAM_TYPE_COUNT > stream_type); ++ KBASE_DEBUG_ASSERT( ++ PACKET_SIZE - PACKET_HEADER_SIZE - PACKET_NUMBER_SIZE >= ++ msg_size); ++ ++ stream = tl_stream[stream_type]; ++ ++ spin_lock_irqsave(&stream->lock, *flags); ++ ++ wb_idx_raw = atomic_read(&stream->wbi); ++ wb_idx = wb_idx_raw % PACKET_COUNT; ++ wb_size = atomic_read(&stream->buffer[wb_idx].size); ++ ++ /* Select next buffer if data will not fit into current one. */ ++ if (PACKET_SIZE < wb_size + msg_size) { ++ wb_size = kbasep_tlstream_msgbuf_submit( ++ stream, wb_idx_raw, wb_size); ++ wb_idx = (wb_idx_raw + 1) % PACKET_COUNT; ++ } ++ ++ /* Reserve space in selected buffer. */ ++ atomic_set(&stream->buffer[wb_idx].size, wb_size + msg_size); ++ ++#if MALI_UNIT_TEST ++ atomic_add(msg_size, &tlstream_bytes_generated); ++#endif /* MALI_UNIT_TEST */ ++ ++ return &stream->buffer[wb_idx].data[wb_size]; ++} ++ ++/** ++ * kbasep_tlstream_msgbuf_release - unlock selected stream ++ * @stream_type: type of the stream that shall be locked ++ * @flags: value obtained during stream acquire ++ * ++ * Function releases stream that has been previously locked with a call to ++ * kbasep_tlstream_msgbuf_acquire(). ++ */ ++static void kbasep_tlstream_msgbuf_release( ++ enum tl_stream_type stream_type, ++ unsigned long flags) __releases(&stream->lock) ++{ ++ struct tl_stream *stream; ++ ++ KBASE_DEBUG_ASSERT(TL_STREAM_TYPE_COUNT > stream_type); ++ ++ stream = tl_stream[stream_type]; ++ ++ /* Mark stream as containing unflushed data. */ ++ atomic_set(&stream->autoflush_counter, 0); ++ ++ spin_unlock_irqrestore(&stream->lock, flags); ++} ++ ++/*****************************************************************************/ ++ ++/** ++ * kbasep_tlstream_flush_stream - flush stream ++ * @stype: type of stream to be flushed ++ * ++ * Flush pending data in timeline stream. ++ */ ++static void kbasep_tlstream_flush_stream(enum tl_stream_type stype) ++{ ++ struct tl_stream *stream = tl_stream[stype]; ++ unsigned long flags; ++ unsigned int wb_idx_raw; ++ unsigned int wb_idx; ++ size_t wb_size; ++ size_t min_size = PACKET_HEADER_SIZE; ++ ++ if (stream->numbered) ++ min_size += PACKET_NUMBER_SIZE; ++ ++ spin_lock_irqsave(&stream->lock, flags); ++ ++ wb_idx_raw = atomic_read(&stream->wbi); ++ wb_idx = wb_idx_raw % PACKET_COUNT; ++ wb_size = atomic_read(&stream->buffer[wb_idx].size); ++ ++ if (wb_size > min_size) { ++ wb_size = kbasep_tlstream_msgbuf_submit( ++ stream, wb_idx_raw, wb_size); ++ wb_idx = (wb_idx_raw + 1) % PACKET_COUNT; ++ atomic_set(&stream->buffer[wb_idx].size, wb_size); ++ } ++ spin_unlock_irqrestore(&stream->lock, flags); ++} ++ ++/** ++ * kbasep_tlstream_autoflush_timer_callback - autoflush timer callback ++ * @data: unused ++ * ++ * Timer is executed periodically to check if any of the stream contains ++ * buffer ready to be submitted to user space. ++ */ ++static void kbasep_tlstream_autoflush_timer_callback(struct timer_list *t) ++{ ++ enum tl_stream_type stype; ++ int rcode; ++ ++ for (stype = 0; stype < TL_STREAM_TYPE_COUNT; stype++) { ++ struct tl_stream *stream = tl_stream[stype]; ++ unsigned long flags; ++ unsigned int wb_idx_raw; ++ unsigned int wb_idx; ++ size_t wb_size; ++ size_t min_size = PACKET_HEADER_SIZE; ++ ++ int af_cnt = atomic_read(&stream->autoflush_counter); ++ ++ /* Check if stream contain unflushed data. */ ++ if (0 > af_cnt) ++ continue; ++ ++ /* Check if stream should be flushed now. */ ++ if (af_cnt != atomic_cmpxchg( ++ &stream->autoflush_counter, ++ af_cnt, ++ af_cnt + 1)) ++ continue; ++ if (!af_cnt) ++ continue; ++ ++ /* Autoflush this stream. */ ++ if (stream->numbered) ++ min_size += PACKET_NUMBER_SIZE; ++ ++ spin_lock_irqsave(&stream->lock, flags); ++ ++ wb_idx_raw = atomic_read(&stream->wbi); ++ wb_idx = wb_idx_raw % PACKET_COUNT; ++ wb_size = atomic_read(&stream->buffer[wb_idx].size); ++ ++ if (wb_size > min_size) { ++ wb_size = kbasep_tlstream_msgbuf_submit( ++ stream, wb_idx_raw, wb_size); ++ wb_idx = (wb_idx_raw + 1) % PACKET_COUNT; ++ atomic_set(&stream->buffer[wb_idx].size, ++ wb_size); ++ } ++ spin_unlock_irqrestore(&stream->lock, flags); ++ } ++ ++ if (atomic_read(&autoflush_timer_active)) ++ rcode = mod_timer( ++ &autoflush_timer, ++ jiffies + msecs_to_jiffies(AUTOFLUSH_INTERVAL)); ++ CSTD_UNUSED(rcode); ++} ++ ++/** ++ * kbasep_tlstream_packet_pending - check timeline streams for pending packets ++ * @stype: pointer to variable where stream type will be placed ++ * @rb_idx_raw: pointer to variable where read buffer index will be placed ++ * ++ * Function checks all streams for pending packets. It will stop as soon as ++ * packet ready to be submitted to user space is detected. Variables under ++ * pointers, passed as the parameters to this function will be updated with ++ * values pointing to right stream and buffer. ++ * ++ * Return: non-zero if any of timeline streams has at last one packet ready ++ */ ++static int kbasep_tlstream_packet_pending( ++ enum tl_stream_type *stype, ++ unsigned int *rb_idx_raw) ++{ ++ int pending = 0; ++ ++ KBASE_DEBUG_ASSERT(stype); ++ KBASE_DEBUG_ASSERT(rb_idx_raw); ++ ++ for ( ++ *stype = 0; ++ (*stype < TL_STREAM_TYPE_COUNT) && !pending; ++ (*stype)++) { ++ if (NULL != tl_stream[*stype]) { ++ *rb_idx_raw = atomic_read(&tl_stream[*stype]->rbi); ++ /* Read buffer index may be updated by writer in case of ++ * overflow. Read and write buffer indexes must be ++ * loaded in correct order. */ ++ smp_rmb(); ++ if (atomic_read(&tl_stream[*stype]->wbi) != *rb_idx_raw) ++ pending = 1; ++ } ++ } ++ (*stype)--; ++ ++ return pending; ++} ++ ++/** ++ * kbasep_tlstream_read - copy data from streams to buffer provided by user ++ * @filp: pointer to file structure (unused) ++ * @buffer: pointer to the buffer provided by user ++ * @size: maximum amount of data that can be stored in the buffer ++ * @f_pos: pointer to file offset (unused) ++ * ++ * Return: number of bytes stored in the buffer ++ */ ++static ssize_t kbasep_tlstream_read( ++ struct file *filp, ++ char __user *buffer, ++ size_t size, ++ loff_t *f_pos) ++{ ++ ssize_t copy_len = 0; ++ ++ KBASE_DEBUG_ASSERT(filp); ++ KBASE_DEBUG_ASSERT(f_pos); ++ ++ if (!buffer) ++ return -EINVAL; ++ ++ if ((0 > *f_pos) || (PACKET_SIZE > size)) ++ return -EINVAL; ++ ++ mutex_lock(&tl_reader_lock); ++ ++ while (copy_len < size) { ++ enum tl_stream_type stype; ++ unsigned int rb_idx_raw = 0; ++ unsigned int rb_idx; ++ size_t rb_size; ++ ++ /* If we don't have any data yet, wait for packet to be ++ * submitted. If we already read some packets and there is no ++ * packet pending return back to user. */ ++ if (0 < copy_len) { ++ if (!kbasep_tlstream_packet_pending( ++ &stype, ++ &rb_idx_raw)) ++ break; ++ } else { ++ if (wait_event_interruptible( ++ tl_event_queue, ++ kbasep_tlstream_packet_pending( ++ &stype, ++ &rb_idx_raw))) { ++ copy_len = -ERESTARTSYS; ++ break; ++ } ++ } ++ ++ /* Check if this packet fits into the user buffer. ++ * If so copy its content. */ ++ rb_idx = rb_idx_raw % PACKET_COUNT; ++ rb_size = atomic_read(&tl_stream[stype]->buffer[rb_idx].size); ++ if (rb_size > size - copy_len) ++ break; ++ if (copy_to_user( ++ &buffer[copy_len], ++ tl_stream[stype]->buffer[rb_idx].data, ++ rb_size)) { ++ copy_len = -EFAULT; ++ break; ++ } ++ ++ /* If the rbi still points to the packet we just processed ++ * then there was no overflow so we add the copied size to ++ * copy_len and move rbi on to the next packet ++ */ ++ smp_rmb(); ++ if (atomic_read(&tl_stream[stype]->rbi) == rb_idx_raw) { ++ copy_len += rb_size; ++ atomic_inc(&tl_stream[stype]->rbi); ++ ++#if MALI_UNIT_TEST ++ atomic_add(rb_size, &tlstream_bytes_collected); ++#endif /* MALI_UNIT_TEST */ ++ } ++ } ++ ++ mutex_unlock(&tl_reader_lock); ++ ++ return copy_len; ++} ++ ++/** ++ * kbasep_tlstream_poll - poll timeline stream for packets ++ * @filp: pointer to file structure ++ * @wait: pointer to poll table ++ * Return: POLLIN if data can be read without blocking, otherwise zero ++ */ ++static unsigned int kbasep_tlstream_poll(struct file *filp, poll_table *wait) ++{ ++ enum tl_stream_type stream_type; ++ unsigned int rb_idx; ++ ++ KBASE_DEBUG_ASSERT(filp); ++ KBASE_DEBUG_ASSERT(wait); ++ ++ poll_wait(filp, &tl_event_queue, wait); ++ if (kbasep_tlstream_packet_pending(&stream_type, &rb_idx)) ++ return POLLIN; ++ return 0; ++} ++ ++/** ++ * kbasep_tlstream_release - release timeline stream descriptor ++ * @inode: pointer to inode structure ++ * @filp: pointer to file structure ++ * ++ * Return always return zero ++ */ ++static int kbasep_tlstream_release(struct inode *inode, struct file *filp) ++{ ++ KBASE_DEBUG_ASSERT(inode); ++ KBASE_DEBUG_ASSERT(filp); ++ CSTD_UNUSED(inode); ++ CSTD_UNUSED(filp); ++ ++ /* Stop autoflush timer before releasing access to streams. */ ++ atomic_set(&autoflush_timer_active, 0); ++ del_timer_sync(&autoflush_timer); ++ ++ atomic_set(&kbase_tlstream_enabled, 0); ++ return 0; ++} ++ ++/** ++ * kbasep_tlstream_timeline_header - prepare timeline header stream packet ++ * @stream_type: type of the stream that will carry header data ++ * @tp_desc: pointer to array with tracepoint descriptors ++ * @tp_count: number of descriptors in the given array ++ * ++ * Functions fills in information about tracepoints stored in body stream ++ * associated with this header stream. ++ */ ++static void kbasep_tlstream_timeline_header( ++ enum tl_stream_type stream_type, ++ const struct tp_desc *tp_desc, ++ u32 tp_count) ++{ ++ const u8 tv = SWTRACE_VERSION; /* protocol version */ ++ const u8 ps = sizeof(void *); /* pointer size */ ++ size_t msg_size = sizeof(tv) + sizeof(ps) + sizeof(tp_count); ++ char *buffer; ++ size_t pos = 0; ++ unsigned long flags; ++ unsigned int i; ++ ++ KBASE_DEBUG_ASSERT(TL_STREAM_TYPE_COUNT > stream_type); ++ KBASE_DEBUG_ASSERT(tp_desc); ++ ++ /* Calculate the size of the timeline message. */ ++ for (i = 0; i < tp_count; i++) { ++ msg_size += sizeof(tp_desc[i].id); ++ msg_size += ++ strnlen(tp_desc[i].id_str, STRLEN_MAX) + ++ sizeof(char) + sizeof(u32); ++ msg_size += ++ strnlen(tp_desc[i].name, STRLEN_MAX) + ++ sizeof(char) + sizeof(u32); ++ msg_size += ++ strnlen(tp_desc[i].arg_types, STRLEN_MAX) + ++ sizeof(char) + sizeof(u32); ++ msg_size += ++ strnlen(tp_desc[i].arg_names, STRLEN_MAX) + ++ sizeof(char) + sizeof(u32); ++ } ++ ++ KBASE_DEBUG_ASSERT(PACKET_SIZE - PACKET_HEADER_SIZE >= msg_size); ++ ++ buffer = kbasep_tlstream_msgbuf_acquire(stream_type, msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &tv, sizeof(tv)); ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &ps, sizeof(ps)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &tp_count, sizeof(tp_count)); ++ ++ for (i = 0; i < tp_count; i++) { ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, ++ &tp_desc[i].id, sizeof(tp_desc[i].id)); ++ pos = kbasep_tlstream_write_string( ++ buffer, pos, ++ tp_desc[i].id_str, msg_size - pos); ++ pos = kbasep_tlstream_write_string( ++ buffer, pos, ++ tp_desc[i].name, msg_size - pos); ++ pos = kbasep_tlstream_write_string( ++ buffer, pos, ++ tp_desc[i].arg_types, msg_size - pos); ++ pos = kbasep_tlstream_write_string( ++ buffer, pos, ++ tp_desc[i].arg_names, msg_size - pos); ++ } ++ ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(stream_type, flags); ++ ++ /* We don't expect any more data to be read in this stream. ++ * As header stream must be read before its associated body stream, ++ * make this packet visible to the user straightaway. */ ++ kbasep_tlstream_flush_stream(stream_type); ++} ++ ++/*****************************************************************************/ ++ ++int kbase_tlstream_init(void) ++{ ++ enum tl_stream_type i; ++ ++ /* Prepare stream structures. */ ++ for (i = 0; i < TL_STREAM_TYPE_COUNT; i++) { ++ tl_stream[i] = kmalloc(sizeof(**tl_stream), GFP_KERNEL); ++ if (!tl_stream[i]) ++ break; ++ kbasep_timeline_stream_init(tl_stream[i], i); ++ } ++ if (TL_STREAM_TYPE_COUNT > i) { ++ for (; i > 0; i--) { ++ kbasep_timeline_stream_term(tl_stream[i - 1]); ++ kfree(tl_stream[i - 1]); ++ } ++ return -ENOMEM; ++ } ++ ++ /* Initialize autoflush timer. */ ++ timer_setup(&autoflush_timer, ++ kbasep_tlstream_autoflush_timer_callback, ++ 0); ++ ++ return 0; ++} ++ ++void kbase_tlstream_term(void) ++{ ++ enum tl_stream_type i; ++ ++ for (i = 0; i < TL_STREAM_TYPE_COUNT; i++) { ++ kbasep_timeline_stream_term(tl_stream[i]); ++ kfree(tl_stream[i]); ++ } ++} ++ ++static void kbase_create_timeline_objects(struct kbase_context *kctx) ++{ ++ struct kbase_device *kbdev = kctx->kbdev; ++ unsigned int lpu_id; ++ unsigned int as_nr; ++ struct kbasep_kctx_list_element *element; ++ ++ /* Create LPU objects. */ ++ for (lpu_id = 0; lpu_id < kbdev->gpu_props.num_job_slots; lpu_id++) { ++ u32 *lpu = ++ &kbdev->gpu_props.props.raw_props.js_features[lpu_id]; ++ KBASE_TLSTREAM_TL_SUMMARY_NEW_LPU(lpu, lpu_id, *lpu); ++ } ++ ++ /* Create Address Space objects. */ ++ for (as_nr = 0; as_nr < kbdev->nr_hw_address_spaces; as_nr++) ++ KBASE_TLSTREAM_TL_SUMMARY_NEW_AS(&kbdev->as[as_nr], as_nr); ++ ++ /* Create GPU object and make it retain all LPUs and address spaces. */ ++ KBASE_TLSTREAM_TL_SUMMARY_NEW_GPU( ++ kbdev, ++ kbdev->gpu_props.props.raw_props.gpu_id, ++ kbdev->gpu_props.num_cores); ++ ++ for (lpu_id = 0; lpu_id < kbdev->gpu_props.num_job_slots; lpu_id++) { ++ void *lpu = ++ &kbdev->gpu_props.props.raw_props.js_features[lpu_id]; ++ KBASE_TLSTREAM_TL_SUMMARY_LIFELINK_LPU_GPU(lpu, kbdev); ++ } ++ for (as_nr = 0; as_nr < kbdev->nr_hw_address_spaces; as_nr++) ++ KBASE_TLSTREAM_TL_SUMMARY_LIFELINK_AS_GPU( ++ &kbdev->as[as_nr], ++ kbdev); ++ ++ /* Create object for each known context. */ ++ mutex_lock(&kbdev->kctx_list_lock); ++ list_for_each_entry(element, &kbdev->kctx_list, link) { ++ KBASE_TLSTREAM_TL_SUMMARY_NEW_CTX( ++ element->kctx, ++ element->kctx->id, ++ (u32)(element->kctx->tgid)); ++ } ++ /* Before releasing the lock, reset body stream buffers. ++ * This will prevent context creation message to be directed to both ++ * summary and body stream. ++ */ ++ kbase_tlstream_reset_body_streams(); ++ mutex_unlock(&kbdev->kctx_list_lock); ++ /* Static object are placed into summary packet that needs to be ++ * transmitted first. Flush all streams to make it available to ++ * user space. ++ */ ++ kbase_tlstream_flush_streams(); ++} ++ ++int kbase_tlstream_acquire(struct kbase_context *kctx, u32 flags) ++{ ++ int ret; ++ u32 tlstream_enabled = TLSTREAM_ENABLED | flags; ++ ++ if (0 == atomic_cmpxchg(&kbase_tlstream_enabled, 0, tlstream_enabled)) { ++ int rcode; ++ ++ ret = anon_inode_getfd( ++ "[mali_tlstream]", ++ &kbasep_tlstream_fops, ++ kctx, ++ O_RDONLY | O_CLOEXEC); ++ if (ret < 0) { ++ atomic_set(&kbase_tlstream_enabled, 0); ++ return ret; ++ } ++ ++ /* Reset and initialize header streams. */ ++ kbasep_timeline_stream_reset( ++ tl_stream[TL_STREAM_TYPE_OBJ_HEADER]); ++ kbasep_timeline_stream_reset( ++ tl_stream[TL_STREAM_TYPE_OBJ_SUMMARY]); ++ kbasep_timeline_stream_reset( ++ tl_stream[TL_STREAM_TYPE_AUX_HEADER]); ++ kbasep_tlstream_timeline_header( ++ TL_STREAM_TYPE_OBJ_HEADER, ++ tp_desc_obj, ++ ARRAY_SIZE(tp_desc_obj)); ++ kbasep_tlstream_timeline_header( ++ TL_STREAM_TYPE_AUX_HEADER, ++ tp_desc_aux, ++ ARRAY_SIZE(tp_desc_aux)); ++ ++ /* Start autoflush timer. */ ++ atomic_set(&autoflush_timer_active, 1); ++ rcode = mod_timer( ++ &autoflush_timer, ++ jiffies + msecs_to_jiffies(AUTOFLUSH_INTERVAL)); ++ CSTD_UNUSED(rcode); ++ ++ /* If job dumping is enabled, readjust the software event's ++ * timeout as the default value of 3 seconds is often ++ * insufficient. */ ++ if (flags & BASE_TLSTREAM_JOB_DUMPING_ENABLED) { ++ dev_info(kctx->kbdev->dev, ++ "Job dumping is enabled, readjusting the software event's timeout\n"); ++ atomic_set(&kctx->kbdev->js_data.soft_job_timeout_ms, ++ 1800000); ++ } ++ ++ /* Summary stream was cleared during acquire. ++ * Create static timeline objects that will be ++ * read by client. ++ */ ++ kbase_create_timeline_objects(kctx); ++ ++ } else { ++ ret = -EBUSY; ++ } ++ ++ return ret; ++} ++ ++void kbase_tlstream_flush_streams(void) ++{ ++ enum tl_stream_type stype; ++ ++ for (stype = 0; stype < TL_STREAM_TYPE_COUNT; stype++) ++ kbasep_tlstream_flush_stream(stype); ++} ++ ++void kbase_tlstream_reset_body_streams(void) ++{ ++ kbasep_timeline_stream_reset( ++ tl_stream[TL_STREAM_TYPE_OBJ]); ++ kbasep_timeline_stream_reset( ++ tl_stream[TL_STREAM_TYPE_AUX]); ++} ++ ++#if MALI_UNIT_TEST ++void kbase_tlstream_stats(u32 *bytes_collected, u32 *bytes_generated) ++{ ++ KBASE_DEBUG_ASSERT(bytes_collected); ++ KBASE_DEBUG_ASSERT(bytes_generated); ++ *bytes_collected = atomic_read(&tlstream_bytes_collected); ++ *bytes_generated = atomic_read(&tlstream_bytes_generated); ++} ++#endif /* MALI_UNIT_TEST */ ++ ++/*****************************************************************************/ ++ ++void __kbase_tlstream_tl_summary_new_ctx(void *context, u32 nr, u32 tgid) ++{ ++ const u32 msg_id = KBASE_TL_NEW_CTX; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(context) + sizeof(nr) + ++ sizeof(tgid); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ_SUMMARY, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &context, sizeof(context)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &nr, sizeof(nr)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &tgid, sizeof(tgid)); ++ ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags); ++} ++ ++void __kbase_tlstream_tl_summary_new_gpu(void *gpu, u32 id, u32 core_count) ++{ ++ const u32 msg_id = KBASE_TL_NEW_GPU; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(gpu) + sizeof(id) + ++ sizeof(core_count); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ_SUMMARY, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &gpu, sizeof(gpu)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &id, sizeof(id)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &core_count, sizeof(core_count)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags); ++} ++ ++void __kbase_tlstream_tl_summary_new_lpu(void *lpu, u32 nr, u32 fn) ++{ ++ const u32 msg_id = KBASE_TL_NEW_LPU; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(lpu) + sizeof(nr) + ++ sizeof(fn); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ_SUMMARY, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &lpu, sizeof(lpu)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &nr, sizeof(nr)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &fn, sizeof(fn)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags); ++} ++ ++void __kbase_tlstream_tl_summary_lifelink_lpu_gpu(void *lpu, void *gpu) ++{ ++ const u32 msg_id = KBASE_TL_LIFELINK_LPU_GPU; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(lpu) + sizeof(gpu); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ_SUMMARY, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &lpu, sizeof(lpu)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &gpu, sizeof(gpu)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags); ++} ++ ++void __kbase_tlstream_tl_summary_new_as(void *as, u32 nr) ++{ ++ const u32 msg_id = KBASE_TL_NEW_AS; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(as) + sizeof(nr); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ_SUMMARY, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &as, sizeof(as)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &nr, sizeof(nr)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags); ++} ++ ++void __kbase_tlstream_tl_summary_lifelink_as_gpu(void *as, void *gpu) ++{ ++ const u32 msg_id = KBASE_TL_LIFELINK_AS_GPU; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(as) + sizeof(gpu); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ_SUMMARY, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &as, sizeof(as)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &gpu, sizeof(gpu)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags); ++} ++ ++/*****************************************************************************/ ++ ++void __kbase_tlstream_tl_new_ctx(void *context, u32 nr, u32 tgid) ++{ ++ const u32 msg_id = KBASE_TL_NEW_CTX; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(context) + sizeof(nr) + ++ sizeof(tgid); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &context, sizeof(context)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &nr, sizeof(nr)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &tgid, sizeof(tgid)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_new_atom(void *atom, u32 nr) ++{ ++ const u32 msg_id = KBASE_TL_NEW_ATOM; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) + sizeof(atom) + ++ sizeof(nr); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &atom, sizeof(atom)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &nr, sizeof(nr)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_del_ctx(void *context) ++{ ++ const u32 msg_id = KBASE_TL_DEL_CTX; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(context); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &context, sizeof(context)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_del_atom(void *atom) ++{ ++ const u32 msg_id = KBASE_TL_DEL_ATOM; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(atom); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &atom, sizeof(atom)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_ret_ctx_lpu(void *context, void *lpu) ++{ ++ const u32 msg_id = KBASE_TL_RET_CTX_LPU; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(context) + sizeof(lpu); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &context, sizeof(context)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &lpu, sizeof(lpu)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_ret_atom_ctx(void *atom, void *context) ++{ ++ const u32 msg_id = KBASE_TL_RET_ATOM_CTX; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(context); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &atom, sizeof(atom)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &context, sizeof(context)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_ret_atom_lpu( ++ void *atom, void *lpu, const char *attrib_match_list) ++{ ++ const u32 msg_id = KBASE_TL_RET_ATOM_LPU; ++ const size_t msg_s0 = sizeof(u32) + sizeof(char) + ++ strnlen(attrib_match_list, STRLEN_MAX); ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + ++ sizeof(atom) + sizeof(lpu) + msg_s0; ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &atom, sizeof(atom)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &lpu, sizeof(lpu)); ++ pos = kbasep_tlstream_write_string( ++ buffer, pos, attrib_match_list, msg_s0); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_nret_ctx_lpu(void *context, void *lpu) ++{ ++ const u32 msg_id = KBASE_TL_NRET_CTX_LPU; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(context) + sizeof(lpu); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &context, sizeof(context)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &lpu, sizeof(lpu)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_nret_atom_ctx(void *atom, void *context) ++{ ++ const u32 msg_id = KBASE_TL_NRET_ATOM_CTX; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(context); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &atom, sizeof(atom)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &context, sizeof(context)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_dep_atom_atom(void *atom1, void *atom2) ++{ ++ const u32 msg_id = KBASE_TL_DEP_ATOM_ATOM; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(atom1) + sizeof(atom2); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &atom1, sizeof(atom1)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &atom2, sizeof(atom2)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_ndep_atom_atom(void *atom1, void *atom2) ++{ ++ const u32 msg_id = KBASE_TL_NDEP_ATOM_ATOM; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(atom1) + sizeof(atom2); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &atom1, sizeof(atom1)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &atom2, sizeof(atom2)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_rdep_atom_atom(void *atom1, void *atom2) ++{ ++ const u32 msg_id = KBASE_TL_RDEP_ATOM_ATOM; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(atom1) + sizeof(atom2); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &atom1, sizeof(atom1)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &atom2, sizeof(atom2)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_nret_atom_lpu(void *atom, void *lpu) ++{ ++ const u32 msg_id = KBASE_TL_NRET_ATOM_LPU; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(lpu); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &atom, sizeof(atom)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &lpu, sizeof(lpu)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_ret_as_ctx(void *as, void *ctx) ++{ ++ const u32 msg_id = KBASE_TL_RET_AS_CTX; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(as) + sizeof(ctx); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &as, sizeof(as)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &ctx, sizeof(ctx)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_nret_as_ctx(void *as, void *ctx) ++{ ++ const u32 msg_id = KBASE_TL_NRET_AS_CTX; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(as) + sizeof(ctx); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &as, sizeof(as)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &ctx, sizeof(ctx)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_ret_atom_as(void *atom, void *as) ++{ ++ const u32 msg_id = KBASE_TL_RET_ATOM_AS; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(as); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &atom, sizeof(atom)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &as, sizeof(as)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_nret_atom_as(void *atom, void *as) ++{ ++ const u32 msg_id = KBASE_TL_NRET_ATOM_AS; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(as); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &atom, sizeof(atom)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &as, sizeof(as)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_attrib_atom_config( ++ void *atom, u64 jd, u64 affinity, u32 config) ++{ ++ const u32 msg_id = KBASE_TL_ATTRIB_ATOM_CONFIG; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(atom) + ++ sizeof(jd) + sizeof(affinity) + sizeof(config); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &atom, sizeof(atom)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &jd, sizeof(jd)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &affinity, sizeof(affinity)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &config, sizeof(config)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_attrib_atom_priority(void *atom, u32 prio) ++{ ++ const u32 msg_id = KBASE_TL_ATTRIB_ATOM_PRIORITY; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(prio); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &atom, sizeof(atom)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &prio, sizeof(prio)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_attrib_atom_state(void *atom, u32 state) ++{ ++ const u32 msg_id = KBASE_TL_ATTRIB_ATOM_STATE; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(state); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &atom, sizeof(atom)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &state, sizeof(state)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_attrib_atom_priority_change(void *atom) ++{ ++ const u32 msg_id = KBASE_TL_ATTRIB_ATOM_PRIORITY_CHANGE; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(atom); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &atom, sizeof(atom)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_attrib_atom_jit( ++ void *atom, u64 edit_addr, u64 new_addr) ++{ ++ const u32 msg_id = KBASE_TL_ATTRIB_ATOM_JIT; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(atom) ++ + sizeof(edit_addr) + sizeof(new_addr); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &atom, sizeof(atom)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &edit_addr, sizeof(edit_addr)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &new_addr, sizeof(new_addr)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_attrib_as_config( ++ void *as, u64 transtab, u64 memattr, u64 transcfg) ++{ ++ const u32 msg_id = KBASE_TL_ATTRIB_AS_CONFIG; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(as) + ++ sizeof(transtab) + sizeof(memattr) + sizeof(transcfg); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &as, sizeof(as)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &transtab, sizeof(transtab)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &memattr, sizeof(memattr)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &transcfg, sizeof(transcfg)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_event_lpu_softstop(void *lpu) ++{ ++ const u32 msg_id = KBASE_TL_EVENT_LPU_SOFTSTOP; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(lpu); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &lpu, sizeof(lpu)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_event_atom_softstop_ex(void *atom) ++{ ++ const u32 msg_id = KBASE_TL_EVENT_ATOM_SOFTSTOP_EX; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(atom); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &atom, sizeof(atom)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_event_atom_softstop_issue(void *atom) ++{ ++ const u32 msg_id = KBASE_TL_EVENT_ATOM_SOFTSTOP_ISSUE; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(atom); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &atom, sizeof(atom)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_jd_gpu_soft_reset(void *gpu) ++{ ++ const u32 msg_id = KBASE_JD_GPU_SOFT_RESET; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(gpu); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &gpu, sizeof(gpu)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++/*****************************************************************************/ ++ ++void __kbase_tlstream_aux_pm_state(u32 core_type, u64 state) ++{ ++ const u32 msg_id = KBASE_AUX_PM_STATE; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(core_type) + ++ sizeof(state); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_AUX, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &core_type, sizeof(core_type)); ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &state, sizeof(state)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags); ++} ++ ++void __kbase_tlstream_aux_pagefault(u32 ctx_nr, u64 page_count_change) ++{ ++ const u32 msg_id = KBASE_AUX_PAGEFAULT; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(ctx_nr) + ++ sizeof(page_count_change); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_AUX, msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &ctx_nr, sizeof(ctx_nr)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, ++ &page_count_change, sizeof(page_count_change)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags); ++} ++ ++void __kbase_tlstream_aux_pagesalloc(u32 ctx_nr, u64 page_count) ++{ ++ const u32 msg_id = KBASE_AUX_PAGESALLOC; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(ctx_nr) + ++ sizeof(page_count); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_AUX, msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &ctx_nr, sizeof(ctx_nr)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &page_count, sizeof(page_count)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags); ++} ++ ++void __kbase_tlstream_aux_devfreq_target(u64 target_freq) ++{ ++ const u32 msg_id = KBASE_AUX_DEVFREQ_TARGET; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(target_freq); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_AUX, msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &target_freq, sizeof(target_freq)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags); ++} ++ ++void __kbase_tlstream_aux_protected_enter_start(void *gpu) ++{ ++ const u32 msg_id = KBASE_AUX_PROTECTED_ENTER_START; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(gpu); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_AUX, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &gpu, sizeof(gpu)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags); ++} ++void __kbase_tlstream_aux_protected_enter_end(void *gpu) ++{ ++ const u32 msg_id = KBASE_AUX_PROTECTED_ENTER_END; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(gpu); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_AUX, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &gpu, sizeof(gpu)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags); ++} ++ ++void __kbase_tlstream_aux_protected_leave_start(void *gpu) ++{ ++ const u32 msg_id = KBASE_AUX_PROTECTED_LEAVE_START; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(gpu); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_AUX, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &gpu, sizeof(gpu)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags); ++} ++void __kbase_tlstream_aux_protected_leave_end(void *gpu) ++{ ++ const u32 msg_id = KBASE_AUX_PROTECTED_LEAVE_END; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(gpu); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_AUX, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &gpu, sizeof(gpu)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags); ++} +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_tlstream.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_tlstream.h +new file mode 100755 +index 000000000000..c0a1117d5f25 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_tlstream.h +@@ -0,0 +1,623 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#if !defined(_KBASE_TLSTREAM_H) ++#define _KBASE_TLSTREAM_H ++ ++#include ++ ++/*****************************************************************************/ ++ ++/** ++ * kbase_tlstream_init - initialize timeline infrastructure in kernel ++ * Return: zero on success, negative number on error ++ */ ++int kbase_tlstream_init(void); ++ ++/** ++ * kbase_tlstream_term - terminate timeline infrastructure in kernel ++ * ++ * Timeline need have to been previously enabled with kbase_tlstream_init(). ++ */ ++void kbase_tlstream_term(void); ++ ++/** ++ * kbase_tlstream_acquire - acquire timeline stream file descriptor ++ * @kctx: kernel common context ++ * @flags: timeline stream flags ++ * ++ * This descriptor is meant to be used by userspace timeline to gain access to ++ * kernel timeline stream. This stream is later broadcasted by user space to the ++ * timeline client. ++ * Only one entity can own the descriptor at any given time. Descriptor shall be ++ * closed if unused. If descriptor cannot be obtained (i.e. when it is already ++ * being used) return will be a negative value. ++ * ++ * Return: file descriptor on success, negative number on error ++ */ ++int kbase_tlstream_acquire(struct kbase_context *kctx, u32 flags); ++ ++/** ++ * kbase_tlstream_flush_streams - flush timeline streams. ++ * ++ * Function will flush pending data in all timeline streams. ++ */ ++void kbase_tlstream_flush_streams(void); ++ ++/** ++ * kbase_tlstream_reset_body_streams - reset timeline body streams. ++ * ++ * Function will discard pending data in all timeline body streams. ++ */ ++void kbase_tlstream_reset_body_streams(void); ++ ++#if MALI_UNIT_TEST ++/** ++ * kbase_tlstream_test - start timeline stream data generator ++ * @tpw_count: number of trace point writers in each context ++ * @msg_delay: time delay in milliseconds between trace points written by one ++ * writer ++ * @msg_count: number of trace points written by one writer ++ * @aux_msg: if non-zero aux messages will be included ++ * ++ * This test starts a requested number of asynchronous writers in both IRQ and ++ * thread context. Each writer will generate required number of test ++ * tracepoints (tracepoints with embedded information about writer that ++ * should be verified by user space reader). Tracepoints will be emitted in ++ * all timeline body streams. If aux_msg is non-zero writer will also ++ * generate not testable tracepoints (tracepoints without information about ++ * writer). These tracepoints are used to check correctness of remaining ++ * timeline message generating functions. Writer will wait requested time ++ * between generating another set of messages. This call blocks until all ++ * writers finish. ++ */ ++void kbase_tlstream_test( ++ unsigned int tpw_count, ++ unsigned int msg_delay, ++ unsigned int msg_count, ++ int aux_msg); ++ ++/** ++ * kbase_tlstream_stats - read timeline stream statistics ++ * @bytes_collected: will hold number of bytes read by the user ++ * @bytes_generated: will hold number of bytes generated by trace points ++ */ ++void kbase_tlstream_stats(u32 *bytes_collected, u32 *bytes_generated); ++#endif /* MALI_UNIT_TEST */ ++ ++/*****************************************************************************/ ++ ++#define TL_ATOM_STATE_IDLE 0 ++#define TL_ATOM_STATE_READY 1 ++#define TL_ATOM_STATE_DONE 2 ++#define TL_ATOM_STATE_POSTED 3 ++ ++void __kbase_tlstream_tl_summary_new_ctx(void *context, u32 nr, u32 tgid); ++void __kbase_tlstream_tl_summary_new_gpu(void *gpu, u32 id, u32 core_count); ++void __kbase_tlstream_tl_summary_new_lpu(void *lpu, u32 nr, u32 fn); ++void __kbase_tlstream_tl_summary_lifelink_lpu_gpu(void *lpu, void *gpu); ++void __kbase_tlstream_tl_summary_new_as(void *as, u32 nr); ++void __kbase_tlstream_tl_summary_lifelink_as_gpu(void *as, void *gpu); ++void __kbase_tlstream_tl_new_ctx(void *context, u32 nr, u32 tgid); ++void __kbase_tlstream_tl_new_atom(void *atom, u32 nr); ++void __kbase_tlstream_tl_del_ctx(void *context); ++void __kbase_tlstream_tl_del_atom(void *atom); ++void __kbase_tlstream_tl_ret_ctx_lpu(void *context, void *lpu); ++void __kbase_tlstream_tl_ret_atom_ctx(void *atom, void *context); ++void __kbase_tlstream_tl_ret_atom_lpu( ++ void *atom, void *lpu, const char *attrib_match_list); ++void __kbase_tlstream_tl_nret_ctx_lpu(void *context, void *lpu); ++void __kbase_tlstream_tl_nret_atom_ctx(void *atom, void *context); ++void __kbase_tlstream_tl_nret_atom_lpu(void *atom, void *lpu); ++void __kbase_tlstream_tl_ret_as_ctx(void *as, void *ctx); ++void __kbase_tlstream_tl_nret_as_ctx(void *as, void *ctx); ++void __kbase_tlstream_tl_ret_atom_as(void *atom, void *as); ++void __kbase_tlstream_tl_nret_atom_as(void *atom, void *as); ++void __kbase_tlstream_tl_dep_atom_atom(void *atom1, void *atom2); ++void __kbase_tlstream_tl_ndep_atom_atom(void *atom1, void *atom2); ++void __kbase_tlstream_tl_rdep_atom_atom(void *atom1, void *atom2); ++void __kbase_tlstream_tl_attrib_atom_config( ++ void *atom, u64 jd, u64 affinity, u32 config); ++void __kbase_tlstream_tl_attrib_atom_priority(void *atom, u32 prio); ++void __kbase_tlstream_tl_attrib_atom_state(void *atom, u32 state); ++void __kbase_tlstream_tl_attrib_atom_priority_change(void *atom); ++void __kbase_tlstream_tl_attrib_atom_jit( ++ void *atom, u64 edit_addr, u64 new_addr); ++void __kbase_tlstream_tl_attrib_as_config( ++ void *as, u64 transtab, u64 memattr, u64 transcfg); ++void __kbase_tlstream_tl_event_atom_softstop_ex(void *atom); ++void __kbase_tlstream_tl_event_lpu_softstop(void *lpu); ++void __kbase_tlstream_tl_event_atom_softstop_issue(void *atom); ++void __kbase_tlstream_jd_gpu_soft_reset(void *gpu); ++void __kbase_tlstream_aux_pm_state(u32 core_type, u64 state); ++void __kbase_tlstream_aux_pagefault(u32 ctx_nr, u64 page_count_change); ++void __kbase_tlstream_aux_pagesalloc(u32 ctx_nr, u64 page_count); ++void __kbase_tlstream_aux_devfreq_target(u64 target_freq); ++void __kbase_tlstream_aux_protected_enter_start(void *gpu); ++void __kbase_tlstream_aux_protected_enter_end(void *gpu); ++void __kbase_tlstream_aux_protected_leave_start(void *gpu); ++void __kbase_tlstream_aux_protected_leave_end(void *gpu); ++ ++#define TLSTREAM_ENABLED (1 << 31) ++ ++extern atomic_t kbase_tlstream_enabled; ++ ++#define __TRACE_IF_ENABLED(trace_name, ...) \ ++ do { \ ++ int enabled = atomic_read(&kbase_tlstream_enabled); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_##trace_name(__VA_ARGS__); \ ++ } while (0) ++ ++#define __TRACE_IF_ENABLED_LATENCY(trace_name, ...) \ ++ do { \ ++ int enabled = atomic_read(&kbase_tlstream_enabled); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS) \ ++ __kbase_tlstream_##trace_name(__VA_ARGS__); \ ++ } while (0) ++ ++#define __TRACE_IF_ENABLED_JD(trace_name, ...) \ ++ do { \ ++ int enabled = atomic_read(&kbase_tlstream_enabled); \ ++ if (enabled & BASE_TLSTREAM_JOB_DUMPING_ENABLED) \ ++ __kbase_tlstream_##trace_name(__VA_ARGS__); \ ++ } while (0) ++ ++/*****************************************************************************/ ++ ++/** ++ * KBASE_TLSTREAM_TL_SUMMARY_NEW_CTX - create context object in timeline ++ * summary ++ * @context: name of the context object ++ * @nr: context number ++ * @tgid: thread Group Id ++ * ++ * Function emits a timeline message informing about context creation. Context ++ * is created with context number (its attribute), that can be used to link ++ * kbase context with userspace context. ++ * This message is directed to timeline summary stream. ++ */ ++#define KBASE_TLSTREAM_TL_SUMMARY_NEW_CTX(context, nr, tgid) \ ++ __TRACE_IF_ENABLED(tl_summary_new_ctx, context, nr, tgid) ++ ++/** ++ * KBASE_TLSTREAM_TL_SUMMARY_NEW_GPU - create GPU object in timeline summary ++ * @gpu: name of the GPU object ++ * @id: id value of this GPU ++ * @core_count: number of cores this GPU hosts ++ * ++ * Function emits a timeline message informing about GPU creation. GPU is ++ * created with two attributes: id and core count. ++ * This message is directed to timeline summary stream. ++ */ ++#define KBASE_TLSTREAM_TL_SUMMARY_NEW_GPU(gpu, id, core_count) \ ++ __TRACE_IF_ENABLED(tl_summary_new_gpu, gpu, id, core_count) ++ ++/** ++ * KBASE_TLSTREAM_TL_SUMMARY_NEW_LPU - create LPU object in timeline summary ++ * @lpu: name of the Logical Processing Unit object ++ * @nr: sequential number assigned to this LPU ++ * @fn: property describing this LPU's functional abilities ++ * ++ * Function emits a timeline message informing about LPU creation. LPU is ++ * created with two attributes: number linking this LPU with GPU's job slot ++ * and function bearing information about this LPU abilities. ++ * This message is directed to timeline summary stream. ++ */ ++#define KBASE_TLSTREAM_TL_SUMMARY_NEW_LPU(lpu, nr, fn) \ ++ __TRACE_IF_ENABLED(tl_summary_new_lpu, lpu, nr, fn) ++ ++/** ++ * KBASE_TLSTREAM_TL_SUMMARY_LIFELINK_LPU_GPU - lifelink LPU object to GPU ++ * @lpu: name of the Logical Processing Unit object ++ * @gpu: name of the GPU object ++ * ++ * Function emits a timeline message informing that LPU object shall be deleted ++ * along with GPU object. ++ * This message is directed to timeline summary stream. ++ */ ++#define KBASE_TLSTREAM_TL_SUMMARY_LIFELINK_LPU_GPU(lpu, gpu) \ ++ __TRACE_IF_ENABLED(tl_summary_lifelink_lpu_gpu, lpu, gpu) ++ ++/** ++ * KBASE_TLSTREAM_TL_SUMMARY_NEW_AS - create address space object in timeline summary ++ * @as: name of the address space object ++ * @nr: sequential number assigned to this address space ++ * ++ * Function emits a timeline message informing about address space creation. ++ * Address space is created with one attribute: number identifying this ++ * address space. ++ * This message is directed to timeline summary stream. ++ */ ++#define KBASE_TLSTREAM_TL_SUMMARY_NEW_AS(as, nr) \ ++ __TRACE_IF_ENABLED(tl_summary_new_as, as, nr) ++ ++/** ++ * KBASE_TLSTREAM_TL_SUMMARY_LIFELINK_AS_GPU - lifelink address space object to GPU ++ * @as: name of the address space object ++ * @gpu: name of the GPU object ++ * ++ * Function emits a timeline message informing that address space object ++ * shall be deleted along with GPU object. ++ * This message is directed to timeline summary stream. ++ */ ++#define KBASE_TLSTREAM_TL_SUMMARY_LIFELINK_AS_GPU(as, gpu) \ ++ __TRACE_IF_ENABLED(tl_summary_lifelink_as_gpu, as, gpu) ++ ++/** ++ * KBASE_TLSTREAM_TL_NEW_CTX - create context object in timeline ++ * @context: name of the context object ++ * @nr: context number ++ * @tgid: thread Group Id ++ * ++ * Function emits a timeline message informing about context creation. Context ++ * is created with context number (its attribute), that can be used to link ++ * kbase context with userspace context. ++ */ ++#define KBASE_TLSTREAM_TL_NEW_CTX(context, nr, tgid) \ ++ __TRACE_IF_ENABLED(tl_new_ctx, context, nr, tgid) ++ ++/** ++ * KBASE_TLSTREAM_TL_NEW_ATOM - create atom object in timeline ++ * @atom: name of the atom object ++ * @nr: sequential number assigned to this atom ++ * ++ * Function emits a timeline message informing about atom creation. Atom is ++ * created with atom number (its attribute) that links it with actual work ++ * bucket id understood by hardware. ++ */ ++#define KBASE_TLSTREAM_TL_NEW_ATOM(atom, nr) \ ++ __TRACE_IF_ENABLED(tl_new_atom, atom, nr) ++ ++/** ++ * KBASE_TLSTREAM_TL_DEL_CTX - destroy context object in timeline ++ * @context: name of the context object ++ * ++ * Function emits a timeline message informing that context object ceased to ++ * exist. ++ */ ++#define KBASE_TLSTREAM_TL_DEL_CTX(context) \ ++ __TRACE_IF_ENABLED(tl_del_ctx, context) ++ ++/** ++ * KBASE_TLSTREAM_TL_DEL_ATOM - destroy atom object in timeline ++ * @atom: name of the atom object ++ * ++ * Function emits a timeline message informing that atom object ceased to ++ * exist. ++ */ ++#define KBASE_TLSTREAM_TL_DEL_ATOM(atom) \ ++ __TRACE_IF_ENABLED(tl_del_atom, atom) ++ ++/** ++ * KBASE_TLSTREAM_TL_RET_CTX_LPU - retain context by LPU ++ * @context: name of the context object ++ * @lpu: name of the Logical Processing Unit object ++ * ++ * Function emits a timeline message informing that context is being held ++ * by LPU and must not be deleted unless it is released. ++ */ ++#define KBASE_TLSTREAM_TL_RET_CTX_LPU(context, lpu) \ ++ __TRACE_IF_ENABLED(tl_ret_ctx_lpu, context, lpu) ++ ++/** ++ * KBASE_TLSTREAM_TL_RET_ATOM_CTX - retain atom by context ++ * @atom: name of the atom object ++ * @context: name of the context object ++ * ++ * Function emits a timeline message informing that atom object is being held ++ * by context and must not be deleted unless it is released. ++ */ ++#define KBASE_TLSTREAM_TL_RET_ATOM_CTX(atom, context) \ ++ __TRACE_IF_ENABLED(tl_ret_atom_ctx, atom, context) ++ ++/** ++ * KBASE_TLSTREAM_TL_RET_ATOM_LPU - retain atom by LPU ++ * @atom: name of the atom object ++ * @lpu: name of the Logical Processing Unit object ++ * @attrib_match_list: list containing match operator attributes ++ * ++ * Function emits a timeline message informing that atom object is being held ++ * by LPU and must not be deleted unless it is released. ++ */ ++#define KBASE_TLSTREAM_TL_RET_ATOM_LPU(atom, lpu, attrib_match_list) \ ++ __TRACE_IF_ENABLED(tl_ret_atom_lpu, atom, lpu, attrib_match_list) ++ ++/** ++ * KBASE_TLSTREAM_TL_NRET_CTX_LPU - release context by LPU ++ * @context: name of the context object ++ * @lpu: name of the Logical Processing Unit object ++ * ++ * Function emits a timeline message informing that context is being released ++ * by LPU object. ++ */ ++#define KBASE_TLSTREAM_TL_NRET_CTX_LPU(context, lpu) \ ++ __TRACE_IF_ENABLED(tl_nret_ctx_lpu, context, lpu) ++ ++/** ++ * KBASE_TLSTREAM_TL_NRET_ATOM_CTX - release atom by context ++ * @atom: name of the atom object ++ * @context: name of the context object ++ * ++ * Function emits a timeline message informing that atom object is being ++ * released by context. ++ */ ++#define KBASE_TLSTREAM_TL_NRET_ATOM_CTX(atom, context) \ ++ __TRACE_IF_ENABLED(tl_nret_atom_ctx, atom, context) ++ ++/** ++ * KBASE_TLSTREAM_TL_NRET_ATOM_LPU - release atom by LPU ++ * @atom: name of the atom object ++ * @lpu: name of the Logical Processing Unit object ++ * ++ * Function emits a timeline message informing that atom object is being ++ * released by LPU. ++ */ ++#define KBASE_TLSTREAM_TL_NRET_ATOM_LPU(atom, lpu) \ ++ __TRACE_IF_ENABLED(tl_nret_atom_lpu, atom, lpu) ++ ++/** ++ * KBASE_TLSTREAM_TL_RET_AS_CTX - lifelink address space object to context ++ * @as: name of the address space object ++ * @ctx: name of the context object ++ * ++ * Function emits a timeline message informing that address space object ++ * is being held by the context object. ++ */ ++#define KBASE_TLSTREAM_TL_RET_AS_CTX(as, ctx) \ ++ __TRACE_IF_ENABLED(tl_ret_as_ctx, as, ctx) ++ ++/** ++ * KBASE_TLSTREAM_TL_NRET_AS_CTX - release address space by context ++ * @as: name of the address space object ++ * @ctx: name of the context object ++ * ++ * Function emits a timeline message informing that address space object ++ * is being released by atom. ++ */ ++#define KBASE_TLSTREAM_TL_NRET_AS_CTX(as, ctx) \ ++ __TRACE_IF_ENABLED(tl_nret_as_ctx, as, ctx) ++ ++/** ++ * KBASE_TLSTREAM_TL_RET_ATOM_AS - retain atom by address space ++ * @atom: name of the atom object ++ * @as: name of the address space object ++ * ++ * Function emits a timeline message informing that atom object is being held ++ * by address space and must not be deleted unless it is released. ++ */ ++#define KBASE_TLSTREAM_TL_RET_ATOM_AS(atom, as) \ ++ __TRACE_IF_ENABLED(tl_ret_atom_as, atom, as) ++ ++/** ++ * KBASE_TLSTREAM_TL_NRET_ATOM_AS - release atom by address space ++ * @atom: name of the atom object ++ * @as: name of the address space object ++ * ++ * Function emits a timeline message informing that atom object is being ++ * released by address space. ++ */ ++#define KBASE_TLSTREAM_TL_NRET_ATOM_AS(atom, as) \ ++ __TRACE_IF_ENABLED(tl_nret_atom_as, atom, as) ++ ++/** ++ * KBASE_TLSTREAM_TL_DEP_ATOM_ATOM - parent atom depends on child atom ++ * @atom1: name of the child atom object ++ * @atom2: name of the parent atom object that depends on child atom ++ * ++ * Function emits a timeline message informing that parent atom waits for ++ * child atom object to be completed before start its execution. ++ */ ++#define KBASE_TLSTREAM_TL_DEP_ATOM_ATOM(atom1, atom2) \ ++ __TRACE_IF_ENABLED(tl_dep_atom_atom, atom1, atom2) ++ ++/** ++ * KBASE_TLSTREAM_TL_NDEP_ATOM_ATOM - dependency between atoms resolved ++ * @atom1: name of the child atom object ++ * @atom2: name of the parent atom object that depended on child atom ++ * ++ * Function emits a timeline message informing that parent atom execution ++ * dependency on child atom has been resolved. ++ */ ++#define KBASE_TLSTREAM_TL_NDEP_ATOM_ATOM(atom1, atom2) \ ++ __TRACE_IF_ENABLED(tl_ndep_atom_atom, atom1, atom2) ++ ++/** ++ * KBASE_TLSTREAM_TL_RDEP_ATOM_ATOM - information about already resolved dependency between atoms ++ * @atom1: name of the child atom object ++ * @atom2: name of the parent atom object that depended on child atom ++ * ++ * Function emits a timeline message informing that parent atom execution ++ * dependency on child atom has been resolved. ++ */ ++#define KBASE_TLSTREAM_TL_RDEP_ATOM_ATOM(atom1, atom2) \ ++ __TRACE_IF_ENABLED(tl_rdep_atom_atom, atom1, atom2) ++ ++/** ++ * KBASE_TLSTREAM_TL_ATTRIB_ATOM_CONFIG - atom job slot attributes ++ * @atom: name of the atom object ++ * @jd: job descriptor address ++ * @affinity: job affinity ++ * @config: job config ++ * ++ * Function emits a timeline message containing atom attributes. ++ */ ++#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_CONFIG(atom, jd, affinity, config) \ ++ __TRACE_IF_ENABLED(tl_attrib_atom_config, atom, jd, affinity, config) ++ ++/** ++ * KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY - atom priority ++ * @atom: name of the atom object ++ * @prio: atom priority ++ * ++ * Function emits a timeline message containing atom priority. ++ */ ++#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY(atom, prio) \ ++ __TRACE_IF_ENABLED_LATENCY(tl_attrib_atom_priority, atom, prio) ++ ++/** ++ * KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE - atom state ++ * @atom: name of the atom object ++ * @state: atom state ++ * ++ * Function emits a timeline message containing atom state. ++ */ ++#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(atom, state) \ ++ __TRACE_IF_ENABLED_LATENCY(tl_attrib_atom_state, atom, state) ++ ++/** ++ * KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY_CHANGE - atom caused priority change ++ * @atom: name of the atom object ++ * ++ * Function emits a timeline message signalling priority change ++ */ ++#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY_CHANGE(atom) \ ++ __TRACE_IF_ENABLED_LATENCY(tl_attrib_atom_priority_change, atom) ++ ++/** ++ * KBASE_TLSTREAM_TL_ATTRIB_ATOM_JIT - jit happened on atom ++ * @atom: atom identifier ++ * @edit_addr: address edited by jit ++ * @new_addr: address placed into the edited location ++ */ ++#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_JIT(atom, edit_addr, new_addr) \ ++ __TRACE_IF_ENABLED_JD(tl_attrib_atom_jit, atom, edit_addr, new_addr) ++ ++/** ++ * KBASE_TLSTREAM_TL_ATTRIB_AS_CONFIG - address space attributes ++ * @as: assigned address space ++ * @transtab: configuration of the TRANSTAB register ++ * @memattr: configuration of the MEMATTR register ++ * @transcfg: configuration of the TRANSCFG register (or zero if not present) ++ * ++ * Function emits a timeline message containing address space attributes. ++ */ ++#define KBASE_TLSTREAM_TL_ATTRIB_AS_CONFIG(as, transtab, memattr, transcfg) \ ++ __TRACE_IF_ENABLED(tl_attrib_as_config, as, transtab, memattr, transcfg) ++ ++/** ++ * KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_ex ++ * @atom: atom identifier ++ */ ++#define KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_EX(atom) \ ++ __TRACE_IF_ENABLED(tl_event_atom_softstop_ex, atom) ++ ++/** ++ * KBASE_TLSTREAM_TL_EVENT_LPU_softstop ++ * @lpu: name of the LPU object ++ */ ++#define KBASE_TLSTREAM_TL_EVENT_LPU_SOFTSTOP(lpu) \ ++ __TRACE_IF_ENABLED(tl_event_lpu_softstop, lpu) ++ ++/** ++ * KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_issue ++ * @atom: atom identifier ++ */ ++#define KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_ISSUE(atom) \ ++ __TRACE_IF_ENABLED(tl_event_atom_softstop_issue, atom) ++ ++/** ++ * KBASE_TLSTREAM_JD_GPU_SOFT_RESET - The GPU is being soft reset ++ * @gpu: name of the GPU object ++ * ++ * This imperative tracepoint is specific to job dumping. ++ * Function emits a timeline message indicating GPU soft reset. ++ */ ++#define KBASE_TLSTREAM_JD_GPU_SOFT_RESET(gpu) \ ++ __TRACE_IF_ENABLED(jd_gpu_soft_reset, gpu) ++ ++ ++/** ++ * KBASE_TLSTREAM_AUX_PM_STATE - timeline message: power management state ++ * @core_type: core type (shader, tiler, l2 cache, l3 cache) ++ * @state: 64bits bitmask reporting power state of the cores (1-ON, 0-OFF) ++ */ ++#define KBASE_TLSTREAM_AUX_PM_STATE(core_type, state) \ ++ __TRACE_IF_ENABLED(aux_pm_state, core_type, state) ++ ++/** ++ * KBASE_TLSTREAM_AUX_PAGEFAULT - timeline message: MMU page fault event ++ * resulting in new pages being mapped ++ * @ctx_nr: kernel context number ++ * @page_count_change: number of pages to be added ++ */ ++#define KBASE_TLSTREAM_AUX_PAGEFAULT(ctx_nr, page_count_change) \ ++ __TRACE_IF_ENABLED(aux_pagefault, ctx_nr, page_count_change) ++ ++/** ++ * KBASE_TLSTREAM_AUX_PAGESALLOC - timeline message: total number of allocated ++ * pages is changed ++ * @ctx_nr: kernel context number ++ * @page_count: number of pages used by the context ++ */ ++#define KBASE_TLSTREAM_AUX_PAGESALLOC(ctx_nr, page_count) \ ++ __TRACE_IF_ENABLED(aux_pagesalloc, ctx_nr, page_count) ++ ++/** ++ * KBASE_TLSTREAM_AUX_DEVFREQ_TARGET - timeline message: new target DVFS ++ * frequency ++ * @target_freq: new target frequency ++ */ ++#define KBASE_TLSTREAM_AUX_DEVFREQ_TARGET(target_freq) \ ++ __TRACE_IF_ENABLED(aux_devfreq_target, target_freq) ++ ++/** ++ * KBASE_TLSTREAM_AUX_PROTECTED_ENTER_START - The GPU has started transitioning ++ * to protected mode ++ * @gpu: name of the GPU object ++ * ++ * Function emits a timeline message indicating the GPU is starting to ++ * transition to protected mode. ++ */ ++#define KBASE_TLSTREAM_AUX_PROTECTED_ENTER_START(gpu) \ ++ __TRACE_IF_ENABLED_LATENCY(aux_protected_enter_start, gpu) ++ ++/** ++ * KBASE_TLSTREAM_AUX_PROTECTED_ENTER_END - The GPU has finished transitioning ++ * to protected mode ++ * @gpu: name of the GPU object ++ * ++ * Function emits a timeline message indicating the GPU has finished ++ * transitioning to protected mode. ++ */ ++#define KBASE_TLSTREAM_AUX_PROTECTED_ENTER_END(gpu) \ ++ __TRACE_IF_ENABLED_LATENCY(aux_protected_enter_end, gpu) ++ ++/** ++ * KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_START - The GPU has started transitioning ++ * to non-protected mode ++ * @gpu: name of the GPU object ++ * ++ * Function emits a timeline message indicating the GPU is starting to ++ * transition to non-protected mode. ++ */ ++#define KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_START(gpu) \ ++ __TRACE_IF_ENABLED_LATENCY(aux_protected_leave_start, gpu) ++ ++/** ++ * KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_END - The GPU has finished transitioning ++ * to non-protected mode ++ * @gpu: name of the GPU object ++ * ++ * Function emits a timeline message indicating the GPU has finished ++ * transitioning to non-protected mode. ++ */ ++#define KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_END(gpu) \ ++ __TRACE_IF_ENABLED_LATENCY(aux_protected_leave_end, gpu) ++ ++#endif /* _KBASE_TLSTREAM_H */ ++ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_trace_defs.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_trace_defs.h +new file mode 100755 +index 000000000000..e2e0544208ce +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_trace_defs.h +@@ -0,0 +1,264 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2014 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ***** IMPORTANT: THIS IS NOT A NORMAL HEADER FILE ***** ++ * ***** DO NOT INCLUDE DIRECTLY ***** ++ * ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** */ ++ ++/* ++ * The purpose of this header file is just to contain a list of trace code idenitifers ++ * ++ * Each identifier is wrapped in a macro, so that its string form and enum form can be created ++ * ++ * Each macro is separated with a comma, to allow insertion into an array initializer or enum definition block. ++ * ++ * This allows automatic creation of an enum and a corresponding array of strings ++ * ++ * Before #including, the includer MUST #define KBASE_TRACE_CODE_MAKE_CODE. ++ * After #including, the includer MUST #under KBASE_TRACE_CODE_MAKE_CODE. ++ * ++ * e.g.: ++ * #define KBASE_TRACE_CODE( X ) KBASE_TRACE_CODE_ ## X ++ * typedef enum ++ * { ++ * #define KBASE_TRACE_CODE_MAKE_CODE( X ) KBASE_TRACE_CODE( X ) ++ * #include "mali_kbase_trace_defs.h" ++ * #undef KBASE_TRACE_CODE_MAKE_CODE ++ * } kbase_trace_code; ++ * ++ * IMPORTANT: THIS FILE MUST NOT BE USED FOR ANY OTHER PURPOSE OTHER THAN THE ABOVE ++ * ++ * ++ * The use of the macro here is: ++ * - KBASE_TRACE_CODE_MAKE_CODE( X ) ++ * ++ * Which produces: ++ * - For an enum, KBASE_TRACE_CODE_X ++ * - For a string, "X" ++ * ++ * ++ * For example: ++ * - KBASE_TRACE_CODE_MAKE_CODE( JM_JOB_COMPLETE ) expands to: ++ * - KBASE_TRACE_CODE_JM_JOB_COMPLETE for the enum ++ * - "JM_JOB_COMPLETE" for the string ++ * - To use it to trace an event, do: ++ * - KBASE_TRACE_ADD( kbdev, JM_JOB_COMPLETE, subcode, kctx, uatom, val ); ++ */ ++ ++#if 0 /* Dummy section to avoid breaking formatting */ ++int dummy_array[] = { ++#endif ++ ++/* ++ * Core events ++ */ ++ /* no info_val, no gpu_addr, no atom */ ++ KBASE_TRACE_CODE_MAKE_CODE(CORE_CTX_DESTROY), ++ /* no info_val, no gpu_addr, no atom */ ++ KBASE_TRACE_CODE_MAKE_CODE(CORE_CTX_HWINSTR_TERM), ++ /* info_val == GPU_IRQ_STATUS register */ ++ KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_IRQ), ++ /* info_val == bits cleared */ ++ KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_IRQ_CLEAR), ++ /* info_val == GPU_IRQ_STATUS register */ ++ KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_IRQ_DONE), ++ KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_SOFT_RESET), ++ KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_HARD_RESET), ++ KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_PRFCNT_CLEAR), ++ /* GPU addr==dump address */ ++ KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_PRFCNT_SAMPLE), ++ KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_CLEAN_INV_CACHES), ++/* ++ * Job Slot management events ++ */ ++ /* info_val==irq rawstat at start */ ++ KBASE_TRACE_CODE_MAKE_CODE(JM_IRQ), ++ /* info_val==jobs processed */ ++ KBASE_TRACE_CODE_MAKE_CODE(JM_IRQ_END), ++/* In the following: ++ * ++ * - ctx is set if a corresponding job found (NULL otherwise, e.g. some soft-stop cases) ++ * - uatom==kernel-side mapped uatom address (for correlation with user-side) ++ */ ++ /* info_val==exit code; gpu_addr==chain gpuaddr */ ++ KBASE_TRACE_CODE_MAKE_CODE(JM_JOB_DONE), ++ /* gpu_addr==JS_HEAD_NEXT written, info_val==lower 32 bits of affinity */ ++ KBASE_TRACE_CODE_MAKE_CODE(JM_SUBMIT), ++ /* gpu_addr is as follows: ++ * - If JS_STATUS active after soft-stop, val==gpu addr written to ++ * JS_HEAD on submit ++ * - otherwise gpu_addr==0 */ ++ KBASE_TRACE_CODE_MAKE_CODE(JM_SOFTSTOP), ++ KBASE_TRACE_CODE_MAKE_CODE(JM_SOFTSTOP_0), ++ KBASE_TRACE_CODE_MAKE_CODE(JM_SOFTSTOP_1), ++ /* gpu_addr==JS_HEAD read */ ++ KBASE_TRACE_CODE_MAKE_CODE(JM_HARDSTOP), ++ /* gpu_addr==JS_HEAD read */ ++ KBASE_TRACE_CODE_MAKE_CODE(JM_HARDSTOP_0), ++ /* gpu_addr==JS_HEAD read */ ++ KBASE_TRACE_CODE_MAKE_CODE(JM_HARDSTOP_1), ++ /* gpu_addr==JS_TAIL read */ ++ KBASE_TRACE_CODE_MAKE_CODE(JM_UPDATE_HEAD), ++/* gpu_addr is as follows: ++ * - If JS_STATUS active before soft-stop, val==JS_HEAD ++ * - otherwise gpu_addr==0 ++ */ ++ /* gpu_addr==JS_HEAD read */ ++ KBASE_TRACE_CODE_MAKE_CODE(JM_CHECK_HEAD), ++ KBASE_TRACE_CODE_MAKE_CODE(JM_FLUSH_WORKQS), ++ KBASE_TRACE_CODE_MAKE_CODE(JM_FLUSH_WORKQS_DONE), ++ /* info_val == is_scheduled */ ++ KBASE_TRACE_CODE_MAKE_CODE(JM_ZAP_NON_SCHEDULED), ++ /* info_val == is_scheduled */ ++ KBASE_TRACE_CODE_MAKE_CODE(JM_ZAP_SCHEDULED), ++ KBASE_TRACE_CODE_MAKE_CODE(JM_ZAP_DONE), ++ /* info_val == nr jobs submitted */ ++ KBASE_TRACE_CODE_MAKE_CODE(JM_SLOT_SOFT_OR_HARD_STOP), ++ /* gpu_addr==JS_HEAD_NEXT last written */ ++ KBASE_TRACE_CODE_MAKE_CODE(JM_SLOT_EVICT), ++ KBASE_TRACE_CODE_MAKE_CODE(JM_SUBMIT_AFTER_RESET), ++ KBASE_TRACE_CODE_MAKE_CODE(JM_BEGIN_RESET_WORKER), ++ KBASE_TRACE_CODE_MAKE_CODE(JM_END_RESET_WORKER), ++/* ++ * Job dispatch events ++ */ ++ /* gpu_addr==value to write into JS_HEAD */ ++ KBASE_TRACE_CODE_MAKE_CODE(JD_DONE), ++ /* gpu_addr==value to write into JS_HEAD */ ++ KBASE_TRACE_CODE_MAKE_CODE(JD_DONE_WORKER), ++ /* gpu_addr==value to write into JS_HEAD */ ++ KBASE_TRACE_CODE_MAKE_CODE(JD_DONE_WORKER_END), ++ /* gpu_addr==value to write into JS_HEAD */ ++ KBASE_TRACE_CODE_MAKE_CODE(JD_DONE_TRY_RUN_NEXT_JOB), ++ /* gpu_addr==0, info_val==0, uatom==0 */ ++ KBASE_TRACE_CODE_MAKE_CODE(JD_ZAP_CONTEXT), ++ /* gpu_addr==value to write into JS_HEAD */ ++ KBASE_TRACE_CODE_MAKE_CODE(JD_CANCEL), ++ /* gpu_addr==value to write into JS_HEAD */ ++ KBASE_TRACE_CODE_MAKE_CODE(JD_CANCEL_WORKER), ++/* ++ * Scheduler Core events ++ */ ++ KBASE_TRACE_CODE_MAKE_CODE(JS_RETAIN_CTX_NOLOCK), ++ /* gpu_addr==value to write into JS_HEAD */ ++ KBASE_TRACE_CODE_MAKE_CODE(JS_ADD_JOB), ++ /* gpu_addr==last value written/would be written to JS_HEAD */ ++ KBASE_TRACE_CODE_MAKE_CODE(JS_REMOVE_JOB), ++ KBASE_TRACE_CODE_MAKE_CODE(JS_RETAIN_CTX), ++ KBASE_TRACE_CODE_MAKE_CODE(JS_RELEASE_CTX), ++ KBASE_TRACE_CODE_MAKE_CODE(JS_TRY_SCHEDULE_HEAD_CTX), ++ /* gpu_addr==value to write into JS_HEAD */ ++ KBASE_TRACE_CODE_MAKE_CODE(JS_JOB_DONE_TRY_RUN_NEXT_JOB), ++ /* gpu_addr==value to write into JS_HEAD */ ++ KBASE_TRACE_CODE_MAKE_CODE(JS_JOB_DONE_RETRY_NEEDED), ++ /* kctx is the one being evicted, info_val == kctx to put in */ ++ KBASE_TRACE_CODE_MAKE_CODE(JS_FAST_START_EVICTS_CTX), ++ KBASE_TRACE_CODE_MAKE_CODE(JS_AFFINITY_SUBMIT_TO_BLOCKED), ++ /* info_val == lower 32 bits of affinity */ ++ KBASE_TRACE_CODE_MAKE_CODE(JS_AFFINITY_CURRENT), ++ /* info_val == lower 32 bits of affinity */ ++ KBASE_TRACE_CODE_MAKE_CODE(JS_CORE_REF_REQUEST_CORES_FAILED), ++ /* info_val == lower 32 bits of affinity */ ++ KBASE_TRACE_CODE_MAKE_CODE(JS_CORE_REF_REGISTER_INUSE_FAILED), ++ /* info_val == lower 32 bits of rechecked affinity */ ++ KBASE_TRACE_CODE_MAKE_CODE(JS_CORE_REF_REQUEST_ON_RECHECK_FAILED), ++ /* info_val == lower 32 bits of rechecked affinity */ ++ KBASE_TRACE_CODE_MAKE_CODE(JS_CORE_REF_REGISTER_ON_RECHECK_FAILED), ++ /* info_val == lower 32 bits of affinity */ ++ KBASE_TRACE_CODE_MAKE_CODE(JS_CORE_REF_AFFINITY_WOULD_VIOLATE), ++ /* info_val == the ctx attribute now on ctx */ ++ KBASE_TRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_ON_CTX), ++ /* info_val == the ctx attribute now on runpool */ ++ KBASE_TRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_ON_RUNPOOL), ++ /* info_val == the ctx attribute now off ctx */ ++ KBASE_TRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_OFF_CTX), ++ /* info_val == the ctx attribute now off runpool */ ++ KBASE_TRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_OFF_RUNPOOL), ++/* ++ * Scheduler Policy events ++ */ ++ KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_INIT_CTX), ++ KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_TERM_CTX), ++ /* info_val == whether it was evicted */ ++ KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_TRY_EVICT_CTX), ++ KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_FOREACH_CTX_JOBS), ++ KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_ENQUEUE_CTX), ++ KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_DEQUEUE_HEAD_CTX), ++ KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_RUNPOOL_ADD_CTX), ++ KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_RUNPOOL_REMOVE_CTX), ++ KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_DEQUEUE_JOB), ++ KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_DEQUEUE_JOB_IRQ), ++ /* gpu_addr==JS_HEAD to write if the job were run */ ++ KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_ENQUEUE_JOB), ++ KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_TIMER_START), ++ KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_TIMER_END), ++/* ++ * Power Management Events ++ */ ++ KBASE_TRACE_CODE_MAKE_CODE(PM_JOB_SUBMIT_AFTER_POWERING_UP), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_JOB_SUBMIT_AFTER_POWERED_UP), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_PWRON), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_PWRON_TILER), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_PWRON_L2), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_PWROFF), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_PWROFF_TILER), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_PWROFF_L2), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_POWERED), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_POWERED_TILER), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_POWERED_L2), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_DESIRED), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_DESIRED_TILER), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_AVAILABLE), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_AVAILABLE_TILER), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_AVAILABLE), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_AVAILABLE_TILER), ++ /* PM_DESIRED_REACHED: gpu_addr == pm.gpu_in_desired_state */ ++ KBASE_TRACE_CODE_MAKE_CODE(PM_DESIRED_REACHED), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_DESIRED_REACHED_TILER), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_REGISTER_CHANGE_SHADER_INUSE), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_REGISTER_CHANGE_TILER_INUSE), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_REGISTER_CHANGE_SHADER_NEEDED), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_REGISTER_CHANGE_TILER_NEEDED), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_RELEASE_CHANGE_SHADER_INUSE), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_RELEASE_CHANGE_TILER_INUSE), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_UNREQUEST_CHANGE_SHADER_NEEDED), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_UNREQUEST_CHANGE_TILER_NEEDED), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_REQUEST_CHANGE_SHADER_NEEDED), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_REQUEST_CHANGE_TILER_NEEDED), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_WAKE_WAITERS), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_CONTEXT_ACTIVE), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_CONTEXT_IDLE), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_GPU_ON), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_GPU_OFF), ++ /* info_val == policy number, or -1 for "Already changing" */ ++ KBASE_TRACE_CODE_MAKE_CODE(PM_SET_POLICY), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_CA_SET_POLICY), ++ /* info_val == policy number */ ++ KBASE_TRACE_CODE_MAKE_CODE(PM_CURRENT_POLICY_INIT), ++ /* info_val == policy number */ ++ KBASE_TRACE_CODE_MAKE_CODE(PM_CURRENT_POLICY_TERM), ++/* Unused code just to make it easier to not have a comma at the end. ++ * All other codes MUST come before this */ ++ KBASE_TRACE_CODE_MAKE_CODE(DUMMY) ++ ++#if 0 /* Dummy section to avoid breaking formatting */ ++}; ++#endif ++ ++/* ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_trace_timeline.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_trace_timeline.c +new file mode 100755 +index 000000000000..d9854749f45b +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_trace_timeline.c +@@ -0,0 +1,236 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#include ++#include ++#include ++ ++#define CREATE_TRACE_POINTS ++ ++#ifdef CONFIG_MALI_BIFROST_TRACE_TIMELINE ++#include "mali_timeline.h" ++ ++#include ++#include ++ ++EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_atoms_in_flight); ++EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_atom); ++EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_gpu_slot_active); ++EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_gpu_slot_action); ++EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_gpu_power_active); ++EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_l2_power_active); ++EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_pm_event); ++EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_slot_atom); ++EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_pm_checktrans); ++EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_context_active); ++ ++struct kbase_trace_timeline_desc { ++ char *enum_str; ++ char *desc; ++ char *format; ++ char *format_desc; ++}; ++ ++static struct kbase_trace_timeline_desc kbase_trace_timeline_desc_table[] = { ++ #define KBASE_TIMELINE_TRACE_CODE(enum_val, desc, format, format_desc) { #enum_val, desc, format, format_desc } ++ #include "mali_kbase_trace_timeline_defs.h" ++ #undef KBASE_TIMELINE_TRACE_CODE ++}; ++ ++#define KBASE_NR_TRACE_CODES ARRAY_SIZE(kbase_trace_timeline_desc_table) ++ ++static void *kbasep_trace_timeline_seq_start(struct seq_file *s, loff_t *pos) ++{ ++ if (*pos >= KBASE_NR_TRACE_CODES) ++ return NULL; ++ ++ return &kbase_trace_timeline_desc_table[*pos]; ++} ++ ++static void kbasep_trace_timeline_seq_stop(struct seq_file *s, void *data) ++{ ++} ++ ++static void *kbasep_trace_timeline_seq_next(struct seq_file *s, void *data, loff_t *pos) ++{ ++ (*pos)++; ++ ++ if (*pos == KBASE_NR_TRACE_CODES) ++ return NULL; ++ ++ return &kbase_trace_timeline_desc_table[*pos]; ++} ++ ++static int kbasep_trace_timeline_seq_show(struct seq_file *s, void *data) ++{ ++ struct kbase_trace_timeline_desc *trace_desc = data; ++ ++ seq_printf(s, "%s#%s#%s#%s\n", trace_desc->enum_str, trace_desc->desc, trace_desc->format, trace_desc->format_desc); ++ return 0; ++} ++ ++ ++static const struct seq_operations kbasep_trace_timeline_seq_ops = { ++ .start = kbasep_trace_timeline_seq_start, ++ .next = kbasep_trace_timeline_seq_next, ++ .stop = kbasep_trace_timeline_seq_stop, ++ .show = kbasep_trace_timeline_seq_show, ++}; ++ ++static int kbasep_trace_timeline_debugfs_open(struct inode *inode, struct file *file) ++{ ++ return seq_open(file, &kbasep_trace_timeline_seq_ops); ++} ++ ++static const struct file_operations kbasep_trace_timeline_debugfs_fops = { ++ .open = kbasep_trace_timeline_debugfs_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = seq_release, ++}; ++ ++#ifdef CONFIG_DEBUG_FS ++ ++void kbasep_trace_timeline_debugfs_init(struct kbase_device *kbdev) ++{ ++ debugfs_create_file("mali_timeline_defs", ++ S_IRUGO, kbdev->mali_debugfs_directory, NULL, ++ &kbasep_trace_timeline_debugfs_fops); ++} ++ ++#endif /* CONFIG_DEBUG_FS */ ++ ++void kbase_timeline_job_slot_submit(struct kbase_device *kbdev, struct kbase_context *kctx, ++ struct kbase_jd_atom *katom, int js) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (kbdev->timeline.slot_atoms_submitted[js] > 0) { ++ KBASE_TIMELINE_JOB_START_NEXT(kctx, js, 1); ++ } else { ++ base_atom_id atom_number = kbase_jd_atom_id(kctx, katom); ++ ++ KBASE_TIMELINE_JOB_START_HEAD(kctx, js, 1); ++ KBASE_TIMELINE_JOB_START(kctx, js, atom_number); ++ } ++ ++kbdev->timeline.slot_atoms_submitted[js]; ++ ++ KBASE_TIMELINE_ATOMS_SUBMITTED(kctx, js, kbdev->timeline.slot_atoms_submitted[js]); ++} ++ ++void kbase_timeline_job_slot_done(struct kbase_device *kbdev, struct kbase_context *kctx, ++ struct kbase_jd_atom *katom, int js, ++ kbasep_js_atom_done_code done_code) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (done_code & KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT) { ++ KBASE_TIMELINE_JOB_START_NEXT(kctx, js, 0); ++ } else { ++ /* Job finished in JS_HEAD */ ++ base_atom_id atom_number = kbase_jd_atom_id(kctx, katom); ++ ++ KBASE_TIMELINE_JOB_START_HEAD(kctx, js, 0); ++ KBASE_TIMELINE_JOB_STOP(kctx, js, atom_number); ++ ++ /* see if we need to trace the job in JS_NEXT moving to JS_HEAD */ ++ if (kbase_backend_nr_atoms_submitted(kbdev, js)) { ++ struct kbase_jd_atom *next_katom; ++ struct kbase_context *next_kctx; ++ ++ /* Peek the next atom - note that the atom in JS_HEAD will already ++ * have been dequeued */ ++ next_katom = kbase_backend_inspect_head(kbdev, js); ++ WARN_ON(!next_katom); ++ next_kctx = next_katom->kctx; ++ KBASE_TIMELINE_JOB_START_NEXT(next_kctx, js, 0); ++ KBASE_TIMELINE_JOB_START_HEAD(next_kctx, js, 1); ++ KBASE_TIMELINE_JOB_START(next_kctx, js, kbase_jd_atom_id(next_kctx, next_katom)); ++ } ++ } ++ ++ --kbdev->timeline.slot_atoms_submitted[js]; ++ ++ KBASE_TIMELINE_ATOMS_SUBMITTED(kctx, js, kbdev->timeline.slot_atoms_submitted[js]); ++} ++ ++void kbase_timeline_pm_send_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event_sent) ++{ ++ int uid = 0; ++ int old_uid; ++ ++ /* If a producer already exists for the event, try to use their UID (multiple-producers) */ ++ uid = atomic_read(&kbdev->timeline.pm_event_uid[event_sent]); ++ old_uid = uid; ++ ++ /* Get a new non-zero UID if we don't have one yet */ ++ while (!uid) ++ uid = atomic_inc_return(&kbdev->timeline.pm_event_uid_counter); ++ ++ /* Try to use this UID */ ++ if (old_uid != atomic_cmpxchg(&kbdev->timeline.pm_event_uid[event_sent], old_uid, uid)) ++ /* If it changed, raced with another producer: we've lost this UID */ ++ uid = 0; ++ ++ KBASE_TIMELINE_PM_SEND_EVENT(kbdev, event_sent, uid); ++} ++ ++void kbase_timeline_pm_check_handle_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event) ++{ ++ int uid = atomic_read(&kbdev->timeline.pm_event_uid[event]); ++ ++ if (uid != 0) { ++ if (uid != atomic_cmpxchg(&kbdev->timeline.pm_event_uid[event], uid, 0)) ++ /* If it changed, raced with another consumer: we've lost this UID */ ++ uid = 0; ++ ++ KBASE_TIMELINE_PM_HANDLE_EVENT(kbdev, event, uid); ++ } ++} ++ ++void kbase_timeline_pm_handle_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event) ++{ ++ int uid = atomic_read(&kbdev->timeline.pm_event_uid[event]); ++ ++ if (uid != atomic_cmpxchg(&kbdev->timeline.pm_event_uid[event], uid, 0)) ++ /* If it changed, raced with another consumer: we've lost this UID */ ++ uid = 0; ++ ++ KBASE_TIMELINE_PM_HANDLE_EVENT(kbdev, event, uid); ++} ++ ++void kbase_timeline_pm_l2_transition_start(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ /* Simply log the start of the transition */ ++ kbdev->timeline.l2_transitioning = true; ++ KBASE_TIMELINE_POWERING_L2(kbdev); ++} ++ ++void kbase_timeline_pm_l2_transition_done(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ /* Simply log the end of the transition */ ++ if (kbdev->timeline.l2_transitioning) { ++ kbdev->timeline.l2_transitioning = false; ++ KBASE_TIMELINE_POWERED_L2(kbdev); ++ } ++} ++ ++#endif /* CONFIG_MALI_BIFROST_TRACE_TIMELINE */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_trace_timeline.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_trace_timeline.h +new file mode 100755 +index 000000000000..4b517f396f8c +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_trace_timeline.h +@@ -0,0 +1,363 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#if !defined(_KBASE_TRACE_TIMELINE_H) ++#define _KBASE_TRACE_TIMELINE_H ++ ++#ifdef CONFIG_MALI_BIFROST_TRACE_TIMELINE ++ ++enum kbase_trace_timeline_code { ++ #define KBASE_TIMELINE_TRACE_CODE(enum_val, desc, format, format_desc) enum_val ++ #include "mali_kbase_trace_timeline_defs.h" ++ #undef KBASE_TIMELINE_TRACE_CODE ++}; ++ ++#ifdef CONFIG_DEBUG_FS ++ ++/** Initialize Timeline DebugFS entries */ ++void kbasep_trace_timeline_debugfs_init(struct kbase_device *kbdev); ++ ++#else /* CONFIG_DEBUG_FS */ ++ ++#define kbasep_trace_timeline_debugfs_init CSTD_NOP ++ ++#endif /* CONFIG_DEBUG_FS */ ++ ++/* mali_timeline.h defines kernel tracepoints used by the KBASE_TIMELINE ++ * functions. ++ * Output is timestamped by either sched_clock() (default), local_clock(), or ++ * cpu_clock(), depending on /sys/kernel/debug/tracing/trace_clock */ ++#include "mali_timeline.h" ++ ++/* Trace number of atoms in flight for kctx (atoms either not completed, or in ++ process of being returned to user */ ++#define KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, count) \ ++ do { \ ++ struct timespec ts; \ ++ getrawmonotonic(&ts); \ ++ trace_mali_timeline_atoms_in_flight(ts.tv_sec, ts.tv_nsec, \ ++ (int)kctx->timeline.owner_tgid, \ ++ count); \ ++ } while (0) ++ ++/* Trace atom_id being Ready to Run */ ++#define KBASE_TIMELINE_ATOM_READY(kctx, atom_id) \ ++ do { \ ++ struct timespec ts; \ ++ getrawmonotonic(&ts); \ ++ trace_mali_timeline_atom(ts.tv_sec, ts.tv_nsec, \ ++ CTX_FLOW_ATOM_READY, \ ++ (int)kctx->timeline.owner_tgid, \ ++ atom_id); \ ++ } while (0) ++ ++/* Trace number of atoms submitted to job slot js ++ * ++ * NOTE: This uses a different tracepoint to the head/next/soft-stop actions, ++ * so that those actions can be filtered out separately from this ++ * ++ * This is because this is more useful, as we can use it to calculate general ++ * utilization easily and accurately */ ++#define KBASE_TIMELINE_ATOMS_SUBMITTED(kctx, js, count) \ ++ do { \ ++ struct timespec ts; \ ++ getrawmonotonic(&ts); \ ++ trace_mali_timeline_gpu_slot_active(ts.tv_sec, ts.tv_nsec, \ ++ SW_SET_GPU_SLOT_ACTIVE, \ ++ (int)kctx->timeline.owner_tgid, \ ++ js, count); \ ++ } while (0) ++ ++ ++/* Trace atoms present in JS_NEXT */ ++#define KBASE_TIMELINE_JOB_START_NEXT(kctx, js, count) \ ++ do { \ ++ struct timespec ts; \ ++ getrawmonotonic(&ts); \ ++ trace_mali_timeline_gpu_slot_action(ts.tv_sec, ts.tv_nsec, \ ++ SW_SET_GPU_SLOT_NEXT, \ ++ (int)kctx->timeline.owner_tgid, \ ++ js, count); \ ++ } while (0) ++ ++/* Trace atoms present in JS_HEAD */ ++#define KBASE_TIMELINE_JOB_START_HEAD(kctx, js, count) \ ++ do { \ ++ struct timespec ts; \ ++ getrawmonotonic(&ts); \ ++ trace_mali_timeline_gpu_slot_action(ts.tv_sec, ts.tv_nsec, \ ++ SW_SET_GPU_SLOT_HEAD, \ ++ (int)kctx->timeline.owner_tgid, \ ++ js, count); \ ++ } while (0) ++ ++/* Trace that a soft stop/evict from next is being attempted on a slot */ ++#define KBASE_TIMELINE_TRY_SOFT_STOP(kctx, js, count) \ ++ do { \ ++ struct timespec ts; \ ++ getrawmonotonic(&ts); \ ++ trace_mali_timeline_gpu_slot_action(ts.tv_sec, ts.tv_nsec, \ ++ SW_SET_GPU_SLOT_STOPPING, \ ++ (kctx) ? (int)kctx->timeline.owner_tgid : 0, \ ++ js, count); \ ++ } while (0) ++ ++ ++ ++/* Trace state of overall GPU power */ ++#define KBASE_TIMELINE_GPU_POWER(kbdev, active) \ ++ do { \ ++ struct timespec ts; \ ++ getrawmonotonic(&ts); \ ++ trace_mali_timeline_gpu_power_active(ts.tv_sec, ts.tv_nsec, \ ++ SW_SET_GPU_POWER_ACTIVE, active); \ ++ } while (0) ++ ++/* Trace state of tiler power */ ++#define KBASE_TIMELINE_POWER_TILER(kbdev, bitmap) \ ++ do { \ ++ struct timespec ts; \ ++ getrawmonotonic(&ts); \ ++ trace_mali_timeline_gpu_power_active(ts.tv_sec, ts.tv_nsec, \ ++ SW_SET_GPU_POWER_TILER_ACTIVE, \ ++ hweight64(bitmap)); \ ++ } while (0) ++ ++/* Trace number of shaders currently powered */ ++#define KBASE_TIMELINE_POWER_SHADER(kbdev, bitmap) \ ++ do { \ ++ struct timespec ts; \ ++ getrawmonotonic(&ts); \ ++ trace_mali_timeline_gpu_power_active(ts.tv_sec, ts.tv_nsec, \ ++ SW_SET_GPU_POWER_SHADER_ACTIVE, \ ++ hweight64(bitmap)); \ ++ } while (0) ++ ++/* Trace state of L2 power */ ++#define KBASE_TIMELINE_POWER_L2(kbdev, bitmap) \ ++ do { \ ++ struct timespec ts; \ ++ getrawmonotonic(&ts); \ ++ trace_mali_timeline_gpu_power_active(ts.tv_sec, ts.tv_nsec, \ ++ SW_SET_GPU_POWER_L2_ACTIVE, \ ++ hweight64(bitmap)); \ ++ } while (0) ++ ++/* Trace state of L2 cache*/ ++#define KBASE_TIMELINE_POWERING_L2(kbdev) \ ++ do { \ ++ struct timespec ts; \ ++ getrawmonotonic(&ts); \ ++ trace_mali_timeline_l2_power_active(ts.tv_sec, ts.tv_nsec, \ ++ SW_FLOW_GPU_POWER_L2_POWERING, \ ++ 1); \ ++ } while (0) ++ ++#define KBASE_TIMELINE_POWERED_L2(kbdev) \ ++ do { \ ++ struct timespec ts; \ ++ getrawmonotonic(&ts); \ ++ trace_mali_timeline_l2_power_active(ts.tv_sec, ts.tv_nsec, \ ++ SW_FLOW_GPU_POWER_L2_ACTIVE, \ ++ 1); \ ++ } while (0) ++ ++/* Trace kbase_pm_send_event message send */ ++#define KBASE_TIMELINE_PM_SEND_EVENT(kbdev, event_type, pm_event_id) \ ++ do { \ ++ struct timespec ts; \ ++ getrawmonotonic(&ts); \ ++ trace_mali_timeline_pm_event(ts.tv_sec, ts.tv_nsec, \ ++ SW_FLOW_PM_SEND_EVENT, \ ++ event_type, pm_event_id); \ ++ } while (0) ++ ++/* Trace kbase_pm_worker message receive */ ++#define KBASE_TIMELINE_PM_HANDLE_EVENT(kbdev, event_type, pm_event_id) \ ++ do { \ ++ struct timespec ts; \ ++ getrawmonotonic(&ts); \ ++ trace_mali_timeline_pm_event(ts.tv_sec, ts.tv_nsec, \ ++ SW_FLOW_PM_HANDLE_EVENT, \ ++ event_type, pm_event_id); \ ++ } while (0) ++ ++ ++/* Trace atom_id starting in JS_HEAD */ ++#define KBASE_TIMELINE_JOB_START(kctx, js, _consumerof_atom_number) \ ++ do { \ ++ struct timespec ts; \ ++ getrawmonotonic(&ts); \ ++ trace_mali_timeline_slot_atom(ts.tv_sec, ts.tv_nsec, \ ++ HW_START_GPU_JOB_CHAIN_SW_APPROX, \ ++ (int)kctx->timeline.owner_tgid, \ ++ js, _consumerof_atom_number); \ ++ } while (0) ++ ++/* Trace atom_id stopping on JS_HEAD */ ++#define KBASE_TIMELINE_JOB_STOP(kctx, js, _producerof_atom_number_completed) \ ++ do { \ ++ struct timespec ts; \ ++ getrawmonotonic(&ts); \ ++ trace_mali_timeline_slot_atom(ts.tv_sec, ts.tv_nsec, \ ++ HW_STOP_GPU_JOB_CHAIN_SW_APPROX, \ ++ (int)kctx->timeline.owner_tgid, \ ++ js, _producerof_atom_number_completed); \ ++ } while (0) ++ ++/** Trace beginning/end of a call to kbase_pm_check_transitions_nolock from a ++ * certin caller */ ++#define KBASE_TIMELINE_PM_CHECKTRANS(kbdev, trace_code) \ ++ do { \ ++ struct timespec ts; \ ++ getrawmonotonic(&ts); \ ++ trace_mali_timeline_pm_checktrans(ts.tv_sec, ts.tv_nsec, \ ++ trace_code, 1); \ ++ } while (0) ++ ++/* Trace number of contexts active */ ++#define KBASE_TIMELINE_CONTEXT_ACTIVE(kbdev, count) \ ++ do { \ ++ struct timespec ts; \ ++ getrawmonotonic(&ts); \ ++ trace_mali_timeline_context_active(ts.tv_sec, ts.tv_nsec, \ ++ count); \ ++ } while (0) ++ ++/* NOTE: kbase_timeline_pm_cores_func() is in mali_kbase_pm_policy.c */ ++ ++/** ++ * Trace that an atom is starting on a job slot ++ * ++ * The caller must be holding hwaccess_lock ++ */ ++void kbase_timeline_job_slot_submit(struct kbase_device *kbdev, struct kbase_context *kctx, ++ struct kbase_jd_atom *katom, int js); ++ ++/** ++ * Trace that an atom has done on a job slot ++ * ++ * 'Done' in this sense can occur either because: ++ * - the atom in JS_HEAD finished ++ * - the atom in JS_NEXT was evicted ++ * ++ * Whether the atom finished or was evicted is passed in @a done_code ++ * ++ * It is assumed that the atom has already been removed from the submit slot, ++ * with either: ++ * - kbasep_jm_dequeue_submit_slot() ++ * - kbasep_jm_dequeue_tail_submit_slot() ++ * ++ * The caller must be holding hwaccess_lock ++ */ ++void kbase_timeline_job_slot_done(struct kbase_device *kbdev, struct kbase_context *kctx, ++ struct kbase_jd_atom *katom, int js, ++ kbasep_js_atom_done_code done_code); ++ ++ ++/** Trace a pm event starting */ ++void kbase_timeline_pm_send_event(struct kbase_device *kbdev, ++ enum kbase_timeline_pm_event event_sent); ++ ++/** Trace a pm event finishing */ ++void kbase_timeline_pm_check_handle_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event); ++ ++/** Check whether a pm event was present, and if so trace finishing it */ ++void kbase_timeline_pm_handle_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event); ++ ++/** Trace L2 power-up start */ ++void kbase_timeline_pm_l2_transition_start(struct kbase_device *kbdev); ++ ++/** Trace L2 power-up done */ ++void kbase_timeline_pm_l2_transition_done(struct kbase_device *kbdev); ++ ++#else ++ ++#define KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, count) CSTD_NOP() ++ ++#define KBASE_TIMELINE_ATOM_READY(kctx, atom_id) CSTD_NOP() ++ ++#define KBASE_TIMELINE_ATOMS_SUBMITTED(kctx, js, count) CSTD_NOP() ++ ++#define KBASE_TIMELINE_JOB_START_NEXT(kctx, js, count) CSTD_NOP() ++ ++#define KBASE_TIMELINE_JOB_START_HEAD(kctx, js, count) CSTD_NOP() ++ ++#define KBASE_TIMELINE_TRY_SOFT_STOP(kctx, js, count) CSTD_NOP() ++ ++#define KBASE_TIMELINE_GPU_POWER(kbdev, active) CSTD_NOP() ++ ++#define KBASE_TIMELINE_POWER_TILER(kbdev, bitmap) CSTD_NOP() ++ ++#define KBASE_TIMELINE_POWER_SHADER(kbdev, bitmap) CSTD_NOP() ++ ++#define KBASE_TIMELINE_POWER_L2(kbdev, active) CSTD_NOP() ++ ++#define KBASE_TIMELINE_POWERING_L2(kbdev) CSTD_NOP() ++ ++#define KBASE_TIMELINE_POWERED_L2(kbdev) CSTD_NOP() ++ ++#define KBASE_TIMELINE_PM_SEND_EVENT(kbdev, event_type, pm_event_id) CSTD_NOP() ++ ++#define KBASE_TIMELINE_PM_HANDLE_EVENT(kbdev, event_type, pm_event_id) CSTD_NOP() ++ ++#define KBASE_TIMELINE_JOB_START(kctx, js, _consumerof_atom_number) CSTD_NOP() ++ ++#define KBASE_TIMELINE_JOB_STOP(kctx, js, _producerof_atom_number_completed) CSTD_NOP() ++ ++#define KBASE_TIMELINE_PM_CHECKTRANS(kbdev, trace_code) CSTD_NOP() ++ ++#define KBASE_TIMELINE_CONTEXT_ACTIVE(kbdev, count) CSTD_NOP() ++ ++static inline void kbase_timeline_job_slot_submit(struct kbase_device *kbdev, struct kbase_context *kctx, ++ struct kbase_jd_atom *katom, int js) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++} ++ ++static inline void kbase_timeline_job_slot_done(struct kbase_device *kbdev, struct kbase_context *kctx, ++ struct kbase_jd_atom *katom, int js, ++ kbasep_js_atom_done_code done_code) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++} ++ ++static inline void kbase_timeline_pm_send_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event_sent) ++{ ++} ++ ++static inline void kbase_timeline_pm_check_handle_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event) ++{ ++} ++ ++static inline void kbase_timeline_pm_handle_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event) ++{ ++} ++ ++static inline void kbase_timeline_pm_l2_transition_start(struct kbase_device *kbdev) ++{ ++} ++ ++static inline void kbase_timeline_pm_l2_transition_done(struct kbase_device *kbdev) ++{ ++} ++#endif /* CONFIG_MALI_BIFROST_TRACE_TIMELINE */ ++ ++#endif /* _KBASE_TRACE_TIMELINE_H */ ++ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_trace_timeline_defs.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_trace_timeline_defs.h +new file mode 100755 +index 000000000000..156a95a67f4a +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_trace_timeline_defs.h +@@ -0,0 +1,140 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ***** IMPORTANT: THIS IS NOT A NORMAL HEADER FILE ***** ++ * ***** DO NOT INCLUDE DIRECTLY ***** ++ * ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** */ ++ ++/* ++ * Conventions on Event Names: ++ * ++ * - The prefix determines something about how the timeline should be ++ * displayed, and is split up into various parts, separated by underscores: ++ * - 'SW' and 'HW' as the first part will be used to determine whether a ++ * timeline is to do with Software or Hardware - effectively, separate ++ * 'channels' for Software and Hardware ++ * - 'START', 'STOP', 'ENTER', 'LEAVE' can be used in the second part, and ++ * signify related pairs of events - these are optional. ++ * - 'FLOW' indicates a generic event, which can use dependencies ++ * - This gives events such as: ++ * - 'SW_ENTER_FOO' ++ * - 'SW_LEAVE_FOO' ++ * - 'SW_FLOW_BAR_1' ++ * - 'SW_FLOW_BAR_2' ++ * - 'HW_START_BAZ' ++ * - 'HW_STOP_BAZ' ++ * - And an unadorned HW event: ++ * - 'HW_BAZ_FROZBOZ' ++ */ ++ ++/* ++ * Conventions on parameter names: ++ * - anything with 'instance' in the name will have a separate timeline based ++ * on that instances. ++ * - underscored-prefixed parameters will by hidden by default on timelines ++ * ++ * Hence: ++ * - Different job slots have their own 'instance', based on the instance value ++ * - Per-context info (e.g. atoms on a context) have their own 'instance' ++ * (i.e. each context should be on a different timeline) ++ * ++ * Note that globally-shared resources can be tagged with a tgid, but we don't ++ * want an instance per context: ++ * - There's no point having separate Job Slot timelines for each context, that ++ * would be confusing - there's only really 3 job slots! ++ * - There's no point having separate Shader-powered timelines for each ++ * context, that would be confusing - all shader cores (whether it be 4, 8, ++ * etc) are shared in the system. ++ */ ++ ++ /* ++ * CTX events ++ */ ++ /* Separate timelines for each context 'instance'*/ ++ KBASE_TIMELINE_TRACE_CODE(CTX_SET_NR_ATOMS_IN_FLIGHT, "CTX: Atoms in flight", "%d,%d", "_instance_tgid,_value_number_of_atoms"), ++ KBASE_TIMELINE_TRACE_CODE(CTX_FLOW_ATOM_READY, "CTX: Atoms Ready to Run", "%d,%d,%d", "_instance_tgid,_consumerof_atom_number,_producerof_atom_number_ready"), ++ ++ /* ++ * SW Events ++ */ ++ /* Separate timelines for each slot 'instance' */ ++ KBASE_TIMELINE_TRACE_CODE(SW_SET_GPU_SLOT_ACTIVE, "SW: GPU slot active", "%d,%d,%d", "_tgid,_instance_slot,_value_number_of_atoms"), ++ KBASE_TIMELINE_TRACE_CODE(SW_SET_GPU_SLOT_NEXT, "SW: GPU atom in NEXT", "%d,%d,%d", "_tgid,_instance_slot,_value_is_an_atom_in_next"), ++ KBASE_TIMELINE_TRACE_CODE(SW_SET_GPU_SLOT_HEAD, "SW: GPU atom in HEAD", "%d,%d,%d", "_tgid,_instance_slot,_value_is_an_atom_in_head"), ++ KBASE_TIMELINE_TRACE_CODE(SW_SET_GPU_SLOT_STOPPING, "SW: Try Soft-Stop on GPU slot", "%d,%d,%d", "_tgid,_instance_slot,_value_is_slot_stopping"), ++ /* Shader and overall power is shared - can't have separate instances of ++ * it, just tagging with the context */ ++ KBASE_TIMELINE_TRACE_CODE(SW_SET_GPU_POWER_ACTIVE, "SW: GPU power active", "%d,%d", "_tgid,_value_is_power_active"), ++ KBASE_TIMELINE_TRACE_CODE(SW_SET_GPU_POWER_TILER_ACTIVE, "SW: GPU tiler powered", "%d,%d", "_tgid,_value_number_of_tilers"), ++ KBASE_TIMELINE_TRACE_CODE(SW_SET_GPU_POWER_SHADER_ACTIVE, "SW: GPU shaders powered", "%d,%d", "_tgid,_value_number_of_shaders"), ++ KBASE_TIMELINE_TRACE_CODE(SW_SET_GPU_POWER_L2_ACTIVE, "SW: GPU L2 powered", "%d,%d", "_tgid,_value_number_of_l2"), ++ ++ /* SW Power event messaging. _event_type is one from the kbase_pm_event enum */ ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_SEND_EVENT, "SW: PM Send Event", "%d,%d,%d", "_tgid,_event_type,_writerof_pm_event_id"), ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_HANDLE_EVENT, "SW: PM Handle Event", "%d,%d,%d", "_tgid,_event_type,_finalconsumerof_pm_event_id"), ++ /* SW L2 power events */ ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_GPU_POWER_L2_POWERING, "SW: GPU L2 powering", "%d,%d", "_tgid,_writerof_l2_transitioning"), ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_GPU_POWER_L2_ACTIVE, "SW: GPU L2 powering done", "%d,%d", "_tgid,_finalconsumerof_l2_transitioning"), ++ ++ KBASE_TIMELINE_TRACE_CODE(SW_SET_CONTEXT_ACTIVE, "SW: Context Active", "%d,%d", "_tgid,_value_active"), ++ ++ /* ++ * BEGIN: Significant SW Functions that call kbase_pm_check_transitions_nolock() ++ */ ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_DO_POWEROFF_START, "SW: PM CheckTrans from kbase_pm_do_poweroff", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_do_poweroff"), ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_DO_POWEROFF_END, "SW: PM CheckTrans from kbase_pm_do_poweroff", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_do_poweroff"), ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_DO_POWERON_START, "SW: PM CheckTrans from kbase_pm_do_poweron", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_do_poweron"), ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_DO_POWERON_END, "SW: PM CheckTrans from kbase_pm_do_poweron", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_do_poweron"), ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_GPU_INTERRUPT_START, "SW: PM CheckTrans from kbase_gpu_interrupt", "%d,%d", "_tgid,_writerof_pm_checktrans_gpu_interrupt"), ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_GPU_INTERRUPT_END, "SW: PM CheckTrans from kbase_gpu_interrupt", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_gpu_interrupt"), ++ ++ /* ++ * Significant Indirect callers of kbase_pm_check_transitions_nolock() ++ */ ++ /* kbase_pm_request_cores */ ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_START, "SW: PM CheckTrans from kbase_pm_request_cores(shader)", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_request_cores_shader"), ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_END, "SW: PM CheckTrans from kbase_pm_request_cores(shader)", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_request_cores_shader"), ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_TILER_START, "SW: PM CheckTrans from kbase_pm_request_cores(tiler)", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_request_cores_tiler"), ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_TILER_END, "SW: PM CheckTrans from kbase_pm_request_cores(tiler)", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_request_cores_tiler"), ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_TILER_START, "SW: PM CheckTrans from kbase_pm_request_cores(shader+tiler)", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_request_cores_shader_tiler"), ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_TILER_END, "SW: PM CheckTrans from kbase_pm_request_cores(shader+tiler)", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_request_cores_shader_tiler"), ++ /* kbase_pm_release_cores */ ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_START, "SW: PM CheckTrans from kbase_pm_release_cores(shader)", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_release_cores_shader"), ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_END, "SW: PM CheckTrans from kbase_pm_release_cores(shader)", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_release_cores_shader"), ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_TILER_START, "SW: PM CheckTrans from kbase_pm_release_cores(tiler)", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_release_cores_tiler"), ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_TILER_END, "SW: PM CheckTrans from kbase_pm_release_cores(tiler)", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_release_cores_tiler"), ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_TILER_START, "SW: PM CheckTrans from kbase_pm_release_cores(shader+tiler)", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_release_cores_shader_tiler"), ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_TILER_END, "SW: PM CheckTrans from kbase_pm_release_cores(shader+tiler)", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_release_cores_shader_tiler"), ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_DEFERRED_START, "SW: PM CheckTrans from kbasep_pm_do_shader_poweroff_callback", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_do_shader_poweroff_callback"), ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_DEFERRED_END, "SW: PM CheckTrans from kbasep_pm_do_shader_poweroff_callback", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_do_shader_poweroff_callback"), ++ /* ++ * END: SW Functions that call kbase_pm_check_transitions_nolock() ++ */ ++ ++ /* ++ * HW Events ++ */ ++ KBASE_TIMELINE_TRACE_CODE(HW_MMU_FAULT, ++"HW: MMU Fault", "%d,%d,%d", "_tgid,fault_type,fault_stage,asid"), ++ KBASE_TIMELINE_TRACE_CODE(HW_START_GPU_JOB_CHAIN_SW_APPROX, ++"HW: Job Chain start (SW approximated)", "%d,%d,%d", ++"_tgid,job_slot,_consumerof_atom_number_ready"), ++ KBASE_TIMELINE_TRACE_CODE(HW_STOP_GPU_JOB_CHAIN_SW_APPROX, ++"HW: Job Chain stop (SW approximated)", "%d,%d,%d", ++"_tgid,job_slot,_producerof_atom_number_completed") +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_uku.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_uku.h +new file mode 100755 +index 000000000000..cf8ee0572dc5 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_uku.h +@@ -0,0 +1,532 @@ ++/* ++ * ++ * (C) COPYRIGHT 2008-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#ifndef _KBASE_UKU_H_ ++#define _KBASE_UKU_H_ ++ ++#include "mali_uk.h" ++#include "mali_base_kernel.h" ++ ++/* This file needs to support being included from kernel and userside (which use different defines) */ ++#if defined(CONFIG_MALI_BIFROST_ERROR_INJECT) || MALI_ERROR_INJECT_ON ++#define SUPPORT_MALI_ERROR_INJECT ++#endif /* defined(CONFIG_MALI_BIFROST_ERROR_INJECT) || MALI_ERROR_INJECT_ON */ ++#if defined(CONFIG_MALI_BIFROST_NO_MALI) ++#define SUPPORT_MALI_NO_MALI ++#elif defined(MALI_BIFROST_NO_MALI) ++#if MALI_BIFROST_NO_MALI ++#define SUPPORT_MALI_NO_MALI ++#endif ++#endif ++ ++#if defined(SUPPORT_MALI_NO_MALI) || defined(SUPPORT_MALI_ERROR_INJECT) ++#include "backend/gpu/mali_kbase_model_dummy.h" ++#endif ++ ++#include "mali_kbase_gpuprops_types.h" ++ ++/* ++ * 10.1: ++ * - Do mmap in kernel for SAME_VA memory allocations rather then ++ * calling back into the kernel as a 2nd stage of the allocation request. ++ * ++ * 10.2: ++ * - Add KBASE_FUNC_MEM_JIT_INIT which allows clients to request a custom VA ++ * region for use with JIT (ignored on 32-bit platforms) ++ * ++ * 10.3: ++ * - base_jd_core_req typedef-ed to u32 (instead of to u16) ++ * - two flags added: BASE_JD_REQ_SKIP_CACHE_STAT / _END ++ * ++ * 10.4: ++ * - Removed KBASE_FUNC_EXT_BUFFER_LOCK used only in internal tests ++ * ++ * 10.5: ++ * - Reverted to performing mmap in user space so that tools like valgrind work. ++ * ++ * 10.6: ++ * - Add flags input variable to KBASE_FUNC_TLSTREAM_ACQUIRE ++ */ ++#define BASE_UK_VERSION_MAJOR 10 ++#define BASE_UK_VERSION_MINOR 6 ++ ++#define LINUX_UK_BASE_MAGIC 0x80 ++ ++struct kbase_uk_mem_alloc { ++ union uk_header header; ++ /* IN */ ++ u64 va_pages; ++ u64 commit_pages; ++ u64 extent; ++ /* IN/OUT */ ++ u64 flags; ++ /* OUT */ ++ u64 gpu_va; ++ u16 va_alignment; ++ u8 padding[6]; ++}; ++ ++struct kbase_uk_mem_free { ++ union uk_header header; ++ /* IN */ ++ u64 gpu_addr; ++ /* OUT */ ++}; ++ ++struct kbase_uk_mem_alias { ++ union uk_header header; ++ /* IN/OUT */ ++ u64 flags; ++ /* IN */ ++ u64 stride; ++ u64 nents; ++ u64 ai; ++ /* OUT */ ++ u64 gpu_va; ++ u64 va_pages; ++}; ++ ++struct kbase_uk_mem_import { ++ union uk_header header; ++ /* IN */ ++ u64 phandle; ++ u32 type; ++ u32 padding; ++ /* IN/OUT */ ++ u64 flags; ++ /* OUT */ ++ u64 gpu_va; ++ u64 va_pages; ++}; ++ ++struct kbase_uk_mem_flags_change { ++ union uk_header header; ++ /* IN */ ++ u64 gpu_va; ++ u64 flags; ++ u64 mask; ++}; ++ ++struct kbase_uk_job_submit { ++ union uk_header header; ++ /* IN */ ++ u64 addr; ++ u32 nr_atoms; ++ u32 stride; /* bytes between atoms, i.e. sizeof(base_jd_atom_v2) */ ++ /* OUT */ ++}; ++ ++struct kbase_uk_post_term { ++ union uk_header header; ++}; ++ ++struct kbase_uk_sync_now { ++ union uk_header header; ++ ++ /* IN */ ++ struct base_syncset sset; ++ ++ /* OUT */ ++}; ++ ++struct kbase_uk_hwcnt_setup { ++ union uk_header header; ++ ++ /* IN */ ++ u64 dump_buffer; ++ u32 jm_bm; ++ u32 shader_bm; ++ u32 tiler_bm; ++ u32 unused_1; /* keep for backwards compatibility */ ++ u32 mmu_l2_bm; ++ u32 padding; ++ /* OUT */ ++}; ++ ++/** ++ * struct kbase_uk_hwcnt_reader_setup - User/Kernel space data exchange structure ++ * @header: UK structure header ++ * @buffer_count: requested number of dumping buffers ++ * @jm_bm: counters selection bitmask (JM) ++ * @shader_bm: counters selection bitmask (Shader) ++ * @tiler_bm: counters selection bitmask (Tiler) ++ * @mmu_l2_bm: counters selection bitmask (MMU_L2) ++ * @fd: dumping notification file descriptor ++ * ++ * This structure sets up HWC dumper/reader for this context. ++ * Multiple instances can be created for single context. ++ */ ++struct kbase_uk_hwcnt_reader_setup { ++ union uk_header header; ++ ++ /* IN */ ++ u32 buffer_count; ++ u32 jm_bm; ++ u32 shader_bm; ++ u32 tiler_bm; ++ u32 mmu_l2_bm; ++ ++ /* OUT */ ++ s32 fd; ++}; ++ ++struct kbase_uk_hwcnt_dump { ++ union uk_header header; ++}; ++ ++struct kbase_uk_hwcnt_clear { ++ union uk_header header; ++}; ++ ++struct kbase_uk_fence_validate { ++ union uk_header header; ++ /* IN */ ++ s32 fd; ++ u32 padding; ++ /* OUT */ ++}; ++ ++struct kbase_uk_stream_create { ++ union uk_header header; ++ /* IN */ ++ char name[32]; ++ /* OUT */ ++ s32 fd; ++ u32 padding; ++}; ++ ++struct kbase_uk_gpuprops { ++ union uk_header header; ++ ++ /* IN */ ++ struct mali_base_gpu_props props; ++ /* OUT */ ++}; ++ ++struct kbase_uk_mem_query { ++ union uk_header header; ++ /* IN */ ++ u64 gpu_addr; ++#define KBASE_MEM_QUERY_COMMIT_SIZE 1 ++#define KBASE_MEM_QUERY_VA_SIZE 2 ++#define KBASE_MEM_QUERY_FLAGS 3 ++ u64 query; ++ /* OUT */ ++ u64 value; ++}; ++ ++struct kbase_uk_mem_commit { ++ union uk_header header; ++ /* IN */ ++ u64 gpu_addr; ++ u64 pages; ++ /* OUT */ ++ u32 result_subcode; ++ u32 padding; ++}; ++ ++struct kbase_uk_find_cpu_offset { ++ union uk_header header; ++ /* IN */ ++ u64 gpu_addr; ++ u64 cpu_addr; ++ u64 size; ++ /* OUT */ ++ u64 offset; ++}; ++ ++#define KBASE_GET_VERSION_BUFFER_SIZE 64 ++struct kbase_uk_get_ddk_version { ++ union uk_header header; ++ /* OUT */ ++ char version_buffer[KBASE_GET_VERSION_BUFFER_SIZE]; ++ u32 version_string_size; ++ u32 padding; ++}; ++ ++struct kbase_uk_disjoint_query { ++ union uk_header header; ++ /* OUT */ ++ u32 counter; ++ u32 padding; ++}; ++ ++struct kbase_uk_set_flags { ++ union uk_header header; ++ /* IN */ ++ u32 create_flags; ++ u32 padding; ++}; ++ ++#if MALI_UNIT_TEST ++#define TEST_ADDR_COUNT 4 ++#define KBASE_TEST_BUFFER_SIZE 128 ++struct kbase_exported_test_data { ++ u64 test_addr[TEST_ADDR_COUNT]; /**< memory address */ ++ u32 test_addr_pages[TEST_ADDR_COUNT]; /**< memory size in pages */ ++ u64 kctx; /**< base context created by process */ ++ u64 mm; /**< pointer to process address space */ ++ u8 buffer1[KBASE_TEST_BUFFER_SIZE]; /**< unit test defined parameter */ ++ u8 buffer2[KBASE_TEST_BUFFER_SIZE]; /**< unit test defined parameter */ ++}; ++ ++struct kbase_uk_set_test_data { ++ union uk_header header; ++ /* IN */ ++ struct kbase_exported_test_data test_data; ++}; ++ ++#endif /* MALI_UNIT_TEST */ ++ ++#ifdef SUPPORT_MALI_ERROR_INJECT ++struct kbase_uk_error_params { ++ union uk_header header; ++ /* IN */ ++ struct kbase_error_params params; ++}; ++#endif /* SUPPORT_MALI_ERROR_INJECT */ ++ ++#ifdef SUPPORT_MALI_NO_MALI ++struct kbase_uk_model_control_params { ++ union uk_header header; ++ /* IN */ ++ struct kbase_model_control_params params; ++}; ++#endif /* SUPPORT_MALI_NO_MALI */ ++ ++struct kbase_uk_profiling_controls { ++ union uk_header header; ++ u32 profiling_controls[FBDUMP_CONTROL_MAX]; ++}; ++ ++struct kbase_uk_debugfs_mem_profile_add { ++ union uk_header header; ++ u32 len; ++ u32 padding; ++ u64 buf; ++}; ++ ++struct kbase_uk_context_id { ++ union uk_header header; ++ /* OUT */ ++ int id; ++}; ++ ++/** ++ * struct kbase_uk_tlstream_acquire - User/Kernel space data exchange structure ++ * @header: UK structure header ++ * @flags: timeline stream flags ++ * @fd: timeline stream file descriptor ++ * ++ * This structure is used when performing a call to acquire kernel side timeline ++ * stream file descriptor. ++ */ ++struct kbase_uk_tlstream_acquire { ++ union uk_header header; ++ /* IN */ ++ u32 flags; ++ /* OUT */ ++ s32 fd; ++}; ++ ++/** ++ * struct kbase_uk_tlstream_acquire_v10_4 - User/Kernel space data exchange ++ * structure ++ * @header: UK structure header ++ * @fd: timeline stream file descriptor ++ * ++ * This structure is used when performing a call to acquire kernel side timeline ++ * stream file descriptor. ++ */ ++struct kbase_uk_tlstream_acquire_v10_4 { ++ union uk_header header; ++ /* IN */ ++ /* OUT */ ++ s32 fd; ++}; ++ ++/** ++ * struct kbase_uk_tlstream_flush - User/Kernel space data exchange structure ++ * @header: UK structure header ++ * ++ * This structure is used when performing a call to flush kernel side ++ * timeline streams. ++ */ ++struct kbase_uk_tlstream_flush { ++ union uk_header header; ++ /* IN */ ++ /* OUT */ ++}; ++ ++#if MALI_UNIT_TEST ++/** ++ * struct kbase_uk_tlstream_test - User/Kernel space data exchange structure ++ * @header: UK structure header ++ * @tpw_count: number of trace point writers in each context ++ * @msg_delay: time delay between tracepoints from one writer in milliseconds ++ * @msg_count: number of trace points written by one writer ++ * @aux_msg: if non-zero aux messages will be included ++ * ++ * This structure is used when performing a call to start timeline stream test ++ * embedded in kernel. ++ */ ++struct kbase_uk_tlstream_test { ++ union uk_header header; ++ /* IN */ ++ u32 tpw_count; ++ u32 msg_delay; ++ u32 msg_count; ++ u32 aux_msg; ++ /* OUT */ ++}; ++ ++/** ++ * struct kbase_uk_tlstream_stats - User/Kernel space data exchange structure ++ * @header: UK structure header ++ * @bytes_collected: number of bytes read by user ++ * @bytes_generated: number of bytes generated by tracepoints ++ * ++ * This structure is used when performing a call to obtain timeline stream ++ * statistics. ++ */ ++struct kbase_uk_tlstream_stats { ++ union uk_header header; /**< UK structure header. */ ++ /* IN */ ++ /* OUT */ ++ u32 bytes_collected; ++ u32 bytes_generated; ++}; ++#endif /* MALI_UNIT_TEST */ ++ ++/** ++ * struct struct kbase_uk_prfcnt_value for the KBASE_FUNC_SET_PRFCNT_VALUES ioctl ++ * @header: UK structure header ++ * @data: Counter samples for the dummy model ++ * @size:............Size of the counter sample data ++ */ ++struct kbase_uk_prfcnt_values { ++ union uk_header header; ++ /* IN */ ++ u32 *data; ++ u32 size; ++}; ++ ++/** ++ * struct kbase_uk_soft_event_update - User/Kernel space data exchange structure ++ * @header: UK structure header ++ * @evt: the GPU address containing the event ++ * @new_status: the new event status, must be either BASE_JD_SOFT_EVENT_SET or ++ * BASE_JD_SOFT_EVENT_RESET ++ * @flags: reserved for future uses, must be set to 0 ++ * ++ * This structure is used to update the status of a software event. If the ++ * event's status is set to BASE_JD_SOFT_EVENT_SET, any job currently waiting ++ * on this event will complete. ++ */ ++struct kbase_uk_soft_event_update { ++ union uk_header header; ++ /* IN */ ++ u64 evt; ++ u32 new_status; ++ u32 flags; ++}; ++ ++/** ++ * struct kbase_uk_mem_jit_init - User/Kernel space data exchange structure ++ * @header: UK structure header ++ * @va_pages: Number of virtual pages required for JIT ++ * ++ * This structure is used when requesting initialization of JIT. ++ */ ++struct kbase_uk_mem_jit_init { ++ union uk_header header; ++ /* IN */ ++ u64 va_pages; ++}; ++ ++enum kbase_uk_function_id { ++ KBASE_FUNC_MEM_ALLOC = (UK_FUNC_ID + 0), ++ KBASE_FUNC_MEM_IMPORT = (UK_FUNC_ID + 1), ++ KBASE_FUNC_MEM_COMMIT = (UK_FUNC_ID + 2), ++ KBASE_FUNC_MEM_QUERY = (UK_FUNC_ID + 3), ++ KBASE_FUNC_MEM_FREE = (UK_FUNC_ID + 4), ++ KBASE_FUNC_MEM_FLAGS_CHANGE = (UK_FUNC_ID + 5), ++ KBASE_FUNC_MEM_ALIAS = (UK_FUNC_ID + 6), ++ ++ /* UK_FUNC_ID + 7 not in use since BASE_LEGACY_UK6_SUPPORT dropped */ ++ ++ KBASE_FUNC_SYNC = (UK_FUNC_ID + 8), ++ ++ KBASE_FUNC_POST_TERM = (UK_FUNC_ID + 9), ++ ++ KBASE_FUNC_HWCNT_SETUP = (UK_FUNC_ID + 10), ++ KBASE_FUNC_HWCNT_DUMP = (UK_FUNC_ID + 11), ++ KBASE_FUNC_HWCNT_CLEAR = (UK_FUNC_ID + 12), ++ ++ KBASE_FUNC_GPU_PROPS_REG_DUMP = (UK_FUNC_ID + 14), ++ ++ KBASE_FUNC_FIND_CPU_OFFSET = (UK_FUNC_ID + 15), ++ ++ KBASE_FUNC_GET_VERSION = (UK_FUNC_ID + 16), ++ KBASE_FUNC_SET_FLAGS = (UK_FUNC_ID + 18), ++ ++ KBASE_FUNC_SET_TEST_DATA = (UK_FUNC_ID + 19), ++ KBASE_FUNC_INJECT_ERROR = (UK_FUNC_ID + 20), ++ KBASE_FUNC_MODEL_CONTROL = (UK_FUNC_ID + 21), ++ ++ /* UK_FUNC_ID + 22 not in use since BASE_LEGACY_UK8_SUPPORT dropped */ ++ ++ KBASE_FUNC_FENCE_VALIDATE = (UK_FUNC_ID + 23), ++ KBASE_FUNC_STREAM_CREATE = (UK_FUNC_ID + 24), ++ KBASE_FUNC_GET_PROFILING_CONTROLS = (UK_FUNC_ID + 25), ++ KBASE_FUNC_SET_PROFILING_CONTROLS = (UK_FUNC_ID + 26), ++ /* to be used only for testing ++ * purposes, otherwise these controls ++ * are set through gator API */ ++ ++ KBASE_FUNC_DEBUGFS_MEM_PROFILE_ADD = (UK_FUNC_ID + 27), ++ KBASE_FUNC_JOB_SUBMIT = (UK_FUNC_ID + 28), ++ KBASE_FUNC_DISJOINT_QUERY = (UK_FUNC_ID + 29), ++ ++ KBASE_FUNC_GET_CONTEXT_ID = (UK_FUNC_ID + 31), ++ ++ KBASE_FUNC_TLSTREAM_ACQUIRE_V10_4 = (UK_FUNC_ID + 32), ++#if MALI_UNIT_TEST ++ KBASE_FUNC_TLSTREAM_TEST = (UK_FUNC_ID + 33), ++ KBASE_FUNC_TLSTREAM_STATS = (UK_FUNC_ID + 34), ++#endif /* MALI_UNIT_TEST */ ++ KBASE_FUNC_TLSTREAM_FLUSH = (UK_FUNC_ID + 35), ++ ++ KBASE_FUNC_HWCNT_READER_SETUP = (UK_FUNC_ID + 36), ++ ++#ifdef SUPPORT_MALI_NO_MALI ++ KBASE_FUNC_SET_PRFCNT_VALUES = (UK_FUNC_ID + 37), ++#endif ++ ++ KBASE_FUNC_SOFT_EVENT_UPDATE = (UK_FUNC_ID + 38), ++ ++ KBASE_FUNC_MEM_JIT_INIT = (UK_FUNC_ID + 39), ++ ++ KBASE_FUNC_TLSTREAM_ACQUIRE = (UK_FUNC_ID + 40), ++ ++ KBASE_FUNC_MAX ++}; ++ ++#endif /* _KBASE_UKU_H_ */ ++ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_utility.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_utility.c +new file mode 100755 +index 000000000000..be474ff87401 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_utility.c +@@ -0,0 +1,33 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2013, 2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#include ++ ++bool kbasep_list_member_of(const struct list_head *base, struct list_head *entry) ++{ ++ struct list_head *pos = base->next; ++ ++ while (pos != base) { ++ if (pos == entry) ++ return true; ++ ++ pos = pos->next; ++ } ++ return false; ++} +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_utility.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_utility.h +new file mode 100755 +index 000000000000..fd7252dab0de +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_utility.h +@@ -0,0 +1,37 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2013, 2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#ifndef _KBASE_UTILITY_H ++#define _KBASE_UTILITY_H ++ ++#ifndef _KBASE_H_ ++#error "Don't include this file directly, use mali_kbase.h instead" ++#endif ++ ++/** Test whether the given list entry is a member of the given list. ++ * ++ * @param base The head of the list to be tested ++ * @param entry The list entry to be tested ++ * ++ * @return true if entry is a member of base ++ * false otherwise ++ */ ++bool kbasep_list_member_of(const struct list_head *base, struct list_head *entry); ++ ++#endif /* _KBASE_UTILITY_H */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_vinstr.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_vinstr.c +new file mode 100755 +index 000000000000..9c5b2e46c0e5 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_vinstr.c +@@ -0,0 +1,2072 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++/*****************************************************************************/ ++ ++/* Hwcnt reader API version */ ++#define HWCNT_READER_API 1 ++ ++/* The number of nanoseconds in a second. */ ++#define NSECS_IN_SEC 1000000000ull /* ns */ ++ ++/* The time resolution of dumping service. */ ++#define DUMPING_RESOLUTION 500000ull /* ns */ ++ ++/* The maximal supported number of dumping buffers. */ ++#define MAX_BUFFER_COUNT 32 ++ ++/* Size and number of hw counters blocks. */ ++#define NR_CNT_BLOCKS_PER_GROUP 8 ++#define NR_CNT_PER_BLOCK 64 ++#define NR_BYTES_PER_CNT 4 ++#define NR_BYTES_PER_HDR 16 ++#define PRFCNT_EN_MASK_OFFSET 0x8 ++ ++/*****************************************************************************/ ++ ++enum { ++ SHADER_HWCNT_BM, ++ TILER_HWCNT_BM, ++ MMU_L2_HWCNT_BM, ++ JM_HWCNT_BM ++}; ++ ++enum vinstr_state { ++ VINSTR_IDLE, ++ VINSTR_DUMPING, ++ VINSTR_SUSPENDING, ++ VINSTR_SUSPENDED, ++ VINSTR_RESUMING ++}; ++ ++/** ++ * struct kbase_vinstr_context - vinstr context per device ++ * @lock: protects the entire vinstr context ++ * @kbdev: pointer to kbase device ++ * @kctx: pointer to kbase context ++ * @vmap: vinstr vmap for mapping hwcnt dump buffer ++ * @gpu_va: GPU hwcnt dump buffer address ++ * @cpu_va: the CPU side mapping of the hwcnt dump buffer ++ * @dump_size: size of the dump buffer in bytes ++ * @bitmap: current set of counters monitored, not always in sync ++ * with hardware ++ * @reprogram: when true, reprogram hwcnt block with the new set of ++ * counters ++ * @state: vinstr state ++ * @state_lock: protects information about vinstr state ++ * @suspend_waitq: notification queue to trigger state re-validation ++ * @suspend_cnt: reference counter of vinstr's suspend state ++ * @suspend_work: worker to execute on entering suspended state ++ * @resume_work: worker to execute on leaving suspended state ++ * @nclients: number of attached clients, pending or otherwise ++ * @waiting_clients: head of list of clients being periodically sampled ++ * @idle_clients: head of list of clients being idle ++ * @suspended_clients: head of list of clients being suspended ++ * @thread: periodic sampling thread ++ * @waitq: notification queue of sampling thread ++ * @request_pending: request for action for sampling thread ++ * @clients_present: when true, we have at least one client ++ * Note: this variable is in sync. with nclients and is ++ * present to preserve simplicity. Protected by state_lock. ++ */ ++struct kbase_vinstr_context { ++ struct mutex lock; ++ struct kbase_device *kbdev; ++ struct kbase_context *kctx; ++ ++ struct kbase_vmap_struct vmap; ++ u64 gpu_va; ++ void *cpu_va; ++ size_t dump_size; ++ u32 bitmap[4]; ++ bool reprogram; ++ ++ enum vinstr_state state; ++ struct spinlock state_lock; ++ wait_queue_head_t suspend_waitq; ++ unsigned int suspend_cnt; ++ struct work_struct suspend_work; ++ struct work_struct resume_work; ++ ++ u32 nclients; ++ struct list_head waiting_clients; ++ struct list_head idle_clients; ++ struct list_head suspended_clients; ++ ++ struct task_struct *thread; ++ wait_queue_head_t waitq; ++ atomic_t request_pending; ++ ++ bool clients_present; ++}; ++ ++/** ++ * struct kbase_vinstr_client - a vinstr client attached to a vinstr context ++ * @vinstr_ctx: vinstr context client is attached to ++ * @list: node used to attach this client to list in vinstr context ++ * @buffer_count: number of buffers this client is using ++ * @event_mask: events this client reacts to ++ * @dump_size: size of one dump buffer in bytes ++ * @bitmap: bitmap request for JM, TILER, SHADER and MMU counters ++ * @legacy_buffer: userspace hwcnt dump buffer (legacy interface) ++ * @kernel_buffer: kernel hwcnt dump buffer (kernel client interface) ++ * @accum_buffer: temporary accumulation buffer for preserving counters ++ * @dump_time: next time this clients shall request hwcnt dump ++ * @dump_interval: interval between periodic hwcnt dumps ++ * @dump_buffers: kernel hwcnt dump buffers allocated by this client ++ * @dump_buffers_meta: metadata of dump buffers ++ * @meta_idx: index of metadata being accessed by userspace ++ * @read_idx: index of buffer read by userspace ++ * @write_idx: index of buffer being written by dumping service ++ * @waitq: client's notification queue ++ * @pending: when true, client has attached but hwcnt not yet updated ++ */ ++struct kbase_vinstr_client { ++ struct kbase_vinstr_context *vinstr_ctx; ++ struct list_head list; ++ unsigned int buffer_count; ++ u32 event_mask; ++ size_t dump_size; ++ u32 bitmap[4]; ++ void __user *legacy_buffer; ++ void *kernel_buffer; ++ void *accum_buffer; ++ u64 dump_time; ++ u32 dump_interval; ++ char *dump_buffers; ++ struct kbase_hwcnt_reader_metadata *dump_buffers_meta; ++ atomic_t meta_idx; ++ atomic_t read_idx; ++ atomic_t write_idx; ++ wait_queue_head_t waitq; ++ bool pending; ++}; ++ ++/** ++ * struct kbasep_vinstr_wake_up_timer - vinstr service thread wake up timer ++ * @hrtimer: high resolution timer ++ * @vinstr_ctx: vinstr context ++ */ ++struct kbasep_vinstr_wake_up_timer { ++ struct hrtimer hrtimer; ++ struct kbase_vinstr_context *vinstr_ctx; ++}; ++ ++/*****************************************************************************/ ++ ++static int kbasep_vinstr_service_task(void *data); ++ ++static unsigned int kbasep_vinstr_hwcnt_reader_poll( ++ struct file *filp, ++ poll_table *wait); ++static long kbasep_vinstr_hwcnt_reader_ioctl( ++ struct file *filp, ++ unsigned int cmd, ++ unsigned long arg); ++static int kbasep_vinstr_hwcnt_reader_mmap( ++ struct file *filp, ++ struct vm_area_struct *vma); ++static int kbasep_vinstr_hwcnt_reader_release( ++ struct inode *inode, ++ struct file *filp); ++ ++/* The timeline stream file operations structure. */ ++static const struct file_operations vinstr_client_fops = { ++ .poll = kbasep_vinstr_hwcnt_reader_poll, ++ .unlocked_ioctl = kbasep_vinstr_hwcnt_reader_ioctl, ++ .compat_ioctl = kbasep_vinstr_hwcnt_reader_ioctl, ++ .mmap = kbasep_vinstr_hwcnt_reader_mmap, ++ .release = kbasep_vinstr_hwcnt_reader_release, ++}; ++ ++/*****************************************************************************/ ++ ++static int enable_hwcnt(struct kbase_vinstr_context *vinstr_ctx) ++{ ++ struct kbase_context *kctx = vinstr_ctx->kctx; ++ struct kbase_device *kbdev = kctx->kbdev; ++ struct kbase_uk_hwcnt_setup setup; ++ int err; ++ ++ setup.dump_buffer = vinstr_ctx->gpu_va; ++ setup.jm_bm = vinstr_ctx->bitmap[JM_HWCNT_BM]; ++ setup.tiler_bm = vinstr_ctx->bitmap[TILER_HWCNT_BM]; ++ setup.shader_bm = vinstr_ctx->bitmap[SHADER_HWCNT_BM]; ++ setup.mmu_l2_bm = vinstr_ctx->bitmap[MMU_L2_HWCNT_BM]; ++ ++ /* Mark the context as active so the GPU is kept turned on */ ++ /* A suspend won't happen here, because we're in a syscall from a ++ * userspace thread. */ ++ kbase_pm_context_active(kbdev); ++ ++ /* Schedule the context in */ ++ kbasep_js_schedule_privileged_ctx(kbdev, kctx); ++ err = kbase_instr_hwcnt_enable_internal(kbdev, kctx, &setup); ++ if (err) { ++ /* Release the context. This had its own Power Manager Active ++ * reference */ ++ kbasep_js_release_privileged_ctx(kbdev, kctx); ++ ++ /* Also release our Power Manager Active reference */ ++ kbase_pm_context_idle(kbdev); ++ } ++ ++ return err; ++} ++ ++static void disable_hwcnt(struct kbase_vinstr_context *vinstr_ctx) ++{ ++ struct kbase_context *kctx = vinstr_ctx->kctx; ++ struct kbase_device *kbdev = kctx->kbdev; ++ int err; ++ ++ err = kbase_instr_hwcnt_disable_internal(kctx); ++ if (err) { ++ dev_warn(kbdev->dev, "Failed to disable HW counters (ctx:%p)", ++ kctx); ++ return; ++ } ++ ++ /* Release the context. This had its own Power Manager Active reference. */ ++ kbasep_js_release_privileged_ctx(kbdev, kctx); ++ ++ /* Also release our Power Manager Active reference. */ ++ kbase_pm_context_idle(kbdev); ++ ++ dev_dbg(kbdev->dev, "HW counters dumping disabled for context %p", kctx); ++} ++ ++static int reprogram_hwcnt(struct kbase_vinstr_context *vinstr_ctx) ++{ ++ disable_hwcnt(vinstr_ctx); ++ return enable_hwcnt(vinstr_ctx); ++} ++ ++static void hwcnt_bitmap_set(u32 dst[4], u32 src[4]) ++{ ++ dst[JM_HWCNT_BM] = src[JM_HWCNT_BM]; ++ dst[TILER_HWCNT_BM] = src[TILER_HWCNT_BM]; ++ dst[SHADER_HWCNT_BM] = src[SHADER_HWCNT_BM]; ++ dst[MMU_L2_HWCNT_BM] = src[MMU_L2_HWCNT_BM]; ++} ++ ++static void hwcnt_bitmap_union(u32 dst[4], u32 src[4]) ++{ ++ dst[JM_HWCNT_BM] |= src[JM_HWCNT_BM]; ++ dst[TILER_HWCNT_BM] |= src[TILER_HWCNT_BM]; ++ dst[SHADER_HWCNT_BM] |= src[SHADER_HWCNT_BM]; ++ dst[MMU_L2_HWCNT_BM] |= src[MMU_L2_HWCNT_BM]; ++} ++ ++size_t kbase_vinstr_dump_size(struct kbase_device *kbdev) ++{ ++ size_t dump_size; ++ ++#ifndef CONFIG_MALI_BIFROST_NO_MALI ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_V4)) { ++ u32 nr_cg; ++ ++ nr_cg = kbdev->gpu_props.num_core_groups; ++ dump_size = nr_cg * NR_CNT_BLOCKS_PER_GROUP * ++ NR_CNT_PER_BLOCK * ++ NR_BYTES_PER_CNT; ++ } else ++#endif /* CONFIG_MALI_BIFROST_NO_MALI */ ++ { ++ /* assume v5 for now */ ++ base_gpu_props *props = &kbdev->gpu_props.props; ++ u32 nr_l2 = props->l2_props.num_l2_slices; ++ u64 core_mask = props->coherency_info.group[0].core_mask; ++ u32 nr_blocks = fls64(core_mask); ++ ++ /* JM and tiler counter blocks are always present */ ++ dump_size = (2 + nr_l2 + nr_blocks) * ++ NR_CNT_PER_BLOCK * ++ NR_BYTES_PER_CNT; ++ } ++ return dump_size; ++} ++KBASE_EXPORT_TEST_API(kbase_vinstr_dump_size); ++ ++static size_t kbasep_vinstr_dump_size_ctx( ++ struct kbase_vinstr_context *vinstr_ctx) ++{ ++ return kbase_vinstr_dump_size(vinstr_ctx->kctx->kbdev); ++} ++ ++static int kbasep_vinstr_map_kernel_dump_buffer( ++ struct kbase_vinstr_context *vinstr_ctx) ++{ ++ struct kbase_va_region *reg; ++ struct kbase_context *kctx = vinstr_ctx->kctx; ++ u64 flags, nr_pages; ++ ++ flags = BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_GPU_WR; ++ vinstr_ctx->dump_size = kbasep_vinstr_dump_size_ctx(vinstr_ctx); ++ nr_pages = PFN_UP(vinstr_ctx->dump_size); ++ ++ reg = kbase_mem_alloc(kctx, nr_pages, nr_pages, 0, &flags, ++ &vinstr_ctx->gpu_va); ++ if (!reg) ++ return -ENOMEM; ++ ++ vinstr_ctx->cpu_va = kbase_vmap( ++ kctx, ++ vinstr_ctx->gpu_va, ++ vinstr_ctx->dump_size, ++ &vinstr_ctx->vmap); ++ if (!vinstr_ctx->cpu_va) { ++ kbase_mem_free(kctx, vinstr_ctx->gpu_va); ++ return -ENOMEM; ++ } ++ ++ return 0; ++} ++ ++static void kbasep_vinstr_unmap_kernel_dump_buffer( ++ struct kbase_vinstr_context *vinstr_ctx) ++{ ++ struct kbase_context *kctx = vinstr_ctx->kctx; ++ ++ kbase_vunmap(kctx, &vinstr_ctx->vmap); ++ kbase_mem_free(kctx, vinstr_ctx->gpu_va); ++} ++ ++/** ++ * kbasep_vinstr_create_kctx - create kernel context for vinstr ++ * @vinstr_ctx: vinstr context ++ * Return: zero on success ++ */ ++static int kbasep_vinstr_create_kctx(struct kbase_vinstr_context *vinstr_ctx) ++{ ++ struct kbase_device *kbdev = vinstr_ctx->kbdev; ++ struct kbasep_kctx_list_element *element = NULL; ++ unsigned long flags; ++ bool enable_backend = false; ++ int err; ++ ++ vinstr_ctx->kctx = kbase_create_context(vinstr_ctx->kbdev, true); ++ if (!vinstr_ctx->kctx) ++ return -ENOMEM; ++ ++ /* Map the master kernel dump buffer. The HW dumps the counters ++ * into this memory region. */ ++ err = kbasep_vinstr_map_kernel_dump_buffer(vinstr_ctx); ++ if (err) ++ goto failed_map; ++ ++ /* Add kernel context to list of contexts associated with device. */ ++ element = kzalloc(sizeof(*element), GFP_KERNEL); ++ if (element) { ++ element->kctx = vinstr_ctx->kctx; ++ mutex_lock(&kbdev->kctx_list_lock); ++ list_add(&element->link, &kbdev->kctx_list); ++ ++ /* Inform timeline client about new context. ++ * Do this while holding the lock to avoid tracepoint ++ * being created in both body and summary stream. */ ++ KBASE_TLSTREAM_TL_NEW_CTX( ++ vinstr_ctx->kctx, ++ vinstr_ctx->kctx->id, ++ (u32)(vinstr_ctx->kctx->tgid)); ++ ++ mutex_unlock(&kbdev->kctx_list_lock); ++ } else { ++ /* Don't treat this as a fail - just warn about it. */ ++ dev_warn(kbdev->dev, ++ "couldn't add kctx to kctx_list\n"); ++ } ++ ++ /* Don't enable hardware counters if vinstr is suspended. ++ * Note that vinstr resume code is run under vinstr context lock, ++ * lower layer will be enabled as needed on resume. */ ++ spin_lock_irqsave(&vinstr_ctx->state_lock, flags); ++ if (VINSTR_IDLE == vinstr_ctx->state) ++ enable_backend = true; ++ vinstr_ctx->clients_present = true; ++ spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); ++ if (enable_backend) ++ err = enable_hwcnt(vinstr_ctx); ++ if (err) ++ goto failed_enable; ++ ++ vinstr_ctx->thread = kthread_run( ++ kbasep_vinstr_service_task, ++ vinstr_ctx, ++ "mali_vinstr_service"); ++ if (IS_ERR(vinstr_ctx->thread)) { ++ err = PTR_ERR(vinstr_ctx->thread); ++ goto failed_kthread; ++ } ++ ++ return 0; ++ ++failed_kthread: ++ disable_hwcnt(vinstr_ctx); ++failed_enable: ++ spin_lock_irqsave(&vinstr_ctx->state_lock, flags); ++ vinstr_ctx->clients_present = false; ++ spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); ++ kbasep_vinstr_unmap_kernel_dump_buffer(vinstr_ctx); ++ if (element) { ++ mutex_lock(&kbdev->kctx_list_lock); ++ list_del(&element->link); ++ kfree(element); ++ mutex_unlock(&kbdev->kctx_list_lock); ++ KBASE_TLSTREAM_TL_DEL_CTX(vinstr_ctx->kctx); ++ } ++failed_map: ++ kbase_destroy_context(vinstr_ctx->kctx); ++ vinstr_ctx->kctx = NULL; ++ return err; ++} ++ ++/** ++ * kbasep_vinstr_destroy_kctx - destroy vinstr's kernel context ++ * @vinstr_ctx: vinstr context ++ */ ++static void kbasep_vinstr_destroy_kctx(struct kbase_vinstr_context *vinstr_ctx) ++{ ++ struct kbase_device *kbdev = vinstr_ctx->kbdev; ++ struct kbasep_kctx_list_element *element; ++ struct kbasep_kctx_list_element *tmp; ++ bool found = false; ++ unsigned long flags; ++ ++ /* Release hw counters dumping resources. */ ++ vinstr_ctx->thread = NULL; ++ disable_hwcnt(vinstr_ctx); ++ kbasep_vinstr_unmap_kernel_dump_buffer(vinstr_ctx); ++ ++ /* Simplify state transitions by specifying that we have no clients. */ ++ spin_lock_irqsave(&vinstr_ctx->state_lock, flags); ++ vinstr_ctx->clients_present = false; ++ spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); ++ ++ /* Remove kernel context from the device's contexts list. */ ++ mutex_lock(&kbdev->kctx_list_lock); ++ list_for_each_entry_safe(element, tmp, &kbdev->kctx_list, link) { ++ if (element->kctx == vinstr_ctx->kctx) { ++ list_del(&element->link); ++ kfree(element); ++ found = true; ++ } ++ } ++ mutex_unlock(&kbdev->kctx_list_lock); ++ ++ if (!found) ++ dev_warn(kbdev->dev, "kctx not in kctx_list\n"); ++ ++ /* Destroy context. */ ++ kbase_destroy_context(vinstr_ctx->kctx); ++ ++ /* Inform timeline client about context destruction. */ ++ KBASE_TLSTREAM_TL_DEL_CTX(vinstr_ctx->kctx); ++ ++ vinstr_ctx->kctx = NULL; ++} ++ ++/** ++ * kbasep_vinstr_attach_client - Attach a client to the vinstr core ++ * @vinstr_ctx: vinstr context ++ * @buffer_count: requested number of dump buffers ++ * @bitmap: bitmaps describing which counters should be enabled ++ * @argp: pointer where notification descriptor shall be stored ++ * @kernel_buffer: pointer to kernel side buffer ++ * ++ * Return: vinstr opaque client handle or NULL on failure ++ */ ++static struct kbase_vinstr_client *kbasep_vinstr_attach_client( ++ struct kbase_vinstr_context *vinstr_ctx, u32 buffer_count, ++ u32 bitmap[4], void *argp, void *kernel_buffer) ++{ ++ struct task_struct *thread = NULL; ++ struct kbase_vinstr_client *cli; ++ ++ KBASE_DEBUG_ASSERT(vinstr_ctx); ++ ++ if (buffer_count > MAX_BUFFER_COUNT ++ || (buffer_count & (buffer_count - 1))) ++ return NULL; ++ ++ cli = kzalloc(sizeof(*cli), GFP_KERNEL); ++ if (!cli) ++ return NULL; ++ ++ cli->vinstr_ctx = vinstr_ctx; ++ cli->buffer_count = buffer_count; ++ cli->event_mask = ++ (1 << BASE_HWCNT_READER_EVENT_MANUAL) | ++ (1 << BASE_HWCNT_READER_EVENT_PERIODIC); ++ cli->pending = true; ++ ++ hwcnt_bitmap_set(cli->bitmap, bitmap); ++ ++ mutex_lock(&vinstr_ctx->lock); ++ ++ hwcnt_bitmap_union(vinstr_ctx->bitmap, cli->bitmap); ++ vinstr_ctx->reprogram = true; ++ ++ /* If this is the first client, create the vinstr kbase ++ * context. This context is permanently resident until the ++ * last client exits. */ ++ if (!vinstr_ctx->nclients) { ++ hwcnt_bitmap_set(vinstr_ctx->bitmap, cli->bitmap); ++ if (kbasep_vinstr_create_kctx(vinstr_ctx) < 0) ++ goto error; ++ ++ vinstr_ctx->reprogram = false; ++ cli->pending = false; ++ } ++ ++ /* The GPU resets the counter block every time there is a request ++ * to dump it. We need a per client kernel buffer for accumulating ++ * the counters. */ ++ cli->dump_size = kbasep_vinstr_dump_size_ctx(vinstr_ctx); ++ cli->accum_buffer = kzalloc(cli->dump_size, GFP_KERNEL); ++ if (!cli->accum_buffer) ++ goto error; ++ ++ /* Prepare buffers. */ ++ if (cli->buffer_count) { ++ int *fd = (int *)argp; ++ size_t tmp; ++ ++ /* Allocate area for buffers metadata storage. */ ++ tmp = sizeof(struct kbase_hwcnt_reader_metadata) * ++ cli->buffer_count; ++ cli->dump_buffers_meta = kmalloc(tmp, GFP_KERNEL); ++ if (!cli->dump_buffers_meta) ++ goto error; ++ ++ /* Allocate required number of dumping buffers. */ ++ cli->dump_buffers = (char *)__get_free_pages( ++ GFP_KERNEL | __GFP_ZERO, ++ get_order(cli->dump_size * cli->buffer_count)); ++ if (!cli->dump_buffers) ++ goto error; ++ ++ /* Create descriptor for user-kernel data exchange. */ ++ *fd = anon_inode_getfd( ++ "[mali_vinstr_desc]", ++ &vinstr_client_fops, ++ cli, ++ O_RDONLY | O_CLOEXEC); ++ if (0 > *fd) ++ goto error; ++ } else if (kernel_buffer) { ++ cli->kernel_buffer = kernel_buffer; ++ } else { ++ cli->legacy_buffer = (void __user *)argp; ++ } ++ ++ atomic_set(&cli->read_idx, 0); ++ atomic_set(&cli->meta_idx, 0); ++ atomic_set(&cli->write_idx, 0); ++ init_waitqueue_head(&cli->waitq); ++ ++ vinstr_ctx->nclients++; ++ list_add(&cli->list, &vinstr_ctx->idle_clients); ++ ++ mutex_unlock(&vinstr_ctx->lock); ++ ++ return cli; ++ ++error: ++ kfree(cli->dump_buffers_meta); ++ if (cli->dump_buffers) ++ free_pages( ++ (unsigned long)cli->dump_buffers, ++ get_order(cli->dump_size * cli->buffer_count)); ++ kfree(cli->accum_buffer); ++ if (!vinstr_ctx->nclients && vinstr_ctx->kctx) { ++ thread = vinstr_ctx->thread; ++ kbasep_vinstr_destroy_kctx(vinstr_ctx); ++ } ++ kfree(cli); ++ ++ mutex_unlock(&vinstr_ctx->lock); ++ ++ /* Thread must be stopped after lock is released. */ ++ if (thread) ++ kthread_stop(thread); ++ ++ return NULL; ++} ++ ++void kbase_vinstr_detach_client(struct kbase_vinstr_client *cli) ++{ ++ struct kbase_vinstr_context *vinstr_ctx; ++ struct kbase_vinstr_client *iter, *tmp; ++ struct task_struct *thread = NULL; ++ u32 zerobitmap[4] = { 0 }; ++ int cli_found = 0; ++ ++ KBASE_DEBUG_ASSERT(cli); ++ vinstr_ctx = cli->vinstr_ctx; ++ KBASE_DEBUG_ASSERT(vinstr_ctx); ++ ++ mutex_lock(&vinstr_ctx->lock); ++ ++ list_for_each_entry_safe(iter, tmp, &vinstr_ctx->idle_clients, list) { ++ if (iter == cli) { ++ vinstr_ctx->reprogram = true; ++ cli_found = 1; ++ list_del(&iter->list); ++ break; ++ } ++ } ++ if (!cli_found) { ++ list_for_each_entry_safe( ++ iter, tmp, &vinstr_ctx->waiting_clients, list) { ++ if (iter == cli) { ++ vinstr_ctx->reprogram = true; ++ cli_found = 1; ++ list_del(&iter->list); ++ break; ++ } ++ } ++ } ++ KBASE_DEBUG_ASSERT(cli_found); ++ ++ kfree(cli->dump_buffers_meta); ++ free_pages( ++ (unsigned long)cli->dump_buffers, ++ get_order(cli->dump_size * cli->buffer_count)); ++ kfree(cli->accum_buffer); ++ kfree(cli); ++ ++ vinstr_ctx->nclients--; ++ if (!vinstr_ctx->nclients) { ++ thread = vinstr_ctx->thread; ++ kbasep_vinstr_destroy_kctx(vinstr_ctx); ++ } ++ ++ /* Rebuild context bitmap now that the client has detached */ ++ hwcnt_bitmap_set(vinstr_ctx->bitmap, zerobitmap); ++ list_for_each_entry(iter, &vinstr_ctx->idle_clients, list) ++ hwcnt_bitmap_union(vinstr_ctx->bitmap, iter->bitmap); ++ list_for_each_entry(iter, &vinstr_ctx->waiting_clients, list) ++ hwcnt_bitmap_union(vinstr_ctx->bitmap, iter->bitmap); ++ ++ mutex_unlock(&vinstr_ctx->lock); ++ ++ /* Thread must be stopped after lock is released. */ ++ if (thread) ++ kthread_stop(thread); ++} ++KBASE_EXPORT_TEST_API(kbase_vinstr_detach_client); ++ ++/* Accumulate counters in the dump buffer */ ++static void accum_dump_buffer(void *dst, void *src, size_t dump_size) ++{ ++ size_t block_size = NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT; ++ u32 *d = dst; ++ u32 *s = src; ++ size_t i, j; ++ ++ for (i = 0; i < dump_size; i += block_size) { ++ /* skip over the header block */ ++ d += NR_BYTES_PER_HDR / sizeof(u32); ++ s += NR_BYTES_PER_HDR / sizeof(u32); ++ for (j = 0; j < (block_size - NR_BYTES_PER_HDR) / sizeof(u32); j++) { ++ /* saturate result if addition would result in wraparound */ ++ if (U32_MAX - *d < *s) ++ *d = U32_MAX; ++ else ++ *d += *s; ++ d++; ++ s++; ++ } ++ } ++} ++ ++/* This is the Midgard v4 patch function. It copies the headers for each ++ * of the defined blocks from the master kernel buffer and then patches up ++ * the performance counter enable mask for each of the blocks to exclude ++ * counters that were not requested by the client. */ ++static void patch_dump_buffer_hdr_v4( ++ struct kbase_vinstr_context *vinstr_ctx, ++ struct kbase_vinstr_client *cli) ++{ ++ u32 *mask; ++ u8 *dst = cli->accum_buffer; ++ u8 *src = vinstr_ctx->cpu_va; ++ u32 nr_cg = vinstr_ctx->kctx->kbdev->gpu_props.num_core_groups; ++ size_t i, group_size, group; ++ enum { ++ SC0_BASE = 0 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT, ++ SC1_BASE = 1 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT, ++ SC2_BASE = 2 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT, ++ SC3_BASE = 3 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT, ++ TILER_BASE = 4 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT, ++ MMU_L2_BASE = 5 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT, ++ JM_BASE = 7 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT ++ }; ++ ++ group_size = NR_CNT_BLOCKS_PER_GROUP * ++ NR_CNT_PER_BLOCK * ++ NR_BYTES_PER_CNT; ++ for (i = 0; i < nr_cg; i++) { ++ group = i * group_size; ++ /* copy shader core headers */ ++ memcpy(&dst[group + SC0_BASE], &src[group + SC0_BASE], ++ NR_BYTES_PER_HDR); ++ memcpy(&dst[group + SC1_BASE], &src[group + SC1_BASE], ++ NR_BYTES_PER_HDR); ++ memcpy(&dst[group + SC2_BASE], &src[group + SC2_BASE], ++ NR_BYTES_PER_HDR); ++ memcpy(&dst[group + SC3_BASE], &src[group + SC3_BASE], ++ NR_BYTES_PER_HDR); ++ ++ /* copy tiler header */ ++ memcpy(&dst[group + TILER_BASE], &src[group + TILER_BASE], ++ NR_BYTES_PER_HDR); ++ ++ /* copy mmu header */ ++ memcpy(&dst[group + MMU_L2_BASE], &src[group + MMU_L2_BASE], ++ NR_BYTES_PER_HDR); ++ ++ /* copy job manager header */ ++ memcpy(&dst[group + JM_BASE], &src[group + JM_BASE], ++ NR_BYTES_PER_HDR); ++ ++ /* patch the shader core enable mask */ ++ mask = (u32 *)&dst[group + SC0_BASE + PRFCNT_EN_MASK_OFFSET]; ++ *mask &= cli->bitmap[SHADER_HWCNT_BM]; ++ mask = (u32 *)&dst[group + SC1_BASE + PRFCNT_EN_MASK_OFFSET]; ++ *mask &= cli->bitmap[SHADER_HWCNT_BM]; ++ mask = (u32 *)&dst[group + SC2_BASE + PRFCNT_EN_MASK_OFFSET]; ++ *mask &= cli->bitmap[SHADER_HWCNT_BM]; ++ mask = (u32 *)&dst[group + SC3_BASE + PRFCNT_EN_MASK_OFFSET]; ++ *mask &= cli->bitmap[SHADER_HWCNT_BM]; ++ ++ /* patch the tiler core enable mask */ ++ mask = (u32 *)&dst[group + TILER_BASE + PRFCNT_EN_MASK_OFFSET]; ++ *mask &= cli->bitmap[TILER_HWCNT_BM]; ++ ++ /* patch the mmu core enable mask */ ++ mask = (u32 *)&dst[group + MMU_L2_BASE + PRFCNT_EN_MASK_OFFSET]; ++ *mask &= cli->bitmap[MMU_L2_HWCNT_BM]; ++ ++ /* patch the job manager enable mask */ ++ mask = (u32 *)&dst[group + JM_BASE + PRFCNT_EN_MASK_OFFSET]; ++ *mask &= cli->bitmap[JM_HWCNT_BM]; ++ } ++} ++ ++/* This is the Midgard v5 patch function. It copies the headers for each ++ * of the defined blocks from the master kernel buffer and then patches up ++ * the performance counter enable mask for each of the blocks to exclude ++ * counters that were not requested by the client. */ ++static void patch_dump_buffer_hdr_v5( ++ struct kbase_vinstr_context *vinstr_ctx, ++ struct kbase_vinstr_client *cli) ++{ ++ struct kbase_device *kbdev = vinstr_ctx->kctx->kbdev; ++ u32 i, nr_l2; ++ u64 core_mask; ++ u32 *mask; ++ u8 *dst = cli->accum_buffer; ++ u8 *src = vinstr_ctx->cpu_va; ++ size_t block_size = NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT; ++ ++ /* copy and patch job manager header */ ++ memcpy(dst, src, NR_BYTES_PER_HDR); ++ mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET]; ++ *mask &= cli->bitmap[JM_HWCNT_BM]; ++ dst += block_size; ++ src += block_size; ++ ++ /* copy and patch tiler header */ ++ memcpy(dst, src, NR_BYTES_PER_HDR); ++ mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET]; ++ *mask &= cli->bitmap[TILER_HWCNT_BM]; ++ dst += block_size; ++ src += block_size; ++ ++ /* copy and patch MMU/L2C headers */ ++ nr_l2 = kbdev->gpu_props.props.l2_props.num_l2_slices; ++ for (i = 0; i < nr_l2; i++) { ++ memcpy(dst, src, NR_BYTES_PER_HDR); ++ mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET]; ++ *mask &= cli->bitmap[MMU_L2_HWCNT_BM]; ++ dst += block_size; ++ src += block_size; ++ } ++ ++ /* copy and patch shader core headers */ ++ core_mask = kbdev->gpu_props.props.coherency_info.group[0].core_mask; ++ while (0ull != core_mask) { ++ memcpy(dst, src, NR_BYTES_PER_HDR); ++ if (0ull != (core_mask & 1ull)) { ++ /* if block is not reserved update header */ ++ mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET]; ++ *mask &= cli->bitmap[SHADER_HWCNT_BM]; ++ } ++ dst += block_size; ++ src += block_size; ++ ++ core_mask >>= 1; ++ } ++} ++ ++/** ++ * accum_clients - accumulate dumped hw counters for all known clients ++ * @vinstr_ctx: vinstr context ++ */ ++static void accum_clients(struct kbase_vinstr_context *vinstr_ctx) ++{ ++ struct kbase_vinstr_client *iter; ++ int v4 = 0; ++ ++#ifndef CONFIG_MALI_BIFROST_NO_MALI ++ v4 = kbase_hw_has_feature(vinstr_ctx->kbdev, BASE_HW_FEATURE_V4); ++#endif ++ ++ list_for_each_entry(iter, &vinstr_ctx->idle_clients, list) { ++ /* Don't bother accumulating clients whose hwcnt requests ++ * have not yet been honoured. */ ++ if (iter->pending) ++ continue; ++ if (v4) ++ patch_dump_buffer_hdr_v4(vinstr_ctx, iter); ++ else ++ patch_dump_buffer_hdr_v5(vinstr_ctx, iter); ++ accum_dump_buffer( ++ iter->accum_buffer, ++ vinstr_ctx->cpu_va, ++ iter->dump_size); ++ } ++ list_for_each_entry(iter, &vinstr_ctx->waiting_clients, list) { ++ /* Don't bother accumulating clients whose hwcnt requests ++ * have not yet been honoured. */ ++ if (iter->pending) ++ continue; ++ if (v4) ++ patch_dump_buffer_hdr_v4(vinstr_ctx, iter); ++ else ++ patch_dump_buffer_hdr_v5(vinstr_ctx, iter); ++ accum_dump_buffer( ++ iter->accum_buffer, ++ vinstr_ctx->cpu_va, ++ iter->dump_size); ++ } ++} ++ ++/*****************************************************************************/ ++ ++/** ++ * kbasep_vinstr_get_timestamp - return timestamp ++ * ++ * Function returns timestamp value based on raw monotonic timer. Value will ++ * wrap around zero in case of overflow. ++ * ++ * Return: timestamp value ++ */ ++static u64 kbasep_vinstr_get_timestamp(void) ++{ ++ struct timespec ts; ++ ++ getrawmonotonic(&ts); ++ return (u64)ts.tv_sec * NSECS_IN_SEC + ts.tv_nsec; ++} ++ ++/** ++ * kbasep_vinstr_add_dump_request - register client's dumping request ++ * @cli: requesting client ++ * @waiting_clients: list of pending dumping requests ++ */ ++static void kbasep_vinstr_add_dump_request( ++ struct kbase_vinstr_client *cli, ++ struct list_head *waiting_clients) ++{ ++ struct kbase_vinstr_client *tmp; ++ ++ if (list_empty(waiting_clients)) { ++ list_add(&cli->list, waiting_clients); ++ return; ++ } ++ list_for_each_entry(tmp, waiting_clients, list) { ++ if (tmp->dump_time > cli->dump_time) { ++ list_add_tail(&cli->list, &tmp->list); ++ return; ++ } ++ } ++ list_add_tail(&cli->list, waiting_clients); ++} ++ ++/** ++ * kbasep_vinstr_collect_and_accumulate - collect hw counters via low level ++ * dump and accumulate them for known ++ * clients ++ * @vinstr_ctx: vinstr context ++ * @timestamp: pointer where collection timestamp will be recorded ++ * ++ * Return: zero on success ++ */ ++static int kbasep_vinstr_collect_and_accumulate( ++ struct kbase_vinstr_context *vinstr_ctx, u64 *timestamp) ++{ ++ unsigned long flags; ++ int rcode; ++ ++#ifdef CONFIG_MALI_BIFROST_NO_MALI ++ /* The dummy model needs the CPU mapping. */ ++ gpu_model_set_dummy_prfcnt_base_cpu(vinstr_ctx->cpu_va); ++#endif ++ ++ spin_lock_irqsave(&vinstr_ctx->state_lock, flags); ++ if (VINSTR_IDLE != vinstr_ctx->state) { ++ spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); ++ return -EAGAIN; ++ } else { ++ vinstr_ctx->state = VINSTR_DUMPING; ++ } ++ spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); ++ ++ /* Request HW counters dump. ++ * Disable preemption to make dump timestamp more accurate. */ ++ preempt_disable(); ++ *timestamp = kbasep_vinstr_get_timestamp(); ++ rcode = kbase_instr_hwcnt_request_dump(vinstr_ctx->kctx); ++ preempt_enable(); ++ ++ if (!rcode) ++ rcode = kbase_instr_hwcnt_wait_for_dump(vinstr_ctx->kctx); ++ WARN_ON(rcode); ++ ++ spin_lock_irqsave(&vinstr_ctx->state_lock, flags); ++ switch (vinstr_ctx->state) ++ { ++ case VINSTR_SUSPENDING: ++ schedule_work(&vinstr_ctx->suspend_work); ++ break; ++ case VINSTR_DUMPING: ++ vinstr_ctx->state = VINSTR_IDLE; ++ wake_up_all(&vinstr_ctx->suspend_waitq); ++ break; ++ default: ++ break; ++ } ++ spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); ++ ++ /* Accumulate values of collected counters. */ ++ if (!rcode) ++ accum_clients(vinstr_ctx); ++ ++ return rcode; ++} ++ ++/** ++ * kbasep_vinstr_fill_dump_buffer - copy accumulated counters to empty kernel ++ * buffer ++ * @cli: requesting client ++ * @timestamp: timestamp when counters were collected ++ * @event_id: id of event that caused triggered counters collection ++ * ++ * Return: zero on success ++ */ ++static int kbasep_vinstr_fill_dump_buffer( ++ struct kbase_vinstr_client *cli, u64 timestamp, ++ enum base_hwcnt_reader_event event_id) ++{ ++ unsigned int write_idx = atomic_read(&cli->write_idx); ++ unsigned int read_idx = atomic_read(&cli->read_idx); ++ ++ struct kbase_hwcnt_reader_metadata *meta; ++ void *buffer; ++ ++ /* Check if there is a place to copy HWC block into. */ ++ if (write_idx - read_idx == cli->buffer_count) ++ return -1; ++ write_idx %= cli->buffer_count; ++ ++ /* Fill in dump buffer and its metadata. */ ++ buffer = &cli->dump_buffers[write_idx * cli->dump_size]; ++ meta = &cli->dump_buffers_meta[write_idx]; ++ meta->timestamp = timestamp; ++ meta->event_id = event_id; ++ meta->buffer_idx = write_idx; ++ memcpy(buffer, cli->accum_buffer, cli->dump_size); ++ return 0; ++} ++ ++/** ++ * kbasep_vinstr_fill_dump_buffer_legacy - copy accumulated counters to buffer ++ * allocated in userspace ++ * @cli: requesting client ++ * ++ * Return: zero on success ++ * ++ * This is part of legacy ioctl interface. ++ */ ++static int kbasep_vinstr_fill_dump_buffer_legacy( ++ struct kbase_vinstr_client *cli) ++{ ++ void __user *buffer = cli->legacy_buffer; ++ int rcode; ++ ++ /* Copy data to user buffer. */ ++ rcode = copy_to_user(buffer, cli->accum_buffer, cli->dump_size); ++ if (rcode) { ++ pr_warn("error while copying buffer to user\n"); ++ return -EFAULT; ++ } ++ return 0; ++} ++ ++/** ++ * kbasep_vinstr_fill_dump_buffer_kernel - copy accumulated counters to buffer ++ * allocated in kernel space ++ * @cli: requesting client ++ * ++ * Return: zero on success ++ * ++ * This is part of the kernel client interface. ++ */ ++static int kbasep_vinstr_fill_dump_buffer_kernel( ++ struct kbase_vinstr_client *cli) ++{ ++ memcpy(cli->kernel_buffer, cli->accum_buffer, cli->dump_size); ++ ++ return 0; ++} ++ ++/** ++ * kbasep_vinstr_reprogram - reprogram hwcnt set collected by inst ++ * @vinstr_ctx: vinstr context ++ */ ++static void kbasep_vinstr_reprogram( ++ struct kbase_vinstr_context *vinstr_ctx) ++{ ++ unsigned long flags; ++ bool suspended = false; ++ ++ /* Don't enable hardware counters if vinstr is suspended. */ ++ spin_lock_irqsave(&vinstr_ctx->state_lock, flags); ++ if (VINSTR_IDLE != vinstr_ctx->state) ++ suspended = true; ++ spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); ++ if (suspended) ++ return; ++ ++ /* Change to suspended state is done while holding vinstr context ++ * lock. Below code will then no re-enable the instrumentation. */ ++ ++ if (vinstr_ctx->reprogram) { ++ struct kbase_vinstr_client *iter; ++ ++ if (!reprogram_hwcnt(vinstr_ctx)) { ++ vinstr_ctx->reprogram = false; ++ list_for_each_entry( ++ iter, ++ &vinstr_ctx->idle_clients, ++ list) ++ iter->pending = false; ++ list_for_each_entry( ++ iter, ++ &vinstr_ctx->waiting_clients, ++ list) ++ iter->pending = false; ++ } ++ } ++} ++ ++/** ++ * kbasep_vinstr_update_client - copy accumulated counters to user readable ++ * buffer and notify the user ++ * @cli: requesting client ++ * @timestamp: timestamp when counters were collected ++ * @event_id: id of event that caused triggered counters collection ++ * ++ * Return: zero on success ++ */ ++static int kbasep_vinstr_update_client( ++ struct kbase_vinstr_client *cli, u64 timestamp, ++ enum base_hwcnt_reader_event event_id) ++{ ++ int rcode = 0; ++ ++ /* Copy collected counters to user readable buffer. */ ++ if (cli->buffer_count) ++ rcode = kbasep_vinstr_fill_dump_buffer( ++ cli, timestamp, event_id); ++ else if (cli->kernel_buffer) ++ rcode = kbasep_vinstr_fill_dump_buffer_kernel(cli); ++ else ++ rcode = kbasep_vinstr_fill_dump_buffer_legacy(cli); ++ ++ if (rcode) ++ goto exit; ++ ++ ++ /* Notify client. Make sure all changes to memory are visible. */ ++ wmb(); ++ atomic_inc(&cli->write_idx); ++ wake_up_interruptible(&cli->waitq); ++ ++ /* Prepare for next request. */ ++ memset(cli->accum_buffer, 0, cli->dump_size); ++ ++exit: ++ return rcode; ++} ++ ++/** ++ * kbasep_vinstr_wake_up_callback - vinstr wake up timer wake up function ++ * ++ * @hrtimer: high resolution timer ++ * ++ * Return: High resolution timer restart enum. ++ */ ++static enum hrtimer_restart kbasep_vinstr_wake_up_callback( ++ struct hrtimer *hrtimer) ++{ ++ struct kbasep_vinstr_wake_up_timer *timer = ++ container_of( ++ hrtimer, ++ struct kbasep_vinstr_wake_up_timer, ++ hrtimer); ++ ++ KBASE_DEBUG_ASSERT(timer); ++ ++ atomic_set(&timer->vinstr_ctx->request_pending, 1); ++ wake_up_all(&timer->vinstr_ctx->waitq); ++ ++ return HRTIMER_NORESTART; ++} ++ ++/** ++ * kbasep_vinstr_service_task - HWC dumping service thread ++ * ++ * @data: Pointer to vinstr context structure. ++ * ++ * Return: 0 on success; -ENOMEM if timer allocation fails ++ */ ++static int kbasep_vinstr_service_task(void *data) ++{ ++ struct kbase_vinstr_context *vinstr_ctx = data; ++ struct kbasep_vinstr_wake_up_timer *timer; ++ ++ KBASE_DEBUG_ASSERT(vinstr_ctx); ++ ++ timer = kmalloc(sizeof(*timer), GFP_KERNEL); ++ ++ if (!timer) { ++ dev_warn(vinstr_ctx->kbdev->dev, "Timer allocation failed!\n"); ++ return -ENOMEM; ++ } ++ ++ hrtimer_init(&timer->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); ++ ++ timer->hrtimer.function = kbasep_vinstr_wake_up_callback; ++ timer->vinstr_ctx = vinstr_ctx; ++ ++ while (!kthread_should_stop()) { ++ struct kbase_vinstr_client *cli = NULL; ++ struct kbase_vinstr_client *tmp; ++ int rcode; ++ ++ u64 timestamp = kbasep_vinstr_get_timestamp(); ++ u64 dump_time = 0; ++ struct list_head expired_requests; ++ ++ /* Hold lock while performing operations on lists of clients. */ ++ mutex_lock(&vinstr_ctx->lock); ++ ++ /* Closing thread must not interact with client requests. */ ++ if (current == vinstr_ctx->thread) { ++ atomic_set(&vinstr_ctx->request_pending, 0); ++ ++ if (!list_empty(&vinstr_ctx->waiting_clients)) { ++ cli = list_first_entry( ++ &vinstr_ctx->waiting_clients, ++ struct kbase_vinstr_client, ++ list); ++ dump_time = cli->dump_time; ++ } ++ } ++ ++ if (!cli || ((s64)timestamp - (s64)dump_time < 0ll)) { ++ mutex_unlock(&vinstr_ctx->lock); ++ ++ /* Sleep until next dumping event or service request. */ ++ if (cli) { ++ u64 diff = dump_time - timestamp; ++ ++ hrtimer_start( ++ &timer->hrtimer, ++ ns_to_ktime(diff), ++ HRTIMER_MODE_REL); ++ } ++ wait_event( ++ vinstr_ctx->waitq, ++ atomic_read( ++ &vinstr_ctx->request_pending) || ++ kthread_should_stop()); ++ hrtimer_cancel(&timer->hrtimer); ++ continue; ++ } ++ ++ rcode = kbasep_vinstr_collect_and_accumulate(vinstr_ctx, ++ ×tamp); ++ ++ INIT_LIST_HEAD(&expired_requests); ++ ++ /* Find all expired requests. */ ++ list_for_each_entry_safe( ++ cli, ++ tmp, ++ &vinstr_ctx->waiting_clients, ++ list) { ++ s64 tdiff = ++ (s64)(timestamp + DUMPING_RESOLUTION) - ++ (s64)cli->dump_time; ++ if (tdiff >= 0ll) { ++ list_del(&cli->list); ++ list_add(&cli->list, &expired_requests); ++ } else { ++ break; ++ } ++ } ++ ++ /* Fill data for each request found. */ ++ list_for_each_entry_safe(cli, tmp, &expired_requests, list) { ++ /* Ensure that legacy buffer will not be used from ++ * this kthread context. */ ++ BUG_ON(0 == cli->buffer_count); ++ /* Expect only periodically sampled clients. */ ++ BUG_ON(0 == cli->dump_interval); ++ ++ if (!rcode) ++ kbasep_vinstr_update_client( ++ cli, ++ timestamp, ++ BASE_HWCNT_READER_EVENT_PERIODIC); ++ ++ /* Set new dumping time. Drop missed probing times. */ ++ do { ++ cli->dump_time += cli->dump_interval; ++ } while (cli->dump_time < timestamp); ++ ++ list_del(&cli->list); ++ kbasep_vinstr_add_dump_request( ++ cli, ++ &vinstr_ctx->waiting_clients); ++ } ++ ++ /* Reprogram counters set if required. */ ++ kbasep_vinstr_reprogram(vinstr_ctx); ++ ++ mutex_unlock(&vinstr_ctx->lock); ++ } ++ ++ kfree(timer); ++ ++ return 0; ++} ++ ++/*****************************************************************************/ ++ ++/** ++ * kbasep_vinstr_hwcnt_reader_buffer_ready - check if client has ready buffers ++ * @cli: pointer to vinstr client structure ++ * ++ * Return: non-zero if client has at least one dumping buffer filled that was ++ * not notified to user yet ++ */ ++static int kbasep_vinstr_hwcnt_reader_buffer_ready( ++ struct kbase_vinstr_client *cli) ++{ ++ KBASE_DEBUG_ASSERT(cli); ++ return atomic_read(&cli->write_idx) != atomic_read(&cli->meta_idx); ++} ++ ++/** ++ * kbasep_vinstr_hwcnt_reader_ioctl_get_buffer - hwcnt reader's ioctl command ++ * @cli: pointer to vinstr client structure ++ * @buffer: pointer to userspace buffer ++ * @size: size of buffer ++ * ++ * Return: zero on success ++ */ ++static long kbasep_vinstr_hwcnt_reader_ioctl_get_buffer( ++ struct kbase_vinstr_client *cli, void __user *buffer, ++ size_t size) ++{ ++ unsigned int meta_idx = atomic_read(&cli->meta_idx); ++ unsigned int idx = meta_idx % cli->buffer_count; ++ ++ struct kbase_hwcnt_reader_metadata *meta = &cli->dump_buffers_meta[idx]; ++ ++ /* Metadata sanity check. */ ++ KBASE_DEBUG_ASSERT(idx == meta->buffer_idx); ++ ++ if (sizeof(struct kbase_hwcnt_reader_metadata) != size) ++ return -EINVAL; ++ ++ /* Check if there is any buffer available. */ ++ if (atomic_read(&cli->write_idx) == meta_idx) ++ return -EAGAIN; ++ ++ /* Check if previously taken buffer was put back. */ ++ if (atomic_read(&cli->read_idx) != meta_idx) ++ return -EBUSY; ++ ++ /* Copy next available buffer's metadata to user. */ ++ if (copy_to_user(buffer, meta, size)) ++ return -EFAULT; ++ ++ atomic_inc(&cli->meta_idx); ++ ++ return 0; ++} ++ ++/** ++ * kbasep_vinstr_hwcnt_reader_ioctl_put_buffer - hwcnt reader's ioctl command ++ * @cli: pointer to vinstr client structure ++ * @buffer: pointer to userspace buffer ++ * @size: size of buffer ++ * ++ * Return: zero on success ++ */ ++static long kbasep_vinstr_hwcnt_reader_ioctl_put_buffer( ++ struct kbase_vinstr_client *cli, void __user *buffer, ++ size_t size) ++{ ++ unsigned int read_idx = atomic_read(&cli->read_idx); ++ unsigned int idx = read_idx % cli->buffer_count; ++ ++ struct kbase_hwcnt_reader_metadata meta; ++ ++ if (sizeof(struct kbase_hwcnt_reader_metadata) != size) ++ return -EINVAL; ++ ++ /* Check if any buffer was taken. */ ++ if (atomic_read(&cli->meta_idx) == read_idx) ++ return -EPERM; ++ ++ /* Check if correct buffer is put back. */ ++ if (copy_from_user(&meta, buffer, size)) ++ return -EFAULT; ++ if (idx != meta.buffer_idx) ++ return -EINVAL; ++ ++ atomic_inc(&cli->read_idx); ++ ++ return 0; ++} ++ ++/** ++ * kbasep_vinstr_hwcnt_reader_ioctl_set_interval - hwcnt reader's ioctl command ++ * @cli: pointer to vinstr client structure ++ * @interval: periodic dumping interval (disable periodic dumping if zero) ++ * ++ * Return: zero on success ++ */ ++static long kbasep_vinstr_hwcnt_reader_ioctl_set_interval( ++ struct kbase_vinstr_client *cli, u32 interval) ++{ ++ struct kbase_vinstr_context *vinstr_ctx = cli->vinstr_ctx; ++ ++ KBASE_DEBUG_ASSERT(vinstr_ctx); ++ ++ mutex_lock(&vinstr_ctx->lock); ++ ++ list_del(&cli->list); ++ ++ cli->dump_interval = interval; ++ ++ /* If interval is non-zero, enable periodic dumping for this client. */ ++ if (cli->dump_interval) { ++ if (DUMPING_RESOLUTION > cli->dump_interval) ++ cli->dump_interval = DUMPING_RESOLUTION; ++ cli->dump_time = ++ kbasep_vinstr_get_timestamp() + cli->dump_interval; ++ ++ kbasep_vinstr_add_dump_request( ++ cli, &vinstr_ctx->waiting_clients); ++ ++ atomic_set(&vinstr_ctx->request_pending, 1); ++ wake_up_all(&vinstr_ctx->waitq); ++ } else { ++ list_add(&cli->list, &vinstr_ctx->idle_clients); ++ } ++ ++ mutex_unlock(&vinstr_ctx->lock); ++ ++ return 0; ++} ++ ++/** ++ * kbasep_vinstr_hwcnt_reader_event_mask - return event mask for event id ++ * @event_id: id of event ++ * Return: event_mask or zero if event is not supported or maskable ++ */ ++static u32 kbasep_vinstr_hwcnt_reader_event_mask( ++ enum base_hwcnt_reader_event event_id) ++{ ++ u32 event_mask = 0; ++ ++ switch (event_id) { ++ case BASE_HWCNT_READER_EVENT_PREJOB: ++ case BASE_HWCNT_READER_EVENT_POSTJOB: ++ /* These event are maskable. */ ++ event_mask = (1 << event_id); ++ break; ++ ++ case BASE_HWCNT_READER_EVENT_MANUAL: ++ case BASE_HWCNT_READER_EVENT_PERIODIC: ++ /* These event are non-maskable. */ ++ default: ++ /* These event are not supported. */ ++ break; ++ } ++ ++ return event_mask; ++} ++ ++/** ++ * kbasep_vinstr_hwcnt_reader_ioctl_enable_event - hwcnt reader's ioctl command ++ * @cli: pointer to vinstr client structure ++ * @event_id: id of event to enable ++ * ++ * Return: zero on success ++ */ ++static long kbasep_vinstr_hwcnt_reader_ioctl_enable_event( ++ struct kbase_vinstr_client *cli, ++ enum base_hwcnt_reader_event event_id) ++{ ++ struct kbase_vinstr_context *vinstr_ctx = cli->vinstr_ctx; ++ u32 event_mask; ++ ++ KBASE_DEBUG_ASSERT(vinstr_ctx); ++ ++ event_mask = kbasep_vinstr_hwcnt_reader_event_mask(event_id); ++ if (!event_mask) ++ return -EINVAL; ++ ++ mutex_lock(&vinstr_ctx->lock); ++ cli->event_mask |= event_mask; ++ mutex_unlock(&vinstr_ctx->lock); ++ ++ return 0; ++} ++ ++/** ++ * kbasep_vinstr_hwcnt_reader_ioctl_disable_event - hwcnt reader's ioctl command ++ * @cli: pointer to vinstr client structure ++ * @event_id: id of event to disable ++ * ++ * Return: zero on success ++ */ ++static long kbasep_vinstr_hwcnt_reader_ioctl_disable_event( ++ struct kbase_vinstr_client *cli, ++ enum base_hwcnt_reader_event event_id) ++{ ++ struct kbase_vinstr_context *vinstr_ctx = cli->vinstr_ctx; ++ u32 event_mask; ++ ++ KBASE_DEBUG_ASSERT(vinstr_ctx); ++ ++ event_mask = kbasep_vinstr_hwcnt_reader_event_mask(event_id); ++ if (!event_mask) ++ return -EINVAL; ++ ++ mutex_lock(&vinstr_ctx->lock); ++ cli->event_mask &= ~event_mask; ++ mutex_unlock(&vinstr_ctx->lock); ++ ++ return 0; ++} ++ ++/** ++ * kbasep_vinstr_hwcnt_reader_ioctl_get_hwver - hwcnt reader's ioctl command ++ * @cli: pointer to vinstr client structure ++ * @hwver: pointer to user buffer where hw version will be stored ++ * ++ * Return: zero on success ++ */ ++static long kbasep_vinstr_hwcnt_reader_ioctl_get_hwver( ++ struct kbase_vinstr_client *cli, u32 __user *hwver) ++{ ++#ifndef CONFIG_MALI_BIFROST_NO_MALI ++ struct kbase_vinstr_context *vinstr_ctx = cli->vinstr_ctx; ++#endif ++ ++ u32 ver = 5; ++ ++#ifndef CONFIG_MALI_BIFROST_NO_MALI ++ KBASE_DEBUG_ASSERT(vinstr_ctx); ++ if (kbase_hw_has_feature(vinstr_ctx->kbdev, BASE_HW_FEATURE_V4)) ++ ver = 4; ++#endif ++ ++ return put_user(ver, hwver); ++} ++ ++/** ++ * kbasep_vinstr_hwcnt_reader_ioctl - hwcnt reader's ioctl ++ * @filp: pointer to file structure ++ * @cmd: user command ++ * @arg: command's argument ++ * ++ * Return: zero on success ++ */ ++static long kbasep_vinstr_hwcnt_reader_ioctl(struct file *filp, ++ unsigned int cmd, unsigned long arg) ++{ ++ long rcode = 0; ++ struct kbase_vinstr_client *cli; ++ ++ KBASE_DEBUG_ASSERT(filp); ++ ++ cli = filp->private_data; ++ KBASE_DEBUG_ASSERT(cli); ++ ++ if (unlikely(KBASE_HWCNT_READER != _IOC_TYPE(cmd))) ++ return -EINVAL; ++ ++ switch (cmd) { ++ case KBASE_HWCNT_READER_GET_API_VERSION: ++ rcode = put_user(HWCNT_READER_API, (u32 __user *)arg); ++ break; ++ case KBASE_HWCNT_READER_GET_HWVER: ++ rcode = kbasep_vinstr_hwcnt_reader_ioctl_get_hwver( ++ cli, (u32 __user *)arg); ++ break; ++ case KBASE_HWCNT_READER_GET_BUFFER_SIZE: ++ KBASE_DEBUG_ASSERT(cli->vinstr_ctx); ++ rcode = put_user( ++ (u32)cli->vinstr_ctx->dump_size, ++ (u32 __user *)arg); ++ break; ++ case KBASE_HWCNT_READER_DUMP: ++ rcode = kbase_vinstr_hwc_dump( ++ cli, BASE_HWCNT_READER_EVENT_MANUAL); ++ break; ++ case KBASE_HWCNT_READER_CLEAR: ++ rcode = kbase_vinstr_hwc_clear(cli); ++ break; ++ case KBASE_HWCNT_READER_GET_BUFFER: ++ rcode = kbasep_vinstr_hwcnt_reader_ioctl_get_buffer( ++ cli, (void __user *)arg, _IOC_SIZE(cmd)); ++ break; ++ case KBASE_HWCNT_READER_PUT_BUFFER: ++ rcode = kbasep_vinstr_hwcnt_reader_ioctl_put_buffer( ++ cli, (void __user *)arg, _IOC_SIZE(cmd)); ++ break; ++ case KBASE_HWCNT_READER_SET_INTERVAL: ++ rcode = kbasep_vinstr_hwcnt_reader_ioctl_set_interval( ++ cli, (u32)arg); ++ break; ++ case KBASE_HWCNT_READER_ENABLE_EVENT: ++ rcode = kbasep_vinstr_hwcnt_reader_ioctl_enable_event( ++ cli, (enum base_hwcnt_reader_event)arg); ++ break; ++ case KBASE_HWCNT_READER_DISABLE_EVENT: ++ rcode = kbasep_vinstr_hwcnt_reader_ioctl_disable_event( ++ cli, (enum base_hwcnt_reader_event)arg); ++ break; ++ default: ++ rcode = -EINVAL; ++ break; ++ } ++ ++ return rcode; ++} ++ ++/** ++ * kbasep_vinstr_hwcnt_reader_poll - hwcnt reader's poll ++ * @filp: pointer to file structure ++ * @wait: pointer to poll table ++ * Return: POLLIN if data can be read without blocking, otherwise zero ++ */ ++static unsigned int kbasep_vinstr_hwcnt_reader_poll(struct file *filp, ++ poll_table *wait) ++{ ++ struct kbase_vinstr_client *cli; ++ ++ KBASE_DEBUG_ASSERT(filp); ++ KBASE_DEBUG_ASSERT(wait); ++ ++ cli = filp->private_data; ++ KBASE_DEBUG_ASSERT(cli); ++ ++ poll_wait(filp, &cli->waitq, wait); ++ if (kbasep_vinstr_hwcnt_reader_buffer_ready(cli)) ++ return POLLIN; ++ return 0; ++} ++ ++/** ++ * kbasep_vinstr_hwcnt_reader_mmap - hwcnt reader's mmap ++ * @filp: pointer to file structure ++ * @vma: pointer to vma structure ++ * Return: zero on success ++ */ ++static int kbasep_vinstr_hwcnt_reader_mmap(struct file *filp, ++ struct vm_area_struct *vma) ++{ ++ struct kbase_vinstr_client *cli; ++ unsigned long size, addr, pfn, offset; ++ unsigned long vm_size = vma->vm_end - vma->vm_start; ++ ++ KBASE_DEBUG_ASSERT(filp); ++ KBASE_DEBUG_ASSERT(vma); ++ ++ cli = filp->private_data; ++ KBASE_DEBUG_ASSERT(cli); ++ ++ size = cli->buffer_count * cli->dump_size; ++ ++ if (vma->vm_pgoff > (size >> PAGE_SHIFT)) ++ return -EINVAL; ++ ++ offset = vma->vm_pgoff << PAGE_SHIFT; ++ if (vm_size > size - offset) ++ return -EINVAL; ++ ++ addr = __pa((unsigned long)cli->dump_buffers + offset); ++ pfn = addr >> PAGE_SHIFT; ++ ++ return remap_pfn_range( ++ vma, ++ vma->vm_start, ++ pfn, ++ vm_size, ++ vma->vm_page_prot); ++} ++ ++/** ++ * kbasep_vinstr_hwcnt_reader_release - hwcnt reader's release ++ * @inode: pointer to inode structure ++ * @filp: pointer to file structure ++ * Return always return zero ++ */ ++static int kbasep_vinstr_hwcnt_reader_release(struct inode *inode, ++ struct file *filp) ++{ ++ struct kbase_vinstr_client *cli; ++ ++ KBASE_DEBUG_ASSERT(inode); ++ KBASE_DEBUG_ASSERT(filp); ++ ++ cli = filp->private_data; ++ KBASE_DEBUG_ASSERT(cli); ++ ++ kbase_vinstr_detach_client(cli); ++ return 0; ++} ++ ++/*****************************************************************************/ ++ ++/** ++ * kbasep_vinstr_kick_scheduler - trigger scheduler cycle ++ * @kbdev: pointer to kbase device structure ++ */ ++static void kbasep_vinstr_kick_scheduler(struct kbase_device *kbdev) ++{ ++ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; ++ unsigned long flags; ++ ++ down(&js_devdata->schedule_sem); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_backend_slot_update(kbdev); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ up(&js_devdata->schedule_sem); ++} ++ ++/** ++ * kbasep_vinstr_suspend_worker - worker suspending vinstr module ++ * @data: pointer to work structure ++ */ ++static void kbasep_vinstr_suspend_worker(struct work_struct *data) ++{ ++ struct kbase_vinstr_context *vinstr_ctx; ++ unsigned long flags; ++ ++ vinstr_ctx = container_of(data, struct kbase_vinstr_context, ++ suspend_work); ++ ++ mutex_lock(&vinstr_ctx->lock); ++ ++ if (vinstr_ctx->kctx) ++ disable_hwcnt(vinstr_ctx); ++ ++ spin_lock_irqsave(&vinstr_ctx->state_lock, flags); ++ vinstr_ctx->state = VINSTR_SUSPENDED; ++ wake_up_all(&vinstr_ctx->suspend_waitq); ++ spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); ++ ++ mutex_unlock(&vinstr_ctx->lock); ++ ++ /* Kick GPU scheduler to allow entering protected mode. ++ * This must happen after vinstr was suspended. */ ++ kbasep_vinstr_kick_scheduler(vinstr_ctx->kbdev); ++} ++ ++/** ++ * kbasep_vinstr_suspend_worker - worker resuming vinstr module ++ * @data: pointer to work structure ++ */ ++static void kbasep_vinstr_resume_worker(struct work_struct *data) ++{ ++ struct kbase_vinstr_context *vinstr_ctx; ++ unsigned long flags; ++ ++ vinstr_ctx = container_of(data, struct kbase_vinstr_context, ++ resume_work); ++ ++ mutex_lock(&vinstr_ctx->lock); ++ ++ if (vinstr_ctx->kctx) ++ enable_hwcnt(vinstr_ctx); ++ ++ spin_lock_irqsave(&vinstr_ctx->state_lock, flags); ++ vinstr_ctx->state = VINSTR_IDLE; ++ wake_up_all(&vinstr_ctx->suspend_waitq); ++ spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); ++ ++ mutex_unlock(&vinstr_ctx->lock); ++ ++ /* Kick GPU scheduler to allow entering protected mode. ++ * Note that scheduler state machine might requested re-entry to ++ * protected mode before vinstr was resumed. ++ * This must happen after vinstr was release. */ ++ kbasep_vinstr_kick_scheduler(vinstr_ctx->kbdev); ++} ++ ++/*****************************************************************************/ ++ ++struct kbase_vinstr_context *kbase_vinstr_init(struct kbase_device *kbdev) ++{ ++ struct kbase_vinstr_context *vinstr_ctx; ++ ++ vinstr_ctx = kzalloc(sizeof(*vinstr_ctx), GFP_KERNEL); ++ if (!vinstr_ctx) ++ return NULL; ++ ++ INIT_LIST_HEAD(&vinstr_ctx->idle_clients); ++ INIT_LIST_HEAD(&vinstr_ctx->waiting_clients); ++ mutex_init(&vinstr_ctx->lock); ++ spin_lock_init(&vinstr_ctx->state_lock); ++ vinstr_ctx->kbdev = kbdev; ++ vinstr_ctx->thread = NULL; ++ vinstr_ctx->state = VINSTR_IDLE; ++ vinstr_ctx->suspend_cnt = 0; ++ INIT_WORK(&vinstr_ctx->suspend_work, kbasep_vinstr_suspend_worker); ++ INIT_WORK(&vinstr_ctx->resume_work, kbasep_vinstr_resume_worker); ++ init_waitqueue_head(&vinstr_ctx->suspend_waitq); ++ ++ atomic_set(&vinstr_ctx->request_pending, 0); ++ init_waitqueue_head(&vinstr_ctx->waitq); ++ ++ return vinstr_ctx; ++} ++ ++void kbase_vinstr_term(struct kbase_vinstr_context *vinstr_ctx) ++{ ++ struct kbase_vinstr_client *cli; ++ ++ /* Stop service thread first. */ ++ if (vinstr_ctx->thread) ++ kthread_stop(vinstr_ctx->thread); ++ ++ /* Wait for workers. */ ++ flush_work(&vinstr_ctx->suspend_work); ++ flush_work(&vinstr_ctx->resume_work); ++ ++ while (1) { ++ struct list_head *list = &vinstr_ctx->idle_clients; ++ ++ if (list_empty(list)) { ++ list = &vinstr_ctx->waiting_clients; ++ if (list_empty(list)) ++ break; ++ } ++ ++ cli = list_first_entry(list, struct kbase_vinstr_client, list); ++ list_del(&cli->list); ++ kfree(cli->accum_buffer); ++ kfree(cli); ++ vinstr_ctx->nclients--; ++ } ++ KBASE_DEBUG_ASSERT(!vinstr_ctx->nclients); ++ if (vinstr_ctx->kctx) ++ kbasep_vinstr_destroy_kctx(vinstr_ctx); ++ kfree(vinstr_ctx); ++} ++ ++int kbase_vinstr_hwcnt_reader_setup(struct kbase_vinstr_context *vinstr_ctx, ++ struct kbase_uk_hwcnt_reader_setup *setup) ++{ ++ struct kbase_vinstr_client *cli; ++ u32 bitmap[4]; ++ ++ KBASE_DEBUG_ASSERT(vinstr_ctx); ++ KBASE_DEBUG_ASSERT(setup); ++ KBASE_DEBUG_ASSERT(setup->buffer_count); ++ ++ bitmap[SHADER_HWCNT_BM] = setup->shader_bm; ++ bitmap[TILER_HWCNT_BM] = setup->tiler_bm; ++ bitmap[MMU_L2_HWCNT_BM] = setup->mmu_l2_bm; ++ bitmap[JM_HWCNT_BM] = setup->jm_bm; ++ ++ cli = kbasep_vinstr_attach_client( ++ vinstr_ctx, ++ setup->buffer_count, ++ bitmap, ++ &setup->fd, ++ NULL); ++ ++ if (!cli) ++ return -ENOMEM; ++ ++ return 0; ++} ++ ++int kbase_vinstr_legacy_hwc_setup( ++ struct kbase_vinstr_context *vinstr_ctx, ++ struct kbase_vinstr_client **cli, ++ struct kbase_uk_hwcnt_setup *setup) ++{ ++ KBASE_DEBUG_ASSERT(vinstr_ctx); ++ KBASE_DEBUG_ASSERT(setup); ++ KBASE_DEBUG_ASSERT(cli); ++ ++ if (setup->dump_buffer) { ++ u32 bitmap[4]; ++ ++ bitmap[SHADER_HWCNT_BM] = setup->shader_bm; ++ bitmap[TILER_HWCNT_BM] = setup->tiler_bm; ++ bitmap[MMU_L2_HWCNT_BM] = setup->mmu_l2_bm; ++ bitmap[JM_HWCNT_BM] = setup->jm_bm; ++ ++ if (*cli) ++ return -EBUSY; ++ ++ *cli = kbasep_vinstr_attach_client( ++ vinstr_ctx, ++ 0, ++ bitmap, ++ (void *)(long)setup->dump_buffer, ++ NULL); ++ ++ if (!(*cli)) ++ return -ENOMEM; ++ } else { ++ if (!*cli) ++ return -EINVAL; ++ ++ kbase_vinstr_detach_client(*cli); ++ *cli = NULL; ++ } ++ ++ return 0; ++} ++ ++struct kbase_vinstr_client *kbase_vinstr_hwcnt_kernel_setup( ++ struct kbase_vinstr_context *vinstr_ctx, ++ struct kbase_uk_hwcnt_reader_setup *setup, ++ void *kernel_buffer) ++{ ++ u32 bitmap[4]; ++ ++ if (!vinstr_ctx || !setup || !kernel_buffer) ++ return NULL; ++ ++ bitmap[SHADER_HWCNT_BM] = setup->shader_bm; ++ bitmap[TILER_HWCNT_BM] = setup->tiler_bm; ++ bitmap[MMU_L2_HWCNT_BM] = setup->mmu_l2_bm; ++ bitmap[JM_HWCNT_BM] = setup->jm_bm; ++ ++ return kbasep_vinstr_attach_client( ++ vinstr_ctx, ++ 0, ++ bitmap, ++ NULL, ++ kernel_buffer); ++} ++KBASE_EXPORT_TEST_API(kbase_vinstr_hwcnt_kernel_setup); ++ ++int kbase_vinstr_hwc_dump(struct kbase_vinstr_client *cli, ++ enum base_hwcnt_reader_event event_id) ++{ ++ int rcode = 0; ++ struct kbase_vinstr_context *vinstr_ctx; ++ u64 timestamp; ++ u32 event_mask; ++ ++ if (!cli) ++ return -EINVAL; ++ ++ vinstr_ctx = cli->vinstr_ctx; ++ KBASE_DEBUG_ASSERT(vinstr_ctx); ++ ++ KBASE_DEBUG_ASSERT(event_id < BASE_HWCNT_READER_EVENT_COUNT); ++ event_mask = 1 << event_id; ++ ++ mutex_lock(&vinstr_ctx->lock); ++ ++ if (event_mask & cli->event_mask) { ++ rcode = kbasep_vinstr_collect_and_accumulate( ++ vinstr_ctx, ++ ×tamp); ++ if (rcode) ++ goto exit; ++ ++ rcode = kbasep_vinstr_update_client(cli, timestamp, event_id); ++ if (rcode) ++ goto exit; ++ ++ kbasep_vinstr_reprogram(vinstr_ctx); ++ } ++ ++exit: ++ mutex_unlock(&vinstr_ctx->lock); ++ ++ return rcode; ++} ++KBASE_EXPORT_TEST_API(kbase_vinstr_hwc_dump); ++ ++int kbase_vinstr_hwc_clear(struct kbase_vinstr_client *cli) ++{ ++ struct kbase_vinstr_context *vinstr_ctx; ++ int rcode; ++ u64 unused; ++ ++ if (!cli) ++ return -EINVAL; ++ ++ vinstr_ctx = cli->vinstr_ctx; ++ KBASE_DEBUG_ASSERT(vinstr_ctx); ++ ++ mutex_lock(&vinstr_ctx->lock); ++ ++ rcode = kbasep_vinstr_collect_and_accumulate(vinstr_ctx, &unused); ++ if (rcode) ++ goto exit; ++ rcode = kbase_instr_hwcnt_clear(vinstr_ctx->kctx); ++ if (rcode) ++ goto exit; ++ memset(cli->accum_buffer, 0, cli->dump_size); ++ ++ kbasep_vinstr_reprogram(vinstr_ctx); ++ ++exit: ++ mutex_unlock(&vinstr_ctx->lock); ++ ++ return rcode; ++} ++ ++int kbase_vinstr_try_suspend(struct kbase_vinstr_context *vinstr_ctx) ++{ ++ unsigned long flags; ++ int ret = -EAGAIN; ++ ++ KBASE_DEBUG_ASSERT(vinstr_ctx); ++ ++ spin_lock_irqsave(&vinstr_ctx->state_lock, flags); ++ switch (vinstr_ctx->state) { ++ case VINSTR_SUSPENDED: ++ vinstr_ctx->suspend_cnt++; ++ /* overflow shall not happen */ ++ BUG_ON(0 == vinstr_ctx->suspend_cnt); ++ ret = 0; ++ break; ++ ++ case VINSTR_IDLE: ++ if (vinstr_ctx->clients_present) { ++ vinstr_ctx->state = VINSTR_SUSPENDING; ++ schedule_work(&vinstr_ctx->suspend_work); ++ } else { ++ vinstr_ctx->state = VINSTR_SUSPENDED; ++ ++ vinstr_ctx->suspend_cnt++; ++ /* overflow shall not happen */ ++ WARN_ON(0 == vinstr_ctx->suspend_cnt); ++ ret = 0; ++ } ++ break; ++ ++ case VINSTR_DUMPING: ++ vinstr_ctx->state = VINSTR_SUSPENDING; ++ break; ++ ++ case VINSTR_SUSPENDING: ++ /* fall through */ ++ case VINSTR_RESUMING: ++ break; ++ ++ default: ++ BUG(); ++ break; ++ } ++ spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); ++ ++ return ret; ++} ++ ++void kbase_vinstr_suspend(struct kbase_vinstr_context *vinstr_ctx) ++{ ++ wait_event(vinstr_ctx->suspend_waitq, ++ (0 == kbase_vinstr_try_suspend(vinstr_ctx))); ++} ++ ++void kbase_vinstr_resume(struct kbase_vinstr_context *vinstr_ctx) ++{ ++ unsigned long flags; ++ ++ KBASE_DEBUG_ASSERT(vinstr_ctx); ++ ++ spin_lock_irqsave(&vinstr_ctx->state_lock, flags); ++ BUG_ON(VINSTR_SUSPENDING == vinstr_ctx->state); ++ if (VINSTR_SUSPENDED == vinstr_ctx->state) { ++ BUG_ON(0 == vinstr_ctx->suspend_cnt); ++ vinstr_ctx->suspend_cnt--; ++ if (0 == vinstr_ctx->suspend_cnt) { ++ if (vinstr_ctx->clients_present) { ++ vinstr_ctx->state = VINSTR_RESUMING; ++ schedule_work(&vinstr_ctx->resume_work); ++ } else { ++ vinstr_ctx->state = VINSTR_IDLE; ++ } ++ } ++ } ++ spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); ++} +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_vinstr.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_vinstr.h +new file mode 100755 +index 000000000000..6207d25aef06 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_vinstr.h +@@ -0,0 +1,155 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KBASE_VINSTR_H_ ++#define _KBASE_VINSTR_H_ ++ ++#include ++#include ++ ++/*****************************************************************************/ ++ ++struct kbase_vinstr_context; ++struct kbase_vinstr_client; ++ ++/*****************************************************************************/ ++ ++/** ++ * kbase_vinstr_init() - initialize the vinstr core ++ * @kbdev: kbase device ++ * ++ * Return: pointer to the vinstr context on success or NULL on failure ++ */ ++struct kbase_vinstr_context *kbase_vinstr_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_vinstr_term() - terminate the vinstr core ++ * @vinstr_ctx: vinstr context ++ */ ++void kbase_vinstr_term(struct kbase_vinstr_context *vinstr_ctx); ++ ++/** ++ * kbase_vinstr_hwcnt_reader_setup - configure hw counters reader ++ * @vinstr_ctx: vinstr context ++ * @setup: reader's configuration ++ * ++ * Return: zero on success ++ */ ++int kbase_vinstr_hwcnt_reader_setup( ++ struct kbase_vinstr_context *vinstr_ctx, ++ struct kbase_uk_hwcnt_reader_setup *setup); ++ ++/** ++ * kbase_vinstr_legacy_hwc_setup - configure hw counters for dumping ++ * @vinstr_ctx: vinstr context ++ * @cli: pointer where to store pointer to new vinstr client structure ++ * @setup: hwc configuration ++ * ++ * Return: zero on success ++ */ ++int kbase_vinstr_legacy_hwc_setup( ++ struct kbase_vinstr_context *vinstr_ctx, ++ struct kbase_vinstr_client **cli, ++ struct kbase_uk_hwcnt_setup *setup); ++ ++/** ++ * kbase_vinstr_hwcnt_kernel_setup - configure hw counters for kernel side ++ * client ++ * @vinstr_ctx: vinstr context ++ * @setup: reader's configuration ++ * @kernel_buffer: pointer to dump buffer ++ * ++ * setup->buffer_count and setup->fd are not used for kernel side clients. ++ * ++ * Return: pointer to client structure, or NULL on failure ++ */ ++struct kbase_vinstr_client *kbase_vinstr_hwcnt_kernel_setup( ++ struct kbase_vinstr_context *vinstr_ctx, ++ struct kbase_uk_hwcnt_reader_setup *setup, ++ void *kernel_buffer); ++ ++/** ++ * kbase_vinstr_hwc_dump - issue counter dump for vinstr client ++ * @cli: pointer to vinstr client ++ * @event_id: id of event that triggered hwcnt dump ++ * ++ * Return: zero on success ++ */ ++int kbase_vinstr_hwc_dump( ++ struct kbase_vinstr_client *cli, ++ enum base_hwcnt_reader_event event_id); ++ ++/** ++ * kbase_vinstr_hwc_clear - performs a reset of the hardware counters for ++ * a given kbase context ++ * @cli: pointer to vinstr client ++ * ++ * Return: zero on success ++ */ ++int kbase_vinstr_hwc_clear(struct kbase_vinstr_client *cli); ++ ++/** ++ * kbase_vinstr_try_suspend - try suspending operation of a given vinstr context ++ * @vinstr_ctx: vinstr context ++ * ++ * Return: 0 on success, or negative if state change is in progress ++ * ++ * Warning: This API call is non-generic. It is meant to be used only by ++ * job scheduler state machine. ++ * ++ * Function initiates vinstr switch to suspended state. Once it was called ++ * vinstr enters suspending state. If function return non-zero value, it ++ * indicates that state switch is not complete and function must be called ++ * again. On state switch vinstr will trigger job scheduler state machine ++ * cycle. ++ */ ++int kbase_vinstr_try_suspend(struct kbase_vinstr_context *vinstr_ctx); ++ ++/** ++ * kbase_vinstr_suspend - suspends operation of a given vinstr context ++ * @vinstr_ctx: vinstr context ++ * ++ * Function initiates vinstr switch to suspended state. Then it blocks until ++ * operation is completed. ++ */ ++void kbase_vinstr_suspend(struct kbase_vinstr_context *vinstr_ctx); ++ ++/** ++ * kbase_vinstr_resume - resumes operation of a given vinstr context ++ * @vinstr_ctx: vinstr context ++ * ++ * Function can be called only if it was preceded by a successful call ++ * to kbase_vinstr_suspend. ++ */ ++void kbase_vinstr_resume(struct kbase_vinstr_context *vinstr_ctx); ++ ++/** ++ * kbase_vinstr_dump_size - Return required size of dump buffer ++ * @kbdev: device pointer ++ * ++ * Return : buffer size in bytes ++ */ ++size_t kbase_vinstr_dump_size(struct kbase_device *kbdev); ++ ++/** ++ * kbase_vinstr_detach_client - Detach a client from the vinstr core ++ * @cli: pointer to vinstr client ++ */ ++void kbase_vinstr_detach_client(struct kbase_vinstr_client *cli); ++ ++#endif /* _KBASE_VINSTR_H_ */ ++ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_linux_kbase_trace.h b/drivers/gpu/arm/bifrost_for_linux/mali_linux_kbase_trace.h +new file mode 100755 +index 000000000000..5d6b4021d626 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_linux_kbase_trace.h +@@ -0,0 +1,201 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++#if !defined(_TRACE_MALI_KBASE_H) || defined(TRACE_HEADER_MULTI_READ) ++#define _TRACE_MALI_KBASE_H ++ ++#undef TRACE_SYSTEM ++#define TRACE_SYSTEM mali ++ ++#include ++ ++DECLARE_EVENT_CLASS(mali_slot_template, ++ TP_PROTO(int jobslot, unsigned int info_val), ++ TP_ARGS(jobslot, info_val), ++ TP_STRUCT__entry( ++ __field(unsigned int, jobslot) ++ __field(unsigned int, info_val) ++ ), ++ TP_fast_assign( ++ __entry->jobslot = jobslot; ++ __entry->info_val = info_val; ++ ), ++ TP_printk("jobslot=%u info=%u", __entry->jobslot, __entry->info_val) ++); ++ ++#define DEFINE_MALI_SLOT_EVENT(name) \ ++DEFINE_EVENT(mali_slot_template, mali_##name, \ ++ TP_PROTO(int jobslot, unsigned int info_val), \ ++ TP_ARGS(jobslot, info_val)) ++DEFINE_MALI_SLOT_EVENT(JM_SUBMIT); ++DEFINE_MALI_SLOT_EVENT(JM_JOB_DONE); ++DEFINE_MALI_SLOT_EVENT(JM_UPDATE_HEAD); ++DEFINE_MALI_SLOT_EVENT(JM_CHECK_HEAD); ++DEFINE_MALI_SLOT_EVENT(JM_SOFTSTOP); ++DEFINE_MALI_SLOT_EVENT(JM_SOFTSTOP_0); ++DEFINE_MALI_SLOT_EVENT(JM_SOFTSTOP_1); ++DEFINE_MALI_SLOT_EVENT(JM_HARDSTOP); ++DEFINE_MALI_SLOT_EVENT(JM_HARDSTOP_0); ++DEFINE_MALI_SLOT_EVENT(JM_HARDSTOP_1); ++DEFINE_MALI_SLOT_EVENT(JM_SLOT_SOFT_OR_HARD_STOP); ++DEFINE_MALI_SLOT_EVENT(JM_SLOT_EVICT); ++DEFINE_MALI_SLOT_EVENT(JM_BEGIN_RESET_WORKER); ++DEFINE_MALI_SLOT_EVENT(JM_END_RESET_WORKER); ++DEFINE_MALI_SLOT_EVENT(JS_CORE_REF_REGISTER_ON_RECHECK_FAILED); ++DEFINE_MALI_SLOT_EVENT(JS_AFFINITY_SUBMIT_TO_BLOCKED); ++DEFINE_MALI_SLOT_EVENT(JS_AFFINITY_CURRENT); ++DEFINE_MALI_SLOT_EVENT(JD_DONE_TRY_RUN_NEXT_JOB); ++DEFINE_MALI_SLOT_EVENT(JS_CORE_REF_REQUEST_CORES_FAILED); ++DEFINE_MALI_SLOT_EVENT(JS_CORE_REF_REGISTER_INUSE_FAILED); ++DEFINE_MALI_SLOT_EVENT(JS_CORE_REF_REQUEST_ON_RECHECK_FAILED); ++DEFINE_MALI_SLOT_EVENT(JS_CORE_REF_AFFINITY_WOULD_VIOLATE); ++DEFINE_MALI_SLOT_EVENT(JS_JOB_DONE_TRY_RUN_NEXT_JOB); ++DEFINE_MALI_SLOT_EVENT(JS_JOB_DONE_RETRY_NEEDED); ++DEFINE_MALI_SLOT_EVENT(JS_POLICY_DEQUEUE_JOB); ++DEFINE_MALI_SLOT_EVENT(JS_POLICY_DEQUEUE_JOB_IRQ); ++#undef DEFINE_MALI_SLOT_EVENT ++ ++DECLARE_EVENT_CLASS(mali_refcount_template, ++ TP_PROTO(int refcount, unsigned int info_val), ++ TP_ARGS(refcount, info_val), ++ TP_STRUCT__entry( ++ __field(unsigned int, refcount) ++ __field(unsigned int, info_val) ++ ), ++ TP_fast_assign( ++ __entry->refcount = refcount; ++ __entry->info_val = info_val; ++ ), ++ TP_printk("refcount=%u info=%u", __entry->refcount, __entry->info_val) ++); ++ ++#define DEFINE_MALI_REFCOUNT_EVENT(name) \ ++DEFINE_EVENT(mali_refcount_template, mali_##name, \ ++ TP_PROTO(int refcount, unsigned int info_val), \ ++ TP_ARGS(refcount, info_val)) ++DEFINE_MALI_REFCOUNT_EVENT(JS_RETAIN_CTX_NOLOCK); ++DEFINE_MALI_REFCOUNT_EVENT(JS_ADD_JOB); ++DEFINE_MALI_REFCOUNT_EVENT(JS_REMOVE_JOB); ++DEFINE_MALI_REFCOUNT_EVENT(JS_RETAIN_CTX); ++DEFINE_MALI_REFCOUNT_EVENT(JS_RELEASE_CTX); ++DEFINE_MALI_REFCOUNT_EVENT(JS_TRY_SCHEDULE_HEAD_CTX); ++DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_INIT_CTX); ++DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_TERM_CTX); ++DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_ENQUEUE_CTX); ++DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_DEQUEUE_HEAD_CTX); ++DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_TRY_EVICT_CTX); ++DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_RUNPOOL_ADD_CTX); ++DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_RUNPOOL_REMOVE_CTX); ++DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_FOREACH_CTX_JOBS); ++DEFINE_MALI_REFCOUNT_EVENT(PM_CONTEXT_ACTIVE); ++DEFINE_MALI_REFCOUNT_EVENT(PM_CONTEXT_IDLE); ++#undef DEFINE_MALI_REFCOUNT_EVENT ++ ++DECLARE_EVENT_CLASS(mali_add_template, ++ TP_PROTO(int gpu_addr, unsigned int info_val), ++ TP_ARGS(gpu_addr, info_val), ++ TP_STRUCT__entry( ++ __field(unsigned int, gpu_addr) ++ __field(unsigned int, info_val) ++ ), ++ TP_fast_assign( ++ __entry->gpu_addr = gpu_addr; ++ __entry->info_val = info_val; ++ ), ++ TP_printk("gpu_addr=%u info=%u", __entry->gpu_addr, __entry->info_val) ++); ++ ++#define DEFINE_MALI_ADD_EVENT(name) \ ++DEFINE_EVENT(mali_add_template, mali_##name, \ ++ TP_PROTO(int gpu_addr, unsigned int info_val), \ ++ TP_ARGS(gpu_addr, info_val)) ++DEFINE_MALI_ADD_EVENT(CORE_CTX_DESTROY); ++DEFINE_MALI_ADD_EVENT(CORE_CTX_HWINSTR_TERM); ++DEFINE_MALI_ADD_EVENT(CORE_GPU_IRQ); ++DEFINE_MALI_ADD_EVENT(CORE_GPU_IRQ_CLEAR); ++DEFINE_MALI_ADD_EVENT(CORE_GPU_IRQ_DONE); ++DEFINE_MALI_ADD_EVENT(CORE_GPU_SOFT_RESET); ++DEFINE_MALI_ADD_EVENT(CORE_GPU_HARD_RESET); ++DEFINE_MALI_ADD_EVENT(CORE_GPU_PRFCNT_SAMPLE); ++DEFINE_MALI_ADD_EVENT(CORE_GPU_PRFCNT_CLEAR); ++DEFINE_MALI_ADD_EVENT(CORE_GPU_CLEAN_INV_CACHES); ++DEFINE_MALI_ADD_EVENT(JD_DONE_WORKER); ++DEFINE_MALI_ADD_EVENT(JD_DONE_WORKER_END); ++DEFINE_MALI_ADD_EVENT(JD_CANCEL_WORKER); ++DEFINE_MALI_ADD_EVENT(JD_DONE); ++DEFINE_MALI_ADD_EVENT(JD_CANCEL); ++DEFINE_MALI_ADD_EVENT(JD_ZAP_CONTEXT); ++DEFINE_MALI_ADD_EVENT(JM_IRQ); ++DEFINE_MALI_ADD_EVENT(JM_IRQ_END); ++DEFINE_MALI_ADD_EVENT(JM_FLUSH_WORKQS); ++DEFINE_MALI_ADD_EVENT(JM_FLUSH_WORKQS_DONE); ++DEFINE_MALI_ADD_EVENT(JM_ZAP_NON_SCHEDULED); ++DEFINE_MALI_ADD_EVENT(JM_ZAP_SCHEDULED); ++DEFINE_MALI_ADD_EVENT(JM_ZAP_DONE); ++DEFINE_MALI_ADD_EVENT(JM_SUBMIT_AFTER_RESET); ++DEFINE_MALI_ADD_EVENT(JM_JOB_COMPLETE); ++DEFINE_MALI_ADD_EVENT(JS_FAST_START_EVICTS_CTX); ++DEFINE_MALI_ADD_EVENT(JS_CTX_ATTR_NOW_ON_RUNPOOL); ++DEFINE_MALI_ADD_EVENT(JS_CTX_ATTR_NOW_OFF_RUNPOOL); ++DEFINE_MALI_ADD_EVENT(JS_CTX_ATTR_NOW_ON_CTX); ++DEFINE_MALI_ADD_EVENT(JS_CTX_ATTR_NOW_OFF_CTX); ++DEFINE_MALI_ADD_EVENT(JS_POLICY_TIMER_END); ++DEFINE_MALI_ADD_EVENT(JS_POLICY_TIMER_START); ++DEFINE_MALI_ADD_EVENT(JS_POLICY_ENQUEUE_JOB); ++DEFINE_MALI_ADD_EVENT(PM_CORES_CHANGE_DESIRED); ++DEFINE_MALI_ADD_EVENT(PM_JOB_SUBMIT_AFTER_POWERING_UP); ++DEFINE_MALI_ADD_EVENT(PM_JOB_SUBMIT_AFTER_POWERED_UP); ++DEFINE_MALI_ADD_EVENT(PM_PWRON); ++DEFINE_MALI_ADD_EVENT(PM_PWRON_TILER); ++DEFINE_MALI_ADD_EVENT(PM_PWRON_L2); ++DEFINE_MALI_ADD_EVENT(PM_PWROFF); ++DEFINE_MALI_ADD_EVENT(PM_PWROFF_TILER); ++DEFINE_MALI_ADD_EVENT(PM_PWROFF_L2); ++DEFINE_MALI_ADD_EVENT(PM_CORES_POWERED); ++DEFINE_MALI_ADD_EVENT(PM_CORES_POWERED_TILER); ++DEFINE_MALI_ADD_EVENT(PM_CORES_POWERED_L2); ++DEFINE_MALI_ADD_EVENT(PM_DESIRED_REACHED); ++DEFINE_MALI_ADD_EVENT(PM_DESIRED_REACHED_TILER); ++DEFINE_MALI_ADD_EVENT(PM_UNREQUEST_CHANGE_SHADER_NEEDED); ++DEFINE_MALI_ADD_EVENT(PM_REQUEST_CHANGE_SHADER_NEEDED); ++DEFINE_MALI_ADD_EVENT(PM_REGISTER_CHANGE_SHADER_NEEDED); ++DEFINE_MALI_ADD_EVENT(PM_REGISTER_CHANGE_SHADER_INUSE); ++DEFINE_MALI_ADD_EVENT(PM_RELEASE_CHANGE_SHADER_INUSE); ++DEFINE_MALI_ADD_EVENT(PM_CORES_AVAILABLE); ++DEFINE_MALI_ADD_EVENT(PM_CORES_AVAILABLE_TILER); ++DEFINE_MALI_ADD_EVENT(PM_CORES_CHANGE_AVAILABLE); ++DEFINE_MALI_ADD_EVENT(PM_CORES_CHANGE_AVAILABLE_TILER); ++DEFINE_MALI_ADD_EVENT(PM_GPU_ON); ++DEFINE_MALI_ADD_EVENT(PM_GPU_OFF); ++DEFINE_MALI_ADD_EVENT(PM_SET_POLICY); ++DEFINE_MALI_ADD_EVENT(PM_CURRENT_POLICY_INIT); ++DEFINE_MALI_ADD_EVENT(PM_CURRENT_POLICY_TERM); ++DEFINE_MALI_ADD_EVENT(PM_CA_SET_POLICY); ++DEFINE_MALI_ADD_EVENT(PM_WAKE_WAITERS); ++#undef DEFINE_MALI_ADD_EVENT ++ ++#endif /* _TRACE_MALI_KBASE_H */ ++ ++#undef TRACE_INCLUDE_PATH ++#undef linux ++#define TRACE_INCLUDE_PATH . ++#undef TRACE_INCLUDE_FILE ++#define TRACE_INCLUDE_FILE mali_linux_kbase_trace ++ ++/* This part must be outside protection */ ++#include +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_linux_trace.h b/drivers/gpu/arm/bifrost_for_linux/mali_linux_trace.h +new file mode 100755 +index 000000000000..2be06a552768 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_linux_trace.h +@@ -0,0 +1,189 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#if !defined(_TRACE_MALI_H) || defined(TRACE_HEADER_MULTI_READ) ++#define _TRACE_MALI_H ++ ++#undef TRACE_SYSTEM ++#define TRACE_SYSTEM mali ++#define TRACE_INCLUDE_FILE mali_linux_trace ++ ++#include ++ ++#define MALI_JOB_SLOTS_EVENT_CHANGED ++ ++/** ++ * mali_job_slots_event - called from mali_kbase_core_linux.c ++ * @event_id: ORed together bitfields representing a type of event, made with the GATOR_MAKE_EVENT() macro. ++ */ ++TRACE_EVENT(mali_job_slots_event, ++ TP_PROTO(unsigned int event_id, unsigned int tgid, unsigned int pid, ++ unsigned char job_id), ++ TP_ARGS(event_id, tgid, pid, job_id), ++ TP_STRUCT__entry( ++ __field(unsigned int, event_id) ++ __field(unsigned int, tgid) ++ __field(unsigned int, pid) ++ __field(unsigned char, job_id) ++ ), ++ TP_fast_assign( ++ __entry->event_id = event_id; ++ __entry->tgid = tgid; ++ __entry->pid = pid; ++ __entry->job_id = job_id; ++ ), ++ TP_printk("event=%u tgid=%u pid=%u job_id=%u", ++ __entry->event_id, __entry->tgid, __entry->pid, __entry->job_id) ++); ++ ++/** ++ * mali_pm_status - Called by mali_kbase_pm_driver.c ++ * @event_id: core type (shader, tiler, l2 cache) ++ * @value: 64bits bitmask reporting either power status of the cores (1-ON, 0-OFF) ++ */ ++TRACE_EVENT(mali_pm_status, ++ TP_PROTO(unsigned int event_id, unsigned long long value), ++ TP_ARGS(event_id, value), ++ TP_STRUCT__entry( ++ __field(unsigned int, event_id) ++ __field(unsigned long long, value) ++ ), ++ TP_fast_assign( ++ __entry->event_id = event_id; ++ __entry->value = value; ++ ), ++ TP_printk("event %u = %llu", __entry->event_id, __entry->value) ++); ++ ++/** ++ * mali_pm_power_on - Called by mali_kbase_pm_driver.c ++ * @event_id: core type (shader, tiler, l2 cache) ++ * @value: 64bits bitmask reporting the cores to power up ++ */ ++TRACE_EVENT(mali_pm_power_on, ++ TP_PROTO(unsigned int event_id, unsigned long long value), ++ TP_ARGS(event_id, value), ++ TP_STRUCT__entry( ++ __field(unsigned int, event_id) ++ __field(unsigned long long, value) ++ ), ++ TP_fast_assign( ++ __entry->event_id = event_id; ++ __entry->value = value; ++ ), ++ TP_printk("event %u = %llu", __entry->event_id, __entry->value) ++); ++ ++/** ++ * mali_pm_power_off - Called by mali_kbase_pm_driver.c ++ * @event_id: core type (shader, tiler, l2 cache) ++ * @value: 64bits bitmask reporting the cores to power down ++ */ ++TRACE_EVENT(mali_pm_power_off, ++ TP_PROTO(unsigned int event_id, unsigned long long value), ++ TP_ARGS(event_id, value), ++ TP_STRUCT__entry( ++ __field(unsigned int, event_id) ++ __field(unsigned long long, value) ++ ), ++ TP_fast_assign( ++ __entry->event_id = event_id; ++ __entry->value = value; ++ ), ++ TP_printk("event %u = %llu", __entry->event_id, __entry->value) ++); ++ ++/** ++ * mali_page_fault_insert_pages - Called by page_fault_worker() ++ * it reports an MMU page fault resulting in new pages being mapped. ++ * @event_id: MMU address space number. ++ * @value: number of newly allocated pages ++ */ ++TRACE_EVENT(mali_page_fault_insert_pages, ++ TP_PROTO(int event_id, unsigned long value), ++ TP_ARGS(event_id, value), ++ TP_STRUCT__entry( ++ __field(int, event_id) ++ __field(unsigned long, value) ++ ), ++ TP_fast_assign( ++ __entry->event_id = event_id; ++ __entry->value = value; ++ ), ++ TP_printk("event %d = %lu", __entry->event_id, __entry->value) ++); ++ ++/** ++ * mali_mmu_as_in_use - Called by assign_and_activate_kctx_addr_space() ++ * it reports that a certain MMU address space is in use now. ++ * @event_id: MMU address space number. ++ */ ++TRACE_EVENT(mali_mmu_as_in_use, ++ TP_PROTO(int event_id), ++ TP_ARGS(event_id), ++ TP_STRUCT__entry( ++ __field(int, event_id) ++ ), ++ TP_fast_assign( ++ __entry->event_id = event_id; ++ ), ++ TP_printk("event=%d", __entry->event_id) ++); ++ ++/** ++ * mali_mmu_as_released - Called by kbasep_js_runpool_release_ctx_internal() ++ * it reports that a certain MMU address space has been released now. ++ * @event_id: MMU address space number. ++ */ ++TRACE_EVENT(mali_mmu_as_released, ++ TP_PROTO(int event_id), ++ TP_ARGS(event_id), ++ TP_STRUCT__entry( ++ __field(int, event_id) ++ ), ++ TP_fast_assign( ++ __entry->event_id = event_id; ++ ), ++ TP_printk("event=%d", __entry->event_id) ++); ++ ++/** ++ * mali_total_alloc_pages_change - Called by kbase_atomic_add_pages() ++ * and by kbase_atomic_sub_pages() ++ * it reports that the total number of allocated pages is changed. ++ * @event_id: number of pages to be added or subtracted (according to the sign). ++ */ ++TRACE_EVENT(mali_total_alloc_pages_change, ++ TP_PROTO(long long int event_id), ++ TP_ARGS(event_id), ++ TP_STRUCT__entry( ++ __field(long long int, event_id) ++ ), ++ TP_fast_assign( ++ __entry->event_id = event_id; ++ ), ++ TP_printk("event=%lld", __entry->event_id) ++); ++ ++#endif /* _TRACE_MALI_H */ ++ ++#undef TRACE_INCLUDE_PATH ++#undef linux ++#define TRACE_INCLUDE_PATH . ++ ++/* This part must be outside protection */ ++#include +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_malisw.h b/drivers/gpu/arm/bifrost_for_linux/mali_malisw.h +new file mode 100755 +index 000000000000..99452933eab4 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_malisw.h +@@ -0,0 +1,131 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/** ++ * Kernel-wide include for common macros and types. ++ */ ++ ++#ifndef _MALISW_H_ ++#define _MALISW_H_ ++ ++#include ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 0) ++#define U8_MAX ((u8)~0U) ++#define S8_MAX ((s8)(U8_MAX>>1)) ++#define S8_MIN ((s8)(-S8_MAX - 1)) ++#define U16_MAX ((u16)~0U) ++#define S16_MAX ((s16)(U16_MAX>>1)) ++#define S16_MIN ((s16)(-S16_MAX - 1)) ++#define U32_MAX ((u32)~0U) ++#define S32_MAX ((s32)(U32_MAX>>1)) ++#define S32_MIN ((s32)(-S32_MAX - 1)) ++#define U64_MAX ((u64)~0ULL) ++#define S64_MAX ((s64)(U64_MAX>>1)) ++#define S64_MIN ((s64)(-S64_MAX - 1)) ++#endif /* LINUX_VERSION_CODE */ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 5, 0) ++#define SIZE_MAX (~(size_t)0) ++#endif /* LINUX_VERSION_CODE */ ++ ++/** ++ * MIN - Return the lesser of two values. ++ * ++ * As a macro it may evaluate its arguments more than once. ++ * Refer to MAX macro for more details ++ */ ++#define MIN(x, y) ((x) < (y) ? (x) : (y)) ++ ++/** ++ * MAX - Return the greater of two values. ++ * ++ * As a macro it may evaluate its arguments more than once. ++ * If called on the same two arguments as MIN it is guaranteed to return ++ * the one that MIN didn't return. This is significant for types where not ++ * all values are comparable e.g. NaNs in floating-point types. But if you want ++ * to retrieve the min and max of two values, consider using a conditional swap ++ * instead. ++ */ ++#define MAX(x, y) ((x) < (y) ? (y) : (x)) ++ ++/** ++ * @hideinitializer ++ * Function-like macro for suppressing unused variable warnings. Where possible ++ * such variables should be removed; this macro is present for cases where we ++ * much support API backwards compatibility. ++ */ ++#define CSTD_UNUSED(x) ((void)(x)) ++ ++/** ++ * @hideinitializer ++ * Function-like macro for use where "no behavior" is desired. This is useful ++ * when compile time macros turn a function-like macro in to a no-op, but ++ * where having no statement is otherwise invalid. ++ */ ++#define CSTD_NOP(...) ((void)#__VA_ARGS__) ++ ++/** ++ * Function-like macro for converting a pointer in to a u64 for storing into ++ * an external data structure. This is commonly used when pairing a 32-bit ++ * CPU with a 64-bit peripheral, such as a Midgard GPU. C's type promotion ++ * is complex and a straight cast does not work reliably as pointers are ++ * often considered as signed. ++ */ ++#define PTR_TO_U64(x) ((uint64_t)((uintptr_t)(x))) ++ ++/** ++ * @hideinitializer ++ * Function-like macro for stringizing a single level macro. ++ * @code ++ * #define MY_MACRO 32 ++ * CSTD_STR1( MY_MACRO ) ++ * > "MY_MACRO" ++ * @endcode ++ */ ++#define CSTD_STR1(x) #x ++ ++/** ++ * @hideinitializer ++ * Function-like macro for stringizing a macro's value. This should not be used ++ * if the macro is defined in a way which may have no value; use the ++ * alternative @c CSTD_STR2N macro should be used instead. ++ * @code ++ * #define MY_MACRO 32 ++ * CSTD_STR2( MY_MACRO ) ++ * > "32" ++ * @endcode ++ */ ++#define CSTD_STR2(x) CSTD_STR1(x) ++ ++/** ++ * Specify an assertion value which is evaluated at compile time. Recommended ++ * usage is specification of a @c static @c INLINE function containing all of ++ * the assertions thus: ++ * ++ * @code ++ * static INLINE [module]_compile_time_assertions( void ) ++ * { ++ * COMPILE_TIME_ASSERT( sizeof(uintptr_t) == sizeof(intptr_t) ); ++ * } ++ * @endcode ++ * ++ * @note Use @c static not @c STATIC. We never want to turn off this @c static ++ * specification for testing purposes. ++ */ ++#define CSTD_COMPILE_TIME_ASSERT(expr) \ ++ do { switch (0) { case 0: case (expr):; } } while (false) ++ ++#endif /* _MALISW_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_midg_coherency.h b/drivers/gpu/arm/bifrost_for_linux/mali_midg_coherency.h +new file mode 100755 +index 000000000000..a509cbd5f175 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_midg_coherency.h +@@ -0,0 +1,26 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _MIDG_COHERENCY_H_ ++#define _MIDG_COHERENCY_H_ ++ ++#define COHERENCY_ACE_LITE 0 ++#define COHERENCY_ACE 1 ++#define COHERENCY_NONE 31 ++#define COHERENCY_FEATURE_BIT(x) (1 << (x)) ++ ++#endif /* _MIDG_COHERENCY_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_midg_regmap.h b/drivers/gpu/arm/bifrost_for_linux/mali_midg_regmap.h +new file mode 100755 +index 000000000000..554ed8dcb3eb +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_midg_regmap.h +@@ -0,0 +1,611 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _MIDGARD_REGMAP_H_ ++#define _MIDGARD_REGMAP_H_ ++ ++#include "mali_midg_coherency.h" ++#include "mali_kbase_gpu_id.h" ++ ++/* ++ * Begin Register Offsets ++ */ ++ ++#define GPU_CONTROL_BASE 0x0000 ++#define GPU_CONTROL_REG(r) (GPU_CONTROL_BASE + (r)) ++#define GPU_ID 0x000 /* (RO) GPU and revision identifier */ ++#define L2_FEATURES 0x004 /* (RO) Level 2 cache features */ ++#define SUSPEND_SIZE 0x008 /* (RO) Fixed-function suspend buffer ++ size */ ++#define TILER_FEATURES 0x00C /* (RO) Tiler Features */ ++#define MEM_FEATURES 0x010 /* (RO) Memory system features */ ++#define MMU_FEATURES 0x014 /* (RO) MMU features */ ++#define AS_PRESENT 0x018 /* (RO) Address space slots present */ ++#define JS_PRESENT 0x01C /* (RO) Job slots present */ ++#define GPU_IRQ_RAWSTAT 0x020 /* (RW) */ ++#define GPU_IRQ_CLEAR 0x024 /* (WO) */ ++#define GPU_IRQ_MASK 0x028 /* (RW) */ ++#define GPU_IRQ_STATUS 0x02C /* (RO) */ ++ ++/* IRQ flags */ ++#define GPU_FAULT (1 << 0) /* A GPU Fault has occurred */ ++#define MULTIPLE_GPU_FAULTS (1 << 7) /* More than one GPU Fault occurred. */ ++#define RESET_COMPLETED (1 << 8) /* Set when a reset has completed. Intended to use with SOFT_RESET ++ commands which may take time. */ ++#define POWER_CHANGED_SINGLE (1 << 9) /* Set when a single core has finished powering up or down. */ ++#define POWER_CHANGED_ALL (1 << 10) /* Set when all cores have finished powering up or down ++ and the power manager is idle. */ ++ ++#define PRFCNT_SAMPLE_COMPLETED (1 << 16) /* Set when a performance count sample has completed. */ ++#define CLEAN_CACHES_COMPLETED (1 << 17) /* Set when a cache clean operation has completed. */ ++ ++#define GPU_IRQ_REG_ALL (GPU_FAULT | MULTIPLE_GPU_FAULTS | RESET_COMPLETED \ ++ | POWER_CHANGED_ALL | PRFCNT_SAMPLE_COMPLETED) ++ ++#define GPU_COMMAND 0x030 /* (WO) */ ++#define GPU_STATUS 0x034 /* (RO) */ ++#define LATEST_FLUSH 0x038 /* (RO) */ ++ ++#define GROUPS_L2_COHERENT (1 << 0) /* Cores groups are l2 coherent */ ++#define GPU_DBGEN (1 << 8) /* DBGEN wire status */ ++ ++#define GPU_FAULTSTATUS 0x03C /* (RO) GPU exception type and fault status */ ++#define GPU_FAULTADDRESS_LO 0x040 /* (RO) GPU exception fault address, low word */ ++#define GPU_FAULTADDRESS_HI 0x044 /* (RO) GPU exception fault address, high word */ ++ ++#define PWR_KEY 0x050 /* (WO) Power manager key register */ ++#define PWR_OVERRIDE0 0x054 /* (RW) Power manager override settings */ ++#define PWR_OVERRIDE1 0x058 /* (RW) Power manager override settings */ ++ ++#define PRFCNT_BASE_LO 0x060 /* (RW) Performance counter memory region base address, low word */ ++#define PRFCNT_BASE_HI 0x064 /* (RW) Performance counter memory region base address, high word */ ++#define PRFCNT_CONFIG 0x068 /* (RW) Performance counter configuration */ ++#define PRFCNT_JM_EN 0x06C /* (RW) Performance counter enable flags for Job Manager */ ++#define PRFCNT_SHADER_EN 0x070 /* (RW) Performance counter enable flags for shader cores */ ++#define PRFCNT_TILER_EN 0x074 /* (RW) Performance counter enable flags for tiler */ ++#define PRFCNT_MMU_L2_EN 0x07C /* (RW) Performance counter enable flags for MMU/L2 cache */ ++ ++#define CYCLE_COUNT_LO 0x090 /* (RO) Cycle counter, low word */ ++#define CYCLE_COUNT_HI 0x094 /* (RO) Cycle counter, high word */ ++#define TIMESTAMP_LO 0x098 /* (RO) Global time stamp counter, low word */ ++#define TIMESTAMP_HI 0x09C /* (RO) Global time stamp counter, high word */ ++ ++#define THREAD_MAX_THREADS 0x0A0 /* (RO) Maximum number of threads per core */ ++#define THREAD_MAX_WORKGROUP_SIZE 0x0A4 /* (RO) Maximum workgroup size */ ++#define THREAD_MAX_BARRIER_SIZE 0x0A8 /* (RO) Maximum threads waiting at a barrier */ ++#define THREAD_FEATURES 0x0AC /* (RO) Thread features */ ++ ++#define TEXTURE_FEATURES_0 0x0B0 /* (RO) Support flags for indexed texture formats 0..31 */ ++#define TEXTURE_FEATURES_1 0x0B4 /* (RO) Support flags for indexed texture formats 32..63 */ ++#define TEXTURE_FEATURES_2 0x0B8 /* (RO) Support flags for indexed texture formats 64..95 */ ++ ++#define TEXTURE_FEATURES_REG(n) GPU_CONTROL_REG(TEXTURE_FEATURES_0 + ((n) << 2)) ++ ++#define JS0_FEATURES 0x0C0 /* (RO) Features of job slot 0 */ ++#define JS1_FEATURES 0x0C4 /* (RO) Features of job slot 1 */ ++#define JS2_FEATURES 0x0C8 /* (RO) Features of job slot 2 */ ++#define JS3_FEATURES 0x0CC /* (RO) Features of job slot 3 */ ++#define JS4_FEATURES 0x0D0 /* (RO) Features of job slot 4 */ ++#define JS5_FEATURES 0x0D4 /* (RO) Features of job slot 5 */ ++#define JS6_FEATURES 0x0D8 /* (RO) Features of job slot 6 */ ++#define JS7_FEATURES 0x0DC /* (RO) Features of job slot 7 */ ++#define JS8_FEATURES 0x0E0 /* (RO) Features of job slot 8 */ ++#define JS9_FEATURES 0x0E4 /* (RO) Features of job slot 9 */ ++#define JS10_FEATURES 0x0E8 /* (RO) Features of job slot 10 */ ++#define JS11_FEATURES 0x0EC /* (RO) Features of job slot 11 */ ++#define JS12_FEATURES 0x0F0 /* (RO) Features of job slot 12 */ ++#define JS13_FEATURES 0x0F4 /* (RO) Features of job slot 13 */ ++#define JS14_FEATURES 0x0F8 /* (RO) Features of job slot 14 */ ++#define JS15_FEATURES 0x0FC /* (RO) Features of job slot 15 */ ++ ++#define JS_FEATURES_REG(n) GPU_CONTROL_REG(JS0_FEATURES + ((n) << 2)) ++ ++#define SHADER_PRESENT_LO 0x100 /* (RO) Shader core present bitmap, low word */ ++#define SHADER_PRESENT_HI 0x104 /* (RO) Shader core present bitmap, high word */ ++ ++#define TILER_PRESENT_LO 0x110 /* (RO) Tiler core present bitmap, low word */ ++#define TILER_PRESENT_HI 0x114 /* (RO) Tiler core present bitmap, high word */ ++ ++#define L2_PRESENT_LO 0x120 /* (RO) Level 2 cache present bitmap, low word */ ++#define L2_PRESENT_HI 0x124 /* (RO) Level 2 cache present bitmap, high word */ ++ ++#define STACK_PRESENT_LO 0xE00 /* (RO) Core stack present bitmap, low word */ ++#define STACK_PRESENT_HI 0xE04 /* (RO) Core stack present bitmap, high word */ ++ ++ ++#define SHADER_READY_LO 0x140 /* (RO) Shader core ready bitmap, low word */ ++#define SHADER_READY_HI 0x144 /* (RO) Shader core ready bitmap, high word */ ++ ++#define TILER_READY_LO 0x150 /* (RO) Tiler core ready bitmap, low word */ ++#define TILER_READY_HI 0x154 /* (RO) Tiler core ready bitmap, high word */ ++ ++#define L2_READY_LO 0x160 /* (RO) Level 2 cache ready bitmap, low word */ ++#define L2_READY_HI 0x164 /* (RO) Level 2 cache ready bitmap, high word */ ++ ++#define STACK_READY_LO 0xE10 /* (RO) Core stack ready bitmap, low word */ ++#define STACK_READY_HI 0xE14 /* (RO) Core stack ready bitmap, high word */ ++ ++ ++#define SHADER_PWRON_LO 0x180 /* (WO) Shader core power on bitmap, low word */ ++#define SHADER_PWRON_HI 0x184 /* (WO) Shader core power on bitmap, high word */ ++ ++#define TILER_PWRON_LO 0x190 /* (WO) Tiler core power on bitmap, low word */ ++#define TILER_PWRON_HI 0x194 /* (WO) Tiler core power on bitmap, high word */ ++ ++#define L2_PWRON_LO 0x1A0 /* (WO) Level 2 cache power on bitmap, low word */ ++#define L2_PWRON_HI 0x1A4 /* (WO) Level 2 cache power on bitmap, high word */ ++ ++#define STACK_PWRON_LO 0xE20 /* (RO) Core stack power on bitmap, low word */ ++#define STACK_PWRON_HI 0xE24 /* (RO) Core stack power on bitmap, high word */ ++ ++ ++#define SHADER_PWROFF_LO 0x1C0 /* (WO) Shader core power off bitmap, low word */ ++#define SHADER_PWROFF_HI 0x1C4 /* (WO) Shader core power off bitmap, high word */ ++ ++#define TILER_PWROFF_LO 0x1D0 /* (WO) Tiler core power off bitmap, low word */ ++#define TILER_PWROFF_HI 0x1D4 /* (WO) Tiler core power off bitmap, high word */ ++ ++#define L2_PWROFF_LO 0x1E0 /* (WO) Level 2 cache power off bitmap, low word */ ++#define L2_PWROFF_HI 0x1E4 /* (WO) Level 2 cache power off bitmap, high word */ ++ ++#define STACK_PWROFF_LO 0xE30 /* (RO) Core stack power off bitmap, low word */ ++#define STACK_PWROFF_HI 0xE34 /* (RO) Core stack power off bitmap, high word */ ++ ++ ++#define SHADER_PWRTRANS_LO 0x200 /* (RO) Shader core power transition bitmap, low word */ ++#define SHADER_PWRTRANS_HI 0x204 /* (RO) Shader core power transition bitmap, high word */ ++ ++#define TILER_PWRTRANS_LO 0x210 /* (RO) Tiler core power transition bitmap, low word */ ++#define TILER_PWRTRANS_HI 0x214 /* (RO) Tiler core power transition bitmap, high word */ ++ ++#define L2_PWRTRANS_LO 0x220 /* (RO) Level 2 cache power transition bitmap, low word */ ++#define L2_PWRTRANS_HI 0x224 /* (RO) Level 2 cache power transition bitmap, high word */ ++ ++#define STACK_PWRTRANS_LO 0xE40 /* (RO) Core stack power transition bitmap, low word */ ++#define STACK_PWRTRANS_HI 0xE44 /* (RO) Core stack power transition bitmap, high word */ ++ ++ ++#define SHADER_PWRACTIVE_LO 0x240 /* (RO) Shader core active bitmap, low word */ ++#define SHADER_PWRACTIVE_HI 0x244 /* (RO) Shader core active bitmap, high word */ ++ ++#define TILER_PWRACTIVE_LO 0x250 /* (RO) Tiler core active bitmap, low word */ ++#define TILER_PWRACTIVE_HI 0x254 /* (RO) Tiler core active bitmap, high word */ ++ ++#define L2_PWRACTIVE_LO 0x260 /* (RO) Level 2 cache active bitmap, low word */ ++#define L2_PWRACTIVE_HI 0x264 /* (RO) Level 2 cache active bitmap, high word */ ++ ++#define COHERENCY_FEATURES 0x300 /* (RO) Coherency features present */ ++#define COHERENCY_ENABLE 0x304 /* (RW) Coherency enable */ ++ ++#define JM_CONFIG 0xF00 /* (RW) Job Manager configuration register (Implementation specific register) */ ++#define SHADER_CONFIG 0xF04 /* (RW) Shader core configuration settings (Implementation specific register) */ ++#define TILER_CONFIG 0xF08 /* (RW) Tiler core configuration settings (Implementation specific register) */ ++#define L2_MMU_CONFIG 0xF0C /* (RW) Configuration of the L2 cache and MMU (Implementation specific register) */ ++ ++#define JOB_CONTROL_BASE 0x1000 ++ ++#define JOB_CONTROL_REG(r) (JOB_CONTROL_BASE + (r)) ++ ++#define JOB_IRQ_RAWSTAT 0x000 /* Raw interrupt status register */ ++#define JOB_IRQ_CLEAR 0x004 /* Interrupt clear register */ ++#define JOB_IRQ_MASK 0x008 /* Interrupt mask register */ ++#define JOB_IRQ_STATUS 0x00C /* Interrupt status register */ ++#define JOB_IRQ_JS_STATE 0x010 /* status==active and _next == busy snapshot from last JOB_IRQ_CLEAR */ ++#define JOB_IRQ_THROTTLE 0x014 /* cycles to delay delivering an interrupt externally. The JOB_IRQ_STATUS is NOT affected by this, just the delivery of the interrupt. */ ++ ++#define JOB_SLOT0 0x800 /* Configuration registers for job slot 0 */ ++#define JOB_SLOT1 0x880 /* Configuration registers for job slot 1 */ ++#define JOB_SLOT2 0x900 /* Configuration registers for job slot 2 */ ++#define JOB_SLOT3 0x980 /* Configuration registers for job slot 3 */ ++#define JOB_SLOT4 0xA00 /* Configuration registers for job slot 4 */ ++#define JOB_SLOT5 0xA80 /* Configuration registers for job slot 5 */ ++#define JOB_SLOT6 0xB00 /* Configuration registers for job slot 6 */ ++#define JOB_SLOT7 0xB80 /* Configuration registers for job slot 7 */ ++#define JOB_SLOT8 0xC00 /* Configuration registers for job slot 8 */ ++#define JOB_SLOT9 0xC80 /* Configuration registers for job slot 9 */ ++#define JOB_SLOT10 0xD00 /* Configuration registers for job slot 10 */ ++#define JOB_SLOT11 0xD80 /* Configuration registers for job slot 11 */ ++#define JOB_SLOT12 0xE00 /* Configuration registers for job slot 12 */ ++#define JOB_SLOT13 0xE80 /* Configuration registers for job slot 13 */ ++#define JOB_SLOT14 0xF00 /* Configuration registers for job slot 14 */ ++#define JOB_SLOT15 0xF80 /* Configuration registers for job slot 15 */ ++ ++#define JOB_SLOT_REG(n, r) (JOB_CONTROL_REG(JOB_SLOT0 + ((n) << 7)) + (r)) ++ ++#define JS_HEAD_LO 0x00 /* (RO) Job queue head pointer for job slot n, low word */ ++#define JS_HEAD_HI 0x04 /* (RO) Job queue head pointer for job slot n, high word */ ++#define JS_TAIL_LO 0x08 /* (RO) Job queue tail pointer for job slot n, low word */ ++#define JS_TAIL_HI 0x0C /* (RO) Job queue tail pointer for job slot n, high word */ ++#define JS_AFFINITY_LO 0x10 /* (RO) Core affinity mask for job slot n, low word */ ++#define JS_AFFINITY_HI 0x14 /* (RO) Core affinity mask for job slot n, high word */ ++#define JS_CONFIG 0x18 /* (RO) Configuration settings for job slot n */ ++#define JS_XAFFINITY 0x1C /* (RO) Extended affinity mask for job ++ slot n */ ++ ++#define JS_COMMAND 0x20 /* (WO) Command register for job slot n */ ++#define JS_STATUS 0x24 /* (RO) Status register for job slot n */ ++ ++#define JS_HEAD_NEXT_LO 0x40 /* (RW) Next job queue head pointer for job slot n, low word */ ++#define JS_HEAD_NEXT_HI 0x44 /* (RW) Next job queue head pointer for job slot n, high word */ ++ ++#define JS_AFFINITY_NEXT_LO 0x50 /* (RW) Next core affinity mask for job slot n, low word */ ++#define JS_AFFINITY_NEXT_HI 0x54 /* (RW) Next core affinity mask for job slot n, high word */ ++#define JS_CONFIG_NEXT 0x58 /* (RW) Next configuration settings for job slot n */ ++#define JS_XAFFINITY_NEXT 0x5C /* (RW) Next extended affinity mask for ++ job slot n */ ++ ++#define JS_COMMAND_NEXT 0x60 /* (RW) Next command register for job slot n */ ++ ++#define JS_FLUSH_ID_NEXT 0x70 /* (RW) Next job slot n cache flush ID */ ++ ++#define MEMORY_MANAGEMENT_BASE 0x2000 ++#define MMU_REG(r) (MEMORY_MANAGEMENT_BASE + (r)) ++ ++#define MMU_IRQ_RAWSTAT 0x000 /* (RW) Raw interrupt status register */ ++#define MMU_IRQ_CLEAR 0x004 /* (WO) Interrupt clear register */ ++#define MMU_IRQ_MASK 0x008 /* (RW) Interrupt mask register */ ++#define MMU_IRQ_STATUS 0x00C /* (RO) Interrupt status register */ ++ ++#define MMU_AS0 0x400 /* Configuration registers for address space 0 */ ++#define MMU_AS1 0x440 /* Configuration registers for address space 1 */ ++#define MMU_AS2 0x480 /* Configuration registers for address space 2 */ ++#define MMU_AS3 0x4C0 /* Configuration registers for address space 3 */ ++#define MMU_AS4 0x500 /* Configuration registers for address space 4 */ ++#define MMU_AS5 0x540 /* Configuration registers for address space 5 */ ++#define MMU_AS6 0x580 /* Configuration registers for address space 6 */ ++#define MMU_AS7 0x5C0 /* Configuration registers for address space 7 */ ++#define MMU_AS8 0x600 /* Configuration registers for address space 8 */ ++#define MMU_AS9 0x640 /* Configuration registers for address space 9 */ ++#define MMU_AS10 0x680 /* Configuration registers for address space 10 */ ++#define MMU_AS11 0x6C0 /* Configuration registers for address space 11 */ ++#define MMU_AS12 0x700 /* Configuration registers for address space 12 */ ++#define MMU_AS13 0x740 /* Configuration registers for address space 13 */ ++#define MMU_AS14 0x780 /* Configuration registers for address space 14 */ ++#define MMU_AS15 0x7C0 /* Configuration registers for address space 15 */ ++ ++#define MMU_AS_REG(n, r) (MMU_REG(MMU_AS0 + ((n) << 6)) + (r)) ++ ++#define AS_TRANSTAB_LO 0x00 /* (RW) Translation Table Base Address for address space n, low word */ ++#define AS_TRANSTAB_HI 0x04 /* (RW) Translation Table Base Address for address space n, high word */ ++#define AS_MEMATTR_LO 0x08 /* (RW) Memory attributes for address space n, low word. */ ++#define AS_MEMATTR_HI 0x0C /* (RW) Memory attributes for address space n, high word. */ ++#define AS_LOCKADDR_LO 0x10 /* (RW) Lock region address for address space n, low word */ ++#define AS_LOCKADDR_HI 0x14 /* (RW) Lock region address for address space n, high word */ ++#define AS_COMMAND 0x18 /* (WO) MMU command register for address space n */ ++#define AS_FAULTSTATUS 0x1C /* (RO) MMU fault status register for address space n */ ++#define AS_FAULTADDRESS_LO 0x20 /* (RO) Fault Address for address space n, low word */ ++#define AS_FAULTADDRESS_HI 0x24 /* (RO) Fault Address for address space n, high word */ ++#define AS_STATUS 0x28 /* (RO) Status flags for address space n */ ++ ++ ++/* (RW) Translation table configuration for address space n, low word */ ++#define AS_TRANSCFG_LO 0x30 ++/* (RW) Translation table configuration for address space n, high word */ ++#define AS_TRANSCFG_HI 0x34 ++/* (RO) Secondary fault address for address space n, low word */ ++#define AS_FAULTEXTRA_LO 0x38 ++/* (RO) Secondary fault address for address space n, high word */ ++#define AS_FAULTEXTRA_HI 0x3C ++ ++/* End Register Offsets */ ++ ++/* ++ * MMU_IRQ_RAWSTAT register values. Values are valid also for ++ MMU_IRQ_CLEAR, MMU_IRQ_MASK, MMU_IRQ_STATUS registers. ++ */ ++ ++#define MMU_PAGE_FAULT_FLAGS 16 ++ ++/* Macros returning a bitmask to retrieve page fault or bus error flags from ++ * MMU registers */ ++#define MMU_PAGE_FAULT(n) (1UL << (n)) ++#define MMU_BUS_ERROR(n) (1UL << ((n) + MMU_PAGE_FAULT_FLAGS)) ++ ++/* ++ * Begin LPAE MMU TRANSTAB register values ++ */ ++#define AS_TRANSTAB_LPAE_ADDR_SPACE_MASK 0xfffff000 ++#define AS_TRANSTAB_LPAE_ADRMODE_UNMAPPED (0u << 0) ++#define AS_TRANSTAB_LPAE_ADRMODE_IDENTITY (1u << 1) ++#define AS_TRANSTAB_LPAE_ADRMODE_TABLE (3u << 0) ++#define AS_TRANSTAB_LPAE_READ_INNER (1u << 2) ++#define AS_TRANSTAB_LPAE_SHARE_OUTER (1u << 4) ++ ++#define AS_TRANSTAB_LPAE_ADRMODE_MASK 0x00000003 ++ ++/* ++ * Begin AARCH64 MMU TRANSTAB register values ++ */ ++#define MMU_HW_OUTA_BITS 40 ++#define AS_TRANSTAB_BASE_MASK ((1ULL << MMU_HW_OUTA_BITS) - (1ULL << 4)) ++ ++/* ++ * Begin MMU STATUS register values ++ */ ++#define AS_STATUS_AS_ACTIVE 0x01 ++ ++#define AS_FAULTSTATUS_EXCEPTION_CODE_MASK (0x7<<3) ++#define AS_FAULTSTATUS_EXCEPTION_CODE_TRANSLATION_FAULT (0x0<<3) ++#define AS_FAULTSTATUS_EXCEPTION_CODE_PERMISSION_FAULT (0x1<<3) ++#define AS_FAULTSTATUS_EXCEPTION_CODE_TRANSTAB_BUS_FAULT (0x2<<3) ++#define AS_FAULTSTATUS_EXCEPTION_CODE_ACCESS_FLAG (0x3<<3) ++ ++#define AS_FAULTSTATUS_EXCEPTION_CODE_ADDRESS_SIZE_FAULT (0x4<<3) ++#define AS_FAULTSTATUS_EXCEPTION_CODE_MEMORY_ATTRIBUTES_FAULT (0x5<<3) ++ ++#define AS_FAULTSTATUS_ACCESS_TYPE_MASK (0x3<<8) ++#define AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC (0x0<<8) ++#define AS_FAULTSTATUS_ACCESS_TYPE_EX (0x1<<8) ++#define AS_FAULTSTATUS_ACCESS_TYPE_READ (0x2<<8) ++#define AS_FAULTSTATUS_ACCESS_TYPE_WRITE (0x3<<8) ++ ++/* ++ * Begin MMU TRANSCFG register values ++ */ ++ ++#define AS_TRANSCFG_ADRMODE_LEGACY 0 ++#define AS_TRANSCFG_ADRMODE_UNMAPPED 1 ++#define AS_TRANSCFG_ADRMODE_IDENTITY 2 ++#define AS_TRANSCFG_ADRMODE_AARCH64_4K 6 ++#define AS_TRANSCFG_ADRMODE_AARCH64_64K 8 ++ ++#define AS_TRANSCFG_ADRMODE_MASK 0xF ++ ++ ++/* ++ * Begin TRANSCFG register values ++ */ ++#define AS_TRANSCFG_PTW_MEMATTR_MASK (3 << 24) ++#define AS_TRANSCFG_PTW_MEMATTR_NON_CACHEABLE (1 << 24) ++#define AS_TRANSCFG_PTW_MEMATTR_WRITE_BACK (2 << 24) ++ ++#define AS_TRANSCFG_PTW_SH_MASK ((3 << 28)) ++#define AS_TRANSCFG_PTW_SH_OS (2 << 28) ++#define AS_TRANSCFG_PTW_SH_IS (3 << 28) ++ ++/* ++ * Begin Command Values ++ */ ++ ++/* JS_COMMAND register commands */ ++#define JS_COMMAND_NOP 0x00 /* NOP Operation. Writing this value is ignored */ ++#define JS_COMMAND_START 0x01 /* Start processing a job chain. Writing this value is ignored */ ++#define JS_COMMAND_SOFT_STOP 0x02 /* Gently stop processing a job chain */ ++#define JS_COMMAND_HARD_STOP 0x03 /* Rudely stop processing a job chain */ ++#define JS_COMMAND_SOFT_STOP_0 0x04 /* Execute SOFT_STOP if JOB_CHAIN_FLAG is 0 */ ++#define JS_COMMAND_HARD_STOP_0 0x05 /* Execute HARD_STOP if JOB_CHAIN_FLAG is 0 */ ++#define JS_COMMAND_SOFT_STOP_1 0x06 /* Execute SOFT_STOP if JOB_CHAIN_FLAG is 1 */ ++#define JS_COMMAND_HARD_STOP_1 0x07 /* Execute HARD_STOP if JOB_CHAIN_FLAG is 1 */ ++ ++#define JS_COMMAND_MASK 0x07 /* Mask of bits currently in use by the HW */ ++ ++/* AS_COMMAND register commands */ ++#define AS_COMMAND_NOP 0x00 /* NOP Operation */ ++#define AS_COMMAND_UPDATE 0x01 /* Broadcasts the values in AS_TRANSTAB and ASn_MEMATTR to all MMUs */ ++#define AS_COMMAND_LOCK 0x02 /* Issue a lock region command to all MMUs */ ++#define AS_COMMAND_UNLOCK 0x03 /* Issue a flush region command to all MMUs */ ++#define AS_COMMAND_FLUSH 0x04 /* Flush all L2 caches then issue a flush region command to all MMUs ++ (deprecated - only for use with T60x) */ ++#define AS_COMMAND_FLUSH_PT 0x04 /* Flush all L2 caches then issue a flush region command to all MMUs */ ++#define AS_COMMAND_FLUSH_MEM 0x05 /* Wait for memory accesses to complete, flush all the L1s cache then ++ flush all L2 caches then issue a flush region command to all MMUs */ ++ ++/* Possible values of JS_CONFIG and JS_CONFIG_NEXT registers */ ++#define JS_CONFIG_START_FLUSH_NO_ACTION (0u << 0) ++#define JS_CONFIG_START_FLUSH_CLEAN (1u << 8) ++#define JS_CONFIG_START_FLUSH_CLEAN_INVALIDATE (3u << 8) ++#define JS_CONFIG_START_MMU (1u << 10) ++#define JS_CONFIG_JOB_CHAIN_FLAG (1u << 11) ++#define JS_CONFIG_END_FLUSH_NO_ACTION JS_CONFIG_START_FLUSH_NO_ACTION ++#define JS_CONFIG_END_FLUSH_CLEAN (1u << 12) ++#define JS_CONFIG_END_FLUSH_CLEAN_INVALIDATE (3u << 12) ++#define JS_CONFIG_ENABLE_FLUSH_REDUCTION (1u << 14) ++#define JS_CONFIG_DISABLE_DESCRIPTOR_WR_BK (1u << 15) ++#define JS_CONFIG_THREAD_PRI(n) ((n) << 16) ++ ++/* JS_XAFFINITY register values */ ++#define JS_XAFFINITY_XAFFINITY_ENABLE (1u << 0) ++#define JS_XAFFINITY_TILER_ENABLE (1u << 8) ++#define JS_XAFFINITY_CACHE_ENABLE (1u << 16) ++ ++/* JS_STATUS register values */ ++ ++/* NOTE: Please keep this values in sync with enum base_jd_event_code in mali_base_kernel.h. ++ * The values are separated to avoid dependency of userspace and kernel code. ++ */ ++ ++/* Group of values representing the job status insead a particular fault */ ++#define JS_STATUS_NO_EXCEPTION_BASE 0x00 ++#define JS_STATUS_INTERRUPTED (JS_STATUS_NO_EXCEPTION_BASE + 0x02) /* 0x02 means INTERRUPTED */ ++#define JS_STATUS_STOPPED (JS_STATUS_NO_EXCEPTION_BASE + 0x03) /* 0x03 means STOPPED */ ++#define JS_STATUS_TERMINATED (JS_STATUS_NO_EXCEPTION_BASE + 0x04) /* 0x04 means TERMINATED */ ++ ++/* General fault values */ ++#define JS_STATUS_FAULT_BASE 0x40 ++#define JS_STATUS_CONFIG_FAULT (JS_STATUS_FAULT_BASE) /* 0x40 means CONFIG FAULT */ ++#define JS_STATUS_POWER_FAULT (JS_STATUS_FAULT_BASE + 0x01) /* 0x41 means POWER FAULT */ ++#define JS_STATUS_READ_FAULT (JS_STATUS_FAULT_BASE + 0x02) /* 0x42 means READ FAULT */ ++#define JS_STATUS_WRITE_FAULT (JS_STATUS_FAULT_BASE + 0x03) /* 0x43 means WRITE FAULT */ ++#define JS_STATUS_AFFINITY_FAULT (JS_STATUS_FAULT_BASE + 0x04) /* 0x44 means AFFINITY FAULT */ ++#define JS_STATUS_BUS_FAULT (JS_STATUS_FAULT_BASE + 0x08) /* 0x48 means BUS FAULT */ ++ ++/* Instruction or data faults */ ++#define JS_STATUS_INSTRUCTION_FAULT_BASE 0x50 ++#define JS_STATUS_INSTR_INVALID_PC (JS_STATUS_INSTRUCTION_FAULT_BASE) /* 0x50 means INSTR INVALID PC */ ++#define JS_STATUS_INSTR_INVALID_ENC (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x01) /* 0x51 means INSTR INVALID ENC */ ++#define JS_STATUS_INSTR_TYPE_MISMATCH (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x02) /* 0x52 means INSTR TYPE MISMATCH */ ++#define JS_STATUS_INSTR_OPERAND_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x03) /* 0x53 means INSTR OPERAND FAULT */ ++#define JS_STATUS_INSTR_TLS_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x04) /* 0x54 means INSTR TLS FAULT */ ++#define JS_STATUS_INSTR_BARRIER_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x05) /* 0x55 means INSTR BARRIER FAULT */ ++#define JS_STATUS_INSTR_ALIGN_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x06) /* 0x56 means INSTR ALIGN FAULT */ ++/* NOTE: No fault with 0x57 code defined in spec. */ ++#define JS_STATUS_DATA_INVALID_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x08) /* 0x58 means DATA INVALID FAULT */ ++#define JS_STATUS_TILE_RANGE_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x09) /* 0x59 means TILE RANGE FAULT */ ++#define JS_STATUS_ADDRESS_RANGE_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x0A) /* 0x5A means ADDRESS RANGE FAULT */ ++ ++/* Other faults */ ++#define JS_STATUS_MEMORY_FAULT_BASE 0x60 ++#define JS_STATUS_OUT_OF_MEMORY (JS_STATUS_MEMORY_FAULT_BASE) /* 0x60 means OUT OF MEMORY */ ++#define JS_STATUS_UNKNOWN 0x7F /* 0x7F means UNKNOWN */ ++ ++/* GPU_COMMAND values */ ++#define GPU_COMMAND_NOP 0x00 /* No operation, nothing happens */ ++#define GPU_COMMAND_SOFT_RESET 0x01 /* Stop all external bus interfaces, and then reset the entire GPU. */ ++#define GPU_COMMAND_HARD_RESET 0x02 /* Immediately reset the entire GPU. */ ++#define GPU_COMMAND_PRFCNT_CLEAR 0x03 /* Clear all performance counters, setting them all to zero. */ ++#define GPU_COMMAND_PRFCNT_SAMPLE 0x04 /* Sample all performance counters, writing them out to memory */ ++#define GPU_COMMAND_CYCLE_COUNT_START 0x05 /* Starts the cycle counter, and system timestamp propagation */ ++#define GPU_COMMAND_CYCLE_COUNT_STOP 0x06 /* Stops the cycle counter, and system timestamp propagation */ ++#define GPU_COMMAND_CLEAN_CACHES 0x07 /* Clean all caches */ ++#define GPU_COMMAND_CLEAN_INV_CACHES 0x08 /* Clean and invalidate all caches */ ++#define GPU_COMMAND_SET_PROTECTED_MODE 0x09 /* Places the GPU in protected mode */ ++ ++/* End Command Values */ ++ ++/* GPU_STATUS values */ ++#define GPU_STATUS_PRFCNT_ACTIVE (1 << 2) /* Set if the performance counters are active. */ ++#define GPU_STATUS_PROTECTED_MODE_ACTIVE (1 << 7) /* Set if protected mode is active */ ++ ++/* PRFCNT_CONFIG register values */ ++#define PRFCNT_CONFIG_MODE_SHIFT 0 /* Counter mode position. */ ++#define PRFCNT_CONFIG_AS_SHIFT 4 /* Address space bitmap position. */ ++#define PRFCNT_CONFIG_SETSELECT_SHIFT 8 /* Set select position. */ ++ ++#define PRFCNT_CONFIG_MODE_OFF 0 /* The performance counters are disabled. */ ++#define PRFCNT_CONFIG_MODE_MANUAL 1 /* The performance counters are enabled, but are only written out when a PRFCNT_SAMPLE command is issued using the GPU_COMMAND register. */ ++#define PRFCNT_CONFIG_MODE_TILE 2 /* The performance counters are enabled, and are written out each time a tile finishes rendering. */ ++ ++/* AS_MEMATTR values: */ ++/* Use GPU implementation-defined caching policy. */ ++#define AS_MEMATTR_IMPL_DEF_CACHE_POLICY 0x88ull ++/* The attribute set to force all resources to be cached. */ ++#define AS_MEMATTR_FORCE_TO_CACHE_ALL 0x8Full ++/* Inner write-alloc cache setup, no outer caching */ ++#define AS_MEMATTR_WRITE_ALLOC 0x8Dull ++ ++/* Set to implementation defined, outer caching */ ++#define AS_MEMATTR_AARCH64_OUTER_IMPL_DEF 0x88ull ++/* Set to write back memory, outer caching */ ++#define AS_MEMATTR_AARCH64_OUTER_WA 0x8Dull ++ ++/* Use GPU implementation-defined caching policy. */ ++#define AS_MEMATTR_LPAE_IMPL_DEF_CACHE_POLICY 0x48ull ++/* The attribute set to force all resources to be cached. */ ++#define AS_MEMATTR_LPAE_FORCE_TO_CACHE_ALL 0x4Full ++/* Inner write-alloc cache setup, no outer caching */ ++#define AS_MEMATTR_LPAE_WRITE_ALLOC 0x4Dull ++/* Set to implementation defined, outer caching */ ++#define AS_MEMATTR_LPAE_OUTER_IMPL_DEF 0x88ull ++/* Set to write back memory, outer caching */ ++#define AS_MEMATTR_LPAE_OUTER_WA 0x8Dull ++ ++/* Symbols for default MEMATTR to use ++ * Default is - HW implementation defined caching */ ++#define AS_MEMATTR_INDEX_DEFAULT 0 ++#define AS_MEMATTR_INDEX_DEFAULT_ACE 3 ++ ++/* HW implementation defined caching */ ++#define AS_MEMATTR_INDEX_IMPL_DEF_CACHE_POLICY 0 ++/* Force cache on */ ++#define AS_MEMATTR_INDEX_FORCE_TO_CACHE_ALL 1 ++/* Write-alloc */ ++#define AS_MEMATTR_INDEX_WRITE_ALLOC 2 ++/* Outer coherent, inner implementation defined policy */ ++#define AS_MEMATTR_INDEX_OUTER_IMPL_DEF 3 ++/* Outer coherent, write alloc inner */ ++#define AS_MEMATTR_INDEX_OUTER_WA 4 ++ ++/* JS_FEATURES register */ ++ ++#define JS_FEATURE_NULL_JOB (1u << 1) ++#define JS_FEATURE_SET_VALUE_JOB (1u << 2) ++#define JS_FEATURE_CACHE_FLUSH_JOB (1u << 3) ++#define JS_FEATURE_COMPUTE_JOB (1u << 4) ++#define JS_FEATURE_VERTEX_JOB (1u << 5) ++#define JS_FEATURE_GEOMETRY_JOB (1u << 6) ++#define JS_FEATURE_TILER_JOB (1u << 7) ++#define JS_FEATURE_FUSED_JOB (1u << 8) ++#define JS_FEATURE_FRAGMENT_JOB (1u << 9) ++ ++/* End JS_FEATURES register */ ++ ++/* L2_MMU_CONFIG register */ ++#define L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY_SHIFT (23) ++#define L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY (0x1 << L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY_SHIFT) ++#define L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT (24) ++#define L2_MMU_CONFIG_LIMIT_EXTERNAL_READS (0x3 << L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT) ++#define L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_OCTANT (0x1 << L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT) ++#define L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_QUARTER (0x2 << L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT) ++#define L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_HALF (0x3 << L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT) ++ ++#define L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT (26) ++#define L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES (0x3 << L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT) ++#define L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_OCTANT (0x1 << L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT) ++#define L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_QUARTER (0x2 << L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT) ++#define L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_HALF (0x3 << L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT) ++/* End L2_MMU_CONFIG register */ ++ ++/* THREAD_* registers */ ++ ++/* THREAD_FEATURES IMPLEMENTATION_TECHNOLOGY values */ ++#define IMPLEMENTATION_UNSPECIFIED 0 ++#define IMPLEMENTATION_SILICON 1 ++#define IMPLEMENTATION_FPGA 2 ++#define IMPLEMENTATION_MODEL 3 ++ ++/* Default values when registers are not supported by the implemented hardware */ ++#define THREAD_MT_DEFAULT 256 ++#define THREAD_MWS_DEFAULT 256 ++#define THREAD_MBS_DEFAULT 256 ++#define THREAD_MR_DEFAULT 1024 ++#define THREAD_MTQ_DEFAULT 4 ++#define THREAD_MTGS_DEFAULT 10 ++ ++/* End THREAD_* registers */ ++ ++/* SHADER_CONFIG register */ ++ ++#define SC_ALT_COUNTERS (1ul << 3) ++#define SC_OVERRIDE_FWD_PIXEL_KILL (1ul << 4) ++#define SC_SDC_DISABLE_OQ_DISCARD (1ul << 6) ++#define SC_LS_ALLOW_ATTR_TYPES (1ul << 16) ++#define SC_LS_PAUSEBUFFER_DISABLE (1ul << 16) ++#define SC_TLS_HASH_ENABLE (1ul << 17) ++#define SC_LS_ATTR_CHECK_DISABLE (1ul << 18) ++#define SC_ENABLE_TEXGRD_FLAGS (1ul << 25) ++/* End SHADER_CONFIG register */ ++ ++/* TILER_CONFIG register */ ++ ++#define TC_CLOCK_GATE_OVERRIDE (1ul << 0) ++ ++/* End TILER_CONFIG register */ ++ ++/* JM_CONFIG register */ ++ ++#define JM_TIMESTAMP_OVERRIDE (1ul << 0) ++#define JM_CLOCK_GATE_OVERRIDE (1ul << 1) ++#define JM_JOB_THROTTLE_ENABLE (1ul << 2) ++#define JM_JOB_THROTTLE_LIMIT_SHIFT (3) ++#define JM_MAX_JOB_THROTTLE_LIMIT (0x3F) ++#define JM_FORCE_COHERENCY_FEATURES_SHIFT (2) ++#define JM_IDVS_GROUP_SIZE_SHIFT (16) ++#define JM_MAX_IDVS_GROUP_SIZE (0x3F) ++/* End JM_CONFIG register */ ++ ++ ++#endif /* _MIDGARD_REGMAP_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_timeline.h b/drivers/gpu/arm/bifrost_for_linux/mali_timeline.h +new file mode 100755 +index 000000000000..bd5f6614b6bb +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_timeline.h +@@ -0,0 +1,396 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#undef TRACE_SYSTEM ++#define TRACE_SYSTEM mali_timeline ++ ++#if !defined(_MALI_TIMELINE_H) || defined(TRACE_HEADER_MULTI_READ) ++#define _MALI_TIMELINE_H ++ ++#include ++ ++TRACE_EVENT(mali_timeline_atoms_in_flight, ++ ++ TP_PROTO(u64 ts_sec, ++ u32 ts_nsec, ++ int tgid, ++ int count), ++ ++ TP_ARGS(ts_sec, ++ ts_nsec, ++ tgid, ++ count), ++ ++ TP_STRUCT__entry( ++ __field(u64, ts_sec) ++ __field(u32, ts_nsec) ++ __field(int, tgid) ++ __field(int, count) ++ ), ++ ++ TP_fast_assign( ++ __entry->ts_sec = ts_sec; ++ __entry->ts_nsec = ts_nsec; ++ __entry->tgid = tgid; ++ __entry->count = count; ++ ), ++ ++ TP_printk("%i,%i.%.9i,%i,%i", CTX_SET_NR_ATOMS_IN_FLIGHT, ++ (int)__entry->ts_sec, ++ (int)__entry->ts_nsec, ++ __entry->tgid, ++ __entry->count) ++); ++ ++ ++TRACE_EVENT(mali_timeline_atom, ++ ++ TP_PROTO(u64 ts_sec, ++ u32 ts_nsec, ++ int event_type, ++ int tgid, ++ int atom_id), ++ ++ TP_ARGS(ts_sec, ++ ts_nsec, ++ event_type, ++ tgid, ++ atom_id), ++ ++ TP_STRUCT__entry( ++ __field(u64, ts_sec) ++ __field(u32, ts_nsec) ++ __field(int, event_type) ++ __field(int, tgid) ++ __field(int, atom_id) ++ ), ++ ++ TP_fast_assign( ++ __entry->ts_sec = ts_sec; ++ __entry->ts_nsec = ts_nsec; ++ __entry->event_type = event_type; ++ __entry->tgid = tgid; ++ __entry->atom_id = atom_id; ++ ), ++ ++ TP_printk("%i,%i.%.9i,%i,%i,%i", __entry->event_type, ++ (int)__entry->ts_sec, ++ (int)__entry->ts_nsec, ++ __entry->tgid, ++ __entry->atom_id, ++ __entry->atom_id) ++); ++ ++TRACE_EVENT(mali_timeline_gpu_slot_active, ++ ++ TP_PROTO(u64 ts_sec, ++ u32 ts_nsec, ++ int event_type, ++ int tgid, ++ int js, ++ int count), ++ ++ TP_ARGS(ts_sec, ++ ts_nsec, ++ event_type, ++ tgid, ++ js, ++ count), ++ ++ TP_STRUCT__entry( ++ __field(u64, ts_sec) ++ __field(u32, ts_nsec) ++ __field(int, event_type) ++ __field(int, tgid) ++ __field(int, js) ++ __field(int, count) ++ ), ++ ++ TP_fast_assign( ++ __entry->ts_sec = ts_sec; ++ __entry->ts_nsec = ts_nsec; ++ __entry->event_type = event_type; ++ __entry->tgid = tgid; ++ __entry->js = js; ++ __entry->count = count; ++ ), ++ ++ TP_printk("%i,%i.%.9i,%i,%i,%i", __entry->event_type, ++ (int)__entry->ts_sec, ++ (int)__entry->ts_nsec, ++ __entry->tgid, ++ __entry->js, ++ __entry->count) ++); ++ ++TRACE_EVENT(mali_timeline_gpu_slot_action, ++ ++ TP_PROTO(u64 ts_sec, ++ u32 ts_nsec, ++ int event_type, ++ int tgid, ++ int js, ++ int count), ++ ++ TP_ARGS(ts_sec, ++ ts_nsec, ++ event_type, ++ tgid, ++ js, ++ count), ++ ++ TP_STRUCT__entry( ++ __field(u64, ts_sec) ++ __field(u32, ts_nsec) ++ __field(int, event_type) ++ __field(int, tgid) ++ __field(int, js) ++ __field(int, count) ++ ), ++ ++ TP_fast_assign( ++ __entry->ts_sec = ts_sec; ++ __entry->ts_nsec = ts_nsec; ++ __entry->event_type = event_type; ++ __entry->tgid = tgid; ++ __entry->js = js; ++ __entry->count = count; ++ ), ++ ++ TP_printk("%i,%i.%.9i,%i,%i,%i", __entry->event_type, ++ (int)__entry->ts_sec, ++ (int)__entry->ts_nsec, ++ __entry->tgid, ++ __entry->js, ++ __entry->count) ++); ++ ++TRACE_EVENT(mali_timeline_gpu_power_active, ++ ++ TP_PROTO(u64 ts_sec, ++ u32 ts_nsec, ++ int event_type, ++ int active), ++ ++ TP_ARGS(ts_sec, ++ ts_nsec, ++ event_type, ++ active), ++ ++ TP_STRUCT__entry( ++ __field(u64, ts_sec) ++ __field(u32, ts_nsec) ++ __field(int, event_type) ++ __field(int, active) ++ ), ++ ++ TP_fast_assign( ++ __entry->ts_sec = ts_sec; ++ __entry->ts_nsec = ts_nsec; ++ __entry->event_type = event_type; ++ __entry->active = active; ++ ), ++ ++ TP_printk("%i,%i.%.9i,0,%i", __entry->event_type, ++ (int)__entry->ts_sec, ++ (int)__entry->ts_nsec, ++ __entry->active) ++ ++); ++ ++TRACE_EVENT(mali_timeline_l2_power_active, ++ ++ TP_PROTO(u64 ts_sec, ++ u32 ts_nsec, ++ int event_type, ++ int state), ++ ++ TP_ARGS(ts_sec, ++ ts_nsec, ++ event_type, ++ state), ++ ++ TP_STRUCT__entry( ++ __field(u64, ts_sec) ++ __field(u32, ts_nsec) ++ __field(int, event_type) ++ __field(int, state) ++ ), ++ ++ TP_fast_assign( ++ __entry->ts_sec = ts_sec; ++ __entry->ts_nsec = ts_nsec; ++ __entry->event_type = event_type; ++ __entry->state = state; ++ ), ++ ++ TP_printk("%i,%i.%.9i,0,%i", __entry->event_type, ++ (int)__entry->ts_sec, ++ (int)__entry->ts_nsec, ++ __entry->state) ++ ++); ++TRACE_EVENT(mali_timeline_pm_event, ++ ++ TP_PROTO(u64 ts_sec, ++ u32 ts_nsec, ++ int event_type, ++ int pm_event_type, ++ unsigned int pm_event_id), ++ ++ TP_ARGS(ts_sec, ++ ts_nsec, ++ event_type, ++ pm_event_type, ++ pm_event_id), ++ ++ TP_STRUCT__entry( ++ __field(u64, ts_sec) ++ __field(u32, ts_nsec) ++ __field(int, event_type) ++ __field(int, pm_event_type) ++ __field(unsigned int, pm_event_id) ++ ), ++ ++ TP_fast_assign( ++ __entry->ts_sec = ts_sec; ++ __entry->ts_nsec = ts_nsec; ++ __entry->event_type = event_type; ++ __entry->pm_event_type = pm_event_type; ++ __entry->pm_event_id = pm_event_id; ++ ), ++ ++ TP_printk("%i,%i.%.9i,0,%i,%u", __entry->event_type, ++ (int)__entry->ts_sec, ++ (int)__entry->ts_nsec, ++ __entry->pm_event_type, __entry->pm_event_id) ++ ++); ++ ++TRACE_EVENT(mali_timeline_slot_atom, ++ ++ TP_PROTO(u64 ts_sec, ++ u32 ts_nsec, ++ int event_type, ++ int tgid, ++ int js, ++ int atom_id), ++ ++ TP_ARGS(ts_sec, ++ ts_nsec, ++ event_type, ++ tgid, ++ js, ++ atom_id), ++ ++ TP_STRUCT__entry( ++ __field(u64, ts_sec) ++ __field(u32, ts_nsec) ++ __field(int, event_type) ++ __field(int, tgid) ++ __field(int, js) ++ __field(int, atom_id) ++ ), ++ ++ TP_fast_assign( ++ __entry->ts_sec = ts_sec; ++ __entry->ts_nsec = ts_nsec; ++ __entry->event_type = event_type; ++ __entry->tgid = tgid; ++ __entry->js = js; ++ __entry->atom_id = atom_id; ++ ), ++ ++ TP_printk("%i,%i.%.9i,%i,%i,%i", __entry->event_type, ++ (int)__entry->ts_sec, ++ (int)__entry->ts_nsec, ++ __entry->tgid, ++ __entry->js, ++ __entry->atom_id) ++); ++ ++TRACE_EVENT(mali_timeline_pm_checktrans, ++ ++ TP_PROTO(u64 ts_sec, ++ u32 ts_nsec, ++ int trans_code, ++ int trans_id), ++ ++ TP_ARGS(ts_sec, ++ ts_nsec, ++ trans_code, ++ trans_id), ++ ++ TP_STRUCT__entry( ++ __field(u64, ts_sec) ++ __field(u32, ts_nsec) ++ __field(int, trans_code) ++ __field(int, trans_id) ++ ), ++ ++ TP_fast_assign( ++ __entry->ts_sec = ts_sec; ++ __entry->ts_nsec = ts_nsec; ++ __entry->trans_code = trans_code; ++ __entry->trans_id = trans_id; ++ ), ++ ++ TP_printk("%i,%i.%.9i,0,%i", __entry->trans_code, ++ (int)__entry->ts_sec, ++ (int)__entry->ts_nsec, ++ __entry->trans_id) ++ ++); ++ ++TRACE_EVENT(mali_timeline_context_active, ++ ++ TP_PROTO(u64 ts_sec, ++ u32 ts_nsec, ++ int count), ++ ++ TP_ARGS(ts_sec, ++ ts_nsec, ++ count), ++ ++ TP_STRUCT__entry( ++ __field(u64, ts_sec) ++ __field(u32, ts_nsec) ++ __field(int, count) ++ ), ++ ++ TP_fast_assign( ++ __entry->ts_sec = ts_sec; ++ __entry->ts_nsec = ts_nsec; ++ __entry->count = count; ++ ), ++ ++ TP_printk("%i,%i.%.9i,0,%i", SW_SET_CONTEXT_ACTIVE, ++ (int)__entry->ts_sec, ++ (int)__entry->ts_nsec, ++ __entry->count) ++); ++ ++#endif /* _MALI_TIMELINE_H */ ++ ++#undef TRACE_INCLUDE_PATH ++#define TRACE_INCLUDE_PATH . ++ ++/* This part must be outside protection */ ++#include ++ +diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_uk.h b/drivers/gpu/arm/bifrost_for_linux/mali_uk.h +new file mode 100755 +index 000000000000..841d03fb5873 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/mali_uk.h +@@ -0,0 +1,141 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010, 2012-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/** ++ * @file mali_uk.h ++ * Types and definitions that are common across OSs for both the user ++ * and kernel side of the User-Kernel interface. ++ */ ++ ++#ifndef _UK_H_ ++#define _UK_H_ ++ ++#ifdef __cplusplus ++extern "C" { ++#endif /* __cplusplus */ ++ ++/** ++ * @addtogroup base_api ++ * @{ ++ */ ++ ++/** ++ * @defgroup uk_api User-Kernel Interface API ++ * ++ * The User-Kernel Interface abstracts the communication mechanism between the user and kernel-side code of device ++ * drivers developed as part of the Midgard DDK. Currently that includes the Base driver and the UMP driver. ++ * ++ * It exposes an OS independent API to user-side code (UKU) which routes functions calls to an OS-independent ++ * kernel-side API (UKK) via an OS-specific communication mechanism. ++ * ++ * This API is internal to the Midgard DDK and is not exposed to any applications. ++ * ++ * @{ ++ */ ++ ++/** ++ * These are identifiers for kernel-side drivers implementing a UK interface, aka UKK clients. The ++ * UK module maps this to an OS specific device name, e.g. "gpu_base" -> "GPU0:". Specify this ++ * identifier to select a UKK client to the uku_open() function. ++ * ++ * When a new UKK client driver is created a new identifier needs to be added to the uk_client_id ++ * enumeration and the uku_open() implemenation for the various OS ports need to be updated to ++ * provide a mapping of the identifier to the OS specific device name. ++ * ++ */ ++enum uk_client_id { ++ /** ++ * Value used to identify the Base driver UK client. ++ */ ++ UK_CLIENT_MALI_T600_BASE, ++ ++ /** The number of uk clients supported. This must be the last member of the enum */ ++ UK_CLIENT_COUNT ++}; ++ ++/** ++ * Each function callable through the UK interface has a unique number. ++ * Functions provided by UK clients start from number UK_FUNC_ID. ++ * Numbers below UK_FUNC_ID are used for internal UK functions. ++ */ ++enum uk_func { ++ UKP_FUNC_ID_CHECK_VERSION, /**< UKK Core internal function */ ++ /** ++ * Each UK client numbers the functions they provide starting from ++ * number UK_FUNC_ID. This number is then eventually assigned to the ++ * id field of the union uk_header structure when preparing to make a ++ * UK call. See your UK client for a list of their function numbers. ++ */ ++ UK_FUNC_ID = 512 ++}; ++ ++/** ++ * Arguments for a UK call are stored in a structure. This structure consists ++ * of a fixed size header and a payload. The header carries a 32-bit number ++ * identifying the UK function to be called (see uk_func). When the UKK client ++ * receives this header and executed the requested UK function, it will use ++ * the same header to store the result of the function in the form of a ++ * int return code. The size of this structure is such that the ++ * first member of the payload following the header can be accessed efficiently ++ * on a 32 and 64-bit kernel and the structure has the same size regardless ++ * of a 32 or 64-bit kernel. The uk_kernel_size_type type should be defined ++ * accordingly in the OS specific mali_uk_os.h header file. ++ */ ++union uk_header { ++ /** ++ * 32-bit number identifying the UK function to be called. ++ * Also see uk_func. ++ */ ++ u32 id; ++ /** ++ * The int return code returned by the called UK function. ++ * See the specification of the particular UK function you are ++ * calling for the meaning of the error codes returned. All ++ * UK functions return 0 on success. ++ */ ++ u32 ret; ++ /* ++ * Used to ensure 64-bit alignment of this union. Do not remove. ++ * This field is used for padding and does not need to be initialized. ++ */ ++ u64 sizer; ++}; ++ ++/** ++ * This structure carries a 16-bit major and minor number and is sent along with an internal UK call ++ * used during uku_open to identify the versions of the UK module in use by the user-side and kernel-side. ++ */ ++struct uku_version_check_args { ++ union uk_header header; ++ /**< UK call header */ ++ u16 major; ++ /**< This field carries the user-side major version on input and the kernel-side major version on output */ ++ u16 minor; ++ /**< This field carries the user-side minor version on input and the kernel-side minor version on output. */ ++ u8 padding[4]; ++}; ++ ++/** @} end group uk_api */ ++ ++/** @} *//* end group base_api */ ++ ++#ifdef __cplusplus ++} ++#endif /* __cplusplus */ ++#endif /* _UK_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/Kconfig b/drivers/gpu/arm/bifrost_for_linux/platform/Kconfig +new file mode 100755 +index 000000000000..38835d3d1531 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/platform/Kconfig +@@ -0,0 +1,24 @@ ++# ++# (C) COPYRIGHT 2012-2013, 2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++ ++ ++# Add your platform specific Kconfig file here ++# ++# "drivers/gpu/arm/midgard/platform/xxx/Kconfig" ++# ++# Where xxx is the platform name is the name set in MALI_PLATFORM_NAME ++# ++ +diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/devicetree/Kbuild b/drivers/gpu/arm/bifrost_for_linux/platform/devicetree/Kbuild +new file mode 100755 +index 000000000000..d40d7982ff04 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/platform/devicetree/Kbuild +@@ -0,0 +1,18 @@ ++# ++# (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++mali_kbase-y += \ ++ $(MALI_PLATFORM_DIR)/mali_kbase_config_devicetree.o \ ++ $(MALI_PLATFORM_DIR)/mali_kbase_runtime_pm.o +diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/devicetree/mali_kbase_config_devicetree.c b/drivers/gpu/arm/bifrost_for_linux/platform/devicetree/mali_kbase_config_devicetree.c +new file mode 100755 +index 000000000000..29ccc29e4125 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/platform/devicetree/mali_kbase_config_devicetree.c +@@ -0,0 +1,40 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015, 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include ++ ++int kbase_platform_early_init(void) ++{ ++ /* Nothing needed at this stage */ ++ return 0; ++} ++ ++static struct kbase_platform_config dummy_platform_config; ++ ++struct kbase_platform_config *kbase_get_platform_config(void) ++{ ++ return &dummy_platform_config; ++} ++ ++int kbase_platform_register(void) ++{ ++ return 0; ++} ++ ++void kbase_platform_unregister(void) ++{ ++} +diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/devicetree/mali_kbase_config_platform.h b/drivers/gpu/arm/bifrost_for_linux/platform/devicetree/mali_kbase_config_platform.h +new file mode 100755 +index 000000000000..2ceca34945b9 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/platform/devicetree/mali_kbase_config_platform.h +@@ -0,0 +1,80 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/** ++ * Maximum frequency GPU will be clocked at. Given in kHz. ++ * This must be specified as there is no default value. ++ * ++ * Attached value: number in kHz ++ * Default value: NA ++ */ ++#define GPU_FREQ_KHZ_MAX (5000) ++/** ++ * Minimum frequency GPU will be clocked at. Given in kHz. ++ * This must be specified as there is no default value. ++ * ++ * Attached value: number in kHz ++ * Default value: NA ++ */ ++#define GPU_FREQ_KHZ_MIN (5000) ++ ++/** ++ * CPU_SPEED_FUNC - A pointer to a function that calculates the CPU clock ++ * ++ * CPU clock speed of the platform is in MHz - see kbase_cpu_clk_speed_func ++ * for the function prototype. ++ * ++ * Attached value: A kbase_cpu_clk_speed_func. ++ * Default Value: NA ++ */ ++#define CPU_SPEED_FUNC (NULL) ++ ++/** ++ * GPU_SPEED_FUNC - A pointer to a function that calculates the GPU clock ++ * ++ * GPU clock speed of the platform in MHz - see kbase_gpu_clk_speed_func ++ * for the function prototype. ++ * ++ * Attached value: A kbase_gpu_clk_speed_func. ++ * Default Value: NA ++ */ ++#define GPU_SPEED_FUNC (NULL) ++ ++/** ++ * Power management configuration ++ * ++ * Attached value: pointer to @ref kbase_pm_callback_conf ++ * Default value: See @ref kbase_pm_callback_conf ++ */ ++#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks) ++ ++/** ++ * Platform specific configuration functions ++ * ++ * Attached value: pointer to @ref kbase_platform_funcs_conf ++ * Default value: See @ref kbase_platform_funcs_conf ++ */ ++#define PLATFORM_FUNCS (NULL) ++ ++extern struct kbase_pm_callback_conf pm_callbacks; ++ ++/** ++ * Autosuspend delay ++ * ++ * The delay time (in milliseconds) to be used for autosuspend ++ */ ++#define AUTO_SUSPEND_DELAY (100) +diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/devicetree/mali_kbase_runtime_pm.c b/drivers/gpu/arm/bifrost_for_linux/platform/devicetree/mali_kbase_runtime_pm.c +new file mode 100755 +index 000000000000..9fe37c8d835e +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/platform/devicetree/mali_kbase_runtime_pm.c +@@ -0,0 +1,121 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015, 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include ++#include ++#include ++#include "mali_kbase_config_platform.h" ++ ++static int pm_callback_power_on(struct kbase_device *kbdev) ++{ ++ int ret = 1; /* Assume GPU has been powered off */ ++ int error; ++ ++ dev_dbg(kbdev->dev, "pm_callback_power_on %p\n", ++ (void *)kbdev->dev->pm_domain); ++ ++ error = pm_runtime_get_sync(kbdev->dev); ++ if (error == 1) { ++ /* ++ * Let core know that the chip has not been ++ * powered off, so we can save on re-initialization. ++ */ ++ ret = 0; ++ } ++ ++ dev_dbg(kbdev->dev, "pm_runtime_get_sync returned %d\n", error); ++ ++ return ret; ++} ++ ++static void pm_callback_power_off(struct kbase_device *kbdev) ++{ ++ dev_dbg(kbdev->dev, "pm_callback_power_off\n"); ++ ++ pm_runtime_mark_last_busy(kbdev->dev); ++ pm_runtime_put_autosuspend(kbdev->dev); ++} ++ ++int kbase_device_runtime_init(struct kbase_device *kbdev) ++{ ++ int ret = 0; ++ ++ dev_dbg(kbdev->dev, "kbase_device_runtime_init\n"); ++ ++ pm_runtime_set_autosuspend_delay(kbdev->dev, AUTO_SUSPEND_DELAY); ++ pm_runtime_use_autosuspend(kbdev->dev); ++ ++ pm_runtime_set_active(kbdev->dev); ++ pm_runtime_enable(kbdev->dev); ++ ++ if (!pm_runtime_enabled(kbdev->dev)) { ++ dev_warn(kbdev->dev, "pm_runtime not enabled"); ++ ret = -ENOSYS; ++ } ++ ++ return ret; ++} ++ ++void kbase_device_runtime_disable(struct kbase_device *kbdev) ++{ ++ dev_dbg(kbdev->dev, "kbase_device_runtime_disable\n"); ++ pm_runtime_disable(kbdev->dev); ++} ++ ++static int pm_callback_runtime_on(struct kbase_device *kbdev) ++{ ++ dev_dbg(kbdev->dev, "pm_callback_runtime_on\n"); ++ ++ return 0; ++} ++ ++static void pm_callback_runtime_off(struct kbase_device *kbdev) ++{ ++ dev_dbg(kbdev->dev, "pm_callback_runtime_off\n"); ++} ++ ++static void pm_callback_resume(struct kbase_device *kbdev) ++{ ++ int ret = pm_callback_runtime_on(kbdev); ++ ++ WARN_ON(ret); ++} ++ ++static void pm_callback_suspend(struct kbase_device *kbdev) ++{ ++ pm_callback_runtime_off(kbdev); ++} ++ ++struct kbase_pm_callback_conf pm_callbacks = { ++ .power_on_callback = pm_callback_power_on, ++ .power_off_callback = pm_callback_power_off, ++ .power_suspend_callback = pm_callback_suspend, ++ .power_resume_callback = pm_callback_resume, ++#ifdef KBASE_PM_RUNTIME ++ .power_runtime_init_callback = kbase_device_runtime_init, ++ .power_runtime_term_callback = kbase_device_runtime_disable, ++ .power_runtime_on_callback = pm_callback_runtime_on, ++ .power_runtime_off_callback = pm_callback_runtime_off, ++#else /* KBASE_PM_RUNTIME */ ++ .power_runtime_init_callback = NULL, ++ .power_runtime_term_callback = NULL, ++ .power_runtime_on_callback = NULL, ++ .power_runtime_off_callback = NULL, ++#endif /* KBASE_PM_RUNTIME */ ++}; ++ ++ +diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/mali_kbase_platform_common.h b/drivers/gpu/arm/bifrost_for_linux/platform/mali_kbase_platform_common.h +new file mode 100755 +index 000000000000..7cb3be7f78ce +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/platform/mali_kbase_platform_common.h +@@ -0,0 +1,26 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2013 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/** ++ * @brief Entry point to transfer control to a platform for early initialization ++ * ++ * This function is called early on in the initialization during execution of ++ * @ref kbase_driver_init. ++ * ++ * @return Zero to indicate success non-zero for failure. ++ */ ++int kbase_platform_early_init(void); +diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/rk/Kbuild b/drivers/gpu/arm/bifrost_for_linux/platform/rk/Kbuild +new file mode 100755 +index 000000000000..7cc6c59d969f +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/platform/rk/Kbuild +@@ -0,0 +1,17 @@ ++# ++# (C) COPYRIGHT 2012-2013 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++bifrost_kbase-y += \ ++ $(MALI_PLATFORM_DIR)/mali_kbase_config_rk.o \ ++ +diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/rk/custom_log.h b/drivers/gpu/arm/bifrost_for_linux/platform/rk/custom_log.h +new file mode 100755 +index 000000000000..5de70ee13d25 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/platform/rk/custom_log.h +@@ -0,0 +1,192 @@ ++/* ++ * (C) COPYRIGHT RockChip Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ */ ++ ++#ifndef __CUSTOM_LOG_H__ ++#define __CUSTOM_LOG_H__ ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++/* ----------------------------------------------------------------------------- ++ * Include Files ++ * ----------------------------------------------------------------------------- ++ */ ++#include ++#include ++ ++/* ----------------------------------------------------------------------------- ++ * Macros Definition ++ * ----------------------------------------------------------------------------- ++ */ ++ ++/** 若下列 macro 有被定义, æ‰ ä½¿èƒ½ log 输出. */ ++/* #define ENABLE_DEBUG_LOG */ ++ ++/*----------------------------------------------------------------------------*/ ++ ++#ifdef ENABLE_VERBOSE_LOG ++/** Verbose log. */ ++#define V(fmt, args...) \ ++ pr_debug("V : [File] : %s; [Line] : %d; [Func] : %s(); " fmt \ ++ "\n", \ ++ __FILE__, \ ++ __LINE__, \ ++ __func__, \ ++ ## args) ++#else ++#define V(...) ((void)0) ++#endif ++ ++#ifdef ENABLE_DEBUG_LOG ++/** Debug log. */ ++#define D(fmt, args...) \ ++ pr_info("D : [File] : %s; [Line] : %d; [Func] : %s(); " fmt \ ++ "\n", \ ++ __FILE__, \ ++ __LINE__, \ ++ __func__, \ ++ ## args) ++#else ++#define D(...) ((void)0) ++#endif ++ ++#define I(fmt, args...) \ ++ pr_info("I : [File] : %s; [Line] : %d; [Func] : %s(); " fmt \ ++ "\n", \ ++ __FILE__, \ ++ __LINE__, \ ++ __func__, \ ++ ## args) ++ ++#define W(fmt, args...) \ ++ pr_warn("W : [File] : %s; [Line] : %d; [Func] : %s(); " \ ++ fmt "\n", \ ++ __FILE__, \ ++ __LINE__, \ ++ __func__, \ ++ ## args) ++ ++#define E(fmt, args...) \ ++ pr_err("E : [File] : %s; [Line] : %d; [Func] : %s(); " fmt \ ++ "\n", \ ++ __FILE__, \ ++ __LINE__, \ ++ __func__, \ ++ ## args) ++ ++/*-------------------------------------------------------*/ ++ ++/** 使用 D(), 以å进制的形å¼æ‰“å°å˜é‡ 'var' çš„ value. */ ++#define D_DEC(var) D(#var " = %d.", var) ++ ++#define E_DEC(var) E(#var " = %d.", var) ++ ++/** 使用 D(), 以å六进制的形å¼æ‰“å°å˜é‡ 'var' çš„ value. */ ++#define D_HEX(var) D(#var " = 0x%x.", var) ++ ++#define E_HEX(var) E(#var " = 0x%x.", var) ++ ++/** ++ * 使用 D(), 以å六进制的形å¼, ++ * 打å°æŒ‡é’ˆç±»åž‹å˜é‡ 'ptr' çš„ value. ++ */ ++#define D_PTR(ptr) D(#ptr " = %p.", ptr) ++ ++#define E_PTR(ptr) E(#ptr " = %p.", ptr) ++ ++/** 使用 D(), æ‰“å° char 字串. */ ++#define D_STR(p_str) \ ++do { \ ++ if (!p_str) { \ ++ D(#p_str " = NULL."); \ ++ else \ ++ D(#p_str " = '%s'.", p_str); \ ++} while (0) ++ ++#define E_STR(p_str) \ ++do { \ ++ if (!p_str) \ ++ E(#p_str " = NULL."); \ ++ else \ ++ E(#p_str " = '%s'.", p_str); \ ++} while (0) ++ ++#ifdef ENABLE_DEBUG_LOG ++/** ++ * log 从 'p_start' 地å€å¼€å§‹çš„ 'len' 个字节的数æ®. ++ */ ++#define D_MEM(p_start, len) \ ++do { \ ++ int i = 0; \ ++ char *p = (char *)(p_start); \ ++ D("dump memory from addr of '" #p_start "', from %p, length %d' : ", \ ++ (p_start), \ ++ (len)); \ ++ pr_debug("\t\t"); \ ++ for (i = 0; i < (len); i++) \ ++ pr_debug("0x%02x, ", p[i]); \ ++ pr_debug("\n"); \ ++} while (0) ++#else ++#define D_MEM(...) ((void)0) ++#endif ++ ++/*-------------------------------------------------------*/ ++ ++/** ++ * 在特定æ¡ä»¶ä¸‹, 判定 error å‘生, ++ * å°†å˜é‡ 'ret_var' 设置 'err_code', ++ * log 输出对应的 Error Caution, ++ * 然åŽè·³è½¬ 'label' 指定的代ç å¤„执行. ++ * @param msg ++ * 纯字串形å¼çš„æ示信æ¯. ++ * @param ret_var ++ * 标识函数执行状æ€æˆ–者结果的å˜é‡, ++ * 将被设置具体的 Error Code. ++ * 通常是 'ret' or 'result'. ++ * @param err_code ++ * 表å¾ç‰¹å®š error 的常数标识, ++ * 通常是 å®çš„å½¢æ€. ++ * @param label ++ * 程åºå°†è¦è·³è½¬åˆ°çš„错误处ç†ä»£ç çš„æ ‡å·, ++ * 通常就是 'EXIT'. ++ * @param args... ++ * 对应 'msg_fmt' 实å‚中, ++ * '%s', '%d', ... 等转æ¢è¯´æ˜Žç¬¦çš„具体å¯å˜é•¿å®žå‚. ++ */ ++#define SET_ERROR_AND_JUMP(msg_fmt, ret_var, err_code, label, args...) \ ++do { \ ++ E("To set '" #ret_var "' to %d('" #err_code "'), because : " msg_fmt, \ ++ (err_code), \ ++ ## args); \ ++ (ret_var) = (err_code); \ ++ goto label; \ ++} while (0) ++ ++/* ----------------------------------------------------------------------------- ++ * Types and Structures Definition ++ * ----------------------------------------------------------------------------- ++ */ ++ ++/* ----------------------------------------------------------------------------- ++ * Global Functions' Prototype ++ * ----------------------------------------------------------------------------- ++ */ ++ ++/* ----------------------------------------------------------------------------- ++ * Inline Functions Implementation ++ * ----------------------------------------------------------------------------- ++ */ ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif /* __CUSTOM_LOG_H__ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/rk/mali_kbase_config_platform.h b/drivers/gpu/arm/bifrost_for_linux/platform/rk/mali_kbase_config_platform.h +new file mode 100755 +index 000000000000..07c5b6f8a760 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/platform/rk/mali_kbase_config_platform.h +@@ -0,0 +1,88 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ */ ++ ++/** ++ * @file mali_kbase_config_platform.h ++ * 声明 platform_config_of_rk (platform_rk çš„ platform_config). ++ */ ++ ++/** ++ * Maximum frequency GPU will be clocked at. ++ * Given in kHz. ++ * This must be specified as there is no default value. ++ * ++ * Attached value: number in kHz ++ * Default value: NA ++ */ ++#define GPU_FREQ_KHZ_MAX (5000) ++ ++/** ++ * Minimum frequency GPU will be clocked at. ++ * Given in kHz. ++ * This must be specified as there is no default value. ++ * ++ * Attached value: number in kHz ++ * Default value: NA ++ */ ++#define GPU_FREQ_KHZ_MIN (5000) ++ ++/** ++ * CPU_SPEED_FUNC ++ * - A pointer to a function that calculates the CPU clock ++ * ++ * CPU clock speed of the platform is in MHz ++ * - see kbase_cpu_clk_speed_func for the function prototype. ++ * ++ * Attached value: A kbase_cpu_clk_speed_func. ++ * Default Value: NA ++ */ ++#define CPU_SPEED_FUNC (NULL) ++ ++/** ++ * GPU_SPEED_FUNC ++ * - A pointer to a function that calculates the GPU clock ++ * ++ * GPU clock speed of the platform in MHz ++ * - see kbase_gpu_clk_speed_func for the function prototype. ++ * ++ * Attached value: A kbase_gpu_clk_speed_func. ++ * Default Value: NA ++ */ ++#define GPU_SPEED_FUNC (NULL) ++ ++/** ++ * Power management configuration ++ * ++ * Attached value: ++ * pointer to @ref kbase_pm_callback_conf ++ * Default value: ++ * See @ref kbase_pm_callback_conf ++ */ ++#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks) ++extern struct kbase_pm_callback_conf pm_callbacks; ++ ++/** ++ * Platform specific configuration functions ++ * ++ * Attached value: ++ * pointer to @ref kbase_platform_funcs_conf ++ * Default value: ++ * See @ref kbase_platform_funcs_conf ++ */ ++#define PLATFORM_FUNCS (&platform_funcs) ++extern struct kbase_platform_funcs_conf platform_funcs; ++ ++/** ++ * Secure mode switch ++ * ++ * Attached value: pointer to @ref kbase_secure_ops ++ */ ++#define SECURE_CALLBACKS (NULL) ++ +diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/rk/mali_kbase_config_rk.c b/drivers/gpu/arm/bifrost_for_linux/platform/rk/mali_kbase_config_rk.c +new file mode 100755 +index 000000000000..926c2dd3f8c9 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/platform/rk/mali_kbase_config_rk.c +@@ -0,0 +1,459 @@ ++/* ++ * (C) COPYRIGHT RockChip Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ */ ++ ++/* #define ENABLE_DEBUG_LOG */ ++#include "custom_log.h" ++ ++#include ++#include ++#include ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "mali_kbase_rk.h" ++ ++#define MAX_PROP_NAME_LEN 3 ++#define LEAKAGE_TABLE_END ~1 ++#define LEAKAGE_INVALID 0xff ++ ++struct pvtm_config { ++ unsigned int freq; ++ unsigned int volt; ++ unsigned int ch[2]; ++ unsigned int sample_time; ++ unsigned int num; ++ unsigned int err; ++ unsigned int ref_temp; ++ int temp_prop[2]; ++ const char *tz_name; ++ struct thermal_zone_device *tz; ++}; ++ ++struct volt_sel_table { ++ int min; ++ int max; ++ int sel; ++}; ++ ++/** ++ * @file mali_kbase_config_rk.c ++ * 对 platform_config_of_rk 的具体实现. ++ * ++ * mali_device_driver 包å«ä¸¤éƒ¨åˆ† : ++ * .DP : platform_dependent_part_in_mdd : ++ * ä¾èµ– platform 部分, ++ * æºç åœ¨ /platform// ++ * 在 mali_device_driver 内部, ++ * 记为 platform_dependent_part, ++ * 也被记为 platform_specific_code. ++ * .DP : common_parts_in_mdd : ++ * arm 实现的通用的部分, ++ * æºç åœ¨ / 下. ++ * 在 mali_device_driver 内部, 记为 common_parts. ++ */ ++ ++/*---------------------------------------------------------------------------*/ ++ ++#ifdef CONFIG_REGULATOR ++static int rk_pm_enable_regulator(struct kbase_device *kbdev); ++static void rk_pm_disable_regulator(struct kbase_device *kbdev); ++#else ++static inline int rk_pm_enable_regulator(struct kbase_device *kbdev) ++{ ++ return 0; ++} ++ ++static inline void rk_pm_disable_regulator(struct kbase_device *kbdev) ++{ ++} ++#endif ++ ++static int rk_pm_enable_clk(struct kbase_device *kbdev); ++ ++static void rk_pm_disable_clk(struct kbase_device *kbdev); ++ ++static int kbase_platform_rk_create_sysfs_files(struct device *dev); ++ ++static void kbase_platform_rk_remove_sysfs_files(struct device *dev); ++ ++/*---------------------------------------------------------------------------*/ ++ ++static void rk_pm_power_off_delay_work(struct work_struct *work) ++{ ++ struct rk_context *platform = ++ container_of(to_delayed_work(work), struct rk_context, work); ++ struct kbase_device *kbdev = platform->kbdev; ++ ++ if (!platform->is_powered) { ++ D("mali_dev is already powered off."); ++ return; ++ } ++ ++ if (pm_runtime_enabled(kbdev->dev)) { ++ D("to put_sync_suspend mali_dev."); ++ pm_runtime_put_sync_suspend(kbdev->dev); ++ } ++ ++ rk_pm_disable_regulator(kbdev); ++ ++ platform->is_powered = false; ++ KBASE_TIMELINE_GPU_POWER(kbdev, 0); ++ wake_unlock(&platform->wake_lock); ++} ++ ++static int kbase_platform_rk_init(struct kbase_device *kbdev) ++{ ++ int ret = 0; ++ struct rk_context *platform; ++ ++ platform = kzalloc(sizeof(*platform), GFP_KERNEL); ++ if (!platform) { ++ E("err."); ++ return -ENOMEM; ++ } ++ ++ platform->is_powered = false; ++ platform->kbdev = kbdev; ++ ++ platform->delay_ms = 200; ++ if (of_property_read_u32(kbdev->dev->of_node, "power-off-delay-ms", ++ &platform->delay_ms)) ++ W("power-off-delay-ms not available."); ++ ++ platform->power_off_wq = create_freezable_workqueue("gpu_power_off_wq"); ++ if (!platform->power_off_wq) { ++ E("couldn't create workqueue"); ++ ret = -ENOMEM; ++ goto err_wq; ++ } ++ INIT_DEFERRABLE_WORK(&platform->work, rk_pm_power_off_delay_work); ++ ++ wake_lock_init(&platform->wake_lock, WAKE_LOCK_SUSPEND, "gpu"); ++ ++ platform->utilisation_period = DEFAULT_UTILISATION_PERIOD_IN_MS; ++ ++ ret = kbase_platform_rk_create_sysfs_files(kbdev->dev); ++ if (ret) { ++ E("fail to create sysfs_files. ret = %d.", ret); ++ goto err_sysfs_files; ++ } ++ ++ kbdev->platform_context = (void *)platform; ++ pm_runtime_enable(kbdev->dev); ++ ++ return 0; ++ ++err_sysfs_files: ++ wake_lock_destroy(&platform->wake_lock); ++ destroy_workqueue(platform->power_off_wq); ++err_wq: ++ return ret; ++} ++ ++static void kbase_platform_rk_term(struct kbase_device *kbdev) ++{ ++ struct rk_context *platform = ++ (struct rk_context *)kbdev->platform_context; ++ ++ pm_runtime_disable(kbdev->dev); ++ kbdev->platform_context = NULL; ++ ++ if (platform) { ++ cancel_delayed_work_sync(&platform->work); ++ wake_lock_destroy(&platform->wake_lock); ++ destroy_workqueue(platform->power_off_wq); ++ platform->is_powered = false; ++ platform->kbdev = NULL; ++ kfree(platform); ++ } ++ kbase_platform_rk_remove_sysfs_files(kbdev->dev); ++} ++ ++struct kbase_platform_funcs_conf platform_funcs = { ++ .platform_init_func = &kbase_platform_rk_init, ++ .platform_term_func = &kbase_platform_rk_term, ++}; ++ ++/*---------------------------------------------------------------------------*/ ++ ++static int rk_pm_callback_runtime_on(struct kbase_device *kbdev) ++{ ++ return 0; ++} ++ ++static void rk_pm_callback_runtime_off(struct kbase_device *kbdev) ++{ ++} ++ ++static int rk_pm_callback_power_on(struct kbase_device *kbdev) ++{ ++ int ret = 1; /* Assume GPU has been powered off */ ++ int err = 0; ++ struct rk_context *platform = get_rk_context(kbdev); ++ ++ cancel_delayed_work_sync(&platform->work); ++ ++ err = rk_pm_enable_clk(kbdev); ++ if (err) { ++ E("failed to enable clk: %d", err); ++ return err; ++ } ++ ++ if (platform->is_powered) { ++ D("mali_device is already powered."); ++ return 0; ++ } ++ ++ /* we must enable vdd_gpu before pd_gpu_in_chip. */ ++ err = rk_pm_enable_regulator(kbdev); ++ if (err) { ++ E("fail to enable regulator, err : %d.", err); ++ return err; ++ } ++ ++ /* è‹¥ mali_dev çš„ runtime_pm 是 enabled çš„, 则... */ ++ if (pm_runtime_enabled(kbdev->dev)) { ++ D("to resume mali_dev syncly."); ++ /* 对 pd_in_chip çš„ on æ“作, ++ * 将在 pm_domain çš„ runtime_pm_callbacks 中完æˆ. ++ */ ++ err = pm_runtime_get_sync(kbdev->dev); ++ if (err < 0) { ++ E("failed to runtime resume device: %d.", err); ++ return err; ++ } else if (err == 1) { /* runtime_pm_status is still active */ ++ D("chip has NOT been powered off, no need to re-init."); ++ ret = 0; ++ } ++ } ++ ++ platform->is_powered = true; ++ KBASE_TIMELINE_GPU_POWER(kbdev, 1); ++ wake_lock(&platform->wake_lock); ++ ++ return ret; ++} ++ ++static void rk_pm_callback_power_off(struct kbase_device *kbdev) ++{ ++ struct rk_context *platform = get_rk_context(kbdev); ++ ++ rk_pm_disable_clk(kbdev); ++ queue_delayed_work(platform->power_off_wq, &platform->work, ++ msecs_to_jiffies(platform->delay_ms)); ++} ++ ++int rk_kbase_device_runtime_init(struct kbase_device *kbdev) ++{ ++ return 0; ++} ++ ++void rk_kbase_device_runtime_disable(struct kbase_device *kbdev) ++{ ++} ++ ++struct kbase_pm_callback_conf pm_callbacks = { ++ .power_on_callback = rk_pm_callback_power_on, ++ .power_off_callback = rk_pm_callback_power_off, ++#ifdef CONFIG_PM ++ .power_runtime_init_callback = rk_kbase_device_runtime_init, ++ .power_runtime_term_callback = rk_kbase_device_runtime_disable, ++ .power_runtime_on_callback = rk_pm_callback_runtime_on, ++ .power_runtime_off_callback = rk_pm_callback_runtime_off, ++#else /* CONFIG_PM */ ++ .power_runtime_init_callback = NULL, ++ .power_runtime_term_callback = NULL, ++ .power_runtime_on_callback = NULL, ++ .power_runtime_off_callback = NULL, ++#endif /* CONFIG_PM */ ++}; ++ ++int kbase_platform_early_init(void) ++{ ++ /* Nothing needed at this stage */ ++ return 0; ++} ++ ++/*---------------------------------------------------------------------------*/ ++ ++void kbase_platform_rk_shutdown(struct kbase_device *kbdev) ++{ ++ I("to make vdd_gpu enabled for turning off pd_gpu in pm_framework."); ++ rk_pm_enable_regulator(kbdev); ++} ++ ++/*---------------------------------------------------------------------------*/ ++ ++#ifdef CONFIG_REGULATOR ++static int rk_pm_enable_regulator(struct kbase_device *kbdev) ++{ ++ int ret = 0; ++ ++ if (!kbdev->regulator) { ++ W("no mali regulator control, no need to enable."); ++ goto EXIT; ++ } ++ ++ D("to enable regulator."); ++ ret = regulator_enable(kbdev->regulator); ++ if (ret) { ++ E("fail to enable regulator, ret : %d.", ret); ++ goto EXIT; ++ } ++ ++EXIT: ++ return ret; ++} ++ ++static void rk_pm_disable_regulator(struct kbase_device *kbdev) ++{ ++ if (!(kbdev->regulator)) { ++ W("no mali regulator control, no need to disable."); ++ return; ++ } ++ ++ D("to disable regulator."); ++ regulator_disable(kbdev->regulator); ++} ++#endif ++ ++static int rk_pm_enable_clk(struct kbase_device *kbdev) ++{ ++ int err = 0; ++ ++ if (!(kbdev->clock)) { ++ W("no mali clock control, no need to enable."); ++ } else { ++ D("to enable clk."); ++ err = clk_enable(kbdev->clock); ++ if (err) ++ E("failed to enable clk: %d.", err); ++ } ++ ++ return err; ++} ++ ++static void rk_pm_disable_clk(struct kbase_device *kbdev) ++{ ++ if (!(kbdev->clock)) { ++ W("no mali clock control, no need to disable."); ++ } else { ++ D("to disable clk."); ++ clk_disable(kbdev->clock); ++ } ++} ++ ++/*---------------------------------------------------------------------------*/ ++ ++static ssize_t utilisation_period_show(struct device *dev, ++ struct device_attribute *attr, ++ char *buf) ++{ ++ struct kbase_device *kbdev = dev_get_drvdata(dev); ++ struct rk_context *platform = get_rk_context(kbdev); ++ ssize_t ret = 0; ++ ++ ret += snprintf(buf, PAGE_SIZE, "%u\n", platform->utilisation_period); ++ ++ return ret; ++} ++ ++static ssize_t utilisation_period_store(struct device *dev, ++ struct device_attribute *attr, ++ const char *buf, ++ size_t count) ++{ ++ struct kbase_device *kbdev = dev_get_drvdata(dev); ++ struct rk_context *platform = get_rk_context(kbdev); ++ int ret = 0; ++ ++ ret = kstrtouint(buf, 0, &platform->utilisation_period); ++ if (ret) { ++ E("invalid input period : %s.", buf); ++ return ret; ++ } ++ D("set utilisation_period to '%d'.", platform->utilisation_period); ++ ++ return count; ++} ++ ++static ssize_t utilisation_show(struct device *dev, ++ struct device_attribute *attr, ++ char *buf) ++{ ++ struct kbase_device *kbdev = dev_get_drvdata(dev); ++ struct rk_context *platform = get_rk_context(kbdev); ++ ssize_t ret = 0; ++ unsigned long period_in_us = platform->utilisation_period * 1000; ++ unsigned long total_time; ++ unsigned long busy_time; ++ unsigned long utilisation; ++ ++ kbase_pm_reset_dvfs_utilisation(kbdev); ++ usleep_range(period_in_us, period_in_us + 100); ++ kbase_pm_get_dvfs_utilisation(kbdev, &total_time, &busy_time); ++ /* 'devfreq_dev_profile' instance registered to devfreq ++ * also uses kbase_pm_reset_dvfs_utilisation ++ * and kbase_pm_get_dvfs_utilisation. ++ * it's better to cat this file when DVFS is disabled. ++ */ ++ D("total_time : %lu, busy_time : %lu.", total_time, busy_time); ++ ++ utilisation = busy_time * 100 / total_time; ++ ret += snprintf(buf, PAGE_SIZE, "%ld\n", utilisation); ++ ++ return ret; ++} ++ ++static DEVICE_ATTR_RW(utilisation_period); ++static DEVICE_ATTR_RO(utilisation); ++ ++static int kbase_platform_rk_create_sysfs_files(struct device *dev) ++{ ++ int ret = 0; ++ ++ ret = device_create_file(dev, &dev_attr_utilisation_period); ++ if (ret) { ++ E("fail to create sysfs file 'utilisation_period'."); ++ goto out; ++ } ++ ++ ret = device_create_file(dev, &dev_attr_utilisation); ++ if (ret) { ++ E("fail to create sysfs file 'utilisation'."); ++ goto remove_utilisation_period; ++ } ++ ++ return 0; ++ ++remove_utilisation_period: ++ device_remove_file(dev, &dev_attr_utilisation_period); ++out: ++ return ret; ++} ++ ++static void kbase_platform_rk_remove_sysfs_files(struct device *dev) ++{ ++ device_remove_file(dev, &dev_attr_utilisation_period); ++ device_remove_file(dev, &dev_attr_utilisation); ++} ++ ++int kbase_platform_rk_init_opp_table(struct kbase_device *kbdev) ++{ ++ return rockchip_init_opp_table(kbdev->dev, NULL, ++ "gpu_leakage", "mali"); ++} +diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/rk/mali_kbase_rk.h b/drivers/gpu/arm/bifrost_for_linux/platform/rk/mali_kbase_rk.h +new file mode 100755 +index 000000000000..6eab25014d21 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/platform/rk/mali_kbase_rk.h +@@ -0,0 +1,62 @@ ++/* drivers/gpu/t6xx/kbase/src/platform/rk/mali_kbase_platform.h ++ * Rockchip SoC Mali-Midgard platform-dependent codes ++ * ++ * This program is free software; you can redistribute it and/or modify ++ * it under the terms of the GNU General Public License version 2 as ++ * published by the Free Software FoundatIon. ++ */ ++ ++/** ++ * @file mali_kbase_rk.h ++ * ++ * defines work_context type of platform_dependent_part. ++ */ ++ ++#ifndef _MALI_KBASE_RK_H_ ++#define _MALI_KBASE_RK_H_ ++ ++#include ++ ++/*---------------------------------------------------------------------------*/ ++ ++#define DEFAULT_UTILISATION_PERIOD_IN_MS (100) ++ ++/*---------------------------------------------------------------------------*/ ++ ++/* ++ * struct rk_context - work_context of platform_dependent_part_of_rk. ++ */ ++struct rk_context { ++ /* ++ * record the status of common_parts calling 'power_on_callback' ++ * and 'power_off_callback'. ++ */ ++ bool is_powered; ++ ++ struct kbase_device *kbdev; ++ ++ struct workqueue_struct *power_off_wq; ++ /* delayed_work_to_power_off_gpu. */ ++ struct delayed_work work; ++ unsigned int delay_ms; ++ ++ /* ++ * WAKE_LOCK_SUSPEND for ensuring to run ++ * delayed_work_to_power_off_gpu before suspend. ++ */ ++ struct wake_lock wake_lock; ++ ++ /* debug only, the period in ms to count gpu_utilisation. */ ++ unsigned int utilisation_period; ++}; ++ ++/*---------------------------------------------------------------------------*/ ++ ++static inline struct rk_context *get_rk_context( ++ const struct kbase_device *kbdev) ++{ ++ return (struct rk_context *)(kbdev->platform_context); ++} ++ ++#endif /* _MALI_KBASE_RK_H_ */ ++ +diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/vexpress/Kbuild b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress/Kbuild +new file mode 100755 +index 000000000000..d9d5e9085231 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress/Kbuild +@@ -0,0 +1,19 @@ ++# ++# (C) COPYRIGHT 2012-2013, 2016-2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++mali_kbase-y += \ ++ $(MALI_PLATFORM_DIR)/mali_kbase_config_vexpress.o \ ++ $(MALI_PLATFORM_DIR)/mali_kbase_cpu_vexpress.o \ ++ mali_kbase_platform_fake.o +diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/vexpress/mali_kbase_config_platform.h b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress/mali_kbase_config_platform.h +new file mode 100755 +index 000000000000..02835f129aa3 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress/mali_kbase_config_platform.h +@@ -0,0 +1,75 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include "mali_kbase_cpu_vexpress.h" ++ ++/** ++ * Maximum frequency GPU will be clocked at. Given in kHz. ++ * This must be specified as there is no default value. ++ * ++ * Attached value: number in kHz ++ * Default value: NA ++ */ ++#define GPU_FREQ_KHZ_MAX kbase_get_platform_max_freq() ++/** ++ * Minimum frequency GPU will be clocked at. Given in kHz. ++ * This must be specified as there is no default value. ++ * ++ * Attached value: number in kHz ++ * Default value: NA ++ */ ++#define GPU_FREQ_KHZ_MIN kbase_get_platform_min_freq() ++ ++/** ++ * CPU_SPEED_FUNC - A pointer to a function that calculates the CPU clock ++ * ++ * CPU clock speed of the platform is in MHz - see kbase_cpu_clk_speed_func ++ * for the function prototype. ++ * ++ * Attached value: A kbase_cpu_clk_speed_func. ++ * Default Value: NA ++ */ ++#define CPU_SPEED_FUNC (&kbase_get_vexpress_cpu_clock_speed) ++ ++/** ++ * GPU_SPEED_FUNC - A pointer to a function that calculates the GPU clock ++ * ++ * GPU clock speed of the platform in MHz - see kbase_gpu_clk_speed_func ++ * for the function prototype. ++ * ++ * Attached value: A kbase_gpu_clk_speed_func. ++ * Default Value: NA ++ */ ++#define GPU_SPEED_FUNC (NULL) ++ ++/** ++ * Power management configuration ++ * ++ * Attached value: pointer to @ref kbase_pm_callback_conf ++ * Default value: See @ref kbase_pm_callback_conf ++ */ ++#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks) ++ ++/** ++ * Platform specific configuration functions ++ * ++ * Attached value: pointer to @ref kbase_platform_funcs_conf ++ * Default value: See @ref kbase_platform_funcs_conf ++ */ ++#define PLATFORM_FUNCS (NULL) ++ ++extern struct kbase_pm_callback_conf pm_callbacks; +diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/vexpress/mali_kbase_config_vexpress.c b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress/mali_kbase_config_vexpress.c +new file mode 100755 +index 000000000000..15ce2bc5eea5 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress/mali_kbase_config_vexpress.c +@@ -0,0 +1,85 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#include ++#include ++#include ++#include ++#include "mali_kbase_cpu_vexpress.h" ++#include "mali_kbase_config_platform.h" ++ ++#define HARD_RESET_AT_POWER_OFF 0 ++ ++#ifndef CONFIG_OF ++static struct kbase_io_resources io_resources = { ++ .job_irq_number = 68, ++ .mmu_irq_number = 69, ++ .gpu_irq_number = 70, ++ .io_memory_region = { ++ .start = 0xFC010000, ++ .end = 0xFC010000 + (4096 * 4) - 1 ++ } ++}; ++#endif /* CONFIG_OF */ ++ ++static int pm_callback_power_on(struct kbase_device *kbdev) ++{ ++ /* Nothing is needed on VExpress, but we may have destroyed GPU state (if the below HARD_RESET code is active) */ ++ return 1; ++} ++ ++static void pm_callback_power_off(struct kbase_device *kbdev) ++{ ++#if HARD_RESET_AT_POWER_OFF ++ /* Cause a GPU hard reset to test whether we have actually idled the GPU ++ * and that we properly reconfigure the GPU on power up. ++ * Usually this would be dangerous, but if the GPU is working correctly it should ++ * be completely safe as the GPU should not be active at this point. ++ * However this is disabled normally because it will most likely interfere with ++ * bus logging etc. ++ */ ++ KBASE_TRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, NULL, 0u, 0); ++ kbase_os_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_HARD_RESET); ++#endif ++} ++ ++struct kbase_pm_callback_conf pm_callbacks = { ++ .power_on_callback = pm_callback_power_on, ++ .power_off_callback = pm_callback_power_off, ++ .power_suspend_callback = NULL, ++ .power_resume_callback = NULL ++}; ++ ++static struct kbase_platform_config versatile_platform_config = { ++#ifndef CONFIG_OF ++ .io_resources = &io_resources ++#endif ++}; ++ ++struct kbase_platform_config *kbase_get_platform_config(void) ++{ ++ return &versatile_platform_config; ++} ++ ++ ++int kbase_platform_early_init(void) ++{ ++ /* Nothing needed at this stage */ ++ return 0; ++} +diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/vexpress/mali_kbase_cpu_vexpress.c b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress/mali_kbase_cpu_vexpress.c +new file mode 100755 +index 000000000000..4665f98cbbe4 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress/mali_kbase_cpu_vexpress.c +@@ -0,0 +1,279 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include ++#include ++#include "mali_kbase_cpu_vexpress.h" ++ ++#define HZ_IN_MHZ (1000000) ++ ++#define CORETILE_EXPRESS_A9X4_SCC_START (0x100E2000) ++#define MOTHERBOARD_SYS_CFG_START (0x10000000) ++#define SYS_CFGDATA_OFFSET (0x000000A0) ++#define SYS_CFGCTRL_OFFSET (0x000000A4) ++#define SYS_CFGSTAT_OFFSET (0x000000A8) ++ ++#define SYS_CFGCTRL_START_BIT_VALUE (1 << 31) ++#define READ_REG_BIT_VALUE (0 << 30) ++#define DCC_DEFAULT_BIT_VALUE (0 << 26) ++#define SYS_CFG_OSC_FUNC_BIT_VALUE (1 << 20) ++#define SITE_DEFAULT_BIT_VALUE (1 << 16) ++#define BOARD_STACK_POS_DEFAULT_BIT_VALUE (0 << 12) ++#define DEVICE_DEFAULT_BIT_VALUE (2 << 0) ++#define SYS_CFG_COMPLETE_BIT_VALUE (1 << 0) ++#define SYS_CFG_ERROR_BIT_VALUE (1 << 1) ++ ++#define FEED_REG_BIT_MASK (0x0F) ++#define FCLK_PA_DIVIDE_BIT_SHIFT (0x03) ++#define FCLK_PB_DIVIDE_BIT_SHIFT (0x07) ++#define FCLK_PC_DIVIDE_BIT_SHIFT (0x0B) ++#define AXICLK_PA_DIVIDE_BIT_SHIFT (0x0F) ++#define AXICLK_PB_DIVIDE_BIT_SHIFT (0x13) ++ ++/* the following three values used for reading ++ * HBI value of the LogicTile daughterboard */ ++#define VE_MOTHERBOARD_PERIPHERALS_SMB_CS7 (0x10000000) ++#define VE_SYS_PROC_ID1_OFFSET (0x00000088) ++#define VE_LOGIC_TILE_HBI_MASK (0x00000FFF) ++ ++#define IS_SINGLE_BIT_SET(val, pos) (val&(1<> ++ FCLK_PA_DIVIDE_BIT_SHIFT); ++ /* CFGRW0[10:7] */ ++ pb_divide = ((reg_val & (FEED_REG_BIT_MASK << ++ FCLK_PB_DIVIDE_BIT_SHIFT)) >> ++ FCLK_PB_DIVIDE_BIT_SHIFT); ++ *cpu_clock = osc2_value * (pa_divide + 1) / (pb_divide + 1); ++ } else if (IS_SINGLE_BIT_SET(reg_val, 1)) { ++ /* CFGRW0[1] - CLKOC */ ++ /* CFGRW0[6:3] */ ++ pa_divide = ((reg_val & (FEED_REG_BIT_MASK << ++ FCLK_PA_DIVIDE_BIT_SHIFT)) >> ++ FCLK_PA_DIVIDE_BIT_SHIFT); ++ /* CFGRW0[14:11] */ ++ pc_divide = ((reg_val & (FEED_REG_BIT_MASK << ++ FCLK_PC_DIVIDE_BIT_SHIFT)) >> ++ FCLK_PC_DIVIDE_BIT_SHIFT); ++ *cpu_clock = osc2_value * (pa_divide + 1) / (pc_divide + 1); ++ } else if (IS_SINGLE_BIT_SET(reg_val, 2)) { ++ /* CFGRW0[2] - FACLK */ ++ /* CFGRW0[18:15] */ ++ pa_divide = ((reg_val & (FEED_REG_BIT_MASK << ++ AXICLK_PA_DIVIDE_BIT_SHIFT)) >> ++ AXICLK_PA_DIVIDE_BIT_SHIFT); ++ /* CFGRW0[22:19] */ ++ pb_divide = ((reg_val & (FEED_REG_BIT_MASK << ++ AXICLK_PB_DIVIDE_BIT_SHIFT)) >> ++ AXICLK_PB_DIVIDE_BIT_SHIFT); ++ *cpu_clock = osc2_value * (pa_divide + 1) / (pb_divide + 1); ++ } else { ++ err = -EIO; ++ } ++ ++set_reg_error: ++ongoing_request: ++ raw_spin_unlock(&syscfg_lock); ++ *cpu_clock /= HZ_IN_MHZ; ++ ++ if (!err) ++ cpu_clock_speed = *cpu_clock; ++ ++ iounmap(scc_reg); ++ ++scc_reg_map_failed: ++ iounmap(syscfg_reg); ++ ++syscfg_reg_map_failed: ++ ++ return err; ++} ++ ++/** ++ * kbase_get_platform_logic_tile_type - determines which LogicTile type ++ * is used by Versatile Express ++ * ++ * When platform_config build parameter is specified as vexpress, i.e., ++ * platform_config=vexpress, GPU frequency may vary dependent on the ++ * particular platform. The GPU frequency depends on the LogicTile type. ++ * ++ * This function determines which LogicTile type is used by the platform by ++ * reading the HBI value of the daughterboard which holds the LogicTile: ++ * ++ * 0x217 HBI0217 Virtex-6 ++ * 0x192 HBI0192 Virtex-5 ++ * 0x247 HBI0247 Virtex-7 ++ * ++ * Return: HBI value of the logic tile daughterboard, zero if not accessible ++ */ ++static u32 kbase_get_platform_logic_tile_type(void) ++{ ++ void __iomem *syscfg_reg = NULL; ++ u32 sys_procid1 = 0; ++ ++ syscfg_reg = ioremap(VE_MOTHERBOARD_PERIPHERALS_SMB_CS7 + VE_SYS_PROC_ID1_OFFSET, 4); ++ if (NULL != syscfg_reg) { ++ sys_procid1 = readl(syscfg_reg); ++ iounmap(syscfg_reg); ++ } ++ ++ return sys_procid1 & VE_LOGIC_TILE_HBI_MASK; ++} ++ ++u32 kbase_get_platform_min_freq(void) ++{ ++ u32 ve_logic_tile = kbase_get_platform_logic_tile_type(); ++ ++ switch (ve_logic_tile) { ++ case 0x217: ++ /* Virtex 6, HBI0217 */ ++ return VE_VIRTEX6_GPU_FREQ_MIN; ++ case 0x247: ++ /* Virtex 7, HBI0247 */ ++ return VE_VIRTEX7_GPU_FREQ_MIN; ++ default: ++ /* all other logic tiles, i.e., Virtex 5 HBI0192 ++ * or unsuccessful reading from the platform - ++ * fall back to some default value */ ++ return VE_DEFAULT_GPU_FREQ_MIN; ++ } ++} ++ ++u32 kbase_get_platform_max_freq(void) ++{ ++ u32 ve_logic_tile = kbase_get_platform_logic_tile_type(); ++ ++ switch (ve_logic_tile) { ++ case 0x217: ++ /* Virtex 6, HBI0217 */ ++ return VE_VIRTEX6_GPU_FREQ_MAX; ++ case 0x247: ++ /* Virtex 7, HBI0247 */ ++ return VE_VIRTEX7_GPU_FREQ_MAX; ++ default: ++ /* all other logic tiles, i.e., Virtex 5 HBI0192 ++ * or unsuccessful reading from the platform - ++ * fall back to some default value */ ++ return VE_DEFAULT_GPU_FREQ_MAX; ++ } ++} +diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/vexpress/mali_kbase_cpu_vexpress.h b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress/mali_kbase_cpu_vexpress.h +new file mode 100755 +index 000000000000..da865698133a +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress/mali_kbase_cpu_vexpress.h +@@ -0,0 +1,38 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2013, 2015-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#ifndef _KBASE_CPU_VEXPRESS_H_ ++#define _KBASE_CPU_VEXPRESS_H_ ++ ++/** ++ * Versatile Express implementation of @ref kbase_cpu_clk_speed_func. ++ */ ++int kbase_get_vexpress_cpu_clock_speed(u32 *cpu_clock); ++ ++/** ++ * Get the minimum GPU frequency for the attached logic tile ++ */ ++u32 kbase_get_platform_min_freq(void); ++ ++/** ++ * Get the maximum GPU frequency for the attached logic tile ++ */ ++u32 kbase_get_platform_max_freq(void); ++ ++#endif /* _KBASE_CPU_VEXPRESS_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/vexpress_1xv7_a57/Kbuild b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress_1xv7_a57/Kbuild +new file mode 100755 +index 000000000000..df87c74f43ba +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress_1xv7_a57/Kbuild +@@ -0,0 +1,18 @@ ++# ++# (C) COPYRIGHT 2013-2014, 2016-2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++mali_kbase-y += \ ++ $(MALI_PLATFORM_DIR)/mali_kbase_config_vexpress.o \ ++ mali_kbase_platform_fake.o +diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/vexpress_1xv7_a57/mali_kbase_config_platform.h b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress_1xv7_a57/mali_kbase_config_platform.h +new file mode 100755 +index 000000000000..0efbf3962f98 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress_1xv7_a57/mali_kbase_config_platform.h +@@ -0,0 +1,73 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/** ++ * Maximum frequency GPU will be clocked at. Given in kHz. ++ * This must be specified as there is no default value. ++ * ++ * Attached value: number in kHz ++ * Default value: NA ++ */ ++#define GPU_FREQ_KHZ_MAX 5000 ++/** ++ * Minimum frequency GPU will be clocked at. Given in kHz. ++ * This must be specified as there is no default value. ++ * ++ * Attached value: number in kHz ++ * Default value: NA ++ */ ++#define GPU_FREQ_KHZ_MIN 5000 ++ ++/** ++ * CPU_SPEED_FUNC - A pointer to a function that calculates the CPU clock ++ * ++ * CPU clock speed of the platform is in MHz - see kbase_cpu_clk_speed_func ++ * for the function prototype. ++ * ++ * Attached value: A kbase_cpu_clk_speed_func. ++ * Default Value: NA ++ */ ++#define CPU_SPEED_FUNC (&kbase_cpuprops_get_default_clock_speed) ++ ++/** ++ * GPU_SPEED_FUNC - A pointer to a function that calculates the GPU clock ++ * ++ * GPU clock speed of the platform in MHz - see kbase_gpu_clk_speed_func ++ * for the function prototype. ++ * ++ * Attached value: A kbase_gpu_clk_speed_func. ++ * Default Value: NA ++ */ ++#define GPU_SPEED_FUNC (NULL) ++ ++/** ++ * Power management configuration ++ * ++ * Attached value: pointer to @ref kbase_pm_callback_conf ++ * Default value: See @ref kbase_pm_callback_conf ++ */ ++#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks) ++ ++/** ++ * Platform specific configuration functions ++ * ++ * Attached value: pointer to @ref kbase_platform_funcs_conf ++ * Default value: See @ref kbase_platform_funcs_conf ++ */ ++#define PLATFORM_FUNCS (NULL) ++ ++extern struct kbase_pm_callback_conf pm_callbacks; +diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c +new file mode 100755 +index 000000000000..3ff0930fb4a3 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c +@@ -0,0 +1,79 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2014 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include ++#include ++#include ++#include ++ ++#define HARD_RESET_AT_POWER_OFF 0 ++ ++#ifndef CONFIG_OF ++static struct kbase_io_resources io_resources = { ++ .job_irq_number = 68, ++ .mmu_irq_number = 69, ++ .gpu_irq_number = 70, ++ .io_memory_region = { ++ .start = 0x2f010000, ++ .end = 0x2f010000 + (4096 * 4) - 1} ++}; ++#endif ++ ++static int pm_callback_power_on(struct kbase_device *kbdev) ++{ ++ /* Nothing is needed on VExpress, but we may have destroyed GPU state (if the below HARD_RESET code is active) */ ++ return 1; ++} ++ ++static void pm_callback_power_off(struct kbase_device *kbdev) ++{ ++#if HARD_RESET_AT_POWER_OFF ++ /* Cause a GPU hard reset to test whether we have actually idled the GPU ++ * and that we properly reconfigure the GPU on power up. ++ * Usually this would be dangerous, but if the GPU is working correctly it should ++ * be completely safe as the GPU should not be active at this point. ++ * However this is disabled normally because it will most likely interfere with ++ * bus logging etc. ++ */ ++ KBASE_TRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, NULL, 0u, 0); ++ kbase_os_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_HARD_RESET); ++#endif ++} ++ ++struct kbase_pm_callback_conf pm_callbacks = { ++ .power_on_callback = pm_callback_power_on, ++ .power_off_callback = pm_callback_power_off, ++ .power_suspend_callback = NULL, ++ .power_resume_callback = NULL ++}; ++ ++static struct kbase_platform_config versatile_platform_config = { ++#ifndef CONFIG_OF ++ .io_resources = &io_resources ++#endif ++}; ++ ++struct kbase_platform_config *kbase_get_platform_config(void) ++{ ++ return &versatile_platform_config; ++} ++ ++int kbase_platform_early_init(void) ++{ ++ /* Nothing needed at this stage */ ++ return 0; ++} +diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/vexpress_6xvirtex7_10mhz/Kbuild b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress_6xvirtex7_10mhz/Kbuild +new file mode 100755 +index 000000000000..d9d5e9085231 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress_6xvirtex7_10mhz/Kbuild +@@ -0,0 +1,19 @@ ++# ++# (C) COPYRIGHT 2012-2013, 2016-2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++mali_kbase-y += \ ++ $(MALI_PLATFORM_DIR)/mali_kbase_config_vexpress.o \ ++ $(MALI_PLATFORM_DIR)/mali_kbase_cpu_vexpress.o \ ++ mali_kbase_platform_fake.o +diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_platform.h b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_platform.h +new file mode 100755 +index 000000000000..dbdf21e009f9 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_platform.h +@@ -0,0 +1,75 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include "mali_kbase_cpu_vexpress.h" ++ ++/** ++ * Maximum frequency GPU will be clocked at. Given in kHz. ++ * This must be specified as there is no default value. ++ * ++ * Attached value: number in kHz ++ * Default value: NA ++ */ ++#define GPU_FREQ_KHZ_MAX 10000 ++/** ++ * Minimum frequency GPU will be clocked at. Given in kHz. ++ * This must be specified as there is no default value. ++ * ++ * Attached value: number in kHz ++ * Default value: NA ++ */ ++#define GPU_FREQ_KHZ_MIN 10000 ++ ++/** ++ * CPU_SPEED_FUNC - A pointer to a function that calculates the CPU clock ++ * ++ * CPU clock speed of the platform is in MHz - see kbase_cpu_clk_speed_func ++ * for the function prototype. ++ * ++ * Attached value: A kbase_cpu_clk_speed_func. ++ * Default Value: NA ++ */ ++#define CPU_SPEED_FUNC (&kbase_get_vexpress_cpu_clock_speed) ++ ++/** ++ * GPU_SPEED_FUNC - A pointer to a function that calculates the GPU clock ++ * ++ * GPU clock speed of the platform in MHz - see kbase_gpu_clk_speed_func ++ * for the function prototype. ++ * ++ * Attached value: A kbase_gpu_clk_speed_func. ++ * Default Value: NA ++ */ ++#define GPU_SPEED_FUNC (NULL) ++ ++/** ++ * Power management configuration ++ * ++ * Attached value: pointer to @ref kbase_pm_callback_conf ++ * Default value: See @ref kbase_pm_callback_conf ++ */ ++#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks) ++ ++/** ++ * Platform specific configuration functions ++ * ++ * Attached value: pointer to @ref kbase_platform_funcs_conf ++ * Default value: See @ref kbase_platform_funcs_conf ++ */ ++#define PLATFORM_FUNCS (NULL) ++ ++extern struct kbase_pm_callback_conf pm_callbacks; +diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_vexpress.c b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_vexpress.c +new file mode 100755 +index 000000000000..76ffe4a1e59e +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_vexpress.c +@@ -0,0 +1,83 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2014 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#include ++#include ++#include ++#include ++#include "mali_kbase_cpu_vexpress.h" ++ ++#define HARD_RESET_AT_POWER_OFF 0 ++ ++#ifndef CONFIG_OF ++static struct kbase_io_resources io_resources = { ++ .job_irq_number = 75, ++ .mmu_irq_number = 76, ++ .gpu_irq_number = 77, ++ .io_memory_region = { ++ .start = 0x2F000000, ++ .end = 0x2F000000 + (4096 * 4) - 1} ++}; ++#endif ++ ++static int pm_callback_power_on(struct kbase_device *kbdev) ++{ ++ /* Nothing is needed on VExpress, but we may have destroyed GPU state (if the below HARD_RESET code is active) */ ++ return 1; ++} ++ ++static void pm_callback_power_off(struct kbase_device *kbdev) ++{ ++#if HARD_RESET_AT_POWER_OFF ++ /* Cause a GPU hard reset to test whether we have actually idled the GPU ++ * and that we properly reconfigure the GPU on power up. ++ * Usually this would be dangerous, but if the GPU is working correctly it should ++ * be completely safe as the GPU should not be active at this point. ++ * However this is disabled normally because it will most likely interfere with ++ * bus logging etc. ++ */ ++ KBASE_TRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, NULL, 0u, 0); ++ kbase_os_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_HARD_RESET); ++#endif ++} ++ ++struct kbase_pm_callback_conf pm_callbacks = { ++ .power_on_callback = pm_callback_power_on, ++ .power_off_callback = pm_callback_power_off, ++ .power_suspend_callback = NULL, ++ .power_resume_callback = NULL ++}; ++ ++static struct kbase_platform_config versatile_platform_config = { ++#ifndef CONFIG_OF ++ .io_resources = &io_resources ++#endif ++}; ++ ++struct kbase_platform_config *kbase_get_platform_config(void) ++{ ++ return &versatile_platform_config; ++} ++ ++int kbase_platform_early_init(void) ++{ ++ /* Nothing needed at this stage */ ++ return 0; ++} ++ +diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/vexpress_6xvirtex7_10mhz/mali_kbase_cpu_vexpress.c b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress_6xvirtex7_10mhz/mali_kbase_cpu_vexpress.c +new file mode 100755 +index 000000000000..816dff49835f +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress_6xvirtex7_10mhz/mali_kbase_cpu_vexpress.c +@@ -0,0 +1,71 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2013 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#include ++#include ++#include "mali_kbase_cpu_vexpress.h" ++ ++#define HZ_IN_MHZ (1000000) ++ ++#define CORETILE_EXPRESS_A9X4_SCC_START (0x100E2000) ++#define MOTHERBOARD_SYS_CFG_START (0x10000000) ++#define SYS_CFGDATA_OFFSET (0x000000A0) ++#define SYS_CFGCTRL_OFFSET (0x000000A4) ++#define SYS_CFGSTAT_OFFSET (0x000000A8) ++ ++#define SYS_CFGCTRL_START_BIT_VALUE (1 << 31) ++#define READ_REG_BIT_VALUE (0 << 30) ++#define DCC_DEFAULT_BIT_VALUE (0 << 26) ++#define SYS_CFG_OSC_FUNC_BIT_VALUE (1 << 20) ++#define SITE_DEFAULT_BIT_VALUE (1 << 16) ++#define BOARD_STACK_POS_DEFAULT_BIT_VALUE (0 << 12) ++#define DEVICE_DEFAULT_BIT_VALUE (2 << 0) ++#define SYS_CFG_COMPLETE_BIT_VALUE (1 << 0) ++#define SYS_CFG_ERROR_BIT_VALUE (1 << 1) ++ ++#define FEED_REG_BIT_MASK (0x0F) ++#define FCLK_PA_DIVIDE_BIT_SHIFT (0x03) ++#define FCLK_PB_DIVIDE_BIT_SHIFT (0x07) ++#define FCLK_PC_DIVIDE_BIT_SHIFT (0x0B) ++#define AXICLK_PA_DIVIDE_BIT_SHIFT (0x0F) ++#define AXICLK_PB_DIVIDE_BIT_SHIFT (0x13) ++ ++#define IS_SINGLE_BIT_SET(val, pos) (val&(1< ++ ++/** ++ * @addtogroup uk_api User-Kernel Interface API ++ * @{ ++ */ ++ ++/** ++ * @addtogroup uk_api_kernel UKK (Kernel side) ++ * @{ ++ */ ++ ++/** ++ * Internal OS specific data structure associated with each UKK session. Part ++ * of a ukk_session object. ++ */ ++typedef struct ukkp_session { ++ int dummy; /**< No internal OS specific data at this time */ ++} ukkp_session; ++ ++/** @} end group uk_api_kernel */ ++ ++/** @} end group uk_api */ ++ ++#endif /* _UKK_OS_H__ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/protected_mode_switcher.h b/drivers/gpu/arm/bifrost_for_linux/protected_mode_switcher.h +new file mode 100755 +index 000000000000..5dc2f3ba8cf6 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/protected_mode_switcher.h +@@ -0,0 +1,64 @@ ++/* ++ * ++ * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _PROTECTED_MODE_SWITCH_H_ ++#define _PROTECTED_MODE_SWITCH_H_ ++ ++struct protected_mode_device; ++ ++/** ++ * struct protected_mode_ops - Callbacks for protected mode switch operations ++ * ++ * @protected_mode_enable: Callback to enable protected mode for device ++ * @protected_mode_disable: Callback to disable protected mode for device ++ */ ++struct protected_mode_ops { ++ /** ++ * protected_mode_enable() - Enable protected mode on device ++ * @dev: The struct device ++ * ++ * Return: 0 on success, non-zero on error ++ */ ++ int (*protected_mode_enable)( ++ struct protected_mode_device *protected_dev); ++ ++ /** ++ * protected_mode_disable() - Disable protected mode on device, and ++ * reset device ++ * @dev: The struct device ++ * ++ * Return: 0 on success, non-zero on error ++ */ ++ int (*protected_mode_disable)( ++ struct protected_mode_device *protected_dev); ++}; ++ ++/** ++ * struct protected_mode_device - Device structure for protected mode devices ++ * ++ * @ops - Callbacks associated with this device ++ * @data - Pointer to device private data ++ * ++ * This structure should be registered with the platform device using ++ * platform_set_drvdata(). ++ */ ++struct protected_mode_device { ++ struct protected_mode_ops ops; ++ void *data; ++}; ++ ++#endif /* _PROTECTED_MODE_SWITCH_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/sconscript b/drivers/gpu/arm/bifrost_for_linux/sconscript +new file mode 100755 +index 000000000000..e738dd7a3869 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/sconscript +@@ -0,0 +1,72 @@ ++# ++# (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++import sys ++Import('env') ++ ++SConscript( 'tests/sconscript' ) ++ ++mock_test = 0 ++ ++# Source files required for kbase. ++kbase_src = [ ++ Glob('*.c'), ++ Glob('backend/*/*.c'), ++ Glob('internal/*/*.c'), ++ Glob('ipa/*.c'), ++ Glob('platform/%s/*.c' % env['platform_config']), ++] ++ ++if env['platform_config']=='juno_soc': ++ kbase_src += [Glob('platform/devicetree/*.c')] ++else: ++ kbase_src += [Glob('platform/%s/*.c' % env['platform_config'])] ++ ++if Glob('#kernel/drivers/gpu/arm/midgard/tests/internal/src/mock') and env['unit'] == '1': ++ kbase_src += [Glob('#kernel/drivers/gpu/arm/midgard/tests/internal/src/mock/*.c')] ++ mock_test = 1 ++ ++make_args = env.kernel_get_config_defines(ret_list = True) + [ ++ 'PLATFORM=%s' % env['platform'], ++ 'MALI_ERROR_INJECT_ON=%s' % env['error_inject'], ++ 'MALI_KERNEL_TEST_API=%s' % env['debug'], ++ 'MALI_UNIT_TEST=%s' % env['unit'], ++ 'MALI_RELEASE_NAME=%s' % env['mali_release_name'], ++ 'MALI_MOCK_TEST=%s' % mock_test, ++ 'MALI_CUSTOMER_RELEASE=%s' % env['release'], ++ 'MALI_INSTRUMENTATION_LEVEL=%s' % env['instr'], ++ 'MALI_COVERAGE=%s' % env['coverage'], ++] ++ ++kbase = env.BuildKernelModule('$STATIC_LIB_PATH/mali_kbase.ko', kbase_src, ++ make_args = make_args) ++ ++# Add a dependency on kds.ko. ++# Only necessary when KDS is not built into the kernel. ++# ++if env['os'] != 'android': ++ if not env.KernelConfigEnabled("CONFIG_KDS"): ++ env.Depends(kbase, '$STATIC_LIB_PATH/kds.ko') ++ ++# need Module.symvers from ump.ko build ++if int(env['ump']) == 1: ++ env.Depends(kbase, '$STATIC_LIB_PATH/ump.ko') ++ ++if 'smc_protected_mode_switcher' in env: ++ env.Depends('$STATIC_LIB_PATH/mali_kbase.ko', '$STATIC_LIB_PATH/smc_protected_mode_switcher.ko') ++ ++env.KernelObjTarget('kbase', kbase) ++ ++env.AppendUnique(BASE=['cutils_linked_list']) +diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/Kbuild b/drivers/gpu/arm/bifrost_for_linux/tests/Kbuild +new file mode 100755 +index 000000000000..b4bed0473439 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/tests/Kbuild +@@ -0,0 +1,17 @@ ++# ++# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++obj-$(CONFIG_MALI_KUTF) += kutf/ ++obj-$(CONFIG_MALI_IRQ_LATENCY) += mali_kutf_irq_test/ +diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/Kconfig b/drivers/gpu/arm/bifrost_for_linux/tests/Kconfig +new file mode 100755 +index 000000000000..da0515c065de +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/tests/Kconfig +@@ -0,0 +1,17 @@ ++# ++# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++source "drivers/gpu/arm/midgard/tests/kutf/Kconfig" ++source "drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Kconfig" +diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/include/kutf/kutf_helpers.h b/drivers/gpu/arm/bifrost_for_linux/tests/include/kutf/kutf_helpers.h +new file mode 100755 +index 000000000000..3f1dfc244d30 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/tests/include/kutf/kutf_helpers.h +@@ -0,0 +1,216 @@ ++/* ++ * ++ * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KERNEL_UTF_HELPERS_H_ ++#define _KERNEL_UTF_HELPERS_H_ ++ ++/* kutf_helpers.h ++ * Test helper functions for the kernel UTF test infrastructure. ++ * ++ * This collection of helper functions are provided as 'stock' implementation ++ * helpers for certain features of kutf. Tests can implement common/boilerplate ++ * functionality using these, whilst still providing them the option of ++ * implementing completely custom functions themselves to use those kutf ++ * features. ++ */ ++ ++#include ++#include ++#include ++ ++/** ++ * enum kutf_helper_textbuf_flag - flags for textbufs ++ * @KUTF_HELPER_TEXTBUF_FLAG_DYING: Test is dying, textbuf should not allow ++ * writes, nor block on empty. ++ */ ++enum kutf_helper_textbuf_flag { ++ KUTF_HELPER_TEXTBUF_FLAG_DYING = (1u << 0), ++}; ++ ++/** ++ * struct kutf_helper_textbuf_line - Structure representing a line of text ++ * ++ * The string itself is stored immediately after this. ++ * ++ * @node: List node for the textbuf's textbuf_list ++ * @str_size: Length of the string buffer, including the \0 terminator ++ * @str: 'Flexible array' for the string representing the line ++ */ ++struct kutf_helper_textbuf_line { ++ struct list_head node; ++ int str_size; ++ char str[]; ++}; ++ ++/** ++ * struct kutf_helper_textbuf - Structure to representing sequential lines of ++ * text ++ * @lock: mutex to hold whilst accessing the structure ++ * @nr_user_clients: Number of userspace clients connected via an open() ++ * call ++ * @mempool: mempool for allocating lines ++ * @scratchpad: scratch area for receiving text of size max_line_size ++ * @used_bytes: number of valid bytes in the scratchpad ++ * @prev_pos: Previous position userspace has accessed ++ * @prev_line_pos: Previous start of line position userspace has accessed ++ * @textbuf_list: List head to store all the lines of text ++ * @max_line_size: Maximum size in memory allowed for a line of text ++ * @max_nr_lines: Maximum number of lines permitted in this textbuf ++ * @nr_lines: Number of entries in textbuf_list ++ * @flags: Flags indicating state of the textbuf, using values ++ * from enum kutf_helper_textbuf_flag ++ * @user_opened_wq: Waitq for when there's at least one userspace client ++ * connected to the textbuf via an open() call ++ * @not_full_wq: Waitq for when the textbuf can be enqueued into/can ++ * consume data from userspace ++ * @not_empty_wq: Waitq for when the textbuf can be dequeued from/can ++ * produce data for userspace ++ */ ++ ++struct kutf_helper_textbuf { ++ struct mutex lock; ++ int nr_user_clients; ++ struct kutf_mempool *mempool; ++ char *scratchpad; ++ int used_bytes; ++ loff_t prev_pos; ++ loff_t prev_line_pos; ++ struct list_head textbuf_list; ++ int max_line_size; ++ int max_nr_lines; ++ int nr_lines; ++ unsigned long flags; ++ wait_queue_head_t user_opened_wq; ++ wait_queue_head_t not_full_wq; ++ wait_queue_head_t not_empty_wq; ++ ++}; ++ ++/* stock callbacks for userspace to read from/write to the 'data' file as a ++ * textbuf */ ++extern struct kutf_userdata_ops kutf_helper_textbuf_userdata_ops; ++ ++/** ++ * kutf_helper_textbuf_init() - init a textbuf for use as a 'data' file ++ * consumer/producer ++ * @textbuf: textbuf to initialize ++ * @mempool: mempool to allocate from ++ * @max_line_size: maximum line size expected to/from userspace ++ * @max_nr_lines: maximum number of lines to expect to/from userspace ++ * ++ * Initialize a textbuf so that it can consume writes made to the 'data' file, ++ * and produce reads for userspace on the 'data' file. Tests may then read the ++ * lines written by userspace, or fill the buffer so it may be read back by ++ * userspace. ++ * ++ * The caller should write the @textbuf pointer into the kutf_context's ++ * userdata_producer_priv or userdata_consumer_priv member during fixture ++ * creation. ++ * ++ * Usually a test will have separate textbufs for userspace to write to and ++ * read from. Using the same one for both will echo back to the user what they ++ * are writing. ++ * ++ * Lines are understood as being separated by the '\n' character, but no '\n' ++ * characters will be observed by the test ++ * ++ * @max_line_size puts an upper bound on the size of lines in a textbuf, ++ * including the \0 terminator. Lines exceeding this will be truncated, ++ * effectively ignoring incoming data until the next '\n' ++ * ++ * Combining this with @max_nr_lines puts an upper bound on the size of the ++ * file read in ++ * ++ * Return: 0 on success, or negative value on error. ++ */ ++int kutf_helper_textbuf_init(struct kutf_helper_textbuf *textbuf, ++ struct kutf_mempool *mempool, int max_line_size, ++ int max_nr_lines); ++ ++/** ++ * kutf_helper_textbuf_wait_for_user() - wait for userspace to open the 'data' ++ * file ++ * @textbuf: textbuf to wait on ++ * ++ * This can be used to synchronize with userspace so that subsequent calls to ++ * kutf_helper_textbuf_dequeue() and kutf_helper_textbuf_enqueue() should ++ * succeed. ++ * ++ * Waiting is done on a timeout. ++ * ++ * There is of course no guarantee that userspace will keep the file open after ++ * this, but any error in the dequeue/enqueue functions afterwards can be ++ * treated as such rather than "we're still waiting for userspace to begin" ++ * ++ * Return: 0 if waited successfully, -ETIMEDOUT if we exceeded the ++ * timeout, or some other negative value if there was an ++ * error during waiting. ++ */ ++ ++int kutf_helper_textbuf_wait_for_user(struct kutf_helper_textbuf *textbuf); ++ ++ ++/** ++ * kutf_helper_textbuf_dequeue() - dequeue a line from a textbuf ++ * @textbuf: textbuf dequeue a line as a string from ++ * @str_size: pointer to storage to receive the size of the string, ++ * which includes the '\0' terminator, or NULL if not ++ * required ++ * ++ * Dequeue (remove) a line from the start of the textbuf as a string, and ++ * return it. ++ * ++ * If no lines are available, then this will block until a line has been ++ * submitted. If a userspace client is not connected and there are no remaining ++ * lines, then this function returns NULL instead. ++ * ++ * The memory for the string comes from the kutf_mempool given during ++ * initialization of the textbuf, and shares the same lifetime as it. ++ * ++ * Return: pointer to the next line of the textbuf. NULL indicated ++ * all userspace clients disconnected. An error value to be ++ * checked with IS_ERR() family of functions if a signal or ++ * some other error occurred ++ */ ++char *kutf_helper_textbuf_dequeue(struct kutf_helper_textbuf *textbuf, ++ int *str_size); ++ ++/** ++ * kutf_helper_textbuf_enqueue() - enqueue a line to a textbuf ++ * @textbuf: textbuf to enqueue a line as a string to ++ * @enqueue_str: pointer to the string to enqueue to the textbuf ++ * @buf_max_size: maximum size of the buffer holding @enqueue_str ++ * ++ * Enqueue (add) a line to the end of a textbuf as a string. ++ * ++ * The caller should avoid placing '\n' characters in their strings, as these ++ * will not be split into multiple lines. ++ * ++ * A copy of the string will be made into the textbuf, so @enqueue_str can be ++ * freed immediately after if.the caller wishes to do so. ++ * ++ * If the maximum amount of lines has been reached, then this will block until ++ * a line has been removed to make space. If a userspace client is not ++ * connected and there is no space available, then this function returns ++ * -EBUSY. ++ * ++ * Return: 0 on success, or negative value on error ++ */ ++int kutf_helper_textbuf_enqueue(struct kutf_helper_textbuf *textbuf, ++ char *enqueue_str, int buf_max_size); ++ ++#endif /* _KERNEL_UTF_HELPERS_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/include/kutf/kutf_helpers_user.h b/drivers/gpu/arm/bifrost_for_linux/tests/include/kutf/kutf_helpers_user.h +new file mode 100755 +index 000000000000..759bf717c7cd +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/tests/include/kutf/kutf_helpers_user.h +@@ -0,0 +1,179 @@ ++/* ++ * ++ * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KERNEL_UTF_HELPERS_USER_H_ ++#define _KERNEL_UTF_HELPERS_USER_H_ ++ ++/* kutf_helpers.h ++ * Test helper functions for the kernel UTF test infrastructure, whose ++ * implementation mirrors that of similar functions for kutf-userside ++ */ ++ ++#include ++#include ++ ++ ++#define KUTF_HELPER_MAX_VAL_NAME_LEN 255 ++ ++enum kutf_helper_valtype { ++ KUTF_HELPER_VALTYPE_INVALID, ++ KUTF_HELPER_VALTYPE_U64, ++ KUTF_HELPER_VALTYPE_STR, ++ ++ KUTF_HELPER_VALTYPE_COUNT /* Must be last */ ++}; ++ ++struct kutf_helper_named_val { ++ enum kutf_helper_valtype type; ++ char *val_name; ++ union { ++ u64 val_u64; ++ char *val_str; ++ } u; ++}; ++ ++/* Extra error values for certain helpers when we want to distinguish between ++ * Linux's own error values too. ++ * ++ * These can only be used on certain functions returning an int type that are ++ * documented as returning one of these potential values, they cannot be used ++ * from functions return a ptr type, since we can't decode it with PTR_ERR ++ * ++ * No negative values are used - Linux error codes should be used instead, and ++ * indicate a problem in accessing the data file itself (are generally ++ * unrecoverable) ++ * ++ * Positive values indicate correct access but invalid parsing (can be ++ * recovered from assuming data in the future is correct) */ ++enum kutf_helper_err { ++ /* No error - must be zero */ ++ KUTF_HELPER_ERR_NONE = 0, ++ /* Named value parsing encountered an invalid name */ ++ KUTF_HELPER_ERR_INVALID_NAME, ++ /* Named value parsing of string or u64 type encountered extra ++ * characters after the value (after the last digit for a u64 type or ++ * after the string end delimiter for string type) */ ++ KUTF_HELPER_ERR_CHARS_AFTER_VAL, ++ /* Named value parsing of string type couldn't find the string end ++ * delimiter. ++ * ++ * This cannot be encountered when the NAME="value" message exceeds the ++ * textbuf's maximum line length, because such messages are not checked ++ * for an end string delimiter */ ++ KUTF_HELPER_ERR_NO_END_DELIMITER, ++ /* Named value didn't parse as any of the known types */ ++ KUTF_HELPER_ERR_INVALID_VALUE, ++}; ++ ++ ++/* textbuf Send named NAME=value pair, u64 value ++ * ++ * NAME must match [A-Z0-9_]\+ and can be up to MAX_VAL_NAME_LEN characters long ++ * ++ * This is assuming the kernel-side test is using the 'textbuf' helpers ++ * ++ * Any failure will be logged on the suite's current test fixture ++ * ++ * Returns 0 on success, non-zero on failure ++ */ ++int kutf_helper_textbuf_send_named_u64(struct kutf_context *context, ++ struct kutf_helper_textbuf *textbuf, char *val_name, u64 val); ++ ++/* Get the maximum length of a string that can be represented as a particular ++ * NAME="value" pair without string-value truncation in the kernel's buffer ++ * ++ * Given val_name and the kernel buffer's size, this can be used to determine ++ * the maximum length of a string that can be sent as val_name="value" pair ++ * without having the string value truncated. Any string longer than this will ++ * be truncated at some point during communication to this size. ++ * ++ * The calculation is valid both for sending strings of val_str_len to kernel, ++ * and for receiving a string that was originally val_str_len from the kernel. ++ * ++ * It is assumed that valname is a valid name for ++ * kutf_test_helpers_textbuf_send_named_str(), and no checking will be made to ++ * ensure this. ++ * ++ * Returns the maximum string length that can be represented, or a negative ++ * value if the NAME="value" encoding itself wouldn't fit in kern_buf_sz ++ */ ++int kutf_helper_textbuf_max_str_len_for_kern(char *val_name, int kern_buf_sz); ++ ++/* textbuf Send named NAME="str" pair ++ * ++ * no escaping allowed in str. Any of the following characters will terminate ++ * the string: '"' '\\' '\n' ++ * ++ * NAME must match [A-Z0-9_]\+ and can be up to MAX_VAL_NAME_LEN characters long ++ * ++ * This is assuming the kernel-side test is using the 'textbuf' helpers ++ * ++ * Any failure will be logged on the suite's current test fixture ++ * ++ * Returns 0 on success, non-zero on failure */ ++int kutf_helper_textbuf_send_named_str(struct kutf_context *context, ++ struct kutf_helper_textbuf *textbuf, char *val_name, ++ char *val_str); ++ ++/* textbuf Receive named NAME=value pair ++ * ++ * This can receive u64 and string values - check named_val->type ++ * ++ * If you are not planning on dynamic handling of the named value's name and ++ * type, then kutf_test_helpers_textbuf_receive_check_val() is more useful as a ++ * convenience function. ++ * ++ * String members of named_val will come from memory allocated on the fixture's mempool ++ * ++ * Returns 0 on success. Negative value on failure to receive from the 'data' ++ * file, positive value indicates an enum kutf_helper_err value for correct ++ * reception of data but invalid parsing */ ++int kutf_helper_textbuf_receive_named_val(struct kutf_helper_named_val *named_val, ++ struct kutf_helper_textbuf *textbuf); ++ ++/* textbuf Receive and validate NAME=value pair ++ * ++ * As with kutf_test_helpers_textbuf_receive_named_val, but validate that the ++ * name and type are as expected, as a convenience for a common pattern found ++ * in tests. ++ * ++ * NOTE: this only returns an error value if there was actually a problem ++ * receiving data. ++ * ++ * NOTE: If the underlying data was received correctly, but: ++ * - isn't of the expected name ++ * - isn't the expected type ++ * - isn't correctly parsed for the type ++ * then the following happens: ++ * - failure result is recorded ++ * - named_val->type will be KUTF_HELPER_VALTYPE_INVALID ++ * - named_val->u will contain some default value that should be relatively ++ * harmless for the test, including being writable in the case of string ++ * values ++ * - return value will be 0 to indicate success ++ * ++ * The rationale behind this is that we'd prefer to continue the rest of the ++ * test with failures propagated, rather than hitting a timeout */ ++int kutf_helper_textbuf_receive_check_val(struct kutf_helper_named_val *named_val, ++ struct kutf_context *context, struct kutf_helper_textbuf *textbuf, ++ char *expect_val_name, enum kutf_helper_valtype expect_val_type); ++ ++/* Output a named value to kmsg */ ++void kutf_helper_output_named_val(struct kutf_helper_named_val *named_val); ++ ++ ++#endif /* _KERNEL_UTF_HELPERS_USER_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/include/kutf/kutf_mem.h b/drivers/gpu/arm/bifrost_for_linux/tests/include/kutf/kutf_mem.h +new file mode 100755 +index 000000000000..584c9dd4bc13 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/tests/include/kutf/kutf_mem.h +@@ -0,0 +1,68 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KERNEL_UTF_MEM_H_ ++#define _KERNEL_UTF_MEM_H_ ++ ++/* kutf_mem.h ++ * Functions for management of memory pools in the kernel. ++ * ++ * This module implements a memory pool allocator, allowing a test ++ * implementation to allocate linked allocations which can then be freed by a ++ * single free which releases all of the resources held by the entire pool. ++ * ++ * Note that it is not possible to free single resources within the pool once ++ * allocated. ++ */ ++ ++#include ++#include ++ ++/** ++ * struct kutf_mempool - the memory pool context management structure ++ * @head: list head on which the allocations in this context are added to ++ * @lock: mutex for concurrent allocation from multiple threads ++ * ++ */ ++struct kutf_mempool { ++ struct list_head head; ++ struct mutex lock; ++}; ++ ++/** ++ * kutf_mempool_init() - Initialize a memory pool. ++ * @pool: Memory pool structure to initialize, provided by the user ++ * ++ * Return: zero on success ++ */ ++int kutf_mempool_init(struct kutf_mempool *pool); ++ ++/** ++ * kutf_mempool_alloc() - Allocate memory from a pool ++ * @pool: Memory pool to allocate from ++ * @size: Size of memory wanted in number of bytes ++ * ++ * Return: Pointer to memory on success, NULL on failure. ++ */ ++void *kutf_mempool_alloc(struct kutf_mempool *pool, size_t size); ++ ++/** ++ * kutf_mempool_destroy() - Destroy a memory pool, freeing all memory within it. ++ * @pool: The memory pool to free ++ */ ++void kutf_mempool_destroy(struct kutf_mempool *pool); ++#endif /* _KERNEL_UTF_MEM_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/include/kutf/kutf_resultset.h b/drivers/gpu/arm/bifrost_for_linux/tests/include/kutf/kutf_resultset.h +new file mode 100755 +index 000000000000..1cc85f1b7a46 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/tests/include/kutf/kutf_resultset.h +@@ -0,0 +1,121 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KERNEL_UTF_RESULTSET_H_ ++#define _KERNEL_UTF_RESULTSET_H_ ++ ++/* kutf_resultset.h ++ * Functions and structures for handling test results and result sets. ++ * ++ * This section of the kernel UTF contains structures and functions used for the ++ * management of Results and Result Sets. ++ */ ++ ++/** ++ * enum kutf_result_status - Status values for a single Test error. ++ * @KUTF_RESULT_BENCHMARK: Result is a meta-result containing benchmark ++ * results. ++ * @KUTF_RESULT_SKIP: The test was skipped. ++ * @KUTF_RESULT_UNKNOWN: The test has an unknown result. ++ * @KUTF_RESULT_PASS: The test result passed. ++ * @KUTF_RESULT_DEBUG: The test result passed, but raised a debug ++ * message. ++ * @KUTF_RESULT_INFO: The test result passed, but raised ++ * an informative message. ++ * @KUTF_RESULT_WARN: The test result passed, but raised a warning ++ * message. ++ * @KUTF_RESULT_FAIL: The test result failed with a non-fatal error. ++ * @KUTF_RESULT_FATAL: The test result failed with a fatal error. ++ * @KUTF_RESULT_ABORT: The test result failed due to a non-UTF ++ * assertion failure. ++ * @KUTF_RESULT_COUNT: The current number of possible status messages. ++ */ ++enum kutf_result_status { ++ KUTF_RESULT_BENCHMARK = -3, ++ KUTF_RESULT_SKIP = -2, ++ KUTF_RESULT_UNKNOWN = -1, ++ ++ KUTF_RESULT_PASS = 0, ++ KUTF_RESULT_DEBUG = 1, ++ KUTF_RESULT_INFO = 2, ++ KUTF_RESULT_WARN = 3, ++ KUTF_RESULT_FAIL = 4, ++ KUTF_RESULT_FATAL = 5, ++ KUTF_RESULT_ABORT = 6, ++ ++ KUTF_RESULT_COUNT ++}; ++ ++/* The maximum size of a kutf_result_status result when ++ * converted to a string ++ */ ++#define KUTF_ERROR_MAX_NAME_SIZE 21 ++ ++#ifdef __KERNEL__ ++ ++#include ++ ++/** ++ * struct kutf_result - Represents a single test result. ++ * @node: Next result in the list of results. ++ * @status: The status summary (pass / warn / fail / etc). ++ * @message: A more verbose status message. ++ */ ++struct kutf_result { ++ struct list_head node; ++ enum kutf_result_status status; ++ const char *message; ++}; ++ ++/** ++ * kutf_create_result_set() - Create a new result set ++ * to which results can be added. ++ * ++ * Return: The created resultset. ++ */ ++struct kutf_result_set *kutf_create_result_set(void); ++ ++/** ++ * kutf_add_result() - Add a result to the end of an existing resultset. ++ * ++ * @mempool: The memory pool to allocate the result storage from. ++ * @set: The resultset to add the result to. ++ * @status: The result status to add. ++ * @message: The result message to add. ++ */ ++void kutf_add_result(struct kutf_mempool *mempool, struct kutf_result_set *set, ++ enum kutf_result_status status, const char *message); ++ ++/** ++ * kutf_remove_result() - Remove a result from the head of a resultset. ++ * @set: The resultset. ++ * ++ * Return: result or NULL if there are no further results in the resultset. ++ */ ++struct kutf_result *kutf_remove_result( ++ struct kutf_result_set *set); ++ ++/** ++ * kutf_destroy_result_set() - Free a previously created resultset. ++ * ++ * @results: The result set whose resources to free. ++ */ ++void kutf_destroy_result_set(struct kutf_result_set *results); ++ ++#endif /* __KERNEL__ */ ++ ++#endif /* _KERNEL_UTF_RESULTSET_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/include/kutf/kutf_suite.h b/drivers/gpu/arm/bifrost_for_linux/tests/include/kutf/kutf_suite.h +new file mode 100755 +index 000000000000..cba2b2d84d62 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/tests/include/kutf/kutf_suite.h +@@ -0,0 +1,568 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KERNEL_UTF_SUITE_H_ ++#define _KERNEL_UTF_SUITE_H_ ++ ++/* kutf_suite.h ++ * Functions for management of test suites. ++ * ++ * This collection of data structures, macros, and functions are used to ++ * create Test Suites, Tests within those Test Suites, and Fixture variants ++ * of each test. ++ */ ++ ++#include ++ ++#include ++#include ++ ++/** ++ * Pseudo-flag indicating an absence of any specified test class. Note that ++ * tests should not be annotated with this constant as it is simply a zero ++ * value; tests without a more specific class must be marked with the flag ++ * KUTF_F_TEST_GENERIC. ++ */ ++#define KUTF_F_TEST_NONE ((unsigned int)(0)) ++ ++/** ++ * Class indicating this test is a smoke test. ++ * A given set of smoke tests should be quick to run, enabling rapid turn-around ++ * of "regress-on-commit" test runs. ++ */ ++#define KUTF_F_TEST_SMOKETEST ((unsigned int)(1 << 1)) ++ ++/** ++ * Class indicating this test is a performance test. ++ * These tests typically produce a performance metric, such as "time to run" or ++ * "frames per second", ++ */ ++#define KUTF_F_TEST_PERFORMANCE ((unsigned int)(1 << 2)) ++ ++/** ++ * Class indicating that this test is a deprecated test. ++ * These tests have typically been replaced by an alternative test which is ++ * more efficient, or has better coverage. ++ */ ++#define KUTF_F_TEST_DEPRECATED ((unsigned int)(1 << 3)) ++ ++/** ++ * Class indicating that this test is a known failure. ++ * These tests have typically been run and failed, but marking them as a known ++ * failure means it is easier to triage results. ++ * ++ * It is typically more convenient to triage known failures using the ++ * results database and web UI, as this means there is no need to modify the ++ * test code. ++ */ ++#define KUTF_F_TEST_EXPECTED_FAILURE ((unsigned int)(1 << 4)) ++ ++/** ++ * Class indicating that this test is a generic test, which is not a member of ++ * a more specific test class. Tests which are not created with a specific set ++ * of filter flags by the user are assigned this test class by default. ++ */ ++#define KUTF_F_TEST_GENERIC ((unsigned int)(1 << 5)) ++ ++/** ++ * Class indicating this test is a resource allocation failure test. ++ * A resource allocation failure test will test that an error code is ++ * correctly propagated when an allocation fails. ++ */ ++#define KUTF_F_TEST_RESFAIL ((unsigned int)(1 << 6)) ++ ++/** ++ * Additional flag indicating that this test is an expected failure when ++ * run in resource failure mode. These tests are never run when running ++ * the low resource mode. ++ */ ++#define KUTF_F_TEST_EXPECTED_FAILURE_RF ((unsigned int)(1 << 7)) ++ ++/** ++ * Flag reserved for user-defined filter zero. ++ */ ++#define KUTF_F_TEST_USER_0 ((unsigned int)(1 << 24)) ++ ++/** ++ * Flag reserved for user-defined filter one. ++ */ ++#define KUTF_F_TEST_USER_1 ((unsigned int)(1 << 25)) ++ ++/** ++ * Flag reserved for user-defined filter two. ++ */ ++#define KUTF_F_TEST_USER_2 ((unsigned int)(1 << 26)) ++ ++/** ++ * Flag reserved for user-defined filter three. ++ */ ++#define KUTF_F_TEST_USER_3 ((unsigned int)(1 << 27)) ++ ++/** ++ * Flag reserved for user-defined filter four. ++ */ ++#define KUTF_F_TEST_USER_4 ((unsigned int)(1 << 28)) ++ ++/** ++ * Flag reserved for user-defined filter five. ++ */ ++#define KUTF_F_TEST_USER_5 ((unsigned int)(1 << 29)) ++ ++/** ++ * Flag reserved for user-defined filter six. ++ */ ++#define KUTF_F_TEST_USER_6 ((unsigned int)(1 << 30)) ++ ++/** ++ * Flag reserved for user-defined filter seven. ++ */ ++#define KUTF_F_TEST_USER_7 ((unsigned int)(1 << 31)) ++ ++/** ++ * Pseudo-flag indicating that all test classes should be executed. ++ */ ++#define KUTF_F_TEST_ALL ((unsigned int)(0xFFFFFFFFU)) ++ ++/** ++ * union kutf_callback_data - Union used to store test callback data ++ * @ptr_value: pointer to the location where test callback data ++ * are stored ++ * @u32_value: a number which represents test callback data ++ */ ++union kutf_callback_data { ++ void *ptr_value; ++ u32 u32_value; ++}; ++ ++/** ++ * struct kutf_userdata_ops- Structure defining methods to exchange data ++ * with userspace via the 'data' file ++ * @open: Function used to notify when the 'data' file was opened ++ * @release: Function used to notify when the 'data' file was closed ++ * @notify_ended: Function used to notify when the test has ended. ++ * @consumer: Function used to consume writes from userspace ++ * @producer: Function used to produce data for userspace to read ++ * ++ * All ops can be NULL. ++ */ ++struct kutf_userdata_ops { ++ int (*open)(void *priv); ++ void (*release)(void *priv); ++ void (*notify_ended)(void *priv); ++ ssize_t (*consumer)(void *priv, const char __user *userbuf, ++ size_t userbuf_len, loff_t *ppos); ++ ssize_t (*producer)(void *priv, char __user *userbuf, ++ size_t userbuf_len, loff_t *ppos); ++}; ++ ++/** ++ * struct kutf_context - Structure representing a kernel test context ++ * @kref: Refcount for number of users of this context ++ * @suite: Convenience pointer to the suite this context ++ * is running ++ * @test_fix: The fixture that is being run in this context ++ * @fixture_pool: The memory pool used for the duration of ++ * the fixture/text context. ++ * @fixture: The user provided fixture structure. ++ * @fixture_index: The index (id) of the current fixture. ++ * @fixture_name: The name of the current fixture (or NULL if unnamed). ++ * @test_data: Any user private data associated with this test ++ * @result_set: All the results logged by this test context ++ * @status: The status of the currently running fixture. ++ * @expected_status: The expected status on exist of the currently ++ * running fixture. ++ * @userdata_consumer_priv: Parameter to pass into kutf_userdata_ops ++ * consumer function. Must not be NULL if a ++ * consumer function was specified ++ * @userdata_producer_priv: Parameter to pass into kutf_userdata_ops ++ * producer function. Must not be NULL if a ++ * producer function was specified ++ * @userdata_dentry: The debugfs file for userdata exchange ++ */ ++struct kutf_context { ++ struct kref kref; ++ struct kutf_suite *suite; ++ struct kutf_test_fixture *test_fix; ++ struct kutf_mempool fixture_pool; ++ void *fixture; ++ unsigned int fixture_index; ++ const char *fixture_name; ++ union kutf_callback_data test_data; ++ struct kutf_result_set *result_set; ++ enum kutf_result_status status; ++ enum kutf_result_status expected_status; ++ void *userdata_consumer_priv; ++ void *userdata_producer_priv; ++ struct dentry *userdata_dentry; ++}; ++ ++/** ++ * struct kutf_suite - Structure representing a kernel test suite ++ * @app: The application this suite belongs to. ++ * @name: The name of this suite. ++ * @suite_data: Any user private data associated with this ++ * suite. ++ * @create_fixture: Function used to create a new fixture instance ++ * @remove_fixture: Function used to destroy a new fixture instance ++ * @fixture_variants: The number of variants (must be at least 1). ++ * @suite_default_flags: Suite global filter flags which are set on ++ * all tests. ++ * @node: List node for suite_list ++ * @dir: The debugfs directory for this suite ++ * @test_list: List head to store all the tests which are ++ * part of this suite ++ */ ++struct kutf_suite { ++ struct kutf_application *app; ++ const char *name; ++ union kutf_callback_data suite_data; ++ void *(*create_fixture)(struct kutf_context *context); ++ void (*remove_fixture)(struct kutf_context *context); ++ unsigned int fixture_variants; ++ unsigned int suite_default_flags; ++ struct list_head node; ++ struct dentry *dir; ++ struct list_head test_list; ++}; ++ ++/* ============================================================================ ++ Application functions ++============================================================================ */ ++ ++/** ++ * kutf_create_application() - Create an in kernel test application. ++ * @name: The name of the test application. ++ * ++ * Return: pointer to the kutf_application on success or NULL ++ * on failure ++ */ ++struct kutf_application *kutf_create_application(const char *name); ++ ++/** ++ * kutf_destroy_application() - Destroy an in kernel test application. ++ * ++ * @app: The test application to destroy. ++ */ ++void kutf_destroy_application(struct kutf_application *app); ++ ++/* ============================================================================ ++ Suite functions ++============================================================================ */ ++ ++/** ++ * kutf_create_suite() - Create a kernel test suite. ++ * @app: The test application to create the suite in. ++ * @name: The name of the suite. ++ * @fixture_count: The number of fixtures to run over the test ++ * functions in this suite ++ * @create_fixture: Callback used to create a fixture. The returned value ++ * is stored in the fixture pointer in the context for ++ * use in the test functions. ++ * @remove_fixture: Callback used to remove a previously created fixture. ++ * ++ * Suite names must be unique. Should two suites with the same name be ++ * registered with the same application then this function will fail, if they ++ * are registered with different applications then the function will not detect ++ * this and the call will succeed. ++ * ++ * Return: pointer to the created kutf_suite on success or NULL ++ * on failure ++ */ ++struct kutf_suite *kutf_create_suite( ++ struct kutf_application *app, ++ const char *name, ++ unsigned int fixture_count, ++ void *(*create_fixture)(struct kutf_context *context), ++ void (*remove_fixture)(struct kutf_context *context)); ++ ++/** ++ * kutf_create_suite_with_filters() - Create a kernel test suite with user ++ * defined default filters. ++ * @app: The test application to create the suite in. ++ * @name: The name of the suite. ++ * @fixture_count: The number of fixtures to run over the test ++ * functions in this suite ++ * @create_fixture: Callback used to create a fixture. The returned value ++ * is stored in the fixture pointer in the context for ++ * use in the test functions. ++ * @remove_fixture: Callback used to remove a previously created fixture. ++ * @filters: Filters to apply to a test if it doesn't provide its own ++ * ++ * Suite names must be unique. Should two suites with the same name be ++ * registered with the same application then this function will fail, if they ++ * are registered with different applications then the function will not detect ++ * this and the call will succeed. ++ * ++ * Return: pointer to the created kutf_suite on success or NULL on failure ++ */ ++struct kutf_suite *kutf_create_suite_with_filters( ++ struct kutf_application *app, ++ const char *name, ++ unsigned int fixture_count, ++ void *(*create_fixture)(struct kutf_context *context), ++ void (*remove_fixture)(struct kutf_context *context), ++ unsigned int filters); ++ ++/** ++ * kutf_create_suite_with_filters_and_data() - Create a kernel test suite with ++ * user defined default filters. ++ * @app: The test application to create the suite in. ++ * @name: The name of the suite. ++ * @fixture_count: The number of fixtures to run over the test ++ * functions in this suite ++ * @create_fixture: Callback used to create a fixture. The returned value ++ * is stored in the fixture pointer in the context for ++ * use in the test functions. ++ * @remove_fixture: Callback used to remove a previously created fixture. ++ * @filters: Filters to apply to a test if it doesn't provide its own ++ * @suite_data: Suite specific callback data, provided during the ++ * running of the test in the kutf_context ++ * ++ * Return: pointer to the created kutf_suite on success or NULL ++ * on failure ++ */ ++struct kutf_suite *kutf_create_suite_with_filters_and_data( ++ struct kutf_application *app, ++ const char *name, ++ unsigned int fixture_count, ++ void *(*create_fixture)(struct kutf_context *context), ++ void (*remove_fixture)(struct kutf_context *context), ++ unsigned int filters, ++ union kutf_callback_data suite_data); ++ ++/** ++ * kutf_add_test() - Add a test to a kernel test suite. ++ * @suite: The suite to add the test to. ++ * @id: The ID of the test. ++ * @name: The name of the test. ++ * @execute: Callback to the test function to run. ++ * ++ * Note: As no filters are provided the test will use the suite filters instead ++ */ ++void kutf_add_test(struct kutf_suite *suite, ++ unsigned int id, ++ const char *name, ++ void (*execute)(struct kutf_context *context)); ++ ++/** ++ * kutf_add_test_with_filters() - Add a test to a kernel test suite with filters ++ * @suite: The suite to add the test to. ++ * @id: The ID of the test. ++ * @name: The name of the test. ++ * @execute: Callback to the test function to run. ++ * @filters: A set of filtering flags, assigning test categories. ++ */ ++void kutf_add_test_with_filters(struct kutf_suite *suite, ++ unsigned int id, ++ const char *name, ++ void (*execute)(struct kutf_context *context), ++ unsigned int filters); ++ ++/** ++ * kutf_add_test_with_filters_and_data() - Add a test to a kernel test suite ++ * with filters. ++ * @suite: The suite to add the test to. ++ * @id: The ID of the test. ++ * @name: The name of the test. ++ * @execute: Callback to the test function to run. ++ * @filters: A set of filtering flags, assigning test categories. ++ * @test_data: Test specific callback data, provided during the ++ * running of the test in the kutf_context ++ */ ++void kutf_add_test_with_filters_and_data( ++ struct kutf_suite *suite, ++ unsigned int id, ++ const char *name, ++ void (*execute)(struct kutf_context *context), ++ unsigned int filters, ++ union kutf_callback_data test_data); ++ ++/** ++ * kutf_add_test_with_filters_data_and_userdata() - Add a test to a kernel test suite with filters and setup for ++ * receiving data from userside ++ * @suite: The suite to add the test to. ++ * @id: The ID of the test. ++ * @name: The name of the test. ++ * @execute: Callback to the test function to run. ++ * @filters: A set of filtering flags, assigning test categories. ++ * @test_data: Test specific callback data, provided during the ++ * running of the test in the kutf_context ++ * @userdata_ops: Callbacks to use for sending and receiving data to ++ * userspace. A copy of the struct kutf_userdata_ops is ++ * taken. Each callback can be NULL. ++ * ++ */ ++void kutf_add_test_with_filters_data_and_userdata( ++ struct kutf_suite *suite, ++ unsigned int id, ++ const char *name, ++ void (*execute)(struct kutf_context *context), ++ unsigned int filters, ++ union kutf_callback_data test_data, ++ struct kutf_userdata_ops *userdata_ops); ++ ++ ++/* ============================================================================ ++ Test functions ++============================================================================ */ ++/** ++ * kutf_test_log_result_external() - Log a result which has been created ++ * externally into a in a standard form ++ * recognized by the log parser. ++ * @context: The test context the test is running in ++ * @message: The message for this result ++ * @new_status: The result status of this log message ++ */ ++void kutf_test_log_result_external( ++ struct kutf_context *context, ++ const char *message, ++ enum kutf_result_status new_status); ++ ++/** ++ * kutf_test_expect_abort() - Tell the kernel that you expect the current ++ * fixture to produce an abort. ++ * @context: The test context this test is running in. ++ */ ++void kutf_test_expect_abort(struct kutf_context *context); ++ ++/** ++ * kutf_test_expect_fatal() - Tell the kernel that you expect the current ++ * fixture to produce a fatal error. ++ * @context: The test context this test is running in. ++ */ ++void kutf_test_expect_fatal(struct kutf_context *context); ++ ++/** ++ * kutf_test_expect_fail() - Tell the kernel that you expect the current ++ * fixture to fail. ++ * @context: The test context this test is running in. ++ */ ++void kutf_test_expect_fail(struct kutf_context *context); ++ ++/** ++ * kutf_test_expect_warn() - Tell the kernel that you expect the current ++ * fixture to produce a warning. ++ * @context: The test context this test is running in. ++ */ ++void kutf_test_expect_warn(struct kutf_context *context); ++ ++/** ++ * kutf_test_expect_pass() - Tell the kernel that you expect the current ++ * fixture to pass. ++ * @context: The test context this test is running in. ++ */ ++void kutf_test_expect_pass(struct kutf_context *context); ++ ++/** ++ * kutf_test_skip() - Tell the kernel that the test should be skipped. ++ * @context: The test context this test is running in. ++ */ ++void kutf_test_skip(struct kutf_context *context); ++ ++/** ++ * kutf_test_skip_msg() - Tell the kernel that this test has been skipped, ++ * supplying a reason string. ++ * @context: The test context this test is running in. ++ * @message: A message string containing the reason for the skip. ++ * ++ * Note: The message must not be freed during the lifetime of the test run. ++ * This means it should either be a prebaked string, or if a dynamic string ++ * is required it must be created with kutf_dsprintf which will store ++ * the resultant string in a buffer who's lifetime is the same as the test run. ++ */ ++void kutf_test_skip_msg(struct kutf_context *context, const char *message); ++ ++/** ++ * kutf_test_pass() - Tell the kernel that this test has passed. ++ * @context: The test context this test is running in. ++ * @message: A message string containing the reason for the pass. ++ * ++ * Note: The message must not be freed during the lifetime of the test run. ++ * This means it should either be a pre-baked string, or if a dynamic string ++ * is required it must be created with kutf_dsprintf which will store ++ * the resultant string in a buffer who's lifetime is the same as the test run. ++ */ ++void kutf_test_pass(struct kutf_context *context, char const *message); ++ ++/** ++ * kutf_test_debug() - Send a debug message ++ * @context: The test context this test is running in. ++ * @message: A message string containing the debug information. ++ * ++ * Note: The message must not be freed during the lifetime of the test run. ++ * This means it should either be a pre-baked string, or if a dynamic string ++ * is required it must be created with kutf_dsprintf which will store ++ * the resultant string in a buffer who's lifetime is the same as the test run. ++ */ ++void kutf_test_debug(struct kutf_context *context, char const *message); ++ ++/** ++ * kutf_test_info() - Send an information message ++ * @context: The test context this test is running in. ++ * @message: A message string containing the information message. ++ * ++ * Note: The message must not be freed during the lifetime of the test run. ++ * This means it should either be a pre-baked string, or if a dynamic string ++ * is required it must be created with kutf_dsprintf which will store ++ * the resultant string in a buffer who's lifetime is the same as the test run. ++ */ ++void kutf_test_info(struct kutf_context *context, char const *message); ++ ++/** ++ * kutf_test_warn() - Send a warning message ++ * @context: The test context this test is running in. ++ * @message: A message string containing the warning message. ++ * ++ * Note: The message must not be freed during the lifetime of the test run. ++ * This means it should either be a pre-baked string, or if a dynamic string ++ * is required it must be created with kutf_dsprintf which will store ++ * the resultant string in a buffer who's lifetime is the same as the test run. ++ */ ++void kutf_test_warn(struct kutf_context *context, char const *message); ++ ++/** ++ * kutf_test_fail() - Tell the kernel that a test has failed ++ * @context: The test context this test is running in. ++ * @message: A message string containing the failure message. ++ * ++ * Note: The message must not be freed during the lifetime of the test run. ++ * This means it should either be a pre-baked string, or if a dynamic string ++ * is required it must be created with kutf_dsprintf which will store ++ * the resultant string in a buffer who's lifetime is the same as the test run. ++ */ ++void kutf_test_fail(struct kutf_context *context, char const *message); ++ ++/** ++ * kutf_test_fatal() - Tell the kernel that a test has triggered a fatal error ++ * @context: The test context this test is running in. ++ * @message: A message string containing the fatal error message. ++ * ++ * Note: The message must not be freed during the lifetime of the test run. ++ * This means it should either be a pre-baked string, or if a dynamic string ++ * is required it must be created with kutf_dsprintf which will store ++ * the resultant string in a buffer who's lifetime is the same as the test run. ++ */ ++void kutf_test_fatal(struct kutf_context *context, char const *message); ++ ++/** ++ * kutf_test_abort() - Tell the kernel that a test triggered an abort in the test ++ * ++ * @context: The test context this test is running in. ++ */ ++void kutf_test_abort(struct kutf_context *context); ++ ++#endif /* _KERNEL_UTF_SUITE_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/include/kutf/kutf_utils.h b/drivers/gpu/arm/bifrost_for_linux/tests/include/kutf/kutf_utils.h +new file mode 100755 +index 000000000000..c458c1f73802 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/tests/include/kutf/kutf_utils.h +@@ -0,0 +1,55 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KERNEL_UTF_UTILS_H_ ++#define _KERNEL_UTF_UTILS_H_ ++ ++/* kutf_utils.h ++ * Utilities for the kernel UTF test infrastructure. ++ * ++ * This collection of library functions are provided for use by kernel UTF ++ * and users of kernel UTF which don't directly fit within the other ++ * code modules. ++ */ ++ ++#include ++ ++/** ++ * Maximum size of the message strings within kernel UTF, messages longer then ++ * this will be truncated. ++ */ ++#define KUTF_MAX_DSPRINTF_LEN 1024 ++ ++/** ++ * kutf_dsprintf() - dynamic sprintf ++ * @pool: memory pool to allocate from ++ * @fmt: The format string describing the string to document. ++ * @... The parameters to feed in to the format string. ++ * ++ * This function implements sprintf which dynamically allocates memory to store ++ * the string. The library will free the memory containing the string when the ++ * result set is cleared or destroyed. ++ * ++ * Note The returned string may be truncated to fit an internal temporary ++ * buffer, which is KUTF_MAX_DSPRINTF_LEN bytes in length. ++ * ++ * Return: Returns pointer to allocated string, or NULL on error. ++ */ ++const char *kutf_dsprintf(struct kutf_mempool *pool, ++ const char *fmt, ...); ++ ++#endif /* _KERNEL_UTF_UTILS_H_ */ +diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/kutf/Kbuild b/drivers/gpu/arm/bifrost_for_linux/tests/kutf/Kbuild +new file mode 100755 +index 000000000000..97f80057224f +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/tests/kutf/Kbuild +@@ -0,0 +1,20 @@ ++# ++# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++ccflags-y += -I$(src)/../include ++ ++obj-$(CONFIG_MALI_KUTF) += kutf.o ++ ++kutf-y := kutf_mem.o kutf_resultset.o kutf_suite.o kutf_utils.o kutf_helpers.o kutf_helpers_user.o +diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/kutf/Kconfig b/drivers/gpu/arm/bifrost_for_linux/tests/kutf/Kconfig +new file mode 100755 +index 000000000000..6a87bdbf746e +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/tests/kutf/Kconfig +@@ -0,0 +1,22 @@ ++# ++# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++ ++config MALI_KUTF ++ tristate "Mali Kernel Unit Test Framework" ++ default m ++ help ++ Enables MALI testing framework. To compile it as a module, ++ choose M here - this will generate a single module called kutf. +diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/kutf/Makefile b/drivers/gpu/arm/bifrost_for_linux/tests/kutf/Makefile +new file mode 100755 +index 000000000000..010c92ca39b9 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/tests/kutf/Makefile +@@ -0,0 +1,29 @@ ++# ++# (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++# linux build system bootstrap for out-of-tree module ++ ++# default to building for the host ++ARCH ?= $(shell uname -m) ++ ++ifeq ($(KDIR),) ++$(error Must specify KDIR to point to the kernel to target)) ++endif ++ ++all: ++ $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) $(SCONS_CONFIGS) EXTRA_CFLAGS=-I$(CURDIR)/../include modules ++ ++clean: ++ $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) clean +diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/kutf/kutf_helpers.c b/drivers/gpu/arm/bifrost_for_linux/tests/kutf/kutf_helpers.c +new file mode 100755 +index 000000000000..793d58c789ff +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/tests/kutf/kutf_helpers.c +@@ -0,0 +1,768 @@ ++/* ++ * ++ * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* Kernel UTF test helpers */ ++#include ++ ++/* 10s timeout for user thread to open the 'data' file once the test is started */ ++#define USERDATA_WAIT_TIMEOUT_MS 10000 ++#include ++#include ++#include ++#include ++#include ++#include ++ ++ ++int kutf_helper_textbuf_init(struct kutf_helper_textbuf *textbuf, ++ struct kutf_mempool *mempool, int max_line_size, ++ int max_nr_lines) ++{ ++ textbuf->scratchpad = kutf_mempool_alloc(mempool, max_line_size); ++ ++ if (!textbuf->scratchpad) ++ return -ENOMEM; ++ ++ mutex_init(&textbuf->lock); ++ textbuf->nr_user_clients = 0; ++ textbuf->mempool = mempool; ++ textbuf->used_bytes = 0; ++ textbuf->prev_pos = 0; ++ textbuf->prev_line_pos = 0; ++ INIT_LIST_HEAD(&textbuf->textbuf_list); ++ textbuf->max_line_size = max_line_size; ++ textbuf->max_nr_lines = max_nr_lines; ++ textbuf->nr_lines = 0; ++ textbuf->flags = 0ul; ++ init_waitqueue_head(&textbuf->user_opened_wq); ++ init_waitqueue_head(&textbuf->not_full_wq); ++ init_waitqueue_head(&textbuf->not_empty_wq); ++ ++ return 0; ++} ++EXPORT_SYMBOL(kutf_helper_textbuf_init); ++ ++/** ++ * kutf_helper_textbuf_open() - Notify that userspace has opened the 'data' ++ * file for a textbuf ++ * ++ * @priv: private pointer from a kutf_userdata_exchange, which ++ * should be a pointer to a struct kutf_helper_textbuf ++ * ++ * Return: 0 on success, or negative value on error. ++ */ ++static int kutf_helper_textbuf_open(void *priv) ++{ ++ struct kutf_helper_textbuf *textbuf = priv; ++ int ret; ++ ++ ret = mutex_lock_interruptible(&textbuf->lock); ++ if (ret) ++ return -ERESTARTSYS; ++ ++ ++(textbuf->nr_user_clients); ++ wake_up(&textbuf->user_opened_wq); ++ ++ mutex_unlock(&textbuf->lock); ++ return ret; ++} ++ ++/** ++ * kutf_helper_textbuf_release() - Notify that userspace has closed the 'data' ++ * file for a textbuf ++ * ++ * @priv: private pointer from a kutf_userdata_exchange, which ++ * should be a pointer to a struct kutf_helper_textbuf ++ */ ++static void kutf_helper_textbuf_release(void *priv) ++{ ++ struct kutf_helper_textbuf *textbuf = priv; ++ ++ /* Shouldn't use interruptible variants here because if a signal is ++ * pending, we can't abort and restart the call */ ++ mutex_lock(&textbuf->lock); ++ ++ --(textbuf->nr_user_clients); ++ if (!textbuf->nr_user_clients) { ++ /* All clients disconnected, wakeup kernel-side waiters */ ++ wake_up(&textbuf->not_full_wq); ++ wake_up(&textbuf->not_empty_wq); ++ } ++ ++ mutex_unlock(&textbuf->lock); ++} ++ ++/** ++ * kutf_helper_textbuf_notify_test_ended() - Notify that the test has ended ++ * ++ * @priv: private pointer from a kutf_userdata_exchange, which ++ * should be a pointer to a struct kutf_helper_textbuf ++ * ++ * After this call, userspace should be allowed to finish remaining reads but ++ * not make new ones, and not be allowed to make new writes. ++ */ ++static void kutf_helper_textbuf_notify_test_ended(void *priv) ++{ ++ struct kutf_helper_textbuf *textbuf = priv; ++ ++ /* Shouldn't use interruptible variants here because if a signal is ++ * pending, we can't abort and restart the call */ ++ mutex_lock(&textbuf->lock); ++ ++ textbuf->flags |= KUTF_HELPER_TEXTBUF_FLAG_DYING; ++ ++ /* Consumers waiting due to being full should wake up and abort */ ++ wake_up(&textbuf->not_full_wq); ++ /* Producers waiting due to being empty should wake up and abort */ ++ wake_up(&textbuf->not_empty_wq); ++ ++ mutex_unlock(&textbuf->lock); ++} ++ ++/* Collect text in a textbuf scratchpad up to (but excluding) specified ++ * newline_off, and add it as a textbuf_line ++ * ++ * newline_off is permissible to be at the character after the end of the ++ * scratchpad (i.e. equal to textbuf->max_line_size), for handling when the ++ * line was longer than the size of the scratchpad. Nevertheless, the resulting ++ * size of the line is kept at textbuf->max_line_size, including the '\0' ++ * terminator. That is, the string length will be textbuf->max_line_size-1. ++ * ++ * Remaining characters strictly after newline_off are moved to the beginning ++ * of the scratchpad, to allow space for a longer line to be collected. This ++ * means the character specified at newline_off will be removed from/no longer ++ * be within the valid region of the scratchpad ++ * ++ * Returns number of bytes the scratchpad was shortened by, or an error ++ * otherwise ++ */ ++static size_t collect_line(struct kutf_helper_textbuf *textbuf, int newline_off) ++{ ++ /* '\n' terminator will be replaced as '\0' */ ++ int str_buf_size; ++ struct kutf_helper_textbuf_line *textbuf_line; ++ char *str_start; ++ int bytes_remain; ++ char *scratch = textbuf->scratchpad; ++ int nextline_off; ++ ++ str_buf_size = newline_off + 1; ++ if (str_buf_size > textbuf->max_line_size) ++ str_buf_size = textbuf->max_line_size; ++ ++ /* String is stored immediately after the line */ ++ textbuf_line = kutf_mempool_alloc(textbuf->mempool, str_buf_size + sizeof(struct kutf_helper_textbuf_line)); ++ if (!textbuf_line) ++ return -ENOMEM; ++ ++ str_start = &textbuf_line->str[0]; ++ ++ /* Copy in string, excluding the terminating '\n' character, replacing ++ * it with '\0' */ ++ strncpy(str_start, scratch, str_buf_size - 1); ++ str_start[str_buf_size-1] = '\0'; ++ textbuf_line->str_size = str_buf_size; ++ ++ /* Append to the textbuf */ ++ list_add_tail(&textbuf_line->node, &textbuf->textbuf_list); ++ ++(textbuf->nr_lines); ++ ++ /* Move the rest of the scratchpad to the start */ ++ nextline_off = newline_off + 1; ++ if (nextline_off > textbuf->used_bytes) ++ nextline_off = textbuf->used_bytes; ++ ++ bytes_remain = textbuf->used_bytes - nextline_off; ++ memmove(scratch, scratch + nextline_off, bytes_remain); ++ textbuf->used_bytes = bytes_remain; ++ ++ /* Wakeup anyone blocked on empty */ ++ wake_up(&textbuf->not_empty_wq); ++ ++ return nextline_off; ++} ++ ++/* Buffer size for truncating a string to its newline. ++ * Allocated on the stack, so keep it moderately small (within PAGE_SIZE) */ ++#define TRUNCATE_BUF_SZ 512 ++ ++/* Discard input from a userbuf up to a newline, then collect what was in the ++ * scratchpad into a new textbuf line */ ++static ssize_t collect_longline_truncate(struct kutf_helper_textbuf *textbuf, ++ const char __user *userbuf, size_t userbuf_len) ++{ ++ ssize_t bytes_processed = 0; ++ ++ while (userbuf_len > 0) { ++ int userbuf_copy_sz = userbuf_len; ++ size_t res; ++ char *newline_ptr; ++ char truncate_buf[TRUNCATE_BUF_SZ]; ++ ++ if (userbuf_len > TRUNCATE_BUF_SZ) ++ userbuf_copy_sz = TRUNCATE_BUF_SZ; ++ else ++ userbuf_copy_sz = (int)userbuf_len; ++ ++ /* copy what we can */ ++ res = copy_from_user(truncate_buf, userbuf, userbuf_copy_sz); ++ if (res == userbuf_copy_sz) ++ return -EFAULT; ++ userbuf_copy_sz -= res; ++ ++ /* Search for newline in what was copied */ ++ newline_ptr = strnchr(truncate_buf, userbuf_copy_sz, '\n'); ++ ++ if (newline_ptr) { ++ ssize_t sres; ++ /* Newline found: collect scratchpad and exit out */ ++ int newline_off = newline_ptr - truncate_buf; ++ ++ sres = collect_line(textbuf, textbuf->used_bytes); ++ if (sres < 0) ++ return sres; ++ ++ bytes_processed += newline_off + 1; ++ break; ++ } ++ ++ /* Newline not yet found: advance to the next part to copy */ ++ userbuf += userbuf_copy_sz; ++ userbuf_len -= userbuf_copy_sz; ++ bytes_processed += userbuf_copy_sz; ++ } ++ ++ return bytes_processed; ++} ++ ++/** ++ * kutf_helper_textbuf_consume() - 'data' file consumer function for writing to ++ * a textbuf ++ * @priv: private pointer from a kutf_userdata_exchange, which ++ * should be a pointer to a struct kutf_helper_textbuf to ++ * write into ++ * @userbuf: the userspace buffer to read from ++ * @userbuf_len: size of the userspace buffer ++ * @ppos: the current position in the buffer ++ * ++ * This consumer function is used as a write consumer for the 'data' file, ++ * receiving data that has been written to the 'data' file by userspace. It ++ * will read from the userspace buffer @userbuf and separates it into '\n' ++ * delimited lines for the textbuf pointed to by @priv . ++ * ++ * If there is insufficient space in textbuf, then it will block until there is ++ * space - for example, a kernel-side test calls ++ * kutf_helper_textbuf_dequeue(). Since this is expected to be called in the ++ * context of a syscall, the call can only be cancelled by sending an ++ * appropriate signal to the userspace process. ++ * ++ * The current position @ppos is advanced by the number of bytes successfully ++ * read. ++ * ++ * Return: the number of bytes read, or negative value on error. ++ */ ++static ssize_t kutf_helper_textbuf_consume(void *priv, ++ const char __user *userbuf, size_t userbuf_len, loff_t *ppos) ++{ ++ struct kutf_helper_textbuf *textbuf = priv; ++ int userbuf_copy_sz; ++ char *next_newline_ptr; ++ size_t bytes_processed = 0; ++ int newdata_off; ++ ssize_t ret; ++ ++ ret = mutex_lock_interruptible(&textbuf->lock); ++ if (ret) ++ return -ERESTARTSYS; ++ ++ /* Validate input */ ++ if (*ppos < 0) { ++ ret = -EINVAL; ++ goto out_unlock; ++ } ++ if (!userbuf_len) { ++ ret = 0; ++ goto out_unlock; ++ } ++ ++ while (textbuf->nr_lines >= textbuf->max_nr_lines && ++ !(textbuf->flags & KUTF_HELPER_TEXTBUF_FLAG_DYING)) { ++ /* Block on kernel-side dequeue making space available ++ * NOTE: should also handle O_NONBLOCK */ ++ mutex_unlock(&textbuf->lock); ++ ret = wait_event_interruptible(textbuf->not_full_wq, ++ (textbuf->nr_lines < textbuf->max_nr_lines || ++ (textbuf->flags & KUTF_HELPER_TEXTBUF_FLAG_DYING))); ++ if (ret) ++ return -ERESTARTSYS; ++ ret = mutex_lock_interruptible(&textbuf->lock); ++ if (ret) ++ return -ERESTARTSYS; ++ } ++ ++ if (textbuf->flags & KUTF_HELPER_TEXTBUF_FLAG_DYING) { ++ ret = -ENODEV; ++ goto out_unlock; ++ } ++ ++ if (textbuf->prev_pos != *ppos && textbuf->used_bytes) { ++ /* Seeking causes a new line to occur: ++ * Truncate what data was there into a textbuf-line, and reset ++ * the buffer */ ++ ret = collect_line(textbuf, textbuf->used_bytes); ++ if (ret < 0) ++ goto finish; ++ } else if (textbuf->used_bytes >= (textbuf->max_line_size - 1)) { ++ /* Line too long discard input until we find a '\n' */ ++ ret = collect_longline_truncate(textbuf, userbuf, userbuf_len); ++ ++ if (ret < 0) ++ goto finish; ++ ++ /* Update userbuf with how much was processed, which may be the ++ * entire buffer now */ ++ userbuf += ret; ++ userbuf_len -= ret; ++ bytes_processed += ret; ++ ++ /* If there's buffer remaining and we fault later (e.g. can't ++ * read or OOM) ensure ppos is updated */ ++ *ppos += ret; ++ ++ /* recheck in case entire buffer processed */ ++ if (!userbuf_len) ++ goto finish; ++ } ++ ++ /* An extra line may've been added, ensure we don't overfill */ ++ if (textbuf->nr_lines >= textbuf->max_nr_lines) ++ goto finish_noerr; ++ ++ userbuf_copy_sz = userbuf_len; ++ ++ /* Copy in as much as we can */ ++ if (userbuf_copy_sz > textbuf->max_line_size - textbuf->used_bytes) ++ userbuf_copy_sz = textbuf->max_line_size - textbuf->used_bytes; ++ ++ ret = copy_from_user(textbuf->scratchpad + textbuf->used_bytes, userbuf, userbuf_copy_sz); ++ if (ret == userbuf_copy_sz) { ++ ret = -EFAULT; ++ goto finish; ++ } ++ userbuf_copy_sz -= ret; ++ ++ newdata_off = textbuf->used_bytes; ++ textbuf->used_bytes += userbuf_copy_sz; ++ ++ while (textbuf->used_bytes && textbuf->nr_lines < textbuf->max_nr_lines) { ++ int new_bytes_remain = textbuf->used_bytes - newdata_off; ++ /* Find a new line - only the new part should be checked */ ++ next_newline_ptr = strnchr(textbuf->scratchpad + newdata_off, new_bytes_remain, '\n'); ++ ++ if (next_newline_ptr) { ++ int newline_off = next_newline_ptr - textbuf->scratchpad; ++ ++ /* if found, collect up to it, then memmove the rest */ ++ /* reset positions and see if we can fill any further */ ++ /* repeat until run out of data or line is filled */ ++ ret = collect_line(textbuf, newline_off); ++ ++ /* If filled up or OOM, rollback the remaining new ++ * data. Instead we'll try to grab it next time we're ++ * called */ ++ if (textbuf->nr_lines >= textbuf->max_nr_lines || ret < 0) ++ textbuf->used_bytes = newdata_off; ++ ++ if (ret < 0) ++ goto finish; ++ ++ /* Fix up ppos etc in case we'll be ending the loop */ ++ *ppos += ret - newdata_off; ++ bytes_processed += ret - newdata_off; ++ newdata_off = 0; ++ } else { ++ /* there's bytes left, but no new-line, so try to fill up next time */ ++ *ppos += new_bytes_remain; ++ bytes_processed += new_bytes_remain; ++ break; ++ } ++ } ++ ++finish_noerr: ++ ret = bytes_processed; ++finish: ++ textbuf->prev_pos = *ppos; ++out_unlock: ++ mutex_unlock(&textbuf->lock); ++ ++ return ret; ++} ++ ++/** ++ * kutf_helper_textbuf_produce() - 'data' file producer function for reading ++ * from a textbuf ++ * @priv: private pointer from a kutf_userdata_exchange, which ++ * should be a pointer to a struct kutf_helper_textbuf to ++ * read from ++ * @userbuf: the userspace buffer to write to ++ * @userbuf_len: size of the userspace buffer ++ * @ppos: the current position in the buffer ++ * ++ * This producer function is used as a read producer for the 'data' file, ++ * allowing userspace to read from the 'data' file. It will write to the ++ * userspace buffer @userbuf, taking lines from the textbuf pointed to by ++ * @priv, separating each line with '\n'. ++ * ++ * If there is no data in the textbuf, then it will block until some appears - ++ * for example, a kernel-side test calls kutf_helper_textbuf_enqueue(). Since ++ * this is expected to be called in the context of a syscall, the call can only ++ * be cancelled by sending an appropriate signal to the userspace process. ++ * ++ * The current position @ppos is advanced by the number of bytes successfully ++ * written. ++ * ++ * Return: the number of bytes written, or negative value on error ++ */ ++static ssize_t kutf_helper_textbuf_produce(void *priv, char __user *userbuf, ++ size_t userbuf_len, loff_t *ppos) ++{ ++ struct kutf_helper_textbuf *textbuf = priv; ++ loff_t pos_offset; ++ struct kutf_helper_textbuf_line *line = NULL; ++ int line_start_pos; ++ size_t bytes_processed = 0; ++ ssize_t ret; ++ int copy_length; ++ ++ ret = mutex_lock_interruptible(&textbuf->lock); ++ if (ret) ++ return -ERESTARTSYS; ++ ++ /* Validate input */ ++ if (*ppos < 0) { ++ ret = -EINVAL; ++ goto finish; ++ } ++ if (!userbuf_len) { ++ ret = 0; ++ goto finish; ++ } ++ ++ /* Seeking to before the beginning of the line will have the effect of ++ * resetting the position to the start of the current data, since we've ++ * already discarded previous data */ ++ if (*ppos < textbuf->prev_line_pos) ++ textbuf->prev_line_pos = *ppos; ++ ++ while (!line) { ++ int needs_wake = 0; ++ ++ pos_offset = *ppos - textbuf->prev_line_pos; ++ line_start_pos = 0; ++ ++ /* Find the line for the offset, emptying the textbuf as we go */ ++ while (!list_empty(&textbuf->textbuf_list)) { ++ int line_end_pos; ++ ++ line = list_first_entry(&textbuf->textbuf_list, struct kutf_helper_textbuf_line, node); ++ ++ /* str_size used in line_end_pos because lines implicitly have ++ * a '\n', but we count the '\0' string terminator as that */ ++ line_end_pos = line_start_pos + line->str_size; ++ ++ if (pos_offset < line_end_pos) ++ break; ++ ++ line_start_pos += line->str_size; ++ /* Only discard a line when we're sure it's finished ++ * with, to avoid awkward rollback conditions if we've ++ * had to block */ ++ list_del(&line->node); ++ --(textbuf->nr_lines); ++ line = NULL; ++ needs_wake = 1; ++ } ++ ++ /* Update the start of the line pos for next time we're called */ ++ textbuf->prev_line_pos += line_start_pos; ++ ++ /* If space was freed up, wake waiters */ ++ if (needs_wake) ++ wake_up(&textbuf->not_full_wq); ++; ++ if (!line) { ++ /* Only check before waiting, to ensure if the test ++ * does the last enqueue and immediately finishes, then ++ * we'll go back round the loop to receive the line ++ * instead of just dying straight away */ ++ if (textbuf->flags & KUTF_HELPER_TEXTBUF_FLAG_DYING) { ++ /* Indicate EOF rather than an error */ ++ ret = 0; ++ goto finish; ++ } ++ ++ /* No lines found, block for new ones ++ * NOTE: should also handle O_NONBLOCK */ ++ mutex_unlock(&textbuf->lock); ++ ret = wait_event_interruptible(textbuf->not_empty_wq, ++ (textbuf->nr_lines > 0 || ++ (textbuf->flags & KUTF_HELPER_TEXTBUF_FLAG_DYING))); ++ ++ /* signals here are not restartable */ ++ if (ret) ++ return ret; ++ ret = mutex_lock_interruptible(&textbuf->lock); ++ if (ret) ++ return ret; ++ } ++ ++ } ++ ++ ++ /* Find offset within the line, guaranteed to be within line->str_size */ ++ pos_offset -= line_start_pos; ++ ++ while (userbuf_len && line) { ++ /* Copy at most to the end of string, excluding terminator */ ++ copy_length = line->str_size - 1 - pos_offset; ++ if (copy_length > userbuf_len) ++ copy_length = userbuf_len; ++ ++ if (copy_length) { ++ ret = copy_to_user(userbuf, &line->str[pos_offset], copy_length); ++ if (ret == copy_length) { ++ ret = -EFAULT; ++ goto finish; ++ } ++ copy_length -= ret; ++ ++ userbuf += copy_length; ++ userbuf_len -= copy_length; ++ bytes_processed += copy_length; ++ *ppos += copy_length; ++ if (ret) ++ goto finish_noerr; ++ } ++ ++ /* Add terminator if one was needed */ ++ if (userbuf_len) { ++ copy_length = 1; ++ ret = copy_to_user(userbuf, "\n", copy_length); ++ if (ret == copy_length) { ++ ret = -EFAULT; ++ goto finish; ++ } ++ copy_length -= ret; ++ ++ userbuf += copy_length; ++ userbuf_len -= copy_length; ++ bytes_processed += copy_length; ++ *ppos += copy_length; ++ } else { ++ /* string wasn't completely copied this time - try to ++ * finish it next call */ ++ break; ++ } ++ ++ /* Line Completed - only now can safely delete it */ ++ textbuf->prev_line_pos += line->str_size; ++ list_del(&line->node); ++ --(textbuf->nr_lines); ++ line = NULL; ++ /* Space freed up, wake up waiters */ ++ wake_up(&textbuf->not_full_wq); ++ ++ /* Pick the next line */ ++ if (!list_empty(&textbuf->textbuf_list)) { ++ line = list_first_entry(&textbuf->textbuf_list, struct kutf_helper_textbuf_line, node); ++ pos_offset = 0; ++ } ++ /* if no more lines, we've copied at least some bytes, so only ++ * need to block on new lines the next time we're called */ ++ } ++ ++finish_noerr: ++ ret = bytes_processed; ++finish: ++ mutex_unlock(&textbuf->lock); ++ ++ return ret; ++} ++ ++int kutf_helper_textbuf_wait_for_user(struct kutf_helper_textbuf *textbuf) ++{ ++ int err; ++ unsigned long now; ++ unsigned long timeout_jiffies = msecs_to_jiffies(USERDATA_WAIT_TIMEOUT_MS); ++ unsigned long time_end; ++ int ret = 0; ++ ++ /* Mutex locking using non-interruptible variants, since a signal to ++ * the user process will generally have to wait until we finish the ++ * test, because we can't restart the test. The exception is where ++ * we're blocked on a waitq */ ++ mutex_lock(&textbuf->lock); ++ ++ now = jiffies; ++ time_end = now + timeout_jiffies; ++ ++ while (!textbuf->nr_user_clients && time_before_eq(now, time_end)) { ++ unsigned long time_to_wait = time_end - now; ++ /* No users yet, block or timeout */ ++ mutex_unlock(&textbuf->lock); ++ /* Use interruptible here - in case we block for a long time ++ * and want to kill the user process */ ++ err = wait_event_interruptible_timeout(textbuf->user_opened_wq, ++ (textbuf->nr_user_clients > 0), time_to_wait); ++ /* Any error is not restartable due to how kutf runs tests */ ++ if (err < 0) ++ return -EINTR; ++ mutex_lock(&textbuf->lock); ++ ++ now = jiffies; ++ } ++ if (!textbuf->nr_user_clients) ++ ret = -ETIMEDOUT; ++ ++ mutex_unlock(&textbuf->lock); ++ ++ return ret; ++} ++EXPORT_SYMBOL(kutf_helper_textbuf_wait_for_user); ++ ++char *kutf_helper_textbuf_dequeue(struct kutf_helper_textbuf *textbuf, ++ int *str_size) ++{ ++ struct kutf_helper_textbuf_line *line; ++ char *ret = NULL; ++ ++ /* Mutex locking using non-interruptible variants, since a signal to ++ * the user process will generally have to wait until we finish the ++ * test, because we can't restart the test. The exception is where ++ * we're blocked on a waitq */ ++ mutex_lock(&textbuf->lock); ++ ++ while (list_empty(&textbuf->textbuf_list)) { ++ int err; ++ ++ if (!textbuf->nr_user_clients) { ++ /* No user-side clients - error */ ++ goto out; ++ } ++ ++ /* No lines found, block for new ones from user-side consumer */ ++ mutex_unlock(&textbuf->lock); ++ /* Use interruptible here - in case we block for a long time ++ * and want to kill the user process */ ++ err = wait_event_interruptible(textbuf->not_empty_wq, ++ (textbuf->nr_lines > 0 || !textbuf->nr_user_clients)); ++ /* Any error is not restartable due to how kutf runs tests */ ++ if (err) ++ return ERR_PTR(-EINTR); ++ mutex_lock(&textbuf->lock); ++ } ++ ++ line = list_first_entry(&textbuf->textbuf_list, struct kutf_helper_textbuf_line, node); ++ list_del(&line->node); ++ --(textbuf->nr_lines); ++ /* Space freed up, wake up waiters */ ++ wake_up(&textbuf->not_full_wq); ++ ++ if (str_size) ++ *str_size = line->str_size; ++ ++ ret = &line->str[0]; ++ ++out: ++ mutex_unlock(&textbuf->lock); ++ return ret; ++} ++EXPORT_SYMBOL(kutf_helper_textbuf_dequeue); ++ ++int kutf_helper_textbuf_enqueue(struct kutf_helper_textbuf *textbuf, ++ char *enqueue_str, int buf_max_size) ++{ ++ struct kutf_helper_textbuf_line *textbuf_line; ++ int str_size = strnlen(enqueue_str, buf_max_size) + 1; ++ char *str_start; ++ int ret = 0; ++ ++ /* Mutex locking using non-interruptible variants, since a signal to ++ * the user process will generally have to wait until we finish the ++ * test, because we can't restart the test. The exception is where ++ * we're blocked on a waitq */ ++ mutex_lock(&textbuf->lock); ++ ++ if (str_size > textbuf->max_line_size) ++ str_size = textbuf->max_line_size; ++ ++ while (textbuf->nr_lines >= textbuf->max_nr_lines) { ++ if (!textbuf->nr_user_clients) { ++ /* No user-side clients - error */ ++ ret = -EBUSY; ++ goto out; ++ } ++ ++ /* Block on user-side producer making space available */ ++ mutex_unlock(&textbuf->lock); ++ /* Use interruptible here - in case we block for a long time ++ * and want to kill the user process */ ++ ret = wait_event_interruptible(textbuf->not_full_wq, ++ (textbuf->nr_lines < textbuf->max_nr_lines || !textbuf->nr_user_clients)); ++ /* Any error is not restartable due to how kutf runs tests */ ++ if (ret) ++ return -EINTR; ++ mutex_lock(&textbuf->lock); ++ } ++ ++ /* String is stored immediately after the line */ ++ textbuf_line = kutf_mempool_alloc(textbuf->mempool, str_size + sizeof(struct kutf_helper_textbuf_line)); ++ if (!textbuf_line) { ++ ret = -ENOMEM; ++ goto out; ++ } ++ ++ str_start = &textbuf_line->str[0]; ++ ++ /* Copy in string */ ++ strncpy(str_start, enqueue_str, str_size); ++ /* Enforce the '\0' termination */ ++ str_start[str_size-1] = '\0'; ++ textbuf_line->str_size = str_size; ++ ++ /* Append to the textbuf */ ++ list_add_tail(&textbuf_line->node, &textbuf->textbuf_list); ++ ++(textbuf->nr_lines); ++ ++ /* Wakeup anyone blocked on empty */ ++ wake_up(&textbuf->not_empty_wq); ++ ++out: ++ mutex_unlock(&textbuf->lock); ++ return ret; ++} ++EXPORT_SYMBOL(kutf_helper_textbuf_enqueue); ++ ++ ++struct kutf_userdata_ops kutf_helper_textbuf_userdata_ops = { ++ .open = kutf_helper_textbuf_open, ++ .release = kutf_helper_textbuf_release, ++ .notify_ended = kutf_helper_textbuf_notify_test_ended, ++ .consumer = kutf_helper_textbuf_consume, ++ .producer = kutf_helper_textbuf_produce, ++}; ++EXPORT_SYMBOL(kutf_helper_textbuf_userdata_ops); +diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/kutf/kutf_helpers_user.c b/drivers/gpu/arm/bifrost_for_linux/tests/kutf/kutf_helpers_user.c +new file mode 100755 +index 000000000000..cf3b00563c5f +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/tests/kutf/kutf_helpers_user.c +@@ -0,0 +1,460 @@ ++/* ++ * ++ * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* Kernel UTF test helpers that mirror those for kutf-userside */ ++#include ++#include ++ ++#include ++#include ++ ++const char *valtype_names[] = { ++ "INVALID", ++ "U64", ++ "STR", ++}; ++ ++static const char *get_val_type_name(enum kutf_helper_valtype valtype) ++{ ++ /* enums can be signed or unsigned (implementation dependant), so ++ * enforce it to prevent: ++ * a) "<0 comparison on unsigned type" warning - if we did both upper ++ * and lower bound check ++ * b) incorrect range checking if it was a signed type - if we did ++ * upper bound check only */ ++ unsigned int type_idx = (unsigned int)valtype; ++ ++ if (type_idx >= (unsigned int)KUTF_HELPER_VALTYPE_COUNT) ++ type_idx = (unsigned int)KUTF_HELPER_VALTYPE_INVALID; ++ ++ return valtype_names[type_idx]; ++} ++ ++/* Check up to str_len chars of val_str to see if it's a valid value name: ++ * ++ * - Has between 1 and KUTF_HELPER_MAX_VAL_NAME_LEN characters before the \0 terminator ++ * - And, each char is in the character set [A-Z0-9_] */ ++static int validate_val_name(char *val_str, int str_len) ++{ ++ int i = 0; ++ ++ for (i = 0; str_len && i <= KUTF_HELPER_MAX_VAL_NAME_LEN && val_str[i] != '\0'; ++i, --str_len) { ++ char val_chr = val_str[i]; ++ ++ if (val_chr >= 'A' && val_chr <= 'Z') ++ continue; ++ if (val_chr >= '0' && val_chr <= '9') ++ continue; ++ if (val_chr == '_') ++ continue; ++ ++ /* Character not in the set [A-Z0-9_] - report error */ ++ return 1; ++ } ++ ++ /* Names of 0 length are not valid */ ++ if (i == 0) ++ return 1; ++ /* Length greater than KUTF_HELPER_MAX_VAL_NAME_LEN not allowed */ ++ if (i > KUTF_HELPER_MAX_VAL_NAME_LEN || (i == KUTF_HELPER_MAX_VAL_NAME_LEN && val_str[i] != '\0')) ++ return 1; ++ ++ return 0; ++} ++ ++/* Find the length of the valid part of the string when it will be in quotes ++ * e.g. "str" ++ * ++ * That is, before any '\\', '\n' or '"' characters. This is so we don't have ++ * to escape the string */ ++static int find_quoted_string_valid_len(char *str) ++{ ++ char *ptr; ++ const char *check_chars = "\\\n\""; ++ ++ ptr = strpbrk(str, check_chars); ++ if (ptr) ++ return ptr-str; ++ ++ return strlen(str); ++} ++ ++#define MAX_U64_HEX_LEN 16 ++/* (Name size) + ("=0x" size) + (64-bit hex value size) + (terminator) */ ++#define NAMED_U64_VAL_BUF_SZ (KUTF_HELPER_MAX_VAL_NAME_LEN + 3 + MAX_U64_HEX_LEN + 1) ++ ++int kutf_helper_textbuf_send_named_u64(struct kutf_context *context, ++ struct kutf_helper_textbuf *textbuf, char *val_name, u64 val) ++{ ++ int ret = 1; ++ char msgbuf[NAMED_U64_VAL_BUF_SZ]; ++ const char *errmsg = NULL; ++ ++ if (validate_val_name(val_name, KUTF_HELPER_MAX_VAL_NAME_LEN + 1)) { ++ errmsg = kutf_dsprintf(&context->fixture_pool, ++ "Failed to send u64 value named '%s': Invalid value name", val_name); ++ goto out_err; ++ } ++ ++ ret = snprintf(msgbuf, NAMED_U64_VAL_BUF_SZ, "%s=0x%llx", val_name, val); ++ if (ret >= NAMED_U64_VAL_BUF_SZ || ret < 0) { ++ errmsg = kutf_dsprintf(&context->fixture_pool, ++ "Failed to send u64 value named '%s': snprintf() problem buffer size==%d ret=%d", ++ val_name, NAMED_U64_VAL_BUF_SZ, ret); ++ goto out_err; ++ } ++ msgbuf[NAMED_U64_VAL_BUF_SZ-1] = '\0'; ++ ++ ret = kutf_helper_textbuf_enqueue(textbuf, msgbuf, NAMED_U64_VAL_BUF_SZ); ++ if (ret) { ++ errmsg = kutf_dsprintf(&context->fixture_pool, ++ "Failed to send u64 value named '%s': send returned %d", ++ val_name, ret); ++ goto out_err; ++ } ++ ++ return ret; ++out_err: ++ kutf_test_fail(context, errmsg); ++ return ret; ++} ++EXPORT_SYMBOL(kutf_helper_textbuf_send_named_u64); ++ ++#define NAMED_VALUE_SEP "=" ++#define NAMED_STR_START_DELIM NAMED_VALUE_SEP "\"" ++#define NAMED_STR_END_DELIM "\"" ++ ++int kutf_helper_textbuf_max_str_len_for_kern(char *val_name, ++ int kern_buf_sz) ++{ ++ int val_name_len = strlen(val_name); ++ int start_delim_len = strlen(NAMED_STR_START_DELIM); ++ int max_msg_len = kern_buf_sz - 1; ++ int max_str_len; ++ ++ /* We do not include the end delimiter. Providing there is a line ++ * ending character when sending the message, the end delimiter can be ++ * truncated off safely to allow proper NAME="value" reception when ++ * value's length is too long */ ++ max_str_len = max_msg_len - val_name_len - start_delim_len; ++ ++ return max_str_len; ++} ++EXPORT_SYMBOL(kutf_helper_textbuf_max_str_len_for_kern); ++ ++int kutf_helper_textbuf_send_named_str(struct kutf_context *context, ++ struct kutf_helper_textbuf *textbuf, char *val_name, ++ char *val_str) ++{ ++ int val_str_len; ++ int str_buf_sz; ++ char *str_buf = NULL; ++ int ret = 1; ++ char *copy_ptr; ++ int val_name_len; ++ int start_delim_len = strlen(NAMED_STR_START_DELIM); ++ int end_delim_len = strlen(NAMED_STR_END_DELIM); ++ const char *errmsg = NULL; ++ ++ if (validate_val_name(val_name, KUTF_HELPER_MAX_VAL_NAME_LEN + 1)) { ++ errmsg = kutf_dsprintf(&context->fixture_pool, ++ "Failed to send u64 value named '%s': Invalid value name", val_name); ++ goto out_err; ++ } ++ val_name_len = strlen(val_name); ++ ++ val_str_len = find_quoted_string_valid_len(val_str); ++ ++ /* (name length) + ("=\"" length) + (val_str len) + ("\"" length) + terminator */ ++ str_buf_sz = val_name_len + start_delim_len + val_str_len + end_delim_len + 1; ++ ++ /* Using kmalloc() here instead of mempool since we know we need to free ++ * before we return */ ++ str_buf = kmalloc(str_buf_sz, GFP_KERNEL); ++ if (!str_buf) { ++ errmsg = kutf_dsprintf(&context->fixture_pool, ++ "Failed to send str value named '%s': kmalloc failed, str_buf_sz=%d", ++ val_name, str_buf_sz); ++ goto out_err; ++ } ++ copy_ptr = str_buf; ++ ++ /* Manually copy each string component instead of snprintf because ++ * val_str may need to end early, and less error path handling */ ++ ++ /* name */ ++ memcpy(copy_ptr, val_name, val_name_len); ++ copy_ptr += val_name_len; ++ ++ /* str start delimiter */ ++ memcpy(copy_ptr, NAMED_STR_START_DELIM, start_delim_len); ++ copy_ptr += start_delim_len; ++ ++ /* str value */ ++ memcpy(copy_ptr, val_str, val_str_len); ++ copy_ptr += val_str_len; ++ ++ /* str end delimiter */ ++ memcpy(copy_ptr, NAMED_STR_END_DELIM, end_delim_len); ++ copy_ptr += end_delim_len; ++ ++ /* Terminator */ ++ *copy_ptr = '\0'; ++ ++ ret = kutf_helper_textbuf_enqueue(textbuf, str_buf, str_buf_sz); ++ ++ if (ret) { ++ errmsg = kutf_dsprintf(&context->fixture_pool, ++ "Failed to send str value named '%s': send returned %d", ++ val_name, ret); ++ goto out_err; ++ } ++ ++ kfree(str_buf); ++ return ret; ++ ++out_err: ++ kutf_test_fail(context, errmsg); ++ kfree(str_buf); ++ return ret; ++} ++EXPORT_SYMBOL(kutf_helper_textbuf_send_named_str); ++ ++int kutf_helper_textbuf_receive_named_val(struct kutf_helper_named_val *named_val, ++ struct kutf_helper_textbuf *textbuf) ++{ ++ int recv_sz; ++ char *recv_str; ++ char *search_ptr; ++ char *name_str = NULL; ++ int name_len; ++ int strval_len; ++ enum kutf_helper_valtype type = KUTF_HELPER_VALTYPE_INVALID; ++ char *strval = NULL; ++ u64 u64val = 0; ++ int orig_recv_sz; ++ int err = KUTF_HELPER_ERR_INVALID_VALUE; ++ ++ recv_str = kutf_helper_textbuf_dequeue(textbuf, &recv_sz); ++ if (!recv_str) ++ return -EBUSY; ++ else if (IS_ERR(recv_str)) ++ return PTR_ERR(recv_str); ++ orig_recv_sz = recv_sz; ++ ++ /* Find the '=', grab the name and validate it */ ++ search_ptr = strnchr(recv_str, recv_sz, NAMED_VALUE_SEP[0]); ++ if (search_ptr) { ++ name_len = search_ptr - recv_str; ++ if (!validate_val_name(recv_str, name_len)) { ++ /* no need to reallocate - just modify string in place */ ++ name_str = recv_str; ++ name_str[name_len] = '\0'; ++ ++ /* Move until after the '=' */ ++ recv_str += (name_len + 1); ++ recv_sz -= (name_len + 1); ++ } ++ } ++ if (!name_str) { ++ pr_err("Invalid name part for recevied string '%s'\n", recv_str); ++ return KUTF_HELPER_ERR_INVALID_NAME; ++ } ++ ++ /* detect value type */ ++ if (*recv_str == NAMED_STR_START_DELIM[1]) { ++ /* string delimiter start*/ ++ ++recv_str; ++ --recv_sz; ++ ++ /* Find end of string */ ++ search_ptr = strnchr(recv_str, recv_sz, NAMED_STR_END_DELIM[0]); ++ if (search_ptr) { ++ strval_len = search_ptr - recv_str; ++ /* Validate the string to ensure it contains no quotes */ ++ if (strval_len == find_quoted_string_valid_len(recv_str)) { ++ /* no need to reallocate - just modify string in place */ ++ strval = recv_str; ++ strval[strval_len] = '\0'; ++ ++ /* Move until after the end delimiter */ ++ recv_str += (strval_len + 1); ++ recv_sz -= (strval_len + 1); ++ type = KUTF_HELPER_VALTYPE_STR; ++ } else { ++ pr_err("String value contains invalid characters in rest of received string '%s'\n", recv_str); ++ err = KUTF_HELPER_ERR_CHARS_AFTER_VAL; ++ } ++ } else if (orig_recv_sz == textbuf->max_line_size) { ++ /* No end-delimiter found, but the line is at ++ * the max line size. Assume that before ++ * truncation the line had a closing delimiter ++ * anyway */ ++ strval_len = strlen(recv_str); ++ /* Validate the string to ensure it contains no quotes */ ++ if (strval_len == find_quoted_string_valid_len(recv_str)) { ++ strval = recv_str; ++ ++ /* Move to the end of the string */ ++ recv_str += strval_len; ++ recv_sz -= strval_len; ++ type = KUTF_HELPER_VALTYPE_STR; ++ } else { ++ pr_err("String value contains invalid characters in rest of received string '%s'\n", recv_str); ++ err = KUTF_HELPER_ERR_CHARS_AFTER_VAL; ++ } ++ } else { ++ pr_err("End of string delimiter not found in rest of received string '%s'\n", recv_str); ++ err = KUTF_HELPER_ERR_NO_END_DELIMITER; ++ } ++ } else { ++ /* possibly a number value - strtoull will parse it */ ++ err = kstrtoull(recv_str, 0, &u64val); ++ /* unlike userspace can't get an end ptr, but if kstrtoull() ++ * reads characters after the number it'll report -EINVAL */ ++ if (!err) { ++ int len_remain = strnlen(recv_str, recv_sz); ++ ++ type = KUTF_HELPER_VALTYPE_U64; ++ recv_str += len_remain; ++ recv_sz -= len_remain; ++ } else { ++ /* special case: not a number, report as such */ ++ pr_err("Rest of received string was not a numeric value or quoted string value: '%s'\n", recv_str); ++ } ++ } ++ ++ if (type == KUTF_HELPER_VALTYPE_INVALID) ++ return err; ++ ++ /* Any remaining characters - error */ ++ if (strnlen(recv_str, recv_sz) != 0) { ++ pr_err("Characters remain after value of type %s: '%s'\n", ++ get_val_type_name(type), recv_str); ++ return KUTF_HELPER_ERR_CHARS_AFTER_VAL; ++ } ++ ++ /* Success - write into the output structure */ ++ switch (type) { ++ case KUTF_HELPER_VALTYPE_U64: ++ named_val->u.val_u64 = u64val; ++ break; ++ case KUTF_HELPER_VALTYPE_STR: ++ named_val->u.val_str = strval; ++ break; ++ default: ++ pr_err("Unreachable, fix textbuf_receive_named_val\n"); ++ /* Coding error, report as though 'data' file failed */ ++ return -EINVAL; ++ } ++ ++ named_val->val_name = name_str; ++ named_val->type = type; ++ ++ return KUTF_HELPER_ERR_NONE; ++} ++EXPORT_SYMBOL(kutf_helper_textbuf_receive_named_val); ++ ++#define DUMMY_MSG "" ++int kutf_helper_textbuf_receive_check_val(struct kutf_helper_named_val *named_val, ++ struct kutf_context *context, struct kutf_helper_textbuf *textbuf, ++ char *expect_val_name, enum kutf_helper_valtype expect_val_type) ++{ ++ int err; ++ ++ err = kutf_helper_textbuf_receive_named_val(named_val, textbuf); ++ if (err < 0) { ++ const char *msg = kutf_dsprintf(&context->fixture_pool, ++ "Failed to receive value named '%s'", ++ expect_val_name); ++ kutf_test_fail(context, msg); ++ return err; ++ } else if (err > 0) { ++ const char *msg = kutf_dsprintf(&context->fixture_pool, ++ "Named-value parse error when expecting value named '%s'", ++ expect_val_name); ++ kutf_test_fail(context, msg); ++ goto out_fail_and_fixup; ++ } ++ ++ if (strcmp(named_val->val_name, expect_val_name) != 0) { ++ const char *msg = kutf_dsprintf(&context->fixture_pool, ++ "Expecting to receive value named '%s' but got '%s'", ++ expect_val_name, named_val->val_name); ++ kutf_test_fail(context, msg); ++ goto out_fail_and_fixup; ++ } ++ ++ ++ if (named_val->type != expect_val_type) { ++ const char *msg = kutf_dsprintf(&context->fixture_pool, ++ "Expecting value named '%s' to be of type %s but got %s", ++ expect_val_name, get_val_type_name(expect_val_type), ++ get_val_type_name(named_val->type)); ++ kutf_test_fail(context, msg); ++ goto out_fail_and_fixup; ++ } ++ ++ return err; ++ ++out_fail_and_fixup: ++ /* Produce a valid but incorrect value */ ++ switch (expect_val_type) { ++ case KUTF_HELPER_VALTYPE_U64: ++ named_val->u.val_u64 = 0ull; ++ break; ++ case KUTF_HELPER_VALTYPE_STR: ++ { ++ char *str = kutf_mempool_alloc(&context->fixture_pool, sizeof(DUMMY_MSG)); ++ ++ if (!str) ++ return -1; ++ ++ strcpy(str, DUMMY_MSG); ++ named_val->u.val_str = str; ++ break; ++ } ++ default: ++ break; ++ } ++ ++ /* Indicate that this is invalid */ ++ named_val->type = KUTF_HELPER_VALTYPE_INVALID; ++ ++ /* But at least allow the caller to continue in the test with failures */ ++ return 0; ++} ++EXPORT_SYMBOL(kutf_helper_textbuf_receive_check_val); ++ ++void kutf_helper_output_named_val(struct kutf_helper_named_val *named_val) ++{ ++ switch (named_val->type) { ++ case KUTF_HELPER_VALTYPE_U64: ++ pr_warn("%s=0x%llx\n", named_val->val_name, named_val->u.val_u64); ++ break; ++ case KUTF_HELPER_VALTYPE_STR: ++ pr_warn("%s=\"%s\"\n", named_val->val_name, named_val->u.val_str); ++ break; ++ case KUTF_HELPER_VALTYPE_INVALID: ++ pr_warn("%s is invalid\n", named_val->val_name); ++ break; ++ default: ++ pr_warn("%s has unknown type %d\n", named_val->val_name, named_val->type); ++ break; ++ } ++} ++EXPORT_SYMBOL(kutf_helper_output_named_val); +diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/kutf/kutf_mem.c b/drivers/gpu/arm/bifrost_for_linux/tests/kutf/kutf_mem.c +new file mode 100755 +index 000000000000..a75e15fde05f +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/tests/kutf/kutf_mem.c +@@ -0,0 +1,102 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* Kernel UTF memory management functions */ ++ ++#include ++#include ++ ++#include ++ ++ ++/** ++ * struct kutf_alloc_entry - Structure representing an allocation. ++ * @node: List node for use with kutf_mempool. ++ * @data: Data area of the allocation ++ */ ++struct kutf_alloc_entry { ++ struct list_head node; ++ u8 data[0]; ++}; ++ ++int kutf_mempool_init(struct kutf_mempool *pool) ++{ ++ if (!pool) { ++ pr_err("NULL pointer passed to %s\n", __func__); ++ return -1; ++ } ++ ++ INIT_LIST_HEAD(&pool->head); ++ mutex_init(&pool->lock); ++ ++ return 0; ++} ++EXPORT_SYMBOL(kutf_mempool_init); ++ ++void kutf_mempool_destroy(struct kutf_mempool *pool) ++{ ++ struct list_head *remove; ++ struct list_head *tmp; ++ ++ if (!pool) { ++ pr_err("NULL pointer passed to %s\n", __func__); ++ return; ++ } ++ ++ mutex_lock(&pool->lock); ++ list_for_each_safe(remove, tmp, &pool->head) { ++ struct kutf_alloc_entry *remove_alloc; ++ ++ remove_alloc = list_entry(remove, struct kutf_alloc_entry, node); ++ list_del(&remove_alloc->node); ++ kfree(remove_alloc); ++ } ++ mutex_unlock(&pool->lock); ++ ++} ++EXPORT_SYMBOL(kutf_mempool_destroy); ++ ++void *kutf_mempool_alloc(struct kutf_mempool *pool, size_t size) ++{ ++ struct kutf_alloc_entry *ret; ++ ++ if (!pool) { ++ pr_err("NULL pointer passed to %s\n", __func__); ++ goto fail_pool; ++ } ++ ++ mutex_lock(&pool->lock); ++ ++ ret = kmalloc(sizeof(*ret) + size, GFP_KERNEL); ++ if (!ret) { ++ pr_err("Failed to allocate memory\n"); ++ goto fail_alloc; ++ } ++ ++ INIT_LIST_HEAD(&ret->node); ++ list_add(&ret->node, &pool->head); ++ ++ mutex_unlock(&pool->lock); ++ ++ return &ret->data[0]; ++ ++fail_alloc: ++ mutex_unlock(&pool->lock); ++fail_pool: ++ return NULL; ++} ++EXPORT_SYMBOL(kutf_mempool_alloc); +diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/kutf/kutf_resultset.c b/drivers/gpu/arm/bifrost_for_linux/tests/kutf/kutf_resultset.c +new file mode 100755 +index 000000000000..5bd04969fd55 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/tests/kutf/kutf_resultset.c +@@ -0,0 +1,95 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* Kernel UTF result management functions */ ++ ++#include ++#include ++#include ++ ++#include ++ ++/** ++ * struct kutf_result_set - Represents a set of results. ++ * @results: Pointer to the linked list where the results are stored. ++ */ ++struct kutf_result_set { ++ struct list_head results; ++}; ++ ++struct kutf_result_set *kutf_create_result_set(void) ++{ ++ struct kutf_result_set *set; ++ ++ set = kmalloc(sizeof(*set), GFP_KERNEL); ++ if (!set) { ++ pr_err("Failed to allocate resultset"); ++ goto fail_alloc; ++ } ++ ++ INIT_LIST_HEAD(&set->results); ++ ++ return set; ++ ++fail_alloc: ++ return NULL; ++} ++ ++void kutf_add_result(struct kutf_mempool *mempool, ++ struct kutf_result_set *set, ++ enum kutf_result_status status, ++ const char *message) ++{ ++ /* Create the new result */ ++ struct kutf_result *new_result; ++ ++ BUG_ON(set == NULL); ++ ++ new_result = kutf_mempool_alloc(mempool, sizeof(*new_result)); ++ if (!new_result) { ++ pr_err("Result allocation failed\n"); ++ return; ++ } ++ ++ INIT_LIST_HEAD(&new_result->node); ++ new_result->status = status; ++ new_result->message = message; ++ ++ list_add_tail(&new_result->node, &set->results); ++} ++ ++void kutf_destroy_result_set(struct kutf_result_set *set) ++{ ++ if (!list_empty(&set->results)) ++ pr_err("kutf_destroy_result_set: Unread results from test\n"); ++ ++ kfree(set); ++} ++ ++struct kutf_result *kutf_remove_result(struct kutf_result_set *set) ++{ ++ if (!list_empty(&set->results)) { ++ struct kutf_result *ret; ++ ++ ret = list_first_entry(&set->results, struct kutf_result, node); ++ list_del(&ret->node); ++ return ret; ++ } ++ ++ return NULL; ++} ++ +diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/kutf/kutf_suite.c b/drivers/gpu/arm/bifrost_for_linux/tests/kutf/kutf_suite.c +new file mode 100755 +index 000000000000..ad30cc86a3b0 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/tests/kutf/kutf_suite.c +@@ -0,0 +1,1398 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* Kernel UTF suite, test and fixture management including user to kernel ++ * interaction */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++ ++#include ++#include ++#include ++ ++#if defined(CONFIG_DEBUG_FS) ++ ++/** ++ * struct kutf_application - Structure which represents kutf application ++ * @name: The name of this test application. ++ * @dir: The debugfs directory for this test ++ * @suite_list: List head to store all the suites which are part of this ++ * application ++ */ ++struct kutf_application { ++ const char *name; ++ struct dentry *dir; ++ struct list_head suite_list; ++}; ++ ++/** ++ * struct kutf_test_function - Structure which represents kutf test function ++ * @suite: Back reference to the suite this test function ++ * belongs to ++ * @filters: Filters that apply to this test function ++ * @test_id: Test ID ++ * @execute: Function to run for this test ++ * @test_data: Static data for this test ++ * @node: List node for test_list ++ * @variant_list: List head to store all the variants which can run on ++ * this function ++ * @dir: debugfs directory for this test function ++ * @userdata_ops: Callbacks to use for sending and receiving data to ++ * userspace. ++ */ ++struct kutf_test_function { ++ struct kutf_suite *suite; ++ unsigned int filters; ++ unsigned int test_id; ++ void (*execute)(struct kutf_context *context); ++ union kutf_callback_data test_data; ++ struct list_head node; ++ struct list_head variant_list; ++ struct dentry *dir; ++ struct kutf_userdata_ops userdata_ops; ++}; ++ ++/** ++ * struct kutf_test_fixture - Structure which holds information on the kutf ++ * test fixture ++ * @test_func: Test function this fixture belongs to ++ * @fixture_index: Index of this fixture ++ * @node: List node for variant_list ++ * @dir: debugfs directory for this test fixture ++ * @nr_running: Current count of user-clients running this fixture ++ */ ++struct kutf_test_fixture { ++ struct kutf_test_function *test_func; ++ unsigned int fixture_index; ++ struct list_head node; ++ struct dentry *dir; ++ atomic_t nr_running; ++}; ++ ++struct dentry *base_dir; ++ ++/** ++ * struct kutf_convert_table - Structure which keeps test results ++ * @result_name: Status of the test result ++ * @result: Status value for a single test ++ */ ++struct kutf_convert_table { ++ char result_name[50]; ++ enum kutf_result_status result; ++}; ++ ++struct kutf_convert_table kutf_convert[] = { ++#define ADD_UTF_RESULT(_name) \ ++{ \ ++ #_name, \ ++ _name, \ ++}, ++ADD_UTF_RESULT(KUTF_RESULT_BENCHMARK) ++ADD_UTF_RESULT(KUTF_RESULT_SKIP) ++ADD_UTF_RESULT(KUTF_RESULT_UNKNOWN) ++ADD_UTF_RESULT(KUTF_RESULT_PASS) ++ADD_UTF_RESULT(KUTF_RESULT_DEBUG) ++ADD_UTF_RESULT(KUTF_RESULT_INFO) ++ADD_UTF_RESULT(KUTF_RESULT_WARN) ++ADD_UTF_RESULT(KUTF_RESULT_FAIL) ++ADD_UTF_RESULT(KUTF_RESULT_FATAL) ++ADD_UTF_RESULT(KUTF_RESULT_ABORT) ++}; ++ ++#define UTF_CONVERT_SIZE (ARRAY_SIZE(kutf_convert)) ++ ++/** ++ * kutf_create_context() - Create a test context in which a specific fixture ++ * of an application will be run and its results ++ * reported back to the user ++ * @test_fix: Test fixture to be run. ++ * ++ * The context's refcount will be initialized to 1. ++ * ++ * Return: Returns the created test context on success or NULL on failure ++ */ ++static struct kutf_context *kutf_create_context( ++ struct kutf_test_fixture *test_fix); ++ ++/** ++ * kutf_destroy_context() - Destroy a previously created test context, only ++ * once its refcount has become zero ++ * @kref: pointer to kref member within the context ++ * ++ * This should only be used via a kref_put() call on the context's kref member ++ */ ++static void kutf_destroy_context(struct kref *kref); ++ ++/** ++ * kutf_context_get() - increment refcount on a context ++ * @context: the kutf context ++ * ++ * This must be used when the lifetime of the context might exceed that of the ++ * thread creating @context ++ */ ++static void kutf_context_get(struct kutf_context *context); ++ ++/** ++ * kutf_context_put() - decrement refcount on a context, destroying it when it ++ * reached zero ++ * @context: the kutf context ++ * ++ * This must be used only after a corresponding kutf_context_get() call on ++ * @context, and the caller no longer needs access to @context. ++ */ ++static void kutf_context_put(struct kutf_context *context); ++ ++/** ++ * kutf_set_result() - Set the test result against the specified test context ++ * @context: Test context ++ * @status: Result status ++ */ ++static void kutf_set_result(struct kutf_context *context, ++ enum kutf_result_status status); ++ ++/** ++ * kutf_set_expected_result() - Set the expected test result for the specified ++ * test context ++ * @context: Test context ++ * @expected_status: Expected result status ++ */ ++static void kutf_set_expected_result(struct kutf_context *context, ++ enum kutf_result_status expected_status); ++ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 4, 0)) ++/* Pre 3.4.0 kernels don't have the simple_open helper */ ++ ++/** ++ * simple_open() - Helper for file opening which stores the inode private data ++ * into the file private data ++ * @inode: File entry representation ++ * @file: A specific opening of the file ++ * ++ * Return: always 0; if inode private data do not exist, the file will not ++ * be assigned private data ++ */ ++static int simple_open(struct inode *inode, struct file *file) ++{ ++ if (inode->i_private) ++ file->private_data = inode->i_private; ++ return 0; ++} ++#endif ++ ++/** ++ * kutf_result_to_string() - Converts a KUTF result into a string ++ * @result_str: Output result string ++ * @result: Result status to convert ++ * ++ * Return: 1 if test result was successfully converted to string, 0 otherwise ++ */ ++static int kutf_result_to_string(char **result_str, ++ enum kutf_result_status result) ++{ ++ int i; ++ int ret = 0; ++ ++ for (i = 0; i < UTF_CONVERT_SIZE; i++) { ++ if (result == kutf_convert[i].result) { ++ *result_str = kutf_convert[i].result_name; ++ ret = 1; ++ } ++ } ++ return ret; ++} ++ ++/** ++ * kutf_debugfs_const_string_read() - Simple debugfs read callback which ++ * returns a constant string ++ * @file: Opened file to read from ++ * @buf: User buffer to write the data into ++ * @len: Amount of data to read ++ * @ppos: Offset into file to read from ++ * ++ * Return: On success, the number of bytes read and offset @ppos advanced by ++ * this number; on error, negative value ++ */ ++static ssize_t kutf_debugfs_const_string_read(struct file *file, ++ char __user *buf, size_t len, loff_t *ppos) ++{ ++ char *str = file->private_data; ++ ++ return simple_read_from_buffer(buf, len, ppos, str, strlen(str)); ++} ++ ++static const struct file_operations kutf_debugfs_const_string_ops = { ++ .owner = THIS_MODULE, ++ .open = simple_open, ++ .read = kutf_debugfs_const_string_read, ++ .llseek = default_llseek, ++}; ++ ++/** ++ * kutf_debugfs_data_open() Debugfs open callback for the "data" entry. ++ * @inode: inode of the opened file ++ * @file: Opened file to read from ++ * ++ * This function notifies the userdata callbacks that the userdata file has ++ * been opened, for tracking purposes. ++ * ++ * It is called on both the context's userdata_consumer_priv and ++ * userdata_producer_priv. ++ * ++ * This takes a refcount on the kutf_context ++ * ++ * Return: 0 on success ++ */ ++static int kutf_debugfs_data_open(struct inode *inode, struct file *file) ++{ ++ struct kutf_context *test_context = inode->i_private; ++ struct kutf_test_fixture *test_fix = test_context->test_fix; ++ struct kutf_test_function *test_func = test_fix->test_func; ++ int err; ++ ++ simple_open(inode, file); ++ ++ /* This is not an error */ ++ if (!test_func->userdata_ops.open) ++ goto out_no_ops; ++ ++ /* This is safe here - the 'data' file is only openable whilst the ++ * initial refcount is still present, and the initial refcount is only ++ * dropped strictly after the 'data' file is removed */ ++ kutf_context_get(test_context); ++ ++ if (test_context->userdata_consumer_priv) { ++ err = test_func->userdata_ops.open(test_context->userdata_consumer_priv); ++ if (err) ++ goto out_consumer_fail; ++ } ++ ++ if (test_context->userdata_producer_priv) { ++ err = test_func->userdata_ops.open(test_context->userdata_producer_priv); ++ if (err) ++ goto out_producer_fail; ++ } ++ ++out_no_ops: ++ return 0; ++ ++out_producer_fail: ++ if (test_func->userdata_ops.release && test_context->userdata_consumer_priv) ++ test_func->userdata_ops.release(test_context->userdata_consumer_priv); ++out_consumer_fail: ++ kutf_context_put(test_context); ++ ++ return err; ++} ++ ++ ++/** ++ * kutf_debugfs_data_read() Debugfs read callback for the "data" entry. ++ * @file: Opened file to read from ++ * @buf: User buffer to write the data into ++ * @len: Amount of data to read ++ * @ppos: Offset into file to read from ++ * ++ * This function allows user and kernel to exchange extra data necessary for ++ * the test fixture. ++ * ++ * The data is read from the first struct kutf_context running the fixture ++ * ++ * Return: Number of bytes read ++ */ ++static ssize_t kutf_debugfs_data_read(struct file *file, char __user *buf, ++ size_t len, loff_t *ppos) ++{ ++ struct kutf_context *test_context = file->private_data; ++ struct kutf_test_fixture *test_fix = test_context->test_fix; ++ struct kutf_test_function *test_func = test_fix->test_func; ++ ssize_t (*producer)(void *private, char __user *userbuf, ++ size_t userbuf_len, loff_t *ppos); ++ ssize_t count; ++ ++ producer = test_func->userdata_ops.producer; ++ /* Can only read if there's a producer callback */ ++ if (!producer) ++ return -ENODEV; ++ ++ count = producer(test_context->userdata_producer_priv, buf, len, ppos); ++ ++ return count; ++} ++ ++/** ++ * kutf_debugfs_data_write() Debugfs write callback for the "data" entry. ++ * @file: Opened file to write to ++ * @buf: User buffer to read the data from ++ * @len: Amount of data to write ++ * @ppos: Offset into file to write to ++ * ++ * This function allows user and kernel to exchange extra data necessary for ++ * the test fixture. ++ * ++ * The data is added to the first struct kutf_context running the fixture ++ * ++ * Return: Number of bytes written ++ */ ++static ssize_t kutf_debugfs_data_write(struct file *file, ++ const char __user *buf, size_t len, loff_t *ppos) ++{ ++ struct kutf_context *test_context = file->private_data; ++ struct kutf_test_fixture *test_fix = test_context->test_fix; ++ struct kutf_test_function *test_func = test_fix->test_func; ++ ssize_t (*consumer)(void *private, const char __user *userbuf, ++ size_t userbuf_len, loff_t *ppos); ++ ssize_t count; ++ ++ consumer = test_func->userdata_ops.consumer; ++ /* Can only write if there's a consumer callback */ ++ if (!consumer) ++ return -ENODEV; ++ ++ count = consumer(test_context->userdata_consumer_priv, buf, len, ppos); ++ ++ return count; ++} ++ ++ ++/** ++ * kutf_debugfs_data_release() - Debugfs release callback for the "data" entry. ++ * @inode: File entry representation ++ * @file: A specific opening of the file ++ * ++ * This function notifies the userdata callbacks that the userdata file has ++ * been closed, for tracking purposes. ++ * ++ * It is called on both the context's userdata_consumer_priv and ++ * userdata_producer_priv. ++ * ++ * It also drops the refcount on the kutf_context that was taken during ++ * kutf_debugfs_data_open() ++ */ ++static int kutf_debugfs_data_release(struct inode *inode, struct file *file) ++{ ++ struct kutf_context *test_context = file->private_data; ++ struct kutf_test_fixture *test_fix = test_context->test_fix; ++ struct kutf_test_function *test_func = test_fix->test_func; ++ ++ if (!test_func->userdata_ops.release) ++ return 0; ++ ++ if (test_context->userdata_consumer_priv) ++ test_func->userdata_ops.release(test_context->userdata_consumer_priv); ++ if (test_context->userdata_producer_priv) ++ test_func->userdata_ops.release(test_context->userdata_producer_priv); ++ ++ kutf_context_put(test_context); ++ ++ return 0; ++} ++ ++ ++static const struct file_operations kutf_debugfs_data_ops = { ++ .owner = THIS_MODULE, ++ .open = kutf_debugfs_data_open, ++ .read = kutf_debugfs_data_read, ++ .write = kutf_debugfs_data_write, ++ .release = kutf_debugfs_data_release, ++ .llseek = default_llseek, ++}; ++ ++/** ++ * userdata_init() - Initialize userspace data exchange for a test, if ++ * specified by that test ++ * @test_context: Test context ++ * ++ * Note that this allows new refcounts to be made on test_context by userspace ++ * threads opening the 'data' file. ++ * ++ * Return: 0 on success, negative value corresponding to error code in failure ++ * and kutf result will be set appropriately to indicate the error ++ */ ++static int userdata_init(struct kutf_context *test_context) ++{ ++ struct kutf_test_fixture *test_fix = test_context->test_fix; ++ struct kutf_test_function *test_func = test_fix->test_func; ++ int err = 0; ++ struct dentry *userdata_dentry; ++ ++ /* Valid to have neither a producer or consumer, which is the case for ++ * tests not requiring usersdata */ ++ if ((!test_func->userdata_ops.consumer) && (!test_func->userdata_ops.producer)) ++ return err; ++ ++ if (test_func->userdata_ops.consumer && !test_context->userdata_consumer_priv) { ++ kutf_test_fatal(test_context, ++ "incorrect test setup - userdata consumer provided without private data"); ++ return -EFAULT; ++ } ++ ++ if (test_func->userdata_ops.producer && !test_context->userdata_producer_priv) { ++ kutf_test_fatal(test_context, ++ "incorrect test setup - userdata producer provided without private data"); ++ return -EFAULT; ++ } ++ ++ userdata_dentry = debugfs_create_file("data", S_IROTH, test_fix->dir, ++ test_context, &kutf_debugfs_data_ops); ++ ++ if (!userdata_dentry) { ++ pr_err("Failed to create debugfs file \"data\" when running fixture\n"); ++ /* Not using Fatal (which stops other tests running), ++ * nor Abort (which indicates teardown should not be done) */ ++ kutf_test_fail(test_context, ++ "failed to create 'data' file for userside data exchange"); ++ ++ /* Error code is discarded by caller, but consistent with other ++ * debugfs_create_file failures */ ++ err = -EEXIST; ++ } else { ++ test_context->userdata_dentry = userdata_dentry; ++ } ++ ++ ++ return err; ++} ++ ++/** ++ * userdata_term() - Terminate userspace data exchange for a test, if specified ++ * by that test ++ * @test_context: Test context ++ * ++ * Note This also prevents new refcounts being made on @test_context by userspace ++ * threads opening the 'data' file for this test. Any existing open file descriptors ++ * to the 'data' file will still be safe to use by userspace. ++ */ ++static void userdata_term(struct kutf_context *test_context) ++{ ++ struct kutf_test_fixture *test_fix = test_context->test_fix; ++ struct kutf_test_function *test_func = test_fix->test_func; ++ void (*notify_ended)(void *priv) = test_func->userdata_ops.notify_ended; ++ ++ /* debugfs_remove() is safe when parameter is error or NULL */ ++ debugfs_remove(test_context->userdata_dentry); ++ ++ /* debugfs_remove() doesn't kill any currently open file descriptors on ++ * this file, and such fds are still safe to use providing test_context ++ * is properly refcounted */ ++ ++ if (notify_ended) { ++ if (test_context->userdata_consumer_priv) ++ notify_ended(test_context->userdata_consumer_priv); ++ if (test_context->userdata_producer_priv) ++ notify_ended(test_context->userdata_producer_priv); ++ } ++ ++} ++ ++/** ++ * kutf_add_explicit_result() - Check if an explicit result needs to be added ++ * @context: KUTF test context ++ */ ++static void kutf_add_explicit_result(struct kutf_context *context) ++{ ++ switch (context->expected_status) { ++ case KUTF_RESULT_UNKNOWN: ++ if (context->status == KUTF_RESULT_UNKNOWN) ++ kutf_test_pass(context, "(implicit pass)"); ++ break; ++ ++ case KUTF_RESULT_WARN: ++ if (context->status == KUTF_RESULT_WARN) ++ kutf_test_pass(context, ++ "Pass (expected warn occurred)"); ++ else if (context->status != KUTF_RESULT_SKIP) ++ kutf_test_fail(context, ++ "Fail (expected warn missing)"); ++ break; ++ ++ case KUTF_RESULT_FAIL: ++ if (context->status == KUTF_RESULT_FAIL) ++ kutf_test_pass(context, ++ "Pass (expected fail occurred)"); ++ else if (context->status != KUTF_RESULT_SKIP) { ++ /* Force the expected status so the fail gets logged */ ++ context->expected_status = KUTF_RESULT_PASS; ++ kutf_test_fail(context, ++ "Fail (expected fail missing)"); ++ } ++ break; ++ ++ case KUTF_RESULT_FATAL: ++ if (context->status == KUTF_RESULT_FATAL) ++ kutf_test_pass(context, ++ "Pass (expected fatal occurred)"); ++ else if (context->status != KUTF_RESULT_SKIP) ++ kutf_test_fail(context, ++ "Fail (expected fatal missing)"); ++ break; ++ ++ case KUTF_RESULT_ABORT: ++ if (context->status == KUTF_RESULT_ABORT) ++ kutf_test_pass(context, ++ "Pass (expected abort occurred)"); ++ else if (context->status != KUTF_RESULT_SKIP) ++ kutf_test_fail(context, ++ "Fail (expected abort missing)"); ++ break; ++ default: ++ break; ++ } ++} ++ ++/** ++ * kutf_debugfs_run_open() Debugfs open callback for the "run" entry. ++ * @inode: inode of the opened file ++ * @file: Opened file to read from ++ * ++ * This function retrieves the test fixture data that is associated with the ++ * opened file and works back to get the test, suite and application so ++ * it can then run the test that is associated with the file entry. ++ * ++ * Return: 0 on success ++ */ ++static int kutf_debugfs_run_open(struct inode *inode, struct file *file) ++{ ++ struct kutf_test_fixture *test_fix = inode->i_private; ++ struct kutf_test_function *test_func = test_fix->test_func; ++ struct kutf_suite *suite = test_func->suite; ++ struct kutf_context *test_context; ++ int err = 0; ++ ++ /* For the moment, only one user-client should be attempting to run ++ * this at a time. This simplifies how we lookup the kutf_context when ++ * using the 'data' file. ++ * Removing this restriction would require a rewrite of the mechanism ++ * of the 'data' file to pass data in, perhaps 'data' created here and ++ * based upon userspace thread's pid */ ++ if (atomic_inc_return(&test_fix->nr_running) != 1) { ++ err = -EBUSY; ++ goto finish; ++ } ++ ++ test_context = kutf_create_context(test_fix); ++ if (!test_context) { ++ err = -ENODEV; ++ goto finish; ++ } ++ ++ file->private_data = test_context; ++ ++ /* ++ * Call the create fixture function if required before the ++ * fixture is run ++ */ ++ if (suite->create_fixture) ++ test_context->fixture = suite->create_fixture(test_context); ++ ++ /* Only run the test if the fixture was created (if required) */ ++ if ((suite->create_fixture && test_context->fixture) || ++ (!suite->create_fixture)) { ++ int late_err; ++ /* Setup any userdata exchange */ ++ late_err = userdata_init(test_context); ++ ++ if (!late_err) ++ /* Run this fixture */ ++ test_func->execute(test_context); ++ ++ userdata_term(test_context); ++ ++ if (suite->remove_fixture) ++ suite->remove_fixture(test_context); ++ ++ kutf_add_explicit_result(test_context); ++ } ++ ++finish: ++ atomic_dec(&test_fix->nr_running); ++ return err; ++} ++ ++/** ++ * kutf_debugfs_run_read() - Debugfs read callback for the "run" entry. ++ * @file: Opened file to read from ++ * @buf: User buffer to write the data into ++ * @len: Amount of data to read ++ * @ppos: Offset into file to read from ++ * ++ * This function emits the results which where logged during the opening of ++ * the file kutf_debugfs_run_open. ++ * Results will be emitted one at a time, once all the results have been read ++ * 0 will be returned to indicate there is no more data. ++ * ++ * Return: Number of bytes read. ++ */ ++static ssize_t kutf_debugfs_run_read(struct file *file, char __user *buf, ++ size_t len, loff_t *ppos) ++{ ++ struct kutf_context *test_context = file->private_data; ++ struct kutf_result *res; ++ unsigned long bytes_not_copied; ++ ssize_t bytes_copied = 0; ++ ++ /* Note: This code assumes a result is read completely */ ++ res = kutf_remove_result(test_context->result_set); ++ if (res) { ++ char *kutf_str_ptr = NULL; ++ unsigned int kutf_str_len = 0; ++ unsigned int message_len = 0; ++ char separator = ':'; ++ char terminator = '\n'; ++ ++ kutf_result_to_string(&kutf_str_ptr, res->status); ++ if (kutf_str_ptr) ++ kutf_str_len = strlen(kutf_str_ptr); ++ ++ if (res->message) ++ message_len = strlen(res->message); ++ ++ if ((kutf_str_len + 1 + message_len + 1) > len) { ++ pr_err("Not enough space in user buffer for a single result"); ++ return 0; ++ } ++ ++ /* First copy the result string */ ++ if (kutf_str_ptr) { ++ bytes_not_copied = copy_to_user(&buf[0], kutf_str_ptr, ++ kutf_str_len); ++ bytes_copied += kutf_str_len - bytes_not_copied; ++ if (bytes_not_copied) ++ goto exit; ++ } ++ ++ /* Then the separator */ ++ bytes_not_copied = copy_to_user(&buf[bytes_copied], ++ &separator, 1); ++ bytes_copied += 1 - bytes_not_copied; ++ if (bytes_not_copied) ++ goto exit; ++ ++ /* Finally Next copy the result string */ ++ if (res->message) { ++ bytes_not_copied = copy_to_user(&buf[bytes_copied], ++ res->message, message_len); ++ bytes_copied += message_len - bytes_not_copied; ++ if (bytes_not_copied) ++ goto exit; ++ } ++ ++ /* Finally the terminator */ ++ bytes_not_copied = copy_to_user(&buf[bytes_copied], ++ &terminator, 1); ++ bytes_copied += 1 - bytes_not_copied; ++ } ++exit: ++ return bytes_copied; ++} ++ ++/** ++ * kutf_debugfs_run_release() - Debugfs release callback for the "run" entry. ++ * @inode: File entry representation ++ * @file: A specific opening of the file ++ * ++ * Release any resources that where created during the opening of the file ++ * ++ * Note that resources may not be released immediately, that might only happen ++ * later when other users of the kutf_context release their refcount. ++ * ++ * Return: 0 on success ++ */ ++static int kutf_debugfs_run_release(struct inode *inode, struct file *file) ++{ ++ struct kutf_context *test_context = file->private_data; ++ ++ kutf_context_put(test_context); ++ return 0; ++} ++ ++static const struct file_operations kutf_debugfs_run_ops = { ++ .owner = THIS_MODULE, ++ .open = kutf_debugfs_run_open, ++ .read = kutf_debugfs_run_read, ++ .release = kutf_debugfs_run_release, ++ .llseek = default_llseek, ++}; ++ ++/** ++ * create_fixture_variant() - Creates a fixture variant for the specified ++ * test function and index and the debugfs entries ++ * that represent it. ++ * @test_func: Test function ++ * @fixture_index: Fixture index ++ * ++ * Return: 0 on success, negative value corresponding to error code in failure ++ */ ++static int create_fixture_variant(struct kutf_test_function *test_func, ++ unsigned int fixture_index) ++{ ++ struct kutf_test_fixture *test_fix; ++ char name[11]; /* Enough to print the MAX_UINT32 + the null terminator */ ++ struct dentry *tmp; ++ int err; ++ ++ test_fix = kmalloc(sizeof(*test_fix), GFP_KERNEL); ++ if (!test_fix) { ++ pr_err("Failed to create debugfs directory when adding fixture\n"); ++ err = -ENOMEM; ++ goto fail_alloc; ++ } ++ ++ test_fix->test_func = test_func; ++ test_fix->fixture_index = fixture_index; ++ atomic_set(&test_fix->nr_running, 0); ++ ++ snprintf(name, sizeof(name), "%d", fixture_index); ++ test_fix->dir = debugfs_create_dir(name, test_func->dir); ++ if (!test_func->dir) { ++ pr_err("Failed to create debugfs directory when adding fixture\n"); ++ /* Might not be the right error, we don't get it passed back to us */ ++ err = -EEXIST; ++ goto fail_dir; ++ } ++ ++ tmp = debugfs_create_file("type", S_IROTH, test_fix->dir, "fixture\n", ++ &kutf_debugfs_const_string_ops); ++ if (!tmp) { ++ pr_err("Failed to create debugfs file \"type\" when adding fixture\n"); ++ /* Might not be the right error, we don't get it passed back to us */ ++ err = -EEXIST; ++ goto fail_file; ++ } ++ ++ tmp = debugfs_create_file("run", S_IROTH, test_fix->dir, test_fix, ++ &kutf_debugfs_run_ops); ++ if (!tmp) { ++ pr_err("Failed to create debugfs file \"run\" when adding fixture\n"); ++ /* Might not be the right error, we don't get it passed back to us */ ++ err = -EEXIST; ++ goto fail_file; ++ } ++ ++ list_add(&test_fix->node, &test_func->variant_list); ++ return 0; ++ ++fail_file: ++ debugfs_remove_recursive(test_fix->dir); ++fail_dir: ++ kfree(test_fix); ++fail_alloc: ++ return err; ++} ++ ++/** ++ * kutf_remove_test_variant() - Destroy a previously created fixture variant. ++ * @test_fix: Test fixture ++ */ ++static void kutf_remove_test_variant(struct kutf_test_fixture *test_fix) ++{ ++ debugfs_remove_recursive(test_fix->dir); ++ kfree(test_fix); ++} ++ ++void kutf_add_test_with_filters_data_and_userdata( ++ struct kutf_suite *suite, ++ unsigned int id, ++ const char *name, ++ void (*execute)(struct kutf_context *context), ++ unsigned int filters, ++ union kutf_callback_data test_data, ++ struct kutf_userdata_ops *userdata_ops) ++{ ++ struct kutf_test_function *test_func; ++ struct dentry *tmp; ++ unsigned int i; ++ ++ test_func = kmalloc(sizeof(*test_func), GFP_KERNEL); ++ if (!test_func) { ++ pr_err("Failed to allocate memory when adding test %s\n", name); ++ goto fail_alloc; ++ } ++ ++ INIT_LIST_HEAD(&test_func->variant_list); ++ ++ test_func->dir = debugfs_create_dir(name, suite->dir); ++ if (!test_func->dir) { ++ pr_err("Failed to create debugfs directory when adding test %s\n", name); ++ goto fail_dir; ++ } ++ ++ tmp = debugfs_create_file("type", S_IROTH, test_func->dir, "test\n", ++ &kutf_debugfs_const_string_ops); ++ if (!tmp) { ++ pr_err("Failed to create debugfs file \"type\" when adding test %s\n", name); ++ goto fail_file; ++ } ++ ++ test_func->filters = filters; ++ tmp = debugfs_create_x32("filters", S_IROTH, test_func->dir, ++ &test_func->filters); ++ if (!tmp) { ++ pr_err("Failed to create debugfs file \"filters\" when adding test %s\n", name); ++ goto fail_file; ++ } ++ ++ test_func->test_id = id; ++ tmp = debugfs_create_u32("test_id", S_IROTH, test_func->dir, ++ &test_func->test_id); ++ if (!tmp) { ++ pr_err("Failed to create debugfs file \"test_id\" when adding test %s\n", name); ++ goto fail_file; ++ } ++ ++ for (i = 0; i < suite->fixture_variants; i++) { ++ if (create_fixture_variant(test_func, i)) { ++ pr_err("Failed to create fixture %d when adding test %s\n", i, name); ++ goto fail_file; ++ } ++ } ++ ++ test_func->suite = suite; ++ test_func->execute = execute; ++ test_func->test_data = test_data; ++ memcpy(&test_func->userdata_ops, userdata_ops, sizeof(*userdata_ops)); ++ ++ list_add(&test_func->node, &suite->test_list); ++ return; ++ ++fail_file: ++ debugfs_remove_recursive(test_func->dir); ++fail_dir: ++ kfree(test_func); ++fail_alloc: ++ return; ++} ++EXPORT_SYMBOL(kutf_add_test_with_filters_data_and_userdata); ++ ++void kutf_add_test_with_filters_and_data( ++ struct kutf_suite *suite, ++ unsigned int id, ++ const char *name, ++ void (*execute)(struct kutf_context *context), ++ unsigned int filters, ++ union kutf_callback_data test_data) ++{ ++ struct kutf_userdata_ops userdata_ops = { ++ .open = NULL, ++ .release = NULL, ++ .consumer = NULL, ++ .producer = NULL, ++ }; ++ ++ kutf_add_test_with_filters_data_and_userdata(suite, id, name, execute, ++ filters, test_data, &userdata_ops); ++} ++ ++EXPORT_SYMBOL(kutf_add_test_with_filters_and_data); ++ ++void kutf_add_test_with_filters( ++ struct kutf_suite *suite, ++ unsigned int id, ++ const char *name, ++ void (*execute)(struct kutf_context *context), ++ unsigned int filters) ++{ ++ union kutf_callback_data data; ++ ++ data.ptr_value = NULL; ++ ++ kutf_add_test_with_filters_and_data(suite, ++ id, ++ name, ++ execute, ++ suite->suite_default_flags, ++ data); ++} ++EXPORT_SYMBOL(kutf_add_test_with_filters); ++ ++void kutf_add_test(struct kutf_suite *suite, ++ unsigned int id, ++ const char *name, ++ void (*execute)(struct kutf_context *context)) ++{ ++ union kutf_callback_data data; ++ ++ data.ptr_value = NULL; ++ ++ kutf_add_test_with_filters_and_data(suite, ++ id, ++ name, ++ execute, ++ suite->suite_default_flags, ++ data); ++} ++EXPORT_SYMBOL(kutf_add_test); ++ ++/** ++ * kutf_remove_test(): Remove a previously added test function. ++ * @test_func: Test function ++ */ ++static void kutf_remove_test(struct kutf_test_function *test_func) ++{ ++ struct list_head *pos; ++ struct list_head *tmp; ++ ++ list_for_each_safe(pos, tmp, &test_func->variant_list) { ++ struct kutf_test_fixture *test_fix; ++ ++ test_fix = list_entry(pos, struct kutf_test_fixture, node); ++ kutf_remove_test_variant(test_fix); ++ } ++ ++ list_del(&test_func->node); ++ debugfs_remove_recursive(test_func->dir); ++ kfree(test_func); ++} ++ ++struct kutf_suite *kutf_create_suite_with_filters_and_data( ++ struct kutf_application *app, ++ const char *name, ++ unsigned int fixture_count, ++ void *(*create_fixture)(struct kutf_context *context), ++ void (*remove_fixture)(struct kutf_context *context), ++ unsigned int filters, ++ union kutf_callback_data suite_data) ++{ ++ struct kutf_suite *suite; ++ struct dentry *tmp; ++ ++ suite = kmalloc(sizeof(*suite), GFP_KERNEL); ++ if (!suite) { ++ pr_err("Failed to allocate memory when creating suite %s\n", name); ++ goto fail_kmalloc; ++ } ++ ++ suite->dir = debugfs_create_dir(name, app->dir); ++ if (!suite->dir) { ++ pr_err("Failed to create debugfs directory when adding test %s\n", name); ++ goto fail_debugfs; ++ } ++ ++ tmp = debugfs_create_file("type", S_IROTH, suite->dir, "suite\n", ++ &kutf_debugfs_const_string_ops); ++ if (!tmp) { ++ pr_err("Failed to create debugfs file \"type\" when adding test %s\n", name); ++ goto fail_file; ++ } ++ ++ INIT_LIST_HEAD(&suite->test_list); ++ suite->app = app; ++ suite->name = name; ++ suite->fixture_variants = fixture_count; ++ suite->create_fixture = create_fixture; ++ suite->remove_fixture = remove_fixture; ++ suite->suite_default_flags = filters; ++ suite->suite_data = suite_data; ++ ++ list_add(&suite->node, &app->suite_list); ++ ++ return suite; ++ ++fail_file: ++ debugfs_remove_recursive(suite->dir); ++fail_debugfs: ++ kfree(suite); ++fail_kmalloc: ++ return NULL; ++} ++EXPORT_SYMBOL(kutf_create_suite_with_filters_and_data); ++ ++struct kutf_suite *kutf_create_suite_with_filters( ++ struct kutf_application *app, ++ const char *name, ++ unsigned int fixture_count, ++ void *(*create_fixture)(struct kutf_context *context), ++ void (*remove_fixture)(struct kutf_context *context), ++ unsigned int filters) ++{ ++ union kutf_callback_data data; ++ ++ data.ptr_value = NULL; ++ return kutf_create_suite_with_filters_and_data(app, ++ name, ++ fixture_count, ++ create_fixture, ++ remove_fixture, ++ filters, ++ data); ++} ++EXPORT_SYMBOL(kutf_create_suite_with_filters); ++ ++struct kutf_suite *kutf_create_suite( ++ struct kutf_application *app, ++ const char *name, ++ unsigned int fixture_count, ++ void *(*create_fixture)(struct kutf_context *context), ++ void (*remove_fixture)(struct kutf_context *context)) ++{ ++ union kutf_callback_data data; ++ ++ data.ptr_value = NULL; ++ return kutf_create_suite_with_filters_and_data(app, ++ name, ++ fixture_count, ++ create_fixture, ++ remove_fixture, ++ KUTF_F_TEST_GENERIC, ++ data); ++} ++EXPORT_SYMBOL(kutf_create_suite); ++ ++/** ++ * kutf_destroy_suite() - Destroy a previously added test suite. ++ * @suite: Test suite ++ */ ++static void kutf_destroy_suite(struct kutf_suite *suite) ++{ ++ struct list_head *pos; ++ struct list_head *tmp; ++ ++ list_for_each_safe(pos, tmp, &suite->test_list) { ++ struct kutf_test_function *test_func; ++ ++ test_func = list_entry(pos, struct kutf_test_function, node); ++ kutf_remove_test(test_func); ++ } ++ ++ list_del(&suite->node); ++ debugfs_remove_recursive(suite->dir); ++ kfree(suite); ++} ++ ++struct kutf_application *kutf_create_application(const char *name) ++{ ++ struct kutf_application *app; ++ struct dentry *tmp; ++ ++ app = kmalloc(sizeof(*app), GFP_KERNEL); ++ if (!app) { ++ pr_err("Failed to create allocate memory when creating application %s\n", name); ++ goto fail_kmalloc; ++ } ++ ++ app->dir = debugfs_create_dir(name, base_dir); ++ if (!app->dir) { ++ pr_err("Failed to create debugfs direcotry when creating application %s\n", name); ++ goto fail_debugfs; ++ } ++ ++ tmp = debugfs_create_file("type", S_IROTH, app->dir, "application\n", ++ &kutf_debugfs_const_string_ops); ++ if (!tmp) { ++ pr_err("Failed to create debugfs file \"type\" when creating application %s\n", name); ++ goto fail_file; ++ } ++ ++ INIT_LIST_HEAD(&app->suite_list); ++ app->name = name; ++ ++ return app; ++ ++fail_file: ++ debugfs_remove_recursive(app->dir); ++fail_debugfs: ++ kfree(app); ++fail_kmalloc: ++ return NULL; ++} ++EXPORT_SYMBOL(kutf_create_application); ++ ++void kutf_destroy_application(struct kutf_application *app) ++{ ++ struct list_head *pos; ++ struct list_head *tmp; ++ ++ list_for_each_safe(pos, tmp, &app->suite_list) { ++ struct kutf_suite *suite; ++ ++ suite = list_entry(pos, struct kutf_suite, node); ++ kutf_destroy_suite(suite); ++ } ++ ++ debugfs_remove_recursive(app->dir); ++ kfree(app); ++} ++EXPORT_SYMBOL(kutf_destroy_application); ++ ++static struct kutf_context *kutf_create_context( ++ struct kutf_test_fixture *test_fix) ++{ ++ struct kutf_context *new_context; ++ ++ new_context = kmalloc(sizeof(*new_context), GFP_KERNEL); ++ if (!new_context) { ++ pr_err("Failed to allocate test context"); ++ goto fail_alloc; ++ } ++ ++ new_context->result_set = kutf_create_result_set(); ++ if (!new_context->result_set) { ++ pr_err("Failed to create resultset"); ++ goto fail_result_set; ++ } ++ ++ new_context->test_fix = test_fix; ++ /* Save the pointer to the suite as the callbacks will require it */ ++ new_context->suite = test_fix->test_func->suite; ++ new_context->status = KUTF_RESULT_UNKNOWN; ++ new_context->expected_status = KUTF_RESULT_UNKNOWN; ++ ++ kutf_mempool_init(&new_context->fixture_pool); ++ new_context->fixture = NULL; ++ new_context->fixture_index = test_fix->fixture_index; ++ new_context->fixture_name = NULL; ++ new_context->test_data = test_fix->test_func->test_data; ++ new_context->userdata_consumer_priv = NULL; ++ new_context->userdata_producer_priv = NULL; ++ new_context->userdata_dentry = NULL; ++ ++ kref_init(&new_context->kref); ++ ++ return new_context; ++ ++fail_result_set: ++ kfree(new_context); ++fail_alloc: ++ return NULL; ++} ++ ++static void kutf_destroy_context(struct kref *kref) ++{ ++ struct kutf_context *context; ++ ++ context = container_of(kref, struct kutf_context, kref); ++ kutf_destroy_result_set(context->result_set); ++ kutf_mempool_destroy(&context->fixture_pool); ++ kfree(context); ++} ++ ++static void kutf_context_get(struct kutf_context *context) ++{ ++ kref_get(&context->kref); ++} ++ ++static void kutf_context_put(struct kutf_context *context) ++{ ++ kref_put(&context->kref, kutf_destroy_context); ++} ++ ++ ++static void kutf_set_result(struct kutf_context *context, ++ enum kutf_result_status status) ++{ ++ context->status = status; ++} ++ ++static void kutf_set_expected_result(struct kutf_context *context, ++ enum kutf_result_status expected_status) ++{ ++ context->expected_status = expected_status; ++} ++ ++/** ++ * kutf_test_log_result() - Log a result for the specified test context ++ * @context: Test context ++ * @message: Result string ++ * @new_status: Result status ++ */ ++static void kutf_test_log_result( ++ struct kutf_context *context, ++ const char *message, ++ enum kutf_result_status new_status) ++{ ++ if (context->status < new_status) ++ context->status = new_status; ++ ++ if (context->expected_status != new_status) ++ kutf_add_result(&context->fixture_pool, context->result_set, ++ new_status, message); ++} ++ ++void kutf_test_log_result_external( ++ struct kutf_context *context, ++ const char *message, ++ enum kutf_result_status new_status) ++{ ++ kutf_test_log_result(context, message, new_status); ++} ++EXPORT_SYMBOL(kutf_test_log_result_external); ++ ++void kutf_test_expect_abort(struct kutf_context *context) ++{ ++ kutf_set_expected_result(context, KUTF_RESULT_ABORT); ++} ++EXPORT_SYMBOL(kutf_test_expect_abort); ++ ++void kutf_test_expect_fatal(struct kutf_context *context) ++{ ++ kutf_set_expected_result(context, KUTF_RESULT_FATAL); ++} ++EXPORT_SYMBOL(kutf_test_expect_fatal); ++ ++void kutf_test_expect_fail(struct kutf_context *context) ++{ ++ kutf_set_expected_result(context, KUTF_RESULT_FAIL); ++} ++EXPORT_SYMBOL(kutf_test_expect_fail); ++ ++void kutf_test_expect_warn(struct kutf_context *context) ++{ ++ kutf_set_expected_result(context, KUTF_RESULT_WARN); ++} ++EXPORT_SYMBOL(kutf_test_expect_warn); ++ ++void kutf_test_expect_pass(struct kutf_context *context) ++{ ++ kutf_set_expected_result(context, KUTF_RESULT_PASS); ++} ++EXPORT_SYMBOL(kutf_test_expect_pass); ++ ++void kutf_test_skip(struct kutf_context *context) ++{ ++ kutf_set_result(context, KUTF_RESULT_SKIP); ++ kutf_set_expected_result(context, KUTF_RESULT_UNKNOWN); ++ ++ kutf_test_log_result(context, "Test skipped", KUTF_RESULT_SKIP); ++} ++EXPORT_SYMBOL(kutf_test_skip); ++ ++void kutf_test_skip_msg(struct kutf_context *context, const char *message) ++{ ++ kutf_set_result(context, KUTF_RESULT_SKIP); ++ kutf_set_expected_result(context, KUTF_RESULT_UNKNOWN); ++ ++ kutf_test_log_result(context, kutf_dsprintf(&context->fixture_pool, ++ "Test skipped: %s", message), KUTF_RESULT_SKIP); ++ kutf_test_log_result(context, "!!!Test skipped!!!", KUTF_RESULT_SKIP); ++} ++EXPORT_SYMBOL(kutf_test_skip_msg); ++ ++void kutf_test_debug(struct kutf_context *context, char const *message) ++{ ++ kutf_test_log_result(context, message, KUTF_RESULT_DEBUG); ++} ++EXPORT_SYMBOL(kutf_test_debug); ++ ++void kutf_test_pass(struct kutf_context *context, char const *message) ++{ ++ static const char explicit_message[] = "(explicit pass)"; ++ ++ if (!message) ++ message = explicit_message; ++ ++ kutf_test_log_result(context, message, KUTF_RESULT_PASS); ++} ++EXPORT_SYMBOL(kutf_test_pass); ++ ++void kutf_test_info(struct kutf_context *context, char const *message) ++{ ++ kutf_test_log_result(context, message, KUTF_RESULT_INFO); ++} ++EXPORT_SYMBOL(kutf_test_info); ++ ++void kutf_test_warn(struct kutf_context *context, char const *message) ++{ ++ kutf_test_log_result(context, message, KUTF_RESULT_WARN); ++} ++EXPORT_SYMBOL(kutf_test_warn); ++ ++void kutf_test_fail(struct kutf_context *context, char const *message) ++{ ++ kutf_test_log_result(context, message, KUTF_RESULT_FAIL); ++} ++EXPORT_SYMBOL(kutf_test_fail); ++ ++void kutf_test_fatal(struct kutf_context *context, char const *message) ++{ ++ kutf_test_log_result(context, message, KUTF_RESULT_FATAL); ++} ++EXPORT_SYMBOL(kutf_test_fatal); ++ ++void kutf_test_abort(struct kutf_context *context) ++{ ++ kutf_test_log_result(context, "", KUTF_RESULT_ABORT); ++} ++EXPORT_SYMBOL(kutf_test_abort); ++ ++/** ++ * init_kutf_core() - Module entry point. ++ * ++ * Create the base entry point in debugfs. ++ */ ++static int __init init_kutf_core(void) ++{ ++ int ret; ++ ++ base_dir = debugfs_create_dir("kutf_tests", NULL); ++ if (!base_dir) { ++ ret = -ENODEV; ++ goto exit_dir; ++ } ++ ++ return 0; ++ ++exit_dir: ++ return ret; ++} ++ ++/** ++ * exit_kutf_core() - Module exit point. ++ * ++ * Remove the base entry point in debugfs. ++ */ ++static void __exit exit_kutf_core(void) ++{ ++ debugfs_remove_recursive(base_dir); ++} ++ ++#else /* defined(CONFIG_DEBUG_FS) */ ++ ++/** ++ * init_kutf_core() - Module entry point. ++ * ++ * Stub for when build against a kernel without debugfs support ++ */ ++static int __init init_kutf_core(void) ++{ ++ pr_debug("KUTF requires a kernel with debug fs support"); ++ ++ return -ENODEV; ++} ++ ++/** ++ * exit_kutf_core() - Module exit point. ++ * ++ * Stub for when build against a kernel without debugfs support ++ */ ++static void __exit exit_kutf_core(void) ++{ ++} ++#endif /* defined(CONFIG_DEBUG_FS) */ ++ ++MODULE_LICENSE("GPL"); ++ ++module_init(init_kutf_core); ++module_exit(exit_kutf_core); +diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/kutf/kutf_utils.c b/drivers/gpu/arm/bifrost_for_linux/tests/kutf/kutf_utils.c +new file mode 100755 +index 000000000000..a429a2dbf788 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/tests/kutf/kutf_utils.c +@@ -0,0 +1,71 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* Kernel UTF utility functions */ ++ ++#include ++#include ++#include ++#include ++ ++#include ++#include ++ ++static char tmp_buffer[KUTF_MAX_DSPRINTF_LEN]; ++ ++DEFINE_MUTEX(buffer_lock); ++ ++const char *kutf_dsprintf(struct kutf_mempool *pool, ++ const char *fmt, ...) ++{ ++ va_list args; ++ int len; ++ int size; ++ void *buffer; ++ ++ mutex_lock(&buffer_lock); ++ va_start(args, fmt); ++ len = vsnprintf(tmp_buffer, sizeof(tmp_buffer), fmt, args); ++ va_end(args); ++ ++ if (len < 0) { ++ pr_err("kutf_dsprintf: Bad format dsprintf format %s\n", fmt); ++ goto fail_format; ++ } ++ ++ if (len >= sizeof(tmp_buffer)) { ++ pr_warn("kutf_dsprintf: Truncated dsprintf message %s\n", fmt); ++ size = sizeof(tmp_buffer); ++ } else { ++ size = len + 1; ++ } ++ ++ buffer = kutf_mempool_alloc(pool, size); ++ if (!buffer) ++ goto fail_alloc; ++ ++ memcpy(buffer, tmp_buffer, size); ++ mutex_unlock(&buffer_lock); ++ ++ return buffer; ++ ++fail_alloc: ++fail_format: ++ mutex_unlock(&buffer_lock); ++ return NULL; ++} ++EXPORT_SYMBOL(kutf_dsprintf); +diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/kutf/sconscript b/drivers/gpu/arm/bifrost_for_linux/tests/kutf/sconscript +new file mode 100755 +index 000000000000..d7f112448e42 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/tests/kutf/sconscript +@@ -0,0 +1,21 @@ ++# ++# (C) COPYRIGHT 2014-2016, 2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++Import('kutf_env') ++ ++make_args = kutf_env.kernel_get_config_defines(ret_list = True) ++ ++mod = kutf_env.BuildKernelModule('$STATIC_LIB_PATH/kutf.ko', Glob('*.c'), make_args = make_args) ++kutf_env.KernelObjTarget('kutf', mod) +diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/mali_kutf_irq_test/Kbuild b/drivers/gpu/arm/bifrost_for_linux/tests/mali_kutf_irq_test/Kbuild +new file mode 100755 +index 000000000000..0cd9cebe9d8b +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/tests/mali_kutf_irq_test/Kbuild +@@ -0,0 +1,20 @@ ++# ++# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++ccflags-y += -I$(src)/../include -I$(src)/../../../ -I$(src)/../../ -I$(src)/../../backend/gpu -I$(srctree)/drivers/staging/android ++ ++obj-$(CONFIG_MALI_IRQ_LATENCY) += mali_kutf_irq_test.o ++ ++mali_kutf_irq_test-y := mali_kutf_irq_test_main.o +diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/mali_kutf_irq_test/Kconfig b/drivers/gpu/arm/bifrost_for_linux/tests/mali_kutf_irq_test/Kconfig +new file mode 100755 +index 000000000000..4caa8ec8a0e2 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/tests/mali_kutf_irq_test/Kconfig +@@ -0,0 +1,23 @@ ++# ++# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++config MALI_IRQ_LATENCY ++ tristate "Mali GPU IRQ latency measurement" ++ depends on MALI_BIFROST && MALI_BIFROST_DEBUG && MALI_KUTF ++ default m ++ help ++ This option will build a test module mali_kutf_irq_test that ++ can determine the latency of the Mali GPU IRQ on your system. ++ Choosing M here will generate a single module called mali_kutf_irq_test. +diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/mali_kutf_irq_test/Makefile b/drivers/gpu/arm/bifrost_for_linux/tests/mali_kutf_irq_test/Makefile +new file mode 100755 +index 000000000000..ced37b08e532 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/tests/mali_kutf_irq_test/Makefile +@@ -0,0 +1,47 @@ ++# ++# (C) COPYRIGHT 2015, 2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++# linux build system bootstrap for out-of-tree module ++ ++# default to building for the host ++ARCH ?= $(shell uname -m) ++ ++ifeq ($(KDIR),) ++$(error Must specify KDIR to point to the kernel to target)) ++endif ++ ++TEST_CCFLAGS := \ ++ -DMALI_DEBUG=$(MALI_BIFROST_DEBUG) \ ++ -DMALI_BACKEND_KERNEL=$(MALI_BACKEND_KERNEL) \ ++ -DMALI_NO_MALI=$(MALI_BIFROST_NO_MALI) \ ++ -DMALI_UNIT_TEST=$(MALI_UNIT_TEST) \ ++ -DMALI_USE_UMP=$(MALI_USE_UMP) \ ++ -DMALI_ERROR_INJECT_ON=$(MALI_ERROR_INJECT_ON) \ ++ -DMALI_CUSTOMER_RELEASE=$(MALI_CUSTOMER_RELEASE) \ ++ $(SCONS_CFLAGS) \ ++ -I$(CURDIR)/../include \ ++ -I$(CURDIR)/../../../../../../include \ ++ -I$(CURDIR)/../../../ \ ++ -I$(CURDIR)/../../ \ ++ -I$(CURDIR)/../../backend/gpu \ ++ -I$(CURDIR)/ \ ++ -I$(srctree)/drivers/staging/android \ ++ -I$(srctree)/include/linux ++ ++all: ++ $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) $(SCONS_CONFIGS) EXTRA_CFLAGS="$(TEST_CCFLAGS)" KBUILD_EXTRA_SYMBOLS="$(CURDIR)/../kutf/Module.symvers $(CURDIR)/../../Module.symvers" modules ++ ++clean: ++ $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) clean +diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c b/drivers/gpu/arm/bifrost_for_linux/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c +new file mode 100755 +index 000000000000..c9cc4447cf37 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c +@@ -0,0 +1,269 @@ ++/* ++ * ++ * (C) COPYRIGHT 2016, 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include ++#include ++#include ++ ++#include "mali_kbase.h" ++#include ++ ++#include ++#include ++ ++/* ++ * This file contains the code which is used for measuring interrupt latency ++ * of the Mali GPU IRQ. In particular, function mali_kutf_irq_latency() is ++ * used with this purpose and it is called within KUTF framework - a kernel ++ * unit test framework. The measured latency provided by this test should ++ * be representative for the latency of the Mali JOB/MMU IRQs as well. ++ */ ++ ++/* KUTF test application pointer for this test */ ++struct kutf_application *irq_app; ++ ++/** ++ * struct kutf_irq_fixture data - test fixture used by the test functions. ++ * @kbdev: kbase device for the GPU. ++ * ++ */ ++struct kutf_irq_fixture_data { ++ struct kbase_device *kbdev; ++}; ++ ++#define SEC_TO_NANO(s) ((s)*1000000000LL) ++ ++/* ID for the GPU IRQ */ ++#define GPU_IRQ_HANDLER 2 ++ ++#define NR_TEST_IRQS 1000000 ++ ++/* IRQ for the test to trigger. Currently MULTIPLE_GPU_FAULTS as we would not ++ * expect to see this in normal use (e.g., when Android is running). */ ++#define TEST_IRQ MULTIPLE_GPU_FAULTS ++ ++#define IRQ_TIMEOUT HZ ++ ++/* Kernel API for setting irq throttle hook callback and irq time in us*/ ++extern int kbase_set_custom_irq_handler(struct kbase_device *kbdev, ++ irq_handler_t custom_handler, ++ int irq_type); ++extern irqreturn_t kbase_gpu_irq_handler(int irq, void *data); ++ ++static DECLARE_WAIT_QUEUE_HEAD(wait); ++static bool triggered; ++static u64 irq_time; ++ ++static void *kbase_untag(void *ptr) ++{ ++ return (void *)(((uintptr_t) ptr) & ~3); ++} ++ ++/** ++ * kbase_gpu_irq_custom_handler - Custom IRQ throttle handler ++ * @irq: IRQ number ++ * @data: Data associated with this IRQ ++ * ++ * Return: state of the IRQ ++ */ ++static irqreturn_t kbase_gpu_irq_custom_handler(int irq, void *data) ++{ ++ struct kbase_device *kbdev = kbase_untag(data); ++ u32 val; ++ ++ val = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_STATUS), NULL); ++ if (val & TEST_IRQ) { ++ struct timespec tval; ++ ++ getnstimeofday(&tval); ++ irq_time = SEC_TO_NANO(tval.tv_sec) + (tval.tv_nsec); ++ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), val, ++ NULL); ++ ++ triggered = true; ++ wake_up(&wait); ++ ++ return IRQ_HANDLED; ++ } ++ ++ /* Trigger main irq handler */ ++ return kbase_gpu_irq_handler(irq, data); ++} ++ ++/** ++ * mali_kutf_irq_default_create_fixture() - Creates the fixture data required ++ * for all the tests in the irq suite. ++ * @context: KUTF context. ++ * ++ * Return: Fixture data created on success or NULL on failure ++ */ ++static void *mali_kutf_irq_default_create_fixture( ++ struct kutf_context *context) ++{ ++ struct kutf_irq_fixture_data *data; ++ ++ data = kutf_mempool_alloc(&context->fixture_pool, ++ sizeof(struct kutf_irq_fixture_data)); ++ ++ if (!data) ++ goto fail; ++ ++ /* Acquire the kbase device */ ++ data->kbdev = kbase_find_device(-1); ++ if (data->kbdev == NULL) { ++ kutf_test_fail(context, "Failed to find kbase device"); ++ goto fail; ++ } ++ ++ return data; ++ ++fail: ++ return NULL; ++} ++ ++/** ++ * mali_kutf_irq_default_remove_fixture() - Destroy fixture data previously ++ * created by mali_kutf_irq_default_create_fixture. ++ * ++ * @context: KUTF context. ++ */ ++static void mali_kutf_irq_default_remove_fixture( ++ struct kutf_context *context) ++{ ++ struct kutf_irq_fixture_data *data = context->fixture; ++ struct kbase_device *kbdev = data->kbdev; ++ ++ kbase_release_device(kbdev); ++} ++ ++/** ++ * mali_kutf_irq_latency() - measure GPU IRQ latency ++ * @context: kutf context within which to perform the test ++ * ++ * The test triggers IRQs manually, and measures the ++ * time between triggering the IRQ and the IRQ handler being executed. ++ * ++ * This is not a traditional test, in that the pass/fail status has little ++ * meaning (other than indicating that the IRQ handler executed at all). Instead ++ * the results are in the latencies provided with the test result. There is no ++ * meaningful pass/fail result that can be obtained here, instead the latencies ++ * are provided for manual analysis only. ++ */ ++static void mali_kutf_irq_latency(struct kutf_context *context) ++{ ++ struct kutf_irq_fixture_data *data = context->fixture; ++ struct kbase_device *kbdev = data->kbdev; ++ u64 min_time = U64_MAX, max_time = 0, average_time = 0; ++ int i; ++ bool test_failed = false; ++ ++ /* Force GPU to be powered */ ++ kbase_pm_context_active(kbdev); ++ ++ kbase_set_custom_irq_handler(kbdev, kbase_gpu_irq_custom_handler, ++ GPU_IRQ_HANDLER); ++ ++ for (i = 0; i < NR_TEST_IRQS; i++) { ++ struct timespec tval; ++ u64 start_time; ++ int ret; ++ ++ triggered = false; ++ getnstimeofday(&tval); ++ start_time = SEC_TO_NANO(tval.tv_sec) + (tval.tv_nsec); ++ ++ /* Trigger fake IRQ */ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), ++ TEST_IRQ, NULL); ++ ++ ret = wait_event_timeout(wait, triggered != false, IRQ_TIMEOUT); ++ ++ if (ret == 0) { ++ kutf_test_fail(context, "Timed out waiting for IRQ\n"); ++ test_failed = true; ++ break; ++ } ++ ++ if ((irq_time - start_time) < min_time) ++ min_time = irq_time - start_time; ++ if ((irq_time - start_time) > max_time) ++ max_time = irq_time - start_time; ++ average_time += irq_time - start_time; ++ ++ udelay(10); ++ } ++ ++ /* Go back to default handler */ ++ kbase_set_custom_irq_handler(kbdev, NULL, GPU_IRQ_HANDLER); ++ ++ kbase_pm_context_idle(kbdev); ++ ++ if (!test_failed) { ++ const char *results; ++ ++ do_div(average_time, NR_TEST_IRQS); ++ results = kutf_dsprintf(&context->fixture_pool, ++ "Min latency = %lldns, Max latency = %lldns, Average latency = %lldns\n", ++ min_time, max_time, average_time); ++ kutf_test_pass(context, results); ++ } ++} ++ ++/** ++ * Module entry point for this test. ++ */ ++int mali_kutf_irq_test_main_init(void) ++{ ++ struct kutf_suite *suite; ++ ++ irq_app = kutf_create_application("irq"); ++ ++ if (NULL == irq_app) { ++ pr_warn("Creation of test application failed!\n"); ++ return -ENOMEM; ++ } ++ ++ suite = kutf_create_suite(irq_app, "irq_default", ++ 1, mali_kutf_irq_default_create_fixture, ++ mali_kutf_irq_default_remove_fixture); ++ ++ if (NULL == suite) { ++ pr_warn("Creation of test suite failed!\n"); ++ kutf_destroy_application(irq_app); ++ return -ENOMEM; ++ } ++ ++ kutf_add_test(suite, 0x0, "irq_latency", ++ mali_kutf_irq_latency); ++ return 0; ++} ++ ++/** ++ * Module exit point for this test. ++ */ ++void mali_kutf_irq_test_main_exit(void) ++{ ++ kutf_destroy_application(irq_app); ++} ++ ++module_init(mali_kutf_irq_test_main_init); ++module_exit(mali_kutf_irq_test_main_exit); ++ ++MODULE_LICENSE("GPL"); ++MODULE_AUTHOR("ARM Ltd."); ++MODULE_VERSION("1.0"); +diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/mali_kutf_irq_test/sconscript b/drivers/gpu/arm/bifrost_for_linux/tests/mali_kutf_irq_test/sconscript +new file mode 100755 +index 000000000000..b06d9ea32924 +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/tests/mali_kutf_irq_test/sconscript +@@ -0,0 +1,30 @@ ++# ++# (C) COPYRIGHT 2015, 2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++import os ++Import('env') ++ ++src = [Glob('#kernel/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/*.c'), Glob('#kernel/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Makefile')] ++ ++if env.GetOption('clean') : ++ env.Execute(Action("make clean", '[CLEAN] mali_kutf_irq_test')) ++ cmd = env.Command('$STATIC_LIB_PATH/mali_kutf_irq_test.ko', src, []) ++ env.KernelObjTarget('mali_kutf_irq_test', cmd) ++else: ++ makeAction=Action("cd ${SOURCE.dir} && make MALI_BIFROST_DEBUG=${debug} MALI_BACKEND_KERNEL=1 MALI_ERROR_INJECT_ON=${error_inject} MALI_BIFROST_NO_MALI=${no_mali} MALI_UNIT_TEST=${unit} MALI_USE_UMP=${ump} MALI_CUSTOMER_RELEASE=${release} %s && ( ( [ -f mali_kutf_irq_test.ko ] && cp mali_kutf_irq_test.ko $STATIC_LIB_PATH/ ) || touch $STATIC_LIB_PATH/mali_kutf_irq_test.ko)" % env.kernel_get_config_defines(), '$MAKECOMSTR') ++ cmd = env.Command('$STATIC_LIB_PATH/mali_kutf_irq_test.ko', src, [makeAction]) ++ env.Depends('$STATIC_LIB_PATH/mali_kutf_irq_test.ko', '$STATIC_LIB_PATH/kutf.ko') ++ env.Depends('$STATIC_LIB_PATH/mali_kutf_irq_test.ko', '$STATIC_LIB_PATH/mali_kbase.ko') ++ env.KernelObjTarget('mali_kutf_irq_test', cmd) +diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/sconscript b/drivers/gpu/arm/bifrost_for_linux/tests/sconscript +new file mode 100755 +index 000000000000..04584117ccef +--- /dev/null ++++ b/drivers/gpu/arm/bifrost_for_linux/tests/sconscript +@@ -0,0 +1,38 @@ ++# ++# (C) COPYRIGHT 2010-2011, 2013, 2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++Import ('env') ++ ++kutf_env = env.Clone() ++kutf_env.Append(CPPPATH = '#kernel/drivers/gpu/arm/midgard/tests/include') ++Export('kutf_env') ++ ++if Glob('internal/sconscript'): ++ SConscript('internal/sconscript') ++ ++if kutf_env['debug'] == '1': ++ SConscript('kutf/sconscript') ++ SConscript('mali_kutf_irq_test/sconscript') ++ ++ if Glob('kutf_test/sconscript'): ++ SConscript('kutf_test/sconscript') ++ ++ if Glob('kutf_test_runner/sconscript'): ++ SConscript('kutf_test_runner/sconscript') ++ ++if env['unit'] == '1': ++ SConscript('mali_kutf_ipa_test/sconscript') ++ SConscript('mali_kutf_ipa_unit_test/sconscript') ++ SConscript('mali_kutf_vinstr_test/sconscript') +diff --git a/drivers/gpu/arm/mali400/.gitignore b/drivers/gpu/arm/mali400/.gitignore +new file mode 100755 +index 000000000000..d91c8078a009 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/.gitignore +@@ -0,0 +1 @@ ++./mali/__malidrv_build_info.c +diff --git a/drivers/gpu/arm/mali400/Kbuild b/drivers/gpu/arm/mali400/Kbuild +new file mode 100755 +index 000000000000..dbb7ad3e5d85 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/Kbuild +@@ -0,0 +1,2 @@ ++# SPDX-License-Identifier: GPL-2.0 ++obj-y += mali/ +diff --git a/drivers/gpu/arm/mali400/mali/.gitignore b/drivers/gpu/arm/mali400/mali/.gitignore +new file mode 100755 +index 000000000000..6b1a3ed27a7f +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/.gitignore +@@ -0,0 +1 @@ ++__malidrv_build_info.c +diff --git a/drivers/gpu/arm/mali400/mali/Kbuild b/drivers/gpu/arm/mali400/mali/Kbuild +new file mode 100755 +index 000000000000..7390ab758f22 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/Kbuild +@@ -0,0 +1,254 @@ ++# ++# Copyright (C) 2010-2011 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++# as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained from Free Software ++# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++# ++ ++# This file is called by the Linux build system. ++ ++# make $(src) as absolute path if it isn't already, by prefixing $(srctree) ++src:=$(if $(patsubst /%,,$(src)),$(srctree)/$(src),$(src)) ++ ++# set up defaults if not defined by the user ++TIMESTAMP ?= default ++OS_MEMORY_KERNEL_BUFFER_SIZE_IN_MB ?= 16 ++USING_GPU_UTILIZATION ?= 1 ++PROFILING_SKIP_PP_JOBS ?= 0 ++PROFILING_SKIP_PP_AND_GP_JOBS ?= 0 ++MALI_PP_SCHEDULER_FORCE_NO_JOB_OVERLAP ?= 0 ++MALI_PP_SCHEDULER_KEEP_SUB_JOB_STARTS_ALIGNED ?= 0 ++MALI_PP_SCHEDULER_FORCE_NO_JOB_OVERLAP_BETWEEN_APPS ?= 0 ++MALI_UPPER_HALF_SCHEDULING ?= 1 ++MALI_ENABLE_CPU_CYCLES ?= 0 ++MALI_PLATFORM ?= rk ++ ++# For customer releases the Linux Device Drivers will be provided as ARM proprietary and GPL releases: ++# The ARM proprietary product will only include the license/proprietary directory ++# The GPL product will only include the license/gpl directory ++ccflags-y += -I$(src)/linux/license/gpl ++ ++ ++ifeq ($(USING_GPU_UTILIZATION), 1) ++ ifeq ($(USING_DVFS), 1) ++ $(error USING_GPU_UTILIZATION conflict with USING_DVFS you can read the Integration Guide to choose which one do you need) ++ endif ++endif ++ ++ifneq ($(MALI_PLATFORM),) ++ EXTRA_DEFINES += -DMALI_FAKE_PLATFORM_DEVICE=1 ++ #MALI_PLATFORM_FILES = $(wildcard platform/$(MALI_PLATFORM)/*.c) ++ mali-y += \ ++ platform/$(MALI_PLATFORM)/rk.o ++endif ++ ++ifeq ($(MALI_PLATFORM_FILES),) ++ifeq ($(CONFIG_ARCH_EXYNOS4),y) ++EXTRA_DEFINES += -DMALI_FAKE_PLATFORM_DEVICE=1 ++export MALI_PLATFORM=exynos4 ++export MALI_PLATFORM_FILES_BUILDIN = $(notdir $(wildcard $(src)/platform/$(MALI_PLATFORM)/*.c)) ++export MALI_PLATFORM_FILES_ADD_PREFIX = $(addprefix platform/$(MALI_PLATFORM)/,$(MALI_PLATFORM_FILES_BUILDIN)) ++endif ++endif ++ ++mali-y += \ ++ linux/mali_osk_atomics.o \ ++ linux/mali_osk_irq.o \ ++ linux/mali_osk_wq.o \ ++ linux/mali_osk_locks.o \ ++ linux/mali_osk_wait_queue.o \ ++ linux/mali_osk_low_level_mem.o \ ++ linux/mali_osk_math.o \ ++ linux/mali_osk_memory.o \ ++ linux/mali_osk_misc.o \ ++ linux/mali_osk_mali.o \ ++ linux/mali_osk_notification.o \ ++ linux/mali_osk_time.o \ ++ linux/mali_osk_timers.o \ ++ linux/mali_osk_bitmap.o ++ ++mali-y += linux/mali_memory.o linux/mali_memory_os_alloc.o ++mali-y += linux/mali_memory_external.o ++mali-y += linux/mali_memory_block_alloc.o ++mali-y += linux/mali_memory_swap_alloc.o ++ ++mali-y += \ ++ linux/mali_memory_manager.o \ ++ linux/mali_memory_virtual.o \ ++ linux/mali_memory_util.o \ ++ linux/mali_memory_cow.o \ ++ linux/mali_memory_defer_bind.o ++ ++mali-y += \ ++ linux/mali_ukk_mem.o \ ++ linux/mali_ukk_gp.o \ ++ linux/mali_ukk_pp.o \ ++ linux/mali_ukk_core.o \ ++ linux/mali_ukk_soft_job.o \ ++ linux/mali_ukk_timeline.o ++ ++mali-$(CONFIG_MALI_DEVFREQ) += \ ++ linux/mali_devfreq.o \ ++ common/mali_pm_metrics.o ++ ++# Source files which always are included in a build ++mali-y += \ ++ common/mali_kernel_core.o \ ++ linux/mali_kernel_linux.o \ ++ common/mali_session.o \ ++ linux/mali_device_pause_resume.o \ ++ common/mali_kernel_vsync.o \ ++ linux/mali_ukk_vsync.o \ ++ linux/mali_kernel_sysfs.o \ ++ common/mali_mmu.o \ ++ common/mali_mmu_page_directory.o \ ++ common/mali_mem_validation.o \ ++ common/mali_hw_core.o \ ++ common/mali_gp.o \ ++ common/mali_pp.o \ ++ common/mali_pp_job.o \ ++ common/mali_gp_job.o \ ++ common/mali_soft_job.o \ ++ common/mali_scheduler.o \ ++ common/mali_executor.o \ ++ common/mali_group.o \ ++ common/mali_dlbu.o \ ++ common/mali_broadcast.o \ ++ common/mali_pm.o \ ++ common/mali_pmu.o \ ++ common/mali_user_settings_db.o \ ++ common/mali_kernel_utilization.o \ ++ common/mali_control_timer.o \ ++ common/mali_l2_cache.o \ ++ common/mali_timeline.o \ ++ common/mali_timeline_fence_wait.o \ ++ common/mali_timeline_sync_fence.o \ ++ common/mali_spinlock_reentrant.o \ ++ common/mali_pm_domain.o \ ++ linux/mali_osk_pm.o \ ++ linux/mali_pmu_power_up_down.o \ ++ __malidrv_build_info.o ++ ++ifneq ($(wildcard $(src)/linux/mali_slp_global_lock.c),) ++ mali-y += linux/mali_slp_global_lock.o ++endif ++ ++ifneq ($(MALI_PLATFORM_FILES),) ++ mali-y += $(MALI_PLATFORM_FILES:.c=.o) ++endif ++ ++ifneq ($(MALI_PLATFORM_FILES_ADD_PREFIX),) ++ mali-y += $(MALI_PLATFORM_FILES_ADD_PREFIX:.c=.o) ++endif ++ ++mali-$(CONFIG_MALI400_PROFILING) += linux/mali_ukk_profiling.o ++mali-$(CONFIG_MALI400_PROFILING) += linux/mali_osk_profiling.o ++ ++mali-$(CONFIG_MALI400_INTERNAL_PROFILING) += linux/mali_profiling_internal.o timestamp-$(TIMESTAMP)/mali_timestamp.o ++ccflags-$(CONFIG_MALI400_INTERNAL_PROFILING) += -I$(src)/timestamp-$(TIMESTAMP) ++ ++mali-$(CONFIG_DMA_SHARED_BUFFER) += linux/mali_memory_dma_buf.o ++mali-$(CONFIG_DMA_SHARED_BUFFER) += linux/mali_memory_secure.o ++mali-$(CONFIG_SYNC) += linux/mali_sync.o ++mali-$(CONFIG_SYNC) += linux/mali_internal_sync.o ++mali-$(CONFIG_SYNC_FILE) += linux/mali_sync.o ++mali-$(CONFIG_SYNC_FILE) += linux/mali_internal_sync.o ++mali-$(CONFIG_MALI_DMA_BUF_FENCE) += linux/mali_dma_fence.o ++ccflags-$(CONFIG_SYNC) += -Idrivers/staging/android ++ccflags-$(CONFIG_SYNC_FILE) += -Idrivers/staging/android ++ ++mali-$(CONFIG_MALI400_UMP) += linux/mali_memory_ump.o ++ ++mali-$(CONFIG_MALI_DVFS) += common/mali_dvfs_policy.o ++ ++# Tell the Linux build system from which .o file to create the kernel module ++obj-$(CONFIG_MALI400) := mali.o ++ ++ccflags-y += $(EXTRA_DEFINES) ++ ++# Set up our defines, which will be passed to gcc ++ccflags-y += -DMALI_PP_SCHEDULER_FORCE_NO_JOB_OVERLAP=$(MALI_PP_SCHEDULER_FORCE_NO_JOB_OVERLAP) ++ccflags-y += -DMALI_PP_SCHEDULER_KEEP_SUB_JOB_STARTS_ALIGNED=$(MALI_PP_SCHEDULER_KEEP_SUB_JOB_STARTS_ALIGNED) ++ccflags-y += -DMALI_PP_SCHEDULER_FORCE_NO_JOB_OVERLAP_BETWEEN_APPS=$(MALI_PP_SCHEDULER_FORCE_NO_JOB_OVERLAP_BETWEEN_APPS) ++ccflags-y += -DMALI_STATE_TRACKING=1 ++ccflags-y += -DMALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_MB=$(OS_MEMORY_KERNEL_BUFFER_SIZE_IN_MB) ++ccflags-y += -DUSING_GPU_UTILIZATION=$(USING_GPU_UTILIZATION) ++ccflags-y += -DMALI_ENABLE_CPU_CYCLES=$(MALI_ENABLE_CPU_CYCLES) ++ ++ifeq ($(MALI_UPPER_HALF_SCHEDULING),1) ++ ccflags-y += -DMALI_UPPER_HALF_SCHEDULING ++endif ++ ++#build-in include path is different ++ifeq ($(MALI_PLATFORM_FILES),) ++ccflags-$(CONFIG_MALI400_UMP) += -I$(src)/../ump/include/ ++else ++ccflags-$(CONFIG_MALI400_UMP) += -I$(src)/../../ump/include/ump ++endif ++ccflags-$(CONFIG_MALI400_DEBUG) += -DDEBUG ++ ++# Use our defines when compiling ++ccflags-y += -I$(src) -I$(src)/include -I$(src)/common -I$(src)/linux -I$(src)/platform -Wno-date-time ++ ++# Get subversion revision number, fall back to only ${MALI_RELEASE_NAME} if no svn info is available ++MALI_RELEASE_NAME=$(shell cat $(src)/.version 2> /dev/null) ++ ++SVN_INFO = (cd $(src); svn info 2>/dev/null) ++ ++ifneq ($(shell $(SVN_INFO) 2>/dev/null),) ++# SVN detected ++SVN_REV := $(shell $(SVN_INFO) | grep '^Revision: '| sed -e 's/^Revision: //' 2>/dev/null) ++DRIVER_REV := $(MALI_RELEASE_NAME)-r$(SVN_REV) ++CHANGE_DATE := $(shell $(SVN_INFO) | grep '^Last Changed Date: ' | cut -d: -f2- | cut -b2-) ++CHANGED_REVISION := $(shell $(SVN_INFO) | grep '^Last Changed Rev: ' | cut -d: -f2- | cut -b2-) ++REPO_URL := $(shell $(SVN_INFO) | grep '^URL: ' | cut -d: -f2- | cut -b2-) ++ ++else # SVN ++# GIT_REV := $(shell cd $(src); git describe --always 2>/dev/null) ++ifneq ($(GIT_REV),) ++# Git detected ++DRIVER_REV := $(MALI_RELEASE_NAME)-$(GIT_REV) ++CHANGE_DATE := $(shell cd $(src); git log -1 --format="%ci") ++CHANGED_REVISION := $(GIT_REV) ++REPO_URL := $(shell cd $(src); git describe --all --always 2>/dev/null) ++ ++else # Git ++# No Git or SVN detected ++DRIVER_REV := $(MALI_RELEASE_NAME) ++CHANGE_DATE := $(MALI_RELEASE_NAME) ++CHANGED_REVISION := $(MALI_RELEASE_NAME) ++endif ++endif ++ ++ccflags-y += -DSVN_REV_STRING=\"$(DRIVER_REV)\" ++ ++VERSION_STRINGS := ++VERSION_STRINGS += API_VERSION=$(shell cd $(src); grep "\#define _MALI_API_VERSION" $(FILES_PREFIX)include/linux/mali/mali_utgard_uk_types.h | cut -d' ' -f 3 ) ++VERSION_STRINGS += REPO_URL=$(REPO_URL) ++VERSION_STRINGS += REVISION=$(DRIVER_REV) ++VERSION_STRINGS += CHANGED_REVISION=$(CHANGED_REVISION) ++VERSION_STRINGS += CHANGE_DATE=$(CHANGE_DATE) ++VERSION_STRINGS += BUILD_DATE=$(shell date) ++ifdef CONFIG_MALI400_DEBUG ++VERSION_STRINGS += BUILD=debug ++else ++VERSION_STRINGS += BUILD=release ++endif ++VERSION_STRINGS += TARGET_PLATFORM=$(TARGET_PLATFORM) ++VERSION_STRINGS += MALI_PLATFORM=$(MALI_PLATFORM) ++VERSION_STRINGS += KDIR=$(KDIR) ++VERSION_STRINGS += OS_MEMORY_KERNEL_BUFFER_SIZE_IN_MB=$(OS_MEMORY_KERNEL_BUFFER_SIZE_IN_MB) ++VERSION_STRINGS += USING_UMP=$(CONFIG_MALI400_UMP) ++VERSION_STRINGS += USING_PROFILING=$(CONFIG_MALI400_PROFILING) ++VERSION_STRINGS += USING_INTERNAL_PROFILING=$(CONFIG_MALI400_INTERNAL_PROFILING) ++VERSION_STRINGS += USING_GPU_UTILIZATION=$(USING_GPU_UTILIZATION) ++VERSION_STRINGS += USING_DVFS=$(CONFIG_MALI_DVFS) ++VERSION_STRINGS += USING_DMA_BUF_FENCE = $(CONFIG_MALI_DMA_BUF_FENCE) ++VERSION_STRINGS += MALI_UPPER_HALF_SCHEDULING=$(MALI_UPPER_HALF_SCHEDULING) ++ ++# Create file with Mali driver configuration ++$(src)/__malidrv_build_info.c: ++ @echo 'const char *__malidrv_build_info(void) { return "malidrv: $(VERSION_STRINGS)";}' > $(src)/__malidrv_build_info.c +diff --git a/drivers/gpu/arm/mali400/mali/Kconfig b/drivers/gpu/arm/mali400/mali/Kconfig +new file mode 100755 +index 000000000000..34c5f72edcda +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/Kconfig +@@ -0,0 +1,118 @@ ++# SPDX-License-Identifier: GPL-2.0 ++config MALI400 ++ tristate "Mali-300/400/450 support" ++ depends on ARM || ARM64 ++ select DMA_SHARED_BUFFER ++ help ++ This enables support for the ARM Mali-300, Mali-400, and Mali-450 ++ GPUs. ++ ++ To compile this driver as a module, choose M here: the module will be ++ called mali. ++ ++config MALI450 ++ bool "Enable Mali-450 support" ++ depends on MALI400 ++ help ++ This enables support for Mali-450 specific features. ++ ++config MALI470 ++ bool "Enable Mali-470 support" ++ depends on MALI400 ++ help ++ This enables support for Mali-470 specific features. ++ ++config MALI400_DEBUG ++ bool "Enable debug in Mali driver" ++ depends on MALI400 ++ help ++ This enabled extra debug checks and messages in the Mali driver. ++ ++config MALI400_PROFILING ++ bool "Enable Mali profiling" ++ depends on MALI400 ++ select TRACEPOINTS ++ default y ++ help ++ This enables gator profiling of Mali GPU events. ++ ++config MALI400_INTERNAL_PROFILING ++ bool "Enable internal Mali profiling API" ++ depends on MALI400_PROFILING ++ default n ++ help ++ This enables the internal legacy Mali profiling API. ++ ++config MALI400_UMP ++ bool "Enable UMP support" ++ depends on MALI400 ++ help ++ This enables support for the UMP memory sharing API in the Mali driver. ++ ++config MALI_DVFS ++ bool "Enable Mali dynamically frequency change" ++ depends on MALI400 && !MALI_DEVFREQ ++ default y ++ help ++ This enables support for dynamic change frequency of Mali with the goal of lowering power consumption. ++ ++config MALI_DMA_BUF_MAP_ON_ATTACH ++ bool "Map dma-buf attachments on attach" ++ depends on MALI400 && DMA_SHARED_BUFFER ++ default y ++ help ++ This makes the Mali driver map dma-buf attachments after doing ++ attach. If this is not set the dma-buf attachments will be mapped for ++ every time the GPU need to access the buffer. ++ ++ Mapping for each access can cause lower performance. ++ ++config MALI_SHARED_INTERRUPTS ++ bool "Support for shared interrupts" ++ depends on MALI400 ++ default n ++ help ++ Adds functionality required to properly support shared interrupts. Without this support, ++ the device driver will fail during insmod if it detects shared interrupts. This also ++ works when the GPU is not using shared interrupts, but might have a slight performance ++ impact. ++ ++config MALI_PMU_PARALLEL_POWER_UP ++ bool "Power up Mali PMU domains in parallel" ++ depends on MALI400 ++ default n ++ help ++ This makes the Mali driver power up all PMU power domains in parallel, instead of ++ powering up domains one by one, with a slight delay in between. Powering on all power ++ domains at the same time may cause peak currents higher than what some systems can handle. ++ These systems must not enable this option. ++ ++config MALI_DT ++ bool "Using device tree to initialize module" ++ depends on MALI400 && OF ++ default n ++ help ++ This enable the Mali driver to choose the device tree path to get platform resoures ++ and disable the old config method. Mali driver could run on the platform which the ++ device tree is enabled in kernel and corresponding hardware description is implemented ++ properly in device DTS file. ++ ++config MALI_DEVFREQ ++ bool "Using devfreq to tuning frequency" ++ depends on MALI400 && PM_DEVFREQ ++ default n ++ help ++ Support devfreq for Mali. ++ ++ Using the devfreq framework and, by default, the simpleondemand ++ governor, the frequency of Mali will be dynamically selected from the ++ available OPPs. ++ ++config MALI_QUIET ++ bool "Make Mali driver very quiet" ++ depends on MALI400 && !MALI400_DEBUG ++ default n ++ help ++ This forces the Mali driver to never print any messages. ++ ++ If unsure, say N. +diff --git a/drivers/gpu/arm/mali400/mali/Makefile b/drivers/gpu/arm/mali400/mali/Makefile +new file mode 100755 +index 000000000000..0b91321a5af1 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/Makefile +@@ -0,0 +1,206 @@ ++# ++# Copyright (C) 2010-2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++# as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained from Free Software ++# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++# ++ ++USE_UMPV2=0 ++USING_PROFILING ?= 1 ++USING_INTERNAL_PROFILING ?= 0 ++USING_DVFS ?= 1 ++USING_DMA_BUF_FENCE ?= 0 ++MALI_HEATMAPS_ENABLED ?= 0 ++MALI_DMA_BUF_MAP_ON_ATTACH ?= 1 ++MALI_PMU_PARALLEL_POWER_UP ?= 0 ++USING_DT ?= 0 ++MALI_MEM_SWAP_TRACKING ?= 0 ++USING_DEVFREQ ?= 0 ++ ++# The Makefile sets up "arch" based on the CONFIG, creates the version info ++# string and the __malidrv_build_info.c file, and then call the Linux build ++# system to actually build the driver. After that point the Kbuild file takes ++# over. ++ ++# set up defaults if not defined by the user ++ARCH ?= arm ++ ++OSKOS=linux ++FILES_PREFIX= ++ ++check_cc2 = \ ++ $(shell if $(1) -S -o /dev/null -xc /dev/null > /dev/null 2>&1; \ ++ then \ ++ echo "$(2)"; \ ++ else \ ++ echo "$(3)"; \ ++ fi ;) ++ ++# This conditional makefile exports the global definition ARM_INTERNAL_BUILD. Customer releases will not include arm_internal.mak ++-include ../../../arm_internal.mak ++ ++# Give warning of old config parameters are used ++ifneq ($(CONFIG),) ++$(warning "You have specified the CONFIG variable which is no longer in used. Use TARGET_PLATFORM instead.") ++endif ++ ++ifneq ($(CPU),) ++$(warning "You have specified the CPU variable which is no longer in used. Use TARGET_PLATFORM instead.") ++endif ++ ++# Include the mapping between TARGET_PLATFORM and KDIR + MALI_PLATFORM ++-include MALI_CONFIGURATION ++export KDIR ?= $(KDIR-$(TARGET_PLATFORM)) ++export MALI_PLATFORM ?= $(MALI_PLATFORM-$(TARGET_PLATFORM)) ++ ++ifneq ($(TARGET_PLATFORM),) ++ifeq ($(MALI_PLATFORM),) ++$(error "Invalid TARGET_PLATFORM: $(TARGET_PLATFORM)") ++endif ++endif ++ ++# validate lookup result ++ifeq ($(KDIR),) ++$(error No KDIR found for platform $(TARGET_PLATFORM)) ++endif ++ ++ifeq ($(USING_GPU_UTILIZATION), 1) ++ ifeq ($(USING_DVFS), 1) ++ $(error USING_GPU_UTILIZATION conflict with USING_DVFS you can read the Integration Guide to choose which one do you need) ++ endif ++endif ++ ++ifeq ($(USING_UMP),1) ++export CONFIG_MALI400_UMP=y ++export EXTRA_DEFINES += -DCONFIG_MALI400_UMP=1 ++ifeq ($(USE_UMPV2),1) ++UMP_SYMVERS_FILE ?= ../umpv2/Module.symvers ++else ++UMP_SYMVERS_FILE ?= ../ump/Module.symvers ++endif ++KBUILD_EXTRA_SYMBOLS = $(realpath $(UMP_SYMVERS_FILE)) ++$(warning $(KBUILD_EXTRA_SYMBOLS)) ++endif ++ ++# Define host system directory ++KDIR-$(shell uname -m):=/lib/modules/$(shell uname -r)/build ++ ++include $(KDIR)/.config ++ ++ifeq ($(ARCH), arm) ++# when compiling for ARM we're cross compiling ++export CROSS_COMPILE ?= $(call check_cc2, arm-linux-gnueabi-gcc, arm-linux-gnueabi-, arm-none-linux-gnueabi-) ++endif ++ ++# report detected/selected settings ++ifdef ARM_INTERNAL_BUILD ++$(warning TARGET_PLATFORM $(TARGET_PLATFORM)) ++$(warning KDIR $(KDIR)) ++$(warning MALI_PLATFORM $(MALI_PLATFORM)) ++endif ++ ++# Set up build config ++export CONFIG_MALI400=m ++export CONFIG_MALI450=y ++export CONFIG_MALI470=y ++ ++export EXTRA_DEFINES += -DCONFIG_MALI400=1 ++export EXTRA_DEFINES += -DCONFIG_MALI450=1 ++export EXTRA_DEFINES += -DCONFIG_MALI470=1 ++ ++ifneq ($(MALI_PLATFORM),) ++export EXTRA_DEFINES += -DMALI_FAKE_PLATFORM_DEVICE=1 ++export MALI_PLATFORM_FILES = $(wildcard platform/$(MALI_PLATFORM)/*.c) ++endif ++ ++ifeq ($(USING_PROFILING),1) ++ifeq ($(CONFIG_TRACEPOINTS),) ++$(warning CONFIG_TRACEPOINTS required for profiling) ++else ++export CONFIG_MALI400_PROFILING=y ++export EXTRA_DEFINES += -DCONFIG_MALI400_PROFILING=1 ++ifeq ($(USING_INTERNAL_PROFILING),1) ++export CONFIG_MALI400_INTERNAL_PROFILING=y ++export EXTRA_DEFINES += -DCONFIG_MALI400_INTERNAL_PROFILING=1 ++endif ++ifeq ($(MALI_HEATMAPS_ENABLED),1) ++export MALI_HEATMAPS_ENABLED=y ++export EXTRA_DEFINES += -DCONFIG_MALI400_HEATMAPS_ENABLED ++endif ++endif ++endif ++ ++ifeq ($(MALI_DMA_BUF_MAP_ON_ATTACH),1) ++export CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH=y ++export EXTRA_DEFINES += -DCONFIG_MALI_DMA_BUF_MAP_ON_ATTACH ++endif ++ ++ifeq ($(MALI_SHARED_INTERRUPTS),1) ++export CONFIG_MALI_SHARED_INTERRUPTS=y ++export EXTRA_DEFINES += -DCONFIG_MALI_SHARED_INTERRUPTS ++endif ++ ++ifeq ($(USING_DVFS),1) ++export CONFIG_MALI_DVFS=y ++export EXTRA_DEFINES += -DCONFIG_MALI_DVFS ++endif ++ ++ifeq ($(USING_DMA_BUF_FENCE),1) ++export CONFIG_MALI_DMA_BUF_FENCE=y ++export EXTRA_DEFINES += -DCONFIG_MALI_DMA_BUF_FENCE ++endif ++ ++ifeq ($(MALI_PMU_PARALLEL_POWER_UP),1) ++export CONFIG_MALI_PMU_PARALLEL_POWER_UP=y ++export EXTRA_DEFINES += -DCONFIG_MALI_PMU_PARALLEL_POWER_UP ++endif ++ ++ifdef CONFIG_OF ++ifeq ($(USING_DT),1) ++export CONFIG_MALI_DT=y ++export EXTRA_DEFINES += -DCONFIG_MALI_DT ++endif ++endif ++ ++ifeq ($(USING_DEVFREQ), 1) ++ifdef CONFIG_PM_DEVFREQ ++export CONFIG_MALI_DEVFREQ=y ++export EXTRA_DEFINES += -DCONFIG_MALI_DEVFREQ=1 ++else ++$(warning "You want to support DEVFREQ but kernel didn't support DEVFREQ.") ++endif ++endif ++ ++ifneq ($(BUILD),release) ++# Debug ++export CONFIG_MALI400_DEBUG=y ++else ++# Release ++ifeq ($(MALI_QUIET),1) ++export CONFIG_MALI_QUIET=y ++export EXTRA_DEFINES += -DCONFIG_MALI_QUIET ++endif ++endif ++ ++ifeq ($(MALI_SKIP_JOBS),1) ++EXTRA_DEFINES += -DPROFILING_SKIP_PP_JOBS=1 -DPROFILING_SKIP_GP_JOBS=1 ++endif ++ ++ifeq ($(MALI_MEM_SWAP_TRACKING),1) ++EXTRA_DEFINES += -DMALI_MEM_SWAP_TRACKING=1 ++endif ++ ++all: $(UMP_SYMVERS_FILE) ++ $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) modules ++ @rm $(FILES_PREFIX)__malidrv_build_info.c $(FILES_PREFIX)__malidrv_build_info.o ++ ++clean: ++ $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) clean ++ ++kernelrelease: ++ $(MAKE) ARCH=$(ARCH) -C $(KDIR) kernelrelease ++ ++export CONFIG KBUILD_EXTRA_SYMBOLS +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_broadcast.c b/drivers/gpu/arm/mali400/mali/common/mali_broadcast.c +new file mode 100755 +index 000000000000..79a418c36ccb +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_broadcast.c +@@ -0,0 +1,142 @@ ++/* ++ * Copyright (C) 2012-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include "mali_broadcast.h" ++#include "mali_kernel_common.h" ++#include "mali_osk.h" ++ ++#define MALI_BROADCAST_REGISTER_SIZE 0x1000 ++#define MALI_BROADCAST_REG_BROADCAST_MASK 0x0 ++#define MALI_BROADCAST_REG_INTERRUPT_MASK 0x4 ++ ++struct mali_bcast_unit { ++ struct mali_hw_core hw_core; ++ u32 current_mask; ++}; ++ ++struct mali_bcast_unit *mali_bcast_unit_create(const _mali_osk_resource_t *resource) ++{ ++ struct mali_bcast_unit *bcast_unit = NULL; ++ ++ MALI_DEBUG_ASSERT_POINTER(resource); ++ MALI_DEBUG_PRINT(2, ("Broadcast: Creating Mali Broadcast unit: %s\n", ++ resource->description)); ++ ++ bcast_unit = _mali_osk_malloc(sizeof(struct mali_bcast_unit)); ++ if (NULL == bcast_unit) { ++ MALI_PRINT_ERROR(("Broadcast: Failed to allocate memory for Broadcast unit\n")); ++ return NULL; ++ } ++ ++ if (_MALI_OSK_ERR_OK == mali_hw_core_create(&bcast_unit->hw_core, ++ resource, MALI_BROADCAST_REGISTER_SIZE)) { ++ bcast_unit->current_mask = 0; ++ mali_bcast_reset(bcast_unit); ++ ++ return bcast_unit; ++ } else { ++ MALI_PRINT_ERROR(("Broadcast: Failed map broadcast unit\n")); ++ } ++ ++ _mali_osk_free(bcast_unit); ++ ++ return NULL; ++} ++ ++void mali_bcast_unit_delete(struct mali_bcast_unit *bcast_unit) ++{ ++ MALI_DEBUG_ASSERT_POINTER(bcast_unit); ++ mali_hw_core_delete(&bcast_unit->hw_core); ++ _mali_osk_free(bcast_unit); ++} ++ ++/* Call this function to add the @group's id into bcast mask ++ * Note: redundant calling this function with same @group ++ * doesn't make any difference as calling it once ++ */ ++void mali_bcast_add_group(struct mali_bcast_unit *bcast_unit, ++ struct mali_group *group) ++{ ++ u32 bcast_id; ++ u32 broadcast_mask; ++ ++ MALI_DEBUG_ASSERT_POINTER(bcast_unit); ++ MALI_DEBUG_ASSERT_POINTER(group); ++ ++ bcast_id = mali_pp_core_get_bcast_id(mali_group_get_pp_core(group)); ++ ++ broadcast_mask = bcast_unit->current_mask; ++ ++ broadcast_mask |= (bcast_id); /* add PP core to broadcast */ ++ broadcast_mask |= (bcast_id << 16); /* add MMU to broadcast */ ++ ++ /* store mask so we can restore on reset */ ++ bcast_unit->current_mask = broadcast_mask; ++} ++ ++/* Call this function to remove @group's id from bcast mask ++ * Note: redundant calling this function with same @group ++ * doesn't make any difference as calling it once ++ */ ++void mali_bcast_remove_group(struct mali_bcast_unit *bcast_unit, ++ struct mali_group *group) ++{ ++ u32 bcast_id; ++ u32 broadcast_mask; ++ ++ MALI_DEBUG_ASSERT_POINTER(bcast_unit); ++ MALI_DEBUG_ASSERT_POINTER(group); ++ ++ bcast_id = mali_pp_core_get_bcast_id(mali_group_get_pp_core(group)); ++ ++ broadcast_mask = bcast_unit->current_mask; ++ ++ broadcast_mask &= ~((bcast_id << 16) | bcast_id); ++ ++ /* store mask so we can restore on reset */ ++ bcast_unit->current_mask = broadcast_mask; ++} ++ ++void mali_bcast_reset(struct mali_bcast_unit *bcast_unit) ++{ ++ MALI_DEBUG_ASSERT_POINTER(bcast_unit); ++ ++ MALI_DEBUG_PRINT(4, ++ ("Broadcast: setting mask 0x%08X + 0x%08X (reset)\n", ++ bcast_unit->current_mask, ++ bcast_unit->current_mask & 0xFF)); ++ ++ /* set broadcast mask */ ++ mali_hw_core_register_write(&bcast_unit->hw_core, ++ MALI_BROADCAST_REG_BROADCAST_MASK, ++ bcast_unit->current_mask); ++ ++ /* set IRQ override mask */ ++ mali_hw_core_register_write(&bcast_unit->hw_core, ++ MALI_BROADCAST_REG_INTERRUPT_MASK, ++ bcast_unit->current_mask & 0xFF); ++} ++ ++void mali_bcast_disable(struct mali_bcast_unit *bcast_unit) ++{ ++ MALI_DEBUG_ASSERT_POINTER(bcast_unit); ++ ++ MALI_DEBUG_PRINT(4, ("Broadcast: setting mask 0x0 + 0x0 (disable)\n")); ++ ++ /* set broadcast mask */ ++ mali_hw_core_register_write(&bcast_unit->hw_core, ++ MALI_BROADCAST_REG_BROADCAST_MASK, ++ 0x0); ++ ++ /* set IRQ override mask */ ++ mali_hw_core_register_write(&bcast_unit->hw_core, ++ MALI_BROADCAST_REG_INTERRUPT_MASK, ++ 0x0); ++} +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_broadcast.h b/drivers/gpu/arm/mali400/mali/common/mali_broadcast.h +new file mode 100755 +index 000000000000..0475b7171d8d +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_broadcast.h +@@ -0,0 +1,57 @@ ++/* ++ * Copyright (C) 2012-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_BROADCAST_H__ ++#define __MALI_BROADCAST_H__ ++ ++/* ++ * Interface for the broadcast unit on Mali-450. ++ * ++ * - Represents up to 8 × (MMU + PP) pairs. ++ * - Supports dynamically changing which (MMU + PP) pairs receive the broadcast by ++ * setting a mask. ++ */ ++ ++#include "mali_hw_core.h" ++#include "mali_group.h" ++ ++struct mali_bcast_unit; ++ ++struct mali_bcast_unit *mali_bcast_unit_create(const _mali_osk_resource_t *resource); ++void mali_bcast_unit_delete(struct mali_bcast_unit *bcast_unit); ++ ++/* Add a group to the list of (MMU + PP) pairs broadcasts go out to. */ ++void mali_bcast_add_group(struct mali_bcast_unit *bcast_unit, struct mali_group *group); ++ ++/* Remove a group to the list of (MMU + PP) pairs broadcasts go out to. */ ++void mali_bcast_remove_group(struct mali_bcast_unit *bcast_unit, struct mali_group *group); ++ ++/* Re-set cached mask. This needs to be called after having been suspended. */ ++void mali_bcast_reset(struct mali_bcast_unit *bcast_unit); ++ ++/** ++ * Disable broadcast unit ++ * ++ * mali_bcast_enable must be called to re-enable the unit. Cores may not be ++ * added or removed when the unit is disabled. ++ */ ++void mali_bcast_disable(struct mali_bcast_unit *bcast_unit); ++ ++/** ++ * Re-enable broadcast unit ++ * ++ * This resets the masks to include the cores present when mali_bcast_disable was called. ++ */ ++MALI_STATIC_INLINE void mali_bcast_enable(struct mali_bcast_unit *bcast_unit) ++{ ++ mali_bcast_reset(bcast_unit); ++} ++ ++#endif /* __MALI_BROADCAST_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_control_timer.c b/drivers/gpu/arm/mali400/mali/common/mali_control_timer.c +new file mode 100755 +index 000000000000..5bed27a8c5c9 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_control_timer.c +@@ -0,0 +1,139 @@ ++/* ++ * Copyright (C) 2010-2012, 2014-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include "mali_kernel_utilization.h" ++#include "mali_osk.h" ++#include "mali_osk_mali.h" ++#include "mali_kernel_common.h" ++#include "mali_session.h" ++#include "mali_dvfs_policy.h" ++#include "mali_control_timer.h" ++ ++static u64 period_start_time = 0; ++ ++/** .KP : mali_control_timer */ ++static _mali_osk_timer_t *mali_control_timer = NULL; ++static mali_bool timer_running = MALI_FALSE; ++ ++/** ++ * period_of_notifying_mali_utilization_to_platform_dependent_part, ++ * ms 为å•ä½. ++ */ ++static u32 mali_control_timeout = 20; ++ ++void mali_control_timer_add(u32 timeout)/* 'timeout' : 以 ms 为å•ä½. */ ++{ ++ _mali_osk_timer_add(mali_control_timer, _mali_osk_time_mstoticks(timeout)); ++} ++ ++void mali_control_timer_mod(u32 timeout_in_ms) ++{ ++ _mali_osk_timer_mod(mali_control_timer, _mali_osk_time_mstoticks(timeout_in_ms)); ++} ++ ++static void mali_control_timer_callback(void *arg) ++{ ++ if (mali_utilization_enabled()) { ++ struct mali_gpu_utilization_data *util_data = NULL; ++ u64 time_period = 0; ++ mali_bool need_add_timer = MALI_TRUE; ++ ++ /* Calculate gpu utilization */ ++ util_data = mali_utilization_calculate(&period_start_time, &time_period, &need_add_timer); ++ ++ if (util_data) { ++#if defined(CONFIG_MALI_DVFS) ++ mali_dvfs_policy_realize(util_data, time_period); ++#else ++ mali_utilization_platform_realize(util_data); ++#endif ++ ++ if (MALI_TRUE == timer_running) ++ if (MALI_TRUE == need_add_timer) { ++ mali_control_timer_mod(mali_control_timeout); ++ } ++ } ++ } ++} ++ ++/* Init a timer (for now it is used for GPU utilization and dvfs) */ ++_mali_osk_errcode_t mali_control_timer_init(void) ++{ ++ _mali_osk_device_data data; ++ ++ if (_MALI_OSK_ERR_OK == _mali_osk_device_data_get(&data)) { ++ /* Use device specific settings (if defined) */ ++ if (0 != data.control_interval) { ++ mali_control_timeout = data.control_interval; ++ MALI_DEBUG_PRINT(2, ("Mali GPU Timer: %u\n", mali_control_timeout)); ++ } ++ } ++ ++ mali_control_timer = _mali_osk_timer_init(mali_control_timer_callback); ++ if (NULL == mali_control_timer) { ++ return _MALI_OSK_ERR_FAULT; ++ } ++ _mali_osk_timer_setcallback(mali_control_timer, mali_control_timer_callback, NULL); ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++void mali_control_timer_term(void) ++{ ++ if (NULL != mali_control_timer) { ++ _mali_osk_timer_del(mali_control_timer); ++ timer_running = MALI_FALSE; ++ _mali_osk_timer_term(mali_control_timer); ++ mali_control_timer = NULL; ++ } ++} ++ ++mali_bool mali_control_timer_resume(u64 time_now) ++{ ++ mali_utilization_data_assert_locked(); ++ ++ if (timer_running != MALI_TRUE) { ++ timer_running = MALI_TRUE; ++ ++ period_start_time = time_now; ++ ++ mali_utilization_reset(); ++ ++ return MALI_TRUE; ++ } ++ ++ return MALI_FALSE; ++} ++ ++void mali_control_timer_pause(void) ++{ ++ mali_utilization_data_assert_locked(); ++ if (timer_running == MALI_TRUE) { ++ timer_running = MALI_FALSE; ++ } ++} ++ ++void mali_control_timer_suspend(mali_bool suspend) ++{ ++ mali_utilization_data_lock(); ++ ++ if (timer_running == MALI_TRUE) { ++ timer_running = MALI_FALSE; ++ ++ mali_utilization_data_unlock(); ++ ++ if (suspend == MALI_TRUE) { ++ _mali_osk_timer_del(mali_control_timer); ++ mali_utilization_reset(); ++ } ++ } else { ++ mali_utilization_data_unlock(); ++ } ++} +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_control_timer.h b/drivers/gpu/arm/mali400/mali/common/mali_control_timer.h +new file mode 100755 +index 000000000000..c9e6e058ea8e +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_control_timer.h +@@ -0,0 +1,30 @@ ++/* ++ * Copyright (C) 2010-2012, 2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_CONTROL_TIMER_H__ ++#define __MALI_CONTROL_TIMER_H__ ++ ++#include "mali_osk.h" ++ ++_mali_osk_errcode_t mali_control_timer_init(void); ++ ++void mali_control_timer_term(void); ++ ++mali_bool mali_control_timer_resume(u64 time_now); ++ ++void mali_control_timer_suspend(mali_bool suspend); ++void mali_control_timer_pause(void); ++ ++void mali_control_timer_add(u32 timeout); ++ ++void mali_control_timer_mod(u32 timeout_in_ms); ++ ++#endif /* __MALI_CONTROL_TIMER_H__ */ ++ +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_dlbu.c b/drivers/gpu/arm/mali400/mali/common/mali_dlbu.c +new file mode 100755 +index 000000000000..99b7f360768b +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_dlbu.c +@@ -0,0 +1,213 @@ ++/* ++ * Copyright (C) 2012-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include "mali_dlbu.h" ++#include "mali_memory.h" ++#include "mali_pp.h" ++#include "mali_group.h" ++#include "mali_osk.h" ++#include "mali_hw_core.h" ++ ++/** ++ * Size of DLBU registers in bytes ++ */ ++#define MALI_DLBU_SIZE 0x400 ++ ++mali_dma_addr mali_dlbu_phys_addr = 0; ++static mali_io_address mali_dlbu_cpu_addr = NULL; ++ ++/** ++ * DLBU register numbers ++ * Used in the register read/write routines. ++ * See the hardware documentation for more information about each register ++ */ ++typedef enum mali_dlbu_register { ++ MALI_DLBU_REGISTER_MASTER_TLLIST_PHYS_ADDR = 0x0000, /**< Master tile list physical base address; ++ 31:12 Physical address to the page used for the DLBU ++ 0 DLBU enable - set this bit to 1 enables the AXI bus ++ between PPs and L2s, setting to 0 disables the router and ++ no further transactions are sent to DLBU */ ++ MALI_DLBU_REGISTER_MASTER_TLLIST_VADDR = 0x0004, /**< Master tile list virtual base address; ++ 31:12 Virtual address to the page used for the DLBU */ ++ MALI_DLBU_REGISTER_TLLIST_VBASEADDR = 0x0008, /**< Tile list virtual base address; ++ 31:12 Virtual address to the tile list. This address is used when ++ calculating the call address sent to PP.*/ ++ MALI_DLBU_REGISTER_FB_DIM = 0x000C, /**< Framebuffer dimension; ++ 23:16 Number of tiles in Y direction-1 ++ 7:0 Number of tiles in X direction-1 */ ++ MALI_DLBU_REGISTER_TLLIST_CONF = 0x0010, /**< Tile list configuration; ++ 29:28 select the size of each allocated block: 0=128 bytes, 1=256, 2=512, 3=1024 ++ 21:16 2^n number of tiles to be binned to one tile list in Y direction ++ 5:0 2^n number of tiles to be binned to one tile list in X direction */ ++ MALI_DLBU_REGISTER_START_TILE_POS = 0x0014, /**< Start tile positions; ++ 31:24 start position in Y direction for group 1 ++ 23:16 start position in X direction for group 1 ++ 15:8 start position in Y direction for group 0 ++ 7:0 start position in X direction for group 0 */ ++ MALI_DLBU_REGISTER_PP_ENABLE_MASK = 0x0018, /**< PP enable mask; ++ 7 enable PP7 for load balancing ++ 6 enable PP6 for load balancing ++ 5 enable PP5 for load balancing ++ 4 enable PP4 for load balancing ++ 3 enable PP3 for load balancing ++ 2 enable PP2 for load balancing ++ 1 enable PP1 for load balancing ++ 0 enable PP0 for load balancing */ ++} mali_dlbu_register; ++ ++typedef enum { ++ PP0ENABLE = 0, ++ PP1ENABLE, ++ PP2ENABLE, ++ PP3ENABLE, ++ PP4ENABLE, ++ PP5ENABLE, ++ PP6ENABLE, ++ PP7ENABLE ++} mali_dlbu_pp_enable; ++ ++struct mali_dlbu_core { ++ struct mali_hw_core hw_core; /**< Common for all HW cores */ ++ u32 pp_cores_mask; /**< This is a mask for the PP cores whose operation will be controlled by LBU ++ see MALI_DLBU_REGISTER_PP_ENABLE_MASK register */ ++}; ++ ++_mali_osk_errcode_t mali_dlbu_initialize(void) ++{ ++ MALI_DEBUG_PRINT(2, ("Mali DLBU: Initializing\n")); ++ ++ if (_MALI_OSK_ERR_OK == ++ mali_mmu_get_table_page(&mali_dlbu_phys_addr, ++ &mali_dlbu_cpu_addr)) { ++ return _MALI_OSK_ERR_OK; ++ } ++ ++ return _MALI_OSK_ERR_FAULT; ++} ++ ++void mali_dlbu_terminate(void) ++{ ++ MALI_DEBUG_PRINT(3, ("Mali DLBU: terminating\n")); ++ ++ if (0 != mali_dlbu_phys_addr && 0 != mali_dlbu_cpu_addr) { ++ mali_mmu_release_table_page(mali_dlbu_phys_addr, ++ mali_dlbu_cpu_addr); ++ mali_dlbu_phys_addr = 0; ++ mali_dlbu_cpu_addr = 0; ++ } ++} ++ ++struct mali_dlbu_core *mali_dlbu_create(const _mali_osk_resource_t *resource) ++{ ++ struct mali_dlbu_core *core = NULL; ++ ++ MALI_DEBUG_PRINT(2, ("Mali DLBU: Creating Mali dynamic load balancing unit: %s\n", resource->description)); ++ ++ core = _mali_osk_malloc(sizeof(struct mali_dlbu_core)); ++ if (NULL != core) { ++ if (_MALI_OSK_ERR_OK == mali_hw_core_create(&core->hw_core, resource, MALI_DLBU_SIZE)) { ++ core->pp_cores_mask = 0; ++ if (_MALI_OSK_ERR_OK == mali_dlbu_reset(core)) { ++ return core; ++ } ++ MALI_PRINT_ERROR(("Failed to reset DLBU %s\n", core->hw_core.description)); ++ mali_hw_core_delete(&core->hw_core); ++ } ++ ++ _mali_osk_free(core); ++ } else { ++ MALI_PRINT_ERROR(("Mali DLBU: Failed to allocate memory for DLBU core\n")); ++ } ++ ++ return NULL; ++} ++ ++void mali_dlbu_delete(struct mali_dlbu_core *dlbu) ++{ ++ MALI_DEBUG_ASSERT_POINTER(dlbu); ++ mali_hw_core_delete(&dlbu->hw_core); ++ _mali_osk_free(dlbu); ++} ++ ++_mali_osk_errcode_t mali_dlbu_reset(struct mali_dlbu_core *dlbu) ++{ ++ u32 dlbu_registers[7]; ++ _mali_osk_errcode_t err = _MALI_OSK_ERR_FAULT; ++ MALI_DEBUG_ASSERT_POINTER(dlbu); ++ ++ MALI_DEBUG_PRINT(4, ("Mali DLBU: mali_dlbu_reset: %s\n", dlbu->hw_core.description)); ++ ++ dlbu_registers[0] = mali_dlbu_phys_addr | 1; /* bit 0 enables the whole core */ ++ dlbu_registers[1] = MALI_DLBU_VIRT_ADDR; ++ dlbu_registers[2] = 0; ++ dlbu_registers[3] = 0; ++ dlbu_registers[4] = 0; ++ dlbu_registers[5] = 0; ++ dlbu_registers[6] = dlbu->pp_cores_mask; ++ ++ /* write reset values to core registers */ ++ mali_hw_core_register_write_array_relaxed(&dlbu->hw_core, MALI_DLBU_REGISTER_MASTER_TLLIST_PHYS_ADDR, dlbu_registers, 7); ++ ++ err = _MALI_OSK_ERR_OK; ++ ++ return err; ++} ++ ++void mali_dlbu_update_mask(struct mali_dlbu_core *dlbu) ++{ ++ MALI_DEBUG_ASSERT_POINTER(dlbu); ++ ++ mali_hw_core_register_write(&dlbu->hw_core, MALI_DLBU_REGISTER_PP_ENABLE_MASK, dlbu->pp_cores_mask); ++} ++ ++void mali_dlbu_add_group(struct mali_dlbu_core *dlbu, struct mali_group *group) ++{ ++ struct mali_pp_core *pp_core; ++ u32 bcast_id; ++ ++ MALI_DEBUG_ASSERT_POINTER(dlbu); ++ MALI_DEBUG_ASSERT_POINTER(group); ++ ++ pp_core = mali_group_get_pp_core(group); ++ bcast_id = mali_pp_core_get_bcast_id(pp_core); ++ ++ dlbu->pp_cores_mask |= bcast_id; ++ MALI_DEBUG_PRINT(3, ("Mali DLBU: Adding core[%d] New mask= 0x%02x\n", bcast_id , dlbu->pp_cores_mask)); ++} ++ ++/* Remove a group from the DLBU */ ++void mali_dlbu_remove_group(struct mali_dlbu_core *dlbu, struct mali_group *group) ++{ ++ struct mali_pp_core *pp_core; ++ u32 bcast_id; ++ ++ MALI_DEBUG_ASSERT_POINTER(dlbu); ++ MALI_DEBUG_ASSERT_POINTER(group); ++ ++ pp_core = mali_group_get_pp_core(group); ++ bcast_id = mali_pp_core_get_bcast_id(pp_core); ++ ++ dlbu->pp_cores_mask &= ~bcast_id; ++ MALI_DEBUG_PRINT(3, ("Mali DLBU: Removing core[%d] New mask= 0x%02x\n", bcast_id, dlbu->pp_cores_mask)); ++} ++ ++/* Configure the DLBU for \a job. This needs to be done before the job is started on the groups in the DLBU. */ ++void mali_dlbu_config_job(struct mali_dlbu_core *dlbu, struct mali_pp_job *job) ++{ ++ u32 *registers; ++ MALI_DEBUG_ASSERT(job); ++ registers = mali_pp_job_get_dlbu_registers(job); ++ MALI_DEBUG_PRINT(4, ("Mali DLBU: Starting job\n")); ++ ++ /* Writing 4 registers: ++ * DLBU registers except the first two (written once at DLBU initialisation / reset) and the PP_ENABLE_MASK register */ ++ mali_hw_core_register_write_array_relaxed(&dlbu->hw_core, MALI_DLBU_REGISTER_TLLIST_VBASEADDR, registers, 4); ++ ++} +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_dlbu.h b/drivers/gpu/arm/mali400/mali/common/mali_dlbu.h +new file mode 100755 +index 000000000000..a7ecf41471d8 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_dlbu.h +@@ -0,0 +1,45 @@ ++/* ++ * Copyright (C) 2012-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_DLBU_H__ ++#define __MALI_DLBU_H__ ++ ++#define MALI_DLBU_VIRT_ADDR 0xFFF00000 /* master tile virtual address fixed at this value and mapped into every session */ ++ ++#include "mali_osk.h" ++ ++struct mali_pp_job; ++struct mali_group; ++struct mali_dlbu_core; ++ ++extern mali_dma_addr mali_dlbu_phys_addr; ++ ++_mali_osk_errcode_t mali_dlbu_initialize(void); ++void mali_dlbu_terminate(void); ++ ++struct mali_dlbu_core *mali_dlbu_create(const _mali_osk_resource_t *resource); ++void mali_dlbu_delete(struct mali_dlbu_core *dlbu); ++ ++_mali_osk_errcode_t mali_dlbu_reset(struct mali_dlbu_core *dlbu); ++ ++void mali_dlbu_add_group(struct mali_dlbu_core *dlbu, struct mali_group *group); ++void mali_dlbu_remove_group(struct mali_dlbu_core *dlbu, struct mali_group *group); ++ ++/** @brief Called to update HW after DLBU state changed ++ * ++ * This function must be called after \a mali_dlbu_add_group or \a ++ * mali_dlbu_remove_group to write the updated mask to hardware, unless the ++ * same is accomplished by calling \a mali_dlbu_reset. ++ */ ++void mali_dlbu_update_mask(struct mali_dlbu_core *dlbu); ++ ++void mali_dlbu_config_job(struct mali_dlbu_core *dlbu, struct mali_pp_job *job); ++ ++#endif /* __MALI_DLBU_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_dvfs_policy.c b/drivers/gpu/arm/mali400/mali/common/mali_dvfs_policy.c +new file mode 100755 +index 000000000000..55b21a410754 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_dvfs_policy.c +@@ -0,0 +1,308 @@ ++/* ++ * Copyright (C) 2010-2012, 2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include ++#include "mali_kernel_common.h" ++#include "mali_scheduler.h" ++#include "mali_dvfs_policy.h" ++#include "mali_osk_mali.h" ++#include "mali_osk_profiling.h" ++ ++#define CLOCK_TUNING_TIME_DEBUG 0 ++ ++#define MAX_PERFORMANCE_VALUE 256 ++#define MALI_PERCENTAGE_TO_UTILIZATION_FRACTION(percent) ((int) ((percent)*(MAX_PERFORMANCE_VALUE)/100.0 + 0.5)) ++ ++/** The max fps the same as display vsync default 60, can set by module insert parameter */ ++int mali_max_system_fps = 60; ++/** A lower limit on their desired FPS default 58, can set by module insert parameter */ ++int mali_desired_fps = 58; ++ ++static int mali_fps_step1 = 0; ++static int mali_fps_step2 = 0; ++ ++static int clock_step = -1; ++static int cur_clk_step = -1; ++static struct mali_gpu_clock *gpu_clk = NULL; ++ ++/*Function prototype */ ++static int (*mali_gpu_set_freq)(int) = NULL; ++static int (*mali_gpu_get_freq)(void) = NULL; ++ ++static mali_bool mali_dvfs_enabled = MALI_FALSE; ++ ++#define NUMBER_OF_NANOSECONDS_PER_SECOND 1000000000ULL ++static u32 calculate_window_render_fps(u64 time_period) ++{ ++ u32 max_window_number; ++ u64 tmp; ++ u64 max = time_period; ++ u32 leading_zeroes; ++ u32 shift_val; ++ u32 time_period_shift; ++ u32 max_window_number_shift; ++ u32 ret_val; ++ ++ max_window_number = mali_session_max_window_num(); ++ ++ /* To avoid float division, extend the dividend to ns unit */ ++ tmp = (u64)max_window_number * NUMBER_OF_NANOSECONDS_PER_SECOND; ++ if (tmp > time_period) { ++ max = tmp; ++ } ++ ++ /* ++ * We may have 64-bit values, a dividend or a divisor or both ++ * To avoid dependencies to a 64-bit divider, we shift down the two values ++ * equally first. ++ */ ++ leading_zeroes = _mali_osk_clz((u32)(max >> 32)); ++ shift_val = 32 - leading_zeroes; ++ ++ time_period_shift = (u32)(time_period >> shift_val); ++ max_window_number_shift = (u32)(tmp >> shift_val); ++ ++ ret_val = max_window_number_shift / time_period_shift; ++ ++ return ret_val; ++} ++ ++static bool mali_pickup_closest_avail_clock(int target_clock_mhz, mali_bool pick_clock_up) ++{ ++ int i = 0; ++ bool clock_changed = false; ++ ++ /* Round up the closest available frequency step for target_clock_hz */ ++ for (i = 0; i < gpu_clk->num_of_steps; i++) { ++ /* Find the first item > target_clock_hz */ ++ if (((int)(gpu_clk->item[i].clock) - target_clock_mhz) > 0) { ++ break; ++ } ++ } ++ ++ /* If the target clock greater than the maximum clock just pick the maximum one*/ ++ if (i == gpu_clk->num_of_steps) { ++ i = gpu_clk->num_of_steps - 1; ++ } else { ++ if ((!pick_clock_up) && (i > 0)) { ++ i = i - 1; ++ } ++ } ++ ++ clock_step = i; ++ if (cur_clk_step != clock_step) { ++ clock_changed = true; ++ } ++ ++ return clock_changed; ++} ++ ++void mali_dvfs_policy_realize(struct mali_gpu_utilization_data *data, u64 time_period) ++{ ++ int under_perform_boundary_value = 0; ++ int over_perform_boundary_value = 0; ++ int current_fps = 0; ++ int current_gpu_util = 0; ++ bool clock_changed = false; ++#if CLOCK_TUNING_TIME_DEBUG ++ struct timeval start; ++ struct timeval stop; ++ unsigned int elapse_time; ++ do_gettimeofday(&start); ++#endif ++ u32 window_render_fps; ++ ++ if (NULL == gpu_clk) { ++ MALI_DEBUG_PRINT(2, ("Enable DVFS but patform doesn't Support freq change. \n")); ++ return; ++ } ++ ++ window_render_fps = calculate_window_render_fps(time_period); ++ ++ current_fps = window_render_fps; ++ current_gpu_util = data->utilization_gpu; ++ ++ /* Get the specific under_perform_boundary_value and over_perform_boundary_value */ ++ if ((mali_desired_fps <= current_fps) && (current_fps < mali_max_system_fps)) { ++ under_perform_boundary_value = MALI_PERCENTAGE_TO_UTILIZATION_FRACTION(90); ++ over_perform_boundary_value = MALI_PERCENTAGE_TO_UTILIZATION_FRACTION(70); ++ } else if ((mali_fps_step1 <= current_fps) && (current_fps < mali_desired_fps)) { ++ under_perform_boundary_value = MALI_PERCENTAGE_TO_UTILIZATION_FRACTION(55); ++ over_perform_boundary_value = MALI_PERCENTAGE_TO_UTILIZATION_FRACTION(35); ++ } else if ((mali_fps_step2 <= current_fps) && (current_fps < mali_fps_step1)) { ++ under_perform_boundary_value = MALI_PERCENTAGE_TO_UTILIZATION_FRACTION(70); ++ over_perform_boundary_value = MALI_PERCENTAGE_TO_UTILIZATION_FRACTION(50); ++ } else { ++ under_perform_boundary_value = MALI_PERCENTAGE_TO_UTILIZATION_FRACTION(55); ++ over_perform_boundary_value = MALI_PERCENTAGE_TO_UTILIZATION_FRACTION(35); ++ } ++ ++ MALI_DEBUG_PRINT(5, ("Using ARM power policy: gpu util = %d \n", current_gpu_util)); ++ MALI_DEBUG_PRINT(5, ("Using ARM power policy: under_perform = %d, over_perform = %d \n", under_perform_boundary_value, over_perform_boundary_value)); ++ MALI_DEBUG_PRINT(5, ("Using ARM power policy: render fps = %d, pressure render fps = %d \n", current_fps, window_render_fps)); ++ ++ /* Get current clock value */ ++ cur_clk_step = mali_gpu_get_freq(); ++ ++ /* Consider offscreen */ ++ if (0 == current_fps) { ++ /* GP or PP under perform, need to give full power */ ++ if (current_gpu_util > over_perform_boundary_value) { ++ if (cur_clk_step != gpu_clk->num_of_steps - 1) { ++ clock_changed = true; ++ clock_step = gpu_clk->num_of_steps - 1; ++ } ++ } ++ ++ /* If GPU is idle, use lowest power */ ++ if (0 == current_gpu_util) { ++ if (cur_clk_step != 0) { ++ clock_changed = true; ++ clock_step = 0; ++ } ++ } ++ ++ goto real_setting; ++ } ++ ++ /* 2. Calculate target clock if the GPU clock can be tuned */ ++ if (-1 != cur_clk_step) { ++ int target_clk_mhz = -1; ++ mali_bool pick_clock_up = MALI_TRUE; ++ ++ if (current_gpu_util > under_perform_boundary_value) { ++ /* when under perform, need to consider the fps part */ ++ target_clk_mhz = gpu_clk->item[cur_clk_step].clock * current_gpu_util * mali_desired_fps / under_perform_boundary_value / current_fps; ++ pick_clock_up = MALI_TRUE; ++ } else if (current_gpu_util < over_perform_boundary_value) { ++ /* when over perform, did't need to consider fps, system didn't want to reach desired fps */ ++ target_clk_mhz = gpu_clk->item[cur_clk_step].clock * current_gpu_util / under_perform_boundary_value; ++ pick_clock_up = MALI_FALSE; ++ } ++ ++ if (-1 != target_clk_mhz) { ++ clock_changed = mali_pickup_closest_avail_clock(target_clk_mhz, pick_clock_up); ++ } ++ } ++ ++real_setting: ++ if (clock_changed) { ++ mali_gpu_set_freq(clock_step); ++ ++ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE | ++ MALI_PROFILING_EVENT_CHANNEL_GPU | ++ MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE, ++ gpu_clk->item[clock_step].clock, ++ gpu_clk->item[clock_step].vol / 1000, ++ 0, 0, 0); ++ } ++ ++#if CLOCK_TUNING_TIME_DEBUG ++ do_gettimeofday(&stop); ++ ++ elapse_time = timeval_to_ns(&stop) - timeval_to_ns(&start); ++ MALI_DEBUG_PRINT(2, ("Using ARM power policy: eclapse time = %d\n", elapse_time)); ++#endif ++} ++ ++_mali_osk_errcode_t mali_dvfs_policy_init(void) ++{ ++ _mali_osk_device_data data; ++ _mali_osk_errcode_t err = _MALI_OSK_ERR_OK; ++ ++ if (_MALI_OSK_ERR_OK == _mali_osk_device_data_get(&data)) { ++ if ((NULL != data.get_clock_info) && (NULL != data.set_freq) && (NULL != data.get_freq)) { ++ MALI_DEBUG_PRINT(2, ("Mali DVFS init: using arm dvfs policy \n")); ++ ++ ++ mali_fps_step1 = mali_max_system_fps / 3; ++ mali_fps_step2 = mali_max_system_fps / 5; ++ ++ data.get_clock_info(&gpu_clk); ++ ++ if (gpu_clk != NULL) { ++#ifdef DEBUG ++ int i; ++ for (i = 0; i < gpu_clk->num_of_steps; i++) { ++ MALI_DEBUG_PRINT(5, ("mali gpu clock info: step%d clock(%d)Hz,vol(%d) \n", ++ i, gpu_clk->item[i].clock, gpu_clk->item[i].vol)); ++ } ++#endif ++ } else { ++ MALI_DEBUG_PRINT(2, ("Mali DVFS init: platform didn't define enough info for ddk to do DVFS \n")); ++ } ++ ++ mali_gpu_get_freq = data.get_freq; ++ mali_gpu_set_freq = data.set_freq; ++ ++ if ((NULL != gpu_clk) && (gpu_clk->num_of_steps > 0) ++ && (NULL != mali_gpu_get_freq) && (NULL != mali_gpu_set_freq)) { ++ mali_dvfs_enabled = MALI_TRUE; ++ } ++ } else { ++ MALI_DEBUG_PRINT(2, ("Mali DVFS init: platform function callback incomplete, need check mali_gpu_device_data in platform .\n")); ++ } ++ } else { ++ err = _MALI_OSK_ERR_FAULT; ++ MALI_DEBUG_PRINT(2, ("Mali DVFS init: get platform data error .\n")); ++ } ++ ++ return err; ++} ++ ++/* ++ * Always give full power when start a new period, ++ * if mali dvfs enabled, for performance consideration ++ */ ++void mali_dvfs_policy_new_period(void) ++{ ++ /* Always give full power when start a new period */ ++ unsigned int cur_clk_step = 0; ++ ++ cur_clk_step = mali_gpu_get_freq(); ++ ++ if (cur_clk_step != (gpu_clk->num_of_steps - 1)) { ++ mali_gpu_set_freq(gpu_clk->num_of_steps - 1); ++ ++ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE | ++ MALI_PROFILING_EVENT_CHANNEL_GPU | ++ MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE, gpu_clk->item[gpu_clk->num_of_steps - 1].clock, ++ gpu_clk->item[gpu_clk->num_of_steps - 1].vol / 1000, 0, 0, 0); ++ } ++} ++ ++mali_bool mali_dvfs_policy_enabled(void) ++{ ++ return mali_dvfs_enabled; ++} ++ ++#if defined(CONFIG_MALI400_PROFILING) ++void mali_get_current_gpu_clk_item(struct mali_gpu_clk_item *clk_item) ++{ ++ if (mali_platform_device != NULL) { ++ ++ struct mali_gpu_device_data *device_data = NULL; ++ device_data = (struct mali_gpu_device_data *)mali_platform_device->dev.platform_data; ++ ++ if ((NULL != device_data->get_clock_info) && (NULL != device_data->get_freq)) { ++ ++ int cur_clk_step = device_data->get_freq(); ++ struct mali_gpu_clock *mali_gpu_clk = NULL; ++ ++ device_data->get_clock_info(&mali_gpu_clk); ++ clk_item->clock = mali_gpu_clk->item[cur_clk_step].clock; ++ clk_item->vol = mali_gpu_clk->item[cur_clk_step].vol; ++ } else { ++ MALI_DEBUG_PRINT(2, ("Mali GPU Utilization: platform function callback incomplete, need check mali_gpu_device_data in platform .\n")); ++ } ++ } ++} ++#endif ++ +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_dvfs_policy.h b/drivers/gpu/arm/mali400/mali/common/mali_dvfs_policy.h +new file mode 100755 +index 000000000000..662348c4e6ac +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_dvfs_policy.h +@@ -0,0 +1,34 @@ ++/* ++ * Copyright (C) 2010-2012, 2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_DVFS_POLICY_H__ ++#define __MALI_DVFS_POLICY_H__ ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++void mali_dvfs_policy_realize(struct mali_gpu_utilization_data *data, u64 time_period); ++ ++_mali_osk_errcode_t mali_dvfs_policy_init(void); ++ ++void mali_dvfs_policy_new_period(void); ++ ++mali_bool mali_dvfs_policy_enabled(void); ++ ++#if defined(CONFIG_MALI400_PROFILING) ++void mali_get_current_gpu_clk_item(struct mali_gpu_clk_item *clk_item); ++#endif ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif/* __MALI_DVFS_POLICY_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_executor.c b/drivers/gpu/arm/mali400/mali/common/mali_executor.c +new file mode 100755 +index 000000000000..ee40520ed0ce +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_executor.c +@@ -0,0 +1,2707 @@ ++/* ++ * Copyright (C) 2012-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include "mali_executor.h" ++#include "mali_scheduler.h" ++#include "mali_kernel_common.h" ++#include "mali_kernel_core.h" ++#include "mali_osk.h" ++#include "mali_osk_list.h" ++#include "mali_pp.h" ++#include "mali_pp_job.h" ++#include "mali_group.h" ++#include "mali_pm.h" ++#include "mali_timeline.h" ++#include "mali_osk_profiling.h" ++#include "mali_session.h" ++#include "mali_osk_mali.h" ++ ++/*Add for voltage scan function*/ ++extern u32 mali_group_error; ++ ++/* ++ * If dma_buf with map on demand is used, we defer job deletion and job queue ++ * if in atomic context, since both might sleep. ++ */ ++#if defined(CONFIG_DMA_SHARED_BUFFER) && !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) ++#define MALI_EXECUTOR_USE_DEFERRED_PP_JOB_DELETE 1 ++#define MALI_EXECUTOR_USE_DEFERRED_PP_JOB_QUEUE 1 ++#endif /* !defined(CONFIG_DMA_SHARED_BUFFER) && !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) */ ++ ++/* ++ * ---------- static type definitions (structs, enums, etc) ---------- ++ */ ++ ++enum mali_executor_state_t { ++ EXEC_STATE_NOT_PRESENT, /* Virtual group on Mali-300/400 (do not use) */ ++ EXEC_STATE_DISABLED, /* Disabled by core scaling (do not use) */ ++ EXEC_STATE_EMPTY, /* No child groups for virtual group (do not use) */ ++ EXEC_STATE_INACTIVE, /* Can be used, but must be activate first */ ++ EXEC_STATE_IDLE, /* Active and ready to be used */ ++ EXEC_STATE_WORKING, /* Executing a job */ ++}; ++ ++/* ++ * ---------- global variables (exported due to inline functions) ---------- ++ */ ++ ++/* Lock for this module (protecting all HW access except L2 caches) */ ++_mali_osk_spinlock_irq_t *mali_executor_lock_obj = NULL; ++ ++mali_bool mali_executor_hints[MALI_EXECUTOR_HINT_MAX]; ++ ++/* ++ * ---------- static variables ---------- ++ */ ++ ++/* Used to defer job scheduling */ ++static _mali_osk_wq_work_t *executor_wq_high_pri = NULL; ++ ++/* Store version from GP and PP (user space wants to know this) */ ++static u32 pp_version = 0; ++static u32 gp_version = 0; ++ ++/* List of physical PP groups which are disabled by some external source */ ++static _MALI_OSK_LIST_HEAD_STATIC_INIT(group_list_disabled); ++static u32 group_list_disabled_count = 0; ++ ++/* List of groups which can be used, but activate first */ ++static _MALI_OSK_LIST_HEAD_STATIC_INIT(group_list_inactive); ++static u32 group_list_inactive_count = 0; ++ ++/* List of groups which are active and ready to be used */ ++static _MALI_OSK_LIST_HEAD_STATIC_INIT(group_list_idle); ++static u32 group_list_idle_count = 0; ++ ++/* List of groups which are executing a job */ ++static _MALI_OSK_LIST_HEAD_STATIC_INIT(group_list_working); ++static u32 group_list_working_count = 0; ++ ++/* Virtual group (if any) */ ++static struct mali_group *virtual_group = NULL; ++ ++/* Virtual group state is tracked with a state variable instead of 4 lists */ ++static enum mali_executor_state_t virtual_group_state = EXEC_STATE_NOT_PRESENT; ++ ++/* GP group */ ++static struct mali_group *gp_group = NULL; ++ ++/* GP group state is tracked with a state variable instead of 4 lists */ ++static enum mali_executor_state_t gp_group_state = EXEC_STATE_NOT_PRESENT; ++ ++static u32 gp_returned_cookie = 0; ++ ++/* Total number of physical PP cores present */ ++static u32 num_physical_pp_cores_total = 0; ++ ++/* Number of physical cores which are enabled */ ++static u32 num_physical_pp_cores_enabled = 0; ++ ++/* Enable or disable core scaling */ ++static mali_bool core_scaling_enabled = MALI_TRUE; ++ ++/* Variables to allow safe pausing of the scheduler */ ++static _mali_osk_wait_queue_t *executor_working_wait_queue = NULL; ++static u32 pause_count = 0; ++ ++/* PP cores haven't been enabled because of some pp cores haven't been disabled. */ ++static int core_scaling_delay_up_mask[MALI_MAX_NUMBER_OF_DOMAINS] = { 0 }; ++ ++/* Variables used to implement notify pp core changes to userspace when core scaling ++ * is finished in mali_executor_complete_group() function. */ ++static _mali_osk_wq_work_t *executor_wq_notify_core_change = NULL; ++static _mali_osk_wait_queue_t *executor_notify_core_change_wait_queue = NULL; ++ ++/* ++ * ---------- Forward declaration of static functions ---------- ++ */ ++static mali_bool mali_executor_is_suspended(void *data); ++static mali_bool mali_executor_is_working(void); ++static void mali_executor_disable_empty_virtual(void); ++static mali_bool mali_executor_physical_rejoin_virtual(struct mali_group *group); ++static mali_bool mali_executor_has_virtual_group(void); ++static mali_bool mali_executor_virtual_group_is_usable(void); ++static void mali_executor_schedule(void); ++static void mali_executor_wq_schedule(void *arg); ++static void mali_executor_send_gp_oom_to_user(struct mali_gp_job *job); ++static void mali_executor_complete_group(struct mali_group *group, ++ mali_bool success, ++ struct mali_gp_job **gp_job_done, ++ struct mali_pp_job **pp_job_done); ++static void mali_executor_change_state_pp_physical(struct mali_group *group, ++ _mali_osk_list_t *old_list, ++ u32 *old_count, ++ _mali_osk_list_t *new_list, ++ u32 *new_count); ++static mali_bool mali_executor_group_is_in_state(struct mali_group *group, ++ enum mali_executor_state_t state); ++ ++static void mali_executor_group_enable_internal(struct mali_group *group); ++static void mali_executor_group_disable_internal(struct mali_group *group); ++static void mali_executor_core_scale(unsigned int target_core_nr); ++static void mali_executor_core_scale_in_group_complete(struct mali_group *group); ++static void mali_executor_notify_core_change(u32 num_cores); ++static void mali_executor_wq_notify_core_change(void *arg); ++static void mali_executor_change_group_status_disabled(struct mali_group *group); ++static mali_bool mali_executor_deactivate_list_idle(mali_bool deactivate_idle_group); ++static void mali_executor_set_state_pp_physical(struct mali_group *group, ++ _mali_osk_list_t *new_list, ++ u32 *new_count); ++ ++/* ++ * ---------- Actual implementation ---------- ++ */ ++ ++_mali_osk_errcode_t mali_executor_initialize(void) ++{ ++ mali_executor_lock_obj = _mali_osk_spinlock_irq_init(_MALI_OSK_LOCKFLAG_ORDERED, _MALI_OSK_LOCK_ORDER_EXECUTOR); ++ if (NULL == mali_executor_lock_obj) { ++ mali_executor_terminate(); ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ ++ executor_wq_high_pri = _mali_osk_wq_create_work_high_pri(mali_executor_wq_schedule, NULL); ++ if (NULL == executor_wq_high_pri) { ++ mali_executor_terminate(); ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ ++ executor_working_wait_queue = _mali_osk_wait_queue_init(); ++ if (NULL == executor_working_wait_queue) { ++ mali_executor_terminate(); ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ ++ executor_wq_notify_core_change = _mali_osk_wq_create_work(mali_executor_wq_notify_core_change, NULL); ++ if (NULL == executor_wq_notify_core_change) { ++ mali_executor_terminate(); ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ ++ executor_notify_core_change_wait_queue = _mali_osk_wait_queue_init(); ++ if (NULL == executor_notify_core_change_wait_queue) { ++ mali_executor_terminate(); ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++void mali_executor_terminate(void) ++{ ++ if (NULL != executor_notify_core_change_wait_queue) { ++ _mali_osk_wait_queue_term(executor_notify_core_change_wait_queue); ++ executor_notify_core_change_wait_queue = NULL; ++ } ++ ++ if (NULL != executor_wq_notify_core_change) { ++ _mali_osk_wq_delete_work(executor_wq_notify_core_change); ++ executor_wq_notify_core_change = NULL; ++ } ++ ++ if (NULL != executor_working_wait_queue) { ++ _mali_osk_wait_queue_term(executor_working_wait_queue); ++ executor_working_wait_queue = NULL; ++ } ++ ++ if (NULL != executor_wq_high_pri) { ++ _mali_osk_wq_delete_work(executor_wq_high_pri); ++ executor_wq_high_pri = NULL; ++ } ++ ++ if (NULL != mali_executor_lock_obj) { ++ _mali_osk_spinlock_irq_term(mali_executor_lock_obj); ++ mali_executor_lock_obj = NULL; ++ } ++} ++ ++void mali_executor_populate(void) ++{ ++ u32 num_groups; ++ u32 i; ++ ++ num_groups = mali_group_get_glob_num_groups(); ++ ++ /* Do we have a virtual group? */ ++ for (i = 0; i < num_groups; i++) { ++ struct mali_group *group = mali_group_get_glob_group(i); ++ ++ if (mali_group_is_virtual(group)) { ++ virtual_group = group; ++ virtual_group_state = EXEC_STATE_INACTIVE; ++ break; ++ } ++ } ++ ++ /* Find all the available physical GP and PP cores */ ++ for (i = 0; i < num_groups; i++) { ++ struct mali_group *group = mali_group_get_glob_group(i); ++ ++ if (NULL != group) { ++ struct mali_pp_core *pp_core = mali_group_get_pp_core(group); ++ struct mali_gp_core *gp_core = mali_group_get_gp_core(group); ++ ++ if (!mali_group_is_virtual(group)) { ++ if (NULL != pp_core) { ++ if (0 == pp_version) { ++ /* Retrieve PP version from the first available PP core */ ++ pp_version = mali_pp_core_get_version(pp_core); ++ } ++ ++ if (NULL != virtual_group) { ++ mali_executor_lock(); ++ mali_group_add_group(virtual_group, group); ++ mali_executor_unlock(); ++ } else { ++ _mali_osk_list_add(&group->executor_list, &group_list_inactive); ++ group_list_inactive_count++; ++ } ++ ++ num_physical_pp_cores_total++; ++ } else { ++ MALI_DEBUG_ASSERT_POINTER(gp_core); ++ ++ if (0 == gp_version) { ++ /* Retrieve GP version */ ++ gp_version = mali_gp_core_get_version(gp_core); ++ } ++ ++ gp_group = group; ++ gp_group_state = EXEC_STATE_INACTIVE; ++ } ++ ++ } ++ } ++ } ++ ++ num_physical_pp_cores_enabled = num_physical_pp_cores_total; ++} ++ ++void mali_executor_depopulate(void) ++{ ++ struct mali_group *group; ++ struct mali_group *temp; ++ ++ MALI_DEBUG_ASSERT(EXEC_STATE_WORKING != gp_group_state); ++ ++ if (NULL != gp_group) { ++ mali_group_delete(gp_group); ++ gp_group = NULL; ++ } ++ ++ MALI_DEBUG_ASSERT(EXEC_STATE_WORKING != virtual_group_state); ++ ++ if (NULL != virtual_group) { ++ mali_group_delete(virtual_group); ++ virtual_group = NULL; ++ } ++ ++ MALI_DEBUG_ASSERT(_mali_osk_list_empty(&group_list_working)); ++ ++ _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_idle, struct mali_group, executor_list) { ++ mali_group_delete(group); ++ } ++ ++ _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_inactive, struct mali_group, executor_list) { ++ mali_group_delete(group); ++ } ++ ++ _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_disabled, struct mali_group, executor_list) { ++ mali_group_delete(group); ++ } ++} ++ ++void mali_executor_suspend(void) ++{ ++ mali_executor_lock(); ++ ++ /* Increment the pause_count so that no more jobs will be scheduled */ ++ pause_count++; ++ ++ mali_executor_unlock(); ++ ++ _mali_osk_wait_queue_wait_event(executor_working_wait_queue, ++ mali_executor_is_suspended, NULL); ++ ++ /* ++ * mali_executor_complete_XX() leaves jobs in idle state. ++ * deactivate option is used when we are going to power down ++ * the entire GPU (OS suspend) and want a consistent SW vs HW ++ * state. ++ */ ++ mali_executor_lock(); ++ ++ mali_executor_deactivate_list_idle(MALI_TRUE); ++ ++ /* ++ * The following steps are used to deactive all of activated ++ * (MALI_GROUP_STATE_ACTIVE) and activating (MALI_GROUP ++ * _STAET_ACTIVATION_PENDING) groups, to make sure the variable ++ * pd_mask_wanted is equal with 0. */ ++ if (MALI_GROUP_STATE_INACTIVE != mali_group_get_state(gp_group)) { ++ gp_group_state = EXEC_STATE_INACTIVE; ++ mali_group_deactivate(gp_group); ++ } ++ ++ if (mali_executor_has_virtual_group()) { ++ if (MALI_GROUP_STATE_INACTIVE ++ != mali_group_get_state(virtual_group)) { ++ virtual_group_state = EXEC_STATE_INACTIVE; ++ mali_group_deactivate(virtual_group); ++ } ++ } ++ ++ if (0 < group_list_inactive_count) { ++ struct mali_group *group; ++ struct mali_group *temp; ++ ++ _MALI_OSK_LIST_FOREACHENTRY(group, temp, ++ &group_list_inactive, ++ struct mali_group, executor_list) { ++ if (MALI_GROUP_STATE_ACTIVATION_PENDING ++ == mali_group_get_state(group)) { ++ mali_group_deactivate(group); ++ } ++ ++ /* ++ * On mali-450 platform, we may have physical group in the group inactive ++ * list, and its state is MALI_GROUP_STATE_ACTIVATION_PENDING, so we only ++ * deactivate it is not enough, we still also need add it back to virtual group. ++ * And now, virtual group must be in INACTIVE state, so it's safe to add ++ * physical group to virtual group at this point. ++ */ ++ if (NULL != virtual_group) { ++ _mali_osk_list_delinit(&group->executor_list); ++ group_list_inactive_count--; ++ ++ mali_group_add_group(virtual_group, group); ++ } ++ } ++ } ++ ++ mali_executor_unlock(); ++} ++ ++void mali_executor_resume(void) ++{ ++ mali_executor_lock(); ++ ++ /* Decrement pause_count to allow scheduling again (if it reaches 0) */ ++ pause_count--; ++ if (0 == pause_count) { ++ mali_executor_schedule(); ++ } ++ ++ mali_executor_unlock(); ++} ++ ++u32 mali_executor_get_num_cores_total(void) ++{ ++ return num_physical_pp_cores_total; ++} ++ ++u32 mali_executor_get_num_cores_enabled(void) ++{ ++ return num_physical_pp_cores_enabled; ++} ++ ++struct mali_pp_core *mali_executor_get_virtual_pp(void) ++{ ++ MALI_DEBUG_ASSERT_POINTER(virtual_group); ++ MALI_DEBUG_ASSERT_POINTER(virtual_group->pp_core); ++ return virtual_group->pp_core; ++} ++ ++struct mali_group *mali_executor_get_virtual_group(void) ++{ ++ return virtual_group; ++} ++ ++void mali_executor_zap_all_active(struct mali_session_data *session) ++{ ++ struct mali_group *group; ++ struct mali_group *temp; ++ mali_bool ret; ++ ++ mali_executor_lock(); ++ ++ /* ++ * This function is a bit complicated because ++ * mali_group_zap_session() can fail. This only happens because the ++ * group is in an unhandled page fault status. ++ * We need to make sure this page fault is handled before we return, ++ * so that we know every single outstanding MMU transactions have ++ * completed. This will allow caller to safely remove physical pages ++ * when we have returned. ++ */ ++ ++ MALI_DEBUG_ASSERT(NULL != gp_group); ++ ret = mali_group_zap_session(gp_group, session); ++ if (MALI_FALSE == ret) { ++ struct mali_gp_job *gp_job = NULL; ++ ++ mali_executor_complete_group(gp_group, MALI_FALSE, &gp_job, NULL); ++ ++ MALI_DEBUG_ASSERT_POINTER(gp_job); ++ ++ /* GP job completed, make sure it is freed */ ++ mali_scheduler_complete_gp_job(gp_job, MALI_FALSE, ++ MALI_TRUE, MALI_TRUE); ++ } ++ ++ if (mali_executor_has_virtual_group()) { ++ ret = mali_group_zap_session(virtual_group, session); ++ if (MALI_FALSE == ret) { ++ struct mali_pp_job *pp_job = NULL; ++ ++ mali_executor_complete_group(virtual_group, MALI_FALSE, NULL, &pp_job); ++ ++ if (NULL != pp_job) { ++ /* PP job completed, make sure it is freed */ ++ mali_scheduler_complete_pp_job(pp_job, 0, ++ MALI_FALSE, MALI_TRUE); ++ } ++ } ++ } ++ ++ _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_working, ++ struct mali_group, executor_list) { ++ ret = mali_group_zap_session(group, session); ++ if (MALI_FALSE == ret) { ++ ret = mali_group_zap_session(group, session); ++ if (MALI_FALSE == ret) { ++ struct mali_pp_job *pp_job = NULL; ++ ++ mali_executor_complete_group(group, MALI_FALSE, NULL, &pp_job); ++ ++ if (NULL != pp_job) { ++ /* PP job completed, free it */ ++ mali_scheduler_complete_pp_job(pp_job, ++ 0, MALI_FALSE, ++ MALI_TRUE); ++ } ++ } ++ } ++ } ++ ++ mali_executor_unlock(); ++} ++ ++void mali_executor_schedule_from_mask(mali_scheduler_mask mask, mali_bool deferred_schedule) ++{ ++ if (MALI_SCHEDULER_MASK_EMPTY != mask) { ++ if (MALI_TRUE == deferred_schedule) { ++ _mali_osk_wq_schedule_work_high_pri(executor_wq_high_pri); ++ } else { ++ /* Schedule from this thread*/ ++ mali_executor_lock(); ++ mali_executor_schedule(); ++ mali_executor_unlock(); ++ } ++ } ++} ++ ++_mali_osk_errcode_t mali_executor_interrupt_gp(struct mali_group *group, ++ mali_bool in_upper_half) ++{ ++ enum mali_interrupt_result int_result; ++ mali_bool time_out = MALI_FALSE; ++ ++ MALI_DEBUG_PRINT(4, ("Executor: GP interrupt from %s in %s half\n", ++ mali_group_core_description(group), ++ in_upper_half ? "upper" : "bottom")); ++ ++ mali_executor_lock(); ++ if (!mali_group_is_working(group)) { ++ /* Not working, so nothing to do */ ++ mali_executor_unlock(); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ MALI_DEBUG_ASSERT(mali_group_is_working(group)); ++ ++ if (mali_group_has_timed_out(group)) { ++ int_result = MALI_INTERRUPT_RESULT_ERROR; ++ time_out = MALI_TRUE; ++ MALI_PRINT(("Executor GP: Job %d Timeout on %s\n", ++ mali_gp_job_get_id(group->gp_running_job), ++ mali_group_core_description(group))); ++ } else { ++ int_result = mali_group_get_interrupt_result_gp(group); ++ if (MALI_INTERRUPT_RESULT_NONE == int_result) { ++ mali_executor_unlock(); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ } ++ ++#if defined(CONFIG_MALI_SHARED_INTERRUPTS) ++ if (MALI_INTERRUPT_RESULT_NONE == int_result) { ++ /* No interrupts signalled, so nothing to do */ ++ mali_executor_unlock(); ++ return _MALI_OSK_ERR_FAULT; ++ } ++#else ++ MALI_DEBUG_ASSERT(MALI_INTERRUPT_RESULT_NONE != int_result); ++#endif ++ ++ mali_group_mask_all_interrupts_gp(group); ++ ++ if (MALI_INTERRUPT_RESULT_SUCCESS_VS == int_result) { ++ if (mali_group_gp_is_active(group)) { ++ /* Only VS completed so far, while PLBU is still active */ ++ ++ /* Enable all but the current interrupt */ ++ mali_group_enable_interrupts_gp(group, int_result); ++ ++ mali_executor_unlock(); ++ return _MALI_OSK_ERR_OK; ++ } ++ } else if (MALI_INTERRUPT_RESULT_SUCCESS_PLBU == int_result) { ++ if (mali_group_gp_is_active(group)) { ++ /* Only PLBU completed so far, while VS is still active */ ++ ++ /* Enable all but the current interrupt */ ++ mali_group_enable_interrupts_gp(group, int_result); ++ ++ mali_executor_unlock(); ++ return _MALI_OSK_ERR_OK; ++ } ++ } else if (MALI_INTERRUPT_RESULT_OOM == int_result) { ++ struct mali_gp_job *job = mali_group_get_running_gp_job(group); ++ ++ /* PLBU out of mem */ ++ MALI_DEBUG_PRINT(3, ("Executor: PLBU needs more heap memory\n")); ++ ++#if defined(CONFIG_MALI400_PROFILING) ++ /* Give group a chance to generate a SUSPEND event */ ++ mali_group_oom(group); ++#endif ++ ++ /* ++ * no need to hold interrupt raised while ++ * waiting for more memory. ++ */ ++ mali_executor_send_gp_oom_to_user(job); ++ ++ mali_executor_unlock(); ++ ++ return _MALI_OSK_ERR_OK; ++ } ++ ++ /*Add for voltage scan function*/ ++ if (MALI_INTERRUPT_RESULT_ERROR == int_result) ++ mali_group_error++; ++ ++ /* We should now have a real interrupt to handle */ ++ ++ MALI_DEBUG_PRINT(4, ("Executor: Group %s completed with %s\n", ++ mali_group_core_description(group), ++ (MALI_INTERRUPT_RESULT_ERROR == int_result) ? ++ "ERROR" : "success")); ++ ++ if (in_upper_half && MALI_INTERRUPT_RESULT_ERROR == int_result) { ++ /* Don't bother to do processing of errors in upper half */ ++ mali_executor_unlock(); ++ ++ if (MALI_FALSE == time_out) { ++ mali_group_schedule_bottom_half_gp(group); ++ } ++ } else { ++ struct mali_gp_job *job; ++ mali_bool success; ++ ++ /* ++ if (MALI_TRUE == time_out) { ++ mali_group_dump_status(group); ++ } ++ */ ++ ++ success = (int_result != MALI_INTERRUPT_RESULT_ERROR) ? ++ MALI_TRUE : MALI_FALSE; ++ ++ mali_executor_complete_group(group, success, &job, NULL); ++ ++ mali_executor_unlock(); ++ ++ /* GP jobs always fully complete */ ++ MALI_DEBUG_ASSERT(NULL != job); ++ ++ /* This will notify user space and close the job object */ ++ mali_scheduler_complete_gp_job(job, success, ++ MALI_TRUE, MALI_TRUE); ++ } ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++_mali_osk_errcode_t mali_executor_interrupt_pp(struct mali_group *group, ++ mali_bool in_upper_half) ++{ ++ enum mali_interrupt_result int_result; ++ mali_bool time_out = MALI_FALSE; ++ ++ MALI_DEBUG_PRINT(4, ("Executor: PP interrupt from %s in %s half\n", ++ mali_group_core_description(group), ++ in_upper_half ? "upper" : "bottom")); ++ ++ mali_executor_lock(); ++ ++ if (!mali_group_is_working(group)) { ++ /* Not working, so nothing to do */ ++ mali_executor_unlock(); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ if (in_upper_half) { ++ if (mali_group_is_in_virtual(group)) { ++ /* Child groups should never handle PP interrupts */ ++ MALI_DEBUG_ASSERT(!mali_group_has_timed_out(group)); ++ mali_executor_unlock(); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ } ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ MALI_DEBUG_ASSERT(mali_group_is_working(group)); ++ MALI_DEBUG_ASSERT(!mali_group_is_in_virtual(group)); ++ ++ if (mali_group_has_timed_out(group)) { ++ int_result = MALI_INTERRUPT_RESULT_ERROR; ++ time_out = MALI_TRUE; ++ MALI_PRINT(("Executor PP: Job %d Timeout on %s\n", ++ mali_pp_job_get_id(group->pp_running_job), ++ mali_group_core_description(group))); ++ } else { ++ int_result = mali_group_get_interrupt_result_pp(group); ++ if (MALI_INTERRUPT_RESULT_NONE == int_result) { ++ mali_executor_unlock(); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ } ++ ++#if defined(CONFIG_MALI_SHARED_INTERRUPTS) ++ if (MALI_INTERRUPT_RESULT_NONE == int_result) { ++ /* No interrupts signalled, so nothing to do */ ++ mali_executor_unlock(); ++ return _MALI_OSK_ERR_FAULT; ++ } else if (MALI_INTERRUPT_RESULT_SUCCESS == int_result) { ++ if (mali_group_is_virtual(group) && mali_group_pp_is_active(group)) { ++ /* Some child groups are still working, so nothing to do right now */ ++ mali_executor_unlock(); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ } ++#else ++ MALI_DEBUG_ASSERT(MALI_INTERRUPT_RESULT_NONE != int_result); ++#endif ++ ++ /*Add voltage scan function*/ ++ ++ if (MALI_INTERRUPT_RESULT_ERROR == int_result) ++ mali_group_error++; ++ ++ /* We should now have a real interrupt to handle */ ++ ++ MALI_DEBUG_PRINT(4, ("Executor: Group %s completed with %s\n", ++ mali_group_core_description(group), ++ (MALI_INTERRUPT_RESULT_ERROR == int_result) ? ++ "ERROR" : "success")); ++ ++ if (in_upper_half && MALI_INTERRUPT_RESULT_ERROR == int_result) { ++ /* Don't bother to do processing of errors in upper half */ ++ mali_group_mask_all_interrupts_pp(group); ++ mali_executor_unlock(); ++ ++ if (MALI_FALSE == time_out) { ++ mali_group_schedule_bottom_half_pp(group); ++ } ++ } else { ++ struct mali_pp_job *job = NULL; ++ mali_bool success; ++ ++ if (MALI_TRUE == time_out) { ++ mali_group_dump_status(group); ++ } ++ ++ success = (int_result == MALI_INTERRUPT_RESULT_SUCCESS) ? ++ MALI_TRUE : MALI_FALSE; ++ ++ mali_executor_complete_group(group, success, NULL, &job); ++ ++ mali_executor_unlock(); ++ ++ if (NULL != job) { ++ /* Notify user space and close the job object */ ++ mali_scheduler_complete_pp_job(job, ++ num_physical_pp_cores_total, ++ MALI_TRUE, MALI_TRUE); ++ } ++ } ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++_mali_osk_errcode_t mali_executor_interrupt_mmu(struct mali_group *group, ++ mali_bool in_upper_half) ++{ ++ enum mali_interrupt_result int_result; ++ ++ MALI_DEBUG_PRINT(4, ("Executor: MMU interrupt from %s in %s half\n", ++ mali_group_core_description(group), ++ in_upper_half ? "upper" : "bottom")); ++ ++ mali_executor_lock(); ++ if (!mali_group_is_working(group)) { ++ /* Not working, so nothing to do */ ++ mali_executor_unlock(); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ MALI_DEBUG_ASSERT(mali_group_is_working(group)); ++ ++ int_result = mali_group_get_interrupt_result_mmu(group); ++ if (MALI_INTERRUPT_RESULT_NONE == int_result) { ++ mali_executor_unlock(); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++#if defined(CONFIG_MALI_SHARED_INTERRUPTS) ++ if (MALI_INTERRUPT_RESULT_NONE == int_result) { ++ /* No interrupts signalled, so nothing to do */ ++ mali_executor_unlock(); ++ return _MALI_OSK_ERR_FAULT; ++ } ++#else ++ MALI_DEBUG_ASSERT(MALI_INTERRUPT_RESULT_ERROR == int_result); ++#endif ++ ++ /* We should now have a real interrupt to handle */ ++ ++ if (in_upper_half) { ++ /* Don't bother to do processing of errors in upper half */ ++ ++ struct mali_group *parent = group->parent_group; ++ ++ mali_mmu_mask_all_interrupts(group->mmu); ++ ++ mali_executor_unlock(); ++ ++ if (NULL == parent) { ++ mali_group_schedule_bottom_half_mmu(group); ++ } else { ++ mali_group_schedule_bottom_half_mmu(parent); ++ } ++ ++ } else { ++ struct mali_gp_job *gp_job = NULL; ++ struct mali_pp_job *pp_job = NULL; ++ ++#ifdef DEBUG ++ ++ u32 fault_address = mali_mmu_get_page_fault_addr(group->mmu); ++ u32 status = mali_mmu_get_status(group->mmu); ++ MALI_DEBUG_PRINT(2, ("Executor: Mali page fault detected at 0x%x from bus id %d of type %s on %s\n", ++ (void *)(uintptr_t)fault_address, ++ (status >> 6) & 0x1F, ++ (status & 32) ? "write" : "read", ++ group->mmu->hw_core.description)); ++ MALI_DEBUG_PRINT(3, ("Executor: MMU rawstat = 0x%08X, MMU status = 0x%08X\n", ++ mali_mmu_get_rawstat(group->mmu), status)); ++ mali_mmu_pagedir_diag(mali_session_get_page_directory(group->session), fault_address); ++#endif ++ ++ mali_executor_complete_group(group, MALI_FALSE, &gp_job, &pp_job); ++ ++ mali_executor_unlock(); ++ ++ if (NULL != gp_job) { ++ MALI_DEBUG_ASSERT(NULL == pp_job); ++ ++ /* Notify user space and close the job object */ ++ mali_scheduler_complete_gp_job(gp_job, MALI_FALSE, ++ MALI_TRUE, MALI_TRUE); ++ } else if (NULL != pp_job) { ++ MALI_DEBUG_ASSERT(NULL == gp_job); ++ ++ /* Notify user space and close the job object */ ++ mali_scheduler_complete_pp_job(pp_job, ++ num_physical_pp_cores_total, ++ MALI_TRUE, MALI_TRUE); ++ } ++ } ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++void mali_executor_group_power_up(struct mali_group *groups[], u32 num_groups) ++{ ++ u32 i; ++ mali_bool child_groups_activated = MALI_FALSE; ++ mali_bool do_schedule = MALI_FALSE; ++#if defined(DEBUG) ++ u32 num_activated = 0; ++#endif ++ ++ MALI_DEBUG_ASSERT_POINTER(groups); ++ MALI_DEBUG_ASSERT(0 < num_groups); ++ ++ mali_executor_lock(); ++ ++ MALI_DEBUG_PRINT(3, ("Executor: powering up %u groups\n", num_groups)); ++ ++ for (i = 0; i < num_groups; i++) { ++ MALI_DEBUG_PRINT(3, ("Executor: powering up group %s\n", ++ mali_group_core_description(groups[i]))); ++ ++ mali_group_power_up(groups[i]); ++ ++ if ((MALI_GROUP_STATE_ACTIVATION_PENDING != mali_group_get_state(groups[i]) || ++ (MALI_TRUE != mali_executor_group_is_in_state(groups[i], EXEC_STATE_INACTIVE)))) { ++ /* nothing more to do for this group */ ++ continue; ++ } ++ ++ MALI_DEBUG_PRINT(3, ("Executor: activating group %s\n", ++ mali_group_core_description(groups[i]))); ++ ++#if defined(DEBUG) ++ num_activated++; ++#endif ++ ++ if (mali_group_is_in_virtual(groups[i])) { ++ /* ++ * At least one child group of virtual group is powered on. ++ */ ++ child_groups_activated = MALI_TRUE; ++ } else if (MALI_FALSE == mali_group_is_virtual(groups[i])) { ++ /* Set gp and pp not in virtual to active. */ ++ mali_group_set_active(groups[i]); ++ } ++ ++ /* Move group from inactive to idle list */ ++ if (groups[i] == gp_group) { ++ MALI_DEBUG_ASSERT(EXEC_STATE_INACTIVE == ++ gp_group_state); ++ gp_group_state = EXEC_STATE_IDLE; ++ } else if (MALI_FALSE == mali_group_is_in_virtual(groups[i]) ++ && MALI_FALSE == mali_group_is_virtual(groups[i])) { ++ MALI_DEBUG_ASSERT(MALI_TRUE == mali_executor_group_is_in_state(groups[i], ++ EXEC_STATE_INACTIVE)); ++ ++ mali_executor_change_state_pp_physical(groups[i], ++ &group_list_inactive, ++ &group_list_inactive_count, ++ &group_list_idle, ++ &group_list_idle_count); ++ } ++ ++ do_schedule = MALI_TRUE; ++ } ++ ++ if (mali_executor_has_virtual_group() && ++ MALI_TRUE == child_groups_activated && ++ MALI_GROUP_STATE_ACTIVATION_PENDING == ++ mali_group_get_state(virtual_group)) { ++ /* ++ * Try to active virtual group while it may be not sucessful every time, ++ * because there is one situation that not all of child groups are powered on ++ * in one time and virtual group is in activation pending state. ++ */ ++ if (mali_group_set_active(virtual_group)) { ++ /* Move group from inactive to idle */ ++ MALI_DEBUG_ASSERT(EXEC_STATE_INACTIVE == ++ virtual_group_state); ++ virtual_group_state = EXEC_STATE_IDLE; ++ ++ MALI_DEBUG_PRINT(3, ("Executor: powering up %u groups completed, %u physical activated, 1 virtual activated.\n", num_groups, num_activated)); ++ } else { ++ MALI_DEBUG_PRINT(3, ("Executor: powering up %u groups completed, %u physical activated\n", num_groups, num_activated)); ++ } ++ } else { ++ MALI_DEBUG_PRINT(3, ("Executor: powering up %u groups completed, %u physical activated\n", num_groups, num_activated)); ++ } ++ ++ if (MALI_TRUE == do_schedule) { ++ /* Trigger a schedule */ ++ mali_executor_schedule(); ++ } ++ ++ mali_executor_unlock(); ++} ++ ++void mali_executor_group_power_down(struct mali_group *groups[], ++ u32 num_groups) ++{ ++ u32 i; ++ ++ MALI_DEBUG_ASSERT_POINTER(groups); ++ MALI_DEBUG_ASSERT(0 < num_groups); ++ ++ mali_executor_lock(); ++ ++ MALI_DEBUG_PRINT(3, ("Executor: powering down %u groups\n", num_groups)); ++ ++ for (i = 0; i < num_groups; i++) { ++ /* Groups must be either disabled or inactive. while for virtual group, ++ * it maybe in empty state, because when we meet pm_runtime_suspend, ++ * virtual group could be powered off, and before we acquire mali_executor_lock, ++ * we must release mali_pm_state_lock, if there is a new physical job was queued, ++ * all of physical groups in virtual group could be pulled out, so we only can ++ * powered down an empty virtual group. Those physical groups will be powered ++ * up in following pm_runtime_resume callback function. ++ */ ++ MALI_DEBUG_ASSERT(mali_executor_group_is_in_state(groups[i], ++ EXEC_STATE_DISABLED) || ++ mali_executor_group_is_in_state(groups[i], ++ EXEC_STATE_INACTIVE) || ++ mali_executor_group_is_in_state(groups[i], ++ EXEC_STATE_EMPTY)); ++ ++ MALI_DEBUG_PRINT(3, ("Executor: powering down group %s\n", ++ mali_group_core_description(groups[i]))); ++ ++ mali_group_power_down(groups[i]); ++ } ++ ++ MALI_DEBUG_PRINT(3, ("Executor: powering down %u groups completed\n", num_groups)); ++ ++ mali_executor_unlock(); ++} ++ ++void mali_executor_abort_session(struct mali_session_data *session) ++{ ++ struct mali_group *group; ++ struct mali_group *tmp_group; ++ ++ MALI_DEBUG_ASSERT_POINTER(session); ++ MALI_DEBUG_ASSERT(session->is_aborting); ++ ++ MALI_DEBUG_PRINT(3, ++ ("Executor: Aborting all jobs from session 0x%08X.\n", ++ session)); ++ ++ mali_executor_lock(); ++ ++ if (mali_group_get_session(gp_group) == session) { ++ if (EXEC_STATE_WORKING == gp_group_state) { ++ struct mali_gp_job *gp_job = NULL; ++ ++ mali_executor_complete_group(gp_group, MALI_FALSE, &gp_job, NULL); ++ ++ MALI_DEBUG_ASSERT_POINTER(gp_job); ++ ++ /* GP job completed, make sure it is freed */ ++ mali_scheduler_complete_gp_job(gp_job, MALI_FALSE, ++ MALI_FALSE, MALI_TRUE); ++ } else { ++ /* Same session, but not working, so just clear it */ ++ mali_group_clear_session(gp_group); ++ } ++ } ++ ++ if (mali_executor_has_virtual_group()) { ++ if (EXEC_STATE_WORKING == virtual_group_state ++ && mali_group_get_session(virtual_group) == session) { ++ struct mali_pp_job *pp_job = NULL; ++ ++ mali_executor_complete_group(virtual_group, MALI_FALSE, NULL, &pp_job); ++ ++ if (NULL != pp_job) { ++ /* PP job completed, make sure it is freed */ ++ mali_scheduler_complete_pp_job(pp_job, 0, ++ MALI_FALSE, MALI_TRUE); ++ } ++ } ++ } ++ ++ _MALI_OSK_LIST_FOREACHENTRY(group, tmp_group, &group_list_working, ++ struct mali_group, executor_list) { ++ if (mali_group_get_session(group) == session) { ++ struct mali_pp_job *pp_job = NULL; ++ ++ mali_executor_complete_group(group, MALI_FALSE, NULL, &pp_job); ++ ++ if (NULL != pp_job) { ++ /* PP job completed, make sure it is freed */ ++ mali_scheduler_complete_pp_job(pp_job, 0, ++ MALI_FALSE, MALI_TRUE); ++ } ++ } ++ } ++ ++ _MALI_OSK_LIST_FOREACHENTRY(group, tmp_group, &group_list_idle, struct mali_group, executor_list) { ++ mali_group_clear_session(group); ++ } ++ ++ _MALI_OSK_LIST_FOREACHENTRY(group, tmp_group, &group_list_inactive, struct mali_group, executor_list) { ++ mali_group_clear_session(group); ++ } ++ ++ _MALI_OSK_LIST_FOREACHENTRY(group, tmp_group, &group_list_disabled, struct mali_group, executor_list) { ++ mali_group_clear_session(group); ++ } ++ ++ mali_executor_unlock(); ++} ++ ++ ++void mali_executor_core_scaling_enable(void) ++{ ++ /* PS: Core scaling is by default enabled */ ++ core_scaling_enabled = MALI_TRUE; ++} ++ ++void mali_executor_core_scaling_disable(void) ++{ ++ core_scaling_enabled = MALI_FALSE; ++} ++ ++mali_bool mali_executor_core_scaling_is_enabled(void) ++{ ++ return core_scaling_enabled; ++} ++ ++void mali_executor_group_enable(struct mali_group *group) ++{ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ ++ mali_executor_lock(); ++ ++ if ((NULL != mali_group_get_gp_core(group) || NULL != mali_group_get_pp_core(group)) ++ && (mali_executor_group_is_in_state(group, EXEC_STATE_DISABLED))) { ++ mali_executor_group_enable_internal(group); ++ } ++ ++ mali_executor_schedule(); ++ mali_executor_unlock(); ++ ++ _mali_osk_wq_schedule_work(executor_wq_notify_core_change); ++} ++ ++/* ++ * If a physical group is inactive or idle, we should disable it immediately, ++ * if group is in virtual, and virtual group is idle, disable given physical group in it. ++ */ ++void mali_executor_group_disable(struct mali_group *group) ++{ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ ++ mali_executor_lock(); ++ ++ if ((NULL != mali_group_get_gp_core(group) || NULL != mali_group_get_pp_core(group)) ++ && (!mali_executor_group_is_in_state(group, EXEC_STATE_DISABLED))) { ++ mali_executor_group_disable_internal(group); ++ } ++ ++ mali_executor_schedule(); ++ mali_executor_unlock(); ++ ++ _mali_osk_wq_schedule_work(executor_wq_notify_core_change); ++} ++ ++mali_bool mali_executor_group_is_disabled(struct mali_group *group) ++{ ++ /* NB: This function is not optimized for time critical usage */ ++ ++ mali_bool ret; ++ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ ++ mali_executor_lock(); ++ ret = mali_executor_group_is_in_state(group, EXEC_STATE_DISABLED); ++ mali_executor_unlock(); ++ ++ return ret; ++} ++ ++int mali_executor_set_perf_level(unsigned int target_core_nr, mali_bool override) ++{ ++ if (target_core_nr == num_physical_pp_cores_enabled) return 0; ++ if (MALI_FALSE == core_scaling_enabled && MALI_FALSE == override) return -EPERM; ++ if (target_core_nr > num_physical_pp_cores_total) return -EINVAL; ++ if (0 == target_core_nr) return -EINVAL; ++ ++ mali_executor_core_scale(target_core_nr); ++ ++ _mali_osk_wq_schedule_work(executor_wq_notify_core_change); ++ ++ return 0; ++} ++ ++#if MALI_STATE_TRACKING ++u32 mali_executor_dump_state(char *buf, u32 size) ++{ ++ int n = 0; ++ struct mali_group *group; ++ struct mali_group *temp; ++ ++ mali_executor_lock(); ++ ++ switch (gp_group_state) { ++ case EXEC_STATE_INACTIVE: ++ n += _mali_osk_snprintf(buf + n, size - n, ++ "GP group is in state INACTIVE\n"); ++ break; ++ case EXEC_STATE_IDLE: ++ n += _mali_osk_snprintf(buf + n, size - n, ++ "GP group is in state IDLE\n"); ++ break; ++ case EXEC_STATE_WORKING: ++ n += _mali_osk_snprintf(buf + n, size - n, ++ "GP group is in state WORKING\n"); ++ break; ++ default: ++ n += _mali_osk_snprintf(buf + n, size - n, ++ "GP group is in unknown/illegal state %u\n", ++ gp_group_state); ++ break; ++ } ++ ++ n += mali_group_dump_state(gp_group, buf + n, size - n); ++ ++ n += _mali_osk_snprintf(buf + n, size - n, ++ "Physical PP groups in WORKING state (count = %u):\n", ++ group_list_working_count); ++ ++ _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_working, struct mali_group, executor_list) { ++ n += mali_group_dump_state(group, buf + n, size - n); ++ } ++ ++ n += _mali_osk_snprintf(buf + n, size - n, ++ "Physical PP groups in IDLE state (count = %u):\n", ++ group_list_idle_count); ++ ++ _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_idle, struct mali_group, executor_list) { ++ n += mali_group_dump_state(group, buf + n, size - n); ++ } ++ ++ n += _mali_osk_snprintf(buf + n, size - n, ++ "Physical PP groups in INACTIVE state (count = %u):\n", ++ group_list_inactive_count); ++ ++ _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_inactive, struct mali_group, executor_list) { ++ n += mali_group_dump_state(group, buf + n, size - n); ++ } ++ ++ n += _mali_osk_snprintf(buf + n, size - n, ++ "Physical PP groups in DISABLED state (count = %u):\n", ++ group_list_disabled_count); ++ ++ _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_disabled, struct mali_group, executor_list) { ++ n += mali_group_dump_state(group, buf + n, size - n); ++ } ++ ++ if (mali_executor_has_virtual_group()) { ++ switch (virtual_group_state) { ++ case EXEC_STATE_EMPTY: ++ n += _mali_osk_snprintf(buf + n, size - n, ++ "Virtual PP group is in state EMPTY\n"); ++ break; ++ case EXEC_STATE_INACTIVE: ++ n += _mali_osk_snprintf(buf + n, size - n, ++ "Virtual PP group is in state INACTIVE\n"); ++ break; ++ case EXEC_STATE_IDLE: ++ n += _mali_osk_snprintf(buf + n, size - n, ++ "Virtual PP group is in state IDLE\n"); ++ break; ++ case EXEC_STATE_WORKING: ++ n += _mali_osk_snprintf(buf + n, size - n, ++ "Virtual PP group is in state WORKING\n"); ++ break; ++ default: ++ n += _mali_osk_snprintf(buf + n, size - n, ++ "Virtual PP group is in unknown/illegal state %u\n", ++ virtual_group_state); ++ break; ++ } ++ ++ n += mali_group_dump_state(virtual_group, buf + n, size - n); ++ } ++ ++ mali_executor_unlock(); ++ ++ n += _mali_osk_snprintf(buf + n, size - n, "\n"); ++ ++ return n; ++} ++#endif ++ ++_mali_osk_errcode_t _mali_ukk_get_pp_number_of_cores(_mali_uk_get_pp_number_of_cores_s *args) ++{ ++ MALI_DEBUG_ASSERT_POINTER(args); ++ MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx); ++ args->number_of_total_cores = num_physical_pp_cores_total; ++ args->number_of_enabled_cores = num_physical_pp_cores_enabled; ++ return _MALI_OSK_ERR_OK; ++} ++ ++_mali_osk_errcode_t _mali_ukk_get_pp_core_version(_mali_uk_get_pp_core_version_s *args) ++{ ++ MALI_DEBUG_ASSERT_POINTER(args); ++ MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx); ++ args->version = pp_version; ++ return _MALI_OSK_ERR_OK; ++} ++ ++_mali_osk_errcode_t _mali_ukk_get_gp_number_of_cores(_mali_uk_get_gp_number_of_cores_s *args) ++{ ++ MALI_DEBUG_ASSERT_POINTER(args); ++ MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx); ++ args->number_of_cores = 1; ++ return _MALI_OSK_ERR_OK; ++} ++ ++_mali_osk_errcode_t _mali_ukk_get_gp_core_version(_mali_uk_get_gp_core_version_s *args) ++{ ++ MALI_DEBUG_ASSERT_POINTER(args); ++ MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx); ++ args->version = gp_version; ++ return _MALI_OSK_ERR_OK; ++} ++ ++_mali_osk_errcode_t _mali_ukk_gp_suspend_response(_mali_uk_gp_suspend_response_s *args) ++{ ++ struct mali_session_data *session; ++ struct mali_gp_job *job; ++ ++ MALI_DEBUG_ASSERT_POINTER(args); ++ MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx); ++ ++ session = (struct mali_session_data *)(uintptr_t)args->ctx; ++ ++ if (_MALIGP_JOB_RESUME_WITH_NEW_HEAP == args->code) { ++ _mali_osk_notification_t *new_notification = NULL; ++ ++ new_notification = _mali_osk_notification_create( ++ _MALI_NOTIFICATION_GP_STALLED, ++ sizeof(_mali_uk_gp_job_suspended_s)); ++ ++ if (NULL != new_notification) { ++ MALI_DEBUG_PRINT(3, ("Executor: Resuming job %u with new heap; 0x%08X - 0x%08X\n", ++ args->cookie, args->arguments[0], args->arguments[1])); ++ ++ mali_executor_lock(); ++ ++ /* Resume the job in question if it is still running */ ++ job = mali_group_get_running_gp_job(gp_group); ++ if (NULL != job && ++ args->cookie == mali_gp_job_get_id(job) && ++ session == mali_gp_job_get_session(job)) { ++ /* ++ * Correct job is running, resume with new heap ++ */ ++ ++ mali_gp_job_set_oom_notification(job, ++ new_notification); ++ ++ /* This will also re-enable interrupts */ ++ mali_group_resume_gp_with_new_heap(gp_group, ++ args->cookie, ++ args->arguments[0], ++ args->arguments[1]); ++ ++ mali_executor_unlock(); ++ return _MALI_OSK_ERR_OK; ++ } else { ++ MALI_DEBUG_PRINT(2, ("Executor: Unable to resume gp job becasue gp time out or any other unexpected reason!\n")); ++ ++ _mali_osk_notification_delete(new_notification); ++ ++ mali_executor_unlock(); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ } else { ++ MALI_PRINT_ERROR(("Executor: Failed to allocate notification object. Will abort GP job.\n")); ++ } ++ } else { ++ MALI_DEBUG_PRINT(2, ("Executor: Aborting job %u, no new heap provided\n", args->cookie)); ++ } ++ ++ mali_executor_lock(); ++ ++ /* Abort the job in question if it is still running */ ++ job = mali_group_get_running_gp_job(gp_group); ++ if (NULL != job && ++ args->cookie == mali_gp_job_get_id(job) && ++ session == mali_gp_job_get_session(job)) { ++ /* Correct job is still running */ ++ struct mali_gp_job *job_done = NULL; ++ ++ mali_executor_complete_group(gp_group, MALI_FALSE, &job_done, NULL); ++ ++ /* The same job should have completed */ ++ MALI_DEBUG_ASSERT(job_done == job); ++ ++ /* GP job completed, make sure it is freed */ ++ mali_scheduler_complete_gp_job(job_done, MALI_FALSE, ++ MALI_TRUE, MALI_TRUE); ++ } ++ ++ mali_executor_unlock(); ++ return _MALI_OSK_ERR_FAULT; ++} ++ ++ ++/* ++ * ---------- Implementation of static functions ---------- ++ */ ++ ++void mali_executor_lock(void) ++{ ++ _mali_osk_spinlock_irq_lock(mali_executor_lock_obj); ++ MALI_DEBUG_PRINT(5, ("Executor: lock taken\n")); ++} ++ ++void mali_executor_unlock(void) ++{ ++ MALI_DEBUG_PRINT(5, ("Executor: Releasing lock\n")); ++ _mali_osk_spinlock_irq_unlock(mali_executor_lock_obj); ++} ++ ++static mali_bool mali_executor_is_suspended(void *data) ++{ ++ mali_bool ret; ++ ++ /* This callback does not use the data pointer. */ ++ MALI_IGNORE(data); ++ ++ mali_executor_lock(); ++ ++ ret = pause_count > 0 && !mali_executor_is_working(); ++ ++ mali_executor_unlock(); ++ ++ return ret; ++} ++ ++static mali_bool mali_executor_is_working() ++{ ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ ++ return (0 != group_list_working_count || ++ EXEC_STATE_WORKING == gp_group_state || ++ EXEC_STATE_WORKING == virtual_group_state); ++} ++ ++static void mali_executor_disable_empty_virtual(void) ++{ ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ MALI_DEBUG_ASSERT(virtual_group_state != EXEC_STATE_EMPTY); ++ MALI_DEBUG_ASSERT(virtual_group_state != EXEC_STATE_WORKING); ++ ++ if (mali_group_is_empty(virtual_group)) { ++ virtual_group_state = EXEC_STATE_EMPTY; ++ } ++} ++ ++static mali_bool mali_executor_physical_rejoin_virtual(struct mali_group *group) ++{ ++ mali_bool trigger_pm_update = MALI_FALSE; ++ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ /* Only rejoining after job has completed (still active) */ ++ MALI_DEBUG_ASSERT(MALI_GROUP_STATE_ACTIVE == ++ mali_group_get_state(group)); ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ MALI_DEBUG_ASSERT(MALI_TRUE == mali_executor_has_virtual_group()); ++ MALI_DEBUG_ASSERT(MALI_FALSE == mali_group_is_virtual(group)); ++ ++ /* Make sure group and virtual group have same status */ ++ ++ if (MALI_GROUP_STATE_INACTIVE == mali_group_get_state(virtual_group)) { ++ if (mali_group_deactivate(group)) { ++ trigger_pm_update = MALI_TRUE; ++ } ++ ++ if (virtual_group_state == EXEC_STATE_EMPTY) { ++ virtual_group_state = EXEC_STATE_INACTIVE; ++ } ++ } else if (MALI_GROUP_STATE_ACTIVATION_PENDING == ++ mali_group_get_state(virtual_group)) { ++ /* ++ * Activation is pending for virtual group, leave ++ * this child group as active. ++ */ ++ if (virtual_group_state == EXEC_STATE_EMPTY) { ++ virtual_group_state = EXEC_STATE_INACTIVE; ++ } ++ } else { ++ MALI_DEBUG_ASSERT(MALI_GROUP_STATE_ACTIVE == ++ mali_group_get_state(virtual_group)); ++ ++ if (virtual_group_state == EXEC_STATE_EMPTY) { ++ virtual_group_state = EXEC_STATE_IDLE; ++ } ++ } ++ ++ /* Remove group from idle list */ ++ MALI_DEBUG_ASSERT(mali_executor_group_is_in_state(group, ++ EXEC_STATE_IDLE)); ++ _mali_osk_list_delinit(&group->executor_list); ++ group_list_idle_count--; ++ ++ /* ++ * And finally rejoin the virtual group ++ * group will start working on same job as virtual_group, ++ * if virtual_group is working on a job ++ */ ++ mali_group_add_group(virtual_group, group); ++ ++ return trigger_pm_update; ++} ++ ++static mali_bool mali_executor_has_virtual_group(void) ++{ ++#if (defined(CONFIG_MALI450) || defined(CONFIG_MALI470)) ++ return (NULL != virtual_group) ? MALI_TRUE : MALI_FALSE; ++#else ++ return MALI_FALSE; ++#endif /* (defined(CONFIG_MALI450) || defined(CONFIG_MALI470)) */ ++} ++ ++static mali_bool mali_executor_virtual_group_is_usable(void) ++{ ++#if (defined(CONFIG_MALI450) || defined(CONFIG_MALI470)) ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ return ((EXEC_STATE_INACTIVE == virtual_group_state || ++ EXEC_STATE_IDLE == virtual_group_state) && (virtual_group->state != MALI_GROUP_STATE_ACTIVATION_PENDING)) ? ++ MALI_TRUE : MALI_FALSE; ++#else ++ return MALI_FALSE; ++#endif /* (defined(CONFIG_MALI450) || defined(CONFIG_MALI470)) */ ++} ++ ++static mali_bool mali_executor_tackle_gp_bound(void) ++{ ++ struct mali_pp_job *job; ++ ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ ++ job = mali_scheduler_job_pp_physical_peek(); ++ ++ if (NULL != job && MALI_TRUE == mali_is_mali400()) { ++ if (0 < group_list_working_count && ++ mali_pp_job_is_large_and_unstarted(job)) { ++ return MALI_TRUE; ++ } ++ } ++ ++ return MALI_FALSE; ++} ++ ++static mali_bool mali_executor_schedule_is_early_out(mali_bool *gpu_secure_mode_is_needed) ++{ ++ struct mali_pp_job *next_pp_job_to_start = NULL; ++ struct mali_group *group; ++ struct mali_group *tmp_group; ++ struct mali_pp_job *physical_pp_job_working = NULL; ++ struct mali_pp_job *virtual_pp_job_working = NULL; ++ mali_bool gpu_working_in_protected_mode = MALI_FALSE; ++ mali_bool gpu_working_in_non_protected_mode = MALI_FALSE; ++ ++ MALI_DEBUG_ASSERT_LOCK_HELD(mali_scheduler_lock_obj); ++ ++ *gpu_secure_mode_is_needed = MALI_FALSE; ++ ++ /* Check if the gpu secure mode is supported, exit if not.*/ ++ if (MALI_FALSE == _mali_osk_gpu_secure_mode_is_supported()) { ++ return MALI_FALSE; ++ } ++ ++ /* Check if need to set gpu secure mode for the next pp job, ++ * get the next pp job that will be scheduled if exist. ++ */ ++ next_pp_job_to_start = mali_scheduler_job_pp_next(); ++ ++ /* Check current pp physical/virtual running job is protected job or not if exist.*/ ++ _MALI_OSK_LIST_FOREACHENTRY(group, tmp_group, &group_list_working, ++ struct mali_group, executor_list) { ++ physical_pp_job_working = group->pp_running_job; ++ break; ++ } ++ ++ if (EXEC_STATE_WORKING == virtual_group_state) { ++ virtual_pp_job_working = virtual_group->pp_running_job; ++ } ++ ++ if (NULL != physical_pp_job_working) { ++ if (MALI_TRUE == mali_pp_job_is_protected_job(physical_pp_job_working)) { ++ gpu_working_in_protected_mode = MALI_TRUE; ++ } else { ++ gpu_working_in_non_protected_mode = MALI_TRUE; ++ } ++ } else if (NULL != virtual_pp_job_working) { ++ if (MALI_TRUE == mali_pp_job_is_protected_job(virtual_pp_job_working)) { ++ gpu_working_in_protected_mode = MALI_TRUE; ++ } else { ++ gpu_working_in_non_protected_mode = MALI_TRUE; ++ } ++ } else if (EXEC_STATE_WORKING == gp_group_state) { ++ gpu_working_in_non_protected_mode = MALI_TRUE; ++ } ++ ++ /* If the next pp job is the protected pp job.*/ ++ if ((NULL != next_pp_job_to_start) && MALI_TRUE == mali_pp_job_is_protected_job(next_pp_job_to_start)) { ++ /* if gp is working or any non-protected pp job is working now, unable to schedule protected pp job. */ ++ if (MALI_TRUE == gpu_working_in_non_protected_mode) ++ return MALI_TRUE; ++ ++ *gpu_secure_mode_is_needed = MALI_TRUE; ++ return MALI_FALSE; ++ ++ } ++ ++ if (MALI_TRUE == gpu_working_in_protected_mode) { ++ /* Unable to schedule non-protected pp job/gp job if exist protected pp running jobs*/ ++ return MALI_TRUE; ++ } ++ ++ return MALI_FALSE; ++} ++/* ++ * This is where jobs are actually started. ++ */ ++static void mali_executor_schedule(void) ++{ ++ u32 i; ++ u32 num_physical_needed = 0; ++ u32 num_physical_to_process = 0; ++ mali_bool trigger_pm_update = MALI_FALSE; ++ mali_bool deactivate_idle_group = MALI_TRUE; ++ mali_bool gpu_secure_mode_is_needed = MALI_FALSE; ++ mali_bool is_gpu_secure_mode = MALI_FALSE; ++ /* Physical groups + jobs to start in this function */ ++ struct mali_group *groups_to_start[MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS]; ++ struct mali_pp_job *jobs_to_start[MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS]; ++ u32 sub_jobs_to_start[MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS]; ++ int num_jobs_to_start = 0; ++ ++ /* Virtual job to start in this function */ ++ struct mali_pp_job *virtual_job_to_start = NULL; ++ ++ /* GP job to start in this function */ ++ struct mali_gp_job *gp_job_to_start = NULL; ++ ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ ++ if (pause_count > 0) { ++ /* Execution is suspended, don't schedule any jobs. */ ++ return; ++ } ++ ++ /* Lock needed in order to safely handle the job queues */ ++ mali_scheduler_lock(); ++ ++ /* 1. Check the schedule if need to early out. */ ++ if (MALI_TRUE == mali_executor_schedule_is_early_out(&gpu_secure_mode_is_needed)) { ++ mali_scheduler_unlock(); ++ return; ++ } ++ ++ /* 2. Activate gp firstly if have gp job queued. */ ++ if ((EXEC_STATE_INACTIVE == gp_group_state) ++ && (0 < mali_scheduler_job_gp_count()) ++ && (gpu_secure_mode_is_needed == MALI_FALSE)) { ++ ++ enum mali_group_state state = ++ mali_group_activate(gp_group); ++ if (MALI_GROUP_STATE_ACTIVE == state) { ++ /* Set GP group state to idle */ ++ gp_group_state = EXEC_STATE_IDLE; ++ } else { ++ trigger_pm_update = MALI_TRUE; ++ } ++ } ++ ++ /* 3. Prepare as many physical groups as needed/possible */ ++ ++ num_physical_needed = mali_scheduler_job_physical_head_count(gpu_secure_mode_is_needed); ++ ++ /* On mali-450 platform, we don't need to enter in this block frequently. */ ++ if (0 < num_physical_needed) { ++ ++ if (num_physical_needed <= group_list_idle_count) { ++ /* We have enough groups on idle list already */ ++ num_physical_to_process = num_physical_needed; ++ num_physical_needed = 0; ++ } else { ++ /* We need to get a hold of some more groups */ ++ num_physical_to_process = group_list_idle_count; ++ num_physical_needed -= group_list_idle_count; ++ } ++ ++ if (0 < num_physical_needed) { ++ ++ /* 3.1. Activate groups which are inactive */ ++ ++ struct mali_group *group; ++ struct mali_group *temp; ++ ++ _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_inactive, ++ struct mali_group, executor_list) { ++ enum mali_group_state state = ++ mali_group_activate(group); ++ if (MALI_GROUP_STATE_ACTIVE == state) { ++ /* Move from inactive to idle */ ++ mali_executor_change_state_pp_physical(group, ++ &group_list_inactive, ++ &group_list_inactive_count, ++ &group_list_idle, ++ &group_list_idle_count); ++ num_physical_to_process++; ++ } else { ++ trigger_pm_update = MALI_TRUE; ++ } ++ ++ num_physical_needed--; ++ if (0 == num_physical_needed) { ++ /* We have activated all the groups we need */ ++ break; ++ } ++ } ++ } ++ ++ if (mali_executor_virtual_group_is_usable()) { ++ ++ /* ++ * 3.2. And finally, steal and activate groups ++ * from virtual group if we need even more ++ */ ++ while (0 < num_physical_needed) { ++ struct mali_group *group; ++ ++ group = mali_group_acquire_group(virtual_group); ++ if (NULL != group) { ++ enum mali_group_state state; ++ ++ mali_executor_disable_empty_virtual(); ++ ++ state = mali_group_activate(group); ++ if (MALI_GROUP_STATE_ACTIVE == state) { ++ /* Group is ready, add to idle list */ ++ _mali_osk_list_add( ++ &group->executor_list, ++ &group_list_idle); ++ group_list_idle_count++; ++ num_physical_to_process++; ++ } else { ++ /* ++ * Group is not ready yet, ++ * add to inactive list ++ */ ++ _mali_osk_list_add( ++ &group->executor_list, ++ &group_list_inactive); ++ group_list_inactive_count++; ++ ++ trigger_pm_update = MALI_TRUE; ++ } ++ num_physical_needed--; ++ } else { ++ /* ++ * We could not get enough groups ++ * from the virtual group. ++ */ ++ break; ++ } ++ } ++ } ++ ++ /* 3.3. Assign physical jobs to groups */ ++ ++ if (0 < num_physical_to_process) { ++ struct mali_group *group; ++ struct mali_group *temp; ++ ++ _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_idle, ++ struct mali_group, executor_list) { ++ struct mali_pp_job *job = NULL; ++ u32 sub_job = MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS; ++ ++ MALI_DEBUG_ASSERT(num_jobs_to_start < ++ MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS); ++ ++ MALI_DEBUG_ASSERT(0 < ++ mali_scheduler_job_physical_head_count(gpu_secure_mode_is_needed)); ++ ++ /* If the next pp job is non-protected, check if gp bound now. */ ++ if ((MALI_FALSE == gpu_secure_mode_is_needed) ++ && (mali_executor_hint_is_enabled(MALI_EXECUTOR_HINT_GP_BOUND)) ++ && (MALI_TRUE == mali_executor_tackle_gp_bound())) { ++ /* ++ * We're gp bound, ++ * don't start this right now. ++ */ ++ deactivate_idle_group = MALI_FALSE; ++ num_physical_to_process = 0; ++ break; ++ } ++ ++ job = mali_scheduler_job_pp_physical_get( ++ &sub_job); ++ ++ if (MALI_FALSE == gpu_secure_mode_is_needed) { ++ MALI_DEBUG_ASSERT(MALI_FALSE == mali_pp_job_is_protected_job(job)); ++ } else { ++ MALI_DEBUG_ASSERT(MALI_TRUE == mali_pp_job_is_protected_job(job)); ++ } ++ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ MALI_DEBUG_ASSERT(sub_job <= MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS); ++ ++ /* Put job + group on list of jobs to start later on */ ++ ++ groups_to_start[num_jobs_to_start] = group; ++ jobs_to_start[num_jobs_to_start] = job; ++ sub_jobs_to_start[num_jobs_to_start] = sub_job; ++ num_jobs_to_start++; ++ ++ /* Move group from idle to working */ ++ mali_executor_change_state_pp_physical(group, ++ &group_list_idle, ++ &group_list_idle_count, ++ &group_list_working, ++ &group_list_working_count); ++ ++ num_physical_to_process--; ++ if (0 == num_physical_to_process) { ++ /* Got all we needed */ ++ break; ++ } ++ } ++ } ++ } ++ ++ /* 4. Deactivate idle pp group , must put deactive here before active vitual group ++ * for cover case first only has physical job in normal queue but group inactive, ++ * so delay the job start go to active group, when group activated, ++ * call scheduler again, but now if we get high queue virtual job, ++ * we will do nothing in schedule cause executor schedule stop ++ */ ++ ++ if (MALI_TRUE == mali_executor_deactivate_list_idle(deactivate_idle_group ++ && (!mali_timeline_has_physical_pp_job()))) { ++ trigger_pm_update = MALI_TRUE; ++ } ++ ++ /* 5. Activate virtual group, if needed */ ++ if (EXEC_STATE_INACTIVE == virtual_group_state && ++ MALI_TRUE == mali_scheduler_job_next_is_virtual()) { ++ struct mali_pp_job *virtual_job = mali_scheduler_job_pp_virtual_peek(); ++ if ((MALI_FALSE == gpu_secure_mode_is_needed && MALI_FALSE == mali_pp_job_is_protected_job(virtual_job)) ++ || (MALI_TRUE == gpu_secure_mode_is_needed && MALI_TRUE == mali_pp_job_is_protected_job(virtual_job))) { ++ enum mali_group_state state = ++ mali_group_activate(virtual_group); ++ if (MALI_GROUP_STATE_ACTIVE == state) { ++ /* Set virtual group state to idle */ ++ virtual_group_state = EXEC_STATE_IDLE; ++ } else { ++ trigger_pm_update = MALI_TRUE; ++ } ++ } ++ } ++ ++ /* 6. To power up group asap, trigger pm update only when no need to swith the gpu mode. */ ++ ++ is_gpu_secure_mode = _mali_osk_gpu_secure_mode_is_enabled(); ++ ++ if ((MALI_FALSE == gpu_secure_mode_is_needed && MALI_FALSE == is_gpu_secure_mode) ++ || (MALI_TRUE == gpu_secure_mode_is_needed && MALI_TRUE == is_gpu_secure_mode)) { ++ if (MALI_TRUE == trigger_pm_update) { ++ trigger_pm_update = MALI_FALSE; ++ mali_pm_update_async(); ++ } ++ } ++ ++ /* 7. Assign jobs to idle virtual group (or deactivate if no job) */ ++ ++ if (EXEC_STATE_IDLE == virtual_group_state) { ++ if (MALI_TRUE == mali_scheduler_job_next_is_virtual()) { ++ struct mali_pp_job *virtual_job = mali_scheduler_job_pp_virtual_peek(); ++ if ((MALI_FALSE == gpu_secure_mode_is_needed && MALI_FALSE == mali_pp_job_is_protected_job(virtual_job)) ++ || (MALI_TRUE == gpu_secure_mode_is_needed && MALI_TRUE == mali_pp_job_is_protected_job(virtual_job))) { ++ virtual_job_to_start = ++ mali_scheduler_job_pp_virtual_get(); ++ virtual_group_state = EXEC_STATE_WORKING; ++ } ++ } else if (!mali_timeline_has_virtual_pp_job()) { ++ virtual_group_state = EXEC_STATE_INACTIVE; ++ ++ if (mali_group_deactivate(virtual_group)) { ++ trigger_pm_update = MALI_TRUE; ++ } ++ } ++ } ++ ++ /* 8. Assign job to idle GP group (or deactivate if no job) */ ++ ++ if (EXEC_STATE_IDLE == gp_group_state && MALI_FALSE == gpu_secure_mode_is_needed) { ++ if (0 < mali_scheduler_job_gp_count()) { ++ gp_job_to_start = mali_scheduler_job_gp_get(); ++ gp_group_state = EXEC_STATE_WORKING; ++ } else if (!mali_timeline_has_gp_job()) { ++ gp_group_state = EXEC_STATE_INACTIVE; ++ if (mali_group_deactivate(gp_group)) { ++ trigger_pm_update = MALI_TRUE; ++ } ++ } ++ } ++ ++ /* 9. We no longer need the schedule/queue lock */ ++ ++ mali_scheduler_unlock(); ++ ++ /* 10. start jobs */ ++ if (NULL != virtual_job_to_start) { ++ MALI_DEBUG_ASSERT(!mali_group_pp_is_active(virtual_group)); ++ mali_group_start_pp_job(virtual_group, ++ virtual_job_to_start, 0, is_gpu_secure_mode); ++ } ++ ++ for (i = 0; i < num_jobs_to_start; i++) { ++ MALI_DEBUG_ASSERT(!mali_group_pp_is_active( ++ groups_to_start[i])); ++ mali_group_start_pp_job(groups_to_start[i], ++ jobs_to_start[i], ++ sub_jobs_to_start[i], is_gpu_secure_mode); ++ } ++ ++ MALI_DEBUG_ASSERT_POINTER(gp_group); ++ ++ if (NULL != gp_job_to_start) { ++ MALI_DEBUG_ASSERT(!mali_group_gp_is_active(gp_group)); ++ mali_group_start_gp_job(gp_group, gp_job_to_start, is_gpu_secure_mode); ++ } ++ ++ /* 11. Trigger any pending PM updates */ ++ if (MALI_TRUE == trigger_pm_update) { ++ mali_pm_update_async(); ++ } ++} ++ ++/* Handler for deferred schedule requests */ ++static void mali_executor_wq_schedule(void *arg) ++{ ++ MALI_IGNORE(arg); ++ mali_executor_lock(); ++ mali_executor_schedule(); ++ mali_executor_unlock(); ++} ++ ++static void mali_executor_send_gp_oom_to_user(struct mali_gp_job *job) ++{ ++ _mali_uk_gp_job_suspended_s *jobres; ++ _mali_osk_notification_t *notification; ++ ++ notification = mali_gp_job_get_oom_notification(job); ++ ++ /* ++ * Remember the id we send to user space, so we have something to ++ * verify when we get a response ++ */ ++ gp_returned_cookie = mali_gp_job_get_id(job); ++ ++ jobres = (_mali_uk_gp_job_suspended_s *)notification->result_buffer; ++ jobres->user_job_ptr = mali_gp_job_get_user_id(job); ++ jobres->cookie = gp_returned_cookie; ++ ++ mali_session_send_notification(mali_gp_job_get_session(job), ++ notification); ++} ++static struct mali_gp_job *mali_executor_complete_gp(struct mali_group *group, ++ mali_bool success) ++{ ++ struct mali_gp_job *job; ++ ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ ++ /* Extracts the needed HW status from core and reset */ ++ job = mali_group_complete_gp(group, success); ++ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ ++ /* Core is now ready to go into idle list */ ++ gp_group_state = EXEC_STATE_IDLE; ++ ++ /* This will potentially queue more GP and PP jobs */ ++ mali_timeline_tracker_release(&job->tracker); ++ ++ /* Signal PP job */ ++ mali_gp_job_signal_pp_tracker(job, success); ++ ++ return job; ++} ++ ++static struct mali_pp_job *mali_executor_complete_pp(struct mali_group *group, ++ mali_bool success) ++{ ++ struct mali_pp_job *job; ++ u32 sub_job; ++ mali_bool job_is_done; ++ ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ ++ /* Extracts the needed HW status from core and reset */ ++ job = mali_group_complete_pp(group, success, &sub_job); ++ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ ++ /* Core is now ready to go into idle list */ ++ if (mali_group_is_virtual(group)) { ++ virtual_group_state = EXEC_STATE_IDLE; ++ } else { ++ /* Move from working to idle state */ ++ mali_executor_change_state_pp_physical(group, ++ &group_list_working, ++ &group_list_working_count, ++ &group_list_idle, ++ &group_list_idle_count); ++ } ++ ++ /* It is the executor module which owns the jobs themselves by now */ ++ mali_pp_job_mark_sub_job_completed(job, success); ++ job_is_done = mali_pp_job_is_complete(job); ++ ++ if (job_is_done) { ++ /* This will potentially queue more GP and PP jobs */ ++ mali_timeline_tracker_release(&job->tracker); ++ } ++ ++ return job; ++} ++ ++static void mali_executor_complete_group(struct mali_group *group, ++ mali_bool success, ++ struct mali_gp_job **gp_job_done, ++ struct mali_pp_job **pp_job_done) ++{ ++ struct mali_gp_core *gp_core = mali_group_get_gp_core(group); ++ struct mali_pp_core *pp_core = mali_group_get_pp_core(group); ++ struct mali_gp_job *gp_job = NULL; ++ struct mali_pp_job *pp_job = NULL; ++ mali_bool pp_job_is_done = MALI_TRUE; ++ ++ if (NULL != gp_core) { ++ gp_job = mali_executor_complete_gp(group, success); ++ } else { ++ MALI_DEBUG_ASSERT_POINTER(pp_core); ++ MALI_IGNORE(pp_core); ++ pp_job = mali_executor_complete_pp(group, success); ++ ++ pp_job_is_done = mali_pp_job_is_complete(pp_job); ++ } ++ ++ if (pause_count > 0) { ++ /* Execution has been suspended */ ++ ++ if (!mali_executor_is_working()) { ++ /* Last job completed, wake up sleepers */ ++ _mali_osk_wait_queue_wake_up( ++ executor_working_wait_queue); ++ } ++ } else if (MALI_TRUE == mali_group_disable_requested(group)) { ++ mali_executor_core_scale_in_group_complete(group); ++ ++ mali_executor_schedule(); ++ } else { ++ /* try to schedule new jobs */ ++ mali_executor_schedule(); ++ } ++ ++ if (NULL != gp_job) { ++ MALI_DEBUG_ASSERT_POINTER(gp_job_done); ++ *gp_job_done = gp_job; ++ } else if (pp_job_is_done) { ++ MALI_DEBUG_ASSERT_POINTER(pp_job); ++ MALI_DEBUG_ASSERT_POINTER(pp_job_done); ++ *pp_job_done = pp_job; ++ } ++} ++ ++static void mali_executor_change_state_pp_physical(struct mali_group *group, ++ _mali_osk_list_t *old_list, ++ u32 *old_count, ++ _mali_osk_list_t *new_list, ++ u32 *new_count) ++{ ++ /* ++ * It's a bit more complicated to change the state for the physical PP ++ * groups since their state is determined by the list they are on. ++ */ ++#if defined(DEBUG) ++ mali_bool found = MALI_FALSE; ++ struct mali_group *group_iter; ++ struct mali_group *temp; ++ u32 old_counted = 0; ++ u32 new_counted = 0; ++ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ MALI_DEBUG_ASSERT_POINTER(old_list); ++ MALI_DEBUG_ASSERT_POINTER(old_count); ++ MALI_DEBUG_ASSERT_POINTER(new_list); ++ MALI_DEBUG_ASSERT_POINTER(new_count); ++ ++ /* ++ * Verify that group is present on old list, ++ * and that the count is correct ++ */ ++ ++ _MALI_OSK_LIST_FOREACHENTRY(group_iter, temp, old_list, ++ struct mali_group, executor_list) { ++ old_counted++; ++ if (group == group_iter) { ++ found = MALI_TRUE; ++ } ++ } ++ ++ _MALI_OSK_LIST_FOREACHENTRY(group_iter, temp, new_list, ++ struct mali_group, executor_list) { ++ new_counted++; ++ } ++ ++ if (MALI_FALSE == found) { ++ if (old_list == &group_list_idle) { ++ MALI_DEBUG_PRINT(1, (" old Group list is idle,")); ++ } else if (old_list == &group_list_inactive) { ++ MALI_DEBUG_PRINT(1, (" old Group list is inactive,")); ++ } else if (old_list == &group_list_working) { ++ MALI_DEBUG_PRINT(1, (" old Group list is working,")); ++ } else if (old_list == &group_list_disabled) { ++ MALI_DEBUG_PRINT(1, (" old Group list is disable,")); ++ } ++ ++ if (MALI_TRUE == mali_executor_group_is_in_state(group, EXEC_STATE_WORKING)) { ++ MALI_DEBUG_PRINT(1, (" group in working \n")); ++ } else if (MALI_TRUE == mali_executor_group_is_in_state(group, EXEC_STATE_INACTIVE)) { ++ MALI_DEBUG_PRINT(1, (" group in inactive \n")); ++ } else if (MALI_TRUE == mali_executor_group_is_in_state(group, EXEC_STATE_IDLE)) { ++ MALI_DEBUG_PRINT(1, (" group in idle \n")); ++ } else if (MALI_TRUE == mali_executor_group_is_in_state(group, EXEC_STATE_DISABLED)) { ++ MALI_DEBUG_PRINT(1, (" but group in disabled \n")); ++ } ++ } ++ ++ MALI_DEBUG_ASSERT(MALI_TRUE == found); ++ MALI_DEBUG_ASSERT(0 < (*old_count)); ++ MALI_DEBUG_ASSERT((*old_count) == old_counted); ++ MALI_DEBUG_ASSERT((*new_count) == new_counted); ++#endif ++ ++ _mali_osk_list_move(&group->executor_list, new_list); ++ (*old_count)--; ++ (*new_count)++; ++} ++ ++static void mali_executor_set_state_pp_physical(struct mali_group *group, ++ _mali_osk_list_t *new_list, ++ u32 *new_count) ++{ ++ _mali_osk_list_add(&group->executor_list, new_list); ++ (*new_count)++; ++} ++ ++static mali_bool mali_executor_group_is_in_state(struct mali_group *group, ++ enum mali_executor_state_t state) ++{ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ ++ if (gp_group == group) { ++ if (gp_group_state == state) { ++ return MALI_TRUE; ++ } ++ } else if (virtual_group == group || mali_group_is_in_virtual(group)) { ++ if (virtual_group_state == state) { ++ return MALI_TRUE; ++ } ++ } else { ++ /* Physical PP group */ ++ struct mali_group *group_iter; ++ struct mali_group *temp; ++ _mali_osk_list_t *list; ++ ++ if (EXEC_STATE_DISABLED == state) { ++ list = &group_list_disabled; ++ } else if (EXEC_STATE_INACTIVE == state) { ++ list = &group_list_inactive; ++ } else if (EXEC_STATE_IDLE == state) { ++ list = &group_list_idle; ++ } else { ++ MALI_DEBUG_ASSERT(EXEC_STATE_WORKING == state); ++ list = &group_list_working; ++ } ++ ++ _MALI_OSK_LIST_FOREACHENTRY(group_iter, temp, list, ++ struct mali_group, executor_list) { ++ if (group_iter == group) { ++ return MALI_TRUE; ++ } ++ } ++ } ++ ++ /* group not in correct state */ ++ return MALI_FALSE; ++} ++ ++static void mali_executor_group_enable_internal(struct mali_group *group) ++{ ++ MALI_DEBUG_ASSERT(group); ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ MALI_DEBUG_ASSERT(mali_executor_group_is_in_state(group, EXEC_STATE_DISABLED)); ++ ++ /* Put into inactive state (== "lowest" enabled state) */ ++ if (group == gp_group) { ++ MALI_DEBUG_ASSERT(EXEC_STATE_DISABLED == gp_group_state); ++ gp_group_state = EXEC_STATE_INACTIVE; ++ } else { ++ mali_executor_change_state_pp_physical(group, ++ &group_list_disabled, ++ &group_list_disabled_count, ++ &group_list_inactive, ++ &group_list_inactive_count); ++ ++ ++num_physical_pp_cores_enabled; ++ MALI_DEBUG_PRINT(4, ("Enabling group id %d \n", group->pp_core->core_id)); ++ } ++ ++ if (MALI_GROUP_STATE_ACTIVE == mali_group_activate(group)) { ++ MALI_DEBUG_ASSERT(MALI_TRUE == mali_group_power_is_on(group)); ++ ++ /* Move from inactive to idle */ ++ if (group == gp_group) { ++ gp_group_state = EXEC_STATE_IDLE; ++ } else { ++ mali_executor_change_state_pp_physical(group, ++ &group_list_inactive, ++ &group_list_inactive_count, ++ &group_list_idle, ++ &group_list_idle_count); ++ ++ if (mali_executor_has_virtual_group()) { ++ if (mali_executor_physical_rejoin_virtual(group)) { ++ mali_pm_update_async(); ++ } ++ } ++ } ++ } else { ++ mali_pm_update_async(); ++ } ++} ++ ++static void mali_executor_group_disable_internal(struct mali_group *group) ++{ ++ mali_bool working; ++ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ MALI_DEBUG_ASSERT(!mali_executor_group_is_in_state(group, EXEC_STATE_DISABLED)); ++ ++ working = mali_executor_group_is_in_state(group, EXEC_STATE_WORKING); ++ if (MALI_TRUE == working) { ++ /** Group to be disabled once it completes current work, ++ * when virtual group completes, also check child groups for this flag */ ++ mali_group_set_disable_request(group, MALI_TRUE); ++ return; ++ } ++ ++ /* Put into disabled state */ ++ if (group == gp_group) { ++ /* GP group */ ++ MALI_DEBUG_ASSERT(EXEC_STATE_WORKING != gp_group_state); ++ gp_group_state = EXEC_STATE_DISABLED; ++ } else { ++ if (mali_group_is_in_virtual(group)) { ++ /* A child group of virtual group. move the specific group from virtual group */ ++ MALI_DEBUG_ASSERT(EXEC_STATE_WORKING != virtual_group_state); ++ ++ mali_executor_set_state_pp_physical(group, ++ &group_list_disabled, ++ &group_list_disabled_count); ++ ++ mali_group_remove_group(virtual_group, group); ++ mali_executor_disable_empty_virtual(); ++ } else { ++ mali_executor_change_group_status_disabled(group); ++ } ++ ++ --num_physical_pp_cores_enabled; ++ MALI_DEBUG_PRINT(4, ("Disabling group id %d \n", group->pp_core->core_id)); ++ } ++ ++ if (MALI_GROUP_STATE_INACTIVE != group->state) { ++ if (MALI_TRUE == mali_group_deactivate(group)) { ++ mali_pm_update_async(); ++ } ++ } ++} ++ ++static void mali_executor_notify_core_change(u32 num_cores) ++{ ++ mali_bool done = MALI_FALSE; ++ ++ if (mali_is_mali450() || mali_is_mali470()) { ++ return; ++ } ++ ++ /* ++ * This function gets a bit complicated because we can't hold the session lock while ++ * allocating notification objects. ++ */ ++ while (!done) { ++ u32 i; ++ u32 num_sessions_alloc; ++ u32 num_sessions_with_lock; ++ u32 used_notification_objects = 0; ++ _mali_osk_notification_t **notobjs; ++ ++ /* Pre allocate the number of notifications objects we need right now (might change after lock has been taken) */ ++ num_sessions_alloc = mali_session_get_count(); ++ if (0 == num_sessions_alloc) { ++ /* No sessions to report to */ ++ return; ++ } ++ ++ notobjs = (_mali_osk_notification_t **)_mali_osk_malloc(sizeof(_mali_osk_notification_t *) * num_sessions_alloc); ++ if (NULL == notobjs) { ++ MALI_PRINT_ERROR(("Failed to notify user space session about num PP core change (alloc failure)\n")); ++ /* there is probably no point in trying again, system must be really low on memory and probably unusable now anyway */ ++ return; ++ } ++ ++ for (i = 0; i < num_sessions_alloc; i++) { ++ notobjs[i] = _mali_osk_notification_create(_MALI_NOTIFICATION_PP_NUM_CORE_CHANGE, sizeof(_mali_uk_pp_num_cores_changed_s)); ++ if (NULL != notobjs[i]) { ++ _mali_uk_pp_num_cores_changed_s *data = notobjs[i]->result_buffer; ++ data->number_of_enabled_cores = num_cores; ++ } else { ++ MALI_PRINT_ERROR(("Failed to notify user space session about num PP core change (alloc failure %u)\n", i)); ++ } ++ } ++ ++ mali_session_lock(); ++ ++ /* number of sessions will not change while we hold the lock */ ++ num_sessions_with_lock = mali_session_get_count(); ++ ++ if (num_sessions_alloc >= num_sessions_with_lock) { ++ /* We have allocated enough notification objects for all the sessions atm */ ++ struct mali_session_data *session, *tmp; ++ MALI_SESSION_FOREACH(session, tmp, link) { ++ MALI_DEBUG_ASSERT(used_notification_objects < num_sessions_alloc); ++ if (NULL != notobjs[used_notification_objects]) { ++ mali_session_send_notification(session, notobjs[used_notification_objects]); ++ notobjs[used_notification_objects] = NULL; /* Don't track this notification object any more */ ++ } ++ used_notification_objects++; ++ } ++ done = MALI_TRUE; ++ } ++ ++ mali_session_unlock(); ++ ++ /* Delete any remaining/unused notification objects */ ++ for (; used_notification_objects < num_sessions_alloc; used_notification_objects++) { ++ if (NULL != notobjs[used_notification_objects]) { ++ _mali_osk_notification_delete(notobjs[used_notification_objects]); ++ } ++ } ++ ++ _mali_osk_free(notobjs); ++ } ++} ++ ++static mali_bool mali_executor_core_scaling_is_done(void *data) ++{ ++ u32 i; ++ u32 num_groups; ++ mali_bool ret = MALI_TRUE; ++ ++ MALI_IGNORE(data); ++ ++ mali_executor_lock(); ++ ++ num_groups = mali_group_get_glob_num_groups(); ++ ++ for (i = 0; i < num_groups; i++) { ++ struct mali_group *group = mali_group_get_glob_group(i); ++ ++ if (NULL != group) { ++ if (MALI_TRUE == group->disable_requested && NULL != mali_group_get_pp_core(group)) { ++ ret = MALI_FALSE; ++ break; ++ } ++ } ++ } ++ mali_executor_unlock(); ++ ++ return ret; ++} ++ ++static void mali_executor_wq_notify_core_change(void *arg) ++{ ++ MALI_IGNORE(arg); ++ ++ if (mali_is_mali450() || mali_is_mali470()) { ++ return; ++ } ++ ++ _mali_osk_wait_queue_wait_event(executor_notify_core_change_wait_queue, ++ mali_executor_core_scaling_is_done, NULL); ++ ++ mali_executor_notify_core_change(num_physical_pp_cores_enabled); ++} ++ ++/** ++ * Clear all disable request from the _last_ core scaling behavior. ++ */ ++static void mali_executor_core_scaling_reset(void) ++{ ++ u32 i; ++ u32 num_groups; ++ ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ ++ num_groups = mali_group_get_glob_num_groups(); ++ ++ for (i = 0; i < num_groups; i++) { ++ struct mali_group *group = mali_group_get_glob_group(i); ++ ++ if (NULL != group) { ++ group->disable_requested = MALI_FALSE; ++ } ++ } ++ ++ for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) { ++ core_scaling_delay_up_mask[i] = 0; ++ } ++} ++ ++static void mali_executor_core_scale(unsigned int target_core_nr) ++{ ++ int current_core_scaling_mask[MALI_MAX_NUMBER_OF_DOMAINS] = { 0 }; ++ int target_core_scaling_mask[MALI_MAX_NUMBER_OF_DOMAINS] = { 0 }; ++ int i; ++ ++ MALI_DEBUG_ASSERT(0 < target_core_nr); ++ MALI_DEBUG_ASSERT(num_physical_pp_cores_total >= target_core_nr); ++ ++ mali_executor_lock(); ++ ++ if (target_core_nr < num_physical_pp_cores_enabled) { ++ MALI_DEBUG_PRINT(2, ("Requesting %d cores: disabling %d cores\n", target_core_nr, num_physical_pp_cores_enabled - target_core_nr)); ++ } else { ++ MALI_DEBUG_PRINT(2, ("Requesting %d cores: enabling %d cores\n", target_core_nr, target_core_nr - num_physical_pp_cores_enabled)); ++ } ++ ++ /* When a new core scaling request is comming, we should remove the un-doing ++ * part of the last core scaling request. It's safe because we have only one ++ * lock(executor lock) protection. */ ++ mali_executor_core_scaling_reset(); ++ ++ mali_pm_get_best_power_cost_mask(num_physical_pp_cores_enabled, current_core_scaling_mask); ++ mali_pm_get_best_power_cost_mask(target_core_nr, target_core_scaling_mask); ++ ++ for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) { ++ target_core_scaling_mask[i] = target_core_scaling_mask[i] - current_core_scaling_mask[i]; ++ MALI_DEBUG_PRINT(5, ("target_core_scaling_mask[%d] = %d\n", i, target_core_scaling_mask[i])); ++ } ++ ++ for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) { ++ if (0 > target_core_scaling_mask[i]) { ++ struct mali_pm_domain *domain; ++ ++ domain = mali_pm_domain_get_from_index(i); ++ ++ /* Domain is valid and has pp cores */ ++ if ((NULL != domain) && !(_mali_osk_list_empty(&domain->group_list))) { ++ struct mali_group *group; ++ struct mali_group *temp; ++ ++ _MALI_OSK_LIST_FOREACHENTRY(group, temp, &domain->group_list, struct mali_group, pm_domain_list) { ++ if (NULL != mali_group_get_pp_core(group) && (!mali_executor_group_is_in_state(group, EXEC_STATE_DISABLED)) ++ && (!mali_group_is_virtual(group))) { ++ mali_executor_group_disable_internal(group); ++ target_core_scaling_mask[i]++; ++ if ((0 == target_core_scaling_mask[i])) { ++ break; ++ } ++ ++ } ++ } ++ } ++ } ++ } ++ ++ for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) { ++ /** ++ * Target_core_scaling_mask[i] is bigger than 0, ++ * means we need to enable some pp cores in ++ * this domain whose domain index is i. ++ */ ++ if (0 < target_core_scaling_mask[i]) { ++ struct mali_pm_domain *domain; ++ ++ if (num_physical_pp_cores_enabled >= target_core_nr) { ++ break; ++ } ++ ++ domain = mali_pm_domain_get_from_index(i); ++ ++ /* Domain is valid and has pp cores */ ++ if ((NULL != domain) && !(_mali_osk_list_empty(&domain->group_list))) { ++ struct mali_group *group; ++ struct mali_group *temp; ++ ++ _MALI_OSK_LIST_FOREACHENTRY(group, temp, &domain->group_list, struct mali_group, pm_domain_list) { ++ if (NULL != mali_group_get_pp_core(group) && mali_executor_group_is_in_state(group, EXEC_STATE_DISABLED) ++ && (!mali_group_is_virtual(group))) { ++ mali_executor_group_enable_internal(group); ++ target_core_scaling_mask[i]--; ++ ++ if ((0 == target_core_scaling_mask[i]) || num_physical_pp_cores_enabled == target_core_nr) { ++ break; ++ } ++ } ++ } ++ } ++ } ++ } ++ ++ /** ++ * Here, we may still have some pp cores not been enabled because of some ++ * pp cores need to be disabled are still in working state. ++ */ ++ for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) { ++ if (0 < target_core_scaling_mask[i]) { ++ core_scaling_delay_up_mask[i] = target_core_scaling_mask[i]; ++ } ++ } ++ ++ mali_executor_schedule(); ++ mali_executor_unlock(); ++} ++ ++static void mali_executor_core_scale_in_group_complete(struct mali_group *group) ++{ ++ int num_pp_cores_disabled = 0; ++ int num_pp_cores_to_enable = 0; ++ int i; ++ ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ MALI_DEBUG_ASSERT(MALI_TRUE == mali_group_disable_requested(group)); ++ ++ /* Disable child group of virtual group */ ++ if (mali_group_is_virtual(group)) { ++ struct mali_group *child; ++ struct mali_group *temp; ++ ++ _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, struct mali_group, group_list) { ++ if (MALI_TRUE == mali_group_disable_requested(child)) { ++ mali_group_set_disable_request(child, MALI_FALSE); ++ mali_executor_group_disable_internal(child); ++ num_pp_cores_disabled++; ++ } ++ } ++ mali_group_set_disable_request(group, MALI_FALSE); ++ } else { ++ mali_executor_group_disable_internal(group); ++ mali_group_set_disable_request(group, MALI_FALSE); ++ if (NULL != mali_group_get_pp_core(group)) { ++ num_pp_cores_disabled++; ++ } ++ } ++ ++ num_pp_cores_to_enable = num_pp_cores_disabled; ++ ++ for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) { ++ if (0 < core_scaling_delay_up_mask[i]) { ++ struct mali_pm_domain *domain; ++ ++ if (0 == num_pp_cores_to_enable) { ++ break; ++ } ++ ++ domain = mali_pm_domain_get_from_index(i); ++ ++ /* Domain is valid and has pp cores */ ++ if ((NULL != domain) && !(_mali_osk_list_empty(&domain->group_list))) { ++ struct mali_group *disabled_group; ++ struct mali_group *temp; ++ ++ _MALI_OSK_LIST_FOREACHENTRY(disabled_group, temp, &domain->group_list, struct mali_group, pm_domain_list) { ++ if (NULL != mali_group_get_pp_core(disabled_group) && mali_executor_group_is_in_state(disabled_group, EXEC_STATE_DISABLED)) { ++ mali_executor_group_enable_internal(disabled_group); ++ core_scaling_delay_up_mask[i]--; ++ num_pp_cores_to_enable--; ++ ++ if ((0 == core_scaling_delay_up_mask[i]) || 0 == num_pp_cores_to_enable) { ++ break; ++ } ++ } ++ } ++ } ++ } ++ } ++ ++ _mali_osk_wait_queue_wake_up(executor_notify_core_change_wait_queue); ++} ++ ++static void mali_executor_change_group_status_disabled(struct mali_group *group) ++{ ++ /* Physical PP group */ ++ mali_bool idle; ++ ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ ++ idle = mali_executor_group_is_in_state(group, EXEC_STATE_IDLE); ++ if (MALI_TRUE == idle) { ++ mali_executor_change_state_pp_physical(group, ++ &group_list_idle, ++ &group_list_idle_count, ++ &group_list_disabled, ++ &group_list_disabled_count); ++ } else { ++ mali_executor_change_state_pp_physical(group, ++ &group_list_inactive, ++ &group_list_inactive_count, ++ &group_list_disabled, ++ &group_list_disabled_count); ++ } ++} ++ ++static mali_bool mali_executor_deactivate_list_idle(mali_bool deactivate_idle_group) ++{ ++ mali_bool trigger_pm_update = MALI_FALSE; ++ ++ if (group_list_idle_count > 0) { ++ if (mali_executor_has_virtual_group()) { ++ ++ /* Rejoin virtual group on Mali-450 */ ++ ++ struct mali_group *group; ++ struct mali_group *temp; ++ ++ _MALI_OSK_LIST_FOREACHENTRY(group, temp, ++ &group_list_idle, ++ struct mali_group, executor_list) { ++ if (mali_executor_physical_rejoin_virtual( ++ group)) { ++ trigger_pm_update = MALI_TRUE; ++ } ++ } ++ } else if (deactivate_idle_group) { ++ struct mali_group *group; ++ struct mali_group *temp; ++ ++ /* Deactivate group on Mali-300/400 */ ++ ++ _MALI_OSK_LIST_FOREACHENTRY(group, temp, ++ &group_list_idle, ++ struct mali_group, executor_list) { ++ if (mali_group_deactivate(group)) { ++ trigger_pm_update = MALI_TRUE; ++ } ++ ++ /* Move from idle to inactive */ ++ mali_executor_change_state_pp_physical(group, ++ &group_list_idle, ++ &group_list_idle_count, ++ &group_list_inactive, ++ &group_list_inactive_count); ++ } ++ } ++ } ++ ++ return trigger_pm_update; ++} ++ ++void mali_executor_running_status_print(void) ++{ ++ struct mali_group *group = NULL; ++ struct mali_group *temp = NULL; ++ ++ MALI_PRINT(("GP running job: %p\n", gp_group->gp_running_job)); ++ if ((gp_group->gp_core) && (gp_group->is_working)) { ++ mali_group_dump_status(gp_group); ++ } ++ MALI_PRINT(("Physical PP groups in WORKING state (count = %u):\n", group_list_working_count)); ++ _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_working, struct mali_group, executor_list) { ++ MALI_PRINT(("PP running job: %p, subjob %d \n", group->pp_running_job, group->pp_running_sub_job)); ++ mali_group_dump_status(group); ++ } ++ MALI_PRINT(("Physical PP groups in INACTIVE state (count = %u):\n", group_list_inactive_count)); ++ _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_inactive, struct mali_group, executor_list) { ++ MALI_PRINT(("\tPP status %d, SW power: %s\n", group->state, group->power_is_on ? "On" : "Off")); ++ MALI_PRINT(("\tPP #%d: %s\n", group->pp_core->core_id, group->pp_core->hw_core.description)); ++ } ++ MALI_PRINT(("Physical PP groups in IDLE state (count = %u):\n", group_list_idle_count)); ++ _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_idle, struct mali_group, executor_list) { ++ MALI_PRINT(("\tPP status %d, SW power: %s\n", group->state, group->power_is_on ? "On" : "Off")); ++ MALI_PRINT(("\tPP #%d: %s\n", group->pp_core->core_id, group->pp_core->hw_core.description)); ++ } ++ MALI_PRINT(("Physical PP groups in DISABLED state (count = %u):\n", group_list_disabled_count)); ++ _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_disabled, struct mali_group, executor_list) { ++ MALI_PRINT(("\tPP status %d, SW power: %s\n", group->state, group->power_is_on ? "On" : "Off")); ++ MALI_PRINT(("\tPP #%d: %s\n", group->pp_core->core_id, group->pp_core->hw_core.description)); ++ } ++ ++ if (mali_executor_has_virtual_group()) { ++ MALI_PRINT(("Virtual group running job: %p\n", virtual_group->pp_running_job)); ++ MALI_PRINT(("Virtual group status: %d\n", virtual_group_state)); ++ MALI_PRINT(("Virtual group->status: %d\n", virtual_group->state)); ++ MALI_PRINT(("\tSW power: %s\n", virtual_group->power_is_on ? "On" : "Off")); ++ _MALI_OSK_LIST_FOREACHENTRY(group, temp, &virtual_group->group_list, ++ struct mali_group, group_list) { ++ int i = 0; ++ MALI_PRINT(("\tchild group(%s) running job: %p\n", group->pp_core->hw_core.description, group->pp_running_job)); ++ MALI_PRINT(("\tchild group(%s)->status: %d\n", group->pp_core->hw_core.description, group->state)); ++ MALI_PRINT(("\tchild group(%s) SW power: %s\n", group->pp_core->hw_core.description, group->power_is_on ? "On" : "Off")); ++ if (group->pm_domain) { ++ MALI_PRINT(("\tPower domain: id %u\n", mali_pm_domain_get_id(group->pm_domain))); ++ MALI_PRINT(("\tMask:0x%04x \n", mali_pm_domain_get_mask(group->pm_domain))); ++ MALI_PRINT(("\tUse-count:%u \n", mali_pm_domain_get_use_count(group->pm_domain))); ++ MALI_PRINT(("\tCurrent power status:%s \n", (mali_pm_domain_get_mask(group->pm_domain)& mali_pm_get_current_mask()) ? "On" : "Off")); ++ MALI_PRINT(("\tWanted power status:%s \n", (mali_pm_domain_get_mask(group->pm_domain)& mali_pm_get_wanted_mask()) ? "On" : "Off")); ++ } ++ ++ for (i = 0; i < 2; i++) { ++ if (NULL != group->l2_cache_core[i]) { ++ struct mali_pm_domain *domain; ++ domain = mali_l2_cache_get_pm_domain(group->l2_cache_core[i]); ++ MALI_PRINT(("\t L2(index %d) group SW power: %s\n", i, group->l2_cache_core[i]->power_is_on ? "On" : "Off")); ++ if (domain) { ++ MALI_PRINT(("\tL2 Power domain: id %u\n", mali_pm_domain_get_id(domain))); ++ MALI_PRINT(("\tL2 Mask:0x%04x \n", mali_pm_domain_get_mask(domain))); ++ MALI_PRINT(("\tL2 Use-count:%u \n", mali_pm_domain_get_use_count(domain))); ++ MALI_PRINT(("\tL2 Current power status:%s \n", (mali_pm_domain_get_mask(domain) & mali_pm_get_current_mask()) ? "On" : "Off")); ++ MALI_PRINT(("\tL2 Wanted power status:%s \n", (mali_pm_domain_get_mask(domain) & mali_pm_get_wanted_mask()) ? "On" : "Off")); ++ } ++ } ++ } ++ } ++ if (EXEC_STATE_WORKING == virtual_group_state) { ++ mali_group_dump_status(virtual_group); ++ } ++ } ++} ++ ++void mali_executor_status_dump(void) ++{ ++ mali_executor_lock(); ++ mali_scheduler_lock(); ++ ++ /* print schedule queue status */ ++ mali_scheduler_gp_pp_job_queue_print(); ++ ++ mali_scheduler_unlock(); ++ mali_executor_unlock(); ++} +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_executor.h b/drivers/gpu/arm/mali400/mali/common/mali_executor.h +new file mode 100755 +index 000000000000..4224d6a6cdc4 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_executor.h +@@ -0,0 +1,102 @@ ++/* ++ * Copyright (C) 2012, 2014-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_EXECUTOR_H__ ++#define __MALI_EXECUTOR_H__ ++ ++#include "mali_osk.h" ++#include "mali_scheduler_types.h" ++#include "mali_kernel_common.h" ++ ++typedef enum { ++ MALI_EXECUTOR_HINT_GP_BOUND = 0 ++#define MALI_EXECUTOR_HINT_MAX 1 ++} mali_executor_hint; ++ ++extern mali_bool mali_executor_hints[MALI_EXECUTOR_HINT_MAX]; ++ ++/* forward declare struct instead of using include */ ++struct mali_session_data; ++struct mali_group; ++struct mali_pp_core; ++ ++extern _mali_osk_spinlock_irq_t *mali_executor_lock_obj; ++ ++#define MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD() MALI_DEBUG_ASSERT_LOCK_HELD(mali_executor_lock_obj); ++ ++_mali_osk_errcode_t mali_executor_initialize(void); ++void mali_executor_terminate(void); ++ ++void mali_executor_populate(void); ++void mali_executor_depopulate(void); ++ ++void mali_executor_suspend(void); ++void mali_executor_resume(void); ++ ++u32 mali_executor_get_num_cores_total(void); ++u32 mali_executor_get_num_cores_enabled(void); ++struct mali_pp_core *mali_executor_get_virtual_pp(void); ++struct mali_group *mali_executor_get_virtual_group(void); ++ ++void mali_executor_zap_all_active(struct mali_session_data *session); ++ ++/** ++ * Schedule GP and PP according to bitmask. ++ * ++ * @param mask A scheduling bitmask. ++ * @param deferred_schedule MALI_TRUE if schedule should be deferred, MALI_FALSE if not. ++ */ ++void mali_executor_schedule_from_mask(mali_scheduler_mask mask, mali_bool deferred_schedule); ++ ++_mali_osk_errcode_t mali_executor_interrupt_gp(struct mali_group *group, mali_bool in_upper_half); ++_mali_osk_errcode_t mali_executor_interrupt_pp(struct mali_group *group, mali_bool in_upper_half); ++_mali_osk_errcode_t mali_executor_interrupt_mmu(struct mali_group *group, mali_bool in_upper_half); ++void mali_executor_group_power_up(struct mali_group *groups[], u32 num_groups); ++void mali_executor_group_power_down(struct mali_group *groups[], u32 num_groups); ++ ++void mali_executor_abort_session(struct mali_session_data *session); ++ ++void mali_executor_core_scaling_enable(void); ++void mali_executor_core_scaling_disable(void); ++mali_bool mali_executor_core_scaling_is_enabled(void); ++ ++void mali_executor_group_enable(struct mali_group *group); ++void mali_executor_group_disable(struct mali_group *group); ++mali_bool mali_executor_group_is_disabled(struct mali_group *group); ++ ++int mali_executor_set_perf_level(unsigned int target_core_nr, mali_bool override); ++ ++#if MALI_STATE_TRACKING ++u32 mali_executor_dump_state(char *buf, u32 size); ++#endif ++ ++MALI_STATIC_INLINE void mali_executor_hint_enable(mali_executor_hint hint) ++{ ++ MALI_DEBUG_ASSERT(hint < MALI_EXECUTOR_HINT_MAX); ++ mali_executor_hints[hint] = MALI_TRUE; ++} ++ ++MALI_STATIC_INLINE void mali_executor_hint_disable(mali_executor_hint hint) ++{ ++ MALI_DEBUG_ASSERT(hint < MALI_EXECUTOR_HINT_MAX); ++ mali_executor_hints[hint] = MALI_FALSE; ++} ++ ++MALI_STATIC_INLINE mali_bool mali_executor_hint_is_enabled(mali_executor_hint hint) ++{ ++ MALI_DEBUG_ASSERT(hint < MALI_EXECUTOR_HINT_MAX); ++ return mali_executor_hints[hint]; ++} ++ ++void mali_executor_running_status_print(void); ++void mali_executor_status_dump(void); ++void mali_executor_lock(void); ++void mali_executor_unlock(void); ++#endif /* __MALI_EXECUTOR_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_gp.c b/drivers/gpu/arm/mali400/mali/common/mali_gp.c +new file mode 100755 +index 000000000000..7d3d4aff7c3f +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_gp.c +@@ -0,0 +1,357 @@ ++/* ++ * Copyright (C) 2011-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include "mali_gp.h" ++#include "mali_hw_core.h" ++#include "mali_group.h" ++#include "mali_osk.h" ++#include "regs/mali_gp_regs.h" ++#include "mali_kernel_common.h" ++#include "mali_kernel_core.h" ++#if defined(CONFIG_MALI400_PROFILING) ++#include "mali_osk_profiling.h" ++#endif ++ ++static struct mali_gp_core *mali_global_gp_core = NULL; ++ ++/* Interrupt handlers */ ++static void mali_gp_irq_probe_trigger(void *data); ++static _mali_osk_errcode_t mali_gp_irq_probe_ack(void *data); ++ ++struct mali_gp_core *mali_gp_create(const _mali_osk_resource_t *resource, struct mali_group *group) ++{ ++ struct mali_gp_core *core = NULL; ++ ++ MALI_DEBUG_ASSERT(NULL == mali_global_gp_core); ++ MALI_DEBUG_PRINT(2, ("Mali GP: Creating Mali GP core: %s\n", resource->description)); ++ ++ core = _mali_osk_malloc(sizeof(struct mali_gp_core)); ++ if (NULL != core) { ++ if (_MALI_OSK_ERR_OK == mali_hw_core_create(&core->hw_core, resource, MALIGP2_REGISTER_ADDRESS_SPACE_SIZE)) { ++ _mali_osk_errcode_t ret; ++ ++ ret = mali_gp_reset(core); ++ ++ if (_MALI_OSK_ERR_OK == ret) { ++ ret = mali_group_add_gp_core(group, core); ++ if (_MALI_OSK_ERR_OK == ret) { ++ /* Setup IRQ handlers (which will do IRQ probing if needed) */ ++ core->irq = _mali_osk_irq_init(resource->irq, ++ mali_group_upper_half_gp, ++ group, ++ mali_gp_irq_probe_trigger, ++ mali_gp_irq_probe_ack, ++ core, ++ resource->description); ++ if (NULL != core->irq) { ++ MALI_DEBUG_PRINT(4, ("Mali GP: set global gp core from 0x%08X to 0x%08X\n", mali_global_gp_core, core)); ++ mali_global_gp_core = core; ++ ++ return core; ++ } else { ++ MALI_PRINT_ERROR(("Mali GP: Failed to setup interrupt handlers for GP core %s\n", core->hw_core.description)); ++ } ++ mali_group_remove_gp_core(group); ++ } else { ++ MALI_PRINT_ERROR(("Mali GP: Failed to add core %s to group\n", core->hw_core.description)); ++ } ++ } ++ mali_hw_core_delete(&core->hw_core); ++ } ++ ++ _mali_osk_free(core); ++ } else { ++ MALI_PRINT_ERROR(("Failed to allocate memory for GP core\n")); ++ } ++ ++ return NULL; ++} ++ ++void mali_gp_delete(struct mali_gp_core *core) ++{ ++ MALI_DEBUG_ASSERT_POINTER(core); ++ ++ _mali_osk_irq_term(core->irq); ++ mali_hw_core_delete(&core->hw_core); ++ mali_global_gp_core = NULL; ++ _mali_osk_free(core); ++} ++ ++void mali_gp_stop_bus(struct mali_gp_core *core) ++{ ++ MALI_DEBUG_ASSERT_POINTER(core); ++ ++ mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_CMD, MALIGP2_REG_VAL_CMD_STOP_BUS); ++} ++ ++_mali_osk_errcode_t mali_gp_stop_bus_wait(struct mali_gp_core *core) ++{ ++ int i; ++ ++ MALI_DEBUG_ASSERT_POINTER(core); ++ ++ /* Send the stop bus command. */ ++ mali_gp_stop_bus(core); ++ ++ /* Wait for bus to be stopped */ ++ for (i = 0; i < MALI_REG_POLL_COUNT_SLOW; i++) { ++ if (mali_hw_core_register_read(&core->hw_core, MALIGP2_REG_ADDR_MGMT_STATUS) & MALIGP2_REG_VAL_STATUS_BUS_STOPPED) { ++ break; ++ } ++ } ++ ++ if (MALI_REG_POLL_COUNT_SLOW == i) { ++ MALI_PRINT_ERROR(("Mali GP: Failed to stop bus on %s\n", core->hw_core.description)); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ return _MALI_OSK_ERR_OK; ++} ++ ++void mali_gp_hard_reset(struct mali_gp_core *core) ++{ ++ const u32 reset_wait_target_register = MALIGP2_REG_ADDR_MGMT_PERF_CNT_0_LIMIT; ++ const u32 reset_invalid_value = 0xC0FFE000; ++ const u32 reset_check_value = 0xC01A0000; ++ const u32 reset_default_value = 0; ++ int i; ++ ++ MALI_DEBUG_ASSERT_POINTER(core); ++ MALI_DEBUG_PRINT(4, ("Mali GP: Hard reset of core %s\n", core->hw_core.description)); ++ ++ mali_hw_core_register_write(&core->hw_core, reset_wait_target_register, reset_invalid_value); ++ ++ mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_CMD, MALIGP2_REG_VAL_CMD_RESET); ++ ++ for (i = 0; i < MALI_REG_POLL_COUNT_FAST; i++) { ++ mali_hw_core_register_write(&core->hw_core, reset_wait_target_register, reset_check_value); ++ if (reset_check_value == mali_hw_core_register_read(&core->hw_core, reset_wait_target_register)) { ++ break; ++ } ++ } ++ ++ if (MALI_REG_POLL_COUNT_FAST == i) { ++ MALI_PRINT_ERROR(("Mali GP: The hard reset loop didn't work, unable to recover\n")); ++ } ++ ++ mali_hw_core_register_write(&core->hw_core, reset_wait_target_register, reset_default_value); /* set it back to the default */ ++ /* Re-enable interrupts */ ++ mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_CLEAR, MALIGP2_REG_VAL_IRQ_MASK_ALL); ++ mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_MASK, MALIGP2_REG_VAL_IRQ_MASK_USED); ++ ++} ++ ++void mali_gp_reset_async(struct mali_gp_core *core) ++{ ++ MALI_DEBUG_ASSERT_POINTER(core); ++ ++ MALI_DEBUG_PRINT(4, ("Mali GP: Reset of core %s\n", core->hw_core.description)); ++ ++ mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_MASK, 0); /* disable the IRQs */ ++ mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_CLEAR, MALI400GP_REG_VAL_IRQ_RESET_COMPLETED); ++ mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_CMD, MALI400GP_REG_VAL_CMD_SOFT_RESET); ++ ++} ++ ++_mali_osk_errcode_t mali_gp_reset_wait(struct mali_gp_core *core) ++{ ++ int i; ++ u32 rawstat = 0; ++ ++ MALI_DEBUG_ASSERT_POINTER(core); ++ ++ for (i = 0; i < MALI_REG_POLL_COUNT_FAST; i++) { ++ rawstat = mali_hw_core_register_read(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_RAWSTAT); ++ if (rawstat & MALI400GP_REG_VAL_IRQ_RESET_COMPLETED) { ++ break; ++ } ++ } ++ ++ if (i == MALI_REG_POLL_COUNT_FAST) { ++ MALI_PRINT_ERROR(("Mali GP: Failed to reset core %s, rawstat: 0x%08x\n", ++ core->hw_core.description, rawstat)); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ /* Re-enable interrupts */ ++ mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_CLEAR, MALIGP2_REG_VAL_IRQ_MASK_ALL); ++ mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_MASK, MALIGP2_REG_VAL_IRQ_MASK_USED); ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++_mali_osk_errcode_t mali_gp_reset(struct mali_gp_core *core) ++{ ++ mali_gp_reset_async(core); ++ return mali_gp_reset_wait(core); ++} ++ ++void mali_gp_job_start(struct mali_gp_core *core, struct mali_gp_job *job) ++{ ++ u32 startcmd = 0; ++ u32 *frame_registers = mali_gp_job_get_frame_registers(job); ++ u32 counter_src0 = mali_gp_job_get_perf_counter_src0(job); ++ u32 counter_src1 = mali_gp_job_get_perf_counter_src1(job); ++ ++ MALI_DEBUG_ASSERT_POINTER(core); ++ ++ if (mali_gp_job_has_vs_job(job)) { ++ startcmd |= (u32) MALIGP2_REG_VAL_CMD_START_VS; ++ } ++ ++ if (mali_gp_job_has_plbu_job(job)) { ++ startcmd |= (u32) MALIGP2_REG_VAL_CMD_START_PLBU; ++ } ++ ++ MALI_DEBUG_ASSERT(0 != startcmd); ++ ++ mali_hw_core_register_write_array_relaxed(&core->hw_core, MALIGP2_REG_ADDR_MGMT_VSCL_START_ADDR, frame_registers, MALIGP2_NUM_REGS_FRAME); ++ ++ if (MALI_HW_CORE_NO_COUNTER != counter_src0) { ++ mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_PERF_CNT_0_SRC, counter_src0); ++ mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_PERF_CNT_0_ENABLE, MALIGP2_REG_VAL_PERF_CNT_ENABLE); ++ } ++ if (MALI_HW_CORE_NO_COUNTER != counter_src1) { ++ mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_PERF_CNT_1_SRC, counter_src1); ++ mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_PERF_CNT_1_ENABLE, MALIGP2_REG_VAL_PERF_CNT_ENABLE); ++ } ++ ++ MALI_DEBUG_PRINT(3, ("Mali GP: Starting job (0x%08x) on core %s with command 0x%08X\n", job, core->hw_core.description, startcmd)); ++ ++ mali_hw_core_register_write_relaxed(&core->hw_core, MALIGP2_REG_ADDR_MGMT_CMD, MALIGP2_REG_VAL_CMD_UPDATE_PLBU_ALLOC); ++ ++ /* Barrier to make sure the previous register write is finished */ ++ _mali_osk_write_mem_barrier(); ++ ++ /* This is the command that starts the core. ++ * ++ * Don't actually run the job if PROFILING_SKIP_PP_JOBS are set, just ++ * force core to assert the completion interrupt. ++ */ ++#if !defined(PROFILING_SKIP_GP_JOBS) ++ mali_hw_core_register_write_relaxed(&core->hw_core, MALIGP2_REG_ADDR_MGMT_CMD, startcmd); ++#else ++ { ++ u32 bits = 0; ++ ++ if (mali_gp_job_has_vs_job(job)) ++ bits = MALIGP2_REG_VAL_IRQ_VS_END_CMD_LST; ++ if (mali_gp_job_has_plbu_job(job)) ++ bits |= MALIGP2_REG_VAL_IRQ_PLBU_END_CMD_LST; ++ ++ mali_hw_core_register_write_relaxed(&core->hw_core, ++ MALIGP2_REG_ADDR_MGMT_INT_RAWSTAT, bits); ++ } ++#endif ++ ++ /* Barrier to make sure the previous register write is finished */ ++ _mali_osk_write_mem_barrier(); ++} ++ ++void mali_gp_resume_with_new_heap(struct mali_gp_core *core, u32 start_addr, u32 end_addr) ++{ ++ u32 irq_readout; ++ ++ MALI_DEBUG_ASSERT_POINTER(core); ++ ++ irq_readout = mali_hw_core_register_read(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_RAWSTAT); ++ ++ if (irq_readout & MALIGP2_REG_VAL_IRQ_PLBU_OUT_OF_MEM) { ++ mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_CLEAR, (MALIGP2_REG_VAL_IRQ_PLBU_OUT_OF_MEM | MALIGP2_REG_VAL_IRQ_HANG)); ++ mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_MASK, MALIGP2_REG_VAL_IRQ_MASK_USED); /* re-enable interrupts */ ++ mali_hw_core_register_write_relaxed(&core->hw_core, MALIGP2_REG_ADDR_MGMT_PLBU_ALLOC_START_ADDR, start_addr); ++ mali_hw_core_register_write_relaxed(&core->hw_core, MALIGP2_REG_ADDR_MGMT_PLBU_ALLOC_END_ADDR, end_addr); ++ ++ MALI_DEBUG_PRINT(3, ("Mali GP: Resuming job\n")); ++ ++ mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_CMD, MALIGP2_REG_VAL_CMD_UPDATE_PLBU_ALLOC); ++ _mali_osk_write_mem_barrier(); ++ } ++ /* ++ * else: core has been reset between PLBU_OUT_OF_MEM interrupt and this new heap response. ++ * A timeout or a page fault on Mali-200 PP core can cause this behaviour. ++ */ ++} ++ ++u32 mali_gp_core_get_version(struct mali_gp_core *core) ++{ ++ MALI_DEBUG_ASSERT_POINTER(core); ++ return mali_hw_core_register_read(&core->hw_core, MALIGP2_REG_ADDR_MGMT_VERSION); ++} ++ ++struct mali_gp_core *mali_gp_get_global_gp_core(void) ++{ ++ return mali_global_gp_core; ++} ++ ++/* ------------- interrupt handling below ------------------ */ ++static void mali_gp_irq_probe_trigger(void *data) ++{ ++ struct mali_gp_core *core = (struct mali_gp_core *)data; ++ ++ mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_MASK, MALIGP2_REG_VAL_IRQ_MASK_USED); ++ mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_RAWSTAT, MALIGP2_REG_VAL_IRQ_AXI_BUS_ERROR); ++ _mali_osk_mem_barrier(); ++} ++ ++static _mali_osk_errcode_t mali_gp_irq_probe_ack(void *data) ++{ ++ struct mali_gp_core *core = (struct mali_gp_core *)data; ++ u32 irq_readout; ++ ++ irq_readout = mali_hw_core_register_read(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_STAT); ++ if (MALIGP2_REG_VAL_IRQ_AXI_BUS_ERROR & irq_readout) { ++ mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_CLEAR, MALIGP2_REG_VAL_IRQ_AXI_BUS_ERROR); ++ _mali_osk_mem_barrier(); ++ return _MALI_OSK_ERR_OK; ++ } ++ ++ return _MALI_OSK_ERR_FAULT; ++} ++ ++/* ------ local helper functions below --------- */ ++#if MALI_STATE_TRACKING ++u32 mali_gp_dump_state(struct mali_gp_core *core, char *buf, u32 size) ++{ ++ int n = 0; ++ ++ n += _mali_osk_snprintf(buf + n, size - n, "\tGP: %s\n", core->hw_core.description); ++ ++ return n; ++} ++#endif ++ ++void mali_gp_update_performance_counters(struct mali_gp_core *core, struct mali_gp_job *job) ++{ ++ u32 val0 = 0; ++ u32 val1 = 0; ++ u32 counter_src0 = mali_gp_job_get_perf_counter_src0(job); ++ u32 counter_src1 = mali_gp_job_get_perf_counter_src1(job); ++ ++ if (MALI_HW_CORE_NO_COUNTER != counter_src0) { ++ val0 = mali_hw_core_register_read(&core->hw_core, MALIGP2_REG_ADDR_MGMT_PERF_CNT_0_VALUE); ++ mali_gp_job_set_perf_counter_value0(job, val0); ++ ++#if defined(CONFIG_MALI400_PROFILING) ++ _mali_osk_profiling_report_hw_counter(COUNTER_VP_0_C0, val0); ++ _mali_osk_profiling_record_global_counters(COUNTER_VP_0_C0, val0); ++#endif ++ ++ } ++ ++ if (MALI_HW_CORE_NO_COUNTER != counter_src1) { ++ val1 = mali_hw_core_register_read(&core->hw_core, MALIGP2_REG_ADDR_MGMT_PERF_CNT_1_VALUE); ++ mali_gp_job_set_perf_counter_value1(job, val1); ++ ++#if defined(CONFIG_MALI400_PROFILING) ++ _mali_osk_profiling_report_hw_counter(COUNTER_VP_0_C1, val1); ++ _mali_osk_profiling_record_global_counters(COUNTER_VP_0_C1, val1); ++#endif ++ } ++} +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_gp.h b/drivers/gpu/arm/mali400/mali/common/mali_gp.h +new file mode 100755 +index 000000000000..3156310f21c7 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_gp.h +@@ -0,0 +1,127 @@ ++/* ++ * Copyright (C) 2011-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_GP_H__ ++#define __MALI_GP_H__ ++ ++#include "mali_osk.h" ++#include "mali_gp_job.h" ++#include "mali_hw_core.h" ++#include "regs/mali_gp_regs.h" ++ ++struct mali_group; ++ ++/** ++ * Definition of the GP core struct ++ * Used to track a GP core in the system. ++ */ ++struct mali_gp_core { ++ struct mali_hw_core hw_core; /**< Common for all HW cores */ ++ _mali_osk_irq_t *irq; /**< IRQ handler */ ++}; ++ ++_mali_osk_errcode_t mali_gp_initialize(void); ++void mali_gp_terminate(void); ++ ++struct mali_gp_core *mali_gp_create(const _mali_osk_resource_t *resource, struct mali_group *group); ++void mali_gp_delete(struct mali_gp_core *core); ++ ++void mali_gp_stop_bus(struct mali_gp_core *core); ++_mali_osk_errcode_t mali_gp_stop_bus_wait(struct mali_gp_core *core); ++void mali_gp_reset_async(struct mali_gp_core *core); ++_mali_osk_errcode_t mali_gp_reset_wait(struct mali_gp_core *core); ++void mali_gp_hard_reset(struct mali_gp_core *core); ++_mali_osk_errcode_t mali_gp_reset(struct mali_gp_core *core); ++ ++void mali_gp_job_start(struct mali_gp_core *core, struct mali_gp_job *job); ++void mali_gp_resume_with_new_heap(struct mali_gp_core *core, u32 start_addr, u32 end_addr); ++ ++u32 mali_gp_core_get_version(struct mali_gp_core *core); ++ ++struct mali_gp_core *mali_gp_get_global_gp_core(void); ++ ++#if MALI_STATE_TRACKING ++u32 mali_gp_dump_state(struct mali_gp_core *core, char *buf, u32 size); ++#endif ++ ++void mali_gp_update_performance_counters(struct mali_gp_core *core, struct mali_gp_job *job); ++ ++MALI_STATIC_INLINE const char *mali_gp_core_description(struct mali_gp_core *core) ++{ ++ return core->hw_core.description; ++} ++ ++MALI_STATIC_INLINE enum mali_interrupt_result mali_gp_get_interrupt_result(struct mali_gp_core *core) ++{ ++ u32 stat_used = mali_hw_core_register_read(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_STAT) & ++ MALIGP2_REG_VAL_IRQ_MASK_USED; ++ ++ if (0 == stat_used) { ++ return MALI_INTERRUPT_RESULT_NONE; ++ } else if ((MALIGP2_REG_VAL_IRQ_VS_END_CMD_LST | ++ MALIGP2_REG_VAL_IRQ_PLBU_END_CMD_LST) == stat_used) { ++ return MALI_INTERRUPT_RESULT_SUCCESS; ++ } else if (MALIGP2_REG_VAL_IRQ_VS_END_CMD_LST == stat_used) { ++ return MALI_INTERRUPT_RESULT_SUCCESS_VS; ++ } else if (MALIGP2_REG_VAL_IRQ_PLBU_END_CMD_LST == stat_used) { ++ return MALI_INTERRUPT_RESULT_SUCCESS_PLBU; ++ } else if (MALIGP2_REG_VAL_IRQ_PLBU_OUT_OF_MEM & stat_used) { ++ return MALI_INTERRUPT_RESULT_OOM; ++ } ++ ++ return MALI_INTERRUPT_RESULT_ERROR; ++} ++ ++MALI_STATIC_INLINE u32 mali_gp_get_rawstat(struct mali_gp_core *core) ++{ ++ MALI_DEBUG_ASSERT_POINTER(core); ++ return mali_hw_core_register_read(&core->hw_core, ++ MALIGP2_REG_ADDR_MGMT_INT_RAWSTAT); ++} ++ ++MALI_STATIC_INLINE u32 mali_gp_is_active(struct mali_gp_core *core) ++{ ++ u32 status = mali_hw_core_register_read(&core->hw_core, MALIGP2_REG_ADDR_MGMT_STATUS); ++ return (status & MALIGP2_REG_VAL_STATUS_MASK_ACTIVE) ? MALI_TRUE : MALI_FALSE; ++} ++ ++MALI_STATIC_INLINE void mali_gp_mask_all_interrupts(struct mali_gp_core *core) ++{ ++ mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_MASK, MALIGP2_REG_VAL_IRQ_MASK_NONE); ++} ++ ++MALI_STATIC_INLINE void mali_gp_enable_interrupts(struct mali_gp_core *core, enum mali_interrupt_result exceptions) ++{ ++ /* Enable all interrupts, except those specified in exceptions */ ++ u32 value; ++ ++ if (MALI_INTERRUPT_RESULT_SUCCESS_VS == exceptions) { ++ /* Enable all used except VS complete */ ++ value = MALIGP2_REG_VAL_IRQ_MASK_USED & ++ ~MALIGP2_REG_VAL_IRQ_VS_END_CMD_LST; ++ } else { ++ MALI_DEBUG_ASSERT(MALI_INTERRUPT_RESULT_SUCCESS_PLBU == ++ exceptions); ++ /* Enable all used except PLBU complete */ ++ value = MALIGP2_REG_VAL_IRQ_MASK_USED & ++ ~MALIGP2_REG_VAL_IRQ_PLBU_END_CMD_LST; ++ } ++ ++ mali_hw_core_register_write(&core->hw_core, ++ MALIGP2_REG_ADDR_MGMT_INT_MASK, ++ value); ++} ++ ++MALI_STATIC_INLINE u32 mali_gp_read_plbu_alloc_start_addr(struct mali_gp_core *core) ++{ ++ return mali_hw_core_register_read(&core->hw_core, MALIGP2_REG_ADDR_MGMT_PLBU_ALLOC_START_ADDR); ++} ++ ++#endif /* __MALI_GP_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_gp_job.c b/drivers/gpu/arm/mali400/mali/common/mali_gp_job.c +new file mode 100755 +index 000000000000..5d4d9f2530d3 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_gp_job.c +@@ -0,0 +1,306 @@ ++/* ++ * Copyright (C) 2011-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include "mali_gp_job.h" ++#include "mali_osk.h" ++#include "mali_osk_list.h" ++#include "mali_uk_types.h" ++#include "mali_memory_virtual.h" ++#include "mali_memory_defer_bind.h" ++ ++static u32 gp_counter_src0 = MALI_HW_CORE_NO_COUNTER; /**< Performance counter 0, MALI_HW_CORE_NO_COUNTER for disabled */ ++static u32 gp_counter_src1 = MALI_HW_CORE_NO_COUNTER; /**< Performance counter 1, MALI_HW_CORE_NO_COUNTER for disabled */ ++static void _mali_gp_del_varying_allocations(struct mali_gp_job *job); ++ ++ ++static int _mali_gp_add_varying_allocations(struct mali_session_data *session, ++ struct mali_gp_job *job, ++ u32 *alloc, ++ u32 num) ++{ ++ int i = 0; ++ struct mali_gp_allocation_node *alloc_node; ++ mali_mem_allocation *mali_alloc = NULL; ++ struct mali_vma_node *mali_vma_node = NULL; ++ ++ for (i = 0 ; i < num ; i++) { ++ MALI_DEBUG_ASSERT(alloc[i]); ++ alloc_node = _mali_osk_calloc(1, sizeof(struct mali_gp_allocation_node)); ++ if (alloc_node) { ++ INIT_LIST_HEAD(&alloc_node->node); ++ /* find mali allocation structure by vaddress*/ ++ mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, alloc[i], 0); ++ ++ if (likely(mali_vma_node)) { ++ mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); ++ MALI_DEBUG_ASSERT(alloc[i] == mali_vma_node->vm_node.start); ++ } else { ++ MALI_DEBUG_PRINT(1, ("ERROE!_mali_gp_add_varying_allocations,can't find allocation %d by address =0x%x, num=%d\n", i, alloc[i], num)); ++ _mali_osk_free(alloc_node); ++ goto fail; ++ } ++ alloc_node->alloc = mali_alloc; ++ /* add to gp job varying alloc list*/ ++ list_move(&alloc_node->node, &job->varying_alloc); ++ } else ++ goto fail; ++ } ++ ++ return 0; ++fail: ++ MALI_DEBUG_PRINT(1, ("ERROE!_mali_gp_add_varying_allocations,failed to alloc memory!\n")); ++ _mali_gp_del_varying_allocations(job); ++ return -1; ++} ++ ++ ++static void _mali_gp_del_varying_allocations(struct mali_gp_job *job) ++{ ++ struct mali_gp_allocation_node *alloc_node, *tmp_node; ++ ++ list_for_each_entry_safe(alloc_node, tmp_node, &job->varying_alloc, node) { ++ list_del(&alloc_node->node); ++ kfree(alloc_node); ++ } ++ INIT_LIST_HEAD(&job->varying_alloc); ++} ++ ++struct mali_gp_job *mali_gp_job_create(struct mali_session_data *session, _mali_uk_gp_start_job_s *uargs, u32 id, struct mali_timeline_tracker *pp_tracker) ++{ ++ struct mali_gp_job *job; ++ u32 perf_counter_flag; ++ u32 __user *memory_list = NULL; ++ struct mali_gp_allocation_node *alloc_node, *tmp_node; ++ _mali_uk_gp_start_job_s copy_of_uargs; ++ ++ job = _mali_osk_calloc(1, sizeof(struct mali_gp_job)); ++ if (NULL != job) { ++ job->finished_notification = _mali_osk_notification_create(_MALI_NOTIFICATION_GP_FINISHED, sizeof(_mali_uk_gp_job_finished_s)); ++ if (NULL == job->finished_notification) { ++ goto fail3; ++ } ++ ++ job->oom_notification = _mali_osk_notification_create(_MALI_NOTIFICATION_GP_STALLED, sizeof(_mali_uk_gp_job_suspended_s)); ++ if (NULL == job->oom_notification) { ++ goto fail2; ++ } ++ ++ if (0 != _mali_osk_copy_from_user(&job->uargs, uargs, sizeof(_mali_uk_gp_start_job_s))) { ++ goto fail1; ++ } ++ ++ perf_counter_flag = mali_gp_job_get_perf_counter_flag(job); ++ ++ /* case when no counters came from user space ++ * so pass the debugfs / DS-5 provided global ones to the job object */ ++ if (!((perf_counter_flag & _MALI_PERFORMANCE_COUNTER_FLAG_SRC0_ENABLE) || ++ (perf_counter_flag & _MALI_PERFORMANCE_COUNTER_FLAG_SRC1_ENABLE))) { ++ mali_gp_job_set_perf_counter_src0(job, mali_gp_job_get_gp_counter_src0()); ++ mali_gp_job_set_perf_counter_src1(job, mali_gp_job_get_gp_counter_src1()); ++ } ++ ++ _mali_osk_list_init(&job->list); ++ job->session = session; ++ job->id = id; ++ job->heap_current_addr = job->uargs.frame_registers[4]; ++ job->perf_counter_value0 = 0; ++ job->perf_counter_value1 = 0; ++ job->pid = _mali_osk_get_pid(); ++ job->tid = _mali_osk_get_tid(); ++ ++ ++ INIT_LIST_HEAD(&job->varying_alloc); ++ INIT_LIST_HEAD(&job->vary_todo); ++ job->dmem = NULL; ++ ++ if (job->uargs.deferred_mem_num > session->allocation_mgr.mali_allocation_num) { ++ MALI_PRINT_ERROR(("Mali GP job: The number of varying buffer to defer bind is invalid !\n")); ++ goto fail1; ++ } ++ ++ /* add varying allocation list*/ ++ if (job->uargs.deferred_mem_num > 0) { ++ /* copy varying list from user space*/ ++ job->varying_list = _mali_osk_calloc(1, sizeof(u32) * job->uargs.deferred_mem_num); ++ if (!job->varying_list) { ++ MALI_PRINT_ERROR(("Mali GP job: allocate varying_list failed varying_alloc_num = %d !\n", job->uargs.deferred_mem_num)); ++ goto fail1; ++ } ++ ++ if (0 != _mali_osk_copy_from_user(©_of_uargs, uargs, sizeof(_mali_uk_gp_start_job_s))) { ++ goto fail1; ++ } ++ memory_list = (u32 __user *)(uintptr_t)copy_of_uargs.deferred_mem_list; ++ ++ if (0 != _mali_osk_copy_from_user(job->varying_list, memory_list, sizeof(u32) * job->uargs.deferred_mem_num)) { ++ MALI_PRINT_ERROR(("Mali GP job: Failed to copy varying list from user space!\n")); ++ goto fail; ++ } ++ ++ if (unlikely(_mali_gp_add_varying_allocations(session, job, job->varying_list, ++ job->uargs.deferred_mem_num))) { ++ MALI_PRINT_ERROR(("Mali GP job: _mali_gp_add_varying_allocations failed!\n")); ++ goto fail; ++ } ++ ++ /* do preparetion for each allocation */ ++ list_for_each_entry_safe(alloc_node, tmp_node, &job->varying_alloc, node) { ++ if (unlikely(_MALI_OSK_ERR_OK != mali_mem_defer_bind_allocation_prepare(alloc_node->alloc, &job->vary_todo, &job->required_varying_memsize))) { ++ MALI_PRINT_ERROR(("Mali GP job: mali_mem_defer_bind_allocation_prepare failed!\n")); ++ goto fail; ++ } ++ } ++ ++ _mali_gp_del_varying_allocations(job); ++ ++ /* bind varying here, to avoid memory latency issue. */ ++ { ++ struct mali_defer_mem_block dmem_block; ++ ++ INIT_LIST_HEAD(&dmem_block.free_pages); ++ atomic_set(&dmem_block.num_free_pages, 0); ++ ++ if (mali_mem_prepare_mem_for_job(job, &dmem_block)) { ++ MALI_PRINT_ERROR(("Mali GP job: mali_mem_prepare_mem_for_job failed!\n")); ++ goto fail; ++ } ++ if (_MALI_OSK_ERR_OK != mali_mem_defer_bind(job, &dmem_block)) { ++ MALI_PRINT_ERROR(("gp job create, mali_mem_defer_bind failed! GP %x fail!", job)); ++ goto fail; ++ } ++ } ++ ++ if (job->uargs.varying_memsize > MALI_UK_BIG_VARYING_SIZE) { ++ job->big_job = 1; ++ } ++ } ++ job->pp_tracker = pp_tracker; ++ if (NULL != job->pp_tracker) { ++ /* Take a reference on PP job's tracker that will be released when the GP ++ job is done. */ ++ mali_timeline_system_tracker_get(session->timeline_system, pp_tracker); ++ } ++ ++ mali_timeline_tracker_init(&job->tracker, MALI_TIMELINE_TRACKER_GP, NULL, job); ++ mali_timeline_fence_copy_uk_fence(&(job->tracker.fence), &(job->uargs.fence)); ++ ++ return job; ++ } else { ++ MALI_PRINT_ERROR(("Mali GP job: _mali_osk_calloc failed!\n")); ++ return NULL; ++ } ++ ++ ++fail: ++ _mali_osk_free(job->varying_list); ++ /* Handle allocate fail here, free all varying node */ ++ { ++ struct mali_backend_bind_list *bkn, *bkn_tmp; ++ list_for_each_entry_safe(bkn, bkn_tmp , &job->vary_todo, node) { ++ list_del(&bkn->node); ++ _mali_osk_free(bkn); ++ } ++ } ++fail1: ++ _mali_osk_notification_delete(job->oom_notification); ++fail2: ++ _mali_osk_notification_delete(job->finished_notification); ++fail3: ++ _mali_osk_free(job); ++ return NULL; ++} ++ ++void mali_gp_job_delete(struct mali_gp_job *job) ++{ ++ struct mali_backend_bind_list *bkn, *bkn_tmp; ++ MALI_DEBUG_ASSERT_POINTER(job); ++ MALI_DEBUG_ASSERT(NULL == job->pp_tracker); ++ MALI_DEBUG_ASSERT(_mali_osk_list_empty(&job->list)); ++ _mali_osk_free(job->varying_list); ++ ++ /* Handle allocate fail here, free all varying node */ ++ list_for_each_entry_safe(bkn, bkn_tmp , &job->vary_todo, node) { ++ list_del(&bkn->node); ++ _mali_osk_free(bkn); ++ } ++ ++ mali_mem_defer_dmem_free(job); ++ ++ /* de-allocate the pre-allocated oom notifications */ ++ if (NULL != job->oom_notification) { ++ _mali_osk_notification_delete(job->oom_notification); ++ job->oom_notification = NULL; ++ } ++ if (NULL != job->finished_notification) { ++ _mali_osk_notification_delete(job->finished_notification); ++ job->finished_notification = NULL; ++ } ++ ++ _mali_osk_free(job); ++} ++ ++void mali_gp_job_list_add(struct mali_gp_job *job, _mali_osk_list_t *list) ++{ ++ struct mali_gp_job *iter; ++ struct mali_gp_job *tmp; ++ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); ++ ++ /* Find position in list/queue where job should be added. */ ++ _MALI_OSK_LIST_FOREACHENTRY_REVERSE(iter, tmp, list, ++ struct mali_gp_job, list) { ++ ++ /* A span is used to handle job ID wrapping. */ ++ bool job_is_after = (mali_gp_job_get_id(job) - ++ mali_gp_job_get_id(iter)) < ++ MALI_SCHEDULER_JOB_ID_SPAN; ++ ++ if (job_is_after) { ++ break; ++ } ++ } ++ ++ _mali_osk_list_add(&job->list, &iter->list); ++} ++ ++u32 mali_gp_job_get_gp_counter_src0(void) ++{ ++ return gp_counter_src0; ++} ++ ++void mali_gp_job_set_gp_counter_src0(u32 counter) ++{ ++ gp_counter_src0 = counter; ++} ++ ++u32 mali_gp_job_get_gp_counter_src1(void) ++{ ++ return gp_counter_src1; ++} ++ ++void mali_gp_job_set_gp_counter_src1(u32 counter) ++{ ++ gp_counter_src1 = counter; ++} ++ ++mali_scheduler_mask mali_gp_job_signal_pp_tracker(struct mali_gp_job *job, mali_bool success) ++{ ++ mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; ++ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ ++ if (NULL != job->pp_tracker) { ++ schedule_mask |= mali_timeline_system_tracker_put(job->session->timeline_system, job->pp_tracker, MALI_FALSE == success); ++ job->pp_tracker = NULL; ++ } ++ ++ return schedule_mask; ++} +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_gp_job.h b/drivers/gpu/arm/mali400/mali/common/mali_gp_job.h +new file mode 100755 +index 000000000000..b84333f9f810 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_gp_job.h +@@ -0,0 +1,324 @@ ++/* ++ * Copyright (C) 2011-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_GP_JOB_H__ ++#define __MALI_GP_JOB_H__ ++ ++#include "mali_osk.h" ++#include "mali_osk_list.h" ++#include "mali_uk_types.h" ++#include "mali_session.h" ++#include "mali_timeline.h" ++#include "mali_scheduler_types.h" ++#include "mali_scheduler.h" ++#include "mali_executor.h" ++#include "mali_timeline.h" ++ ++struct mali_defer_mem; ++/** ++ * This structure represents a GP job ++ * ++ * The GP job object itself is not protected by any single lock, ++ * but relies on other locks instead (scheduler, executor and timeline lock). ++ * Think of the job object as moving between these sub systems through-out ++ * its lifetime. Different part of the GP job struct is used by different ++ * subsystems. Accessor functions ensure that correct lock is taken. ++ * Do NOT access any data members directly from outside this module! ++ */ ++struct mali_gp_job { ++ /* ++ * These members are typically only set at creation, ++ * and only read later on. ++ * They do not require any lock protection. ++ */ ++ _mali_uk_gp_start_job_s uargs; /**< Arguments from user space */ ++ struct mali_session_data *session; /**< Session which submitted this job */ ++ u32 pid; /**< Process ID of submitting process */ ++ u32 tid; /**< Thread ID of submitting thread */ ++ u32 id; /**< Identifier for this job in kernel space (sequential numbering) */ ++ u32 cache_order; /**< Cache order used for L2 cache flushing (sequential numbering) */ ++ struct mali_timeline_tracker tracker; /**< Timeline tracker for this job */ ++ struct mali_timeline_tracker *pp_tracker; /**< Pointer to Timeline tracker for PP job that depends on this job. */ ++ _mali_osk_notification_t *finished_notification; /**< Notification sent back to userspace on job complete */ ++ ++ /* ++ * These members are used by the scheduler, ++ * protected by scheduler lock ++ */ ++ _mali_osk_list_t list; /**< Used to link jobs together in the scheduler queue */ ++ ++ /* ++ * These members are used by the executor and/or group, ++ * protected by executor lock ++ */ ++ _mali_osk_notification_t *oom_notification; /**< Notification sent back to userspace on OOM */ ++ ++ /* ++ * Set by executor/group on job completion, read by scheduler when ++ * returning job to user. Hold executor lock when setting, ++ * no lock needed when reading ++ */ ++ u32 heap_current_addr; /**< Holds the current HEAP address when the job has completed */ ++ u32 perf_counter_value0; /**< Value of performance counter 0 (to be returned to user space) */ ++ u32 perf_counter_value1; /**< Value of performance counter 1 (to be returned to user space) */ ++ struct mali_defer_mem *dmem; /** < used for defer bind to store dmem info */ ++ struct list_head varying_alloc; /**< hold the list of varying allocations */ ++ u32 bind_flag; /** < flag for deferbind*/ ++ u32 *varying_list; /**< varying memory list need to to defer bind*/ ++ struct list_head vary_todo; /**< list of backend list need to do defer bind*/ ++ u32 required_varying_memsize; /** < size of varying memory to reallocate*/ ++ u32 big_job; /** < if the gp job have large varying output and may take long time*/ ++}; ++ ++#define MALI_DEFER_BIND_MEMORY_PREPARED (0x1 << 0) ++#define MALI_DEFER_BIND_MEMORY_BINDED (0x1 << 2) ++ ++struct mali_gp_allocation_node { ++ struct list_head node; ++ mali_mem_allocation *alloc; ++}; ++ ++struct mali_gp_job *mali_gp_job_create(struct mali_session_data *session, _mali_uk_gp_start_job_s *uargs, u32 id, struct mali_timeline_tracker *pp_tracker); ++void mali_gp_job_delete(struct mali_gp_job *job); ++ ++u32 mali_gp_job_get_gp_counter_src0(void); ++void mali_gp_job_set_gp_counter_src0(u32 counter); ++u32 mali_gp_job_get_gp_counter_src1(void); ++void mali_gp_job_set_gp_counter_src1(u32 counter); ++ ++MALI_STATIC_INLINE u32 mali_gp_job_get_id(struct mali_gp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return (NULL == job) ? 0 : job->id; ++} ++ ++MALI_STATIC_INLINE void mali_gp_job_set_cache_order(struct mali_gp_job *job, ++ u32 cache_order) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); ++ job->cache_order = cache_order; ++} ++ ++MALI_STATIC_INLINE u32 mali_gp_job_get_cache_order(struct mali_gp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return (NULL == job) ? 0 : job->cache_order; ++} ++ ++MALI_STATIC_INLINE u64 mali_gp_job_get_user_id(struct mali_gp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return job->uargs.user_job_ptr; ++} ++ ++MALI_STATIC_INLINE u32 mali_gp_job_get_frame_builder_id(struct mali_gp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return job->uargs.frame_builder_id; ++} ++ ++MALI_STATIC_INLINE u32 mali_gp_job_get_flush_id(struct mali_gp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return job->uargs.flush_id; ++} ++ ++MALI_STATIC_INLINE u32 mali_gp_job_get_pid(struct mali_gp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return job->pid; ++} ++ ++MALI_STATIC_INLINE u32 mali_gp_job_get_tid(struct mali_gp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return job->tid; ++} ++ ++MALI_STATIC_INLINE u32 *mali_gp_job_get_frame_registers(struct mali_gp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return job->uargs.frame_registers; ++} ++ ++MALI_STATIC_INLINE struct mali_session_data *mali_gp_job_get_session(struct mali_gp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return job->session; ++} ++ ++MALI_STATIC_INLINE mali_bool mali_gp_job_has_vs_job(struct mali_gp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return (job->uargs.frame_registers[0] != job->uargs.frame_registers[1]) ? MALI_TRUE : MALI_FALSE; ++} ++ ++MALI_STATIC_INLINE mali_bool mali_gp_job_has_plbu_job(struct mali_gp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return (job->uargs.frame_registers[2] != job->uargs.frame_registers[3]) ? MALI_TRUE : MALI_FALSE; ++} ++ ++MALI_STATIC_INLINE u32 mali_gp_job_get_current_heap_addr(struct mali_gp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return job->heap_current_addr; ++} ++ ++MALI_STATIC_INLINE void mali_gp_job_set_current_heap_addr(struct mali_gp_job *job, u32 heap_addr) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ job->heap_current_addr = heap_addr; ++} ++ ++MALI_STATIC_INLINE u32 mali_gp_job_get_perf_counter_flag(struct mali_gp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return job->uargs.perf_counter_flag; ++} ++ ++MALI_STATIC_INLINE u32 mali_gp_job_get_perf_counter_src0(struct mali_gp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return job->uargs.perf_counter_src0; ++} ++ ++MALI_STATIC_INLINE u32 mali_gp_job_get_perf_counter_src1(struct mali_gp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return job->uargs.perf_counter_src1; ++} ++ ++MALI_STATIC_INLINE u32 mali_gp_job_get_perf_counter_value0(struct mali_gp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return job->perf_counter_value0; ++} ++ ++MALI_STATIC_INLINE u32 mali_gp_job_get_perf_counter_value1(struct mali_gp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return job->perf_counter_value1; ++} ++ ++MALI_STATIC_INLINE void mali_gp_job_set_perf_counter_src0(struct mali_gp_job *job, u32 src) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ job->uargs.perf_counter_src0 = src; ++} ++ ++MALI_STATIC_INLINE void mali_gp_job_set_perf_counter_src1(struct mali_gp_job *job, u32 src) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ job->uargs.perf_counter_src1 = src; ++} ++ ++MALI_STATIC_INLINE void mali_gp_job_set_perf_counter_value0(struct mali_gp_job *job, u32 value) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ job->perf_counter_value0 = value; ++} ++ ++MALI_STATIC_INLINE void mali_gp_job_set_perf_counter_value1(struct mali_gp_job *job, u32 value) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ job->perf_counter_value1 = value; ++} ++ ++void mali_gp_job_list_add(struct mali_gp_job *job, _mali_osk_list_t *list); ++ ++MALI_STATIC_INLINE void mali_gp_job_list_move(struct mali_gp_job *job, ++ _mali_osk_list_t *list) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); ++ MALI_DEBUG_ASSERT(!_mali_osk_list_empty(&job->list)); ++ _mali_osk_list_move(&job->list, list); ++} ++ ++MALI_STATIC_INLINE void mali_gp_job_list_remove(struct mali_gp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); ++ _mali_osk_list_delinit(&job->list); ++} ++ ++MALI_STATIC_INLINE _mali_osk_notification_t * ++mali_gp_job_get_finished_notification(struct mali_gp_job *job) ++{ ++ _mali_osk_notification_t *notification; ++ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ MALI_DEBUG_ASSERT_POINTER(job->finished_notification); ++ ++ notification = job->finished_notification; ++ job->finished_notification = NULL; ++ ++ return notification; ++} ++ ++MALI_STATIC_INLINE _mali_osk_notification_t *mali_gp_job_get_oom_notification( ++ struct mali_gp_job *job) ++{ ++ _mali_osk_notification_t *notification; ++ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ MALI_DEBUG_ASSERT_POINTER(job->oom_notification); ++ ++ notification = job->oom_notification; ++ job->oom_notification = NULL; ++ ++ return notification; ++} ++ ++MALI_STATIC_INLINE void mali_gp_job_set_oom_notification( ++ struct mali_gp_job *job, ++ _mali_osk_notification_t *notification) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ MALI_DEBUG_ASSERT(NULL == job->oom_notification); ++ job->oom_notification = notification; ++} ++ ++MALI_STATIC_INLINE struct mali_timeline_tracker *mali_gp_job_get_tracker( ++ struct mali_gp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return &(job->tracker); ++} ++ ++ ++MALI_STATIC_INLINE u32 *mali_gp_job_get_timeline_point_ptr( ++ struct mali_gp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return (u32 __user *)(uintptr_t)job->uargs.timeline_point_ptr; ++} ++ ++ ++/** ++ * Release reference on tracker for PP job that depends on this GP job. ++ * ++ * @note If GP job has a reference on tracker, this function MUST be called before the GP job is ++ * deleted. ++ * ++ * @param job GP job that is done. ++ * @param success MALI_TRUE if job completed successfully, MALI_FALSE if not. ++ * @return A scheduling bitmask indicating whether scheduling needs to be done. ++ */ ++mali_scheduler_mask mali_gp_job_signal_pp_tracker(struct mali_gp_job *job, mali_bool success); ++ ++#endif /* __MALI_GP_JOB_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_group.c b/drivers/gpu/arm/mali400/mali/common/mali_group.c +new file mode 100755 +index 000000000000..47979a2f1a35 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_group.c +@@ -0,0 +1,1875 @@ ++/* ++ * Copyright (C) 2011-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++#include "mali_kernel_common.h" ++#include "mali_group.h" ++#include "mali_osk.h" ++#include "mali_l2_cache.h" ++#include "mali_gp.h" ++#include "mali_pp.h" ++#include "mali_mmu.h" ++#include "mali_dlbu.h" ++#include "mali_broadcast.h" ++#include "mali_scheduler.h" ++#include "mali_osk_profiling.h" ++#include "mali_osk_mali.h" ++#include "mali_pm_domain.h" ++#include "mali_pm.h" ++#include "mali_executor.h" ++ ++#if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS) ++#include ++#include ++#endif ++ ++#define MALI_MAX_NUM_DOMAIN_REFS (MALI_MAX_NUMBER_OF_GROUPS * 2) ++ ++#if defined(CONFIG_MALI400_PROFILING) ++static void mali_group_report_l2_cache_counters_per_core(struct mali_group *group, u32 core_num); ++#endif /* #if defined(CONFIG_MALI400_PROFILING) */ ++ ++static struct mali_group *mali_global_groups[MALI_MAX_NUMBER_OF_GROUPS] = { NULL, }; ++static u32 mali_global_num_groups = 0; ++ ++/* SW timer for job execution */ ++int mali_max_job_runtime = MALI_MAX_JOB_RUNTIME_DEFAULT; ++ ++/* local helper functions */ ++static void mali_group_bottom_half_mmu(void *data); ++static void mali_group_bottom_half_gp(void *data); ++static void mali_group_bottom_half_pp(void *data); ++static void mali_group_timeout(void *data); ++static void mali_group_reset_pp(struct mali_group *group); ++static void mali_group_reset_mmu(struct mali_group *group); ++ ++static void mali_group_activate_page_directory(struct mali_group *group, struct mali_session_data *session, mali_bool is_reload); ++static void mali_group_recovery_reset(struct mali_group *group); ++ ++struct mali_group *mali_group_create(struct mali_l2_cache_core *core, ++ struct mali_dlbu_core *dlbu, ++ struct mali_bcast_unit *bcast, ++ u32 domain_index) ++{ ++ struct mali_group *group = NULL; ++ ++ if (mali_global_num_groups >= MALI_MAX_NUMBER_OF_GROUPS) { ++ MALI_PRINT_ERROR(("Mali group: Too many group objects created\n")); ++ return NULL; ++ } ++ ++ group = _mali_osk_calloc(1, sizeof(struct mali_group)); ++ if (NULL != group) { ++ group->timeout_timer = _mali_osk_timer_init(mali_group_timeout); ++ if (NULL != group->timeout_timer) { ++ _mali_osk_timer_setcallback(group->timeout_timer, mali_group_timeout, (void *)group); ++ ++ group->l2_cache_core[0] = core; ++ _mali_osk_list_init(&group->group_list); ++ _mali_osk_list_init(&group->executor_list); ++ _mali_osk_list_init(&group->pm_domain_list); ++ group->bcast_core = bcast; ++ group->dlbu_core = dlbu; ++ ++ /* register this object as a part of the correct power domain */ ++ if ((NULL != core) || (NULL != dlbu) || (NULL != bcast)) ++ group->pm_domain = mali_pm_register_group(domain_index, group); ++ ++ mali_global_groups[mali_global_num_groups] = group; ++ mali_global_num_groups++; ++ ++ return group; ++ } ++ _mali_osk_free(group); ++ } ++ ++ return NULL; ++} ++ ++void mali_group_delete(struct mali_group *group) ++{ ++ u32 i; ++ ++ MALI_DEBUG_PRINT(4, ("Deleting group %s\n", ++ mali_group_core_description(group))); ++ ++ MALI_DEBUG_ASSERT(NULL == group->parent_group); ++ MALI_DEBUG_ASSERT((MALI_GROUP_STATE_INACTIVE == group->state) || ((MALI_GROUP_STATE_ACTIVATION_PENDING == group->state))); ++ ++ /* Delete the resources that this group owns */ ++ if (NULL != group->gp_core) { ++ mali_gp_delete(group->gp_core); ++ } ++ ++ if (NULL != group->pp_core) { ++ mali_pp_delete(group->pp_core); ++ } ++ ++ if (NULL != group->mmu) { ++ mali_mmu_delete(group->mmu); ++ } ++ ++ if (mali_group_is_virtual(group)) { ++ /* Remove all groups from virtual group */ ++ struct mali_group *child; ++ struct mali_group *temp; ++ ++ _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, struct mali_group, group_list) { ++ child->parent_group = NULL; ++ mali_group_delete(child); ++ } ++ ++ mali_dlbu_delete(group->dlbu_core); ++ ++ if (NULL != group->bcast_core) { ++ mali_bcast_unit_delete(group->bcast_core); ++ } ++ } ++ ++ for (i = 0; i < mali_global_num_groups; i++) { ++ if (mali_global_groups[i] == group) { ++ mali_global_groups[i] = NULL; ++ mali_global_num_groups--; ++ ++ if (i != mali_global_num_groups) { ++ /* We removed a group from the middle of the array -- move the last ++ * group to the current position to close the gap */ ++ mali_global_groups[i] = mali_global_groups[mali_global_num_groups]; ++ mali_global_groups[mali_global_num_groups] = NULL; ++ } ++ ++ break; ++ } ++ } ++ ++ if (NULL != group->timeout_timer) { ++ _mali_osk_timer_del(group->timeout_timer); ++ _mali_osk_timer_term(group->timeout_timer); ++ } ++ ++ if (NULL != group->bottom_half_work_mmu) { ++ _mali_osk_wq_delete_work(group->bottom_half_work_mmu); ++ } ++ ++ if (NULL != group->bottom_half_work_gp) { ++ _mali_osk_wq_delete_work(group->bottom_half_work_gp); ++ } ++ ++ if (NULL != group->bottom_half_work_pp) { ++ _mali_osk_wq_delete_work(group->bottom_half_work_pp); ++ } ++ ++ _mali_osk_free(group); ++} ++ ++_mali_osk_errcode_t mali_group_add_mmu_core(struct mali_group *group, struct mali_mmu_core *mmu_core) ++{ ++ /* This group object now owns the MMU core object */ ++ group->mmu = mmu_core; ++ group->bottom_half_work_mmu = _mali_osk_wq_create_work(mali_group_bottom_half_mmu, group); ++ if (NULL == group->bottom_half_work_mmu) { ++ return _MALI_OSK_ERR_FAULT; ++ } ++ return _MALI_OSK_ERR_OK; ++} ++ ++void mali_group_remove_mmu_core(struct mali_group *group) ++{ ++ /* This group object no longer owns the MMU core object */ ++ group->mmu = NULL; ++ if (NULL != group->bottom_half_work_mmu) { ++ _mali_osk_wq_delete_work(group->bottom_half_work_mmu); ++ } ++} ++ ++_mali_osk_errcode_t mali_group_add_gp_core(struct mali_group *group, struct mali_gp_core *gp_core) ++{ ++ /* This group object now owns the GP core object */ ++ group->gp_core = gp_core; ++ group->bottom_half_work_gp = _mali_osk_wq_create_work(mali_group_bottom_half_gp, group); ++ if (NULL == group->bottom_half_work_gp) { ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++void mali_group_remove_gp_core(struct mali_group *group) ++{ ++ /* This group object no longer owns the GP core object */ ++ group->gp_core = NULL; ++ if (NULL != group->bottom_half_work_gp) { ++ _mali_osk_wq_delete_work(group->bottom_half_work_gp); ++ } ++} ++ ++_mali_osk_errcode_t mali_group_add_pp_core(struct mali_group *group, struct mali_pp_core *pp_core) ++{ ++ /* This group object now owns the PP core object */ ++ group->pp_core = pp_core; ++ group->bottom_half_work_pp = _mali_osk_wq_create_work(mali_group_bottom_half_pp, group); ++ if (NULL == group->bottom_half_work_pp) { ++ return _MALI_OSK_ERR_FAULT; ++ } ++ return _MALI_OSK_ERR_OK; ++} ++ ++void mali_group_remove_pp_core(struct mali_group *group) ++{ ++ /* This group object no longer owns the PP core object */ ++ group->pp_core = NULL; ++ if (NULL != group->bottom_half_work_pp) { ++ _mali_osk_wq_delete_work(group->bottom_half_work_pp); ++ } ++} ++ ++enum mali_group_state mali_group_activate(struct mali_group *group) ++{ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ ++ MALI_DEBUG_PRINT(4, ("Group: Activating group %s\n", ++ mali_group_core_description(group))); ++ ++ if (MALI_GROUP_STATE_INACTIVE == group->state) { ++ /* Group is inactive, get PM refs in order to power up */ ++ ++ /* ++ * We'll take a maximum of 2 power domain references pr group, ++ * one for the group itself, and one for it's L2 cache. ++ */ ++ struct mali_pm_domain *domains[MALI_MAX_NUM_DOMAIN_REFS]; ++ struct mali_group *groups[MALI_MAX_NUM_DOMAIN_REFS]; ++ u32 num_domains = 0; ++ mali_bool all_groups_on; ++ ++ /* Deal with child groups first */ ++ if (mali_group_is_virtual(group)) { ++ /* ++ * The virtual group might have 0, 1 or 2 L2s in ++ * its l2_cache_core array, but we ignore these and ++ * let the child groups take the needed L2 cache ref ++ * on behalf of the virtual group. ++ * In other words; The L2 refs are taken in pair with ++ * the physical group which the L2 is attached to. ++ */ ++ struct mali_group *child; ++ struct mali_group *temp; ++ ++ /* ++ * Child group is inactive, get PM ++ * refs in order to power up. ++ */ ++ _MALI_OSK_LIST_FOREACHENTRY(child, temp, ++ &group->group_list, ++ struct mali_group, group_list) { ++ MALI_DEBUG_ASSERT(MALI_GROUP_STATE_INACTIVE ++ == child->state); ++ ++ child->state = MALI_GROUP_STATE_ACTIVATION_PENDING; ++ ++ MALI_DEBUG_ASSERT_POINTER( ++ child->pm_domain); ++ domains[num_domains] = child->pm_domain; ++ groups[num_domains] = child; ++ num_domains++; ++ ++ /* ++ * Take L2 domain ref for child group. ++ */ ++ MALI_DEBUG_ASSERT(MALI_MAX_NUM_DOMAIN_REFS ++ > num_domains); ++ domains[num_domains] = mali_l2_cache_get_pm_domain( ++ child->l2_cache_core[0]); ++ groups[num_domains] = NULL; ++ MALI_DEBUG_ASSERT(NULL == ++ child->l2_cache_core[1]); ++ num_domains++; ++ } ++ } else { ++ /* Take L2 domain ref for physical groups. */ ++ MALI_DEBUG_ASSERT(MALI_MAX_NUM_DOMAIN_REFS > ++ num_domains); ++ ++ domains[num_domains] = mali_l2_cache_get_pm_domain( ++ group->l2_cache_core[0]); ++ groups[num_domains] = NULL; ++ MALI_DEBUG_ASSERT(NULL == group->l2_cache_core[1]); ++ num_domains++; ++ } ++ ++ /* Do the group itself last (it's dependencies first) */ ++ ++ group->state = MALI_GROUP_STATE_ACTIVATION_PENDING; ++ ++ MALI_DEBUG_ASSERT_POINTER(group->pm_domain); ++ domains[num_domains] = group->pm_domain; ++ groups[num_domains] = group; ++ num_domains++; ++ ++ all_groups_on = mali_pm_get_domain_refs(domains, groups, ++ num_domains); ++ ++ /* ++ * Complete activation for group, include ++ * virtual group or physical group. ++ */ ++ if (MALI_TRUE == all_groups_on) { ++ ++ mali_group_set_active(group); ++ } ++ } else if (MALI_GROUP_STATE_ACTIVE == group->state) { ++ /* Already active */ ++ MALI_DEBUG_ASSERT(MALI_TRUE == group->power_is_on); ++ } else { ++ /* ++ * Activation already pending, group->power_is_on could ++ * be both true or false. We need to wait for power up ++ * notification anyway. ++ */ ++ MALI_DEBUG_ASSERT(MALI_GROUP_STATE_ACTIVATION_PENDING ++ == group->state); ++ } ++ ++ MALI_DEBUG_PRINT(4, ("Group: group %s activation result: %s\n", ++ mali_group_core_description(group), ++ MALI_GROUP_STATE_ACTIVE == group->state ? ++ "ACTIVE" : "PENDING")); ++ ++ return group->state; ++} ++ ++mali_bool mali_group_set_active(struct mali_group *group) ++{ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ MALI_DEBUG_ASSERT(MALI_GROUP_STATE_ACTIVATION_PENDING == group->state); ++ MALI_DEBUG_ASSERT(MALI_TRUE == group->power_is_on); ++ ++ MALI_DEBUG_PRINT(4, ("Group: Activation completed for %s\n", ++ mali_group_core_description(group))); ++ ++ if (mali_group_is_virtual(group)) { ++ struct mali_group *child; ++ struct mali_group *temp; ++ ++ _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, ++ struct mali_group, group_list) { ++ if (MALI_TRUE != child->power_is_on) { ++ return MALI_FALSE; ++ } ++ ++ child->state = MALI_GROUP_STATE_ACTIVE; ++ } ++ ++ mali_group_reset(group); ++ } ++ ++ /* Go to ACTIVE state */ ++ group->state = MALI_GROUP_STATE_ACTIVE; ++ ++ return MALI_TRUE; ++} ++ ++mali_bool mali_group_deactivate(struct mali_group *group) ++{ ++ struct mali_pm_domain *domains[MALI_MAX_NUM_DOMAIN_REFS]; ++ u32 num_domains = 0; ++ mali_bool power_down = MALI_FALSE; ++ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ MALI_DEBUG_ASSERT(MALI_GROUP_STATE_INACTIVE != group->state); ++ ++ MALI_DEBUG_PRINT(3, ("Group: Deactivating group %s\n", ++ mali_group_core_description(group))); ++ ++ group->state = MALI_GROUP_STATE_INACTIVE; ++ ++ MALI_DEBUG_ASSERT_POINTER(group->pm_domain); ++ domains[num_domains] = group->pm_domain; ++ num_domains++; ++ ++ if (mali_group_is_virtual(group)) { ++ /* Release refs for all child groups */ ++ struct mali_group *child; ++ struct mali_group *temp; ++ ++ _MALI_OSK_LIST_FOREACHENTRY(child, temp, ++ &group->group_list, ++ struct mali_group, group_list) { ++ child->state = MALI_GROUP_STATE_INACTIVE; ++ ++ MALI_DEBUG_ASSERT_POINTER(child->pm_domain); ++ domains[num_domains] = child->pm_domain; ++ num_domains++; ++ ++ /* Release L2 cache domain for child groups */ ++ MALI_DEBUG_ASSERT(MALI_MAX_NUM_DOMAIN_REFS > ++ num_domains); ++ domains[num_domains] = mali_l2_cache_get_pm_domain( ++ child->l2_cache_core[0]); ++ MALI_DEBUG_ASSERT(NULL == child->l2_cache_core[1]); ++ num_domains++; ++ } ++ ++ /* ++ * Must do mali_group_power_down() steps right here for ++ * virtual group, because virtual group itself is likely to ++ * stay powered on, however child groups are now very likely ++ * to be powered off (and thus lose their state). ++ */ ++ ++ mali_group_clear_session(group); ++ /* ++ * Disable the broadcast unit (clear it's mask). ++ * This is needed in case the GPU isn't actually ++ * powered down at this point and groups are ++ * removed from an inactive virtual group. ++ * If not, then the broadcast unit will intercept ++ * their interrupts! ++ */ ++ mali_bcast_disable(group->bcast_core); ++ } else { ++ /* Release L2 cache domain for physical groups */ ++ MALI_DEBUG_ASSERT(MALI_MAX_NUM_DOMAIN_REFS > ++ num_domains); ++ domains[num_domains] = mali_l2_cache_get_pm_domain( ++ group->l2_cache_core[0]); ++ MALI_DEBUG_ASSERT(NULL == group->l2_cache_core[1]); ++ num_domains++; ++ } ++ ++ power_down = mali_pm_put_domain_refs(domains, num_domains); ++ ++ return power_down; ++} ++ ++void mali_group_power_up(struct mali_group *group) ++{ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ ++ MALI_DEBUG_PRINT(3, ("Group: Power up for %s\n", ++ mali_group_core_description(group))); ++ ++ group->power_is_on = MALI_TRUE; ++ ++ if (MALI_FALSE == mali_group_is_virtual(group) ++ && MALI_FALSE == mali_group_is_in_virtual(group)) { ++ mali_group_reset(group); ++ } ++ ++ /* ++ * When we just acquire only one physical group form virt group, ++ * we should remove the bcast&dlbu mask from virt group and ++ * reset bcast and dlbu core, although part of pp cores in virt ++ * group maybe not be powered on. ++ */ ++ if (MALI_TRUE == mali_group_is_virtual(group)) { ++ mali_bcast_reset(group->bcast_core); ++ mali_dlbu_update_mask(group->dlbu_core); ++ } ++} ++ ++void mali_group_power_down(struct mali_group *group) ++{ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ MALI_DEBUG_ASSERT(MALI_TRUE == group->power_is_on); ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ ++ MALI_DEBUG_PRINT(3, ("Group: Power down for %s\n", ++ mali_group_core_description(group))); ++ ++ group->power_is_on = MALI_FALSE; ++ ++ if (mali_group_is_virtual(group)) { ++ /* ++ * What we do for physical jobs in this function should ++ * already have been done in mali_group_deactivate() ++ * for virtual group. ++ */ ++ MALI_DEBUG_ASSERT(NULL == group->session); ++ } else { ++ mali_group_clear_session(group); ++ } ++} ++ ++MALI_DEBUG_CODE(static void mali_group_print_virtual(struct mali_group *vgroup) ++{ ++ u32 i; ++ struct mali_group *group; ++ struct mali_group *temp; ++ ++ MALI_DEBUG_PRINT(4, ("Virtual group %s (%p)\n", ++ mali_group_core_description(vgroup), ++ vgroup)); ++ MALI_DEBUG_PRINT(4, ("l2_cache_core[0] = %p, ref = %d\n", vgroup->l2_cache_core[0], vgroup->l2_cache_core_ref_count[0])); ++ MALI_DEBUG_PRINT(4, ("l2_cache_core[1] = %p, ref = %d\n", vgroup->l2_cache_core[1], vgroup->l2_cache_core_ref_count[1])); ++ ++ i = 0; ++ _MALI_OSK_LIST_FOREACHENTRY(group, temp, &vgroup->group_list, struct mali_group, group_list) { ++ MALI_DEBUG_PRINT(4, ("[%d] %s (%p), l2_cache_core[0] = %p\n", ++ i, mali_group_core_description(group), ++ group, group->l2_cache_core[0])); ++ i++; ++ } ++}) ++ ++static void mali_group_dump_core_status(struct mali_group *group) ++{ ++ u32 i; ++ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ MALI_DEBUG_ASSERT(NULL != group->gp_core || (NULL != group->pp_core && !mali_group_is_virtual(group))); ++ ++ if (NULL != group->gp_core) { ++ MALI_PRINT(("Dump Group %s\n", group->gp_core->hw_core.description)); ++ ++ for (i = 0; i < 0xA8; i += 0x10) { ++ MALI_PRINT(("0x%04x: 0x%08x 0x%08x 0x%08x 0x%08x\n", i, mali_hw_core_register_read(&group->gp_core->hw_core, i), ++ mali_hw_core_register_read(&group->gp_core->hw_core, i + 4), ++ mali_hw_core_register_read(&group->gp_core->hw_core, i + 8), ++ mali_hw_core_register_read(&group->gp_core->hw_core, i + 12))); ++ } ++ ++ ++ } else { ++ MALI_PRINT(("Dump Group %s\n", group->pp_core->hw_core.description)); ++ ++ for (i = 0; i < 0x5c; i += 0x10) { ++ MALI_PRINT(("0x%04x: 0x%08x 0x%08x 0x%08x 0x%08x\n", i, mali_hw_core_register_read(&group->pp_core->hw_core, i), ++ mali_hw_core_register_read(&group->pp_core->hw_core, i + 4), ++ mali_hw_core_register_read(&group->pp_core->hw_core, i + 8), ++ mali_hw_core_register_read(&group->pp_core->hw_core, i + 12))); ++ } ++ ++ /* Ignore some minor registers */ ++ for (i = 0x1000; i < 0x1068; i += 0x10) { ++ MALI_PRINT(("0x%04x: 0x%08x 0x%08x 0x%08x 0x%08x\n", i, mali_hw_core_register_read(&group->pp_core->hw_core, i), ++ mali_hw_core_register_read(&group->pp_core->hw_core, i + 4), ++ mali_hw_core_register_read(&group->pp_core->hw_core, i + 8), ++ mali_hw_core_register_read(&group->pp_core->hw_core, i + 12))); ++ } ++ } ++ ++ MALI_PRINT(("Dump Group MMU\n")); ++ for (i = 0; i < 0x24; i += 0x10) { ++ MALI_PRINT(("0x%04x: 0x%08x 0x%08x 0x%08x 0x%08x\n", i, mali_hw_core_register_read(&group->mmu->hw_core, i), ++ mali_hw_core_register_read(&group->mmu->hw_core, i + 4), ++ mali_hw_core_register_read(&group->mmu->hw_core, i + 8), ++ mali_hw_core_register_read(&group->mmu->hw_core, i + 12))); ++ } ++} ++ ++ ++/** ++ * @Dump group status ++ */ ++void mali_group_dump_status(struct mali_group *group) ++{ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ ++ if (mali_group_is_virtual(group)) { ++ struct mali_group *group_c; ++ struct mali_group *temp; ++ _MALI_OSK_LIST_FOREACHENTRY(group_c, temp, &group->group_list, struct mali_group, group_list) { ++ mali_group_dump_core_status(group_c); ++ } ++ } else { ++ mali_group_dump_core_status(group); ++ } ++} ++ ++/** ++ * @brief Add child group to virtual group parent ++ */ ++void mali_group_add_group(struct mali_group *parent, struct mali_group *child) ++{ ++ mali_bool found; ++ u32 i; ++ ++ MALI_DEBUG_PRINT(3, ("Adding group %s to virtual group %s\n", ++ mali_group_core_description(child), ++ mali_group_core_description(parent))); ++ ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ MALI_DEBUG_ASSERT(mali_group_is_virtual(parent)); ++ MALI_DEBUG_ASSERT(!mali_group_is_virtual(child)); ++ MALI_DEBUG_ASSERT(NULL == child->parent_group); ++ ++ _mali_osk_list_addtail(&child->group_list, &parent->group_list); ++ ++ child->parent_group = parent; ++ ++ MALI_DEBUG_ASSERT_POINTER(child->l2_cache_core[0]); ++ ++ MALI_DEBUG_PRINT(4, ("parent->l2_cache_core: [0] = %p, [1] = %p\n", parent->l2_cache_core[0], parent->l2_cache_core[1])); ++ MALI_DEBUG_PRINT(4, ("child->l2_cache_core: [0] = %p, [1] = %p\n", child->l2_cache_core[0], child->l2_cache_core[1])); ++ ++ /* Keep track of the L2 cache cores of child groups */ ++ found = MALI_FALSE; ++ for (i = 0; i < 2; i++) { ++ if (parent->l2_cache_core[i] == child->l2_cache_core[0]) { ++ MALI_DEBUG_ASSERT(parent->l2_cache_core_ref_count[i] > 0); ++ parent->l2_cache_core_ref_count[i]++; ++ found = MALI_TRUE; ++ } ++ } ++ ++ if (!found) { ++ /* First time we see this L2 cache, add it to our list */ ++ i = (NULL == parent->l2_cache_core[0]) ? 0 : 1; ++ ++ MALI_DEBUG_PRINT(4, ("First time we see l2_cache %p. Adding to [%d] = %p\n", child->l2_cache_core[0], i, parent->l2_cache_core[i])); ++ ++ MALI_DEBUG_ASSERT(NULL == parent->l2_cache_core[i]); ++ ++ parent->l2_cache_core[i] = child->l2_cache_core[0]; ++ parent->l2_cache_core_ref_count[i]++; ++ } ++ ++ /* Update Broadcast Unit and DLBU */ ++ mali_bcast_add_group(parent->bcast_core, child); ++ mali_dlbu_add_group(parent->dlbu_core, child); ++ ++ if (MALI_TRUE == parent->power_is_on) { ++ mali_bcast_reset(parent->bcast_core); ++ mali_dlbu_update_mask(parent->dlbu_core); ++ } ++ ++ if (MALI_TRUE == child->power_is_on) { ++ if (NULL == parent->session) { ++ if (NULL != child->session) { ++ /* ++ * Parent has no session, so clear ++ * child session as well. ++ */ ++ mali_mmu_activate_empty_page_directory(child->mmu); ++ } ++ } else { ++ if (parent->session == child->session) { ++ /* We already have same session as parent, ++ * so a simple zap should be enough. ++ */ ++ mali_mmu_zap_tlb(child->mmu); ++ } else { ++ /* ++ * Parent has a different session, so we must ++ * switch to that sessions page table ++ */ ++ mali_mmu_activate_page_directory(child->mmu, mali_session_get_page_directory(parent->session)); ++ } ++ ++ /* It is the parent which keeps the session from now on */ ++ child->session = NULL; ++ } ++ } else { ++ /* should have been cleared when child was powered down */ ++ MALI_DEBUG_ASSERT(NULL == child->session); ++ } ++ ++ /* Start job on child when parent is active */ ++ if (NULL != parent->pp_running_job) { ++ struct mali_pp_job *job = parent->pp_running_job; ++ ++ MALI_DEBUG_PRINT(3, ("Group %x joining running job %d on virtual group %x\n", ++ child, mali_pp_job_get_id(job), parent)); ++ ++ /* Only allowed to add active child to an active parent */ ++ MALI_DEBUG_ASSERT(MALI_GROUP_STATE_ACTIVE == parent->state); ++ MALI_DEBUG_ASSERT(MALI_GROUP_STATE_ACTIVE == child->state); ++ ++ mali_pp_job_start(child->pp_core, job, mali_pp_core_get_id(child->pp_core), MALI_TRUE); ++ ++ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE | ++ MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(mali_pp_core_get_id(child->pp_core)) | ++ MALI_PROFILING_EVENT_REASON_SINGLE_HW_FLUSH, ++ mali_pp_job_get_frame_builder_id(job), mali_pp_job_get_flush_id(job), 0, 0, 0); ++ ++ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START | ++ MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(mali_pp_core_get_id(child->pp_core)) | ++ MALI_PROFILING_EVENT_REASON_START_STOP_HW_VIRTUAL, ++ mali_pp_job_get_pid(job), mali_pp_job_get_tid(job), 0, 0, 0); ++#if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS) ++ if (child->pp_core) { ++ trace_gpu_sched_switch( ++ mali_pp_core_description(child->pp_core), ++ sched_clock(), mali_pp_job_get_tid(job), ++ 0, mali_pp_job_get_id(job)); ++ } ++#endif ++ ++#if defined(CONFIG_MALI400_PROFILING) ++ trace_mali_core_active(mali_pp_job_get_pid(job), 1 /* active */, 0 /* PP */, mali_pp_core_get_id(child->pp_core), ++ mali_pp_job_get_frame_builder_id(job), mali_pp_job_get_flush_id(job)); ++#endif ++ } ++ ++ MALI_DEBUG_CODE(mali_group_print_virtual(parent);) ++} ++ ++/** ++ * @brief Remove child group from virtual group parent ++ */ ++void mali_group_remove_group(struct mali_group *parent, struct mali_group *child) ++{ ++ u32 i; ++ ++ MALI_DEBUG_PRINT(3, ("Removing group %s from virtual group %s\n", ++ mali_group_core_description(child), ++ mali_group_core_description(parent))); ++ ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ MALI_DEBUG_ASSERT(mali_group_is_virtual(parent)); ++ MALI_DEBUG_ASSERT(!mali_group_is_virtual(child)); ++ MALI_DEBUG_ASSERT(parent == child->parent_group); ++ ++ /* Update Broadcast Unit and DLBU */ ++ mali_bcast_remove_group(parent->bcast_core, child); ++ mali_dlbu_remove_group(parent->dlbu_core, child); ++ ++ if (MALI_TRUE == parent->power_is_on) { ++ mali_bcast_reset(parent->bcast_core); ++ mali_dlbu_update_mask(parent->dlbu_core); ++ } ++ ++ child->session = parent->session; ++ child->parent_group = NULL; ++ ++ _mali_osk_list_delinit(&child->group_list); ++ if (_mali_osk_list_empty(&parent->group_list)) { ++ parent->session = NULL; ++ } ++ ++ /* Keep track of the L2 cache cores of child groups */ ++ i = (child->l2_cache_core[0] == parent->l2_cache_core[0]) ? 0 : 1; ++ ++ MALI_DEBUG_ASSERT(child->l2_cache_core[0] == parent->l2_cache_core[i]); ++ ++ parent->l2_cache_core_ref_count[i]--; ++ if (parent->l2_cache_core_ref_count[i] == 0) { ++ parent->l2_cache_core[i] = NULL; ++ } ++ ++ MALI_DEBUG_CODE(mali_group_print_virtual(parent)); ++} ++ ++struct mali_group *mali_group_acquire_group(struct mali_group *parent) ++{ ++ struct mali_group *child = NULL; ++ ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ MALI_DEBUG_ASSERT(mali_group_is_virtual(parent)); ++ ++ if (!_mali_osk_list_empty(&parent->group_list)) { ++ child = _MALI_OSK_LIST_ENTRY(parent->group_list.prev, struct mali_group, group_list); ++ mali_group_remove_group(parent, child); ++ } ++ ++ if (NULL != child) { ++ if (MALI_GROUP_STATE_ACTIVE != parent->state ++ && MALI_TRUE == child->power_is_on) { ++ mali_group_reset(child); ++ } ++ } ++ ++ return child; ++} ++ ++void mali_group_reset(struct mali_group *group) ++{ ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ MALI_DEBUG_ASSERT(NULL == group->gp_running_job); ++ MALI_DEBUG_ASSERT(NULL == group->pp_running_job); ++ ++ MALI_DEBUG_PRINT(3, ("Group: reset of %s\n", ++ mali_group_core_description(group))); ++ ++ if (NULL != group->dlbu_core) { ++ mali_dlbu_reset(group->dlbu_core); ++ } ++ ++ if (NULL != group->bcast_core) { ++ mali_bcast_reset(group->bcast_core); ++ } ++ ++ MALI_DEBUG_ASSERT(NULL != group->mmu); ++ mali_group_reset_mmu(group); ++ ++ if (NULL != group->gp_core) { ++ MALI_DEBUG_ASSERT(NULL == group->pp_core); ++ mali_gp_reset(group->gp_core); ++ } else { ++ MALI_DEBUG_ASSERT(NULL != group->pp_core); ++ mali_group_reset_pp(group); ++ } ++} ++ ++void mali_group_start_gp_job(struct mali_group *group, struct mali_gp_job *job, mali_bool gpu_secure_mode_pre_enabled) ++{ ++ struct mali_session_data *session; ++ ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ ++ MALI_DEBUG_PRINT(3, ("Group: Starting GP job 0x%08X on group %s\n", ++ job, ++ mali_group_core_description(group))); ++ ++ session = mali_gp_job_get_session(job); ++ ++ MALI_DEBUG_ASSERT_POINTER(group->l2_cache_core[0]); ++ mali_l2_cache_invalidate_conditional(group->l2_cache_core[0], mali_gp_job_get_cache_order(job)); ++ ++ /* Reset GPU and disable gpu secure mode if needed. */ ++ if (MALI_TRUE == _mali_osk_gpu_secure_mode_is_enabled()) { ++ struct mali_pmu_core *pmu = mali_pmu_get_global_pmu_core(); ++ _mali_osk_gpu_reset_and_secure_mode_disable(); ++ /* Need to disable the pmu interrupt mask register */ ++ if (NULL != pmu) { ++ mali_pmu_reset(pmu); ++ } ++ } ++ ++ /* Reload mmu page table if needed */ ++ if (MALI_TRUE == gpu_secure_mode_pre_enabled) { ++ mali_group_reset(group); ++ mali_group_activate_page_directory(group, session, MALI_TRUE); ++ } else { ++ mali_group_activate_page_directory(group, session, MALI_FALSE); ++ } ++ ++ mali_gp_job_start(group->gp_core, job); ++ ++ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE | ++ MALI_PROFILING_MAKE_EVENT_CHANNEL_GP(0) | ++ MALI_PROFILING_EVENT_REASON_SINGLE_HW_FLUSH, ++ mali_gp_job_get_frame_builder_id(job), mali_gp_job_get_flush_id(job), 0, 0, 0); ++ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START | ++ MALI_PROFILING_MAKE_EVENT_CHANNEL_GP(0), ++ mali_gp_job_get_pid(job), mali_gp_job_get_tid(job), 0, 0, 0); ++ ++#if defined(CONFIG_MALI400_PROFILING) ++ trace_mali_core_active(mali_gp_job_get_pid(job), 1 /* active */, 1 /* GP */, 0 /* core */, ++ mali_gp_job_get_frame_builder_id(job), mali_gp_job_get_flush_id(job)); ++#endif ++ ++#if defined(CONFIG_MALI400_PROFILING) ++ if ((MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src0(group->l2_cache_core[0])) && ++ (MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src1(group->l2_cache_core[0]))) { ++ mali_group_report_l2_cache_counters_per_core(group, 0); ++ } ++#endif /* #if defined(CONFIG_MALI400_PROFILING) */ ++ ++#if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS) ++ if (group->gp_core) { ++ trace_gpu_sched_switch(mali_gp_core_description(group->gp_core), ++ sched_clock(), mali_gp_job_get_tid(job), ++ 0, mali_gp_job_get_id(job)); ++ } ++#endif ++ ++ group->gp_running_job = job; ++ group->is_working = MALI_TRUE; ++ ++ /* Setup SW timer and record start time */ ++ group->start_time = _mali_osk_time_tickcount(); ++ _mali_osk_timer_mod(group->timeout_timer, _mali_osk_time_mstoticks(mali_max_job_runtime)); ++ ++ MALI_DEBUG_PRINT(4, ("Group: Started GP job 0x%08X on group %s at %u\n", ++ job, ++ mali_group_core_description(group), ++ group->start_time)); ++} ++ ++/* Used to set all the registers except frame renderer list address and fragment shader stack address ++ * It means the caller must set these two registers properly before calling this function ++ */ ++void mali_group_start_pp_job(struct mali_group *group, struct mali_pp_job *job, u32 sub_job, mali_bool gpu_secure_mode_pre_enabled) ++{ ++ struct mali_session_data *session; ++ ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ ++ MALI_DEBUG_PRINT(3, ("Group: Starting PP job 0x%08X part %u/%u on group %s\n", ++ job, sub_job + 1, ++ mali_pp_job_get_sub_job_count(job), ++ mali_group_core_description(group))); ++ ++ session = mali_pp_job_get_session(job); ++ ++ if (NULL != group->l2_cache_core[0]) { ++ mali_l2_cache_invalidate_conditional(group->l2_cache_core[0], mali_pp_job_get_cache_order(job)); ++ } ++ ++ if (NULL != group->l2_cache_core[1]) { ++ mali_l2_cache_invalidate_conditional(group->l2_cache_core[1], mali_pp_job_get_cache_order(job)); ++ } ++ ++ /* Reset GPU and change gpu secure mode if needed. */ ++ if (MALI_TRUE == mali_pp_job_is_protected_job(job) && MALI_FALSE == _mali_osk_gpu_secure_mode_is_enabled()) { ++ struct mali_pmu_core *pmu = mali_pmu_get_global_pmu_core(); ++ _mali_osk_gpu_reset_and_secure_mode_enable(); ++ /* Need to disable the pmu interrupt mask register */ ++ if (NULL != pmu) { ++ mali_pmu_reset(pmu); ++ } ++ } else if (MALI_FALSE == mali_pp_job_is_protected_job(job) && MALI_TRUE == _mali_osk_gpu_secure_mode_is_enabled()) { ++ struct mali_pmu_core *pmu = mali_pmu_get_global_pmu_core(); ++ _mali_osk_gpu_reset_and_secure_mode_disable(); ++ /* Need to disable the pmu interrupt mask register */ ++ if (NULL != pmu) { ++ mali_pmu_reset(pmu); ++ } ++ } ++ ++ /* Reload the mmu page table if needed */ ++ if ((MALI_TRUE == mali_pp_job_is_protected_job(job) && MALI_FALSE == gpu_secure_mode_pre_enabled) ++ || (MALI_FALSE == mali_pp_job_is_protected_job(job) && MALI_TRUE == gpu_secure_mode_pre_enabled)) { ++ mali_group_reset(group); ++ mali_group_activate_page_directory(group, session, MALI_TRUE); ++ } else { ++ mali_group_activate_page_directory(group, session, MALI_FALSE); ++ } ++ ++ if (mali_group_is_virtual(group)) { ++ struct mali_group *child; ++ struct mali_group *temp; ++ u32 core_num = 0; ++ ++ MALI_DEBUG_ASSERT(mali_pp_job_is_virtual(job)); ++ ++ /* Configure DLBU for the job */ ++ mali_dlbu_config_job(group->dlbu_core, job); ++ ++ /* Write stack address for each child group */ ++ _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, struct mali_group, group_list) { ++ mali_pp_write_addr_stack(child->pp_core, job); ++ core_num++; ++ } ++ ++ mali_pp_job_start(group->pp_core, job, sub_job, MALI_FALSE); ++ } else { ++ mali_pp_job_start(group->pp_core, job, sub_job, MALI_FALSE); ++ } ++ ++ /* if the group is virtual, loop through physical groups which belong to this group ++ * and call profiling events for its cores as virtual */ ++ if (MALI_TRUE == mali_group_is_virtual(group)) { ++ struct mali_group *child; ++ struct mali_group *temp; ++ ++ _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, struct mali_group, group_list) { ++ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE | ++ MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(mali_pp_core_get_id(child->pp_core)) | ++ MALI_PROFILING_EVENT_REASON_SINGLE_HW_FLUSH, ++ mali_pp_job_get_frame_builder_id(job), mali_pp_job_get_flush_id(job), 0, 0, 0); ++ ++ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START | ++ MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(mali_pp_core_get_id(child->pp_core)) | ++ MALI_PROFILING_EVENT_REASON_START_STOP_HW_VIRTUAL, ++ mali_pp_job_get_pid(job), mali_pp_job_get_tid(job), 0, 0, 0); ++ ++#if defined(CONFIG_MALI400_PROFILING) ++ trace_mali_core_active(mali_pp_job_get_pid(job), 1 /* active */, 0 /* PP */, mali_pp_core_get_id(child->pp_core), ++ mali_pp_job_get_frame_builder_id(job), mali_pp_job_get_flush_id(job)); ++#endif ++ } ++ ++#if defined(CONFIG_MALI400_PROFILING) ++ if (0 != group->l2_cache_core_ref_count[0]) { ++ if ((MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src0(group->l2_cache_core[0])) && ++ (MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src1(group->l2_cache_core[0]))) { ++ mali_group_report_l2_cache_counters_per_core(group, mali_l2_cache_get_id(group->l2_cache_core[0])); ++ } ++ } ++ if (0 != group->l2_cache_core_ref_count[1]) { ++ if ((MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src0(group->l2_cache_core[1])) && ++ (MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src1(group->l2_cache_core[1]))) { ++ mali_group_report_l2_cache_counters_per_core(group, mali_l2_cache_get_id(group->l2_cache_core[1])); ++ } ++ } ++#endif /* #if defined(CONFIG_MALI400_PROFILING) */ ++ ++ } else { /* group is physical - call profiling events for physical cores */ ++ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE | ++ MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(mali_pp_core_get_id(group->pp_core)) | ++ MALI_PROFILING_EVENT_REASON_SINGLE_HW_FLUSH, ++ mali_pp_job_get_frame_builder_id(job), mali_pp_job_get_flush_id(job), 0, 0, 0); ++ ++ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START | ++ MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(mali_pp_core_get_id(group->pp_core)) | ++ MALI_PROFILING_EVENT_REASON_START_STOP_HW_PHYSICAL, ++ mali_pp_job_get_pid(job), mali_pp_job_get_tid(job), 0, 0, 0); ++ ++#if defined(CONFIG_MALI400_PROFILING) ++ trace_mali_core_active(mali_pp_job_get_pid(job), 1 /* active */, 0 /* PP */, mali_pp_core_get_id(group->pp_core), ++ mali_pp_job_get_frame_builder_id(job), mali_pp_job_get_flush_id(job)); ++#endif ++ ++#if defined(CONFIG_MALI400_PROFILING) ++ if ((MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src0(group->l2_cache_core[0])) && ++ (MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src1(group->l2_cache_core[0]))) { ++ mali_group_report_l2_cache_counters_per_core(group, mali_l2_cache_get_id(group->l2_cache_core[0])); ++ } ++#endif /* #if defined(CONFIG_MALI400_PROFILING) */ ++ } ++ ++#if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS) ++ if (group->pp_core) { ++ trace_gpu_sched_switch(mali_pp_core_description(group->pp_core), ++ sched_clock(), mali_pp_job_get_tid(job), ++ 0, mali_pp_job_get_id(job)); ++ } ++#endif ++ ++ group->pp_running_job = job; ++ group->pp_running_sub_job = sub_job; ++ group->is_working = MALI_TRUE; ++ ++ /* Setup SW timer and record start time */ ++ group->start_time = _mali_osk_time_tickcount(); ++ _mali_osk_timer_mod(group->timeout_timer, _mali_osk_time_mstoticks(mali_max_job_runtime)); ++ ++ MALI_DEBUG_PRINT(4, ("Group: Started PP job 0x%08X part %u/%u on group %s at %u\n", ++ job, sub_job + 1, ++ mali_pp_job_get_sub_job_count(job), ++ mali_group_core_description(group), ++ group->start_time)); ++ ++} ++ ++void mali_group_resume_gp_with_new_heap(struct mali_group *group, u32 job_id, u32 start_addr, u32 end_addr) ++{ ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ ++ MALI_DEBUG_ASSERT_POINTER(group->l2_cache_core[0]); ++ mali_l2_cache_invalidate(group->l2_cache_core[0]); ++ ++ mali_mmu_zap_tlb_without_stall(group->mmu); ++ ++ mali_gp_resume_with_new_heap(group->gp_core, start_addr, end_addr); ++ ++ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_RESUME | ++ MALI_PROFILING_MAKE_EVENT_CHANNEL_GP(0), ++ 0, 0, 0, 0, 0); ++ ++#if defined(CONFIG_MALI400_PROFILING) ++ trace_mali_core_active(mali_gp_job_get_pid(group->gp_running_job), 1 /* active */, 1 /* GP */, 0 /* core */, ++ mali_gp_job_get_frame_builder_id(group->gp_running_job), mali_gp_job_get_flush_id(group->gp_running_job)); ++#endif ++} ++ ++static void mali_group_reset_mmu(struct mali_group *group) ++{ ++ struct mali_group *child; ++ struct mali_group *temp; ++ _mali_osk_errcode_t err; ++ ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ ++ if (!mali_group_is_virtual(group)) { ++ /* This is a physical group or an idle virtual group -- simply wait for ++ * the reset to complete. */ ++ err = mali_mmu_reset(group->mmu); ++ MALI_DEBUG_ASSERT(_MALI_OSK_ERR_OK == err); ++ } else { /* virtual group */ ++ /* Loop through all members of this virtual group and wait ++ * until they are done resetting. ++ */ ++ _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, struct mali_group, group_list) { ++ err = mali_mmu_reset(child->mmu); ++ MALI_DEBUG_ASSERT(_MALI_OSK_ERR_OK == err); ++ } ++ } ++} ++ ++static void mali_group_reset_pp(struct mali_group *group) ++{ ++ struct mali_group *child; ++ struct mali_group *temp; ++ ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ ++ mali_pp_reset_async(group->pp_core); ++ ++ if (!mali_group_is_virtual(group) || NULL == group->pp_running_job) { ++ /* This is a physical group or an idle virtual group -- simply wait for ++ * the reset to complete. */ ++ mali_pp_reset_wait(group->pp_core); ++ } else { ++ /* Loop through all members of this virtual group and wait until they ++ * are done resetting. ++ */ ++ _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, struct mali_group, group_list) { ++ mali_pp_reset_wait(child->pp_core); ++ } ++ } ++} ++ ++struct mali_pp_job *mali_group_complete_pp(struct mali_group *group, mali_bool success, u32 *sub_job) ++{ ++ struct mali_pp_job *pp_job_to_return; ++ ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ MALI_DEBUG_ASSERT_POINTER(group); ++ MALI_DEBUG_ASSERT_POINTER(group->pp_core); ++ MALI_DEBUG_ASSERT_POINTER(group->pp_running_job); ++ MALI_DEBUG_ASSERT_POINTER(sub_job); ++ MALI_DEBUG_ASSERT(MALI_TRUE == group->is_working); ++ ++ /* Stop/clear the timeout timer. */ ++ _mali_osk_timer_del_async(group->timeout_timer); ++ ++ if (NULL != group->pp_running_job) { ++ ++ /* Deal with HW counters and profiling */ ++ ++ if (MALI_TRUE == mali_group_is_virtual(group)) { ++ struct mali_group *child; ++ struct mali_group *temp; ++ ++ /* update performance counters from each physical pp core within this virtual group */ ++ _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, struct mali_group, group_list) { ++ mali_pp_update_performance_counters(group->pp_core, child->pp_core, group->pp_running_job, mali_pp_core_get_id(child->pp_core)); ++ } ++ ++#if defined(CONFIG_MALI400_PROFILING) ++ /* send profiling data per physical core */ ++ _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, struct mali_group, group_list) { ++ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | ++ MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(mali_pp_core_get_id(child->pp_core)) | ++ MALI_PROFILING_EVENT_REASON_START_STOP_HW_VIRTUAL, ++ mali_pp_job_get_perf_counter_value0(group->pp_running_job, mali_pp_core_get_id(child->pp_core)), ++ mali_pp_job_get_perf_counter_value1(group->pp_running_job, mali_pp_core_get_id(child->pp_core)), ++ mali_pp_job_get_perf_counter_src0(group->pp_running_job, group->pp_running_sub_job) | (mali_pp_job_get_perf_counter_src1(group->pp_running_job, group->pp_running_sub_job) << 8), ++ 0, 0); ++ ++ trace_mali_core_active(mali_pp_job_get_pid(group->pp_running_job), ++ 0 /* active */, 0 /* PP */, mali_pp_core_get_id(child->pp_core), ++ mali_pp_job_get_frame_builder_id(group->pp_running_job), ++ mali_pp_job_get_flush_id(group->pp_running_job)); ++ } ++ if (0 != group->l2_cache_core_ref_count[0]) { ++ if ((MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src0(group->l2_cache_core[0])) && ++ (MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src1(group->l2_cache_core[0]))) { ++ mali_group_report_l2_cache_counters_per_core(group, mali_l2_cache_get_id(group->l2_cache_core[0])); ++ } ++ } ++ if (0 != group->l2_cache_core_ref_count[1]) { ++ if ((MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src0(group->l2_cache_core[1])) && ++ (MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src1(group->l2_cache_core[1]))) { ++ mali_group_report_l2_cache_counters_per_core(group, mali_l2_cache_get_id(group->l2_cache_core[1])); ++ } ++ } ++ ++#endif ++ } else { ++ /* update performance counters for a physical group's pp core */ ++ mali_pp_update_performance_counters(group->pp_core, group->pp_core, group->pp_running_job, group->pp_running_sub_job); ++ ++#if defined(CONFIG_MALI400_PROFILING) ++ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | ++ MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(mali_pp_core_get_id(group->pp_core)) | ++ MALI_PROFILING_EVENT_REASON_START_STOP_HW_PHYSICAL, ++ mali_pp_job_get_perf_counter_value0(group->pp_running_job, group->pp_running_sub_job), ++ mali_pp_job_get_perf_counter_value1(group->pp_running_job, group->pp_running_sub_job), ++ mali_pp_job_get_perf_counter_src0(group->pp_running_job, group->pp_running_sub_job) | (mali_pp_job_get_perf_counter_src1(group->pp_running_job, group->pp_running_sub_job) << 8), ++ 0, 0); ++ ++ trace_mali_core_active(mali_pp_job_get_pid(group->pp_running_job), ++ 0 /* active */, 0 /* PP */, mali_pp_core_get_id(group->pp_core), ++ mali_pp_job_get_frame_builder_id(group->pp_running_job), ++ mali_pp_job_get_flush_id(group->pp_running_job)); ++ ++ if ((MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src0(group->l2_cache_core[0])) && ++ (MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src1(group->l2_cache_core[0]))) { ++ mali_group_report_l2_cache_counters_per_core(group, mali_l2_cache_get_id(group->l2_cache_core[0])); ++ } ++#endif ++ } ++ ++#if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS) ++ if (group->gp_core) { ++ trace_gpu_sched_switch( ++ mali_gp_core_description(group->gp_core), ++ sched_clock(), 0, 0, 0); ++ } ++#endif ++ ++ } ++ ++ if (success) { ++ /* Only do soft reset for successful jobs, a full recovery ++ * reset will be done for failed jobs. */ ++ mali_pp_reset_async(group->pp_core); ++ } ++ ++ pp_job_to_return = group->pp_running_job; ++ group->pp_running_job = NULL; ++ group->is_working = MALI_FALSE; ++ *sub_job = group->pp_running_sub_job; ++ ++ if (!success) { ++ MALI_DEBUG_PRINT(2, ("Mali group: Executing recovery reset due to job failure\n")); ++ mali_group_recovery_reset(group); ++ } else if (_MALI_OSK_ERR_OK != mali_pp_reset_wait(group->pp_core)) { ++ MALI_PRINT_ERROR(("Mali group: Executing recovery reset due to reset failure\n")); ++ mali_group_recovery_reset(group); ++ } ++ ++ return pp_job_to_return; ++} ++ ++struct mali_gp_job *mali_group_complete_gp(struct mali_group *group, mali_bool success) ++{ ++ struct mali_gp_job *gp_job_to_return; ++ ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ MALI_DEBUG_ASSERT_POINTER(group); ++ MALI_DEBUG_ASSERT_POINTER(group->gp_core); ++ MALI_DEBUG_ASSERT_POINTER(group->gp_running_job); ++ MALI_DEBUG_ASSERT(MALI_TRUE == group->is_working); ++ ++ /* Stop/clear the timeout timer. */ ++ _mali_osk_timer_del_async(group->timeout_timer); ++ ++ if (NULL != group->gp_running_job) { ++ mali_gp_update_performance_counters(group->gp_core, group->gp_running_job); ++ ++#if defined(CONFIG_MALI400_PROFILING) ++ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | MALI_PROFILING_MAKE_EVENT_CHANNEL_GP(0), ++ mali_gp_job_get_perf_counter_value0(group->gp_running_job), ++ mali_gp_job_get_perf_counter_value1(group->gp_running_job), ++ mali_gp_job_get_perf_counter_src0(group->gp_running_job) | (mali_gp_job_get_perf_counter_src1(group->gp_running_job) << 8), ++ 0, 0); ++ ++ if ((MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src0(group->l2_cache_core[0])) && ++ (MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src1(group->l2_cache_core[0]))) ++ mali_group_report_l2_cache_counters_per_core(group, 0); ++#endif ++ ++#if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS) ++ if (group->pp_core) { ++ trace_gpu_sched_switch( ++ mali_pp_core_description(group->pp_core), ++ sched_clock(), 0, 0, 0); ++ } ++#endif ++ ++#if defined(CONFIG_MALI400_PROFILING) ++ trace_mali_core_active(mali_gp_job_get_pid(group->gp_running_job), 0 /* active */, 1 /* GP */, 0 /* core */, ++ mali_gp_job_get_frame_builder_id(group->gp_running_job), mali_gp_job_get_flush_id(group->gp_running_job)); ++#endif ++ ++ mali_gp_job_set_current_heap_addr(group->gp_running_job, ++ mali_gp_read_plbu_alloc_start_addr(group->gp_core)); ++ } ++ ++ if (success) { ++ /* Only do soft reset for successful jobs, a full recovery ++ * reset will be done for failed jobs. */ ++ mali_gp_reset_async(group->gp_core); ++ } ++ ++ gp_job_to_return = group->gp_running_job; ++ group->gp_running_job = NULL; ++ group->is_working = MALI_FALSE; ++ ++ if (!success) { ++ MALI_DEBUG_PRINT(2, ("Mali group: Executing recovery reset due to job failure\n")); ++ mali_group_recovery_reset(group); ++ } else if (_MALI_OSK_ERR_OK != mali_gp_reset_wait(group->gp_core)) { ++ MALI_PRINT_ERROR(("Mali group: Executing recovery reset due to reset failure\n")); ++ mali_group_recovery_reset(group); ++ } ++ ++ return gp_job_to_return; ++} ++ ++struct mali_group *mali_group_get_glob_group(u32 index) ++{ ++ if (mali_global_num_groups > index) { ++ return mali_global_groups[index]; ++ } ++ ++ return NULL; ++} ++ ++u32 mali_group_get_glob_num_groups(void) ++{ ++ return mali_global_num_groups; ++} ++ ++static void mali_group_activate_page_directory(struct mali_group *group, struct mali_session_data *session, mali_bool is_reload) ++{ ++ MALI_DEBUG_PRINT(5, ("Mali group: Activating page directory 0x%08X from session 0x%08X on group %s\n", ++ mali_session_get_page_directory(session), session, ++ mali_group_core_description(group))); ++ ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ ++ if (group->session != session || MALI_TRUE == is_reload) { ++ /* Different session than last time, so we need to do some work */ ++ MALI_DEBUG_PRINT(5, ("Mali group: Activate session: %08x previous: %08x on group %s\n", ++ session, group->session, ++ mali_group_core_description(group))); ++ mali_mmu_activate_page_directory(group->mmu, mali_session_get_page_directory(session)); ++ group->session = session; ++ } else { ++ /* Same session as last time, so no work required */ ++ MALI_DEBUG_PRINT(4, ("Mali group: Activate existing session 0x%08X on group %s\n", ++ session->page_directory, ++ mali_group_core_description(group))); ++ mali_mmu_zap_tlb_without_stall(group->mmu); ++ } ++} ++ ++static void mali_group_recovery_reset(struct mali_group *group) ++{ ++ _mali_osk_errcode_t err; ++ ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ ++ /* Stop cores, bus stop */ ++ if (NULL != group->pp_core) { ++ mali_pp_stop_bus(group->pp_core); ++ } else { ++ mali_gp_stop_bus(group->gp_core); ++ } ++ ++ /* Flush MMU and clear page fault (if any) */ ++ mali_mmu_activate_fault_flush_page_directory(group->mmu); ++ mali_mmu_page_fault_done(group->mmu); ++ ++ /* Wait for cores to stop bus, then do a hard reset on them */ ++ if (NULL != group->pp_core) { ++ if (mali_group_is_virtual(group)) { ++ struct mali_group *child, *temp; ++ ++ /* Disable the broadcast unit while we do reset directly on the member cores. */ ++ mali_bcast_disable(group->bcast_core); ++ ++ _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, struct mali_group, group_list) { ++ mali_pp_stop_bus_wait(child->pp_core); ++ mali_pp_hard_reset(child->pp_core); ++ } ++ ++ mali_bcast_enable(group->bcast_core); ++ } else { ++ mali_pp_stop_bus_wait(group->pp_core); ++ mali_pp_hard_reset(group->pp_core); ++ } ++ } else { ++ mali_gp_stop_bus_wait(group->gp_core); ++ mali_gp_hard_reset(group->gp_core); ++ } ++ ++ /* Reset MMU */ ++ err = mali_mmu_reset(group->mmu); ++ MALI_DEBUG_ASSERT(_MALI_OSK_ERR_OK == err); ++ MALI_IGNORE(err); ++ ++ group->session = NULL; ++} ++ ++#if MALI_STATE_TRACKING ++u32 mali_group_dump_state(struct mali_group *group, char *buf, u32 size) ++{ ++ int n = 0; ++ int i; ++ struct mali_group *child; ++ struct mali_group *temp; ++ ++ if (mali_group_is_virtual(group)) { ++ n += _mali_osk_snprintf(buf + n, size - n, ++ "Virtual PP Group: %p\n", group); ++ } else if (mali_group_is_in_virtual(group)) { ++ n += _mali_osk_snprintf(buf + n, size - n, ++ "Child PP Group: %p\n", group); ++ } else if (NULL != group->pp_core) { ++ n += _mali_osk_snprintf(buf + n, size - n, ++ "Physical PP Group: %p\n", group); ++ } else { ++ MALI_DEBUG_ASSERT_POINTER(group->gp_core); ++ n += _mali_osk_snprintf(buf + n, size - n, ++ "GP Group: %p\n", group); ++ } ++ ++ switch (group->state) { ++ case MALI_GROUP_STATE_INACTIVE: ++ n += _mali_osk_snprintf(buf + n, size - n, ++ "\tstate: INACTIVE\n"); ++ break; ++ case MALI_GROUP_STATE_ACTIVATION_PENDING: ++ n += _mali_osk_snprintf(buf + n, size - n, ++ "\tstate: ACTIVATION_PENDING\n"); ++ break; ++ case MALI_GROUP_STATE_ACTIVE: ++ n += _mali_osk_snprintf(buf + n, size - n, ++ "\tstate: MALI_GROUP_STATE_ACTIVE\n"); ++ break; ++ default: ++ n += _mali_osk_snprintf(buf + n, size - n, ++ "\tstate: UNKNOWN (%d)\n", group->state); ++ MALI_DEBUG_ASSERT(0); ++ break; ++ } ++ ++ n += _mali_osk_snprintf(buf + n, size - n, ++ "\tSW power: %s\n", ++ group->power_is_on ? "On" : "Off"); ++ ++ n += mali_pm_dump_state_domain(group->pm_domain, buf + n, size - n); ++ ++ for (i = 0; i < 2; i++) { ++ if (NULL != group->l2_cache_core[i]) { ++ struct mali_pm_domain *domain; ++ domain = mali_l2_cache_get_pm_domain( ++ group->l2_cache_core[i]); ++ n += mali_pm_dump_state_domain(domain, ++ buf + n, size - n); ++ } ++ } ++ ++ if (group->gp_core) { ++ n += mali_gp_dump_state(group->gp_core, buf + n, size - n); ++ n += _mali_osk_snprintf(buf + n, size - n, ++ "\tGP running job: %p\n", group->gp_running_job); ++ } ++ ++ if (group->pp_core) { ++ n += mali_pp_dump_state(group->pp_core, buf + n, size - n); ++ n += _mali_osk_snprintf(buf + n, size - n, ++ "\tPP running job: %p, subjob %d \n", ++ group->pp_running_job, ++ group->pp_running_sub_job); ++ } ++ ++ _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, ++ struct mali_group, group_list) { ++ n += mali_group_dump_state(child, buf + n, size - n); ++ } ++ ++ return n; ++} ++#endif ++ ++_mali_osk_errcode_t mali_group_upper_half_mmu(void *data) ++{ ++ struct mali_group *group = (struct mali_group *)data; ++ _mali_osk_errcode_t ret; ++ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ MALI_DEBUG_ASSERT_POINTER(group->mmu); ++ ++#if defined(CONFIG_MALI400_PROFILING) && defined (CONFIG_TRACEPOINTS) ++#if defined(CONFIG_MALI_SHARED_INTERRUPTS) ++ mali_executor_lock(); ++ if (!mali_group_is_working(group)) { ++ /* Not working, so nothing to do */ ++ mali_executor_unlock(); ++ return _MALI_OSK_ERR_FAULT; ++ } ++#endif ++ if (NULL != group->gp_core) { ++ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START | ++ MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | ++ MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF, ++ 0, 0, /* No pid and tid for interrupt handler */ ++ MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP_MMU(0), ++ mali_mmu_get_rawstat(group->mmu), 0); ++ } else { ++ MALI_DEBUG_ASSERT_POINTER(group->pp_core); ++ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START | ++ MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | ++ MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF, ++ 0, 0, /* No pid and tid for interrupt handler */ ++ MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP_MMU( ++ mali_pp_core_get_id(group->pp_core)), ++ mali_mmu_get_rawstat(group->mmu), 0); ++ } ++#if defined(CONFIG_MALI_SHARED_INTERRUPTS) ++ mali_executor_unlock(); ++#endif ++#endif ++ ++ ret = mali_executor_interrupt_mmu(group, MALI_TRUE); ++ ++#if defined(CONFIG_MALI400_PROFILING) && defined (CONFIG_TRACEPOINTS) ++#if defined(CONFIG_MALI_SHARED_INTERRUPTS) ++ mali_executor_lock(); ++ if (!mali_group_is_working(group) && (!mali_group_power_is_on(group))) { ++ /* group complete and on job shedule on it, it already power off */ ++ if (NULL != group->gp_core) { ++ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | ++ MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | ++ MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF, ++ 0, 0, /* No pid and tid for interrupt handler */ ++ MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP_MMU(0), ++ 0xFFFFFFFF, 0); ++ } else { ++ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | ++ MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | ++ MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF, ++ 0, 0, /* No pid and tid for interrupt handler */ ++ MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP_MMU( ++ mali_pp_core_get_id(group->pp_core)), ++ 0xFFFFFFFF, 0); ++ } ++ ++ mali_executor_unlock(); ++ return ret; ++ } ++#endif ++ ++ if (NULL != group->gp_core) { ++ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | ++ MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | ++ MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF, ++ 0, 0, /* No pid and tid for interrupt handler */ ++ MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP_MMU(0), ++ mali_mmu_get_rawstat(group->mmu), 0); ++ } else { ++ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | ++ MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | ++ MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF, ++ 0, 0, /* No pid and tid for interrupt handler */ ++ MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP_MMU( ++ mali_pp_core_get_id(group->pp_core)), ++ mali_mmu_get_rawstat(group->mmu), 0); ++ } ++#if defined(CONFIG_MALI_SHARED_INTERRUPTS) ++ mali_executor_unlock(); ++#endif ++#endif ++ ++ return ret; ++} ++ ++static void mali_group_bottom_half_mmu(void *data) ++{ ++ struct mali_group *group = (struct mali_group *)data; ++ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ MALI_DEBUG_ASSERT_POINTER(group->mmu); ++ ++ if (NULL != group->gp_core) { ++ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START | ++ MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | ++ MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF, ++ 0, _mali_osk_get_tid(), /* pid and tid */ ++ MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP_MMU(0), ++ mali_mmu_get_rawstat(group->mmu), 0); ++ } else { ++ MALI_DEBUG_ASSERT_POINTER(group->pp_core); ++ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START | ++ MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | ++ MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF, ++ 0, _mali_osk_get_tid(), /* pid and tid */ ++ MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP_MMU( ++ mali_pp_core_get_id(group->pp_core)), ++ mali_mmu_get_rawstat(group->mmu), 0); ++ } ++ ++ mali_executor_interrupt_mmu(group, MALI_FALSE); ++ ++ if (NULL != group->gp_core) { ++ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | ++ MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | ++ MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF, ++ 0, _mali_osk_get_tid(), /* pid and tid */ ++ MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP_MMU(0), ++ mali_mmu_get_rawstat(group->mmu), 0); ++ } else { ++ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | ++ MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | ++ MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF, ++ 0, _mali_osk_get_tid(), /* pid and tid */ ++ MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP_MMU( ++ mali_pp_core_get_id(group->pp_core)), ++ mali_mmu_get_rawstat(group->mmu), 0); ++ } ++} ++ ++_mali_osk_errcode_t mali_group_upper_half_gp(void *data) ++{ ++ struct mali_group *group = (struct mali_group *)data; ++ _mali_osk_errcode_t ret; ++ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ MALI_DEBUG_ASSERT_POINTER(group->gp_core); ++ MALI_DEBUG_ASSERT_POINTER(group->mmu); ++ ++#if defined(CONFIG_MALI400_PROFILING) && defined (CONFIG_TRACEPOINTS) ++#if defined(CONFIG_MALI_SHARED_INTERRUPTS) ++ mali_executor_lock(); ++ if (!mali_group_is_working(group)) { ++ /* Not working, so nothing to do */ ++ mali_executor_unlock(); ++ return _MALI_OSK_ERR_FAULT; ++ } ++#endif ++ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START | ++ MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | ++ MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF, ++ 0, 0, /* No pid and tid for interrupt handler */ ++ MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP(0), ++ mali_gp_get_rawstat(group->gp_core), 0); ++ ++ MALI_DEBUG_PRINT(4, ("Group: Interrupt 0x%08X from %s\n", ++ mali_gp_get_rawstat(group->gp_core), ++ mali_group_core_description(group))); ++#if defined(CONFIG_MALI_SHARED_INTERRUPTS) ++ mali_executor_unlock(); ++#endif ++#endif ++ ret = mali_executor_interrupt_gp(group, MALI_TRUE); ++ ++#if defined(CONFIG_MALI400_PROFILING) && defined (CONFIG_TRACEPOINTS) ++#if defined(CONFIG_MALI_SHARED_INTERRUPTS) ++ mali_executor_lock(); ++ if (!mali_group_is_working(group) && (!mali_group_power_is_on(group))) { ++ /* group complete and on job shedule on it, it already power off */ ++ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | ++ MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | ++ MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF, ++ 0, 0, /* No pid and tid for interrupt handler */ ++ MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP(0), ++ 0xFFFFFFFF, 0); ++ mali_executor_unlock(); ++ return ret; ++ } ++#endif ++ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | ++ MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | ++ MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF, ++ 0, 0, /* No pid and tid for interrupt handler */ ++ MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP(0), ++ mali_gp_get_rawstat(group->gp_core), 0); ++#if defined(CONFIG_MALI_SHARED_INTERRUPTS) ++ mali_executor_unlock(); ++#endif ++#endif ++ return ret; ++} ++ ++static void mali_group_bottom_half_gp(void *data) ++{ ++ struct mali_group *group = (struct mali_group *)data; ++ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ MALI_DEBUG_ASSERT_POINTER(group->gp_core); ++ MALI_DEBUG_ASSERT_POINTER(group->mmu); ++ ++ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START | ++ MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | ++ MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF, ++ 0, _mali_osk_get_tid(), /* pid and tid */ ++ MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP(0), ++ mali_gp_get_rawstat(group->gp_core), 0); ++ ++ mali_executor_interrupt_gp(group, MALI_FALSE); ++ ++ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | ++ MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | ++ MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF, ++ 0, _mali_osk_get_tid(), /* pid and tid */ ++ MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP(0), ++ mali_gp_get_rawstat(group->gp_core), 0); ++} ++ ++_mali_osk_errcode_t mali_group_upper_half_pp(void *data) ++{ ++ struct mali_group *group = (struct mali_group *)data; ++ _mali_osk_errcode_t ret; ++ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ MALI_DEBUG_ASSERT_POINTER(group->pp_core); ++ MALI_DEBUG_ASSERT_POINTER(group->mmu); ++ ++#if defined(CONFIG_MALI400_PROFILING) && defined (CONFIG_TRACEPOINTS) ++#if defined(CONFIG_MALI_SHARED_INTERRUPTS) ++ mali_executor_lock(); ++ if (!mali_group_is_working(group)) { ++ /* Not working, so nothing to do */ ++ mali_executor_unlock(); ++ return _MALI_OSK_ERR_FAULT; ++ } ++#endif ++ ++ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START | ++ MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | ++ MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF, ++ 0, 0, /* No pid and tid for interrupt handler */ ++ MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP( ++ mali_pp_core_get_id(group->pp_core)), ++ mali_pp_get_rawstat(group->pp_core), 0); ++ ++ MALI_DEBUG_PRINT(4, ("Group: Interrupt 0x%08X from %s\n", ++ mali_pp_get_rawstat(group->pp_core), ++ mali_group_core_description(group))); ++#if defined(CONFIG_MALI_SHARED_INTERRUPTS) ++ mali_executor_unlock(); ++#endif ++#endif ++ ++ ret = mali_executor_interrupt_pp(group, MALI_TRUE); ++ ++#if defined(CONFIG_MALI400_PROFILING) && defined (CONFIG_TRACEPOINTS) ++#if defined(CONFIG_MALI_SHARED_INTERRUPTS) ++ mali_executor_lock(); ++ if (!mali_group_is_working(group) && (!mali_group_power_is_on(group))) { ++ /* group complete and on job shedule on it, it already power off */ ++ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | ++ MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | ++ MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF, ++ 0, 0, /* No pid and tid for interrupt handler */ ++ MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP( ++ mali_pp_core_get_id(group->pp_core)), ++ 0xFFFFFFFF, 0); ++ mali_executor_unlock(); ++ return ret; ++ } ++#endif ++ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | ++ MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | ++ MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF, ++ 0, 0, /* No pid and tid for interrupt handler */ ++ MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP( ++ mali_pp_core_get_id(group->pp_core)), ++ mali_pp_get_rawstat(group->pp_core), 0); ++#if defined(CONFIG_MALI_SHARED_INTERRUPTS) ++ mali_executor_unlock(); ++#endif ++#endif ++ return ret; ++} ++ ++static void mali_group_bottom_half_pp(void *data) ++{ ++ struct mali_group *group = (struct mali_group *)data; ++ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ MALI_DEBUG_ASSERT_POINTER(group->pp_core); ++ MALI_DEBUG_ASSERT_POINTER(group->mmu); ++ ++ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START | ++ MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | ++ MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF, ++ 0, _mali_osk_get_tid(), /* pid and tid */ ++ MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP( ++ mali_pp_core_get_id(group->pp_core)), ++ mali_pp_get_rawstat(group->pp_core), 0); ++ ++ mali_executor_interrupt_pp(group, MALI_FALSE); ++ ++ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | ++ MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | ++ MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF, ++ 0, _mali_osk_get_tid(), /* pid and tid */ ++ MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP( ++ mali_pp_core_get_id(group->pp_core)), ++ mali_pp_get_rawstat(group->pp_core), 0); ++} ++ ++static void mali_group_timeout(void *data) ++{ ++ struct mali_group *group = (struct mali_group *)data; ++ MALI_DEBUG_ASSERT_POINTER(group); ++ ++ MALI_DEBUG_PRINT(2, ("Group: timeout handler for %s at %u\n", ++ mali_group_core_description(group), ++ _mali_osk_time_tickcount())); ++ ++ if (NULL != group->gp_core) { ++ mali_group_schedule_bottom_half_gp(group); ++ } else { ++ MALI_DEBUG_ASSERT_POINTER(group->pp_core); ++ mali_group_schedule_bottom_half_pp(group); ++ } ++} ++ ++mali_bool mali_group_zap_session(struct mali_group *group, ++ struct mali_session_data *session) ++{ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ MALI_DEBUG_ASSERT_POINTER(session); ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ ++ if (group->session != session) { ++ /* not running from this session */ ++ return MALI_TRUE; /* success */ ++ } ++ ++ if (group->is_working) { ++ /* The Zap also does the stall and disable_stall */ ++ mali_bool zap_success = mali_mmu_zap_tlb(group->mmu); ++ return zap_success; ++ } else { ++ /* Just remove the session instead of zapping */ ++ mali_group_clear_session(group); ++ return MALI_TRUE; /* success */ ++ } ++} ++ ++#if defined(CONFIG_MALI400_PROFILING) ++static void mali_group_report_l2_cache_counters_per_core(struct mali_group *group, u32 core_num) ++{ ++ u32 source0 = 0; ++ u32 value0 = 0; ++ u32 source1 = 0; ++ u32 value1 = 0; ++ u32 profiling_channel = 0; ++ ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ ++ switch (core_num) { ++ case 0: ++ profiling_channel = MALI_PROFILING_EVENT_TYPE_SINGLE | ++ MALI_PROFILING_EVENT_CHANNEL_GPU | ++ MALI_PROFILING_EVENT_REASON_SINGLE_GPU_L20_COUNTERS; ++ break; ++ case 1: ++ profiling_channel = MALI_PROFILING_EVENT_TYPE_SINGLE | ++ MALI_PROFILING_EVENT_CHANNEL_GPU | ++ MALI_PROFILING_EVENT_REASON_SINGLE_GPU_L21_COUNTERS; ++ break; ++ case 2: ++ profiling_channel = MALI_PROFILING_EVENT_TYPE_SINGLE | ++ MALI_PROFILING_EVENT_CHANNEL_GPU | ++ MALI_PROFILING_EVENT_REASON_SINGLE_GPU_L22_COUNTERS; ++ break; ++ default: ++ profiling_channel = MALI_PROFILING_EVENT_TYPE_SINGLE | ++ MALI_PROFILING_EVENT_CHANNEL_GPU | ++ MALI_PROFILING_EVENT_REASON_SINGLE_GPU_L20_COUNTERS; ++ break; ++ } ++ ++ if (0 == core_num) { ++ mali_l2_cache_core_get_counter_values(group->l2_cache_core[0], &source0, &value0, &source1, &value1); ++ } ++ if (1 == core_num) { ++ if (1 == mali_l2_cache_get_id(group->l2_cache_core[0])) { ++ mali_l2_cache_core_get_counter_values(group->l2_cache_core[0], &source0, &value0, &source1, &value1); ++ } else if (1 == mali_l2_cache_get_id(group->l2_cache_core[1])) { ++ mali_l2_cache_core_get_counter_values(group->l2_cache_core[1], &source0, &value0, &source1, &value1); ++ } ++ } ++ if (2 == core_num) { ++ if (2 == mali_l2_cache_get_id(group->l2_cache_core[0])) { ++ mali_l2_cache_core_get_counter_values(group->l2_cache_core[0], &source0, &value0, &source1, &value1); ++ } else if (2 == mali_l2_cache_get_id(group->l2_cache_core[1])) { ++ mali_l2_cache_core_get_counter_values(group->l2_cache_core[1], &source0, &value0, &source1, &value1); ++ } ++ } ++ ++ _mali_osk_profiling_add_event(profiling_channel, source1 << 8 | source0, value0, value1, 0, 0); ++} ++#endif /* #if defined(CONFIG_MALI400_PROFILING) */ +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_group.h b/drivers/gpu/arm/mali400/mali/common/mali_group.h +new file mode 100755 +index 000000000000..32481e4a6748 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_group.h +@@ -0,0 +1,460 @@ ++/* ++ * Copyright (C) 2011-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_GROUP_H__ ++#define __MALI_GROUP_H__ ++ ++#include "mali_osk.h" ++#include "mali_l2_cache.h" ++#include "mali_mmu.h" ++#include "mali_gp.h" ++#include "mali_pp.h" ++#include "mali_session.h" ++#include "mali_osk_profiling.h" ++ ++/** ++ * @brief Default max runtime [ms] for a core job - used by timeout timers ++ */ ++#define MALI_MAX_JOB_RUNTIME_DEFAULT 5000 ++ ++extern int mali_max_job_runtime; ++ ++#define MALI_MAX_NUMBER_OF_GROUPS 10 ++#define MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS 8 ++ ++enum mali_group_state { ++ MALI_GROUP_STATE_INACTIVE, ++ MALI_GROUP_STATE_ACTIVATION_PENDING, ++ MALI_GROUP_STATE_ACTIVE, ++}; ++ ++/** ++ * The structure represents a render group ++ * A render group is defined by all the cores that share the same Mali MMU ++ */ ++ ++struct mali_group { ++ struct mali_mmu_core *mmu; ++ struct mali_session_data *session; ++ ++ enum mali_group_state state; ++ mali_bool power_is_on; ++ ++ mali_bool is_working; ++ unsigned long start_time; /* in ticks */ ++ ++ struct mali_gp_core *gp_core; ++ struct mali_gp_job *gp_running_job; ++ ++ struct mali_pp_core *pp_core; ++ struct mali_pp_job *pp_running_job; ++ u32 pp_running_sub_job; ++ ++ struct mali_pm_domain *pm_domain; ++ ++ struct mali_l2_cache_core *l2_cache_core[2]; ++ u32 l2_cache_core_ref_count[2]; ++ ++ /* Parent virtual group (if any) */ ++ struct mali_group *parent_group; ++ ++ struct mali_dlbu_core *dlbu_core; ++ struct mali_bcast_unit *bcast_core; ++ ++ /* Used for working groups which needs to be disabled */ ++ mali_bool disable_requested; ++ ++ /* Used by group to link child groups (for virtual group) */ ++ _mali_osk_list_t group_list; ++ ++ /* Used by executor module in order to link groups of same state */ ++ _mali_osk_list_t executor_list; ++ ++ /* Used by PM domains to link groups of same domain */ ++ _mali_osk_list_t pm_domain_list; ++ ++ _mali_osk_wq_work_t *bottom_half_work_mmu; ++ _mali_osk_wq_work_t *bottom_half_work_gp; ++ _mali_osk_wq_work_t *bottom_half_work_pp; ++ ++ _mali_osk_timer_t *timeout_timer; ++}; ++ ++/** @brief Create a new Mali group object ++ * ++ * @return A pointer to a new group object ++ */ ++struct mali_group *mali_group_create(struct mali_l2_cache_core *core, ++ struct mali_dlbu_core *dlbu, ++ struct mali_bcast_unit *bcast, ++ u32 domain_index); ++ ++void mali_group_dump_status(struct mali_group *group); ++ ++void mali_group_delete(struct mali_group *group); ++ ++_mali_osk_errcode_t mali_group_add_mmu_core(struct mali_group *group, ++ struct mali_mmu_core *mmu_core); ++void mali_group_remove_mmu_core(struct mali_group *group); ++ ++_mali_osk_errcode_t mali_group_add_gp_core(struct mali_group *group, ++ struct mali_gp_core *gp_core); ++void mali_group_remove_gp_core(struct mali_group *group); ++ ++_mali_osk_errcode_t mali_group_add_pp_core(struct mali_group *group, ++ struct mali_pp_core *pp_core); ++void mali_group_remove_pp_core(struct mali_group *group); ++ ++MALI_STATIC_INLINE const char *mali_group_core_description( ++ struct mali_group *group) ++{ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ if (NULL != group->pp_core) { ++ return mali_pp_core_description(group->pp_core); ++ } else { ++ MALI_DEBUG_ASSERT_POINTER(group->gp_core); ++ return mali_gp_core_description(group->gp_core); ++ } ++} ++ ++MALI_STATIC_INLINE mali_bool mali_group_is_virtual(struct mali_group *group) ++{ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ ++#if (defined(CONFIG_MALI450) || defined(CONFIG_MALI470)) ++ return (NULL != group->dlbu_core); ++#else ++ return MALI_FALSE; ++#endif ++} ++ ++/** @brief Check if a group is a part of a virtual group or not ++ */ ++MALI_STATIC_INLINE mali_bool mali_group_is_in_virtual(struct mali_group *group) ++{ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ ++#if (defined(CONFIG_MALI450) || defined(CONFIG_MALI470)) ++ return (NULL != group->parent_group) ? MALI_TRUE : MALI_FALSE; ++#else ++ return MALI_FALSE; ++#endif ++} ++ ++/** @brief Reset group ++ * ++ * This function will reset the entire group, ++ * including all the cores present in the group. ++ * ++ * @param group Pointer to the group to reset ++ */ ++void mali_group_reset(struct mali_group *group); ++ ++MALI_STATIC_INLINE struct mali_session_data *mali_group_get_session( ++ struct mali_group *group) ++{ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ ++ return group->session; ++} ++ ++MALI_STATIC_INLINE void mali_group_clear_session(struct mali_group *group) ++{ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ ++ if (NULL != group->session) { ++ mali_mmu_activate_empty_page_directory(group->mmu); ++ group->session = NULL; ++ } ++} ++ ++enum mali_group_state mali_group_activate(struct mali_group *group); ++ ++/* ++ * Change state from ACTIVATION_PENDING to ACTIVE ++ * For virtual group, all childs need to be ACTIVE first ++ */ ++mali_bool mali_group_set_active(struct mali_group *group); ++ ++/* ++ * @return MALI_TRUE means one or more domains can now be powered off, ++ * and caller should call either mali_pm_update_async() or ++ * mali_pm_update_sync() in order to do so. ++ */ ++mali_bool mali_group_deactivate(struct mali_group *group); ++ ++MALI_STATIC_INLINE enum mali_group_state mali_group_get_state(struct mali_group *group) ++{ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ return group->state; ++} ++ ++MALI_STATIC_INLINE mali_bool mali_group_power_is_on(struct mali_group *group) ++{ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ return group->power_is_on; ++} ++ ++void mali_group_power_up(struct mali_group *group); ++void mali_group_power_down(struct mali_group *group); ++ ++MALI_STATIC_INLINE void mali_group_set_disable_request( ++ struct mali_group *group, mali_bool disable) ++{ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ group->disable_requested = disable; ++ ++ /** ++ * When one of child group's disable_requeset is set TRUE, then ++ * the disable_request of parent group should also be set to TRUE. ++ * While, the disable_request of parent group should only be set to FALSE ++ * only when all of its child group's disable_request are set to FALSE. ++ */ ++ if (NULL != group->parent_group && MALI_TRUE == disable) { ++ group->parent_group->disable_requested = disable; ++ } ++} ++ ++MALI_STATIC_INLINE mali_bool mali_group_disable_requested( ++ struct mali_group *group) ++{ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ return group->disable_requested; ++} ++ ++/** @brief Virtual groups */ ++void mali_group_add_group(struct mali_group *parent, struct mali_group *child); ++struct mali_group *mali_group_acquire_group(struct mali_group *parent); ++void mali_group_remove_group(struct mali_group *parent, struct mali_group *child); ++ ++/** @brief Checks if the group is working. ++ */ ++MALI_STATIC_INLINE mali_bool mali_group_is_working(struct mali_group *group) ++{ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ if (mali_group_is_in_virtual(group)) { ++ struct mali_group *tmp_group = mali_executor_get_virtual_group(); ++ return tmp_group->is_working; ++ } ++ return group->is_working; ++} ++ ++MALI_STATIC_INLINE struct mali_gp_job *mali_group_get_running_gp_job(struct mali_group *group) ++{ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ return group->gp_running_job; ++} ++ ++/** @brief Zap MMU TLB on all groups ++ * ++ * Zap TLB on group if \a session is active. ++ */ ++mali_bool mali_group_zap_session(struct mali_group *group, ++ struct mali_session_data *session); ++ ++/** @brief Get pointer to GP core object ++ */ ++MALI_STATIC_INLINE struct mali_gp_core *mali_group_get_gp_core(struct mali_group *group) ++{ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ return group->gp_core; ++} ++ ++/** @brief Get pointer to PP core object ++ */ ++MALI_STATIC_INLINE struct mali_pp_core *mali_group_get_pp_core(struct mali_group *group) ++{ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ return group->pp_core; ++} ++ ++/** @brief Start GP job ++ */ ++void mali_group_start_gp_job(struct mali_group *group, struct mali_gp_job *job, mali_bool gpu_secure_mode_pre_enabled); ++ ++void mali_group_start_pp_job(struct mali_group *group, struct mali_pp_job *job, u32 sub_job, mali_bool gpu_secure_mode_pre_enabled); ++ ++/** @brief Start virtual group Job on a virtual group ++*/ ++void mali_group_start_job_on_virtual(struct mali_group *group, struct mali_pp_job *job, u32 first_subjob, u32 last_subjob); ++ ++ ++/** @brief Start a subjob from a particular on a specific PP group ++*/ ++void mali_group_start_job_on_group(struct mali_group *group, struct mali_pp_job *job, u32 subjob); ++ ++ ++/** @brief remove all the unused groups in tmp_unused group list, so that the group is in consistent status. ++ */ ++void mali_group_non_dlbu_job_done_virtual(struct mali_group *group); ++ ++ ++/** @brief Resume GP job that suspended waiting for more heap memory ++ */ ++void mali_group_resume_gp_with_new_heap(struct mali_group *group, u32 job_id, u32 start_addr, u32 end_addr); ++ ++MALI_STATIC_INLINE enum mali_interrupt_result mali_group_get_interrupt_result_gp(struct mali_group *group) ++{ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ MALI_DEBUG_ASSERT_POINTER(group->gp_core); ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ return mali_gp_get_interrupt_result(group->gp_core); ++} ++ ++MALI_STATIC_INLINE enum mali_interrupt_result mali_group_get_interrupt_result_pp(struct mali_group *group) ++{ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ MALI_DEBUG_ASSERT_POINTER(group->pp_core); ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ return mali_pp_get_interrupt_result(group->pp_core); ++} ++ ++MALI_STATIC_INLINE enum mali_interrupt_result mali_group_get_interrupt_result_mmu(struct mali_group *group) ++{ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ MALI_DEBUG_ASSERT_POINTER(group->mmu); ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ return mali_mmu_get_interrupt_result(group->mmu); ++} ++ ++MALI_STATIC_INLINE mali_bool mali_group_gp_is_active(struct mali_group *group) ++{ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ MALI_DEBUG_ASSERT_POINTER(group->gp_core); ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ return mali_gp_is_active(group->gp_core); ++} ++ ++MALI_STATIC_INLINE mali_bool mali_group_pp_is_active(struct mali_group *group) ++{ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ MALI_DEBUG_ASSERT_POINTER(group->pp_core); ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ return mali_pp_is_active(group->pp_core); ++} ++ ++MALI_STATIC_INLINE mali_bool mali_group_has_timed_out(struct mali_group *group) ++{ ++ unsigned long time_cost; ++ struct mali_group *tmp_group = group; ++ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ ++ /* if the group is in virtual need to use virtual_group's start time */ ++ if (mali_group_is_in_virtual(group)) { ++ tmp_group = mali_executor_get_virtual_group(); ++ } ++ ++ time_cost = _mali_osk_time_tickcount() - tmp_group->start_time; ++ if (_mali_osk_time_mstoticks(mali_max_job_runtime) <= time_cost) { ++ /* ++ * current tick is at or after timeout end time, ++ * so this is a valid timeout ++ */ ++ return MALI_TRUE; ++ } else { ++ /* ++ * Not a valid timeout. A HW interrupt probably beat ++ * us to it, and the timer wasn't properly deleted ++ * (async deletion used due to atomic context). ++ */ ++ return MALI_FALSE; ++ } ++} ++ ++MALI_STATIC_INLINE void mali_group_mask_all_interrupts_gp(struct mali_group *group) ++{ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ MALI_DEBUG_ASSERT_POINTER(group->gp_core); ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ return mali_gp_mask_all_interrupts(group->gp_core); ++} ++ ++MALI_STATIC_INLINE void mali_group_mask_all_interrupts_pp(struct mali_group *group) ++{ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ MALI_DEBUG_ASSERT_POINTER(group->pp_core); ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ return mali_pp_mask_all_interrupts(group->pp_core); ++} ++ ++MALI_STATIC_INLINE void mali_group_enable_interrupts_gp( ++ struct mali_group *group, ++ enum mali_interrupt_result exceptions) ++{ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ MALI_DEBUG_ASSERT_POINTER(group->gp_core); ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ mali_gp_enable_interrupts(group->gp_core, exceptions); ++} ++ ++MALI_STATIC_INLINE void mali_group_schedule_bottom_half_gp(struct mali_group *group) ++{ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ MALI_DEBUG_ASSERT_POINTER(group->gp_core); ++ _mali_osk_wq_schedule_work(group->bottom_half_work_gp); ++} ++ ++ ++MALI_STATIC_INLINE void mali_group_schedule_bottom_half_pp(struct mali_group *group) ++{ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ MALI_DEBUG_ASSERT_POINTER(group->pp_core); ++ _mali_osk_wq_schedule_work(group->bottom_half_work_pp); ++} ++ ++MALI_STATIC_INLINE void mali_group_schedule_bottom_half_mmu(struct mali_group *group) ++{ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ MALI_DEBUG_ASSERT_POINTER(group->mmu); ++ _mali_osk_wq_schedule_work(group->bottom_half_work_mmu); ++} ++ ++struct mali_pp_job *mali_group_complete_pp(struct mali_group *group, mali_bool success, u32 *sub_job); ++ ++struct mali_gp_job *mali_group_complete_gp(struct mali_group *group, mali_bool success); ++ ++#if defined(CONFIG_MALI400_PROFILING) ++MALI_STATIC_INLINE void mali_group_oom(struct mali_group *group) ++{ ++ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SUSPEND | ++ MALI_PROFILING_MAKE_EVENT_CHANNEL_GP(0), ++ 0, 0, 0, 0, 0); ++} ++#endif ++ ++struct mali_group *mali_group_get_glob_group(u32 index); ++u32 mali_group_get_glob_num_groups(void); ++ ++u32 mali_group_dump_state(struct mali_group *group, char *buf, u32 size); ++ ++ ++_mali_osk_errcode_t mali_group_upper_half_mmu(void *data); ++_mali_osk_errcode_t mali_group_upper_half_gp(void *data); ++_mali_osk_errcode_t mali_group_upper_half_pp(void *data); ++ ++MALI_STATIC_INLINE mali_bool mali_group_is_empty(struct mali_group *group) ++{ ++ MALI_DEBUG_ASSERT_POINTER(group); ++ MALI_DEBUG_ASSERT(mali_group_is_virtual(group)); ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ return _mali_osk_list_empty(&group->group_list); ++} ++ ++#endif /* __MALI_GROUP_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_hw_core.c b/drivers/gpu/arm/mali400/mali/common/mali_hw_core.c +new file mode 100755 +index 000000000000..a813816e998d +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_hw_core.c +@@ -0,0 +1,47 @@ ++/* ++ * Copyright (C) 2011-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include "mali_hw_core.h" ++#include "mali_osk.h" ++#include "mali_kernel_common.h" ++#include "mali_osk_mali.h" ++ ++_mali_osk_errcode_t mali_hw_core_create(struct mali_hw_core *core, const _mali_osk_resource_t *resource, u32 reg_size) ++{ ++ core->phys_addr = resource->base; ++ core->phys_offset = resource->base - _mali_osk_resource_base_address(); ++ core->description = resource->description; ++ core->size = reg_size; ++ ++ MALI_DEBUG_ASSERT(core->phys_offset < core->phys_addr); ++ ++ if (_MALI_OSK_ERR_OK == _mali_osk_mem_reqregion(core->phys_addr, core->size, core->description)) { ++ core->mapped_registers = _mali_osk_mem_mapioregion(core->phys_addr, core->size, core->description); ++ if (NULL != core->mapped_registers) { ++ return _MALI_OSK_ERR_OK; ++ } else { ++ MALI_PRINT_ERROR(("Failed to map memory region for core %s at phys_addr 0x%08X\n", core->description, core->phys_addr)); ++ } ++ _mali_osk_mem_unreqregion(core->phys_addr, core->size); ++ } else { ++ MALI_PRINT_ERROR(("Failed to request memory region for core %s at phys_addr 0x%08X\n", core->description, core->phys_addr)); ++ } ++ ++ return _MALI_OSK_ERR_FAULT; ++} ++ ++void mali_hw_core_delete(struct mali_hw_core *core) ++{ ++ if (NULL != core->mapped_registers) { ++ _mali_osk_mem_unmapioregion(core->phys_addr, core->size, core->mapped_registers); ++ core->mapped_registers = NULL; ++ } ++ _mali_osk_mem_unreqregion(core->phys_addr, core->size); ++} +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_hw_core.h b/drivers/gpu/arm/mali400/mali/common/mali_hw_core.h +new file mode 100755 +index 000000000000..38d96e240a20 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_hw_core.h +@@ -0,0 +1,111 @@ ++/* ++ * Copyright (C) 2011-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_HW_CORE_H__ ++#define __MALI_HW_CORE_H__ ++ ++#include "mali_osk.h" ++#include "mali_kernel_common.h" ++ ++/** ++ * The common parts for all Mali HW cores (GP, PP, MMU, L2 and PMU) ++ * This struct is embedded inside all core specific structs. ++ */ ++struct mali_hw_core { ++ uintptr_t phys_addr; /**< Physical address of the registers */ ++ u32 phys_offset; /**< Offset from start of Mali to registers */ ++ u32 size; /**< Size of registers */ ++ mali_io_address mapped_registers; /**< Virtual mapping of the registers */ ++ const char *description; /**< Name of unit (as specified in device configuration) */ ++}; ++ ++#define MALI_REG_POLL_COUNT_FAST 1000000 ++#define MALI_REG_POLL_COUNT_SLOW 1000000 ++ ++/* ++ * GP and PP core translate their int_stat/rawstat into one of these ++ */ ++enum mali_interrupt_result { ++ MALI_INTERRUPT_RESULT_NONE, ++ MALI_INTERRUPT_RESULT_SUCCESS, ++ MALI_INTERRUPT_RESULT_SUCCESS_VS, ++ MALI_INTERRUPT_RESULT_SUCCESS_PLBU, ++ MALI_INTERRUPT_RESULT_OOM, ++ MALI_INTERRUPT_RESULT_ERROR ++}; ++ ++_mali_osk_errcode_t mali_hw_core_create(struct mali_hw_core *core, const _mali_osk_resource_t *resource, u32 reg_size); ++void mali_hw_core_delete(struct mali_hw_core *core); ++ ++MALI_STATIC_INLINE u32 mali_hw_core_register_read(struct mali_hw_core *core, u32 relative_address) ++{ ++ u32 read_val; ++ read_val = _mali_osk_mem_ioread32(core->mapped_registers, relative_address); ++ MALI_DEBUG_PRINT(6, ("register_read for core %s, relative addr=0x%04X, val=0x%08X\n", ++ core->description, relative_address, read_val)); ++ return read_val; ++} ++ ++MALI_STATIC_INLINE void mali_hw_core_register_write_relaxed(struct mali_hw_core *core, u32 relative_address, u32 new_val) ++{ ++ MALI_DEBUG_PRINT(6, ("register_write_relaxed for core %s, relative addr=0x%04X, val=0x%08X\n", ++ core->description, relative_address, new_val)); ++ _mali_osk_mem_iowrite32_relaxed(core->mapped_registers, relative_address, new_val); ++} ++ ++/* Conditionally write a register. ++ * The register will only be written if the new value is different from the old_value. ++ * If the new value is different, the old value will also be updated */ ++MALI_STATIC_INLINE void mali_hw_core_register_write_relaxed_conditional(struct mali_hw_core *core, u32 relative_address, u32 new_val, const u32 old_val) ++{ ++ MALI_DEBUG_PRINT(6, ("register_write_relaxed for core %s, relative addr=0x%04X, val=0x%08X\n", ++ core->description, relative_address, new_val)); ++ if (old_val != new_val) { ++ _mali_osk_mem_iowrite32_relaxed(core->mapped_registers, relative_address, new_val); ++ } ++} ++ ++MALI_STATIC_INLINE void mali_hw_core_register_write(struct mali_hw_core *core, u32 relative_address, u32 new_val) ++{ ++ MALI_DEBUG_PRINT(6, ("register_write for core %s, relative addr=0x%04X, val=0x%08X\n", ++ core->description, relative_address, new_val)); ++ _mali_osk_mem_iowrite32(core->mapped_registers, relative_address, new_val); ++} ++ ++MALI_STATIC_INLINE void mali_hw_core_register_write_array_relaxed(struct mali_hw_core *core, u32 relative_address, u32 *write_array, u32 nr_of_regs) ++{ ++ u32 i; ++ MALI_DEBUG_PRINT(6, ("register_write_array: for core %s, relative addr=0x%04X, nr of regs=%u\n", ++ core->description, relative_address, nr_of_regs)); ++ ++ /* Do not use burst writes against the registers */ ++ for (i = 0; i < nr_of_regs; i++) { ++ mali_hw_core_register_write_relaxed(core, relative_address + i * 4, write_array[i]); ++ } ++} ++ ++/* Conditionally write a set of registers. ++ * The register will only be written if the new value is different from the old_value. ++ * If the new value is different, the old value will also be updated */ ++MALI_STATIC_INLINE void mali_hw_core_register_write_array_relaxed_conditional(struct mali_hw_core *core, u32 relative_address, u32 *write_array, u32 nr_of_regs, const u32 *old_array) ++{ ++ u32 i; ++ MALI_DEBUG_PRINT(6, ("register_write_array: for core %s, relative addr=0x%04X, nr of regs=%u\n", ++ core->description, relative_address, nr_of_regs)); ++ ++ /* Do not use burst writes against the registers */ ++ for (i = 0; i < nr_of_regs; i++) { ++ if (old_array[i] != write_array[i]) { ++ mali_hw_core_register_write_relaxed(core, relative_address + i * 4, write_array[i]); ++ } ++ } ++} ++ ++#endif /* __MALI_HW_CORE_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_kernel_common.h b/drivers/gpu/arm/mali400/mali/common/mali_kernel_common.h +new file mode 100755 +index 000000000000..6a8f0f0116a4 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_kernel_common.h +@@ -0,0 +1,181 @@ ++/* ++ * Copyright (C) 2010, 2012-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_KERNEL_COMMON_H__ ++#define __MALI_KERNEL_COMMON_H__ ++ ++#include "mali_osk.h" ++ ++/* Make sure debug is defined when it should be */ ++#ifndef DEBUG ++#if defined(_DEBUG) ++#define DEBUG ++#endif ++#endif ++ ++/* The file include several useful macros for error checking, debugging and printing. ++ * - MALI_PRINTF(...) Do not use this function: Will be included in Release builds. ++ * - MALI_DEBUG_PRINT(nr, (X) ) Prints the second argument if nr<=MALI_DEBUG_LEVEL. ++ * - MALI_DEBUG_ERROR( (X) ) Prints an errortext, a source trace, and the given error message. ++ * - MALI_DEBUG_ASSERT(exp,(X)) If the asserted expr is false, the program will exit. ++ * - MALI_DEBUG_ASSERT_POINTER(pointer) Triggers if the pointer is a zero pointer. ++ * - MALI_DEBUG_CODE( X ) The code inside the macro is only compiled in Debug builds. ++ * ++ * The (X) means that you must add an extra parenthesis around the argumentlist. ++ * ++ * The printf function: MALI_PRINTF(...) is routed to _mali_osk_debugmsg ++ * ++ * Suggested range for the DEBUG-LEVEL is [1:6] where ++ * [1:2] Is messages with highest priority, indicate possible errors. ++ * [3:4] Is messages with medium priority, output important variables. ++ * [5:6] Is messages with low priority, used during extensive debugging. ++ */ ++ ++/** ++* Fundamental error macro. Reports an error code. This is abstracted to allow us to ++* easily switch to a different error reporting method if we want, and also to allow ++* us to search for error returns easily. ++* ++* Note no closing semicolon - this is supplied in typical usage: ++* ++* MALI_ERROR(MALI_ERROR_OUT_OF_MEMORY); ++*/ ++#define MALI_ERROR(error_code) return (error_code) ++ ++/** ++ * Basic error macro, to indicate success. ++ * Note no closing semicolon - this is supplied in typical usage: ++ * ++ * MALI_SUCCESS; ++ */ ++#define MALI_SUCCESS MALI_ERROR(_MALI_OSK_ERR_OK) ++ ++/** ++ * Basic error macro. This checks whether the given condition is true, and if not returns ++ * from this function with the supplied error code. This is a macro so that we can override it ++ * for stress testing. ++ * ++ * Note that this uses the do-while-0 wrapping to ensure that we don't get problems with dangling ++ * else clauses. Note also no closing semicolon - this is supplied in typical usage: ++ * ++ * MALI_CHECK((p!=NULL), ERROR_NO_OBJECT); ++ */ ++#define MALI_CHECK(condition, error_code) do { if(!(condition)) MALI_ERROR(error_code); } while(0) ++ ++/** ++ * Error propagation macro. If the expression given is anything other than ++ * _MALI_OSK_NO_ERROR, then the value is returned from the enclosing function ++ * as an error code. This effectively acts as a guard clause, and propagates ++ * error values up the call stack. This uses a temporary value to ensure that ++ * the error expression is not evaluated twice. ++ * If the counter for forcing a failure has been set using _mali_force_error, ++ * this error will be returned without evaluating the expression in ++ * MALI_CHECK_NO_ERROR ++ */ ++#define MALI_CHECK_NO_ERROR(expression) \ ++ do { _mali_osk_errcode_t _check_no_error_result=(expression); \ ++ if(_check_no_error_result != _MALI_OSK_ERR_OK) \ ++ MALI_ERROR(_check_no_error_result); \ ++ } while(0) ++ ++/** ++ * Pointer check macro. Checks non-null pointer. ++ */ ++#define MALI_CHECK_NON_NULL(pointer, error_code) MALI_CHECK( ((pointer)!=NULL), (error_code) ) ++ ++/** ++ * Error macro with goto. This checks whether the given condition is true, and if not jumps ++ * to the specified label using a goto. The label must therefore be local to the function in ++ * which this macro appears. This is most usually used to execute some clean-up code before ++ * exiting with a call to ERROR. ++ * ++ * Like the other macros, this is a macro to allow us to override the condition if we wish, ++ * e.g. to force an error during stress testing. ++ */ ++#define MALI_CHECK_GOTO(condition, label) do { if(!(condition)) goto label; } while(0) ++ ++/** ++ * Explicitly ignore a parameter passed into a function, to suppress compiler warnings. ++ * Should only be used with parameter names. ++ */ ++#define MALI_IGNORE(x) x=x ++ ++#if defined(CONFIG_MALI_QUIET) ++#define MALI_PRINTF(args) ++#else ++#define MALI_PRINTF(args) _mali_osk_dbgmsg args; ++#endif ++ ++#define MALI_PRINT_ERROR(args) do{ \ ++ MALI_PRINTF(("Mali: ERR: %s\n" ,__FILE__)); \ ++ MALI_PRINTF((" %s()%4d\n ", __FUNCTION__, __LINE__)) ; \ ++ MALI_PRINTF(args); \ ++ MALI_PRINTF(("\n")); \ ++ } while(0) ++ ++#define MALI_PRINT(args) do{ \ ++ MALI_PRINTF(("Mali: ")); \ ++ MALI_PRINTF(args); \ ++ } while (0) ++ ++#ifdef DEBUG ++#ifndef mali_debug_level ++extern int mali_debug_level; ++#endif ++ ++#define MALI_DEBUG_CODE(code) code ++#define MALI_DEBUG_PRINT(level, args) do { \ ++ if((level) <= mali_debug_level)\ ++ {MALI_PRINTF(("Mali<" #level ">: ")); MALI_PRINTF(args); } \ ++ } while (0) ++ ++#define MALI_DEBUG_PRINT_ERROR(args) MALI_PRINT_ERROR(args) ++ ++#define MALI_DEBUG_PRINT_IF(level,condition,args) \ ++ if((condition)&&((level) <= mali_debug_level))\ ++ {MALI_PRINTF(("Mali<" #level ">: ")); MALI_PRINTF(args); } ++ ++#define MALI_DEBUG_PRINT_ELSE(level, args)\ ++ else if((level) <= mali_debug_level)\ ++ { MALI_PRINTF(("Mali<" #level ">: ")); MALI_PRINTF(args); } ++ ++/** ++ * @note these variants of DEBUG ASSERTS will cause a debugger breakpoint ++ * to be entered (see _mali_osk_break() ). An alternative would be to call ++ * _mali_osk_abort(), on OSs that support it. ++ */ ++#define MALI_DEBUG_PRINT_ASSERT(condition, args) do {if( !(condition)) { MALI_PRINT_ERROR(args); _mali_osk_break(); } } while(0) ++#define MALI_DEBUG_ASSERT_POINTER(pointer) do {if( (pointer)== NULL) {MALI_PRINT_ERROR(("NULL pointer " #pointer)); _mali_osk_break();} } while(0) ++#define MALI_DEBUG_ASSERT(condition) do {if( !(condition)) {MALI_PRINT_ERROR(("ASSERT failed: " #condition )); _mali_osk_break();} } while(0) ++ ++#else /* DEBUG */ ++ ++#define MALI_DEBUG_CODE(code) ++#define MALI_DEBUG_PRINT(string,args) do {} while(0) ++#define MALI_DEBUG_PRINT_ERROR(args) do {} while(0) ++#define MALI_DEBUG_PRINT_IF(level,condition,args) do {} while(0) ++#define MALI_DEBUG_PRINT_ELSE(level,condition,args) do {} while(0) ++#define MALI_DEBUG_PRINT_ASSERT(condition,args) do {} while(0) ++#define MALI_DEBUG_ASSERT_POINTER(pointer) do {} while(0) ++#define MALI_DEBUG_ASSERT(condition) do {} while(0) ++ ++#endif /* DEBUG */ ++ ++/** ++ * variables from user space cannot be dereferenced from kernel space; tagging them ++ * with __user allows the GCC compiler to generate a warning. Other compilers may ++ * not support this so we define it here as an empty macro if the compiler doesn't ++ * define it. ++ */ ++#ifndef __user ++#define __user ++#endif ++ ++#endif /* __MALI_KERNEL_COMMON_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_kernel_core.c b/drivers/gpu/arm/mali400/mali/common/mali_kernel_core.c +new file mode 100755 +index 000000000000..87f97b710257 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_kernel_core.c +@@ -0,0 +1,1349 @@ ++/* ++ * Copyright (C) 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include "mali_kernel_common.h" ++#include "mali_session.h" ++#include "mali_osk.h" ++#include "mali_osk_mali.h" ++#include "mali_ukk.h" ++#include "mali_kernel_core.h" ++#include "mali_memory.h" ++#include "mali_mem_validation.h" ++#include "mali_mmu.h" ++#include "mali_mmu_page_directory.h" ++#include "mali_dlbu.h" ++#include "mali_broadcast.h" ++#include "mali_gp.h" ++#include "mali_pp.h" ++#include "mali_executor.h" ++#include "mali_pp_job.h" ++#include "mali_group.h" ++#include "mali_pm.h" ++#include "mali_pmu.h" ++#include "mali_scheduler.h" ++#include "mali_kernel_utilization.h" ++#include "mali_l2_cache.h" ++#include "mali_timeline.h" ++#include "mali_soft_job.h" ++#include "mali_pm_domain.h" ++#if defined(CONFIG_MALI400_PROFILING) ++#include "mali_osk_profiling.h" ++#endif ++#if defined(CONFIG_MALI400_INTERNAL_PROFILING) ++#include "mali_profiling_internal.h" ++#endif ++#include "mali_control_timer.h" ++#include "mali_dvfs_policy.h" ++#include ++#include ++#if defined(CONFIG_MALI_DMA_BUF_FENCE) ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++#include ++#else ++#include ++#endif ++#endif ++ ++#define MALI_SHARED_MEMORY_DEFAULT_SIZE 0xffffffff ++ ++/* Mali GPU memory. Real values come from module parameter or from device specific data */ ++unsigned int mali_dedicated_mem_start = 0; ++unsigned int mali_dedicated_mem_size = 0; ++ ++/* Default shared memory size is set to 4G. */ ++unsigned int mali_shared_mem_size = MALI_SHARED_MEMORY_DEFAULT_SIZE; ++ ++/* Frame buffer memory to be accessible by Mali GPU */ ++int mali_fb_start = 0; ++int mali_fb_size = 0; ++ ++/* Mali max job runtime */ ++extern int mali_max_job_runtime; ++ ++/** Start profiling from module load? */ ++int mali_boot_profiling = 0; ++ ++/** Limits for the number of PP cores behind each L2 cache. */ ++int mali_max_pp_cores_group_1 = 0xFF; ++int mali_max_pp_cores_group_2 = 0xFF; ++ ++int mali_inited_pp_cores_group_1 = 0; ++int mali_inited_pp_cores_group_2 = 0; ++ ++static _mali_product_id_t global_product_id = _MALI_PRODUCT_ID_UNKNOWN; ++static uintptr_t global_gpu_base_address = 0; ++static u32 global_gpu_major_version = 0; ++static u32 global_gpu_minor_version = 0; ++ ++mali_bool mali_gpu_class_is_mali450 = MALI_FALSE; ++mali_bool mali_gpu_class_is_mali470 = MALI_FALSE; ++ ++static _mali_osk_errcode_t mali_set_global_gpu_base_address(void) ++{ ++ _mali_osk_errcode_t err = _MALI_OSK_ERR_OK; ++ ++ global_gpu_base_address = _mali_osk_resource_base_address(); ++ if (0 == global_gpu_base_address) { ++ err = _MALI_OSK_ERR_ITEM_NOT_FOUND; ++ } ++ ++ return err; ++} ++ ++static u32 mali_get_bcast_id(_mali_osk_resource_t *resource_pp) ++{ ++ switch (resource_pp->base - global_gpu_base_address) { ++ case 0x08000: ++ case 0x20000: /* fall-through for aliased mapping */ ++ return 0x01; ++ case 0x0A000: ++ case 0x22000: /* fall-through for aliased mapping */ ++ return 0x02; ++ case 0x0C000: ++ case 0x24000: /* fall-through for aliased mapping */ ++ return 0x04; ++ case 0x0E000: ++ case 0x26000: /* fall-through for aliased mapping */ ++ return 0x08; ++ case 0x28000: ++ return 0x10; ++ case 0x2A000: ++ return 0x20; ++ case 0x2C000: ++ return 0x40; ++ case 0x2E000: ++ return 0x80; ++ default: ++ return 0; ++ } ++} ++ ++static _mali_osk_errcode_t mali_parse_product_info(void) ++{ ++ _mali_osk_resource_t first_pp_resource; ++ ++ /* Find the first PP core resource (again) */ ++ if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(MALI_OFFSET_PP0, &first_pp_resource)) { ++ /* Create a dummy PP object for this core so that we can read the version register */ ++ struct mali_group *group = mali_group_create(NULL, NULL, NULL, MALI_DOMAIN_INDEX_PP0); ++ if (NULL != group) { ++ struct mali_pp_core *pp_core = mali_pp_create(&first_pp_resource, group, MALI_FALSE, mali_get_bcast_id(&first_pp_resource)); ++ if (NULL != pp_core) { ++ u32 pp_version; ++ ++ pp_version = mali_pp_core_get_version(pp_core); ++ ++ mali_group_delete(group); ++ ++ global_gpu_major_version = (pp_version >> 8) & 0xFF; ++ global_gpu_minor_version = pp_version & 0xFF; ++ ++ switch (pp_version >> 16) { ++ case MALI200_PP_PRODUCT_ID: ++ global_product_id = _MALI_PRODUCT_ID_MALI200; ++ MALI_DEBUG_PRINT(2, ("Found Mali GPU Mali-200 r%up%u\n", global_gpu_major_version, global_gpu_minor_version)); ++ MALI_PRINT_ERROR(("Mali-200 is not supported by this driver.\n")); ++ _mali_osk_abort(); ++ break; ++ case MALI300_PP_PRODUCT_ID: ++ global_product_id = _MALI_PRODUCT_ID_MALI300; ++ MALI_DEBUG_PRINT(2, ("Found Mali GPU Mali-300 r%up%u\n", global_gpu_major_version, global_gpu_minor_version)); ++ break; ++ case MALI400_PP_PRODUCT_ID: ++ global_product_id = _MALI_PRODUCT_ID_MALI400; ++ MALI_DEBUG_PRINT(2, ("Found Mali GPU Mali-400 MP r%up%u\n", global_gpu_major_version, global_gpu_minor_version)); ++ break; ++ case MALI450_PP_PRODUCT_ID: ++ global_product_id = _MALI_PRODUCT_ID_MALI450; ++ MALI_DEBUG_PRINT(2, ("Found Mali GPU Mali-450 MP r%up%u\n", global_gpu_major_version, global_gpu_minor_version)); ++ break; ++ case MALI470_PP_PRODUCT_ID: ++ global_product_id = _MALI_PRODUCT_ID_MALI470; ++ MALI_DEBUG_PRINT(2, ("Found Mali GPU Mali-470 MP r%up%u\n", global_gpu_major_version, global_gpu_minor_version)); ++ break; ++ default: ++ MALI_DEBUG_PRINT(2, ("Found unknown Mali GPU (r%up%u)\n", global_gpu_major_version, global_gpu_minor_version)); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ return _MALI_OSK_ERR_OK; ++ } else { ++ MALI_PRINT_ERROR(("Failed to create initial PP object\n")); ++ } ++ } else { ++ MALI_PRINT_ERROR(("Failed to create initial group object\n")); ++ } ++ } else { ++ MALI_PRINT_ERROR(("First PP core not specified in config file\n")); ++ } ++ ++ return _MALI_OSK_ERR_FAULT; ++} ++ ++static void mali_delete_groups(void) ++{ ++ struct mali_group *group; ++ ++ group = mali_group_get_glob_group(0); ++ while (NULL != group) { ++ mali_group_delete(group); ++ group = mali_group_get_glob_group(0); ++ } ++ ++ MALI_DEBUG_ASSERT(0 == mali_group_get_glob_num_groups()); ++} ++ ++static void mali_delete_l2_cache_cores(void) ++{ ++ struct mali_l2_cache_core *l2; ++ ++ l2 = mali_l2_cache_core_get_glob_l2_core(0); ++ while (NULL != l2) { ++ mali_l2_cache_delete(l2); ++ l2 = mali_l2_cache_core_get_glob_l2_core(0); ++ } ++ ++ MALI_DEBUG_ASSERT(0 == mali_l2_cache_core_get_glob_num_l2_cores()); ++} ++ ++static struct mali_l2_cache_core *mali_create_l2_cache_core(_mali_osk_resource_t *resource, u32 domain_index) ++{ ++ struct mali_l2_cache_core *l2_cache = NULL; ++ ++ if (NULL != resource) { ++ ++ MALI_DEBUG_PRINT(3, ("Found L2 cache %s\n", resource->description)); ++ ++ l2_cache = mali_l2_cache_create(resource, domain_index); ++ if (NULL == l2_cache) { ++ MALI_PRINT_ERROR(("Failed to create L2 cache object\n")); ++ return NULL; ++ } ++ } ++ MALI_DEBUG_PRINT(3, ("Created L2 cache core object\n")); ++ ++ return l2_cache; ++} ++ ++static _mali_osk_errcode_t mali_parse_config_l2_cache(void) ++{ ++ struct mali_l2_cache_core *l2_cache = NULL; ++ ++ if (mali_is_mali400()) { ++ _mali_osk_resource_t l2_resource; ++ if (_MALI_OSK_ERR_OK != _mali_osk_resource_find(MALI400_OFFSET_L2_CACHE0, &l2_resource)) { ++ MALI_DEBUG_PRINT(3, ("Did not find required Mali L2 cache in config file\n")); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ l2_cache = mali_create_l2_cache_core(&l2_resource, MALI_DOMAIN_INDEX_L20); ++ if (NULL == l2_cache) { ++ return _MALI_OSK_ERR_FAULT; ++ } ++ } else if (mali_is_mali450()) { ++ /* ++ * L2 for GP at 0x10000 ++ * L2 for PP0-3 at 0x01000 ++ * L2 for PP4-7 at 0x11000 (optional) ++ */ ++ ++ _mali_osk_resource_t l2_gp_resource; ++ _mali_osk_resource_t l2_pp_grp0_resource; ++ _mali_osk_resource_t l2_pp_grp1_resource; ++ ++ /* Make cluster for GP's L2 */ ++ if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(MALI450_OFFSET_L2_CACHE0, &l2_gp_resource)) { ++ MALI_DEBUG_PRINT(3, ("Creating Mali-450 L2 cache core for GP\n")); ++ l2_cache = mali_create_l2_cache_core(&l2_gp_resource, MALI_DOMAIN_INDEX_L20); ++ if (NULL == l2_cache) { ++ return _MALI_OSK_ERR_FAULT; ++ } ++ } else { ++ MALI_DEBUG_PRINT(3, ("Did not find required Mali L2 cache for GP in config file\n")); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ /* Find corresponding l2 domain */ ++ if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(MALI450_OFFSET_L2_CACHE1, &l2_pp_grp0_resource)) { ++ MALI_DEBUG_PRINT(3, ("Creating Mali-450 L2 cache core for PP group 0\n")); ++ l2_cache = mali_create_l2_cache_core(&l2_pp_grp0_resource, MALI_DOMAIN_INDEX_L21); ++ if (NULL == l2_cache) { ++ return _MALI_OSK_ERR_FAULT; ++ } ++ } else { ++ MALI_DEBUG_PRINT(3, ("Did not find required Mali L2 cache for PP group 0 in config file\n")); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ /* Second PP core group is optional, don't fail if we don't find it */ ++ if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(MALI450_OFFSET_L2_CACHE2, &l2_pp_grp1_resource)) { ++ MALI_DEBUG_PRINT(3, ("Creating Mali-450 L2 cache core for PP group 1\n")); ++ l2_cache = mali_create_l2_cache_core(&l2_pp_grp1_resource, MALI_DOMAIN_INDEX_L22); ++ if (NULL == l2_cache) { ++ return _MALI_OSK_ERR_FAULT; ++ } ++ } ++ } else if (mali_is_mali470()) { ++ _mali_osk_resource_t l2c1_resource; ++ ++ /* Make cluster for L2C1 */ ++ if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(MALI470_OFFSET_L2_CACHE1, &l2c1_resource)) { ++ MALI_DEBUG_PRINT(3, ("Creating Mali-470 L2 cache 1\n")); ++ l2_cache = mali_create_l2_cache_core(&l2c1_resource, MALI_DOMAIN_INDEX_L21); ++ if (NULL == l2_cache) { ++ return _MALI_OSK_ERR_FAULT; ++ } ++ } else { ++ MALI_DEBUG_PRINT(3, ("Did not find required Mali L2 cache for L2C1\n")); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ } ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++static struct mali_group *mali_create_group(struct mali_l2_cache_core *cache, ++ _mali_osk_resource_t *resource_mmu, ++ _mali_osk_resource_t *resource_gp, ++ _mali_osk_resource_t *resource_pp, ++ u32 domain_index) ++{ ++ struct mali_mmu_core *mmu; ++ struct mali_group *group; ++ ++ MALI_DEBUG_PRINT(3, ("Starting new group for MMU %s\n", resource_mmu->description)); ++ ++ /* Create the group object */ ++ group = mali_group_create(cache, NULL, NULL, domain_index); ++ if (NULL == group) { ++ MALI_PRINT_ERROR(("Failed to create group object for MMU %s\n", resource_mmu->description)); ++ return NULL; ++ } ++ ++ /* Create the MMU object inside group */ ++ mmu = mali_mmu_create(resource_mmu, group, MALI_FALSE); ++ if (NULL == mmu) { ++ MALI_PRINT_ERROR(("Failed to create MMU object\n")); ++ mali_group_delete(group); ++ return NULL; ++ } ++ ++ if (NULL != resource_gp) { ++ /* Create the GP core object inside this group */ ++ struct mali_gp_core *gp_core = mali_gp_create(resource_gp, group); ++ if (NULL == gp_core) { ++ /* No need to clean up now, as we will clean up everything linked in from the cluster when we fail this function */ ++ MALI_PRINT_ERROR(("Failed to create GP object\n")); ++ mali_group_delete(group); ++ return NULL; ++ } ++ } ++ ++ if (NULL != resource_pp) { ++ struct mali_pp_core *pp_core; ++ ++ /* Create the PP core object inside this group */ ++ pp_core = mali_pp_create(resource_pp, group, MALI_FALSE, mali_get_bcast_id(resource_pp)); ++ if (NULL == pp_core) { ++ /* No need to clean up now, as we will clean up everything linked in from the cluster when we fail this function */ ++ MALI_PRINT_ERROR(("Failed to create PP object\n")); ++ mali_group_delete(group); ++ return NULL; ++ } ++ } ++ ++ return group; ++} ++ ++static _mali_osk_errcode_t mali_create_virtual_group(_mali_osk_resource_t *resource_mmu_pp_bcast, ++ _mali_osk_resource_t *resource_pp_bcast, ++ _mali_osk_resource_t *resource_dlbu, ++ _mali_osk_resource_t *resource_bcast) ++{ ++ struct mali_mmu_core *mmu_pp_bcast_core; ++ struct mali_pp_core *pp_bcast_core; ++ struct mali_dlbu_core *dlbu_core; ++ struct mali_bcast_unit *bcast_core; ++ struct mali_group *group; ++ ++ MALI_DEBUG_PRINT(2, ("Starting new virtual group for MMU PP broadcast core %s\n", resource_mmu_pp_bcast->description)); ++ ++ /* Create the DLBU core object */ ++ dlbu_core = mali_dlbu_create(resource_dlbu); ++ if (NULL == dlbu_core) { ++ MALI_PRINT_ERROR(("Failed to create DLBU object \n")); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ /* Create the Broadcast unit core */ ++ bcast_core = mali_bcast_unit_create(resource_bcast); ++ if (NULL == bcast_core) { ++ MALI_PRINT_ERROR(("Failed to create Broadcast unit object!\n")); ++ mali_dlbu_delete(dlbu_core); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ /* Create the group object */ ++#if defined(DEBUG) ++ /* Get a physical PP group to temporarily add to broadcast unit. IRQ ++ * verification needs a physical group in the broadcast unit to test ++ * the broadcast unit interrupt line. */ ++ { ++ struct mali_group *phys_group = NULL; ++ int i; ++ for (i = 0; i < mali_group_get_glob_num_groups(); i++) { ++ phys_group = mali_group_get_glob_group(i); ++ if (NULL != mali_group_get_pp_core(phys_group)) break; ++ } ++ MALI_DEBUG_ASSERT(NULL != mali_group_get_pp_core(phys_group)); ++ ++ /* Add the group temporarily to the broadcast, and update the ++ * broadcast HW. Since the HW is not updated when removing the ++ * group the IRQ check will work when the virtual PP is created ++ * later. ++ * ++ * When the virtual group gets populated, the actually used ++ * groups will be added to the broadcast unit and the HW will ++ * be updated. ++ */ ++ mali_bcast_add_group(bcast_core, phys_group); ++ mali_bcast_reset(bcast_core); ++ mali_bcast_remove_group(bcast_core, phys_group); ++ } ++#endif /* DEBUG */ ++ group = mali_group_create(NULL, dlbu_core, bcast_core, MALI_DOMAIN_INDEX_DUMMY); ++ if (NULL == group) { ++ MALI_PRINT_ERROR(("Failed to create group object for MMU PP broadcast core %s\n", resource_mmu_pp_bcast->description)); ++ mali_bcast_unit_delete(bcast_core); ++ mali_dlbu_delete(dlbu_core); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ /* Create the MMU object inside group */ ++ mmu_pp_bcast_core = mali_mmu_create(resource_mmu_pp_bcast, group, MALI_TRUE); ++ if (NULL == mmu_pp_bcast_core) { ++ MALI_PRINT_ERROR(("Failed to create MMU PP broadcast object\n")); ++ mali_group_delete(group); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ /* Create the PP core object inside this group */ ++ pp_bcast_core = mali_pp_create(resource_pp_bcast, group, MALI_TRUE, 0); ++ if (NULL == pp_bcast_core) { ++ /* No need to clean up now, as we will clean up everything linked in from the cluster when we fail this function */ ++ MALI_PRINT_ERROR(("Failed to create PP object\n")); ++ mali_group_delete(group); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++static _mali_osk_errcode_t mali_parse_config_groups(void) ++{ ++ struct mali_group *group; ++ int cluster_id_gp = 0; ++ int cluster_id_pp_grp0 = 0; ++ int cluster_id_pp_grp1 = 0; ++ int i; ++ ++ _mali_osk_resource_t resource_gp; ++ _mali_osk_resource_t resource_gp_mmu; ++ _mali_osk_resource_t resource_pp[8]; ++ _mali_osk_resource_t resource_pp_mmu[8]; ++ _mali_osk_resource_t resource_pp_mmu_bcast; ++ _mali_osk_resource_t resource_pp_bcast; ++ _mali_osk_resource_t resource_dlbu; ++ _mali_osk_resource_t resource_bcast; ++ _mali_osk_errcode_t resource_gp_found; ++ _mali_osk_errcode_t resource_gp_mmu_found; ++ _mali_osk_errcode_t resource_pp_found[8]; ++ _mali_osk_errcode_t resource_pp_mmu_found[8]; ++ _mali_osk_errcode_t resource_pp_mmu_bcast_found; ++ _mali_osk_errcode_t resource_pp_bcast_found; ++ _mali_osk_errcode_t resource_dlbu_found; ++ _mali_osk_errcode_t resource_bcast_found; ++ ++ if (!(mali_is_mali400() || mali_is_mali450() || mali_is_mali470())) { ++ /* No known HW core */ ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ if (MALI_MAX_JOB_RUNTIME_DEFAULT == mali_max_job_runtime) { ++ /* Group settings are not overridden by module parameters, so use device settings */ ++ _mali_osk_device_data data = { 0, }; ++ ++ if (_MALI_OSK_ERR_OK == _mali_osk_device_data_get(&data)) { ++ /* Use device specific settings (if defined) */ ++ if (0 != data.max_job_runtime) { ++ mali_max_job_runtime = data.max_job_runtime; ++ } ++ } ++ } ++ ++ if (mali_is_mali450()) { ++ /* Mali-450 have separate L2s for GP, and PP core group(s) */ ++ cluster_id_pp_grp0 = 1; ++ cluster_id_pp_grp1 = 2; ++ } ++ ++ resource_gp_found = _mali_osk_resource_find(MALI_OFFSET_GP, &resource_gp); ++ resource_gp_mmu_found = _mali_osk_resource_find(MALI_OFFSET_GP_MMU, &resource_gp_mmu); ++ resource_pp_found[0] = _mali_osk_resource_find(MALI_OFFSET_PP0, &(resource_pp[0])); ++ resource_pp_found[1] = _mali_osk_resource_find(MALI_OFFSET_PP1, &(resource_pp[1])); ++ resource_pp_found[2] = _mali_osk_resource_find(MALI_OFFSET_PP2, &(resource_pp[2])); ++ resource_pp_found[3] = _mali_osk_resource_find(MALI_OFFSET_PP3, &(resource_pp[3])); ++ resource_pp_found[4] = _mali_osk_resource_find(MALI_OFFSET_PP4, &(resource_pp[4])); ++ resource_pp_found[5] = _mali_osk_resource_find(MALI_OFFSET_PP5, &(resource_pp[5])); ++ resource_pp_found[6] = _mali_osk_resource_find(MALI_OFFSET_PP6, &(resource_pp[6])); ++ resource_pp_found[7] = _mali_osk_resource_find(MALI_OFFSET_PP7, &(resource_pp[7])); ++ resource_pp_mmu_found[0] = _mali_osk_resource_find(MALI_OFFSET_PP0_MMU, &(resource_pp_mmu[0])); ++ resource_pp_mmu_found[1] = _mali_osk_resource_find(MALI_OFFSET_PP1_MMU, &(resource_pp_mmu[1])); ++ resource_pp_mmu_found[2] = _mali_osk_resource_find(MALI_OFFSET_PP2_MMU, &(resource_pp_mmu[2])); ++ resource_pp_mmu_found[3] = _mali_osk_resource_find(MALI_OFFSET_PP3_MMU, &(resource_pp_mmu[3])); ++ resource_pp_mmu_found[4] = _mali_osk_resource_find(MALI_OFFSET_PP4_MMU, &(resource_pp_mmu[4])); ++ resource_pp_mmu_found[5] = _mali_osk_resource_find(MALI_OFFSET_PP5_MMU, &(resource_pp_mmu[5])); ++ resource_pp_mmu_found[6] = _mali_osk_resource_find(MALI_OFFSET_PP6_MMU, &(resource_pp_mmu[6])); ++ resource_pp_mmu_found[7] = _mali_osk_resource_find(MALI_OFFSET_PP7_MMU, &(resource_pp_mmu[7])); ++ ++ ++ if (mali_is_mali450() || mali_is_mali470()) { ++ resource_bcast_found = _mali_osk_resource_find(MALI_OFFSET_BCAST, &resource_bcast); ++ resource_dlbu_found = _mali_osk_resource_find(MALI_OFFSET_DLBU, &resource_dlbu); ++ resource_pp_mmu_bcast_found = _mali_osk_resource_find(MALI_OFFSET_PP_BCAST_MMU, &resource_pp_mmu_bcast); ++ resource_pp_bcast_found = _mali_osk_resource_find(MALI_OFFSET_PP_BCAST, &resource_pp_bcast); ++ ++ if (_MALI_OSK_ERR_OK != resource_bcast_found || ++ _MALI_OSK_ERR_OK != resource_dlbu_found || ++ _MALI_OSK_ERR_OK != resource_pp_mmu_bcast_found || ++ _MALI_OSK_ERR_OK != resource_pp_bcast_found) { ++ /* Missing mandatory core(s) for Mali-450 or Mali-470 */ ++ MALI_DEBUG_PRINT(2, ("Missing mandatory resources, Mali-450 needs DLBU, Broadcast unit, virtual PP core and virtual MMU\n")); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ } ++ ++ if (_MALI_OSK_ERR_OK != resource_gp_found || ++ _MALI_OSK_ERR_OK != resource_gp_mmu_found || ++ _MALI_OSK_ERR_OK != resource_pp_found[0] || ++ _MALI_OSK_ERR_OK != resource_pp_mmu_found[0]) { ++ /* Missing mandatory core(s) */ ++ MALI_DEBUG_PRINT(2, ("Missing mandatory resource, need at least one GP and one PP, both with a separate MMU\n")); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ MALI_DEBUG_ASSERT(1 <= mali_l2_cache_core_get_glob_num_l2_cores()); ++ group = mali_create_group(mali_l2_cache_core_get_glob_l2_core(cluster_id_gp), &resource_gp_mmu, &resource_gp, NULL, MALI_DOMAIN_INDEX_GP); ++ if (NULL == group) { ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ /* Create group for first (and mandatory) PP core */ ++ MALI_DEBUG_ASSERT(mali_l2_cache_core_get_glob_num_l2_cores() >= (cluster_id_pp_grp0 + 1)); /* >= 1 on Mali-300 and Mali-400, >= 2 on Mali-450 */ ++ group = mali_create_group(mali_l2_cache_core_get_glob_l2_core(cluster_id_pp_grp0), &resource_pp_mmu[0], NULL, &resource_pp[0], MALI_DOMAIN_INDEX_PP0); ++ if (NULL == group) { ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ mali_inited_pp_cores_group_1++; ++ ++ /* Create groups for rest of the cores in the first PP core group */ ++ for (i = 1; i < 4; i++) { /* First half of the PP cores belong to first core group */ ++ if (mali_inited_pp_cores_group_1 < mali_max_pp_cores_group_1) { ++ if (_MALI_OSK_ERR_OK == resource_pp_found[i] && _MALI_OSK_ERR_OK == resource_pp_mmu_found[i]) { ++ group = mali_create_group(mali_l2_cache_core_get_glob_l2_core(cluster_id_pp_grp0), &resource_pp_mmu[i], NULL, &resource_pp[i], MALI_DOMAIN_INDEX_PP0 + i); ++ if (NULL == group) { ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ mali_inited_pp_cores_group_1++; ++ } ++ } ++ } ++ ++ /* Create groups for cores in the second PP core group */ ++ for (i = 4; i < 8; i++) { /* Second half of the PP cores belong to second core group */ ++ if (mali_inited_pp_cores_group_2 < mali_max_pp_cores_group_2) { ++ if (_MALI_OSK_ERR_OK == resource_pp_found[i] && _MALI_OSK_ERR_OK == resource_pp_mmu_found[i]) { ++ MALI_DEBUG_ASSERT(mali_l2_cache_core_get_glob_num_l2_cores() >= 2); /* Only Mali-450 have a second core group */ ++ group = mali_create_group(mali_l2_cache_core_get_glob_l2_core(cluster_id_pp_grp1), &resource_pp_mmu[i], NULL, &resource_pp[i], MALI_DOMAIN_INDEX_PP0 + i); ++ if (NULL == group) { ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ mali_inited_pp_cores_group_2++; ++ } ++ } ++ } ++ ++ if (mali_is_mali450() || mali_is_mali470()) { ++ _mali_osk_errcode_t err = mali_create_virtual_group(&resource_pp_mmu_bcast, &resource_pp_bcast, &resource_dlbu, &resource_bcast); ++ if (_MALI_OSK_ERR_OK != err) { ++ return err; ++ } ++ } ++ ++ mali_max_pp_cores_group_1 = mali_inited_pp_cores_group_1; ++ mali_max_pp_cores_group_2 = mali_inited_pp_cores_group_2; ++ MALI_DEBUG_PRINT(2, ("%d+%d PP cores initialized\n", mali_inited_pp_cores_group_1, mali_inited_pp_cores_group_2)); ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++static _mali_osk_errcode_t mali_check_shared_interrupts(void) ++{ ++#if !defined(CONFIG_MALI_SHARED_INTERRUPTS) ++ if (MALI_TRUE == _mali_osk_shared_interrupts()) { ++ MALI_PRINT_ERROR(("Shared interrupts detected, but driver support is not enabled\n")); ++ return _MALI_OSK_ERR_FAULT; ++ } ++#endif /* !defined(CONFIG_MALI_SHARED_INTERRUPTS) */ ++ ++ /* It is OK to compile support for shared interrupts even if Mali is not using it. */ ++ return _MALI_OSK_ERR_OK; ++} ++ ++static _mali_osk_errcode_t mali_parse_config_pmu(void) ++{ ++ _mali_osk_resource_t resource_pmu; ++ ++ MALI_DEBUG_ASSERT(0 != global_gpu_base_address); ++ ++ if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(MALI_OFFSET_PMU, &resource_pmu)) { ++ struct mali_pmu_core *pmu; ++ ++ pmu = mali_pmu_create(&resource_pmu); ++ if (NULL == pmu) { ++ MALI_PRINT_ERROR(("Failed to create PMU\n")); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ } ++ ++ /* It's ok if the PMU doesn't exist */ ++ return _MALI_OSK_ERR_OK; ++} ++ ++static _mali_osk_errcode_t mali_parse_config_memory(void) ++{ ++ _mali_osk_device_data data = { 0, }; ++ _mali_osk_errcode_t ret; ++ ++ /* The priority of setting the value of mali_shared_mem_size, ++ * mali_dedicated_mem_start and mali_dedicated_mem_size: ++ * 1. module parameter; ++ * 2. platform data; ++ * 3. default value; ++ **/ ++ if (_MALI_OSK_ERR_OK == _mali_osk_device_data_get(&data)) { ++ /* Memory settings are not overridden by module parameters, so use device settings */ ++ if (0 == mali_dedicated_mem_start && 0 == mali_dedicated_mem_size) { ++ /* Use device specific settings (if defined) */ ++ mali_dedicated_mem_start = data.dedicated_mem_start; ++ mali_dedicated_mem_size = data.dedicated_mem_size; ++ } ++ ++ if (MALI_SHARED_MEMORY_DEFAULT_SIZE == mali_shared_mem_size && ++ 0 != data.shared_mem_size) { ++ mali_shared_mem_size = data.shared_mem_size; ++ } ++ } ++ ++ if (0 < mali_dedicated_mem_size && 0 != mali_dedicated_mem_start) { ++ MALI_DEBUG_PRINT(2, ("Mali memory settings (dedicated: 0x%08X@0x%08X)\n", ++ mali_dedicated_mem_size, mali_dedicated_mem_start)); ++ ++ /* Dedicated memory */ ++ ret = mali_memory_core_resource_dedicated_memory(mali_dedicated_mem_start, mali_dedicated_mem_size); ++ if (_MALI_OSK_ERR_OK != ret) { ++ MALI_PRINT_ERROR(("Failed to register dedicated memory\n")); ++ mali_memory_terminate(); ++ return ret; ++ } ++ } ++ ++ if (0 < mali_shared_mem_size) { ++ MALI_DEBUG_PRINT(2, ("Mali memory settings (shared: 0x%08X)\n", mali_shared_mem_size)); ++ ++ /* Shared OS memory */ ++ ret = mali_memory_core_resource_os_memory(mali_shared_mem_size); ++ if (_MALI_OSK_ERR_OK != ret) { ++ MALI_PRINT_ERROR(("Failed to register shared OS memory\n")); ++ mali_memory_terminate(); ++ return ret; ++ } ++ } ++ ++ if (0 == mali_fb_start && 0 == mali_fb_size) { ++ /* Frame buffer settings are not overridden by module parameters, so use device settings */ ++ _mali_osk_device_data data = { 0, }; ++ ++ if (_MALI_OSK_ERR_OK == _mali_osk_device_data_get(&data)) { ++ /* Use device specific settings (if defined) */ ++ mali_fb_start = data.fb_start; ++ mali_fb_size = data.fb_size; ++ } ++ ++ MALI_DEBUG_PRINT(2, ("Using device defined frame buffer settings (0x%08X@0x%08X)\n", ++ mali_fb_size, mali_fb_start)); ++ } else { ++ MALI_DEBUG_PRINT(2, ("Using module defined frame buffer settings (0x%08X@0x%08X)\n", ++ mali_fb_size, mali_fb_start)); ++ } ++ ++ if (0 != mali_fb_size) { ++ /* Register frame buffer */ ++ ret = mali_mem_validation_add_range(mali_fb_start, mali_fb_size); ++ if (_MALI_OSK_ERR_OK != ret) { ++ MALI_PRINT_ERROR(("Failed to register frame buffer memory region\n")); ++ mali_memory_terminate(); ++ return ret; ++ } ++ } ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++static void mali_detect_gpu_class(void) ++{ ++ if (_mali_osk_identify_gpu_resource() == 0x450) ++ mali_gpu_class_is_mali450 = MALI_TRUE; ++ ++ if (_mali_osk_identify_gpu_resource() == 0x470) ++ mali_gpu_class_is_mali470 = MALI_TRUE; ++} ++ ++static _mali_osk_errcode_t mali_init_hw_reset(void) ++{ ++#if (defined(CONFIG_MALI450) || defined(CONFIG_MALI470)) ++ _mali_osk_resource_t resource_bcast; ++ ++ /* Ensure broadcast unit is in a good state before we start creating ++ * groups and cores. ++ */ ++ if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(MALI_OFFSET_BCAST, &resource_bcast)) { ++ struct mali_bcast_unit *bcast_core; ++ ++ bcast_core = mali_bcast_unit_create(&resource_bcast); ++ if (NULL == bcast_core) { ++ MALI_PRINT_ERROR(("Failed to create Broadcast unit object!\n")); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ mali_bcast_unit_delete(bcast_core); ++ } ++#endif /* (defined(CONFIG_MALI450) || defined(CONFIG_MALI470)) */ ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++_mali_osk_errcode_t mali_initialize_subsystems(void) ++{ ++ _mali_osk_errcode_t err; ++ ++#ifdef CONFIG_MALI_DT ++ err = _mali_osk_resource_initialize(); ++ if (_MALI_OSK_ERR_OK != err) { ++ mali_terminate_subsystems(); ++ return err; ++ } ++#endif ++ ++ mali_pp_job_initialize(); ++ ++ err = mali_timeline_initialize(); ++ if (_MALI_OSK_ERR_OK != err) { ++ mali_terminate_subsystems(); ++ return err; ++ } ++ ++ err = mali_session_initialize(); ++ if (_MALI_OSK_ERR_OK != err) { ++ mali_terminate_subsystems(); ++ return err; ++ } ++ ++ /*Try to init gpu secure mode */ ++ _mali_osk_gpu_secure_mode_init(); ++ ++#if defined(CONFIG_MALI400_PROFILING) ++ err = _mali_osk_profiling_init(mali_boot_profiling ? MALI_TRUE : MALI_FALSE); ++ if (_MALI_OSK_ERR_OK != err) { ++ /* No biggie if we weren't able to initialize the profiling */ ++ MALI_PRINT_ERROR(("Failed to initialize profiling, feature will be unavailable\n")); ++ } ++#endif ++ ++ err = mali_memory_initialize(); ++ if (_MALI_OSK_ERR_OK != err) { ++ mali_terminate_subsystems(); ++ return err; ++ } ++ ++ err = mali_executor_initialize(); ++ if (_MALI_OSK_ERR_OK != err) { ++ mali_terminate_subsystems(); ++ return err; ++ } ++ ++ err = mali_scheduler_initialize(); ++ if (_MALI_OSK_ERR_OK != err) { ++ mali_terminate_subsystems(); ++ return err; ++ } ++ ++ /* Configure memory early, needed by mali_mmu_initialize. */ ++ err = mali_parse_config_memory(); ++ if (_MALI_OSK_ERR_OK != err) { ++ mali_terminate_subsystems(); ++ return err; ++ } ++ ++ err = mali_set_global_gpu_base_address(); ++ if (_MALI_OSK_ERR_OK != err) { ++ mali_terminate_subsystems(); ++ return err; ++ } ++ ++ /* Detect GPU class (uses L2 cache count) */ ++ mali_detect_gpu_class(); ++ ++ err = mali_check_shared_interrupts(); ++ if (_MALI_OSK_ERR_OK != err) { ++ mali_terminate_subsystems(); ++ return err; ++ } ++ ++ /* Initialize the MALI PMU (will not touch HW!) */ ++ err = mali_parse_config_pmu(); ++ if (_MALI_OSK_ERR_OK != err) { ++ mali_terminate_subsystems(); ++ return err; ++ } ++ ++ /* Initialize the power management module */ ++ err = mali_pm_initialize(); ++ if (_MALI_OSK_ERR_OK != err) { ++ mali_terminate_subsystems(); ++ return err; ++ } ++ ++ /* Make sure the entire GPU stays on for the rest of this function */ ++ mali_pm_init_begin(); ++ ++ /* Ensure HW is in a good state before starting to access cores. */ ++ err = mali_init_hw_reset(); ++ if (_MALI_OSK_ERR_OK != err) { ++ mali_terminate_subsystems(); ++ return err; ++ } ++ ++ /* Detect which Mali GPU we are dealing with */ ++ err = mali_parse_product_info(); ++ if (_MALI_OSK_ERR_OK != err) { ++ mali_pm_init_end(); ++ mali_terminate_subsystems(); ++ return err; ++ } ++ ++ /* The global_product_id is now populated with the correct Mali GPU */ ++ ++ /* Start configuring the actual Mali hardware. */ ++ ++ err = mali_mmu_initialize(); ++ if (_MALI_OSK_ERR_OK != err) { ++ mali_pm_init_end(); ++ mali_terminate_subsystems(); ++ return err; ++ } ++ ++ if (mali_is_mali450() || mali_is_mali470()) { ++ err = mali_dlbu_initialize(); ++ if (_MALI_OSK_ERR_OK != err) { ++ mali_pm_init_end(); ++ mali_terminate_subsystems(); ++ return err; ++ } ++ } ++ ++ err = mali_parse_config_l2_cache(); ++ if (_MALI_OSK_ERR_OK != err) { ++ mali_pm_init_end(); ++ mali_terminate_subsystems(); ++ return err; ++ } ++ ++ err = mali_parse_config_groups(); ++ if (_MALI_OSK_ERR_OK != err) { ++ mali_pm_init_end(); ++ mali_terminate_subsystems(); ++ return err; ++ } ++ ++ /* Move groups into executor */ ++ mali_executor_populate(); ++ ++ /* Need call after all group has assigned a domain */ ++ mali_pm_power_cost_setup(); ++ ++ /* Initialize the GPU timer */ ++ err = mali_control_timer_init(); ++ if (_MALI_OSK_ERR_OK != err) { ++ mali_pm_init_end(); ++ mali_terminate_subsystems(); ++ return err; ++ } ++ ++ /* Initialize the GPU utilization tracking */ ++ err = mali_utilization_init(); ++ if (_MALI_OSK_ERR_OK != err) { ++ mali_pm_init_end(); ++ mali_terminate_subsystems(); ++ return err; ++ } ++ ++#if defined(CONFIG_MALI_DVFS) ++ err = mali_dvfs_policy_init(); ++ if (_MALI_OSK_ERR_OK != err) { ++ mali_pm_init_end(); ++ mali_terminate_subsystems(); ++ return err; ++ } ++#endif ++ ++ /* Allowing the system to be turned off */ ++ mali_pm_init_end(); ++ ++ return _MALI_OSK_ERR_OK; /* all ok */ ++} ++ ++void mali_terminate_subsystems(void) ++{ ++ struct mali_pmu_core *pmu = mali_pmu_get_global_pmu_core(); ++ ++ MALI_DEBUG_PRINT(2, ("terminate_subsystems() called\n")); ++ ++ mali_utilization_term(); ++ mali_control_timer_term(); ++ ++ mali_executor_depopulate(); ++ mali_delete_groups(); /* Delete groups not added to executor */ ++ mali_executor_terminate(); ++ ++ mali_scheduler_terminate(); ++ mali_pp_job_terminate(); ++ mali_delete_l2_cache_cores(); ++ mali_mmu_terminate(); ++ ++ if (mali_is_mali450() || mali_is_mali470()) { ++ mali_dlbu_terminate(); ++ } ++ ++ mali_pm_terminate(); ++ ++ if (NULL != pmu) { ++ mali_pmu_delete(pmu); ++ } ++ ++#if defined(CONFIG_MALI400_PROFILING) ++ _mali_osk_profiling_term(); ++#endif ++ ++ _mali_osk_gpu_secure_mode_deinit(); ++ ++ mali_memory_terminate(); ++ ++ mali_session_terminate(); ++ ++ mali_timeline_terminate(); ++ ++ global_gpu_base_address = 0; ++} ++ ++_mali_product_id_t mali_kernel_core_get_product_id(void) ++{ ++ return global_product_id; ++} ++ ++u32 mali_kernel_core_get_gpu_major_version(void) ++{ ++ return global_gpu_major_version; ++} ++ ++u32 mali_kernel_core_get_gpu_minor_version(void) ++{ ++ return global_gpu_minor_version; ++} ++ ++_mali_osk_errcode_t _mali_ukk_get_api_version(_mali_uk_get_api_version_s *args) ++{ ++ MALI_DEBUG_ASSERT_POINTER(args); ++ MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx); ++ ++ /* check compatability */ ++ if (args->version == _MALI_UK_API_VERSION) { ++ args->compatible = 1; ++ } else { ++ args->compatible = 0; ++ } ++ ++ args->version = _MALI_UK_API_VERSION; /* report our version */ ++ ++ /* success regardless of being compatible or not */ ++ MALI_SUCCESS; ++} ++ ++_mali_osk_errcode_t _mali_ukk_get_api_version_v2(_mali_uk_get_api_version_v2_s *args) ++{ ++ MALI_DEBUG_ASSERT_POINTER(args); ++ MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx); ++ ++ /* check compatability */ ++ if (args->version == _MALI_UK_API_VERSION) { ++ args->compatible = 1; ++ } else { ++ args->compatible = 0; ++ } ++ ++ args->version = _MALI_UK_API_VERSION; /* report our version */ ++ ++ /* success regardless of being compatible or not */ ++ return _MALI_OSK_ERR_OK; ++} ++ ++_mali_osk_errcode_t _mali_ukk_wait_for_notification(_mali_uk_wait_for_notification_s *args) ++{ ++ _mali_osk_errcode_t err; ++ _mali_osk_notification_t *notification; ++ _mali_osk_notification_queue_t *queue; ++ struct mali_session_data *session; ++ ++ /* check input */ ++ MALI_DEBUG_ASSERT_POINTER(args); ++ MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx); ++ ++ session = (struct mali_session_data *)(uintptr_t)args->ctx; ++ queue = session->ioctl_queue; ++ ++ /* if the queue does not exist we're currently shutting down */ ++ if (NULL == queue) { ++ MALI_DEBUG_PRINT(1, ("No notification queue registered with the session. Asking userspace to stop querying\n")); ++ args->type = _MALI_NOTIFICATION_CORE_SHUTDOWN_IN_PROGRESS; ++ return _MALI_OSK_ERR_OK; ++ } ++ ++ /* receive a notification, might sleep */ ++ err = _mali_osk_notification_queue_receive(queue, ¬ification); ++ if (_MALI_OSK_ERR_OK != err) { ++ MALI_ERROR(err); /* errcode returned, pass on to caller */ ++ } ++ ++ /* copy the buffer to the user */ ++ args->type = (_mali_uk_notification_type)notification->notification_type; ++ _mali_osk_memcpy(&args->data, notification->result_buffer, notification->result_buffer_size); ++ ++ /* finished with the notification */ ++ _mali_osk_notification_delete(notification); ++ ++ return _MALI_OSK_ERR_OK; /* all ok */ ++} ++ ++_mali_osk_errcode_t _mali_ukk_post_notification(_mali_uk_post_notification_s *args) ++{ ++ _mali_osk_notification_t *notification; ++ _mali_osk_notification_queue_t *queue; ++ struct mali_session_data *session; ++ ++ /* check input */ ++ MALI_DEBUG_ASSERT_POINTER(args); ++ MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx); ++ ++ session = (struct mali_session_data *)(uintptr_t)args->ctx; ++ queue = session->ioctl_queue; ++ ++ /* if the queue does not exist we're currently shutting down */ ++ if (NULL == queue) { ++ MALI_DEBUG_PRINT(1, ("No notification queue registered with the session. Asking userspace to stop querying\n")); ++ return _MALI_OSK_ERR_OK; ++ } ++ ++ notification = _mali_osk_notification_create(args->type, 0); ++ if (NULL == notification) { ++ MALI_PRINT_ERROR(("Failed to create notification object\n")); ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ ++ _mali_osk_notification_queue_send(queue, notification); ++ ++ return _MALI_OSK_ERR_OK; /* all ok */ ++} ++ ++_mali_osk_errcode_t _mali_ukk_pending_submit(_mali_uk_pending_submit_s *args) ++{ ++ wait_queue_head_t *queue; ++ ++ /* check input */ ++ MALI_DEBUG_ASSERT_POINTER(args); ++ MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx); ++ ++ queue = mali_session_get_wait_queue(); ++ ++ /* check pending big job number, might sleep if larger than MAX allowed number */ ++ if (wait_event_interruptible(*queue, MALI_MAX_PENDING_BIG_JOB > mali_scheduler_job_gp_big_job_count())) { ++ return _MALI_OSK_ERR_RESTARTSYSCALL; ++ } ++ ++ return _MALI_OSK_ERR_OK; /* all ok */ ++} ++ ++ ++_mali_osk_errcode_t _mali_ukk_request_high_priority(_mali_uk_request_high_priority_s *args) ++{ ++ struct mali_session_data *session; ++ ++ MALI_DEBUG_ASSERT_POINTER(args); ++ MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx); ++ ++ session = (struct mali_session_data *)(uintptr_t)args->ctx; ++ ++ if (!session->use_high_priority_job_queue) { ++ session->use_high_priority_job_queue = MALI_TRUE; ++ MALI_DEBUG_PRINT(2, ("Session 0x%08X with pid %d was granted higher priority.\n", session, _mali_osk_get_pid())); ++ } ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++_mali_osk_errcode_t _mali_ukk_open(void **context) ++{ ++ u32 i; ++ struct mali_session_data *session; ++ ++ /* allocated struct to track this session */ ++ session = (struct mali_session_data *)_mali_osk_calloc(1, sizeof(struct mali_session_data)); ++ MALI_CHECK_NON_NULL(session, _MALI_OSK_ERR_NOMEM); ++ ++ MALI_DEBUG_PRINT(3, ("Session starting\n")); ++ ++ /* create a response queue for this session */ ++ session->ioctl_queue = _mali_osk_notification_queue_init(); ++ if (NULL == session->ioctl_queue) { ++ goto err; ++ } ++ ++ /*create a wait queue for this session */ ++ session->wait_queue = _mali_osk_wait_queue_init(); ++ if (NULL == session->wait_queue) { ++ goto err_wait_queue; ++ } ++ ++ session->page_directory = mali_mmu_pagedir_alloc(); ++ if (NULL == session->page_directory) { ++ goto err_mmu; ++ } ++ ++ if (_MALI_OSK_ERR_OK != mali_mmu_pagedir_map(session->page_directory, MALI_DLBU_VIRT_ADDR, _MALI_OSK_MALI_PAGE_SIZE)) { ++ MALI_PRINT_ERROR(("Failed to map DLBU page into session\n")); ++ goto err_mmu; ++ } ++ ++ if (0 != mali_dlbu_phys_addr) { ++ mali_mmu_pagedir_update(session->page_directory, MALI_DLBU_VIRT_ADDR, mali_dlbu_phys_addr, ++ _MALI_OSK_MALI_PAGE_SIZE, MALI_MMU_FLAGS_DEFAULT); ++ } ++ ++ if (_MALI_OSK_ERR_OK != mali_memory_session_begin(session)) { ++ goto err_session; ++ } ++ ++ /* Create soft system. */ ++ session->soft_job_system = mali_soft_job_system_create(session); ++ if (NULL == session->soft_job_system) { ++ goto err_soft; ++ } ++ ++ /* Initialize the dma fence context.*/ ++#if defined(CONFIG_MALI_DMA_BUF_FENCE) ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++ session->fence_context = dma_fence_context_alloc(1); ++#elif LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0) ++ session->fence_context = fence_context_alloc(1); ++ _mali_osk_atomic_init(&session->fence_seqno, 0); ++#else ++ MALI_PRINT_ERROR(("The kernel version not support dma fence!\n")); ++ goto err_time_line; ++#endif ++#endif ++ ++ /* Create timeline system. */ ++ session->timeline_system = mali_timeline_system_create(session); ++ if (NULL == session->timeline_system) { ++ goto err_time_line; ++ } ++ ++#if defined(CONFIG_MALI_DVFS) ++ _mali_osk_atomic_init(&session->number_of_window_jobs, 0); ++#endif ++ ++ _mali_osk_atomic_init(&session->number_of_pp_jobs, 0); ++ ++ session->use_high_priority_job_queue = MALI_FALSE; ++ ++ /* Initialize list of PP jobs on this session. */ ++ _MALI_OSK_INIT_LIST_HEAD(&session->pp_job_list); ++ ++ /* Initialize the pp_job_fb_lookup_list array used to quickly lookup jobs from a given frame builder */ ++ for (i = 0; i < MALI_PP_JOB_FB_LOOKUP_LIST_SIZE; ++i) { ++ _MALI_OSK_INIT_LIST_HEAD(&session->pp_job_fb_lookup_list[i]); ++ } ++ ++ session->pid = _mali_osk_get_pid(); ++ session->comm = _mali_osk_get_comm(); ++ session->max_mali_mem_allocated_size = 0; ++ for (i = 0; i < MALI_MEM_TYPE_MAX; i ++) { ++ atomic_set(&session->mali_mem_array[i], 0); ++ } ++ atomic_set(&session->mali_mem_allocated_pages, 0); ++ *context = (void *)session; ++ ++ /* Add session to the list of all sessions. */ ++ mali_session_add(session); ++ ++ MALI_DEBUG_PRINT(3, ("Session started\n")); ++ return _MALI_OSK_ERR_OK; ++ ++err_time_line: ++ mali_soft_job_system_destroy(session->soft_job_system); ++err_soft: ++ mali_memory_session_end(session); ++err_session: ++ mali_mmu_pagedir_free(session->page_directory); ++err_mmu: ++ _mali_osk_wait_queue_term(session->wait_queue); ++err_wait_queue: ++ _mali_osk_notification_queue_term(session->ioctl_queue); ++err: ++ _mali_osk_free(session); ++ MALI_ERROR(_MALI_OSK_ERR_NOMEM); ++ ++} ++ ++#if defined(DEBUG) ++/* parameter used for debug */ ++extern u32 num_pm_runtime_resume; ++extern u32 num_pm_updates; ++extern u32 num_pm_updates_up; ++extern u32 num_pm_updates_down; ++#endif ++ ++_mali_osk_errcode_t _mali_ukk_close(void **context) ++{ ++ struct mali_session_data *session; ++ MALI_CHECK_NON_NULL(context, _MALI_OSK_ERR_INVALID_ARGS); ++ session = (struct mali_session_data *)*context; ++ ++ MALI_DEBUG_PRINT(3, ("Session ending\n")); ++ ++ MALI_DEBUG_ASSERT_POINTER(session->soft_job_system); ++ MALI_DEBUG_ASSERT_POINTER(session->timeline_system); ++ ++ /* Remove session from list of all sessions. */ ++ mali_session_remove(session); ++ ++ /* This flag is used to prevent queueing of jobs due to activation. */ ++ session->is_aborting = MALI_TRUE; ++ ++ /* Stop the soft job timer. */ ++ mali_timeline_system_stop_timer(session->timeline_system); ++ ++ /* Abort queued jobs */ ++ mali_scheduler_abort_session(session); ++ ++ /* Abort executing jobs */ ++ mali_executor_abort_session(session); ++ ++ /* Abort the soft job system. */ ++ mali_soft_job_system_abort(session->soft_job_system); ++ ++ /* Force execution of all pending bottom half processing for GP and PP. */ ++ _mali_osk_wq_flush(); ++ ++ /* The session PP list should now be empty. */ ++ MALI_DEBUG_ASSERT(_mali_osk_list_empty(&session->pp_job_list)); ++ ++ /* At this point the GP and PP scheduler no longer has any jobs queued or running from this ++ * session, and all soft jobs in the soft job system has been destroyed. */ ++ ++ /* Any trackers left in the timeline system are directly or indirectly waiting on external ++ * sync fences. Cancel all sync fence waiters to trigger activation of all remaining ++ * trackers. This call will sleep until all timelines are empty. */ ++ mali_timeline_system_abort(session->timeline_system); ++ ++ /* Flush pending work. ++ * Needed to make sure all bottom half processing related to this ++ * session has been completed, before we free internal data structures. ++ */ ++ _mali_osk_wq_flush(); ++ ++ /* Destroy timeline system. */ ++ mali_timeline_system_destroy(session->timeline_system); ++ session->timeline_system = NULL; ++ ++ /* Destroy soft system. */ ++ mali_soft_job_system_destroy(session->soft_job_system); ++ session->soft_job_system = NULL; ++ ++ /*Wait for the session job lists become empty.*/ ++ _mali_osk_wait_queue_wait_event(session->wait_queue, mali_session_pp_job_is_empty, (void *) session); ++ ++ /* Free remaining memory allocated to this session */ ++ mali_memory_session_end(session); ++ ++#if defined(CONFIG_MALI_DVFS) ++ _mali_osk_atomic_term(&session->number_of_window_jobs); ++#endif ++ ++#if defined(CONFIG_MALI400_PROFILING) ++ _mali_osk_profiling_stop_sampling(session->pid); ++#endif ++ ++ /* Free session data structures */ ++ mali_mmu_pagedir_unmap(session->page_directory, MALI_DLBU_VIRT_ADDR, _MALI_OSK_MALI_PAGE_SIZE); ++ mali_mmu_pagedir_free(session->page_directory); ++ _mali_osk_wait_queue_term(session->wait_queue); ++ _mali_osk_notification_queue_term(session->ioctl_queue); ++ _mali_osk_free(session); ++ ++ *context = NULL; ++ ++ MALI_DEBUG_PRINT(3, ("Session has ended\n")); ++ ++#if defined(DEBUG) ++ MALI_DEBUG_PRINT(3, ("Stats: # runtime resumes: %u\n", num_pm_runtime_resume)); ++ MALI_DEBUG_PRINT(3, (" # PM updates: .... %u (up %u, down %u)\n", num_pm_updates, num_pm_updates_up, num_pm_updates_down)); ++ ++ num_pm_runtime_resume = 0; ++ num_pm_updates = 0; ++ num_pm_updates_up = 0; ++ num_pm_updates_down = 0; ++#endif ++ ++ return _MALI_OSK_ERR_OK;; ++} ++ ++#if MALI_STATE_TRACKING ++u32 _mali_kernel_core_dump_state(char *buf, u32 size) ++{ ++ int n = 0; /* Number of bytes written to buf */ ++ ++ n += mali_scheduler_dump_state(buf + n, size - n); ++ n += mali_executor_dump_state(buf + n, size - n); ++ ++ return n; ++} ++#endif +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_kernel_core.h b/drivers/gpu/arm/mali400/mali/common/mali_kernel_core.h +new file mode 100755 +index 000000000000..c471fc955107 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_kernel_core.h +@@ -0,0 +1,57 @@ ++/* ++ * Copyright (C) 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_KERNEL_CORE_H__ ++#define __MALI_KERNEL_CORE_H__ ++ ++#include "mali_osk.h" ++ ++typedef enum { ++ _MALI_PRODUCT_ID_UNKNOWN, ++ _MALI_PRODUCT_ID_MALI200, ++ _MALI_PRODUCT_ID_MALI300, ++ _MALI_PRODUCT_ID_MALI400, ++ _MALI_PRODUCT_ID_MALI450, ++ _MALI_PRODUCT_ID_MALI470, ++} _mali_product_id_t; ++ ++extern mali_bool mali_gpu_class_is_mali450; ++extern mali_bool mali_gpu_class_is_mali470; ++ ++_mali_osk_errcode_t mali_initialize_subsystems(void); ++ ++void mali_terminate_subsystems(void); ++ ++_mali_product_id_t mali_kernel_core_get_product_id(void); ++ ++u32 mali_kernel_core_get_gpu_major_version(void); ++ ++u32 mali_kernel_core_get_gpu_minor_version(void); ++ ++u32 _mali_kernel_core_dump_state(char *buf, u32 size); ++ ++MALI_STATIC_INLINE mali_bool mali_is_mali470(void) ++{ ++ return mali_gpu_class_is_mali470; ++} ++ ++MALI_STATIC_INLINE mali_bool mali_is_mali450(void) ++{ ++ return mali_gpu_class_is_mali450; ++} ++ ++MALI_STATIC_INLINE mali_bool mali_is_mali400(void) ++{ ++ if (mali_gpu_class_is_mali450 || mali_gpu_class_is_mali470) ++ return MALI_FALSE; ++ ++ return MALI_TRUE; ++} ++#endif /* __MALI_KERNEL_CORE_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_kernel_utilization.c b/drivers/gpu/arm/mali400/mali/common/mali_kernel_utilization.c +new file mode 100755 +index 000000000000..d1b8dc3b0b0e +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_kernel_utilization.c +@@ -0,0 +1,440 @@ ++/* ++ * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include "mali_kernel_utilization.h" ++#include "mali_osk.h" ++#include "mali_osk_mali.h" ++#include "mali_kernel_common.h" ++#include "mali_session.h" ++#include "mali_scheduler.h" ++ ++#include "mali_executor.h" ++#include "mali_dvfs_policy.h" ++#include "mali_control_timer.h" ++ ++/* Thresholds for GP bound detection. */ ++#define MALI_GP_BOUND_GP_UTILIZATION_THRESHOLD 240 ++#define MALI_GP_BOUND_PP_UTILIZATION_THRESHOLD 250 ++ ++static _mali_osk_spinlock_irq_t *utilization_data_lock; ++ ++static u32 num_running_gp_cores = 0; ++static u32 num_running_pp_cores = 0; ++ ++static u64 work_start_time_gpu = 0; ++static u64 work_start_time_gp = 0; ++static u64 work_start_time_pp = 0; ++static u64 accumulated_work_time_gpu = 0; ++static u64 accumulated_work_time_gp = 0; ++static u64 accumulated_work_time_pp = 0; ++ ++static u32 last_utilization_gpu = 0 ; ++static u32 last_utilization_gp = 0 ; ++static u32 last_utilization_pp = 0 ; ++ ++void (*mali_utilization_callback)(struct mali_gpu_utilization_data *data) = NULL; ++ ++/* Define the first timer control timer timeout in milliseconds */ ++static u32 mali_control_first_timeout = 100; ++static struct mali_gpu_utilization_data mali_util_data = {0, }; ++ ++struct mali_gpu_utilization_data *mali_utilization_calculate(u64 *start_time, u64 *time_period, mali_bool *need_add_timer) ++{ ++ u64 time_now; ++ u32 leading_zeroes; ++ u32 shift_val; ++ u32 work_normalized_gpu; ++ u32 work_normalized_gp; ++ u32 work_normalized_pp; ++ u32 period_normalized; ++ u32 utilization_gpu; ++ u32 utilization_gp; ++ u32 utilization_pp; ++ ++ mali_utilization_data_lock(); ++ ++ time_now = _mali_osk_time_get_ns(); ++ ++ *time_period = time_now - *start_time; ++ ++ if (accumulated_work_time_gpu == 0 && work_start_time_gpu == 0) { ++ mali_control_timer_pause(); ++ /* ++ * No work done for this period ++ * - No need to reschedule timer ++ * - Report zero usage ++ */ ++ last_utilization_gpu = 0; ++ last_utilization_gp = 0; ++ last_utilization_pp = 0; ++ ++ mali_util_data.utilization_gpu = last_utilization_gpu; ++ mali_util_data.utilization_gp = last_utilization_gp; ++ mali_util_data.utilization_pp = last_utilization_pp; ++ ++ mali_utilization_data_unlock(); ++ ++ *need_add_timer = MALI_FALSE; ++ ++ mali_executor_hint_disable(MALI_EXECUTOR_HINT_GP_BOUND); ++ ++ MALI_DEBUG_PRINT(4, ("last_utilization_gpu = %d \n", last_utilization_gpu)); ++ MALI_DEBUG_PRINT(4, ("last_utilization_gp = %d \n", last_utilization_gp)); ++ MALI_DEBUG_PRINT(4, ("last_utilization_pp = %d \n", last_utilization_pp)); ++ ++ return &mali_util_data; ++ } ++ ++ /* If we are currently busy, update working period up to now */ ++ if (work_start_time_gpu != 0) { ++ accumulated_work_time_gpu += (time_now - work_start_time_gpu); ++ work_start_time_gpu = time_now; ++ ++ /* GP and/or PP will also be busy if the GPU is busy at this point */ ++ ++ if (work_start_time_gp != 0) { ++ accumulated_work_time_gp += (time_now - work_start_time_gp); ++ work_start_time_gp = time_now; ++ } ++ ++ if (work_start_time_pp != 0) { ++ accumulated_work_time_pp += (time_now - work_start_time_pp); ++ work_start_time_pp = time_now; ++ } ++ } ++ ++ /* ++ * We have two 64-bit values, a dividend and a divisor. ++ * To avoid dependencies to a 64-bit divider, we shift down the two values ++ * equally first. ++ * We shift the dividend up and possibly the divisor down, making the result X in 256. ++ */ ++ ++ /* Shift the 64-bit values down so they fit inside a 32-bit integer */ ++ leading_zeroes = _mali_osk_clz((u32)(*time_period >> 32)); ++ shift_val = 32 - leading_zeroes; ++ work_normalized_gpu = (u32)(accumulated_work_time_gpu >> shift_val); ++ work_normalized_gp = (u32)(accumulated_work_time_gp >> shift_val); ++ work_normalized_pp = (u32)(accumulated_work_time_pp >> shift_val); ++ period_normalized = (u32)(*time_period >> shift_val); ++ ++ /* ++ * Now, we should report the usage in parts of 256 ++ * this means we must shift up the dividend or down the divisor by 8 ++ * (we could do a combination, but we just use one for simplicity, ++ * but the end result should be good enough anyway) ++ */ ++ if (period_normalized > 0x00FFFFFF) { ++ /* The divisor is so big that it is safe to shift it down */ ++ period_normalized >>= 8; ++ } else { ++ /* ++ * The divisor is so small that we can shift up the dividend, without loosing any data. ++ * (dividend is always smaller than the divisor) ++ */ ++ work_normalized_gpu <<= 8; ++ work_normalized_gp <<= 8; ++ work_normalized_pp <<= 8; ++ } ++ ++ utilization_gpu = work_normalized_gpu / period_normalized; ++ utilization_gp = work_normalized_gp / period_normalized; ++ utilization_pp = work_normalized_pp / period_normalized; ++ ++ last_utilization_gpu = utilization_gpu; ++ last_utilization_gp = utilization_gp; ++ last_utilization_pp = utilization_pp; ++ ++ if ((MALI_GP_BOUND_GP_UTILIZATION_THRESHOLD < last_utilization_gp) && ++ (MALI_GP_BOUND_PP_UTILIZATION_THRESHOLD > last_utilization_pp)) { ++ mali_executor_hint_enable(MALI_EXECUTOR_HINT_GP_BOUND); ++ } else { ++ mali_executor_hint_disable(MALI_EXECUTOR_HINT_GP_BOUND); ++ } ++ ++ /* starting a new period */ ++ accumulated_work_time_gpu = 0; ++ accumulated_work_time_gp = 0; ++ accumulated_work_time_pp = 0; ++ ++ *start_time = time_now; ++ ++ mali_util_data.utilization_gp = last_utilization_gp; ++ mali_util_data.utilization_gpu = last_utilization_gpu; ++ mali_util_data.utilization_pp = last_utilization_pp; ++ ++ mali_utilization_data_unlock(); ++ ++ *need_add_timer = MALI_TRUE; ++ ++ MALI_DEBUG_PRINT(4, ("last_utilization_gpu = %d \n", last_utilization_gpu)); ++ MALI_DEBUG_PRINT(4, ("last_utilization_gp = %d \n", last_utilization_gp)); ++ MALI_DEBUG_PRINT(4, ("last_utilization_pp = %d \n", last_utilization_pp)); ++ ++ return &mali_util_data; ++} ++ ++_mali_osk_errcode_t mali_utilization_init(void) ++{ ++#if USING_GPU_UTILIZATION ++ _mali_osk_device_data data; ++ ++ if (_MALI_OSK_ERR_OK == _mali_osk_device_data_get(&data)) { ++ if (NULL != data.utilization_callback) { ++ mali_utilization_callback = data.utilization_callback; ++ MALI_DEBUG_PRINT(2, ("Mali GPU Utilization: Utilization handler installed \n")); ++ } ++ } ++#endif /* defined(USING_GPU_UTILIZATION) */ ++ ++ if (NULL == mali_utilization_callback) { ++ MALI_DEBUG_PRINT(2, ("Mali GPU Utilization: No platform utilization handler installed\n")); ++ } ++ ++ utilization_data_lock = _mali_osk_spinlock_irq_init(_MALI_OSK_LOCKFLAG_ORDERED, _MALI_OSK_LOCK_ORDER_UTILIZATION); ++ if (NULL == utilization_data_lock) { ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ num_running_gp_cores = 0; ++ num_running_pp_cores = 0; ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++void mali_utilization_term(void) ++{ ++ if (NULL != utilization_data_lock) { ++ _mali_osk_spinlock_irq_term(utilization_data_lock); ++ } ++} ++ ++void mali_utilization_gp_start(void) ++{ ++ mali_utilization_data_lock(); ++ ++ ++num_running_gp_cores; ++ if (1 == num_running_gp_cores) { ++ u64 time_now = _mali_osk_time_get_ns(); ++ ++ /* First GP core started, consider GP busy from now and onwards */ ++ work_start_time_gp = time_now; ++ ++ if (0 == num_running_pp_cores) { ++ mali_bool is_resume = MALI_FALSE; ++ /* ++ * There are no PP cores running, so this is also the point ++ * at which we consider the GPU to be busy as well. ++ */ ++ work_start_time_gpu = time_now; ++ ++ is_resume = mali_control_timer_resume(time_now); ++ ++ mali_utilization_data_unlock(); ++ ++ if (is_resume) { ++ /* Do some policy in new period for performance consideration */ ++#if defined(CONFIG_MALI_DVFS) ++ /* Clear session->number_of_window_jobs, prepare parameter for dvfs */ ++ mali_session_max_window_num(); ++ if (0 == last_utilization_gpu) { ++ /* ++ * for mali_dev_pause is called in set clock, ++ * so each time we change clock, we will set clock to ++ * highest step even if under down clock case, ++ * it is not nessesary, so we only set the clock under ++ * last time utilization equal 0, we stop the timer then ++ * start the GPU again case ++ */ ++ mali_dvfs_policy_new_period(); ++ } ++#endif ++ /* ++ * First timeout using short interval for power consideration ++ * because we give full power in the new period, but if the ++ * job loading is light, finish in 10ms, the other time all keep ++ * in high freq it will wast time. ++ */ ++ mali_control_timer_add(mali_control_first_timeout); ++ } ++ } else { ++ mali_utilization_data_unlock(); ++ } ++ ++ } else { ++ /* Nothing to do */ ++ mali_utilization_data_unlock(); ++ } ++} ++ ++void mali_utilization_pp_start(void) ++{ ++ mali_utilization_data_lock(); ++ ++ ++num_running_pp_cores; ++ if (1 == num_running_pp_cores) { ++ u64 time_now = _mali_osk_time_get_ns(); ++ ++ /* First PP core started, consider PP busy from now and onwards */ ++ work_start_time_pp = time_now; ++ ++ if (0 == num_running_gp_cores) { ++ mali_bool is_resume = MALI_FALSE; ++ /* ++ * There are no GP cores running, so this is also the point ++ * at which we consider the GPU to be busy as well. ++ */ ++ work_start_time_gpu = time_now; ++ ++ /* Start a new period if stoped */ ++ is_resume = mali_control_timer_resume(time_now); ++ ++ mali_utilization_data_unlock(); ++ ++ if (is_resume) { ++#if defined(CONFIG_MALI_DVFS) ++ /* Clear session->number_of_window_jobs, prepare parameter for dvfs */ ++ mali_session_max_window_num(); ++ if (0 == last_utilization_gpu) { ++ /* ++ * for mali_dev_pause is called in set clock, ++ * so each time we change clock, we will set clock to ++ * highest step even if under down clock case, ++ * it is not nessesary, so we only set the clock under ++ * last time utilization equal 0, we stop the timer then ++ * start the GPU again case ++ */ ++ mali_dvfs_policy_new_period(); ++ } ++#endif ++ ++ /* ++ * First timeout using short interval for power consideration ++ * because we give full power in the new period, but if the ++ * job loading is light, finish in 10ms, the other time all keep ++ * in high freq it will wast time. ++ */ ++ mali_control_timer_add(mali_control_first_timeout); ++ } ++ } else { ++ mali_utilization_data_unlock(); ++ } ++ } else { ++ /* Nothing to do */ ++ mali_utilization_data_unlock(); ++ } ++} ++ ++void mali_utilization_gp_end(void) ++{ ++ mali_utilization_data_lock(); ++ ++ --num_running_gp_cores; ++ if (0 == num_running_gp_cores) { ++ u64 time_now = _mali_osk_time_get_ns(); ++ ++ /* Last GP core ended, consider GP idle from now and onwards */ ++ accumulated_work_time_gp += (time_now - work_start_time_gp); ++ work_start_time_gp = 0; ++ ++ if (0 == num_running_pp_cores) { ++ /* ++ * There are no PP cores running, so this is also the point ++ * at which we consider the GPU to be idle as well. ++ */ ++ accumulated_work_time_gpu += (time_now - work_start_time_gpu); ++ work_start_time_gpu = 0; ++ } ++ } ++ ++ mali_utilization_data_unlock(); ++} ++ ++void mali_utilization_pp_end(void) ++{ ++ mali_utilization_data_lock(); ++ ++ --num_running_pp_cores; ++ if (0 == num_running_pp_cores) { ++ u64 time_now = _mali_osk_time_get_ns(); ++ ++ /* Last PP core ended, consider PP idle from now and onwards */ ++ accumulated_work_time_pp += (time_now - work_start_time_pp); ++ work_start_time_pp = 0; ++ ++ if (0 == num_running_gp_cores) { ++ /* ++ * There are no GP cores running, so this is also the point ++ * at which we consider the GPU to be idle as well. ++ */ ++ accumulated_work_time_gpu += (time_now - work_start_time_gpu); ++ work_start_time_gpu = 0; ++ } ++ } ++ ++ mali_utilization_data_unlock(); ++} ++ ++mali_bool mali_utilization_enabled(void) ++{ ++#if defined(CONFIG_MALI_DVFS) ++ return mali_dvfs_policy_enabled(); ++#else ++ return (NULL != mali_utilization_callback); ++#endif /* defined(CONFIG_MALI_DVFS) */ ++} ++ ++void mali_utilization_platform_realize(struct mali_gpu_utilization_data *util_data) ++{ ++ MALI_DEBUG_ASSERT_POINTER(mali_utilization_callback); ++ ++ mali_utilization_callback(util_data); ++} ++ ++void mali_utilization_reset(void) ++{ ++ accumulated_work_time_gpu = 0; ++ accumulated_work_time_gp = 0; ++ accumulated_work_time_pp = 0; ++ ++ last_utilization_gpu = 0; ++ last_utilization_gp = 0; ++ last_utilization_pp = 0; ++} ++ ++void mali_utilization_data_lock(void) ++{ ++ _mali_osk_spinlock_irq_lock(utilization_data_lock); ++} ++ ++void mali_utilization_data_unlock(void) ++{ ++ _mali_osk_spinlock_irq_unlock(utilization_data_lock); ++} ++ ++void mali_utilization_data_assert_locked(void) ++{ ++ MALI_DEBUG_ASSERT_LOCK_HELD(utilization_data_lock); ++} ++ ++u32 _mali_ukk_utilization_gp_pp(void) ++{ ++ return last_utilization_gpu; ++} ++ ++u32 _mali_ukk_utilization_gp(void) ++{ ++ return last_utilization_gp; ++} ++ ++u32 _mali_ukk_utilization_pp(void) ++{ ++ return last_utilization_pp; ++} +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_kernel_utilization.h b/drivers/gpu/arm/mali400/mali/common/mali_kernel_utilization.h +new file mode 100755 +index 000000000000..06f585dcb238 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_kernel_utilization.h +@@ -0,0 +1,72 @@ ++/* ++ * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_KERNEL_UTILIZATION_H__ ++#define __MALI_KERNEL_UTILIZATION_H__ ++ ++#include ++#include "mali_osk.h" ++ ++/** ++ * Initialize/start the Mali GPU utilization metrics reporting. ++ * ++ * @return _MALI_OSK_ERR_OK on success, otherwise failure. ++ */ ++_mali_osk_errcode_t mali_utilization_init(void); ++ ++/** ++ * Terminate the Mali GPU utilization metrics reporting ++ */ ++void mali_utilization_term(void); ++ ++/** ++ * Check if Mali utilization is enabled ++ */ ++mali_bool mali_utilization_enabled(void); ++ ++/** ++ * Should be called when a job is about to execute a GP job ++ */ ++void mali_utilization_gp_start(void); ++ ++/** ++ * Should be called when a job has completed executing a GP job ++ */ ++void mali_utilization_gp_end(void); ++ ++/** ++ * Should be called when a job is about to execute a PP job ++ */ ++void mali_utilization_pp_start(void); ++ ++/** ++ * Should be called when a job has completed executing a PP job ++ */ ++void mali_utilization_pp_end(void); ++ ++/** ++ * Should be called to calcution the GPU utilization ++ */ ++struct mali_gpu_utilization_data *mali_utilization_calculate(u64 *start_time, u64 *time_period, mali_bool *need_add_timer); ++ ++_mali_osk_spinlock_irq_t *mali_utilization_get_lock(void); ++ ++void mali_utilization_platform_realize(struct mali_gpu_utilization_data *util_data); ++ ++void mali_utilization_data_lock(void); ++ ++void mali_utilization_data_unlock(void); ++ ++void mali_utilization_data_assert_locked(void); ++ ++void mali_utilization_reset(void); ++ ++ ++#endif /* __MALI_KERNEL_UTILIZATION_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_kernel_vsync.c b/drivers/gpu/arm/mali400/mali/common/mali_kernel_vsync.c +new file mode 100755 +index 000000000000..dd44e5e7fa03 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_kernel_vsync.c +@@ -0,0 +1,45 @@ ++/* ++ * Copyright (C) 2011-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include "mali_kernel_common.h" ++#include "mali_osk.h" ++#include "mali_ukk.h" ++ ++#include "mali_osk_profiling.h" ++ ++_mali_osk_errcode_t _mali_ukk_vsync_event_report(_mali_uk_vsync_event_report_s *args) ++{ ++ _mali_uk_vsync_event event = (_mali_uk_vsync_event)args->event; ++ MALI_IGNORE(event); /* event is not used for release code, and that is OK */ ++ ++ /* ++ * Manually generate user space events in kernel space. ++ * This saves user space from calling kernel space twice in this case. ++ * We just need to remember to add pid and tid manually. ++ */ ++ if (event == _MALI_UK_VSYNC_EVENT_BEGIN_WAIT) { ++ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SUSPEND | ++ MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | ++ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_VSYNC, ++ _mali_osk_get_pid(), _mali_osk_get_tid(), 0, 0, 0); ++ } ++ ++ if (event == _MALI_UK_VSYNC_EVENT_END_WAIT) { ++ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_RESUME | ++ MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | ++ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_VSYNC, ++ _mali_osk_get_pid(), _mali_osk_get_tid(), 0, 0, 0); ++ } ++ ++ ++ MALI_DEBUG_PRINT(4, ("Received VSYNC event: %d\n", event)); ++ MALI_SUCCESS; ++} ++ +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_l2_cache.c b/drivers/gpu/arm/mali400/mali/common/mali_l2_cache.c +new file mode 100755 +index 000000000000..fe33f561b2aa +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_l2_cache.c +@@ -0,0 +1,534 @@ ++/* ++ * Copyright (C) 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++#include "mali_kernel_common.h" ++#include "mali_osk.h" ++#include "mali_l2_cache.h" ++#include "mali_hw_core.h" ++#include "mali_scheduler.h" ++#include "mali_pm.h" ++#include "mali_pm_domain.h" ++ ++/** ++ * Size of the Mali L2 cache registers in bytes ++ */ ++#define MALI400_L2_CACHE_REGISTERS_SIZE 0x30 ++ ++/** ++ * Mali L2 cache register numbers ++ * Used in the register read/write routines. ++ * See the hardware documentation for more information about each register ++ */ ++typedef enum mali_l2_cache_register { ++ MALI400_L2_CACHE_REGISTER_SIZE = 0x0004, ++ MALI400_L2_CACHE_REGISTER_STATUS = 0x0008, ++ /*unused = 0x000C */ ++ MALI400_L2_CACHE_REGISTER_COMMAND = 0x0010, ++ MALI400_L2_CACHE_REGISTER_CLEAR_PAGE = 0x0014, ++ MALI400_L2_CACHE_REGISTER_MAX_READS = 0x0018, ++ MALI400_L2_CACHE_REGISTER_ENABLE = 0x001C, ++ MALI400_L2_CACHE_REGISTER_PERFCNT_SRC0 = 0x0020, ++ MALI400_L2_CACHE_REGISTER_PERFCNT_VAL0 = 0x0024, ++ MALI400_L2_CACHE_REGISTER_PERFCNT_SRC1 = 0x0028, ++ MALI400_L2_CACHE_REGISTER_PERFCNT_VAL1 = 0x002C, ++} mali_l2_cache_register; ++ ++/** ++ * Mali L2 cache commands ++ * These are the commands that can be sent to the Mali L2 cache unit ++ */ ++typedef enum mali_l2_cache_command { ++ MALI400_L2_CACHE_COMMAND_CLEAR_ALL = 0x01, ++} mali_l2_cache_command; ++ ++/** ++ * Mali L2 cache commands ++ * These are the commands that can be sent to the Mali L2 cache unit ++ */ ++typedef enum mali_l2_cache_enable { ++ MALI400_L2_CACHE_ENABLE_DEFAULT = 0x0, /* Default */ ++ MALI400_L2_CACHE_ENABLE_ACCESS = 0x01, ++ MALI400_L2_CACHE_ENABLE_READ_ALLOCATE = 0x02, ++} mali_l2_cache_enable; ++ ++/** ++ * Mali L2 cache status bits ++ */ ++typedef enum mali_l2_cache_status { ++ MALI400_L2_CACHE_STATUS_COMMAND_BUSY = 0x01, ++ MALI400_L2_CACHE_STATUS_DATA_BUSY = 0x02, ++} mali_l2_cache_status; ++ ++#define MALI400_L2_MAX_READS_NOT_SET -1 ++ ++static struct mali_l2_cache_core * ++ mali_global_l2s[MALI_MAX_NUMBER_OF_L2_CACHE_CORES] = { NULL, }; ++static u32 mali_global_num_l2s = 0; ++ ++int mali_l2_max_reads = MALI400_L2_MAX_READS_NOT_SET; ++ ++ ++/* Local helper functions */ ++ ++static void mali_l2_cache_reset(struct mali_l2_cache_core *cache); ++ ++static _mali_osk_errcode_t mali_l2_cache_send_command( ++ struct mali_l2_cache_core *cache, u32 reg, u32 val); ++ ++static void mali_l2_cache_lock(struct mali_l2_cache_core *cache) ++{ ++ MALI_DEBUG_ASSERT_POINTER(cache); ++ _mali_osk_spinlock_irq_lock(cache->lock); ++} ++ ++static void mali_l2_cache_unlock(struct mali_l2_cache_core *cache) ++{ ++ MALI_DEBUG_ASSERT_POINTER(cache); ++ _mali_osk_spinlock_irq_unlock(cache->lock); ++} ++ ++/* Implementation of the L2 cache interface */ ++ ++struct mali_l2_cache_core *mali_l2_cache_create( ++ _mali_osk_resource_t *resource, u32 domain_index) ++{ ++ struct mali_l2_cache_core *cache = NULL; ++#if defined(DEBUG) ++ u32 cache_size; ++#endif ++ ++ MALI_DEBUG_PRINT(4, ("Mali L2 cache: Creating Mali L2 cache: %s\n", ++ resource->description)); ++ ++ if (mali_global_num_l2s >= MALI_MAX_NUMBER_OF_L2_CACHE_CORES) { ++ MALI_PRINT_ERROR(("Mali L2 cache: Too many L2 caches\n")); ++ return NULL; ++ } ++ ++ cache = _mali_osk_malloc(sizeof(struct mali_l2_cache_core)); ++ if (NULL == cache) { ++ MALI_PRINT_ERROR(("Mali L2 cache: Failed to allocate memory for L2 cache core\n")); ++ return NULL; ++ } ++ ++ cache->core_id = mali_global_num_l2s; ++ cache->counter_src0 = MALI_HW_CORE_NO_COUNTER; ++ cache->counter_src1 = MALI_HW_CORE_NO_COUNTER; ++ cache->counter_value0_base = 0; ++ cache->counter_value1_base = 0; ++ cache->pm_domain = NULL; ++ cache->power_is_on = MALI_FALSE; ++ cache->last_invalidated_id = 0; ++ ++ if (_MALI_OSK_ERR_OK != mali_hw_core_create(&cache->hw_core, ++ resource, MALI400_L2_CACHE_REGISTERS_SIZE)) { ++ _mali_osk_free(cache); ++ return NULL; ++ } ++ ++#if defined(DEBUG) ++ cache_size = mali_hw_core_register_read(&cache->hw_core, ++ MALI400_L2_CACHE_REGISTER_SIZE); ++ MALI_DEBUG_PRINT(2, ("Mali L2 cache: Created %s: % 3uK, %u-way, % 2ubyte cache line, % 3ubit external bus\n", ++ resource->description, ++ 1 << (((cache_size >> 16) & 0xff) - 10), ++ 1 << ((cache_size >> 8) & 0xff), ++ 1 << (cache_size & 0xff), ++ 1 << ((cache_size >> 24) & 0xff))); ++#endif ++ ++ cache->lock = _mali_osk_spinlock_irq_init(_MALI_OSK_LOCKFLAG_ORDERED, ++ _MALI_OSK_LOCK_ORDER_L2); ++ if (NULL == cache->lock) { ++ MALI_PRINT_ERROR(("Mali L2 cache: Failed to create counter lock for L2 cache core %s\n", ++ cache->hw_core.description)); ++ mali_hw_core_delete(&cache->hw_core); ++ _mali_osk_free(cache); ++ return NULL; ++ } ++ ++ /* register with correct power domain */ ++ cache->pm_domain = mali_pm_register_l2_cache( ++ domain_index, cache); ++ ++ mali_global_l2s[mali_global_num_l2s] = cache; ++ mali_global_num_l2s++; ++ ++ return cache; ++} ++ ++void mali_l2_cache_delete(struct mali_l2_cache_core *cache) ++{ ++ u32 i; ++ for (i = 0; i < mali_global_num_l2s; i++) { ++ if (mali_global_l2s[i] != cache) { ++ continue; ++ } ++ ++ mali_global_l2s[i] = NULL; ++ mali_global_num_l2s--; ++ ++ if (i == mali_global_num_l2s) { ++ /* Removed last element, nothing more to do */ ++ break; ++ } ++ ++ /* ++ * We removed a l2 cache from the middle of the array, ++ * so move the last l2 cache to current position ++ */ ++ mali_global_l2s[i] = mali_global_l2s[mali_global_num_l2s]; ++ mali_global_l2s[mali_global_num_l2s] = NULL; ++ ++ /* All good */ ++ break; ++ } ++ ++ _mali_osk_spinlock_irq_term(cache->lock); ++ mali_hw_core_delete(&cache->hw_core); ++ _mali_osk_free(cache); ++} ++ ++void mali_l2_cache_power_up(struct mali_l2_cache_core *cache) ++{ ++ MALI_DEBUG_ASSERT_POINTER(cache); ++ ++ mali_l2_cache_lock(cache); ++ ++ mali_l2_cache_reset(cache); ++ ++ if ((1 << MALI_DOMAIN_INDEX_DUMMY) != cache->pm_domain->pmu_mask) ++ MALI_DEBUG_ASSERT(MALI_FALSE == cache->power_is_on); ++ cache->power_is_on = MALI_TRUE; ++ ++ mali_l2_cache_unlock(cache); ++} ++ ++void mali_l2_cache_power_down(struct mali_l2_cache_core *cache) ++{ ++ MALI_DEBUG_ASSERT_POINTER(cache); ++ ++ mali_l2_cache_lock(cache); ++ ++ MALI_DEBUG_ASSERT(MALI_TRUE == cache->power_is_on); ++ ++ /* ++ * The HW counters will start from zero again when we resume, ++ * but we should report counters as always increasing. ++ * Take a copy of the HW values now in order to add this to ++ * the values we report after being powered up. ++ * ++ * The physical power off of the L2 cache might be outside our ++ * own control (e.g. runtime PM). That is why we must manually ++ * set set the counter value to zero as well. ++ */ ++ ++ if (cache->counter_src0 != MALI_HW_CORE_NO_COUNTER) { ++ cache->counter_value0_base += mali_hw_core_register_read( ++ &cache->hw_core, ++ MALI400_L2_CACHE_REGISTER_PERFCNT_VAL0); ++ mali_hw_core_register_write(&cache->hw_core, ++ MALI400_L2_CACHE_REGISTER_PERFCNT_VAL0, 0); ++ } ++ ++ if (cache->counter_src1 != MALI_HW_CORE_NO_COUNTER) { ++ cache->counter_value1_base += mali_hw_core_register_read( ++ &cache->hw_core, ++ MALI400_L2_CACHE_REGISTER_PERFCNT_VAL1); ++ mali_hw_core_register_write(&cache->hw_core, ++ MALI400_L2_CACHE_REGISTER_PERFCNT_VAL1, 0); ++ } ++ ++ ++ cache->power_is_on = MALI_FALSE; ++ ++ mali_l2_cache_unlock(cache); ++} ++ ++void mali_l2_cache_core_set_counter_src( ++ struct mali_l2_cache_core *cache, u32 source_id, u32 counter) ++{ ++ u32 reg_offset_src; ++ u32 reg_offset_val; ++ ++ MALI_DEBUG_ASSERT_POINTER(cache); ++ MALI_DEBUG_ASSERT(source_id >= 0 && source_id <= 1); ++ ++ mali_l2_cache_lock(cache); ++ ++ if (0 == source_id) { ++ /* start counting from 0 */ ++ cache->counter_value0_base = 0; ++ cache->counter_src0 = counter; ++ reg_offset_src = MALI400_L2_CACHE_REGISTER_PERFCNT_SRC0; ++ reg_offset_val = MALI400_L2_CACHE_REGISTER_PERFCNT_VAL0; ++ } else { ++ /* start counting from 0 */ ++ cache->counter_value1_base = 0; ++ cache->counter_src1 = counter; ++ reg_offset_src = MALI400_L2_CACHE_REGISTER_PERFCNT_SRC1; ++ reg_offset_val = MALI400_L2_CACHE_REGISTER_PERFCNT_VAL1; ++ } ++ ++ if (cache->power_is_on) { ++ u32 hw_src; ++ ++ if (MALI_HW_CORE_NO_COUNTER != counter) { ++ hw_src = counter; ++ } else { ++ hw_src = 0; /* disable value for HW */ ++ } ++ ++ /* Set counter src */ ++ mali_hw_core_register_write(&cache->hw_core, ++ reg_offset_src, hw_src); ++ ++ /* Make sure the HW starts counting from 0 again */ ++ mali_hw_core_register_write(&cache->hw_core, ++ reg_offset_val, 0); ++ } ++ ++ mali_l2_cache_unlock(cache); ++} ++ ++void mali_l2_cache_core_get_counter_values( ++ struct mali_l2_cache_core *cache, ++ u32 *src0, u32 *value0, u32 *src1, u32 *value1) ++{ ++ MALI_DEBUG_ASSERT_POINTER(cache); ++ MALI_DEBUG_ASSERT(NULL != src0); ++ MALI_DEBUG_ASSERT(NULL != value0); ++ MALI_DEBUG_ASSERT(NULL != src1); ++ MALI_DEBUG_ASSERT(NULL != value1); ++ ++ mali_l2_cache_lock(cache); ++ ++ *src0 = cache->counter_src0; ++ *src1 = cache->counter_src1; ++ ++ if (cache->counter_src0 != MALI_HW_CORE_NO_COUNTER) { ++ if (MALI_TRUE == cache->power_is_on) { ++ *value0 = mali_hw_core_register_read(&cache->hw_core, ++ MALI400_L2_CACHE_REGISTER_PERFCNT_VAL0); ++ } else { ++ *value0 = 0; ++ } ++ ++ /* Add base offset value (in case we have been power off) */ ++ *value0 += cache->counter_value0_base; ++ } ++ ++ if (cache->counter_src1 != MALI_HW_CORE_NO_COUNTER) { ++ if (MALI_TRUE == cache->power_is_on) { ++ *value1 = mali_hw_core_register_read(&cache->hw_core, ++ MALI400_L2_CACHE_REGISTER_PERFCNT_VAL1); ++ } else { ++ *value1 = 0; ++ } ++ ++ /* Add base offset value (in case we have been power off) */ ++ *value1 += cache->counter_value1_base; ++ } ++ ++ mali_l2_cache_unlock(cache); ++} ++ ++struct mali_l2_cache_core *mali_l2_cache_core_get_glob_l2_core(u32 index) ++{ ++ if (mali_global_num_l2s > index) { ++ return mali_global_l2s[index]; ++ } ++ ++ return NULL; ++} ++ ++u32 mali_l2_cache_core_get_glob_num_l2_cores(void) ++{ ++ return mali_global_num_l2s; ++} ++ ++void mali_l2_cache_invalidate(struct mali_l2_cache_core *cache) ++{ ++ MALI_DEBUG_ASSERT_POINTER(cache); ++ ++ if (NULL == cache) { ++ return; ++ } ++ ++ mali_l2_cache_lock(cache); ++ ++ cache->last_invalidated_id = mali_scheduler_get_new_cache_order(); ++ mali_l2_cache_send_command(cache, MALI400_L2_CACHE_REGISTER_COMMAND, ++ MALI400_L2_CACHE_COMMAND_CLEAR_ALL); ++ ++ mali_l2_cache_unlock(cache); ++} ++ ++void mali_l2_cache_invalidate_conditional( ++ struct mali_l2_cache_core *cache, u32 id) ++{ ++ MALI_DEBUG_ASSERT_POINTER(cache); ++ ++ if (NULL == cache) { ++ return; ++ } ++ ++ /* ++ * If the last cache invalidation was done by a job with a higher id we ++ * don't have to flush. Since user space will store jobs w/ their ++ * corresponding memory in sequence (first job #0, then job #1, ...), ++ * we don't have to flush for job n-1 if job n has already invalidated ++ * the cache since we know for sure that job n-1's memory was already ++ * written when job n was started. ++ */ ++ ++ mali_l2_cache_lock(cache); ++ ++ if (((s32)id) > ((s32)cache->last_invalidated_id)) { ++ /* Set latest invalidated id to current "point in time" */ ++ cache->last_invalidated_id = ++ mali_scheduler_get_new_cache_order(); ++ mali_l2_cache_send_command(cache, ++ MALI400_L2_CACHE_REGISTER_COMMAND, ++ MALI400_L2_CACHE_COMMAND_CLEAR_ALL); ++ } ++ ++ mali_l2_cache_unlock(cache); ++} ++ ++void mali_l2_cache_invalidate_all(void) ++{ ++ u32 i; ++ for (i = 0; i < mali_global_num_l2s; i++) { ++ struct mali_l2_cache_core *cache = mali_global_l2s[i]; ++ _mali_osk_errcode_t ret; ++ ++ MALI_DEBUG_ASSERT_POINTER(cache); ++ ++ mali_l2_cache_lock(cache); ++ ++ if (MALI_TRUE != cache->power_is_on) { ++ mali_l2_cache_unlock(cache); ++ continue; ++ } ++ ++ cache->last_invalidated_id = ++ mali_scheduler_get_new_cache_order(); ++ ++ ret = mali_l2_cache_send_command(cache, ++ MALI400_L2_CACHE_REGISTER_COMMAND, ++ MALI400_L2_CACHE_COMMAND_CLEAR_ALL); ++ if (_MALI_OSK_ERR_OK != ret) { ++ MALI_PRINT_ERROR(("Failed to invalidate cache\n")); ++ } ++ ++ mali_l2_cache_unlock(cache); ++ } ++} ++ ++void mali_l2_cache_invalidate_all_pages(u32 *pages, u32 num_pages) ++{ ++ u32 i; ++ for (i = 0; i < mali_global_num_l2s; i++) { ++ struct mali_l2_cache_core *cache = mali_global_l2s[i]; ++ u32 j; ++ ++ MALI_DEBUG_ASSERT_POINTER(cache); ++ ++ mali_l2_cache_lock(cache); ++ ++ if (MALI_TRUE != cache->power_is_on) { ++ mali_l2_cache_unlock(cache); ++ continue; ++ } ++ ++ for (j = 0; j < num_pages; j++) { ++ _mali_osk_errcode_t ret; ++ ++ ret = mali_l2_cache_send_command(cache, ++ MALI400_L2_CACHE_REGISTER_CLEAR_PAGE, ++ pages[j]); ++ if (_MALI_OSK_ERR_OK != ret) { ++ MALI_PRINT_ERROR(("Failed to invalidate cache (page)\n")); ++ } ++ } ++ ++ mali_l2_cache_unlock(cache); ++ } ++} ++ ++/* -------- local helper functions below -------- */ ++ ++static void mali_l2_cache_reset(struct mali_l2_cache_core *cache) ++{ ++ MALI_DEBUG_ASSERT_POINTER(cache); ++ MALI_DEBUG_ASSERT_LOCK_HELD(cache->lock); ++ ++ /* Invalidate cache (just to keep it in a known state at startup) */ ++ mali_l2_cache_send_command(cache, MALI400_L2_CACHE_REGISTER_COMMAND, ++ MALI400_L2_CACHE_COMMAND_CLEAR_ALL); ++ ++ /* Enable cache */ ++ mali_hw_core_register_write(&cache->hw_core, ++ MALI400_L2_CACHE_REGISTER_ENABLE, ++ (u32)MALI400_L2_CACHE_ENABLE_ACCESS | ++ (u32)MALI400_L2_CACHE_ENABLE_READ_ALLOCATE); ++ ++ if (MALI400_L2_MAX_READS_NOT_SET != mali_l2_max_reads) { ++ mali_hw_core_register_write(&cache->hw_core, ++ MALI400_L2_CACHE_REGISTER_MAX_READS, ++ (u32)mali_l2_max_reads); ++ } ++ ++ /* Restart any performance counters (if enabled) */ ++ if (cache->counter_src0 != MALI_HW_CORE_NO_COUNTER) { ++ ++ mali_hw_core_register_write(&cache->hw_core, ++ MALI400_L2_CACHE_REGISTER_PERFCNT_SRC0, ++ cache->counter_src0); ++ } ++ ++ if (cache->counter_src1 != MALI_HW_CORE_NO_COUNTER) { ++ mali_hw_core_register_write(&cache->hw_core, ++ MALI400_L2_CACHE_REGISTER_PERFCNT_SRC1, ++ cache->counter_src1); ++ } ++} ++ ++static _mali_osk_errcode_t mali_l2_cache_send_command( ++ struct mali_l2_cache_core *cache, u32 reg, u32 val) ++{ ++ int i = 0; ++ const int loop_count = 100000; ++ ++ MALI_DEBUG_ASSERT_POINTER(cache); ++ MALI_DEBUG_ASSERT_LOCK_HELD(cache->lock); ++ ++ /* ++ * First, wait for L2 cache command handler to go idle. ++ * (Commands received while processing another command will be ignored) ++ */ ++ for (i = 0; i < loop_count; i++) { ++ if (!(mali_hw_core_register_read(&cache->hw_core, ++ MALI400_L2_CACHE_REGISTER_STATUS) & ++ (u32)MALI400_L2_CACHE_STATUS_COMMAND_BUSY)) { ++ break; ++ } ++ } ++ ++ if (i == loop_count) { ++ MALI_DEBUG_PRINT(1, ("Mali L2 cache: aborting wait for command interface to go idle\n")); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ /* then issue the command */ ++ mali_hw_core_register_write(&cache->hw_core, reg, val); ++ ++ return _MALI_OSK_ERR_OK; ++} +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_l2_cache.h b/drivers/gpu/arm/mali400/mali/common/mali_l2_cache.h +new file mode 100755 +index 000000000000..c48a8844075f +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_l2_cache.h +@@ -0,0 +1,124 @@ ++/* ++ * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_KERNEL_L2_CACHE_H__ ++#define __MALI_KERNEL_L2_CACHE_H__ ++ ++#include "mali_osk.h" ++#include "mali_hw_core.h" ++ ++#define MALI_MAX_NUMBER_OF_L2_CACHE_CORES 3 ++/* Maximum 1 GP and 4 PP for an L2 cache core (Mali-400 MP4) */ ++#define MALI_MAX_NUMBER_OF_GROUPS_PER_L2_CACHE 5 ++ ++/** ++ * Definition of the L2 cache core struct ++ * Used to track a L2 cache unit in the system. ++ * Contains information about the mapping of the registers ++ */ ++struct mali_l2_cache_core { ++ /* Common HW core functionality */ ++ struct mali_hw_core hw_core; ++ ++ /* Synchronize L2 cache access */ ++ _mali_osk_spinlock_irq_t *lock; ++ ++ /* Unique core ID */ ++ u32 core_id; ++ ++ /* The power domain this L2 cache belongs to */ ++ struct mali_pm_domain *pm_domain; ++ ++ /* MALI_TRUE if power is on for this L2 cache */ ++ mali_bool power_is_on; ++ ++ /* A "timestamp" to avoid unnecessary flushes */ ++ u32 last_invalidated_id; ++ ++ /* Performance counter 0, MALI_HW_CORE_NO_COUNTER for disabled */ ++ u32 counter_src0; ++ ++ /* Performance counter 1, MALI_HW_CORE_NO_COUNTER for disabled */ ++ u32 counter_src1; ++ ++ /* ++ * Performance counter 0 value base/offset ++ * (allows accumulative reporting even after power off) ++ */ ++ u32 counter_value0_base; ++ ++ /* ++ * Performance counter 0 value base/offset ++ * (allows accumulative reporting even after power off) ++ */ ++ u32 counter_value1_base; ++ ++ /* Used by PM domains to link L2 caches of same domain */ ++ _mali_osk_list_t pm_domain_list; ++}; ++ ++_mali_osk_errcode_t mali_l2_cache_initialize(void); ++void mali_l2_cache_terminate(void); ++ ++struct mali_l2_cache_core *mali_l2_cache_create( ++ _mali_osk_resource_t *resource, u32 domain_index); ++void mali_l2_cache_delete(struct mali_l2_cache_core *cache); ++ ++MALI_STATIC_INLINE u32 mali_l2_cache_get_id(struct mali_l2_cache_core *cache) ++{ ++ MALI_DEBUG_ASSERT_POINTER(cache); ++ return cache->core_id; ++} ++ ++MALI_STATIC_INLINE struct mali_pm_domain *mali_l2_cache_get_pm_domain( ++ struct mali_l2_cache_core *cache) ++{ ++ MALI_DEBUG_ASSERT_POINTER(cache); ++ return cache->pm_domain; ++} ++ ++void mali_l2_cache_power_up(struct mali_l2_cache_core *cache); ++void mali_l2_cache_power_down(struct mali_l2_cache_core *cache); ++ ++void mali_l2_cache_core_set_counter_src( ++ struct mali_l2_cache_core *cache, u32 source_id, u32 counter); ++ ++MALI_STATIC_INLINE u32 mali_l2_cache_core_get_counter_src0( ++ struct mali_l2_cache_core *cache) ++{ ++ MALI_DEBUG_ASSERT_POINTER(cache); ++ return cache->counter_src0; ++} ++ ++MALI_STATIC_INLINE u32 mali_l2_cache_core_get_counter_src1( ++ struct mali_l2_cache_core *cache) ++{ ++ MALI_DEBUG_ASSERT_POINTER(cache); ++ return cache->counter_src1; ++} ++ ++void mali_l2_cache_core_get_counter_values( ++ struct mali_l2_cache_core *cache, ++ u32 *src0, u32 *value0, u32 *src1, u32 *value1); ++ ++struct mali_l2_cache_core *mali_l2_cache_core_get_glob_l2_core(u32 index); ++u32 mali_l2_cache_core_get_glob_num_l2_cores(void); ++ ++struct mali_group *mali_l2_cache_get_group( ++ struct mali_l2_cache_core *cache, u32 index); ++ ++void mali_l2_cache_invalidate(struct mali_l2_cache_core *cache); ++void mali_l2_cache_invalidate_conditional( ++ struct mali_l2_cache_core *cache, u32 id); ++ ++void mali_l2_cache_invalidate_all(void); ++void mali_l2_cache_invalidate_all_pages(u32 *pages, u32 num_pages); ++ ++#endif /* __MALI_KERNEL_L2_CACHE_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_mem_validation.c b/drivers/gpu/arm/mali400/mali/common/mali_mem_validation.c +new file mode 100755 +index 000000000000..eb95998f1469 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_mem_validation.c +@@ -0,0 +1,68 @@ ++/* ++ * Copyright (C) 2011-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include "mali_mem_validation.h" ++#include "mali_osk.h" ++#include "mali_kernel_common.h" ++ ++#define MALI_INVALID_MEM_ADDR 0xFFFFFFFF ++ ++typedef struct { ++ u32 phys_base; /**< Mali physical base of the memory, page aligned */ ++ u32 size; /**< size in bytes of the memory, multiple of page size */ ++} _mali_mem_validation_t; ++ ++static _mali_mem_validation_t mali_mem_validator = { MALI_INVALID_MEM_ADDR, MALI_INVALID_MEM_ADDR }; ++ ++_mali_osk_errcode_t mali_mem_validation_add_range(u32 start, u32 size) ++{ ++ /* Check that no other MEM_VALIDATION resources exist */ ++ if (MALI_INVALID_MEM_ADDR != mali_mem_validator.phys_base) { ++ MALI_PRINT_ERROR(("Failed to add frame buffer memory; another range is already specified\n")); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ /* Check restrictions on page alignment */ ++ if ((0 != (start & (~_MALI_OSK_CPU_PAGE_MASK))) || ++ (0 != (size & (~_MALI_OSK_CPU_PAGE_MASK)))) { ++ MALI_PRINT_ERROR(("Failed to add frame buffer memory; incorrect alignment\n")); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ mali_mem_validator.phys_base = start; ++ mali_mem_validator.size = size; ++ MALI_DEBUG_PRINT(2, ("Memory Validator installed for Mali physical address base=0x%08X, size=0x%08X\n", ++ mali_mem_validator.phys_base, mali_mem_validator.size)); ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++_mali_osk_errcode_t mali_mem_validation_check(u32 phys_addr, u32 size) ++{ ++#if 0 ++ if (phys_addr < (phys_addr + size)) { /* Don't allow overflow (or zero size) */ ++ if ((0 == (phys_addr & (~_MALI_OSK_CPU_PAGE_MASK))) && ++ (0 == (size & (~_MALI_OSK_CPU_PAGE_MASK)))) { ++ if ((phys_addr >= mali_mem_validator.phys_base) && ++ ((phys_addr + (size - 1)) >= mali_mem_validator.phys_base) && ++ (phys_addr <= (mali_mem_validator.phys_base + (mali_mem_validator.size - 1))) && ++ ((phys_addr + (size - 1)) <= (mali_mem_validator.phys_base + (mali_mem_validator.size - 1)))) { ++ MALI_DEBUG_PRINT(3, ("Accepted range 0x%08X + size 0x%08X (= 0x%08X)\n", phys_addr, size, (phys_addr + size - 1))); ++ return _MALI_OSK_ERR_OK; ++ } ++ } ++ } ++ ++ MALI_PRINT_ERROR(("MALI PHYSICAL RANGE VALIDATION ERROR: The range supplied was: phys_base=0x%08X, size=0x%08X\n", phys_addr, size)); ++ ++ return _MALI_OSK_ERR_FAULT; ++#endif ++ return _MALI_OSK_ERR_OK; ++} +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_mem_validation.h b/drivers/gpu/arm/mali400/mali/common/mali_mem_validation.h +new file mode 100755 +index 000000000000..05013f46f901 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_mem_validation.h +@@ -0,0 +1,19 @@ ++/* ++ * Copyright (C) 2011-2013, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_MEM_VALIDATION_H__ ++#define __MALI_MEM_VALIDATION_H__ ++ ++#include "mali_osk.h" ++ ++_mali_osk_errcode_t mali_mem_validation_add_range(u32 start, u32 size); ++_mali_osk_errcode_t mali_mem_validation_check(u32 phys_addr, u32 size); ++ ++#endif /* __MALI_MEM_VALIDATION_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_mmu.c b/drivers/gpu/arm/mali400/mali/common/mali_mmu.c +new file mode 100755 +index 000000000000..b82486fa66c0 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_mmu.c +@@ -0,0 +1,433 @@ ++/* ++ * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include "mali_kernel_common.h" ++#include "mali_osk.h" ++#include "mali_osk_list.h" ++#include "mali_ukk.h" ++ ++#include "mali_mmu.h" ++#include "mali_hw_core.h" ++#include "mali_group.h" ++#include "mali_mmu_page_directory.h" ++ ++/** ++ * Size of the MMU registers in bytes ++ */ ++#define MALI_MMU_REGISTERS_SIZE 0x24 ++ ++/** ++ * MMU commands ++ * These are the commands that can be sent ++ * to the MMU unit. ++ */ ++typedef enum mali_mmu_command { ++ MALI_MMU_COMMAND_ENABLE_PAGING = 0x00, /**< Enable paging (memory translation) */ ++ MALI_MMU_COMMAND_DISABLE_PAGING = 0x01, /**< Disable paging (memory translation) */ ++ MALI_MMU_COMMAND_ENABLE_STALL = 0x02, /**< Enable stall on page fault */ ++ MALI_MMU_COMMAND_DISABLE_STALL = 0x03, /**< Disable stall on page fault */ ++ MALI_MMU_COMMAND_ZAP_CACHE = 0x04, /**< Zap the entire page table cache */ ++ MALI_MMU_COMMAND_PAGE_FAULT_DONE = 0x05, /**< Page fault processed */ ++ MALI_MMU_COMMAND_HARD_RESET = 0x06 /**< Reset the MMU back to power-on settings */ ++} mali_mmu_command; ++ ++static void mali_mmu_probe_trigger(void *data); ++static _mali_osk_errcode_t mali_mmu_probe_ack(void *data); ++ ++MALI_STATIC_INLINE _mali_osk_errcode_t mali_mmu_raw_reset(struct mali_mmu_core *mmu); ++ ++/* page fault queue flush helper pages ++ * note that the mapping pointers are currently unused outside of the initialization functions */ ++static mali_dma_addr mali_page_fault_flush_page_directory = MALI_INVALID_PAGE; ++static mali_io_address mali_page_fault_flush_page_directory_mapping = NULL; ++static mali_dma_addr mali_page_fault_flush_page_table = MALI_INVALID_PAGE; ++static mali_io_address mali_page_fault_flush_page_table_mapping = NULL; ++static mali_dma_addr mali_page_fault_flush_data_page = MALI_INVALID_PAGE; ++static mali_io_address mali_page_fault_flush_data_page_mapping = NULL; ++ ++/* an empty page directory (no address valid) which is active on any MMU not currently marked as in use */ ++static mali_dma_addr mali_empty_page_directory_phys = MALI_INVALID_PAGE; ++static mali_io_address mali_empty_page_directory_virt = NULL; ++ ++ ++_mali_osk_errcode_t mali_mmu_initialize(void) ++{ ++ /* allocate the helper pages */ ++ mali_empty_page_directory_phys = mali_allocate_empty_page(&mali_empty_page_directory_virt); ++ if (0 == mali_empty_page_directory_phys) { ++ MALI_DEBUG_PRINT_ERROR(("Mali MMU: Could not allocate empty page directory.\n")); ++ mali_empty_page_directory_phys = MALI_INVALID_PAGE; ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ ++ if (_MALI_OSK_ERR_OK != mali_create_fault_flush_pages(&mali_page_fault_flush_page_directory, ++ &mali_page_fault_flush_page_directory_mapping, ++ &mali_page_fault_flush_page_table, ++ &mali_page_fault_flush_page_table_mapping, ++ &mali_page_fault_flush_data_page, ++ &mali_page_fault_flush_data_page_mapping)) { ++ MALI_DEBUG_PRINT_ERROR(("Mali MMU: Could not allocate fault flush pages\n")); ++ mali_free_empty_page(mali_empty_page_directory_phys, mali_empty_page_directory_virt); ++ mali_empty_page_directory_phys = MALI_INVALID_PAGE; ++ mali_empty_page_directory_virt = NULL; ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++void mali_mmu_terminate(void) ++{ ++ MALI_DEBUG_PRINT(3, ("Mali MMU: terminating\n")); ++ ++ /* Free global helper pages */ ++ mali_free_empty_page(mali_empty_page_directory_phys, mali_empty_page_directory_virt); ++ mali_empty_page_directory_phys = MALI_INVALID_PAGE; ++ mali_empty_page_directory_virt = NULL; ++ ++ /* Free the page fault flush pages */ ++ mali_destroy_fault_flush_pages(&mali_page_fault_flush_page_directory, ++ &mali_page_fault_flush_page_directory_mapping, ++ &mali_page_fault_flush_page_table, ++ &mali_page_fault_flush_page_table_mapping, ++ &mali_page_fault_flush_data_page, ++ &mali_page_fault_flush_data_page_mapping); ++} ++ ++struct mali_mmu_core *mali_mmu_create(_mali_osk_resource_t *resource, struct mali_group *group, mali_bool is_virtual) ++{ ++ struct mali_mmu_core *mmu = NULL; ++ ++ MALI_DEBUG_ASSERT_POINTER(resource); ++ ++ MALI_DEBUG_PRINT(2, ("Mali MMU: Creating Mali MMU: %s\n", resource->description)); ++ ++ mmu = _mali_osk_calloc(1, sizeof(struct mali_mmu_core)); ++ if (NULL != mmu) { ++ if (_MALI_OSK_ERR_OK == mali_hw_core_create(&mmu->hw_core, resource, MALI_MMU_REGISTERS_SIZE)) { ++ if (_MALI_OSK_ERR_OK == mali_group_add_mmu_core(group, mmu)) { ++ if (is_virtual) { ++ /* Skip reset and IRQ setup for virtual MMU */ ++ return mmu; ++ } ++ ++ if (_MALI_OSK_ERR_OK == mali_mmu_reset(mmu)) { ++ /* Setup IRQ handlers (which will do IRQ probing if needed) */ ++ mmu->irq = _mali_osk_irq_init(resource->irq, ++ mali_group_upper_half_mmu, ++ group, ++ mali_mmu_probe_trigger, ++ mali_mmu_probe_ack, ++ mmu, ++ resource->description); ++ if (NULL != mmu->irq) { ++ return mmu; ++ } else { ++ MALI_PRINT_ERROR(("Mali MMU: Failed to setup interrupt handlers for MMU %s\n", mmu->hw_core.description)); ++ } ++ } ++ mali_group_remove_mmu_core(group); ++ } else { ++ MALI_PRINT_ERROR(("Mali MMU: Failed to add core %s to group\n", mmu->hw_core.description)); ++ } ++ mali_hw_core_delete(&mmu->hw_core); ++ } ++ ++ _mali_osk_free(mmu); ++ } else { ++ MALI_PRINT_ERROR(("Failed to allocate memory for MMU\n")); ++ } ++ ++ return NULL; ++} ++ ++void mali_mmu_delete(struct mali_mmu_core *mmu) ++{ ++ if (NULL != mmu->irq) { ++ _mali_osk_irq_term(mmu->irq); ++ } ++ ++ mali_hw_core_delete(&mmu->hw_core); ++ _mali_osk_free(mmu); ++} ++ ++static void mali_mmu_enable_paging(struct mali_mmu_core *mmu) ++{ ++ int i; ++ ++ mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_ENABLE_PAGING); ++ ++ for (i = 0; i < MALI_REG_POLL_COUNT_FAST; ++i) { ++ if (mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS) & MALI_MMU_STATUS_BIT_PAGING_ENABLED) { ++ break; ++ } ++ } ++ if (MALI_REG_POLL_COUNT_FAST == i) { ++ MALI_PRINT_ERROR(("Enable paging request failed, MMU status is 0x%08X\n", mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS))); ++ } ++} ++ ++/** ++ * Issues the enable stall command to the MMU and waits for HW to complete the request ++ * @param mmu The MMU to enable paging for ++ * @return MALI_TRUE if HW stall was successfully engaged, otherwise MALI_FALSE (req timed out) ++ */ ++static mali_bool mali_mmu_enable_stall(struct mali_mmu_core *mmu) ++{ ++ int i; ++ u32 mmu_status = mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS); ++ ++ if (0 == (mmu_status & MALI_MMU_STATUS_BIT_PAGING_ENABLED)) { ++ MALI_DEBUG_PRINT(4, ("MMU stall is implicit when Paging is not enabled.\n")); ++ return MALI_TRUE; ++ } ++ ++ if (mmu_status & MALI_MMU_STATUS_BIT_PAGE_FAULT_ACTIVE) { ++ MALI_DEBUG_PRINT(3, ("Aborting MMU stall request since it is in pagefault state.\n")); ++ return MALI_FALSE; ++ } ++ ++ mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_ENABLE_STALL); ++ ++ for (i = 0; i < MALI_REG_POLL_COUNT_FAST; ++i) { ++ mmu_status = mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS); ++ if (mmu_status & MALI_MMU_STATUS_BIT_PAGE_FAULT_ACTIVE) { ++ break; ++ } ++ if ((mmu_status & MALI_MMU_STATUS_BIT_STALL_ACTIVE) && (0 == (mmu_status & MALI_MMU_STATUS_BIT_STALL_NOT_ACTIVE))) { ++ break; ++ } ++ if (0 == (mmu_status & (MALI_MMU_STATUS_BIT_PAGING_ENABLED))) { ++ break; ++ } ++ } ++ if (MALI_REG_POLL_COUNT_FAST == i) { ++ MALI_DEBUG_PRINT(2, ("Enable stall request failed, MMU status is 0x%08X\n", mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS))); ++ return MALI_FALSE; ++ } ++ ++ if (mmu_status & MALI_MMU_STATUS_BIT_PAGE_FAULT_ACTIVE) { ++ MALI_DEBUG_PRINT(2, ("Aborting MMU stall request since it has a pagefault.\n")); ++ return MALI_FALSE; ++ } ++ ++ return MALI_TRUE; ++} ++ ++/** ++ * Issues the disable stall command to the MMU and waits for HW to complete the request ++ * @param mmu The MMU to enable paging for ++ */ ++static void mali_mmu_disable_stall(struct mali_mmu_core *mmu) ++{ ++ int i; ++ u32 mmu_status = mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS); ++ ++ if (0 == (mmu_status & MALI_MMU_STATUS_BIT_PAGING_ENABLED)) { ++ MALI_DEBUG_PRINT(3, ("MMU disable skipped since it was not enabled.\n")); ++ return; ++ } ++ if (mmu_status & MALI_MMU_STATUS_BIT_PAGE_FAULT_ACTIVE) { ++ MALI_DEBUG_PRINT(2, ("Aborting MMU disable stall request since it is in pagefault state.\n")); ++ return; ++ } ++ ++ mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_DISABLE_STALL); ++ ++ for (i = 0; i < MALI_REG_POLL_COUNT_FAST; ++i) { ++ u32 status = mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS); ++ if (0 == (status & MALI_MMU_STATUS_BIT_STALL_ACTIVE)) { ++ break; ++ } ++ if (status & MALI_MMU_STATUS_BIT_PAGE_FAULT_ACTIVE) { ++ break; ++ } ++ if (0 == (mmu_status & MALI_MMU_STATUS_BIT_PAGING_ENABLED)) { ++ break; ++ } ++ } ++ if (MALI_REG_POLL_COUNT_FAST == i) MALI_DEBUG_PRINT(1, ("Disable stall request failed, MMU status is 0x%08X\n", mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS))); ++} ++ ++void mali_mmu_page_fault_done(struct mali_mmu_core *mmu) ++{ ++ MALI_DEBUG_PRINT(4, ("Mali MMU: %s: Leaving page fault mode\n", mmu->hw_core.description)); ++ mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_PAGE_FAULT_DONE); ++} ++ ++MALI_STATIC_INLINE _mali_osk_errcode_t mali_mmu_raw_reset(struct mali_mmu_core *mmu) ++{ ++ int i; ++ ++ mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_DTE_ADDR, 0xCAFEBABE); ++ MALI_DEBUG_ASSERT(0xCAFEB000 == mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_DTE_ADDR)); ++ mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_HARD_RESET); ++ ++ for (i = 0; i < MALI_REG_POLL_COUNT_FAST; ++i) { ++ if (mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_DTE_ADDR) == 0) { ++ break; ++ } ++ } ++ if (MALI_REG_POLL_COUNT_FAST == i) { ++ MALI_PRINT_ERROR(("Reset request failed, MMU status is 0x%08X\n", mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS))); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++_mali_osk_errcode_t mali_mmu_reset(struct mali_mmu_core *mmu) ++{ ++ _mali_osk_errcode_t err = _MALI_OSK_ERR_FAULT; ++ mali_bool stall_success; ++ MALI_DEBUG_ASSERT_POINTER(mmu); ++ ++ stall_success = mali_mmu_enable_stall(mmu); ++ if (!stall_success) { ++ err = _MALI_OSK_ERR_BUSY; ++ } ++ ++ MALI_DEBUG_PRINT(3, ("Mali MMU: mali_kernel_mmu_reset: %s\n", mmu->hw_core.description)); ++ ++ if (_MALI_OSK_ERR_OK == mali_mmu_raw_reset(mmu)) { ++ mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_INT_MASK, MALI_MMU_INTERRUPT_PAGE_FAULT | MALI_MMU_INTERRUPT_READ_BUS_ERROR); ++ /* no session is active, so just activate the empty page directory */ ++ mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_DTE_ADDR, mali_empty_page_directory_phys); ++ mali_mmu_enable_paging(mmu); ++ err = _MALI_OSK_ERR_OK; ++ } ++ mali_mmu_disable_stall(mmu); ++ ++ return err; ++} ++ ++mali_bool mali_mmu_zap_tlb(struct mali_mmu_core *mmu) ++{ ++ mali_bool stall_success = mali_mmu_enable_stall(mmu); ++ ++ mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_ZAP_CACHE); ++ ++ if (MALI_FALSE == stall_success) { ++ /* False means that it is in Pagefault state. Not possible to disable_stall then */ ++ return MALI_FALSE; ++ } ++ ++ mali_mmu_disable_stall(mmu); ++ return MALI_TRUE; ++} ++ ++void mali_mmu_zap_tlb_without_stall(struct mali_mmu_core *mmu) ++{ ++ mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_ZAP_CACHE); ++} ++ ++ ++void mali_mmu_invalidate_page(struct mali_mmu_core *mmu, u32 mali_address) ++{ ++ mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_ZAP_ONE_LINE, MALI_MMU_PDE_ENTRY(mali_address)); ++} ++ ++static void mali_mmu_activate_address_space(struct mali_mmu_core *mmu, u32 page_directory) ++{ ++ /* The MMU must be in stalled or page fault mode, for this writing to work */ ++ MALI_DEBUG_ASSERT(0 != (mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS) ++ & (MALI_MMU_STATUS_BIT_STALL_ACTIVE | MALI_MMU_STATUS_BIT_PAGE_FAULT_ACTIVE))); ++ mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_DTE_ADDR, page_directory); ++ mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_ZAP_CACHE); ++ ++} ++ ++void mali_mmu_activate_page_directory(struct mali_mmu_core *mmu, struct mali_page_directory *pagedir) ++{ ++ mali_bool stall_success; ++ MALI_DEBUG_ASSERT_POINTER(mmu); ++ ++ MALI_DEBUG_PRINT(5, ("Asked to activate page directory 0x%x on MMU %s\n", pagedir, mmu->hw_core.description)); ++ ++ stall_success = mali_mmu_enable_stall(mmu); ++ MALI_DEBUG_ASSERT(stall_success); ++ MALI_IGNORE(stall_success); ++ mali_mmu_activate_address_space(mmu, pagedir->page_directory); ++ mali_mmu_disable_stall(mmu); ++} ++ ++void mali_mmu_activate_empty_page_directory(struct mali_mmu_core *mmu) ++{ ++ mali_bool stall_success; ++ ++ MALI_DEBUG_ASSERT_POINTER(mmu); ++ MALI_DEBUG_PRINT(3, ("Activating the empty page directory on MMU %s\n", mmu->hw_core.description)); ++ ++ stall_success = mali_mmu_enable_stall(mmu); ++ ++ /* This function can only be called when the core is idle, so it could not fail. */ ++ MALI_DEBUG_ASSERT(stall_success); ++ MALI_IGNORE(stall_success); ++ ++ mali_mmu_activate_address_space(mmu, mali_empty_page_directory_phys); ++ mali_mmu_disable_stall(mmu); ++} ++ ++void mali_mmu_activate_fault_flush_page_directory(struct mali_mmu_core *mmu) ++{ ++ mali_bool stall_success; ++ MALI_DEBUG_ASSERT_POINTER(mmu); ++ ++ MALI_DEBUG_PRINT(3, ("Activating the page fault flush page directory on MMU %s\n", mmu->hw_core.description)); ++ stall_success = mali_mmu_enable_stall(mmu); ++ /* This function is expect to fail the stalling, since it might be in PageFault mode when it is called */ ++ mali_mmu_activate_address_space(mmu, mali_page_fault_flush_page_directory); ++ if (MALI_TRUE == stall_success) mali_mmu_disable_stall(mmu); ++} ++ ++/* Is called when we want the mmu to give an interrupt */ ++static void mali_mmu_probe_trigger(void *data) ++{ ++ struct mali_mmu_core *mmu = (struct mali_mmu_core *)data; ++ mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_INT_RAWSTAT, MALI_MMU_INTERRUPT_PAGE_FAULT | MALI_MMU_INTERRUPT_READ_BUS_ERROR); ++} ++ ++/* Is called when the irq probe wants the mmu to acknowledge an interrupt from the hw */ ++static _mali_osk_errcode_t mali_mmu_probe_ack(void *data) ++{ ++ struct mali_mmu_core *mmu = (struct mali_mmu_core *)data; ++ u32 int_stat; ++ ++ int_stat = mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_INT_STATUS); ++ ++ MALI_DEBUG_PRINT(2, ("mali_mmu_probe_irq_acknowledge: intstat 0x%x\n", int_stat)); ++ if (int_stat & MALI_MMU_INTERRUPT_PAGE_FAULT) { ++ MALI_DEBUG_PRINT(2, ("Probe: Page fault detect: PASSED\n")); ++ mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_INT_CLEAR, MALI_MMU_INTERRUPT_PAGE_FAULT); ++ } else { ++ MALI_DEBUG_PRINT(1, ("Probe: Page fault detect: FAILED\n")); ++ } ++ ++ if (int_stat & MALI_MMU_INTERRUPT_READ_BUS_ERROR) { ++ MALI_DEBUG_PRINT(2, ("Probe: Bus read error detect: PASSED\n")); ++ mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_INT_CLEAR, MALI_MMU_INTERRUPT_READ_BUS_ERROR); ++ } else { ++ MALI_DEBUG_PRINT(1, ("Probe: Bus read error detect: FAILED\n")); ++ } ++ ++ if ((int_stat & (MALI_MMU_INTERRUPT_PAGE_FAULT | MALI_MMU_INTERRUPT_READ_BUS_ERROR)) == ++ (MALI_MMU_INTERRUPT_PAGE_FAULT | MALI_MMU_INTERRUPT_READ_BUS_ERROR)) { ++ return _MALI_OSK_ERR_OK; ++ } ++ ++ return _MALI_OSK_ERR_FAULT; ++} ++ ++#if 0 ++void mali_mmu_print_state(struct mali_mmu_core *mmu) ++{ ++ MALI_DEBUG_PRINT(2, ("MMU: State of %s is 0x%08x\n", mmu->hw_core.description, mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS))); ++} ++#endif +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_mmu.h b/drivers/gpu/arm/mali400/mali/common/mali_mmu.h +new file mode 100755 +index 000000000000..6ed48585f3d2 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_mmu.h +@@ -0,0 +1,124 @@ ++/* ++ * Copyright (C) 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_MMU_H__ ++#define __MALI_MMU_H__ ++ ++#include "mali_osk.h" ++#include "mali_mmu_page_directory.h" ++#include "mali_hw_core.h" ++ ++/* Forward declaration from mali_group.h */ ++struct mali_group; ++ ++/** ++ * MMU register numbers ++ * Used in the register read/write routines. ++ * See the hardware documentation for more information about each register ++ */ ++typedef enum mali_mmu_register { ++ MALI_MMU_REGISTER_DTE_ADDR = 0x0000, /**< Current Page Directory Pointer */ ++ MALI_MMU_REGISTER_STATUS = 0x0004, /**< Status of the MMU */ ++ MALI_MMU_REGISTER_COMMAND = 0x0008, /**< Command register, used to control the MMU */ ++ MALI_MMU_REGISTER_PAGE_FAULT_ADDR = 0x000C, /**< Logical address of the last page fault */ ++ MALI_MMU_REGISTER_ZAP_ONE_LINE = 0x010, /**< Used to invalidate the mapping of a single page from the MMU */ ++ MALI_MMU_REGISTER_INT_RAWSTAT = 0x0014, /**< Raw interrupt status, all interrupts visible */ ++ MALI_MMU_REGISTER_INT_CLEAR = 0x0018, /**< Indicate to the MMU that the interrupt has been received */ ++ MALI_MMU_REGISTER_INT_MASK = 0x001C, /**< Enable/disable types of interrupts */ ++ MALI_MMU_REGISTER_INT_STATUS = 0x0020 /**< Interrupt status based on the mask */ ++} mali_mmu_register; ++ ++/** ++ * MMU interrupt register bits ++ * Each cause of the interrupt is reported ++ * through the (raw) interrupt status registers. ++ * Multiple interrupts can be pending, so multiple bits ++ * can be set at once. ++ */ ++typedef enum mali_mmu_interrupt { ++ MALI_MMU_INTERRUPT_PAGE_FAULT = 0x01, /**< A page fault occured */ ++ MALI_MMU_INTERRUPT_READ_BUS_ERROR = 0x02 /**< A bus read error occured */ ++} mali_mmu_interrupt; ++ ++typedef enum mali_mmu_status_bits { ++ MALI_MMU_STATUS_BIT_PAGING_ENABLED = 1 << 0, ++ MALI_MMU_STATUS_BIT_PAGE_FAULT_ACTIVE = 1 << 1, ++ MALI_MMU_STATUS_BIT_STALL_ACTIVE = 1 << 2, ++ MALI_MMU_STATUS_BIT_IDLE = 1 << 3, ++ MALI_MMU_STATUS_BIT_REPLAY_BUFFER_EMPTY = 1 << 4, ++ MALI_MMU_STATUS_BIT_PAGE_FAULT_IS_WRITE = 1 << 5, ++ MALI_MMU_STATUS_BIT_STALL_NOT_ACTIVE = 1 << 31, ++} mali_mmu_status_bits; ++ ++/** ++ * Definition of the MMU struct ++ * Used to track a MMU unit in the system. ++ * Contains information about the mapping of the registers ++ */ ++struct mali_mmu_core { ++ struct mali_hw_core hw_core; /**< Common for all HW cores */ ++ _mali_osk_irq_t *irq; /**< IRQ handler */ ++}; ++ ++_mali_osk_errcode_t mali_mmu_initialize(void); ++ ++void mali_mmu_terminate(void); ++ ++struct mali_mmu_core *mali_mmu_create(_mali_osk_resource_t *resource, struct mali_group *group, mali_bool is_virtual); ++void mali_mmu_delete(struct mali_mmu_core *mmu); ++ ++_mali_osk_errcode_t mali_mmu_reset(struct mali_mmu_core *mmu); ++mali_bool mali_mmu_zap_tlb(struct mali_mmu_core *mmu); ++void mali_mmu_zap_tlb_without_stall(struct mali_mmu_core *mmu); ++void mali_mmu_invalidate_page(struct mali_mmu_core *mmu, u32 mali_address); ++ ++void mali_mmu_activate_page_directory(struct mali_mmu_core *mmu, struct mali_page_directory *pagedir); ++void mali_mmu_activate_empty_page_directory(struct mali_mmu_core *mmu); ++void mali_mmu_activate_fault_flush_page_directory(struct mali_mmu_core *mmu); ++ ++void mali_mmu_page_fault_done(struct mali_mmu_core *mmu); ++ ++MALI_STATIC_INLINE enum mali_interrupt_result mali_mmu_get_interrupt_result(struct mali_mmu_core *mmu) ++{ ++ u32 rawstat_used = mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_INT_RAWSTAT); ++ if (0 == rawstat_used) { ++ return MALI_INTERRUPT_RESULT_NONE; ++ } ++ ++ return MALI_INTERRUPT_RESULT_ERROR; ++} ++ ++ ++MALI_STATIC_INLINE u32 mali_mmu_get_int_status(struct mali_mmu_core *mmu) ++{ ++ return mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_INT_STATUS); ++} ++ ++MALI_STATIC_INLINE u32 mali_mmu_get_rawstat(struct mali_mmu_core *mmu) ++{ ++ return mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_INT_RAWSTAT); ++} ++ ++MALI_STATIC_INLINE void mali_mmu_mask_all_interrupts(struct mali_mmu_core *mmu) ++{ ++ mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_INT_MASK, 0); ++} ++ ++MALI_STATIC_INLINE u32 mali_mmu_get_status(struct mali_mmu_core *mmu) ++{ ++ return mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS); ++} ++ ++MALI_STATIC_INLINE u32 mali_mmu_get_page_fault_addr(struct mali_mmu_core *mmu) ++{ ++ return mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_PAGE_FAULT_ADDR); ++} ++ ++#endif /* __MALI_MMU_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_mmu_page_directory.c b/drivers/gpu/arm/mali400/mali/common/mali_mmu_page_directory.c +new file mode 100755 +index 000000000000..9ad3e8970b7d +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_mmu_page_directory.c +@@ -0,0 +1,495 @@ ++/* ++ * Copyright (C) 2011-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include "mali_kernel_common.h" ++#include "mali_osk.h" ++#include "mali_ukk.h" ++#include "mali_uk_types.h" ++#include "mali_mmu_page_directory.h" ++#include "mali_memory.h" ++#include "mali_l2_cache.h" ++ ++static _mali_osk_errcode_t fill_page(mali_io_address mapping, u32 data); ++ ++u32 mali_allocate_empty_page(mali_io_address *virt_addr) ++{ ++ _mali_osk_errcode_t err; ++ mali_io_address mapping; ++ mali_dma_addr address; ++ ++ if (_MALI_OSK_ERR_OK != mali_mmu_get_table_page(&address, &mapping)) { ++ /* Allocation failed */ ++ MALI_DEBUG_PRINT(2, ("Mali MMU: Failed to get table page for empty pgdir\n")); ++ return 0; ++ } ++ ++ MALI_DEBUG_ASSERT_POINTER(mapping); ++ ++ err = fill_page(mapping, 0); ++ if (_MALI_OSK_ERR_OK != err) { ++ mali_mmu_release_table_page(address, mapping); ++ MALI_DEBUG_PRINT(2, ("Mali MMU: Failed to zero page\n")); ++ return 0; ++ } ++ ++ *virt_addr = mapping; ++ return address; ++} ++ ++void mali_free_empty_page(mali_dma_addr address, mali_io_address virt_addr) ++{ ++ if (MALI_INVALID_PAGE != address) { ++ mali_mmu_release_table_page(address, virt_addr); ++ } ++} ++ ++_mali_osk_errcode_t mali_create_fault_flush_pages(mali_dma_addr *page_directory, ++ mali_io_address *page_directory_mapping, ++ mali_dma_addr *page_table, mali_io_address *page_table_mapping, ++ mali_dma_addr *data_page, mali_io_address *data_page_mapping) ++{ ++ _mali_osk_errcode_t err; ++ ++ err = mali_mmu_get_table_page(data_page, data_page_mapping); ++ if (_MALI_OSK_ERR_OK == err) { ++ err = mali_mmu_get_table_page(page_table, page_table_mapping); ++ if (_MALI_OSK_ERR_OK == err) { ++ err = mali_mmu_get_table_page(page_directory, page_directory_mapping); ++ if (_MALI_OSK_ERR_OK == err) { ++ fill_page(*data_page_mapping, 0); ++ fill_page(*page_table_mapping, *data_page | MALI_MMU_FLAGS_DEFAULT); ++ fill_page(*page_directory_mapping, *page_table | MALI_MMU_FLAGS_PRESENT); ++ MALI_SUCCESS; ++ } ++ mali_mmu_release_table_page(*page_table, *page_table_mapping); ++ *page_table = MALI_INVALID_PAGE; ++ } ++ mali_mmu_release_table_page(*data_page, *data_page_mapping); ++ *data_page = MALI_INVALID_PAGE; ++ } ++ return err; ++} ++ ++void mali_destroy_fault_flush_pages( ++ mali_dma_addr *page_directory, mali_io_address *page_directory_mapping, ++ mali_dma_addr *page_table, mali_io_address *page_table_mapping, ++ mali_dma_addr *data_page, mali_io_address *data_page_mapping) ++{ ++ if (MALI_INVALID_PAGE != *page_directory) { ++ mali_mmu_release_table_page(*page_directory, *page_directory_mapping); ++ *page_directory = MALI_INVALID_PAGE; ++ *page_directory_mapping = NULL; ++ } ++ ++ if (MALI_INVALID_PAGE != *page_table) { ++ mali_mmu_release_table_page(*page_table, *page_table_mapping); ++ *page_table = MALI_INVALID_PAGE; ++ *page_table_mapping = NULL; ++ } ++ ++ if (MALI_INVALID_PAGE != *data_page) { ++ mali_mmu_release_table_page(*data_page, *data_page_mapping); ++ *data_page = MALI_INVALID_PAGE; ++ *data_page_mapping = NULL; ++ } ++} ++ ++static _mali_osk_errcode_t fill_page(mali_io_address mapping, u32 data) ++{ ++ int i; ++ MALI_DEBUG_ASSERT_POINTER(mapping); ++ ++ for (i = 0; i < MALI_MMU_PAGE_SIZE / 4; i++) { ++ _mali_osk_mem_iowrite32_relaxed(mapping, i * sizeof(u32), data); ++ } ++ _mali_osk_mem_barrier(); ++ MALI_SUCCESS; ++} ++ ++_mali_osk_errcode_t mali_mmu_pagedir_map(struct mali_page_directory *pagedir, u32 mali_address, u32 size) ++{ ++ const int first_pde = MALI_MMU_PDE_ENTRY(mali_address); ++ const int last_pde = MALI_MMU_PDE_ENTRY(mali_address + size - 1); ++ _mali_osk_errcode_t err; ++ mali_io_address pde_mapping; ++ mali_dma_addr pde_phys; ++ int i, page_count; ++ u32 start_address; ++ if (last_pde < first_pde) ++ return _MALI_OSK_ERR_INVALID_ARGS; ++ ++ for (i = first_pde; i <= last_pde; i++) { ++ if (0 == (_mali_osk_mem_ioread32(pagedir->page_directory_mapped, ++ i * sizeof(u32)) & MALI_MMU_FLAGS_PRESENT)) { ++ /* Page table not present */ ++ MALI_DEBUG_ASSERT(0 == pagedir->page_entries_usage_count[i]); ++ MALI_DEBUG_ASSERT(NULL == pagedir->page_entries_mapped[i]); ++ ++ err = mali_mmu_get_table_page(&pde_phys, &pde_mapping); ++ if (_MALI_OSK_ERR_OK != err) { ++ MALI_PRINT_ERROR(("Failed to allocate page table page.\n")); ++ return err; ++ } ++ pagedir->page_entries_mapped[i] = pde_mapping; ++ ++ /* Update PDE, mark as present */ ++ _mali_osk_mem_iowrite32_relaxed(pagedir->page_directory_mapped, i * sizeof(u32), ++ pde_phys | MALI_MMU_FLAGS_PRESENT); ++ ++ MALI_DEBUG_ASSERT(0 == pagedir->page_entries_usage_count[i]); ++ } ++ ++ if (first_pde == last_pde) { ++ pagedir->page_entries_usage_count[i] += size / MALI_MMU_PAGE_SIZE; ++ } else if (i == first_pde) { ++ start_address = i * MALI_MMU_VIRTUAL_PAGE_SIZE; ++ page_count = (start_address + MALI_MMU_VIRTUAL_PAGE_SIZE - mali_address) / MALI_MMU_PAGE_SIZE; ++ pagedir->page_entries_usage_count[i] += page_count; ++ } else if (i == last_pde) { ++ start_address = i * MALI_MMU_VIRTUAL_PAGE_SIZE; ++ page_count = (mali_address + size - start_address) / MALI_MMU_PAGE_SIZE; ++ pagedir->page_entries_usage_count[i] += page_count; ++ } else { ++ pagedir->page_entries_usage_count[i] = 1024; ++ } ++ } ++ _mali_osk_write_mem_barrier(); ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++MALI_STATIC_INLINE void mali_mmu_zero_pte(mali_io_address page_table, u32 mali_address, u32 size) ++{ ++ int i; ++ const int first_pte = MALI_MMU_PTE_ENTRY(mali_address); ++ const int last_pte = MALI_MMU_PTE_ENTRY(mali_address + size - 1); ++ ++ for (i = first_pte; i <= last_pte; i++) { ++ _mali_osk_mem_iowrite32_relaxed(page_table, i * sizeof(u32), 0); ++ } ++} ++ ++static u32 mali_page_directory_get_phys_address(struct mali_page_directory *pagedir, u32 index) ++{ ++ return (_mali_osk_mem_ioread32(pagedir->page_directory_mapped, ++ index * sizeof(u32)) & ~MALI_MMU_FLAGS_MASK); ++} ++ ++ ++_mali_osk_errcode_t mali_mmu_pagedir_unmap(struct mali_page_directory *pagedir, u32 mali_address, u32 size) ++{ ++ const int first_pde = MALI_MMU_PDE_ENTRY(mali_address); ++ const int last_pde = MALI_MMU_PDE_ENTRY(mali_address + size - 1); ++ u32 left = size; ++ int i; ++ mali_bool pd_changed = MALI_FALSE; ++ u32 pages_to_invalidate[3]; /* hard-coded to 3: max two pages from the PT level plus max one page from PD level */ ++ u32 num_pages_inv = 0; ++ mali_bool invalidate_all = MALI_FALSE; /* safety mechanism in case page_entries_usage_count is unreliable */ ++ ++ /* For all page directory entries in range. */ ++ for (i = first_pde; i <= last_pde; i++) { ++ u32 size_in_pde, offset; ++ ++ MALI_DEBUG_ASSERT_POINTER(pagedir->page_entries_mapped[i]); ++ MALI_DEBUG_ASSERT(0 != pagedir->page_entries_usage_count[i]); ++ ++ /* Offset into page table, 0 if mali_address is 4MiB aligned */ ++ offset = (mali_address & (MALI_MMU_VIRTUAL_PAGE_SIZE - 1)); ++ if (left < MALI_MMU_VIRTUAL_PAGE_SIZE - offset) { ++ size_in_pde = left; ++ } else { ++ size_in_pde = MALI_MMU_VIRTUAL_PAGE_SIZE - offset; ++ } ++ ++ pagedir->page_entries_usage_count[i] -= size_in_pde / MALI_MMU_PAGE_SIZE; ++ ++ /* If entire page table is unused, free it */ ++ if (0 == pagedir->page_entries_usage_count[i]) { ++ u32 page_phys; ++ void *page_virt; ++ MALI_DEBUG_PRINT(4, ("Releasing page table as this is the last reference\n")); ++ /* last reference removed, no need to zero out each PTE */ ++ ++ page_phys = MALI_MMU_ENTRY_ADDRESS(_mali_osk_mem_ioread32(pagedir->page_directory_mapped, i * sizeof(u32))); ++ page_virt = pagedir->page_entries_mapped[i]; ++ pagedir->page_entries_mapped[i] = NULL; ++ _mali_osk_mem_iowrite32_relaxed(pagedir->page_directory_mapped, i * sizeof(u32), 0); ++ ++ mali_mmu_release_table_page(page_phys, page_virt); ++ pd_changed = MALI_TRUE; ++ } else { ++ MALI_DEBUG_ASSERT(num_pages_inv < 2); ++ if (num_pages_inv < 2) { ++ pages_to_invalidate[num_pages_inv] = mali_page_directory_get_phys_address(pagedir, i); ++ num_pages_inv++; ++ } else { ++ invalidate_all = MALI_TRUE; ++ } ++ ++ /* If part of the page table is still in use, zero the relevant PTEs */ ++ mali_mmu_zero_pte(pagedir->page_entries_mapped[i], mali_address, size_in_pde); ++ } ++ ++ left -= size_in_pde; ++ mali_address += size_in_pde; ++ } ++ _mali_osk_write_mem_barrier(); ++ ++ /* L2 pages invalidation */ ++ if (MALI_TRUE == pd_changed) { ++ MALI_DEBUG_ASSERT(num_pages_inv < 3); ++ if (num_pages_inv < 3) { ++ pages_to_invalidate[num_pages_inv] = pagedir->page_directory; ++ num_pages_inv++; ++ } else { ++ invalidate_all = MALI_TRUE; ++ } ++ } ++ ++ if (invalidate_all) { ++ mali_l2_cache_invalidate_all(); ++ } else { ++ mali_l2_cache_invalidate_all_pages(pages_to_invalidate, num_pages_inv); ++ } ++ ++ MALI_SUCCESS; ++} ++ ++struct mali_page_directory *mali_mmu_pagedir_alloc(void) ++{ ++ struct mali_page_directory *pagedir; ++ _mali_osk_errcode_t err; ++ mali_dma_addr phys; ++ ++ pagedir = _mali_osk_calloc(1, sizeof(struct mali_page_directory)); ++ if (NULL == pagedir) { ++ return NULL; ++ } ++ ++ err = mali_mmu_get_table_page(&phys, &pagedir->page_directory_mapped); ++ if (_MALI_OSK_ERR_OK != err) { ++ _mali_osk_free(pagedir); ++ return NULL; ++ } ++ ++ pagedir->page_directory = (u32)phys; ++ ++ /* Zero page directory */ ++ fill_page(pagedir->page_directory_mapped, 0); ++ ++ return pagedir; ++} ++ ++void mali_mmu_pagedir_free(struct mali_page_directory *pagedir) ++{ ++ const int num_page_table_entries = sizeof(pagedir->page_entries_mapped) / sizeof(pagedir->page_entries_mapped[0]); ++ int i; ++ ++ /* Free referenced page tables and zero PDEs. */ ++ for (i = 0; i < num_page_table_entries; i++) { ++ if (pagedir->page_directory_mapped && (_mali_osk_mem_ioread32( ++ pagedir->page_directory_mapped, ++ sizeof(u32)*i) & MALI_MMU_FLAGS_PRESENT)) { ++ mali_dma_addr phys = _mali_osk_mem_ioread32(pagedir->page_directory_mapped, ++ i * sizeof(u32)) & ~MALI_MMU_FLAGS_MASK; ++ _mali_osk_mem_iowrite32_relaxed(pagedir->page_directory_mapped, i * sizeof(u32), 0); ++ mali_mmu_release_table_page(phys, pagedir->page_entries_mapped[i]); ++ } ++ } ++ _mali_osk_write_mem_barrier(); ++ ++ /* Free the page directory page. */ ++ mali_mmu_release_table_page(pagedir->page_directory, pagedir->page_directory_mapped); ++ ++ _mali_osk_free(pagedir); ++} ++ ++ ++void mali_mmu_pagedir_update(struct mali_page_directory *pagedir, u32 mali_address, ++ mali_dma_addr phys_address, u32 size, u32 permission_bits) ++{ ++ u32 end_address = mali_address + size; ++ u32 mali_phys = (u32)phys_address; ++ ++ /* Map physical pages into MMU page tables */ ++ for (; mali_address < end_address; mali_address += MALI_MMU_PAGE_SIZE, mali_phys += MALI_MMU_PAGE_SIZE) { ++ MALI_DEBUG_ASSERT_POINTER(pagedir->page_entries_mapped[MALI_MMU_PDE_ENTRY(mali_address)]); ++ _mali_osk_mem_iowrite32_relaxed(pagedir->page_entries_mapped[MALI_MMU_PDE_ENTRY(mali_address)], ++ MALI_MMU_PTE_ENTRY(mali_address) * sizeof(u32), ++ mali_phys | permission_bits); ++ } ++} ++ ++void mali_mmu_pagedir_diag(struct mali_page_directory *pagedir, u32 fault_addr) ++{ ++#if defined(DEBUG) ++ u32 pde_index, pte_index; ++ u32 pde, pte; ++ ++ pde_index = MALI_MMU_PDE_ENTRY(fault_addr); ++ pte_index = MALI_MMU_PTE_ENTRY(fault_addr); ++ ++ ++ pde = _mali_osk_mem_ioread32(pagedir->page_directory_mapped, ++ pde_index * sizeof(u32)); ++ ++ ++ if (pde & MALI_MMU_FLAGS_PRESENT) { ++ u32 pte_addr = MALI_MMU_ENTRY_ADDRESS(pde); ++ ++ pte = _mali_osk_mem_ioread32(pagedir->page_entries_mapped[pde_index], ++ pte_index * sizeof(u32)); ++ ++ MALI_DEBUG_PRINT(2, ("\tMMU: %08x: Page table present: %08x\n" ++ "\t\tPTE: %08x, page %08x is %s\n", ++ fault_addr, pte_addr, pte, ++ MALI_MMU_ENTRY_ADDRESS(pte), ++ pte & MALI_MMU_FLAGS_DEFAULT ? "rw" : "not present")); ++ } else { ++ MALI_DEBUG_PRINT(2, ("\tMMU: %08x: Page table not present: %08x\n", ++ fault_addr, pde)); ++ } ++#else ++ MALI_IGNORE(pagedir); ++ MALI_IGNORE(fault_addr); ++#endif ++} ++ ++/* For instrumented */ ++struct dump_info { ++ u32 buffer_left; ++ u32 register_writes_size; ++ u32 page_table_dump_size; ++ u32 *buffer; ++}; ++ ++static _mali_osk_errcode_t writereg(u32 where, u32 what, const char *comment, struct dump_info *info) ++{ ++ if (NULL != info) { ++ info->register_writes_size += sizeof(u32) * 2; /* two 32-bit words */ ++ ++ if (NULL != info->buffer) { ++ /* check that we have enough space */ ++ if (info->buffer_left < sizeof(u32) * 2) MALI_ERROR(_MALI_OSK_ERR_NOMEM); ++ ++ *info->buffer = where; ++ info->buffer++; ++ ++ *info->buffer = what; ++ info->buffer++; ++ ++ info->buffer_left -= sizeof(u32) * 2; ++ } ++ } ++ ++ MALI_SUCCESS; ++} ++ ++static _mali_osk_errcode_t mali_mmu_dump_page(mali_io_address page, u32 phys_addr, struct dump_info *info) ++{ ++ if (NULL != info) { ++ /* 4096 for the page and 4 bytes for the address */ ++ const u32 page_size_in_elements = MALI_MMU_PAGE_SIZE / 4; ++ const u32 page_size_in_bytes = MALI_MMU_PAGE_SIZE; ++ const u32 dump_size_in_bytes = MALI_MMU_PAGE_SIZE + 4; ++ ++ info->page_table_dump_size += dump_size_in_bytes; ++ ++ if (NULL != info->buffer) { ++ if (info->buffer_left < dump_size_in_bytes) MALI_ERROR(_MALI_OSK_ERR_NOMEM); ++ ++ *info->buffer = phys_addr; ++ info->buffer++; ++ ++ _mali_osk_memcpy(info->buffer, page, page_size_in_bytes); ++ info->buffer += page_size_in_elements; ++ ++ info->buffer_left -= dump_size_in_bytes; ++ } ++ } ++ ++ MALI_SUCCESS; ++} ++ ++static _mali_osk_errcode_t dump_mmu_page_table(struct mali_page_directory *pagedir, struct dump_info *info) ++{ ++ MALI_DEBUG_ASSERT_POINTER(pagedir); ++ MALI_DEBUG_ASSERT_POINTER(info); ++ ++ if (NULL != pagedir->page_directory_mapped) { ++ int i; ++ ++ MALI_CHECK_NO_ERROR( ++ mali_mmu_dump_page(pagedir->page_directory_mapped, pagedir->page_directory, info) ++ ); ++ ++ for (i = 0; i < 1024; i++) { ++ if (NULL != pagedir->page_entries_mapped[i]) { ++ MALI_CHECK_NO_ERROR( ++ mali_mmu_dump_page(pagedir->page_entries_mapped[i], ++ _mali_osk_mem_ioread32(pagedir->page_directory_mapped, ++ i * sizeof(u32)) & ~MALI_MMU_FLAGS_MASK, info) ++ ); ++ } ++ } ++ } ++ ++ MALI_SUCCESS; ++} ++ ++static _mali_osk_errcode_t dump_mmu_registers(struct mali_page_directory *pagedir, struct dump_info *info) ++{ ++ MALI_CHECK_NO_ERROR(writereg(0x00000000, pagedir->page_directory, ++ "set the page directory address", info)); ++ MALI_CHECK_NO_ERROR(writereg(0x00000008, 4, "zap???", info)); ++ MALI_CHECK_NO_ERROR(writereg(0x00000008, 0, "enable paging", info)); ++ MALI_SUCCESS; ++} ++ ++_mali_osk_errcode_t _mali_ukk_query_mmu_page_table_dump_size(_mali_uk_query_mmu_page_table_dump_size_s *args) ++{ ++ struct dump_info info = { 0, 0, 0, NULL }; ++ struct mali_session_data *session_data; ++ ++ session_data = (struct mali_session_data *)(uintptr_t)(args->ctx); ++ MALI_DEBUG_ASSERT_POINTER(session_data); ++ MALI_DEBUG_ASSERT_POINTER(args); ++ ++ MALI_CHECK_NO_ERROR(dump_mmu_registers(session_data->page_directory, &info)); ++ MALI_CHECK_NO_ERROR(dump_mmu_page_table(session_data->page_directory, &info)); ++ args->size = info.register_writes_size + info.page_table_dump_size; ++ MALI_SUCCESS; ++} ++ ++_mali_osk_errcode_t _mali_ukk_dump_mmu_page_table(_mali_uk_dump_mmu_page_table_s *args) ++{ ++ struct dump_info info = { 0, 0, 0, NULL }; ++ struct mali_session_data *session_data; ++ ++ MALI_DEBUG_ASSERT_POINTER(args); ++ ++ session_data = (struct mali_session_data *)(uintptr_t)(args->ctx); ++ MALI_DEBUG_ASSERT_POINTER(session_data); ++ ++ info.buffer_left = args->size; ++ info.buffer = (u32 *)(uintptr_t)args->buffer; ++ ++ args->register_writes = (uintptr_t)info.buffer; ++ MALI_CHECK_NO_ERROR(dump_mmu_registers(session_data->page_directory, &info)); ++ ++ args->page_table_dump = (uintptr_t)info.buffer; ++ MALI_CHECK_NO_ERROR(dump_mmu_page_table(session_data->page_directory, &info)); ++ ++ args->register_writes_size = info.register_writes_size; ++ args->page_table_dump_size = info.page_table_dump_size; ++ ++ MALI_SUCCESS; ++} +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_mmu_page_directory.h b/drivers/gpu/arm/mali400/mali/common/mali_mmu_page_directory.h +new file mode 100755 +index 000000000000..3fdf07210259 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_mmu_page_directory.h +@@ -0,0 +1,110 @@ ++/* ++ * Copyright (C) 2011-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_MMU_PAGE_DIRECTORY_H__ ++#define __MALI_MMU_PAGE_DIRECTORY_H__ ++ ++#include "mali_osk.h" ++ ++/** ++ * Size of an MMU page in bytes ++ */ ++#define MALI_MMU_PAGE_SIZE 0x1000 ++ ++/* ++ * Size of the address space referenced by a page table page ++ */ ++#define MALI_MMU_VIRTUAL_PAGE_SIZE 0x400000 /* 4 MiB */ ++ ++/** ++ * Page directory index from address ++ * Calculates the page directory index from the given address ++ */ ++#define MALI_MMU_PDE_ENTRY(address) (((address)>>22) & 0x03FF) ++ ++/** ++ * Page table index from address ++ * Calculates the page table index from the given address ++ */ ++#define MALI_MMU_PTE_ENTRY(address) (((address)>>12) & 0x03FF) ++ ++/** ++ * Extract the memory address from an PDE/PTE entry ++ */ ++#define MALI_MMU_ENTRY_ADDRESS(value) ((value) & 0xFFFFFC00) ++ ++#define MALI_INVALID_PAGE ((u32)(~0)) ++ ++/** ++ * ++ */ ++typedef enum mali_mmu_entry_flags { ++ MALI_MMU_FLAGS_PRESENT = 0x01, ++ MALI_MMU_FLAGS_READ_PERMISSION = 0x02, ++ MALI_MMU_FLAGS_WRITE_PERMISSION = 0x04, ++ MALI_MMU_FLAGS_OVERRIDE_CACHE = 0x8, ++ MALI_MMU_FLAGS_WRITE_CACHEABLE = 0x10, ++ MALI_MMU_FLAGS_WRITE_ALLOCATE = 0x20, ++ MALI_MMU_FLAGS_WRITE_BUFFERABLE = 0x40, ++ MALI_MMU_FLAGS_READ_CACHEABLE = 0x80, ++ MALI_MMU_FLAGS_READ_ALLOCATE = 0x100, ++ MALI_MMU_FLAGS_MASK = 0x1FF, ++} mali_mmu_entry_flags; ++ ++ ++#define MALI_MMU_FLAGS_FORCE_GP_READ_ALLOCATE ( \ ++ MALI_MMU_FLAGS_PRESENT | \ ++ MALI_MMU_FLAGS_READ_PERMISSION | \ ++ MALI_MMU_FLAGS_WRITE_PERMISSION | \ ++ MALI_MMU_FLAGS_OVERRIDE_CACHE | \ ++ MALI_MMU_FLAGS_WRITE_CACHEABLE | \ ++ MALI_MMU_FLAGS_WRITE_BUFFERABLE | \ ++ MALI_MMU_FLAGS_READ_CACHEABLE | \ ++ MALI_MMU_FLAGS_READ_ALLOCATE ) ++ ++#define MALI_MMU_FLAGS_DEFAULT ( \ ++ MALI_MMU_FLAGS_PRESENT | \ ++ MALI_MMU_FLAGS_READ_PERMISSION | \ ++ MALI_MMU_FLAGS_WRITE_PERMISSION ) ++ ++ ++struct mali_page_directory { ++ u32 page_directory; /**< Physical address of the memory session's page directory */ ++ mali_io_address page_directory_mapped; /**< Pointer to the mapped version of the page directory into the kernel's address space */ ++ ++ mali_io_address page_entries_mapped[1024]; /**< Pointers to the page tables which exists in the page directory mapped into the kernel's address space */ ++ u32 page_entries_usage_count[1024]; /**< Tracks usage count of the page table pages, so they can be releases on the last reference */ ++}; ++ ++/* Map Mali virtual address space (i.e. ensure page tables exist for the virtual range) */ ++_mali_osk_errcode_t mali_mmu_pagedir_map(struct mali_page_directory *pagedir, u32 mali_address, u32 size); ++_mali_osk_errcode_t mali_mmu_pagedir_unmap(struct mali_page_directory *pagedir, u32 mali_address, u32 size); ++ ++/* Back virtual address space with actual pages. Assumes input is contiguous and 4k aligned. */ ++void mali_mmu_pagedir_update(struct mali_page_directory *pagedir, u32 mali_address, ++ mali_dma_addr phys_address, u32 size, u32 permission_bits); ++ ++u32 mali_allocate_empty_page(mali_io_address *virtual); ++void mali_free_empty_page(mali_dma_addr address, mali_io_address virt_addr); ++_mali_osk_errcode_t mali_create_fault_flush_pages(mali_dma_addr *page_directory, ++ mali_io_address *page_directory_mapping, ++ mali_dma_addr *page_table, mali_io_address *page_table_mapping, ++ mali_dma_addr *data_page, mali_io_address *data_page_mapping); ++void mali_destroy_fault_flush_pages( ++ mali_dma_addr *page_directory, mali_io_address *page_directory_mapping, ++ mali_dma_addr *page_table, mali_io_address *page_table_mapping, ++ mali_dma_addr *data_page, mali_io_address *data_page_mapping); ++ ++struct mali_page_directory *mali_mmu_pagedir_alloc(void); ++void mali_mmu_pagedir_free(struct mali_page_directory *pagedir); ++ ++void mali_mmu_pagedir_diag(struct mali_page_directory *pagedir, u32 fault_addr); ++ ++#endif /* __MALI_MMU_PAGE_DIRECTORY_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_osk.h b/drivers/gpu/arm/mali400/mali/common/mali_osk.h +new file mode 100755 +index 000000000000..9ade362d6b28 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_osk.h +@@ -0,0 +1,1389 @@ ++/* ++ * Copyright (C) 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file mali_osk.h ++ * Defines the OS abstraction layer for the kernel device driver (OSK) ++ */ ++ ++#ifndef __MALI_OSK_H__ ++#define __MALI_OSK_H__ ++ ++#include ++#include "mali_osk_types.h" ++#include "mali_osk_specific.h" /* include any per-os specifics */ ++#include "mali_osk_locks.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++/** ++ * @addtogroup uddapi Unified Device Driver (UDD) APIs ++ * ++ * @{ ++ */ ++ ++/** ++ * @addtogroup oskapi UDD OS Abstraction for Kernel-side (OSK) APIs ++ * ++ * @{ ++ */ ++ ++/** @addtogroup _mali_osk_lock OSK Mutual Exclusion Locks ++ * @{ */ ++ ++#ifdef DEBUG ++/** @brief Macro for asserting that the current thread holds a given lock ++ */ ++#define MALI_DEBUG_ASSERT_LOCK_HELD(l) MALI_DEBUG_ASSERT(_mali_osk_lock_get_owner((_mali_osk_lock_debug_t *)l) == _mali_osk_get_tid()); ++ ++/** @brief returns a lock's owner (thread id) if debugging is enabled ++ */ ++#else ++#define MALI_DEBUG_ASSERT_LOCK_HELD(l) do {} while(0) ++#endif ++ ++#define _mali_osk_ctxprintf seq_printf ++ ++/** @} */ /* end group _mali_osk_lock */ ++ ++/** @addtogroup _mali_osk_miscellaneous ++ * @{ */ ++ ++/** @brief Find the containing structure of another structure ++ * ++ * This is the reverse of the operation 'offsetof'. This means that the ++ * following condition is satisfied: ++ * ++ * ptr == _MALI_OSK_CONTAINER_OF( &ptr->member, type, member ) ++ * ++ * When ptr is of type 'type'. ++ * ++ * Its purpose it to recover a larger structure that has wrapped a smaller one. ++ * ++ * @note no type or memory checking occurs to ensure that a wrapper structure ++ * does in fact exist, and that it is being recovered with respect to the ++ * correct member. ++ * ++ * @param ptr the pointer to the member that is contained within the larger ++ * structure ++ * @param type the type of the structure that contains the member ++ * @param member the name of the member in the structure that ptr points to. ++ * @return a pointer to a \a type object which contains \a member, as pointed ++ * to by \a ptr. ++ */ ++#define _MALI_OSK_CONTAINER_OF(ptr, type, member) \ ++ ((type *)( ((char *)ptr) - offsetof(type,member) )) ++ ++/** @addtogroup _mali_osk_wq ++ * @{ */ ++ ++/** @brief Initialize work queues (for deferred work) ++ * ++ * @return _MALI_OSK_ERR_OK on success, otherwise failure. ++ */ ++_mali_osk_errcode_t _mali_osk_wq_init(void); ++ ++/** @brief Terminate work queues (for deferred work) ++ */ ++void _mali_osk_wq_term(void); ++ ++/** @brief Create work in the work queue ++ * ++ * Creates a work object which can be scheduled in the work queue. When ++ * scheduled, \a handler will be called with \a data as the argument. ++ * ++ * Refer to \ref _mali_osk_wq_schedule_work() for details on how work ++ * is scheduled in the queue. ++ * ++ * The returned pointer must be freed with \ref _mali_osk_wq_delete_work() ++ * when no longer needed. ++ */ ++_mali_osk_wq_work_t *_mali_osk_wq_create_work(_mali_osk_wq_work_handler_t handler, void *data); ++ ++/** @brief A high priority version of \a _mali_osk_wq_create_work() ++ * ++ * Creates a work object which can be scheduled in the high priority work queue. ++ * ++ * This is unfortunately needed to get low latency scheduling of the Mali cores. Normally we would ++ * schedule the next job in hw_irq or tasklet, but often we can't since we need to synchronously map ++ * and unmap shared memory when a job is connected to external fences (timelines). And this requires ++ * taking a mutex. ++ * ++ * We do signal a lot of other (low priority) work also as part of the job being finished, and if we ++ * don't set this Mali scheduling thread as high priority, we see that the CPU scheduler often runs ++ * random things instead of starting the next GPU job when the GPU is idle. So setting the gpu ++ * scheduler to high priority does give a visually more responsive system. ++ * ++ * Start the high priority work with: \a _mali_osk_wq_schedule_work_high_pri() ++ */ ++_mali_osk_wq_work_t *_mali_osk_wq_create_work_high_pri(_mali_osk_wq_work_handler_t handler, void *data); ++ ++/** @brief Delete a work object ++ * ++ * This will flush the work queue to ensure that the work handler will not ++ * be called after deletion. ++ */ ++void _mali_osk_wq_delete_work(_mali_osk_wq_work_t *work); ++ ++/** @brief Delete a work object ++ * ++ * This will NOT flush the work queue, so only call this if you are sure that the work handler will ++ * not be called after deletion. ++ */ ++void _mali_osk_wq_delete_work_nonflush(_mali_osk_wq_work_t *work); ++ ++/** @brief Cause a queued, deferred call of the work handler ++ * ++ * _mali_osk_wq_schedule_work provides a mechanism for enqueuing deferred calls ++ * to the work handler. After calling \ref _mali_osk_wq_schedule_work(), the ++ * work handler will be scheduled to run at some point in the future. ++ * ++ * Typically this is called by the IRQ upper-half to defer further processing of ++ * IRQ-related work to the IRQ bottom-half handler. This is necessary for work ++ * that cannot be done in an IRQ context by the IRQ upper-half handler. Timer ++ * callbacks also use this mechanism, because they are treated as though they ++ * operate in an IRQ context. Refer to \ref _mali_osk_timer_t for more ++ * information. ++ * ++ * Code that operates in a kernel-process context (with no IRQ context ++ * restrictions) may also enqueue deferred calls to the IRQ bottom-half. The ++ * advantage over direct calling is that deferred calling allows the caller and ++ * IRQ bottom half to hold the same mutex, with a guarantee that they will not ++ * deadlock just by using this mechanism. ++ * ++ * _mali_osk_wq_schedule_work() places deferred call requests on a queue, to ++ * allow for more than one thread to make a deferred call. Therfore, if it is ++ * called 'K' times, then the IRQ bottom-half will be scheduled 'K' times too. ++ * 'K' is a number that is implementation-specific. ++ * ++ * _mali_osk_wq_schedule_work() is guaranteed to not block on: ++ * - enqueuing a deferred call request. ++ * - the completion of the work handler. ++ * ++ * This is to prevent deadlock. For example, if _mali_osk_wq_schedule_work() ++ * blocked, then it would cause a deadlock when the following two conditions ++ * hold: ++ * - The work handler callback (of type _mali_osk_wq_work_handler_t) locks ++ * a mutex ++ * - And, at the same time, the caller of _mali_osk_wq_schedule_work() also ++ * holds the same mutex ++ * ++ * @note care must be taken to not overflow the queue that ++ * _mali_osk_wq_schedule_work() operates on. Code must be structured to ++ * ensure that the number of requests made to the queue is bounded. Otherwise, ++ * work will be lost. ++ * ++ * The queue that _mali_osk_wq_schedule_work implements is a FIFO of N-writer, ++ * 1-reader type. The writers are the callers of _mali_osk_wq_schedule_work ++ * (all OSK-registered IRQ upper-half handlers in the system, watchdog timers, ++ * callers from a Kernel-process context). The reader is a single thread that ++ * handles all OSK-registered work. ++ * ++ * @param work a pointer to the _mali_osk_wq_work_t object corresponding to the ++ * work to begin processing. ++ */ ++void _mali_osk_wq_schedule_work(_mali_osk_wq_work_t *work); ++ ++/** @brief Cause a queued, deferred call of the high priority work handler ++ * ++ * Function is the same as \a _mali_osk_wq_schedule_work() with the only ++ * difference that it runs in a high (real time) priority on the system. ++ * ++ * Should only be used as a substitue for doing the same work in interrupts. ++ * ++ * This is allowed to sleep, but the work should be small since it will block ++ * all other applications. ++*/ ++void _mali_osk_wq_schedule_work_high_pri(_mali_osk_wq_work_t *work); ++ ++/** @brief Flush the work queue ++ * ++ * This will flush the OSK work queue, ensuring all work in the queue has ++ * completed before returning. ++ * ++ * Since this blocks on the completion of work in the work-queue, the ++ * caller of this function \b must \b not hold any mutexes that are taken by ++ * any registered work handler. To do so may cause a deadlock. ++ * ++ */ ++void _mali_osk_wq_flush(void); ++ ++/** @brief Create work in the delayed work queue ++ * ++ * Creates a work object which can be scheduled in the work queue. When ++ * scheduled, a timer will be start and the \a handler will be called with ++ * \a data as the argument when timer out ++ * ++ * Refer to \ref _mali_osk_wq_delayed_schedule_work() for details on how work ++ * is scheduled in the queue. ++ * ++ * The returned pointer must be freed with \ref _mali_osk_wq_delayed_delete_work_nonflush() ++ * when no longer needed. ++ */ ++_mali_osk_wq_delayed_work_t *_mali_osk_wq_delayed_create_work(_mali_osk_wq_work_handler_t handler, void *data); ++ ++/** @brief Delete a work object ++ * ++ * This will NOT flush the work queue, so only call this if you are sure that the work handler will ++ * not be called after deletion. ++ */ ++void _mali_osk_wq_delayed_delete_work_nonflush(_mali_osk_wq_delayed_work_t *work); ++ ++/** @brief Cancel a delayed work without waiting for it to finish ++ * ++ * Note that the \a work callback function may still be running on return from ++ * _mali_osk_wq_delayed_cancel_work_async(). ++ * ++ * @param work The delayed work to be cancelled ++ */ ++void _mali_osk_wq_delayed_cancel_work_async(_mali_osk_wq_delayed_work_t *work); ++ ++/** @brief Cancel a delayed work and wait for it to finish ++ * ++ * When this function returns, the \a work was either cancelled or it finished running. ++ * ++ * @param work The delayed work to be cancelled ++ */ ++void _mali_osk_wq_delayed_cancel_work_sync(_mali_osk_wq_delayed_work_t *work); ++ ++/** @brief Put \a work task in global workqueue after delay ++ * ++ * After waiting for a given time this puts a job in the kernel-global ++ * workqueue. ++ * ++ * If \a work was already on a queue, this function will return without doing anything ++ * ++ * @param work job to be done ++ * @param delay number of jiffies to wait or 0 for immediate execution ++ */ ++void _mali_osk_wq_delayed_schedule_work(_mali_osk_wq_delayed_work_t *work, u32 delay); ++ ++/** @} */ /* end group _mali_osk_wq */ ++ ++ ++/** @addtogroup _mali_osk_irq ++ * @{ */ ++ ++/** @brief Initialize IRQ handling for a resource ++ * ++ * Registers an interrupt handler \a uhandler for the given IRQ number \a irqnum. ++ * \a data will be passed as argument to the handler when an interrupt occurs. ++ * ++ * If \a irqnum is -1, _mali_osk_irq_init will probe for the IRQ number using ++ * the supplied \a trigger_func and \a ack_func. These functions will also ++ * receive \a data as their argument. ++ * ++ * @param irqnum The IRQ number that the resource uses, as seen by the CPU. ++ * The value -1 has a special meaning which indicates the use of probing, and ++ * trigger_func and ack_func must be non-NULL. ++ * @param uhandler The interrupt handler, corresponding to a ISR handler for ++ * the resource ++ * @param int_data resource specific data, which will be passed to uhandler ++ * @param trigger_func Optional: a function to trigger the resource's irq, to ++ * probe for the interrupt. Use NULL if irqnum != -1. ++ * @param ack_func Optional: a function to acknowledge the resource's irq, to ++ * probe for the interrupt. Use NULL if irqnum != -1. ++ * @param probe_data resource-specific data, which will be passed to ++ * (if present) trigger_func and ack_func ++ * @param description textual description of the IRQ resource. ++ * @return on success, a pointer to a _mali_osk_irq_t object, which represents ++ * the IRQ handling on this resource. NULL on failure. ++ */ ++_mali_osk_irq_t *_mali_osk_irq_init(u32 irqnum, _mali_osk_irq_uhandler_t uhandler, void *int_data, _mali_osk_irq_trigger_t trigger_func, _mali_osk_irq_ack_t ack_func, void *probe_data, const char *description); ++ ++/** @brief Terminate IRQ handling on a resource. ++ * ++ * This will disable the interrupt from the device, and then waits for any ++ * currently executing IRQ handlers to complete. ++ * ++ * @note If work is deferred to an IRQ bottom-half handler through ++ * \ref _mali_osk_wq_schedule_work(), be sure to flush any remaining work ++ * with \ref _mali_osk_wq_flush() or (implicitly) with \ref _mali_osk_wq_delete_work() ++ * ++ * @param irq a pointer to the _mali_osk_irq_t object corresponding to the ++ * resource whose IRQ handling is to be terminated. ++ */ ++void _mali_osk_irq_term(_mali_osk_irq_t *irq); ++ ++/** @} */ /* end group _mali_osk_irq */ ++ ++ ++/** @addtogroup _mali_osk_atomic ++ * @{ */ ++ ++/** @brief Decrement an atomic counter ++ * ++ * @note It is an error to decrement the counter beyond -(1<<23) ++ * ++ * @param atom pointer to an atomic counter */ ++void _mali_osk_atomic_dec(_mali_osk_atomic_t *atom); ++ ++/** @brief Decrement an atomic counter, return new value ++ * ++ * @param atom pointer to an atomic counter ++ * @return The new value, after decrement */ ++u32 _mali_osk_atomic_dec_return(_mali_osk_atomic_t *atom); ++ ++/** @brief Increment an atomic counter ++ * ++ * @note It is an error to increment the counter beyond (1<<23)-1 ++ * ++ * @param atom pointer to an atomic counter */ ++void _mali_osk_atomic_inc(_mali_osk_atomic_t *atom); ++ ++/** @brief Increment an atomic counter, return new value ++ * ++ * @param atom pointer to an atomic counter */ ++u32 _mali_osk_atomic_inc_return(_mali_osk_atomic_t *atom); ++ ++/** @brief Initialize an atomic counter ++ * ++ * @note the parameter required is a u32, and so signed integers should be ++ * cast to u32. ++ * ++ * @param atom pointer to an atomic counter ++ * @param val the value to initialize the atomic counter. ++ */ ++void _mali_osk_atomic_init(_mali_osk_atomic_t *atom, u32 val); ++ ++/** @brief Read a value from an atomic counter ++ * ++ * This can only be safely used to determine the value of the counter when it ++ * is guaranteed that other threads will not be modifying the counter. This ++ * makes its usefulness limited. ++ * ++ * @param atom pointer to an atomic counter ++ */ ++u32 _mali_osk_atomic_read(_mali_osk_atomic_t *atom); ++ ++/** @brief Terminate an atomic counter ++ * ++ * @param atom pointer to an atomic counter ++ */ ++void _mali_osk_atomic_term(_mali_osk_atomic_t *atom); ++ ++/** @brief Assign a new val to atomic counter, and return the old atomic counter ++ * ++ * @param atom pointer to an atomic counter ++ * @param val the new value assign to the atomic counter ++ * @return the old value of the atomic counter ++ */ ++u32 _mali_osk_atomic_xchg(_mali_osk_atomic_t *atom, u32 val); ++/** @} */ /* end group _mali_osk_atomic */ ++ ++ ++/** @defgroup _mali_osk_memory OSK Memory Allocation ++ * @{ */ ++ ++/** @brief Allocate zero-initialized memory. ++ * ++ * Returns a buffer capable of containing at least \a n elements of \a size ++ * bytes each. The buffer is initialized to zero. ++ * ++ * If there is a need for a bigger block of memory (16KB or bigger), then ++ * consider to use _mali_osk_vmalloc() instead, as this function might ++ * map down to a OS function with size limitations. ++ * ++ * The buffer is suitably aligned for storage and subsequent access of every ++ * type that the compiler supports. Therefore, the pointer to the start of the ++ * buffer may be cast into any pointer type, and be subsequently accessed from ++ * such a pointer, without loss of information. ++ * ++ * When the buffer is no longer in use, it must be freed with _mali_osk_free(). ++ * Failure to do so will cause a memory leak. ++ * ++ * @note Most toolchains supply memory allocation functions that meet the ++ * compiler's alignment requirements. ++ * ++ * @param n Number of elements to allocate ++ * @param size Size of each element ++ * @return On success, the zero-initialized buffer allocated. NULL on failure ++ */ ++void *_mali_osk_calloc(u32 n, u32 size); ++ ++/** @brief Allocate memory. ++ * ++ * Returns a buffer capable of containing at least \a size bytes. The ++ * contents of the buffer are undefined. ++ * ++ * If there is a need for a bigger block of memory (16KB or bigger), then ++ * consider to use _mali_osk_vmalloc() instead, as this function might ++ * map down to a OS function with size limitations. ++ * ++ * The buffer is suitably aligned for storage and subsequent access of every ++ * type that the compiler supports. Therefore, the pointer to the start of the ++ * buffer may be cast into any pointer type, and be subsequently accessed from ++ * such a pointer, without loss of information. ++ * ++ * When the buffer is no longer in use, it must be freed with _mali_osk_free(). ++ * Failure to do so will cause a memory leak. ++ * ++ * @note Most toolchains supply memory allocation functions that meet the ++ * compiler's alignment requirements. ++ * ++ * Remember to free memory using _mali_osk_free(). ++ * @param size Number of bytes to allocate ++ * @return On success, the buffer allocated. NULL on failure. ++ */ ++void *_mali_osk_malloc(u32 size); ++ ++/** @brief Free memory. ++ * ++ * Reclaims the buffer pointed to by the parameter \a ptr for the system. ++ * All memory returned from _mali_osk_malloc() and _mali_osk_calloc() ++ * must be freed before the application exits. Otherwise, ++ * a memory leak will occur. ++ * ++ * Memory must be freed once. It is an error to free the same non-NULL pointer ++ * more than once. ++ * ++ * It is legal to free the NULL pointer. ++ * ++ * @param ptr Pointer to buffer to free ++ */ ++void _mali_osk_free(void *ptr); ++ ++/** @brief Allocate memory. ++ * ++ * Returns a buffer capable of containing at least \a size bytes. The ++ * contents of the buffer are undefined. ++ * ++ * This function is potentially slower than _mali_osk_malloc() and _mali_osk_calloc(), ++ * but do support bigger sizes. ++ * ++ * The buffer is suitably aligned for storage and subsequent access of every ++ * type that the compiler supports. Therefore, the pointer to the start of the ++ * buffer may be cast into any pointer type, and be subsequently accessed from ++ * such a pointer, without loss of information. ++ * ++ * When the buffer is no longer in use, it must be freed with _mali_osk_free(). ++ * Failure to do so will cause a memory leak. ++ * ++ * @note Most toolchains supply memory allocation functions that meet the ++ * compiler's alignment requirements. ++ * ++ * Remember to free memory using _mali_osk_free(). ++ * @param size Number of bytes to allocate ++ * @return On success, the buffer allocated. NULL on failure. ++ */ ++void *_mali_osk_valloc(u32 size); ++ ++/** @brief Free memory. ++ * ++ * Reclaims the buffer pointed to by the parameter \a ptr for the system. ++ * All memory returned from _mali_osk_valloc() must be freed before the ++ * application exits. Otherwise a memory leak will occur. ++ * ++ * Memory must be freed once. It is an error to free the same non-NULL pointer ++ * more than once. ++ * ++ * It is legal to free the NULL pointer. ++ * ++ * @param ptr Pointer to buffer to free ++ */ ++void _mali_osk_vfree(void *ptr); ++ ++/** @brief Copies memory. ++ * ++ * Copies the \a len bytes from the buffer pointed by the parameter \a src ++ * directly to the buffer pointed by \a dst. ++ * ++ * It is an error for \a src to overlap \a dst anywhere in \a len bytes. ++ * ++ * @param dst Pointer to the destination array where the content is to be ++ * copied. ++ * @param src Pointer to the source of data to be copied. ++ * @param len Number of bytes to copy. ++ * @return \a dst is always passed through unmodified. ++ */ ++void *_mali_osk_memcpy(void *dst, const void *src, u32 len); ++ ++/** @brief Fills memory. ++ * ++ * Sets the first \a n bytes of the block of memory pointed to by \a s to ++ * the specified value ++ * @param s Pointer to the block of memory to fill. ++ * @param c Value to be set, passed as u32. Only the 8 Least Significant Bits (LSB) ++ * are used. ++ * @param n Number of bytes to be set to the value. ++ * @return \a s is always passed through unmodified ++ */ ++void *_mali_osk_memset(void *s, u32 c, u32 n); ++/** @} */ /* end group _mali_osk_memory */ ++ ++ ++/** @brief Checks the amount of memory allocated ++ * ++ * Checks that not more than \a max_allocated bytes are allocated. ++ * ++ * Some OS bring up an interactive out of memory dialogue when the ++ * system runs out of memory. This can stall non-interactive ++ * apps (e.g. automated test runs). This function can be used to ++ * not trigger the OOM dialogue by keeping allocations ++ * within a certain limit. ++ * ++ * @return MALI_TRUE when \a max_allocated bytes are not in use yet. MALI_FALSE ++ * when at least \a max_allocated bytes are in use. ++ */ ++mali_bool _mali_osk_mem_check_allocated(u32 max_allocated); ++ ++ ++/** @addtogroup _mali_osk_low_level_memory ++ * @{ */ ++ ++/** @brief Issue a memory barrier ++ * ++ * This defines an arbitrary memory barrier operation, which forces an ordering constraint ++ * on memory read and write operations. ++ */ ++void _mali_osk_mem_barrier(void); ++ ++/** @brief Issue a write memory barrier ++ * ++ * This defines an write memory barrier operation which forces an ordering constraint ++ * on memory write operations. ++ */ ++void _mali_osk_write_mem_barrier(void); ++ ++/** @brief Map a physically contiguous region into kernel space ++ * ++ * This is primarily used for mapping in registers from resources, and Mali-MMU ++ * page tables. The mapping is only visable from kernel-space. ++ * ++ * Access has to go through _mali_osk_mem_ioread32 and _mali_osk_mem_iowrite32 ++ * ++ * @param phys CPU-physical base address of the memory to map in. This must ++ * be aligned to the system's page size, which is assumed to be 4K. ++ * @param size the number of bytes of physically contiguous address space to ++ * map in ++ * @param description A textual description of the memory being mapped in. ++ * @return On success, a Mali IO address through which the mapped-in ++ * memory/registers can be accessed. NULL on failure. ++ */ ++mali_io_address _mali_osk_mem_mapioregion(uintptr_t phys, u32 size, const char *description); ++ ++/** @brief Unmap a physically contiguous address range from kernel space. ++ * ++ * The address range should be one previously mapped in through ++ * _mali_osk_mem_mapioregion. ++ * ++ * It is a programming error to do (but not limited to) the following: ++ * - attempt an unmap twice ++ * - unmap only part of a range obtained through _mali_osk_mem_mapioregion ++ * - unmap more than the range obtained through _mali_osk_mem_mapioregion ++ * - unmap an address range that was not successfully mapped using ++ * _mali_osk_mem_mapioregion ++ * - provide a mapping that does not map to phys. ++ * ++ * @param phys CPU-physical base address of the memory that was originally ++ * mapped in. This must be aligned to the system's page size, which is assumed ++ * to be 4K ++ * @param size The number of bytes that were originally mapped in. ++ * @param mapping The Mali IO address through which the mapping is ++ * accessed. ++ */ ++void _mali_osk_mem_unmapioregion(uintptr_t phys, u32 size, mali_io_address mapping); ++ ++/** @brief Allocate and Map a physically contiguous region into kernel space ++ * ++ * This is used for allocating physically contiguous regions (such as Mali-MMU ++ * page tables) and mapping them into kernel space. The mapping is only ++ * visible from kernel-space. ++ * ++ * The alignment of the returned memory is guaranteed to be at least ++ * _MALI_OSK_CPU_PAGE_SIZE. ++ * ++ * Access must go through _mali_osk_mem_ioread32 and _mali_osk_mem_iowrite32 ++ * ++ * @note This function is primarily to provide support for OSs that are ++ * incapable of separating the tasks 'allocate physically contiguous memory' ++ * and 'map it into kernel space' ++ * ++ * @param[out] phys CPU-physical base address of memory that was allocated. ++ * (*phys) will be guaranteed to be aligned to at least ++ * _MALI_OSK_CPU_PAGE_SIZE on success. ++ * ++ * @param[in] size the number of bytes of physically contiguous memory to ++ * allocate. This must be a multiple of _MALI_OSK_CPU_PAGE_SIZE. ++ * ++ * @return On success, a Mali IO address through which the mapped-in ++ * memory/registers can be accessed. NULL on failure, and (*phys) is unmodified. ++ */ ++mali_io_address _mali_osk_mem_allocioregion(u32 *phys, u32 size); ++ ++/** @brief Free a physically contiguous address range from kernel space. ++ * ++ * The address range should be one previously mapped in through ++ * _mali_osk_mem_allocioregion. ++ * ++ * It is a programming error to do (but not limited to) the following: ++ * - attempt a free twice on the same ioregion ++ * - free only part of a range obtained through _mali_osk_mem_allocioregion ++ * - free more than the range obtained through _mali_osk_mem_allocioregion ++ * - free an address range that was not successfully mapped using ++ * _mali_osk_mem_allocioregion ++ * - provide a mapping that does not map to phys. ++ * ++ * @param phys CPU-physical base address of the memory that was originally ++ * mapped in, which was aligned to _MALI_OSK_CPU_PAGE_SIZE. ++ * @param size The number of bytes that were originally mapped in, which was ++ * a multiple of _MALI_OSK_CPU_PAGE_SIZE. ++ * @param mapping The Mali IO address through which the mapping is ++ * accessed. ++ */ ++void _mali_osk_mem_freeioregion(u32 phys, u32 size, mali_io_address mapping); ++ ++/** @brief Request a region of physically contiguous memory ++ * ++ * This is used to ensure exclusive access to a region of physically contigous ++ * memory. ++ * ++ * It is acceptable to implement this as a stub. However, it is then the job ++ * of the System Integrator to ensure that no other device driver will be using ++ * the physical address ranges used by Mali, while the Mali device driver is ++ * loaded. ++ * ++ * @param phys CPU-physical base address of the memory to request. This must ++ * be aligned to the system's page size, which is assumed to be 4K. ++ * @param size the number of bytes of physically contiguous address space to ++ * request. ++ * @param description A textual description of the memory being requested. ++ * @return _MALI_OSK_ERR_OK on success. Otherwise, a suitable ++ * _mali_osk_errcode_t on failure. ++ */ ++_mali_osk_errcode_t _mali_osk_mem_reqregion(uintptr_t phys, u32 size, const char *description); ++ ++/** @brief Un-request a region of physically contiguous memory ++ * ++ * This is used to release a regious of physically contiguous memory previously ++ * requested through _mali_osk_mem_reqregion, so that other device drivers may ++ * use it. This will be called at time of Mali device driver termination. ++ * ++ * It is a programming error to attempt to: ++ * - unrequest a region twice ++ * - unrequest only part of a range obtained through _mali_osk_mem_reqregion ++ * - unrequest more than the range obtained through _mali_osk_mem_reqregion ++ * - unrequest an address range that was not successfully requested using ++ * _mali_osk_mem_reqregion ++ * ++ * @param phys CPU-physical base address of the memory to un-request. This must ++ * be aligned to the system's page size, which is assumed to be 4K ++ * @param size the number of bytes of physically contiguous address space to ++ * un-request. ++ */ ++void _mali_osk_mem_unreqregion(uintptr_t phys, u32 size); ++ ++/** @brief Read from a location currently mapped in through ++ * _mali_osk_mem_mapioregion ++ * ++ * This reads a 32-bit word from a 32-bit aligned location. It is a programming ++ * error to provide unaligned locations, or to read from memory that is not ++ * mapped in, or not mapped through either _mali_osk_mem_mapioregion() or ++ * _mali_osk_mem_allocioregion(). ++ * ++ * @param mapping Mali IO address to read from ++ * @param offset Byte offset from the given IO address to operate on, must be a multiple of 4 ++ * @return the 32-bit word from the specified location. ++ */ ++u32 _mali_osk_mem_ioread32(volatile mali_io_address mapping, u32 offset); ++ ++/** @brief Write to a location currently mapped in through ++ * _mali_osk_mem_mapioregion without memory barriers ++ * ++ * This write a 32-bit word to a 32-bit aligned location without using memory barrier. ++ * It is a programming error to provide unaligned locations, or to write to memory that is not ++ * mapped in, or not mapped through either _mali_osk_mem_mapioregion() or ++ * _mali_osk_mem_allocioregion(). ++ * ++ * @param mapping Mali IO address to write to ++ * @param offset Byte offset from the given IO address to operate on, must be a multiple of 4 ++ * @param val the 32-bit word to write. ++ */ ++void _mali_osk_mem_iowrite32_relaxed(volatile mali_io_address addr, u32 offset, u32 val); ++ ++/** @brief Write to a location currently mapped in through ++ * _mali_osk_mem_mapioregion with write memory barrier ++ * ++ * This write a 32-bit word to a 32-bit aligned location. It is a programming ++ * error to provide unaligned locations, or to write to memory that is not ++ * mapped in, or not mapped through either _mali_osk_mem_mapioregion() or ++ * _mali_osk_mem_allocioregion(). ++ * ++ * @param mapping Mali IO address to write to ++ * @param offset Byte offset from the given IO address to operate on, must be a multiple of 4 ++ * @param val the 32-bit word to write. ++ */ ++void _mali_osk_mem_iowrite32(volatile mali_io_address mapping, u32 offset, u32 val); ++ ++/** @brief Flush all CPU caches ++ * ++ * This should only be implemented if flushing of the cache is required for ++ * memory mapped in through _mali_osk_mem_mapregion. ++ */ ++void _mali_osk_cache_flushall(void); ++ ++/** @brief Flush any caches necessary for the CPU and MALI to have the same view of a range of uncached mapped memory ++ * ++ * This should only be implemented if your OS doesn't do a full cache flush (inner & outer) ++ * after allocating uncached mapped memory. ++ * ++ * Some OS do not perform a full cache flush (including all outer caches) for uncached mapped memory. ++ * They zero the memory through a cached mapping, then flush the inner caches but not the outer caches. ++ * This is required for MALI to have the correct view of the memory. ++ */ ++void _mali_osk_cache_ensure_uncached_range_flushed(void *uncached_mapping, u32 offset, u32 size); ++ ++/** @brief Safely copy as much data as possible from src to dest ++ * ++ * Do not crash if src or dest isn't available. ++ * ++ * @param dest Destination buffer (limited to user space mapped Mali memory) ++ * @param src Source buffer ++ * @param size Number of bytes to copy ++ * @return Number of bytes actually copied ++ */ ++u32 _mali_osk_mem_write_safe(void *dest, const void *src, u32 size); ++ ++/** @} */ /* end group _mali_osk_low_level_memory */ ++ ++ ++/** @addtogroup _mali_osk_notification ++ * ++ * User space notification framework ++ * ++ * Communication with user space of asynchronous events is performed through a ++ * synchronous call to the \ref u_k_api. ++ * ++ * Since the events are asynchronous, the events have to be queued until a ++ * synchronous U/K API call can be made by user-space. A U/K API call might also ++ * be received before any event has happened. Therefore the notifications the ++ * different subsystems wants to send to user space has to be queued for later ++ * reception, or a U/K API call has to be blocked until an event has occured. ++ * ++ * Typical uses of notifications are after running of jobs on the hardware or ++ * when changes to the system is detected that needs to be relayed to user ++ * space. ++ * ++ * After an event has occured user space has to be notified using some kind of ++ * message. The notification framework supports sending messages to waiting ++ * threads or queueing of messages until a U/K API call is made. ++ * ++ * The notification queue is a FIFO. There are no restrictions on the numbers ++ * of readers or writers in the queue. ++ * ++ * A message contains what user space needs to identifiy how to handle an ++ * event. This includes a type field and a possible type specific payload. ++ * ++ * A notification to user space is represented by a ++ * \ref _mali_osk_notification_t object. A sender gets hold of such an object ++ * using _mali_osk_notification_create(). The buffer given by the ++ * _mali_osk_notification_t::result_buffer field in the object is used to store ++ * any type specific data. The other fields are internal to the queue system ++ * and should not be touched. ++ * ++ * @{ */ ++ ++/** @brief Create a notification object ++ * ++ * Returns a notification object which can be added to the queue of ++ * notifications pending for user space transfer. ++ * ++ * The implementation will initialize all members of the ++ * \ref _mali_osk_notification_t object. In particular, the ++ * _mali_osk_notification_t::result_buffer member will be initialized to point ++ * to \a size bytes of storage, and that storage will be suitably aligned for ++ * storage of any structure. That is, the created buffer meets the same ++ * requirements as _mali_osk_malloc(). ++ * ++ * The notification object must be deleted when not in use. Use ++ * _mali_osk_notification_delete() for deleting it. ++ * ++ * @note You \b must \b not call _mali_osk_free() on a \ref _mali_osk_notification_t, ++ * object, or on a _mali_osk_notification_t::result_buffer. You must only use ++ * _mali_osk_notification_delete() to free the resources assocaited with a ++ * \ref _mali_osk_notification_t object. ++ * ++ * @param type The notification type ++ * @param size The size of the type specific buffer to send ++ * @return Pointer to a notification object with a suitable buffer, or NULL on error. ++ */ ++_mali_osk_notification_t *_mali_osk_notification_create(u32 type, u32 size); ++ ++/** @brief Delete a notification object ++ * ++ * This must be called to reclaim the resources of a notification object. This ++ * includes: ++ * - The _mali_osk_notification_t::result_buffer ++ * - The \ref _mali_osk_notification_t itself. ++ * ++ * A notification object \b must \b not be used after it has been deleted by ++ * _mali_osk_notification_delete(). ++ * ++ * In addition, the notification object may not be deleted while it is in a ++ * queue. That is, if it has been placed on a queue with ++ * _mali_osk_notification_queue_send(), then it must not be deleted until ++ * it has been received by a call to _mali_osk_notification_queue_receive(). ++ * Otherwise, the queue may be corrupted. ++ * ++ * @param object the notification object to delete. ++ */ ++void _mali_osk_notification_delete(_mali_osk_notification_t *object); ++ ++/** @brief Create a notification queue ++ * ++ * Creates a notification queue which can be used to queue messages for user ++ * delivery and get queued messages from ++ * ++ * The queue is a FIFO, and has no restrictions on the numbers of readers or ++ * writers. ++ * ++ * When the queue is no longer in use, it must be terminated with ++ * \ref _mali_osk_notification_queue_term(). Failure to do so will result in a ++ * memory leak. ++ * ++ * @return Pointer to a new notification queue or NULL on error. ++ */ ++_mali_osk_notification_queue_t *_mali_osk_notification_queue_init(void); ++ ++/** @brief Destroy a notification queue ++ * ++ * Destroys a notification queue and frees associated resources from the queue. ++ * ++ * A notification queue \b must \b not be destroyed in the following cases: ++ * - while there are \ref _mali_osk_notification_t objects in the queue. ++ * - while there are writers currently acting upon the queue. That is, while ++ * a thread is currently calling \ref _mali_osk_notification_queue_send() on ++ * the queue, or while a thread may call ++ * \ref _mali_osk_notification_queue_send() on the queue in the future. ++ * - while there are readers currently waiting upon the queue. That is, while ++ * a thread is currently calling \ref _mali_osk_notification_queue_receive() on ++ * the queue, or while a thread may call ++ * \ref _mali_osk_notification_queue_receive() on the queue in the future. ++ * ++ * Therefore, all \ref _mali_osk_notification_t objects must be flushed and ++ * deleted by the code that makes use of the notification queues, since only ++ * they know the structure of the _mali_osk_notification_t::result_buffer ++ * (even if it may only be a flat sturcture). ++ * ++ * @note Since the queue is a FIFO, the code using notification queues may ++ * create its own 'flush' type of notification, to assist in flushing the ++ * queue. ++ * ++ * Once the queue has been destroyed, it must not be used again. ++ * ++ * @param queue The queue to destroy ++ */ ++void _mali_osk_notification_queue_term(_mali_osk_notification_queue_t *queue); ++ ++/** @brief Schedule notification for delivery ++ * ++ * When a \ref _mali_osk_notification_t object has been created successfully ++ * and set up, it may be added to the queue of objects waiting for user space ++ * transfer. ++ * ++ * The sending will not block if the queue is full. ++ * ++ * A \ref _mali_osk_notification_t object \b must \b not be put on two different ++ * queues at the same time, or enqueued twice onto a single queue before ++ * reception. However, it is acceptable for it to be requeued \em after reception ++ * from a call to _mali_osk_notification_queue_receive(), even onto the same queue. ++ * ++ * Again, requeuing must also not enqueue onto two different queues at the same ++ * time, or enqueue onto the same queue twice before reception. ++ * ++ * @param queue The notification queue to add this notification to ++ * @param object The entry to add ++ */ ++void _mali_osk_notification_queue_send(_mali_osk_notification_queue_t *queue, _mali_osk_notification_t *object); ++ ++/** @brief Receive a notification from a queue ++ * ++ * Receives a single notification from the given queue. ++ * ++ * If no notifciations are ready the thread will sleep until one becomes ready. ++ * Therefore, notifications may not be received into an ++ * IRQ or 'atomic' context (that is, a context where sleeping is disallowed). ++ * ++ * @param queue The queue to receive from ++ * @param result Pointer to storage of a pointer of type ++ * \ref _mali_osk_notification_t*. \a result will be written to such that the ++ * expression \a (*result) will evaluate to a pointer to a valid ++ * \ref _mali_osk_notification_t object, or NULL if none were received. ++ * @return _MALI_OSK_ERR_OK on success. _MALI_OSK_ERR_RESTARTSYSCALL if the sleep was interrupted. ++ */ ++_mali_osk_errcode_t _mali_osk_notification_queue_receive(_mali_osk_notification_queue_t *queue, _mali_osk_notification_t **result); ++ ++/** @brief Dequeues a notification from a queue ++ * ++ * Receives a single notification from the given queue. ++ * ++ * If no notifciations are ready the function call will return an error code. ++ * ++ * @param queue The queue to receive from ++ * @param result Pointer to storage of a pointer of type ++ * \ref _mali_osk_notification_t*. \a result will be written to such that the ++ * expression \a (*result) will evaluate to a pointer to a valid ++ * \ref _mali_osk_notification_t object, or NULL if none were received. ++ * @return _MALI_OSK_ERR_OK on success, _MALI_OSK_ERR_ITEM_NOT_FOUND if queue was empty. ++ */ ++_mali_osk_errcode_t _mali_osk_notification_queue_dequeue(_mali_osk_notification_queue_t *queue, _mali_osk_notification_t **result); ++ ++/** @} */ /* end group _mali_osk_notification */ ++ ++ ++/** @addtogroup _mali_osk_timer ++ * ++ * Timers use the OS's representation of time, which are 'ticks'. This is to ++ * prevent aliasing problems between the internal timer time, and the time ++ * asked for. ++ * ++ * @{ */ ++ ++/** @brief Initialize a timer ++ * ++ * Allocates resources for a new timer, and initializes them. This does not ++ * start the timer. ++ * ++ * @return a pointer to the allocated timer object, or NULL on failure. ++ */ ++_mali_osk_timer_t *_mali_osk_timer_init(_mali_osk_timer_callback_t callback); ++ ++/** @brief Start a timer ++ * ++ * It is an error to start a timer without setting the callback via ++ * _mali_osk_timer_setcallback(). ++ * ++ * It is an error to use this to start an already started timer. ++ * ++ * The timer will expire in \a ticks_to_expire ticks, at which point, the ++ * callback function will be invoked with the callback-specific data, ++ * as registered by _mali_osk_timer_setcallback(). ++ * ++ * @param tim the timer to start ++ * @param ticks_to_expire the amount of time in ticks for the timer to run ++ * before triggering. ++ */ ++void _mali_osk_timer_add(_mali_osk_timer_t *tim, unsigned long ticks_to_expire); ++ ++/** @brief Modify a timer ++ * ++ * Set the relative time at which a timer will expire, and start it if it is ++ * stopped. If \a ticks_to_expire 0 the timer fires immediately. ++ * ++ * It is an error to modify a timer without setting the callback via ++ * _mali_osk_timer_setcallback(). ++ * ++ * The timer will expire at \a ticks_to_expire from the time of the call, at ++ * which point, the callback function will be invoked with the ++ * callback-specific data, as set by _mali_osk_timer_setcallback(). ++ * ++ * @param tim the timer to modify, and start if necessary ++ * @param ticks_to_expire the \em absolute time in ticks at which this timer ++ * should trigger. ++ * ++ */ ++void _mali_osk_timer_mod(_mali_osk_timer_t *tim, unsigned long ticks_to_expire); ++ ++/** @brief Stop a timer, and block on its completion. ++ * ++ * Stop the timer. When the function returns, it is guaranteed that the timer's ++ * callback will not be running on any CPU core. ++ * ++ * Since stoping the timer blocks on compeletion of the callback, the callback ++ * may not obtain any mutexes that the caller holds. Otherwise, a deadlock will ++ * occur. ++ * ++ * @note While the callback itself is guaranteed to not be running, work ++ * enqueued on the work-queue by the timer (with ++ * \ref _mali_osk_wq_schedule_work()) may still run. The timer callback and ++ * work handler must take this into account. ++ * ++ * It is legal to stop an already stopped timer. ++ * ++ * @param tim the timer to stop. ++ * ++ */ ++void _mali_osk_timer_del(_mali_osk_timer_t *tim); ++ ++/** @brief Stop a timer. ++ * ++ * Stop the timer. When the function returns, the timer's callback may still be ++ * running on any CPU core. ++ * ++ * It is legal to stop an already stopped timer. ++ * ++ * @param tim the timer to stop. ++ */ ++void _mali_osk_timer_del_async(_mali_osk_timer_t *tim); ++ ++/** @brief Check if timer is pending. ++ * ++ * Check if timer is active. ++ * ++ * @param tim the timer to check ++ * @return MALI_TRUE if time is active, MALI_FALSE if it is not active ++ */ ++mali_bool _mali_osk_timer_pending(_mali_osk_timer_t *tim); ++ ++/** @brief Set a timer's callback parameters. ++ * ++ * This must be called at least once before a timer is started/modified. ++ * ++ * After a timer has been stopped or expires, the callback remains set. This ++ * means that restarting the timer will call the same function with the same ++ * parameters on expiry. ++ * ++ * @param tim the timer to set callback on. ++ * @param callback Function to call when timer expires ++ * @param data Function-specific data to supply to the function on expiry. ++ */ ++void _mali_osk_timer_setcallback(_mali_osk_timer_t *tim, _mali_osk_timer_callback_t callback, void *data); ++ ++/** @brief Terminate a timer, and deallocate resources. ++ * ++ * The timer must first be stopped by calling _mali_osk_timer_del(). ++ * ++ * It is a programming error for _mali_osk_timer_term() to be called on: ++ * - timer that is currently running ++ * - a timer that is currently executing its callback. ++ * ++ * @param tim the timer to deallocate. ++ */ ++void _mali_osk_timer_term(_mali_osk_timer_t *tim); ++/** @} */ /* end group _mali_osk_timer */ ++ ++ ++/** @defgroup _mali_osk_time OSK Time functions ++ * ++ * \ref _mali_osk_time use the OS's representation of time, which are ++ * 'ticks'. This is to prevent aliasing problems between the internal timer ++ * time, and the time asked for. ++ * ++ * OS tick time is measured as a u32. The time stored in a u32 may either be ++ * an absolute time, or a time delta between two events. Whilst it is valid to ++ * use math opeartors to \em change the tick value represented as a u32, it ++ * is often only meaningful to do such operations on time deltas, rather than ++ * on absolute time. However, it is meaningful to add/subtract time deltas to ++ * absolute times. ++ * ++ * Conversion between tick time and milliseconds (ms) may not be loss-less, ++ * and are \em implementation \em depenedant. ++ * ++ * Code use OS time must take this into account, since: ++ * - a small OS time may (or may not) be rounded ++ * - a large time may (or may not) overflow ++ * ++ * @{ */ ++ ++/** @brief Return whether ticka occurs after or at the same time as tickb ++ * ++ * Systems where ticks can wrap must handle that. ++ * ++ * @param ticka ticka ++ * @param tickb tickb ++ * @return MALI_TRUE if ticka represents a time that occurs at or after tickb. ++ */ ++mali_bool _mali_osk_time_after_eq(unsigned long ticka, unsigned long tickb); ++ ++/** @brief Convert milliseconds to OS 'ticks' ++ * ++ * @param ms time interval in milliseconds ++ * @return the corresponding time interval in OS ticks. ++ */ ++unsigned long _mali_osk_time_mstoticks(u32 ms); ++ ++/** @brief Convert OS 'ticks' to milliseconds ++ * ++ * @param ticks time interval in OS ticks. ++ * @return the corresponding time interval in milliseconds ++ */ ++u32 _mali_osk_time_tickstoms(unsigned long ticks); ++ ++ ++/** @brief Get the current time in OS 'ticks'. ++ * @return the current time in OS 'ticks'. ++ */ ++unsigned long _mali_osk_time_tickcount(void); ++ ++/** @brief Cause a microsecond delay ++ * ++ * The delay will have microsecond resolution, and is necessary for correct ++ * operation of the driver. At worst, the delay will be \b at least \a usecs ++ * microseconds, and so may be (significantly) more. ++ * ++ * This function may be implemented as a busy-wait, which is the most sensible ++ * implementation. On OSs where there are situations in which a thread must not ++ * sleep, this is definitely implemented as a busy-wait. ++ * ++ * @param usecs the number of microseconds to wait for. ++ */ ++void _mali_osk_time_ubusydelay(u32 usecs); ++ ++/** @brief Return time in nano seconds, since any given reference. ++ * ++ * @return Time in nano seconds ++ */ ++u64 _mali_osk_time_get_ns(void); ++ ++/** @brief Return time in nano seconds, since boot time. ++ * ++ * @return Time in nano seconds ++ */ ++u64 _mali_osk_boot_time_get_ns(void); ++ ++/** @} */ /* end group _mali_osk_time */ ++ ++/** @defgroup _mali_osk_math OSK Math ++ * @{ */ ++ ++/** @brief Count Leading Zeros (Little-endian) ++ * ++ * @note This function must be implemented to support the reference ++ * implementation of _mali_osk_find_first_zero_bit, as defined in ++ * mali_osk_bitops.h. ++ * ++ * @param val 32-bit words to count leading zeros on ++ * @return the number of leading zeros. ++ */ ++u32 _mali_osk_clz(u32 val); ++ ++/** @brief find last (most-significant) bit set ++ * ++ * @param val 32-bit words to count last bit set on ++ * @return last bit set. ++ */ ++u32 _mali_osk_fls(u32 val); ++ ++/** @} */ /* end group _mali_osk_math */ ++ ++/** @addtogroup _mali_osk_wait_queue OSK Wait Queue functionality ++ * @{ */ ++ ++/** @brief Initialize an empty Wait Queue */ ++_mali_osk_wait_queue_t *_mali_osk_wait_queue_init(void); ++ ++/** @brief Sleep if condition is false ++ * ++ * @param queue the queue to use ++ * @param condition function pointer to a boolean function ++ * @param data data parameter for condition function ++ * ++ * Put thread to sleep if the given \a condition function returns false. When ++ * being asked to wake up again, the condition will be re-checked and the ++ * thread only woken up if the condition is now true. ++ */ ++void _mali_osk_wait_queue_wait_event(_mali_osk_wait_queue_t *queue, mali_bool(*condition)(void *), void *data); ++ ++/** @brief Sleep if condition is false ++ * ++ * @param queue the queue to use ++ * @param condition function pointer to a boolean function ++ * @param data data parameter for condition function ++ * @param timeout timeout in ms ++ * ++ * Put thread to sleep if the given \a condition function returns false. When ++ * being asked to wake up again, the condition will be re-checked and the ++ * thread only woken up if the condition is now true. Will return if time ++ * exceeds timeout. ++ */ ++void _mali_osk_wait_queue_wait_event_timeout(_mali_osk_wait_queue_t *queue, mali_bool(*condition)(void *), void *data, u32 timeout); ++ ++/** @brief Wake up all threads in wait queue if their respective conditions are ++ * true ++ * ++ * @param queue the queue whose threads should be woken up ++ * ++ * Wake up all threads in wait queue \a queue whose condition is now true. ++ */ ++void _mali_osk_wait_queue_wake_up(_mali_osk_wait_queue_t *queue); ++ ++/** @brief terminate a wait queue ++ * ++ * @param queue the queue to terminate. ++ */ ++void _mali_osk_wait_queue_term(_mali_osk_wait_queue_t *queue); ++/** @} */ /* end group _mali_osk_wait_queue */ ++ ++ ++/** @addtogroup _mali_osk_miscellaneous ++ * @{ */ ++ ++/** @brief Output a device driver debug message. ++ * ++ * The interpretation of \a fmt is the same as the \c format parameter in ++ * _mali_osu_vsnprintf(). ++ * ++ * @param fmt a _mali_osu_vsnprintf() style format string ++ * @param ... a variable-number of parameters suitable for \a fmt ++ */ ++void _mali_osk_dbgmsg(const char *fmt, ...); ++ ++/** @brief Print fmt into buf. ++ * ++ * The interpretation of \a fmt is the same as the \c format parameter in ++ * _mali_osu_vsnprintf(). ++ * ++ * @param buf a pointer to the result buffer ++ * @param size the total number of bytes allowed to write to \a buf ++ * @param fmt a _mali_osu_vsnprintf() style format string ++ * @param ... a variable-number of parameters suitable for \a fmt ++ * @return The number of bytes written to \a buf ++ */ ++u32 _mali_osk_snprintf(char *buf, u32 size, const char *fmt, ...); ++ ++/** @brief Abnormal process abort. ++ * ++ * Terminates the caller-process if this function is called. ++ * ++ * This function will be called from Debug assert-macros in mali_kernel_common.h. ++ * ++ * This function will never return - because to continue from a Debug assert ++ * could cause even more problems, and hinder debugging of the initial problem. ++ * ++ * This function is only used in Debug builds, and is not used in Release builds. ++ */ ++void _mali_osk_abort(void); ++ ++/** @brief Sets breakpoint at point where function is called. ++ * ++ * This function will be called from Debug assert-macros in mali_kernel_common.h, ++ * to assist in debugging. If debugging at this level is not required, then this ++ * function may be implemented as a stub. ++ * ++ * This function is only used in Debug builds, and is not used in Release builds. ++ */ ++void _mali_osk_break(void); ++ ++/** @brief Return an identificator for calling process. ++ * ++ * @return Identificator for calling process. ++ */ ++u32 _mali_osk_get_pid(void); ++ ++/** @brief Return an name for calling process. ++ * ++ * @return name for calling process. ++ */ ++char *_mali_osk_get_comm(void); ++ ++/** @brief Return an identificator for calling thread. ++ * ++ * @return Identificator for calling thread. ++ */ ++u32 _mali_osk_get_tid(void); ++ ++ ++/** @brief Take a reference to the power manager system for the Mali device (synchronously). ++ * ++ * When function returns successfully, Mali is ON. ++ * ++ * @note Call \a _mali_osk_pm_dev_ref_put() to release this reference. ++ */ ++_mali_osk_errcode_t _mali_osk_pm_dev_ref_get_sync(void); ++ ++/** @brief Take a reference to the external power manager system for the Mali device (asynchronously). ++ * ++ * Mali might not yet be on after this function as returned. ++ * Please use \a _mali_osk_pm_dev_barrier() or \a _mali_osk_pm_dev_ref_get_sync() ++ * to wait for Mali to be powered on. ++ * ++ * @note Call \a _mali_osk_pm_dev_ref_dec() to release this reference. ++ */ ++_mali_osk_errcode_t _mali_osk_pm_dev_ref_get_async(void); ++ ++/** @brief Release the reference to the external power manger system for the Mali device. ++ * ++ * When reference count reach zero, the cores can be off. ++ * ++ * @note This must be used to release references taken with ++ * \a _mali_osk_pm_dev_ref_get_sync() or \a _mali_osk_pm_dev_ref_get_sync(). ++ */ ++void _mali_osk_pm_dev_ref_put(void); ++ ++/** @brief Block until pending PM operations are done ++ */ ++void _mali_osk_pm_dev_barrier(void); ++ ++/** @} */ /* end group _mali_osk_miscellaneous */ ++ ++/** @defgroup _mali_osk_bitmap OSK Bitmap ++ * @{ */ ++ ++/** @brief Allocate a unique number from the bitmap object. ++ * ++ * @param bitmap Initialized bitmap object. ++ * @return An unique existence in the bitmap object. ++ */ ++u32 _mali_osk_bitmap_alloc(struct _mali_osk_bitmap *bitmap); ++ ++/** @brief Free a interger to the bitmap object. ++ * ++ * @param bitmap Initialized bitmap object. ++ * @param obj An number allocated from bitmap object. ++ */ ++void _mali_osk_bitmap_free(struct _mali_osk_bitmap *bitmap, u32 obj); ++ ++/** @brief Allocate continuous number from the bitmap object. ++ * ++ * @param bitmap Initialized bitmap object. ++ * @return start number of the continuous number block. ++ */ ++u32 _mali_osk_bitmap_alloc_range(struct _mali_osk_bitmap *bitmap, int cnt); ++ ++/** @brief Free a block of continuous number block to the bitmap object. ++ * ++ * @param bitmap Initialized bitmap object. ++ * @param obj Start number. ++ * @param cnt The size of the continuous number block. ++ */ ++void _mali_osk_bitmap_free_range(struct _mali_osk_bitmap *bitmap, u32 obj, int cnt); ++ ++/** @brief Available count could be used to allocate in the given bitmap object. ++ * ++ */ ++u32 _mali_osk_bitmap_avail(struct _mali_osk_bitmap *bitmap); ++ ++/** @brief Initialize an bitmap object.. ++ * ++ * @param bitmap An poiter of uninitialized bitmap object. ++ * @param num Size of thei bitmap object and decide the memory size allocated. ++ * @param reserve start number used to allocate. ++ */ ++int _mali_osk_bitmap_init(struct _mali_osk_bitmap *bitmap, u32 num, u32 reserve); ++ ++/** @brief Free the given bitmap object. ++ * ++ * @param bitmap Initialized bitmap object. ++ */ ++void _mali_osk_bitmap_term(struct _mali_osk_bitmap *bitmap); ++/** @} */ /* end group _mali_osk_bitmap */ ++ ++/** @} */ /* end group osuapi */ ++ ++/** @} */ /* end group uddapi */ ++ ++ ++ ++#ifdef __cplusplus ++} ++#endif ++ ++/* Check standard inlines */ ++#ifndef MALI_STATIC_INLINE ++#error MALI_STATIC_INLINE not defined on your OS ++#endif ++ ++#ifndef MALI_NON_STATIC_INLINE ++#error MALI_NON_STATIC_INLINE not defined on your OS ++#endif ++ ++#endif /* __MALI_OSK_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_osk_bitops.h b/drivers/gpu/arm/mali400/mali/common/mali_osk_bitops.h +new file mode 100755 +index 000000000000..bb1831753a40 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_osk_bitops.h +@@ -0,0 +1,162 @@ ++/* ++ * Copyright (C) 2010, 2013-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file mali_osk_bitops.h ++ * Implementation of the OS abstraction layer for the kernel device driver ++ */ ++ ++#ifndef __MALI_OSK_BITOPS_H__ ++#define __MALI_OSK_BITOPS_H__ ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++MALI_STATIC_INLINE void _mali_internal_clear_bit(u32 bit, u32 *addr) ++{ ++ MALI_DEBUG_ASSERT(bit < 32); ++ MALI_DEBUG_ASSERT(NULL != addr); ++ ++ (*addr) &= ~(1 << bit); ++} ++ ++MALI_STATIC_INLINE void _mali_internal_set_bit(u32 bit, u32 *addr) ++{ ++ MALI_DEBUG_ASSERT(bit < 32); ++ MALI_DEBUG_ASSERT(NULL != addr); ++ ++ (*addr) |= (1 << bit); ++} ++ ++MALI_STATIC_INLINE u32 _mali_internal_test_bit(u32 bit, u32 value) ++{ ++ MALI_DEBUG_ASSERT(bit < 32); ++ return value & (1 << bit); ++} ++ ++MALI_STATIC_INLINE int _mali_internal_find_first_zero_bit(u32 value) ++{ ++ u32 inverted; ++ u32 negated; ++ u32 isolated; ++ u32 leading_zeros; ++ ++ /* Begin with xxx...x0yyy...y, where ys are 1, number of ys is in range 0..31 */ ++ inverted = ~value; /* zzz...z1000...0 */ ++ /* Using count_trailing_zeros on inverted value - ++ * See ARM System Developers Guide for details of count_trailing_zeros */ ++ ++ /* Isolate the zero: it is preceeded by a run of 1s, so add 1 to it */ ++ negated = (u32) - inverted ; /* -a == ~a + 1 (mod 2^n) for n-bit numbers */ ++ /* negated = xxx...x1000...0 */ ++ ++ isolated = negated & inverted ; /* xxx...x1000...0 & zzz...z1000...0, zs are ~xs */ ++ /* And so the first zero bit is in the same position as the 1 == number of 1s that preceeded it ++ * Note that the output is zero if value was all 1s */ ++ ++ leading_zeros = _mali_osk_clz(isolated); ++ ++ return 31 - leading_zeros; ++} ++ ++ ++/** @defgroup _mali_osk_bitops OSK Non-atomic Bit-operations ++ * @{ */ ++ ++/** ++ * These bit-operations do not work atomically, and so locks must be used if ++ * atomicity is required. ++ * ++ * Reference implementations for Little Endian are provided, and so it should ++ * not normally be necessary to re-implement these. Efficient bit-twiddling ++ * techniques are used where possible, implemented in portable C. ++ * ++ * Note that these reference implementations rely on _mali_osk_clz() being ++ * implemented. ++ */ ++ ++/** @brief Clear a bit in a sequence of 32-bit words ++ * @param nr bit number to clear, starting from the (Little-endian) least ++ * significant bit ++ * @param addr starting point for counting. ++ */ ++MALI_STATIC_INLINE void _mali_osk_clear_nonatomic_bit(u32 nr, u32 *addr) ++{ ++ addr += nr >> 5; /* find the correct word */ ++ nr = nr & ((1 << 5) - 1); /* The bit number within the word */ ++ ++ _mali_internal_clear_bit(nr, addr); ++} ++ ++/** @brief Set a bit in a sequence of 32-bit words ++ * @param nr bit number to set, starting from the (Little-endian) least ++ * significant bit ++ * @param addr starting point for counting. ++ */ ++MALI_STATIC_INLINE void _mali_osk_set_nonatomic_bit(u32 nr, u32 *addr) ++{ ++ addr += nr >> 5; /* find the correct word */ ++ nr = nr & ((1 << 5) - 1); /* The bit number within the word */ ++ ++ _mali_internal_set_bit(nr, addr); ++} ++ ++/** @brief Test a bit in a sequence of 32-bit words ++ * @param nr bit number to test, starting from the (Little-endian) least ++ * significant bit ++ * @param addr starting point for counting. ++ * @return zero if bit was clear, non-zero if set. Do not rely on the return ++ * value being related to the actual word under test. ++ */ ++MALI_STATIC_INLINE u32 _mali_osk_test_bit(u32 nr, u32 *addr) ++{ ++ addr += nr >> 5; /* find the correct word */ ++ nr = nr & ((1 << 5) - 1); /* The bit number within the word */ ++ ++ return _mali_internal_test_bit(nr, *addr); ++} ++ ++/* Return maxbit if not found */ ++/** @brief Find the first zero bit in a sequence of 32-bit words ++ * @param addr starting point for search. ++ * @param maxbit the maximum number of bits to search ++ * @return the number of the first zero bit found, or maxbit if none were found ++ * in the specified range. ++ */ ++MALI_STATIC_INLINE u32 _mali_osk_find_first_zero_bit(const u32 *addr, u32 maxbit) ++{ ++ u32 total; ++ ++ for (total = 0; total < maxbit; total += 32, ++addr) { ++ int result; ++ result = _mali_internal_find_first_zero_bit(*addr); ++ ++ /* non-negative signifies the bit was found */ ++ if (result >= 0) { ++ total += (u32)result; ++ break; ++ } ++ } ++ ++ /* Now check if we reached maxbit or above */ ++ if (total >= maxbit) { ++ total = maxbit; ++ } ++ ++ return total; /* either the found bit nr, or maxbit if not found */ ++} ++/** @} */ /* end group _mali_osk_bitops */ ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif /* __MALI_OSK_BITOPS_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_osk_list.h b/drivers/gpu/arm/mali400/mali/common/mali_osk_list.h +new file mode 100755 +index 000000000000..9af2d7d4d621 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_osk_list.h +@@ -0,0 +1,273 @@ ++/* ++ * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file mali_osk_list.h ++ * Implementation of the OS abstraction layer for the kernel device driver ++ */ ++ ++#ifndef __MALI_OSK_LIST_H__ ++#define __MALI_OSK_LIST_H__ ++ ++#include "mali_osk.h" ++#include "mali_kernel_common.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++MALI_STATIC_INLINE void __mali_osk_list_add(_mali_osk_list_t *new_entry, _mali_osk_list_t *prev, _mali_osk_list_t *next) ++{ ++ next->prev = new_entry; ++ new_entry->next = next; ++ new_entry->prev = prev; ++ prev->next = new_entry; ++} ++ ++MALI_STATIC_INLINE void __mali_osk_list_del(_mali_osk_list_t *prev, _mali_osk_list_t *next) ++{ ++ next->prev = prev; ++ prev->next = next; ++} ++ ++/** @addtogroup _mali_osk_list OSK Doubly-Linked Circular Lists ++ * @{ */ ++ ++/** Reference implementations of Doubly-linked Circular Lists are provided. ++ * There is often no need to re-implement these. ++ * ++ * @note The implementation may differ subtly from any lists the OS provides. ++ * For this reason, these lists should not be mixed with OS-specific lists ++ * inside the OSK/UKK implementation. */ ++ ++/** @brief Initialize a list to be a head of an empty list ++ * @param exp the list to initialize. */ ++#define _MALI_OSK_INIT_LIST_HEAD(exp) _mali_osk_list_init(exp) ++ ++/** @brief Define a list variable, which is uninitialized. ++ * @param exp the name of the variable that the list will be defined as. */ ++#define _MALI_OSK_LIST_HEAD(exp) _mali_osk_list_t exp ++ ++/** @brief Define a list variable, which is initialized. ++ * @param exp the name of the variable that the list will be defined as. */ ++#define _MALI_OSK_LIST_HEAD_STATIC_INIT(exp) _mali_osk_list_t exp = { &exp, &exp } ++ ++/** @brief Initialize a list element. ++ * ++ * All list elements must be initialized before use. ++ * ++ * Do not use on any list element that is present in a list without using ++ * _mali_osk_list_del first, otherwise this will break the list. ++ * ++ * @param list the list element to initialize ++ */ ++MALI_STATIC_INLINE void _mali_osk_list_init(_mali_osk_list_t *list) ++{ ++ list->next = list; ++ list->prev = list; ++} ++ ++/** @brief Insert a single list element after an entry in a list ++ * ++ * As an example, if this is inserted to the head of a list, then this becomes ++ * the first element of the list. ++ * ++ * Do not use to move list elements from one list to another, as it will break ++ * the originating list. ++ * ++ * ++ * @param newlist the list element to insert ++ * @param list the list in which to insert. The new element will be the next ++ * entry in this list ++ */ ++MALI_STATIC_INLINE void _mali_osk_list_add(_mali_osk_list_t *new_entry, _mali_osk_list_t *list) ++{ ++ __mali_osk_list_add(new_entry, list, list->next); ++} ++ ++/** @brief Insert a single list element before an entry in a list ++ * ++ * As an example, if this is inserted to the head of a list, then this becomes ++ * the last element of the list. ++ * ++ * Do not use to move list elements from one list to another, as it will break ++ * the originating list. ++ * ++ * @param newlist the list element to insert ++ * @param list the list in which to insert. The new element will be the previous ++ * entry in this list ++ */ ++MALI_STATIC_INLINE void _mali_osk_list_addtail(_mali_osk_list_t *new_entry, _mali_osk_list_t *list) ++{ ++ __mali_osk_list_add(new_entry, list->prev, list); ++} ++ ++/** @brief Remove a single element from a list ++ * ++ * The element will no longer be present in the list. The removed list element ++ * will be uninitialized, and so should not be traversed. It must be ++ * initialized before further use. ++ * ++ * @param list the list element to remove. ++ */ ++MALI_STATIC_INLINE void _mali_osk_list_del(_mali_osk_list_t *list) ++{ ++ __mali_osk_list_del(list->prev, list->next); ++} ++ ++/** @brief Remove a single element from a list, and re-initialize it ++ * ++ * The element will no longer be present in the list. The removed list element ++ * will initialized, and so can be used as normal. ++ * ++ * @param list the list element to remove and initialize. ++ */ ++MALI_STATIC_INLINE void _mali_osk_list_delinit(_mali_osk_list_t *list) ++{ ++ __mali_osk_list_del(list->prev, list->next); ++ _mali_osk_list_init(list); ++} ++ ++/** @brief Determine whether a list is empty. ++ * ++ * An empty list is one that contains a single element that points to itself. ++ * ++ * @param list the list to check. ++ * @return non-zero if the list is empty, and zero otherwise. ++ */ ++MALI_STATIC_INLINE mali_bool _mali_osk_list_empty(_mali_osk_list_t *list) ++{ ++ return list->next == list; ++} ++ ++/** @brief Move a list element from one list to another. ++ * ++ * The list element must be initialized. ++ * ++ * As an example, moving a list item to the head of a new list causes this item ++ * to be the first element in the new list. ++ * ++ * @param move the list element to move ++ * @param list the new list into which the element will be inserted, as the next ++ * element in the list. ++ */ ++MALI_STATIC_INLINE void _mali_osk_list_move(_mali_osk_list_t *move_entry, _mali_osk_list_t *list) ++{ ++ __mali_osk_list_del(move_entry->prev, move_entry->next); ++ _mali_osk_list_add(move_entry, list); ++} ++ ++/** @brief Move an entire list ++ * ++ * The list element must be initialized. ++ * ++ * Allows you to move a list from one list head to another list head ++ * ++ * @param old_list The existing list head ++ * @param new_list The new list head (must be an empty list) ++ */ ++MALI_STATIC_INLINE void _mali_osk_list_move_list(_mali_osk_list_t *old_list, _mali_osk_list_t *new_list) ++{ ++ MALI_DEBUG_ASSERT(_mali_osk_list_empty(new_list)); ++ if (!_mali_osk_list_empty(old_list)) { ++ new_list->next = old_list->next; ++ new_list->prev = old_list->prev; ++ new_list->next->prev = new_list; ++ new_list->prev->next = new_list; ++ old_list->next = old_list; ++ old_list->prev = old_list; ++ } ++} ++ ++/** @brief Find the containing structure of a list ++ * ++ * When traversing a list, this is used to recover the containing structure, ++ * given that is contains a _mali_osk_list_t member. ++ * ++ * Each list must be of structures of one type, and must link the same members ++ * together, otherwise it will not be possible to correctly recover the ++ * sturctures that the lists link. ++ * ++ * @note no type or memory checking occurs to ensure that a structure does in ++ * fact exist for the list entry, and that it is being recovered with respect ++ * to the correct list member. ++ * ++ * @param ptr the pointer to the _mali_osk_list_t member in this structure ++ * @param type the type of the structure that contains the member ++ * @param member the member of the structure that ptr points to. ++ * @return a pointer to a \a type object which contains the _mali_osk_list_t ++ * \a member, as pointed to by the _mali_osk_list_t \a *ptr. ++ */ ++#define _MALI_OSK_LIST_ENTRY(ptr, type, member) \ ++ _MALI_OSK_CONTAINER_OF(ptr, type, member) ++ ++/** @brief Enumerate a list safely ++ * ++ * With this macro, lists can be enumerated in a 'safe' manner. That is, ++ * entries can be deleted from the list without causing an error during ++ * enumeration. To achieve this, a 'temporary' pointer is required, which must ++ * be provided to the macro. ++ * ++ * Use it like a 'for()', 'while()' or 'do()' construct, and so it must be ++ * followed by a statement or compound-statement which will be executed for ++ * each list entry. ++ * ++ * Upon loop completion, providing that an early out was not taken in the ++ * loop body, then it is guaranteed that ptr->member == list, even if the loop ++ * body never executed. ++ * ++ * @param ptr a pointer to an object of type 'type', which points to the ++ * structure that contains the currently enumerated list entry. ++ * @param tmp a pointer to an object of type 'type', which must not be used ++ * inside the list-execution statement. ++ * @param list a pointer to a _mali_osk_list_t, from which enumeration will ++ * begin ++ * @param type the type of the structure that contains the _mali_osk_list_t ++ * member that is part of the list to be enumerated. ++ * @param member the _mali_osk_list_t member of the structure that is part of ++ * the list to be enumerated. ++ */ ++#define _MALI_OSK_LIST_FOREACHENTRY(ptr, tmp, list, type, member) \ ++ for (ptr = _MALI_OSK_LIST_ENTRY((list)->next, type, member), \ ++ tmp = _MALI_OSK_LIST_ENTRY(ptr->member.next, type, member); \ ++ &ptr->member != (list); \ ++ ptr = tmp, \ ++ tmp = _MALI_OSK_LIST_ENTRY(tmp->member.next, type, member)) ++ ++/** @brief Enumerate a list in reverse order safely ++ * ++ * This macro is identical to @ref _MALI_OSK_LIST_FOREACHENTRY, except that ++ * entries are enumerated in reverse order. ++ * ++ * @param ptr a pointer to an object of type 'type', which points to the ++ * structure that contains the currently enumerated list entry. ++ * @param tmp a pointer to an object of type 'type', which must not be used ++ * inside the list-execution statement. ++ * @param list a pointer to a _mali_osk_list_t, from which enumeration will ++ * begin ++ * @param type the type of the structure that contains the _mali_osk_list_t ++ * member that is part of the list to be enumerated. ++ * @param member the _mali_osk_list_t member of the structure that is part of ++ * the list to be enumerated. ++ */ ++#define _MALI_OSK_LIST_FOREACHENTRY_REVERSE(ptr, tmp, list, type, member) \ ++ for (ptr = _MALI_OSK_LIST_ENTRY((list)->prev, type, member), \ ++ tmp = _MALI_OSK_LIST_ENTRY(ptr->member.prev, type, member); \ ++ &ptr->member != (list); \ ++ ptr = tmp, \ ++ tmp = _MALI_OSK_LIST_ENTRY(tmp->member.prev, type, member)) ++ ++/** @} */ /* end group _mali_osk_list */ ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif /* __MALI_OSK_LIST_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_osk_mali.h b/drivers/gpu/arm/mali400/mali/common/mali_osk_mali.h +new file mode 100755 +index 000000000000..bf69925a43a7 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_osk_mali.h +@@ -0,0 +1,152 @@ ++/* ++ * Copyright (C) 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file mali_osk_mali.h ++ * Defines the OS abstraction layer which is specific for the Mali kernel device driver (OSK) ++ */ ++ ++#ifndef __MALI_OSK_MALI_H__ ++#define __MALI_OSK_MALI_H__ ++ ++#include ++#include ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++#ifdef CONFIG_MALI_DEVFREQ ++struct mali_device { ++ struct device *dev; ++#ifdef CONFIG_HAVE_CLK ++ struct clk *clock; ++#endif ++#ifdef CONFIG_REGULATOR ++ struct regulator *regulator; ++#endif ++#ifdef CONFIG_PM_DEVFREQ ++ struct devfreq_dev_profile devfreq_profile; ++ struct devfreq *devfreq; ++ unsigned long current_freq; ++ unsigned long current_voltage; ++ struct monitor_dev_info *mdev_info; ++#ifdef CONFIG_DEVFREQ_THERMAL ++ struct thermal_cooling_device *devfreq_cooling; ++#endif ++#endif ++ struct mali_pm_metrics_data mali_metrics; ++}; ++#endif ++ ++/** @addtogroup _mali_osk_miscellaneous ++ * @{ */ ++ ++/** @brief Struct with device specific configuration data ++ */ ++typedef struct mali_gpu_device_data _mali_osk_device_data; ++ ++#ifdef CONFIG_MALI_DT ++/** @brief Initialize those device resources when we use device tree ++ * ++ * @return _MALI_OSK_ERR_OK on success, otherwise failure. ++ */ ++_mali_osk_errcode_t _mali_osk_resource_initialize(void); ++#endif ++ ++/** @brief Find Mali GPU HW resource ++ * ++ * @param addr Address of Mali GPU resource to find ++ * @param res Storage for resource information if resource is found. ++ * @return _MALI_OSK_ERR_OK on success, _MALI_OSK_ERR_ITEM_NOT_FOUND if resource is not found ++ */ ++_mali_osk_errcode_t _mali_osk_resource_find(u32 addr, _mali_osk_resource_t *res); ++ ++ ++/** @brief Find Mali GPU HW base address ++ * ++ * @return 0 if resources are found, otherwise the Mali GPU component with lowest address. ++ */ ++uintptr_t _mali_osk_resource_base_address(void); ++ ++/** @brief Find the specific GPU resource. ++ * ++ * @return value ++ * 0x400 if Mali 400 specific GPU resource identified ++ * 0x450 if Mali 450 specific GPU resource identified ++ * 0x470 if Mali 470 specific GPU resource identified ++ * ++ */ ++u32 _mali_osk_identify_gpu_resource(void); ++ ++/** @brief Retrieve the Mali GPU specific data ++ * ++ * @return _MALI_OSK_ERR_OK on success, otherwise failure. ++ */ ++_mali_osk_errcode_t _mali_osk_device_data_get(_mali_osk_device_data *data); ++ ++/** @brief Find the pmu domain config from device data. ++ * ++ * @param domain_config_array used to store pmu domain config found in device data. ++ * @param array_size is the size of array domain_config_array. ++ */ ++void _mali_osk_device_data_pmu_config_get(u16 *domain_config_array, int array_size); ++ ++/** @brief Get Mali PMU switch delay ++ * ++ *@return pmu switch delay if it is configured ++ */ ++u32 _mali_osk_get_pmu_switch_delay(void); ++ ++/** @brief Determines if Mali GPU has been configured with shared interrupts. ++ * ++ * @return MALI_TRUE if shared interrupts, MALI_FALSE if not. ++ */ ++mali_bool _mali_osk_shared_interrupts(void); ++ ++/** @brief Initialize the gpu secure mode. ++ * The gpu secure mode will initially be in a disabled state. ++ * @return _MALI_OSK_ERR_OK on success, otherwise failure. ++ */ ++_mali_osk_errcode_t _mali_osk_gpu_secure_mode_init(void); ++ ++/** @brief Deinitialize the gpu secure mode. ++ * @return _MALI_OSK_ERR_OK on success, otherwise failure. ++ */ ++_mali_osk_errcode_t _mali_osk_gpu_secure_mode_deinit(void); ++ ++/** @brief Reset GPU and enable the gpu secure mode. ++ * @return _MALI_OSK_ERR_OK on success, otherwise failure. ++ */ ++_mali_osk_errcode_t _mali_osk_gpu_reset_and_secure_mode_enable(void); ++ ++/** @brief Reset GPU and disable the gpu secure mode. ++ * @return _MALI_OSK_ERR_OK on success, otherwise failure. ++ */ ++_mali_osk_errcode_t _mali_osk_gpu_reset_and_secure_mode_disable(void); ++ ++/** @brief Check if the gpu secure mode has been enabled. ++ * @return MALI_TRUE if enabled, otherwise MALI_FALSE. ++ */ ++mali_bool _mali_osk_gpu_secure_mode_is_enabled(void); ++ ++/** @brief Check if the gpu secure mode is supported. ++ * @return MALI_TRUE if supported, otherwise MALI_FALSE. ++ */ ++mali_bool _mali_osk_gpu_secure_mode_is_supported(void); ++ ++ ++/** @} */ /* end group _mali_osk_miscellaneous */ ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif /* __MALI_OSK_MALI_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_osk_profiling.h b/drivers/gpu/arm/mali400/mali/common/mali_osk_profiling.h +new file mode 100755 +index 000000000000..6e4583db1c80 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_osk_profiling.h +@@ -0,0 +1,146 @@ ++/* ++ * Copyright (C) 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_OSK_PROFILING_H__ ++#define __MALI_OSK_PROFILING_H__ ++ ++#if defined(CONFIG_MALI400_PROFILING) && defined (CONFIG_TRACEPOINTS) ++ ++#include "mali_linux_trace.h" ++#include "mali_profiling_events.h" ++#include "mali_profiling_gator_api.h" ++ ++#define MALI_PROFILING_MAX_BUFFER_ENTRIES 1048576 ++ ++#define MALI_PROFILING_NO_HW_COUNTER = ((u32)-1) ++ ++/** @defgroup _mali_osk_profiling External profiling connectivity ++ * @{ */ ++ ++/** ++ * Initialize the profiling module. ++ * @return _MALI_OSK_ERR_OK on success, otherwise failure. ++ */ ++_mali_osk_errcode_t _mali_osk_profiling_init(mali_bool auto_start); ++ ++/* ++ * Terminate the profiling module. ++ */ ++void _mali_osk_profiling_term(void); ++ ++/** ++ * Stop the profile sampling operation. ++ */ ++void _mali_osk_profiling_stop_sampling(u32 pid); ++ ++/** ++ * Start recording profiling data ++ * ++ * The specified limit will determine how large the capture buffer is. ++ * MALI_PROFILING_MAX_BUFFER_ENTRIES determines the maximum size allowed by the device driver. ++ * ++ * @param limit The desired maximum number of events to record on input, the actual maximum on output. ++ * @return _MALI_OSK_ERR_OK on success, otherwise failure. ++ */ ++_mali_osk_errcode_t _mali_osk_profiling_start(u32 *limit); ++ ++/** ++ * Add an profiling event ++ * ++ * @param event_id The event identificator. ++ * @param data0 First data parameter, depending on event_id specified. ++ * @param data1 Second data parameter, depending on event_id specified. ++ * @param data2 Third data parameter, depending on event_id specified. ++ * @param data3 Fourth data parameter, depending on event_id specified. ++ * @param data4 Fifth data parameter, depending on event_id specified. ++ */ ++void _mali_osk_profiling_add_event(u32 event_id, u32 data0, u32 data1, u32 data2, u32 data3, u32 data4); ++ ++/** ++ * Report a hardware counter event. ++ * ++ * @param counter_id The ID of the counter. ++ * @param value The value of the counter. ++ */ ++ ++/* Call Linux tracepoint directly */ ++#define _mali_osk_profiling_report_hw_counter(counter_id, value) trace_mali_hw_counter(counter_id, value) ++ ++/** ++ * Report SW counters ++ * ++ * @param counters array of counter values ++ */ ++void _mali_osk_profiling_report_sw_counters(u32 *counters); ++ ++void _mali_osk_profiling_record_global_counters(int counter_id, u32 value); ++ ++/** ++ * Stop recording profiling data ++ * ++ * @param count Returns the number of recorded events. ++ * @return _MALI_OSK_ERR_OK on success, otherwise failure. ++ */ ++_mali_osk_errcode_t _mali_osk_profiling_stop(u32 *count); ++ ++/** ++ * Retrieves the number of events that can be retrieved ++ * ++ * @return The number of recorded events that can be retrieved. ++ */ ++u32 _mali_osk_profiling_get_count(void); ++ ++/** ++ * Retrieve an event ++ * ++ * @param index Event index (start with 0 and continue until this function fails to retrieve all events) ++ * @param timestamp The timestamp for the retrieved event will be stored here. ++ * @param event_id The event ID for the retrieved event will be stored here. ++ * @param data The 5 data values for the retrieved event will be stored here. ++ * @return _MALI_OSK_ERR_OK on success, otherwise failure. ++ */ ++_mali_osk_errcode_t _mali_osk_profiling_get_event(u32 index, u64 *timestamp, u32 *event_id, u32 data[5]); ++ ++/** ++ * Clear the recorded buffer. ++ * ++ * This is needed in order to start another recording. ++ * ++ * @return _MALI_OSK_ERR_OK on success, otherwise failure. ++ */ ++_mali_osk_errcode_t _mali_osk_profiling_clear(void); ++ ++/** ++ * Checks if a recording of profiling data is in progress ++ * ++ * @return MALI_TRUE if recording of profiling data is in progress, MALI_FALSE if not ++ */ ++mali_bool _mali_osk_profiling_is_recording(void); ++ ++/** ++ * Checks if profiling data is available for retrival ++ * ++ * @return MALI_TRUE if profiling data is avaiable, MALI_FALSE if not ++ */ ++mali_bool _mali_osk_profiling_have_recording(void); ++ ++/** @} */ /* end group _mali_osk_profiling */ ++ ++#else /* defined(CONFIG_MALI400_PROFILING) && defined(CONFIG_TRACEPOINTS) */ ++ ++/* Dummy add_event, for when profiling is disabled. */ ++ ++#define _mali_osk_profiling_add_event(event_id, data0, data1, data2, data3, data4) ++ ++#endif /* defined(CONFIG_MALI400_PROFILING) && defined(CONFIG_TRACEPOINTS) */ ++ ++#endif /* __MALI_OSK_PROFILING_H__ */ ++ ++ +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_osk_types.h b/drivers/gpu/arm/mali400/mali/common/mali_osk_types.h +new file mode 100755 +index 000000000000..b6fa94ce16b3 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_osk_types.h +@@ -0,0 +1,471 @@ ++/* ++ * Copyright (C) 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file mali_osk_types.h ++ * Defines types of the OS abstraction layer for the kernel device driver (OSK) ++ */ ++ ++#ifndef __MALI_OSK_TYPES_H__ ++#define __MALI_OSK_TYPES_H__ ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++/** ++ * @addtogroup uddapi Unified Device Driver (UDD) APIs ++ * ++ * @{ ++ */ ++ ++/** ++ * @addtogroup oskapi UDD OS Abstraction for Kernel-side (OSK) APIs ++ * ++ * @{ ++ */ ++ ++/** @defgroup _mali_osk_miscellaneous OSK Miscellaneous functions, constants and types ++ * @{ */ ++ ++/* Define integer types used by OSK. Note: these currently clash with Linux so we only define them if not defined already */ ++#ifndef __KERNEL__ ++typedef unsigned char u8; ++typedef signed char s8; ++typedef unsigned short u16; ++typedef signed short s16; ++typedef unsigned int u32; ++typedef signed int s32; ++typedef unsigned long long u64; ++#define BITS_PER_LONG (sizeof(long)*8) ++#else ++/* Ensure Linux types u32, etc. are defined */ ++#include ++#endif ++ ++/** @brief Mali Boolean type which uses MALI_TRUE and MALI_FALSE ++ */ ++typedef unsigned long mali_bool; ++ ++#ifndef MALI_TRUE ++#define MALI_TRUE ((mali_bool)1) ++#endif ++ ++#ifndef MALI_FALSE ++#define MALI_FALSE ((mali_bool)0) ++#endif ++ ++#define MALI_HW_CORE_NO_COUNTER ((u32)-1) ++ ++ ++#define MALI_S32_MAX 0x7fffffff ++ ++/** ++ * @brief OSK Error codes ++ * ++ * Each OS may use its own set of error codes, and may require that the ++ * User/Kernel interface take certain error code. This means that the common ++ * error codes need to be sufficiently rich to pass the correct error code ++ * thorugh from the OSK to U/K layer, across all OSs. ++ * ++ * The result is that some error codes will appear redundant on some OSs. ++ * Under all OSs, the OSK layer must translate native OS error codes to ++ * _mali_osk_errcode_t codes. Similarly, the U/K layer must translate from ++ * _mali_osk_errcode_t codes to native OS error codes. ++ */ ++typedef enum { ++ _MALI_OSK_ERR_OK = 0, /**< Success. */ ++ _MALI_OSK_ERR_FAULT = -1, /**< General non-success */ ++ _MALI_OSK_ERR_INVALID_FUNC = -2, /**< Invalid function requested through User/Kernel interface (e.g. bad IOCTL number) */ ++ _MALI_OSK_ERR_INVALID_ARGS = -3, /**< Invalid arguments passed through User/Kernel interface */ ++ _MALI_OSK_ERR_NOMEM = -4, /**< Insufficient memory */ ++ _MALI_OSK_ERR_TIMEOUT = -5, /**< Timeout occurred */ ++ _MALI_OSK_ERR_RESTARTSYSCALL = -6, /**< Special: On certain OSs, must report when an interruptable mutex is interrupted. Ignore otherwise. */ ++ _MALI_OSK_ERR_ITEM_NOT_FOUND = -7, /**< Table Lookup failed */ ++ _MALI_OSK_ERR_BUSY = -8, /**< Device/operation is busy. Try again later */ ++ _MALI_OSK_ERR_UNSUPPORTED = -9, /**< Optional part of the interface used, and is unsupported */ ++} _mali_osk_errcode_t; ++ ++/** @} */ /* end group _mali_osk_miscellaneous */ ++ ++/** @defgroup _mali_osk_wq OSK work queues ++ * @{ */ ++ ++/** @brief Private type for work objects */ ++typedef struct _mali_osk_wq_work_s _mali_osk_wq_work_t; ++typedef struct _mali_osk_wq_delayed_work_s _mali_osk_wq_delayed_work_t; ++ ++/** @brief Work queue handler function ++ * ++ * This function type is called when the work is scheduled by the work queue, ++ * e.g. as an IRQ bottom-half handler. ++ * ++ * Refer to \ref _mali_osk_wq_schedule_work() for more information on the ++ * work-queue and work handlers. ++ * ++ * @param arg resource-specific data ++ */ ++typedef void (*_mali_osk_wq_work_handler_t)(void *arg); ++ ++/* @} */ /* end group _mali_osk_wq */ ++ ++/** @defgroup _mali_osk_irq OSK IRQ handling ++ * @{ */ ++ ++/** @brief Private type for IRQ handling objects */ ++typedef struct _mali_osk_irq_t_struct _mali_osk_irq_t; ++ ++/** @brief Optional function to trigger an irq from a resource ++ * ++ * This function is implemented by the common layer to allow probing of a resource's IRQ. ++ * @param arg resource-specific data */ ++typedef void (*_mali_osk_irq_trigger_t)(void *arg); ++ ++/** @brief Optional function to acknowledge an irq from a resource ++ * ++ * This function is implemented by the common layer to allow probing of a resource's IRQ. ++ * @param arg resource-specific data ++ * @return _MALI_OSK_ERR_OK if the IRQ was successful, or a suitable _mali_osk_errcode_t on failure. */ ++typedef _mali_osk_errcode_t (*_mali_osk_irq_ack_t)(void *arg); ++ ++/** @brief IRQ 'upper-half' handler callback. ++ * ++ * This function is implemented by the common layer to do the initial handling of a ++ * resource's IRQ. This maps on to the concept of an ISR that does the minimum ++ * work necessary before handing off to an IST. ++ * ++ * The communication of the resource-specific data from the ISR to the IST is ++ * handled by the OSK implementation. ++ * ++ * On most systems, the IRQ upper-half handler executes in IRQ context. ++ * Therefore, the system may have restrictions about what can be done in this ++ * context ++ * ++ * If an IRQ upper-half handler requires more work to be done than can be ++ * acheived in an IRQ context, then it may defer the work with ++ * _mali_osk_wq_schedule_work(). Refer to \ref _mali_osk_wq_create_work() for ++ * more information. ++ * ++ * @param arg resource-specific data ++ * @return _MALI_OSK_ERR_OK if the IRQ was correctly handled, or a suitable ++ * _mali_osk_errcode_t otherwise. ++ */ ++typedef _mali_osk_errcode_t (*_mali_osk_irq_uhandler_t)(void *arg); ++ ++ ++/** @} */ /* end group _mali_osk_irq */ ++ ++ ++/** @defgroup _mali_osk_atomic OSK Atomic counters ++ * @{ */ ++ ++/** @brief Public type of atomic counters ++ * ++ * This is public for allocation on stack. On systems that support it, this is just a single 32-bit value. ++ * On others, it could be encapsulating an object stored elsewhere. ++ * ++ * Regardless of implementation, the \ref _mali_osk_atomic functions \b must be used ++ * for all accesses to the variable's value, even if atomicity is not required. ++ * Do not access u.val or u.obj directly. ++ */ ++typedef struct { ++ union { ++ u32 val; ++ void *obj; ++ } u; ++} _mali_osk_atomic_t; ++/** @} */ /* end group _mali_osk_atomic */ ++ ++ ++/** @defgroup _mali_osk_lock OSK Mutual Exclusion Locks ++ * @{ */ ++ ++ ++/** @brief OSK Mutual Exclusion Lock ordered list ++ * ++ * This lists the various types of locks in the system and is used to check ++ * that locks are taken in the correct order. ++ * ++ * - Holding more than one lock of the same order at the same time is not ++ * allowed. ++ * - Taking a lock of a lower order than the highest-order lock currently held ++ * is not allowed. ++ * ++ */ ++typedef enum { ++ /* || Locks || */ ++ /* || must be || */ ++ /* _||_ taken in _||_ */ ++ /* \ / this \ / */ ++ /* \/ order! \/ */ ++ ++ _MALI_OSK_LOCK_ORDER_FIRST = 0, ++ ++ _MALI_OSK_LOCK_ORDER_SESSIONS, ++ _MALI_OSK_LOCK_ORDER_MEM_SESSION, ++ _MALI_OSK_LOCK_ORDER_MEM_INFO, ++ _MALI_OSK_LOCK_ORDER_MEM_PT_CACHE, ++ _MALI_OSK_LOCK_ORDER_DESCRIPTOR_MAP, ++ _MALI_OSK_LOCK_ORDER_PM_EXECUTION, ++ _MALI_OSK_LOCK_ORDER_EXECUTOR, ++ _MALI_OSK_LOCK_ORDER_TIMELINE_SYSTEM, ++ _MALI_OSK_LOCK_ORDER_SCHEDULER, ++ _MALI_OSK_LOCK_ORDER_SCHEDULER_DEFERRED, ++ _MALI_OSK_LOCK_ORDER_PROFILING, ++ _MALI_OSK_LOCK_ORDER_L2, ++ _MALI_OSK_LOCK_ORDER_L2_COMMAND, ++ _MALI_OSK_LOCK_ORDER_UTILIZATION, ++ _MALI_OSK_LOCK_ORDER_SESSION_PENDING_JOBS, ++ _MALI_OSK_LOCK_ORDER_PM_STATE, ++ ++ _MALI_OSK_LOCK_ORDER_LAST, ++} _mali_osk_lock_order_t; ++ ++ ++/** @brief OSK Mutual Exclusion Lock flags type ++ * ++ * - Any lock can use the order parameter. ++ */ ++typedef enum { ++ _MALI_OSK_LOCKFLAG_UNORDERED = 0x1, /**< Indicate that the order of this lock should not be checked */ ++ _MALI_OSK_LOCKFLAG_ORDERED = 0x2, ++ /** @enum _mali_osk_lock_flags_t ++ * ++ * Flags from 0x10000--0x80000000 are RESERVED for User-mode */ ++ ++} _mali_osk_lock_flags_t; ++ ++/** @brief Mutual Exclusion Lock Mode Optimization hint ++ * ++ * The lock mode is used to implement the read/write locking of locks when we call ++ * functions _mali_osk_mutex_rw_init/wait/signal/term/. In this case, the RO mode can ++ * be used to allow multiple concurrent readers, but no writers. The RW mode is used for ++ * writers, and so will wait for all readers to release the lock (if any present). ++ * Further readers and writers will wait until the writer releases the lock. ++ * ++ * The mode is purely an optimization hint: for example, it is permissible for ++ * all locks to behave in RW mode, regardless of that supplied. ++ * ++ * It is an error to attempt to use locks in anything other that RW mode when ++ * call functions _mali_osk_mutex_rw_wait/signal(). ++ * ++ */ ++typedef enum { ++ _MALI_OSK_LOCKMODE_UNDEF = -1, /**< Undefined lock mode. For internal use only */ ++ _MALI_OSK_LOCKMODE_RW = 0x0, /**< Read-write mode, default. All readers and writers are mutually-exclusive */ ++ _MALI_OSK_LOCKMODE_RO, /**< Read-only mode, to support multiple concurrent readers, but mutual exclusion in the presence of writers. */ ++ /** @enum _mali_osk_lock_mode_t ++ * ++ * Lock modes 0x40--0x7F are RESERVED for User-mode */ ++} _mali_osk_lock_mode_t; ++ ++/** @brief Private types for Mutual Exclusion lock objects */ ++typedef struct _mali_osk_lock_debug_s _mali_osk_lock_debug_t; ++typedef struct _mali_osk_spinlock_s _mali_osk_spinlock_t; ++typedef struct _mali_osk_spinlock_irq_s _mali_osk_spinlock_irq_t; ++typedef struct _mali_osk_mutex_s _mali_osk_mutex_t; ++typedef struct _mali_osk_mutex_rw_s _mali_osk_mutex_rw_t; ++ ++/** @} */ /* end group _mali_osk_lock */ ++ ++/** @defgroup _mali_osk_low_level_memory OSK Low-level Memory Operations ++ * @{ */ ++ ++/** ++ * @brief Private data type for use in IO accesses to/from devices. ++ * ++ * This represents some range that is accessible from the device. Examples ++ * include: ++ * - Device Registers, which could be readable and/or writeable. ++ * - Memory that the device has access to, for storing configuration structures. ++ * ++ * Access to this range must be made through the _mali_osk_mem_ioread32() and ++ * _mali_osk_mem_iowrite32() functions. ++ */ ++typedef struct _mali_io_address *mali_io_address; ++ ++/** @defgroup _MALI_OSK_CPU_PAGE CPU Physical page size macros. ++ * ++ * The order of the page size is supplied for ++ * ease of use by algorithms that might require it, since it is easier to know ++ * it ahead of time rather than calculating it. ++ * ++ * The Mali Page Mask macro masks off the lower bits of a physical address to ++ * give the start address of the page for that physical address. ++ * ++ * @note The Mali device driver code is designed for systems with 4KB page size. ++ * Changing these macros will not make the entire Mali device driver work with ++ * page sizes other than 4KB. ++ * ++ * @note The CPU Physical Page Size has been assumed to be the same as the Mali ++ * Physical Page Size. ++ * ++ * @{ ++ */ ++ ++/** CPU Page Order, as log to base 2 of the Page size. @see _MALI_OSK_CPU_PAGE_SIZE */ ++#define _MALI_OSK_CPU_PAGE_ORDER ((u32)12) ++/** CPU Page Size, in bytes. */ ++#define _MALI_OSK_CPU_PAGE_SIZE (((u32)1) << (_MALI_OSK_CPU_PAGE_ORDER)) ++/** CPU Page Mask, which masks off the offset within a page */ ++#define _MALI_OSK_CPU_PAGE_MASK (~((((u32)1) << (_MALI_OSK_CPU_PAGE_ORDER)) - ((u32)1))) ++/** @} */ /* end of group _MALI_OSK_CPU_PAGE */ ++ ++/** @defgroup _MALI_OSK_MALI_PAGE Mali Physical Page size macros ++ * ++ * Mali Physical page size macros. The order of the page size is supplied for ++ * ease of use by algorithms that might require it, since it is easier to know ++ * it ahead of time rather than calculating it. ++ * ++ * The Mali Page Mask macro masks off the lower bits of a physical address to ++ * give the start address of the page for that physical address. ++ * ++ * @note The Mali device driver code is designed for systems with 4KB page size. ++ * Changing these macros will not make the entire Mali device driver work with ++ * page sizes other than 4KB. ++ * ++ * @note The Mali Physical Page Size has been assumed to be the same as the CPU ++ * Physical Page Size. ++ * ++ * @{ ++ */ ++ ++/** Mali Page Order, as log to base 2 of the Page size. @see _MALI_OSK_MALI_PAGE_SIZE */ ++#define _MALI_OSK_MALI_PAGE_ORDER PAGE_SHIFT ++/** Mali Page Size, in bytes. */ ++#define _MALI_OSK_MALI_PAGE_SIZE PAGE_SIZE ++/** Mali Page Mask, which masks off the offset within a page */ ++#define _MALI_OSK_MALI_PAGE_MASK PAGE_MASK ++/** @} */ /* end of group _MALI_OSK_MALI_PAGE*/ ++ ++/** @brief flags for mapping a user-accessible memory range ++ * ++ * Where a function with prefix '_mali_osk_mem_mapregion' accepts flags as one ++ * of the function parameters, it will use one of these. These allow per-page ++ * control over mappings. Compare with the mali_memory_allocation_flag type, ++ * which acts over an entire range ++ * ++ * These may be OR'd together with bitwise OR (|), but must be cast back into ++ * the type after OR'ing. ++ */ ++typedef enum { ++ _MALI_OSK_MEM_MAPREGION_FLAG_OS_ALLOCATED_PHYSADDR = 0x1, /**< Physical address is OS Allocated */ ++} _mali_osk_mem_mapregion_flags_t; ++/** @} */ /* end group _mali_osk_low_level_memory */ ++ ++/** @defgroup _mali_osk_notification OSK Notification Queues ++ * @{ */ ++ ++/** @brief Private type for notification queue objects */ ++typedef struct _mali_osk_notification_queue_t_struct _mali_osk_notification_queue_t; ++ ++/** @brief Public notification data object type */ ++typedef struct _mali_osk_notification_t_struct { ++ u32 notification_type; /**< The notification type */ ++ u32 result_buffer_size; /**< Size of the result buffer to copy to user space */ ++ void *result_buffer; /**< Buffer containing any type specific data */ ++} _mali_osk_notification_t; ++ ++/** @} */ /* end group _mali_osk_notification */ ++ ++ ++/** @defgroup _mali_osk_timer OSK Timer Callbacks ++ * @{ */ ++ ++/** @brief Function to call when a timer expires ++ * ++ * When a timer expires, this function is called. Note that on many systems, ++ * a timer callback will be executed in IRQ context. Therefore, restrictions ++ * may apply on what can be done inside the timer callback. ++ * ++ * If a timer requires more work to be done than can be acheived in an IRQ ++ * context, then it may defer the work with a work-queue. For example, it may ++ * use \ref _mali_osk_wq_schedule_work() to make use of a bottom-half handler ++ * to carry out the remaining work. ++ * ++ * Stopping the timer with \ref _mali_osk_timer_del() blocks on compeletion of ++ * the callback. Therefore, the callback may not obtain any mutexes also held ++ * by any callers of _mali_osk_timer_del(). Otherwise, a deadlock may occur. ++ * ++ * @param arg Function-specific data */ ++typedef void (*_mali_osk_timer_callback_t)(void *arg); ++ ++/** @brief Private type for Timer Callback Objects */ ++typedef struct _mali_osk_timer_t_struct _mali_osk_timer_t; ++/** @} */ /* end group _mali_osk_timer */ ++ ++ ++/** @addtogroup _mali_osk_list OSK Doubly-Linked Circular Lists ++ * @{ */ ++ ++/** @brief Public List objects. ++ * ++ * To use, add a _mali_osk_list_t member to the structure that may become part ++ * of a list. When traversing the _mali_osk_list_t objects, use the ++ * _MALI_OSK_CONTAINER_OF() macro to recover the structure from its ++ *_mali_osk_list_t member ++ * ++ * Each structure may have multiple _mali_osk_list_t members, so that the ++ * structure is part of multiple lists. When traversing lists, ensure that the ++ * correct _mali_osk_list_t member is used, because type-checking will be ++ * lost by the compiler. ++ */ ++typedef struct _mali_osk_list_s { ++ struct _mali_osk_list_s *next; ++ struct _mali_osk_list_s *prev; ++} _mali_osk_list_t; ++/** @} */ /* end group _mali_osk_list */ ++ ++/** @addtogroup _mali_osk_miscellaneous ++ * @{ */ ++ ++/** @brief resource description struct ++ * ++ * Platform independent representation of a Mali HW resource ++ */ ++typedef struct _mali_osk_resource { ++ const char *description; /**< short description of the resource */ ++ uintptr_t base; /**< Physical base address of the resource, as seen by Mali resources. */ ++ const char *irq_name; /**< Name of irq belong to this resource */ ++ u32 irq; /**< IRQ number delivered to the CPU, or -1 to tell the driver to probe for it (if possible) */ ++} _mali_osk_resource_t; ++/** @} */ /* end group _mali_osk_miscellaneous */ ++ ++/** @defgroup _mali_osk_wait_queue OSK Wait Queue functionality ++ * @{ */ ++/** @brief Private type for wait queue objects */ ++typedef struct _mali_osk_wait_queue_t_struct _mali_osk_wait_queue_t; ++/** @} */ /* end group _mali_osk_wait_queue */ ++ ++/** @} */ /* end group osuapi */ ++ ++/** @} */ /* end group uddapi */ ++ ++/** @brief Mali print ctx type which uses seq_file ++ */ ++typedef struct seq_file _mali_osk_print_ctx; ++ ++#define _MALI_OSK_BITMAP_INVALIDATE_INDEX -1 ++ ++typedef struct _mali_osk_bitmap { ++ u32 reserve; ++ u32 last; ++ u32 max; ++ u32 avail; ++ _mali_osk_spinlock_t *lock; ++ unsigned long *table; ++} _mali_osk_bitmap_t; ++ ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif /* __MALI_OSK_TYPES_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_pm.c b/drivers/gpu/arm/mali400/mali/common/mali_pm.c +new file mode 100755 +index 000000000000..3989a33aeaef +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_pm.c +@@ -0,0 +1,1362 @@ ++/* ++ * Copyright (C) 2011-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include "mali_pm.h" ++#include "mali_kernel_common.h" ++#include "mali_osk.h" ++#include "mali_osk_mali.h" ++#include "mali_scheduler.h" ++#include "mali_group.h" ++#include "mali_pm_domain.h" ++#include "mali_pmu.h" ++ ++#include "mali_executor.h" ++#include "mali_control_timer.h" ++ ++#if defined(DEBUG) ++u32 num_pm_runtime_resume = 0; ++u32 num_pm_updates = 0; ++u32 num_pm_updates_up = 0; ++u32 num_pm_updates_down = 0; ++#endif ++ ++#define MALI_PM_DOMAIN_DUMMY_MASK (1 << MALI_DOMAIN_INDEX_DUMMY) ++ ++/* lock protecting power state (including pm_domains) */ ++static _mali_osk_spinlock_irq_t *pm_lock_state = NULL; ++ ++/* the wanted domain mask (protected by pm_lock_state) */ ++static u32 pd_mask_wanted = 0; ++ ++/* used to deferring the actual power changes */ ++static _mali_osk_wq_work_t *pm_work = NULL; ++ ++/* lock protecting power change execution */ ++static _mali_osk_mutex_t *pm_lock_exec = NULL; ++ ++/* PMU domains which are actually powered on (protected by pm_lock_exec) */ ++static u32 pmu_mask_current = 0; ++ ++/* ++ * domains which marked as powered on (protected by pm_lock_exec) ++ * This can be different from pmu_mask_current right after GPU power on ++ * if the PMU domains default to powered up. ++ */ ++static u32 pd_mask_current = 0; ++ ++static u16 domain_config[MALI_MAX_NUMBER_OF_DOMAINS] = { ++ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ++ 1 << MALI_DOMAIN_INDEX_DUMMY ++}; ++ ++/* The relative core power cost */ ++#define MALI_GP_COST 3 ++#define MALI_PP_COST 6 ++#define MALI_L2_COST 1 ++ ++/* ++ *We have MALI_MAX_NUMBER_OF_PP_PHYSICAL_CORES + 1 rows in this matrix ++ *because we mush store the mask of different pp cores: 0, 1, 2, 3, 4, 5, 6, 7, 8. ++ */ ++static int mali_pm_domain_power_cost_result[MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS + 1][MALI_MAX_NUMBER_OF_DOMAINS]; ++/* ++ * Keep track of runtime PM state, so that we know ++ * how to resume during OS resume. ++ */ ++#ifdef CONFIG_PM_RUNTIME ++static mali_bool mali_pm_runtime_active = MALI_FALSE; ++#else ++/* when kernel don't enable PM_RUNTIME, set the flag always true, ++ * for GPU will not power off by runtime */ ++static mali_bool mali_pm_runtime_active = MALI_TRUE; ++#endif ++ ++static void mali_pm_state_lock(void); ++static void mali_pm_state_unlock(void); ++static _mali_osk_errcode_t mali_pm_create_pm_domains(void); ++static void mali_pm_set_pmu_domain_config(void); ++static u32 mali_pm_get_registered_cores_mask(void); ++static void mali_pm_update_sync_internal(void); ++static mali_bool mali_pm_common_suspend(void); ++static void mali_pm_update_work(void *data); ++#if defined(DEBUG) ++const char *mali_pm_mask_to_string(u32 mask); ++const char *mali_pm_group_stats_to_string(void); ++#endif ++ ++_mali_osk_errcode_t mali_pm_initialize(void) ++{ ++ _mali_osk_errcode_t err; ++ struct mali_pmu_core *pmu; ++ ++ pm_lock_state = _mali_osk_spinlock_irq_init(_MALI_OSK_LOCKFLAG_ORDERED, ++ _MALI_OSK_LOCK_ORDER_PM_STATE); ++ if (NULL == pm_lock_state) { ++ mali_pm_terminate(); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ pm_lock_exec = _mali_osk_mutex_init(_MALI_OSK_LOCKFLAG_ORDERED, ++ _MALI_OSK_LOCK_ORDER_PM_STATE); ++ if (NULL == pm_lock_exec) { ++ mali_pm_terminate(); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ pm_work = _mali_osk_wq_create_work(mali_pm_update_work, NULL); ++ if (NULL == pm_work) { ++ mali_pm_terminate(); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ pmu = mali_pmu_get_global_pmu_core(); ++ if (NULL != pmu) { ++ /* ++ * We have a Mali PMU, set the correct domain ++ * configuration (default or custom) ++ */ ++ ++ u32 registered_cores_mask; ++ ++ mali_pm_set_pmu_domain_config(); ++ ++ registered_cores_mask = mali_pm_get_registered_cores_mask(); ++ mali_pmu_set_registered_cores_mask(pmu, registered_cores_mask); ++ ++ MALI_DEBUG_ASSERT(0 == pd_mask_wanted); ++ } ++ ++ /* Create all power domains needed (at least one dummy domain) */ ++ err = mali_pm_create_pm_domains(); ++ if (_MALI_OSK_ERR_OK != err) { ++ mali_pm_terminate(); ++ return err; ++ } ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++void mali_pm_terminate(void) ++{ ++ if (NULL != pm_work) { ++ _mali_osk_wq_delete_work(pm_work); ++ pm_work = NULL; ++ } ++ ++ mali_pm_domain_terminate(); ++ ++ if (NULL != pm_lock_exec) { ++ _mali_osk_mutex_term(pm_lock_exec); ++ pm_lock_exec = NULL; ++ } ++ ++ if (NULL != pm_lock_state) { ++ _mali_osk_spinlock_irq_term(pm_lock_state); ++ pm_lock_state = NULL; ++ } ++} ++ ++struct mali_pm_domain *mali_pm_register_l2_cache(u32 domain_index, ++ struct mali_l2_cache_core *l2_cache) ++{ ++ struct mali_pm_domain *domain; ++ ++ domain = mali_pm_domain_get_from_mask(domain_config[domain_index]); ++ if (NULL == domain) { ++ MALI_DEBUG_ASSERT(0 == domain_config[domain_index]); ++ domain = mali_pm_domain_get_from_index( ++ MALI_DOMAIN_INDEX_DUMMY); ++ domain_config[domain_index] = MALI_PM_DOMAIN_DUMMY_MASK; ++ } else { ++ MALI_DEBUG_ASSERT(0 != domain_config[domain_index]); ++ } ++ ++ MALI_DEBUG_ASSERT(NULL != domain); ++ ++ mali_pm_domain_add_l2_cache(domain, l2_cache); ++ ++ return domain; /* return the actual domain this was registered in */ ++} ++ ++struct mali_pm_domain *mali_pm_register_group(u32 domain_index, ++ struct mali_group *group) ++{ ++ struct mali_pm_domain *domain; ++ ++ domain = mali_pm_domain_get_from_mask(domain_config[domain_index]); ++ if (NULL == domain) { ++ MALI_DEBUG_ASSERT(0 == domain_config[domain_index]); ++ domain = mali_pm_domain_get_from_index( ++ MALI_DOMAIN_INDEX_DUMMY); ++ domain_config[domain_index] = MALI_PM_DOMAIN_DUMMY_MASK; ++ } else { ++ MALI_DEBUG_ASSERT(0 != domain_config[domain_index]); ++ } ++ ++ MALI_DEBUG_ASSERT(NULL != domain); ++ ++ mali_pm_domain_add_group(domain, group); ++ ++ return domain; /* return the actual domain this was registered in */ ++} ++ ++mali_bool mali_pm_get_domain_refs(struct mali_pm_domain **domains, ++ struct mali_group **groups, ++ u32 num_domains) ++{ ++ mali_bool ret = MALI_TRUE; /* Assume all is powered on instantly */ ++ u32 i; ++ ++ mali_pm_state_lock(); ++ ++ for (i = 0; i < num_domains; i++) { ++ MALI_DEBUG_ASSERT_POINTER(domains[i]); ++ pd_mask_wanted |= mali_pm_domain_ref_get(domains[i]); ++ if (MALI_FALSE == mali_pm_domain_power_is_on(domains[i])) { ++ /* ++ * Tell caller that the corresponding group ++ * was not already powered on. ++ */ ++ ret = MALI_FALSE; ++ } else { ++ /* ++ * There is a time gap between we power on the domain and ++ * set the power state of the corresponding groups to be on. ++ */ ++ if (NULL != groups[i] && ++ MALI_FALSE == mali_group_power_is_on(groups[i])) { ++ ret = MALI_FALSE; ++ } ++ } ++ } ++ ++ MALI_DEBUG_PRINT(3, ("PM: wanted domain mask = 0x%08X (get refs)\n", pd_mask_wanted)); ++ ++ mali_pm_state_unlock(); ++ ++ return ret; ++} ++ ++mali_bool mali_pm_put_domain_refs(struct mali_pm_domain **domains, ++ u32 num_domains) ++{ ++ u32 mask = 0; ++ mali_bool ret; ++ u32 i; ++ ++ mali_pm_state_lock(); ++ ++ for (i = 0; i < num_domains; i++) { ++ MALI_DEBUG_ASSERT_POINTER(domains[i]); ++ mask |= mali_pm_domain_ref_put(domains[i]); ++ } ++ ++ if (0 == mask) { ++ /* return false, all domains should still stay on */ ++ ret = MALI_FALSE; ++ } else { ++ /* Assert that we are dealing with a change */ ++ MALI_DEBUG_ASSERT((pd_mask_wanted & mask) == mask); ++ ++ /* Update our desired domain mask */ ++ pd_mask_wanted &= ~mask; ++ ++ /* return true; one or more domains can now be powered down */ ++ ret = MALI_TRUE; ++ } ++ ++ MALI_DEBUG_PRINT(3, ("PM: wanted domain mask = 0x%08X (put refs)\n", pd_mask_wanted)); ++ ++ mali_pm_state_unlock(); ++ ++ return ret; ++} ++ ++void mali_pm_init_begin(void) ++{ ++ struct mali_pmu_core *pmu = mali_pmu_get_global_pmu_core(); ++ ++ _mali_osk_pm_dev_ref_get_sync(); ++ ++ /* Ensure all PMU domains are on */ ++ if (NULL != pmu) { ++ mali_pmu_power_up_all(pmu); ++ } ++} ++ ++void mali_pm_init_end(void) ++{ ++ struct mali_pmu_core *pmu = mali_pmu_get_global_pmu_core(); ++ ++ /* Ensure all PMU domains are off */ ++ if (NULL != pmu) { ++ mali_pmu_power_down_all(pmu); ++ } ++ ++ _mali_osk_pm_dev_ref_put(); ++} ++ ++void mali_pm_update_sync(void) ++{ ++ mali_pm_exec_lock(); ++ ++ if (MALI_TRUE == mali_pm_runtime_active) { ++ /* ++ * Only update if GPU is powered on. ++ * Deactivation of the last group will result in both a ++ * deferred runtime PM suspend operation and ++ * deferred execution of this function. ++ * mali_pm_runtime_active will be false if runtime PM ++ * executed first and thus the GPU is now fully powered off. ++ */ ++ mali_pm_update_sync_internal(); ++ } ++ ++ mali_pm_exec_unlock(); ++} ++ ++void mali_pm_update_async(void) ++{ ++ _mali_osk_wq_schedule_work(pm_work); ++} ++ ++void mali_pm_os_suspend(mali_bool os_suspend) ++{ ++ int ret; ++ ++ MALI_DEBUG_PRINT(3, ("Mali PM: OS suspend\n")); ++ ++ /* Suspend execution of all jobs, and go to inactive state */ ++ mali_executor_suspend(); ++ ++ if (os_suspend) { ++ mali_control_timer_suspend(MALI_TRUE); ++ } ++ ++ mali_pm_exec_lock(); ++ ++ ret = mali_pm_common_suspend(); ++ ++ MALI_DEBUG_ASSERT(MALI_TRUE == ret); ++ MALI_IGNORE(ret); ++ ++ mali_pm_exec_unlock(); ++} ++ ++void mali_pm_os_resume(void) ++{ ++ struct mali_pmu_core *pmu = mali_pmu_get_global_pmu_core(); ++ ++ MALI_DEBUG_PRINT(3, ("Mali PM: OS resume\n")); ++ ++ mali_pm_exec_lock(); ++ ++#if defined(DEBUG) ++ mali_pm_state_lock(); ++ ++ /* Assert that things are as we left them in os_suspend(). */ ++ MALI_DEBUG_ASSERT(0 == pd_mask_wanted); ++ MALI_DEBUG_ASSERT(0 == pd_mask_current); ++ MALI_DEBUG_ASSERT(0 == pmu_mask_current); ++ ++ MALI_DEBUG_ASSERT(MALI_TRUE == mali_pm_domain_all_unused()); ++ ++ mali_pm_state_unlock(); ++#endif ++ ++ if (MALI_TRUE == mali_pm_runtime_active) { ++ /* Runtime PM was active, so reset PMU */ ++ if (NULL != pmu) { ++ mali_pmu_reset(pmu); ++ pmu_mask_current = mali_pmu_get_mask(pmu); ++ ++ MALI_DEBUG_PRINT(3, ("Mali PM: OS resume 0x%x \n", pmu_mask_current)); ++ } ++ ++ mali_pm_update_sync_internal(); ++ } ++ ++ mali_pm_exec_unlock(); ++ ++ /* Start executing jobs again */ ++ mali_executor_resume(); ++} ++ ++mali_bool mali_pm_runtime_suspend(void) ++{ ++ mali_bool ret; ++ ++ MALI_DEBUG_PRINT(3, ("Mali PM: Runtime suspend\n")); ++ ++ mali_pm_exec_lock(); ++ ++ /* ++ * Put SW state directly into "off" state, and do not bother to power ++ * down each power domain, because entire GPU will be powered off ++ * when we return. ++ * For runtime PM suspend, in contrast to OS suspend, there is a race ++ * between this function and the mali_pm_update_sync_internal(), which ++ * is fine... ++ */ ++ ret = mali_pm_common_suspend(); ++ if (MALI_TRUE == ret) { ++ mali_pm_runtime_active = MALI_FALSE; ++ } else { ++ /* ++ * Process the "power up" instead, ++ * which could have been "lost" ++ */ ++ mali_pm_update_sync_internal(); ++ } ++ ++ mali_pm_exec_unlock(); ++ ++ return ret; ++} ++ ++void mali_pm_runtime_resume(void) ++{ ++ struct mali_pmu_core *pmu = mali_pmu_get_global_pmu_core(); ++ ++ mali_pm_exec_lock(); ++ ++ mali_pm_runtime_active = MALI_TRUE; ++ ++#if defined(DEBUG) ++ ++num_pm_runtime_resume; ++ ++ mali_pm_state_lock(); ++ ++ /* ++ * Assert that things are as we left them in runtime_suspend(), ++ * except for pd_mask_wanted which normally will be the reason we ++ * got here (job queued => domains wanted) ++ */ ++ MALI_DEBUG_ASSERT(0 == pd_mask_current); ++ MALI_DEBUG_ASSERT(0 == pmu_mask_current); ++ ++ mali_pm_state_unlock(); ++#endif ++ ++ if (NULL != pmu) { ++ mali_pmu_reset(pmu); ++ pmu_mask_current = mali_pmu_get_mask(pmu); ++ MALI_DEBUG_PRINT(3, ("Mali PM: Runtime resume 0x%x \n", pmu_mask_current)); ++ } ++ ++ /* ++ * Normally we are resumed because a job has just been queued. ++ * pd_mask_wanted should thus be != 0. ++ * It is however possible for others to take a Mali Runtime PM ref ++ * without having a job queued. ++ * We should however always call mali_pm_update_sync_internal(), ++ * because this will take care of any potential mismatch between ++ * pmu_mask_current and pd_mask_current. ++ */ ++ mali_pm_update_sync_internal(); ++ ++ mali_pm_exec_unlock(); ++} ++ ++#if MALI_STATE_TRACKING ++u32 mali_pm_dump_state_domain(struct mali_pm_domain *domain, ++ char *buf, u32 size) ++{ ++ int n = 0; ++ ++ n += _mali_osk_snprintf(buf + n, size - n, ++ "\tPower domain: id %u\n", ++ mali_pm_domain_get_id(domain)); ++ ++ n += _mali_osk_snprintf(buf + n, size - n, ++ "\t\tMask: 0x%04x\n", ++ mali_pm_domain_get_mask(domain)); ++ ++ n += _mali_osk_snprintf(buf + n, size - n, ++ "\t\tUse count: %u\n", ++ mali_pm_domain_get_use_count(domain)); ++ ++ n += _mali_osk_snprintf(buf + n, size - n, ++ "\t\tCurrent power state: %s\n", ++ (mali_pm_domain_get_mask(domain) & pd_mask_current) ? ++ "On" : "Off"); ++ ++ n += _mali_osk_snprintf(buf + n, size - n, ++ "\t\tWanted power state: %s\n", ++ (mali_pm_domain_get_mask(domain) & pd_mask_wanted) ? ++ "On" : "Off"); ++ ++ return n; ++} ++#endif ++ ++static void mali_pm_state_lock(void) ++{ ++ _mali_osk_spinlock_irq_lock(pm_lock_state); ++} ++ ++static void mali_pm_state_unlock(void) ++{ ++ _mali_osk_spinlock_irq_unlock(pm_lock_state); ++} ++ ++void mali_pm_exec_lock(void) ++{ ++ _mali_osk_mutex_wait(pm_lock_exec); ++} ++ ++void mali_pm_exec_unlock(void) ++{ ++ _mali_osk_mutex_signal(pm_lock_exec); ++} ++ ++static void mali_pm_domain_power_up(u32 power_up_mask, ++ struct mali_group *groups_up[MALI_MAX_NUMBER_OF_GROUPS], ++ u32 *num_groups_up, ++ struct mali_l2_cache_core *l2_up[MALI_MAX_NUMBER_OF_L2_CACHE_CORES], ++ u32 *num_l2_up) ++{ ++ u32 domain_bit; ++ u32 notify_mask = power_up_mask; ++ ++ MALI_DEBUG_ASSERT(0 != power_up_mask); ++ MALI_DEBUG_ASSERT_POINTER(groups_up); ++ MALI_DEBUG_ASSERT_POINTER(num_groups_up); ++ MALI_DEBUG_ASSERT(0 == *num_groups_up); ++ MALI_DEBUG_ASSERT_POINTER(l2_up); ++ MALI_DEBUG_ASSERT_POINTER(num_l2_up); ++ MALI_DEBUG_ASSERT(0 == *num_l2_up); ++ ++ MALI_DEBUG_ASSERT_LOCK_HELD(pm_lock_exec); ++ MALI_DEBUG_ASSERT_LOCK_HELD(pm_lock_state); ++ ++ MALI_DEBUG_PRINT(5, ++ ("PM update: Powering up domains: . [%s]\n", ++ mali_pm_mask_to_string(power_up_mask))); ++ ++ pd_mask_current |= power_up_mask; ++ ++ domain_bit = _mali_osk_fls(notify_mask); ++ while (0 != domain_bit) { ++ u32 domain_id = domain_bit - 1; ++ struct mali_pm_domain *domain = ++ mali_pm_domain_get_from_index( ++ domain_id); ++ struct mali_l2_cache_core *l2_cache; ++ struct mali_l2_cache_core *l2_cache_tmp; ++ struct mali_group *group; ++ struct mali_group *group_tmp; ++ ++ /* Mark domain as powered up */ ++ mali_pm_domain_set_power_on(domain, MALI_TRUE); ++ ++ /* ++ * Make a note of the L2 and/or group(s) to notify ++ * (need to release the PM state lock before doing so) ++ */ ++ ++ _MALI_OSK_LIST_FOREACHENTRY(l2_cache, ++ l2_cache_tmp, ++ mali_pm_domain_get_l2_cache_list( ++ domain), ++ struct mali_l2_cache_core, ++ pm_domain_list) { ++ MALI_DEBUG_ASSERT(*num_l2_up < ++ MALI_MAX_NUMBER_OF_L2_CACHE_CORES); ++ l2_up[*num_l2_up] = l2_cache; ++ (*num_l2_up)++; ++ } ++ ++ _MALI_OSK_LIST_FOREACHENTRY(group, ++ group_tmp, ++ mali_pm_domain_get_group_list(domain), ++ struct mali_group, ++ pm_domain_list) { ++ MALI_DEBUG_ASSERT(*num_groups_up < ++ MALI_MAX_NUMBER_OF_GROUPS); ++ groups_up[*num_groups_up] = group; ++ ++ (*num_groups_up)++; ++ } ++ ++ /* Remove current bit and find next */ ++ notify_mask &= ~(1 << (domain_id)); ++ domain_bit = _mali_osk_fls(notify_mask); ++ } ++} ++static void mali_pm_domain_power_down(u32 power_down_mask, ++ struct mali_group *groups_down[MALI_MAX_NUMBER_OF_GROUPS], ++ u32 *num_groups_down, ++ struct mali_l2_cache_core *l2_down[MALI_MAX_NUMBER_OF_L2_CACHE_CORES], ++ u32 *num_l2_down) ++{ ++ u32 domain_bit; ++ u32 notify_mask = power_down_mask; ++ ++ MALI_DEBUG_ASSERT(0 != power_down_mask); ++ MALI_DEBUG_ASSERT_POINTER(groups_down); ++ MALI_DEBUG_ASSERT_POINTER(num_groups_down); ++ MALI_DEBUG_ASSERT(0 == *num_groups_down); ++ MALI_DEBUG_ASSERT_POINTER(l2_down); ++ MALI_DEBUG_ASSERT_POINTER(num_l2_down); ++ MALI_DEBUG_ASSERT(0 == *num_l2_down); ++ ++ MALI_DEBUG_ASSERT_LOCK_HELD(pm_lock_exec); ++ MALI_DEBUG_ASSERT_LOCK_HELD(pm_lock_state); ++ ++ MALI_DEBUG_PRINT(5, ++ ("PM update: Powering down domains: [%s]\n", ++ mali_pm_mask_to_string(power_down_mask))); ++ ++ pd_mask_current &= ~power_down_mask; ++ ++ domain_bit = _mali_osk_fls(notify_mask); ++ while (0 != domain_bit) { ++ u32 domain_id = domain_bit - 1; ++ struct mali_pm_domain *domain = ++ mali_pm_domain_get_from_index(domain_id); ++ struct mali_l2_cache_core *l2_cache; ++ struct mali_l2_cache_core *l2_cache_tmp; ++ struct mali_group *group; ++ struct mali_group *group_tmp; ++ ++ /* Mark domain as powered down */ ++ mali_pm_domain_set_power_on(domain, MALI_FALSE); ++ ++ /* ++ * Make a note of the L2s and/or groups to notify ++ * (need to release the PM state lock before doing so) ++ */ ++ ++ _MALI_OSK_LIST_FOREACHENTRY(l2_cache, ++ l2_cache_tmp, ++ mali_pm_domain_get_l2_cache_list(domain), ++ struct mali_l2_cache_core, ++ pm_domain_list) { ++ MALI_DEBUG_ASSERT(*num_l2_down < ++ MALI_MAX_NUMBER_OF_L2_CACHE_CORES); ++ l2_down[*num_l2_down] = l2_cache; ++ (*num_l2_down)++; ++ } ++ ++ _MALI_OSK_LIST_FOREACHENTRY(group, ++ group_tmp, ++ mali_pm_domain_get_group_list(domain), ++ struct mali_group, ++ pm_domain_list) { ++ MALI_DEBUG_ASSERT(*num_groups_down < ++ MALI_MAX_NUMBER_OF_GROUPS); ++ groups_down[*num_groups_down] = group; ++ (*num_groups_down)++; ++ } ++ ++ /* Remove current bit and find next */ ++ notify_mask &= ~(1 << (domain_id)); ++ domain_bit = _mali_osk_fls(notify_mask); ++ } ++} ++ ++/* ++ * Execute pending power domain changes ++ * pm_lock_exec lock must be taken by caller. ++ */ ++static void mali_pm_update_sync_internal(void) ++{ ++ /* ++ * This should only be called in non-atomic context ++ * (normally as deferred work) ++ * ++ * Look at the pending power domain changes, and execute these. ++ * Make sure group and schedulers are notified about changes. ++ */ ++ ++ struct mali_pmu_core *pmu = mali_pmu_get_global_pmu_core(); ++ ++ u32 power_down_mask; ++ u32 power_up_mask; ++ ++ MALI_DEBUG_ASSERT_LOCK_HELD(pm_lock_exec); ++ ++#if defined(DEBUG) ++ ++num_pm_updates; ++#endif ++ ++ /* Hold PM state lock while we look at (and obey) the wanted state */ ++ mali_pm_state_lock(); ++ ++ MALI_DEBUG_PRINT(5, ("PM update pre: Wanted domain mask: .. [%s]\n", ++ mali_pm_mask_to_string(pd_mask_wanted))); ++ MALI_DEBUG_PRINT(5, ("PM update pre: Current domain mask: . [%s]\n", ++ mali_pm_mask_to_string(pd_mask_current))); ++ MALI_DEBUG_PRINT(5, ("PM update pre: Current PMU mask: .... [%s]\n", ++ mali_pm_mask_to_string(pmu_mask_current))); ++ MALI_DEBUG_PRINT(5, ("PM update pre: Group power stats: ... <%s>\n", ++ mali_pm_group_stats_to_string())); ++ ++ /* Figure out which cores we need to power on */ ++ power_up_mask = pd_mask_wanted & ++ (pd_mask_wanted ^ pd_mask_current); ++ ++ if (0 != power_up_mask) { ++ u32 power_up_mask_pmu; ++ struct mali_group *groups_up[MALI_MAX_NUMBER_OF_GROUPS]; ++ u32 num_groups_up = 0; ++ struct mali_l2_cache_core * ++ l2_up[MALI_MAX_NUMBER_OF_L2_CACHE_CORES]; ++ u32 num_l2_up = 0; ++ u32 i; ++ ++#if defined(DEBUG) ++ ++num_pm_updates_up; ++#endif ++ ++ /* ++ * Make sure dummy/global domain is always included when ++ * powering up, since this is controlled by runtime PM, ++ * and device power is on at this stage. ++ */ ++ power_up_mask |= MALI_PM_DOMAIN_DUMMY_MASK; ++ ++ /* Power up only real PMU domains */ ++ power_up_mask_pmu = power_up_mask & ~MALI_PM_DOMAIN_DUMMY_MASK; ++ ++ /* But not those that happen to be powered on already */ ++ power_up_mask_pmu &= (power_up_mask ^ pmu_mask_current) & ++ power_up_mask; ++ ++ if (0 != power_up_mask_pmu) { ++ MALI_DEBUG_ASSERT(NULL != pmu); ++ pmu_mask_current |= power_up_mask_pmu; ++ mali_pmu_power_up(pmu, power_up_mask_pmu); ++ } ++ ++ /* ++ * Put the domains themselves in power up state. ++ * We get the groups and L2s to notify in return. ++ */ ++ mali_pm_domain_power_up(power_up_mask, ++ groups_up, &num_groups_up, ++ l2_up, &num_l2_up); ++ ++ /* Need to unlock PM state lock before notifying L2 + groups */ ++ mali_pm_state_unlock(); ++ ++ /* Notify each L2 cache that we have be powered up */ ++ for (i = 0; i < num_l2_up; i++) { ++ mali_l2_cache_power_up(l2_up[i]); ++ } ++ ++ /* ++ * Tell execution module about all the groups we have ++ * powered up. Groups will be notified as a result of this. ++ */ ++ mali_executor_group_power_up(groups_up, num_groups_up); ++ ++ /* Lock state again before checking for power down */ ++ mali_pm_state_lock(); ++ } ++ ++ /* Figure out which cores we need to power off */ ++ power_down_mask = pd_mask_current & ++ (pd_mask_wanted ^ pd_mask_current); ++ ++ /* ++ * Never power down the dummy/global domain here. This is to be done ++ * from a suspend request (since this domain is only physicall powered ++ * down at that point) ++ */ ++ power_down_mask &= ~MALI_PM_DOMAIN_DUMMY_MASK; ++ ++ if (0 != power_down_mask) { ++ u32 power_down_mask_pmu; ++ struct mali_group *groups_down[MALI_MAX_NUMBER_OF_GROUPS]; ++ u32 num_groups_down = 0; ++ struct mali_l2_cache_core * ++ l2_down[MALI_MAX_NUMBER_OF_L2_CACHE_CORES]; ++ u32 num_l2_down = 0; ++ u32 i; ++ ++#if defined(DEBUG) ++ ++num_pm_updates_down; ++#endif ++ ++ /* ++ * Put the domains themselves in power down state. ++ * We get the groups and L2s to notify in return. ++ */ ++ mali_pm_domain_power_down(power_down_mask, ++ groups_down, &num_groups_down, ++ l2_down, &num_l2_down); ++ ++ /* Need to unlock PM state lock before notifying L2 + groups */ ++ mali_pm_state_unlock(); ++ ++ /* ++ * Tell execution module about all the groups we will be ++ * powering down. Groups will be notified as a result of this. ++ */ ++ if (0 < num_groups_down) { ++ mali_executor_group_power_down(groups_down, num_groups_down); ++ } ++ ++ /* Notify each L2 cache that we will be powering down */ ++ for (i = 0; i < num_l2_down; i++) { ++ mali_l2_cache_power_down(l2_down[i]); ++ } ++ ++ /* ++ * Power down only PMU domains which should not stay on ++ * Some domains might for instance currently be incorrectly ++ * powered up if default domain power state is all on. ++ */ ++ power_down_mask_pmu = pmu_mask_current & (~pd_mask_current); ++ ++ if (0 != power_down_mask_pmu) { ++ MALI_DEBUG_ASSERT(NULL != pmu); ++ pmu_mask_current &= ~power_down_mask_pmu; ++ mali_pmu_power_down(pmu, power_down_mask_pmu); ++ ++ } ++ } else { ++ /* ++ * Power down only PMU domains which should not stay on ++ * Some domains might for instance currently be incorrectly ++ * powered up if default domain power state is all on. ++ */ ++ u32 power_down_mask_pmu; ++ ++ /* No need for state lock since we'll only update PMU */ ++ mali_pm_state_unlock(); ++ ++ power_down_mask_pmu = pmu_mask_current & (~pd_mask_current); ++ ++ if (0 != power_down_mask_pmu) { ++ MALI_DEBUG_ASSERT(NULL != pmu); ++ pmu_mask_current &= ~power_down_mask_pmu; ++ mali_pmu_power_down(pmu, power_down_mask_pmu); ++ } ++ } ++ ++ MALI_DEBUG_PRINT(5, ("PM update post: Current domain mask: . [%s]\n", ++ mali_pm_mask_to_string(pd_mask_current))); ++ MALI_DEBUG_PRINT(5, ("PM update post: Current PMU mask: .... [%s]\n", ++ mali_pm_mask_to_string(pmu_mask_current))); ++ MALI_DEBUG_PRINT(5, ("PM update post: Group power stats: ... <%s>\n", ++ mali_pm_group_stats_to_string())); ++} ++ ++static mali_bool mali_pm_common_suspend(void) ++{ ++ mali_pm_state_lock(); ++ ++ if (0 != pd_mask_wanted) { ++ MALI_DEBUG_PRINT(5, ("PM: Aborting suspend operation\n\n\n")); ++ mali_pm_state_unlock(); ++ return MALI_FALSE; ++ } ++ ++ MALI_DEBUG_PRINT(5, ("PM suspend pre: Wanted domain mask: .. [%s]\n", ++ mali_pm_mask_to_string(pd_mask_wanted))); ++ MALI_DEBUG_PRINT(5, ("PM suspend pre: Current domain mask: . [%s]\n", ++ mali_pm_mask_to_string(pd_mask_current))); ++ MALI_DEBUG_PRINT(5, ("PM suspend pre: Current PMU mask: .... [%s]\n", ++ mali_pm_mask_to_string(pmu_mask_current))); ++ MALI_DEBUG_PRINT(5, ("PM suspend pre: Group power stats: ... <%s>\n", ++ mali_pm_group_stats_to_string())); ++ ++ if (0 != pd_mask_current) { ++ /* ++ * We have still some domains powered on. ++ * It is for instance very normal that at least the ++ * dummy/global domain is marked as powered on at this point. ++ * (because it is physically powered on until this function ++ * returns) ++ */ ++ ++ struct mali_group *groups_down[MALI_MAX_NUMBER_OF_GROUPS]; ++ u32 num_groups_down = 0; ++ struct mali_l2_cache_core * ++ l2_down[MALI_MAX_NUMBER_OF_L2_CACHE_CORES]; ++ u32 num_l2_down = 0; ++ u32 i; ++ ++ /* ++ * Put the domains themselves in power down state. ++ * We get the groups and L2s to notify in return. ++ */ ++ mali_pm_domain_power_down(pd_mask_current, ++ groups_down, ++ &num_groups_down, ++ l2_down, ++ &num_l2_down); ++ ++ MALI_DEBUG_ASSERT(0 == pd_mask_current); ++ MALI_DEBUG_ASSERT(MALI_TRUE == mali_pm_domain_all_unused()); ++ ++ /* Need to unlock PM state lock before notifying L2 + groups */ ++ mali_pm_state_unlock(); ++ ++ /* ++ * Tell execution module about all the groups we will be ++ * powering down. Groups will be notified as a result of this. ++ */ ++ if (0 < num_groups_down) { ++ mali_executor_group_power_down(groups_down, num_groups_down); ++ } ++ ++ /* Notify each L2 cache that we will be powering down */ ++ for (i = 0; i < num_l2_down; i++) { ++ mali_l2_cache_power_down(l2_down[i]); ++ } ++ ++ pmu_mask_current = 0; ++ } else { ++ MALI_DEBUG_ASSERT(0 == pmu_mask_current); ++ ++ MALI_DEBUG_ASSERT(MALI_TRUE == mali_pm_domain_all_unused()); ++ ++ mali_pm_state_unlock(); ++ } ++ ++ MALI_DEBUG_PRINT(5, ("PM suspend post: Current domain mask: [%s]\n", ++ mali_pm_mask_to_string(pd_mask_current))); ++ MALI_DEBUG_PRINT(5, ("PM suspend post: Current PMU mask: ... [%s]\n", ++ mali_pm_mask_to_string(pmu_mask_current))); ++ MALI_DEBUG_PRINT(5, ("PM suspend post: Group power stats: .. <%s>\n", ++ mali_pm_group_stats_to_string())); ++ ++ return MALI_TRUE; ++} ++ ++static void mali_pm_update_work(void *data) ++{ ++ MALI_IGNORE(data); ++ mali_pm_update_sync(); ++} ++ ++static _mali_osk_errcode_t mali_pm_create_pm_domains(void) ++{ ++ int i; ++ ++ /* Create all domains (including dummy domain) */ ++ for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) { ++ if (0x0 == domain_config[i]) continue; ++ ++ if (NULL == mali_pm_domain_create(domain_config[i])) { ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ } ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++static void mali_pm_set_default_pm_domain_config(void) ++{ ++ MALI_DEBUG_ASSERT(0 != _mali_osk_resource_base_address()); ++ ++ /* GP core */ ++ if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( ++ MALI_OFFSET_GP, NULL)) { ++ domain_config[MALI_DOMAIN_INDEX_GP] = 0x01; ++ } ++ ++ /* PP0 - PP3 core */ ++ if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( ++ MALI_OFFSET_PP0, NULL)) { ++ if (mali_is_mali400()) { ++ domain_config[MALI_DOMAIN_INDEX_PP0] = 0x01 << 2; ++ } else if (mali_is_mali450()) { ++ domain_config[MALI_DOMAIN_INDEX_PP0] = 0x01 << 1; ++ } else if (mali_is_mali470()) { ++ domain_config[MALI_DOMAIN_INDEX_PP0] = 0x01 << 0; ++ } ++ } ++ ++ if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( ++ MALI_OFFSET_PP1, NULL)) { ++ if (mali_is_mali400()) { ++ domain_config[MALI_DOMAIN_INDEX_PP1] = 0x01 << 3; ++ } else if (mali_is_mali450()) { ++ domain_config[MALI_DOMAIN_INDEX_PP1] = 0x01 << 2; ++ } else if (mali_is_mali470()) { ++ domain_config[MALI_DOMAIN_INDEX_PP1] = 0x01 << 1; ++ } ++ } ++ ++ if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( ++ MALI_OFFSET_PP2, NULL)) { ++ if (mali_is_mali400()) { ++ domain_config[MALI_DOMAIN_INDEX_PP2] = 0x01 << 4; ++ } else if (mali_is_mali450()) { ++ domain_config[MALI_DOMAIN_INDEX_PP2] = 0x01 << 2; ++ } else if (mali_is_mali470()) { ++ domain_config[MALI_DOMAIN_INDEX_PP2] = 0x01 << 1; ++ } ++ } ++ ++ if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( ++ MALI_OFFSET_PP3, NULL)) { ++ if (mali_is_mali400()) { ++ domain_config[MALI_DOMAIN_INDEX_PP3] = 0x01 << 5; ++ } else if (mali_is_mali450()) { ++ domain_config[MALI_DOMAIN_INDEX_PP3] = 0x01 << 2; ++ } else if (mali_is_mali470()) { ++ domain_config[MALI_DOMAIN_INDEX_PP3] = 0x01 << 1; ++ } ++ } ++ ++ /* PP4 - PP7 */ ++ if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( ++ MALI_OFFSET_PP4, NULL)) { ++ domain_config[MALI_DOMAIN_INDEX_PP4] = 0x01 << 3; ++ } ++ ++ if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( ++ MALI_OFFSET_PP5, NULL)) { ++ domain_config[MALI_DOMAIN_INDEX_PP5] = 0x01 << 3; ++ } ++ ++ if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( ++ MALI_OFFSET_PP6, NULL)) { ++ domain_config[MALI_DOMAIN_INDEX_PP6] = 0x01 << 3; ++ } ++ ++ if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( ++ MALI_OFFSET_PP7, NULL)) { ++ domain_config[MALI_DOMAIN_INDEX_PP7] = 0x01 << 3; ++ } ++ ++ /* L2gp/L2PP0/L2PP4 */ ++ if (mali_is_mali400()) { ++ if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( ++ MALI400_OFFSET_L2_CACHE0, NULL)) { ++ domain_config[MALI_DOMAIN_INDEX_L20] = 0x01 << 1; ++ } ++ } else if (mali_is_mali450()) { ++ if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( ++ MALI450_OFFSET_L2_CACHE0, NULL)) { ++ domain_config[MALI_DOMAIN_INDEX_L20] = 0x01 << 0; ++ } ++ ++ if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( ++ MALI450_OFFSET_L2_CACHE1, NULL)) { ++ domain_config[MALI_DOMAIN_INDEX_L21] = 0x01 << 1; ++ } ++ ++ if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( ++ MALI450_OFFSET_L2_CACHE2, NULL)) { ++ domain_config[MALI_DOMAIN_INDEX_L22] = 0x01 << 3; ++ } ++ } else if (mali_is_mali470()) { ++ if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( ++ MALI470_OFFSET_L2_CACHE1, NULL)) { ++ domain_config[MALI_DOMAIN_INDEX_L21] = 0x01 << 0; ++ } ++ } ++} ++ ++static u32 mali_pm_get_registered_cores_mask(void) ++{ ++ int i = 0; ++ u32 mask = 0; ++ ++ for (i = 0; i < MALI_DOMAIN_INDEX_DUMMY; i++) { ++ mask |= domain_config[i]; ++ } ++ ++ return mask; ++} ++ ++static void mali_pm_set_pmu_domain_config(void) ++{ ++ int i = 0; ++ ++ _mali_osk_device_data_pmu_config_get(domain_config, MALI_MAX_NUMBER_OF_DOMAINS - 1); ++ ++ for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS - 1; i++) { ++ if (0 != domain_config[i]) { ++ MALI_DEBUG_PRINT(2, ("Using customer pmu config:\n")); ++ break; ++ } ++ } ++ ++ if (MALI_MAX_NUMBER_OF_DOMAINS - 1 == i) { ++ MALI_DEBUG_PRINT(2, ("Using hw detect pmu config:\n")); ++ mali_pm_set_default_pm_domain_config(); ++ } ++ ++ for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS - 1; i++) { ++ if (domain_config[i]) { ++ MALI_DEBUG_PRINT(2, ("domain_config[%d] = 0x%x \n", i, domain_config[i])); ++ } ++ } ++ /* Can't override dummy domain mask */ ++ domain_config[MALI_DOMAIN_INDEX_DUMMY] = ++ 1 << MALI_DOMAIN_INDEX_DUMMY; ++} ++ ++#if defined(DEBUG) ++const char *mali_pm_mask_to_string(u32 mask) ++{ ++ static char bit_str[MALI_MAX_NUMBER_OF_DOMAINS + 1]; ++ int bit; ++ int str_pos = 0; ++ ++ /* Must be protected by lock since we use shared string buffer */ ++ if (NULL != pm_lock_exec) { ++ MALI_DEBUG_ASSERT_LOCK_HELD(pm_lock_exec); ++ } ++ ++ for (bit = MALI_MAX_NUMBER_OF_DOMAINS - 1; bit >= 0; bit--) { ++ if (mask & (1 << bit)) { ++ bit_str[str_pos] = 'X'; ++ } else { ++ bit_str[str_pos] = '-'; ++ } ++ str_pos++; ++ } ++ ++ bit_str[MALI_MAX_NUMBER_OF_DOMAINS] = '\0'; ++ ++ return bit_str; ++} ++ ++const char *mali_pm_group_stats_to_string(void) ++{ ++ static char bit_str[MALI_MAX_NUMBER_OF_GROUPS + 1]; ++ u32 num_groups = mali_group_get_glob_num_groups(); ++ u32 i; ++ ++ /* Must be protected by lock since we use shared string buffer */ ++ if (NULL != pm_lock_exec) { ++ MALI_DEBUG_ASSERT_LOCK_HELD(pm_lock_exec); ++ } ++ ++ for (i = 0; i < num_groups && i < MALI_MAX_NUMBER_OF_GROUPS; i++) { ++ struct mali_group *group; ++ ++ group = mali_group_get_glob_group(i); ++ ++ if (MALI_TRUE == mali_group_power_is_on(group)) { ++ bit_str[i] = 'X'; ++ } else { ++ bit_str[i] = '-'; ++ } ++ } ++ ++ bit_str[i] = '\0'; ++ ++ return bit_str; ++} ++#endif ++ ++/* ++ * num_pp is the number of PP cores which will be powered on given this mask ++ * cost is the total power cost of cores which will be powered on given this mask ++ */ ++static void mali_pm_stat_from_mask(u32 mask, u32 *num_pp, u32 *cost) ++{ ++ u32 i; ++ ++ /* loop through all cores */ ++ for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) { ++ if (!(domain_config[i] & mask)) { ++ continue; ++ } ++ ++ switch (i) { ++ case MALI_DOMAIN_INDEX_GP: ++ *cost += MALI_GP_COST; ++ ++ break; ++ case MALI_DOMAIN_INDEX_PP0: /* Fall through */ ++ case MALI_DOMAIN_INDEX_PP1: /* Fall through */ ++ case MALI_DOMAIN_INDEX_PP2: /* Fall through */ ++ case MALI_DOMAIN_INDEX_PP3: ++ if (mali_is_mali400()) { ++ if ((domain_config[MALI_DOMAIN_INDEX_L20] & mask) ++ || (domain_config[MALI_DOMAIN_INDEX_DUMMY] ++ == domain_config[MALI_DOMAIN_INDEX_L20])) { ++ *num_pp += 1; ++ } ++ } else { ++ if ((domain_config[MALI_DOMAIN_INDEX_L21] & mask) ++ || (domain_config[MALI_DOMAIN_INDEX_DUMMY] ++ == domain_config[MALI_DOMAIN_INDEX_L21])) { ++ *num_pp += 1; ++ } ++ } ++ ++ *cost += MALI_PP_COST; ++ break; ++ case MALI_DOMAIN_INDEX_PP4: /* Fall through */ ++ case MALI_DOMAIN_INDEX_PP5: /* Fall through */ ++ case MALI_DOMAIN_INDEX_PP6: /* Fall through */ ++ case MALI_DOMAIN_INDEX_PP7: ++ MALI_DEBUG_ASSERT(mali_is_mali450()); ++ ++ if ((domain_config[MALI_DOMAIN_INDEX_L22] & mask) ++ || (domain_config[MALI_DOMAIN_INDEX_DUMMY] ++ == domain_config[MALI_DOMAIN_INDEX_L22])) { ++ *num_pp += 1; ++ } ++ ++ *cost += MALI_PP_COST; ++ break; ++ case MALI_DOMAIN_INDEX_L20: /* Fall through */ ++ case MALI_DOMAIN_INDEX_L21: /* Fall through */ ++ case MALI_DOMAIN_INDEX_L22: ++ *cost += MALI_L2_COST; ++ ++ break; ++ } ++ } ++} ++ ++void mali_pm_power_cost_setup(void) ++{ ++ /* ++ * Two parallel arrays which store the best domain mask and its cost ++ * The index is the number of PP cores, E.g. Index 0 is for 1 PP option, ++ * might have mask 0x2 and with cost of 1, lower cost is better ++ */ ++ u32 best_mask[MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS] = { 0 }; ++ u32 best_cost[MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS] = { 0 }; ++ /* Array cores_in_domain is used to store the total pp cores in each pm domain. */ ++ u32 cores_in_domain[MALI_MAX_NUMBER_OF_DOMAINS] = { 0 }; ++ /* Domain_count is used to represent the max domain we have.*/ ++ u32 max_domain_mask = 0; ++ u32 max_domain_id = 0; ++ u32 always_on_pp_cores = 0; ++ ++ u32 num_pp, cost, mask; ++ u32 i, j , k; ++ ++ /* Initialize statistics */ ++ for (i = 0; i < MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS; i++) { ++ best_mask[i] = 0; ++ best_cost[i] = 0xFFFFFFFF; /* lower cost is better */ ++ } ++ ++ for (i = 0; i < MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS + 1; i++) { ++ for (j = 0; j < MALI_MAX_NUMBER_OF_DOMAINS; j++) { ++ mali_pm_domain_power_cost_result[i][j] = 0; ++ } ++ } ++ ++ /* Caculate number of pp cores of a given domain config. */ ++ for (i = MALI_DOMAIN_INDEX_PP0; i <= MALI_DOMAIN_INDEX_PP7; i++) { ++ if (0 < domain_config[i]) { ++ /* Get the max domain mask value used to caculate power cost ++ * and we don't count in always on pp cores. */ ++ if (MALI_PM_DOMAIN_DUMMY_MASK != domain_config[i] ++ && max_domain_mask < domain_config[i]) { ++ max_domain_mask = domain_config[i]; ++ } ++ ++ if (MALI_PM_DOMAIN_DUMMY_MASK == domain_config[i]) { ++ always_on_pp_cores++; ++ } ++ } ++ } ++ max_domain_id = _mali_osk_fls(max_domain_mask); ++ ++ /* ++ * Try all combinations of power domains and check how many PP cores ++ * they have and their power cost. ++ */ ++ for (mask = 0; mask < (1 << max_domain_id); mask++) { ++ num_pp = 0; ++ cost = 0; ++ ++ mali_pm_stat_from_mask(mask, &num_pp, &cost); ++ ++ /* This mask is usable for all MP1 up to num_pp PP cores, check statistics for all */ ++ for (i = 0; i < num_pp; i++) { ++ if (best_cost[i] >= cost) { ++ best_cost[i] = cost; ++ best_mask[i] = mask; ++ } ++ } ++ } ++ ++ /* ++ * If we want to enable x pp cores, if x is less than number of always_on pp cores, ++ * all of pp cores we will enable must be always_on pp cores. ++ */ ++ for (i = 0; i < mali_executor_get_num_cores_total(); i++) { ++ if (i < always_on_pp_cores) { ++ mali_pm_domain_power_cost_result[i + 1][MALI_MAX_NUMBER_OF_DOMAINS - 1] ++ = i + 1; ++ } else { ++ mali_pm_domain_power_cost_result[i + 1][MALI_MAX_NUMBER_OF_DOMAINS - 1] ++ = always_on_pp_cores; ++ } ++ } ++ ++ /* In this loop, variable i represent for the number of non-always on pp cores we want to enabled. */ ++ for (i = 0; i < (mali_executor_get_num_cores_total() - always_on_pp_cores); i++) { ++ if (best_mask[i] == 0) { ++ /* This MP variant is not available */ ++ continue; ++ } ++ ++ for (j = 0; j < MALI_MAX_NUMBER_OF_DOMAINS; j++) { ++ cores_in_domain[j] = 0; ++ } ++ ++ for (j = MALI_DOMAIN_INDEX_PP0; j <= MALI_DOMAIN_INDEX_PP7; j++) { ++ if (0 < domain_config[j] ++ && (MALI_PM_DOMAIN_DUMMY_MASK != domain_config[i])) { ++ cores_in_domain[_mali_osk_fls(domain_config[j]) - 1]++; ++ } ++ } ++ ++ /* In this loop, j represent for the number we have already enabled.*/ ++ for (j = 0; j <= i;) { ++ /* j used to visit all of domain to get the number of pp cores remained in it. */ ++ for (k = 0; k < max_domain_id; k++) { ++ /* If domain k in best_mask[i] is enabled and this domain has extra pp cores, ++ * we know we must pick at least one pp core from this domain. ++ * And then we move to next enabled pm domain. */ ++ if ((best_mask[i] & (0x1 << k)) && (0 < cores_in_domain[k])) { ++ cores_in_domain[k]--; ++ mali_pm_domain_power_cost_result[always_on_pp_cores + i + 1][k]++; ++ j++; ++ if (j > i) { ++ break; ++ } ++ } ++ } ++ } ++ } ++} ++ ++/* ++ * When we are doing core scaling, ++ * this function is called to return the best mask to ++ * achieve the best pp group power cost. ++ */ ++void mali_pm_get_best_power_cost_mask(int num_requested, int *dst) ++{ ++ MALI_DEBUG_ASSERT((mali_executor_get_num_cores_total() >= num_requested) && (0 <= num_requested)); ++ ++ _mali_osk_memcpy(dst, mali_pm_domain_power_cost_result[num_requested], MALI_MAX_NUMBER_OF_DOMAINS * sizeof(int)); ++} ++ ++u32 mali_pm_get_current_mask(void) ++{ ++ return pd_mask_current; ++} ++ ++u32 mali_pm_get_wanted_mask(void) ++{ ++ return pd_mask_wanted; ++} +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_pm.h b/drivers/gpu/arm/mali400/mali/common/mali_pm.h +new file mode 100755 +index 000000000000..dac69958e034 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_pm.h +@@ -0,0 +1,91 @@ ++/* ++ * Copyright (C) 2011-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_PM_H__ ++#define __MALI_PM_H__ ++ ++#include "mali_osk.h" ++#include "mali_pm_domain.h" ++ ++#define MALI_DOMAIN_INDEX_GP 0 ++#define MALI_DOMAIN_INDEX_PP0 1 ++#define MALI_DOMAIN_INDEX_PP1 2 ++#define MALI_DOMAIN_INDEX_PP2 3 ++#define MALI_DOMAIN_INDEX_PP3 4 ++#define MALI_DOMAIN_INDEX_PP4 5 ++#define MALI_DOMAIN_INDEX_PP5 6 ++#define MALI_DOMAIN_INDEX_PP6 7 ++#define MALI_DOMAIN_INDEX_PP7 8 ++#define MALI_DOMAIN_INDEX_L20 9 ++#define MALI_DOMAIN_INDEX_L21 10 ++#define MALI_DOMAIN_INDEX_L22 11 ++/* ++ * The dummy domain is used when there is no physical power domain ++ * (e.g. no PMU or always on cores) ++ */ ++#define MALI_DOMAIN_INDEX_DUMMY 12 ++#define MALI_MAX_NUMBER_OF_DOMAINS 13 ++ ++/** ++ * Initialize the Mali PM module ++ * ++ * PM module covers Mali PM core, PM domains and Mali PMU ++ */ ++_mali_osk_errcode_t mali_pm_initialize(void); ++ ++/** ++ * Terminate the Mali PM module ++ */ ++void mali_pm_terminate(void); ++ ++void mali_pm_exec_lock(void); ++void mali_pm_exec_unlock(void); ++ ++ ++struct mali_pm_domain *mali_pm_register_l2_cache(u32 domain_index, ++ struct mali_l2_cache_core *l2_cache); ++struct mali_pm_domain *mali_pm_register_group(u32 domain_index, ++ struct mali_group *group); ++ ++mali_bool mali_pm_get_domain_refs(struct mali_pm_domain **domains, ++ struct mali_group **groups, ++ u32 num_domains); ++mali_bool mali_pm_put_domain_refs(struct mali_pm_domain **domains, ++ u32 num_domains); ++ ++void mali_pm_init_begin(void); ++void mali_pm_init_end(void); ++ ++void mali_pm_update_sync(void); ++void mali_pm_update_async(void); ++ ++/* Callback functions for system power management */ ++void mali_pm_os_suspend(mali_bool os_suspend); ++void mali_pm_os_resume(void); ++ ++mali_bool mali_pm_runtime_suspend(void); ++void mali_pm_runtime_resume(void); ++ ++#if MALI_STATE_TRACKING ++u32 mali_pm_dump_state_domain(struct mali_pm_domain *domain, ++ char *buf, u32 size); ++#endif ++ ++void mali_pm_power_cost_setup(void); ++ ++void mali_pm_get_best_power_cost_mask(int num_requested, int *dst); ++ ++#if defined(DEBUG) ++const char *mali_pm_mask_to_string(u32 mask); ++#endif ++ ++u32 mali_pm_get_current_mask(void); ++u32 mali_pm_get_wanted_mask(void); ++#endif /* __MALI_PM_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_pm_domain.c b/drivers/gpu/arm/mali400/mali/common/mali_pm_domain.c +new file mode 100755 +index 000000000000..8290f7d88f6a +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_pm_domain.c +@@ -0,0 +1,209 @@ ++/* ++ * Copyright (C) 2013-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include "mali_kernel_common.h" ++#include "mali_osk.h" ++#include "mali_pm_domain.h" ++#include "mali_pmu.h" ++#include "mali_group.h" ++#include "mali_pm.h" ++ ++static struct mali_pm_domain *mali_pm_domains[MALI_MAX_NUMBER_OF_DOMAINS] = ++{ NULL, }; ++ ++void mali_pm_domain_initialize(void) ++{ ++ /* Domains will be initialized/created on demand */ ++} ++ ++void mali_pm_domain_terminate(void) ++{ ++ int i; ++ ++ /* Delete all domains that has been created */ ++ for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) { ++ mali_pm_domain_delete(mali_pm_domains[i]); ++ mali_pm_domains[i] = NULL; ++ } ++} ++ ++struct mali_pm_domain *mali_pm_domain_create(u32 pmu_mask) ++{ ++ struct mali_pm_domain *domain = NULL; ++ u32 domain_id = 0; ++ ++ domain = mali_pm_domain_get_from_mask(pmu_mask); ++ if (NULL != domain) return domain; ++ ++ MALI_DEBUG_PRINT(2, ++ ("Mali PM domain: Creating Mali PM domain (mask=0x%08X)\n", ++ pmu_mask)); ++ ++ domain = (struct mali_pm_domain *)_mali_osk_malloc( ++ sizeof(struct mali_pm_domain)); ++ if (NULL != domain) { ++ domain->power_is_on = MALI_FALSE; ++ domain->pmu_mask = pmu_mask; ++ domain->use_count = 0; ++ _mali_osk_list_init(&domain->group_list); ++ _mali_osk_list_init(&domain->l2_cache_list); ++ ++ domain_id = _mali_osk_fls(pmu_mask) - 1; ++ /* Verify the domain_id */ ++ MALI_DEBUG_ASSERT(MALI_MAX_NUMBER_OF_DOMAINS > domain_id); ++ /* Verify that pmu_mask only one bit is set */ ++ MALI_DEBUG_ASSERT((1 << domain_id) == pmu_mask); ++ mali_pm_domains[domain_id] = domain; ++ ++ return domain; ++ } else { ++ MALI_DEBUG_PRINT_ERROR(("Unable to create PM domain\n")); ++ } ++ ++ return NULL; ++} ++ ++void mali_pm_domain_delete(struct mali_pm_domain *domain) ++{ ++ if (NULL == domain) { ++ return; ++ } ++ ++ _mali_osk_list_delinit(&domain->group_list); ++ _mali_osk_list_delinit(&domain->l2_cache_list); ++ ++ _mali_osk_free(domain); ++} ++ ++void mali_pm_domain_add_group(struct mali_pm_domain *domain, ++ struct mali_group *group) ++{ ++ MALI_DEBUG_ASSERT_POINTER(domain); ++ MALI_DEBUG_ASSERT_POINTER(group); ++ ++ /* ++ * Use addtail because virtual group is created last and it needs ++ * to be at the end of the list (in order to be activated after ++ * all children. ++ */ ++ _mali_osk_list_addtail(&group->pm_domain_list, &domain->group_list); ++} ++ ++void mali_pm_domain_add_l2_cache(struct mali_pm_domain *domain, ++ struct mali_l2_cache_core *l2_cache) ++{ ++ MALI_DEBUG_ASSERT_POINTER(domain); ++ MALI_DEBUG_ASSERT_POINTER(l2_cache); ++ _mali_osk_list_add(&l2_cache->pm_domain_list, &domain->l2_cache_list); ++} ++ ++struct mali_pm_domain *mali_pm_domain_get_from_mask(u32 mask) ++{ ++ u32 id = 0; ++ ++ if (0 == mask) { ++ return NULL; ++ } ++ ++ id = _mali_osk_fls(mask) - 1; ++ ++ MALI_DEBUG_ASSERT(MALI_MAX_NUMBER_OF_DOMAINS > id); ++ /* Verify that pmu_mask only one bit is set */ ++ MALI_DEBUG_ASSERT((1 << id) == mask); ++ ++ return mali_pm_domains[id]; ++} ++ ++struct mali_pm_domain *mali_pm_domain_get_from_index(u32 id) ++{ ++ MALI_DEBUG_ASSERT(MALI_MAX_NUMBER_OF_DOMAINS > id); ++ ++ return mali_pm_domains[id]; ++} ++ ++u32 mali_pm_domain_ref_get(struct mali_pm_domain *domain) ++{ ++ MALI_DEBUG_ASSERT_POINTER(domain); ++ ++ if (0 == domain->use_count) { ++ _mali_osk_pm_dev_ref_get_async(); ++ } ++ ++ ++domain->use_count; ++ MALI_DEBUG_PRINT(4, ("PM domain %p: ref_get, use_count => %u\n", domain, domain->use_count)); ++ ++ /* Return our mask so caller can check this against wanted mask */ ++ return domain->pmu_mask; ++} ++ ++u32 mali_pm_domain_ref_put(struct mali_pm_domain *domain) ++{ ++ MALI_DEBUG_ASSERT_POINTER(domain); ++ ++ --domain->use_count; ++ MALI_DEBUG_PRINT(4, ("PM domain %p: ref_put, use_count => %u\n", domain, domain->use_count)); ++ ++ if (0 == domain->use_count) { ++ _mali_osk_pm_dev_ref_put(); ++ } ++ ++ /* ++ * Return the PMU mask which now could be be powered down ++ * (the bit for this domain). ++ * This is the responsibility of the caller (mali_pm) ++ */ ++ return (0 == domain->use_count ? domain->pmu_mask : 0); ++} ++ ++#if MALI_STATE_TRACKING ++u32 mali_pm_domain_get_id(struct mali_pm_domain *domain) ++{ ++ u32 id = 0; ++ ++ MALI_DEBUG_ASSERT_POINTER(domain); ++ MALI_DEBUG_ASSERT(0 != domain->pmu_mask); ++ ++ id = _mali_osk_fls(domain->pmu_mask) - 1; ++ ++ MALI_DEBUG_ASSERT(MALI_MAX_NUMBER_OF_DOMAINS > id); ++ /* Verify that pmu_mask only one bit is set */ ++ MALI_DEBUG_ASSERT((1 << id) == domain->pmu_mask); ++ /* Verify that we have stored the domain at right id/index */ ++ MALI_DEBUG_ASSERT(domain == mali_pm_domains[id]); ++ ++ return id; ++} ++#endif ++ ++#if defined(DEBUG) ++mali_bool mali_pm_domain_all_unused(void) ++{ ++ int i; ++ ++ for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) { ++ if (NULL == mali_pm_domains[i]) { ++ /* Nothing to check */ ++ continue; ++ } ++ ++ if (MALI_TRUE == mali_pm_domains[i]->power_is_on) { ++ /* Not ready for suspend! */ ++ return MALI_FALSE; ++ } ++ ++ if (0 != mali_pm_domains[i]->use_count) { ++ /* Not ready for suspend! */ ++ return MALI_FALSE; ++ } ++ } ++ ++ return MALI_TRUE; ++} ++#endif +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_pm_domain.h b/drivers/gpu/arm/mali400/mali/common/mali_pm_domain.h +new file mode 100755 +index 000000000000..5776abe39f3d +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_pm_domain.h +@@ -0,0 +1,104 @@ ++/* ++ * Copyright (C) 2013-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_PM_DOMAIN_H__ ++#define __MALI_PM_DOMAIN_H__ ++ ++#include "mali_kernel_common.h" ++#include "mali_osk.h" ++ ++#include "mali_l2_cache.h" ++#include "mali_group.h" ++#include "mali_pmu.h" ++ ++/* Instances are protected by PM state lock */ ++struct mali_pm_domain { ++ mali_bool power_is_on; ++ s32 use_count; ++ u32 pmu_mask; ++ ++ /* Zero or more groups can belong to this domain */ ++ _mali_osk_list_t group_list; ++ ++ /* Zero or more L2 caches can belong to this domain */ ++ _mali_osk_list_t l2_cache_list; ++}; ++ ++ ++void mali_pm_domain_initialize(void); ++void mali_pm_domain_terminate(void); ++ ++struct mali_pm_domain *mali_pm_domain_create(u32 pmu_mask); ++void mali_pm_domain_delete(struct mali_pm_domain *domain); ++ ++void mali_pm_domain_add_l2_cache( ++ struct mali_pm_domain *domain, ++ struct mali_l2_cache_core *l2_cache); ++void mali_pm_domain_add_group(struct mali_pm_domain *domain, ++ struct mali_group *group); ++ ++struct mali_pm_domain *mali_pm_domain_get_from_mask(u32 mask); ++struct mali_pm_domain *mali_pm_domain_get_from_index(u32 id); ++ ++/* Ref counting */ ++u32 mali_pm_domain_ref_get(struct mali_pm_domain *domain); ++u32 mali_pm_domain_ref_put(struct mali_pm_domain *domain); ++ ++MALI_STATIC_INLINE _mali_osk_list_t *mali_pm_domain_get_group_list( ++ struct mali_pm_domain *domain) ++{ ++ MALI_DEBUG_ASSERT_POINTER(domain); ++ return &domain->group_list; ++} ++ ++MALI_STATIC_INLINE _mali_osk_list_t *mali_pm_domain_get_l2_cache_list( ++ struct mali_pm_domain *domain) ++{ ++ MALI_DEBUG_ASSERT_POINTER(domain); ++ return &domain->l2_cache_list; ++} ++ ++MALI_STATIC_INLINE mali_bool mali_pm_domain_power_is_on( ++ struct mali_pm_domain *domain) ++{ ++ MALI_DEBUG_ASSERT_POINTER(domain); ++ return domain->power_is_on; ++} ++ ++MALI_STATIC_INLINE void mali_pm_domain_set_power_on( ++ struct mali_pm_domain *domain, ++ mali_bool power_is_on) ++{ ++ MALI_DEBUG_ASSERT_POINTER(domain); ++ domain->power_is_on = power_is_on; ++} ++ ++MALI_STATIC_INLINE u32 mali_pm_domain_get_use_count( ++ struct mali_pm_domain *domain) ++{ ++ MALI_DEBUG_ASSERT_POINTER(domain); ++ return domain->use_count; ++} ++ ++#if MALI_STATE_TRACKING ++u32 mali_pm_domain_get_id(struct mali_pm_domain *domain); ++ ++MALI_STATIC_INLINE u32 mali_pm_domain_get_mask(struct mali_pm_domain *domain) ++{ ++ MALI_DEBUG_ASSERT_POINTER(domain); ++ return domain->pmu_mask; ++} ++#endif ++ ++#if defined(DEBUG) ++mali_bool mali_pm_domain_all_unused(void); ++#endif ++ ++#endif /* __MALI_PM_DOMAIN_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_pm_metrics.c b/drivers/gpu/arm/mali400/mali/common/mali_pm_metrics.c +new file mode 100755 +index 000000000000..cf74823230f7 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_pm_metrics.c +@@ -0,0 +1,255 @@ ++/* ++ * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++#include "mali_pm_metrics.h" ++#include "mali_osk_locks.h" ++#include "mali_osk_mali.h" ++#include ++ ++#define MALI_PM_TIME_SHIFT 0 ++#define MALI_UTILIZATION_MAX_PERIOD 80000000/* ns = 100ms */ ++ ++_mali_osk_errcode_t mali_pm_metrics_init(struct mali_device *mdev) ++{ ++ int i = 0; ++ ++ MALI_DEBUG_ASSERT(mdev != NULL); ++ ++ mdev->mali_metrics.time_period_start = ktime_get(); ++ mdev->mali_metrics.time_period_start_gp = mdev->mali_metrics.time_period_start; ++ mdev->mali_metrics.time_period_start_pp = mdev->mali_metrics.time_period_start; ++ ++ mdev->mali_metrics.time_busy = 0; ++ mdev->mali_metrics.time_idle = 0; ++ mdev->mali_metrics.prev_busy = 0; ++ mdev->mali_metrics.prev_idle = 0; ++ mdev->mali_metrics.num_running_gp_cores = 0; ++ mdev->mali_metrics.num_running_pp_cores = 0; ++ mdev->mali_metrics.time_busy_gp = 0; ++ mdev->mali_metrics.time_idle_gp = 0; ++ ++ for (i = 0; i < MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS; i++) { ++ mdev->mali_metrics.time_busy_pp[i] = 0; ++ mdev->mali_metrics.time_idle_pp[i] = 0; ++ } ++ mdev->mali_metrics.gpu_active = MALI_FALSE; ++ ++ mdev->mali_metrics.lock = _mali_osk_spinlock_irq_init(_MALI_OSK_LOCKFLAG_UNORDERED, _MALI_OSK_LOCK_ORDER_FIRST); ++ if (NULL == mdev->mali_metrics.lock) { ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++void mali_pm_metrics_term(struct mali_device *mdev) ++{ ++ _mali_osk_spinlock_irq_term(mdev->mali_metrics.lock); ++} ++ ++/*caller needs to hold mdev->mali_metrics.lock before calling this function*/ ++void mali_pm_record_job_status(struct mali_device *mdev) ++{ ++ ktime_t now; ++ ktime_t diff; ++ u64 ns_time; ++ ++ MALI_DEBUG_ASSERT(mdev != NULL); ++ ++ now = ktime_get(); ++ diff = ktime_sub(now, mdev->mali_metrics.time_period_start); ++ ++ ns_time = (u64)(ktime_to_ns(diff) >> MALI_PM_TIME_SHIFT); ++ mdev->mali_metrics.time_busy += ns_time; ++ mdev->mali_metrics.time_period_start = now; ++} ++ ++void mali_pm_record_gpu_idle(mali_bool is_gp) ++{ ++ ktime_t now; ++ ktime_t diff; ++ u64 ns_time; ++ struct mali_device *mdev = dev_get_drvdata(&mali_platform_device->dev); ++ ++ MALI_DEBUG_ASSERT(mdev != NULL); ++ ++ _mali_osk_spinlock_irq_lock(mdev->mali_metrics.lock); ++ now = ktime_get(); ++ ++ if (MALI_TRUE == is_gp) { ++ --mdev->mali_metrics.num_running_gp_cores; ++ if (0 == mdev->mali_metrics.num_running_gp_cores) { ++ diff = ktime_sub(now, mdev->mali_metrics.time_period_start_gp); ++ ns_time = (u64)(ktime_to_ns(diff) >> MALI_PM_TIME_SHIFT); ++ mdev->mali_metrics.time_busy_gp += ns_time; ++ mdev->mali_metrics.time_period_start_gp = now; ++ ++ if (0 == mdev->mali_metrics.num_running_pp_cores) { ++ MALI_DEBUG_ASSERT(mdev->mali_metrics.gpu_active == MALI_TRUE); ++ diff = ktime_sub(now, mdev->mali_metrics.time_period_start); ++ ns_time = (u64)(ktime_to_ns(diff) >> MALI_PM_TIME_SHIFT); ++ mdev->mali_metrics.time_busy += ns_time; ++ mdev->mali_metrics.time_period_start = now; ++ mdev->mali_metrics.gpu_active = MALI_FALSE; ++ } ++ } ++ } else { ++ --mdev->mali_metrics.num_running_pp_cores; ++ if (0 == mdev->mali_metrics.num_running_pp_cores) { ++ diff = ktime_sub(now, mdev->mali_metrics.time_period_start_pp); ++ ns_time = (u64)(ktime_to_ns(diff) >> MALI_PM_TIME_SHIFT); ++ mdev->mali_metrics.time_busy_pp[0] += ns_time; ++ mdev->mali_metrics.time_period_start_pp = now; ++ ++ if (0 == mdev->mali_metrics.num_running_gp_cores) { ++ MALI_DEBUG_ASSERT(mdev->mali_metrics.gpu_active == MALI_TRUE); ++ diff = ktime_sub(now, mdev->mali_metrics.time_period_start); ++ ns_time = (u64)(ktime_to_ns(diff) >> MALI_PM_TIME_SHIFT); ++ mdev->mali_metrics.time_busy += ns_time; ++ mdev->mali_metrics.time_period_start = now; ++ mdev->mali_metrics.gpu_active = MALI_FALSE; ++ } ++ } ++ } ++ ++ _mali_osk_spinlock_irq_unlock(mdev->mali_metrics.lock); ++} ++ ++void mali_pm_record_gpu_active(mali_bool is_gp) ++{ ++ ktime_t now; ++ ktime_t diff; ++ struct mali_device *mdev = dev_get_drvdata(&mali_platform_device->dev); ++ ++ MALI_DEBUG_ASSERT(mdev != NULL); ++ ++ _mali_osk_spinlock_irq_lock(mdev->mali_metrics.lock); ++ now = ktime_get(); ++ ++ if (MALI_TRUE == is_gp) { ++ mdev->mali_metrics.num_running_gp_cores++; ++ if (1 == mdev->mali_metrics.num_running_gp_cores) { ++ diff = ktime_sub(now, mdev->mali_metrics.time_period_start_gp); ++ mdev->mali_metrics.time_idle_gp += (u64)(ktime_to_ns(diff) >> MALI_PM_TIME_SHIFT); ++ mdev->mali_metrics.time_period_start_gp = now; ++ if (0 == mdev->mali_metrics.num_running_pp_cores) { ++ MALI_DEBUG_ASSERT(mdev->mali_metrics.gpu_active == MALI_FALSE); ++ diff = ktime_sub(now, mdev->mali_metrics.time_period_start); ++ mdev->mali_metrics.time_idle += (u64)(ktime_to_ns(diff) >> MALI_PM_TIME_SHIFT); ++ mdev->mali_metrics.time_period_start = now; ++ mdev->mali_metrics.gpu_active = MALI_TRUE; ++ } ++ } else { ++ MALI_DEBUG_ASSERT(mdev->mali_metrics.gpu_active == MALI_TRUE); ++ } ++ } else { ++ mdev->mali_metrics.num_running_pp_cores++; ++ if (1 == mdev->mali_metrics.num_running_pp_cores) { ++ diff = ktime_sub(now, mdev->mali_metrics.time_period_start_pp); ++ mdev->mali_metrics.time_idle_pp[0] += (u64)(ktime_to_ns(diff) >> MALI_PM_TIME_SHIFT); ++ mdev->mali_metrics.time_period_start_pp = now; ++ if (0 == mdev->mali_metrics.num_running_gp_cores) { ++ MALI_DEBUG_ASSERT(mdev->mali_metrics.gpu_active == MALI_FALSE); ++ diff = ktime_sub(now, mdev->mali_metrics.time_period_start); ++ mdev->mali_metrics.time_idle += (u64)(ktime_to_ns(diff) >> MALI_PM_TIME_SHIFT); ++ mdev->mali_metrics.time_period_start = now; ++ mdev->mali_metrics.gpu_active = MALI_TRUE; ++ } ++ } else { ++ MALI_DEBUG_ASSERT(mdev->mali_metrics.gpu_active == MALI_TRUE); ++ } ++ } ++ ++ _mali_osk_spinlock_irq_unlock(mdev->mali_metrics.lock); ++} ++ ++ ++/*caller needs to hold mdev->mali_metrics.lock before calling this function*/ ++static void mali_pm_get_dvfs_utilisation_calc(struct mali_device *mdev, ktime_t now) ++{ ++ ktime_t diff; ++ ++ MALI_DEBUG_ASSERT(mdev != NULL); ++ ++ diff = ktime_sub(now, mdev->mali_metrics.time_period_start); ++ ++ if (mdev->mali_metrics.gpu_active) { ++ mdev->mali_metrics.time_busy += (u64)(ktime_to_ns(diff) >> MALI_PM_TIME_SHIFT); ++ } else { ++ mdev->mali_metrics.time_idle += (u64)(ktime_to_ns(diff) >> MALI_PM_TIME_SHIFT); ++ } ++} ++ ++/* Caller needs to hold mdev->mali_metrics.lock before calling this function. */ ++static void mali_pm_reset_dvfs_utilisation_unlocked(struct mali_device *mdev, ktime_t now) ++{ ++ /* Store previous value */ ++ mdev->mali_metrics.prev_idle = mdev->mali_metrics.time_idle; ++ mdev->mali_metrics.prev_busy = mdev->mali_metrics.time_busy; ++ ++ /* Reset current values */ ++ mdev->mali_metrics.time_period_start = now; ++ mdev->mali_metrics.time_period_start_gp = now; ++ mdev->mali_metrics.time_period_start_pp = now; ++ mdev->mali_metrics.time_idle = 0; ++ mdev->mali_metrics.time_busy = 0; ++ ++ mdev->mali_metrics.time_busy_gp = 0; ++ mdev->mali_metrics.time_idle_gp = 0; ++ mdev->mali_metrics.time_busy_pp[0] = 0; ++ mdev->mali_metrics.time_idle_pp[0] = 0; ++} ++ ++void mali_pm_reset_dvfs_utilisation(struct mali_device *mdev) ++{ ++ _mali_osk_spinlock_irq_lock(mdev->mali_metrics.lock); ++ mali_pm_reset_dvfs_utilisation_unlocked(mdev, ktime_get()); ++ _mali_osk_spinlock_irq_unlock(mdev->mali_metrics.lock); ++} ++ ++void mali_pm_get_dvfs_utilisation(struct mali_device *mdev, ++ unsigned long *total_out, unsigned long *busy_out) ++{ ++ ktime_t now = ktime_get(); ++ u64 busy = 0; ++ u64 total = 0; ++ ++ _mali_osk_spinlock_irq_lock(mdev->mali_metrics.lock); ++ ++ mali_pm_get_dvfs_utilisation_calc(mdev, now); ++ ++ busy = mdev->mali_metrics.time_busy; ++ total = busy + mdev->mali_metrics.time_idle; ++ ++ /* Reset stats if older than MALI_UTILIZATION_MAX_PERIOD (default ++ * 100ms) */ ++ if (total >= MALI_UTILIZATION_MAX_PERIOD) { ++ mali_pm_reset_dvfs_utilisation_unlocked(mdev, now); ++ } else if (total < (MALI_UTILIZATION_MAX_PERIOD / 2)) { ++ total += mdev->mali_metrics.prev_idle + ++ mdev->mali_metrics.prev_busy; ++ busy += mdev->mali_metrics.prev_busy; ++ } ++ ++ *total_out = (unsigned long)total; ++ *busy_out = (unsigned long)busy; ++ _mali_osk_spinlock_irq_unlock(mdev->mali_metrics.lock); ++} ++ ++void mali_pm_metrics_spin_lock(void) ++{ ++ struct mali_device *mdev = dev_get_drvdata(&mali_platform_device->dev); ++ _mali_osk_spinlock_irq_lock(mdev->mali_metrics.lock); ++} ++ ++void mali_pm_metrics_spin_unlock(void) ++{ ++ struct mali_device *mdev = dev_get_drvdata(&mali_platform_device->dev); ++ _mali_osk_spinlock_irq_unlock(mdev->mali_metrics.lock); ++} +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_pm_metrics.h b/drivers/gpu/arm/mali400/mali/common/mali_pm_metrics.h +new file mode 100755 +index 000000000000..2b136b0de4e3 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_pm_metrics.h +@@ -0,0 +1,74 @@ ++/* ++ * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_PM_METRICS_H__ ++#define __MALI_PM_METRICS_H__ ++ ++#ifdef CONFIG_MALI_DEVFREQ ++#include "mali_osk_locks.h" ++#include "mali_group.h" ++ ++struct mali_device; ++ ++/** ++ * Metrics data collected for use by the power management framework. ++ */ ++struct mali_pm_metrics_data { ++ ktime_t time_period_start; ++ u64 time_busy; ++ u64 time_idle; ++ u64 prev_busy; ++ u64 prev_idle; ++ u32 num_running_gp_cores; ++ u32 num_running_pp_cores; ++ ktime_t time_period_start_gp; ++ u64 time_busy_gp; ++ u64 time_idle_gp; ++ ktime_t time_period_start_pp; ++ u64 time_busy_pp[MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS]; ++ u64 time_idle_pp[MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS]; ++ mali_bool gpu_active; ++ _mali_osk_spinlock_irq_t *lock; ++}; ++ ++/** ++ * Initialize/start the Mali GPU pm_metrics metrics reporting. ++ * ++ * @return _MALI_OSK_ERR_OK on success, otherwise failure. ++ */ ++_mali_osk_errcode_t mali_pm_metrics_init(struct mali_device *mdev); ++ ++/** ++ * Terminate the Mali GPU pm_metrics metrics reporting ++ */ ++void mali_pm_metrics_term(struct mali_device *mdev); ++ ++/** ++ * Should be called when a job is about to execute a GPU job ++ */ ++void mali_pm_record_gpu_active(mali_bool is_gp); ++ ++/** ++ * Should be called when a job is finished ++ */ ++void mali_pm_record_gpu_idle(mali_bool is_gp); ++ ++void mali_pm_reset_dvfs_utilisation(struct mali_device *mdev); ++ ++void mali_pm_get_dvfs_utilisation(struct mali_device *mdev, unsigned long *total_out, unsigned long *busy_out); ++ ++void mali_pm_metrics_spin_lock(void); ++ ++void mali_pm_metrics_spin_unlock(void); ++#else ++void mali_pm_record_gpu_idle(mali_bool is_gp) {} ++void mali_pm_record_gpu_active(mali_bool is_gp) {} ++#endif ++#endif /* __MALI_PM_METRICS_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_pmu.c b/drivers/gpu/arm/mali400/mali/common/mali_pmu.c +new file mode 100755 +index 000000000000..6f0af59f6fd4 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_pmu.c +@@ -0,0 +1,270 @@ ++/* ++ * Copyright (C) 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file mali_pmu.c ++ * Mali driver functions for Mali 400 PMU hardware ++ */ ++#include "mali_hw_core.h" ++#include "mali_pmu.h" ++#include "mali_pp.h" ++#include "mali_kernel_common.h" ++#include "mali_osk.h" ++#include "mali_pm.h" ++#include "mali_osk_mali.h" ++ ++struct mali_pmu_core *mali_global_pmu_core = NULL; ++ ++static _mali_osk_errcode_t mali_pmu_wait_for_command_finish( ++ struct mali_pmu_core *pmu); ++ ++struct mali_pmu_core *mali_pmu_create(_mali_osk_resource_t *resource) ++{ ++ struct mali_pmu_core *pmu; ++ ++ MALI_DEBUG_ASSERT(NULL == mali_global_pmu_core); ++ MALI_DEBUG_PRINT(2, ("Mali PMU: Creating Mali PMU core\n")); ++ ++ pmu = (struct mali_pmu_core *)_mali_osk_malloc( ++ sizeof(struct mali_pmu_core)); ++ if (NULL != pmu) { ++ pmu->registered_cores_mask = 0; /* to be set later */ ++ ++ if (_MALI_OSK_ERR_OK == mali_hw_core_create(&pmu->hw_core, ++ resource, PMU_REGISTER_ADDRESS_SPACE_SIZE)) { ++ ++ pmu->switch_delay = _mali_osk_get_pmu_switch_delay(); ++ ++ mali_global_pmu_core = pmu; ++ ++ return pmu; ++ } ++ _mali_osk_free(pmu); ++ } ++ ++ return NULL; ++} ++ ++void mali_pmu_delete(struct mali_pmu_core *pmu) ++{ ++ MALI_DEBUG_ASSERT_POINTER(pmu); ++ MALI_DEBUG_ASSERT(pmu == mali_global_pmu_core); ++ ++ MALI_DEBUG_PRINT(2, ("Mali PMU: Deleting Mali PMU core\n")); ++ ++ mali_global_pmu_core = NULL; ++ ++ mali_hw_core_delete(&pmu->hw_core); ++ _mali_osk_free(pmu); ++} ++ ++void mali_pmu_set_registered_cores_mask(struct mali_pmu_core *pmu, u32 mask) ++{ ++ pmu->registered_cores_mask = mask; ++} ++ ++void mali_pmu_reset(struct mali_pmu_core *pmu) ++{ ++ MALI_DEBUG_ASSERT_POINTER(pmu); ++ MALI_DEBUG_ASSERT(pmu->registered_cores_mask != 0); ++ ++ /* Setup the desired defaults */ ++ mali_hw_core_register_write_relaxed(&pmu->hw_core, ++ PMU_REG_ADDR_MGMT_INT_MASK, 0); ++ mali_hw_core_register_write_relaxed(&pmu->hw_core, ++ PMU_REG_ADDR_MGMT_SW_DELAY, pmu->switch_delay); ++} ++ ++void mali_pmu_power_up_all(struct mali_pmu_core *pmu) ++{ ++ u32 stat; ++ ++ MALI_DEBUG_ASSERT_POINTER(pmu); ++ MALI_DEBUG_ASSERT(pmu->registered_cores_mask != 0); ++ ++ mali_pm_exec_lock(); ++ ++ mali_pmu_reset(pmu); ++ ++ /* Now simply power up the domains which are marked as powered down */ ++ stat = mali_hw_core_register_read(&pmu->hw_core, ++ PMU_REG_ADDR_MGMT_STATUS); ++ mali_pmu_power_up(pmu, stat); ++ ++ mali_pm_exec_unlock(); ++} ++ ++void mali_pmu_power_down_all(struct mali_pmu_core *pmu) ++{ ++ u32 stat; ++ ++ MALI_DEBUG_ASSERT_POINTER(pmu); ++ MALI_DEBUG_ASSERT(pmu->registered_cores_mask != 0); ++ ++ mali_pm_exec_lock(); ++ ++ /* Now simply power down the domains which are marked as powered up */ ++ stat = mali_hw_core_register_read(&pmu->hw_core, ++ PMU_REG_ADDR_MGMT_STATUS); ++ mali_pmu_power_down(pmu, (~stat) & pmu->registered_cores_mask); ++ ++ mali_pm_exec_unlock(); ++} ++ ++_mali_osk_errcode_t mali_pmu_power_down(struct mali_pmu_core *pmu, u32 mask) ++{ ++ u32 stat; ++ _mali_osk_errcode_t err; ++ ++ MALI_DEBUG_ASSERT_POINTER(pmu); ++ MALI_DEBUG_ASSERT(pmu->registered_cores_mask != 0); ++ MALI_DEBUG_ASSERT(mask <= pmu->registered_cores_mask); ++ MALI_DEBUG_ASSERT(0 == (mali_hw_core_register_read(&pmu->hw_core, ++ PMU_REG_ADDR_MGMT_INT_RAWSTAT) & ++ PMU_REG_VAL_IRQ)); ++ ++ MALI_DEBUG_PRINT(3, ++ ("PMU power down: ...................... [%s]\n", ++ mali_pm_mask_to_string(mask))); ++ ++ stat = mali_hw_core_register_read(&pmu->hw_core, ++ PMU_REG_ADDR_MGMT_STATUS); ++ ++ /* ++ * Assert that we are not powering down domains which are already ++ * powered down. ++ */ ++ MALI_DEBUG_ASSERT(0 == (stat & mask)); ++ ++ mask &= ~(0x1 << MALI_DOMAIN_INDEX_DUMMY); ++ ++ if (0 == mask || 0 == ((~stat) & mask)) return _MALI_OSK_ERR_OK; ++ ++ mali_hw_core_register_write(&pmu->hw_core, ++ PMU_REG_ADDR_MGMT_POWER_DOWN, mask); ++ ++ /* ++ * Do not wait for interrupt on Mali-300/400 if all domains are ++ * powered off by our power down command, because the HW will simply ++ * not generate an interrupt in this case. ++ */ ++ if (mali_is_mali450() || mali_is_mali470() || pmu->registered_cores_mask != (mask | stat)) { ++ err = mali_pmu_wait_for_command_finish(pmu); ++ if (_MALI_OSK_ERR_OK != err) { ++ return err; ++ } ++ } else { ++ mali_hw_core_register_write(&pmu->hw_core, ++ PMU_REG_ADDR_MGMT_INT_CLEAR, PMU_REG_VAL_IRQ); ++ } ++ ++#if defined(DEBUG) ++ /* Verify power status of domains after power down */ ++ stat = mali_hw_core_register_read(&pmu->hw_core, ++ PMU_REG_ADDR_MGMT_STATUS); ++ MALI_DEBUG_ASSERT(mask == (stat & mask)); ++#endif ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++_mali_osk_errcode_t mali_pmu_power_up(struct mali_pmu_core *pmu, u32 mask) ++{ ++ u32 stat; ++ _mali_osk_errcode_t err; ++#if !defined(CONFIG_MALI_PMU_PARALLEL_POWER_UP) ++ u32 current_domain; ++#endif ++ ++ MALI_DEBUG_ASSERT_POINTER(pmu); ++ MALI_DEBUG_ASSERT(pmu->registered_cores_mask != 0); ++ MALI_DEBUG_ASSERT(mask <= pmu->registered_cores_mask); ++ MALI_DEBUG_ASSERT(0 == (mali_hw_core_register_read(&pmu->hw_core, ++ PMU_REG_ADDR_MGMT_INT_RAWSTAT) & ++ PMU_REG_VAL_IRQ)); ++ ++ MALI_DEBUG_PRINT(3, ++ ("PMU power up: ........................ [%s]\n", ++ mali_pm_mask_to_string(mask))); ++ ++ stat = mali_hw_core_register_read(&pmu->hw_core, ++ PMU_REG_ADDR_MGMT_STATUS); ++ stat &= pmu->registered_cores_mask; ++ ++ mask &= ~(0x1 << MALI_DOMAIN_INDEX_DUMMY); ++ if (0 == mask || 0 == (stat & mask)) return _MALI_OSK_ERR_OK; ++ ++ /* ++ * Assert that we are only powering up domains which are currently ++ * powered down. ++ */ ++ MALI_DEBUG_ASSERT(mask == (stat & mask)); ++ ++#if defined(CONFIG_MALI_PMU_PARALLEL_POWER_UP) ++ mali_hw_core_register_write(&pmu->hw_core, ++ PMU_REG_ADDR_MGMT_POWER_UP, mask); ++ ++ err = mali_pmu_wait_for_command_finish(pmu); ++ if (_MALI_OSK_ERR_OK != err) { ++ return err; ++ } ++#else ++ for (current_domain = 1; ++ current_domain <= pmu->registered_cores_mask; ++ current_domain <<= 1) { ++ if (current_domain & mask & stat) { ++ mali_hw_core_register_write(&pmu->hw_core, ++ PMU_REG_ADDR_MGMT_POWER_UP, ++ current_domain); ++ ++ err = mali_pmu_wait_for_command_finish(pmu); ++ if (_MALI_OSK_ERR_OK != err) { ++ return err; ++ } ++ } ++ } ++#endif ++ ++#if defined(DEBUG) ++ /* Verify power status of domains after power up */ ++ stat = mali_hw_core_register_read(&pmu->hw_core, ++ PMU_REG_ADDR_MGMT_STATUS); ++ MALI_DEBUG_ASSERT(0 == (stat & mask)); ++#endif /* defined(DEBUG) */ ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++static _mali_osk_errcode_t mali_pmu_wait_for_command_finish( ++ struct mali_pmu_core *pmu) ++{ ++ u32 rawstat; ++ u32 timeout = MALI_REG_POLL_COUNT_SLOW; ++ ++ MALI_DEBUG_ASSERT(pmu); ++ ++ /* Wait for the command to complete */ ++ do { ++ rawstat = mali_hw_core_register_read(&pmu->hw_core, ++ PMU_REG_ADDR_MGMT_INT_RAWSTAT); ++ --timeout; ++ } while (0 == (rawstat & PMU_REG_VAL_IRQ) && 0 < timeout); ++ ++ MALI_DEBUG_ASSERT(0 < timeout); ++ ++ if (0 == timeout) { ++ return _MALI_OSK_ERR_TIMEOUT; ++ } ++ ++ mali_hw_core_register_write(&pmu->hw_core, ++ PMU_REG_ADDR_MGMT_INT_CLEAR, PMU_REG_VAL_IRQ); ++ ++ return _MALI_OSK_ERR_OK; ++} +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_pmu.h b/drivers/gpu/arm/mali400/mali/common/mali_pmu.h +new file mode 100755 +index 000000000000..5b856240fdac +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_pmu.h +@@ -0,0 +1,123 @@ ++/* ++ * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file mali_platform.h ++ * Platform specific Mali driver functions ++ */ ++ ++#ifndef __MALI_PMU_H__ ++#define __MALI_PMU_H__ ++ ++#include "mali_osk.h" ++#include "mali_kernel_common.h" ++#include "mali_hw_core.h" ++ ++/** @brief MALI inbuilt PMU hardware info and PMU hardware has knowledge of cores power mask ++ */ ++struct mali_pmu_core { ++ struct mali_hw_core hw_core; ++ u32 registered_cores_mask; ++ u32 switch_delay; ++}; ++ ++/** @brief Register layout for hardware PMU ++ */ ++typedef enum { ++ PMU_REG_ADDR_MGMT_POWER_UP = 0x00, /*< Power up register */ ++ PMU_REG_ADDR_MGMT_POWER_DOWN = 0x04, /*< Power down register */ ++ PMU_REG_ADDR_MGMT_STATUS = 0x08, /*< Core sleep status register */ ++ PMU_REG_ADDR_MGMT_INT_MASK = 0x0C, /*< Interrupt mask register */ ++ PMU_REG_ADDR_MGMT_INT_RAWSTAT = 0x10, /*< Interrupt raw status register */ ++ PMU_REG_ADDR_MGMT_INT_CLEAR = 0x18, /*< Interrupt clear register */ ++ PMU_REG_ADDR_MGMT_SW_DELAY = 0x1C, /*< Switch delay register */ ++ PMU_REGISTER_ADDRESS_SPACE_SIZE = 0x28, /*< Size of register space */ ++} pmu_reg_addr_mgmt_addr; ++ ++#define PMU_REG_VAL_IRQ 1 ++ ++extern struct mali_pmu_core *mali_global_pmu_core; ++ ++/** @brief Initialisation of MALI PMU ++ * ++ * This is called from entry point of the driver in order to create and intialize the PMU resource ++ * ++ * @param resource it will be a pointer to a PMU resource ++ * @param number_of_pp_cores Number of found PP resources in configuration ++ * @param number_of_l2_caches Number of found L2 cache resources in configuration ++ * @return The created PMU object, or NULL in case of failure. ++ */ ++struct mali_pmu_core *mali_pmu_create(_mali_osk_resource_t *resource); ++ ++/** @brief It deallocates the PMU resource ++ * ++ * This is called on the exit of the driver to terminate the PMU resource ++ * ++ * @param pmu Pointer to PMU core object to delete ++ */ ++void mali_pmu_delete(struct mali_pmu_core *pmu); ++ ++/** @brief Set registered cores mask ++ * ++ * @param pmu Pointer to PMU core object ++ * @param mask All available/valid domain bits ++ */ ++void mali_pmu_set_registered_cores_mask(struct mali_pmu_core *pmu, u32 mask); ++ ++/** @brief Retrieves the Mali PMU core object (if any) ++ * ++ * @return The Mali PMU object, or NULL if no PMU exists. ++ */ ++MALI_STATIC_INLINE struct mali_pmu_core *mali_pmu_get_global_pmu_core(void) ++{ ++ return mali_global_pmu_core; ++} ++ ++/** @brief Reset PMU core ++ * ++ * @param pmu Pointer to PMU core object to reset ++ */ ++void mali_pmu_reset(struct mali_pmu_core *pmu); ++ ++void mali_pmu_power_up_all(struct mali_pmu_core *pmu); ++ ++void mali_pmu_power_down_all(struct mali_pmu_core *pmu); ++ ++/** @brief Returns a mask of the currently powered up domains ++ * ++ * @param pmu Pointer to PMU core object ++ */ ++MALI_STATIC_INLINE u32 mali_pmu_get_mask(struct mali_pmu_core *pmu) ++{ ++ u32 stat = mali_hw_core_register_read(&pmu->hw_core, PMU_REG_ADDR_MGMT_STATUS); ++ return ((~stat) & pmu->registered_cores_mask); ++} ++ ++/** @brief MALI GPU power down using MALI in-built PMU ++ * ++ * Called to power down the specified cores. ++ * ++ * @param pmu Pointer to PMU core object to power down ++ * @param mask Mask specifying which power domains to power down ++ * @return _MALI_OSK_ERR_OK on success otherwise, a suitable _mali_osk_errcode_t error. ++ */ ++_mali_osk_errcode_t mali_pmu_power_down(struct mali_pmu_core *pmu, u32 mask); ++ ++/** @brief MALI GPU power up using MALI in-built PMU ++ * ++ * Called to power up the specified cores. ++ * ++ * @param pmu Pointer to PMU core object to power up ++ * @param mask Mask specifying which power domains to power up ++ * @return _MALI_OSK_ERR_OK on success otherwise, a suitable _mali_osk_errcode_t error. ++ */ ++_mali_osk_errcode_t mali_pmu_power_up(struct mali_pmu_core *pmu, u32 mask); ++ ++#endif /* __MALI_PMU_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_pp.c b/drivers/gpu/arm/mali400/mali/common/mali_pp.c +new file mode 100755 +index 000000000000..2dd8b8766f8e +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_pp.c +@@ -0,0 +1,502 @@ ++/* ++ * Copyright (C) 2011-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include "mali_pp_job.h" ++#include "mali_pp.h" ++#include "mali_hw_core.h" ++#include "mali_group.h" ++#include "regs/mali_200_regs.h" ++#include "mali_kernel_common.h" ++#include "mali_kernel_core.h" ++ ++#if defined(CONFIG_MALI400_PROFILING) ++#include "mali_osk_profiling.h" ++#endif ++ ++/* Number of frame registers on Mali-200 */ ++#define MALI_PP_MALI200_NUM_FRAME_REGISTERS ((0x04C/4)+1) ++/* Number of frame registers on Mali-300 and later */ ++#define MALI_PP_MALI400_NUM_FRAME_REGISTERS ((0x058/4)+1) ++ ++static struct mali_pp_core *mali_global_pp_cores[MALI_MAX_NUMBER_OF_PP_CORES] = { NULL }; ++static u32 mali_global_num_pp_cores = 0; ++ ++/* Interrupt handlers */ ++static void mali_pp_irq_probe_trigger(void *data); ++static _mali_osk_errcode_t mali_pp_irq_probe_ack(void *data); ++ ++struct mali_pp_core *mali_pp_create(const _mali_osk_resource_t *resource, struct mali_group *group, mali_bool is_virtual, u32 bcast_id) ++{ ++ struct mali_pp_core *core = NULL; ++ ++ MALI_DEBUG_PRINT(2, ("Mali PP: Creating Mali PP core: %s\n", resource->description)); ++ MALI_DEBUG_PRINT(2, ("Mali PP: Base address of PP core: 0x%x\n", resource->base)); ++ ++ if (mali_global_num_pp_cores >= MALI_MAX_NUMBER_OF_PP_CORES) { ++ MALI_PRINT_ERROR(("Mali PP: Too many PP core objects created\n")); ++ return NULL; ++ } ++ ++ core = _mali_osk_calloc(1, sizeof(struct mali_pp_core)); ++ if (NULL != core) { ++ core->core_id = mali_global_num_pp_cores; ++ core->bcast_id = bcast_id; ++ ++ if (_MALI_OSK_ERR_OK == mali_hw_core_create(&core->hw_core, resource, MALI200_REG_SIZEOF_REGISTER_BANK)) { ++ _mali_osk_errcode_t ret; ++ ++ if (!is_virtual) { ++ ret = mali_pp_reset(core); ++ } else { ++ ret = _MALI_OSK_ERR_OK; ++ } ++ ++ if (_MALI_OSK_ERR_OK == ret) { ++ ret = mali_group_add_pp_core(group, core); ++ if (_MALI_OSK_ERR_OK == ret) { ++ /* Setup IRQ handlers (which will do IRQ probing if needed) */ ++ MALI_DEBUG_ASSERT(!is_virtual || -1 != resource->irq); ++ ++ core->irq = _mali_osk_irq_init(resource->irq, ++ mali_group_upper_half_pp, ++ group, ++ mali_pp_irq_probe_trigger, ++ mali_pp_irq_probe_ack, ++ core, ++ resource->description); ++ if (NULL != core->irq) { ++ mali_global_pp_cores[mali_global_num_pp_cores] = core; ++ mali_global_num_pp_cores++; ++ ++ return core; ++ } else { ++ MALI_PRINT_ERROR(("Mali PP: Failed to setup interrupt handlers for PP core %s\n", core->hw_core.description)); ++ } ++ mali_group_remove_pp_core(group); ++ } else { ++ MALI_PRINT_ERROR(("Mali PP: Failed to add core %s to group\n", core->hw_core.description)); ++ } ++ } ++ mali_hw_core_delete(&core->hw_core); ++ } ++ ++ _mali_osk_free(core); ++ } else { ++ MALI_PRINT_ERROR(("Mali PP: Failed to allocate memory for PP core\n")); ++ } ++ ++ return NULL; ++} ++ ++void mali_pp_delete(struct mali_pp_core *core) ++{ ++ u32 i; ++ ++ MALI_DEBUG_ASSERT_POINTER(core); ++ ++ _mali_osk_irq_term(core->irq); ++ mali_hw_core_delete(&core->hw_core); ++ ++ /* Remove core from global list */ ++ for (i = 0; i < mali_global_num_pp_cores; i++) { ++ if (mali_global_pp_cores[i] == core) { ++ mali_global_pp_cores[i] = NULL; ++ mali_global_num_pp_cores--; ++ ++ if (i != mali_global_num_pp_cores) { ++ /* We removed a PP core from the middle of the array -- move the last ++ * PP core to the current position to close the gap */ ++ mali_global_pp_cores[i] = mali_global_pp_cores[mali_global_num_pp_cores]; ++ mali_global_pp_cores[mali_global_num_pp_cores] = NULL; ++ } ++ ++ break; ++ } ++ } ++ ++ _mali_osk_free(core); ++} ++ ++void mali_pp_stop_bus(struct mali_pp_core *core) ++{ ++ MALI_DEBUG_ASSERT_POINTER(core); ++ /* Will only send the stop bus command, and not wait for it to complete */ ++ mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_CTRL_MGMT, MALI200_REG_VAL_CTRL_MGMT_STOP_BUS); ++} ++ ++_mali_osk_errcode_t mali_pp_stop_bus_wait(struct mali_pp_core *core) ++{ ++ int i; ++ ++ MALI_DEBUG_ASSERT_POINTER(core); ++ ++ /* Send the stop bus command. */ ++ mali_pp_stop_bus(core); ++ ++ /* Wait for bus to be stopped */ ++ for (i = 0; i < MALI_REG_POLL_COUNT_FAST; i++) { ++ if (mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_STATUS) & MALI200_REG_VAL_STATUS_BUS_STOPPED) ++ break; ++ } ++ ++ if (MALI_REG_POLL_COUNT_FAST == i) { ++ MALI_PRINT_ERROR(("Mali PP: Failed to stop bus on %s. Status: 0x%08x\n", core->hw_core.description, mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_STATUS))); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ return _MALI_OSK_ERR_OK; ++} ++ ++/* Frame register reset values. ++ * Taken from the Mali400 TRM, 3.6. Pixel processor control register summary */ ++static const u32 mali_frame_registers_reset_values[_MALI_PP_MAX_FRAME_REGISTERS] = { ++ 0x0, /* Renderer List Address Register */ ++ 0x0, /* Renderer State Word Base Address Register */ ++ 0x0, /* Renderer Vertex Base Register */ ++ 0x2, /* Feature Enable Register */ ++ 0x0, /* Z Clear Value Register */ ++ 0x0, /* Stencil Clear Value Register */ ++ 0x0, /* ABGR Clear Value 0 Register */ ++ 0x0, /* ABGR Clear Value 1 Register */ ++ 0x0, /* ABGR Clear Value 2 Register */ ++ 0x0, /* ABGR Clear Value 3 Register */ ++ 0x0, /* Bounding Box Left Right Register */ ++ 0x0, /* Bounding Box Bottom Register */ ++ 0x0, /* FS Stack Address Register */ ++ 0x0, /* FS Stack Size and Initial Value Register */ ++ 0x0, /* Reserved */ ++ 0x0, /* Reserved */ ++ 0x0, /* Origin Offset X Register */ ++ 0x0, /* Origin Offset Y Register */ ++ 0x75, /* Subpixel Specifier Register */ ++ 0x0, /* Tiebreak mode Register */ ++ 0x0, /* Polygon List Format Register */ ++ 0x0, /* Scaling Register */ ++ 0x0 /* Tilebuffer configuration Register */ ++}; ++ ++/* WBx register reset values */ ++static const u32 mali_wb_registers_reset_values[_MALI_PP_MAX_WB_REGISTERS] = { ++ 0x0, /* WBx Source Select Register */ ++ 0x0, /* WBx Target Address Register */ ++ 0x0, /* WBx Target Pixel Format Register */ ++ 0x0, /* WBx Target AA Format Register */ ++ 0x0, /* WBx Target Layout */ ++ 0x0, /* WBx Target Scanline Length */ ++ 0x0, /* WBx Target Flags Register */ ++ 0x0, /* WBx MRT Enable Register */ ++ 0x0, /* WBx MRT Offset Register */ ++ 0x0, /* WBx Global Test Enable Register */ ++ 0x0, /* WBx Global Test Reference Value Register */ ++ 0x0 /* WBx Global Test Compare Function Register */ ++}; ++ ++/* Performance Counter 0 Enable Register reset value */ ++static const u32 mali_perf_cnt_enable_reset_value = 0; ++ ++_mali_osk_errcode_t mali_pp_hard_reset(struct mali_pp_core *core) ++{ ++ /* Bus must be stopped before calling this function */ ++ const u32 reset_wait_target_register = MALI200_REG_ADDR_MGMT_PERF_CNT_0_LIMIT; ++ const u32 reset_invalid_value = 0xC0FFE000; ++ const u32 reset_check_value = 0xC01A0000; ++ int i; ++ ++ MALI_DEBUG_ASSERT_POINTER(core); ++ MALI_DEBUG_PRINT(2, ("Mali PP: Hard reset of core %s\n", core->hw_core.description)); ++ ++ /* Set register to a bogus value. The register will be used to detect when reset is complete */ ++ mali_hw_core_register_write_relaxed(&core->hw_core, reset_wait_target_register, reset_invalid_value); ++ mali_hw_core_register_write_relaxed(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_MASK, MALI200_REG_VAL_IRQ_MASK_NONE); ++ ++ /* Force core to reset */ ++ mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_CTRL_MGMT, MALI200_REG_VAL_CTRL_MGMT_FORCE_RESET); ++ ++ /* Wait for reset to be complete */ ++ for (i = 0; i < MALI_REG_POLL_COUNT_FAST; i++) { ++ mali_hw_core_register_write(&core->hw_core, reset_wait_target_register, reset_check_value); ++ if (reset_check_value == mali_hw_core_register_read(&core->hw_core, reset_wait_target_register)) { ++ break; ++ } ++ } ++ ++ if (MALI_REG_POLL_COUNT_FAST == i) { ++ MALI_PRINT_ERROR(("Mali PP: The hard reset loop didn't work, unable to recover\n")); ++ } ++ ++ mali_hw_core_register_write(&core->hw_core, reset_wait_target_register, 0x00000000); /* set it back to the default */ ++ /* Re-enable interrupts */ ++ mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_CLEAR, MALI200_REG_VAL_IRQ_MASK_ALL); ++ mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_MASK, MALI200_REG_VAL_IRQ_MASK_USED); ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++void mali_pp_reset_async(struct mali_pp_core *core) ++{ ++ MALI_DEBUG_ASSERT_POINTER(core); ++ ++ MALI_DEBUG_PRINT(4, ("Mali PP: Reset of core %s\n", core->hw_core.description)); ++ ++ mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_MASK, 0); /* disable the IRQs */ ++ mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_RAWSTAT, MALI200_REG_VAL_IRQ_MASK_ALL); ++ mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_CTRL_MGMT, MALI400PP_REG_VAL_CTRL_MGMT_SOFT_RESET); ++} ++ ++_mali_osk_errcode_t mali_pp_reset_wait(struct mali_pp_core *core) ++{ ++ int i; ++ u32 rawstat = 0; ++ ++ for (i = 0; i < MALI_REG_POLL_COUNT_FAST; i++) { ++ u32 status = mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_STATUS); ++ if (!(status & MALI200_REG_VAL_STATUS_RENDERING_ACTIVE)) { ++ rawstat = mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_RAWSTAT); ++ if (rawstat == MALI400PP_REG_VAL_IRQ_RESET_COMPLETED) { ++ break; ++ } ++ } ++ } ++ ++ if (i == MALI_REG_POLL_COUNT_FAST) { ++ MALI_PRINT_ERROR(("Mali PP: Failed to reset core %s, rawstat: 0x%08x\n", ++ core->hw_core.description, rawstat)); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ /* Re-enable interrupts */ ++ mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_CLEAR, MALI200_REG_VAL_IRQ_MASK_ALL); ++ mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_MASK, MALI200_REG_VAL_IRQ_MASK_USED); ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++_mali_osk_errcode_t mali_pp_reset(struct mali_pp_core *core) ++{ ++ mali_pp_reset_async(core); ++ return mali_pp_reset_wait(core); ++} ++ ++void mali_pp_job_start(struct mali_pp_core *core, struct mali_pp_job *job, u32 sub_job, mali_bool restart_virtual) ++{ ++ u32 relative_address; ++ u32 start_index; ++ u32 nr_of_regs; ++ u32 *frame_registers = mali_pp_job_get_frame_registers(job); ++ u32 *wb0_registers = mali_pp_job_get_wb0_registers(job); ++ u32 *wb1_registers = mali_pp_job_get_wb1_registers(job); ++ u32 *wb2_registers = mali_pp_job_get_wb2_registers(job); ++ u32 counter_src0 = mali_pp_job_get_perf_counter_src0(job, sub_job); ++ u32 counter_src1 = mali_pp_job_get_perf_counter_src1(job, sub_job); ++ ++ MALI_DEBUG_ASSERT_POINTER(core); ++ ++ /* Write frame registers */ ++ ++ /* ++ * There are two frame registers which are different for each sub job: ++ * 1. The Renderer List Address Register (MALI200_REG_ADDR_FRAME) ++ * 2. The FS Stack Address Register (MALI200_REG_ADDR_STACK) ++ */ ++ mali_hw_core_register_write_relaxed_conditional(&core->hw_core, MALI200_REG_ADDR_FRAME, mali_pp_job_get_addr_frame(job, sub_job), mali_frame_registers_reset_values[MALI200_REG_ADDR_FRAME / sizeof(u32)]); ++ ++ /* For virtual jobs, the stack address shouldn't be broadcast but written individually */ ++ if (!mali_pp_job_is_virtual(job) || restart_virtual) { ++ mali_hw_core_register_write_relaxed_conditional(&core->hw_core, MALI200_REG_ADDR_STACK, mali_pp_job_get_addr_stack(job, sub_job), mali_frame_registers_reset_values[MALI200_REG_ADDR_STACK / sizeof(u32)]); ++ } ++ ++ /* Write registers between MALI200_REG_ADDR_FRAME and MALI200_REG_ADDR_STACK */ ++ relative_address = MALI200_REG_ADDR_RSW; ++ start_index = MALI200_REG_ADDR_RSW / sizeof(u32); ++ nr_of_regs = (MALI200_REG_ADDR_STACK - MALI200_REG_ADDR_RSW) / sizeof(u32); ++ ++ mali_hw_core_register_write_array_relaxed_conditional(&core->hw_core, ++ relative_address, &frame_registers[start_index], ++ nr_of_regs, &mali_frame_registers_reset_values[start_index]); ++ ++ /* MALI200_REG_ADDR_STACK_SIZE */ ++ relative_address = MALI200_REG_ADDR_STACK_SIZE; ++ start_index = MALI200_REG_ADDR_STACK_SIZE / sizeof(u32); ++ ++ mali_hw_core_register_write_relaxed_conditional(&core->hw_core, ++ relative_address, frame_registers[start_index], ++ mali_frame_registers_reset_values[start_index]); ++ ++ /* Skip 2 reserved registers */ ++ ++ /* Write remaining registers */ ++ relative_address = MALI200_REG_ADDR_ORIGIN_OFFSET_X; ++ start_index = MALI200_REG_ADDR_ORIGIN_OFFSET_X / sizeof(u32); ++ nr_of_regs = MALI_PP_MALI400_NUM_FRAME_REGISTERS - MALI200_REG_ADDR_ORIGIN_OFFSET_X / sizeof(u32); ++ ++ mali_hw_core_register_write_array_relaxed_conditional(&core->hw_core, ++ relative_address, &frame_registers[start_index], ++ nr_of_regs, &mali_frame_registers_reset_values[start_index]); ++ ++ /* Write WBx registers */ ++ if (wb0_registers[0]) { /* M200_WB0_REG_SOURCE_SELECT register */ ++ mali_hw_core_register_write_array_relaxed_conditional(&core->hw_core, MALI200_REG_ADDR_WB0, wb0_registers, _MALI_PP_MAX_WB_REGISTERS, mali_wb_registers_reset_values); ++ } ++ ++ if (wb1_registers[0]) { /* M200_WB1_REG_SOURCE_SELECT register */ ++ mali_hw_core_register_write_array_relaxed_conditional(&core->hw_core, MALI200_REG_ADDR_WB1, wb1_registers, _MALI_PP_MAX_WB_REGISTERS, mali_wb_registers_reset_values); ++ } ++ ++ if (wb2_registers[0]) { /* M200_WB2_REG_SOURCE_SELECT register */ ++ mali_hw_core_register_write_array_relaxed_conditional(&core->hw_core, MALI200_REG_ADDR_WB2, wb2_registers, _MALI_PP_MAX_WB_REGISTERS, mali_wb_registers_reset_values); ++ } ++ ++ if (MALI_HW_CORE_NO_COUNTER != counter_src0) { ++ mali_hw_core_register_write_relaxed(&core->hw_core, MALI200_REG_ADDR_MGMT_PERF_CNT_0_SRC, counter_src0); ++ mali_hw_core_register_write_relaxed_conditional(&core->hw_core, MALI200_REG_ADDR_MGMT_PERF_CNT_0_ENABLE, MALI200_REG_VAL_PERF_CNT_ENABLE, mali_perf_cnt_enable_reset_value); ++ } ++ if (MALI_HW_CORE_NO_COUNTER != counter_src1) { ++ mali_hw_core_register_write_relaxed(&core->hw_core, MALI200_REG_ADDR_MGMT_PERF_CNT_1_SRC, counter_src1); ++ mali_hw_core_register_write_relaxed_conditional(&core->hw_core, MALI200_REG_ADDR_MGMT_PERF_CNT_1_ENABLE, MALI200_REG_VAL_PERF_CNT_ENABLE, mali_perf_cnt_enable_reset_value); ++ } ++ ++#ifdef CONFIG_MALI400_HEATMAPS_ENABLED ++ if (job->uargs.perf_counter_flag & _MALI_PERFORMANCE_COUNTER_FLAG_HEATMAP_ENABLE) { ++ mali_hw_core_register_write_relaxed(&core->hw_core, MALI200_REG_ADDR_MGMT_PERFMON_CONTR, ((job->uargs.tilesx & 0x3FF) << 16) | 1); ++ mali_hw_core_register_write_relaxed(&core->hw_core, MALI200_REG_ADDR_MGMT_PERFMON_BASE, job->uargs.heatmap_mem & 0xFFFFFFF8); ++ } ++#endif /* CONFIG_MALI400_HEATMAPS_ENABLED */ ++ ++ MALI_DEBUG_PRINT(3, ("Mali PP: Starting job 0x%08X part %u/%u on PP core %s\n", job, sub_job + 1, mali_pp_job_get_sub_job_count(job), core->hw_core.description)); ++ ++ /* Adding barrier to make sure all rester writes are finished */ ++ _mali_osk_write_mem_barrier(); ++ ++ /* This is the command that starts the core. ++ * ++ * Don't actually run the job if PROFILING_SKIP_PP_JOBS are set, just ++ * force core to assert the completion interrupt. ++ */ ++#if !defined(PROFILING_SKIP_PP_JOBS) ++ mali_hw_core_register_write_relaxed(&core->hw_core, MALI200_REG_ADDR_MGMT_CTRL_MGMT, MALI200_REG_VAL_CTRL_MGMT_START_RENDERING); ++#else ++ mali_hw_core_register_write_relaxed(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_RAWSTAT, MALI200_REG_VAL_IRQ_END_OF_FRAME); ++#endif ++ ++ /* Adding barrier to make sure previous rester writes is finished */ ++ _mali_osk_write_mem_barrier(); ++} ++ ++u32 mali_pp_core_get_version(struct mali_pp_core *core) ++{ ++ MALI_DEBUG_ASSERT_POINTER(core); ++ return mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_VERSION); ++} ++ ++struct mali_pp_core *mali_pp_get_global_pp_core(u32 index) ++{ ++ if (mali_global_num_pp_cores > index) { ++ return mali_global_pp_cores[index]; ++ } ++ ++ return NULL; ++} ++ ++u32 mali_pp_get_glob_num_pp_cores(void) ++{ ++ return mali_global_num_pp_cores; ++} ++ ++/* ------------- interrupt handling below ------------------ */ ++static void mali_pp_irq_probe_trigger(void *data) ++{ ++ struct mali_pp_core *core = (struct mali_pp_core *)data; ++ mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_MASK, MALI200_REG_VAL_IRQ_MASK_USED); ++ mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_RAWSTAT, MALI200_REG_VAL_IRQ_BUS_ERROR); ++ _mali_osk_mem_barrier(); ++} ++ ++static _mali_osk_errcode_t mali_pp_irq_probe_ack(void *data) ++{ ++ struct mali_pp_core *core = (struct mali_pp_core *)data; ++ u32 irq_readout; ++ ++ irq_readout = mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_STATUS); ++ if (MALI200_REG_VAL_IRQ_BUS_ERROR & irq_readout) { ++ mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_CLEAR, MALI200_REG_VAL_IRQ_BUS_ERROR); ++ _mali_osk_mem_barrier(); ++ return _MALI_OSK_ERR_OK; ++ } ++ ++ return _MALI_OSK_ERR_FAULT; ++} ++ ++ ++#if 0 ++static void mali_pp_print_registers(struct mali_pp_core *core) ++{ ++ MALI_DEBUG_PRINT(2, ("Mali PP: Register MALI200_REG_ADDR_MGMT_VERSION = 0x%08X\n", mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_VERSION))); ++ MALI_DEBUG_PRINT(2, ("Mali PP: Register MALI200_REG_ADDR_MGMT_CURRENT_REND_LIST_ADDR = 0x%08X\n", mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_CURRENT_REND_LIST_ADDR))); ++ MALI_DEBUG_PRINT(2, ("Mali PP: Register MALI200_REG_ADDR_MGMT_STATUS = 0x%08X\n", mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_STATUS))); ++ MALI_DEBUG_PRINT(2, ("Mali PP: Register MALI200_REG_ADDR_MGMT_INT_RAWSTAT = 0x%08X\n", mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_RAWSTAT))); ++ MALI_DEBUG_PRINT(2, ("Mali PP: Register MALI200_REG_ADDR_MGMT_INT_MASK = 0x%08X\n", mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_MASK))); ++ MALI_DEBUG_PRINT(2, ("Mali PP: Register MALI200_REG_ADDR_MGMT_INT_STATUS = 0x%08X\n", mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_STATUS))); ++ MALI_DEBUG_PRINT(2, ("Mali PP: Register MALI200_REG_ADDR_MGMT_BUS_ERROR_STATUS = 0x%08X\n", mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_BUS_ERROR_STATUS))); ++ MALI_DEBUG_PRINT(2, ("Mali PP: Register MALI200_REG_ADDR_MGMT_PERF_CNT_0_ENABLE = 0x%08X\n", mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_PERF_CNT_0_ENABLE))); ++ MALI_DEBUG_PRINT(2, ("Mali PP: Register MALI200_REG_ADDR_MGMT_PERF_CNT_0_SRC = 0x%08X\n", mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_PERF_CNT_0_SRC))); ++ MALI_DEBUG_PRINT(2, ("Mali PP: Register MALI200_REG_ADDR_MGMT_PERF_CNT_0_VALUE = 0x%08X\n", mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_PERF_CNT_0_VALUE))); ++ MALI_DEBUG_PRINT(2, ("Mali PP: Register MALI200_REG_ADDR_MGMT_PERF_CNT_1_ENABLE = 0x%08X\n", mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_PERF_CNT_1_ENABLE))); ++ MALI_DEBUG_PRINT(2, ("Mali PP: Register MALI200_REG_ADDR_MGMT_PERF_CNT_1_SRC = 0x%08X\n", mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_PERF_CNT_1_SRC))); ++ MALI_DEBUG_PRINT(2, ("Mali PP: Register MALI200_REG_ADDR_MGMT_PERF_CNT_1_VALUE = 0x%08X\n", mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_PERF_CNT_1_VALUE))); ++} ++#endif ++ ++#if 0 ++void mali_pp_print_state(struct mali_pp_core *core) ++{ ++ MALI_DEBUG_PRINT(2, ("Mali PP: State: 0x%08x\n", mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_STATUS))); ++} ++#endif ++ ++void mali_pp_update_performance_counters(struct mali_pp_core *parent, struct mali_pp_core *child, struct mali_pp_job *job, u32 subjob) ++{ ++ u32 val0 = 0; ++ u32 val1 = 0; ++ u32 counter_src0 = mali_pp_job_get_perf_counter_src0(job, subjob); ++ u32 counter_src1 = mali_pp_job_get_perf_counter_src1(job, subjob); ++#if defined(CONFIG_MALI400_PROFILING) ++ int counter_index = COUNTER_FP_0_C0 + (2 * child->core_id); ++#endif ++ ++ if (MALI_HW_CORE_NO_COUNTER != counter_src0) { ++ val0 = mali_hw_core_register_read(&child->hw_core, MALI200_REG_ADDR_MGMT_PERF_CNT_0_VALUE); ++ mali_pp_job_set_perf_counter_value0(job, subjob, val0); ++ ++#if defined(CONFIG_MALI400_PROFILING) ++ _mali_osk_profiling_report_hw_counter(counter_index, val0); ++ _mali_osk_profiling_record_global_counters(counter_index, val0); ++#endif ++ } ++ ++ if (MALI_HW_CORE_NO_COUNTER != counter_src1) { ++ val1 = mali_hw_core_register_read(&child->hw_core, MALI200_REG_ADDR_MGMT_PERF_CNT_1_VALUE); ++ mali_pp_job_set_perf_counter_value1(job, subjob, val1); ++ ++#if defined(CONFIG_MALI400_PROFILING) ++ _mali_osk_profiling_report_hw_counter(counter_index + 1, val1); ++ _mali_osk_profiling_record_global_counters(counter_index + 1, val1); ++#endif ++ } ++} ++ ++#if MALI_STATE_TRACKING ++u32 mali_pp_dump_state(struct mali_pp_core *core, char *buf, u32 size) ++{ ++ int n = 0; ++ ++ n += _mali_osk_snprintf(buf + n, size - n, "\tPP #%d: %s\n", core->core_id, core->hw_core.description); ++ ++ return n; ++} ++#endif +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_pp.h b/drivers/gpu/arm/mali400/mali/common/mali_pp.h +new file mode 100755 +index 000000000000..f98b29866ffa +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_pp.h +@@ -0,0 +1,138 @@ ++/* ++ * Copyright (C) 2011-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_PP_H__ ++#define __MALI_PP_H__ ++ ++#include "mali_osk.h" ++#include "mali_pp_job.h" ++#include "mali_hw_core.h" ++ ++struct mali_group; ++ ++#define MALI_MAX_NUMBER_OF_PP_CORES 9 ++ ++/** ++ * Definition of the PP core struct ++ * Used to track a PP core in the system. ++ */ ++struct mali_pp_core { ++ struct mali_hw_core hw_core; /**< Common for all HW cores */ ++ _mali_osk_irq_t *irq; /**< IRQ handler */ ++ u32 core_id; /**< Unique core ID */ ++ u32 bcast_id; /**< The "flag" value used by the Mali-450 broadcast and DLBU unit */ ++}; ++ ++_mali_osk_errcode_t mali_pp_initialize(void); ++void mali_pp_terminate(void); ++ ++struct mali_pp_core *mali_pp_create(const _mali_osk_resource_t *resource, struct mali_group *group, mali_bool is_virtual, u32 bcast_id); ++void mali_pp_delete(struct mali_pp_core *core); ++ ++void mali_pp_stop_bus(struct mali_pp_core *core); ++_mali_osk_errcode_t mali_pp_stop_bus_wait(struct mali_pp_core *core); ++void mali_pp_reset_async(struct mali_pp_core *core); ++_mali_osk_errcode_t mali_pp_reset_wait(struct mali_pp_core *core); ++_mali_osk_errcode_t mali_pp_reset(struct mali_pp_core *core); ++_mali_osk_errcode_t mali_pp_hard_reset(struct mali_pp_core *core); ++ ++void mali_pp_job_start(struct mali_pp_core *core, struct mali_pp_job *job, u32 sub_job, mali_bool restart_virtual); ++ ++u32 mali_pp_core_get_version(struct mali_pp_core *core); ++ ++MALI_STATIC_INLINE u32 mali_pp_core_get_id(struct mali_pp_core *core) ++{ ++ MALI_DEBUG_ASSERT_POINTER(core); ++ return core->core_id; ++} ++ ++MALI_STATIC_INLINE u32 mali_pp_core_get_bcast_id(struct mali_pp_core *core) ++{ ++ MALI_DEBUG_ASSERT_POINTER(core); ++ return core->bcast_id; ++} ++ ++struct mali_pp_core *mali_pp_get_global_pp_core(u32 index); ++u32 mali_pp_get_glob_num_pp_cores(void); ++ ++/* Debug */ ++u32 mali_pp_dump_state(struct mali_pp_core *core, char *buf, u32 size); ++ ++/** ++ * Put instrumented HW counters from the core(s) to the job object (if enabled) ++ * ++ * parent and child is always the same, except for virtual jobs on Mali-450. ++ * In this case, the counters will be enabled on the virtual core (parent), ++ * but values need to be read from the child cores. ++ * ++ * @param parent The core used to see if the counters was enabled ++ * @param child The core to actually read the values from ++ * @job Job object to update with counter values (if enabled) ++ * @subjob Which subjob the counters are applicable for (core ID for virtual jobs) ++ */ ++void mali_pp_update_performance_counters(struct mali_pp_core *parent, struct mali_pp_core *child, struct mali_pp_job *job, u32 subjob); ++ ++MALI_STATIC_INLINE const char *mali_pp_core_description(struct mali_pp_core *core) ++{ ++ return core->hw_core.description; ++} ++ ++MALI_STATIC_INLINE enum mali_interrupt_result mali_pp_get_interrupt_result(struct mali_pp_core *core) ++{ ++ u32 rawstat_used = mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_RAWSTAT) & ++ MALI200_REG_VAL_IRQ_MASK_USED; ++ if (0 == rawstat_used) { ++ return MALI_INTERRUPT_RESULT_NONE; ++ } else if (MALI200_REG_VAL_IRQ_END_OF_FRAME == rawstat_used) { ++ return MALI_INTERRUPT_RESULT_SUCCESS; ++ } ++ ++ return MALI_INTERRUPT_RESULT_ERROR; ++} ++ ++MALI_STATIC_INLINE u32 mali_pp_get_rawstat(struct mali_pp_core *core) ++{ ++ MALI_DEBUG_ASSERT_POINTER(core); ++ return mali_hw_core_register_read(&core->hw_core, ++ MALI200_REG_ADDR_MGMT_INT_RAWSTAT); ++} ++ ++ ++MALI_STATIC_INLINE u32 mali_pp_is_active(struct mali_pp_core *core) ++{ ++ u32 status = mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_STATUS); ++ return (status & MALI200_REG_VAL_STATUS_RENDERING_ACTIVE) ? MALI_TRUE : MALI_FALSE; ++} ++ ++MALI_STATIC_INLINE void mali_pp_mask_all_interrupts(struct mali_pp_core *core) ++{ ++ mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_MASK, MALI200_REG_VAL_IRQ_MASK_NONE); ++} ++ ++MALI_STATIC_INLINE void mali_pp_enable_interrupts(struct mali_pp_core *core) ++{ ++ mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_MASK, MALI200_REG_VAL_IRQ_MASK_USED); ++} ++ ++MALI_STATIC_INLINE void mali_pp_write_addr_renderer_list(struct mali_pp_core *core, ++ struct mali_pp_job *job, u32 subjob) ++{ ++ u32 addr = mali_pp_job_get_addr_frame(job, subjob); ++ mali_hw_core_register_write_relaxed(&core->hw_core, MALI200_REG_ADDR_FRAME, addr); ++} ++ ++ ++MALI_STATIC_INLINE void mali_pp_write_addr_stack(struct mali_pp_core *core, struct mali_pp_job *job) ++{ ++ u32 addr = mali_pp_job_get_addr_stack(job, core->core_id); ++ mali_hw_core_register_write_relaxed(&core->hw_core, MALI200_REG_ADDR_STACK, addr); ++} ++ ++#endif /* __MALI_PP_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_pp_job.c b/drivers/gpu/arm/mali400/mali/common/mali_pp_job.c +new file mode 100755 +index 000000000000..b0216d4c1ac8 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_pp_job.c +@@ -0,0 +1,316 @@ ++/* ++ * Copyright (C) 2011-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include "mali_pp.h" ++#include "mali_pp_job.h" ++#include "mali_osk.h" ++#include "mali_osk_list.h" ++#include "mali_kernel_common.h" ++#include "mali_uk_types.h" ++#include "mali_executor.h" ++#if defined(CONFIG_DMA_SHARED_BUFFER) && !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) ++#include "linux/mali_memory_dma_buf.h" ++#endif ++#include "mali_memory_swap_alloc.h" ++#include "mali_scheduler.h" ++ ++static u32 pp_counter_src0 = MALI_HW_CORE_NO_COUNTER; /**< Performance counter 0, MALI_HW_CORE_NO_COUNTER for disabled */ ++static u32 pp_counter_src1 = MALI_HW_CORE_NO_COUNTER; /**< Performance counter 1, MALI_HW_CORE_NO_COUNTER for disabled */ ++static _mali_osk_atomic_t pp_counter_per_sub_job_count; /**< Number of values in the two arrays which is != MALI_HW_CORE_NO_COUNTER */ ++static u32 pp_counter_per_sub_job_src0[_MALI_PP_MAX_SUB_JOBS] = { MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER }; ++static u32 pp_counter_per_sub_job_src1[_MALI_PP_MAX_SUB_JOBS] = { MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER }; ++ ++void mali_pp_job_initialize(void) ++{ ++ _mali_osk_atomic_init(&pp_counter_per_sub_job_count, 0); ++} ++ ++void mali_pp_job_terminate(void) ++{ ++ _mali_osk_atomic_term(&pp_counter_per_sub_job_count); ++} ++ ++struct mali_pp_job *mali_pp_job_create(struct mali_session_data *session, ++ _mali_uk_pp_start_job_s __user *uargs, u32 id) ++{ ++ struct mali_pp_job *job; ++ u32 perf_counter_flag; ++ ++ job = _mali_osk_calloc(1, sizeof(struct mali_pp_job)); ++ if (NULL != job) { ++ ++ _mali_osk_list_init(&job->list); ++ _mali_osk_list_init(&job->session_fb_lookup_list); ++ _mali_osk_atomic_inc(&session->number_of_pp_jobs); ++ ++ if (0 != _mali_osk_copy_from_user(&job->uargs, uargs, sizeof(_mali_uk_pp_start_job_s))) { ++ goto fail; ++ } ++ ++ if (job->uargs.num_cores > _MALI_PP_MAX_SUB_JOBS) { ++ MALI_PRINT_ERROR(("Mali PP job: Too many sub jobs specified in job object\n")); ++ goto fail; ++ } ++ ++ if (!mali_pp_job_use_no_notification(job)) { ++ job->finished_notification = _mali_osk_notification_create(_MALI_NOTIFICATION_PP_FINISHED, sizeof(_mali_uk_pp_job_finished_s)); ++ if (NULL == job->finished_notification) goto fail; ++ } ++ ++ perf_counter_flag = mali_pp_job_get_perf_counter_flag(job); ++ ++ /* case when no counters came from user space ++ * so pass the debugfs / DS-5 provided global ones to the job object */ ++ if (!((perf_counter_flag & _MALI_PERFORMANCE_COUNTER_FLAG_SRC0_ENABLE) || ++ (perf_counter_flag & _MALI_PERFORMANCE_COUNTER_FLAG_SRC1_ENABLE))) { ++ u32 sub_job_count = _mali_osk_atomic_read(&pp_counter_per_sub_job_count); ++ ++ /* These counters apply for all virtual jobs, and where no per sub job counter is specified */ ++ job->uargs.perf_counter_src0 = pp_counter_src0; ++ job->uargs.perf_counter_src1 = pp_counter_src1; ++ ++ /* We only copy the per sub job array if it is enabled with at least one counter */ ++ if (0 < sub_job_count) { ++ job->perf_counter_per_sub_job_count = sub_job_count; ++ _mali_osk_memcpy(job->perf_counter_per_sub_job_src0, pp_counter_per_sub_job_src0, sizeof(pp_counter_per_sub_job_src0)); ++ _mali_osk_memcpy(job->perf_counter_per_sub_job_src1, pp_counter_per_sub_job_src1, sizeof(pp_counter_per_sub_job_src1)); ++ } ++ } ++ ++ job->session = session; ++ job->id = id; ++ ++ job->sub_jobs_num = job->uargs.num_cores ? job->uargs.num_cores : 1; ++ job->pid = _mali_osk_get_pid(); ++ job->tid = _mali_osk_get_tid(); ++ ++ _mali_osk_atomic_init(&job->sub_jobs_completed, 0); ++ _mali_osk_atomic_init(&job->sub_job_errors, 0); ++ job->swap_status = MALI_NO_SWAP_IN; ++ job->user_notification = MALI_FALSE; ++ job->num_pp_cores_in_virtual = 0; ++ ++ if (job->uargs.num_memory_cookies > session->allocation_mgr.mali_allocation_num) { ++ MALI_PRINT_ERROR(("Mali PP job: The number of memory cookies is invalid !\n")); ++ goto fail; ++ } ++ ++ if (job->uargs.num_memory_cookies > 0) { ++ u32 size; ++ u32 __user *memory_cookies = (u32 __user *)(uintptr_t)job->uargs.memory_cookies; ++ ++ size = sizeof(*memory_cookies) * (job->uargs.num_memory_cookies); ++ ++ job->memory_cookies = _mali_osk_malloc(size); ++ if (NULL == job->memory_cookies) { ++ MALI_PRINT_ERROR(("Mali PP job: Failed to allocate %d bytes of memory cookies!\n", size)); ++ goto fail; ++ } ++ ++ if (0 != _mali_osk_copy_from_user(job->memory_cookies, memory_cookies, size)) { ++ MALI_PRINT_ERROR(("Mali PP job: Failed to copy %d bytes of memory cookies from user!\n", size)); ++ goto fail; ++ } ++ } ++ ++ if (_MALI_OSK_ERR_OK != mali_pp_job_check(job)) { ++ /* Not a valid job. */ ++ goto fail; ++ } ++ ++ mali_timeline_tracker_init(&job->tracker, MALI_TIMELINE_TRACKER_PP, NULL, job); ++ mali_timeline_fence_copy_uk_fence(&(job->tracker.fence), &(job->uargs.fence)); ++ ++ mali_mem_swap_in_pages(job); ++ ++ return job; ++ } ++ ++fail: ++ if (NULL != job) { ++ mali_pp_job_delete(job); ++ } ++ ++ return NULL; ++} ++ ++void mali_pp_job_delete(struct mali_pp_job *job) ++{ ++ struct mali_session_data *session; ++ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ MALI_DEBUG_ASSERT(_mali_osk_list_empty(&job->list)); ++ MALI_DEBUG_ASSERT(_mali_osk_list_empty(&job->session_fb_lookup_list)); ++ ++ session = mali_pp_job_get_session(job); ++ MALI_DEBUG_ASSERT_POINTER(session); ++ ++ if (NULL != job->memory_cookies) { ++#if defined(CONFIG_DMA_SHARED_BUFFER) && !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) ++ /* Unmap buffers attached to job */ ++ mali_dma_buf_unmap_job(job); ++#endif ++ if (MALI_NO_SWAP_IN != job->swap_status) { ++ mali_mem_swap_out_pages(job); ++ } ++ ++ _mali_osk_free(job->memory_cookies); ++ } ++ ++ if (job->user_notification) { ++ mali_scheduler_return_pp_job_to_user(job, ++ job->num_pp_cores_in_virtual); ++ } ++ ++ if (NULL != job->finished_notification) { ++ _mali_osk_notification_delete(job->finished_notification); ++ } ++ ++ _mali_osk_atomic_term(&job->sub_jobs_completed); ++ _mali_osk_atomic_term(&job->sub_job_errors); ++ _mali_osk_atomic_dec(&session->number_of_pp_jobs); ++ _mali_osk_free(job); ++ ++ _mali_osk_wait_queue_wake_up(session->wait_queue); ++} ++ ++void mali_pp_job_list_add(struct mali_pp_job *job, _mali_osk_list_t *list) ++{ ++ struct mali_pp_job *iter; ++ struct mali_pp_job *tmp; ++ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); ++ ++ /* Find position in list/queue where job should be added. */ ++ _MALI_OSK_LIST_FOREACHENTRY_REVERSE(iter, tmp, list, ++ struct mali_pp_job, list) { ++ /* job should be started after iter if iter is in progress. */ ++ if (0 < iter->sub_jobs_started) { ++ break; ++ } ++ ++ /* ++ * job should be started after iter if it has a higher ++ * job id. A span is used to handle job id wrapping. ++ */ ++ if ((mali_pp_job_get_id(job) - ++ mali_pp_job_get_id(iter)) < ++ MALI_SCHEDULER_JOB_ID_SPAN) { ++ break; ++ } ++ } ++ ++ _mali_osk_list_add(&job->list, &iter->list); ++} ++ ++ ++u32 mali_pp_job_get_perf_counter_src0(struct mali_pp_job *job, u32 sub_job) ++{ ++ /* Virtual jobs always use the global job counter (or if there are per sub job counters at all) */ ++ if (mali_pp_job_is_virtual(job) || 0 == job->perf_counter_per_sub_job_count) { ++ return job->uargs.perf_counter_src0; ++ } ++ ++ /* Use per sub job counter if enabled... */ ++ if (MALI_HW_CORE_NO_COUNTER != job->perf_counter_per_sub_job_src0[sub_job]) { ++ return job->perf_counter_per_sub_job_src0[sub_job]; ++ } ++ ++ /* ...else default to global job counter */ ++ return job->uargs.perf_counter_src0; ++} ++ ++u32 mali_pp_job_get_perf_counter_src1(struct mali_pp_job *job, u32 sub_job) ++{ ++ /* Virtual jobs always use the global job counter (or if there are per sub job counters at all) */ ++ if (mali_pp_job_is_virtual(job) || 0 == job->perf_counter_per_sub_job_count) { ++ /* Virtual jobs always use the global job counter */ ++ return job->uargs.perf_counter_src1; ++ } ++ ++ /* Use per sub job counter if enabled... */ ++ if (MALI_HW_CORE_NO_COUNTER != job->perf_counter_per_sub_job_src1[sub_job]) { ++ return job->perf_counter_per_sub_job_src1[sub_job]; ++ } ++ ++ /* ...else default to global job counter */ ++ return job->uargs.perf_counter_src1; ++} ++ ++void mali_pp_job_set_pp_counter_global_src0(u32 counter) ++{ ++ pp_counter_src0 = counter; ++} ++ ++void mali_pp_job_set_pp_counter_global_src1(u32 counter) ++{ ++ pp_counter_src1 = counter; ++} ++ ++void mali_pp_job_set_pp_counter_sub_job_src0(u32 sub_job, u32 counter) ++{ ++ MALI_DEBUG_ASSERT(sub_job < _MALI_PP_MAX_SUB_JOBS); ++ ++ if (MALI_HW_CORE_NO_COUNTER == pp_counter_per_sub_job_src0[sub_job]) { ++ /* increment count since existing counter was disabled */ ++ _mali_osk_atomic_inc(&pp_counter_per_sub_job_count); ++ } ++ ++ if (MALI_HW_CORE_NO_COUNTER == counter) { ++ /* decrement count since new counter is disabled */ ++ _mali_osk_atomic_dec(&pp_counter_per_sub_job_count); ++ } ++ ++ /* PS: A change from MALI_HW_CORE_NO_COUNTER to MALI_HW_CORE_NO_COUNTER will inc and dec, result will be 0 change */ ++ ++ pp_counter_per_sub_job_src0[sub_job] = counter; ++} ++ ++void mali_pp_job_set_pp_counter_sub_job_src1(u32 sub_job, u32 counter) ++{ ++ MALI_DEBUG_ASSERT(sub_job < _MALI_PP_MAX_SUB_JOBS); ++ ++ if (MALI_HW_CORE_NO_COUNTER == pp_counter_per_sub_job_src1[sub_job]) { ++ /* increment count since existing counter was disabled */ ++ _mali_osk_atomic_inc(&pp_counter_per_sub_job_count); ++ } ++ ++ if (MALI_HW_CORE_NO_COUNTER == counter) { ++ /* decrement count since new counter is disabled */ ++ _mali_osk_atomic_dec(&pp_counter_per_sub_job_count); ++ } ++ ++ /* PS: A change from MALI_HW_CORE_NO_COUNTER to MALI_HW_CORE_NO_COUNTER will inc and dec, result will be 0 change */ ++ ++ pp_counter_per_sub_job_src1[sub_job] = counter; ++} ++ ++u32 mali_pp_job_get_pp_counter_global_src0(void) ++{ ++ return pp_counter_src0; ++} ++ ++u32 mali_pp_job_get_pp_counter_global_src1(void) ++{ ++ return pp_counter_src1; ++} ++ ++u32 mali_pp_job_get_pp_counter_sub_job_src0(u32 sub_job) ++{ ++ MALI_DEBUG_ASSERT(sub_job < _MALI_PP_MAX_SUB_JOBS); ++ return pp_counter_per_sub_job_src0[sub_job]; ++} ++ ++u32 mali_pp_job_get_pp_counter_sub_job_src1(u32 sub_job) ++{ ++ MALI_DEBUG_ASSERT(sub_job < _MALI_PP_MAX_SUB_JOBS); ++ return pp_counter_per_sub_job_src1[sub_job]; ++} +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_pp_job.h b/drivers/gpu/arm/mali400/mali/common/mali_pp_job.h +new file mode 100755 +index 000000000000..d0331f398ff9 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_pp_job.h +@@ -0,0 +1,594 @@ ++/* ++ * Copyright (C) 2011-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_PP_JOB_H__ ++#define __MALI_PP_JOB_H__ ++ ++#include "mali_osk.h" ++#include "mali_osk_list.h" ++#include "mali_uk_types.h" ++#include "mali_session.h" ++#include "mali_kernel_common.h" ++#include "regs/mali_200_regs.h" ++#include "mali_kernel_core.h" ++#include "mali_dlbu.h" ++#include "mali_timeline.h" ++#include "mali_scheduler.h" ++#include "mali_executor.h" ++#if defined(CONFIG_DMA_SHARED_BUFFER) && !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) ++#include "linux/mali_memory_dma_buf.h" ++#endif ++#if defined(CONFIG_MALI_DMA_BUF_FENCE) ++#include "linux/mali_dma_fence.h" ++#endif ++ ++typedef enum pp_job_status { ++ MALI_NO_SWAP_IN, ++ MALI_SWAP_IN_FAIL, ++ MALI_SWAP_IN_SUCC, ++} pp_job_status; ++ ++/** ++ * This structure represents a PP job, including all sub jobs. ++ * ++ * The PP job object itself is not protected by any single lock, ++ * but relies on other locks instead (scheduler, executor and timeline lock). ++ * Think of the job object as moving between these sub systems through-out ++ * its lifetime. Different part of the PP job struct is used by different ++ * subsystems. Accessor functions ensure that correct lock is taken. ++ * Do NOT access any data members directly from outside this module! ++ */ ++struct mali_pp_job { ++ /* ++ * These members are typically only set at creation, ++ * and only read later on. ++ * They do not require any lock protection. ++ */ ++ _mali_uk_pp_start_job_s uargs; /**< Arguments from user space */ ++ struct mali_session_data *session; /**< Session which submitted this job */ ++ u32 pid; /**< Process ID of submitting process */ ++ u32 tid; /**< Thread ID of submitting thread */ ++ u32 id; /**< Identifier for this job in kernel space (sequential numbering) */ ++ u32 cache_order; /**< Cache order used for L2 cache flushing (sequential numbering) */ ++ struct mali_timeline_tracker tracker; /**< Timeline tracker for this job */ ++ _mali_osk_notification_t *finished_notification; /**< Notification sent back to userspace on job complete */ ++ u32 perf_counter_per_sub_job_count; /**< Number of values in the two arrays which is != MALI_HW_CORE_NO_COUNTER */ ++ u32 perf_counter_per_sub_job_src0[_MALI_PP_MAX_SUB_JOBS]; /**< Per sub job counters src0 */ ++ u32 perf_counter_per_sub_job_src1[_MALI_PP_MAX_SUB_JOBS]; /**< Per sub job counters src1 */ ++ u32 sub_jobs_num; /**< Number of subjobs; set to 1 for Mali-450 if DLBU is used, otherwise equals number of PP cores */ ++ ++ pp_job_status swap_status; /**< Used to track each PP job swap status, if fail, we need to drop them in scheduler part */ ++ mali_bool user_notification; /**< When we deferred delete PP job, we need to judge if we need to send job finish notification to user space */ ++ u32 num_pp_cores_in_virtual; /**< How many PP cores we have when job finished */ ++ ++ /* ++ * These members are used by both scheduler and executor. ++ * They are "protected" by atomic operations. ++ */ ++ _mali_osk_atomic_t sub_jobs_completed; /**< Number of completed sub-jobs in this superjob */ ++ _mali_osk_atomic_t sub_job_errors; /**< Bitfield with errors (errors for each single sub-job is or'ed together) */ ++ ++ /* ++ * These members are used by scheduler, but only when no one else ++ * knows about this job object but the working function. ++ * No lock is thus needed for these. ++ */ ++ u32 *memory_cookies; /**< Memory cookies attached to job */ ++ ++ /* ++ * These members are used by the scheduler, ++ * protected by scheduler lock ++ */ ++ _mali_osk_list_t list; /**< Used to link jobs together in the scheduler queue */ ++ _mali_osk_list_t session_fb_lookup_list; /**< Used to link jobs together from the same frame builder in the session */ ++ ++ u32 sub_jobs_started; /**< Total number of sub-jobs started (always started in ascending order) */ ++ ++ /* ++ * Set by executor/group on job completion, read by scheduler when ++ * returning job to user. Hold executor lock when setting, ++ * no lock needed when reading ++ */ ++ u32 perf_counter_value0[_MALI_PP_MAX_SUB_JOBS]; /**< Value of performance counter 0 (to be returned to user space), one for each sub job */ ++ u32 perf_counter_value1[_MALI_PP_MAX_SUB_JOBS]; /**< Value of performance counter 1 (to be returned to user space), one for each sub job */ ++ ++#if defined(CONFIG_MALI_DMA_BUF_FENCE) ++ struct mali_dma_fence_context dma_fence_context; /**< The mali dma fence context to record dma fence waiters that this job wait for */ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++ struct dma_fence *rendered_dma_fence; /**< the new dma fence link to this job */ ++#else ++ struct fence *rendered_dma_fence; /**< the new dma fence link to this job */ ++#endif ++#endif ++}; ++ ++void mali_pp_job_initialize(void); ++void mali_pp_job_terminate(void); ++ ++struct mali_pp_job *mali_pp_job_create(struct mali_session_data *session, _mali_uk_pp_start_job_s *uargs, u32 id); ++void mali_pp_job_delete(struct mali_pp_job *job); ++ ++u32 mali_pp_job_get_perf_counter_src0(struct mali_pp_job *job, u32 sub_job); ++u32 mali_pp_job_get_perf_counter_src1(struct mali_pp_job *job, u32 sub_job); ++ ++void mali_pp_job_set_pp_counter_global_src0(u32 counter); ++void mali_pp_job_set_pp_counter_global_src1(u32 counter); ++void mali_pp_job_set_pp_counter_sub_job_src0(u32 sub_job, u32 counter); ++void mali_pp_job_set_pp_counter_sub_job_src1(u32 sub_job, u32 counter); ++ ++u32 mali_pp_job_get_pp_counter_global_src0(void); ++u32 mali_pp_job_get_pp_counter_global_src1(void); ++u32 mali_pp_job_get_pp_counter_sub_job_src0(u32 sub_job); ++u32 mali_pp_job_get_pp_counter_sub_job_src1(u32 sub_job); ++ ++MALI_STATIC_INLINE u32 mali_pp_job_get_id(struct mali_pp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return (NULL == job) ? 0 : job->id; ++} ++ ++MALI_STATIC_INLINE void mali_pp_job_set_cache_order(struct mali_pp_job *job, ++ u32 cache_order) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); ++ job->cache_order = cache_order; ++} ++ ++MALI_STATIC_INLINE u32 mali_pp_job_get_cache_order(struct mali_pp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return (NULL == job) ? 0 : job->cache_order; ++} ++ ++MALI_STATIC_INLINE u64 mali_pp_job_get_user_id(struct mali_pp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return job->uargs.user_job_ptr; ++} ++ ++MALI_STATIC_INLINE u32 mali_pp_job_get_frame_builder_id(struct mali_pp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return job->uargs.frame_builder_id; ++} ++ ++MALI_STATIC_INLINE u32 mali_pp_job_get_flush_id(struct mali_pp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return job->uargs.flush_id; ++} ++ ++MALI_STATIC_INLINE u32 mali_pp_job_get_pid(struct mali_pp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return job->pid; ++} ++ ++MALI_STATIC_INLINE u32 mali_pp_job_get_tid(struct mali_pp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return job->tid; ++} ++ ++MALI_STATIC_INLINE u32 *mali_pp_job_get_frame_registers(struct mali_pp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return job->uargs.frame_registers; ++} ++ ++MALI_STATIC_INLINE u32 *mali_pp_job_get_dlbu_registers(struct mali_pp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return job->uargs.dlbu_registers; ++} ++ ++MALI_STATIC_INLINE mali_bool mali_pp_job_is_virtual(struct mali_pp_job *job) ++{ ++#if (defined(CONFIG_MALI450) || defined(CONFIG_MALI470)) ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return (0 == job->uargs.num_cores) ? MALI_TRUE : MALI_FALSE; ++#else ++ return MALI_FALSE; ++#endif ++} ++ ++MALI_STATIC_INLINE u32 mali_pp_job_get_addr_frame(struct mali_pp_job *job, u32 sub_job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ ++ if (mali_pp_job_is_virtual(job)) { ++ return MALI_DLBU_VIRT_ADDR; ++ } else if (0 == sub_job) { ++ return job->uargs.frame_registers[MALI200_REG_ADDR_FRAME / sizeof(u32)]; ++ } else if (sub_job < _MALI_PP_MAX_SUB_JOBS) { ++ return job->uargs.frame_registers_addr_frame[sub_job - 1]; ++ } ++ ++ return 0; ++} ++ ++MALI_STATIC_INLINE u32 mali_pp_job_get_addr_stack(struct mali_pp_job *job, u32 sub_job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ ++ if (0 == sub_job) { ++ return job->uargs.frame_registers[MALI200_REG_ADDR_STACK / sizeof(u32)]; ++ } else if (sub_job < _MALI_PP_MAX_SUB_JOBS) { ++ return job->uargs.frame_registers_addr_stack[sub_job - 1]; ++ } ++ ++ return 0; ++} ++ ++void mali_pp_job_list_add(struct mali_pp_job *job, _mali_osk_list_t *list); ++ ++MALI_STATIC_INLINE void mali_pp_job_list_addtail(struct mali_pp_job *job, ++ _mali_osk_list_t *list) ++{ ++ _mali_osk_list_addtail(&job->list, list); ++} ++ ++MALI_STATIC_INLINE void mali_pp_job_list_move(struct mali_pp_job *job, ++ _mali_osk_list_t *list) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); ++ MALI_DEBUG_ASSERT(!_mali_osk_list_empty(&job->list)); ++ _mali_osk_list_move(&job->list, list); ++} ++ ++MALI_STATIC_INLINE void mali_pp_job_list_remove(struct mali_pp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); ++ _mali_osk_list_delinit(&job->list); ++} ++ ++MALI_STATIC_INLINE u32 *mali_pp_job_get_wb0_registers(struct mali_pp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return job->uargs.wb0_registers; ++} ++ ++MALI_STATIC_INLINE u32 *mali_pp_job_get_wb1_registers(struct mali_pp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return job->uargs.wb1_registers; ++} ++ ++MALI_STATIC_INLINE u32 *mali_pp_job_get_wb2_registers(struct mali_pp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return job->uargs.wb2_registers; ++} ++ ++MALI_STATIC_INLINE u32 mali_pp_job_get_wb0_source_addr(struct mali_pp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return job->uargs.wb0_registers[MALI200_REG_ADDR_WB_SOURCE_ADDR / sizeof(u32)]; ++} ++ ++MALI_STATIC_INLINE u32 mali_pp_job_get_wb1_source_addr(struct mali_pp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return job->uargs.wb1_registers[MALI200_REG_ADDR_WB_SOURCE_ADDR / sizeof(u32)]; ++} ++ ++MALI_STATIC_INLINE u32 mali_pp_job_get_wb2_source_addr(struct mali_pp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return job->uargs.wb2_registers[MALI200_REG_ADDR_WB_SOURCE_ADDR / sizeof(u32)]; ++} ++ ++MALI_STATIC_INLINE void mali_pp_job_disable_wb0(struct mali_pp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ job->uargs.wb0_registers[MALI200_REG_ADDR_WB_SOURCE_SELECT] = 0; ++} ++ ++MALI_STATIC_INLINE void mali_pp_job_disable_wb1(struct mali_pp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ job->uargs.wb1_registers[MALI200_REG_ADDR_WB_SOURCE_SELECT] = 0; ++} ++ ++MALI_STATIC_INLINE void mali_pp_job_disable_wb2(struct mali_pp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ job->uargs.wb2_registers[MALI200_REG_ADDR_WB_SOURCE_SELECT] = 0; ++} ++ ++MALI_STATIC_INLINE mali_bool mali_pp_job_all_writeback_unit_disabled(struct mali_pp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ ++ if (job->uargs.wb0_registers[MALI200_REG_ADDR_WB_SOURCE_SELECT] || ++ job->uargs.wb1_registers[MALI200_REG_ADDR_WB_SOURCE_SELECT] || ++ job->uargs.wb2_registers[MALI200_REG_ADDR_WB_SOURCE_SELECT] ++ ) { ++ /* At least one output unit active */ ++ return MALI_FALSE; ++ } ++ ++ /* All outputs are disabled - we can abort the job */ ++ return MALI_TRUE; ++} ++ ++MALI_STATIC_INLINE void mali_pp_job_fb_lookup_add(struct mali_pp_job *job) ++{ ++ u32 fb_lookup_id; ++ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); ++ ++ fb_lookup_id = MALI_PP_JOB_FB_LOOKUP_LIST_MASK & job->uargs.frame_builder_id; ++ ++ MALI_DEBUG_ASSERT(MALI_PP_JOB_FB_LOOKUP_LIST_SIZE > fb_lookup_id); ++ ++ _mali_osk_list_addtail(&job->session_fb_lookup_list, ++ &job->session->pp_job_fb_lookup_list[fb_lookup_id]); ++} ++ ++MALI_STATIC_INLINE void mali_pp_job_fb_lookup_remove(struct mali_pp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); ++ _mali_osk_list_delinit(&job->session_fb_lookup_list); ++} ++ ++MALI_STATIC_INLINE struct mali_session_data *mali_pp_job_get_session(struct mali_pp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return job->session; ++} ++ ++MALI_STATIC_INLINE mali_bool mali_pp_job_has_started_sub_jobs(struct mali_pp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); ++ return (0 < job->sub_jobs_started) ? MALI_TRUE : MALI_FALSE; ++} ++ ++MALI_STATIC_INLINE mali_bool mali_pp_job_has_unstarted_sub_jobs(struct mali_pp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); ++ return (job->sub_jobs_started < job->sub_jobs_num) ? MALI_TRUE : MALI_FALSE; ++} ++ ++/* Function used when we are terminating a session with jobs. Return TRUE if it has a rendering job. ++ Makes sure that no new subjobs are started. */ ++MALI_STATIC_INLINE void mali_pp_job_mark_unstarted_failed(struct mali_pp_job *job) ++{ ++ u32 jobs_remaining; ++ u32 i; ++ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); ++ ++ jobs_remaining = job->sub_jobs_num - job->sub_jobs_started; ++ job->sub_jobs_started += jobs_remaining; ++ ++ /* Not the most optimal way, but this is only used in error cases */ ++ for (i = 0; i < jobs_remaining; i++) { ++ _mali_osk_atomic_inc(&job->sub_jobs_completed); ++ _mali_osk_atomic_inc(&job->sub_job_errors); ++ } ++} ++ ++MALI_STATIC_INLINE mali_bool mali_pp_job_is_complete(struct mali_pp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return (job->sub_jobs_num == ++ _mali_osk_atomic_read(&job->sub_jobs_completed)) ? ++ MALI_TRUE : MALI_FALSE; ++} ++ ++MALI_STATIC_INLINE u32 mali_pp_job_get_first_unstarted_sub_job(struct mali_pp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); ++ return job->sub_jobs_started; ++} ++ ++MALI_STATIC_INLINE u32 mali_pp_job_get_sub_job_count(struct mali_pp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return job->sub_jobs_num; ++} ++ ++MALI_STATIC_INLINE u32 mali_pp_job_unstarted_sub_job_count(struct mali_pp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); ++ MALI_DEBUG_ASSERT(job->sub_jobs_num >= job->sub_jobs_started); ++ return (job->sub_jobs_num - job->sub_jobs_started); ++} ++ ++MALI_STATIC_INLINE u32 mali_pp_job_num_memory_cookies(struct mali_pp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return job->uargs.num_memory_cookies; ++} ++ ++MALI_STATIC_INLINE u32 mali_pp_job_get_memory_cookie( ++ struct mali_pp_job *job, u32 index) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ MALI_DEBUG_ASSERT(index < job->uargs.num_memory_cookies); ++ MALI_DEBUG_ASSERT_POINTER(job->memory_cookies); ++ return job->memory_cookies[index]; ++} ++ ++MALI_STATIC_INLINE mali_bool mali_pp_job_needs_dma_buf_mapping(struct mali_pp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ ++ if (0 < job->uargs.num_memory_cookies) { ++ return MALI_TRUE; ++ } ++ ++ return MALI_FALSE; ++} ++ ++MALI_STATIC_INLINE void mali_pp_job_mark_sub_job_started(struct mali_pp_job *job, u32 sub_job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); ++ ++ /* Assert that we are marking the "first unstarted sub job" as started */ ++ MALI_DEBUG_ASSERT(job->sub_jobs_started == sub_job); ++ ++ job->sub_jobs_started++; ++} ++ ++MALI_STATIC_INLINE void mali_pp_job_mark_sub_job_completed(struct mali_pp_job *job, mali_bool success) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ ++ _mali_osk_atomic_inc(&job->sub_jobs_completed); ++ if (MALI_FALSE == success) { ++ _mali_osk_atomic_inc(&job->sub_job_errors); ++ } ++} ++ ++MALI_STATIC_INLINE mali_bool mali_pp_job_was_success(struct mali_pp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ if (0 == _mali_osk_atomic_read(&job->sub_job_errors)) { ++ return MALI_TRUE; ++ } ++ return MALI_FALSE; ++} ++ ++MALI_STATIC_INLINE mali_bool mali_pp_job_use_no_notification( ++ struct mali_pp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return (job->uargs.flags & _MALI_PP_JOB_FLAG_NO_NOTIFICATION) ? ++ MALI_TRUE : MALI_FALSE; ++} ++ ++MALI_STATIC_INLINE mali_bool mali_pp_job_is_pilot_job(struct mali_pp_job *job) ++{ ++ /* ++ * A pilot job is currently identified as jobs which ++ * require no callback notification. ++ */ ++ return mali_pp_job_use_no_notification(job); ++} ++ ++MALI_STATIC_INLINE _mali_osk_notification_t * ++mali_pp_job_get_finished_notification(struct mali_pp_job *job) ++{ ++ _mali_osk_notification_t *notification; ++ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ MALI_DEBUG_ASSERT_POINTER(job->finished_notification); ++ ++ notification = job->finished_notification; ++ job->finished_notification = NULL; ++ ++ return notification; ++} ++ ++MALI_STATIC_INLINE mali_bool mali_pp_job_is_window_surface( ++ struct mali_pp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return (job->uargs.flags & _MALI_PP_JOB_FLAG_IS_WINDOW_SURFACE) ++ ? MALI_TRUE : MALI_FALSE; ++} ++ ++MALI_STATIC_INLINE mali_bool mali_pp_job_is_protected_job(struct mali_pp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return (job->uargs.flags & _MALI_PP_JOB_FLAG_PROTECTED) ++ ? MALI_TRUE : MALI_FALSE; ++} ++ ++MALI_STATIC_INLINE u32 mali_pp_job_get_perf_counter_flag(struct mali_pp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return job->uargs.perf_counter_flag; ++} ++ ++MALI_STATIC_INLINE u32 mali_pp_job_get_perf_counter_value0(struct mali_pp_job *job, u32 sub_job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return job->perf_counter_value0[sub_job]; ++} ++ ++MALI_STATIC_INLINE u32 mali_pp_job_get_perf_counter_value1(struct mali_pp_job *job, u32 sub_job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return job->perf_counter_value1[sub_job]; ++} ++ ++MALI_STATIC_INLINE void mali_pp_job_set_perf_counter_value0(struct mali_pp_job *job, u32 sub_job, u32 value) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ job->perf_counter_value0[sub_job] = value; ++} ++ ++MALI_STATIC_INLINE void mali_pp_job_set_perf_counter_value1(struct mali_pp_job *job, u32 sub_job, u32 value) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); ++ job->perf_counter_value1[sub_job] = value; ++} ++ ++MALI_STATIC_INLINE _mali_osk_errcode_t mali_pp_job_check(struct mali_pp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ if (mali_pp_job_is_virtual(job) && job->sub_jobs_num != 1) { ++ return _MALI_OSK_ERR_FAULT; ++ } ++ return _MALI_OSK_ERR_OK; ++} ++ ++/** ++ * Returns MALI_TRUE if this job has more than two sub jobs and all sub jobs are unstarted. ++ * ++ * @param job Job to check. ++ * @return MALI_TRUE if job has more than two sub jobs and all sub jobs are unstarted, MALI_FALSE if not. ++ */ ++MALI_STATIC_INLINE mali_bool mali_pp_job_is_large_and_unstarted(struct mali_pp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); ++ MALI_DEBUG_ASSERT(!mali_pp_job_is_virtual(job)); ++ ++ return (0 == job->sub_jobs_started && 2 < job->sub_jobs_num); ++} ++ ++/** ++ * Get PP job's Timeline tracker. ++ * ++ * @param job PP job. ++ * @return Pointer to Timeline tracker for the job. ++ */ ++MALI_STATIC_INLINE struct mali_timeline_tracker *mali_pp_job_get_tracker(struct mali_pp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return &(job->tracker); ++} ++ ++MALI_STATIC_INLINE u32 *mali_pp_job_get_timeline_point_ptr( ++ struct mali_pp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ return (u32 __user *)(uintptr_t)job->uargs.timeline_point_ptr; ++} ++ ++ ++#endif /* __MALI_PP_JOB_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_scheduler.c b/drivers/gpu/arm/mali400/mali/common/mali_scheduler.c +new file mode 100755 +index 000000000000..b5e6cfddbb0e +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_scheduler.c +@@ -0,0 +1,1548 @@ ++/* ++ * Copyright (C) 2012-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include "mali_scheduler.h" ++#include "mali_kernel_common.h" ++#include "mali_osk.h" ++#include "mali_osk_profiling.h" ++#include "mali_kernel_utilization.h" ++#include "mali_timeline.h" ++#include "mali_gp_job.h" ++#include "mali_pp_job.h" ++#include "mali_executor.h" ++#include "mali_group.h" ++#include ++#include ++#include "mali_pm_metrics.h" ++ ++#if defined(CONFIG_DMA_SHARED_BUFFER) ++#include "mali_memory_dma_buf.h" ++#if defined(CONFIG_MALI_DMA_BUF_FENCE) ++#include "mali_dma_fence.h" ++#include ++#endif ++#endif ++ ++#if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS) ++#include ++#include ++#endif ++/* ++ * ---------- static defines/constants ---------- ++ */ ++ ++/* ++ * If dma_buf with map on demand is used, we defer job queue ++ * if in atomic context, since both might sleep. ++ */ ++#if defined(CONFIG_DMA_SHARED_BUFFER) ++#if !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) ++#define MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE 1 ++#endif ++#endif ++ ++ ++/* ++ * ---------- global variables (exported due to inline functions) ---------- ++ */ ++ ++/* Lock protecting this module */ ++_mali_osk_spinlock_irq_t *mali_scheduler_lock_obj = NULL; ++ ++/* Queue of jobs to be executed on the GP group */ ++struct mali_scheduler_job_queue job_queue_gp; ++ ++/* Queue of PP jobs */ ++struct mali_scheduler_job_queue job_queue_pp; ++ ++_mali_osk_atomic_t mali_job_id_autonumber; ++_mali_osk_atomic_t mali_job_cache_order_autonumber; ++/* ++ * ---------- static variables ---------- ++ */ ++ ++_mali_osk_wq_work_t *scheduler_wq_pp_job_delete = NULL; ++_mali_osk_spinlock_irq_t *scheduler_pp_job_delete_lock = NULL; ++static _MALI_OSK_LIST_HEAD_STATIC_INIT(scheduler_pp_job_deletion_queue); ++ ++#if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) ++static _mali_osk_wq_work_t *scheduler_wq_pp_job_queue = NULL; ++static _mali_osk_spinlock_irq_t *scheduler_pp_job_queue_lock = NULL; ++static _MALI_OSK_LIST_HEAD_STATIC_INIT(scheduler_pp_job_queue_list); ++#endif ++ ++/* ++ * ---------- Forward declaration of static functions ---------- ++ */ ++ ++static mali_timeline_point mali_scheduler_submit_gp_job( ++ struct mali_session_data *session, struct mali_gp_job *job); ++static _mali_osk_errcode_t mali_scheduler_submit_pp_job( ++ struct mali_session_data *session, struct mali_pp_job *job, mali_timeline_point *point); ++ ++static mali_bool mali_scheduler_queue_gp_job(struct mali_gp_job *job); ++static mali_bool mali_scheduler_queue_pp_job(struct mali_pp_job *job); ++ ++static void mali_scheduler_return_gp_job_to_user(struct mali_gp_job *job, ++ mali_bool success); ++ ++static void mali_scheduler_deferred_pp_job_delete(struct mali_pp_job *job); ++void mali_scheduler_do_pp_job_delete(void *arg); ++ ++#if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) ++static void mali_scheduler_deferred_pp_job_queue(struct mali_pp_job *job); ++static void mali_scheduler_do_pp_job_queue(void *arg); ++#endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) */ ++ ++/* ++ * ---------- Actual implementation ---------- ++ */ ++ ++_mali_osk_errcode_t mali_scheduler_initialize(void) ++{ ++ _mali_osk_atomic_init(&mali_job_id_autonumber, 0); ++ _mali_osk_atomic_init(&mali_job_cache_order_autonumber, 0); ++ ++ _MALI_OSK_INIT_LIST_HEAD(&job_queue_gp.normal_pri); ++ _MALI_OSK_INIT_LIST_HEAD(&job_queue_gp.high_pri); ++ job_queue_gp.depth = 0; ++ job_queue_gp.big_job_num = 0; ++ ++ _MALI_OSK_INIT_LIST_HEAD(&job_queue_pp.normal_pri); ++ _MALI_OSK_INIT_LIST_HEAD(&job_queue_pp.high_pri); ++ job_queue_pp.depth = 0; ++ job_queue_pp.big_job_num = 0; ++ ++ mali_scheduler_lock_obj = _mali_osk_spinlock_irq_init( ++ _MALI_OSK_LOCKFLAG_ORDERED, ++ _MALI_OSK_LOCK_ORDER_SCHEDULER); ++ if (NULL == mali_scheduler_lock_obj) { ++ mali_scheduler_terminate(); ++ } ++ ++ scheduler_wq_pp_job_delete = _mali_osk_wq_create_work( ++ mali_scheduler_do_pp_job_delete, NULL); ++ if (NULL == scheduler_wq_pp_job_delete) { ++ mali_scheduler_terminate(); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ scheduler_pp_job_delete_lock = _mali_osk_spinlock_irq_init( ++ _MALI_OSK_LOCKFLAG_ORDERED, ++ _MALI_OSK_LOCK_ORDER_SCHEDULER_DEFERRED); ++ if (NULL == scheduler_pp_job_delete_lock) { ++ mali_scheduler_terminate(); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++#if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) ++ scheduler_wq_pp_job_queue = _mali_osk_wq_create_work( ++ mali_scheduler_do_pp_job_queue, NULL); ++ if (NULL == scheduler_wq_pp_job_queue) { ++ mali_scheduler_terminate(); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ scheduler_pp_job_queue_lock = _mali_osk_spinlock_irq_init( ++ _MALI_OSK_LOCKFLAG_ORDERED, ++ _MALI_OSK_LOCK_ORDER_SCHEDULER_DEFERRED); ++ if (NULL == scheduler_pp_job_queue_lock) { ++ mali_scheduler_terminate(); ++ return _MALI_OSK_ERR_FAULT; ++ } ++#endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) */ ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++void mali_scheduler_terminate(void) ++{ ++#if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) ++ if (NULL != scheduler_pp_job_queue_lock) { ++ _mali_osk_spinlock_irq_term(scheduler_pp_job_queue_lock); ++ scheduler_pp_job_queue_lock = NULL; ++ } ++ ++ if (NULL != scheduler_wq_pp_job_queue) { ++ _mali_osk_wq_delete_work(scheduler_wq_pp_job_queue); ++ scheduler_wq_pp_job_queue = NULL; ++ } ++#endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) */ ++ ++ if (NULL != scheduler_pp_job_delete_lock) { ++ _mali_osk_spinlock_irq_term(scheduler_pp_job_delete_lock); ++ scheduler_pp_job_delete_lock = NULL; ++ } ++ ++ if (NULL != scheduler_wq_pp_job_delete) { ++ _mali_osk_wq_delete_work(scheduler_wq_pp_job_delete); ++ scheduler_wq_pp_job_delete = NULL; ++ } ++ ++ if (NULL != mali_scheduler_lock_obj) { ++ _mali_osk_spinlock_irq_term(mali_scheduler_lock_obj); ++ mali_scheduler_lock_obj = NULL; ++ } ++ ++ _mali_osk_atomic_term(&mali_job_cache_order_autonumber); ++ _mali_osk_atomic_term(&mali_job_id_autonumber); ++} ++ ++u32 mali_scheduler_job_physical_head_count(mali_bool gpu_mode_is_secure) ++{ ++ /* ++ * Count how many physical sub jobs are present from the head of queue ++ * until the first virtual job is present. ++ * Early out when we have reached maximum number of PP cores (8) ++ */ ++ u32 count = 0; ++ struct mali_pp_job *job; ++ struct mali_pp_job *temp; ++ ++ /* Check for partially started normal pri jobs */ ++ if (!_mali_osk_list_empty(&job_queue_pp.normal_pri)) { ++ MALI_DEBUG_ASSERT(0 < job_queue_pp.depth); ++ ++ job = _MALI_OSK_LIST_ENTRY(job_queue_pp.normal_pri.next, ++ struct mali_pp_job, list); ++ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ ++ if (MALI_TRUE == mali_pp_job_has_started_sub_jobs(job)) { ++ /* ++ * Remember; virtual jobs can't be queued and started ++ * at the same time, so this must be a physical job ++ */ ++ if ((MALI_FALSE == gpu_mode_is_secure && MALI_FALSE == mali_pp_job_is_protected_job(job)) ++ || (MALI_TRUE == gpu_mode_is_secure && MALI_TRUE == mali_pp_job_is_protected_job(job))) { ++ ++ count += mali_pp_job_unstarted_sub_job_count(job); ++ if (MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS <= count) { ++ return MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS; ++ } ++ } ++ } ++ } ++ ++ _MALI_OSK_LIST_FOREACHENTRY(job, temp, &job_queue_pp.high_pri, ++ struct mali_pp_job, list) { ++ if ((MALI_FALSE == mali_pp_job_is_virtual(job)) ++ && ((MALI_FALSE == gpu_mode_is_secure && MALI_FALSE == mali_pp_job_is_protected_job(job)) ++ || (MALI_TRUE == gpu_mode_is_secure && MALI_TRUE == mali_pp_job_is_protected_job(job)))) { ++ ++ count += mali_pp_job_unstarted_sub_job_count(job); ++ if (MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS <= count) { ++ return MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS; ++ } ++ } else { ++ /* Came across a virtual job, so stop counting */ ++ return count; ++ } ++ } ++ ++ _MALI_OSK_LIST_FOREACHENTRY(job, temp, &job_queue_pp.normal_pri, ++ struct mali_pp_job, list) { ++ if ((MALI_FALSE == mali_pp_job_is_virtual(job)) ++ && (MALI_FALSE == mali_pp_job_has_started_sub_jobs(job)) ++ && ((MALI_FALSE == gpu_mode_is_secure && MALI_FALSE == mali_pp_job_is_protected_job(job)) ++ || (MALI_TRUE == gpu_mode_is_secure && MALI_TRUE == mali_pp_job_is_protected_job(job)))) { ++ ++ count += mali_pp_job_unstarted_sub_job_count(job); ++ if (MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS <= count) { ++ return MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS; ++ } ++ } else { ++ /* Came across a virtual job, so stop counting */ ++ return count; ++ } ++ } ++ return count; ++} ++ ++struct mali_pp_job *mali_scheduler_job_pp_next(void) ++{ ++ struct mali_pp_job *job; ++ struct mali_pp_job *temp; ++ ++ MALI_DEBUG_ASSERT_LOCK_HELD(mali_scheduler_lock_obj); ++ ++ /* Check for partially started normal pri jobs */ ++ if (!_mali_osk_list_empty(&job_queue_pp.normal_pri)) { ++ MALI_DEBUG_ASSERT(0 < job_queue_pp.depth); ++ ++ job = _MALI_OSK_LIST_ENTRY(job_queue_pp.normal_pri.next, ++ struct mali_pp_job, list); ++ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ ++ if (MALI_TRUE == mali_pp_job_has_started_sub_jobs(job)) { ++ return job; ++ } ++ } ++ ++ _MALI_OSK_LIST_FOREACHENTRY(job, temp, &job_queue_pp.high_pri, ++ struct mali_pp_job, list) { ++ return job; ++ } ++ ++ _MALI_OSK_LIST_FOREACHENTRY(job, temp, &job_queue_pp.normal_pri, ++ struct mali_pp_job, list) { ++ return job; ++ } ++ ++ return NULL; ++} ++ ++mali_bool mali_scheduler_job_next_is_virtual(void) ++{ ++ struct mali_pp_job *job; ++ ++ job = mali_scheduler_job_pp_virtual_peek(); ++ if (NULL != job) { ++ MALI_DEBUG_ASSERT(mali_pp_job_is_virtual(job)); ++ ++ return MALI_TRUE; ++ } ++ ++ return MALI_FALSE; ++} ++ ++struct mali_gp_job *mali_scheduler_job_gp_get(void) ++{ ++ _mali_osk_list_t *queue; ++ struct mali_gp_job *job = NULL; ++ ++ MALI_DEBUG_ASSERT_LOCK_HELD(mali_scheduler_lock_obj); ++ MALI_DEBUG_ASSERT(0 < job_queue_gp.depth); ++ MALI_DEBUG_ASSERT(job_queue_gp.big_job_num <= job_queue_gp.depth); ++ ++ if (!_mali_osk_list_empty(&job_queue_gp.high_pri)) { ++ queue = &job_queue_gp.high_pri; ++ } else { ++ queue = &job_queue_gp.normal_pri; ++ MALI_DEBUG_ASSERT(!_mali_osk_list_empty(queue)); ++ } ++ ++ job = _MALI_OSK_LIST_ENTRY(queue->next, struct mali_gp_job, list); ++ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ ++ mali_gp_job_list_remove(job); ++ job_queue_gp.depth--; ++ if (job->big_job) { ++ job_queue_gp.big_job_num --; ++ if (job_queue_gp.big_job_num < MALI_MAX_PENDING_BIG_JOB) { ++ /* wake up process */ ++ wait_queue_head_t *queue = mali_session_get_wait_queue(); ++ wake_up(queue); ++ } ++ } ++ return job; ++} ++ ++struct mali_pp_job *mali_scheduler_job_pp_physical_peek(void) ++{ ++ struct mali_pp_job *job = NULL; ++ struct mali_pp_job *tmp_job = NULL; ++ ++ MALI_DEBUG_ASSERT_LOCK_HELD(mali_scheduler_lock_obj); ++ ++ /* ++ * For PP jobs we favour partially started jobs in normal ++ * priority queue over unstarted jobs in high priority queue ++ */ ++ ++ if (!_mali_osk_list_empty(&job_queue_pp.normal_pri)) { ++ MALI_DEBUG_ASSERT(0 < job_queue_pp.depth); ++ ++ tmp_job = _MALI_OSK_LIST_ENTRY(job_queue_pp.normal_pri.next, ++ struct mali_pp_job, list); ++ MALI_DEBUG_ASSERT(NULL != tmp_job); ++ ++ if (MALI_FALSE == mali_pp_job_is_virtual(tmp_job)) { ++ job = tmp_job; ++ } ++ } ++ ++ if (NULL == job || ++ MALI_FALSE == mali_pp_job_has_started_sub_jobs(job)) { ++ /* ++ * There isn't a partially started job in normal queue, so ++ * look in high priority queue. ++ */ ++ if (!_mali_osk_list_empty(&job_queue_pp.high_pri)) { ++ MALI_DEBUG_ASSERT(0 < job_queue_pp.depth); ++ ++ tmp_job = _MALI_OSK_LIST_ENTRY(job_queue_pp.high_pri.next, ++ struct mali_pp_job, list); ++ MALI_DEBUG_ASSERT(NULL != tmp_job); ++ ++ if (MALI_FALSE == mali_pp_job_is_virtual(tmp_job)) { ++ job = tmp_job; ++ } ++ } ++ } ++ ++ return job; ++} ++ ++struct mali_pp_job *mali_scheduler_job_pp_virtual_peek(void) ++{ ++ struct mali_pp_job *job = NULL; ++ struct mali_pp_job *tmp_job = NULL; ++ ++ MALI_DEBUG_ASSERT_LOCK_HELD(mali_scheduler_lock_obj); ++ ++ if (!_mali_osk_list_empty(&job_queue_pp.high_pri)) { ++ MALI_DEBUG_ASSERT(0 < job_queue_pp.depth); ++ ++ tmp_job = _MALI_OSK_LIST_ENTRY(job_queue_pp.high_pri.next, ++ struct mali_pp_job, list); ++ ++ if (MALI_TRUE == mali_pp_job_is_virtual(tmp_job)) { ++ job = tmp_job; ++ } ++ } ++ ++ if (NULL == job) { ++ if (!_mali_osk_list_empty(&job_queue_pp.normal_pri)) { ++ MALI_DEBUG_ASSERT(0 < job_queue_pp.depth); ++ ++ tmp_job = _MALI_OSK_LIST_ENTRY(job_queue_pp.normal_pri.next, ++ struct mali_pp_job, list); ++ ++ if (MALI_TRUE == mali_pp_job_is_virtual(tmp_job)) { ++ job = tmp_job; ++ } ++ } ++ } ++ ++ return job; ++} ++ ++struct mali_pp_job *mali_scheduler_job_pp_physical_get(u32 *sub_job) ++{ ++ struct mali_pp_job *job = mali_scheduler_job_pp_physical_peek(); ++ ++ MALI_DEBUG_ASSERT(MALI_FALSE == mali_pp_job_is_virtual(job)); ++ ++ if (NULL != job) { ++ *sub_job = mali_pp_job_get_first_unstarted_sub_job(job); ++ ++ mali_pp_job_mark_sub_job_started(job, *sub_job); ++ if (MALI_FALSE == mali_pp_job_has_unstarted_sub_jobs(job)) { ++ /* Remove from queue when last sub job has been retrieved */ ++ mali_pp_job_list_remove(job); ++ } ++ ++ job_queue_pp.depth--; ++ ++ /* ++ * Job about to start so it is no longer be ++ * possible to discard WB ++ */ ++ mali_pp_job_fb_lookup_remove(job); ++ } ++ ++ return job; ++} ++ ++struct mali_pp_job *mali_scheduler_job_pp_virtual_get(void) ++{ ++ struct mali_pp_job *job = mali_scheduler_job_pp_virtual_peek(); ++ ++ MALI_DEBUG_ASSERT(MALI_TRUE == mali_pp_job_is_virtual(job)); ++ ++ if (NULL != job) { ++ MALI_DEBUG_ASSERT(0 == ++ mali_pp_job_get_first_unstarted_sub_job(job)); ++ MALI_DEBUG_ASSERT(1 == ++ mali_pp_job_get_sub_job_count(job)); ++ ++ mali_pp_job_mark_sub_job_started(job, 0); ++ ++ mali_pp_job_list_remove(job); ++ ++ job_queue_pp.depth--; ++ ++ /* ++ * Job about to start so it is no longer be ++ * possible to discard WB ++ */ ++ mali_pp_job_fb_lookup_remove(job); ++ } ++ ++ return job; ++} ++ ++mali_scheduler_mask mali_scheduler_activate_gp_job(struct mali_gp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ ++ MALI_DEBUG_PRINT(4, ("Mali GP scheduler: Timeline activation for job %u (0x%08X).\n", ++ mali_gp_job_get_id(job), job)); ++ ++ mali_scheduler_lock(); ++ ++ if (!mali_scheduler_queue_gp_job(job)) { ++ /* Failed to enqueue job, release job (with error) */ ++ ++ mali_scheduler_unlock(); ++ ++ mali_timeline_tracker_release(mali_gp_job_get_tracker(job)); ++ mali_gp_job_signal_pp_tracker(job, MALI_FALSE); ++ ++ /* This will notify user space and close the job object */ ++ mali_scheduler_complete_gp_job(job, MALI_FALSE, ++ MALI_TRUE, MALI_FALSE); ++ ++ return MALI_SCHEDULER_MASK_EMPTY; ++ } ++ ++ mali_scheduler_unlock(); ++ ++ return MALI_SCHEDULER_MASK_GP; ++} ++ ++mali_scheduler_mask mali_scheduler_activate_pp_job(struct mali_pp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ ++ MALI_DEBUG_PRINT(4, ("Mali PP scheduler: Timeline activation for job %u (0x%08X).\n", ++ mali_pp_job_get_id(job), job)); ++ ++ if (MALI_TRUE == mali_timeline_tracker_activation_error( ++ mali_pp_job_get_tracker(job))) { ++ MALI_DEBUG_PRINT(3, ("Mali PP scheduler: Job %u (0x%08X) activated with error, aborting.\n", ++ mali_pp_job_get_id(job), job)); ++ ++ mali_scheduler_lock(); ++ mali_pp_job_fb_lookup_remove(job); ++ mali_pp_job_mark_unstarted_failed(job); ++ mali_scheduler_unlock(); ++ ++ mali_timeline_tracker_release(mali_pp_job_get_tracker(job)); ++ ++ /* This will notify user space and close the job object */ ++ mali_scheduler_complete_pp_job(job, 0, MALI_TRUE, MALI_FALSE); ++ ++ return MALI_SCHEDULER_MASK_EMPTY; ++ } ++ ++#if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) ++ if (mali_pp_job_needs_dma_buf_mapping(job)) { ++ mali_scheduler_deferred_pp_job_queue(job); ++ return MALI_SCHEDULER_MASK_EMPTY; ++ } ++#endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) */ ++ ++ mali_scheduler_lock(); ++ ++ if (!mali_scheduler_queue_pp_job(job)) { ++ /* Failed to enqueue job, release job (with error) */ ++ mali_pp_job_fb_lookup_remove(job); ++ mali_pp_job_mark_unstarted_failed(job); ++ mali_scheduler_unlock(); ++ ++ mali_timeline_tracker_release(mali_pp_job_get_tracker(job)); ++ ++ /* This will notify user space and close the job object */ ++ mali_scheduler_complete_pp_job(job, 0, MALI_TRUE, MALI_FALSE); ++ ++ return MALI_SCHEDULER_MASK_EMPTY; ++ } ++ ++ mali_scheduler_unlock(); ++ return MALI_SCHEDULER_MASK_PP; ++} ++ ++void mali_scheduler_complete_gp_job(struct mali_gp_job *job, ++ mali_bool success, ++ mali_bool user_notification, ++ mali_bool dequeued) ++{ ++ if (user_notification) { ++ mali_scheduler_return_gp_job_to_user(job, success); ++ } ++ ++ if (dequeued) { ++ _mali_osk_pm_dev_ref_put(); ++ ++ if (mali_utilization_enabled()) { ++ mali_utilization_gp_end(); ++ } ++ mali_pm_record_gpu_idle(MALI_TRUE); ++ } ++ ++ mali_gp_job_delete(job); ++} ++ ++void mali_scheduler_complete_pp_job(struct mali_pp_job *job, ++ u32 num_cores_in_virtual, ++ mali_bool user_notification, ++ mali_bool dequeued) ++{ ++ job->user_notification = user_notification; ++ job->num_pp_cores_in_virtual = num_cores_in_virtual; ++ ++#if defined(CONFIG_MALI_DMA_BUF_FENCE) ++ if (NULL != job->rendered_dma_fence) ++ mali_dma_fence_signal_and_put(&job->rendered_dma_fence); ++#endif ++ ++ if (dequeued) { ++#if defined(CONFIG_MALI_DVFS) ++ if (mali_pp_job_is_window_surface(job)) { ++ struct mali_session_data *session; ++ session = mali_pp_job_get_session(job); ++ mali_session_inc_num_window_jobs(session); ++ } ++#endif ++ _mali_osk_pm_dev_ref_put(); ++ ++ if (mali_utilization_enabled()) { ++ mali_utilization_pp_end(); ++ } ++ mali_pm_record_gpu_idle(MALI_FALSE); ++ } ++ ++ /* With ZRAM feature enabled, all pp jobs will be force to use deferred delete. */ ++ mali_scheduler_deferred_pp_job_delete(job); ++} ++ ++void mali_scheduler_abort_session(struct mali_session_data *session) ++{ ++ struct mali_gp_job *gp_job; ++ struct mali_gp_job *gp_tmp; ++ struct mali_pp_job *pp_job; ++ struct mali_pp_job *pp_tmp; ++ _MALI_OSK_LIST_HEAD_STATIC_INIT(removed_jobs_gp); ++ _MALI_OSK_LIST_HEAD_STATIC_INIT(removed_jobs_pp); ++ ++ MALI_DEBUG_ASSERT_POINTER(session); ++ MALI_DEBUG_ASSERT(session->is_aborting); ++ ++ MALI_DEBUG_PRINT(3, ("Mali scheduler: Aborting all queued jobs from session 0x%08X.\n", ++ session)); ++ ++ mali_scheduler_lock(); ++ ++ /* Remove from GP normal priority queue */ ++ _MALI_OSK_LIST_FOREACHENTRY(gp_job, gp_tmp, &job_queue_gp.normal_pri, ++ struct mali_gp_job, list) { ++ if (mali_gp_job_get_session(gp_job) == session) { ++ mali_gp_job_list_move(gp_job, &removed_jobs_gp); ++ job_queue_gp.depth--; ++ job_queue_gp.big_job_num -= gp_job->big_job ? 1 : 0; ++ } ++ } ++ ++ /* Remove from GP high priority queue */ ++ _MALI_OSK_LIST_FOREACHENTRY(gp_job, gp_tmp, &job_queue_gp.high_pri, ++ struct mali_gp_job, list) { ++ if (mali_gp_job_get_session(gp_job) == session) { ++ mali_gp_job_list_move(gp_job, &removed_jobs_gp); ++ job_queue_gp.depth--; ++ job_queue_gp.big_job_num -= gp_job->big_job ? 1 : 0; ++ } ++ } ++ ++ /* Remove from PP normal priority queue */ ++ _MALI_OSK_LIST_FOREACHENTRY(pp_job, pp_tmp, ++ &job_queue_pp.normal_pri, ++ struct mali_pp_job, list) { ++ if (mali_pp_job_get_session(pp_job) == session) { ++ mali_pp_job_fb_lookup_remove(pp_job); ++ ++ job_queue_pp.depth -= ++ mali_pp_job_unstarted_sub_job_count( ++ pp_job); ++ mali_pp_job_mark_unstarted_failed(pp_job); ++ ++ if (MALI_FALSE == mali_pp_job_has_unstarted_sub_jobs(pp_job)) { ++ if (mali_pp_job_is_complete(pp_job)) { ++ mali_pp_job_list_move(pp_job, ++ &removed_jobs_pp); ++ } else { ++ mali_pp_job_list_remove(pp_job); ++ } ++ } ++ } ++ } ++ ++ /* Remove from PP high priority queue */ ++ _MALI_OSK_LIST_FOREACHENTRY(pp_job, pp_tmp, ++ &job_queue_pp.high_pri, ++ struct mali_pp_job, list) { ++ if (mali_pp_job_get_session(pp_job) == session) { ++ mali_pp_job_fb_lookup_remove(pp_job); ++ ++ job_queue_pp.depth -= ++ mali_pp_job_unstarted_sub_job_count( ++ pp_job); ++ mali_pp_job_mark_unstarted_failed(pp_job); ++ ++ if (MALI_FALSE == mali_pp_job_has_unstarted_sub_jobs(pp_job)) { ++ if (mali_pp_job_is_complete(pp_job)) { ++ mali_pp_job_list_move(pp_job, ++ &removed_jobs_pp); ++ } else { ++ mali_pp_job_list_remove(pp_job); ++ } ++ } ++ } ++ } ++ ++ /* ++ * Release scheduler lock so we can release trackers ++ * (which will potentially queue new jobs) ++ */ ++ mali_scheduler_unlock(); ++ ++ /* Release and complete all (non-running) found GP jobs */ ++ _MALI_OSK_LIST_FOREACHENTRY(gp_job, gp_tmp, &removed_jobs_gp, ++ struct mali_gp_job, list) { ++ mali_timeline_tracker_release(mali_gp_job_get_tracker(gp_job)); ++ mali_gp_job_signal_pp_tracker(gp_job, MALI_FALSE); ++ _mali_osk_list_delinit(&gp_job->list); ++ mali_scheduler_complete_gp_job(gp_job, ++ MALI_FALSE, MALI_FALSE, MALI_TRUE); ++ } ++ ++ /* Release and complete non-running PP jobs */ ++ _MALI_OSK_LIST_FOREACHENTRY(pp_job, pp_tmp, &removed_jobs_pp, ++ struct mali_pp_job, list) { ++ mali_timeline_tracker_release(mali_pp_job_get_tracker(pp_job)); ++ _mali_osk_list_delinit(&pp_job->list); ++ mali_scheduler_complete_pp_job(pp_job, 0, ++ MALI_FALSE, MALI_TRUE); ++ } ++} ++ ++_mali_osk_errcode_t _mali_ukk_gp_start_job(void *ctx, ++ _mali_uk_gp_start_job_s *uargs) ++{ ++ struct mali_session_data *session; ++ struct mali_gp_job *job; ++ mali_timeline_point point; ++ u32 __user *point_ptr = NULL; ++ ++ MALI_DEBUG_ASSERT_POINTER(uargs); ++ MALI_DEBUG_ASSERT_POINTER(ctx); ++ ++ session = (struct mali_session_data *)(uintptr_t)ctx; ++ ++ job = mali_gp_job_create(session, uargs, mali_scheduler_get_new_id(), ++ NULL); ++ if (NULL == job) { ++ MALI_PRINT_ERROR(("Failed to create GP job.\n")); ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ ++ point_ptr = (u32 __user *)(uintptr_t)mali_gp_job_get_timeline_point_ptr(job); ++ ++ point = mali_scheduler_submit_gp_job(session, job); ++ ++ if (0 != _mali_osk_put_user(((u32) point), point_ptr)) { ++ /* ++ * Let user space know that something failed ++ * after the job was started. ++ */ ++ return _MALI_OSK_ERR_ITEM_NOT_FOUND; ++ } ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++_mali_osk_errcode_t _mali_ukk_pp_start_job(void *ctx, ++ _mali_uk_pp_start_job_s *uargs) ++{ ++ _mali_osk_errcode_t ret; ++ struct mali_session_data *session; ++ struct mali_pp_job *job; ++ mali_timeline_point point; ++ u32 __user *point_ptr = NULL; ++ ++ MALI_DEBUG_ASSERT_POINTER(uargs); ++ MALI_DEBUG_ASSERT_POINTER(ctx); ++ ++ session = (struct mali_session_data *)(uintptr_t)ctx; ++ ++ job = mali_pp_job_create(session, uargs, mali_scheduler_get_new_id()); ++ if (NULL == job) { ++ MALI_PRINT_ERROR(("Failed to create PP job.\n")); ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ ++ point_ptr = (u32 __user *)(uintptr_t)mali_pp_job_get_timeline_point_ptr(job); ++ ++ /* Submit PP job. */ ++ ret = mali_scheduler_submit_pp_job(session, job, &point); ++ job = NULL; ++ ++ if (_MALI_OSK_ERR_OK == ret) { ++ if (0 != _mali_osk_put_user(((u32) point), point_ptr)) { ++ /* ++ * Let user space know that something failed ++ * after the jobs were started. ++ */ ++ return _MALI_OSK_ERR_ITEM_NOT_FOUND; ++ } ++ } ++ ++ return ret; ++} ++ ++_mali_osk_errcode_t _mali_ukk_pp_and_gp_start_job(void *ctx, ++ _mali_uk_pp_and_gp_start_job_s *uargs) ++{ ++ _mali_osk_errcode_t ret; ++ struct mali_session_data *session; ++ _mali_uk_pp_and_gp_start_job_s kargs; ++ struct mali_pp_job *pp_job; ++ struct mali_gp_job *gp_job; ++ u32 __user *point_ptr = NULL; ++ mali_timeline_point point; ++ _mali_uk_pp_start_job_s __user *pp_args; ++ _mali_uk_gp_start_job_s __user *gp_args; ++ ++ MALI_DEBUG_ASSERT_POINTER(ctx); ++ MALI_DEBUG_ASSERT_POINTER(uargs); ++ ++ session = (struct mali_session_data *) ctx; ++ ++ if (0 != _mali_osk_copy_from_user(&kargs, uargs, ++ sizeof(_mali_uk_pp_and_gp_start_job_s))) { ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ ++ pp_args = (_mali_uk_pp_start_job_s __user *)(uintptr_t)kargs.pp_args; ++ gp_args = (_mali_uk_gp_start_job_s __user *)(uintptr_t)kargs.gp_args; ++ ++ pp_job = mali_pp_job_create(session, pp_args, ++ mali_scheduler_get_new_id()); ++ if (NULL == pp_job) { ++ MALI_PRINT_ERROR(("Failed to create PP job.\n")); ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ ++ gp_job = mali_gp_job_create(session, gp_args, ++ mali_scheduler_get_new_id(), ++ mali_pp_job_get_tracker(pp_job)); ++ if (NULL == gp_job) { ++ MALI_PRINT_ERROR(("Failed to create GP job.\n")); ++ mali_pp_job_delete(pp_job); ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ ++ point_ptr = (u32 __user *)(uintptr_t)mali_pp_job_get_timeline_point_ptr(pp_job); ++ ++ /* Submit GP job. */ ++ mali_scheduler_submit_gp_job(session, gp_job); ++ gp_job = NULL; ++ ++ /* Submit PP job. */ ++ ret = mali_scheduler_submit_pp_job(session, pp_job, &point); ++ pp_job = NULL; ++ ++ if (_MALI_OSK_ERR_OK == ret) { ++ if (0 != _mali_osk_put_user(((u32) point), point_ptr)) { ++ /* ++ * Let user space know that something failed ++ * after the jobs were started. ++ */ ++ return _MALI_OSK_ERR_ITEM_NOT_FOUND; ++ } ++ } ++ ++ return ret; ++} ++ ++void _mali_ukk_pp_job_disable_wb(_mali_uk_pp_disable_wb_s *args) ++{ ++ struct mali_session_data *session; ++ struct mali_pp_job *job; ++ struct mali_pp_job *tmp; ++ u32 fb_lookup_id; ++ ++ MALI_DEBUG_ASSERT_POINTER(args); ++ MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx); ++ ++ session = (struct mali_session_data *)(uintptr_t)args->ctx; ++ ++ fb_lookup_id = args->fb_id & MALI_PP_JOB_FB_LOOKUP_LIST_MASK; ++ ++ mali_scheduler_lock(); ++ ++ /* Iterate over all jobs for given frame builder_id. */ ++ _MALI_OSK_LIST_FOREACHENTRY(job, tmp, ++ &session->pp_job_fb_lookup_list[fb_lookup_id], ++ struct mali_pp_job, session_fb_lookup_list) { ++ MALI_DEBUG_CODE(u32 disable_mask = 0); ++ ++ if (mali_pp_job_get_frame_builder_id(job) != ++ (u32) args->fb_id) { ++ MALI_DEBUG_PRINT(4, ("Mali PP scheduler: Disable WB mismatching FB.\n")); ++ continue; ++ } ++ ++ MALI_DEBUG_CODE(disable_mask |= 0xD << (4 * 3)); ++ ++ if (mali_pp_job_get_wb0_source_addr(job) == args->wb0_memory) { ++ MALI_DEBUG_CODE(disable_mask |= 0x1 << (4 * 1)); ++ mali_pp_job_disable_wb0(job); ++ } ++ ++ if (mali_pp_job_get_wb1_source_addr(job) == args->wb1_memory) { ++ MALI_DEBUG_CODE(disable_mask |= 0x2 << (4 * 2)); ++ mali_pp_job_disable_wb1(job); ++ } ++ ++ if (mali_pp_job_get_wb2_source_addr(job) == args->wb2_memory) { ++ MALI_DEBUG_CODE(disable_mask |= 0x3 << (4 * 3)); ++ mali_pp_job_disable_wb2(job); ++ } ++ MALI_DEBUG_PRINT(3, ("Mali PP scheduler: Disable WB: 0x%X.\n", ++ disable_mask)); ++ } ++ ++ mali_scheduler_unlock(); ++} ++ ++#if MALI_STATE_TRACKING ++u32 mali_scheduler_dump_state(char *buf, u32 size) ++{ ++ int n = 0; ++ ++ n += _mali_osk_snprintf(buf + n, size - n, "GP queues\n"); ++ n += _mali_osk_snprintf(buf + n, size - n, ++ "\tQueue depth: %u\n", job_queue_gp.depth); ++ n += _mali_osk_snprintf(buf + n, size - n, ++ "\tNormal priority queue is %s\n", ++ _mali_osk_list_empty(&job_queue_gp.normal_pri) ? ++ "empty" : "not empty"); ++ n += _mali_osk_snprintf(buf + n, size - n, ++ "\tHigh priority queue is %s\n", ++ _mali_osk_list_empty(&job_queue_gp.high_pri) ? ++ "empty" : "not empty"); ++ ++ n += _mali_osk_snprintf(buf + n, size - n, ++ "PP queues\n"); ++ n += _mali_osk_snprintf(buf + n, size - n, ++ "\tQueue depth: %u\n", job_queue_pp.depth); ++ n += _mali_osk_snprintf(buf + n, size - n, ++ "\tNormal priority queue is %s\n", ++ _mali_osk_list_empty(&job_queue_pp.normal_pri) ++ ? "empty" : "not empty"); ++ n += _mali_osk_snprintf(buf + n, size - n, ++ "\tHigh priority queue is %s\n", ++ _mali_osk_list_empty(&job_queue_pp.high_pri) ++ ? "empty" : "not empty"); ++ ++ n += _mali_osk_snprintf(buf + n, size - n, "\n"); ++ ++ return n; ++} ++#endif ++ ++/* ++ * ---------- Implementation of static functions ---------- ++ */ ++ ++static mali_timeline_point mali_scheduler_submit_gp_job( ++ struct mali_session_data *session, struct mali_gp_job *job) ++{ ++ mali_timeline_point point; ++ ++ MALI_DEBUG_ASSERT_POINTER(session); ++ MALI_DEBUG_ASSERT_POINTER(job); ++ ++ /* Add job to Timeline system. */ ++ point = mali_timeline_system_add_tracker(session->timeline_system, ++ mali_gp_job_get_tracker(job), MALI_TIMELINE_GP); ++ ++ return point; ++} ++ ++static _mali_osk_errcode_t mali_scheduler_submit_pp_job( ++ struct mali_session_data *session, struct mali_pp_job *job, mali_timeline_point *point) ++ ++{ ++ _mali_osk_errcode_t ret = _MALI_OSK_ERR_OK; ++ ++#if defined(CONFIG_MALI_DMA_BUF_FENCE) ++ struct ww_acquire_ctx ww_actx; ++ u32 i; ++ u32 num_memory_cookies = 0; ++ struct reservation_object **reservation_object_list = NULL; ++ unsigned int num_reservation_object = 0; ++#endif ++ ++ MALI_DEBUG_ASSERT_POINTER(session); ++ MALI_DEBUG_ASSERT_POINTER(job); ++ ++ mali_scheduler_lock(); ++ /* ++ * Adding job to the lookup list used to quickly discard ++ * writeback units of queued jobs. ++ */ ++ mali_pp_job_fb_lookup_add(job); ++ mali_scheduler_unlock(); ++ ++#if defined(CONFIG_MALI_DMA_BUF_FENCE) ++ ++ /* Allocate the reservation_object_list to list the dma reservation object of dependent dma buffer */ ++ num_memory_cookies = mali_pp_job_num_memory_cookies(job); ++ if (0 < num_memory_cookies) { ++ reservation_object_list = kzalloc(sizeof(struct reservation_object *) * num_memory_cookies, GFP_KERNEL); ++ if (NULL == reservation_object_list) { ++ MALI_PRINT_ERROR(("Failed to alloc the reservation object list.\n")); ++ ret = _MALI_OSK_ERR_NOMEM; ++ goto failed_to_alloc_reservation_object_list; ++ } ++ } ++ ++ /* Add the dma reservation object into reservation_object_list*/ ++ for (i = 0; i < num_memory_cookies; i++) { ++ mali_mem_backend *mem_backend = NULL; ++ struct reservation_object *tmp_reservation_object = NULL; ++ u32 mali_addr = mali_pp_job_get_memory_cookie(job, i); ++ ++ mem_backend = mali_mem_backend_struct_search(session, mali_addr); ++ ++ MALI_DEBUG_ASSERT_POINTER(mem_backend); ++ ++ if (NULL == mem_backend) { ++ MALI_PRINT_ERROR(("Failed to find the memory backend for memory cookie[%d].\n", i)); ++ goto failed_to_find_mem_backend; ++ } ++ ++ if (MALI_MEM_DMA_BUF != mem_backend->type) ++ continue; ++ ++ tmp_reservation_object = mem_backend->dma_buf.attachment->buf->resv; ++ ++ if (NULL != tmp_reservation_object) { ++ mali_dma_fence_add_reservation_object_list(tmp_reservation_object, ++ reservation_object_list, &num_reservation_object); ++ } ++ } ++ ++ /* ++ * Add the mali dma fence callback to wait for all dependent dma buf, ++ * and extend the timeline system to support dma fence, ++ * then create the new internal dma fence to replace all last dma fence for dependent dma buf. ++ */ ++ if (0 < num_reservation_object) { ++ int error; ++ int num_dma_fence_waiter = 0; ++ /* Create one new dma fence.*/ ++ job->rendered_dma_fence = mali_dma_fence_new(job->session->fence_context, ++ _mali_osk_atomic_inc_return(&job->session->fence_seqno)); ++ ++ if (NULL == job->rendered_dma_fence) { ++ MALI_PRINT_ERROR(("Failed to creat one new dma fence.\n")); ++ ret = _MALI_OSK_ERR_FAULT; ++ goto failed_to_create_dma_fence; ++ } ++ ++ /* In order to avoid deadlock, wait/wound mutex lock to lock all dma buffers*/ ++ ++ error = mali_dma_fence_lock_reservation_object_list(reservation_object_list, ++ num_reservation_object, &ww_actx); ++ ++ if (0 != error) { ++ MALI_PRINT_ERROR(("Failed to lock all reservation objects.\n")); ++ ret = _MALI_OSK_ERR_FAULT; ++ goto failed_to_lock_reservation_object_list; ++ } ++ ++ mali_dma_fence_context_init(&job->dma_fence_context, ++ mali_timeline_dma_fence_callback, (void *)job); ++ ++ /* Add dma fence waiters and dma fence callback. */ ++ for (i = 0; i < num_reservation_object; i++) { ++ ret = mali_dma_fence_context_add_waiters(&job->dma_fence_context, reservation_object_list[i]); ++ if (_MALI_OSK_ERR_OK != ret) { ++ MALI_PRINT_ERROR(("Failed to add waiter into mali dma fence context.\n")); ++ goto failed_to_add_dma_fence_waiter; ++ } ++ } ++ ++ for (i = 0; i < num_reservation_object; i++) { ++ reservation_object_add_excl_fence(reservation_object_list[i], job->rendered_dma_fence); ++ } ++ ++ num_dma_fence_waiter = job->dma_fence_context.num_dma_fence_waiter; ++ ++ /* Add job to Timeline system. */ ++ (*point) = mali_timeline_system_add_tracker(session->timeline_system, ++ mali_pp_job_get_tracker(job), MALI_TIMELINE_PP); ++ ++ if (0 != num_dma_fence_waiter) { ++ mali_dma_fence_context_dec_count(&job->dma_fence_context); ++ } ++ ++ /* Unlock all wait/wound mutex lock. */ ++ mali_dma_fence_unlock_reservation_object_list(reservation_object_list, ++ num_reservation_object, &ww_actx); ++ } else { ++ /* Add job to Timeline system. */ ++ (*point) = mali_timeline_system_add_tracker(session->timeline_system, ++ mali_pp_job_get_tracker(job), MALI_TIMELINE_PP); ++ } ++ ++ kfree(reservation_object_list); ++ return ret; ++#else ++ /* Add job to Timeline system. */ ++ (*point) = mali_timeline_system_add_tracker(session->timeline_system, ++ mali_pp_job_get_tracker(job), MALI_TIMELINE_PP); ++#endif ++ ++#if defined(CONFIG_MALI_DMA_BUF_FENCE) ++failed_to_add_dma_fence_waiter: ++ mali_dma_fence_context_term(&job->dma_fence_context); ++ mali_dma_fence_unlock_reservation_object_list(reservation_object_list, ++ num_reservation_object, &ww_actx); ++failed_to_lock_reservation_object_list: ++ mali_dma_fence_signal_and_put(&job->rendered_dma_fence); ++failed_to_create_dma_fence: ++failed_to_find_mem_backend: ++ if (NULL != reservation_object_list) ++ kfree(reservation_object_list); ++failed_to_alloc_reservation_object_list: ++ mali_pp_job_fb_lookup_remove(job); ++#endif ++ return ret; ++} ++ ++static mali_bool mali_scheduler_queue_gp_job(struct mali_gp_job *job) ++{ ++ struct mali_session_data *session; ++ _mali_osk_list_t *queue; ++ ++ MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); ++ MALI_DEBUG_ASSERT_POINTER(job); ++ ++ session = mali_gp_job_get_session(job); ++ MALI_DEBUG_ASSERT_POINTER(session); ++ ++ if (unlikely(session->is_aborting)) { ++ MALI_DEBUG_PRINT(4, ("Mali GP scheduler: Job %u (0x%08X) queued while session is aborting.\n", ++ mali_gp_job_get_id(job), job)); ++ return MALI_FALSE; /* job not queued */ ++ } ++ ++ mali_gp_job_set_cache_order(job, mali_scheduler_get_new_cache_order()); ++ ++ /* Determine which queue the job should be added to. */ ++ if (session->use_high_priority_job_queue) { ++ queue = &job_queue_gp.high_pri; ++ } else { ++ queue = &job_queue_gp.normal_pri; ++ } ++ ++ job_queue_gp.depth += 1; ++ job_queue_gp.big_job_num += (job->big_job) ? 1 : 0; ++ ++ /* Add job to queue (mali_gp_job_queue_add find correct place). */ ++ mali_gp_job_list_add(job, queue); ++ ++ /* ++ * We hold a PM reference for every job we hold queued (and running) ++ * It is important that we take this reference after job has been ++ * added the the queue so that any runtime resume could schedule this ++ * job right there and then. ++ */ ++ _mali_osk_pm_dev_ref_get_async(); ++ ++ if (mali_utilization_enabled()) { ++ /* ++ * We cheat a little bit by counting the GP as busy from the ++ * time a GP job is queued. This will be fine because we only ++ * loose the tiny idle gap between jobs, but we will instead ++ * get less utilization work to do (less locks taken) ++ */ ++ mali_utilization_gp_start(); ++ } ++ ++ mali_pm_record_gpu_active(MALI_TRUE); ++ ++ /* Add profiling events for job enqueued */ ++ _mali_osk_profiling_add_event( ++ MALI_PROFILING_EVENT_TYPE_SINGLE | ++ MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | ++ MALI_PROFILING_EVENT_REASON_SINGLE_SW_GP_ENQUEUE, ++ mali_gp_job_get_pid(job), ++ mali_gp_job_get_tid(job), ++ mali_gp_job_get_frame_builder_id(job), ++ mali_gp_job_get_flush_id(job), ++ 0); ++ ++#if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS) ++ trace_gpu_job_enqueue(mali_gp_job_get_tid(job), ++ mali_gp_job_get_id(job), "GP"); ++#endif ++ ++ MALI_DEBUG_PRINT(3, ("Mali GP scheduler: Job %u (0x%08X) queued\n", ++ mali_gp_job_get_id(job), job)); ++ ++ return MALI_TRUE; /* job queued */ ++} ++ ++static mali_bool mali_scheduler_queue_pp_job(struct mali_pp_job *job) ++{ ++ struct mali_session_data *session; ++ _mali_osk_list_t *queue = NULL; ++ ++ MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); ++ MALI_DEBUG_ASSERT_POINTER(job); ++ ++ session = mali_pp_job_get_session(job); ++ MALI_DEBUG_ASSERT_POINTER(session); ++ ++ if (unlikely(session->is_aborting)) { ++ MALI_DEBUG_PRINT(2, ("Mali PP scheduler: Job %u (0x%08X) queued while session is aborting.\n", ++ mali_pp_job_get_id(job), job)); ++ return MALI_FALSE; /* job not queued */ ++ } else if (unlikely(MALI_SWAP_IN_FAIL == job->swap_status)) { ++ MALI_DEBUG_PRINT(2, ("Mali PP scheduler: Job %u (0x%08X) queued while swap in failed.\n", ++ mali_pp_job_get_id(job), job)); ++ return MALI_FALSE; ++ } ++ ++ mali_pp_job_set_cache_order(job, mali_scheduler_get_new_cache_order()); ++ ++ if (session->use_high_priority_job_queue) { ++ queue = &job_queue_pp.high_pri; ++ } else { ++ queue = &job_queue_pp.normal_pri; ++ } ++ ++ job_queue_pp.depth += ++ mali_pp_job_get_sub_job_count(job); ++ ++ /* Add job to queue (mali_gp_job_queue_add find correct place). */ ++ mali_pp_job_list_add(job, queue); ++ ++ /* ++ * We hold a PM reference for every job we hold queued (and running) ++ * It is important that we take this reference after job has been ++ * added the the queue so that any runtime resume could schedule this ++ * job right there and then. ++ */ ++ _mali_osk_pm_dev_ref_get_async(); ++ ++ if (mali_utilization_enabled()) { ++ /* ++ * We cheat a little bit by counting the PP as busy from the ++ * time a PP job is queued. This will be fine because we only ++ * loose the tiny idle gap between jobs, but we will instead ++ * get less utilization work to do (less locks taken) ++ */ ++ mali_utilization_pp_start(); ++ } ++ ++ mali_pm_record_gpu_active(MALI_FALSE); ++ ++ /* Add profiling events for job enqueued */ ++ _mali_osk_profiling_add_event( ++ MALI_PROFILING_EVENT_TYPE_SINGLE | ++ MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | ++ MALI_PROFILING_EVENT_REASON_SINGLE_SW_PP_ENQUEUE, ++ mali_pp_job_get_pid(job), ++ mali_pp_job_get_tid(job), ++ mali_pp_job_get_frame_builder_id(job), ++ mali_pp_job_get_flush_id(job), ++ 0); ++ ++#if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS) ++ trace_gpu_job_enqueue(mali_pp_job_get_tid(job), ++ mali_pp_job_get_id(job), "PP"); ++#endif ++ ++ MALI_DEBUG_PRINT(3, ("Mali PP scheduler: %s job %u (0x%08X) with %u parts queued.\n", ++ mali_pp_job_is_virtual(job) ++ ? "Virtual" : "Physical", ++ mali_pp_job_get_id(job), job, ++ mali_pp_job_get_sub_job_count(job))); ++ ++ return MALI_TRUE; /* job queued */ ++} ++ ++static void mali_scheduler_return_gp_job_to_user(struct mali_gp_job *job, ++ mali_bool success) ++{ ++ _mali_uk_gp_job_finished_s *jobres; ++ struct mali_session_data *session; ++ _mali_osk_notification_t *notification; ++ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ ++ session = mali_gp_job_get_session(job); ++ MALI_DEBUG_ASSERT_POINTER(session); ++ ++ notification = mali_gp_job_get_finished_notification(job); ++ MALI_DEBUG_ASSERT_POINTER(notification); ++ ++ jobres = notification->result_buffer; ++ MALI_DEBUG_ASSERT_POINTER(jobres); ++ ++ jobres->pending_big_job_num = mali_scheduler_job_gp_big_job_count(); ++ ++ jobres->user_job_ptr = mali_gp_job_get_user_id(job); ++ if (MALI_TRUE == success) { ++ jobres->status = _MALI_UK_JOB_STATUS_END_SUCCESS; ++ } else { ++ jobres->status = _MALI_UK_JOB_STATUS_END_UNKNOWN_ERR; ++ } ++ jobres->heap_current_addr = mali_gp_job_get_current_heap_addr(job); ++ jobres->perf_counter0 = mali_gp_job_get_perf_counter_value0(job); ++ jobres->perf_counter1 = mali_gp_job_get_perf_counter_value1(job); ++ ++ mali_session_send_notification(session, notification); ++} ++ ++void mali_scheduler_return_pp_job_to_user(struct mali_pp_job *job, ++ u32 num_cores_in_virtual) ++{ ++ u32 i; ++ u32 num_counters_to_copy; ++ _mali_uk_pp_job_finished_s *jobres; ++ struct mali_session_data *session; ++ _mali_osk_notification_t *notification; ++ ++ if (MALI_TRUE == mali_pp_job_use_no_notification(job)) { ++ return; ++ } ++ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ ++ session = mali_pp_job_get_session(job); ++ MALI_DEBUG_ASSERT_POINTER(session); ++ ++ notification = mali_pp_job_get_finished_notification(job); ++ MALI_DEBUG_ASSERT_POINTER(notification); ++ ++ jobres = notification->result_buffer; ++ MALI_DEBUG_ASSERT_POINTER(jobres); ++ ++ jobres->user_job_ptr = mali_pp_job_get_user_id(job); ++ if (MALI_TRUE == mali_pp_job_was_success(job)) { ++ jobres->status = _MALI_UK_JOB_STATUS_END_SUCCESS; ++ } else { ++ jobres->status = _MALI_UK_JOB_STATUS_END_UNKNOWN_ERR; ++ } ++ ++ if (mali_pp_job_is_virtual(job)) { ++ num_counters_to_copy = num_cores_in_virtual; ++ } else { ++ num_counters_to_copy = mali_pp_job_get_sub_job_count(job); ++ } ++ ++ for (i = 0; i < num_counters_to_copy; i++) { ++ jobres->perf_counter0[i] = ++ mali_pp_job_get_perf_counter_value0(job, i); ++ jobres->perf_counter1[i] = ++ mali_pp_job_get_perf_counter_value1(job, i); ++ jobres->perf_counter_src0 = ++ mali_pp_job_get_pp_counter_global_src0(); ++ jobres->perf_counter_src1 = ++ mali_pp_job_get_pp_counter_global_src1(); ++ } ++ ++ mali_session_send_notification(session, notification); ++} ++ ++static void mali_scheduler_deferred_pp_job_delete(struct mali_pp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ ++ _mali_osk_spinlock_irq_lock(scheduler_pp_job_delete_lock); ++ mali_pp_job_list_addtail(job, &scheduler_pp_job_deletion_queue); ++ _mali_osk_spinlock_irq_unlock(scheduler_pp_job_delete_lock); ++ ++ _mali_osk_wq_schedule_work(scheduler_wq_pp_job_delete); ++} ++ ++void mali_scheduler_do_pp_job_delete(void *arg) ++{ ++ _MALI_OSK_LIST_HEAD_STATIC_INIT(list); ++ struct mali_pp_job *job; ++ struct mali_pp_job *tmp; ++ ++ MALI_IGNORE(arg); ++ ++ /* ++ * Quickly "unhook" the jobs pending to be deleted, so we can release ++ * the lock before we start deleting the job objects ++ * (without any locks held) ++ */ ++ _mali_osk_spinlock_irq_lock(scheduler_pp_job_delete_lock); ++ _mali_osk_list_move_list(&scheduler_pp_job_deletion_queue, &list); ++ _mali_osk_spinlock_irq_unlock(scheduler_pp_job_delete_lock); ++ ++ _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &list, ++ struct mali_pp_job, list) { ++ _mali_osk_list_delinit(&job->list); ++ ++#if defined(CONFIG_MALI_DMA_BUF_FENCE) ++ mali_dma_fence_context_term(&job->dma_fence_context); ++#endif ++ ++ mali_pp_job_delete(job); /* delete the job object itself */ ++ } ++} ++ ++#if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) ++ ++static void mali_scheduler_deferred_pp_job_queue(struct mali_pp_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ ++ _mali_osk_spinlock_irq_lock(scheduler_pp_job_queue_lock); ++ mali_pp_job_list_addtail(job, &scheduler_pp_job_queue_list); ++ _mali_osk_spinlock_irq_unlock(scheduler_pp_job_queue_lock); ++ ++ _mali_osk_wq_schedule_work(scheduler_wq_pp_job_queue); ++} ++ ++static void mali_scheduler_do_pp_job_queue(void *arg) ++{ ++ _MALI_OSK_LIST_HEAD_STATIC_INIT(list); ++ struct mali_pp_job *job; ++ struct mali_pp_job *tmp; ++ mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; ++ ++ MALI_IGNORE(arg); ++ ++ /* ++ * Quickly "unhook" the jobs pending to be queued, so we can release ++ * the lock before we start queueing the job objects ++ * (without any locks held) ++ */ ++ _mali_osk_spinlock_irq_lock(scheduler_pp_job_queue_lock); ++ _mali_osk_list_move_list(&scheduler_pp_job_queue_list, &list); ++ _mali_osk_spinlock_irq_unlock(scheduler_pp_job_queue_lock); ++ ++ /* First loop through all jobs and do the pre-work (no locks needed) */ ++ _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &list, ++ struct mali_pp_job, list) { ++ if (mali_pp_job_needs_dma_buf_mapping(job)) { ++ /* ++ * This operation could fail, but we continue anyway, ++ * because the worst that could happen is that this ++ * job will fail due to a Mali page fault. ++ */ ++ mali_dma_buf_map_job(job); ++ } ++ } ++ ++ mali_scheduler_lock(); ++ ++ /* Then loop through all jobs again to queue them (lock needed) */ ++ _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &list, ++ struct mali_pp_job, list) { ++ ++ /* Remove from scheduler_pp_job_queue_list before queueing */ ++ mali_pp_job_list_remove(job); ++ ++ if (mali_scheduler_queue_pp_job(job)) { ++ /* Job queued successfully */ ++ schedule_mask |= MALI_SCHEDULER_MASK_PP; ++ } else { ++ /* Failed to enqueue job, release job (with error) */ ++ mali_pp_job_fb_lookup_remove(job); ++ mali_pp_job_mark_unstarted_failed(job); ++ ++ /* unlock scheduler in this uncommon case */ ++ mali_scheduler_unlock(); ++ ++ schedule_mask |= mali_timeline_tracker_release( ++ mali_pp_job_get_tracker(job)); ++ ++ /* Notify user space and close the job object */ ++ mali_scheduler_complete_pp_job(job, 0, MALI_TRUE, ++ MALI_FALSE); ++ ++ mali_scheduler_lock(); ++ } ++ } ++ ++ mali_scheduler_unlock(); ++ ++ /* Trigger scheduling of jobs */ ++ mali_executor_schedule_from_mask(schedule_mask, MALI_FALSE); ++} ++ ++#endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) */ ++ ++void mali_scheduler_gp_pp_job_queue_print(void) ++{ ++ struct mali_gp_job *gp_job = NULL; ++ struct mali_gp_job *tmp_gp_job = NULL; ++ struct mali_pp_job *pp_job = NULL; ++ struct mali_pp_job *tmp_pp_job = NULL; ++ ++ MALI_DEBUG_ASSERT_LOCK_HELD(mali_scheduler_lock_obj); ++ MALI_DEBUG_ASSERT_LOCK_HELD(mali_executor_lock_obj); ++ ++ /* dump job queup status */ ++ if ((0 == job_queue_gp.depth) && (0 == job_queue_pp.depth)) { ++ MALI_PRINT(("No GP&PP job in the job queue.\n")); ++ return; ++ } ++ ++ MALI_PRINT(("Total (%d) GP job in the job queue.\n", job_queue_gp.depth)); ++ if (job_queue_gp.depth > 0) { ++ if (!_mali_osk_list_empty(&job_queue_gp.high_pri)) { ++ _MALI_OSK_LIST_FOREACHENTRY(gp_job, tmp_gp_job, &job_queue_gp.high_pri, ++ struct mali_gp_job, list) { ++ MALI_PRINT(("GP job(%p) id = %d tid = %d pid = %d in the gp job high_pri queue\n", gp_job, gp_job->id, gp_job->tid, gp_job->pid)); ++ } ++ } ++ ++ if (!_mali_osk_list_empty(&job_queue_gp.normal_pri)) { ++ _MALI_OSK_LIST_FOREACHENTRY(gp_job, tmp_gp_job, &job_queue_gp.normal_pri, ++ struct mali_gp_job, list) { ++ MALI_PRINT(("GP job(%p) id = %d tid = %d pid = %d in the gp job normal_pri queue\n", gp_job, gp_job->id, gp_job->tid, gp_job->pid)); ++ } ++ } ++ } ++ ++ MALI_PRINT(("Total (%d) PP job in the job queue.\n", job_queue_pp.depth)); ++ if (job_queue_pp.depth > 0) { ++ if (!_mali_osk_list_empty(&job_queue_pp.high_pri)) { ++ _MALI_OSK_LIST_FOREACHENTRY(pp_job, tmp_pp_job, &job_queue_pp.high_pri, ++ struct mali_pp_job, list) { ++ if (mali_pp_job_is_virtual(pp_job)) { ++ MALI_PRINT(("PP Virtual job(%p) id = %d tid = %d pid = %d in the pp job high_pri queue\n", pp_job, pp_job->id, pp_job->tid, pp_job->pid)); ++ } else { ++ MALI_PRINT(("PP Physical job(%p) id = %d tid = %d pid = %d in the pp job high_pri queue\n", pp_job, pp_job->id, pp_job->tid, pp_job->pid)); ++ } ++ } ++ } ++ ++ if (!_mali_osk_list_empty(&job_queue_pp.normal_pri)) { ++ _MALI_OSK_LIST_FOREACHENTRY(pp_job, tmp_pp_job, &job_queue_pp.normal_pri, ++ struct mali_pp_job, list) { ++ if (mali_pp_job_is_virtual(pp_job)) { ++ MALI_PRINT(("PP Virtual job(%p) id = %d tid = %d pid = %d in the pp job normal_pri queue\n", pp_job, pp_job->id, pp_job->tid, pp_job->pid)); ++ } else { ++ MALI_PRINT(("PP Physical job(%p) id = %d tid = %d pid = %d in the pp job normal_pri queue\n", pp_job, pp_job->id, pp_job->tid, pp_job->pid)); ++ } ++ } ++ } ++ } ++ ++ /* dump group running job status */ ++ mali_executor_running_status_print(); ++} +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_scheduler.h b/drivers/gpu/arm/mali400/mali/common/mali_scheduler.h +new file mode 100755 +index 000000000000..de81a421ea9a +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_scheduler.h +@@ -0,0 +1,131 @@ ++/* ++ * Copyright (C) 2012-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_SCHEDULER_H__ ++#define __MALI_SCHEDULER_H__ ++ ++#include "mali_osk.h" ++#include "mali_osk_list.h" ++#include "mali_scheduler_types.h" ++#include "mali_session.h" ++ ++struct mali_scheduler_job_queue { ++ _MALI_OSK_LIST_HEAD(normal_pri); /* Queued jobs with normal priority */ ++ _MALI_OSK_LIST_HEAD(high_pri); /* Queued jobs with high priority */ ++ u32 depth; /* Depth of combined queues. */ ++ u32 big_job_num; ++}; ++ ++extern _mali_osk_spinlock_irq_t *mali_scheduler_lock_obj; ++ ++/* Queue of jobs to be executed on the GP group */ ++extern struct mali_scheduler_job_queue job_queue_gp; ++ ++/* Queue of PP jobs */ ++extern struct mali_scheduler_job_queue job_queue_pp; ++ ++extern _mali_osk_atomic_t mali_job_id_autonumber; ++extern _mali_osk_atomic_t mali_job_cache_order_autonumber; ++ ++#define MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD() MALI_DEBUG_ASSERT_LOCK_HELD(mali_scheduler_lock_obj); ++ ++_mali_osk_errcode_t mali_scheduler_initialize(void); ++void mali_scheduler_terminate(void); ++ ++MALI_STATIC_INLINE void mali_scheduler_lock(void) ++{ ++ _mali_osk_spinlock_irq_lock(mali_scheduler_lock_obj); ++ MALI_DEBUG_PRINT(5, ("Mali scheduler: scheduler lock taken.\n")); ++} ++ ++MALI_STATIC_INLINE void mali_scheduler_unlock(void) ++{ ++ MALI_DEBUG_PRINT(5, ("Mali scheduler: Releasing scheduler lock.\n")); ++ _mali_osk_spinlock_irq_unlock(mali_scheduler_lock_obj); ++} ++ ++MALI_STATIC_INLINE u32 mali_scheduler_job_gp_count(void) ++{ ++ return job_queue_gp.depth; ++} ++MALI_STATIC_INLINE u32 mali_scheduler_job_gp_big_job_count(void) ++{ ++ return job_queue_gp.big_job_num; ++} ++ ++u32 mali_scheduler_job_physical_head_count(mali_bool gpu_mode_is_secure); ++ ++mali_bool mali_scheduler_job_next_is_virtual(void); ++struct mali_pp_job *mali_scheduler_job_pp_next(void); ++ ++struct mali_gp_job *mali_scheduler_job_gp_get(void); ++struct mali_pp_job *mali_scheduler_job_pp_physical_peek(void); ++struct mali_pp_job *mali_scheduler_job_pp_virtual_peek(void); ++struct mali_pp_job *mali_scheduler_job_pp_physical_get(u32 *sub_job); ++struct mali_pp_job *mali_scheduler_job_pp_virtual_get(void); ++ ++MALI_STATIC_INLINE u32 mali_scheduler_get_new_id(void) ++{ ++ return _mali_osk_atomic_inc_return(&mali_job_id_autonumber); ++} ++ ++MALI_STATIC_INLINE u32 mali_scheduler_get_new_cache_order(void) ++{ ++ return _mali_osk_atomic_inc_return(&mali_job_cache_order_autonumber); ++} ++ ++/** ++ * @brief Used by the Timeline system to queue a GP job. ++ * ++ * @note @ref mali_executor_schedule_from_mask() should be called if this ++ * function returns non-zero. ++ * ++ * @param job The GP job that is being activated. ++ * ++ * @return A scheduling bitmask that can be used to decide if scheduling is ++ * necessary after this call. ++ */ ++mali_scheduler_mask mali_scheduler_activate_gp_job(struct mali_gp_job *job); ++ ++/** ++ * @brief Used by the Timeline system to queue a PP job. ++ * ++ * @note @ref mali_executor_schedule_from_mask() should be called if this ++ * function returns non-zero. ++ * ++ * @param job The PP job that is being activated. ++ * ++ * @return A scheduling bitmask that can be used to decide if scheduling is ++ * necessary after this call. ++ */ ++mali_scheduler_mask mali_scheduler_activate_pp_job(struct mali_pp_job *job); ++ ++void mali_scheduler_complete_gp_job(struct mali_gp_job *job, ++ mali_bool success, ++ mali_bool user_notification, ++ mali_bool dequeued); ++ ++void mali_scheduler_complete_pp_job(struct mali_pp_job *job, ++ u32 num_cores_in_virtual, ++ mali_bool user_notification, ++ mali_bool dequeued); ++ ++void mali_scheduler_abort_session(struct mali_session_data *session); ++ ++void mali_scheduler_return_pp_job_to_user(struct mali_pp_job *job, ++ u32 num_cores_in_virtual); ++ ++#if MALI_STATE_TRACKING ++u32 mali_scheduler_dump_state(char *buf, u32 size); ++#endif ++ ++void mali_scheduler_gp_pp_job_queue_print(void); ++ ++#endif /* __MALI_SCHEDULER_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_scheduler_types.h b/drivers/gpu/arm/mali400/mali/common/mali_scheduler_types.h +new file mode 100755 +index 000000000000..ba1d71d01d46 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_scheduler_types.h +@@ -0,0 +1,29 @@ ++/* ++ * Copyright (C) 2013-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_SCHEDULER_TYPES_H__ ++#define __MALI_SCHEDULER_TYPES_H__ ++ ++#include "mali_osk.h" ++ ++#define MALI_SCHEDULER_JOB_ID_SPAN 65535 ++ ++/** ++ * Bitmask used for defered scheduling of subsystems. ++ */ ++typedef u32 mali_scheduler_mask; ++ ++#define MALI_SCHEDULER_MASK_GP (1<<0) ++#define MALI_SCHEDULER_MASK_PP (1<<1) ++ ++#define MALI_SCHEDULER_MASK_EMPTY 0 ++#define MALI_SCHEDULER_MASK_ALL (MALI_SCHEDULER_MASK_GP | MALI_SCHEDULER_MASK_PP) ++ ++#endif /* __MALI_SCHEDULER_TYPES_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_session.c b/drivers/gpu/arm/mali400/mali/common/mali_session.c +new file mode 100755 +index 000000000000..7504fb108779 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_session.c +@@ -0,0 +1,155 @@ ++/* ++ * Copyright (C) 2012-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include "mali_osk.h" ++#include "mali_osk_list.h" ++#include "mali_session.h" ++#include "mali_ukk.h" ++#ifdef MALI_MEM_SWAP_TRACKING ++#include "mali_memory_swap_alloc.h" ++#endif ++ ++_MALI_OSK_LIST_HEAD(mali_sessions); ++static u32 mali_session_count = 0; ++ ++_mali_osk_spinlock_irq_t *mali_sessions_lock = NULL; ++wait_queue_head_t pending_queue; ++ ++_mali_osk_errcode_t mali_session_initialize(void) ++{ ++ _MALI_OSK_INIT_LIST_HEAD(&mali_sessions); ++ /* init wait queue for big varying job */ ++ init_waitqueue_head(&pending_queue); ++ ++ mali_sessions_lock = _mali_osk_spinlock_irq_init( ++ _MALI_OSK_LOCKFLAG_ORDERED, ++ _MALI_OSK_LOCK_ORDER_SESSIONS); ++ if (NULL == mali_sessions_lock) { ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++void mali_session_terminate(void) ++{ ++ if (NULL != mali_sessions_lock) { ++ _mali_osk_spinlock_irq_term(mali_sessions_lock); ++ mali_sessions_lock = NULL; ++ } ++} ++ ++void mali_session_add(struct mali_session_data *session) ++{ ++ mali_session_lock(); ++ _mali_osk_list_add(&session->link, &mali_sessions); ++ mali_session_count++; ++ mali_session_unlock(); ++} ++ ++void mali_session_remove(struct mali_session_data *session) ++{ ++ mali_session_lock(); ++ _mali_osk_list_delinit(&session->link); ++ mali_session_count--; ++ mali_session_unlock(); ++} ++ ++u32 mali_session_get_count(void) ++{ ++ return mali_session_count; ++} ++ ++mali_bool mali_session_pp_job_is_empty(void *data) ++{ ++ struct mali_session_data *session = (struct mali_session_data *)data; ++ MALI_DEBUG_ASSERT_POINTER(session); ++ ++ if ( 0 == _mali_osk_atomic_read(&session->number_of_pp_jobs)) { ++ return MALI_TRUE; ++ } ++ return MALI_FALSE; ++} ++ ++wait_queue_head_t *mali_session_get_wait_queue(void) ++{ ++ return &pending_queue; ++} ++ ++/* ++ * Get the max completed window jobs from all active session, ++ * which will be used in window render frame per sec calculate ++ */ ++#if defined(CONFIG_MALI_DVFS) ++u32 mali_session_max_window_num(void) ++{ ++ struct mali_session_data *session, *tmp; ++ u32 max_window_num = 0; ++ u32 tmp_number = 0; ++ ++ mali_session_lock(); ++ ++ MALI_SESSION_FOREACH(session, tmp, link) { ++ tmp_number = _mali_osk_atomic_xchg( ++ &session->number_of_window_jobs, 0); ++ if (max_window_num < tmp_number) { ++ max_window_num = tmp_number; ++ } ++ } ++ ++ mali_session_unlock(); ++ ++ return max_window_num; ++} ++#endif ++ ++void mali_session_memory_tracking(_mali_osk_print_ctx *print_ctx) ++{ ++ struct mali_session_data *session, *tmp; ++ u32 mali_mem_usage; ++ u32 total_mali_mem_size; ++#ifdef MALI_MEM_SWAP_TRACKING ++ u32 swap_pool_size; ++ u32 swap_unlock_size; ++#endif ++ ++ MALI_DEBUG_ASSERT_POINTER(print_ctx); ++ mali_session_lock(); ++ MALI_SESSION_FOREACH(session, tmp, link) { ++#ifdef MALI_MEM_SWAP_TRACKING ++ _mali_osk_ctxprintf(print_ctx, " %-25s %-10u %-10u %-15u %-15u %-10u %-10u %-10u\n", ++ session->comm, session->pid, ++ (atomic_read(&session->mali_mem_allocated_pages)) * _MALI_OSK_MALI_PAGE_SIZE, ++ (unsigned int)session->max_mali_mem_allocated_size, ++ (unsigned int)((atomic_read(&session->mali_mem_array[MALI_MEM_EXTERNAL])) * _MALI_OSK_MALI_PAGE_SIZE), ++ (unsigned int)((atomic_read(&session->mali_mem_array[MALI_MEM_UMP])) * _MALI_OSK_MALI_PAGE_SIZE), ++ (unsigned int)((atomic_read(&session->mali_mem_array[MALI_MEM_DMA_BUF])) * _MALI_OSK_MALI_PAGE_SIZE), ++ (unsigned int)((atomic_read(&session->mali_mem_array[MALI_MEM_SWAP])) * _MALI_OSK_MALI_PAGE_SIZE) ++ ); ++#else ++ _mali_osk_ctxprintf(print_ctx, " %-25s %-10u %-10u %-15u %-15u %-10u %-10u \n", ++ session->comm, session->pid, ++ (unsigned int)((atomic_read(&session->mali_mem_allocated_pages)) * _MALI_OSK_MALI_PAGE_SIZE), ++ (unsigned int)session->max_mali_mem_allocated_size, ++ (unsigned int)((atomic_read(&session->mali_mem_array[MALI_MEM_EXTERNAL])) * _MALI_OSK_MALI_PAGE_SIZE), ++ (unsigned int)((atomic_read(&session->mali_mem_array[MALI_MEM_UMP])) * _MALI_OSK_MALI_PAGE_SIZE), ++ (unsigned int)((atomic_read(&session->mali_mem_array[MALI_MEM_DMA_BUF])) * _MALI_OSK_MALI_PAGE_SIZE) ++ ); ++#endif ++ } ++ mali_session_unlock(); ++ mali_mem_usage = _mali_ukk_report_memory_usage(); ++ total_mali_mem_size = _mali_ukk_report_total_memory_size(); ++ _mali_osk_ctxprintf(print_ctx, "Mali mem usage: %u\nMali mem limit: %u\n", mali_mem_usage, total_mali_mem_size); ++#ifdef MALI_MEM_SWAP_TRACKING ++ mali_mem_swap_tracking(&swap_pool_size, &swap_unlock_size); ++ _mali_osk_ctxprintf(print_ctx, "Mali swap mem pool : %u\nMali swap mem unlock: %u\n", swap_pool_size, swap_unlock_size); ++#endif ++} +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_session.h b/drivers/gpu/arm/mali400/mali/common/mali_session.h +new file mode 100755 +index 000000000000..da8b9927ee60 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_session.h +@@ -0,0 +1,136 @@ ++/* ++ * Copyright (C) 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_SESSION_H__ ++#define __MALI_SESSION_H__ ++ ++#include "mali_mmu_page_directory.h" ++#include "mali_osk.h" ++#include "mali_osk_list.h" ++#include "mali_memory_types.h" ++#include "mali_memory_manager.h" ++ ++struct mali_timeline_system; ++struct mali_soft_system; ++ ++/* Number of frame builder job lists per session. */ ++#define MALI_PP_JOB_FB_LOOKUP_LIST_SIZE 16 ++#define MALI_PP_JOB_FB_LOOKUP_LIST_MASK (MALI_PP_JOB_FB_LOOKUP_LIST_SIZE - 1) ++/*Max pending big job allowed in kernel*/ ++#define MALI_MAX_PENDING_BIG_JOB (2) ++ ++struct mali_session_data { ++ _mali_osk_notification_queue_t *ioctl_queue; ++ ++ _mali_osk_wait_queue_t *wait_queue; /**The wait queue to wait for the number of pp job become 0.*/ ++ ++ _mali_osk_mutex_t *memory_lock; /**< Lock protecting the vm manipulation */ ++ _mali_osk_mutex_t *cow_lock; /** < Lock protecting the cow memory free manipulation */ ++#if 0 ++ _mali_osk_list_t memory_head; /**< Track all the memory allocated in this session, for freeing on abnormal termination */ ++#endif ++ struct mali_page_directory *page_directory; /**< MMU page directory for this session */ ++ ++ _MALI_OSK_LIST_HEAD(link); /**< Link for list of all sessions */ ++ _MALI_OSK_LIST_HEAD(pp_job_list); /**< List of all PP jobs on this session */ ++ ++#if defined(CONFIG_MALI_DVFS) ++ _mali_osk_atomic_t number_of_window_jobs; /**< Record the window jobs completed on this session in a period */ ++#endif ++ _mali_osk_atomic_t number_of_pp_jobs; /** < Record the pp jobs on this session */ ++ ++ _mali_osk_list_t pp_job_fb_lookup_list[MALI_PP_JOB_FB_LOOKUP_LIST_SIZE]; /**< List of PP job lists per frame builder id. Used to link jobs from same frame builder. */ ++ struct mali_soft_job_system *soft_job_system; /**< Soft job system for this session. */ ++ struct mali_timeline_system *timeline_system; /**< Timeline system for this session. */ ++ ++ mali_bool is_aborting; /**< MALI_TRUE if the session is aborting, MALI_FALSE if not. */ ++ mali_bool use_high_priority_job_queue; /**< If MALI_TRUE, jobs added from this session will use the high priority job queues. */ ++ u32 pid; ++ char *comm; ++ atomic_t mali_mem_array[MALI_MEM_TYPE_MAX]; /**< The array to record mem types' usage for this session. */ ++ atomic_t mali_mem_allocated_pages; /** The current allocated mali memory pages, which include mali os memory and mali dedicated memory.*/ ++ size_t max_mali_mem_allocated_size; /**< The past max mali memory allocated size, which include mali os memory and mali dedicated memory. */ ++ /* Added for new memroy system */ ++ struct mali_allocation_manager allocation_mgr; ++ ++#if defined(CONFIG_MALI_DMA_BUF_FENCE) ++ u32 fence_context; /** < The execution dma fence context this fence is run on. */ ++ _mali_osk_atomic_t fence_seqno; /** < Alinear increasing sequence number for this dma fence context. */ ++#endif ++}; ++ ++_mali_osk_errcode_t mali_session_initialize(void); ++void mali_session_terminate(void); ++ ++/* List of all sessions. Actual list head in mali_kernel_core.c */ ++extern _mali_osk_list_t mali_sessions; ++/* Lock to protect modification and access to the mali_sessions list */ ++extern _mali_osk_spinlock_irq_t *mali_sessions_lock; ++ ++MALI_STATIC_INLINE void mali_session_lock(void) ++{ ++ _mali_osk_spinlock_irq_lock(mali_sessions_lock); ++} ++ ++MALI_STATIC_INLINE void mali_session_unlock(void) ++{ ++ _mali_osk_spinlock_irq_unlock(mali_sessions_lock); ++} ++ ++void mali_session_add(struct mali_session_data *session); ++void mali_session_remove(struct mali_session_data *session); ++u32 mali_session_get_count(void); ++mali_bool mali_session_pp_job_is_empty(void *data); ++wait_queue_head_t *mali_session_get_wait_queue(void); ++ ++#define MALI_SESSION_FOREACH(session, tmp, link) \ ++ _MALI_OSK_LIST_FOREACHENTRY(session, tmp, &mali_sessions, struct mali_session_data, link) ++ ++MALI_STATIC_INLINE struct mali_page_directory *mali_session_get_page_directory(struct mali_session_data *session) ++{ ++ return session->page_directory; ++} ++ ++MALI_STATIC_INLINE void mali_session_memory_lock(struct mali_session_data *session) ++{ ++ MALI_DEBUG_ASSERT_POINTER(session); ++ _mali_osk_mutex_wait(session->memory_lock); ++} ++ ++MALI_STATIC_INLINE void mali_session_memory_unlock(struct mali_session_data *session) ++{ ++ MALI_DEBUG_ASSERT_POINTER(session); ++ _mali_osk_mutex_signal(session->memory_lock); ++} ++ ++MALI_STATIC_INLINE void mali_session_send_notification(struct mali_session_data *session, _mali_osk_notification_t *object) ++{ ++ _mali_osk_notification_queue_send(session->ioctl_queue, object); ++} ++ ++#if defined(CONFIG_MALI_DVFS) ++ ++MALI_STATIC_INLINE void mali_session_inc_num_window_jobs(struct mali_session_data *session) ++{ ++ MALI_DEBUG_ASSERT_POINTER(session); ++ _mali_osk_atomic_inc(&session->number_of_window_jobs); ++} ++ ++/* ++ * Get the max completed window jobs from all active session, ++ * which will be used in window render frame per sec calculate ++ */ ++u32 mali_session_max_window_num(void); ++ ++#endif ++ ++void mali_session_memory_tracking(_mali_osk_print_ctx *print_ctx); ++ ++#endif /* __MALI_SESSION_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_soft_job.c b/drivers/gpu/arm/mali400/mali/common/mali_soft_job.c +new file mode 100755 +index 000000000000..35cd830bc83a +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_soft_job.c +@@ -0,0 +1,438 @@ ++/* ++ * Copyright (C) 2013-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include "mali_soft_job.h" ++#include "mali_osk.h" ++#include "mali_timeline.h" ++#include "mali_session.h" ++#include "mali_kernel_common.h" ++#include "mali_uk_types.h" ++#include "mali_scheduler.h" ++#include "mali_executor.h" ++ ++MALI_STATIC_INLINE void mali_soft_job_system_lock(struct mali_soft_job_system *system) ++{ ++ MALI_DEBUG_ASSERT_POINTER(system); ++ _mali_osk_spinlock_irq_lock(system->lock); ++ MALI_DEBUG_PRINT(5, ("Mali Soft Job: soft system %p lock taken\n", system)); ++ MALI_DEBUG_ASSERT(0 == system->lock_owner); ++ MALI_DEBUG_CODE(system->lock_owner = _mali_osk_get_tid()); ++} ++ ++MALI_STATIC_INLINE void mali_soft_job_system_unlock(struct mali_soft_job_system *system) ++{ ++ MALI_DEBUG_ASSERT_POINTER(system); ++ MALI_DEBUG_PRINT(5, ("Mali Soft Job: releasing soft system %p lock\n", system)); ++ MALI_DEBUG_ASSERT(_mali_osk_get_tid() == system->lock_owner); ++ MALI_DEBUG_CODE(system->lock_owner = 0); ++ _mali_osk_spinlock_irq_unlock(system->lock); ++} ++ ++#if defined(DEBUG) ++MALI_STATIC_INLINE void mali_soft_job_system_assert_locked(struct mali_soft_job_system *system) ++{ ++ MALI_DEBUG_ASSERT_POINTER(system); ++ MALI_DEBUG_ASSERT(_mali_osk_get_tid() == system->lock_owner); ++} ++#define MALI_ASSERT_SOFT_JOB_SYSTEM_LOCKED(system) mali_soft_job_system_assert_locked(system) ++#else ++#define MALI_ASSERT_SOFT_JOB_SYSTEM_LOCKED(system) ++#endif /* defined(DEBUG) */ ++ ++struct mali_soft_job_system *mali_soft_job_system_create(struct mali_session_data *session) ++{ ++ struct mali_soft_job_system *system; ++ ++ MALI_DEBUG_ASSERT_POINTER(session); ++ ++ system = (struct mali_soft_job_system *) _mali_osk_calloc(1, sizeof(struct mali_soft_job_system)); ++ if (NULL == system) { ++ return NULL; ++ } ++ ++ system->session = session; ++ ++ system->lock = _mali_osk_spinlock_irq_init(_MALI_OSK_LOCKFLAG_ORDERED, _MALI_OSK_LOCK_ORDER_SCHEDULER); ++ if (NULL == system->lock) { ++ mali_soft_job_system_destroy(system); ++ return NULL; ++ } ++ system->lock_owner = 0; ++ system->last_job_id = 0; ++ ++ _MALI_OSK_INIT_LIST_HEAD(&(system->jobs_used)); ++ ++ return system; ++} ++ ++void mali_soft_job_system_destroy(struct mali_soft_job_system *system) ++{ ++ MALI_DEBUG_ASSERT_POINTER(system); ++ ++ /* All jobs should be free at this point. */ ++ MALI_DEBUG_ASSERT(_mali_osk_list_empty(&(system->jobs_used))); ++ ++ if (NULL != system) { ++ if (NULL != system->lock) { ++ _mali_osk_spinlock_irq_term(system->lock); ++ } ++ _mali_osk_free(system); ++ } ++} ++ ++static void mali_soft_job_system_free_job(struct mali_soft_job_system *system, struct mali_soft_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ MALI_DEBUG_ASSERT_POINTER(system); ++ ++ mali_soft_job_system_lock(job->system); ++ ++ MALI_DEBUG_ASSERT(MALI_SOFT_JOB_INVALID_ID != job->id); ++ MALI_DEBUG_ASSERT(system == job->system); ++ ++ _mali_osk_list_del(&(job->system_list)); ++ ++ mali_soft_job_system_unlock(job->system); ++ ++ _mali_osk_free(job); ++} ++ ++MALI_STATIC_INLINE struct mali_soft_job *mali_soft_job_system_lookup_job(struct mali_soft_job_system *system, u32 job_id) ++{ ++ struct mali_soft_job *job, *tmp; ++ ++ MALI_DEBUG_ASSERT_POINTER(system); ++ MALI_ASSERT_SOFT_JOB_SYSTEM_LOCKED(system); ++ ++ _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &system->jobs_used, struct mali_soft_job, system_list) { ++ if (job->id == job_id) ++ return job; ++ } ++ ++ return NULL; ++} ++ ++void mali_soft_job_destroy(struct mali_soft_job *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ MALI_DEBUG_ASSERT_POINTER(job->system); ++ ++ MALI_DEBUG_PRINT(4, ("Mali Soft Job: destroying soft job %u (0x%08X)\n", job->id, job)); ++ ++ if (NULL != job) { ++ if (0 < _mali_osk_atomic_dec_return(&job->refcount)) return; ++ ++ _mali_osk_atomic_term(&job->refcount); ++ ++ if (NULL != job->activated_notification) { ++ _mali_osk_notification_delete(job->activated_notification); ++ job->activated_notification = NULL; ++ } ++ ++ mali_soft_job_system_free_job(job->system, job); ++ } ++} ++ ++struct mali_soft_job *mali_soft_job_create(struct mali_soft_job_system *system, mali_soft_job_type type, u64 user_job) ++{ ++ struct mali_soft_job *job; ++ _mali_osk_notification_t *notification = NULL; ++ ++ MALI_DEBUG_ASSERT_POINTER(system); ++ MALI_DEBUG_ASSERT((MALI_SOFT_JOB_TYPE_USER_SIGNALED == type) || ++ (MALI_SOFT_JOB_TYPE_SELF_SIGNALED == type)); ++ ++ notification = _mali_osk_notification_create(_MALI_NOTIFICATION_SOFT_ACTIVATED, sizeof(_mali_uk_soft_job_activated_s)); ++ if (unlikely(NULL == notification)) { ++ MALI_PRINT_ERROR(("Mali Soft Job: failed to allocate notification")); ++ return NULL; ++ } ++ ++ job = _mali_osk_malloc(sizeof(struct mali_soft_job)); ++ if (unlikely(NULL == job)) { ++ MALI_DEBUG_PRINT(2, ("Mali Soft Job: system alloc job failed. \n")); ++ return NULL; ++ } ++ ++ mali_soft_job_system_lock(system); ++ ++ job->system = system; ++ job->id = system->last_job_id++; ++ job->state = MALI_SOFT_JOB_STATE_ALLOCATED; ++ ++ _mali_osk_list_add(&(job->system_list), &(system->jobs_used)); ++ ++ job->type = type; ++ job->user_job = user_job; ++ job->activated = MALI_FALSE; ++ ++ job->activated_notification = notification; ++ ++ _mali_osk_atomic_init(&job->refcount, 1); ++ ++ MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_ALLOCATED == job->state); ++ MALI_DEBUG_ASSERT(system == job->system); ++ MALI_DEBUG_ASSERT(MALI_SOFT_JOB_INVALID_ID != job->id); ++ ++ mali_soft_job_system_unlock(system); ++ ++ return job; ++} ++ ++mali_timeline_point mali_soft_job_start(struct mali_soft_job *job, struct mali_timeline_fence *fence) ++{ ++ mali_timeline_point point; ++ struct mali_soft_job_system *system; ++ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ MALI_DEBUG_ASSERT_POINTER(fence); ++ ++ MALI_DEBUG_ASSERT_POINTER(job->system); ++ system = job->system; ++ ++ MALI_DEBUG_ASSERT_POINTER(system->session); ++ MALI_DEBUG_ASSERT_POINTER(system->session->timeline_system); ++ ++ mali_soft_job_system_lock(system); ++ ++ MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_ALLOCATED == job->state); ++ job->state = MALI_SOFT_JOB_STATE_STARTED; ++ ++ mali_soft_job_system_unlock(system); ++ ++ MALI_DEBUG_PRINT(4, ("Mali Soft Job: starting soft job %u (0x%08X)\n", job->id, job)); ++ ++ mali_timeline_tracker_init(&job->tracker, MALI_TIMELINE_TRACKER_SOFT, fence, job); ++ point = mali_timeline_system_add_tracker(system->session->timeline_system, &job->tracker, MALI_TIMELINE_SOFT); ++ ++ return point; ++} ++ ++static mali_bool mali_soft_job_is_activated(void *data) ++{ ++ struct mali_soft_job *job; ++ ++ job = (struct mali_soft_job *) data; ++ MALI_DEBUG_ASSERT_POINTER(job); ++ ++ return job->activated; ++} ++ ++_mali_osk_errcode_t mali_soft_job_system_signal_job(struct mali_soft_job_system *system, u32 job_id) ++{ ++ struct mali_soft_job *job; ++ struct mali_timeline_system *timeline_system; ++ mali_scheduler_mask schedule_mask; ++ ++ MALI_DEBUG_ASSERT_POINTER(system); ++ ++ mali_soft_job_system_lock(system); ++ ++ job = mali_soft_job_system_lookup_job(system, job_id); ++ ++ if ((NULL == job) || (MALI_SOFT_JOB_TYPE_USER_SIGNALED != job->type) ++ || !(MALI_SOFT_JOB_STATE_STARTED == job->state || MALI_SOFT_JOB_STATE_TIMED_OUT == job->state)) { ++ mali_soft_job_system_unlock(system); ++ MALI_PRINT_ERROR(("Mali Soft Job: invalid soft job id %u", job_id)); ++ return _MALI_OSK_ERR_ITEM_NOT_FOUND; ++ } ++ ++ if (MALI_SOFT_JOB_STATE_TIMED_OUT == job->state) { ++ job->state = MALI_SOFT_JOB_STATE_SIGNALED; ++ mali_soft_job_system_unlock(system); ++ ++ MALI_DEBUG_ASSERT(MALI_TRUE == job->activated); ++ MALI_DEBUG_PRINT(4, ("Mali Soft Job: soft job %u (0x%08X) was timed out\n", job->id, job)); ++ mali_soft_job_destroy(job); ++ ++ return _MALI_OSK_ERR_TIMEOUT; ++ } ++ ++ MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_STARTED == job->state); ++ ++ job->state = MALI_SOFT_JOB_STATE_SIGNALED; ++ mali_soft_job_system_unlock(system); ++ ++ /* Since the job now is in signaled state, timeouts from the timeline system will be ++ * ignored, and it is not possible to signal this job again. */ ++ ++ timeline_system = system->session->timeline_system; ++ MALI_DEBUG_ASSERT_POINTER(timeline_system); ++ ++ /* Wait until activated. */ ++ _mali_osk_wait_queue_wait_event(timeline_system->wait_queue, mali_soft_job_is_activated, (void *) job); ++ ++ MALI_DEBUG_PRINT(4, ("Mali Soft Job: signaling soft job %u (0x%08X)\n", job->id, job)); ++ ++ schedule_mask = mali_timeline_tracker_release(&job->tracker); ++ mali_executor_schedule_from_mask(schedule_mask, MALI_FALSE); ++ ++ mali_soft_job_destroy(job); ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++static void mali_soft_job_send_activated_notification(struct mali_soft_job *job) ++{ ++ if (NULL != job->activated_notification) { ++ _mali_uk_soft_job_activated_s *res = job->activated_notification->result_buffer; ++ res->user_job = job->user_job; ++ mali_session_send_notification(job->system->session, job->activated_notification); ++ } ++ job->activated_notification = NULL; ++} ++ ++mali_scheduler_mask mali_soft_job_system_activate_job(struct mali_soft_job *job) ++{ ++ mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; ++ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ MALI_DEBUG_ASSERT_POINTER(job->system); ++ MALI_DEBUG_ASSERT_POINTER(job->system->session); ++ ++ MALI_DEBUG_PRINT(4, ("Mali Soft Job: Timeline activation for soft job %u (0x%08X).\n", job->id, job)); ++ ++ mali_soft_job_system_lock(job->system); ++ ++ if (unlikely(job->system->session->is_aborting)) { ++ MALI_DEBUG_PRINT(3, ("Mali Soft Job: Soft job %u (0x%08X) activated while session is aborting.\n", job->id, job)); ++ ++ mali_soft_job_system_unlock(job->system); ++ ++ /* Since we are in shutdown, we can ignore the scheduling bitmask. */ ++ mali_timeline_tracker_release(&job->tracker); ++ mali_soft_job_destroy(job); ++ return schedule_mask; ++ } ++ ++ /* Send activated notification. */ ++ mali_soft_job_send_activated_notification(job); ++ ++ /* Wake up sleeping signaler. */ ++ job->activated = MALI_TRUE; ++ ++ /* If job type is self signaled, release tracker, move soft job to free list, and scheduler at once */ ++ if (MALI_SOFT_JOB_TYPE_SELF_SIGNALED == job->type) { ++ MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_STARTED == job->state); ++ ++ job->state = MALI_SOFT_JOB_STATE_SIGNALED; ++ mali_soft_job_system_unlock(job->system); ++ ++ schedule_mask |= mali_timeline_tracker_release(&job->tracker); ++ ++ mali_soft_job_destroy(job); ++ } else { ++ _mali_osk_wait_queue_wake_up(job->tracker.system->wait_queue); ++ ++ mali_soft_job_system_unlock(job->system); ++ } ++ ++ return schedule_mask; ++} ++ ++mali_scheduler_mask mali_soft_job_system_timeout_job(struct mali_soft_job *job) ++{ ++ mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; ++ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ MALI_DEBUG_ASSERT_POINTER(job->system); ++ MALI_DEBUG_ASSERT_POINTER(job->system->session); ++ MALI_DEBUG_ASSERT(MALI_TRUE == job->activated); ++ ++ MALI_DEBUG_PRINT(4, ("Mali Soft Job: Timeline timeout for soft job %u (0x%08X).\n", job->id, job)); ++ ++ mali_soft_job_system_lock(job->system); ++ ++ MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_STARTED == job->state || ++ MALI_SOFT_JOB_STATE_SIGNALED == job->state); ++ ++ if (unlikely(job->system->session->is_aborting)) { ++ /* The session is aborting. This job will be released and destroyed by @ref ++ * mali_soft_job_system_abort(). */ ++ mali_soft_job_system_unlock(job->system); ++ ++ return MALI_SCHEDULER_MASK_EMPTY; ++ } ++ ++ if (MALI_SOFT_JOB_STATE_STARTED != job->state) { ++ MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_SIGNALED == job->state); ++ ++ /* The job is about to be signaled, ignore timeout. */ ++ MALI_DEBUG_PRINT(4, ("Mali Soft Job: Timeout on soft job %u (0x%08X) in signaled state.\n", job->id, job)); ++ mali_soft_job_system_unlock(job->system); ++ return schedule_mask; ++ } ++ ++ MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_STARTED == job->state); ++ ++ job->state = MALI_SOFT_JOB_STATE_TIMED_OUT; ++ _mali_osk_atomic_inc(&job->refcount); ++ ++ mali_soft_job_system_unlock(job->system); ++ ++ schedule_mask = mali_timeline_tracker_release(&job->tracker); ++ ++ mali_soft_job_destroy(job); ++ ++ return schedule_mask; ++} ++ ++void mali_soft_job_system_abort(struct mali_soft_job_system *system) ++{ ++ struct mali_soft_job *job, *tmp; ++ _MALI_OSK_LIST_HEAD_STATIC_INIT(jobs); ++ ++ MALI_DEBUG_ASSERT_POINTER(system); ++ MALI_DEBUG_ASSERT_POINTER(system->session); ++ MALI_DEBUG_ASSERT(system->session->is_aborting); ++ ++ MALI_DEBUG_PRINT(3, ("Mali Soft Job: Aborting soft job system for session 0x%08X.\n", system->session)); ++ ++ mali_soft_job_system_lock(system); ++ ++ _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &system->jobs_used, struct mali_soft_job, system_list) { ++ MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_STARTED == job->state || ++ MALI_SOFT_JOB_STATE_TIMED_OUT == job->state); ++ ++ if (MALI_SOFT_JOB_STATE_STARTED == job->state) { ++ /* If the job has been activated, we have to release the tracker and destroy ++ * the job. If not, the tracker will be released and the job destroyed when ++ * it is activated. */ ++ if (MALI_TRUE == job->activated) { ++ MALI_DEBUG_PRINT(3, ("Mali Soft Job: Aborting unsignaled soft job %u (0x%08X).\n", job->id, job)); ++ ++ job->state = MALI_SOFT_JOB_STATE_SIGNALED; ++ _mali_osk_list_move(&job->system_list, &jobs); ++ } ++ } else if (MALI_SOFT_JOB_STATE_TIMED_OUT == job->state) { ++ MALI_DEBUG_PRINT(3, ("Mali Soft Job: Aborting timed out soft job %u (0x%08X).\n", job->id, job)); ++ ++ /* We need to destroy this soft job. */ ++ _mali_osk_list_move(&job->system_list, &jobs); ++ } ++ } ++ ++ mali_soft_job_system_unlock(system); ++ ++ /* Release and destroy jobs. */ ++ _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &jobs, struct mali_soft_job, system_list) { ++ MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_SIGNALED == job->state || ++ MALI_SOFT_JOB_STATE_TIMED_OUT == job->state); ++ ++ if (MALI_SOFT_JOB_STATE_SIGNALED == job->state) { ++ mali_timeline_tracker_release(&job->tracker); ++ } ++ ++ /* Move job back to used list before destroying. */ ++ _mali_osk_list_move(&job->system_list, &system->jobs_used); ++ ++ mali_soft_job_destroy(job); ++ } ++} +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_soft_job.h b/drivers/gpu/arm/mali400/mali/common/mali_soft_job.h +new file mode 100755 +index 000000000000..018ef4c527d9 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_soft_job.h +@@ -0,0 +1,190 @@ ++/* ++ * Copyright (C) 2013-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_SOFT_JOB_H__ ++#define __MALI_SOFT_JOB_H__ ++ ++#include "mali_osk.h" ++ ++#include "mali_timeline.h" ++ ++struct mali_timeline_fence; ++struct mali_session_data; ++struct mali_soft_job; ++struct mali_soft_job_system; ++ ++/** ++ * Soft job types. ++ * ++ * Soft jobs of type MALI_SOFT_JOB_TYPE_USER_SIGNALED will only complete after activation if either ++ * they are signaled by user-space (@ref mali_soft_job_system_signaled_job) or if they are timed out ++ * by the Timeline system. ++ * Soft jobs of type MALI_SOFT_JOB_TYPE_SELF_SIGNALED will release job resource automatically ++ * in kernel when the job is activated. ++ */ ++typedef enum mali_soft_job_type { ++ MALI_SOFT_JOB_TYPE_SELF_SIGNALED, ++ MALI_SOFT_JOB_TYPE_USER_SIGNALED, ++} mali_soft_job_type; ++ ++/** ++ * Soft job state. ++ * ++ * mali_soft_job_system_start_job a job will first be allocated.The job's state set to MALI_SOFT_JOB_STATE_ALLOCATED. ++ * Once the job is added to the timeline system, the state changes to MALI_SOFT_JOB_STATE_STARTED. ++ * ++ * For soft jobs of type MALI_SOFT_JOB_TYPE_USER_SIGNALED the state is changed to ++ * MALI_SOFT_JOB_STATE_SIGNALED when @ref mali_soft_job_system_signal_job is called and the soft ++ * job's state is MALI_SOFT_JOB_STATE_STARTED or MALI_SOFT_JOB_STATE_TIMED_OUT. ++ * ++ * If a soft job of type MALI_SOFT_JOB_TYPE_USER_SIGNALED is timed out before being signaled, the ++ * state is changed to MALI_SOFT_JOB_STATE_TIMED_OUT. This can only happen to soft jobs in state ++ * MALI_SOFT_JOB_STATE_STARTED. ++ * ++ */ ++typedef enum mali_soft_job_state { ++ MALI_SOFT_JOB_STATE_ALLOCATED, ++ MALI_SOFT_JOB_STATE_STARTED, ++ MALI_SOFT_JOB_STATE_SIGNALED, ++ MALI_SOFT_JOB_STATE_TIMED_OUT, ++} mali_soft_job_state; ++ ++#define MALI_SOFT_JOB_INVALID_ID ((u32) -1) ++ ++/** ++ * Soft job struct. ++ * ++ * Soft job can be used to represent any kind of CPU work done in kernel-space. ++ */ ++typedef struct mali_soft_job { ++ mali_soft_job_type type; /**< Soft job type. Must be one of MALI_SOFT_JOB_TYPE_*. */ ++ u64 user_job; /**< Identifier for soft job in user space. */ ++ _mali_osk_atomic_t refcount; /**< Soft jobs are reference counted to prevent premature deletion. */ ++ struct mali_timeline_tracker tracker; /**< Timeline tracker for soft job. */ ++ mali_bool activated; /**< MALI_TRUE if the job has been activated, MALI_FALSE if not. */ ++ _mali_osk_notification_t *activated_notification; /**< Pre-allocated notification object for ACTIVATED_NOTIFICATION. */ ++ ++ /* Protected by soft job system lock. */ ++ u32 id; /**< Used by user-space to find corresponding soft job in kernel-space. */ ++ mali_soft_job_state state; /**< State of soft job, must be one of MALI_SOFT_JOB_STATE_*. */ ++ struct mali_soft_job_system *system; /**< The soft job system this job is in. */ ++ _mali_osk_list_t system_list; /**< List element used by soft job system. */ ++} mali_soft_job; ++ ++/** ++ * Per-session soft job system. ++ * ++ * The soft job system is used to manage all soft jobs that belongs to a session. ++ */ ++typedef struct mali_soft_job_system { ++ struct mali_session_data *session; /**< The session this soft job system belongs to. */ ++ _MALI_OSK_LIST_HEAD(jobs_used); /**< List of all allocated soft jobs. */ ++ ++ _mali_osk_spinlock_irq_t *lock; /**< Lock used to protect soft job system and its soft jobs. */ ++ u32 lock_owner; /**< Contains tid of thread that locked the system or 0, if not locked. */ ++ u32 last_job_id; /**< Recored the last job id protected by lock. */ ++} mali_soft_job_system; ++ ++/** ++ * Create a soft job system. ++ * ++ * @param session The session this soft job system will belong to. ++ * @return The new soft job system, or NULL if unsuccessful. ++ */ ++struct mali_soft_job_system *mali_soft_job_system_create(struct mali_session_data *session); ++ ++/** ++ * Destroy a soft job system. ++ * ++ * @note The soft job must not have any started or activated jobs. Call @ref ++ * mali_soft_job_system_abort first. ++ * ++ * @param system The soft job system we are destroying. ++ */ ++void mali_soft_job_system_destroy(struct mali_soft_job_system *system); ++ ++/** ++ * Create a soft job. ++ * ++ * @param system Soft job system to create soft job from. ++ * @param type Type of the soft job. ++ * @param user_job Identifier for soft job in user space. ++ * @return New soft job if successful, NULL if not. ++ */ ++struct mali_soft_job *mali_soft_job_create(struct mali_soft_job_system *system, mali_soft_job_type type, u64 user_job); ++ ++/** ++ * Destroy soft job. ++ * ++ * @param job Soft job to destroy. ++ */ ++void mali_soft_job_destroy(struct mali_soft_job *job); ++ ++/** ++ * Start a soft job. ++ * ++ * The soft job will be added to the Timeline system which will then activate it after all ++ * dependencies have been resolved. ++ * ++ * Create soft jobs with @ref mali_soft_job_create before starting them. ++ * ++ * @param job Soft job to start. ++ * @param fence Fence representing dependencies for this soft job. ++ * @return Point on soft job timeline. ++ */ ++mali_timeline_point mali_soft_job_start(struct mali_soft_job *job, struct mali_timeline_fence *fence); ++ ++/** ++ * Use by user-space to signal that a soft job has completed. ++ * ++ * @note Only valid for soft jobs with type MALI_SOFT_JOB_TYPE_USER_SIGNALED. ++ * ++ * @note The soft job must be in state MALI_SOFT_JOB_STATE_STARTED for the signal to be successful. ++ * ++ * @note If the soft job was signaled successfully, or it received a time out, the soft job will be ++ * destroyed after this call and should no longer be used. ++ * ++ * @note This function will block until the soft job has been activated. ++ * ++ * @param system The soft job system the job was started in. ++ * @param job_id ID of soft job we are signaling. ++ * ++ * @return _MALI_OSK_ERR_ITEM_NOT_FOUND if the soft job ID was invalid, _MALI_OSK_ERR_TIMEOUT if the ++ * soft job was timed out or _MALI_OSK_ERR_OK if we successfully signaled the soft job. ++ */ ++_mali_osk_errcode_t mali_soft_job_system_signal_job(struct mali_soft_job_system *system, u32 job_id); ++ ++/** ++ * Used by the Timeline system to activate a soft job. ++ * ++ * @param job The soft job that is being activated. ++ * @return A scheduling bitmask. ++ */ ++mali_scheduler_mask mali_soft_job_system_activate_job(struct mali_soft_job *job); ++ ++/** ++ * Used by the Timeline system to timeout a soft job. ++ * ++ * A soft job is timed out if it completes or is signaled later than MALI_TIMELINE_TIMEOUT_HZ after ++ * activation. ++ * ++ * @param job The soft job that is being timed out. ++ * @return A scheduling bitmask. ++ */ ++mali_scheduler_mask mali_soft_job_system_timeout_job(struct mali_soft_job *job); ++ ++/** ++ * Used to cleanup activated soft jobs in the soft job system on session abort. ++ * ++ * @param system The soft job system that is being aborted. ++ */ ++void mali_soft_job_system_abort(struct mali_soft_job_system *system); ++ ++#endif /* __MALI_SOFT_JOB_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_spinlock_reentrant.c b/drivers/gpu/arm/mali400/mali/common/mali_spinlock_reentrant.c +new file mode 100755 +index 000000000000..f829e99f02ab +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_spinlock_reentrant.c +@@ -0,0 +1,77 @@ ++/* ++ * Copyright (C) 2013, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include "mali_spinlock_reentrant.h" ++ ++#include "mali_osk.h" ++#include "mali_kernel_common.h" ++ ++struct mali_spinlock_reentrant *mali_spinlock_reentrant_init(_mali_osk_lock_order_t lock_order) ++{ ++ struct mali_spinlock_reentrant *spinlock; ++ ++ spinlock = _mali_osk_calloc(1, sizeof(struct mali_spinlock_reentrant)); ++ if (NULL == spinlock) { ++ return NULL; ++ } ++ ++ spinlock->lock = _mali_osk_spinlock_irq_init(_MALI_OSK_LOCKFLAG_ORDERED, lock_order); ++ if (NULL == spinlock->lock) { ++ mali_spinlock_reentrant_term(spinlock); ++ return NULL; ++ } ++ ++ return spinlock; ++} ++ ++void mali_spinlock_reentrant_term(struct mali_spinlock_reentrant *spinlock) ++{ ++ MALI_DEBUG_ASSERT_POINTER(spinlock); ++ MALI_DEBUG_ASSERT(0 == spinlock->counter && 0 == spinlock->owner); ++ ++ if (NULL != spinlock->lock) { ++ _mali_osk_spinlock_irq_term(spinlock->lock); ++ } ++ ++ _mali_osk_free(spinlock); ++} ++ ++void mali_spinlock_reentrant_wait(struct mali_spinlock_reentrant *spinlock, u32 tid) ++{ ++ MALI_DEBUG_ASSERT_POINTER(spinlock); ++ MALI_DEBUG_ASSERT_POINTER(spinlock->lock); ++ MALI_DEBUG_ASSERT(0 != tid); ++ ++ MALI_DEBUG_PRINT(5, ("%s ^\n", __FUNCTION__)); ++ ++ if (tid != spinlock->owner) { ++ _mali_osk_spinlock_irq_lock(spinlock->lock); ++ MALI_DEBUG_ASSERT(0 == spinlock->owner && 0 == spinlock->counter); ++ spinlock->owner = tid; ++ } ++ ++ MALI_DEBUG_PRINT(5, ("%s v\n", __FUNCTION__)); ++ ++ ++spinlock->counter; ++} ++ ++void mali_spinlock_reentrant_signal(struct mali_spinlock_reentrant *spinlock, u32 tid) ++{ ++ MALI_DEBUG_ASSERT_POINTER(spinlock); ++ MALI_DEBUG_ASSERT_POINTER(spinlock->lock); ++ MALI_DEBUG_ASSERT(0 != tid && tid == spinlock->owner); ++ ++ --spinlock->counter; ++ if (0 == spinlock->counter) { ++ spinlock->owner = 0; ++ MALI_DEBUG_PRINT(5, ("%s release last\n", __FUNCTION__)); ++ _mali_osk_spinlock_irq_unlock(spinlock->lock); ++ } ++} +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_spinlock_reentrant.h b/drivers/gpu/arm/mali400/mali/common/mali_spinlock_reentrant.h +new file mode 100755 +index 000000000000..4d788ec1bbe4 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_spinlock_reentrant.h +@@ -0,0 +1,70 @@ ++/* ++ * Copyright (C) 2013, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_SPINLOCK_REENTRANT_H__ ++#define __MALI_SPINLOCK_REENTRANT_H__ ++ ++#include "mali_osk.h" ++#include "mali_kernel_common.h" ++ ++/** ++ * Reentrant spinlock. ++ */ ++struct mali_spinlock_reentrant { ++ _mali_osk_spinlock_irq_t *lock; ++ u32 owner; ++ u32 counter; ++}; ++ ++/** ++ * Create a new reentrant spinlock. ++ * ++ * @param lock_order Lock order. ++ * @return New reentrant spinlock. ++ */ ++struct mali_spinlock_reentrant *mali_spinlock_reentrant_init(_mali_osk_lock_order_t lock_order); ++ ++/** ++ * Terminate reentrant spinlock and free any associated resources. ++ * ++ * @param spinlock Reentrant spinlock to terminate. ++ */ ++void mali_spinlock_reentrant_term(struct mali_spinlock_reentrant *spinlock); ++ ++/** ++ * Wait for reentrant spinlock to be signaled. ++ * ++ * @param spinlock Reentrant spinlock. ++ * @param tid Thread ID. ++ */ ++void mali_spinlock_reentrant_wait(struct mali_spinlock_reentrant *spinlock, u32 tid); ++ ++/** ++ * Signal reentrant spinlock. ++ * ++ * @param spinlock Reentrant spinlock. ++ * @param tid Thread ID. ++ */ ++void mali_spinlock_reentrant_signal(struct mali_spinlock_reentrant *spinlock, u32 tid); ++ ++/** ++ * Check if thread is holding reentrant spinlock. ++ * ++ * @param spinlock Reentrant spinlock. ++ * @param tid Thread ID. ++ * @return MALI_TRUE if thread is holding spinlock, MALI_FALSE if not. ++ */ ++MALI_STATIC_INLINE mali_bool mali_spinlock_reentrant_is_held(struct mali_spinlock_reentrant *spinlock, u32 tid) ++{ ++ MALI_DEBUG_ASSERT_POINTER(spinlock->lock); ++ return (tid == spinlock->owner && 0 < spinlock->counter); ++} ++ ++#endif /* __MALI_SPINLOCK_REENTRANT_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_timeline.c b/drivers/gpu/arm/mali400/mali/common/mali_timeline.c +new file mode 100755 +index 000000000000..ffffee9306ce +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_timeline.c +@@ -0,0 +1,1964 @@ ++/* ++ * Copyright (C) 2013-2018 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++#include ++#include "mali_timeline.h" ++#include "mali_kernel_common.h" ++#include "mali_scheduler.h" ++#include "mali_soft_job.h" ++#include "mali_timeline_fence_wait.h" ++#include "mali_timeline_sync_fence.h" ++#include "mali_executor.h" ++#include "mali_pp_job.h" ++ ++#define MALI_TIMELINE_SYSTEM_LOCKED(system) (mali_spinlock_reentrant_is_held((system)->spinlock, _mali_osk_get_tid())) ++ ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++_mali_osk_wq_work_t *sync_fence_callback_work_t = NULL; ++_mali_osk_spinlock_irq_t *sync_fence_callback_list_lock = NULL; ++static _MALI_OSK_LIST_HEAD_STATIC_INIT(sync_fence_callback_queue); ++#endif ++ ++/* ++ * Following three elements are used to record how many ++ * gp, physical pp or virtual pp jobs are delayed in the whole ++ * timeline system, we can use these three value to decide ++ * if need to deactivate idle group. ++ */ ++_mali_osk_atomic_t gp_tracker_count; ++_mali_osk_atomic_t phy_pp_tracker_count; ++_mali_osk_atomic_t virt_pp_tracker_count; ++ ++static mali_scheduler_mask mali_timeline_system_release_waiter(struct mali_timeline_system *system, ++ struct mali_timeline_waiter *waiter); ++ ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++#include ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0) ++#include ++#include ++#include ++ ++struct mali_deferred_fence_put_entry { ++ struct hlist_node list; ++ struct sync_fence *fence; ++}; ++ ++static HLIST_HEAD(mali_timeline_sync_fence_to_free_list); ++static DEFINE_SPINLOCK(mali_timeline_sync_fence_to_free_lock); ++ ++static void put_sync_fences(struct work_struct *ignore) ++{ ++ struct hlist_head list; ++ struct hlist_node *tmp, *pos; ++ unsigned long flags; ++ struct mali_deferred_fence_put_entry *o; ++ ++ spin_lock_irqsave(&mali_timeline_sync_fence_to_free_lock, flags); ++ hlist_move_list(&mali_timeline_sync_fence_to_free_list, &list); ++ spin_unlock_irqrestore(&mali_timeline_sync_fence_to_free_lock, flags); ++ ++ hlist_for_each_entry_safe(o, pos, tmp, &list, list) { ++ sync_fence_put(o->fence); ++ kfree(o); ++ } ++} ++ ++static DECLARE_DELAYED_WORK(delayed_sync_fence_put, put_sync_fences); ++#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0) */ ++ ++/* Callback that is called when a sync fence a tracker is waiting on is signaled. */ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) ++static void mali_timeline_sync_fence_callback(struct sync_fence *sync_fence, struct sync_fence_waiter *sync_fence_waiter) ++#else ++static void mali_timeline_sync_fence_callback(struct mali_internal_sync_fence *sync_fence, struct mali_internal_sync_fence_waiter *sync_fence_waiter) ++#endif ++{ ++ struct mali_timeline_tracker *tracker; ++ ++ MALI_IGNORE(sync_fence); ++ MALI_DEBUG_ASSERT_POINTER(sync_fence_waiter); ++ ++ tracker = _MALI_OSK_CONTAINER_OF(sync_fence_waiter, struct mali_timeline_tracker, sync_fence_waiter); ++ MALI_DEBUG_ASSERT_POINTER(tracker); ++ ++ _mali_osk_spinlock_irq_lock(sync_fence_callback_list_lock); ++ _mali_osk_list_addtail(&tracker->sync_fence_signal_list, &sync_fence_callback_queue); ++ _mali_osk_spinlock_irq_unlock(sync_fence_callback_list_lock); ++ ++ _mali_osk_wq_schedule_work(sync_fence_callback_work_t); ++} ++#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ ++ ++static mali_scheduler_mask mali_timeline_tracker_time_out(struct mali_timeline_tracker *tracker) ++{ ++ MALI_DEBUG_ASSERT_POINTER(tracker); ++ MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_SOFT == tracker->type); ++ ++ return mali_soft_job_system_timeout_job((struct mali_soft_job *) tracker->job); ++} ++ ++static void mali_timeline_timer_callback(void *data) ++{ ++ struct mali_timeline_system *system; ++ struct mali_timeline_tracker *tracker; ++ struct mali_timeline *timeline; ++ mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; ++ u32 tid = _mali_osk_get_tid(); ++ ++ timeline = (struct mali_timeline *) data; ++ MALI_DEBUG_ASSERT_POINTER(timeline); ++ ++ system = timeline->system; ++ MALI_DEBUG_ASSERT_POINTER(system); ++ ++ mali_spinlock_reentrant_wait(system->spinlock, tid); ++ ++ if (!system->timer_enabled) { ++ mali_spinlock_reentrant_signal(system->spinlock, tid); ++ return; ++ } ++ ++ tracker = timeline->tracker_tail; ++ timeline->timer_active = MALI_FALSE; ++ ++ if (NULL != tracker && MALI_TRUE == tracker->timer_active) { ++ /* This is likely the delayed work that has been schedule out before cancelled. */ ++ if (MALI_TIMELINE_TIMEOUT_HZ > (_mali_osk_time_tickcount() - tracker->os_tick_activate)) { ++ mali_spinlock_reentrant_signal(system->spinlock, tid); ++ return; ++ } ++ ++ schedule_mask = mali_timeline_tracker_time_out(tracker); ++ tracker->timer_active = MALI_FALSE; ++ } else { ++ MALI_PRINT_ERROR(("Mali Timeline: Soft job timer callback without a waiting tracker.\n")); ++ } ++ ++ mali_spinlock_reentrant_signal(system->spinlock, tid); ++ ++ mali_executor_schedule_from_mask(schedule_mask, MALI_FALSE); ++} ++ ++void mali_timeline_system_stop_timer(struct mali_timeline_system *system) ++{ ++ u32 i; ++ u32 tid = _mali_osk_get_tid(); ++ ++ MALI_DEBUG_ASSERT_POINTER(system); ++ ++ mali_spinlock_reentrant_wait(system->spinlock, tid); ++ system->timer_enabled = MALI_FALSE; ++ mali_spinlock_reentrant_signal(system->spinlock, tid); ++ ++ for (i = 0; i < MALI_TIMELINE_MAX; ++i) { ++ struct mali_timeline *timeline = system->timelines[i]; ++ ++ MALI_DEBUG_ASSERT_POINTER(timeline); ++ ++ if (NULL != timeline->delayed_work) { ++ _mali_osk_wq_delayed_cancel_work_sync(timeline->delayed_work); ++ timeline->timer_active = MALI_FALSE; ++ } ++ } ++} ++ ++static void mali_timeline_destroy(struct mali_timeline *timeline) ++{ ++ MALI_DEBUG_ASSERT_POINTER(timeline); ++ if (NULL != timeline) { ++ /* Assert that the timeline object has been properly cleaned up before destroying it. */ ++ MALI_DEBUG_ASSERT(timeline->point_oldest == timeline->point_next); ++ MALI_DEBUG_ASSERT(NULL == timeline->tracker_head); ++ MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail); ++ MALI_DEBUG_ASSERT(NULL == timeline->waiter_head); ++ MALI_DEBUG_ASSERT(NULL == timeline->waiter_tail); ++ MALI_DEBUG_ASSERT(NULL != timeline->system); ++ MALI_DEBUG_ASSERT(MALI_TIMELINE_MAX > timeline->id); ++ ++ if (NULL != timeline->delayed_work) { ++ _mali_osk_wq_delayed_cancel_work_sync(timeline->delayed_work); ++ _mali_osk_wq_delayed_delete_work_nonflush(timeline->delayed_work); ++ } ++ ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ if (NULL != timeline->sync_tl) { ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) ++ sync_timeline_destroy(timeline->sync_tl); ++#else ++ mali_internal_sync_timeline_destroy(timeline->sync_tl); ++#endif ++ } ++#else ++ _mali_osk_free(timeline); ++#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ ++ } ++} ++ ++static struct mali_timeline *mali_timeline_create(struct mali_timeline_system *system, enum mali_timeline_id id) ++{ ++ struct mali_timeline *timeline; ++ ++ MALI_DEBUG_ASSERT_POINTER(system); ++ MALI_DEBUG_ASSERT(id < MALI_TIMELINE_MAX); ++ ++ timeline = (struct mali_timeline *) _mali_osk_calloc(1, sizeof(struct mali_timeline)); ++ if (NULL == timeline) { ++ return NULL; ++ } ++ ++ /* Initially the timeline is empty. */ ++#if defined(MALI_TIMELINE_DEBUG_START_POINT) ++ /* Start the timeline a bit before wrapping when debugging. */ ++ timeline->point_next = UINT_MAX - MALI_TIMELINE_MAX_POINT_SPAN - 128; ++#else ++ timeline->point_next = 1; ++#endif ++ timeline->point_oldest = timeline->point_next; ++ ++ /* The tracker and waiter lists will initially be empty. */ ++ ++ timeline->system = system; ++ timeline->id = id; ++ ++ timeline->delayed_work = _mali_osk_wq_delayed_create_work(mali_timeline_timer_callback, timeline); ++ if (NULL == timeline->delayed_work) { ++ mali_timeline_destroy(timeline); ++ return NULL; ++ } ++ ++ timeline->timer_active = MALI_FALSE; ++ ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ { ++ char timeline_name[32]; ++ ++ switch (id) { ++ case MALI_TIMELINE_GP: ++ _mali_osk_snprintf(timeline_name, 32, "mali-%u-gp", _mali_osk_get_pid()); ++ break; ++ case MALI_TIMELINE_PP: ++ _mali_osk_snprintf(timeline_name, 32, "mali-%u-pp", _mali_osk_get_pid()); ++ break; ++ case MALI_TIMELINE_SOFT: ++ _mali_osk_snprintf(timeline_name, 32, "mali-%u-soft", _mali_osk_get_pid()); ++ break; ++ default: ++ MALI_PRINT_ERROR(("Mali Timeline: Invalid timeline id %d\n", id)); ++ mali_timeline_destroy(timeline); ++ return NULL; ++ } ++ ++ timeline->destroyed = MALI_FALSE; ++ ++ timeline->sync_tl = mali_sync_timeline_create(timeline, timeline_name); ++ if (NULL == timeline->sync_tl) { ++ mali_timeline_destroy(timeline); ++ return NULL; ++ } ++ ++ timeline->spinlock = mali_spinlock_reentrant_init(_MALI_OSK_LOCK_ORDER_TIMELINE_SYSTEM); ++ if (NULL == timeline->spinlock) { ++ mali_timeline_destroy(timeline); ++ return NULL; ++ } ++ } ++#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ ++ ++ return timeline; ++} ++ ++static void mali_timeline_insert_tracker(struct mali_timeline *timeline, struct mali_timeline_tracker *tracker) ++{ ++ MALI_DEBUG_ASSERT_POINTER(timeline); ++ MALI_DEBUG_ASSERT_POINTER(tracker); ++ ++ if (mali_timeline_is_full(timeline)) { ++ /* Don't add tracker if timeline is full. */ ++ tracker->point = MALI_TIMELINE_NO_POINT; ++ return; ++ } ++ ++ tracker->timeline = timeline; ++ tracker->point = timeline->point_next; ++ ++ /* Find next available point. */ ++ timeline->point_next++; ++ if (MALI_TIMELINE_NO_POINT == timeline->point_next) { ++ timeline->point_next++; ++ } ++ ++ MALI_DEBUG_ASSERT(!mali_timeline_is_empty(timeline)); ++ ++ if (MALI_TIMELINE_TRACKER_GP == tracker->type) { ++ _mali_osk_atomic_inc(&gp_tracker_count); ++ } else if (MALI_TIMELINE_TRACKER_PP == tracker->type) { ++ if (mali_pp_job_is_virtual((struct mali_pp_job *)tracker->job)) { ++ _mali_osk_atomic_inc(&virt_pp_tracker_count); ++ } else { ++ _mali_osk_atomic_inc(&phy_pp_tracker_count); ++ } ++ } ++ ++ /* Add tracker as new head on timeline's tracker list. */ ++ if (NULL == timeline->tracker_head) { ++ /* Tracker list is empty. */ ++ MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail); ++ ++ timeline->tracker_tail = tracker; ++ ++ MALI_DEBUG_ASSERT(NULL == tracker->timeline_next); ++ MALI_DEBUG_ASSERT(NULL == tracker->timeline_prev); ++ } else { ++ MALI_DEBUG_ASSERT(NULL == timeline->tracker_head->timeline_next); ++ ++ tracker->timeline_prev = timeline->tracker_head; ++ timeline->tracker_head->timeline_next = tracker; ++ ++ MALI_DEBUG_ASSERT(NULL == tracker->timeline_next); ++ } ++ timeline->tracker_head = tracker; ++ ++ MALI_DEBUG_ASSERT(NULL == timeline->tracker_head->timeline_next); ++ MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail->timeline_prev); ++} ++ ++/* Inserting the waiter object into the given timeline */ ++static void mali_timeline_insert_waiter(struct mali_timeline *timeline, struct mali_timeline_waiter *waiter_new) ++{ ++ struct mali_timeline_waiter *waiter_prev; ++ struct mali_timeline_waiter *waiter_next; ++ ++ /* Waiter time must be between timeline head and tail, and there must ++ * be less than MALI_TIMELINE_MAX_POINT_SPAN elements between */ ++ MALI_DEBUG_ASSERT((waiter_new->point - timeline->point_oldest) < MALI_TIMELINE_MAX_POINT_SPAN); ++ MALI_DEBUG_ASSERT((-waiter_new->point + timeline->point_next) < MALI_TIMELINE_MAX_POINT_SPAN); ++ ++ /* Finding out where to put this waiter, in the linked waiter list of the given timeline **/ ++ waiter_prev = timeline->waiter_head; /* Insert new after waiter_prev */ ++ waiter_next = NULL; /* Insert new before waiter_next */ ++ ++ /* Iterating backwards from head (newest) to tail (oldest) until we ++ * find the correct spot to insert the new waiter */ ++ while (waiter_prev && mali_timeline_point_after(waiter_prev->point, waiter_new->point)) { ++ waiter_next = waiter_prev; ++ waiter_prev = waiter_prev->timeline_prev; ++ } ++ ++ if (NULL == waiter_prev && NULL == waiter_next) { ++ /* list is empty */ ++ timeline->waiter_head = waiter_new; ++ timeline->waiter_tail = waiter_new; ++ } else if (NULL == waiter_next) { ++ /* insert at head */ ++ waiter_new->timeline_prev = timeline->waiter_head; ++ timeline->waiter_head->timeline_next = waiter_new; ++ timeline->waiter_head = waiter_new; ++ } else if (NULL == waiter_prev) { ++ /* insert at tail */ ++ waiter_new->timeline_next = timeline->waiter_tail; ++ timeline->waiter_tail->timeline_prev = waiter_new; ++ timeline->waiter_tail = waiter_new; ++ } else { ++ /* insert between */ ++ waiter_new->timeline_next = waiter_next; ++ waiter_new->timeline_prev = waiter_prev; ++ waiter_next->timeline_prev = waiter_new; ++ waiter_prev->timeline_next = waiter_new; ++ } ++} ++ ++static void mali_timeline_update_delayed_work(struct mali_timeline *timeline) ++{ ++ struct mali_timeline_system *system; ++ struct mali_timeline_tracker *oldest_tracker; ++ ++ MALI_DEBUG_ASSERT_POINTER(timeline); ++ MALI_DEBUG_ASSERT(MALI_TIMELINE_SOFT == timeline->id); ++ ++ system = timeline->system; ++ MALI_DEBUG_ASSERT_POINTER(system); ++ ++ MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); ++ ++ /* Timer is disabled, early out. */ ++ if (!system->timer_enabled) return; ++ ++ oldest_tracker = timeline->tracker_tail; ++ if (NULL != oldest_tracker && 0 == oldest_tracker->trigger_ref_count) { ++ if (MALI_FALSE == oldest_tracker->timer_active) { ++ if (MALI_TRUE == timeline->timer_active) { ++ _mali_osk_wq_delayed_cancel_work_async(timeline->delayed_work); ++ } ++ _mali_osk_wq_delayed_schedule_work(timeline->delayed_work, MALI_TIMELINE_TIMEOUT_HZ); ++ oldest_tracker->timer_active = MALI_TRUE; ++ timeline->timer_active = MALI_TRUE; ++ } ++ } else if (MALI_TRUE == timeline->timer_active) { ++ _mali_osk_wq_delayed_cancel_work_async(timeline->delayed_work); ++ timeline->timer_active = MALI_FALSE; ++ } ++} ++ ++static mali_scheduler_mask mali_timeline_update_oldest_point(struct mali_timeline *timeline) ++{ ++ mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; ++ ++ MALI_DEBUG_ASSERT_POINTER(timeline); ++ ++ MALI_DEBUG_CODE({ ++ struct mali_timeline_system *system = timeline->system; ++ MALI_DEBUG_ASSERT_POINTER(system); ++ ++ MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); ++ }); ++ ++ if (NULL != timeline->tracker_tail) { ++ /* Set oldest point to oldest tracker's point */ ++ timeline->point_oldest = timeline->tracker_tail->point; ++ } else { ++ /* No trackers, mark point list as empty */ ++ timeline->point_oldest = timeline->point_next; ++ } ++ ++ /* Release all waiters no longer on the timeline's point list. ++ * Releasing a waiter can trigger this function to be called again, so ++ * we do not store any pointers on stack. */ ++ while (NULL != timeline->waiter_tail) { ++ u32 waiter_time_relative; ++ u32 time_head_relative; ++ struct mali_timeline_waiter *waiter = timeline->waiter_tail; ++ ++ time_head_relative = timeline->point_next - timeline->point_oldest; ++ waiter_time_relative = waiter->point - timeline->point_oldest; ++ ++ if (waiter_time_relative < time_head_relative) { ++ /* This and all following waiters are on the point list, so we are done. */ ++ break; ++ } ++ ++ /* Remove waiter from timeline's waiter list. */ ++ if (NULL != waiter->timeline_next) { ++ waiter->timeline_next->timeline_prev = NULL; ++ } else { ++ /* This was the last waiter */ ++ timeline->waiter_head = NULL; ++ } ++ timeline->waiter_tail = waiter->timeline_next; ++ ++ /* Release waiter. This could activate a tracker, if this was ++ * the last waiter for the tracker. */ ++ schedule_mask |= mali_timeline_system_release_waiter(timeline->system, waiter); ++ } ++ ++ return schedule_mask; ++} ++ ++static mali_scheduler_mask mali_timeline_release_with_depended_point(struct mali_timeline_tracker *tracker) ++{ ++ struct mali_timeline *timeline; ++ struct mali_timeline_waiter *waiter; ++ mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; ++ ++ timeline = tracker->timeline; ++ MALI_DEBUG_ASSERT_POINTER(timeline); ++ MALI_DEBUG_ASSERT(MALI_TIMELINE_SOFT == timeline->id); ++ ++ MALI_DEBUG_CODE({ ++ struct mali_timeline_system *system = timeline->system; ++ MALI_DEBUG_ASSERT_POINTER(system); ++ ++ MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); ++ }); ++ ++ /* Only release the waiter that wait for the tracker. */ ++ waiter = timeline->waiter_tail; ++ while (NULL != waiter) { ++ if (waiter->point == tracker->point) { ++ ++ struct mali_timeline_waiter *waiter_next; ++ struct mali_timeline_waiter *waiter_prev; ++ ++ waiter_next = waiter->timeline_next; ++ waiter_prev = waiter->timeline_prev; ++ waiter->timeline_next = NULL; ++ waiter->timeline_prev = NULL; ++ ++ if (NULL != waiter_prev) { ++ waiter_prev->timeline_next = waiter_next; ++ } ++ ++ if (NULL != waiter_next) { ++ waiter_next->timeline_prev = waiter_prev; ++ } ++ ++ if (waiter == timeline->waiter_tail) ++ timeline->waiter_tail = waiter_next; ++ ++ if (waiter == timeline->waiter_head) ++ timeline->waiter_head = NULL; ++ ++ schedule_mask |= mali_timeline_system_release_waiter(timeline->system, waiter); ++ waiter = waiter_next; ++ }else { ++ ++ waiter = waiter->timeline_next; ++ } ++ } ++ ++ return schedule_mask; ++} ++ ++void mali_timeline_tracker_init(struct mali_timeline_tracker *tracker, ++ mali_timeline_tracker_type type, ++ struct mali_timeline_fence *fence, ++ void *job) ++{ ++ MALI_DEBUG_ASSERT_POINTER(tracker); ++ MALI_DEBUG_ASSERT_POINTER(job); ++ ++ MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAX > type); ++ ++ /* Zero out all tracker members. */ ++ _mali_osk_memset(tracker, 0, sizeof(*tracker)); ++ ++ tracker->type = type; ++ tracker->job = job; ++ tracker->trigger_ref_count = 1; /* Prevents any callback from trigging while adding it */ ++ tracker->os_tick_create = _mali_osk_time_tickcount(); ++ MALI_DEBUG_CODE(tracker->magic = MALI_TIMELINE_TRACKER_MAGIC); ++ ++ tracker->activation_error = MALI_TIMELINE_ACTIVATION_ERROR_NONE; ++ ++ /* Copy fence. */ ++ if (NULL != fence) { ++ _mali_osk_memcpy(&tracker->fence, fence, sizeof(struct mali_timeline_fence)); ++ } ++} ++ ++mali_scheduler_mask mali_timeline_tracker_release(struct mali_timeline_tracker *tracker) ++{ ++ struct mali_timeline *timeline; ++ struct mali_timeline_system *system; ++ struct mali_timeline_tracker *tracker_next, *tracker_prev; ++ mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; ++ u32 tid = _mali_osk_get_tid(); ++ ++ /* Upon entry a group lock will be held, but not a scheduler lock. */ ++ MALI_DEBUG_ASSERT_POINTER(tracker); ++ MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAGIC == tracker->magic); ++ ++ /* Tracker should have been triggered */ ++ MALI_DEBUG_ASSERT(0 == tracker->trigger_ref_count); ++ ++ /* All waiters should have been released at this point */ ++ MALI_DEBUG_ASSERT(NULL == tracker->waiter_head); ++ MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail); ++ ++ MALI_DEBUG_PRINT(3, ("Mali Timeline: releasing tracker for job 0x%08X\n", tracker->job)); ++ ++ timeline = tracker->timeline; ++ if (NULL == timeline) { ++ /* Tracker was not on a timeline, there is nothing to release. */ ++ return MALI_SCHEDULER_MASK_EMPTY; ++ } ++ ++ system = timeline->system; ++ MALI_DEBUG_ASSERT_POINTER(system); ++ ++ mali_spinlock_reentrant_wait(system->spinlock, tid); ++ ++ /* Tracker should still be on timeline */ ++ MALI_DEBUG_ASSERT(!mali_timeline_is_empty(timeline)); ++ MALI_DEBUG_ASSERT(mali_timeline_is_point_on(timeline, tracker->point)); ++ ++ /* Tracker is no longer valid. */ ++ MALI_DEBUG_CODE(tracker->magic = 0); ++ ++ tracker_next = tracker->timeline_next; ++ tracker_prev = tracker->timeline_prev; ++ tracker->timeline_next = NULL; ++ tracker->timeline_prev = NULL; ++ ++ /* Removing tracker from timeline's tracker list */ ++ if (NULL == tracker_next) { ++ /* This tracker was the head */ ++ timeline->tracker_head = tracker_prev; ++ } else { ++ tracker_next->timeline_prev = tracker_prev; ++ } ++ ++ if (NULL == tracker_prev) { ++ /* This tracker was the tail */ ++ timeline->tracker_tail = tracker_next; ++ MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); ++ /* Update the timeline's oldest time and release any waiters */ ++ schedule_mask |= mali_timeline_update_oldest_point(timeline); ++ MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); ++ } else { ++ tracker_prev->timeline_next = tracker_next; ++ if (MALI_TIMELINE_SOFT == tracker->timeline->id) { ++ /* Use the signaled soft tracker to release the depended soft waiter */ ++ schedule_mask |= mali_timeline_release_with_depended_point(tracker); ++ MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); ++ } ++ } ++ ++ MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); ++ ++ /* Update delayed work only when it is the soft job timeline */ ++ if (MALI_TIMELINE_SOFT == tracker->timeline->id) { ++ mali_timeline_update_delayed_work(tracker->timeline); ++ } ++ ++ mali_spinlock_reentrant_signal(system->spinlock, tid); ++ ++ return schedule_mask; ++} ++ ++void mali_timeline_system_release_waiter_list(struct mali_timeline_system *system, ++ struct mali_timeline_waiter *tail, ++ struct mali_timeline_waiter *head) ++{ ++ MALI_DEBUG_ASSERT_POINTER(system); ++ MALI_DEBUG_ASSERT_POINTER(head); ++ MALI_DEBUG_ASSERT_POINTER(tail); ++ MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); ++ ++ head->tracker_next = system->waiter_empty_list; ++ system->waiter_empty_list = tail; ++} ++ ++static mali_scheduler_mask mali_timeline_tracker_activate(struct mali_timeline_tracker *tracker) ++{ ++ mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; ++ struct mali_timeline_system *system; ++ struct mali_timeline *timeline; ++ u32 tid = _mali_osk_get_tid(); ++ ++ MALI_DEBUG_ASSERT_POINTER(tracker); ++ MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAGIC == tracker->magic); ++ ++ system = tracker->system; ++ MALI_DEBUG_ASSERT_POINTER(system); ++ MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); ++ ++ tracker->os_tick_activate = _mali_osk_time_tickcount(); ++ ++ if (NULL != tracker->waiter_head) { ++ mali_timeline_system_release_waiter_list(system, tracker->waiter_tail, tracker->waiter_head); ++ tracker->waiter_head = NULL; ++ tracker->waiter_tail = NULL; ++ } ++ ++ switch (tracker->type) { ++ case MALI_TIMELINE_TRACKER_GP: ++ schedule_mask = mali_scheduler_activate_gp_job((struct mali_gp_job *) tracker->job); ++ ++ _mali_osk_atomic_dec(&gp_tracker_count); ++ break; ++ case MALI_TIMELINE_TRACKER_PP: ++ if (mali_pp_job_is_virtual((struct mali_pp_job *)tracker->job)) { ++ _mali_osk_atomic_dec(&virt_pp_tracker_count); ++ } else { ++ _mali_osk_atomic_dec(&phy_pp_tracker_count); ++ } ++ schedule_mask = mali_scheduler_activate_pp_job((struct mali_pp_job *) tracker->job); ++ break; ++ case MALI_TIMELINE_TRACKER_SOFT: ++ timeline = tracker->timeline; ++ MALI_DEBUG_ASSERT_POINTER(timeline); ++ ++ schedule_mask |= mali_soft_job_system_activate_job((struct mali_soft_job *) tracker->job); ++ ++ /* Start a soft timer to make sure the soft job be released in a limited time */ ++ mali_spinlock_reentrant_wait(system->spinlock, tid); ++ mali_timeline_update_delayed_work(timeline); ++ mali_spinlock_reentrant_signal(system->spinlock, tid); ++ break; ++ case MALI_TIMELINE_TRACKER_WAIT: ++ mali_timeline_fence_wait_activate((struct mali_timeline_fence_wait_tracker *) tracker->job); ++ break; ++ case MALI_TIMELINE_TRACKER_SYNC: ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ mali_timeline_sync_fence_activate((struct mali_timeline_sync_fence_tracker *) tracker->job); ++#else ++ MALI_PRINT_ERROR(("Mali Timeline: sync tracker not supported\n", tracker->type)); ++#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ ++ break; ++ default: ++ MALI_PRINT_ERROR(("Mali Timeline - Illegal tracker type: %d\n", tracker->type)); ++ break; ++ } ++ ++ return schedule_mask; ++} ++ ++void mali_timeline_system_tracker_get(struct mali_timeline_system *system, struct mali_timeline_tracker *tracker) ++{ ++ u32 tid = _mali_osk_get_tid(); ++ ++ MALI_DEBUG_ASSERT_POINTER(tracker); ++ MALI_DEBUG_ASSERT_POINTER(system); ++ ++ mali_spinlock_reentrant_wait(system->spinlock, tid); ++ ++ MALI_DEBUG_ASSERT(0 < tracker->trigger_ref_count); ++ tracker->trigger_ref_count++; ++ ++ mali_spinlock_reentrant_signal(system->spinlock, tid); ++} ++ ++mali_scheduler_mask mali_timeline_system_tracker_put(struct mali_timeline_system *system, struct mali_timeline_tracker *tracker, mali_timeline_activation_error activation_error) ++{ ++ u32 tid = _mali_osk_get_tid(); ++ mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; ++ ++ MALI_DEBUG_ASSERT_POINTER(tracker); ++ MALI_DEBUG_ASSERT_POINTER(system); ++ ++ mali_spinlock_reentrant_wait(system->spinlock, tid); ++ ++ MALI_DEBUG_ASSERT(0 < tracker->trigger_ref_count); ++ tracker->trigger_ref_count--; ++ ++ tracker->activation_error |= activation_error; ++ ++ if (0 == tracker->trigger_ref_count) { ++ schedule_mask |= mali_timeline_tracker_activate(tracker); ++ tracker = NULL; ++ } ++ ++ mali_spinlock_reentrant_signal(system->spinlock, tid); ++ ++ return schedule_mask; ++} ++ ++void mali_timeline_fence_copy_uk_fence(struct mali_timeline_fence *fence, _mali_uk_fence_t *uk_fence) ++{ ++ u32 i; ++ ++ MALI_DEBUG_ASSERT_POINTER(fence); ++ MALI_DEBUG_ASSERT_POINTER(uk_fence); ++ ++ for (i = 0; i < MALI_TIMELINE_MAX; ++i) { ++ fence->points[i] = uk_fence->points[i]; ++ } ++ ++ fence->sync_fd = uk_fence->sync_fd; ++} ++ ++struct mali_timeline_system *mali_timeline_system_create(struct mali_session_data *session) ++{ ++ u32 i; ++ struct mali_timeline_system *system; ++ ++ MALI_DEBUG_ASSERT_POINTER(session); ++ MALI_DEBUG_PRINT(4, ("Mali Timeline: creating timeline system\n")); ++ ++ system = (struct mali_timeline_system *) _mali_osk_calloc(1, sizeof(struct mali_timeline_system)); ++ if (NULL == system) { ++ return NULL; ++ } ++ ++ system->spinlock = mali_spinlock_reentrant_init(_MALI_OSK_LOCK_ORDER_TIMELINE_SYSTEM); ++ if (NULL == system->spinlock) { ++ mali_timeline_system_destroy(system); ++ return NULL; ++ } ++ ++ for (i = 0; i < MALI_TIMELINE_MAX; ++i) { ++ system->timelines[i] = mali_timeline_create(system, (enum mali_timeline_id)i); ++ if (NULL == system->timelines[i]) { ++ mali_timeline_system_destroy(system); ++ return NULL; ++ } ++ } ++ ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ system->signaled_sync_tl = mali_sync_timeline_create(NULL, "mali-always-signaled"); ++ if (NULL == system->signaled_sync_tl) { ++ mali_timeline_system_destroy(system); ++ return NULL; ++ } ++#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ ++ ++ system->waiter_empty_list = NULL; ++ system->session = session; ++ system->timer_enabled = MALI_TRUE; ++ ++ system->wait_queue = _mali_osk_wait_queue_init(); ++ if (NULL == system->wait_queue) { ++ mali_timeline_system_destroy(system); ++ return NULL; ++ } ++ ++ return system; ++} ++ ++#if defined(CONFIG_MALI_DMA_BUF_FENCE) ||defined(CONFIG_SYNC) ||defined(CONFIG_SYNC_FILE) ++/** ++ * Check if there are any trackers left on timeline. ++ * ++ * Used as a wait queue conditional. ++ * ++ * @param data Timeline. ++ * @return MALI_TRUE if there are no trackers on timeline, MALI_FALSE if not. ++ */ ++static mali_bool mali_timeline_has_no_trackers(void *data) ++{ ++ struct mali_timeline *timeline = (struct mali_timeline *) data; ++ ++ MALI_DEBUG_ASSERT_POINTER(timeline); ++ ++ return mali_timeline_is_empty(timeline); ++} ++#if defined(CONFIG_SYNC) ||defined(CONFIG_SYNC_FILE) ++/** ++ * Cancel sync fence waiters waited upon by trackers on all timelines. ++ * ++ * Will return after all timelines have no trackers left. ++ * ++ * @param system Timeline system. ++ */ ++static void mali_timeline_cancel_sync_fence_waiters(struct mali_timeline_system *system) ++{ ++ u32 i; ++ u32 tid = _mali_osk_get_tid(); ++ struct mali_timeline_tracker *tracker, *tracker_next; ++ _MALI_OSK_LIST_HEAD_STATIC_INIT(tracker_list); ++ ++ MALI_DEBUG_ASSERT_POINTER(system); ++ MALI_DEBUG_ASSERT_POINTER(system->session); ++ MALI_DEBUG_ASSERT(system->session->is_aborting); ++ ++ mali_spinlock_reentrant_wait(system->spinlock, tid); ++ ++ /* Cancel sync fence waiters. */ ++ for (i = 0; i < MALI_TIMELINE_MAX; ++i) { ++ struct mali_timeline *timeline = system->timelines[i]; ++ ++ MALI_DEBUG_ASSERT_POINTER(timeline); ++ ++ tracker_next = timeline->tracker_tail; ++ while (NULL != tracker_next) { ++ tracker = tracker_next; ++ tracker_next = tracker->timeline_next; ++ ++ if (NULL == tracker->sync_fence) continue; ++ ++ MALI_DEBUG_PRINT(3, ("Mali Timeline: Cancelling sync fence wait for tracker 0x%08X.\n", tracker)); ++ ++ /* Cancel sync fence waiter. */ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) ++ if (0 == sync_fence_cancel_async(tracker->sync_fence, &tracker->sync_fence_waiter)) { ++#else ++ if (0 == mali_internal_sync_fence_cancel_async(tracker->sync_fence, &tracker->sync_fence_waiter)) { ++#endif ++ /* Callback was not called, move tracker to local list. */ ++ _mali_osk_list_add(&tracker->sync_fence_cancel_list, &tracker_list); ++ } ++ } ++ } ++ ++ mali_spinlock_reentrant_signal(system->spinlock, tid); ++ ++ /* Manually call sync fence callback in order to release waiter and trigger activation of tracker. */ ++ _MALI_OSK_LIST_FOREACHENTRY(tracker, tracker_next, &tracker_list, struct mali_timeline_tracker, sync_fence_cancel_list) { ++ mali_timeline_sync_fence_callback(tracker->sync_fence, &tracker->sync_fence_waiter); ++ } ++ ++ /* Sleep until all sync fence callbacks are done and all timelines are empty. */ ++ for (i = 0; i < MALI_TIMELINE_MAX; ++i) { ++ struct mali_timeline *timeline = system->timelines[i]; ++ ++ MALI_DEBUG_ASSERT_POINTER(timeline); ++ ++ _mali_osk_wait_queue_wait_event(system->wait_queue, mali_timeline_has_no_trackers, (void *) timeline); ++ } ++} ++ ++#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ ++ ++#if defined(CONFIG_MALI_DMA_BUF_FENCE) ++static void mali_timeline_cancel_dma_fence_waiters(struct mali_timeline_system *system) ++{ ++ u32 i, j; ++ u32 tid = _mali_osk_get_tid(); ++ struct mali_pp_job *pp_job = NULL; ++ struct mali_pp_job *next_pp_job = NULL; ++ struct mali_timeline *timeline = NULL; ++ struct mali_timeline_tracker *tracker, *tracker_next; ++ _MALI_OSK_LIST_HEAD_STATIC_INIT(pp_job_list); ++ ++ MALI_DEBUG_ASSERT_POINTER(system); ++ MALI_DEBUG_ASSERT_POINTER(system->session); ++ MALI_DEBUG_ASSERT(system->session->is_aborting); ++ ++ mali_spinlock_reentrant_wait(system->spinlock, tid); ++ ++ /* Cancel dma fence waiters. */ ++ timeline = system->timelines[MALI_TIMELINE_PP]; ++ MALI_DEBUG_ASSERT_POINTER(timeline); ++ ++ tracker_next = timeline->tracker_tail; ++ while (NULL != tracker_next) { ++ mali_bool fence_is_signaled = MALI_TRUE; ++ tracker = tracker_next; ++ tracker_next = tracker->timeline_next; ++ ++ if (NULL == tracker->waiter_dma_fence) continue; ++ pp_job = (struct mali_pp_job *)tracker->job; ++ MALI_DEBUG_ASSERT_POINTER(pp_job); ++ MALI_DEBUG_PRINT(3, ("Mali Timeline: Cancelling dma fence waiter for tracker 0x%08X.\n", tracker)); ++ ++ for (j = 0; j < pp_job->dma_fence_context.num_dma_fence_waiter; j++) { ++ if (pp_job->dma_fence_context.mali_dma_fence_waiters[j]) { ++ /* Cancel a previously callback from the fence. ++ * This function returns true if the callback is successfully removed, ++ * or false if the fence has already been signaled. ++ */ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++ bool ret = dma_fence_remove_callback(pp_job->dma_fence_context.mali_dma_fence_waiters[j]->fence, ++ &pp_job->dma_fence_context.mali_dma_fence_waiters[j]->base); ++ ++#else ++ bool ret = fence_remove_callback(pp_job->dma_fence_context.mali_dma_fence_waiters[j]->fence, ++ &pp_job->dma_fence_context.mali_dma_fence_waiters[j]->base); ++#endif ++ if (ret) { ++ fence_is_signaled = MALI_FALSE; ++ } ++ } ++ } ++ ++ /* Callbacks were not called, move pp job to local list. */ ++ if (MALI_FALSE == fence_is_signaled) ++ _mali_osk_list_add(&pp_job->list, &pp_job_list); ++ } ++ ++ mali_spinlock_reentrant_signal(system->spinlock, tid); ++ ++ /* Manually call dma fence callback in order to release waiter and trigger activation of tracker. */ ++ _MALI_OSK_LIST_FOREACHENTRY(pp_job, next_pp_job, &pp_job_list, struct mali_pp_job, list) { ++ mali_timeline_dma_fence_callback((void *)pp_job); ++ } ++ ++ /* Sleep until all dma fence callbacks are done and all timelines are empty. */ ++ for (i = 0; i < MALI_TIMELINE_MAX; ++i) { ++ struct mali_timeline *timeline = system->timelines[i]; ++ MALI_DEBUG_ASSERT_POINTER(timeline); ++ _mali_osk_wait_queue_wait_event(system->wait_queue, mali_timeline_has_no_trackers, (void *) timeline); ++ } ++} ++#endif ++#endif ++void mali_timeline_system_abort(struct mali_timeline_system *system) ++{ ++ MALI_DEBUG_CODE(u32 tid = _mali_osk_get_tid();); ++ ++ MALI_DEBUG_ASSERT_POINTER(system); ++ MALI_DEBUG_ASSERT_POINTER(system->session); ++ MALI_DEBUG_ASSERT(system->session->is_aborting); ++ ++ MALI_DEBUG_PRINT(3, ("Mali Timeline: Aborting timeline system for session 0x%08X.\n", system->session)); ++ ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ mali_timeline_cancel_sync_fence_waiters(system); ++#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ ++ ++#if defined(CONFIG_MALI_DMA_BUF_FENCE) ++ mali_timeline_cancel_dma_fence_waiters(system); ++#endif ++ ++ /* Should not be any waiters or trackers left at this point. */ ++ MALI_DEBUG_CODE({ ++ u32 i; ++ mali_spinlock_reentrant_wait(system->spinlock, tid); ++ for (i = 0; i < MALI_TIMELINE_MAX; ++i) ++ { ++ struct mali_timeline *timeline = system->timelines[i]; ++ MALI_DEBUG_ASSERT_POINTER(timeline); ++ MALI_DEBUG_ASSERT(timeline->point_oldest == timeline->point_next); ++ MALI_DEBUG_ASSERT(NULL == timeline->tracker_head); ++ MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail); ++ MALI_DEBUG_ASSERT(NULL == timeline->waiter_head); ++ MALI_DEBUG_ASSERT(NULL == timeline->waiter_tail); ++ } ++ mali_spinlock_reentrant_signal(system->spinlock, tid); ++ }); ++} ++ ++void mali_timeline_system_destroy(struct mali_timeline_system *system) ++{ ++ u32 i; ++ struct mali_timeline_waiter *waiter, *next; ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ u32 tid = _mali_osk_get_tid(); ++#endif ++ ++ MALI_DEBUG_ASSERT_POINTER(system); ++ MALI_DEBUG_ASSERT_POINTER(system->session); ++ ++ MALI_DEBUG_PRINT(4, ("Mali Timeline: destroying timeline system\n")); ++ ++ if (NULL != system) { ++ ++ /* There should be no waiters left on this queue. */ ++ if (NULL != system->wait_queue) { ++ _mali_osk_wait_queue_term(system->wait_queue); ++ system->wait_queue = NULL; ++ } ++ ++ /* Free all waiters in empty list */ ++ waiter = system->waiter_empty_list; ++ while (NULL != waiter) { ++ next = waiter->tracker_next; ++ _mali_osk_free(waiter); ++ waiter = next; ++ } ++ ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ if (NULL != system->signaled_sync_tl) { ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) ++ sync_timeline_destroy(system->signaled_sync_tl); ++#else ++ mali_internal_sync_timeline_destroy(system->signaled_sync_tl); ++#endif ++ } ++ ++ for (i = 0; i < MALI_TIMELINE_MAX; ++i) { ++ if ((NULL != system->timelines[i]) && (NULL != system->timelines[i]->spinlock)) { ++ mali_spinlock_reentrant_wait(system->timelines[i]->spinlock, tid); ++ system->timelines[i]->destroyed = MALI_TRUE; ++ mali_spinlock_reentrant_signal(system->timelines[i]->spinlock, tid); ++ } ++ } ++#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ ++ ++ for (i = 0; i < MALI_TIMELINE_MAX; ++i) { ++ if (NULL != system->timelines[i]) { ++ mali_timeline_destroy(system->timelines[i]); ++ } ++ } ++ ++ if (NULL != system->spinlock) { ++ mali_spinlock_reentrant_term(system->spinlock); ++ } ++ ++ _mali_osk_free(system); ++ } ++} ++ ++/** ++ * Find how many waiters are needed for a given fence. ++ * ++ * @param fence The fence to check. ++ * @return Number of waiters needed for fence. ++ */ ++static u32 mali_timeline_fence_num_waiters(struct mali_timeline_fence *fence) ++{ ++ u32 i, num_waiters = 0; ++ ++ MALI_DEBUG_ASSERT_POINTER(fence); ++ ++ for (i = 0; i < MALI_TIMELINE_MAX; ++i) { ++ if (MALI_TIMELINE_NO_POINT != fence->points[i]) { ++ ++num_waiters; ++ } ++ } ++ ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ if (-1 != fence->sync_fd) ++num_waiters; ++#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ ++ ++ return num_waiters; ++} ++ ++static struct mali_timeline_waiter *mali_timeline_system_get_zeroed_waiter(struct mali_timeline_system *system) ++{ ++ struct mali_timeline_waiter *waiter; ++ ++ MALI_DEBUG_ASSERT_POINTER(system); ++ MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); ++ ++ waiter = system->waiter_empty_list; ++ if (NULL != waiter) { ++ /* Remove waiter from empty list and zero it */ ++ system->waiter_empty_list = waiter->tracker_next; ++ _mali_osk_memset(waiter, 0, sizeof(*waiter)); ++ } ++ ++ /* Return NULL if list was empty. */ ++ return waiter; ++} ++ ++static void mali_timeline_system_allocate_waiters(struct mali_timeline_system *system, ++ struct mali_timeline_waiter **tail, ++ struct mali_timeline_waiter **head, ++ int max_num_waiters) ++{ ++ u32 i, tid = _mali_osk_get_tid(); ++ mali_bool do_alloc; ++ struct mali_timeline_waiter *waiter; ++ ++ MALI_DEBUG_ASSERT_POINTER(system); ++ MALI_DEBUG_ASSERT_POINTER(tail); ++ MALI_DEBUG_ASSERT_POINTER(head); ++ ++ MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); ++ ++ *head = *tail = NULL; ++ do_alloc = MALI_FALSE; ++ i = 0; ++ while (i < max_num_waiters) { ++ if (MALI_FALSE == do_alloc) { ++ waiter = mali_timeline_system_get_zeroed_waiter(system); ++ if (NULL == waiter) { ++ do_alloc = MALI_TRUE; ++ mali_spinlock_reentrant_signal(system->spinlock, tid); ++ continue; ++ } ++ } else { ++ waiter = _mali_osk_calloc(1, sizeof(struct mali_timeline_waiter)); ++ if (NULL == waiter) break; ++ } ++ ++i; ++ if (NULL == *tail) { ++ *tail = waiter; ++ *head = waiter; ++ } else { ++ (*head)->tracker_next = waiter; ++ *head = waiter; ++ } ++ } ++ if (MALI_TRUE == do_alloc) { ++ mali_spinlock_reentrant_wait(system->spinlock, tid); ++ } ++} ++ ++/** ++ * Create waiters for the given tracker. The tracker is activated when all waiters are release. ++ * ++ * @note Tracker can potentially be activated before this function returns. ++ * ++ * @param system Timeline system. ++ * @param tracker Tracker we will create waiters for. ++ * @param waiter_tail List of pre-allocated waiters. ++ * @param waiter_head List of pre-allocated waiters. ++ */ ++static void mali_timeline_system_create_waiters_and_unlock(struct mali_timeline_system *system, ++ struct mali_timeline_tracker *tracker, ++ struct mali_timeline_waiter *waiter_tail, ++ struct mali_timeline_waiter *waiter_head) ++{ ++ int i; ++ u32 tid = _mali_osk_get_tid(); ++ mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) ++ struct sync_fence *sync_fence = NULL; ++#else ++ struct mali_internal_sync_fence *sync_fence = NULL; ++#endif ++#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ ++ ++ MALI_DEBUG_ASSERT_POINTER(system); ++ MALI_DEBUG_ASSERT_POINTER(tracker); ++ ++ MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); ++ ++ MALI_DEBUG_ASSERT(NULL == tracker->waiter_head); ++ MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail); ++ MALI_DEBUG_ASSERT(NULL != tracker->job); ++ ++ /* Creating waiter object for all the timelines the fence is put on. Inserting this waiter ++ * into the timelines sorted list of waiters */ ++ for (i = 0; i < MALI_TIMELINE_MAX; ++i) { ++ mali_timeline_point point; ++ struct mali_timeline *timeline; ++ struct mali_timeline_waiter *waiter; ++ ++ /* Get point on current timeline from tracker's fence. */ ++ point = tracker->fence.points[i]; ++ ++ if (likely(MALI_TIMELINE_NO_POINT == point)) { ++ /* Fence contains no point on this timeline so we don't need a waiter. */ ++ continue; ++ } ++ ++ timeline = system->timelines[i]; ++ MALI_DEBUG_ASSERT_POINTER(timeline); ++ ++ if (unlikely(!mali_timeline_is_point_valid(timeline, point))) { ++ MALI_PRINT_ERROR(("Mali Timeline: point %d is not valid (oldest=%d, next=%d)\n", ++ point, timeline->point_oldest, timeline->point_next)); ++ continue; ++ } ++ ++ if (likely(mali_timeline_is_point_released(timeline, point))) { ++ /* Tracker representing the point has been released so we don't need a ++ * waiter. */ ++ continue; ++ } ++ ++ if ((MALI_TIMELINE_SOFT == timeline->id) && mali_timeline_is_tracker_released(timeline, point)) { ++ /* The tracker that the point related to has already been released, so no need to a waiter. */ ++ continue; ++ } ++ ++ /* The point is on timeline. */ ++ MALI_DEBUG_ASSERT(mali_timeline_is_point_on(timeline, point)); ++ ++ /* Get a new zeroed waiter object. */ ++ if (likely(NULL != waiter_tail)) { ++ waiter = waiter_tail; ++ waiter_tail = waiter_tail->tracker_next; ++ } else { ++ MALI_PRINT_ERROR(("Mali Timeline: failed to allocate memory for waiter\n")); ++ continue; ++ } ++ ++ /* Yanking the trigger ref count of the tracker. */ ++ tracker->trigger_ref_count++; ++ ++ waiter->point = point; ++ waiter->tracker = tracker; ++ ++ /* Insert waiter on tracker's singly-linked waiter list. */ ++ if (NULL == tracker->waiter_head) { ++ /* list is empty */ ++ MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail); ++ tracker->waiter_tail = waiter; ++ } else { ++ tracker->waiter_head->tracker_next = waiter; ++ } ++ tracker->waiter_head = waiter; ++ ++ /* Add waiter to timeline. */ ++ mali_timeline_insert_waiter(timeline, waiter); ++ } ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ if (-1 != tracker->fence.sync_fd) { ++ int ret; ++ struct mali_timeline_waiter *waiter; ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) ++ sync_fence = sync_fence_fdget(tracker->fence.sync_fd); ++#else ++ sync_fence = mali_internal_sync_fence_fdget(tracker->fence.sync_fd); ++#endif ++ if (unlikely(NULL == sync_fence)) { ++ MALI_PRINT_ERROR(("Mali Timeline: failed to get sync fence from fd %d\n", tracker->fence.sync_fd)); ++ goto exit; ++ } ++ ++ /* Check if we have a zeroed waiter object available. */ ++ if (unlikely(NULL == waiter_tail)) { ++ MALI_PRINT_ERROR(("Mali Timeline: failed to allocate memory for waiter\n")); ++ goto exit; ++ } ++ ++ /* Start asynchronous wait that will release waiter when the fence is signaled. */ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) ++ sync_fence_waiter_init(&tracker->sync_fence_waiter, mali_timeline_sync_fence_callback); ++ ret = sync_fence_wait_async(sync_fence, &tracker->sync_fence_waiter); ++#else ++ mali_internal_sync_fence_waiter_init(&tracker->sync_fence_waiter, mali_timeline_sync_fence_callback); ++ ret = mali_internal_sync_fence_wait_async(sync_fence, &tracker->sync_fence_waiter); ++#endif ++ if (1 == ret) { ++ /* Fence already signaled, no waiter needed. */ ++ tracker->fence.sync_fd = -1; ++ goto exit; ++ } else if (0 != ret) { ++ MALI_PRINT_ERROR(("Mali Timeline: sync fence fd %d signaled with error %d\n", tracker->fence.sync_fd, ret)); ++ tracker->activation_error |= MALI_TIMELINE_ACTIVATION_ERROR_SYNC_BIT; ++ goto exit; ++ } ++ ++ /* Grab new zeroed waiter object. */ ++ waiter = waiter_tail; ++ waiter_tail = waiter_tail->tracker_next; ++ ++ /* Increase the trigger ref count of the tracker. */ ++ tracker->trigger_ref_count++; ++ ++ waiter->point = MALI_TIMELINE_NO_POINT; ++ waiter->tracker = tracker; ++ ++ /* Insert waiter on tracker's singly-linked waiter list. */ ++ if (NULL == tracker->waiter_head) { ++ /* list is empty */ ++ MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail); ++ tracker->waiter_tail = waiter; ++ } else { ++ tracker->waiter_head->tracker_next = waiter; ++ } ++ tracker->waiter_head = waiter; ++ ++ /* Also store waiter in separate field for easy access by sync callback. */ ++ tracker->waiter_sync = waiter; ++ ++ /* Store the sync fence in tracker so we can retrieve in abort session, if needed. */ ++ tracker->sync_fence = sync_fence; ++ ++ sync_fence = NULL; ++ } ++#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE)*/ ++#if defined(CONFIG_MALI_DMA_BUF_FENCE) ++ if ((NULL != tracker->timeline) && (MALI_TIMELINE_PP == tracker->timeline->id)) { ++ ++ struct mali_pp_job *job = (struct mali_pp_job *)tracker->job; ++ ++ if (0 < job->dma_fence_context.num_dma_fence_waiter) { ++ struct mali_timeline_waiter *waiter; ++ /* Check if we have a zeroed waiter object available. */ ++ if (unlikely(NULL == waiter_tail)) { ++ MALI_PRINT_ERROR(("Mali Timeline: failed to allocate memory for waiter\n")); ++ goto exit; ++ } ++ ++ /* Grab new zeroed waiter object. */ ++ waiter = waiter_tail; ++ waiter_tail = waiter_tail->tracker_next; ++ ++ /* Increase the trigger ref count of the tracker. */ ++ tracker->trigger_ref_count++; ++ ++ waiter->point = MALI_TIMELINE_NO_POINT; ++ waiter->tracker = tracker; ++ ++ /* Insert waiter on tracker's singly-linked waiter list. */ ++ if (NULL == tracker->waiter_head) { ++ /* list is empty */ ++ MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail); ++ tracker->waiter_tail = waiter; ++ } else { ++ tracker->waiter_head->tracker_next = waiter; ++ } ++ tracker->waiter_head = waiter; ++ ++ /* Also store waiter in separate field for easy access by sync callback. */ ++ tracker->waiter_dma_fence = waiter; ++ } ++ } ++#endif /* defined(CONFIG_MALI_DMA_BUF_FENCE)*/ ++ ++#if defined(CONFIG_MALI_DMA_BUF_FENCE) ||defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++exit: ++#endif /* defined(CONFIG_MALI_DMA_BUF_FENCE) || defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ ++ ++ if (NULL != waiter_tail) { ++ mali_timeline_system_release_waiter_list(system, waiter_tail, waiter_head); ++ } ++ ++ /* Release the initial trigger ref count. */ ++ tracker->trigger_ref_count--; ++ ++ /* If there were no waiters added to this tracker we activate immediately. */ ++ if (0 == tracker->trigger_ref_count) { ++ schedule_mask |= mali_timeline_tracker_activate(tracker); ++ } ++ ++ mali_spinlock_reentrant_signal(system->spinlock, tid); ++ ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ if (NULL != sync_fence) { ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) ++ sync_fence_put(sync_fence); ++#else ++ fput(sync_fence->file); ++#endif ++ } ++#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ ++ ++ mali_executor_schedule_from_mask(schedule_mask, MALI_FALSE); ++} ++ ++mali_timeline_point mali_timeline_system_add_tracker(struct mali_timeline_system *system, ++ struct mali_timeline_tracker *tracker, ++ enum mali_timeline_id timeline_id) ++{ ++ int num_waiters = 0; ++ struct mali_timeline_waiter *waiter_tail, *waiter_head; ++ u32 tid = _mali_osk_get_tid(); ++ ++ mali_timeline_point point = MALI_TIMELINE_NO_POINT; ++ ++ MALI_DEBUG_ASSERT_POINTER(system); ++ MALI_DEBUG_ASSERT_POINTER(system->session); ++ MALI_DEBUG_ASSERT_POINTER(tracker); ++ ++ MALI_DEBUG_ASSERT(MALI_FALSE == system->session->is_aborting); ++ MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAX > tracker->type); ++ MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAGIC == tracker->magic); ++ ++ MALI_DEBUG_PRINT(4, ("Mali Timeline: adding tracker for job %p, timeline: %d\n", tracker->job, timeline_id)); ++ ++ MALI_DEBUG_ASSERT(0 < tracker->trigger_ref_count); ++ tracker->system = system; ++ ++ mali_spinlock_reentrant_wait(system->spinlock, tid); ++ ++ num_waiters = mali_timeline_fence_num_waiters(&tracker->fence); ++ ++#if defined(CONFIG_MALI_DMA_BUF_FENCE) ++ if (MALI_TIMELINE_PP == timeline_id) { ++ struct mali_pp_job *job = (struct mali_pp_job *)tracker->job; ++ if (0 < job->dma_fence_context.num_dma_fence_waiter) ++ num_waiters++; ++ } ++#endif ++ ++ /* Allocate waiters. */ ++ mali_timeline_system_allocate_waiters(system, &waiter_tail, &waiter_head, num_waiters); ++ MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); ++ ++ /* Add tracker to timeline. This will allocate a point for the tracker on the timeline. If ++ * timeline ID is MALI_TIMELINE_NONE the tracker will NOT be added to a timeline and the ++ * point will be MALI_TIMELINE_NO_POINT. ++ * ++ * NOTE: the tracker can fail to be added if the timeline is full. If this happens, the ++ * point will be MALI_TIMELINE_NO_POINT. */ ++ MALI_DEBUG_ASSERT(timeline_id < MALI_TIMELINE_MAX || timeline_id == MALI_TIMELINE_NONE); ++ if (likely(timeline_id < MALI_TIMELINE_MAX)) { ++ struct mali_timeline *timeline = system->timelines[timeline_id]; ++ mali_timeline_insert_tracker(timeline, tracker); ++ MALI_DEBUG_ASSERT(!mali_timeline_is_empty(timeline)); ++ } ++ ++ point = tracker->point; ++ ++ /* Create waiters for tracker based on supplied fence. Each waiter will increase the ++ * trigger ref count. */ ++ mali_timeline_system_create_waiters_and_unlock(system, tracker, waiter_tail, waiter_head); ++ tracker = NULL; ++ ++ /* At this point the tracker object might have been freed so we should no longer ++ * access it. */ ++ ++ ++ /* The tracker will always be activated after calling add_tracker, even if NO_POINT is ++ * returned. */ ++ return point; ++} ++ ++static mali_scheduler_mask mali_timeline_system_release_waiter(struct mali_timeline_system *system, ++ struct mali_timeline_waiter *waiter) ++{ ++ struct mali_timeline_tracker *tracker; ++ mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; ++ ++ MALI_DEBUG_ASSERT_POINTER(system); ++ MALI_DEBUG_ASSERT_POINTER(waiter); ++ ++ MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); ++ ++ tracker = waiter->tracker; ++ MALI_DEBUG_ASSERT_POINTER(tracker); ++ ++ /* At this point the waiter has been removed from the timeline's waiter list, but it is ++ * still on the tracker's waiter list. All of the tracker's waiters will be released when ++ * the tracker is activated. */ ++ ++ waiter->point = MALI_TIMELINE_NO_POINT; ++ waiter->tracker = NULL; ++ ++ tracker->trigger_ref_count--; ++ if (0 == tracker->trigger_ref_count) { ++ /* This was the last waiter; activate tracker */ ++ schedule_mask |= mali_timeline_tracker_activate(tracker); ++ tracker = NULL; ++ } ++ ++ return schedule_mask; ++} ++ ++mali_timeline_point mali_timeline_system_get_latest_point(struct mali_timeline_system *system, ++ enum mali_timeline_id timeline_id) ++{ ++ mali_timeline_point point; ++ struct mali_timeline *timeline; ++ u32 tid = _mali_osk_get_tid(); ++ ++ MALI_DEBUG_ASSERT_POINTER(system); ++ ++ if (MALI_TIMELINE_MAX <= timeline_id) { ++ return MALI_TIMELINE_NO_POINT; ++ } ++ ++ mali_spinlock_reentrant_wait(system->spinlock, tid); ++ ++ timeline = system->timelines[timeline_id]; ++ MALI_DEBUG_ASSERT_POINTER(timeline); ++ ++ point = MALI_TIMELINE_NO_POINT; ++ if (timeline->point_oldest != timeline->point_next) { ++ point = timeline->point_next - 1; ++ if (MALI_TIMELINE_NO_POINT == point) point--; ++ } ++ ++ mali_spinlock_reentrant_signal(system->spinlock, tid); ++ ++ return point; ++} ++ ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++static void mali_timeline_do_sync_fence_callback(void *arg) ++{ ++ _MALI_OSK_LIST_HEAD_STATIC_INIT(list); ++ struct mali_timeline_tracker *tracker; ++ struct mali_timeline_tracker *tmp_tracker; ++ u32 tid = _mali_osk_get_tid(); ++ ++ MALI_IGNORE(arg); ++ ++ /* ++ * Quickly "unhook" the jobs pending to be deleted, so we can release ++ * the lock before we start deleting the job objects ++ * (without any locks held) ++ */ ++ _mali_osk_spinlock_irq_lock(sync_fence_callback_list_lock); ++ _mali_osk_list_move_list(&sync_fence_callback_queue, &list); ++ _mali_osk_spinlock_irq_unlock(sync_fence_callback_list_lock); ++ ++ _MALI_OSK_LIST_FOREACHENTRY(tracker, tmp_tracker, &list, ++ struct mali_timeline_tracker, sync_fence_signal_list) { ++ mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; ++ mali_bool is_aborting = MALI_FALSE; ++ int fence_status = 0; ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) ++ struct sync_fence *sync_fence = NULL; ++#else ++ struct mali_internal_sync_fence *sync_fence = NULL; ++#endif ++ struct mali_timeline_system *system = NULL; ++ struct mali_timeline_waiter *waiter = NULL; ++ ++ _mali_osk_list_delinit(&tracker->sync_fence_signal_list); ++ ++ sync_fence = tracker->sync_fence; ++ MALI_DEBUG_ASSERT_POINTER(sync_fence); ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) ++ fence_status = sync_fence->status; ++#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) ++ fence_status = atomic_read(&sync_fence->status); ++#else ++ fence_status = sync_fence->fence->ops->signaled(sync_fence->fence); ++#endif ++ ++ system = tracker->system; ++ MALI_DEBUG_ASSERT_POINTER(system); ++ MALI_DEBUG_ASSERT_POINTER(system->session); ++ ++ mali_spinlock_reentrant_wait(system->spinlock, tid); ++ ++ is_aborting = system->session->is_aborting; ++ if (!is_aborting && (0 > fence_status)) { ++ MALI_PRINT_ERROR(("Mali Timeline: sync fence fd %d signaled with error %d\n", tracker->fence.sync_fd, fence_status)); ++ tracker->activation_error |= MALI_TIMELINE_ACTIVATION_ERROR_SYNC_BIT; ++ } ++ ++ waiter = tracker->waiter_sync; ++ MALI_DEBUG_ASSERT_POINTER(waiter); ++ ++ tracker->sync_fence = NULL; ++ tracker->fence.sync_fd = -1; ++ ++ schedule_mask |= mali_timeline_system_release_waiter(system, waiter); ++ ++ /* If aborting, wake up sleepers that are waiting for sync fence callbacks to complete. */ ++ if (is_aborting) { ++ _mali_osk_wait_queue_wake_up(system->wait_queue); ++ } ++ ++ mali_spinlock_reentrant_signal(system->spinlock, tid); ++ ++ /* ++ * Older versions of Linux, before 3.5, doesn't support fput() in interrupt ++ * context. For those older kernels, allocate a list object and put the ++ * fence object on that and defer the call to sync_fence_put() to a workqueue. ++ */ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0) ++ { ++ struct mali_deferred_fence_put_entry *obj; ++ ++ obj = kzalloc(sizeof(struct mali_deferred_fence_put_entry), GFP_ATOMIC); ++ if (obj) { ++ unsigned long flags; ++ mali_bool schedule = MALI_FALSE; ++ ++ obj->fence = sync_fence; ++ ++ spin_lock_irqsave(&mali_timeline_sync_fence_to_free_lock, flags); ++ if (hlist_empty(&mali_timeline_sync_fence_to_free_list)) ++ schedule = MALI_TRUE; ++ hlist_add_head(&obj->list, &mali_timeline_sync_fence_to_free_list); ++ spin_unlock_irqrestore(&mali_timeline_sync_fence_to_free_lock, flags); ++ ++ if (schedule) ++ schedule_delayed_work(&delayed_sync_fence_put, 0); ++ } ++ } ++#else ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) ++ sync_fence_put(sync_fence); ++#else ++ fput(sync_fence->file); ++#endif ++#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0) */ ++ ++ if (!is_aborting) { ++ mali_executor_schedule_from_mask(schedule_mask, MALI_TRUE); ++ } ++ } ++} ++#endif ++_mali_osk_errcode_t mali_timeline_initialize(void) ++{ ++ _mali_osk_atomic_init(&gp_tracker_count, 0); ++ _mali_osk_atomic_init(&phy_pp_tracker_count, 0); ++ _mali_osk_atomic_init(&virt_pp_tracker_count, 0); ++ ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ sync_fence_callback_list_lock = _mali_osk_spinlock_irq_init(_MALI_OSK_LOCKFLAG_UNORDERED, _MALI_OSK_LOCK_ORDER_FIRST); ++ if (NULL == sync_fence_callback_list_lock) { ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ ++ sync_fence_callback_work_t = _mali_osk_wq_create_work( ++ mali_timeline_do_sync_fence_callback, NULL); ++ ++ if (NULL == sync_fence_callback_work_t) { ++ return _MALI_OSK_ERR_FAULT; ++ } ++#endif ++ return _MALI_OSK_ERR_OK; ++} ++ ++ ++void mali_timeline_terminate(void) ++{ ++ _mali_osk_atomic_term(&gp_tracker_count); ++ _mali_osk_atomic_term(&phy_pp_tracker_count); ++ _mali_osk_atomic_term(&virt_pp_tracker_count); ++ ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ if (NULL != sync_fence_callback_list_lock) { ++ _mali_osk_spinlock_irq_term(sync_fence_callback_list_lock); ++ sync_fence_callback_list_lock = NULL; ++ } ++ ++ if (NULL != sync_fence_callback_work_t) { ++ _mali_osk_wq_delete_work(sync_fence_callback_work_t); ++ sync_fence_callback_work_t = NULL; ++ } ++#endif ++} ++ ++#if defined(MALI_TIMELINE_DEBUG_FUNCTIONS) ++ ++static mali_bool is_waiting_on_timeline(struct mali_timeline_tracker *tracker, enum mali_timeline_id id) ++{ ++ struct mali_timeline *timeline; ++ struct mali_timeline_system *system; ++ ++ MALI_DEBUG_ASSERT_POINTER(tracker); ++ ++ MALI_DEBUG_ASSERT_POINTER(tracker->timeline); ++ timeline = tracker->timeline; ++ ++ MALI_DEBUG_ASSERT_POINTER(timeline->system); ++ system = timeline->system; ++ ++ if (MALI_TIMELINE_MAX > id) { ++ if (MALI_TIMELINE_NO_POINT != tracker->fence.points[id]) { ++ return mali_timeline_is_point_on(system->timelines[id], tracker->fence.points[id]); ++ } else { ++ return MALI_FALSE; ++ } ++ } else { ++ MALI_DEBUG_ASSERT(MALI_TIMELINE_NONE == id); ++ return MALI_FALSE; ++ } ++} ++ ++static const char *timeline_id_to_string(enum mali_timeline_id id) ++{ ++ switch (id) { ++ case MALI_TIMELINE_GP: ++ return "GP"; ++ case MALI_TIMELINE_PP: ++ return "PP"; ++ case MALI_TIMELINE_SOFT: ++ return "SOFT"; ++ default: ++ return "NONE"; ++ } ++} ++ ++static const char *timeline_tracker_type_to_string(enum mali_timeline_tracker_type type) ++{ ++ switch (type) { ++ case MALI_TIMELINE_TRACKER_GP: ++ return "GP"; ++ case MALI_TIMELINE_TRACKER_PP: ++ return "PP"; ++ case MALI_TIMELINE_TRACKER_SOFT: ++ return "SOFT"; ++ case MALI_TIMELINE_TRACKER_WAIT: ++ return "WAIT"; ++ case MALI_TIMELINE_TRACKER_SYNC: ++ return "SYNC"; ++ default: ++ return "INVALID"; ++ } ++} ++ ++mali_timeline_tracker_state mali_timeline_debug_get_tracker_state(struct mali_timeline_tracker *tracker) ++{ ++ struct mali_timeline *timeline = NULL; ++ ++ MALI_DEBUG_ASSERT_POINTER(tracker); ++ timeline = tracker->timeline; ++ ++ if (0 != tracker->trigger_ref_count) { ++ return MALI_TIMELINE_TS_WAITING; ++ } ++ ++ if (timeline && (timeline->tracker_tail == tracker || NULL != tracker->timeline_prev)) { ++ return MALI_TIMELINE_TS_ACTIVE; ++ } ++ ++ if (timeline && (MALI_TIMELINE_NO_POINT == tracker->point)) { ++ return MALI_TIMELINE_TS_INIT; ++ } ++ ++ return MALI_TIMELINE_TS_FINISH; ++} ++ ++void mali_timeline_debug_print_tracker(struct mali_timeline_tracker *tracker, _mali_osk_print_ctx *print_ctx) ++{ ++ const char *tracker_state = "IWAF"; ++ char state_char = 'I'; ++ char tracker_type[32] = {0}; ++ ++ MALI_DEBUG_ASSERT_POINTER(tracker); ++ ++ state_char = *(tracker_state + mali_timeline_debug_get_tracker_state(tracker)); ++ _mali_osk_snprintf(tracker_type, sizeof(tracker_type), "%s", timeline_tracker_type_to_string(tracker->type)); ++ ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ if (0 != tracker->trigger_ref_count) { ++ if (print_ctx) ++ _mali_osk_ctxprintf(print_ctx, "TL: %s %u %c - ref_wait:%u [%s(%u),%s(%u),%s(%u), fd:%d, fence:(0x%08X)] job:(0x%08X)\n", ++ tracker_type, tracker->point, state_char, tracker->trigger_ref_count, ++ is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "WaitGP" : " ", tracker->fence.points[0], ++ is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "WaitPP" : " ", tracker->fence.points[1], ++ is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "WaitSOFT" : " ", tracker->fence.points[2], ++ tracker->fence.sync_fd, (unsigned int)(uintptr_t)(tracker->sync_fence), (unsigned int)(uintptr_t)(tracker->job)); ++ else ++ MALI_DEBUG_PRINT(2, ("TL: %s %u %c - ref_wait:%u [%s(%u),%s(%u),%s(%u), fd:%d, fence:(0x%08X)] job:(0x%08X)\n", ++ tracker_type, tracker->point, state_char, tracker->trigger_ref_count, ++ is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "WaitGP" : " ", tracker->fence.points[0], ++ is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "WaitPP" : " ", tracker->fence.points[1], ++ is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "WaitSOFT" : " ", tracker->fence.points[2], ++ tracker->fence.sync_fd, (unsigned int)(uintptr_t)(tracker->sync_fence), (unsigned int)(uintptr_t)(tracker->job))); ++ } else { ++ if (print_ctx) ++ _mali_osk_ctxprintf(print_ctx, "TL: %s %u %c fd:%d fence:(0x%08X) job:(0x%08X)\n", ++ tracker_type, tracker->point, state_char, ++ tracker->fence.sync_fd, (unsigned int)(uintptr_t)(tracker->sync_fence), (unsigned int)(uintptr_t)(tracker->job)); ++ else ++ MALI_DEBUG_PRINT(2, ("TL: %s %u %c fd:%d fence:(0x%08X) job:(0x%08X)\n", ++ tracker_type, tracker->point, state_char, ++ tracker->fence.sync_fd, (unsigned int)(uintptr_t)(tracker->sync_fence), (unsigned int)(uintptr_t)(tracker->job))); ++ ++ } ++#else ++ if (0 != tracker->trigger_ref_count) { ++ if (print_ctx) ++ _mali_osk_ctxprintf(print_ctx, "TL: %s %u %c - ref_wait:%u [%s(%u),%s(%u),%s(%u)] job:(0x%08X)\n", ++ tracker_type, tracker->point, state_char, tracker->trigger_ref_count, ++ is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "WaitGP" : " ", tracker->fence.points[0], ++ is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "WaitPP" : " ", tracker->fence.points[1], ++ is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "WaitSOFT" : " ", tracker->fence.points[2], ++ (unsigned int)(uintptr_t)(tracker->job)); ++ else ++ MALI_DEBUG_PRINT(2, ("TL: %s %u %c - ref_wait:%u [%s(%u),%s(%u),%s(%u)] job:(0x%08X)\n", ++ tracker_type, tracker->point, state_char, tracker->trigger_ref_count, ++ is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "WaitGP" : " ", tracker->fence.points[0], ++ is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "WaitPP" : " ", tracker->fence.points[1], ++ is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "WaitSOFT" : " ", tracker->fence.points[2], ++ (unsigned int)(uintptr_t)(tracker->job))); ++ } else { ++ if (print_ctx) ++ _mali_osk_ctxprintf(print_ctx, "TL: %s %u %c job:(0x%08X)\n", ++ tracker_type, tracker->point, state_char, ++ (unsigned int)(uintptr_t)(tracker->job)); ++ else ++ MALI_DEBUG_PRINT(2, ("TL: %s %u %c job:(0x%08X)\n", ++ tracker_type, tracker->point, state_char, ++ (unsigned int)(uintptr_t)(tracker->job))); ++ ++ } ++#endif ++} ++ ++void mali_timeline_debug_print_timeline(struct mali_timeline *timeline, _mali_osk_print_ctx *print_ctx) ++{ ++ struct mali_timeline_tracker *tracker = NULL; ++ ++ MALI_DEBUG_ASSERT_POINTER(timeline); ++ ++ tracker = timeline->tracker_tail; ++ while (NULL != tracker) { ++ mali_timeline_debug_print_tracker(tracker, print_ctx); ++ tracker = tracker->timeline_next; ++ } ++} ++ ++#if !(LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0)) ++void mali_timeline_debug_direct_print_tracker(struct mali_timeline_tracker *tracker) ++{ ++ const char *tracker_state = "IWAF"; ++ char state_char = 'I'; ++ char tracker_type[32] = {0}; ++ ++ MALI_DEBUG_ASSERT_POINTER(tracker); ++ ++ state_char = *(tracker_state + mali_timeline_debug_get_tracker_state(tracker)); ++ _mali_osk_snprintf(tracker_type, sizeof(tracker_type), "%s", timeline_tracker_type_to_string(tracker->type)); ++ ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ if (0 != tracker->trigger_ref_count) { ++ MALI_PRINT(("TL: %s %u %c - ref_wait:%u [%s(%u),%s(%u),%s(%u), fd:%d, fence:(0x%08X)] job:(0x%08X)\n", ++ tracker_type, tracker->point, state_char, tracker->trigger_ref_count, ++ is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "WaitGP" : " ", tracker->fence.points[0], ++ is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "WaitPP" : " ", tracker->fence.points[1], ++ is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "WaitSOFT" : " ", tracker->fence.points[2], ++ tracker->fence.sync_fd, tracker->sync_fence, tracker->job)); ++ } else { ++ MALI_PRINT(("TL: %s %u %c fd:%d fence:(0x%08X) job:(0x%08X)\n", ++ tracker_type, tracker->point, state_char, ++ tracker->fence.sync_fd, tracker->sync_fence, tracker->job)); ++ } ++#else ++ if (0 != tracker->trigger_ref_count) { ++ MALI_PRINT(("TL: %s %u %c - ref_wait:%u [%s(%u),%s(%u),%s(%u)] job:(0x%08X)\n", ++ tracker_type, tracker->point, state_char, tracker->trigger_ref_count, ++ is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "WaitGP" : " ", tracker->fence.points[0], ++ is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "WaitPP" : " ", tracker->fence.points[1], ++ is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "WaitSOFT" : " ", tracker->fence.points[2], ++ tracker->job)); ++ } else { ++ MALI_PRINT(("TL: %s %u %c job:(0x%08X)\n", ++ tracker_type, tracker->point, state_char, ++ tracker->job)); ++ } ++#endif ++} ++ ++void mali_timeline_debug_direct_print_timeline(struct mali_timeline *timeline) ++{ ++ struct mali_timeline_tracker *tracker = NULL; ++ ++ MALI_DEBUG_ASSERT_POINTER(timeline); ++ ++ tracker = timeline->tracker_tail; ++ while (NULL != tracker) { ++ mali_timeline_debug_direct_print_tracker(tracker); ++ tracker = tracker->timeline_next; ++ } ++} ++ ++#endif ++ ++void mali_timeline_debug_print_system(struct mali_timeline_system *system, _mali_osk_print_ctx *print_ctx) ++{ ++ int i; ++ int num_printed = 0; ++ u32 tid = _mali_osk_get_tid(); ++ ++ MALI_DEBUG_ASSERT_POINTER(system); ++ ++ mali_spinlock_reentrant_wait(system->spinlock, tid); ++ ++ /* Print all timelines */ ++ for (i = 0; i < MALI_TIMELINE_MAX; ++i) { ++ struct mali_timeline *timeline = system->timelines[i]; ++ ++ MALI_DEBUG_ASSERT_POINTER(timeline); ++ ++ if (NULL == timeline->tracker_head) continue; ++ if (print_ctx) ++ _mali_osk_ctxprintf(print_ctx, "TL: Timeline %s:\n", ++ timeline_id_to_string((enum mali_timeline_id)i)); ++ else ++ MALI_DEBUG_PRINT(2, ("TL: Timeline %s: oldest (%u) next(%u)\n", ++ timeline_id_to_string((enum mali_timeline_id)i), timeline->point_oldest, timeline->point_next)); ++ ++ mali_timeline_debug_print_timeline(timeline, print_ctx); ++ num_printed++; ++ } ++ ++ if (0 == num_printed) { ++ if (print_ctx) ++ _mali_osk_ctxprintf(print_ctx, "TL: All timelines empty\n"); ++ else ++ MALI_DEBUG_PRINT(2, ("TL: All timelines empty\n")); ++ } ++ ++ mali_spinlock_reentrant_signal(system->spinlock, tid); ++} ++ ++#endif /* defined(MALI_TIMELINE_DEBUG_FUNCTIONS) */ ++ ++#if defined(CONFIG_MALI_DMA_BUF_FENCE) ++void mali_timeline_dma_fence_callback(void *pp_job_ptr) ++{ ++ struct mali_timeline_system *system; ++ struct mali_timeline_waiter *waiter; ++ struct mali_timeline_tracker *tracker; ++ struct mali_pp_job *pp_job = (struct mali_pp_job *)pp_job_ptr; ++ mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; ++ u32 tid = _mali_osk_get_tid(); ++ mali_bool is_aborting = MALI_FALSE; ++ ++ MALI_DEBUG_ASSERT_POINTER(pp_job); ++ ++ tracker = &pp_job->tracker; ++ MALI_DEBUG_ASSERT_POINTER(tracker); ++ ++ system = tracker->system; ++ MALI_DEBUG_ASSERT_POINTER(system); ++ MALI_DEBUG_ASSERT_POINTER(system->session); ++ ++ mali_spinlock_reentrant_wait(system->spinlock, tid); ++ ++ waiter = tracker->waiter_dma_fence; ++ MALI_DEBUG_ASSERT_POINTER(waiter); ++ ++ schedule_mask |= mali_timeline_system_release_waiter(system, waiter); ++ ++ is_aborting = system->session->is_aborting; ++ ++ /* If aborting, wake up sleepers that are waiting for dma fence callbacks to complete. */ ++ if (is_aborting) { ++ _mali_osk_wait_queue_wake_up(system->wait_queue); ++ } ++ ++ mali_spinlock_reentrant_signal(system->spinlock, tid); ++ ++ if (!is_aborting) { ++ mali_executor_schedule_from_mask(schedule_mask, MALI_TRUE); ++ } ++} ++#endif +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_timeline.h b/drivers/gpu/arm/mali400/mali/common/mali_timeline.h +new file mode 100755 +index 000000000000..3e8bfc8fb733 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_timeline.h +@@ -0,0 +1,587 @@ ++/* ++ * Copyright (C) 2013-2018 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_TIMELINE_H__ ++#define __MALI_TIMELINE_H__ ++ ++#include "mali_osk.h" ++#include "mali_ukk.h" ++#include "mali_session.h" ++#include "mali_kernel_common.h" ++#include "mali_spinlock_reentrant.h" ++#include "mali_sync.h" ++#include "mali_scheduler_types.h" ++#include ++ ++/** ++ * Soft job timeout. ++ * ++ * Soft jobs have to be signaled as complete after activation. Normally this is done by user space, ++ * but in order to guarantee that every soft job is completed, we also have a timer. ++ */ ++#define MALI_TIMELINE_TIMEOUT_HZ ((unsigned long) (HZ * 3 / 2)) /* 1500 ms. */ ++ ++/** ++ * Timeline type. ++ */ ++typedef enum mali_timeline_id { ++ MALI_TIMELINE_GP = MALI_UK_TIMELINE_GP, /**< GP job timeline. */ ++ MALI_TIMELINE_PP = MALI_UK_TIMELINE_PP, /**< PP job timeline. */ ++ MALI_TIMELINE_SOFT = MALI_UK_TIMELINE_SOFT, /**< Soft job timeline. */ ++ MALI_TIMELINE_MAX = MALI_UK_TIMELINE_MAX ++} mali_timeline_id; ++ ++/** ++ * Used by trackers that should not be added to a timeline (@ref mali_timeline_system_add_tracker). ++ */ ++#define MALI_TIMELINE_NONE MALI_TIMELINE_MAX ++ ++/** ++ * Tracker type. ++ */ ++typedef enum mali_timeline_tracker_type { ++ MALI_TIMELINE_TRACKER_GP = 0, /**< Tracker used by GP jobs. */ ++ MALI_TIMELINE_TRACKER_PP = 1, /**< Tracker used by PP jobs. */ ++ MALI_TIMELINE_TRACKER_SOFT = 2, /**< Tracker used by soft jobs. */ ++ MALI_TIMELINE_TRACKER_WAIT = 3, /**< Tracker used for fence wait. */ ++ MALI_TIMELINE_TRACKER_SYNC = 4, /**< Tracker used for sync fence. */ ++ MALI_TIMELINE_TRACKER_MAX = 5, ++} mali_timeline_tracker_type; ++ ++/** ++ * Tracker activation error. ++ */ ++typedef u32 mali_timeline_activation_error; ++#define MALI_TIMELINE_ACTIVATION_ERROR_NONE 0 ++#define MALI_TIMELINE_ACTIVATION_ERROR_SYNC_BIT (1<<1) ++#define MALI_TIMELINE_ACTIVATION_ERROR_FATAL_BIT (1<<0) ++ ++/** ++ * Type used to represent a point on a timeline. ++ */ ++typedef u32 mali_timeline_point; ++ ++/** ++ * Used to represent that no point on a timeline. ++ */ ++#define MALI_TIMELINE_NO_POINT ((mali_timeline_point) 0) ++ ++/** ++ * The maximum span of points on a timeline. A timeline will be considered full if the difference ++ * between the oldest and newest points is equal or larger to this value. ++ */ ++#define MALI_TIMELINE_MAX_POINT_SPAN 65536 ++ ++/** ++ * Magic value used to assert on validity of trackers. ++ */ ++#define MALI_TIMELINE_TRACKER_MAGIC 0xabcdabcd ++ ++struct mali_timeline; ++struct mali_timeline_waiter; ++struct mali_timeline_tracker; ++ ++/** ++ * Timeline fence. ++ */ ++struct mali_timeline_fence { ++ mali_timeline_point points[MALI_TIMELINE_MAX]; /**< For each timeline, a point or MALI_TIMELINE_NO_POINT. */ ++ s32 sync_fd; /**< A file descriptor representing a sync fence, or -1. */ ++}; ++ ++/** ++ * Timeline system. ++ * ++ * The Timeline system has a set of timelines associated with a session. ++ */ ++struct mali_timeline_system { ++ struct mali_spinlock_reentrant *spinlock; /**< Spin lock protecting the timeline system */ ++ struct mali_timeline *timelines[MALI_TIMELINE_MAX]; /**< The timelines in this system */ ++ ++ /* Single-linked list of unused waiter objects. Uses the tracker_next field in tracker. */ ++ struct mali_timeline_waiter *waiter_empty_list; ++ ++ struct mali_session_data *session; /**< Session that owns this system. */ ++ ++ mali_bool timer_enabled; /**< Set to MALI_TRUE if soft job timer should be enabled, MALI_FALSE if not. */ ++ ++ _mali_osk_wait_queue_t *wait_queue; /**< Wait queue. */ ++ ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) ++ struct sync_timeline *signaled_sync_tl; /**< Special sync timeline used to create pre-signaled sync fences */ ++#else ++ struct mali_internal_sync_timeline *signaled_sync_tl; /**< Special sync timeline used to create pre-signaled sync fences */ ++#endif ++#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ ++}; ++ ++/** ++ * Timeline. Each Timeline system will have MALI_TIMELINE_MAX timelines. ++ */ ++struct mali_timeline { ++ mali_timeline_point point_next; /**< The next available point. */ ++ mali_timeline_point point_oldest; /**< The oldest point not released. */ ++ ++ /* Double-linked list of trackers. Sorted in ascending order by tracker->time_number with ++ * tail pointing to the tracker with the oldest time. */ ++ struct mali_timeline_tracker *tracker_head; ++ struct mali_timeline_tracker *tracker_tail; ++ ++ /* Double-linked list of waiters. Sorted in ascending order by waiter->time_number_wait ++ * with tail pointing to the waiter with oldest wait time. */ ++ struct mali_timeline_waiter *waiter_head; ++ struct mali_timeline_waiter *waiter_tail; ++ ++ struct mali_timeline_system *system; /**< Timeline system this timeline belongs to. */ ++ enum mali_timeline_id id; /**< Timeline type. */ ++ ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) ++ struct sync_timeline *sync_tl; /**< Sync timeline that corresponds to this timeline. */ ++#else ++ struct mali_internal_sync_timeline *sync_tl; ++#endif ++ mali_bool destroyed; ++ struct mali_spinlock_reentrant *spinlock; /**< Spin lock protecting the timeline system */ ++#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ ++ ++ /* The following fields are used to time out soft job trackers. */ ++ _mali_osk_wq_delayed_work_t *delayed_work; ++ mali_bool timer_active; ++}; ++ ++/** ++ * Timeline waiter. ++ */ ++struct mali_timeline_waiter { ++ mali_timeline_point point; /**< Point on timeline we are waiting for to be released. */ ++ struct mali_timeline_tracker *tracker; /**< Tracker that is waiting. */ ++ ++ struct mali_timeline_waiter *timeline_next; /**< Next waiter on timeline's waiter list. */ ++ struct mali_timeline_waiter *timeline_prev; /**< Previous waiter on timeline's waiter list. */ ++ ++ struct mali_timeline_waiter *tracker_next; /**< Next waiter on tracker's waiter list. */ ++}; ++ ++/** ++ * Timeline tracker. ++ */ ++struct mali_timeline_tracker { ++ MALI_DEBUG_CODE(u32 magic); /**< Should always be MALI_TIMELINE_TRACKER_MAGIC for a valid tracker. */ ++ ++ mali_timeline_point point; /**< Point on timeline for this tracker */ ++ ++ struct mali_timeline_tracker *timeline_next; /**< Next tracker on timeline's tracker list */ ++ struct mali_timeline_tracker *timeline_prev; /**< Previous tracker on timeline's tracker list */ ++ ++ u32 trigger_ref_count; /**< When zero tracker will be activated */ ++ mali_timeline_activation_error activation_error; /**< Activation error. */ ++ struct mali_timeline_fence fence; /**< Fence used to create this tracker */ ++ ++ /* Single-linked list of waiters. Sorted in order of insertions with ++ * tail pointing to first waiter. */ ++ struct mali_timeline_waiter *waiter_head; ++ struct mali_timeline_waiter *waiter_tail; ++ ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ /* These are only used if the tracker is waiting on a sync fence. */ ++ struct mali_timeline_waiter *waiter_sync; /**< A direct pointer to timeline waiter representing sync fence. */ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) ++ struct sync_fence_waiter sync_fence_waiter; /**< Used to connect sync fence and tracker in sync fence wait callback. */ ++ struct sync_fence *sync_fence; /**< The sync fence this tracker is waiting on. */ ++#else ++ struct mali_internal_sync_fence_waiter sync_fence_waiter; /**< Used to connect sync fence and tracker in sync fence wait callback. */ ++ struct mali_internal_sync_fence *sync_fence; /**< The sync fence this tracker is waiting on. */ ++#endif ++ _mali_osk_list_t sync_fence_cancel_list; /**< List node used to cancel sync fence waiters. */ ++ _mali_osk_list_t sync_fence_signal_list; /** < List node used to singal sync fence callback function. */ ++ ++#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ ++ ++#if defined(CONFIG_MALI_DMA_BUF_FENCE) ++ struct mali_timeline_waiter *waiter_dma_fence; /**< A direct pointer to timeline waiter representing dma fence. */ ++#endif ++ ++ struct mali_timeline_system *system; /**< Timeline system. */ ++ struct mali_timeline *timeline; /**< Timeline, or NULL if not on a timeline. */ ++ enum mali_timeline_tracker_type type; /**< Type of tracker. */ ++ void *job; /**< Owner of tracker. */ ++ ++ /* The following fields are used to time out soft job trackers. */ ++ unsigned long os_tick_create; ++ unsigned long os_tick_activate; ++ mali_bool timer_active; ++}; ++ ++extern _mali_osk_atomic_t gp_tracker_count; ++extern _mali_osk_atomic_t phy_pp_tracker_count; ++extern _mali_osk_atomic_t virt_pp_tracker_count; ++ ++/** ++ * What follows is a set of functions to check the state of a timeline and to determine where on a ++ * timeline a given point is. Most of these checks will translate the timeline so the oldest point ++ * on the timeline is aligned with zero. Remember that all of these calculation are done on ++ * unsigned integers. ++ * ++ * The following example illustrates the three different states a point can be in. The timeline has ++ * been translated to put the oldest point at zero: ++ * ++ * ++ * ++ * [ point is in forbidden zone ] ++ * 64k wide ++ * MALI_TIMELINE_MAX_POINT_SPAN ++ * ++ * [ point is on timeline ) ( point is released ] ++ * ++ * 0--------------------------##############################--------------------2^32 - 1 ++ * ^ ^ ++ * \ | ++ * oldest point on timeline | ++ * \ ++ * next point on timeline ++ */ ++ ++/** ++ * Compare two timeline points ++ * ++ * Returns true if a is after b, false if a is before or equal to b. ++ * ++ * This funcion ignores MALI_TIMELINE_MAX_POINT_SPAN. Wrapping is supported and ++ * the result will be correct if the points is less then UINT_MAX/2 apart. ++ * ++ * @param a Point on timeline ++ * @param b Point on timeline ++ * @return MALI_TRUE if a is after b ++ */ ++MALI_STATIC_INLINE mali_bool mali_timeline_point_after(mali_timeline_point a, mali_timeline_point b) ++{ ++ return 0 > ((s32)b) - ((s32)a); ++} ++ ++/** ++ * Check if a point is on timeline. A point is on a timeline if it is greater than, or equal to, ++ * the oldest point, and less than the next point. ++ * ++ * @param timeline Timeline. ++ * @param point Point on timeline. ++ * @return MALI_TRUE if point is on timeline, MALI_FALSE if not. ++ */ ++MALI_STATIC_INLINE mali_bool mali_timeline_is_point_on(struct mali_timeline *timeline, mali_timeline_point point) ++{ ++ MALI_DEBUG_ASSERT_POINTER(timeline); ++ MALI_DEBUG_ASSERT(MALI_TIMELINE_NO_POINT != point); ++ ++ return (point - timeline->point_oldest) < (timeline->point_next - timeline->point_oldest); ++} ++ ++/** ++ * Check if a point has been released. A point is released if it is older than the oldest point on ++ * the timeline, newer than the next point, and also not in the forbidden zone. ++ * ++ * @param timeline Timeline. ++ * @param point Point on timeline. ++ * @return MALI_TRUE if point has been release, MALI_FALSE if not. ++ */ ++MALI_STATIC_INLINE mali_bool mali_timeline_is_point_released(struct mali_timeline *timeline, mali_timeline_point point) ++{ ++ mali_timeline_point point_normalized; ++ mali_timeline_point next_normalized; ++ ++ MALI_DEBUG_ASSERT_POINTER(timeline); ++ MALI_DEBUG_ASSERT(MALI_TIMELINE_NO_POINT != point); ++ ++ point_normalized = point - timeline->point_oldest; ++ next_normalized = timeline->point_next - timeline->point_oldest; ++ ++ return point_normalized > (next_normalized + MALI_TIMELINE_MAX_POINT_SPAN); ++} ++ ++/** ++ * Check if the tracker that the point relate to has been released. A point is released if the tracker is not on the timeline. ++ * @param timeline Timeline. ++ * @param point Point on timeline. ++ * @return MALI_TRUE if the tracker has been release, MALI_FALSE if not. ++ */ ++MALI_STATIC_INLINE mali_bool mali_timeline_is_tracker_released(struct mali_timeline *timeline, mali_timeline_point point) ++{ ++ struct mali_timeline_tracker *tracker; ++ ++ MALI_DEBUG_ASSERT_POINTER(timeline); ++ MALI_DEBUG_ASSERT(MALI_TIMELINE_NO_POINT != point); ++ ++ tracker = timeline->tracker_tail; ++ ++ while (NULL != tracker) { ++ if (point == tracker->point) ++ return MALI_FALSE; ++ tracker = tracker->timeline_next; ++ } ++ ++ return MALI_TRUE; ++} ++ ++/** ++ * Check if a point is valid. A point is valid if is on the timeline or has been released. ++ * ++ * @param timeline Timeline. ++ * @param point Point on timeline. ++ * @return MALI_TRUE if point is valid, MALI_FALSE if not. ++ */ ++MALI_STATIC_INLINE mali_bool mali_timeline_is_point_valid(struct mali_timeline *timeline, mali_timeline_point point) ++{ ++ MALI_DEBUG_ASSERT_POINTER(timeline); ++ return mali_timeline_is_point_on(timeline, point) || mali_timeline_is_point_released(timeline, point); ++} ++ ++/** ++ * Check if timeline is empty (has no points on it). A timeline is empty if next == oldest. ++ * ++ * @param timeline Timeline. ++ * @return MALI_TRUE if timeline is empty, MALI_FALSE if not. ++ */ ++MALI_STATIC_INLINE mali_bool mali_timeline_is_empty(struct mali_timeline *timeline) ++{ ++ MALI_DEBUG_ASSERT_POINTER(timeline); ++ return timeline->point_next == timeline->point_oldest; ++} ++ ++/** ++ * Check if timeline is full. A valid timeline cannot span more than 64k points (@ref ++ * MALI_TIMELINE_MAX_POINT_SPAN). ++ * ++ * @param timeline Timeline. ++ * @return MALI_TRUE if timeline is full, MALI_FALSE if not. ++ */ ++MALI_STATIC_INLINE mali_bool mali_timeline_is_full(struct mali_timeline *timeline) ++{ ++ MALI_DEBUG_ASSERT_POINTER(timeline); ++ return MALI_TIMELINE_MAX_POINT_SPAN <= (timeline->point_next - timeline->point_oldest); ++} ++ ++/** ++ * Create a new timeline system. ++ * ++ * @param session The session this timeline system will belong to. ++ * @return New timeline system. ++ */ ++struct mali_timeline_system *mali_timeline_system_create(struct mali_session_data *session); ++ ++/** ++ * Abort timeline system. ++ * ++ * This will release all pending waiters in the timeline system causing all trackers to be ++ * activated. ++ * ++ * @param system Timeline system to abort all jobs from. ++ */ ++void mali_timeline_system_abort(struct mali_timeline_system *system); ++ ++/** ++ * Destroy an empty timeline system. ++ * ++ * @note @ref mali_timeline_system_abort() should be called prior to this function. ++ * ++ * @param system Timeline system to destroy. ++ */ ++void mali_timeline_system_destroy(struct mali_timeline_system *system); ++ ++/** ++ * Stop the soft job timer. ++ * ++ * @param system Timeline system ++ */ ++void mali_timeline_system_stop_timer(struct mali_timeline_system *system); ++ ++/** ++ * Add a tracker to a timeline system and optionally also on a timeline. ++ * ++ * Once added to the timeline system, the tracker is guaranteed to be activated. The tracker can be ++ * activated before this function returns. Thus, it is also possible that the tracker is released ++ * before this function returns, depending on the tracker type. ++ * ++ * @note Tracker must be initialized (@ref mali_timeline_tracker_init) before being added to the ++ * timeline system. ++ * ++ * @param system Timeline system the tracker will be added to. ++ * @param tracker The tracker to be added. ++ * @param timeline_id Id of the timeline the tracker will be added to, or ++ * MALI_TIMELINE_NONE if it should not be added on a timeline. ++ * @return Point on timeline identifying this tracker, or MALI_TIMELINE_NO_POINT if not on timeline. ++ */ ++mali_timeline_point mali_timeline_system_add_tracker(struct mali_timeline_system *system, ++ struct mali_timeline_tracker *tracker, ++ enum mali_timeline_id timeline_id); ++ ++/** ++ * Get latest point on timeline. ++ * ++ * @param system Timeline system. ++ * @param timeline_id Id of timeline to get latest point from. ++ * @return Latest point on timeline, or MALI_TIMELINE_NO_POINT if the timeline is empty. ++ */ ++mali_timeline_point mali_timeline_system_get_latest_point(struct mali_timeline_system *system, ++ enum mali_timeline_id timeline_id); ++ ++/** ++ * Initialize tracker. ++ * ++ * Must be called before tracker is added to timeline system (@ref mali_timeline_system_add_tracker). ++ * ++ * @param tracker Tracker to initialize. ++ * @param type Type of tracker. ++ * @param fence Fence used to set up dependencies for tracker. ++ * @param job Pointer to job struct this tracker is associated with. ++ */ ++void mali_timeline_tracker_init(struct mali_timeline_tracker *tracker, ++ mali_timeline_tracker_type type, ++ struct mali_timeline_fence *fence, ++ void *job); ++ ++/** ++ * Grab trigger ref count on tracker. ++ * ++ * This will prevent tracker from being activated until the trigger ref count reaches zero. ++ * ++ * @note Tracker must have been initialized (@ref mali_timeline_tracker_init). ++ * ++ * @param system Timeline system. ++ * @param tracker Tracker. ++ */ ++void mali_timeline_system_tracker_get(struct mali_timeline_system *system, struct mali_timeline_tracker *tracker); ++ ++/** ++ * Release trigger ref count on tracker. ++ * ++ * If the trigger ref count reaches zero, the tracker will be activated. ++ * ++ * @param system Timeline system. ++ * @param tracker Tracker. ++ * @param activation_error Error bitmask if activated with error, or MALI_TIMELINE_ACTIVATION_ERROR_NONE if no error. ++ * @return Scheduling bitmask. ++ */ ++mali_scheduler_mask mali_timeline_system_tracker_put(struct mali_timeline_system *system, struct mali_timeline_tracker *tracker, mali_timeline_activation_error activation_error); ++ ++/** ++ * Release a tracker from the timeline system. ++ * ++ * This is used to signal that the job being tracker is finished, either due to normal circumstances ++ * (job complete/abort) or due to a timeout. ++ * ++ * We may need to schedule some subsystems after a tracker has been released and the returned ++ * bitmask will tell us if it is necessary. If the return value is non-zero, this value needs to be ++ * sent as an input parameter to @ref mali_scheduler_schedule_from_mask() to do the scheduling. ++ * ++ * @note Tracker must have been activated before being released. ++ * @warning Not calling @ref mali_scheduler_schedule_from_mask() after releasing a tracker can lead ++ * to a deadlock. ++ * ++ * @param tracker Tracker being released. ++ * @return Scheduling bitmask. ++ */ ++mali_scheduler_mask mali_timeline_tracker_release(struct mali_timeline_tracker *tracker); ++ ++MALI_STATIC_INLINE mali_bool mali_timeline_tracker_activation_error( ++ struct mali_timeline_tracker *tracker) ++{ ++ MALI_DEBUG_ASSERT_POINTER(tracker); ++ return (MALI_TIMELINE_ACTIVATION_ERROR_FATAL_BIT & ++ tracker->activation_error) ? MALI_TRUE : MALI_FALSE; ++} ++ ++/** ++ * Copy data from a UK fence to a Timeline fence. ++ * ++ * @param fence Timeline fence. ++ * @param uk_fence UK fence. ++ */ ++void mali_timeline_fence_copy_uk_fence(struct mali_timeline_fence *fence, _mali_uk_fence_t *uk_fence); ++ ++_mali_osk_errcode_t mali_timeline_initialize(void); ++ ++void mali_timeline_terminate(void); ++ ++MALI_STATIC_INLINE mali_bool mali_timeline_has_gp_job(void) ++{ ++ return 0 < _mali_osk_atomic_read(&gp_tracker_count); ++} ++ ++MALI_STATIC_INLINE mali_bool mali_timeline_has_physical_pp_job(void) ++{ ++ return 0 < _mali_osk_atomic_read(&phy_pp_tracker_count); ++} ++ ++MALI_STATIC_INLINE mali_bool mali_timeline_has_virtual_pp_job(void) ++{ ++ return 0 < _mali_osk_atomic_read(&virt_pp_tracker_count); ++} ++ ++#if defined(DEBUG) ++#define MALI_TIMELINE_DEBUG_FUNCTIONS ++#endif /* DEBUG */ ++#if defined(MALI_TIMELINE_DEBUG_FUNCTIONS) ++ ++/** ++ * Tracker state. Used for debug printing. ++ */ ++typedef enum mali_timeline_tracker_state { ++ MALI_TIMELINE_TS_INIT = 0, ++ MALI_TIMELINE_TS_WAITING = 1, ++ MALI_TIMELINE_TS_ACTIVE = 2, ++ MALI_TIMELINE_TS_FINISH = 3, ++} mali_timeline_tracker_state; ++ ++/** ++ * Get tracker state. ++ * ++ * @param tracker Tracker to check. ++ * @return State of tracker. ++ */ ++mali_timeline_tracker_state mali_timeline_debug_get_tracker_state(struct mali_timeline_tracker *tracker); ++ ++/** ++ * Print debug information about tracker. ++ * ++ * @param tracker Tracker to print. ++ */ ++void mali_timeline_debug_print_tracker(struct mali_timeline_tracker *tracker, _mali_osk_print_ctx *print_ctx); ++ ++/** ++ * Print debug information about timeline. ++ * ++ * @param timeline Timeline to print. ++ */ ++void mali_timeline_debug_print_timeline(struct mali_timeline *timeline, _mali_osk_print_ctx *print_ctx); ++ ++#if !(LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0)) ++void mali_timeline_debug_direct_print_tracker(struct mali_timeline_tracker *tracker); ++void mali_timeline_debug_direct_print_timeline(struct mali_timeline *timeline); ++#endif ++ ++/** ++ * Print debug information about timeline system. ++ * ++ * @param system Timeline system to print. ++ */ ++void mali_timeline_debug_print_system(struct mali_timeline_system *system, _mali_osk_print_ctx *print_ctx); ++ ++#endif /* defined(MALI_TIMELINE_DEBUG_FUNCTIONS) */ ++ ++#if defined(CONFIG_MALI_DMA_BUF_FENCE) ++/** ++ * The timeline dma fence callback when dma fence signal. ++ * ++ * @param pp_job_ptr The pointer to pp job that link to the signaled dma fence. ++ */ ++void mali_timeline_dma_fence_callback(void *pp_job_ptr); ++#endif ++ ++#endif /* __MALI_TIMELINE_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_timeline_fence_wait.c b/drivers/gpu/arm/mali400/mali/common/mali_timeline_fence_wait.c +new file mode 100755 +index 000000000000..1ab13f50997f +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_timeline_fence_wait.c +@@ -0,0 +1,218 @@ ++/* ++ * Copyright (C) 2013-2014, 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++#include ++#include "mali_timeline_fence_wait.h" ++#include "mali_osk.h" ++#include "mali_kernel_common.h" ++#include "mali_spinlock_reentrant.h" ++ ++/** ++ * Allocate a fence waiter tracker. ++ * ++ * @return New fence waiter if successful, NULL if not. ++ */ ++static struct mali_timeline_fence_wait_tracker *mali_timeline_fence_wait_tracker_alloc(void) ++{ ++ return (struct mali_timeline_fence_wait_tracker *) _mali_osk_calloc(1, sizeof(struct mali_timeline_fence_wait_tracker)); ++} ++ ++/** ++ * Free fence waiter tracker. ++ * ++ * @param wait Fence wait tracker to free. ++ */ ++static void mali_timeline_fence_wait_tracker_free(struct mali_timeline_fence_wait_tracker *wait) ++{ ++ MALI_DEBUG_ASSERT_POINTER(wait); ++ _mali_osk_atomic_term(&wait->refcount); ++ _mali_osk_free(wait); ++} ++ ++/** ++ * Check if fence wait tracker has been activated. Used as a wait queue condition. ++ * ++ * @param data Fence waiter. ++ * @return MALI_TRUE if tracker has been activated, MALI_FALSE if not. ++ */ ++static mali_bool mali_timeline_fence_wait_tracker_is_activated(void *data) ++{ ++ struct mali_timeline_fence_wait_tracker *wait; ++ ++ wait = (struct mali_timeline_fence_wait_tracker *) data; ++ MALI_DEBUG_ASSERT_POINTER(wait); ++ ++ return wait->activated; ++} ++ ++/** ++ * Check if fence has been signaled. ++ * ++ * @param system Timeline system. ++ * @param fence Timeline fence. ++ * @return MALI_TRUE if fence is signaled, MALI_FALSE if not. ++ */ ++static mali_bool mali_timeline_fence_wait_check_status(struct mali_timeline_system *system, struct mali_timeline_fence *fence) ++{ ++ int i; ++ u32 tid = _mali_osk_get_tid(); ++ mali_bool ret = MALI_TRUE; ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) ++ struct sync_fence *sync_fence = NULL; ++#else ++ struct mali_internal_sync_fence *sync_fence = NULL; ++#endif ++#endif ++ ++ MALI_DEBUG_ASSERT_POINTER(system); ++ MALI_DEBUG_ASSERT_POINTER(fence); ++ ++ mali_spinlock_reentrant_wait(system->spinlock, tid); ++ ++ for (i = 0; i < MALI_TIMELINE_MAX; ++i) { ++ struct mali_timeline *timeline; ++ mali_timeline_point point; ++ ++ point = fence->points[i]; ++ ++ if (likely(MALI_TIMELINE_NO_POINT == point)) { ++ /* Fence contains no point on this timeline. */ ++ continue; ++ } ++ ++ timeline = system->timelines[i]; ++ MALI_DEBUG_ASSERT_POINTER(timeline); ++ ++ if (unlikely(!mali_timeline_is_point_valid(timeline, point))) { ++ MALI_PRINT_ERROR(("Mali Timeline: point %d is not valid (oldest=%d, next=%d)\n", point, timeline->point_oldest, timeline->point_next)); ++ } ++ ++ if (!mali_timeline_is_point_released(timeline, point)) { ++ ret = MALI_FALSE; ++ goto exit; ++ } ++ } ++ ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ if (-1 != fence->sync_fd) { ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) ++ sync_fence = sync_fence_fdget(fence->sync_fd); ++#else ++ sync_fence = mali_internal_sync_fence_fdget(fence->sync_fd); ++#endif ++ if (likely(NULL != sync_fence)) { ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) ++ if (0 == sync_fence->status) { ++#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) ++ if (0 < atomic_read(&sync_fence->status)) { ++#else ++ if (0 == sync_fence->fence->ops->signaled(sync_fence->fence)) { ++#endif ++ ret = MALI_FALSE; ++ ++ } else { ++ ret = MALI_TRUE; ++ } ++ } else { ++ MALI_PRINT_ERROR(("Mali Timeline: failed to get sync fence from fd %d\n", fence->sync_fd)); ++ } ++ } ++#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ ++ ++exit: ++ mali_spinlock_reentrant_signal(system->spinlock, tid); ++ ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ if (NULL != sync_fence) { ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) ++ sync_fence_put(sync_fence); ++#else ++ fput(sync_fence->file); ++#endif ++ } ++#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ ++ ++ return ret; ++} ++ ++mali_bool mali_timeline_fence_wait(struct mali_timeline_system *system, struct mali_timeline_fence *fence, u32 timeout) ++{ ++ struct mali_timeline_fence_wait_tracker *wait; ++ mali_timeline_point point; ++ mali_bool ret; ++ ++ MALI_DEBUG_ASSERT_POINTER(system); ++ MALI_DEBUG_ASSERT_POINTER(fence); ++ ++ MALI_DEBUG_PRINT(4, ("Mali Timeline: wait on fence\n")); ++ ++ if (MALI_TIMELINE_FENCE_WAIT_TIMEOUT_IMMEDIATELY == timeout) { ++ return mali_timeline_fence_wait_check_status(system, fence); ++ } ++ ++ wait = mali_timeline_fence_wait_tracker_alloc(); ++ if (unlikely(NULL == wait)) { ++ MALI_PRINT_ERROR(("Mali Timeline: failed to allocate data for fence wait\n")); ++ return MALI_FALSE; ++ } ++ ++ wait->activated = MALI_FALSE; ++ wait->system = system; ++ ++ /* Initialize refcount to two references. The reference first will be released by this ++ * function after the wait is over. The second reference will be released when the tracker ++ * is activated. */ ++ _mali_osk_atomic_init(&wait->refcount, 2); ++ ++ /* Add tracker to timeline system, but not to a timeline. */ ++ mali_timeline_tracker_init(&wait->tracker, MALI_TIMELINE_TRACKER_WAIT, fence, wait); ++ point = mali_timeline_system_add_tracker(system, &wait->tracker, MALI_TIMELINE_NONE); ++ MALI_DEBUG_ASSERT(MALI_TIMELINE_NO_POINT == point); ++ MALI_IGNORE(point); ++ ++ /* Wait for the tracker to be activated or time out. */ ++ if (MALI_TIMELINE_FENCE_WAIT_TIMEOUT_NEVER == timeout) { ++ _mali_osk_wait_queue_wait_event(system->wait_queue, mali_timeline_fence_wait_tracker_is_activated, (void *) wait); ++ } else { ++ _mali_osk_wait_queue_wait_event_timeout(system->wait_queue, mali_timeline_fence_wait_tracker_is_activated, (void *) wait, timeout); ++ } ++ ++ ret = wait->activated; ++ ++ if (0 == _mali_osk_atomic_dec_return(&wait->refcount)) { ++ mali_timeline_fence_wait_tracker_free(wait); ++ } ++ ++ return ret; ++} ++ ++void mali_timeline_fence_wait_activate(struct mali_timeline_fence_wait_tracker *wait) ++{ ++ mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; ++ ++ MALI_DEBUG_ASSERT_POINTER(wait); ++ MALI_DEBUG_ASSERT_POINTER(wait->system); ++ ++ MALI_DEBUG_PRINT(4, ("Mali Timeline: activation for fence wait tracker\n")); ++ ++ MALI_DEBUG_ASSERT(MALI_FALSE == wait->activated); ++ wait->activated = MALI_TRUE; ++ ++ _mali_osk_wait_queue_wake_up(wait->system->wait_queue); ++ ++ /* Nothing can wait on this tracker, so nothing to schedule after release. */ ++ schedule_mask = mali_timeline_tracker_release(&wait->tracker); ++ MALI_DEBUG_ASSERT(MALI_SCHEDULER_MASK_EMPTY == schedule_mask); ++ MALI_IGNORE(schedule_mask); ++ ++ if (0 == _mali_osk_atomic_dec_return(&wait->refcount)) { ++ mali_timeline_fence_wait_tracker_free(wait); ++ } ++} +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_timeline_fence_wait.h b/drivers/gpu/arm/mali400/mali/common/mali_timeline_fence_wait.h +new file mode 100755 +index 000000000000..9da12baeef1a +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_timeline_fence_wait.h +@@ -0,0 +1,67 @@ ++/* ++ * Copyright (C) 2013, 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file mali_timeline_fence_wait.h ++ * ++ * This file contains functions used to wait until a Timeline fence is signaled. ++ */ ++ ++#ifndef __MALI_TIMELINE_FENCE_WAIT_H__ ++#define __MALI_TIMELINE_FENCE_WAIT_H__ ++ ++#include "mali_osk.h" ++#include "mali_timeline.h" ++ ++/** ++ * If used as the timeout argument in @ref mali_timeline_fence_wait, a timer is not used and the ++ * function only returns when the fence is signaled. ++ */ ++#define MALI_TIMELINE_FENCE_WAIT_TIMEOUT_NEVER ((u32) -1) ++ ++/** ++ * If used as the timeout argument in @ref mali_timeline_fence_wait, the function will return ++ * immediately with the current state of the fence. ++ */ ++#define MALI_TIMELINE_FENCE_WAIT_TIMEOUT_IMMEDIATELY 0 ++ ++/** ++ * Fence wait tracker. ++ * ++ * The fence wait tracker is added to the Timeline system with the fence we are waiting on as a ++ * dependency. We will then perform a blocking wait, possibly with a timeout, until the tracker is ++ * activated, which happens when the fence is signaled. ++ */ ++struct mali_timeline_fence_wait_tracker { ++ mali_bool activated; /**< MALI_TRUE if the tracker has been activated, MALI_FALSE if not. */ ++ _mali_osk_atomic_t refcount; /**< Reference count. */ ++ struct mali_timeline_system *system; /**< Timeline system. */ ++ struct mali_timeline_tracker tracker; /**< Timeline tracker. */ ++}; ++ ++/** ++ * Wait for a fence to be signaled, or timeout is reached. ++ * ++ * @param system Timeline system. ++ * @param fence Fence to wait on. ++ * @param timeout Timeout in ms, or MALI_TIMELINE_FENCE_WAIT_TIMEOUT_NEVER or ++ * MALI_TIMELINE_FENCE_WAIT_TIMEOUT_IMMEDIATELY. ++ * @return MALI_TRUE if signaled, MALI_FALSE if timed out. ++ */ ++mali_bool mali_timeline_fence_wait(struct mali_timeline_system *system, struct mali_timeline_fence *fence, u32 timeout); ++ ++/** ++ * Used by the Timeline system to activate a fence wait tracker. ++ * ++ * @param fence_wait_tracker Fence waiter tracker. ++ */ ++void mali_timeline_fence_wait_activate(struct mali_timeline_fence_wait_tracker *fence_wait_tracker); ++ ++#endif /* __MALI_TIMELINE_FENCE_WAIT_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_timeline_sync_fence.c b/drivers/gpu/arm/mali400/mali/common/mali_timeline_sync_fence.c +new file mode 100755 +index 000000000000..bb7f6a04e8bd +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_timeline_sync_fence.c +@@ -0,0 +1,179 @@ ++/* ++ * Copyright (C) 2013, 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++#include ++#include "mali_timeline_sync_fence.h" ++ ++#include "mali_osk.h" ++#include "mali_kernel_common.h" ++#include "mali_sync.h" ++ ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++/** ++ * Creates a sync fence tracker and a sync fence. Adds sync fence tracker to Timeline system and ++ * returns sync fence. The sync fence will be signaled when the sync fence tracker is activated. ++ * ++ * @param timeline Timeline. ++ * @param point Point on timeline. ++ * @return Sync fence that will be signaled when tracker is activated. ++ */ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) ++static struct sync_fence *mali_timeline_sync_fence_create_and_add_tracker(struct mali_timeline *timeline, mali_timeline_point point) ++#else ++static struct mali_internal_sync_fence *mali_timeline_sync_fence_create_and_add_tracker(struct mali_timeline *timeline, mali_timeline_point point) ++#endif ++{ ++ struct mali_timeline_sync_fence_tracker *sync_fence_tracker; ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) ++ struct sync_fence *sync_fence; ++#else ++ struct mali_internal_sync_fence *sync_fence; ++#endif ++ struct mali_timeline_fence fence; ++ ++ MALI_DEBUG_ASSERT_POINTER(timeline); ++ MALI_DEBUG_ASSERT(MALI_TIMELINE_NO_POINT != point); ++ ++ /* Allocate sync fence tracker. */ ++ sync_fence_tracker = _mali_osk_calloc(1, sizeof(struct mali_timeline_sync_fence_tracker)); ++ if (NULL == sync_fence_tracker) { ++ MALI_PRINT_ERROR(("Mali Timeline: sync_fence_tracker allocation failed\n")); ++ return NULL; ++ } ++ ++ /* Create sync flag. */ ++ MALI_DEBUG_ASSERT_POINTER(timeline->sync_tl); ++ sync_fence_tracker->flag = mali_sync_flag_create(timeline->sync_tl, point); ++ if (NULL == sync_fence_tracker->flag) { ++ MALI_PRINT_ERROR(("Mali Timeline: sync_flag creation failed\n")); ++ _mali_osk_free(sync_fence_tracker); ++ return NULL; ++ } ++ ++ /* Create sync fence from sync flag. */ ++ sync_fence = mali_sync_flag_create_fence(sync_fence_tracker->flag); ++ if (NULL == sync_fence) { ++ MALI_PRINT_ERROR(("Mali Timeline: sync_fence creation failed\n")); ++ mali_sync_flag_put(sync_fence_tracker->flag); ++ _mali_osk_free(sync_fence_tracker); ++ return NULL; ++ } ++ ++ /* Setup fence for tracker. */ ++ _mali_osk_memset(&fence, 0, sizeof(struct mali_timeline_fence)); ++ fence.sync_fd = -1; ++ fence.points[timeline->id] = point; ++ ++ /* Finally, add the tracker to Timeline system. */ ++ mali_timeline_tracker_init(&sync_fence_tracker->tracker, MALI_TIMELINE_TRACKER_SYNC, &fence, sync_fence_tracker); ++ point = mali_timeline_system_add_tracker(timeline->system, &sync_fence_tracker->tracker, MALI_TIMELINE_NONE); ++ MALI_DEBUG_ASSERT(MALI_TIMELINE_NO_POINT == point); ++ ++ return sync_fence; ++} ++ ++s32 mali_timeline_sync_fence_create(struct mali_timeline_system *system, struct mali_timeline_fence *fence) ++{ ++ u32 i; ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) ++ struct sync_fence *sync_fence_acc = NULL; ++#else ++ struct mali_internal_sync_fence *sync_fence_acc = NULL; ++#endif ++ MALI_DEBUG_ASSERT_POINTER(system); ++ MALI_DEBUG_ASSERT_POINTER(fence); ++ ++ for (i = 0; i < MALI_TIMELINE_MAX; ++i) { ++ struct mali_timeline *timeline; ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) ++ struct sync_fence *sync_fence; ++#else ++ struct mali_internal_sync_fence *sync_fence; ++#endif ++ if (MALI_TIMELINE_NO_POINT == fence->points[i]) continue; ++ ++ timeline = system->timelines[i]; ++ MALI_DEBUG_ASSERT_POINTER(timeline); ++ ++ sync_fence = mali_timeline_sync_fence_create_and_add_tracker(timeline, fence->points[i]); ++ if (NULL == sync_fence) goto error; ++ ++ if (NULL != sync_fence_acc) { ++ /* Merge sync fences. */ ++ sync_fence_acc = mali_sync_fence_merge(sync_fence_acc, sync_fence); ++ if (NULL == sync_fence_acc) goto error; ++ } else { ++ /* This was the first sync fence created. */ ++ sync_fence_acc = sync_fence; ++ } ++ } ++ ++ if (-1 != fence->sync_fd) { ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) ++ struct sync_fence *sync_fence; ++ sync_fence = sync_fence_fdget(fence->sync_fd); ++#else ++ struct mali_internal_sync_fence *sync_fence; ++ sync_fence = mali_internal_sync_fence_fdget(fence->sync_fd); ++#endif ++ ++ if (NULL == sync_fence) goto error; ++ ++ if (NULL != sync_fence_acc) { ++ sync_fence_acc = mali_sync_fence_merge(sync_fence_acc, sync_fence); ++ if (NULL == sync_fence_acc) goto error; ++ } else { ++ sync_fence_acc = sync_fence; ++ } ++ } ++ ++ if (NULL == sync_fence_acc) { ++ MALI_DEBUG_ASSERT_POINTER(system->signaled_sync_tl); ++ ++ /* There was nothing to wait on, so return an already signaled fence. */ ++ ++ sync_fence_acc = mali_sync_timeline_create_signaled_fence(system->signaled_sync_tl); ++ if (NULL == sync_fence_acc) goto error; ++ } ++ ++ /* Return file descriptor for the accumulated sync fence. */ ++ return mali_sync_fence_fd_alloc(sync_fence_acc); ++ ++error: ++ if (NULL != sync_fence_acc) { ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) ++ sync_fence_put(sync_fence_acc); ++#else ++ fput(sync_fence_acc->file); ++#endif ++ } ++ ++ return -1; ++} ++ ++void mali_timeline_sync_fence_activate(struct mali_timeline_sync_fence_tracker *sync_fence_tracker) ++{ ++ mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; ++ ++ MALI_DEBUG_ASSERT_POINTER(sync_fence_tracker); ++ MALI_DEBUG_ASSERT_POINTER(sync_fence_tracker->flag); ++ ++ MALI_DEBUG_PRINT(4, ("Mali Timeline: activation for sync fence tracker\n")); ++ ++ /* Signal flag and release reference. */ ++ mali_sync_flag_signal(sync_fence_tracker->flag, 0); ++ mali_sync_flag_put(sync_fence_tracker->flag); ++ ++ /* Nothing can wait on this tracker, so nothing to schedule after release. */ ++ schedule_mask = mali_timeline_tracker_release(&sync_fence_tracker->tracker); ++ MALI_DEBUG_ASSERT(MALI_SCHEDULER_MASK_EMPTY == schedule_mask); ++ ++ _mali_osk_free(sync_fence_tracker); ++} ++#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_timeline_sync_fence.h b/drivers/gpu/arm/mali400/mali/common/mali_timeline_sync_fence.h +new file mode 100755 +index 000000000000..65e368ae7c9e +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_timeline_sync_fence.h +@@ -0,0 +1,51 @@ ++/* ++ * Copyright (C) 2013, 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file mali_timeline_sync_fence.h ++ * ++ * This file contains code related to creating sync fences from timeline fences. ++ */ ++ ++#ifndef __MALI_TIMELINE_SYNC_FENCE_H__ ++#define __MALI_TIMELINE_SYNC_FENCE_H__ ++ ++#include "mali_timeline.h" ++ ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ ++/** ++ * Sync fence tracker. ++ */ ++struct mali_timeline_sync_fence_tracker { ++ struct mali_sync_flag *flag; /**< Sync flag used to connect tracker and sync fence. */ ++ struct mali_timeline_tracker tracker; /**< Timeline tracker. */ ++}; ++ ++/** ++ * Create a sync fence that will be signaled when @ref fence is signaled. ++ * ++ * @param system Timeline system. ++ * @param fence Fence to create sync fence from. ++ * @return File descriptor for new sync fence, or -1 on error. ++ */ ++s32 mali_timeline_sync_fence_create(struct mali_timeline_system *system, struct mali_timeline_fence *fence); ++ ++/** ++ * Used by the Timeline system to activate a sync fence tracker. ++ * ++ * @param sync_fence_tracker Sync fence tracker. ++ * ++ */ ++void mali_timeline_sync_fence_activate(struct mali_timeline_sync_fence_tracker *sync_fence_tracker); ++ ++#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ ++ ++#endif /* __MALI_TIMELINE_SYNC_FENCE_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_ukk.h b/drivers/gpu/arm/mali400/mali/common/mali_ukk.h +new file mode 100755 +index 000000000000..55a05c50436a +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_ukk.h +@@ -0,0 +1,551 @@ ++/* ++ * Copyright (C) 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file mali_ukk.h ++ * Defines the kernel-side interface of the user-kernel interface ++ */ ++ ++#ifndef __MALI_UKK_H__ ++#define __MALI_UKK_H__ ++ ++#include "mali_osk.h" ++#include "mali_uk_types.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++/** ++ * @addtogroup uddapi Unified Device Driver (UDD) APIs ++ * ++ * @{ ++ */ ++ ++/** ++ * @addtogroup u_k_api UDD User/Kernel Interface (U/K) APIs ++ * ++ * - The _mali_uk functions are an abstraction of the interface to the device ++ * driver. On certain OSs, this would be implemented via the IOCTL interface. ++ * On other OSs, it could be via extension of some Device Driver Class, or ++ * direct function call for Bare metal/RTOSs. ++ * - It is important to note that: ++ * - The Device Driver has implemented the _mali_ukk set of functions ++ * - The Base Driver calls the corresponding set of _mali_uku functions. ++ * - What requires porting is solely the calling mechanism from User-side to ++ * Kernel-side, and propagating back the results. ++ * - Each U/K function is associated with a (group, number) pair from ++ * \ref _mali_uk_functions to make it possible for a common function in the ++ * Base Driver and Device Driver to route User/Kernel calls from/to the ++ * correct _mali_uk function. For example, in an IOCTL system, the IOCTL number ++ * would be formed based on the group and number assigned to the _mali_uk ++ * function, as listed in \ref _mali_uk_functions. On the user-side, each ++ * _mali_uku function would just make an IOCTL with the IOCTL-code being an ++ * encoded form of the (group, number) pair. On the kernel-side, the Device ++ * Driver's IOCTL handler decodes the IOCTL-code back into a (group, number) ++ * pair, and uses this to determine which corresponding _mali_ukk should be ++ * called. ++ * - Refer to \ref _mali_uk_functions for more information about this ++ * (group, number) pairing. ++ * - In a system where there is no distinction between user and kernel-side, ++ * the U/K interface may be implemented as:@code ++ * MALI_STATIC_INLINE _mali_osk_errcode_t _mali_uku_examplefunction( _mali_uk_examplefunction_s *args ) ++ * { ++ * return mali_ukk_examplefunction( args ); ++ * } ++ * @endcode ++ * - Therefore, all U/K calls behave \em as \em though they were direct ++ * function calls (but the \b implementation \em need \em not be a direct ++ * function calls) ++ * ++ * @note Naming the _mali_uk functions the same on both User and Kernel sides ++ * on non-RTOS systems causes debugging issues when setting breakpoints. In ++ * this case, it is not clear which function the breakpoint is put on. ++ * Therefore the _mali_uk functions in user space are prefixed with \c _mali_uku ++ * and in kernel space with \c _mali_ukk. The naming for the argument ++ * structures is unaffected. ++ * ++ * - The _mali_uk functions are synchronous. ++ * - Arguments to the _mali_uk functions are passed in a structure. The only ++ * parameter passed to the _mali_uk functions is a pointer to this structure. ++ * This first member of this structure, ctx, is a pointer to a context returned ++ * by _mali_uku_open(). For example:@code ++ * typedef struct ++ * { ++ * void *ctx; ++ * u32 number_of_cores; ++ * } _mali_uk_get_gp_number_of_cores_s; ++ * @endcode ++ * ++ * - Each _mali_uk function has its own argument structure named after the ++ * function. The argument is distinguished by the _s suffix. ++ * - The argument types are defined by the base driver and user-kernel ++ * interface. ++ * - All _mali_uk functions return a standard \ref _mali_osk_errcode_t. ++ * - Only arguments of type input or input/output need be initialized before ++ * calling a _mali_uk function. ++ * - Arguments of type output and input/output are only valid when the ++ * _mali_uk function returns \ref _MALI_OSK_ERR_OK. ++ * - The \c ctx member is always invalid after it has been used by a ++ * _mali_uk function, except for the context management functions ++ * ++ * ++ * \b Interface \b restrictions ++ * ++ * The requirements of the interface mean that an implementation of the ++ * User-kernel interface may do no 'real' work. For example, the following are ++ * illegal in the User-kernel implementation: ++ * - Calling functions necessary for operation on all systems, which would ++ * not otherwise get called on RTOS systems. ++ * - For example, a U/K interface that calls multiple _mali_ukk functions ++ * during one particular U/K call. This could not be achieved by the same code ++ * which uses direct function calls for the U/K interface. ++ * - Writing in values to the args members, when otherwise these members would ++ * not hold a useful value for a direct function call U/K interface. ++ * - For example, U/K interface implementation that take NULL members in ++ * their arguments structure from the user side, but those members are ++ * replaced with non-NULL values in the kernel-side of the U/K interface ++ * implementation. A scratch area for writing data is one such example. In this ++ * case, a direct function call U/K interface would segfault, because no code ++ * would be present to replace the NULL pointer with a meaningful pointer. ++ * - Note that we discourage the case where the U/K implementation changes ++ * a NULL argument member to non-NULL, and then the Device Driver code (outside ++ * of the U/K layer) re-checks this member for NULL, and corrects it when ++ * necessary. Whilst such code works even on direct function call U/K ++ * intefaces, it reduces the testing coverage of the Device Driver code. This ++ * is because we have no way of testing the NULL == value path on an OS ++ * implementation. ++ * ++ * A number of allowable examples exist where U/K interfaces do 'real' work: ++ * - The 'pointer switching' technique for \ref _mali_ukk_get_system_info ++ * - In this case, without the pointer switching on direct function call ++ * U/K interface, the Device Driver code still sees the same thing: a pointer ++ * to which it can write memory. This is because such a system has no ++ * distinction between a user and kernel pointer. ++ * - Writing an OS-specific value into the ukk_private member for ++ * _mali_ukk_mem_mmap(). ++ * - In this case, this value is passed around by Device Driver code, but ++ * its actual value is never checked. Device Driver code simply passes it from ++ * the U/K layer to the OSK layer, where it can be acted upon. In this case, ++ * \em some OS implementations of the U/K (_mali_ukk_mem_mmap()) and OSK ++ * (_mali_osk_mem_mapregion_init()) functions will collaborate on the ++ * meaning of ukk_private member. On other OSs, it may be unused by both ++ * U/K and OSK layers ++ * - Therefore, on error inside the U/K interface implementation itself, ++ * it will be as though the _mali_ukk function itself had failed, and cleaned ++ * up after itself. ++ * - Compare this to a direct function call U/K implementation, where all ++ * error cleanup is handled by the _mali_ukk function itself. The direct ++ * function call U/K interface implementation is automatically atomic. ++ * ++ * The last example highlights a consequence of all U/K interface ++ * implementations: they must be atomic with respect to the Device Driver code. ++ * And therefore, should Device Driver code succeed but the U/K implementation ++ * fail afterwards (but before return to user-space), then the U/K ++ * implementation must cause appropriate cleanup actions to preserve the ++ * atomicity of the interface. ++ * ++ * @{ ++ */ ++ ++ ++/** @defgroup _mali_uk_context U/K Context management ++ * ++ * These functions allow for initialisation of the user-kernel interface once per process. ++ * ++ * Generally the context will store the OS specific object to communicate with the kernel device driver and further ++ * state information required by the specific implementation. The context is shareable among all threads in the caller process. ++ * ++ * On IOCTL systems, this is likely to be a file descriptor as a result of opening the kernel device driver. ++ * ++ * On a bare-metal/RTOS system with no distinction between kernel and ++ * user-space, the U/K interface simply calls the _mali_ukk variant of the ++ * function by direct function call. In this case, the context returned is the ++ * mali_session_data from _mali_ukk_open(). ++ * ++ * The kernel side implementations of the U/K interface expect the first member of the argument structure to ++ * be the context created by _mali_uku_open(). On some OS implementations, the meaning of this context ++ * will be different between user-side and kernel-side. In which case, the kernel-side will need to replace this context ++ * with the kernel-side equivalent, because user-side will not have access to kernel-side data. The context parameter ++ * in the argument structure therefore has to be of type input/output. ++ * ++ * It should be noted that the caller cannot reuse the \c ctx member of U/K ++ * argument structure after a U/K call, because it may be overwritten. Instead, ++ * the context handle must always be stored elsewhere, and copied into ++ * the appropriate U/K argument structure for each user-side call to ++ * the U/K interface. This is not usually a problem, since U/K argument ++ * structures are usually placed on the stack. ++ * ++ * @{ */ ++ ++/** @brief Begin a new Mali Device Driver session ++ * ++ * This is used to obtain a per-process context handle for all future U/K calls. ++ * ++ * @param context pointer to storage to return a (void*)context handle. ++ * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. ++ */ ++_mali_osk_errcode_t _mali_ukk_open(void **context); ++ ++/** @brief End a Mali Device Driver session ++ * ++ * This should be called when the process no longer requires use of the Mali Device Driver. ++ * ++ * The context handle must not be used after it has been closed. ++ * ++ * @param context pointer to a stored (void*)context handle. ++ * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. ++ */ ++_mali_osk_errcode_t _mali_ukk_close(void **context); ++ ++/** @} */ /* end group _mali_uk_context */ ++ ++ ++/** @addtogroup _mali_uk_core U/K Core ++ * ++ * The core functions provide the following functionality: ++ * - verify that the user and kernel API are compatible ++ * - retrieve information about the cores and memory banks in the system ++ * - wait for the result of jobs started on a core ++ * ++ * @{ */ ++ ++/** @brief Waits for a job notification. ++ * ++ * Sleeps until notified or a timeout occurs. Returns information about the notification. ++ * ++ * @param args see _mali_uk_wait_for_notification_s in "mali_utgard_uk_types.h" ++ * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. ++ */ ++_mali_osk_errcode_t _mali_ukk_wait_for_notification(_mali_uk_wait_for_notification_s *args); ++ ++/** @brief Post a notification to the notification queue of this application. ++ * ++ * @param args see _mali_uk_post_notification_s in "mali_utgard_uk_types.h" ++ * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. ++ */ ++_mali_osk_errcode_t _mali_ukk_post_notification(_mali_uk_post_notification_s *args); ++ ++/** @brief Verifies if the user and kernel side of this API are compatible. ++ * ++ * This function is obsolete, but kept to allow old, incompatible user space ++ * clients to robustly detect the incompatibility. ++ * ++ * @param args see _mali_uk_get_api_version_s in "mali_utgard_uk_types.h" ++ * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. ++ */ ++_mali_osk_errcode_t _mali_ukk_get_api_version(_mali_uk_get_api_version_s *args); ++ ++/** @brief Verifies if the user and kernel side of this API are compatible. ++ * ++ * @param args see _mali_uk_get_api_version_v2_s in "mali_utgard_uk_types.h" ++ * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. ++ */ ++_mali_osk_errcode_t _mali_ukk_get_api_version_v2(_mali_uk_get_api_version_v2_s *args); ++ ++/** @brief Get the user space settings applicable for calling process. ++ * ++ * @param args see _mali_uk_get_user_settings_s in "mali_utgard_uk_types.h" ++ * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. ++ */ ++_mali_osk_errcode_t _mali_ukk_get_user_settings(_mali_uk_get_user_settings_s *args); ++ ++/** @brief Get a user space setting applicable for calling process. ++ * ++ * @param args see _mali_uk_get_user_setting_s in "mali_utgard_uk_types.h" ++ * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. ++ */ ++_mali_osk_errcode_t _mali_ukk_get_user_setting(_mali_uk_get_user_setting_s *args); ++ ++/* @brief Grant or deny high priority scheduling for this session. ++ * ++ * @param args see _mali_uk_request_high_priority_s in "mali_utgard_uk_types.h" ++ * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. ++ */ ++_mali_osk_errcode_t _mali_ukk_request_high_priority(_mali_uk_request_high_priority_s *args); ++ ++/** @brief Make process sleep if the pending big job in kernel >= MALI_MAX_PENDING_BIG_JOB ++ * ++ */ ++_mali_osk_errcode_t _mali_ukk_pending_submit(_mali_uk_pending_submit_s *args); ++ ++/** @} */ /* end group _mali_uk_core */ ++ ++ ++/** @addtogroup _mali_uk_memory U/K Memory ++ * ++ * The memory functions provide functionality with and without a Mali-MMU present. ++ * ++ * For Mali-MMU based systems, the following functionality is provided: ++ * - Initialize and terminate MALI virtual address space ++ * - Allocate/deallocate physical memory to a MALI virtual address range and map into/unmap from the ++ * current process address space ++ * - Map/unmap external physical memory into the MALI virtual address range ++ * ++ * For Mali-nonMMU based systems: ++ * - Allocate/deallocate MALI memory ++ * ++ * @{ */ ++ ++/** @brief Map Mali Memory into the current user process ++ * ++ * Maps Mali memory into the current user process in a generic way. ++ * ++ * This function is to be used for Mali-MMU mode. The function is available in both Mali-MMU and Mali-nonMMU modes, ++ * but should not be called by a user process in Mali-nonMMU mode. ++ * ++ * The implementation and operation of _mali_ukk_mem_mmap() is dependant on whether the driver is built for Mali-MMU ++ * or Mali-nonMMU: ++ * - In the nonMMU case, _mali_ukk_mem_mmap() requires a physical address to be specified. For this reason, an OS U/K ++ * implementation should not allow this to be called from user-space. In any case, nonMMU implementations are ++ * inherently insecure, and so the overall impact is minimal. Mali-MMU mode should be used if security is desired. ++ * - In the MMU case, _mali_ukk_mem_mmap() the _mali_uk_mem_mmap_s::phys_addr ++ * member is used for the \em Mali-virtual address desired for the mapping. The ++ * implementation of _mali_ukk_mem_mmap() will allocate both the CPU-virtual ++ * and CPU-physical addresses, and can cope with mapping a contiguous virtual ++ * address range to a sequence of non-contiguous physical pages. In this case, ++ * the CPU-physical addresses are not communicated back to the user-side, as ++ * they are unnecsessary; the \em Mali-virtual address range must be used for ++ * programming Mali structures. ++ * ++ * In the second (MMU) case, _mali_ukk_mem_mmap() handles management of ++ * CPU-virtual and CPU-physical ranges, but the \em caller must manage the ++ * \em Mali-virtual address range from the user-side. ++ * ++ * @note Mali-virtual address ranges are entirely separate between processes. ++ * It is not possible for a process to accidentally corrupt another process' ++ * \em Mali-virtual address space. ++ * ++ * @param args see _mali_uk_mem_mmap_s in "mali_utgard_uk_types.h" ++ * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. ++ */ ++_mali_osk_errcode_t _mali_ukk_mem_mmap(_mali_uk_mem_mmap_s *args); ++ ++/** @brief Unmap Mali Memory from the current user process ++ * ++ * Unmaps Mali memory from the current user process in a generic way. This only operates on Mali memory supplied ++ * from _mali_ukk_mem_mmap(). ++ * ++ * @param args see _mali_uk_mem_munmap_s in "mali_utgard_uk_types.h" ++ * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. ++ */ ++_mali_osk_errcode_t _mali_ukk_mem_munmap(_mali_uk_mem_munmap_s *args); ++ ++/** @brief Determine the buffer size necessary for an MMU page table dump. ++ * @param args see _mali_uk_query_mmu_page_table_dump_size_s in mali_utgard_uk_types.h ++ * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. ++ */ ++_mali_osk_errcode_t _mali_ukk_query_mmu_page_table_dump_size(_mali_uk_query_mmu_page_table_dump_size_s *args); ++/** @brief Dump MMU Page tables. ++ * @param args see _mali_uk_dump_mmu_page_table_s in mali_utgard_uk_types.h ++ * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. ++ */ ++_mali_osk_errcode_t _mali_ukk_dump_mmu_page_table(_mali_uk_dump_mmu_page_table_s *args); ++ ++/** @brief Write user data to specified Mali memory without causing segfaults. ++ * @param args see _mali_uk_mem_write_safe_s in mali_utgard_uk_types.h ++ * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. ++ */ ++_mali_osk_errcode_t _mali_ukk_mem_write_safe(_mali_uk_mem_write_safe_s *args); ++ ++/** @} */ /* end group _mali_uk_memory */ ++ ++ ++/** @addtogroup _mali_uk_pp U/K Fragment Processor ++ * ++ * The Fragment Processor (aka PP (Pixel Processor)) functions provide the following functionality: ++ * - retrieving version of the fragment processors ++ * - determine number of fragment processors ++ * - starting a job on a fragment processor ++ * ++ * @{ */ ++ ++/** @brief Issue a request to start a new job on a Fragment Processor. ++ * ++ * If the request fails args->status is set to _MALI_UK_START_JOB_NOT_STARTED_DO_REQUEUE and you can ++ * try to start the job again. ++ * ++ * An existing job could be returned for requeueing if the new job has a higher priority than a previously started job ++ * which the hardware hasn't actually started processing yet. In this case the new job will be started instead and the ++ * existing one returned, otherwise the new job is started and the status field args->status is set to ++ * _MALI_UK_START_JOB_STARTED. ++ * ++ * Job completion can be awaited with _mali_ukk_wait_for_notification(). ++ * ++ * @param ctx user-kernel context (mali_session) ++ * @param uargs see _mali_uk_pp_start_job_s in "mali_utgard_uk_types.h". Use _mali_osk_copy_from_user to retrieve data! ++ * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. ++ */ ++_mali_osk_errcode_t _mali_ukk_pp_start_job(void *ctx, _mali_uk_pp_start_job_s *uargs); ++ ++/** ++ * @brief Issue a request to start new jobs on both Vertex Processor and Fragment Processor. ++ * ++ * @note Will call into @ref _mali_ukk_pp_start_job and @ref _mali_ukk_gp_start_job. ++ * ++ * @param ctx user-kernel context (mali_session) ++ * @param uargs see _mali_uk_pp_and_gp_start_job_s in "mali_utgard_uk_types.h". Use _mali_osk_copy_from_user to retrieve data! ++ * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. ++ */ ++_mali_osk_errcode_t _mali_ukk_pp_and_gp_start_job(void *ctx, _mali_uk_pp_and_gp_start_job_s *uargs); ++ ++/** @brief Returns the number of Fragment Processors in the system ++ * ++ * @param args see _mali_uk_get_pp_number_of_cores_s in "mali_utgard_uk_types.h" ++ * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. ++ */ ++_mali_osk_errcode_t _mali_ukk_get_pp_number_of_cores(_mali_uk_get_pp_number_of_cores_s *args); ++ ++/** @brief Returns the version that all Fragment Processor cores are compatible with. ++ * ++ * This function may only be called when _mali_ukk_get_pp_number_of_cores() indicated at least one Fragment ++ * Processor core is available. ++ * ++ * @param args see _mali_uk_get_pp_core_version_s in "mali_utgard_uk_types.h" ++ * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. ++ */ ++_mali_osk_errcode_t _mali_ukk_get_pp_core_version(_mali_uk_get_pp_core_version_s *args); ++ ++/** @brief Disable Write-back unit(s) on specified job ++ * ++ * @param args see _mali_uk_get_pp_core_version_s in "mali_utgard_uk_types.h" ++ */ ++void _mali_ukk_pp_job_disable_wb(_mali_uk_pp_disable_wb_s *args); ++ ++ ++/** @} */ /* end group _mali_uk_pp */ ++ ++ ++/** @addtogroup _mali_uk_gp U/K Vertex Processor ++ * ++ * The Vertex Processor (aka GP (Geometry Processor)) functions provide the following functionality: ++ * - retrieving version of the Vertex Processors ++ * - determine number of Vertex Processors available ++ * - starting a job on a Vertex Processor ++ * ++ * @{ */ ++ ++/** @brief Issue a request to start a new job on a Vertex Processor. ++ * ++ * If the request fails args->status is set to _MALI_UK_START_JOB_NOT_STARTED_DO_REQUEUE and you can ++ * try to start the job again. ++ * ++ * An existing job could be returned for requeueing if the new job has a higher priority than a previously started job ++ * which the hardware hasn't actually started processing yet. In this case the new job will be started and the ++ * existing one returned, otherwise the new job is started and the status field args->status is set to ++ * _MALI_UK_START_JOB_STARTED. ++ * ++ * Job completion can be awaited with _mali_ukk_wait_for_notification(). ++ * ++ * @param ctx user-kernel context (mali_session) ++ * @param uargs see _mali_uk_gp_start_job_s in "mali_utgard_uk_types.h". Use _mali_osk_copy_from_user to retrieve data! ++ * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. ++ */ ++_mali_osk_errcode_t _mali_ukk_gp_start_job(void *ctx, _mali_uk_gp_start_job_s *uargs); ++ ++/** @brief Returns the number of Vertex Processors in the system. ++ * ++ * @param args see _mali_uk_get_gp_number_of_cores_s in "mali_utgard_uk_types.h" ++ * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. ++ */ ++_mali_osk_errcode_t _mali_ukk_get_gp_number_of_cores(_mali_uk_get_gp_number_of_cores_s *args); ++ ++/** @brief Returns the version that all Vertex Processor cores are compatible with. ++ * ++ * This function may only be called when _mali_uk_get_gp_number_of_cores() indicated at least one Vertex ++ * Processor core is available. ++ * ++ * @param args see _mali_uk_get_gp_core_version_s in "mali_utgard_uk_types.h" ++ * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. ++ */ ++_mali_osk_errcode_t _mali_ukk_get_gp_core_version(_mali_uk_get_gp_core_version_s *args); ++ ++/** @brief Resume or abort suspended Vertex Processor jobs. ++ * ++ * After receiving notification that a Vertex Processor job was suspended from ++ * _mali_ukk_wait_for_notification() you can use this function to resume or abort the job. ++ * ++ * @param args see _mali_uk_gp_suspend_response_s in "mali_utgard_uk_types.h" ++ * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. ++ */ ++_mali_osk_errcode_t _mali_ukk_gp_suspend_response(_mali_uk_gp_suspend_response_s *args); ++ ++/** @} */ /* end group _mali_uk_gp */ ++ ++#if defined(CONFIG_MALI400_PROFILING) ++/** @addtogroup _mali_uk_profiling U/K Timeline profiling module ++ * @{ */ ++ ++/** @brief Add event to profiling buffer. ++ * ++ * @param args see _mali_uk_profiling_add_event_s in "mali_utgard_uk_types.h" ++ */ ++_mali_osk_errcode_t _mali_ukk_profiling_add_event(_mali_uk_profiling_add_event_s *args); ++ ++/** @brief Get profiling stream fd. ++ * ++ * @param args see _mali_uk_profiling_stream_fd_get_s in "mali_utgard_uk_types.h" ++ */ ++_mali_osk_errcode_t _mali_ukk_profiling_stream_fd_get(_mali_uk_profiling_stream_fd_get_s *args); ++ ++/** @brief Profiling control set. ++ * ++ * @param args see _mali_uk_profiling_control_set_s in "mali_utgard_uk_types.h" ++ */ ++_mali_osk_errcode_t _mali_ukk_profiling_control_set(_mali_uk_profiling_control_set_s *args); ++ ++/** @} */ /* end group _mali_uk_profiling */ ++#endif ++ ++/** @addtogroup _mali_uk_vsync U/K VSYNC reporting module ++ * @{ */ ++ ++/** @brief Report events related to vsync. ++ * ++ * @note Events should be reported when starting to wait for vsync and when the ++ * waiting is finished. This information can then be used in kernel space to ++ * complement the GPU utilization metric. ++ * ++ * @param args see _mali_uk_vsync_event_report_s in "mali_utgard_uk_types.h" ++ */ ++_mali_osk_errcode_t _mali_ukk_vsync_event_report(_mali_uk_vsync_event_report_s *args); ++ ++/** @} */ /* end group _mali_uk_vsync */ ++ ++/** @addtogroup _mali_sw_counters_report U/K Software counter reporting ++ * @{ */ ++ ++/** @brief Report software counters. ++ * ++ * @param args see _mali_uk_sw_counters_report_s in "mali_uk_types.h" ++ */ ++_mali_osk_errcode_t _mali_ukk_sw_counters_report(_mali_uk_sw_counters_report_s *args); ++ ++/** @} */ /* end group _mali_sw_counters_report */ ++ ++/** @} */ /* end group u_k_api */ ++ ++/** @} */ /* end group uddapi */ ++ ++u32 _mali_ukk_report_memory_usage(void); ++ ++u32 _mali_ukk_report_total_memory_size(void); ++ ++u32 _mali_ukk_utilization_gp_pp(void); ++ ++u32 _mali_ukk_utilization_gp(void); ++ ++u32 _mali_ukk_utilization_pp(void); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif /* __MALI_UKK_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_user_settings_db.c b/drivers/gpu/arm/mali400/mali/common/mali_user_settings_db.c +new file mode 100755 +index 000000000000..1911eff87a72 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_user_settings_db.c +@@ -0,0 +1,147 @@ ++/** ++ * Copyright (C) 2012-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include "mali_kernel_common.h" ++#include "mali_osk.h" ++#include "mali_ukk.h" ++#include "mali_uk_types.h" ++#include "mali_user_settings_db.h" ++#include "mali_session.h" ++ ++static u32 mali_user_settings[_MALI_UK_USER_SETTING_MAX]; ++const char *_mali_uk_user_setting_descriptions[] = _MALI_UK_USER_SETTING_DESCRIPTIONS; ++ ++static void mali_user_settings_notify(_mali_uk_user_setting_t setting, u32 value) ++{ ++ mali_bool done = MALI_FALSE; ++ ++ /* ++ * This function gets a bit complicated because we can't hold the session lock while ++ * allocating notification objects. ++ */ ++ ++ while (!done) { ++ u32 i; ++ u32 num_sessions_alloc; ++ u32 num_sessions_with_lock; ++ u32 used_notification_objects = 0; ++ _mali_osk_notification_t **notobjs; ++ ++ /* Pre allocate the number of notifications objects we need right now (might change after lock has been taken) */ ++ num_sessions_alloc = mali_session_get_count(); ++ if (0 == num_sessions_alloc) { ++ /* No sessions to report to */ ++ return; ++ } ++ ++ notobjs = (_mali_osk_notification_t **)_mali_osk_malloc(sizeof(_mali_osk_notification_t *) * num_sessions_alloc); ++ if (NULL == notobjs) { ++ MALI_PRINT_ERROR(("Failed to notify user space session about num PP core change (alloc failure)\n")); ++ return; ++ } ++ ++ for (i = 0; i < num_sessions_alloc; i++) { ++ notobjs[i] = _mali_osk_notification_create(_MALI_NOTIFICATION_SETTINGS_CHANGED, ++ sizeof(_mali_uk_settings_changed_s)); ++ if (NULL != notobjs[i]) { ++ _mali_uk_settings_changed_s *data; ++ data = notobjs[i]->result_buffer; ++ ++ data->setting = setting; ++ data->value = value; ++ } else { ++ MALI_PRINT_ERROR(("Failed to notify user space session about setting change (alloc failure %u)\n", i)); ++ } ++ } ++ ++ mali_session_lock(); ++ ++ /* number of sessions will not change while we hold the lock */ ++ num_sessions_with_lock = mali_session_get_count(); ++ ++ if (num_sessions_alloc >= num_sessions_with_lock) { ++ /* We have allocated enough notification objects for all the sessions atm */ ++ struct mali_session_data *session, *tmp; ++ MALI_SESSION_FOREACH(session, tmp, link) { ++ MALI_DEBUG_ASSERT(used_notification_objects < num_sessions_alloc); ++ if (NULL != notobjs[used_notification_objects]) { ++ mali_session_send_notification(session, notobjs[used_notification_objects]); ++ notobjs[used_notification_objects] = NULL; /* Don't track this notification object any more */ ++ } ++ used_notification_objects++; ++ } ++ done = MALI_TRUE; ++ } ++ ++ mali_session_unlock(); ++ ++ /* Delete any remaining/unused notification objects */ ++ for (; used_notification_objects < num_sessions_alloc; used_notification_objects++) { ++ if (NULL != notobjs[used_notification_objects]) { ++ _mali_osk_notification_delete(notobjs[used_notification_objects]); ++ } ++ } ++ ++ _mali_osk_free(notobjs); ++ } ++} ++ ++void mali_set_user_setting(_mali_uk_user_setting_t setting, u32 value) ++{ ++ mali_bool notify = MALI_FALSE; ++ ++ if (setting >= _MALI_UK_USER_SETTING_MAX) { ++ MALI_DEBUG_PRINT_ERROR(("Invalid user setting %ud\n")); ++ return; ++ } ++ ++ if (mali_user_settings[setting] != value) { ++ notify = MALI_TRUE; ++ } ++ ++ mali_user_settings[setting] = value; ++ ++ if (notify) { ++ mali_user_settings_notify(setting, value); ++ } ++} ++ ++u32 mali_get_user_setting(_mali_uk_user_setting_t setting) ++{ ++ if (setting >= _MALI_UK_USER_SETTING_MAX) { ++ return 0; ++ } ++ ++ return mali_user_settings[setting]; ++} ++ ++_mali_osk_errcode_t _mali_ukk_get_user_setting(_mali_uk_get_user_setting_s *args) ++{ ++ _mali_uk_user_setting_t setting; ++ MALI_DEBUG_ASSERT_POINTER(args); ++ ++ setting = args->setting; ++ ++ if (_MALI_UK_USER_SETTING_MAX > setting) { ++ args->value = mali_user_settings[setting]; ++ return _MALI_OSK_ERR_OK; ++ } else { ++ return _MALI_OSK_ERR_INVALID_ARGS; ++ } ++} ++ ++_mali_osk_errcode_t _mali_ukk_get_user_settings(_mali_uk_get_user_settings_s *args) ++{ ++ MALI_DEBUG_ASSERT_POINTER(args); ++ ++ _mali_osk_memcpy(args->settings, mali_user_settings, sizeof(mali_user_settings)); ++ ++ return _MALI_OSK_ERR_OK; ++} +diff --git a/drivers/gpu/arm/mali400/mali/common/mali_user_settings_db.h b/drivers/gpu/arm/mali400/mali/common/mali_user_settings_db.h +new file mode 100755 +index 000000000000..da9c0630e371 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/common/mali_user_settings_db.h +@@ -0,0 +1,39 @@ ++/** ++ * Copyright (C) 2012-2013, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_USER_SETTINGS_DB_H__ ++#define __MALI_USER_SETTINGS_DB_H__ ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++#include "mali_uk_types.h" ++ ++/** @brief Set Mali user setting in DB ++ * ++ * Update the DB with a new value for \a setting. If the value is different from theprevious set value running sessions will be notified of the change. ++ * ++ * @param setting the setting to be changed ++ * @param value the new value to set ++ */ ++void mali_set_user_setting(_mali_uk_user_setting_t setting, u32 value); ++ ++/** @brief Get current Mali user setting value from DB ++ * ++ * @param setting the setting to extract ++ * @return the value of the selected setting ++ */ ++u32 mali_get_user_setting(_mali_uk_user_setting_t setting); ++ ++#ifdef __cplusplus ++} ++#endif ++#endif /* __MALI_KERNEL_USER_SETTING__ */ +diff --git a/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard.h b/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard.h +new file mode 100755 +index 000000000000..7df55c951d6f +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard.h +@@ -0,0 +1,526 @@ ++/* ++ * Copyright (C) 2012-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file mali_utgard.h ++ * Defines types and interface exposed by the Mali Utgard device driver ++ */ ++ ++#ifndef __MALI_UTGARD_H__ ++#define __MALI_UTGARD_H__ ++ ++#include "mali_osk_types.h" ++#ifdef CONFIG_MALI_DEVFREQ ++#include ++#include "mali_pm_metrics.h" ++#ifdef CONFIG_DEVFREQ_THERMAL ++#include ++#endif ++#endif ++ ++#define MALI_GPU_NAME_UTGARD "mali-utgard" ++ ++ ++#define MALI_OFFSET_GP 0x00000 ++#define MALI_OFFSET_GP_MMU 0x03000 ++ ++#define MALI_OFFSET_PP0 0x08000 ++#define MALI_OFFSET_PP0_MMU 0x04000 ++#define MALI_OFFSET_PP1 0x0A000 ++#define MALI_OFFSET_PP1_MMU 0x05000 ++#define MALI_OFFSET_PP2 0x0C000 ++#define MALI_OFFSET_PP2_MMU 0x06000 ++#define MALI_OFFSET_PP3 0x0E000 ++#define MALI_OFFSET_PP3_MMU 0x07000 ++ ++#define MALI_OFFSET_PP4 0x28000 ++#define MALI_OFFSET_PP4_MMU 0x1C000 ++#define MALI_OFFSET_PP5 0x2A000 ++#define MALI_OFFSET_PP5_MMU 0x1D000 ++#define MALI_OFFSET_PP6 0x2C000 ++#define MALI_OFFSET_PP6_MMU 0x1E000 ++#define MALI_OFFSET_PP7 0x2E000 ++#define MALI_OFFSET_PP7_MMU 0x1F000 ++ ++#define MALI_OFFSET_L2_RESOURCE0 0x01000 ++#define MALI_OFFSET_L2_RESOURCE1 0x10000 ++#define MALI_OFFSET_L2_RESOURCE2 0x11000 ++ ++#define MALI400_OFFSET_L2_CACHE0 MALI_OFFSET_L2_RESOURCE0 ++#define MALI450_OFFSET_L2_CACHE0 MALI_OFFSET_L2_RESOURCE1 ++#define MALI450_OFFSET_L2_CACHE1 MALI_OFFSET_L2_RESOURCE0 ++#define MALI450_OFFSET_L2_CACHE2 MALI_OFFSET_L2_RESOURCE2 ++#define MALI470_OFFSET_L2_CACHE1 MALI_OFFSET_L2_RESOURCE0 ++ ++#define MALI_OFFSET_BCAST 0x13000 ++#define MALI_OFFSET_DLBU 0x14000 ++ ++#define MALI_OFFSET_PP_BCAST 0x16000 ++#define MALI_OFFSET_PP_BCAST_MMU 0x15000 ++ ++#define MALI_OFFSET_PMU 0x02000 ++#define MALI_OFFSET_DMA 0x12000 ++ ++/* Mali-300 */ ++ ++#define MALI_GPU_RESOURCES_MALI300(base_addr, gp_irq, gp_mmu_irq, pp_irq, pp_mmu_irq) \ ++ MALI_GPU_RESOURCES_MALI400_MP1(base_addr, gp_irq, gp_mmu_irq, pp_irq, pp_mmu_irq) ++ ++#define MALI_GPU_RESOURCES_MALI300_PMU(base_addr, gp_irq, gp_mmu_irq, pp_irq, pp_mmu_irq) \ ++ MALI_GPU_RESOURCES_MALI400_MP1_PMU(base_addr, gp_irq, gp_mmu_irq, pp_irq, pp_mmu_irq) ++ ++/* Mali-400 */ ++ ++#define MALI_GPU_RESOURCES_MALI400_MP1(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq) \ ++ MALI_GPU_RESOURCE_L2(base_addr + MALI400_OFFSET_L2_CACHE0) \ ++ MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ ++ MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) ++ ++#define MALI_GPU_RESOURCES_MALI400_MP1_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq) \ ++ MALI_GPU_RESOURCES_MALI400_MP1(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq) \ ++ MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) ++ ++#define MALI_GPU_RESOURCES_MALI400_MP2(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq) \ ++ MALI_GPU_RESOURCE_L2(base_addr + MALI400_OFFSET_L2_CACHE0) \ ++ MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ ++ MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ ++ MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) ++ ++#define MALI_GPU_RESOURCES_MALI400_MP2_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq) \ ++ MALI_GPU_RESOURCES_MALI400_MP2(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq) \ ++ MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) ++ ++#define MALI_GPU_RESOURCES_MALI400_MP3(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq) \ ++ MALI_GPU_RESOURCE_L2(base_addr + MALI400_OFFSET_L2_CACHE0) \ ++ MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ ++ MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ ++ MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) \ ++ MALI_GPU_RESOURCE_PP_WITH_MMU(2, base_addr + MALI_OFFSET_PP2, pp2_irq, base_addr + MALI_OFFSET_PP2_MMU, pp2_mmu_irq) ++ ++#define MALI_GPU_RESOURCES_MALI400_MP3_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq) \ ++ MALI_GPU_RESOURCES_MALI400_MP3(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq) \ ++ MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) ++ ++#define MALI_GPU_RESOURCES_MALI400_MP4(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq) \ ++ MALI_GPU_RESOURCE_L2(base_addr + MALI400_OFFSET_L2_CACHE0) \ ++ MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ ++ MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ ++ MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) \ ++ MALI_GPU_RESOURCE_PP_WITH_MMU(2, base_addr + MALI_OFFSET_PP2, pp2_irq, base_addr + MALI_OFFSET_PP2_MMU, pp2_mmu_irq) \ ++ MALI_GPU_RESOURCE_PP_WITH_MMU(3, base_addr + MALI_OFFSET_PP3, pp3_irq, base_addr + MALI_OFFSET_PP3_MMU, pp3_mmu_irq) ++ ++#define MALI_GPU_RESOURCES_MALI400_MP4_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq) \ ++ MALI_GPU_RESOURCES_MALI400_MP4(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq) \ ++ MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ ++ ++ /* Mali-450 */ ++#define MALI_GPU_RESOURCES_MALI450_MP2(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp_bcast_irq) \ ++ MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE0) \ ++ MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ ++ MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE1) \ ++ MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ ++ MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) \ ++ MALI_GPU_RESOURCE_BCAST(base_addr + MALI_OFFSET_BCAST) \ ++ MALI_GPU_RESOURCE_DLBU(base_addr + MALI_OFFSET_DLBU) \ ++ MALI_GPU_RESOURCE_PP_BCAST(base_addr + MALI_OFFSET_PP_BCAST, pp_bcast_irq) \ ++ MALI_GPU_RESOURCE_PP_MMU_BCAST(base_addr + MALI_OFFSET_PP_BCAST_MMU) \ ++ MALI_GPU_RESOURCE_DMA(base_addr + MALI_OFFSET_DMA) ++ ++#define MALI_GPU_RESOURCES_MALI450_MP2_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp_bcast_irq) \ ++ MALI_GPU_RESOURCES_MALI450_MP2(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp_bcast_irq) \ ++ MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ ++ ++#define MALI_GPU_RESOURCES_MALI450_MP3(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp_bcast_irq) \ ++ MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE0) \ ++ MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ ++ MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE1) \ ++ MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ ++ MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) \ ++ MALI_GPU_RESOURCE_PP_WITH_MMU(2, base_addr + MALI_OFFSET_PP2, pp2_irq, base_addr + MALI_OFFSET_PP2_MMU, pp2_mmu_irq) \ ++ MALI_GPU_RESOURCE_BCAST(base_addr + MALI_OFFSET_BCAST) \ ++ MALI_GPU_RESOURCE_DLBU(base_addr + MALI_OFFSET_DLBU) \ ++ MALI_GPU_RESOURCE_PP_BCAST(base_addr + MALI_OFFSET_PP_BCAST, pp_bcast_irq) \ ++ MALI_GPU_RESOURCE_PP_MMU_BCAST(base_addr + MALI_OFFSET_PP_BCAST_MMU) ++ ++#define MALI_GPU_RESOURCES_MALI450_MP3_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp_bcast_irq) \ ++ MALI_GPU_RESOURCES_MALI450_MP3(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp_bcast_irq) \ ++ MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ ++ ++#define MALI_GPU_RESOURCES_MALI450_MP4(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp_bcast_irq) \ ++ MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE0) \ ++ MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ ++ MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE1) \ ++ MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ ++ MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) \ ++ MALI_GPU_RESOURCE_PP_WITH_MMU(2, base_addr + MALI_OFFSET_PP2, pp2_irq, base_addr + MALI_OFFSET_PP2_MMU, pp2_mmu_irq) \ ++ MALI_GPU_RESOURCE_PP_WITH_MMU(3, base_addr + MALI_OFFSET_PP3, pp3_irq, base_addr + MALI_OFFSET_PP3_MMU, pp3_mmu_irq) \ ++ MALI_GPU_RESOURCE_BCAST(base_addr + MALI_OFFSET_BCAST) \ ++ MALI_GPU_RESOURCE_DLBU(base_addr + MALI_OFFSET_DLBU) \ ++ MALI_GPU_RESOURCE_PP_BCAST(base_addr + MALI_OFFSET_PP_BCAST, pp_bcast_irq) \ ++ MALI_GPU_RESOURCE_PP_MMU_BCAST(base_addr + MALI_OFFSET_PP_BCAST_MMU) \ ++ MALI_GPU_RESOURCE_DMA(base_addr + MALI_OFFSET_DMA) ++ ++#define MALI_GPU_RESOURCES_MALI450_MP4_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp_bcast_irq) \ ++ MALI_GPU_RESOURCES_MALI450_MP4(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp_bcast_irq) \ ++ MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ ++ ++#define MALI_GPU_RESOURCES_MALI450_MP6(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp4_irq, pp4_mmu_irq, pp5_irq, pp5_mmu_irq, pp_bcast_irq) \ ++ MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE0) \ ++ MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ ++ MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE1) \ ++ MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ ++ MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) \ ++ MALI_GPU_RESOURCE_PP_WITH_MMU(2, base_addr + MALI_OFFSET_PP2, pp2_irq, base_addr + MALI_OFFSET_PP2_MMU, pp2_mmu_irq) \ ++ MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE2) \ ++ MALI_GPU_RESOURCE_PP_WITH_MMU(3, base_addr + MALI_OFFSET_PP4, pp3_irq, base_addr + MALI_OFFSET_PP4_MMU, pp3_mmu_irq) \ ++ MALI_GPU_RESOURCE_PP_WITH_MMU(4, base_addr + MALI_OFFSET_PP5, pp4_irq, base_addr + MALI_OFFSET_PP5_MMU, pp4_mmu_irq) \ ++ MALI_GPU_RESOURCE_PP_WITH_MMU(5, base_addr + MALI_OFFSET_PP6, pp5_irq, base_addr + MALI_OFFSET_PP6_MMU, pp5_mmu_irq) \ ++ MALI_GPU_RESOURCE_BCAST(base_addr + MALI_OFFSET_BCAST) \ ++ MALI_GPU_RESOURCE_DLBU(base_addr + MALI_OFFSET_DLBU) \ ++ MALI_GPU_RESOURCE_PP_BCAST(base_addr + MALI_OFFSET_PP_BCAST, pp_bcast_irq) \ ++ MALI_GPU_RESOURCE_PP_MMU_BCAST(base_addr + MALI_OFFSET_PP_BCAST_MMU) \ ++ MALI_GPU_RESOURCE_DMA(base_addr + MALI_OFFSET_DMA) ++ ++#define MALI_GPU_RESOURCES_MALI450_MP6_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp4_irq, pp4_mmu_irq, pp5_irq, pp5_mmu_irq, pp_bcast_irq) \ ++ MALI_GPU_RESOURCES_MALI450_MP6(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp4_irq, pp4_mmu_irq, pp5_irq, pp5_mmu_irq, pp_bcast_irq) \ ++ MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ ++ ++#define MALI_GPU_RESOURCES_MALI450_MP8(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp4_irq, pp4_mmu_irq, pp5_irq, pp5_mmu_irq, pp6_irq, pp6_mmu_irq, pp7_irq, pp7_mmu_irq, pp_bcast_irq) \ ++ MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE0) \ ++ MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ ++ MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE1) \ ++ MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ ++ MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) \ ++ MALI_GPU_RESOURCE_PP_WITH_MMU(2, base_addr + MALI_OFFSET_PP2, pp2_irq, base_addr + MALI_OFFSET_PP2_MMU, pp2_mmu_irq) \ ++ MALI_GPU_RESOURCE_PP_WITH_MMU(3, base_addr + MALI_OFFSET_PP3, pp3_irq, base_addr + MALI_OFFSET_PP3_MMU, pp3_mmu_irq) \ ++ MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE2) \ ++ MALI_GPU_RESOURCE_PP_WITH_MMU(4, base_addr + MALI_OFFSET_PP4, pp4_irq, base_addr + MALI_OFFSET_PP4_MMU, pp4_mmu_irq) \ ++ MALI_GPU_RESOURCE_PP_WITH_MMU(5, base_addr + MALI_OFFSET_PP5, pp5_irq, base_addr + MALI_OFFSET_PP5_MMU, pp5_mmu_irq) \ ++ MALI_GPU_RESOURCE_PP_WITH_MMU(6, base_addr + MALI_OFFSET_PP6, pp6_irq, base_addr + MALI_OFFSET_PP6_MMU, pp6_mmu_irq) \ ++ MALI_GPU_RESOURCE_PP_WITH_MMU(7, base_addr + MALI_OFFSET_PP7, pp7_irq, base_addr + MALI_OFFSET_PP7_MMU, pp7_mmu_irq) \ ++ MALI_GPU_RESOURCE_BCAST(base_addr + MALI_OFFSET_BCAST) \ ++ MALI_GPU_RESOURCE_DLBU(base_addr + MALI_OFFSET_DLBU) \ ++ MALI_GPU_RESOURCE_PP_BCAST(base_addr + MALI_OFFSET_PP_BCAST, pp_bcast_irq) \ ++ MALI_GPU_RESOURCE_PP_MMU_BCAST(base_addr + MALI_OFFSET_PP_BCAST_MMU) \ ++ MALI_GPU_RESOURCE_DMA(base_addr + MALI_OFFSET_DMA) ++ ++#define MALI_GPU_RESOURCES_MALI450_MP8_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp4_irq, pp4_mmu_irq, pp5_irq, pp5_mmu_irq, pp6_irq, pp6_mmu_irq, pp7_irq, pp7_mmu_irq, pp_bcast_irq) \ ++ MALI_GPU_RESOURCES_MALI450_MP8(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp4_irq, pp4_mmu_irq, pp5_irq, pp5_mmu_irq, pp6_irq, pp6_mmu_irq, pp7_irq, pp7_mmu_irq, pp_bcast_irq) \ ++ MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ ++ ++ /* Mali - 470 */ ++#define MALI_GPU_RESOURCES_MALI470_MP1(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp_bcast_irq) \ ++ MALI_GPU_RESOURCE_L2(base_addr + MALI470_OFFSET_L2_CACHE1) \ ++ MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ ++ MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ ++ MALI_GPU_RESOURCE_BCAST(base_addr + MALI_OFFSET_BCAST) \ ++ MALI_GPU_RESOURCE_DLBU(base_addr + MALI_OFFSET_DLBU) \ ++ MALI_GPU_RESOURCE_PP_BCAST(base_addr + MALI_OFFSET_PP_BCAST, pp_bcast_irq) \ ++ MALI_GPU_RESOURCE_PP_MMU_BCAST(base_addr + MALI_OFFSET_PP_BCAST_MMU) ++ ++#define MALI_GPU_RESOURCES_MALI470_MP1_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp_bcast_irq) \ ++ MALI_GPU_RESOURCES_MALI470_MP1(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp_bcast_irq) \ ++ MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ ++ ++#define MALI_GPU_RESOURCES_MALI470_MP2(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp_bcast_irq) \ ++ MALI_GPU_RESOURCE_L2(base_addr + MALI470_OFFSET_L2_CACHE1) \ ++ MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ ++ MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ ++ MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) \ ++ MALI_GPU_RESOURCE_BCAST(base_addr + MALI_OFFSET_BCAST) \ ++ MALI_GPU_RESOURCE_DLBU(base_addr + MALI_OFFSET_DLBU) \ ++ MALI_GPU_RESOURCE_PP_BCAST(base_addr + MALI_OFFSET_PP_BCAST, pp_bcast_irq) \ ++ MALI_GPU_RESOURCE_PP_MMU_BCAST(base_addr + MALI_OFFSET_PP_BCAST_MMU) ++ ++#define MALI_GPU_RESOURCES_MALI470_MP2_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp_bcast_irq) \ ++ MALI_GPU_RESOURCES_MALI470_MP2(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp_bcast_irq) \ ++ MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ ++ ++#define MALI_GPU_RESOURCES_MALI470_MP3(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp_bcast_irq) \ ++ MALI_GPU_RESOURCE_L2(base_addr + MALI470_OFFSET_L2_CACHE1) \ ++ MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ ++ MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ ++ MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) \ ++ MALI_GPU_RESOURCE_PP_WITH_MMU(2, base_addr + MALI_OFFSET_PP2, pp2_irq, base_addr + MALI_OFFSET_PP2_MMU, pp2_mmu_irq) \ ++ MALI_GPU_RESOURCE_BCAST(base_addr + MALI_OFFSET_BCAST) \ ++ MALI_GPU_RESOURCE_DLBU(base_addr + MALI_OFFSET_DLBU) \ ++ MALI_GPU_RESOURCE_PP_BCAST(base_addr + MALI_OFFSET_PP_BCAST, pp_bcast_irq) \ ++ MALI_GPU_RESOURCE_PP_MMU_BCAST(base_addr + MALI_OFFSET_PP_BCAST_MMU) ++ ++#define MALI_GPU_RESOURCES_MALI470_MP3_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp_bcast_irq) \ ++ MALI_GPU_RESOURCES_MALI470_MP3(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp_bcast_irq) \ ++ MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ ++ ++#define MALI_GPU_RESOURCES_MALI470_MP4(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp_bcast_irq) \ ++ MALI_GPU_RESOURCE_L2(base_addr + MALI470_OFFSET_L2_CACHE1) \ ++ MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ ++ MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ ++ MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) \ ++ MALI_GPU_RESOURCE_PP_WITH_MMU(2, base_addr + MALI_OFFSET_PP2, pp2_irq, base_addr + MALI_OFFSET_PP2_MMU, pp2_mmu_irq) \ ++ MALI_GPU_RESOURCE_PP_WITH_MMU(3, base_addr + MALI_OFFSET_PP3, pp3_irq, base_addr + MALI_OFFSET_PP3_MMU, pp3_mmu_irq) \ ++ MALI_GPU_RESOURCE_BCAST(base_addr + MALI_OFFSET_BCAST) \ ++ MALI_GPU_RESOURCE_DLBU(base_addr + MALI_OFFSET_DLBU) \ ++ MALI_GPU_RESOURCE_PP_BCAST(base_addr + MALI_OFFSET_PP_BCAST, pp_bcast_irq) \ ++ MALI_GPU_RESOURCE_PP_MMU_BCAST(base_addr + MALI_OFFSET_PP_BCAST_MMU) ++ ++#define MALI_GPU_RESOURCES_MALI470_MP4_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp_bcast_irq) \ ++ MALI_GPU_RESOURCES_MALI470_MP4(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp_bcast_irq) \ ++ MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ ++ ++#define MALI_GPU_RESOURCE_L2(addr) \ ++ { \ ++ .name = "Mali_L2", \ ++ .flags = IORESOURCE_MEM, \ ++ .start = addr, \ ++ .end = addr + 0x200, \ ++ }, ++ ++#define MALI_GPU_RESOURCE_GP(gp_addr, gp_irq) \ ++ { \ ++ .name = "Mali_GP", \ ++ .flags = IORESOURCE_MEM, \ ++ .start = gp_addr, \ ++ .end = gp_addr + 0x100, \ ++ }, \ ++ { \ ++ .name = "Mali_GP_IRQ", \ ++ .flags = IORESOURCE_IRQ, \ ++ .start = gp_irq, \ ++ .end = gp_irq, \ ++ }, \ ++ ++#define MALI_GPU_RESOURCE_GP_WITH_MMU(gp_addr, gp_irq, gp_mmu_addr, gp_mmu_irq) \ ++ { \ ++ .name = "Mali_GP", \ ++ .flags = IORESOURCE_MEM, \ ++ .start = gp_addr, \ ++ .end = gp_addr + 0x100, \ ++ }, \ ++ { \ ++ .name = "Mali_GP_IRQ", \ ++ .flags = IORESOURCE_IRQ, \ ++ .start = gp_irq, \ ++ .end = gp_irq, \ ++ }, \ ++ { \ ++ .name = "Mali_GP_MMU", \ ++ .flags = IORESOURCE_MEM, \ ++ .start = gp_mmu_addr, \ ++ .end = gp_mmu_addr + 0x100, \ ++ }, \ ++ { \ ++ .name = "Mali_GP_MMU_IRQ", \ ++ .flags = IORESOURCE_IRQ, \ ++ .start = gp_mmu_irq, \ ++ .end = gp_mmu_irq, \ ++ }, ++ ++#define MALI_GPU_RESOURCE_PP(pp_addr, pp_irq) \ ++ { \ ++ .name = "Mali_PP", \ ++ .flags = IORESOURCE_MEM, \ ++ .start = pp_addr, \ ++ .end = pp_addr + 0x1100, \ ++ }, \ ++ { \ ++ .name = "Mali_PP_IRQ", \ ++ .flags = IORESOURCE_IRQ, \ ++ .start = pp_irq, \ ++ .end = pp_irq, \ ++ }, \ ++ ++#define MALI_GPU_RESOURCE_PP_WITH_MMU(id, pp_addr, pp_irq, pp_mmu_addr, pp_mmu_irq) \ ++ { \ ++ .name = "Mali_PP" #id, \ ++ .flags = IORESOURCE_MEM, \ ++ .start = pp_addr, \ ++ .end = pp_addr + 0x1100, \ ++ }, \ ++ { \ ++ .name = "Mali_PP" #id "_IRQ", \ ++ .flags = IORESOURCE_IRQ, \ ++ .start = pp_irq, \ ++ .end = pp_irq, \ ++ }, \ ++ { \ ++ .name = "Mali_PP" #id "_MMU", \ ++ .flags = IORESOURCE_MEM, \ ++ .start = pp_mmu_addr, \ ++ .end = pp_mmu_addr + 0x100, \ ++ }, \ ++ { \ ++ .name = "Mali_PP" #id "_MMU_IRQ", \ ++ .flags = IORESOURCE_IRQ, \ ++ .start = pp_mmu_irq, \ ++ .end = pp_mmu_irq, \ ++ }, ++ ++#define MALI_GPU_RESOURCE_MMU(mmu_addr, mmu_irq) \ ++ { \ ++ .name = "Mali_MMU", \ ++ .flags = IORESOURCE_MEM, \ ++ .start = mmu_addr, \ ++ .end = mmu_addr + 0x100, \ ++ }, \ ++ { \ ++ .name = "Mali_MMU_IRQ", \ ++ .flags = IORESOURCE_IRQ, \ ++ .start = mmu_irq, \ ++ .end = mmu_irq, \ ++ }, ++ ++#define MALI_GPU_RESOURCE_PMU(pmu_addr) \ ++ { \ ++ .name = "Mali_PMU", \ ++ .flags = IORESOURCE_MEM, \ ++ .start = pmu_addr, \ ++ .end = pmu_addr + 0x100, \ ++ }, ++ ++#define MALI_GPU_RESOURCE_DMA(dma_addr) \ ++ { \ ++ .name = "Mali_DMA", \ ++ .flags = IORESOURCE_MEM, \ ++ .start = dma_addr, \ ++ .end = dma_addr + 0x100, \ ++ }, ++ ++#define MALI_GPU_RESOURCE_DLBU(dlbu_addr) \ ++ { \ ++ .name = "Mali_DLBU", \ ++ .flags = IORESOURCE_MEM, \ ++ .start = dlbu_addr, \ ++ .end = dlbu_addr + 0x100, \ ++ }, ++ ++#define MALI_GPU_RESOURCE_BCAST(bcast_addr) \ ++ { \ ++ .name = "Mali_Broadcast", \ ++ .flags = IORESOURCE_MEM, \ ++ .start = bcast_addr, \ ++ .end = bcast_addr + 0x100, \ ++ }, ++ ++#define MALI_GPU_RESOURCE_PP_BCAST(pp_addr, pp_irq) \ ++ { \ ++ .name = "Mali_PP_Broadcast", \ ++ .flags = IORESOURCE_MEM, \ ++ .start = pp_addr, \ ++ .end = pp_addr + 0x1100, \ ++ }, \ ++ { \ ++ .name = "Mali_PP_Broadcast_IRQ", \ ++ .flags = IORESOURCE_IRQ, \ ++ .start = pp_irq, \ ++ .end = pp_irq, \ ++ }, \ ++ ++#define MALI_GPU_RESOURCE_PP_MMU_BCAST(pp_mmu_bcast_addr) \ ++ { \ ++ .name = "Mali_PP_MMU_Broadcast", \ ++ .flags = IORESOURCE_MEM, \ ++ .start = pp_mmu_bcast_addr, \ ++ .end = pp_mmu_bcast_addr + 0x100, \ ++ }, ++ ++ struct mali_gpu_utilization_data { ++ unsigned int utilization_gpu; /* Utilization for GP and all PP cores combined, 0 = no utilization, 256 = full utilization */ ++ unsigned int utilization_gp; /* Utilization for GP core only, 0 = no utilization, 256 = full utilization */ ++ unsigned int utilization_pp; /* Utilization for all PP cores combined, 0 = no utilization, 256 = full utilization */ ++ }; ++ ++ struct mali_gpu_clk_item { ++ unsigned int clock; /* unit(MHz) */ ++ unsigned int vol; ++ }; ++ ++ struct mali_gpu_clock { ++ struct mali_gpu_clk_item *item; ++ unsigned int num_of_steps; ++ }; ++ ++ struct mali_gpu_device_data { ++ /* Shared GPU memory */ ++ unsigned long shared_mem_size; ++ ++ /* ++ * Mali PMU switch delay. ++ * Only needed if the power gates are connected to the PMU in a high fanout ++ * network. This value is the number of Mali clock cycles it takes to ++ * enable the power gates and turn on the power mesh. ++ * This value will have no effect if a daisy chain implementation is used. ++ */ ++ u32 pmu_switch_delay; ++ ++ /* Mali Dynamic power domain configuration in sequence from 0-11 ++ * GP PP0 PP1 PP2 PP3 PP4 PP5 PP6 PP7, L2$0 L2$1 L2$2 ++ */ ++ u16 pmu_domain_config[12]; ++ ++ /* Dedicated GPU memory range (physical). */ ++ unsigned long dedicated_mem_start; ++ unsigned long dedicated_mem_size; ++ ++ /* Frame buffer memory to be accessible by Mali GPU (physical) */ ++ unsigned long fb_start; ++ unsigned long fb_size; ++ ++ /* Max runtime [ms] for jobs */ ++ int max_job_runtime; ++ ++ /* Report GPU utilization and related control in this interval (specified in ms) */ ++ unsigned long control_interval; ++ ++ /* Function that will receive periodic GPU utilization numbers */ ++ void (*utilization_callback)(struct mali_gpu_utilization_data *data); ++ ++ /* Fuction that platform callback for freq setting, needed when CONFIG_MALI_DVFS enabled */ ++ int (*set_freq)(int setting_clock_step); ++ /* Function that platfrom report it's clock info which driver can set, needed when CONFIG_MALI_DVFS enabled */ ++ void (*get_clock_info)(struct mali_gpu_clock **data); ++ /* Function that get the current clock info, needed when CONFIG_MALI_DVFS enabled */ ++ int (*get_freq)(void); ++ /* Function that init the mali gpu secure mode */ ++ int (*secure_mode_init)(void); ++ /* Function that deinit the mali gpu secure mode */ ++ void (*secure_mode_deinit)(void); ++ /* Function that reset GPU and enable gpu secure mode */ ++ int (*gpu_reset_and_secure_mode_enable)(void); ++ /* Function that Reset GPU and disable gpu secure mode */ ++ int (*gpu_reset_and_secure_mode_disable)(void); ++ /* ipa related interface customer need register */ ++#if defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL) ++ struct devfreq_cooling_power *gpu_cooling_ops; ++#endif ++ }; ++ ++ /** ++ * Pause the scheduling and power state changes of Mali device driver. ++ * mali_dev_resume() must always be called as soon as possible after this function ++ * in order to resume normal operation of the Mali driver. ++ */ ++ void mali_dev_pause(void); ++ ++ /** ++ * Resume scheduling and allow power changes in Mali device driver. ++ * This must always be called after mali_dev_pause(). ++ */ ++ void mali_dev_resume(void); ++ ++ /** @brief Set the desired number of PP cores to use. ++ * ++ * The internal Mali PMU will be used, if present, to physically power off the PP cores. ++ * ++ * @param num_cores The number of desired cores ++ * @return 0 on success, otherwise error. -EINVAL means an invalid number of cores was specified. ++ */ ++ int mali_perf_set_num_pp_cores(unsigned int num_cores); ++ ++#endif +diff --git a/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_ioctl.h b/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_ioctl.h +new file mode 100755 +index 000000000000..686708eaef75 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_ioctl.h +@@ -0,0 +1,97 @@ ++/* ++ * Copyright (C) 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_UTGARD_IOCTL_H__ ++#define __MALI_UTGARD_IOCTL_H__ ++ ++#include ++#include ++#include /* file system operations */ ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++/** ++ * @file mali_kernel_ioctl.h ++ * Interface to the Linux device driver. ++ * This file describes the interface needed to use the Linux device driver. ++ * Its interface is designed to used by the HAL implementation through a thin arch layer. ++ */ ++ ++/** ++ * ioctl commands ++ */ ++ ++#define MALI_IOC_BASE 0x82 ++#define MALI_IOC_CORE_BASE (_MALI_UK_CORE_SUBSYSTEM + MALI_IOC_BASE) ++#define MALI_IOC_MEMORY_BASE (_MALI_UK_MEMORY_SUBSYSTEM + MALI_IOC_BASE) ++#define MALI_IOC_PP_BASE (_MALI_UK_PP_SUBSYSTEM + MALI_IOC_BASE) ++#define MALI_IOC_GP_BASE (_MALI_UK_GP_SUBSYSTEM + MALI_IOC_BASE) ++#define MALI_IOC_PROFILING_BASE (_MALI_UK_PROFILING_SUBSYSTEM + MALI_IOC_BASE) ++#define MALI_IOC_VSYNC_BASE (_MALI_UK_VSYNC_SUBSYSTEM + MALI_IOC_BASE) ++ ++#define MALI_IOC_WAIT_FOR_NOTIFICATION _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_WAIT_FOR_NOTIFICATION, _mali_uk_wait_for_notification_s) ++#define MALI_IOC_GET_API_VERSION _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_GET_API_VERSION, u32) ++#define MALI_IOC_GET_API_VERSION_V2 _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_GET_API_VERSION, _mali_uk_get_api_version_v2_s) ++/* rk_ext. */ ++#define MALI_IOC_GET_RK_KO_VERSION _IOWR(MALI_IOC_CORE_BASE, _MALI_GET_RK_KO_VERSION, _mali_rk_ko_version_s) ++#define MALI_IOC_POST_NOTIFICATION _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_POST_NOTIFICATION, _mali_uk_post_notification_s) ++#define MALI_IOC_GET_USER_SETTING _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_GET_USER_SETTING, _mali_uk_get_user_setting_s) ++#define MALI_IOC_GET_USER_SETTINGS _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_GET_USER_SETTINGS, _mali_uk_get_user_settings_s) ++#define MALI_IOC_REQUEST_HIGH_PRIORITY _IOW (MALI_IOC_CORE_BASE, _MALI_UK_REQUEST_HIGH_PRIORITY, _mali_uk_request_high_priority_s) ++#define MALI_IOC_TIMELINE_GET_LATEST_POINT _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_TIMELINE_GET_LATEST_POINT, _mali_uk_timeline_get_latest_point_s) ++#define MALI_IOC_TIMELINE_WAIT _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_TIMELINE_WAIT, _mali_uk_timeline_wait_s) ++#define MALI_IOC_TIMELINE_CREATE_SYNC_FENCE _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_TIMELINE_CREATE_SYNC_FENCE, _mali_uk_timeline_create_sync_fence_s) ++#define MALI_IOC_SOFT_JOB_START _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_SOFT_JOB_START, _mali_uk_soft_job_start_s) ++#define MALI_IOC_SOFT_JOB_SIGNAL _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_SOFT_JOB_SIGNAL, _mali_uk_soft_job_signal_s) ++#define MALI_IOC_PENDING_SUBMIT _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_PENDING_SUBMIT, _mali_uk_pending_submit_s) ++ ++#define MALI_IOC_MEM_ALLOC _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_ALLOC_MEM, _mali_uk_alloc_mem_s) ++#define MALI_IOC_MEM_FREE _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_FREE_MEM, _mali_uk_free_mem_s) ++#define MALI_IOC_MEM_BIND _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_BIND_MEM, _mali_uk_bind_mem_s) ++#define MALI_IOC_MEM_UNBIND _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_UNBIND_MEM, _mali_uk_unbind_mem_s) ++#define MALI_IOC_MEM_COW _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_COW_MEM, _mali_uk_cow_mem_s) ++#define MALI_IOC_MEM_COW_MODIFY_RANGE _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_COW_MODIFY_RANGE, _mali_uk_cow_modify_range_s) ++#define MALI_IOC_MEM_RESIZE _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_RESIZE_MEM, _mali_uk_mem_resize_s) ++#define MALI_IOC_MEM_DMA_BUF_GET_SIZE _IOR(MALI_IOC_MEMORY_BASE, _MALI_UK_DMA_BUF_GET_SIZE, _mali_uk_dma_buf_get_size_s) ++#define MALI_IOC_MEM_QUERY_MMU_PAGE_TABLE_DUMP_SIZE _IOR (MALI_IOC_MEMORY_BASE, _MALI_UK_QUERY_MMU_PAGE_TABLE_DUMP_SIZE, _mali_uk_query_mmu_page_table_dump_size_s) ++#define MALI_IOC_MEM_DUMP_MMU_PAGE_TABLE _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_DUMP_MMU_PAGE_TABLE, _mali_uk_dump_mmu_page_table_s) ++#define MALI_IOC_MEM_WRITE_SAFE _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_MEM_WRITE_SAFE, _mali_uk_mem_write_safe_s) ++ ++#define MALI_IOC_PP_START_JOB _IOWR(MALI_IOC_PP_BASE, _MALI_UK_PP_START_JOB, _mali_uk_pp_start_job_s) ++#define MALI_IOC_PP_AND_GP_START_JOB _IOWR(MALI_IOC_PP_BASE, _MALI_UK_PP_AND_GP_START_JOB, _mali_uk_pp_and_gp_start_job_s) ++#define MALI_IOC_PP_NUMBER_OF_CORES_GET _IOR (MALI_IOC_PP_BASE, _MALI_UK_GET_PP_NUMBER_OF_CORES, _mali_uk_get_pp_number_of_cores_s) ++#define MALI_IOC_PP_CORE_VERSION_GET _IOR (MALI_IOC_PP_BASE, _MALI_UK_GET_PP_CORE_VERSION, _mali_uk_get_pp_core_version_s) ++#define MALI_IOC_PP_DISABLE_WB _IOW (MALI_IOC_PP_BASE, _MALI_UK_PP_DISABLE_WB, _mali_uk_pp_disable_wb_s) ++ ++#define MALI_IOC_GP2_START_JOB _IOWR(MALI_IOC_GP_BASE, _MALI_UK_GP_START_JOB, _mali_uk_gp_start_job_s) ++#define MALI_IOC_GP2_NUMBER_OF_CORES_GET _IOR (MALI_IOC_GP_BASE, _MALI_UK_GET_GP_NUMBER_OF_CORES, _mali_uk_get_gp_number_of_cores_s) ++#define MALI_IOC_GP2_CORE_VERSION_GET _IOR (MALI_IOC_GP_BASE, _MALI_UK_GET_GP_CORE_VERSION, _mali_uk_get_gp_core_version_s) ++#define MALI_IOC_GP2_SUSPEND_RESPONSE _IOW (MALI_IOC_GP_BASE, _MALI_UK_GP_SUSPEND_RESPONSE,_mali_uk_gp_suspend_response_s) ++ ++#define MALI_IOC_PROFILING_ADD_EVENT _IOWR(MALI_IOC_PROFILING_BASE, _MALI_UK_PROFILING_ADD_EVENT, _mali_uk_profiling_add_event_s) ++#define MALI_IOC_PROFILING_REPORT_SW_COUNTERS _IOW (MALI_IOC_PROFILING_BASE, _MALI_UK_PROFILING_REPORT_SW_COUNTERS, _mali_uk_sw_counters_report_s) ++#define MALI_IOC_PROFILING_MEMORY_USAGE_GET _IOR(MALI_IOC_PROFILING_BASE, _MALI_UK_PROFILING_MEMORY_USAGE_GET, _mali_uk_profiling_memory_usage_get_s) ++#define MALI_IOC_PROFILING_STREAM_FD_GET _IOR(MALI_IOC_PROFILING_BASE, _MALI_UK_PROFILING_STREAM_FD_GET, _mali_uk_profiling_stream_fd_get_s) ++#define MALI_IOC_PROILING_CONTROL_SET _IOR(MALI_IOC_PROFILING_BASE, _MALI_UK_PROFILING_CONTROL_SET, _mali_uk_profiling_control_set_s) ++ ++#define MALI_IOC_VSYNC_EVENT_REPORT _IOW (MALI_IOC_VSYNC_BASE, _MALI_UK_VSYNC_EVENT_REPORT, _mali_uk_vsync_event_report_s) ++ ++/* rk_ext : 对 r5p0 集æˆä¹‹åŽ, mali_so ä¸å†ä½¿ç”¨ä¸‹é¢çš„ ioctl, 而使用 MALI_IOC_GET_RK_KO_VERSION. */ ++#if 0 ++#define MALI_IOC_GET_MALI_VERSION_IN_RK30 _IOWR(MALI_IOC_CORE_BASE,_MALI_UK_GET_MALI_VERSION_IN_RK30,_mali_uk_get_mali_version_in_rk30_s *) ++#endif ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif /* __MALI_UTGARD_IOCTL_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_profiling_events.h b/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_profiling_events.h +new file mode 100755 +index 000000000000..17d31de931d0 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_profiling_events.h +@@ -0,0 +1,190 @@ ++/* ++ * Copyright (C) 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef _MALI_UTGARD_PROFILING_EVENTS_H_ ++#define _MALI_UTGARD_PROFILING_EVENTS_H_ ++ ++/* ++ * The event ID is a 32 bit value consisting of different fields ++ * reserved, 4 bits, for future use ++ * event type, 4 bits, cinstr_profiling_event_type_t ++ * event channel, 8 bits, the source of the event. ++ * event data, 16 bit field, data depending on event type ++ */ ++ ++/** ++ * Specifies what kind of event this is ++ */ ++typedef enum { ++ MALI_PROFILING_EVENT_TYPE_SINGLE = 0 << 24, ++ MALI_PROFILING_EVENT_TYPE_START = 1 << 24, ++ MALI_PROFILING_EVENT_TYPE_STOP = 2 << 24, ++ MALI_PROFILING_EVENT_TYPE_SUSPEND = 3 << 24, ++ MALI_PROFILING_EVENT_TYPE_RESUME = 4 << 24, ++} cinstr_profiling_event_type_t; ++ ++ ++/** ++ * Secifies the channel/source of the event ++ */ ++typedef enum { ++ MALI_PROFILING_EVENT_CHANNEL_SOFTWARE = 0 << 16, ++ MALI_PROFILING_EVENT_CHANNEL_GP0 = 1 << 16, ++ MALI_PROFILING_EVENT_CHANNEL_PP0 = 5 << 16, ++ MALI_PROFILING_EVENT_CHANNEL_PP1 = 6 << 16, ++ MALI_PROFILING_EVENT_CHANNEL_PP2 = 7 << 16, ++ MALI_PROFILING_EVENT_CHANNEL_PP3 = 8 << 16, ++ MALI_PROFILING_EVENT_CHANNEL_PP4 = 9 << 16, ++ MALI_PROFILING_EVENT_CHANNEL_PP5 = 10 << 16, ++ MALI_PROFILING_EVENT_CHANNEL_PP6 = 11 << 16, ++ MALI_PROFILING_EVENT_CHANNEL_PP7 = 12 << 16, ++ MALI_PROFILING_EVENT_CHANNEL_GPU = 21 << 16, ++} cinstr_profiling_event_channel_t; ++ ++ ++#define MALI_PROFILING_MAKE_EVENT_CHANNEL_GP(num) (((MALI_PROFILING_EVENT_CHANNEL_GP0 >> 16) + (num)) << 16) ++#define MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(num) (((MALI_PROFILING_EVENT_CHANNEL_PP0 >> 16) + (num)) << 16) ++ ++/** ++ * These events are applicable when the type MALI_PROFILING_EVENT_TYPE_SINGLE is used from software channel ++ */ ++typedef enum { ++ MALI_PROFILING_EVENT_REASON_SINGLE_SW_NONE = 0, ++ MALI_PROFILING_EVENT_REASON_SINGLE_SW_EGL_NEW_FRAME = 1, ++ MALI_PROFILING_EVENT_REASON_SINGLE_SW_FLUSH = 2, ++ MALI_PROFILING_EVENT_REASON_SINGLE_SW_EGL_SWAP_BUFFERS = 3, ++ MALI_PROFILING_EVENT_REASON_SINGLE_SW_FB_EVENT = 4, ++ MALI_PROFILING_EVENT_REASON_SINGLE_SW_GP_ENQUEUE = 5, ++ MALI_PROFILING_EVENT_REASON_SINGLE_SW_PP_ENQUEUE = 6, ++ MALI_PROFILING_EVENT_REASON_SINGLE_SW_READBACK = 7, ++ MALI_PROFILING_EVENT_REASON_SINGLE_SW_WRITEBACK = 8, ++ MALI_PROFILING_EVENT_REASON_SINGLE_SW_ENTER_API_FUNC = 10, ++ MALI_PROFILING_EVENT_REASON_SINGLE_SW_LEAVE_API_FUNC = 11, ++ MALI_PROFILING_EVENT_REASON_SINGLE_SW_DISCARD_ATTACHMENTS = 13, ++ MALI_PROFILING_EVENT_REASON_SINGLE_SW_UMP_TRY_LOCK = 53, ++ MALI_PROFILING_EVENT_REASON_SINGLE_SW_UMP_LOCK = 54, ++ MALI_PROFILING_EVENT_REASON_SINGLE_SW_UMP_UNLOCK = 55, ++ MALI_PROFILING_EVENT_REASON_SINGLE_LOCK_CONTENDED = 56, ++ MALI_PROFILING_EVENT_REASON_SINGLE_SW_EGL_MALI_FENCE_DUP = 57, ++ MALI_PROFILING_EVENT_REASON_SINGLE_SW_EGL_SET_PP_JOB_FENCE = 58, ++ MALI_PROFILING_EVENT_REASON_SINGLE_SW_EGL_WAIT_SYNC = 59, ++ MALI_PROFILING_EVENT_REASON_SINGLE_SW_EGL_CREATE_FENCE_SYNC = 60, ++ MALI_PROFILING_EVENT_REASON_SINGLE_SW_EGL_CREATE_NATIVE_FENCE_SYNC = 61, ++ MALI_PROFILING_EVENT_REASON_SINGLE_SW_EGL_FENCE_FLUSH = 62, ++ MALI_PROFILING_EVENT_REASON_SINGLE_SW_EGL_FLUSH_SERVER_WAITS = 63, ++} cinstr_profiling_event_reason_single_sw_t; ++ ++/** ++ * These events are applicable when the type MALI_PROFILING_EVENT_TYPE_START/STOP is used from software channel ++ * to inform whether the core is physical or virtual ++ */ ++typedef enum { ++ MALI_PROFILING_EVENT_REASON_START_STOP_HW_PHYSICAL = 0, ++ MALI_PROFILING_EVENT_REASON_START_STOP_HW_VIRTUAL = 1, ++} cinstr_profiling_event_reason_start_stop_hw_t; ++ ++/** ++ * These events are applicable when the type MALI_PROFILING_EVENT_TYPE_START/STOP is used from software channel ++ */ ++typedef enum { ++ /*MALI_PROFILING_EVENT_REASON_START_STOP_SW_NONE = 0,*/ ++ MALI_PROFILING_EVENT_REASON_START_STOP_SW_MALI = 1, ++ MALI_PROFILING_EVENT_REASON_START_STOP_SW_CALLBACK_THREAD = 2, ++ MALI_PROFILING_EVENT_REASON_START_STOP_SW_WORKER_THREAD = 3, ++ MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF = 4, ++ MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF = 5, ++} cinstr_profiling_event_reason_start_stop_sw_t; ++ ++/** ++ * These events are applicable when the type MALI_PROFILING_EVENT_TYPE_SUSPEND/RESUME is used from software channel ++ */ ++typedef enum { ++ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_NONE = 0, /* used */ ++ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_PIPELINE_FULL = 1, /* NOT used */ ++ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_VSYNC = 26, /* used in some build configurations */ ++ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_FB_IFRAME_WAIT = 27, /* USED */ ++ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_FB_IFRAME_SYNC = 28, /* USED */ ++ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_VG_WAIT_FILTER_CLEANUP = 29, /* used */ ++ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_VG_WAIT_TEXTURE = 30, /* used */ ++ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_GLES_WAIT_MIPLEVEL = 31, /* used */ ++ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_GLES_WAIT_READPIXELS = 32, /* used */ ++ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_EGL_WAIT_SWAP_IMMEDIATE = 33, /* NOT used */ ++ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_ICS_QUEUE_BUFFER = 34, /* USED */ ++ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_ICS_DEQUEUE_BUFFER = 35, /* USED */ ++ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_UMP_LOCK = 36, /* Not currently used */ ++ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_X11_GLOBAL_LOCK = 37, /* Not currently used */ ++ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_X11_SWAP = 38, /* Not currently used */ ++ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_MALI_EGL_IMAGE_SYNC_WAIT = 39, /* USED */ ++ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_GP_JOB_HANDLING = 40, /* USED */ ++ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_PP_JOB_HANDLING = 41, /* USED */ ++ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_EGL_MALI_FENCE_MERGE = 42, /* USED */ ++ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_EGL_MALI_FENCE_DUP = 43, ++ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_EGL_FLUSH_SERVER_WAITS = 44, ++ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_EGL_WAIT_SYNC = 45, /* USED */ ++ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_FB_JOBS_WAIT = 46, /* USED */ ++ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_FB_NOFRAMES_WAIT = 47, /* USED */ ++ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_FB_NOJOBS_WAIT = 48, /* USED */ ++ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_SUBMIT_LIMITER_WAIT = 49, /* USED */ ++} cinstr_profiling_event_reason_suspend_resume_sw_t; ++ ++/** ++ * These events are applicable when the type MALI_PROFILING_EVENT_TYPE_SINGLE is used from a HW channel (GPx+PPx) ++ */ ++typedef enum { ++ MALI_PROFILING_EVENT_REASON_SINGLE_HW_NONE = 0, ++ MALI_PROFILING_EVENT_REASON_SINGLE_HW_INTERRUPT = 1, ++ MALI_PROFILING_EVENT_REASON_SINGLE_HW_FLUSH = 2, ++} cinstr_profiling_event_reason_single_hw_t; ++ ++/** ++ * These events are applicable when the type MALI_PROFILING_EVENT_TYPE_SINGLE is used from the GPU channel ++ */ ++typedef enum { ++ MALI_PROFILING_EVENT_REASON_SINGLE_GPU_NONE = 0, ++ MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE = 1, ++ MALI_PROFILING_EVENT_REASON_SINGLE_GPU_L20_COUNTERS = 2, ++ MALI_PROFILING_EVENT_REASON_SINGLE_GPU_L21_COUNTERS = 3, ++ MALI_PROFILING_EVENT_REASON_SINGLE_GPU_L22_COUNTERS = 4, ++} cinstr_profiling_event_reason_single_gpu_t; ++ ++/** ++ * These values are applicable for the 3rd data parameter when ++ * the type MALI_PROFILING_EVENT_TYPE_START is used from the software channel ++ * with the MALI_PROFILING_EVENT_REASON_START_STOP_BOTTOM_HALF reason. ++ */ ++typedef enum { ++ MALI_PROFILING_EVENT_DATA_CORE_GP0 = 1, ++ MALI_PROFILING_EVENT_DATA_CORE_PP0 = 5, ++ MALI_PROFILING_EVENT_DATA_CORE_PP1 = 6, ++ MALI_PROFILING_EVENT_DATA_CORE_PP2 = 7, ++ MALI_PROFILING_EVENT_DATA_CORE_PP3 = 8, ++ MALI_PROFILING_EVENT_DATA_CORE_PP4 = 9, ++ MALI_PROFILING_EVENT_DATA_CORE_PP5 = 10, ++ MALI_PROFILING_EVENT_DATA_CORE_PP6 = 11, ++ MALI_PROFILING_EVENT_DATA_CORE_PP7 = 12, ++ MALI_PROFILING_EVENT_DATA_CORE_GP0_MMU = 22, /* GP0 + 21 */ ++ MALI_PROFILING_EVENT_DATA_CORE_PP0_MMU = 26, /* PP0 + 21 */ ++ MALI_PROFILING_EVENT_DATA_CORE_PP1_MMU = 27, /* PP1 + 21 */ ++ MALI_PROFILING_EVENT_DATA_CORE_PP2_MMU = 28, /* PP2 + 21 */ ++ MALI_PROFILING_EVENT_DATA_CORE_PP3_MMU = 29, /* PP3 + 21 */ ++ MALI_PROFILING_EVENT_DATA_CORE_PP4_MMU = 30, /* PP4 + 21 */ ++ MALI_PROFILING_EVENT_DATA_CORE_PP5_MMU = 31, /* PP5 + 21 */ ++ MALI_PROFILING_EVENT_DATA_CORE_PP6_MMU = 32, /* PP6 + 21 */ ++ MALI_PROFILING_EVENT_DATA_CORE_PP7_MMU = 33, /* PP7 + 21 */ ++ ++} cinstr_profiling_event_data_core_t; ++ ++#define MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP(num) (MALI_PROFILING_EVENT_DATA_CORE_GP0 + (num)) ++#define MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP_MMU(num) (MALI_PROFILING_EVENT_DATA_CORE_GP0_MMU + (num)) ++#define MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP(num) (MALI_PROFILING_EVENT_DATA_CORE_PP0 + (num)) ++#define MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP_MMU(num) (MALI_PROFILING_EVENT_DATA_CORE_PP0_MMU + (num)) ++ ++ ++#endif /*_MALI_UTGARD_PROFILING_EVENTS_H_*/ +diff --git a/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_profiling_gator_api.h b/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_profiling_gator_api.h +new file mode 100755 +index 000000000000..c1927d1450dc +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_profiling_gator_api.h +@@ -0,0 +1,305 @@ ++/* ++ * Copyright (C) 2013, 2015-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_UTGARD_PROFILING_GATOR_API_H__ ++#define __MALI_UTGARD_PROFILING_GATOR_API_H__ ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++#define MALI_PROFILING_API_VERSION 4 ++ ++#define MAX_NUM_L2_CACHE_CORES 3 ++#define MAX_NUM_FP_CORES 8 ++#define MAX_NUM_VP_CORES 1 ++ ++#define _MALI_SPCIAL_COUNTER_DESCRIPTIONS \ ++ { \ ++ "Filmstrip_cnt0", \ ++ "Frequency", \ ++ "Voltage", \ ++ "vertex", \ ++ "fragment", \ ++ "Total_alloc_pages", \ ++ }; ++ ++#define _MALI_MEM_COUTNER_DESCRIPTIONS \ ++ { \ ++ "untyped_memory", \ ++ "vertex_index_buffer", \ ++ "texture_buffer", \ ++ "varying_buffer", \ ++ "render_target", \ ++ "pbuffer_buffer", \ ++ "plbu_heap", \ ++ "pointer_array_buffer", \ ++ "slave_tilelist", \ ++ "untyped_gp_cmdlist", \ ++ "polygon_cmdlist", \ ++ "texture_descriptor", \ ++ "render_state_word", \ ++ "shader", \ ++ "stream_buffer", \ ++ "fragment_stack", \ ++ "uniform", \ ++ "untyped_frame_pool", \ ++ "untyped_surface", \ ++ }; ++ ++/** The list of events supported by the Mali DDK. */ ++typedef enum { ++ /* Vertex processor activity */ ++ ACTIVITY_VP_0 = 0, ++ ++ /* Fragment processor activity */ ++ ACTIVITY_FP_0, ++ ACTIVITY_FP_1, ++ ACTIVITY_FP_2, ++ ACTIVITY_FP_3, ++ ACTIVITY_FP_4, ++ ACTIVITY_FP_5, ++ ACTIVITY_FP_6, ++ ACTIVITY_FP_7, ++ ++ /* L2 cache counters */ ++ COUNTER_L2_0_C0, ++ COUNTER_L2_0_C1, ++ COUNTER_L2_1_C0, ++ COUNTER_L2_1_C1, ++ COUNTER_L2_2_C0, ++ COUNTER_L2_2_C1, ++ ++ /* Vertex processor counters */ ++ COUNTER_VP_0_C0, ++ COUNTER_VP_0_C1, ++ ++ /* Fragment processor counters */ ++ COUNTER_FP_0_C0, ++ COUNTER_FP_0_C1, ++ COUNTER_FP_1_C0, ++ COUNTER_FP_1_C1, ++ COUNTER_FP_2_C0, ++ COUNTER_FP_2_C1, ++ COUNTER_FP_3_C0, ++ COUNTER_FP_3_C1, ++ COUNTER_FP_4_C0, ++ COUNTER_FP_4_C1, ++ COUNTER_FP_5_C0, ++ COUNTER_FP_5_C1, ++ COUNTER_FP_6_C0, ++ COUNTER_FP_6_C1, ++ COUNTER_FP_7_C0, ++ COUNTER_FP_7_C1, ++ ++ /* ++ * If more hardware counters are added, the _mali_osk_hw_counter_table ++ * below should also be updated. ++ */ ++ ++ /* EGL software counters */ ++ COUNTER_EGL_BLIT_TIME, ++ ++ /* GLES software counters */ ++ COUNTER_GLES_DRAW_ELEMENTS_CALLS, ++ COUNTER_GLES_DRAW_ELEMENTS_NUM_INDICES, ++ COUNTER_GLES_DRAW_ELEMENTS_NUM_TRANSFORMED, ++ COUNTER_GLES_DRAW_ARRAYS_CALLS, ++ COUNTER_GLES_DRAW_ARRAYS_NUM_TRANSFORMED, ++ COUNTER_GLES_DRAW_POINTS, ++ COUNTER_GLES_DRAW_LINES, ++ COUNTER_GLES_DRAW_LINE_LOOP, ++ COUNTER_GLES_DRAW_LINE_STRIP, ++ COUNTER_GLES_DRAW_TRIANGLES, ++ COUNTER_GLES_DRAW_TRIANGLE_STRIP, ++ COUNTER_GLES_DRAW_TRIANGLE_FAN, ++ COUNTER_GLES_NON_VBO_DATA_COPY_TIME, ++ COUNTER_GLES_UNIFORM_BYTES_COPIED_TO_MALI, ++ COUNTER_GLES_UPLOAD_TEXTURE_TIME, ++ COUNTER_GLES_UPLOAD_VBO_TIME, ++ COUNTER_GLES_NUM_FLUSHES, ++ COUNTER_GLES_NUM_VSHADERS_GENERATED, ++ COUNTER_GLES_NUM_FSHADERS_GENERATED, ++ COUNTER_GLES_VSHADER_GEN_TIME, ++ COUNTER_GLES_FSHADER_GEN_TIME, ++ COUNTER_GLES_INPUT_TRIANGLES, ++ COUNTER_GLES_VXCACHE_HIT, ++ COUNTER_GLES_VXCACHE_MISS, ++ COUNTER_GLES_VXCACHE_COLLISION, ++ COUNTER_GLES_CULLED_TRIANGLES, ++ COUNTER_GLES_CULLED_LINES, ++ COUNTER_GLES_BACKFACE_TRIANGLES, ++ COUNTER_GLES_GBCLIP_TRIANGLES, ++ COUNTER_GLES_GBCLIP_LINES, ++ COUNTER_GLES_TRIANGLES_DRAWN, ++ COUNTER_GLES_DRAWCALL_TIME, ++ COUNTER_GLES_TRIANGLES_COUNT, ++ COUNTER_GLES_INDEPENDENT_TRIANGLES_COUNT, ++ COUNTER_GLES_STRIP_TRIANGLES_COUNT, ++ COUNTER_GLES_FAN_TRIANGLES_COUNT, ++ COUNTER_GLES_LINES_COUNT, ++ COUNTER_GLES_INDEPENDENT_LINES_COUNT, ++ COUNTER_GLES_STRIP_LINES_COUNT, ++ COUNTER_GLES_LOOP_LINES_COUNT, ++ ++ /* Special counter */ ++ ++ /* Framebuffer capture pseudo-counter */ ++ COUNTER_FILMSTRIP, ++ COUNTER_FREQUENCY, ++ COUNTER_VOLTAGE, ++ COUNTER_VP_ACTIVITY, ++ COUNTER_FP_ACTIVITY, ++ COUNTER_TOTAL_ALLOC_PAGES, ++ ++ /* Memory usage counter */ ++ COUNTER_MEM_UNTYPED, ++ COUNTER_MEM_VB_IB, ++ COUNTER_MEM_TEXTURE, ++ COUNTER_MEM_VARYING, ++ COUNTER_MEM_RT, ++ COUNTER_MEM_PBUFFER, ++ /* memory usages for gp command */ ++ COUNTER_MEM_PLBU_HEAP, ++ COUNTER_MEM_POINTER_ARRAY, ++ COUNTER_MEM_SLAVE_TILELIST, ++ COUNTER_MEM_UNTYPE_GP_CMDLIST, ++ /* memory usages for polygon list command */ ++ COUNTER_MEM_POLYGON_CMDLIST, ++ /* memory usages for pp command */ ++ COUNTER_MEM_TD, ++ COUNTER_MEM_RSW, ++ /* other memory usages */ ++ COUNTER_MEM_SHADER, ++ COUNTER_MEM_STREAMS, ++ COUNTER_MEM_FRAGMENT_STACK, ++ COUNTER_MEM_UNIFORM, ++ /* Special mem usage, which is used for mem pool allocation */ ++ COUNTER_MEM_UNTYPE_MEM_POOL, ++ COUNTER_MEM_UNTYPE_SURFACE, ++ ++ NUMBER_OF_EVENTS ++} _mali_osk_counter_id; ++ ++#define FIRST_ACTIVITY_EVENT ACTIVITY_VP_0 ++#define LAST_ACTIVITY_EVENT ACTIVITY_FP_7 ++ ++#define FIRST_HW_COUNTER COUNTER_L2_0_C0 ++#define LAST_HW_COUNTER COUNTER_FP_7_C1 ++ ++#define FIRST_SW_COUNTER COUNTER_EGL_BLIT_TIME ++#define LAST_SW_COUNTER COUNTER_GLES_LOOP_LINES_COUNT ++ ++#define FIRST_SPECIAL_COUNTER COUNTER_FILMSTRIP ++#define LAST_SPECIAL_COUNTER COUNTER_TOTAL_ALLOC_PAGES ++ ++#define FIRST_MEM_COUNTER COUNTER_MEM_UNTYPED ++#define LAST_MEM_COUNTER COUNTER_MEM_UNTYPE_SURFACE ++ ++#define MALI_PROFILING_MEM_COUNTERS_NUM (LAST_MEM_COUNTER - FIRST_MEM_COUNTER + 1) ++#define MALI_PROFILING_SPECIAL_COUNTERS_NUM (LAST_SPECIAL_COUNTER - FIRST_SPECIAL_COUNTER + 1) ++#define MALI_PROFILING_SW_COUNTERS_NUM (LAST_SW_COUNTER - FIRST_SW_COUNTER + 1) ++ ++/** ++ * Define the stream header type for porfiling stream. ++ */ ++#define STREAM_HEADER_FRAMEBUFFER 0x05 /* The stream packet header type for framebuffer dumping. */ ++#define STREAM_HEADER_COUNTER_VALUE 0x09 /* The stream packet header type for hw/sw/memory counter sampling. */ ++#define STREAM_HEADER_CORE_ACTIVITY 0x0a /* The stream packet header type for activity counter sampling. */ ++#define STREAM_HEADER_SIZE 5 ++ ++/** ++ * Define the packet header type of profiling control packet. ++ */ ++#define PACKET_HEADER_ERROR 0x80 /* The response packet header type if error. */ ++#define PACKET_HEADER_ACK 0x81 /* The response packet header type if OK. */ ++#define PACKET_HEADER_COUNTERS_REQUEST 0x82 /* The control packet header type to request counter information from ddk. */ ++#define PACKET_HEADER_COUNTERS_ACK 0x83 /* The response packet header type to send out counter information. */ ++#define PACKET_HEADER_COUNTERS_ENABLE 0x84 /* The control packet header type to enable counters. */ ++#define PACKET_HEADER_START_CAPTURE_VALUE 0x85 /* The control packet header type to start capture values. */ ++ ++#define PACKET_HEADER_SIZE 5 ++ ++/** ++ * Structure to pass performance counter data of a Mali core ++ */ ++typedef struct _mali_profiling_core_counters { ++ u32 source0; ++ u32 value0; ++ u32 source1; ++ u32 value1; ++} _mali_profiling_core_counters; ++ ++/** ++ * Structure to pass performance counter data of Mali L2 cache cores ++ */ ++typedef struct _mali_profiling_l2_counter_values { ++ struct _mali_profiling_core_counters cores[MAX_NUM_L2_CACHE_CORES]; ++} _mali_profiling_l2_counter_values; ++ ++/** ++ * Structure to pass data defining Mali instance in use: ++ * ++ * mali_product_id - Mali product id ++ * mali_version_major - Mali version major number ++ * mali_version_minor - Mali version minor number ++ * num_of_l2_cores - number of L2 cache cores ++ * num_of_fp_cores - number of fragment processor cores ++ * num_of_vp_cores - number of vertex processor cores ++ */ ++typedef struct _mali_profiling_mali_version { ++ u32 mali_product_id; ++ u32 mali_version_major; ++ u32 mali_version_minor; ++ u32 num_of_l2_cores; ++ u32 num_of_fp_cores; ++ u32 num_of_vp_cores; ++} _mali_profiling_mali_version; ++ ++/** ++ * Structure to define the mali profiling counter struct. ++ */ ++typedef struct mali_profiling_counter { ++ char counter_name[40]; ++ u32 counter_id; ++ u32 counter_event; ++ u32 prev_counter_value; ++ u32 current_counter_value; ++ u32 key; ++ int enabled; ++} mali_profiling_counter; ++ ++/* ++ * List of possible actions to be controlled by Streamline. ++ * The following numbers are used by gator to control the frame buffer dumping and s/w counter reporting. ++ * We cannot use the enums in mali_uk_types.h because they are unknown inside gator. ++ */ ++#define FBDUMP_CONTROL_ENABLE (1) ++#define FBDUMP_CONTROL_RATE (2) ++#define SW_COUNTER_ENABLE (3) ++#define FBDUMP_CONTROL_RESIZE_FACTOR (4) ++#define MEM_COUNTER_ENABLE (5) ++#define ANNOTATE_PROFILING_ENABLE (6) ++ ++void _mali_profiling_control(u32 action, u32 value); ++ ++u32 _mali_profiling_get_l2_counters(_mali_profiling_l2_counter_values *values); ++ ++int _mali_profiling_set_event(u32 counter_id, s32 event_id); ++ ++u32 _mali_profiling_get_api_version(void); ++ ++void _mali_profiling_get_mali_version(struct _mali_profiling_mali_version *values); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif /* __MALI_UTGARD_PROFILING_GATOR_API_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_uk_types.h b/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_uk_types.h +new file mode 100755 +index 000000000000..34656f09b2ab +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_uk_types.h +@@ -0,0 +1,1108 @@ ++/* ++ * Copyright (C) 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file mali_uk_types.h ++ * Defines the types and constants used in the user-kernel interface ++ */ ++ ++#ifndef __MALI_UTGARD_UK_TYPES_H__ ++#define __MALI_UTGARD_UK_TYPES_H__ ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++/* Iteration functions depend on these values being consecutive. */ ++#define MALI_UK_TIMELINE_GP 0 ++#define MALI_UK_TIMELINE_PP 1 ++#define MALI_UK_TIMELINE_SOFT 2 ++#define MALI_UK_TIMELINE_MAX 3 ++ ++#define MALI_UK_BIG_VARYING_SIZE (1024*1024*2) ++ ++typedef struct { ++ u32 points[MALI_UK_TIMELINE_MAX]; ++ s32 sync_fd; ++} _mali_uk_fence_t; ++ ++/** ++ * @addtogroup uddapi Unified Device Driver (UDD) APIs ++ * ++ * @{ ++ */ ++ ++/** ++ * @addtogroup u_k_api UDD User/Kernel Interface (U/K) APIs ++ * ++ * @{ ++ */ ++ ++/** @defgroup _mali_uk_core U/K Core ++ * @{ */ ++ ++/** Definition of subsystem numbers, to assist in creating a unique identifier ++ * for each U/K call. ++ * ++ * @see _mali_uk_functions */ ++typedef enum { ++ _MALI_UK_CORE_SUBSYSTEM, /**< Core Group of U/K calls */ ++ _MALI_UK_MEMORY_SUBSYSTEM, /**< Memory Group of U/K calls */ ++ _MALI_UK_PP_SUBSYSTEM, /**< Fragment Processor Group of U/K calls */ ++ _MALI_UK_GP_SUBSYSTEM, /**< Vertex Processor Group of U/K calls */ ++ _MALI_UK_PROFILING_SUBSYSTEM, /**< Profiling Group of U/K calls */ ++ _MALI_UK_VSYNC_SUBSYSTEM, /**< VSYNC Group of U/K calls */ ++} _mali_uk_subsystem_t; ++ ++/** Within a function group each function has its unique sequence number ++ * to assist in creating a unique identifier for each U/K call. ++ * ++ * An ordered pair of numbers selected from ++ * ( \ref _mali_uk_subsystem_t,\ref _mali_uk_functions) will uniquely identify the ++ * U/K call across all groups of functions, and all functions. */ ++typedef enum { ++ /** Core functions */ ++ ++ _MALI_UK_OPEN = 0, /**< _mali_ukk_open() */ ++ _MALI_UK_CLOSE, /**< _mali_ukk_close() */ ++ _MALI_UK_WAIT_FOR_NOTIFICATION, /**< _mali_ukk_wait_for_notification() */ ++ _MALI_UK_GET_API_VERSION, /**< _mali_ukk_get_api_version() */ ++ _MALI_UK_POST_NOTIFICATION, /**< _mali_ukk_post_notification() */ ++ _MALI_UK_GET_USER_SETTING, /**< _mali_ukk_get_user_setting() *//**< [out] */ ++ _MALI_UK_GET_USER_SETTINGS, /**< _mali_ukk_get_user_settings() *//**< [out] */ ++ _MALI_UK_REQUEST_HIGH_PRIORITY, /**< _mali_ukk_request_high_priority() */ ++ _MALI_UK_TIMELINE_GET_LATEST_POINT, /**< _mali_ukk_timeline_get_latest_point() */ ++ _MALI_UK_TIMELINE_WAIT, /**< _mali_ukk_timeline_wait() */ ++ _MALI_UK_TIMELINE_CREATE_SYNC_FENCE, /**< _mali_ukk_timeline_create_sync_fence() */ ++ _MALI_UK_SOFT_JOB_START, /**< _mali_ukk_soft_job_start() */ ++ _MALI_UK_SOFT_JOB_SIGNAL, /**< _mali_ukk_soft_job_signal() */ ++ _MALI_UK_PENDING_SUBMIT, /**< _mali_ukk_pending_submit() */ ++ ++ _MALI_GET_RK_KO_VERSION, /* rk_ext */ ++ _MALI_UK_GET_MALI_VERSION_IN_RK30, ++ ++ /** Memory functions */ ++ ++ _MALI_UK_ALLOC_MEM = 0, /**< _mali_ukk_alloc_mem() */ ++ _MALI_UK_FREE_MEM, /**< _mali_ukk_free_mem() */ ++ _MALI_UK_BIND_MEM, /**< _mali_ukk_mem_bind() */ ++ _MALI_UK_UNBIND_MEM, /**< _mali_ukk_mem_unbind() */ ++ _MALI_UK_COW_MEM, /**< _mali_ukk_mem_cow() */ ++ _MALI_UK_COW_MODIFY_RANGE, /**< _mali_ukk_mem_cow_modify_range() */ ++ _MALI_UK_RESIZE_MEM, /**<._mali_ukk_mem_resize() */ ++ _MALI_UK_QUERY_MMU_PAGE_TABLE_DUMP_SIZE, /**< _mali_ukk_mem_get_mmu_page_table_dump_size() */ ++ _MALI_UK_DUMP_MMU_PAGE_TABLE, /**< _mali_ukk_mem_dump_mmu_page_table() */ ++ _MALI_UK_DMA_BUF_GET_SIZE, /**< _mali_ukk_dma_buf_get_size() */ ++ _MALI_UK_MEM_WRITE_SAFE, /**< _mali_uku_mem_write_safe() */ ++ ++ /** Common functions for each core */ ++ ++ _MALI_UK_START_JOB = 0, /**< Start a Fragment/Vertex Processor Job on a core */ ++ _MALI_UK_GET_NUMBER_OF_CORES, /**< Get the number of Fragment/Vertex Processor cores */ ++ _MALI_UK_GET_CORE_VERSION, /**< Get the Fragment/Vertex Processor version compatible with all cores */ ++ ++ /** Fragment Processor Functions */ ++ ++ _MALI_UK_PP_START_JOB = _MALI_UK_START_JOB, /**< _mali_ukk_pp_start_job() */ ++ _MALI_UK_GET_PP_NUMBER_OF_CORES = _MALI_UK_GET_NUMBER_OF_CORES, /**< _mali_ukk_get_pp_number_of_cores() */ ++ _MALI_UK_GET_PP_CORE_VERSION = _MALI_UK_GET_CORE_VERSION, /**< _mali_ukk_get_pp_core_version() */ ++ _MALI_UK_PP_DISABLE_WB, /**< _mali_ukk_pp_job_disable_wb() */ ++ _MALI_UK_PP_AND_GP_START_JOB, /**< _mali_ukk_pp_and_gp_start_job() */ ++ ++ /** Vertex Processor Functions */ ++ ++ _MALI_UK_GP_START_JOB = _MALI_UK_START_JOB, /**< _mali_ukk_gp_start_job() */ ++ _MALI_UK_GET_GP_NUMBER_OF_CORES = _MALI_UK_GET_NUMBER_OF_CORES, /**< _mali_ukk_get_gp_number_of_cores() */ ++ _MALI_UK_GET_GP_CORE_VERSION = _MALI_UK_GET_CORE_VERSION, /**< _mali_ukk_get_gp_core_version() */ ++ _MALI_UK_GP_SUSPEND_RESPONSE, /**< _mali_ukk_gp_suspend_response() */ ++ ++ /** Profiling functions */ ++ ++ _MALI_UK_PROFILING_ADD_EVENT = 0, /**< __mali_uku_profiling_add_event() */ ++ _MALI_UK_PROFILING_REPORT_SW_COUNTERS,/**< __mali_uku_profiling_report_sw_counters() */ ++ _MALI_UK_PROFILING_MEMORY_USAGE_GET, /**< __mali_uku_profiling_memory_usage_get() */ ++ _MALI_UK_PROFILING_STREAM_FD_GET, /** < __mali_uku_profiling_stream_fd_get() */ ++ _MALI_UK_PROFILING_CONTROL_SET, /** < __mali_uku_profiling_control_set() */ ++ ++ /** VSYNC reporting fuctions */ ++ _MALI_UK_VSYNC_EVENT_REPORT = 0, /**< _mali_ukk_vsync_event_report() */ ++} _mali_uk_functions; ++ ++/** @defgroup _mali_uk_getsysteminfo U/K Get System Info ++ * @{ */ ++ ++/** ++ * Type definition for the core version number. ++ * Used when returning the version number read from a core ++ * ++ * Its format is that of the 32-bit Version register for a particular core. ++ * Refer to the "Mali200 and MaliGP2 3D Graphics Processor Technical Reference ++ * Manual", ARM DDI 0415C, for more information. ++ */ ++typedef u32 _mali_core_version; ++ ++/** @} */ /* end group _mali_uk_core */ ++ ++ ++/** @defgroup _mali_uk_gp U/K Vertex Processor ++ * @{ */ ++ ++/** @defgroup _mali_uk_gp_suspend_response_s Vertex Processor Suspend Response ++ * @{ */ ++ ++/** @brief Arguments for _mali_ukk_gp_suspend_response() ++ * ++ * When _mali_wait_for_notification() receives notification that a ++ * Vertex Processor job was suspended, you need to send a response to indicate ++ * what needs to happen with this job. You can either abort or resume the job. ++ * ++ * - set @c code to indicate response code. This is either @c _MALIGP_JOB_ABORT or ++ * @c _MALIGP_JOB_RESUME_WITH_NEW_HEAP to indicate you will provide a new heap ++ * for the job that will resolve the out of memory condition for the job. ++ * - copy the @c cookie value from the @c _mali_uk_gp_job_suspended_s notification; ++ * this is an identifier for the suspended job ++ * - set @c arguments[0] and @c arguments[1] to zero if you abort the job. If ++ * you resume it, @c argument[0] should specify the Mali start address for the new ++ * heap and @c argument[1] the Mali end address of the heap. ++ * - pass in the user-kernel context @c ctx that was returned from _mali_ukk_open() ++ * ++ */ ++typedef enum _maligp_job_suspended_response_code { ++ _MALIGP_JOB_ABORT, /**< Abort the Vertex Processor job */ ++ _MALIGP_JOB_RESUME_WITH_NEW_HEAP /**< Resume the Vertex Processor job with a new heap */ ++} _maligp_job_suspended_response_code; ++ ++typedef struct { ++ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ u32 cookie; /**< [in] cookie from the _mali_uk_gp_job_suspended_s notification */ ++ _maligp_job_suspended_response_code code; /**< [in] abort or resume response code, see \ref _maligp_job_suspended_response_code */ ++ u32 arguments[2]; /**< [in] 0 when aborting a job. When resuming a job, the Mali start and end address for a new heap to resume the job with */ ++} _mali_uk_gp_suspend_response_s; ++ ++/** @} */ /* end group _mali_uk_gp_suspend_response_s */ ++ ++/** @defgroup _mali_uk_gpstartjob_s Vertex Processor Start Job ++ * @{ */ ++ ++/** @brief Status indicating the result of the execution of a Vertex or Fragment processor job */ ++typedef enum { ++ _MALI_UK_JOB_STATUS_END_SUCCESS = 1 << (16 + 0), ++ _MALI_UK_JOB_STATUS_END_OOM = 1 << (16 + 1), ++ _MALI_UK_JOB_STATUS_END_ABORT = 1 << (16 + 2), ++ _MALI_UK_JOB_STATUS_END_TIMEOUT_SW = 1 << (16 + 3), ++ _MALI_UK_JOB_STATUS_END_HANG = 1 << (16 + 4), ++ _MALI_UK_JOB_STATUS_END_SEG_FAULT = 1 << (16 + 5), ++ _MALI_UK_JOB_STATUS_END_ILLEGAL_JOB = 1 << (16 + 6), ++ _MALI_UK_JOB_STATUS_END_UNKNOWN_ERR = 1 << (16 + 7), ++ _MALI_UK_JOB_STATUS_END_SHUTDOWN = 1 << (16 + 8), ++ _MALI_UK_JOB_STATUS_END_SYSTEM_UNUSABLE = 1 << (16 + 9) ++} _mali_uk_job_status; ++ ++#define MALIGP2_NUM_REGS_FRAME (6) ++ ++/** @brief Arguments for _mali_ukk_gp_start_job() ++ * ++ * To start a Vertex Processor job ++ * - associate the request with a reference to a @c mali_gp_job_info by setting ++ * user_job_ptr to the address of the @c mali_gp_job_info of the job. ++ * - set @c priority to the priority of the @c mali_gp_job_info ++ * - specify a timeout for the job by setting @c watchdog_msecs to the number of ++ * milliseconds the job is allowed to run. Specifying a value of 0 selects the ++ * default timeout in use by the device driver. ++ * - copy the frame registers from the @c mali_gp_job_info into @c frame_registers. ++ * - set the @c perf_counter_flag, @c perf_counter_src0 and @c perf_counter_src1 to zero ++ * for a non-instrumented build. For an instrumented build you can use up ++ * to two performance counters. Set the corresponding bit in @c perf_counter_flag ++ * to enable them. @c perf_counter_src0 and @c perf_counter_src1 specify ++ * the source of what needs to get counted (e.g. number of vertex loader ++ * cache hits). For source id values, see ARM DDI0415A, Table 3-60. ++ * - pass in the user-kernel context @c ctx that was returned from _mali_ukk_open() ++ * ++ * When @c _mali_ukk_gp_start_job() returns @c _MALI_OSK_ERR_OK, status contains the ++ * result of the request (see \ref _mali_uk_start_job_status). If the job could ++ * not get started (@c _MALI_UK_START_JOB_NOT_STARTED_DO_REQUEUE) it should be ++ * tried again. ++ * ++ * After the job has started, @c _mali_wait_for_notification() will be notified ++ * that the job finished or got suspended. It may get suspended due to ++ * resource shortage. If it finished (see _mali_ukk_wait_for_notification()) ++ * the notification will contain a @c _mali_uk_gp_job_finished_s result. If ++ * it got suspended the notification will contain a @c _mali_uk_gp_job_suspended_s ++ * result. ++ * ++ * The @c _mali_uk_gp_job_finished_s contains the job status (see \ref _mali_uk_job_status), ++ * the number of milliseconds the job took to render, and values of core registers ++ * when the job finished (irq status, performance counters, renderer list ++ * address). A job has finished succesfully when its status is ++ * @c _MALI_UK_JOB_STATUS_FINISHED. If the hardware detected a timeout while rendering ++ * the job, or software detected the job is taking more than watchdog_msecs to ++ * complete, the status will indicate @c _MALI_UK_JOB_STATUS_HANG. ++ * If the hardware detected a bus error while accessing memory associated with the ++ * job, status will indicate @c _MALI_UK_JOB_STATUS_SEG_FAULT. ++ * status will indicate @c _MALI_UK_JOB_STATUS_NOT_STARTED if the driver had to ++ * stop the job but the job didn't start on the hardware yet, e.g. when the ++ * driver shutdown. ++ * ++ * In case the job got suspended, @c _mali_uk_gp_job_suspended_s contains ++ * the @c user_job_ptr identifier used to start the job with, the @c reason ++ * why the job stalled (see \ref _maligp_job_suspended_reason) and a @c cookie ++ * to identify the core on which the job stalled. This @c cookie will be needed ++ * when responding to this nofication by means of _mali_ukk_gp_suspend_response(). ++ * (see _mali_ukk_gp_suspend_response()). The response is either to abort or ++ * resume the job. If the job got suspended due to an out of memory condition ++ * you may be able to resolve this by providing more memory and resuming the job. ++ * ++ */ ++typedef struct { ++ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ u64 user_job_ptr; /**< [in] identifier for the job in user space, a @c mali_gp_job_info* */ ++ u32 priority; /**< [in] job priority. A lower number means higher priority */ ++ u32 frame_registers[MALIGP2_NUM_REGS_FRAME]; /**< [in] core specific registers associated with this job */ ++ u32 perf_counter_flag; /**< [in] bitmask indicating which performance counters to enable, see \ref _MALI_PERFORMANCE_COUNTER_FLAG_SRC0_ENABLE and related macro definitions */ ++ u32 perf_counter_src0; /**< [in] source id for performance counter 0 (see ARM DDI0415A, Table 3-60) */ ++ u32 perf_counter_src1; /**< [in] source id for performance counter 1 (see ARM DDI0415A, Table 3-60) */ ++ u32 frame_builder_id; /**< [in] id of the originating frame builder */ ++ u32 flush_id; /**< [in] flush id within the originating frame builder */ ++ _mali_uk_fence_t fence; /**< [in] fence this job must wait on */ ++ u64 timeline_point_ptr; /**< [in,out] pointer to u32: location where point on gp timeline for this job will be written */ ++ u32 varying_memsize; /** < [in] size of varying memory to use deffer bind*/ ++ u32 deferred_mem_num; ++ u64 deferred_mem_list; /** < [in] memory hanlde list of varying buffer to use deffer bind */ ++} _mali_uk_gp_start_job_s; ++ ++#define _MALI_PERFORMANCE_COUNTER_FLAG_SRC0_ENABLE (1<<0) /**< Enable performance counter SRC0 for a job */ ++#define _MALI_PERFORMANCE_COUNTER_FLAG_SRC1_ENABLE (1<<1) /**< Enable performance counter SRC1 for a job */ ++#define _MALI_PERFORMANCE_COUNTER_FLAG_HEATMAP_ENABLE (1<<2) /**< Enable per tile (aka heatmap) generation with for a job (using the enabled counter sources) */ ++ ++/** @} */ /* end group _mali_uk_gpstartjob_s */ ++ ++typedef struct { ++ u64 user_job_ptr; /**< [out] identifier for the job in user space */ ++ _mali_uk_job_status status; /**< [out] status of finished job */ ++ u32 heap_current_addr; /**< [out] value of the GP PLB PL heap start address register */ ++ u32 perf_counter0; /**< [out] value of performance counter 0 (see ARM DDI0415A) */ ++ u32 perf_counter1; /**< [out] value of performance counter 1 (see ARM DDI0415A) */ ++ u32 pending_big_job_num; ++} _mali_uk_gp_job_finished_s; ++ ++typedef struct { ++ u64 user_job_ptr; /**< [out] identifier for the job in user space */ ++ u32 cookie; /**< [out] identifier for the core in kernel space on which the job stalled */ ++} _mali_uk_gp_job_suspended_s; ++ ++/** @} */ /* end group _mali_uk_gp */ ++ ++ ++/** @defgroup _mali_uk_pp U/K Fragment Processor ++ * @{ */ ++ ++#define _MALI_PP_MAX_SUB_JOBS 8 ++ ++#define _MALI_PP_MAX_FRAME_REGISTERS ((0x058/4)+1) ++ ++#define _MALI_PP_MAX_WB_REGISTERS ((0x02C/4)+1) ++ ++#define _MALI_DLBU_MAX_REGISTERS 4 ++ ++/** Flag for _mali_uk_pp_start_job_s */ ++#define _MALI_PP_JOB_FLAG_NO_NOTIFICATION (1<<0) ++#define _MALI_PP_JOB_FLAG_IS_WINDOW_SURFACE (1<<1) ++#define _MALI_PP_JOB_FLAG_PROTECTED (1<<2) ++ ++/** @defgroup _mali_uk_ppstartjob_s Fragment Processor Start Job ++ * @{ */ ++ ++/** @brief Arguments for _mali_ukk_pp_start_job() ++ * ++ * To start a Fragment Processor job ++ * - associate the request with a reference to a mali_pp_job by setting ++ * @c user_job_ptr to the address of the @c mali_pp_job of the job. ++ * - set @c priority to the priority of the mali_pp_job ++ * - specify a timeout for the job by setting @c watchdog_msecs to the number of ++ * milliseconds the job is allowed to run. Specifying a value of 0 selects the ++ * default timeout in use by the device driver. ++ * - copy the frame registers from the @c mali_pp_job into @c frame_registers. ++ * For MALI200 you also need to copy the write back 0,1 and 2 registers. ++ * - set the @c perf_counter_flag, @c perf_counter_src0 and @c perf_counter_src1 to zero ++ * for a non-instrumented build. For an instrumented build you can use up ++ * to two performance counters. Set the corresponding bit in @c perf_counter_flag ++ * to enable them. @c perf_counter_src0 and @c perf_counter_src1 specify ++ * the source of what needs to get counted (e.g. number of vertex loader ++ * cache hits). For source id values, see ARM DDI0415A, Table 3-60. ++ * - pass in the user-kernel context in @c ctx that was returned from _mali_ukk_open() ++ * ++ * When _mali_ukk_pp_start_job() returns @c _MALI_OSK_ERR_OK, @c status contains the ++ * result of the request (see \ref _mali_uk_start_job_status). If the job could ++ * not get started (@c _MALI_UK_START_JOB_NOT_STARTED_DO_REQUEUE) it should be ++ * tried again. ++ * ++ * After the job has started, _mali_wait_for_notification() will be notified ++ * when the job finished. The notification will contain a ++ * @c _mali_uk_pp_job_finished_s result. It contains the @c user_job_ptr ++ * identifier used to start the job with, the job @c status (see \ref _mali_uk_job_status), ++ * the number of milliseconds the job took to render, and values of core registers ++ * when the job finished (irq status, performance counters, renderer list ++ * address). A job has finished succesfully when its status is ++ * @c _MALI_UK_JOB_STATUS_FINISHED. If the hardware detected a timeout while rendering ++ * the job, or software detected the job is taking more than @c watchdog_msecs to ++ * complete, the status will indicate @c _MALI_UK_JOB_STATUS_HANG. ++ * If the hardware detected a bus error while accessing memory associated with the ++ * job, status will indicate @c _MALI_UK_JOB_STATUS_SEG_FAULT. ++ * status will indicate @c _MALI_UK_JOB_STATUS_NOT_STARTED if the driver had to ++ * stop the job but the job didn't start on the hardware yet, e.g. when the ++ * driver shutdown. ++ * ++ */ ++typedef struct { ++ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ u64 user_job_ptr; /**< [in] identifier for the job in user space */ ++ u32 priority; /**< [in] job priority. A lower number means higher priority */ ++ u32 frame_registers[_MALI_PP_MAX_FRAME_REGISTERS]; /**< [in] core specific registers associated with first sub job, see ARM DDI0415A */ ++ u32 frame_registers_addr_frame[_MALI_PP_MAX_SUB_JOBS - 1]; /**< [in] ADDR_FRAME registers for sub job 1-7 */ ++ u32 frame_registers_addr_stack[_MALI_PP_MAX_SUB_JOBS - 1]; /**< [in] ADDR_STACK registers for sub job 1-7 */ ++ u32 wb0_registers[_MALI_PP_MAX_WB_REGISTERS]; ++ u32 wb1_registers[_MALI_PP_MAX_WB_REGISTERS]; ++ u32 wb2_registers[_MALI_PP_MAX_WB_REGISTERS]; ++ u32 dlbu_registers[_MALI_DLBU_MAX_REGISTERS]; /**< [in] Dynamic load balancing unit registers */ ++ u32 num_cores; /**< [in] Number of cores to set up (valid range: 1-8(M450) or 4(M400)) */ ++ u32 perf_counter_flag; /**< [in] bitmask indicating which performance counters to enable, see \ref _MALI_PERFORMANCE_COUNTER_FLAG_SRC0_ENABLE and related macro definitions */ ++ u32 perf_counter_src0; /**< [in] source id for performance counter 0 (see ARM DDI0415A, Table 3-60) */ ++ u32 perf_counter_src1; /**< [in] source id for performance counter 1 (see ARM DDI0415A, Table 3-60) */ ++ u32 frame_builder_id; /**< [in] id of the originating frame builder */ ++ u32 flush_id; /**< [in] flush id within the originating frame builder */ ++ u32 flags; /**< [in] See _MALI_PP_JOB_FLAG_* for a list of avaiable flags */ ++ u32 tilesx; /**< [in] number of tiles in the x direction (needed for heatmap generation */ ++ u32 tilesy; /**< [in] number of tiles in y direction (needed for reading the heatmap memory) */ ++ u32 heatmap_mem; /**< [in] memory address to store counter values per tile (aka heatmap) */ ++ u32 num_memory_cookies; /**< [in] number of memory cookies attached to job */ ++ u64 memory_cookies; /**< [in] pointer to array of u32 memory cookies attached to job */ ++ _mali_uk_fence_t fence; /**< [in] fence this job must wait on */ ++ u64 timeline_point_ptr; /**< [in,out] pointer to location of u32 where point on pp timeline for this job will be written */ ++} _mali_uk_pp_start_job_s; ++ ++typedef struct { ++ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ u64 gp_args; /**< [in,out] GP uk arguments (see _mali_uk_gp_start_job_s) */ ++ u64 pp_args; /**< [in,out] PP uk arguments (see _mali_uk_pp_start_job_s) */ ++} _mali_uk_pp_and_gp_start_job_s; ++ ++/** @} */ /* end group _mali_uk_ppstartjob_s */ ++ ++typedef struct { ++ u64 user_job_ptr; /**< [out] identifier for the job in user space */ ++ _mali_uk_job_status status; /**< [out] status of finished job */ ++ u32 perf_counter0[_MALI_PP_MAX_SUB_JOBS]; /**< [out] value of perfomance counter 0 (see ARM DDI0415A), one for each sub job */ ++ u32 perf_counter1[_MALI_PP_MAX_SUB_JOBS]; /**< [out] value of perfomance counter 1 (see ARM DDI0415A), one for each sub job */ ++ u32 perf_counter_src0; ++ u32 perf_counter_src1; ++} _mali_uk_pp_job_finished_s; ++ ++typedef struct { ++ u32 number_of_enabled_cores; /**< [out] the new number of enabled cores */ ++} _mali_uk_pp_num_cores_changed_s; ++ ++ ++ ++/** ++ * Flags to indicate write-back units ++ */ ++typedef enum { ++ _MALI_UK_PP_JOB_WB0 = 1, ++ _MALI_UK_PP_JOB_WB1 = 2, ++ _MALI_UK_PP_JOB_WB2 = 4, ++} _mali_uk_pp_job_wbx_flag; ++ ++typedef struct { ++ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ u32 fb_id; /**< [in] Frame builder ID of job to disable WB units for */ ++ u32 wb0_memory; ++ u32 wb1_memory; ++ u32 wb2_memory; ++} _mali_uk_pp_disable_wb_s; ++ ++ ++/** @} */ /* end group _mali_uk_pp */ ++ ++/** @defgroup _mali_uk_soft_job U/K Soft Job ++ * @{ */ ++ ++typedef struct { ++ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ u64 user_job; /**< [in] identifier for the job in user space */ ++ u64 job_id_ptr; /**< [in,out] pointer to location of u32 where job id will be written */ ++ _mali_uk_fence_t fence; /**< [in] fence this job must wait on */ ++ u32 point; /**< [out] point on soft timeline for this job */ ++ u32 type; /**< [in] type of soft job */ ++} _mali_uk_soft_job_start_s; ++ ++typedef struct { ++ u64 user_job; /**< [out] identifier for the job in user space */ ++} _mali_uk_soft_job_activated_s; ++ ++typedef struct { ++ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ u32 job_id; /**< [in] id for soft job */ ++} _mali_uk_soft_job_signal_s; ++ ++/** @} */ /* end group _mali_uk_soft_job */ ++ ++typedef struct { ++ u32 counter_id; ++ u32 key; ++ int enable; ++} _mali_uk_annotate_profiling_mem_counter_s; ++ ++typedef struct { ++ u32 sampling_rate; ++ int enable; ++} _mali_uk_annotate_profiling_enable_s; ++ ++ ++/** @addtogroup _mali_uk_core U/K Core ++ * @{ */ ++ ++/** @defgroup _mali_uk_waitfornotification_s Wait For Notification ++ * @{ */ ++ ++/** @brief Notification type encodings ++ * ++ * Each Notification type is an ordered pair of (subsystem,id), and is unique. ++ * ++ * The encoding of subsystem,id into a 32-bit word is: ++ * encoding = (( subsystem << _MALI_NOTIFICATION_SUBSYSTEM_SHIFT ) & _MALI_NOTIFICATION_SUBSYSTEM_MASK) ++ * | (( id << _MALI_NOTIFICATION_ID_SHIFT ) & _MALI_NOTIFICATION_ID_MASK) ++ * ++ * @see _mali_uk_wait_for_notification_s ++ */ ++typedef enum { ++ /** core notifications */ ++ ++ _MALI_NOTIFICATION_CORE_SHUTDOWN_IN_PROGRESS = (_MALI_UK_CORE_SUBSYSTEM << 16) | 0x20, ++ _MALI_NOTIFICATION_APPLICATION_QUIT = (_MALI_UK_CORE_SUBSYSTEM << 16) | 0x40, ++ _MALI_NOTIFICATION_SETTINGS_CHANGED = (_MALI_UK_CORE_SUBSYSTEM << 16) | 0x80, ++ _MALI_NOTIFICATION_SOFT_ACTIVATED = (_MALI_UK_CORE_SUBSYSTEM << 16) | 0x100, ++ ++ /** Fragment Processor notifications */ ++ ++ _MALI_NOTIFICATION_PP_FINISHED = (_MALI_UK_PP_SUBSYSTEM << 16) | 0x10, ++ _MALI_NOTIFICATION_PP_NUM_CORE_CHANGE = (_MALI_UK_PP_SUBSYSTEM << 16) | 0x20, ++ ++ /** Vertex Processor notifications */ ++ ++ _MALI_NOTIFICATION_GP_FINISHED = (_MALI_UK_GP_SUBSYSTEM << 16) | 0x10, ++ _MALI_NOTIFICATION_GP_STALLED = (_MALI_UK_GP_SUBSYSTEM << 16) | 0x20, ++ ++ /** Profiling notifications */ ++ _MALI_NOTIFICATION_ANNOTATE_PROFILING_MEM_COUNTER = (_MALI_UK_PROFILING_SUBSYSTEM << 16) | 0x10, ++ _MALI_NOTIFICATION_ANNOTATE_PROFILING_ENABLE = (_MALI_UK_PROFILING_SUBSYSTEM << 16) | 0x20, ++} _mali_uk_notification_type; ++ ++/** to assist in splitting up 32-bit notification value in subsystem and id value */ ++#define _MALI_NOTIFICATION_SUBSYSTEM_MASK 0xFFFF0000 ++#define _MALI_NOTIFICATION_SUBSYSTEM_SHIFT 16 ++#define _MALI_NOTIFICATION_ID_MASK 0x0000FFFF ++#define _MALI_NOTIFICATION_ID_SHIFT 0 ++ ++ ++/** @brief Enumeration of possible settings which match mali_setting_t in user space ++ * ++ * ++ */ ++typedef enum { ++ _MALI_UK_USER_SETTING_SW_EVENTS_ENABLE = 0, ++ _MALI_UK_USER_SETTING_COLORBUFFER_CAPTURE_ENABLED, ++ _MALI_UK_USER_SETTING_DEPTHBUFFER_CAPTURE_ENABLED, ++ _MALI_UK_USER_SETTING_STENCILBUFFER_CAPTURE_ENABLED, ++ _MALI_UK_USER_SETTING_PER_TILE_COUNTERS_CAPTURE_ENABLED, ++ _MALI_UK_USER_SETTING_BUFFER_CAPTURE_COMPOSITOR, ++ _MALI_UK_USER_SETTING_BUFFER_CAPTURE_WINDOW, ++ _MALI_UK_USER_SETTING_BUFFER_CAPTURE_OTHER, ++ _MALI_UK_USER_SETTING_BUFFER_CAPTURE_N_FRAMES, ++ _MALI_UK_USER_SETTING_BUFFER_CAPTURE_RESIZE_FACTOR, ++ _MALI_UK_USER_SETTING_SW_COUNTER_ENABLED, ++ _MALI_UK_USER_SETTING_MAX, ++} _mali_uk_user_setting_t; ++ ++/* See mali_user_settings_db.c */ ++extern const char *_mali_uk_user_setting_descriptions[]; ++#define _MALI_UK_USER_SETTING_DESCRIPTIONS \ ++ { \ ++ "sw_events_enable", \ ++ "colorbuffer_capture_enable", \ ++ "depthbuffer_capture_enable", \ ++ "stencilbuffer_capture_enable", \ ++ "per_tile_counters_enable", \ ++ "buffer_capture_compositor", \ ++ "buffer_capture_window", \ ++ "buffer_capture_other", \ ++ "buffer_capture_n_frames", \ ++ "buffer_capture_resize_factor", \ ++ "sw_counters_enable", \ ++ }; ++ ++/** @brief struct to hold the value to a particular setting as seen in the kernel space ++ */ ++typedef struct { ++ _mali_uk_user_setting_t setting; ++ u32 value; ++} _mali_uk_settings_changed_s; ++ ++/** @brief Arguments for _mali_ukk_wait_for_notification() ++ * ++ * On successful return from _mali_ukk_wait_for_notification(), the members of ++ * this structure will indicate the reason for notification. ++ * ++ * Specifically, the source of the notification can be identified by the ++ * subsystem and id fields of the mali_uk_notification_type in the code.type ++ * member. The type member is encoded in a way to divide up the types into a ++ * subsystem field, and a per-subsystem ID field. See ++ * _mali_uk_notification_type for more information. ++ * ++ * Interpreting the data union member depends on the notification type: ++ * ++ * - type == _MALI_NOTIFICATION_CORE_SHUTDOWN_IN_PROGRESS ++ * - The kernel side is shutting down. No further ++ * _mali_uk_wait_for_notification() calls should be made. ++ * - In this case, the value of the data union member is undefined. ++ * - This is used to indicate to the user space client that it should close ++ * the connection to the Mali Device Driver. ++ * - type == _MALI_NOTIFICATION_PP_FINISHED ++ * - The notification data is of type _mali_uk_pp_job_finished_s. It contains the user_job_ptr ++ * identifier used to start the job with, the job status, the number of milliseconds the job took to render, ++ * and values of core registers when the job finished (irq status, performance counters, renderer list ++ * address). ++ * - A job has finished succesfully when its status member is _MALI_UK_JOB_STATUS_FINISHED. ++ * - If the hardware detected a timeout while rendering the job, or software detected the job is ++ * taking more than watchdog_msecs (see _mali_ukk_pp_start_job()) to complete, the status member will ++ * indicate _MALI_UK_JOB_STATUS_HANG. ++ * - If the hardware detected a bus error while accessing memory associated with the job, status will ++ * indicate _MALI_UK_JOB_STATUS_SEG_FAULT. ++ * - Status will indicate MALI_UK_JOB_STATUS_NOT_STARTED if the driver had to stop the job but the job ++ * didn't start the hardware yet, e.g. when the driver closes. ++ * - type == _MALI_NOTIFICATION_GP_FINISHED ++ * - The notification data is of type _mali_uk_gp_job_finished_s. The notification is similar to that of ++ * type == _MALI_NOTIFICATION_PP_FINISHED, except that several other GP core register values are returned. ++ * The status values have the same meaning for type == _MALI_NOTIFICATION_PP_FINISHED. ++ * - type == _MALI_NOTIFICATION_GP_STALLED ++ * - The nofication data is of type _mali_uk_gp_job_suspended_s. It contains the user_job_ptr ++ * identifier used to start the job with, the reason why the job stalled and a cookie to identify the core on ++ * which the job stalled. ++ * - The reason member of gp_job_suspended is set to _MALIGP_JOB_SUSPENDED_OUT_OF_MEMORY ++ * when the polygon list builder unit has run out of memory. ++ */ ++typedef struct { ++ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ _mali_uk_notification_type type; /**< [out] Type of notification available */ ++ union { ++ _mali_uk_gp_job_suspended_s gp_job_suspended;/**< [out] Notification data for _MALI_NOTIFICATION_GP_STALLED notification type */ ++ _mali_uk_gp_job_finished_s gp_job_finished; /**< [out] Notification data for _MALI_NOTIFICATION_GP_FINISHED notification type */ ++ _mali_uk_pp_job_finished_s pp_job_finished; /**< [out] Notification data for _MALI_NOTIFICATION_PP_FINISHED notification type */ ++ _mali_uk_settings_changed_s setting_changed;/**< [out] Notification data for _MALI_NOTIFICAATION_SETTINGS_CHANGED notification type */ ++ _mali_uk_soft_job_activated_s soft_job_activated; /**< [out] Notification data for _MALI_NOTIFICATION_SOFT_ACTIVATED notification type */ ++ _mali_uk_annotate_profiling_mem_counter_s profiling_mem_counter; ++ _mali_uk_annotate_profiling_enable_s profiling_enable; ++ } data; ++} _mali_uk_wait_for_notification_s; ++ ++/** @brief Arguments for _mali_ukk_post_notification() ++ * ++ * Posts the specified notification to the notification queue for this application. ++ * This is used to send a quit message to the callback thread. ++ */ ++typedef struct { ++ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ _mali_uk_notification_type type; /**< [in] Type of notification to post */ ++} _mali_uk_post_notification_s; ++ ++/** @} */ /* end group _mali_uk_waitfornotification_s */ ++ ++/** @defgroup _mali_uk_getapiversion_s Get API Version ++ * @{ */ ++ ++/** helpers for Device Driver API version handling */ ++ ++/** @brief Encode a version ID from a 16-bit input ++ * ++ * @note the input is assumed to be 16 bits. It must not exceed 16 bits. */ ++#define _MAKE_VERSION_ID(x) (((x) << 16UL) | (x)) ++ ++/** @brief Check whether a 32-bit value is likely to be Device Driver API ++ * version ID. */ ++#define _IS_VERSION_ID(x) (((x) & 0xFFFF) == (((x) >> 16UL) & 0xFFFF)) ++ ++/** @brief Decode a 16-bit version number from a 32-bit Device Driver API version ++ * ID */ ++#define _GET_VERSION(x) (((x) >> 16UL) & 0xFFFF) ++ ++/** @brief Determine whether two 32-bit encoded version IDs match */ ++#define _IS_API_MATCH(x, y) (IS_VERSION_ID((x)) && IS_VERSION_ID((y)) && (GET_VERSION((x)) == GET_VERSION((y)))) ++ /** ++ * RK MALI version code ++ */ ++#define _MALI_RK_LIBS_VERSION 1 ++ ++/** ++ * API version define. ++ * Indicates the version of the kernel API ++ * The version is a 16bit integer incremented on each API change. ++ * The 16bit integer is stored twice in a 32bit integer ++ * For example, for version 1 the value would be 0x00010001 ++ */ ++#define _MALI_API_VERSION 900 ++#define _MALI_UK_API_VERSION _MAKE_VERSION_ID(_MALI_API_VERSION) ++ ++/** ++ * The API version is a 16-bit integer stored in both the lower and upper 16-bits ++ * of a 32-bit value. The 16-bit API version value is incremented on each API ++ * change. Version 1 would be 0x00010001. Used in _mali_uk_get_api_version_s. ++ */ ++typedef u32 _mali_uk_api_version; ++ ++/** @brief Arguments for _mali_uk_get_api_version() ++ * ++ * The user-side interface version must be written into the version member, ++ * encoded using _MAKE_VERSION_ID(). It will be compared to the API version of ++ * the kernel-side interface. ++ * ++ * On successful return, the version member will be the API version of the ++ * kernel-side interface. _MALI_UK_API_VERSION macro defines the current version ++ * of the API. ++ * ++ * The compatible member must be checked to see if the version of the user-side ++ * interface is compatible with the kernel-side interface, since future versions ++ * of the interface may be backwards compatible. ++ */ ++typedef struct { ++ u32 ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ _mali_uk_api_version version; /**< [in,out] API version of user-side interface. */ ++ int compatible; /**< [out] @c 1 when @version is compatible, @c 0 otherwise */ ++} _mali_uk_get_api_version_s; ++ ++/** @brief Arguments for _mali_uk_get_api_version_v2() ++ * ++ * The user-side interface version must be written into the version member, ++ * encoded using _MAKE_VERSION_ID(). It will be compared to the API version of ++ * the kernel-side interface. ++ * ++ * On successful return, the version member will be the API version of the ++ * kernel-side interface. _MALI_UK_API_VERSION macro defines the current version ++ * of the API. ++ * ++ * The compatible member must be checked to see if the version of the user-side ++ * interface is compatible with the kernel-side interface, since future versions ++ * of the interface may be backwards compatible. ++ */ ++typedef struct { ++ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ _mali_uk_api_version version; /**< [in,out] API version of user-side interface. */ ++ int compatible; /**< [out] @c 1 when @version is compatible, @c 0 otherwise */ ++} _mali_uk_get_api_version_v2_s; ++ ++typedef struct ++{ ++ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ _mali_uk_api_version version; /**< [in,out] API version of user-side interface. */ ++} _mali_uk_get_mali_version_in_rk30_s; ++ ++/* rk_ext : rk_ko_ver_t. */ ++typedef struct { ++ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ _mali_uk_api_version version; /**< [in,out] API version of user-side interface. */ ++} _mali_rk_ko_version_s; ++/** @} */ /* end group _mali_uk_getapiversion_s */ ++ ++/** @defgroup _mali_uk_get_user_settings_s Get user space settings */ ++ ++/** @brief struct to keep the matching values of the user space settings within certain context ++ * ++ * Each member of the settings array corresponds to a matching setting in the user space and its value is the value ++ * of that particular setting. ++ * ++ * All settings are given reference to the context pointed to by the ctx pointer. ++ * ++ */ ++typedef struct { ++ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ u32 settings[_MALI_UK_USER_SETTING_MAX]; /**< [out] The values for all settings */ ++} _mali_uk_get_user_settings_s; ++ ++/** @brief struct to hold the value of a particular setting from the user space within a given context ++ */ ++typedef struct { ++ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ _mali_uk_user_setting_t setting; /**< [in] setting to get */ ++ u32 value; /**< [out] value of setting */ ++} _mali_uk_get_user_setting_s; ++ ++/** @brief Arguments for _mali_ukk_request_high_priority() */ ++typedef struct { ++ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ ++} _mali_uk_request_high_priority_s; ++ ++/** @brief Arguments for _mali_ukk_pending_submit() */ ++typedef struct { ++ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ ++} _mali_uk_pending_submit_s; ++ ++/** @} */ /* end group _mali_uk_core */ ++ ++ ++/** @defgroup _mali_uk_memory U/K Memory ++ * @{ */ ++ ++#define _MALI_MEMORY_ALLOCATE_RESIZEABLE (1<<4) /* BUFFER can trim dow/grow*/ ++#define _MALI_MEMORY_ALLOCATE_NO_BIND_GPU (1<<5) /*Not map to GPU when allocate, must call bind later*/ ++#define _MALI_MEMORY_ALLOCATE_SWAPPABLE (1<<6) /* Allocate swappale memory. */ ++#define _MALI_MEMORY_ALLOCATE_DEFER_BIND (1<<7) /*Not map to GPU when allocate, must call bind later*/ ++#define _MALI_MEMORY_ALLOCATE_SECURE (1<<8) /* Allocate secure memory. */ ++ ++ ++typedef struct { ++ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ u32 gpu_vaddr; /**< [in] GPU virtual address */ ++ u32 vsize; /**< [in] vitrual size of the allocation */ ++ u32 psize; /**< [in] physical size of the allocation */ ++ u32 flags; ++ u64 backend_handle; /**< [out] backend handle */ ++ s32 secure_shared_fd; /** < [in] the mem handle for secure mem */ ++} _mali_uk_alloc_mem_s; ++ ++ ++typedef struct { ++ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ u32 gpu_vaddr; /**< [in] use as handle to free allocation */ ++ u32 free_pages_nr; /** < [out] record the number of free pages */ ++} _mali_uk_free_mem_s; ++ ++ ++#define _MALI_MEMORY_BIND_BACKEND_UMP (1<<8) ++#define _MALI_MEMORY_BIND_BACKEND_DMA_BUF (1<<9) ++#define _MALI_MEMORY_BIND_BACKEND_MALI_MEMORY (1<<10) ++#define _MALI_MEMORY_BIND_BACKEND_EXTERNAL_MEMORY (1<<11) ++#define _MALI_MEMORY_BIND_BACKEND_EXT_COW (1<<12) ++#define _MALI_MEMORY_BIND_BACKEND_HAVE_ALLOCATION (1<<13) ++ ++ ++#define _MALI_MEMORY_BIND_BACKEND_MASK (_MALI_MEMORY_BIND_BACKEND_UMP| \ ++ _MALI_MEMORY_BIND_BACKEND_DMA_BUF |\ ++ _MALI_MEMORY_BIND_BACKEND_MALI_MEMORY |\ ++ _MALI_MEMORY_BIND_BACKEND_EXTERNAL_MEMORY |\ ++ _MALI_MEMORY_BIND_BACKEND_EXT_COW |\ ++ _MALI_MEMORY_BIND_BACKEND_HAVE_ALLOCATION) ++ ++ ++#define _MALI_MEMORY_GPU_READ_ALLOCATE (1<<16) ++ ++ ++typedef struct { ++ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ u32 vaddr; /**< [in] mali address to map the physical memory to */ ++ u32 size; /**< [in] size */ ++ u32 flags; /**< [in] see_MALI_MEMORY_BIND_BACKEND_* */ ++ u32 padding; /** padding for 32/64 struct alignment */ ++ union { ++ struct { ++ u32 secure_id; /**< [in] secure id */ ++ u32 rights; /**< [in] rights necessary for accessing memory */ ++ u32 flags; /**< [in] flags, see \ref _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE */ ++ } bind_ump; ++ struct { ++ u32 mem_fd; /**< [in] Memory descriptor */ ++ u32 rights; /**< [in] rights necessary for accessing memory */ ++ u32 flags; /**< [in] flags, see \ref _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE */ ++ } bind_dma_buf; ++ struct { ++ u32 phys_addr; /**< [in] physical address */ ++ u32 rights; /**< [in] rights necessary for accessing memory */ ++ u32 flags; /**< [in] flags, see \ref _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE */ ++ } bind_ext_memory; ++ } mem_union; ++} _mali_uk_bind_mem_s; ++ ++typedef struct { ++ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ u32 flags; /**< [in] see_MALI_MEMORY_BIND_BACKEND_* */ ++ u32 vaddr; /**< [in] identifier for mapped memory object in kernel space */ ++} _mali_uk_unbind_mem_s; ++ ++typedef struct { ++ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ u32 target_handle; /**< [in] handle of allocation need to do COW */ ++ u32 target_offset; /**< [in] offset in target allocation to do COW(for support COW a memory allocated from memory_bank, PAGE_SIZE align)*/ ++ u32 target_size; /**< [in] size of target allocation to do COW (for support memory bank, PAGE_SIZE align)(in byte) */ ++ u32 range_start; /**< [in] re allocate range start offset, offset from the start of allocation (PAGE_SIZE align)*/ ++ u32 range_size; /**< [in] re allocate size (PAGE_SIZE align)*/ ++ u32 vaddr; /**< [in] mali address for the new allocaiton */ ++ u32 backend_handle; /**< [out] backend handle */ ++ u32 flags; ++} _mali_uk_cow_mem_s; ++ ++typedef struct { ++ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ u32 range_start; /**< [in] re allocate range start offset, offset from the start of allocation */ ++ u32 size; /**< [in] re allocate size*/ ++ u32 vaddr; /**< [in] mali address for the new allocaiton */ ++ s32 change_pages_nr; /**< [out] record the page number change for cow operation */ ++} _mali_uk_cow_modify_range_s; ++ ++ ++typedef struct { ++ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ u32 mem_fd; /**< [in] Memory descriptor */ ++ u32 size; /**< [out] size */ ++} _mali_uk_dma_buf_get_size_s; ++ ++/** Flag for _mali_uk_map_external_mem_s, _mali_uk_attach_ump_mem_s and _mali_uk_attach_dma_buf_s */ ++#define _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE (1<<0) ++ ++ ++typedef struct { ++ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ u64 vaddr; /* the buffer to do resize*/ ++ u32 psize; /* wanted physical size of this memory */ ++} _mali_uk_mem_resize_s; ++ ++/** ++ * @brief Arguments for _mali_uk[uk]_mem_write_safe() ++ */ ++typedef struct { ++ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ u64 src; /**< [in] Pointer to source data */ ++ u64 dest; /**< [in] Destination Mali buffer */ ++ u32 size; /**< [in,out] Number of bytes to write/copy on input, number of bytes actually written/copied on output */ ++} _mali_uk_mem_write_safe_s; ++ ++typedef struct { ++ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ u32 size; /**< [out] size of MMU page table information (registers + page tables) */ ++} _mali_uk_query_mmu_page_table_dump_size_s; ++ ++typedef struct { ++ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ u32 size; /**< [in] size of buffer to receive mmu page table information */ ++ u64 buffer; /**< [in,out] buffer to receive mmu page table information */ ++ u32 register_writes_size; /**< [out] size of MMU register dump */ ++ u64 register_writes; /**< [out] pointer within buffer where MMU register dump is stored */ ++ u32 page_table_dump_size; /**< [out] size of MMU page table dump */ ++ u64 page_table_dump; /**< [out] pointer within buffer where MMU page table dump is stored */ ++} _mali_uk_dump_mmu_page_table_s; ++ ++/** @} */ /* end group _mali_uk_memory */ ++ ++ ++/** @addtogroup _mali_uk_pp U/K Fragment Processor ++ * @{ */ ++ ++/** @brief Arguments for _mali_ukk_get_pp_number_of_cores() ++ * ++ * - pass in the user-kernel context @c ctx that was returned from _mali_ukk_open() ++ * - Upon successful return from _mali_ukk_get_pp_number_of_cores(), @c number_of_cores ++ * will contain the number of Fragment Processor cores in the system. ++ */ ++typedef struct { ++ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ u32 number_of_total_cores; /**< [out] Total number of Fragment Processor cores in the system */ ++ u32 number_of_enabled_cores; /**< [out] Number of enabled Fragment Processor cores */ ++} _mali_uk_get_pp_number_of_cores_s; ++ ++/** @brief Arguments for _mali_ukk_get_pp_core_version() ++ * ++ * - pass in the user-kernel context @c ctx that was returned from _mali_ukk_open() ++ * - Upon successful return from _mali_ukk_get_pp_core_version(), @c version contains ++ * the version that all Fragment Processor cores are compatible with. ++ */ ++typedef struct { ++ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ _mali_core_version version; /**< [out] version returned from core, see \ref _mali_core_version */ ++ u32 padding; ++} _mali_uk_get_pp_core_version_s; ++ ++/** @} */ /* end group _mali_uk_pp */ ++ ++ ++/** @addtogroup _mali_uk_gp U/K Vertex Processor ++ * @{ */ ++ ++/** @brief Arguments for _mali_ukk_get_gp_number_of_cores() ++ * ++ * - pass in the user-kernel context @c ctx that was returned from _mali_ukk_open() ++ * - Upon successful return from _mali_ukk_get_gp_number_of_cores(), @c number_of_cores ++ * will contain the number of Vertex Processor cores in the system. ++ */ ++typedef struct { ++ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ u32 number_of_cores; /**< [out] number of Vertex Processor cores in the system */ ++} _mali_uk_get_gp_number_of_cores_s; ++ ++/** @brief Arguments for _mali_ukk_get_gp_core_version() ++ * ++ * - pass in the user-kernel context @c ctx that was returned from _mali_ukk_open() ++ * - Upon successful return from _mali_ukk_get_gp_core_version(), @c version contains ++ * the version that all Vertex Processor cores are compatible with. ++ */ ++typedef struct { ++ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ _mali_core_version version; /**< [out] version returned from core, see \ref _mali_core_version */ ++} _mali_uk_get_gp_core_version_s; ++ ++/** @} */ /* end group _mali_uk_gp */ ++ ++typedef struct { ++ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ u32 event_id; /**< [in] event id to register (see enum mali_profiling_events for values) */ ++ u32 data[5]; /**< [in] event specific data */ ++} _mali_uk_profiling_add_event_s; ++ ++typedef struct { ++ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ u32 memory_usage; /**< [out] total memory usage */ ++ u32 vaddr; /**< [in] mali address for the cow allocaiton */ ++ s32 change_pages_nr; /**< [out] record the page number change for cow operation */ ++} _mali_uk_profiling_memory_usage_get_s; ++ ++ ++/** @addtogroup _mali_uk_memory U/K Memory ++ * @{ */ ++ ++/** @brief Arguments to _mali_ukk_mem_mmap() ++ * ++ * Use of the phys_addr member depends on whether the driver is compiled for ++ * Mali-MMU or nonMMU: ++ * - in the nonMMU case, this is the physical address of the memory as seen by ++ * the CPU (which may be a constant offset from that used by Mali) ++ * - in the MMU case, this is the Mali Virtual base address of the memory to ++ * allocate, and the particular physical pages used to back the memory are ++ * entirely determined by _mali_ukk_mem_mmap(). The details of the physical pages ++ * are not reported to user-space for security reasons. ++ * ++ * The cookie member must be stored for use later when freeing the memory by ++ * calling _mali_ukk_mem_munmap(). In the Mali-MMU case, the cookie is secure. ++ * ++ * The ukk_private word must be set to zero when calling from user-space. On ++ * Kernel-side, the OS implementation of the U/K interface can use it to ++ * communicate data to the OS implementation of the OSK layer. In particular, ++ * _mali_ukk_get_big_block() directly calls _mali_ukk_mem_mmap directly, and ++ * will communicate its own ukk_private word through the ukk_private member ++ * here. The common code itself will not inspect or modify the ukk_private ++ * word, and so it may be safely used for whatever purposes necessary to ++ * integrate Mali Memory handling into the OS. ++ * ++ * The uku_private member is currently reserved for use by the user-side ++ * implementation of the U/K interface. Its value must be zero. ++ */ ++typedef struct { ++ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ void *mapping; /**< [out] Returns user-space virtual address for the mapping */ ++ u32 size; /**< [in] Size of the requested mapping */ ++ u32 phys_addr; /**< [in] Physical address - could be offset, depending on caller+callee convention */ ++ mali_bool writeable; ++} _mali_uk_mem_mmap_s; ++ ++/** @brief Arguments to _mali_ukk_mem_munmap() ++ * ++ * The cookie and mapping members must be that returned from the same previous ++ * call to _mali_ukk_mem_mmap(). The size member must correspond to cookie ++ * and mapping - that is, it must be the value originally supplied to a call to ++ * _mali_ukk_mem_mmap that returned the values of mapping and cookie. ++ * ++ * An error will be returned if an attempt is made to unmap only part of the ++ * originally obtained range, or to unmap more than was originally obtained. ++ */ ++typedef struct { ++ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ void *mapping; /**< [in] The mapping returned from mmap call */ ++ u32 size; /**< [in] The size passed to mmap call */ ++} _mali_uk_mem_munmap_s; ++/** @} */ /* end group _mali_uk_memory */ ++ ++/** @defgroup _mali_uk_vsync U/K VSYNC Wait Reporting Module ++ * @{ */ ++ ++/** @brief VSYNC events ++ * ++ * These events are reported when DDK starts to wait for vsync and when the ++ * vsync has occured and the DDK can continue on the next frame. ++ */ ++typedef enum _mali_uk_vsync_event { ++ _MALI_UK_VSYNC_EVENT_BEGIN_WAIT = 0, ++ _MALI_UK_VSYNC_EVENT_END_WAIT ++} _mali_uk_vsync_event; ++ ++/** @brief Arguments to _mali_ukk_vsync_event() ++ * ++ */ ++typedef struct { ++ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ _mali_uk_vsync_event event; /**< [in] VSYNCH event type */ ++} _mali_uk_vsync_event_report_s; ++ ++/** @} */ /* end group _mali_uk_vsync */ ++ ++/** @defgroup _mali_uk_sw_counters_report U/K Software Counter Reporting ++ * @{ */ ++ ++/** @brief Software counter values ++ * ++ * Values recorded for each of the software counters during a single renderpass. ++ */ ++typedef struct { ++ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ u64 counters; /**< [in] The array of u32 counter values */ ++ u32 num_counters; /**< [in] The number of elements in counters array */ ++} _mali_uk_sw_counters_report_s; ++ ++/** @} */ /* end group _mali_uk_sw_counters_report */ ++ ++/** @defgroup _mali_uk_timeline U/K Mali Timeline ++ * @{ */ ++ ++typedef struct { ++ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ u32 timeline; /**< [in] timeline id */ ++ u32 point; /**< [out] latest point on timeline */ ++} _mali_uk_timeline_get_latest_point_s; ++ ++typedef struct { ++ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ _mali_uk_fence_t fence; /**< [in] fence */ ++ u32 timeout; /**< [in] timeout (0 for no wait, -1 for blocking) */ ++ u32 status; /**< [out] status of fence (1 if signaled, 0 if timeout) */ ++} _mali_uk_timeline_wait_s; ++ ++typedef struct { ++ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ _mali_uk_fence_t fence; /**< [in] mali fence to create linux sync fence from */ ++ s32 sync_fd; /**< [out] file descriptor for new linux sync fence */ ++} _mali_uk_timeline_create_sync_fence_s; ++ ++/** @} */ /* end group _mali_uk_timeline */ ++ ++/** @} */ /* end group u_k_api */ ++ ++/** @} */ /* end group uddapi */ ++ ++typedef struct { ++ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ s32 stream_fd; /**< [in] The profiling kernel base stream fd handle */ ++} _mali_uk_profiling_stream_fd_get_s; ++ ++typedef struct { ++ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ u64 control_packet_data; /**< [in] the control packet data for control settings */ ++ u32 control_packet_size; /**< [in] The control packet size */ ++ u64 response_packet_data; /** < [out] The response packet data */ ++ u32 response_packet_size; /** < [in,out] The response packet data */ ++} _mali_uk_profiling_control_set_s; ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif /* __MALI_UTGARD_UK_TYPES_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/linux/license/gpl/mali_kernel_license.h b/drivers/gpu/arm/mali400/mali/linux/license/gpl/mali_kernel_license.h +new file mode 100755 +index 000000000000..6fafc6777e48 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/license/gpl/mali_kernel_license.h +@@ -0,0 +1,30 @@ ++/* ++ * Copyright (C) 2010, 2013, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file mali_kernel_license.h ++ * Defines for the macro MODULE_LICENSE. ++ */ ++ ++#ifndef __MALI_KERNEL_LICENSE_H__ ++#define __MALI_KERNEL_LICENSE_H__ ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++#define MALI_KERNEL_LINUX_LICENSE "GPL" ++#define MALI_LICENSE_IS_GPL 1 ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif /* __MALI_KERNEL_LICENSE_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_devfreq.c b/drivers/gpu/arm/mali400/mali/linux/mali_devfreq.c +new file mode 100755 +index 000000000000..260c2a8227a9 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_devfreq.c +@@ -0,0 +1,354 @@ ++/* ++ * Copyright (C) 2011-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include "mali_osk_mali.h" ++#include "mali_kernel_common.h" ++ ++#include ++#include ++#include ++#include ++#ifdef CONFIG_DEVFREQ_THERMAL ++#include ++#endif ++ ++#include ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0) ++#include ++#else /* Linux >= 3.13 */ ++/* In 3.13 the OPP include header file, types, and functions were all ++ * renamed. Use the old filename for the include, and define the new names to ++ * the old, when an old kernel is detected. ++ */ ++#include ++#define dev_pm_opp opp ++#define dev_pm_opp_get_voltage opp_get_voltage ++#define dev_pm_opp_get_opp_count opp_get_opp_count ++#define dev_pm_opp_find_freq_ceil opp_find_freq_ceil ++#endif /* Linux >= 3.13 */ ++ ++#include "mali_pm_metrics.h" ++ ++#include ++#include ++ ++static struct monitor_dev_profile mali_mdevp = { ++ .type = MONITOR_TPYE_DEV, ++ .low_temp_adjust = rockchip_monitor_dev_low_temp_adjust, ++ .high_temp_adjust = rockchip_monitor_dev_high_temp_adjust, ++}; ++ ++static struct devfreq_simple_ondemand_data ondemand_data; ++ ++static int ++mali_devfreq_target(struct device *dev, unsigned long *target_freq, u32 flags) ++{ ++ struct mali_device *mdev = dev_get_drvdata(dev); ++ struct dev_pm_opp *opp; ++ unsigned long freq = 0; ++ unsigned long old_freq = mdev->current_freq; ++ unsigned long voltage; ++ int err; ++ ++ freq = *target_freq; ++ ++ opp = devfreq_recommended_opp(dev, &freq, flags); ++ if (IS_ERR(opp)) { ++ MALI_PRINT_ERROR(("Failed to get opp (%ld)\n", PTR_ERR(opp))); ++ return PTR_ERR(opp); ++ } ++ voltage = dev_pm_opp_get_voltage(opp); ++ dev_pm_opp_put(opp); ++ ++ MALI_DEBUG_PRINT(2, ("mali_devfreq_target:set_freq = %lld flags = 0x%x\n", freq, flags)); ++ /* ++ * Only update if there is a change of frequency ++ */ ++ if (old_freq == freq) { ++ *target_freq = freq; ++ mali_pm_reset_dvfs_utilisation(mdev); ++#ifdef CONFIG_REGULATOR ++ if (mdev->current_voltage == voltage) ++ return 0; ++ err = regulator_set_voltage(mdev->regulator, voltage, INT_MAX); ++ if (err) { ++ dev_err(dev, "Failed to set voltage (%d)\n", err); ++ return err; ++ } ++ mdev->current_voltage = voltage; ++#endif ++ return 0; ++ } ++ ++#ifdef CONFIG_REGULATOR ++ if (mdev->regulator && mdev->current_voltage != voltage && ++ old_freq < freq) { ++ err = regulator_set_voltage(mdev->regulator, voltage, INT_MAX); ++ if (err) { ++ MALI_PRINT_ERROR(("Failed to increase voltage (%d)\n", err)); ++ return err; ++ } ++ } ++#endif ++ ++ err = clk_set_rate(mdev->clock, freq); ++ if (err) { ++ MALI_PRINT_ERROR(("Failed to set clock %lu (target %lu)\n", freq, *target_freq)); ++ return err; ++ } ++ *target_freq = freq; ++ mdev->current_freq = freq; ++ if (mdev->devfreq) ++ mdev->devfreq->last_status.current_frequency = freq; ++ ++#ifdef CONFIG_REGULATOR ++ if (mdev->regulator && mdev->current_voltage != voltage && ++ old_freq > freq) { ++ err = regulator_set_voltage(mdev->regulator, voltage, INT_MAX); ++ if (err) { ++ MALI_PRINT_ERROR(("Failed to decrease voltage (%d)\n", err)); ++ return err; ++ } ++ } ++#endif ++ ++ mdev->current_voltage = voltage; ++ ++ mali_pm_reset_dvfs_utilisation(mdev); ++ ++ return err; ++} ++ ++static int ++mali_devfreq_cur_freq(struct device *dev, unsigned long *freq) ++{ ++ struct mali_device *mdev = dev_get_drvdata(dev); ++ ++ *freq = mdev->current_freq; ++ ++ MALI_DEBUG_PRINT(2, ("mali_devfreq_cur_freq: freq = %d \n", *freq)); ++ return 0; ++} ++ ++static int ++mali_devfreq_status(struct device *dev, struct devfreq_dev_status *stat) ++{ ++ struct mali_device *mdev = dev_get_drvdata(dev); ++ ++ stat->current_frequency = mdev->current_freq; ++ ++ mali_pm_get_dvfs_utilisation(mdev, ++ &stat->total_time, &stat->busy_time); ++ ++ stat->private_data = NULL; ++ ++#ifdef CONFIG_DEVFREQ_THERMAL ++ memcpy(&mdev->devfreq->last_status, stat, sizeof(*stat)); ++#endif ++ ++ return 0; ++} ++ ++/* setup platform specific opp in platform.c*/ ++int __weak setup_opps(void) ++{ ++ return 0; ++} ++ ++/* term platform specific opp in platform.c*/ ++int __weak term_opps(struct device *dev) ++{ ++ return 0; ++} ++ ++static int mali_devfreq_init_freq_table(struct mali_device *mdev, ++ struct devfreq_dev_profile *dp) ++{ ++ int err, count; ++ int i = 0; ++ unsigned long freq = 0; ++ struct dev_pm_opp *opp; ++ ++ err = setup_opps(); ++ if (err) ++ return err; ++ ++ count = dev_pm_opp_get_opp_count(mdev->dev); ++ if (count < 0) { ++ return count; ++ } ++ ++ MALI_DEBUG_PRINT(2, ("mali devfreq table count %d\n", count)); ++ ++ dp->freq_table = kmalloc_array(count, sizeof(dp->freq_table[0]), ++ GFP_KERNEL); ++ if (!dp->freq_table) ++ return -ENOMEM; ++ ++ for (i = 0; i < count; i++, freq++) { ++ opp = dev_pm_opp_find_freq_ceil(mdev->dev, &freq); ++ if (IS_ERR(opp)) ++ break; ++ dev_pm_opp_put(opp); ++ ++ dp->freq_table[i] = freq; ++ MALI_DEBUG_PRINT(2, ("mali devfreq table array[%d] = %d\n", i, freq)); ++ } ++ ++ if (count != i) ++ MALI_PRINT_ERROR(("Unable to enumerate all OPPs (%d!=%d)\n", ++ count, i)); ++ ++ dp->max_state = i; ++ ++ return 0; ++} ++ ++static void mali_devfreq_term_freq_table(struct mali_device *mdev) ++{ ++ struct devfreq_dev_profile *dp = mdev->devfreq->profile; ++ ++ kfree(dp->freq_table); ++ term_opps(mdev->dev); ++} ++ ++static void mali_devfreq_exit(struct device *dev) ++{ ++ struct mali_device *mdev = dev_get_drvdata(dev); ++ ++ mali_devfreq_term_freq_table(mdev); ++} ++ ++int mali_devfreq_init(struct mali_device *mdev) ++{ ++ struct device_node *np = mdev->dev->of_node; ++#ifdef CONFIG_DEVFREQ_THERMAL ++ struct devfreq_cooling_power *callbacks = NULL; ++ _mali_osk_device_data data; ++#endif ++ struct devfreq_dev_profile *dp; ++ struct dev_pm_opp *opp; ++ unsigned long opp_rate; ++ int err; ++ ++ MALI_DEBUG_PRINT(2, ("Init Mali devfreq\n")); ++ ++ if (!mdev->clock) ++ return -ENODEV; ++ ++ mdev->current_freq = clk_get_rate(mdev->clock); ++ ++ dp = &mdev->devfreq_profile; ++ ++ dp->initial_freq = mdev->current_freq; ++ dp->polling_ms = 100; ++ dp->target = mali_devfreq_target; ++ dp->get_dev_status = mali_devfreq_status; ++ dp->get_cur_freq = mali_devfreq_cur_freq; ++ dp->exit = mali_devfreq_exit; ++ ++ if (mali_devfreq_init_freq_table(mdev, dp)) ++ return -EFAULT; ++ ++ of_property_read_u32(np, "upthreshold", ++ &ondemand_data.upthreshold); ++ of_property_read_u32(np, "downdifferential", ++ &ondemand_data.downdifferential); ++ ++ mdev->devfreq = devfreq_add_device(mdev->dev, dp, ++ "simple_ondemand", &ondemand_data); ++ if (IS_ERR(mdev->devfreq)) { ++ mali_devfreq_term_freq_table(mdev); ++ return PTR_ERR(mdev->devfreq); ++ } ++ ++ err = devfreq_register_opp_notifier(mdev->dev, mdev->devfreq); ++ if (err) { ++ MALI_PRINT_ERROR(("Failed to register OPP notifier (%d)\n", err)); ++ goto opp_notifier_failed; ++ } ++ ++ opp_rate = mdev->current_freq; ++ opp = devfreq_recommended_opp(mdev->dev, &opp_rate, 0); ++ if (!IS_ERR(opp)) ++ dev_pm_opp_put(opp); ++ mdev->devfreq->last_status.current_frequency = opp_rate; ++ ++ mali_mdevp.data = mdev->devfreq; ++ mdev->mdev_info = rockchip_system_monitor_register(mdev->dev, ++ &mali_mdevp); ++ if (IS_ERR(mdev->mdev_info)) { ++ dev_dbg(mdev->dev, "without system monitor\n"); ++ mdev->mdev_info = NULL; ++ } ++#ifdef CONFIG_DEVFREQ_THERMAL ++ if (of_machine_is_compatible("rockchip,rk3036")) ++ return 0; ++ ++ /* Initilization last_status it will be used when first power allocate called */ ++ mdev->devfreq->last_status.current_frequency = mdev->current_freq; ++ ++ if (_MALI_OSK_ERR_OK == _mali_osk_device_data_get(&data)) { ++ if (NULL != data.gpu_cooling_ops) { ++ callbacks = data.gpu_cooling_ops; ++ MALI_DEBUG_PRINT(2, ("Mali GPU Thermal: Callback handler installed \n")); ++ } ++ } ++ ++ if (callbacks) { ++ mdev->devfreq_cooling = of_devfreq_cooling_register_power( ++ mdev->dev->of_node, ++ mdev->devfreq, ++ callbacks); ++ if (IS_ERR_OR_NULL(mdev->devfreq_cooling)) { ++ err = PTR_ERR(mdev->devfreq_cooling); ++ MALI_PRINT_ERROR(("Failed to register cooling device (%d)\n", err)); ++ goto cooling_failed; ++ } else { ++ MALI_DEBUG_PRINT(2, ("Mali GPU Thermal Cooling installed \n")); ++ } ++ } ++#endif ++ ++ return 0; ++ ++#ifdef CONFIG_DEVFREQ_THERMAL ++cooling_failed: ++ devfreq_unregister_opp_notifier(mdev->dev, mdev->devfreq); ++#endif /* CONFIG_DEVFREQ_THERMAL */ ++opp_notifier_failed: ++ err = devfreq_remove_device(mdev->devfreq); ++ if (err) ++ MALI_PRINT_ERROR(("Failed to terminate devfreq (%d)\n", err)); ++ else ++ mdev->devfreq = NULL; ++ ++ return err; ++} ++ ++void mali_devfreq_term(struct mali_device *mdev) ++{ ++ int err; ++ ++ MALI_DEBUG_PRINT(2, ("Term Mali devfreq\n")); ++ ++ rockchip_system_monitor_unregister(mdev->mdev_info); ++#ifdef CONFIG_DEVFREQ_THERMAL ++ devfreq_cooling_unregister(mdev->devfreq_cooling); ++#endif ++ ++ devfreq_unregister_opp_notifier(mdev->dev, mdev->devfreq); ++ ++ err = devfreq_remove_device(mdev->devfreq); ++ if (err) ++ MALI_PRINT_ERROR(("Failed to terminate devfreq (%d)\n", err)); ++ else ++ mdev->devfreq = NULL; ++} +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_devfreq.h b/drivers/gpu/arm/mali400/mali/linux/mali_devfreq.h +new file mode 100755 +index 000000000000..ba7c017d88dc +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_devfreq.h +@@ -0,0 +1,17 @@ ++/* ++ * Copyright (C) 2011-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++#ifndef _MALI_DEVFREQ_H_ ++#define _MALI_DEVFREQ_H_ ++ ++int mali_devfreq_init(struct mali_device *mdev); ++ ++void mali_devfreq_term(struct mali_device *mdev); ++ ++#endif +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_device_pause_resume.c b/drivers/gpu/arm/mali400/mali/linux/mali_device_pause_resume.c +new file mode 100755 +index 000000000000..95c3ea12d645 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_device_pause_resume.c +@@ -0,0 +1,36 @@ ++/** ++ * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file mali_device_pause_resume.c ++ * Implementation of the Mali pause/resume functionality ++ */ ++ ++#include ++#include ++#include "mali_pm.h" ++ ++void mali_dev_pause(void) ++{ ++ /* ++ * Deactive all groups to prevent hardware being touched ++ * during the period of mali device pausing ++ */ ++ mali_pm_os_suspend(MALI_FALSE); ++} ++ ++EXPORT_SYMBOL(mali_dev_pause); ++ ++void mali_dev_resume(void) ++{ ++ mali_pm_os_resume(); ++} ++ ++EXPORT_SYMBOL(mali_dev_resume); +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_dma_fence.c b/drivers/gpu/arm/mali400/mali/linux/mali_dma_fence.c +new file mode 100755 +index 000000000000..e026e11e4bc5 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_dma_fence.c +@@ -0,0 +1,439 @@ ++/* ++ * Copyright (C) 2012-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++#include ++#include "mali_osk.h" ++#include "mali_kernel_common.h" ++ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0) ++#include "mali_dma_fence.h" ++#include ++#include ++#endif ++ ++static DEFINE_SPINLOCK(mali_dma_fence_lock); ++ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++static bool mali_dma_fence_enable_signaling(struct dma_fence *fence) ++{ ++ MALI_IGNORE(fence); ++ return true; ++} ++ ++static const char *mali_dma_fence_get_driver_name(struct dma_fence *fence) ++{ ++ MALI_IGNORE(fence); ++ return "mali"; ++} ++ ++static const char *mali_dma_fence_get_timeline_name(struct dma_fence *fence) ++{ ++ MALI_IGNORE(fence); ++ return "mali_dma_fence"; ++} ++ ++static const struct dma_fence_ops mali_dma_fence_ops = { ++ .get_driver_name = mali_dma_fence_get_driver_name, ++ .get_timeline_name = mali_dma_fence_get_timeline_name, ++ .enable_signaling = mali_dma_fence_enable_signaling, ++ .signaled = NULL, ++ .wait = dma_fence_default_wait, ++ .release = NULL ++}; ++#else ++static bool mali_dma_fence_enable_signaling(struct fence *fence) ++{ ++ MALI_IGNORE(fence); ++ return true; ++} ++ ++static const char *mali_dma_fence_get_driver_name(struct fence *fence) ++{ ++ MALI_IGNORE(fence); ++ return "mali"; ++} ++ ++static const char *mali_dma_fence_get_timeline_name(struct fence *fence) ++{ ++ MALI_IGNORE(fence); ++ return "mali_dma_fence"; ++} ++ ++static const struct fence_ops mali_dma_fence_ops = { ++ .get_driver_name = mali_dma_fence_get_driver_name, ++ .get_timeline_name = mali_dma_fence_get_timeline_name, ++ .enable_signaling = mali_dma_fence_enable_signaling, ++ .signaled = NULL, ++ .wait = fence_default_wait, ++ .release = NULL ++}; ++#endif ++ ++static void mali_dma_fence_context_cleanup(struct mali_dma_fence_context *dma_fence_context) ++{ ++ u32 i; ++ ++ MALI_DEBUG_ASSERT_POINTER(dma_fence_context); ++ ++ for (i = 0; i < dma_fence_context->num_dma_fence_waiter; i++) { ++ if (dma_fence_context->mali_dma_fence_waiters[i]) { ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++ dma_fence_remove_callback(dma_fence_context->mali_dma_fence_waiters[i]->fence, ++ &dma_fence_context->mali_dma_fence_waiters[i]->base); ++ dma_fence_put(dma_fence_context->mali_dma_fence_waiters[i]->fence); ++ ++#else ++ fence_remove_callback(dma_fence_context->mali_dma_fence_waiters[i]->fence, ++ &dma_fence_context->mali_dma_fence_waiters[i]->base); ++ fence_put(dma_fence_context->mali_dma_fence_waiters[i]->fence); ++#endif ++ kfree(dma_fence_context->mali_dma_fence_waiters[i]); ++ dma_fence_context->mali_dma_fence_waiters[i] = NULL; ++ } ++ } ++ ++ if (NULL != dma_fence_context->mali_dma_fence_waiters) ++ kfree(dma_fence_context->mali_dma_fence_waiters); ++ ++ dma_fence_context->mali_dma_fence_waiters = NULL; ++ dma_fence_context->num_dma_fence_waiter = 0; ++} ++ ++static void mali_dma_fence_context_work_func(struct work_struct *work_handle) ++{ ++ struct mali_dma_fence_context *dma_fence_context; ++ ++ MALI_DEBUG_ASSERT_POINTER(work_handle); ++ ++ dma_fence_context = container_of(work_handle, struct mali_dma_fence_context, work_handle); ++ ++ dma_fence_context->cb_func(dma_fence_context->pp_job_ptr); ++} ++ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++static void mali_dma_fence_callback(struct dma_fence *fence, struct dma_fence_cb *cb) ++#else ++static void mali_dma_fence_callback(struct fence *fence, struct fence_cb *cb) ++#endif ++{ ++ struct mali_dma_fence_waiter *dma_fence_waiter = NULL; ++ struct mali_dma_fence_context *dma_fence_context = NULL; ++ ++ MALI_DEBUG_ASSERT_POINTER(fence); ++ MALI_DEBUG_ASSERT_POINTER(cb); ++ ++ MALI_IGNORE(fence); ++ ++ dma_fence_waiter = container_of(cb, struct mali_dma_fence_waiter, base); ++ dma_fence_context = dma_fence_waiter->parent; ++ ++ MALI_DEBUG_ASSERT_POINTER(dma_fence_context); ++ ++ if (atomic_dec_and_test(&dma_fence_context->count)) ++ schedule_work(&dma_fence_context->work_handle); ++} ++ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++static _mali_osk_errcode_t mali_dma_fence_add_callback(struct mali_dma_fence_context *dma_fence_context, struct dma_fence *fence) ++#else ++static _mali_osk_errcode_t mali_dma_fence_add_callback(struct mali_dma_fence_context *dma_fence_context, struct fence *fence) ++#endif ++{ ++ int ret = 0; ++ struct mali_dma_fence_waiter *dma_fence_waiter; ++ struct mali_dma_fence_waiter **dma_fence_waiters; ++ ++ MALI_DEBUG_ASSERT_POINTER(dma_fence_context); ++ MALI_DEBUG_ASSERT_POINTER(fence); ++ ++ dma_fence_waiters = krealloc(dma_fence_context->mali_dma_fence_waiters, ++ (dma_fence_context->num_dma_fence_waiter + 1) ++ * sizeof(struct mali_dma_fence_waiter *), ++ GFP_KERNEL); ++ ++ if (NULL == dma_fence_waiters) { ++ MALI_DEBUG_PRINT(1, ("Mali dma fence: failed to realloc the dma fence waiters.\n")); ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ ++ dma_fence_context->mali_dma_fence_waiters = dma_fence_waiters; ++ ++ dma_fence_waiter = kzalloc(sizeof(struct mali_dma_fence_waiter), GFP_KERNEL); ++ ++ if (NULL == dma_fence_waiter) { ++ MALI_DEBUG_PRINT(1, ("Mali dma fence: failed to create mali dma fence waiter.\n")); ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++ dma_fence_get(fence); ++#else ++ fence_get(fence); ++#endif ++ dma_fence_waiter->fence = fence; ++ dma_fence_waiter->parent = dma_fence_context; ++ atomic_inc(&dma_fence_context->count); ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++ ret = dma_fence_add_callback(fence, &dma_fence_waiter->base, ++ mali_dma_fence_callback); ++#else ++ ret = fence_add_callback(fence, &dma_fence_waiter->base, ++ mali_dma_fence_callback); ++#endif ++ if (0 > ret) { ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++ dma_fence_put(fence); ++#else ++ fence_put(fence); ++#endif ++ kfree(dma_fence_waiter); ++ atomic_dec(&dma_fence_context->count); ++ if (-ENOENT == ret) { ++ /*-ENOENT if fence has already been signaled, return _MALI_OSK_ERR_OK*/ ++ return _MALI_OSK_ERR_OK; ++ } ++ /* Failed to add the fence callback into fence, return _MALI_OSK_ERR_FAULT*/ ++ MALI_DEBUG_PRINT(1, ("Mali dma fence: failed to add callback into fence.\n")); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ dma_fence_context->mali_dma_fence_waiters[dma_fence_context->num_dma_fence_waiter] = dma_fence_waiter; ++ dma_fence_context->num_dma_fence_waiter++; ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++struct dma_fence *mali_dma_fence_new(u32 context, u32 seqno) ++#else ++struct fence *mali_dma_fence_new(u32 context, u32 seqno) ++#endif ++{ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++ struct dma_fence *fence = NULL; ++ fence = kzalloc(sizeof(struct dma_fence), GFP_KERNEL); ++#else ++ struct fence *fence = NULL; ++ fence = kzalloc(sizeof(struct fence), GFP_KERNEL); ++#endif ++ if (NULL == fence) { ++ MALI_DEBUG_PRINT(1, ("Mali dma fence: failed to create dma fence.\n")); ++ return fence; ++ } ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++ dma_fence_init(fence, ++ &mali_dma_fence_ops, ++ &mali_dma_fence_lock, ++ context, seqno); ++#else ++ fence_init(fence, ++ &mali_dma_fence_ops, ++ &mali_dma_fence_lock, ++ context, seqno); ++#endif ++ return fence; ++} ++ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++void mali_dma_fence_signal_and_put(struct dma_fence **fence) ++#else ++void mali_dma_fence_signal_and_put(struct fence **fence) ++#endif ++{ ++ MALI_DEBUG_ASSERT_POINTER(fence); ++ MALI_DEBUG_ASSERT_POINTER(*fence); ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++ dma_fence_signal(*fence); ++ dma_fence_put(*fence); ++#else ++ fence_signal(*fence); ++ fence_put(*fence); ++#endif ++ *fence = NULL; ++} ++ ++void mali_dma_fence_context_init(struct mali_dma_fence_context *dma_fence_context, ++ mali_dma_fence_context_callback_func_t cb_func, ++ void *pp_job_ptr) ++{ ++ MALI_DEBUG_ASSERT_POINTER(dma_fence_context); ++ ++ INIT_WORK(&dma_fence_context->work_handle, mali_dma_fence_context_work_func); ++ atomic_set(&dma_fence_context->count, 1); ++ dma_fence_context->num_dma_fence_waiter = 0; ++ dma_fence_context->mali_dma_fence_waiters = NULL; ++ dma_fence_context->cb_func = cb_func; ++ dma_fence_context->pp_job_ptr = pp_job_ptr; ++} ++ ++_mali_osk_errcode_t mali_dma_fence_context_add_waiters(struct mali_dma_fence_context *dma_fence_context, ++ struct reservation_object *dma_reservation_object) ++{ ++ _mali_osk_errcode_t ret = _MALI_OSK_ERR_OK; ++ u32 shared_count = 0, i; ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++ struct dma_fence *exclusive_fence = NULL; ++ struct dma_fence **shared_fences = NULL; ++#else ++ struct fence *exclusive_fence = NULL; ++ struct fence **shared_fences = NULL; ++#endif ++ MALI_DEBUG_ASSERT_POINTER(dma_fence_context); ++ MALI_DEBUG_ASSERT_POINTER(dma_reservation_object); ++ ++ /* Get all the shared/exclusive fences in the reservation object of dma buf*/ ++ ret = reservation_object_get_fences_rcu(dma_reservation_object, &exclusive_fence, ++ &shared_count, &shared_fences); ++ if (ret < 0) { ++ MALI_DEBUG_PRINT(1, ("Mali dma fence: failed to get shared or exclusive_fence dma fences from the reservation object of dma buf.\n")); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ if (exclusive_fence) { ++ ret = mali_dma_fence_add_callback(dma_fence_context, exclusive_fence); ++ if (_MALI_OSK_ERR_OK != ret) { ++ MALI_DEBUG_PRINT(1, ("Mali dma fence: failed to add callback into exclusive fence.\n")); ++ mali_dma_fence_context_cleanup(dma_fence_context); ++ goto ended; ++ } ++ } ++ ++ ++ for (i = 0; i < shared_count; i++) { ++ ret = mali_dma_fence_add_callback(dma_fence_context, shared_fences[i]); ++ if (_MALI_OSK_ERR_OK != ret) { ++ MALI_DEBUG_PRINT(1, ("Mali dma fence: failed to add callback into shared fence [%d].\n", i)); ++ mali_dma_fence_context_cleanup(dma_fence_context); ++ break; ++ } ++ } ++ ++ended: ++ ++ if (exclusive_fence) ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++ dma_fence_put(exclusive_fence); ++#else ++ fence_put(exclusive_fence); ++#endif ++ ++ if (shared_fences) { ++ for (i = 0; i < shared_count; i++) { ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++ dma_fence_put(shared_fences[i]); ++#else ++ fence_put(shared_fences[i]); ++#endif ++ } ++ kfree(shared_fences); ++ } ++ ++ return ret; ++} ++ ++ ++void mali_dma_fence_context_term(struct mali_dma_fence_context *dma_fence_context) ++{ ++ MALI_DEBUG_ASSERT_POINTER(dma_fence_context); ++ atomic_set(&dma_fence_context->count, 0); ++ if (dma_fence_context->work_handle.func) { ++ cancel_work_sync(&dma_fence_context->work_handle); ++ } ++ mali_dma_fence_context_cleanup(dma_fence_context); ++} ++ ++void mali_dma_fence_context_dec_count(struct mali_dma_fence_context *dma_fence_context) ++{ ++ MALI_DEBUG_ASSERT_POINTER(dma_fence_context); ++ ++ if (atomic_dec_and_test(&dma_fence_context->count)) ++ schedule_work(&dma_fence_context->work_handle); ++} ++ ++ ++void mali_dma_fence_add_reservation_object_list(struct reservation_object *dma_reservation_object, ++ struct reservation_object **dma_reservation_object_list, ++ u32 *num_dma_reservation_object) ++{ ++ u32 i; ++ ++ MALI_DEBUG_ASSERT_POINTER(dma_reservation_object); ++ MALI_DEBUG_ASSERT_POINTER(dma_reservation_object_list); ++ MALI_DEBUG_ASSERT_POINTER(num_dma_reservation_object); ++ ++ for (i = 0; i < *num_dma_reservation_object; i++) { ++ if (dma_reservation_object_list[i] == dma_reservation_object) ++ return; ++ } ++ ++ dma_reservation_object_list[*num_dma_reservation_object] = dma_reservation_object; ++ (*num_dma_reservation_object)++; ++} ++ ++int mali_dma_fence_lock_reservation_object_list(struct reservation_object **dma_reservation_object_list, ++ u32 num_dma_reservation_object, struct ww_acquire_ctx *ww_actx) ++{ ++ u32 i; ++ ++ struct reservation_object *reservation_object_to_slow_lock = NULL; ++ ++ MALI_DEBUG_ASSERT_POINTER(dma_reservation_object_list); ++ MALI_DEBUG_ASSERT_POINTER(ww_actx); ++ ++ ww_acquire_init(ww_actx, &reservation_ww_class); ++ ++again: ++ for (i = 0; i < num_dma_reservation_object; i++) { ++ int ret; ++ ++ if (dma_reservation_object_list[i] == reservation_object_to_slow_lock) { ++ reservation_object_to_slow_lock = NULL; ++ continue; ++ } ++ ++ ret = ww_mutex_lock(&dma_reservation_object_list[i]->lock, ww_actx); ++ ++ if (ret < 0) { ++ u32 slow_lock_index = i; ++ ++ /* unlock all pre locks we have already locked.*/ ++ while (i > 0) { ++ i--; ++ ww_mutex_unlock(&dma_reservation_object_list[i]->lock); ++ } ++ ++ if (NULL != reservation_object_to_slow_lock) ++ ww_mutex_unlock(&reservation_object_to_slow_lock->lock); ++ ++ if (ret == -EDEADLK) { ++ reservation_object_to_slow_lock = dma_reservation_object_list[slow_lock_index]; ++ ww_mutex_lock_slow(&reservation_object_to_slow_lock->lock, ww_actx); ++ goto again; ++ } ++ ww_acquire_fini(ww_actx); ++ MALI_DEBUG_PRINT(1, ("Mali dma fence: failed to lock all dma reservation objects.\n", i)); ++ return ret; ++ } ++ } ++ ++ ww_acquire_done(ww_actx); ++ return 0; ++} ++ ++void mali_dma_fence_unlock_reservation_object_list(struct reservation_object **dma_reservation_object_list, ++ u32 num_dma_reservation_object, struct ww_acquire_ctx *ww_actx) ++{ ++ u32 i; ++ ++ for (i = 0; i < num_dma_reservation_object; i++) ++ ww_mutex_unlock(&dma_reservation_object_list[i]->lock); ++ ++ ww_acquire_fini(ww_actx); ++} +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_dma_fence.h b/drivers/gpu/arm/mali400/mali/linux/mali_dma_fence.h +new file mode 100755 +index 000000000000..d44f6d1a8926 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_dma_fence.h +@@ -0,0 +1,124 @@ ++/* ++ * Copyright (C) 2012-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file mali_dma_fence.h ++ * ++ * Mali interface for Linux dma buf fence objects. ++ */ ++ ++#ifndef _MALI_DMA_FENCE_H_ ++#define _MALI_DMA_FENCE_H_ ++ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0) ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++#include ++#else ++#include ++#endif ++#include ++#endif ++ ++struct mali_dma_fence_context; ++ ++/* The mali dma fence context callback function */ ++typedef void (*mali_dma_fence_context_callback_func_t)(void *pp_job_ptr); ++ ++struct mali_dma_fence_waiter { ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++ struct dma_fence *fence; ++ struct dma_fence_cb base; ++#else ++ struct fence_cb base; ++ struct fence *fence; ++#endif ++ struct mali_dma_fence_context *parent; ++}; ++ ++struct mali_dma_fence_context { ++ struct work_struct work_handle; ++ struct mali_dma_fence_waiter **mali_dma_fence_waiters; ++ u32 num_dma_fence_waiter; ++ atomic_t count; ++ void *pp_job_ptr; /* the mali pp job pointer */; ++ mali_dma_fence_context_callback_func_t cb_func; ++}; ++ ++/* Create a dma fence ++ * @param context The execution context this fence is run on ++ * @param seqno A linearly increasing sequence number for this context ++ * @return the new dma fence if success, or NULL on failure. ++ */ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++struct dma_fence *mali_dma_fence_new(u32 context, u32 seqno); ++#else ++struct fence *mali_dma_fence_new(u32 context, u32 seqno); ++#endif ++/* Signal and put dma fence ++ * @param fence The dma fence to signal and put ++ */ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++void mali_dma_fence_signal_and_put(struct dma_fence **fence); ++#else ++void mali_dma_fence_signal_and_put(struct fence **fence); ++#endif ++/** ++ * Initialize a mali dma fence context for pp job. ++ * @param dma_fence_context The mali dma fence context to initialize. ++ * @param cb_func The dma fence context callback function to call when all dma fence release. ++ * @param pp_job_ptr The pp_job to call function with. ++ */ ++void mali_dma_fence_context_init(struct mali_dma_fence_context *dma_fence_context, ++ mali_dma_fence_context_callback_func_t cb_func, ++ void *pp_job_ptr); ++ ++/** ++ * Add new mali dma fence waiter into mali dma fence context ++ * @param dma_fence_context The mali dma fence context ++ * @param dma_reservation_object the reservation object to create new mali dma fence waiters ++ * @return _MALI_OSK_ERR_OK if success, or not. ++ */ ++_mali_osk_errcode_t mali_dma_fence_context_add_waiters(struct mali_dma_fence_context *dma_fence_context, ++ struct reservation_object *dma_reservation_object); ++ ++/** ++ * Release the dma fence context ++ * @param dma_fence_text The mali dma fence context. ++ */ ++void mali_dma_fence_context_term(struct mali_dma_fence_context *dma_fence_context); ++ ++/** ++ * Decrease the dma fence context atomic count ++ * @param dma_fence_text The mali dma fence context. ++ */ ++void mali_dma_fence_context_dec_count(struct mali_dma_fence_context *dma_fence_context); ++ ++/** ++ * Get all reservation object ++ * @param dma_reservation_object The reservation object to add into the reservation object list ++ * @param dma_reservation_object_list The reservation object list to store all reservation object ++ * @param num_dma_reservation_object The number of all reservation object ++ */ ++void mali_dma_fence_add_reservation_object_list(struct reservation_object *dma_reservation_object, ++ struct reservation_object **dma_reservation_object_list, ++ u32 *num_dma_reservation_object); ++ ++/** ++ * Wait/wound mutex lock to lock all reservation object. ++ */ ++int mali_dma_fence_lock_reservation_object_list(struct reservation_object **dma_reservation_object_list, ++ u32 num_dma_reservation_object, struct ww_acquire_ctx *ww_actx); ++ ++/** ++ * Wait/wound mutex lock to unlock all reservation object. ++ */ ++void mali_dma_fence_unlock_reservation_object_list(struct reservation_object **dma_reservation_object_list, ++ u32 num_dma_reservation_object, struct ww_acquire_ctx *ww_actx); ++#endif +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_internal_sync.c b/drivers/gpu/arm/mali400/mali/linux/mali_internal_sync.c +new file mode 100755 +index 000000000000..e13cbad3e513 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_internal_sync.c +@@ -0,0 +1,783 @@ ++/* ++ * Copyright (C) 2012-2018 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include "mali_internal_sync.h" ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 6, 0) ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "mali_osk.h" ++#include "mali_kernel_common.h" ++#if defined(DEBUG) ++#include "mali_session.h" ++#include "mali_timeline.h" ++#endif ++ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++static const struct dma_fence_ops fence_ops; ++#else ++static const struct fence_ops fence_ops; ++#endif ++ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++static struct mali_internal_sync_point *mali_internal_fence_to_sync_pt(struct dma_fence *fence) ++#else ++static struct mali_internal_sync_point *mali_internal_fence_to_sync_pt(struct fence *fence) ++#endif ++{ ++ MALI_DEBUG_ASSERT_POINTER(fence); ++ return container_of(fence, struct mali_internal_sync_point, base); ++} ++ ++static inline struct mali_internal_sync_timeline *mali_internal_sync_pt_to_sync_timeline(struct mali_internal_sync_point *sync_pt) ++{ ++ MALI_DEBUG_ASSERT_POINTER(sync_pt); ++ return container_of(sync_pt->base.lock, struct mali_internal_sync_timeline, sync_pt_list_lock); ++} ++ ++static void mali_internal_sync_timeline_free(struct kref *kref_count) ++{ ++ struct mali_internal_sync_timeline *sync_timeline; ++ ++ MALI_DEBUG_ASSERT_POINTER(kref_count); ++ ++ sync_timeline = container_of(kref_count, struct mali_internal_sync_timeline, kref_count); ++ ++ if (sync_timeline->ops->release_obj) ++ sync_timeline->ops->release_obj(sync_timeline); ++ ++ kfree(sync_timeline); ++} ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) ++static void mali_internal_fence_check_cb_func(struct fence *fence, struct fence_cb *cb) ++#else ++static void mali_internal_fence_check_cb_func(struct dma_fence *fence, struct dma_fence_cb *cb) ++#endif ++{ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) ++ struct mali_internal_sync_fence_cb *check; ++#else ++ struct mali_internal_sync_fence_waiter *waiter; ++#endif ++ struct mali_internal_sync_fence *sync_fence; ++ int ret; ++ MALI_DEBUG_ASSERT_POINTER(cb); ++ MALI_IGNORE(fence); ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) ++ check = container_of(cb, struct mali_internal_sync_fence_cb, cb); ++ sync_fence = check->sync_file; ++#else ++ waiter = container_of(cb, struct mali_internal_sync_fence_waiter, cb); ++ sync_fence = (struct mali_internal_sync_fence *)waiter->work.private; ++#endif ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) ++ ret = atomic_dec_and_test(&sync_fence->status); ++ if (ret) ++ wake_up_all(&sync_fence->wq); ++#else ++ ret = sync_fence->fence->ops->signaled(sync_fence->fence); ++ ++ if (0 > ret) ++ MALI_PRINT_ERROR(("Mali internal sync:Failed to wait fence 0x%x for sync_fence 0x%x.\n", fence, sync_fence)); ++ if (1 == ret) ++ wake_up_all(&sync_fence->wq); ++#endif ++} ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) ++static void mali_internal_sync_fence_add_fence(struct mali_internal_sync_fence *sync_fence, struct fence *sync_pt) ++{ ++ int fence_num = 0; ++ MALI_DEBUG_ASSERT_POINTER(sync_fence); ++ MALI_DEBUG_ASSERT_POINTER(sync_pt); ++ ++ fence_num = sync_fence->num_fences; ++ ++ sync_fence->cbs[fence_num].fence = sync_pt; ++ sync_fence->cbs[fence_num].sync_file = sync_fence; ++ ++ if (!fence_add_callback(sync_pt, &sync_fence->cbs[fence_num].cb, mali_internal_fence_check_cb_func)) { ++ fence_get(sync_pt); ++ sync_fence->num_fences++; ++ atomic_inc(&sync_fence->status); ++ } ++} ++#endif ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 13, 0) ++static int mali_internal_sync_fence_wake_up_wq(wait_queue_entry_t *curr, unsigned mode, ++ int wake_flags, void *key) ++#else ++static int mali_internal_sync_fence_wake_up_wq(wait_queue_t *curr, unsigned mode, ++ int wake_flags, void *key) ++#endif ++{ ++ struct mali_internal_sync_fence_waiter *wait; ++ MALI_IGNORE(mode); ++ MALI_IGNORE(wake_flags); ++ MALI_IGNORE(key); ++ ++ wait = container_of(curr, struct mali_internal_sync_fence_waiter, work); ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 13, 0) ++ list_del_init(&wait->work.entry); ++#else ++ list_del_init(&wait->work.task_list); ++#endif ++ wait->callback(wait->work.private, wait); ++ return 1; ++} ++ ++struct mali_internal_sync_timeline *mali_internal_sync_timeline_create(const struct mali_internal_sync_timeline_ops *ops, ++ int size, const char *name) ++{ ++ struct mali_internal_sync_timeline *sync_timeline = NULL; ++ ++ MALI_DEBUG_ASSERT_POINTER(ops); ++ ++ if (size < sizeof(struct mali_internal_sync_timeline)) { ++ MALI_PRINT_ERROR(("Mali internal sync:Invalid size to create the mali internal sync timeline.\n")); ++ goto err; ++ } ++ ++ sync_timeline = kzalloc(size, GFP_KERNEL); ++ if (NULL == sync_timeline) { ++ MALI_PRINT_ERROR(("Mali internal sync:Failed to allocate buffer for the mali internal sync timeline.\n")); ++ goto err; ++ } ++ kref_init(&sync_timeline->kref_count); ++ sync_timeline->ops = ops; ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++ sync_timeline->fence_context = dma_fence_context_alloc(1); ++#else ++ sync_timeline->fence_context = fence_context_alloc(1); ++#endif ++ strlcpy(sync_timeline->name, name, sizeof(sync_timeline->name)); ++ ++ INIT_LIST_HEAD(&sync_timeline->sync_pt_list_head); ++ spin_lock_init(&sync_timeline->sync_pt_list_lock); ++ ++ return sync_timeline; ++err: ++ if (NULL != sync_timeline) { ++ kfree(sync_timeline); ++ } ++ return NULL; ++} ++ ++void mali_internal_sync_timeline_destroy(struct mali_internal_sync_timeline *sync_timeline) ++{ ++ MALI_DEBUG_ASSERT_POINTER(sync_timeline); ++ ++ sync_timeline->destroyed = MALI_TRUE; ++ ++ smp_wmb(); ++ ++ mali_internal_sync_timeline_signal(sync_timeline); ++ kref_put(&sync_timeline->kref_count, mali_internal_sync_timeline_free); ++} ++ ++void mali_internal_sync_timeline_signal(struct mali_internal_sync_timeline *sync_timeline) ++{ ++ unsigned long flags; ++ struct mali_internal_sync_point *sync_pt, *next; ++ ++ MALI_DEBUG_ASSERT_POINTER(sync_timeline); ++ ++ spin_lock_irqsave(&sync_timeline->sync_pt_list_lock, flags); ++ ++ list_for_each_entry_safe(sync_pt, next, &sync_timeline->sync_pt_list_head, ++ sync_pt_list) { ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++ if (dma_fence_is_signaled_locked(&sync_pt->base)) ++#else ++ if (fence_is_signaled_locked(&sync_pt->base)) ++#endif ++ list_del_init(&sync_pt->sync_pt_list); ++ } ++ ++ spin_unlock_irqrestore(&sync_timeline->sync_pt_list_lock, flags); ++} ++ ++struct mali_internal_sync_point *mali_internal_sync_point_create(struct mali_internal_sync_timeline *sync_timeline, int size) ++{ ++ unsigned long flags; ++ struct mali_internal_sync_point *sync_pt = NULL; ++ ++ MALI_DEBUG_ASSERT_POINTER(sync_timeline); ++ ++ if (size < sizeof(struct mali_internal_sync_point)) { ++ MALI_PRINT_ERROR(("Mali internal sync:Invalid size to create the mali internal sync point.\n")); ++ goto err; ++ } ++ ++ sync_pt = kzalloc(size, GFP_KERNEL); ++ if (NULL == sync_pt) { ++ MALI_PRINT_ERROR(("Mali internal sync:Failed to allocate buffer for the mali internal sync point.\n")); ++ goto err; ++ } ++ spin_lock_irqsave(&sync_timeline->sync_pt_list_lock, flags); ++ kref_get(&sync_timeline->kref_count); ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++ dma_fence_init(&sync_pt->base, &fence_ops, &sync_timeline->sync_pt_list_lock, ++ sync_timeline->fence_context, ++sync_timeline->value); ++#else ++ fence_init(&sync_pt->base, &fence_ops, &sync_timeline->sync_pt_list_lock, ++ sync_timeline->fence_context, ++sync_timeline->value); ++#endif ++ INIT_LIST_HEAD(&sync_pt->sync_pt_list); ++ spin_unlock_irqrestore(&sync_timeline->sync_pt_list_lock, flags); ++ ++ return sync_pt; ++err: ++ if (NULL != sync_pt) { ++ kfree(sync_pt); ++ } ++ return NULL; ++} ++ ++struct mali_internal_sync_fence *mali_internal_sync_fence_fdget(int fd) ++{ ++ struct file *file = fget(fd); ++ ++ if (NULL == file) { ++ return NULL; ++ } ++ ++ return file->private_data; ++} ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) ++struct mali_internal_sync_fence *mali_internal_sync_fence_merge( ++ struct mali_internal_sync_fence *sync_fence1, struct mali_internal_sync_fence *sync_fence2) ++{ ++ struct mali_internal_sync_fence *new_sync_fence; ++ int i, j, num_fence1, num_fence2, total_fences; ++ struct fence *fence0 = NULL; ++ ++ MALI_DEBUG_ASSERT_POINTER(sync_fence1); ++ MALI_DEBUG_ASSERT_POINTER(sync_fence2); ++ ++ num_fence1 = sync_fence1->num_fences; ++ num_fence2 = sync_fence2->num_fences; ++ ++ total_fences = num_fence1 + num_fence2; ++ ++ i = 0; ++ j = 0; ++ ++ if (num_fence1 > 0) { ++ fence0 = sync_fence1->cbs[i].fence; ++ i = 1; ++ } else if (num_fence2 > 0) { ++ fence0 = sync_fence2->cbs[i].fence; ++ j = 1; ++ } ++ ++ new_sync_fence = (struct mali_internal_sync_fence *)sync_file_create(fence0); ++ if (NULL == new_sync_fence) { ++ MALI_PRINT_ERROR(("Mali internal sync:Failed to create the mali internal sync fence when merging sync fence.\n")); ++ return NULL; ++ } ++ ++ fence_remove_callback(new_sync_fence->cb[0].fence, &new_sync_fence->cb[0].cb); ++ new_sync_fence->num_fences = 0; ++ atomic_dec(&new_sync_fence->status); ++ ++ for (; i < num_fence1 && j < num_fence2;) { ++ struct fence *fence1 = sync_fence1->cbs[i].fence; ++ struct fence *fence2 = sync_fence2->cbs[j].fence; ++ ++ if (fence1->context < fence2->context) { ++ mali_internal_sync_fence_add_fence(new_sync_fence, fence1); ++ ++ i++; ++ } else if (fence1->context > fence2->context) { ++ mali_internal_sync_fence_add_fence(new_sync_fence, fence2); ++ ++ j++; ++ } else { ++ if (fence1->seqno - fence2->seqno <= INT_MAX) ++ mali_internal_sync_fence_add_fence(new_sync_fence, fence1); ++ else ++ mali_internal_sync_fence_add_fence(new_sync_fence, fence2); ++ i++; ++ j++; ++ } ++ } ++ ++ for (; i < num_fence1; i++) ++ mali_internal_sync_fence_add_fence(new_sync_fence, sync_fence1->cbs[i].fence); ++ ++ for (; j < num_fence2; j++) ++ mali_internal_sync_fence_add_fence(new_sync_fence, sync_fence2->cbs[j].fence); ++ ++ return new_sync_fence; ++} ++#else ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) ++static struct fence **mali_internal_get_fences(struct mali_internal_sync_fence *sync_fence, int *num_fences) ++#else ++static struct dma_fence **mali_internal_get_fences(struct mali_internal_sync_fence *sync_fence, int *num_fences) ++#endif ++{ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) ++ if (sync_fence->fence->ops == &fence_array_ops) { ++ struct fence_array *fence_array = container_of(sync_fence->fence, struct fence_array, base); ++ *num_fences = fence_array->num_fences; ++ return fence_array->fences; ++ } ++#else ++ if (sync_fence->fence->ops == &dma_fence_array_ops) { ++ struct dma_fence_array *fence_array = container_of(sync_fence->fence, struct dma_fence_array, base); ++ *num_fences = fence_array->num_fences; ++ return fence_array->fences; ++ } ++#endif ++ *num_fences = 1; ++ return &sync_fence->fence; ++} ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) ++static void mali_internal_add_fence_array(struct fence **fences, int *num_fences, struct fence *fence) ++#else ++static void mali_internal_add_fence_array(struct dma_fence **fences, int *num_fences, struct dma_fence *fence) ++#endif ++{ ++ fences[*num_fences] = fence; ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) ++ if (!fence_is_signaled(fence)) { ++ fence_get(fence); ++ (*num_fences)++; ++ } ++#else ++ if (!dma_fence_is_signaled(fence)) { ++ dma_fence_get(fence); ++ (*num_fences)++; ++ } ++#endif ++} ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) ++static int mali_internal_sync_fence_set_fence_array(struct mali_internal_sync_fence *sync_fence, ++ struct fence **fences, int num_fences) ++#else ++static int mali_internal_sync_fence_set_fence_array(struct mali_internal_sync_fence *sync_fence, ++ struct dma_fence **fences, int num_fences) ++#endif ++{ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) ++ struct fence_array *array; ++#else ++ struct dma_fence_array *array; ++#endif ++ if(num_fences == 1) { ++ sync_fence->fence =fences[0]; ++ kfree(fences); ++ } else { ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) ++ array = fence_array_create(num_fences, fences, ++ fence_context_alloc(1), 1, false); ++#else ++ array = dma_fence_array_create(num_fences, fences, ++ dma_fence_context_alloc(1), 1, false); ++#endif ++ if (!array){ ++ return -ENOMEM; ++ } ++ sync_fence->fence = &array->base; ++ } ++ return 0; ++} ++ ++struct mali_internal_sync_fence *mali_internal_sync_fence_merge( ++ struct mali_internal_sync_fence *sync_fence1, struct mali_internal_sync_fence *sync_fence2) ++{ ++ struct mali_internal_sync_fence *sync_fence; ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) ++ struct fence **fences, **nfences, **fences1, **fences2; ++#else ++ struct dma_fence **fences, **nfences, **fences1, **fences2; ++#endif ++ int real_num_fences, i, j, num_fences, num_fences1, num_fences2; ++ ++ fences1 = mali_internal_get_fences(sync_fence1, &num_fences1); ++ fences2 = mali_internal_get_fences(sync_fence2, &num_fences2); ++ ++ num_fences = num_fences1 + num_fences2; ++ ++ fences = kcalloc(num_fences, sizeof(*fences), GFP_KERNEL); ++ if (!fences) { ++ MALI_PRINT_ERROR(("Mali internal sync:Failed to alloc buffer for fences.\n")); ++ goto fences_alloc_failed; ++ } ++ ++ for (real_num_fences = i = j = 0; i < num_fences1 && j < num_fences2;) { ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) ++ struct fence *fence1 = fences1[i]; ++ struct fence *fence2 = fences2[j]; ++#else ++ struct dma_fence *fence1 = fences1[i]; ++ struct dma_fence *fence2 = fences2[j]; ++#endif ++ if (fence1->context < fence2->context) { ++ mali_internal_add_fence_array(fences, &real_num_fences, fence1); ++ ++ i++; ++ } else if (fence1->context > fence2->context) { ++ mali_internal_add_fence_array(fences, &real_num_fences, fence2); ++ ++ j++; ++ } else { ++ if (fence1->seqno - fence2->seqno <= INT_MAX) ++ mali_internal_add_fence_array(fences, &real_num_fences, fence1); ++ else ++ mali_internal_add_fence_array(fences, &real_num_fences, fence2); ++ ++ i++; ++ j++; ++ } ++ } ++ ++ for (; i < num_fences1; i++) ++ mali_internal_add_fence_array(fences, &real_num_fences, fences1[i]); ++ ++ for (; j < num_fences2; j++) ++ mali_internal_add_fence_array(fences, &real_num_fences, fences2[j]); ++ ++ if (0 == real_num_fences) ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) ++ fences[real_num_fences++] = fence_get(fences1[0]); ++#else ++ fences[real_num_fences++] = dma_fence_get(fences1[0]); ++#endif ++ ++ if (num_fences > real_num_fences) { ++ nfences = krealloc(fences, real_num_fences * sizeof(*fences), ++ GFP_KERNEL); ++ if (!nfences) ++ goto nfences_alloc_failed; ++ ++ fences = nfences; ++ } ++ ++ sync_fence = (struct mali_internal_sync_fence *)sync_file_create(fences[0]); ++ if (NULL == sync_fence) { ++ MALI_PRINT_ERROR(("Mali internal sync:Failed to create the mali internal sync fence when merging sync fence.\n")); ++ goto sync_fence_alloc_failed; ++ } ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) ++ fence_put(fences[0]); ++#else ++ dma_fence_put(fences[0]); ++#endif ++ ++ if (mali_internal_sync_fence_set_fence_array(sync_fence, fences, real_num_fences) < 0) { ++ MALI_PRINT_ERROR(("Mali internal sync:Failed to set fence for sync fence.\n")); ++ goto sync_fence_set_failed; ++ } ++ ++ return sync_fence; ++ ++sync_fence_set_failed: ++ fput(sync_fence->file); ++sync_fence_alloc_failed: ++ for (i = 0; i < real_num_fences; i++) ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) ++ fence_put(fences[i]); ++#else ++ dma_fence_put(fences[i]); ++#endif ++nfences_alloc_failed: ++ kfree(fences); ++fences_alloc_failed: ++ return NULL; ++} ++#endif ++ ++void mali_internal_sync_fence_waiter_init(struct mali_internal_sync_fence_waiter *waiter, ++ mali_internal_sync_callback_t callback) ++{ ++ MALI_DEBUG_ASSERT_POINTER(waiter); ++ MALI_DEBUG_ASSERT_POINTER(callback); ++ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 13, 0) ++ INIT_LIST_HEAD(&waiter->work.entry); ++#else ++ INIT_LIST_HEAD(&waiter->work.task_list); ++#endif ++ waiter->callback = callback; ++} ++ ++int mali_internal_sync_fence_wait_async(struct mali_internal_sync_fence *sync_fence, ++ struct mali_internal_sync_fence_waiter *waiter) ++{ ++ int err; ++ unsigned long flags; ++ ++ MALI_DEBUG_ASSERT_POINTER(sync_fence); ++ MALI_DEBUG_ASSERT_POINTER(waiter); ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) ++ err = atomic_read(&sync_fence->status); ++ ++ if (0 > err) ++ return err; ++ ++ if (!err) ++ return 1; ++ ++ init_waitqueue_func_entry(&waiter->work, mali_internal_sync_fence_wake_up_wq); ++ waiter->work.private = sync_fence; ++ ++ spin_lock_irqsave(&sync_fence->wq.lock, flags); ++ err = atomic_read(&sync_fence->status); ++ ++ if (0 < err) ++ __add_wait_queue_tail(&sync_fence->wq, &waiter->work); ++ spin_unlock_irqrestore(&sync_fence->wq.lock, flags); ++ ++ if (0 > err) ++ return err; ++ ++ return !err; ++#else ++ if ((sync_fence->fence) && (sync_fence->fence->ops) && (sync_fence->fence->ops->signaled)) ++ err = sync_fence->fence->ops->signaled(sync_fence->fence); ++ else ++ err = -1; ++ ++ if (0 > err) ++ return err; ++ ++ if (1 == err) ++ return err; ++ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++ err = dma_fence_add_callback(sync_fence->fence, &waiter->cb, mali_internal_fence_check_cb_func); ++#elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0) ++ err = fence_add_callback(sync_fence->fence, &waiter->cb, mali_internal_fence_check_cb_func); ++#endif ++ ++ if (0 != err) { ++ if (-ENOENT == err) ++ err = 1; ++ return err; ++ } ++ init_waitqueue_func_entry(&waiter->work, mali_internal_sync_fence_wake_up_wq); ++ waiter->work.private = sync_fence; ++ ++ spin_lock_irqsave(&sync_fence->wq.lock, flags); ++ err = sync_fence->fence->ops->signaled(sync_fence->fence); ++ ++ if (0 == err){ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 13, 0) ++ __add_wait_queue_entry_tail(&sync_fence->wq, &waiter->work); ++#else ++ __add_wait_queue_tail(&sync_fence->wq, &waiter->work); ++#endif ++ } ++ spin_unlock_irqrestore(&sync_fence->wq.lock, flags); ++ ++ return err; ++#endif ++} ++ ++int mali_internal_sync_fence_cancel_async(struct mali_internal_sync_fence *sync_fence, ++ struct mali_internal_sync_fence_waiter *waiter) ++{ ++ unsigned long flags; ++ int ret = 0; ++ ++ MALI_DEBUG_ASSERT_POINTER(sync_fence); ++ MALI_DEBUG_ASSERT_POINTER(waiter); ++ ++ spin_lock_irqsave(&sync_fence->wq.lock, flags); ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 13, 0) ++ if (!list_empty(&waiter->work.entry)) ++ list_del_init(&waiter->work.entry); ++#else ++ if (!list_empty(&waiter->work.task_list)) ++ list_del_init(&waiter->work.task_list); ++#endif ++ else ++ ret = -ENOENT; ++ spin_unlock_irqrestore(&sync_fence->wq.lock, flags); ++ ++ if (0 == ret) { ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++ dma_fence_remove_callback(sync_fence->fence, &waiter->cb); ++#elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0) ++ fence_remove_callback(sync_fence->fence, &waiter->cb); ++#endif ++ ++ } ++ ++ return ret; ++} ++ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++static const char *mali_internal_fence_get_driver_name(struct dma_fence *fence) ++#else ++static const char *mali_internal_fence_get_driver_name(struct fence *fence) ++#endif ++{ ++ struct mali_internal_sync_point *sync_pt; ++ struct mali_internal_sync_timeline *parent; ++ ++ MALI_DEBUG_ASSERT_POINTER(fence); ++ ++ sync_pt = mali_internal_fence_to_sync_pt(fence); ++ parent = mali_internal_sync_pt_to_sync_timeline(sync_pt); ++ ++ return parent->ops->driver_name; ++} ++ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++static const char *mali_internal_fence_get_timeline_name(struct dma_fence *fence) ++#else ++static const char *mali_internal_fence_get_timeline_name(struct fence *fence) ++#endif ++{ ++ struct mali_internal_sync_point *sync_pt; ++ struct mali_internal_sync_timeline *parent; ++ ++ MALI_DEBUG_ASSERT_POINTER(fence); ++ ++ sync_pt = mali_internal_fence_to_sync_pt(fence); ++ parent = mali_internal_sync_pt_to_sync_timeline(sync_pt); ++ ++ return parent->name; ++} ++ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++static void mali_internal_fence_release(struct dma_fence *fence) ++#else ++static void mali_internal_fence_release(struct fence *fence) ++#endif ++{ ++ unsigned long flags; ++ struct mali_internal_sync_point *sync_pt; ++ struct mali_internal_sync_timeline *parent; ++ ++ MALI_DEBUG_ASSERT_POINTER(fence); ++ ++ sync_pt = mali_internal_fence_to_sync_pt(fence); ++ parent = mali_internal_sync_pt_to_sync_timeline(sync_pt); ++ ++ ++ spin_lock_irqsave(fence->lock, flags); ++ if (WARN_ON_ONCE(!list_empty(&sync_pt->sync_pt_list))) ++ list_del(&sync_pt->sync_pt_list); ++ spin_unlock_irqrestore(fence->lock, flags); ++ ++ if (parent->ops->free_pt) ++ parent->ops->free_pt(sync_pt); ++ ++ kref_put(&parent->kref_count, mali_internal_sync_timeline_free); ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++ dma_fence_free(&sync_pt->base); ++#else ++ fence_free(&sync_pt->base); ++#endif ++} ++ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++static bool mali_internal_fence_signaled(struct dma_fence *fence) ++#else ++static bool mali_internal_fence_signaled(struct fence *fence) ++#endif ++{ ++ int ret; ++ struct mali_internal_sync_point *sync_pt; ++ struct mali_internal_sync_timeline *parent; ++ ++ MALI_DEBUG_ASSERT_POINTER(fence); ++ ++ sync_pt = mali_internal_fence_to_sync_pt(fence); ++ parent = mali_internal_sync_pt_to_sync_timeline(sync_pt); ++ ++ ret = parent->ops->has_signaled(sync_pt); ++ if (0 > ret) ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0) \ ++ || (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 68))) ++ fence->error = ret; ++#else ++ fence->status = ret; ++#endif ++ return ret; ++} ++ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++static bool mali_internal_fence_enable_signaling(struct dma_fence *fence) ++#else ++static bool mali_internal_fence_enable_signaling(struct fence *fence) ++#endif ++{ ++ struct mali_internal_sync_point *sync_pt; ++ struct mali_internal_sync_timeline *parent; ++ ++ MALI_DEBUG_ASSERT_POINTER(fence); ++ ++ sync_pt = mali_internal_fence_to_sync_pt(fence); ++ parent = mali_internal_sync_pt_to_sync_timeline(sync_pt); ++ ++ if (mali_internal_fence_signaled(fence)) ++ return false; ++ ++ list_add_tail(&sync_pt->sync_pt_list, &parent->sync_pt_list_head); ++ return true; ++} ++ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++static void mali_internal_fence_value_str(struct dma_fence *fence, char *str, int size) ++#else ++static void mali_internal_fence_value_str(struct fence *fence, char *str, int size) ++#endif ++{ ++ struct mali_internal_sync_point *sync_pt; ++ struct mali_internal_sync_timeline *parent; ++ ++ MALI_DEBUG_ASSERT_POINTER(fence); ++ MALI_IGNORE(str); ++ MALI_IGNORE(size); ++ ++ sync_pt = mali_internal_fence_to_sync_pt(fence); ++ parent = mali_internal_sync_pt_to_sync_timeline(sync_pt); ++ ++ parent->ops->print_sync_pt(sync_pt); ++} ++ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++static const struct dma_fence_ops fence_ops = { ++#else ++static const struct fence_ops fence_ops = { ++#endif ++ .get_driver_name = mali_internal_fence_get_driver_name, ++ .get_timeline_name = mali_internal_fence_get_timeline_name, ++ .enable_signaling = mali_internal_fence_enable_signaling, ++ .signaled = mali_internal_fence_signaled, ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++ .wait = dma_fence_default_wait, ++#else ++ .wait = fence_default_wait, ++#endif ++ .release = mali_internal_fence_release, ++ .fence_value_str = mali_internal_fence_value_str, ++}; ++#endif +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_internal_sync.h b/drivers/gpu/arm/mali400/mali/linux/mali_internal_sync.h +new file mode 100755 +index 000000000000..dbb29222ba98 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_internal_sync.h +@@ -0,0 +1,191 @@ ++/* ++ * Copyright (C) 2012-2015, 2017-2018 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file mali_internal_sync.h ++ * ++ * Mali internal structure/interface for sync. ++ */ ++ ++#ifndef _MALI_INTERNAL_SYNC_H ++#define _MALI_INTERNAL_SYNC_H ++#include ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 6, 0) ++#include ++#include ++#include ++#include ++#include ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0) ++#include ++#else ++#include ++#endif ++ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++#include ++#else ++#include ++#endif ++ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0) ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) ++#include ++#else ++#include ++#endif ++#endif ++ ++struct mali_internal_sync_timeline; ++struct mali_internal_sync_point; ++struct mali_internal_sync_fence; ++ ++struct mali_internal_sync_timeline_ops { ++ const char *driver_name; ++ int (*has_signaled)(struct mali_internal_sync_point *pt); ++ void (*free_pt)(struct mali_internal_sync_point *sync_pt); ++ void (*release_obj)(struct mali_internal_sync_timeline *sync_timeline); ++ void (*print_sync_pt)(struct mali_internal_sync_point *sync_pt); ++}; ++ ++struct mali_internal_sync_timeline { ++ struct kref kref_count; ++ const struct mali_internal_sync_timeline_ops *ops; ++ char name[32]; ++ bool destroyed; ++ int fence_context; ++ int value; ++ spinlock_t sync_pt_list_lock; ++ struct list_head sync_pt_list_head; ++}; ++ ++struct mali_internal_sync_point { ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++ struct dma_fence base; ++#else ++ struct fence base; ++#endif ++ struct list_head sync_pt_list; ++}; ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) ++struct mali_internal_sync_fence_cb { ++ struct fence_cb cb; ++ struct fence *fence; ++ struct mali_internal_sync_fence *sync_file; ++}; ++#endif ++ ++struct mali_internal_sync_fence { ++ struct file *file; ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0) ++ struct kref kref; ++#endif ++ char name[32]; ++#ifdef CONFIG_DEBUG_FS ++ struct list_head sync_file_list; ++#endif ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) ++ int num_fences; ++#endif ++ wait_queue_head_t wq; ++#if LINUX_VERSION_CODE > KERNEL_VERSION(4, 12, 0) ++ unsigned long flags; ++#endif ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) ++ atomic_t status; ++ struct mali_internal_sync_fence_cb cbs[]; ++#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) ++ struct fence *fence; ++ struct fence_cb cb; ++#else ++ struct dma_fence *fence; ++ struct dma_fence_cb cb; ++#endif ++}; ++ ++struct mali_internal_sync_fence_waiter; ++ ++typedef void (*mali_internal_sync_callback_t)(struct mali_internal_sync_fence *sync_fence, ++ struct mali_internal_sync_fence_waiter *waiter); ++ ++struct mali_internal_sync_fence_waiter { ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 13, 0) ++ wait_queue_entry_t work; ++#else ++ wait_queue_t work; ++#endif ++ mali_internal_sync_callback_t callback; ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0) ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) ++ struct fence_cb cb; ++#else ++ struct dma_fence_cb cb; ++#endif ++#endif ++}; ++ ++/** ++ * Create a mali internal sync timeline. ++ * @param ops The implementation ops for the mali internal sync timeline ++ * @param size The size to allocate ++ * @param name The sync_timeline name ++ * @return The new mali internal sync timeline if successful, NULL if not. ++ */ ++struct mali_internal_sync_timeline *mali_internal_sync_timeline_create(const struct mali_internal_sync_timeline_ops *ops, ++ int size, const char *name); ++ ++/** ++ * Destroy one mali internal sync timeline. ++ * @param sync_timeline The mali internal sync timeline to destroy. ++ */ ++void mali_internal_sync_timeline_destroy(struct mali_internal_sync_timeline *sync_timeline); ++ ++/** ++ * Signal one mali internal sync timeline. ++ * @param sync_timeline The mali internal sync timeline to signal. ++ */ ++void mali_internal_sync_timeline_signal(struct mali_internal_sync_timeline *sync_timeline); ++ ++/** ++ * Create one mali internal sync point. ++ * @param sync_timeline The mali internal sync timeline to add this mali internal sync point. ++ * @return the new mali internal sync point if successful, NULL if not. ++ */ ++struct mali_internal_sync_point *mali_internal_sync_point_create(struct mali_internal_sync_timeline *sync_timeline, int size); ++ ++/** ++ * Merge mali internal sync fences ++ * @param sync_fence1 The mali internal sync fence to merge ++ * @param sync_fence2 The mali internal sync fence to merge ++ * @return the new mali internal sync fence if successful, NULL if not. ++ */ ++struct mali_internal_sync_fence *mali_internal_sync_fence_merge(struct mali_internal_sync_fence *sync_fence1, ++ struct mali_internal_sync_fence *sync_fence2); ++ ++/** ++ * Get the mali internal sync fence from sync fd ++ * @param fd The sync handle to get the mali internal sync fence ++ * @return the mali internal sync fence if successful, NULL if not. ++ */ ++struct mali_internal_sync_fence *mali_internal_sync_fence_fdget(int fd); ++ ++ ++void mali_internal_sync_fence_waiter_init(struct mali_internal_sync_fence_waiter *waiter, ++ mali_internal_sync_callback_t callback); ++ ++int mali_internal_sync_fence_wait_async(struct mali_internal_sync_fence *sync_fence, ++ struct mali_internal_sync_fence_waiter *waiter); ++ ++int mali_internal_sync_fence_cancel_async(struct mali_internal_sync_fence *sync_fence, ++ struct mali_internal_sync_fence_waiter *waiter); ++ ++#endif /*LINUX_VERSION_CODE >= KERNEL_VERSION(4, 6, 0)*/ ++#endif /* _MALI_INTERNAL_SYNC_H */ +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_kernel_linux.c b/drivers/gpu/arm/mali400/mali/linux/mali_kernel_linux.c +new file mode 100755 +index 000000000000..e45c7d2f2b1e +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_kernel_linux.c +@@ -0,0 +1,1154 @@ ++/** ++ * Copyright (C) 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++ ++/** ++ * @file mali_kernel_linux.c ++ * Implementation of the Linux device driver entrypoints ++ */ ++#include "../platform/rk/custom_log.h" ++#include "../platform/rk/rk_ext.h" ++ ++#include /* kernel module definitions */ ++#include /* file system operations */ ++#include /* character device definitions */ ++#include /* memory manager definitions */ ++#include ++#include ++#include ++#include "mali_kernel_license.h" ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "mali_kernel_common.h" ++#include "mali_session.h" ++#include "mali_kernel_core.h" ++#include "mali_osk.h" ++#include "mali_kernel_linux.h" ++#include "mali_ukk.h" ++#include "mali_ukk_wrappers.h" ++#include "mali_kernel_sysfs.h" ++#include "mali_pm.h" ++#include "mali_kernel_license.h" ++#include "mali_memory.h" ++#include "mali_memory_dma_buf.h" ++#include "mali_memory_manager.h" ++#include "mali_memory_swap_alloc.h" ++#if defined(CONFIG_MALI400_INTERNAL_PROFILING) ++#include "mali_profiling_internal.h" ++#endif ++#if defined(CONFIG_MALI400_PROFILING) && defined(CONFIG_MALI_DVFS) ++#include "mali_osk_profiling.h" ++#include "mali_dvfs_policy.h" ++ ++static int is_first_resume = 1; ++/*Store the clk and vol for boot/insmod and mali_resume*/ ++static struct mali_gpu_clk_item mali_gpu_clk[2]; ++#endif ++ ++/* Streamline support for the Mali driver */ ++#if defined(CONFIG_TRACEPOINTS) && defined(CONFIG_MALI400_PROFILING) ++/* Ask Linux to create the tracepoints */ ++#define CREATE_TRACE_POINTS ++#include "mali_linux_trace.h" ++ ++EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_event); ++EXPORT_TRACEPOINT_SYMBOL_GPL(mali_hw_counter); ++EXPORT_TRACEPOINT_SYMBOL_GPL(mali_sw_counters); ++#endif /* CONFIG_TRACEPOINTS */ ++ ++#ifdef CONFIG_MALI_DEVFREQ ++#include "mali_devfreq.h" ++#include "mali_osk_mali.h" ++ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) ++#include ++#else ++/* In 3.13 the OPP include header file, types, and functions were all ++ * renamed. Use the old filename for the include, and define the new names to ++ * the old, when an old kernel is detected. ++ */ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0) ++#include ++#else ++#include ++#endif /* Linux >= 3.13*/ ++#define dev_pm_opp_of_add_table of_init_opp_table ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0) ++#define dev_pm_opp_of_remove_table of_free_opp_table ++#endif /* Linux >= 3.19 */ ++#endif /* Linux >= 4.4.0 */ ++#endif ++ ++/* from the __malidrv_build_info.c file that is generated during build */ ++extern const char *__malidrv_build_info(void); ++ ++/* Module parameter to control log level */ ++int mali_debug_level = 2; ++module_param(mali_debug_level, int, S_IRUSR | S_IWUSR | S_IWGRP | S_IRGRP | S_IROTH); /* rw-rw-r-- */ ++MODULE_PARM_DESC(mali_debug_level, "Higher number, more dmesg output"); ++ ++extern int mali_max_job_runtime; ++module_param(mali_max_job_runtime, int, S_IRUSR | S_IWUSR | S_IWGRP | S_IRGRP | S_IROTH); ++MODULE_PARM_DESC(mali_max_job_runtime, "Maximum allowed job runtime in msecs.\nJobs will be killed after this no matter what"); ++ ++extern int mali_l2_max_reads; ++module_param(mali_l2_max_reads, int, S_IRUSR | S_IRGRP | S_IROTH); ++MODULE_PARM_DESC(mali_l2_max_reads, "Maximum reads for Mali L2 cache"); ++ ++extern unsigned int mali_dedicated_mem_start; ++module_param(mali_dedicated_mem_start, uint, S_IRUSR | S_IRGRP | S_IROTH); ++MODULE_PARM_DESC(mali_dedicated_mem_start, "Physical start address of dedicated Mali GPU memory."); ++ ++extern unsigned int mali_dedicated_mem_size; ++module_param(mali_dedicated_mem_size, uint, S_IRUSR | S_IRGRP | S_IROTH); ++MODULE_PARM_DESC(mali_dedicated_mem_size, "Size of dedicated Mali GPU memory."); ++ ++extern unsigned int mali_shared_mem_size; ++module_param(mali_shared_mem_size, uint, S_IRUSR | S_IRGRP | S_IROTH); ++MODULE_PARM_DESC(mali_shared_mem_size, "Size of shared Mali GPU memory."); ++ ++#if defined(CONFIG_MALI400_PROFILING) ++extern int mali_boot_profiling; ++module_param(mali_boot_profiling, int, S_IRUSR | S_IRGRP | S_IROTH); ++MODULE_PARM_DESC(mali_boot_profiling, "Start profiling as a part of Mali driver initialization"); ++#endif ++ ++extern int mali_max_pp_cores_group_1; ++module_param(mali_max_pp_cores_group_1, int, S_IRUSR | S_IRGRP | S_IROTH); ++MODULE_PARM_DESC(mali_max_pp_cores_group_1, "Limit the number of PP cores to use from first PP group."); ++ ++extern int mali_max_pp_cores_group_2; ++module_param(mali_max_pp_cores_group_2, int, S_IRUSR | S_IRGRP | S_IROTH); ++MODULE_PARM_DESC(mali_max_pp_cores_group_2, "Limit the number of PP cores to use from second PP group (Mali-450 only)."); ++ ++extern unsigned int mali_mem_swap_out_threshold_value; ++module_param(mali_mem_swap_out_threshold_value, uint, S_IRUSR | S_IRGRP | S_IROTH); ++MODULE_PARM_DESC(mali_mem_swap_out_threshold_value, "Threshold value used to limit how much swappable memory cached in Mali driver."); ++ ++#if defined(CONFIG_MALI_DVFS) ++/** the max fps the same as display vsync default 60, can set by module insert parameter */ ++extern int mali_max_system_fps; ++module_param(mali_max_system_fps, int, S_IRUSR | S_IWUSR | S_IWGRP | S_IRGRP | S_IROTH); ++MODULE_PARM_DESC(mali_max_system_fps, "Max system fps the same as display VSYNC."); ++ ++/** a lower limit on their desired FPS default 58, can set by module insert parameter*/ ++extern int mali_desired_fps; ++module_param(mali_desired_fps, int, S_IRUSR | S_IWUSR | S_IWGRP | S_IRGRP | S_IROTH); ++MODULE_PARM_DESC(mali_desired_fps, "A bit lower than max_system_fps which user desired fps"); ++#endif ++ ++#if MALI_ENABLE_CPU_CYCLES ++#include ++#include ++#include ++static struct timer_list mali_init_cpu_clock_timers[8]; ++static u32 mali_cpu_clock_last_value[8] = {0,}; ++#endif ++ ++/* Export symbols from common code: mali_user_settings.c */ ++#include "mali_user_settings_db.h" ++EXPORT_SYMBOL(mali_set_user_setting); ++EXPORT_SYMBOL(mali_get_user_setting); ++ ++static char mali_dev_name[] = "mali"; /* should be const, but the functions we call requires non-cost */ ++ ++/* This driver only supports one Mali device, and this variable stores this single platform device */ ++struct platform_device *mali_platform_device = NULL; ++ ++/* This driver only supports one Mali device, and this variable stores the exposed misc device (/dev/mali) */ ++static struct miscdevice mali_miscdevice = { 0, }; ++ ++static int mali_miscdevice_register(struct platform_device *pdev); ++static void mali_miscdevice_unregister(void); ++ ++static int mali_open(struct inode *inode, struct file *filp); ++static int mali_release(struct inode *inode, struct file *filp); ++static long mali_ioctl(struct file *filp, unsigned int cmd, unsigned long arg); ++ ++static int mali_probe(struct platform_device *pdev); ++static int mali_remove(struct platform_device *pdev); ++ ++static int mali_driver_suspend_scheduler(struct device *dev); ++static int mali_driver_resume_scheduler(struct device *dev); ++ ++#ifdef CONFIG_PM_RUNTIME ++static int mali_driver_runtime_suspend(struct device *dev); ++static int mali_driver_runtime_resume(struct device *dev); ++static int mali_driver_runtime_idle(struct device *dev); ++#endif ++ ++#if defined(MALI_FAKE_PLATFORM_DEVICE) ++#if defined(CONFIG_MALI_DT) ++extern int mali_platform_device_init(struct platform_device *device); ++extern int mali_platform_device_deinit(struct platform_device *device); ++#else ++extern int mali_platform_device_register(void); ++extern int mali_platform_device_unregister(void); ++#endif ++#endif ++ ++extern int rk_platform_init_opp_table(struct device *dev); ++ ++/* Linux power management operations provided by the Mali device driver */ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 29)) ++struct pm_ext_ops mali_dev_ext_pm_ops = { ++ .base = ++ { ++ .suspend = mali_driver_suspend_scheduler, ++ .resume = mali_driver_resume_scheduler, ++ .freeze = mali_driver_suspend_scheduler, ++ .thaw = mali_driver_resume_scheduler, ++ }, ++}; ++#else ++static const struct dev_pm_ops mali_dev_pm_ops = { ++#ifdef CONFIG_PM_RUNTIME ++ .runtime_suspend = mali_driver_runtime_suspend, ++ .runtime_resume = mali_driver_runtime_resume, ++ .runtime_idle = mali_driver_runtime_idle, ++#endif ++ .suspend = mali_driver_suspend_scheduler, ++ .resume = mali_driver_resume_scheduler, ++ .freeze = mali_driver_suspend_scheduler, ++ .thaw = mali_driver_resume_scheduler, ++ .poweroff = mali_driver_suspend_scheduler, ++}; ++#endif ++ ++#ifdef CONFIG_MALI_DT ++static struct of_device_id base_dt_ids[] = { ++ {.compatible = "arm,mali-300"}, ++ /*-------------------------------------------------------*/ ++ /* rk_ext : to use dts_for_mali_ko_befor_r5p0-01rel0. */ ++ // {.compatible = "arm,mali-400"}, ++ {.compatible = "arm,mali400"}, ++ /*-------------------------------------------------------*/ ++ {.compatible = "arm,mali-450"}, ++ {.compatible = "arm,mali-470"}, ++ {}, ++}; ++ ++MODULE_DEVICE_TABLE(of, base_dt_ids); ++#endif ++ ++/* The Mali device driver struct */ ++static struct platform_driver mali_platform_driver = { ++ .probe = mali_probe, ++ .remove = mali_remove, ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 29)) ++ .pm = &mali_dev_ext_pm_ops, ++#endif ++ .driver = ++ { ++ .name = MALI_GPU_NAME_UTGARD, ++ .owner = THIS_MODULE, ++ .bus = &platform_bus_type, ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29)) ++ .pm = &mali_dev_pm_ops, ++#endif ++#ifdef CONFIG_MALI_DT ++ .of_match_table = of_match_ptr(base_dt_ids), ++#endif ++ }, ++}; ++ ++/* Linux misc device operations (/dev/mali) */ ++struct file_operations mali_fops = { ++ .owner = THIS_MODULE, ++ .open = mali_open, ++ .release = mali_release, ++ .unlocked_ioctl = mali_ioctl, ++ .compat_ioctl = mali_ioctl, ++ .mmap = mali_mmap ++}; ++ ++#if MALI_ENABLE_CPU_CYCLES ++void mali_init_cpu_time_counters(int reset, int enable_divide_by_64) ++{ ++ /* The CPU assembly reference used is: ARM Architecture Reference Manual ARMv7-AR C.b */ ++ u32 write_value; ++ ++ /* See B4.1.116 PMCNTENSET, Performance Monitors Count Enable Set register, VMSA */ ++ /* setting p15 c9 c12 1 to 0x8000000f==CPU_CYCLE_ENABLE |EVENT_3_ENABLE|EVENT_2_ENABLE|EVENT_1_ENABLE|EVENT_0_ENABLE */ ++ asm volatile("mcr p15, 0, %0, c9, c12, 1" :: "r"(0x8000000f)); ++ ++ ++ /* See B4.1.117 PMCR, Performance Monitors Control Register. Writing to p15, c9, c12, 0 */ ++ write_value = 1 << 0; /* Bit 0 set. Enable counters */ ++ if (reset) { ++ write_value |= 1 << 1; /* Reset event counters */ ++ write_value |= 1 << 2; /* Reset cycle counter */ ++ } ++ if (enable_divide_by_64) { ++ write_value |= 1 << 3; /* Enable the Clock divider by 64 */ ++ } ++ write_value |= 1 << 4; /* Export enable. Not needed */ ++ asm volatile("MCR p15, 0, %0, c9, c12, 0\t\n" :: "r"(write_value)); ++ ++ /* PMOVSR Overflow Flag Status Register - Clear Clock and Event overflows */ ++ asm volatile("MCR p15, 0, %0, c9, c12, 3\t\n" :: "r"(0x8000000f)); ++ ++ ++ /* See B4.1.124 PMUSERENR - setting p15 c9 c14 to 1" */ ++ /* User mode access to the Performance Monitors enabled. */ ++ /* Lets User space read cpu clock cycles */ ++ asm volatile("mcr p15, 0, %0, c9, c14, 0" :: "r"(1)); ++} ++ ++/** A timer function that configures the cycle clock counter on current CPU. ++ * The function \a mali_init_cpu_time_counters_on_all_cpus sets up this ++ * function to trigger on all Cpus during module load. ++ */ ++static void mali_init_cpu_clock_timer_func(unsigned long data) ++{ ++ int reset_counters, enable_divide_clock_counter_by_64; ++ int current_cpu = raw_smp_processor_id(); ++ unsigned int sample0; ++ unsigned int sample1; ++ ++ MALI_IGNORE(data); ++ ++ reset_counters = 1; ++ enable_divide_clock_counter_by_64 = 0; ++ mali_init_cpu_time_counters(reset_counters, enable_divide_clock_counter_by_64); ++ ++ sample0 = mali_get_cpu_cyclecount(); ++ sample1 = mali_get_cpu_cyclecount(); ++ ++ MALI_DEBUG_PRINT(3, ("Init Cpu %d cycle counter- First two samples: %08x %08x \n", current_cpu, sample0, sample1)); ++} ++ ++/** A timer functions for storing current time on all cpus. ++ * Used for checking if the clocks have similar values or if they are drifting. ++ */ ++static void mali_print_cpu_clock_timer_func(unsigned long data) ++{ ++ int current_cpu = raw_smp_processor_id(); ++ unsigned int sample0; ++ ++ MALI_IGNORE(data); ++ sample0 = mali_get_cpu_cyclecount(); ++ if (current_cpu < 8) { ++ mali_cpu_clock_last_value[current_cpu] = sample0; ++ } ++} ++ ++/** Init the performance registers on all CPUs to count clock cycles. ++ * For init \a print_only should be 0. ++ * If \a print_only is 1, it will intead print the current clock value of all CPUs. ++ */ ++void mali_init_cpu_time_counters_on_all_cpus(int print_only) ++{ ++ int i = 0; ++ int cpu_number; ++ int jiffies_trigger; ++ int jiffies_wait; ++ ++ jiffies_wait = 2; ++ jiffies_trigger = jiffies + jiffies_wait; ++ ++ for (i = 0 ; i < 8 ; i++) { ++ init_timer(&mali_init_cpu_clock_timers[i]); ++ if (print_only) mali_init_cpu_clock_timers[i].function = mali_print_cpu_clock_timer_func; ++ else mali_init_cpu_clock_timers[i].function = mali_init_cpu_clock_timer_func; ++ mali_init_cpu_clock_timers[i].expires = jiffies_trigger ; ++ } ++ cpu_number = cpumask_first(cpu_online_mask); ++ for (i = 0 ; i < 8 ; i++) { ++ int next_cpu; ++ add_timer_on(&mali_init_cpu_clock_timers[i], cpu_number); ++ next_cpu = cpumask_next(cpu_number, cpu_online_mask); ++ if (next_cpu >= nr_cpu_ids) break; ++ cpu_number = next_cpu; ++ } ++ ++ while (jiffies_wait) jiffies_wait = schedule_timeout_uninterruptible(jiffies_wait); ++ ++ for (i = 0 ; i < 8 ; i++) { ++ del_timer_sync(&mali_init_cpu_clock_timers[i]); ++ } ++ ++ if (print_only) { ++ if ((0 == mali_cpu_clock_last_value[2]) && (0 == mali_cpu_clock_last_value[3])) { ++ /* Diff can be printed if we want to check if the clocks are in sync ++ int diff = mali_cpu_clock_last_value[0] - mali_cpu_clock_last_value[1];*/ ++ MALI_DEBUG_PRINT(2, ("CPU cycle counters readout all: %08x %08x\n", mali_cpu_clock_last_value[0], mali_cpu_clock_last_value[1])); ++ } else { ++ MALI_DEBUG_PRINT(2, ("CPU cycle counters readout all: %08x %08x %08x %08x\n", mali_cpu_clock_last_value[0], mali_cpu_clock_last_value[1], mali_cpu_clock_last_value[2], mali_cpu_clock_last_value[3])); ++ } ++ } ++} ++#endif ++ ++int mali_module_init(void) ++{ ++ int err = 0; ++ ++ MALI_DEBUG_PRINT(2, ("Inserting Mali v%d device driver. \n", _MALI_API_VERSION)); ++ MALI_DEBUG_PRINT(2, ("Compiled: %s, time: %s.\n", __DATE__, __TIME__)); ++ MALI_DEBUG_PRINT(2, ("Driver revision: %s\n", SVN_REV_STRING)); ++ ++ I("svn_rev_string_from_arm of this mali_ko is '%s', rk_ko_ver is '%d', built at '%s', on '%s'.", ++ SVN_REV_STRING, ++ RK_KO_VER, ++ __TIME__, ++ __DATE__); ++ ++#if MALI_ENABLE_CPU_CYCLES ++ mali_init_cpu_time_counters_on_all_cpus(0); ++ MALI_DEBUG_PRINT(2, ("CPU cycle counter setup complete\n")); ++ /* Printing the current cpu counters */ ++ mali_init_cpu_time_counters_on_all_cpus(1); ++#endif ++ ++ /* Initialize module wide settings */ ++#ifdef MALI_FAKE_PLATFORM_DEVICE ++#ifndef CONFIG_MALI_DT ++ MALI_DEBUG_PRINT(2, ("mali_module_init() registering device\n")); ++ err = mali_platform_device_register(); ++ if (0 != err) { ++ return err; ++ } ++#endif ++#endif ++ ++ MALI_DEBUG_PRINT(2, ("mali_module_init() registering driver\n")); ++ ++ err = platform_driver_register(&mali_platform_driver); ++ ++ if (0 != err) { ++ MALI_DEBUG_PRINT(2, ("mali_module_init() Failed to register driver (%d)\n", err)); ++#ifdef MALI_FAKE_PLATFORM_DEVICE ++#ifndef CONFIG_MALI_DT ++ mali_platform_device_unregister(); ++#endif ++#endif ++ mali_platform_device = NULL; ++ return err; ++ } ++ ++#if defined(CONFIG_MALI400_INTERNAL_PROFILING) ++ err = _mali_internal_profiling_init(mali_boot_profiling ? MALI_TRUE : MALI_FALSE); ++ if (0 != err) { ++ /* No biggie if we wheren't able to initialize the profiling */ ++ MALI_PRINT_ERROR(("Failed to initialize profiling, feature will be unavailable\n")); ++ } ++#endif ++ ++ /* Tracing the current frequency and voltage from boot/insmod*/ ++#if defined(CONFIG_MALI400_PROFILING) && defined(CONFIG_MALI_DVFS) ++ /* Just call mali_get_current_gpu_clk_item(),to record current clk info.*/ ++ mali_get_current_gpu_clk_item(&mali_gpu_clk[0]); ++ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE | ++ MALI_PROFILING_EVENT_CHANNEL_GPU | ++ MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE, ++ mali_gpu_clk[0].clock, ++ mali_gpu_clk[0].vol / 1000, ++ 0, 0, 0); ++#endif ++ ++ MALI_PRINT(("Mali device driver loaded\n")); ++ ++ return 0; /* Success */ ++} ++ ++void mali_module_exit(void) ++{ ++ MALI_DEBUG_PRINT(2, ("Unloading Mali v%d device driver.\n", _MALI_API_VERSION)); ++ ++ MALI_DEBUG_PRINT(2, ("mali_module_exit() unregistering driver\n")); ++ ++ platform_driver_unregister(&mali_platform_driver); ++ ++#if defined(MALI_FAKE_PLATFORM_DEVICE) ++#ifndef CONFIG_MALI_DT ++ MALI_DEBUG_PRINT(2, ("mali_module_exit() unregistering device\n")); ++ mali_platform_device_unregister(); ++#endif ++#endif ++ ++ /* Tracing the current frequency and voltage from rmmod*/ ++ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE | ++ MALI_PROFILING_EVENT_CHANNEL_GPU | ++ MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE, ++ 0, ++ 0, ++ 0, 0, 0); ++ ++#if defined(CONFIG_MALI400_INTERNAL_PROFILING) ++ _mali_internal_profiling_term(); ++#endif ++ ++ MALI_PRINT(("Mali device driver unloaded\n")); ++} ++ ++#ifdef CONFIG_MALI_DEVFREQ ++struct mali_device *mali_device_alloc(void) ++{ ++ return kzalloc(sizeof(struct mali_device), GFP_KERNEL); ++} ++ ++void mali_device_free(struct mali_device *mdev) ++{ ++ kfree(mdev); ++} ++#endif ++ ++static int mali_probe(struct platform_device *pdev) ++{ ++ int err; ++#ifdef CONFIG_MALI_DEVFREQ ++ struct mali_device *mdev; ++#endif ++ ++ MALI_DEBUG_PRINT(2, ("mali_probe(): Called for platform device %s\n", pdev->name)); ++ ++ if (NULL != mali_platform_device) { ++ /* Already connected to a device, return error */ ++ MALI_PRINT_ERROR(("mali_probe(): The Mali driver is already connected with a Mali device.")); ++ return -EEXIST; ++ } ++ ++ mali_platform_device = pdev; ++ ++ dev_info(&pdev->dev, "mali_platform_device->num_resources = %d\n", ++ mali_platform_device->num_resources); ++ ++ { ++ int i = 0; ++ ++ for(i = 0; i < mali_platform_device->num_resources; i++) ++ dev_info(&pdev->dev, ++ "resource[%d].start = 0x%pa\n", ++ i, ++ &mali_platform_device->resource[i].start); ++ } ++ ++#ifdef CONFIG_MALI_DT ++ /* If we use DT to initialize our DDK, we have to prepare somethings. */ ++ err = mali_platform_device_init(mali_platform_device); ++ if (0 != err) { ++ MALI_PRINT_ERROR(("mali_probe(): Failed to initialize platform device.")); ++ mali_platform_device = NULL; ++ return -EFAULT; ++ } ++#endif ++ ++#ifdef CONFIG_MALI_DEVFREQ ++ mdev = mali_device_alloc(); ++ if (!mdev) { ++ MALI_PRINT_ERROR(("Can't allocate mali device private data\n")); ++ return -ENOMEM; ++ } ++ ++ mdev->dev = &pdev->dev; ++ dev_set_drvdata(mdev->dev, mdev); ++ ++ /*Initilization clock and regulator*/ ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0)) && defined(CONFIG_OF) \ ++ && defined(CONFIG_REGULATOR) ++ mdev->regulator = regulator_get_optional(mdev->dev, "mali"); ++ if (IS_ERR_OR_NULL(mdev->regulator)) { ++ MALI_DEBUG_PRINT(2, ("Continuing without Mali regulator control\n")); ++ mdev->regulator = NULL; ++ /* Allow probe to continue without regulator */ ++ } ++#endif /* LINUX_VERSION_CODE >= 3, 12, 0 */ ++ ++ err = rk_platform_init_opp_table(mdev->dev); ++ if (err) ++ MALI_DEBUG_PRINT(3, ("Failed to init_opp_table\n")); ++ ++ /* Need to name the gpu clock "clk_mali" in the device tree */ ++ mdev->clock = clk_get(mdev->dev, "clk_mali"); ++ if (IS_ERR_OR_NULL(mdev->clock)) { ++ MALI_DEBUG_PRINT(2, ("Continuing without Mali clock control\n")); ++ mdev->clock = NULL; ++ /* Allow probe to continue without clock. */ ++ } else { ++ err = clk_prepare(mdev->clock); ++ if (err) { ++ MALI_PRINT_ERROR(("Failed to prepare clock (%d)\n", err)); ++ goto clock_prepare_failed; ++ } ++ } ++ ++ /* initilize pm metrics related */ ++ if (mali_pm_metrics_init(mdev) < 0) { ++ MALI_DEBUG_PRINT(2, ("mali pm metrics init failed\n")); ++ goto pm_metrics_init_failed; ++ } ++ ++ if (mali_devfreq_init(mdev) < 0) { ++ MALI_DEBUG_PRINT(2, ("mali devfreq init failed\n")); ++ goto devfreq_init_failed; ++ } ++#endif ++ ++ ++ if (_MALI_OSK_ERR_OK == _mali_osk_wq_init()) { ++ /* Initialize the Mali GPU HW specified by pdev */ ++ if (_MALI_OSK_ERR_OK == mali_initialize_subsystems()) { ++ /* Register a misc device (so we are accessible from user space) */ ++ err = mali_miscdevice_register(pdev); ++ if (0 == err) { ++ /* Setup sysfs entries */ ++ err = mali_sysfs_register(mali_dev_name); ++ ++ if (0 == err) { ++ MALI_DEBUG_PRINT(2, ("mali_probe(): Successfully initialized driver for platform device %s\n", pdev->name)); ++ ++ return 0; ++ } else { ++ MALI_PRINT_ERROR(("mali_probe(): failed to register sysfs entries")); ++ } ++ mali_miscdevice_unregister(); ++ } else { ++ MALI_PRINT_ERROR(("mali_probe(): failed to register Mali misc device.")); ++ } ++ mali_terminate_subsystems(); ++ } else { ++ MALI_PRINT_ERROR(("mali_probe(): Failed to initialize Mali device driver.")); ++ } ++ _mali_osk_wq_term(); ++ } ++ ++#ifdef CONFIG_MALI_DEVFREQ ++ mali_devfreq_term(mdev); ++devfreq_init_failed: ++ mali_pm_metrics_term(mdev); ++pm_metrics_init_failed: ++ clk_unprepare(mdev->clock); ++clock_prepare_failed: ++ clk_put(mdev->clock); ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0)) && defined(CONFIG_OF) \ ++ && defined(CONFIG_PM_OPP) ++ dev_pm_opp_of_remove_table(mdev->dev); ++#endif ++ ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0)) && defined(CONFIG_OF) \ ++ && defined(CONFIG_REGULATOR) ++ regulator_put(mdev->regulator); ++#endif /* LINUX_VERSION_CODE >= 3, 12, 0 */ ++ mali_device_free(mdev); ++#endif ++ ++#ifdef CONFIG_MALI_DT ++ mali_platform_device_deinit(mali_platform_device); ++#endif ++ mali_platform_device = NULL; ++ return -EFAULT; ++} ++ ++static int mali_remove(struct platform_device *pdev) ++{ ++#ifdef CONFIG_MALI_DEVFREQ ++ struct mali_device *mdev = dev_get_drvdata(&pdev->dev); ++#endif ++ ++ MALI_DEBUG_PRINT(2, ("mali_remove() called for platform device %s\n", pdev->name)); ++ mali_sysfs_unregister(); ++ mali_miscdevice_unregister(); ++ mali_terminate_subsystems(); ++ _mali_osk_wq_term(); ++ ++#ifdef CONFIG_MALI_DEVFREQ ++ mali_devfreq_term(mdev); ++ ++ mali_pm_metrics_term(mdev); ++ ++ if (mdev->clock) { ++ clk_unprepare(mdev->clock); ++ clk_put(mdev->clock); ++ mdev->clock = NULL; ++ } ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0)) && defined(CONFIG_OF) \ ++ && defined(CONFIG_PM_OPP) ++ dev_pm_opp_of_remove_table(mdev->dev); ++#endif ++ ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0)) && defined(CONFIG_OF) \ ++ && defined(CONFIG_REGULATOR) ++ regulator_put(mdev->regulator); ++#endif /* LINUX_VERSION_CODE >= 3, 12, 0 */ ++ mali_device_free(mdev); ++#endif ++ ++#ifdef CONFIG_MALI_DT ++ mali_platform_device_deinit(mali_platform_device); ++#endif ++ mali_platform_device = NULL; ++ return 0; ++} ++ ++static int mali_miscdevice_register(struct platform_device *pdev) ++{ ++ int err; ++ ++ mali_miscdevice.minor = MISC_DYNAMIC_MINOR; ++ mali_miscdevice.name = mali_dev_name; ++ mali_miscdevice.fops = &mali_fops; ++ mali_miscdevice.parent = get_device(&pdev->dev); ++ ++ err = misc_register(&mali_miscdevice); ++ if (0 != err) { ++ MALI_PRINT_ERROR(("Failed to register misc device, misc_register() returned %d\n", err)); ++ } ++ ++ return err; ++} ++ ++static void mali_miscdevice_unregister(void) ++{ ++ misc_deregister(&mali_miscdevice); ++} ++ ++static int mali_driver_suspend_scheduler(struct device *dev) ++{ ++#ifdef CONFIG_MALI_DEVFREQ ++ struct mali_device *mdev = dev_get_drvdata(dev); ++ if (!mdev) ++ return -ENODEV; ++#endif ++ ++#if defined(CONFIG_MALI_DEVFREQ) && \ ++ (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) ++ devfreq_suspend_device(mdev->devfreq); ++#endif ++ ++ mali_pm_os_suspend(MALI_TRUE); ++ /* Tracing the frequency and voltage after mali is suspended */ ++ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE | ++ MALI_PROFILING_EVENT_CHANNEL_GPU | ++ MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE, ++ 0, ++ 0, ++ 0, 0, 0); ++ return 0; ++} ++ ++static int mali_driver_resume_scheduler(struct device *dev) ++{ ++#ifdef CONFIG_MALI_DEVFREQ ++ struct mali_device *mdev = dev_get_drvdata(dev); ++ if (!mdev) ++ return -ENODEV; ++#endif ++ ++ /* Tracing the frequency and voltage after mali is resumed */ ++#if defined(CONFIG_MALI400_PROFILING) && defined(CONFIG_MALI_DVFS) ++ /* Just call mali_get_current_gpu_clk_item() once,to record current clk info.*/ ++ if (is_first_resume == 1) { ++ mali_get_current_gpu_clk_item(&mali_gpu_clk[1]); ++ is_first_resume = 0; ++ } ++ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE | ++ MALI_PROFILING_EVENT_CHANNEL_GPU | ++ MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE, ++ mali_gpu_clk[1].clock, ++ mali_gpu_clk[1].vol / 1000, ++ 0, 0, 0); ++#endif ++ mali_pm_os_resume(); ++ ++#if defined(CONFIG_MALI_DEVFREQ) && \ ++ (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) ++ devfreq_resume_device(mdev->devfreq); ++#endif ++ ++ return 0; ++} ++ ++#ifdef CONFIG_PM_RUNTIME ++static int mali_driver_runtime_suspend(struct device *dev) ++{ ++#ifdef CONFIG_MALI_DEVFREQ ++ struct mali_device *mdev = dev_get_drvdata(dev); ++ if (!mdev) ++ return -ENODEV; ++#endif ++ ++ if (MALI_TRUE == mali_pm_runtime_suspend()) { ++ /* Tracing the frequency and voltage after mali is suspended */ ++ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE | ++ MALI_PROFILING_EVENT_CHANNEL_GPU | ++ MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE, ++ 0, ++ 0, ++ 0, 0, 0); ++ ++#if defined(CONFIG_MALI_DEVFREQ) && \ ++ (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) ++ MALI_DEBUG_PRINT(4, ("devfreq_suspend_device: stop devfreq monitor\n")); ++ devfreq_suspend_device(mdev->devfreq); ++#endif ++ ++ return 0; ++ } else { ++ return -EBUSY; ++ } ++} ++ ++static int mali_driver_runtime_resume(struct device *dev) ++{ ++#ifdef CONFIG_MALI_DEVFREQ ++ struct mali_device *mdev = dev_get_drvdata(dev); ++ if (!mdev) ++ return -ENODEV; ++#endif ++ ++ /* Tracing the frequency and voltage after mali is resumed */ ++#if defined(CONFIG_MALI400_PROFILING) && defined(CONFIG_MALI_DVFS) ++ /* Just call mali_get_current_gpu_clk_item() once,to record current clk info.*/ ++ if (is_first_resume == 1) { ++ mali_get_current_gpu_clk_item(&mali_gpu_clk[1]); ++ is_first_resume = 0; ++ } ++ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE | ++ MALI_PROFILING_EVENT_CHANNEL_GPU | ++ MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE, ++ mali_gpu_clk[1].clock, ++ mali_gpu_clk[1].vol / 1000, ++ 0, 0, 0); ++#endif ++ ++ mali_pm_runtime_resume(); ++ ++#if defined(CONFIG_MALI_DEVFREQ) && \ ++ (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) ++ MALI_DEBUG_PRINT(4, ("devfreq_resume_device: start devfreq monitor\n")); ++ devfreq_resume_device(mdev->devfreq); ++#endif ++ return 0; ++} ++ ++static int mali_driver_runtime_idle(struct device *dev) ++{ ++ /* Nothing to do */ ++ return 0; ++} ++#endif ++ ++static int mali_open(struct inode *inode, struct file *filp) ++{ ++ struct mali_session_data *session_data; ++ _mali_osk_errcode_t err; ++ ++ /* input validation */ ++ if (mali_miscdevice.minor != iminor(inode)) { ++ MALI_PRINT_ERROR(("mali_open() Minor does not match\n")); ++ return -ENODEV; ++ } ++ ++ /* allocated struct to track this session */ ++ err = _mali_ukk_open((void **)&session_data); ++ if (_MALI_OSK_ERR_OK != err) return map_errcode(err); ++ ++ /* initialize file pointer */ ++ filp->f_pos = 0; ++ ++ /* link in our session data */ ++ filp->private_data = (void *)session_data; ++ ++ filp->f_mapping = mali_mem_swap_get_global_swap_file()->f_mapping; ++ ++ return 0; ++} ++ ++static int mali_release(struct inode *inode, struct file *filp) ++{ ++ _mali_osk_errcode_t err; ++ ++ /* input validation */ ++ if (mali_miscdevice.minor != iminor(inode)) { ++ MALI_PRINT_ERROR(("mali_release() Minor does not match\n")); ++ return -ENODEV; ++ } ++ ++ err = _mali_ukk_close((void **)&filp->private_data); ++ if (_MALI_OSK_ERR_OK != err) return map_errcode(err); ++ ++ return 0; ++} ++ ++int map_errcode(_mali_osk_errcode_t err) ++{ ++ switch (err) { ++ case _MALI_OSK_ERR_OK : ++ return 0; ++ case _MALI_OSK_ERR_FAULT: ++ return -EFAULT; ++ case _MALI_OSK_ERR_INVALID_FUNC: ++ return -ENOTTY; ++ case _MALI_OSK_ERR_INVALID_ARGS: ++ return -EINVAL; ++ case _MALI_OSK_ERR_NOMEM: ++ return -ENOMEM; ++ case _MALI_OSK_ERR_TIMEOUT: ++ return -ETIMEDOUT; ++ case _MALI_OSK_ERR_RESTARTSYSCALL: ++ return -ERESTARTSYS; ++ case _MALI_OSK_ERR_ITEM_NOT_FOUND: ++ return -ENOENT; ++ default: ++ return -EFAULT; ++ } ++} ++ ++static long mali_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) ++{ ++ int err; ++ struct mali_session_data *session_data; ++ ++ MALI_DEBUG_PRINT(7, ("Ioctl received 0x%08X 0x%08lX\n", cmd, arg)); ++ ++ session_data = (struct mali_session_data *)filp->private_data; ++ if (NULL == session_data) { ++ MALI_DEBUG_PRINT(7, ("filp->private_data was NULL\n")); ++ return -ENOTTY; ++ } ++ ++ if (NULL == (void *)arg) { ++ MALI_DEBUG_PRINT(7, ("arg was NULL\n")); ++ return -ENOTTY; ++ } ++ ++ switch (cmd) { ++ case MALI_IOC_WAIT_FOR_NOTIFICATION: ++ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_wait_for_notification_s), sizeof(u64))); ++ err = wait_for_notification_wrapper(session_data, (_mali_uk_wait_for_notification_s __user *)arg); ++ break; ++ ++ case MALI_IOC_GET_API_VERSION_V2: ++ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_api_version_v2_s), sizeof(u64))); ++ err = get_api_version_v2_wrapper(session_data, (_mali_uk_get_api_version_v2_s __user *)arg); ++ break; ++ ++ case MALI_IOC_GET_API_VERSION: ++ err = get_api_version_wrapper(session_data, (_mali_uk_get_api_version_s __user *)arg); ++ break; ++ ++ case MALI_IOC_POST_NOTIFICATION: ++ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_post_notification_s), sizeof(u64))); ++ err = post_notification_wrapper(session_data, (_mali_uk_post_notification_s __user *)arg); ++ break; ++ ++ /* rk_ext : 从对 r5p0-01rel0 集æˆå¼€å§‹, ä¸å†ä½¿ç”¨. */ ++#if 0 ++ case MALI_IOC_GET_MALI_VERSION_IN_RK30: ++ err = get_mali_version_in_rk30_wrapper(session_data, (_mali_uk_get_mali_version_in_rk30_s __user *)arg); ++ break; ++#else ++ case MALI_IOC_GET_RK_KO_VERSION: ++ err = get_rk_ko_version_wrapper(session_data, (_mali_rk_ko_version_s __user *)arg); ++ break; ++#endif ++ ++ case MALI_IOC_GET_USER_SETTINGS: ++ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_user_settings_s), sizeof(u64))); ++ err = get_user_settings_wrapper(session_data, (_mali_uk_get_user_settings_s __user *)arg); ++ break; ++ ++ case MALI_IOC_REQUEST_HIGH_PRIORITY: ++ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_request_high_priority_s), sizeof(u64))); ++ err = request_high_priority_wrapper(session_data, (_mali_uk_request_high_priority_s __user *)arg); ++ break; ++ ++ case MALI_IOC_PENDING_SUBMIT: ++ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_pending_submit_s), sizeof(u64))); ++ err = pending_submit_wrapper(session_data, (_mali_uk_pending_submit_s __user *)arg); ++ break; ++ ++#if defined(CONFIG_MALI400_PROFILING) ++ case MALI_IOC_PROFILING_ADD_EVENT: ++ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_profiling_add_event_s), sizeof(u64))); ++ err = profiling_add_event_wrapper(session_data, (_mali_uk_profiling_add_event_s __user *)arg); ++ break; ++ ++ case MALI_IOC_PROFILING_REPORT_SW_COUNTERS: ++ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_sw_counters_report_s), sizeof(u64))); ++ err = profiling_report_sw_counters_wrapper(session_data, (_mali_uk_sw_counters_report_s __user *)arg); ++ break; ++ ++ case MALI_IOC_PROFILING_STREAM_FD_GET: ++ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_profiling_stream_fd_get_s), sizeof(u64))); ++ err = profiling_get_stream_fd_wrapper(session_data, (_mali_uk_profiling_stream_fd_get_s __user *)arg); ++ break; ++ ++ case MALI_IOC_PROILING_CONTROL_SET: ++ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_profiling_control_set_s), sizeof(u64))); ++ err = profiling_control_set_wrapper(session_data, (_mali_uk_profiling_control_set_s __user *)arg); ++ break; ++#else ++ ++ case MALI_IOC_PROFILING_ADD_EVENT: /* FALL-THROUGH */ ++ case MALI_IOC_PROFILING_REPORT_SW_COUNTERS: /* FALL-THROUGH */ ++ MALI_DEBUG_PRINT(2, ("Profiling not supported\n")); ++ err = -ENOTTY; ++ break; ++#endif ++ ++ case MALI_IOC_PROFILING_MEMORY_USAGE_GET: ++ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_profiling_memory_usage_get_s), sizeof(u64))); ++ err = mem_usage_get_wrapper(session_data, (_mali_uk_profiling_memory_usage_get_s __user *)arg); ++ break; ++ ++ case MALI_IOC_MEM_ALLOC: ++ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_alloc_mem_s), sizeof(u64))); ++ err = mem_alloc_wrapper(session_data, (_mali_uk_alloc_mem_s __user *)arg); ++ break; ++ ++ case MALI_IOC_MEM_FREE: ++ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_free_mem_s), sizeof(u64))); ++ err = mem_free_wrapper(session_data, (_mali_uk_free_mem_s __user *)arg); ++ break; ++ ++ case MALI_IOC_MEM_BIND: ++ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_bind_mem_s), sizeof(u64))); ++ err = mem_bind_wrapper(session_data, (_mali_uk_bind_mem_s __user *)arg); ++ break; ++ ++ case MALI_IOC_MEM_UNBIND: ++ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_unbind_mem_s), sizeof(u64))); ++ err = mem_unbind_wrapper(session_data, (_mali_uk_unbind_mem_s __user *)arg); ++ break; ++ ++ case MALI_IOC_MEM_COW: ++ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_cow_mem_s), sizeof(u64))); ++ err = mem_cow_wrapper(session_data, (_mali_uk_cow_mem_s __user *)arg); ++ break; ++ ++ case MALI_IOC_MEM_COW_MODIFY_RANGE: ++ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_cow_modify_range_s), sizeof(u64))); ++ err = mem_cow_modify_range_wrapper(session_data, (_mali_uk_cow_modify_range_s __user *)arg); ++ break; ++ ++ case MALI_IOC_MEM_RESIZE: ++ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_mem_resize_s), sizeof(u64))); ++ err = mem_resize_mem_wrapper(session_data, (_mali_uk_mem_resize_s __user *)arg); ++ break; ++ ++ case MALI_IOC_MEM_WRITE_SAFE: ++ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_mem_write_safe_s), sizeof(u64))); ++ err = mem_write_safe_wrapper(session_data, (_mali_uk_mem_write_safe_s __user *)arg); ++ break; ++ ++ case MALI_IOC_MEM_QUERY_MMU_PAGE_TABLE_DUMP_SIZE: ++ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_query_mmu_page_table_dump_size_s), sizeof(u64))); ++ err = mem_query_mmu_page_table_dump_size_wrapper(session_data, (_mali_uk_query_mmu_page_table_dump_size_s __user *)arg); ++ break; ++ ++ case MALI_IOC_MEM_DUMP_MMU_PAGE_TABLE: ++ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_dump_mmu_page_table_s), sizeof(u64))); ++ err = mem_dump_mmu_page_table_wrapper(session_data, (_mali_uk_dump_mmu_page_table_s __user *)arg); ++ break; ++ ++ case MALI_IOC_MEM_DMA_BUF_GET_SIZE: ++#ifdef CONFIG_DMA_SHARED_BUFFER ++ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_dma_buf_get_size_s), sizeof(u64))); ++ err = mali_dma_buf_get_size(session_data, (_mali_uk_dma_buf_get_size_s __user *)arg); ++#else ++ MALI_DEBUG_PRINT(2, ("DMA-BUF not supported\n")); ++ err = -ENOTTY; ++#endif ++ break; ++ ++ case MALI_IOC_PP_START_JOB: ++ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_pp_start_job_s), sizeof(u64))); ++ err = pp_start_job_wrapper(session_data, (_mali_uk_pp_start_job_s __user *)arg); ++ break; ++ ++ case MALI_IOC_PP_AND_GP_START_JOB: ++ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_pp_and_gp_start_job_s), sizeof(u64))); ++ err = pp_and_gp_start_job_wrapper(session_data, (_mali_uk_pp_and_gp_start_job_s __user *)arg); ++ break; ++ ++ case MALI_IOC_PP_NUMBER_OF_CORES_GET: ++ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_pp_number_of_cores_s), sizeof(u64))); ++ err = pp_get_number_of_cores_wrapper(session_data, (_mali_uk_get_pp_number_of_cores_s __user *)arg); ++ break; ++ ++ case MALI_IOC_PP_CORE_VERSION_GET: ++ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_pp_core_version_s), sizeof(u64))); ++ err = pp_get_core_version_wrapper(session_data, (_mali_uk_get_pp_core_version_s __user *)arg); ++ break; ++ ++ case MALI_IOC_PP_DISABLE_WB: ++ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_pp_disable_wb_s), sizeof(u64))); ++ err = pp_disable_wb_wrapper(session_data, (_mali_uk_pp_disable_wb_s __user *)arg); ++ break; ++ ++ case MALI_IOC_GP2_START_JOB: ++ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_gp_start_job_s), sizeof(u64))); ++ err = gp_start_job_wrapper(session_data, (_mali_uk_gp_start_job_s __user *)arg); ++ break; ++ ++ case MALI_IOC_GP2_NUMBER_OF_CORES_GET: ++ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_gp_number_of_cores_s), sizeof(u64))); ++ err = gp_get_number_of_cores_wrapper(session_data, (_mali_uk_get_gp_number_of_cores_s __user *)arg); ++ break; ++ ++ case MALI_IOC_GP2_CORE_VERSION_GET: ++ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_gp_core_version_s), sizeof(u64))); ++ err = gp_get_core_version_wrapper(session_data, (_mali_uk_get_gp_core_version_s __user *)arg); ++ break; ++ ++ case MALI_IOC_GP2_SUSPEND_RESPONSE: ++ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_gp_suspend_response_s), sizeof(u64))); ++ err = gp_suspend_response_wrapper(session_data, (_mali_uk_gp_suspend_response_s __user *)arg); ++ break; ++ ++ case MALI_IOC_VSYNC_EVENT_REPORT: ++ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_vsync_event_report_s), sizeof(u64))); ++ err = vsync_event_report_wrapper(session_data, (_mali_uk_vsync_event_report_s __user *)arg); ++ break; ++ ++ case MALI_IOC_TIMELINE_GET_LATEST_POINT: ++ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_timeline_get_latest_point_s), sizeof(u64))); ++ err = timeline_get_latest_point_wrapper(session_data, (_mali_uk_timeline_get_latest_point_s __user *)arg); ++ break; ++ case MALI_IOC_TIMELINE_WAIT: ++ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_timeline_wait_s), sizeof(u64))); ++ err = timeline_wait_wrapper(session_data, (_mali_uk_timeline_wait_s __user *)arg); ++ break; ++ case MALI_IOC_TIMELINE_CREATE_SYNC_FENCE: ++ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_timeline_create_sync_fence_s), sizeof(u64))); ++ err = timeline_create_sync_fence_wrapper(session_data, (_mali_uk_timeline_create_sync_fence_s __user *)arg); ++ break; ++ case MALI_IOC_SOFT_JOB_START: ++ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_soft_job_start_s), sizeof(u64))); ++ err = soft_job_start_wrapper(session_data, (_mali_uk_soft_job_start_s __user *)arg); ++ break; ++ case MALI_IOC_SOFT_JOB_SIGNAL: ++ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_soft_job_signal_s), sizeof(u64))); ++ err = soft_job_signal_wrapper(session_data, (_mali_uk_soft_job_signal_s __user *)arg); ++ break; ++ ++ default: ++ MALI_DEBUG_PRINT(2, ("No handler for ioctl 0x%08X 0x%08lX\n", cmd, arg)); ++ err = -ENOTTY; ++ }; ++ ++ return err; ++} ++ ++late_initcall_sync(mali_module_init); ++module_exit(mali_module_exit); ++ ++MODULE_LICENSE(MALI_KERNEL_LINUX_LICENSE); ++MODULE_AUTHOR("ARM Ltd."); ++MODULE_VERSION(SVN_REV_STRING); +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_kernel_linux.h b/drivers/gpu/arm/mali400/mali/linux/mali_kernel_linux.h +new file mode 100755 +index 000000000000..be754cb15646 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_kernel_linux.h +@@ -0,0 +1,36 @@ ++/* ++ * Copyright (C) 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_KERNEL_LINUX_H__ ++#define __MALI_KERNEL_LINUX_H__ ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++#include /* character device definitions */ ++#include ++#include ++#include "mali_kernel_license.h" ++#include "mali_osk_types.h" ++#include ++ ++extern struct platform_device *mali_platform_device; ++ ++/* After 3.19.0 kenrel droped CONFIG_PM_RUNTIME define,define by ourself */ ++#if defined(CONFIG_PM) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0) ++#define CONFIG_PM_RUNTIME 1 ++#endif ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif /* __MALI_KERNEL_LINUX_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_kernel_sysfs.c b/drivers/gpu/arm/mali400/mali/linux/mali_kernel_sysfs.c +new file mode 100755 +index 000000000000..7bda438fef50 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_kernel_sysfs.c +@@ -0,0 +1,1410 @@ ++/** ++ * Copyright (C) 2011-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++ ++/** ++ * @file mali_kernel_sysfs.c ++ * Implementation of some sysfs data exports ++ */ ++ ++#include ++#include ++#include ++#include ++#include "mali_kernel_license.h" ++#include "mali_kernel_common.h" ++#include "mali_ukk.h" ++ ++#if MALI_LICENSE_IS_GPL ++ ++#include ++#include ++#include ++#include ++#include ++#include "mali_kernel_sysfs.h" ++#if defined(CONFIG_MALI400_INTERNAL_PROFILING) ++#include ++#include "mali_osk_profiling.h" ++#endif ++ ++#include ++#include "mali_pm.h" ++#include "mali_pmu.h" ++#include "mali_group.h" ++#include "mali_gp.h" ++#include "mali_pp.h" ++#include "mali_l2_cache.h" ++#include "mali_hw_core.h" ++#include "mali_kernel_core.h" ++#include "mali_user_settings_db.h" ++#include "mali_profiling_internal.h" ++#include "mali_gp_job.h" ++#include "mali_pp_job.h" ++#include "mali_executor.h" ++ ++#define PRIVATE_DATA_COUNTER_MAKE_GP(src) (src) ++#define PRIVATE_DATA_COUNTER_MAKE_PP(src) ((1 << 24) | src) ++#define PRIVATE_DATA_COUNTER_MAKE_PP_SUB_JOB(src, sub_job) ((1 << 24) | (1 << 16) | (sub_job << 8) | src) ++#define PRIVATE_DATA_COUNTER_IS_PP(a) ((((a) >> 24) & 0xFF) ? MALI_TRUE : MALI_FALSE) ++#define PRIVATE_DATA_COUNTER_GET_SRC(a) (a & 0xFF) ++#define PRIVATE_DATA_COUNTER_IS_SUB_JOB(a) ((((a) >> 16) & 0xFF) ? MALI_TRUE : MALI_FALSE) ++#define PRIVATE_DATA_COUNTER_GET_SUB_JOB(a) (((a) >> 8) & 0xFF) ++ ++#define POWER_BUFFER_SIZE 3 ++ ++static struct dentry *mali_debugfs_dir = NULL; ++ ++typedef enum { ++ _MALI_DEVICE_SUSPEND, ++ _MALI_DEVICE_RESUME, ++ _MALI_DEVICE_DVFS_PAUSE, ++ _MALI_DEVICE_DVFS_RESUME, ++ _MALI_MAX_EVENTS ++} _mali_device_debug_power_events; ++ ++static const char *const mali_power_events[_MALI_MAX_EVENTS] = { ++ [_MALI_DEVICE_SUSPEND] = "suspend", ++ [_MALI_DEVICE_RESUME] = "resume", ++ [_MALI_DEVICE_DVFS_PAUSE] = "dvfs_pause", ++ [_MALI_DEVICE_DVFS_RESUME] = "dvfs_resume", ++}; ++ ++static mali_bool power_always_on_enabled = MALI_FALSE; ++ ++static int open_copy_private_data(struct inode *inode, struct file *filp) ++{ ++ filp->private_data = inode->i_private; ++ return 0; ++} ++ ++static ssize_t group_enabled_read(struct file *filp, char __user *buf, size_t count, loff_t *offp) ++{ ++ int r; ++ char buffer[64]; ++ struct mali_group *group; ++ ++ group = (struct mali_group *)filp->private_data; ++ MALI_DEBUG_ASSERT_POINTER(group); ++ ++ r = snprintf(buffer, 64, "%u\n", ++ mali_executor_group_is_disabled(group) ? 0 : 1); ++ ++ return simple_read_from_buffer(buf, count, offp, buffer, r); ++} ++ ++static ssize_t group_enabled_write(struct file *filp, const char __user *buf, size_t count, loff_t *offp) ++{ ++ int r; ++ char buffer[64]; ++ unsigned long val; ++ struct mali_group *group; ++ ++ group = (struct mali_group *)filp->private_data; ++ MALI_DEBUG_ASSERT_POINTER(group); ++ ++ if (count >= sizeof(buffer)) { ++ return -ENOMEM; ++ } ++ ++ if (copy_from_user(&buffer[0], buf, count)) { ++ return -EFAULT; ++ } ++ buffer[count] = '\0'; ++ ++ r = kstrtoul(&buffer[0], 10, &val); ++ if (0 != r) { ++ return -EINVAL; ++ } ++ ++ switch (val) { ++ case 1: ++ mali_executor_group_enable(group); ++ break; ++ case 0: ++ mali_executor_group_disable(group); ++ break; ++ default: ++ return -EINVAL; ++ break; ++ } ++ ++ *offp += count; ++ return count; ++} ++ ++static const struct file_operations group_enabled_fops = { ++ .owner = THIS_MODULE, ++ .open = open_copy_private_data, ++ .read = group_enabled_read, ++ .write = group_enabled_write, ++}; ++ ++static ssize_t hw_core_base_addr_read(struct file *filp, char __user *buf, size_t count, loff_t *offp) ++{ ++ int r; ++ char buffer[64]; ++ struct mali_hw_core *hw_core; ++ ++ hw_core = (struct mali_hw_core *)filp->private_data; ++ MALI_DEBUG_ASSERT_POINTER(hw_core); ++ ++ r = snprintf(buffer, 64, "0x%lX\n", hw_core->phys_addr); ++ ++ return simple_read_from_buffer(buf, count, offp, buffer, r); ++} ++ ++static const struct file_operations hw_core_base_addr_fops = { ++ .owner = THIS_MODULE, ++ .open = open_copy_private_data, ++ .read = hw_core_base_addr_read, ++}; ++ ++static ssize_t profiling_counter_src_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) ++{ ++ u32 is_pp = PRIVATE_DATA_COUNTER_IS_PP((uintptr_t)filp->private_data); ++ u32 src_id = PRIVATE_DATA_COUNTER_GET_SRC((uintptr_t)filp->private_data); ++ mali_bool is_sub_job = PRIVATE_DATA_COUNTER_IS_SUB_JOB((uintptr_t)filp->private_data); ++ u32 sub_job = PRIVATE_DATA_COUNTER_GET_SUB_JOB((uintptr_t)filp->private_data); ++ char buf[64]; ++ int r; ++ u32 val; ++ ++ if (MALI_TRUE == is_pp) { ++ /* PP counter */ ++ if (MALI_TRUE == is_sub_job) { ++ /* Get counter for a particular sub job */ ++ if (0 == src_id) { ++ val = mali_pp_job_get_pp_counter_sub_job_src0(sub_job); ++ } else { ++ val = mali_pp_job_get_pp_counter_sub_job_src1(sub_job); ++ } ++ } else { ++ /* Get default counter for all PP sub jobs */ ++ if (0 == src_id) { ++ val = mali_pp_job_get_pp_counter_global_src0(); ++ } else { ++ val = mali_pp_job_get_pp_counter_global_src1(); ++ } ++ } ++ } else { ++ /* GP counter */ ++ if (0 == src_id) { ++ val = mali_gp_job_get_gp_counter_src0(); ++ } else { ++ val = mali_gp_job_get_gp_counter_src1(); ++ } ++ } ++ ++ if (MALI_HW_CORE_NO_COUNTER == val) { ++ r = snprintf(buf, 64, "-1\n"); ++ } else { ++ r = snprintf(buf, 64, "%u\n", val); ++ } ++ ++ return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); ++} ++ ++static ssize_t profiling_counter_src_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) ++{ ++ u32 is_pp = PRIVATE_DATA_COUNTER_IS_PP((uintptr_t)filp->private_data); ++ u32 src_id = PRIVATE_DATA_COUNTER_GET_SRC((uintptr_t)filp->private_data); ++ mali_bool is_sub_job = PRIVATE_DATA_COUNTER_IS_SUB_JOB((uintptr_t)filp->private_data); ++ u32 sub_job = PRIVATE_DATA_COUNTER_GET_SUB_JOB((uintptr_t)filp->private_data); ++ char buf[64]; ++ long val; ++ int ret; ++ ++ if (cnt >= sizeof(buf)) { ++ return -EINVAL; ++ } ++ ++ if (copy_from_user(&buf, ubuf, cnt)) { ++ return -EFAULT; ++ } ++ ++ buf[cnt] = 0; ++ ++ ret = kstrtol(buf, 10, &val); ++ if (ret < 0) { ++ return ret; ++ } ++ ++ if (val < 0) { ++ /* any negative input will disable counter */ ++ val = MALI_HW_CORE_NO_COUNTER; ++ } ++ ++ if (MALI_TRUE == is_pp) { ++ /* PP counter */ ++ if (MALI_TRUE == is_sub_job) { ++ /* Set counter for a particular sub job */ ++ if (0 == src_id) { ++ mali_pp_job_set_pp_counter_sub_job_src0(sub_job, (u32)val); ++ } else { ++ mali_pp_job_set_pp_counter_sub_job_src1(sub_job, (u32)val); ++ } ++ } else { ++ /* Set default counter for all PP sub jobs */ ++ if (0 == src_id) { ++ mali_pp_job_set_pp_counter_global_src0((u32)val); ++ } else { ++ mali_pp_job_set_pp_counter_global_src1((u32)val); ++ } ++ } ++ } else { ++ /* GP counter */ ++ if (0 == src_id) { ++ mali_gp_job_set_gp_counter_src0((u32)val); ++ } else { ++ mali_gp_job_set_gp_counter_src1((u32)val); ++ } ++ } ++ ++ *ppos += cnt; ++ return cnt; ++} ++ ++static const struct file_operations profiling_counter_src_fops = { ++ .owner = THIS_MODULE, ++ .open = open_copy_private_data, ++ .read = profiling_counter_src_read, ++ .write = profiling_counter_src_write, ++}; ++ ++static ssize_t l2_l2x_counter_srcx_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos, u32 src_id) ++{ ++ char buf[64]; ++ int r; ++ u32 val; ++ struct mali_l2_cache_core *l2_core = (struct mali_l2_cache_core *)filp->private_data; ++ ++ if (0 == src_id) { ++ val = mali_l2_cache_core_get_counter_src0(l2_core); ++ } else { ++ val = mali_l2_cache_core_get_counter_src1(l2_core); ++ } ++ ++ if (MALI_HW_CORE_NO_COUNTER == val) { ++ r = snprintf(buf, 64, "-1\n"); ++ } else { ++ r = snprintf(buf, 64, "%u\n", val); ++ } ++ return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); ++} ++ ++static ssize_t l2_l2x_counter_srcx_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos, u32 src_id) ++{ ++ struct mali_l2_cache_core *l2_core = (struct mali_l2_cache_core *)filp->private_data; ++ char buf[64]; ++ long val; ++ int ret; ++ ++ if (cnt >= sizeof(buf)) { ++ return -EINVAL; ++ } ++ ++ if (copy_from_user(&buf, ubuf, cnt)) { ++ return -EFAULT; ++ } ++ ++ buf[cnt] = 0; ++ ++ ret = kstrtol(buf, 10, &val); ++ if (ret < 0) { ++ return ret; ++ } ++ ++ if (val < 0) { ++ /* any negative input will disable counter */ ++ val = MALI_HW_CORE_NO_COUNTER; ++ } ++ ++ mali_l2_cache_core_set_counter_src(l2_core, src_id, (u32)val); ++ ++ *ppos += cnt; ++ return cnt; ++} ++ ++static ssize_t l2_all_counter_srcx_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos, u32 src_id) ++{ ++ char buf[64]; ++ long val; ++ int ret; ++ u32 l2_id; ++ struct mali_l2_cache_core *l2_cache; ++ ++ if (cnt >= sizeof(buf)) { ++ return -EINVAL; ++ } ++ ++ if (copy_from_user(&buf, ubuf, cnt)) { ++ return -EFAULT; ++ } ++ ++ buf[cnt] = 0; ++ ++ ret = kstrtol(buf, 10, &val); ++ if (ret < 0) { ++ return ret; ++ } ++ ++ if (val < 0) { ++ /* any negative input will disable counter */ ++ val = MALI_HW_CORE_NO_COUNTER; ++ } ++ ++ l2_id = 0; ++ l2_cache = mali_l2_cache_core_get_glob_l2_core(l2_id); ++ while (NULL != l2_cache) { ++ mali_l2_cache_core_set_counter_src(l2_cache, src_id, (u32)val); ++ ++ /* try next L2 */ ++ l2_id++; ++ l2_cache = mali_l2_cache_core_get_glob_l2_core(l2_id); ++ } ++ ++ *ppos += cnt; ++ return cnt; ++} ++ ++static ssize_t l2_l2x_counter_src0_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) ++{ ++ return l2_l2x_counter_srcx_read(filp, ubuf, cnt, ppos, 0); ++} ++ ++static ssize_t l2_l2x_counter_src1_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) ++{ ++ return l2_l2x_counter_srcx_read(filp, ubuf, cnt, ppos, 1); ++} ++ ++static ssize_t l2_l2x_counter_src0_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) ++{ ++ return l2_l2x_counter_srcx_write(filp, ubuf, cnt, ppos, 0); ++} ++ ++static ssize_t l2_l2x_counter_src1_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) ++{ ++ return l2_l2x_counter_srcx_write(filp, ubuf, cnt, ppos, 1); ++} ++ ++static ssize_t l2_all_counter_src0_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) ++{ ++ return l2_all_counter_srcx_write(filp, ubuf, cnt, ppos, 0); ++} ++ ++static ssize_t l2_all_counter_src1_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) ++{ ++ return l2_all_counter_srcx_write(filp, ubuf, cnt, ppos, 1); ++} ++ ++static const struct file_operations l2_l2x_counter_src0_fops = { ++ .owner = THIS_MODULE, ++ .open = open_copy_private_data, ++ .read = l2_l2x_counter_src0_read, ++ .write = l2_l2x_counter_src0_write, ++}; ++ ++static const struct file_operations l2_l2x_counter_src1_fops = { ++ .owner = THIS_MODULE, ++ .open = open_copy_private_data, ++ .read = l2_l2x_counter_src1_read, ++ .write = l2_l2x_counter_src1_write, ++}; ++ ++static const struct file_operations l2_all_counter_src0_fops = { ++ .owner = THIS_MODULE, ++ .write = l2_all_counter_src0_write, ++}; ++ ++static const struct file_operations l2_all_counter_src1_fops = { ++ .owner = THIS_MODULE, ++ .write = l2_all_counter_src1_write, ++}; ++ ++static ssize_t l2_l2x_counter_valx_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos, u32 src_id) ++{ ++ char buf[64]; ++ int r; ++ u32 src0 = 0; ++ u32 val0 = 0; ++ u32 src1 = 0; ++ u32 val1 = 0; ++ u32 val = -1; ++ struct mali_l2_cache_core *l2_core = (struct mali_l2_cache_core *)filp->private_data; ++ ++ mali_l2_cache_core_get_counter_values(l2_core, &src0, &val0, &src1, &val1); ++ ++ if (0 == src_id) { ++ if (MALI_HW_CORE_NO_COUNTER != val0) { ++ val = val0; ++ } ++ } else { ++ if (MALI_HW_CORE_NO_COUNTER != val1) { ++ val = val1; ++ } ++ } ++ ++ r = snprintf(buf, 64, "%u\n", val); ++ ++ return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); ++} ++ ++static ssize_t l2_l2x_counter_val0_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) ++{ ++ return l2_l2x_counter_valx_read(filp, ubuf, cnt, ppos, 0); ++} ++ ++static ssize_t l2_l2x_counter_val1_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) ++{ ++ return l2_l2x_counter_valx_read(filp, ubuf, cnt, ppos, 1); ++} ++ ++static const struct file_operations l2_l2x_counter_val0_fops = { ++ .owner = THIS_MODULE, ++ .open = open_copy_private_data, ++ .read = l2_l2x_counter_val0_read, ++}; ++ ++static const struct file_operations l2_l2x_counter_val1_fops = { ++ .owner = THIS_MODULE, ++ .open = open_copy_private_data, ++ .read = l2_l2x_counter_val1_read, ++}; ++ ++static ssize_t power_always_on_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) ++{ ++ unsigned long val; ++ int ret; ++ char buf[32]; ++ ++ cnt = min(cnt, sizeof(buf) - 1); ++ if (copy_from_user(buf, ubuf, cnt)) { ++ return -EFAULT; ++ } ++ buf[cnt] = '\0'; ++ ++ ret = kstrtoul(buf, 10, &val); ++ if (0 != ret) { ++ return ret; ++ } ++ ++ /* Update setting (not exactly thread safe) */ ++ if (1 == val && MALI_FALSE == power_always_on_enabled) { ++ power_always_on_enabled = MALI_TRUE; ++ _mali_osk_pm_dev_ref_get_sync(); ++ } else if (0 == val && MALI_TRUE == power_always_on_enabled) { ++ power_always_on_enabled = MALI_FALSE; ++ _mali_osk_pm_dev_ref_put(); ++ } ++ ++ *ppos += cnt; ++ return cnt; ++} ++ ++static ssize_t power_always_on_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) ++{ ++ if (MALI_TRUE == power_always_on_enabled) { ++ return simple_read_from_buffer(ubuf, cnt, ppos, "1\n", 2); ++ } else { ++ return simple_read_from_buffer(ubuf, cnt, ppos, "0\n", 2); ++ } ++} ++ ++static const struct file_operations power_always_on_fops = { ++ .owner = THIS_MODULE, ++ .read = power_always_on_read, ++ .write = power_always_on_write, ++}; ++ ++static ssize_t power_power_events_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) ++{ ++ if (!strncmp(ubuf, mali_power_events[_MALI_DEVICE_SUSPEND], strlen(mali_power_events[_MALI_DEVICE_SUSPEND]) - 1)) { ++ mali_pm_os_suspend(MALI_TRUE); ++ } else if (!strncmp(ubuf, mali_power_events[_MALI_DEVICE_RESUME], strlen(mali_power_events[_MALI_DEVICE_RESUME]) - 1)) { ++ mali_pm_os_resume(); ++ } else if (!strncmp(ubuf, mali_power_events[_MALI_DEVICE_DVFS_PAUSE], strlen(mali_power_events[_MALI_DEVICE_DVFS_PAUSE]) - 1)) { ++ mali_dev_pause(); ++ } else if (!strncmp(ubuf, mali_power_events[_MALI_DEVICE_DVFS_RESUME], strlen(mali_power_events[_MALI_DEVICE_DVFS_RESUME]) - 1)) { ++ mali_dev_resume(); ++ } ++ *ppos += cnt; ++ return cnt; ++} ++ ++static loff_t power_power_events_seek(struct file *file, loff_t offset, int orig) ++{ ++ file->f_pos = offset; ++ return 0; ++} ++ ++static const struct file_operations power_power_events_fops = { ++ .owner = THIS_MODULE, ++ .write = power_power_events_write, ++ .llseek = power_power_events_seek, ++}; ++ ++#if MALI_STATE_TRACKING ++static int mali_seq_internal_state_show(struct seq_file *seq_file, void *v) ++{ ++ u32 len = 0; ++ u32 size; ++ char *buf; ++ ++ size = seq_get_buf(seq_file, &buf); ++ ++ if (!size) { ++ return -ENOMEM; ++ } ++ ++ /* Create the internal state dump. */ ++ len = snprintf(buf + len, size - len, "Mali device driver %s\n", SVN_REV_STRING); ++ len += snprintf(buf + len, size - len, "License: %s\n\n", MALI_KERNEL_LINUX_LICENSE); ++ ++ len += _mali_kernel_core_dump_state(buf + len, size - len); ++ ++ seq_commit(seq_file, len); ++ ++ return 0; ++} ++ ++static int mali_seq_internal_state_open(struct inode *inode, struct file *file) ++{ ++ return single_open(file, mali_seq_internal_state_show, NULL); ++} ++ ++static const struct file_operations mali_seq_internal_state_fops = { ++ .owner = THIS_MODULE, ++ .open = mali_seq_internal_state_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = single_release, ++}; ++#endif /* MALI_STATE_TRACKING */ ++ ++#if defined(CONFIG_MALI400_INTERNAL_PROFILING) ++static ssize_t profiling_record_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) ++{ ++ char buf[64]; ++ int r; ++ ++ r = snprintf(buf, 64, "%u\n", _mali_internal_profiling_is_recording() ? 1 : 0); ++ return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); ++} ++ ++static ssize_t profiling_record_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) ++{ ++ char buf[64]; ++ unsigned long val; ++ int ret; ++ ++ if (cnt >= sizeof(buf)) { ++ return -EINVAL; ++ } ++ ++ if (copy_from_user(&buf, ubuf, cnt)) { ++ return -EFAULT; ++ } ++ ++ buf[cnt] = 0; ++ ++ ret = kstrtoul(buf, 10, &val); ++ if (ret < 0) { ++ return ret; ++ } ++ ++ if (val != 0) { ++ u32 limit = MALI_PROFILING_MAX_BUFFER_ENTRIES; /* This can be made configurable at a later stage if we need to */ ++ ++ /* check if we are already recording */ ++ if (MALI_TRUE == _mali_internal_profiling_is_recording()) { ++ MALI_DEBUG_PRINT(3, ("Recording of profiling events already in progress\n")); ++ return -EFAULT; ++ } ++ ++ /* check if we need to clear out an old recording first */ ++ if (MALI_TRUE == _mali_internal_profiling_have_recording()) { ++ if (_MALI_OSK_ERR_OK != _mali_internal_profiling_clear()) { ++ MALI_DEBUG_PRINT(3, ("Failed to clear existing recording of profiling events\n")); ++ return -EFAULT; ++ } ++ } ++ ++ /* start recording profiling data */ ++ if (_MALI_OSK_ERR_OK != _mali_internal_profiling_start(&limit)) { ++ MALI_DEBUG_PRINT(3, ("Failed to start recording of profiling events\n")); ++ return -EFAULT; ++ } ++ ++ MALI_DEBUG_PRINT(3, ("Profiling recording started (max %u events)\n", limit)); ++ } else { ++ /* stop recording profiling data */ ++ u32 count = 0; ++ if (_MALI_OSK_ERR_OK != _mali_internal_profiling_stop(&count)) { ++ MALI_DEBUG_PRINT(2, ("Failed to stop recording of profiling events\n")); ++ return -EFAULT; ++ } ++ ++ MALI_DEBUG_PRINT(2, ("Profiling recording stopped (recorded %u events)\n", count)); ++ } ++ ++ *ppos += cnt; ++ return cnt; ++} ++ ++static const struct file_operations profiling_record_fops = { ++ .owner = THIS_MODULE, ++ .read = profiling_record_read, ++ .write = profiling_record_write, ++}; ++ ++static void *profiling_events_start(struct seq_file *s, loff_t *pos) ++{ ++ loff_t *spos; ++ ++ /* check if we have data avaiable */ ++ if (MALI_TRUE != _mali_internal_profiling_have_recording()) { ++ return NULL; ++ } ++ ++ spos = kmalloc(sizeof(loff_t), GFP_KERNEL); ++ if (NULL == spos) { ++ return NULL; ++ } ++ ++ *spos = *pos; ++ return spos; ++} ++ ++static void *profiling_events_next(struct seq_file *s, void *v, loff_t *pos) ++{ ++ loff_t *spos = v; ++ ++ /* check if we have data avaiable */ ++ if (MALI_TRUE != _mali_internal_profiling_have_recording()) { ++ return NULL; ++ } ++ ++ /* check if the next entry actually is avaiable */ ++ if (_mali_internal_profiling_get_count() <= (u32)(*spos + 1)) { ++ return NULL; ++ } ++ ++ *pos = ++*spos; ++ return spos; ++} ++ ++static void profiling_events_stop(struct seq_file *s, void *v) ++{ ++ kfree(v); ++} ++ ++static int profiling_events_show(struct seq_file *seq_file, void *v) ++{ ++ loff_t *spos = v; ++ u32 index; ++ u64 timestamp; ++ u32 event_id; ++ u32 data[5]; ++ ++ index = (u32) * spos; ++ ++ /* Retrieve all events */ ++ if (_MALI_OSK_ERR_OK == _mali_internal_profiling_get_event(index, ×tamp, &event_id, data)) { ++ seq_printf(seq_file, "%llu %u %u %u %u %u %u\n", timestamp, event_id, data[0], data[1], data[2], data[3], data[4]); ++ return 0; ++ } ++ ++ return 0; ++} ++ ++static int profiling_events_show_human_readable(struct seq_file *seq_file, void *v) ++{ ++#define MALI_EVENT_ID_IS_HW(event_id) (((event_id & 0x00FF0000) >= MALI_PROFILING_EVENT_CHANNEL_GP0) && ((event_id & 0x00FF0000) <= MALI_PROFILING_EVENT_CHANNEL_PP7)) ++ ++ static u64 start_time = 0; ++ loff_t *spos = v; ++ u32 index; ++ u64 timestamp; ++ u32 event_id; ++ u32 data[5]; ++ ++ index = (u32) * spos; ++ ++ /* Retrieve all events */ ++ if (_MALI_OSK_ERR_OK == _mali_internal_profiling_get_event(index, ×tamp, &event_id, data)) { ++ seq_printf(seq_file, "%llu %u %u %u %u %u %u # ", timestamp, event_id, data[0], data[1], data[2], data[3], data[4]); ++ ++ if (0 == index) { ++ start_time = timestamp; ++ } ++ ++ seq_printf(seq_file, "[%06u] ", index); ++ ++ switch (event_id & 0x0F000000) { ++ case MALI_PROFILING_EVENT_TYPE_SINGLE: ++ seq_printf(seq_file, "SINGLE | "); ++ break; ++ case MALI_PROFILING_EVENT_TYPE_START: ++ seq_printf(seq_file, "START | "); ++ break; ++ case MALI_PROFILING_EVENT_TYPE_STOP: ++ seq_printf(seq_file, "STOP | "); ++ break; ++ case MALI_PROFILING_EVENT_TYPE_SUSPEND: ++ seq_printf(seq_file, "SUSPEND | "); ++ break; ++ case MALI_PROFILING_EVENT_TYPE_RESUME: ++ seq_printf(seq_file, "RESUME | "); ++ break; ++ default: ++ seq_printf(seq_file, "0x%01X | ", (event_id & 0x0F000000) >> 24); ++ break; ++ } ++ ++ switch (event_id & 0x00FF0000) { ++ case MALI_PROFILING_EVENT_CHANNEL_SOFTWARE: ++ seq_printf(seq_file, "SW | "); ++ break; ++ case MALI_PROFILING_EVENT_CHANNEL_GP0: ++ seq_printf(seq_file, "GP0 | "); ++ break; ++ case MALI_PROFILING_EVENT_CHANNEL_PP0: ++ seq_printf(seq_file, "PP0 | "); ++ break; ++ case MALI_PROFILING_EVENT_CHANNEL_PP1: ++ seq_printf(seq_file, "PP1 | "); ++ break; ++ case MALI_PROFILING_EVENT_CHANNEL_PP2: ++ seq_printf(seq_file, "PP2 | "); ++ break; ++ case MALI_PROFILING_EVENT_CHANNEL_PP3: ++ seq_printf(seq_file, "PP3 | "); ++ break; ++ case MALI_PROFILING_EVENT_CHANNEL_PP4: ++ seq_printf(seq_file, "PP4 | "); ++ break; ++ case MALI_PROFILING_EVENT_CHANNEL_PP5: ++ seq_printf(seq_file, "PP5 | "); ++ break; ++ case MALI_PROFILING_EVENT_CHANNEL_PP6: ++ seq_printf(seq_file, "PP6 | "); ++ break; ++ case MALI_PROFILING_EVENT_CHANNEL_PP7: ++ seq_printf(seq_file, "PP7 | "); ++ break; ++ case MALI_PROFILING_EVENT_CHANNEL_GPU: ++ seq_printf(seq_file, "GPU | "); ++ break; ++ default: ++ seq_printf(seq_file, "0x%02X | ", (event_id & 0x00FF0000) >> 16); ++ break; ++ } ++ ++ if (MALI_EVENT_ID_IS_HW(event_id)) { ++ if (((event_id & 0x0F000000) == MALI_PROFILING_EVENT_TYPE_START) || ((event_id & 0x0F000000) == MALI_PROFILING_EVENT_TYPE_STOP)) { ++ switch (event_id & 0x0000FFFF) { ++ case MALI_PROFILING_EVENT_REASON_START_STOP_HW_PHYSICAL: ++ seq_printf(seq_file, "PHYSICAL | "); ++ break; ++ case MALI_PROFILING_EVENT_REASON_START_STOP_HW_VIRTUAL: ++ seq_printf(seq_file, "VIRTUAL | "); ++ break; ++ default: ++ seq_printf(seq_file, "0x%04X | ", event_id & 0x0000FFFF); ++ break; ++ } ++ } else { ++ seq_printf(seq_file, "0x%04X | ", event_id & 0x0000FFFF); ++ } ++ } else { ++ seq_printf(seq_file, "0x%04X | ", event_id & 0x0000FFFF); ++ } ++ ++ seq_printf(seq_file, "T0 + 0x%016llX\n", timestamp - start_time); ++ ++ return 0; ++ } ++ ++ return 0; ++} ++ ++static const struct seq_operations profiling_events_seq_ops = { ++ .start = profiling_events_start, ++ .next = profiling_events_next, ++ .stop = profiling_events_stop, ++ .show = profiling_events_show ++}; ++ ++static int profiling_events_open(struct inode *inode, struct file *file) ++{ ++ return seq_open(file, &profiling_events_seq_ops); ++} ++ ++static const struct file_operations profiling_events_fops = { ++ .owner = THIS_MODULE, ++ .open = profiling_events_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = seq_release, ++}; ++ ++static const struct seq_operations profiling_events_human_readable_seq_ops = { ++ .start = profiling_events_start, ++ .next = profiling_events_next, ++ .stop = profiling_events_stop, ++ .show = profiling_events_show_human_readable ++}; ++ ++static int profiling_events_human_readable_open(struct inode *inode, struct file *file) ++{ ++ return seq_open(file, &profiling_events_human_readable_seq_ops); ++} ++ ++static const struct file_operations profiling_events_human_readable_fops = { ++ .owner = THIS_MODULE, ++ .open = profiling_events_human_readable_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = seq_release, ++}; ++ ++#endif ++ ++static int memory_debugfs_show(struct seq_file *s, void *private_data) ++{ ++#ifdef MALI_MEM_SWAP_TRACKING ++ seq_printf(s, " %-25s %-10s %-10s %-15s %-15s %-10s %-10s %-10s \n"\ ++ "=================================================================================================================================\n", ++ "Name (:bytes)", "pid", "mali_mem", "max_mali_mem", ++ "external_mem", "ump_mem", "dma_mem", "swap_mem"); ++#else ++ seq_printf(s, " %-25s %-10s %-10s %-15s %-15s %-10s %-10s \n"\ ++ "========================================================================================================================\n", ++ "Name (:bytes)", "pid", "mali_mem", "max_mali_mem", ++ "external_mem", "ump_mem", "dma_mem"); ++#endif ++ mali_session_memory_tracking(s); ++ return 0; ++} ++ ++static int memory_debugfs_open(struct inode *inode, struct file *file) ++{ ++ return single_open(file, memory_debugfs_show, inode->i_private); ++} ++ ++static const struct file_operations memory_usage_fops = { ++ .owner = THIS_MODULE, ++ .open = memory_debugfs_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = single_release, ++}; ++ ++static ssize_t utilization_gp_pp_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) ++{ ++ char buf[64]; ++ size_t r; ++ u32 uval = _mali_ukk_utilization_gp_pp(); ++ ++ r = snprintf(buf, 64, "%u\n", uval); ++ return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); ++} ++ ++static ssize_t utilization_gp_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) ++{ ++ char buf[64]; ++ size_t r; ++ u32 uval = _mali_ukk_utilization_gp(); ++ ++ r = snprintf(buf, 64, "%u\n", uval); ++ return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); ++} ++ ++static ssize_t utilization_pp_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) ++{ ++ char buf[64]; ++ size_t r; ++ u32 uval = _mali_ukk_utilization_pp(); ++ ++ r = snprintf(buf, 64, "%u\n", uval); ++ return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); ++} ++ ++ ++static const struct file_operations utilization_gp_pp_fops = { ++ .owner = THIS_MODULE, ++ .read = utilization_gp_pp_read, ++}; ++ ++static const struct file_operations utilization_gp_fops = { ++ .owner = THIS_MODULE, ++ .read = utilization_gp_read, ++}; ++ ++static const struct file_operations utilization_pp_fops = { ++ .owner = THIS_MODULE, ++ .read = utilization_pp_read, ++}; ++ ++static ssize_t user_settings_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) ++{ ++ unsigned long val; ++ int ret; ++ _mali_uk_user_setting_t setting; ++ char buf[32]; ++ ++ cnt = min(cnt, sizeof(buf) - 1); ++ if (copy_from_user(buf, ubuf, cnt)) { ++ return -EFAULT; ++ } ++ buf[cnt] = '\0'; ++ ++ ret = kstrtoul(buf, 10, &val); ++ if (0 != ret) { ++ return ret; ++ } ++ ++ /* Update setting */ ++ setting = (_mali_uk_user_setting_t)(filp->private_data); ++ mali_set_user_setting(setting, val); ++ ++ *ppos += cnt; ++ return cnt; ++} ++ ++static ssize_t user_settings_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) ++{ ++ char buf[64]; ++ size_t r; ++ u32 value; ++ _mali_uk_user_setting_t setting; ++ ++ setting = (_mali_uk_user_setting_t)(filp->private_data); ++ value = mali_get_user_setting(setting); ++ ++ r = snprintf(buf, 64, "%u\n", value); ++ return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); ++} ++ ++static const struct file_operations user_settings_fops = { ++ .owner = THIS_MODULE, ++ .open = open_copy_private_data, ++ .read = user_settings_read, ++ .write = user_settings_write, ++}; ++ ++static int mali_sysfs_user_settings_register(void) ++{ ++ struct dentry *mali_user_settings_dir = debugfs_create_dir("userspace_settings", mali_debugfs_dir); ++ ++ if (mali_user_settings_dir != NULL) { ++ long i; ++ for (i = 0; i < _MALI_UK_USER_SETTING_MAX; i++) { ++ debugfs_create_file(_mali_uk_user_setting_descriptions[i], ++ 0600, mali_user_settings_dir, (void *)i, ++ &user_settings_fops); ++ } ++ } ++ ++ return 0; ++} ++ ++static ssize_t pp_num_cores_enabled_write(struct file *filp, const char __user *buf, size_t count, loff_t *offp) ++{ ++ int ret; ++ char buffer[32]; ++ unsigned long val; ++ ++ if (count >= sizeof(buffer)) { ++ return -ENOMEM; ++ } ++ ++ if (copy_from_user(&buffer[0], buf, count)) { ++ return -EFAULT; ++ } ++ buffer[count] = '\0'; ++ ++ ret = kstrtoul(&buffer[0], 10, &val); ++ if (0 != ret) { ++ return -EINVAL; ++ } ++ ++ ret = mali_executor_set_perf_level(val, MALI_TRUE); /* override even if core scaling is disabled */ ++ if (ret) { ++ return ret; ++ } ++ ++ *offp += count; ++ return count; ++} ++ ++static ssize_t pp_num_cores_enabled_read(struct file *filp, char __user *buf, size_t count, loff_t *offp) ++{ ++ int r; ++ char buffer[64]; ++ ++ r = snprintf(buffer, 64, "%u\n", mali_executor_get_num_cores_enabled()); ++ ++ return simple_read_from_buffer(buf, count, offp, buffer, r); ++} ++ ++static const struct file_operations pp_num_cores_enabled_fops = { ++ .owner = THIS_MODULE, ++ .write = pp_num_cores_enabled_write, ++ .read = pp_num_cores_enabled_read, ++ .llseek = default_llseek, ++}; ++ ++static ssize_t pp_num_cores_total_read(struct file *filp, char __user *buf, size_t count, loff_t *offp) ++{ ++ int r; ++ char buffer[64]; ++ ++ r = snprintf(buffer, 64, "%u\n", mali_executor_get_num_cores_total()); ++ ++ return simple_read_from_buffer(buf, count, offp, buffer, r); ++} ++ ++static const struct file_operations pp_num_cores_total_fops = { ++ .owner = THIS_MODULE, ++ .read = pp_num_cores_total_read, ++}; ++ ++static ssize_t pp_core_scaling_enabled_write(struct file *filp, const char __user *buf, size_t count, loff_t *offp) ++{ ++ int ret; ++ char buffer[32]; ++ unsigned long val; ++ ++ if (count >= sizeof(buffer)) { ++ return -ENOMEM; ++ } ++ ++ if (copy_from_user(&buffer[0], buf, count)) { ++ return -EFAULT; ++ } ++ buffer[count] = '\0'; ++ ++ ret = kstrtoul(&buffer[0], 10, &val); ++ if (0 != ret) { ++ return -EINVAL; ++ } ++ ++ switch (val) { ++ case 1: ++ mali_executor_core_scaling_enable(); ++ break; ++ case 0: ++ mali_executor_core_scaling_disable(); ++ break; ++ default: ++ return -EINVAL; ++ break; ++ } ++ ++ *offp += count; ++ return count; ++} ++ ++static ssize_t pp_core_scaling_enabled_read(struct file *filp, char __user *buf, size_t count, loff_t *offp) ++{ ++ return simple_read_from_buffer(buf, count, offp, mali_executor_core_scaling_is_enabled() ? "1\n" : "0\n", 2); ++} ++static const struct file_operations pp_core_scaling_enabled_fops = { ++ .owner = THIS_MODULE, ++ .write = pp_core_scaling_enabled_write, ++ .read = pp_core_scaling_enabled_read, ++ .llseek = default_llseek, ++}; ++ ++static ssize_t version_read(struct file *filp, char __user *buf, size_t count, loff_t *offp) ++{ ++ int r = 0; ++ char buffer[64]; ++ ++ switch (mali_kernel_core_get_product_id()) { ++ case _MALI_PRODUCT_ID_MALI200: ++ r = snprintf(buffer, 64, "Mali-200\n"); ++ break; ++ case _MALI_PRODUCT_ID_MALI300: ++ r = snprintf(buffer, 64, "Mali-300\n"); ++ break; ++ case _MALI_PRODUCT_ID_MALI400: ++ r = snprintf(buffer, 64, "Mali-400 MP\n"); ++ break; ++ case _MALI_PRODUCT_ID_MALI450: ++ r = snprintf(buffer, 64, "Mali-450 MP\n"); ++ break; ++ case _MALI_PRODUCT_ID_MALI470: ++ r = snprintf(buffer, 64, "Mali-470 MP\n"); ++ break; ++ case _MALI_PRODUCT_ID_UNKNOWN: ++ return -EINVAL; ++ break; ++ }; ++ ++ return simple_read_from_buffer(buf, count, offp, buffer, r); ++} ++ ++static const struct file_operations version_fops = { ++ .owner = THIS_MODULE, ++ .read = version_read, ++}; ++ ++#if defined(DEBUG) ++static int timeline_debugfs_show(struct seq_file *s, void *private_data) ++{ ++ struct mali_session_data *session, *tmp; ++ u32 session_seq = 1; ++ ++ seq_printf(s, "timeline system info: \n=================\n\n"); ++ ++ mali_session_lock(); ++ MALI_SESSION_FOREACH(session, tmp, link) { ++ seq_printf(s, "session %d <%p> start:\n", session_seq, session); ++ mali_timeline_debug_print_system(session->timeline_system, s); ++ seq_printf(s, "session %d end\n\n\n", session_seq++); ++ } ++ mali_session_unlock(); ++ ++ return 0; ++} ++ ++static int timeline_debugfs_open(struct inode *inode, struct file *file) ++{ ++ return single_open(file, timeline_debugfs_show, inode->i_private); ++} ++ ++static const struct file_operations timeline_dump_fops = { ++ .owner = THIS_MODULE, ++ .open = timeline_debugfs_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = single_release ++}; ++#endif ++ ++int mali_sysfs_register(const char *mali_dev_name) ++{ ++ mali_debugfs_dir = debugfs_create_dir(mali_dev_name, NULL); ++ if (ERR_PTR(-ENODEV) == mali_debugfs_dir) { ++ /* Debugfs not supported. */ ++ mali_debugfs_dir = NULL; ++ } else { ++ if (NULL != mali_debugfs_dir) { ++ /* Debugfs directory created successfully; create files now */ ++ struct dentry *mali_power_dir; ++ struct dentry *mali_gp_dir; ++ struct dentry *mali_pp_dir; ++ struct dentry *mali_l2_dir; ++ struct dentry *mali_profiling_dir; ++ ++ debugfs_create_file("version", 0400, mali_debugfs_dir, NULL, &version_fops); ++ ++ mali_power_dir = debugfs_create_dir("power", mali_debugfs_dir); ++ if (mali_power_dir != NULL) { ++ debugfs_create_file("always_on", 0600, mali_power_dir, NULL, &power_always_on_fops); ++ debugfs_create_file("power_events", 0200, mali_power_dir, NULL, &power_power_events_fops); ++ } ++ ++ mali_gp_dir = debugfs_create_dir("gp", mali_debugfs_dir); ++ if (mali_gp_dir != NULL) { ++ u32 num_groups; ++ long i; ++ ++ num_groups = mali_group_get_glob_num_groups(); ++ for (i = 0; i < num_groups; i++) { ++ struct mali_group *group = mali_group_get_glob_group(i); ++ ++ struct mali_gp_core *gp_core = mali_group_get_gp_core(group); ++ if (NULL != gp_core) { ++ struct dentry *mali_gp_gpx_dir; ++ mali_gp_gpx_dir = debugfs_create_dir("gp0", mali_gp_dir); ++ if (NULL != mali_gp_gpx_dir) { ++ debugfs_create_file("base_addr", 0400, mali_gp_gpx_dir, &gp_core->hw_core, &hw_core_base_addr_fops); ++ debugfs_create_file("enabled", 0600, mali_gp_gpx_dir, group, &group_enabled_fops); ++ } ++ break; /* no need to look for any other GP cores */ ++ } ++ ++ } ++ } ++ ++ mali_pp_dir = debugfs_create_dir("pp", mali_debugfs_dir); ++ if (mali_pp_dir != NULL) { ++ u32 num_groups; ++ long i; ++ ++ debugfs_create_file("num_cores_total", 0400, mali_pp_dir, NULL, &pp_num_cores_total_fops); ++ debugfs_create_file("num_cores_enabled", 0600, mali_pp_dir, NULL, &pp_num_cores_enabled_fops); ++ debugfs_create_file("core_scaling_enabled", 0600, mali_pp_dir, NULL, &pp_core_scaling_enabled_fops); ++ ++ num_groups = mali_group_get_glob_num_groups(); ++ for (i = 0; i < num_groups; i++) { ++ struct mali_group *group = mali_group_get_glob_group(i); ++ ++ struct mali_pp_core *pp_core = mali_group_get_pp_core(group); ++ if (NULL != pp_core) { ++ char buf[16]; ++ struct dentry *mali_pp_ppx_dir; ++ _mali_osk_snprintf(buf, sizeof(buf), "pp%u", mali_pp_core_get_id(pp_core)); ++ mali_pp_ppx_dir = debugfs_create_dir(buf, mali_pp_dir); ++ if (NULL != mali_pp_ppx_dir) { ++ debugfs_create_file("base_addr", 0400, mali_pp_ppx_dir, &pp_core->hw_core, &hw_core_base_addr_fops); ++ if (!mali_group_is_virtual(group)) { ++ debugfs_create_file("enabled", 0600, mali_pp_ppx_dir, group, &group_enabled_fops); ++ } ++ } ++ } ++ } ++ } ++ ++ mali_l2_dir = debugfs_create_dir("l2", mali_debugfs_dir); ++ if (mali_l2_dir != NULL) { ++ struct dentry *mali_l2_all_dir; ++ u32 l2_id; ++ struct mali_l2_cache_core *l2_cache; ++ ++ mali_l2_all_dir = debugfs_create_dir("all", mali_l2_dir); ++ if (mali_l2_all_dir != NULL) { ++ debugfs_create_file("counter_src0", 0200, mali_l2_all_dir, NULL, &l2_all_counter_src0_fops); ++ debugfs_create_file("counter_src1", 0200, mali_l2_all_dir, NULL, &l2_all_counter_src1_fops); ++ } ++ ++ l2_id = 0; ++ l2_cache = mali_l2_cache_core_get_glob_l2_core(l2_id); ++ while (NULL != l2_cache) { ++ char buf[16]; ++ struct dentry *mali_l2_l2x_dir; ++ _mali_osk_snprintf(buf, sizeof(buf), "l2%u", l2_id); ++ mali_l2_l2x_dir = debugfs_create_dir(buf, mali_l2_dir); ++ if (NULL != mali_l2_l2x_dir) { ++ debugfs_create_file("counter_src0", 0600, mali_l2_l2x_dir, l2_cache, &l2_l2x_counter_src0_fops); ++ debugfs_create_file("counter_src1", 0600, mali_l2_l2x_dir, l2_cache, &l2_l2x_counter_src1_fops); ++ debugfs_create_file("counter_val0", 0600, mali_l2_l2x_dir, l2_cache, &l2_l2x_counter_val0_fops); ++ debugfs_create_file("counter_val1", 0600, mali_l2_l2x_dir, l2_cache, &l2_l2x_counter_val1_fops); ++ debugfs_create_file("base_addr", 0400, mali_l2_l2x_dir, &l2_cache->hw_core, &hw_core_base_addr_fops); ++ } ++ ++ /* try next L2 */ ++ l2_id++; ++ l2_cache = mali_l2_cache_core_get_glob_l2_core(l2_id); ++ } ++ } ++ ++ debugfs_create_file("gpu_memory", 0444, mali_debugfs_dir, NULL, &memory_usage_fops); ++ ++ debugfs_create_file("utilization_gp_pp", 0400, mali_debugfs_dir, NULL, &utilization_gp_pp_fops); ++ debugfs_create_file("utilization_gp", 0400, mali_debugfs_dir, NULL, &utilization_gp_fops); ++ debugfs_create_file("utilization_pp", 0400, mali_debugfs_dir, NULL, &utilization_pp_fops); ++ ++ mali_profiling_dir = debugfs_create_dir("profiling", mali_debugfs_dir); ++ if (mali_profiling_dir != NULL) { ++ u32 max_sub_jobs; ++ long i; ++ struct dentry *mali_profiling_gp_dir; ++ struct dentry *mali_profiling_pp_dir; ++#if defined(CONFIG_MALI400_INTERNAL_PROFILING) ++ struct dentry *mali_profiling_proc_dir; ++#endif ++ /* ++ * Create directory where we can set GP HW counters. ++ */ ++ mali_profiling_gp_dir = debugfs_create_dir("gp", mali_profiling_dir); ++ if (mali_profiling_gp_dir != NULL) { ++ debugfs_create_file("counter_src0", 0600, mali_profiling_gp_dir, (void *)PRIVATE_DATA_COUNTER_MAKE_GP(0), &profiling_counter_src_fops); ++ debugfs_create_file("counter_src1", 0600, mali_profiling_gp_dir, (void *)PRIVATE_DATA_COUNTER_MAKE_GP(1), &profiling_counter_src_fops); ++ } ++ ++ /* ++ * Create directory where we can set PP HW counters. ++ * Possible override with specific HW counters for a particular sub job ++ * (Disable core scaling before using the override!) ++ */ ++ mali_profiling_pp_dir = debugfs_create_dir("pp", mali_profiling_dir); ++ if (mali_profiling_pp_dir != NULL) { ++ debugfs_create_file("counter_src0", 0600, mali_profiling_pp_dir, (void *)PRIVATE_DATA_COUNTER_MAKE_PP(0), &profiling_counter_src_fops); ++ debugfs_create_file("counter_src1", 0600, mali_profiling_pp_dir, (void *)PRIVATE_DATA_COUNTER_MAKE_PP(1), &profiling_counter_src_fops); ++ } ++ ++ max_sub_jobs = mali_executor_get_num_cores_total(); ++ for (i = 0; i < max_sub_jobs; i++) { ++ char buf[16]; ++ struct dentry *mali_profiling_pp_x_dir; ++ _mali_osk_snprintf(buf, sizeof(buf), "%u", i); ++ mali_profiling_pp_x_dir = debugfs_create_dir(buf, mali_profiling_pp_dir); ++ if (NULL != mali_profiling_pp_x_dir) { ++ debugfs_create_file("counter_src0", ++ 0600, mali_profiling_pp_x_dir, ++ (void *)PRIVATE_DATA_COUNTER_MAKE_PP_SUB_JOB(0, i), ++ &profiling_counter_src_fops); ++ debugfs_create_file("counter_src1", ++ 0600, mali_profiling_pp_x_dir, ++ (void *)PRIVATE_DATA_COUNTER_MAKE_PP_SUB_JOB(1, i), ++ &profiling_counter_src_fops); ++ } ++ } ++ ++#if defined(CONFIG_MALI400_INTERNAL_PROFILING) ++ mali_profiling_proc_dir = debugfs_create_dir("proc", mali_profiling_dir); ++ if (mali_profiling_proc_dir != NULL) { ++ struct dentry *mali_profiling_proc_default_dir = debugfs_create_dir("default", mali_profiling_proc_dir); ++ if (mali_profiling_proc_default_dir != NULL) { ++ debugfs_create_file("enable", 0600, mali_profiling_proc_default_dir, (void *)_MALI_UK_USER_SETTING_SW_EVENTS_ENABLE, &user_settings_fops); ++ } ++ } ++ debugfs_create_file("record", 0600, mali_profiling_dir, NULL, &profiling_record_fops); ++ debugfs_create_file("events", 0400, mali_profiling_dir, NULL, &profiling_events_fops); ++ debugfs_create_file("events_human_readable", 0400, mali_profiling_dir, NULL, &profiling_events_human_readable_fops); ++#endif ++ } ++ ++#if MALI_STATE_TRACKING ++ debugfs_create_file("state_dump", 0400, mali_debugfs_dir, NULL, &mali_seq_internal_state_fops); ++#endif ++ ++#if defined(DEBUG) ++ debugfs_create_file("timeline_dump", 0400, mali_debugfs_dir, NULL, &timeline_dump_fops); ++#endif ++ if (mali_sysfs_user_settings_register()) { ++ /* Failed to create the debugfs entries for the user settings DB. */ ++ MALI_DEBUG_PRINT(2, ("Failed to create user setting debugfs files. Ignoring...\n")); ++ } ++ } ++ } ++ ++ /* Success! */ ++ return 0; ++} ++ ++int mali_sysfs_unregister(void) ++{ ++ if (NULL != mali_debugfs_dir) { ++ debugfs_remove_recursive(mali_debugfs_dir); ++ } ++ return 0; ++} ++ ++#else /* MALI_LICENSE_IS_GPL */ ++ ++/* Dummy implementations for non-GPL */ ++ ++int mali_sysfs_register(struct mali_dev *device, dev_t dev, const char *mali_dev_name) ++{ ++ return 0; ++} ++ ++int mali_sysfs_unregister(void) ++{ ++ return 0; ++} ++ ++#endif /* MALI_LICENSE_IS_GPL */ +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_kernel_sysfs.h b/drivers/gpu/arm/mali400/mali/linux/mali_kernel_sysfs.h +new file mode 100755 +index 000000000000..91580a87c1e1 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_kernel_sysfs.h +@@ -0,0 +1,29 @@ ++/* ++ * Copyright (C) 2011-2013, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_KERNEL_SYSFS_H__ ++#define __MALI_KERNEL_SYSFS_H__ ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++#include ++ ++#define MALI_PROC_DIR "driver/mali" ++ ++int mali_sysfs_register(const char *mali_dev_name); ++int mali_sysfs_unregister(void); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif /* __MALI_KERNEL_LINUX_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_linux_trace.h b/drivers/gpu/arm/mali400/mali/linux/mali_linux_trace.h +new file mode 100755 +index 000000000000..222260823c81 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_linux_trace.h +@@ -0,0 +1,161 @@ ++/* ++ * Copyright (C) 2012-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#if !defined (MALI_LINUX_TRACE_H) || defined (TRACE_HEADER_MULTI_READ) ++#define MALI_LINUX_TRACE_H ++ ++#include ++ ++#include ++#include ++ ++#undef TRACE_SYSTEM ++#define TRACE_SYSTEM mali ++ ++#define TRACE_INCLUDE_PATH . ++#define TRACE_INCLUDE_FILE mali_linux_trace ++ ++/** ++ * Define the tracepoint used to communicate the status of a GPU. Called ++ * when a GPU turns on or turns off. ++ * ++ * @param event_id The type of the event. This parameter is a bitfield ++ * encoding the type of the event. ++ * ++ * @param d0 First data parameter. ++ * @param d1 Second data parameter. ++ * @param d2 Third data parameter. ++ * @param d3 Fourth data parameter. ++ * @param d4 Fifth data parameter. ++ */ ++TRACE_EVENT(mali_timeline_event, ++ ++ TP_PROTO(unsigned int event_id, unsigned int d0, unsigned int d1, ++ unsigned int d2, unsigned int d3, unsigned int d4), ++ ++ TP_ARGS(event_id, d0, d1, d2, d3, d4), ++ ++ TP_STRUCT__entry( ++ __field(unsigned int, event_id) ++ __field(unsigned int, d0) ++ __field(unsigned int, d1) ++ __field(unsigned int, d2) ++ __field(unsigned int, d3) ++ __field(unsigned int, d4) ++ ), ++ ++ TP_fast_assign( ++ __entry->event_id = event_id; ++ __entry->d0 = d0; ++ __entry->d1 = d1; ++ __entry->d2 = d2; ++ __entry->d3 = d3; ++ __entry->d4 = d4; ++ ), ++ ++ TP_printk("event=%d", __entry->event_id) ++ ); ++ ++/** ++ * Define a tracepoint used to regsiter the value of a hardware counter. ++ * Hardware counters belonging to the vertex or fragment processor are ++ * reported via this tracepoint each frame, whilst L2 cache hardware ++ * counters are reported continuously. ++ * ++ * @param counter_id The counter ID. ++ * @param value The value of the counter. ++ */ ++TRACE_EVENT(mali_hw_counter, ++ ++ TP_PROTO(unsigned int counter_id, unsigned int value), ++ ++ TP_ARGS(counter_id, value), ++ ++ TP_STRUCT__entry( ++ __field(unsigned int, counter_id) ++ __field(unsigned int, value) ++ ), ++ ++ TP_fast_assign( ++ __entry->counter_id = counter_id; ++ ), ++ ++ TP_printk("event %d = %d", __entry->counter_id, __entry->value) ++ ); ++ ++/** ++ * Define a tracepoint used to send a bundle of software counters. ++ * ++ * @param counters The bundle of counters. ++ */ ++TRACE_EVENT(mali_sw_counters, ++ ++ TP_PROTO(pid_t pid, pid_t tid, void *surface_id, unsigned int *counters), ++ ++ TP_ARGS(pid, tid, surface_id, counters), ++ ++ TP_STRUCT__entry( ++ __field(pid_t, pid) ++ __field(pid_t, tid) ++ __field(void *, surface_id) ++ __field(unsigned int *, counters) ++ ), ++ ++ TP_fast_assign( ++ __entry->pid = pid; ++ __entry->tid = tid; ++ __entry->surface_id = surface_id; ++ __entry->counters = counters; ++ ), ++ ++ TP_printk("counters were %s", __entry->counters == NULL ? "NULL" : "not NULL") ++ ); ++ ++/** ++ * Define a tracepoint used to gather core activity for systrace ++ * @param pid The process id for which the core activity originates from ++ * @param active If the core is active (1) or not (0) ++ * @param core_type The type of core active, either GP (1) or PP (0) ++ * @param core_id The core id that is active for the core_type ++ * @param frame_builder_id The frame builder id associated with this core activity ++ * @param flush_id The flush id associated with this core activity ++ */ ++TRACE_EVENT(mali_core_active, ++ ++ TP_PROTO(pid_t pid, unsigned int active, unsigned int core_type, unsigned int core_id, unsigned int frame_builder_id, unsigned int flush_id), ++ ++ TP_ARGS(pid, active, core_type, core_id, frame_builder_id, flush_id), ++ ++ TP_STRUCT__entry( ++ __field(pid_t, pid) ++ __field(unsigned int, active) ++ __field(unsigned int, core_type) ++ __field(unsigned int, core_id) ++ __field(unsigned int, frame_builder_id) ++ __field(unsigned int, flush_id) ++ ), ++ ++ TP_fast_assign( ++ __entry->pid = pid; ++ __entry->active = active; ++ __entry->core_type = core_type; ++ __entry->core_id = core_id; ++ __entry->frame_builder_id = frame_builder_id; ++ __entry->flush_id = flush_id; ++ ), ++ ++ TP_printk("%s|%d|%s%i:%x|%d", __entry->active ? "S" : "F", __entry->pid, __entry->core_type ? "GP" : "PP", __entry->core_id, __entry->flush_id, __entry->frame_builder_id) ++ ); ++ ++#endif /* MALI_LINUX_TRACE_H */ ++ ++/* This part must exist outside the header guard. */ ++#include ++ +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory.c +new file mode 100755 +index 000000000000..dfc769e6cc40 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory.c +@@ -0,0 +1,531 @@ ++/* ++ * Copyright (C) 2013-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "mali_osk.h" ++#include "mali_executor.h" ++ ++#include "mali_memory.h" ++#include "mali_memory_os_alloc.h" ++#include "mali_memory_block_alloc.h" ++#include "mali_memory_util.h" ++#include "mali_memory_virtual.h" ++#include "mali_memory_manager.h" ++#include "mali_memory_cow.h" ++#include "mali_memory_swap_alloc.h" ++#include "mali_memory_defer_bind.h" ++#if defined(CONFIG_DMA_SHARED_BUFFER) ++#include "mali_memory_secure.h" ++#endif ++ ++extern unsigned int mali_dedicated_mem_size; ++extern unsigned int mali_shared_mem_size; ++ ++#define MALI_VM_NUM_FAULT_PREFETCH (0x8) ++ ++static void mali_mem_vma_open(struct vm_area_struct *vma) ++{ ++ mali_mem_allocation *alloc = (mali_mem_allocation *)vma->vm_private_data; ++ MALI_DEBUG_PRINT(4, ("Open called on vma %p\n", vma)); ++ ++ /* If need to share the allocation, add ref_count here */ ++ mali_allocation_ref(alloc); ++ return; ++} ++static void mali_mem_vma_close(struct vm_area_struct *vma) ++{ ++ /* If need to share the allocation, unref ref_count here */ ++ mali_mem_allocation *alloc = (mali_mem_allocation *)vma->vm_private_data; ++ ++ mali_allocation_unref(&alloc); ++ vma->vm_private_data = NULL; ++} ++ ++static vm_fault_t mali_mem_vma_fault(struct vm_fault *vmf) ++{ ++ struct vm_area_struct *vma = vmf->vma; ++ mali_mem_allocation *alloc = (mali_mem_allocation *)vma->vm_private_data; ++ mali_mem_backend *mem_bkend = NULL; ++ int ret; ++ int prefetch_num = MALI_VM_NUM_FAULT_PREFETCH; ++ ++ unsigned long address = (unsigned long)vmf->address; ++ MALI_DEBUG_ASSERT(alloc->backend_handle); ++ MALI_DEBUG_ASSERT((unsigned long)alloc->cpu_mapping.addr <= address); ++ ++ /* Get backend memory & Map on CPU */ ++ mutex_lock(&mali_idr_mutex); ++ if (!(mem_bkend = idr_find(&mali_backend_idr, alloc->backend_handle))) { ++ MALI_DEBUG_PRINT(1, ("Can't find memory backend in mmap!\n")); ++ mutex_unlock(&mali_idr_mutex); ++ return VM_FAULT_SIGBUS; ++ } ++ mutex_unlock(&mali_idr_mutex); ++ MALI_DEBUG_ASSERT(mem_bkend->type == alloc->type); ++ ++ if ((mem_bkend->type == MALI_MEM_COW && (MALI_MEM_BACKEND_FLAG_SWAP_COWED != ++ (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED))) && ++ (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_COW_CPU_NO_WRITE)) { ++ /*check if use page fault to do COW*/ ++ MALI_DEBUG_PRINT(4, ("mali_vma_fault: do cow allocate on demand!, address=0x%x\n", address)); ++ mutex_lock(&mem_bkend->mutex); ++ ret = mali_mem_cow_allocate_on_demand(mem_bkend, ++ (address - vma->vm_start) / PAGE_SIZE); ++ mutex_unlock(&mem_bkend->mutex); ++ ++ if (ret != _MALI_OSK_ERR_OK) { ++ return VM_FAULT_OOM; ++ } ++ prefetch_num = 1; ++ ++ /* handle COW modified range cpu mapping ++ we zap the mapping in cow_modify_range, it will trigger page fault ++ when CPU access it, so here we map it to CPU*/ ++ mutex_lock(&mem_bkend->mutex); ++ ret = mali_mem_cow_cpu_map_pages_locked(mem_bkend, vma, address, prefetch_num); ++ mutex_unlock(&mem_bkend->mutex); ++ ++ if (unlikely(ret != _MALI_OSK_ERR_OK)) { ++ return VM_FAULT_SIGBUS; ++ } ++ } else if ((mem_bkend->type == MALI_MEM_SWAP) || ++ (mem_bkend->type == MALI_MEM_COW && (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED))) { ++ u32 offset_in_bkend = (address - vma->vm_start) / PAGE_SIZE; ++ int ret = _MALI_OSK_ERR_OK; ++ ++ mutex_lock(&mem_bkend->mutex); ++ if (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_COW_CPU_NO_WRITE) { ++ ret = mali_mem_swap_cow_page_on_demand(mem_bkend, offset_in_bkend, &vmf->page); ++ } else { ++ ret = mali_mem_swap_allocate_page_on_demand(mem_bkend, offset_in_bkend, &vmf->page); ++ } ++ mutex_unlock(&mem_bkend->mutex); ++ ++ if (ret != _MALI_OSK_ERR_OK) { ++ MALI_DEBUG_PRINT(2, ("Mali swap memory page fault process failed, address=0x%x\n", address)); ++ return VM_FAULT_OOM; ++ } else { ++ return VM_FAULT_LOCKED; ++ } ++ } else { ++ MALI_PRINT_ERROR(("Mali vma fault! It never happen, indicating some logic errors in caller.\n")); ++ /*NOT support yet or OOM*/ ++ return VM_FAULT_OOM; ++ } ++ return VM_FAULT_NOPAGE; ++} ++ ++static struct vm_operations_struct mali_kernel_vm_ops = { ++ .open = mali_mem_vma_open, ++ .close = mali_mem_vma_close, ++ .fault = mali_mem_vma_fault, ++}; ++ ++ ++/** @ map mali allocation to CPU address ++* ++* Supported backend types: ++* --MALI_MEM_OS ++* -- need to add COW? ++ *Not supported backend types: ++* -_MALI_MEMORY_BIND_BACKEND_UMP ++* -_MALI_MEMORY_BIND_BACKEND_DMA_BUF ++* -_MALI_MEMORY_BIND_BACKEND_EXTERNAL_MEMORY ++* ++*/ ++int mali_mmap(struct file *filp, struct vm_area_struct *vma) ++{ ++ struct mali_session_data *session; ++ mali_mem_allocation *mali_alloc = NULL; ++ u32 mali_addr = vma->vm_pgoff << PAGE_SHIFT; ++ struct mali_vma_node *mali_vma_node = NULL; ++ mali_mem_backend *mem_bkend = NULL; ++ int ret = -EFAULT; ++ ++ session = (struct mali_session_data *)filp->private_data; ++ if (NULL == session) { ++ MALI_PRINT_ERROR(("mmap called without any session data available\n")); ++ return -EFAULT; ++ } ++ ++ MALI_DEBUG_PRINT(4, ("MMap() handler: start=0x%08X, phys=0x%08X, size=0x%08X vma->flags 0x%08x\n", ++ (unsigned int)vma->vm_start, (unsigned int)(vma->vm_pgoff << PAGE_SHIFT), ++ (unsigned int)(vma->vm_end - vma->vm_start), vma->vm_flags)); ++ ++ /* Operations used on any memory system */ ++ /* do not need to anything in vm open/close now */ ++ ++ /* find mali allocation structure by vaddress*/ ++ mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_addr, 0); ++ if (likely(mali_vma_node)) { ++ mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); ++ MALI_DEBUG_ASSERT(mali_addr == mali_vma_node->vm_node.start); ++ if (unlikely(mali_addr != mali_vma_node->vm_node.start)) { ++ /* only allow to use start address for mmap */ ++ MALI_DEBUG_PRINT(1, ("mali_addr != mali_vma_node->vm_node.start\n")); ++ return -EFAULT; ++ } ++ } else { ++ MALI_DEBUG_ASSERT(NULL == mali_vma_node); ++ return -EFAULT; ++ } ++ ++ mali_alloc->cpu_mapping.addr = (void __user *)vma->vm_start; ++ ++ if (mali_alloc->flags & _MALI_MEMORY_ALLOCATE_DEFER_BIND) { ++ MALI_DEBUG_PRINT(1, ("ERROR : trying to access varying memory by CPU!\n")); ++ return -EFAULT; ++ } ++ ++ /* Get backend memory & Map on CPU */ ++ mutex_lock(&mali_idr_mutex); ++ if (!(mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle))) { ++ MALI_DEBUG_PRINT(1, ("Can't find memory backend in mmap!\n")); ++ mutex_unlock(&mali_idr_mutex); ++ return -EFAULT; ++ } ++ mutex_unlock(&mali_idr_mutex); ++ ++ if (!(MALI_MEM_SWAP == mali_alloc->type || ++ (MALI_MEM_COW == mali_alloc->type && (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED)))) { ++ /* Set some bits which indicate that, the memory is IO memory, meaning ++ * that no paging is to be performed and the memory should not be ++ * included in crash dumps. And that the memory is reserved, meaning ++ * that it's present and can never be paged out (see also previous ++ * entry) ++ */ ++ vma->vm_flags |= VM_IO; ++ vma->vm_flags |= VM_DONTCOPY; ++ vma->vm_flags |= VM_PFNMAP; ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 7, 0) ++ vma->vm_flags |= VM_RESERVED; ++#else ++ vma->vm_flags |= VM_DONTDUMP; ++ vma->vm_flags |= VM_DONTEXPAND; ++#endif ++ } else if (MALI_MEM_SWAP == mali_alloc->type) { ++ vma->vm_pgoff = mem_bkend->start_idx; ++ } ++ ++ vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); ++ vma->vm_ops = &mali_kernel_vm_ops; ++ ++ mali_alloc->cpu_mapping.addr = (void __user *)vma->vm_start; ++ ++ /* If it's a copy-on-write mapping, map to read only */ ++ if (!(vma->vm_flags & VM_WRITE)) { ++ MALI_DEBUG_PRINT(4, ("mmap allocation with read only !\n")); ++ /* add VM_WRITE for do_page_fault will check this when a write fault */ ++ vma->vm_flags |= VM_WRITE | VM_READ; ++ vma->vm_page_prot = PAGE_READONLY; ++ vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); ++ mem_bkend->flags |= MALI_MEM_BACKEND_FLAG_COW_CPU_NO_WRITE; ++ goto out; ++ } ++ ++ if (mem_bkend->type == MALI_MEM_OS) { ++ ret = mali_mem_os_cpu_map(mem_bkend, vma); ++ } else if (mem_bkend->type == MALI_MEM_COW && ++ (MALI_MEM_BACKEND_FLAG_SWAP_COWED != (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED))) { ++ ret = mali_mem_cow_cpu_map(mem_bkend, vma); ++ } else if (mem_bkend->type == MALI_MEM_BLOCK) { ++ ret = mali_mem_block_cpu_map(mem_bkend, vma); ++ } else if ((mem_bkend->type == MALI_MEM_SWAP) || (mem_bkend->type == MALI_MEM_COW && ++ (MALI_MEM_BACKEND_FLAG_SWAP_COWED == (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED)))) { ++ /*For swappable memory, CPU page table will be created by page fault handler. */ ++ ret = 0; ++ } else if (mem_bkend->type == MALI_MEM_SECURE) { ++#if defined(CONFIG_DMA_SHARED_BUFFER) ++ ret = mali_mem_secure_cpu_map(mem_bkend, vma); ++#else ++ MALI_DEBUG_PRINT(1, ("DMA not supported for mali secure memory\n")); ++ return -EFAULT; ++#endif ++ } else { ++ /* Not support yet*/ ++ MALI_DEBUG_PRINT_ERROR(("Invalid type of backend memory! \n")); ++ return -EFAULT; ++ } ++ ++ if (ret != 0) { ++ MALI_DEBUG_PRINT(1, ("ret != 0\n")); ++ return -EFAULT; ++ } ++out: ++ MALI_DEBUG_ASSERT(MALI_MEM_ALLOCATION_VALID_MAGIC == mali_alloc->magic); ++ ++ vma->vm_private_data = (void *)mali_alloc; ++ mali_alloc->cpu_mapping.vma = vma; ++ ++ mali_allocation_ref(mali_alloc); ++ ++ return 0; ++} ++ ++_mali_osk_errcode_t mali_mem_mali_map_prepare(mali_mem_allocation *descriptor) ++{ ++ u32 size = descriptor->psize; ++ struct mali_session_data *session = descriptor->session; ++ ++ MALI_DEBUG_ASSERT(MALI_MEM_ALLOCATION_VALID_MAGIC == descriptor->magic); ++ ++ /* Map dma-buf into this session's page tables */ ++ ++ if (descriptor->flags & MALI_MEM_FLAG_MALI_GUARD_PAGE) { ++ size += MALI_MMU_PAGE_SIZE; ++ } ++ ++ return mali_mmu_pagedir_map(session->page_directory, descriptor->mali_vma_node.vm_node.start, size); ++} ++ ++_mali_osk_errcode_t mali_mem_mali_map_resize(mali_mem_allocation *descriptor, u32 new_size) ++{ ++ u32 old_size = descriptor->psize; ++ struct mali_session_data *session = descriptor->session; ++ ++ MALI_DEBUG_ASSERT(MALI_MEM_ALLOCATION_VALID_MAGIC == descriptor->magic); ++ ++ if (descriptor->flags & MALI_MEM_FLAG_MALI_GUARD_PAGE) { ++ new_size += MALI_MMU_PAGE_SIZE; ++ } ++ ++ if (new_size > old_size) { ++ MALI_DEBUG_ASSERT(new_size <= descriptor->mali_vma_node.vm_node.size); ++ return mali_mmu_pagedir_map(session->page_directory, descriptor->mali_vma_node.vm_node.start + old_size, new_size - old_size); ++ } ++ return _MALI_OSK_ERR_OK; ++} ++ ++void mali_mem_mali_map_free(struct mali_session_data *session, u32 size, mali_address_t vaddr, u32 flags) ++{ ++ if (flags & MALI_MEM_FLAG_MALI_GUARD_PAGE) { ++ size += MALI_MMU_PAGE_SIZE; ++ } ++ ++ /* Umap and flush L2 */ ++ mali_mmu_pagedir_unmap(session->page_directory, vaddr, size); ++ mali_executor_zap_all_active(session); ++} ++ ++u32 _mali_ukk_report_memory_usage(void) ++{ ++ u32 sum = 0; ++ ++ if (MALI_TRUE == mali_memory_have_dedicated_memory()) { ++ sum += mali_mem_block_allocator_stat(); ++ } ++ ++ sum += mali_mem_os_stat(); ++ ++ return sum; ++} ++ ++u32 _mali_ukk_report_total_memory_size(void) ++{ ++ return mali_dedicated_mem_size + mali_shared_mem_size; ++} ++ ++ ++/** ++ * Per-session memory descriptor mapping table sizes ++ */ ++#define MALI_MEM_DESCRIPTORS_INIT 64 ++#define MALI_MEM_DESCRIPTORS_MAX 65536 ++ ++_mali_osk_errcode_t mali_memory_session_begin(struct mali_session_data *session_data) ++{ ++ MALI_DEBUG_PRINT(5, ("Memory session begin\n")); ++ ++ session_data->memory_lock = _mali_osk_mutex_init(_MALI_OSK_LOCKFLAG_ORDERED, ++ _MALI_OSK_LOCK_ORDER_MEM_SESSION); ++ ++ if (NULL == session_data->memory_lock) { ++ MALI_ERROR(_MALI_OSK_ERR_FAULT); ++ } ++ ++ session_data->cow_lock = _mali_osk_mutex_init(_MALI_OSK_LOCKFLAG_UNORDERED, 0); ++ if (NULL == session_data->cow_lock) { ++ _mali_osk_mutex_term(session_data->memory_lock); ++ MALI_ERROR(_MALI_OSK_ERR_FAULT); ++ } ++ ++ mali_memory_manager_init(&session_data->allocation_mgr); ++ ++ MALI_DEBUG_PRINT(5, ("MMU session begin: success\n")); ++ MALI_SUCCESS; ++} ++ ++void mali_memory_session_end(struct mali_session_data *session) ++{ ++ MALI_DEBUG_PRINT(3, ("MMU session end\n")); ++ ++ if (NULL == session) { ++ MALI_DEBUG_PRINT(1, ("No session data found during session end\n")); ++ return; ++ } ++ /* free allocation */ ++ mali_free_session_allocations(session); ++ /* do some check in unint*/ ++ mali_memory_manager_uninit(&session->allocation_mgr); ++ ++ /* Free the lock */ ++ _mali_osk_mutex_term(session->memory_lock); ++ _mali_osk_mutex_term(session->cow_lock); ++ return; ++} ++ ++_mali_osk_errcode_t mali_memory_initialize(void) ++{ ++ _mali_osk_errcode_t err; ++ ++ idr_init(&mali_backend_idr); ++ mutex_init(&mali_idr_mutex); ++ ++ err = mali_mem_swap_init(); ++ if (err != _MALI_OSK_ERR_OK) { ++ return err; ++ } ++ err = mali_mem_os_init(); ++ if (_MALI_OSK_ERR_OK == err) { ++ err = mali_mem_defer_bind_manager_init(); ++ } ++ ++ return err; ++} ++ ++void mali_memory_terminate(void) ++{ ++ mali_mem_swap_term(); ++ mali_mem_defer_bind_manager_destory(); ++ mali_mem_os_term(); ++ if (mali_memory_have_dedicated_memory()) { ++ mali_mem_block_allocator_destroy(); ++ } ++} ++ ++ ++struct mali_page_node *_mali_page_node_allocate(mali_page_node_type type) ++{ ++ mali_page_node *page_node = NULL; ++ ++ page_node = kzalloc(sizeof(mali_page_node), GFP_KERNEL); ++ MALI_DEBUG_ASSERT(NULL != page_node); ++ ++ if (page_node) { ++ page_node->type = type; ++ INIT_LIST_HEAD(&page_node->list); ++ } ++ ++ return page_node; ++} ++ ++void _mali_page_node_ref(struct mali_page_node *node) ++{ ++ if (node->type == MALI_PAGE_NODE_OS) { ++ /* add ref to this page */ ++ get_page(node->page); ++ } else if (node->type == MALI_PAGE_NODE_BLOCK) { ++ mali_mem_block_add_ref(node); ++ } else if (node->type == MALI_PAGE_NODE_SWAP) { ++ atomic_inc(&node->swap_it->ref_count); ++ } else { ++ MALI_DEBUG_PRINT_ERROR(("Invalid type of mali page node! \n")); ++ } ++} ++ ++void _mali_page_node_unref(struct mali_page_node *node) ++{ ++ if (node->type == MALI_PAGE_NODE_OS) { ++ /* unref to this page */ ++ put_page(node->page); ++ } else if (node->type == MALI_PAGE_NODE_BLOCK) { ++ mali_mem_block_dec_ref(node); ++ } else { ++ MALI_DEBUG_PRINT_ERROR(("Invalid type of mali page node! \n")); ++ } ++} ++ ++ ++void _mali_page_node_add_page(struct mali_page_node *node, struct page *page) ++{ ++ MALI_DEBUG_ASSERT(MALI_PAGE_NODE_OS == node->type); ++ node->page = page; ++} ++ ++ ++void _mali_page_node_add_swap_item(struct mali_page_node *node, struct mali_swap_item *item) ++{ ++ MALI_DEBUG_ASSERT(MALI_PAGE_NODE_SWAP == node->type); ++ node->swap_it = item; ++} ++ ++void _mali_page_node_add_block_item(struct mali_page_node *node, mali_block_item *item) ++{ ++ MALI_DEBUG_ASSERT(MALI_PAGE_NODE_BLOCK == node->type); ++ node->blk_it = item; ++} ++ ++ ++int _mali_page_node_get_ref_count(struct mali_page_node *node) ++{ ++ if (node->type == MALI_PAGE_NODE_OS) { ++ /* get ref count of this page */ ++ return page_count(node->page); ++ } else if (node->type == MALI_PAGE_NODE_BLOCK) { ++ return mali_mem_block_get_ref_count(node); ++ } else if (node->type == MALI_PAGE_NODE_SWAP) { ++ return atomic_read(&node->swap_it->ref_count); ++ } else { ++ MALI_DEBUG_PRINT_ERROR(("Invalid type of mali page node! \n")); ++ } ++ return -1; ++} ++ ++ ++dma_addr_t _mali_page_node_get_dma_addr(struct mali_page_node *node) ++{ ++ if (node->type == MALI_PAGE_NODE_OS) { ++ return page_private(node->page); ++ } else if (node->type == MALI_PAGE_NODE_BLOCK) { ++ return _mali_blk_item_get_phy_addr(node->blk_it); ++ } else if (node->type == MALI_PAGE_NODE_SWAP) { ++ return node->swap_it->dma_addr; ++ } else { ++ MALI_DEBUG_PRINT_ERROR(("Invalid type of mali page node! \n")); ++ } ++ return 0; ++} ++ ++ ++unsigned long _mali_page_node_get_pfn(struct mali_page_node *node) ++{ ++ if (node->type == MALI_PAGE_NODE_OS) { ++ return page_to_pfn(node->page); ++ } else if (node->type == MALI_PAGE_NODE_BLOCK) { ++ /* get phy addr for BLOCK page*/ ++ return _mali_blk_item_get_pfn(node->blk_it); ++ } else if (node->type == MALI_PAGE_NODE_SWAP) { ++ return page_to_pfn(node->swap_it->page); ++ } else { ++ MALI_DEBUG_PRINT_ERROR(("Invalid type of mali page node! \n")); ++ } ++ return 0; ++} ++ ++ +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory.h +new file mode 100755 +index 000000000000..efebbef235d8 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory.h +@@ -0,0 +1,143 @@ ++/* ++ * Copyright (C) 2013-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_MEMORY_H__ ++#define __MALI_MEMORY_H__ ++ ++#include "mali_osk.h" ++#include "mali_session.h" ++ ++#include ++#include ++ ++#include "mali_memory_types.h" ++#include "mali_memory_os_alloc.h" ++ ++_mali_osk_errcode_t mali_memory_initialize(void); ++void mali_memory_terminate(void); ++ ++/** @brief Allocate a page table page ++ * ++ * Allocate a page for use as a page directory or page table. The page is ++ * mapped into kernel space. ++ * ++ * @return _MALI_OSK_ERR_OK on success, otherwise an error code ++ * @param table_page GPU pointer to the allocated page ++ * @param mapping CPU pointer to the mapping of the allocated page ++ */ ++MALI_STATIC_INLINE _mali_osk_errcode_t ++mali_mmu_get_table_page(mali_dma_addr *table_page, mali_io_address *mapping) ++{ ++ return mali_mem_os_get_table_page(table_page, mapping); ++} ++ ++/** @brief Release a page table page ++ * ++ * Release a page table page allocated through \a mali_mmu_get_table_page ++ * ++ * @param pa the GPU address of the page to release ++ */ ++MALI_STATIC_INLINE void ++mali_mmu_release_table_page(mali_dma_addr phys, void *virt) ++{ ++ mali_mem_os_release_table_page(phys, virt); ++} ++ ++/** @brief mmap function ++ * ++ * mmap syscalls on the Mali device node will end up here. ++ * ++ * This function allocates Mali memory and maps it on CPU and Mali. ++ */ ++int mali_mmap(struct file *filp, struct vm_area_struct *vma); ++ ++/** @brief Start a new memory session ++ * ++ * Called when a process opens the Mali device node. ++ * ++ * @param session Pointer to session to initialize ++ */ ++_mali_osk_errcode_t mali_memory_session_begin(struct mali_session_data *session); ++ ++/** @brief Close a memory session ++ * ++ * Called when a process closes the Mali device node. ++ * ++ * Memory allocated by the session will be freed ++ * ++ * @param session Pointer to the session to terminate ++ */ ++void mali_memory_session_end(struct mali_session_data *session); ++ ++/** @brief Prepare Mali page tables for mapping ++ * ++ * This function will prepare the Mali page tables for mapping the memory ++ * described by \a descriptor. ++ * ++ * Page tables will be reference counted and allocated, if not yet present. ++ * ++ * @param descriptor Pointer to the memory descriptor to the mapping ++ */ ++_mali_osk_errcode_t mali_mem_mali_map_prepare(mali_mem_allocation *descriptor); ++ ++/** @brief Resize Mali page tables for mapping ++ * ++ * This function will Resize the Mali page tables for mapping the memory ++ * described by \a descriptor. ++ * ++ * Page tables will be reference counted and allocated, if not yet present. ++ * ++ * @param descriptor Pointer to the memory descriptor to the mapping ++ * @param new_size The new size of descriptor ++ */ ++_mali_osk_errcode_t mali_mem_mali_map_resize(mali_mem_allocation *descriptor, u32 new_size); ++ ++/** @brief Free Mali page tables for mapping ++ * ++ * This function will unmap pages from Mali memory and free the page tables ++ * that are now unused. ++ * ++ * The updated pages in the Mali L2 cache will be invalidated, and the MMU TLBs will be zapped if necessary. ++ * ++ * @param descriptor Pointer to the memory descriptor to unmap ++ */ ++void mali_mem_mali_map_free(struct mali_session_data *session, u32 size, mali_address_t vaddr, u32 flags); ++ ++/** @brief Parse resource and prepare the OS memory allocator ++ * ++ * @param size Maximum size to allocate for Mali GPU. ++ * @return _MALI_OSK_ERR_OK on success, otherwise failure. ++ */ ++_mali_osk_errcode_t mali_memory_core_resource_os_memory(u32 size); ++ ++/** @brief Parse resource and prepare the dedicated memory allocator ++ * ++ * @param start Physical start address of dedicated Mali GPU memory. ++ * @param size Size of dedicated Mali GPU memory. ++ * @return _MALI_OSK_ERR_OK on success, otherwise failure. ++ */ ++_mali_osk_errcode_t mali_memory_core_resource_dedicated_memory(u32 start, u32 size); ++ ++ ++struct mali_page_node *_mali_page_node_allocate(mali_page_node_type type); ++ ++void _mali_page_node_ref(struct mali_page_node *node); ++void _mali_page_node_unref(struct mali_page_node *node); ++void _mali_page_node_add_page(struct mali_page_node *node, struct page *page); ++ ++void _mali_page_node_add_block_item(struct mali_page_node *node, mali_block_item *item); ++ ++void _mali_page_node_add_swap_item(struct mali_page_node *node, struct mali_swap_item *item); ++ ++int _mali_page_node_get_ref_count(struct mali_page_node *node); ++dma_addr_t _mali_page_node_get_dma_addr(struct mali_page_node *node); ++unsigned long _mali_page_node_get_pfn(struct mali_page_node *node); ++ ++#endif /* __MALI_MEMORY_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_block_alloc.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_block_alloc.c +new file mode 100755 +index 000000000000..bccef3576914 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_block_alloc.c +@@ -0,0 +1,362 @@ ++/* ++ * Copyright (C) 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include "mali_kernel_common.h" ++#include "mali_memory.h" ++#include "mali_memory_block_alloc.h" ++#include "mali_osk.h" ++#include ++ ++ ++static mali_block_allocator *mali_mem_block_gobal_allocator = NULL; ++ ++unsigned long _mali_blk_item_get_phy_addr(mali_block_item *item) ++{ ++ return (item->phy_addr & ~(MALI_BLOCK_REF_MASK)); ++} ++ ++ ++unsigned long _mali_blk_item_get_pfn(mali_block_item *item) ++{ ++ return (item->phy_addr / MALI_BLOCK_SIZE); ++} ++ ++ ++u32 mali_mem_block_get_ref_count(mali_page_node *node) ++{ ++ MALI_DEBUG_ASSERT(node->type == MALI_PAGE_NODE_BLOCK); ++ return (node->blk_it->phy_addr & MALI_BLOCK_REF_MASK); ++} ++ ++ ++/* Increase the refence count ++* It not atomic, so it need to get sp_lock before call this function ++*/ ++ ++u32 mali_mem_block_add_ref(mali_page_node *node) ++{ ++ MALI_DEBUG_ASSERT(node->type == MALI_PAGE_NODE_BLOCK); ++ MALI_DEBUG_ASSERT(mali_mem_block_get_ref_count(node) < MALI_BLOCK_MAX_REF_COUNT); ++ return (node->blk_it->phy_addr++ & MALI_BLOCK_REF_MASK); ++} ++ ++/* Decase the refence count ++* It not atomic, so it need to get sp_lock before call this function ++*/ ++u32 mali_mem_block_dec_ref(mali_page_node *node) ++{ ++ MALI_DEBUG_ASSERT(node->type == MALI_PAGE_NODE_BLOCK); ++ MALI_DEBUG_ASSERT(mali_mem_block_get_ref_count(node) > 0); ++ return (node->blk_it->phy_addr-- & MALI_BLOCK_REF_MASK); ++} ++ ++ ++static mali_block_allocator *mali_mem_block_allocator_create(u32 base_address, u32 size) ++{ ++ mali_block_allocator *info; ++ u32 usable_size; ++ u32 num_blocks; ++ mali_page_node *m_node; ++ mali_block_item *mali_blk_items = NULL; ++ int i = 0; ++ ++ usable_size = size & ~(MALI_BLOCK_SIZE - 1); ++ MALI_DEBUG_PRINT(3, ("Mali block allocator create for region starting at 0x%08X length 0x%08X\n", base_address, size)); ++ MALI_DEBUG_PRINT(4, ("%d usable bytes\n", usable_size)); ++ num_blocks = usable_size / MALI_BLOCK_SIZE; ++ MALI_DEBUG_PRINT(4, ("which becomes %d blocks\n", num_blocks)); ++ ++ if (usable_size == 0) { ++ MALI_DEBUG_PRINT(1, ("Memory block of size %d is unusable\n", size)); ++ return NULL; ++ } ++ ++ info = _mali_osk_calloc(1, sizeof(mali_block_allocator)); ++ if (NULL != info) { ++ INIT_LIST_HEAD(&info->free); ++ spin_lock_init(&info->sp_lock); ++ info->total_num = num_blocks; ++ mali_blk_items = _mali_osk_calloc(1, sizeof(mali_block_item) * num_blocks); ++ ++ if (mali_blk_items) { ++ info->items = mali_blk_items; ++ /* add blocks(4k size) to free list*/ ++ for (i = 0 ; i < num_blocks ; i++) { ++ /* add block information*/ ++ mali_blk_items[i].phy_addr = base_address + (i * MALI_BLOCK_SIZE); ++ /* add to free list */ ++ m_node = _mali_page_node_allocate(MALI_PAGE_NODE_BLOCK); ++ if (m_node == NULL) ++ goto fail; ++ _mali_page_node_add_block_item(m_node, &(mali_blk_items[i])); ++ list_add_tail(&m_node->list, &info->free); ++ atomic_add(1, &info->free_num); ++ } ++ return info; ++ } ++ } ++fail: ++ mali_mem_block_allocator_destroy(); ++ return NULL; ++} ++ ++void mali_mem_block_allocator_destroy(void) ++{ ++ struct mali_page_node *m_page, *m_tmp; ++ mali_block_allocator *info = mali_mem_block_gobal_allocator; ++ MALI_DEBUG_ASSERT_POINTER(info); ++ MALI_DEBUG_PRINT(4, ("Memory block destroy !\n")); ++ ++ if (NULL == info) ++ return; ++ ++ list_for_each_entry_safe(m_page, m_tmp , &info->free, list) { ++ MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_BLOCK); ++ list_del(&m_page->list); ++ kfree(m_page); ++ } ++ ++ _mali_osk_free(info->items); ++ _mali_osk_free(info); ++} ++ ++u32 mali_mem_block_release(mali_mem_backend *mem_bkend) ++{ ++ mali_mem_allocation *alloc = mem_bkend->mali_allocation; ++ u32 free_pages_nr = 0; ++ MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_BLOCK); ++ ++ /* Unmap the memory from the mali virtual address space. */ ++ mali_mem_block_mali_unmap(alloc); ++ mutex_lock(&mem_bkend->mutex); ++ free_pages_nr = mali_mem_block_free(&mem_bkend->block_mem); ++ mutex_unlock(&mem_bkend->mutex); ++ return free_pages_nr; ++} ++ ++ ++int mali_mem_block_alloc(mali_mem_block_mem *block_mem, u32 size) ++{ ++ struct mali_page_node *m_page, *m_tmp; ++ size_t page_count = PAGE_ALIGN(size) / _MALI_OSK_MALI_PAGE_SIZE; ++ mali_block_allocator *info = mali_mem_block_gobal_allocator; ++ MALI_DEBUG_ASSERT_POINTER(info); ++ ++ MALI_DEBUG_PRINT(4, ("BLOCK Mem: Allocate size = 0x%x\n", size)); ++ /*do some init */ ++ INIT_LIST_HEAD(&block_mem->pfns); ++ ++ spin_lock(&info->sp_lock); ++ /*check if have enough space*/ ++ if (atomic_read(&info->free_num) > page_count) { ++ list_for_each_entry_safe(m_page, m_tmp , &info->free, list) { ++ if (page_count > 0) { ++ MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_BLOCK); ++ MALI_DEBUG_ASSERT(mali_mem_block_get_ref_count(m_page) == 0); ++ list_move(&m_page->list, &block_mem->pfns); ++ block_mem->count++; ++ atomic_dec(&info->free_num); ++ _mali_page_node_ref(m_page); ++ } else { ++ break; ++ } ++ page_count--; ++ } ++ } else { ++ /* can't allocate from BLOCK memory*/ ++ spin_unlock(&info->sp_lock); ++ return -1; ++ } ++ ++ spin_unlock(&info->sp_lock); ++ return 0; ++} ++ ++u32 mali_mem_block_free(mali_mem_block_mem *block_mem) ++{ ++ u32 free_pages_nr = 0; ++ ++ free_pages_nr = mali_mem_block_free_list(&block_mem->pfns); ++ MALI_DEBUG_PRINT(4, ("BLOCK Mem free : allocated size = 0x%x, free size = 0x%x\n", block_mem->count * _MALI_OSK_MALI_PAGE_SIZE, ++ free_pages_nr * _MALI_OSK_MALI_PAGE_SIZE)); ++ block_mem->count = 0; ++ MALI_DEBUG_ASSERT(list_empty(&block_mem->pfns)); ++ ++ return free_pages_nr; ++} ++ ++ ++u32 mali_mem_block_free_list(struct list_head *list) ++{ ++ struct mali_page_node *m_page, *m_tmp; ++ mali_block_allocator *info = mali_mem_block_gobal_allocator; ++ u32 free_pages_nr = 0; ++ ++ if (info) { ++ spin_lock(&info->sp_lock); ++ list_for_each_entry_safe(m_page, m_tmp , list, list) { ++ if (1 == _mali_page_node_get_ref_count(m_page)) { ++ free_pages_nr++; ++ } ++ mali_mem_block_free_node(m_page); ++ } ++ spin_unlock(&info->sp_lock); ++ } ++ return free_pages_nr; ++} ++ ++/* free the node,*/ ++void mali_mem_block_free_node(struct mali_page_node *node) ++{ ++ mali_block_allocator *info = mali_mem_block_gobal_allocator; ++ ++ /* only handle BLOCK node */ ++ if (node->type == MALI_PAGE_NODE_BLOCK && info) { ++ /*Need to make this atomic?*/ ++ if (1 == _mali_page_node_get_ref_count(node)) { ++ /*Move to free list*/ ++ _mali_page_node_unref(node); ++ list_move_tail(&node->list, &info->free); ++ atomic_add(1, &info->free_num); ++ } else { ++ _mali_page_node_unref(node); ++ list_del(&node->list); ++ kfree(node); ++ } ++ } ++} ++ ++/* unref the node, but not free it */ ++_mali_osk_errcode_t mali_mem_block_unref_node(struct mali_page_node *node) ++{ ++ mali_block_allocator *info = mali_mem_block_gobal_allocator; ++ mali_page_node *new_node; ++ ++ /* only handle BLOCK node */ ++ if (node->type == MALI_PAGE_NODE_BLOCK && info) { ++ /*Need to make this atomic?*/ ++ if (1 == _mali_page_node_get_ref_count(node)) { ++ /* allocate a new node, Add to free list, keep the old node*/ ++ _mali_page_node_unref(node); ++ new_node = _mali_page_node_allocate(MALI_PAGE_NODE_BLOCK); ++ if (new_node) { ++ memcpy(new_node, node, sizeof(mali_page_node)); ++ list_add(&new_node->list, &info->free); ++ atomic_add(1, &info->free_num); ++ } else ++ return _MALI_OSK_ERR_FAULT; ++ ++ } else { ++ _mali_page_node_unref(node); ++ } ++ } ++ return _MALI_OSK_ERR_OK; ++} ++ ++ ++int mali_mem_block_mali_map(mali_mem_block_mem *block_mem, struct mali_session_data *session, u32 vaddr, u32 props) ++{ ++ struct mali_page_directory *pagedir = session->page_directory; ++ struct mali_page_node *m_page; ++ dma_addr_t phys; ++ u32 virt = vaddr; ++ u32 prop = props; ++ ++ list_for_each_entry(m_page, &block_mem->pfns, list) { ++ MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_BLOCK); ++ phys = _mali_page_node_get_dma_addr(m_page); ++#if defined(CONFIG_ARCH_DMA_ADDR_T_64BIT) ++ /* Verify that the "physical" address is 32-bit and ++ * usable for Mali, when on a system with bus addresses ++ * wider than 32-bit. */ ++ MALI_DEBUG_ASSERT(0 == (phys >> 32)); ++#endif ++ mali_mmu_pagedir_update(pagedir, virt, (mali_dma_addr)phys, MALI_MMU_PAGE_SIZE, prop); ++ virt += MALI_MMU_PAGE_SIZE; ++ } ++ ++ return 0; ++} ++ ++void mali_mem_block_mali_unmap(mali_mem_allocation *alloc) ++{ ++ struct mali_session_data *session; ++ MALI_DEBUG_ASSERT_POINTER(alloc); ++ session = alloc->session; ++ MALI_DEBUG_ASSERT_POINTER(session); ++ ++ mali_session_memory_lock(session); ++ mali_mem_mali_map_free(session, alloc->psize, alloc->mali_vma_node.vm_node.start, ++ alloc->flags); ++ mali_session_memory_unlock(session); ++} ++ ++ ++int mali_mem_block_cpu_map(mali_mem_backend *mem_bkend, struct vm_area_struct *vma) ++{ ++ int ret; ++ mali_mem_block_mem *block_mem = &mem_bkend->block_mem; ++ unsigned long addr = vma->vm_start; ++ struct mali_page_node *m_page; ++ MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_BLOCK); ++ ++ list_for_each_entry(m_page, &block_mem->pfns, list) { ++ MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_BLOCK); ++ ret = vmf_insert_pfn(vma, addr, _mali_page_node_get_pfn(m_page)); ++ ++ if (unlikely(0 != ret)) { ++ return -EFAULT; ++ } ++ addr += _MALI_OSK_MALI_PAGE_SIZE; ++ ++ } ++ ++ return 0; ++} ++ ++ ++_mali_osk_errcode_t mali_memory_core_resource_dedicated_memory(u32 start, u32 size) ++{ ++ mali_block_allocator *allocator; ++ ++ /* Do the low level linux operation first */ ++ ++ /* Request ownership of the memory */ ++ if (_MALI_OSK_ERR_OK != _mali_osk_mem_reqregion(start, size, "Dedicated Mali GPU memory")) { ++ MALI_DEBUG_PRINT(1, ("Failed to request memory region for frame buffer (0x%08X - 0x%08X)\n", start, start + size - 1)); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ /* Create generic block allocator object to handle it */ ++ allocator = mali_mem_block_allocator_create(start, size); ++ ++ if (NULL == allocator) { ++ MALI_DEBUG_PRINT(1, ("Memory bank registration failed\n")); ++ _mali_osk_mem_unreqregion(start, size); ++ MALI_ERROR(_MALI_OSK_ERR_FAULT); ++ } ++ ++ mali_mem_block_gobal_allocator = (mali_block_allocator *)allocator; ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++mali_bool mali_memory_have_dedicated_memory(void) ++{ ++ return mali_mem_block_gobal_allocator ? MALI_TRUE : MALI_FALSE; ++} ++ ++u32 mali_mem_block_allocator_stat(void) ++{ ++ mali_block_allocator *allocator = mali_mem_block_gobal_allocator; ++ MALI_DEBUG_ASSERT_POINTER(allocator); ++ ++ return (allocator->total_num - atomic_read(&allocator->free_num)) * _MALI_OSK_MALI_PAGE_SIZE; ++} +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_block_alloc.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_block_alloc.h +new file mode 100755 +index 000000000000..70fd9ec25f50 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_block_alloc.h +@@ -0,0 +1,58 @@ ++/* ++ * Copyright (C) 2010, 2013, 2015-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_BLOCK_ALLOCATOR_H__ ++#define __MALI_BLOCK_ALLOCATOR_H__ ++ ++#include "mali_session.h" ++#include "mali_memory.h" ++#include ++ ++#include "mali_memory_types.h" ++ ++#define MALI_BLOCK_SIZE (PAGE_SIZE) /* 4 kB, manage BLOCK memory as page size */ ++#define MALI_BLOCK_REF_MASK (0xFFF) ++#define MALI_BLOCK_MAX_REF_COUNT (0xFFF) ++ ++ ++ ++typedef struct mali_block_allocator { ++ /* ++ * In free list, each node's ref_count is 0, ++ * ref_count added when allocated or referenced in COW ++ */ ++ mali_block_item *items; /* information for each block item*/ ++ struct list_head free; /*free list of mali_memory_node*/ ++ spinlock_t sp_lock; /*lock for reference count & free list opertion*/ ++ u32 total_num; /* Number of total pages*/ ++ atomic_t free_num; /*number of free pages*/ ++} mali_block_allocator; ++ ++unsigned long _mali_blk_item_get_phy_addr(mali_block_item *item); ++unsigned long _mali_blk_item_get_pfn(mali_block_item *item); ++u32 mali_mem_block_get_ref_count(mali_page_node *node); ++u32 mali_mem_block_add_ref(mali_page_node *node); ++u32 mali_mem_block_dec_ref(mali_page_node *node); ++u32 mali_mem_block_release(mali_mem_backend *mem_bkend); ++int mali_mem_block_alloc(mali_mem_block_mem *block_mem, u32 size); ++int mali_mem_block_mali_map(mali_mem_block_mem *block_mem, struct mali_session_data *session, u32 vaddr, u32 props); ++void mali_mem_block_mali_unmap(mali_mem_allocation *alloc); ++ ++int mali_mem_block_cpu_map(mali_mem_backend *mem_bkend, struct vm_area_struct *vma); ++_mali_osk_errcode_t mali_memory_core_resource_dedicated_memory(u32 start, u32 size); ++mali_bool mali_memory_have_dedicated_memory(void); ++u32 mali_mem_block_free(mali_mem_block_mem *block_mem); ++u32 mali_mem_block_free_list(struct list_head *list); ++void mali_mem_block_free_node(struct mali_page_node *node); ++void mali_mem_block_allocator_destroy(void); ++_mali_osk_errcode_t mali_mem_block_unref_node(struct mali_page_node *node); ++u32 mali_mem_block_allocator_stat(void); ++ ++#endif /* __MALI_BLOCK_ALLOCATOR_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_cow.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_cow.c +new file mode 100755 +index 000000000000..0bdf90b167d6 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_cow.c +@@ -0,0 +1,776 @@ ++/* ++ * Copyright (C) 2013-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#ifdef CONFIG_ARM ++#include ++#endif ++#include ++ ++#include "mali_memory.h" ++#include "mali_kernel_common.h" ++#include "mali_uk_types.h" ++#include "mali_osk.h" ++#include "mali_kernel_linux.h" ++#include "mali_memory_cow.h" ++#include "mali_memory_block_alloc.h" ++#include "mali_memory_swap_alloc.h" ++ ++/** ++* allocate pages for COW backend and flush cache ++*/ ++static struct page *mali_mem_cow_alloc_page(void) ++ ++{ ++ mali_mem_os_mem os_mem; ++ struct mali_page_node *node; ++ struct page *new_page; ++ ++ int ret = 0; ++ /* allocate pages from os mem */ ++ ret = mali_mem_os_alloc_pages(&os_mem, _MALI_OSK_MALI_PAGE_SIZE); ++ ++ if (ret) { ++ return NULL; ++ } ++ ++ MALI_DEBUG_ASSERT(1 == os_mem.count); ++ ++ node = _MALI_OSK_CONTAINER_OF(os_mem.pages.next, struct mali_page_node, list); ++ new_page = node->page; ++ node->page = NULL; ++ list_del(&node->list); ++ kfree(node); ++ ++ return new_page; ++} ++ ++ ++static struct list_head *_mali_memory_cow_get_node_list(mali_mem_backend *target_bk, ++ u32 target_offset, ++ u32 target_size) ++{ ++ MALI_DEBUG_ASSERT(MALI_MEM_OS == target_bk->type || MALI_MEM_COW == target_bk->type || ++ MALI_MEM_BLOCK == target_bk->type || MALI_MEM_SWAP == target_bk->type); ++ ++ if (MALI_MEM_OS == target_bk->type) { ++ MALI_DEBUG_ASSERT(&target_bk->os_mem); ++ MALI_DEBUG_ASSERT(((target_size + target_offset) / _MALI_OSK_MALI_PAGE_SIZE) <= target_bk->os_mem.count); ++ return &target_bk->os_mem.pages; ++ } else if (MALI_MEM_COW == target_bk->type) { ++ MALI_DEBUG_ASSERT(&target_bk->cow_mem); ++ MALI_DEBUG_ASSERT(((target_size + target_offset) / _MALI_OSK_MALI_PAGE_SIZE) <= target_bk->cow_mem.count); ++ return &target_bk->cow_mem.pages; ++ } else if (MALI_MEM_BLOCK == target_bk->type) { ++ MALI_DEBUG_ASSERT(&target_bk->block_mem); ++ MALI_DEBUG_ASSERT(((target_size + target_offset) / _MALI_OSK_MALI_PAGE_SIZE) <= target_bk->block_mem.count); ++ return &target_bk->block_mem.pfns; ++ } else if (MALI_MEM_SWAP == target_bk->type) { ++ MALI_DEBUG_ASSERT(&target_bk->swap_mem); ++ MALI_DEBUG_ASSERT(((target_size + target_offset) / _MALI_OSK_MALI_PAGE_SIZE) <= target_bk->swap_mem.count); ++ return &target_bk->swap_mem.pages; ++ } ++ ++ return NULL; ++} ++ ++/** ++* Do COW for os memory - support do COW for memory from bank memory ++* The range_start/size can be zero, which means it will call cow_modify_range ++* latter. ++* This function allocate new pages for COW backend from os mem for a modified range ++* It will keep the page which not in the modified range and Add ref to it ++* ++* @target_bk - target allocation's backend(the allocation need to do COW) ++* @target_offset - the offset in target allocation to do COW(for support COW a memory allocated from memory_bank, 4K align) ++* @target_size - size of target allocation to do COW (for support memory bank) ++* @backend -COW backend ++* @range_start - offset of modified range (4K align) ++* @range_size - size of modified range ++*/ ++_mali_osk_errcode_t mali_memory_cow_os_memory(mali_mem_backend *target_bk, ++ u32 target_offset, ++ u32 target_size, ++ mali_mem_backend *backend, ++ u32 range_start, ++ u32 range_size) ++{ ++ mali_mem_cow *cow = &backend->cow_mem; ++ struct mali_page_node *m_page, *m_tmp, *page_node; ++ int target_page = 0; ++ struct page *new_page; ++ struct list_head *pages = NULL; ++ ++ pages = _mali_memory_cow_get_node_list(target_bk, target_offset, target_size); ++ ++ if (NULL == pages) { ++ MALI_DEBUG_PRINT_ERROR(("No memory page need to cow ! \n")); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ MALI_DEBUG_ASSERT(0 == cow->count); ++ ++ INIT_LIST_HEAD(&cow->pages); ++ mutex_lock(&target_bk->mutex); ++ list_for_each_entry_safe(m_page, m_tmp, pages, list) { ++ /* add page from (target_offset,target_offset+size) to cow backend */ ++ if ((target_page >= target_offset / _MALI_OSK_MALI_PAGE_SIZE) && ++ (target_page < ((target_size + target_offset) / _MALI_OSK_MALI_PAGE_SIZE))) { ++ ++ /* allocate a new page node, alway use OS memory for COW */ ++ page_node = _mali_page_node_allocate(MALI_PAGE_NODE_OS); ++ ++ if (NULL == page_node) { ++ mutex_unlock(&target_bk->mutex); ++ goto error; ++ } ++ ++ INIT_LIST_HEAD(&page_node->list); ++ ++ /* check if in the modified range*/ ++ if ((cow->count >= range_start / _MALI_OSK_MALI_PAGE_SIZE) && ++ (cow->count < (range_start + range_size) / _MALI_OSK_MALI_PAGE_SIZE)) { ++ /* need to allocate a new page */ ++ /* To simplify the case, All COW memory is allocated from os memory ?*/ ++ new_page = mali_mem_cow_alloc_page(); ++ ++ if (NULL == new_page) { ++ kfree(page_node); ++ mutex_unlock(&target_bk->mutex); ++ goto error; ++ } ++ ++ _mali_page_node_add_page(page_node, new_page); ++ } else { ++ /*Add Block memory case*/ ++ if (m_page->type != MALI_PAGE_NODE_BLOCK) { ++ _mali_page_node_add_page(page_node, m_page->page); ++ } else { ++ page_node->type = MALI_PAGE_NODE_BLOCK; ++ _mali_page_node_add_block_item(page_node, m_page->blk_it); ++ } ++ ++ /* add ref to this page */ ++ _mali_page_node_ref(m_page); ++ } ++ ++ /* add it to COW backend page list */ ++ list_add_tail(&page_node->list, &cow->pages); ++ cow->count++; ++ } ++ target_page++; ++ } ++ mutex_unlock(&target_bk->mutex); ++ return _MALI_OSK_ERR_OK; ++error: ++ mali_mem_cow_release(backend, MALI_FALSE); ++ return _MALI_OSK_ERR_FAULT; ++} ++ ++_mali_osk_errcode_t mali_memory_cow_swap_memory(mali_mem_backend *target_bk, ++ u32 target_offset, ++ u32 target_size, ++ mali_mem_backend *backend, ++ u32 range_start, ++ u32 range_size) ++{ ++ mali_mem_cow *cow = &backend->cow_mem; ++ struct mali_page_node *m_page, *m_tmp, *page_node; ++ int target_page = 0; ++ struct mali_swap_item *swap_item; ++ struct list_head *pages = NULL; ++ ++ pages = _mali_memory_cow_get_node_list(target_bk, target_offset, target_size); ++ if (NULL == pages) { ++ MALI_DEBUG_PRINT_ERROR(("No swap memory page need to cow ! \n")); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ MALI_DEBUG_ASSERT(0 == cow->count); ++ ++ INIT_LIST_HEAD(&cow->pages); ++ mutex_lock(&target_bk->mutex); ++ ++ backend->flags |= MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN; ++ ++ list_for_each_entry_safe(m_page, m_tmp, pages, list) { ++ /* add page from (target_offset,target_offset+size) to cow backend */ ++ if ((target_page >= target_offset / _MALI_OSK_MALI_PAGE_SIZE) && ++ (target_page < ((target_size + target_offset) / _MALI_OSK_MALI_PAGE_SIZE))) { ++ ++ /* allocate a new page node, use swap memory for COW memory swap cowed flag. */ ++ page_node = _mali_page_node_allocate(MALI_PAGE_NODE_SWAP); ++ ++ if (NULL == page_node) { ++ mutex_unlock(&target_bk->mutex); ++ goto error; ++ } ++ ++ /* check if in the modified range*/ ++ if ((cow->count >= range_start / _MALI_OSK_MALI_PAGE_SIZE) && ++ (cow->count < (range_start + range_size) / _MALI_OSK_MALI_PAGE_SIZE)) { ++ /* need to allocate a new page */ ++ /* To simplify the case, All COW memory is allocated from os memory ?*/ ++ swap_item = mali_mem_swap_alloc_swap_item(); ++ ++ if (NULL == swap_item) { ++ kfree(page_node); ++ mutex_unlock(&target_bk->mutex); ++ goto error; ++ } ++ ++ swap_item->idx = mali_mem_swap_idx_alloc(); ++ ++ if (_MALI_OSK_BITMAP_INVALIDATE_INDEX == swap_item->idx) { ++ MALI_DEBUG_PRINT(1, ("Failed to allocate swap index in swap CoW.\n")); ++ kfree(page_node); ++ kfree(swap_item); ++ mutex_unlock(&target_bk->mutex); ++ goto error; ++ } ++ ++ _mali_page_node_add_swap_item(page_node, swap_item); ++ } else { ++ _mali_page_node_add_swap_item(page_node, m_page->swap_it); ++ ++ /* add ref to this page */ ++ _mali_page_node_ref(m_page); ++ } ++ ++ list_add_tail(&page_node->list, &cow->pages); ++ cow->count++; ++ } ++ target_page++; ++ } ++ mutex_unlock(&target_bk->mutex); ++ ++ return _MALI_OSK_ERR_OK; ++error: ++ mali_mem_swap_release(backend, MALI_FALSE); ++ return _MALI_OSK_ERR_FAULT; ++ ++} ++ ++ ++_mali_osk_errcode_t _mali_mem_put_page_node(mali_page_node *node) ++{ ++ if (node->type == MALI_PAGE_NODE_OS) { ++ return mali_mem_os_put_page(node->page); ++ } else if (node->type == MALI_PAGE_NODE_BLOCK) { ++ return mali_mem_block_unref_node(node); ++ } else if (node->type == MALI_PAGE_NODE_SWAP) { ++ return _mali_mem_swap_put_page_node(node); ++ } else ++ MALI_DEBUG_ASSERT(0); ++ return _MALI_OSK_ERR_FAULT; ++} ++ ++ ++/** ++* Modify a range of a exist COW backend ++* @backend -COW backend ++* @range_start - offset of modified range (4K align) ++* @range_size - size of modified range(in byte) ++*/ ++_mali_osk_errcode_t mali_memory_cow_modify_range(mali_mem_backend *backend, ++ u32 range_start, ++ u32 range_size) ++{ ++ mali_mem_allocation *alloc = NULL; ++ struct mali_session_data *session; ++ mali_mem_cow *cow = &backend->cow_mem; ++ struct mali_page_node *m_page, *m_tmp; ++ LIST_HEAD(pages); ++ struct page *new_page; ++ u32 count = 0; ++ s32 change_pages_nr = 0; ++ _mali_osk_errcode_t ret = _MALI_OSK_ERR_OK; ++ ++ if (range_start % _MALI_OSK_MALI_PAGE_SIZE) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS); ++ if (range_size % _MALI_OSK_MALI_PAGE_SIZE) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS); ++ ++ alloc = backend->mali_allocation; ++ MALI_DEBUG_ASSERT_POINTER(alloc); ++ ++ session = alloc->session; ++ MALI_DEBUG_ASSERT_POINTER(session); ++ ++ MALI_DEBUG_ASSERT(MALI_MEM_COW == backend->type); ++ MALI_DEBUG_ASSERT(((range_start + range_size) / _MALI_OSK_MALI_PAGE_SIZE) <= cow->count); ++ ++ mutex_lock(&backend->mutex); ++ ++ /* free pages*/ ++ list_for_each_entry_safe(m_page, m_tmp, &cow->pages, list) { ++ ++ /* check if in the modified range*/ ++ if ((count >= range_start / _MALI_OSK_MALI_PAGE_SIZE) && ++ (count < (range_start + range_size) / _MALI_OSK_MALI_PAGE_SIZE)) { ++ if (MALI_PAGE_NODE_SWAP != m_page->type) { ++ new_page = mali_mem_cow_alloc_page(); ++ ++ if (NULL == new_page) { ++ goto error; ++ } ++ if (1 != _mali_page_node_get_ref_count(m_page)) ++ change_pages_nr++; ++ /* unref old page*/ ++ _mali_osk_mutex_wait(session->cow_lock); ++ if (_mali_mem_put_page_node(m_page)) { ++ __free_page(new_page); ++ _mali_osk_mutex_signal(session->cow_lock); ++ goto error; ++ } ++ _mali_osk_mutex_signal(session->cow_lock); ++ /* add new page*/ ++ /* always use OS for COW*/ ++ m_page->type = MALI_PAGE_NODE_OS; ++ _mali_page_node_add_page(m_page, new_page); ++ } else { ++ struct mali_swap_item *swap_item; ++ ++ swap_item = mali_mem_swap_alloc_swap_item(); ++ ++ if (NULL == swap_item) { ++ goto error; ++ } ++ ++ swap_item->idx = mali_mem_swap_idx_alloc(); ++ ++ if (_MALI_OSK_BITMAP_INVALIDATE_INDEX == swap_item->idx) { ++ MALI_DEBUG_PRINT(1, ("Failed to allocate swap index in swap CoW modify range.\n")); ++ kfree(swap_item); ++ goto error; ++ } ++ ++ if (1 != _mali_page_node_get_ref_count(m_page)) { ++ change_pages_nr++; ++ } ++ ++ if (_mali_mem_put_page_node(m_page)) { ++ mali_mem_swap_free_swap_item(swap_item); ++ goto error; ++ } ++ ++ _mali_page_node_add_swap_item(m_page, swap_item); ++ } ++ } ++ count++; ++ } ++ cow->change_pages_nr = change_pages_nr; ++ ++ MALI_DEBUG_ASSERT(MALI_MEM_COW == alloc->type); ++ ++ /* ZAP cpu mapping(modified range), and do cpu mapping here if need */ ++ if (NULL != alloc->cpu_mapping.vma) { ++ MALI_DEBUG_ASSERT(0 != alloc->backend_handle); ++ MALI_DEBUG_ASSERT(NULL != alloc->cpu_mapping.vma); ++ MALI_DEBUG_ASSERT(alloc->cpu_mapping.vma->vm_end - alloc->cpu_mapping.vma->vm_start >= range_size); ++ ++ if (MALI_MEM_BACKEND_FLAG_SWAP_COWED != (backend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED)) { ++ zap_vma_ptes(alloc->cpu_mapping.vma, alloc->cpu_mapping.vma->vm_start + range_start, range_size); ++ ++ ret = mali_mem_cow_cpu_map_pages_locked(backend, alloc->cpu_mapping.vma, alloc->cpu_mapping.vma->vm_start + range_start, range_size / _MALI_OSK_MALI_PAGE_SIZE); ++ ++ if (unlikely(ret != _MALI_OSK_ERR_OK)) { ++ MALI_DEBUG_PRINT(2, ("mali_memory_cow_modify_range: cpu mapping failed !\n")); ++ ret = _MALI_OSK_ERR_FAULT; ++ } ++ } else { ++ /* used to trigger page fault for swappable cowed memory. */ ++ alloc->cpu_mapping.vma->vm_flags |= VM_PFNMAP; ++ alloc->cpu_mapping.vma->vm_flags |= VM_MIXEDMAP; ++ ++ zap_vma_ptes(alloc->cpu_mapping.vma, alloc->cpu_mapping.vma->vm_start + range_start, range_size); ++ /* delete this flag to let swappble is ummapped regard to stauct page not page frame. */ ++ alloc->cpu_mapping.vma->vm_flags &= ~VM_PFNMAP; ++ alloc->cpu_mapping.vma->vm_flags &= ~VM_MIXEDMAP; ++ } ++ } ++ ++error: ++ mutex_unlock(&backend->mutex); ++ return ret; ++ ++} ++ ++ ++/** ++* Allocate pages for COW backend ++* @alloc -allocation for COW allocation ++* @target_bk - target allocation's backend(the allocation need to do COW) ++* @target_offset - the offset in target allocation to do COW(for support COW a memory allocated from memory_bank, 4K align) ++* @target_size - size of target allocation to do COW (for support memory bank)(in byte) ++* @backend -COW backend ++* @range_start - offset of modified range (4K align) ++* @range_size - size of modified range(in byte) ++*/ ++_mali_osk_errcode_t mali_memory_do_cow(mali_mem_backend *target_bk, ++ u32 target_offset, ++ u32 target_size, ++ mali_mem_backend *backend, ++ u32 range_start, ++ u32 range_size) ++{ ++ struct mali_session_data *session = backend->mali_allocation->session; ++ ++ MALI_CHECK_NON_NULL(session, _MALI_OSK_ERR_INVALID_ARGS); ++ ++ /* size & offset must be a multiple of the system page size */ ++ if (target_size % _MALI_OSK_MALI_PAGE_SIZE) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS); ++ if (range_size % _MALI_OSK_MALI_PAGE_SIZE) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS); ++ if (target_offset % _MALI_OSK_MALI_PAGE_SIZE) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS); ++ if (range_start % _MALI_OSK_MALI_PAGE_SIZE) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS); ++ ++ /* check backend type */ ++ MALI_DEBUG_ASSERT(MALI_MEM_COW == backend->type); ++ ++ switch (target_bk->type) { ++ case MALI_MEM_OS: ++ case MALI_MEM_BLOCK: ++ return mali_memory_cow_os_memory(target_bk, target_offset, target_size, backend, range_start, range_size); ++ break; ++ case MALI_MEM_COW: ++ if (backend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED) { ++ return mali_memory_cow_swap_memory(target_bk, target_offset, target_size, backend, range_start, range_size); ++ } else { ++ return mali_memory_cow_os_memory(target_bk, target_offset, target_size, backend, range_start, range_size); ++ } ++ break; ++ case MALI_MEM_SWAP: ++ return mali_memory_cow_swap_memory(target_bk, target_offset, target_size, backend, range_start, range_size); ++ break; ++ case MALI_MEM_EXTERNAL: ++ /*NOT support yet*/ ++ MALI_DEBUG_PRINT_ERROR(("External physical memory not supported ! \n")); ++ return _MALI_OSK_ERR_UNSUPPORTED; ++ break; ++ case MALI_MEM_DMA_BUF: ++ /*NOT support yet*/ ++ MALI_DEBUG_PRINT_ERROR(("DMA buffer not supported ! \n")); ++ return _MALI_OSK_ERR_UNSUPPORTED; ++ break; ++ case MALI_MEM_UMP: ++ /*NOT support yet*/ ++ MALI_DEBUG_PRINT_ERROR(("UMP buffer not supported ! \n")); ++ return _MALI_OSK_ERR_UNSUPPORTED; ++ break; ++ default: ++ /*Not support yet*/ ++ MALI_DEBUG_PRINT_ERROR(("Invalid memory type not supported ! \n")); ++ return _MALI_OSK_ERR_UNSUPPORTED; ++ break; ++ } ++ return _MALI_OSK_ERR_OK; ++} ++ ++ ++/** ++* Map COW backend memory to mali ++* Support OS/BLOCK for mali_page_node ++*/ ++int mali_mem_cow_mali_map(mali_mem_backend *mem_bkend, u32 range_start, u32 range_size) ++{ ++ mali_mem_allocation *cow_alloc; ++ struct mali_page_node *m_page; ++ struct mali_session_data *session; ++ struct mali_page_directory *pagedir; ++ u32 virt, start; ++ ++ cow_alloc = mem_bkend->mali_allocation; ++ virt = cow_alloc->mali_vma_node.vm_node.start; ++ start = virt; ++ ++ MALI_DEBUG_ASSERT_POINTER(mem_bkend); ++ MALI_DEBUG_ASSERT(MALI_MEM_COW == mem_bkend->type); ++ MALI_DEBUG_ASSERT_POINTER(cow_alloc); ++ ++ session = cow_alloc->session; ++ pagedir = session->page_directory; ++ MALI_CHECK_NON_NULL(session, _MALI_OSK_ERR_INVALID_ARGS); ++ list_for_each_entry(m_page, &mem_bkend->cow_mem.pages, list) { ++ if ((virt - start >= range_start) && (virt - start < range_start + range_size)) { ++ dma_addr_t phys = _mali_page_node_get_dma_addr(m_page); ++#if defined(CONFIG_ARCH_DMA_ADDR_T_64BIT) ++ MALI_DEBUG_ASSERT(0 == (phys >> 32)); ++#endif ++ mali_mmu_pagedir_update(pagedir, virt, (mali_dma_addr)phys, ++ MALI_MMU_PAGE_SIZE, MALI_MMU_FLAGS_DEFAULT); ++ } ++ virt += MALI_MMU_PAGE_SIZE; ++ } ++ return 0; ++} ++ ++/** ++* Map COW backend to cpu ++* support OS/BLOCK memory ++*/ ++int mali_mem_cow_cpu_map(mali_mem_backend *mem_bkend, struct vm_area_struct *vma) ++{ ++ mali_mem_cow *cow = &mem_bkend->cow_mem; ++ struct mali_page_node *m_page; ++ int ret; ++ unsigned long addr = vma->vm_start; ++ MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_COW); ++ ++ list_for_each_entry(m_page, &cow->pages, list) { ++ /* We should use vm_insert_page, but it does a dcache ++ * flush which makes it way slower than remap_pfn_range or vmf_insert_pfn. ++ ret = vm_insert_page(vma, addr, page); ++ */ ++ ret = vmf_insert_pfn(vma, addr, _mali_page_node_get_pfn(m_page)); ++ ++ if (unlikely(0 != ret)) { ++ return ret; ++ } ++ addr += _MALI_OSK_MALI_PAGE_SIZE; ++ } ++ ++ return 0; ++} ++ ++/** ++* Map some pages(COW backend) to CPU vma@vaddr ++*@ mem_bkend - COW backend ++*@ vma ++*@ vaddr -start CPU vaddr mapped to ++*@ num - max number of pages to map to CPU vaddr ++*/ ++_mali_osk_errcode_t mali_mem_cow_cpu_map_pages_locked(mali_mem_backend *mem_bkend, ++ struct vm_area_struct *vma, ++ unsigned long vaddr, ++ int num) ++{ ++ mali_mem_cow *cow = &mem_bkend->cow_mem; ++ struct mali_page_node *m_page; ++ int ret; ++ int offset; ++ int count ; ++ unsigned long vstart = vma->vm_start; ++ count = 0; ++ MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_COW); ++ MALI_DEBUG_ASSERT(0 == vaddr % _MALI_OSK_MALI_PAGE_SIZE); ++ MALI_DEBUG_ASSERT(0 == vstart % _MALI_OSK_MALI_PAGE_SIZE); ++ offset = (vaddr - vstart) / _MALI_OSK_MALI_PAGE_SIZE; ++ ++ list_for_each_entry(m_page, &cow->pages, list) { ++ if ((count >= offset) && (count < offset + num)) { ++ ret = vmf_insert_pfn(vma, vaddr, _mali_page_node_get_pfn(m_page)); ++ ++ if (unlikely(0 != ret)) { ++ if (count == offset) { ++ return _MALI_OSK_ERR_FAULT; ++ } else { ++ /* ret is EBUSY when page isn't in modify range, but now it's OK*/ ++ return _MALI_OSK_ERR_OK; ++ } ++ } ++ vaddr += _MALI_OSK_MALI_PAGE_SIZE; ++ } ++ count++; ++ } ++ return _MALI_OSK_ERR_OK; ++} ++ ++/** ++* Release COW backend memory ++* free it directly(put_page--unref page), not put into pool ++*/ ++u32 mali_mem_cow_release(mali_mem_backend *mem_bkend, mali_bool is_mali_mapped) ++{ ++ mali_mem_allocation *alloc; ++ struct mali_session_data *session; ++ u32 free_pages_nr = 0; ++ MALI_DEBUG_ASSERT_POINTER(mem_bkend); ++ MALI_DEBUG_ASSERT(MALI_MEM_COW == mem_bkend->type); ++ alloc = mem_bkend->mali_allocation; ++ MALI_DEBUG_ASSERT_POINTER(alloc); ++ ++ session = alloc->session; ++ MALI_DEBUG_ASSERT_POINTER(session); ++ ++ if (MALI_MEM_BACKEND_FLAG_SWAP_COWED != (MALI_MEM_BACKEND_FLAG_SWAP_COWED & mem_bkend->flags)) { ++ /* Unmap the memory from the mali virtual address space. */ ++ if (MALI_TRUE == is_mali_mapped) ++ mali_mem_os_mali_unmap(alloc); ++ /* free cow backend list*/ ++ _mali_osk_mutex_wait(session->cow_lock); ++ free_pages_nr = mali_mem_os_free(&mem_bkend->cow_mem.pages, mem_bkend->cow_mem.count, MALI_TRUE); ++ _mali_osk_mutex_signal(session->cow_lock); ++ ++ free_pages_nr += mali_mem_block_free_list(&mem_bkend->cow_mem.pages); ++ ++ MALI_DEBUG_ASSERT(list_empty(&mem_bkend->cow_mem.pages)); ++ } else { ++ free_pages_nr = mali_mem_swap_release(mem_bkend, is_mali_mapped); ++ } ++ ++ ++ MALI_DEBUG_PRINT(4, ("COW Mem free : allocated size = 0x%x, free size = 0x%x\n", mem_bkend->cow_mem.count * _MALI_OSK_MALI_PAGE_SIZE, ++ free_pages_nr * _MALI_OSK_MALI_PAGE_SIZE)); ++ ++ mem_bkend->cow_mem.count = 0; ++ return free_pages_nr; ++} ++ ++ ++/* Dst node could os node or swap node. */ ++void _mali_mem_cow_copy_page(mali_page_node *src_node, mali_page_node *dst_node) ++{ ++ void *dst, *src; ++ struct page *dst_page; ++ dma_addr_t dma_addr; ++ ++ MALI_DEBUG_ASSERT(src_node != NULL); ++ MALI_DEBUG_ASSERT(dst_node != NULL); ++ MALI_DEBUG_ASSERT(dst_node->type == MALI_PAGE_NODE_OS ++ || dst_node->type == MALI_PAGE_NODE_SWAP); ++ ++ if (dst_node->type == MALI_PAGE_NODE_OS) { ++ dst_page = dst_node->page; ++ } else { ++ dst_page = dst_node->swap_it->page; ++ } ++ ++ dma_unmap_page(&mali_platform_device->dev, _mali_page_node_get_dma_addr(dst_node), ++ _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL); ++ ++ /* map it , and copy the content*/ ++ dst = kmap_atomic(dst_page); ++ ++ if (src_node->type == MALI_PAGE_NODE_OS || ++ src_node->type == MALI_PAGE_NODE_SWAP) { ++ struct page *src_page; ++ ++ if (src_node->type == MALI_PAGE_NODE_OS) { ++ src_page = src_node->page; ++ } else { ++ src_page = src_node->swap_it->page; ++ } ++ ++ /* Clear and invaliate cache */ ++ /* In ARM architecture, speculative read may pull stale data into L1 cache ++ * for kernel linear mapping page table. DMA_BIDIRECTIONAL could ++ * invalidate the L1 cache so that following read get the latest data ++ */ ++ dma_unmap_page(&mali_platform_device->dev, _mali_page_node_get_dma_addr(src_node), ++ _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL); ++ ++ src = kmap_atomic(src_page); ++ memcpy(dst, src , _MALI_OSK_MALI_PAGE_SIZE); ++ kunmap_atomic(src); ++ dma_addr = dma_map_page(&mali_platform_device->dev, src_page, ++ 0, _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL); ++ ++ if (src_node->type == MALI_PAGE_NODE_SWAP) { ++ src_node->swap_it->dma_addr = dma_addr; ++ } ++ } else if (src_node->type == MALI_PAGE_NODE_BLOCK) { ++ /* ++ * use ioremap to map src for BLOCK memory ++ */ ++ src = ioremap(_mali_page_node_get_dma_addr(src_node), _MALI_OSK_MALI_PAGE_SIZE); ++ memcpy(dst, src , _MALI_OSK_MALI_PAGE_SIZE); ++ iounmap(src); ++ } ++ kunmap_atomic(dst); ++ dma_addr = dma_map_page(&mali_platform_device->dev, dst_page, ++ 0, _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL); ++ ++ if (dst_node->type == MALI_PAGE_NODE_SWAP) { ++ dst_node->swap_it->dma_addr = dma_addr; ++ } ++} ++ ++ ++/* ++* allocate page on demand when CPU access it, ++* THis used in page fault handler ++*/ ++_mali_osk_errcode_t mali_mem_cow_allocate_on_demand(mali_mem_backend *mem_bkend, u32 offset_page) ++{ ++ struct page *new_page = NULL; ++ struct mali_page_node *new_node = NULL; ++ int i = 0; ++ struct mali_page_node *m_page, *found_node = NULL; ++ struct mali_session_data *session = NULL; ++ mali_mem_cow *cow = &mem_bkend->cow_mem; ++ MALI_DEBUG_ASSERT(MALI_MEM_COW == mem_bkend->type); ++ MALI_DEBUG_ASSERT(offset_page < mem_bkend->size / _MALI_OSK_MALI_PAGE_SIZE); ++ MALI_DEBUG_PRINT(4, ("mali_mem_cow_allocate_on_demand !, offset_page =0x%x\n", offset_page)); ++ ++ /* allocate new page here */ ++ new_page = mali_mem_cow_alloc_page(); ++ if (!new_page) ++ return _MALI_OSK_ERR_NOMEM; ++ ++ new_node = _mali_page_node_allocate(MALI_PAGE_NODE_OS); ++ if (!new_node) { ++ __free_page(new_page); ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ ++ /* find the page in backend*/ ++ list_for_each_entry(m_page, &cow->pages, list) { ++ if (i == offset_page) { ++ found_node = m_page; ++ break; ++ } ++ i++; ++ } ++ MALI_DEBUG_ASSERT(found_node); ++ if (NULL == found_node) { ++ __free_page(new_page); ++ kfree(new_node); ++ return _MALI_OSK_ERR_ITEM_NOT_FOUND; ++ } ++ ++ _mali_page_node_add_page(new_node, new_page); ++ ++ /* Copy the src page's content to new page */ ++ _mali_mem_cow_copy_page(found_node, new_node); ++ ++ MALI_DEBUG_ASSERT_POINTER(mem_bkend->mali_allocation); ++ session = mem_bkend->mali_allocation->session; ++ MALI_DEBUG_ASSERT_POINTER(session); ++ if (1 != _mali_page_node_get_ref_count(found_node)) { ++ atomic_add(1, &session->mali_mem_allocated_pages); ++ if (atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > session->max_mali_mem_allocated_size) { ++ session->max_mali_mem_allocated_size = atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE; ++ } ++ mem_bkend->cow_mem.change_pages_nr++; ++ } ++ ++ _mali_osk_mutex_wait(session->cow_lock); ++ if (_mali_mem_put_page_node(found_node)) { ++ __free_page(new_page); ++ kfree(new_node); ++ _mali_osk_mutex_signal(session->cow_lock); ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ _mali_osk_mutex_signal(session->cow_lock); ++ ++ list_replace(&found_node->list, &new_node->list); ++ ++ kfree(found_node); ++ ++ /* map to GPU side*/ ++ _mali_osk_mutex_wait(session->memory_lock); ++ mali_mem_cow_mali_map(mem_bkend, offset_page * _MALI_OSK_MALI_PAGE_SIZE, _MALI_OSK_MALI_PAGE_SIZE); ++ _mali_osk_mutex_signal(session->memory_lock); ++ return _MALI_OSK_ERR_OK; ++} +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_cow.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_cow.h +new file mode 100755 +index 000000000000..5f83a37fc8f8 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_cow.h +@@ -0,0 +1,48 @@ ++/* ++ * Copyright (C) 2013-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_MEMORY_COW_H__ ++#define __MALI_MEMORY_COW_H__ ++ ++#include "mali_osk.h" ++#include "mali_session.h" ++#include "mali_memory_types.h" ++ ++int mali_mem_cow_cpu_map(mali_mem_backend *mem_bkend, struct vm_area_struct *vma); ++_mali_osk_errcode_t mali_mem_cow_cpu_map_pages_locked(mali_mem_backend *mem_bkend, ++ struct vm_area_struct *vma, ++ unsigned long vaddr, ++ int num); ++ ++_mali_osk_errcode_t mali_memory_do_cow(mali_mem_backend *target_bk, ++ u32 target_offset, ++ u32 target_size, ++ mali_mem_backend *backend, ++ u32 range_start, ++ u32 range_size); ++ ++_mali_osk_errcode_t mali_memory_cow_modify_range(mali_mem_backend *backend, ++ u32 range_start, ++ u32 range_size); ++ ++_mali_osk_errcode_t mali_memory_cow_os_memory(mali_mem_backend *target_bk, ++ u32 target_offset, ++ u32 target_size, ++ mali_mem_backend *backend, ++ u32 range_start, ++ u32 range_size); ++ ++void _mali_mem_cow_copy_page(mali_page_node *src_node, mali_page_node *dst_node); ++ ++int mali_mem_cow_mali_map(mali_mem_backend *mem_bkend, u32 range_start, u32 range_size); ++u32 mali_mem_cow_release(mali_mem_backend *mem_bkend, mali_bool is_mali_mapped); ++_mali_osk_errcode_t mali_mem_cow_allocate_on_demand(mali_mem_backend *mem_bkend, u32 offset_page); ++#endif ++ +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_defer_bind.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_defer_bind.c +new file mode 100755 +index 000000000000..a9db577cb851 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_defer_bind.c +@@ -0,0 +1,262 @@ ++/* ++ * Copyright (C) 2013-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#ifdef CONFIG_ARM ++#include ++#endif ++#include ++ ++#include "mali_memory.h" ++#include "mali_kernel_common.h" ++#include "mali_uk_types.h" ++#include "mali_osk.h" ++#include "mali_kernel_linux.h" ++#include "mali_memory_defer_bind.h" ++#include "mali_executor.h" ++#include "mali_osk.h" ++#include "mali_scheduler.h" ++#include "mali_gp_job.h" ++ ++mali_defer_bind_manager *mali_dmem_man = NULL; ++ ++static u32 mali_dmem_get_gp_varying_size(struct mali_gp_job *gp_job) ++{ ++ return gp_job->required_varying_memsize / _MALI_OSK_MALI_PAGE_SIZE; ++} ++ ++_mali_osk_errcode_t mali_mem_defer_bind_manager_init(void) ++{ ++ mali_dmem_man = _mali_osk_calloc(1, sizeof(struct mali_defer_bind_manager)); ++ if (!mali_dmem_man) ++ return _MALI_OSK_ERR_NOMEM; ++ ++ atomic_set(&mali_dmem_man->num_used_pages, 0); ++ atomic_set(&mali_dmem_man->num_dmem, 0); ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++ ++void mali_mem_defer_bind_manager_destory(void) ++{ ++ if (mali_dmem_man) { ++ MALI_DEBUG_ASSERT(0 == atomic_read(&mali_dmem_man->num_dmem)); ++ kfree(mali_dmem_man); ++ } ++ mali_dmem_man = NULL; ++} ++ ++ ++/*allocate pages from OS memory*/ ++_mali_osk_errcode_t mali_mem_defer_alloc_mem(u32 require, struct mali_session_data *session, mali_defer_mem_block *dblock) ++{ ++ int retval = 0; ++ u32 num_pages = require; ++ mali_mem_os_mem os_mem; ++ ++ retval = mali_mem_os_alloc_pages(&os_mem, num_pages * _MALI_OSK_MALI_PAGE_SIZE); ++ ++ /* add to free pages list */ ++ if (0 == retval) { ++ MALI_DEBUG_PRINT(4, ("mali_mem_defer_alloc_mem ,,*** pages allocate = 0x%x \n", num_pages)); ++ list_splice(&os_mem.pages, &dblock->free_pages); ++ atomic_add(os_mem.count, &dblock->num_free_pages); ++ atomic_add(os_mem.count, &session->mali_mem_allocated_pages); ++ if (atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > session->max_mali_mem_allocated_size) { ++ session->max_mali_mem_allocated_size = atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE; ++ } ++ return _MALI_OSK_ERR_OK; ++ } else ++ return _MALI_OSK_ERR_FAULT; ++} ++ ++_mali_osk_errcode_t mali_mem_prepare_mem_for_job(struct mali_gp_job *next_gp_job, mali_defer_mem_block *dblock) ++{ ++ u32 require_page; ++ ++ if (!next_gp_job) ++ return _MALI_OSK_ERR_FAULT; ++ ++ require_page = mali_dmem_get_gp_varying_size(next_gp_job); ++ ++ MALI_DEBUG_PRINT(4, ("mali_mem_defer_prepare_mem_work, require alloc page 0x%x\n", ++ require_page)); ++ /* allocate more pages from OS */ ++ if (_MALI_OSK_ERR_OK != mali_mem_defer_alloc_mem(require_page, next_gp_job->session, dblock)) { ++ MALI_DEBUG_PRINT(1, ("ERROR##mali_mem_defer_prepare_mem_work, allocate page failed!!")); ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ ++ next_gp_job->bind_flag = MALI_DEFER_BIND_MEMORY_PREPARED; ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++ ++/* do preparetion for allocation before defer bind */ ++_mali_osk_errcode_t mali_mem_defer_bind_allocation_prepare(mali_mem_allocation *alloc, struct list_head *list, u32 *required_varying_memsize) ++{ ++ mali_mem_backend *mem_bkend = NULL; ++ struct mali_backend_bind_list *bk_list = _mali_osk_calloc(1, sizeof(struct mali_backend_bind_list)); ++ if (NULL == bk_list) ++ return _MALI_OSK_ERR_FAULT; ++ ++ INIT_LIST_HEAD(&bk_list->node); ++ /* Get backend memory */ ++ mutex_lock(&mali_idr_mutex); ++ if (!(mem_bkend = idr_find(&mali_backend_idr, alloc->backend_handle))) { ++ MALI_DEBUG_PRINT(1, ("Can't find memory backend in defer bind!\n")); ++ mutex_unlock(&mali_idr_mutex); ++ _mali_osk_free(bk_list); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ mutex_unlock(&mali_idr_mutex); ++ ++ /* If the mem backend has already been bound, no need to bind again.*/ ++ if (mem_bkend->os_mem.count > 0) { ++ _mali_osk_free(bk_list); ++ return _MALI_OSK_ERR_OK; ++ } ++ ++ MALI_DEBUG_PRINT(4, ("bind_allocation_prepare:: allocation =%x vaddr=0x%x!\n", alloc, alloc->mali_vma_node.vm_node.start)); ++ ++ INIT_LIST_HEAD(&mem_bkend->os_mem.pages); ++ ++ bk_list->bkend = mem_bkend; ++ bk_list->vaddr = alloc->mali_vma_node.vm_node.start; ++ bk_list->session = alloc->session; ++ bk_list->page_num = mem_bkend->size / _MALI_OSK_MALI_PAGE_SIZE; ++ *required_varying_memsize += mem_bkend->size; ++ MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_OS); ++ ++ /* add to job to do list */ ++ list_add(&bk_list->node, list); ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++ ++ ++/* bind phyiscal memory to allocation ++This function will be called in IRQ handler*/ ++static _mali_osk_errcode_t mali_mem_defer_bind_allocation(struct mali_backend_bind_list *bk_node, ++ struct list_head *pages) ++{ ++ struct mali_session_data *session = bk_node->session; ++ mali_mem_backend *mem_bkend = bk_node->bkend; ++ MALI_DEBUG_PRINT(4, ("mali_mem_defer_bind_allocation, bind bkend = %x page num=0x%x vaddr=%x session=%x\n", mem_bkend, bk_node->page_num, bk_node->vaddr, session)); ++ ++ MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_OS); ++ list_splice(pages, &mem_bkend->os_mem.pages); ++ mem_bkend->os_mem.count = bk_node->page_num; ++ ++ if (mem_bkend->type == MALI_MEM_OS) { ++ mali_mem_os_mali_map(&mem_bkend->os_mem, session, bk_node->vaddr, 0, ++ mem_bkend->os_mem.count, MALI_MMU_FLAGS_DEFAULT); ++ } ++ smp_wmb(); ++ bk_node->flag = MALI_DEFER_BIND_MEMORY_BINDED; ++ mem_bkend->flags &= ~MALI_MEM_BACKEND_FLAG_NOT_BINDED; ++ mem_bkend->flags |= MALI_MEM_BACKEND_FLAG_BINDED; ++ return _MALI_OSK_ERR_OK; ++} ++ ++ ++static struct list_head *mali_mem_defer_get_free_page_list(u32 count, struct list_head *pages, mali_defer_mem_block *dblock) ++{ ++ int i = 0; ++ struct mali_page_node *m_page, *m_tmp; ++ ++ if (atomic_read(&dblock->num_free_pages) < count) { ++ return NULL; ++ } else { ++ list_for_each_entry_safe(m_page, m_tmp, &dblock->free_pages, list) { ++ if (i < count) { ++ list_move_tail(&m_page->list, pages); ++ } else { ++ break; ++ } ++ i++; ++ } ++ MALI_DEBUG_ASSERT(i == count); ++ atomic_sub(count, &dblock->num_free_pages); ++ return pages; ++ } ++} ++ ++ ++/* called in job start IOCTL to bind physical memory for each allocations ++@ bk_list backend list to do defer bind ++@ pages page list to do this bind ++@ count number of pages ++*/ ++_mali_osk_errcode_t mali_mem_defer_bind(struct mali_gp_job *gp, ++ struct mali_defer_mem_block *dmem_block) ++{ ++ struct mali_defer_mem *dmem = NULL; ++ struct mali_backend_bind_list *bkn, *bkn_tmp; ++ LIST_HEAD(pages); ++ ++ if (gp->required_varying_memsize != (atomic_read(&dmem_block->num_free_pages) * _MALI_OSK_MALI_PAGE_SIZE)) { ++ MALI_DEBUG_PRINT_ERROR(("#BIND: The memsize of varying buffer not match to the pagesize of the dmem_block!!## \n")); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ MALI_DEBUG_PRINT(4, ("#BIND: GP job=%x## \n", gp)); ++ dmem = (mali_defer_mem *)_mali_osk_calloc(1, sizeof(struct mali_defer_mem)); ++ if (dmem) { ++ INIT_LIST_HEAD(&dmem->node); ++ gp->dmem = dmem; ++ } else { ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ ++ atomic_add(1, &mali_dmem_man->num_dmem); ++ /* for each bk_list backend, do bind */ ++ list_for_each_entry_safe(bkn, bkn_tmp , &gp->vary_todo, node) { ++ INIT_LIST_HEAD(&pages); ++ if (likely(mali_mem_defer_get_free_page_list(bkn->page_num, &pages, dmem_block))) { ++ list_del(&bkn->node); ++ mali_mem_defer_bind_allocation(bkn, &pages); ++ _mali_osk_free(bkn); ++ } else { ++ /* not enough memory will not happen */ ++ MALI_DEBUG_PRINT_ERROR(("#BIND: NOT enough memory when binded !!## \n")); ++ _mali_osk_free(gp->dmem); ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ } ++ ++ if (!list_empty(&gp->vary_todo)) { ++ MALI_DEBUG_PRINT_ERROR(("#BIND: The deferbind backend list isn't empty !!## \n")); ++ _mali_osk_free(gp->dmem); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ dmem->flag = MALI_DEFER_BIND_MEMORY_BINDED; ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++void mali_mem_defer_dmem_free(struct mali_gp_job *gp) ++{ ++ if (gp->dmem) { ++ atomic_dec(&mali_dmem_man->num_dmem); ++ _mali_osk_free(gp->dmem); ++ } ++} ++ +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_defer_bind.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_defer_bind.h +new file mode 100755 +index 000000000000..defa08d52a46 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_defer_bind.h +@@ -0,0 +1,64 @@ ++/* ++ * Copyright (C) 2013-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++#ifndef __MALI_MEMORY_DEFER_BIND_H_ ++#define __MALI_MEMORY_DEFER_BIND_H_ ++ ++ ++#include "mali_osk.h" ++#include "mali_session.h" ++ ++#include ++#include ++#include ++#include ++#include ++ ++ ++#include "mali_memory_types.h" ++#include "mali_memory_os_alloc.h" ++#include "mali_uk_types.h" ++ ++struct mali_gp_job; ++ ++typedef struct mali_defer_mem { ++ struct list_head node; /*dlist node in bind manager */ ++ u32 flag; ++} mali_defer_mem; ++ ++ ++typedef struct mali_defer_mem_block { ++ struct list_head free_pages; /* page pool */ ++ atomic_t num_free_pages; ++} mali_defer_mem_block; ++ ++/* varying memory list need to bind */ ++typedef struct mali_backend_bind_list { ++ struct list_head node; ++ struct mali_mem_backend *bkend; ++ u32 vaddr; ++ u32 page_num; ++ struct mali_session_data *session; ++ u32 flag; ++} mali_backend_bind_lists; ++ ++ ++typedef struct mali_defer_bind_manager { ++ atomic_t num_used_pages; ++ atomic_t num_dmem; ++} mali_defer_bind_manager; ++ ++_mali_osk_errcode_t mali_mem_defer_bind_manager_init(void); ++void mali_mem_defer_bind_manager_destory(void); ++_mali_osk_errcode_t mali_mem_defer_bind(struct mali_gp_job *gp, struct mali_defer_mem_block *dmem_block); ++_mali_osk_errcode_t mali_mem_defer_bind_allocation_prepare(mali_mem_allocation *alloc, struct list_head *list, u32 *required_varying_memsize); ++_mali_osk_errcode_t mali_mem_prepare_mem_for_job(struct mali_gp_job *next_gp_job, mali_defer_mem_block *dblock); ++void mali_mem_defer_dmem_free(struct mali_gp_job *gp); ++ ++#endif +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_dma_buf.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_dma_buf.c +new file mode 100755 +index 000000000000..1f4565127a6b +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_dma_buf.c +@@ -0,0 +1,369 @@ ++/* ++ * Copyright (C) 2012-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include /* file system operations */ ++#include /* user space access */ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "mali_ukk.h" ++#include "mali_osk.h" ++#include "mali_kernel_common.h" ++#include "mali_session.h" ++#include "mali_kernel_linux.h" ++ ++#include "mali_memory.h" ++#include "mali_memory_dma_buf.h" ++#include "mali_memory_virtual.h" ++#include "mali_pp_job.h" ++ ++/* ++ * Map DMA buf attachment \a mem into \a session at virtual address \a virt. ++ */ ++static int mali_dma_buf_map(mali_mem_backend *mem_backend) ++{ ++ mali_mem_allocation *alloc; ++ struct mali_dma_buf_attachment *mem; ++ struct mali_session_data *session; ++ struct mali_page_directory *pagedir; ++ _mali_osk_errcode_t err; ++ struct scatterlist *sg; ++ u32 virt, flags; ++ int i; ++ ++ MALI_DEBUG_ASSERT_POINTER(mem_backend); ++ ++ alloc = mem_backend->mali_allocation; ++ MALI_DEBUG_ASSERT_POINTER(alloc); ++ ++ mem = mem_backend->dma_buf.attachment; ++ MALI_DEBUG_ASSERT_POINTER(mem); ++ ++ session = alloc->session; ++ MALI_DEBUG_ASSERT_POINTER(session); ++ MALI_DEBUG_ASSERT(mem->session == session); ++ ++ virt = alloc->mali_vma_node.vm_node.start; ++ flags = alloc->flags; ++ ++ mali_session_memory_lock(session); ++ mem->map_ref++; ++ ++ MALI_DEBUG_PRINT(5, ("Mali DMA-buf: map attachment %p, new map_ref = %d\n", mem, mem->map_ref)); ++ ++ if (1 == mem->map_ref) { ++ ++ /* First reference taken, so we need to map the dma buf */ ++ MALI_DEBUG_ASSERT(!mem->is_mapped); ++ ++ mem->sgt = dma_buf_map_attachment(mem->attachment, DMA_BIDIRECTIONAL); ++ if (IS_ERR_OR_NULL(mem->sgt)) { ++ MALI_DEBUG_PRINT_ERROR(("Failed to map dma-buf attachment\n")); ++ mem->map_ref--; ++ mali_session_memory_unlock(session); ++ return -EFAULT; ++ } ++ ++ err = mali_mem_mali_map_prepare(alloc); ++ if (_MALI_OSK_ERR_OK != err) { ++ MALI_DEBUG_PRINT(1, ("Mapping of DMA memory failed\n")); ++ mem->map_ref--; ++ mali_session_memory_unlock(session); ++ return -ENOMEM; ++ } ++ ++ pagedir = mali_session_get_page_directory(session); ++ MALI_DEBUG_ASSERT_POINTER(pagedir); ++ ++ for_each_sg(mem->sgt->sgl, sg, mem->sgt->nents, i) { ++ u32 size = sg_dma_len(sg); ++ dma_addr_t phys = sg_dma_address(sg); ++ ++ /* sg must be page aligned. */ ++ MALI_DEBUG_ASSERT(0 == size % MALI_MMU_PAGE_SIZE); ++ MALI_DEBUG_ASSERT(0 == (phys & ~(uintptr_t)0xFFFFFFFF)); ++ ++ mali_mmu_pagedir_update(pagedir, virt, phys, size, MALI_MMU_FLAGS_DEFAULT); ++ ++ virt += size; ++ } ++ ++ if (flags & MALI_MEM_FLAG_MALI_GUARD_PAGE) { ++ u32 guard_phys; ++ MALI_DEBUG_PRINT(7, ("Mapping in extra guard page\n")); ++ ++ guard_phys = sg_dma_address(mem->sgt->sgl); ++ mali_mmu_pagedir_update(pagedir, virt, guard_phys, MALI_MMU_PAGE_SIZE, MALI_MMU_FLAGS_DEFAULT); ++ } ++ ++ mem->is_mapped = MALI_TRUE; ++ mali_session_memory_unlock(session); ++ /* Wake up any thread waiting for buffer to become mapped */ ++ wake_up_all(&mem->wait_queue); ++ } else { ++ MALI_DEBUG_ASSERT(mem->is_mapped); ++ mali_session_memory_unlock(session); ++ } ++ ++ return 0; ++} ++ ++static void mali_dma_buf_unmap(mali_mem_allocation *alloc, struct mali_dma_buf_attachment *mem) ++{ ++ MALI_DEBUG_ASSERT_POINTER(alloc); ++ MALI_DEBUG_ASSERT_POINTER(mem); ++ MALI_DEBUG_ASSERT_POINTER(mem->attachment); ++ MALI_DEBUG_ASSERT_POINTER(mem->buf); ++ MALI_DEBUG_ASSERT_POINTER(alloc->session); ++ ++ mali_session_memory_lock(alloc->session); ++ mem->map_ref--; ++ ++ MALI_DEBUG_PRINT(5, ("Mali DMA-buf: unmap attachment %p, new map_ref = %d\n", mem, mem->map_ref)); ++ ++ if (0 == mem->map_ref) { ++ dma_buf_unmap_attachment(mem->attachment, mem->sgt, DMA_BIDIRECTIONAL); ++ if (MALI_TRUE == mem->is_mapped) { ++ mali_mem_mali_map_free(alloc->session, alloc->psize, alloc->mali_vma_node.vm_node.start, ++ alloc->flags); ++ } ++ mem->is_mapped = MALI_FALSE; ++ } ++ mali_session_memory_unlock(alloc->session); ++ /* Wake up any thread waiting for buffer to become unmapped */ ++ wake_up_all(&mem->wait_queue); ++} ++ ++#if !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) ++int mali_dma_buf_map_job(struct mali_pp_job *job) ++{ ++ struct mali_dma_buf_attachment *mem; ++ _mali_osk_errcode_t err; ++ int i; ++ int ret = 0; ++ u32 num_memory_cookies; ++ struct mali_session_data *session; ++ struct mali_vma_node *mali_vma_node = NULL; ++ mali_mem_allocation *mali_alloc = NULL; ++ mali_mem_backend *mem_bkend = NULL; ++ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ ++ num_memory_cookies = mali_pp_job_num_memory_cookies(job); ++ ++ session = mali_pp_job_get_session(job); ++ ++ MALI_DEBUG_ASSERT_POINTER(session); ++ ++ for (i = 0; i < num_memory_cookies; i++) { ++ u32 mali_addr = mali_pp_job_get_memory_cookie(job, i); ++ mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_addr, 0); ++ MALI_DEBUG_ASSERT(NULL != mali_vma_node); ++ mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); ++ MALI_DEBUG_ASSERT(NULL != mali_alloc); ++ if (MALI_MEM_DMA_BUF != mali_alloc->type) { ++ continue; ++ } ++ ++ /* Get backend memory & Map on CPU */ ++ mutex_lock(&mali_idr_mutex); ++ mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle); ++ mutex_unlock(&mali_idr_mutex); ++ MALI_DEBUG_ASSERT(NULL != mem_bkend); ++ ++ mem = mem_bkend->dma_buf.attachment; ++ ++ MALI_DEBUG_ASSERT_POINTER(mem); ++ MALI_DEBUG_ASSERT(mem->session == mali_pp_job_get_session(job)); ++ ++ err = mali_dma_buf_map(mem_bkend); ++ if (0 != err) { ++ MALI_DEBUG_PRINT_ERROR(("Mali DMA-buf: Failed to map dma-buf for mali address %x\n", mali_addr)); ++ ret = -EFAULT; ++ continue; ++ } ++ } ++ return ret; ++} ++ ++void mali_dma_buf_unmap_job(struct mali_pp_job *job) ++{ ++ struct mali_dma_buf_attachment *mem; ++ int i; ++ u32 num_memory_cookies; ++ struct mali_session_data *session; ++ struct mali_vma_node *mali_vma_node = NULL; ++ mali_mem_allocation *mali_alloc = NULL; ++ mali_mem_backend *mem_bkend = NULL; ++ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ ++ num_memory_cookies = mali_pp_job_num_memory_cookies(job); ++ ++ session = mali_pp_job_get_session(job); ++ ++ MALI_DEBUG_ASSERT_POINTER(session); ++ ++ for (i = 0; i < num_memory_cookies; i++) { ++ u32 mali_addr = mali_pp_job_get_memory_cookie(job, i); ++ mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_addr, 0); ++ MALI_DEBUG_ASSERT(NULL != mali_vma_node); ++ mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); ++ MALI_DEBUG_ASSERT(NULL != mali_alloc); ++ if (MALI_MEM_DMA_BUF != mali_alloc->type) { ++ continue; ++ } ++ ++ /* Get backend memory & Map on CPU */ ++ mutex_lock(&mali_idr_mutex); ++ mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle); ++ mutex_unlock(&mali_idr_mutex); ++ MALI_DEBUG_ASSERT(NULL != mem_bkend); ++ ++ mem = mem_bkend->dma_buf.attachment; ++ ++ MALI_DEBUG_ASSERT_POINTER(mem); ++ MALI_DEBUG_ASSERT(mem->session == mali_pp_job_get_session(job)); ++ mali_dma_buf_unmap(mem_bkend->mali_allocation, mem); ++ } ++} ++#endif /* !CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH */ ++ ++int mali_dma_buf_get_size(struct mali_session_data *session, _mali_uk_dma_buf_get_size_s __user *user_arg) ++{ ++ _mali_uk_dma_buf_get_size_s args; ++ int fd; ++ struct dma_buf *buf; ++ ++ /* get call arguments from user space. copy_from_user returns how many bytes which where NOT copied */ ++ if (0 != copy_from_user(&args, (void __user *)user_arg, sizeof(_mali_uk_dma_buf_get_size_s))) { ++ return -EFAULT; ++ } ++ ++ /* Do DMA-BUF stuff */ ++ fd = args.mem_fd; ++ ++ buf = dma_buf_get(fd); ++ if (IS_ERR_OR_NULL(buf)) { ++ MALI_DEBUG_PRINT_ERROR(("Failed to get dma-buf from fd: %d\n", fd)); ++ return PTR_ERR_OR_ZERO(buf); ++ } ++ ++ if (0 != put_user(buf->size, &user_arg->size)) { ++ dma_buf_put(buf); ++ return -EFAULT; ++ } ++ ++ dma_buf_put(buf); ++ ++ return 0; ++} ++ ++_mali_osk_errcode_t mali_mem_bind_dma_buf(mali_mem_allocation *alloc, ++ mali_mem_backend *mem_backend, ++ int fd, u32 flags) ++{ ++ struct dma_buf *buf; ++ struct mali_dma_buf_attachment *dma_mem; ++ struct mali_session_data *session = alloc->session; ++ ++ MALI_DEBUG_ASSERT_POINTER(session); ++ MALI_DEBUG_ASSERT_POINTER(mem_backend); ++ MALI_DEBUG_ASSERT_POINTER(alloc); ++ ++ /* get dma buffer */ ++ buf = dma_buf_get(fd); ++ if (IS_ERR_OR_NULL(buf)) { ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ /* Currently, mapping of the full buffer are supported. */ ++ if (alloc->psize != buf->size) { ++ goto failed_alloc_mem; ++ } ++ ++ dma_mem = _mali_osk_calloc(1, sizeof(struct mali_dma_buf_attachment)); ++ if (NULL == dma_mem) { ++ goto failed_alloc_mem; ++ } ++ ++ dma_mem->buf = buf; ++ dma_mem->session = session; ++ dma_mem->map_ref = 0; ++ init_waitqueue_head(&dma_mem->wait_queue); ++ ++ dma_mem->attachment = dma_buf_attach(dma_mem->buf, &mali_platform_device->dev); ++ if (NULL == dma_mem->attachment) { ++ goto failed_dma_attach; ++ } ++ ++ mem_backend->dma_buf.attachment = dma_mem; ++ ++ alloc->flags |= MALI_MEM_FLAG_DONT_CPU_MAP; ++ if (flags & _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE) { ++ alloc->flags |= MALI_MEM_FLAG_MALI_GUARD_PAGE; ++ } ++ ++ ++#if defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) ++ /* Map memory into session's Mali virtual address space. */ ++ if (0 != mali_dma_buf_map(mem_backend)) { ++ goto Failed_dma_map; ++ } ++#endif ++ ++ return _MALI_OSK_ERR_OK; ++ ++#if defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) ++Failed_dma_map: ++ mali_dma_buf_unmap(alloc, dma_mem); ++#endif ++ /* Wait for buffer to become unmapped */ ++ wait_event(dma_mem->wait_queue, !dma_mem->is_mapped); ++ MALI_DEBUG_ASSERT(!dma_mem->is_mapped); ++ dma_buf_detach(dma_mem->buf, dma_mem->attachment); ++failed_dma_attach: ++ _mali_osk_free(dma_mem); ++failed_alloc_mem: ++ dma_buf_put(buf); ++ return _MALI_OSK_ERR_FAULT; ++} ++ ++void mali_mem_unbind_dma_buf(mali_mem_backend *mem_backend) ++{ ++ struct mali_dma_buf_attachment *mem; ++ MALI_DEBUG_ASSERT_POINTER(mem_backend); ++ MALI_DEBUG_ASSERT(MALI_MEM_DMA_BUF == mem_backend->type); ++ ++ mem = mem_backend->dma_buf.attachment; ++ MALI_DEBUG_ASSERT_POINTER(mem); ++ MALI_DEBUG_ASSERT_POINTER(mem->attachment); ++ MALI_DEBUG_ASSERT_POINTER(mem->buf); ++ MALI_DEBUG_PRINT(3, ("Mali DMA-buf: release attachment %p\n", mem)); ++ ++#if defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) ++ MALI_DEBUG_ASSERT_POINTER(mem_backend->mali_allocation); ++ /* We mapped implicitly on attach, so we need to unmap on release */ ++ mali_dma_buf_unmap(mem_backend->mali_allocation, mem); ++#endif ++ /* Wait for buffer to become unmapped */ ++ wait_event(mem->wait_queue, !mem->is_mapped); ++ MALI_DEBUG_ASSERT(!mem->is_mapped); ++ ++ dma_buf_detach(mem->buf, mem->attachment); ++ dma_buf_put(mem->buf); ++ ++ _mali_osk_free(mem); ++} +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_dma_buf.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_dma_buf.h +new file mode 100755 +index 000000000000..a9b2870389ff +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_dma_buf.h +@@ -0,0 +1,53 @@ ++/* ++ * Copyright (C) 2011-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_MEMORY_DMA_BUF_H__ ++#define __MALI_MEMORY_DMA_BUF_H__ ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++#include "mali_uk_types.h" ++#include "mali_osk.h" ++#include "mali_memory.h" ++ ++struct mali_pp_job; ++ ++struct mali_dma_buf_attachment; ++struct mali_dma_buf_attachment { ++ struct dma_buf *buf; ++ struct dma_buf_attachment *attachment; ++ struct sg_table *sgt; ++ struct mali_session_data *session; ++ int map_ref; ++ struct mutex map_lock; ++ mali_bool is_mapped; ++ wait_queue_head_t wait_queue; ++}; ++ ++int mali_dma_buf_get_size(struct mali_session_data *session, _mali_uk_dma_buf_get_size_s __user *arg); ++ ++void mali_mem_unbind_dma_buf(mali_mem_backend *mem_backend); ++ ++_mali_osk_errcode_t mali_mem_bind_dma_buf(mali_mem_allocation *alloc, ++ mali_mem_backend *mem_backend, ++ int fd, u32 flags); ++ ++#if !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) ++int mali_dma_buf_map_job(struct mali_pp_job *job); ++void mali_dma_buf_unmap_job(struct mali_pp_job *job); ++#endif ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif /* __MALI_MEMORY_DMA_BUF_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_external.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_external.c +new file mode 100755 +index 000000000000..76018b7ab90b +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_external.c +@@ -0,0 +1,89 @@ ++/* ++ * Copyright (C) 2013-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include "mali_kernel_common.h" ++#include "mali_osk.h" ++#include "mali_ukk.h" ++#include "mali_memory.h" ++#include "mali_mem_validation.h" ++#include "mali_uk_types.h" ++ ++void mali_mem_unbind_ext_buf(mali_mem_backend *mem_backend) ++{ ++ mali_mem_allocation *alloc; ++ struct mali_session_data *session; ++ MALI_DEBUG_ASSERT_POINTER(mem_backend); ++ alloc = mem_backend->mali_allocation; ++ MALI_DEBUG_ASSERT_POINTER(alloc); ++ MALI_DEBUG_ASSERT(MALI_MEM_EXTERNAL == mem_backend->type); ++ ++ session = alloc->session; ++ MALI_DEBUG_ASSERT_POINTER(session); ++ mali_session_memory_lock(session); ++ mali_mem_mali_map_free(session, alloc->psize, alloc->mali_vma_node.vm_node.start, ++ alloc->flags); ++ mali_session_memory_unlock(session); ++} ++ ++_mali_osk_errcode_t mali_mem_bind_ext_buf(mali_mem_allocation *alloc, ++ mali_mem_backend *mem_backend, ++ u32 phys_addr, ++ u32 flag) ++{ ++ struct mali_session_data *session; ++ _mali_osk_errcode_t err; ++ u32 virt, phys, size; ++ MALI_DEBUG_ASSERT_POINTER(mem_backend); ++ MALI_DEBUG_ASSERT_POINTER(alloc); ++ size = alloc->psize; ++ session = (struct mali_session_data *)(uintptr_t)alloc->session; ++ MALI_CHECK_NON_NULL(session, _MALI_OSK_ERR_INVALID_ARGS); ++ ++ /* check arguments */ ++ /* NULL might be a valid Mali address */ ++ if (!size) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS); ++ ++ /* size must be a multiple of the system page size */ ++ if (size % _MALI_OSK_MALI_PAGE_SIZE) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS); ++ ++ /* Validate the mali physical range */ ++ if (_MALI_OSK_ERR_OK != mali_mem_validation_check(phys_addr, size)) { ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ if (flag & _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE) { ++ alloc->flags |= MALI_MEM_FLAG_MALI_GUARD_PAGE; ++ } ++ ++ mali_session_memory_lock(session); ++ ++ virt = alloc->mali_vma_node.vm_node.start; ++ phys = phys_addr; ++ ++ err = mali_mem_mali_map_prepare(alloc); ++ if (_MALI_OSK_ERR_OK != err) { ++ mali_session_memory_unlock(session); ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ ++ mali_mmu_pagedir_update(session->page_directory, virt, phys, size, MALI_MMU_FLAGS_DEFAULT); ++ ++ if (alloc->flags & MALI_MEM_FLAG_MALI_GUARD_PAGE) { ++ mali_mmu_pagedir_update(session->page_directory, virt + size, phys, _MALI_OSK_MALI_PAGE_SIZE, MALI_MMU_FLAGS_DEFAULT); ++ } ++ MALI_DEBUG_PRINT(3, ++ ("Requested to map physical memory 0x%x-0x%x into virtual memory 0x%x\n", ++ phys_addr, (phys_addr + size - 1), ++ virt)); ++ mali_session_memory_unlock(session); ++ ++ MALI_SUCCESS; ++} ++ +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_external.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_external.h +new file mode 100755 +index 000000000000..2db178d96233 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_external.h +@@ -0,0 +1,29 @@ ++ ++/* ++ * Copyright (C) 2011-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_MEMORY_EXTERNAL_H__ ++#define __MALI_MEMORY_EXTERNAL_H__ ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++_mali_osk_errcode_t mali_mem_bind_ext_buf(mali_mem_allocation *alloc, ++ mali_mem_backend *mem_backend, ++ u32 phys_addr, ++ u32 flag); ++void mali_mem_unbind_ext_buf(mali_mem_backend *mem_backend); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_manager.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_manager.c +new file mode 100755 +index 000000000000..27dee0f19c81 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_manager.c +@@ -0,0 +1,993 @@ ++/* ++ * Copyright (C) 2013-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#if defined(CONFIG_DMA_SHARED_BUFFER) ++#include ++#endif ++#include ++ ++#include "mali_osk.h" ++#include "mali_osk_mali.h" ++#include "mali_kernel_linux.h" ++#include "mali_scheduler.h" ++#include "mali_memory.h" ++#include "mali_memory_os_alloc.h" ++#if defined(CONFIG_DMA_SHARED_BUFFER) ++#include "mali_memory_dma_buf.h" ++#include "mali_memory_secure.h" ++#endif ++#if defined(CONFIG_MALI400_UMP) ++#include "mali_memory_ump.h" ++#endif ++#include "mali_memory_manager.h" ++#include "mali_memory_virtual.h" ++#include "mali_memory_util.h" ++#include "mali_memory_external.h" ++#include "mali_memory_cow.h" ++#include "mali_memory_block_alloc.h" ++#include "mali_ukk.h" ++#include "mali_memory_swap_alloc.h" ++ ++/* ++* New memory system interface ++*/ ++ ++/*inti idr for backend memory */ ++struct idr mali_backend_idr; ++struct mutex mali_idr_mutex; ++ ++/* init allocation manager */ ++int mali_memory_manager_init(struct mali_allocation_manager *mgr) ++{ ++ /* init Locks */ ++ rwlock_init(&mgr->vm_lock); ++ mutex_init(&mgr->list_mutex); ++ ++ /* init link */ ++ INIT_LIST_HEAD(&mgr->head); ++ ++ /* init RB tree */ ++ mgr->allocation_mgr_rb = RB_ROOT; ++ mgr->mali_allocation_num = 0; ++ return 0; ++} ++ ++/* Deinit allocation manager ++* Do some check for debug ++*/ ++void mali_memory_manager_uninit(struct mali_allocation_manager *mgr) ++{ ++ /* check RB tree is empty */ ++ MALI_DEBUG_ASSERT(((void *)(mgr->allocation_mgr_rb.rb_node) == (void *)rb_last(&mgr->allocation_mgr_rb))); ++ /* check allocation List */ ++ MALI_DEBUG_ASSERT(list_empty(&mgr->head)); ++} ++ ++/* Prepare memory descriptor */ ++static mali_mem_allocation *mali_mem_allocation_struct_create(struct mali_session_data *session) ++{ ++ mali_mem_allocation *mali_allocation; ++ ++ /* Allocate memory */ ++ mali_allocation = (mali_mem_allocation *)kzalloc(sizeof(mali_mem_allocation), GFP_KERNEL); ++ if (NULL == mali_allocation) { ++ MALI_DEBUG_PRINT(1, ("mali_mem_allocation_struct_create: descriptor was NULL\n")); ++ return NULL; ++ } ++ ++ MALI_DEBUG_CODE(mali_allocation->magic = MALI_MEM_ALLOCATION_VALID_MAGIC); ++ ++ /* do init */ ++ mali_allocation->flags = 0; ++ mali_allocation->session = session; ++ ++ INIT_LIST_HEAD(&mali_allocation->list); ++ _mali_osk_atomic_init(&mali_allocation->mem_alloc_refcount, 1); ++ ++ /** ++ *add to session list ++ */ ++ mutex_lock(&session->allocation_mgr.list_mutex); ++ list_add_tail(&mali_allocation->list, &session->allocation_mgr.head); ++ session->allocation_mgr.mali_allocation_num++; ++ mutex_unlock(&session->allocation_mgr.list_mutex); ++ ++ return mali_allocation; ++} ++ ++void mali_mem_allocation_struct_destory(mali_mem_allocation *alloc) ++{ ++ MALI_DEBUG_ASSERT_POINTER(alloc); ++ MALI_DEBUG_ASSERT_POINTER(alloc->session); ++ mutex_lock(&alloc->session->allocation_mgr.list_mutex); ++ list_del(&alloc->list); ++ alloc->session->allocation_mgr.mali_allocation_num--; ++ mutex_unlock(&alloc->session->allocation_mgr.list_mutex); ++ ++ kfree(alloc); ++} ++ ++int mali_mem_backend_struct_create(mali_mem_backend **backend, u32 psize) ++{ ++ mali_mem_backend *mem_backend = NULL; ++ s32 ret = -ENOSPC; ++ s32 index = -1; ++ *backend = (mali_mem_backend *)kzalloc(sizeof(mali_mem_backend), GFP_KERNEL); ++ if (NULL == *backend) { ++ MALI_DEBUG_PRINT(1, ("mali_mem_backend_struct_create: backend descriptor was NULL\n")); ++ return -1; ++ } ++ mem_backend = *backend; ++ mem_backend->size = psize; ++ mutex_init(&mem_backend->mutex); ++ INIT_LIST_HEAD(&mem_backend->list); ++ mem_backend->using_count = 0; ++ ++ ++ /* link backend with id */ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 9, 0) ++again: ++ if (!idr_pre_get(&mali_backend_idr, GFP_KERNEL)) { ++ kfree(mem_backend); ++ return -ENOMEM; ++ } ++ mutex_lock(&mali_idr_mutex); ++ ret = idr_get_new_above(&mali_backend_idr, mem_backend, 1, &index); ++ mutex_unlock(&mali_idr_mutex); ++ ++ if (-ENOSPC == ret) { ++ kfree(mem_backend); ++ return -ENOSPC; ++ } ++ if (-EAGAIN == ret) ++ goto again; ++#else ++ mutex_lock(&mali_idr_mutex); ++ ret = idr_alloc(&mali_backend_idr, mem_backend, 1, MALI_S32_MAX, GFP_KERNEL); ++ mutex_unlock(&mali_idr_mutex); ++ index = ret; ++ if (ret < 0) { ++ MALI_DEBUG_PRINT(1, ("mali_mem_backend_struct_create: Can't allocate idr for backend! \n")); ++ kfree(mem_backend); ++ return -ENOSPC; ++ } ++#endif ++ return index; ++} ++ ++ ++static void mali_mem_backend_struct_destory(mali_mem_backend **backend, s32 backend_handle) ++{ ++ mali_mem_backend *mem_backend = *backend; ++ ++ mutex_lock(&mali_idr_mutex); ++ idr_remove(&mali_backend_idr, backend_handle); ++ mutex_unlock(&mali_idr_mutex); ++ kfree(mem_backend); ++ *backend = NULL; ++} ++ ++mali_mem_backend *mali_mem_backend_struct_search(struct mali_session_data *session, u32 mali_address) ++{ ++ struct mali_vma_node *mali_vma_node = NULL; ++ mali_mem_backend *mem_bkend = NULL; ++ mali_mem_allocation *mali_alloc = NULL; ++ MALI_DEBUG_ASSERT_POINTER(session); ++ mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_address, 0); ++ if (NULL == mali_vma_node) { ++ MALI_DEBUG_PRINT(1, ("mali_mem_backend_struct_search:vma node was NULL\n")); ++ return NULL; ++ } ++ mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); ++ /* Get backend memory & Map on CPU */ ++ mutex_lock(&mali_idr_mutex); ++ mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle); ++ mutex_unlock(&mali_idr_mutex); ++ MALI_DEBUG_ASSERT(NULL != mem_bkend); ++ return mem_bkend; ++} ++ ++static _mali_osk_errcode_t mali_mem_resize(struct mali_session_data *session, mali_mem_backend *mem_backend, u32 physical_size) ++{ ++ _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT; ++ int retval = 0; ++ mali_mem_allocation *mali_allocation = NULL; ++ mali_mem_os_mem tmp_os_mem; ++ s32 change_page_count; ++ ++ MALI_DEBUG_ASSERT_POINTER(session); ++ MALI_DEBUG_ASSERT_POINTER(mem_backend); ++ MALI_DEBUG_PRINT(4, (" mali_mem_resize_memory called! \n")); ++ MALI_DEBUG_ASSERT(0 == physical_size % MALI_MMU_PAGE_SIZE); ++ ++ mali_allocation = mem_backend->mali_allocation; ++ MALI_DEBUG_ASSERT_POINTER(mali_allocation); ++ ++ MALI_DEBUG_ASSERT(MALI_MEM_FLAG_CAN_RESIZE & mali_allocation->flags); ++ MALI_DEBUG_ASSERT(MALI_MEM_OS == mali_allocation->type); ++ ++ mutex_lock(&mem_backend->mutex); ++ ++ /* Do resize*/ ++ if (physical_size > mem_backend->size) { ++ u32 add_size = physical_size - mem_backend->size; ++ ++ MALI_DEBUG_ASSERT(0 == add_size % MALI_MMU_PAGE_SIZE); ++ ++ /* Allocate new pages from os mem */ ++ retval = mali_mem_os_alloc_pages(&tmp_os_mem, add_size); ++ ++ if (retval) { ++ if (-ENOMEM == retval) { ++ ret = _MALI_OSK_ERR_NOMEM; ++ } else { ++ ret = _MALI_OSK_ERR_FAULT; ++ } ++ MALI_DEBUG_PRINT(2, ("_mali_ukk_mem_resize: memory allocation failed !\n")); ++ goto failed_alloc_memory; ++ } ++ ++ MALI_DEBUG_ASSERT(tmp_os_mem.count == add_size / MALI_MMU_PAGE_SIZE); ++ ++ /* Resize the memory of the backend */ ++ ret = mali_mem_os_resize_pages(&tmp_os_mem, &mem_backend->os_mem, 0, tmp_os_mem.count); ++ ++ if (ret) { ++ MALI_DEBUG_PRINT(2, ("_mali_ukk_mem_resize: memory resizing failed !\n")); ++ goto failed_resize_pages; ++ } ++ ++ /*Resize cpu mapping */ ++ if (NULL != mali_allocation->cpu_mapping.vma) { ++ ret = mali_mem_os_resize_cpu_map_locked(mem_backend, mali_allocation->cpu_mapping.vma, mali_allocation->cpu_mapping.vma->vm_start + mem_backend->size, add_size); ++ if (unlikely(ret != _MALI_OSK_ERR_OK)) { ++ MALI_DEBUG_PRINT(2, ("_mali_ukk_mem_resize: cpu mapping failed !\n")); ++ goto failed_cpu_map; ++ } ++ } ++ ++ /* Resize mali mapping */ ++ _mali_osk_mutex_wait(session->memory_lock); ++ ret = mali_mem_mali_map_resize(mali_allocation, physical_size); ++ ++ if (ret) { ++ MALI_DEBUG_PRINT(1, ("_mali_ukk_mem_resize: mali map resize fail !\n")); ++ goto failed_gpu_map; ++ } ++ ++ ret = mali_mem_os_mali_map(&mem_backend->os_mem, session, mali_allocation->mali_vma_node.vm_node.start, ++ mali_allocation->psize / MALI_MMU_PAGE_SIZE, add_size / MALI_MMU_PAGE_SIZE, mali_allocation->mali_mapping.properties); ++ if (ret) { ++ MALI_DEBUG_PRINT(2, ("_mali_ukk_mem_resize: mali mapping failed !\n")); ++ goto failed_gpu_map; ++ } ++ ++ _mali_osk_mutex_signal(session->memory_lock); ++ } else { ++ u32 dec_size, page_count; ++ u32 vaddr = 0; ++ INIT_LIST_HEAD(&tmp_os_mem.pages); ++ tmp_os_mem.count = 0; ++ ++ dec_size = mem_backend->size - physical_size; ++ MALI_DEBUG_ASSERT(0 == dec_size % MALI_MMU_PAGE_SIZE); ++ ++ page_count = dec_size / MALI_MMU_PAGE_SIZE; ++ vaddr = mali_allocation->mali_vma_node.vm_node.start + physical_size; ++ ++ /* Resize the memory of the backend */ ++ ret = mali_mem_os_resize_pages(&mem_backend->os_mem, &tmp_os_mem, physical_size / MALI_MMU_PAGE_SIZE, page_count); ++ ++ if (ret) { ++ MALI_DEBUG_PRINT(4, ("_mali_ukk_mem_resize: mali map resize failed!\n")); ++ goto failed_resize_pages; ++ } ++ ++ /* Resize mali map */ ++ _mali_osk_mutex_wait(session->memory_lock); ++ mali_mem_mali_map_free(session, dec_size, vaddr, mali_allocation->flags); ++ _mali_osk_mutex_signal(session->memory_lock); ++ ++ /* Zap cpu mapping */ ++ if (0 != mali_allocation->cpu_mapping.addr) { ++ MALI_DEBUG_ASSERT(NULL != mali_allocation->cpu_mapping.vma); ++ zap_vma_ptes(mali_allocation->cpu_mapping.vma, mali_allocation->cpu_mapping.vma->vm_start + physical_size, dec_size); ++ } ++ ++ /* Free those extra pages */ ++ mali_mem_os_free(&tmp_os_mem.pages, tmp_os_mem.count, MALI_FALSE); ++ } ++ ++ /* Resize memory allocation and memory backend */ ++ change_page_count = (s32)(physical_size - mem_backend->size) / MALI_MMU_PAGE_SIZE; ++ mali_allocation->psize = physical_size; ++ mem_backend->size = physical_size; ++ mutex_unlock(&mem_backend->mutex); ++ ++ if (change_page_count > 0) { ++ atomic_add(change_page_count, &session->mali_mem_allocated_pages); ++ if (atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > session->max_mali_mem_allocated_size) { ++ session->max_mali_mem_allocated_size = atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE; ++ } ++ ++ } else { ++ atomic_sub((s32)(-change_page_count), &session->mali_mem_allocated_pages); ++ } ++ ++ return _MALI_OSK_ERR_OK; ++ ++failed_gpu_map: ++ _mali_osk_mutex_signal(session->memory_lock); ++failed_cpu_map: ++ if (physical_size > mem_backend->size) { ++ mali_mem_os_resize_pages(&mem_backend->os_mem, &tmp_os_mem, mem_backend->size / MALI_MMU_PAGE_SIZE, ++ (physical_size - mem_backend->size) / MALI_MMU_PAGE_SIZE); ++ } else { ++ mali_mem_os_resize_pages(&tmp_os_mem, &mem_backend->os_mem, 0, tmp_os_mem.count); ++ } ++failed_resize_pages: ++ if (0 != tmp_os_mem.count) ++ mali_mem_os_free(&tmp_os_mem.pages, tmp_os_mem.count, MALI_FALSE); ++failed_alloc_memory: ++ ++ mutex_unlock(&mem_backend->mutex); ++ return ret; ++} ++ ++ ++/* Set GPU MMU properties */ ++static void _mali_memory_gpu_map_property_set(u32 *properties, u32 flags) ++{ ++ if (_MALI_MEMORY_GPU_READ_ALLOCATE & flags) { ++ *properties = MALI_MMU_FLAGS_FORCE_GP_READ_ALLOCATE; ++ } else { ++ *properties = MALI_MMU_FLAGS_DEFAULT; ++ } ++} ++ ++_mali_osk_errcode_t mali_mem_add_mem_size(struct mali_session_data *session, u32 mali_addr, u32 add_size) ++{ ++ mali_mem_backend *mem_backend = NULL; ++ _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT; ++ mali_mem_allocation *mali_allocation = NULL; ++ u32 new_physical_size; ++ MALI_DEBUG_ASSERT_POINTER(session); ++ MALI_DEBUG_ASSERT(0 == add_size % MALI_MMU_PAGE_SIZE); ++ ++ /* Get the memory backend that need to be resize. */ ++ mem_backend = mali_mem_backend_struct_search(session, mali_addr); ++ ++ if (NULL == mem_backend) { ++ MALI_DEBUG_PRINT(2, ("_mali_ukk_mem_resize: memory backend = NULL!\n")); ++ return ret; ++ } ++ ++ mali_allocation = mem_backend->mali_allocation; ++ ++ MALI_DEBUG_ASSERT_POINTER(mali_allocation); ++ ++ new_physical_size = add_size + mem_backend->size; ++ ++ if (new_physical_size > (mali_allocation->mali_vma_node.vm_node.size)) ++ return ret; ++ ++ MALI_DEBUG_ASSERT(new_physical_size != mem_backend->size); ++ ++ ret = mali_mem_resize(session, mem_backend, new_physical_size); ++ ++ return ret; ++} ++ ++/** ++* function@_mali_ukk_mem_allocate - allocate mali memory ++*/ ++_mali_osk_errcode_t _mali_ukk_mem_allocate(_mali_uk_alloc_mem_s *args) ++{ ++ struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; ++ mali_mem_backend *mem_backend = NULL; ++ _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT; ++ int retval = 0; ++ mali_mem_allocation *mali_allocation = NULL; ++ struct mali_vma_node *mali_vma_node = NULL; ++ ++ MALI_DEBUG_PRINT(4, (" _mali_ukk_mem_allocate, vaddr=0x%x, size =0x%x! \n", args->gpu_vaddr, args->psize)); ++ ++ /* Check if the address is allocated ++ */ ++ mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, args->gpu_vaddr, 0); ++ ++ if (unlikely(mali_vma_node)) { ++ MALI_DEBUG_PRINT_ERROR(("The mali virtual address has already been used ! \n")); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ /** ++ *create mali memory allocation ++ */ ++ ++ mali_allocation = mali_mem_allocation_struct_create(session); ++ ++ if (mali_allocation == NULL) { ++ MALI_DEBUG_PRINT(1, ("_mali_ukk_mem_allocate: Failed to create allocation struct! \n")); ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ mali_allocation->psize = args->psize; ++ mali_allocation->vsize = args->vsize; ++ ++ /* MALI_MEM_OS if need to support mem resize, ++ * or MALI_MEM_BLOCK if have dedicated memory, ++ * or MALI_MEM_OS, ++ * or MALI_MEM_SWAP. ++ */ ++ if (args->flags & _MALI_MEMORY_ALLOCATE_SWAPPABLE) { ++ mali_allocation->type = MALI_MEM_SWAP; ++ } else if (args->flags & _MALI_MEMORY_ALLOCATE_RESIZEABLE) { ++ mali_allocation->type = MALI_MEM_OS; ++ mali_allocation->flags |= MALI_MEM_FLAG_CAN_RESIZE; ++ } else if (args->flags & _MALI_MEMORY_ALLOCATE_SECURE) { ++ mali_allocation->type = MALI_MEM_SECURE; ++ } else if (MALI_TRUE == mali_memory_have_dedicated_memory()) { ++ mali_allocation->type = MALI_MEM_BLOCK; ++ } else { ++ mali_allocation->type = MALI_MEM_OS; ++ } ++ ++ /** ++ *add allocation node to RB tree for index ++ */ ++ mali_allocation->mali_vma_node.vm_node.start = args->gpu_vaddr; ++ mali_allocation->mali_vma_node.vm_node.size = args->vsize; ++ ++ mali_vma_offset_add(&session->allocation_mgr, &mali_allocation->mali_vma_node); ++ ++ mali_allocation->backend_handle = mali_mem_backend_struct_create(&mem_backend, args->psize); ++ if (mali_allocation->backend_handle < 0) { ++ ret = _MALI_OSK_ERR_NOMEM; ++ MALI_DEBUG_PRINT(1, ("mali_allocation->backend_handle < 0! \n")); ++ goto failed_alloc_backend; ++ } ++ ++ ++ mem_backend->mali_allocation = mali_allocation; ++ mem_backend->type = mali_allocation->type; ++ ++ mali_allocation->mali_mapping.addr = args->gpu_vaddr; ++ ++ /* set gpu mmu propery */ ++ _mali_memory_gpu_map_property_set(&mali_allocation->mali_mapping.properties, args->flags); ++ /* do prepare for MALI mapping */ ++ if (!(args->flags & _MALI_MEMORY_ALLOCATE_NO_BIND_GPU) && mali_allocation->psize > 0) { ++ _mali_osk_mutex_wait(session->memory_lock); ++ ++ ret = mali_mem_mali_map_prepare(mali_allocation); ++ if (0 != ret) { ++ _mali_osk_mutex_signal(session->memory_lock); ++ goto failed_prepare_map; ++ } ++ _mali_osk_mutex_signal(session->memory_lock); ++ } ++ ++ if (mali_allocation->psize == 0) { ++ mem_backend->os_mem.count = 0; ++ INIT_LIST_HEAD(&mem_backend->os_mem.pages); ++ goto done; ++ } ++ ++ if (args->flags & _MALI_MEMORY_ALLOCATE_DEFER_BIND) { ++ mali_allocation->flags |= _MALI_MEMORY_ALLOCATE_DEFER_BIND; ++ mem_backend->flags |= MALI_MEM_BACKEND_FLAG_NOT_BINDED; ++ /* init for defer bind backend*/ ++ mem_backend->os_mem.count = 0; ++ INIT_LIST_HEAD(&mem_backend->os_mem.pages); ++ ++ goto done; ++ } ++ ++ if (likely(mali_allocation->psize > 0)) { ++ ++ if (MALI_MEM_SECURE == mem_backend->type) { ++#if defined(CONFIG_DMA_SHARED_BUFFER) ++ ret = mali_mem_secure_attach_dma_buf(&mem_backend->secure_mem, mem_backend->size, args->secure_shared_fd); ++ if (_MALI_OSK_ERR_OK != ret) { ++ MALI_DEBUG_PRINT(1, ("Failed to attach dma buf for secure memory! \n")); ++ goto failed_alloc_pages; ++ } ++#else ++ ret = _MALI_OSK_ERR_UNSUPPORTED; ++ MALI_DEBUG_PRINT(1, ("DMA not supported for mali secure memory! \n")); ++ goto failed_alloc_pages; ++#endif ++ } else { ++ ++ /** ++ *allocate physical memory ++ */ ++ if (mem_backend->type == MALI_MEM_OS) { ++ retval = mali_mem_os_alloc_pages(&mem_backend->os_mem, mem_backend->size); ++ } else if (mem_backend->type == MALI_MEM_BLOCK) { ++ /* try to allocated from BLOCK memory first, then try OS memory if failed.*/ ++ if (mali_mem_block_alloc(&mem_backend->block_mem, mem_backend->size)) { ++ retval = mali_mem_os_alloc_pages(&mem_backend->os_mem, mem_backend->size); ++ mem_backend->type = MALI_MEM_OS; ++ mali_allocation->type = MALI_MEM_OS; ++ } ++ } else if (MALI_MEM_SWAP == mem_backend->type) { ++ retval = mali_mem_swap_alloc_pages(&mem_backend->swap_mem, mali_allocation->mali_vma_node.vm_node.size, &mem_backend->start_idx); ++ } else { ++ /* ONLY support mem_os type */ ++ MALI_DEBUG_ASSERT(0); ++ } ++ ++ if (retval) { ++ ret = _MALI_OSK_ERR_NOMEM; ++ MALI_DEBUG_PRINT(1, (" can't allocate enough pages! \n")); ++ goto failed_alloc_pages; ++ } ++ } ++ } ++ ++ /** ++ *map to GPU side ++ */ ++ if (!(args->flags & _MALI_MEMORY_ALLOCATE_NO_BIND_GPU) && mali_allocation->psize > 0) { ++ _mali_osk_mutex_wait(session->memory_lock); ++ /* Map on Mali */ ++ ++ if (mem_backend->type == MALI_MEM_OS) { ++ ret = mali_mem_os_mali_map(&mem_backend->os_mem, session, args->gpu_vaddr, 0, ++ mem_backend->size / MALI_MMU_PAGE_SIZE, mali_allocation->mali_mapping.properties); ++ ++ } else if (mem_backend->type == MALI_MEM_BLOCK) { ++ mali_mem_block_mali_map(&mem_backend->block_mem, session, args->gpu_vaddr, ++ mali_allocation->mali_mapping.properties); ++ } else if (mem_backend->type == MALI_MEM_SWAP) { ++ ret = mali_mem_swap_mali_map(&mem_backend->swap_mem, session, args->gpu_vaddr, ++ mali_allocation->mali_mapping.properties); ++ } else if (mem_backend->type == MALI_MEM_SECURE) { ++#if defined(CONFIG_DMA_SHARED_BUFFER) ++ ret = mali_mem_secure_mali_map(&mem_backend->secure_mem, session, args->gpu_vaddr, mali_allocation->mali_mapping.properties); ++#endif ++ } else { /* unsupport type */ ++ MALI_DEBUG_ASSERT(0); ++ } ++ ++ _mali_osk_mutex_signal(session->memory_lock); ++ } ++done: ++ if (MALI_MEM_OS == mem_backend->type) { ++ atomic_add(mem_backend->os_mem.count, &session->mali_mem_allocated_pages); ++ } else if (MALI_MEM_BLOCK == mem_backend->type) { ++ atomic_add(mem_backend->block_mem.count, &session->mali_mem_allocated_pages); ++ } else if (MALI_MEM_SECURE == mem_backend->type) { ++ atomic_add(mem_backend->secure_mem.count, &session->mali_mem_allocated_pages); ++ } else { ++ MALI_DEBUG_ASSERT(MALI_MEM_SWAP == mem_backend->type); ++ atomic_add(mem_backend->swap_mem.count, &session->mali_mem_allocated_pages); ++ atomic_add(mem_backend->swap_mem.count, &session->mali_mem_array[mem_backend->type]); ++ } ++ ++ if (atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > session->max_mali_mem_allocated_size) { ++ session->max_mali_mem_allocated_size = atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE; ++ } ++ return _MALI_OSK_ERR_OK; ++ ++failed_alloc_pages: ++ mali_mem_mali_map_free(session, mali_allocation->psize, mali_allocation->mali_vma_node.vm_node.start, mali_allocation->flags); ++failed_prepare_map: ++ mali_mem_backend_struct_destory(&mem_backend, mali_allocation->backend_handle); ++failed_alloc_backend: ++ ++ mali_vma_offset_remove(&session->allocation_mgr, &mali_allocation->mali_vma_node); ++ mali_mem_allocation_struct_destory(mali_allocation); ++ ++ return ret; ++} ++ ++ ++_mali_osk_errcode_t _mali_ukk_mem_free(_mali_uk_free_mem_s *args) ++{ ++ struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; ++ u32 vaddr = args->gpu_vaddr; ++ mali_mem_allocation *mali_alloc = NULL; ++ struct mali_vma_node *mali_vma_node = NULL; ++ ++ /* find mali allocation structure by vaddress*/ ++ mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, vaddr, 0); ++ if (NULL == mali_vma_node) { ++ MALI_DEBUG_PRINT(1, ("_mali_ukk_mem_free: invalid addr: 0x%x\n", vaddr)); ++ return _MALI_OSK_ERR_INVALID_ARGS; ++ } ++ MALI_DEBUG_ASSERT(NULL != mali_vma_node); ++ mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); ++ ++ if (mali_alloc) ++ /* check ref_count */ ++ args->free_pages_nr = mali_allocation_unref(&mali_alloc); ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++ ++/** ++* Function _mali_ukk_mem_bind -- bind a external memory to a new GPU address ++* It will allocate a new mem allocation and bind external memory to it. ++* Supported backend type are: ++* _MALI_MEMORY_BIND_BACKEND_UMP ++* _MALI_MEMORY_BIND_BACKEND_DMA_BUF ++* _MALI_MEMORY_BIND_BACKEND_EXTERNAL_MEMORY ++* CPU access is not supported yet ++*/ ++_mali_osk_errcode_t _mali_ukk_mem_bind(_mali_uk_bind_mem_s *args) ++{ ++ struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; ++ mali_mem_backend *mem_backend = NULL; ++ _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT; ++ mali_mem_allocation *mali_allocation = NULL; ++ MALI_DEBUG_PRINT(5, (" _mali_ukk_mem_bind, vaddr=0x%x, size =0x%x! \n", args->vaddr, args->size)); ++ ++ /** ++ * allocate mali allocation. ++ */ ++ mali_allocation = mali_mem_allocation_struct_create(session); ++ ++ if (mali_allocation == NULL) { ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ mali_allocation->psize = args->size; ++ mali_allocation->vsize = args->size; ++ mali_allocation->mali_mapping.addr = args->vaddr; ++ ++ /* add allocation node to RB tree for index */ ++ mali_allocation->mali_vma_node.vm_node.start = args->vaddr; ++ mali_allocation->mali_vma_node.vm_node.size = args->size; ++ mali_vma_offset_add(&session->allocation_mgr, &mali_allocation->mali_vma_node); ++ ++ /* allocate backend*/ ++ if (mali_allocation->psize > 0) { ++ mali_allocation->backend_handle = mali_mem_backend_struct_create(&mem_backend, mali_allocation->psize); ++ if (mali_allocation->backend_handle < 0) { ++ goto Failed_alloc_backend; ++ } ++ ++ } else { ++ goto Failed_alloc_backend; ++ } ++ ++ mem_backend->size = mali_allocation->psize; ++ mem_backend->mali_allocation = mali_allocation; ++ ++ switch (args->flags & _MALI_MEMORY_BIND_BACKEND_MASK) { ++ case _MALI_MEMORY_BIND_BACKEND_UMP: ++#if defined(CONFIG_MALI400_UMP) ++ mali_allocation->type = MALI_MEM_UMP; ++ mem_backend->type = MALI_MEM_UMP; ++ ret = mali_mem_bind_ump_buf(mali_allocation, mem_backend, ++ args->mem_union.bind_ump.secure_id, args->mem_union.bind_ump.flags); ++ if (_MALI_OSK_ERR_OK != ret) { ++ MALI_DEBUG_PRINT(1, ("Bind ump buf failed\n")); ++ goto Failed_bind_backend; ++ } ++#else ++ MALI_DEBUG_PRINT(1, ("UMP not supported\n")); ++ goto Failed_bind_backend; ++#endif ++ break; ++ case _MALI_MEMORY_BIND_BACKEND_DMA_BUF: ++#if defined(CONFIG_DMA_SHARED_BUFFER) ++ mali_allocation->type = MALI_MEM_DMA_BUF; ++ mem_backend->type = MALI_MEM_DMA_BUF; ++ ret = mali_mem_bind_dma_buf(mali_allocation, mem_backend, ++ args->mem_union.bind_dma_buf.mem_fd, args->mem_union.bind_dma_buf.flags); ++ if (_MALI_OSK_ERR_OK != ret) { ++ MALI_DEBUG_PRINT(1, ("Bind dma buf failed\n")); ++ goto Failed_bind_backend; ++ } ++#else ++ MALI_DEBUG_PRINT(1, ("DMA not supported\n")); ++ goto Failed_bind_backend; ++#endif ++ break; ++ case _MALI_MEMORY_BIND_BACKEND_MALI_MEMORY: ++ /* not allowed */ ++ MALI_DEBUG_PRINT_ERROR(("Mali internal memory type not supported !\n")); ++ goto Failed_bind_backend; ++ break; ++ ++ case _MALI_MEMORY_BIND_BACKEND_EXTERNAL_MEMORY: ++ mali_allocation->type = MALI_MEM_EXTERNAL; ++ mem_backend->type = MALI_MEM_EXTERNAL; ++ ret = mali_mem_bind_ext_buf(mali_allocation, mem_backend, args->mem_union.bind_ext_memory.phys_addr, ++ args->mem_union.bind_ext_memory.flags); ++ if (_MALI_OSK_ERR_OK != ret) { ++ MALI_DEBUG_PRINT(1, ("Bind external buf failed\n")); ++ goto Failed_bind_backend; ++ } ++ break; ++ ++ case _MALI_MEMORY_BIND_BACKEND_EXT_COW: ++ /* not allowed */ ++ MALI_DEBUG_PRINT_ERROR(("External cow memory type not supported !\n")); ++ goto Failed_bind_backend; ++ break; ++ ++ default: ++ MALI_DEBUG_PRINT_ERROR(("Invalid memory type not supported !\n")); ++ goto Failed_bind_backend; ++ break; ++ } ++ MALI_DEBUG_ASSERT(0 == mem_backend->size % MALI_MMU_PAGE_SIZE); ++ atomic_add(mem_backend->size / MALI_MMU_PAGE_SIZE, &session->mali_mem_array[mem_backend->type]); ++ return _MALI_OSK_ERR_OK; ++ ++Failed_bind_backend: ++ mali_mem_backend_struct_destory(&mem_backend, mali_allocation->backend_handle); ++ ++Failed_alloc_backend: ++ mali_vma_offset_remove(&session->allocation_mgr, &mali_allocation->mali_vma_node); ++ mali_mem_allocation_struct_destory(mali_allocation); ++ ++ MALI_DEBUG_PRINT(1, (" _mali_ukk_mem_bind, return ERROR! \n")); ++ return ret; ++} ++ ++ ++/* ++* Function _mali_ukk_mem_unbind -- unbind a external memory to a new GPU address ++* This function unbind the backend memory and free the allocation ++* no ref_count for this type of memory ++*/ ++_mali_osk_errcode_t _mali_ukk_mem_unbind(_mali_uk_unbind_mem_s *args) ++{ ++ /**/ ++ struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; ++ mali_mem_allocation *mali_allocation = NULL; ++ struct mali_vma_node *mali_vma_node = NULL; ++ u32 mali_addr = args->vaddr; ++ MALI_DEBUG_PRINT(5, (" _mali_ukk_mem_unbind, vaddr=0x%x! \n", args->vaddr)); ++ ++ /* find the allocation by vaddr */ ++ mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_addr, 0); ++ if (likely(mali_vma_node)) { ++ MALI_DEBUG_ASSERT(mali_addr == mali_vma_node->vm_node.start); ++ mali_allocation = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); ++ } else { ++ MALI_DEBUG_ASSERT(NULL != mali_vma_node); ++ return _MALI_OSK_ERR_INVALID_ARGS; ++ } ++ ++ if (NULL != mali_allocation) ++ /* check ref_count */ ++ mali_allocation_unref(&mali_allocation); ++ return _MALI_OSK_ERR_OK; ++} ++ ++/* ++* Function _mali_ukk_mem_cow -- COW for an allocation ++* This function allocate new pages for a range (range, range+size) of allocation ++* And Map it(keep use the not in range pages from target allocation ) to an GPU vaddr ++*/ ++_mali_osk_errcode_t _mali_ukk_mem_cow(_mali_uk_cow_mem_s *args) ++{ ++ _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT; ++ mali_mem_backend *target_backend = NULL; ++ mali_mem_backend *mem_backend = NULL; ++ struct mali_vma_node *mali_vma_node = NULL; ++ mali_mem_allocation *mali_allocation = NULL; ++ ++ struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; ++ /* Get the target backend for cow */ ++ target_backend = mali_mem_backend_struct_search(session, args->target_handle); ++ ++ if (NULL == target_backend || 0 == target_backend->size) { ++ MALI_DEBUG_ASSERT_POINTER(target_backend); ++ MALI_DEBUG_ASSERT(0 != target_backend->size); ++ return ret; ++ } ++ ++ /*Cow not support resized mem */ ++ MALI_DEBUG_ASSERT(MALI_MEM_FLAG_CAN_RESIZE != (MALI_MEM_FLAG_CAN_RESIZE & target_backend->mali_allocation->flags)); ++ ++ /* Check if the new mali address is allocated */ ++ mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, args->vaddr, 0); ++ ++ if (unlikely(mali_vma_node)) { ++ MALI_DEBUG_PRINT_ERROR(("The mali virtual address has already been used ! \n")); ++ return ret; ++ } ++ ++ /* create new alloction for COW*/ ++ mali_allocation = mali_mem_allocation_struct_create(session); ++ if (mali_allocation == NULL) { ++ MALI_DEBUG_PRINT(1, ("_mali_ukk_mem_cow: Failed to create allocation struct!\n")); ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ mali_allocation->psize = args->target_size; ++ mali_allocation->vsize = args->target_size; ++ mali_allocation->type = MALI_MEM_COW; ++ ++ /*add allocation node to RB tree for index*/ ++ mali_allocation->mali_vma_node.vm_node.start = args->vaddr; ++ mali_allocation->mali_vma_node.vm_node.size = mali_allocation->vsize; ++ mali_vma_offset_add(&session->allocation_mgr, &mali_allocation->mali_vma_node); ++ ++ /* create new backend for COW memory */ ++ mali_allocation->backend_handle = mali_mem_backend_struct_create(&mem_backend, mali_allocation->psize); ++ if (mali_allocation->backend_handle < 0) { ++ ret = _MALI_OSK_ERR_NOMEM; ++ MALI_DEBUG_PRINT(1, ("mali_allocation->backend_handle < 0! \n")); ++ goto failed_alloc_backend; ++ } ++ mem_backend->mali_allocation = mali_allocation; ++ mem_backend->type = mali_allocation->type; ++ ++ if (target_backend->type == MALI_MEM_SWAP || ++ (MALI_MEM_COW == target_backend->type && (MALI_MEM_BACKEND_FLAG_SWAP_COWED & target_backend->flags))) { ++ mem_backend->flags |= MALI_MEM_BACKEND_FLAG_SWAP_COWED; ++ /** ++ * CoWed swap backends couldn't be mapped as non-linear vma, because if one ++ * vma is set with flag VM_NONLINEAR, the vma->vm_private_data will be used by kernel, ++ * while in mali driver, we use this variable to store the pointer of mali_allocation, so there ++ * is a conflict. ++ * To resolve this problem, we have to do some fake things, we reserved about 64MB ++ * space from index 0, there isn't really page's index will be set from 0 to (64MB>>PAGE_SHIFT_NUM), ++ * and all of CoWed swap memory backends' start_idx will be assigned with 0, and these ++ * backends will be mapped as linear and will add to priority tree of global swap file, while ++ * these vmas will never be found by using normal page->index, these pages in those vma ++ * also couldn't be swapped out. ++ */ ++ mem_backend->start_idx = 0; ++ } ++ ++ /* Add the target backend's cow count, also allocate new pages for COW backend from os mem ++ *for a modified range and keep the page which not in the modified range and Add ref to it ++ */ ++ MALI_DEBUG_PRINT(3, ("Cow mapping: target_addr: 0x%x; cow_addr: 0x%x, size: %u\n", target_backend->mali_allocation->mali_vma_node.vm_node.start, ++ mali_allocation->mali_vma_node.vm_node.start, mali_allocation->mali_vma_node.vm_node.size)); ++ ++ ret = mali_memory_do_cow(target_backend, args->target_offset, args->target_size, mem_backend, args->range_start, args->range_size); ++ if (_MALI_OSK_ERR_OK != ret) { ++ MALI_DEBUG_PRINT(1, ("_mali_ukk_mem_cow: Failed to cow!\n")); ++ goto failed_do_cow; ++ } ++ ++ /** ++ *map to GPU side ++ */ ++ mali_allocation->mali_mapping.addr = args->vaddr; ++ /* set gpu mmu propery */ ++ _mali_memory_gpu_map_property_set(&mali_allocation->mali_mapping.properties, args->flags); ++ ++ _mali_osk_mutex_wait(session->memory_lock); ++ /* Map on Mali */ ++ ret = mali_mem_mali_map_prepare(mali_allocation); ++ if (0 != ret) { ++ MALI_DEBUG_PRINT(1, (" prepare map fail! \n")); ++ goto failed_gpu_map; ++ } ++ ++ if (!(mem_backend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED)) { ++ mali_mem_cow_mali_map(mem_backend, 0, mem_backend->size); ++ } ++ ++ _mali_osk_mutex_signal(session->memory_lock); ++ ++ mutex_lock(&target_backend->mutex); ++ target_backend->flags |= MALI_MEM_BACKEND_FLAG_COWED; ++ mutex_unlock(&target_backend->mutex); ++ ++ atomic_add(args->range_size / MALI_MMU_PAGE_SIZE, &session->mali_mem_allocated_pages); ++ if (atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > session->max_mali_mem_allocated_size) { ++ session->max_mali_mem_allocated_size = atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE; ++ } ++ return _MALI_OSK_ERR_OK; ++ ++failed_gpu_map: ++ _mali_osk_mutex_signal(session->memory_lock); ++ mali_mem_cow_release(mem_backend, MALI_FALSE); ++ mem_backend->cow_mem.count = 0; ++failed_do_cow: ++ mali_mem_backend_struct_destory(&mem_backend, mali_allocation->backend_handle); ++failed_alloc_backend: ++ mali_vma_offset_remove(&session->allocation_mgr, &mali_allocation->mali_vma_node); ++ mali_mem_allocation_struct_destory(mali_allocation); ++ ++ return ret; ++} ++ ++_mali_osk_errcode_t _mali_ukk_mem_cow_modify_range(_mali_uk_cow_modify_range_s *args) ++{ ++ _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT; ++ mali_mem_backend *mem_backend = NULL; ++ struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; ++ ++ MALI_DEBUG_PRINT(4, (" _mali_ukk_mem_cow_modify_range called! \n")); ++ /* Get the backend that need to be modified. */ ++ mem_backend = mali_mem_backend_struct_search(session, args->vaddr); ++ ++ if (NULL == mem_backend || 0 == mem_backend->size) { ++ MALI_DEBUG_ASSERT_POINTER(mem_backend); ++ MALI_DEBUG_ASSERT(0 != mem_backend->size); ++ return ret; ++ } ++ ++ MALI_DEBUG_ASSERT(MALI_MEM_COW == mem_backend->type); ++ ++ ret = mali_memory_cow_modify_range(mem_backend, args->range_start, args->size); ++ args->change_pages_nr = mem_backend->cow_mem.change_pages_nr; ++ if (_MALI_OSK_ERR_OK != ret) ++ return ret; ++ _mali_osk_mutex_wait(session->memory_lock); ++ if (!(mem_backend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED)) { ++ mali_mem_cow_mali_map(mem_backend, args->range_start, args->size); ++ } ++ _mali_osk_mutex_signal(session->memory_lock); ++ ++ atomic_add(args->change_pages_nr, &session->mali_mem_allocated_pages); ++ if (atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > session->max_mali_mem_allocated_size) { ++ session->max_mali_mem_allocated_size = atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE; ++ } ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++ ++_mali_osk_errcode_t _mali_ukk_mem_resize(_mali_uk_mem_resize_s *args) ++{ ++ mali_mem_backend *mem_backend = NULL; ++ _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT; ++ ++ struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; ++ ++ MALI_DEBUG_ASSERT_POINTER(session); ++ MALI_DEBUG_PRINT(4, (" mali_mem_resize_memory called! \n")); ++ MALI_DEBUG_ASSERT(0 == args->psize % MALI_MMU_PAGE_SIZE); ++ ++ /* Get the memory backend that need to be resize. */ ++ mem_backend = mali_mem_backend_struct_search(session, args->vaddr); ++ ++ if (NULL == mem_backend) { ++ MALI_DEBUG_PRINT(2, ("_mali_ukk_mem_resize: memory backend = NULL!\n")); ++ return ret; ++ } ++ ++ MALI_DEBUG_ASSERT(args->psize != mem_backend->size); ++ ++ ret = mali_mem_resize(session, mem_backend, args->psize); ++ ++ return ret; ++} ++ ++_mali_osk_errcode_t _mali_ukk_mem_usage_get(_mali_uk_profiling_memory_usage_get_s *args) ++{ ++ args->memory_usage = _mali_ukk_report_memory_usage(); ++ if (0 != args->vaddr) { ++ mali_mem_backend *mem_backend = NULL; ++ struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; ++ /* Get the backend that need to be modified. */ ++ mem_backend = mali_mem_backend_struct_search(session, args->vaddr); ++ if (NULL == mem_backend) { ++ MALI_DEBUG_ASSERT_POINTER(mem_backend); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ if (MALI_MEM_COW == mem_backend->type) ++ args->change_pages_nr = mem_backend->cow_mem.change_pages_nr; ++ } ++ return _MALI_OSK_ERR_OK; ++} +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_manager.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_manager.h +new file mode 100755 +index 000000000000..23d8cde753a1 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_manager.h +@@ -0,0 +1,51 @@ ++/* ++ * Copyright (C) 2013-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_MEMORY_MANAGER_H__ ++#define __MALI_MEMORY_MANAGER_H__ ++ ++#include "mali_osk.h" ++#include ++#include ++#include ++#include ++#include ++#include "mali_memory_types.h" ++#include "mali_memory_os_alloc.h" ++#include "mali_uk_types.h" ++ ++struct mali_allocation_manager { ++ rwlock_t vm_lock; ++ struct rb_root allocation_mgr_rb; ++ struct list_head head; ++ struct mutex list_mutex; ++ u32 mali_allocation_num; ++}; ++ ++extern struct idr mali_backend_idr; ++extern struct mutex mali_idr_mutex; ++ ++int mali_memory_manager_init(struct mali_allocation_manager *mgr); ++void mali_memory_manager_uninit(struct mali_allocation_manager *mgr); ++ ++void mali_mem_allocation_struct_destory(mali_mem_allocation *alloc); ++_mali_osk_errcode_t mali_mem_add_mem_size(struct mali_session_data *session, u32 mali_addr, u32 add_size); ++mali_mem_backend *mali_mem_backend_struct_search(struct mali_session_data *session, u32 mali_address); ++_mali_osk_errcode_t _mali_ukk_mem_allocate(_mali_uk_alloc_mem_s *args); ++_mali_osk_errcode_t _mali_ukk_mem_free(_mali_uk_free_mem_s *args); ++_mali_osk_errcode_t _mali_ukk_mem_bind(_mali_uk_bind_mem_s *args); ++_mali_osk_errcode_t _mali_ukk_mem_unbind(_mali_uk_unbind_mem_s *args); ++_mali_osk_errcode_t _mali_ukk_mem_cow(_mali_uk_cow_mem_s *args); ++_mali_osk_errcode_t _mali_ukk_mem_cow_modify_range(_mali_uk_cow_modify_range_s *args); ++_mali_osk_errcode_t _mali_ukk_mem_usage_get(_mali_uk_profiling_memory_usage_get_s *args); ++_mali_osk_errcode_t _mali_ukk_mem_resize(_mali_uk_mem_resize_s *args); ++ ++#endif ++ +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_os_alloc.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_os_alloc.c +new file mode 100755 +index 000000000000..1e1f5eb4a0f7 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_os_alloc.c +@@ -0,0 +1,810 @@ ++/* ++ * Copyright (C) 2013-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include "../platform/rk/custom_log.h" ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "mali_osk.h" ++#include "mali_memory.h" ++#include "mali_memory_os_alloc.h" ++#include "mali_kernel_linux.h" ++ ++/* Minimum size of allocator page pool */ ++#define MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES (MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_MB * 256) ++#define MALI_OS_MEMORY_POOL_TRIM_JIFFIES (10 * CONFIG_HZ) /* Default to 10s */ ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 0, 0) ++#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35) ++static int mali_mem_os_shrink(int nr_to_scan, gfp_t gfp_mask); ++#else ++static int mali_mem_os_shrink(struct shrinker *shrinker, int nr_to_scan, gfp_t gfp_mask); ++#endif ++#else ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) ++static int mali_mem_os_shrink(struct shrinker *shrinker, struct shrink_control *sc); ++#else ++static unsigned long mali_mem_os_shrink(struct shrinker *shrinker, struct shrink_control *sc); ++static unsigned long mali_mem_os_shrink_count(struct shrinker *shrinker, struct shrink_control *sc); ++#endif ++#endif ++static void mali_mem_os_trim_pool(struct work_struct *work); ++ ++struct mali_mem_os_allocator mali_mem_os_allocator = { ++ .pool_lock = __SPIN_LOCK_UNLOCKED(pool_lock), ++ .pool_pages = LIST_HEAD_INIT(mali_mem_os_allocator.pool_pages), ++ .pool_count = 0, ++ ++ .allocated_pages = ATOMIC_INIT(0), ++ .allocation_limit = 0, ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) ++ .shrinker.shrink = mali_mem_os_shrink, ++#else ++ .shrinker.count_objects = mali_mem_os_shrink_count, ++ .shrinker.scan_objects = mali_mem_os_shrink, ++#endif ++ .shrinker.seeks = DEFAULT_SEEKS, ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0) ++ .timed_shrinker = __DELAYED_WORK_INITIALIZER(mali_mem_os_allocator.timed_shrinker, mali_mem_os_trim_pool, TIMER_DEFERRABLE), ++#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 38) ++ .timed_shrinker = __DEFERRED_WORK_INITIALIZER(mali_mem_os_allocator.timed_shrinker, mali_mem_os_trim_pool), ++#else ++ .timed_shrinker = __DELAYED_WORK_INITIALIZER(mali_mem_os_allocator.timed_shrinker, mali_mem_os_trim_pool), ++#endif ++}; ++ ++u32 mali_mem_os_free(struct list_head *os_pages, u32 pages_count, mali_bool cow_flag) ++{ ++ LIST_HEAD(pages); ++ struct mali_page_node *m_page, *m_tmp; ++ u32 free_pages_nr = 0; ++ ++ if (MALI_TRUE == cow_flag) { ++ list_for_each_entry_safe(m_page, m_tmp, os_pages, list) { ++ /*only handle OS node here */ ++ if (m_page->type == MALI_PAGE_NODE_OS) { ++ if (1 == _mali_page_node_get_ref_count(m_page)) { ++ list_move(&m_page->list, &pages); ++ atomic_sub(1, &mali_mem_os_allocator.allocated_pages); ++ free_pages_nr ++; ++ } else { ++ _mali_page_node_unref(m_page); ++ m_page->page = NULL; ++ list_del(&m_page->list); ++ kfree(m_page); ++ } ++ } ++ } ++ } else { ++ list_cut_position(&pages, os_pages, os_pages->prev); ++ atomic_sub(pages_count, &mali_mem_os_allocator.allocated_pages); ++ free_pages_nr = pages_count; ++ } ++ ++ /* Put pages on pool. */ ++ spin_lock(&mali_mem_os_allocator.pool_lock); ++ list_splice(&pages, &mali_mem_os_allocator.pool_pages); ++ mali_mem_os_allocator.pool_count += free_pages_nr; ++ spin_unlock(&mali_mem_os_allocator.pool_lock); ++ ++ if (MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES < mali_mem_os_allocator.pool_count) { ++ MALI_DEBUG_PRINT(5, ("OS Mem: Starting pool trim timer %u\n", mali_mem_os_allocator.pool_count)); ++ queue_delayed_work(mali_mem_os_allocator.wq, &mali_mem_os_allocator.timed_shrinker, MALI_OS_MEMORY_POOL_TRIM_JIFFIES); ++ } ++ return free_pages_nr; ++} ++ ++/** ++* put page without put it into page pool ++*/ ++_mali_osk_errcode_t mali_mem_os_put_page(struct page *page) ++{ ++ MALI_DEBUG_ASSERT_POINTER(page); ++ if (1 == page_count(page)) { ++ atomic_sub(1, &mali_mem_os_allocator.allocated_pages); ++ dma_unmap_page(&mali_platform_device->dev, page_private(page), ++ _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL); ++ ClearPagePrivate(page); ++ } ++ put_page(page); ++ return _MALI_OSK_ERR_OK; ++} ++ ++_mali_osk_errcode_t mali_mem_os_resize_pages(mali_mem_os_mem *mem_from, mali_mem_os_mem *mem_to, u32 start_page, u32 page_count) ++{ ++ struct mali_page_node *m_page, *m_tmp; ++ u32 i = 0; ++ ++ MALI_DEBUG_ASSERT_POINTER(mem_from); ++ MALI_DEBUG_ASSERT_POINTER(mem_to); ++ ++ if (mem_from->count < start_page + page_count) { ++ return _MALI_OSK_ERR_INVALID_ARGS; ++ } ++ ++ list_for_each_entry_safe(m_page, m_tmp, &mem_from->pages, list) { ++ if (i >= start_page && i < start_page + page_count) { ++ list_move_tail(&m_page->list, &mem_to->pages); ++ mem_from->count--; ++ mem_to->count++; ++ } ++ i++; ++ } ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++ ++int mali_mem_os_alloc_pages(mali_mem_os_mem *os_mem, u32 size) ++{ ++ struct page *new_page; ++ LIST_HEAD(pages_list); ++ size_t page_count = PAGE_ALIGN(size) / _MALI_OSK_MALI_PAGE_SIZE; ++ size_t remaining = page_count; ++ struct mali_page_node *m_page, *m_tmp; ++ u32 i; ++ ++ MALI_DEBUG_ASSERT_POINTER(os_mem); ++ ++ if (atomic_read(&mali_mem_os_allocator.allocated_pages) * _MALI_OSK_MALI_PAGE_SIZE + size > mali_mem_os_allocator.allocation_limit) { ++ MALI_DEBUG_PRINT(2, ("Mali Mem: Unable to allocate %u bytes. Currently allocated: %lu, max limit %lu\n", ++ size, ++ atomic_read(&mali_mem_os_allocator.allocated_pages) * _MALI_OSK_MALI_PAGE_SIZE, ++ mali_mem_os_allocator.allocation_limit)); ++ return -ENOMEM; ++ } ++ ++ INIT_LIST_HEAD(&os_mem->pages); ++ os_mem->count = page_count; ++ ++ /* Grab pages from pool. */ ++ { ++ size_t pool_pages; ++ spin_lock(&mali_mem_os_allocator.pool_lock); ++ pool_pages = min(remaining, mali_mem_os_allocator.pool_count); ++ for (i = pool_pages; i > 0; i--) { ++ BUG_ON(list_empty(&mali_mem_os_allocator.pool_pages)); ++ list_move(mali_mem_os_allocator.pool_pages.next, &pages_list); ++ } ++ mali_mem_os_allocator.pool_count -= pool_pages; ++ remaining -= pool_pages; ++ spin_unlock(&mali_mem_os_allocator.pool_lock); ++ } ++ ++ /* Process pages from pool. */ ++ i = 0; ++ list_for_each_entry_safe(m_page, m_tmp, &pages_list, list) { ++ BUG_ON(NULL == m_page); ++ ++ list_move_tail(&m_page->list, &os_mem->pages); ++ } ++ ++ /* Allocate new pages, if needed. */ ++ for (i = 0; i < remaining; i++) { ++ dma_addr_t dma_addr; ++ gfp_t flags = __GFP_ZERO | GFP_HIGHUSER; ++ int err; ++ ++#if defined(CONFIG_ARM) && !defined(CONFIG_ARM_LPAE) ++ flags |= GFP_HIGHUSER; ++#else ++#ifdef CONFIG_ZONE_DMA32 ++ flags |= GFP_DMA32; ++#else ++#ifdef CONFIG_ZONE_DMA ++#else ++ /* arm64 utgard only work on < 4G, but the kernel ++ * didn't provide method to allocte memory < 4G ++ */ ++ MALI_DEBUG_ASSERT(0); ++#endif ++#endif ++#endif ++ ++ new_page = alloc_page(flags); ++ ++ if (unlikely(NULL == new_page)) { ++ E("err."); ++ /* Calculate the number of pages actually allocated, and free them. */ ++ os_mem->count = (page_count - remaining) + i; ++ atomic_add(os_mem->count, &mali_mem_os_allocator.allocated_pages); ++ mali_mem_os_free(&os_mem->pages, os_mem->count, MALI_FALSE); ++ return -ENOMEM; ++ } ++ ++ /* Ensure page is flushed from CPU caches. */ ++ dma_addr = dma_map_page(&mali_platform_device->dev, new_page, ++ 0, _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL); ++ dma_unmap_page(&mali_platform_device->dev, dma_addr, ++ _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL); ++ dma_addr = dma_map_page(&mali_platform_device->dev, new_page, ++ 0, _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL); ++ ++ err = dma_mapping_error(&mali_platform_device->dev, dma_addr); ++ if (unlikely(err)) { ++ MALI_DEBUG_PRINT_ERROR(("OS Mem: Failed to DMA map page %p: %u", ++ new_page, err)); ++ __free_page(new_page); ++ os_mem->count = (page_count - remaining) + i; ++ atomic_add(os_mem->count, &mali_mem_os_allocator.allocated_pages); ++ mali_mem_os_free(&os_mem->pages, os_mem->count, MALI_FALSE); ++ return -EFAULT; ++ } ++ ++ /* Store page phys addr */ ++ SetPagePrivate(new_page); ++ set_page_private(new_page, dma_addr); ++ ++ m_page = _mali_page_node_allocate(MALI_PAGE_NODE_OS); ++ if (unlikely(NULL == m_page)) { ++ MALI_PRINT_ERROR(("OS Mem: Can't allocate mali_page node! \n")); ++ dma_unmap_page(&mali_platform_device->dev, page_private(new_page), ++ _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL); ++ ClearPagePrivate(new_page); ++ __free_page(new_page); ++ os_mem->count = (page_count - remaining) + i; ++ atomic_add(os_mem->count, &mali_mem_os_allocator.allocated_pages); ++ mali_mem_os_free(&os_mem->pages, os_mem->count, MALI_FALSE); ++ return -EFAULT; ++ } ++ m_page->page = new_page; ++ ++ list_add_tail(&m_page->list, &os_mem->pages); ++ } ++ ++ atomic_add(page_count, &mali_mem_os_allocator.allocated_pages); ++ ++ if (MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES > mali_mem_os_allocator.pool_count) { ++ MALI_DEBUG_PRINT(4, ("OS Mem: Stopping pool trim timer, only %u pages on pool\n", mali_mem_os_allocator.pool_count)); ++ cancel_delayed_work(&mali_mem_os_allocator.timed_shrinker); ++ } ++ ++ return 0; ++} ++ ++ ++_mali_osk_errcode_t mali_mem_os_mali_map(mali_mem_os_mem *os_mem, struct mali_session_data *session, u32 vaddr, u32 start_page, u32 mapping_pgae_num, u32 props) ++{ ++ struct mali_page_directory *pagedir = session->page_directory; ++ struct mali_page_node *m_page; ++ u32 virt; ++ u32 prop = props; ++ ++ MALI_DEBUG_ASSERT_POINTER(session); ++ MALI_DEBUG_ASSERT_POINTER(os_mem); ++ ++ MALI_DEBUG_ASSERT(start_page <= os_mem->count); ++ MALI_DEBUG_ASSERT((start_page + mapping_pgae_num) <= os_mem->count); ++ ++ if ((start_page + mapping_pgae_num) == os_mem->count) { ++ ++ virt = vaddr + MALI_MMU_PAGE_SIZE * (start_page + mapping_pgae_num); ++ ++ list_for_each_entry_reverse(m_page, &os_mem->pages, list) { ++ ++ virt -= MALI_MMU_PAGE_SIZE; ++ if (mapping_pgae_num > 0) { ++ dma_addr_t phys = page_private(m_page->page); ++#if defined(CONFIG_ARCH_DMA_ADDR_T_64BIT) ++ /* Verify that the "physical" address is 32-bit and ++ * usable for Mali, when on a system with bus addresses ++ * wider than 32-bit. */ ++ MALI_DEBUG_ASSERT(0 == (phys >> 32)); ++#endif ++ mali_mmu_pagedir_update(pagedir, virt, (mali_dma_addr)phys, MALI_MMU_PAGE_SIZE, prop); ++ } else { ++ break; ++ } ++ mapping_pgae_num--; ++ } ++ ++ } else { ++ u32 i = 0; ++ virt = vaddr; ++ list_for_each_entry(m_page, &os_mem->pages, list) { ++ ++ if (i >= start_page) { ++ dma_addr_t phys = page_private(m_page->page); ++ ++#if defined(CONFIG_ARCH_DMA_ADDR_T_64BIT) ++ /* Verify that the "physical" address is 32-bit and ++ * usable for Mali, when on a system with bus addresses ++ * wider than 32-bit. */ ++ MALI_DEBUG_ASSERT(0 == (phys >> 32)); ++#endif ++ mali_mmu_pagedir_update(pagedir, virt, (mali_dma_addr)phys, MALI_MMU_PAGE_SIZE, prop); ++ } ++ i++; ++ virt += MALI_MMU_PAGE_SIZE; ++ } ++ } ++ return _MALI_OSK_ERR_OK; ++} ++ ++ ++void mali_mem_os_mali_unmap(mali_mem_allocation *alloc) ++{ ++ struct mali_session_data *session; ++ MALI_DEBUG_ASSERT_POINTER(alloc); ++ session = alloc->session; ++ MALI_DEBUG_ASSERT_POINTER(session); ++ ++ mali_session_memory_lock(session); ++ mali_mem_mali_map_free(session, alloc->psize, alloc->mali_vma_node.vm_node.start, ++ alloc->flags); ++ mali_session_memory_unlock(session); ++} ++ ++int mali_mem_os_cpu_map(mali_mem_backend *mem_bkend, struct vm_area_struct *vma) ++{ ++ mali_mem_os_mem *os_mem = &mem_bkend->os_mem; ++ struct mali_page_node *m_page; ++ struct page *page; ++ int ret; ++ unsigned long addr = vma->vm_start; ++ MALI_DEBUG_ASSERT(MALI_MEM_OS == mem_bkend->type); ++ ++ list_for_each_entry(m_page, &os_mem->pages, list) { ++ /* We should use vm_insert_page, but it does a dcache ++ * flush which makes it way slower than remap_pfn_range or vmf_insert_pfn. ++ ret = vm_insert_page(vma, addr, page); ++ */ ++ page = m_page->page; ++ ret = vmf_insert_pfn(vma, addr, page_to_pfn(page)); ++ ++ if (unlikely(0 != ret)) { ++ return -EFAULT; ++ } ++ addr += _MALI_OSK_MALI_PAGE_SIZE; ++ } ++ ++ return 0; ++} ++ ++_mali_osk_errcode_t mali_mem_os_resize_cpu_map_locked(mali_mem_backend *mem_bkend, struct vm_area_struct *vma, unsigned long start_vaddr, u32 mappig_size) ++{ ++ mali_mem_os_mem *os_mem = &mem_bkend->os_mem; ++ struct mali_page_node *m_page; ++ int ret; ++ int offset; ++ int mapping_page_num; ++ int count ; ++ ++ unsigned long vstart = vma->vm_start; ++ count = 0; ++ MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_OS); ++ MALI_DEBUG_ASSERT(0 == start_vaddr % _MALI_OSK_MALI_PAGE_SIZE); ++ MALI_DEBUG_ASSERT(0 == vstart % _MALI_OSK_MALI_PAGE_SIZE); ++ offset = (start_vaddr - vstart) / _MALI_OSK_MALI_PAGE_SIZE; ++ MALI_DEBUG_ASSERT(offset <= os_mem->count); ++ mapping_page_num = mappig_size / _MALI_OSK_MALI_PAGE_SIZE; ++ MALI_DEBUG_ASSERT((offset + mapping_page_num) <= os_mem->count); ++ ++ if ((offset + mapping_page_num) == os_mem->count) { ++ ++ unsigned long vm_end = start_vaddr + mappig_size; ++ ++ list_for_each_entry_reverse(m_page, &os_mem->pages, list) { ++ ++ vm_end -= _MALI_OSK_MALI_PAGE_SIZE; ++ if (mapping_page_num > 0) { ++ ret = vmf_insert_pfn(vma, vm_end, page_to_pfn(m_page->page)); ++ ++ if (unlikely(0 != ret)) { ++ /*will return -EBUSY If the page has already been mapped into table, but it's OK*/ ++ if (-EBUSY == ret) { ++ break; ++ } else { ++ MALI_DEBUG_PRINT(1, ("OS Mem: mali_mem_os_resize_cpu_map_locked failed, ret = %d, offset is %d,page_count is %d\n", ++ ret, offset + mapping_page_num, os_mem->count)); ++ } ++ return _MALI_OSK_ERR_FAULT; ++ } ++ } else { ++ break; ++ } ++ mapping_page_num--; ++ ++ } ++ } else { ++ ++ list_for_each_entry(m_page, &os_mem->pages, list) { ++ if (count >= offset) { ++ ++ ret = vmf_insert_pfn(vma, vstart, page_to_pfn(m_page->page)); ++ ++ if (unlikely(0 != ret)) { ++ /*will return -EBUSY If the page has already been mapped into table, but it's OK*/ ++ if (-EBUSY == ret) { ++ break; ++ } else { ++ MALI_DEBUG_PRINT(1, ("OS Mem: mali_mem_os_resize_cpu_map_locked failed, ret = %d, count is %d, offset is %d,page_count is %d\n", ++ ret, count, offset, os_mem->count)); ++ } ++ return _MALI_OSK_ERR_FAULT; ++ } ++ } ++ count++; ++ vstart += _MALI_OSK_MALI_PAGE_SIZE; ++ } ++ } ++ return _MALI_OSK_ERR_OK; ++} ++ ++u32 mali_mem_os_release(mali_mem_backend *mem_bkend) ++{ ++ ++ mali_mem_allocation *alloc; ++ struct mali_session_data *session; ++ u32 free_pages_nr = 0; ++ MALI_DEBUG_ASSERT_POINTER(mem_bkend); ++ MALI_DEBUG_ASSERT(MALI_MEM_OS == mem_bkend->type); ++ ++ alloc = mem_bkend->mali_allocation; ++ MALI_DEBUG_ASSERT_POINTER(alloc); ++ ++ session = alloc->session; ++ MALI_DEBUG_ASSERT_POINTER(session); ++ ++ /* Unmap the memory from the mali virtual address space. */ ++ mali_mem_os_mali_unmap(alloc); ++ mutex_lock(&mem_bkend->mutex); ++ /* Free pages */ ++ if (MALI_MEM_BACKEND_FLAG_COWED & mem_bkend->flags) { ++ /* Lock to avoid the free race condition for the cow shared memory page node. */ ++ _mali_osk_mutex_wait(session->cow_lock); ++ free_pages_nr = mali_mem_os_free(&mem_bkend->os_mem.pages, mem_bkend->os_mem.count, MALI_TRUE); ++ _mali_osk_mutex_signal(session->cow_lock); ++ } else { ++ free_pages_nr = mali_mem_os_free(&mem_bkend->os_mem.pages, mem_bkend->os_mem.count, MALI_FALSE); ++ } ++ mutex_unlock(&mem_bkend->mutex); ++ ++ MALI_DEBUG_PRINT(4, ("OS Mem free : allocated size = 0x%x, free size = 0x%x\n", mem_bkend->os_mem.count * _MALI_OSK_MALI_PAGE_SIZE, ++ free_pages_nr * _MALI_OSK_MALI_PAGE_SIZE)); ++ ++ mem_bkend->os_mem.count = 0; ++ return free_pages_nr; ++} ++ ++ ++#define MALI_MEM_OS_PAGE_TABLE_PAGE_POOL_SIZE 128 ++static struct { ++ struct { ++ mali_dma_addr phys; ++ mali_io_address mapping; ++ } page[MALI_MEM_OS_PAGE_TABLE_PAGE_POOL_SIZE]; ++ size_t count; ++ spinlock_t lock; ++} mali_mem_page_table_page_pool = { ++ .count = 0, ++ .lock = __SPIN_LOCK_UNLOCKED(pool_lock), ++}; ++ ++_mali_osk_errcode_t mali_mem_os_get_table_page(mali_dma_addr *phys, mali_io_address *mapping) ++{ ++ _mali_osk_errcode_t ret = _MALI_OSK_ERR_NOMEM; ++ dma_addr_t tmp_phys; ++ ++ spin_lock(&mali_mem_page_table_page_pool.lock); ++ if (0 < mali_mem_page_table_page_pool.count) { ++ u32 i = --mali_mem_page_table_page_pool.count; ++ *phys = mali_mem_page_table_page_pool.page[i].phys; ++ *mapping = mali_mem_page_table_page_pool.page[i].mapping; ++ ++ ret = _MALI_OSK_ERR_OK; ++ } ++ spin_unlock(&mali_mem_page_table_page_pool.lock); ++ ++ if (_MALI_OSK_ERR_OK != ret) { ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0) ++ *mapping = dma_alloc_attrs(&mali_platform_device->dev, ++ _MALI_OSK_MALI_PAGE_SIZE, &tmp_phys, ++ GFP_KERNEL, DMA_ATTR_WRITE_COMBINE); ++#else ++ *mapping = dma_alloc_writecombine(&mali_platform_device->dev, ++ _MALI_OSK_MALI_PAGE_SIZE, &tmp_phys, GFP_KERNEL); ++#endif ++ if (NULL != *mapping) { ++ ret = _MALI_OSK_ERR_OK; ++ ++#if defined(CONFIG_ARCH_DMA_ADDR_T_64BIT) ++ /* Verify that the "physical" address is 32-bit and ++ * usable for Mali, when on a system with bus addresses ++ * wider than 32-bit. */ ++ MALI_DEBUG_ASSERT(0 == (tmp_phys >> 32)); ++#endif ++ ++ *phys = (mali_dma_addr)tmp_phys; ++ } ++ } ++ ++ return ret; ++} ++ ++void mali_mem_os_release_table_page(mali_dma_addr phys, void *virt) ++{ ++ spin_lock(&mali_mem_page_table_page_pool.lock); ++ if (MALI_MEM_OS_PAGE_TABLE_PAGE_POOL_SIZE > mali_mem_page_table_page_pool.count) { ++ u32 i = mali_mem_page_table_page_pool.count; ++ mali_mem_page_table_page_pool.page[i].phys = phys; ++ mali_mem_page_table_page_pool.page[i].mapping = virt; ++ ++ ++mali_mem_page_table_page_pool.count; ++ ++ spin_unlock(&mali_mem_page_table_page_pool.lock); ++ } else { ++ spin_unlock(&mali_mem_page_table_page_pool.lock); ++ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0) ++ dma_free_attrs(&mali_platform_device->dev, ++ _MALI_OSK_MALI_PAGE_SIZE, virt, phys, ++ DMA_ATTR_WRITE_COMBINE); ++#else ++ dma_free_writecombine(&mali_platform_device->dev, ++ _MALI_OSK_MALI_PAGE_SIZE, virt, phys); ++#endif ++ } ++} ++ ++void mali_mem_os_free_page_node(struct mali_page_node *m_page) ++{ ++ struct page *page = m_page->page; ++ MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_OS); ++ ++ if (1 == page_count(page)) { ++ dma_unmap_page(&mali_platform_device->dev, page_private(page), ++ _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL); ++ ClearPagePrivate(page); ++ } ++ __free_page(page); ++ m_page->page = NULL; ++ list_del(&m_page->list); ++ kfree(m_page); ++} ++ ++/* The maximum number of page table pool pages to free in one go. */ ++#define MALI_MEM_OS_CHUNK_TO_FREE 64UL ++ ++/* Free a certain number of pages from the page table page pool. ++ * The pool lock must be held when calling the function, and the lock will be ++ * released before returning. ++ */ ++static void mali_mem_os_page_table_pool_free(size_t nr_to_free) ++{ ++ mali_dma_addr phys_arr[MALI_MEM_OS_CHUNK_TO_FREE]; ++ void *virt_arr[MALI_MEM_OS_CHUNK_TO_FREE]; ++ u32 i; ++ ++ MALI_DEBUG_ASSERT(nr_to_free <= MALI_MEM_OS_CHUNK_TO_FREE); ++ ++ /* Remove nr_to_free pages from the pool and store them locally on stack. */ ++ for (i = 0; i < nr_to_free; i++) { ++ u32 pool_index = mali_mem_page_table_page_pool.count - i - 1; ++ ++ phys_arr[i] = mali_mem_page_table_page_pool.page[pool_index].phys; ++ virt_arr[i] = mali_mem_page_table_page_pool.page[pool_index].mapping; ++ } ++ ++ mali_mem_page_table_page_pool.count -= nr_to_free; ++ ++ spin_unlock(&mali_mem_page_table_page_pool.lock); ++ ++ /* After releasing the spinlock: free the pages we removed from the pool. */ ++ for (i = 0; i < nr_to_free; i++) { ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0) ++ dma_free_attrs(&mali_platform_device->dev, _MALI_OSK_MALI_PAGE_SIZE, ++ virt_arr[i], (dma_addr_t)phys_arr[i], ++ DMA_ATTR_WRITE_COMBINE); ++#else ++ dma_free_writecombine(&mali_platform_device->dev, ++ _MALI_OSK_MALI_PAGE_SIZE, ++ virt_arr[i], (dma_addr_t)phys_arr[i]); ++#endif ++ } ++} ++ ++static void mali_mem_os_trim_page_table_page_pool(void) ++{ ++ size_t nr_to_free = 0; ++ size_t nr_to_keep; ++ ++ /* Keep 2 page table pages for each 1024 pages in the page cache. */ ++ nr_to_keep = mali_mem_os_allocator.pool_count / 512; ++ /* And a minimum of eight pages, to accomodate new sessions. */ ++ nr_to_keep += 8; ++ ++ if (0 == spin_trylock(&mali_mem_page_table_page_pool.lock)) return; ++ ++ if (nr_to_keep < mali_mem_page_table_page_pool.count) { ++ nr_to_free = mali_mem_page_table_page_pool.count - nr_to_keep; ++ nr_to_free = min((size_t)MALI_MEM_OS_CHUNK_TO_FREE, nr_to_free); ++ } ++ ++ /* Pool lock will be released by the callee. */ ++ mali_mem_os_page_table_pool_free(nr_to_free); ++} ++ ++static unsigned long mali_mem_os_shrink_count(struct shrinker *shrinker, struct shrink_control *sc) ++{ ++ return mali_mem_os_allocator.pool_count; ++} ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 0, 0) ++#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35) ++static int mali_mem_os_shrink(int nr_to_scan, gfp_t gfp_mask) ++#else ++static int mali_mem_os_shrink(struct shrinker *shrinker, int nr_to_scan, gfp_t gfp_mask) ++#endif /* Linux < 2.6.35 */ ++#else ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) ++static int mali_mem_os_shrink(struct shrinker *shrinker, struct shrink_control *sc) ++#else ++static unsigned long mali_mem_os_shrink(struct shrinker *shrinker, struct shrink_control *sc) ++#endif /* Linux < 3.12.0 */ ++#endif /* Linux < 3.0.0 */ ++{ ++ struct mali_page_node *m_page, *m_tmp; ++ unsigned long flags; ++ struct list_head *le, pages; ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 0, 0) ++ int nr = nr_to_scan; ++#else ++ int nr = sc->nr_to_scan; ++#endif ++ ++ if (0 == nr) { ++ return mali_mem_os_shrink_count(shrinker, sc); ++ } ++ ++ if (0 == spin_trylock_irqsave(&mali_mem_os_allocator.pool_lock, flags)) { ++ /* Not able to lock. */ ++ return -1; ++ } ++ ++ if (0 == mali_mem_os_allocator.pool_count) { ++ /* No pages availble */ ++ spin_unlock_irqrestore(&mali_mem_os_allocator.pool_lock, flags); ++ return 0; ++ } ++ ++ /* Release from general page pool */ ++ nr = min((size_t)nr, mali_mem_os_allocator.pool_count); ++ mali_mem_os_allocator.pool_count -= nr; ++ list_for_each(le, &mali_mem_os_allocator.pool_pages) { ++ --nr; ++ if (0 == nr) break; ++ } ++ list_cut_position(&pages, &mali_mem_os_allocator.pool_pages, le); ++ spin_unlock_irqrestore(&mali_mem_os_allocator.pool_lock, flags); ++ ++ list_for_each_entry_safe(m_page, m_tmp, &pages, list) { ++ mali_mem_os_free_page_node(m_page); ++ } ++ ++ if (MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES > mali_mem_os_allocator.pool_count) { ++ /* Pools are empty, stop timer */ ++ MALI_DEBUG_PRINT(5, ("Stopping timer, only %u pages on pool\n", mali_mem_os_allocator.pool_count)); ++ cancel_delayed_work(&mali_mem_os_allocator.timed_shrinker); ++ } ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) ++ return mali_mem_os_shrink_count(shrinker, sc); ++#else ++ return nr; ++#endif ++} ++ ++static void mali_mem_os_trim_pool(struct work_struct *data) ++{ ++ struct mali_page_node *m_page, *m_tmp; ++ struct list_head *le; ++ LIST_HEAD(pages); ++ size_t nr_to_free; ++ ++ MALI_IGNORE(data); ++ ++ MALI_DEBUG_PRINT(3, ("OS Mem: Trimming pool %u\n", mali_mem_os_allocator.pool_count)); ++ ++ /* Release from general page pool */ ++ spin_lock(&mali_mem_os_allocator.pool_lock); ++ if (MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES < mali_mem_os_allocator.pool_count) { ++ size_t count = mali_mem_os_allocator.pool_count - MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES; ++ const size_t min_to_free = min(64, MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES); ++ ++ /* Free half the pages on the pool above the static limit. Or 64 pages, 256KB. */ ++ nr_to_free = max(count / 2, min_to_free); ++ ++ mali_mem_os_allocator.pool_count -= nr_to_free; ++ list_for_each(le, &mali_mem_os_allocator.pool_pages) { ++ --nr_to_free; ++ if (0 == nr_to_free) break; ++ } ++ list_cut_position(&pages, &mali_mem_os_allocator.pool_pages, le); ++ } ++ spin_unlock(&mali_mem_os_allocator.pool_lock); ++ ++ list_for_each_entry_safe(m_page, m_tmp, &pages, list) { ++ mali_mem_os_free_page_node(m_page); ++ } ++ ++ /* Release some pages from page table page pool */ ++ mali_mem_os_trim_page_table_page_pool(); ++ ++ if (MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES < mali_mem_os_allocator.pool_count) { ++ MALI_DEBUG_PRINT(4, ("OS Mem: Starting pool trim timer %u\n", mali_mem_os_allocator.pool_count)); ++ queue_delayed_work(mali_mem_os_allocator.wq, &mali_mem_os_allocator.timed_shrinker, MALI_OS_MEMORY_POOL_TRIM_JIFFIES); ++ } ++} ++ ++_mali_osk_errcode_t mali_mem_os_init(void) ++{ ++ mali_mem_os_allocator.wq = alloc_workqueue("mali-mem", WQ_UNBOUND, 1); ++ if (NULL == mali_mem_os_allocator.wq) { ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ ++ register_shrinker(&mali_mem_os_allocator.shrinker); ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++void mali_mem_os_term(void) ++{ ++ struct mali_page_node *m_page, *m_tmp; ++ unregister_shrinker(&mali_mem_os_allocator.shrinker); ++ cancel_delayed_work_sync(&mali_mem_os_allocator.timed_shrinker); ++ ++ if (NULL != mali_mem_os_allocator.wq) { ++ destroy_workqueue(mali_mem_os_allocator.wq); ++ mali_mem_os_allocator.wq = NULL; ++ } ++ ++ spin_lock(&mali_mem_os_allocator.pool_lock); ++ list_for_each_entry_safe(m_page, m_tmp, &mali_mem_os_allocator.pool_pages, list) { ++ mali_mem_os_free_page_node(m_page); ++ ++ --mali_mem_os_allocator.pool_count; ++ } ++ BUG_ON(mali_mem_os_allocator.pool_count); ++ spin_unlock(&mali_mem_os_allocator.pool_lock); ++ ++ /* Release from page table page pool */ ++ do { ++ u32 nr_to_free; ++ ++ spin_lock(&mali_mem_page_table_page_pool.lock); ++ ++ nr_to_free = min((size_t)MALI_MEM_OS_CHUNK_TO_FREE, mali_mem_page_table_page_pool.count); ++ ++ /* Pool lock will be released by the callee. */ ++ mali_mem_os_page_table_pool_free(nr_to_free); ++ } while (0 != mali_mem_page_table_page_pool.count); ++} ++ ++_mali_osk_errcode_t mali_memory_core_resource_os_memory(u32 size) ++{ ++ mali_mem_os_allocator.allocation_limit = size; ++ ++ MALI_SUCCESS; ++} ++ ++u32 mali_mem_os_stat(void) ++{ ++ return atomic_read(&mali_mem_os_allocator.allocated_pages) * _MALI_OSK_MALI_PAGE_SIZE; ++} +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_os_alloc.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_os_alloc.h +new file mode 100755 +index 000000000000..8c9b35d0b230 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_os_alloc.h +@@ -0,0 +1,54 @@ ++/* ++ * Copyright (C) 2013-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_MEMORY_OS_ALLOC_H__ ++#define __MALI_MEMORY_OS_ALLOC_H__ ++ ++#include "mali_osk.h" ++#include "mali_memory_types.h" ++ ++ ++/** @brief Release Mali OS memory ++ * ++ * The session memory_lock must be held when calling this function. ++ * ++ * @param mem_bkend Pointer to the mali_mem_backend to release ++ */ ++u32 mali_mem_os_release(mali_mem_backend *mem_bkend); ++ ++_mali_osk_errcode_t mali_mem_os_get_table_page(mali_dma_addr *phys, mali_io_address *mapping); ++ ++void mali_mem_os_release_table_page(mali_dma_addr phys, void *virt); ++ ++_mali_osk_errcode_t mali_mem_os_init(void); ++ ++void mali_mem_os_term(void); ++ ++u32 mali_mem_os_stat(void); ++ ++void mali_mem_os_free_page_node(struct mali_page_node *m_page); ++ ++int mali_mem_os_alloc_pages(mali_mem_os_mem *os_mem, u32 size); ++ ++u32 mali_mem_os_free(struct list_head *os_pages, u32 pages_count, mali_bool cow_flag); ++ ++_mali_osk_errcode_t mali_mem_os_put_page(struct page *page); ++ ++_mali_osk_errcode_t mali_mem_os_resize_pages(mali_mem_os_mem *mem_from, mali_mem_os_mem *mem_to, u32 start_page, u32 page_count); ++ ++_mali_osk_errcode_t mali_mem_os_mali_map(mali_mem_os_mem *os_mem, struct mali_session_data *session, u32 vaddr, u32 start_page, u32 mapping_pgae_num, u32 props); ++ ++void mali_mem_os_mali_unmap(mali_mem_allocation *alloc); ++ ++int mali_mem_os_cpu_map(mali_mem_backend *mem_bkend, struct vm_area_struct *vma); ++ ++_mali_osk_errcode_t mali_mem_os_resize_cpu_map_locked(mali_mem_backend *mem_bkend, struct vm_area_struct *vma, unsigned long start_vaddr, u32 mappig_size); ++ ++#endif /* __MALI_MEMORY_OS_ALLOC_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_secure.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_secure.c +new file mode 100755 +index 000000000000..0b4f828680d0 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_secure.c +@@ -0,0 +1,170 @@ ++/* ++ * Copyright (C) 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include "mali_kernel_common.h" ++#include "mali_memory.h" ++#include "mali_memory_secure.h" ++#include "mali_osk.h" ++#include ++#include ++#include ++#include ++ ++_mali_osk_errcode_t mali_mem_secure_attach_dma_buf(mali_mem_secure *secure_mem, u32 size, int mem_fd) ++{ ++ struct dma_buf *buf; ++ MALI_DEBUG_ASSERT_POINTER(secure_mem); ++ ++ /* get dma buffer */ ++ buf = dma_buf_get(mem_fd); ++ if (IS_ERR_OR_NULL(buf)) { ++ MALI_DEBUG_PRINT_ERROR(("Failed to get dma buf!\n")); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ if (size != buf->size) { ++ MALI_DEBUG_PRINT_ERROR(("The secure mem size not match to the dma buf size!\n")); ++ goto failed_alloc_mem; ++ } ++ ++ secure_mem->buf = buf; ++ secure_mem->attachment = dma_buf_attach(secure_mem->buf, &mali_platform_device->dev); ++ if (NULL == secure_mem->attachment) { ++ MALI_DEBUG_PRINT_ERROR(("Failed to get dma buf attachment!\n")); ++ goto failed_dma_attach; ++ } ++ ++ secure_mem->sgt = dma_buf_map_attachment(secure_mem->attachment, DMA_BIDIRECTIONAL); ++ if (IS_ERR_OR_NULL(secure_mem->sgt)) { ++ MALI_DEBUG_PRINT_ERROR(("Failed to map dma buf attachment\n")); ++ goto failed_dma_map; ++ } ++ ++ secure_mem->count = size / MALI_MMU_PAGE_SIZE; ++ ++ return _MALI_OSK_ERR_OK; ++ ++failed_dma_map: ++ dma_buf_detach(secure_mem->buf, secure_mem->attachment); ++failed_dma_attach: ++failed_alloc_mem: ++ dma_buf_put(buf); ++ return _MALI_OSK_ERR_FAULT; ++} ++ ++_mali_osk_errcode_t mali_mem_secure_mali_map(mali_mem_secure *secure_mem, struct mali_session_data *session, u32 vaddr, u32 props) ++{ ++ struct mali_page_directory *pagedir; ++ struct scatterlist *sg; ++ u32 virt = vaddr; ++ u32 prop = props; ++ int i; ++ ++ MALI_DEBUG_ASSERT_POINTER(secure_mem); ++ MALI_DEBUG_ASSERT_POINTER(secure_mem->sgt); ++ MALI_DEBUG_ASSERT_POINTER(session); ++ ++ pagedir = session->page_directory; ++ ++ for_each_sg(secure_mem->sgt->sgl, sg, secure_mem->sgt->nents, i) { ++ u32 size = sg_dma_len(sg); ++ dma_addr_t phys = sg_dma_address(sg); ++ ++ /* sg must be page aligned. */ ++ MALI_DEBUG_ASSERT(0 == size % MALI_MMU_PAGE_SIZE); ++ MALI_DEBUG_ASSERT(0 == (phys & ~(uintptr_t)0xFFFFFFFF)); ++ ++ mali_mmu_pagedir_update(pagedir, virt, phys, size, prop); ++ ++ MALI_DEBUG_PRINT(3, ("The secure mem physical address: 0x%x gpu virtual address: 0x%x! \n", phys, virt)); ++ virt += size; ++ } ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++void mali_mem_secure_mali_unmap(mali_mem_allocation *alloc) ++{ ++ struct mali_session_data *session; ++ MALI_DEBUG_ASSERT_POINTER(alloc); ++ session = alloc->session; ++ MALI_DEBUG_ASSERT_POINTER(session); ++ ++ mali_session_memory_lock(session); ++ mali_mem_mali_map_free(session, alloc->psize, alloc->mali_vma_node.vm_node.start, ++ alloc->flags); ++ mali_session_memory_unlock(session); ++} ++ ++ ++int mali_mem_secure_cpu_map(mali_mem_backend *mem_bkend, struct vm_area_struct *vma) ++{ ++ ++ int ret = 0; ++ struct scatterlist *sg; ++ mali_mem_secure *secure_mem = &mem_bkend->secure_mem; ++ unsigned long addr = vma->vm_start; ++ int i; ++ ++ MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_SECURE); ++ ++ for_each_sg(secure_mem->sgt->sgl, sg, secure_mem->sgt->nents, i) { ++ phys_addr_t phys; ++ dma_addr_t dev_addr; ++ u32 size, j; ++ dev_addr = sg_dma_address(sg); ++#if defined(CONFIG_ARM64) ||LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0) ++ phys = dma_to_phys(&mali_platform_device->dev, dev_addr); ++#else ++ phys = page_to_phys(pfn_to_page(dma_to_pfn(&mali_platform_device->dev, dev_addr))); ++#endif ++ size = sg_dma_len(sg); ++ MALI_DEBUG_ASSERT(0 == size % _MALI_OSK_MALI_PAGE_SIZE); ++ ++ for (j = 0; j < size / _MALI_OSK_MALI_PAGE_SIZE; j++) { ++ ret = vmf_insert_pfn(vma, addr, PFN_DOWN(phys)); ++ ++ if (unlikely(0 != ret)) { ++ return -EFAULT; ++ } ++ addr += _MALI_OSK_MALI_PAGE_SIZE; ++ phys += _MALI_OSK_MALI_PAGE_SIZE; ++ ++ MALI_DEBUG_PRINT(3, ("The secure mem physical address: 0x%x , cpu virtual address: 0x%x! \n", phys, addr)); ++ } ++ } ++ return ret; ++} ++ ++u32 mali_mem_secure_release(mali_mem_backend *mem_bkend) ++{ ++ struct mali_mem_secure *mem; ++ mali_mem_allocation *alloc = mem_bkend->mali_allocation; ++ u32 free_pages_nr = 0; ++ MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_SECURE); ++ ++ mem = &mem_bkend->secure_mem; ++ MALI_DEBUG_ASSERT_POINTER(mem->attachment); ++ MALI_DEBUG_ASSERT_POINTER(mem->buf); ++ MALI_DEBUG_ASSERT_POINTER(mem->sgt); ++ /* Unmap the memory from the mali virtual address space. */ ++ mali_mem_secure_mali_unmap(alloc); ++ mutex_lock(&mem_bkend->mutex); ++ dma_buf_unmap_attachment(mem->attachment, mem->sgt, DMA_BIDIRECTIONAL); ++ dma_buf_detach(mem->buf, mem->attachment); ++ dma_buf_put(mem->buf); ++ mutex_unlock(&mem_bkend->mutex); ++ ++ free_pages_nr = mem->count; ++ ++ return free_pages_nr; ++} ++ ++ +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_secure.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_secure.h +new file mode 100755 +index 000000000000..48691d4790fe +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_secure.h +@@ -0,0 +1,30 @@ ++/* ++ * Copyright (C) 2010, 2013, 2015-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_MEMORY_SECURE_H__ ++#define __MALI_MEMORY_SECURE_H__ ++ ++#include "mali_session.h" ++#include "mali_memory.h" ++#include ++ ++#include "mali_memory_types.h" ++ ++_mali_osk_errcode_t mali_mem_secure_attach_dma_buf(mali_mem_secure *secure_mem, u32 size, int mem_fd); ++ ++_mali_osk_errcode_t mali_mem_secure_mali_map(mali_mem_secure *secure_mem, struct mali_session_data *session, u32 vaddr, u32 props); ++ ++void mali_mem_secure_mali_unmap(mali_mem_allocation *alloc); ++ ++int mali_mem_secure_cpu_map(mali_mem_backend *mem_bkend, struct vm_area_struct *vma); ++ ++u32 mali_mem_secure_release(mali_mem_backend *mem_bkend); ++ ++#endif /* __MALI_MEMORY_SECURE_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_swap_alloc.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_swap_alloc.c +new file mode 100755 +index 000000000000..d682785b9673 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_swap_alloc.c +@@ -0,0 +1,943 @@ ++/* ++ * Copyright (C) 2013-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include "mali_osk.h" ++#include "mali_osk_mali.h" ++#include "mali_memory.h" ++#include "mali_memory_manager.h" ++#include "mali_memory_virtual.h" ++#include "mali_memory_cow.h" ++#include "mali_ukk.h" ++#include "mali_kernel_utilization.h" ++#include "mali_memory_swap_alloc.h" ++ ++ ++static struct _mali_osk_bitmap idx_mgr; ++static struct file *global_swap_file; ++static struct address_space *global_swap_space; ++static _mali_osk_wq_work_t *mali_mem_swap_out_workq = NULL; ++static u32 mem_backend_swapped_pool_size; ++#ifdef MALI_MEM_SWAP_TRACKING ++static u32 mem_backend_swapped_unlock_size; ++#endif ++/* Lock order: mem_backend_swapped_pool_lock > each memory backend's mutex lock. ++ * This lock used to protect mem_backend_swapped_pool_size and mem_backend_swapped_pool. */ ++static struct mutex mem_backend_swapped_pool_lock; ++static struct list_head mem_backend_swapped_pool; ++ ++extern struct mali_mem_os_allocator mali_mem_os_allocator; ++ ++#define MALI_SWAP_LOW_MEM_DEFAULT_VALUE (60*1024*1024) ++#define MALI_SWAP_INVALIDATE_MALI_ADDRESS (0) /* Used to mark the given memory cookie is invalidate. */ ++#define MALI_SWAP_GLOBAL_SWAP_FILE_SIZE (0xFFFFFFFF) ++#define MALI_SWAP_GLOBAL_SWAP_FILE_INDEX \ ++ ((MALI_SWAP_GLOBAL_SWAP_FILE_SIZE) >> PAGE_SHIFT) ++#define MALI_SWAP_GLOBAL_SWAP_FILE_INDEX_RESERVE (1 << 15) /* Reserved for CoW nonlinear swap backend memory, the space size is 128MB. */ ++ ++unsigned int mali_mem_swap_out_threshold_value = MALI_SWAP_LOW_MEM_DEFAULT_VALUE; ++ ++/** ++ * We have two situations to do shrinking things, one is we met low GPU utilization which shows GPU needn't touch too ++ * swappable backends in short time, and the other one is we add new swappable backends, the total pool size exceed ++ * the threshold value of the swapped pool size. ++ */ ++typedef enum { ++ MALI_MEM_SWAP_SHRINK_WITH_LOW_UTILIZATION = 100, ++ MALI_MEM_SWAP_SHRINK_FOR_ADDING_NEW_BACKENDS = 257, ++} _mali_mem_swap_pool_shrink_type_t; ++ ++static void mali_mem_swap_swapped_bkend_pool_check_for_low_utilization(void *arg); ++ ++_mali_osk_errcode_t mali_mem_swap_init(void) ++{ ++ gfp_t flags = __GFP_NORETRY | __GFP_NOWARN; ++ ++ if (_MALI_OSK_ERR_OK != _mali_osk_bitmap_init(&idx_mgr, MALI_SWAP_GLOBAL_SWAP_FILE_INDEX, MALI_SWAP_GLOBAL_SWAP_FILE_INDEX_RESERVE)) { ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ ++ global_swap_file = shmem_file_setup("mali_swap", MALI_SWAP_GLOBAL_SWAP_FILE_SIZE, VM_NORESERVE); ++ if (IS_ERR(global_swap_file)) { ++ _mali_osk_bitmap_term(&idx_mgr); ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ ++ global_swap_space = global_swap_file->f_path.dentry->d_inode->i_mapping; ++ ++ mali_mem_swap_out_workq = _mali_osk_wq_create_work(mali_mem_swap_swapped_bkend_pool_check_for_low_utilization, NULL); ++ if (NULL == mali_mem_swap_out_workq) { ++ _mali_osk_bitmap_term(&idx_mgr); ++ fput(global_swap_file); ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ ++#if defined(CONFIG_ARM) && !defined(CONFIG_ARM_LPAE) ++ flags |= GFP_HIGHUSER; ++#else ++#ifdef CONFIG_ZONE_DMA32 ++ flags |= GFP_DMA32; ++#else ++#ifdef CONFIG_ZONE_DMA ++ flags |= GFP_DMA; ++#else ++ /* arm64 utgard only work on < 4G, but the kernel ++ * didn't provide method to allocte memory < 4G ++ */ ++ MALI_DEBUG_ASSERT(0); ++#endif ++#endif ++#endif ++ ++ /* When we use shmem_read_mapping_page to allocate/swap-in, it will ++ * use these flags to allocate new page if need.*/ ++ mapping_set_gfp_mask(global_swap_space, flags); ++ ++ mem_backend_swapped_pool_size = 0; ++#ifdef MALI_MEM_SWAP_TRACKING ++ mem_backend_swapped_unlock_size = 0; ++#endif ++ mutex_init(&mem_backend_swapped_pool_lock); ++ INIT_LIST_HEAD(&mem_backend_swapped_pool); ++ ++ MALI_DEBUG_PRINT(2, ("Mali SWAP: Swap out threshold vaule is %uM\n", mali_mem_swap_out_threshold_value >> 20)); ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++void mali_mem_swap_term(void) ++{ ++ _mali_osk_bitmap_term(&idx_mgr); ++ ++ fput(global_swap_file); ++ ++ _mali_osk_wq_delete_work(mali_mem_swap_out_workq); ++ ++ MALI_DEBUG_ASSERT(list_empty(&mem_backend_swapped_pool)); ++ MALI_DEBUG_ASSERT(0 == mem_backend_swapped_pool_size); ++ ++ return; ++} ++ ++struct file *mali_mem_swap_get_global_swap_file(void) ++{ ++ return global_swap_file; ++} ++ ++/* Judge if swappable backend in swapped pool. */ ++static mali_bool mali_memory_swap_backend_in_swapped_pool(mali_mem_backend *mem_bkend) ++{ ++ MALI_DEBUG_ASSERT_POINTER(mem_bkend); ++ ++ return !list_empty(&mem_bkend->list); ++} ++ ++void mali_memory_swap_list_backend_delete(mali_mem_backend *mem_bkend) ++{ ++ MALI_DEBUG_ASSERT_POINTER(mem_bkend); ++ ++ mutex_lock(&mem_backend_swapped_pool_lock); ++ mutex_lock(&mem_bkend->mutex); ++ ++ if (MALI_FALSE == mali_memory_swap_backend_in_swapped_pool(mem_bkend)) { ++ mutex_unlock(&mem_bkend->mutex); ++ mutex_unlock(&mem_backend_swapped_pool_lock); ++ return; ++ } ++ ++ MALI_DEBUG_ASSERT(!list_empty(&mem_bkend->list)); ++ ++ list_del_init(&mem_bkend->list); ++ ++ mutex_unlock(&mem_bkend->mutex); ++ ++ mem_backend_swapped_pool_size -= mem_bkend->size; ++ ++ mutex_unlock(&mem_backend_swapped_pool_lock); ++} ++ ++static void mali_mem_swap_out_page_node(mali_page_node *page_node) ++{ ++ MALI_DEBUG_ASSERT(page_node); ++ ++ dma_unmap_page(&mali_platform_device->dev, page_node->swap_it->dma_addr, ++ _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE); ++ set_page_dirty(page_node->swap_it->page); ++ put_page(page_node->swap_it->page); ++} ++ ++void mali_mem_swap_unlock_single_mem_backend(mali_mem_backend *mem_bkend) ++{ ++ mali_page_node *m_page; ++ ++ MALI_DEBUG_ASSERT(1 == mutex_is_locked(&mem_bkend->mutex)); ++ ++ if (MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN == (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN)) { ++ return; ++ } ++ ++ mem_bkend->flags |= MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN; ++ ++ list_for_each_entry(m_page, &mem_bkend->swap_mem.pages, list) { ++ mali_mem_swap_out_page_node(m_page); ++ } ++ ++ return; ++} ++ ++static void mali_mem_swap_unlock_partial_locked_mem_backend(mali_mem_backend *mem_bkend, mali_page_node *page_node) ++{ ++ mali_page_node *m_page; ++ ++ MALI_DEBUG_ASSERT(1 == mutex_is_locked(&mem_bkend->mutex)); ++ ++ list_for_each_entry(m_page, &mem_bkend->swap_mem.pages, list) { ++ if (m_page == page_node) { ++ break; ++ } ++ mali_mem_swap_out_page_node(m_page); ++ } ++} ++ ++static void mali_mem_swap_swapped_bkend_pool_shrink(_mali_mem_swap_pool_shrink_type_t shrink_type) ++{ ++ mali_mem_backend *bkend, *tmp_bkend; ++ long system_free_size; ++ u32 last_gpu_utilization, gpu_utilization_threshold_value, temp_swap_out_threshold_value; ++ ++ MALI_DEBUG_ASSERT(1 == mutex_is_locked(&mem_backend_swapped_pool_lock)); ++ ++ if (MALI_MEM_SWAP_SHRINK_WITH_LOW_UTILIZATION == shrink_type) { ++ /** ++ * When we met that system memory is very low and Mali locked swappable memory size is less than ++ * threshold value, and at the same time, GPU load is very low and don't need high performance, ++ * at this condition, we can unlock more swap memory backend from swapped backends pool. ++ */ ++ gpu_utilization_threshold_value = MALI_MEM_SWAP_SHRINK_WITH_LOW_UTILIZATION; ++ temp_swap_out_threshold_value = (mali_mem_swap_out_threshold_value >> 2); ++ } else { ++ /* When we add swappable memory backends to swapped pool, we need to think that we couldn't ++ * hold too much swappable backends in Mali driver, and also we need considering performance. ++ * So there is a balance for swapping out memory backend, we should follow the following conditions: ++ * 1. Total memory size in global mem backend swapped pool is more than the defined threshold value. ++ * 2. System level free memory size is less than the defined threshold value. ++ * 3. Please note that GPU utilization problem isn't considered in this condition. ++ */ ++ gpu_utilization_threshold_value = MALI_MEM_SWAP_SHRINK_FOR_ADDING_NEW_BACKENDS; ++ temp_swap_out_threshold_value = mali_mem_swap_out_threshold_value; ++ } ++ ++ /* Get system free pages number. */ ++ system_free_size = global_zone_page_state(NR_FREE_PAGES) * PAGE_SIZE; ++ last_gpu_utilization = _mali_ukk_utilization_gp_pp(); ++ ++ if ((last_gpu_utilization < gpu_utilization_threshold_value) ++ && (system_free_size < mali_mem_swap_out_threshold_value) ++ && (mem_backend_swapped_pool_size > temp_swap_out_threshold_value)) { ++ list_for_each_entry_safe(bkend, tmp_bkend, &mem_backend_swapped_pool, list) { ++ if (mem_backend_swapped_pool_size <= temp_swap_out_threshold_value) { ++ break; ++ } ++ ++ mutex_lock(&bkend->mutex); ++ ++ /* check if backend is in use. */ ++ if (0 < bkend->using_count) { ++ mutex_unlock(&bkend->mutex); ++ continue; ++ } ++ ++ mali_mem_swap_unlock_single_mem_backend(bkend); ++ list_del_init(&bkend->list); ++ mem_backend_swapped_pool_size -= bkend->size; ++#ifdef MALI_MEM_SWAP_TRACKING ++ mem_backend_swapped_unlock_size += bkend->size; ++#endif ++ mutex_unlock(&bkend->mutex); ++ } ++ } ++ ++ return; ++} ++ ++static void mali_mem_swap_swapped_bkend_pool_check_for_low_utilization(void *arg) ++{ ++ MALI_IGNORE(arg); ++ ++ mutex_lock(&mem_backend_swapped_pool_lock); ++ ++ mali_mem_swap_swapped_bkend_pool_shrink(MALI_MEM_SWAP_SHRINK_WITH_LOW_UTILIZATION); ++ ++ mutex_unlock(&mem_backend_swapped_pool_lock); ++} ++ ++/** ++ * After PP job finished, we add all of swappable memory backend used by this PP ++ * job to the tail of the global swapped pool, and if the total size of swappable memory is more than threshold ++ * value, we also need to shrink the swapped pool start from the head of the list. ++ */ ++void mali_memory_swap_list_backend_add(mali_mem_backend *mem_bkend) ++{ ++ mutex_lock(&mem_backend_swapped_pool_lock); ++ mutex_lock(&mem_bkend->mutex); ++ ++ if (mali_memory_swap_backend_in_swapped_pool(mem_bkend)) { ++ MALI_DEBUG_ASSERT(!list_empty(&mem_bkend->list)); ++ ++ list_del_init(&mem_bkend->list); ++ list_add_tail(&mem_bkend->list, &mem_backend_swapped_pool); ++ mutex_unlock(&mem_bkend->mutex); ++ mutex_unlock(&mem_backend_swapped_pool_lock); ++ return; ++ } ++ ++ list_add_tail(&mem_bkend->list, &mem_backend_swapped_pool); ++ ++ mutex_unlock(&mem_bkend->mutex); ++ mem_backend_swapped_pool_size += mem_bkend->size; ++ ++ mali_mem_swap_swapped_bkend_pool_shrink(MALI_MEM_SWAP_SHRINK_FOR_ADDING_NEW_BACKENDS); ++ ++ mutex_unlock(&mem_backend_swapped_pool_lock); ++ return; ++} ++ ++ ++u32 mali_mem_swap_idx_alloc(void) ++{ ++ return _mali_osk_bitmap_alloc(&idx_mgr); ++} ++ ++void mali_mem_swap_idx_free(u32 idx) ++{ ++ _mali_osk_bitmap_free(&idx_mgr, idx); ++} ++ ++static u32 mali_mem_swap_idx_range_alloc(u32 count) ++{ ++ u32 index; ++ ++ index = _mali_osk_bitmap_alloc_range(&idx_mgr, count); ++ ++ return index; ++} ++ ++static void mali_mem_swap_idx_range_free(u32 idx, int num) ++{ ++ _mali_osk_bitmap_free_range(&idx_mgr, idx, num); ++} ++ ++struct mali_swap_item *mali_mem_swap_alloc_swap_item(void) ++{ ++ mali_swap_item *swap_item; ++ ++ swap_item = kzalloc(sizeof(mali_swap_item), GFP_KERNEL); ++ ++ if (NULL == swap_item) { ++ return NULL; ++ } ++ ++ atomic_set(&swap_item->ref_count, 1); ++ swap_item->page = NULL; ++ atomic_add(1, &mali_mem_os_allocator.allocated_pages); ++ ++ return swap_item; ++} ++ ++void mali_mem_swap_free_swap_item(mali_swap_item *swap_item) ++{ ++ struct inode *file_node; ++ long long start, end; ++ ++ /* If this swap item is shared, we just reduce the reference counter. */ ++ if (0 == atomic_dec_return(&swap_item->ref_count)) { ++ file_node = global_swap_file->f_path.dentry->d_inode; ++ start = swap_item->idx; ++ start = start << 12; ++ end = start + PAGE_SIZE; ++ ++ shmem_truncate_range(file_node, start, (end - 1)); ++ ++ mali_mem_swap_idx_free(swap_item->idx); ++ ++ atomic_sub(1, &mali_mem_os_allocator.allocated_pages); ++ ++ kfree(swap_item); ++ } ++} ++ ++/* Used to allocate new swap item for new memory allocation and cow page for write. */ ++struct mali_page_node *_mali_mem_swap_page_node_allocate(void) ++{ ++ struct mali_page_node *m_page; ++ ++ m_page = _mali_page_node_allocate(MALI_PAGE_NODE_SWAP); ++ ++ if (NULL == m_page) { ++ return NULL; ++ } ++ ++ m_page->swap_it = mali_mem_swap_alloc_swap_item(); ++ ++ if (NULL == m_page->swap_it) { ++ kfree(m_page); ++ return NULL; ++ } ++ ++ return m_page; ++} ++ ++_mali_osk_errcode_t _mali_mem_swap_put_page_node(struct mali_page_node *m_page) ++{ ++ ++ mali_mem_swap_free_swap_item(m_page->swap_it); ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++void _mali_mem_swap_page_node_free(struct mali_page_node *m_page) ++{ ++ _mali_mem_swap_put_page_node(m_page); ++ ++ kfree(m_page); ++ ++ return; ++} ++ ++u32 mali_mem_swap_free(mali_mem_swap *swap_mem) ++{ ++ struct mali_page_node *m_page, *m_tmp; ++ u32 free_pages_nr = 0; ++ ++ MALI_DEBUG_ASSERT_POINTER(swap_mem); ++ ++ list_for_each_entry_safe(m_page, m_tmp, &swap_mem->pages, list) { ++ MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_SWAP); ++ ++ /* free the page node and release the swap item, if the ref count is 1, ++ * then need also free the swap item. */ ++ list_del(&m_page->list); ++ if (1 == _mali_page_node_get_ref_count(m_page)) { ++ free_pages_nr++; ++ } ++ ++ _mali_mem_swap_page_node_free(m_page); ++ } ++ ++ return free_pages_nr; ++} ++ ++static u32 mali_mem_swap_cow_free(mali_mem_cow *cow_mem) ++{ ++ struct mali_page_node *m_page, *m_tmp; ++ u32 free_pages_nr = 0; ++ ++ MALI_DEBUG_ASSERT_POINTER(cow_mem); ++ ++ list_for_each_entry_safe(m_page, m_tmp, &cow_mem->pages, list) { ++ MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_SWAP); ++ ++ /* free the page node and release the swap item, if the ref count is 1, ++ * then need also free the swap item. */ ++ list_del(&m_page->list); ++ if (1 == _mali_page_node_get_ref_count(m_page)) { ++ free_pages_nr++; ++ } ++ ++ _mali_mem_swap_page_node_free(m_page); ++ } ++ ++ return free_pages_nr; ++} ++ ++u32 mali_mem_swap_release(mali_mem_backend *mem_bkend, mali_bool is_mali_mapped) ++{ ++ mali_mem_allocation *alloc; ++ u32 free_pages_nr = 0; ++ ++ MALI_DEBUG_ASSERT_POINTER(mem_bkend); ++ alloc = mem_bkend->mali_allocation; ++ MALI_DEBUG_ASSERT_POINTER(alloc); ++ ++ if (is_mali_mapped) { ++ mali_mem_swap_mali_unmap(alloc); ++ } ++ ++ mali_memory_swap_list_backend_delete(mem_bkend); ++ ++ mutex_lock(&mem_bkend->mutex); ++ /* To make sure the given memory backend was unlocked from Mali side, ++ * and then free this memory block. */ ++ mali_mem_swap_unlock_single_mem_backend(mem_bkend); ++ mutex_unlock(&mem_bkend->mutex); ++ ++ if (MALI_MEM_SWAP == mem_bkend->type) { ++ free_pages_nr = mali_mem_swap_free(&mem_bkend->swap_mem); ++ } else { ++ free_pages_nr = mali_mem_swap_cow_free(&mem_bkend->cow_mem); ++ } ++ ++ return free_pages_nr; ++} ++ ++mali_bool mali_mem_swap_in_page_node(struct mali_page_node *page_node) ++{ ++ MALI_DEBUG_ASSERT(NULL != page_node); ++ ++ page_node->swap_it->page = shmem_read_mapping_page(global_swap_space, page_node->swap_it->idx); ++ ++ if (IS_ERR(page_node->swap_it->page)) { ++ MALI_DEBUG_PRINT_ERROR(("SWAP Mem: failed to swap in page with index: %d.\n", page_node->swap_it->idx)); ++ return MALI_FALSE; ++ } ++ ++ /* Ensure page is flushed from CPU caches. */ ++ page_node->swap_it->dma_addr = dma_map_page(&mali_platform_device->dev, page_node->swap_it->page, ++ 0, _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE); ++ ++ return MALI_TRUE; ++} ++ ++int mali_mem_swap_alloc_pages(mali_mem_swap *swap_mem, u32 size, u32 *bkend_idx) ++{ ++ size_t page_count = PAGE_ALIGN(size) / PAGE_SIZE; ++ struct mali_page_node *m_page; ++ long system_free_size; ++ u32 i, index; ++ mali_bool ret; ++ ++ MALI_DEBUG_ASSERT(NULL != swap_mem); ++ MALI_DEBUG_ASSERT(NULL != bkend_idx); ++ MALI_DEBUG_ASSERT(page_count <= MALI_SWAP_GLOBAL_SWAP_FILE_INDEX_RESERVE); ++ ++ if (atomic_read(&mali_mem_os_allocator.allocated_pages) * _MALI_OSK_MALI_PAGE_SIZE + size > mali_mem_os_allocator.allocation_limit) { ++ MALI_DEBUG_PRINT(2, ("Mali Mem: Unable to allocate %u bytes. Currently allocated: %lu, max limit %lu\n", ++ size, ++ atomic_read(&mali_mem_os_allocator.allocated_pages) * _MALI_OSK_MALI_PAGE_SIZE, ++ mali_mem_os_allocator.allocation_limit)); ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ ++ INIT_LIST_HEAD(&swap_mem->pages); ++ swap_mem->count = page_count; ++ index = mali_mem_swap_idx_range_alloc(page_count); ++ ++ if (_MALI_OSK_BITMAP_INVALIDATE_INDEX == index) { ++ MALI_PRINT_ERROR(("Mali Swap: Failed to allocate continuous index for swappable Mali memory.")); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ for (i = 0; i < page_count; i++) { ++ m_page = _mali_mem_swap_page_node_allocate(); ++ ++ if (NULL == m_page) { ++ MALI_DEBUG_PRINT_ERROR(("SWAP Mem: Failed to allocate mali page node.")); ++ swap_mem->count = i; ++ ++ mali_mem_swap_free(swap_mem); ++ mali_mem_swap_idx_range_free(index + i, page_count - i); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ m_page->swap_it->idx = index + i; ++ ++ ret = mali_mem_swap_in_page_node(m_page); ++ ++ if (MALI_FALSE == ret) { ++ MALI_DEBUG_PRINT_ERROR(("SWAP Mem: Allocate new page from SHMEM file failed.")); ++ _mali_mem_swap_page_node_free(m_page); ++ mali_mem_swap_idx_range_free(index + i + 1, page_count - i - 1); ++ ++ swap_mem->count = i; ++ mali_mem_swap_free(swap_mem); ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ ++ list_add_tail(&m_page->list, &swap_mem->pages); ++ } ++ ++ system_free_size = global_zone_page_state(NR_FREE_PAGES) * PAGE_SIZE; ++ ++ if ((system_free_size < mali_mem_swap_out_threshold_value) ++ && (mem_backend_swapped_pool_size > (mali_mem_swap_out_threshold_value >> 2)) ++ && mali_utilization_enabled()) { ++ _mali_osk_wq_schedule_work(mali_mem_swap_out_workq); ++ } ++ ++ *bkend_idx = index; ++ return 0; ++} ++ ++void mali_mem_swap_mali_unmap(mali_mem_allocation *alloc) ++{ ++ struct mali_session_data *session; ++ ++ MALI_DEBUG_ASSERT_POINTER(alloc); ++ session = alloc->session; ++ MALI_DEBUG_ASSERT_POINTER(session); ++ ++ mali_session_memory_lock(session); ++ mali_mem_mali_map_free(session, alloc->psize, alloc->mali_vma_node.vm_node.start, ++ alloc->flags); ++ mali_session_memory_unlock(session); ++} ++ ++ ++/* Insert these pages from shmem to mali page table*/ ++_mali_osk_errcode_t mali_mem_swap_mali_map(mali_mem_swap *swap_mem, struct mali_session_data *session, u32 vaddr, u32 props) ++{ ++ struct mali_page_directory *pagedir = session->page_directory; ++ struct mali_page_node *m_page; ++ dma_addr_t phys; ++ u32 virt = vaddr; ++ u32 prop = props; ++ ++ list_for_each_entry(m_page, &swap_mem->pages, list) { ++ MALI_DEBUG_ASSERT(NULL != m_page->swap_it->page); ++ phys = m_page->swap_it->dma_addr; ++ ++ mali_mmu_pagedir_update(pagedir, virt, phys, MALI_MMU_PAGE_SIZE, prop); ++ virt += MALI_MMU_PAGE_SIZE; ++ } ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++int mali_mem_swap_in_pages(struct mali_pp_job *job) ++{ ++ u32 num_memory_cookies; ++ struct mali_session_data *session; ++ struct mali_vma_node *mali_vma_node = NULL; ++ mali_mem_allocation *mali_alloc = NULL; ++ mali_mem_backend *mem_bkend = NULL; ++ struct mali_page_node *m_page; ++ mali_bool swap_in_success = MALI_TRUE; ++ int i; ++ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ ++ num_memory_cookies = mali_pp_job_num_memory_cookies(job); ++ session = mali_pp_job_get_session(job); ++ ++ MALI_DEBUG_ASSERT_POINTER(session); ++ ++ for (i = 0; i < num_memory_cookies; i++) { ++ ++ u32 mali_addr = mali_pp_job_get_memory_cookie(job, i); ++ ++ mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_addr, 0); ++ if (NULL == mali_vma_node) { ++ job->memory_cookies[i] = MALI_SWAP_INVALIDATE_MALI_ADDRESS; ++ swap_in_success = MALI_FALSE; ++ MALI_PRINT_ERROR(("SWAP Mem: failed to find mali_vma_node through Mali address: 0x%08x.\n", mali_addr)); ++ continue; ++ } ++ ++ mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); ++ MALI_DEBUG_ASSERT(NULL != mali_alloc); ++ ++ if (MALI_MEM_SWAP != mali_alloc->type && ++ MALI_MEM_COW != mali_alloc->type) { ++ continue; ++ } ++ ++ /* Get backend memory & Map on GPU */ ++ mutex_lock(&mali_idr_mutex); ++ mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle); ++ mutex_unlock(&mali_idr_mutex); ++ MALI_DEBUG_ASSERT(NULL != mem_bkend); ++ ++ /* We neednot hold backend's lock here, race safe.*/ ++ if ((MALI_MEM_COW == mem_bkend->type) && ++ (!(mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED))) { ++ continue; ++ } ++ ++ mutex_lock(&mem_bkend->mutex); ++ ++ /* When swap_in_success is MALI_FALSE, it means this job has memory backend that could not be swapped in, ++ * and it will be aborted in mali scheduler, so here, we just mark those memory cookies which ++ * should not be swapped out when delete job to invalide */ ++ if (MALI_FALSE == swap_in_success) { ++ job->memory_cookies[i] = MALI_SWAP_INVALIDATE_MALI_ADDRESS; ++ mutex_unlock(&mem_bkend->mutex); ++ continue; ++ } ++ ++ /* Before swap in, checking if this memory backend has been swapped in by the latest flushed jobs. */ ++ ++mem_bkend->using_count; ++ ++ if (1 < mem_bkend->using_count) { ++ MALI_DEBUG_ASSERT(MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN != (MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN & mem_bkend->flags)); ++ mutex_unlock(&mem_bkend->mutex); ++ continue; ++ } ++ ++ if (MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN != (MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN & mem_bkend->flags)) { ++ mutex_unlock(&mem_bkend->mutex); ++ continue; ++ } ++ ++ ++ list_for_each_entry(m_page, &mem_bkend->swap_mem.pages, list) { ++ if (MALI_FALSE == mali_mem_swap_in_page_node(m_page)) { ++ /* Don't have enough memory to swap in page, so release pages have already been swapped ++ * in and then mark this pp job to be fail. */ ++ mali_mem_swap_unlock_partial_locked_mem_backend(mem_bkend, m_page); ++ swap_in_success = MALI_FALSE; ++ break; ++ } ++ } ++ ++ if (swap_in_success) { ++#ifdef MALI_MEM_SWAP_TRACKING ++ mem_backend_swapped_unlock_size -= mem_bkend->size; ++#endif ++ _mali_osk_mutex_wait(session->memory_lock); ++ mali_mem_swap_mali_map(&mem_bkend->swap_mem, session, mali_alloc->mali_mapping.addr, mali_alloc->mali_mapping.properties); ++ _mali_osk_mutex_signal(session->memory_lock); ++ ++ /* Remove the unlock flag from mem backend flags, mark this backend has been swapped in. */ ++ mem_bkend->flags &= ~(MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN); ++ mutex_unlock(&mem_bkend->mutex); ++ } else { ++ --mem_bkend->using_count; ++ /* Marking that this backend is not swapped in, need not to be processed anymore. */ ++ job->memory_cookies[i] = MALI_SWAP_INVALIDATE_MALI_ADDRESS; ++ mutex_unlock(&mem_bkend->mutex); ++ } ++ } ++ ++ job->swap_status = swap_in_success ? MALI_SWAP_IN_SUCC : MALI_SWAP_IN_FAIL; ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++int mali_mem_swap_out_pages(struct mali_pp_job *job) ++{ ++ u32 num_memory_cookies; ++ struct mali_session_data *session; ++ struct mali_vma_node *mali_vma_node = NULL; ++ mali_mem_allocation *mali_alloc = NULL; ++ mali_mem_backend *mem_bkend = NULL; ++ int i; ++ ++ MALI_DEBUG_ASSERT_POINTER(job); ++ ++ num_memory_cookies = mali_pp_job_num_memory_cookies(job); ++ session = mali_pp_job_get_session(job); ++ ++ MALI_DEBUG_ASSERT_POINTER(session); ++ ++ ++ for (i = 0; i < num_memory_cookies; i++) { ++ u32 mali_addr = mali_pp_job_get_memory_cookie(job, i); ++ ++ if (MALI_SWAP_INVALIDATE_MALI_ADDRESS == mali_addr) { ++ continue; ++ } ++ ++ mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_addr, 0); ++ ++ if (NULL == mali_vma_node) { ++ MALI_PRINT_ERROR(("SWAP Mem: failed to find mali_vma_node through Mali address: 0x%08x.\n", mali_addr)); ++ continue; ++ } ++ ++ mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); ++ MALI_DEBUG_ASSERT(NULL != mali_alloc); ++ ++ if (MALI_MEM_SWAP != mali_alloc->type && ++ MALI_MEM_COW != mali_alloc->type) { ++ continue; ++ } ++ ++ mutex_lock(&mali_idr_mutex); ++ mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle); ++ mutex_unlock(&mali_idr_mutex); ++ MALI_DEBUG_ASSERT(NULL != mem_bkend); ++ ++ /* We neednot hold backend's lock here, race safe.*/ ++ if ((MALI_MEM_COW == mem_bkend->type) && ++ (!(mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED))) { ++ continue; ++ } ++ ++ mutex_lock(&mem_bkend->mutex); ++ ++ MALI_DEBUG_ASSERT(0 < mem_bkend->using_count); ++ ++ /* Reducing the using_count of mem backend means less pp job are using this memory backend, ++ * if this count get to zero, it means no pp job is using it now, could put it to swap out list. */ ++ --mem_bkend->using_count; ++ ++ if (0 < mem_bkend->using_count) { ++ mutex_unlock(&mem_bkend->mutex); ++ continue; ++ } ++ mutex_unlock(&mem_bkend->mutex); ++ ++ mali_memory_swap_list_backend_add(mem_bkend); ++ } ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++int mali_mem_swap_allocate_page_on_demand(mali_mem_backend *mem_bkend, u32 offset, struct page **pagep) ++{ ++ struct mali_page_node *m_page, *found_node = NULL; ++ struct page *found_page; ++ mali_mem_swap *swap = NULL; ++ mali_mem_cow *cow = NULL; ++ dma_addr_t dma_addr; ++ u32 i = 0; ++ ++ if (MALI_MEM_SWAP == mem_bkend->type) { ++ swap = &mem_bkend->swap_mem; ++ list_for_each_entry(m_page, &swap->pages, list) { ++ if (i == offset) { ++ found_node = m_page; ++ break; ++ } ++ i++; ++ } ++ } else { ++ MALI_DEBUG_ASSERT(MALI_MEM_COW == mem_bkend->type); ++ MALI_DEBUG_ASSERT(MALI_MEM_BACKEND_FLAG_SWAP_COWED == (MALI_MEM_BACKEND_FLAG_SWAP_COWED & mem_bkend->flags)); ++ ++ cow = &mem_bkend->cow_mem; ++ list_for_each_entry(m_page, &cow->pages, list) { ++ if (i == offset) { ++ found_node = m_page; ++ break; ++ } ++ i++; ++ } ++ } ++ ++ if (NULL == found_node) { ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ found_page = shmem_read_mapping_page(global_swap_space, found_node->swap_it->idx); ++ ++ if (!IS_ERR(found_page)) { ++ lock_page(found_page); ++ dma_addr = dma_map_page(&mali_platform_device->dev, found_page, ++ 0, _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE); ++ dma_unmap_page(&mali_platform_device->dev, dma_addr, ++ _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE); ++ ++ *pagep = found_page; ++ } else { ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++int mali_mem_swap_cow_page_on_demand(mali_mem_backend *mem_bkend, u32 offset, struct page **pagep) ++{ ++ struct mali_page_node *m_page, *found_node = NULL, *new_node = NULL; ++ mali_mem_cow *cow = NULL; ++ u32 i = 0; ++ ++ MALI_DEBUG_ASSERT(MALI_MEM_COW == mem_bkend->type); ++ MALI_DEBUG_ASSERT(MALI_MEM_BACKEND_FLAG_SWAP_COWED == (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED)); ++ MALI_DEBUG_ASSERT(MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN == (MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN & mem_bkend->flags)); ++ MALI_DEBUG_ASSERT(!mali_memory_swap_backend_in_swapped_pool(mem_bkend)); ++ ++ cow = &mem_bkend->cow_mem; ++ list_for_each_entry(m_page, &cow->pages, list) { ++ if (i == offset) { ++ found_node = m_page; ++ break; ++ } ++ i++; ++ } ++ ++ if (NULL == found_node) { ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ new_node = _mali_mem_swap_page_node_allocate(); ++ ++ if (NULL == new_node) { ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ new_node->swap_it->idx = mali_mem_swap_idx_alloc(); ++ ++ if (_MALI_OSK_BITMAP_INVALIDATE_INDEX == new_node->swap_it->idx) { ++ MALI_DEBUG_PRINT(1, ("Failed to allocate swap index in swap CoW on demand.\n")); ++ kfree(new_node->swap_it); ++ kfree(new_node); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ if (MALI_FALSE == mali_mem_swap_in_page_node(new_node)) { ++ _mali_mem_swap_page_node_free(new_node); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ /* swap in found node for copy in kernel. */ ++ if (MALI_FALSE == mali_mem_swap_in_page_node(found_node)) { ++ mali_mem_swap_out_page_node(new_node); ++ _mali_mem_swap_page_node_free(new_node); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ _mali_mem_cow_copy_page(found_node, new_node); ++ ++ list_replace(&found_node->list, &new_node->list); ++ ++ if (1 != _mali_page_node_get_ref_count(found_node)) { ++ atomic_add(1, &mem_bkend->mali_allocation->session->mali_mem_allocated_pages); ++ if (atomic_read(&mem_bkend->mali_allocation->session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > mem_bkend->mali_allocation->session->max_mali_mem_allocated_size) { ++ mem_bkend->mali_allocation->session->max_mali_mem_allocated_size = atomic_read(&mem_bkend->mali_allocation->session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE; ++ } ++ mem_bkend->cow_mem.change_pages_nr++; ++ } ++ ++ mali_mem_swap_out_page_node(found_node); ++ _mali_mem_swap_page_node_free(found_node); ++ ++ /* When swap in the new page node, we have called dma_map_page for this page.\n */ ++ dma_unmap_page(&mali_platform_device->dev, new_node->swap_it->dma_addr, ++ _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE); ++ ++ lock_page(new_node->swap_it->page); ++ ++ *pagep = new_node->swap_it->page; ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++#ifdef MALI_MEM_SWAP_TRACKING ++void mali_mem_swap_tracking(u32 *swap_pool_size, u32 *unlock_size) ++{ ++ *swap_pool_size = mem_backend_swapped_pool_size; ++ *unlock_size = mem_backend_swapped_unlock_size; ++} ++#endif +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_swap_alloc.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_swap_alloc.h +new file mode 100755 +index 000000000000..5810960e204a +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_swap_alloc.h +@@ -0,0 +1,121 @@ ++/* ++ * Copyright (C) 2013-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_MEMORY_SWAP_ALLOC_H__ ++#define __MALI_MEMORY_SWAP_ALLOC_H__ ++ ++#include "mali_osk.h" ++#include "mali_session.h" ++ ++#include "mali_memory_types.h" ++#include "mali_pp_job.h" ++ ++/** ++ * Initialize memory swapping module. ++ */ ++_mali_osk_errcode_t mali_mem_swap_init(void); ++ ++void mali_mem_swap_term(void); ++ ++/** ++ * Return global share memory file to other modules. ++ */ ++struct file *mali_mem_swap_get_global_swap_file(void); ++ ++/** ++ * Unlock the given memory backend and pages in it could be swapped out by kernel. ++ */ ++void mali_mem_swap_unlock_single_mem_backend(mali_mem_backend *mem_bkend); ++ ++/** ++ * Remove the given memory backend from global swap list. ++ */ ++void mali_memory_swap_list_backend_delete(mali_mem_backend *mem_bkend); ++ ++/** ++ * Add the given memory backend to global swap list. ++ */ ++void mali_memory_swap_list_backend_add(mali_mem_backend *mem_bkend); ++ ++/** ++ * Allocate 1 index from bitmap used as page index in global swap file. ++ */ ++u32 mali_mem_swap_idx_alloc(void); ++ ++void mali_mem_swap_idx_free(u32 idx); ++ ++/** ++ * Allocate a new swap item without page index. ++ */ ++struct mali_swap_item *mali_mem_swap_alloc_swap_item(void); ++ ++/** ++ * Free a swap item, truncate the corresponding space in page cache and free index of page. ++ */ ++void mali_mem_swap_free_swap_item(mali_swap_item *swap_item); ++ ++/** ++ * Allocate a page node with swap item. ++ */ ++struct mali_page_node *_mali_mem_swap_page_node_allocate(void); ++ ++/** ++ * Reduce the reference count of given page node and if return 0, just free this page node. ++ */ ++_mali_osk_errcode_t _mali_mem_swap_put_page_node(struct mali_page_node *m_page); ++ ++void _mali_mem_swap_page_node_free(struct mali_page_node *m_page); ++ ++/** ++ * Free a swappable memory backend. ++ */ ++u32 mali_mem_swap_free(mali_mem_swap *swap_mem); ++ ++/** ++ * Ummap and free. ++ */ ++u32 mali_mem_swap_release(mali_mem_backend *mem_bkend, mali_bool is_mali_mapped); ++ ++/** ++ * Read in a page from global swap file with the pre-allcated page index. ++ */ ++mali_bool mali_mem_swap_in_page_node(struct mali_page_node *page_node); ++ ++int mali_mem_swap_alloc_pages(mali_mem_swap *swap_mem, u32 size, u32 *bkend_idx); ++ ++_mali_osk_errcode_t mali_mem_swap_mali_map(mali_mem_swap *swap_mem, struct mali_session_data *session, u32 vaddr, u32 props); ++ ++void mali_mem_swap_mali_unmap(mali_mem_allocation *alloc); ++ ++/** ++ * When pp job created, we need swap in all of memory backend needed by this pp job. ++ */ ++int mali_mem_swap_in_pages(struct mali_pp_job *job); ++ ++/** ++ * Put all of memory backends used this pp job to the global swap list. ++ */ ++int mali_mem_swap_out_pages(struct mali_pp_job *job); ++ ++/** ++ * This will be called in page fault to process CPU read&write. ++ */ ++int mali_mem_swap_allocate_page_on_demand(mali_mem_backend *mem_bkend, u32 offset, struct page **pagep) ; ++ ++/** ++ * Used to process cow on demand for swappable memory backend. ++ */ ++int mali_mem_swap_cow_page_on_demand(mali_mem_backend *mem_bkend, u32 offset, struct page **pagep); ++ ++#ifdef MALI_MEM_SWAP_TRACKING ++void mali_mem_swap_tracking(u32 *swap_pool_size, u32 *unlock_size); ++#endif ++#endif /* __MALI_MEMORY_SWAP_ALLOC_H__ */ ++ +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_types.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_types.h +new file mode 100755 +index 000000000000..33db40929642 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_types.h +@@ -0,0 +1,219 @@ ++/* ++ * Copyright (C) 2013-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_MEMORY_TYPES_H__ ++#define __MALI_MEMORY_TYPES_H__ ++ ++#include ++ ++#if defined(CONFIG_MALI400_UMP) ++#include "ump_kernel_interface.h" ++#endif ++ ++typedef u32 mali_address_t; ++ ++typedef enum mali_mem_type { ++ MALI_MEM_OS, ++ MALI_MEM_EXTERNAL, ++ MALI_MEM_SWAP, ++ MALI_MEM_DMA_BUF, ++ MALI_MEM_UMP, ++ MALI_MEM_BLOCK, ++ MALI_MEM_COW, ++ MALI_MEM_SECURE, ++ MALI_MEM_TYPE_MAX, ++} mali_mem_type; ++ ++typedef struct mali_block_item { ++ /* for block type, the block_phy is alway page size align ++ * so use low 12bit used for ref_cout. ++ */ ++ unsigned long phy_addr; ++} mali_block_item; ++ ++/** ++ * idx is used to locate the given page in the address space of swap file. ++ * ref_count is used to mark how many memory backends are using this item. ++ */ ++typedef struct mali_swap_item { ++ u32 idx; ++ atomic_t ref_count; ++ struct page *page; ++ dma_addr_t dma_addr; ++} mali_swap_item; ++ ++typedef enum mali_page_node_type { ++ MALI_PAGE_NODE_OS, ++ MALI_PAGE_NODE_BLOCK, ++ MALI_PAGE_NODE_SWAP, ++} mali_page_node_type; ++ ++typedef struct mali_page_node { ++ struct list_head list; ++ union { ++ struct page *page; ++ mali_block_item *blk_it; /*pointer to block item*/ ++ mali_swap_item *swap_it; ++ }; ++ ++ u32 type; ++} mali_page_node; ++ ++typedef struct mali_mem_os_mem { ++ struct list_head pages; ++ u32 count; ++} mali_mem_os_mem; ++ ++typedef struct mali_mem_dma_buf { ++#if defined(CONFIG_DMA_SHARED_BUFFER) ++ struct mali_dma_buf_attachment *attachment; ++#endif ++} mali_mem_dma_buf; ++ ++typedef struct mali_mem_external { ++ dma_addr_t phys; ++ u32 size; ++} mali_mem_external; ++ ++typedef struct mali_mem_ump { ++#if defined(CONFIG_MALI400_UMP) ++ ump_dd_handle handle; ++#endif ++} mali_mem_ump; ++ ++typedef struct block_allocator_allocation { ++ /* The list will be released in reverse order */ ++ struct block_info *last_allocated; ++ u32 mapping_length; ++ struct block_allocator *info; ++} block_allocator_allocation; ++ ++typedef struct mali_mem_block_mem { ++ struct list_head pfns; ++ u32 count; ++} mali_mem_block_mem; ++ ++typedef struct mali_mem_virt_mali_mapping { ++ mali_address_t addr; /* Virtual Mali address */ ++ u32 properties; /* MMU Permissions + cache, must match MMU HW */ ++} mali_mem_virt_mali_mapping; ++ ++typedef struct mali_mem_virt_cpu_mapping { ++ void __user *addr; ++ struct vm_area_struct *vma; ++} mali_mem_virt_cpu_mapping; ++ ++#define MALI_MEM_ALLOCATION_VALID_MAGIC 0xdeda110c ++#define MALI_MEM_ALLOCATION_FREED_MAGIC 0x10101010 ++ ++typedef struct mali_mm_node { ++ /* MALI GPU vaddr start, use u32 for mmu only support 32bit address*/ ++ uint32_t start; /* GPU vaddr */ ++ uint32_t size; /* GPU allocation virtual size */ ++ unsigned allocated : 1; ++} mali_mm_node; ++ ++typedef struct mali_vma_node { ++ struct mali_mm_node vm_node; ++ struct rb_node vm_rb; ++} mali_vma_node; ++ ++ ++typedef struct mali_mem_allocation { ++ MALI_DEBUG_CODE(u32 magic); ++ mali_mem_type type; /**< Type of memory */ ++ u32 flags; /**< Flags for this allocation */ ++ ++ struct mali_session_data *session; /**< Pointer to session that owns the allocation */ ++ ++ mali_mem_virt_cpu_mapping cpu_mapping; /**< CPU mapping */ ++ mali_mem_virt_mali_mapping mali_mapping; /**< Mali mapping */ ++ ++ /* add for new memory system */ ++ struct mali_vma_node mali_vma_node; ++ u32 vsize; /* virtual size*/ ++ u32 psize; /* physical backend memory size*/ ++ struct list_head list; ++ s32 backend_handle; /* idr for mem_backend */ ++ _mali_osk_atomic_t mem_alloc_refcount; ++} mali_mem_allocation; ++ ++struct mali_mem_os_allocator { ++ spinlock_t pool_lock; ++ struct list_head pool_pages; ++ size_t pool_count; ++ ++ atomic_t allocated_pages; ++ size_t allocation_limit; ++ ++ struct shrinker shrinker; ++ struct delayed_work timed_shrinker; ++ struct workqueue_struct *wq; ++}; ++ ++/* COW backend memory type */ ++typedef struct mali_mem_cow { ++ struct list_head pages; /**< all pages for this cow backend allocation, ++ including new allocated pages for modified range*/ ++ u32 count; /**< number of pages */ ++ s32 change_pages_nr; ++} mali_mem_cow; ++ ++typedef struct mali_mem_swap { ++ struct list_head pages; ++ u32 count; ++} mali_mem_swap; ++ ++typedef struct mali_mem_secure { ++#if defined(CONFIG_DMA_SHARED_BUFFER) ++ struct dma_buf *buf; ++ struct dma_buf_attachment *attachment; ++ struct sg_table *sgt; ++#endif ++ u32 count; ++} mali_mem_secure; ++ ++#define MALI_MEM_BACKEND_FLAG_COWED (0x1) /* COW has happen on this backend */ ++#define MALI_MEM_BACKEND_FLAG_COW_CPU_NO_WRITE (0x2) /* This is an COW backend, mapped as not allowed cpu to write */ ++#define MALI_MEM_BACKEND_FLAG_SWAP_COWED (0x4) /* Mark the given backend is cowed from swappable memory. */ ++/* Mark this backend is not swapped_in in MALI driver, and before using it, ++ * we should swap it in and set up corresponding page table. */ ++#define MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN (0x8) ++#define MALI_MEM_BACKEND_FLAG_NOT_BINDED (0x1 << 5) /* this backend it not back with physical memory, used for defer bind */ ++#define MALI_MEM_BACKEND_FLAG_BINDED (0x1 << 6) /* this backend it back with physical memory, used for defer bind */ ++ ++typedef struct mali_mem_backend { ++ mali_mem_type type; /**< Type of backend memory */ ++ u32 flags; /**< Flags for this allocation */ ++ u32 size; ++ /* Union selected by type. */ ++ union { ++ mali_mem_os_mem os_mem; /**< MALI_MEM_OS */ ++ mali_mem_external ext_mem; /**< MALI_MEM_EXTERNAL */ ++ mali_mem_dma_buf dma_buf; /**< MALI_MEM_DMA_BUF */ ++ mali_mem_ump ump_mem; /**< MALI_MEM_UMP */ ++ mali_mem_block_mem block_mem; /**< MALI_MEM_BLOCK */ ++ mali_mem_cow cow_mem; ++ mali_mem_swap swap_mem; ++ mali_mem_secure secure_mem; ++ }; ++ mali_mem_allocation *mali_allocation; ++ struct mutex mutex; ++ mali_mem_type cow_type; ++ ++ struct list_head list; /**< Used to link swappable memory backend to the global swappable list */ ++ int using_count; /**< Mark how many PP jobs are using this memory backend */ ++ u32 start_idx; /**< If the correspondign vma of this backend is linear, this value will be used to set vma->vm_pgoff */ ++} mali_mem_backend; ++ ++#define MALI_MEM_FLAG_MALI_GUARD_PAGE (_MALI_MAP_EXTERNAL_MAP_GUARD_PAGE) ++#define MALI_MEM_FLAG_DONT_CPU_MAP (1 << 1) ++#define MALI_MEM_FLAG_CAN_RESIZE (_MALI_MEMORY_ALLOCATE_RESIZEABLE) ++#endif /* __MALI_MEMORY_TYPES__ */ +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_ump.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_ump.c +new file mode 100755 +index 000000000000..666d4b0fb1cd +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_ump.c +@@ -0,0 +1,154 @@ ++/* ++ * Copyright (C) 2012-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include "mali_ukk.h" ++#include "mali_osk.h" ++#include "mali_kernel_common.h" ++#include "mali_session.h" ++#include "mali_kernel_linux.h" ++#include "mali_memory.h" ++#include "ump_kernel_interface.h" ++ ++static int mali_mem_ump_map(mali_mem_backend *mem_backend) ++{ ++ ump_dd_handle ump_mem; ++ mali_mem_allocation *alloc; ++ struct mali_session_data *session; ++ u32 nr_blocks; ++ u32 i; ++ ump_dd_physical_block *ump_blocks; ++ struct mali_page_directory *pagedir; ++ u32 offset = 0; ++ _mali_osk_errcode_t err; ++ ++ MALI_DEBUG_ASSERT_POINTER(mem_backend); ++ MALI_DEBUG_ASSERT(MALI_MEM_UMP == mem_backend->type); ++ ++ alloc = mem_backend->mali_allocation; ++ MALI_DEBUG_ASSERT_POINTER(alloc); ++ ++ session = alloc->session; ++ MALI_DEBUG_ASSERT_POINTER(session); ++ ++ ump_mem = mem_backend->ump_mem.handle; ++ MALI_DEBUG_ASSERT(UMP_DD_HANDLE_INVALID != ump_mem); ++ ++ nr_blocks = ump_dd_phys_block_count_get(ump_mem); ++ if (nr_blocks == 0) { ++ MALI_DEBUG_PRINT(1, ("No block count\n")); ++ return -EINVAL; ++ } ++ ++ ump_blocks = _mali_osk_malloc(sizeof(*ump_blocks) * nr_blocks); ++ if (NULL == ump_blocks) { ++ return -ENOMEM; ++ } ++ ++ if (UMP_DD_INVALID == ump_dd_phys_blocks_get(ump_mem, ump_blocks, nr_blocks)) { ++ _mali_osk_free(ump_blocks); ++ return -EFAULT; ++ } ++ ++ pagedir = session->page_directory; ++ ++ mali_session_memory_lock(session); ++ ++ err = mali_mem_mali_map_prepare(alloc); ++ if (_MALI_OSK_ERR_OK != err) { ++ MALI_DEBUG_PRINT(1, ("Mapping of UMP memory failed\n")); ++ ++ _mali_osk_free(ump_blocks); ++ mali_session_memory_unlock(session); ++ return -ENOMEM; ++ } ++ ++ for (i = 0; i < nr_blocks; ++i) { ++ u32 virt = alloc->mali_vma_node.vm_node.start + offset; ++ ++ MALI_DEBUG_PRINT(7, ("Mapping in 0x%08x size %d\n", ump_blocks[i].addr , ump_blocks[i].size)); ++ ++ mali_mmu_pagedir_update(pagedir, virt, ump_blocks[i].addr, ++ ump_blocks[i].size, MALI_MMU_FLAGS_DEFAULT); ++ ++ offset += ump_blocks[i].size; ++ } ++ ++ if (alloc->flags & _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE) { ++ u32 virt = alloc->mali_vma_node.vm_node.start + offset; ++ ++ /* Map in an extra virtual guard page at the end of the VMA */ ++ MALI_DEBUG_PRINT(6, ("Mapping in extra guard page\n")); ++ ++ mali_mmu_pagedir_update(pagedir, virt, ump_blocks[0].addr, _MALI_OSK_MALI_PAGE_SIZE, MALI_MMU_FLAGS_DEFAULT); ++ ++ offset += _MALI_OSK_MALI_PAGE_SIZE; ++ } ++ mali_session_memory_unlock(session); ++ _mali_osk_free(ump_blocks); ++ return 0; ++} ++ ++static void mali_mem_ump_unmap(mali_mem_allocation *alloc) ++{ ++ struct mali_session_data *session; ++ MALI_DEBUG_ASSERT_POINTER(alloc); ++ session = alloc->session; ++ MALI_DEBUG_ASSERT_POINTER(session); ++ mali_session_memory_lock(session); ++ mali_mem_mali_map_free(session, alloc->psize, alloc->mali_vma_node.vm_node.start, ++ alloc->flags); ++ mali_session_memory_unlock(session); ++} ++ ++int mali_mem_bind_ump_buf(mali_mem_allocation *alloc, mali_mem_backend *mem_backend, u32 secure_id, u32 flags) ++{ ++ ump_dd_handle ump_mem; ++ int ret; ++ MALI_DEBUG_ASSERT_POINTER(alloc); ++ MALI_DEBUG_ASSERT_POINTER(mem_backend); ++ MALI_DEBUG_ASSERT(MALI_MEM_UMP == mem_backend->type); ++ ++ MALI_DEBUG_PRINT(3, ++ ("Requested to map ump memory with secure id %d into virtual memory 0x%08X, size 0x%08X\n", ++ secure_id, alloc->mali_vma_node.vm_node.start, alloc->mali_vma_node.vm_node.size)); ++ ++ ump_mem = ump_dd_handle_create_from_secure_id(secure_id); ++ if (UMP_DD_HANDLE_INVALID == ump_mem) MALI_ERROR(_MALI_OSK_ERR_FAULT); ++ alloc->flags |= MALI_MEM_FLAG_DONT_CPU_MAP; ++ if (flags & _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE) { ++ alloc->flags |= MALI_MEM_FLAG_MALI_GUARD_PAGE; ++ } ++ ++ mem_backend->ump_mem.handle = ump_mem; ++ ++ ret = mali_mem_ump_map(mem_backend); ++ if (0 != ret) { ++ ump_dd_reference_release(ump_mem); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ MALI_DEBUG_PRINT(3, ("Returning from UMP bind\n")); ++ return _MALI_OSK_ERR_OK; ++} ++ ++void mali_mem_unbind_ump_buf(mali_mem_backend *mem_backend) ++{ ++ ump_dd_handle ump_mem; ++ mali_mem_allocation *alloc; ++ MALI_DEBUG_ASSERT_POINTER(mem_backend); ++ MALI_DEBUG_ASSERT(MALI_MEM_UMP == mem_backend->type); ++ ump_mem = mem_backend->ump_mem.handle; ++ MALI_DEBUG_ASSERT(UMP_DD_HANDLE_INVALID != ump_mem); ++ ++ alloc = mem_backend->mali_allocation; ++ MALI_DEBUG_ASSERT_POINTER(alloc); ++ mali_mem_ump_unmap(alloc); ++ ump_dd_reference_release(ump_mem); ++} ++ +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_ump.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_ump.h +new file mode 100755 +index 000000000000..c314c8dcbf1c +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_ump.h +@@ -0,0 +1,29 @@ ++/* ++ * Copyright (C) 2011-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_MEMORY_UMP_BUF_H__ ++#define __MALI_MEMORY_UMP_BUF_H__ ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++#include "mali_uk_types.h" ++#include "mali_osk.h" ++#include "mali_memory.h" ++ ++int mali_mem_bind_ump_buf(mali_mem_allocation *alloc, mali_mem_backend *mem_backend, u32 secure_id, u32 flags); ++void mali_mem_unbind_ump_buf(mali_mem_backend *mem_backend); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif /* __MALI_MEMORY_DMA_BUF_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_util.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_util.c +new file mode 100755 +index 000000000000..8e13e923c3fb +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_util.c +@@ -0,0 +1,158 @@ ++/* ++ * Copyright (C) 2013-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "mali_osk.h" ++#include "mali_osk_mali.h" ++#include "mali_kernel_linux.h" ++#include "mali_scheduler.h" ++ ++#include "mali_memory.h" ++#include "mali_memory_os_alloc.h" ++#if defined(CONFIG_DMA_SHARED_BUFFER) ++#include "mali_memory_dma_buf.h" ++#include "mali_memory_secure.h" ++#endif ++#if defined(CONFIG_MALI400_UMP) ++#include "mali_memory_ump.h" ++#endif ++#include "mali_memory_external.h" ++#include "mali_memory_manager.h" ++#include "mali_memory_virtual.h" ++#include "mali_memory_cow.h" ++#include "mali_memory_block_alloc.h" ++#include "mali_memory_swap_alloc.h" ++ ++ ++ ++/** ++*function @_mali_free_allocation_mem - free a memory allocation ++*/ ++static u32 _mali_free_allocation_mem(mali_mem_allocation *mali_alloc) ++{ ++ mali_mem_backend *mem_bkend = NULL; ++ u32 free_pages_nr = 0; ++ ++ struct mali_session_data *session = mali_alloc->session; ++ MALI_DEBUG_PRINT(4, (" _mali_free_allocation_mem, psize =0x%x! \n", mali_alloc->psize)); ++ if (0 == mali_alloc->psize) ++ goto out; ++ ++ /* Get backend memory & Map on CPU */ ++ mutex_lock(&mali_idr_mutex); ++ mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle); ++ mutex_unlock(&mali_idr_mutex); ++ MALI_DEBUG_ASSERT(NULL != mem_bkend); ++ ++ switch (mem_bkend->type) { ++ case MALI_MEM_OS: ++ free_pages_nr = mali_mem_os_release(mem_bkend); ++ atomic_sub(free_pages_nr, &session->mali_mem_allocated_pages); ++ break; ++ case MALI_MEM_UMP: ++#if defined(CONFIG_MALI400_UMP) ++ mali_mem_unbind_ump_buf(mem_bkend); ++ atomic_sub(mem_bkend->size / MALI_MMU_PAGE_SIZE, &session->mali_mem_array[mem_bkend->type]); ++#else ++ MALI_DEBUG_PRINT(1, ("UMP not supported\n")); ++#endif ++ break; ++ case MALI_MEM_DMA_BUF: ++#if defined(CONFIG_DMA_SHARED_BUFFER) ++ mali_mem_unbind_dma_buf(mem_bkend); ++ atomic_sub(mem_bkend->size / MALI_MMU_PAGE_SIZE, &session->mali_mem_array[mem_bkend->type]); ++#else ++ MALI_DEBUG_PRINT(1, ("DMA not supported\n")); ++#endif ++ break; ++ case MALI_MEM_EXTERNAL: ++ mali_mem_unbind_ext_buf(mem_bkend); ++ atomic_sub(mem_bkend->size / MALI_MMU_PAGE_SIZE, &session->mali_mem_array[mem_bkend->type]); ++ break; ++ ++ case MALI_MEM_BLOCK: ++ free_pages_nr = mali_mem_block_release(mem_bkend); ++ atomic_sub(free_pages_nr, &session->mali_mem_allocated_pages); ++ break; ++ ++ case MALI_MEM_COW: ++ if (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED) { ++ free_pages_nr = mali_mem_swap_release(mem_bkend, MALI_TRUE); ++ } else { ++ free_pages_nr = mali_mem_cow_release(mem_bkend, MALI_TRUE); ++ } ++ atomic_sub(free_pages_nr, &session->mali_mem_allocated_pages); ++ break; ++ case MALI_MEM_SWAP: ++ free_pages_nr = mali_mem_swap_release(mem_bkend, MALI_TRUE); ++ atomic_sub(free_pages_nr, &session->mali_mem_allocated_pages); ++ atomic_sub(free_pages_nr, &session->mali_mem_array[mem_bkend->type]); ++ break; ++ case MALI_MEM_SECURE: ++#if defined(CONFIG_DMA_SHARED_BUFFER) ++ free_pages_nr = mali_mem_secure_release(mem_bkend); ++ atomic_sub(free_pages_nr, &session->mali_mem_allocated_pages); ++#else ++ MALI_DEBUG_PRINT(1, ("DMA not supported for mali secure memory\n")); ++#endif ++ break; ++ default: ++ MALI_DEBUG_PRINT(1, ("mem type %d is not in the mali_mem_type enum.\n", mem_bkend->type)); ++ break; ++ } ++ ++ /*Remove backend memory idex */ ++ mutex_lock(&mali_idr_mutex); ++ idr_remove(&mali_backend_idr, mali_alloc->backend_handle); ++ mutex_unlock(&mali_idr_mutex); ++ kfree(mem_bkend); ++out: ++ /* remove memory allocation */ ++ mali_vma_offset_remove(&session->allocation_mgr, &mali_alloc->mali_vma_node); ++ mali_mem_allocation_struct_destory(mali_alloc); ++ return free_pages_nr; ++} ++ ++/** ++* ref_count for allocation ++*/ ++u32 mali_allocation_unref(struct mali_mem_allocation **alloc) ++{ ++ u32 free_pages_nr = 0; ++ mali_mem_allocation *mali_alloc = *alloc; ++ *alloc = NULL; ++ if (0 == _mali_osk_atomic_dec_return(&mali_alloc->mem_alloc_refcount)) { ++ free_pages_nr = _mali_free_allocation_mem(mali_alloc); ++ } ++ return free_pages_nr; ++} ++ ++void mali_allocation_ref(struct mali_mem_allocation *alloc) ++{ ++ _mali_osk_atomic_inc(&alloc->mem_alloc_refcount); ++} ++ ++void mali_free_session_allocations(struct mali_session_data *session) ++{ ++ struct mali_mem_allocation *entry, *next; ++ ++ MALI_DEBUG_PRINT(4, (" mali_free_session_allocations! \n")); ++ ++ list_for_each_entry_safe(entry, next, &session->allocation_mgr.head, list) { ++ mali_allocation_unref(&entry); ++ } ++} +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_util.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_util.h +new file mode 100755 +index 000000000000..33ac99509740 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_util.h +@@ -0,0 +1,20 @@ ++/* ++ * Copyright (C) 2013-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_MEMORY_UTIL_H__ ++#define __MALI_MEMORY_UTIL_H__ ++ ++u32 mali_allocation_unref(struct mali_mem_allocation **alloc); ++ ++void mali_allocation_ref(struct mali_mem_allocation *alloc); ++ ++void mali_free_session_allocations(struct mali_session_data *session); ++ ++#endif +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_virtual.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_virtual.c +new file mode 100755 +index 000000000000..0b31e3a23432 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_virtual.c +@@ -0,0 +1,127 @@ ++/* ++ * Copyright (C) 2013-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "mali_osk.h" ++#include "mali_osk_mali.h" ++#include "mali_kernel_linux.h" ++#include "mali_scheduler.h" ++#include "mali_memory_os_alloc.h" ++#include "mali_memory_manager.h" ++#include "mali_memory_virtual.h" ++ ++ ++/** ++*internal helper to link node into the rb-tree ++*/ ++static inline void _mali_vma_offset_add_rb(struct mali_allocation_manager *mgr, ++ struct mali_vma_node *node) ++{ ++ struct rb_node **iter = &mgr->allocation_mgr_rb.rb_node; ++ struct rb_node *parent = NULL; ++ struct mali_vma_node *iter_node; ++ ++ while (likely(*iter)) { ++ parent = *iter; ++ iter_node = rb_entry(*iter, struct mali_vma_node, vm_rb); ++ ++ if (node->vm_node.start < iter_node->vm_node.start) ++ iter = &(*iter)->rb_left; ++ else if (node->vm_node.start > iter_node->vm_node.start) ++ iter = &(*iter)->rb_right; ++ else ++ MALI_DEBUG_ASSERT(0); ++ } ++ ++ rb_link_node(&node->vm_rb, parent, iter); ++ rb_insert_color(&node->vm_rb, &mgr->allocation_mgr_rb); ++} ++ ++/** ++ * mali_vma_offset_add() - Add offset node to RB Tree ++ */ ++int mali_vma_offset_add(struct mali_allocation_manager *mgr, ++ struct mali_vma_node *node) ++{ ++ int ret = 0; ++ write_lock(&mgr->vm_lock); ++ ++ if (node->vm_node.allocated) { ++ goto out; ++ } ++ ++ _mali_vma_offset_add_rb(mgr, node); ++ /* set to allocated */ ++ node->vm_node.allocated = 1; ++ ++out: ++ write_unlock(&mgr->vm_lock); ++ return ret; ++} ++ ++/** ++ * mali_vma_offset_remove() - Remove offset node from RB tree ++ */ ++void mali_vma_offset_remove(struct mali_allocation_manager *mgr, ++ struct mali_vma_node *node) ++{ ++ write_lock(&mgr->vm_lock); ++ ++ if (node->vm_node.allocated) { ++ rb_erase(&node->vm_rb, &mgr->allocation_mgr_rb); ++ memset(&node->vm_node, 0, sizeof(node->vm_node)); ++ } ++ write_unlock(&mgr->vm_lock); ++} ++ ++/** ++* mali_vma_offset_search - Search the node in RB tree ++*/ ++struct mali_vma_node *mali_vma_offset_search(struct mali_allocation_manager *mgr, ++ unsigned long start, unsigned long pages) ++{ ++ struct mali_vma_node *node, *best; ++ struct rb_node *iter; ++ unsigned long offset; ++ read_lock(&mgr->vm_lock); ++ ++ iter = mgr->allocation_mgr_rb.rb_node; ++ best = NULL; ++ ++ while (likely(iter)) { ++ node = rb_entry(iter, struct mali_vma_node, vm_rb); ++ offset = node->vm_node.start; ++ if (start >= offset) { ++ iter = iter->rb_right; ++ best = node; ++ if (start == offset) ++ break; ++ } else { ++ iter = iter->rb_left; ++ } ++ } ++ ++ if (best) { ++ offset = best->vm_node.start + best->vm_node.size; ++ if (offset <= start + pages) ++ best = NULL; ++ } ++ read_unlock(&mgr->vm_lock); ++ ++ return best; ++} ++ +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_virtual.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_virtual.h +new file mode 100755 +index 000000000000..fd03ed9f2bbb +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_virtual.h +@@ -0,0 +1,35 @@ ++/* ++ * Copyright (C) 2013-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++#ifndef __MALI_GPU_VMEM_H__ ++#define __MALI_GPU_VMEM_H__ ++ ++#include "mali_osk.h" ++#include "mali_session.h" ++#include ++#include ++#include ++#include ++#include ++#include "mali_memory_types.h" ++#include "mali_memory_os_alloc.h" ++#include "mali_memory_manager.h" ++ ++ ++ ++int mali_vma_offset_add(struct mali_allocation_manager *mgr, ++ struct mali_vma_node *node); ++ ++void mali_vma_offset_remove(struct mali_allocation_manager *mgr, ++ struct mali_vma_node *node); ++ ++struct mali_vma_node *mali_vma_offset_search(struct mali_allocation_manager *mgr, ++ unsigned long start, unsigned long pages); ++ ++#endif +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_atomics.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_atomics.c +new file mode 100755 +index 000000000000..5bc0e52ebe23 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_atomics.c +@@ -0,0 +1,59 @@ ++/* ++ * Copyright (C) 2010, 2013-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file mali_osk_atomics.c ++ * Implementation of the OS abstraction layer for the kernel device driver ++ */ ++ ++#include "mali_osk.h" ++#include ++#include "mali_kernel_common.h" ++ ++void _mali_osk_atomic_dec(_mali_osk_atomic_t *atom) ++{ ++ atomic_dec((atomic_t *)&atom->u.val); ++} ++ ++u32 _mali_osk_atomic_dec_return(_mali_osk_atomic_t *atom) ++{ ++ return atomic_dec_return((atomic_t *)&atom->u.val); ++} ++ ++void _mali_osk_atomic_inc(_mali_osk_atomic_t *atom) ++{ ++ atomic_inc((atomic_t *)&atom->u.val); ++} ++ ++u32 _mali_osk_atomic_inc_return(_mali_osk_atomic_t *atom) ++{ ++ return atomic_inc_return((atomic_t *)&atom->u.val); ++} ++ ++void _mali_osk_atomic_init(_mali_osk_atomic_t *atom, u32 val) ++{ ++ MALI_DEBUG_ASSERT_POINTER(atom); ++ atomic_set((atomic_t *)&atom->u.val, val); ++} ++ ++u32 _mali_osk_atomic_read(_mali_osk_atomic_t *atom) ++{ ++ return atomic_read((atomic_t *)&atom->u.val); ++} ++ ++void _mali_osk_atomic_term(_mali_osk_atomic_t *atom) ++{ ++ MALI_IGNORE(atom); ++} ++ ++u32 _mali_osk_atomic_xchg(_mali_osk_atomic_t *atom, u32 val) ++{ ++ return atomic_xchg((atomic_t *)&atom->u.val, val); ++} +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_bitmap.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_bitmap.c +new file mode 100755 +index 000000000000..fb9ccd2ad1e2 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_bitmap.c +@@ -0,0 +1,152 @@ ++/* ++ * Copyright (C) 2010, 2013-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file mali_osk_bitmap.c ++ * Implementation of the OS abstraction layer for the kernel device driver ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include "common/mali_kernel_common.h" ++#include "mali_osk_types.h" ++#include "mali_osk.h" ++ ++u32 _mali_osk_bitmap_alloc(struct _mali_osk_bitmap *bitmap) ++{ ++ u32 obj; ++ ++ MALI_DEBUG_ASSERT_POINTER(bitmap); ++ ++ _mali_osk_spinlock_lock(bitmap->lock); ++ ++ obj = find_next_zero_bit(bitmap->table, bitmap->max, bitmap->reserve); ++ ++ if (obj < bitmap->max) { ++ set_bit(obj, bitmap->table); ++ } else { ++ obj = -1; ++ } ++ ++ if (obj != -1) ++ --bitmap->avail; ++ _mali_osk_spinlock_unlock(bitmap->lock); ++ ++ return obj; ++} ++ ++void _mali_osk_bitmap_free(struct _mali_osk_bitmap *bitmap, u32 obj) ++{ ++ MALI_DEBUG_ASSERT_POINTER(bitmap); ++ ++ _mali_osk_bitmap_free_range(bitmap, obj, 1); ++} ++ ++u32 _mali_osk_bitmap_alloc_range(struct _mali_osk_bitmap *bitmap, int cnt) ++{ ++ u32 obj; ++ ++ MALI_DEBUG_ASSERT_POINTER(bitmap); ++ ++ if (0 >= cnt) { ++ return -1; ++ } ++ ++ if (1 == cnt) { ++ return _mali_osk_bitmap_alloc(bitmap); ++ } ++ ++ _mali_osk_spinlock_lock(bitmap->lock); ++ obj = bitmap_find_next_zero_area(bitmap->table, bitmap->max, ++ bitmap->last, cnt, 0); ++ ++ if (obj >= bitmap->max) { ++ obj = bitmap_find_next_zero_area(bitmap->table, bitmap->max, ++ bitmap->reserve, cnt, 0); ++ } ++ ++ if (obj < bitmap->max) { ++ bitmap_set(bitmap->table, obj, cnt); ++ ++ bitmap->last = (obj + cnt); ++ if (bitmap->last >= bitmap->max) { ++ bitmap->last = bitmap->reserve; ++ } ++ } else { ++ obj = -1; ++ } ++ ++ if (obj != -1) { ++ bitmap->avail -= cnt; ++ } ++ ++ _mali_osk_spinlock_unlock(bitmap->lock); ++ ++ return obj; ++} ++ ++u32 _mali_osk_bitmap_avail(struct _mali_osk_bitmap *bitmap) ++{ ++ MALI_DEBUG_ASSERT_POINTER(bitmap); ++ ++ return bitmap->avail; ++} ++ ++void _mali_osk_bitmap_free_range(struct _mali_osk_bitmap *bitmap, u32 obj, int cnt) ++{ ++ MALI_DEBUG_ASSERT_POINTER(bitmap); ++ ++ _mali_osk_spinlock_lock(bitmap->lock); ++ bitmap_clear(bitmap->table, obj, cnt); ++ bitmap->last = min(bitmap->last, obj); ++ ++ bitmap->avail += cnt; ++ _mali_osk_spinlock_unlock(bitmap->lock); ++} ++ ++int _mali_osk_bitmap_init(struct _mali_osk_bitmap *bitmap, u32 num, u32 reserve) ++{ ++ MALI_DEBUG_ASSERT_POINTER(bitmap); ++ MALI_DEBUG_ASSERT(reserve <= num); ++ ++ bitmap->reserve = reserve; ++ bitmap->last = reserve; ++ bitmap->max = num; ++ bitmap->avail = num - reserve; ++ bitmap->lock = _mali_osk_spinlock_init(_MALI_OSK_LOCKFLAG_UNORDERED, _MALI_OSK_LOCK_ORDER_FIRST); ++ if (!bitmap->lock) { ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ bitmap->table = kzalloc(BITS_TO_LONGS(bitmap->max) * ++ sizeof(long), GFP_KERNEL); ++ if (!bitmap->table) { ++ _mali_osk_spinlock_term(bitmap->lock); ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++void _mali_osk_bitmap_term(struct _mali_osk_bitmap *bitmap) ++{ ++ MALI_DEBUG_ASSERT_POINTER(bitmap); ++ ++ if (NULL != bitmap->lock) { ++ _mali_osk_spinlock_term(bitmap->lock); ++ } ++ ++ if (NULL != bitmap->table) { ++ kfree(bitmap->table); ++ } ++} ++ +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_irq.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_irq.c +new file mode 100755 +index 000000000000..5c8b9ceab9ab +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_irq.c +@@ -0,0 +1,200 @@ ++/* ++ * Copyright (C) 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file mali_osk_irq.c ++ * Implementation of the OS abstraction layer for the kernel device driver ++ */ ++ ++#include /* For memory allocation */ ++#include ++#include ++#include ++ ++#include "mali_osk.h" ++#include "mali_kernel_common.h" ++ ++typedef struct _mali_osk_irq_t_struct { ++ u32 irqnum; ++ void *data; ++ _mali_osk_irq_uhandler_t uhandler; ++} mali_osk_irq_object_t; ++ ++typedef irqreturn_t (*irq_handler_func_t)(int, void *, struct pt_regs *); ++static irqreturn_t irq_handler_upper_half(int port_name, void *dev_id); /* , struct pt_regs *regs*/ ++ ++#if defined(DEBUG) ++ ++struct test_interrupt_data { ++ _mali_osk_irq_ack_t ack_func; ++ void *probe_data; ++ mali_bool interrupt_received; ++ wait_queue_head_t wq; ++}; ++ ++static irqreturn_t test_interrupt_upper_half(int port_name, void *dev_id) ++{ ++ irqreturn_t ret = IRQ_NONE; ++ struct test_interrupt_data *data = (struct test_interrupt_data *)dev_id; ++ ++ if (_MALI_OSK_ERR_OK == data->ack_func(data->probe_data)) { ++ data->interrupt_received = MALI_TRUE; ++ wake_up(&data->wq); ++ ret = IRQ_HANDLED; ++ } ++ ++ return ret; ++} ++ ++static _mali_osk_errcode_t test_interrupt(u32 irqnum, ++ _mali_osk_irq_trigger_t trigger_func, ++ _mali_osk_irq_ack_t ack_func, ++ void *probe_data, ++ const char *description) ++{ ++ unsigned long irq_flags = 0; ++ struct test_interrupt_data data = { ++ .ack_func = ack_func, ++ .probe_data = probe_data, ++ .interrupt_received = MALI_FALSE, ++ }; ++ ++#if defined(CONFIG_MALI_SHARED_INTERRUPTS) ++ irq_flags |= IRQF_SHARED; ++#endif /* defined(CONFIG_MALI_SHARED_INTERRUPTS) */ ++ ++ if (0 != request_irq(irqnum, test_interrupt_upper_half, irq_flags, description, &data)) { ++ MALI_DEBUG_PRINT(2, ("Unable to install test IRQ handler for core '%s'\n", description)); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ init_waitqueue_head(&data.wq); ++ ++ trigger_func(probe_data); ++ wait_event_timeout(data.wq, data.interrupt_received, 100); ++ ++ free_irq(irqnum, &data); ++ ++ if (data.interrupt_received) { ++ MALI_DEBUG_PRINT(3, ("%s: Interrupt test OK\n", description)); ++ return _MALI_OSK_ERR_OK; ++ } else { ++ MALI_PRINT_ERROR(("%s: Failed interrupt test on %u\n", description, irqnum)); ++ return _MALI_OSK_ERR_FAULT; ++ } ++} ++ ++#endif /* defined(DEBUG) */ ++ ++_mali_osk_irq_t *_mali_osk_irq_init(u32 irqnum, _mali_osk_irq_uhandler_t uhandler, void *int_data, _mali_osk_irq_trigger_t trigger_func, _mali_osk_irq_ack_t ack_func, void *probe_data, const char *description) ++{ ++ mali_osk_irq_object_t *irq_object; ++ unsigned long irq_flags = 0; ++ ++#if defined(CONFIG_MALI_SHARED_INTERRUPTS) ++ irq_flags |= IRQF_SHARED; ++#endif /* defined(CONFIG_MALI_SHARED_INTERRUPTS) */ ++ ++ irq_object = kmalloc(sizeof(mali_osk_irq_object_t), GFP_KERNEL); ++ if (NULL == irq_object) { ++ return NULL; ++ } ++ ++ if (-1 == irqnum) { ++ /* Probe for IRQ */ ++ if ((NULL != trigger_func) && (NULL != ack_func)) { ++ unsigned long probe_count = 3; ++ _mali_osk_errcode_t err; ++ int irq; ++ ++ MALI_DEBUG_PRINT(2, ("Probing for irq\n")); ++ ++ do { ++ unsigned long mask; ++ ++ mask = probe_irq_on(); ++ trigger_func(probe_data); ++ ++ _mali_osk_time_ubusydelay(5); ++ ++ irq = probe_irq_off(mask); ++ err = ack_func(probe_data); ++ } while (irq < 0 && (err == _MALI_OSK_ERR_OK) && probe_count--); ++ ++ if (irq < 0 || (_MALI_OSK_ERR_OK != err)) irqnum = -1; ++ else irqnum = irq; ++ } else irqnum = -1; /* no probe functions, fault */ ++ ++ if (-1 != irqnum) { ++ /* found an irq */ ++ MALI_DEBUG_PRINT(2, ("Found irq %d\n", irqnum)); ++ } else { ++ MALI_DEBUG_PRINT(2, ("Probe for irq failed\n")); ++ } ++ } ++ ++ irq_object->irqnum = irqnum; ++ irq_object->uhandler = uhandler; ++ irq_object->data = int_data; ++ ++ if (-1 == irqnum) { ++ MALI_DEBUG_PRINT(2, ("No IRQ for core '%s' found during probe\n", description)); ++ kfree(irq_object); ++ return NULL; ++ } ++ ++#if defined(DEBUG) ++ /* Verify that the configured interrupt settings are working */ ++ if (_MALI_OSK_ERR_OK != test_interrupt(irqnum, trigger_func, ack_func, probe_data, description)) { ++ MALI_DEBUG_PRINT(2, ("Test of IRQ(%d) handler for core '%s' failed\n", irqnum, description)); ++ kfree(irq_object); ++ return NULL; ++ } ++#endif ++ ++ if (0 != request_irq(irqnum, irq_handler_upper_half, irq_flags, description, irq_object)) { ++ MALI_DEBUG_PRINT(2, ("Unable to install IRQ handler for core '%s'\n", description)); ++ kfree(irq_object); ++ return NULL; ++ } ++ ++ return irq_object; ++} ++ ++void _mali_osk_irq_term(_mali_osk_irq_t *irq) ++{ ++ mali_osk_irq_object_t *irq_object = (mali_osk_irq_object_t *)irq; ++ free_irq(irq_object->irqnum, irq_object); ++ kfree(irq_object); ++} ++ ++ ++/** This function is called directly in interrupt context from the OS just after ++ * the CPU get the hw-irq from mali, or other devices on the same IRQ-channel. ++ * It is registered one of these function for each mali core. When an interrupt ++ * arrives this function will be called equal times as registered mali cores. ++ * That means that we only check one mali core in one function call, and the ++ * core we check for each turn is given by the \a dev_id variable. ++ * If we detect an pending interrupt on the given core, we mask the interrupt ++ * out by settging the core's IRQ_MASK register to zero. ++ * Then we schedule the mali_core_irq_handler_bottom_half to run as high priority ++ * work queue job. ++ */ ++static irqreturn_t irq_handler_upper_half(int port_name, void *dev_id) /* , struct pt_regs *regs*/ ++{ ++ irqreturn_t ret = IRQ_NONE; ++ mali_osk_irq_object_t *irq_object = (mali_osk_irq_object_t *)dev_id; ++ ++ if (_MALI_OSK_ERR_OK == irq_object->uhandler(irq_object->data)) { ++ ret = IRQ_HANDLED; ++ } ++ ++ return ret; ++} +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_locks.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_locks.c +new file mode 100755 +index 000000000000..ed5f0b0da7cb +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_locks.c +@@ -0,0 +1,287 @@ ++/* ++ * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file mali_osk_locks.c ++ * Implemenation of the OS abstraction layer for the kernel device driver ++ */ ++ ++#include "mali_osk_locks.h" ++#include "mali_kernel_common.h" ++#include "mali_osk.h" ++ ++ ++#ifdef DEBUG ++#ifdef LOCK_ORDER_CHECKING ++static DEFINE_SPINLOCK(lock_tracking_lock); ++static mali_bool add_lock_to_log_and_check(struct _mali_osk_lock_debug_s *lock, uint32_t tid); ++static void remove_lock_from_log(struct _mali_osk_lock_debug_s *lock, uint32_t tid); ++static const char *const lock_order_to_string(_mali_osk_lock_order_t order); ++#endif /* LOCK_ORDER_CHECKING */ ++ ++void _mali_osk_locks_debug_init(struct _mali_osk_lock_debug_s *checker, _mali_osk_lock_flags_t flags, _mali_osk_lock_order_t order) ++{ ++ checker->orig_flags = flags; ++ checker->owner = 0; ++ ++#ifdef LOCK_ORDER_CHECKING ++ checker->order = order; ++ checker->next = NULL; ++#endif ++} ++ ++void _mali_osk_locks_debug_add(struct _mali_osk_lock_debug_s *checker) ++{ ++ checker->owner = _mali_osk_get_tid(); ++ ++#ifdef LOCK_ORDER_CHECKING ++ if (!(checker->orig_flags & _MALI_OSK_LOCKFLAG_UNORDERED)) { ++ if (!add_lock_to_log_and_check(checker, _mali_osk_get_tid())) { ++ printk(KERN_ERR "%d: ERROR lock %p taken while holding a lock of a higher order.\n", ++ _mali_osk_get_tid(), checker); ++ dump_stack(); ++ } ++ } ++#endif ++} ++ ++void _mali_osk_locks_debug_remove(struct _mali_osk_lock_debug_s *checker) ++{ ++ ++#ifdef LOCK_ORDER_CHECKING ++ if (!(checker->orig_flags & _MALI_OSK_LOCKFLAG_UNORDERED)) { ++ remove_lock_from_log(checker, _mali_osk_get_tid()); ++ } ++#endif ++ checker->owner = 0; ++} ++ ++ ++#ifdef LOCK_ORDER_CHECKING ++/* Lock order checking ++ * ------------------- ++ * ++ * To assure that lock ordering scheme defined by _mali_osk_lock_order_t is strictly adhered to, the ++ * following function will, together with a linked list and some extra members in _mali_osk_lock_debug_s, ++ * make sure that a lock that is taken has a higher order than the current highest-order lock a ++ * thread holds. ++ * ++ * This is done in the following manner: ++ * - A linked list keeps track of locks held by a thread. ++ * - A `next' pointer is added to each lock. This is used to chain the locks together. ++ * - When taking a lock, the `add_lock_to_log_and_check' makes sure that taking ++ * the given lock is legal. It will follow the linked list to find the last ++ * lock taken by this thread. If the last lock's order was lower than the ++ * lock that is to be taken, it appends the new lock to the list and returns ++ * true, if not, it return false. This return value is assert()'ed on in ++ * _mali_osk_lock_wait(). ++ */ ++ ++static struct _mali_osk_lock_debug_s *lock_lookup_list; ++ ++static void dump_lock_tracking_list(void) ++{ ++ struct _mali_osk_lock_debug_s *l; ++ u32 n = 1; ++ ++ /* print list for debugging purposes */ ++ l = lock_lookup_list; ++ ++ while (NULL != l) { ++ printk(" [lock: %p, tid_owner: %d, order: %d] ->", l, l->owner, l->order); ++ l = l->next; ++ MALI_DEBUG_ASSERT(n++ < 100); ++ } ++ printk(" NULL\n"); ++} ++ ++static int tracking_list_length(void) ++{ ++ struct _mali_osk_lock_debug_s *l; ++ u32 n = 0; ++ l = lock_lookup_list; ++ ++ while (NULL != l) { ++ l = l->next; ++ n++; ++ MALI_DEBUG_ASSERT(n < 100); ++ } ++ return n; ++} ++ ++static mali_bool add_lock_to_log_and_check(struct _mali_osk_lock_debug_s *lock, uint32_t tid) ++{ ++ mali_bool ret = MALI_FALSE; ++ _mali_osk_lock_order_t highest_order_for_tid = _MALI_OSK_LOCK_ORDER_FIRST; ++ struct _mali_osk_lock_debug_s *highest_order_lock = (struct _mali_osk_lock_debug_s *)0xbeefbabe; ++ struct _mali_osk_lock_debug_s *l; ++ unsigned long local_lock_flag; ++ u32 len; ++ ++ spin_lock_irqsave(&lock_tracking_lock, local_lock_flag); ++ len = tracking_list_length(); ++ ++ l = lock_lookup_list; ++ if (NULL == l) { /* This is the first lock taken by this thread -- record and return true */ ++ lock_lookup_list = lock; ++ spin_unlock_irqrestore(&lock_tracking_lock, local_lock_flag); ++ return MALI_TRUE; ++ } else { ++ /* Traverse the locks taken and find the lock of the highest order. ++ * Since several threads may hold locks, each lock's owner must be ++ * checked so that locks not owned by this thread can be ignored. */ ++ for (;;) { ++ MALI_DEBUG_ASSERT_POINTER(l); ++ if (tid == l->owner && l->order >= highest_order_for_tid) { ++ highest_order_for_tid = l->order; ++ highest_order_lock = l; ++ } ++ ++ if (NULL != l->next) { ++ l = l->next; ++ } else { ++ break; ++ } ++ } ++ ++ l->next = lock; ++ l->next = NULL; ++ } ++ ++ /* We have now found the highest order lock currently held by this thread and can see if it is ++ * legal to take the requested lock. */ ++ ret = highest_order_for_tid < lock->order; ++ ++ if (!ret) { ++ printk(KERN_ERR "Took lock of order %d (%s) while holding lock of order %d (%s)\n", ++ lock->order, lock_order_to_string(lock->order), ++ highest_order_for_tid, lock_order_to_string(highest_order_for_tid)); ++ dump_lock_tracking_list(); ++ } ++ ++ if (len + 1 != tracking_list_length()) { ++ printk(KERN_ERR "************ lock: %p\n", lock); ++ printk(KERN_ERR "************ before: %d *** after: %d ****\n", len, tracking_list_length()); ++ dump_lock_tracking_list(); ++ MALI_DEBUG_ASSERT_POINTER(NULL); ++ } ++ ++ spin_unlock_irqrestore(&lock_tracking_lock, local_lock_flag); ++ return ret; ++} ++ ++static void remove_lock_from_log(struct _mali_osk_lock_debug_s *lock, uint32_t tid) ++{ ++ struct _mali_osk_lock_debug_s *curr; ++ struct _mali_osk_lock_debug_s *prev = NULL; ++ unsigned long local_lock_flag; ++ u32 len; ++ u32 n = 0; ++ ++ spin_lock_irqsave(&lock_tracking_lock, local_lock_flag); ++ len = tracking_list_length(); ++ curr = lock_lookup_list; ++ ++ if (NULL == curr) { ++ printk(KERN_ERR "Error: Lock tracking list was empty on call to remove_lock_from_log\n"); ++ dump_lock_tracking_list(); ++ } ++ ++ MALI_DEBUG_ASSERT_POINTER(curr); ++ ++ ++ while (lock != curr) { ++ prev = curr; ++ ++ MALI_DEBUG_ASSERT_POINTER(curr); ++ curr = curr->next; ++ MALI_DEBUG_ASSERT(n++ < 100); ++ } ++ ++ if (NULL == prev) { ++ lock_lookup_list = curr->next; ++ } else { ++ MALI_DEBUG_ASSERT_POINTER(curr); ++ MALI_DEBUG_ASSERT_POINTER(prev); ++ prev->next = curr->next; ++ } ++ ++ lock->next = NULL; ++ ++ if (len - 1 != tracking_list_length()) { ++ printk(KERN_ERR "************ lock: %p\n", lock); ++ printk(KERN_ERR "************ before: %d *** after: %d ****\n", len, tracking_list_length()); ++ dump_lock_tracking_list(); ++ MALI_DEBUG_ASSERT_POINTER(NULL); ++ } ++ ++ spin_unlock_irqrestore(&lock_tracking_lock, local_lock_flag); ++} ++ ++static const char *const lock_order_to_string(_mali_osk_lock_order_t order) ++{ ++ switch (order) { ++ case _MALI_OSK_LOCK_ORDER_SESSIONS: ++ return "_MALI_OSK_LOCK_ORDER_SESSIONS"; ++ break; ++ case _MALI_OSK_LOCK_ORDER_MEM_SESSION: ++ return "_MALI_OSK_LOCK_ORDER_MEM_SESSION"; ++ break; ++ case _MALI_OSK_LOCK_ORDER_MEM_INFO: ++ return "_MALI_OSK_LOCK_ORDER_MEM_INFO"; ++ break; ++ case _MALI_OSK_LOCK_ORDER_MEM_PT_CACHE: ++ return "_MALI_OSK_LOCK_ORDER_MEM_PT_CACHE"; ++ break; ++ case _MALI_OSK_LOCK_ORDER_DESCRIPTOR_MAP: ++ return "_MALI_OSK_LOCK_ORDER_DESCRIPTOR_MAP"; ++ break; ++ case _MALI_OSK_LOCK_ORDER_PM_EXECUTION: ++ return "_MALI_OSK_LOCK_ORDER_PM_EXECUTION"; ++ break; ++ case _MALI_OSK_LOCK_ORDER_EXECUTOR: ++ return "_MALI_OSK_LOCK_ORDER_EXECUTOR"; ++ break; ++ case _MALI_OSK_LOCK_ORDER_TIMELINE_SYSTEM: ++ return "_MALI_OSK_LOCK_ORDER_TIMELINE_SYSTEM"; ++ break; ++ case _MALI_OSK_LOCK_ORDER_SCHEDULER: ++ return "_MALI_OSK_LOCK_ORDER_SCHEDULER"; ++ break; ++ case _MALI_OSK_LOCK_ORDER_SCHEDULER_DEFERRED: ++ return "_MALI_OSK_LOCK_ORDER_SCHEDULER_DEFERRED"; ++ break; ++ case _MALI_OSK_LOCK_ORDER_DMA_COMMAND: ++ return "_MALI_OSK_LOCK_ORDER_DMA_COMMAND"; ++ break; ++ case _MALI_OSK_LOCK_ORDER_PROFILING: ++ return "_MALI_OSK_LOCK_ORDER_PROFILING"; ++ break; ++ case _MALI_OSK_LOCK_ORDER_L2: ++ return "_MALI_OSK_LOCK_ORDER_L2"; ++ break; ++ case _MALI_OSK_LOCK_ORDER_L2_COMMAND: ++ return "_MALI_OSK_LOCK_ORDER_L2_COMMAND"; ++ break; ++ case _MALI_OSK_LOCK_ORDER_UTILIZATION: ++ return "_MALI_OSK_LOCK_ORDER_UTILIZATION"; ++ break; ++ case _MALI_OSK_LOCK_ORDER_SESSION_PENDING_JOBS: ++ return "_MALI_OSK_LOCK_ORDER_SESSION_PENDING_JOBS"; ++ break; ++ case _MALI_OSK_LOCK_ORDER_PM_STATE: ++ return "_MALI_OSK_LOCK_ORDER_PM_STATE"; ++ break; ++ default: ++ return ""; ++ } ++} ++#endif /* LOCK_ORDER_CHECKING */ ++#endif /* DEBUG */ +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_locks.h b/drivers/gpu/arm/mali400/mali/linux/mali_osk_locks.h +new file mode 100755 +index 000000000000..6fd5af95285b +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_locks.h +@@ -0,0 +1,326 @@ ++/* ++ * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file mali_osk_locks.h ++ * Defines OS abstraction of lock and mutex ++ */ ++#ifndef _MALI_OSK_LOCKS_H ++#define _MALI_OSK_LOCKS_H ++ ++#include ++#include ++#include ++ ++#include ++ ++#include "mali_osk_types.h" ++ ++#ifdef _cplusplus ++extern "C" { ++#endif ++ ++ /* When DEBUG is enabled, this struct will be used to track owner, mode and order checking */ ++#ifdef DEBUG ++ struct _mali_osk_lock_debug_s { ++ u32 owner; ++ _mali_osk_lock_flags_t orig_flags; ++ _mali_osk_lock_order_t order; ++ struct _mali_osk_lock_debug_s *next; ++ }; ++#endif ++ ++ /* Anstraction of spinlock_t */ ++ struct _mali_osk_spinlock_s { ++#ifdef DEBUG ++ struct _mali_osk_lock_debug_s checker; ++#endif ++ spinlock_t spinlock; ++ }; ++ ++ /* Abstration of spinlock_t and lock flag which is used to store register's state before locking */ ++ struct _mali_osk_spinlock_irq_s { ++#ifdef DEBUG ++ struct _mali_osk_lock_debug_s checker; ++#endif ++ ++ spinlock_t spinlock; ++ unsigned long flags; ++ }; ++ ++ /* Abstraction of rw_semaphore in OS */ ++ struct _mali_osk_mutex_rw_s { ++#ifdef DEBUG ++ struct _mali_osk_lock_debug_s checker; ++ _mali_osk_lock_mode_t mode; ++#endif ++ ++ struct rw_semaphore rw_sema; ++ }; ++ ++ /* Mutex and mutex_interruptible functions share the same osk mutex struct */ ++ struct _mali_osk_mutex_s { ++#ifdef DEBUG ++ struct _mali_osk_lock_debug_s checker; ++#endif ++ struct mutex mutex; ++ }; ++ ++#ifdef DEBUG ++ /** @brief _mali_osk_locks_debug_init/add/remove() functions are declared when DEBUG is enabled and ++ * defined in file mali_osk_locks.c. When LOCK_ORDER_CHECKING is enabled, calling these functions when we ++ * init/lock/unlock a lock/mutex, we could track lock order of a given tid. */ ++ void _mali_osk_locks_debug_init(struct _mali_osk_lock_debug_s *checker, _mali_osk_lock_flags_t flags, _mali_osk_lock_order_t order); ++ void _mali_osk_locks_debug_add(struct _mali_osk_lock_debug_s *checker); ++ void _mali_osk_locks_debug_remove(struct _mali_osk_lock_debug_s *checker); ++ ++ /** @brief This function can return a given lock's owner when DEBUG is enabled. */ ++ static inline u32 _mali_osk_lock_get_owner(struct _mali_osk_lock_debug_s *lock) ++ { ++ return lock->owner; ++ } ++#else ++#define _mali_osk_locks_debug_init(x, y, z) do {} while (0) ++#define _mali_osk_locks_debug_add(x) do {} while (0) ++#define _mali_osk_locks_debug_remove(x) do {} while (0) ++#endif ++ ++ /** @brief Before use _mali_osk_spin_lock, init function should be used to allocate memory and initial spinlock*/ ++ static inline _mali_osk_spinlock_t *_mali_osk_spinlock_init(_mali_osk_lock_flags_t flags, _mali_osk_lock_order_t order) ++ { ++ _mali_osk_spinlock_t *lock = NULL; ++ ++ lock = kmalloc(sizeof(_mali_osk_spinlock_t), GFP_KERNEL); ++ if (NULL == lock) { ++ return NULL; ++ } ++ spin_lock_init(&lock->spinlock); ++ _mali_osk_locks_debug_init((struct _mali_osk_lock_debug_s *)lock, flags, order); ++ return lock; ++ } ++ ++ /** @brief Lock a spinlock */ ++ static inline void _mali_osk_spinlock_lock(_mali_osk_spinlock_t *lock) ++ { ++ BUG_ON(NULL == lock); ++ spin_lock(&lock->spinlock); ++ _mali_osk_locks_debug_add((struct _mali_osk_lock_debug_s *)lock); ++ } ++ ++ /** @brief Unlock a spinlock */ ++ static inline void _mali_osk_spinlock_unlock(_mali_osk_spinlock_t *lock) ++ { ++ BUG_ON(NULL == lock); ++ _mali_osk_locks_debug_remove((struct _mali_osk_lock_debug_s *)lock); ++ spin_unlock(&lock->spinlock); ++ } ++ ++ /** @brief Free a memory block which the argument lock pointed to and its type must be ++ * _mali_osk_spinlock_t *. */ ++ static inline void _mali_osk_spinlock_term(_mali_osk_spinlock_t *lock) ++ { ++ /* Parameter validation */ ++ BUG_ON(NULL == lock); ++ ++ /* Linux requires no explicit termination of spinlocks, semaphores, or rw_semaphores */ ++ kfree(lock); ++ } ++ ++ /** @brief Before _mali_osk_spinlock_irq_lock/unlock/term() is called, init function should be ++ * called to initial spinlock and flags in struct _mali_osk_spinlock_irq_t. */ ++ static inline _mali_osk_spinlock_irq_t *_mali_osk_spinlock_irq_init(_mali_osk_lock_flags_t flags, _mali_osk_lock_order_t order) ++ { ++ _mali_osk_spinlock_irq_t *lock = NULL; ++ lock = kmalloc(sizeof(_mali_osk_spinlock_irq_t), GFP_KERNEL); ++ ++ if (NULL == lock) { ++ return NULL; ++ } ++ ++ lock->flags = 0; ++ spin_lock_init(&lock->spinlock); ++ _mali_osk_locks_debug_init((struct _mali_osk_lock_debug_s *)lock, flags, order); ++ return lock; ++ } ++ ++ /** @brief Lock spinlock and save the register's state */ ++ static inline void _mali_osk_spinlock_irq_lock(_mali_osk_spinlock_irq_t *lock) ++ { ++ unsigned long tmp_flags; ++ ++ BUG_ON(NULL == lock); ++ spin_lock_irqsave(&lock->spinlock, tmp_flags); ++ lock->flags = tmp_flags; ++ _mali_osk_locks_debug_add((struct _mali_osk_lock_debug_s *)lock); ++ } ++ ++ /** @brief Unlock spinlock with saved register's state */ ++ static inline void _mali_osk_spinlock_irq_unlock(_mali_osk_spinlock_irq_t *lock) ++ { ++ BUG_ON(NULL == lock); ++ _mali_osk_locks_debug_remove((struct _mali_osk_lock_debug_s *)lock); ++ spin_unlock_irqrestore(&lock->spinlock, lock->flags); ++ } ++ ++ /** @brief Destroy a given memory block which lock pointed to, and the lock type must be ++ * _mali_osk_spinlock_irq_t *. */ ++ static inline void _mali_osk_spinlock_irq_term(_mali_osk_spinlock_irq_t *lock) ++ { ++ /* Parameter validation */ ++ BUG_ON(NULL == lock); ++ ++ /* Linux requires no explicit termination of spinlocks, semaphores, or rw_semaphores */ ++ kfree(lock); ++ } ++ ++ /** @brief Before _mali_osk_mutex_rw_wait/signal/term() is called, we should call ++ * _mali_osk_mutex_rw_init() to kmalloc a memory block and initial part of elements in it. */ ++ static inline _mali_osk_mutex_rw_t *_mali_osk_mutex_rw_init(_mali_osk_lock_flags_t flags, _mali_osk_lock_order_t order) ++ { ++ _mali_osk_mutex_rw_t *lock = NULL; ++ ++ lock = kmalloc(sizeof(_mali_osk_mutex_rw_t), GFP_KERNEL); ++ ++ if (NULL == lock) { ++ return NULL; ++ } ++ ++ init_rwsem(&lock->rw_sema); ++ _mali_osk_locks_debug_init((struct _mali_osk_lock_debug_s *)lock, flags, order); ++ return lock; ++ } ++ ++ /** @brief When call _mali_osk_mutex_rw_wait/signal() functions, the second argument mode ++ * should be assigned with value _MALI_OSK_LOCKMODE_RO or _MALI_OSK_LOCKMODE_RW */ ++ static inline void _mali_osk_mutex_rw_wait(_mali_osk_mutex_rw_t *lock, _mali_osk_lock_mode_t mode) ++ { ++ BUG_ON(NULL == lock); ++ BUG_ON(!(_MALI_OSK_LOCKMODE_RO == mode || _MALI_OSK_LOCKMODE_RW == mode)); ++ ++ if (mode == _MALI_OSK_LOCKMODE_RO) { ++ down_read(&lock->rw_sema); ++ } else { ++ down_write(&lock->rw_sema); ++ } ++ ++#ifdef DEBUG ++ if (mode == _MALI_OSK_LOCKMODE_RW) { ++ lock->mode = mode; ++ } else { /* mode == _MALI_OSK_LOCKMODE_RO */ ++ lock->mode = mode; ++ } ++ _mali_osk_locks_debug_add((struct _mali_osk_lock_debug_s *)lock); ++#endif ++ } ++ ++ /** @brief Up lock->rw_sema with up_read/write() accordinf argument mode's value. */ ++ static inline void _mali_osk_mutex_rw_signal(_mali_osk_mutex_rw_t *lock, _mali_osk_lock_mode_t mode) ++ { ++ BUG_ON(NULL == lock); ++ BUG_ON(!(_MALI_OSK_LOCKMODE_RO == mode || _MALI_OSK_LOCKMODE_RW == mode)); ++#ifdef DEBUG ++ /* make sure the thread releasing the lock actually was the owner */ ++ if (mode == _MALI_OSK_LOCKMODE_RW) { ++ _mali_osk_locks_debug_remove((struct _mali_osk_lock_debug_s *)lock); ++ /* This lock now has no owner */ ++ lock->checker.owner = 0; ++ } ++#endif ++ ++ if (mode == _MALI_OSK_LOCKMODE_RO) { ++ up_read(&lock->rw_sema); ++ } else { ++ up_write(&lock->rw_sema); ++ } ++ } ++ ++ /** @brief Free a given memory block which lock pointed to and its type must be ++ * _mali_sok_mutex_rw_t *. */ ++ static inline void _mali_osk_mutex_rw_term(_mali_osk_mutex_rw_t *lock) ++ { ++ /* Parameter validation */ ++ BUG_ON(NULL == lock); ++ ++ /* Linux requires no explicit termination of spinlocks, semaphores, or rw_semaphores */ ++ kfree(lock); ++ } ++ ++ /** @brief Mutex & mutex_interruptible share the same init and term function, because they have the ++ * same osk mutex struct, and the difference between them is which locking function they use */ ++ static inline _mali_osk_mutex_t *_mali_osk_mutex_init(_mali_osk_lock_flags_t flags, _mali_osk_lock_order_t order) ++ { ++ _mali_osk_mutex_t *lock = NULL; ++ ++ lock = kmalloc(sizeof(_mali_osk_mutex_t), GFP_KERNEL); ++ ++ if (NULL == lock) { ++ return NULL; ++ } ++ mutex_init(&lock->mutex); ++ ++ _mali_osk_locks_debug_init((struct _mali_osk_lock_debug_s *)lock, flags, order); ++ return lock; ++ } ++ ++ /** @brief Lock the lock->mutex with mutex_lock_interruptible function */ ++ static inline _mali_osk_errcode_t _mali_osk_mutex_wait_interruptible(_mali_osk_mutex_t *lock) ++ { ++ _mali_osk_errcode_t err = _MALI_OSK_ERR_OK; ++ ++ BUG_ON(NULL == lock); ++ ++ if (mutex_lock_interruptible(&lock->mutex)) { ++ printk(KERN_WARNING "Mali: Can not lock mutex\n"); ++ err = _MALI_OSK_ERR_RESTARTSYSCALL; ++ } ++ ++ _mali_osk_locks_debug_add((struct _mali_osk_lock_debug_s *)lock); ++ return err; ++ } ++ ++ /** @brief Unlock the lock->mutex which is locked with mutex_lock_interruptible() function. */ ++ static inline void _mali_osk_mutex_signal_interruptible(_mali_osk_mutex_t *lock) ++ { ++ BUG_ON(NULL == lock); ++ _mali_osk_locks_debug_remove((struct _mali_osk_lock_debug_s *)lock); ++ mutex_unlock(&lock->mutex); ++ } ++ ++ /** @brief Lock the lock->mutex just with mutex_lock() function which could not be interruptted. */ ++ static inline void _mali_osk_mutex_wait(_mali_osk_mutex_t *lock) ++ { ++ BUG_ON(NULL == lock); ++ mutex_lock(&lock->mutex); ++ _mali_osk_locks_debug_add((struct _mali_osk_lock_debug_s *)lock); ++ } ++ ++ /** @brief Unlock the lock->mutex which is locked with mutex_lock() function. */ ++ static inline void _mali_osk_mutex_signal(_mali_osk_mutex_t *lock) ++ { ++ BUG_ON(NULL == lock); ++ _mali_osk_locks_debug_remove((struct _mali_osk_lock_debug_s *)lock); ++ mutex_unlock(&lock->mutex); ++ } ++ ++ /** @brief Free a given memory block which lock point. */ ++ static inline void _mali_osk_mutex_term(_mali_osk_mutex_t *lock) ++ { ++ /* Parameter validation */ ++ BUG_ON(NULL == lock); ++ ++ /* Linux requires no explicit termination of spinlocks, semaphores, or rw_semaphores */ ++ kfree(lock); ++ } ++ ++#ifdef _cplusplus ++} ++#endif ++ ++#endif +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_low_level_mem.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_low_level_mem.c +new file mode 100755 +index 000000000000..994b04dad745 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_low_level_mem.c +@@ -0,0 +1,146 @@ ++/* ++ * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file mali_osk_low_level_mem.c ++ * Implementation of the OS abstraction layer for the kernel device driver ++ */ ++ ++#include ++#include ++#include ++ ++#include "mali_kernel_common.h" ++#include "mali_osk.h" ++#include "mali_ukk.h" ++ ++void _mali_osk_mem_barrier(void) ++{ ++ mb(); ++} ++ ++void _mali_osk_write_mem_barrier(void) ++{ ++ wmb(); ++} ++ ++mali_io_address _mali_osk_mem_mapioregion(uintptr_t phys, u32 size, const char *description) ++{ ++ return (mali_io_address)ioremap(phys, size); ++} ++ ++void _mali_osk_mem_unmapioregion(uintptr_t phys, u32 size, mali_io_address virt) ++{ ++ iounmap((void *)virt); ++} ++ ++_mali_osk_errcode_t inline _mali_osk_mem_reqregion(uintptr_t phys, u32 size, const char *description) ++{ ++#if MALI_LICENSE_IS_GPL ++ return _MALI_OSK_ERR_OK; /* GPL driver gets the mem region for the resources registered automatically */ ++#else ++ return ((NULL == request_mem_region(phys, size, description)) ? _MALI_OSK_ERR_NOMEM : _MALI_OSK_ERR_OK); ++#endif ++} ++ ++void inline _mali_osk_mem_unreqregion(uintptr_t phys, u32 size) ++{ ++#if !MALI_LICENSE_IS_GPL ++ release_mem_region(phys, size); ++#endif ++} ++ ++void inline _mali_osk_mem_iowrite32_relaxed(volatile mali_io_address addr, u32 offset, u32 val) ++{ ++ __raw_writel(cpu_to_le32(val), ((u8 *)addr) + offset); ++} ++ ++u32 inline _mali_osk_mem_ioread32(volatile mali_io_address addr, u32 offset) ++{ ++ return ioread32(((u8 *)addr) + offset); ++} ++ ++void inline _mali_osk_mem_iowrite32(volatile mali_io_address addr, u32 offset, u32 val) ++{ ++ iowrite32(val, ((u8 *)addr) + offset); ++} ++ ++void _mali_osk_cache_flushall(void) ++{ ++ /** @note Cached memory is not currently supported in this implementation */ ++} ++ ++void _mali_osk_cache_ensure_uncached_range_flushed(void *uncached_mapping, u32 offset, u32 size) ++{ ++ _mali_osk_write_mem_barrier(); ++} ++ ++u32 _mali_osk_mem_write_safe(void __user *dest, const void __user *src, u32 size) ++{ ++#define MALI_MEM_SAFE_COPY_BLOCK_SIZE 4096 ++ u32 retval = 0; ++ void *temp_buf; ++ ++ temp_buf = kmalloc(MALI_MEM_SAFE_COPY_BLOCK_SIZE, GFP_KERNEL); ++ if (NULL != temp_buf) { ++ u32 bytes_left_to_copy = size; ++ u32 i; ++ for (i = 0; i < size; i += MALI_MEM_SAFE_COPY_BLOCK_SIZE) { ++ u32 size_to_copy; ++ u32 size_copied; ++ u32 bytes_left; ++ ++ if (bytes_left_to_copy > MALI_MEM_SAFE_COPY_BLOCK_SIZE) { ++ size_to_copy = MALI_MEM_SAFE_COPY_BLOCK_SIZE; ++ } else { ++ size_to_copy = bytes_left_to_copy; ++ } ++ ++ bytes_left = copy_from_user(temp_buf, ((char *)src) + i, size_to_copy); ++ size_copied = size_to_copy - bytes_left; ++ ++ bytes_left = copy_to_user(((char *)dest) + i, temp_buf, size_copied); ++ size_copied -= bytes_left; ++ ++ bytes_left_to_copy -= size_copied; ++ retval += size_copied; ++ ++ if (size_copied != size_to_copy) { ++ break; /* Early out, we was not able to copy this entire block */ ++ } ++ } ++ ++ kfree(temp_buf); ++ } ++ ++ return retval; ++} ++ ++_mali_osk_errcode_t _mali_ukk_mem_write_safe(_mali_uk_mem_write_safe_s *args) ++{ ++ void __user *src; ++ void __user *dst; ++ struct mali_session_data *session; ++ ++ MALI_DEBUG_ASSERT_POINTER(args); ++ ++ session = (struct mali_session_data *)(uintptr_t)args->ctx; ++ ++ if (NULL == session) { ++ return _MALI_OSK_ERR_INVALID_ARGS; ++ } ++ ++ src = (void __user *)(uintptr_t)args->src; ++ dst = (void __user *)(uintptr_t)args->dest; ++ ++ /* Return number of bytes actually copied */ ++ args->size = _mali_osk_mem_write_safe(dst, src, args->size); ++ return _MALI_OSK_ERR_OK; ++} +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_mali.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_mali.c +new file mode 100755 +index 000000000000..a729d0499869 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_mali.c +@@ -0,0 +1,505 @@ ++/* ++ * Copyright (C) 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++ ++/** ++ * @file mali_osk_mali.c ++ * Implementation of the OS abstraction layer which is specific for the Mali kernel device driver ++ */ ++#include "../platform/rk/custom_log.h" ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "mali_osk_mali.h" ++#include "mali_kernel_common.h" /* MALI_xxx macros */ ++#include "mali_osk.h" /* kernel side OS functions */ ++#include "mali_kernel_linux.h" ++ ++static mali_bool mali_secure_mode_enabled = MALI_FALSE; ++static mali_bool mali_secure_mode_supported = MALI_FALSE; ++ ++/* Function that init the mali gpu secure mode */ ++void (*mali_secure_mode_deinit)(void) = NULL; ++/* Function that reset GPU and enable the mali gpu secure mode */ ++int (*mali_gpu_reset_and_secure_mode_enable)(void) = NULL; ++/* Function that reset GPU and disable the mali gpu secure mode */ ++int (*mali_gpu_reset_and_secure_mode_disable)(void) = NULL; ++ ++ ++#ifdef CONFIG_MALI_DT ++ ++#define MALI_OSK_INVALID_RESOURCE_ADDRESS 0xFFFFFFFF ++ ++/** ++ * Define the max number of resource we could have. ++ */ ++#define MALI_OSK_MAX_RESOURCE_NUMBER 27 ++ ++/** ++ * Define the max number of resource with interrupts, and they are ++ * the first 20 elements in array mali_osk_resource_bank. ++ */ ++#define MALI_OSK_RESOURCE_WITH_IRQ_NUMBER 20 ++ ++/** ++ * pp core start and end location in mali_osk_resource_bank array. ++ */ ++#define MALI_OSK_RESOURCE_PP_LOCATION_START 2 ++#define MALI_OSK_RESOURCE_PP_LOCATION_END 17 ++ ++/** ++ * L2 cache start and end location in mali_osk_resource_bank array. ++ */ ++#define MALI_OSK_RESOURCE_L2_LOCATION_START 20 ++#define MALI_OSK_RESOURCE_l2_LOCATION_END 22 ++ ++/** ++ * DMA unit location. ++ */ ++#define MALI_OSK_RESOURCE_DMA_LOCATION 26 ++ ++static _mali_osk_resource_t mali_osk_resource_bank[MALI_OSK_MAX_RESOURCE_NUMBER] = { ++ /*-------------------------------------------------------*/ ++ /* rk_ext : to use dts_for_mali_ko_befor_r5p0-01rel0. */ ++ /* {.description = "Mali_GP", .base = MALI_OFFSET_GP, .irq_name = "IRQGP",}, */ ++ {.description = "Mali_GP", .base = MALI_OFFSET_GP, .irq_name = "Mali_GP_IRQ",}, ++ /* {.description = "Mali_GP_MMU", .base = MALI_OFFSET_GP_MMU, .irq_name = "IRQGPMMU",}, */ ++ {.description = "Mali_GP_MMU", .base = MALI_OFFSET_GP_MMU, .irq_name = "Mali_GP_MMU_IRQ",}, ++ /* {.description = "Mali_PP0", .base = MALI_OFFSET_PP0, .irq_name = "IRQPP0",}, */ ++ {.description = "Mali_PP0", .base = MALI_OFFSET_PP0, .irq_name = "Mali_PP0_IRQ",}, ++ /* {.description = "Mali_PP0_MMU", .base = MALI_OFFSET_PP0_MMU, .irq_name = "IRQPPMMU0",}, */ ++ {.description = "Mali_PP0_MMU", .base = MALI_OFFSET_PP0_MMU, .irq_name = "Mali_PP0_MMU_IRQ",}, ++ /* {.description = "Mali_PP1", .base = MALI_OFFSET_PP1, .irq_name = "IRQPP1",}, */ ++ {.description = "Mali_PP1", .base = MALI_OFFSET_PP1, .irq_name = "Mali_PP1_IRQ",}, ++ /* {.description = "Mali_PP1_MMU", .base = MALI_OFFSET_PP1_MMU, .irq_name = "IRQPPMMU1",}, */ ++ {.description = "Mali_PP1_MMU", .base = MALI_OFFSET_PP1_MMU, .irq_name = "Mali_PP1_MMU_IRQ",}, ++ ++ {.description = "Mali_PP2", .base = MALI_OFFSET_PP2, .irq_name = "Mali_PP2_IRQ",}, ++ {.description = "Mali_PP2_MMU", .base = MALI_OFFSET_PP2_MMU, .irq_name = "Mali_PP2_MMU_IRQ",}, ++ {.description = "Mali_PP3", .base = MALI_OFFSET_PP3, .irq_name = "Mali_PP3_IRQ",}, ++ {.description = "Mali_PP3_MMU", .base = MALI_OFFSET_PP3_MMU, .irq_name = "Mali_PP3_MMU_IRQ",}, ++ /*-------------------------------------------------------*/ ++ {.description = "Mali_PP4", .base = MALI_OFFSET_PP4, .irq_name = "IRQPP4",}, ++ {.description = "Mali_PP4_MMU", .base = MALI_OFFSET_PP4_MMU, .irq_name = "IRQPPMMU4",}, ++ {.description = "Mali_PP5", .base = MALI_OFFSET_PP5, .irq_name = "IRQPP5",}, ++ {.description = "Mali_PP5_MMU", .base = MALI_OFFSET_PP5_MMU, .irq_name = "IRQPPMMU5",}, ++ {.description = "Mali_PP6", .base = MALI_OFFSET_PP6, .irq_name = "IRQPP6",}, ++ {.description = "Mali_PP6_MMU", .base = MALI_OFFSET_PP6_MMU, .irq_name = "IRQPPMMU6",}, ++ {.description = "Mali_PP7", .base = MALI_OFFSET_PP7, .irq_name = "IRQPP7",}, ++ {.description = "Mali_PP7_MMU", .base = MALI_OFFSET_PP7_MMU, .irq_name = "IRQPPMMU",}, ++ {.description = "Mali_PP_Broadcast", .base = MALI_OFFSET_PP_BCAST, .irq_name = "IRQPP",}, ++ {.description = "Mali_PMU", .base = MALI_OFFSET_PMU, .irq_name = "IRQPMU",}, ++ {.description = "Mali_L2", .base = MALI_OFFSET_L2_RESOURCE0,}, ++ {.description = "Mali_L2", .base = MALI_OFFSET_L2_RESOURCE1,}, ++ {.description = "Mali_L2", .base = MALI_OFFSET_L2_RESOURCE2,}, ++ {.description = "Mali_PP_MMU_Broadcast", .base = MALI_OFFSET_PP_BCAST_MMU,}, ++ {.description = "Mali_Broadcast", .base = MALI_OFFSET_BCAST,}, ++ {.description = "Mali_DLBU", .base = MALI_OFFSET_DLBU,}, ++ {.description = "Mali_DMA", .base = MALI_OFFSET_DMA,}, ++}; ++ ++static int _mali_osk_get_compatible_name(const char **out_string) ++{ ++ struct device_node *node = mali_platform_device->dev.of_node; ++ ++ MALI_DEBUG_ASSERT(NULL != node); ++ ++ return of_property_read_string(node, "compatible", out_string); ++} ++ ++_mali_osk_errcode_t _mali_osk_resource_initialize(void) ++{ ++ mali_bool mali_is_450 = MALI_FALSE, mali_is_470 = MALI_FALSE; ++ int i, pp_core_num = 0, l2_core_num = 0; ++ struct resource *res; ++ const char *compatible_name = NULL; ++ ++ if (0 == _mali_osk_get_compatible_name(&compatible_name)) { ++ if (0 == strncmp(compatible_name, "arm,mali-450", strlen("arm,mali-450"))) { ++ mali_is_450 = MALI_TRUE; ++ MALI_DEBUG_PRINT(2, ("mali-450 device tree detected.")); ++ } else if (0 == strncmp(compatible_name, "arm,mali-470", strlen("arm,mali-470"))) { ++ mali_is_470 = MALI_TRUE; ++ MALI_DEBUG_PRINT(2, ("mali-470 device tree detected.")); ++ } ++ } ++ ++ for (i = 0; i < MALI_OSK_RESOURCE_WITH_IRQ_NUMBER; i++) { ++ res = platform_get_resource_byname(mali_platform_device, IORESOURCE_IRQ, mali_osk_resource_bank[i].irq_name); ++ if (res) { ++ mali_osk_resource_bank[i].irq = res->start; ++ } else { ++ mali_osk_resource_bank[i].base = MALI_OSK_INVALID_RESOURCE_ADDRESS; ++ } ++ } ++ ++ for (i = MALI_OSK_RESOURCE_PP_LOCATION_START; i <= MALI_OSK_RESOURCE_PP_LOCATION_END; i++) { ++ if (MALI_OSK_INVALID_RESOURCE_ADDRESS != mali_osk_resource_bank[i].base) { ++ pp_core_num++; ++ } ++ } ++ ++ /* We have to divide by 2, because we caculate twice for only one pp(pp_core and pp_mmu_core). */ ++ if (0 != pp_core_num % 2) { ++ MALI_DEBUG_PRINT(2, ("The value of pp core number isn't normal.")); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ pp_core_num /= 2; ++ ++ /** ++ * we can caculate the number of l2 cache core according the number of pp core number ++ * and device type(mali400/mali450/mali470). ++ */ ++ l2_core_num = 1; ++ if (mali_is_450) { ++ if (pp_core_num > 4) { ++ l2_core_num = 3; ++ } else if (pp_core_num <= 4) { ++ l2_core_num = 2; ++ } ++ } ++ ++ for (i = MALI_OSK_RESOURCE_l2_LOCATION_END; i > MALI_OSK_RESOURCE_L2_LOCATION_START + l2_core_num - 1; i--) { ++ mali_osk_resource_bank[i].base = MALI_OSK_INVALID_RESOURCE_ADDRESS; ++ } ++ ++ /* If device is not mali-450 type, we have to remove related resource from resource bank. */ ++ if (!(mali_is_450 || mali_is_470)) { ++ for (i = MALI_OSK_RESOURCE_l2_LOCATION_END + 1; i < MALI_OSK_MAX_RESOURCE_NUMBER; i++) { ++ mali_osk_resource_bank[i].base = MALI_OSK_INVALID_RESOURCE_ADDRESS; ++ } ++ } ++ ++ if (mali_is_470) ++ mali_osk_resource_bank[MALI_OSK_RESOURCE_DMA_LOCATION].base = MALI_OSK_INVALID_RESOURCE_ADDRESS; ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++_mali_osk_errcode_t _mali_osk_resource_find(u32 addr, _mali_osk_resource_t *res) ++{ ++ int i; ++ ++ if (NULL == mali_platform_device) { ++ return _MALI_OSK_ERR_ITEM_NOT_FOUND; ++ } ++ ++ /* Traverse all of resources in resources bank to find the matching one. */ ++ for (i = 0; i < MALI_OSK_MAX_RESOURCE_NUMBER; i++) { ++ if (mali_osk_resource_bank[i].base == addr) { ++ if (NULL != res) { ++ res->base = addr + _mali_osk_resource_base_address(); ++ res->description = mali_osk_resource_bank[i].description; ++ res->irq = mali_osk_resource_bank[i].irq; ++ } ++ return _MALI_OSK_ERR_OK; ++ } ++ } ++ ++ return _MALI_OSK_ERR_ITEM_NOT_FOUND; ++} ++ ++uintptr_t _mali_osk_resource_base_address(void) ++{ ++ struct resource *reg_res = NULL; ++ uintptr_t ret = 0; ++ ++ reg_res = platform_get_resource(mali_platform_device, IORESOURCE_MEM, 0); ++ ++ if (NULL != reg_res) { ++ ret = reg_res->start; ++ } ++ ++ return ret; ++} ++ ++void _mali_osk_device_data_pmu_config_get(u16 *domain_config_array, int array_size) ++{ ++ struct device_node *node = mali_platform_device->dev.of_node; ++ struct property *prop; ++ const __be32 *p; ++ int length = 0, i = 0; ++ u32 u; ++ ++ MALI_DEBUG_PRINT(2, ("Get pmu config from device tree configuration.\n")); ++ ++ MALI_DEBUG_ASSERT(NULL != node); ++ ++ if (!of_get_property(node, "pmu_domain_config", &length)) { ++ return; ++ } ++ ++ if (array_size != length / sizeof(u32)) { ++ MALI_PRINT_ERROR(("Wrong pmu domain config in device tree.")); ++ return; ++ } ++ ++ of_property_for_each_u32(node, "pmu_domain_config", prop, p, u) { ++ domain_config_array[i] = (u16)u; ++ i++; ++ } ++ ++ return; ++} ++ ++u32 _mali_osk_get_pmu_switch_delay(void) ++{ ++ struct device_node *node = mali_platform_device->dev.of_node; ++ u32 switch_delay; ++ ++ MALI_DEBUG_ASSERT(NULL != node); ++ ++ if (0 == of_property_read_u32(node, "pmu_switch_delay", &switch_delay)) { ++ return switch_delay; ++ } else { ++ MALI_DEBUG_PRINT(2, ("Couldn't find pmu_switch_delay in device tree configuration.\n")); ++ } ++ ++ return 0; ++} ++ ++#else /* CONFIG_MALI_DT */ /* 若未 定义 CONFIG_MALI_DT. */ ++ ++_mali_osk_errcode_t _mali_osk_resource_find(u32 addr, _mali_osk_resource_t *res) ++{ ++ int i; ++ uintptr_t phys_addr; ++ ++ if (NULL == mali_platform_device) { ++ /* Not connected to a device */ ++ return _MALI_OSK_ERR_ITEM_NOT_FOUND; ++ } ++ ++ phys_addr = addr + _mali_osk_resource_base_address(); ++ for (i = 0; i < mali_platform_device->num_resources; i++) { ++ if (IORESOURCE_MEM == resource_type(&(mali_platform_device->resource[i])) && ++ mali_platform_device->resource[i].start == phys_addr) { ++ if (NULL != res) { ++ res->base = phys_addr; ++ res->description = mali_platform_device->resource[i].name; ++ ++ /* Any (optional) IRQ resource belonging to this resource will follow */ ++ if ((i + 1) < mali_platform_device->num_resources && ++ IORESOURCE_IRQ == resource_type(&(mali_platform_device->resource[i + 1]))) { ++ res->irq = mali_platform_device->resource[i + 1].start; ++ } else { ++ res->irq = -1; ++ } ++ } ++ return _MALI_OSK_ERR_OK; ++ } ++ } ++ ++ return _MALI_OSK_ERR_ITEM_NOT_FOUND; ++} ++ ++uintptr_t _mali_osk_resource_base_address(void) ++{ ++ uintptr_t lowest_addr = (uintptr_t)(0 - 1); ++ uintptr_t ret = 0; ++ ++ if (NULL != mali_platform_device) { ++ int i; ++ for (i = 0; i < mali_platform_device->num_resources; i++) { ++ if (mali_platform_device->resource[i].flags & IORESOURCE_MEM && ++ mali_platform_device->resource[i].start < lowest_addr) { ++ lowest_addr = mali_platform_device->resource[i].start; ++ ret = lowest_addr; ++ } ++ } ++ } ++ ++ return ret; ++} ++ ++void _mali_osk_device_data_pmu_config_get(u16 *domain_config_array, int array_size) ++{ ++ _mali_osk_device_data data = { 0, }; ++ ++ MALI_DEBUG_PRINT(2, ("Get pmu config from platform device data.\n")); ++ if (_MALI_OSK_ERR_OK == _mali_osk_device_data_get(&data)) { ++ /* Copy the custom customer power domain config */ ++ _mali_osk_memcpy(domain_config_array, data.pmu_domain_config, sizeof(data.pmu_domain_config)); ++ } ++ ++ return; ++} ++ ++u32 _mali_osk_get_pmu_switch_delay(void) ++{ ++ _mali_osk_errcode_t err; ++ _mali_osk_device_data data = { 0, }; ++ ++ err = _mali_osk_device_data_get(&data); ++ ++ if (_MALI_OSK_ERR_OK == err) { ++ return data.pmu_switch_delay; ++ } ++ ++ return 0; ++} ++#endif /* CONFIG_MALI_DT */ ++ ++_mali_osk_errcode_t _mali_osk_device_data_get(_mali_osk_device_data *data) ++{ ++ MALI_DEBUG_ASSERT_POINTER(data); ++ ++ if (NULL != mali_platform_device) { ++ struct mali_gpu_device_data *os_data = NULL; ++ ++ os_data = (struct mali_gpu_device_data *)mali_platform_device->dev.platform_data; ++ if (NULL != os_data) { ++ /* Copy data from OS dependant struct to Mali neutral struct (identical!) */ ++ BUILD_BUG_ON(sizeof(*os_data) != sizeof(*data)); ++ _mali_osk_memcpy(data, os_data, sizeof(*os_data)); ++ ++ return _MALI_OSK_ERR_OK; ++ } ++ } ++ ++ return _MALI_OSK_ERR_ITEM_NOT_FOUND; ++} ++ ++u32 _mali_osk_identify_gpu_resource(void) ++{ ++ if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(MALI_OFFSET_L2_RESOURCE1, NULL)) ++ /* Mali 450 */ ++ return 0x450; ++ ++ if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(MALI_OFFSET_DLBU, NULL)) ++ /* Mali 470 */ ++ return 0x470; ++ ++ /* Mali 400 */ ++ return 0x400; ++} ++ ++mali_bool _mali_osk_shared_interrupts(void) ++{ ++ u32 irqs[128]; ++ u32 i, j, irq, num_irqs_found = 0; ++ ++ MALI_DEBUG_ASSERT_POINTER(mali_platform_device); ++ MALI_DEBUG_ASSERT(128 >= mali_platform_device->num_resources); ++ ++ for (i = 0; i < mali_platform_device->num_resources; i++) { ++ if (IORESOURCE_IRQ & mali_platform_device->resource[i].flags) { ++ irq = mali_platform_device->resource[i].start; ++ ++ for (j = 0; j < num_irqs_found; ++j) { ++ if (irq == irqs[j]) { ++ return MALI_TRUE; ++ } ++ } ++ ++ irqs[num_irqs_found++] = irq; ++ } ++ } ++ ++ return MALI_FALSE; ++} ++ ++_mali_osk_errcode_t _mali_osk_gpu_secure_mode_init(void) ++{ ++ _mali_osk_device_data data = { 0, }; ++ ++ if (_MALI_OSK_ERR_OK == _mali_osk_device_data_get(&data)) { ++ if ((NULL != data.secure_mode_init) && (NULL != data.secure_mode_deinit) ++ && (NULL != data.gpu_reset_and_secure_mode_enable) && (NULL != data.gpu_reset_and_secure_mode_disable)) { ++ int err = data.secure_mode_init(); ++ if (err) { ++ MALI_DEBUG_PRINT(1, ("Failed to init gpu secure mode.\n")); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ mali_secure_mode_deinit = data.secure_mode_deinit; ++ mali_gpu_reset_and_secure_mode_enable = data.gpu_reset_and_secure_mode_enable; ++ mali_gpu_reset_and_secure_mode_disable = data.gpu_reset_and_secure_mode_disable; ++ ++ mali_secure_mode_supported = MALI_TRUE; ++ mali_secure_mode_enabled = MALI_FALSE; ++ return _MALI_OSK_ERR_OK; ++ } ++ } ++ MALI_DEBUG_PRINT(3, ("GPU secure mode not supported.\n")); ++ return _MALI_OSK_ERR_UNSUPPORTED; ++ ++} ++ ++_mali_osk_errcode_t _mali_osk_gpu_secure_mode_deinit(void) ++{ ++ if (NULL != mali_secure_mode_deinit) { ++ mali_secure_mode_deinit(); ++ mali_secure_mode_enabled = MALI_FALSE; ++ mali_secure_mode_supported = MALI_FALSE; ++ return _MALI_OSK_ERR_OK; ++ } ++ MALI_DEBUG_PRINT(3, ("GPU secure mode not supported.\n")); ++ return _MALI_OSK_ERR_UNSUPPORTED; ++ ++} ++ ++ ++_mali_osk_errcode_t _mali_osk_gpu_reset_and_secure_mode_enable(void) ++{ ++ /* the mali executor lock must be held before enter this function. */ ++ ++ MALI_DEBUG_ASSERT(MALI_FALSE == mali_secure_mode_enabled); ++ ++ if (NULL != mali_gpu_reset_and_secure_mode_enable) { ++ if (mali_gpu_reset_and_secure_mode_enable()) { ++ MALI_DEBUG_PRINT(1, ("Failed to reset GPU or enable gpu secure mode.\n")); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ mali_secure_mode_enabled = MALI_TRUE; ++ return _MALI_OSK_ERR_OK; ++ } ++ MALI_DEBUG_PRINT(1, ("GPU secure mode not supported.\n")); ++ return _MALI_OSK_ERR_UNSUPPORTED; ++} ++ ++_mali_osk_errcode_t _mali_osk_gpu_reset_and_secure_mode_disable(void) ++{ ++ /* the mali executor lock must be held before enter this function. */ ++ ++ MALI_DEBUG_ASSERT(MALI_TRUE == mali_secure_mode_enabled); ++ ++ if (NULL != mali_gpu_reset_and_secure_mode_disable) { ++ if (mali_gpu_reset_and_secure_mode_disable()) { ++ MALI_DEBUG_PRINT(1, ("Failed to reset GPU or disable gpu secure mode.\n")); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ mali_secure_mode_enabled = MALI_FALSE; ++ ++ return _MALI_OSK_ERR_OK; ++ ++ } ++ MALI_DEBUG_PRINT(1, ("GPU secure mode not supported.\n")); ++ return _MALI_OSK_ERR_UNSUPPORTED; ++ ++} ++ ++mali_bool _mali_osk_gpu_secure_mode_is_enabled(void) ++{ ++ return mali_secure_mode_enabled; ++} ++ ++mali_bool _mali_osk_gpu_secure_mode_is_supported(void) ++{ ++ return mali_secure_mode_supported; ++} ++ ++ +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_math.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_math.c +new file mode 100755 +index 000000000000..0b2d00762771 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_math.c +@@ -0,0 +1,27 @@ ++/* ++ * Copyright (C) 2010, 2013-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file mali_osk_math.c ++ * Implementation of the OS abstraction layer for the kernel device driver ++ */ ++ ++#include "mali_osk.h" ++#include ++ ++u32 _mali_osk_clz(u32 input) ++{ ++ return 32 - fls(input); ++} ++ ++u32 _mali_osk_fls(u32 input) ++{ ++ return fls(input); ++} +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_memory.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_memory.c +new file mode 100755 +index 000000000000..174616b566c4 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_memory.c +@@ -0,0 +1,61 @@ ++/* ++ * Copyright (C) 2010-2011, 2013-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file mali_osk_memory.c ++ * Implementation of the OS abstraction layer for the kernel device driver ++ */ ++ ++#include "mali_osk.h" ++#include ++#include ++ ++void inline *_mali_osk_calloc(u32 n, u32 size) ++{ ++ return kcalloc(n, size, GFP_KERNEL); ++} ++ ++void inline *_mali_osk_malloc(u32 size) ++{ ++ return kmalloc(size, GFP_KERNEL); ++} ++ ++void inline _mali_osk_free(void *ptr) ++{ ++ kfree(ptr); ++} ++ ++void inline *_mali_osk_valloc(u32 size) ++{ ++ return vmalloc(size); ++} ++ ++void inline _mali_osk_vfree(void *ptr) ++{ ++ vfree(ptr); ++} ++ ++void inline *_mali_osk_memcpy(void *dst, const void *src, u32 len) ++{ ++ return memcpy(dst, src, len); ++} ++ ++void inline *_mali_osk_memset(void *s, u32 c, u32 n) ++{ ++ return memset(s, c, n); ++} ++ ++mali_bool _mali_osk_mem_check_allocated(u32 max_allocated) ++{ ++ /* No need to prevent an out-of-memory dialogue appearing on Linux, ++ * so we always return MALI_TRUE. ++ */ ++ return MALI_TRUE; ++} +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_misc.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_misc.c +new file mode 100755 +index 000000000000..9845187f8122 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_misc.c +@@ -0,0 +1,81 @@ ++/* ++ * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file mali_osk_misc.c ++ * Implementation of the OS abstraction layer for the kernel device driver ++ */ ++#include ++#include ++#include ++#include ++#include ++#include ++#include "mali_osk.h" ++ ++#if !defined(CONFIG_MALI_QUIET) ++void _mali_osk_dbgmsg(const char *fmt, ...) ++{ ++ va_list args; ++ va_start(args, fmt); ++ vprintk(fmt, args); ++ va_end(args); ++} ++#endif /* !defined(CONFIG_MALI_QUIET) */ ++ ++u32 _mali_osk_snprintf(char *buf, u32 size, const char *fmt, ...) ++{ ++ int res; ++ va_list args; ++ va_start(args, fmt); ++ ++ res = vscnprintf(buf, (size_t)size, fmt, args); ++ ++ va_end(args); ++ return res; ++} ++ ++void _mali_osk_abort(void) ++{ ++ /* make a simple fault by dereferencing a NULL pointer */ ++ dump_stack(); ++ *(volatile int *)0 = 0; ++} ++ ++void _mali_osk_break(void) ++{ ++ _mali_osk_abort(); ++} ++ ++u32 _mali_osk_get_pid(void) ++{ ++ /* Thread group ID is the process ID on Linux */ ++ return (u32)current->tgid; ++} ++ ++char *_mali_osk_get_comm(void) ++{ ++ return (char *)current->comm; ++} ++ ++ ++u32 _mali_osk_get_tid(void) ++{ ++ /* pid is actually identifying the thread on Linux */ ++ u32 tid = current->pid; ++ ++ /* If the pid is 0 the core was idle. Instead of returning 0 we return a special number ++ * identifying which core we are on. */ ++ if (0 == tid) { ++ tid = -(1 + raw_smp_processor_id()); ++ } ++ ++ return tid; ++} +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_notification.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_notification.c +new file mode 100755 +index 000000000000..a05f8f066964 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_notification.c +@@ -0,0 +1,182 @@ ++/* ++ * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file mali_osk_notification.c ++ * Implementation of the OS abstraction layer for the kernel device driver ++ */ ++ ++#include "mali_osk.h" ++#include "mali_kernel_common.h" ++ ++#include ++#include ++#include ++ ++/** ++ * Declaration of the notification queue object type ++ * Contains a linked list of notification pending delivery to user space. ++ * It also contains a wait queue of exclusive waiters blocked in the ioctl ++ * When a new notification is posted a single thread is resumed. ++ */ ++struct _mali_osk_notification_queue_t_struct { ++ spinlock_t mutex; /**< Mutex protecting the list */ ++ wait_queue_head_t receive_queue; /**< Threads waiting for new entries to the queue */ ++ struct list_head head; /**< List of notifications waiting to be picked up */ ++}; ++ ++typedef struct _mali_osk_notification_wrapper_t_struct { ++ struct list_head list; /**< Internal linked list variable */ ++ _mali_osk_notification_t data; /**< Notification data */ ++} _mali_osk_notification_wrapper_t; ++ ++_mali_osk_notification_queue_t *_mali_osk_notification_queue_init(void) ++{ ++ _mali_osk_notification_queue_t *result; ++ ++ result = (_mali_osk_notification_queue_t *)kmalloc(sizeof(_mali_osk_notification_queue_t), GFP_KERNEL); ++ if (NULL == result) return NULL; ++ ++ spin_lock_init(&result->mutex); ++ init_waitqueue_head(&result->receive_queue); ++ INIT_LIST_HEAD(&result->head); ++ ++ return result; ++} ++ ++_mali_osk_notification_t *_mali_osk_notification_create(u32 type, u32 size) ++{ ++ /* OPT Recycling of notification objects */ ++ _mali_osk_notification_wrapper_t *notification; ++ ++ notification = (_mali_osk_notification_wrapper_t *)kmalloc(sizeof(_mali_osk_notification_wrapper_t) + size, ++ GFP_KERNEL | __GFP_HIGH | __GFP_RETRY_MAYFAIL); ++ if (NULL == notification) { ++ MALI_DEBUG_PRINT(1, ("Failed to create a notification object\n")); ++ return NULL; ++ } ++ ++ /* Init the list */ ++ INIT_LIST_HEAD(¬ification->list); ++ ++ if (0 != size) { ++ notification->data.result_buffer = ((u8 *)notification) + sizeof(_mali_osk_notification_wrapper_t); ++ } else { ++ notification->data.result_buffer = NULL; ++ } ++ ++ /* set up the non-allocating fields */ ++ notification->data.notification_type = type; ++ notification->data.result_buffer_size = size; ++ ++ /* all ok */ ++ return &(notification->data); ++} ++ ++void _mali_osk_notification_delete(_mali_osk_notification_t *object) ++{ ++ _mali_osk_notification_wrapper_t *notification; ++ MALI_DEBUG_ASSERT_POINTER(object); ++ ++ notification = container_of(object, _mali_osk_notification_wrapper_t, data); ++ ++ /* Free the container */ ++ kfree(notification); ++} ++ ++void _mali_osk_notification_queue_term(_mali_osk_notification_queue_t *queue) ++{ ++ _mali_osk_notification_t *result; ++ MALI_DEBUG_ASSERT_POINTER(queue); ++ ++ while (_MALI_OSK_ERR_OK == _mali_osk_notification_queue_dequeue(queue, &result)) { ++ _mali_osk_notification_delete(result); ++ } ++ ++ /* not much to do, just free the memory */ ++ kfree(queue); ++} ++void _mali_osk_notification_queue_send(_mali_osk_notification_queue_t *queue, _mali_osk_notification_t *object) ++{ ++#if defined(MALI_UPPER_HALF_SCHEDULING) ++ unsigned long irq_flags; ++#endif ++ ++ _mali_osk_notification_wrapper_t *notification; ++ MALI_DEBUG_ASSERT_POINTER(queue); ++ MALI_DEBUG_ASSERT_POINTER(object); ++ ++ notification = container_of(object, _mali_osk_notification_wrapper_t, data); ++ ++#if defined(MALI_UPPER_HALF_SCHEDULING) ++ spin_lock_irqsave(&queue->mutex, irq_flags); ++#else ++ spin_lock(&queue->mutex); ++#endif ++ ++ list_add_tail(¬ification->list, &queue->head); ++ ++#if defined(MALI_UPPER_HALF_SCHEDULING) ++ spin_unlock_irqrestore(&queue->mutex, irq_flags); ++#else ++ spin_unlock(&queue->mutex); ++#endif ++ ++ /* and wake up one possible exclusive waiter */ ++ wake_up(&queue->receive_queue); ++} ++ ++_mali_osk_errcode_t _mali_osk_notification_queue_dequeue(_mali_osk_notification_queue_t *queue, _mali_osk_notification_t **result) ++{ ++#if defined(MALI_UPPER_HALF_SCHEDULING) ++ unsigned long irq_flags; ++#endif ++ ++ _mali_osk_errcode_t ret = _MALI_OSK_ERR_ITEM_NOT_FOUND; ++ _mali_osk_notification_wrapper_t *wrapper_object; ++ ++#if defined(MALI_UPPER_HALF_SCHEDULING) ++ spin_lock_irqsave(&queue->mutex, irq_flags); ++#else ++ spin_lock(&queue->mutex); ++#endif ++ ++ if (!list_empty(&queue->head)) { ++ wrapper_object = list_entry(queue->head.next, _mali_osk_notification_wrapper_t, list); ++ *result = &(wrapper_object->data); ++ list_del_init(&wrapper_object->list); ++ ret = _MALI_OSK_ERR_OK; ++ } ++ ++#if defined(MALI_UPPER_HALF_SCHEDULING) ++ spin_unlock_irqrestore(&queue->mutex, irq_flags); ++#else ++ spin_unlock(&queue->mutex); ++#endif ++ ++ return ret; ++} ++ ++_mali_osk_errcode_t _mali_osk_notification_queue_receive(_mali_osk_notification_queue_t *queue, _mali_osk_notification_t **result) ++{ ++ /* check input */ ++ MALI_DEBUG_ASSERT_POINTER(queue); ++ MALI_DEBUG_ASSERT_POINTER(result); ++ ++ /* default result */ ++ *result = NULL; ++ ++ if (wait_event_interruptible(queue->receive_queue, ++ _MALI_OSK_ERR_OK == _mali_osk_notification_queue_dequeue(queue, result))) { ++ return _MALI_OSK_ERR_RESTARTSYSCALL; ++ } ++ ++ return _MALI_OSK_ERR_OK; /* all ok */ ++} +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_pm.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_pm.c +new file mode 100755 +index 000000000000..e28e2eb21fe2 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_pm.c +@@ -0,0 +1,83 @@ ++/** ++ * Copyright (C) 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file mali_osk_pm.c ++ * Implementation of the callback functions from common power management ++ */ ++ ++#include ++ ++#include "mali_kernel_linux.h" ++#ifdef CONFIG_PM_RUNTIME ++#include ++#endif /* CONFIG_PM_RUNTIME */ ++#include ++#include ++#include "mali_osk.h" ++#include "mali_kernel_common.h" ++ ++/* Can NOT run in atomic context */ ++_mali_osk_errcode_t _mali_osk_pm_dev_ref_get_sync(void) ++{ ++#ifdef CONFIG_PM_RUNTIME ++ int err; ++ MALI_DEBUG_ASSERT_POINTER(mali_platform_device); ++ err = pm_runtime_get_sync(&(mali_platform_device->dev)); ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) ++ pm_runtime_mark_last_busy(&(mali_platform_device->dev)); ++#endif ++ if (0 > err) { ++ MALI_PRINT_ERROR(("Mali OSK PM: pm_runtime_get_sync() returned error code %d\n", err)); ++ return _MALI_OSK_ERR_FAULT; ++ } ++#endif ++ return _MALI_OSK_ERR_OK; ++} ++ ++/* Can run in atomic context */ ++_mali_osk_errcode_t _mali_osk_pm_dev_ref_get_async(void) ++{ ++#ifdef CONFIG_PM_RUNTIME ++ int err; ++ MALI_DEBUG_ASSERT_POINTER(mali_platform_device); ++ err = pm_runtime_get(&(mali_platform_device->dev)); ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) ++ pm_runtime_mark_last_busy(&(mali_platform_device->dev)); ++#endif ++ if (0 > err && -EINPROGRESS != err) { ++ MALI_PRINT_ERROR(("Mali OSK PM: pm_runtime_get() returned error code %d\n", err)); ++ return _MALI_OSK_ERR_FAULT; ++ } ++#endif ++ return _MALI_OSK_ERR_OK; ++} ++ ++ ++/* Can run in atomic context */ ++void _mali_osk_pm_dev_ref_put(void) ++{ ++#ifdef CONFIG_PM_RUNTIME ++ MALI_DEBUG_ASSERT_POINTER(mali_platform_device); ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) ++ pm_runtime_mark_last_busy(&(mali_platform_device->dev)); ++ pm_runtime_put_autosuspend(&(mali_platform_device->dev)); ++#else ++ pm_runtime_put(&(mali_platform_device->dev)); ++#endif ++#endif ++} ++ ++void _mali_osk_pm_dev_barrier(void) ++{ ++#ifdef CONFIG_PM_RUNTIME ++ pm_runtime_barrier(&(mali_platform_device->dev)); ++#endif ++} +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_profiling.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_profiling.c +new file mode 100755 +index 000000000000..9e977ea4d0ff +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_profiling.c +@@ -0,0 +1,1282 @@ ++/* ++ * Copyright (C) 2012-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#include "mali_kernel_common.h" ++#include "mali_osk.h" ++#include "mali_ukk.h" ++#include "mali_uk_types.h" ++#include "mali_osk_profiling.h" ++#include "mali_linux_trace.h" ++#include "mali_gp.h" ++#include "mali_pp.h" ++#include "mali_l2_cache.h" ++#include "mali_user_settings_db.h" ++#include "mali_executor.h" ++#include "mali_memory_manager.h" ++ ++#define MALI_PROFILING_STREAM_DATA_DEFAULT_SIZE 100 ++#define MALI_PROFILING_STREAM_HOLD_TIME 1000000 /*1 ms */ ++ ++#define MALI_PROFILING_STREAM_BUFFER_SIZE (1 << 12) ++#define MALI_PROFILING_STREAM_BUFFER_NUM 100 ++ ++/** ++ * Define the mali profiling stream struct. ++ */ ++typedef struct mali_profiling_stream { ++ u8 data[MALI_PROFILING_STREAM_BUFFER_SIZE]; ++ u32 used_size; ++ struct list_head list; ++} mali_profiling_stream; ++ ++typedef struct mali_profiling_stream_list { ++ spinlock_t spin_lock; ++ struct list_head free_list; ++ struct list_head queue_list; ++} mali_profiling_stream_list; ++ ++static const char mali_name[] = "4xx"; ++static const char utgard_setup_version[] = "ANNOTATE_SETUP 1\n"; ++ ++static u32 profiling_sample_rate = 0; ++static u32 first_sw_counter_index = 0; ++ ++static mali_bool l2_cache_counter_if_enabled = MALI_FALSE; ++static u32 num_counters_enabled = 0; ++static u32 mem_counters_enabled = 0; ++ ++static _mali_osk_atomic_t stream_fd_if_used; ++ ++static wait_queue_head_t stream_fd_wait_queue; ++static mali_profiling_counter *global_mali_profiling_counters = NULL; ++static u32 num_global_mali_profiling_counters = 0; ++ ++static mali_profiling_stream_list *global_mali_stream_list = NULL; ++static mali_profiling_stream *mali_counter_stream = NULL; ++static mali_profiling_stream *mali_core_activity_stream = NULL; ++static u64 mali_core_activity_stream_dequeue_time = 0; ++static spinlock_t mali_activity_lock; ++static u32 mali_activity_cores_num = 0; ++static struct hrtimer profiling_sampling_timer; ++ ++const char *_mali_mem_counter_descriptions[] = _MALI_MEM_COUTNER_DESCRIPTIONS; ++const char *_mali_special_counter_descriptions[] = _MALI_SPCIAL_COUNTER_DESCRIPTIONS; ++ ++static u32 current_profiling_pid = 0; ++ ++static void _mali_profiling_stream_list_destory(mali_profiling_stream_list *profiling_stream_list) ++{ ++ mali_profiling_stream *profiling_stream, *tmp_profiling_stream; ++ MALI_DEBUG_ASSERT_POINTER(profiling_stream_list); ++ ++ list_for_each_entry_safe(profiling_stream, tmp_profiling_stream, &profiling_stream_list->free_list, list) { ++ list_del(&profiling_stream->list); ++ kfree(profiling_stream); ++ } ++ ++ list_for_each_entry_safe(profiling_stream, tmp_profiling_stream, &profiling_stream_list->queue_list, list) { ++ list_del(&profiling_stream->list); ++ kfree(profiling_stream); ++ } ++ ++ kfree(profiling_stream_list); ++} ++ ++static void _mali_profiling_global_stream_list_free(void) ++{ ++ mali_profiling_stream *profiling_stream, *tmp_profiling_stream; ++ unsigned long irq_flags; ++ ++ MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list); ++ spin_lock_irqsave(&global_mali_stream_list->spin_lock, irq_flags); ++ list_for_each_entry_safe(profiling_stream, tmp_profiling_stream, &global_mali_stream_list->queue_list, list) { ++ profiling_stream->used_size = 0; ++ list_move(&profiling_stream->list, &global_mali_stream_list->free_list); ++ } ++ spin_unlock_irqrestore(&global_mali_stream_list->spin_lock, irq_flags); ++} ++ ++static _mali_osk_errcode_t _mali_profiling_global_stream_list_dequeue(struct list_head *stream_list, mali_profiling_stream **new_mali_profiling_stream) ++{ ++ unsigned long irq_flags; ++ _mali_osk_errcode_t ret = _MALI_OSK_ERR_OK; ++ MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list); ++ MALI_DEBUG_ASSERT_POINTER(stream_list); ++ ++ spin_lock_irqsave(&global_mali_stream_list->spin_lock, irq_flags); ++ ++ if (!list_empty(stream_list)) { ++ *new_mali_profiling_stream = list_entry(stream_list->next, mali_profiling_stream, list); ++ list_del_init(&(*new_mali_profiling_stream)->list); ++ } else { ++ ret = _MALI_OSK_ERR_NOMEM; ++ } ++ ++ spin_unlock_irqrestore(&global_mali_stream_list->spin_lock, irq_flags); ++ ++ return ret; ++} ++ ++static void _mali_profiling_global_stream_list_queue(struct list_head *stream_list, mali_profiling_stream *current_mali_profiling_stream) ++{ ++ unsigned long irq_flags; ++ MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list); ++ MALI_DEBUG_ASSERT_POINTER(stream_list); ++ ++ spin_lock_irqsave(&global_mali_stream_list->spin_lock, irq_flags); ++ list_add_tail(¤t_mali_profiling_stream->list, stream_list); ++ spin_unlock_irqrestore(&global_mali_stream_list->spin_lock, irq_flags); ++} ++ ++static mali_bool _mali_profiling_global_stream_queue_list_if_empty(void) ++{ ++ MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list); ++ return list_empty(&global_mali_stream_list->queue_list); ++} ++ ++static u32 _mali_profiling_global_stream_queue_list_next_size(void) ++{ ++ unsigned long irq_flags; ++ u32 size = 0; ++ MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list); ++ ++ spin_lock_irqsave(&global_mali_stream_list->spin_lock, irq_flags); ++ if (!list_empty(&global_mali_stream_list->queue_list)) { ++ mali_profiling_stream *next_mali_profiling_stream = ++ list_entry(global_mali_stream_list->queue_list.next, mali_profiling_stream, list); ++ size = next_mali_profiling_stream->used_size; ++ } ++ spin_unlock_irqrestore(&global_mali_stream_list->spin_lock, irq_flags); ++ return size; ++} ++ ++/* The mali profiling stream file operations functions. */ ++static ssize_t _mali_profiling_stream_read( ++ struct file *filp, ++ char __user *buffer, ++ size_t size, ++ loff_t *f_pos); ++ ++static unsigned int _mali_profiling_stream_poll(struct file *filp, poll_table *wait); ++ ++static int _mali_profiling_stream_release(struct inode *inode, struct file *filp); ++ ++/* The timeline stream file operations structure. */ ++static const struct file_operations mali_profiling_stream_fops = { ++ .release = _mali_profiling_stream_release, ++ .read = _mali_profiling_stream_read, ++ .poll = _mali_profiling_stream_poll, ++}; ++ ++static ssize_t _mali_profiling_stream_read( ++ struct file *filp, ++ char __user *buffer, ++ size_t size, ++ loff_t *f_pos) ++{ ++ u32 copy_len = 0; ++ mali_profiling_stream *current_mali_profiling_stream; ++ u32 used_size; ++ MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list); ++ ++ while (!_mali_profiling_global_stream_queue_list_if_empty()) { ++ used_size = _mali_profiling_global_stream_queue_list_next_size(); ++ if (used_size <= ((u32)size - copy_len)) { ++ current_mali_profiling_stream = NULL; ++ _mali_profiling_global_stream_list_dequeue(&global_mali_stream_list->queue_list, ++ ¤t_mali_profiling_stream); ++ MALI_DEBUG_ASSERT_POINTER(current_mali_profiling_stream); ++ if (copy_to_user(&buffer[copy_len], current_mali_profiling_stream->data, current_mali_profiling_stream->used_size)) { ++ current_mali_profiling_stream->used_size = 0; ++ _mali_profiling_global_stream_list_queue(&global_mali_stream_list->free_list, current_mali_profiling_stream); ++ return -EFAULT; ++ } ++ copy_len += current_mali_profiling_stream->used_size; ++ current_mali_profiling_stream->used_size = 0; ++ _mali_profiling_global_stream_list_queue(&global_mali_stream_list->free_list, current_mali_profiling_stream); ++ } else { ++ break; ++ } ++ } ++ return (ssize_t)copy_len; ++} ++ ++static unsigned int _mali_profiling_stream_poll(struct file *filp, poll_table *wait) ++{ ++ poll_wait(filp, &stream_fd_wait_queue, wait); ++ if (!_mali_profiling_global_stream_queue_list_if_empty()) ++ return POLLIN; ++ return 0; ++} ++ ++static int _mali_profiling_stream_release(struct inode *inode, struct file *filp) ++{ ++ _mali_osk_atomic_init(&stream_fd_if_used, 0); ++ return 0; ++} ++ ++/* The funs for control packet and stream data.*/ ++static void _mali_profiling_set_packet_size(unsigned char *const buf, const u32 size) ++{ ++ u32 i; ++ ++ for (i = 0; i < sizeof(size); ++i) ++ buf[i] = (size >> 8 * i) & 0xFF; ++} ++ ++static u32 _mali_profiling_get_packet_size(unsigned char *const buf) ++{ ++ u32 i; ++ u32 size = 0; ++ for (i = 0; i < sizeof(size); ++i) ++ size |= (u32)buf[i] << 8 * i; ++ return size; ++} ++ ++static u32 _mali_profiling_read_packet_int(unsigned char *const buf, u32 *const pos, u32 const packet_size) ++{ ++ u64 int_value = 0; ++ u8 shift = 0; ++ u8 byte_value = ~0; ++ ++ while ((byte_value & 0x80) != 0) { ++ if ((*pos) >= packet_size) ++ return -1; ++ byte_value = buf[*pos]; ++ *pos += 1; ++ int_value |= (u32)(byte_value & 0x7f) << shift; ++ shift += 7; ++ } ++ ++ if (shift < 8 * sizeof(int_value) && (byte_value & 0x40) != 0) { ++ int_value |= -(1 << shift); ++ } ++ ++ return int_value; ++} ++ ++static u32 _mali_profiling_pack_int(u8 *const buf, u32 const buf_size, u32 const pos, s32 value) ++{ ++ u32 add_bytes = 0; ++ int more = 1; ++ while (more) { ++ /* low order 7 bits of val */ ++ char byte_value = value & 0x7f; ++ value >>= 7; ++ ++ if ((value == 0 && (byte_value & 0x40) == 0) || (value == -1 && (byte_value & 0x40) != 0)) { ++ more = 0; ++ } else { ++ byte_value |= 0x80; ++ } ++ ++ if ((pos + add_bytes) >= buf_size) ++ return 0; ++ buf[pos + add_bytes] = byte_value; ++ add_bytes++; ++ } ++ ++ return add_bytes; ++} ++ ++static int _mali_profiling_pack_long(uint8_t *const buf, u32 const buf_size, u32 const pos, s64 val) ++{ ++ int add_bytes = 0; ++ int more = 1; ++ while (more) { ++ /* low order 7 bits of x */ ++ char byte_value = val & 0x7f; ++ val >>= 7; ++ ++ if ((val == 0 && (byte_value & 0x40) == 0) || (val == -1 && (byte_value & 0x40) != 0)) { ++ more = 0; ++ } else { ++ byte_value |= 0x80; ++ } ++ ++ MALI_DEBUG_ASSERT((pos + add_bytes) < buf_size); ++ buf[pos + add_bytes] = byte_value; ++ add_bytes++; ++ } ++ ++ return add_bytes; ++} ++ ++static void _mali_profiling_stream_add_counter(mali_profiling_stream *profiling_stream, s64 current_time, u32 key, u32 counter_value) ++{ ++ u32 add_size = STREAM_HEADER_SIZE; ++ MALI_DEBUG_ASSERT_POINTER(profiling_stream); ++ MALI_DEBUG_ASSERT((profiling_stream->used_size) < MALI_PROFILING_STREAM_BUFFER_SIZE); ++ ++ profiling_stream->data[profiling_stream->used_size] = STREAM_HEADER_COUNTER_VALUE; ++ ++ add_size += _mali_profiling_pack_long(profiling_stream->data, MALI_PROFILING_STREAM_BUFFER_SIZE, ++ profiling_stream->used_size + add_size, current_time); ++ add_size += _mali_profiling_pack_int(profiling_stream->data, MALI_PROFILING_STREAM_BUFFER_SIZE, ++ profiling_stream->used_size + add_size, (s32)0); ++ add_size += _mali_profiling_pack_int(profiling_stream->data, MALI_PROFILING_STREAM_BUFFER_SIZE, ++ profiling_stream->used_size + add_size, (s32)key); ++ add_size += _mali_profiling_pack_int(profiling_stream->data, MALI_PROFILING_STREAM_BUFFER_SIZE, ++ profiling_stream->used_size + add_size, (s32)counter_value); ++ ++ _mali_profiling_set_packet_size(profiling_stream->data + profiling_stream->used_size + 1, ++ add_size - STREAM_HEADER_SIZE); ++ ++ profiling_stream->used_size += add_size; ++} ++ ++/* The callback function for sampling timer.*/ ++static enum hrtimer_restart _mali_profiling_sampling_counters(struct hrtimer *timer) ++{ ++ u32 counter_index; ++ s64 current_time; ++ MALI_DEBUG_ASSERT_POINTER(global_mali_profiling_counters); ++ MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list); ++ ++ MALI_DEBUG_ASSERT(NULL == mali_counter_stream); ++ if (_MALI_OSK_ERR_OK == _mali_profiling_global_stream_list_dequeue( ++ &global_mali_stream_list->free_list, &mali_counter_stream)) { ++ ++ MALI_DEBUG_ASSERT_POINTER(mali_counter_stream); ++ MALI_DEBUG_ASSERT(0 == mali_counter_stream->used_size); ++ ++ /* Capture l2 cache counter values if enabled */ ++ if (MALI_TRUE == l2_cache_counter_if_enabled) { ++ int i, j = 0; ++ _mali_profiling_l2_counter_values l2_counters_values; ++ _mali_profiling_get_l2_counters(&l2_counters_values); ++ ++ for (i = COUNTER_L2_0_C0; i <= COUNTER_L2_2_C1; i++) { ++ if (0 == (j % 2)) ++ _mali_osk_profiling_record_global_counters(i, l2_counters_values.cores[j / 2].value0); ++ else ++ _mali_osk_profiling_record_global_counters(i, l2_counters_values.cores[j / 2].value1); ++ j++; ++ } ++ } ++ ++ current_time = (s64)_mali_osk_boot_time_get_ns(); ++ ++ /* Add all enabled counter values into stream */ ++ for (counter_index = 0; counter_index < num_global_mali_profiling_counters; counter_index++) { ++ /* No need to sample these couners here. */ ++ if (global_mali_profiling_counters[counter_index].enabled) { ++ if ((global_mali_profiling_counters[counter_index].counter_id >= FIRST_MEM_COUNTER && ++ global_mali_profiling_counters[counter_index].counter_id <= LAST_MEM_COUNTER) ++ || (global_mali_profiling_counters[counter_index].counter_id == COUNTER_VP_ACTIVITY) ++ || (global_mali_profiling_counters[counter_index].counter_id == COUNTER_FP_ACTIVITY) ++ || (global_mali_profiling_counters[counter_index].counter_id == COUNTER_FILMSTRIP)) { ++ ++ continue; ++ } ++ ++ if (global_mali_profiling_counters[counter_index].counter_id >= COUNTER_L2_0_C0 && ++ global_mali_profiling_counters[counter_index].counter_id <= COUNTER_L2_2_C1) { ++ ++ u32 prev_val = global_mali_profiling_counters[counter_index].prev_counter_value; ++ ++ _mali_profiling_stream_add_counter(mali_counter_stream, current_time, global_mali_profiling_counters[counter_index].key, ++ global_mali_profiling_counters[counter_index].current_counter_value - prev_val); ++ ++ prev_val = global_mali_profiling_counters[counter_index].current_counter_value; ++ ++ global_mali_profiling_counters[counter_index].prev_counter_value = prev_val; ++ } else { ++ ++ if (global_mali_profiling_counters[counter_index].counter_id == COUNTER_TOTAL_ALLOC_PAGES) { ++ u32 total_alloc_mem = _mali_ukk_report_memory_usage(); ++ global_mali_profiling_counters[counter_index].current_counter_value = total_alloc_mem / _MALI_OSK_MALI_PAGE_SIZE; ++ } ++ _mali_profiling_stream_add_counter(mali_counter_stream, current_time, global_mali_profiling_counters[counter_index].key, ++ global_mali_profiling_counters[counter_index].current_counter_value); ++ if (global_mali_profiling_counters[counter_index].counter_id < FIRST_SPECIAL_COUNTER) ++ global_mali_profiling_counters[counter_index].current_counter_value = 0; ++ } ++ } ++ } ++ _mali_profiling_global_stream_list_queue(&global_mali_stream_list->queue_list, mali_counter_stream); ++ mali_counter_stream = NULL; ++ } else { ++ MALI_DEBUG_PRINT(1, ("Not enough mali profiling stream buffer!\n")); ++ } ++ ++ wake_up_interruptible(&stream_fd_wait_queue); ++ ++ /*Enable the sampling timer again*/ ++ if (0 != num_counters_enabled && 0 != profiling_sample_rate) { ++ hrtimer_forward_now(&profiling_sampling_timer, ns_to_ktime(profiling_sample_rate)); ++ return HRTIMER_RESTART; ++ } ++ return HRTIMER_NORESTART; ++} ++ ++static void _mali_profiling_sampling_core_activity_switch(int counter_id, int core, u32 activity, u32 pid) ++{ ++ unsigned long irq_flags; ++ ++ spin_lock_irqsave(&mali_activity_lock, irq_flags); ++ if (activity == 0) ++ mali_activity_cores_num--; ++ else ++ mali_activity_cores_num++; ++ spin_unlock_irqrestore(&mali_activity_lock, irq_flags); ++ ++ if (NULL != global_mali_profiling_counters) { ++ int i ; ++ for (i = 0; i < num_global_mali_profiling_counters; i++) { ++ if (counter_id == global_mali_profiling_counters[i].counter_id && global_mali_profiling_counters[i].enabled) { ++ u64 current_time = _mali_osk_boot_time_get_ns(); ++ u32 add_size = STREAM_HEADER_SIZE; ++ ++ if (NULL != mali_core_activity_stream) { ++ if ((mali_core_activity_stream_dequeue_time + MALI_PROFILING_STREAM_HOLD_TIME < current_time) || ++ (MALI_PROFILING_STREAM_DATA_DEFAULT_SIZE > MALI_PROFILING_STREAM_BUFFER_SIZE ++ - mali_core_activity_stream->used_size)) { ++ _mali_profiling_global_stream_list_queue(&global_mali_stream_list->queue_list, mali_core_activity_stream); ++ mali_core_activity_stream = NULL; ++ wake_up_interruptible(&stream_fd_wait_queue); ++ } ++ } ++ ++ if (NULL == mali_core_activity_stream) { ++ if (_MALI_OSK_ERR_OK == _mali_profiling_global_stream_list_dequeue( ++ &global_mali_stream_list->free_list, &mali_core_activity_stream)) { ++ mali_core_activity_stream_dequeue_time = current_time; ++ } else { ++ MALI_DEBUG_PRINT(1, ("Not enough mali profiling stream buffer!\n")); ++ wake_up_interruptible(&stream_fd_wait_queue); ++ break; ++ } ++ ++ } ++ ++ mali_core_activity_stream->data[mali_core_activity_stream->used_size] = STREAM_HEADER_CORE_ACTIVITY; ++ ++ add_size += _mali_profiling_pack_long(mali_core_activity_stream->data, ++ MALI_PROFILING_STREAM_BUFFER_SIZE, mali_core_activity_stream->used_size + add_size, (s64)current_time); ++ add_size += _mali_profiling_pack_int(mali_core_activity_stream->data, ++ MALI_PROFILING_STREAM_BUFFER_SIZE, mali_core_activity_stream->used_size + add_size, core); ++ add_size += _mali_profiling_pack_int(mali_core_activity_stream->data, ++ MALI_PROFILING_STREAM_BUFFER_SIZE, mali_core_activity_stream->used_size + add_size, (s32)global_mali_profiling_counters[i].key); ++ add_size += _mali_profiling_pack_int(mali_core_activity_stream->data, ++ MALI_PROFILING_STREAM_BUFFER_SIZE, mali_core_activity_stream->used_size + add_size, activity); ++ add_size += _mali_profiling_pack_int(mali_core_activity_stream->data, ++ MALI_PROFILING_STREAM_BUFFER_SIZE, mali_core_activity_stream->used_size + add_size, pid); ++ ++ _mali_profiling_set_packet_size(mali_core_activity_stream->data + mali_core_activity_stream->used_size + 1, ++ add_size - STREAM_HEADER_SIZE); ++ ++ mali_core_activity_stream->used_size += add_size; ++ ++ if (0 == mali_activity_cores_num) { ++ _mali_profiling_global_stream_list_queue(&global_mali_stream_list->queue_list, mali_core_activity_stream); ++ mali_core_activity_stream = NULL; ++ wake_up_interruptible(&stream_fd_wait_queue); ++ } ++ ++ break; ++ } ++ } ++ } ++} ++ ++static mali_bool _mali_profiling_global_counters_init(void) ++{ ++ int core_id, counter_index, counter_number, counter_id; ++ u32 num_l2_cache_cores; ++ u32 num_pp_cores; ++ u32 num_gp_cores = 1; ++ ++ MALI_DEBUG_ASSERT(NULL == global_mali_profiling_counters); ++ num_pp_cores = mali_pp_get_glob_num_pp_cores(); ++ num_l2_cache_cores = mali_l2_cache_core_get_glob_num_l2_cores(); ++ ++ num_global_mali_profiling_counters = 3 * (num_gp_cores + num_pp_cores) + 2 * num_l2_cache_cores ++ + MALI_PROFILING_SW_COUNTERS_NUM ++ + MALI_PROFILING_SPECIAL_COUNTERS_NUM ++ + MALI_PROFILING_MEM_COUNTERS_NUM; ++ global_mali_profiling_counters = _mali_osk_calloc(num_global_mali_profiling_counters, sizeof(mali_profiling_counter)); ++ ++ if (NULL == global_mali_profiling_counters) ++ return MALI_FALSE; ++ ++ counter_index = 0; ++ /*Vertex processor counters */ ++ for (core_id = 0; core_id < num_gp_cores; core_id ++) { ++ global_mali_profiling_counters[counter_index].counter_id = ACTIVITY_VP_0 + core_id; ++ _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name, ++ sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_VP_%d_active", mali_name, core_id); ++ ++ for (counter_number = 0; counter_number < 2; counter_number++) { ++ counter_index++; ++ global_mali_profiling_counters[counter_index].counter_id = COUNTER_VP_0_C0 + (2 * core_id) + counter_number; ++ _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name, ++ sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_VP_%d_cnt%d", mali_name, core_id, counter_number); ++ } ++ } ++ ++ /* Fragment processors' counters */ ++ for (core_id = 0; core_id < num_pp_cores; core_id++) { ++ counter_index++; ++ global_mali_profiling_counters[counter_index].counter_id = ACTIVITY_FP_0 + core_id; ++ _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name, ++ sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_FP_%d_active", mali_name, core_id); ++ ++ for (counter_number = 0; counter_number < 2; counter_number++) { ++ counter_index++; ++ global_mali_profiling_counters[counter_index].counter_id = COUNTER_FP_0_C0 + (2 * core_id) + counter_number; ++ _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name, ++ sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_FP_%d_cnt%d", mali_name, core_id, counter_number); ++ } ++ } ++ ++ /* L2 Cache counters */ ++ for (core_id = 0; core_id < num_l2_cache_cores; core_id++) { ++ for (counter_number = 0; counter_number < 2; counter_number++) { ++ counter_index++; ++ global_mali_profiling_counters[counter_index].counter_id = COUNTER_L2_0_C0 + (2 * core_id) + counter_number; ++ _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name, ++ sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_L2_%d_cnt%d", mali_name, core_id, counter_number); ++ } ++ } ++ ++ /* Now set up the software counter entries */ ++ for (counter_id = FIRST_SW_COUNTER; counter_id <= LAST_SW_COUNTER; counter_id++) { ++ counter_index++; ++ ++ if (0 == first_sw_counter_index) ++ first_sw_counter_index = counter_index; ++ ++ global_mali_profiling_counters[counter_index].counter_id = counter_id; ++ _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name, ++ sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_SW_%d", mali_name, counter_id - FIRST_SW_COUNTER); ++ } ++ ++ /* Now set up the special counter entries */ ++ for (counter_id = FIRST_SPECIAL_COUNTER; counter_id <= LAST_SPECIAL_COUNTER; counter_id++) { ++ ++ counter_index++; ++ _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name, ++ sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_%s", ++ mali_name, _mali_special_counter_descriptions[counter_id - FIRST_SPECIAL_COUNTER]); ++ ++ global_mali_profiling_counters[counter_index].counter_id = counter_id; ++ } ++ ++ /* Now set up the mem counter entries*/ ++ for (counter_id = FIRST_MEM_COUNTER; counter_id <= LAST_MEM_COUNTER; counter_id++) { ++ ++ counter_index++; ++ _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name, ++ sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_%s", ++ mali_name, _mali_mem_counter_descriptions[counter_id - FIRST_MEM_COUNTER]); ++ ++ global_mali_profiling_counters[counter_index].counter_id = counter_id; ++ } ++ ++ MALI_DEBUG_ASSERT((counter_index + 1) == num_global_mali_profiling_counters); ++ ++ return MALI_TRUE; ++} ++ ++void _mali_profiling_notification_mem_counter(struct mali_session_data *session, u32 counter_id, u32 key, int enable) ++{ ++ ++ MALI_DEBUG_ASSERT_POINTER(session); ++ ++ if (NULL != session) { ++ _mali_osk_notification_t *notification; ++ _mali_osk_notification_queue_t *queue; ++ ++ queue = session->ioctl_queue; ++ MALI_DEBUG_ASSERT(NULL != queue); ++ ++ notification = _mali_osk_notification_create(_MALI_NOTIFICATION_ANNOTATE_PROFILING_MEM_COUNTER, ++ sizeof(_mali_uk_annotate_profiling_mem_counter_s)); ++ ++ if (NULL != notification) { ++ _mali_uk_annotate_profiling_mem_counter_s *data = notification->result_buffer; ++ data->counter_id = counter_id; ++ data->key = key; ++ data->enable = enable; ++ ++ _mali_osk_notification_queue_send(queue, notification); ++ } else { ++ MALI_PRINT_ERROR(("Failed to create notification object!\n")); ++ } ++ } else { ++ MALI_PRINT_ERROR(("Failed to find the right session!\n")); ++ } ++} ++ ++void _mali_profiling_notification_enable(struct mali_session_data *session, u32 sampling_rate, int enable) ++{ ++ MALI_DEBUG_ASSERT_POINTER(session); ++ ++ if (NULL != session) { ++ _mali_osk_notification_t *notification; ++ _mali_osk_notification_queue_t *queue; ++ ++ queue = session->ioctl_queue; ++ MALI_DEBUG_ASSERT(NULL != queue); ++ ++ notification = _mali_osk_notification_create(_MALI_NOTIFICATION_ANNOTATE_PROFILING_ENABLE, ++ sizeof(_mali_uk_annotate_profiling_enable_s)); ++ ++ if (NULL != notification) { ++ _mali_uk_annotate_profiling_enable_s *data = notification->result_buffer; ++ data->sampling_rate = sampling_rate; ++ data->enable = enable; ++ ++ _mali_osk_notification_queue_send(queue, notification); ++ } else { ++ MALI_PRINT_ERROR(("Failed to create notification object!\n")); ++ } ++ } else { ++ MALI_PRINT_ERROR(("Failed to find the right session!\n")); ++ } ++} ++ ++ ++_mali_osk_errcode_t _mali_osk_profiling_init(mali_bool auto_start) ++{ ++ int i; ++ mali_profiling_stream *new_mali_profiling_stream = NULL; ++ mali_profiling_stream_list *new_mali_profiling_stream_list = NULL; ++ if (MALI_TRUE == auto_start) { ++ mali_set_user_setting(_MALI_UK_USER_SETTING_SW_EVENTS_ENABLE, MALI_TRUE); ++ } ++ ++ /*Init the global_mali_stream_list*/ ++ MALI_DEBUG_ASSERT(NULL == global_mali_stream_list); ++ new_mali_profiling_stream_list = (mali_profiling_stream_list *)kmalloc(sizeof(mali_profiling_stream_list), GFP_KERNEL); ++ ++ if (NULL == new_mali_profiling_stream_list) { ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ ++ spin_lock_init(&new_mali_profiling_stream_list->spin_lock); ++ INIT_LIST_HEAD(&new_mali_profiling_stream_list->free_list); ++ INIT_LIST_HEAD(&new_mali_profiling_stream_list->queue_list); ++ ++ spin_lock_init(&mali_activity_lock); ++ mali_activity_cores_num = 0; ++ ++ for (i = 0; i < MALI_PROFILING_STREAM_BUFFER_NUM; i++) { ++ new_mali_profiling_stream = (mali_profiling_stream *)kmalloc(sizeof(mali_profiling_stream), GFP_KERNEL); ++ if (NULL == new_mali_profiling_stream) { ++ _mali_profiling_stream_list_destory(new_mali_profiling_stream_list); ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ ++ INIT_LIST_HEAD(&new_mali_profiling_stream->list); ++ new_mali_profiling_stream->used_size = 0; ++ list_add_tail(&new_mali_profiling_stream->list, &new_mali_profiling_stream_list->free_list); ++ ++ } ++ ++ _mali_osk_atomic_init(&stream_fd_if_used, 0); ++ init_waitqueue_head(&stream_fd_wait_queue); ++ ++ hrtimer_init(&profiling_sampling_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); ++ ++ profiling_sampling_timer.function = _mali_profiling_sampling_counters; ++ ++ global_mali_stream_list = new_mali_profiling_stream_list; ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++void _mali_osk_profiling_term(void) ++{ ++ if (0 != profiling_sample_rate) { ++ hrtimer_cancel(&profiling_sampling_timer); ++ profiling_sample_rate = 0; ++ } ++ _mali_osk_atomic_term(&stream_fd_if_used); ++ ++ if (NULL != global_mali_profiling_counters) { ++ _mali_osk_free(global_mali_profiling_counters); ++ global_mali_profiling_counters = NULL; ++ num_global_mali_profiling_counters = 0; ++ } ++ ++ if (NULL != global_mali_stream_list) { ++ _mali_profiling_stream_list_destory(global_mali_stream_list); ++ global_mali_stream_list = NULL; ++ } ++ ++} ++ ++void _mali_osk_profiling_stop_sampling(u32 pid) ++{ ++ if (pid == current_profiling_pid) { ++ ++ int i; ++ /* Reset all counter states when closing connection.*/ ++ for (i = 0; i < num_global_mali_profiling_counters; ++i) { ++ _mali_profiling_set_event(global_mali_profiling_counters[i].counter_id, MALI_HW_CORE_NO_COUNTER); ++ global_mali_profiling_counters[i].enabled = 0; ++ global_mali_profiling_counters[i].prev_counter_value = 0; ++ global_mali_profiling_counters[i].current_counter_value = 0; ++ } ++ l2_cache_counter_if_enabled = MALI_FALSE; ++ num_counters_enabled = 0; ++ mem_counters_enabled = 0; ++ _mali_profiling_control(FBDUMP_CONTROL_ENABLE, 0); ++ _mali_profiling_control(SW_COUNTER_ENABLE, 0); ++ /* Delete sampling timer when closing connection. */ ++ if (0 != profiling_sample_rate) { ++ hrtimer_cancel(&profiling_sampling_timer); ++ profiling_sample_rate = 0; ++ } ++ current_profiling_pid = 0; ++ } ++} ++ ++void _mali_osk_profiling_add_event(u32 event_id, u32 data0, u32 data1, u32 data2, u32 data3, u32 data4) ++{ ++ /*Record the freq & volt to global_mali_profiling_counters here. */ ++ if (0 != profiling_sample_rate) { ++ u32 channel; ++ u32 state; ++ channel = (event_id >> 16) & 0xFF; ++ state = ((event_id >> 24) & 0xF) << 24; ++ ++ switch (state) { ++ case MALI_PROFILING_EVENT_TYPE_SINGLE: ++ if ((MALI_PROFILING_EVENT_CHANNEL_GPU >> 16) == channel) { ++ u32 reason = (event_id & 0xFFFF); ++ if (MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE == reason) { ++ _mali_osk_profiling_record_global_counters(COUNTER_FREQUENCY, data0); ++ _mali_osk_profiling_record_global_counters(COUNTER_VOLTAGE, data1); ++ } ++ } ++ break; ++ case MALI_PROFILING_EVENT_TYPE_START: ++ if ((MALI_PROFILING_EVENT_CHANNEL_GP0 >> 16) == channel) { ++ _mali_profiling_sampling_core_activity_switch(COUNTER_VP_ACTIVITY, 0, 1, data1); ++ } else if (channel >= (MALI_PROFILING_EVENT_CHANNEL_PP0 >> 16) && ++ (MALI_PROFILING_EVENT_CHANNEL_PP7 >> 16) >= channel) { ++ u32 core_id = channel - (MALI_PROFILING_EVENT_CHANNEL_PP0 >> 16); ++ _mali_profiling_sampling_core_activity_switch(COUNTER_FP_ACTIVITY, core_id, 1, data1); ++ } ++ break; ++ case MALI_PROFILING_EVENT_TYPE_STOP: ++ if ((MALI_PROFILING_EVENT_CHANNEL_GP0 >> 16) == channel) { ++ _mali_profiling_sampling_core_activity_switch(COUNTER_VP_ACTIVITY, 0, 0, 0); ++ } else if (channel >= (MALI_PROFILING_EVENT_CHANNEL_PP0 >> 16) && ++ (MALI_PROFILING_EVENT_CHANNEL_PP7 >> 16) >= channel) { ++ u32 core_id = channel - (MALI_PROFILING_EVENT_CHANNEL_PP0 >> 16); ++ _mali_profiling_sampling_core_activity_switch(COUNTER_FP_ACTIVITY, core_id, 0, 0); ++ } ++ break; ++ default: ++ break; ++ } ++ } ++ trace_mali_timeline_event(event_id, data0, data1, data2, data3, data4); ++} ++ ++void _mali_osk_profiling_report_sw_counters(u32 *counters) ++{ ++ trace_mali_sw_counters(_mali_osk_get_pid(), _mali_osk_get_tid(), NULL, counters); ++} ++ ++void _mali_osk_profiling_record_global_counters(int counter_id, u32 value) ++{ ++ if (NULL != global_mali_profiling_counters) { ++ int i ; ++ for (i = 0; i < num_global_mali_profiling_counters; i++) { ++ if (counter_id == global_mali_profiling_counters[i].counter_id && global_mali_profiling_counters[i].enabled) { ++ global_mali_profiling_counters[i].current_counter_value = value; ++ break; ++ } ++ } ++ } ++} ++ ++_mali_osk_errcode_t _mali_ukk_profiling_add_event(_mali_uk_profiling_add_event_s *args) ++{ ++ /* Always add process and thread identificator in the first two data elements for events from user space */ ++ _mali_osk_profiling_add_event(args->event_id, _mali_osk_get_pid(), _mali_osk_get_tid(), args->data[2], args->data[3], args->data[4]); ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++_mali_osk_errcode_t _mali_ukk_sw_counters_report(_mali_uk_sw_counters_report_s *args) ++{ ++ u32 *counters = (u32 *)(uintptr_t)args->counters; ++ ++ _mali_osk_profiling_report_sw_counters(counters); ++ ++ if (NULL != global_mali_profiling_counters) { ++ int i; ++ for (i = 0; i < MALI_PROFILING_SW_COUNTERS_NUM; i ++) { ++ if (global_mali_profiling_counters[first_sw_counter_index + i].enabled) { ++ global_mali_profiling_counters[first_sw_counter_index + i].current_counter_value = *(counters + i); ++ } ++ } ++ } ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++_mali_osk_errcode_t _mali_ukk_profiling_stream_fd_get(_mali_uk_profiling_stream_fd_get_s *args) ++{ ++ struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; ++ MALI_DEBUG_ASSERT_POINTER(session); ++ ++ if (1 == _mali_osk_atomic_inc_return(&stream_fd_if_used)) { ++ ++ s32 fd = anon_inode_getfd("[mali_profiling_stream]", &mali_profiling_stream_fops, ++ session, ++ O_RDONLY | O_CLOEXEC); ++ ++ args->stream_fd = fd; ++ if (0 > fd) { ++ _mali_osk_atomic_dec(&stream_fd_if_used); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ args->stream_fd = fd; ++ } else { ++ _mali_osk_atomic_dec(&stream_fd_if_used); ++ args->stream_fd = -1; ++ return _MALI_OSK_ERR_BUSY; ++ } ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++_mali_osk_errcode_t _mali_ukk_profiling_control_set(_mali_uk_profiling_control_set_s *args) ++{ ++ u32 control_packet_size; ++ u32 output_buffer_size; ++ ++ struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; ++ MALI_DEBUG_ASSERT_POINTER(session); ++ ++ if (NULL == global_mali_profiling_counters && MALI_FALSE == _mali_profiling_global_counters_init()) { ++ MALI_PRINT_ERROR(("Failed to create global_mali_profiling_counters.\n")); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ control_packet_size = args->control_packet_size; ++ output_buffer_size = args->response_packet_size; ++ ++ if (0 != control_packet_size) { ++ u8 control_type; ++ u8 *control_packet_data; ++ u8 *response_packet_data; ++ u32 version_length = sizeof(utgard_setup_version) - 1; ++ ++ control_packet_data = (u8 *)(uintptr_t)args->control_packet_data; ++ MALI_DEBUG_ASSERT_POINTER(control_packet_data); ++ response_packet_data = (u8 *)(uintptr_t)args->response_packet_data; ++ MALI_DEBUG_ASSERT_POINTER(response_packet_data); ++ ++ /*Decide if need to ignore Utgard setup version.*/ ++ if (control_packet_size >= version_length) { ++ if (0 == memcmp(control_packet_data, utgard_setup_version, version_length)) { ++ if (control_packet_size == version_length) { ++ args->response_packet_size = 0; ++ return _MALI_OSK_ERR_OK; ++ } else { ++ control_packet_data += version_length; ++ control_packet_size -= version_length; ++ } ++ } ++ } ++ ++ current_profiling_pid = _mali_osk_get_pid(); ++ ++ control_type = control_packet_data[0]; ++ switch (control_type) { ++ case PACKET_HEADER_COUNTERS_REQUEST: { ++ int i; ++ ++ if (PACKET_HEADER_SIZE > control_packet_size || ++ control_packet_size != _mali_profiling_get_packet_size(control_packet_data + 1)) { ++ MALI_PRINT_ERROR(("Wrong control packet size, type 0x%x,size 0x%x.\n", control_packet_data[0], control_packet_size)); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ /* Send supported counters */ ++ if (PACKET_HEADER_SIZE > output_buffer_size) ++ return _MALI_OSK_ERR_FAULT; ++ ++ *response_packet_data = PACKET_HEADER_COUNTERS_ACK; ++ args->response_packet_size = PACKET_HEADER_SIZE; ++ ++ for (i = 0; i < num_global_mali_profiling_counters; ++i) { ++ u32 name_size = strlen(global_mali_profiling_counters[i].counter_name); ++ ++ if ((args->response_packet_size + name_size + 1) > output_buffer_size) { ++ MALI_PRINT_ERROR(("Response packet data is too large..\n")); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ memcpy(response_packet_data + args->response_packet_size, ++ global_mali_profiling_counters[i].counter_name, name_size + 1); ++ ++ args->response_packet_size += (name_size + 1); ++ ++ if (global_mali_profiling_counters[i].counter_id == COUNTER_VP_ACTIVITY) { ++ args->response_packet_size += _mali_profiling_pack_int(response_packet_data, ++ output_buffer_size, args->response_packet_size, (s32)1); ++ } else if (global_mali_profiling_counters[i].counter_id == COUNTER_FP_ACTIVITY) { ++ args->response_packet_size += _mali_profiling_pack_int(response_packet_data, ++ output_buffer_size, args->response_packet_size, (s32)mali_pp_get_glob_num_pp_cores()); ++ } else { ++ args->response_packet_size += _mali_profiling_pack_int(response_packet_data, ++ output_buffer_size, args->response_packet_size, (s32) - 1); ++ } ++ } ++ ++ _mali_profiling_set_packet_size(response_packet_data + 1, args->response_packet_size); ++ break; ++ } ++ ++ case PACKET_HEADER_COUNTERS_ENABLE: { ++ int i; ++ u32 request_pos = PACKET_HEADER_SIZE; ++ mali_bool sw_counter_if_enabled = MALI_FALSE; ++ ++ if (PACKET_HEADER_SIZE > control_packet_size || ++ control_packet_size != _mali_profiling_get_packet_size(control_packet_data + 1)) { ++ MALI_PRINT_ERROR(("Wrong control packet size , type 0x%x,size 0x%x.\n", control_packet_data[0], control_packet_size)); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ /* Init all counter states before enable requested counters.*/ ++ for (i = 0; i < num_global_mali_profiling_counters; ++i) { ++ _mali_profiling_set_event(global_mali_profiling_counters[i].counter_id, MALI_HW_CORE_NO_COUNTER); ++ global_mali_profiling_counters[i].enabled = 0; ++ global_mali_profiling_counters[i].prev_counter_value = 0; ++ global_mali_profiling_counters[i].current_counter_value = 0; ++ ++ if (global_mali_profiling_counters[i].counter_id >= FIRST_MEM_COUNTER && ++ global_mali_profiling_counters[i].counter_id <= LAST_MEM_COUNTER) { ++ _mali_profiling_notification_mem_counter(session, global_mali_profiling_counters[i].counter_id, 0, 0); ++ } ++ } ++ ++ l2_cache_counter_if_enabled = MALI_FALSE; ++ num_counters_enabled = 0; ++ mem_counters_enabled = 0; ++ _mali_profiling_control(FBDUMP_CONTROL_ENABLE, 0); ++ _mali_profiling_control(SW_COUNTER_ENABLE, 0); ++ _mali_profiling_notification_enable(session, 0, 0); ++ ++ /* Enable requested counters */ ++ while (request_pos < control_packet_size) { ++ u32 begin = request_pos; ++ u32 event; ++ u32 key; ++ ++ /* Check the counter name which should be ended with null */ ++ while (request_pos < control_packet_size && control_packet_data[request_pos] != '\0') { ++ ++request_pos; ++ } ++ ++ if (request_pos >= control_packet_size) ++ return _MALI_OSK_ERR_FAULT; ++ ++ ++request_pos; ++ event = _mali_profiling_read_packet_int(control_packet_data, &request_pos, control_packet_size); ++ key = _mali_profiling_read_packet_int(control_packet_data, &request_pos, control_packet_size); ++ ++ for (i = 0; i < num_global_mali_profiling_counters; ++i) { ++ u32 name_size = strlen((char *)(control_packet_data + begin)); ++ ++ if (strncmp(global_mali_profiling_counters[i].counter_name, (char *)(control_packet_data + begin), name_size) == 0) { ++ if (!sw_counter_if_enabled && (FIRST_SW_COUNTER <= global_mali_profiling_counters[i].counter_id ++ && global_mali_profiling_counters[i].counter_id <= LAST_SW_COUNTER)) { ++ sw_counter_if_enabled = MALI_TRUE; ++ _mali_profiling_control(SW_COUNTER_ENABLE, 1); ++ } ++ ++ if (COUNTER_FILMSTRIP == global_mali_profiling_counters[i].counter_id) { ++ _mali_profiling_control(FBDUMP_CONTROL_ENABLE, 1); ++ _mali_profiling_control(FBDUMP_CONTROL_RATE, event & 0xff); ++ _mali_profiling_control(FBDUMP_CONTROL_RESIZE_FACTOR, (event >> 8) & 0xff); ++ } ++ ++ if (global_mali_profiling_counters[i].counter_id >= FIRST_MEM_COUNTER && ++ global_mali_profiling_counters[i].counter_id <= LAST_MEM_COUNTER) { ++ _mali_profiling_notification_mem_counter(session, global_mali_profiling_counters[i].counter_id, ++ key, 1); ++ mem_counters_enabled++; ++ } ++ ++ global_mali_profiling_counters[i].counter_event = event; ++ global_mali_profiling_counters[i].key = key; ++ global_mali_profiling_counters[i].enabled = 1; ++ ++ _mali_profiling_set_event(global_mali_profiling_counters[i].counter_id, ++ global_mali_profiling_counters[i].counter_event); ++ num_counters_enabled++; ++ break; ++ } ++ } ++ ++ if (i == num_global_mali_profiling_counters) { ++ MALI_PRINT_ERROR(("Counter name does not match for type %u.\n", control_type)); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ } ++ ++ if (PACKET_HEADER_SIZE <= output_buffer_size) { ++ *response_packet_data = PACKET_HEADER_ACK; ++ _mali_profiling_set_packet_size(response_packet_data + 1, PACKET_HEADER_SIZE); ++ args->response_packet_size = PACKET_HEADER_SIZE; ++ } else { ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ break; ++ } ++ ++ case PACKET_HEADER_START_CAPTURE_VALUE: { ++ u32 live_rate; ++ u32 request_pos = PACKET_HEADER_SIZE; ++ ++ if (PACKET_HEADER_SIZE > control_packet_size || ++ control_packet_size != _mali_profiling_get_packet_size(control_packet_data + 1)) { ++ MALI_PRINT_ERROR(("Wrong control packet size , type 0x%x,size 0x%x.\n", control_packet_data[0], control_packet_size)); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ /* Read samping rate in nanoseconds and live rate, start capture.*/ ++ profiling_sample_rate = _mali_profiling_read_packet_int(control_packet_data, ++ &request_pos, control_packet_size); ++ ++ live_rate = _mali_profiling_read_packet_int(control_packet_data, &request_pos, control_packet_size); ++ ++ if (PACKET_HEADER_SIZE <= output_buffer_size) { ++ *response_packet_data = PACKET_HEADER_ACK; ++ _mali_profiling_set_packet_size(response_packet_data + 1, PACKET_HEADER_SIZE); ++ args->response_packet_size = PACKET_HEADER_SIZE; ++ } else { ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ if (0 != num_counters_enabled && 0 != profiling_sample_rate) { ++ _mali_profiling_global_stream_list_free(); ++ if (mem_counters_enabled > 0) { ++ _mali_profiling_notification_enable(session, profiling_sample_rate, 1); ++ } ++ hrtimer_start(&profiling_sampling_timer, ++ ktime_set(profiling_sample_rate / 1000000000, profiling_sample_rate % 1000000000), ++ HRTIMER_MODE_REL_PINNED); ++ } ++ ++ break; ++ } ++ default: ++ MALI_PRINT_ERROR(("Unsupported profiling packet header type %u.\n", control_type)); ++ args->response_packet_size = 0; ++ return _MALI_OSK_ERR_FAULT; ++ } ++ } else { ++ _mali_osk_profiling_stop_sampling(current_profiling_pid); ++ _mali_profiling_notification_enable(session, 0, 0); ++ } ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++/** ++ * Called by gator.ko to set HW counters ++ * ++ * @param counter_id The counter ID. ++ * @param event_id Event ID that the counter should count (HW counter value from TRM). ++ * ++ * @return 1 on success, 0 on failure. ++ */ ++int _mali_profiling_set_event(u32 counter_id, s32 event_id) ++{ ++ if (COUNTER_VP_0_C0 == counter_id) { ++ mali_gp_job_set_gp_counter_src0(event_id); ++ } else if (COUNTER_VP_0_C1 == counter_id) { ++ mali_gp_job_set_gp_counter_src1(event_id); ++ } else if (COUNTER_FP_0_C0 <= counter_id && COUNTER_FP_7_C1 >= counter_id) { ++ /* ++ * Two compatibility notes for this function: ++ * ++ * 1) Previously the DDK allowed per core counters. ++ * ++ * This did not make much sense on Mali-450 with the "virtual PP core" concept, ++ * so this option was removed, and only the same pair of HW counters was allowed on all cores, ++ * beginning with r3p2 release. ++ * ++ * Starting with r4p0, it is now possible to set different HW counters for the different sub jobs. ++ * This should be almost the same, since sub job 0 is designed to run on core 0, ++ * sub job 1 on core 1, and so on. ++ * ++ * The scheduling of PP sub jobs is not predictable, and this often led to situations where core 0 ran 2 ++ * sub jobs, while for instance core 1 ran zero. Having the counters set per sub job would thus increase ++ * the predictability of the returned data (as you would be guaranteed data for all the selected HW counters). ++ * ++ * PS: Core scaling needs to be disabled in order to use this reliably (goes for both solutions). ++ * ++ * The framework/#defines with Gator still indicates that the counter is for a particular core, ++ * but this is internally used as a sub job ID instead (no translation needed). ++ * ++ * 2) Global/default vs per sub job counters ++ * ++ * Releases before r3p2 had only per PP core counters. ++ * r3p2 releases had only one set of default/global counters which applied to all PP cores ++ * Starting with r4p0, we have both a set of default/global counters, ++ * and individual counters per sub job (equal to per core). ++ * ++ * To keep compatibility with Gator/DS-5/streamline, the following scheme is used: ++ * ++ * r3p2 release; only counters set for core 0 is handled, ++ * this is applied as the default/global set of counters, and will thus affect all cores. ++ * ++ * r4p0 release; counters set for core 0 is applied as both the global/default set of counters, ++ * and counters for sub job 0. ++ * Counters set for core 1-7 is only applied for the corresponding sub job. ++ * ++ * This should allow the DS-5/Streamline GUI to have a simple mode where it only allows setting the ++ * values for core 0, and thus this will be applied to all PP sub jobs/cores. ++ * Advanced mode will also be supported, where individual pairs of HW counters can be selected. ++ * ++ * The GUI will (until it is updated) still refer to cores instead of sub jobs, but this is probably ++ * something we can live with! ++ * ++ * Mali-450 note: Each job is not divided into a deterministic number of sub jobs, as the HW DLBU ++ * automatically distributes the load between whatever number of cores is available at this particular time. ++ * A normal PP job on Mali-450 is thus considered a single (virtual) job, and it will thus only be possible ++ * to use a single pair of HW counters (even if the job ran on multiple PP cores). ++ * In other words, only the global/default pair of PP HW counters will be used for normal Mali-450 jobs. ++ */ ++ u32 sub_job = (counter_id - COUNTER_FP_0_C0) >> 1; ++ u32 counter_src = (counter_id - COUNTER_FP_0_C0) & 1; ++ if (0 == counter_src) { ++ mali_pp_job_set_pp_counter_sub_job_src0(sub_job, event_id); ++ if (0 == sub_job) { ++ mali_pp_job_set_pp_counter_global_src0(event_id); ++ } ++ } else { ++ mali_pp_job_set_pp_counter_sub_job_src1(sub_job, event_id); ++ if (0 == sub_job) { ++ mali_pp_job_set_pp_counter_global_src1(event_id); ++ } ++ } ++ } else if (COUNTER_L2_0_C0 <= counter_id && COUNTER_L2_2_C1 >= counter_id) { ++ u32 core_id = (counter_id - COUNTER_L2_0_C0) >> 1; ++ struct mali_l2_cache_core *l2_cache_core = mali_l2_cache_core_get_glob_l2_core(core_id); ++ ++ if (NULL != l2_cache_core) { ++ u32 counter_src = (counter_id - COUNTER_L2_0_C0) & 1; ++ mali_l2_cache_core_set_counter_src(l2_cache_core, ++ counter_src, event_id); ++ l2_cache_counter_if_enabled = MALI_TRUE; ++ } ++ } else { ++ return 0; /* Failure, unknown event */ ++ } ++ ++ return 1; /* success */ ++} ++ ++/** ++ * Called by gator.ko to retrieve the L2 cache counter values for all L2 cache cores. ++ * The L2 cache counters are unique in that they are polled by gator, rather than being ++ * transmitted via the tracepoint mechanism. ++ * ++ * @param values Pointer to a _mali_profiling_l2_counter_values structure where ++ * the counter sources and values will be output ++ * @return 0 if all went well; otherwise, return the mask with the bits set for the powered off cores ++ */ ++u32 _mali_profiling_get_l2_counters(_mali_profiling_l2_counter_values *values) ++{ ++ u32 l2_cores_num = mali_l2_cache_core_get_glob_num_l2_cores(); ++ u32 i; ++ ++ MALI_DEBUG_ASSERT(l2_cores_num <= 3); ++ ++ for (i = 0; i < l2_cores_num; i++) { ++ struct mali_l2_cache_core *l2_cache = mali_l2_cache_core_get_glob_l2_core(i); ++ ++ if (NULL == l2_cache) { ++ continue; ++ } ++ ++ mali_l2_cache_core_get_counter_values(l2_cache, ++ &values->cores[i].source0, ++ &values->cores[i].value0, ++ &values->cores[i].source1, ++ &values->cores[i].value1); ++ } ++ ++ return 0; ++} ++ ++/** ++ * Called by gator to control the production of profiling information at runtime. ++ */ ++void _mali_profiling_control(u32 action, u32 value) ++{ ++ switch (action) { ++ case FBDUMP_CONTROL_ENABLE: ++ mali_set_user_setting(_MALI_UK_USER_SETTING_COLORBUFFER_CAPTURE_ENABLED, (value == 0 ? MALI_FALSE : MALI_TRUE)); ++ break; ++ case FBDUMP_CONTROL_RATE: ++ mali_set_user_setting(_MALI_UK_USER_SETTING_BUFFER_CAPTURE_N_FRAMES, value); ++ break; ++ case SW_COUNTER_ENABLE: ++ mali_set_user_setting(_MALI_UK_USER_SETTING_SW_COUNTER_ENABLED, value); ++ break; ++ case FBDUMP_CONTROL_RESIZE_FACTOR: ++ mali_set_user_setting(_MALI_UK_USER_SETTING_BUFFER_CAPTURE_RESIZE_FACTOR, value); ++ break; ++ default: ++ break; /* Ignore unimplemented actions */ ++ } ++} ++ ++/** ++ * Called by gator to get mali api version. ++ */ ++u32 _mali_profiling_get_api_version(void) ++{ ++ return MALI_PROFILING_API_VERSION; ++} ++ ++/** ++* Called by gator to get the data about Mali instance in use: ++* product id, version, number of cores ++*/ ++void _mali_profiling_get_mali_version(struct _mali_profiling_mali_version *values) ++{ ++ values->mali_product_id = (u32)mali_kernel_core_get_product_id(); ++ values->mali_version_major = mali_kernel_core_get_gpu_major_version(); ++ values->mali_version_minor = mali_kernel_core_get_gpu_minor_version(); ++ values->num_of_l2_cores = mali_l2_cache_core_get_glob_num_l2_cores(); ++ values->num_of_fp_cores = mali_executor_get_num_cores_total(); ++ values->num_of_vp_cores = 1; ++} ++ ++ ++EXPORT_SYMBOL(_mali_profiling_set_event); ++EXPORT_SYMBOL(_mali_profiling_get_l2_counters); ++EXPORT_SYMBOL(_mali_profiling_control); ++EXPORT_SYMBOL(_mali_profiling_get_api_version); ++EXPORT_SYMBOL(_mali_profiling_get_mali_version); +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_specific.h b/drivers/gpu/arm/mali400/mali/linux/mali_osk_specific.h +new file mode 100755 +index 000000000000..af51161f9da1 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_specific.h +@@ -0,0 +1,74 @@ ++/* ++ * Copyright (C) 2010, 2012-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file mali_osk_specific.h ++ * Defines per-OS Kernel level specifics, such as unusual workarounds for ++ * certain OSs. ++ */ ++ ++#ifndef __MALI_OSK_SPECIFIC_H__ ++#define __MALI_OSK_SPECIFIC_H__ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++ ++#include "mali_osk_types.h" ++#include "mali_kernel_linux.h" ++ ++#define MALI_STATIC_INLINE static inline ++#define MALI_NON_STATIC_INLINE inline ++ ++typedef struct dma_pool *mali_dma_pool; ++ ++typedef u32 mali_dma_addr; ++ ++#if MALI_ENABLE_CPU_CYCLES ++/* Reads out the clock cycle performance counter of the current cpu. ++ It is useful for cost-free (2 cycle) measuring of the time spent ++ in a code path. Sample before and after, the diff number of cycles. ++ When the CPU is idle it will not increase this clock counter. ++ It means that the counter is accurate if only spin-locks are used, ++ but mutexes may lead to too low values since the cpu might "idle" ++ waiting for the mutex to become available. ++ The clock source is configured on the CPU during mali module load, ++ but will not give useful output after a CPU has been power cycled. ++ It is therefore important to configure the system to not turn of ++ the cpu cores when using this functionallity.*/ ++static inline unsigned int mali_get_cpu_cyclecount(void) ++{ ++ unsigned int value; ++ /* Reading the CCNT Register - CPU clock counter */ ++ asm volatile("MRC p15, 0, %0, c9, c13, 0\t\n": "=r"(value)); ++ return value; ++} ++ ++void mali_init_cpu_time_counters(int reset, int enable_divide_by_64); ++#endif ++ ++ ++MALI_STATIC_INLINE u32 _mali_osk_copy_from_user(void *to, void *from, u32 n) ++{ ++ return (u32)copy_from_user(to, from, (unsigned long)n); ++} ++ ++MALI_STATIC_INLINE mali_bool _mali_osk_in_atomic(void) ++{ ++ return in_atomic(); ++} ++ ++#define _mali_osk_put_user(x, ptr) put_user(x, ptr) ++ ++#endif /* __MALI_OSK_SPECIFIC_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_time.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_time.c +new file mode 100755 +index 000000000000..d295e712ac7a +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_time.c +@@ -0,0 +1,59 @@ ++/* ++ * Copyright (C) 2010, 2013-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file mali_osk_time.c ++ * Implementation of the OS abstraction layer for the kernel device driver ++ */ ++ ++#include "mali_osk.h" ++#include ++#include ++#include ++ ++mali_bool _mali_osk_time_after_eq(unsigned long ticka, unsigned long tickb) ++{ ++ return time_after_eq(ticka, tickb) ? ++ MALI_TRUE : MALI_FALSE; ++} ++ ++unsigned long _mali_osk_time_mstoticks(u32 ms) ++{ ++ return msecs_to_jiffies(ms); ++} ++ ++u32 _mali_osk_time_tickstoms(unsigned long ticks) ++{ ++ return jiffies_to_msecs(ticks); ++} ++ ++unsigned long _mali_osk_time_tickcount(void) ++{ ++ return jiffies; ++} ++ ++void _mali_osk_time_ubusydelay(u32 usecs) ++{ ++ udelay(usecs); ++} ++ ++u64 _mali_osk_time_get_ns(void) ++{ ++ struct timespec64 tsval; ++ ktime_get_real_ts64(&tsval); ++ return (u64)timespec64_to_ns(&tsval); ++} ++ ++u64 _mali_osk_boot_time_get_ns(void) ++{ ++ struct timespec64 tsval; ++ ktime_get_boottime_ts64(&tsval); ++ return (u64)timespec64_to_ns(&tsval); ++} +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_timers.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_timers.c +new file mode 100755 +index 000000000000..d01c1148272a +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_timers.c +@@ -0,0 +1,76 @@ ++/* ++ * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file mali_osk_timers.c ++ * Implementation of the OS abstraction layer for the kernel device driver ++ */ ++ ++#include ++#include ++#include "mali_osk.h" ++#include "mali_kernel_common.h" ++ ++struct _mali_osk_timer_t_struct { ++ struct timer_list timer; ++}; ++ ++typedef void (*timer_timeout_function_t)(unsigned long); ++ ++_mali_osk_timer_t *_mali_osk_timer_init(_mali_osk_timer_callback_t callback) ++{ ++ _mali_osk_timer_t *t = (_mali_osk_timer_t *)kmalloc(sizeof(_mali_osk_timer_t), GFP_KERNEL); ++ if (NULL != t) ++ timer_setup(&t->timer, ++ (void (*)(struct timer_list *))callback, 0); ++ return t; ++} ++ ++void _mali_osk_timer_add(_mali_osk_timer_t *tim, unsigned long ticks_to_expire) ++{ ++ MALI_DEBUG_ASSERT_POINTER(tim); ++ tim->timer.expires = jiffies + ticks_to_expire; ++ add_timer(&(tim->timer)); ++} ++ ++void _mali_osk_timer_mod(_mali_osk_timer_t *tim, unsigned long ticks_to_expire) ++{ ++ MALI_DEBUG_ASSERT_POINTER(tim); ++ mod_timer(&(tim->timer), jiffies + ticks_to_expire); ++} ++ ++void _mali_osk_timer_del(_mali_osk_timer_t *tim) ++{ ++ MALI_DEBUG_ASSERT_POINTER(tim); ++ del_timer_sync(&(tim->timer)); ++} ++ ++void _mali_osk_timer_del_async(_mali_osk_timer_t *tim) ++{ ++ MALI_DEBUG_ASSERT_POINTER(tim); ++ del_timer(&(tim->timer)); ++} ++ ++mali_bool _mali_osk_timer_pending(_mali_osk_timer_t *tim) ++{ ++ MALI_DEBUG_ASSERT_POINTER(tim); ++ return 1 == timer_pending(&(tim->timer)); ++} ++ ++void _mali_osk_timer_setcallback(_mali_osk_timer_t *tim, _mali_osk_timer_callback_t callback, void *data) ++{ ++ MALI_DEBUG_ASSERT_POINTER(tim); ++} ++ ++void _mali_osk_timer_term(_mali_osk_timer_t *tim) ++{ ++ MALI_DEBUG_ASSERT_POINTER(tim); ++ kfree(tim); ++} +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_wait_queue.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_wait_queue.c +new file mode 100755 +index 000000000000..fa12abd3f5dc +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_wait_queue.c +@@ -0,0 +1,78 @@ ++/* ++ * Copyright (C) 2012-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file mali_osk_wait_queue.c ++ * Implemenation of the OS abstraction layer for the kernel device driver ++ */ ++ ++#include ++#include ++#include ++ ++#include "mali_osk.h" ++#include "mali_kernel_common.h" ++ ++struct _mali_osk_wait_queue_t_struct { ++ wait_queue_head_t wait_queue; ++}; ++ ++_mali_osk_wait_queue_t *_mali_osk_wait_queue_init(void) ++{ ++ _mali_osk_wait_queue_t *ret = NULL; ++ ++ ret = kmalloc(sizeof(_mali_osk_wait_queue_t), GFP_KERNEL); ++ ++ if (NULL == ret) { ++ return ret; ++ } ++ ++ init_waitqueue_head(&ret->wait_queue); ++ MALI_DEBUG_ASSERT(!waitqueue_active(&ret->wait_queue)); ++ ++ return ret; ++} ++ ++void _mali_osk_wait_queue_wait_event(_mali_osk_wait_queue_t *queue, mali_bool(*condition)(void *), void *data) ++{ ++ MALI_DEBUG_ASSERT_POINTER(queue); ++ MALI_DEBUG_PRINT(6, ("Adding to wait queue %p\n", queue)); ++ wait_event(queue->wait_queue, condition(data)); ++} ++ ++void _mali_osk_wait_queue_wait_event_timeout(_mali_osk_wait_queue_t *queue, mali_bool(*condition)(void *), void *data, u32 timeout) ++{ ++ MALI_DEBUG_ASSERT_POINTER(queue); ++ MALI_DEBUG_PRINT(6, ("Adding to wait queue %p\n", queue)); ++ wait_event_timeout(queue->wait_queue, condition(data), _mali_osk_time_mstoticks(timeout)); ++} ++ ++void _mali_osk_wait_queue_wake_up(_mali_osk_wait_queue_t *queue) ++{ ++ MALI_DEBUG_ASSERT_POINTER(queue); ++ ++ /* if queue is empty, don't attempt to wake up its elements */ ++ if (!waitqueue_active(&queue->wait_queue)) return; ++ ++ MALI_DEBUG_PRINT(6, ("Waking up elements in wait queue %p ....\n", queue)); ++ ++ wake_up_all(&queue->wait_queue); ++ ++ MALI_DEBUG_PRINT(6, ("... elements in wait queue %p woken up\n", queue)); ++} ++ ++void _mali_osk_wait_queue_term(_mali_osk_wait_queue_t *queue) ++{ ++ /* Parameter validation */ ++ MALI_DEBUG_ASSERT_POINTER(queue); ++ ++ /* Linux requires no explicit termination of wait queues */ ++ kfree(queue); ++} +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_wq.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_wq.c +new file mode 100755 +index 000000000000..d5e258a83a29 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_wq.c +@@ -0,0 +1,240 @@ ++/* ++ * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file mali_osk_wq.c ++ * Implementation of the OS abstraction layer for the kernel device driver ++ */ ++ ++#include /* For memory allocation */ ++#include ++#include ++#include ++ ++#include "mali_osk.h" ++#include "mali_kernel_common.h" ++#include "mali_kernel_license.h" ++#include "mali_kernel_linux.h" ++ ++typedef struct _mali_osk_wq_work_s { ++ _mali_osk_wq_work_handler_t handler; ++ void *data; ++ mali_bool high_pri; ++ struct work_struct work_handle; ++} mali_osk_wq_work_object_t; ++ ++typedef struct _mali_osk_wq_delayed_work_s { ++ _mali_osk_wq_work_handler_t handler; ++ void *data; ++ struct delayed_work work; ++} mali_osk_wq_delayed_work_object_t; ++ ++#if MALI_LICENSE_IS_GPL ++static struct workqueue_struct *mali_wq_normal = NULL; ++static struct workqueue_struct *mali_wq_high = NULL; ++#endif ++ ++static void _mali_osk_wq_work_func(struct work_struct *work); ++ ++_mali_osk_errcode_t _mali_osk_wq_init(void) ++{ ++#if MALI_LICENSE_IS_GPL ++ MALI_DEBUG_ASSERT(NULL == mali_wq_normal); ++ MALI_DEBUG_ASSERT(NULL == mali_wq_high); ++ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 36) ++ mali_wq_normal = alloc_workqueue("mali", WQ_UNBOUND, 0); ++ mali_wq_high = alloc_workqueue("mali_high_pri", WQ_HIGHPRI | WQ_UNBOUND, 0); ++#else ++ mali_wq_normal = create_workqueue("mali"); ++ mali_wq_high = create_workqueue("mali_high_pri"); ++#endif ++ if (NULL == mali_wq_normal || NULL == mali_wq_high) { ++ MALI_PRINT_ERROR(("Unable to create Mali workqueues\n")); ++ ++ if (mali_wq_normal) destroy_workqueue(mali_wq_normal); ++ if (mali_wq_high) destroy_workqueue(mali_wq_high); ++ ++ mali_wq_normal = NULL; ++ mali_wq_high = NULL; ++ ++ return _MALI_OSK_ERR_FAULT; ++ } ++#endif /* MALI_LICENSE_IS_GPL */ ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++void _mali_osk_wq_flush(void) ++{ ++#if MALI_LICENSE_IS_GPL ++ flush_workqueue(mali_wq_high); ++ flush_workqueue(mali_wq_normal); ++#else ++ flush_scheduled_work(); ++#endif ++} ++ ++void _mali_osk_wq_term(void) ++{ ++#if MALI_LICENSE_IS_GPL ++ MALI_DEBUG_ASSERT(NULL != mali_wq_normal); ++ MALI_DEBUG_ASSERT(NULL != mali_wq_high); ++ ++ flush_workqueue(mali_wq_normal); ++ destroy_workqueue(mali_wq_normal); ++ ++ flush_workqueue(mali_wq_high); ++ destroy_workqueue(mali_wq_high); ++ ++ mali_wq_normal = NULL; ++ mali_wq_high = NULL; ++#else ++ flush_scheduled_work(); ++#endif ++} ++ ++_mali_osk_wq_work_t *_mali_osk_wq_create_work(_mali_osk_wq_work_handler_t handler, void *data) ++{ ++ mali_osk_wq_work_object_t *work = kmalloc(sizeof(mali_osk_wq_work_object_t), GFP_KERNEL); ++ ++ if (NULL == work) return NULL; ++ ++ work->handler = handler; ++ work->data = data; ++ work->high_pri = MALI_FALSE; ++ ++ INIT_WORK(&work->work_handle, _mali_osk_wq_work_func); ++ ++ return work; ++} ++ ++_mali_osk_wq_work_t *_mali_osk_wq_create_work_high_pri(_mali_osk_wq_work_handler_t handler, void *data) ++{ ++ mali_osk_wq_work_object_t *work = kmalloc(sizeof(mali_osk_wq_work_object_t), GFP_KERNEL); ++ ++ if (NULL == work) return NULL; ++ ++ work->handler = handler; ++ work->data = data; ++ work->high_pri = MALI_TRUE; ++ ++ INIT_WORK(&work->work_handle, _mali_osk_wq_work_func); ++ ++ return work; ++} ++ ++void _mali_osk_wq_delete_work(_mali_osk_wq_work_t *work) ++{ ++ mali_osk_wq_work_object_t *work_object = (mali_osk_wq_work_object_t *)work; ++ _mali_osk_wq_flush(); ++ kfree(work_object); ++} ++ ++void _mali_osk_wq_delete_work_nonflush(_mali_osk_wq_work_t *work) ++{ ++ mali_osk_wq_work_object_t *work_object = (mali_osk_wq_work_object_t *)work; ++ kfree(work_object); ++} ++ ++void _mali_osk_wq_schedule_work(_mali_osk_wq_work_t *work) ++{ ++ mali_osk_wq_work_object_t *work_object = (mali_osk_wq_work_object_t *)work; ++#if MALI_LICENSE_IS_GPL ++ queue_work(mali_wq_normal, &work_object->work_handle); ++#else ++ schedule_work(&work_object->work_handle); ++#endif ++} ++ ++void _mali_osk_wq_schedule_work_high_pri(_mali_osk_wq_work_t *work) ++{ ++ mali_osk_wq_work_object_t *work_object = (mali_osk_wq_work_object_t *)work; ++#if MALI_LICENSE_IS_GPL ++ queue_work(mali_wq_high, &work_object->work_handle); ++#else ++ schedule_work(&work_object->work_handle); ++#endif ++} ++ ++static void _mali_osk_wq_work_func(struct work_struct *work) ++{ ++ mali_osk_wq_work_object_t *work_object; ++ ++ work_object = _MALI_OSK_CONTAINER_OF(work, mali_osk_wq_work_object_t, work_handle); ++ ++#if MALI_LICENSE_IS_GPL ++#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,36) ++ /* We want highest Dynamic priority of the thread so that the Jobs depending ++ ** on this thread could be scheduled in time. Without this, this thread might ++ ** sometimes need to wait for some threads in user mode to finish its round-robin ++ ** time, causing *bubble* in the Mali pipeline. Thanks to the new implementation ++ ** of high-priority workqueue in new kernel, this only happens in older kernel. ++ */ ++ if (MALI_TRUE == work_object->high_pri) { ++ set_user_nice(current, -19); ++ } ++#endif ++#endif /* MALI_LICENSE_IS_GPL */ ++ ++ work_object->handler(work_object->data); ++} ++ ++static void _mali_osk_wq_delayed_work_func(struct work_struct *work) ++{ ++ mali_osk_wq_delayed_work_object_t *work_object; ++ ++ work_object = _MALI_OSK_CONTAINER_OF(work, mali_osk_wq_delayed_work_object_t, work.work); ++ work_object->handler(work_object->data); ++} ++ ++mali_osk_wq_delayed_work_object_t *_mali_osk_wq_delayed_create_work(_mali_osk_wq_work_handler_t handler, void *data) ++{ ++ mali_osk_wq_delayed_work_object_t *work = kmalloc(sizeof(mali_osk_wq_delayed_work_object_t), GFP_KERNEL); ++ ++ if (NULL == work) return NULL; ++ ++ work->handler = handler; ++ work->data = data; ++ ++ INIT_DELAYED_WORK(&work->work, _mali_osk_wq_delayed_work_func); ++ ++ return work; ++} ++ ++void _mali_osk_wq_delayed_delete_work_nonflush(_mali_osk_wq_delayed_work_t *work) ++{ ++ mali_osk_wq_delayed_work_object_t *work_object = (mali_osk_wq_delayed_work_object_t *)work; ++ kfree(work_object); ++} ++ ++void _mali_osk_wq_delayed_cancel_work_async(_mali_osk_wq_delayed_work_t *work) ++{ ++ mali_osk_wq_delayed_work_object_t *work_object = (mali_osk_wq_delayed_work_object_t *)work; ++ cancel_delayed_work(&work_object->work); ++} ++ ++void _mali_osk_wq_delayed_cancel_work_sync(_mali_osk_wq_delayed_work_t *work) ++{ ++ mali_osk_wq_delayed_work_object_t *work_object = (mali_osk_wq_delayed_work_object_t *)work; ++ cancel_delayed_work_sync(&work_object->work); ++} ++ ++void _mali_osk_wq_delayed_schedule_work(_mali_osk_wq_delayed_work_t *work, u32 delay) ++{ ++ mali_osk_wq_delayed_work_object_t *work_object = (mali_osk_wq_delayed_work_object_t *)work; ++ ++#if MALI_LICENSE_IS_GPL ++ queue_delayed_work(mali_wq_normal, &work_object->work, delay); ++#else ++ schedule_delayed_work(&work_object->work, delay); ++#endif ++ ++} +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_pmu_power_up_down.c b/drivers/gpu/arm/mali400/mali/linux/mali_pmu_power_up_down.c +new file mode 100755 +index 000000000000..931d7f07a1d2 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_pmu_power_up_down.c +@@ -0,0 +1,23 @@ ++/** ++ * Copyright (C) 2010, 2012-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file mali_pmu_power_up_down.c ++ */ ++ ++#include ++#include "mali_executor.h" ++ ++int mali_perf_set_num_pp_cores(unsigned int num_cores) ++{ ++ return mali_executor_set_perf_level(num_cores, MALI_FALSE); ++} ++ ++EXPORT_SYMBOL(mali_perf_set_num_pp_cores); +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_profiling_events.h b/drivers/gpu/arm/mali400/mali/linux/mali_profiling_events.h +new file mode 100755 +index 000000000000..4661cac42b3f +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_profiling_events.h +@@ -0,0 +1,17 @@ ++/* ++ * Copyright (C) 2012, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_PROFILING_EVENTS_H__ ++#define __MALI_PROFILING_EVENTS_H__ ++ ++/* Simple wrapper in order to find the OS specific location of this file */ ++#include ++ ++#endif /* __MALI_PROFILING_EVENTS_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_profiling_gator_api.h b/drivers/gpu/arm/mali400/mali/linux/mali_profiling_gator_api.h +new file mode 100755 +index 000000000000..6fdaa427c4cf +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_profiling_gator_api.h +@@ -0,0 +1,17 @@ ++/* ++ * Copyright (C) 2012-2013, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_PROFILING_GATOR_API_H__ ++#define __MALI_PROFILING_GATOR_API_H__ ++ ++/* Simple wrapper in order to find the OS specific location of this file */ ++#include ++ ++#endif /* __MALI_PROFILING_GATOR_API_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_profiling_internal.c b/drivers/gpu/arm/mali400/mali/linux/mali_profiling_internal.c +new file mode 100755 +index 000000000000..c3a526f0ad90 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_profiling_internal.c +@@ -0,0 +1,275 @@ ++/* ++ * Copyright (C) 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include "mali_kernel_common.h" ++#include "mali_osk.h" ++#include "mali_osk_mali.h" ++#include "mali_ukk.h" ++#include "mali_timestamp.h" ++#include "mali_osk_profiling.h" ++#include "mali_user_settings_db.h" ++#include "mali_profiling_internal.h" ++ ++typedef struct mali_profiling_entry { ++ u64 timestamp; ++ u32 event_id; ++ u32 data[5]; ++} mali_profiling_entry; ++ ++typedef enum mali_profiling_state { ++ MALI_PROFILING_STATE_UNINITIALIZED, ++ MALI_PROFILING_STATE_IDLE, ++ MALI_PROFILING_STATE_RUNNING, ++ MALI_PROFILING_STATE_RETURN, ++} mali_profiling_state; ++ ++static _mali_osk_mutex_t *lock = NULL; ++static mali_profiling_state prof_state = MALI_PROFILING_STATE_UNINITIALIZED; ++static mali_profiling_entry *profile_entries = NULL; ++static _mali_osk_atomic_t profile_insert_index; ++static u32 profile_mask = 0; ++ ++static inline void add_event(u32 event_id, u32 data0, u32 data1, u32 data2, u32 data3, u32 data4); ++ ++void probe_mali_timeline_event(void *data, TP_PROTO(unsigned int event_id, unsigned int d0, unsigned int d1, unsigned ++ int d2, unsigned int d3, unsigned int d4)) ++{ ++ add_event(event_id, d0, d1, d2, d3, d4); ++} ++ ++_mali_osk_errcode_t _mali_internal_profiling_init(mali_bool auto_start) ++{ ++ profile_entries = NULL; ++ profile_mask = 0; ++ _mali_osk_atomic_init(&profile_insert_index, 0); ++ ++ lock = _mali_osk_mutex_init(_MALI_OSK_LOCKFLAG_ORDERED, _MALI_OSK_LOCK_ORDER_PROFILING); ++ if (NULL == lock) { ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ prof_state = MALI_PROFILING_STATE_IDLE; ++ ++ if (MALI_TRUE == auto_start) { ++ u32 limit = MALI_PROFILING_MAX_BUFFER_ENTRIES; /* Use maximum buffer size */ ++ ++ mali_set_user_setting(_MALI_UK_USER_SETTING_SW_EVENTS_ENABLE, MALI_TRUE); ++ if (_MALI_OSK_ERR_OK != _mali_internal_profiling_start(&limit)) { ++ return _MALI_OSK_ERR_FAULT; ++ } ++ } ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++void _mali_internal_profiling_term(void) ++{ ++ u32 count; ++ ++ /* Ensure profiling is stopped */ ++ _mali_internal_profiling_stop(&count); ++ ++ prof_state = MALI_PROFILING_STATE_UNINITIALIZED; ++ ++ if (NULL != profile_entries) { ++ _mali_osk_vfree(profile_entries); ++ profile_entries = NULL; ++ } ++ ++ if (NULL != lock) { ++ _mali_osk_mutex_term(lock); ++ lock = NULL; ++ } ++} ++ ++_mali_osk_errcode_t _mali_internal_profiling_start(u32 *limit) ++{ ++ _mali_osk_errcode_t ret; ++ mali_profiling_entry *new_profile_entries; ++ ++ _mali_osk_mutex_wait(lock); ++ ++ if (MALI_PROFILING_STATE_RUNNING == prof_state) { ++ _mali_osk_mutex_signal(lock); ++ return _MALI_OSK_ERR_BUSY; ++ } ++ ++ new_profile_entries = _mali_osk_valloc(*limit * sizeof(mali_profiling_entry)); ++ ++ if (NULL == new_profile_entries) { ++ _mali_osk_mutex_signal(lock); ++ _mali_osk_vfree(new_profile_entries); ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ ++ if (MALI_PROFILING_MAX_BUFFER_ENTRIES < *limit) { ++ *limit = MALI_PROFILING_MAX_BUFFER_ENTRIES; ++ } ++ ++ profile_mask = 1; ++ while (profile_mask <= *limit) { ++ profile_mask <<= 1; ++ } ++ profile_mask >>= 1; ++ ++ *limit = profile_mask; ++ ++ profile_mask--; /* turns the power of two into a mask of one less */ ++ ++ if (MALI_PROFILING_STATE_IDLE != prof_state) { ++ _mali_osk_mutex_signal(lock); ++ _mali_osk_vfree(new_profile_entries); ++ return _MALI_OSK_ERR_INVALID_ARGS; /* invalid to call this function in this state */ ++ } ++ ++ profile_entries = new_profile_entries; ++ ++ ret = _mali_timestamp_reset(); ++ ++ if (_MALI_OSK_ERR_OK == ret) { ++ prof_state = MALI_PROFILING_STATE_RUNNING; ++ } else { ++ _mali_osk_vfree(profile_entries); ++ profile_entries = NULL; ++ } ++ ++ register_trace_mali_timeline_event(probe_mali_timeline_event, NULL); ++ ++ _mali_osk_mutex_signal(lock); ++ return ret; ++} ++ ++static inline void add_event(u32 event_id, u32 data0, u32 data1, u32 data2, u32 data3, u32 data4) ++{ ++ u32 cur_index = (_mali_osk_atomic_inc_return(&profile_insert_index) - 1) & profile_mask; ++ ++ profile_entries[cur_index].timestamp = _mali_timestamp_get(); ++ profile_entries[cur_index].event_id = event_id; ++ profile_entries[cur_index].data[0] = data0; ++ profile_entries[cur_index].data[1] = data1; ++ profile_entries[cur_index].data[2] = data2; ++ profile_entries[cur_index].data[3] = data3; ++ profile_entries[cur_index].data[4] = data4; ++ ++ /* If event is "leave API function", add current memory usage to the event ++ * as data point 4. This is used in timeline profiling to indicate how ++ * much memory was used when leaving a function. */ ++ if (event_id == (MALI_PROFILING_EVENT_TYPE_SINGLE | MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | MALI_PROFILING_EVENT_REASON_SINGLE_SW_LEAVE_API_FUNC)) { ++ profile_entries[cur_index].data[4] = _mali_ukk_report_memory_usage(); ++ } ++} ++ ++_mali_osk_errcode_t _mali_internal_profiling_stop(u32 *count) ++{ ++ _mali_osk_mutex_wait(lock); ++ ++ if (MALI_PROFILING_STATE_RUNNING != prof_state) { ++ _mali_osk_mutex_signal(lock); ++ return _MALI_OSK_ERR_INVALID_ARGS; /* invalid to call this function in this state */ ++ } ++ ++ /* go into return state (user to retreive events), no more events will be added after this */ ++ prof_state = MALI_PROFILING_STATE_RETURN; ++ ++ unregister_trace_mali_timeline_event(probe_mali_timeline_event, NULL); ++ ++ _mali_osk_mutex_signal(lock); ++ ++ tracepoint_synchronize_unregister(); ++ ++ *count = _mali_osk_atomic_read(&profile_insert_index); ++ if (*count > profile_mask) *count = profile_mask; ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++u32 _mali_internal_profiling_get_count(void) ++{ ++ u32 retval = 0; ++ ++ _mali_osk_mutex_wait(lock); ++ if (MALI_PROFILING_STATE_RETURN == prof_state) { ++ retval = _mali_osk_atomic_read(&profile_insert_index); ++ if (retval > profile_mask) retval = profile_mask; ++ } ++ _mali_osk_mutex_signal(lock); ++ ++ return retval; ++} ++ ++_mali_osk_errcode_t _mali_internal_profiling_get_event(u32 index, u64 *timestamp, u32 *event_id, u32 data[5]) ++{ ++ u32 raw_index = _mali_osk_atomic_read(&profile_insert_index); ++ ++ _mali_osk_mutex_wait(lock); ++ ++ if (index < profile_mask) { ++ if ((raw_index & ~profile_mask) != 0) { ++ index += raw_index; ++ index &= profile_mask; ++ } ++ ++ if (prof_state != MALI_PROFILING_STATE_RETURN) { ++ _mali_osk_mutex_signal(lock); ++ return _MALI_OSK_ERR_INVALID_ARGS; /* invalid to call this function in this state */ ++ } ++ ++ if (index >= raw_index) { ++ _mali_osk_mutex_signal(lock); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ *timestamp = profile_entries[index].timestamp; ++ *event_id = profile_entries[index].event_id; ++ data[0] = profile_entries[index].data[0]; ++ data[1] = profile_entries[index].data[1]; ++ data[2] = profile_entries[index].data[2]; ++ data[3] = profile_entries[index].data[3]; ++ data[4] = profile_entries[index].data[4]; ++ } else { ++ _mali_osk_mutex_signal(lock); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ _mali_osk_mutex_signal(lock); ++ return _MALI_OSK_ERR_OK; ++} ++ ++_mali_osk_errcode_t _mali_internal_profiling_clear(void) ++{ ++ _mali_osk_mutex_wait(lock); ++ ++ if (MALI_PROFILING_STATE_RETURN != prof_state) { ++ _mali_osk_mutex_signal(lock); ++ return _MALI_OSK_ERR_INVALID_ARGS; /* invalid to call this function in this state */ ++ } ++ ++ prof_state = MALI_PROFILING_STATE_IDLE; ++ profile_mask = 0; ++ _mali_osk_atomic_init(&profile_insert_index, 0); ++ ++ if (NULL != profile_entries) { ++ _mali_osk_vfree(profile_entries); ++ profile_entries = NULL; ++ } ++ ++ _mali_osk_mutex_signal(lock); ++ return _MALI_OSK_ERR_OK; ++} ++ ++mali_bool _mali_internal_profiling_is_recording(void) ++{ ++ return prof_state == MALI_PROFILING_STATE_RUNNING ? MALI_TRUE : MALI_FALSE; ++} ++ ++mali_bool _mali_internal_profiling_have_recording(void) ++{ ++ return prof_state == MALI_PROFILING_STATE_RETURN ? MALI_TRUE : MALI_FALSE; ++} +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_profiling_internal.h b/drivers/gpu/arm/mali400/mali/linux/mali_profiling_internal.h +new file mode 100755 +index 000000000000..f17b4583307a +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_profiling_internal.h +@@ -0,0 +1,35 @@ ++/* ++ * Copyright (C) 2012-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_PROFILING_INTERNAL_H__ ++#define __MALI_PROFILING_INTERNAL_H__ ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++#include "mali_osk.h" ++ ++int _mali_internal_profiling_init(mali_bool auto_start); ++void _mali_internal_profiling_term(void); ++ ++mali_bool _mali_internal_profiling_is_recording(void); ++mali_bool _mali_internal_profiling_have_recording(void); ++_mali_osk_errcode_t _mali_internal_profiling_clear(void); ++_mali_osk_errcode_t _mali_internal_profiling_get_event(u32 index, u64 *timestamp, u32 *event_id, u32 data[5]); ++u32 _mali_internal_profiling_get_count(void); ++int _mali_internal_profiling_stop(u32 *count); ++int _mali_internal_profiling_start(u32 *limit); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif /* __MALI_PROFILING_INTERNAL_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_sync.c b/drivers/gpu/arm/mali400/mali/linux/mali_sync.c +new file mode 100755 +index 000000000000..0d98b518f1ac +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_sync.c +@@ -0,0 +1,665 @@ ++/* ++ * Copyright (C) 2012-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include "mali_sync.h" ++ ++#include "mali_osk.h" ++#include "mali_kernel_common.h" ++#include "mali_timeline.h" ++#include "mali_executor.h" ++ ++#include ++#include ++#include ++ ++struct mali_sync_pt { ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) ++ struct sync_pt sync_pt; ++#else ++ struct mali_internal_sync_point sync_pt; ++#endif ++ struct mali_sync_flag *flag; ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) ++ struct sync_timeline *sync_tl; /**< Sync timeline this pt is connected to. */ ++#else ++ struct mali_internal_sync_timeline *sync_tl; /**< Sync timeline this pt is connected to. */ ++#endif ++}; ++ ++/** ++ * The sync flag is used to connect sync fences to the Mali Timeline system. Sync fences can be ++ * created from a sync flag, and when the flag is signaled, the sync fences will also be signaled. ++ */ ++struct mali_sync_flag { ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) ++ struct sync_timeline *sync_tl; /**< Sync timeline this flag is connected to. */ ++#else ++ struct mali_internal_sync_timeline *sync_tl; /**< Sync timeline this flag is connected to. */ ++#endif ++ u32 point; /**< Point on timeline. */ ++ int status; /**< 0 if unsignaled, 1 if signaled without error or negative if signaled with error. */ ++ struct kref refcount; /**< Reference count. */ ++}; ++ ++/** ++ * Mali sync timeline is used to connect mali timeline to sync_timeline. ++ * When fence timeout can print more detailed mali timeline system info. ++ */ ++struct mali_sync_timeline_container { ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) ++ struct sync_timeline sync_timeline; ++#else ++ struct mali_internal_sync_timeline sync_timeline; ++#endif ++ struct mali_timeline *timeline; ++}; ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) ++MALI_STATIC_INLINE struct mali_sync_pt *to_mali_sync_pt(struct sync_pt *pt) ++#else ++MALI_STATIC_INLINE struct mali_sync_pt *to_mali_sync_pt(struct mali_internal_sync_point *pt) ++#endif ++{ ++ return container_of(pt, struct mali_sync_pt, sync_pt); ++} ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) ++MALI_STATIC_INLINE struct mali_sync_timeline_container *to_mali_sync_tl_container(struct sync_timeline *sync_tl) ++#else ++MALI_STATIC_INLINE struct mali_sync_timeline_container *to_mali_sync_tl_container(struct mali_internal_sync_timeline *sync_tl) ++#endif ++{ ++ return container_of(sync_tl, struct mali_sync_timeline_container, sync_timeline); ++} ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) ++static int timeline_has_signaled(struct sync_pt *pt) ++#else ++static int timeline_has_signaled(struct mali_internal_sync_point *pt) ++#endif ++{ ++ struct mali_sync_pt *mpt; ++ ++ MALI_DEBUG_ASSERT_POINTER(pt); ++ mpt = to_mali_sync_pt(pt); ++ ++ MALI_DEBUG_ASSERT_POINTER(mpt->flag); ++ ++ return mpt->flag->status; ++} ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) ++static void timeline_free_pt(struct sync_pt *pt) ++#else ++static void timeline_free_pt(struct mali_internal_sync_point *pt) ++#endif ++{ ++ struct mali_sync_pt *mpt; ++ ++ MALI_DEBUG_ASSERT_POINTER(pt); ++ mpt = to_mali_sync_pt(pt); ++ ++ mali_sync_flag_put(mpt->flag); ++} ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) ++static void timeline_release(struct sync_timeline *sync_timeline) ++#else ++static void timeline_release(struct mali_internal_sync_timeline *sync_timeline) ++#endif ++{ ++ struct mali_sync_timeline_container *mali_sync_tl = NULL; ++ struct mali_timeline *mali_tl = NULL; ++ ++ MALI_DEBUG_ASSERT_POINTER(sync_timeline); ++ ++ mali_sync_tl = to_mali_sync_tl_container(sync_timeline); ++ MALI_DEBUG_ASSERT_POINTER(mali_sync_tl); ++ ++ mali_tl = mali_sync_tl->timeline; ++ ++ /* always signaled timeline didn't have mali container */ ++ if (mali_tl) { ++ if (NULL != mali_tl->spinlock) { ++ mali_spinlock_reentrant_term(mali_tl->spinlock); ++ } ++ _mali_osk_free(mali_tl); ++ } ++ ++ module_put(THIS_MODULE); ++} ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) ++static struct sync_pt *timeline_dup(struct sync_pt *pt) ++{ ++ struct mali_sync_pt *mpt, *new_mpt; ++ struct sync_pt *new_pt; ++ MALI_DEBUG_ASSERT_POINTER(pt); ++ ++ mpt = to_mali_sync_pt(pt); ++ ++ new_pt = sync_pt_create(mpt->sync_tl, sizeof(struct mali_sync_pt)); ++ if (NULL == new_pt) return NULL; ++ ++ new_mpt = to_mali_sync_pt(new_pt); ++ ++ mali_sync_flag_get(mpt->flag); ++ new_mpt->flag = mpt->flag; ++ new_mpt->sync_tl = mpt->sync_tl; ++ ++ return new_pt; ++} ++ ++static int timeline_compare(struct sync_pt *pta, struct sync_pt *ptb) ++{ ++ struct mali_sync_pt *mpta; ++ struct mali_sync_pt *mptb; ++ u32 a, b; ++ ++ MALI_DEBUG_ASSERT_POINTER(pta); ++ MALI_DEBUG_ASSERT_POINTER(ptb); ++ mpta = to_mali_sync_pt(pta); ++ mptb = to_mali_sync_pt(ptb); ++ ++ MALI_DEBUG_ASSERT_POINTER(mpta->flag); ++ MALI_DEBUG_ASSERT_POINTER(mptb->flag); ++ ++ a = mpta->flag->point; ++ b = mptb->flag->point; ++ ++ if (a == b) return 0; ++ ++ return ((b - a) < (a - b) ? -1 : 1); ++} ++#endif ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) ++static void timeline_print_pt(struct seq_file *s, struct sync_pt *sync_pt) ++{ ++ struct mali_sync_pt *mpt; ++ ++ MALI_DEBUG_ASSERT_POINTER(s); ++ MALI_DEBUG_ASSERT_POINTER(sync_pt); ++ ++ mpt = to_mali_sync_pt(sync_pt); ++ ++ /* It is possible this sync point is just under construct, ++ * make sure the flag is valid before accessing it ++ */ ++ if (mpt->flag) { ++ seq_printf(s, "%u", mpt->flag->point); ++ } else { ++ seq_printf(s, "uninitialized"); ++ } ++} ++ ++static void timeline_print_obj(struct seq_file *s, struct sync_timeline *sync_tl) ++{ ++ struct mali_sync_timeline_container *mali_sync_tl = NULL; ++ struct mali_timeline *mali_tl = NULL; ++ ++ MALI_DEBUG_ASSERT_POINTER(sync_tl); ++ ++ mali_sync_tl = to_mali_sync_tl_container(sync_tl); ++ MALI_DEBUG_ASSERT_POINTER(mali_sync_tl); ++ ++ mali_tl = mali_sync_tl->timeline; ++ ++ if (NULL != mali_tl) { ++ seq_printf(s, "oldest (%u) ", mali_tl->point_oldest); ++ seq_printf(s, "next (%u)", mali_tl->point_next); ++ seq_printf(s, "\n"); ++ ++#if defined(MALI_TIMELINE_DEBUG_FUNCTIONS) ++ { ++ u32 tid = _mali_osk_get_tid(); ++ struct mali_timeline_system *system = mali_tl->system; ++ ++ mali_spinlock_reentrant_wait(mali_tl->spinlock, tid); ++ if (!mali_tl->destroyed) { ++ mali_spinlock_reentrant_wait(system->spinlock, tid); ++ mali_timeline_debug_print_timeline(mali_tl, s); ++ mali_spinlock_reentrant_signal(system->spinlock, tid); ++ } ++ mali_spinlock_reentrant_signal(mali_tl->spinlock, tid); ++ ++ /* dump job queue status and group running status */ ++ mali_executor_status_dump(); ++ } ++#endif ++ } ++} ++#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) ++static void timeline_pt_value_str(struct sync_pt *pt, char *str, int size) ++{ ++ struct mali_sync_pt *mpt; ++ ++ MALI_DEBUG_ASSERT_POINTER(str); ++ MALI_DEBUG_ASSERT_POINTER(pt); ++ ++ mpt = to_mali_sync_pt(pt); ++ ++ /* It is possible this sync point is just under construct, ++ * make sure the flag is valid before accessing it ++ */ ++ if (mpt->flag) { ++ _mali_osk_snprintf(str, size, "%u", mpt->flag->point); ++ } else { ++ _mali_osk_snprintf(str, size, "uninitialized"); ++ } ++} ++ ++static void timeline_value_str(struct sync_timeline *timeline, char *str, int size) ++{ ++ struct mali_sync_timeline_container *mali_sync_tl = NULL; ++ struct mali_timeline *mali_tl = NULL; ++ ++ MALI_DEBUG_ASSERT_POINTER(timeline); ++ ++ mali_sync_tl = to_mali_sync_tl_container(timeline); ++ MALI_DEBUG_ASSERT_POINTER(mali_sync_tl); ++ ++ mali_tl = mali_sync_tl->timeline; ++ ++ if (NULL != mali_tl) { ++ _mali_osk_snprintf(str, size, "oldest (%u) ", mali_tl->point_oldest); ++ _mali_osk_snprintf(str, size, "next (%u)", mali_tl->point_next); ++ _mali_osk_snprintf(str, size, "\n"); ++ ++#if defined(MALI_TIMELINE_DEBUG_FUNCTIONS) ++ { ++ u32 tid = _mali_osk_get_tid(); ++ struct mali_timeline_system *system = mali_tl->system; ++ ++ mali_spinlock_reentrant_wait(mali_tl->spinlock, tid); ++ if (!mali_tl->destroyed) { ++ mali_spinlock_reentrant_wait(system->spinlock, tid); ++ mali_timeline_debug_direct_print_timeline(mali_tl); ++ mali_spinlock_reentrant_signal(system->spinlock, tid); ++ } ++ mali_spinlock_reentrant_signal(mali_tl->spinlock, tid); ++ ++ /* dump job queue status and group running status */ ++ mali_executor_status_dump(); ++ } ++#endif ++ } ++} ++#else ++static void timeline_print_sync_pt(struct mali_internal_sync_point *sync_pt) ++{ ++ struct mali_sync_pt *mpt; ++ ++ MALI_DEBUG_ASSERT_POINTER(sync_pt); ++ ++ mpt = to_mali_sync_pt(sync_pt); ++ ++ if (mpt->flag) { ++ MALI_DEBUG_PRINT(2, ("mali_internal_sync_pt: %u\n", mpt->flag->point)); ++ } else { ++ MALI_DEBUG_PRINT(2, ("uninitialized\n", mpt->flag->point)); ++ } ++} ++#endif ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) ++static struct sync_timeline_ops mali_timeline_ops = { ++ .driver_name = "Mali", ++ .dup = timeline_dup, ++ .has_signaled = timeline_has_signaled, ++ .compare = timeline_compare, ++ .free_pt = timeline_free_pt, ++ .release_obj = timeline_release, ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) ++ .print_pt = timeline_print_pt, ++ .print_obj = timeline_print_obj, ++#else ++ .pt_value_str = timeline_pt_value_str, ++ .timeline_value_str = timeline_value_str, ++#endif ++}; ++ ++struct sync_timeline *mali_sync_timeline_create(struct mali_timeline *timeline, const char *name) ++{ ++ struct sync_timeline *sync_tl; ++ struct mali_sync_timeline_container *mali_sync_tl; ++ ++ sync_tl = sync_timeline_create(&mali_timeline_ops, sizeof(struct mali_sync_timeline_container), name); ++ if (NULL == sync_tl) return NULL; ++ ++ mali_sync_tl = to_mali_sync_tl_container(sync_tl); ++ mali_sync_tl->timeline = timeline; ++ ++ /* Grab a reference on the module to ensure the callbacks are present ++ * as long some timeline exists. The reference is released when the ++ * timeline is freed. ++ * Since this function is called from a ioctl on an open file we know ++ * we already have a reference, so using __module_get is safe. */ ++ __module_get(THIS_MODULE); ++ ++ return sync_tl; ++} ++ ++s32 mali_sync_fence_fd_alloc(struct sync_fence *sync_fence) ++{ ++ s32 fd = -1; ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 19, 0) ++ fd = get_unused_fd(); ++#else ++ fd = get_unused_fd_flags(0); ++#endif ++ ++ if (fd < 0) { ++ sync_fence_put(sync_fence); ++ return -1; ++ } ++ sync_fence_install(sync_fence, fd); ++ ++ return fd; ++} ++ ++struct sync_fence *mali_sync_fence_merge(struct sync_fence *sync_fence1, struct sync_fence *sync_fence2) ++{ ++ struct sync_fence *sync_fence; ++ ++ MALI_DEBUG_ASSERT_POINTER(sync_fence1); ++ MALI_DEBUG_ASSERT_POINTER(sync_fence1); ++ ++ sync_fence = sync_fence_merge("mali_merge_fence", sync_fence1, sync_fence2); ++ sync_fence_put(sync_fence1); ++ sync_fence_put(sync_fence2); ++ ++ return sync_fence; ++} ++ ++struct sync_fence *mali_sync_timeline_create_signaled_fence(struct sync_timeline *sync_tl) ++{ ++ struct mali_sync_flag *flag; ++ struct sync_fence *sync_fence; ++ ++ MALI_DEBUG_ASSERT_POINTER(sync_tl); ++ ++ flag = mali_sync_flag_create(sync_tl, 0); ++ if (NULL == flag) return NULL; ++ ++ sync_fence = mali_sync_flag_create_fence(flag); ++ ++ mali_sync_flag_signal(flag, 0); ++ mali_sync_flag_put(flag); ++ ++ return sync_fence; ++} ++ ++struct mali_sync_flag *mali_sync_flag_create(struct sync_timeline *sync_tl, mali_timeline_point point) ++{ ++ struct mali_sync_flag *flag; ++ ++ if (NULL == sync_tl) return NULL; ++ ++ flag = _mali_osk_calloc(1, sizeof(*flag)); ++ if (NULL == flag) return NULL; ++ ++ flag->sync_tl = sync_tl; ++ flag->point = point; ++ ++ flag->status = 0; ++ kref_init(&flag->refcount); ++ ++ return flag; ++} ++ ++/** ++ * Create a sync point attached to given sync flag. ++ * ++ * @note Sync points must be triggered in *exactly* the same order as they are created. ++ * ++ * @param flag Sync flag. ++ * @return New sync point if successful, NULL if not. ++ */ ++static struct sync_pt *mali_sync_flag_create_pt(struct mali_sync_flag *flag) ++{ ++ struct sync_pt *pt; ++ struct mali_sync_pt *mpt; ++ ++ MALI_DEBUG_ASSERT_POINTER(flag); ++ MALI_DEBUG_ASSERT_POINTER(flag->sync_tl); ++ ++ pt = sync_pt_create(flag->sync_tl, sizeof(struct mali_sync_pt)); ++ if (NULL == pt) return NULL; ++ ++ mali_sync_flag_get(flag); ++ ++ mpt = to_mali_sync_pt(pt); ++ mpt->flag = flag; ++ mpt->sync_tl = flag->sync_tl; ++ ++ return pt; ++} ++ ++struct sync_fence *mali_sync_flag_create_fence(struct mali_sync_flag *flag) ++{ ++ struct sync_pt *sync_pt; ++ struct sync_fence *sync_fence; ++ ++ MALI_DEBUG_ASSERT_POINTER(flag); ++ MALI_DEBUG_ASSERT_POINTER(flag->sync_tl); ++ ++ sync_pt = mali_sync_flag_create_pt(flag); ++ if (NULL == sync_pt) return NULL; ++ ++ sync_fence = sync_fence_create("mali_flag_fence", sync_pt); ++ if (NULL == sync_fence) { ++ sync_pt_free(sync_pt); ++ return NULL; ++ } ++ ++ return sync_fence; ++} ++#else ++static struct mali_internal_sync_timeline_ops mali_timeline_ops = { ++ .driver_name = "Mali", ++ .has_signaled = timeline_has_signaled, ++ .free_pt = timeline_free_pt, ++ .release_obj = timeline_release, ++ .print_sync_pt = timeline_print_sync_pt, ++}; ++ ++struct mali_internal_sync_timeline *mali_sync_timeline_create(struct mali_timeline *timeline, const char *name) ++{ ++ struct mali_internal_sync_timeline *sync_tl; ++ struct mali_sync_timeline_container *mali_sync_tl; ++ ++ sync_tl = mali_internal_sync_timeline_create(&mali_timeline_ops, sizeof(struct mali_sync_timeline_container), name); ++ if (NULL == sync_tl) return NULL; ++ ++ mali_sync_tl = to_mali_sync_tl_container(sync_tl); ++ mali_sync_tl->timeline = timeline; ++ ++ /* Grab a reference on the module to ensure the callbacks are present ++ * as long some timeline exists. The reference is released when the ++ * timeline is freed. ++ * Since this function is called from a ioctl on an open file we know ++ * we already have a reference, so using __module_get is safe. */ ++ __module_get(THIS_MODULE); ++ ++ return sync_tl; ++} ++ ++s32 mali_sync_fence_fd_alloc(struct mali_internal_sync_fence *sync_fence) ++{ ++ s32 fd = -1; ++ ++ fd = get_unused_fd_flags(0); ++ ++ if (fd < 0) { ++ fput(sync_fence->file); ++ return -1; ++ } ++ fd_install(fd, sync_fence->file); ++ return fd; ++} ++ ++struct mali_internal_sync_fence *mali_sync_fence_merge(struct mali_internal_sync_fence *sync_fence1, struct mali_internal_sync_fence *sync_fence2) ++{ ++ struct mali_internal_sync_fence *sync_fence; ++ ++ MALI_DEBUG_ASSERT_POINTER(sync_fence1); ++ MALI_DEBUG_ASSERT_POINTER(sync_fence1); ++ ++ sync_fence = mali_internal_sync_fence_merge(sync_fence1, sync_fence2); ++ fput(sync_fence1->file); ++ fput(sync_fence2->file); ++ ++ return sync_fence; ++} ++ ++struct mali_internal_sync_fence *mali_sync_timeline_create_signaled_fence(struct mali_internal_sync_timeline *sync_tl) ++{ ++ struct mali_sync_flag *flag; ++ struct mali_internal_sync_fence *sync_fence; ++ ++ MALI_DEBUG_ASSERT_POINTER(sync_tl); ++ ++ flag = mali_sync_flag_create(sync_tl, 0); ++ if (NULL == flag) return NULL; ++ ++ sync_fence = mali_sync_flag_create_fence(flag); ++ ++ mali_sync_flag_signal(flag, 0); ++ mali_sync_flag_put(flag); ++ ++ return sync_fence; ++} ++ ++struct mali_sync_flag *mali_sync_flag_create(struct mali_internal_sync_timeline *sync_tl, mali_timeline_point point) ++{ ++ struct mali_sync_flag *flag; ++ ++ if (NULL == sync_tl) return NULL; ++ ++ flag = _mali_osk_calloc(1, sizeof(*flag)); ++ if (NULL == flag) return NULL; ++ ++ flag->sync_tl = sync_tl; ++ flag->point = point; ++ ++ flag->status = 0; ++ kref_init(&flag->refcount); ++ ++ return flag; ++} ++ ++/** ++ * Create a sync point attached to given sync flag. ++ * ++ * @note Sync points must be triggered in *exactly* the same order as they are created. ++ * ++ * @param flag Sync flag. ++ * @return New sync point if successful, NULL if not. ++ */ ++static struct mali_internal_sync_point *mali_sync_flag_create_pt(struct mali_sync_flag *flag) ++{ ++ struct mali_internal_sync_point *pt; ++ struct mali_sync_pt *mpt; ++ ++ MALI_DEBUG_ASSERT_POINTER(flag); ++ MALI_DEBUG_ASSERT_POINTER(flag->sync_tl); ++ ++ pt = mali_internal_sync_point_create(flag->sync_tl, sizeof(struct mali_sync_pt)); ++ ++ if (pt == NULL) { ++ MALI_PRINT_ERROR(("Mali sync: sync_pt creation failed\n")); ++ return NULL; ++ } ++ mali_sync_flag_get(flag); ++ ++ mpt = to_mali_sync_pt(pt); ++ mpt->flag = flag; ++ mpt->sync_tl = flag->sync_tl; ++ ++ return pt; ++} ++ ++struct mali_internal_sync_fence *mali_sync_flag_create_fence(struct mali_sync_flag *flag) ++{ ++ struct mali_internal_sync_point *sync_pt; ++ struct mali_internal_sync_fence *sync_fence; ++ ++ MALI_DEBUG_ASSERT_POINTER(flag); ++ MALI_DEBUG_ASSERT_POINTER(flag->sync_tl); ++ ++ sync_pt = mali_sync_flag_create_pt(flag); ++ if (NULL == sync_pt) { ++ MALI_PRINT_ERROR(("Mali sync: sync_pt creation failed\n")); ++ return NULL; ++ } ++ sync_fence = (struct mali_internal_sync_fence *)sync_file_create(&sync_pt->base); ++ if (NULL == sync_fence) { ++ MALI_PRINT_ERROR(("Mali sync: sync_fence creation failed\n")); ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) ++ dma_fence_put(&sync_pt->base); ++#else ++ fence_put(&sync_pt->base); ++#endif ++ return NULL; ++ } ++ ++ /* 'sync_pt' no longer needs to hold a refcount of '*sync_pt', to put it off. */ ++ dma_fence_put(&sync_pt->base); ++ sync_pt = NULL; ++ ++ return sync_fence; ++} ++#endif ++ ++void mali_sync_flag_get(struct mali_sync_flag *flag) ++{ ++ MALI_DEBUG_ASSERT_POINTER(flag); ++ kref_get(&flag->refcount); ++} ++ ++/** ++ * Free sync flag. ++ * ++ * @param ref kref object embedded in sync flag that should be freed. ++ */ ++static void mali_sync_flag_free(struct kref *ref) ++{ ++ struct mali_sync_flag *flag; ++ ++ MALI_DEBUG_ASSERT_POINTER(ref); ++ flag = container_of(ref, struct mali_sync_flag, refcount); ++ ++ _mali_osk_free(flag); ++} ++ ++void mali_sync_flag_put(struct mali_sync_flag *flag) ++{ ++ MALI_DEBUG_ASSERT_POINTER(flag); ++ kref_put(&flag->refcount, mali_sync_flag_free); ++} ++ ++void mali_sync_flag_signal(struct mali_sync_flag *flag, int error) ++{ ++ MALI_DEBUG_ASSERT_POINTER(flag); ++ ++ MALI_DEBUG_ASSERT(0 == flag->status); ++ flag->status = (0 > error) ? error : 1; ++ ++ _mali_osk_write_mem_barrier(); ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) ++ sync_timeline_signal(flag->sync_tl); ++#else ++ mali_internal_sync_timeline_signal(flag->sync_tl); ++#endif ++} ++ ++ +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_sync.h b/drivers/gpu/arm/mali400/mali/linux/mali_sync.h +new file mode 100755 +index 000000000000..91be8b9cf314 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_sync.h +@@ -0,0 +1,169 @@ ++/* ++ * Copyright (C) 2012-2015, 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file mali_sync.h ++ * ++ * Mali interface for Linux sync objects. ++ */ ++ ++#ifndef _MALI_SYNC_H_ ++#define _MALI_SYNC_H_ ++ ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ ++#include ++#include ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 10, 0) ++#include ++#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) ++#include ++#else ++#include "mali_internal_sync.h" ++#endif ++ ++ ++#include "mali_osk.h" ++ ++struct mali_sync_flag; ++struct mali_timeline; ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) ++/** ++ * Create a sync timeline. ++ * ++ * @param name Name of the sync timeline. ++ * @return The new sync timeline if successful, NULL if not. ++ */ ++struct sync_timeline *mali_sync_timeline_create(struct mali_timeline *timeline, const char *name); ++ ++/** ++ * Creates a file descriptor representing the sync fence. Will release sync fence if allocation of ++ * file descriptor fails. ++ * ++ * @param sync_fence Sync fence. ++ * @return File descriptor representing sync fence if successful, or -1 if not. ++ */ ++s32 mali_sync_fence_fd_alloc(struct sync_fence *sync_fence); ++ ++/** ++ * Merges two sync fences. Both input sync fences will be released. ++ * ++ * @param sync_fence1 First sync fence. ++ * @param sync_fence2 Second sync fence. ++ * @return New sync fence that is the result of the merger if successful, or NULL if not. ++ */ ++struct sync_fence *mali_sync_fence_merge(struct sync_fence *sync_fence1, struct sync_fence *sync_fence2); ++ ++/** ++ * Create a sync fence that is already signaled. ++ * ++ * @param tl Sync timeline. ++ * @return New signaled sync fence if successful, NULL if not. ++ */ ++struct sync_fence *mali_sync_timeline_create_signaled_fence(struct sync_timeline *sync_tl); ++ ++ ++/** ++ * Create a sync flag. ++ * ++ * @param sync_tl Sync timeline. ++ * @param point Point on Mali timeline. ++ * @return New sync flag if successful, NULL if not. ++ */ ++struct mali_sync_flag *mali_sync_flag_create(struct sync_timeline *sync_tl, u32 point); ++ ++/** ++ * Create a sync fence attached to given sync flag. ++ * ++ * @param flag Sync flag. ++ * @return New sync fence if successful, NULL if not. ++ */ ++struct sync_fence *mali_sync_flag_create_fence(struct mali_sync_flag *flag); ++#else ++/** ++ * Create a sync timeline. ++ * ++ * @param name Name of the sync timeline. ++ * @return The new sync timeline if successful, NULL if not. ++ */ ++struct mali_internal_sync_timeline *mali_sync_timeline_create(struct mali_timeline *timeline, const char *name); ++ ++/** ++ * Creates a file descriptor representing the sync fence. Will release sync fence if allocation of ++ * file descriptor fails. ++ * ++ * @param sync_fence Sync fence. ++ * @return File descriptor representing sync fence if successful, or -1 if not. ++ */ ++s32 mali_sync_fence_fd_alloc(struct mali_internal_sync_fence *sync_fence); ++ ++/** ++ * Merges two sync fences. Both input sync fences will be released. ++ * ++ * @param sync_fence1 First sync fence. ++ * @param sync_fence2 Second sync fence. ++ * @return New sync fence that is the result of the merger if successful, or NULL if not. ++ */ ++struct mali_internal_sync_fence *mali_sync_fence_merge(struct mali_internal_sync_fence *sync_fence1, struct mali_internal_sync_fence *sync_fence2); ++ ++/** ++ * Create a sync fence that is already signaled. ++ * ++ * @param tl Sync timeline. ++ * @return New signaled sync fence if successful, NULL if not. ++ */ ++struct mali_internal_sync_fence *mali_sync_timeline_create_signaled_fence(struct mali_internal_sync_timeline *sync_tl); ++ ++ ++/** ++ * Create a sync flag. ++ * ++ * @param sync_tl Sync timeline. ++ * @param point Point on Mali timeline. ++ * @return New sync flag if successful, NULL if not. ++ */ ++struct mali_sync_flag *mali_sync_flag_create(struct mali_internal_sync_timeline *sync_tl, u32 point); ++ ++/** ++ * Create a sync fence attached to given sync flag. ++ * ++ * @param flag Sync flag. ++ * @return New sync fence if successful, NULL if not. ++ */ ++struct mali_internal_sync_fence *mali_sync_flag_create_fence(struct mali_sync_flag *flag); ++ ++#endif ++/** ++ * Grab sync flag reference. ++ * ++ * @param flag Sync flag. ++ */ ++void mali_sync_flag_get(struct mali_sync_flag *flag); ++ ++/** ++ * Release sync flag reference. If this was the last reference, the sync flag will be freed. ++ * ++ * @param flag Sync flag. ++ */ ++void mali_sync_flag_put(struct mali_sync_flag *flag); ++ ++/** ++ * Signal sync flag. All sync fences created from this flag will be signaled. ++ * ++ * @param flag Sync flag to signal. ++ * @param error Negative error code, or 0 if no error. ++ */ ++void mali_sync_flag_signal(struct mali_sync_flag *flag, int error); ++ ++#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ ++ ++#endif /* _MALI_SYNC_H_ */ +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_uk_types.h b/drivers/gpu/arm/mali400/mali/linux/mali_uk_types.h +new file mode 100755 +index 000000000000..68b27b8be067 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_uk_types.h +@@ -0,0 +1,17 @@ ++/* ++ * Copyright (C) 2012, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_UK_TYPES_H__ ++#define __MALI_UK_TYPES_H__ ++ ++/* Simple wrapper in order to find the OS specific location of this file */ ++#include ++ ++#endif /* __MALI_UK_TYPES_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_core.c b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_core.c +new file mode 100755 +index 000000000000..0bd1cddb10c6 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_core.c +@@ -0,0 +1,171 @@ ++/* ++ * Copyright (C) 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++#include /* file system operations */ ++#include /* memort allocation functions */ ++#include /* user space access */ ++ ++#include "mali_ukk.h" ++#include "mali_osk.h" ++#include "mali_kernel_common.h" ++#include "mali_session.h" ++#include "mali_ukk_wrappers.h" ++ ++int get_api_version_wrapper(struct mali_session_data *session_data, _mali_uk_get_api_version_s __user *uargs) ++{ ++ _mali_uk_get_api_version_s kargs; ++ _mali_osk_errcode_t err; ++ ++ MALI_CHECK_NON_NULL(uargs, -EINVAL); ++ ++ if (0 != get_user(kargs.version, &uargs->version)) return -EFAULT; ++ ++ kargs.ctx = (uintptr_t)session_data; ++ err = _mali_ukk_get_api_version(&kargs); ++ if (_MALI_OSK_ERR_OK != err) return map_errcode(err); ++ ++ if (0 != put_user(kargs.version, &uargs->version)) return -EFAULT; ++ if (0 != put_user(kargs.compatible, &uargs->compatible)) return -EFAULT; ++ ++ return 0; ++} ++ ++int get_api_version_v2_wrapper(struct mali_session_data *session_data, _mali_uk_get_api_version_v2_s __user *uargs) ++{ ++ _mali_uk_get_api_version_v2_s kargs; ++ _mali_osk_errcode_t err; ++ ++ MALI_CHECK_NON_NULL(uargs, -EINVAL); ++ ++ if (0 != get_user(kargs.version, &uargs->version)) return -EFAULT; ++ ++ kargs.ctx = (uintptr_t)session_data; ++ err = _mali_ukk_get_api_version_v2(&kargs); ++ if (_MALI_OSK_ERR_OK != err) return map_errcode(err); ++ ++ if (0 != put_user(kargs.version, &uargs->version)) return -EFAULT; ++ if (0 != put_user(kargs.compatible, &uargs->compatible)) return -EFAULT; ++ ++ return 0; ++} ++ ++/* rk_ext : 从对 r5p0-01rel0 集æˆå¼€å§‹, ä¸å†ä½¿ç”¨. */ ++#if 0 ++#define mali400_in_rk30_version 0x01 ++int get_mali_version_in_rk30_wrapper(struct mali_session_data *session_data, _mali_uk_get_mali_version_in_rk30_s __user *uargs) ++{ ++ _mali_uk_get_mali_version_in_rk30_s kargs; ++ MALI_CHECK_NON_NULL(uargs, -EINVAL); ++ kargs.ctx = (uintptr_t)session_data; ++ kargs.version = mali400_in_rk30_version; ++ if (0 != put_user(kargs.version, &uargs->version)) return -EFAULT; ++ return 0; ++} ++#else ++#include "../platform/rk/rk_ext.h" ++int get_rk_ko_version_wrapper(struct mali_session_data *session_data, _mali_rk_ko_version_s __user *uargs) ++{ ++ _mali_rk_ko_version_s kargs; ++ MALI_CHECK_NON_NULL(uargs, -EINVAL); ++ kargs.ctx = (uintptr_t)session_data; ++ kargs.version = RK_KO_VER; ++ if (0 != put_user(kargs.version, &uargs->version)) return -EFAULT; ++ return 0; ++} ++#endif ++ ++int wait_for_notification_wrapper(struct mali_session_data *session_data, _mali_uk_wait_for_notification_s __user *uargs) ++{ ++ _mali_uk_wait_for_notification_s kargs; ++ _mali_osk_errcode_t err; ++ ++ MALI_CHECK_NON_NULL(uargs, -EINVAL); ++ ++ kargs.ctx = (uintptr_t)session_data; ++ err = _mali_ukk_wait_for_notification(&kargs); ++ if (_MALI_OSK_ERR_OK != err) return map_errcode(err); ++ ++ if (_MALI_NOTIFICATION_CORE_SHUTDOWN_IN_PROGRESS != kargs.type) { ++ kargs.ctx = (uintptr_t)NULL; /* prevent kernel address to be returned to user space */ ++ if (0 != copy_to_user(uargs, &kargs, sizeof(_mali_uk_wait_for_notification_s))) return -EFAULT; ++ } else { ++ if (0 != put_user(kargs.type, &uargs->type)) return -EFAULT; ++ } ++ ++ return 0; ++} ++ ++int post_notification_wrapper(struct mali_session_data *session_data, _mali_uk_post_notification_s __user *uargs) ++{ ++ _mali_uk_post_notification_s kargs; ++ _mali_osk_errcode_t err; ++ ++ MALI_CHECK_NON_NULL(uargs, -EINVAL); ++ ++ kargs.ctx = (uintptr_t)session_data; ++ ++ if (0 != get_user(kargs.type, &uargs->type)) { ++ return -EFAULT; ++ } ++ ++ err = _mali_ukk_post_notification(&kargs); ++ if (_MALI_OSK_ERR_OK != err) { ++ return map_errcode(err); ++ } ++ ++ return 0; ++} ++ ++int get_user_settings_wrapper(struct mali_session_data *session_data, _mali_uk_get_user_settings_s __user *uargs) ++{ ++ _mali_uk_get_user_settings_s kargs; ++ _mali_osk_errcode_t err; ++ ++ MALI_CHECK_NON_NULL(uargs, -EINVAL); ++ ++ kargs.ctx = (uintptr_t)session_data; ++ err = _mali_ukk_get_user_settings(&kargs); ++ if (_MALI_OSK_ERR_OK != err) { ++ return map_errcode(err); ++ } ++ ++ kargs.ctx = 0; /* prevent kernel address to be returned to user space */ ++ if (0 != copy_to_user(uargs, &kargs, sizeof(_mali_uk_get_user_settings_s))) return -EFAULT; ++ ++ return 0; ++} ++ ++int request_high_priority_wrapper(struct mali_session_data *session_data, _mali_uk_request_high_priority_s __user *uargs) ++{ ++ _mali_uk_request_high_priority_s kargs; ++ _mali_osk_errcode_t err; ++ ++ MALI_CHECK_NON_NULL(uargs, -EINVAL); ++ ++ kargs.ctx = (uintptr_t)session_data; ++ err = _mali_ukk_request_high_priority(&kargs); ++ ++ kargs.ctx = 0; ++ ++ return map_errcode(err); ++} ++ ++int pending_submit_wrapper(struct mali_session_data *session_data, _mali_uk_pending_submit_s __user *uargs) ++{ ++ _mali_uk_pending_submit_s kargs; ++ _mali_osk_errcode_t err; ++ ++ MALI_CHECK_NON_NULL(uargs, -EINVAL); ++ ++ kargs.ctx = (uintptr_t)session_data; ++ err = _mali_ukk_pending_submit(&kargs); ++ if (_MALI_OSK_ERR_OK != err) return map_errcode(err); ++ ++ return 0; ++} +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_gp.c b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_gp.c +new file mode 100755 +index 000000000000..68fcd971938a +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_gp.c +@@ -0,0 +1,91 @@ ++/* ++ * Copyright (C) 2010, 2012-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++#include /* file system operations */ ++#include /* user space access */ ++ ++#include "mali_ukk.h" ++#include "mali_osk.h" ++#include "mali_kernel_common.h" ++#include "mali_session.h" ++#include "mali_ukk_wrappers.h" ++ ++int gp_start_job_wrapper(struct mali_session_data *session_data, _mali_uk_gp_start_job_s __user *uargs) ++{ ++ _mali_osk_errcode_t err; ++ ++ /* If the job was started successfully, 0 is returned. If there was an error, but the job ++ * was started, we return -ENOENT. For anything else returned, the job was not started. */ ++ ++ MALI_CHECK_NON_NULL(uargs, -EINVAL); ++ MALI_CHECK_NON_NULL(session_data, -EINVAL); ++ ++ err = _mali_ukk_gp_start_job(session_data, uargs); ++ if (_MALI_OSK_ERR_OK != err) return map_errcode(err); ++ ++ return 0; ++} ++ ++int gp_get_core_version_wrapper(struct mali_session_data *session_data, _mali_uk_get_gp_core_version_s __user *uargs) ++{ ++ _mali_uk_get_gp_core_version_s kargs; ++ _mali_osk_errcode_t err; ++ ++ MALI_CHECK_NON_NULL(uargs, -EINVAL); ++ MALI_CHECK_NON_NULL(session_data, -EINVAL); ++ ++ kargs.ctx = (uintptr_t)session_data; ++ err = _mali_ukk_get_gp_core_version(&kargs); ++ if (_MALI_OSK_ERR_OK != err) return map_errcode(err); ++ ++ /* no known transactions to roll-back */ ++ ++ if (0 != put_user(kargs.version, &uargs->version)) return -EFAULT; ++ ++ return 0; ++} ++ ++int gp_suspend_response_wrapper(struct mali_session_data *session_data, _mali_uk_gp_suspend_response_s __user *uargs) ++{ ++ _mali_uk_gp_suspend_response_s kargs; ++ _mali_osk_errcode_t err; ++ ++ MALI_CHECK_NON_NULL(uargs, -EINVAL); ++ MALI_CHECK_NON_NULL(session_data, -EINVAL); ++ ++ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_gp_suspend_response_s))) return -EFAULT; ++ ++ kargs.ctx = (uintptr_t)session_data; ++ err = _mali_ukk_gp_suspend_response(&kargs); ++ if (_MALI_OSK_ERR_OK != err) return map_errcode(err); ++ ++ if (0 != put_user(kargs.cookie, &uargs->cookie)) return -EFAULT; ++ ++ /* no known transactions to roll-back */ ++ return 0; ++} ++ ++int gp_get_number_of_cores_wrapper(struct mali_session_data *session_data, _mali_uk_get_gp_number_of_cores_s __user *uargs) ++{ ++ _mali_uk_get_gp_number_of_cores_s kargs; ++ _mali_osk_errcode_t err; ++ ++ MALI_CHECK_NON_NULL(uargs, -EINVAL); ++ MALI_CHECK_NON_NULL(session_data, -EINVAL); ++ ++ kargs.ctx = (uintptr_t)session_data; ++ err = _mali_ukk_get_gp_number_of_cores(&kargs); ++ if (_MALI_OSK_ERR_OK != err) return map_errcode(err); ++ ++ /* no known transactions to roll-back */ ++ ++ if (0 != put_user(kargs.number_of_cores, &uargs->number_of_cores)) return -EFAULT; ++ ++ return 0; ++} +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_mem.c b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_mem.c +new file mode 100755 +index 000000000000..baea4c688db9 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_mem.c +@@ -0,0 +1,333 @@ ++/* ++ * Copyright (C) 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++#include /* file system operations */ ++#include /* user space access */ ++ ++#include "mali_ukk.h" ++#include "mali_osk.h" ++#include "mali_kernel_common.h" ++#include "mali_session.h" ++#include "mali_ukk_wrappers.h" ++ ++int mem_alloc_wrapper(struct mali_session_data *session_data, _mali_uk_alloc_mem_s __user *uargs) ++{ ++ _mali_uk_alloc_mem_s kargs; ++ _mali_osk_errcode_t err; ++ ++ MALI_CHECK_NON_NULL(uargs, -EINVAL); ++ MALI_CHECK_NON_NULL(session_data, -EINVAL); ++ ++ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_alloc_mem_s))) { ++ return -EFAULT; ++ } ++ kargs.ctx = (uintptr_t)session_data; ++ ++ err = _mali_ukk_mem_allocate(&kargs); ++ ++ if (_MALI_OSK_ERR_OK != err) { ++ return map_errcode(err); ++ } ++ ++ if (0 != put_user(kargs.backend_handle, &uargs->backend_handle)) { ++ return -EFAULT; ++ } ++ ++ return 0; ++} ++ ++int mem_free_wrapper(struct mali_session_data *session_data, _mali_uk_free_mem_s __user *uargs) ++{ ++ _mali_uk_free_mem_s kargs; ++ _mali_osk_errcode_t err; ++ ++ MALI_CHECK_NON_NULL(uargs, -EINVAL); ++ MALI_CHECK_NON_NULL(session_data, -EINVAL); ++ ++ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_free_mem_s))) { ++ return -EFAULT; ++ } ++ kargs.ctx = (uintptr_t)session_data; ++ ++ err = _mali_ukk_mem_free(&kargs); ++ ++ if (_MALI_OSK_ERR_OK != err) { ++ return map_errcode(err); ++ } ++ ++ if (0 != put_user(kargs.free_pages_nr, &uargs->free_pages_nr)) { ++ return -EFAULT; ++ } ++ ++ return 0; ++} ++ ++int mem_bind_wrapper(struct mali_session_data *session_data, _mali_uk_bind_mem_s __user *uargs) ++{ ++ _mali_uk_bind_mem_s kargs; ++ _mali_osk_errcode_t err; ++ ++ MALI_CHECK_NON_NULL(uargs, -EINVAL); ++ MALI_CHECK_NON_NULL(session_data, -EINVAL); ++ ++ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_bind_mem_s))) { ++ return -EFAULT; ++ } ++ kargs.ctx = (uintptr_t)session_data; ++ ++ err = _mali_ukk_mem_bind(&kargs); ++ ++ if (_MALI_OSK_ERR_OK != err) { ++ return map_errcode(err); ++ } ++ ++ return 0; ++} ++ ++int mem_unbind_wrapper(struct mali_session_data *session_data, _mali_uk_unbind_mem_s __user *uargs) ++{ ++ _mali_uk_unbind_mem_s kargs; ++ _mali_osk_errcode_t err; ++ ++ MALI_CHECK_NON_NULL(uargs, -EINVAL); ++ MALI_CHECK_NON_NULL(session_data, -EINVAL); ++ ++ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_unbind_mem_s))) { ++ return -EFAULT; ++ } ++ kargs.ctx = (uintptr_t)session_data; ++ ++ err = _mali_ukk_mem_unbind(&kargs); ++ ++ if (_MALI_OSK_ERR_OK != err) { ++ return map_errcode(err); ++ } ++ ++ return 0; ++} ++ ++ ++int mem_cow_wrapper(struct mali_session_data *session_data, _mali_uk_cow_mem_s __user *uargs) ++{ ++ _mali_uk_cow_mem_s kargs; ++ _mali_osk_errcode_t err; ++ ++ MALI_CHECK_NON_NULL(uargs, -EINVAL); ++ MALI_CHECK_NON_NULL(session_data, -EINVAL); ++ ++ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_cow_mem_s))) { ++ return -EFAULT; ++ } ++ kargs.ctx = (uintptr_t)session_data; ++ ++ err = _mali_ukk_mem_cow(&kargs); ++ ++ if (_MALI_OSK_ERR_OK != err) { ++ return map_errcode(err); ++ } ++ ++ if (0 != put_user(kargs.backend_handle, &uargs->backend_handle)) { ++ return -EFAULT; ++ } ++ ++ return 0; ++} ++ ++int mem_cow_modify_range_wrapper(struct mali_session_data *session_data, _mali_uk_cow_modify_range_s __user *uargs) ++{ ++ _mali_uk_cow_modify_range_s kargs; ++ _mali_osk_errcode_t err; ++ ++ MALI_CHECK_NON_NULL(uargs, -EINVAL); ++ MALI_CHECK_NON_NULL(session_data, -EINVAL); ++ ++ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_cow_modify_range_s))) { ++ return -EFAULT; ++ } ++ kargs.ctx = (uintptr_t)session_data; ++ ++ err = _mali_ukk_mem_cow_modify_range(&kargs); ++ ++ if (_MALI_OSK_ERR_OK != err) { ++ return map_errcode(err); ++ } ++ ++ if (0 != put_user(kargs.change_pages_nr, &uargs->change_pages_nr)) { ++ return -EFAULT; ++ } ++ return 0; ++} ++ ++ ++int mem_resize_mem_wrapper(struct mali_session_data *session_data, _mali_uk_mem_resize_s __user *uargs) ++{ ++ _mali_uk_mem_resize_s kargs; ++ _mali_osk_errcode_t err; ++ ++ MALI_CHECK_NON_NULL(uargs, -EINVAL); ++ MALI_CHECK_NON_NULL(session_data, -EINVAL); ++ ++ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_mem_resize_s))) { ++ return -EFAULT; ++ } ++ kargs.ctx = (uintptr_t)session_data; ++ ++ err = _mali_ukk_mem_resize(&kargs); ++ ++ if (_MALI_OSK_ERR_OK != err) { ++ return map_errcode(err); ++ } ++ ++ return 0; ++} ++ ++int mem_write_safe_wrapper(struct mali_session_data *session_data, _mali_uk_mem_write_safe_s __user *uargs) ++{ ++ _mali_uk_mem_write_safe_s kargs; ++ _mali_osk_errcode_t err; ++ ++ MALI_CHECK_NON_NULL(uargs, -EINVAL); ++ MALI_CHECK_NON_NULL(session_data, -EINVAL); ++ ++ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_mem_write_safe_s))) { ++ return -EFAULT; ++ } ++ ++ kargs.ctx = (uintptr_t)session_data; ++ ++ /* Check if we can access the buffers */ ++ if (!access_ok((const void *)(uintptr_t)kargs.dest, kargs.size) || ++ !access_ok((const void *)(uintptr_t)kargs.src, kargs.size)) { ++ return -EINVAL; ++ } ++ ++ /* Check if size wraps */ ++ if ((kargs.size + kargs.dest) <= kargs.dest ++ || (kargs.size + kargs.src) <= kargs.src) { ++ return -EINVAL; ++ } ++ ++ err = _mali_ukk_mem_write_safe(&kargs); ++ if (_MALI_OSK_ERR_OK != err) { ++ return map_errcode(err); ++ } ++ ++ if (0 != put_user(kargs.size, &uargs->size)) { ++ return -EFAULT; ++ } ++ ++ return 0; ++} ++ ++ ++ ++int mem_query_mmu_page_table_dump_size_wrapper(struct mali_session_data *session_data, _mali_uk_query_mmu_page_table_dump_size_s __user *uargs) ++{ ++ _mali_uk_query_mmu_page_table_dump_size_s kargs; ++ _mali_osk_errcode_t err; ++ ++ MALI_CHECK_NON_NULL(uargs, -EINVAL); ++ MALI_CHECK_NON_NULL(session_data, -EINVAL); ++ ++ kargs.ctx = (uintptr_t)session_data; ++ ++ err = _mali_ukk_query_mmu_page_table_dump_size(&kargs); ++ if (_MALI_OSK_ERR_OK != err) return map_errcode(err); ++ ++ if (0 != put_user(kargs.size, &uargs->size)) return -EFAULT; ++ ++ return 0; ++} ++ ++int mem_dump_mmu_page_table_wrapper(struct mali_session_data *session_data, _mali_uk_dump_mmu_page_table_s __user *uargs) ++{ ++ _mali_uk_dump_mmu_page_table_s kargs; ++ _mali_osk_errcode_t err; ++ void __user *user_buffer; ++ void *buffer = NULL; ++ int rc = -EFAULT; ++ ++ /* validate input */ ++ MALI_CHECK_NON_NULL(uargs, -EINVAL); ++ /* the session_data pointer was validated by caller */ ++ ++ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_dump_mmu_page_table_s))) ++ goto err_exit; ++ ++ user_buffer = (void __user *)(uintptr_t)kargs.buffer; ++ if (!access_ok(user_buffer, kargs.size)) ++ goto err_exit; ++ ++ /* allocate temporary buffer (kernel side) to store mmu page table info */ ++ if (kargs.size <= 0) ++ return -EINVAL; ++ /* Allow at most 8MiB buffers, this is more than enough to dump a fully ++ * populated page table. */ ++ if (kargs.size > SZ_8M) ++ return -EINVAL; ++ ++ buffer = (void *)(uintptr_t)_mali_osk_valloc(kargs.size); ++ if (NULL == buffer) { ++ rc = -ENOMEM; ++ goto err_exit; ++ } ++ ++ kargs.ctx = (uintptr_t)session_data; ++ kargs.buffer = (uintptr_t)buffer; ++ err = _mali_ukk_dump_mmu_page_table(&kargs); ++ if (_MALI_OSK_ERR_OK != err) { ++ rc = map_errcode(err); ++ goto err_exit; ++ } ++ ++ /* copy mmu page table info back to user space and update pointers */ ++ if (0 != copy_to_user(user_buffer, buffer, kargs.size)) ++ goto err_exit; ++ ++ kargs.register_writes = kargs.register_writes - ++ (uintptr_t)buffer + (uintptr_t)user_buffer; ++ kargs.page_table_dump = kargs.page_table_dump - ++ (uintptr_t)buffer + (uintptr_t)user_buffer; ++ ++ if (0 != copy_to_user(uargs, &kargs, sizeof(kargs))) ++ goto err_exit; ++ ++ rc = 0; ++ ++err_exit: ++ if (buffer) _mali_osk_vfree(buffer); ++ return rc; ++} ++ ++int mem_usage_get_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_memory_usage_get_s __user *uargs) ++{ ++ _mali_osk_errcode_t err; ++ _mali_uk_profiling_memory_usage_get_s kargs; ++ ++ MALI_CHECK_NON_NULL(uargs, -EINVAL); ++ MALI_CHECK_NON_NULL(session_data, -EINVAL); ++ ++ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_profiling_memory_usage_get_s))) { ++ return -EFAULT; ++ } ++ ++ kargs.ctx = (uintptr_t)session_data; ++ err = _mali_ukk_mem_usage_get(&kargs); ++ if (_MALI_OSK_ERR_OK != err) { ++ return map_errcode(err); ++ } ++ ++ kargs.ctx = (uintptr_t)NULL; /* prevent kernel address to be returned to user space */ ++ if (0 != copy_to_user(uargs, &kargs, sizeof(_mali_uk_profiling_memory_usage_get_s))) { ++ return -EFAULT; ++ } ++ ++ return 0; ++} ++ +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_pp.c b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_pp.c +new file mode 100755 +index 000000000000..a9b0958c06aa +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_pp.c +@@ -0,0 +1,105 @@ ++/* ++ * Copyright (C) 2010, 2012-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++#include /* file system operations */ ++#include /* user space access */ ++ ++#include "mali_ukk.h" ++#include "mali_osk.h" ++#include "mali_kernel_common.h" ++#include "mali_session.h" ++#include "mali_ukk_wrappers.h" ++ ++int pp_start_job_wrapper(struct mali_session_data *session_data, _mali_uk_pp_start_job_s __user *uargs) ++{ ++ _mali_osk_errcode_t err; ++ ++ /* If the job was started successfully, 0 is returned. If there was an error, but the job ++ * was started, we return -ENOENT. For anything else returned, the job was not started. */ ++ ++ MALI_CHECK_NON_NULL(uargs, -EINVAL); ++ MALI_CHECK_NON_NULL(session_data, -EINVAL); ++ ++ err = _mali_ukk_pp_start_job(session_data, uargs); ++ if (_MALI_OSK_ERR_OK != err) return map_errcode(err); ++ ++ return 0; ++} ++ ++int pp_and_gp_start_job_wrapper(struct mali_session_data *session_data, _mali_uk_pp_and_gp_start_job_s __user *uargs) ++{ ++ _mali_osk_errcode_t err; ++ ++ /* If the jobs were started successfully, 0 is returned. If there was an error, but the ++ * jobs were started, we return -ENOENT. For anything else returned, the jobs were not ++ * started. */ ++ ++ MALI_CHECK_NON_NULL(uargs, -EINVAL); ++ MALI_CHECK_NON_NULL(session_data, -EINVAL); ++ ++ err = _mali_ukk_pp_and_gp_start_job(session_data, uargs); ++ if (_MALI_OSK_ERR_OK != err) return map_errcode(err); ++ ++ return 0; ++} ++ ++int pp_get_number_of_cores_wrapper(struct mali_session_data *session_data, _mali_uk_get_pp_number_of_cores_s __user *uargs) ++{ ++ _mali_uk_get_pp_number_of_cores_s kargs; ++ _mali_osk_errcode_t err; ++ ++ MALI_CHECK_NON_NULL(uargs, -EINVAL); ++ MALI_CHECK_NON_NULL(session_data, -EINVAL); ++ ++ kargs.ctx = (uintptr_t)session_data; ++ ++ err = _mali_ukk_get_pp_number_of_cores(&kargs); ++ if (_MALI_OSK_ERR_OK != err) { ++ return map_errcode(err); ++ } ++ ++ kargs.ctx = (uintptr_t)NULL; /* prevent kernel address to be returned to user space */ ++ if (0 != copy_to_user(uargs, &kargs, sizeof(_mali_uk_get_pp_number_of_cores_s))) { ++ return -EFAULT; ++ } ++ ++ return 0; ++} ++ ++int pp_get_core_version_wrapper(struct mali_session_data *session_data, _mali_uk_get_pp_core_version_s __user *uargs) ++{ ++ _mali_uk_get_pp_core_version_s kargs; ++ _mali_osk_errcode_t err; ++ ++ MALI_CHECK_NON_NULL(uargs, -EINVAL); ++ MALI_CHECK_NON_NULL(session_data, -EINVAL); ++ ++ kargs.ctx = (uintptr_t)session_data; ++ err = _mali_ukk_get_pp_core_version(&kargs); ++ if (_MALI_OSK_ERR_OK != err) return map_errcode(err); ++ ++ if (0 != put_user(kargs.version, &uargs->version)) return -EFAULT; ++ ++ return 0; ++} ++ ++int pp_disable_wb_wrapper(struct mali_session_data *session_data, _mali_uk_pp_disable_wb_s __user *uargs) ++{ ++ _mali_uk_pp_disable_wb_s kargs; ++ ++ MALI_CHECK_NON_NULL(uargs, -EINVAL); ++ MALI_CHECK_NON_NULL(session_data, -EINVAL); ++ ++ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_pp_disable_wb_s))) return -EFAULT; ++ ++ kargs.ctx = (uintptr_t)session_data; ++ _mali_ukk_pp_job_disable_wb(&kargs); ++ ++ return 0; ++} +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_profiling.c b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_profiling.c +new file mode 100755 +index 000000000000..8b49ebc50b95 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_profiling.c +@@ -0,0 +1,183 @@ ++/* ++ * Copyright (C) 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++#include /* file system operations */ ++#include /* user space access */ ++#include ++ ++#include "mali_ukk.h" ++#include "mali_osk.h" ++#include "mali_kernel_common.h" ++#include "mali_session.h" ++#include "mali_ukk_wrappers.h" ++ ++int profiling_add_event_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_add_event_s __user *uargs) ++{ ++ _mali_uk_profiling_add_event_s kargs; ++ _mali_osk_errcode_t err; ++ ++ MALI_CHECK_NON_NULL(uargs, -EINVAL); ++ ++ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_profiling_add_event_s))) { ++ return -EFAULT; ++ } ++ ++ kargs.ctx = (uintptr_t)session_data; ++ err = _mali_ukk_profiling_add_event(&kargs); ++ if (_MALI_OSK_ERR_OK != err) { ++ return map_errcode(err); ++ } ++ ++ return 0; ++} ++ ++int profiling_report_sw_counters_wrapper(struct mali_session_data *session_data, _mali_uk_sw_counters_report_s __user *uargs) ++{ ++ _mali_uk_sw_counters_report_s kargs; ++ _mali_osk_errcode_t err; ++ u32 *counter_buffer; ++ u32 __user *counters; ++ ++ MALI_CHECK_NON_NULL(uargs, -EINVAL); ++ ++ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_sw_counters_report_s))) { ++ return -EFAULT; ++ } ++ ++ /* make sure that kargs.num_counters is [at least somewhat] sane */ ++ if (kargs.num_counters > 10000) { ++ MALI_DEBUG_PRINT(1, ("User space attempted to allocate too many counters.\n")); ++ return -EINVAL; ++ } ++ ++ counter_buffer = (u32 *)kmalloc(sizeof(u32) * kargs.num_counters, GFP_KERNEL); ++ if (NULL == counter_buffer) { ++ return -ENOMEM; ++ } ++ ++ counters = (u32 *)(uintptr_t)kargs.counters; ++ ++ if (0 != copy_from_user(counter_buffer, counters, sizeof(u32) * kargs.num_counters)) { ++ kfree(counter_buffer); ++ return -EFAULT; ++ } ++ ++ kargs.ctx = (uintptr_t)session_data; ++ kargs.counters = (uintptr_t)counter_buffer; ++ ++ err = _mali_ukk_sw_counters_report(&kargs); ++ ++ kfree(counter_buffer); ++ ++ if (_MALI_OSK_ERR_OK != err) { ++ return map_errcode(err); ++ } ++ ++ return 0; ++} ++ ++int profiling_get_stream_fd_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_stream_fd_get_s __user *uargs) ++{ ++ _mali_uk_profiling_stream_fd_get_s kargs; ++ _mali_osk_errcode_t err; ++ ++ MALI_CHECK_NON_NULL(uargs, -EINVAL); ++ ++ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_profiling_stream_fd_get_s))) { ++ return -EFAULT; ++ } ++ ++ kargs.ctx = (uintptr_t)session_data; ++ err = _mali_ukk_profiling_stream_fd_get(&kargs); ++ if (_MALI_OSK_ERR_OK != err) { ++ return map_errcode(err); ++ } ++ ++ if (0 != copy_to_user(uargs, &kargs, sizeof(_mali_uk_profiling_stream_fd_get_s))) { ++ return -EFAULT; ++ } ++ ++ return 0; ++} ++ ++int profiling_control_set_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_control_set_s __user *uargs) ++{ ++ _mali_uk_profiling_control_set_s kargs; ++ _mali_osk_errcode_t err; ++ u8 *kernel_control_data = NULL; ++ u8 *kernel_response_data = NULL; ++ ++ MALI_CHECK_NON_NULL(uargs, -EINVAL); ++ ++ if (0 != get_user(kargs.control_packet_size, &uargs->control_packet_size)) return -EFAULT; ++ if (0 != get_user(kargs.response_packet_size, &uargs->response_packet_size)) return -EFAULT; ++ ++ kargs.ctx = (uintptr_t)session_data; ++ ++ ++ /* Sanity check about the size */ ++ if (kargs.control_packet_size > PAGE_SIZE || kargs.response_packet_size > PAGE_SIZE) ++ return -EINVAL; ++ ++ if (0 != kargs.control_packet_size) { ++ ++ if (0 == kargs.response_packet_size) ++ return -EINVAL; ++ ++ kernel_control_data = _mali_osk_calloc(1, kargs.control_packet_size); ++ if (NULL == kernel_control_data) { ++ return -ENOMEM; ++ } ++ ++ kernel_response_data = _mali_osk_calloc(1, kargs.response_packet_size); ++ if (NULL == kernel_response_data) { ++ _mali_osk_free(kernel_control_data); ++ return -ENOMEM; ++ } ++ ++ kargs.control_packet_data = (uintptr_t)kernel_control_data; ++ kargs.response_packet_data = (uintptr_t)kernel_response_data; ++ ++ if (0 != copy_from_user((void *)(uintptr_t)kernel_control_data, (void *)(uintptr_t)uargs->control_packet_data, kargs.control_packet_size)) { ++ _mali_osk_free(kernel_control_data); ++ _mali_osk_free(kernel_response_data); ++ return -EFAULT; ++ } ++ ++ err = _mali_ukk_profiling_control_set(&kargs); ++ if (_MALI_OSK_ERR_OK != err) { ++ _mali_osk_free(kernel_control_data); ++ _mali_osk_free(kernel_response_data); ++ return map_errcode(err); ++ } ++ ++ if (0 != kargs.response_packet_size && 0 != copy_to_user(((void *)(uintptr_t)uargs->response_packet_data), ((void *)(uintptr_t)kargs.response_packet_data), kargs.response_packet_size)) { ++ _mali_osk_free(kernel_control_data); ++ _mali_osk_free(kernel_response_data); ++ return -EFAULT; ++ } ++ ++ if (0 != put_user(kargs.response_packet_size, &uargs->response_packet_size)) { ++ _mali_osk_free(kernel_control_data); ++ _mali_osk_free(kernel_response_data); ++ return -EFAULT; ++ } ++ ++ _mali_osk_free(kernel_control_data); ++ _mali_osk_free(kernel_response_data); ++ } else { ++ ++ err = _mali_ukk_profiling_control_set(&kargs); ++ if (_MALI_OSK_ERR_OK != err) { ++ return map_errcode(err); ++ } ++ ++ } ++ return 0; ++} +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_soft_job.c b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_soft_job.c +new file mode 100755 +index 000000000000..1dd4a7c6fb1e +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_soft_job.c +@@ -0,0 +1,90 @@ ++/* ++ * Copyright (C) 2013-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++#include /* file system operations */ ++#include /* user space access */ ++ ++#include "mali_ukk.h" ++#include "mali_osk.h" ++#include "mali_kernel_common.h" ++#include "mali_session.h" ++#include "mali_ukk_wrappers.h" ++ ++#include "mali_soft_job.h" ++#include "mali_timeline.h" ++ ++int soft_job_start_wrapper(struct mali_session_data *session, _mali_uk_soft_job_start_s __user *uargs) ++{ ++ _mali_uk_soft_job_start_s kargs; ++ u32 type, point; ++ u64 user_job; ++ struct mali_timeline_fence fence; ++ struct mali_soft_job *job = NULL; ++ u32 __user *job_id_ptr = NULL; ++ ++ /* If the job was started successfully, 0 is returned. If there was an error, but the job ++ * was started, we return -ENOENT. For anything else returned, the job was not started. */ ++ ++ MALI_CHECK_NON_NULL(uargs, -EINVAL); ++ MALI_CHECK_NON_NULL(session, -EINVAL); ++ ++ MALI_DEBUG_ASSERT_POINTER(session->soft_job_system); ++ ++ if (0 != copy_from_user(&kargs, uargs, sizeof(kargs))) { ++ return -EFAULT; ++ } ++ ++ type = kargs.type; ++ user_job = kargs.user_job; ++ job_id_ptr = (u32 __user *)(uintptr_t)kargs.job_id_ptr; ++ ++ mali_timeline_fence_copy_uk_fence(&fence, &kargs.fence); ++ ++ if ((MALI_SOFT_JOB_TYPE_USER_SIGNALED != type) && (MALI_SOFT_JOB_TYPE_SELF_SIGNALED != type)) { ++ MALI_DEBUG_PRINT_ERROR(("Invalid soft job type specified\n")); ++ return -EINVAL; ++ } ++ ++ /* Create soft job. */ ++ job = mali_soft_job_create(session->soft_job_system, (enum mali_soft_job_type)type, user_job); ++ if (unlikely(NULL == job)) { ++ return map_errcode(_MALI_OSK_ERR_NOMEM); ++ } ++ ++ /* Write job id back to user space. */ ++ if (0 != put_user(job->id, job_id_ptr)) { ++ MALI_PRINT_ERROR(("Mali Soft Job: failed to put job id")); ++ mali_soft_job_destroy(job); ++ return map_errcode(_MALI_OSK_ERR_NOMEM); ++ } ++ ++ /* Start soft job. */ ++ point = mali_soft_job_start(job, &fence); ++ ++ if (0 != put_user(point, &uargs->point)) { ++ /* Let user space know that something failed after the job was started. */ ++ return -ENOENT; ++ } ++ ++ return 0; ++} ++ ++int soft_job_signal_wrapper(struct mali_session_data *session, _mali_uk_soft_job_signal_s __user *uargs) ++{ ++ u32 job_id; ++ _mali_osk_errcode_t err; ++ ++ MALI_DEBUG_ASSERT_POINTER(session); ++ ++ if (0 != get_user(job_id, &uargs->job_id)) return -EFAULT; ++ ++ err = mali_soft_job_system_signal_job(session->soft_job_system, job_id); ++ ++ return map_errcode(err); ++} +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_timeline.c b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_timeline.c +new file mode 100755 +index 000000000000..ff0c909393a4 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_timeline.c +@@ -0,0 +1,88 @@ ++/* ++ * Copyright (C) 2013, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++#include /* file system operations */ ++#include /* user space access */ ++ ++#include "mali_ukk.h" ++#include "mali_osk.h" ++#include "mali_kernel_common.h" ++#include "mali_session.h" ++#include "mali_ukk_wrappers.h" ++ ++#include "mali_timeline.h" ++#include "mali_timeline_fence_wait.h" ++#include "mali_timeline_sync_fence.h" ++ ++int timeline_get_latest_point_wrapper(struct mali_session_data *session, _mali_uk_timeline_get_latest_point_s __user *uargs) ++{ ++ u32 val; ++ mali_timeline_id timeline; ++ mali_timeline_point point; ++ ++ MALI_DEBUG_ASSERT_POINTER(session); ++ ++ if (0 != get_user(val, &uargs->timeline)) return -EFAULT; ++ ++ if (MALI_UK_TIMELINE_MAX <= val) { ++ return -EINVAL; ++ } ++ ++ timeline = (mali_timeline_id)val; ++ ++ point = mali_timeline_system_get_latest_point(session->timeline_system, timeline); ++ ++ if (0 != put_user(point, &uargs->point)) return -EFAULT; ++ ++ return 0; ++} ++ ++int timeline_wait_wrapper(struct mali_session_data *session, _mali_uk_timeline_wait_s __user *uargs) ++{ ++ u32 timeout, status; ++ mali_bool ret; ++ _mali_uk_fence_t uk_fence; ++ struct mali_timeline_fence fence; ++ ++ MALI_DEBUG_ASSERT_POINTER(session); ++ ++ if (0 != copy_from_user(&uk_fence, &uargs->fence, sizeof(_mali_uk_fence_t))) return -EFAULT; ++ if (0 != get_user(timeout, &uargs->timeout)) return -EFAULT; ++ ++ mali_timeline_fence_copy_uk_fence(&fence, &uk_fence); ++ ++ ret = mali_timeline_fence_wait(session->timeline_system, &fence, timeout); ++ status = (MALI_TRUE == ret ? 1 : 0); ++ ++ if (0 != put_user(status, &uargs->status)) return -EFAULT; ++ ++ return 0; ++} ++ ++int timeline_create_sync_fence_wrapper(struct mali_session_data *session, _mali_uk_timeline_create_sync_fence_s __user *uargs) ++{ ++ s32 sync_fd = -1; ++ _mali_uk_fence_t uk_fence; ++ struct mali_timeline_fence fence; ++ ++ MALI_DEBUG_ASSERT_POINTER(session); ++ ++ if (0 != copy_from_user(&uk_fence, &uargs->fence, sizeof(_mali_uk_fence_t))) return -EFAULT; ++ mali_timeline_fence_copy_uk_fence(&fence, &uk_fence); ++ ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ sync_fd = mali_timeline_sync_fence_create(session->timeline_system, &fence); ++#else ++ sync_fd = -1; ++#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ ++ ++ if (0 != put_user(sync_fd, &uargs->sync_fd)) return -EFAULT; ++ ++ return 0; ++} +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_vsync.c b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_vsync.c +new file mode 100755 +index 000000000000..52519d1f96e0 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_vsync.c +@@ -0,0 +1,39 @@ ++/* ++ * Copyright (C) 2011-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++#include /* file system operations */ ++#include /* user space access */ ++ ++#include "mali_ukk.h" ++#include "mali_osk.h" ++#include "mali_kernel_common.h" ++#include "mali_session.h" ++#include "mali_ukk_wrappers.h" ++ ++ ++int vsync_event_report_wrapper(struct mali_session_data *session_data, _mali_uk_vsync_event_report_s __user *uargs) ++{ ++ _mali_uk_vsync_event_report_s kargs; ++ _mali_osk_errcode_t err; ++ ++ MALI_CHECK_NON_NULL(uargs, -EINVAL); ++ ++ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_vsync_event_report_s))) { ++ return -EFAULT; ++ } ++ ++ kargs.ctx = (uintptr_t)session_data; ++ err = _mali_ukk_vsync_event_report(&kargs); ++ if (_MALI_OSK_ERR_OK != err) { ++ return map_errcode(err); ++ } ++ ++ return 0; ++} ++ +diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_wrappers.h b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_wrappers.h +new file mode 100755 +index 000000000000..1add628fe323 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_wrappers.h +@@ -0,0 +1,82 @@ ++/* ++ * Copyright (C) 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file mali_ukk_wrappers.h ++ * Defines the wrapper functions for each user-kernel function ++ */ ++ ++#ifndef __MALI_UKK_WRAPPERS_H__ ++#define __MALI_UKK_WRAPPERS_H__ ++ ++#include "mali_uk_types.h" ++#include "mali_osk.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++int wait_for_notification_wrapper(struct mali_session_data *session_data, _mali_uk_wait_for_notification_s __user *uargs); ++int get_api_version_wrapper(struct mali_session_data *session_data, _mali_uk_get_api_version_s __user *uargs); ++int get_api_version_v2_wrapper(struct mali_session_data *session_data, _mali_uk_get_api_version_v2_s __user *uargs); ++int get_user_settings_wrapper(struct mali_session_data *session_data, _mali_uk_get_user_settings_s __user *uargs); ++int post_notification_wrapper(struct mali_session_data *session_data, _mali_uk_post_notification_s __user *uargs); ++int request_high_priority_wrapper(struct mali_session_data *session_data, _mali_uk_request_high_priority_s __user *uargs); ++int pending_submit_wrapper(struct mali_session_data *session_data, _mali_uk_pending_submit_s __user *uargs); ++ ++/* rk_ext : 从对 r5p0-01rel0 集æˆå¼€å§‹, ä¸å†ä½¿ç”¨. */ ++#if 0 ++int get_mali_version_in_rk30_wrapper(struct mali_session_data *session_data, _mali_uk_get_mali_version_in_rk30_s __user *uargs); ++#else ++int get_rk_ko_version_wrapper(struct mali_session_data *session_data, _mali_rk_ko_version_s __user *uargs); ++#endif ++ ++int mem_alloc_wrapper(struct mali_session_data *session_data, _mali_uk_alloc_mem_s __user *uargs); ++int mem_free_wrapper(struct mali_session_data *session_data, _mali_uk_free_mem_s __user *uargs); ++int mem_bind_wrapper(struct mali_session_data *session_data, _mali_uk_bind_mem_s __user *uargs); ++int mem_unbind_wrapper(struct mali_session_data *session_data, _mali_uk_unbind_mem_s __user *uargs); ++int mem_cow_wrapper(struct mali_session_data *session_data, _mali_uk_cow_mem_s __user *uargs); ++int mem_cow_modify_range_wrapper(struct mali_session_data *session_data, _mali_uk_cow_modify_range_s __user *uargs); ++int mem_resize_mem_wrapper(struct mali_session_data *session_data, _mali_uk_mem_resize_s __user *uargs); ++int mem_write_safe_wrapper(struct mali_session_data *session_data, _mali_uk_mem_write_safe_s __user *uargs); ++int mem_query_mmu_page_table_dump_size_wrapper(struct mali_session_data *session_data, _mali_uk_query_mmu_page_table_dump_size_s __user *uargs); ++int mem_dump_mmu_page_table_wrapper(struct mali_session_data *session_data, _mali_uk_dump_mmu_page_table_s __user *uargs); ++int mem_usage_get_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_memory_usage_get_s __user *uargs); ++ ++int timeline_get_latest_point_wrapper(struct mali_session_data *session, _mali_uk_timeline_get_latest_point_s __user *uargs); ++int timeline_wait_wrapper(struct mali_session_data *session, _mali_uk_timeline_wait_s __user *uargs); ++int timeline_create_sync_fence_wrapper(struct mali_session_data *session, _mali_uk_timeline_create_sync_fence_s __user *uargs); ++int soft_job_start_wrapper(struct mali_session_data *session, _mali_uk_soft_job_start_s __user *uargs); ++int soft_job_signal_wrapper(struct mali_session_data *session, _mali_uk_soft_job_signal_s __user *uargs); ++int pp_start_job_wrapper(struct mali_session_data *session_data, _mali_uk_pp_start_job_s __user *uargs); ++int pp_and_gp_start_job_wrapper(struct mali_session_data *session_data, _mali_uk_pp_and_gp_start_job_s __user *uargs); ++int pp_get_number_of_cores_wrapper(struct mali_session_data *session_data, _mali_uk_get_pp_number_of_cores_s __user *uargs); ++int pp_get_core_version_wrapper(struct mali_session_data *session_data, _mali_uk_get_pp_core_version_s __user *uargs); ++int pp_disable_wb_wrapper(struct mali_session_data *session_data, _mali_uk_pp_disable_wb_s __user *uargs); ++int gp_start_job_wrapper(struct mali_session_data *session_data, _mali_uk_gp_start_job_s __user *uargs); ++int gp_get_number_of_cores_wrapper(struct mali_session_data *session_data, _mali_uk_get_gp_number_of_cores_s __user *uargs); ++int gp_get_core_version_wrapper(struct mali_session_data *session_data, _mali_uk_get_gp_core_version_s __user *uargs); ++int gp_suspend_response_wrapper(struct mali_session_data *session_data, _mali_uk_gp_suspend_response_s __user *uargs); ++ ++int profiling_add_event_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_add_event_s __user *uargs); ++int profiling_report_sw_counters_wrapper(struct mali_session_data *session_data, _mali_uk_sw_counters_report_s __user *uargs); ++int profiling_get_stream_fd_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_stream_fd_get_s __user *uargs); ++int profiling_control_set_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_control_set_s __user *uargs); ++ ++int vsync_event_report_wrapper(struct mali_session_data *session_data, _mali_uk_vsync_event_report_s __user *uargs); ++ ++ ++int map_errcode(_mali_osk_errcode_t err); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif /* __MALI_UKK_WRAPPERS_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/platform/arm/arm.c b/drivers/gpu/arm/mali400/mali/platform/arm/arm.c +new file mode 100755 +index 000000000000..fc7017bbfe08 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/platform/arm/arm.c +@@ -0,0 +1,629 @@ ++/* ++ * Copyright (C) 2010, 2012-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file mali_platform.c ++ * Platform specific Mali driver functions for: ++ * - Realview Versatile platforms with ARM11 Mpcore and virtex 5. ++ * - Versatile Express platforms with ARM Cortex-A9 and virtex 6. ++ */ ++#include ++#include ++#include ++#include "mali_kernel_linux.h" ++#ifdef CONFIG_PM_RUNTIME ++#include ++#endif ++#include ++#include ++#include "mali_kernel_common.h" ++#include ++#include ++ ++#include "arm_core_scaling.h" ++#include "mali_executor.h" ++ ++#if defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL) ++#include ++#include ++#endif ++ ++static int mali_core_scaling_enable = 0; ++ ++void mali_gpu_utilization_callback(struct mali_gpu_utilization_data *data); ++static u32 mali_read_phys(u32 phys_addr); ++#if defined(CONFIG_ARCH_REALVIEW) ++static void mali_write_phys(u32 phys_addr, u32 value); ++#endif ++ ++#if defined(CONFIG_ARCH_VEXPRESS) && defined(CONFIG_ARM64) ++ ++#define SECURE_MODE_CONTROL_HANDLER 0x6F02006C ++void *secure_mode_mapped_addr = NULL; ++/** ++ * Reset GPU and enable/disable Mali secure mode. ++ * @Return value: ++ * 0: success ++ * non-0: failure. ++ */ ++ ++static int mali_gpu_reset_and_secure_mode_enable_juno(void) ++{ ++ u32 phys_offset = SECURE_MODE_CONTROL_HANDLER & 0x00001FFF; ++ MALI_DEBUG_ASSERT(NULL != secure_mode_mapped_addr); ++ ++ iowrite32(1, ((u8 *)secure_mode_mapped_addr) + phys_offset); ++ ++ if (1 == (u32)ioread32(((u8 *)secure_mode_mapped_addr) + phys_offset)) { ++ MALI_DEBUG_PRINT(3, ("Mali reset GPU and enable secured mode successfully! \n")); ++ return 0; ++ } ++ ++ MALI_PRINT_ERROR(("Failed to reset GPU and enable Mali secured mode !!! \n")); ++ ++ return -1; ++ ++} ++ ++static int mali_gpu_reset_and_secure_mode_disable_juno(void) ++{ ++ u32 phys_offset = SECURE_MODE_CONTROL_HANDLER & 0x00001FFF; ++ MALI_DEBUG_ASSERT(NULL != secure_mode_mapped_addr); ++ ++ iowrite32(0, ((u8 *)secure_mode_mapped_addr) + phys_offset); ++ ++ if (0 == (u32)ioread32(((u8 *)secure_mode_mapped_addr) + phys_offset)) { ++ MALI_DEBUG_PRINT(3, ("Mali reset GPU and disable secured mode successfully! \n")); ++ return 0; ++ } ++ ++ MALI_PRINT_ERROR(("Failed to reset GPU and disable mali secured mode !!! \n")); ++ return -1; ++} ++ ++static int mali_secure_mode_init_juno(void) ++{ ++ u32 phys_addr_page = SECURE_MODE_CONTROL_HANDLER & 0xFFFFE000; ++ u32 phys_offset = SECURE_MODE_CONTROL_HANDLER & 0x00001FFF; ++ u32 map_size = phys_offset + sizeof(u32); ++ ++ MALI_DEBUG_ASSERT(NULL == secure_mode_mapped_addr); ++ ++ secure_mode_mapped_addr = ioremap(phys_addr_page, map_size); ++ if (NULL != secure_mode_mapped_addr) { ++ return mali_gpu_reset_and_secure_mode_disable_juno(); ++ } ++ MALI_DEBUG_PRINT(2, ("Failed to ioremap for Mali secured mode! \n")); ++ return -1; ++} ++ ++static void mali_secure_mode_deinit_juno(void) ++{ ++ if (NULL != secure_mode_mapped_addr) { ++ mali_gpu_reset_and_secure_mode_disable_juno(); ++ iounmap(secure_mode_mapped_addr); ++ secure_mode_mapped_addr = NULL; ++ } ++} ++#endif ++ ++#ifndef CONFIG_MALI_DT ++static void mali_platform_device_release(struct device *device); ++ ++#if defined(CONFIG_ARCH_VEXPRESS) ++ ++#if defined(CONFIG_ARM64) ++/* Juno + Mali-450 MP6 in V7 FPGA */ ++static struct resource mali_gpu_resources_m450_mp6[] = { ++ MALI_GPU_RESOURCES_MALI450_MP6_PMU(0x6F040000, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200) ++}; ++ ++static struct resource mali_gpu_resources_m470_mp4[] = { ++ MALI_GPU_RESOURCES_MALI470_MP4_PMU(0x6F040000, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200) ++}; ++ ++static struct resource mali_gpu_resources_m470_mp3[] = { ++ MALI_GPU_RESOURCES_MALI470_MP3_PMU(0x6F040000, 200, 200, 200, 200, 200, 200, 200, 200, 200) ++}; ++ ++static struct resource mali_gpu_resources_m470_mp2[] = { ++ MALI_GPU_RESOURCES_MALI470_MP2_PMU(0x6F040000, 200, 200, 200, 200, 200, 200, 200) ++}; ++ ++static struct resource mali_gpu_resources_m470_mp1[] = { ++ MALI_GPU_RESOURCES_MALI470_MP1_PMU(0x6F040000, 200, 200, 200, 200, 200) ++}; ++ ++#else ++static struct resource mali_gpu_resources_m450_mp8[] = { ++ MALI_GPU_RESOURCES_MALI450_MP8_PMU(0xFC040000, -1, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 68) ++}; ++ ++static struct resource mali_gpu_resources_m450_mp6[] = { ++ MALI_GPU_RESOURCES_MALI450_MP6_PMU(0xFC040000, -1, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 68) ++}; ++ ++static struct resource mali_gpu_resources_m450_mp4[] = { ++ MALI_GPU_RESOURCES_MALI450_MP4_PMU(0xFC040000, -1, 70, 70, 70, 70, 70, 70, 70, 70, 70, 68) ++}; ++ ++static struct resource mali_gpu_resources_m470_mp4[] = { ++ MALI_GPU_RESOURCES_MALI470_MP4_PMU(0xFC040000, -1, 70, 70, 70, 70, 70, 70, 70, 70, 70, 68) ++}; ++#endif /* CONFIG_ARM64 */ ++ ++#elif defined(CONFIG_ARCH_REALVIEW) ++ ++static struct resource mali_gpu_resources_m300[] = { ++ MALI_GPU_RESOURCES_MALI300_PMU(0xC0000000, -1, -1, -1, -1) ++}; ++ ++static struct resource mali_gpu_resources_m400_mp1[] = { ++ MALI_GPU_RESOURCES_MALI400_MP1_PMU(0xC0000000, -1, -1, -1, -1) ++}; ++ ++static struct resource mali_gpu_resources_m400_mp2[] = { ++ MALI_GPU_RESOURCES_MALI400_MP2_PMU(0xC0000000, -1, -1, -1, -1, -1, -1) ++}; ++ ++#endif ++#endif ++ ++#if defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL) ++ ++#define FALLBACK_STATIC_TEMPERATURE 55000 ++ ++static struct thermal_zone_device *gpu_tz; ++ ++/* Calculate gpu static power example for reference */ ++static unsigned long arm_model_static_power(struct devfreq *devfreq, ++ unsigned long voltage) ++{ ++ int temperature, temp; ++ int temp_squared, temp_cubed, temp_scaling_factor; ++ const unsigned long coefficient = (410UL << 20) / (729000000UL >> 10); ++ const unsigned long voltage_cubed = (voltage * voltage * voltage) >> 10; ++ unsigned long static_power; ++ ++ if (gpu_tz) { ++ int ret; ++ ++ ret = gpu_tz->ops->get_temp(gpu_tz, &temperature); ++ if (ret) { ++ MALI_DEBUG_PRINT(2, ("Error reading temperature for gpu thermal zone: %d\n", ret)); ++ temperature = FALLBACK_STATIC_TEMPERATURE; ++ } ++ } else { ++ temperature = FALLBACK_STATIC_TEMPERATURE; ++ } ++ ++ /* Calculate the temperature scaling factor. To be applied to the ++ * voltage scaled power. ++ */ ++ temp = temperature / 1000; ++ temp_squared = temp * temp; ++ temp_cubed = temp_squared * temp; ++ temp_scaling_factor = ++ (2 * temp_cubed) ++ - (80 * temp_squared) ++ + (4700 * temp) ++ + 32000; ++ ++ static_power = (((coefficient * voltage_cubed) >> 20) ++ * temp_scaling_factor) ++ / 1000000; ++ ++ return static_power; ++} ++ ++/* Calculate gpu dynamic power example for reference */ ++static unsigned long arm_model_dynamic_power(struct devfreq *devfreq, ++ unsigned long freq, ++ unsigned long voltage) ++{ ++ /* The inputs: freq (f) is in Hz, and voltage (v) in mV. ++ * The coefficient (c) is in mW/(MHz mV mV). ++ * ++ * This function calculates the dynamic power after this formula: ++ * Pdyn (mW) = c (mW/(MHz*mV*mV)) * v (mV) * v (mV) * f (MHz) ++ */ ++ const unsigned long v2 = (voltage * voltage) / 1000; /* m*(V*V) */ ++ const unsigned long f_mhz = freq / 1000000; /* MHz */ ++ const unsigned long coefficient = 3600; /* mW/(MHz*mV*mV) */ ++ unsigned long dynamic_power; ++ ++ dynamic_power = (coefficient * v2 * f_mhz) / 1000000; /* mW */ ++ ++ return dynamic_power; ++} ++ ++struct devfreq_cooling_power arm_cooling_ops = { ++ .get_static_power = arm_model_static_power, ++ .get_dynamic_power = arm_model_dynamic_power, ++}; ++#endif ++ ++static struct mali_gpu_device_data mali_gpu_data = { ++#ifndef CONFIG_MALI_DT ++ .pmu_switch_delay = 0xFF, /* do not have to be this high on FPGA, but it is good for testing to have a delay */ ++#if defined(CONFIG_ARCH_VEXPRESS) ++ .shared_mem_size = 256 * 1024 * 1024, /* 256MB */ ++#endif ++#endif ++ .max_job_runtime = 60000, /* 60 seconds */ ++ ++#if defined(CONFIG_ARCH_REALVIEW) ++ .dedicated_mem_start = 0x80000000, /* Physical start address (use 0xD0000000 for old indirect setup) */ ++ .dedicated_mem_size = 0x10000000, /* 256MB */ ++#endif ++#if defined(CONFIG_ARM64) ++ /* Some framebuffer drivers get the framebuffer dynamically, such as through GEM, ++ * in which the memory resource can't be predicted in advance. ++ */ ++ .fb_start = 0x0, ++ .fb_size = 0xFFFFF000, ++#else ++ .fb_start = 0xe0000000, ++ .fb_size = 0x01000000, ++#endif ++ .control_interval = 1000, /* 1000ms */ ++ .utilization_callback = mali_gpu_utilization_callback, ++ .get_clock_info = NULL, ++ .get_freq = NULL, ++ .set_freq = NULL, ++#if defined(CONFIG_ARCH_VEXPRESS) && defined(CONFIG_ARM64) ++ .secure_mode_init = mali_secure_mode_init_juno, ++ .secure_mode_deinit = mali_secure_mode_deinit_juno, ++ .gpu_reset_and_secure_mode_enable = mali_gpu_reset_and_secure_mode_enable_juno, ++ .gpu_reset_and_secure_mode_disable = mali_gpu_reset_and_secure_mode_disable_juno, ++#else ++ .secure_mode_init = NULL, ++ .secure_mode_deinit = NULL, ++ .gpu_reset_and_secure_mode_enable = NULL, ++ .gpu_reset_and_secure_mode_disable = NULL, ++#endif ++#if defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL) ++ .gpu_cooling_ops = &arm_cooling_ops, ++#endif ++}; ++ ++#ifndef CONFIG_MALI_DT ++static struct platform_device mali_gpu_device = { ++ .name = MALI_GPU_NAME_UTGARD, ++ .id = 0, ++ .dev.release = mali_platform_device_release, ++ .dev.dma_mask = &mali_gpu_device.dev.coherent_dma_mask, ++ .dev.coherent_dma_mask = DMA_BIT_MASK(32), ++ ++ .dev.platform_data = &mali_gpu_data, ++}; ++ ++int mali_platform_device_register(void) ++{ ++ int err = -1; ++ int num_pp_cores = 0; ++#if defined(CONFIG_ARCH_REALVIEW) ++ u32 m400_gp_version; ++#endif ++ ++ MALI_DEBUG_PRINT(4, ("mali_platform_device_register() called\n")); ++ ++ /* Detect present Mali GPU and connect the correct resources to the device */ ++#if defined(CONFIG_ARCH_VEXPRESS) ++ ++#if defined(CONFIG_ARM64) ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) ++ mali_gpu_device.dev.archdata.dma_ops = &dummy_dma_ops; ++#else ++ mali_gpu_device.dev.archdata.dma_ops = dma_ops; ++#endif ++ if ((mali_read_phys(0x6F000000) & 0x00600450) == 0x00600450) { ++ MALI_DEBUG_PRINT(4, ("Registering Mali-450 MP6 device\n")); ++ num_pp_cores = 6; ++ mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m450_mp6); ++ mali_gpu_device.resource = mali_gpu_resources_m450_mp6; ++ } else if ((mali_read_phys(0x6F000000) & 0x00F00430) == 0x00400430) { ++ MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP4 device\n")); ++ num_pp_cores = 4; ++ mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m470_mp4); ++ mali_gpu_device.resource = mali_gpu_resources_m470_mp4; ++ } else if ((mali_read_phys(0x6F000000) & 0x00F00430) == 0x00300430) { ++ MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP3 device\n")); ++ num_pp_cores = 3; ++ mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m470_mp3); ++ mali_gpu_device.resource = mali_gpu_resources_m470_mp3; ++ } else if ((mali_read_phys(0x6F000000) & 0x00F00430) == 0x00200430) { ++ MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP2 device\n")); ++ num_pp_cores = 2; ++ mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m470_mp2); ++ mali_gpu_device.resource = mali_gpu_resources_m470_mp2; ++ } else if ((mali_read_phys(0x6F000000) & 0x00F00430) == 0x00100430) { ++ MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP1 device\n")); ++ num_pp_cores = 1; ++ mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m470_mp1); ++ mali_gpu_device.resource = mali_gpu_resources_m470_mp1; ++ } ++#else ++ if (mali_read_phys(0xFC000000) == 0x00000450) { ++ MALI_DEBUG_PRINT(4, ("Registering Mali-450 MP8 device\n")); ++ num_pp_cores = 8; ++ mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m450_mp8); ++ mali_gpu_device.resource = mali_gpu_resources_m450_mp8; ++ } else if (mali_read_phys(0xFC000000) == 0x40600450) { ++ MALI_DEBUG_PRINT(4, ("Registering Mali-450 MP6 device\n")); ++ num_pp_cores = 6; ++ mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m450_mp6); ++ mali_gpu_device.resource = mali_gpu_resources_m450_mp6; ++ } else if (mali_read_phys(0xFC000000) == 0x40400450) { ++ MALI_DEBUG_PRINT(4, ("Registering Mali-450 MP4 device\n")); ++ num_pp_cores = 4; ++ mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m450_mp4); ++ mali_gpu_device.resource = mali_gpu_resources_m450_mp4; ++ } else if (mali_read_phys(0xFC000000) == 0xFFFFFFFF) { ++ MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP4 device\n")); ++ num_pp_cores = 4; ++ mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m470_mp4); ++ mali_gpu_device.resource = mali_gpu_resources_m470_mp4; ++ } ++#endif /* CONFIG_ARM64 */ ++ ++#elif defined(CONFIG_ARCH_REALVIEW) ++ ++ m400_gp_version = mali_read_phys(0xC000006C); ++ if ((m400_gp_version & 0xFFFF0000) == 0x0C070000) { ++ MALI_DEBUG_PRINT(4, ("Registering Mali-300 device\n")); ++ num_pp_cores = 1; ++ mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m300); ++ mali_gpu_device.resource = mali_gpu_resources_m300; ++ mali_write_phys(0xC0010020, 0xA); /* Enable direct memory mapping for FPGA */ ++ } else if ((m400_gp_version & 0xFFFF0000) == 0x0B070000) { ++ u32 fpga_fw_version = mali_read_phys(0xC0010000); ++ if (fpga_fw_version == 0x130C008F || fpga_fw_version == 0x110C008F) { ++ /* Mali-400 MP1 r1p0 or r1p1 */ ++ MALI_DEBUG_PRINT(4, ("Registering Mali-400 MP1 device\n")); ++ num_pp_cores = 1; ++ mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m400_mp1); ++ mali_gpu_device.resource = mali_gpu_resources_m400_mp1; ++ mali_write_phys(0xC0010020, 0xA); /* Enable direct memory mapping for FPGA */ ++ } else if (fpga_fw_version == 0x130C000F) { ++ /* Mali-400 MP2 r1p1 */ ++ MALI_DEBUG_PRINT(4, ("Registering Mali-400 MP2 device\n")); ++ num_pp_cores = 2; ++ mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m400_mp2); ++ mali_gpu_device.resource = mali_gpu_resources_m400_mp2; ++ mali_write_phys(0xC0010020, 0xA); /* Enable direct memory mapping for FPGA */ ++ } ++ } ++ ++#endif ++ /* Register the platform device */ ++ err = platform_device_register(&mali_gpu_device); ++ if (0 == err) { ++#ifdef CONFIG_PM_RUNTIME ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) ++ pm_runtime_set_autosuspend_delay(&(mali_gpu_device.dev), 1000); ++ pm_runtime_use_autosuspend(&(mali_gpu_device.dev)); ++#endif ++ pm_runtime_enable(&(mali_gpu_device.dev)); ++#endif ++ MALI_DEBUG_ASSERT(0 < num_pp_cores); ++ mali_core_scaling_init(num_pp_cores); ++ ++ return 0; ++ } ++ ++ return err; ++} ++ ++void mali_platform_device_unregister(void) ++{ ++ MALI_DEBUG_PRINT(4, ("mali_platform_device_unregister() called\n")); ++ ++ mali_core_scaling_term(); ++#ifdef CONFIG_PM_RUNTIME ++ pm_runtime_disable(&(mali_gpu_device.dev)); ++#endif ++ platform_device_unregister(&mali_gpu_device); ++ ++ platform_device_put(&mali_gpu_device); ++ ++#if defined(CONFIG_ARCH_REALVIEW) ++ mali_write_phys(0xC0010020, 0x9); /* Restore default (legacy) memory mapping */ ++#endif ++} ++ ++static void mali_platform_device_release(struct device *device) ++{ ++ MALI_DEBUG_PRINT(4, ("mali_platform_device_release() called\n")); ++} ++ ++#else /* CONFIG_MALI_DT */ ++int mali_platform_device_init(struct platform_device *device) ++{ ++ int num_pp_cores = 0; ++ int err = -1; ++#if defined(CONFIG_ARCH_REALVIEW) ++ u32 m400_gp_version; ++#endif ++ ++ /* Detect present Mali GPU and connect the correct resources to the device */ ++#if defined(CONFIG_ARCH_VEXPRESS) ++ ++#if defined(CONFIG_ARM64) ++ if ((mali_read_phys(0x6F000000) & 0x00600450) == 0x00600450) { ++ MALI_DEBUG_PRINT(4, ("Registering Mali-450 MP6 device\n")); ++ num_pp_cores = 6; ++ } else if ((mali_read_phys(0x6F000000) & 0x00F00430) == 0x00400430) { ++ MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP4 device\n")); ++ num_pp_cores = 4; ++ } else if ((mali_read_phys(0x6F000000) & 0x00F00430) == 0x00300430) { ++ MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP3 device\n")); ++ num_pp_cores = 3; ++ } else if ((mali_read_phys(0x6F000000) & 0x00F00430) == 0x00200430) { ++ MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP2 device\n")); ++ num_pp_cores = 2; ++ } else if ((mali_read_phys(0x6F000000) & 0x00F00430) == 0x00100430) { ++ MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP1 device\n")); ++ num_pp_cores = 1; ++ } ++#else ++ if (mali_read_phys(0xFC000000) == 0x00000450) { ++ MALI_DEBUG_PRINT(4, ("Registering Mali-450 MP8 device\n")); ++ num_pp_cores = 8; ++ } else if (mali_read_phys(0xFC000000) == 0x40400450) { ++ MALI_DEBUG_PRINT(4, ("Registering Mali-450 MP4 device\n")); ++ num_pp_cores = 4; ++ } else if (mali_read_phys(0xFC000000) == 0xFFFFFFFF) { ++ MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP4 device\n")); ++ num_pp_cores = 4; ++ } ++#endif ++ ++#elif defined(CONFIG_ARCH_REALVIEW) ++ ++ m400_gp_version = mali_read_phys(0xC000006C); ++ if ((m400_gp_version & 0xFFFF0000) == 0x0C070000) { ++ MALI_DEBUG_PRINT(4, ("Registering Mali-300 device\n")); ++ num_pp_cores = 1; ++ mali_write_phys(0xC0010020, 0xA); /* Enable direct memory mapping for FPGA */ ++ } else if ((m400_gp_version & 0xFFFF0000) == 0x0B070000) { ++ u32 fpga_fw_version = mali_read_phys(0xC0010000); ++ if (fpga_fw_version == 0x130C008F || fpga_fw_version == 0x110C008F) { ++ /* Mali-400 MP1 r1p0 or r1p1 */ ++ MALI_DEBUG_PRINT(4, ("Registering Mali-400 MP1 device\n")); ++ num_pp_cores = 1; ++ mali_write_phys(0xC0010020, 0xA); /* Enable direct memory mapping for FPGA */ ++ } else if (fpga_fw_version == 0x130C000F) { ++ /* Mali-400 MP2 r1p1 */ ++ MALI_DEBUG_PRINT(4, ("Registering Mali-400 MP2 device\n")); ++ num_pp_cores = 2; ++ mali_write_phys(0xC0010020, 0xA); /* Enable direct memory mapping for FPGA */ ++ } ++ } ++#endif ++ ++ /* After kernel 3.15 device tree will default set dev ++ * related parameters in of_platform_device_create_pdata. ++ * But kernel changes from version to version, ++ * For example 3.10 didn't include device->dev.dma_mask parameter setting, ++ * if we didn't include here will cause dma_mapping error, ++ * but in kernel 3.15 it include device->dev.dma_mask parameter setting, ++ * so it's better to set must need paramter by DDK itself. ++ */ ++ if (!device->dev.dma_mask) ++ device->dev.dma_mask = &device->dev.coherent_dma_mask; ++ device->dev.archdata.dma_ops = dma_ops; ++ ++ err = platform_device_add_data(device, &mali_gpu_data, sizeof(mali_gpu_data)); ++ ++ if (0 == err) { ++#ifdef CONFIG_PM_RUNTIME ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) ++ pm_runtime_set_autosuspend_delay(&(device->dev), 1000); ++ pm_runtime_use_autosuspend(&(device->dev)); ++#endif ++ pm_runtime_enable(&(device->dev)); ++#endif ++ MALI_DEBUG_ASSERT(0 < num_pp_cores); ++ mali_core_scaling_init(num_pp_cores); ++ } ++ ++#if defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL) ++ /* Some Socs didn't support the devfreq thermal for mali */ ++ if (of_machine_is_compatible("rockchip,rk3036")) ++ return 0; ++ ++ /* Get thermal zone */ ++ gpu_tz = thermal_zone_get_zone_by_name("soc_thermal"); ++ if (IS_ERR(gpu_tz)) { ++ MALI_DEBUG_PRINT(2, ("Error getting gpu thermal zone (%ld), not yet ready?\n", ++ PTR_ERR(gpu_tz))); ++ gpu_tz = NULL; ++ ++ err = -EPROBE_DEFER; ++ } ++#endif ++ ++ return err; ++} ++ ++int mali_platform_device_deinit(struct platform_device *device) ++{ ++ MALI_IGNORE(device); ++ ++ MALI_DEBUG_PRINT(4, ("mali_platform_device_deinit() called\n")); ++ ++ mali_core_scaling_term(); ++#ifdef CONFIG_PM_RUNTIME ++ pm_runtime_disable(&(device->dev)); ++#endif ++ ++#if defined(CONFIG_ARCH_REALVIEW) ++ mali_write_phys(0xC0010020, 0x9); /* Restore default (legacy) memory mapping */ ++#endif ++ ++ return 0; ++} ++ ++#endif /* CONFIG_MALI_DT */ ++ ++static u32 mali_read_phys(u32 phys_addr) ++{ ++ u32 phys_addr_page = phys_addr & 0xFFFFE000; ++ u32 phys_offset = phys_addr & 0x00001FFF; ++ u32 map_size = phys_offset + sizeof(u32); ++ u32 ret = 0xDEADBEEF; ++ void *mem_mapped = ioremap(phys_addr_page, map_size); ++ if (NULL != mem_mapped) { ++ ret = (u32)ioread32(((u8 *)mem_mapped) + phys_offset); ++ iounmap(mem_mapped); ++ } ++ ++ return ret; ++} ++ ++#if defined(CONFIG_ARCH_REALVIEW) ++static void mali_write_phys(u32 phys_addr, u32 value) ++{ ++ u32 phys_addr_page = phys_addr & 0xFFFFE000; ++ u32 phys_offset = phys_addr & 0x00001FFF; ++ u32 map_size = phys_offset + sizeof(u32); ++ void *mem_mapped = ioremap(phys_addr_page, map_size); ++ if (NULL != mem_mapped) { ++ iowrite32(value, ((u8 *)mem_mapped) + phys_offset); ++ iounmap(mem_mapped); ++ } ++} ++#endif ++ ++static int param_set_core_scaling(const char *val, const struct kernel_param *kp) ++{ ++ int ret = param_set_int(val, kp); ++ ++ if (1 == mali_core_scaling_enable) { ++ mali_core_scaling_sync(mali_executor_get_num_cores_enabled()); ++ } ++ return ret; ++} ++ ++static struct kernel_param_ops param_ops_core_scaling = { ++ .set = param_set_core_scaling, ++ .get = param_get_int, ++}; ++ ++module_param_cb(mali_core_scaling_enable, ¶m_ops_core_scaling, &mali_core_scaling_enable, 0644); ++MODULE_PARM_DESC(mali_core_scaling_enable, "1 means to enable core scaling policy, 0 means to disable core scaling policy"); ++ ++void mali_gpu_utilization_callback(struct mali_gpu_utilization_data *data) ++{ ++ if (1 == mali_core_scaling_enable) { ++ mali_core_scaling_update(data); ++ } ++} +diff --git a/drivers/gpu/arm/mali400/mali/platform/arm/arm_core_scaling.c b/drivers/gpu/arm/mali400/mali/platform/arm/arm_core_scaling.c +new file mode 100755 +index 000000000000..7a2fc8107b4f +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/platform/arm/arm_core_scaling.c +@@ -0,0 +1,122 @@ ++/* ++ * Copyright (C) 2013-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file arm_core_scaling.c ++ * Example core scaling policy. ++ */ ++ ++#include "arm_core_scaling.h" ++ ++#include ++#include "mali_kernel_common.h" ++ ++#include ++ ++static int num_cores_total; ++static int num_cores_enabled; ++ ++static struct work_struct wq_work; ++ ++static void set_num_cores(struct work_struct *work) ++{ ++ int err = mali_perf_set_num_pp_cores(num_cores_enabled); ++ MALI_DEBUG_ASSERT(0 == err); ++ MALI_IGNORE(err); ++} ++ ++static void enable_one_core(void) ++{ ++ if (num_cores_enabled < num_cores_total) { ++ ++num_cores_enabled; ++ schedule_work(&wq_work); ++ MALI_DEBUG_PRINT(3, ("Core scaling: Enabling one more core\n")); ++ } ++ ++ MALI_DEBUG_ASSERT(1 <= num_cores_enabled); ++ MALI_DEBUG_ASSERT(num_cores_total >= num_cores_enabled); ++} ++ ++static void disable_one_core(void) ++{ ++ if (1 < num_cores_enabled) { ++ --num_cores_enabled; ++ schedule_work(&wq_work); ++ MALI_DEBUG_PRINT(3, ("Core scaling: Disabling one core\n")); ++ } ++ ++ MALI_DEBUG_ASSERT(1 <= num_cores_enabled); ++ MALI_DEBUG_ASSERT(num_cores_total >= num_cores_enabled); ++} ++ ++static void enable_max_num_cores(void) ++{ ++ if (num_cores_enabled < num_cores_total) { ++ num_cores_enabled = num_cores_total; ++ schedule_work(&wq_work); ++ MALI_DEBUG_PRINT(3, ("Core scaling: Enabling maximum number of cores\n")); ++ } ++ ++ MALI_DEBUG_ASSERT(num_cores_total == num_cores_enabled); ++} ++ ++void mali_core_scaling_init(int num_pp_cores) ++{ ++ INIT_WORK(&wq_work, set_num_cores); ++ ++ num_cores_total = num_pp_cores; ++ num_cores_enabled = num_pp_cores; ++ ++ /* NOTE: Mali is not fully initialized at this point. */ ++} ++ ++void mali_core_scaling_sync(int num_cores) ++{ ++ num_cores_enabled = num_cores; ++} ++ ++void mali_core_scaling_term(void) ++{ ++ flush_scheduled_work(); ++} ++ ++#define PERCENT_OF(percent, max) ((int) ((percent)*(max)/100.0 + 0.5)) ++ ++void mali_core_scaling_update(struct mali_gpu_utilization_data *data) ++{ ++ /* ++ * This function implements a very trivial PP core scaling algorithm. ++ * ++ * It is _NOT_ of production quality. ++ * The only intention behind this algorithm is to exercise and test the ++ * core scaling functionality of the driver. ++ * It is _NOT_ tuned for neither power saving nor performance! ++ * ++ * Other metrics than PP utilization need to be considered as well ++ * in order to make a good core scaling algorithm. ++ */ ++ ++ MALI_DEBUG_PRINT(3, ("Utilization: (%3d, %3d, %3d), cores enabled: %d/%d\n", data->utilization_gpu, data->utilization_gp, data->utilization_pp, num_cores_enabled, num_cores_total)); ++ ++ /* NOTE: this function is normally called directly from the utilization callback which is in ++ * timer context. */ ++ ++ if (PERCENT_OF(90, 256) < data->utilization_pp) { ++ enable_max_num_cores(); ++ } else if (PERCENT_OF(50, 256) < data->utilization_pp) { ++ enable_one_core(); ++ } else if (PERCENT_OF(40, 256) < data->utilization_pp) { ++ /* do nothing */ ++ } else if (PERCENT_OF(0, 256) < data->utilization_pp) { ++ disable_one_core(); ++ } else { ++ /* do nothing */ ++ } ++} +diff --git a/drivers/gpu/arm/mali400/mali/platform/arm/arm_core_scaling.h b/drivers/gpu/arm/mali400/mali/platform/arm/arm_core_scaling.h +new file mode 100755 +index 000000000000..8e0101830749 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/platform/arm/arm_core_scaling.h +@@ -0,0 +1,44 @@ ++/* ++ * Copyright (C) 2013, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file arm_core_scaling.h ++ * Example core scaling policy. ++ */ ++ ++#ifndef __ARM_CORE_SCALING_H__ ++#define __ARM_CORE_SCALING_H__ ++ ++struct mali_gpu_utilization_data; ++ ++/** ++ * Initialize core scaling policy. ++ * ++ * @note The core scaling policy will assume that all PP cores are on initially. ++ * ++ * @param num_pp_cores Total number of PP cores. ++ */ ++void mali_core_scaling_init(int num_pp_cores); ++ ++/** ++ * Terminate core scaling policy. ++ */ ++void mali_core_scaling_term(void); ++ ++/** ++ * Update core scaling policy with new utilization data. ++ * ++ * @param data Utilization data. ++ */ ++void mali_core_scaling_update(struct mali_gpu_utilization_data *data); ++ ++void mali_core_scaling_sync(int num_cores); ++ ++#endif /* __ARM_CORE_SCALING_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/platform/arm/juno_opp.c b/drivers/gpu/arm/mali400/mali/platform/arm/juno_opp.c +new file mode 100755 +index 000000000000..e4e7ab8b2c2e +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/platform/arm/juno_opp.c +@@ -0,0 +1,127 @@ ++/* ++ * Copyright (C) 2010, 2012-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file juno_opp.c ++ * Example: Set up opp table ++ * Using ARM64 juno specific SCPI_PROTOCOL get frequence inform ++ * Customer need implement your own platform releated logic ++ */ ++#ifdef CONFIG_ARCH_VEXPRESS ++#ifdef CONFIG_MALI_DEVFREQ ++#ifdef CONFIG_ARM64 ++#ifdef CONFIG_ARM_SCPI_PROTOCOL ++#include ++#include ++#include ++#include ++#include ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0) ++#include ++#else /* Linux >= 3.13 */ ++/* In 3.13 the OPP include header file, types, and functions were all ++ * renamed. Use the old filename for the include, and define the new names to ++ * the old, when an old kernel is detected. ++ */ ++#include ++#define dev_pm_opp_add opp_add ++#define dev_pm_opp_remove opp_remove ++#endif /* Linux >= 3.13 */ ++ ++#include "mali_kernel_common.h" ++ ++static int init_juno_opps_from_scpi(struct device *dev) ++{ ++ struct scpi_dvfs_info *sinfo; ++ struct scpi_ops *sops; ++ ++ int i; ++ ++ sops = get_scpi_ops(); ++ if (NULL == sops) { ++ MALI_DEBUG_PRINT(2, ("Mali didn't get any scpi ops \n")); ++ return -1; ++ } ++ ++ /* Hard coded for Juno. 2 is GPU domain */ ++ sinfo = sops->dvfs_get_info(2); ++ if (IS_ERR_OR_NULL(sinfo)) ++ return PTR_ERR(sinfo); ++ ++ for (i = 0; i < sinfo->count; i++) { ++ struct scpi_opp *e = &sinfo->opps[i]; ++ ++ MALI_DEBUG_PRINT(2, ("Mali OPP from SCPI: %u Hz @ %u mV\n", e->freq, e->m_volt)); ++ ++ dev_pm_opp_add(dev, e->freq, e->m_volt * 1000); ++ } ++ ++ return 0; ++} ++ ++int setup_opps(void) ++{ ++ struct device_node *np; ++ struct platform_device *pdev; ++ int err; ++ ++ np = of_find_node_by_name(NULL, "gpu"); ++ if (!np) { ++ pr_err("Failed to find DT entry for Mali\n"); ++ return -EFAULT; ++ } ++ ++ pdev = of_find_device_by_node(np); ++ if (!pdev) { ++ pr_err("Failed to find device for Mali\n"); ++ of_node_put(np); ++ return -EFAULT; ++ } ++ ++ err = init_juno_opps_from_scpi(&pdev->dev); ++ ++ of_node_put(np); ++ ++ return err; ++} ++ ++int term_opps(struct device *dev) ++{ ++ struct scpi_dvfs_info *sinfo; ++ struct scpi_ops *sops; ++ ++ int i; ++ ++ sops = get_scpi_ops(); ++ if (NULL == sops) { ++ MALI_DEBUG_PRINT(2, ("Mali didn't get any scpi ops \n")); ++ return -1; ++ } ++ ++ /* Hard coded for Juno. 2 is GPU domain */ ++ sinfo = sops->dvfs_get_info(2); ++ if (IS_ERR_OR_NULL(sinfo)) ++ return PTR_ERR(sinfo); ++ ++ for (i = 0; i < sinfo->count; i++) { ++ struct scpi_opp *e = &sinfo->opps[i]; ++ ++ MALI_DEBUG_PRINT(2, ("Mali Remove OPP: %u Hz \n", e->freq)); ++ ++ dev_pm_opp_remove(dev, e->freq); ++ } ++ ++ return 0; ++ ++} ++#endif ++#endif ++#endif ++#endif +diff --git a/drivers/gpu/arm/mali400/mali/platform/rk/custom_log.h b/drivers/gpu/arm/mali400/mali/platform/rk/custom_log.h +new file mode 100755 +index 000000000000..fe5e1224149e +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/platform/rk/custom_log.h +@@ -0,0 +1,209 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* ---------------------------------------------------------------------------- ++ * File: custom_log.h ++ * ++ * Desc: ChenZhen å好的 log 输出的定制实现. ++ * ++ * -------------------------------------------------------------------- ++ * < 习语 å’Œ 缩略语 > : ++ * ++ * -------------------------------------------------------------------- ++ * Usage: ++ * ++ * Note: ++ * ++ * Author: ChenZhen ++ * ++ * ---------------------------------------------------------------------------- ++ * Version: ++ * v1.0 ++ * ---------------------------------------------------------------------------- ++ * Log: ++ ----Fri Nov 19 15:20:28 2010 v1.0 ++ * ++ * ---------------------------------------------------------------------------- ++ */ ++ ++#ifndef __CUSTOM_LOG_H__ ++#define __CUSTOM_LOG_H__ ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++/* ----------------------------------------------------------------------------- ++ * Include Files ++ * ----------------------------------------------------------------------------- ++ */ ++#include ++#include ++ ++/* ----------------------------------------------------------------------------- ++ * Macros Definition ++ * ----------------------------------------------------------------------------- ++ */ ++ ++/** 若下列 macro 有被定义, æ‰ ä½¿èƒ½ log 输出. */ ++/* #define ENABLE_DEBUG_LOG */ ++ ++/*----------------------------------------------------------------------------*/ ++ ++#ifdef ENABLE_VERBOSE_LOG ++/** Verbose log. */ ++#define V(fmt, args...) \ ++ pr_debug("V : [File] : %s; [Line] : %d; [Func] : %s(); " fmt \ ++ "\n", \ ++ __FILE__, \ ++ __LINE__, \ ++ __func__, \ ++ ## args) ++#else ++#define V(...) ((void)0) ++#endif ++ ++#ifdef ENABLE_DEBUG_LOG ++/** Debug log. */ ++#define D(fmt, args...) \ ++ pr_info("D : [File] : %s; [Line] : %d; [Func] : %s(); " fmt \ ++ "\n", \ ++ __FILE__, \ ++ __LINE__, \ ++ __func__, \ ++ ## args) ++#else ++#define D(...) ((void)0) ++#endif ++ ++#define I(fmt, args...) \ ++ pr_info("I : [File] : %s; [Line] : %d; [Func] : %s(); " fmt \ ++ "\n", \ ++ __FILE__, \ ++ __LINE__, \ ++ __func__, \ ++ ## args) ++ ++#define W(fmt, args...) \ ++ pr_warn("W : [File] : %s; [Line] : %d; [Func] : %s(); " \ ++ fmt "\n", \ ++ __FILE__, \ ++ __LINE__, \ ++ __func__, \ ++ ## args) ++ ++#define E(fmt, args...) \ ++ pr_err("E : [File] : %s; [Line] : %d; [Func] : %s(); " fmt \ ++ "\n", \ ++ __FILE__, \ ++ __LINE__, \ ++ __func__, \ ++ ## args) ++ ++/*-------------------------------------------------------*/ ++ ++/** 使用 D(), 以å进制的形å¼æ‰“å°å˜é‡ 'var' çš„ value. */ ++#define D_DEC(var) D(#var " = %d.", var) ++ ++#define E_DEC(var) E(#var " = %d.", var) ++ ++/** 使用 D(), 以å六进制的形å¼æ‰“å°å˜é‡ 'var' çš„ value. */ ++#define D_HEX(var) D(#var " = 0x%x.", var) ++ ++#define E_HEX(var) E(#var " = 0x%x.", var) ++ ++/** ++ * 使用 D(), 以å六进制的形å¼, ++ * 打å°æŒ‡é’ˆç±»åž‹å˜é‡ 'ptr' çš„ value. ++ */ ++#define D_PTR(ptr) D(#ptr " = %p.", ptr) ++ ++#define E_PTR(ptr) E(#ptr " = %p.", ptr) ++ ++/** 使用 D(), æ‰“å° char 字串. */ ++#define D_STR(p_str) \ ++do { \ ++ if (!p_str) { \ ++ D(#p_str " = NULL."); \ ++ else \ ++ D(#p_str " = '%s'.", p_str); \ ++} while (0) ++ ++#define E_STR(p_str) \ ++do { \ ++ if (!p_str) \ ++ E(#p_str " = NULL."); \ ++ else \ ++ E(#p_str " = '%s'.", p_str); \ ++} while (0) ++ ++#ifdef ENABLE_DEBUG_LOG ++/** ++ * log 从 'p_start' 地å€å¼€å§‹çš„ 'len' 个字节的数æ®. ++ */ ++#define D_MEM(p_start, len) \ ++do { \ ++ int i = 0; \ ++ char *p = (char *)(p_start); \ ++ D("dump memory from addr of '" #p_start "', from %p, length %d' : ", \ ++ (p_start), \ ++ (len)); \ ++ pr_debug("\t\t"); \ ++ for (i = 0; i < (len); i++) \ ++ pr_debug("0x%02x, ", p[i]); \ ++ pr_debug("\n"); \ ++} while (0) ++#else ++#define D_MEM(...) ((void)0) ++#endif ++ ++/*-------------------------------------------------------*/ ++ ++/** ++ * 在特定æ¡ä»¶ä¸‹, 判定 error å‘生, ++ * å°†å˜é‡ 'ret_var' 设置 'err_code', ++ * log 输出对应的 Error Caution, ++ * 然åŽè·³è½¬ 'label' 指定的代ç å¤„执行. ++ * @param msg ++ * 纯字串形å¼çš„æ示信æ¯. ++ * @param ret_var ++ * 标识函数执行状æ€æˆ–者结果的å˜é‡, ++ * 将被设置具体的 Error Code. ++ * 通常是 'ret' or 'result'. ++ * @param err_code ++ * 表å¾ç‰¹å®š error 的常数标识, ++ * 通常是 å®çš„å½¢æ€. ++ * @param label ++ * 程åºå°†è¦è·³è½¬åˆ°çš„错误处ç†ä»£ç çš„æ ‡å·, ++ * 通常就是 'EXIT'. ++ * @param args... ++ * 对应 'msg_fmt' 实å‚中, ++ * '%s', '%d', ... 等转æ¢è¯´æ˜Žç¬¦çš„具体å¯å˜é•¿å®žå‚. ++ */ ++#define SET_ERROR_AND_JUMP(msg_fmt, ret_var, err_code, label, args...) \ ++do { \ ++ E("To set '" #ret_var "' to %d('" #err_code "'), because : " msg_fmt, \ ++ (err_code), \ ++ ## args); \ ++ (ret_var) = (err_code); \ ++ goto label; \ ++} while (0) ++ ++/* ----------------------------------------------------------------------------- ++ * Types and Structures Definition ++ * ----------------------------------------------------------------------------- ++ */ ++ ++/* ----------------------------------------------------------------------------- ++ * Global Functions' Prototype ++ * ----------------------------------------------------------------------------- ++ */ ++ ++/* ----------------------------------------------------------------------------- ++ * Inline Functions Implementation ++ * ----------------------------------------------------------------------------- ++ */ ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif /* __CUSTOM_LOG_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/platform/rk/rk.c b/drivers/gpu/arm/mali400/mali/platform/rk/rk.c +new file mode 100755 +index 000000000000..9a012fdf8d35 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/platform/rk/rk.c +@@ -0,0 +1,676 @@ ++/* ++ * (C) COPYRIGHT RockChip Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ */ ++ ++/** ++ * @file rk.c ++ * implementation of platform_specific_code on rk platforms, such as rk3328h. ++ * ++ * mali_device_driver(MDD) includes 2 parts : ++ * .DP : platform_dependent_part : ++ * located in /mali/platform// ++ * .DP : common_part : ++ * common part implemented by ARM. ++ */ ++ ++#define ENABLE_DEBUG_LOG ++#include "custom_log.h" ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#ifdef CONFIG_PM ++#include ++#endif ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#include "mali_kernel_common.h" ++#include "../../common/mali_osk_mali.h" ++ ++/*---------------------------------------------------------------------------*/ ++ ++u32 mali_group_error; ++ ++/*---------------------------------------------------------------------------*/ ++ ++#define DEFAULT_UTILISATION_PERIOD_IN_MS (100) ++ ++/* ++ * rk_platform_context_of_mali_device. ++ */ ++struct rk_context { ++ /* mali device. */ ++ struct device *dev; ++ /* is the GPU powered on? */ ++ bool is_powered; ++ /* debug only, the period in ms to count gpu_utilisation. */ ++ unsigned int utilisation_period; ++}; ++ ++struct rk_context *s_rk_context; ++ ++/*---------------------------------------------------------------------------*/ ++ ++#ifdef CONFIG_MALI_DEVFREQ ++static ssize_t utilisation_period_show(struct device *dev, ++ struct device_attribute *attr, ++ char *buf) ++{ ++ struct rk_context *platform = s_rk_context; ++ ssize_t ret = 0; ++ ++ ret += snprintf(buf, PAGE_SIZE, "%u\n", platform->utilisation_period); ++ ++ return ret; ++} ++ ++static ssize_t utilisation_period_store(struct device *dev, ++ struct device_attribute *attr, ++ const char *buf, ++ size_t count) ++{ ++ struct rk_context *platform = s_rk_context; ++ int ret = 0; ++ ++ ret = kstrtouint(buf, 0, &platform->utilisation_period); ++ if (ret) { ++ E("invalid input period : %s.", buf); ++ return ret; ++ } ++ D("set utilisation_period to '%d'.", platform->utilisation_period); ++ ++ return count; ++} ++ ++static ssize_t utilisation_show(struct device *dev, ++ struct device_attribute *attr, ++ char *buf) ++{ ++ struct rk_context *platform = s_rk_context; ++ struct mali_device *mdev = dev_get_drvdata(dev); ++ ssize_t ret = 0; ++ unsigned long period_in_us = platform->utilisation_period * 1000; ++ unsigned long total_time; ++ unsigned long busy_time; ++ unsigned long utilisation; ++ ++ mali_pm_reset_dvfs_utilisation(mdev); ++ usleep_range(period_in_us, period_in_us + 100); ++ mali_pm_get_dvfs_utilisation(mdev, &total_time, &busy_time); ++ ++ /* 'devfreq_dev_profile' instance registered to devfreq ++ * also uses mali_pm_reset_dvfs_utilisation() ++ * and mali_pm_get_dvfs_utilisation(). ++ * So, it's better to disable GPU DVFS before reading this node. ++ */ ++ D("total_time : %lu, busy_time : %lu.", total_time, busy_time); ++ ++ utilisation = busy_time / (total_time / 100); ++ ret += snprintf(buf, PAGE_SIZE, "%lu\n", utilisation); ++ ++ return ret; ++} ++ ++static DEVICE_ATTR_RW(utilisation_period); ++static DEVICE_ATTR_RO(utilisation); ++#endif ++ ++static int rk_context_create_sysfs_files(struct device *dev) ++{ ++#ifdef CONFIG_MALI_DEVFREQ ++ int ret; ++ ++ ret = device_create_file(dev, &dev_attr_utilisation_period); ++ if (ret) { ++ E("fail to create sysfs file 'utilisation_period'."); ++ goto out; ++ } ++ ++ ret = device_create_file(dev, &dev_attr_utilisation); ++ if (ret) { ++ E("fail to create sysfs file 'utilisation'."); ++ goto remove_utilisation_period; ++ } ++ ++ return 0; ++ ++remove_utilisation_period: ++ device_remove_file(dev, &dev_attr_utilisation_period); ++out: ++ return ret; ++#else ++ return 0; ++#endif ++} ++ ++static void rk_context_remove_sysfs_files(struct device *dev) ++{ ++#ifdef CONFIG_MALI_DEVFREQ ++ device_remove_file(dev, &dev_attr_utilisation_period); ++ device_remove_file(dev, &dev_attr_utilisation); ++#endif ++} ++ ++/*---------------------------------------------------------------------------*/ ++ ++/* ++ * Init rk_platform_context of mali_device. ++ */ ++static int rk_context_init(struct platform_device *pdev) ++{ ++ int ret = 0; ++ struct device *dev = &pdev->dev; ++ struct rk_context *platform; /* platform_context */ ++ ++ platform = kzalloc(sizeof(*platform), GFP_KERNEL); ++ if (!platform) { ++ E("no mem."); ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ ++ platform->dev = dev; ++ platform->is_powered = false; ++ ++ platform->utilisation_period = DEFAULT_UTILISATION_PERIOD_IN_MS; ++ ++ ret = rk_context_create_sysfs_files(dev); ++ if (ret) { ++ E("fail to create sysfs files, ret = %d", ret); ++ goto EXIT; ++ } ++ ++ s_rk_context = platform; ++ ++ pm_runtime_set_autosuspend_delay(dev, 1000); ++ pm_runtime_use_autosuspend(dev); ++ pm_runtime_enable(dev); ++ ++EXIT: ++ return ret; ++} ++ ++static void rk_context_deinit(struct platform_device *pdev) ++{ ++ struct device *dev = &pdev->dev; ++ struct rk_context *platform = s_rk_context; ++ ++ pm_runtime_disable(dev); ++ ++ s_rk_context = NULL; ++ ++ rk_context_remove_sysfs_files(dev); ++ ++ if (platform) { ++ platform->is_powered = false; ++ platform->dev = NULL; ++ kfree(platform); ++ } ++} ++ ++/*---------------------------------------------------------------------------*/ ++/* for devfreq cooling. */ ++ ++#if defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL) ++ ++#define FALLBACK_STATIC_TEMPERATURE 55000 ++ ++static u32 dynamic_coefficient; ++static u32 static_coefficient; ++static s32 ts[4]; ++static struct thermal_zone_device *gpu_tz; ++ ++static int power_model_simple_init(struct platform_device *pdev) ++{ ++ struct device_node *power_model_node; ++ const char *tz_name; ++ u32 static_power, dynamic_power; ++ u32 voltage, voltage_squared, voltage_cubed, frequency; ++ ++ power_model_node = of_get_child_by_name(pdev->dev.of_node, ++ "power_model"); ++ if (!power_model_node) { ++ dev_err(&pdev->dev, "could not find power_model node\n"); ++ return -ENODEV; ++ } ++ if (!of_device_is_compatible(power_model_node, ++ "arm,mali-simple-power-model")) { ++ dev_err(&pdev->dev, "power_model incompatible with simple power model\n"); ++ return -ENODEV; ++ } ++ ++ if (of_property_read_string(power_model_node, "thermal-zone", ++ &tz_name)) { ++ dev_err(&pdev->dev, "ts in power_model not available\n"); ++ return -EINVAL; ++ } ++ ++ gpu_tz = thermal_zone_get_zone_by_name(tz_name); ++ if (IS_ERR(gpu_tz)) { ++ pr_warn_ratelimited("Error getting gpu thermal zone '%s'(%ld), not yet ready?\n", ++ tz_name, ++ PTR_ERR(gpu_tz)); ++ gpu_tz = NULL; ++ } ++ ++ if (of_property_read_u32(power_model_node, "static-power", ++ &static_power)) { ++ dev_err(&pdev->dev, "static-power in power_model not available\n"); ++ return -EINVAL; ++ } ++ if (of_property_read_u32(power_model_node, "dynamic-power", ++ &dynamic_power)) { ++ dev_err(&pdev->dev, "dynamic-power in power_model not available\n"); ++ return -EINVAL; ++ } ++ if (of_property_read_u32(power_model_node, "voltage", ++ &voltage)) { ++ dev_err(&pdev->dev, "voltage in power_model not available\n"); ++ return -EINVAL; ++ } ++ if (of_property_read_u32(power_model_node, "frequency", ++ &frequency)) { ++ dev_err(&pdev->dev, "frequency in power_model not available\n"); ++ return -EINVAL; ++ } ++ voltage_squared = (voltage * voltage) / 1000; ++ voltage_cubed = voltage * voltage * voltage; ++ static_coefficient = (static_power << 20) / (voltage_cubed >> 10); ++ dynamic_coefficient = (((dynamic_power * 1000) / voltage_squared) ++ * 1000) / frequency; ++ ++ if (of_property_read_u32_array(power_model_node, "ts", (u32 *)ts, 4)) { ++ dev_err(&pdev->dev, "ts in power_model not available\n"); ++ return -EINVAL; ++ } ++ ++ return 0; ++} ++ ++/* Calculate gpu static power example for reference */ ++static unsigned long rk_model_static_power(struct devfreq *devfreq, ++ unsigned long voltage) ++{ ++ int temperature, temp; ++ int temp_squared, temp_cubed, temp_scaling_factor; ++ const unsigned long voltage_cubed = (voltage * voltage * voltage) >> 10; ++ unsigned long static_power; ++ ++ if (gpu_tz) { ++ int ret; ++ ++ ret = gpu_tz->ops->get_temp(gpu_tz, &temperature); ++ if (ret) { ++ MALI_DEBUG_PRINT(2, ("fail to read temp: %d\n", ret)); ++ temperature = FALLBACK_STATIC_TEMPERATURE; ++ } ++ } else { ++ temperature = FALLBACK_STATIC_TEMPERATURE; ++ } ++ ++ /* Calculate the temperature scaling factor. To be applied to the ++ * voltage scaled power. ++ */ ++ temp = temperature / 1000; ++ temp_squared = temp * temp; ++ temp_cubed = temp_squared * temp; ++ temp_scaling_factor = ++ (ts[3] * temp_cubed) ++ + (ts[2] * temp_squared) ++ + (ts[1] * temp) ++ + ts[0]; ++ ++ static_power = (((static_coefficient * voltage_cubed) >> 20) ++ * temp_scaling_factor) ++ / 1000000; ++ ++ return static_power; ++} ++ ++/* Calculate gpu dynamic power example for reference */ ++static unsigned long rk_model_dynamic_power(struct devfreq *devfreq, ++ unsigned long freq, ++ unsigned long voltage) ++{ ++ /* The inputs: freq (f) is in Hz, and voltage (v) in mV. ++ * The coefficient (c) is in mW/(MHz mV mV). ++ * ++ * This function calculates the dynamic power after this formula: ++ * Pdyn (mW) = c (mW/(MHz*mV*mV)) * v (mV) * v (mV) * f (MHz) ++ */ ++ const unsigned long v2 = (voltage * voltage) / 1000; /* m*(V*V) */ ++ const unsigned long f_mhz = freq / 1000000; /* MHz */ ++ unsigned long dynamic_power; ++ ++ dynamic_power = (dynamic_coefficient * v2 * f_mhz) / 1000000; /* mW */ ++ ++ return dynamic_power; ++} ++ ++struct devfreq_cooling_power rk_cooling_ops = { ++ .get_static_power = rk_model_static_power, ++ .get_dynamic_power = rk_model_dynamic_power, ++}; ++#endif ++ ++/*---------------------------------------------------------------------------*/ ++ ++#ifdef CONFIG_PM ++ ++static int rk_platform_enable_clk_gpu(struct device *dev) ++{ ++ int ret = 0; ++#if defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_HAVE_CLK) ++ struct mali_device *mdev = dev_get_drvdata(dev); ++ ++ if (mdev->clock) ++ ret = clk_enable(mdev->clock); ++#endif ++ return ret; ++} ++ ++static void rk_platform_disable_clk_gpu(struct device *dev) ++{ ++#if defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_HAVE_CLK) ++ struct mali_device *mdev = dev_get_drvdata(dev); ++ ++ if (mdev->clock) ++ clk_disable(mdev->clock); ++#endif ++} ++ ++static int rk_platform_enable_gpu_regulator(struct device *dev) ++{ ++ int ret = 0; ++#if defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_REGULATOR) ++ struct mali_device *mdev = dev_get_drvdata(dev); ++ ++ if (mdev->regulator) ++ ret = regulator_enable(mdev->regulator); ++#endif ++ return ret; ++} ++ ++static void rk_platform_disable_gpu_regulator(struct device *dev) ++{ ++#if defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_REGULATOR) ++ struct mali_device *mdev = dev_get_drvdata(dev); ++ ++ if (mdev->regulator) ++ regulator_disable(mdev->regulator); ++#endif ++} ++ ++static int rk_platform_power_on_gpu(struct device *dev) ++{ ++ struct rk_context *platform = s_rk_context; ++ int ret = 0; ++ ++ if (!(platform->is_powered)) { ++ ret = rk_platform_enable_clk_gpu(dev); ++ if (ret) { ++ E("fail to enable clk_gpu, ret : %d.", ret); ++ goto fail_to_enable_clk; ++ } ++ ++ ret = rk_platform_enable_gpu_regulator(dev); ++ if (ret) { ++ E("fail to enable vdd_gpu, ret : %d.", ret); ++ goto fail_to_enable_regulator; ++ } ++ ++ platform->is_powered = true; ++ } ++ ++ return 0; ++ ++fail_to_enable_regulator: ++ rk_platform_disable_clk_gpu(dev); ++ ++fail_to_enable_clk: ++ return ret; ++} ++ ++static void rk_platform_power_off_gpu(struct device *dev) ++{ ++ struct rk_context *platform = s_rk_context; ++ ++ if (platform->is_powered) { ++ rk_platform_disable_clk_gpu(dev); ++ rk_platform_disable_gpu_regulator(dev); ++ ++ platform->is_powered = false; ++ } ++} ++ ++int rk_platform_init_opp_table(struct device *dev) ++{ ++ return rockchip_init_opp_table(dev, NULL, "gpu_leakage", "mali"); ++} ++ ++static int mali_runtime_suspend(struct device *device) ++{ ++ int ret = 0; ++ ++ MALI_DEBUG_PRINT(4, ("mali_runtime_suspend() called\n")); ++ ++ if (device->driver && ++ device->driver->pm && ++ device->driver->pm->runtime_suspend) { ++ /* Need to notify Mali driver about this event */ ++ ret = device->driver->pm->runtime_suspend(device); ++ } ++ ++ if (!ret) ++ rk_platform_power_off_gpu(device); ++ ++ return ret; ++} ++ ++static int mali_runtime_resume(struct device *device) ++{ ++ int ret = 0; ++ ++ MALI_DEBUG_PRINT(4, ("mali_runtime_resume() called\n")); ++ ++ rk_platform_power_on_gpu(device); ++ ++ if (device->driver && ++ device->driver->pm && ++ device->driver->pm->runtime_resume) { ++ /* Need to notify Mali driver about this event */ ++ ret = device->driver->pm->runtime_resume(device); ++ } ++ ++ return ret; ++} ++ ++static int mali_runtime_idle(struct device *device) ++{ ++ int ret = 0; ++ ++ MALI_DEBUG_PRINT(4, ("mali_runtime_idle() called\n")); ++ ++ if (device->driver && ++ device->driver->pm && ++ device->driver->pm->runtime_idle) { ++ /* Need to notify Mali driver about this event */ ++ ret = device->driver->pm->runtime_idle(device); ++ if (ret) ++ return ret; ++ } ++ ++ return 0; ++} ++#endif ++ ++static int mali_os_suspend(struct device *device) ++{ ++ int ret = 0; ++ ++ MALI_DEBUG_PRINT(4, ("mali_os_suspend() called\n")); ++ ++ if (device->driver && ++ device->driver->pm && ++ device->driver->pm->suspend) { ++ /* Need to notify Mali driver about this event */ ++ ret = device->driver->pm->suspend(device); ++ } ++ ++ if (!ret) ++ rk_platform_power_off_gpu(device); ++ ++ return ret; ++} ++ ++static int mali_os_resume(struct device *device) ++{ ++ int ret = 0; ++ ++ MALI_DEBUG_PRINT(4, ("mali_os_resume() called\n")); ++ ++ rk_platform_power_on_gpu(device); ++ ++ if (device->driver && ++ device->driver->pm && ++ device->driver->pm->resume) { ++ /* Need to notify Mali driver about this event */ ++ ret = device->driver->pm->resume(device); ++ } ++ ++ return ret; ++} ++ ++static int mali_os_freeze(struct device *device) ++{ ++ int ret = 0; ++ ++ MALI_DEBUG_PRINT(4, ("mali_os_freeze() called\n")); ++ ++ if (device->driver && ++ device->driver->pm && ++ device->driver->pm->freeze) { ++ /* Need to notify Mali driver about this event */ ++ ret = device->driver->pm->freeze(device); ++ } ++ ++ return ret; ++} ++ ++static int mali_os_thaw(struct device *device) ++{ ++ int ret = 0; ++ ++ MALI_DEBUG_PRINT(4, ("mali_os_thaw() called\n")); ++ ++ if (device->driver && ++ device->driver->pm && ++ device->driver->pm->thaw) { ++ /* Need to notify Mali driver about this event */ ++ ret = device->driver->pm->thaw(device); ++ } ++ ++ return ret; ++} ++ ++static const struct dev_pm_ops mali_gpu_device_type_pm_ops = { ++ .suspend = mali_os_suspend, ++ .resume = mali_os_resume, ++ .freeze = mali_os_freeze, ++ .thaw = mali_os_thaw, ++#ifdef CONFIG_PM ++ .runtime_suspend = mali_runtime_suspend, ++ .runtime_resume = mali_runtime_resume, ++ .runtime_idle = mali_runtime_idle, ++#endif ++}; ++ ++static const struct device_type mali_gpu_device_device_type = { ++ .pm = &mali_gpu_device_type_pm_ops, ++}; ++ ++/* ++ * platform_specific_data of platform_device of mali_gpu. ++ */ ++static const struct mali_gpu_device_data mali_gpu_data = { ++ .shared_mem_size = 1024 * 1024 * 1024, /* 1GB */ ++ .max_job_runtime = 60000, /* 60 seconds */ ++#if defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL) ++ .gpu_cooling_ops = &rk_cooling_ops, ++#endif ++}; ++ ++static void mali_platform_device_add_config(struct platform_device *pdev) ++{ ++ pdev->name = MALI_GPU_NAME_UTGARD, ++ pdev->id = 0; ++ pdev->dev.type = &mali_gpu_device_device_type; ++ pdev->dev.dma_mask = &pdev->dev.coherent_dma_mask, ++ pdev->dev.coherent_dma_mask = DMA_BIT_MASK(32); ++} ++ ++/*---------------------------------------------------------------------------*/ ++/* platform_device_functions called by common_part. */ ++ ++int mali_platform_device_init(struct platform_device *pdev) ++{ ++ int err = 0; ++ ++ mali_platform_device_add_config(pdev); ++ ++ D("to add platform_specific_data to platform_device_of_mali."); ++ err = platform_device_add_data(pdev, ++ &mali_gpu_data, ++ sizeof(mali_gpu_data)); ++ if (err) { ++ E("fail to add platform_specific_data. err : %d.", err); ++ goto add_data_failed; ++ } ++ ++ err = rk_context_init(pdev); ++ if (err) { ++ E("fail to init rk_context. err : %d.", err); ++ goto init_rk_context_failed; ++ } ++ ++#if defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL) ++ if (of_machine_is_compatible("rockchip,rk3036")) ++ return 0; ++ ++ err = power_model_simple_init(pdev); ++ if (err) { ++ E("fail to init simple_power_model, err : %d.", err); ++ goto init_power_model_failed; ++ } ++#endif ++ ++ return 0; ++ ++#if defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL) ++init_power_model_failed: ++ rk_context_deinit(pdev); ++#endif ++init_rk_context_failed: ++add_data_failed: ++ return err; ++} ++ ++void mali_platform_device_deinit(struct platform_device *pdev) ++{ ++ MALI_DEBUG_PRINT(4, ("mali_platform_device_unregister() called\n")); ++ ++ rk_context_deinit(pdev); ++} +diff --git a/drivers/gpu/arm/mali400/mali/platform/rk/rk_ext.h b/drivers/gpu/arm/mali400/mali/platform/rk/rk_ext.h +new file mode 100755 +index 000000000000..bd939350c425 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/platform/rk/rk_ext.h +@@ -0,0 +1,37 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* ---------------------------------------------------------------------------- ++ * File: rk_ext.h ++ * ++ * Desc: rk_ext_on_mali_ko 中的 通行定义等. ++ * ++ * Usage: ++ * ++ * Note: ++ * ++ * Author: ChenZhen ++ * ++ * Log: ++ * ++ * ---------------------------------------------------------------------------- ++ */ ++ ++#ifndef __RK_EXT_H__ ++#define __RK_EXT_H__ ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++/*---------------------------------------------------------------------------*/ ++ ++/** version of rk_ext on mali_ko, aka. rk_ko_ver. */ ++#define RK_KO_VER (5) ++ ++/*---------------------------------------------------------------------------*/ ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif /* __RK_EXT_H__ */ ++ +diff --git a/drivers/gpu/arm/mali400/mali/readme.txt b/drivers/gpu/arm/mali400/mali/readme.txt +new file mode 100755 +index 000000000000..6785ac933b38 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/readme.txt +@@ -0,0 +1,28 @@ ++Building the Mali Device Driver for Linux ++----------------------------------------- ++ ++Build the Mali Device Driver for Linux by running the following make command: ++ ++KDIR= USING_UMP= BUILD= make ++ ++where ++ kdir_path: Path to your Linux Kernel directory ++ ump_option: 1 = Enable UMP support(*) ++ 0 = disable UMP support ++ build_option: debug = debug build of driver ++ release = release build of driver ++ ++(*) For newer Linux Kernels, the Module.symvers file for the UMP device driver ++ must be available. The UMP_SYMVERS_FILE variable in the Makefile should ++ point to this file. This file is generated when the UMP driver is built. ++ ++The result will be a mali.ko file, which can be loaded into the Linux kernel ++by using the insmod command. ++ ++Use of UMP is not recommended. The dma-buf API in the Linux kernel has ++replaced UMP. The Mali Device Driver will be built with dma-buf support if the ++kernel config includes enabled dma-buf. ++ ++The kernel needs to be provided with a platform_device struct for the Mali GPU ++device. See the mali_utgard.h header file for how to set up the Mali GPU ++resources. +diff --git a/drivers/gpu/arm/mali400/mali/regs/mali_200_regs.h b/drivers/gpu/arm/mali400/mali/regs/mali_200_regs.h +new file mode 100755 +index 000000000000..0345fb169a95 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/regs/mali_200_regs.h +@@ -0,0 +1,131 @@ ++/* ++ * Copyright (C) 2010, 2012-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef _MALI200_REGS_H_ ++#define _MALI200_REGS_H_ ++ ++/** ++ * Enum for management register addresses. ++ */ ++enum mali200_mgmt_reg { ++ MALI200_REG_ADDR_MGMT_VERSION = 0x1000, ++ MALI200_REG_ADDR_MGMT_CURRENT_REND_LIST_ADDR = 0x1004, ++ MALI200_REG_ADDR_MGMT_STATUS = 0x1008, ++ MALI200_REG_ADDR_MGMT_CTRL_MGMT = 0x100c, ++ ++ MALI200_REG_ADDR_MGMT_INT_RAWSTAT = 0x1020, ++ MALI200_REG_ADDR_MGMT_INT_CLEAR = 0x1024, ++ MALI200_REG_ADDR_MGMT_INT_MASK = 0x1028, ++ MALI200_REG_ADDR_MGMT_INT_STATUS = 0x102c, ++ ++ MALI200_REG_ADDR_MGMT_BUS_ERROR_STATUS = 0x1050, ++ ++ MALI200_REG_ADDR_MGMT_PERF_CNT_0_ENABLE = 0x1080, ++ MALI200_REG_ADDR_MGMT_PERF_CNT_0_SRC = 0x1084, ++ MALI200_REG_ADDR_MGMT_PERF_CNT_0_LIMIT = 0x1088, ++ MALI200_REG_ADDR_MGMT_PERF_CNT_0_VALUE = 0x108c, ++ ++ MALI200_REG_ADDR_MGMT_PERF_CNT_1_ENABLE = 0x10a0, ++ MALI200_REG_ADDR_MGMT_PERF_CNT_1_SRC = 0x10a4, ++ MALI200_REG_ADDR_MGMT_PERF_CNT_1_VALUE = 0x10ac, ++ ++ MALI200_REG_ADDR_MGMT_PERFMON_CONTR = 0x10b0, ++ MALI200_REG_ADDR_MGMT_PERFMON_BASE = 0x10b4, ++ ++ MALI200_REG_SIZEOF_REGISTER_BANK = 0x10f0 ++ ++}; ++ ++#define MALI200_REG_VAL_PERF_CNT_ENABLE 1 ++ ++enum mali200_mgmt_ctrl_mgmt { ++ MALI200_REG_VAL_CTRL_MGMT_STOP_BUS = (1 << 0), ++ MALI200_REG_VAL_CTRL_MGMT_FLUSH_CACHES = (1 << 3), ++ MALI200_REG_VAL_CTRL_MGMT_FORCE_RESET = (1 << 5), ++ MALI200_REG_VAL_CTRL_MGMT_START_RENDERING = (1 << 6), ++ MALI400PP_REG_VAL_CTRL_MGMT_SOFT_RESET = (1 << 7), /* Only valid for Mali-300 and later */ ++}; ++ ++enum mali200_mgmt_irq { ++ MALI200_REG_VAL_IRQ_END_OF_FRAME = (1 << 0), ++ MALI200_REG_VAL_IRQ_END_OF_TILE = (1 << 1), ++ MALI200_REG_VAL_IRQ_HANG = (1 << 2), ++ MALI200_REG_VAL_IRQ_FORCE_HANG = (1 << 3), ++ MALI200_REG_VAL_IRQ_BUS_ERROR = (1 << 4), ++ MALI200_REG_VAL_IRQ_BUS_STOP = (1 << 5), ++ MALI200_REG_VAL_IRQ_CNT_0_LIMIT = (1 << 6), ++ MALI200_REG_VAL_IRQ_CNT_1_LIMIT = (1 << 7), ++ MALI200_REG_VAL_IRQ_WRITE_BOUNDARY_ERROR = (1 << 8), ++ MALI400PP_REG_VAL_IRQ_INVALID_PLIST_COMMAND = (1 << 9), ++ MALI400PP_REG_VAL_IRQ_CALL_STACK_UNDERFLOW = (1 << 10), ++ MALI400PP_REG_VAL_IRQ_CALL_STACK_OVERFLOW = (1 << 11), ++ MALI400PP_REG_VAL_IRQ_RESET_COMPLETED = (1 << 12), ++}; ++ ++#define MALI200_REG_VAL_IRQ_MASK_ALL ((enum mali200_mgmt_irq) (\ ++ MALI200_REG_VAL_IRQ_END_OF_FRAME |\ ++ MALI200_REG_VAL_IRQ_END_OF_TILE |\ ++ MALI200_REG_VAL_IRQ_HANG |\ ++ MALI200_REG_VAL_IRQ_FORCE_HANG |\ ++ MALI200_REG_VAL_IRQ_BUS_ERROR |\ ++ MALI200_REG_VAL_IRQ_BUS_STOP |\ ++ MALI200_REG_VAL_IRQ_CNT_0_LIMIT |\ ++ MALI200_REG_VAL_IRQ_CNT_1_LIMIT |\ ++ MALI200_REG_VAL_IRQ_WRITE_BOUNDARY_ERROR |\ ++ MALI400PP_REG_VAL_IRQ_INVALID_PLIST_COMMAND |\ ++ MALI400PP_REG_VAL_IRQ_CALL_STACK_UNDERFLOW |\ ++ MALI400PP_REG_VAL_IRQ_CALL_STACK_OVERFLOW |\ ++ MALI400PP_REG_VAL_IRQ_RESET_COMPLETED)) ++ ++#define MALI200_REG_VAL_IRQ_MASK_USED ((enum mali200_mgmt_irq) (\ ++ MALI200_REG_VAL_IRQ_END_OF_FRAME |\ ++ MALI200_REG_VAL_IRQ_FORCE_HANG |\ ++ MALI200_REG_VAL_IRQ_BUS_ERROR |\ ++ MALI200_REG_VAL_IRQ_WRITE_BOUNDARY_ERROR |\ ++ MALI400PP_REG_VAL_IRQ_INVALID_PLIST_COMMAND |\ ++ MALI400PP_REG_VAL_IRQ_CALL_STACK_UNDERFLOW |\ ++ MALI400PP_REG_VAL_IRQ_CALL_STACK_OVERFLOW)) ++ ++#define MALI200_REG_VAL_IRQ_MASK_NONE ((enum mali200_mgmt_irq)(0)) ++ ++enum mali200_mgmt_status { ++ MALI200_REG_VAL_STATUS_RENDERING_ACTIVE = (1 << 0), ++ MALI200_REG_VAL_STATUS_BUS_STOPPED = (1 << 4), ++}; ++ ++enum mali200_render_unit { ++ MALI200_REG_ADDR_FRAME = 0x0000, ++ MALI200_REG_ADDR_RSW = 0x0004, ++ MALI200_REG_ADDR_STACK = 0x0030, ++ MALI200_REG_ADDR_STACK_SIZE = 0x0034, ++ MALI200_REG_ADDR_ORIGIN_OFFSET_X = 0x0040 ++}; ++ ++enum mali200_wb_unit { ++ MALI200_REG_ADDR_WB0 = 0x0100, ++ MALI200_REG_ADDR_WB1 = 0x0200, ++ MALI200_REG_ADDR_WB2 = 0x0300 ++}; ++ ++enum mali200_wb_unit_regs { ++ MALI200_REG_ADDR_WB_SOURCE_SELECT = 0x0000, ++ MALI200_REG_ADDR_WB_SOURCE_ADDR = 0x0004, ++}; ++ ++/* This should be in the top 16 bit of the version register of Mali PP */ ++#define MALI200_PP_PRODUCT_ID 0xC807 ++#define MALI300_PP_PRODUCT_ID 0xCE07 ++#define MALI400_PP_PRODUCT_ID 0xCD07 ++#define MALI450_PP_PRODUCT_ID 0xCF07 ++#define MALI470_PP_PRODUCT_ID 0xCF08 ++ ++ ++ ++#endif /* _MALI200_REGS_H_ */ +diff --git a/drivers/gpu/arm/mali400/mali/regs/mali_gp_regs.h b/drivers/gpu/arm/mali400/mali/regs/mali_gp_regs.h +new file mode 100755 +index 000000000000..7f8b58fd6c49 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/regs/mali_gp_regs.h +@@ -0,0 +1,172 @@ ++/* ++ * Copyright (C) 2010, 2012-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef _MALIGP2_CONROL_REGS_H_ ++#define _MALIGP2_CONROL_REGS_H_ ++ ++/** ++ * These are the different geometry processor control registers. ++ * Their usage is to control and monitor the operation of the ++ * Vertex Shader and the Polygon List Builder in the geometry processor. ++ * Addresses are in 32-bit word relative sizes. ++ * @see [P0081] "Geometry Processor Data Structures" for details ++ */ ++ ++typedef enum { ++ MALIGP2_REG_ADDR_MGMT_VSCL_START_ADDR = 0x00, ++ MALIGP2_REG_ADDR_MGMT_VSCL_END_ADDR = 0x04, ++ MALIGP2_REG_ADDR_MGMT_PLBUCL_START_ADDR = 0x08, ++ MALIGP2_REG_ADDR_MGMT_PLBUCL_END_ADDR = 0x0c, ++ MALIGP2_REG_ADDR_MGMT_PLBU_ALLOC_START_ADDR = 0x10, ++ MALIGP2_REG_ADDR_MGMT_PLBU_ALLOC_END_ADDR = 0x14, ++ MALIGP2_REG_ADDR_MGMT_CMD = 0x20, ++ MALIGP2_REG_ADDR_MGMT_INT_RAWSTAT = 0x24, ++ MALIGP2_REG_ADDR_MGMT_INT_CLEAR = 0x28, ++ MALIGP2_REG_ADDR_MGMT_INT_MASK = 0x2C, ++ MALIGP2_REG_ADDR_MGMT_INT_STAT = 0x30, ++ MALIGP2_REG_ADDR_MGMT_PERF_CNT_0_ENABLE = 0x3C, ++ MALIGP2_REG_ADDR_MGMT_PERF_CNT_1_ENABLE = 0x40, ++ MALIGP2_REG_ADDR_MGMT_PERF_CNT_0_SRC = 0x44, ++ MALIGP2_REG_ADDR_MGMT_PERF_CNT_1_SRC = 0x48, ++ MALIGP2_REG_ADDR_MGMT_PERF_CNT_0_VALUE = 0x4C, ++ MALIGP2_REG_ADDR_MGMT_PERF_CNT_1_VALUE = 0x50, ++ MALIGP2_REG_ADDR_MGMT_PERF_CNT_0_LIMIT = 0x54, ++ MALIGP2_REG_ADDR_MGMT_STATUS = 0x68, ++ MALIGP2_REG_ADDR_MGMT_VERSION = 0x6C, ++ MALIGP2_REG_ADDR_MGMT_VSCL_START_ADDR_READ = 0x80, ++ MALIGP2_REG_ADDR_MGMT_PLBCL_START_ADDR_READ = 0x84, ++ MALIGP2_CONTR_AXI_BUS_ERROR_STAT = 0x94, ++ MALIGP2_REGISTER_ADDRESS_SPACE_SIZE = 0x98, ++} maligp_reg_addr_mgmt_addr; ++ ++#define MALIGP2_REG_VAL_PERF_CNT_ENABLE 1 ++ ++/** ++ * Commands to geometry processor. ++ * @see MALIGP2_CTRL_REG_CMD ++ */ ++typedef enum { ++ MALIGP2_REG_VAL_CMD_START_VS = (1 << 0), ++ MALIGP2_REG_VAL_CMD_START_PLBU = (1 << 1), ++ MALIGP2_REG_VAL_CMD_UPDATE_PLBU_ALLOC = (1 << 4), ++ MALIGP2_REG_VAL_CMD_RESET = (1 << 5), ++ MALIGP2_REG_VAL_CMD_FORCE_HANG = (1 << 6), ++ MALIGP2_REG_VAL_CMD_STOP_BUS = (1 << 9), ++ MALI400GP_REG_VAL_CMD_SOFT_RESET = (1 << 10), /* only valid for Mali-300 and later */ ++} mgp_contr_reg_val_cmd; ++ ++ ++/** @defgroup MALIGP2_IRQ ++ * Interrupt status of geometry processor. ++ * @see MALIGP2_CTRL_REG_INT_RAWSTAT, MALIGP2_REG_ADDR_MGMT_INT_CLEAR, ++ * MALIGP2_REG_ADDR_MGMT_INT_MASK, MALIGP2_REG_ADDR_MGMT_INT_STAT ++ * @{ ++ */ ++#define MALIGP2_REG_VAL_IRQ_VS_END_CMD_LST (1 << 0) ++#define MALIGP2_REG_VAL_IRQ_PLBU_END_CMD_LST (1 << 1) ++#define MALIGP2_REG_VAL_IRQ_PLBU_OUT_OF_MEM (1 << 2) ++#define MALIGP2_REG_VAL_IRQ_VS_SEM_IRQ (1 << 3) ++#define MALIGP2_REG_VAL_IRQ_PLBU_SEM_IRQ (1 << 4) ++#define MALIGP2_REG_VAL_IRQ_HANG (1 << 5) ++#define MALIGP2_REG_VAL_IRQ_FORCE_HANG (1 << 6) ++#define MALIGP2_REG_VAL_IRQ_PERF_CNT_0_LIMIT (1 << 7) ++#define MALIGP2_REG_VAL_IRQ_PERF_CNT_1_LIMIT (1 << 8) ++#define MALIGP2_REG_VAL_IRQ_WRITE_BOUND_ERR (1 << 9) ++#define MALIGP2_REG_VAL_IRQ_SYNC_ERROR (1 << 10) ++#define MALIGP2_REG_VAL_IRQ_AXI_BUS_ERROR (1 << 11) ++#define MALI400GP_REG_VAL_IRQ_AXI_BUS_STOPPED (1 << 12) ++#define MALI400GP_REG_VAL_IRQ_VS_INVALID_CMD (1 << 13) ++#define MALI400GP_REG_VAL_IRQ_PLB_INVALID_CMD (1 << 14) ++#define MALI400GP_REG_VAL_IRQ_RESET_COMPLETED (1 << 19) ++#define MALI400GP_REG_VAL_IRQ_SEMAPHORE_UNDERFLOW (1 << 20) ++#define MALI400GP_REG_VAL_IRQ_SEMAPHORE_OVERFLOW (1 << 21) ++#define MALI400GP_REG_VAL_IRQ_PTR_ARRAY_OUT_OF_BOUNDS (1 << 22) ++ ++/* Mask defining all IRQs in Mali GP */ ++#define MALIGP2_REG_VAL_IRQ_MASK_ALL \ ++ (\ ++ MALIGP2_REG_VAL_IRQ_VS_END_CMD_LST | \ ++ MALIGP2_REG_VAL_IRQ_PLBU_END_CMD_LST | \ ++ MALIGP2_REG_VAL_IRQ_PLBU_OUT_OF_MEM | \ ++ MALIGP2_REG_VAL_IRQ_VS_SEM_IRQ | \ ++ MALIGP2_REG_VAL_IRQ_PLBU_SEM_IRQ | \ ++ MALIGP2_REG_VAL_IRQ_HANG | \ ++ MALIGP2_REG_VAL_IRQ_FORCE_HANG | \ ++ MALIGP2_REG_VAL_IRQ_PERF_CNT_0_LIMIT | \ ++ MALIGP2_REG_VAL_IRQ_PERF_CNT_1_LIMIT | \ ++ MALIGP2_REG_VAL_IRQ_WRITE_BOUND_ERR | \ ++ MALIGP2_REG_VAL_IRQ_SYNC_ERROR | \ ++ MALIGP2_REG_VAL_IRQ_AXI_BUS_ERROR | \ ++ MALI400GP_REG_VAL_IRQ_AXI_BUS_STOPPED | \ ++ MALI400GP_REG_VAL_IRQ_VS_INVALID_CMD | \ ++ MALI400GP_REG_VAL_IRQ_PLB_INVALID_CMD | \ ++ MALI400GP_REG_VAL_IRQ_RESET_COMPLETED | \ ++ MALI400GP_REG_VAL_IRQ_SEMAPHORE_UNDERFLOW | \ ++ MALI400GP_REG_VAL_IRQ_SEMAPHORE_OVERFLOW | \ ++ MALI400GP_REG_VAL_IRQ_PTR_ARRAY_OUT_OF_BOUNDS) ++ ++/* Mask defining the IRQs in Mali GP which we use */ ++#define MALIGP2_REG_VAL_IRQ_MASK_USED \ ++ (\ ++ MALIGP2_REG_VAL_IRQ_VS_END_CMD_LST | \ ++ MALIGP2_REG_VAL_IRQ_PLBU_END_CMD_LST | \ ++ MALIGP2_REG_VAL_IRQ_PLBU_OUT_OF_MEM | \ ++ MALIGP2_REG_VAL_IRQ_FORCE_HANG | \ ++ MALIGP2_REG_VAL_IRQ_WRITE_BOUND_ERR | \ ++ MALIGP2_REG_VAL_IRQ_SYNC_ERROR | \ ++ MALIGP2_REG_VAL_IRQ_AXI_BUS_ERROR | \ ++ MALI400GP_REG_VAL_IRQ_VS_INVALID_CMD | \ ++ MALI400GP_REG_VAL_IRQ_PLB_INVALID_CMD | \ ++ MALI400GP_REG_VAL_IRQ_SEMAPHORE_UNDERFLOW | \ ++ MALI400GP_REG_VAL_IRQ_SEMAPHORE_OVERFLOW | \ ++ MALI400GP_REG_VAL_IRQ_PTR_ARRAY_OUT_OF_BOUNDS) ++ ++/* Mask defining non IRQs on MaliGP2*/ ++#define MALIGP2_REG_VAL_IRQ_MASK_NONE 0 ++ ++/** }@ defgroup MALIGP2_IRQ*/ ++ ++/** @defgroup MALIGP2_STATUS ++ * The different Status values to the geometry processor. ++ * @see MALIGP2_CTRL_REG_STATUS ++ * @{ ++ */ ++#define MALIGP2_REG_VAL_STATUS_VS_ACTIVE 0x0002 ++#define MALIGP2_REG_VAL_STATUS_BUS_STOPPED 0x0004 ++#define MALIGP2_REG_VAL_STATUS_PLBU_ACTIVE 0x0008 ++#define MALIGP2_REG_VAL_STATUS_BUS_ERROR 0x0040 ++#define MALIGP2_REG_VAL_STATUS_WRITE_BOUND_ERR 0x0100 ++/** }@ defgroup MALIGP2_STATUS*/ ++ ++#define MALIGP2_REG_VAL_STATUS_MASK_ACTIVE (\ ++ MALIGP2_REG_VAL_STATUS_VS_ACTIVE|\ ++ MALIGP2_REG_VAL_STATUS_PLBU_ACTIVE) ++ ++ ++#define MALIGP2_REG_VAL_STATUS_MASK_ERROR (\ ++ MALIGP2_REG_VAL_STATUS_BUS_ERROR |\ ++ MALIGP2_REG_VAL_STATUS_WRITE_BOUND_ERR ) ++ ++/* This should be in the top 16 bit of the version register of gp.*/ ++#define MALI200_GP_PRODUCT_ID 0xA07 ++#define MALI300_GP_PRODUCT_ID 0xC07 ++#define MALI400_GP_PRODUCT_ID 0xB07 ++#define MALI450_GP_PRODUCT_ID 0xD07 ++ ++/** ++ * The different sources for instrumented on the geometry processor. ++ * @see MALIGP2_REG_ADDR_MGMT_PERF_CNT_0_SRC ++ */ ++ ++enum MALIGP2_cont_reg_perf_cnt_src { ++ MALIGP2_REG_VAL_PERF_CNT1_SRC_NUMBER_OF_VERTICES_PROCESSED = 0x0a, ++}; ++ ++#endif +diff --git a/drivers/gpu/arm/mali400/mali/timestamp-arm11-cc/mali_timestamp.c b/drivers/gpu/arm/mali400/mali/timestamp-arm11-cc/mali_timestamp.c +new file mode 100755 +index 000000000000..7df934c12122 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/timestamp-arm11-cc/mali_timestamp.c +@@ -0,0 +1,13 @@ ++/* ++ * Copyright (C) 2010-2011, 2013, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include "mali_timestamp.h" ++ ++/* This file is intentionally left empty, as all functions are inlined in mali_profiling_sampler.h */ +diff --git a/drivers/gpu/arm/mali400/mali/timestamp-arm11-cc/mali_timestamp.h b/drivers/gpu/arm/mali400/mali/timestamp-arm11-cc/mali_timestamp.h +new file mode 100755 +index 000000000000..f52097c1901b +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/timestamp-arm11-cc/mali_timestamp.h +@@ -0,0 +1,48 @@ ++/* ++ * Copyright (C) 2010-2011, 2013-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_TIMESTAMP_H__ ++#define __MALI_TIMESTAMP_H__ ++ ++#include "mali_osk.h" ++ ++MALI_STATIC_INLINE _mali_osk_errcode_t _mali_timestamp_reset(void) ++{ ++ /* ++ * reset counters and overflow flags ++ */ ++ ++ u32 mask = (1 << 0) | /* enable all three counters */ ++ (0 << 1) | /* reset both Count Registers to 0x0 */ ++ (1 << 2) | /* reset the Cycle Counter Register to 0x0 */ ++ (0 << 3) | /* 1 = Cycle Counter Register counts every 64th processor clock cycle */ ++ (0 << 4) | /* Count Register 0 interrupt enable */ ++ (0 << 5) | /* Count Register 1 interrupt enable */ ++ (0 << 6) | /* Cycle Counter interrupt enable */ ++ (0 << 8) | /* Count Register 0 overflow flag (clear or write, flag on read) */ ++ (0 << 9) | /* Count Register 1 overflow flag (clear or write, flag on read) */ ++ (1 << 10); /* Cycle Counter Register overflow flag (clear or write, flag on read) */ ++ ++ __asm__ __volatile__("MCR p15, 0, %0, c15, c12, 0" : : "r"(mask)); ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++MALI_STATIC_INLINE u64 _mali_timestamp_get(void) ++{ ++ u32 result; ++ ++ /* this is for the clock cycles */ ++ __asm__ __volatile__("MRC p15, 0, %0, c15, c12, 1" : "=r"(result)); ++ ++ return (u64)result; ++} ++ ++#endif /* __MALI_TIMESTAMP_H__ */ +diff --git a/drivers/gpu/arm/mali400/mali/timestamp-default/mali_timestamp.c b/drivers/gpu/arm/mali400/mali/timestamp-default/mali_timestamp.c +new file mode 100755 +index 000000000000..7df934c12122 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/timestamp-default/mali_timestamp.c +@@ -0,0 +1,13 @@ ++/* ++ * Copyright (C) 2010-2011, 2013, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include "mali_timestamp.h" ++ ++/* This file is intentionally left empty, as all functions are inlined in mali_profiling_sampler.h */ +diff --git a/drivers/gpu/arm/mali400/mali/timestamp-default/mali_timestamp.h b/drivers/gpu/arm/mali400/mali/timestamp-default/mali_timestamp.h +new file mode 100755 +index 000000000000..709a16a82f31 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/mali/timestamp-default/mali_timestamp.h +@@ -0,0 +1,26 @@ ++/* ++ * Copyright (C) 2010-2011, 2013-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __MALI_TIMESTAMP_H__ ++#define __MALI_TIMESTAMP_H__ ++ ++#include "mali_osk.h" ++ ++MALI_STATIC_INLINE _mali_osk_errcode_t _mali_timestamp_reset(void) ++{ ++ return _MALI_OSK_ERR_OK; ++} ++ ++MALI_STATIC_INLINE u64 _mali_timestamp_get(void) ++{ ++ return _mali_osk_boot_time_get_ns(); ++} ++ ++#endif /* __MALI_TIMESTAMP_H__ */ +diff --git a/drivers/gpu/arm/mali400/rk_ver_info.txt b/drivers/gpu/arm/mali400/rk_ver_info.txt +new file mode 100755 +index 000000000000..2a6cbbbb5a97 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/rk_ver_info.txt +@@ -0,0 +1,11 @@ ++ ++r5p0-01rel0-1-x@0 ++ 对 arm_release_ver r5p0-01rel0 的定制集æˆ. ++ r5p0-01rel0 对 gpu çš„ dts 有大修改, 但这里出于兼容考虑, ä»æ—§ä½¿ç”¨ dts_for_mali_ko_befor_r5p0-01rel0. ++ ++r5p0-01rel0-2-x@0 ++ æ”¯æŒ mali_so æ¥èŽ·å– rk_ko_ver. ++ ++r5p0-01rel0-3-x@0 ++ 在 mali_control_timer_callback_chain 中使用 mod_timer, 而ä¸å†æ˜¯ add_timer. ++ +diff --git a/drivers/gpu/arm/mali400/ump/Kbuild b/drivers/gpu/arm/mali400/ump/Kbuild +new file mode 100755 +index 000000000000..a3067ba72459 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/ump/Kbuild +@@ -0,0 +1,92 @@ ++# ++# Copyright (C) 2010-2012 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++# as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained from Free Software ++# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++# ++ ++# Set default configuration to use, if Makefile didn't provide one. ++# Change this to use a different config.h ++CONFIG ?= default ++ ++# Link arch to the selected arch-config directory ++$(shell [ -L $(src)/arch ] && rm $(src)/arch) ++$(shell ln -sf arch-$(CONFIG) $(src)/arch) ++$(shell touch $(src)/arch/config.h) ++ ++UDD_FILE_PREFIX = ../mali/ ++ ++# Get subversion revision number, fall back to 0000 if no svn info is available ++SVN_INFO = (cd $(src); svn info 2>/dev/null) ++ ++ifneq ($(shell $(SVN_INFO) 2>/dev/null),) ++# SVN detected ++SVN_REV := $(shell $(SVN_INFO) | grep '^Revision: '| sed -e 's/^Revision: //' 2>/dev/null) ++DRIVER_REV := $(MALI_RELEASE_NAME)-r$(SVN_REV) ++CHANGE_DATE := $(shell $(SVN_INFO) | grep '^Last Changed Date: ' | cut -d: -f2- | cut -b2-) ++CHANGED_REVISION := $(shell $(SVN_INFO) | grep '^Last Changed Rev: ' | cut -d: -f2- | cut -b2-) ++REPO_URL := $(shell $(SVN_INFO) | grep '^URL: ' | cut -d: -f2- | cut -b2-) ++ ++else # SVN ++GIT_REV := $(shell cd $(src); git describe --always 2>/dev/null) ++ifneq ($(GIT_REV),) ++# Git detected ++DRIVER_REV := $(MALI_RELEASE_NAME)-$(GIT_REV) ++CHANGE_DATE := $(shell cd $(src); git log -1 --format="%ci") ++CHANGED_REVISION := $(GIT_REV) ++REPO_URL := $(shell cd $(src); git describe --all --always 2>/dev/null) ++ ++else # Git ++# No Git or SVN detected ++DRIVER_REV := $(MALI_RELEASE_NAME) ++CHANGE_DATE := $(MALI_RELEASE_NAME) ++CHANGED_REVISION := $(MALI_RELEASE_NAME) ++endif ++endif ++ ++ccflags-y += -DSVN_REV=$(SVN_REV) ++ccflags-y += -DSVN_REV_STRING=\"$(DRIVER_REV)\" ++ ++ccflags-y += -I$(src) -I$(src)/common -I$(src)/linux -I$(src)/../mali/common -I$(src)/../mali/linux -I$(src)/include -I$(src)/../../ump/include/ump ++ccflags-y += -DMALI_STATE_TRACKING=0 ++ccflags-y += -DMALI_ENABLE_CPU_CYCLES=0 ++ccflags-$(CONFIG_UMP_DEBUG) += -DDEBUG ++ ++# For customer releases the Linux Device Drivers will be provided as ARM proprietary and GPL releases: ++# The ARM proprietary product will only include the license/proprietary directory ++# The GPL product will only include the license/gpl directory ++ ++ifeq ($(wildcard $(src)/linux/license/gpl/*),) ++ccflags-y += -I$(src)/linux/license/proprietary -I$(src)/../mali/linux/license/proprietary ++else ++ccflags-y += -I$(src)/linux/license/gpl -I$(src)/../mali/linux/license/gpl ++endif ++ ++ump-y = common/ump_kernel_common.o \ ++ common/ump_kernel_descriptor_mapping.o \ ++ common/ump_kernel_api.o \ ++ common/ump_kernel_ref_drv.o \ ++ linux/ump_kernel_linux.o \ ++ linux/ump_kernel_memory_backend_os.o \ ++ linux/ump_kernel_memory_backend_dedicated.o \ ++ linux/ump_memory_backend.o \ ++ linux/ump_ukk_wrappers.o \ ++ linux/ump_ukk_ref_wrappers.o \ ++ linux/ump_osk_atomics.o \ ++ linux/ump_osk_low_level_mem.o \ ++ linux/ump_osk_misc.o \ ++ linux/ump_kernel_random_mapping.o ++ ++ifneq ($(CONFIG_MALI400),y) ++ump-y += $(UDD_FILE_PREFIX)linux/mali_osk_atomics.o \ ++ $(UDD_FILE_PREFIX)linux/mali_osk_locks.o \ ++ $(UDD_FILE_PREFIX)linux/mali_osk_memory.o \ ++ $(UDD_FILE_PREFIX)linux/mali_osk_math.o \ ++ $(UDD_FILE_PREFIX)linux/mali_osk_misc.o ++endif ++ ++obj-$(CONFIG_UMP) := ump.o ++ +diff --git a/drivers/gpu/arm/mali400/ump/Kconfig b/drivers/gpu/arm/mali400/ump/Kconfig +new file mode 100755 +index 000000000000..ec3509057732 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/ump/Kconfig +@@ -0,0 +1,17 @@ ++# SPDX-License-Identifier: GPL-2.0 ++config UMP ++ tristate "UMP support" ++ depends on ARM ++ help ++ This enables support for the UMP memory allocation and sharing API. ++ ++ To compile this driver as a module, choose M here: the module will be ++ called ump. ++ ++config UMP_DEBUG ++ bool "Enable extra debug in UMP" ++ depends on UMP ++ default y ++ help ++ This enabled extra debug checks and messages in UMP. ++ +diff --git a/drivers/gpu/arm/mali400/ump/Makefile b/drivers/gpu/arm/mali400/ump/Makefile +new file mode 100755 +index 000000000000..88b02a22fce5 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/ump/Makefile +@@ -0,0 +1,67 @@ ++# ++# Copyright (C) 2010-2012, 2014, 2016-2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++# as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained from Free Software ++# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++# ++ ++# For each arch check: CROSS_COMPILE , KDIR , CFLAGS += -DARCH ++ ++export ARCH ?= arm ++BUILD ?= debug ++ ++check_cc2 = \ ++ $(shell if $(1) -S -o /dev/null -xc /dev/null > /dev/null 2>&1; \ ++ then \ ++ echo "$(2)"; \ ++ else \ ++ echo "$(3)"; \ ++ fi ;) ++ ++# Check that required parameters are supplied. ++ifeq ($(CONFIG),) ++CONFIG := default ++endif ++ifeq ($(CPU)$(KDIR),) ++$(error "KDIR or CPU must be specified.") ++endif ++ ++# Get any user defined KDIR- or maybe even a hardcoded KDIR ++-include KDIR_CONFIGURATION ++ ++# Define host system directory ++KDIR-$(shell uname -m):=/lib/modules/$(shell uname -r)/build ++ ++ifeq ($(ARCH), arm) ++# when compiling for ARM we're cross compiling ++export CROSS_COMPILE ?= $(call check_cc2, arm-linux-gnueabi-gcc, arm-linux-gnueabi-, arm-none-linux-gnueabi-) ++endif ++ ++# look up KDIR based om CPU selection ++KDIR ?= $(KDIR-$(CPU)) ++ ++export CONFIG ++ ++export CONFIG_UMP := m ++ifeq ($(BUILD),debug) ++export CONFIG_UMP_DEBUG := y ++else ++export CONFIG_UMP_DEBUG := n ++endif ++ ++ifeq ($(KDIR),) ++$(error No KDIR found for platform $(CPU)) ++endif ++ ++all: ++ $(MAKE) -C $(KDIR) M=$(CURDIR) modules ++ ++kernelrelease: ++ $(MAKE) -C $(KDIR) kernelrelease ++ ++clean: ++ $(MAKE) -C $(KDIR) M=$(CURDIR) clean ++ $(MAKE) -C $(KDIR) M=$(CURDIR)/../mali clean +diff --git a/drivers/gpu/arm/mali400/ump/Makefile.common b/drivers/gpu/arm/mali400/ump/Makefile.common +new file mode 100755 +index 000000000000..ad2c18da98a0 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/ump/Makefile.common +@@ -0,0 +1,20 @@ ++# ++# Copyright (C) 2010-2011, 2013, 2016-2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++# as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained from Free Software ++# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++# ++ ++SRC = $(UMP_FILE_PREFIX)common/ump_kernel_common.c \ ++ $(UMP_FILE_PREFIX)common/ump_kernel_descriptor_mapping.c \ ++ $(UMP_FILE_PREFIX)common/ump_kernel_api.c \ ++ $(UMP_FILE_PREFIX)common/ump_kernel_ref_drv.c ++ ++# Get subversion revision number, fall back to 0000 if no svn info is available ++SVN_REV:=$(shell ((svnversion | grep -qv exported && echo -n 'Revision: ' && svnversion) || git svn info | sed -e 's/$$$$/M/' | grep '^Revision: ' || echo ${MALI_RELEASE_NAME}) 2>/dev/null | sed -e 's/^Revision: //') ++ ++EXTRA_CFLAGS += -DSVN_REV=$(SVN_REV) ++EXTRA_CFLAGS += -DSVN_REV_STRING=\"$(SVN_REV)\" +diff --git a/drivers/gpu/arm/mali400/ump/arch-default/config.h b/drivers/gpu/arm/mali400/ump/arch-default/config.h +new file mode 100755 +index 000000000000..d4aef9dd09c7 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/ump/arch-default/config.h +@@ -0,0 +1,24 @@ ++/* ++ * Copyright (C) 2010, 2012, 2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __ARCH_CONFIG_H__ ++#define __ARCH_CONFIG_H__ ++ ++/* Use OS memory. */ ++#define ARCH_UMP_BACKEND_DEFAULT 1 ++ ++/* OS memory won't need a base address. */ ++#define ARCH_UMP_MEMORY_ADDRESS_DEFAULT 0x00000000 ++ ++/* 512 MB maximum limit for UMP allocations. */ ++#define ARCH_UMP_MEMORY_SIZE_DEFAULT 512UL * 1024UL * 1024UL ++ ++ ++#endif /* __ARCH_CONFIG_H__ */ +diff --git a/drivers/gpu/arm/mali400/ump/arch-pb-virtex5/config.h b/drivers/gpu/arm/mali400/ump/arch-pb-virtex5/config.h +new file mode 100755 +index 000000000000..182e90c1d64f +--- /dev/null ++++ b/drivers/gpu/arm/mali400/ump/arch-pb-virtex5/config.h +@@ -0,0 +1,18 @@ ++/* ++ * Copyright (C) 2010-2013, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __ARCH_CONFIG_H__ ++#define __ARCH_CONFIG_H__ ++ ++#define ARCH_UMP_BACKEND_DEFAULT 0 ++#define ARCH_UMP_MEMORY_ADDRESS_DEFAULT 0xE1000000 ++#define ARCH_UMP_MEMORY_SIZE_DEFAULT 16UL * 1024UL * 1024UL ++ ++#endif /* __ARCH_CONFIG_H__ */ +diff --git a/drivers/gpu/arm/mali400/ump/arch/config.h b/drivers/gpu/arm/mali400/ump/arch/config.h +new file mode 100755 +index 000000000000..d4aef9dd09c7 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/ump/arch/config.h +@@ -0,0 +1,24 @@ ++/* ++ * Copyright (C) 2010, 2012, 2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __ARCH_CONFIG_H__ ++#define __ARCH_CONFIG_H__ ++ ++/* Use OS memory. */ ++#define ARCH_UMP_BACKEND_DEFAULT 1 ++ ++/* OS memory won't need a base address. */ ++#define ARCH_UMP_MEMORY_ADDRESS_DEFAULT 0x00000000 ++ ++/* 512 MB maximum limit for UMP allocations. */ ++#define ARCH_UMP_MEMORY_SIZE_DEFAULT 512UL * 1024UL * 1024UL ++ ++ ++#endif /* __ARCH_CONFIG_H__ */ +diff --git a/drivers/gpu/arm/mali400/ump/common/ump_kernel_api.c b/drivers/gpu/arm/mali400/ump/common/ump_kernel_api.c +new file mode 100755 +index 000000000000..36adb2f5383e +--- /dev/null ++++ b/drivers/gpu/arm/mali400/ump/common/ump_kernel_api.c +@@ -0,0 +1,455 @@ ++/* ++ * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include "mali_osk.h" ++#include "mali_osk_list.h" ++#include "ump_osk.h" ++#include "ump_uk_types.h" ++#include "ump_kernel_interface.h" ++#include "ump_kernel_common.h" ++#include "ump_kernel_random_mapping.h" ++ ++ ++ ++/* ---------------- UMP kernel space API functions follows ---------------- */ ++ ++ ++ ++UMP_KERNEL_API_EXPORT ump_secure_id ump_dd_secure_id_get(ump_dd_handle memh) ++{ ++ ump_dd_mem *mem = (ump_dd_mem *)memh; ++ ++ DEBUG_ASSERT_POINTER(mem); ++ ++ DBG_MSG(5, ("Returning secure ID. ID: %u\n", mem->secure_id)); ++ ++ return mem->secure_id; ++} ++ ++ ++ ++UMP_KERNEL_API_EXPORT ump_dd_handle ump_dd_handle_create_from_secure_id(ump_secure_id secure_id) ++{ ++ ump_dd_mem *mem; ++ ++ DBG_MSG(5, ("Getting handle from secure ID. ID: %u\n", secure_id)); ++ mem = ump_random_mapping_get(device.secure_id_map, (int)secure_id); ++ if (NULL == mem) { ++ DBG_MSG(1, ("Secure ID not found. ID: %u\n", secure_id)); ++ return UMP_DD_HANDLE_INVALID; ++ } ++ ++ /* Keep the reference taken in ump_random_mapping_get() */ ++ ++ return (ump_dd_handle)mem; ++} ++ ++ ++ ++UMP_KERNEL_API_EXPORT unsigned long ump_dd_phys_block_count_get(ump_dd_handle memh) ++{ ++ ump_dd_mem *mem = (ump_dd_mem *) memh; ++ ++ DEBUG_ASSERT_POINTER(mem); ++ ++ return mem->nr_blocks; ++} ++ ++ ++ ++UMP_KERNEL_API_EXPORT ump_dd_status_code ump_dd_phys_blocks_get(ump_dd_handle memh, ump_dd_physical_block *blocks, unsigned long num_blocks) ++{ ++ ump_dd_mem *mem = (ump_dd_mem *)memh; ++ ++ DEBUG_ASSERT_POINTER(mem); ++ ++ if (blocks == NULL) { ++ DBG_MSG(1, ("NULL parameter in ump_dd_phys_blocks_get()\n")); ++ return UMP_DD_INVALID; ++ } ++ ++ if (mem->nr_blocks != num_blocks) { ++ DBG_MSG(1, ("Specified number of blocks do not match actual number of blocks\n")); ++ return UMP_DD_INVALID; ++ } ++ ++ DBG_MSG(5, ("Returning physical block information. ID: %u\n", mem->secure_id)); ++ ++ _mali_osk_memcpy(blocks, mem->block_array, sizeof(ump_dd_physical_block) * mem->nr_blocks); ++ ++ return UMP_DD_SUCCESS; ++} ++ ++ ++ ++UMP_KERNEL_API_EXPORT ump_dd_status_code ump_dd_phys_block_get(ump_dd_handle memh, unsigned long index, ump_dd_physical_block *block) ++{ ++ ump_dd_mem *mem = (ump_dd_mem *)memh; ++ ++ DEBUG_ASSERT_POINTER(mem); ++ ++ if (block == NULL) { ++ DBG_MSG(1, ("NULL parameter in ump_dd_phys_block_get()\n")); ++ return UMP_DD_INVALID; ++ } ++ ++ if (index >= mem->nr_blocks) { ++ DBG_MSG(5, ("Invalid index specified in ump_dd_phys_block_get()\n")); ++ return UMP_DD_INVALID; ++ } ++ ++ DBG_MSG(5, ("Returning physical block information. ID: %u, index: %lu\n", mem->secure_id, index)); ++ ++ *block = mem->block_array[index]; ++ ++ return UMP_DD_SUCCESS; ++} ++ ++ ++ ++UMP_KERNEL_API_EXPORT unsigned long ump_dd_size_get(ump_dd_handle memh) ++{ ++ ump_dd_mem *mem = (ump_dd_mem *)memh; ++ ++ DEBUG_ASSERT_POINTER(mem); ++ ++ DBG_MSG(5, ("Returning size. ID: %u, size: %lu\n", mem->secure_id, mem->size_bytes)); ++ ++ return mem->size_bytes; ++} ++ ++ ++ ++UMP_KERNEL_API_EXPORT void ump_dd_reference_add(ump_dd_handle memh) ++{ ++ ump_dd_mem *mem = (ump_dd_mem *)memh; ++ int new_ref; ++ ++ DEBUG_ASSERT_POINTER(mem); ++ ++ new_ref = _ump_osk_atomic_inc_and_read(&mem->ref_count); ++ ++ DBG_MSG(5, ("Memory reference incremented. ID: %u, new value: %d\n", mem->secure_id, new_ref)); ++} ++ ++ ++ ++UMP_KERNEL_API_EXPORT void ump_dd_reference_release(ump_dd_handle memh) ++{ ++ ump_dd_mem *mem = (ump_dd_mem *)memh; ++ ++ DEBUG_ASSERT_POINTER(mem); ++ ++ ump_random_mapping_put(mem); ++} ++ ++ ++ ++/* --------------- Handling of user space requests follows --------------- */ ++ ++ ++_mali_osk_errcode_t _ump_uku_get_api_version(_ump_uk_api_version_s *args) ++{ ++ ump_session_data *session_data; ++ ++ DEBUG_ASSERT_POINTER(args); ++ DEBUG_ASSERT_POINTER(args->ctx); ++ ++ session_data = (ump_session_data *)args->ctx; ++ ++ /* check compatability */ ++ if (args->version == UMP_IOCTL_API_VERSION) { ++ DBG_MSG(3, ("API version set to newest %d (compatible)\n", ++ GET_VERSION(args->version))); ++ args->compatible = 1; ++ session_data->api_version = args->version; ++ } else { ++ DBG_MSG(2, ("API version set to %d (incompatible with client version %d)\n", ++ GET_VERSION(UMP_IOCTL_API_VERSION), GET_VERSION(args->version))); ++ args->compatible = 0; ++ args->version = UMP_IOCTL_API_VERSION; /* report our version */ ++ } ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++ ++_mali_osk_errcode_t _ump_ukk_release(_ump_uk_release_s *release_info) ++{ ++ ump_session_memory_list_element *session_memory_element; ++ ump_session_memory_list_element *tmp; ++ ump_session_data *session_data; ++ _mali_osk_errcode_t ret = _MALI_OSK_ERR_INVALID_FUNC; ++ int secure_id; ++ ++ DEBUG_ASSERT_POINTER(release_info); ++ DEBUG_ASSERT_POINTER(release_info->ctx); ++ ++ /* Retreive the session data */ ++ session_data = (ump_session_data *)release_info->ctx; ++ ++ /* If there are many items in the memory session list we ++ * could be de-referencing this pointer a lot so keep a local copy ++ */ ++ secure_id = release_info->secure_id; ++ ++ DBG_MSG(4, ("Releasing memory with IOCTL, ID: %u\n", secure_id)); ++ ++ /* Iterate through the memory list looking for the requested secure ID */ ++ _mali_osk_mutex_wait(session_data->lock); ++ _MALI_OSK_LIST_FOREACHENTRY(session_memory_element, tmp, &session_data->list_head_session_memory_list, ump_session_memory_list_element, list) { ++ if (session_memory_element->mem->secure_id == secure_id) { ++ ump_dd_mem *release_mem; ++ ++ release_mem = session_memory_element->mem; ++ _mali_osk_list_del(&session_memory_element->list); ++ ump_dd_reference_release(release_mem); ++ _mali_osk_free(session_memory_element); ++ ++ ret = _MALI_OSK_ERR_OK; ++ break; ++ } ++ } ++ ++ _mali_osk_mutex_signal(session_data->lock); ++ DBG_MSG_IF(1, _MALI_OSK_ERR_OK != ret, ("UMP memory with ID %u does not belong to this session.\n", secure_id)); ++ ++ DBG_MSG(4, ("_ump_ukk_release() returning 0x%x\n", ret)); ++ return ret; ++} ++ ++_mali_osk_errcode_t _ump_ukk_size_get(_ump_uk_size_get_s *user_interaction) ++{ ++ ump_dd_mem *mem; ++ _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT; ++ ++ DEBUG_ASSERT_POINTER(user_interaction); ++ ++ /* We lock the mappings so things don't get removed while we are looking for the memory */ ++ mem = ump_random_mapping_get(device.secure_id_map, user_interaction->secure_id); ++ if (NULL != mem) { ++ user_interaction->size = mem->size_bytes; ++ DBG_MSG(4, ("Returning size. ID: %u, size: %lu ", ++ (ump_secure_id)user_interaction->secure_id, ++ (unsigned long)user_interaction->size)); ++ ump_random_mapping_put(mem); ++ ret = _MALI_OSK_ERR_OK; ++ } else { ++ user_interaction->size = 0; ++ DBG_MSG(1, ("Failed to look up mapping in ump_ioctl_size_get(). ID: %u\n", ++ (ump_secure_id)user_interaction->secure_id)); ++ } ++ ++ return ret; ++} ++ ++ ++ ++void _ump_ukk_msync(_ump_uk_msync_s *args) ++{ ++ ump_dd_mem *mem = NULL; ++ void *virtual = NULL; ++ u32 size = 0; ++ u32 offset = 0; ++ ++ mem = ump_random_mapping_get(device.secure_id_map, (int)args->secure_id); ++ if (NULL == mem) { ++ DBG_MSG(1, ("Failed to look up mapping in _ump_ukk_msync(). ID: %u\n", ++ (ump_secure_id)args->secure_id)); ++ return; ++ } ++ ++ /* Returns the cache settings back to Userspace */ ++ args->is_cached = mem->is_cached; ++ ++ /* If this flag is the only one set, we should not do the actual flush, only the readout */ ++ if (_UMP_UK_MSYNC_READOUT_CACHE_ENABLED == args->op) { ++ DBG_MSG(3, ("_ump_ukk_msync READOUT ID: %u Enabled: %d\n", (ump_secure_id)args->secure_id, mem->is_cached)); ++ goto msync_release_and_return; ++ } ++ ++ /* Nothing to do if the memory is not caches */ ++ if (0 == mem->is_cached) { ++ DBG_MSG(3, ("_ump_ukk_msync IGNORING ID: %u Enabled: %d OP: %d\n", (ump_secure_id)args->secure_id, mem->is_cached, args->op)); ++ goto msync_release_and_return; ++ } ++ DBG_MSG(3, ("UMP[%02u] _ump_ukk_msync Flush OP: %d Address: 0x%08x Mapping: 0x%08x\n", ++ (ump_secure_id)args->secure_id, args->op, args->address, args->mapping)); ++ ++ if (args->address) { ++ virtual = (void *)((u32)args->address); ++ offset = (u32)((args->address) - (args->mapping)); ++ } else { ++ /* Flush entire mapping when no address is specified. */ ++ virtual = args->mapping; ++ } ++ if (args->size) { ++ size = args->size; ++ } else { ++ /* Flush entire mapping when no size is specified. */ ++ size = mem->size_bytes - offset; ++ } ++ ++ if ((offset + size) > mem->size_bytes) { ++ DBG_MSG(1, ("Trying to flush more than the entire UMP allocation: offset: %u + size: %u > %u\n", offset, size, mem->size_bytes)); ++ goto msync_release_and_return; ++ } ++ ++ /* The actual cache flush - Implemented for each OS*/ ++ _ump_osk_msync(mem, virtual, offset, size, args->op, NULL); ++ ++msync_release_and_return: ++ ump_random_mapping_put(mem); ++ return; ++} ++ ++void _ump_ukk_cache_operations_control(_ump_uk_cache_operations_control_s *args) ++{ ++ ump_session_data *session_data; ++ ump_uk_cache_op_control op; ++ ++ DEBUG_ASSERT_POINTER(args); ++ DEBUG_ASSERT_POINTER(args->ctx); ++ ++ op = args->op; ++ session_data = (ump_session_data *)args->ctx; ++ ++ _mali_osk_mutex_wait(session_data->lock); ++ if (op == _UMP_UK_CACHE_OP_START) { ++ session_data->cache_operations_ongoing++; ++ DBG_MSG(4, ("Cache ops start\n")); ++ if (session_data->cache_operations_ongoing != 1) { ++ DBG_MSG(2, ("UMP: Number of simultanious cache control ops: %d\n", session_data->cache_operations_ongoing)); ++ } ++ } else if (op == _UMP_UK_CACHE_OP_FINISH) { ++ DBG_MSG(4, ("Cache ops finish\n")); ++ session_data->cache_operations_ongoing--; ++#if 0 ++ if (session_data->has_pending_level1_cache_flush) { ++ /* This function will set has_pending_level1_cache_flush=0 */ ++ _ump_osk_msync(NULL, NULL, 0, 0, _UMP_UK_MSYNC_FLUSH_L1, session_data); ++ } ++#endif ++ ++ /* to be on the safe side: always flush l1 cache when cache operations are done */ ++ _ump_osk_msync(NULL, NULL, 0, 0, _UMP_UK_MSYNC_FLUSH_L1, session_data); ++ DBG_MSG(4, ("Cache ops finish end\n")); ++ } else { ++ DBG_MSG(1, ("Illegal call to %s at line %d\n", __FUNCTION__, __LINE__)); ++ } ++ _mali_osk_mutex_signal(session_data->lock); ++ ++} ++ ++void _ump_ukk_switch_hw_usage(_ump_uk_switch_hw_usage_s *args) ++{ ++ ump_dd_mem *mem = NULL; ++ ump_uk_user old_user; ++ ump_uk_msync_op cache_op = _UMP_UK_MSYNC_CLEAN_AND_INVALIDATE; ++ ump_session_data *session_data; ++ ++ DEBUG_ASSERT_POINTER(args); ++ DEBUG_ASSERT_POINTER(args->ctx); ++ ++ session_data = (ump_session_data *)args->ctx; ++ ++ mem = ump_random_mapping_get(device.secure_id_map, (int)args->secure_id); ++ if (NULL == mem) { ++ DBG_MSG(1, ("Failed to look up mapping in _ump_ukk_switch_hw_usage(). ID: %u\n", ++ (ump_secure_id)args->secure_id)); ++ return; ++ } ++ ++ old_user = mem->hw_device; ++ mem->hw_device = args->new_user; ++ ++ DBG_MSG(3, ("UMP[%02u] Switch usage Start New: %s Prev: %s.\n", ++ (ump_secure_id)args->secure_id, ++ args->new_user ? "MALI" : "CPU", ++ old_user ? "MALI" : "CPU")); ++ ++ if (!mem->is_cached) { ++ DBG_MSG(3, ("UMP[%02u] Changing owner of uncached memory. Cache flushing not needed.\n", ++ (ump_secure_id)args->secure_id)); ++ goto out; ++ } ++ ++ if (old_user == args->new_user) { ++ DBG_MSG(4, ("UMP[%02u] Setting the new_user equal to previous for. Cache flushing not needed.\n", ++ (ump_secure_id)args->secure_id)); ++ goto out; ++ } ++ if ( ++ /* Previous AND new is both different from CPU */ ++ (old_user != _UMP_UK_USED_BY_CPU) && (args->new_user != _UMP_UK_USED_BY_CPU) ++ ) { ++ DBG_MSG(4, ("UMP[%02u] Previous and new user is not CPU. Cache flushing not needed.\n", ++ (ump_secure_id)args->secure_id)); ++ goto out; ++ } ++ ++ if ((old_user != _UMP_UK_USED_BY_CPU) && (args->new_user == _UMP_UK_USED_BY_CPU)) { ++ cache_op = _UMP_UK_MSYNC_INVALIDATE; ++ DBG_MSG(4, ("UMP[%02u] Cache invalidation needed\n", (ump_secure_id)args->secure_id)); ++#ifdef UMP_SKIP_INVALIDATION ++#error ++ DBG_MSG(4, ("UMP[%02u] Performing Cache invalidation SKIPPED\n", (ump_secure_id)args->secure_id)); ++ goto out; ++#endif ++ } ++ ++ /* Take lock to protect: session->cache_operations_ongoing and session->has_pending_level1_cache_flush */ ++ _mali_osk_mutex_wait(session_data->lock); ++ /* Actual cache flush */ ++ _ump_osk_msync(mem, NULL, 0, mem->size_bytes, cache_op, session_data); ++ _mali_osk_mutex_signal(session_data->lock); ++ ++out: ++ ump_random_mapping_put(mem); ++ DBG_MSG(4, ("UMP[%02u] Switch usage Finish\n", (ump_secure_id)args->secure_id)); ++ return; ++} ++ ++void _ump_ukk_lock(_ump_uk_lock_s *args) ++{ ++ ump_dd_mem *mem = NULL; ++ ++ mem = ump_random_mapping_get(device.secure_id_map, (int)args->secure_id); ++ if (NULL == mem) { ++ DBG_MSG(1, ("UMP[%02u] Failed to look up mapping in _ump_ukk_lock(). ID: %u\n", ++ (ump_secure_id)args->secure_id)); ++ return; ++ } ++ ++ DBG_MSG(1, ("UMP[%02u] Lock. New lock flag: %d. Old Lock flag:\n", (u32)args->secure_id, (u32)args->lock_usage, (u32) mem->lock_usage)); ++ ++ mem->lock_usage = (ump_lock_usage) args->lock_usage; ++ ++ ump_random_mapping_put(mem); ++} ++ ++void _ump_ukk_unlock(_ump_uk_unlock_s *args) ++{ ++ ump_dd_mem *mem = NULL; ++ ++ mem = ump_random_mapping_get(device.secure_id_map, (int)args->secure_id); ++ if (NULL == mem) { ++ DBG_MSG(1, ("Failed to look up mapping in _ump_ukk_unlock(). ID: %u\n", ++ (ump_secure_id)args->secure_id)); ++ return; ++ } ++ ++ DBG_MSG(1, ("UMP[%02u] Unlocking. Old Lock flag:\n", ++ (u32)args->secure_id, (u32) mem->lock_usage)); ++ ++ mem->lock_usage = (ump_lock_usage) UMP_NOT_LOCKED; ++ ++ ump_random_mapping_put(mem); ++} +diff --git a/drivers/gpu/arm/mali400/ump/common/ump_kernel_common.c b/drivers/gpu/arm/mali400/ump/common/ump_kernel_common.c +new file mode 100755 +index 000000000000..73aa9e4c49f9 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/ump/common/ump_kernel_common.c +@@ -0,0 +1,358 @@ ++/* ++ * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include "mali_kernel_common.h" ++#include "mali_osk.h" ++#include "mali_osk_bitops.h" ++#include "mali_osk_list.h" ++#include "ump_osk.h" ++#include "ump_uk_types.h" ++#include "ump_ukk.h" ++#include "ump_kernel_common.h" ++#include "ump_kernel_descriptor_mapping.h" ++#include "ump_kernel_memory_backend.h" ++ ++ ++ ++/** ++ * Define the initial and maximum size of number of secure_ids on the system ++ */ ++#define UMP_SECURE_ID_TABLE_ENTRIES_INITIAL (128 ) ++#define UMP_SECURE_ID_TABLE_ENTRIES_MAXIMUM (4096 ) ++ ++ ++/** ++ * Define the initial and maximum size of the ump_session_data::cookies_map, ++ * which is a \ref ump_descriptor_mapping. This limits how many secure_ids ++ * may be mapped into a particular process using _ump_ukk_map_mem(). ++ */ ++ ++#define UMP_COOKIES_PER_SESSION_INITIAL (UMP_SECURE_ID_TABLE_ENTRIES_INITIAL ) ++#define UMP_COOKIES_PER_SESSION_MAXIMUM (UMP_SECURE_ID_TABLE_ENTRIES_MAXIMUM) ++ ++struct ump_dev device; ++ ++_mali_osk_errcode_t ump_kernel_constructor(void) ++{ ++ _mali_osk_errcode_t err; ++ ++ /* Perform OS Specific initialization */ ++ err = _ump_osk_init(); ++ if (_MALI_OSK_ERR_OK != err) { ++ MSG_ERR(("Failed to initiaze the UMP Device Driver")); ++ return err; ++ } ++ ++ /* Init the global device */ ++ _mali_osk_memset(&device, 0, sizeof(device)); ++ ++ /* Create the descriptor map, which will be used for mapping secure ID to ump_dd_mem structs */ ++ device.secure_id_map = ump_random_mapping_create(); ++ if (NULL == device.secure_id_map) { ++ MSG_ERR(("Failed to create secure id lookup table\n")); ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ ++ /* Init memory backend */ ++ device.backend = ump_memory_backend_create(); ++ if (NULL == device.backend) { ++ MSG_ERR(("Failed to create memory backend\n")); ++ ump_random_mapping_destroy(device.secure_id_map); ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++void ump_kernel_destructor(void) ++{ ++ DEBUG_ASSERT_POINTER(device.secure_id_map); ++ ++ ump_random_mapping_destroy(device.secure_id_map); ++ device.secure_id_map = NULL; ++ ++ device.backend->shutdown(device.backend); ++ device.backend = NULL; ++ ++ ump_memory_backend_destroy(); ++ ++ _ump_osk_term(); ++} ++ ++/** Creates a new UMP session ++ */ ++_mali_osk_errcode_t _ump_ukk_open(void **context) ++{ ++ struct ump_session_data *session_data; ++ ++ /* allocated struct to track this session */ ++ session_data = (struct ump_session_data *)_mali_osk_malloc(sizeof(struct ump_session_data)); ++ if (NULL == session_data) { ++ MSG_ERR(("Failed to allocate ump_session_data in ump_file_open()\n")); ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ ++ session_data->lock = _mali_osk_mutex_init(_MALI_OSK_LOCKFLAG_UNORDERED, 0); ++ if (NULL == session_data->lock) { ++ MSG_ERR(("Failed to initialize lock for ump_session_data in ump_file_open()\n")); ++ _mali_osk_free(session_data); ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ ++ session_data->cookies_map = ump_descriptor_mapping_create( ++ UMP_COOKIES_PER_SESSION_INITIAL, ++ UMP_COOKIES_PER_SESSION_MAXIMUM); ++ ++ if (NULL == session_data->cookies_map) { ++ MSG_ERR(("Failed to create descriptor mapping for _ump_ukk_map_mem cookies\n")); ++ ++ _mali_osk_mutex_term(session_data->lock); ++ _mali_osk_free(session_data); ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ ++ _MALI_OSK_INIT_LIST_HEAD(&session_data->list_head_session_memory_list); ++ ++ _MALI_OSK_INIT_LIST_HEAD(&session_data->list_head_session_memory_mappings_list); ++ ++ /* Since initial version of the UMP interface did not use the API_VERSION ioctl we have to assume ++ that it is this version, and not the "latest" one: UMP_IOCTL_API_VERSION ++ Current and later API versions would do an additional call to this IOCTL and update this variable ++ to the correct one.*/ ++ session_data->api_version = MAKE_VERSION_ID(1); ++ ++ *context = (void *)session_data; ++ ++ session_data->cache_operations_ongoing = 0 ; ++ session_data->has_pending_level1_cache_flush = 0; ++ ++ DBG_MSG(2, ("New session opened\n")); ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++_mali_osk_errcode_t _ump_ukk_close(void **context) ++{ ++ struct ump_session_data *session_data; ++ ump_session_memory_list_element *item; ++ ump_session_memory_list_element *tmp; ++ ++ session_data = (struct ump_session_data *)*context; ++ if (NULL == session_data) { ++ MSG_ERR(("Session data is NULL in _ump_ukk_close()\n")); ++ return _MALI_OSK_ERR_INVALID_ARGS; ++ } ++ ++ /* Unmap any descriptors mapped in. */ ++ if (0 == _mali_osk_list_empty(&session_data->list_head_session_memory_mappings_list)) { ++ ump_memory_allocation *descriptor; ++ ump_memory_allocation *temp; ++ ++ DBG_MSG(1, ("Memory mappings found on session usage list during session termination\n")); ++ ++ /* use the 'safe' list iterator, since freeing removes the active block from the list we're iterating */ ++ _MALI_OSK_LIST_FOREACHENTRY(descriptor, temp, &session_data->list_head_session_memory_mappings_list, ump_memory_allocation, list) { ++ _ump_uk_unmap_mem_s unmap_args; ++ DBG_MSG(4, ("Freeing block with phys address 0x%x size 0x%x mapped in user space at 0x%x\n", ++ descriptor->phys_addr, descriptor->size, descriptor->mapping)); ++ unmap_args.ctx = (void *)session_data; ++ unmap_args.mapping = descriptor->mapping; ++ unmap_args.size = descriptor->size; ++ unmap_args._ukk_private = NULL; /* NOTE: unused */ ++ unmap_args.cookie = descriptor->cookie; ++ ++ /* NOTE: This modifies the list_head_session_memory_mappings_list */ ++ _ump_ukk_unmap_mem(&unmap_args); ++ } ++ } ++ ++ /* ASSERT that we really did free everything, because _ump_ukk_unmap_mem() ++ * can fail silently. */ ++ DEBUG_ASSERT(_mali_osk_list_empty(&session_data->list_head_session_memory_mappings_list)); ++ ++ _MALI_OSK_LIST_FOREACHENTRY(item, tmp, &session_data->list_head_session_memory_list, ump_session_memory_list_element, list) { ++ _mali_osk_list_del(&item->list); ++ DBG_MSG(2, ("Releasing UMP memory %u as part of file close\n", item->mem->secure_id)); ++ ump_dd_reference_release(item->mem); ++ _mali_osk_free(item); ++ } ++ ++ ump_descriptor_mapping_destroy(session_data->cookies_map); ++ ++ _mali_osk_mutex_term(session_data->lock); ++ _mali_osk_free(session_data); ++ ++ DBG_MSG(2, ("Session closed\n")); ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++_mali_osk_errcode_t _ump_ukk_map_mem(_ump_uk_map_mem_s *args) ++{ ++ struct ump_session_data *session_data; ++ ump_memory_allocation *descriptor; /* Describes current mapping of memory */ ++ _mali_osk_errcode_t err; ++ unsigned long offset = 0; ++ unsigned long left; ++ ump_dd_handle handle; /* The real UMP handle for this memory. Its real datatype is ump_dd_mem* */ ++ ump_dd_mem *mem; /* The real UMP memory. It is equal to the handle, but with exposed struct */ ++ u32 block; ++ int map_id; ++ ++ session_data = (ump_session_data *)args->ctx; ++ if (NULL == session_data) { ++ MSG_ERR(("Session data is NULL in _ump_ukk_map_mem()\n")); ++ return _MALI_OSK_ERR_INVALID_ARGS; ++ } ++ ++ descriptor = (ump_memory_allocation *) _mali_osk_calloc(1, sizeof(ump_memory_allocation)); ++ if (NULL == descriptor) { ++ MSG_ERR(("ump_ukk_map_mem: descriptor allocation failed\n")); ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ ++ handle = ump_dd_handle_create_from_secure_id(args->secure_id); ++ if (UMP_DD_HANDLE_INVALID == handle) { ++ _mali_osk_free(descriptor); ++ DBG_MSG(1, ("Trying to map unknown secure ID %u\n", args->secure_id)); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ mem = (ump_dd_mem *)handle; ++ DEBUG_ASSERT(mem); ++ if (mem->size_bytes != args->size) { ++ _mali_osk_free(descriptor); ++ ump_dd_reference_release(handle); ++ DBG_MSG(1, ("Trying to map too much or little. ID: %u, virtual size=%lu, UMP size: %lu\n", args->secure_id, args->size, mem->size_bytes)); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ map_id = ump_descriptor_mapping_allocate_mapping(session_data->cookies_map, (void *) descriptor); ++ ++ if (map_id < 0) { ++ _mali_osk_free(descriptor); ++ ump_dd_reference_release(handle); ++ DBG_MSG(1, ("ump_ukk_map_mem: unable to allocate a descriptor_mapping for return cookie\n")); ++ ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ ++ descriptor->size = args->size; ++ descriptor->handle = handle; ++ descriptor->phys_addr = args->phys_addr; ++ descriptor->process_mapping_info = args->_ukk_private; ++ descriptor->ump_session = session_data; ++ descriptor->cookie = (u32)map_id; ++ ++ if (mem->is_cached) { ++ descriptor->is_cached = 1; ++ DBG_MSG(3, ("Mapping UMP secure_id: %d as cached.\n", args->secure_id)); ++ } else { ++ descriptor->is_cached = 0; ++ DBG_MSG(3, ("Mapping UMP secure_id: %d as Uncached.\n", args->secure_id)); ++ } ++ ++ _mali_osk_list_init(&descriptor->list); ++ ++ err = _ump_osk_mem_mapregion_init(descriptor); ++ if (_MALI_OSK_ERR_OK != err) { ++ DBG_MSG(1, ("Failed to initialize memory mapping in _ump_ukk_map_mem(). ID: %u\n", args->secure_id)); ++ ump_descriptor_mapping_free(session_data->cookies_map, map_id); ++ _mali_osk_free(descriptor); ++ ump_dd_reference_release(mem); ++ return err; ++ } ++ ++ DBG_MSG(4, ("Mapping virtual to physical memory: ID: %u, size:%lu, first physical addr: 0x%08lx, number of regions: %lu\n", ++ mem->secure_id, ++ mem->size_bytes, ++ ((NULL != mem->block_array) ? mem->block_array->addr : 0), ++ mem->nr_blocks)); ++ ++ left = descriptor->size; ++ /* loop over all blocks and map them in */ ++ for (block = 0; block < mem->nr_blocks; block++) { ++ unsigned long size_to_map; ++ ++ if (left > mem->block_array[block].size) { ++ size_to_map = mem->block_array[block].size; ++ } else { ++ size_to_map = left; ++ } ++ ++ if (_MALI_OSK_ERR_OK != _ump_osk_mem_mapregion_map(descriptor, offset, (u32 *) & (mem->block_array[block].addr), size_to_map)) { ++ DBG_MSG(1, ("WARNING: _ump_ukk_map_mem failed to map memory into userspace\n")); ++ ump_descriptor_mapping_free(session_data->cookies_map, map_id); ++ ump_dd_reference_release(mem); ++ _ump_osk_mem_mapregion_term(descriptor); ++ _mali_osk_free(descriptor); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ left -= size_to_map; ++ offset += size_to_map; ++ } ++ ++ /* Add to the ump_memory_allocation tracking list */ ++ _mali_osk_mutex_wait(session_data->lock); ++ _mali_osk_list_add(&descriptor->list, &session_data->list_head_session_memory_mappings_list); ++ _mali_osk_mutex_signal(session_data->lock); ++ ++ args->mapping = descriptor->mapping; ++ args->cookie = descriptor->cookie; ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++void _ump_ukk_unmap_mem(_ump_uk_unmap_mem_s *args) ++{ ++ struct ump_session_data *session_data; ++ ump_memory_allocation *descriptor; ++ ump_dd_handle handle; ++ ++ session_data = (ump_session_data *)args->ctx; ++ ++ if (NULL == session_data) { ++ MSG_ERR(("Session data is NULL in _ump_ukk_map_mem()\n")); ++ return; ++ } ++ ++ if (0 != ump_descriptor_mapping_get(session_data->cookies_map, (int)args->cookie, (void **)&descriptor)) { ++ MSG_ERR(("_ump_ukk_map_mem: cookie 0x%X not found for this session\n", args->cookie)); ++ return; ++ } ++ ++ DEBUG_ASSERT_POINTER(descriptor); ++ ++ handle = descriptor->handle; ++ if (UMP_DD_HANDLE_INVALID == handle) { ++ DBG_MSG(1, ("WARNING: Trying to unmap unknown handle: UNKNOWN\n")); ++ return; ++ } ++ ++ /* Remove the ump_memory_allocation from the list of tracked mappings */ ++ _mali_osk_mutex_wait(session_data->lock); ++ _mali_osk_list_del(&descriptor->list); ++ _mali_osk_mutex_signal(session_data->lock); ++ ++ ump_descriptor_mapping_free(session_data->cookies_map, (int)args->cookie); ++ ++ ump_dd_reference_release(handle); ++ ++ _ump_osk_mem_mapregion_term(descriptor); ++ _mali_osk_free(descriptor); ++} ++ ++u32 _ump_ukk_report_memory_usage(void) ++{ ++ if (device.backend->stat) ++ return device.backend->stat(device.backend); ++ else ++ return 0; ++} +diff --git a/drivers/gpu/arm/mali400/ump/common/ump_kernel_common.h b/drivers/gpu/arm/mali400/ump/common/ump_kernel_common.h +new file mode 100755 +index 000000000000..aa65f1cb6c88 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/ump/common/ump_kernel_common.h +@@ -0,0 +1,125 @@ ++/* ++ * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __UMP_KERNEL_COMMON_H__ ++#define __UMP_KERNEL_COMMON_H__ ++ ++#include "ump_kernel_types.h" ++#include "ump_kernel_interface.h" ++#include "ump_kernel_descriptor_mapping.h" ++#include "ump_kernel_random_mapping.h" ++#include "ump_kernel_memory_backend.h" ++ ++ ++#ifdef DEBUG ++extern int ump_debug_level; ++#define UMP_DEBUG_PRINT(args) _mali_osk_dbgmsg args ++#define UMP_DEBUG_CODE(args) args ++#define DBG_MSG(level,args) do { /* args should be in brackets */ \ ++ ((level) <= ump_debug_level)?\ ++ UMP_DEBUG_PRINT(("UMP<" #level ">: ")), \ ++ UMP_DEBUG_PRINT(args):0; \ ++ } while (0) ++ ++#define DBG_MSG_IF(level,condition,args) /* args should be in brackets */ \ ++ if((condition)&&((level) <= ump_debug_level)) {\ ++ UMP_DEBUG_PRINT(("UMP<" #level ">: ")); \ ++ UMP_DEBUG_PRINT(args); \ ++ } ++ ++#define DBG_MSG_ELSE(level,args) /* args should be in brackets */ \ ++ else if((level) <= ump_debug_level) { \ ++ UMP_DEBUG_PRINT(("UMP<" #level ">: ")); \ ++ UMP_DEBUG_PRINT(args); \ ++ } ++ ++#define DEBUG_ASSERT_POINTER(pointer) do {if( (pointer)== NULL) MSG_ERR(("NULL pointer " #pointer)); } while(0) ++#define DEBUG_ASSERT(condition) do {if(!(condition)) MSG_ERR(("ASSERT failed: " #condition)); } while(0) ++#else /* DEBUG */ ++#define UMP_DEBUG_PRINT(args) do {} while(0) ++#define UMP_DEBUG_CODE(args) ++#define DBG_MSG(level,args) do {} while(0) ++#define DBG_MSG_IF(level,condition,args) do {} while(0) ++#define DBG_MSG_ELSE(level,args) do {} while(0) ++#define DEBUG_ASSERT(condition) do {} while(0) ++#define DEBUG_ASSERT_POINTER(pointer) do {} while(0) ++#endif /* DEBUG */ ++ ++#define MSG_ERR(args) do{ /* args should be in brackets */ \ ++ _mali_osk_dbgmsg("UMP: ERR: %s\n" ,__FILE__); \ ++ _mali_osk_dbgmsg( " %s()%4d\n", __FUNCTION__, __LINE__) ; \ ++ _mali_osk_dbgmsg args ; \ ++ _mali_osk_dbgmsg("\n"); \ ++ } while(0) ++ ++#define MSG(args) do{ /* args should be in brackets */ \ ++ _mali_osk_dbgmsg("UMP: "); \ ++ _mali_osk_dbgmsg args; \ ++ } while (0) ++ ++ ++ ++/* ++ * This struct is used to store per session data. ++ * A session is created when someone open() the device, and ++ * closed when someone close() it or the user space application terminates. ++ */ ++typedef struct ump_session_data { ++ _mali_osk_list_t list_head_session_memory_list; /**< List of ump allocations made by the process (elements are ump_session_memory_list_element) */ ++ _mali_osk_list_t list_head_session_memory_mappings_list; /**< List of ump_memory_allocations mapped in */ ++ int api_version; ++ _mali_osk_mutex_t *lock; ++ ump_descriptor_mapping *cookies_map; /**< Secure mapping of cookies from _ump_ukk_map_mem() */ ++ int cache_operations_ongoing; ++ int has_pending_level1_cache_flush; ++} ump_session_data; ++ ++ ++ ++/* ++ * This struct is used to track the UMP memory references a session has. ++ * We need to track this in order to be able to clean up after user space processes ++ * which don't do it themself (e.g. due to a crash or premature termination). ++ */ ++typedef struct ump_session_memory_list_element { ++ struct ump_dd_mem *mem; ++ _mali_osk_list_t list; ++} ump_session_memory_list_element; ++ ++ ++ ++/* ++ * Device specific data, created when device driver is loaded, and then kept as the global variable device. ++ */ ++typedef struct ump_dev { ++ ump_random_mapping *secure_id_map; ++ ump_memory_backend *backend; ++} ump_dev; ++ ++ ++ ++extern int ump_debug_level; ++extern struct ump_dev device; ++ ++_mali_osk_errcode_t ump_kernel_constructor(void); ++void ump_kernel_destructor(void); ++int ump_map_errcode(_mali_osk_errcode_t err); ++ ++/** ++ * variables from user space cannot be dereferenced from kernel space; tagging them ++ * with __user allows the GCC compiler to generate a warning. Other compilers may ++ * not support this so we define it here as an empty macro if the compiler doesn't ++ * define it. ++ */ ++#ifndef __user ++#define __user ++#endif ++ ++#endif /* __UMP_KERNEL_COMMON_H__ */ +diff --git a/drivers/gpu/arm/mali400/ump/common/ump_kernel_descriptor_mapping.c b/drivers/gpu/arm/mali400/ump/common/ump_kernel_descriptor_mapping.c +new file mode 100755 +index 000000000000..e4642f0394c2 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/ump/common/ump_kernel_descriptor_mapping.c +@@ -0,0 +1,155 @@ ++/* ++ * Copyright (C) 2010-2011, 2013-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include "mali_kernel_common.h" ++#include "mali_osk.h" ++#include "mali_osk_bitops.h" ++#include "ump_kernel_common.h" ++#include "ump_kernel_descriptor_mapping.h" ++ ++#define MALI_PAD_INT(x) (((x) + (BITS_PER_LONG - 1)) & ~(BITS_PER_LONG - 1)) ++ ++/** ++ * Allocate a descriptor table capable of holding 'count' mappings ++ * @param count Number of mappings in the table ++ * @return Pointer to a new table, NULL on error ++ */ ++static ump_descriptor_table *descriptor_table_alloc(int count); ++ ++/** ++ * Free a descriptor table ++ * @param table The table to free ++ */ ++static void descriptor_table_free(ump_descriptor_table *table); ++ ++ump_descriptor_mapping *ump_descriptor_mapping_create(int init_entries, int max_entries) ++{ ++ ump_descriptor_mapping *map = _mali_osk_calloc(1, sizeof(ump_descriptor_mapping)); ++ ++ init_entries = MALI_PAD_INT(init_entries); ++ max_entries = MALI_PAD_INT(max_entries); ++ ++ if (NULL != map) { ++ map->table = descriptor_table_alloc(init_entries); ++ if (NULL != map->table) { ++ map->lock = _mali_osk_mutex_rw_init(_MALI_OSK_LOCKFLAG_UNORDERED, 0); ++ if (NULL != map->lock) { ++ _mali_osk_set_nonatomic_bit(0, map->table->usage); /* reserve bit 0 to prevent NULL/zero logic to kick in */ ++ map->max_nr_mappings_allowed = max_entries; ++ map->current_nr_mappings = init_entries; ++ return map; ++ } ++ descriptor_table_free(map->table); ++ } ++ _mali_osk_free(map); ++ } ++ return NULL; ++} ++ ++void ump_descriptor_mapping_destroy(ump_descriptor_mapping *map) ++{ ++ descriptor_table_free(map->table); ++ _mali_osk_mutex_rw_term(map->lock); ++ _mali_osk_free(map); ++} ++ ++int ump_descriptor_mapping_allocate_mapping(ump_descriptor_mapping *map, void *target) ++{ ++ int descriptor = -1;/*-EFAULT;*/ ++ _mali_osk_mutex_rw_wait(map->lock, _MALI_OSK_LOCKMODE_RW); ++ descriptor = _mali_osk_find_first_zero_bit(map->table->usage, map->current_nr_mappings); ++ if (descriptor == map->current_nr_mappings) { ++ int nr_mappings_new; ++ /* no free descriptor, try to expand the table */ ++ ump_descriptor_table *new_table; ++ ump_descriptor_table *old_table = map->table; ++ nr_mappings_new = map->current_nr_mappings * 2; ++ ++ if (map->current_nr_mappings >= map->max_nr_mappings_allowed) { ++ descriptor = -1; ++ goto unlock_and_exit; ++ } ++ ++ new_table = descriptor_table_alloc(nr_mappings_new); ++ if (NULL == new_table) { ++ descriptor = -1; ++ goto unlock_and_exit; ++ } ++ ++ _mali_osk_memcpy(new_table->usage, old_table->usage, (sizeof(unsigned long)*map->current_nr_mappings) / BITS_PER_LONG); ++ _mali_osk_memcpy(new_table->mappings, old_table->mappings, map->current_nr_mappings * sizeof(void *)); ++ map->table = new_table; ++ map->current_nr_mappings = nr_mappings_new; ++ descriptor_table_free(old_table); ++ } ++ ++ /* we have found a valid descriptor, set the value and usage bit */ ++ _mali_osk_set_nonatomic_bit(descriptor, map->table->usage); ++ map->table->mappings[descriptor] = target; ++ ++unlock_and_exit: ++ _mali_osk_mutex_rw_signal(map->lock, _MALI_OSK_LOCKMODE_RW); ++ return descriptor; ++} ++ ++int ump_descriptor_mapping_get(ump_descriptor_mapping *map, int descriptor, void **target) ++{ ++ int result = -1;/*-EFAULT;*/ ++ DEBUG_ASSERT(map); ++ _mali_osk_mutex_rw_wait(map->lock, _MALI_OSK_LOCKMODE_RO); ++ if ((descriptor > 0) && (descriptor < map->current_nr_mappings) && _mali_osk_test_bit(descriptor, map->table->usage)) { ++ *target = map->table->mappings[descriptor]; ++ result = 0; ++ } else *target = NULL; ++ _mali_osk_mutex_rw_signal(map->lock, _MALI_OSK_LOCKMODE_RO); ++ return result; ++} ++ ++int ump_descriptor_mapping_set(ump_descriptor_mapping *map, int descriptor, void *target) ++{ ++ int result = -1;/*-EFAULT;*/ ++ _mali_osk_mutex_rw_wait(map->lock, _MALI_OSK_LOCKMODE_RO); ++ if ((descriptor > 0) && (descriptor < map->current_nr_mappings) && _mali_osk_test_bit(descriptor, map->table->usage)) { ++ map->table->mappings[descriptor] = target; ++ result = 0; ++ } ++ _mali_osk_mutex_rw_signal(map->lock, _MALI_OSK_LOCKMODE_RO); ++ return result; ++} ++ ++void ump_descriptor_mapping_free(ump_descriptor_mapping *map, int descriptor) ++{ ++ _mali_osk_mutex_rw_wait(map->lock, _MALI_OSK_LOCKMODE_RW); ++ if ((descriptor > 0) && (descriptor < map->current_nr_mappings) && _mali_osk_test_bit(descriptor, map->table->usage)) { ++ map->table->mappings[descriptor] = NULL; ++ _mali_osk_clear_nonatomic_bit(descriptor, map->table->usage); ++ } ++ _mali_osk_mutex_rw_signal(map->lock, _MALI_OSK_LOCKMODE_RW); ++} ++ ++static ump_descriptor_table *descriptor_table_alloc(int count) ++{ ++ ump_descriptor_table *table; ++ ++ table = _mali_osk_calloc(1, sizeof(ump_descriptor_table) + ((sizeof(unsigned long) * count) / BITS_PER_LONG) + (sizeof(void *) * count)); ++ ++ if (NULL != table) { ++ table->usage = (u32 *)((u8 *)table + sizeof(ump_descriptor_table)); ++ table->mappings = (void **)((u8 *)table + sizeof(ump_descriptor_table) + ((sizeof(unsigned long) * count) / BITS_PER_LONG)); ++ } ++ ++ return table; ++} ++ ++static void descriptor_table_free(ump_descriptor_table *table) ++{ ++ _mali_osk_free(table); ++} ++ +diff --git a/drivers/gpu/arm/mali400/ump/common/ump_kernel_descriptor_mapping.h b/drivers/gpu/arm/mali400/ump/common/ump_kernel_descriptor_mapping.h +new file mode 100755 +index 000000000000..a888ba833fbb +--- /dev/null ++++ b/drivers/gpu/arm/mali400/ump/common/ump_kernel_descriptor_mapping.h +@@ -0,0 +1,89 @@ ++/* ++ * Copyright (C) 2010-2011, 2013-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file ump_kernel_descriptor_mapping.h ++ */ ++ ++#ifndef __UMP_KERNEL_DESCRIPTOR_MAPPING_H__ ++#define __UMP_KERNEL_DESCRIPTOR_MAPPING_H__ ++ ++#include "mali_osk.h" ++ ++/** ++ * The actual descriptor mapping table, never directly accessed by clients ++ */ ++typedef struct ump_descriptor_table { ++ u32 *usage; /**< Pointer to bitpattern indicating if a descriptor is valid/used or not */ ++ void **mappings; /**< Array of the pointers the descriptors map to */ ++} ump_descriptor_table; ++ ++/** ++ * The descriptor mapping object ++ * Provides a separate namespace where we can map an integer to a pointer ++ */ ++typedef struct ump_descriptor_mapping { ++ _mali_osk_mutex_rw_t *lock; /**< Lock protecting access to the mapping object */ ++ int max_nr_mappings_allowed; /**< Max number of mappings to support in this namespace */ ++ int current_nr_mappings; /**< Current number of possible mappings */ ++ ump_descriptor_table *table; /**< Pointer to the current mapping table */ ++} ump_descriptor_mapping; ++ ++/** ++ * Create a descriptor mapping object ++ * Create a descriptor mapping capable of holding init_entries growable to max_entries ++ * @param init_entries Number of entries to preallocate memory for ++ * @param max_entries Number of entries to max support ++ * @return Pointer to a descriptor mapping object, NULL on failure ++ */ ++ump_descriptor_mapping *ump_descriptor_mapping_create(int init_entries, int max_entries); ++ ++/** ++ * Destroy a descriptor mapping object ++ * @param map The map to free ++ */ ++void ump_descriptor_mapping_destroy(ump_descriptor_mapping *map); ++ ++/** ++ * Allocate a new mapping entry (descriptor ID) ++ * Allocates a new entry in the map. ++ * @param map The map to allocate a new entry in ++ * @param target The value to map to ++ * @return The descriptor allocated, a negative value on error ++ */ ++int ump_descriptor_mapping_allocate_mapping(ump_descriptor_mapping *map, void *target); ++ ++/** ++ * Get the value mapped to by a descriptor ID ++ * @param map The map to lookup the descriptor id in ++ * @param descriptor The descriptor ID to lookup ++ * @param target Pointer to a pointer which will receive the stored value ++ * @return 0 on successful lookup, negative on error ++ */ ++int ump_descriptor_mapping_get(ump_descriptor_mapping *map, int descriptor, void **target); ++ ++/** ++ * Set the value mapped to by a descriptor ID ++ * @param map The map to lookup the descriptor id in ++ * @param descriptor The descriptor ID to lookup ++ * @param target Pointer to replace the current value with ++ * @return 0 on successful lookup, negative on error ++ */ ++int ump_descriptor_mapping_set(ump_descriptor_mapping *map, int descriptor, void *target); ++ ++/** ++ * Free the descriptor ID ++ * For the descriptor to be reused it has to be freed ++ * @param map The map to free the descriptor from ++ * @param descriptor The descriptor ID to free ++ */ ++void ump_descriptor_mapping_free(ump_descriptor_mapping *map, int descriptor); ++ ++#endif /* __UMP_KERNEL_DESCRIPTOR_MAPPING_H__ */ +diff --git a/drivers/gpu/arm/mali400/ump/common/ump_kernel_memory_backend.h b/drivers/gpu/arm/mali400/ump/common/ump_kernel_memory_backend.h +new file mode 100755 +index 000000000000..2b69f68e87ac +--- /dev/null ++++ b/drivers/gpu/arm/mali400/ump/common/ump_kernel_memory_backend.h +@@ -0,0 +1,48 @@ ++/* ++ * Copyright (C) 2010-2011, 2013-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file ump_kernel_memory_mapping.h ++ */ ++ ++#ifndef __UMP_KERNEL_MEMORY_BACKEND_H__ ++#define __UMP_KERNEL_MEMORY_BACKEND_H__ ++ ++#include "ump_kernel_interface.h" ++#include "ump_kernel_types.h" ++ ++ ++typedef struct ump_memory_allocation { ++ void *phys_addr; ++ void *mapping; ++ unsigned long size; ++ ump_dd_handle handle; ++ void *process_mapping_info; ++ u32 cookie; /**< necessary on some U/K interface implementations */ ++ struct ump_session_data *ump_session; /**< Session that this allocation belongs to */ ++ _mali_osk_list_t list; /**< List for linking together memory allocations into the session's memory head */ ++ u32 is_cached; ++} ump_memory_allocation; ++ ++typedef struct ump_memory_backend { ++ int (*allocate)(void *ctx, ump_dd_mem *descriptor); ++ void (*release)(void *ctx, ump_dd_mem *descriptor); ++ void (*shutdown)(struct ump_memory_backend *backend); ++ u32(*stat)(struct ump_memory_backend *backend); ++ int (*pre_allocate_physical_check)(void *ctx, u32 size); ++ u32(*adjust_to_mali_phys)(void *ctx, u32 cpu_phys); ++ void *ctx; ++} ump_memory_backend; ++ ++ump_memory_backend *ump_memory_backend_create(void); ++void ump_memory_backend_destroy(void); ++ ++#endif /*__UMP_KERNEL_MEMORY_BACKEND_H__ */ ++ +diff --git a/drivers/gpu/arm/mali400/ump/common/ump_kernel_ref_drv.c b/drivers/gpu/arm/mali400/ump/common/ump_kernel_ref_drv.c +new file mode 100755 +index 000000000000..0b6434bee00f +--- /dev/null ++++ b/drivers/gpu/arm/mali400/ump/common/ump_kernel_ref_drv.c +@@ -0,0 +1,181 @@ ++/* ++ * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include "mali_osk.h" ++#include "mali_osk_list.h" ++#include "ump_osk.h" ++#include "ump_uk_types.h" ++ ++#include "ump_kernel_interface_ref_drv.h" ++#include "ump_kernel_common.h" ++#include "ump_kernel_descriptor_mapping.h" ++ ++#define UMP_MINIMUM_SIZE 4096 ++#define UMP_MINIMUM_SIZE_MASK (~(UMP_MINIMUM_SIZE-1)) ++#define UMP_SIZE_ALIGN(x) (((x)+UMP_MINIMUM_SIZE-1)&UMP_MINIMUM_SIZE_MASK) ++#define UMP_ADDR_ALIGN_OFFSET(x) ((x)&(UMP_MINIMUM_SIZE-1)) ++static void phys_blocks_release(void *ctx, struct ump_dd_mem *descriptor); ++ ++UMP_KERNEL_API_EXPORT ump_dd_handle ump_dd_handle_create_from_phys_blocks(ump_dd_physical_block *blocks, unsigned long num_blocks) ++{ ++ ump_dd_mem *mem; ++ unsigned long size_total = 0; ++ int ret; ++ u32 i; ++ ++ /* Go through the input blocks and verify that they are sane */ ++ for (i = 0; i < num_blocks; i++) { ++ unsigned long addr = blocks[i].addr; ++ unsigned long size = blocks[i].size; ++ ++ DBG_MSG(5, ("Adding physical memory to new handle. Address: 0x%08lx, size: %lu\n", addr, size)); ++ size_total += blocks[i].size; ++ ++ if (0 != UMP_ADDR_ALIGN_OFFSET(addr)) { ++ MSG_ERR(("Trying to create UMP memory from unaligned physical address. Address: 0x%08lx\n", addr)); ++ return UMP_DD_HANDLE_INVALID; ++ } ++ ++ if (0 != UMP_ADDR_ALIGN_OFFSET(size)) { ++ MSG_ERR(("Trying to create UMP memory with unaligned size. Size: %lu\n", size)); ++ return UMP_DD_HANDLE_INVALID; ++ } ++ } ++ ++ /* Allocate the ump_dd_mem struct for this allocation */ ++ mem = _mali_osk_malloc(sizeof(*mem)); ++ if (NULL == mem) { ++ DBG_MSG(1, ("Could not allocate ump_dd_mem in ump_dd_handle_create_from_phys_blocks()\n")); ++ return UMP_DD_HANDLE_INVALID; ++ } ++ ++ /* Now, make a copy of the block information supplied by the user */ ++ mem->block_array = _mali_osk_malloc(sizeof(ump_dd_physical_block) * num_blocks); ++ if (NULL == mem->block_array) { ++ _mali_osk_free(mem); ++ DBG_MSG(1, ("Could not allocate a mem handle for function ump_dd_handle_create_from_phys_blocks().\n")); ++ return UMP_DD_HANDLE_INVALID; ++ } ++ ++ _mali_osk_memcpy(mem->block_array, blocks, sizeof(ump_dd_physical_block) * num_blocks); ++ ++ /* And setup the rest of the ump_dd_mem struct */ ++ _mali_osk_atomic_init(&mem->ref_count, 1); ++ mem->size_bytes = size_total; ++ mem->nr_blocks = num_blocks; ++ mem->backend_info = NULL; ++ mem->ctx = NULL; ++ mem->release_func = phys_blocks_release; ++ /* For now UMP handles created by ump_dd_handle_create_from_phys_blocks() is forced to be Uncached */ ++ mem->is_cached = 0; ++ mem->hw_device = _UMP_UK_USED_BY_CPU; ++ mem->lock_usage = UMP_NOT_LOCKED; ++ ++ /* Find a secure ID for this allocation */ ++ ret = ump_random_mapping_insert(device.secure_id_map, mem); ++ if (unlikely(ret)) { ++ _mali_osk_free(mem->block_array); ++ _mali_osk_free(mem); ++ DBG_MSG(1, ("Failed to allocate secure ID in ump_dd_handle_create_from_phys_blocks()\n")); ++ return UMP_DD_HANDLE_INVALID; ++ } ++ ++ DBG_MSG(3, ("UMP memory created. ID: %u, size: %lu\n", mem->secure_id, mem->size_bytes)); ++ ++ return (ump_dd_handle)mem; ++} ++ ++static void phys_blocks_release(void *ctx, struct ump_dd_mem *descriptor) ++{ ++ _mali_osk_free(descriptor->block_array); ++ descriptor->block_array = NULL; ++} ++ ++_mali_osk_errcode_t _ump_ukk_allocate(_ump_uk_allocate_s *user_interaction) ++{ ++ ump_session_data *session_data = NULL; ++ ump_dd_mem *new_allocation = NULL; ++ ump_session_memory_list_element *session_memory_element = NULL; ++ int ret; ++ ++ DEBUG_ASSERT_POINTER(user_interaction); ++ DEBUG_ASSERT_POINTER(user_interaction->ctx); ++ ++ session_data = (ump_session_data *) user_interaction->ctx; ++ ++ session_memory_element = _mali_osk_calloc(1, sizeof(ump_session_memory_list_element)); ++ if (NULL == session_memory_element) { ++ DBG_MSG(1, ("Failed to allocate ump_session_memory_list_element in ump_ioctl_allocate()\n")); ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ ++ ++ new_allocation = _mali_osk_calloc(1, sizeof(ump_dd_mem)); ++ if (NULL == new_allocation) { ++ _mali_osk_free(session_memory_element); ++ DBG_MSG(1, ("Failed to allocate ump_dd_mem in _ump_ukk_allocate()\n")); ++ return _MALI_OSK_ERR_NOMEM; ++ } ++ ++ /* Initialize the part of the new_allocation that we know so for */ ++ _mali_osk_atomic_init(&new_allocation->ref_count, 1); ++ if (0 == (UMP_REF_DRV_UK_CONSTRAINT_USE_CACHE & user_interaction->constraints)) ++ new_allocation->is_cached = 0; ++ else new_allocation->is_cached = 1; ++ ++ /* Special case a size of 0, we should try to emulate what malloc does ++ * in this case, which is to return a valid pointer that must be freed, ++ * but can't be dereferenced */ ++ if (0 == user_interaction->size) { ++ /* Emulate by actually allocating the minimum block size */ ++ user_interaction->size = 1; ++ } ++ ++ /* Page align the size */ ++ new_allocation->size_bytes = UMP_SIZE_ALIGN(user_interaction->size); ++ new_allocation->lock_usage = UMP_NOT_LOCKED; ++ ++ /* Now, ask the active memory backend to do the actual memory allocation */ ++ if (!device.backend->allocate(device.backend->ctx, new_allocation)) { ++ DBG_MSG(3, ("OOM: No more UMP memory left. Failed to allocate memory in ump_ioctl_allocate(). Size: %lu, requested size: %lu\n", ++ new_allocation->size_bytes, ++ (unsigned long)user_interaction->size)); ++ _mali_osk_free(new_allocation); ++ _mali_osk_free(session_memory_element); ++ return _MALI_OSK_ERR_INVALID_FUNC; ++ } ++ new_allocation->hw_device = _UMP_UK_USED_BY_CPU; ++ new_allocation->ctx = device.backend->ctx; ++ new_allocation->release_func = device.backend->release; ++ ++ /* Initialize the session_memory_element, and add it to the session object */ ++ session_memory_element->mem = new_allocation; ++ _mali_osk_mutex_wait(session_data->lock); ++ _mali_osk_list_add(&(session_memory_element->list), &(session_data->list_head_session_memory_list)); ++ _mali_osk_mutex_signal(session_data->lock); ++ ++ /* Create a secure ID for this allocation */ ++ ret = ump_random_mapping_insert(device.secure_id_map, new_allocation); ++ if (unlikely(ret)) { ++ new_allocation->release_func(new_allocation->ctx, new_allocation); ++ _mali_osk_free(session_memory_element); ++ _mali_osk_free(new_allocation); ++ DBG_MSG(1, ("Failed to allocate secure ID in ump_ioctl_allocate()\n")); ++ return _MALI_OSK_ERR_INVALID_FUNC; ++ } ++ ++ user_interaction->secure_id = new_allocation->secure_id; ++ user_interaction->size = new_allocation->size_bytes; ++ DBG_MSG(3, ("UMP memory allocated. ID: %u, size: %lu\n", ++ new_allocation->secure_id, ++ new_allocation->size_bytes)); ++ ++ return _MALI_OSK_ERR_OK; ++} +diff --git a/drivers/gpu/arm/mali400/ump/common/ump_kernel_types.h b/drivers/gpu/arm/mali400/ump/common/ump_kernel_types.h +new file mode 100755 +index 000000000000..32f32ccbe9fc +--- /dev/null ++++ b/drivers/gpu/arm/mali400/ump/common/ump_kernel_types.h +@@ -0,0 +1,58 @@ ++/* ++ * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __UMP_KERNEL_TYPES_H__ ++#define __UMP_KERNEL_TYPES_H__ ++ ++#include "ump_kernel_interface.h" ++#include "mali_osk.h" ++ ++#include ++#ifdef CONFIG_DMA_SHARED_BUFFER ++#include ++#endif ++ ++typedef enum { ++ UMP_USED_BY_CPU = 0, ++ UMP_USED_BY_MALI = 1, ++ UMP_USED_BY_UNKNOWN_DEVICE = 100, ++} ump_hw_usage; ++ ++typedef enum { ++ UMP_NOT_LOCKED = 0, ++ UMP_READ = 1, ++ UMP_READ_WRITE = 3, ++} ump_lock_usage; ++ ++/* ++ * This struct is what is "behind" a ump_dd_handle ++ */ ++typedef struct ump_dd_mem { ++ struct rb_node node; ++ ump_secure_id secure_id; ++ _mali_osk_atomic_t ref_count; ++ unsigned long size_bytes; ++ unsigned long nr_blocks; ++ ump_dd_physical_block *block_array; ++ void (*release_func)(void *ctx, struct ump_dd_mem *descriptor); ++ void *ctx; ++ void *backend_info; ++ int is_cached; ++ ump_hw_usage hw_device; ++ ump_lock_usage lock_usage; ++#ifdef CONFIG_DMA_SHARED_BUFFER ++ struct dma_buf_attachment *import_attach; ++ struct sg_table *sgt; ++#endif ++} ump_dd_mem; ++ ++ ++ ++#endif /* __UMP_KERNEL_TYPES_H__ */ +diff --git a/drivers/gpu/arm/mali400/ump/common/ump_osk.h b/drivers/gpu/arm/mali400/ump/common/ump_osk.h +new file mode 100755 +index 000000000000..9adc4d3df3f5 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/ump/common/ump_osk.h +@@ -0,0 +1,48 @@ ++/* ++ * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file ump_osk.h ++ * Defines the OS abstraction layer for the UMP kernel device driver (OSK) ++ */ ++ ++#ifndef __UMP_OSK_H__ ++#define __UMP_OSK_H__ ++ ++#include ++#include ++#include "ump_uk_types.h" ++#include "ump_kernel_common.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++_mali_osk_errcode_t _ump_osk_init(void); ++ ++_mali_osk_errcode_t _ump_osk_term(void); ++ ++int _ump_osk_atomic_inc_and_read(_mali_osk_atomic_t *atom); ++ ++int _ump_osk_atomic_dec_and_read(_mali_osk_atomic_t *atom); ++ ++_mali_osk_errcode_t _ump_osk_mem_mapregion_init(ump_memory_allocation *descriptor); ++ ++_mali_osk_errcode_t _ump_osk_mem_mapregion_map(ump_memory_allocation *descriptor, u32 offset, u32 *phys_addr, unsigned long size); ++ ++void _ump_osk_mem_mapregion_term(ump_memory_allocation *descriptor); ++ ++void _ump_osk_msync(ump_dd_mem *mem, void *virt, u32 offset, u32 size, ump_uk_msync_op op, ump_session_data *session_data); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif +diff --git a/drivers/gpu/arm/mali400/ump/common/ump_uk_types.h b/drivers/gpu/arm/mali400/ump/common/ump_uk_types.h +new file mode 100755 +index 000000000000..db842cdcbeff +--- /dev/null ++++ b/drivers/gpu/arm/mali400/ump/common/ump_uk_types.h +@@ -0,0 +1,202 @@ ++/* ++ * Copyright (C) 2010, 2012-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file ump_uk_types.h ++ * Defines the types and constants used in the user-kernel interface ++ */ ++ ++#ifndef __UMP_UK_TYPES_H__ ++#define __UMP_UK_TYPES_H__ ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++/* Helpers for API version handling */ ++#define MAKE_VERSION_ID(x) (((x) << 16UL) | (x)) ++#define IS_VERSION_ID(x) (((x) & 0xFFFF) == (((x) >> 16UL) & 0xFFFF)) ++#define GET_VERSION(x) (((x) >> 16UL) & 0xFFFF) ++#define IS_API_MATCH(x, y) (IS_VERSION_ID((x)) && IS_VERSION_ID((y)) && (GET_VERSION((x)) == GET_VERSION((y)))) ++ ++/** ++ * API version define. ++ * Indicates the version of the kernel API ++ * The version is a 16bit integer incremented on each API change. ++ * The 16bit integer is stored twice in a 32bit integer ++ * So for version 1 the value would be 0x00010001 ++ */ ++#define UMP_IOCTL_API_VERSION MAKE_VERSION_ID(3) ++ ++typedef enum ++{ ++ _UMP_IOC_QUERY_API_VERSION = 1, ++ _UMP_IOC_ALLOCATE, ++ _UMP_IOC_RELEASE, ++ _UMP_IOC_SIZE_GET, ++ _UMP_IOC_MAP_MEM, /* not used in Linux */ ++ _UMP_IOC_UNMAP_MEM, /* not used in Linux */ ++ _UMP_IOC_MSYNC, ++ _UMP_IOC_CACHE_OPERATIONS_CONTROL, ++ _UMP_IOC_SWITCH_HW_USAGE, ++ _UMP_IOC_LOCK, ++ _UMP_IOC_UNLOCK, ++ _UMP_IOC_DMABUF_IMPORT, ++} _ump_uk_functions; ++ ++typedef enum ++{ ++ UMP_REF_DRV_UK_CONSTRAINT_NONE = 0, ++ UMP_REF_DRV_UK_CONSTRAINT_PHYSICALLY_LINEAR = 1, ++ UMP_REF_DRV_UK_CONSTRAINT_USE_CACHE = 4, ++} ump_uk_alloc_constraints; ++ ++typedef enum ++{ ++ _UMP_UK_MSYNC_CLEAN = 0, ++ _UMP_UK_MSYNC_CLEAN_AND_INVALIDATE = 1, ++ _UMP_UK_MSYNC_INVALIDATE = 2, ++ _UMP_UK_MSYNC_FLUSH_L1 = 3, ++ _UMP_UK_MSYNC_READOUT_CACHE_ENABLED = 128, ++} ump_uk_msync_op; ++ ++typedef enum ++{ ++ _UMP_UK_CACHE_OP_START = 0, ++ _UMP_UK_CACHE_OP_FINISH = 1, ++} ump_uk_cache_op_control; ++ ++typedef enum ++{ ++ _UMP_UK_READ = 1, ++ _UMP_UK_READ_WRITE = 3, ++} ump_uk_lock_usage; ++ ++typedef enum ++{ ++ _UMP_UK_USED_BY_CPU = 0, ++ _UMP_UK_USED_BY_MALI = 1, ++ _UMP_UK_USED_BY_UNKNOWN_DEVICE = 100, ++} ump_uk_user; ++ ++/** ++ * Get API version ([in,out] u32 api_version, [out] u32 compatible) ++ */ ++typedef struct _ump_uk_api_version_s ++{ ++ void *ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ u32 version; /**< Set to the user space version on entry, stores the device driver version on exit */ ++ u32 compatible; /**< Non-null if the device is compatible with the client */ ++} _ump_uk_api_version_s; ++ ++/** ++ * ALLOCATE ([out] u32 secure_id, [in,out] u32 size, [in] contraints) ++ */ ++typedef struct _ump_uk_allocate_s ++{ ++ void *ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ u32 secure_id; /**< Return value from DD to Userdriver */ ++ u32 size; /**< Input and output. Requested size; input. Returned size; output */ ++ ump_uk_alloc_constraints constraints; /**< Only input to Devicedriver */ ++} _ump_uk_allocate_s; ++ ++/** ++ * SIZE_GET ([in] u32 secure_id, [out]size ) ++ */ ++typedef struct _ump_uk_size_get_s ++{ ++ void *ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ u32 secure_id; /**< Input to DD */ ++ u32 size; /**< Returned size; output */ ++} _ump_uk_size_get_s; ++ ++/** ++ * Release ([in] u32 secure_id) ++ */ ++typedef struct _ump_uk_release_s ++{ ++ void *ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ u32 secure_id; /**< Input to DD */ ++} _ump_uk_release_s; ++ ++typedef struct _ump_uk_map_mem_s ++{ ++ void *ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ void *mapping; /**< [out] Returns user-space virtual address for the mapping */ ++ void *phys_addr; /**< [in] physical address */ ++ unsigned long size; /**< [in] size */ ++ u32 secure_id; /**< [in] secure_id to assign to mapping */ ++ void *_ukk_private; /**< Only used inside linux port between kernel frontend and common part to store vma */ ++ u32 cookie; ++ u32 is_cached; /**< [in,out] caching of CPU mappings */ ++} _ump_uk_map_mem_s; ++ ++typedef struct _ump_uk_unmap_mem_s ++{ ++ void *ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ void *mapping; ++ u32 size; ++ void *_ukk_private; ++ u32 cookie; ++} _ump_uk_unmap_mem_s; ++ ++typedef struct _ump_uk_msync_s ++{ ++ void *ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ void *mapping; /**< [in] mapping addr */ ++ void *address; /**< [in] flush start addr */ ++ u32 size; /**< [in] size to flush */ ++ ump_uk_msync_op op; /**< [in] flush operation */ ++ u32 cookie; /**< [in] cookie stored with reference to the kernel mapping internals */ ++ u32 secure_id; /**< [in] secure_id that identifies the ump buffer */ ++ u32 is_cached; /**< [out] caching of CPU mappings */ ++} _ump_uk_msync_s; ++ ++typedef struct _ump_uk_cache_operations_control_s ++{ ++ void *ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ ump_uk_cache_op_control op; /**< [in] cache operations start/stop */ ++} _ump_uk_cache_operations_control_s; ++ ++ ++typedef struct _ump_uk_switch_hw_usage_s ++{ ++ void *ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ u32 secure_id; /**< [in] secure_id that identifies the ump buffer */ ++ ump_uk_user new_user; /**< [in] cookie stored with reference to the kernel mapping internals */ ++ ++} _ump_uk_switch_hw_usage_s; ++ ++typedef struct _ump_uk_lock_s ++{ ++ void *ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ u32 secure_id; /**< [in] secure_id that identifies the ump buffer */ ++ ump_uk_lock_usage lock_usage; ++} _ump_uk_lock_s; ++ ++typedef struct _ump_uk_unlock_s ++{ ++ void *ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ u32 secure_id; /**< [in] secure_id that identifies the ump buffer */ ++} _ump_uk_unlock_s; ++ ++typedef struct _ump_uk_dmabuf_s ++{ ++ void *ctx; /**< [in,out] user-kernel context (trashed on output) */ ++ int fd; /**< [in] dmabuf_fd that identifies the dmabuf buffer */ ++ size_t size; /**< [in] size of the buffer */ ++ u32 secure_id; /**< [out] secure_id that identifies the ump buffer */ ++} _ump_uk_dmabuf_s; ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif /* __UMP_UK_TYPES_H__ */ +diff --git a/drivers/gpu/arm/mali400/ump/common/ump_ukk.h b/drivers/gpu/arm/mali400/ump/common/ump_ukk.h +new file mode 100755 +index 000000000000..f2906768c37f +--- /dev/null ++++ b/drivers/gpu/arm/mali400/ump/common/ump_ukk.h +@@ -0,0 +1,60 @@ ++/* ++ * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file ump_ukk.h ++ * Defines the kernel-side interface of the user-kernel interface ++ */ ++ ++#ifndef __UMP_UKK_H__ ++#define __UMP_UKK_H__ ++ ++#include "mali_osk.h" ++#include "ump_uk_types.h" ++ ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++ ++_mali_osk_errcode_t _ump_ukk_open(void **context); ++ ++_mali_osk_errcode_t _ump_ukk_close(void **context); ++ ++_mali_osk_errcode_t _ump_ukk_allocate(_ump_uk_allocate_s *user_interaction); ++ ++_mali_osk_errcode_t _ump_ukk_release(_ump_uk_release_s *release_info); ++ ++_mali_osk_errcode_t _ump_ukk_size_get(_ump_uk_size_get_s *user_interaction); ++ ++_mali_osk_errcode_t _ump_ukk_map_mem(_ump_uk_map_mem_s *args); ++ ++_mali_osk_errcode_t _ump_uku_get_api_version(_ump_uk_api_version_s *args); ++ ++void _ump_ukk_unmap_mem(_ump_uk_unmap_mem_s *args); ++ ++void _ump_ukk_msync(_ump_uk_msync_s *args); ++ ++void _ump_ukk_cache_operations_control(_ump_uk_cache_operations_control_s *args); ++ ++void _ump_ukk_switch_hw_usage(_ump_uk_switch_hw_usage_s *args); ++ ++void _ump_ukk_lock(_ump_uk_lock_s *args); ++ ++void _ump_ukk_unlock(_ump_uk_unlock_s *args); ++ ++u32 _ump_ukk_report_memory_usage(void); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif /* __UMP_UKK_H__ */ +diff --git a/drivers/gpu/arm/mali400/ump/linux/license/gpl/ump_kernel_license.h b/drivers/gpu/arm/mali400/ump/linux/license/gpl/ump_kernel_license.h +new file mode 100755 +index 000000000000..d0174055aa28 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/ump/linux/license/gpl/ump_kernel_license.h +@@ -0,0 +1,30 @@ ++/* ++ * Copyright (C) 2010, 2013, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file ump_kernel_license.h ++ * Defines for the macro MODULE_LICENSE. ++ */ ++ ++#ifndef __UMP_KERNEL_LICENSE_H__ ++#define __UMP_KERNEL_LICENSE_H__ ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++#define UMP_KERNEL_LINUX_LICENSE "GPL" ++#define UMP_LICENSE_IS_GPL 1 ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif /* __UMP_KERNEL_LICENSE_H__ */ +diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_ioctl.h b/drivers/gpu/arm/mali400/ump/linux/ump_ioctl.h +new file mode 100755 +index 000000000000..bfb4e8d64885 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/ump/linux/ump_ioctl.h +@@ -0,0 +1,54 @@ ++/* ++ * Copyright (C) 2010-2013, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __UMP_IOCTL_H__ ++#define __UMP_IOCTL_H__ ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++#include ++#include ++ ++#include ++ ++#ifndef __user ++#define __user ++#endif ++ ++ ++/** ++ * @file UMP_ioctl.h ++ * This file describes the interface needed to use the Linux device driver. ++ * The interface is used by the userpace UMP driver. ++ */ ++ ++#define UMP_IOCTL_NR 0x90 ++ ++ ++#define UMP_IOC_QUERY_API_VERSION _IOR(UMP_IOCTL_NR, _UMP_IOC_QUERY_API_VERSION, _ump_uk_api_version_s) ++#define UMP_IOC_ALLOCATE _IOWR(UMP_IOCTL_NR, _UMP_IOC_ALLOCATE, _ump_uk_allocate_s) ++#define UMP_IOC_RELEASE _IOR(UMP_IOCTL_NR, _UMP_IOC_RELEASE, _ump_uk_release_s) ++#define UMP_IOC_SIZE_GET _IOWR(UMP_IOCTL_NR, _UMP_IOC_SIZE_GET, _ump_uk_size_get_s) ++#define UMP_IOC_MSYNC _IOW(UMP_IOCTL_NR, _UMP_IOC_MSYNC, _ump_uk_msync_s) ++ ++#define UMP_IOC_CACHE_OPERATIONS_CONTROL _IOW(UMP_IOCTL_NR, _UMP_IOC_CACHE_OPERATIONS_CONTROL, _ump_uk_cache_operations_control_s) ++#define UMP_IOC_SWITCH_HW_USAGE _IOW(UMP_IOCTL_NR, _UMP_IOC_SWITCH_HW_USAGE, _ump_uk_switch_hw_usage_s) ++#define UMP_IOC_LOCK _IOW(UMP_IOCTL_NR, _UMP_IOC_LOCK, _ump_uk_lock_s) ++#define UMP_IOC_UNLOCK _IOW(UMP_IOCTL_NR, _UMP_IOC_UNLOCK, _ump_uk_unlock_s) ++ ++#define UMP_IOC_DMABUF_IMPORT _IOW(UMP_IOCTL_NR, _UMP_IOC_DMABUF_IMPORT, _ump_uk_dmabuf_s) ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif /* __UMP_IOCTL_H__ */ +diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_kernel_linux.c b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_linux.c +new file mode 100755 +index 000000000000..71b30830c308 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_linux.c +@@ -0,0 +1,449 @@ ++/* ++ * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include /* kernel module definitions */ ++#include /* file system operations */ ++#include /* character device definitions */ ++#include /* request_mem_region */ ++#include /* memory management functions and types */ ++#include /* user space access */ ++#include ++#include ++#include ++ ++#include "arch/config.h" /* Configuration for current platform. The symlinc for arch is set by Makefile */ ++#include "ump_ioctl.h" ++#include "ump_kernel_common.h" ++#include "ump_kernel_interface.h" ++#include "ump_kernel_interface_ref_drv.h" ++#include "ump_kernel_descriptor_mapping.h" ++#include "ump_kernel_memory_backend.h" ++#include "ump_kernel_memory_backend_os.h" ++#include "ump_kernel_memory_backend_dedicated.h" ++#include "ump_kernel_license.h" ++ ++#include "ump_osk.h" ++#include "ump_ukk.h" ++#include "ump_uk_types.h" ++#include "ump_ukk_wrappers.h" ++#include "ump_ukk_ref_wrappers.h" ++ ++ ++/* Module parameter to control log level */ ++int ump_debug_level = 2; ++module_param(ump_debug_level, int, S_IRUSR | S_IWUSR | S_IWGRP | S_IRGRP | S_IROTH); /* rw-rw-r-- */ ++MODULE_PARM_DESC(ump_debug_level, "Higher number, more dmesg output"); ++ ++/* By default the module uses any available major, but it's possible to set it at load time to a specific number */ ++int ump_major = 0; ++module_param(ump_major, int, S_IRUGO); /* r--r--r-- */ ++MODULE_PARM_DESC(ump_major, "Device major number"); ++ ++/* Name of the UMP device driver */ ++static char ump_dev_name[] = "ump"; /* should be const, but the functions we call requires non-cost */ ++ ++ ++#if UMP_LICENSE_IS_GPL ++static struct dentry *ump_debugfs_dir = NULL; ++#endif ++ ++/* ++ * The data which we attached to each virtual memory mapping request we get. ++ * Each memory mapping has a reference to the UMP memory it maps. ++ * We release this reference when the last memory mapping is unmapped. ++ */ ++typedef struct ump_vma_usage_tracker { ++ int references; ++ ump_dd_handle handle; ++} ump_vma_usage_tracker; ++ ++struct ump_device { ++ struct cdev cdev; ++#if UMP_LICENSE_IS_GPL ++ struct class *ump_class; ++#endif ++}; ++ ++/* The global variable containing the global device data */ ++static struct ump_device ump_device; ++struct device *ump_global_mdev = NULL; ++ ++/* Forward declare static functions */ ++static int ump_file_open(struct inode *inode, struct file *filp); ++static int ump_file_release(struct inode *inode, struct file *filp); ++#ifdef HAVE_UNLOCKED_IOCTL ++static long ump_file_ioctl(struct file *filp, unsigned int cmd, unsigned long arg); ++#else ++static int ump_file_ioctl(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg); ++#endif ++static int ump_file_mmap(struct file *filp, struct vm_area_struct *vma); ++ ++ ++/* This variable defines the file operations this UMP device driver offer */ ++static struct file_operations ump_fops = { ++ .owner = THIS_MODULE, ++ .open = ump_file_open, ++ .release = ump_file_release, ++#ifdef HAVE_UNLOCKED_IOCTL ++ .unlocked_ioctl = ump_file_ioctl, ++#else ++ .ioctl = ump_file_ioctl, ++#endif ++ .mmap = ump_file_mmap ++}; ++ ++ ++/* This function is called by Linux to initialize this module. ++ * All we do is initialize the UMP device driver. ++ */ ++static int ump_initialize_module(void) ++{ ++ _mali_osk_errcode_t err; ++ ++ DBG_MSG(2, ("Inserting UMP device driver. Compiled: %s, time: %s\n", __DATE__, __TIME__)); ++ ++ err = ump_kernel_constructor(); ++ if (_MALI_OSK_ERR_OK != err) { ++ MSG_ERR(("UMP device driver init failed\n")); ++ return ump_map_errcode(err); ++ } ++ ++ MSG(("UMP device driver %s loaded\n", SVN_REV_STRING)); ++ return 0; ++} ++ ++ ++ ++/* ++ * This function is called by Linux to unload/terminate/exit/cleanup this module. ++ * All we do is terminate the UMP device driver. ++ */ ++static void ump_cleanup_module(void) ++{ ++ DBG_MSG(2, ("Unloading UMP device driver\n")); ++ ump_kernel_destructor(); ++ DBG_MSG(2, ("Module unloaded\n")); ++} ++ ++ ++ ++static ssize_t ump_memory_used_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) ++{ ++ char buf[64]; ++ size_t r; ++ u32 mem = _ump_ukk_report_memory_usage(); ++ ++ r = snprintf(buf, 64, "%u\n", mem); ++ return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); ++} ++ ++static const struct file_operations ump_memory_usage_fops = { ++ .owner = THIS_MODULE, ++ .read = ump_memory_used_read, ++}; ++ ++/* ++ * Initialize the UMP device driver. ++ */ ++int ump_kernel_device_initialize(void) ++{ ++ int err; ++ dev_t dev = 0; ++#if UMP_LICENSE_IS_GPL ++ ump_debugfs_dir = debugfs_create_dir(ump_dev_name, NULL); ++ if (ERR_PTR(-ENODEV) == ump_debugfs_dir) { ++ ump_debugfs_dir = NULL; ++ } else { ++ debugfs_create_file("memory_usage", 0400, ump_debugfs_dir, NULL, &ump_memory_usage_fops); ++ } ++#endif ++ ++ if (0 == ump_major) { ++ /* auto select a major */ ++ err = alloc_chrdev_region(&dev, 0, 1, ump_dev_name); ++ ump_major = MAJOR(dev); ++ } else { ++ /* use load time defined major number */ ++ dev = MKDEV(ump_major, 0); ++ err = register_chrdev_region(dev, 1, ump_dev_name); ++ } ++ ++ if (0 == err) { ++ memset(&ump_device, 0, sizeof(ump_device)); ++ ++ /* initialize our char dev data */ ++ cdev_init(&ump_device.cdev, &ump_fops); ++ ump_device.cdev.owner = THIS_MODULE; ++ ump_device.cdev.ops = &ump_fops; ++ ++ /* register char dev with the kernel */ ++ err = cdev_add(&ump_device.cdev, dev, 1/*count*/); ++ if (0 == err) { ++ ++#if UMP_LICENSE_IS_GPL ++ ump_device.ump_class = class_create(THIS_MODULE, ump_dev_name); ++ if (IS_ERR(ump_device.ump_class)) { ++ err = PTR_ERR(ump_device.ump_class); ++ } else { ++ ump_global_mdev = device_create(ump_device.ump_class, NULL, dev, NULL, ump_dev_name); ++ if (!IS_ERR(ump_global_mdev)) { ++ return 0; ++ } ++ ++ err = PTR_ERR(ump_global_mdev); ++ } ++ cdev_del(&ump_device.cdev); ++#else ++ return 0; ++#endif ++ } ++ ++ unregister_chrdev_region(dev, 1); ++ } ++ ++ return err; ++} ++ ++ ++ ++/* ++ * Terminate the UMP device driver ++ */ ++void ump_kernel_device_terminate(void) ++{ ++ dev_t dev = MKDEV(ump_major, 0); ++ ++#if UMP_LICENSE_IS_GPL ++ device_destroy(ump_device.ump_class, dev); ++ class_destroy(ump_device.ump_class); ++#endif ++ ++ /* unregister char device */ ++ cdev_del(&ump_device.cdev); ++ ++ /* free major */ ++ unregister_chrdev_region(dev, 1); ++ ++#if UMP_LICENSE_IS_GPL ++ if (ump_debugfs_dir) ++ debugfs_remove_recursive(ump_debugfs_dir); ++#endif ++} ++ ++/* ++ * Open a new session. User space has called open() on us. ++ */ ++static int ump_file_open(struct inode *inode, struct file *filp) ++{ ++ struct ump_session_data *session_data; ++ _mali_osk_errcode_t err; ++ ++ /* input validation */ ++ if (0 != MINOR(inode->i_rdev)) { ++ MSG_ERR(("Minor not zero in ump_file_open()\n")); ++ return -ENODEV; ++ } ++ ++ /* Call the OS-Independent UMP Open function */ ++ err = _ump_ukk_open((void **) &session_data); ++ if (_MALI_OSK_ERR_OK != err) { ++ MSG_ERR(("Ump failed to open a new session\n")); ++ return ump_map_errcode(err); ++ } ++ ++ filp->private_data = (void *)session_data; ++ filp->f_pos = 0; ++ ++ return 0; /* success */ ++} ++ ++ ++ ++/* ++ * Close a session. User space has called close() or crashed/terminated. ++ */ ++static int ump_file_release(struct inode *inode, struct file *filp) ++{ ++ _mali_osk_errcode_t err; ++ ++ err = _ump_ukk_close((void **) &filp->private_data); ++ if (_MALI_OSK_ERR_OK != err) { ++ return ump_map_errcode(err); ++ } ++ ++ return 0; /* success */ ++} ++ ++ ++ ++/* ++ * Handle IOCTL requests. ++ */ ++#ifdef HAVE_UNLOCKED_IOCTL ++static long ump_file_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) ++#else ++static int ump_file_ioctl(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg) ++#endif ++{ ++ int err = -ENOTTY; ++ void __user *argument; ++ struct ump_session_data *session_data; ++ ++#ifndef HAVE_UNLOCKED_IOCTL ++ (void)inode; /* inode not used */ ++#endif ++ ++ session_data = (struct ump_session_data *)filp->private_data; ++ if (NULL == session_data) { ++ MSG_ERR(("No session data attached to file object\n")); ++ return -ENOTTY; ++ } ++ ++ /* interpret the argument as a user pointer to something */ ++ argument = (void __user *)arg; ++ ++ switch (cmd) { ++ case UMP_IOC_QUERY_API_VERSION: ++ err = ump_get_api_version_wrapper((u32 __user *)argument, session_data); ++ break; ++ ++ case UMP_IOC_ALLOCATE : ++ err = ump_allocate_wrapper((u32 __user *)argument, session_data); ++ break; ++ ++ case UMP_IOC_RELEASE: ++ err = ump_release_wrapper((u32 __user *)argument, session_data); ++ break; ++ ++ case UMP_IOC_SIZE_GET: ++ err = ump_size_get_wrapper((u32 __user *)argument, session_data); ++ break; ++ ++ case UMP_IOC_MSYNC: ++ err = ump_msync_wrapper((u32 __user *)argument, session_data); ++ break; ++ ++ case UMP_IOC_CACHE_OPERATIONS_CONTROL: ++ err = ump_cache_operations_control_wrapper((u32 __user *)argument, session_data); ++ break; ++ ++ case UMP_IOC_SWITCH_HW_USAGE: ++ err = ump_switch_hw_usage_wrapper((u32 __user *)argument, session_data); ++ break; ++ ++ case UMP_IOC_LOCK: ++ err = ump_lock_wrapper((u32 __user *)argument, session_data); ++ break; ++ ++ case UMP_IOC_UNLOCK: ++ err = ump_unlock_wrapper((u32 __user *)argument, session_data); ++ break; ++ ++ case UMP_IOC_DMABUF_IMPORT: ++#ifdef CONFIG_DMA_SHARED_BUFFER ++ err = ump_dmabuf_import_wrapper((u32 __user *)argument, session_data); ++#else ++ err = -EFAULT; ++ DBG_MSG(1, ("User space use dmabuf API, but kernel don't support DMA BUF\n")); ++#endif ++ break; ++ ++ default: ++ DBG_MSG(1, ("No handler for IOCTL. cmd: 0x%08x, arg: 0x%08lx\n", cmd, arg)); ++ err = -EFAULT; ++ break; ++ } ++ ++ return err; ++} ++ ++int ump_map_errcode(_mali_osk_errcode_t err) ++{ ++ switch (err) { ++ case _MALI_OSK_ERR_OK : ++ return 0; ++ case _MALI_OSK_ERR_FAULT: ++ return -EFAULT; ++ case _MALI_OSK_ERR_INVALID_FUNC: ++ return -ENOTTY; ++ case _MALI_OSK_ERR_INVALID_ARGS: ++ return -EINVAL; ++ case _MALI_OSK_ERR_NOMEM: ++ return -ENOMEM; ++ case _MALI_OSK_ERR_TIMEOUT: ++ return -ETIMEDOUT; ++ case _MALI_OSK_ERR_RESTARTSYSCALL: ++ return -ERESTARTSYS; ++ case _MALI_OSK_ERR_ITEM_NOT_FOUND: ++ return -ENOENT; ++ default: ++ return -EFAULT; ++ } ++} ++ ++/* ++ * Handle from OS to map specified virtual memory to specified UMP memory. ++ */ ++static int ump_file_mmap(struct file *filp, struct vm_area_struct *vma) ++{ ++ _ump_uk_map_mem_s args; ++ _mali_osk_errcode_t err; ++ struct ump_session_data *session_data; ++ ++ /* Validate the session data */ ++ session_data = (struct ump_session_data *)filp->private_data; ++ if (NULL == session_data) { ++ MSG_ERR(("mmap() called without any session data available\n")); ++ return -EFAULT; ++ } ++ ++ /* Re-pack the arguments that mmap() packed for us */ ++ args.ctx = session_data; ++ args.phys_addr = 0; ++ args.size = vma->vm_end - vma->vm_start; ++ args._ukk_private = vma; ++ args.secure_id = vma->vm_pgoff; ++ ++ /* By setting this flag, during a process fork; the child process will not have the parent UMP mappings */ ++ vma->vm_flags |= VM_DONTCOPY; ++ ++ DBG_MSG(4, ("UMP vma->flags: %x\n", vma->vm_flags)); ++ ++ /* Call the common mmap handler */ ++ err = _ump_ukk_map_mem(&args); ++ if (_MALI_OSK_ERR_OK != err) { ++ MSG_ERR(("_ump_ukk_map_mem() failed in function ump_file_mmap()")); ++ return ump_map_errcode(err); ++ } ++ ++ return 0; /* success */ ++} ++ ++/* Export UMP kernel space API functions */ ++EXPORT_SYMBOL(ump_dd_secure_id_get); ++EXPORT_SYMBOL(ump_dd_handle_create_from_secure_id); ++EXPORT_SYMBOL(ump_dd_phys_block_count_get); ++EXPORT_SYMBOL(ump_dd_phys_block_get); ++EXPORT_SYMBOL(ump_dd_phys_blocks_get); ++EXPORT_SYMBOL(ump_dd_size_get); ++EXPORT_SYMBOL(ump_dd_reference_add); ++EXPORT_SYMBOL(ump_dd_reference_release); ++ ++/* Export our own extended kernel space allocator */ ++EXPORT_SYMBOL(ump_dd_handle_create_from_phys_blocks); ++ ++/* Setup init and exit functions for this module */ ++module_init(ump_initialize_module); ++module_exit(ump_cleanup_module); ++ ++/* And some module informatio */ ++MODULE_LICENSE(UMP_KERNEL_LINUX_LICENSE); ++MODULE_AUTHOR("ARM Ltd."); ++MODULE_VERSION(SVN_REV_STRING); +diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_kernel_linux.h b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_linux.h +new file mode 100755 +index 000000000000..8d32ddbb5449 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_linux.h +@@ -0,0 +1,18 @@ ++/* ++ * Copyright (C) 2010-2013, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __UMP_KERNEL_LINUX_H__ ++#define __UMP_KERNEL_LINUX_H__ ++ ++int ump_kernel_device_initialize(void); ++void ump_kernel_device_terminate(void); ++ ++ ++#endif /* __UMP_KERNEL_H__ */ +diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_dedicated.c b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_dedicated.c +new file mode 100755 +index 000000000000..5a1257a25b82 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_dedicated.c +@@ -0,0 +1,271 @@ ++/* ++ * Copyright (C) 2010-2011, 2013-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/* needed to detect kernel version specific code */ ++#include ++ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,26) ++#include ++#else /* pre 2.6.26 the file was in the arch specific location */ ++#include ++#endif ++ ++#include ++#include ++#include ++#include ++#include "ump_kernel_common.h" ++#include "ump_kernel_memory_backend.h" ++ ++ ++ ++#define UMP_BLOCK_SIZE (256UL * 1024UL) /* 256kB, remember to keep the ()s */ ++ ++ ++ ++typedef struct block_info { ++ struct block_info *next; ++} block_info; ++ ++ ++ ++typedef struct block_allocator { ++ struct semaphore mutex; ++ block_info *all_blocks; ++ block_info *first_free; ++ u32 base; ++ u32 num_blocks; ++ u32 num_free; ++} block_allocator; ++ ++ ++static void block_allocator_shutdown(ump_memory_backend *backend); ++static int block_allocator_allocate(void *ctx, ump_dd_mem *mem); ++static void block_allocator_release(void *ctx, ump_dd_mem *handle); ++static inline u32 get_phys(block_allocator *allocator, block_info *block); ++static u32 block_allocator_stat(struct ump_memory_backend *backend); ++ ++ ++ ++/* ++ * Create dedicated memory backend ++ */ ++ump_memory_backend *ump_block_allocator_create(u32 base_address, u32 size) ++{ ++ ump_memory_backend *backend; ++ block_allocator *allocator; ++ u32 usable_size; ++ u32 num_blocks; ++ ++ usable_size = (size + UMP_BLOCK_SIZE - 1) & ~(UMP_BLOCK_SIZE - 1); ++ num_blocks = usable_size / UMP_BLOCK_SIZE; ++ ++ if (0 == usable_size) { ++ DBG_MSG(1, ("Memory block of size %u is unusable\n", size)); ++ return NULL; ++ } ++ ++ DBG_MSG(5, ("Creating dedicated UMP memory backend. Base address: 0x%08x, size: 0x%08x\n", base_address, size)); ++ DBG_MSG(6, ("%u usable bytes which becomes %u blocks\n", usable_size, num_blocks)); ++ ++ backend = kzalloc(sizeof(ump_memory_backend), GFP_KERNEL); ++ if (NULL != backend) { ++ allocator = kmalloc(sizeof(block_allocator), GFP_KERNEL); ++ if (NULL != allocator) { ++ allocator->all_blocks = kmalloc(sizeof(block_info) * num_blocks, GFP_KERNEL); ++ if (NULL != allocator->all_blocks) { ++ int i; ++ ++ allocator->first_free = NULL; ++ allocator->num_blocks = num_blocks; ++ allocator->num_free = num_blocks; ++ allocator->base = base_address; ++ sema_init(&allocator->mutex, 1); ++ ++ for (i = 0; i < num_blocks; i++) { ++ allocator->all_blocks[i].next = allocator->first_free; ++ allocator->first_free = &allocator->all_blocks[i]; ++ } ++ ++ backend->ctx = allocator; ++ backend->allocate = block_allocator_allocate; ++ backend->release = block_allocator_release; ++ backend->shutdown = block_allocator_shutdown; ++ backend->stat = block_allocator_stat; ++ backend->pre_allocate_physical_check = NULL; ++ backend->adjust_to_mali_phys = NULL; ++ ++ return backend; ++ } ++ kfree(allocator); ++ } ++ kfree(backend); ++ } ++ ++ return NULL; ++} ++ ++ ++ ++/* ++ * Destroy specified dedicated memory backend ++ */ ++static void block_allocator_shutdown(ump_memory_backend *backend) ++{ ++ block_allocator *allocator; ++ ++ BUG_ON(!backend); ++ BUG_ON(!backend->ctx); ++ ++ allocator = (block_allocator *)backend->ctx; ++ ++ DBG_MSG_IF(1, allocator->num_free != allocator->num_blocks, ("%u blocks still in use during shutdown\n", allocator->num_blocks - allocator->num_free)); ++ ++ kfree(allocator->all_blocks); ++ kfree(allocator); ++ kfree(backend); ++} ++ ++ ++ ++static int block_allocator_allocate(void *ctx, ump_dd_mem *mem) ++{ ++ block_allocator *allocator; ++ u32 left; ++ block_info *last_allocated = NULL; ++ int i = 0; ++ ++ BUG_ON(!ctx); ++ BUG_ON(!mem); ++ ++ allocator = (block_allocator *)ctx; ++ left = mem->size_bytes; ++ ++ BUG_ON(!left); ++ BUG_ON(!&allocator->mutex); ++ ++ mem->nr_blocks = ((left + UMP_BLOCK_SIZE - 1) & ~(UMP_BLOCK_SIZE - 1)) / UMP_BLOCK_SIZE; ++ mem->block_array = (ump_dd_physical_block *)vmalloc(sizeof(ump_dd_physical_block) * mem->nr_blocks); ++ if (NULL == mem->block_array) { ++ MSG_ERR(("Failed to allocate block array\n")); ++ return 0; ++ } ++ ++ if (down_interruptible(&allocator->mutex)) { ++ MSG_ERR(("Could not get mutex to do block_allocate\n")); ++ return 0; ++ } ++ ++ mem->size_bytes = 0; ++ ++ while ((left > 0) && (allocator->first_free)) { ++ block_info *block; ++ ++ block = allocator->first_free; ++ allocator->first_free = allocator->first_free->next; ++ block->next = last_allocated; ++ last_allocated = block; ++ allocator->num_free--; ++ ++ mem->block_array[i].addr = get_phys(allocator, block); ++ mem->block_array[i].size = UMP_BLOCK_SIZE; ++ mem->size_bytes += UMP_BLOCK_SIZE; ++ ++ i++; ++ ++ if (left < UMP_BLOCK_SIZE) left = 0; ++ else left -= UMP_BLOCK_SIZE; ++ } ++ ++ if (left) { ++ block_info *block; ++ /* release all memory back to the pool */ ++ while (last_allocated) { ++ block = last_allocated->next; ++ last_allocated->next = allocator->first_free; ++ allocator->first_free = last_allocated; ++ last_allocated = block; ++ allocator->num_free++; ++ } ++ ++ vfree(mem->block_array); ++ mem->backend_info = NULL; ++ mem->block_array = NULL; ++ ++ DBG_MSG(4, ("Could not find a mem-block for the allocation.\n")); ++ up(&allocator->mutex); ++ ++ return 0; ++ } ++ ++ mem->backend_info = last_allocated; ++ ++ up(&allocator->mutex); ++ mem->is_cached = 0; ++ ++ return 1; ++} ++ ++ ++ ++static void block_allocator_release(void *ctx, ump_dd_mem *handle) ++{ ++ block_allocator *allocator; ++ block_info *block, * next; ++ ++ BUG_ON(!ctx); ++ BUG_ON(!handle); ++ ++ allocator = (block_allocator *)ctx; ++ block = (block_info *)handle->backend_info; ++ BUG_ON(!block); ++ ++ if (down_interruptible(&allocator->mutex)) { ++ MSG_ERR(("Allocator release: Failed to get mutex - memory leak\n")); ++ return; ++ } ++ ++ while (block) { ++ next = block->next; ++ ++ BUG_ON((block < allocator->all_blocks) || (block > (allocator->all_blocks + allocator->num_blocks))); ++ ++ block->next = allocator->first_free; ++ allocator->first_free = block; ++ allocator->num_free++; ++ ++ block = next; ++ } ++ DBG_MSG(3, ("%d blocks free after release call\n", allocator->num_free)); ++ up(&allocator->mutex); ++ ++ vfree(handle->block_array); ++ handle->block_array = NULL; ++} ++ ++ ++ ++/* ++ * Helper function for calculating the physical base adderss of a memory block ++ */ ++static inline u32 get_phys(block_allocator *allocator, block_info *block) ++{ ++ return allocator->base + ((block - allocator->all_blocks) * UMP_BLOCK_SIZE); ++} ++ ++static u32 block_allocator_stat(struct ump_memory_backend *backend) ++{ ++ block_allocator *allocator; ++ BUG_ON(!backend); ++ allocator = (block_allocator *)backend->ctx; ++ BUG_ON(!allocator); ++ ++ return (allocator->num_blocks - allocator->num_free) * UMP_BLOCK_SIZE; ++} +diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_dedicated.h b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_dedicated.h +new file mode 100755 +index 000000000000..949fd245c6af +--- /dev/null ++++ b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_dedicated.h +@@ -0,0 +1,23 @@ ++/* ++ * Copyright (C) 2010, 2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file ump_kernel_memory_backend_dedicated.h ++ */ ++ ++#ifndef __UMP_KERNEL_MEMORY_BACKEND_DEDICATED_H__ ++#define __UMP_KERNEL_MEMORY_BACKEND_DEDICATED_H__ ++ ++#include "ump_kernel_memory_backend.h" ++ ++ump_memory_backend *ump_block_allocator_create(u32 base_address, u32 size); ++ ++#endif /* __UMP_KERNEL_MEMORY_BACKEND_DEDICATED_H__ */ ++ +diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_os.c b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_os.c +new file mode 100755 +index 000000000000..7cd8d5d381cf +--- /dev/null ++++ b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_os.c +@@ -0,0 +1,235 @@ ++/* ++ * Copyright (C) 2010-2011, 2013-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/* needed to detect kernel version specific code */ ++#include ++ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,26) ++#include ++#else /* pre 2.6.26 the file was in the arch specific location */ ++#include ++#endif ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include "ump_kernel_common.h" ++#include "ump_kernel_memory_backend.h" ++ ++ ++ ++typedef struct os_allocator { ++ struct semaphore mutex; ++ u32 num_pages_max; /**< Maximum number of pages to allocate from the OS */ ++ u32 num_pages_allocated; /**< Number of pages allocated from the OS */ ++} os_allocator; ++ ++ ++ ++static void os_free(void *ctx, ump_dd_mem *descriptor); ++static int os_allocate(void *ctx, ump_dd_mem *descriptor); ++static void os_memory_backend_destroy(ump_memory_backend *backend); ++static u32 os_stat(struct ump_memory_backend *backend); ++ ++ ++ ++/* ++ * Create OS memory backend ++ */ ++ump_memory_backend *ump_os_memory_backend_create(const int max_allocation) ++{ ++ ump_memory_backend *backend; ++ os_allocator *info; ++ ++ info = kmalloc(sizeof(os_allocator), GFP_KERNEL); ++ if (NULL == info) { ++ return NULL; ++ } ++ ++ info->num_pages_max = max_allocation >> PAGE_SHIFT; ++ info->num_pages_allocated = 0; ++ ++ sema_init(&info->mutex, 1); ++ ++ backend = kmalloc(sizeof(ump_memory_backend), GFP_KERNEL); ++ if (NULL == backend) { ++ kfree(info); ++ return NULL; ++ } ++ ++ backend->ctx = info; ++ backend->allocate = os_allocate; ++ backend->release = os_free; ++ backend->shutdown = os_memory_backend_destroy; ++ backend->stat = os_stat; ++ backend->pre_allocate_physical_check = NULL; ++ backend->adjust_to_mali_phys = NULL; ++ ++ return backend; ++} ++ ++ ++ ++/* ++ * Destroy specified OS memory backend ++ */ ++static void os_memory_backend_destroy(ump_memory_backend *backend) ++{ ++ os_allocator *info = (os_allocator *)backend->ctx; ++ ++ DBG_MSG_IF(1, 0 != info->num_pages_allocated, ("%d pages still in use during shutdown\n", info->num_pages_allocated)); ++ ++ kfree(info); ++ kfree(backend); ++} ++ ++ ++ ++/* ++ * Allocate UMP memory ++ */ ++static int os_allocate(void *ctx, ump_dd_mem *descriptor) ++{ ++ u32 left; ++ os_allocator *info; ++ int pages_allocated = 0; ++ int is_cached; ++ ++ BUG_ON(!descriptor); ++ BUG_ON(!ctx); ++ ++ info = (os_allocator *)ctx; ++ left = descriptor->size_bytes; ++ is_cached = descriptor->is_cached; ++ ++ if (down_interruptible(&info->mutex)) { ++ DBG_MSG(1, ("Failed to get mutex in os_free\n")); ++ return 0; /* failure */ ++ } ++ ++ descriptor->backend_info = NULL; ++ descriptor->nr_blocks = ((left + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1)) >> PAGE_SHIFT; ++ ++ DBG_MSG(5, ("Allocating page array. Size: %lu\n", descriptor->nr_blocks * sizeof(ump_dd_physical_block))); ++ ++ descriptor->block_array = (ump_dd_physical_block *)vmalloc(sizeof(ump_dd_physical_block) * descriptor->nr_blocks); ++ if (NULL == descriptor->block_array) { ++ up(&info->mutex); ++ DBG_MSG(1, ("Block array could not be allocated\n")); ++ return 0; /* failure */ ++ } ++ ++ while (left > 0 && ((info->num_pages_allocated + pages_allocated) < info->num_pages_max)) { ++ struct page *new_page; ++ ++ if (is_cached) { ++ new_page = alloc_page(GFP_HIGHUSER | __GFP_ZERO | __GFP_REPEAT | __GFP_NOWARN); ++ } else { ++ new_page = alloc_page(GFP_HIGHUSER | __GFP_ZERO | __GFP_REPEAT | __GFP_NOWARN | __GFP_COLD); ++ } ++ if (NULL == new_page) { ++ break; ++ } ++ ++ /* Ensure page caches are flushed. */ ++ if (is_cached) { ++ descriptor->block_array[pages_allocated].addr = page_to_phys(new_page); ++ descriptor->block_array[pages_allocated].size = PAGE_SIZE; ++ } else { ++ descriptor->block_array[pages_allocated].addr = dma_map_page(NULL, new_page, 0, PAGE_SIZE, DMA_BIDIRECTIONAL); ++ descriptor->block_array[pages_allocated].size = PAGE_SIZE; ++ } ++ ++ DBG_MSG(5, ("Allocated page 0x%08lx cached: %d\n", descriptor->block_array[pages_allocated].addr, is_cached)); ++ ++ if (left < PAGE_SIZE) { ++ left = 0; ++ } else { ++ left -= PAGE_SIZE; ++ } ++ ++ pages_allocated++; ++ } ++ ++ DBG_MSG(5, ("Alloce for ID:%2d got %d pages, cached: %d\n", descriptor->secure_id, pages_allocated)); ++ ++ if (left) { ++ DBG_MSG(1, ("Failed to allocate needed pages\n")); ++ ++ while (pages_allocated) { ++ pages_allocated--; ++ if (!is_cached) { ++ dma_unmap_page(NULL, descriptor->block_array[pages_allocated].addr, PAGE_SIZE, DMA_BIDIRECTIONAL); ++ } ++ __free_page(pfn_to_page(descriptor->block_array[pages_allocated].addr >> PAGE_SHIFT)); ++ } ++ ++ up(&info->mutex); ++ ++ return 0; /* failure */ ++ } ++ ++ info->num_pages_allocated += pages_allocated; ++ ++ DBG_MSG(6, ("%d out of %d pages now allocated\n", info->num_pages_allocated, info->num_pages_max)); ++ ++ up(&info->mutex); ++ ++ return 1; /* success*/ ++} ++ ++ ++/* ++ * Free specified UMP memory ++ */ ++static void os_free(void *ctx, ump_dd_mem *descriptor) ++{ ++ os_allocator *info; ++ int i; ++ ++ BUG_ON(!ctx); ++ BUG_ON(!descriptor); ++ ++ info = (os_allocator *)ctx; ++ ++ BUG_ON(descriptor->nr_blocks > info->num_pages_allocated); ++ ++ if (down_interruptible(&info->mutex)) { ++ DBG_MSG(1, ("Failed to get mutex in os_free\n")); ++ return; ++ } ++ ++ DBG_MSG(5, ("Releasing %lu OS pages\n", descriptor->nr_blocks)); ++ ++ info->num_pages_allocated -= descriptor->nr_blocks; ++ ++ up(&info->mutex); ++ ++ for (i = 0; i < descriptor->nr_blocks; i++) { ++ DBG_MSG(6, ("Freeing physical page. Address: 0x%08lx\n", descriptor->block_array[i].addr)); ++ if (! descriptor->is_cached) { ++ dma_unmap_page(NULL, descriptor->block_array[i].addr, PAGE_SIZE, DMA_BIDIRECTIONAL); ++ } ++ __free_page(pfn_to_page(descriptor->block_array[i].addr >> PAGE_SHIFT)); ++ } ++ ++ vfree(descriptor->block_array); ++} ++ ++ ++static u32 os_stat(struct ump_memory_backend *backend) ++{ ++ os_allocator *info; ++ info = (os_allocator *)backend->ctx; ++ return info->num_pages_allocated * _MALI_OSK_MALI_PAGE_SIZE; ++} +diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_os.h b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_os.h +new file mode 100755 +index 000000000000..d21d503512ec +--- /dev/null ++++ b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_os.h +@@ -0,0 +1,23 @@ ++/* ++ * Copyright (C) 2010, 2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file ump_kernel_memory_backend_os.h ++ */ ++ ++#ifndef __UMP_KERNEL_MEMORY_BACKEND_OS_H__ ++#define __UMP_KERNEL_MEMORY_BACKEND_OS_H__ ++ ++#include "ump_kernel_memory_backend.h" ++ ++ump_memory_backend *ump_os_memory_backend_create(const int max_allocation); ++ ++#endif /* __UMP_KERNEL_MEMORY_BACKEND_OS_H__ */ ++ +diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_kernel_random_mapping.c b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_random_mapping.c +new file mode 100755 +index 000000000000..6be0f86440de +--- /dev/null ++++ b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_random_mapping.c +@@ -0,0 +1,222 @@ ++/* ++ * Copyright (C) 2010-2011, 2013-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include "mali_kernel_common.h" ++#include "mali_osk.h" ++#include "ump_osk.h" ++#include "ump_kernel_common.h" ++#include "ump_kernel_types.h" ++#include "ump_kernel_random_mapping.h" ++ ++#include ++#include ++#include ++#include ++ ++ ++static ump_dd_mem *search(struct rb_root *root, int id) ++{ ++ struct rb_node *node = root->rb_node; ++ ++ while (node) { ++ ump_dd_mem *e = container_of(node, ump_dd_mem, node); ++ ++ if (id < e->secure_id) { ++ node = node->rb_left; ++ } else if (id > e->secure_id) { ++ node = node->rb_right; ++ } else { ++ return e; ++ } ++ } ++ ++ return NULL; ++} ++ ++static mali_bool insert(struct rb_root *root, int id, ump_dd_mem *mem) ++{ ++ struct rb_node **new = &(root->rb_node); ++ struct rb_node *parent = NULL; ++ ++ while (*new) { ++ ump_dd_mem *this = container_of(*new, ump_dd_mem, node); ++ ++ parent = *new; ++ if (id < this->secure_id) { ++ new = &((*new)->rb_left); ++ } else if (id > this->secure_id) { ++ new = &((*new)->rb_right); ++ } else { ++ printk(KERN_ERR "UMP: ID already used %x\n", id); ++ return MALI_FALSE; ++ } ++ } ++ ++ rb_link_node(&mem->node, parent, new); ++ rb_insert_color(&mem->node, root); ++ ++ return MALI_TRUE; ++} ++ ++ ++ump_random_mapping *ump_random_mapping_create(void) ++{ ++ ump_random_mapping *map = _mali_osk_calloc(1, sizeof(ump_random_mapping)); ++ ++ if (NULL == map) ++ return NULL; ++ ++ map->lock = _mali_osk_mutex_rw_init(_MALI_OSK_LOCKFLAG_ORDERED, ++ _MALI_OSK_LOCK_ORDER_DESCRIPTOR_MAP); ++ if (NULL != map->lock) { ++ map->root = RB_ROOT; ++#if UMP_RANDOM_MAP_DELAY ++ map->failed.count = 0; ++ map->failed.timestamp = jiffies; ++#endif ++ return map; ++ } ++ return NULL; ++} ++ ++void ump_random_mapping_destroy(ump_random_mapping *map) ++{ ++ _mali_osk_mutex_rw_term(map->lock); ++ _mali_osk_free(map); ++} ++ ++int ump_random_mapping_insert(ump_random_mapping *map, ump_dd_mem *mem) ++{ ++ _mali_osk_mutex_rw_wait(map->lock, _MALI_OSK_LOCKMODE_RW); ++ ++ while (1) { ++ u32 id; ++ ++ get_random_bytes(&id, sizeof(id)); ++ ++ /* Try a new random number if id happened to be the invalid ++ * secure ID (-1). */ ++ if (unlikely(id == UMP_INVALID_SECURE_ID)) ++ continue; ++ ++ /* Insert into the tree. If the id was already in use, get a ++ * new random id and try again. */ ++ if (insert(&map->root, id, mem)) { ++ mem->secure_id = id; ++ break; ++ } ++ } ++ _mali_osk_mutex_rw_signal(map->lock, _MALI_OSK_LOCKMODE_RW); ++ ++ return 0; ++} ++ ++ump_dd_mem *ump_random_mapping_get(ump_random_mapping *map, int id) ++{ ++ ump_dd_mem *mem = NULL; ++#if UMP_RANDOM_MAP_DELAY ++ int do_delay = 0; ++#endif ++ ++ DEBUG_ASSERT(map); ++ ++ _mali_osk_mutex_rw_wait(map->lock, _MALI_OSK_LOCKMODE_RO); ++ mem = search(&map->root, id); ++ ++ if (unlikely(NULL == mem)) { ++#if UMP_RANDOM_MAP_DELAY ++ map->failed.count++; ++ ++ if (time_is_before_jiffies(map->failed.timestamp + ++ UMP_FAILED_LOOKUP_DELAY * HZ)) { ++ /* If it is a long time since last failure, reset ++ * the counter and skip the delay this time. */ ++ map->failed.count = 0; ++ } else if (map->failed.count > UMP_FAILED_LOOKUPS_ALLOWED) { ++ do_delay = 1; ++ } ++ ++ map->failed.timestamp = jiffies; ++#endif /* UMP_RANDOM_MAP_DELAY */ ++ } else { ++ ump_dd_reference_add(mem); ++ } ++ _mali_osk_mutex_rw_signal(map->lock, _MALI_OSK_LOCKMODE_RO); ++ ++#if UMP_RANDOM_MAP_DELAY ++ if (do_delay) { ++ /* Apply delay */ ++ schedule_timeout_killable(UMP_FAILED_LOOKUP_DELAY); ++ } ++#endif /* UMP_RANDOM_MAP_DELAY */ ++ ++ return mem; ++} ++ ++static ump_dd_mem *ump_random_mapping_remove_internal(ump_random_mapping *map, int id) ++{ ++ ump_dd_mem *mem = NULL; ++ ++ mem = search(&map->root, id); ++ ++ if (mem) { ++ rb_erase(&mem->node, &map->root); ++ } ++ ++ return mem; ++} ++ ++void ump_random_mapping_put(ump_dd_mem *mem) ++{ ++ int new_ref; ++ ++ _mali_osk_mutex_rw_wait(device.secure_id_map->lock, _MALI_OSK_LOCKMODE_RW); ++ ++ new_ref = _ump_osk_atomic_dec_and_read(&mem->ref_count); ++ DBG_MSG(5, ("Memory reference decremented. ID: %u, new value: %d\n", ++ mem->secure_id, new_ref)); ++ ++ if (0 == new_ref) { ++ DBG_MSG(3, ("Final release of memory. ID: %u\n", mem->secure_id)); ++ ++#ifdef CONFIG_DMA_SHARED_BUFFER ++ if (mem->import_attach) { ++ struct dma_buf_attachment *attach = mem->import_attach; ++ struct dma_buf *dma_buf; ++ ++ if (mem->sgt) ++ dma_buf_unmap_attachment(attach, mem->sgt, ++ DMA_BIDIRECTIONAL); ++ ++ dma_buf = attach->dmabuf; ++ dma_buf_detach(attach->dmabuf, attach); ++ dma_buf_put(dma_buf); ++ ++ } ++#endif ++ ump_random_mapping_remove_internal(device.secure_id_map, mem->secure_id); ++ ++ mem->release_func(mem->ctx, mem); ++ _mali_osk_free(mem); ++ } ++ ++ _mali_osk_mutex_rw_signal(device.secure_id_map->lock, _MALI_OSK_LOCKMODE_RW); ++} ++ ++ump_dd_mem *ump_random_mapping_remove(ump_random_mapping *map, int descriptor) ++{ ++ ump_dd_mem *mem; ++ ++ _mali_osk_mutex_rw_wait(map->lock, _MALI_OSK_LOCKMODE_RW); ++ mem = ump_random_mapping_remove_internal(map, descriptor); ++ _mali_osk_mutex_rw_signal(map->lock, _MALI_OSK_LOCKMODE_RW); ++ ++ return mem; ++} +diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_kernel_random_mapping.h b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_random_mapping.h +new file mode 100755 +index 000000000000..2cea6cedc380 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_random_mapping.h +@@ -0,0 +1,84 @@ ++/* ++ * Copyright (C) 2010-2011, 2013-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file ump_kernel_random_mapping.h ++ */ ++ ++#ifndef __UMP_KERNEL_RANDOM_MAPPING_H__ ++#define __UMP_KERNEL_RANDOM_MAPPING_H__ ++ ++#include "mali_osk.h" ++#include ++ ++#define UMP_RANDOM_MAP_DELAY 1 ++#define UMP_FAILED_LOOKUP_DELAY 10 /* ms */ ++#define UMP_FAILED_LOOKUPS_ALLOWED 10 /* number of allowed failed lookups */ ++ ++/** ++ * The random mapping object ++ * Provides a separate namespace where we can map an integer to a pointer ++ */ ++typedef struct ump_random_mapping { ++ _mali_osk_mutex_rw_t *lock; /**< Lock protecting access to the mapping object */ ++ struct rb_root root; ++#if UMP_RANDOM_MAP_DELAY ++ struct { ++ unsigned long count; ++ unsigned long timestamp; ++ } failed; ++#endif ++} ump_random_mapping; ++ ++/** ++ * Create a random mapping object ++ * Create a random mapping capable of holding 2^20 entries ++ * @return Pointer to a random mapping object, NULL on failure ++ */ ++ump_random_mapping *ump_random_mapping_create(void); ++ ++/** ++ * Destroy a random mapping object ++ * @param map The map to free ++ */ ++void ump_random_mapping_destroy(ump_random_mapping *map); ++ ++/** ++ * Allocate a new mapping entry (random ID) ++ * Allocates a new entry in the map. ++ * @param map The map to allocate a new entry in ++ * @param target The value to map to ++ * @return The random allocated, a negative value on error ++ */ ++int ump_random_mapping_insert(ump_random_mapping *map, ump_dd_mem *mem); ++ ++/** ++ * Get the value mapped to by a random ID ++ * ++ * If the lookup fails, punish the calling thread by applying a delay. ++ * ++ * @param map The map to lookup the random id in ++ * @param id The ID to lookup ++ * @param target Pointer to a pointer which will receive the stored value ++ * @return ump_dd_mem pointer on successful lookup, NULL on error ++ */ ++ump_dd_mem *ump_random_mapping_get(ump_random_mapping *map, int id); ++ ++void ump_random_mapping_put(ump_dd_mem *mem); ++ ++/** ++ * Free the random ID ++ * For the random to be reused it has to be freed ++ * @param map The map to free the random from ++ * @param id The ID to free ++ */ ++ump_dd_mem *ump_random_mapping_remove(ump_random_mapping *map, int id); ++ ++#endif /* __UMP_KERNEL_RANDOM_MAPPING_H__ */ +diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_memory_backend.c b/drivers/gpu/arm/mali400/ump/linux/ump_memory_backend.c +new file mode 100755 +index 000000000000..e41931e1ea75 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/ump/linux/ump_memory_backend.c +@@ -0,0 +1,65 @@ ++/* ++ * Copyright (C) 2010, 2013-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include /* kernel module definitions */ ++#include /* request_mem_region */ ++ ++#include "arch/config.h" /* Configuration for current platform. The symlink for arch is set by Makefile */ ++ ++#include "ump_osk.h" ++#include "ump_kernel_common.h" ++#include "ump_kernel_memory_backend_os.h" ++#include "ump_kernel_memory_backend_dedicated.h" ++ ++/* Configure which dynamic memory allocator to use */ ++int ump_backend = ARCH_UMP_BACKEND_DEFAULT; ++module_param(ump_backend, int, S_IRUGO); /* r--r--r-- */ ++MODULE_PARM_DESC(ump_backend, "0 = dedicated memory backend (default), 1 = OS memory backend"); ++ ++/* The base address of the memory block for the dedicated memory backend */ ++unsigned int ump_memory_address = ARCH_UMP_MEMORY_ADDRESS_DEFAULT; ++module_param(ump_memory_address, uint, S_IRUGO); /* r--r--r-- */ ++MODULE_PARM_DESC(ump_memory_address, "The physical address to map for the dedicated memory backend"); ++ ++/* The size of the memory block for the dedicated memory backend */ ++unsigned int ump_memory_size = ARCH_UMP_MEMORY_SIZE_DEFAULT; ++module_param(ump_memory_size, uint, S_IRUGO); /* r--r--r-- */ ++MODULE_PARM_DESC(ump_memory_size, "The size of fixed memory to map in the dedicated memory backend"); ++ ++ump_memory_backend *ump_memory_backend_create(void) ++{ ++ ump_memory_backend *backend = NULL; ++ ++ /* Create the dynamic memory allocator backend */ ++ if (0 == ump_backend) { ++ DBG_MSG(2, ("Using dedicated memory backend\n")); ++ ++ DBG_MSG(2, ("Requesting dedicated memory: 0x%08x, size: %u\n", ump_memory_address, ump_memory_size)); ++ /* Ask the OS if we can use the specified physical memory */ ++ if (NULL == request_mem_region(ump_memory_address, ump_memory_size, "UMP Memory")) { ++ MSG_ERR(("Failed to request memory region (0x%08X - 0x%08X). Is Mali DD already loaded?\n", ump_memory_address, ump_memory_address + ump_memory_size - 1)); ++ return NULL; ++ } ++ backend = ump_block_allocator_create(ump_memory_address, ump_memory_size); ++ } else if (1 == ump_backend) { ++ DBG_MSG(2, ("Using OS memory backend, allocation limit: %d\n", ump_memory_size)); ++ backend = ump_os_memory_backend_create(ump_memory_size); ++ } ++ ++ return backend; ++} ++ ++void ump_memory_backend_destroy(void) ++{ ++ if (0 == ump_backend) { ++ DBG_MSG(2, ("Releasing dedicated memory: 0x%08x\n", ump_memory_address)); ++ release_mem_region(ump_memory_address, ump_memory_size); ++ } ++} +diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_osk_atomics.c b/drivers/gpu/arm/mali400/ump/linux/ump_osk_atomics.c +new file mode 100755 +index 000000000000..2b634ba79c6e +--- /dev/null ++++ b/drivers/gpu/arm/mali400/ump/linux/ump_osk_atomics.c +@@ -0,0 +1,27 @@ ++/* ++ * Copyright (C) 2010, 2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file ump_osk_atomics.c ++ * Implementation of the OS abstraction layer for the UMP kernel device driver ++ */ ++ ++#include "ump_osk.h" ++#include ++ ++int _ump_osk_atomic_dec_and_read(_mali_osk_atomic_t *atom) ++{ ++ return atomic_dec_return((atomic_t *)&atom->u.val); ++} ++ ++int _ump_osk_atomic_inc_and_read(_mali_osk_atomic_t *atom) ++{ ++ return atomic_inc_return((atomic_t *)&atom->u.val); ++} +diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_osk_low_level_mem.c b/drivers/gpu/arm/mali400/ump/linux/ump_osk_low_level_mem.c +new file mode 100755 +index 000000000000..e08bf25257b0 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/ump/linux/ump_osk_low_level_mem.c +@@ -0,0 +1,314 @@ ++/* ++ * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file ump_osk_memory.c ++ * Implementation of the OS abstraction layer for the kernel device driver ++ */ ++ ++/* needed to detect kernel version specific code */ ++#include ++ ++#include "ump_osk.h" ++#include "ump_uk_types.h" ++#include "ump_ukk.h" ++#include "ump_kernel_common.h" ++#include /* kernel module definitions */ ++#include ++#include ++#include ++ ++#include ++#include /* to verify pointers from user space */ ++#include ++#include ++ ++typedef struct ump_vma_usage_tracker { ++ atomic_t references; ++ ump_memory_allocation *descriptor; ++} ump_vma_usage_tracker; ++ ++static void ump_vma_open(struct vm_area_struct *vma); ++static void ump_vma_close(struct vm_area_struct *vma); ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,26) ++static int ump_cpu_page_fault_handler(struct vm_area_struct *vma, struct vm_fault *vmf); ++#else ++static unsigned long ump_cpu_page_fault_handler(struct vm_area_struct *vma, unsigned long address); ++#endif ++ ++static struct vm_operations_struct ump_vm_ops = { ++ .open = ump_vma_open, ++ .close = ump_vma_close, ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,26) ++ .fault = ump_cpu_page_fault_handler ++#else ++ .nopfn = ump_cpu_page_fault_handler ++#endif ++}; ++ ++/* ++ * Page fault for VMA region ++ * This should never happen since we always map in the entire virtual memory range. ++ */ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,26) ++static int ump_cpu_page_fault_handler(struct vm_area_struct *vma, struct vm_fault *vmf) ++#else ++static unsigned long ump_cpu_page_fault_handler(struct vm_area_struct *vma, unsigned long address) ++#endif ++{ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,26) ++ void __user *address; ++ address = vmf->virtual_address; ++#endif ++ MSG_ERR(("Page-fault in UMP memory region caused by the CPU\n")); ++ MSG_ERR(("VMA: 0x%08lx, virtual address: 0x%08lx\n", (unsigned long)vma, address)); ++ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,26) ++ return VM_FAULT_SIGBUS; ++#else ++ return NOPFN_SIGBUS; ++#endif ++} ++ ++static void ump_vma_open(struct vm_area_struct *vma) ++{ ++ ump_vma_usage_tracker *vma_usage_tracker; ++ int new_val; ++ ++ vma_usage_tracker = (ump_vma_usage_tracker *)vma->vm_private_data; ++ BUG_ON(NULL == vma_usage_tracker); ++ ++ new_val = atomic_inc_return(&vma_usage_tracker->references); ++ ++ DBG_MSG(4, ("VMA open, VMA reference count incremented. VMA: 0x%08lx, reference count: %d\n", (unsigned long)vma, new_val)); ++} ++ ++static void ump_vma_close(struct vm_area_struct *vma) ++{ ++ ump_vma_usage_tracker *vma_usage_tracker; ++ _ump_uk_unmap_mem_s args; ++ int new_val; ++ ++ vma_usage_tracker = (ump_vma_usage_tracker *)vma->vm_private_data; ++ BUG_ON(NULL == vma_usage_tracker); ++ ++ new_val = atomic_dec_return(&vma_usage_tracker->references); ++ ++ DBG_MSG(4, ("VMA close, VMA reference count decremented. VMA: 0x%08lx, reference count: %d\n", (unsigned long)vma, new_val)); ++ ++ if (0 == new_val) { ++ ump_memory_allocation *descriptor; ++ ++ descriptor = vma_usage_tracker->descriptor; ++ ++ args.ctx = descriptor->ump_session; ++ args.cookie = descriptor->cookie; ++ args.mapping = descriptor->mapping; ++ args.size = descriptor->size; ++ ++ args._ukk_private = NULL; /** @note unused */ ++ ++ DBG_MSG(4, ("No more VMA references left, releasing UMP memory\n")); ++ _ump_ukk_unmap_mem(& args); ++ ++ /* vma_usage_tracker is free()d by _ump_osk_mem_mapregion_term() */ ++ } ++} ++ ++_mali_osk_errcode_t _ump_osk_mem_mapregion_init(ump_memory_allocation *descriptor) ++{ ++ ump_vma_usage_tracker *vma_usage_tracker; ++ struct vm_area_struct *vma; ++ ++ if (NULL == descriptor) return _MALI_OSK_ERR_FAULT; ++ ++ vma_usage_tracker = kmalloc(sizeof(ump_vma_usage_tracker), GFP_KERNEL); ++ if (NULL == vma_usage_tracker) { ++ DBG_MSG(1, ("Failed to allocate memory for ump_vma_usage_tracker in _mali_osk_mem_mapregion_init\n")); ++ return -_MALI_OSK_ERR_FAULT; ++ } ++ ++ vma = (struct vm_area_struct *)descriptor->process_mapping_info; ++ if (NULL == vma) { ++ kfree(vma_usage_tracker); ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ vma->vm_private_data = vma_usage_tracker; ++ vma->vm_flags |= VM_IO; ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3,7,0) ++ vma->vm_flags |= VM_RESERVED; ++#else ++ vma->vm_flags |= VM_DONTDUMP; ++ vma->vm_flags |= VM_DONTEXPAND; ++ vma->vm_flags |= VM_PFNMAP; ++#endif ++ ++ ++ if (0 == descriptor->is_cached) { ++ vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); ++ } ++ DBG_MSG(3, ("Mapping with page_prot: 0x%x\n", vma->vm_page_prot)); ++ ++ /* Setup the functions which handle further VMA handling */ ++ vma->vm_ops = &ump_vm_ops; ++ ++ /* Do the va range allocation - in this case, it was done earlier, so we copy in that information */ ++ descriptor->mapping = (void __user *)vma->vm_start; ++ ++ atomic_set(&vma_usage_tracker->references, 1); /*this can later be increased if process is forked, see ump_vma_open() */ ++ vma_usage_tracker->descriptor = descriptor; ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++void _ump_osk_mem_mapregion_term(ump_memory_allocation *descriptor) ++{ ++ struct vm_area_struct *vma; ++ ump_vma_usage_tracker *vma_usage_tracker; ++ ++ if (NULL == descriptor) return; ++ ++ /* Linux does the right thing as part of munmap to remove the mapping ++ * All that remains is that we remove the vma_usage_tracker setup in init() */ ++ vma = (struct vm_area_struct *)descriptor->process_mapping_info; ++ ++ vma_usage_tracker = vma->vm_private_data; ++ ++ /* We only get called if mem_mapregion_init succeeded */ ++ kfree(vma_usage_tracker); ++ return; ++} ++ ++_mali_osk_errcode_t _ump_osk_mem_mapregion_map(ump_memory_allocation *descriptor, u32 offset, u32 *phys_addr, unsigned long size) ++{ ++ struct vm_area_struct *vma; ++ _mali_osk_errcode_t retval; ++ ++ if (NULL == descriptor) return _MALI_OSK_ERR_FAULT; ++ ++ vma = (struct vm_area_struct *)descriptor->process_mapping_info; ++ ++ if (NULL == vma) return _MALI_OSK_ERR_FAULT; ++ ++ retval = remap_pfn_range(vma, ((u32)descriptor->mapping) + offset, (*phys_addr) >> PAGE_SHIFT, size, vma->vm_page_prot) ? _MALI_OSK_ERR_FAULT : _MALI_OSK_ERR_OK;; ++ ++ DBG_MSG(4, ("Mapping virtual to physical memory. ID: %u, vma: 0x%08lx, virtual addr:0x%08lx, physical addr: 0x%08lx, size:%lu, prot:0x%x, vm_flags:0x%x RETVAL: 0x%x\n", ++ ump_dd_secure_id_get(descriptor->handle), ++ (unsigned long)vma, ++ (unsigned long)(vma->vm_start + offset), ++ (unsigned long)*phys_addr, ++ size, ++ (unsigned int)vma->vm_page_prot, vma->vm_flags, retval)); ++ ++ return retval; ++} ++ ++static void level1_cache_flush_all(void) ++{ ++ DBG_MSG(4, ("UMP[xx] Flushing complete L1 cache\n")); ++ __cpuc_flush_kern_all(); ++} ++ ++void _ump_osk_msync(ump_dd_mem *mem, void *virt, u32 offset, u32 size, ump_uk_msync_op op, ump_session_data *session_data) ++{ ++ int i; ++ ++ /* Flush L1 using virtual address, the entire range in one go. ++ * Only flush if user space process has a valid write mapping on given address. */ ++ if ((mem) && (virt != NULL) && (access_ok(virt, size))) { ++ __cpuc_flush_dcache_area(virt, size); ++ DBG_MSG(3, ("UMP[%02u] Flushing CPU L1 Cache. CPU address: %x, size: %x\n", mem->secure_id, virt, size)); ++ } else { ++ if (session_data) { ++ if (op == _UMP_UK_MSYNC_FLUSH_L1) { ++ DBG_MSG(4, ("UMP Pending L1 cache flushes: %d\n", session_data->has_pending_level1_cache_flush)); ++ session_data->has_pending_level1_cache_flush = 0; ++ level1_cache_flush_all(); ++ return; ++ } else { ++ if (session_data->cache_operations_ongoing) { ++ session_data->has_pending_level1_cache_flush++; ++ DBG_MSG(4, ("UMP[%02u] Defering the L1 flush. Nr pending:%d\n", mem->secure_id, session_data->has_pending_level1_cache_flush)); ++ } else { ++ /* Flushing the L1 cache for each switch_user() if ump_cache_operations_control(START) is not called */ ++ level1_cache_flush_all(); ++ } ++ } ++ } else { ++ DBG_MSG(4, ("Unkown state %s %d\n", __FUNCTION__, __LINE__)); ++ level1_cache_flush_all(); ++ } ++ } ++ ++ if (NULL == mem) return; ++ ++ if (mem->size_bytes == size) { ++ DBG_MSG(3, ("UMP[%02u] Flushing CPU L2 Cache\n", mem->secure_id)); ++ } else { ++ DBG_MSG(3, ("UMP[%02u] Flushing CPU L2 Cache. Blocks:%u, TotalSize:%u. FlushSize:%u Offset:0x%x FirstPaddr:0x%08x\n", ++ mem->secure_id, mem->nr_blocks, mem->size_bytes, size, offset, mem->block_array[0].addr)); ++ } ++ ++ ++ /* Flush L2 using physical addresses, block for block. */ ++ for (i = 0 ; i < mem->nr_blocks; i++) { ++ u32 start_p, end_p; ++ ump_dd_physical_block *block; ++ block = &mem->block_array[i]; ++ ++ if (offset >= block->size) { ++ offset -= block->size; ++ continue; ++ } ++ ++ if (offset) { ++ start_p = (u32)block->addr + offset; ++ /* We'll zero the offset later, after using it to calculate end_p. */ ++ } else { ++ start_p = (u32)block->addr; ++ } ++ ++ if (size < block->size - offset) { ++ end_p = start_p + size; ++ size = 0; ++ } else { ++ if (offset) { ++ end_p = start_p + (block->size - offset); ++ size -= block->size - offset; ++ offset = 0; ++ } else { ++ end_p = start_p + block->size; ++ size -= block->size; ++ } ++ } ++ ++ switch (op) { ++ case _UMP_UK_MSYNC_CLEAN: ++ outer_clean_range(start_p, end_p); ++ break; ++ case _UMP_UK_MSYNC_CLEAN_AND_INVALIDATE: ++ outer_flush_range(start_p, end_p); ++ break; ++ case _UMP_UK_MSYNC_INVALIDATE: ++ outer_inv_range(start_p, end_p); ++ break; ++ default: ++ break; ++ } ++ ++ if (0 == size) { ++ /* Nothing left to flush. */ ++ break; ++ } ++ } ++ ++ return; ++} +diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_osk_misc.c b/drivers/gpu/arm/mali400/ump/linux/ump_osk_misc.c +new file mode 100755 +index 000000000000..58c9f1bf27b8 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/ump/linux/ump_osk_misc.c +@@ -0,0 +1,36 @@ ++/* ++ * Copyright (C) 2010, 2013-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file ump_osk_misc.c ++ * Implementation of the OS abstraction layer for the UMP kernel device driver ++ */ ++ ++ ++#include "ump_osk.h" ++ ++#include ++#include "ump_kernel_linux.h" ++ ++/* is called from ump_kernel_constructor in common code */ ++_mali_osk_errcode_t _ump_osk_init(void) ++{ ++ if (0 != ump_kernel_device_initialize()) { ++ return _MALI_OSK_ERR_FAULT; ++ } ++ ++ return _MALI_OSK_ERR_OK; ++} ++ ++_mali_osk_errcode_t _ump_osk_term(void) ++{ ++ ump_kernel_device_terminate(); ++ return _MALI_OSK_ERR_OK; ++} +diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_ukk_ref_wrappers.c b/drivers/gpu/arm/mali400/ump/linux/ump_ukk_ref_wrappers.c +new file mode 100755 +index 000000000000..56a787ff64dc +--- /dev/null ++++ b/drivers/gpu/arm/mali400/ump/linux/ump_ukk_ref_wrappers.c +@@ -0,0 +1,230 @@ ++/* ++ * Copyright (C) 2010, 2013-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file ump_ukk_wrappers.c ++ * Defines the wrapper functions which turn Linux IOCTL calls into _ukk_ calls for the reference implementation ++ */ ++ ++ ++#include /* user space access */ ++ ++#include "ump_osk.h" ++#include "ump_uk_types.h" ++#include "ump_ukk.h" ++#include "ump_kernel_common.h" ++#include ++#include "ump_kernel_interface_ref_drv.h" ++#include "mali_osk_list.h" ++ ++extern struct device *ump_global_mdev; ++ ++/* ++ * IOCTL operation; Allocate UMP memory ++ */ ++int ump_allocate_wrapper(u32 __user *argument, struct ump_session_data *session_data) ++{ ++ _ump_uk_allocate_s user_interaction; ++ _mali_osk_errcode_t err; ++ ++ /* Sanity check input parameters */ ++ if (NULL == argument || NULL == session_data) { ++ MSG_ERR(("NULL parameter in ump_ioctl_allocate()\n")); ++ return -ENOTTY; ++ } ++ ++ /* Copy the user space memory to kernel space (so we safely can read it) */ ++ if (0 != copy_from_user(&user_interaction, argument, sizeof(user_interaction))) { ++ MSG_ERR(("copy_from_user() in ump_ioctl_allocate()\n")); ++ return -EFAULT; ++ } ++ ++ user_interaction.ctx = (void *) session_data; ++ ++ err = _ump_ukk_allocate(&user_interaction); ++ if (_MALI_OSK_ERR_OK != err) { ++ DBG_MSG(1, ("_ump_ukk_allocate() failed in ump_ioctl_allocate()\n")); ++ return ump_map_errcode(err); ++ } ++ user_interaction.ctx = NULL; ++ ++ if (0 != copy_to_user(argument, &user_interaction, sizeof(user_interaction))) { ++ /* If the copy fails then we should release the memory. We can use the IOCTL release to accomplish this */ ++ _ump_uk_release_s release_args; ++ ++ MSG_ERR(("copy_to_user() failed in ump_ioctl_allocate()\n")); ++ ++ release_args.ctx = (void *) session_data; ++ release_args.secure_id = user_interaction.secure_id; ++ ++ err = _ump_ukk_release(&release_args); ++ if (_MALI_OSK_ERR_OK != err) { ++ MSG_ERR(("_ump_ukk_release() also failed when trying to release newly allocated memory in ump_ioctl_allocate()\n")); ++ } ++ ++ return -EFAULT; ++ } ++ ++ return 0; /* success */ ++} ++ ++#ifdef CONFIG_DMA_SHARED_BUFFER ++static ump_dd_handle get_ump_handle_from_dmabuf(struct ump_session_data *session_data, ++ struct dma_buf *dmabuf) ++{ ++ ump_session_memory_list_element *session_mem, *tmp; ++ struct dma_buf_attachment *attach; ++ ump_dd_handle ump_handle; ++ ++ DEBUG_ASSERT_POINTER(session_data); ++ ++ _mali_osk_mutex_wait(session_data->lock); ++ ++ _MALI_OSK_LIST_FOREACHENTRY(session_mem, tmp, ++ &session_data->list_head_session_memory_list, ++ ump_session_memory_list_element, list) { ++ if (session_mem->mem->import_attach) { ++ attach = session_mem->mem->import_attach; ++ if (attach->dmabuf == dmabuf) { ++ _mali_osk_mutex_signal(session_data->lock); ++ ump_handle = (ump_dd_handle)session_mem->mem; ++ ump_random_mapping_get(device.secure_id_map, ump_dd_secure_id_get(ump_handle)); ++ return ump_handle; ++ } ++ } ++ } ++ ++ _mali_osk_mutex_signal(session_data->lock); ++ ++ return NULL; ++} ++ ++int ump_dmabuf_import_wrapper(u32 __user *argument, ++ struct ump_session_data *session_data) ++{ ++ ump_session_memory_list_element *session = NULL; ++ _ump_uk_dmabuf_s ump_dmabuf; ++ ump_dd_handle ump_handle; ++ ump_dd_physical_block *blocks = NULL; ++ struct dma_buf_attachment *attach = NULL; ++ struct dma_buf *dma_buf; ++ struct sg_table *sgt = NULL; ++ struct scatterlist *sgl; ++ unsigned int i = 0; ++ int ret = 0; ++ ++ /* Sanity check input parameters */ ++ if (!argument || !session_data) { ++ MSG_ERR(("NULL parameter.\n")); ++ return -EINVAL; ++ } ++ ++ if (copy_from_user(&ump_dmabuf, argument, ++ sizeof(_ump_uk_dmabuf_s))) { ++ MSG_ERR(("copy_from_user() failed.\n")); ++ return -EFAULT; ++ } ++ ++ dma_buf = dma_buf_get(ump_dmabuf.fd); ++ if (IS_ERR(dma_buf)) ++ return PTR_ERR(dma_buf); ++ ++ /* ++ * if already imported then increase a refcount to the ump descriptor ++ * and call dma_buf_put() and then go to found to return previous ++ * ump secure id. ++ */ ++ ump_handle = get_ump_handle_from_dmabuf(session_data, dma_buf); ++ if (ump_handle) { ++ dma_buf_put(dma_buf); ++ goto found; ++ } ++ ++ attach = dma_buf_attach(dma_buf, ump_global_mdev); ++ if (IS_ERR(attach)) { ++ ret = PTR_ERR(attach); ++ goto err_dma_buf_put; ++ } ++ ++ sgt = dma_buf_map_attachment(attach, DMA_BIDIRECTIONAL); ++ if (IS_ERR(sgt)) { ++ ret = PTR_ERR(sgt); ++ goto err_dma_buf_detach; ++ } ++ ++ blocks = (ump_dd_physical_block *)_mali_osk_malloc(sizeof(ump_dd_physical_block) * sgt->nents); ++ if (!blocks) { ++ DBG_MSG(1, ("Failed to allocate blocks.\n")); ++ ret = -EFAULT; ++ goto err_dma_buf_unmap; ++ } ++ for_each_sg(sgt->sgl, sgl, sgt->nents, i) { ++ blocks[i].addr = sg_phys(sgl); ++ blocks[i].size = sg_dma_len(sgl); ++ } ++ ++ /* ++ * Initialize the session memory list element, and add it ++ * to the session object ++ */ ++ session = _mali_osk_calloc(1, sizeof(*session)); ++ if (!session) { ++ DBG_MSG(1, ("Failed to allocate session.\n")); ++ ret = -EFAULT; ++ goto err_free_block; ++ } ++ ++ ump_handle = ump_dd_handle_create_from_phys_blocks(blocks, i); ++ if (UMP_DD_HANDLE_INVALID == ump_handle) { ++ DBG_MSG(1, ("Failed to create ump handle.\n")); ++ ret = -EFAULT; ++ goto err_free_session; ++ } ++ ++ session->mem = (ump_dd_mem *)ump_handle; ++ session->mem->import_attach = attach; ++ session->mem->sgt = sgt; ++ ++ _mali_osk_mutex_wait(session_data->lock); ++ _mali_osk_list_add(&(session->list), ++ &(session_data->list_head_session_memory_list)); ++ _mali_osk_mutex_signal(session_data->lock); ++ ++ _mali_osk_free(blocks); ++ ++found: ++ ump_dmabuf.ctx = (void *)session_data; ++ ump_dmabuf.secure_id = ump_dd_secure_id_get(ump_handle); ++ ump_dmabuf.size = ump_dd_size_get(ump_handle); ++ ++ if (copy_to_user(argument, &ump_dmabuf, ++ sizeof(_ump_uk_dmabuf_s))) { ++ MSG_ERR(("copy_to_user() failed.\n")); ++ ret = -EFAULT; ++ goto err_release_ump_handle; ++ } ++ ++ return ret; ++ ++err_release_ump_handle: ++ ump_dd_reference_release(ump_handle); ++err_free_session: ++ _mali_osk_free(session); ++err_free_block: ++ _mali_osk_free(blocks); ++err_dma_buf_unmap: ++ dma_buf_unmap_attachment(attach, sgt, DMA_BIDIRECTIONAL); ++err_dma_buf_detach: ++ dma_buf_detach(dma_buf, attach); ++err_dma_buf_put: ++ dma_buf_put(dma_buf); ++ return ret; ++} ++#endif +diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_ukk_ref_wrappers.h b/drivers/gpu/arm/mali400/ump/linux/ump_ukk_ref_wrappers.h +new file mode 100755 +index 000000000000..61a7095a6920 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/ump/linux/ump_ukk_ref_wrappers.h +@@ -0,0 +1,36 @@ ++/* ++ * Copyright (C) 2010, 2013-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file ump_ukk_wrappers.h ++ * Defines the wrapper functions which turn Linux IOCTL calls into _ukk_ calls for the reference implementation ++ */ ++ ++#ifndef __UMP_UKK_REF_WRAPPERS_H__ ++#define __UMP_UKK_REF_WRAPPERS_H__ ++ ++#include ++#include "ump_kernel_common.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++ ++int ump_allocate_wrapper(u32 __user *argument, struct ump_session_data *session_data); ++#ifdef CONFIG_DMA_SHARED_BUFFER ++int ump_dmabuf_import_wrapper(u32 __user *argument, struct ump_session_data *session_data); ++#endif ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif /* __UMP_UKK_REF_WRAPPERS_H__ */ +diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_ukk_wrappers.c b/drivers/gpu/arm/mali400/ump/linux/ump_ukk_wrappers.c +new file mode 100755 +index 000000000000..4d6b69608fd4 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/ump/linux/ump_ukk_wrappers.c +@@ -0,0 +1,280 @@ ++/* ++ * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file ump_ukk_wrappers.c ++ * Defines the wrapper functions which turn Linux IOCTL calls into _ukk_ calls ++ */ ++ ++#include /* user space access */ ++ ++#include "ump_osk.h" ++#include "ump_uk_types.h" ++#include "ump_ukk.h" ++#include "ump_kernel_common.h" ++ ++/* ++ * IOCTL operation; Negotiate version of IOCTL API ++ */ ++int ump_get_api_version_wrapper(u32 __user *argument, struct ump_session_data *session_data) ++{ ++ _ump_uk_api_version_s version_info; ++ _mali_osk_errcode_t err; ++ ++ /* Sanity check input parameters */ ++ if (NULL == argument || NULL == session_data) { ++ MSG_ERR(("NULL parameter in ump_ioctl_get_api_version()\n")); ++ return -ENOTTY; ++ } ++ ++ /* Copy the user space memory to kernel space (so we safely can read it) */ ++ if (0 != copy_from_user(&version_info, argument, sizeof(version_info))) { ++ MSG_ERR(("copy_from_user() in ump_ioctl_get_api_version()\n")); ++ return -EFAULT; ++ } ++ ++ version_info.ctx = (void *) session_data; ++ err = _ump_uku_get_api_version(&version_info); ++ if (_MALI_OSK_ERR_OK != err) { ++ MSG_ERR(("_ump_uku_get_api_version() failed in ump_ioctl_get_api_version()\n")); ++ return ump_map_errcode(err); ++ } ++ ++ version_info.ctx = NULL; ++ ++ /* Copy ouput data back to user space */ ++ if (0 != copy_to_user(argument, &version_info, sizeof(version_info))) { ++ MSG_ERR(("copy_to_user() failed in ump_ioctl_get_api_version()\n")); ++ return -EFAULT; ++ } ++ ++ return 0; /* success */ ++} ++ ++ ++/* ++ * IOCTL operation; Release reference to specified UMP memory. ++ */ ++int ump_release_wrapper(u32 __user *argument, struct ump_session_data *session_data) ++{ ++ _ump_uk_release_s release_args; ++ _mali_osk_errcode_t err; ++ ++ /* Sanity check input parameters */ ++ if (NULL == session_data) { ++ MSG_ERR(("NULL parameter in ump_ioctl_release()\n")); ++ return -ENOTTY; ++ } ++ ++ /* Copy the user space memory to kernel space (so we safely can read it) */ ++ if (0 != copy_from_user(&release_args, argument, sizeof(release_args))) { ++ MSG_ERR(("copy_from_user() in ump_ioctl_get_api_version()\n")); ++ return -EFAULT; ++ } ++ ++ release_args.ctx = (void *) session_data; ++ err = _ump_ukk_release(&release_args); ++ if (_MALI_OSK_ERR_OK != err) { ++ MSG_ERR(("_ump_ukk_release() failed in ump_ioctl_release()\n")); ++ return ump_map_errcode(err); ++ } ++ ++ ++ return 0; /* success */ ++} ++ ++/* ++ * IOCTL operation; Return size for specified UMP memory. ++ */ ++int ump_size_get_wrapper(u32 __user *argument, struct ump_session_data *session_data) ++{ ++ _ump_uk_size_get_s user_interaction; ++ _mali_osk_errcode_t err; ++ ++ /* Sanity check input parameters */ ++ if (NULL == argument || NULL == session_data) { ++ MSG_ERR(("NULL parameter in ump_ioctl_size_get()\n")); ++ return -ENOTTY; ++ } ++ ++ if (0 != copy_from_user(&user_interaction, argument, sizeof(user_interaction))) { ++ MSG_ERR(("copy_from_user() in ump_ioctl_size_get()\n")); ++ return -EFAULT; ++ } ++ ++ user_interaction.ctx = (void *) session_data; ++ err = _ump_ukk_size_get(&user_interaction); ++ if (_MALI_OSK_ERR_OK != err) { ++ MSG_ERR(("_ump_ukk_size_get() failed in ump_ioctl_size_get()\n")); ++ return ump_map_errcode(err); ++ } ++ ++ user_interaction.ctx = NULL; ++ ++ if (0 != copy_to_user(argument, &user_interaction, sizeof(user_interaction))) { ++ MSG_ERR(("copy_to_user() failed in ump_ioctl_size_get()\n")); ++ return -EFAULT; ++ } ++ ++ return 0; /* success */ ++} ++ ++/* ++ * IOCTL operation; Do cache maintenance on specified UMP memory. ++ */ ++int ump_msync_wrapper(u32 __user *argument, struct ump_session_data *session_data) ++{ ++ _ump_uk_msync_s user_interaction; ++ ++ /* Sanity check input parameters */ ++ if (NULL == argument || NULL == session_data) { ++ MSG_ERR(("NULL parameter in ump_ioctl_size_get()\n")); ++ return -ENOTTY; ++ } ++ ++ if (0 != copy_from_user(&user_interaction, argument, sizeof(user_interaction))) { ++ MSG_ERR(("copy_from_user() in ump_ioctl_msync()\n")); ++ return -EFAULT; ++ } ++ ++ user_interaction.ctx = (void *) session_data; ++ ++ _ump_ukk_msync(&user_interaction); ++ ++ user_interaction.ctx = NULL; ++ ++ if (0 != copy_to_user(argument, &user_interaction, sizeof(user_interaction))) { ++ MSG_ERR(("copy_to_user() failed in ump_ioctl_msync()\n")); ++ return -EFAULT; ++ } ++ ++ return 0; /* success */ ++} ++int ump_cache_operations_control_wrapper(u32 __user *argument, struct ump_session_data *session_data) ++{ ++ _ump_uk_cache_operations_control_s user_interaction; ++ ++ /* Sanity check input parameters */ ++ if (NULL == argument || NULL == session_data) { ++ MSG_ERR(("NULL parameter in ump_ioctl_size_get()\n")); ++ return -ENOTTY; ++ } ++ ++ if (0 != copy_from_user(&user_interaction, argument, sizeof(user_interaction))) { ++ MSG_ERR(("copy_from_user() in ump_ioctl_cache_operations_control()\n")); ++ return -EFAULT; ++ } ++ ++ user_interaction.ctx = (void *) session_data; ++ ++ _ump_ukk_cache_operations_control((_ump_uk_cache_operations_control_s *) &user_interaction); ++ ++ user_interaction.ctx = NULL; ++ ++#if 0 /* No data to copy back */ ++ if (0 != copy_to_user(argument, &user_interaction, sizeof(user_interaction))) { ++ MSG_ERR(("copy_to_user() failed in ump_ioctl_cache_operations_control()\n")); ++ return -EFAULT; ++ } ++#endif ++ return 0; /* success */ ++} ++ ++int ump_switch_hw_usage_wrapper(u32 __user *argument, struct ump_session_data *session_data) ++{ ++ _ump_uk_switch_hw_usage_s user_interaction; ++ ++ /* Sanity check input parameters */ ++ if (NULL == argument || NULL == session_data) { ++ MSG_ERR(("NULL parameter in ump_ioctl_size_get()\n")); ++ return -ENOTTY; ++ } ++ ++ if (0 != copy_from_user(&user_interaction, argument, sizeof(user_interaction))) { ++ MSG_ERR(("copy_from_user() in ump_ioctl_switch_hw_usage()\n")); ++ return -EFAULT; ++ } ++ ++ user_interaction.ctx = (void *) session_data; ++ ++ _ump_ukk_switch_hw_usage(&user_interaction); ++ ++ user_interaction.ctx = NULL; ++ ++#if 0 /* No data to copy back */ ++ if (0 != copy_to_user(argument, &user_interaction, sizeof(user_interaction))) { ++ MSG_ERR(("copy_to_user() failed in ump_ioctl_switch_hw_usage()\n")); ++ return -EFAULT; ++ } ++#endif ++ return 0; /* success */ ++} ++ ++int ump_lock_wrapper(u32 __user *argument, struct ump_session_data *session_data) ++{ ++ _ump_uk_lock_s user_interaction; ++ ++ /* Sanity check input parameters */ ++ if (NULL == argument || NULL == session_data) { ++ MSG_ERR(("NULL parameter in ump_ioctl_size_get()\n")); ++ return -ENOTTY; ++ } ++ ++ if (0 != copy_from_user(&user_interaction, argument, sizeof(user_interaction))) { ++ MSG_ERR(("copy_from_user() in ump_ioctl_switch_hw_usage()\n")); ++ return -EFAULT; ++ } ++ ++ user_interaction.ctx = (void *) session_data; ++ ++ _ump_ukk_lock(&user_interaction); ++ ++ user_interaction.ctx = NULL; ++ ++#if 0 /* No data to copy back */ ++ if (0 != copy_to_user(argument, &user_interaction, sizeof(user_interaction))) { ++ MSG_ERR(("copy_to_user() failed in ump_ioctl_switch_hw_usage()\n")); ++ return -EFAULT; ++ } ++#endif ++ ++ return 0; /* success */ ++} ++ ++int ump_unlock_wrapper(u32 __user *argument, struct ump_session_data *session_data) ++{ ++ _ump_uk_unlock_s user_interaction; ++ ++ /* Sanity check input parameters */ ++ if (NULL == argument || NULL == session_data) { ++ MSG_ERR(("NULL parameter in ump_ioctl_size_get()\n")); ++ return -ENOTTY; ++ } ++ ++ if (0 != copy_from_user(&user_interaction, argument, sizeof(user_interaction))) { ++ MSG_ERR(("copy_from_user() in ump_ioctl_switch_hw_usage()\n")); ++ return -EFAULT; ++ } ++ ++ user_interaction.ctx = (void *) session_data; ++ ++ _ump_ukk_unlock(&user_interaction); ++ ++ user_interaction.ctx = NULL; ++ ++#if 0 /* No data to copy back */ ++ if (0 != copy_to_user(argument, &user_interaction, sizeof(user_interaction))) { ++ MSG_ERR(("copy_to_user() failed in ump_ioctl_switch_hw_usage()\n")); ++ return -EFAULT; ++ } ++#endif ++ ++ return 0; /* success */ ++} +diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_ukk_wrappers.h b/drivers/gpu/arm/mali400/ump/linux/ump_ukk_wrappers.h +new file mode 100755 +index 000000000000..5f8fc683c8f5 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/ump/linux/ump_ukk_wrappers.h +@@ -0,0 +1,46 @@ ++/* ++ * Copyright (C) 2010, 2012-2014, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++/** ++ * @file ump_ukk_wrappers.h ++ * Defines the wrapper functions which turn Linux IOCTL calls into _ukk_ calls ++ */ ++ ++#ifndef __UMP_UKK_WRAPPERS_H__ ++#define __UMP_UKK_WRAPPERS_H__ ++ ++#include ++#include "ump_kernel_common.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++ ++ ++int ump_get_api_version_wrapper(u32 __user *argument, struct ump_session_data *session_data); ++int ump_release_wrapper(u32 __user *argument, struct ump_session_data *session_data); ++int ump_size_get_wrapper(u32 __user *argument, struct ump_session_data *session_data); ++int ump_msync_wrapper(u32 __user *argument, struct ump_session_data *session_data); ++int ump_cache_operations_control_wrapper(u32 __user *argument, struct ump_session_data *session_data); ++int ump_switch_hw_usage_wrapper(u32 __user *argument, struct ump_session_data *session_data); ++int ump_lock_wrapper(u32 __user *argument, struct ump_session_data *session_data); ++int ump_unlock_wrapper(u32 __user *argument, struct ump_session_data *session_data); ++ ++ ++ ++ ++#ifdef __cplusplus ++} ++#endif ++ ++ ++ ++#endif /* __UMP_UKK_WRAPPERS_H__ */ +diff --git a/drivers/gpu/arm/mali400/ump/readme.txt b/drivers/gpu/arm/mali400/ump/readme.txt +new file mode 100755 +index 000000000000..c238cf0f2b1f +--- /dev/null ++++ b/drivers/gpu/arm/mali400/ump/readme.txt +@@ -0,0 +1,28 @@ ++Building the UMP Device Driver for Linux ++---------------------------------------- ++ ++Build the UMP Device Driver for Linux by running the following make command: ++ ++KDIR= CONFIG= BUILD= make ++ ++where ++ kdir_path: Path to your Linux Kernel directory ++ your_config: Name of the sub-folder to find the required config.h file ++ ("arch-" will be prepended) ++ build_option: debug or release. Debug is default. ++ ++The config.h contains following configuration parameters: ++ ++ARCH_UMP_BACKEND_DEFAULT ++ 0 specifies the dedicated memory allocator. ++ 1 specifies the OS memory allocator. ++ARCH_UMP_MEMORY_ADDRESS_DEFAULT ++ This is only required for the dedicated memory allocator, and specifies ++ the physical start address of the memory block reserved for UMP. ++ARCH_UMP_MEMORY_SIZE_DEFAULT ++ This specified the size of the memory block reserved for UMP, or the ++ maximum limit for allocations from the OS. ++ ++The result will be a ump.ko file, which can be loaded into the Linux kernel ++by using the insmod command. The driver can also be built as a part of the ++kernel itself. +diff --git a/drivers/gpu/arm/mali400/umplock/Makefile b/drivers/gpu/arm/mali400/umplock/Makefile +new file mode 100755 +index 000000000000..e5549a33f91d +--- /dev/null ++++ b/drivers/gpu/arm/mali400/umplock/Makefile +@@ -0,0 +1,69 @@ ++# ++# Copyright (C) 2012, 2016-2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++# as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained from Free Software ++# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++# ++ ++# default to building for the host ++ARCH ?= $(shell uname -m) ++ ++# linux build system integration ++ ++ifneq ($(KERNELRELEASE),) ++# Inside the kernel build system ++ ++EXTRA_CFLAGS += -I$(KBUILD_EXTMOD) ++ ++SRC = umplock_driver.c ++ ++MODULE:=umplock.ko ++ ++obj-m := $(MODULE:.ko=.o) ++$(MODULE:.ko=-y) := $(SRC:.c=.o) ++ ++$(MODULE:.ko=-objs) := $(SRC:.c=.o) ++ ++else ++# Outside the kernel build system ++# ++# ++ ++# Get any user defined KDIR- or maybe even a hardcoded KDIR ++-include KDIR_CONFIGURATION ++ ++# Define host system directory ++KDIR-$(shell uname -m):=/lib/modules/$(shell uname -r)/build ++ ++ifeq ($(ARCH), arm) ++ # when compiling for ARM we're cross compiling ++ export CROSS_COMPILE ?= arm-none-linux-gnueabi- ++ CONFIG ?= arm ++else ++ # Compiling for the host ++ CONFIG ?= $(shell uname -m) ++endif ++ ++# default cpu to select ++CPU ?= $(shell uname -m) ++ ++# look up KDIR based om CPU selection ++KDIR ?= $(KDIR-$(CPU)) ++ ++ifeq ($(KDIR),) ++$(error No KDIR found for platform $(CPU)) ++endif ++ ++all: ++ $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) ++ ++kernelrelease: ++ $(MAKE) -C $(KDIR) kernelrelease ++ ++clean: ++ $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) clean ++ ++endif +diff --git a/drivers/gpu/arm/mali400/umplock/umplock_driver.c b/drivers/gpu/arm/mali400/umplock/umplock_driver.c +new file mode 100755 +index 000000000000..173f4d9bb5c7 +--- /dev/null ++++ b/drivers/gpu/arm/mali400/umplock/umplock_driver.c +@@ -0,0 +1,618 @@ ++/* ++ * Copyright (C) 2012-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include "umplock_ioctl.h" ++#include ++ ++#define MAX_ITEMS 1024 ++#define MAX_PIDS 128 ++ ++typedef struct lock_cmd_priv { ++ uint32_t msg[128]; /*ioctl args*/ ++ u32 pid; /*process id*/ ++} _lock_cmd_priv; ++ ++typedef struct lock_ref { ++ int ref_count; ++ u32 pid; ++ u32 down_count; ++} _lock_ref; ++ ++typedef struct umplock_item { ++ u32 secure_id; ++ u32 id_ref_count; ++ u32 owner; ++ _lock_access_usage usage; ++ _lock_ref references[MAX_PIDS]; ++ struct semaphore item_lock; ++} umplock_item; ++ ++typedef struct umplock_device_private { ++ struct mutex item_list_lock; ++ atomic_t sessions; ++ umplock_item items[MAX_ITEMS]; ++ u32 pids[MAX_PIDS]; ++} umplock_device_private; ++ ++struct umplock_device { ++ struct cdev cdev; ++ struct class *umplock_class; ++}; ++ ++static struct umplock_device umplock_device; ++static umplock_device_private device; ++static dev_t umplock_dev; ++static char umplock_dev_name[] = "umplock"; ++ ++int umplock_debug_level = 0; ++module_param(umplock_debug_level, int, S_IRUSR | S_IWUSR | S_IWGRP | S_IRGRP | S_IROTH); /* rw-rw-r-- */ ++MODULE_PARM_DESC(umplock_debug_level, "set umplock_debug_level to print debug messages"); ++ ++#define PDEBUG(level, fmt, args...) do { if ((level) <= umplock_debug_level) printk(KERN_DEBUG "umplock: " fmt, ##args); } while (0) ++#define PERROR(fmt, args...) do { printk(KERN_ERR "umplock: " fmt, ##args); } while (0) ++ ++int umplock_find_item(u32 secure_id) ++{ ++ int i; ++ for (i = 0; i < MAX_ITEMS; i++) { ++ if (device.items[i].secure_id == secure_id) { ++ return i; ++ } ++ } ++ ++ return -1; ++} ++ ++static int umplock_find_item_by_pid(_lock_cmd_priv *lock_cmd, int *item_slot, int *ref_slot) ++{ ++ _lock_item_s *lock_item; ++ int i, j; ++ ++ lock_item = (_lock_item_s *)&lock_cmd->msg; ++ ++ i = umplock_find_item(lock_item->secure_id); ++ ++ if (i < 0) { ++ return -1; ++ } ++ ++ for (j = 0; j < MAX_PIDS; j++) { ++ if (device.items[i].references[j].pid == lock_cmd->pid) { ++ *item_slot = i; ++ *ref_slot = j; ++ return 0; ++ } ++ } ++ return -1 ; ++} ++ ++static int umplock_find_client_valid(u32 pid) ++{ ++ int i; ++ ++ if (pid == 0) { ++ return -1; ++ } ++ ++ for (i = 0; i < MAX_PIDS; i++) { ++ if (device.pids[i] == pid) { ++ return i; ++ } ++ } ++ ++ return -1; ++} ++ ++static int do_umplock_create_locked(_lock_cmd_priv *lock_cmd) ++{ ++ int i_index, ref_index; ++ int ret; ++ _lock_item_s *lock_item = (_lock_item_s *)&lock_cmd->msg; ++ ++ i_index = ref_index = -1; ++ ++ ret = umplock_find_client_valid(lock_cmd->pid); ++ if (ret < 0) { ++ /*lock request from an invalid client pid, do nothing*/ ++ return -EINVAL; ++ } ++ ++ ret = umplock_find_item_by_pid(lock_cmd, &i_index, &ref_index); ++ if (ret >= 0) { ++ } else if ((i_index = umplock_find_item(lock_item->secure_id)) >= 0) { ++ for (ref_index = 0; ref_index < MAX_PIDS; ref_index++) { ++ if (device.items[i_index].references[ref_index].pid == 0) { ++ break; ++ } ++ } ++ if (ref_index < MAX_PIDS) { ++ device.items[i_index].references[ref_index].pid = lock_cmd->pid; ++ device.items[i_index].references[ref_index].ref_count = 0; ++ device.items[i_index].references[ref_index].down_count = 0; ++ } else { ++ PERROR("whoops, item ran out of available reference slots\n"); ++ return -EINVAL; ++ ++ } ++ } else { ++ i_index = umplock_find_item(0); ++ ++ if (i_index >= 0) { ++ device.items[i_index].secure_id = lock_item->secure_id; ++ device.items[i_index].id_ref_count = 0; ++ device.items[i_index].usage = lock_item->usage; ++ device.items[i_index].references[0].pid = lock_cmd->pid; ++ device.items[i_index].references[0].ref_count = 0; ++ device.items[i_index].references[0].down_count = 0; ++ sema_init(&device.items[i_index].item_lock, 1); ++ } else { ++ PERROR("whoops, ran out of available slots\n"); ++ return -EINVAL; ++ } ++ } ++ ++ return 0; ++} ++/** IOCTLs **/ ++ ++static int do_umplock_create(_lock_cmd_priv *lock_cmd) ++{ ++ return 0; ++} ++ ++static int do_umplock_process(_lock_cmd_priv *lock_cmd) ++{ ++ int ret, i_index, ref_index; ++ _lock_item_s *lock_item = (_lock_item_s *)&lock_cmd->msg; ++ ++ mutex_lock(&device.item_list_lock); ++ ++ if (0 == lock_item->secure_id) { ++ PERROR("IOCTL_UMPLOCK_PROCESS called with secure_id is 0, pid: %d\n", lock_cmd->pid); ++ mutex_unlock(&device.item_list_lock); ++ return -EINVAL; ++ } ++ ++ ret = do_umplock_create_locked(lock_cmd); ++ if (ret < 0) { ++ mutex_unlock(&device.item_list_lock); ++ return -EINVAL; ++ } ++ ++ ret = umplock_find_item_by_pid(lock_cmd, &i_index, &ref_index); ++ if (ret < 0) { ++ /*fail to find a item*/ ++ PERROR("IOCTL_UMPLOCK_PROCESS called with invalid parameter, pid: %d\n", lock_cmd->pid); ++ mutex_unlock(&device.item_list_lock); ++ return -EINVAL; ++ } ++ device.items[i_index].references[ref_index].ref_count++; ++ device.items[i_index].id_ref_count++; ++ PDEBUG(1, "try to lock, pid: %d, secure_id: 0x%x, ref_count: %d\n", lock_cmd->pid, lock_item->secure_id, device.items[i_index].references[ref_index].ref_count); ++ ++ if (lock_cmd->pid == device.items[i_index].owner) { ++ PDEBUG(1, "already own the lock, pid: %d, secure_id: 0x%x, ref_count: %d\n", lock_cmd->pid, lock_item->secure_id, device.items[i_index].references[ref_index].ref_count); ++ mutex_unlock(&device.item_list_lock); ++ return 0; ++ } ++ ++ device.items[i_index].references[ref_index].down_count++; ++ mutex_unlock(&device.item_list_lock); ++ if (down_interruptible(&device.items[i_index].item_lock)) { ++ /*wait up without hold the umplock. restore previous state and return*/ ++ mutex_lock(&device.item_list_lock); ++ device.items[i_index].references[ref_index].ref_count--; ++ device.items[i_index].id_ref_count--; ++ device.items[i_index].references[ref_index].down_count--; ++ if (0 == device.items[i_index].references[ref_index].ref_count) { ++ device.items[i_index].references[ref_index].pid = 0; ++ if (0 == device.items[i_index].id_ref_count) { ++ PDEBUG(1, "release item, pid: %d, secure_id: 0x%x\n", lock_cmd->pid, lock_item->secure_id); ++ device.items[i_index].secure_id = 0; ++ } ++ } ++ ++ PERROR("failed lock, pid: %d, secure_id: 0x%x, ref_count: %d\n", lock_cmd->pid, lock_item->secure_id, device.items[i_index].references[ref_index].ref_count); ++ ++ mutex_unlock(&device.item_list_lock); ++ return -ERESTARTSYS; ++ } ++ ++ mutex_lock(&device.item_list_lock); ++ PDEBUG(1, "got lock, pid: %d, secure_id: 0x%x, ref_count: %d\n", lock_cmd->pid, lock_item->secure_id, device.items[i_index].references[ref_index].ref_count); ++ device.items[i_index].owner = lock_cmd->pid; ++ mutex_unlock(&device.item_list_lock); ++ ++ return 0; ++} ++ ++static int do_umplock_release(_lock_cmd_priv *lock_cmd) ++{ ++ int ret, i_index, ref_index, call_up; ++ _lock_item_s *lock_item = (_lock_item_s *)&lock_cmd->msg; ++ ++ mutex_lock(&device.item_list_lock); ++ ++ if (0 == lock_item->secure_id) { ++ PERROR("IOCTL_UMPLOCK_RELEASE called with secure_id is 0, pid: %d\n", lock_cmd->pid); ++ mutex_unlock(&device.item_list_lock); ++ return -EINVAL; ++ } ++ ++ ret = umplock_find_client_valid(lock_cmd->pid); ++ if (ret < 0) { ++ /*lock request from an invalid client pid, do nothing*/ ++ mutex_unlock(&device.item_list_lock); ++ return -EPERM; ++ } ++ ++ i_index = ref_index = -1; ++ ++ ret = umplock_find_item_by_pid(lock_cmd, &i_index, &ref_index); ++ if (ret < 0) { ++ /*fail to find item*/ ++ PERROR("IOCTL_UMPLOCK_RELEASE called with invalid parameter pid: %d, secid: 0x%x\n", lock_cmd->pid, lock_item->secure_id); ++ mutex_unlock(&device.item_list_lock); ++ return -EINVAL; ++ } ++ ++ /* if the lock is not owned by this process */ ++ if (lock_cmd->pid != device.items[i_index].owner) { ++ mutex_unlock(&device.item_list_lock); ++ return -EPERM; ++ } ++ ++ /* if the ref_count is 0, that means nothing to unlock, just return */ ++ if (0 == device.items[i_index].references[ref_index].ref_count) { ++ mutex_unlock(&device.item_list_lock); ++ return 0; ++ } ++ ++ device.items[i_index].references[ref_index].ref_count--; ++ device.items[i_index].id_ref_count--; ++ PDEBUG(1, "unlock, pid: %d, secure_id: 0x%x, ref_count: %d\n", lock_cmd->pid, lock_item->secure_id, device.items[i_index].references[ref_index].ref_count); ++ ++ call_up = 0; ++ if (device.items[i_index].references[ref_index].down_count > 1) { ++ call_up = 1; ++ device.items[i_index].references[ref_index].down_count--; ++ } ++ if (0 == device.items[i_index].references[ref_index].ref_count) { ++ device.items[i_index].references[ref_index].pid = 0; ++ if (0 == device.items[i_index].id_ref_count) { ++ PDEBUG(1, "release item, pid: %d, secure_id: 0x%x\n", lock_cmd->pid, lock_item->secure_id); ++ device.items[i_index].secure_id = 0; ++ } ++ device.items[i_index].owner = 0; ++ call_up = 1; ++ } ++ if (call_up) { ++ PDEBUG(1, "call up, pid: %d, secure_id: 0x%x\n", lock_cmd->pid, lock_item->secure_id); ++ up(&device.items[i_index].item_lock); ++ } ++ mutex_unlock(&device.item_list_lock); ++ ++ return 0; ++} ++ ++static int do_umplock_zap(void) ++{ ++ int i; ++ ++ PDEBUG(1, "ZAP ALL ENTRIES!\n"); ++ ++ mutex_lock(&device.item_list_lock); ++ ++ for (i = 0; i < MAX_ITEMS; i++) { ++ device.items[i].secure_id = 0; ++ memset(&device.items[i].references, 0, sizeof(_lock_ref) * MAX_PIDS); ++ sema_init(&device.items[i].item_lock, 1); ++ } ++ ++ for (i = 0; i < MAX_PIDS; i++) { ++ device.pids[i] = 0; ++ } ++ mutex_unlock(&device.item_list_lock); ++ ++ return 0; ++} ++ ++static int do_umplock_dump(void) ++{ ++ int i, j; ++ ++ mutex_lock(&device.item_list_lock); ++ PERROR("dump all the items begin\n"); ++ for (i = 0; i < MAX_ITEMS; i++) { ++ for (j = 0; j < MAX_PIDS; j++) { ++ if (device.items[i].secure_id != 0 && device.items[i].references[j].pid != 0) { ++ PERROR("item[%d]->secure_id=0x%x, owner=%d\t reference[%d].ref_count=%d.pid=%d\n", ++ i, ++ device.items[i].secure_id, ++ device.items[i].owner, ++ j, ++ device.items[i].references[j].ref_count, ++ device.items[i].references[j].pid); ++ } ++ } ++ } ++ PERROR("dump all the items end\n"); ++ mutex_unlock(&device.item_list_lock); ++ ++ return 0; ++} ++ ++int do_umplock_client_add(_lock_cmd_priv *lock_cmd) ++{ ++ int i; ++ mutex_lock(&device.item_list_lock); ++ for (i = 0; i < MAX_PIDS; i++) { ++ if (device.pids[i] == lock_cmd->pid) { ++ mutex_unlock(&device.item_list_lock); ++ return 0; ++ } ++ } ++ for (i = 0; i < MAX_PIDS; i++) { ++ if (device.pids[i] == 0) { ++ device.pids[i] = lock_cmd->pid; ++ break; ++ } ++ } ++ mutex_unlock(&device.item_list_lock); ++ if (i == MAX_PIDS) { ++ PERROR("Oops, Run out of client slots\n "); ++ return -EINVAL; ++ } ++ return 0; ++} ++ ++int do_umplock_client_delete(_lock_cmd_priv *lock_cmd) ++{ ++ int p_index = -1, i_index = -1, ref_index = -1; ++ int ret; ++ _lock_item_s *lock_item; ++ lock_item = (_lock_item_s *)&lock_cmd->msg; ++ ++ mutex_lock(&device.item_list_lock); ++ p_index = umplock_find_client_valid(lock_cmd->pid); ++ /*lock item pid is not valid.*/ ++ if (p_index < 0) { ++ mutex_unlock(&device.item_list_lock); ++ return 0; ++ } ++ ++ /*walk through umplock item list and release reference attached to this client*/ ++ for (i_index = 0; i_index < MAX_ITEMS; i_index++) { ++ lock_item->secure_id = device.items[i_index].secure_id; ++ ++ /*find the item index and reference slot for the lock_item*/ ++ ret = umplock_find_item_by_pid(lock_cmd, &i_index, &ref_index); ++ ++ if (ret < 0) { ++ /*client has no reference on this umplock item, skip*/ ++ continue; ++ } ++ while (device.items[i_index].references[ref_index].ref_count) { ++ /*release references on this client*/ ++ ++ PDEBUG(1, "delete client, pid: %d, ref_count: %d\n", lock_cmd->pid, device.items[i_index].references[ref_index].ref_count); ++ ++ mutex_unlock(&device.item_list_lock); ++ do_umplock_release(lock_cmd); ++ mutex_lock(&device.item_list_lock); ++ } ++ } ++ ++ /*remove the pid from umplock valid pid list*/ ++ device.pids[p_index] = 0; ++ mutex_unlock(&device.item_list_lock); ++ ++ return 0; ++} ++ ++static long umplock_driver_ioctl(struct file *f, unsigned int cmd, unsigned long arg) ++{ ++ int ret; ++ uint32_t size = _IOC_SIZE(cmd); ++ _lock_cmd_priv lock_cmd ; ++ ++ if (_IOC_TYPE(cmd) != LOCK_IOCTL_GROUP) { ++ return -ENOTTY; ++ } ++ ++ if (_IOC_NR(cmd) >= LOCK_IOCTL_MAX_CMDS) { ++ return -ENOTTY; ++ } ++ ++ switch (cmd) { ++ case LOCK_IOCTL_CREATE: ++ if (size != sizeof(_lock_item_s)) { ++ return -ENOTTY; ++ } ++ ++ if (copy_from_user(&lock_cmd.msg, (void __user *)arg, size)) { ++ return -EFAULT; ++ } ++ lock_cmd.pid = (u32)current->tgid; ++ ret = do_umplock_create(&lock_cmd); ++ if (ret) { ++ return ret; ++ } ++ return 0; ++ ++ case LOCK_IOCTL_PROCESS: ++ if (size != sizeof(_lock_item_s)) { ++ return -ENOTTY; ++ } ++ ++ if (copy_from_user(&lock_cmd.msg, (void __user *)arg, size)) { ++ return -EFAULT; ++ } ++ lock_cmd.pid = (u32)current->tgid; ++ return do_umplock_process(&lock_cmd); ++ ++ case LOCK_IOCTL_RELEASE: ++ if (size != sizeof(_lock_item_s)) { ++ return -ENOTTY; ++ } ++ ++ if (copy_from_user(&lock_cmd.msg, (void __user *)arg, size)) { ++ return -EFAULT; ++ } ++ lock_cmd.pid = (u32)current->tgid; ++ ret = do_umplock_release(&lock_cmd); ++ if (ret) { ++ return ret; ++ } ++ return 0; ++ ++ case LOCK_IOCTL_ZAP: ++ do_umplock_zap(); ++ return 0; ++ ++ case LOCK_IOCTL_DUMP: ++ do_umplock_dump(); ++ return 0; ++ } ++ ++ return -ENOIOCTLCMD; ++} ++ ++static int umplock_driver_open(struct inode *inode, struct file *filp) ++{ ++ _lock_cmd_priv lock_cmd; ++ ++ atomic_inc(&device.sessions); ++ PDEBUG(1, "OPEN SESSION (%i references)\n", atomic_read(&device.sessions)); ++ ++ lock_cmd.pid = (u32)current->tgid; ++ do_umplock_client_add(&lock_cmd); ++ ++ return 0; ++} ++ ++static int umplock_driver_release(struct inode *inode, struct file *filp) ++{ ++ int sessions = 0; ++ _lock_cmd_priv lock_cmd; ++ ++ lock_cmd.pid = (u32)current->tgid; ++ do_umplock_client_delete(&lock_cmd); ++ ++ mutex_lock(&device.item_list_lock); ++ atomic_dec(&device.sessions); ++ sessions = atomic_read(&device.sessions); ++ PDEBUG(1, "CLOSE SESSION (%i references)\n", sessions); ++ mutex_unlock(&device.item_list_lock); ++ if (sessions == 0) { ++ do_umplock_zap(); ++ } ++ ++ return 0; ++} ++ ++static struct file_operations umplock_fops = { ++ .owner = THIS_MODULE, ++ .open = umplock_driver_open, ++ .release = umplock_driver_release, ++ .unlocked_ioctl = umplock_driver_ioctl, ++}; ++ ++int umplock_device_initialize(void) ++{ ++ int err; ++ ++ err = alloc_chrdev_region(&umplock_dev, 0, 1, umplock_dev_name); ++ ++ if (0 == err) { ++ memset(&umplock_device, 0, sizeof(umplock_device)); ++ cdev_init(&umplock_device.cdev, &umplock_fops); ++ umplock_device.cdev.owner = THIS_MODULE; ++ umplock_device.cdev.ops = &umplock_fops; ++ ++ err = cdev_add(&umplock_device.cdev, umplock_dev, 1); ++ if (0 == err) { ++ umplock_device.umplock_class = class_create(THIS_MODULE, umplock_dev_name); ++ if (IS_ERR(umplock_device.umplock_class)) { ++ err = PTR_ERR(umplock_device.umplock_class); ++ } else { ++ struct device *mdev; ++ mdev = device_create(umplock_device.umplock_class, NULL, umplock_dev, NULL, umplock_dev_name); ++ if (!IS_ERR(mdev)) { ++ return 0; /* all ok */ ++ } ++ ++ err = PTR_ERR(mdev); ++ class_destroy(umplock_device.umplock_class); ++ } ++ cdev_del(&umplock_device.cdev); ++ } ++ ++ unregister_chrdev_region(umplock_dev, 1); ++ } else { ++ PERROR("alloc chardev region failed\n"); ++ } ++ ++ return err; ++} ++ ++void umplock_device_terminate(void) ++{ ++ device_destroy(umplock_device.umplock_class, umplock_dev); ++ class_destroy(umplock_device.umplock_class); ++ ++ cdev_del(&umplock_device.cdev); ++ unregister_chrdev_region(umplock_dev, 1); ++} ++ ++static int __init umplock_initialize_module(void) ++{ ++ PDEBUG(1, "Inserting UMP lock device driver. Compiled: %s, time: %s\n", __DATE__, __TIME__); ++ ++ mutex_init(&device.item_list_lock); ++ if (umplock_device_initialize() != 0) { ++ PERROR("UMP lock device driver init failed\n"); ++ return -ENOTTY; ++ } ++ memset(&device.items, 0, sizeof(umplock_item) * MAX_ITEMS); ++ memset(&device.pids, 0, sizeof(u32) * MAX_PIDS); ++ atomic_set(&device.sessions, 0); ++ ++ PDEBUG(1, "UMP lock device driver loaded\n"); ++ ++ return 0; ++} ++ ++static void __exit umplock_cleanup_module(void) ++{ ++ PDEBUG(1, "unloading UMP lock module\n"); ++ ++ memset(&device.items, 0, sizeof(umplock_item) * MAX_ITEMS); ++ memset(&device.pids, 0, sizeof(u32) * MAX_PIDS); ++ umplock_device_terminate(); ++ mutex_destroy(&device.item_list_lock); ++ ++ PDEBUG(1, "UMP lock module unloaded\n"); ++} ++ ++module_init(umplock_initialize_module); ++module_exit(umplock_cleanup_module); ++ ++ ++MODULE_LICENSE("GPL"); ++MODULE_AUTHOR("ARM Ltd."); ++MODULE_DESCRIPTION("ARM UMP locker"); +diff --git a/drivers/gpu/arm/mali400/umplock/umplock_ioctl.h b/drivers/gpu/arm/mali400/umplock/umplock_ioctl.h +new file mode 100755 +index 000000000000..8afdaad7000f +--- /dev/null ++++ b/drivers/gpu/arm/mali400/umplock/umplock_ioctl.h +@@ -0,0 +1,66 @@ ++/* ++ * Copyright (C) 2012-2013, 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 ++ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained from Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ */ ++ ++#ifndef __UMPLOCK_IOCTL_H__ ++#define __UMPLOCK_IOCTL_H__ ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++#include ++#include ++ ++#ifndef __user ++#define __user ++#endif ++ ++ ++/** ++ * @file umplock_ioctl.h ++ * This file describes the interface needed to use the Linux device driver. ++ * The interface is used by the userpace Mali DDK. ++ */ ++ ++typedef enum { ++ _LOCK_ACCESS_RENDERABLE = 1, ++ _LOCK_ACCESS_TEXTURE, ++ _LOCK_ACCESS_CPU_WRITE, ++ _LOCK_ACCESS_CPU_READ, ++} _lock_access_usage; ++ ++typedef struct _lock_item_s { ++ unsigned int secure_id; ++ _lock_access_usage usage; ++} _lock_item_s; ++ ++ ++#define LOCK_IOCTL_GROUP 0x91 ++ ++#define _LOCK_IOCTL_CREATE_CMD 0 /* create kernel lock item */ ++#define _LOCK_IOCTL_PROCESS_CMD 1 /* process kernel lock item */ ++#define _LOCK_IOCTL_RELEASE_CMD 2 /* release kernel lock item */ ++#define _LOCK_IOCTL_ZAP_CMD 3 /* clean up all kernel lock items */ ++#define _LOCK_IOCTL_DUMP_CMD 4 /* dump all the items */ ++ ++#define LOCK_IOCTL_MAX_CMDS 5 ++ ++#define LOCK_IOCTL_CREATE _IOW( LOCK_IOCTL_GROUP, _LOCK_IOCTL_CREATE_CMD, _lock_item_s ) ++#define LOCK_IOCTL_PROCESS _IOW( LOCK_IOCTL_GROUP, _LOCK_IOCTL_PROCESS_CMD, _lock_item_s ) ++#define LOCK_IOCTL_RELEASE _IOW( LOCK_IOCTL_GROUP, _LOCK_IOCTL_RELEASE_CMD, _lock_item_s ) ++#define LOCK_IOCTL_ZAP _IO ( LOCK_IOCTL_GROUP, _LOCK_IOCTL_ZAP_CMD ) ++#define LOCK_IOCTL_DUMP _IO ( LOCK_IOCTL_GROUP, _LOCK_IOCTL_DUMP_CMD ) ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif /* __UMPLOCK_IOCTL_H__ */ ++ +diff --git a/drivers/gpu/arm/midgard/Kbuild b/drivers/gpu/arm/midgard/Kbuild +new file mode 100755 +index 000000000000..b2c2bbcda668 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/Kbuild +@@ -0,0 +1,221 @@ ++# ++# (C) COPYRIGHT 2012-2016, 2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++KBUILD_CFLAGS += -include rename.h ++ ++# Driver version string which is returned to userspace via an ioctl ++MALI_RELEASE_NAME ?= "r18p0-01rel0" ++ ++# Paths required for build ++ ++# make $(src) as absolute path if it isn't already, by prefixing $(srctree) ++src:=$(if $(patsubst /%,,$(src)),$(srctree)/$(src),$(src)) ++KBASE_PATH = $(src) ++KBASE_PLATFORM_PATH = $(KBASE_PATH)/platform_dummy ++UMP_PATH = $(src)/../../../base ++ ++ifeq ($(CONFIG_MALI_ERROR_INJECTION),y) ++MALI_ERROR_INJECT_ON = 1 ++endif ++ ++# Set up defaults if not defined by build system ++MALI_CUSTOMER_RELEASE ?= 1 ++MALI_UNIT_TEST ?= 0 ++MALI_KERNEL_TEST_API ?= 0 ++MALI_ERROR_INJECT_ON ?= 0 ++MALI_MOCK_TEST ?= 0 ++MALI_COVERAGE ?= 0 ++MALI_INSTRUMENTATION_LEVEL ?= 0 ++# This workaround is for what seems to be a compiler bug we observed in ++# GCC 4.7 on AOSP 4.3. The bug caused an intermittent failure compiling ++# the "_Pragma" syntax, where an error message is returned: ++# ++# "internal compiler error: unspellable token PRAGMA" ++# ++# This regression has thus far only been seen on the GCC 4.7 compiler bundled ++# with AOSP 4.3.0. So this makefile, intended for in-tree kernel builds ++# which are not known to be used with AOSP, is hardcoded to disable the ++# workaround, i.e. set the define to 0. ++MALI_GCC_WORKAROUND_MIDCOM_4598 ?= 0 ++ ++# Set up our defines, which will be passed to gcc ++DEFINES = \ ++ -DMALI_CUSTOMER_RELEASE=$(MALI_CUSTOMER_RELEASE) \ ++ -DMALI_KERNEL_TEST_API=$(MALI_KERNEL_TEST_API) \ ++ -DMALI_UNIT_TEST=$(MALI_UNIT_TEST) \ ++ -DMALI_ERROR_INJECT_ON=$(MALI_ERROR_INJECT_ON) \ ++ -DMALI_MOCK_TEST=$(MALI_MOCK_TEST) \ ++ -DMALI_COVERAGE=$(MALI_COVERAGE) \ ++ -DMALI_INSTRUMENTATION_LEVEL=$(MALI_INSTRUMENTATION_LEVEL) \ ++ -DMALI_RELEASE_NAME=\"$(MALI_RELEASE_NAME)\" \ ++ -DMALI_GCC_WORKAROUND_MIDCOM_4598=$(MALI_GCC_WORKAROUND_MIDCOM_4598) ++ ++ifeq ($(KBUILD_EXTMOD),) ++# in-tree ++DEFINES +=-DMALI_KBASE_THIRDPARTY_PATH=../../$(src)/platform/$(CONFIG_MALI_PLATFORM_THIRDPARTY_NAME) ++else ++# out-of-tree ++DEFINES +=-DMALI_KBASE_THIRDPARTY_PATH=$(src)/platform/$(CONFIG_MALI_PLATFORM_THIRDPARTY_NAME) ++endif ++ ++DEFINES += -I$(srctree)/drivers/staging/android ++ ++# Use our defines when compiling ++ccflags-y += $(DEFINES) -I$(KBASE_PATH) -I$(KBASE_PLATFORM_PATH) -I$(UMP_PATH) -I$(srctree)/include/linux ++subdir-ccflags-y += $(DEFINES) -I$(KBASE_PATH) -I$(KBASE_PLATFORM_PATH) -I$(OSK_PATH) -I$(UMP_PATH) -I$(srctree)/include/linux ++ ++SRC := \ ++ mali_kbase_device.c \ ++ mali_kbase_cache_policy.c \ ++ mali_kbase_mem.c \ ++ mali_kbase_mmu.c \ ++ mali_kbase_ctx_sched.c \ ++ mali_kbase_jd.c \ ++ mali_kbase_jd_debugfs.c \ ++ mali_kbase_jm.c \ ++ mali_kbase_gpuprops.c \ ++ mali_kbase_js.c \ ++ mali_kbase_js_ctx_attr.c \ ++ mali_kbase_event.c \ ++ mali_kbase_context.c \ ++ mali_kbase_pm.c \ ++ mali_kbase_config.c \ ++ mali_kbase_vinstr.c \ ++ mali_kbase_softjobs.c \ ++ mali_kbase_10969_workaround.c \ ++ mali_kbase_hw.c \ ++ mali_kbase_utility.c \ ++ mali_kbase_debug.c \ ++ mali_kbase_trace_timeline.c \ ++ mali_kbase_gpu_memory_debugfs.c \ ++ mali_kbase_mem_linux.c \ ++ mali_kbase_core_linux.c \ ++ mali_kbase_replay.c \ ++ mali_kbase_mem_profile_debugfs.c \ ++ mali_kbase_mmu_mode_lpae.c \ ++ mali_kbase_mmu_mode_aarch64.c \ ++ mali_kbase_disjoint_events.c \ ++ mali_kbase_gator_api.c \ ++ mali_kbase_debug_mem_view.c \ ++ mali_kbase_debug_job_fault.c \ ++ mali_kbase_smc.c \ ++ mali_kbase_mem_pool.c \ ++ mali_kbase_mem_pool_debugfs.c \ ++ mali_kbase_tlstream.c \ ++ mali_kbase_strings.c \ ++ mali_kbase_as_fault_debugfs.c \ ++ mali_kbase_regs_history_debugfs.c ++ ++ ++ ++ ++ifeq ($(MALI_UNIT_TEST),1) ++ SRC += mali_kbase_tlstream_test.c ++endif ++ ++ifeq ($(MALI_CUSTOMER_RELEASE),0) ++ SRC += mali_kbase_regs_dump_debugfs.c ++endif ++ ++ ++ccflags-y += -I$(KBASE_PATH) ++ ++ifeq ($(CONFIG_MALI_PLATFORM_FAKE),y) ++ SRC += mali_kbase_platform_fake.c ++ ++ ifeq ($(CONFIG_MALI_PLATFORM_VEXPRESS),y) ++ SRC += platform/vexpress/mali_kbase_config_vexpress.c \ ++ platform/vexpress/mali_kbase_cpu_vexpress.c ++ ccflags-y += -I$(src)/platform/vexpress ++ endif ++ ++ ifeq ($(CONFIG_MALI_PLATFORM_RTSM_VE),y) ++ SRC += platform/rtsm_ve/mali_kbase_config_vexpress.c ++ ccflags-y += -I$(src)/platform/rtsm_ve ++ endif ++ ++ ifeq ($(CONFIG_MALI_PLATFORM_VEXPRESS_1XV7_A57),y) ++ SRC += platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c ++ ccflags-y += -I$(src)/platform/vexpress_1xv7_a57 ++ endif ++ ++ ifeq ($(CONFIG_MALI_PLATFORM_VEXPRESS_6XVIRTEX7_10MHZ),y) ++ SRC += platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_vexpress.c \ ++ platform/vexpress_6xvirtex7_10mhz/mali_kbase_cpu_vexpress.c ++ ccflags-y += -I$(src)/platform/vexpress_6xvirtex7_10mhz ++ endif ++endif # CONFIG_MALI_PLATFORM_FAKE=y ++ ++# Tell the Linux build system from which .o file to create the kernel module ++obj-$(CONFIG_MALI_MIDGARD) += midgard_kbase.o ++ ++# Tell the Linux build system to enable building of our .c files ++midgard_kbase-y := $(SRC:.c=.o) ++ ++ifeq ($(CONFIG_MALI_PLATFORM_THIRDPARTY),y) ++ # Kconfig passes in the name with quotes for in-tree builds - remove them. ++ platform_name := $(shell echo $(CONFIG_MALI_PLATFORM_THIRDPARTY_NAME)) ++ MALI_PLATFORM_THIRDPARTY_DIR := platform/$(platform_name) ++ ccflags-y += -I$(src)/$(MALI_PLATFORM_THIRDPARTY_DIR) ++ include $(src)/$(MALI_PLATFORM_THIRDPARTY_DIR)/Kbuild ++endif ++ ++ifeq ($(CONFIG_MALI_DEVFREQ),y) ++ ifeq ($(CONFIG_DEVFREQ_THERMAL),y) ++ include $(src)/ipa/Kbuild ++ endif ++endif ++ ++midgard_kbase-$(CONFIG_MALI_DMA_FENCE) += \ ++ mali_kbase_dma_fence.o \ ++ mali_kbase_fence.o ++midgard_kbase-$(CONFIG_SYNC) += \ ++ mali_kbase_sync_android.o \ ++ mali_kbase_sync_common.o ++midgard_kbase-$(CONFIG_SYNC_FILE) += \ ++ mali_kbase_sync_file.o \ ++ mali_kbase_sync_common.o \ ++ mali_kbase_fence.o ++ ++MALI_BACKEND_PATH ?= backend ++CONFIG_MALI_BACKEND ?= gpu ++CONFIG_MALI_BACKEND_REAL ?= $(CONFIG_MALI_BACKEND) ++ ++ifeq ($(MALI_MOCK_TEST),1) ++ifeq ($(CONFIG_MALI_BACKEND_REAL),gpu) ++# Test functionality ++midgard_kbase-y += tests/internal/src/mock/mali_kbase_pm_driver_mock.o ++endif ++endif ++ ++include $(src)/$(MALI_BACKEND_PATH)/$(CONFIG_MALI_BACKEND_REAL)/Kbuild ++midgard_kbase-y += $(BACKEND:.c=.o) ++ ++ ++ccflags-y += -I$(src)/$(MALI_BACKEND_PATH)/$(CONFIG_MALI_BACKEND_REAL) ++subdir-ccflags-y += -I$(src)/$(MALI_BACKEND_PATH)/$(CONFIG_MALI_BACKEND_REAL) ++ ++# Default to devicetree platform if neither a fake platform or a thirdparty ++# platform is configured. ++ifeq ($(CONFIG_MALI_PLATFORM_THIRDPARTY)$(CONFIG_MALI_PLATFORM_FAKE),) ++CONFIG_MALI_PLATFORM_DEVICETREE := y ++endif ++ ++midgard_kbase-$(CONFIG_MALI_PLATFORM_DEVICETREE) += \ ++ platform/devicetree/mali_kbase_runtime_pm.o \ ++ platform/devicetree/mali_kbase_config_devicetree.o ++ccflags-$(CONFIG_MALI_PLATFORM_DEVICETREE) += -I$(src)/platform/devicetree ++ ++# For kutf and mali_kutf_irq_latency_test ++obj-$(CONFIG_MALI_KUTF) += tests/ +diff --git a/drivers/gpu/arm/midgard/Kconfig b/drivers/gpu/arm/midgard/Kconfig +new file mode 100755 +index 000000000000..1b28bb73ad59 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/Kconfig +@@ -0,0 +1,248 @@ ++# ++# (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++ ++menuconfig MALI_MIDGARD ++ tristate "Mali Midgard series support" ++ select GPU_TRACEPOINTS if ANDROID ++ default n ++ help ++ Enable this option to build support for a ARM Mali Midgard GPU. ++ ++ To compile this driver as a module, choose M here: ++ this will generate a single module, called mali_kbase. ++ ++config MALI_GATOR_SUPPORT ++ bool "Streamline support via Gator" ++ depends on MALI_MIDGARD ++ default n ++ help ++ Adds diagnostic support for use with the ARM Streamline Performance Analyzer. ++ You will need the Gator device driver already loaded before loading this driver when enabling ++ Streamline debug support. ++ This is a legacy interface required by older versions of Streamline. ++ ++config MALI_MIDGARD_DVFS ++ bool "Enable legacy DVFS" ++ depends on MALI_MIDGARD && !MALI_DEVFREQ && !MALI_PLATFORM_DEVICETREE ++ default n ++ help ++ Choose this option to enable legacy DVFS in the Mali Midgard DDK. ++ ++config MALI_MIDGARD_ENABLE_TRACE ++ bool "Enable kbase tracing" ++ depends on MALI_MIDGARD ++ default n ++ help ++ Enables tracing in kbase. Trace log available through ++ the "mali_trace" debugfs file, when the CONFIG_DEBUG_FS is enabled ++ ++config MALI_DEVFREQ ++ bool "devfreq support for Mali" ++ depends on MALI_MIDGARD && PM_DEVFREQ ++ help ++ Support devfreq for Mali. ++ ++ Using the devfreq framework and, by default, the simpleondemand ++ governor, the frequency of Mali will be dynamically selected from the ++ available OPPs. ++ ++config MALI_DMA_FENCE ++ bool "DMA_BUF fence support for Mali" ++ depends on MALI_MIDGARD && !KDS ++ default n ++ help ++ Support DMA_BUF fences for Mali. ++ ++ This option should only be enabled if KDS is not present and ++ the Linux Kernel has built in support for DMA_BUF fences. ++ ++# MALI_EXPERT configuration options ++ ++menuconfig MALI_EXPERT ++ depends on MALI_MIDGARD ++ bool "Enable Expert Settings" ++ default n ++ help ++ Enabling this option and modifying the default settings may produce a driver with performance or ++ other limitations. ++ ++config MALI_CORESTACK ++ bool "Support controlling power to the GPU core stack" ++ depends on MALI_MIDGARD && MALI_EXPERT ++ default n ++ help ++ Enabling this feature on supported GPUs will let the driver powering ++ on/off the GPU core stack independently without involving the Power ++ Domain Controller. This should only be enabled on platforms which ++ integration of the PDC to the Mali GPU is known to be problematic. ++ This feature is currently only supported on t-Six and t-HEx GPUs. ++ ++ If unsure, say N. ++ ++config MALI_PRFCNT_SET_SECONDARY ++ bool "Use secondary set of performance counters" ++ depends on MALI_MIDGARD && MALI_EXPERT ++ default n ++ help ++ Select this option to use secondary set of performance counters. Kernel ++ features that depend on an access to the primary set of counters may ++ become unavailable. Enabling this option will prevent power management ++ from working optimally and may cause instrumentation tools to return ++ bogus results. ++ ++ If unsure, say N. ++ ++config MALI_PLATFORM_FAKE ++ bool "Enable fake platform device support" ++ depends on MALI_MIDGARD && MALI_EXPERT ++ default n ++ help ++ When you start to work with the Mali Midgard series device driver the platform-specific code of ++ the Linux kernel for your platform may not be complete. In this situation the kernel device driver ++ supports creating the platform device outside of the Linux platform-specific code. ++ Enable this option if would like to use a platform device configuration from within the device driver. ++ ++choice ++ prompt "Platform configuration" ++ depends on MALI_MIDGARD && MALI_EXPERT ++ default MALI_PLATFORM_DEVICETREE ++ help ++ Select the SOC platform that contains a Mali Midgard GPU ++ ++config MALI_PLATFORM_DEVICETREE ++ bool "Device Tree platform" ++ depends on OF ++ help ++ Select this option to use Device Tree with the Mali driver. ++ ++ When using this option the Mali driver will get the details of the ++ GPU hardware from the Device Tree. This means that the same driver ++ binary can run on multiple platforms as long as all the GPU hardware ++ details are described in the device tree. ++ ++ Device Tree is the recommended method for the Mali driver platform ++ integration. ++ ++config MALI_PLATFORM_VEXPRESS ++ depends on ARCH_VEXPRESS && (ARCH_VEXPRESS_CA9X4 || ARCH_VEXPRESS_CA15X4) ++ bool "Versatile Express" ++config MALI_PLATFORM_VEXPRESS_VIRTEX7_40MHZ ++ depends on ARCH_VEXPRESS && (ARCH_VEXPRESS_CA9X4 || ARCH_VEXPRESS_CA15X4) ++ bool "Versatile Express w/Virtex7 @ 40Mhz" ++config MALI_PLATFORM_GOLDFISH ++ depends on ARCH_GOLDFISH ++ bool "Android Goldfish virtual CPU" ++config MALI_PLATFORM_PBX ++ depends on ARCH_REALVIEW && REALVIEW_EB_A9MP && MACH_REALVIEW_PBX ++ bool "Realview PBX-A9" ++config MALI_PLATFORM_THIRDPARTY ++ bool "Third Party Platform" ++endchoice ++ ++config MALI_PLATFORM_THIRDPARTY_NAME ++ depends on MALI_MIDGARD && MALI_PLATFORM_THIRDPARTY && MALI_EXPERT ++ string "Third party platform name" ++ help ++ Enter the name of a third party platform that is supported. The third part configuration ++ file must be in midgard/config/tpip/mali_kbase_config_xxx.c where xxx is the name ++ specified here. ++ ++config MALI_DEBUG ++ bool "Debug build" ++ depends on MALI_MIDGARD && MALI_EXPERT ++ default n ++ help ++ Select this option for increased checking and reporting of errors. ++ ++config MALI_FENCE_DEBUG ++ bool "Debug sync fence usage" ++ depends on MALI_MIDGARD && MALI_EXPERT && (SYNC || SYNC_FILE) ++ default y if MALI_DEBUG ++ help ++ Select this option to enable additional checking and reporting on the ++ use of sync fences in the Mali driver. ++ ++ This will add a 3s timeout to all sync fence waits in the Mali ++ driver, so that when work for Mali has been waiting on a sync fence ++ for a long time a debug message will be printed, detailing what fence ++ is causing the block, and which dependent Mali atoms are blocked as a ++ result of this. ++ ++ The timeout can be changed at runtime through the js_soft_timeout ++ device attribute, where the timeout is specified in milliseconds. ++ ++config MALI_NO_MALI ++ bool "No Mali" ++ depends on MALI_MIDGARD && MALI_EXPERT ++ default n ++ help ++ This can be used to test the driver in a simulated environment ++ whereby the hardware is not physically present. If the hardware is physically ++ present it will not be used. This can be used to test the majority of the ++ driver without needing actual hardware or for software benchmarking. ++ All calls to the simulated hardware will complete immediately as if the hardware ++ completed the task. ++ ++config MALI_ERROR_INJECT ++ bool "Error injection" ++ depends on MALI_MIDGARD && MALI_EXPERT && MALI_NO_MALI ++ default n ++ help ++ Enables insertion of errors to test module failure and recovery mechanisms. ++ ++config MALI_TRACE_TIMELINE ++ bool "Timeline tracing" ++ depends on MALI_MIDGARD && MALI_EXPERT ++ default n ++ help ++ Enables timeline tracing through the kernel tracepoint system. ++ ++config MALI_SYSTEM_TRACE ++ bool "Enable system event tracing support" ++ depends on MALI_MIDGARD && MALI_EXPERT ++ default n ++ help ++ Choose this option to enable system trace events for each ++ kbase event. This is typically used for debugging but has ++ minimal overhead when not in use. Enable only if you know what ++ you are doing. ++ ++config MALI_GPU_MMU_AARCH64 ++ bool "Use AArch64 page tables" ++ depends on MALI_MIDGARD && MALI_EXPERT ++ default n ++ help ++ Use AArch64 format page tables for the GPU instead of LPAE-style. ++ The two formats have the same functionality and performance but a ++ future GPU may deprecate or remove the legacy LPAE-style format. ++ ++ The LPAE-style format is supported on all Midgard and current Bifrost ++ GPUs. Enabling AArch64 format restricts the driver to only supporting ++ Bifrost GPUs. ++ ++ If in doubt, say N. ++ ++config MALI_PWRSOFT_765 ++ bool "PWRSOFT-765 ticket" ++ depends on MALI_MIDGARD && MALI_EXPERT ++ default n ++ help ++ PWRSOFT-765 fixes devfreq cooling devices issues. However, they are ++ not merged in mainline kernel yet. So this define helps to guard those ++ parts of the code. ++ ++source "drivers/gpu/arm/midgard/platform/Kconfig" ++source "drivers/gpu/arm/midgard/tests/Kconfig" +diff --git a/drivers/gpu/arm/midgard/Makefile b/drivers/gpu/arm/midgard/Makefile +new file mode 100755 +index 000000000000..9aa242c4f8c4 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/Makefile +@@ -0,0 +1,42 @@ ++# ++# (C) COPYRIGHT 2010-2016, 2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++ ++KDIR ?= /lib/modules/$(shell uname -r)/build ++ ++BUSLOG_PATH_RELATIVE = $(CURDIR)/../../../.. ++UMP_PATH_RELATIVE = $(CURDIR)/../../../base/ump ++KBASE_PATH_RELATIVE = $(CURDIR) ++KDS_PATH_RELATIVE = $(CURDIR)/../../../.. ++EXTRA_SYMBOLS = $(UMP_PATH_RELATIVE)/src/Module.symvers ++ ++ifeq ($(MALI_UNIT_TEST), 1) ++ EXTRA_SYMBOLS += $(KBASE_PATH_RELATIVE)/tests/internal/src/kernel_assert_module/linux/Module.symvers ++endif ++ ++ifeq ($(MALI_BUS_LOG), 1) ++#Add bus logger symbols ++EXTRA_SYMBOLS += $(BUSLOG_PATH_RELATIVE)/drivers/base/bus_logger/Module.symvers ++endif ++ ++# GPL driver supports KDS ++EXTRA_SYMBOLS += $(KDS_PATH_RELATIVE)/drivers/base/kds/Module.symvers ++ ++# we get the symbols from modules using KBUILD_EXTRA_SYMBOLS to prevent warnings about unknown functions ++all: ++ $(MAKE) -C $(KDIR) M=$(CURDIR) EXTRA_CFLAGS="-I$(CURDIR)/../../../../include -I$(CURDIR)/../../../../tests/include $(SCONS_CFLAGS)" $(SCONS_CONFIGS) KBUILD_EXTRA_SYMBOLS="$(EXTRA_SYMBOLS)" modules ++ ++clean: ++ $(MAKE) -C $(KDIR) M=$(CURDIR) clean +diff --git a/drivers/gpu/arm/midgard/Makefile.kbase b/drivers/gpu/arm/midgard/Makefile.kbase +new file mode 100755 +index 000000000000..2bef9c25eaeb +--- /dev/null ++++ b/drivers/gpu/arm/midgard/Makefile.kbase +@@ -0,0 +1,17 @@ ++# ++# (C) COPYRIGHT 2010 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++EXTRA_CFLAGS += -I$(ROOT) -I$(KBASE_PATH) -I$(OSK_PATH)/src/linux/include -I$(KBASE_PATH)/platform_$(PLATFORM) ++ +diff --git a/drivers/gpu/arm/midgard/backend/gpu/Kbuild b/drivers/gpu/arm/midgard/backend/gpu/Kbuild +new file mode 100755 +index 000000000000..5f700e9b6b44 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/Kbuild +@@ -0,0 +1,60 @@ ++# ++# (C) COPYRIGHT 2014,2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++BACKEND += \ ++ backend/gpu/mali_kbase_cache_policy_backend.c \ ++ backend/gpu/mali_kbase_device_hw.c \ ++ backend/gpu/mali_kbase_gpu.c \ ++ backend/gpu/mali_kbase_gpuprops_backend.c \ ++ backend/gpu/mali_kbase_debug_job_fault_backend.c \ ++ backend/gpu/mali_kbase_irq_linux.c \ ++ backend/gpu/mali_kbase_instr_backend.c \ ++ backend/gpu/mali_kbase_jm_as.c \ ++ backend/gpu/mali_kbase_jm_hw.c \ ++ backend/gpu/mali_kbase_jm_rb.c \ ++ backend/gpu/mali_kbase_js_affinity.c \ ++ backend/gpu/mali_kbase_js_backend.c \ ++ backend/gpu/mali_kbase_mmu_hw_direct.c \ ++ backend/gpu/mali_kbase_pm_backend.c \ ++ backend/gpu/mali_kbase_pm_driver.c \ ++ backend/gpu/mali_kbase_pm_metrics.c \ ++ backend/gpu/mali_kbase_pm_ca.c \ ++ backend/gpu/mali_kbase_pm_ca_fixed.c \ ++ backend/gpu/mali_kbase_pm_always_on.c \ ++ backend/gpu/mali_kbase_pm_coarse_demand.c \ ++ backend/gpu/mali_kbase_pm_demand.c \ ++ backend/gpu/mali_kbase_pm_policy.c \ ++ backend/gpu/mali_kbase_time.c ++ ++ifeq ($(MALI_CUSTOMER_RELEASE),0) ++BACKEND += \ ++ backend/gpu/mali_kbase_pm_ca_random.c \ ++ backend/gpu/mali_kbase_pm_demand_always_powered.c \ ++ backend/gpu/mali_kbase_pm_fast_start.c ++endif ++ ++ifeq ($(CONFIG_MALI_DEVFREQ),y) ++BACKEND += \ ++ backend/gpu/mali_kbase_devfreq.c \ ++ backend/gpu/mali_kbase_pm_ca_devfreq.c ++endif ++ ++ifeq ($(CONFIG_MALI_NO_MALI),y) ++ # Dummy model ++ BACKEND += backend/gpu/mali_kbase_model_dummy.c ++ BACKEND += backend/gpu/mali_kbase_model_linux.c ++ # HW error simulation ++ BACKEND += backend/gpu/mali_kbase_model_error_generator.c ++endif +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_backend_config.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_backend_config.h +new file mode 100755 +index 000000000000..c8ae87eb84a2 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_backend_config.h +@@ -0,0 +1,29 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* ++ * Backend specific configuration ++ */ ++ ++#ifndef _KBASE_BACKEND_CONFIG_H_ ++#define _KBASE_BACKEND_CONFIG_H_ ++ ++/* Enable GPU reset API */ ++#define KBASE_GPU_RESET_EN 1 ++ ++#endif /* _KBASE_BACKEND_CONFIG_H_ */ ++ +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_cache_policy_backend.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_cache_policy_backend.c +new file mode 100755 +index 000000000000..fef9a2cb743e +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_cache_policy_backend.c +@@ -0,0 +1,29 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include "backend/gpu/mali_kbase_cache_policy_backend.h" ++#include ++ ++void kbase_cache_set_coherency_mode(struct kbase_device *kbdev, ++ u32 mode) ++{ ++ kbdev->current_gpu_coherency_mode = mode; ++ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_COHERENCY_REG)) ++ kbase_reg_write(kbdev, COHERENCY_ENABLE, mode, NULL); ++} ++ +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_cache_policy_backend.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_cache_policy_backend.h +new file mode 100755 +index 000000000000..fe9869109a82 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_cache_policy_backend.h +@@ -0,0 +1,34 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++#ifndef _KBASE_CACHE_POLICY_BACKEND_H_ ++#define _KBASE_CACHE_POLICY_BACKEND_H_ ++ ++#include "mali_kbase.h" ++#include "mali_base_kernel.h" ++ ++/** ++ * kbase_cache_set_coherency_mode() - Sets the system coherency mode ++ * in the GPU. ++ * @kbdev: Device pointer ++ * @mode: Coherency mode. COHERENCY_ACE/ACE_LITE ++ */ ++void kbase_cache_set_coherency_mode(struct kbase_device *kbdev, ++ u32 mode); ++ ++#endif /* _KBASE_CACHE_POLICY_H_ */ +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_debug_job_fault_backend.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_debug_job_fault_backend.c +new file mode 100755 +index 000000000000..7851ea6466c7 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_debug_job_fault_backend.c +@@ -0,0 +1,157 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include ++#include ++#include "mali_kbase_debug_job_fault.h" ++ ++#ifdef CONFIG_DEBUG_FS ++ ++/*GPU_CONTROL_REG(r)*/ ++static int gpu_control_reg_snapshot[] = { ++ GPU_ID, ++ SHADER_READY_LO, ++ SHADER_READY_HI, ++ TILER_READY_LO, ++ TILER_READY_HI, ++ L2_READY_LO, ++ L2_READY_HI ++}; ++ ++/* JOB_CONTROL_REG(r) */ ++static int job_control_reg_snapshot[] = { ++ JOB_IRQ_MASK, ++ JOB_IRQ_STATUS ++}; ++ ++/* JOB_SLOT_REG(n,r) */ ++static int job_slot_reg_snapshot[] = { ++ JS_HEAD_LO, ++ JS_HEAD_HI, ++ JS_TAIL_LO, ++ JS_TAIL_HI, ++ JS_AFFINITY_LO, ++ JS_AFFINITY_HI, ++ JS_CONFIG, ++ JS_STATUS, ++ JS_HEAD_NEXT_LO, ++ JS_HEAD_NEXT_HI, ++ JS_AFFINITY_NEXT_LO, ++ JS_AFFINITY_NEXT_HI, ++ JS_CONFIG_NEXT ++}; ++ ++/*MMU_REG(r)*/ ++static int mmu_reg_snapshot[] = { ++ MMU_IRQ_MASK, ++ MMU_IRQ_STATUS ++}; ++ ++/* MMU_AS_REG(n,r) */ ++static int as_reg_snapshot[] = { ++ AS_TRANSTAB_LO, ++ AS_TRANSTAB_HI, ++ AS_MEMATTR_LO, ++ AS_MEMATTR_HI, ++ AS_FAULTSTATUS, ++ AS_FAULTADDRESS_LO, ++ AS_FAULTADDRESS_HI, ++ AS_STATUS ++}; ++ ++bool kbase_debug_job_fault_reg_snapshot_init(struct kbase_context *kctx, ++ int reg_range) ++{ ++ int i, j; ++ int offset = 0; ++ int slot_number; ++ int as_number; ++ ++ if (kctx->reg_dump == NULL) ++ return false; ++ ++ slot_number = kctx->kbdev->gpu_props.num_job_slots; ++ as_number = kctx->kbdev->gpu_props.num_address_spaces; ++ ++ /* get the GPU control registers*/ ++ for (i = 0; i < sizeof(gpu_control_reg_snapshot)/4; i++) { ++ kctx->reg_dump[offset] = ++ GPU_CONTROL_REG(gpu_control_reg_snapshot[i]); ++ offset += 2; ++ } ++ ++ /* get the Job control registers*/ ++ for (i = 0; i < sizeof(job_control_reg_snapshot)/4; i++) { ++ kctx->reg_dump[offset] = ++ JOB_CONTROL_REG(job_control_reg_snapshot[i]); ++ offset += 2; ++ } ++ ++ /* get the Job Slot registers*/ ++ for (j = 0; j < slot_number; j++) { ++ for (i = 0; i < sizeof(job_slot_reg_snapshot)/4; i++) { ++ kctx->reg_dump[offset] = ++ JOB_SLOT_REG(j, job_slot_reg_snapshot[i]); ++ offset += 2; ++ } ++ } ++ ++ /* get the MMU registers*/ ++ for (i = 0; i < sizeof(mmu_reg_snapshot)/4; i++) { ++ kctx->reg_dump[offset] = MMU_REG(mmu_reg_snapshot[i]); ++ offset += 2; ++ } ++ ++ /* get the Address space registers*/ ++ for (j = 0; j < as_number; j++) { ++ for (i = 0; i < sizeof(as_reg_snapshot)/4; i++) { ++ kctx->reg_dump[offset] = ++ MMU_AS_REG(j, as_reg_snapshot[i]); ++ offset += 2; ++ } ++ } ++ ++ WARN_ON(offset >= (reg_range*2/4)); ++ ++ /* set the termination flag*/ ++ kctx->reg_dump[offset] = REGISTER_DUMP_TERMINATION_FLAG; ++ kctx->reg_dump[offset + 1] = REGISTER_DUMP_TERMINATION_FLAG; ++ ++ dev_dbg(kctx->kbdev->dev, "kbase_job_fault_reg_snapshot_init:%d\n", ++ offset); ++ ++ return true; ++} ++ ++bool kbase_job_fault_get_reg_snapshot(struct kbase_context *kctx) ++{ ++ int offset = 0; ++ ++ if (kctx->reg_dump == NULL) ++ return false; ++ ++ while (kctx->reg_dump[offset] != REGISTER_DUMP_TERMINATION_FLAG) { ++ kctx->reg_dump[offset+1] = ++ kbase_reg_read(kctx->kbdev, ++ kctx->reg_dump[offset], NULL); ++ offset += 2; ++ } ++ return true; ++} ++ ++ ++#endif +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_devfreq.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_devfreq.c +new file mode 100755 +index 000000000000..ab14bc2e2ae4 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_devfreq.c +@@ -0,0 +1,458 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++#define ENABLE_DEBUG_LOG ++#include "../../platform/rk/custom_log.h" ++ ++ ++#include ++#include ++#include ++#include ++ ++#include ++#include ++#include ++#ifdef CONFIG_DEVFREQ_THERMAL ++#include ++#endif ++ ++#include ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0) ++#include ++#else /* Linux >= 3.13 */ ++/* In 3.13 the OPP include header file, types, and functions were all ++ * renamed. Use the old filename for the include, and define the new names to ++ * the old, when an old kernel is detected. ++ */ ++#include ++#define dev_pm_opp opp ++#define dev_pm_opp_get_voltage opp_get_voltage ++#define dev_pm_opp_get_opp_count opp_get_opp_count ++#define dev_pm_opp_find_freq_ceil opp_find_freq_ceil ++#define dev_pm_opp_find_freq_floor opp_find_freq_floor ++#endif /* Linux >= 3.13 */ ++#include ++#include ++ ++static struct devfreq_simple_ondemand_data ondemand_data; ++ ++static struct monitor_dev_profile mali_mdevp = { ++ .type = MONITOR_TPYE_DEV, ++ .low_temp_adjust = rockchip_monitor_dev_low_temp_adjust, ++ .high_temp_adjust = rockchip_monitor_dev_high_temp_adjust, ++}; ++ ++/** ++ * opp_translate - Translate nominal OPP frequency from devicetree into real ++ * frequency and core mask ++ * @kbdev: Device pointer ++ * @freq: Nominal frequency ++ * @core_mask: Pointer to u64 to store core mask to ++ * ++ * Return: Real target frequency ++ * ++ * This function will only perform translation if an operating-points-v2-mali ++ * table is present in devicetree. If one is not present then it will return an ++ * untranslated frequency and all cores enabled. ++ */ ++static unsigned long opp_translate(struct kbase_device *kbdev, ++ unsigned long freq, u64 *core_mask) ++{ ++ int i; ++ ++ for (i = 0; i < kbdev->num_opps; i++) { ++ if (kbdev->opp_table[i].opp_freq == freq) { ++ *core_mask = kbdev->opp_table[i].core_mask; ++ return kbdev->opp_table[i].real_freq; ++ } ++ } ++ ++ /* Failed to find OPP - return all cores enabled & nominal frequency */ ++ *core_mask = kbdev->gpu_props.props.raw_props.shader_present; ++ ++ return freq; ++} ++ ++static int ++kbase_devfreq_target(struct device *dev, unsigned long *target_freq, u32 flags) ++{ ++ struct kbase_device *kbdev = dev_get_drvdata(dev); ++ struct dev_pm_opp *opp; ++ unsigned long nominal_freq; ++ unsigned long freq = 0; ++ unsigned long old_freq = kbdev->current_freq; ++ unsigned long voltage; ++ int err; ++ u64 core_mask; ++ ++ freq = *target_freq; ++ ++ opp = devfreq_recommended_opp(dev, &freq, flags); ++ if (IS_ERR(opp)) { ++ dev_err(dev, "Failed to get opp (%ld)\n", PTR_ERR(opp)); ++ return PTR_ERR(opp); ++ } ++ voltage = dev_pm_opp_get_voltage(opp); ++ ++ nominal_freq = freq; ++ ++ /* ++ * Only update if there is a change of frequency ++ */ ++ if (kbdev->current_nominal_freq == nominal_freq) { ++ *target_freq = nominal_freq; ++#ifdef CONFIG_REGULATOR ++ if (kbdev->current_voltage == voltage) ++ return 0; ++ err = regulator_set_voltage(kbdev->regulator, voltage, INT_MAX); ++ if (err) { ++ dev_err(dev, "Failed to set voltage (%d)\n", err); ++ return err; ++ } ++ kbdev->current_voltage = voltage; ++#endif ++ return 0; ++ } ++ ++ freq = opp_translate(kbdev, nominal_freq, &core_mask); ++#ifdef CONFIG_REGULATOR ++ if (kbdev->regulator && kbdev->current_voltage != voltage && ++ old_freq < freq) { ++ err = regulator_set_voltage(kbdev->regulator, voltage, INT_MAX); ++ if (err) { ++ dev_err(dev, "Failed to increase voltage (%d)\n", err); ++ return err; ++ } ++ } ++#endif ++ ++ err = clk_set_rate(kbdev->clock, freq); ++ if (err) { ++ dev_err(dev, "Failed to set clock %lu (target %lu)\n", ++ freq, *target_freq); ++ return err; ++ } ++ *target_freq = freq; ++ kbdev->current_freq = freq; ++ if (kbdev->devfreq) ++ kbdev->devfreq->last_status.current_frequency = freq; ++#ifdef CONFIG_REGULATOR ++ if (kbdev->regulator && kbdev->current_voltage != voltage && ++ old_freq > freq) { ++ err = regulator_set_voltage(kbdev->regulator, voltage, INT_MAX); ++ if (err) { ++ dev_err(dev, "Failed to decrease voltage (%d)\n", err); ++ return err; ++ } ++ } ++#endif ++ ++ if (kbdev->pm.backend.ca_current_policy->id == ++ KBASE_PM_CA_POLICY_ID_DEVFREQ) ++ kbase_devfreq_set_core_mask(kbdev, core_mask); ++ ++ *target_freq = nominal_freq; ++ kbdev->current_voltage = voltage; ++ kbdev->current_nominal_freq = nominal_freq; ++ kbdev->current_freq = freq; ++ kbdev->current_core_mask = core_mask; ++ ++ KBASE_TLSTREAM_AUX_DEVFREQ_TARGET((u64)nominal_freq); ++ ++ kbase_pm_reset_dvfs_utilisation(kbdev); ++ ++ return err; ++} ++ ++static int ++kbase_devfreq_cur_freq(struct device *dev, unsigned long *freq) ++{ ++ struct kbase_device *kbdev = dev_get_drvdata(dev); ++ ++ *freq = kbdev->current_nominal_freq; ++ ++ return 0; ++} ++ ++static int ++kbase_devfreq_status(struct device *dev, struct devfreq_dev_status *stat) ++{ ++ struct kbase_device *kbdev = dev_get_drvdata(dev); ++ ++ stat->current_frequency = kbdev->current_nominal_freq; ++ ++ kbase_pm_get_dvfs_utilisation(kbdev, ++ &stat->total_time, &stat->busy_time); ++ ++ stat->private_data = NULL; ++ ++ return 0; ++} ++ ++static int kbase_devfreq_init_freq_table(struct kbase_device *kbdev, ++ struct devfreq_dev_profile *dp) ++{ ++ int count; ++ int i = 0; ++ unsigned long freq; ++ struct dev_pm_opp *opp; ++ ++ count = dev_pm_opp_get_opp_count(kbdev->dev); ++ if (count < 0) { ++ return count; ++ } ++ ++ dp->freq_table = kmalloc_array(count, sizeof(dp->freq_table[0]), ++ GFP_KERNEL); ++ if (!dp->freq_table) ++ return -ENOMEM; ++ ++ for (i = 0, freq = ULONG_MAX; i < count; i++, freq--) { ++ opp = dev_pm_opp_find_freq_floor(kbdev->dev, &freq); ++ if (IS_ERR(opp)) ++ break; ++ dev_pm_opp_put(opp); ++ ++ dp->freq_table[i] = freq; ++ } ++ ++ if (count != i) ++ dev_warn(kbdev->dev, "Unable to enumerate all OPPs (%d!=%d\n", ++ count, i); ++ ++ dp->max_state = i; ++ ++ return 0; ++} ++ ++static void kbase_devfreq_term_freq_table(struct kbase_device *kbdev) ++{ ++ struct devfreq_dev_profile *dp = kbdev->devfreq->profile; ++ ++ kfree(dp->freq_table); ++} ++ ++static void kbase_devfreq_exit(struct device *dev) ++{ ++ struct kbase_device *kbdev = dev_get_drvdata(dev); ++ ++ kbase_devfreq_term_freq_table(kbdev); ++} ++ ++static int kbase_devfreq_init_core_mask_table(struct kbase_device *kbdev) ++{ ++ struct device_node *opp_node = of_parse_phandle(kbdev->dev->of_node, ++ "operating-points-v2", 0); ++ struct device_node *node; ++ int i = 0; ++ int count; ++ ++ if (!opp_node) ++ return 0; ++ if (!of_device_is_compatible(opp_node, "operating-points-v2-mali")) ++ return 0; ++ ++ count = dev_pm_opp_get_opp_count(kbdev->dev); ++ kbdev->opp_table = kmalloc_array(count, ++ sizeof(struct kbase_devfreq_opp), GFP_KERNEL); ++ if (!kbdev->opp_table) ++ return -ENOMEM; ++ ++ for_each_available_child_of_node(opp_node, node) { ++ u64 core_mask; ++ u64 opp_freq, real_freq; ++ const void *core_count_p; ++ ++ if (of_property_read_u64(node, "opp-hz", &opp_freq)) { ++ dev_warn(kbdev->dev, "OPP is missing required opp-hz property\n"); ++ continue; ++ } ++ if (of_property_read_u64(node, "opp-hz-real", &real_freq)) ++ real_freq = opp_freq; ++ if (of_property_read_u64(node, "opp-core-mask", &core_mask)) ++ core_mask = ++ kbdev->gpu_props.props.raw_props.shader_present; ++ core_count_p = of_get_property(node, "opp-core-count", NULL); ++ if (core_count_p) { ++ u64 remaining_core_mask = ++ kbdev->gpu_props.props.raw_props.shader_present; ++ int core_count = be32_to_cpup(core_count_p); ++ ++ core_mask = 0; ++ ++ for (; core_count > 0; core_count--) { ++ int core = ffs(remaining_core_mask); ++ ++ if (!core) { ++ dev_err(kbdev->dev, "OPP has more cores than GPU\n"); ++ return -ENODEV; ++ } ++ ++ core_mask |= (1ull << (core-1)); ++ remaining_core_mask &= ~(1ull << (core-1)); ++ } ++ } ++ ++ if (!core_mask) { ++ dev_err(kbdev->dev, "OPP has invalid core mask of 0\n"); ++ return -ENODEV; ++ } ++ ++ kbdev->opp_table[i].opp_freq = opp_freq; ++ kbdev->opp_table[i].real_freq = real_freq; ++ kbdev->opp_table[i].core_mask = core_mask; ++ ++ dev_info(kbdev->dev, "OPP %d : opp_freq=%llu real_freq=%llu core_mask=%llx\n", ++ i, opp_freq, real_freq, core_mask); ++ ++ i++; ++ } ++ ++ kbdev->num_opps = i; ++ ++ return 0; ++} ++ ++int kbase_devfreq_init(struct kbase_device *kbdev) ++{ ++ struct device_node *np = kbdev->dev->of_node; ++ struct devfreq_dev_profile *dp; ++ struct dev_pm_opp *opp; ++ unsigned long opp_rate; ++ int err; ++ ++ if (!kbdev->clock) { ++ dev_err(kbdev->dev, "Clock not available for devfreq\n"); ++ return -ENODEV; ++ } ++ ++ kbdev->current_freq = clk_get_rate(kbdev->clock); ++ kbdev->current_nominal_freq = kbdev->current_freq; ++ ++ dp = &kbdev->devfreq_profile; ++ ++ dp->initial_freq = kbdev->current_freq; ++ /* .KP : set devfreq_dvfs_interval_in_ms */ ++ dp->polling_ms = 20; ++ dp->target = kbase_devfreq_target; ++ dp->get_dev_status = kbase_devfreq_status; ++ dp->get_cur_freq = kbase_devfreq_cur_freq; ++ dp->exit = kbase_devfreq_exit; ++ ++ if (kbase_devfreq_init_freq_table(kbdev, dp)) ++ return -EFAULT; ++ ++ err = kbase_devfreq_init_core_mask_table(kbdev); ++ if (err) ++ return err; ++ ++ of_property_read_u32(np, "upthreshold", ++ &ondemand_data.upthreshold); ++ of_property_read_u32(np, "downdifferential", ++ &ondemand_data.downdifferential); ++ ++ kbdev->devfreq = devfreq_add_device(kbdev->dev, dp, ++ "simple_ondemand", &ondemand_data); ++ if (IS_ERR(kbdev->devfreq)) { ++ kbase_devfreq_term_freq_table(kbdev); ++ return PTR_ERR(kbdev->devfreq); ++ } ++ ++ /* devfreq_add_device only copies a few of kbdev->dev's fields, so ++ * set drvdata explicitly so IPA models can access kbdev. */ ++ dev_set_drvdata(&kbdev->devfreq->dev, kbdev); ++ ++ err = devfreq_register_opp_notifier(kbdev->dev, kbdev->devfreq); ++ if (err) { ++ dev_err(kbdev->dev, ++ "Failed to register OPP notifier (%d)\n", err); ++ goto opp_notifier_failed; ++ } ++ ++ opp_rate = kbdev->current_freq; ++ opp = devfreq_recommended_opp(kbdev->dev, &opp_rate, 0); ++ if (!IS_ERR(opp)) ++ dev_pm_opp_put(opp); ++ kbdev->devfreq->last_status.current_frequency = opp_rate; ++ ++ mali_mdevp.data = kbdev->devfreq; ++ kbdev->mdev_info = rockchip_system_monitor_register(kbdev->dev, ++ &mali_mdevp); ++ if (IS_ERR(kbdev->mdev_info)) { ++ dev_dbg(kbdev->dev, "without system monitor\n"); ++ kbdev->mdev_info = NULL; ++ } ++#ifdef CONFIG_DEVFREQ_THERMAL ++ err = kbase_ipa_init(kbdev); ++ if (err) { ++ dev_err(kbdev->dev, "IPA initialization failed\n"); ++ goto cooling_failed; ++ } ++ ++ kbdev->devfreq_cooling = of_devfreq_cooling_register_power( ++ kbdev->dev->of_node, ++ kbdev->devfreq, ++ &kbase_ipa_power_model_ops); ++ if (IS_ERR_OR_NULL(kbdev->devfreq_cooling)) { ++ err = PTR_ERR(kbdev->devfreq_cooling); ++ dev_err(kbdev->dev, ++ "Failed to register cooling device (%d)\n", ++ err); ++ goto cooling_failed; ++ } ++ I("success initing power_model_simple."); ++#endif ++ ++ return 0; ++ ++#ifdef CONFIG_DEVFREQ_THERMAL ++cooling_failed: ++ devfreq_unregister_opp_notifier(kbdev->dev, kbdev->devfreq); ++#endif /* CONFIG_DEVFREQ_THERMAL */ ++opp_notifier_failed: ++ if (devfreq_remove_device(kbdev->devfreq)) ++ dev_err(kbdev->dev, "Failed to terminate devfreq (%d)\n", err); ++ else ++ kbdev->devfreq = NULL; ++ ++ return err; ++} ++ ++void kbase_devfreq_term(struct kbase_device *kbdev) ++{ ++ int err; ++ ++ dev_dbg(kbdev->dev, "Term Mali devfreq\n"); ++ ++ rockchip_system_monitor_unregister(kbdev->mdev_info); ++#ifdef CONFIG_DEVFREQ_THERMAL ++ if (kbdev->devfreq_cooling) ++ devfreq_cooling_unregister(kbdev->devfreq_cooling); ++ ++ kbase_ipa_term(kbdev); ++#endif ++ ++ devfreq_unregister_opp_notifier(kbdev->dev, kbdev->devfreq); ++ ++ err = devfreq_remove_device(kbdev->devfreq); ++ if (err) ++ dev_err(kbdev->dev, "Failed to terminate devfreq (%d)\n", err); ++ else ++ kbdev->devfreq = NULL; ++ ++ kfree(kbdev->opp_table); ++} +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_devfreq.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_devfreq.h +new file mode 100755 +index 000000000000..c0bf8b15b3bc +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_devfreq.h +@@ -0,0 +1,24 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _BASE_DEVFREQ_H_ ++#define _BASE_DEVFREQ_H_ ++ ++int kbase_devfreq_init(struct kbase_device *kbdev); ++void kbase_devfreq_term(struct kbase_device *kbdev); ++ ++#endif /* _BASE_DEVFREQ_H_ */ +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_device_hw.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_device_hw.c +new file mode 100755 +index 000000000000..dcdf15cdc3e8 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_device_hw.c +@@ -0,0 +1,255 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++/* ++ * ++ */ ++#include ++#include ++#include ++ ++#include ++ ++#if !defined(CONFIG_MALI_NO_MALI) ++ ++ ++#ifdef CONFIG_DEBUG_FS ++ ++ ++int kbase_io_history_resize(struct kbase_io_history *h, u16 new_size) ++{ ++ struct kbase_io_access *old_buf; ++ struct kbase_io_access *new_buf; ++ unsigned long flags; ++ ++ if (!new_size) ++ goto out_err; /* The new size must not be 0 */ ++ ++ new_buf = vmalloc(new_size * sizeof(*h->buf)); ++ if (!new_buf) ++ goto out_err; ++ ++ spin_lock_irqsave(&h->lock, flags); ++ ++ old_buf = h->buf; ++ ++ /* Note: we won't bother with copying the old data over. The dumping ++ * logic wouldn't work properly as it relies on 'count' both as a ++ * counter and as an index to the buffer which would have changed with ++ * the new array. This is a corner case that we don't need to support. ++ */ ++ h->count = 0; ++ h->size = new_size; ++ h->buf = new_buf; ++ ++ spin_unlock_irqrestore(&h->lock, flags); ++ ++ vfree(old_buf); ++ ++ return 0; ++ ++out_err: ++ return -1; ++} ++ ++ ++int kbase_io_history_init(struct kbase_io_history *h, u16 n) ++{ ++ h->enabled = false; ++ spin_lock_init(&h->lock); ++ h->count = 0; ++ h->size = 0; ++ h->buf = NULL; ++ if (kbase_io_history_resize(h, n)) ++ return -1; ++ ++ return 0; ++} ++ ++ ++void kbase_io_history_term(struct kbase_io_history *h) ++{ ++ vfree(h->buf); ++ h->buf = NULL; ++} ++ ++ ++/* kbase_io_history_add - add new entry to the register access history ++ * ++ * @h: Pointer to the history data structure ++ * @addr: Register address ++ * @value: The value that is either read from or written to the register ++ * @write: 1 if it's a register write, 0 if it's a read ++ */ ++static void kbase_io_history_add(struct kbase_io_history *h, ++ void __iomem const *addr, u32 value, u8 write) ++{ ++ struct kbase_io_access *io; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&h->lock, flags); ++ ++ io = &h->buf[h->count % h->size]; ++ io->addr = (uintptr_t)addr | write; ++ io->value = value; ++ ++h->count; ++ /* If count overflows, move the index by the buffer size so the entire ++ * buffer will still be dumped later */ ++ if (unlikely(!h->count)) ++ h->count = h->size; ++ ++ spin_unlock_irqrestore(&h->lock, flags); ++} ++ ++ ++void kbase_io_history_dump(struct kbase_device *kbdev) ++{ ++ struct kbase_io_history *const h = &kbdev->io_history; ++ u16 i; ++ size_t iters; ++ unsigned long flags; ++ ++ if (!unlikely(h->enabled)) ++ return; ++ ++ spin_lock_irqsave(&h->lock, flags); ++ ++ dev_err(kbdev->dev, "Register IO History:"); ++ iters = (h->size > h->count) ? h->count : h->size; ++ dev_err(kbdev->dev, "Last %zu register accesses of %zu total:\n", iters, ++ h->count); ++ for (i = 0; i < iters; ++i) { ++ struct kbase_io_access *io = ++ &h->buf[(h->count - iters + i) % h->size]; ++ char const access = (io->addr & 1) ? 'w' : 'r'; ++ ++ dev_err(kbdev->dev, "%6i: %c: reg 0x%p val %08x\n", i, access, ++ (void *)(io->addr & ~0x1), io->value); ++ } ++ ++ spin_unlock_irqrestore(&h->lock, flags); ++} ++ ++ ++#endif /* CONFIG_DEBUG_FS */ ++ ++ ++void kbase_reg_write(struct kbase_device *kbdev, u16 offset, u32 value, ++ struct kbase_context *kctx) ++{ ++ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered); ++ KBASE_DEBUG_ASSERT(kctx == NULL || kctx->as_nr != KBASEP_AS_NR_INVALID); ++ KBASE_DEBUG_ASSERT(kbdev->dev != NULL); ++ ++ writel(value, kbdev->reg + offset); ++ ++#ifdef CONFIG_DEBUG_FS ++ if (unlikely(kbdev->io_history.enabled)) ++ kbase_io_history_add(&kbdev->io_history, kbdev->reg + offset, ++ value, 1); ++#endif /* CONFIG_DEBUG_FS */ ++ dev_dbg(kbdev->dev, "w: reg %04x val %08x", offset, value); ++ ++ if (kctx && kctx->jctx.tb) ++ kbase_device_trace_register_access(kctx, REG_WRITE, offset, ++ value); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_reg_write); ++ ++u32 kbase_reg_read(struct kbase_device *kbdev, u16 offset, ++ struct kbase_context *kctx) ++{ ++ u32 val; ++ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered); ++ KBASE_DEBUG_ASSERT(kctx == NULL || kctx->as_nr != KBASEP_AS_NR_INVALID); ++ KBASE_DEBUG_ASSERT(kbdev->dev != NULL); ++ ++ val = readl(kbdev->reg + offset); ++ ++#ifdef CONFIG_DEBUG_FS ++ if (unlikely(kbdev->io_history.enabled)) ++ kbase_io_history_add(&kbdev->io_history, kbdev->reg + offset, ++ val, 0); ++#endif /* CONFIG_DEBUG_FS */ ++ dev_dbg(kbdev->dev, "r: reg %04x val %08x", offset, val); ++ ++ if (kctx && kctx->jctx.tb) ++ kbase_device_trace_register_access(kctx, REG_READ, offset, val); ++ return val; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_reg_read); ++#endif /* !defined(CONFIG_MALI_NO_MALI) */ ++ ++/** ++ * kbase_report_gpu_fault - Report a GPU fault. ++ * @kbdev: Kbase device pointer ++ * @multiple: Zero if only GPU_FAULT was raised, non-zero if MULTIPLE_GPU_FAULTS ++ * was also set ++ * ++ * This function is called from the interrupt handler when a GPU fault occurs. ++ * It reports the details of the fault using dev_warn(). ++ */ ++static void kbase_report_gpu_fault(struct kbase_device *kbdev, int multiple) ++{ ++ u32 status; ++ u64 address; ++ ++ status = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_FAULTSTATUS), NULL); ++ address = (u64) kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(GPU_FAULTADDRESS_HI), NULL) << 32; ++ address |= kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(GPU_FAULTADDRESS_LO), NULL); ++ ++ dev_warn(kbdev->dev, "GPU Fault 0x%08x (%s) at 0x%016llx", ++ status & 0xFF, ++ kbase_exception_name(kbdev, status), ++ address); ++ if (multiple) ++ dev_warn(kbdev->dev, "There were multiple GPU faults - some have not been reported\n"); ++} ++ ++void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val) ++{ ++ KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ, NULL, NULL, 0u, val); ++ if (val & GPU_FAULT) ++ kbase_report_gpu_fault(kbdev, val & MULTIPLE_GPU_FAULTS); ++ ++ if (val & RESET_COMPLETED) ++ kbase_pm_reset_done(kbdev); ++ ++ if (val & PRFCNT_SAMPLE_COMPLETED) ++ kbase_instr_hwcnt_sample_done(kbdev); ++ ++ if (val & CLEAN_CACHES_COMPLETED) ++ kbase_clean_caches_done(kbdev); ++ ++ KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, NULL, 0u, val); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), val, NULL); ++ ++ /* kbase_pm_check_transitions must be called after the IRQ has been ++ * cleared. This is because it might trigger further power transitions ++ * and we don't want to miss the interrupt raised to notify us that ++ * these further transitions have finished. ++ */ ++ if (val & POWER_CHANGED_ALL) ++ kbase_pm_power_changed(kbdev); ++ ++ KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ_DONE, NULL, NULL, 0u, val); ++} +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_device_internal.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_device_internal.h +new file mode 100755 +index 000000000000..5b20445932fb +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_device_internal.h +@@ -0,0 +1,67 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * Backend-specific HW access device APIs ++ */ ++ ++#ifndef _KBASE_DEVICE_INTERNAL_H_ ++#define _KBASE_DEVICE_INTERNAL_H_ ++ ++/** ++ * kbase_reg_write - write to GPU register ++ * @kbdev: Kbase device pointer ++ * @offset: Offset of register ++ * @value: Value to write ++ * @kctx: Kbase context pointer. May be NULL ++ * ++ * Caller must ensure the GPU is powered (@kbdev->pm.gpu_powered != false). If ++ * @kctx is not NULL then the caller must ensure it is scheduled (@kctx->as_nr ++ * != KBASEP_AS_NR_INVALID). ++ */ ++void kbase_reg_write(struct kbase_device *kbdev, u16 offset, u32 value, ++ struct kbase_context *kctx); ++ ++/** ++ * kbase_reg_read - read from GPU register ++ * @kbdev: Kbase device pointer ++ * @offset: Offset of register ++ * @kctx: Kbase context pointer. May be NULL ++ * ++ * Caller must ensure the GPU is powered (@kbdev->pm.gpu_powered != false). If ++ * @kctx is not NULL then the caller must ensure it is scheduled (@kctx->as_nr ++ * != KBASEP_AS_NR_INVALID). ++ * ++ * Return: Value in desired register ++ */ ++u32 kbase_reg_read(struct kbase_device *kbdev, u16 offset, ++ struct kbase_context *kctx); ++ ++ ++/** ++ * kbase_gpu_interrupt - GPU interrupt handler ++ * @kbdev: Kbase device pointer ++ * @val: The value of the GPU IRQ status register which triggered the call ++ * ++ * This function is called from the interrupt handler when a GPU irq is to be ++ * handled. ++ */ ++void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val); ++ ++#endif /* _KBASE_DEVICE_INTERNAL_H_ */ +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_gpu.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_gpu.c +new file mode 100755 +index 000000000000..d578fd78e825 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_gpu.c +@@ -0,0 +1,123 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++/* ++ * Register-based HW access backend APIs ++ */ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++int kbase_backend_early_init(struct kbase_device *kbdev) ++{ ++ int err; ++ ++ err = kbasep_platform_device_init(kbdev); ++ if (err) ++ return err; ++ ++ /* Ensure we can access the GPU registers */ ++ kbase_pm_register_access_enable(kbdev); ++ ++ /* Find out GPU properties based on the GPU feature registers */ ++ kbase_gpuprops_set(kbdev); ++ ++ /* We're done accessing the GPU registers for now. */ ++ kbase_pm_register_access_disable(kbdev); ++ ++ err = kbase_hwaccess_pm_init(kbdev); ++ if (err) ++ goto fail_pm; ++ ++ err = kbase_install_interrupts(kbdev); ++ if (err) ++ goto fail_interrupts; ++ ++ return 0; ++ ++fail_interrupts: ++ kbase_hwaccess_pm_term(kbdev); ++fail_pm: ++ kbasep_platform_device_term(kbdev); ++ ++ return err; ++} ++ ++void kbase_backend_early_term(struct kbase_device *kbdev) ++{ ++ kbase_release_interrupts(kbdev); ++ kbase_hwaccess_pm_term(kbdev); ++ kbasep_platform_device_term(kbdev); ++} ++ ++int kbase_backend_late_init(struct kbase_device *kbdev) ++{ ++ int err; ++ ++ err = kbase_hwaccess_pm_powerup(kbdev, PM_HW_ISSUES_DETECT); ++ if (err) ++ return err; ++ ++ err = kbase_backend_timer_init(kbdev); ++ if (err) ++ goto fail_timer; ++ ++#ifdef CONFIG_MALI_DEBUG ++#ifndef CONFIG_MALI_NO_MALI ++ if (kbasep_common_test_interrupt_handlers(kbdev) != 0) { ++ dev_err(kbdev->dev, "Interrupt assigment check failed.\n"); ++ err = -EINVAL; ++ goto fail_interrupt_test; ++ } ++#endif /* !CONFIG_MALI_NO_MALI */ ++#endif /* CONFIG_MALI_DEBUG */ ++ ++ err = kbase_job_slot_init(kbdev); ++ if (err) ++ goto fail_job_slot; ++ ++ init_waitqueue_head(&kbdev->hwaccess.backend.reset_wait); ++ ++ return 0; ++ ++fail_job_slot: ++ ++#ifdef CONFIG_MALI_DEBUG ++#ifndef CONFIG_MALI_NO_MALI ++fail_interrupt_test: ++#endif /* !CONFIG_MALI_NO_MALI */ ++#endif /* CONFIG_MALI_DEBUG */ ++ ++ kbase_backend_timer_term(kbdev); ++fail_timer: ++ kbase_hwaccess_pm_halt(kbdev); ++ ++ return err; ++} ++ ++void kbase_backend_late_term(struct kbase_device *kbdev) ++{ ++ kbase_job_slot_halt(kbdev); ++ kbase_job_slot_term(kbdev); ++ kbase_backend_timer_term(kbdev); ++ kbase_hwaccess_pm_halt(kbdev); ++} ++ +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_gpuprops_backend.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_gpuprops_backend.c +new file mode 100755 +index 000000000000..b395325b556b +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_gpuprops_backend.c +@@ -0,0 +1,110 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * Base kernel property query backend APIs ++ */ ++ ++#include ++#include ++#include ++#include ++ ++void kbase_backend_gpuprops_get(struct kbase_device *kbdev, ++ struct kbase_gpuprops_regdump *regdump) ++{ ++ int i; ++ ++ /* Fill regdump with the content of the relevant registers */ ++ regdump->gpu_id = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_ID), NULL); ++ ++ regdump->l2_features = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(L2_FEATURES), NULL); ++ regdump->suspend_size = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(SUSPEND_SIZE), NULL); ++ regdump->tiler_features = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(TILER_FEATURES), NULL); ++ regdump->mem_features = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(MEM_FEATURES), NULL); ++ regdump->mmu_features = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(MMU_FEATURES), NULL); ++ regdump->as_present = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(AS_PRESENT), NULL); ++ regdump->js_present = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(JS_PRESENT), NULL); ++ ++ for (i = 0; i < GPU_MAX_JOB_SLOTS; i++) ++ regdump->js_features[i] = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(JS_FEATURES_REG(i)), NULL); ++ ++ for (i = 0; i < BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS; i++) ++ regdump->texture_features[i] = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(TEXTURE_FEATURES_REG(i)), NULL); ++ ++ regdump->thread_max_threads = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(THREAD_MAX_THREADS), NULL); ++ regdump->thread_max_workgroup_size = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(THREAD_MAX_WORKGROUP_SIZE), ++ NULL); ++ regdump->thread_max_barrier_size = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(THREAD_MAX_BARRIER_SIZE), NULL); ++ regdump->thread_features = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(THREAD_FEATURES), NULL); ++ ++ regdump->shader_present_lo = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(SHADER_PRESENT_LO), NULL); ++ regdump->shader_present_hi = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(SHADER_PRESENT_HI), NULL); ++ ++ regdump->tiler_present_lo = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(TILER_PRESENT_LO), NULL); ++ regdump->tiler_present_hi = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(TILER_PRESENT_HI), NULL); ++ ++ regdump->l2_present_lo = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(L2_PRESENT_LO), NULL); ++ regdump->l2_present_hi = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(L2_PRESENT_HI), NULL); ++ ++ regdump->stack_present_lo = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(STACK_PRESENT_LO), NULL); ++ regdump->stack_present_hi = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(STACK_PRESENT_HI), NULL); ++} ++ ++void kbase_backend_gpuprops_get_features(struct kbase_device *kbdev, ++ struct kbase_gpuprops_regdump *regdump) ++{ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_COHERENCY_REG)) { ++ /* Ensure we can access the GPU registers */ ++ kbase_pm_register_access_enable(kbdev); ++ ++ regdump->coherency_features = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(COHERENCY_FEATURES), NULL); ++ ++ /* We're done accessing the GPU registers for now. */ ++ kbase_pm_register_access_disable(kbdev); ++ } else { ++ /* Pre COHERENCY_FEATURES we only supported ACE_LITE */ ++ regdump->coherency_features = ++ COHERENCY_FEATURE_BIT(COHERENCY_NONE) | ++ COHERENCY_FEATURE_BIT(COHERENCY_ACE_LITE); ++ } ++} ++ +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_instr_backend.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_instr_backend.c +new file mode 100755 +index 000000000000..7ad309e8d7f4 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_instr_backend.c +@@ -0,0 +1,492 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * GPU backend instrumentation APIs. ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++/** ++ * kbasep_instr_hwcnt_cacheclean - Issue Cache Clean & Invalidate command to ++ * hardware ++ * ++ * @kbdev: Kbase device ++ */ ++static void kbasep_instr_hwcnt_cacheclean(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ unsigned long pm_flags; ++ u32 irq_mask; ++ ++ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); ++ KBASE_DEBUG_ASSERT(kbdev->hwcnt.backend.state == ++ KBASE_INSTR_STATE_REQUEST_CLEAN); ++ ++ /* Enable interrupt */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, pm_flags); ++ irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), ++ irq_mask | CLEAN_CACHES_COMPLETED, NULL); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); ++ ++ /* clean&invalidate the caches so we're sure the mmu tables for the dump ++ * buffer is valid */ ++ KBASE_TRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, NULL, 0u, 0); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), ++ GPU_COMMAND_CLEAN_INV_CACHES, NULL); ++ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_CLEANING; ++ ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++} ++ ++int kbase_instr_hwcnt_enable_internal(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ struct kbase_uk_hwcnt_setup *setup) ++{ ++ unsigned long flags, pm_flags; ++ int err = -EINVAL; ++ u32 irq_mask; ++ int ret; ++ u64 shader_cores_needed; ++ u32 prfcnt_config; ++ ++ shader_cores_needed = kbase_pm_get_present_cores(kbdev, ++ KBASE_PM_CORE_SHADER); ++ ++ /* alignment failure */ ++ if ((setup->dump_buffer == 0ULL) || (setup->dump_buffer & (2048 - 1))) ++ goto out_err; ++ ++ /* Override core availability policy to ensure all cores are available ++ */ ++ kbase_pm_ca_instr_enable(kbdev); ++ ++ /* Request the cores early on synchronously - we'll release them on any ++ * errors (e.g. instrumentation already active) */ ++ kbase_pm_request_cores_sync(kbdev, true, shader_cores_needed); ++ ++ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); ++ ++ if (kbdev->hwcnt.backend.state != KBASE_INSTR_STATE_DISABLED) { ++ /* Instrumentation is already enabled */ ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++ goto out_unrequest_cores; ++ } ++ ++ /* Enable interrupt */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, pm_flags); ++ irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), irq_mask | ++ PRFCNT_SAMPLE_COMPLETED, NULL); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); ++ ++ /* In use, this context is the owner */ ++ kbdev->hwcnt.kctx = kctx; ++ /* Remember the dump address so we can reprogram it later */ ++ kbdev->hwcnt.addr = setup->dump_buffer; ++ ++ /* Request the clean */ ++ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_REQUEST_CLEAN; ++ kbdev->hwcnt.backend.triggered = 0; ++ /* Clean&invalidate the caches so we're sure the mmu tables for the dump ++ * buffer is valid */ ++ ret = queue_work(kbdev->hwcnt.backend.cache_clean_wq, ++ &kbdev->hwcnt.backend.cache_clean_work); ++ KBASE_DEBUG_ASSERT(ret); ++ ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++ ++ /* Wait for cacheclean to complete */ ++ wait_event(kbdev->hwcnt.backend.wait, ++ kbdev->hwcnt.backend.triggered != 0); ++ ++ KBASE_DEBUG_ASSERT(kbdev->hwcnt.backend.state == ++ KBASE_INSTR_STATE_IDLE); ++ ++ kbase_pm_request_l2_caches(kbdev); ++ ++ /* Configure */ ++ prfcnt_config = kctx->as_nr << PRFCNT_CONFIG_AS_SHIFT; ++#ifdef CONFIG_MALI_PRFCNT_SET_SECONDARY ++ { ++ u32 gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; ++ u32 product_id = (gpu_id & GPU_ID_VERSION_PRODUCT_ID) ++ >> GPU_ID_VERSION_PRODUCT_ID_SHIFT; ++ int arch_v6 = GPU_ID_IS_NEW_FORMAT(product_id); ++ ++ if (arch_v6) ++ prfcnt_config |= 1 << PRFCNT_CONFIG_SETSELECT_SHIFT; ++ } ++#endif ++ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), ++ prfcnt_config | PRFCNT_CONFIG_MODE_OFF, kctx); ++ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_LO), ++ setup->dump_buffer & 0xFFFFFFFF, kctx); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_HI), ++ setup->dump_buffer >> 32, kctx); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_JM_EN), ++ setup->jm_bm, kctx); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_SHADER_EN), ++ setup->shader_bm, kctx); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_MMU_L2_EN), ++ setup->mmu_l2_bm, kctx); ++ /* Due to PRLAM-8186 we need to disable the Tiler before we enable the ++ * HW counter dump. */ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8186)) ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), 0, ++ kctx); ++ else ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), ++ setup->tiler_bm, kctx); ++ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), ++ prfcnt_config | PRFCNT_CONFIG_MODE_MANUAL, kctx); ++ ++ /* If HW has PRLAM-8186 we can now re-enable the tiler HW counters dump ++ */ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8186)) ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), ++ setup->tiler_bm, kctx); ++ ++ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); ++ ++ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; ++ kbdev->hwcnt.backend.triggered = 1; ++ wake_up(&kbdev->hwcnt.backend.wait); ++ ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++ ++ err = 0; ++ ++ dev_dbg(kbdev->dev, "HW counters dumping set-up for context %p", kctx); ++ return err; ++ out_unrequest_cores: ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_pm_unrequest_cores(kbdev, true, shader_cores_needed); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ out_err: ++ return err; ++} ++ ++int kbase_instr_hwcnt_disable_internal(struct kbase_context *kctx) ++{ ++ unsigned long flags, pm_flags; ++ int err = -EINVAL; ++ u32 irq_mask; ++ struct kbase_device *kbdev = kctx->kbdev; ++ ++ while (1) { ++ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); ++ ++ if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_DISABLED) { ++ /* Instrumentation is not enabled */ ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++ goto out; ++ } ++ ++ if (kbdev->hwcnt.kctx != kctx) { ++ /* Instrumentation has been setup for another context */ ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++ goto out; ++ } ++ ++ if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_IDLE) ++ break; ++ ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++ ++ /* Ongoing dump/setup - wait for its completion */ ++ wait_event(kbdev->hwcnt.backend.wait, ++ kbdev->hwcnt.backend.triggered != 0); ++ } ++ ++ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DISABLED; ++ kbdev->hwcnt.backend.triggered = 0; ++ ++ /* Disable interrupt */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, pm_flags); ++ irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), ++ irq_mask & ~PRFCNT_SAMPLE_COMPLETED, NULL); ++ ++ /* Disable the counters */ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), 0, kctx); ++ ++ kbdev->hwcnt.kctx = NULL; ++ kbdev->hwcnt.addr = 0ULL; ++ ++ kbase_pm_ca_instr_disable(kbdev); ++ ++ kbase_pm_unrequest_cores(kbdev, true, ++ kbase_pm_get_present_cores(kbdev, KBASE_PM_CORE_SHADER)); ++ ++ kbase_pm_release_l2_caches(kbdev); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++ ++ dev_dbg(kbdev->dev, "HW counters dumping disabled for context %p", ++ kctx); ++ ++ err = 0; ++ ++ out: ++ return err; ++} ++ ++int kbase_instr_hwcnt_request_dump(struct kbase_context *kctx) ++{ ++ unsigned long flags; ++ int err = -EINVAL; ++ struct kbase_device *kbdev = kctx->kbdev; ++ ++ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); ++ ++ if (kbdev->hwcnt.kctx != kctx) { ++ /* The instrumentation has been setup for another context */ ++ goto unlock; ++ } ++ ++ if (kbdev->hwcnt.backend.state != KBASE_INSTR_STATE_IDLE) { ++ /* HW counters are disabled or another dump is ongoing, or we're ++ * resetting */ ++ goto unlock; ++ } ++ ++ kbdev->hwcnt.backend.triggered = 0; ++ ++ /* Mark that we're dumping - the PF handler can signal that we faulted ++ */ ++ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DUMPING; ++ ++ /* Reconfigure the dump address */ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_LO), ++ kbdev->hwcnt.addr & 0xFFFFFFFF, NULL); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_HI), ++ kbdev->hwcnt.addr >> 32, NULL); ++ ++ /* Start dumping */ ++ KBASE_TRACE_ADD(kbdev, CORE_GPU_PRFCNT_SAMPLE, NULL, NULL, ++ kbdev->hwcnt.addr, 0); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), ++ GPU_COMMAND_PRFCNT_SAMPLE, kctx); ++ ++ dev_dbg(kbdev->dev, "HW counters dumping done for context %p", kctx); ++ ++ err = 0; ++ ++ unlock: ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++ return err; ++} ++KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_request_dump); ++ ++bool kbase_instr_hwcnt_dump_complete(struct kbase_context *kctx, ++ bool * const success) ++{ ++ unsigned long flags; ++ bool complete = false; ++ struct kbase_device *kbdev = kctx->kbdev; ++ ++ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); ++ ++ if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_IDLE) { ++ *success = true; ++ complete = true; ++ } else if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_FAULT) { ++ *success = false; ++ complete = true; ++ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; ++ } ++ ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++ ++ return complete; ++} ++KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_dump_complete); ++ ++void kbasep_cache_clean_worker(struct work_struct *data) ++{ ++ struct kbase_device *kbdev; ++ unsigned long flags; ++ ++ kbdev = container_of(data, struct kbase_device, ++ hwcnt.backend.cache_clean_work); ++ ++ mutex_lock(&kbdev->cacheclean_lock); ++ kbasep_instr_hwcnt_cacheclean(kbdev); ++ ++ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); ++ /* Wait for our condition, and any reset to complete */ ++ while (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_CLEANING) { ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++ wait_event(kbdev->hwcnt.backend.cache_clean_wait, ++ kbdev->hwcnt.backend.state != ++ KBASE_INSTR_STATE_CLEANING); ++ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); ++ } ++ KBASE_DEBUG_ASSERT(kbdev->hwcnt.backend.state == ++ KBASE_INSTR_STATE_CLEANED); ++ ++ /* All finished and idle */ ++ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; ++ kbdev->hwcnt.backend.triggered = 1; ++ wake_up(&kbdev->hwcnt.backend.wait); ++ ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++ mutex_unlock(&kbdev->cacheclean_lock); ++} ++ ++void kbase_instr_hwcnt_sample_done(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); ++ ++ if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_FAULT) { ++ kbdev->hwcnt.backend.triggered = 1; ++ wake_up(&kbdev->hwcnt.backend.wait); ++ } else if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_DUMPING) { ++ int ret; ++ /* Always clean and invalidate the cache after a successful dump ++ */ ++ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_REQUEST_CLEAN; ++ ret = queue_work(kbdev->hwcnt.backend.cache_clean_wq, ++ &kbdev->hwcnt.backend.cache_clean_work); ++ KBASE_DEBUG_ASSERT(ret); ++ } ++ ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++} ++ ++void kbase_clean_caches_done(struct kbase_device *kbdev) ++{ ++ u32 irq_mask; ++ ++ if (kbdev->hwcnt.backend.state != KBASE_INSTR_STATE_DISABLED) { ++ unsigned long flags; ++ unsigned long pm_flags; ++ ++ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); ++ /* Disable interrupt */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, pm_flags); ++ irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), ++ NULL); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), ++ irq_mask & ~CLEAN_CACHES_COMPLETED, NULL); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); ++ ++ /* Wakeup... */ ++ if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_CLEANING) { ++ /* Only wake if we weren't resetting */ ++ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_CLEANED; ++ wake_up(&kbdev->hwcnt.backend.cache_clean_wait); ++ } ++ ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++ } ++} ++ ++int kbase_instr_hwcnt_wait_for_dump(struct kbase_context *kctx) ++{ ++ struct kbase_device *kbdev = kctx->kbdev; ++ unsigned long flags; ++ int err; ++ ++ /* Wait for dump & cacheclean to complete */ ++ wait_event(kbdev->hwcnt.backend.wait, ++ kbdev->hwcnt.backend.triggered != 0); ++ ++ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); ++ ++ if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_FAULT) { ++ err = -EINVAL; ++ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; ++ } else { ++ /* Dump done */ ++ KBASE_DEBUG_ASSERT(kbdev->hwcnt.backend.state == ++ KBASE_INSTR_STATE_IDLE); ++ err = 0; ++ } ++ ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++ ++ return err; ++} ++ ++int kbase_instr_hwcnt_clear(struct kbase_context *kctx) ++{ ++ unsigned long flags; ++ int err = -EINVAL; ++ struct kbase_device *kbdev = kctx->kbdev; ++ ++ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); ++ ++ /* Check it's the context previously set up and we're not already ++ * dumping */ ++ if (kbdev->hwcnt.kctx != kctx || kbdev->hwcnt.backend.state != ++ KBASE_INSTR_STATE_IDLE) ++ goto out; ++ ++ /* Clear the counters */ ++ KBASE_TRACE_ADD(kbdev, CORE_GPU_PRFCNT_CLEAR, NULL, NULL, 0u, 0); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), ++ GPU_COMMAND_PRFCNT_CLEAR, kctx); ++ ++ err = 0; ++ ++out: ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++ return err; ++} ++KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_clear); ++ ++int kbase_instr_backend_init(struct kbase_device *kbdev) ++{ ++ int ret = 0; ++ ++ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DISABLED; ++ ++ init_waitqueue_head(&kbdev->hwcnt.backend.wait); ++ init_waitqueue_head(&kbdev->hwcnt.backend.cache_clean_wait); ++ INIT_WORK(&kbdev->hwcnt.backend.cache_clean_work, ++ kbasep_cache_clean_worker); ++ kbdev->hwcnt.backend.triggered = 0; ++ ++ kbdev->hwcnt.backend.cache_clean_wq = ++ alloc_workqueue("Mali cache cleaning workqueue", 0, 1); ++ if (NULL == kbdev->hwcnt.backend.cache_clean_wq) ++ ret = -EINVAL; ++ ++ return ret; ++} ++ ++void kbase_instr_backend_term(struct kbase_device *kbdev) ++{ ++ destroy_workqueue(kbdev->hwcnt.backend.cache_clean_wq); ++} ++ +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_instr_defs.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_instr_defs.h +new file mode 100755 +index 000000000000..4794672da8f0 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_instr_defs.h +@@ -0,0 +1,58 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014, 2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* ++ * Backend-specific instrumentation definitions ++ */ ++ ++#ifndef _KBASE_INSTR_DEFS_H_ ++#define _KBASE_INSTR_DEFS_H_ ++ ++/* ++ * Instrumentation State Machine States ++ */ ++enum kbase_instr_state { ++ /* State where instrumentation is not active */ ++ KBASE_INSTR_STATE_DISABLED = 0, ++ /* State machine is active and ready for a command. */ ++ KBASE_INSTR_STATE_IDLE, ++ /* Hardware is currently dumping a frame. */ ++ KBASE_INSTR_STATE_DUMPING, ++ /* We've requested a clean to occur on a workqueue */ ++ KBASE_INSTR_STATE_REQUEST_CLEAN, ++ /* Hardware is currently cleaning and invalidating caches. */ ++ KBASE_INSTR_STATE_CLEANING, ++ /* Cache clean completed, and either a) a dump is complete, or ++ * b) instrumentation can now be setup. */ ++ KBASE_INSTR_STATE_CLEANED, ++ /* An error has occured during DUMPING (page fault). */ ++ KBASE_INSTR_STATE_FAULT ++}; ++ ++/* Structure used for instrumentation and HW counters dumping */ ++struct kbase_instr_backend { ++ wait_queue_head_t wait; ++ int triggered; ++ ++ enum kbase_instr_state state; ++ wait_queue_head_t cache_clean_wait; ++ struct workqueue_struct *cache_clean_wq; ++ struct work_struct cache_clean_work; ++}; ++ ++#endif /* _KBASE_INSTR_DEFS_H_ */ ++ +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_instr_internal.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_instr_internal.h +new file mode 100755 +index 000000000000..e96aeae786e1 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_instr_internal.h +@@ -0,0 +1,45 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * Backend-specific HW access instrumentation APIs ++ */ ++ ++#ifndef _KBASE_INSTR_INTERNAL_H_ ++#define _KBASE_INSTR_INTERNAL_H_ ++ ++/** ++ * kbasep_cache_clean_worker() - Workqueue for handling cache cleaning ++ * @data: a &struct work_struct ++ */ ++void kbasep_cache_clean_worker(struct work_struct *data); ++ ++/** ++ * kbase_clean_caches_done() - Cache clean interrupt received ++ * @kbdev: Kbase device ++ */ ++void kbase_clean_caches_done(struct kbase_device *kbdev); ++ ++/** ++ * kbase_instr_hwcnt_sample_done() - Dump complete interrupt received ++ * @kbdev: Kbase device ++ */ ++void kbase_instr_hwcnt_sample_done(struct kbase_device *kbdev); ++ ++#endif /* _KBASE_INSTR_INTERNAL_H_ */ +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_irq_internal.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_irq_internal.h +new file mode 100755 +index 000000000000..8781561e73d0 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_irq_internal.h +@@ -0,0 +1,39 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* ++ * Backend specific IRQ APIs ++ */ ++ ++#ifndef _KBASE_IRQ_INTERNAL_H_ ++#define _KBASE_IRQ_INTERNAL_H_ ++ ++int kbase_install_interrupts(struct kbase_device *kbdev); ++ ++void kbase_release_interrupts(struct kbase_device *kbdev); ++ ++/** ++ * kbase_synchronize_irqs - Ensure that all IRQ handlers have completed ++ * execution ++ * @kbdev: The kbase device ++ */ ++void kbase_synchronize_irqs(struct kbase_device *kbdev); ++ ++int kbasep_common_test_interrupt_handlers( ++ struct kbase_device * const kbdev); ++ ++#endif /* _KBASE_IRQ_INTERNAL_H_ */ +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_irq_linux.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_irq_linux.c +new file mode 100755 +index 000000000000..8416b80e8b77 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_irq_linux.c +@@ -0,0 +1,469 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include ++#include ++#include ++ ++#include ++ ++#if !defined(CONFIG_MALI_NO_MALI) ++ ++/* GPU IRQ Tags */ ++#define JOB_IRQ_TAG 0 ++#define MMU_IRQ_TAG 1 ++#define GPU_IRQ_TAG 2 ++ ++static void *kbase_tag(void *ptr, u32 tag) ++{ ++ return (void *)(((uintptr_t) ptr) | tag); ++} ++ ++static void *kbase_untag(void *ptr) ++{ ++ return (void *)(((uintptr_t) ptr) & ~3); ++} ++ ++static irqreturn_t kbase_job_irq_handler(int irq, void *data) ++{ ++ unsigned long flags; ++ struct kbase_device *kbdev = kbase_untag(data); ++ u32 val; ++ ++ spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags); ++ ++ if (!kbdev->pm.backend.gpu_powered) { ++ /* GPU is turned off - IRQ is not for us */ ++ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, ++ flags); ++ return IRQ_NONE; ++ } ++ ++ val = kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_STATUS), NULL); ++ ++#ifdef CONFIG_MALI_DEBUG ++ if (!kbdev->pm.backend.driver_ready_for_irqs) ++ dev_warn(kbdev->dev, "%s: irq %d irqstatus 0x%x before driver is ready\n", ++ __func__, irq, val); ++#endif /* CONFIG_MALI_DEBUG */ ++ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags); ++ ++ if (!val) ++ return IRQ_NONE; ++ ++ dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val); ++ ++ kbase_job_done(kbdev, val); ++ ++ return IRQ_HANDLED; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_job_irq_handler); ++ ++static irqreturn_t kbase_mmu_irq_handler(int irq, void *data) ++{ ++ unsigned long flags; ++ struct kbase_device *kbdev = kbase_untag(data); ++ u32 val; ++ ++ spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags); ++ ++ if (!kbdev->pm.backend.gpu_powered) { ++ /* GPU is turned off - IRQ is not for us */ ++ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, ++ flags); ++ return IRQ_NONE; ++ } ++ ++ atomic_inc(&kbdev->faults_pending); ++ ++ val = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_STATUS), NULL); ++ ++#ifdef CONFIG_MALI_DEBUG ++ if (!kbdev->pm.backend.driver_ready_for_irqs) ++ dev_warn(kbdev->dev, "%s: irq %d irqstatus 0x%x before driver is ready\n", ++ __func__, irq, val); ++#endif /* CONFIG_MALI_DEBUG */ ++ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags); ++ ++ if (!val) { ++ atomic_dec(&kbdev->faults_pending); ++ return IRQ_NONE; ++ } ++ ++ dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val); ++ ++ kbase_mmu_interrupt(kbdev, val); ++ ++ atomic_dec(&kbdev->faults_pending); ++ ++ return IRQ_HANDLED; ++} ++ ++static irqreturn_t kbase_gpu_irq_handler(int irq, void *data) ++{ ++ unsigned long flags; ++ struct kbase_device *kbdev = kbase_untag(data); ++ u32 val; ++ ++ spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags); ++ ++ if (!kbdev->pm.backend.gpu_powered) { ++ /* GPU is turned off - IRQ is not for us */ ++ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, ++ flags); ++ return IRQ_NONE; ++ } ++ ++ val = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_STATUS), NULL); ++ ++#ifdef CONFIG_MALI_DEBUG ++ if (!kbdev->pm.backend.driver_ready_for_irqs) ++ dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x before driver is ready\n", ++ __func__, irq, val); ++#endif /* CONFIG_MALI_DEBUG */ ++ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags); ++ ++ if (!val) ++ return IRQ_NONE; ++ ++ dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val); ++ ++ kbase_gpu_interrupt(kbdev, val); ++ ++ return IRQ_HANDLED; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_gpu_irq_handler); ++ ++static irq_handler_t kbase_handler_table[] = { ++ [JOB_IRQ_TAG] = kbase_job_irq_handler, ++ [MMU_IRQ_TAG] = kbase_mmu_irq_handler, ++ [GPU_IRQ_TAG] = kbase_gpu_irq_handler, ++}; ++ ++#ifdef CONFIG_MALI_DEBUG ++#define JOB_IRQ_HANDLER JOB_IRQ_TAG ++#define MMU_IRQ_HANDLER MMU_IRQ_TAG ++#define GPU_IRQ_HANDLER GPU_IRQ_TAG ++ ++/** ++ * kbase_set_custom_irq_handler - Set a custom IRQ handler ++ * @kbdev: Device for which the handler is to be registered ++ * @custom_handler: Handler to be registered ++ * @irq_type: Interrupt type ++ * ++ * Registers given interrupt handler for requested interrupt type ++ * In the case where irq handler is not specified, the default handler shall be ++ * registered ++ * ++ * Return: 0 case success, error code otherwise ++ */ ++int kbase_set_custom_irq_handler(struct kbase_device *kbdev, ++ irq_handler_t custom_handler, ++ int irq_type) ++{ ++ int result = 0; ++ irq_handler_t requested_irq_handler = NULL; ++ ++ KBASE_DEBUG_ASSERT((JOB_IRQ_HANDLER <= irq_type) && ++ (GPU_IRQ_HANDLER >= irq_type)); ++ ++ /* Release previous handler */ ++ if (kbdev->irqs[irq_type].irq) ++ free_irq(kbdev->irqs[irq_type].irq, kbase_tag(kbdev, irq_type)); ++ ++ requested_irq_handler = (NULL != custom_handler) ? custom_handler : ++ kbase_handler_table[irq_type]; ++ ++ if (0 != request_irq(kbdev->irqs[irq_type].irq, ++ requested_irq_handler, ++ kbdev->irqs[irq_type].flags | IRQF_SHARED, ++ dev_name(kbdev->dev), kbase_tag(kbdev, irq_type))) { ++ result = -EINVAL; ++ dev_err(kbdev->dev, "Can't request interrupt %d (index %d)\n", ++ kbdev->irqs[irq_type].irq, irq_type); ++#ifdef CONFIG_SPARSE_IRQ ++ dev_err(kbdev->dev, "You have CONFIG_SPARSE_IRQ support enabled - is the interrupt number correct for this configuration?\n"); ++#endif /* CONFIG_SPARSE_IRQ */ ++ } ++ ++ return result; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_set_custom_irq_handler); ++ ++/* test correct interrupt assigment and reception by cpu */ ++struct kbasep_irq_test { ++ struct hrtimer timer; ++ wait_queue_head_t wait; ++ int triggered; ++ u32 timeout; ++}; ++ ++static struct kbasep_irq_test kbasep_irq_test_data; ++ ++#define IRQ_TEST_TIMEOUT 500 ++ ++static irqreturn_t kbase_job_irq_test_handler(int irq, void *data) ++{ ++ unsigned long flags; ++ struct kbase_device *kbdev = kbase_untag(data); ++ u32 val; ++ ++ spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags); ++ ++ if (!kbdev->pm.backend.gpu_powered) { ++ /* GPU is turned off - IRQ is not for us */ ++ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, ++ flags); ++ return IRQ_NONE; ++ } ++ ++ val = kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_STATUS), NULL); ++ ++ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags); ++ ++ if (!val) ++ return IRQ_NONE; ++ ++ dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val); ++ ++ kbasep_irq_test_data.triggered = 1; ++ wake_up(&kbasep_irq_test_data.wait); ++ ++ kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), val, NULL); ++ ++ return IRQ_HANDLED; ++} ++ ++static irqreturn_t kbase_mmu_irq_test_handler(int irq, void *data) ++{ ++ unsigned long flags; ++ struct kbase_device *kbdev = kbase_untag(data); ++ u32 val; ++ ++ spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags); ++ ++ if (!kbdev->pm.backend.gpu_powered) { ++ /* GPU is turned off - IRQ is not for us */ ++ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, ++ flags); ++ return IRQ_NONE; ++ } ++ ++ val = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_STATUS), NULL); ++ ++ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags); ++ ++ if (!val) ++ return IRQ_NONE; ++ ++ dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val); ++ ++ kbasep_irq_test_data.triggered = 1; ++ wake_up(&kbasep_irq_test_data.wait); ++ ++ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), val, NULL); ++ ++ return IRQ_HANDLED; ++} ++ ++static enum hrtimer_restart kbasep_test_interrupt_timeout(struct hrtimer *timer) ++{ ++ struct kbasep_irq_test *test_data = container_of(timer, ++ struct kbasep_irq_test, timer); ++ ++ test_data->timeout = 1; ++ test_data->triggered = 1; ++ wake_up(&test_data->wait); ++ return HRTIMER_NORESTART; ++} ++ ++static int kbasep_common_test_interrupt( ++ struct kbase_device * const kbdev, u32 tag) ++{ ++ int err = 0; ++ irq_handler_t test_handler; ++ ++ u32 old_mask_val; ++ u16 mask_offset; ++ u16 rawstat_offset; ++ ++ switch (tag) { ++ case JOB_IRQ_TAG: ++ test_handler = kbase_job_irq_test_handler; ++ rawstat_offset = JOB_CONTROL_REG(JOB_IRQ_RAWSTAT); ++ mask_offset = JOB_CONTROL_REG(JOB_IRQ_MASK); ++ break; ++ case MMU_IRQ_TAG: ++ test_handler = kbase_mmu_irq_test_handler; ++ rawstat_offset = MMU_REG(MMU_IRQ_RAWSTAT); ++ mask_offset = MMU_REG(MMU_IRQ_MASK); ++ break; ++ case GPU_IRQ_TAG: ++ /* already tested by pm_driver - bail out */ ++ default: ++ return 0; ++ } ++ ++ /* store old mask */ ++ old_mask_val = kbase_reg_read(kbdev, mask_offset, NULL); ++ /* mask interrupts */ ++ kbase_reg_write(kbdev, mask_offset, 0x0, NULL); ++ ++ if (kbdev->irqs[tag].irq) { ++ /* release original handler and install test handler */ ++ if (kbase_set_custom_irq_handler(kbdev, test_handler, tag) != 0) { ++ err = -EINVAL; ++ } else { ++ kbasep_irq_test_data.timeout = 0; ++ hrtimer_init(&kbasep_irq_test_data.timer, ++ CLOCK_MONOTONIC, HRTIMER_MODE_REL); ++ kbasep_irq_test_data.timer.function = ++ kbasep_test_interrupt_timeout; ++ ++ /* trigger interrupt */ ++ kbase_reg_write(kbdev, mask_offset, 0x1, NULL); ++ kbase_reg_write(kbdev, rawstat_offset, 0x1, NULL); ++ ++ hrtimer_start(&kbasep_irq_test_data.timer, ++ HR_TIMER_DELAY_MSEC(IRQ_TEST_TIMEOUT), ++ HRTIMER_MODE_REL); ++ ++ wait_event(kbasep_irq_test_data.wait, ++ kbasep_irq_test_data.triggered != 0); ++ ++ if (kbasep_irq_test_data.timeout != 0) { ++ dev_err(kbdev->dev, "Interrupt %d (index %d) didn't reach CPU.\n", ++ kbdev->irqs[tag].irq, tag); ++ err = -EINVAL; ++ } else { ++ dev_dbg(kbdev->dev, "Interrupt %d (index %d) reached CPU.\n", ++ kbdev->irqs[tag].irq, tag); ++ } ++ ++ hrtimer_cancel(&kbasep_irq_test_data.timer); ++ kbasep_irq_test_data.triggered = 0; ++ ++ /* mask interrupts */ ++ kbase_reg_write(kbdev, mask_offset, 0x0, NULL); ++ ++ /* release test handler */ ++ free_irq(kbdev->irqs[tag].irq, kbase_tag(kbdev, tag)); ++ } ++ ++ /* restore original interrupt */ ++ if (request_irq(kbdev->irqs[tag].irq, kbase_handler_table[tag], ++ kbdev->irqs[tag].flags | IRQF_SHARED, ++ dev_name(kbdev->dev), kbase_tag(kbdev, tag))) { ++ dev_err(kbdev->dev, "Can't restore original interrupt %d (index %d)\n", ++ kbdev->irqs[tag].irq, tag); ++ err = -EINVAL; ++ } ++ } ++ /* restore old mask */ ++ kbase_reg_write(kbdev, mask_offset, old_mask_val, NULL); ++ ++ return err; ++} ++ ++int kbasep_common_test_interrupt_handlers( ++ struct kbase_device * const kbdev) ++{ ++ int err; ++ ++ init_waitqueue_head(&kbasep_irq_test_data.wait); ++ kbasep_irq_test_data.triggered = 0; ++ ++ /* A suspend won't happen during startup/insmod */ ++ kbase_pm_context_active(kbdev); ++ ++ err = kbasep_common_test_interrupt(kbdev, JOB_IRQ_TAG); ++ if (err) { ++ dev_err(kbdev->dev, "Interrupt JOB_IRQ didn't reach CPU. Check interrupt assignments.\n"); ++ goto out; ++ } ++ ++ err = kbasep_common_test_interrupt(kbdev, MMU_IRQ_TAG); ++ if (err) { ++ dev_err(kbdev->dev, "Interrupt MMU_IRQ didn't reach CPU. Check interrupt assignments.\n"); ++ goto out; ++ } ++ ++ dev_dbg(kbdev->dev, "Interrupts are correctly assigned.\n"); ++ ++ out: ++ kbase_pm_context_idle(kbdev); ++ ++ return err; ++} ++#endif /* CONFIG_MALI_DEBUG */ ++ ++int kbase_install_interrupts(struct kbase_device *kbdev) ++{ ++ u32 nr = ARRAY_SIZE(kbase_handler_table); ++ int err; ++ u32 i; ++ ++ for (i = 0; i < nr; i++) { ++ err = request_irq(kbdev->irqs[i].irq, kbase_handler_table[i], ++ kbdev->irqs[i].flags | IRQF_SHARED, ++ dev_name(kbdev->dev), ++ kbase_tag(kbdev, i)); ++ if (err) { ++ dev_err(kbdev->dev, "Can't request interrupt %d (index %d)\n", ++ kbdev->irqs[i].irq, i); ++#ifdef CONFIG_SPARSE_IRQ ++ dev_err(kbdev->dev, "You have CONFIG_SPARSE_IRQ support enabled - is the interrupt number correct for this configuration?\n"); ++#endif /* CONFIG_SPARSE_IRQ */ ++ goto release; ++ } ++ } ++ ++ return 0; ++ ++ release: ++ while (i-- > 0) ++ free_irq(kbdev->irqs[i].irq, kbase_tag(kbdev, i)); ++ ++ return err; ++} ++ ++void kbase_release_interrupts(struct kbase_device *kbdev) ++{ ++ u32 nr = ARRAY_SIZE(kbase_handler_table); ++ u32 i; ++ ++ for (i = 0; i < nr; i++) { ++ if (kbdev->irqs[i].irq) ++ free_irq(kbdev->irqs[i].irq, kbase_tag(kbdev, i)); ++ } ++} ++ ++void kbase_synchronize_irqs(struct kbase_device *kbdev) ++{ ++ u32 nr = ARRAY_SIZE(kbase_handler_table); ++ u32 i; ++ ++ for (i = 0; i < nr; i++) { ++ if (kbdev->irqs[i].irq) ++ synchronize_irq(kbdev->irqs[i].irq); ++ } ++} ++ ++#endif /* !defined(CONFIG_MALI_NO_MALI) */ +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_as.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_as.c +new file mode 100755 +index 000000000000..92358f2bf298 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_as.c +@@ -0,0 +1,237 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++/* ++ * Register backend context / address space management ++ */ ++ ++#include ++#include ++#include ++ ++/** ++ * assign_and_activate_kctx_addr_space - Assign an AS to a context ++ * @kbdev: Kbase device ++ * @kctx: Kbase context ++ * @current_as: Address Space to assign ++ * ++ * Assign an Address Space (AS) to a context, and add the context to the Policy. ++ * ++ * This includes ++ * setting up the global runpool_irq structure and the context on the AS, ++ * Activating the MMU on the AS, ++ * Allowing jobs to be submitted on the AS. ++ * ++ * Context: ++ * kbasep_js_kctx_info.jsctx_mutex held, ++ * kbasep_js_device_data.runpool_mutex held, ++ * AS transaction mutex held, ++ * Runpool IRQ lock held ++ */ ++static void assign_and_activate_kctx_addr_space(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ struct kbase_as *current_as) ++{ ++ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; ++ ++ lockdep_assert_held(&kctx->jctx.sched_info.ctx.jsctx_mutex); ++ lockdep_assert_held(&js_devdata->runpool_mutex); ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ /* Attribute handling */ ++ kbasep_js_ctx_attr_runpool_retain_ctx(kbdev, kctx); ++ ++ /* Allow it to run jobs */ ++ kbasep_js_set_submit_allowed(js_devdata, kctx); ++ ++ kbase_js_runpool_inc_context_count(kbdev, kctx); ++} ++ ++bool kbase_backend_use_ctx_sched(struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++ int i; ++ ++ if (kbdev->hwaccess.active_kctx == kctx) { ++ /* Context is already active */ ++ return true; ++ } ++ ++ for (i = 0; i < kbdev->nr_hw_address_spaces; i++) { ++ if (kbdev->as_to_kctx[i] == kctx) { ++ /* Context already has ASID - mark as active */ ++ return true; ++ } ++ } ++ ++ /* Context does not have address space assigned */ ++ return false; ++} ++ ++void kbase_backend_release_ctx_irq(struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++ int as_nr = kctx->as_nr; ++ ++ if (as_nr == KBASEP_AS_NR_INVALID) { ++ WARN(1, "Attempting to release context without ASID\n"); ++ return; ++ } ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (atomic_read(&kctx->refcount) != 1) { ++ WARN(1, "Attempting to release active ASID\n"); ++ return; ++ } ++ ++ kbasep_js_clear_submit_allowed(&kbdev->js_data, kctx); ++ ++ kbase_ctx_sched_release_ctx(kctx); ++ kbase_js_runpool_dec_context_count(kbdev, kctx); ++} ++ ++void kbase_backend_release_ctx_noirq(struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++} ++ ++int kbase_backend_find_and_release_free_address_space( ++ struct kbase_device *kbdev, struct kbase_context *kctx) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ unsigned long flags; ++ int i; ++ ++ js_devdata = &kbdev->js_data; ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); ++ mutex_lock(&js_devdata->runpool_mutex); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ for (i = 0; i < kbdev->nr_hw_address_spaces; i++) { ++ struct kbasep_js_kctx_info *as_js_kctx_info; ++ struct kbase_context *as_kctx; ++ ++ as_kctx = kbdev->as_to_kctx[i]; ++ as_js_kctx_info = &as_kctx->jctx.sched_info; ++ ++ /* Don't release privileged or active contexts, or contexts with ++ * jobs running. ++ * Note that a context will have at least 1 reference (which ++ * was previously taken by kbasep_js_schedule_ctx()) until ++ * descheduled. ++ */ ++ if (as_kctx && !kbase_ctx_flag(as_kctx, KCTX_PRIVILEGED) && ++ atomic_read(&as_kctx->refcount) == 1) { ++ if (!kbasep_js_runpool_retain_ctx_nolock(kbdev, ++ as_kctx)) { ++ WARN(1, "Failed to retain active context\n"); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, ++ flags); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ ++ return KBASEP_AS_NR_INVALID; ++ } ++ ++ kbasep_js_clear_submit_allowed(js_devdata, as_kctx); ++ ++ /* Drop and retake locks to take the jsctx_mutex on the ++ * context we're about to release without violating lock ++ * ordering ++ */ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ ++ ++ /* Release context from address space */ ++ mutex_lock(&as_js_kctx_info->ctx.jsctx_mutex); ++ mutex_lock(&js_devdata->runpool_mutex); ++ ++ kbasep_js_runpool_release_ctx_nolock(kbdev, as_kctx); ++ ++ if (!kbase_ctx_flag(as_kctx, KCTX_SCHEDULED)) { ++ kbasep_js_runpool_requeue_or_kill_ctx(kbdev, ++ as_kctx, ++ true); ++ ++ mutex_unlock(&js_devdata->runpool_mutex); ++ mutex_unlock(&as_js_kctx_info->ctx.jsctx_mutex); ++ ++ return i; ++ } ++ ++ /* Context was retained while locks were dropped, ++ * continue looking for free AS */ ++ ++ mutex_unlock(&js_devdata->runpool_mutex); ++ mutex_unlock(&as_js_kctx_info->ctx.jsctx_mutex); ++ ++ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); ++ mutex_lock(&js_devdata->runpool_mutex); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ } ++ } ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ mutex_unlock(&js_devdata->runpool_mutex); ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ ++ return KBASEP_AS_NR_INVALID; ++} ++ ++bool kbase_backend_use_ctx(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ int as_nr) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ struct kbase_as *new_address_space = NULL; ++ ++ js_devdata = &kbdev->js_data; ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ if (kbdev->hwaccess.active_kctx == kctx) { ++ WARN(1, "Context is already scheduled in\n"); ++ return false; ++ } ++ ++ new_address_space = &kbdev->as[as_nr]; ++ ++ lockdep_assert_held(&js_devdata->runpool_mutex); ++ lockdep_assert_held(&kbdev->mmu_hw_mutex); ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ assign_and_activate_kctx_addr_space(kbdev, kctx, new_address_space); ++ ++ if (kbase_ctx_flag(kctx, KCTX_PRIVILEGED)) { ++ /* We need to retain it to keep the corresponding address space ++ */ ++ kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx); ++ } ++ ++ return true; ++} ++ +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_defs.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_defs.h +new file mode 100755 +index 000000000000..08a7400e66d5 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_defs.h +@@ -0,0 +1,123 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++/* ++ * Register-based HW access backend specific definitions ++ */ ++ ++#ifndef _KBASE_HWACCESS_GPU_DEFS_H_ ++#define _KBASE_HWACCESS_GPU_DEFS_H_ ++ ++/* SLOT_RB_SIZE must be < 256 */ ++#define SLOT_RB_SIZE 2 ++#define SLOT_RB_MASK (SLOT_RB_SIZE - 1) ++ ++/** ++ * struct rb_entry - Ringbuffer entry ++ * @katom: Atom associated with this entry ++ */ ++struct rb_entry { ++ struct kbase_jd_atom *katom; ++}; ++ ++/** ++ * struct slot_rb - Slot ringbuffer ++ * @entries: Ringbuffer entries ++ * @last_context: The last context to submit a job on this slot ++ * @read_idx: Current read index of buffer ++ * @write_idx: Current write index of buffer ++ * @job_chain_flag: Flag used to implement jobchain disambiguation ++ */ ++struct slot_rb { ++ struct rb_entry entries[SLOT_RB_SIZE]; ++ ++ struct kbase_context *last_context; ++ ++ u8 read_idx; ++ u8 write_idx; ++ ++ u8 job_chain_flag; ++}; ++ ++/** ++ * struct kbase_backend_data - GPU backend specific data for HW access layer ++ * @slot_rb: Slot ringbuffers ++ * @rmu_workaround_flag: When PRLAM-8987 is present, this flag determines ++ * whether slots 0/1 or slot 2 are currently being ++ * pulled from ++ * @scheduling_timer: The timer tick used for rescheduling jobs ++ * @timer_running: Is the timer running? The runpool_mutex must be ++ * held whilst modifying this. ++ * @suspend_timer: Is the timer suspended? Set when a suspend ++ * occurs and cleared on resume. The runpool_mutex ++ * must be held whilst modifying this. ++ * @reset_gpu: Set to a KBASE_RESET_xxx value (see comments) ++ * @reset_workq: Work queue for performing the reset ++ * @reset_work: Work item for performing the reset ++ * @reset_wait: Wait event signalled when the reset is complete ++ * @reset_timer: Timeout for soft-stops before the reset ++ * @timeouts_updated: Have timeout values just been updated? ++ * ++ * The hwaccess_lock (a spinlock) must be held when accessing this structure ++ */ ++struct kbase_backend_data { ++ struct slot_rb slot_rb[BASE_JM_MAX_NR_SLOTS]; ++ ++ bool rmu_workaround_flag; ++ ++ struct hrtimer scheduling_timer; ++ ++ bool timer_running; ++ bool suspend_timer; ++ ++ atomic_t reset_gpu; ++ ++/* The GPU reset isn't pending */ ++#define KBASE_RESET_GPU_NOT_PENDING 0 ++/* kbase_prepare_to_reset_gpu has been called */ ++#define KBASE_RESET_GPU_PREPARED 1 ++/* kbase_reset_gpu has been called - the reset will now definitely happen ++ * within the timeout period */ ++#define KBASE_RESET_GPU_COMMITTED 2 ++/* The GPU reset process is currently occuring (timeout has expired or ++ * kbasep_try_reset_gpu_early was called) */ ++#define KBASE_RESET_GPU_HAPPENING 3 ++/* Reset the GPU silently, used when resetting the GPU as part of normal ++ * behavior (e.g. when exiting protected mode). */ ++#define KBASE_RESET_GPU_SILENT 4 ++ struct workqueue_struct *reset_workq; ++ struct work_struct reset_work; ++ wait_queue_head_t reset_wait; ++ struct hrtimer reset_timer; ++ ++ bool timeouts_updated; ++}; ++ ++/** ++ * struct kbase_jd_atom_backend - GPU backend specific katom data ++ */ ++struct kbase_jd_atom_backend { ++}; ++ ++/** ++ * struct kbase_context_backend - GPU backend specific context data ++ */ ++struct kbase_context_backend { ++}; ++ ++#endif /* _KBASE_HWACCESS_GPU_DEFS_H_ */ +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_hw.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_hw.c +new file mode 100755 +index 000000000000..a6fb097b94f9 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_hw.c +@@ -0,0 +1,1518 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* ++ * Base kernel job manager APIs ++ */ ++ ++#include ++#include ++#include ++#if defined(CONFIG_MALI_GATOR_SUPPORT) ++#include ++#endif ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#define beenthere(kctx, f, a...) \ ++ dev_dbg(kctx->kbdev->dev, "%s:" f, __func__, ##a) ++ ++#if KBASE_GPU_RESET_EN ++static void kbasep_try_reset_gpu_early(struct kbase_device *kbdev); ++static void kbasep_reset_timeout_worker(struct work_struct *data); ++static enum hrtimer_restart kbasep_reset_timer_callback(struct hrtimer *timer); ++#endif /* KBASE_GPU_RESET_EN */ ++ ++static inline int kbasep_jm_is_js_free(struct kbase_device *kbdev, int js, ++ struct kbase_context *kctx) ++{ ++ return !kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT), kctx); ++} ++ ++void kbase_job_hw_submit(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom, ++ int js) ++{ ++ struct kbase_context *kctx; ++ u32 cfg; ++ u64 jc_head = katom->jc; ++ ++ KBASE_DEBUG_ASSERT(kbdev); ++ KBASE_DEBUG_ASSERT(katom); ++ ++ kctx = katom->kctx; ++ ++ /* Command register must be available */ ++ KBASE_DEBUG_ASSERT(kbasep_jm_is_js_free(kbdev, js, kctx)); ++ /* Affinity is not violating */ ++ kbase_js_debug_log_current_affinities(kbdev); ++ KBASE_DEBUG_ASSERT(!kbase_js_affinity_would_violate(kbdev, js, ++ katom->affinity)); ++ ++ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_LO), ++ jc_head & 0xFFFFFFFF, kctx); ++ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_HI), ++ jc_head >> 32, kctx); ++ ++ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_AFFINITY_NEXT_LO), ++ katom->affinity & 0xFFFFFFFF, kctx); ++ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_AFFINITY_NEXT_HI), ++ katom->affinity >> 32, kctx); ++ ++ /* start MMU, medium priority, cache clean/flush on end, clean/flush on ++ * start */ ++ cfg = kctx->as_nr; ++ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_FLUSH_REDUCTION)) ++ cfg |= JS_CONFIG_ENABLE_FLUSH_REDUCTION; ++ ++#ifndef CONFIG_MALI_COH_GPU ++ if (0 != (katom->core_req & BASE_JD_REQ_SKIP_CACHE_START)) ++ cfg |= JS_CONFIG_START_FLUSH_NO_ACTION; ++ else ++ cfg |= JS_CONFIG_START_FLUSH_CLEAN_INVALIDATE; ++ ++ if (0 != (katom->core_req & BASE_JD_REQ_SKIP_CACHE_END)) ++ cfg |= JS_CONFIG_END_FLUSH_NO_ACTION; ++ else ++ cfg |= JS_CONFIG_END_FLUSH_CLEAN_INVALIDATE; ++#endif /* CONFIG_MALI_COH_GPU */ ++ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10649)) ++ cfg |= JS_CONFIG_START_MMU; ++ ++ cfg |= JS_CONFIG_THREAD_PRI(8); ++ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_MODE) && ++ (katom->atom_flags & KBASE_KATOM_FLAG_PROTECTED)) ++ cfg |= JS_CONFIG_DISABLE_DESCRIPTOR_WR_BK; ++ ++ if (kbase_hw_has_feature(kbdev, ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) { ++ if (!kbdev->hwaccess.backend.slot_rb[js].job_chain_flag) { ++ cfg |= JS_CONFIG_JOB_CHAIN_FLAG; ++ katom->atom_flags |= KBASE_KATOM_FLAGS_JOBCHAIN; ++ kbdev->hwaccess.backend.slot_rb[js].job_chain_flag = ++ true; ++ } else { ++ katom->atom_flags &= ~KBASE_KATOM_FLAGS_JOBCHAIN; ++ kbdev->hwaccess.backend.slot_rb[js].job_chain_flag = ++ false; ++ } ++ } ++ ++ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_CONFIG_NEXT), cfg, kctx); ++ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_FLUSH_REDUCTION)) ++ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_FLUSH_ID_NEXT), ++ katom->flush_id, kctx); ++ ++ /* Write an approximate start timestamp. ++ * It's approximate because there might be a job in the HEAD register. ++ */ ++ katom->start_timestamp = ktime_get(); ++ ++ /* GO ! */ ++ dev_dbg(kbdev->dev, "JS: Submitting atom %p from ctx %p to js[%d] with head=0x%llx, affinity=0x%llx", ++ katom, kctx, js, jc_head, katom->affinity); ++ ++ KBASE_TRACE_ADD_SLOT_INFO(kbdev, JM_SUBMIT, kctx, katom, jc_head, js, ++ (u32) katom->affinity); ++ ++#if defined(CONFIG_MALI_GATOR_SUPPORT) ++ kbase_trace_mali_job_slots_event( ++ GATOR_MAKE_EVENT(GATOR_JOB_SLOT_START, js), ++ kctx, kbase_jd_atom_id(kctx, katom)); ++#endif ++ KBASE_TLSTREAM_TL_ATTRIB_ATOM_CONFIG(katom, jc_head, ++ katom->affinity, cfg); ++ KBASE_TLSTREAM_TL_RET_CTX_LPU( ++ kctx, ++ &kbdev->gpu_props.props.raw_props.js_features[ ++ katom->slot_nr]); ++ KBASE_TLSTREAM_TL_RET_ATOM_AS(katom, &kbdev->as[kctx->as_nr]); ++ KBASE_TLSTREAM_TL_RET_ATOM_LPU( ++ katom, ++ &kbdev->gpu_props.props.raw_props.js_features[js], ++ "ctx_nr,atom_nr"); ++#ifdef CONFIG_GPU_TRACEPOINTS ++ if (!kbase_backend_nr_atoms_submitted(kbdev, js)) { ++ /* If this is the only job on the slot, trace it as starting */ ++ char js_string[16]; ++ ++ trace_gpu_sched_switch( ++ kbasep_make_job_slot_string(js, js_string, ++ sizeof(js_string)), ++ ktime_to_ns(katom->start_timestamp), ++ (u32)katom->kctx->id, 0, katom->work_id); ++ kbdev->hwaccess.backend.slot_rb[js].last_context = katom->kctx; ++ } ++#endif ++ kbase_timeline_job_slot_submit(kbdev, kctx, katom, js); ++ ++ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT), ++ JS_COMMAND_START, katom->kctx); ++} ++ ++/** ++ * kbasep_job_slot_update_head_start_timestamp - Update timestamp ++ * @kbdev: kbase device ++ * @js: job slot ++ * @end_timestamp: timestamp ++ * ++ * Update the start_timestamp of the job currently in the HEAD, based on the ++ * fact that we got an IRQ for the previous set of completed jobs. ++ * ++ * The estimate also takes into account the time the job was submitted, to ++ * work out the best estimate (which might still result in an over-estimate to ++ * the calculated time spent) ++ */ ++static void kbasep_job_slot_update_head_start_timestamp( ++ struct kbase_device *kbdev, ++ int js, ++ ktime_t end_timestamp) ++{ ++ if (kbase_backend_nr_atoms_on_slot(kbdev, js) > 0) { ++ struct kbase_jd_atom *katom; ++ ktime_t timestamp_diff; ++ /* The atom in the HEAD */ ++ katom = kbase_gpu_inspect(kbdev, js, 0); ++ ++ KBASE_DEBUG_ASSERT(katom != NULL); ++ ++ timestamp_diff = ktime_sub(end_timestamp, ++ katom->start_timestamp); ++ if (ktime_to_ns(timestamp_diff) >= 0) { ++ /* Only update the timestamp if it's a better estimate ++ * than what's currently stored. This is because our ++ * estimate that accounts for the throttle time may be ++ * too much of an overestimate */ ++ katom->start_timestamp = end_timestamp; ++ } ++ } ++} ++ ++/** ++ * kbasep_trace_tl_event_lpu_softstop - Call event_lpu_softstop timeline ++ * tracepoint ++ * @kbdev: kbase device ++ * @js: job slot ++ * ++ * Make a tracepoint call to the instrumentation module informing that ++ * softstop happened on given lpu (job slot). ++ */ ++static void kbasep_trace_tl_event_lpu_softstop(struct kbase_device *kbdev, ++ int js) ++{ ++ KBASE_TLSTREAM_TL_EVENT_LPU_SOFTSTOP( ++ &kbdev->gpu_props.props.raw_props.js_features[js]); ++} ++ ++void kbase_job_done(struct kbase_device *kbdev, u32 done) ++{ ++ unsigned long flags; ++ int i; ++ u32 count = 0; ++ ktime_t end_timestamp = ktime_get(); ++ struct kbasep_js_device_data *js_devdata; ++ ++ KBASE_DEBUG_ASSERT(kbdev); ++ js_devdata = &kbdev->js_data; ++ ++ KBASE_TRACE_ADD(kbdev, JM_IRQ, NULL, NULL, 0, done); ++ ++ memset(&kbdev->slot_submit_count_irq[0], 0, ++ sizeof(kbdev->slot_submit_count_irq)); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ while (done) { ++ u32 failed = done >> 16; ++ ++ /* treat failed slots as finished slots */ ++ u32 finished = (done & 0xFFFF) | failed; ++ ++ /* Note: This is inherently unfair, as we always check ++ * for lower numbered interrupts before the higher ++ * numbered ones.*/ ++ i = ffs(finished) - 1; ++ KBASE_DEBUG_ASSERT(i >= 0); ++ ++ do { ++ int nr_done; ++ u32 active; ++ u32 completion_code = BASE_JD_EVENT_DONE;/* assume OK */ ++ u64 job_tail = 0; ++ ++ if (failed & (1u << i)) { ++ /* read out the job slot status code if the job ++ * slot reported failure */ ++ completion_code = kbase_reg_read(kbdev, ++ JOB_SLOT_REG(i, JS_STATUS), NULL); ++ ++ switch (completion_code) { ++ case BASE_JD_EVENT_STOPPED: ++#if defined(CONFIG_MALI_GATOR_SUPPORT) ++ kbase_trace_mali_job_slots_event( ++ GATOR_MAKE_EVENT( ++ GATOR_JOB_SLOT_SOFT_STOPPED, i), ++ NULL, 0); ++#endif ++ ++ kbasep_trace_tl_event_lpu_softstop( ++ kbdev, i); ++ ++ /* Soft-stopped job - read the value of ++ * JS_TAIL so that the job chain can ++ * be resumed */ ++ job_tail = (u64)kbase_reg_read(kbdev, ++ JOB_SLOT_REG(i, JS_TAIL_LO), ++ NULL) | ++ ((u64)kbase_reg_read(kbdev, ++ JOB_SLOT_REG(i, JS_TAIL_HI), ++ NULL) << 32); ++ break; ++ case BASE_JD_EVENT_NOT_STARTED: ++ /* PRLAM-10673 can cause a TERMINATED ++ * job to come back as NOT_STARTED, but ++ * the error interrupt helps us detect ++ * it */ ++ completion_code = ++ BASE_JD_EVENT_TERMINATED; ++ /* fall through */ ++ default: ++ dev_warn(kbdev->dev, "error detected from slot %d, job status 0x%08x (%s)", ++ i, completion_code, ++ kbase_exception_name ++ (kbdev, ++ completion_code)); ++ } ++ ++ kbase_gpu_irq_evict(kbdev, i); ++ } ++ ++ kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), ++ done & ((1 << i) | (1 << (i + 16))), ++ NULL); ++ active = kbase_reg_read(kbdev, ++ JOB_CONTROL_REG(JOB_IRQ_JS_STATE), ++ NULL); ++ ++ if (((active >> i) & 1) == 0 && ++ (((done >> (i + 16)) & 1) == 0)) { ++ /* There is a potential race we must work ++ * around: ++ * ++ * 1. A job slot has a job in both current and ++ * next registers ++ * 2. The job in current completes ++ * successfully, the IRQ handler reads ++ * RAWSTAT and calls this function with the ++ * relevant bit set in "done" ++ * 3. The job in the next registers becomes the ++ * current job on the GPU ++ * 4. Sometime before the JOB_IRQ_CLEAR line ++ * above the job on the GPU _fails_ ++ * 5. The IRQ_CLEAR clears the done bit but not ++ * the failed bit. This atomically sets ++ * JOB_IRQ_JS_STATE. However since both jobs ++ * have now completed the relevant bits for ++ * the slot are set to 0. ++ * ++ * If we now did nothing then we'd incorrectly ++ * assume that _both_ jobs had completed ++ * successfully (since we haven't yet observed ++ * the fail bit being set in RAWSTAT). ++ * ++ * So at this point if there are no active jobs ++ * left we check to see if RAWSTAT has a failure ++ * bit set for the job slot. If it does we know ++ * that there has been a new failure that we ++ * didn't previously know about, so we make sure ++ * that we record this in active (but we wait ++ * for the next loop to deal with it). ++ * ++ * If we were handling a job failure (i.e. done ++ * has the relevant high bit set) then we know ++ * that the value read back from ++ * JOB_IRQ_JS_STATE is the correct number of ++ * remaining jobs because the failed job will ++ * have prevented any futher jobs from starting ++ * execution. ++ */ ++ u32 rawstat = kbase_reg_read(kbdev, ++ JOB_CONTROL_REG(JOB_IRQ_RAWSTAT), NULL); ++ ++ if ((rawstat >> (i + 16)) & 1) { ++ /* There is a failed job that we've ++ * missed - add it back to active */ ++ active |= (1u << i); ++ } ++ } ++ ++ dev_dbg(kbdev->dev, "Job ended with status 0x%08X\n", ++ completion_code); ++ ++ nr_done = kbase_backend_nr_atoms_submitted(kbdev, i); ++ nr_done -= (active >> i) & 1; ++ nr_done -= (active >> (i + 16)) & 1; ++ ++ if (nr_done <= 0) { ++ dev_warn(kbdev->dev, "Spurious interrupt on slot %d", ++ i); ++ ++ goto spurious; ++ } ++ ++ count += nr_done; ++ ++ while (nr_done) { ++ if (nr_done == 1) { ++ kbase_gpu_complete_hw(kbdev, i, ++ completion_code, ++ job_tail, ++ &end_timestamp); ++ kbase_jm_try_kick_all(kbdev); ++ } else { ++ /* More than one job has completed. ++ * Since this is not the last job being ++ * reported this time it must have ++ * passed. This is because the hardware ++ * will not allow further jobs in a job ++ * slot to complete until the failed job ++ * is cleared from the IRQ status. ++ */ ++ kbase_gpu_complete_hw(kbdev, i, ++ BASE_JD_EVENT_DONE, ++ 0, ++ &end_timestamp); ++ } ++ nr_done--; ++ } ++ spurious: ++ done = kbase_reg_read(kbdev, ++ JOB_CONTROL_REG(JOB_IRQ_RAWSTAT), NULL); ++ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10883)) { ++ /* Workaround for missing interrupt caused by ++ * PRLAM-10883 */ ++ if (((active >> i) & 1) && (0 == ++ kbase_reg_read(kbdev, ++ JOB_SLOT_REG(i, ++ JS_STATUS), NULL))) { ++ /* Force job slot to be processed again ++ */ ++ done |= (1u << i); ++ } ++ } ++ ++ failed = done >> 16; ++ finished = (done & 0xFFFF) | failed; ++ if (done) ++ end_timestamp = ktime_get(); ++ } while (finished & (1 << i)); ++ ++ kbasep_job_slot_update_head_start_timestamp(kbdev, i, ++ end_timestamp); ++ } ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++#if KBASE_GPU_RESET_EN ++ if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) == ++ KBASE_RESET_GPU_COMMITTED) { ++ /* If we're trying to reset the GPU then we might be able to do ++ * it early (without waiting for a timeout) because some jobs ++ * have completed ++ */ ++ kbasep_try_reset_gpu_early(kbdev); ++ } ++#endif /* KBASE_GPU_RESET_EN */ ++ KBASE_TRACE_ADD(kbdev, JM_IRQ_END, NULL, NULL, 0, count); ++} ++KBASE_EXPORT_TEST_API(kbase_job_done); ++ ++static bool kbasep_soft_stop_allowed(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom) ++{ ++ bool soft_stops_allowed = true; ++ ++ if (kbase_jd_katom_is_protected(katom)) { ++ soft_stops_allowed = false; ++ } else if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408)) { ++ if ((katom->core_req & BASE_JD_REQ_T) != 0) ++ soft_stops_allowed = false; ++ } ++ return soft_stops_allowed; ++} ++ ++static bool kbasep_hard_stop_allowed(struct kbase_device *kbdev, ++ base_jd_core_req core_reqs) ++{ ++ bool hard_stops_allowed = true; ++ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8394)) { ++ if ((core_reqs & BASE_JD_REQ_T) != 0) ++ hard_stops_allowed = false; ++ } ++ return hard_stops_allowed; ++} ++ ++void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev, ++ int js, ++ u32 action, ++ base_jd_core_req core_reqs, ++ struct kbase_jd_atom *target_katom) ++{ ++ struct kbase_context *kctx = target_katom->kctx; ++#if KBASE_TRACE_ENABLE ++ u32 status_reg_before; ++ u64 job_in_head_before; ++ u32 status_reg_after; ++ ++ KBASE_DEBUG_ASSERT(!(action & (~JS_COMMAND_MASK))); ++ ++ /* Check the head pointer */ ++ job_in_head_before = ((u64) kbase_reg_read(kbdev, ++ JOB_SLOT_REG(js, JS_HEAD_LO), NULL)) ++ | (((u64) kbase_reg_read(kbdev, ++ JOB_SLOT_REG(js, JS_HEAD_HI), NULL)) ++ << 32); ++ status_reg_before = kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_STATUS), ++ NULL); ++#endif ++ ++ if (action == JS_COMMAND_SOFT_STOP) { ++ bool soft_stop_allowed = kbasep_soft_stop_allowed(kbdev, ++ target_katom); ++ ++ if (!soft_stop_allowed) { ++#ifdef CONFIG_MALI_DEBUG ++ dev_dbg(kbdev->dev, ++ "Attempt made to soft-stop a job that cannot be soft-stopped. core_reqs = 0x%X", ++ (unsigned int)core_reqs); ++#endif /* CONFIG_MALI_DEBUG */ ++ return; ++ } ++ ++ /* We are about to issue a soft stop, so mark the atom as having ++ * been soft stopped */ ++ target_katom->atom_flags |= KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED; ++ ++ /* Mark the point where we issue the soft-stop command */ ++ KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_ISSUE(target_katom); ++ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316)) { ++ int i; ++ ++ for (i = 0; ++ i < kbase_backend_nr_atoms_submitted(kbdev, js); ++ i++) { ++ struct kbase_jd_atom *katom; ++ ++ katom = kbase_gpu_inspect(kbdev, js, i); ++ ++ KBASE_DEBUG_ASSERT(katom); ++ ++ /* For HW_ISSUE_8316, only 'bad' jobs attacking ++ * the system can cause this issue: normally, ++ * all memory should be allocated in multiples ++ * of 4 pages, and growable memory should be ++ * changed size in multiples of 4 pages. ++ * ++ * Whilst such 'bad' jobs can be cleared by a ++ * GPU reset, the locking up of a uTLB entry ++ * caused by the bad job could also stall other ++ * ASs, meaning that other ASs' jobs don't ++ * complete in the 'grace' period before the ++ * reset. We don't want to lose other ASs' jobs ++ * when they would normally complete fine, so we ++ * must 'poke' the MMU regularly to help other ++ * ASs complete */ ++ kbase_as_poking_timer_retain_atom( ++ kbdev, katom->kctx, katom); ++ } ++ } ++ ++ if (kbase_hw_has_feature( ++ kbdev, ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) { ++ action = (target_katom->atom_flags & ++ KBASE_KATOM_FLAGS_JOBCHAIN) ? ++ JS_COMMAND_SOFT_STOP_1 : ++ JS_COMMAND_SOFT_STOP_0; ++ } ++ } else if (action == JS_COMMAND_HARD_STOP) { ++ bool hard_stop_allowed = kbasep_hard_stop_allowed(kbdev, ++ core_reqs); ++ ++ if (!hard_stop_allowed) { ++ /* Jobs can be hard-stopped for the following reasons: ++ * * CFS decides the job has been running too long (and ++ * soft-stop has not occurred). In this case the GPU ++ * will be reset by CFS if the job remains on the ++ * GPU. ++ * ++ * * The context is destroyed, kbase_jd_zap_context ++ * will attempt to hard-stop the job. However it also ++ * has a watchdog which will cause the GPU to be ++ * reset if the job remains on the GPU. ++ * ++ * * An (unhandled) MMU fault occurred. As long as ++ * BASE_HW_ISSUE_8245 is defined then the GPU will be ++ * reset. ++ * ++ * All three cases result in the GPU being reset if the ++ * hard-stop fails, so it is safe to just return and ++ * ignore the hard-stop request. ++ */ ++ dev_warn(kbdev->dev, ++ "Attempt made to hard-stop a job that cannot be hard-stopped. core_reqs = 0x%X", ++ (unsigned int)core_reqs); ++ return; ++ } ++ target_katom->atom_flags |= KBASE_KATOM_FLAG_BEEN_HARD_STOPPED; ++ ++ if (kbase_hw_has_feature( ++ kbdev, ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) { ++ action = (target_katom->atom_flags & ++ KBASE_KATOM_FLAGS_JOBCHAIN) ? ++ JS_COMMAND_HARD_STOP_1 : ++ JS_COMMAND_HARD_STOP_0; ++ } ++ } ++ ++ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND), action, kctx); ++ ++#if KBASE_TRACE_ENABLE ++ status_reg_after = kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_STATUS), ++ NULL); ++ if (status_reg_after == BASE_JD_EVENT_ACTIVE) { ++ struct kbase_jd_atom *head; ++ struct kbase_context *head_kctx; ++ ++ head = kbase_gpu_inspect(kbdev, js, 0); ++ head_kctx = head->kctx; ++ ++ if (status_reg_before == BASE_JD_EVENT_ACTIVE) ++ KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, head_kctx, ++ head, job_in_head_before, js); ++ else ++ KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, ++ 0, js); ++ ++ switch (action) { ++ case JS_COMMAND_SOFT_STOP: ++ KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP, head_kctx, ++ head, head->jc, js); ++ break; ++ case JS_COMMAND_SOFT_STOP_0: ++ KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_0, head_kctx, ++ head, head->jc, js); ++ break; ++ case JS_COMMAND_SOFT_STOP_1: ++ KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_1, head_kctx, ++ head, head->jc, js); ++ break; ++ case JS_COMMAND_HARD_STOP: ++ KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP, head_kctx, ++ head, head->jc, js); ++ break; ++ case JS_COMMAND_HARD_STOP_0: ++ KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_0, head_kctx, ++ head, head->jc, js); ++ break; ++ case JS_COMMAND_HARD_STOP_1: ++ KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_1, head_kctx, ++ head, head->jc, js); ++ break; ++ default: ++ BUG(); ++ break; ++ } ++ } else { ++ if (status_reg_before == BASE_JD_EVENT_ACTIVE) ++ KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, ++ job_in_head_before, js); ++ else ++ KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, ++ 0, js); ++ ++ switch (action) { ++ case JS_COMMAND_SOFT_STOP: ++ KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP, NULL, NULL, 0, ++ js); ++ break; ++ case JS_COMMAND_SOFT_STOP_0: ++ KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_0, NULL, NULL, ++ 0, js); ++ break; ++ case JS_COMMAND_SOFT_STOP_1: ++ KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_1, NULL, NULL, ++ 0, js); ++ break; ++ case JS_COMMAND_HARD_STOP: ++ KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP, NULL, NULL, 0, ++ js); ++ break; ++ case JS_COMMAND_HARD_STOP_0: ++ KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_0, NULL, NULL, ++ 0, js); ++ break; ++ case JS_COMMAND_HARD_STOP_1: ++ KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_1, NULL, NULL, ++ 0, js); ++ break; ++ default: ++ BUG(); ++ break; ++ } ++ } ++#endif ++} ++ ++void kbase_backend_jm_kill_jobs_from_kctx(struct kbase_context *kctx) ++{ ++ unsigned long flags; ++ struct kbase_device *kbdev; ++ struct kbasep_js_device_data *js_devdata; ++ int i; ++ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ kbdev = kctx->kbdev; ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ js_devdata = &kbdev->js_data; ++ ++ /* Cancel any remaining running jobs for this kctx */ ++ mutex_lock(&kctx->jctx.lock); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ /* Invalidate all jobs in context, to prevent re-submitting */ ++ for (i = 0; i < BASE_JD_ATOM_COUNT; i++) { ++ if (!work_pending(&kctx->jctx.atoms[i].work)) ++ kctx->jctx.atoms[i].event_code = ++ BASE_JD_EVENT_JOB_CANCELLED; ++ } ++ ++ for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) ++ kbase_job_slot_hardstop(kctx, i, NULL); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&kctx->jctx.lock); ++} ++ ++void kbase_job_slot_ctx_priority_check_locked(struct kbase_context *kctx, ++ struct kbase_jd_atom *target_katom) ++{ ++ struct kbase_device *kbdev; ++ int js = target_katom->slot_nr; ++ int priority = target_katom->sched_priority; ++ int i; ++ bool stop_sent = false; ++ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ kbdev = kctx->kbdev; ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ for (i = 0; i < kbase_backend_nr_atoms_on_slot(kbdev, js); i++) { ++ struct kbase_jd_atom *katom; ++ ++ katom = kbase_gpu_inspect(kbdev, js, i); ++ if (!katom) ++ continue; ++ ++ if (katom->kctx != kctx) ++ continue; ++ ++ if (katom->sched_priority > priority) { ++ if (!stop_sent) ++ KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY_CHANGE( ++ target_katom); ++ ++ kbase_job_slot_softstop(kbdev, js, katom); ++ stop_sent = true; ++ } ++ } ++} ++ ++struct zap_reset_data { ++ /* The stages are: ++ * 1. The timer has never been called ++ * 2. The zap has timed out, all slots are soft-stopped - the GPU reset ++ * will happen. The GPU has been reset when ++ * kbdev->hwaccess.backend.reset_waitq is signalled ++ * ++ * (-1 - The timer has been cancelled) ++ */ ++ int stage; ++ struct kbase_device *kbdev; ++ struct hrtimer timer; ++ spinlock_t lock; /* protects updates to stage member */ ++}; ++ ++static enum hrtimer_restart zap_timeout_callback(struct hrtimer *timer) ++{ ++ struct zap_reset_data *reset_data = container_of(timer, ++ struct zap_reset_data, timer); ++ struct kbase_device *kbdev = reset_data->kbdev; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&reset_data->lock, flags); ++ ++ if (reset_data->stage == -1) ++ goto out; ++ ++#if KBASE_GPU_RESET_EN ++ if (kbase_prepare_to_reset_gpu(kbdev)) { ++ dev_err(kbdev->dev, "Issueing GPU soft-reset because jobs failed to be killed (within %d ms) as part of context termination (e.g. process exit)\n", ++ ZAP_TIMEOUT); ++ kbase_reset_gpu(kbdev); ++ } ++#endif /* KBASE_GPU_RESET_EN */ ++ reset_data->stage = 2; ++ ++ out: ++ spin_unlock_irqrestore(&reset_data->lock, flags); ++ ++ return HRTIMER_NORESTART; ++} ++ ++void kbase_jm_wait_for_zero_jobs(struct kbase_context *kctx) ++{ ++ struct kbase_device *kbdev = kctx->kbdev; ++ struct zap_reset_data reset_data; ++ unsigned long flags; ++ ++ hrtimer_init_on_stack(&reset_data.timer, CLOCK_MONOTONIC, ++ HRTIMER_MODE_REL); ++ reset_data.timer.function = zap_timeout_callback; ++ ++ spin_lock_init(&reset_data.lock); ++ ++ reset_data.kbdev = kbdev; ++ reset_data.stage = 1; ++ ++ hrtimer_start(&reset_data.timer, HR_TIMER_DELAY_MSEC(ZAP_TIMEOUT), ++ HRTIMER_MODE_REL); ++ ++ /* Wait for all jobs to finish, and for the context to be not-scheduled ++ * (due to kbase_job_zap_context(), we also guarentee it's not in the JS ++ * policy queue either */ ++ wait_event(kctx->jctx.zero_jobs_wait, kctx->jctx.job_nr == 0); ++ wait_event(kctx->jctx.sched_info.ctx.is_scheduled_wait, ++ !kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ ++ spin_lock_irqsave(&reset_data.lock, flags); ++ if (reset_data.stage == 1) { ++ /* The timer hasn't run yet - so cancel it */ ++ reset_data.stage = -1; ++ } ++ spin_unlock_irqrestore(&reset_data.lock, flags); ++ ++ hrtimer_cancel(&reset_data.timer); ++ ++ if (reset_data.stage == 2) { ++ /* The reset has already started. ++ * Wait for the reset to complete ++ */ ++ wait_event(kbdev->hwaccess.backend.reset_wait, ++ atomic_read(&kbdev->hwaccess.backend.reset_gpu) ++ == KBASE_RESET_GPU_NOT_PENDING); ++ } ++ destroy_hrtimer_on_stack(&reset_data.timer); ++ ++ dev_dbg(kbdev->dev, "Zap: Finished Context %p", kctx); ++ ++ /* Ensure that the signallers of the waitqs have finished */ ++ mutex_lock(&kctx->jctx.lock); ++ mutex_lock(&kctx->jctx.sched_info.ctx.jsctx_mutex); ++ mutex_unlock(&kctx->jctx.sched_info.ctx.jsctx_mutex); ++ mutex_unlock(&kctx->jctx.lock); ++} ++ ++u32 kbase_backend_get_current_flush_id(struct kbase_device *kbdev) ++{ ++ u32 flush_id = 0; ++ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_FLUSH_REDUCTION)) { ++ mutex_lock(&kbdev->pm.lock); ++ if (kbdev->pm.backend.gpu_powered) ++ flush_id = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(LATEST_FLUSH), NULL); ++ mutex_unlock(&kbdev->pm.lock); ++ } ++ ++ return flush_id; ++} ++ ++int kbase_job_slot_init(struct kbase_device *kbdev) ++{ ++#if KBASE_GPU_RESET_EN ++ kbdev->hwaccess.backend.reset_workq = alloc_workqueue( ++ "Mali reset workqueue", 0, 1); ++ if (NULL == kbdev->hwaccess.backend.reset_workq) ++ return -EINVAL; ++ ++ KBASE_DEBUG_ASSERT(0 == ++ object_is_on_stack(&kbdev->hwaccess.backend.reset_work)); ++ INIT_WORK(&kbdev->hwaccess.backend.reset_work, ++ kbasep_reset_timeout_worker); ++ ++ hrtimer_init(&kbdev->hwaccess.backend.reset_timer, CLOCK_MONOTONIC, ++ HRTIMER_MODE_REL); ++ kbdev->hwaccess.backend.reset_timer.function = ++ kbasep_reset_timer_callback; ++#endif ++ ++ return 0; ++} ++KBASE_EXPORT_TEST_API(kbase_job_slot_init); ++ ++void kbase_job_slot_halt(struct kbase_device *kbdev) ++{ ++ CSTD_UNUSED(kbdev); ++} ++ ++void kbase_job_slot_term(struct kbase_device *kbdev) ++{ ++#if KBASE_GPU_RESET_EN ++ destroy_workqueue(kbdev->hwaccess.backend.reset_workq); ++#endif ++} ++KBASE_EXPORT_TEST_API(kbase_job_slot_term); ++ ++#if KBASE_GPU_RESET_EN ++/** ++ * kbasep_check_for_afbc_on_slot() - Check whether AFBC is in use on this slot ++ * @kbdev: kbase device pointer ++ * @kctx: context to check against ++ * @js: slot to check ++ * @target_katom: An atom to check, or NULL if all atoms from @kctx on ++ * slot @js should be checked ++ * ++ * This checks are based upon parameters that would normally be passed to ++ * kbase_job_slot_hardstop(). ++ * ++ * In the event of @target_katom being NULL, this will check the last jobs that ++ * are likely to be running on the slot to see if a) they belong to kctx, and ++ * so would be stopped, and b) whether they have AFBC ++ * ++ * In that case, It's guaranteed that a job currently executing on the HW with ++ * AFBC will be detected. However, this is a conservative check because it also ++ * detects jobs that have just completed too. ++ * ++ * Return: true when hard-stop _might_ stop an afbc atom, else false. ++ */ ++static bool kbasep_check_for_afbc_on_slot(struct kbase_device *kbdev, ++ struct kbase_context *kctx, int js, ++ struct kbase_jd_atom *target_katom) ++{ ++ bool ret = false; ++ int i; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ /* When we have an atom the decision can be made straight away. */ ++ if (target_katom) ++ return !!(target_katom->core_req & BASE_JD_REQ_FS_AFBC); ++ ++ /* Otherwise, we must chweck the hardware to see if it has atoms from ++ * this context with AFBC. */ ++ for (i = 0; i < kbase_backend_nr_atoms_on_slot(kbdev, js); i++) { ++ struct kbase_jd_atom *katom; ++ ++ katom = kbase_gpu_inspect(kbdev, js, i); ++ if (!katom) ++ continue; ++ ++ /* Ignore atoms from other contexts, they won't be stopped when ++ * we use this for checking if we should hard-stop them */ ++ if (katom->kctx != kctx) ++ continue; ++ ++ /* An atom on this slot and this context: check for AFBC */ ++ if (katom->core_req & BASE_JD_REQ_FS_AFBC) { ++ ret = true; ++ break; ++ } ++ } ++ ++ return ret; ++} ++#endif /* KBASE_GPU_RESET_EN */ ++ ++/** ++ * kbase_job_slot_softstop_swflags - Soft-stop a job with flags ++ * @kbdev: The kbase device ++ * @js: The job slot to soft-stop ++ * @target_katom: The job that should be soft-stopped (or NULL for any job) ++ * @sw_flags: Flags to pass in about the soft-stop ++ * ++ * Context: ++ * The job slot lock must be held when calling this function. ++ * The job slot must not already be in the process of being soft-stopped. ++ * ++ * Soft-stop the specified job slot, with extra information about the stop ++ * ++ * Where possible any job in the next register is evicted before the soft-stop. ++ */ ++void kbase_job_slot_softstop_swflags(struct kbase_device *kbdev, int js, ++ struct kbase_jd_atom *target_katom, u32 sw_flags) ++{ ++ KBASE_DEBUG_ASSERT(!(sw_flags & JS_COMMAND_MASK)); ++ kbase_backend_soft_hard_stop_slot(kbdev, NULL, js, target_katom, ++ JS_COMMAND_SOFT_STOP | sw_flags); ++} ++ ++/** ++ * kbase_job_slot_softstop - Soft-stop the specified job slot ++ * @kbdev: The kbase device ++ * @js: The job slot to soft-stop ++ * @target_katom: The job that should be soft-stopped (or NULL for any job) ++ * Context: ++ * The job slot lock must be held when calling this function. ++ * The job slot must not already be in the process of being soft-stopped. ++ * ++ * Where possible any job in the next register is evicted before the soft-stop. ++ */ ++void kbase_job_slot_softstop(struct kbase_device *kbdev, int js, ++ struct kbase_jd_atom *target_katom) ++{ ++ kbase_job_slot_softstop_swflags(kbdev, js, target_katom, 0u); ++} ++ ++/** ++ * kbase_job_slot_hardstop - Hard-stop the specified job slot ++ * @kctx: The kbase context that contains the job(s) that should ++ * be hard-stopped ++ * @js: The job slot to hard-stop ++ * @target_katom: The job that should be hard-stopped (or NULL for all ++ * jobs from the context) ++ * Context: ++ * The job slot lock must be held when calling this function. ++ */ ++void kbase_job_slot_hardstop(struct kbase_context *kctx, int js, ++ struct kbase_jd_atom *target_katom) ++{ ++ struct kbase_device *kbdev = kctx->kbdev; ++ bool stopped; ++#if KBASE_GPU_RESET_EN ++ /* We make the check for AFBC before evicting/stopping atoms. Note ++ * that no other thread can modify the slots whilst we have the ++ * hwaccess_lock. */ ++ int needs_workaround_for_afbc = ++ kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_T76X_3542) ++ && kbasep_check_for_afbc_on_slot(kbdev, kctx, js, ++ target_katom); ++#endif ++ ++ stopped = kbase_backend_soft_hard_stop_slot(kbdev, kctx, js, ++ target_katom, ++ JS_COMMAND_HARD_STOP); ++#if KBASE_GPU_RESET_EN ++ if (stopped && (kbase_hw_has_issue(kctx->kbdev, BASE_HW_ISSUE_8401) || ++ kbase_hw_has_issue(kctx->kbdev, BASE_HW_ISSUE_9510) || ++ needs_workaround_for_afbc)) { ++ /* MIDBASE-2916 if a fragment job with AFBC encoding is ++ * hardstopped, ensure to do a soft reset also in order to ++ * clear the GPU status. ++ * Workaround for HW issue 8401 has an issue,so after ++ * hard-stopping just reset the GPU. This will ensure that the ++ * jobs leave the GPU.*/ ++ if (kbase_prepare_to_reset_gpu_locked(kbdev)) { ++ dev_err(kbdev->dev, "Issueing GPU soft-reset after hard stopping due to hardware issue"); ++ kbase_reset_gpu_locked(kbdev); ++ } ++ } ++#endif ++} ++ ++/** ++ * kbase_job_check_enter_disjoint - potentiall enter disjoint mode ++ * @kbdev: kbase device ++ * @action: the event which has occurred ++ * @core_reqs: core requirements of the atom ++ * @target_katom: the atom which is being affected ++ * ++ * For a certain soft/hard-stop action, work out whether to enter disjoint ++ * state. ++ * ++ * This does not register multiple disjoint events if the atom has already ++ * started a disjoint period ++ * ++ * @core_reqs can be supplied as 0 if the atom had not started on the hardware ++ * (and so a 'real' soft/hard-stop was not required, but it still interrupted ++ * flow, perhaps on another context) ++ * ++ * kbase_job_check_leave_disjoint() should be used to end the disjoint ++ * state when the soft/hard-stop action is complete ++ */ ++void kbase_job_check_enter_disjoint(struct kbase_device *kbdev, u32 action, ++ base_jd_core_req core_reqs, struct kbase_jd_atom *target_katom) ++{ ++ u32 hw_action = action & JS_COMMAND_MASK; ++ ++ /* For hard-stop, don't enter if hard-stop not allowed */ ++ if (hw_action == JS_COMMAND_HARD_STOP && ++ !kbasep_hard_stop_allowed(kbdev, core_reqs)) ++ return; ++ ++ /* For soft-stop, don't enter if soft-stop not allowed, or isn't ++ * causing disjoint */ ++ if (hw_action == JS_COMMAND_SOFT_STOP && ++ !(kbasep_soft_stop_allowed(kbdev, target_katom) && ++ (action & JS_COMMAND_SW_CAUSES_DISJOINT))) ++ return; ++ ++ /* Nothing to do if already logged disjoint state on this atom */ ++ if (target_katom->atom_flags & KBASE_KATOM_FLAG_IN_DISJOINT) ++ return; ++ ++ target_katom->atom_flags |= KBASE_KATOM_FLAG_IN_DISJOINT; ++ kbase_disjoint_state_up(kbdev); ++} ++ ++/** ++ * kbase_job_check_enter_disjoint - potentially leave disjoint state ++ * @kbdev: kbase device ++ * @target_katom: atom which is finishing ++ * ++ * Work out whether to leave disjoint state when finishing an atom that was ++ * originated by kbase_job_check_enter_disjoint(). ++ */ ++void kbase_job_check_leave_disjoint(struct kbase_device *kbdev, ++ struct kbase_jd_atom *target_katom) ++{ ++ if (target_katom->atom_flags & KBASE_KATOM_FLAG_IN_DISJOINT) { ++ target_katom->atom_flags &= ~KBASE_KATOM_FLAG_IN_DISJOINT; ++ kbase_disjoint_state_down(kbdev); ++ } ++} ++ ++ ++#if KBASE_GPU_RESET_EN ++static void kbase_debug_dump_registers(struct kbase_device *kbdev) ++{ ++ int i; ++ ++ kbase_io_history_dump(kbdev); ++ ++ dev_err(kbdev->dev, "Register state:"); ++ dev_err(kbdev->dev, " GPU_IRQ_RAWSTAT=0x%08x GPU_STATUS=0x%08x", ++ kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), NULL), ++ kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_STATUS), NULL)); ++ dev_err(kbdev->dev, " JOB_IRQ_RAWSTAT=0x%08x JOB_IRQ_JS_STATE=0x%08x", ++ kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_RAWSTAT), NULL), ++ kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_JS_STATE), NULL)); ++ for (i = 0; i < 3; i++) { ++ dev_err(kbdev->dev, " JS%d_STATUS=0x%08x JS%d_HEAD_LO=0x%08x", ++ i, kbase_reg_read(kbdev, JOB_SLOT_REG(i, JS_STATUS), ++ NULL), ++ i, kbase_reg_read(kbdev, JOB_SLOT_REG(i, JS_HEAD_LO), ++ NULL)); ++ } ++ dev_err(kbdev->dev, " MMU_IRQ_RAWSTAT=0x%08x GPU_FAULTSTATUS=0x%08x", ++ kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_RAWSTAT), NULL), ++ kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_FAULTSTATUS), NULL)); ++ dev_err(kbdev->dev, " GPU_IRQ_MASK=0x%08x JOB_IRQ_MASK=0x%08x MMU_IRQ_MASK=0x%08x", ++ kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL), ++ kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), NULL), ++ kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK), NULL)); ++ dev_err(kbdev->dev, " PWR_OVERRIDE0=0x%08x PWR_OVERRIDE1=0x%08x", ++ kbase_reg_read(kbdev, GPU_CONTROL_REG(PWR_OVERRIDE0), NULL), ++ kbase_reg_read(kbdev, GPU_CONTROL_REG(PWR_OVERRIDE1), NULL)); ++ dev_err(kbdev->dev, " SHADER_CONFIG=0x%08x L2_MMU_CONFIG=0x%08x", ++ kbase_reg_read(kbdev, GPU_CONTROL_REG(SHADER_CONFIG), NULL), ++ kbase_reg_read(kbdev, GPU_CONTROL_REG(L2_MMU_CONFIG), NULL)); ++ dev_err(kbdev->dev, " TILER_CONFIG=0x%08x JM_CONFIG=0x%08x", ++ kbase_reg_read(kbdev, GPU_CONTROL_REG(TILER_CONFIG), NULL), ++ kbase_reg_read(kbdev, GPU_CONTROL_REG(JM_CONFIG), NULL)); ++} ++ ++static void kbasep_reset_timeout_worker(struct work_struct *data) ++{ ++ unsigned long flags; ++ struct kbase_device *kbdev; ++ ktime_t end_timestamp = ktime_get(); ++ struct kbasep_js_device_data *js_devdata; ++ bool try_schedule = false; ++ bool silent = false; ++ u32 max_loops = KBASE_CLEAN_CACHE_MAX_LOOPS; ++ ++ KBASE_DEBUG_ASSERT(data); ++ ++ kbdev = container_of(data, struct kbase_device, ++ hwaccess.backend.reset_work); ++ ++ KBASE_DEBUG_ASSERT(kbdev); ++ js_devdata = &kbdev->js_data; ++ ++ if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) == ++ KBASE_RESET_GPU_SILENT) ++ silent = true; ++ ++ KBASE_TRACE_ADD(kbdev, JM_BEGIN_RESET_WORKER, NULL, NULL, 0u, 0); ++ ++ /* Suspend vinstr. ++ * This call will block until vinstr is suspended. */ ++ kbase_vinstr_suspend(kbdev->vinstr_ctx); ++ ++ /* Make sure the timer has completed - this cannot be done from ++ * interrupt context, so this cannot be done within ++ * kbasep_try_reset_gpu_early. */ ++ hrtimer_cancel(&kbdev->hwaccess.backend.reset_timer); ++ ++ if (kbase_pm_context_active_handle_suspend(kbdev, ++ KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) { ++ /* This would re-activate the GPU. Since it's already idle, ++ * there's no need to reset it */ ++ atomic_set(&kbdev->hwaccess.backend.reset_gpu, ++ KBASE_RESET_GPU_NOT_PENDING); ++ kbase_disjoint_state_down(kbdev); ++ wake_up(&kbdev->hwaccess.backend.reset_wait); ++ kbase_vinstr_resume(kbdev->vinstr_ctx); ++ return; ++ } ++ ++ KBASE_DEBUG_ASSERT(kbdev->irq_reset_flush == false); ++ ++ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); ++ spin_lock(&kbdev->hwaccess_lock); ++ spin_lock(&kbdev->mmu_mask_change); ++ /* We're about to flush out the IRQs and their bottom half's */ ++ kbdev->irq_reset_flush = true; ++ ++ /* Disable IRQ to avoid IRQ handlers to kick in after releasing the ++ * spinlock; this also clears any outstanding interrupts */ ++ kbase_pm_disable_interrupts_nolock(kbdev); ++ ++ spin_unlock(&kbdev->mmu_mask_change); ++ spin_unlock(&kbdev->hwaccess_lock); ++ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); ++ ++ /* Ensure that any IRQ handlers have finished ++ * Must be done without any locks IRQ handlers will take */ ++ kbase_synchronize_irqs(kbdev); ++ ++ /* Flush out any in-flight work items */ ++ kbase_flush_mmu_wqs(kbdev); ++ ++ /* The flush has completed so reset the active indicator */ ++ kbdev->irq_reset_flush = false; ++ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TMIX_8463)) { ++ /* Ensure that L2 is not transitioning when we send the reset ++ * command */ ++ while (--max_loops && kbase_pm_get_trans_cores(kbdev, ++ KBASE_PM_CORE_L2)) ++ ; ++ ++ WARN(!max_loops, "L2 power transition timed out while trying to reset\n"); ++ } ++ ++ mutex_lock(&kbdev->pm.lock); ++ /* We hold the pm lock, so there ought to be a current policy */ ++ KBASE_DEBUG_ASSERT(kbdev->pm.backend.pm_current_policy); ++ ++ /* All slot have been soft-stopped and we've waited ++ * SOFT_STOP_RESET_TIMEOUT for the slots to clear, at this point we ++ * assume that anything that is still left on the GPU is stuck there and ++ * we'll kill it when we reset the GPU */ ++ ++ if (!silent) ++ dev_err(kbdev->dev, "Resetting GPU (allowing up to %d ms)", ++ RESET_TIMEOUT); ++ ++ /* Output the state of some interesting registers to help in the ++ * debugging of GPU resets */ ++ if (!silent) ++ kbase_debug_dump_registers(kbdev); ++ ++ /* Complete any jobs that were still on the GPU */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbdev->protected_mode = false; ++ kbase_backend_reset(kbdev, &end_timestamp); ++ kbase_pm_metrics_update(kbdev, NULL); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ /* Reset the GPU */ ++ kbase_pm_init_hw(kbdev, 0); ++ ++ mutex_unlock(&kbdev->pm.lock); ++ ++ mutex_lock(&js_devdata->runpool_mutex); ++ ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_ctx_sched_restore_all_as(kbdev); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ ++ kbase_pm_enable_interrupts(kbdev); ++ ++ atomic_set(&kbdev->hwaccess.backend.reset_gpu, ++ KBASE_RESET_GPU_NOT_PENDING); ++ ++ kbase_disjoint_state_down(kbdev); ++ ++ wake_up(&kbdev->hwaccess.backend.reset_wait); ++ if (!silent) ++ dev_err(kbdev->dev, "Reset complete"); ++ ++ if (js_devdata->nr_contexts_pullable > 0 && !kbdev->poweroff_pending) ++ try_schedule = true; ++ ++ mutex_unlock(&js_devdata->runpool_mutex); ++ ++ mutex_lock(&kbdev->pm.lock); ++ ++ /* Find out what cores are required now */ ++ kbase_pm_update_cores_state(kbdev); ++ ++ /* Synchronously request and wait for those cores, because if ++ * instrumentation is enabled it would need them immediately. */ ++ kbase_pm_check_transitions_sync(kbdev); ++ ++ mutex_unlock(&kbdev->pm.lock); ++ ++ /* Try submitting some jobs to restart processing */ ++ if (try_schedule) { ++ KBASE_TRACE_ADD(kbdev, JM_SUBMIT_AFTER_RESET, NULL, NULL, 0u, ++ 0); ++ kbase_js_sched_all(kbdev); ++ } ++ ++ /* Process any pending slot updates */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_backend_slot_update(kbdev); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ kbase_pm_context_idle(kbdev); ++ ++ /* Release vinstr */ ++ kbase_vinstr_resume(kbdev->vinstr_ctx); ++ ++ KBASE_TRACE_ADD(kbdev, JM_END_RESET_WORKER, NULL, NULL, 0u, 0); ++} ++ ++static enum hrtimer_restart kbasep_reset_timer_callback(struct hrtimer *timer) ++{ ++ struct kbase_device *kbdev = container_of(timer, struct kbase_device, ++ hwaccess.backend.reset_timer); ++ ++ KBASE_DEBUG_ASSERT(kbdev); ++ ++ /* Reset still pending? */ ++ if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu, ++ KBASE_RESET_GPU_COMMITTED, KBASE_RESET_GPU_HAPPENING) == ++ KBASE_RESET_GPU_COMMITTED) ++ queue_work(kbdev->hwaccess.backend.reset_workq, ++ &kbdev->hwaccess.backend.reset_work); ++ ++ return HRTIMER_NORESTART; ++} ++ ++/* ++ * If all jobs are evicted from the GPU then we can reset the GPU ++ * immediately instead of waiting for the timeout to elapse ++ */ ++ ++static void kbasep_try_reset_gpu_early_locked(struct kbase_device *kbdev) ++{ ++ int i; ++ int pending_jobs = 0; ++ ++ KBASE_DEBUG_ASSERT(kbdev); ++ ++ /* Count the number of jobs */ ++ for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) ++ pending_jobs += kbase_backend_nr_atoms_submitted(kbdev, i); ++ ++ if (pending_jobs > 0) { ++ /* There are still jobs on the GPU - wait */ ++ return; ++ } ++ ++ /* To prevent getting incorrect registers when dumping failed job, ++ * skip early reset. ++ */ ++ if (kbdev->job_fault_debug != false) ++ return; ++ ++ /* Check that the reset has been committed to (i.e. kbase_reset_gpu has ++ * been called), and that no other thread beat this thread to starting ++ * the reset */ ++ if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu, ++ KBASE_RESET_GPU_COMMITTED, KBASE_RESET_GPU_HAPPENING) != ++ KBASE_RESET_GPU_COMMITTED) { ++ /* Reset has already occurred */ ++ return; ++ } ++ ++ queue_work(kbdev->hwaccess.backend.reset_workq, ++ &kbdev->hwaccess.backend.reset_work); ++} ++ ++static void kbasep_try_reset_gpu_early(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ struct kbasep_js_device_data *js_devdata; ++ ++ js_devdata = &kbdev->js_data; ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbasep_try_reset_gpu_early_locked(kbdev); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++} ++ ++/** ++ * kbase_prepare_to_reset_gpu_locked - Prepare for resetting the GPU ++ * @kbdev: kbase device ++ * ++ * This function just soft-stops all the slots to ensure that as many jobs as ++ * possible are saved. ++ * ++ * Return: ++ * The function returns a boolean which should be interpreted as follows: ++ * true - Prepared for reset, kbase_reset_gpu_locked should be called. ++ * false - Another thread is performing a reset, kbase_reset_gpu should ++ * not be called. ++ */ ++bool kbase_prepare_to_reset_gpu_locked(struct kbase_device *kbdev) ++{ ++ int i; ++ ++ KBASE_DEBUG_ASSERT(kbdev); ++ ++ if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu, ++ KBASE_RESET_GPU_NOT_PENDING, ++ KBASE_RESET_GPU_PREPARED) != ++ KBASE_RESET_GPU_NOT_PENDING) { ++ /* Some other thread is already resetting the GPU */ ++ return false; ++ } ++ ++ kbase_disjoint_state_up(kbdev); ++ ++ for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) ++ kbase_job_slot_softstop(kbdev, i, NULL); ++ ++ return true; ++} ++ ++bool kbase_prepare_to_reset_gpu(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ bool ret; ++ struct kbasep_js_device_data *js_devdata; ++ ++ js_devdata = &kbdev->js_data; ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ret = kbase_prepare_to_reset_gpu_locked(kbdev); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ return ret; ++} ++KBASE_EXPORT_TEST_API(kbase_prepare_to_reset_gpu); ++ ++/* ++ * This function should be called after kbase_prepare_to_reset_gpu if it ++ * returns true. It should never be called without a corresponding call to ++ * kbase_prepare_to_reset_gpu. ++ * ++ * After this function is called (or not called if kbase_prepare_to_reset_gpu ++ * returned false), the caller should wait for ++ * kbdev->hwaccess.backend.reset_waitq to be signalled to know when the reset ++ * has completed. ++ */ ++void kbase_reset_gpu(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev); ++ ++ /* Note this is an assert/atomic_set because it is a software issue for ++ * a race to be occuring here */ ++ KBASE_DEBUG_ASSERT(atomic_read(&kbdev->hwaccess.backend.reset_gpu) == ++ KBASE_RESET_GPU_PREPARED); ++ atomic_set(&kbdev->hwaccess.backend.reset_gpu, ++ KBASE_RESET_GPU_COMMITTED); ++ ++ dev_err(kbdev->dev, "Preparing to soft-reset GPU: Waiting (upto %d ms) for all jobs to complete soft-stop\n", ++ kbdev->reset_timeout_ms); ++ ++ hrtimer_start(&kbdev->hwaccess.backend.reset_timer, ++ HR_TIMER_DELAY_MSEC(kbdev->reset_timeout_ms), ++ HRTIMER_MODE_REL); ++ ++ /* Try resetting early */ ++ kbasep_try_reset_gpu_early(kbdev); ++} ++KBASE_EXPORT_TEST_API(kbase_reset_gpu); ++ ++void kbase_reset_gpu_locked(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev); ++ ++ /* Note this is an assert/atomic_set because it is a software issue for ++ * a race to be occuring here */ ++ KBASE_DEBUG_ASSERT(atomic_read(&kbdev->hwaccess.backend.reset_gpu) == ++ KBASE_RESET_GPU_PREPARED); ++ atomic_set(&kbdev->hwaccess.backend.reset_gpu, ++ KBASE_RESET_GPU_COMMITTED); ++ ++ dev_err(kbdev->dev, "Preparing to soft-reset GPU: Waiting (upto %d ms) for all jobs to complete soft-stop\n", ++ kbdev->reset_timeout_ms); ++ hrtimer_start(&kbdev->hwaccess.backend.reset_timer, ++ HR_TIMER_DELAY_MSEC(kbdev->reset_timeout_ms), ++ HRTIMER_MODE_REL); ++ ++ /* Try resetting early */ ++ kbasep_try_reset_gpu_early_locked(kbdev); ++} ++ ++void kbase_reset_gpu_silent(struct kbase_device *kbdev) ++{ ++ if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu, ++ KBASE_RESET_GPU_NOT_PENDING, ++ KBASE_RESET_GPU_SILENT) != ++ KBASE_RESET_GPU_NOT_PENDING) { ++ /* Some other thread is already resetting the GPU */ ++ return; ++ } ++ ++ kbase_disjoint_state_up(kbdev); ++ ++ queue_work(kbdev->hwaccess.backend.reset_workq, ++ &kbdev->hwaccess.backend.reset_work); ++} ++ ++bool kbase_reset_gpu_active(struct kbase_device *kbdev) ++{ ++ if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) == ++ KBASE_RESET_GPU_NOT_PENDING) ++ return false; ++ ++ return true; ++} ++#endif /* KBASE_GPU_RESET_EN */ +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_internal.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_internal.h +new file mode 100755 +index 000000000000..1f382b3c1af4 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_internal.h +@@ -0,0 +1,164 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * Job Manager backend-specific low-level APIs. ++ */ ++ ++#ifndef _KBASE_JM_HWACCESS_H_ ++#define _KBASE_JM_HWACCESS_H_ ++ ++#include ++#include ++#include ++ ++#include ++ ++/** ++ * kbase_job_submit_nolock() - Submit a job to a certain job-slot ++ * @kbdev: Device pointer ++ * @katom: Atom to submit ++ * @js: Job slot to submit on ++ * ++ * The caller must check kbasep_jm_is_submit_slots_free() != false before ++ * calling this. ++ * ++ * The following locking conditions are made on the caller: ++ * - it must hold the hwaccess_lock ++ */ ++void kbase_job_submit_nolock(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom, int js); ++ ++/** ++ * kbase_job_done_slot() - Complete the head job on a particular job-slot ++ * @kbdev: Device pointer ++ * @s: Job slot ++ * @completion_code: Completion code of job reported by GPU ++ * @job_tail: Job tail address reported by GPU ++ * @end_timestamp: Timestamp of job completion ++ */ ++void kbase_job_done_slot(struct kbase_device *kbdev, int s, u32 completion_code, ++ u64 job_tail, ktime_t *end_timestamp); ++ ++#ifdef CONFIG_GPU_TRACEPOINTS ++static inline char *kbasep_make_job_slot_string(int js, char *js_string, ++ size_t js_size) ++{ ++ snprintf(js_string, js_size, "job_slot_%i", js); ++ return js_string; ++} ++#endif ++ ++/** ++ * kbase_job_hw_submit() - Submit a job to the GPU ++ * @kbdev: Device pointer ++ * @katom: Atom to submit ++ * @js: Job slot to submit on ++ * ++ * The caller must check kbasep_jm_is_submit_slots_free() != false before ++ * calling this. ++ * ++ * The following locking conditions are made on the caller: ++ * - it must hold the hwaccess_lock ++ */ ++void kbase_job_hw_submit(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom, ++ int js); ++ ++/** ++ * kbasep_job_slot_soft_or_hard_stop_do_action() - Perform a soft or hard stop ++ * on the specified atom ++ * @kbdev: Device pointer ++ * @js: Job slot to stop on ++ * @action: The action to perform, either JSn_COMMAND_HARD_STOP or ++ * JSn_COMMAND_SOFT_STOP ++ * @core_reqs: Core requirements of atom to stop ++ * @target_katom: Atom to stop ++ * ++ * The following locking conditions are made on the caller: ++ * - it must hold the hwaccess_lock ++ */ ++void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev, ++ int js, ++ u32 action, ++ base_jd_core_req core_reqs, ++ struct kbase_jd_atom *target_katom); ++ ++/** ++ * kbase_backend_soft_hard_stop_slot() - Soft or hard stop jobs on a given job ++ * slot belonging to a given context. ++ * @kbdev: Device pointer ++ * @kctx: Context pointer. May be NULL ++ * @katom: Specific atom to stop. May be NULL ++ * @js: Job slot to hard stop ++ * @action: The action to perform, either JSn_COMMAND_HARD_STOP or ++ * JSn_COMMAND_SOFT_STOP ++ * ++ * If no context is provided then all jobs on the slot will be soft or hard ++ * stopped. ++ * ++ * If a katom is provided then only that specific atom will be stopped. In this ++ * case the kctx parameter is ignored. ++ * ++ * Jobs that are on the slot but are not yet on the GPU will be unpulled and ++ * returned to the job scheduler. ++ * ++ * Return: true if an atom was stopped, false otherwise ++ */ ++bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ int js, ++ struct kbase_jd_atom *katom, ++ u32 action); ++ ++/** ++ * kbase_job_slot_init - Initialise job slot framework ++ * @kbdev: Device pointer ++ * ++ * Called on driver initialisation ++ * ++ * Return: 0 on success ++ */ ++int kbase_job_slot_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_job_slot_halt - Halt the job slot framework ++ * @kbdev: Device pointer ++ * ++ * Should prevent any further job slot processing ++ */ ++void kbase_job_slot_halt(struct kbase_device *kbdev); ++ ++/** ++ * kbase_job_slot_term - Terminate job slot framework ++ * @kbdev: Device pointer ++ * ++ * Called on driver termination ++ */ ++void kbase_job_slot_term(struct kbase_device *kbdev); ++ ++/** ++ * kbase_gpu_cacheclean - Cause a GPU cache clean & flush ++ * @kbdev: Device pointer ++ * ++ * Caller must not be in IRQ context ++ */ ++void kbase_gpu_cacheclean(struct kbase_device *kbdev); ++ ++#endif /* _KBASE_JM_HWACCESS_H_ */ +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_rb.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_rb.c +new file mode 100755 +index 000000000000..4b4541660ec4 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_rb.c +@@ -0,0 +1,1952 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++/* ++ * Register-based HW access backend specific APIs ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++/* Return whether the specified ringbuffer is empty. HW access lock must be ++ * held */ ++#define SLOT_RB_EMPTY(rb) (rb->write_idx == rb->read_idx) ++/* Return number of atoms currently in the specified ringbuffer. HW access lock ++ * must be held */ ++#define SLOT_RB_ENTRIES(rb) (int)(s8)(rb->write_idx - rb->read_idx) ++ ++static void kbase_gpu_release_atom(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom, ++ ktime_t *end_timestamp); ++ ++/** ++ * kbase_gpu_enqueue_atom - Enqueue an atom in the HW access ringbuffer ++ * @kbdev: Device pointer ++ * @katom: Atom to enqueue ++ * ++ * Context: Caller must hold the HW access lock ++ */ ++static void kbase_gpu_enqueue_atom(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom) ++{ ++ struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[katom->slot_nr]; ++ ++ WARN_ON(SLOT_RB_ENTRIES(rb) >= SLOT_RB_SIZE); ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ rb->entries[rb->write_idx & SLOT_RB_MASK].katom = katom; ++ rb->write_idx++; ++ ++ katom->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_BLOCKED; ++} ++ ++/** ++ * kbase_gpu_dequeue_atom - Remove an atom from the HW access ringbuffer, once ++ * it has been completed ++ * @kbdev: Device pointer ++ * @js: Job slot to remove atom from ++ * @end_timestamp: Pointer to timestamp of atom completion. May be NULL, in ++ * which case current time will be used. ++ * ++ * Context: Caller must hold the HW access lock ++ * ++ * Return: Atom removed from ringbuffer ++ */ ++static struct kbase_jd_atom *kbase_gpu_dequeue_atom(struct kbase_device *kbdev, ++ int js, ++ ktime_t *end_timestamp) ++{ ++ struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js]; ++ struct kbase_jd_atom *katom; ++ ++ if (SLOT_RB_EMPTY(rb)) { ++ WARN(1, "GPU ringbuffer unexpectedly empty\n"); ++ return NULL; ++ } ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ katom = rb->entries[rb->read_idx & SLOT_RB_MASK].katom; ++ ++ kbase_gpu_release_atom(kbdev, katom, end_timestamp); ++ ++ rb->read_idx++; ++ ++ katom->gpu_rb_state = KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB; ++ ++ kbase_js_debug_log_current_affinities(kbdev); ++ ++ return katom; ++} ++ ++struct kbase_jd_atom *kbase_gpu_inspect(struct kbase_device *kbdev, int js, ++ int idx) ++{ ++ struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js]; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if ((SLOT_RB_ENTRIES(rb) - 1) < idx) ++ return NULL; /* idx out of range */ ++ ++ return rb->entries[(rb->read_idx + idx) & SLOT_RB_MASK].katom; ++} ++ ++struct kbase_jd_atom *kbase_backend_inspect_head(struct kbase_device *kbdev, ++ int js) ++{ ++ return kbase_gpu_inspect(kbdev, js, 0); ++} ++ ++struct kbase_jd_atom *kbase_backend_inspect_tail(struct kbase_device *kbdev, ++ int js) ++{ ++ struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js]; ++ ++ if (SLOT_RB_EMPTY(rb)) ++ return NULL; ++ ++ return rb->entries[(rb->write_idx - 1) & SLOT_RB_MASK].katom; ++} ++ ++/** ++ * kbase_gpu_atoms_submitted - Inspect whether a slot has any atoms currently ++ * on the GPU ++ * @kbdev: Device pointer ++ * @js: Job slot to inspect ++ * ++ * Return: true if there are atoms on the GPU for slot js, ++ * false otherwise ++ */ ++static bool kbase_gpu_atoms_submitted(struct kbase_device *kbdev, int js) ++{ ++ int i; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ for (i = 0; i < SLOT_RB_SIZE; i++) { ++ struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i); ++ ++ if (!katom) ++ return false; ++ if (katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED || ++ katom->gpu_rb_state == KBASE_ATOM_GPU_RB_READY) ++ return true; ++ } ++ ++ return false; ++} ++ ++/** ++ * kbase_gpu_atoms_submitted_any() - Inspect whether there are any atoms ++ * currently on the GPU ++ * @kbdev: Device pointer ++ * ++ * Return: true if there are any atoms on the GPU, false otherwise ++ */ ++static bool kbase_gpu_atoms_submitted_any(struct kbase_device *kbdev) ++{ ++ int js; ++ int i; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { ++ for (i = 0; i < SLOT_RB_SIZE; i++) { ++ struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i); ++ ++ if (katom && katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED) ++ return true; ++ } ++ } ++ return false; ++} ++ ++int kbase_backend_nr_atoms_submitted(struct kbase_device *kbdev, int js) ++{ ++ int nr = 0; ++ int i; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ for (i = 0; i < SLOT_RB_SIZE; i++) { ++ struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i); ++ ++ if (katom && (katom->gpu_rb_state == ++ KBASE_ATOM_GPU_RB_SUBMITTED)) ++ nr++; ++ } ++ ++ return nr; ++} ++ ++int kbase_backend_nr_atoms_on_slot(struct kbase_device *kbdev, int js) ++{ ++ int nr = 0; ++ int i; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ for (i = 0; i < SLOT_RB_SIZE; i++) { ++ if (kbase_gpu_inspect(kbdev, js, i)) ++ nr++; ++ } ++ ++ return nr; ++} ++ ++static int kbase_gpu_nr_atoms_on_slot_min(struct kbase_device *kbdev, int js, ++ enum kbase_atom_gpu_rb_state min_rb_state) ++{ ++ int nr = 0; ++ int i; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ for (i = 0; i < SLOT_RB_SIZE; i++) { ++ struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i); ++ ++ if (katom && (katom->gpu_rb_state >= min_rb_state)) ++ nr++; ++ } ++ ++ return nr; ++} ++ ++/** ++ * check_secure_atom - Check if the given atom is in the given secure state and ++ * has a ringbuffer state of at least ++ * KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION ++ * @katom: Atom pointer ++ * @secure: Desired secure state ++ * ++ * Return: true if atom is in the given state, false otherwise ++ */ ++static bool check_secure_atom(struct kbase_jd_atom *katom, bool secure) ++{ ++ if (katom->gpu_rb_state >= ++ KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION && ++ ((kbase_jd_katom_is_protected(katom) && secure) || ++ (!kbase_jd_katom_is_protected(katom) && !secure))) ++ return true; ++ ++ return false; ++} ++ ++/** ++ * kbase_gpu_check_secure_atoms - Check if there are any atoms in the given ++ * secure state in the ringbuffers of at least ++ * state ++ * KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE ++ * @kbdev: Device pointer ++ * @secure: Desired secure state ++ * ++ * Return: true if any atoms are in the given state, false otherwise ++ */ ++static bool kbase_gpu_check_secure_atoms(struct kbase_device *kbdev, ++ bool secure) ++{ ++ int js, i; ++ ++ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { ++ for (i = 0; i < SLOT_RB_SIZE; i++) { ++ struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, ++ js, i); ++ ++ if (katom) { ++ if (check_secure_atom(katom, secure)) ++ return true; ++ } ++ } ++ } ++ ++ return false; ++} ++ ++int kbase_backend_slot_free(struct kbase_device *kbdev, int js) ++{ ++ if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) != ++ KBASE_RESET_GPU_NOT_PENDING) { ++ /* The GPU is being reset - so prevent submission */ ++ return 0; ++ } ++ ++ return SLOT_RB_SIZE - kbase_backend_nr_atoms_on_slot(kbdev, js); ++} ++ ++ ++static void kbasep_js_job_check_deref_cores(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom); ++ ++static bool kbasep_js_job_check_ref_cores(struct kbase_device *kbdev, ++ int js, ++ struct kbase_jd_atom *katom) ++{ ++ /* The most recently checked affinity. Having this at this scope allows ++ * us to guarantee that we've checked the affinity in this function ++ * call. ++ */ ++ u64 recently_chosen_affinity = 0; ++ bool chosen_affinity = false; ++ bool retry; ++ ++ do { ++ retry = false; ++ ++ /* NOTE: The following uses a number of FALLTHROUGHs to optimize ++ * the calls to this function. Ending of the function is ++ * indicated by BREAK OUT */ ++ switch (katom->coreref_state) { ++ /* State when job is first attempted to be run */ ++ case KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED: ++ KBASE_DEBUG_ASSERT(katom->affinity == 0); ++ ++ /* Compute affinity */ ++ if (false == kbase_js_choose_affinity( ++ &recently_chosen_affinity, kbdev, katom, ++ js)) { ++ /* No cores are currently available */ ++ /* *** BREAK OUT: No state transition *** */ ++ break; ++ } ++ ++ chosen_affinity = true; ++ ++ /* Request the cores */ ++ kbase_pm_request_cores(kbdev, ++ katom->core_req & BASE_JD_REQ_T, ++ recently_chosen_affinity); ++ ++ katom->affinity = recently_chosen_affinity; ++ ++ /* Proceed to next state */ ++ katom->coreref_state = ++ KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES; ++ ++ /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ ++ /* fallthrough */ ++ case KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES: ++ { ++ enum kbase_pm_cores_ready cores_ready; ++ ++ KBASE_DEBUG_ASSERT(katom->affinity != 0 || ++ (katom->core_req & BASE_JD_REQ_T)); ++ ++ cores_ready = kbase_pm_register_inuse_cores( ++ kbdev, ++ katom->core_req & BASE_JD_REQ_T, ++ katom->affinity); ++ if (cores_ready == KBASE_NEW_AFFINITY) { ++ /* Affinity no longer valid - return to ++ * previous state */ ++ kbasep_js_job_check_deref_cores(kbdev, ++ katom); ++ KBASE_TRACE_ADD_SLOT_INFO(kbdev, ++ JS_CORE_REF_REGISTER_INUSE_FAILED, ++ katom->kctx, katom, ++ katom->jc, js, ++ (u32) katom->affinity); ++ /* *** BREAK OUT: Return to previous ++ * state, retry *** */ ++ retry = true; ++ break; ++ } ++ if (cores_ready == KBASE_CORES_NOT_READY) { ++ /* Stay in this state and return, to ++ * retry at this state later */ ++ KBASE_TRACE_ADD_SLOT_INFO(kbdev, ++ JS_CORE_REF_REGISTER_INUSE_FAILED, ++ katom->kctx, katom, ++ katom->jc, js, ++ (u32) katom->affinity); ++ /* *** BREAK OUT: No state transition ++ * *** */ ++ break; ++ } ++ /* Proceed to next state */ ++ katom->coreref_state = ++ KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY; ++ } ++ ++ /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ ++ /* fallthrough */ ++ case KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY: ++ KBASE_DEBUG_ASSERT(katom->affinity != 0 || ++ (katom->core_req & BASE_JD_REQ_T)); ++ ++ /* Optimize out choosing the affinity twice in the same ++ * function call */ ++ if (chosen_affinity == false) { ++ /* See if the affinity changed since a previous ++ * call. */ ++ if (false == kbase_js_choose_affinity( ++ &recently_chosen_affinity, ++ kbdev, katom, js)) { ++ /* No cores are currently available */ ++ kbasep_js_job_check_deref_cores(kbdev, ++ katom); ++ KBASE_TRACE_ADD_SLOT_INFO(kbdev, ++ JS_CORE_REF_REQUEST_ON_RECHECK_FAILED, ++ katom->kctx, katom, ++ katom->jc, js, ++ (u32) recently_chosen_affinity); ++ /* *** BREAK OUT: Transition to lower ++ * state *** */ ++ break; ++ } ++ chosen_affinity = true; ++ } ++ ++ /* Now see if this requires a different set of cores */ ++ if (recently_chosen_affinity != katom->affinity) { ++ enum kbase_pm_cores_ready cores_ready; ++ ++ kbase_pm_request_cores(kbdev, ++ katom->core_req & BASE_JD_REQ_T, ++ recently_chosen_affinity); ++ ++ /* Register new cores whilst we still hold the ++ * old ones, to minimize power transitions */ ++ cores_ready = ++ kbase_pm_register_inuse_cores(kbdev, ++ katom->core_req & BASE_JD_REQ_T, ++ recently_chosen_affinity); ++ kbasep_js_job_check_deref_cores(kbdev, katom); ++ ++ /* Fixup the state that was reduced by ++ * deref_cores: */ ++ katom->coreref_state = ++ KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY; ++ katom->affinity = recently_chosen_affinity; ++ if (cores_ready == KBASE_NEW_AFFINITY) { ++ /* Affinity no longer valid - return to ++ * previous state */ ++ katom->coreref_state = ++ KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES; ++ ++ kbasep_js_job_check_deref_cores(kbdev, ++ katom); ++ ++ KBASE_TRACE_ADD_SLOT_INFO(kbdev, ++ JS_CORE_REF_REGISTER_INUSE_FAILED, ++ katom->kctx, katom, ++ katom->jc, js, ++ (u32) katom->affinity); ++ /* *** BREAK OUT: Return to previous ++ * state, retry *** */ ++ retry = true; ++ break; ++ } ++ /* Now might be waiting for powerup again, with ++ * a new affinity */ ++ if (cores_ready == KBASE_CORES_NOT_READY) { ++ /* Return to previous state */ ++ katom->coreref_state = ++ KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES; ++ KBASE_TRACE_ADD_SLOT_INFO(kbdev, ++ JS_CORE_REF_REGISTER_ON_RECHECK_FAILED, ++ katom->kctx, katom, ++ katom->jc, js, ++ (u32) katom->affinity); ++ /* *** BREAK OUT: Transition to lower ++ * state *** */ ++ break; ++ } ++ } ++ /* Proceed to next state */ ++ katom->coreref_state = ++ KBASE_ATOM_COREREF_STATE_CHECK_AFFINITY_VIOLATIONS; ++ ++ /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ ++ /* fallthrough */ ++ case KBASE_ATOM_COREREF_STATE_CHECK_AFFINITY_VIOLATIONS: ++ KBASE_DEBUG_ASSERT(katom->affinity != 0 || ++ (katom->core_req & BASE_JD_REQ_T)); ++ KBASE_DEBUG_ASSERT(katom->affinity == ++ recently_chosen_affinity); ++ ++ /* Note: this is where the caller must've taken the ++ * hwaccess_lock */ ++ ++ /* Check for affinity violations - if there are any, ++ * then we just ask the caller to requeue and try again ++ * later */ ++ if (kbase_js_affinity_would_violate(kbdev, js, ++ katom->affinity) != false) { ++ /* Return to previous state */ ++ katom->coreref_state = ++ KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY; ++ /* *** BREAK OUT: Transition to lower state *** ++ */ ++ KBASE_TRACE_ADD_SLOT_INFO(kbdev, ++ JS_CORE_REF_AFFINITY_WOULD_VIOLATE, ++ katom->kctx, katom, katom->jc, js, ++ (u32) katom->affinity); ++ break; ++ } ++ ++ /* No affinity violations would result, so the cores are ++ * ready */ ++ katom->coreref_state = KBASE_ATOM_COREREF_STATE_READY; ++ /* *** BREAK OUT: Cores Ready *** */ ++ break; ++ ++ default: ++ KBASE_DEBUG_ASSERT_MSG(false, ++ "Unhandled kbase_atom_coreref_state %d", ++ katom->coreref_state); ++ break; ++ } ++ } while (retry != false); ++ ++ return (katom->coreref_state == KBASE_ATOM_COREREF_STATE_READY); ++} ++ ++static void kbasep_js_job_check_deref_cores(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(katom != NULL); ++ ++ switch (katom->coreref_state) { ++ case KBASE_ATOM_COREREF_STATE_READY: ++ /* State where atom was submitted to the HW - just proceed to ++ * power-down */ ++ KBASE_DEBUG_ASSERT(katom->affinity != 0 || ++ (katom->core_req & BASE_JD_REQ_T)); ++ ++ /* fallthrough */ ++ ++ case KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY: ++ /* State where cores were registered */ ++ KBASE_DEBUG_ASSERT(katom->affinity != 0 || ++ (katom->core_req & BASE_JD_REQ_T)); ++ kbase_pm_release_cores(kbdev, katom->core_req & BASE_JD_REQ_T, ++ katom->affinity); ++ ++ break; ++ ++ case KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES: ++ /* State where cores were requested, but not registered */ ++ KBASE_DEBUG_ASSERT(katom->affinity != 0 || ++ (katom->core_req & BASE_JD_REQ_T)); ++ kbase_pm_unrequest_cores(kbdev, katom->core_req & BASE_JD_REQ_T, ++ katom->affinity); ++ break; ++ ++ case KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED: ++ /* Initial state - nothing required */ ++ KBASE_DEBUG_ASSERT(katom->affinity == 0); ++ break; ++ ++ default: ++ KBASE_DEBUG_ASSERT_MSG(false, ++ "Unhandled coreref_state: %d", ++ katom->coreref_state); ++ break; ++ } ++ ++ katom->affinity = 0; ++ katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED; ++} ++ ++static void kbasep_js_job_check_deref_cores_nokatom(struct kbase_device *kbdev, ++ base_jd_core_req core_req, u64 affinity, ++ enum kbase_atom_coreref_state coreref_state) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ switch (coreref_state) { ++ case KBASE_ATOM_COREREF_STATE_READY: ++ /* State where atom was submitted to the HW - just proceed to ++ * power-down */ ++ KBASE_DEBUG_ASSERT(affinity != 0 || ++ (core_req & BASE_JD_REQ_T)); ++ ++ /* fallthrough */ ++ ++ case KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY: ++ /* State where cores were registered */ ++ KBASE_DEBUG_ASSERT(affinity != 0 || ++ (core_req & BASE_JD_REQ_T)); ++ kbase_pm_release_cores(kbdev, core_req & BASE_JD_REQ_T, ++ affinity); ++ ++ break; ++ ++ case KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES: ++ /* State where cores were requested, but not registered */ ++ KBASE_DEBUG_ASSERT(affinity != 0 || ++ (core_req & BASE_JD_REQ_T)); ++ kbase_pm_unrequest_cores(kbdev, core_req & BASE_JD_REQ_T, ++ affinity); ++ break; ++ ++ case KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED: ++ /* Initial state - nothing required */ ++ KBASE_DEBUG_ASSERT(affinity == 0); ++ break; ++ ++ default: ++ KBASE_DEBUG_ASSERT_MSG(false, ++ "Unhandled coreref_state: %d", ++ coreref_state); ++ break; ++ } ++} ++ ++static void kbase_gpu_release_atom(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom, ++ ktime_t *end_timestamp) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ ++ switch (katom->gpu_rb_state) { ++ case KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB: ++ /* Should be impossible */ ++ WARN(1, "Attempting to release atom not in ringbuffer\n"); ++ break; ++ ++ case KBASE_ATOM_GPU_RB_SUBMITTED: ++ /* Inform power management at start/finish of atom so it can ++ * update its GPU utilisation metrics. Mark atom as not ++ * submitted beforehand. */ ++ katom->gpu_rb_state = KBASE_ATOM_GPU_RB_READY; ++ kbase_pm_metrics_update(kbdev, end_timestamp); ++ ++ if (katom->core_req & BASE_JD_REQ_PERMON) ++ kbase_pm_release_gpu_cycle_counter_nolock(kbdev); ++ /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ ++ ++ KBASE_TLSTREAM_TL_NRET_ATOM_LPU(katom, ++ &kbdev->gpu_props.props.raw_props.js_features ++ [katom->slot_nr]); ++ KBASE_TLSTREAM_TL_NRET_ATOM_AS(katom, &kbdev->as[kctx->as_nr]); ++ KBASE_TLSTREAM_TL_NRET_CTX_LPU(kctx, ++ &kbdev->gpu_props.props.raw_props.js_features ++ [katom->slot_nr]); ++ /* fallthrough */ ++ case KBASE_ATOM_GPU_RB_READY: ++ /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ ++ /* fallthrough */ ++ case KBASE_ATOM_GPU_RB_WAITING_AFFINITY: ++ kbase_js_affinity_release_slot_cores(kbdev, katom->slot_nr, ++ katom->affinity); ++ /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ ++ /* fallthrough */ ++ case KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE: ++ break; ++ ++ case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION: ++ if (katom->protected_state.enter != ++ KBASE_ATOM_ENTER_PROTECTED_CHECK || ++ katom->protected_state.exit != ++ KBASE_ATOM_EXIT_PROTECTED_CHECK) ++ kbdev->protected_mode_transition = false; ++ ++ if (kbase_jd_katom_is_protected(katom) && ++ (katom->protected_state.enter == ++ KBASE_ATOM_ENTER_PROTECTED_IDLE_L2)) { ++ kbase_vinstr_resume(kbdev->vinstr_ctx); ++ ++ /* Go back to configured model for IPA */ ++ kbase_ipa_model_use_configured_locked(kbdev); ++ } ++ ++ ++ /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ ++ /* fallthrough */ ++ case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV: ++ /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ ++ /* fallthrough */ ++ case KBASE_ATOM_GPU_RB_WAITING_BLOCKED: ++ /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ ++ /* fallthrough */ ++ case KBASE_ATOM_GPU_RB_RETURN_TO_JS: ++ break; ++ } ++ ++ katom->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_BLOCKED; ++ katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK; ++} ++ ++static void kbase_gpu_mark_atom_for_return(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom) ++{ ++ kbase_gpu_release_atom(kbdev, katom, NULL); ++ katom->gpu_rb_state = KBASE_ATOM_GPU_RB_RETURN_TO_JS; ++} ++ ++static inline bool kbase_gpu_rmu_workaround(struct kbase_device *kbdev, int js) ++{ ++ struct kbase_backend_data *backend = &kbdev->hwaccess.backend; ++ bool slot_busy[3]; ++ ++ if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) ++ return true; ++ slot_busy[0] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 0, ++ KBASE_ATOM_GPU_RB_WAITING_AFFINITY); ++ slot_busy[1] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 1, ++ KBASE_ATOM_GPU_RB_WAITING_AFFINITY); ++ slot_busy[2] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 2, ++ KBASE_ATOM_GPU_RB_WAITING_AFFINITY); ++ ++ if ((js == 2 && !(slot_busy[0] || slot_busy[1])) || ++ (js != 2 && !slot_busy[2])) ++ return true; ++ ++ /* Don't submit slot 2 atom while GPU has jobs on slots 0/1 */ ++ if (js == 2 && (kbase_gpu_atoms_submitted(kbdev, 0) || ++ kbase_gpu_atoms_submitted(kbdev, 1) || ++ backend->rmu_workaround_flag)) ++ return false; ++ ++ /* Don't submit slot 0/1 atom while GPU has jobs on slot 2 */ ++ if (js != 2 && (kbase_gpu_atoms_submitted(kbdev, 2) || ++ !backend->rmu_workaround_flag)) ++ return false; ++ ++ backend->rmu_workaround_flag = !backend->rmu_workaround_flag; ++ ++ return true; ++} ++ ++/** ++ * other_slots_busy - Determine if any job slots other than @js are currently ++ * running atoms ++ * @kbdev: Device pointer ++ * @js: Job slot ++ * ++ * Return: true if any slots other than @js are busy, false otherwise ++ */ ++static inline bool other_slots_busy(struct kbase_device *kbdev, int js) ++{ ++ int slot; ++ ++ for (slot = 0; slot < kbdev->gpu_props.num_job_slots; slot++) { ++ if (slot == js) ++ continue; ++ ++ if (kbase_gpu_nr_atoms_on_slot_min(kbdev, slot, ++ KBASE_ATOM_GPU_RB_SUBMITTED)) ++ return true; ++ } ++ ++ return false; ++} ++ ++static inline bool kbase_gpu_in_protected_mode(struct kbase_device *kbdev) ++{ ++ return kbdev->protected_mode; ++} ++ ++static int kbase_gpu_protected_mode_enter(struct kbase_device *kbdev) ++{ ++ int err = -EINVAL; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ WARN_ONCE(!kbdev->protected_ops, ++ "Cannot enter protected mode: protected callbacks not specified.\n"); ++ ++ /* ++ * When entering into protected mode, we must ensure that the ++ * GPU is not operating in coherent mode as well. This is to ++ * ensure that no protected memory can be leaked. ++ */ ++ if (kbdev->system_coherency == COHERENCY_ACE) ++ kbase_cache_set_coherency_mode(kbdev, COHERENCY_ACE_LITE); ++ ++ if (kbdev->protected_ops) { ++ /* Switch GPU to protected mode */ ++ err = kbdev->protected_ops->protected_mode_enable( ++ kbdev->protected_dev); ++ ++ if (err) ++ dev_warn(kbdev->dev, "Failed to enable protected mode: %d\n", ++ err); ++ else ++ kbdev->protected_mode = true; ++ } ++ ++ return err; ++} ++ ++static int kbase_gpu_protected_mode_reset(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ WARN_ONCE(!kbdev->protected_ops, ++ "Cannot exit protected mode: protected callbacks not specified.\n"); ++ ++ if (!kbdev->protected_ops) ++ return -EINVAL; ++ ++ /* The protected mode disable callback will be called as part of reset ++ */ ++ kbase_reset_gpu_silent(kbdev); ++ ++ return 0; ++} ++ ++static int kbase_jm_enter_protected_mode(struct kbase_device *kbdev, ++ struct kbase_jd_atom **katom, int idx, int js) ++{ ++ int err = 0; ++ ++ switch (katom[idx]->protected_state.enter) { ++ case KBASE_ATOM_ENTER_PROTECTED_CHECK: ++ KBASE_TLSTREAM_AUX_PROTECTED_ENTER_START(kbdev); ++ /* The checks in KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV ++ * should ensure that we are not already transitiong, and that ++ * there are no atoms currently on the GPU. */ ++ WARN_ON(kbdev->protected_mode_transition); ++ WARN_ON(kbase_gpu_atoms_submitted_any(kbdev)); ++ ++ kbdev->protected_mode_transition = true; ++ katom[idx]->protected_state.enter = ++ KBASE_ATOM_ENTER_PROTECTED_VINSTR; ++ ++ /* ***TRANSITION TO HIGHER STATE*** */ ++ /* fallthrough */ ++ case KBASE_ATOM_ENTER_PROTECTED_VINSTR: ++ if (kbase_vinstr_try_suspend(kbdev->vinstr_ctx) < 0) { ++ /* ++ * We can't switch now because ++ * the vinstr core state switch ++ * is not done yet. ++ */ ++ return -EAGAIN; ++ } ++ ++ /* Use generic model for IPA in protected mode */ ++ kbase_ipa_model_use_fallback_locked(kbdev); ++ ++ /* Once reaching this point GPU must be ++ * switched to protected mode or vinstr ++ * re-enabled. */ ++ ++ /* ++ * Not in correct mode, begin protected mode switch. ++ * Entering protected mode requires us to power down the L2, ++ * and drop out of fully coherent mode. ++ */ ++ katom[idx]->protected_state.enter = ++ KBASE_ATOM_ENTER_PROTECTED_IDLE_L2; ++ ++ kbase_pm_update_cores_state_nolock(kbdev); ++ ++ /* ***TRANSITION TO HIGHER STATE*** */ ++ /* fallthrough */ ++ case KBASE_ATOM_ENTER_PROTECTED_IDLE_L2: ++ /* Avoid unnecessary waiting on non-ACE platforms. */ ++ if (kbdev->current_gpu_coherency_mode == COHERENCY_ACE) { ++ if (kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_L2) || ++ kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_L2)) { ++ /* ++ * The L2 is still powered, wait for all the users to ++ * finish with it before doing the actual reset. ++ */ ++ return -EAGAIN; ++ } ++ } ++ ++ katom[idx]->protected_state.enter = ++ KBASE_ATOM_ENTER_PROTECTED_FINISHED; ++ ++ /* ***TRANSITION TO HIGHER STATE*** */ ++ /* fallthrough */ ++ case KBASE_ATOM_ENTER_PROTECTED_FINISHED: ++ ++ /* No jobs running, so we can switch GPU mode right now. */ ++ err = kbase_gpu_protected_mode_enter(kbdev); ++ ++ /* ++ * Regardless of result, we are no longer transitioning ++ * the GPU. ++ */ ++ kbdev->protected_mode_transition = false; ++ KBASE_TLSTREAM_AUX_PROTECTED_ENTER_END(kbdev); ++ if (err) { ++ /* ++ * Failed to switch into protected mode, resume ++ * vinstr core and fail atom. ++ */ ++ kbase_vinstr_resume(kbdev->vinstr_ctx); ++ katom[idx]->event_code = BASE_JD_EVENT_JOB_INVALID; ++ kbase_gpu_mark_atom_for_return(kbdev, katom[idx]); ++ /* Only return if head atom or previous atom ++ * already removed - as atoms must be returned ++ * in order. */ ++ if (idx == 0 || katom[0]->gpu_rb_state == ++ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { ++ kbase_gpu_dequeue_atom(kbdev, js, NULL); ++ kbase_jm_return_atom_to_js(kbdev, katom[idx]); ++ } ++ ++ /* Go back to configured model for IPA */ ++ kbase_ipa_model_use_configured_locked(kbdev); ++ ++ return -EINVAL; ++ } ++ ++ /* Protected mode sanity checks. */ ++ KBASE_DEBUG_ASSERT_MSG( ++ kbase_jd_katom_is_protected(katom[idx]) == ++ kbase_gpu_in_protected_mode(kbdev), ++ "Protected mode of atom (%d) doesn't match protected mode of GPU (%d)", ++ kbase_jd_katom_is_protected(katom[idx]), ++ kbase_gpu_in_protected_mode(kbdev)); ++ katom[idx]->gpu_rb_state = ++ KBASE_ATOM_GPU_RB_READY; ++ } ++ ++ return 0; ++} ++ ++static int kbase_jm_exit_protected_mode(struct kbase_device *kbdev, ++ struct kbase_jd_atom **katom, int idx, int js) ++{ ++ int err = 0; ++ ++ ++ switch (katom[idx]->protected_state.exit) { ++ case KBASE_ATOM_EXIT_PROTECTED_CHECK: ++ KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_START(kbdev); ++ /* The checks in KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV ++ * should ensure that we are not already transitiong, and that ++ * there are no atoms currently on the GPU. */ ++ WARN_ON(kbdev->protected_mode_transition); ++ WARN_ON(kbase_gpu_atoms_submitted_any(kbdev)); ++ ++ /* ++ * Exiting protected mode requires a reset, but first the L2 ++ * needs to be powered down to ensure it's not active when the ++ * reset is issued. ++ */ ++ katom[idx]->protected_state.exit = ++ KBASE_ATOM_EXIT_PROTECTED_IDLE_L2; ++ ++ kbdev->protected_mode_transition = true; ++ kbase_pm_update_cores_state_nolock(kbdev); ++ ++ /* ***TRANSITION TO HIGHER STATE*** */ ++ /* fallthrough */ ++ case KBASE_ATOM_EXIT_PROTECTED_IDLE_L2: ++ if (kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_L2) || ++ kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_L2)) { ++ /* ++ * The L2 is still powered, wait for all the users to ++ * finish with it before doing the actual reset. ++ */ ++ return -EAGAIN; ++ } ++ katom[idx]->protected_state.exit = ++ KBASE_ATOM_EXIT_PROTECTED_RESET; ++ ++ /* ***TRANSITION TO HIGHER STATE*** */ ++ /* fallthrough */ ++ case KBASE_ATOM_EXIT_PROTECTED_RESET: ++ /* Issue the reset to the GPU */ ++ err = kbase_gpu_protected_mode_reset(kbdev); ++ ++ if (err) { ++ kbdev->protected_mode_transition = false; ++ ++ /* Failed to exit protected mode, fail atom */ ++ katom[idx]->event_code = BASE_JD_EVENT_JOB_INVALID; ++ kbase_gpu_mark_atom_for_return(kbdev, katom[idx]); ++ /* Only return if head atom or previous atom ++ * already removed - as atoms must be returned ++ * in order */ ++ if (idx == 0 || katom[0]->gpu_rb_state == ++ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { ++ kbase_gpu_dequeue_atom(kbdev, js, NULL); ++ kbase_jm_return_atom_to_js(kbdev, katom[idx]); ++ } ++ ++ kbase_vinstr_resume(kbdev->vinstr_ctx); ++ ++ /* Use generic model for IPA in protected mode */ ++ kbase_ipa_model_use_fallback_locked(kbdev); ++ ++ return -EINVAL; ++ } ++ ++ katom[idx]->protected_state.exit = ++ KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT; ++ ++ /* ***TRANSITION TO HIGHER STATE*** */ ++ /* fallthrough */ ++ case KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT: ++ /* A GPU reset is issued when exiting protected mode. Once the ++ * reset is done all atoms' state will also be reset. For this ++ * reason, if the atom is still in this state we can safely ++ * say that the reset has not completed i.e., we have not ++ * finished exiting protected mode yet. ++ */ ++ return -EAGAIN; ++ } ++ ++ return 0; ++} ++ ++void kbase_backend_slot_update(struct kbase_device *kbdev) ++{ ++ int js; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { ++ struct kbase_jd_atom *katom[2]; ++ int idx; ++ ++ katom[0] = kbase_gpu_inspect(kbdev, js, 0); ++ katom[1] = kbase_gpu_inspect(kbdev, js, 1); ++ WARN_ON(katom[1] && !katom[0]); ++ ++ for (idx = 0; idx < SLOT_RB_SIZE; idx++) { ++ bool cores_ready; ++ int ret; ++ ++ if (!katom[idx]) ++ continue; ++ ++ switch (katom[idx]->gpu_rb_state) { ++ case KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB: ++ /* Should be impossible */ ++ WARN(1, "Attempting to update atom not in ringbuffer\n"); ++ break; ++ ++ case KBASE_ATOM_GPU_RB_WAITING_BLOCKED: ++ if (katom[idx]->atom_flags & ++ KBASE_KATOM_FLAG_X_DEP_BLOCKED) ++ break; ++ ++ katom[idx]->gpu_rb_state = ++ KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV; ++ ++ /* ***TRANSITION TO HIGHER STATE*** */ ++ /* fallthrough */ ++ case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV: ++ if (kbase_gpu_check_secure_atoms(kbdev, ++ !kbase_jd_katom_is_protected( ++ katom[idx]))) ++ break; ++ ++ if ((idx == 1) && (kbase_jd_katom_is_protected( ++ katom[0]) != ++ kbase_jd_katom_is_protected( ++ katom[1]))) ++ break; ++ ++ if (kbdev->protected_mode_transition) ++ break; ++ ++ katom[idx]->gpu_rb_state = ++ KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION; ++ ++ /* ***TRANSITION TO HIGHER STATE*** */ ++ /* fallthrough */ ++ case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION: ++ ++ /* ++ * Exiting protected mode must be done before ++ * the references on the cores are taken as ++ * a power down the L2 is required which ++ * can't happen after the references for this ++ * atom are taken. ++ */ ++ ++ if (!kbase_gpu_in_protected_mode(kbdev) && ++ kbase_jd_katom_is_protected(katom[idx])) { ++ /* Atom needs to transition into protected mode. */ ++ ret = kbase_jm_enter_protected_mode(kbdev, ++ katom, idx, js); ++ if (ret) ++ break; ++ } else if (kbase_gpu_in_protected_mode(kbdev) && ++ !kbase_jd_katom_is_protected(katom[idx])) { ++ /* Atom needs to transition out of protected mode. */ ++ ret = kbase_jm_exit_protected_mode(kbdev, ++ katom, idx, js); ++ if (ret) ++ break; ++ } ++ katom[idx]->protected_state.exit = ++ KBASE_ATOM_EXIT_PROTECTED_CHECK; ++ ++ /* Atom needs no protected mode transition. */ ++ ++ katom[idx]->gpu_rb_state = ++ KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE; ++ ++ /* ***TRANSITION TO HIGHER STATE*** */ ++ /* fallthrough */ ++ case KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE: ++ if (katom[idx]->will_fail_event_code) { ++ kbase_gpu_mark_atom_for_return(kbdev, ++ katom[idx]); ++ /* Set EVENT_DONE so this atom will be ++ completed, not unpulled. */ ++ katom[idx]->event_code = ++ BASE_JD_EVENT_DONE; ++ /* Only return if head atom or previous ++ * atom already removed - as atoms must ++ * be returned in order. */ ++ if (idx == 0 || katom[0]->gpu_rb_state == ++ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { ++ kbase_gpu_dequeue_atom(kbdev, js, NULL); ++ kbase_jm_return_atom_to_js(kbdev, katom[idx]); ++ } ++ break; ++ } ++ ++ cores_ready = ++ kbasep_js_job_check_ref_cores(kbdev, js, ++ katom[idx]); ++ ++ if (katom[idx]->event_code == ++ BASE_JD_EVENT_PM_EVENT) { ++ katom[idx]->gpu_rb_state = ++ KBASE_ATOM_GPU_RB_RETURN_TO_JS; ++ break; ++ } ++ ++ if (!cores_ready) ++ break; ++ ++ kbase_js_affinity_retain_slot_cores(kbdev, js, ++ katom[idx]->affinity); ++ katom[idx]->gpu_rb_state = ++ KBASE_ATOM_GPU_RB_WAITING_AFFINITY; ++ ++ /* ***TRANSITION TO HIGHER STATE*** */ ++ /* fallthrough */ ++ case KBASE_ATOM_GPU_RB_WAITING_AFFINITY: ++ if (!kbase_gpu_rmu_workaround(kbdev, js)) ++ break; ++ ++ katom[idx]->gpu_rb_state = ++ KBASE_ATOM_GPU_RB_READY; ++ ++ /* ***TRANSITION TO HIGHER STATE*** */ ++ /* fallthrough */ ++ case KBASE_ATOM_GPU_RB_READY: ++ ++ if (idx == 1) { ++ /* Only submit if head atom or previous ++ * atom already submitted */ ++ if ((katom[0]->gpu_rb_state != ++ KBASE_ATOM_GPU_RB_SUBMITTED && ++ katom[0]->gpu_rb_state != ++ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB)) ++ break; ++ ++ /* If intra-slot serialization in use ++ * then don't submit atom to NEXT slot ++ */ ++ if (kbdev->serialize_jobs & ++ KBASE_SERIALIZE_INTRA_SLOT) ++ break; ++ } ++ ++ /* If inter-slot serialization in use then don't ++ * submit atom if any other slots are in use */ ++ if ((kbdev->serialize_jobs & ++ KBASE_SERIALIZE_INTER_SLOT) && ++ other_slots_busy(kbdev, js)) ++ break; ++ ++ if ((kbdev->serialize_jobs & ++ KBASE_SERIALIZE_RESET) && ++ kbase_reset_gpu_active(kbdev)) ++ break; ++ ++ /* Check if this job needs the cycle counter ++ * enabled before submission */ ++ if (katom[idx]->core_req & BASE_JD_REQ_PERMON) ++ kbase_pm_request_gpu_cycle_counter_l2_is_on( ++ kbdev); ++ ++ kbase_job_hw_submit(kbdev, katom[idx], js); ++ katom[idx]->gpu_rb_state = ++ KBASE_ATOM_GPU_RB_SUBMITTED; ++ ++ /* Inform power management at start/finish of ++ * atom so it can update its GPU utilisation ++ * metrics. */ ++ kbase_pm_metrics_update(kbdev, ++ &katom[idx]->start_timestamp); ++ ++ /* ***TRANSITION TO HIGHER STATE*** */ ++ /* fallthrough */ ++ case KBASE_ATOM_GPU_RB_SUBMITTED: ++ /* Atom submitted to HW, nothing else to do */ ++ break; ++ ++ case KBASE_ATOM_GPU_RB_RETURN_TO_JS: ++ /* Only return if head atom or previous atom ++ * already removed - as atoms must be returned ++ * in order */ ++ if (idx == 0 || katom[0]->gpu_rb_state == ++ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { ++ kbase_gpu_dequeue_atom(kbdev, js, NULL); ++ kbase_jm_return_atom_to_js(kbdev, ++ katom[idx]); ++ } ++ break; ++ } ++ } ++ } ++ ++ /* Warn if PRLAM-8987 affinity restrictions are violated */ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) ++ WARN_ON((kbase_gpu_atoms_submitted(kbdev, 0) || ++ kbase_gpu_atoms_submitted(kbdev, 1)) && ++ kbase_gpu_atoms_submitted(kbdev, 2)); ++} ++ ++ ++void kbase_backend_run_atom(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ kbase_gpu_enqueue_atom(kbdev, katom); ++ kbase_backend_slot_update(kbdev); ++} ++ ++#define HAS_DEP(katom) (katom->pre_dep || katom->atom_flags & \ ++ (KBASE_KATOM_FLAG_X_DEP_BLOCKED | KBASE_KATOM_FLAG_FAIL_BLOCKER)) ++ ++bool kbase_gpu_irq_evict(struct kbase_device *kbdev, int js) ++{ ++ struct kbase_jd_atom *katom; ++ struct kbase_jd_atom *next_katom; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ katom = kbase_gpu_inspect(kbdev, js, 0); ++ next_katom = kbase_gpu_inspect(kbdev, js, 1); ++ ++ if (next_katom && katom->kctx == next_katom->kctx && ++ next_katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED && ++ HAS_DEP(next_katom) && ++ (kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_LO), NULL) ++ != 0 || ++ kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_HI), NULL) ++ != 0)) { ++ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT), ++ JS_COMMAND_NOP, NULL); ++ next_katom->gpu_rb_state = KBASE_ATOM_GPU_RB_READY; ++ ++ KBASE_TLSTREAM_TL_NRET_ATOM_LPU(katom, ++ &kbdev->gpu_props.props.raw_props.js_features ++ [katom->slot_nr]); ++ KBASE_TLSTREAM_TL_NRET_ATOM_AS(katom, &kbdev->as ++ [katom->kctx->as_nr]); ++ KBASE_TLSTREAM_TL_NRET_CTX_LPU(katom->kctx, ++ &kbdev->gpu_props.props.raw_props.js_features ++ [katom->slot_nr]); ++ ++ return true; ++ } ++ ++ return false; ++} ++ ++void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js, ++ u32 completion_code, ++ u64 job_tail, ++ ktime_t *end_timestamp) ++{ ++ struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, 0); ++ struct kbase_context *kctx = katom->kctx; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ /* ++ * When a hard-stop is followed close after a soft-stop, the completion ++ * code may be set to STOPPED, even though the job is terminated ++ */ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TMIX_8438)) { ++ if (completion_code == BASE_JD_EVENT_STOPPED && ++ (katom->atom_flags & ++ KBASE_KATOM_FLAG_BEEN_HARD_STOPPED)) { ++ completion_code = BASE_JD_EVENT_TERMINATED; ++ } ++ } ++ ++ if ((kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_6787) || (katom->core_req & ++ BASE_JD_REQ_SKIP_CACHE_END)) && ++ completion_code != BASE_JD_EVENT_DONE && ++ !(completion_code & BASE_JD_SW_EVENT)) { ++ /* When a job chain fails, on a T60x or when ++ * BASE_JD_REQ_SKIP_CACHE_END is set, the GPU cache is not ++ * flushed. To prevent future evictions causing possible memory ++ * corruption we need to flush the cache manually before any ++ * affected memory gets reused. */ ++ katom->need_cache_flush_cores_retained = katom->affinity; ++ kbase_pm_request_cores(kbdev, false, katom->affinity); ++ } else if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10676)) { ++ if (kbdev->gpu_props.num_core_groups > 1 && ++ !(katom->affinity & ++ kbdev->gpu_props.props.coherency_info.group[0].core_mask ++ ) && ++ (katom->affinity & ++ kbdev->gpu_props.props.coherency_info.group[1].core_mask ++ )) { ++ dev_info(kbdev->dev, "JD: Flushing cache due to PRLAM-10676\n"); ++ katom->need_cache_flush_cores_retained = ++ katom->affinity; ++ kbase_pm_request_cores(kbdev, false, ++ katom->affinity); ++ } ++ } ++ ++ katom = kbase_gpu_dequeue_atom(kbdev, js, end_timestamp); ++ kbase_timeline_job_slot_done(kbdev, katom->kctx, katom, js, 0); ++ ++ if (completion_code == BASE_JD_EVENT_STOPPED) { ++ struct kbase_jd_atom *next_katom = kbase_gpu_inspect(kbdev, js, ++ 0); ++ ++ /* ++ * Dequeue next atom from ringbuffers on same slot if required. ++ * This atom will already have been removed from the NEXT ++ * registers by kbase_gpu_soft_hard_stop_slot(), to ensure that ++ * the atoms on this slot are returned in the correct order. ++ */ ++ if (next_katom && katom->kctx == next_katom->kctx && ++ next_katom->sched_priority == ++ katom->sched_priority) { ++ kbase_gpu_dequeue_atom(kbdev, js, end_timestamp); ++ kbase_jm_return_atom_to_js(kbdev, next_katom); ++ } ++ } else if (completion_code != BASE_JD_EVENT_DONE) { ++ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; ++ int i; ++ ++#if KBASE_TRACE_DUMP_ON_JOB_SLOT_ERROR != 0 ++ KBASE_TRACE_DUMP(kbdev); ++#endif ++ kbasep_js_clear_submit_allowed(js_devdata, katom->kctx); ++ ++ /* ++ * Remove all atoms on the same context from ringbuffers. This ++ * will not remove atoms that are already on the GPU, as these ++ * are guaranteed not to have fail dependencies on the failed ++ * atom. ++ */ ++ for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) { ++ struct kbase_jd_atom *katom_idx0 = ++ kbase_gpu_inspect(kbdev, i, 0); ++ struct kbase_jd_atom *katom_idx1 = ++ kbase_gpu_inspect(kbdev, i, 1); ++ ++ if (katom_idx0 && katom_idx0->kctx == katom->kctx && ++ HAS_DEP(katom_idx0) && ++ katom_idx0->gpu_rb_state != ++ KBASE_ATOM_GPU_RB_SUBMITTED) { ++ /* Dequeue katom_idx0 from ringbuffer */ ++ kbase_gpu_dequeue_atom(kbdev, i, end_timestamp); ++ ++ if (katom_idx1 && ++ katom_idx1->kctx == katom->kctx ++ && HAS_DEP(katom_idx1) && ++ katom_idx0->gpu_rb_state != ++ KBASE_ATOM_GPU_RB_SUBMITTED) { ++ /* Dequeue katom_idx1 from ringbuffer */ ++ kbase_gpu_dequeue_atom(kbdev, i, ++ end_timestamp); ++ ++ katom_idx1->event_code = ++ BASE_JD_EVENT_STOPPED; ++ kbase_jm_return_atom_to_js(kbdev, ++ katom_idx1); ++ } ++ katom_idx0->event_code = BASE_JD_EVENT_STOPPED; ++ kbase_jm_return_atom_to_js(kbdev, katom_idx0); ++ ++ } else if (katom_idx1 && ++ katom_idx1->kctx == katom->kctx && ++ HAS_DEP(katom_idx1) && ++ katom_idx1->gpu_rb_state != ++ KBASE_ATOM_GPU_RB_SUBMITTED) { ++ /* Can not dequeue this atom yet - will be ++ * dequeued when atom at idx0 completes */ ++ katom_idx1->event_code = BASE_JD_EVENT_STOPPED; ++ kbase_gpu_mark_atom_for_return(kbdev, ++ katom_idx1); ++ } ++ } ++ } ++ ++ KBASE_TRACE_ADD_SLOT_INFO(kbdev, JM_JOB_DONE, kctx, katom, katom->jc, ++ js, completion_code); ++ ++ if (job_tail != 0 && job_tail != katom->jc) { ++ bool was_updated = (job_tail != katom->jc); ++ ++ /* Some of the job has been executed, so we update the job chain ++ * address to where we should resume from */ ++ katom->jc = job_tail; ++ if (was_updated) ++ KBASE_TRACE_ADD_SLOT(kbdev, JM_UPDATE_HEAD, katom->kctx, ++ katom, job_tail, js); ++ } ++ ++ /* Only update the event code for jobs that weren't cancelled */ ++ if (katom->event_code != BASE_JD_EVENT_JOB_CANCELLED) ++ katom->event_code = (base_jd_event_code)completion_code; ++ ++ kbase_device_trace_register_access(kctx, REG_WRITE, ++ JOB_CONTROL_REG(JOB_IRQ_CLEAR), ++ 1 << js); ++ ++ /* Complete the job, and start new ones ++ * ++ * Also defer remaining work onto the workqueue: ++ * - Re-queue Soft-stopped jobs ++ * - For any other jobs, queue the job back into the dependency system ++ * - Schedule out the parent context if necessary, and schedule a new ++ * one in. ++ */ ++#ifdef CONFIG_GPU_TRACEPOINTS ++ { ++ /* The atom in the HEAD */ ++ struct kbase_jd_atom *next_katom = kbase_gpu_inspect(kbdev, js, ++ 0); ++ ++ if (next_katom && next_katom->gpu_rb_state == ++ KBASE_ATOM_GPU_RB_SUBMITTED) { ++ char js_string[16]; ++ ++ trace_gpu_sched_switch(kbasep_make_job_slot_string(js, ++ js_string, ++ sizeof(js_string)), ++ ktime_to_ns(*end_timestamp), ++ (u32)next_katom->kctx->id, 0, ++ next_katom->work_id); ++ kbdev->hwaccess.backend.slot_rb[js].last_context = ++ next_katom->kctx; ++ } else { ++ char js_string[16]; ++ ++ trace_gpu_sched_switch(kbasep_make_job_slot_string(js, ++ js_string, ++ sizeof(js_string)), ++ ktime_to_ns(ktime_get()), 0, 0, ++ 0); ++ kbdev->hwaccess.backend.slot_rb[js].last_context = 0; ++ } ++ } ++#endif ++ ++ if (kbdev->serialize_jobs & KBASE_SERIALIZE_RESET) ++ kbase_reset_gpu_silent(kbdev); ++ ++ if (completion_code == BASE_JD_EVENT_STOPPED) ++ katom = kbase_jm_return_atom_to_js(kbdev, katom); ++ else ++ katom = kbase_jm_complete(kbdev, katom, end_timestamp); ++ ++ if (katom) { ++ /* Cross-slot dependency has now become runnable. Try to submit ++ * it. */ ++ ++ /* Check if there are lower priority jobs to soft stop */ ++ kbase_job_slot_ctx_priority_check_locked(kctx, katom); ++ ++ kbase_jm_try_kick(kbdev, 1 << katom->slot_nr); ++ } ++ ++ /* Job completion may have unblocked other atoms. Try to update all job ++ * slots */ ++ kbase_backend_slot_update(kbdev); ++} ++ ++void kbase_backend_reset(struct kbase_device *kbdev, ktime_t *end_timestamp) ++{ ++ int js; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ /* Reset should always take the GPU out of protected mode */ ++ WARN_ON(kbase_gpu_in_protected_mode(kbdev)); ++ ++ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { ++ int atom_idx = 0; ++ int idx; ++ ++ for (idx = 0; idx < SLOT_RB_SIZE; idx++) { ++ struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, ++ js, atom_idx); ++ bool keep_in_jm_rb = false; ++ ++ if (!katom) ++ break; ++ if (katom->protected_state.exit == ++ KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT) ++ { ++ KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_END(kbdev); ++ ++ kbase_vinstr_resume(kbdev->vinstr_ctx); ++ ++ /* protected mode sanity checks */ ++ KBASE_DEBUG_ASSERT_MSG( ++ kbase_jd_katom_is_protected(katom) == kbase_gpu_in_protected_mode(kbdev), ++ "Protected mode of atom (%d) doesn't match protected mode of GPU (%d)", ++ kbase_jd_katom_is_protected(katom), kbase_gpu_in_protected_mode(kbdev)); ++ KBASE_DEBUG_ASSERT_MSG( ++ (kbase_jd_katom_is_protected(katom) && js == 0) || ++ !kbase_jd_katom_is_protected(katom), ++ "Protected atom on JS%d not supported", js); ++ } ++ if (katom->gpu_rb_state < KBASE_ATOM_GPU_RB_SUBMITTED) ++ keep_in_jm_rb = true; ++ ++ kbase_gpu_release_atom(kbdev, katom, NULL); ++ ++ /* ++ * If the atom wasn't on HW when the reset was issued ++ * then leave it in the RB and next time we're kicked ++ * it will be processed again from the starting state. ++ */ ++ if (keep_in_jm_rb) { ++ kbasep_js_job_check_deref_cores(kbdev, katom); ++ katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED; ++ katom->affinity = 0; ++ katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK; ++ /* As the atom was not removed, increment the ++ * index so that we read the correct atom in the ++ * next iteration. */ ++ atom_idx++; ++ continue; ++ } ++ ++ /* ++ * The atom was on the HW when the reset was issued ++ * all we can do is fail the atom. ++ */ ++ kbase_gpu_dequeue_atom(kbdev, js, NULL); ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ kbase_jm_complete(kbdev, katom, end_timestamp); ++ } ++ } ++ ++ kbdev->protected_mode_transition = false; ++} ++ ++static inline void kbase_gpu_stop_atom(struct kbase_device *kbdev, ++ int js, ++ struct kbase_jd_atom *katom, ++ u32 action) ++{ ++ u32 hw_action = action & JS_COMMAND_MASK; ++ ++ kbase_job_check_enter_disjoint(kbdev, action, katom->core_req, katom); ++ kbasep_job_slot_soft_or_hard_stop_do_action(kbdev, js, hw_action, ++ katom->core_req, katom); ++ katom->kctx->blocked_js[js][katom->sched_priority] = true; ++} ++ ++static inline void kbase_gpu_remove_atom(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom, ++ u32 action, ++ bool disjoint) ++{ ++ katom->event_code = BASE_JD_EVENT_REMOVED_FROM_NEXT; ++ kbase_gpu_mark_atom_for_return(kbdev, katom); ++ katom->kctx->blocked_js[katom->slot_nr][katom->sched_priority] = true; ++ ++ if (disjoint) ++ kbase_job_check_enter_disjoint(kbdev, action, katom->core_req, ++ katom); ++} ++ ++static int should_stop_x_dep_slot(struct kbase_jd_atom *katom) ++{ ++ if (katom->x_post_dep) { ++ struct kbase_jd_atom *dep_atom = katom->x_post_dep; ++ ++ if (dep_atom->gpu_rb_state != ++ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB && ++ dep_atom->gpu_rb_state != ++ KBASE_ATOM_GPU_RB_RETURN_TO_JS) ++ return dep_atom->slot_nr; ++ } ++ return -1; ++} ++ ++static void kbase_job_evicted(struct kbase_jd_atom *katom) ++{ ++ kbase_timeline_job_slot_done(katom->kctx->kbdev, katom->kctx, katom, ++ katom->slot_nr, KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT); ++} ++ ++bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ int js, ++ struct kbase_jd_atom *katom, ++ u32 action) ++{ ++ struct kbase_jd_atom *katom_idx0; ++ struct kbase_jd_atom *katom_idx1; ++ ++ bool katom_idx0_valid, katom_idx1_valid; ++ ++ bool ret = false; ++ ++ int stop_x_dep_idx0 = -1, stop_x_dep_idx1 = -1; ++ int prio_idx0 = 0, prio_idx1 = 0; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ katom_idx0 = kbase_gpu_inspect(kbdev, js, 0); ++ katom_idx1 = kbase_gpu_inspect(kbdev, js, 1); ++ ++ if (katom_idx0) ++ prio_idx0 = katom_idx0->sched_priority; ++ if (katom_idx1) ++ prio_idx1 = katom_idx1->sched_priority; ++ ++ if (katom) { ++ katom_idx0_valid = (katom_idx0 == katom); ++ /* If idx0 is to be removed and idx1 is on the same context, ++ * then idx1 must also be removed otherwise the atoms might be ++ * returned out of order */ ++ if (katom_idx1) ++ katom_idx1_valid = (katom_idx1 == katom) || ++ (katom_idx0_valid && ++ (katom_idx0->kctx == ++ katom_idx1->kctx)); ++ else ++ katom_idx1_valid = false; ++ } else { ++ katom_idx0_valid = (katom_idx0 && ++ (!kctx || katom_idx0->kctx == kctx)); ++ katom_idx1_valid = (katom_idx1 && ++ (!kctx || katom_idx1->kctx == kctx) && ++ prio_idx0 == prio_idx1); ++ } ++ ++ if (katom_idx0_valid) ++ stop_x_dep_idx0 = should_stop_x_dep_slot(katom_idx0); ++ if (katom_idx1_valid) ++ stop_x_dep_idx1 = should_stop_x_dep_slot(katom_idx1); ++ ++ if (katom_idx0_valid) { ++ if (katom_idx0->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) { ++ /* Simple case - just dequeue and return */ ++ kbase_gpu_dequeue_atom(kbdev, js, NULL); ++ if (katom_idx1_valid) { ++ kbase_gpu_dequeue_atom(kbdev, js, NULL); ++ katom_idx1->event_code = ++ BASE_JD_EVENT_REMOVED_FROM_NEXT; ++ kbase_jm_return_atom_to_js(kbdev, katom_idx1); ++ katom_idx1->kctx->blocked_js[js][prio_idx1] = ++ true; ++ } ++ ++ katom_idx0->event_code = ++ BASE_JD_EVENT_REMOVED_FROM_NEXT; ++ kbase_jm_return_atom_to_js(kbdev, katom_idx0); ++ katom_idx0->kctx->blocked_js[js][prio_idx0] = true; ++ } else { ++ /* katom_idx0 is on GPU */ ++ if (katom_idx1 && katom_idx1->gpu_rb_state == ++ KBASE_ATOM_GPU_RB_SUBMITTED) { ++ /* katom_idx0 and katom_idx1 are on GPU */ ++ ++ if (kbase_reg_read(kbdev, JOB_SLOT_REG(js, ++ JS_COMMAND_NEXT), NULL) == 0) { ++ /* idx0 has already completed - stop ++ * idx1 if needed*/ ++ if (katom_idx1_valid) { ++ kbase_gpu_stop_atom(kbdev, js, ++ katom_idx1, ++ action); ++ ret = true; ++ } ++ } else { ++ /* idx1 is in NEXT registers - attempt ++ * to remove */ ++ kbase_reg_write(kbdev, ++ JOB_SLOT_REG(js, ++ JS_COMMAND_NEXT), ++ JS_COMMAND_NOP, NULL); ++ ++ if (kbase_reg_read(kbdev, ++ JOB_SLOT_REG(js, ++ JS_HEAD_NEXT_LO), NULL) ++ != 0 || ++ kbase_reg_read(kbdev, ++ JOB_SLOT_REG(js, ++ JS_HEAD_NEXT_HI), NULL) ++ != 0) { ++ /* idx1 removed successfully, ++ * will be handled in IRQ */ ++ kbase_job_evicted(katom_idx1); ++ kbase_gpu_remove_atom(kbdev, ++ katom_idx1, ++ action, true); ++ stop_x_dep_idx1 = ++ should_stop_x_dep_slot(katom_idx1); ++ ++ /* stop idx0 if still on GPU */ ++ kbase_gpu_stop_atom(kbdev, js, ++ katom_idx0, ++ action); ++ ret = true; ++ } else if (katom_idx1_valid) { ++ /* idx0 has already completed, ++ * stop idx1 if needed */ ++ kbase_gpu_stop_atom(kbdev, js, ++ katom_idx1, ++ action); ++ ret = true; ++ } ++ } ++ } else if (katom_idx1_valid) { ++ /* idx1 not on GPU but must be dequeued*/ ++ ++ /* idx1 will be handled in IRQ */ ++ kbase_gpu_remove_atom(kbdev, katom_idx1, action, ++ false); ++ /* stop idx0 */ ++ /* This will be repeated for anything removed ++ * from the next registers, since their normal ++ * flow was also interrupted, and this function ++ * might not enter disjoint state e.g. if we ++ * don't actually do a hard stop on the head ++ * atom */ ++ kbase_gpu_stop_atom(kbdev, js, katom_idx0, ++ action); ++ ret = true; ++ } else { ++ /* no atom in idx1 */ ++ /* just stop idx0 */ ++ kbase_gpu_stop_atom(kbdev, js, katom_idx0, ++ action); ++ ret = true; ++ } ++ } ++ } else if (katom_idx1_valid) { ++ if (katom_idx1->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) { ++ /* Mark for return */ ++ /* idx1 will be returned once idx0 completes */ ++ kbase_gpu_remove_atom(kbdev, katom_idx1, action, ++ false); ++ } else { ++ /* idx1 is on GPU */ ++ if (kbase_reg_read(kbdev, JOB_SLOT_REG(js, ++ JS_COMMAND_NEXT), NULL) == 0) { ++ /* idx0 has already completed - stop idx1 */ ++ kbase_gpu_stop_atom(kbdev, js, katom_idx1, ++ action); ++ ret = true; ++ } else { ++ /* idx1 is in NEXT registers - attempt to ++ * remove */ ++ kbase_reg_write(kbdev, JOB_SLOT_REG(js, ++ JS_COMMAND_NEXT), ++ JS_COMMAND_NOP, NULL); ++ ++ if (kbase_reg_read(kbdev, JOB_SLOT_REG(js, ++ JS_HEAD_NEXT_LO), NULL) != 0 || ++ kbase_reg_read(kbdev, JOB_SLOT_REG(js, ++ JS_HEAD_NEXT_HI), NULL) != 0) { ++ /* idx1 removed successfully, will be ++ * handled in IRQ once idx0 completes */ ++ kbase_job_evicted(katom_idx1); ++ kbase_gpu_remove_atom(kbdev, katom_idx1, ++ action, ++ false); ++ } else { ++ /* idx0 has already completed - stop ++ * idx1 */ ++ kbase_gpu_stop_atom(kbdev, js, ++ katom_idx1, ++ action); ++ ret = true; ++ } ++ } ++ } ++ } ++ ++ ++ if (stop_x_dep_idx0 != -1) ++ kbase_backend_soft_hard_stop_slot(kbdev, kctx, stop_x_dep_idx0, ++ NULL, action); ++ ++ if (stop_x_dep_idx1 != -1) ++ kbase_backend_soft_hard_stop_slot(kbdev, kctx, stop_x_dep_idx1, ++ NULL, action); ++ ++ return ret; ++} ++ ++void kbase_gpu_cacheclean(struct kbase_device *kbdev) ++{ ++ /* Limit the number of loops to avoid a hang if the interrupt is missed ++ */ ++ u32 max_loops = KBASE_CLEAN_CACHE_MAX_LOOPS; ++ ++ mutex_lock(&kbdev->cacheclean_lock); ++ ++ /* use GPU_COMMAND completion solution */ ++ /* clean & invalidate the caches */ ++ KBASE_TRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, NULL, 0u, 0); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), ++ GPU_COMMAND_CLEAN_INV_CACHES, NULL); ++ ++ /* wait for cache flush to complete before continuing */ ++ while (--max_loops && ++ (kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), NULL) & ++ CLEAN_CACHES_COMPLETED) == 0) ++ ; ++ ++ /* clear the CLEAN_CACHES_COMPLETED irq */ ++ KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, NULL, 0u, ++ CLEAN_CACHES_COMPLETED); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), ++ CLEAN_CACHES_COMPLETED, NULL); ++ KBASE_DEBUG_ASSERT_MSG(kbdev->hwcnt.backend.state != ++ KBASE_INSTR_STATE_CLEANING, ++ "Instrumentation code was cleaning caches, but Job Management code cleared their IRQ - Instrumentation code will now hang."); ++ ++ mutex_unlock(&kbdev->cacheclean_lock); ++} ++ ++void kbase_backend_cacheclean(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom) ++{ ++ if (katom->need_cache_flush_cores_retained) { ++ unsigned long flags; ++ ++ kbase_gpu_cacheclean(kbdev); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_pm_unrequest_cores(kbdev, false, ++ katom->need_cache_flush_cores_retained); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ katom->need_cache_flush_cores_retained = 0; ++ } ++} ++ ++void kbase_backend_complete_wq(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom) ++{ ++ /* ++ * If cache flush required due to HW workaround then perform the flush ++ * now ++ */ ++ kbase_backend_cacheclean(kbdev, katom); ++ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10969) && ++ (katom->core_req & BASE_JD_REQ_FS) && ++ katom->event_code == BASE_JD_EVENT_TILE_RANGE_FAULT && ++ (katom->atom_flags & KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED) && ++ !(katom->atom_flags & KBASE_KATOM_FLAGS_RERUN)) { ++ dev_dbg(kbdev->dev, "Soft-stopped fragment shader job got a TILE_RANGE_FAULT. Possible HW issue, trying SW workaround\n"); ++ if (kbasep_10969_workaround_clamp_coordinates(katom)) { ++ /* The job had a TILE_RANGE_FAULT after was soft-stopped ++ * Due to an HW issue we try to execute the job again. ++ */ ++ dev_dbg(kbdev->dev, ++ "Clamping has been executed, try to rerun the job\n" ++ ); ++ katom->event_code = BASE_JD_EVENT_STOPPED; ++ katom->atom_flags |= KBASE_KATOM_FLAGS_RERUN; ++ } ++ } ++ ++ /* Clear the coreref_state now - while check_deref_cores() may not have ++ * been called yet, the caller will have taken a copy of this field. If ++ * this is not done, then if the atom is re-scheduled (following a soft ++ * stop) then the core reference would not be retaken. */ ++ katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED; ++ katom->affinity = 0; ++} ++ ++void kbase_backend_complete_wq_post_sched(struct kbase_device *kbdev, ++ base_jd_core_req core_req, u64 affinity, ++ enum kbase_atom_coreref_state coreref_state) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbasep_js_job_check_deref_cores_nokatom(kbdev, core_req, affinity, ++ coreref_state); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ if (!kbdev->pm.active_count) { ++ mutex_lock(&kbdev->js_data.runpool_mutex); ++ mutex_lock(&kbdev->pm.lock); ++ kbase_pm_update_active(kbdev); ++ mutex_unlock(&kbdev->pm.lock); ++ mutex_unlock(&kbdev->js_data.runpool_mutex); ++ } ++} ++ ++void kbase_gpu_dump_slots(struct kbase_device *kbdev) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ unsigned long flags; ++ int js; ++ ++ js_devdata = &kbdev->js_data; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ dev_info(kbdev->dev, "kbase_gpu_dump_slots:\n"); ++ ++ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { ++ int idx; ++ ++ for (idx = 0; idx < SLOT_RB_SIZE; idx++) { ++ struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, ++ js, ++ idx); ++ ++ if (katom) ++ dev_info(kbdev->dev, ++ " js%d idx%d : katom=%p gpu_rb_state=%d\n", ++ js, idx, katom, katom->gpu_rb_state); ++ else ++ dev_info(kbdev->dev, " js%d idx%d : empty\n", ++ js, idx); ++ } ++ } ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++} ++ ++ ++ +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_rb.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_rb.h +new file mode 100755 +index 000000000000..1e0e05ad3ea4 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_rb.h +@@ -0,0 +1,76 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++/* ++ * Register-based HW access backend specific APIs ++ */ ++ ++#ifndef _KBASE_HWACCESS_GPU_H_ ++#define _KBASE_HWACCESS_GPU_H_ ++ ++#include ++ ++/** ++ * kbase_gpu_irq_evict - Evict an atom from a NEXT slot ++ * ++ * @kbdev: Device pointer ++ * @js: Job slot to evict from ++ * ++ * Evict the atom in the NEXT slot for the specified job slot. This function is ++ * called from the job complete IRQ handler when the previous job has failed. ++ * ++ * Return: true if job evicted from NEXT registers, false otherwise ++ */ ++bool kbase_gpu_irq_evict(struct kbase_device *kbdev, int js); ++ ++/** ++ * kbase_gpu_complete_hw - Complete an atom on job slot js ++ * ++ * @kbdev: Device pointer ++ * @js: Job slot that has completed ++ * @completion_code: Event code from job that has completed ++ * @job_tail: The tail address from the hardware if the job has partially ++ * completed ++ * @end_timestamp: Time of completion ++ */ ++void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js, ++ u32 completion_code, ++ u64 job_tail, ++ ktime_t *end_timestamp); ++ ++/** ++ * kbase_gpu_inspect - Inspect the contents of the HW access ringbuffer ++ * ++ * @kbdev: Device pointer ++ * @js: Job slot to inspect ++ * @idx: Index into ringbuffer. 0 is the job currently running on ++ * the slot, 1 is the job waiting, all other values are invalid. ++ * Return: The atom at that position in the ringbuffer ++ * or NULL if no atom present ++ */ ++struct kbase_jd_atom *kbase_gpu_inspect(struct kbase_device *kbdev, int js, ++ int idx); ++ ++/** ++ * kbase_gpu_dump_slots - Print the contents of the slot ringbuffers ++ * ++ * @kbdev: Device pointer ++ */ ++void kbase_gpu_dump_slots(struct kbase_device *kbdev); ++ ++#endif /* _KBASE_HWACCESS_GPU_H_ */ +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_affinity.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_affinity.c +new file mode 100755 +index 000000000000..54d8ddd80097 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_affinity.c +@@ -0,0 +1,303 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * Base kernel affinity manager APIs ++ */ ++ ++#include ++#include "mali_kbase_js_affinity.h" ++#include "mali_kbase_hw.h" ++ ++#include ++ ++ ++bool kbase_js_can_run_job_on_slot_no_lock(struct kbase_device *kbdev, ++ int js) ++{ ++ /* ++ * Here are the reasons for using job slot 2: ++ * - BASE_HW_ISSUE_8987 (which is entirely used for that purpose) ++ * - In absence of the above, then: ++ * - Atoms with BASE_JD_REQ_COHERENT_GROUP ++ * - But, only when there aren't contexts with ++ * KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES, because the atoms that run on ++ * all cores on slot 1 could be blocked by those using a coherent group ++ * on slot 2 ++ * - And, only when you actually have 2 or more coregroups - if you ++ * only have 1 coregroup, then having jobs for slot 2 implies they'd ++ * also be for slot 1, meaning you'll get interference from them. Jobs ++ * able to run on slot 2 could also block jobs that can only run on ++ * slot 1 (tiler jobs) ++ */ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) ++ return true; ++ ++ if (js != 2) ++ return true; ++ ++ /* Only deal with js==2 now: */ ++ if (kbdev->gpu_props.num_core_groups > 1) { ++ /* Only use slot 2 in the 2+ coregroup case */ ++ if (kbasep_js_ctx_attr_is_attr_on_runpool(kbdev, ++ KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES) == ++ false) { ++ /* ...But only when we *don't* have atoms that run on ++ * all cores */ ++ ++ /* No specific check for BASE_JD_REQ_COHERENT_GROUP ++ * atoms - the policy will sort that out */ ++ return true; ++ } ++ } ++ ++ /* Above checks failed mean we shouldn't use slot 2 */ ++ return false; ++} ++ ++/* ++ * As long as it has been decided to have a deeper modification of ++ * what job scheduler, power manager and affinity manager will ++ * implement, this function is just an intermediate step that ++ * assumes: ++ * - all working cores will be powered on when this is called. ++ * - largest current configuration is 2 core groups. ++ * - It has been decided not to have hardcoded values so the low ++ * and high cores in a core split will be evently distributed. ++ * - Odd combinations of core requirements have been filtered out ++ * and do not get to this function (e.g. CS+T+NSS is not ++ * supported here). ++ * - This function is frequently called and can be optimized, ++ * (see notes in loops), but as the functionallity will likely ++ * be modified, optimization has not been addressed. ++*/ ++bool kbase_js_choose_affinity(u64 * const affinity, ++ struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom, int js) ++{ ++ base_jd_core_req core_req = katom->core_req; ++ unsigned int num_core_groups = kbdev->gpu_props.num_core_groups; ++ u64 core_availability_mask; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ core_availability_mask = kbase_pm_ca_get_core_mask(kbdev); ++ ++ /* ++ * If no cores are currently available (core availability policy is ++ * transitioning) then fail. ++ */ ++ if (0 == core_availability_mask) { ++ *affinity = 0; ++ return false; ++ } ++ ++ KBASE_DEBUG_ASSERT(js >= 0); ++ ++ if ((core_req & (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T)) == ++ BASE_JD_REQ_T) { ++ /* If the hardware supports XAFFINITY then we'll only enable ++ * the tiler (which is the default so this is a no-op), ++ * otherwise enable shader core 0. */ ++ if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_XAFFINITY)) ++ *affinity = 1; ++ else ++ *affinity = 0; ++ ++ return true; ++ } ++ ++ if (1 == kbdev->gpu_props.num_cores) { ++ /* trivial case only one core, nothing to do */ ++ *affinity = core_availability_mask & ++ kbdev->pm.debug_core_mask[js]; ++ } else { ++ if ((core_req & (BASE_JD_REQ_COHERENT_GROUP | ++ BASE_JD_REQ_SPECIFIC_COHERENT_GROUP))) { ++ if (js == 0 || num_core_groups == 1) { ++ /* js[0] and single-core-group systems just get ++ * the first core group */ ++ *affinity = ++ kbdev->gpu_props.props.coherency_info.group[0].core_mask ++ & core_availability_mask & ++ kbdev->pm.debug_core_mask[js]; ++ } else { ++ /* js[1], js[2] use core groups 0, 1 for ++ * dual-core-group systems */ ++ u32 core_group_idx = ((u32) js) - 1; ++ ++ KBASE_DEBUG_ASSERT(core_group_idx < ++ num_core_groups); ++ *affinity = ++ kbdev->gpu_props.props.coherency_info.group[core_group_idx].core_mask ++ & core_availability_mask & ++ kbdev->pm.debug_core_mask[js]; ++ ++ /* If the job is specifically targeting core ++ * group 1 and the core availability policy is ++ * keeping that core group off, then fail */ ++ if (*affinity == 0 && core_group_idx == 1 && ++ kbdev->pm.backend.cg1_disabled ++ == true) ++ katom->event_code = ++ BASE_JD_EVENT_PM_EVENT; ++ } ++ } else { ++ /* All cores are available when no core split is ++ * required */ ++ *affinity = core_availability_mask & ++ kbdev->pm.debug_core_mask[js]; ++ } ++ } ++ ++ /* ++ * If no cores are currently available in the desired core group(s) ++ * (core availability policy is transitioning) then fail. ++ */ ++ if (*affinity == 0) ++ return false; ++ ++ /* Enable core 0 if tiler required for hardware without XAFFINITY ++ * support (notes above) */ ++ if (core_req & BASE_JD_REQ_T) { ++ if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_XAFFINITY)) ++ *affinity = *affinity | 1; ++ } ++ ++ return true; ++} ++ ++static inline bool kbase_js_affinity_is_violating( ++ struct kbase_device *kbdev, ++ u64 *affinities) ++{ ++ /* This implementation checks whether the two slots involved in Generic ++ * thread creation have intersecting affinity. This is due to micro- ++ * architectural issues where a job in slot A targetting cores used by ++ * slot B could prevent the job in slot B from making progress until the ++ * job in slot A has completed. ++ */ ++ u64 affinity_set_left; ++ u64 affinity_set_right; ++ u64 intersection; ++ ++ KBASE_DEBUG_ASSERT(affinities != NULL); ++ ++ affinity_set_left = affinities[1]; ++ ++ affinity_set_right = affinities[2]; ++ ++ /* A violation occurs when any bit in the left_set is also in the ++ * right_set */ ++ intersection = affinity_set_left & affinity_set_right; ++ ++ return (bool) (intersection != (u64) 0u); ++} ++ ++bool kbase_js_affinity_would_violate(struct kbase_device *kbdev, int js, ++ u64 affinity) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ u64 new_affinities[BASE_JM_MAX_NR_SLOTS]; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(js < BASE_JM_MAX_NR_SLOTS); ++ js_devdata = &kbdev->js_data; ++ ++ memcpy(new_affinities, js_devdata->runpool_irq.slot_affinities, ++ sizeof(js_devdata->runpool_irq.slot_affinities)); ++ ++ new_affinities[js] |= affinity; ++ ++ return kbase_js_affinity_is_violating(kbdev, new_affinities); ++} ++ ++void kbase_js_affinity_retain_slot_cores(struct kbase_device *kbdev, int js, ++ u64 affinity) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ u64 cores; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(js < BASE_JM_MAX_NR_SLOTS); ++ js_devdata = &kbdev->js_data; ++ ++ KBASE_DEBUG_ASSERT(kbase_js_affinity_would_violate(kbdev, js, affinity) ++ == false); ++ ++ cores = affinity; ++ while (cores) { ++ int bitnum = fls64(cores) - 1; ++ u64 bit = 1ULL << bitnum; ++ s8 cnt; ++ ++ cnt = ++ ++(js_devdata->runpool_irq.slot_affinity_refcount[js][bitnum]); ++ ++ if (cnt == 1) ++ js_devdata->runpool_irq.slot_affinities[js] |= bit; ++ ++ cores &= ~bit; ++ } ++} ++ ++void kbase_js_affinity_release_slot_cores(struct kbase_device *kbdev, int js, ++ u64 affinity) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ u64 cores; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(js < BASE_JM_MAX_NR_SLOTS); ++ js_devdata = &kbdev->js_data; ++ ++ cores = affinity; ++ while (cores) { ++ int bitnum = fls64(cores) - 1; ++ u64 bit = 1ULL << bitnum; ++ s8 cnt; ++ ++ KBASE_DEBUG_ASSERT( ++ js_devdata->runpool_irq.slot_affinity_refcount[js][bitnum] > 0); ++ ++ cnt = ++ --(js_devdata->runpool_irq.slot_affinity_refcount[js][bitnum]); ++ ++ if (0 == cnt) ++ js_devdata->runpool_irq.slot_affinities[js] &= ~bit; ++ ++ cores &= ~bit; ++ } ++} ++ ++#if KBASE_TRACE_ENABLE ++void kbase_js_debug_log_current_affinities(struct kbase_device *kbdev) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ int slot_nr; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ js_devdata = &kbdev->js_data; ++ ++ for (slot_nr = 0; slot_nr < 3; ++slot_nr) ++ KBASE_TRACE_ADD_SLOT_INFO(kbdev, JS_AFFINITY_CURRENT, NULL, ++ NULL, 0u, slot_nr, ++ (u32) js_devdata->runpool_irq.slot_affinities[slot_nr]); ++} ++#endif /* KBASE_TRACE_ENABLE */ +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_affinity.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_affinity.h +new file mode 100755 +index 000000000000..35d9781ae092 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_affinity.h +@@ -0,0 +1,129 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * Affinity Manager internal APIs. ++ */ ++ ++#ifndef _KBASE_JS_AFFINITY_H_ ++#define _KBASE_JS_AFFINITY_H_ ++ ++/** ++ * kbase_js_can_run_job_on_slot_no_lock - Decide whether it is possible to ++ * submit a job to a particular job slot in the current status ++ * ++ * @kbdev: The kbase device structure of the device ++ * @js: Job slot number to check for allowance ++ * ++ * Will check if submitting to the given job slot is allowed in the current ++ * status. For example using job slot 2 while in soft-stoppable state and only ++ * having 1 coregroup is not allowed by the policy. This function should be ++ * called prior to submitting a job to a slot to make sure policy rules are not ++ * violated. ++ * ++ * The following locking conditions are made on the caller ++ * - it must hold hwaccess_lock ++ */ ++bool kbase_js_can_run_job_on_slot_no_lock(struct kbase_device *kbdev, int js); ++ ++/** ++ * kbase_js_choose_affinity - Compute affinity for a given job. ++ * ++ * @affinity: Affinity bitmap computed ++ * @kbdev: The kbase device structure of the device ++ * @katom: Job chain of which affinity is going to be found ++ * @js: Slot the job chain is being submitted ++ * ++ * Currently assumes an all-on/all-off power management policy. ++ * Also assumes there is at least one core with tiler available. ++ * ++ * Returns true if a valid affinity was chosen, false if ++ * no cores were available. ++ */ ++bool kbase_js_choose_affinity(u64 * const affinity, ++ struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom, ++ int js); ++ ++/** ++ * kbase_js_affinity_would_violate - Determine whether a proposed affinity on ++ * job slot @js would cause a violation of affinity restrictions. ++ * ++ * @kbdev: Kbase device structure ++ * @js: The job slot to test ++ * @affinity: The affinity mask to test ++ * ++ * The following locks must be held by the caller ++ * - hwaccess_lock ++ * ++ * Return: true if the affinity would violate the restrictions ++ */ ++bool kbase_js_affinity_would_violate(struct kbase_device *kbdev, int js, ++ u64 affinity); ++ ++/** ++ * kbase_js_affinity_retain_slot_cores - Affinity tracking: retain cores used by ++ * a slot ++ * ++ * @kbdev: Kbase device structure ++ * @js: The job slot retaining the cores ++ * @affinity: The cores to retain ++ * ++ * The following locks must be held by the caller ++ * - hwaccess_lock ++ */ ++void kbase_js_affinity_retain_slot_cores(struct kbase_device *kbdev, int js, ++ u64 affinity); ++ ++/** ++ * kbase_js_affinity_release_slot_cores - Affinity tracking: release cores used ++ * by a slot ++ * ++ * @kbdev: Kbase device structure ++ * @js: Job slot ++ * @affinity: Bit mask of core to be released ++ * ++ * Cores must be released as soon as a job is dequeued from a slot's 'submit ++ * slots', and before another job is submitted to those slots. Otherwise, the ++ * refcount could exceed the maximum number submittable to a slot, ++ * %BASE_JM_SUBMIT_SLOTS. ++ * ++ * The following locks must be held by the caller ++ * - hwaccess_lock ++ */ ++void kbase_js_affinity_release_slot_cores(struct kbase_device *kbdev, int js, ++ u64 affinity); ++ ++/** ++ * kbase_js_debug_log_current_affinities - log the current affinities ++ * ++ * @kbdev: Kbase device structure ++ * ++ * Output to the Trace log the current tracked affinities on all slots ++ */ ++#if KBASE_TRACE_ENABLE ++void kbase_js_debug_log_current_affinities(struct kbase_device *kbdev); ++#else /* KBASE_TRACE_ENABLE */ ++static inline void ++kbase_js_debug_log_current_affinities(struct kbase_device *kbdev) ++{ ++} ++#endif /* KBASE_TRACE_ENABLE */ ++ ++#endif /* _KBASE_JS_AFFINITY_H_ */ +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_backend.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_backend.c +new file mode 100755 +index 000000000000..a8c1af23a369 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_backend.c +@@ -0,0 +1,356 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++/* ++ * Register-based HW access backend specific job scheduler APIs ++ */ ++ ++#include ++#include ++#include ++#include ++ ++/* ++ * Define for when dumping is enabled. ++ * This should not be based on the instrumentation level as whether dumping is ++ * enabled for a particular level is down to the integrator. However this is ++ * being used for now as otherwise the cinstr headers would be needed. ++ */ ++#define CINSTR_DUMPING_ENABLED (2 == MALI_INSTRUMENTATION_LEVEL) ++ ++/* ++ * Hold the runpool_mutex for this ++ */ ++static inline bool timer_callback_should_run(struct kbase_device *kbdev) ++{ ++ struct kbase_backend_data *backend = &kbdev->hwaccess.backend; ++ s8 nr_running_ctxs; ++ ++ lockdep_assert_held(&kbdev->js_data.runpool_mutex); ++ ++ /* Timer must stop if we are suspending */ ++ if (backend->suspend_timer) ++ return false; ++ ++ /* nr_contexts_pullable is updated with the runpool_mutex. However, the ++ * locking in the caller gives us a barrier that ensures ++ * nr_contexts_pullable is up-to-date for reading */ ++ nr_running_ctxs = atomic_read(&kbdev->js_data.nr_contexts_runnable); ++ ++#ifdef CONFIG_MALI_DEBUG ++ if (kbdev->js_data.softstop_always) { ++ /* Debug support for allowing soft-stop on a single context */ ++ return true; ++ } ++#endif /* CONFIG_MALI_DEBUG */ ++ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_9435)) { ++ /* Timeouts would have to be 4x longer (due to micro- ++ * architectural design) to support OpenCL conformance tests, so ++ * only run the timer when there's: ++ * - 2 or more CL contexts ++ * - 1 or more GLES contexts ++ * ++ * NOTE: We will treat a context that has both Compute and Non- ++ * Compute jobs will be treated as an OpenCL context (hence, we ++ * don't check KBASEP_JS_CTX_ATTR_NON_COMPUTE). ++ */ ++ { ++ s8 nr_compute_ctxs = ++ kbasep_js_ctx_attr_count_on_runpool(kbdev, ++ KBASEP_JS_CTX_ATTR_COMPUTE); ++ s8 nr_noncompute_ctxs = nr_running_ctxs - ++ nr_compute_ctxs; ++ ++ return (bool) (nr_compute_ctxs >= 2 || ++ nr_noncompute_ctxs > 0); ++ } ++ } else { ++ /* Run the timer callback whenever you have at least 1 context ++ */ ++ return (bool) (nr_running_ctxs > 0); ++ } ++} ++ ++static enum hrtimer_restart timer_callback(struct hrtimer *timer) ++{ ++ unsigned long flags; ++ struct kbase_device *kbdev; ++ struct kbasep_js_device_data *js_devdata; ++ struct kbase_backend_data *backend; ++ int s; ++ bool reset_needed = false; ++ ++ KBASE_DEBUG_ASSERT(timer != NULL); ++ ++ backend = container_of(timer, struct kbase_backend_data, ++ scheduling_timer); ++ kbdev = container_of(backend, struct kbase_device, hwaccess.backend); ++ js_devdata = &kbdev->js_data; ++ ++ /* Loop through the slots */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ for (s = 0; s < kbdev->gpu_props.num_job_slots; s++) { ++ struct kbase_jd_atom *atom = NULL; ++ ++ if (kbase_backend_nr_atoms_on_slot(kbdev, s) > 0) { ++ atom = kbase_gpu_inspect(kbdev, s, 0); ++ KBASE_DEBUG_ASSERT(atom != NULL); ++ } ++ ++ if (atom != NULL) { ++ /* The current version of the model doesn't support ++ * Soft-Stop */ ++ if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_5736)) { ++ u32 ticks = atom->ticks++; ++ ++#if !CINSTR_DUMPING_ENABLED ++ u32 soft_stop_ticks, hard_stop_ticks, ++ gpu_reset_ticks; ++ if (atom->core_req & BASE_JD_REQ_ONLY_COMPUTE) { ++ soft_stop_ticks = ++ js_devdata->soft_stop_ticks_cl; ++ hard_stop_ticks = ++ js_devdata->hard_stop_ticks_cl; ++ gpu_reset_ticks = ++ js_devdata->gpu_reset_ticks_cl; ++ } else { ++ soft_stop_ticks = ++ js_devdata->soft_stop_ticks; ++ hard_stop_ticks = ++ js_devdata->hard_stop_ticks_ss; ++ gpu_reset_ticks = ++ js_devdata->gpu_reset_ticks_ss; ++ } ++ ++ /* If timeouts have been changed then ensure ++ * that atom tick count is not greater than the ++ * new soft_stop timeout. This ensures that ++ * atoms do not miss any of the timeouts due to ++ * races between this worker and the thread ++ * changing the timeouts. */ ++ if (backend->timeouts_updated && ++ ticks > soft_stop_ticks) ++ ticks = atom->ticks = soft_stop_ticks; ++ ++ /* Job is Soft-Stoppable */ ++ if (ticks == soft_stop_ticks) { ++ int disjoint_threshold = ++ KBASE_DISJOINT_STATE_INTERLEAVED_CONTEXT_COUNT_THRESHOLD; ++ u32 softstop_flags = 0u; ++ /* Job has been scheduled for at least ++ * js_devdata->soft_stop_ticks ticks. ++ * Soft stop the slot so we can run ++ * other jobs. ++ */ ++ dev_dbg(kbdev->dev, "Soft-stop"); ++#if !KBASE_DISABLE_SCHEDULING_SOFT_STOPS ++ /* nr_user_contexts_running is updated ++ * with the runpool_mutex, but we can't ++ * take that here. ++ * ++ * However, if it's about to be ++ * increased then the new context can't ++ * run any jobs until they take the ++ * hwaccess_lock, so it's OK to observe ++ * the older value. ++ * ++ * Similarly, if it's about to be ++ * decreased, the last job from another ++ * context has already finished, so it's ++ * not too bad that we observe the older ++ * value and register a disjoint event ++ * when we try soft-stopping */ ++ if (js_devdata->nr_user_contexts_running ++ >= disjoint_threshold) ++ softstop_flags |= ++ JS_COMMAND_SW_CAUSES_DISJOINT; ++ ++ kbase_job_slot_softstop_swflags(kbdev, ++ s, atom, softstop_flags); ++#endif ++ } else if (ticks == hard_stop_ticks) { ++ /* Job has been scheduled for at least ++ * js_devdata->hard_stop_ticks_ss ticks. ++ * It should have been soft-stopped by ++ * now. Hard stop the slot. ++ */ ++#if !KBASE_DISABLE_SCHEDULING_HARD_STOPS ++ int ms = ++ js_devdata->scheduling_period_ns ++ / 1000000u; ++ dev_warn(kbdev->dev, "JS: Job Hard-Stopped (took more than %lu ticks at %lu ms/tick)", ++ (unsigned long)ticks, ++ (unsigned long)ms); ++ kbase_job_slot_hardstop(atom->kctx, s, ++ atom); ++#endif ++ } else if (ticks == gpu_reset_ticks) { ++ /* Job has been scheduled for at least ++ * js_devdata->gpu_reset_ticks_ss ticks. ++ * It should have left the GPU by now. ++ * Signal that the GPU needs to be ++ * reset. ++ */ ++ reset_needed = true; ++ } ++#else /* !CINSTR_DUMPING_ENABLED */ ++ /* NOTE: During CINSTR_DUMPING_ENABLED, we use ++ * the alternate timeouts, which makes the hard- ++ * stop and GPU reset timeout much longer. We ++ * also ensure that we don't soft-stop at all. ++ */ ++ if (ticks == js_devdata->soft_stop_ticks) { ++ /* Job has been scheduled for at least ++ * js_devdata->soft_stop_ticks. We do ++ * not soft-stop during ++ * CINSTR_DUMPING_ENABLED, however. ++ */ ++ dev_dbg(kbdev->dev, "Soft-stop"); ++ } else if (ticks == ++ js_devdata->hard_stop_ticks_dumping) { ++ /* Job has been scheduled for at least ++ * js_devdata->hard_stop_ticks_dumping ++ * ticks. Hard stop the slot. ++ */ ++#if !KBASE_DISABLE_SCHEDULING_HARD_STOPS ++ int ms = ++ js_devdata->scheduling_period_ns ++ / 1000000u; ++ dev_warn(kbdev->dev, "JS: Job Hard-Stopped (took more than %lu ticks at %lu ms/tick)", ++ (unsigned long)ticks, ++ (unsigned long)ms); ++ kbase_job_slot_hardstop(atom->kctx, s, ++ atom); ++#endif ++ } else if (ticks == ++ js_devdata->gpu_reset_ticks_dumping) { ++ /* Job has been scheduled for at least ++ * js_devdata->gpu_reset_ticks_dumping ++ * ticks. It should have left the GPU by ++ * now. Signal that the GPU needs to be ++ * reset. ++ */ ++ reset_needed = true; ++ } ++#endif /* !CINSTR_DUMPING_ENABLED */ ++ } ++ } ++ } ++#if KBASE_GPU_RESET_EN ++ if (reset_needed) { ++ dev_err(kbdev->dev, "JS: Job has been on the GPU for too long (JS_RESET_TICKS_SS/DUMPING timeout hit). Issueing GPU soft-reset to resolve."); ++ ++ if (kbase_prepare_to_reset_gpu_locked(kbdev)) ++ kbase_reset_gpu_locked(kbdev); ++ } ++#endif /* KBASE_GPU_RESET_EN */ ++ /* the timer is re-issued if there is contexts in the run-pool */ ++ ++ if (backend->timer_running) ++ hrtimer_start(&backend->scheduling_timer, ++ HR_TIMER_DELAY_NSEC(js_devdata->scheduling_period_ns), ++ HRTIMER_MODE_REL); ++ ++ backend->timeouts_updated = false; ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ return HRTIMER_NORESTART; ++} ++ ++void kbase_backend_ctx_count_changed(struct kbase_device *kbdev) ++{ ++ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; ++ struct kbase_backend_data *backend = &kbdev->hwaccess.backend; ++ unsigned long flags; ++ ++ lockdep_assert_held(&js_devdata->runpool_mutex); ++ ++ if (!timer_callback_should_run(kbdev)) { ++ /* Take spinlock to force synchronisation with timer */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ backend->timer_running = false; ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ /* From now on, return value of timer_callback_should_run() will ++ * also cause the timer to not requeue itself. Its return value ++ * cannot change, because it depends on variables updated with ++ * the runpool_mutex held, which the caller of this must also ++ * hold */ ++ hrtimer_cancel(&backend->scheduling_timer); ++ } ++ ++ if (timer_callback_should_run(kbdev) && !backend->timer_running) { ++ /* Take spinlock to force synchronisation with timer */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ backend->timer_running = true; ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ hrtimer_start(&backend->scheduling_timer, ++ HR_TIMER_DELAY_NSEC(js_devdata->scheduling_period_ns), ++ HRTIMER_MODE_REL); ++ ++ KBASE_TRACE_ADD(kbdev, JS_POLICY_TIMER_START, NULL, NULL, 0u, ++ 0u); ++ } ++} ++ ++int kbase_backend_timer_init(struct kbase_device *kbdev) ++{ ++ struct kbase_backend_data *backend = &kbdev->hwaccess.backend; ++ ++ hrtimer_init(&backend->scheduling_timer, CLOCK_MONOTONIC, ++ HRTIMER_MODE_REL); ++ backend->scheduling_timer.function = timer_callback; ++ ++ backend->timer_running = false; ++ ++ return 0; ++} ++ ++void kbase_backend_timer_term(struct kbase_device *kbdev) ++{ ++ struct kbase_backend_data *backend = &kbdev->hwaccess.backend; ++ ++ hrtimer_cancel(&backend->scheduling_timer); ++} ++ ++void kbase_backend_timer_suspend(struct kbase_device *kbdev) ++{ ++ struct kbase_backend_data *backend = &kbdev->hwaccess.backend; ++ ++ backend->suspend_timer = true; ++ ++ kbase_backend_ctx_count_changed(kbdev); ++} ++ ++void kbase_backend_timer_resume(struct kbase_device *kbdev) ++{ ++ struct kbase_backend_data *backend = &kbdev->hwaccess.backend; ++ ++ backend->suspend_timer = false; ++ ++ kbase_backend_ctx_count_changed(kbdev); ++} ++ ++void kbase_backend_timeouts_changed(struct kbase_device *kbdev) ++{ ++ struct kbase_backend_data *backend = &kbdev->hwaccess.backend; ++ ++ backend->timeouts_updated = true; ++} ++ +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_internal.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_internal.h +new file mode 100755 +index 000000000000..3f53779c6747 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_internal.h +@@ -0,0 +1,69 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++/* ++ * Register-based HW access backend specific job scheduler APIs ++ */ ++ ++#ifndef _KBASE_JS_BACKEND_H_ ++#define _KBASE_JS_BACKEND_H_ ++ ++/** ++ * kbase_backend_timer_init() - Initialise the JS scheduling timer ++ * @kbdev: Device pointer ++ * ++ * This function should be called at driver initialisation ++ * ++ * Return: 0 on success ++ */ ++int kbase_backend_timer_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_backend_timer_term() - Terminate the JS scheduling timer ++ * @kbdev: Device pointer ++ * ++ * This function should be called at driver termination ++ */ ++void kbase_backend_timer_term(struct kbase_device *kbdev); ++ ++/** ++ * kbase_backend_timer_suspend - Suspend is happening, stop the JS scheduling ++ * timer ++ * @kbdev: Device pointer ++ * ++ * This function should be called on suspend, after the active count has reached ++ * zero. This is required as the timer may have been started on job submission ++ * to the job scheduler, but before jobs are submitted to the GPU. ++ * ++ * Caller must hold runpool_mutex. ++ */ ++void kbase_backend_timer_suspend(struct kbase_device *kbdev); ++ ++/** ++ * kbase_backend_timer_resume - Resume is happening, re-evaluate the JS ++ * scheduling timer ++ * @kbdev: Device pointer ++ * ++ * This function should be called on resume. Note that is is not guaranteed to ++ * re-start the timer, only evalute whether it should be re-started. ++ * ++ * Caller must hold runpool_mutex. ++ */ ++void kbase_backend_timer_resume(struct kbase_device *kbdev); ++ ++#endif /* _KBASE_JS_BACKEND_H_ */ +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_mmu_hw_direct.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_mmu_hw_direct.c +new file mode 100755 +index 000000000000..ba826184dd3f +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_mmu_hw_direct.c +@@ -0,0 +1,407 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++/* #define ENABLE_DEBUG_LOG */ ++#include "../../platform/rk/custom_log.h" ++ ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++static inline u64 lock_region(struct kbase_device *kbdev, u64 pfn, ++ u32 num_pages) ++{ ++ u64 region; ++ ++ /* can't lock a zero sized range */ ++ KBASE_DEBUG_ASSERT(num_pages); ++ ++ region = pfn << PAGE_SHIFT; ++ /* ++ * fls returns (given the ASSERT above): ++ * 1 .. 32 ++ * ++ * 10 + fls(num_pages) ++ * results in the range (11 .. 42) ++ */ ++ ++ /* gracefully handle num_pages being zero */ ++ if (0 == num_pages) { ++ region |= 11; ++ } else { ++ u8 region_width; ++ ++ region_width = 10 + fls(num_pages); ++ if (num_pages != (1ul << (region_width - 11))) { ++ /* not pow2, so must go up to the next pow2 */ ++ region_width += 1; ++ } ++ KBASE_DEBUG_ASSERT(region_width <= KBASE_LOCK_REGION_MAX_SIZE); ++ KBASE_DEBUG_ASSERT(region_width >= KBASE_LOCK_REGION_MIN_SIZE); ++ region |= region_width; ++ } ++ ++ return region; ++} ++ ++static int wait_ready(struct kbase_device *kbdev, ++ unsigned int as_nr, struct kbase_context *kctx) ++{ ++ unsigned int max_loops = KBASE_AS_INACTIVE_MAX_LOOPS; ++ u32 val = kbase_reg_read(kbdev, MMU_AS_REG(as_nr, AS_STATUS), kctx); ++ ++ /* Wait for the MMU status to indicate there is no active command, in ++ * case one is pending. Do not log remaining register accesses. */ ++ while (--max_loops && (val & AS_STATUS_AS_ACTIVE)) ++ val = kbase_reg_read(kbdev, MMU_AS_REG(as_nr, AS_STATUS), NULL); ++ ++ if (max_loops == 0) { ++ dev_err(kbdev->dev, "AS_ACTIVE bit stuck\n"); ++ return -1; ++ } ++ ++ /* If waiting in loop was performed, log last read value. */ ++ if (KBASE_AS_INACTIVE_MAX_LOOPS - 1 > max_loops) ++ kbase_reg_read(kbdev, MMU_AS_REG(as_nr, AS_STATUS), kctx); ++ ++ return 0; ++} ++ ++static int write_cmd(struct kbase_device *kbdev, int as_nr, u32 cmd, ++ struct kbase_context *kctx) ++{ ++ int status; ++ ++ /* write AS_COMMAND when MMU is ready to accept another command */ ++ status = wait_ready(kbdev, as_nr, kctx); ++ if (status == 0) ++ kbase_reg_write(kbdev, MMU_AS_REG(as_nr, AS_COMMAND), cmd, ++ kctx); ++ ++ return status; ++} ++ ++static void validate_protected_page_fault(struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++ /* GPUs which support (native) protected mode shall not report page ++ * fault addresses unless it has protected debug mode and protected ++ * debug mode is turned on */ ++ u32 protected_debug_mode = 0; ++ ++ if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_MODE)) ++ return; ++ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE)) { ++ protected_debug_mode = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(GPU_STATUS), ++ kctx) & GPU_DBGEN; ++ } ++ ++ if (!protected_debug_mode) { ++ /* fault_addr should never be reported in protected mode. ++ * However, we just continue by printing an error message */ ++ dev_err(kbdev->dev, "Fault address reported in protected mode\n"); ++ } ++} ++ ++void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat) ++{ ++ const int num_as = 16; ++ const int busfault_shift = MMU_PAGE_FAULT_FLAGS; ++ const int pf_shift = 0; ++ const unsigned long as_bit_mask = (1UL << num_as) - 1; ++ unsigned long flags; ++ u32 new_mask; ++ u32 tmp; ++ ++ /* bus faults */ ++ u32 bf_bits = (irq_stat >> busfault_shift) & as_bit_mask; ++ /* page faults (note: Ignore ASes with both pf and bf) */ ++ u32 pf_bits = ((irq_stat >> pf_shift) & as_bit_mask) & ~bf_bits; ++ ++ KBASE_DEBUG_ASSERT(NULL != kbdev); ++ ++ /* remember current mask */ ++ spin_lock_irqsave(&kbdev->mmu_mask_change, flags); ++ new_mask = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK), NULL); ++ /* mask interrupts for now */ ++ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0, NULL); ++ spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); ++ ++ while (bf_bits | pf_bits) { ++ struct kbase_as *as; ++ int as_no; ++ struct kbase_context *kctx; ++ ++ /* ++ * the while logic ensures we have a bit set, no need to check ++ * for not-found here ++ */ ++ as_no = ffs(bf_bits | pf_bits) - 1; ++ as = &kbdev->as[as_no]; ++ ++ /* ++ * Refcount the kctx ASAP - it shouldn't disappear anyway, since ++ * Bus/Page faults _should_ only occur whilst jobs are running, ++ * and a job causing the Bus/Page fault shouldn't complete until ++ * the MMU is updated ++ */ ++ kctx = kbasep_js_runpool_lookup_ctx(kbdev, as_no); ++ if (!kctx) { ++ E("fail to lookup ctx, to break out."); ++ break; ++ } ++ ++ ++ /* find faulting address */ ++ as->fault_addr = kbase_reg_read(kbdev, ++ MMU_AS_REG(as_no, ++ AS_FAULTADDRESS_HI), ++ kctx); ++ as->fault_addr <<= 32; ++ as->fault_addr |= kbase_reg_read(kbdev, ++ MMU_AS_REG(as_no, ++ AS_FAULTADDRESS_LO), ++ kctx); ++ ++ /* Mark the fault protected or not */ ++ as->protected_mode = kbdev->protected_mode; ++ ++ if (kbdev->protected_mode && as->fault_addr) ++ { ++ /* check if address reporting is allowed */ ++ validate_protected_page_fault(kbdev, kctx); ++ } ++ ++ /* report the fault to debugfs */ ++ kbase_as_fault_debugfs_new(kbdev, as_no); ++ ++ /* record the fault status */ ++ as->fault_status = kbase_reg_read(kbdev, ++ MMU_AS_REG(as_no, ++ AS_FAULTSTATUS), ++ kctx); ++ ++ /* find the fault type */ ++ as->fault_type = (bf_bits & (1 << as_no)) ? ++ KBASE_MMU_FAULT_TYPE_BUS : ++ KBASE_MMU_FAULT_TYPE_PAGE; ++ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) { ++ as->fault_extra_addr = kbase_reg_read(kbdev, ++ MMU_AS_REG(as_no, AS_FAULTEXTRA_HI), ++ kctx); ++ as->fault_extra_addr <<= 32; ++ as->fault_extra_addr |= kbase_reg_read(kbdev, ++ MMU_AS_REG(as_no, AS_FAULTEXTRA_LO), ++ kctx); ++ } ++ ++ if (kbase_as_has_bus_fault(as)) { ++ /* Mark bus fault as handled. ++ * Note that a bus fault is processed first in case ++ * where both a bus fault and page fault occur. ++ */ ++ bf_bits &= ~(1UL << as_no); ++ ++ /* remove the queued BF (and PF) from the mask */ ++ new_mask &= ~(MMU_BUS_ERROR(as_no) | ++ MMU_PAGE_FAULT(as_no)); ++ } else { ++ /* Mark page fault as handled */ ++ pf_bits &= ~(1UL << as_no); ++ ++ /* remove the queued PF from the mask */ ++ new_mask &= ~MMU_PAGE_FAULT(as_no); ++ } ++ ++ /* Process the interrupt for this address space */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_mmu_interrupt_process(kbdev, kctx, as); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ } ++ ++ /* reenable interrupts */ ++ spin_lock_irqsave(&kbdev->mmu_mask_change, flags); ++ tmp = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK), NULL); ++ new_mask |= tmp; ++ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), new_mask, NULL); ++ spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); ++} ++ ++void kbase_mmu_hw_configure(struct kbase_device *kbdev, struct kbase_as *as, ++ struct kbase_context *kctx) ++{ ++ struct kbase_mmu_setup *current_setup = &as->current_setup; ++ u32 transcfg = 0; ++ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) { ++ transcfg = current_setup->transcfg & 0xFFFFFFFFUL; ++ ++ /* Set flag AS_TRANSCFG_PTW_MEMATTR_WRITE_BACK */ ++ /* Clear PTW_MEMATTR bits */ ++ transcfg &= ~AS_TRANSCFG_PTW_MEMATTR_MASK; ++ /* Enable correct PTW_MEMATTR bits */ ++ transcfg |= AS_TRANSCFG_PTW_MEMATTR_WRITE_BACK; ++ ++ if (kbdev->system_coherency == COHERENCY_ACE) { ++ /* Set flag AS_TRANSCFG_PTW_SH_OS (outer shareable) */ ++ /* Clear PTW_SH bits */ ++ transcfg = (transcfg & ~AS_TRANSCFG_PTW_SH_MASK); ++ /* Enable correct PTW_SH bits */ ++ transcfg = (transcfg | AS_TRANSCFG_PTW_SH_OS); ++ } ++ ++ kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSCFG_LO), ++ transcfg, kctx); ++ kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSCFG_HI), ++ (current_setup->transcfg >> 32) & 0xFFFFFFFFUL, ++ kctx); ++ } else { ++ if (kbdev->system_coherency == COHERENCY_ACE) ++ current_setup->transtab |= AS_TRANSTAB_LPAE_SHARE_OUTER; ++ } ++ ++ kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSTAB_LO), ++ current_setup->transtab & 0xFFFFFFFFUL, kctx); ++ kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSTAB_HI), ++ (current_setup->transtab >> 32) & 0xFFFFFFFFUL, kctx); ++ ++ kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_MEMATTR_LO), ++ current_setup->memattr & 0xFFFFFFFFUL, kctx); ++ kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_MEMATTR_HI), ++ (current_setup->memattr >> 32) & 0xFFFFFFFFUL, kctx); ++ ++ KBASE_TLSTREAM_TL_ATTRIB_AS_CONFIG(as, ++ current_setup->transtab, ++ current_setup->memattr, ++ transcfg); ++ ++ write_cmd(kbdev, as->number, AS_COMMAND_UPDATE, kctx); ++} ++ ++int kbase_mmu_hw_do_operation(struct kbase_device *kbdev, struct kbase_as *as, ++ struct kbase_context *kctx, u64 vpfn, u32 nr, u32 op, ++ unsigned int handling_irq) ++{ ++ int ret; ++ ++ lockdep_assert_held(&kbdev->mmu_hw_mutex); ++ ++ if (op == AS_COMMAND_UNLOCK) { ++ /* Unlock doesn't require a lock first */ ++ ret = write_cmd(kbdev, as->number, AS_COMMAND_UNLOCK, kctx); ++ } else { ++ u64 lock_addr = lock_region(kbdev, vpfn, nr); ++ ++ /* Lock the region that needs to be updated */ ++ kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_LOCKADDR_LO), ++ lock_addr & 0xFFFFFFFFUL, kctx); ++ kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_LOCKADDR_HI), ++ (lock_addr >> 32) & 0xFFFFFFFFUL, kctx); ++ write_cmd(kbdev, as->number, AS_COMMAND_LOCK, kctx); ++ ++ /* Run the MMU operation */ ++ write_cmd(kbdev, as->number, op, kctx); ++ ++ /* Wait for the flush to complete */ ++ ret = wait_ready(kbdev, as->number, kctx); ++ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_9630)) { ++ /* Issue an UNLOCK command to ensure that valid page ++ tables are re-read by the GPU after an update. ++ Note that, the FLUSH command should perform all the ++ actions necessary, however the bus logs show that if ++ multiple page faults occur within an 8 page region ++ the MMU does not always re-read the updated page ++ table entries for later faults or is only partially ++ read, it subsequently raises the page fault IRQ for ++ the same addresses, the unlock ensures that the MMU ++ cache is flushed, so updates can be re-read. As the ++ region is now unlocked we need to issue 2 UNLOCK ++ commands in order to flush the MMU/uTLB, ++ see PRLAM-8812. ++ */ ++ write_cmd(kbdev, as->number, AS_COMMAND_UNLOCK, kctx); ++ write_cmd(kbdev, as->number, AS_COMMAND_UNLOCK, kctx); ++ } ++ } ++ ++ return ret; ++} ++ ++void kbase_mmu_hw_clear_fault(struct kbase_device *kbdev, struct kbase_as *as, ++ struct kbase_context *kctx, enum kbase_mmu_fault_type type) ++{ ++ unsigned long flags; ++ u32 pf_bf_mask; ++ ++ spin_lock_irqsave(&kbdev->mmu_mask_change, flags); ++ ++ /* ++ * A reset is in-flight and we're flushing the IRQ + bottom half ++ * so don't update anything as it could race with the reset code. ++ */ ++ if (kbdev->irq_reset_flush) ++ goto unlock; ++ ++ /* Clear the page (and bus fault IRQ as well in case one occurred) */ ++ pf_bf_mask = MMU_PAGE_FAULT(as->number); ++ if (type == KBASE_MMU_FAULT_TYPE_BUS || ++ type == KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED) ++ pf_bf_mask |= MMU_BUS_ERROR(as->number); ++ ++ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), pf_bf_mask, kctx); ++ ++unlock: ++ spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); ++} ++ ++void kbase_mmu_hw_enable_fault(struct kbase_device *kbdev, struct kbase_as *as, ++ struct kbase_context *kctx, enum kbase_mmu_fault_type type) ++{ ++ unsigned long flags; ++ u32 irq_mask; ++ ++ /* Enable the page fault IRQ (and bus fault IRQ as well in case one ++ * occurred) */ ++ spin_lock_irqsave(&kbdev->mmu_mask_change, flags); ++ ++ /* ++ * A reset is in-flight and we're flushing the IRQ + bottom half ++ * so don't update anything as it could race with the reset code. ++ */ ++ if (kbdev->irq_reset_flush) ++ goto unlock; ++ ++ irq_mask = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK), kctx) | ++ MMU_PAGE_FAULT(as->number); ++ ++ if (type == KBASE_MMU_FAULT_TYPE_BUS || ++ type == KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED) ++ irq_mask |= MMU_BUS_ERROR(as->number); ++ ++ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), irq_mask, kctx); ++ ++unlock: ++ spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); ++} +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_mmu_hw_direct.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_mmu_hw_direct.h +new file mode 100755 +index 000000000000..c02253c6acc3 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_mmu_hw_direct.h +@@ -0,0 +1,42 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* ++ * Interface file for the direct implementation for MMU hardware access ++ * ++ * Direct MMU hardware interface ++ * ++ * This module provides the interface(s) that are required by the direct ++ * register access implementation of the MMU hardware interface ++ */ ++ ++#ifndef _MALI_KBASE_MMU_HW_DIRECT_H_ ++#define _MALI_KBASE_MMU_HW_DIRECT_H_ ++ ++#include ++ ++/** ++ * kbase_mmu_interrupt - Process an MMU interrupt. ++ * ++ * Process the MMU interrupt that was reported by the &kbase_device. ++ * ++ * @kbdev: kbase context to clear the fault from. ++ * @irq_stat: Value of the MMU_IRQ_STATUS register ++ */ ++void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat); ++ ++#endif /* _MALI_KBASE_MMU_HW_DIRECT_H_ */ +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_always_on.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_always_on.c +new file mode 100755 +index 000000000000..0614348e935a +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_always_on.c +@@ -0,0 +1,63 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * "Always on" power management policy ++ */ ++ ++#include ++#include ++ ++static u64 always_on_get_core_mask(struct kbase_device *kbdev) ++{ ++ return kbdev->gpu_props.props.raw_props.shader_present; ++} ++ ++static bool always_on_get_core_active(struct kbase_device *kbdev) ++{ ++ return true; ++} ++ ++static void always_on_init(struct kbase_device *kbdev) ++{ ++ CSTD_UNUSED(kbdev); ++} ++ ++static void always_on_term(struct kbase_device *kbdev) ++{ ++ CSTD_UNUSED(kbdev); ++} ++ ++/* ++ * The struct kbase_pm_policy structure for the demand power policy. ++ * ++ * This is the static structure that defines the demand power policy's callback ++ * and name. ++ */ ++const struct kbase_pm_policy kbase_pm_always_on_policy_ops = { ++ "always_on", /* name */ ++ always_on_init, /* init */ ++ always_on_term, /* term */ ++ always_on_get_core_mask, /* get_core_mask */ ++ always_on_get_core_active, /* get_core_active */ ++ 0u, /* flags */ ++ KBASE_PM_POLICY_ID_ALWAYS_ON, /* id */ ++}; ++ ++KBASE_EXPORT_TEST_API(kbase_pm_always_on_policy_ops); +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_always_on.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_always_on.h +new file mode 100755 +index 000000000000..f9d244b01bc2 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_always_on.h +@@ -0,0 +1,77 @@ ++ ++/* ++ * ++ * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * "Always on" power management policy ++ */ ++ ++#ifndef MALI_KBASE_PM_ALWAYS_ON_H ++#define MALI_KBASE_PM_ALWAYS_ON_H ++ ++/** ++ * DOC: ++ * The "Always on" power management policy has the following ++ * characteristics: ++ * ++ * - When KBase indicates that the GPU will be powered up, but we don't yet ++ * know which Job Chains are to be run: ++ * All Shader Cores are powered up, regardless of whether or not they will ++ * be needed later. ++ * ++ * - When KBase indicates that a set of Shader Cores are needed to submit the ++ * currently queued Job Chains: ++ * All Shader Cores are kept powered, regardless of whether or not they will ++ * be needed ++ * ++ * - When KBase indicates that the GPU need not be powered: ++ * The Shader Cores are kept powered, regardless of whether or not they will ++ * be needed. The GPU itself is also kept powered, even though it is not ++ * needed. ++ * ++ * This policy is automatically overridden during system suspend: the desired ++ * core state is ignored, and the cores are forced off regardless of what the ++ * policy requests. After resuming from suspend, new changes to the desired ++ * core state made by the policy are honored. ++ * ++ * Note: ++ * ++ * - KBase indicates the GPU will be powered up when it has a User Process that ++ * has just started to submit Job Chains. ++ * ++ * - KBase indicates the GPU need not be powered when all the Job Chains from ++ * User Processes have finished, and it is waiting for a User Process to ++ * submit some more Job Chains. ++ */ ++ ++/** ++ * struct kbasep_pm_policy_always_on - Private struct for policy instance data ++ * @dummy: unused dummy variable ++ * ++ * This contains data that is private to the particular power policy that is ++ * active. ++ */ ++struct kbasep_pm_policy_always_on { ++ int dummy; ++}; ++ ++extern const struct kbase_pm_policy kbase_pm_always_on_policy_ops; ++ ++#endif /* MALI_KBASE_PM_ALWAYS_ON_H */ ++ +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_backend.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_backend.c +new file mode 100755 +index 000000000000..146fd48bab92 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_backend.c +@@ -0,0 +1,482 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++/* ++ * GPU backend implementation of base kernel power management APIs ++ */ ++ ++#include ++#include ++#include ++#ifdef CONFIG_MALI_PLATFORM_DEVICETREE ++#include ++#endif /* CONFIG_MALI_PLATFORM_DEVICETREE */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data); ++ ++void kbase_pm_register_access_enable(struct kbase_device *kbdev) ++{ ++ struct kbase_pm_callback_conf *callbacks; ++ ++ callbacks = (struct kbase_pm_callback_conf *)POWER_MANAGEMENT_CALLBACKS; ++ ++ if (callbacks) ++ callbacks->power_on_callback(kbdev); ++ ++ kbdev->pm.backend.gpu_powered = true; ++} ++ ++void kbase_pm_register_access_disable(struct kbase_device *kbdev) ++{ ++ struct kbase_pm_callback_conf *callbacks; ++ ++ callbacks = (struct kbase_pm_callback_conf *)POWER_MANAGEMENT_CALLBACKS; ++ ++ if (callbacks) ++ callbacks->power_off_callback(kbdev); ++ ++ kbdev->pm.backend.gpu_powered = false; ++} ++ ++int kbase_hwaccess_pm_init(struct kbase_device *kbdev) ++{ ++ int ret = 0; ++ struct kbase_pm_callback_conf *callbacks; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ mutex_init(&kbdev->pm.lock); ++ ++ kbdev->pm.backend.gpu_poweroff_wait_wq = alloc_workqueue("kbase_pm_poweroff_wait", ++ WQ_HIGHPRI | WQ_UNBOUND, 1); ++ if (!kbdev->pm.backend.gpu_poweroff_wait_wq) ++ return -ENOMEM; ++ ++ INIT_WORK(&kbdev->pm.backend.gpu_poweroff_wait_work, ++ kbase_pm_gpu_poweroff_wait_wq); ++ ++ kbdev->pm.backend.gpu_powered = false; ++ kbdev->pm.suspending = false; ++#ifdef CONFIG_MALI_DEBUG ++ kbdev->pm.backend.driver_ready_for_irqs = false; ++#endif /* CONFIG_MALI_DEBUG */ ++ kbdev->pm.backend.gpu_in_desired_state = true; ++ init_waitqueue_head(&kbdev->pm.backend.gpu_in_desired_state_wait); ++ ++ callbacks = (struct kbase_pm_callback_conf *)POWER_MANAGEMENT_CALLBACKS; ++ if (callbacks) { ++ kbdev->pm.backend.callback_power_on = ++ callbacks->power_on_callback; ++ kbdev->pm.backend.callback_power_off = ++ callbacks->power_off_callback; ++ kbdev->pm.backend.callback_power_suspend = ++ callbacks->power_suspend_callback; ++ kbdev->pm.backend.callback_power_resume = ++ callbacks->power_resume_callback; ++ kbdev->pm.callback_power_runtime_init = ++ callbacks->power_runtime_init_callback; ++ kbdev->pm.callback_power_runtime_term = ++ callbacks->power_runtime_term_callback; ++ kbdev->pm.backend.callback_power_runtime_on = ++ callbacks->power_runtime_on_callback; ++ kbdev->pm.backend.callback_power_runtime_off = ++ callbacks->power_runtime_off_callback; ++ kbdev->pm.backend.callback_power_runtime_idle = ++ callbacks->power_runtime_idle_callback; ++ } else { ++ kbdev->pm.backend.callback_power_on = NULL; ++ kbdev->pm.backend.callback_power_off = NULL; ++ kbdev->pm.backend.callback_power_suspend = NULL; ++ kbdev->pm.backend.callback_power_resume = NULL; ++ kbdev->pm.callback_power_runtime_init = NULL; ++ kbdev->pm.callback_power_runtime_term = NULL; ++ kbdev->pm.backend.callback_power_runtime_on = NULL; ++ kbdev->pm.backend.callback_power_runtime_off = NULL; ++ kbdev->pm.backend.callback_power_runtime_idle = NULL; ++ } ++ ++ /* Initialise the metrics subsystem */ ++ ret = kbasep_pm_metrics_init(kbdev); ++ if (ret) ++ return ret; ++ ++ init_waitqueue_head(&kbdev->pm.backend.l2_powered_wait); ++ kbdev->pm.backend.l2_powered = 0; ++ ++ init_waitqueue_head(&kbdev->pm.backend.reset_done_wait); ++ kbdev->pm.backend.reset_done = false; ++ ++ init_waitqueue_head(&kbdev->pm.zero_active_count_wait); ++ kbdev->pm.active_count = 0; ++ ++ spin_lock_init(&kbdev->pm.backend.gpu_cycle_counter_requests_lock); ++ spin_lock_init(&kbdev->pm.backend.gpu_powered_lock); ++ ++ init_waitqueue_head(&kbdev->pm.backend.poweroff_wait); ++ ++ if (kbase_pm_ca_init(kbdev) != 0) ++ goto workq_fail; ++ ++ if (kbase_pm_policy_init(kbdev) != 0) ++ goto pm_policy_fail; ++ ++ return 0; ++ ++pm_policy_fail: ++ kbase_pm_ca_term(kbdev); ++workq_fail: ++ kbasep_pm_metrics_term(kbdev); ++ return -EINVAL; ++} ++ ++void kbase_pm_do_poweron(struct kbase_device *kbdev, bool is_resume) ++{ ++ lockdep_assert_held(&kbdev->pm.lock); ++ ++ /* Turn clocks and interrupts on - no-op if we haven't done a previous ++ * kbase_pm_clock_off() */ ++ kbase_pm_clock_on(kbdev, is_resume); ++ ++ /* Update core status as required by the policy */ ++ KBASE_TIMELINE_PM_CHECKTRANS(kbdev, ++ SW_FLOW_PM_CHECKTRANS_PM_DO_POWERON_START); ++ kbase_pm_update_cores_state(kbdev); ++ KBASE_TIMELINE_PM_CHECKTRANS(kbdev, ++ SW_FLOW_PM_CHECKTRANS_PM_DO_POWERON_END); ++ ++ /* NOTE: We don't wait to reach the desired state, since running atoms ++ * will wait for that state to be reached anyway */ ++} ++ ++static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data) ++{ ++ struct kbase_device *kbdev = container_of(data, struct kbase_device, ++ pm.backend.gpu_poweroff_wait_work); ++ struct kbase_pm_device_data *pm = &kbdev->pm; ++ struct kbase_pm_backend_data *backend = &pm->backend; ++ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; ++ unsigned long flags; ++ ++#if !PLATFORM_POWER_DOWN_ONLY ++ /* Wait for power transitions to complete. We do this with no locks held ++ * so that we don't deadlock with any pending workqueues */ ++ KBASE_TIMELINE_PM_CHECKTRANS(kbdev, ++ SW_FLOW_PM_CHECKTRANS_PM_DO_POWEROFF_START); ++ kbase_pm_check_transitions_sync(kbdev); ++ KBASE_TIMELINE_PM_CHECKTRANS(kbdev, ++ SW_FLOW_PM_CHECKTRANS_PM_DO_POWEROFF_END); ++#endif /* !PLATFORM_POWER_DOWN_ONLY */ ++ ++ mutex_lock(&js_devdata->runpool_mutex); ++ mutex_lock(&kbdev->pm.lock); ++ ++#if PLATFORM_POWER_DOWN_ONLY ++ if (kbdev->pm.backend.gpu_powered) { ++ if (kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_L2)) { ++ /* If L2 cache is powered then we must flush it before ++ * we power off the GPU. Normally this would have been ++ * handled when the L2 was powered off. */ ++ kbase_gpu_cacheclean(kbdev); ++ } ++ } ++#endif /* PLATFORM_POWER_DOWN_ONLY */ ++ ++ if (!backend->poweron_required) { ++#if !PLATFORM_POWER_DOWN_ONLY ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ WARN_ON(kbdev->l2_available_bitmap || ++ kbdev->shader_available_bitmap || ++ kbdev->tiler_available_bitmap); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++#endif /* !PLATFORM_POWER_DOWN_ONLY */ ++ ++ /* Consume any change-state events */ ++ kbase_timeline_pm_check_handle_event(kbdev, ++ KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED); ++ ++ /* Disable interrupts and turn the clock off */ ++ if (!kbase_pm_clock_off(kbdev, backend->poweroff_is_suspend)) { ++ /* ++ * Page/bus faults are pending, must drop locks to ++ * process. Interrupts are disabled so no more faults ++ * should be generated at this point. ++ */ ++ mutex_unlock(&kbdev->pm.lock); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ kbase_flush_mmu_wqs(kbdev); ++ mutex_lock(&js_devdata->runpool_mutex); ++ mutex_lock(&kbdev->pm.lock); ++ ++ /* Turn off clock now that fault have been handled. We ++ * dropped locks so poweron_required may have changed - ++ * power back on if this is the case.*/ ++ if (backend->poweron_required) ++ kbase_pm_clock_on(kbdev, false); ++ else ++ WARN_ON(!kbase_pm_clock_off(kbdev, ++ backend->poweroff_is_suspend)); ++ } ++ } ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ backend->poweroff_wait_in_progress = false; ++ if (backend->poweron_required) { ++ backend->poweron_required = false; ++ kbase_pm_update_cores_state_nolock(kbdev); ++ kbase_backend_slot_update(kbdev); ++ } ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ mutex_unlock(&kbdev->pm.lock); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ ++ wake_up(&kbdev->pm.backend.poweroff_wait); ++} ++ ++void kbase_pm_do_poweroff(struct kbase_device *kbdev, bool is_suspend) ++{ ++ unsigned long flags; ++ ++ lockdep_assert_held(&kbdev->pm.lock); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ if (!kbdev->pm.backend.poweroff_wait_in_progress) { ++ /* Force all cores off */ ++ kbdev->pm.backend.desired_shader_state = 0; ++ kbdev->pm.backend.desired_tiler_state = 0; ++ ++ /* Force all cores to be unavailable, in the situation where ++ * transitions are in progress for some cores but not others, ++ * and kbase_pm_check_transitions_nolock can not immediately ++ * power off the cores */ ++ kbdev->shader_available_bitmap = 0; ++ kbdev->tiler_available_bitmap = 0; ++ kbdev->l2_available_bitmap = 0; ++ ++ kbdev->pm.backend.poweroff_wait_in_progress = true; ++ kbdev->pm.backend.poweroff_is_suspend = is_suspend; ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ /*Kick off wq here. Callers will have to wait*/ ++ queue_work(kbdev->pm.backend.gpu_poweroff_wait_wq, ++ &kbdev->pm.backend.gpu_poweroff_wait_work); ++ } else { ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ } ++} ++ ++static bool is_poweroff_in_progress(struct kbase_device *kbdev) ++{ ++ bool ret; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ret = (kbdev->pm.backend.poweroff_wait_in_progress == false); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ return ret; ++} ++ ++void kbase_pm_wait_for_poweroff_complete(struct kbase_device *kbdev) ++{ ++ wait_event_killable(kbdev->pm.backend.poweroff_wait, ++ is_poweroff_in_progress(kbdev)); ++} ++ ++int kbase_hwaccess_pm_powerup(struct kbase_device *kbdev, ++ unsigned int flags) ++{ ++ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; ++ unsigned long irq_flags; ++ int ret; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ mutex_lock(&js_devdata->runpool_mutex); ++ mutex_lock(&kbdev->pm.lock); ++ ++ /* A suspend won't happen during startup/insmod */ ++ KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev)); ++ ++ /* Power up the GPU, don't enable IRQs as we are not ready to receive ++ * them. */ ++ ret = kbase_pm_init_hw(kbdev, flags); ++ if (ret) { ++ mutex_unlock(&kbdev->pm.lock); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ return ret; ++ } ++ ++ kbasep_pm_init_core_use_bitmaps(kbdev); ++ ++ kbdev->pm.debug_core_mask_all = kbdev->pm.debug_core_mask[0] = ++ kbdev->pm.debug_core_mask[1] = ++ kbdev->pm.debug_core_mask[2] = ++ kbdev->gpu_props.props.raw_props.shader_present; ++ ++ /* Pretend the GPU is active to prevent a power policy turning the GPU ++ * cores off */ ++ kbdev->pm.active_count = 1; ++ ++ spin_lock_irqsave(&kbdev->pm.backend.gpu_cycle_counter_requests_lock, ++ irq_flags); ++ /* Ensure cycle counter is off */ ++ kbdev->pm.backend.gpu_cycle_counter_requests = 0; ++ spin_unlock_irqrestore( ++ &kbdev->pm.backend.gpu_cycle_counter_requests_lock, ++ irq_flags); ++ ++ /* We are ready to receive IRQ's now as power policy is set up, so ++ * enable them now. */ ++#ifdef CONFIG_MALI_DEBUG ++ spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, irq_flags); ++ kbdev->pm.backend.driver_ready_for_irqs = true; ++ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, irq_flags); ++#endif ++ kbase_pm_enable_interrupts(kbdev); ++ ++ /* Turn on the GPU and any cores needed by the policy */ ++ kbase_pm_do_poweron(kbdev, false); ++ mutex_unlock(&kbdev->pm.lock); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ ++ /* Idle the GPU and/or cores, if the policy wants it to */ ++ kbase_pm_context_idle(kbdev); ++ ++ return 0; ++} ++ ++void kbase_hwaccess_pm_halt(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ mutex_lock(&kbdev->pm.lock); ++ kbase_pm_cancel_deferred_poweroff(kbdev); ++ kbase_pm_do_poweroff(kbdev, false); ++ mutex_unlock(&kbdev->pm.lock); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_hwaccess_pm_halt); ++ ++void kbase_hwaccess_pm_term(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kbdev->pm.active_count == 0); ++ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests == 0); ++ ++ /* Free any resources the policy allocated */ ++ kbase_pm_policy_term(kbdev); ++ kbase_pm_ca_term(kbdev); ++ ++ /* Shut down the metrics subsystem */ ++ kbasep_pm_metrics_term(kbdev); ++ ++ destroy_workqueue(kbdev->pm.backend.gpu_poweroff_wait_wq); ++} ++ ++void kbase_pm_power_changed(struct kbase_device *kbdev) ++{ ++ bool cores_are_available; ++ unsigned long flags; ++ ++ KBASE_TIMELINE_PM_CHECKTRANS(kbdev, ++ SW_FLOW_PM_CHECKTRANS_GPU_INTERRUPT_START); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ cores_are_available = kbase_pm_check_transitions_nolock(kbdev); ++ KBASE_TIMELINE_PM_CHECKTRANS(kbdev, ++ SW_FLOW_PM_CHECKTRANS_GPU_INTERRUPT_END); ++ ++ if (cores_are_available) { ++ /* Log timelining information that a change in state has ++ * completed */ ++ kbase_timeline_pm_handle_event(kbdev, ++ KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED); ++ ++ kbase_backend_slot_update(kbdev); ++ } ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++} ++ ++void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev, ++ u64 new_core_mask_js0, u64 new_core_mask_js1, ++ u64 new_core_mask_js2) ++{ ++ kbdev->pm.debug_core_mask[0] = new_core_mask_js0; ++ kbdev->pm.debug_core_mask[1] = new_core_mask_js1; ++ kbdev->pm.debug_core_mask[2] = new_core_mask_js2; ++ kbdev->pm.debug_core_mask_all = new_core_mask_js0 | new_core_mask_js1 | ++ new_core_mask_js2; ++ ++ kbase_pm_update_cores_state_nolock(kbdev); ++} ++ ++void kbase_hwaccess_pm_gpu_active(struct kbase_device *kbdev) ++{ ++ kbase_pm_update_active(kbdev); ++} ++ ++void kbase_hwaccess_pm_gpu_idle(struct kbase_device *kbdev) ++{ ++ kbase_pm_update_active(kbdev); ++} ++ ++void kbase_hwaccess_pm_suspend(struct kbase_device *kbdev) ++{ ++ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; ++ ++ /* Force power off the GPU and all cores (regardless of policy), only ++ * after the PM active count reaches zero (otherwise, we risk turning it ++ * off prematurely) */ ++ mutex_lock(&js_devdata->runpool_mutex); ++ mutex_lock(&kbdev->pm.lock); ++ ++ kbase_pm_cancel_deferred_poweroff(kbdev); ++ kbase_pm_do_poweroff(kbdev, true); ++ ++ kbase_backend_timer_suspend(kbdev); ++ ++ mutex_unlock(&kbdev->pm.lock); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ ++ kbase_pm_wait_for_poweroff_complete(kbdev); ++} ++ ++void kbase_hwaccess_pm_resume(struct kbase_device *kbdev) ++{ ++ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; ++ ++ mutex_lock(&js_devdata->runpool_mutex); ++ mutex_lock(&kbdev->pm.lock); ++ ++ kbdev->pm.suspending = false; ++ kbase_pm_do_poweron(kbdev, true); ++ ++ kbase_backend_timer_resume(kbdev); ++ ++ mutex_unlock(&kbdev->pm.lock); ++ mutex_unlock(&js_devdata->runpool_mutex); ++} +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca.c +new file mode 100755 +index 000000000000..85890f1e85f5 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca.c +@@ -0,0 +1,182 @@ ++/* ++ * ++ * (C) COPYRIGHT 2013-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* ++ * Base kernel core availability APIs ++ */ ++ ++#include ++#include ++#include ++ ++static const struct kbase_pm_ca_policy *const policy_list[] = { ++ &kbase_pm_ca_fixed_policy_ops, ++#ifdef CONFIG_MALI_DEVFREQ ++ &kbase_pm_ca_devfreq_policy_ops, ++#endif ++#if !MALI_CUSTOMER_RELEASE ++ &kbase_pm_ca_random_policy_ops ++#endif ++}; ++ ++/** ++ * POLICY_COUNT - The number of policies available in the system. ++ * ++ * This is derived from the number of functions listed in policy_list. ++ */ ++#define POLICY_COUNT (sizeof(policy_list)/sizeof(*policy_list)) ++ ++int kbase_pm_ca_init(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ kbdev->pm.backend.ca_current_policy = policy_list[0]; ++ ++ kbdev->pm.backend.ca_current_policy->init(kbdev); ++ ++ return 0; ++} ++ ++void kbase_pm_ca_term(struct kbase_device *kbdev) ++{ ++ kbdev->pm.backend.ca_current_policy->term(kbdev); ++} ++ ++int kbase_pm_ca_list_policies(const struct kbase_pm_ca_policy * const **list) ++{ ++ if (!list) ++ return POLICY_COUNT; ++ ++ *list = policy_list; ++ ++ return POLICY_COUNT; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_ca_list_policies); ++ ++const struct kbase_pm_ca_policy ++*kbase_pm_ca_get_policy(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ return kbdev->pm.backend.ca_current_policy; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_ca_get_policy); ++ ++void kbase_pm_ca_set_policy(struct kbase_device *kbdev, ++ const struct kbase_pm_ca_policy *new_policy) ++{ ++ const struct kbase_pm_ca_policy *old_policy; ++ unsigned long flags; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(new_policy != NULL); ++ ++ KBASE_TRACE_ADD(kbdev, PM_CA_SET_POLICY, NULL, NULL, 0u, ++ new_policy->id); ++ ++ /* During a policy change we pretend the GPU is active */ ++ /* A suspend won't happen here, because we're in a syscall from a ++ * userspace thread */ ++ kbase_pm_context_active(kbdev); ++ ++ mutex_lock(&kbdev->pm.lock); ++ ++ /* Remove the policy to prevent IRQ handlers from working on it */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ old_policy = kbdev->pm.backend.ca_current_policy; ++ kbdev->pm.backend.ca_current_policy = NULL; ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ if (old_policy->term) ++ old_policy->term(kbdev); ++ ++ if (new_policy->init) ++ new_policy->init(kbdev); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbdev->pm.backend.ca_current_policy = new_policy; ++ ++ /* If any core power state changes were previously attempted, but ++ * couldn't be made because the policy was changing (current_policy was ++ * NULL), then re-try them here. */ ++ kbase_pm_update_cores_state_nolock(kbdev); ++ ++ kbdev->pm.backend.ca_current_policy->update_core_status(kbdev, ++ kbdev->shader_ready_bitmap, ++ kbdev->shader_transitioning_bitmap); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ mutex_unlock(&kbdev->pm.lock); ++ ++ /* Now the policy change is finished, we release our fake context active ++ * reference */ ++ kbase_pm_context_idle(kbdev); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_ca_set_policy); ++ ++u64 kbase_pm_ca_get_core_mask(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ /* All cores must be enabled when instrumentation is in use */ ++ if (kbdev->pm.backend.instr_enabled) ++ return kbdev->gpu_props.props.raw_props.shader_present & ++ kbdev->pm.debug_core_mask_all; ++ ++ if (kbdev->pm.backend.ca_current_policy == NULL) ++ return kbdev->gpu_props.props.raw_props.shader_present & ++ kbdev->pm.debug_core_mask_all; ++ ++ return kbdev->pm.backend.ca_current_policy->get_core_mask(kbdev) & ++ kbdev->pm.debug_core_mask_all; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_ca_get_core_mask); ++ ++void kbase_pm_ca_update_core_status(struct kbase_device *kbdev, u64 cores_ready, ++ u64 cores_transitioning) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (kbdev->pm.backend.ca_current_policy != NULL) ++ kbdev->pm.backend.ca_current_policy->update_core_status(kbdev, ++ cores_ready, ++ cores_transitioning); ++} ++ ++void kbase_pm_ca_instr_enable(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbdev->pm.backend.instr_enabled = true; ++ ++ kbase_pm_update_cores_state_nolock(kbdev); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++} ++ ++void kbase_pm_ca_instr_disable(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ kbdev->pm.backend.instr_enabled = false; ++ ++ kbase_pm_update_cores_state_nolock(kbdev); ++} +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca.h +new file mode 100755 +index 000000000000..ee9e751f2d79 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca.h +@@ -0,0 +1,92 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* ++ * Base kernel core availability APIs ++ */ ++ ++#ifndef _KBASE_PM_CA_H_ ++#define _KBASE_PM_CA_H_ ++ ++/** ++ * kbase_pm_ca_init - Initialize core availability framework ++ * ++ * Must be called before calling any other core availability function ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * Return: 0 if the core availability framework was successfully initialized, ++ * -errno otherwise ++ */ ++int kbase_pm_ca_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_ca_term - Terminate core availability framework ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_ca_term(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_ca_get_core_mask - Get currently available shaders core mask ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * Returns a mask of the currently available shader cores. ++ * Calls into the core availability policy ++ * ++ * Return: The bit mask of available cores ++ */ ++u64 kbase_pm_ca_get_core_mask(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_ca_update_core_status - Update core status ++ * ++ * @kbdev: The kbase device structure for the device (must be ++ * a valid pointer) ++ * @cores_ready: The bit mask of cores ready for job submission ++ * @cores_transitioning: The bit mask of cores that are transitioning power ++ * state ++ * ++ * Update core availability policy with current core power status ++ * ++ * Calls into the core availability policy ++ */ ++void kbase_pm_ca_update_core_status(struct kbase_device *kbdev, u64 cores_ready, ++ u64 cores_transitioning); ++ ++/** ++ * kbase_pm_ca_instr_enable - Enable override for instrumentation ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * This overrides the output of the core availability policy, ensuring that all ++ * cores are available ++ */ ++void kbase_pm_ca_instr_enable(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_ca_instr_disable - Disable override for instrumentation ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * This disables any previously enabled override, and resumes normal policy ++ * functionality ++ */ ++void kbase_pm_ca_instr_disable(struct kbase_device *kbdev); ++ ++#endif /* _KBASE_PM_CA_H_ */ +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca_devfreq.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca_devfreq.c +new file mode 100755 +index 000000000000..66bf660cffb6 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca_devfreq.c +@@ -0,0 +1,129 @@ ++/* ++ * ++ * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* ++ * A core availability policy implementing core mask selection from devfreq OPPs ++ * ++ */ ++ ++#include ++#include ++#include ++#include ++ ++void kbase_devfreq_set_core_mask(struct kbase_device *kbdev, u64 core_mask) ++{ ++ struct kbasep_pm_ca_policy_devfreq *data = ++ &kbdev->pm.backend.ca_policy_data.devfreq; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ data->cores_desired = core_mask; ++ ++ /* Disable any cores that are now unwanted */ ++ data->cores_enabled &= data->cores_desired; ++ ++ kbdev->pm.backend.ca_in_transition = true; ++ ++ /* If there are no cores to be powered off then power on desired cores ++ */ ++ if (!(data->cores_used & ~data->cores_desired)) { ++ data->cores_enabled = data->cores_desired; ++ kbdev->pm.backend.ca_in_transition = false; ++ } ++ ++ kbase_pm_update_cores_state_nolock(kbdev); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ dev_dbg(kbdev->dev, "Devfreq policy : new core mask=%llX %llX\n", ++ data->cores_desired, data->cores_enabled); ++} ++ ++static void devfreq_init(struct kbase_device *kbdev) ++{ ++ struct kbasep_pm_ca_policy_devfreq *data = ++ &kbdev->pm.backend.ca_policy_data.devfreq; ++ ++ if (kbdev->current_core_mask) { ++ data->cores_enabled = kbdev->current_core_mask; ++ data->cores_desired = kbdev->current_core_mask; ++ } else { ++ data->cores_enabled = ++ kbdev->gpu_props.props.raw_props.shader_present; ++ data->cores_desired = ++ kbdev->gpu_props.props.raw_props.shader_present; ++ } ++ data->cores_used = 0; ++ kbdev->pm.backend.ca_in_transition = false; ++} ++ ++static void devfreq_term(struct kbase_device *kbdev) ++{ ++} ++ ++static u64 devfreq_get_core_mask(struct kbase_device *kbdev) ++{ ++ return kbdev->pm.backend.ca_policy_data.devfreq.cores_enabled; ++} ++ ++static void devfreq_update_core_status(struct kbase_device *kbdev, ++ u64 cores_ready, ++ u64 cores_transitioning) ++{ ++ struct kbasep_pm_ca_policy_devfreq *data = ++ &kbdev->pm.backend.ca_policy_data.devfreq; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ data->cores_used = cores_ready | cores_transitioning; ++ ++ /* If in desired state then clear transition flag */ ++ if (data->cores_enabled == data->cores_desired) ++ kbdev->pm.backend.ca_in_transition = false; ++ ++ /* If all undesired cores are now off then power on desired cores. ++ * The direct comparison against cores_enabled limits potential ++ * recursion to one level */ ++ if (!(data->cores_used & ~data->cores_desired) && ++ data->cores_enabled != data->cores_desired) { ++ data->cores_enabled = data->cores_desired; ++ ++ kbase_pm_update_cores_state_nolock(kbdev); ++ ++ kbdev->pm.backend.ca_in_transition = false; ++ } ++} ++ ++/* ++ * The struct kbase_pm_ca_policy structure for the devfreq core availability ++ * policy. ++ * ++ * This is the static structure that defines the devfreq core availability power ++ * policy's callback and name. ++ */ ++const struct kbase_pm_ca_policy kbase_pm_ca_devfreq_policy_ops = { ++ "devfreq", /* name */ ++ devfreq_init, /* init */ ++ devfreq_term, /* term */ ++ devfreq_get_core_mask, /* get_core_mask */ ++ devfreq_update_core_status, /* update_core_status */ ++ 0u, /* flags */ ++ KBASE_PM_CA_POLICY_ID_DEVFREQ, /* id */ ++}; ++ +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca_devfreq.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca_devfreq.h +new file mode 100755 +index 000000000000..7ab3cd4d8460 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca_devfreq.h +@@ -0,0 +1,55 @@ ++/* ++ * ++ * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* ++ * A core availability policy for use with devfreq, where core masks are ++ * associated with OPPs. ++ */ ++ ++#ifndef MALI_KBASE_PM_CA_DEVFREQ_H ++#define MALI_KBASE_PM_CA_DEVFREQ_H ++ ++/** ++ * struct kbasep_pm_ca_policy_devfreq - Private structure for devfreq ca policy ++ * ++ * This contains data that is private to the devfreq core availability ++ * policy. ++ * ++ * @cores_desired: Cores that the policy wants to be available ++ * @cores_enabled: Cores that the policy is currently returning as available ++ * @cores_used: Cores currently powered or transitioning ++ */ ++struct kbasep_pm_ca_policy_devfreq { ++ u64 cores_desired; ++ u64 cores_enabled; ++ u64 cores_used; ++}; ++ ++extern const struct kbase_pm_ca_policy kbase_pm_ca_devfreq_policy_ops; ++ ++/** ++ * kbase_devfreq_set_core_mask - Set core mask for policy to use ++ * @kbdev: Device pointer ++ * @core_mask: New core mask ++ * ++ * The new core mask will have immediate effect if the GPU is powered, or will ++ * take effect when it is next powered on. ++ */ ++void kbase_devfreq_set_core_mask(struct kbase_device *kbdev, u64 core_mask); ++ ++#endif /* MALI_KBASE_PM_CA_DEVFREQ_H */ ++ +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca_fixed.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca_fixed.c +new file mode 100755 +index 000000000000..864612d31f9b +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca_fixed.c +@@ -0,0 +1,65 @@ ++/* ++ * ++ * (C) COPYRIGHT 2013-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* ++ * A power policy implementing fixed core availability ++ */ ++ ++#include ++#include ++ ++static void fixed_init(struct kbase_device *kbdev) ++{ ++ kbdev->pm.backend.ca_in_transition = false; ++} ++ ++static void fixed_term(struct kbase_device *kbdev) ++{ ++ CSTD_UNUSED(kbdev); ++} ++ ++static u64 fixed_get_core_mask(struct kbase_device *kbdev) ++{ ++ return kbdev->gpu_props.props.raw_props.shader_present; ++} ++ ++static void fixed_update_core_status(struct kbase_device *kbdev, ++ u64 cores_ready, ++ u64 cores_transitioning) ++{ ++ CSTD_UNUSED(kbdev); ++ CSTD_UNUSED(cores_ready); ++ CSTD_UNUSED(cores_transitioning); ++} ++ ++/* ++ * The struct kbase_pm_policy structure for the fixed power policy. ++ * ++ * This is the static structure that defines the fixed power policy's callback ++ * and name. ++ */ ++const struct kbase_pm_ca_policy kbase_pm_ca_fixed_policy_ops = { ++ "fixed", /* name */ ++ fixed_init, /* init */ ++ fixed_term, /* term */ ++ fixed_get_core_mask, /* get_core_mask */ ++ fixed_update_core_status, /* update_core_status */ ++ 0u, /* flags */ ++ KBASE_PM_CA_POLICY_ID_FIXED, /* id */ ++}; ++ ++KBASE_EXPORT_TEST_API(kbase_pm_ca_fixed_policy_ops); +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca_fixed.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca_fixed.h +new file mode 100755 +index 000000000000..a763155cb703 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca_fixed.h +@@ -0,0 +1,40 @@ ++/* ++ * ++ * (C) COPYRIGHT 2013-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* ++ * A power policy implementing fixed core availability ++ */ ++ ++#ifndef MALI_KBASE_PM_CA_FIXED_H ++#define MALI_KBASE_PM_CA_FIXED_H ++ ++/** ++ * struct kbasep_pm_ca_policy_fixed - Private structure for policy instance data ++ * ++ * @dummy: Dummy member - no state is needed ++ * ++ * This contains data that is private to the particular power policy that is ++ * active. ++ */ ++struct kbasep_pm_ca_policy_fixed { ++ int dummy; ++}; ++ ++extern const struct kbase_pm_ca_policy kbase_pm_ca_fixed_policy_ops; ++ ++#endif /* MALI_KBASE_PM_CA_FIXED_H */ ++ +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_coarse_demand.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_coarse_demand.c +new file mode 100755 +index 000000000000..f891fa225a89 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_coarse_demand.c +@@ -0,0 +1,70 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * "Coarse Demand" power management policy ++ */ ++ ++#include ++#include ++ ++static u64 coarse_demand_get_core_mask(struct kbase_device *kbdev) ++{ ++ if (kbdev->pm.active_count == 0) ++ return 0; ++ ++ return kbdev->gpu_props.props.raw_props.shader_present; ++} ++ ++static bool coarse_demand_get_core_active(struct kbase_device *kbdev) ++{ ++ if (0 == kbdev->pm.active_count && !(kbdev->shader_needed_bitmap | ++ kbdev->shader_inuse_bitmap) && !kbdev->tiler_needed_cnt ++ && !kbdev->tiler_inuse_cnt) ++ return false; ++ ++ return true; ++} ++ ++static void coarse_demand_init(struct kbase_device *kbdev) ++{ ++ CSTD_UNUSED(kbdev); ++} ++ ++static void coarse_demand_term(struct kbase_device *kbdev) ++{ ++ CSTD_UNUSED(kbdev); ++} ++ ++/* The struct kbase_pm_policy structure for the demand power policy. ++ * ++ * This is the static structure that defines the demand power policy's callback ++ * and name. ++ */ ++const struct kbase_pm_policy kbase_pm_coarse_demand_policy_ops = { ++ "coarse_demand", /* name */ ++ coarse_demand_init, /* init */ ++ coarse_demand_term, /* term */ ++ coarse_demand_get_core_mask, /* get_core_mask */ ++ coarse_demand_get_core_active, /* get_core_active */ ++ 0u, /* flags */ ++ KBASE_PM_POLICY_ID_COARSE_DEMAND, /* id */ ++}; ++ ++KBASE_EXPORT_TEST_API(kbase_pm_coarse_demand_policy_ops); +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_coarse_demand.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_coarse_demand.h +new file mode 100755 +index 000000000000..749d305eee9a +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_coarse_demand.h +@@ -0,0 +1,64 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * "Coarse Demand" power management policy ++ */ ++ ++#ifndef MALI_KBASE_PM_COARSE_DEMAND_H ++#define MALI_KBASE_PM_COARSE_DEMAND_H ++ ++/** ++ * DOC: ++ * The "Coarse" demand power management policy has the following ++ * characteristics: ++ * - When KBase indicates that the GPU will be powered up, but we don't yet ++ * know which Job Chains are to be run: ++ * - All Shader Cores are powered up, regardless of whether or not they will ++ * be needed later. ++ * - When KBase indicates that a set of Shader Cores are needed to submit the ++ * currently queued Job Chains: ++ * - All Shader Cores are kept powered, regardless of whether or not they will ++ * be needed ++ * - When KBase indicates that the GPU need not be powered: ++ * - The Shader Cores are powered off, and the GPU itself is powered off too. ++ * ++ * @note: ++ * - KBase indicates the GPU will be powered up when it has a User Process that ++ * has just started to submit Job Chains. ++ * - KBase indicates the GPU need not be powered when all the Job Chains from ++ * User Processes have finished, and it is waiting for a User Process to ++ * submit some more Job Chains. ++ */ ++ ++/** ++ * struct kbasep_pm_policy_coarse_demand - Private structure for coarse demand ++ * policy ++ * ++ * This contains data that is private to the coarse demand power policy. ++ * ++ * @dummy: Dummy member - no state needed ++ */ ++struct kbasep_pm_policy_coarse_demand { ++ int dummy; ++}; ++ ++extern const struct kbase_pm_policy kbase_pm_coarse_demand_policy_ops; ++ ++#endif /* MALI_KBASE_PM_COARSE_DEMAND_H */ +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_defs.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_defs.h +new file mode 100755 +index 000000000000..352744ee6d73 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_defs.h +@@ -0,0 +1,519 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* ++ * Backend-specific Power Manager definitions ++ */ ++ ++#ifndef _KBASE_PM_HWACCESS_DEFS_H_ ++#define _KBASE_PM_HWACCESS_DEFS_H_ ++ ++#include "mali_kbase_pm_ca_fixed.h" ++#include "mali_kbase_pm_ca_devfreq.h" ++#if !MALI_CUSTOMER_RELEASE ++#include "mali_kbase_pm_ca_random.h" ++#endif ++ ++#include "mali_kbase_pm_always_on.h" ++#include "mali_kbase_pm_coarse_demand.h" ++#include "mali_kbase_pm_demand.h" ++#if !MALI_CUSTOMER_RELEASE ++#include "mali_kbase_pm_demand_always_powered.h" ++#include "mali_kbase_pm_fast_start.h" ++#endif ++ ++/* Forward definition - see mali_kbase.h */ ++struct kbase_device; ++struct kbase_jd_atom; ++ ++/** ++ * enum kbase_pm_core_type - The types of core in a GPU. ++ * ++ * These enumerated values are used in calls to ++ * - kbase_pm_get_present_cores() ++ * - kbase_pm_get_active_cores() ++ * - kbase_pm_get_trans_cores() ++ * - kbase_pm_get_ready_cores(). ++ * ++ * They specify which type of core should be acted on. These values are set in ++ * a manner that allows core_type_to_reg() function to be simpler and more ++ * efficient. ++ * ++ * @KBASE_PM_CORE_L2: The L2 cache ++ * @KBASE_PM_CORE_SHADER: Shader cores ++ * @KBASE_PM_CORE_TILER: Tiler cores ++ * @KBASE_PM_CORE_STACK: Core stacks ++ */ ++enum kbase_pm_core_type { ++ KBASE_PM_CORE_L2 = L2_PRESENT_LO, ++ KBASE_PM_CORE_SHADER = SHADER_PRESENT_LO, ++ KBASE_PM_CORE_TILER = TILER_PRESENT_LO, ++ KBASE_PM_CORE_STACK = STACK_PRESENT_LO ++}; ++ ++/** ++ * struct kbasep_pm_metrics_data - Metrics data collected for use by the power ++ * management framework. ++ * ++ * @time_period_start: time at which busy/idle measurements started ++ * @time_busy: number of ns the GPU was busy executing jobs since the ++ * @time_period_start timestamp. ++ * @time_idle: number of ns since time_period_start the GPU was not executing ++ * jobs since the @time_period_start timestamp. ++ * @prev_busy: busy time in ns of previous time period. ++ * Updated when metrics are reset. ++ * @prev_idle: idle time in ns of previous time period ++ * Updated when metrics are reset. ++ * @gpu_active: true when the GPU is executing jobs. false when ++ * not. Updated when the job scheduler informs us a job in submitted ++ * or removed from a GPU slot. ++ * @busy_cl: number of ns the GPU was busy executing CL jobs. Note that ++ * if two CL jobs were active for 400ns, this value would be updated ++ * with 800. ++ * @busy_gl: number of ns the GPU was busy executing GL jobs. Note that ++ * if two GL jobs were active for 400ns, this value would be updated ++ * with 800. ++ * @active_cl_ctx: number of CL jobs active on the GPU. Array is per-device. ++ * @active_gl_ctx: number of GL jobs active on the GPU. Array is per-slot. As ++ * GL jobs never run on slot 2 this slot is not recorded. ++ * @lock: spinlock protecting the kbasep_pm_metrics_data structure ++ * @timer: timer to regularly make DVFS decisions based on the power ++ * management metrics. ++ * @timer_active: boolean indicating @timer is running ++ * @platform_data: pointer to data controlled by platform specific code ++ * @kbdev: pointer to kbase device for which metrics are collected ++ * ++ */ ++struct kbasep_pm_metrics_data { ++ ktime_t time_period_start; ++ u32 time_busy; ++ u32 time_idle; ++ u32 prev_busy; ++ u32 prev_idle; ++ bool gpu_active; ++ u32 busy_cl[2]; ++ u32 busy_gl; ++ u32 active_cl_ctx[2]; ++ u32 active_gl_ctx[2]; /* GL jobs can only run on 2 of the 3 job slots */ ++ spinlock_t lock; ++ ++#ifdef CONFIG_MALI_MIDGARD_DVFS ++ struct hrtimer timer; ++ bool timer_active; ++#endif ++ ++ void *platform_data; ++ struct kbase_device *kbdev; ++}; ++ ++union kbase_pm_policy_data { ++ struct kbasep_pm_policy_always_on always_on; ++ struct kbasep_pm_policy_coarse_demand coarse_demand; ++ struct kbasep_pm_policy_demand demand; ++#if !MALI_CUSTOMER_RELEASE ++ struct kbasep_pm_policy_demand_always_powered demand_always_powered; ++ struct kbasep_pm_policy_fast_start fast_start; ++#endif ++}; ++ ++union kbase_pm_ca_policy_data { ++ struct kbasep_pm_ca_policy_fixed fixed; ++ struct kbasep_pm_ca_policy_devfreq devfreq; ++#if !MALI_CUSTOMER_RELEASE ++ struct kbasep_pm_ca_policy_random random; ++#endif ++}; ++ ++/** ++ * struct kbase_pm_backend_data - Data stored per device for power management. ++ * ++ * This structure contains data for the power management framework. There is one ++ * instance of this structure per device in the system. ++ * ++ * @ca_current_policy: The policy that is currently actively controlling core ++ * availability. ++ * @pm_current_policy: The policy that is currently actively controlling the ++ * power state. ++ * @ca_policy_data: Private data for current CA policy ++ * @pm_policy_data: Private data for current PM policy ++ * @ca_in_transition: Flag indicating when core availability policy is ++ * transitioning cores. The core availability policy must ++ * set this when a change in core availability is occurring. ++ * power_change_lock must be held when accessing this. ++ * @reset_done: Flag when a reset is complete ++ * @reset_done_wait: Wait queue to wait for changes to @reset_done ++ * @l2_powered_wait: Wait queue for whether the l2 cache has been powered as ++ * requested ++ * @l2_powered: State indicating whether all the l2 caches are powered. ++ * Non-zero indicates they're *all* powered ++ * Zero indicates that some (or all) are not powered ++ * @gpu_cycle_counter_requests: The reference count of active gpu cycle counter ++ * users ++ * @gpu_cycle_counter_requests_lock: Lock to protect @gpu_cycle_counter_requests ++ * @desired_shader_state: A bit mask identifying the shader cores that the ++ * power policy would like to be on. The current state ++ * of the cores may be different, but there should be ++ * transitions in progress that will eventually achieve ++ * this state (assuming that the policy doesn't change ++ * its mind in the mean time). ++ * @powering_on_shader_state: A bit mask indicating which shader cores are ++ * currently in a power-on transition ++ * @desired_tiler_state: A bit mask identifying the tiler cores that the power ++ * policy would like to be on. See @desired_shader_state ++ * @powering_on_tiler_state: A bit mask indicating which tiler core are ++ * currently in a power-on transition ++ * @powering_on_l2_state: A bit mask indicating which l2-caches are currently ++ * in a power-on transition ++ * @powering_on_stack_state: A bit mask indicating which core stacks are ++ * currently in a power-on transition ++ * @gpu_in_desired_state: This flag is set if the GPU is powered as requested ++ * by the desired_xxx_state variables ++ * @gpu_in_desired_state_wait: Wait queue set when @gpu_in_desired_state != 0 ++ * @gpu_powered: Set to true when the GPU is powered and register ++ * accesses are possible, false otherwise ++ * @instr_enabled: Set to true when instrumentation is enabled, ++ * false otherwise ++ * @cg1_disabled: Set if the policy wants to keep the second core group ++ * powered off ++ * @driver_ready_for_irqs: Debug state indicating whether sufficient ++ * initialization of the driver has occurred to handle ++ * IRQs ++ * @gpu_powered_lock: Spinlock that must be held when writing @gpu_powered or ++ * accessing @driver_ready_for_irqs ++ * @metrics: Structure to hold metrics for the GPU ++ * @gpu_poweroff_pending: number of poweroff timer ticks until the GPU is ++ * powered off ++ * @shader_poweroff_pending_time: number of poweroff timer ticks until shaders ++ * and/or timers are powered off ++ * @gpu_poweroff_timer: Timer for powering off GPU ++ * @gpu_poweroff_wq: Workqueue to power off GPU on when timer fires ++ * @gpu_poweroff_work: Workitem used on @gpu_poweroff_wq ++ * @shader_poweroff_pending: Bit mask of shaders to be powered off on next ++ * timer callback ++ * @tiler_poweroff_pending: Bit mask of tilers to be powered off on next timer ++ * callback ++ * @poweroff_timer_needed: true if the poweroff timer is currently required, ++ * false otherwise ++ * @poweroff_timer_running: true if the poweroff timer is currently running, ++ * false otherwise ++ * power_change_lock should be held when accessing, ++ * unless there is no way the timer can be running (eg ++ * hrtimer_cancel() was called immediately before) ++ * @poweroff_wait_in_progress: true if a wait for GPU power off is in progress. ++ * hwaccess_lock must be held when accessing ++ * @poweron_required: true if a GPU power on is required. Should only be set ++ * when poweroff_wait_in_progress is true, and therefore the ++ * GPU can not immediately be powered on. pm.lock must be ++ * held when accessing ++ * @poweroff_is_suspend: true if the GPU is being powered off due to a suspend ++ * request. pm.lock must be held when accessing ++ * @gpu_poweroff_wait_wq: workqueue for waiting for GPU to power off ++ * @gpu_poweroff_wait_work: work item for use with @gpu_poweroff_wait_wq ++ * @poweroff_wait: waitqueue for waiting for @gpu_poweroff_wait_work to complete ++ * @callback_power_on: Callback when the GPU needs to be turned on. See ++ * &struct kbase_pm_callback_conf ++ * @callback_power_off: Callback when the GPU may be turned off. See ++ * &struct kbase_pm_callback_conf ++ * @callback_power_suspend: Callback when a suspend occurs and the GPU needs to ++ * be turned off. See &struct kbase_pm_callback_conf ++ * @callback_power_resume: Callback when a resume occurs and the GPU needs to ++ * be turned on. See &struct kbase_pm_callback_conf ++ * @callback_power_runtime_on: Callback when the GPU needs to be turned on. See ++ * &struct kbase_pm_callback_conf ++ * @callback_power_runtime_off: Callback when the GPU may be turned off. See ++ * &struct kbase_pm_callback_conf ++ * @callback_power_runtime_idle: Optional callback when the GPU may be idle. See ++ * &struct kbase_pm_callback_conf ++ * ++ * Note: ++ * During an IRQ, @ca_current_policy or @pm_current_policy can be NULL when the ++ * policy is being changed with kbase_pm_ca_set_policy() or ++ * kbase_pm_set_policy(). The change is protected under ++ * kbase_device.pm.power_change_lock. Direct access to this ++ * from IRQ context must therefore check for NULL. If NULL, then ++ * kbase_pm_ca_set_policy() or kbase_pm_set_policy() will re-issue the policy ++ * functions that would have been done under IRQ. ++ */ ++struct kbase_pm_backend_data { ++ const struct kbase_pm_ca_policy *ca_current_policy; ++ const struct kbase_pm_policy *pm_current_policy; ++ union kbase_pm_ca_policy_data ca_policy_data; ++ union kbase_pm_policy_data pm_policy_data; ++ bool ca_in_transition; ++ bool reset_done; ++ wait_queue_head_t reset_done_wait; ++ wait_queue_head_t l2_powered_wait; ++ int l2_powered; ++ int gpu_cycle_counter_requests; ++ spinlock_t gpu_cycle_counter_requests_lock; ++ ++ u64 desired_shader_state; ++ u64 powering_on_shader_state; ++ u64 desired_tiler_state; ++ u64 powering_on_tiler_state; ++ u64 powering_on_l2_state; ++#ifdef CONFIG_MALI_CORESTACK ++ u64 powering_on_stack_state; ++#endif /* CONFIG_MALI_CORESTACK */ ++ ++ bool gpu_in_desired_state; ++ wait_queue_head_t gpu_in_desired_state_wait; ++ ++ bool gpu_powered; ++ ++ bool instr_enabled; ++ ++ bool cg1_disabled; ++ ++#ifdef CONFIG_MALI_DEBUG ++ bool driver_ready_for_irqs; ++#endif /* CONFIG_MALI_DEBUG */ ++ ++ spinlock_t gpu_powered_lock; ++ ++ ++ struct kbasep_pm_metrics_data metrics; ++ ++ int gpu_poweroff_pending; ++ int shader_poweroff_pending_time; ++ ++ struct hrtimer gpu_poweroff_timer; ++ struct workqueue_struct *gpu_poweroff_wq; ++ struct work_struct gpu_poweroff_work; ++ ++ u64 shader_poweroff_pending; ++ u64 tiler_poweroff_pending; ++ ++ bool poweroff_timer_needed; ++ bool poweroff_timer_running; ++ ++ bool poweroff_wait_in_progress; ++ bool poweron_required; ++ bool poweroff_is_suspend; ++ ++ struct workqueue_struct *gpu_poweroff_wait_wq; ++ struct work_struct gpu_poweroff_wait_work; ++ ++ wait_queue_head_t poweroff_wait; ++ ++ int (*callback_power_on)(struct kbase_device *kbdev); ++ void (*callback_power_off)(struct kbase_device *kbdev); ++ void (*callback_power_suspend)(struct kbase_device *kbdev); ++ void (*callback_power_resume)(struct kbase_device *kbdev); ++ int (*callback_power_runtime_on)(struct kbase_device *kbdev); ++ void (*callback_power_runtime_off)(struct kbase_device *kbdev); ++ int (*callback_power_runtime_idle)(struct kbase_device *kbdev); ++}; ++ ++ ++/* List of policy IDs */ ++enum kbase_pm_policy_id { ++ KBASE_PM_POLICY_ID_DEMAND = 1, ++ KBASE_PM_POLICY_ID_ALWAYS_ON, ++ KBASE_PM_POLICY_ID_COARSE_DEMAND, ++#if !MALI_CUSTOMER_RELEASE ++ KBASE_PM_POLICY_ID_DEMAND_ALWAYS_POWERED, ++ KBASE_PM_POLICY_ID_FAST_START ++#endif ++}; ++ ++typedef u32 kbase_pm_policy_flags; ++ ++/** ++ * struct kbase_pm_policy - Power policy structure. ++ * ++ * Each power policy exposes a (static) instance of this structure which ++ * contains function pointers to the policy's methods. ++ * ++ * @name: The name of this policy ++ * @init: Function called when the policy is selected ++ * @term: Function called when the policy is unselected ++ * @get_core_mask: Function called to get the current shader core mask ++ * @get_core_active: Function called to get the current overall GPU power ++ * state ++ * @flags: Field indicating flags for this policy ++ * @id: Field indicating an ID for this policy. This is not ++ * necessarily the same as its index in the list returned ++ * by kbase_pm_list_policies(). ++ * It is used purely for debugging. ++ */ ++struct kbase_pm_policy { ++ char *name; ++ ++ /** ++ * Function called when the policy is selected ++ * ++ * This should initialize the kbdev->pm.pm_policy_data structure. It ++ * should not attempt to make any changes to hardware state. ++ * ++ * It is undefined what state the cores are in when the function is ++ * called. ++ * ++ * @kbdev: The kbase device structure for the device (must be a ++ * valid pointer) ++ */ ++ void (*init)(struct kbase_device *kbdev); ++ ++ /** ++ * Function called when the policy is unselected. ++ * ++ * @kbdev: The kbase device structure for the device (must be a ++ * valid pointer) ++ */ ++ void (*term)(struct kbase_device *kbdev); ++ ++ /** ++ * Function called to get the current shader core mask ++ * ++ * The returned mask should meet or exceed (kbdev->shader_needed_bitmap ++ * | kbdev->shader_inuse_bitmap). ++ * ++ * @kbdev: The kbase device structure for the device (must be a ++ * valid pointer) ++ * ++ * Return: The mask of shader cores to be powered ++ */ ++ u64 (*get_core_mask)(struct kbase_device *kbdev); ++ ++ /** ++ * Function called to get the current overall GPU power state ++ * ++ * This function should consider the state of kbdev->pm.active_count. If ++ * this count is greater than 0 then there is at least one active ++ * context on the device and the GPU should be powered. If it is equal ++ * to 0 then there are no active contexts and the GPU could be powered ++ * off if desired. ++ * ++ * @kbdev: The kbase device structure for the device (must be a ++ * valid pointer) ++ * ++ * Return: true if the GPU should be powered, false otherwise ++ */ ++ bool (*get_core_active)(struct kbase_device *kbdev); ++ ++ kbase_pm_policy_flags flags; ++ enum kbase_pm_policy_id id; ++}; ++ ++ ++enum kbase_pm_ca_policy_id { ++ KBASE_PM_CA_POLICY_ID_FIXED = 1, ++ KBASE_PM_CA_POLICY_ID_DEVFREQ, ++ KBASE_PM_CA_POLICY_ID_RANDOM ++}; ++ ++typedef u32 kbase_pm_ca_policy_flags; ++ ++/** ++ * Maximum length of a CA policy names ++ */ ++#define KBASE_PM_CA_MAX_POLICY_NAME_LEN 15 ++ ++/** ++ * struct kbase_pm_ca_policy - Core availability policy structure. ++ * ++ * Each core availability policy exposes a (static) instance of this structure ++ * which contains function pointers to the policy's methods. ++ * ++ * @name: The name of this policy ++ * @init: Function called when the policy is selected ++ * @term: Function called when the policy is unselected ++ * @get_core_mask: Function called to get the current shader core ++ * availability mask ++ * @update_core_status: Function called to update the current core status ++ * @flags: Field indicating flags for this policy ++ * @id: Field indicating an ID for this policy. This is not ++ * necessarily the same as its index in the list returned ++ * by kbase_pm_list_policies(). ++ * It is used purely for debugging. ++ */ ++struct kbase_pm_ca_policy { ++ char name[KBASE_PM_CA_MAX_POLICY_NAME_LEN + 1]; ++ ++ /** ++ * Function called when the policy is selected ++ * ++ * This should initialize the kbdev->pm.ca_policy_data structure. It ++ * should not attempt to make any changes to hardware state. ++ * ++ * It is undefined what state the cores are in when the function is ++ * called. ++ * ++ * @kbdev The kbase device structure for the device (must be a ++ * valid pointer) ++ */ ++ void (*init)(struct kbase_device *kbdev); ++ ++ /** ++ * Function called when the policy is unselected. ++ * ++ * @kbdev The kbase device structure for the device (must be a ++ * valid pointer) ++ */ ++ void (*term)(struct kbase_device *kbdev); ++ ++ /** ++ * Function called to get the current shader core availability mask ++ * ++ * When a change in core availability is occurring, the policy must set ++ * kbdev->pm.ca_in_transition to true. This is to indicate that ++ * reporting changes in power state cannot be optimized out, even if ++ * kbdev->pm.desired_shader_state remains unchanged. This must be done ++ * by any functions internal to the Core Availability Policy that change ++ * the return value of kbase_pm_ca_policy::get_core_mask. ++ * ++ * @kbdev The kbase device structure for the device (must be a ++ * valid pointer) ++ * ++ * Return: The current core availability mask ++ */ ++ u64 (*get_core_mask)(struct kbase_device *kbdev); ++ ++ /** ++ * Function called to update the current core status ++ * ++ * If none of the cores in core group 0 are ready or transitioning, then ++ * the policy must ensure that the next call to get_core_mask does not ++ * return 0 for all cores in core group 0. It is an error to disable ++ * core group 0 through the core availability policy. ++ * ++ * When a change in core availability has finished, the policy must set ++ * kbdev->pm.ca_in_transition to false. This is to indicate that ++ * changes in power state can once again be optimized out when ++ * kbdev->pm.desired_shader_state is unchanged. ++ * ++ * @kbdev: The kbase device structure for the device ++ * (must be a valid pointer) ++ * @cores_ready: The mask of cores currently powered and ++ * ready to run jobs ++ * @cores_transitioning: The mask of cores currently transitioning ++ * power state ++ */ ++ void (*update_core_status)(struct kbase_device *kbdev, u64 cores_ready, ++ u64 cores_transitioning); ++ ++ kbase_pm_ca_policy_flags flags; ++ ++ /** ++ * Field indicating an ID for this policy. This is not necessarily the ++ * same as its index in the list returned by kbase_pm_list_policies(). ++ * It is used purely for debugging. ++ */ ++ enum kbase_pm_ca_policy_id id; ++}; ++ ++#endif /* _KBASE_PM_HWACCESS_DEFS_H_ */ +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_demand.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_demand.c +new file mode 100755 +index 000000000000..81322fd0dd17 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_demand.c +@@ -0,0 +1,73 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * A simple demand based power management policy ++ */ ++ ++#include ++#include ++ ++static u64 demand_get_core_mask(struct kbase_device *kbdev) ++{ ++ u64 desired = kbdev->shader_needed_bitmap | kbdev->shader_inuse_bitmap; ++ ++ if (0 == kbdev->pm.active_count) ++ return 0; ++ ++ return desired; ++} ++ ++static bool demand_get_core_active(struct kbase_device *kbdev) ++{ ++ if (0 == kbdev->pm.active_count && !(kbdev->shader_needed_bitmap | ++ kbdev->shader_inuse_bitmap) && !kbdev->tiler_needed_cnt ++ && !kbdev->tiler_inuse_cnt) ++ return false; ++ ++ return true; ++} ++ ++static void demand_init(struct kbase_device *kbdev) ++{ ++ CSTD_UNUSED(kbdev); ++} ++ ++static void demand_term(struct kbase_device *kbdev) ++{ ++ CSTD_UNUSED(kbdev); ++} ++ ++/* ++ * The struct kbase_pm_policy structure for the demand power policy. ++ * ++ * This is the static structure that defines the demand power policy's callback ++ * and name. ++ */ ++const struct kbase_pm_policy kbase_pm_demand_policy_ops = { ++ "demand", /* name */ ++ demand_init, /* init */ ++ demand_term, /* term */ ++ demand_get_core_mask, /* get_core_mask */ ++ demand_get_core_active, /* get_core_active */ ++ 0u, /* flags */ ++ KBASE_PM_POLICY_ID_DEMAND, /* id */ ++}; ++ ++KBASE_EXPORT_TEST_API(kbase_pm_demand_policy_ops); +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_demand.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_demand.h +new file mode 100755 +index 000000000000..c0c84b6e9189 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_demand.h +@@ -0,0 +1,64 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * A simple demand based power management policy ++ */ ++ ++#ifndef MALI_KBASE_PM_DEMAND_H ++#define MALI_KBASE_PM_DEMAND_H ++ ++/** ++ * DOC: Demand power management policy ++ * ++ * The demand power management policy has the following characteristics: ++ * - When KBase indicates that the GPU will be powered up, but we don't yet ++ * know which Job Chains are to be run: ++ * - The Shader Cores are not powered up ++ * ++ * - When KBase indicates that a set of Shader Cores are needed to submit the ++ * currently queued Job Chains: ++ * - Only those Shader Cores are powered up ++ * ++ * - When KBase indicates that the GPU need not be powered: ++ * - The Shader Cores are powered off, and the GPU itself is powered off too. ++ * ++ * Note: ++ * - KBase indicates the GPU will be powered up when it has a User Process that ++ * has just started to submit Job Chains. ++ * ++ * - KBase indicates the GPU need not be powered when all the Job Chains from ++ * User Processes have finished, and it is waiting for a User Process to ++ * submit some more Job Chains. ++ */ ++ ++/** ++ * struct kbasep_pm_policy_demand - Private structure for policy instance data ++ * ++ * @dummy: No state is needed, a dummy variable ++ * ++ * This contains data that is private to the demand power policy. ++ */ ++struct kbasep_pm_policy_demand { ++ int dummy; ++}; ++ ++extern const struct kbase_pm_policy kbase_pm_demand_policy_ops; ++ ++#endif /* MALI_KBASE_PM_DEMAND_H */ +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_driver.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_driver.c +new file mode 100755 +index 000000000000..82727937c545 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_driver.c +@@ -0,0 +1,1713 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * Base kernel Power Management hardware control ++ */ ++ ++// #define ENABLE_DEBUG_LOG ++#include "../../platform/rk/custom_log.h" ++ ++#include ++#include ++#include ++#if defined(CONFIG_MALI_GATOR_SUPPORT) ++#include ++#endif ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++ ++#if MALI_MOCK_TEST ++#define MOCKABLE(function) function##_original ++#else ++#define MOCKABLE(function) function ++#endif /* MALI_MOCK_TEST */ ++ ++/** ++ * enum kbasep_pm_action - Actions that can be performed on a core. ++ * ++ * This enumeration is private to the file. Its values are set to allow ++ * core_type_to_reg() function, which decodes this enumeration, to be simpler ++ * and more efficient. ++ * ++ * @ACTION_PRESENT: The cores that are present ++ * @ACTION_READY: The cores that are ready ++ * @ACTION_PWRON: Power on the cores specified ++ * @ACTION_PWROFF: Power off the cores specified ++ * @ACTION_PWRTRANS: The cores that are transitioning ++ * @ACTION_PWRACTIVE: The cores that are active ++ */ ++enum kbasep_pm_action { ++ ACTION_PRESENT = 0, ++ ACTION_READY = (SHADER_READY_LO - SHADER_PRESENT_LO), ++ ACTION_PWRON = (SHADER_PWRON_LO - SHADER_PRESENT_LO), ++ ACTION_PWROFF = (SHADER_PWROFF_LO - SHADER_PRESENT_LO), ++ ACTION_PWRTRANS = (SHADER_PWRTRANS_LO - SHADER_PRESENT_LO), ++ ACTION_PWRACTIVE = (SHADER_PWRACTIVE_LO - SHADER_PRESENT_LO) ++}; ++ ++/*---------------------------------------------------------------------------*/ ++ ++static bool is_action_of_powering_off_l2(enum kbase_pm_core_type core_type, ++ enum kbasep_pm_action active) ++{ ++ return (KBASE_PM_CORE_L2 == core_type) && (ACTION_PWROFF == active); ++} ++ ++static bool is_action_of_powering_off_shader(enum kbase_pm_core_type core_type, ++ enum kbasep_pm_action active) ++{ ++ return (KBASE_PM_CORE_SHADER == core_type) && (ACTION_PWROFF == active); ++} ++ ++static bool is_action_of_powering_off_tiler(enum kbase_pm_core_type core_type, ++ enum kbasep_pm_action active) ++{ ++ return (KBASE_PM_CORE_TILER == core_type) && (ACTION_PWROFF == active); ++} ++ ++static u64 kbase_pm_get_state( ++ struct kbase_device *kbdev, ++ enum kbase_pm_core_type core_type, ++ enum kbasep_pm_action action); ++ ++/** ++ * core_type_to_reg - Decode a core type and action to a register. ++ * ++ * Given a core type (defined by kbase_pm_core_type) and an action (defined ++ * by kbasep_pm_action) this function will return the register offset that ++ * will perform the action on the core type. The register returned is the _LO ++ * register and an offset must be applied to use the _HI register. ++ * ++ * @core_type: The type of core ++ * @action: The type of action ++ * ++ * Return: The register offset of the _LO register that performs an action of ++ * type @action on a core of type @core_type. ++ */ ++static u32 core_type_to_reg(enum kbase_pm_core_type core_type, ++ enum kbasep_pm_action action) ++{ ++#ifdef CONFIG_MALI_CORESTACK ++ if (core_type == KBASE_PM_CORE_STACK) { ++ switch (action) { ++ case ACTION_PRESENT: ++ return STACK_PRESENT_LO; ++ case ACTION_READY: ++ return STACK_READY_LO; ++ case ACTION_PWRON: ++ return STACK_PWRON_LO; ++ case ACTION_PWROFF: ++ return STACK_PWROFF_LO; ++ case ACTION_PWRTRANS: ++ return STACK_PWRTRANS_LO; ++ default: ++ BUG(); ++ } ++ } ++#endif /* CONFIG_MALI_CORESTACK */ ++ ++ return (u32)core_type + (u32)action; ++} ++ ++#ifdef CONFIG_ARM64 ++static void mali_cci_flush_l2(struct kbase_device *kbdev) ++{ ++ const u32 mask = CLEAN_CACHES_COMPLETED | RESET_COMPLETED; ++ u32 loops = KBASE_CLEAN_CACHE_MAX_LOOPS; ++ u32 raw; ++ ++ /* ++ * Note that we don't take the cache flush mutex here since ++ * we expect to be the last user of the L2, all other L2 users ++ * would have dropped their references, to initiate L2 power ++ * down, L2 power down being the only valid place for this ++ * to be called from. ++ */ ++ ++ kbase_reg_write(kbdev, ++ GPU_CONTROL_REG(GPU_COMMAND), ++ GPU_COMMAND_CLEAN_INV_CACHES, ++ NULL); ++ ++ raw = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), ++ NULL); ++ ++ /* Wait for cache flush to complete before continuing, exit on ++ * gpu resets or loop expiry. */ ++ while (((raw & mask) == 0) && --loops) { ++ raw = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), ++ NULL); ++ } ++} ++#endif ++ ++/** ++ * kbase_pm_invoke - Invokes an action on a core set ++ * ++ * This function performs the action given by @action on a set of cores of a ++ * type given by @core_type. It is a static function used by ++ * kbase_pm_transition_core_type() ++ * ++ * @kbdev: The kbase device structure of the device ++ * @core_type: The type of core that the action should be performed on ++ * @cores: A bit mask of cores to perform the action on (low 32 bits) ++ * @action: The action to perform on the cores ++ */ ++static void kbase_pm_invoke(struct kbase_device *kbdev, ++ enum kbase_pm_core_type core_type, ++ u64 cores, ++ enum kbasep_pm_action action) ++{ ++ u32 reg; ++ u32 lo = cores & 0xFFFFFFFF; ++ u32 hi = (cores >> 32) & 0xFFFFFFFF; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ /*-------------------------------------------------------*/ ++ ++ if ( is_action_of_powering_off_l2(core_type, action) ) { ++ D("not to power off l2 actually."); ++ return; ++ } ++ if ( is_action_of_powering_off_shader(core_type, action) ) { ++ D("not to power off shader actually. cores_lo : 0x%x, hi : 0x%x.", ++ lo, ++ hi); ++ return; ++ } ++ if ( is_action_of_powering_off_tiler(core_type, action) ) { ++ D("not to power off tiler actually."); ++ return; ++ } ++ ++ /*-------------------------------------------------------*/ ++ ++ reg = core_type_to_reg(core_type, action); ++ ++ KBASE_DEBUG_ASSERT(reg); ++#if defined(CONFIG_MALI_GATOR_SUPPORT) ++ if (cores) { ++ if (action == ACTION_PWRON) ++ kbase_trace_mali_pm_power_on(core_type, cores); ++ else if (action == ACTION_PWROFF) ++ kbase_trace_mali_pm_power_off(core_type, cores); ++ } ++#endif ++ ++ if (cores) { ++ u64 state = kbase_pm_get_state(kbdev, core_type, ACTION_READY); ++ ++ if (action == ACTION_PWRON) ++ state |= cores; ++ else if (action == ACTION_PWROFF) ++ state &= ~cores; ++ KBASE_TLSTREAM_AUX_PM_STATE(core_type, state); ++ } ++ ++ /* Tracing */ ++ if (cores) { ++ if (action == ACTION_PWRON) ++ switch (core_type) { ++ case KBASE_PM_CORE_SHADER: ++ KBASE_TRACE_ADD(kbdev, PM_PWRON, NULL, NULL, 0u, ++ lo); ++ break; ++ case KBASE_PM_CORE_TILER: ++ KBASE_TRACE_ADD(kbdev, PM_PWRON_TILER, NULL, ++ NULL, 0u, lo); ++ break; ++ case KBASE_PM_CORE_L2: ++ KBASE_TRACE_ADD(kbdev, PM_PWRON_L2, NULL, NULL, ++ 0u, lo); ++ break; ++ default: ++ break; ++ } ++ else if (action == ACTION_PWROFF) ++ switch (core_type) { ++ case KBASE_PM_CORE_SHADER: ++ KBASE_TRACE_ADD(kbdev, PM_PWROFF, NULL, NULL, ++ 0u, lo); ++ break; ++ case KBASE_PM_CORE_TILER: ++ KBASE_TRACE_ADD(kbdev, PM_PWROFF_TILER, NULL, ++ NULL, 0u, lo); ++ break; ++ case KBASE_PM_CORE_L2: ++ KBASE_TRACE_ADD(kbdev, PM_PWROFF_L2, NULL, NULL, ++ 0u, lo); ++ /* disable snoops before L2 is turned off */ ++ kbase_pm_cache_snoop_disable(kbdev); ++ break; ++ default: ++ break; ++ } ++ } ++ ++ if (lo != 0) ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(reg), lo, NULL); ++ ++ if (hi != 0) ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(reg + 4), hi, NULL); ++} ++ ++/** ++ * kbase_pm_get_state - Get information about a core set ++ * ++ * This function gets information (chosen by @action) about a set of cores of ++ * a type given by @core_type. It is a static function used by ++ * kbase_pm_get_active_cores(), kbase_pm_get_trans_cores() and ++ * kbase_pm_get_ready_cores(). ++ * ++ * @kbdev: The kbase device structure of the device ++ * @core_type: The type of core that the should be queried ++ * @action: The property of the cores to query ++ * ++ * Return: A bit mask specifying the state of the cores ++ */ ++static u64 kbase_pm_get_state(struct kbase_device *kbdev, ++ enum kbase_pm_core_type core_type, ++ enum kbasep_pm_action action) ++{ ++ u32 reg; ++ u32 lo, hi; ++ ++ reg = core_type_to_reg(core_type, action); ++ ++ KBASE_DEBUG_ASSERT(reg); ++ ++ lo = kbase_reg_read(kbdev, GPU_CONTROL_REG(reg), NULL); ++ hi = kbase_reg_read(kbdev, GPU_CONTROL_REG(reg + 4), NULL); ++ ++ return (((u64) hi) << 32) | ((u64) lo); ++} ++ ++void kbasep_pm_init_core_use_bitmaps(struct kbase_device *kbdev) ++{ ++ kbdev->shader_inuse_bitmap = 0; ++ kbdev->shader_needed_bitmap = 0; ++ kbdev->shader_available_bitmap = 0; ++ kbdev->tiler_available_bitmap = 0; ++ kbdev->l2_users_count = 0; ++ kbdev->l2_available_bitmap = 0; ++ kbdev->tiler_needed_cnt = 0; ++ kbdev->tiler_inuse_cnt = 0; ++ ++ memset(kbdev->shader_needed_cnt, 0, sizeof(kbdev->shader_needed_cnt)); ++} ++ ++/** ++ * kbase_pm_get_present_cores - Get the cores that are present ++ * ++ * @kbdev: Kbase device ++ * @type: The type of cores to query ++ * ++ * Return: Bitmask of the cores that are present ++ */ ++u64 kbase_pm_get_present_cores(struct kbase_device *kbdev, ++ enum kbase_pm_core_type type) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ switch (type) { ++ case KBASE_PM_CORE_L2: ++ return kbdev->gpu_props.props.raw_props.l2_present; ++ case KBASE_PM_CORE_SHADER: ++ return kbdev->gpu_props.props.raw_props.shader_present; ++ case KBASE_PM_CORE_TILER: ++ return kbdev->gpu_props.props.raw_props.tiler_present; ++#ifdef CONFIG_MALI_CORESTACK ++ case KBASE_PM_CORE_STACK: ++ return kbdev->gpu_props.props.raw_props.stack_present; ++#endif /* CONFIG_MALI_CORESTACK */ ++ default: ++ break; ++ } ++ KBASE_DEBUG_ASSERT(0); ++ ++ return 0; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_get_present_cores); ++ ++/** ++ * kbase_pm_get_active_cores - Get the cores that are "active" ++ * (busy processing work) ++ * ++ * @kbdev: Kbase device ++ * @type: The type of cores to query ++ * ++ * Return: Bitmask of cores that are active ++ */ ++u64 kbase_pm_get_active_cores(struct kbase_device *kbdev, ++ enum kbase_pm_core_type type) ++{ ++ return kbase_pm_get_state(kbdev, type, ACTION_PWRACTIVE); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_get_active_cores); ++ ++/** ++ * kbase_pm_get_trans_cores - Get the cores that are transitioning between ++ * power states ++ * ++ * @kbdev: Kbase device ++ * @type: The type of cores to query ++ * ++ * Return: Bitmask of cores that are transitioning ++ */ ++u64 kbase_pm_get_trans_cores(struct kbase_device *kbdev, ++ enum kbase_pm_core_type type) ++{ ++ return kbase_pm_get_state(kbdev, type, ACTION_PWRTRANS); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_get_trans_cores); ++ ++/** ++ * kbase_pm_get_ready_cores - Get the cores that are powered on ++ * ++ * @kbdev: Kbase device ++ * @type: The type of cores to query ++ * ++ * Return: Bitmask of cores that are ready (powered on) ++ */ ++u64 kbase_pm_get_ready_cores(struct kbase_device *kbdev, ++ enum kbase_pm_core_type type) ++{ ++ u64 result; ++ ++ result = kbase_pm_get_state(kbdev, type, ACTION_READY); ++ ++ switch (type) { ++ case KBASE_PM_CORE_SHADER: ++ KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED, NULL, NULL, 0u, ++ (u32) result); ++ break; ++ case KBASE_PM_CORE_TILER: ++ KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED_TILER, NULL, NULL, 0u, ++ (u32) result); ++ break; ++ case KBASE_PM_CORE_L2: ++ KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED_L2, NULL, NULL, 0u, ++ (u32) result); ++ break; ++ default: ++ break; ++ } ++ ++ return result; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_get_ready_cores); ++ ++/** ++ * kbase_pm_transition_core_type - Perform power transitions for a particular ++ * core type. ++ * ++ * This function will perform any available power transitions to make the actual ++ * hardware state closer to the desired state. If a core is currently ++ * transitioning then changes to the power state of that call cannot be made ++ * until the transition has finished. Cores which are not present in the ++ * hardware are ignored if they are specified in the desired_state bitmask, ++ * however the return value will always be 0 in this case. ++ * ++ * @kbdev: The kbase device ++ * @type: The core type to perform transitions for ++ * @desired_state: A bit mask of the desired state of the cores ++ * @in_use: A bit mask of the cores that are currently running ++ * jobs. These cores have to be kept powered up because ++ * there are jobs running (or about to run) on them. ++ * @available: Receives a bit mask of the cores that the job ++ * scheduler can use to submit jobs to. May be NULL if ++ * this is not needed. ++ * @powering_on: Bit mask to update with cores that are ++ * transitioning to a power-on state. ++ * ++ * Return: true if the desired state has been reached, false otherwise ++ */ ++static bool kbase_pm_transition_core_type(struct kbase_device *kbdev, ++ enum kbase_pm_core_type type, ++ u64 desired_state, ++ u64 in_use, ++ u64 * const available, ++ u64 *powering_on) ++{ ++ u64 present; ++ u64 ready; ++ u64 trans; ++ u64 powerup; ++ u64 powerdown; ++ u64 powering_on_trans; ++ u64 desired_state_in_use; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ /* Get current state */ ++ present = kbase_pm_get_present_cores(kbdev, type); ++ trans = kbase_pm_get_trans_cores(kbdev, type); ++ ready = kbase_pm_get_ready_cores(kbdev, type); ++ /* mask off ready from trans in case transitions finished between the ++ * register reads */ ++ trans &= ~ready; ++ ++ if (trans) /* Do not progress if any cores are transitioning */ ++ return false; ++ ++ powering_on_trans = trans & *powering_on; ++ *powering_on = powering_on_trans; ++ ++ if (available != NULL) ++ *available = (ready | powering_on_trans) & desired_state; ++ ++ /* Update desired state to include the in-use cores. These have to be ++ * kept powered up because there are jobs running or about to run on ++ * these cores ++ */ ++ desired_state_in_use = desired_state | in_use; ++ ++ /* Update state of whether l2 caches are powered */ ++ if (type == KBASE_PM_CORE_L2) { ++ if ((ready == present) && (desired_state_in_use == ready) && ++ (trans == 0)) { ++ /* All are ready, none will be turned off, and none are ++ * transitioning */ ++ kbdev->pm.backend.l2_powered = 1; ++ /* ++ * Ensure snoops are enabled after L2 is powered up, ++ * note that kbase keeps track of the snoop state, so ++ * safe to repeatedly call. ++ */ ++ kbase_pm_cache_snoop_enable(kbdev); ++ if (kbdev->l2_users_count > 0) { ++ /* Notify any registered l2 cache users ++ * (optimized out when no users waiting) */ ++ wake_up(&kbdev->pm.backend.l2_powered_wait); ++ } ++ } else ++ kbdev->pm.backend.l2_powered = 0; ++ } ++ ++ if (desired_state == ready && (trans == 0)) ++ return true; ++ ++ /* Restrict the cores to those that are actually present */ ++ powerup = desired_state_in_use & present; ++ powerdown = (~desired_state_in_use) & present; ++ ++ /* Restrict to cores that are not already in the desired state */ ++ powerup &= ~ready; ++ powerdown &= ready; ++ ++ /* Don't transition any cores that are already transitioning, except for ++ * Mali cores that support the following case: ++ * ++ * If the SHADER_PWRON or TILER_PWRON registers are written to turn on ++ * a core that is currently transitioning to power off, then this is ++ * remembered and the shader core is automatically powered up again once ++ * the original transition completes. Once the automatic power on is ++ * complete any job scheduled on the shader core should start. ++ */ ++ powerdown &= ~trans; ++ ++ if (kbase_hw_has_feature(kbdev, ++ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS)) ++ if (KBASE_PM_CORE_SHADER == type || KBASE_PM_CORE_TILER == type) ++ trans = powering_on_trans; /* for exception cases, only ++ * mask off cores in power on ++ * transitions */ ++ ++ powerup &= ~trans; ++ ++ /* Perform transitions if any */ ++ kbase_pm_invoke(kbdev, type, powerup, ACTION_PWRON); ++#if !PLATFORM_POWER_DOWN_ONLY ++ kbase_pm_invoke(kbdev, type, powerdown, ACTION_PWROFF); ++#endif ++ ++ /* Recalculate cores transitioning on, and re-evaluate our state */ ++ powering_on_trans |= powerup; ++ *powering_on = powering_on_trans; ++ if (available != NULL) ++ *available = (ready | powering_on_trans) & desired_state; ++ ++ return false; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_transition_core_type); ++ ++/** ++ * get_desired_cache_status - Determine which caches should be on for a ++ * particular core state ++ * ++ * This function takes a bit mask of the present caches and the cores (or ++ * caches) that are attached to the caches that will be powered. It then ++ * computes which caches should be turned on to allow the cores requested to be ++ * powered up. ++ * ++ * @present: The bit mask of present caches ++ * @cores_powered: A bit mask of cores (or L2 caches) that are desired to ++ * be powered ++ * @tilers_powered: The bit mask of tilers that are desired to be powered ++ * ++ * Return: A bit mask of the caches that should be turned on ++ */ ++static u64 get_desired_cache_status(u64 present, u64 cores_powered, ++ u64 tilers_powered) ++{ ++ u64 desired = 0; ++ ++ while (present) { ++ /* Find out which is the highest set bit */ ++ u64 bit = fls64(present) - 1; ++ u64 bit_mask = 1ull << bit; ++ /* Create a mask which has all bits from 'bit' upwards set */ ++ ++ u64 mask = ~(bit_mask - 1); ++ ++ /* If there are any cores powered at this bit or above (that ++ * haven't previously been processed) then we need this core on ++ */ ++ if (cores_powered & mask) ++ desired |= bit_mask; ++ ++ /* Remove bits from cores_powered and present */ ++ cores_powered &= ~mask; ++ present &= ~bit_mask; ++ } ++ ++ /* Power up the required L2(s) for the tiler */ ++ if (tilers_powered) ++ desired |= 1; ++ ++ return desired; ++} ++ ++KBASE_EXPORT_TEST_API(get_desired_cache_status); ++ ++#ifdef CONFIG_MALI_CORESTACK ++u64 kbase_pm_core_stack_mask(u64 cores) ++{ ++ u64 stack_mask = 0; ++ size_t const MAX_CORE_ID = 31; ++ size_t const NUM_CORES_PER_STACK = 4; ++ size_t i; ++ ++ for (i = 0; i <= MAX_CORE_ID; ++i) { ++ if (test_bit(i, (unsigned long *)&cores)) { ++ /* Every core which ID >= 16 is filled to stacks 4-7 ++ * instead of 0-3 */ ++ size_t const stack_num = (i > 16) ? ++ (i % NUM_CORES_PER_STACK) + 4 : ++ (i % NUM_CORES_PER_STACK); ++ set_bit(stack_num, (unsigned long *)&stack_mask); ++ } ++ } ++ ++ return stack_mask; ++} ++#endif /* CONFIG_MALI_CORESTACK */ ++ ++bool ++MOCKABLE(kbase_pm_check_transitions_nolock) (struct kbase_device *kbdev) ++{ ++ bool cores_are_available = false; ++ bool in_desired_state = true; ++ u64 desired_l2_state; ++#ifdef CONFIG_MALI_CORESTACK ++ u64 desired_stack_state; ++ u64 stacks_powered; ++#endif /* CONFIG_MALI_CORESTACK */ ++ u64 cores_powered; ++ u64 tilers_powered; ++ u64 tiler_available_bitmap; ++ u64 tiler_transitioning_bitmap; ++ u64 shader_available_bitmap; ++ u64 shader_ready_bitmap; ++ u64 shader_transitioning_bitmap; ++ u64 l2_available_bitmap; ++ u64 prev_l2_available_bitmap; ++ u64 l2_inuse_bitmap; ++ ++ KBASE_DEBUG_ASSERT(NULL != kbdev); ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ spin_lock(&kbdev->pm.backend.gpu_powered_lock); ++ if (kbdev->pm.backend.gpu_powered == false) { ++ spin_unlock(&kbdev->pm.backend.gpu_powered_lock); ++ if (kbdev->pm.backend.desired_shader_state == 0 && ++ kbdev->pm.backend.desired_tiler_state == 0) ++ return true; ++ return false; ++ } ++ ++ /* Trace that a change-state is being requested, and that it took ++ * (effectively) no time to start it. This is useful for counting how ++ * many state changes occurred, in a way that's backwards-compatible ++ * with processing the trace data */ ++ kbase_timeline_pm_send_event(kbdev, ++ KBASE_TIMELINE_PM_EVENT_CHANGE_GPU_STATE); ++ kbase_timeline_pm_handle_event(kbdev, ++ KBASE_TIMELINE_PM_EVENT_CHANGE_GPU_STATE); ++ ++ /* If any cores are already powered then, we must keep the caches on */ ++ shader_transitioning_bitmap = kbase_pm_get_trans_cores(kbdev, ++ KBASE_PM_CORE_SHADER); ++ cores_powered = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER); ++ cores_powered |= kbdev->pm.backend.desired_shader_state; ++ ++#ifdef CONFIG_MALI_CORESTACK ++ /* Work out which core stacks want to be powered */ ++ desired_stack_state = kbase_pm_core_stack_mask(cores_powered); ++ stacks_powered = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_STACK) | ++ desired_stack_state; ++#endif /* CONFIG_MALI_CORESTACK */ ++ ++ /* Work out which tilers want to be powered */ ++ tiler_transitioning_bitmap = kbase_pm_get_trans_cores(kbdev, ++ KBASE_PM_CORE_TILER); ++ tilers_powered = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_TILER); ++ tilers_powered |= kbdev->pm.backend.desired_tiler_state; ++ ++ /* If there are l2 cache users registered, keep all l2s powered even if ++ * all other cores are off. */ ++ if (kbdev->l2_users_count > 0) ++ cores_powered |= kbdev->gpu_props.props.raw_props.l2_present; ++ ++ desired_l2_state = get_desired_cache_status( ++ kbdev->gpu_props.props.raw_props.l2_present, ++ cores_powered, tilers_powered); ++ ++ l2_inuse_bitmap = get_desired_cache_status( ++ kbdev->gpu_props.props.raw_props.l2_present, ++ cores_powered | shader_transitioning_bitmap, ++ tilers_powered | tiler_transitioning_bitmap); ++ ++#ifdef CONFIG_MALI_CORESTACK ++ if (stacks_powered) ++ desired_l2_state |= 1; ++#endif /* CONFIG_MALI_CORESTACK */ ++ ++ /* If any l2 cache is on, then enable l2 #0, for use by job manager */ ++ if (0 != desired_l2_state) ++ desired_l2_state |= 1; ++ ++ prev_l2_available_bitmap = kbdev->l2_available_bitmap; ++ in_desired_state &= kbase_pm_transition_core_type(kbdev, ++ KBASE_PM_CORE_L2, desired_l2_state, l2_inuse_bitmap, ++ &l2_available_bitmap, ++ &kbdev->pm.backend.powering_on_l2_state); ++ ++ if (kbdev->l2_available_bitmap != l2_available_bitmap) ++ KBASE_TIMELINE_POWER_L2(kbdev, l2_available_bitmap); ++ ++ kbdev->l2_available_bitmap = l2_available_bitmap; ++ ++ ++#ifdef CONFIG_MALI_CORESTACK ++ if (in_desired_state) { ++ in_desired_state &= kbase_pm_transition_core_type(kbdev, ++ KBASE_PM_CORE_STACK, desired_stack_state, 0, ++ &kbdev->stack_available_bitmap, ++ &kbdev->pm.backend.powering_on_stack_state); ++ } ++#endif /* CONFIG_MALI_CORESTACK */ ++ ++ if (in_desired_state) { ++ in_desired_state &= kbase_pm_transition_core_type(kbdev, ++ KBASE_PM_CORE_TILER, ++ kbdev->pm.backend.desired_tiler_state, ++ 0, &tiler_available_bitmap, ++ &kbdev->pm.backend.powering_on_tiler_state); ++ in_desired_state &= kbase_pm_transition_core_type(kbdev, ++ KBASE_PM_CORE_SHADER, ++ kbdev->pm.backend.desired_shader_state, ++ kbdev->shader_inuse_bitmap, ++ &shader_available_bitmap, ++ &kbdev->pm.backend.powering_on_shader_state); ++ ++ if (kbdev->shader_available_bitmap != shader_available_bitmap) { ++ KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, ++ NULL, 0u, ++ (u32) shader_available_bitmap); ++ KBASE_TIMELINE_POWER_SHADER(kbdev, ++ shader_available_bitmap); ++ } ++ ++ kbdev->shader_available_bitmap = shader_available_bitmap; ++ ++ if (kbdev->tiler_available_bitmap != tiler_available_bitmap) { ++ KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER, ++ NULL, NULL, 0u, ++ (u32) tiler_available_bitmap); ++ KBASE_TIMELINE_POWER_TILER(kbdev, ++ tiler_available_bitmap); ++ } ++ ++ kbdev->tiler_available_bitmap = tiler_available_bitmap; ++ ++ } else if ((l2_available_bitmap & ++ kbdev->gpu_props.props.raw_props.tiler_present) != ++ kbdev->gpu_props.props.raw_props.tiler_present) { ++ tiler_available_bitmap = 0; ++ ++ if (kbdev->tiler_available_bitmap != tiler_available_bitmap) ++ KBASE_TIMELINE_POWER_TILER(kbdev, ++ tiler_available_bitmap); ++ ++ kbdev->tiler_available_bitmap = tiler_available_bitmap; ++ } ++ ++ /* State updated for slow-path waiters */ ++ kbdev->pm.backend.gpu_in_desired_state = in_desired_state; ++ ++ shader_ready_bitmap = kbase_pm_get_ready_cores(kbdev, ++ KBASE_PM_CORE_SHADER); ++ shader_transitioning_bitmap = kbase_pm_get_trans_cores(kbdev, ++ KBASE_PM_CORE_SHADER); ++ ++ /* Determine whether the cores are now available (even if the set of ++ * available cores is empty). Note that they can be available even if ++ * we've not finished transitioning to the desired state */ ++ if ((kbdev->shader_available_bitmap & ++ kbdev->pm.backend.desired_shader_state) ++ == kbdev->pm.backend.desired_shader_state && ++ (kbdev->tiler_available_bitmap & ++ kbdev->pm.backend.desired_tiler_state) ++ == kbdev->pm.backend.desired_tiler_state) { ++ cores_are_available = true; ++ ++ KBASE_TRACE_ADD(kbdev, PM_CORES_AVAILABLE, NULL, NULL, 0u, ++ (u32)(kbdev->shader_available_bitmap & ++ kbdev->pm.backend.desired_shader_state)); ++ KBASE_TRACE_ADD(kbdev, PM_CORES_AVAILABLE_TILER, NULL, NULL, 0u, ++ (u32)(kbdev->tiler_available_bitmap & ++ kbdev->pm.backend.desired_tiler_state)); ++ ++ /* Log timelining information about handling events that power ++ * up cores, to match up either with immediate submission either ++ * because cores already available, or from PM IRQ */ ++ if (!in_desired_state) ++ kbase_timeline_pm_send_event(kbdev, ++ KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED); ++ } ++ ++ if (in_desired_state) { ++ KBASE_DEBUG_ASSERT(cores_are_available); ++ ++#if defined(CONFIG_MALI_GATOR_SUPPORT) ++ kbase_trace_mali_pm_status(KBASE_PM_CORE_L2, ++ kbase_pm_get_ready_cores(kbdev, ++ KBASE_PM_CORE_L2)); ++ kbase_trace_mali_pm_status(KBASE_PM_CORE_SHADER, ++ kbase_pm_get_ready_cores(kbdev, ++ KBASE_PM_CORE_SHADER)); ++ kbase_trace_mali_pm_status(KBASE_PM_CORE_TILER, ++ kbase_pm_get_ready_cores(kbdev, ++ KBASE_PM_CORE_TILER)); ++#ifdef CONFIG_MALI_CORESTACK ++ kbase_trace_mali_pm_status(KBASE_PM_CORE_STACK, ++ kbase_pm_get_ready_cores(kbdev, ++ KBASE_PM_CORE_STACK)); ++#endif /* CONFIG_MALI_CORESTACK */ ++#endif ++ ++ KBASE_TLSTREAM_AUX_PM_STATE( ++ KBASE_PM_CORE_L2, ++ kbase_pm_get_ready_cores( ++ kbdev, KBASE_PM_CORE_L2)); ++ KBASE_TLSTREAM_AUX_PM_STATE( ++ KBASE_PM_CORE_SHADER, ++ kbase_pm_get_ready_cores( ++ kbdev, KBASE_PM_CORE_SHADER)); ++ KBASE_TLSTREAM_AUX_PM_STATE( ++ KBASE_PM_CORE_TILER, ++ kbase_pm_get_ready_cores( ++ kbdev, ++ KBASE_PM_CORE_TILER)); ++#ifdef CONFIG_MALI_CORESTACK ++ KBASE_TLSTREAM_AUX_PM_STATE( ++ KBASE_PM_CORE_STACK, ++ kbase_pm_get_ready_cores( ++ kbdev, ++ KBASE_PM_CORE_STACK)); ++#endif /* CONFIG_MALI_CORESTACK */ ++ ++ KBASE_TRACE_ADD(kbdev, PM_DESIRED_REACHED, NULL, NULL, ++ kbdev->pm.backend.gpu_in_desired_state, ++ (u32)kbdev->pm.backend.desired_shader_state); ++ KBASE_TRACE_ADD(kbdev, PM_DESIRED_REACHED_TILER, NULL, NULL, 0u, ++ (u32)kbdev->pm.backend.desired_tiler_state); ++ ++ /* Log timelining information for synchronous waiters */ ++ kbase_timeline_pm_send_event(kbdev, ++ KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED); ++ /* Wake slow-path waiters. Job scheduler does not use this. */ ++ KBASE_TRACE_ADD(kbdev, PM_WAKE_WAITERS, NULL, NULL, 0u, 0); ++ ++ wake_up(&kbdev->pm.backend.gpu_in_desired_state_wait); ++ } ++ ++ spin_unlock(&kbdev->pm.backend.gpu_powered_lock); ++ ++ /* kbase_pm_ca_update_core_status can cause one-level recursion into ++ * this function, so it must only be called once all changes to kbdev ++ * have been committed, and after the gpu_powered_lock has been ++ * dropped. */ ++ if (kbdev->shader_ready_bitmap != shader_ready_bitmap || ++ kbdev->shader_transitioning_bitmap != shader_transitioning_bitmap) { ++ kbdev->shader_ready_bitmap = shader_ready_bitmap; ++ kbdev->shader_transitioning_bitmap = ++ shader_transitioning_bitmap; ++ ++ kbase_pm_ca_update_core_status(kbdev, shader_ready_bitmap, ++ shader_transitioning_bitmap); ++ } ++ ++ /* The core availability policy is not allowed to keep core group 0 ++ * turned off (unless it was changing the l2 power state) */ ++ if (!((shader_ready_bitmap | shader_transitioning_bitmap) & ++ kbdev->gpu_props.props.coherency_info.group[0].core_mask) && ++ (prev_l2_available_bitmap == desired_l2_state) && ++ !(kbase_pm_ca_get_core_mask(kbdev) & ++ kbdev->gpu_props.props.coherency_info.group[0].core_mask)) ++ BUG(); ++ ++ /* The core availability policy is allowed to keep core group 1 off, ++ * but all jobs specifically targeting CG1 must fail */ ++ if (!((shader_ready_bitmap | shader_transitioning_bitmap) & ++ kbdev->gpu_props.props.coherency_info.group[1].core_mask) && ++ !(kbase_pm_ca_get_core_mask(kbdev) & ++ kbdev->gpu_props.props.coherency_info.group[1].core_mask)) ++ kbdev->pm.backend.cg1_disabled = true; ++ else ++ kbdev->pm.backend.cg1_disabled = false; ++ ++ return cores_are_available; ++} ++KBASE_EXPORT_TEST_API(kbase_pm_check_transitions_nolock); ++ ++/* Timeout for kbase_pm_check_transitions_sync when wait_event_killable has ++ * aborted due to a fatal signal. If the time spent waiting has exceeded this ++ * threshold then there is most likely a hardware issue. */ ++#define PM_TIMEOUT (5*HZ) /* 5s */ ++ ++void kbase_pm_check_transitions_sync(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ unsigned long timeout; ++ bool cores_are_available; ++ int ret; ++ ++ /* Force the transition to be checked and reported - the cores may be ++ * 'available' (for job submission) but not fully powered up. */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ cores_are_available = kbase_pm_check_transitions_nolock(kbdev); ++ ++ /* Don't need 'cores_are_available', because we don't return anything */ ++ CSTD_UNUSED(cores_are_available); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ timeout = jiffies + PM_TIMEOUT; ++ ++ /* Wait for cores */ ++ ret = wait_event_killable(kbdev->pm.backend.gpu_in_desired_state_wait, ++ kbdev->pm.backend.gpu_in_desired_state); ++ ++ if (ret < 0 && time_after(jiffies, timeout)) { ++ dev_err(kbdev->dev, "Power transition timed out unexpectedly\n"); ++ dev_err(kbdev->dev, "Desired state :\n"); ++ dev_err(kbdev->dev, "\tShader=%016llx\n", ++ kbdev->pm.backend.desired_shader_state); ++ dev_err(kbdev->dev, "\tTiler =%016llx\n", ++ kbdev->pm.backend.desired_tiler_state); ++ dev_err(kbdev->dev, "Current state :\n"); ++ dev_err(kbdev->dev, "\tShader=%08x%08x\n", ++ kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(SHADER_READY_HI), NULL), ++ kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(SHADER_READY_LO), ++ NULL)); ++ dev_err(kbdev->dev, "\tTiler =%08x%08x\n", ++ kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(TILER_READY_HI), NULL), ++ kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(TILER_READY_LO), NULL)); ++ dev_err(kbdev->dev, "\tL2 =%08x%08x\n", ++ kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(L2_READY_HI), NULL), ++ kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(L2_READY_LO), NULL)); ++ dev_err(kbdev->dev, "Cores transitioning :\n"); ++ dev_err(kbdev->dev, "\tShader=%08x%08x\n", ++ kbase_reg_read(kbdev, GPU_CONTROL_REG( ++ SHADER_PWRTRANS_HI), NULL), ++ kbase_reg_read(kbdev, GPU_CONTROL_REG( ++ SHADER_PWRTRANS_LO), NULL)); ++ dev_err(kbdev->dev, "\tTiler =%08x%08x\n", ++ kbase_reg_read(kbdev, GPU_CONTROL_REG( ++ TILER_PWRTRANS_HI), NULL), ++ kbase_reg_read(kbdev, GPU_CONTROL_REG( ++ TILER_PWRTRANS_LO), NULL)); ++ dev_err(kbdev->dev, "\tL2 =%08x%08x\n", ++ kbase_reg_read(kbdev, GPU_CONTROL_REG( ++ L2_PWRTRANS_HI), NULL), ++ kbase_reg_read(kbdev, GPU_CONTROL_REG( ++ L2_PWRTRANS_LO), NULL)); ++#if KBASE_GPU_RESET_EN ++ dev_err(kbdev->dev, "Sending reset to GPU - all running jobs will be lost\n"); ++ if (kbase_prepare_to_reset_gpu(kbdev)) ++ kbase_reset_gpu(kbdev); ++#endif /* KBASE_GPU_RESET_EN */ ++ } else { ++ /* Log timelining information that a change in state has ++ * completed */ ++ kbase_timeline_pm_handle_event(kbdev, ++ KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED); ++ } ++} ++KBASE_EXPORT_TEST_API(kbase_pm_check_transitions_sync); ++ ++void kbase_pm_enable_interrupts(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ ++ KBASE_DEBUG_ASSERT(NULL != kbdev); ++ /* ++ * Clear all interrupts, ++ * and unmask them all. ++ */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), GPU_IRQ_REG_ALL, ++ NULL); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), GPU_IRQ_REG_ALL, ++ NULL); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), 0xFFFFFFFF, ++ NULL); ++ kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), 0xFFFFFFFF, NULL); ++ ++ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), 0xFFFFFFFF, NULL); ++ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0xFFFFFFFF, NULL); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_enable_interrupts); ++ ++void kbase_pm_disable_interrupts_nolock(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(NULL != kbdev); ++ /* ++ * Mask all interrupts, ++ * and clear them all. ++ */ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), 0, NULL); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), GPU_IRQ_REG_ALL, ++ NULL); ++ kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), 0, NULL); ++ kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), 0xFFFFFFFF, ++ NULL); ++ ++ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0, NULL); ++ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), 0xFFFFFFFF, NULL); ++} ++ ++void kbase_pm_disable_interrupts(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_pm_disable_interrupts_nolock(kbdev); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_disable_interrupts); ++ ++ ++/* ++ * pmu layout: ++ * 0x0000: PMU TAG (RO) (0xCAFECAFE) ++ * 0x0004: PMU VERSION ID (RO) (0x00000000) ++ * 0x0008: CLOCK ENABLE (RW) (31:1 SBZ, 0 CLOCK STATE) ++ */ ++void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume) ++{ ++ bool reset_required = is_resume; ++ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; ++ unsigned long flags; ++ ++ KBASE_DEBUG_ASSERT(NULL != kbdev); ++ lockdep_assert_held(&js_devdata->runpool_mutex); ++ lockdep_assert_held(&kbdev->pm.lock); ++ ++ if (kbdev->pm.backend.gpu_powered) { ++ /* Already turned on */ ++ if (kbdev->poweroff_pending) ++ kbase_pm_enable_interrupts(kbdev); ++ kbdev->poweroff_pending = false; ++ KBASE_DEBUG_ASSERT(!is_resume); ++ return; ++ } ++ ++ kbdev->poweroff_pending = false; ++ ++ KBASE_TRACE_ADD(kbdev, PM_GPU_ON, NULL, NULL, 0u, 0u); ++ ++ if (is_resume && kbdev->pm.backend.callback_power_resume) { ++ kbdev->pm.backend.callback_power_resume(kbdev); ++ return; ++ } else if (kbdev->pm.backend.callback_power_on) { ++ kbdev->pm.backend.callback_power_on(kbdev); ++ /* If your platform properly keeps the GPU state you may use the ++ * return value of the callback_power_on function to ++ * conditionally reset the GPU on power up. Currently we are ++ * conservative and always reset the GPU. */ ++ reset_required = true; ++ } ++ ++ spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags); ++ kbdev->pm.backend.gpu_powered = true; ++ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags); ++ ++ if (reset_required) { ++ /* GPU state was lost, reset GPU to ensure it is in a ++ * consistent state */ ++ kbase_pm_init_hw(kbdev, PM_ENABLE_IRQS); ++ } ++ ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_ctx_sched_restore_all_as(kbdev); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ ++ /* Lastly, enable the interrupts */ ++ kbase_pm_enable_interrupts(kbdev); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_clock_on); ++ ++bool kbase_pm_clock_off(struct kbase_device *kbdev, bool is_suspend) ++{ ++ unsigned long flags; ++ ++ KBASE_DEBUG_ASSERT(NULL != kbdev); ++ lockdep_assert_held(&kbdev->pm.lock); ++ ++ /* ASSERT that the cores should now be unavailable. No lock needed. */ ++ KBASE_DEBUG_ASSERT(kbdev->shader_available_bitmap == 0u); ++ ++ kbdev->poweroff_pending = true; ++ ++ if (!kbdev->pm.backend.gpu_powered) { ++ /* Already turned off */ ++ if (is_suspend && kbdev->pm.backend.callback_power_suspend) ++ kbdev->pm.backend.callback_power_suspend(kbdev); ++ return true; ++ } ++ ++ KBASE_TRACE_ADD(kbdev, PM_GPU_OFF, NULL, NULL, 0u, 0u); ++ ++ /* Disable interrupts. This also clears any outstanding interrupts */ ++ kbase_pm_disable_interrupts(kbdev); ++ /* Ensure that any IRQ handlers have finished */ ++ kbase_synchronize_irqs(kbdev); ++ ++ spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags); ++ ++ if (atomic_read(&kbdev->faults_pending)) { ++ /* Page/bus faults are still being processed. The GPU can not ++ * be powered off until they have completed */ ++ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, ++ flags); ++ return false; ++ } ++ ++ kbase_pm_cache_snoop_disable(kbdev); ++ ++ /* The GPU power may be turned off from this point */ ++ kbdev->pm.backend.gpu_powered = false; ++ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags); ++ ++ if (is_suspend && kbdev->pm.backend.callback_power_suspend) ++ kbdev->pm.backend.callback_power_suspend(kbdev); ++ else if (kbdev->pm.backend.callback_power_off) ++ kbdev->pm.backend.callback_power_off(kbdev); ++ return true; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_clock_off); ++ ++struct kbasep_reset_timeout_data { ++ struct hrtimer timer; ++ bool timed_out; ++ struct kbase_device *kbdev; ++}; ++ ++void kbase_pm_reset_done(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ kbdev->pm.backend.reset_done = true; ++ wake_up(&kbdev->pm.backend.reset_done_wait); ++} ++ ++/** ++ * kbase_pm_wait_for_reset - Wait for a reset to happen ++ * ++ * Wait for the %RESET_COMPLETED IRQ to occur, then reset the waiting state. ++ * ++ * @kbdev: Kbase device ++ */ ++static void kbase_pm_wait_for_reset(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->pm.lock); ++ ++ wait_event(kbdev->pm.backend.reset_done_wait, ++ (kbdev->pm.backend.reset_done)); ++ kbdev->pm.backend.reset_done = false; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_reset_done); ++ ++static enum hrtimer_restart kbasep_reset_timeout(struct hrtimer *timer) ++{ ++ struct kbasep_reset_timeout_data *rtdata = ++ container_of(timer, struct kbasep_reset_timeout_data, timer); ++ ++ rtdata->timed_out = 1; ++ ++ /* Set the wait queue to wake up kbase_pm_init_hw even though the reset ++ * hasn't completed */ ++ kbase_pm_reset_done(rtdata->kbdev); ++ ++ return HRTIMER_NORESTART; ++} ++ ++static void kbase_pm_hw_issues_detect(struct kbase_device *kbdev) ++{ ++ struct device_node *np = kbdev->dev->of_node; ++ u32 jm_values[4]; ++ const u32 gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; ++ const u32 prod_id = (gpu_id & GPU_ID_VERSION_PRODUCT_ID) >> ++ GPU_ID_VERSION_PRODUCT_ID_SHIFT; ++ const u32 major = (gpu_id & GPU_ID_VERSION_MAJOR) >> ++ GPU_ID_VERSION_MAJOR_SHIFT; ++ ++ kbdev->hw_quirks_sc = 0; ++ ++ /* Needed due to MIDBASE-1494: LS_PAUSEBUFFER_DISABLE. See PRLAM-8443. ++ * and needed due to MIDGLES-3539. See PRLAM-11035 */ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8443) || ++ kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_11035)) ++ kbdev->hw_quirks_sc |= SC_LS_PAUSEBUFFER_DISABLE; ++ ++ /* Needed due to MIDBASE-2054: SDC_DISABLE_OQ_DISCARD. See PRLAM-10327. ++ */ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10327)) ++ kbdev->hw_quirks_sc |= SC_SDC_DISABLE_OQ_DISCARD; ++ ++#ifdef CONFIG_MALI_PRFCNT_SET_SECONDARY ++ /* Enable alternative hardware counter selection if configured. */ ++ if (!GPU_ID_IS_NEW_FORMAT(prod_id)) ++ kbdev->hw_quirks_sc |= SC_ALT_COUNTERS; ++#endif ++ ++ /* Needed due to MIDBASE-2795. ENABLE_TEXGRD_FLAGS. See PRLAM-10797. */ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10797)) ++ kbdev->hw_quirks_sc |= SC_ENABLE_TEXGRD_FLAGS; ++ ++ if (!kbase_hw_has_issue(kbdev, GPUCORE_1619)) { ++ if (prod_id < 0x750 || prod_id == 0x6956) /* T60x, T62x, T72x */ ++ kbdev->hw_quirks_sc |= SC_LS_ATTR_CHECK_DISABLE; ++ else if (prod_id >= 0x750 && prod_id <= 0x880) /* T76x, T8xx */ ++ kbdev->hw_quirks_sc |= SC_LS_ALLOW_ATTR_TYPES; ++ } ++ ++ if (!kbdev->hw_quirks_sc) ++ kbdev->hw_quirks_sc = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(SHADER_CONFIG), NULL); ++ ++ kbdev->hw_quirks_tiler = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(TILER_CONFIG), NULL); ++ ++ /* Set tiler clock gate override if required */ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_T76X_3953)) ++ kbdev->hw_quirks_tiler |= TC_CLOCK_GATE_OVERRIDE; ++ ++ /* Limit the GPU bus bandwidth if the platform needs this. */ ++ kbdev->hw_quirks_mmu = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(L2_MMU_CONFIG), NULL); ++ ++ /* Limit read ID width for AXI */ ++ kbdev->hw_quirks_mmu &= ~(L2_MMU_CONFIG_LIMIT_EXTERNAL_READS); ++ kbdev->hw_quirks_mmu |= (DEFAULT_ARID_LIMIT & 0x3) << ++ L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT; ++ ++ /* Limit write ID width for AXI */ ++ kbdev->hw_quirks_mmu &= ~(L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES); ++ kbdev->hw_quirks_mmu |= (DEFAULT_AWID_LIMIT & 0x3) << ++ L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT; ++ ++ if (kbdev->system_coherency == COHERENCY_ACE) { ++ /* Allow memory configuration disparity to be ignored, we ++ * optimize the use of shared memory and thus we expect ++ * some disparity in the memory configuration */ ++ kbdev->hw_quirks_mmu |= L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY; ++ } ++ ++ kbdev->hw_quirks_jm = 0; ++ /* Only for T86x/T88x-based products after r2p0 */ ++ if (prod_id >= 0x860 && prod_id <= 0x880 && major >= 2) { ++ ++ if (of_property_read_u32_array(np, ++ "jm_config", ++ &jm_values[0], ++ ARRAY_SIZE(jm_values))) { ++ /* Entry not in device tree, use defaults */ ++ jm_values[0] = 0; ++ jm_values[1] = 0; ++ jm_values[2] = 0; ++ jm_values[3] = JM_MAX_JOB_THROTTLE_LIMIT; ++ } ++ ++ /* Limit throttle limit to 6 bits*/ ++ if (jm_values[3] > JM_MAX_JOB_THROTTLE_LIMIT) { ++ dev_dbg(kbdev->dev, "JOB_THROTTLE_LIMIT supplied in device tree is too large. Limiting to MAX (63)."); ++ jm_values[3] = JM_MAX_JOB_THROTTLE_LIMIT; ++ } ++ ++ /* Aggregate to one integer. */ ++ kbdev->hw_quirks_jm |= (jm_values[0] ? ++ JM_TIMESTAMP_OVERRIDE : 0); ++ kbdev->hw_quirks_jm |= (jm_values[1] ? ++ JM_CLOCK_GATE_OVERRIDE : 0); ++ kbdev->hw_quirks_jm |= (jm_values[2] ? ++ JM_JOB_THROTTLE_ENABLE : 0); ++ kbdev->hw_quirks_jm |= (jm_values[3] << ++ JM_JOB_THROTTLE_LIMIT_SHIFT); ++ ++ } else if (GPU_ID_IS_NEW_FORMAT(prod_id) && ++ (GPU_ID2_MODEL_MATCH_VALUE(prod_id) == ++ GPU_ID2_PRODUCT_TMIX)) { ++ /* Only for tMIx */ ++ u32 coherency_features; ++ ++ coherency_features = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(COHERENCY_FEATURES), NULL); ++ ++ /* (COHERENCY_ACE_LITE | COHERENCY_ACE) was incorrectly ++ * documented for tMIx so force correct value here. ++ */ ++ if (coherency_features == ++ COHERENCY_FEATURE_BIT(COHERENCY_ACE)) { ++ kbdev->hw_quirks_jm |= ++ (COHERENCY_ACE_LITE | COHERENCY_ACE) << ++ JM_FORCE_COHERENCY_FEATURES_SHIFT; ++ } ++ } ++ ++ if (!kbdev->hw_quirks_jm) ++ kbdev->hw_quirks_jm = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(JM_CONFIG), NULL); ++ ++#ifdef CONFIG_MALI_CORESTACK ++#define MANUAL_POWER_CONTROL ((u32)(1 << 8)) ++ kbdev->hw_quirks_jm |= MANUAL_POWER_CONTROL; ++#endif /* CONFIG_MALI_CORESTACK */ ++} ++ ++static void kbase_pm_hw_issues_apply(struct kbase_device *kbdev) ++{ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(SHADER_CONFIG), ++ kbdev->hw_quirks_sc, NULL); ++ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(TILER_CONFIG), ++ kbdev->hw_quirks_tiler, NULL); ++ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(L2_MMU_CONFIG), ++ kbdev->hw_quirks_mmu, NULL); ++ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(JM_CONFIG), ++ kbdev->hw_quirks_jm, NULL); ++ ++} ++ ++void kbase_pm_cache_snoop_enable(struct kbase_device *kbdev) ++{ ++ if ((kbdev->current_gpu_coherency_mode == COHERENCY_ACE) && ++ !kbdev->cci_snoop_enabled) { ++#ifdef CONFIG_ARM64 ++ if (kbdev->snoop_enable_smc != 0) ++ kbase_invoke_smc_fid(kbdev->snoop_enable_smc, 0, 0, 0); ++#endif /* CONFIG_ARM64 */ ++ dev_dbg(kbdev->dev, "MALI - CCI Snoops - Enabled\n"); ++ kbdev->cci_snoop_enabled = true; ++ } ++} ++ ++void kbase_pm_cache_snoop_disable(struct kbase_device *kbdev) ++{ ++ if (kbdev->cci_snoop_enabled) { ++#ifdef CONFIG_ARM64 ++ if (kbdev->snoop_disable_smc != 0) { ++ mali_cci_flush_l2(kbdev); ++ kbase_invoke_smc_fid(kbdev->snoop_disable_smc, 0, 0, 0); ++ } ++#endif /* CONFIG_ARM64 */ ++ dev_dbg(kbdev->dev, "MALI - CCI Snoops Disabled\n"); ++ kbdev->cci_snoop_enabled = false; ++ } ++} ++ ++static int kbase_pm_do_reset(struct kbase_device *kbdev) ++{ ++ struct kbasep_reset_timeout_data rtdata; ++ ++ KBASE_TRACE_ADD(kbdev, CORE_GPU_SOFT_RESET, NULL, NULL, 0u, 0); ++ ++ KBASE_TLSTREAM_JD_GPU_SOFT_RESET(kbdev); ++ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), ++ GPU_COMMAND_SOFT_RESET, NULL); ++ ++ /* Unmask the reset complete interrupt only */ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), RESET_COMPLETED, ++ NULL); ++ ++ /* Initialize a structure for tracking the status of the reset */ ++ rtdata.kbdev = kbdev; ++ rtdata.timed_out = 0; ++ ++ /* Create a timer to use as a timeout on the reset */ ++ hrtimer_init_on_stack(&rtdata.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); ++ rtdata.timer.function = kbasep_reset_timeout; ++ ++ hrtimer_start(&rtdata.timer, HR_TIMER_DELAY_MSEC(RESET_TIMEOUT), ++ HRTIMER_MODE_REL); ++ ++ /* Wait for the RESET_COMPLETED interrupt to be raised */ ++ kbase_pm_wait_for_reset(kbdev); ++ ++ if (rtdata.timed_out == 0) { ++ /* GPU has been reset */ ++ hrtimer_cancel(&rtdata.timer); ++ destroy_hrtimer_on_stack(&rtdata.timer); ++ return 0; ++ } ++ ++ /* No interrupt has been received - check if the RAWSTAT register says ++ * the reset has completed */ ++ if (kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), NULL) & ++ RESET_COMPLETED) { ++ /* The interrupt is set in the RAWSTAT; this suggests that the ++ * interrupts are not getting to the CPU */ ++ dev_err(kbdev->dev, "Reset interrupt didn't reach CPU. Check interrupt assignments.\n"); ++ /* If interrupts aren't working we can't continue. */ ++ destroy_hrtimer_on_stack(&rtdata.timer); ++ return -EINVAL; ++ } ++ ++ /* The GPU doesn't seem to be responding to the reset so try a hard ++ * reset */ ++ dev_err(kbdev->dev, "Failed to soft-reset GPU (timed out after %d ms), now attempting a hard reset\n", ++ RESET_TIMEOUT); ++ KBASE_TRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, NULL, 0u, 0); ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), ++ GPU_COMMAND_HARD_RESET, NULL); ++ ++ /* Restart the timer to wait for the hard reset to complete */ ++ rtdata.timed_out = 0; ++ ++ hrtimer_start(&rtdata.timer, HR_TIMER_DELAY_MSEC(RESET_TIMEOUT), ++ HRTIMER_MODE_REL); ++ ++ /* Wait for the RESET_COMPLETED interrupt to be raised */ ++ kbase_pm_wait_for_reset(kbdev); ++ ++ if (rtdata.timed_out == 0) { ++ /* GPU has been reset */ ++ hrtimer_cancel(&rtdata.timer); ++ destroy_hrtimer_on_stack(&rtdata.timer); ++ return 0; ++ } ++ ++ destroy_hrtimer_on_stack(&rtdata.timer); ++ ++ dev_err(kbdev->dev, "Failed to hard-reset the GPU (timed out after %d ms)\n", ++ RESET_TIMEOUT); ++ ++ return -EINVAL; ++} ++ ++static int kbasep_protected_mode_enable(struct protected_mode_device *pdev) ++{ ++ struct kbase_device *kbdev = pdev->data; ++ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), ++ GPU_COMMAND_SET_PROTECTED_MODE, NULL); ++ return 0; ++} ++ ++static int kbasep_protected_mode_disable(struct protected_mode_device *pdev) ++{ ++ struct kbase_device *kbdev = pdev->data; ++ ++ lockdep_assert_held(&kbdev->pm.lock); ++ ++ return kbase_pm_do_reset(kbdev); ++} ++ ++struct protected_mode_ops kbase_native_protected_ops = { ++ .protected_mode_enable = kbasep_protected_mode_enable, ++ .protected_mode_disable = kbasep_protected_mode_disable ++}; ++ ++int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags) ++{ ++ unsigned long irq_flags; ++ int err; ++ bool resume_vinstr = false; ++ ++ KBASE_DEBUG_ASSERT(NULL != kbdev); ++ lockdep_assert_held(&kbdev->pm.lock); ++ ++ /* Ensure the clock is on before attempting to access the hardware */ ++ if (!kbdev->pm.backend.gpu_powered) { ++ if (kbdev->pm.backend.callback_power_on) ++ kbdev->pm.backend.callback_power_on(kbdev); ++ ++ spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, ++ irq_flags); ++ kbdev->pm.backend.gpu_powered = true; ++ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, ++ irq_flags); ++ } ++ ++ /* Ensure interrupts are off to begin with, this also clears any ++ * outstanding interrupts */ ++ kbase_pm_disable_interrupts(kbdev); ++ /* Ensure cache snoops are disabled before reset. */ ++ kbase_pm_cache_snoop_disable(kbdev); ++ /* Prepare for the soft-reset */ ++ kbdev->pm.backend.reset_done = false; ++ ++ /* The cores should be made unavailable due to the reset */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); ++ if (kbdev->shader_available_bitmap != 0u) ++ KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, ++ NULL, 0u, (u32)0u); ++ if (kbdev->tiler_available_bitmap != 0u) ++ KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER, ++ NULL, NULL, 0u, (u32)0u); ++ kbdev->shader_available_bitmap = 0u; ++ kbdev->tiler_available_bitmap = 0u; ++ kbdev->l2_available_bitmap = 0u; ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); ++ ++ /* Soft reset the GPU */ ++ if (kbdev->protected_mode_support) ++ err = kbdev->protected_ops->protected_mode_disable( ++ kbdev->protected_dev); ++ else ++ err = kbase_pm_do_reset(kbdev); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); ++ if (kbdev->protected_mode) ++ resume_vinstr = true; ++ kbdev->protected_mode = false; ++ kbase_ipa_model_use_configured_locked(kbdev); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); ++ ++ if (err) ++ goto exit; ++ ++ if (flags & PM_HW_ISSUES_DETECT) ++ kbase_pm_hw_issues_detect(kbdev); ++ ++ kbase_pm_hw_issues_apply(kbdev); ++ kbase_cache_set_coherency_mode(kbdev, kbdev->system_coherency); ++ ++ /* Sanity check protected mode was left after reset */ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_MODE)) { ++ u32 gpu_status = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(GPU_STATUS), NULL); ++ ++ WARN_ON(gpu_status & GPU_STATUS_PROTECTED_MODE_ACTIVE); ++ } ++ ++ /* If cycle counter was in use re-enable it, enable_irqs will only be ++ * false when called from kbase_pm_powerup */ ++ if (kbdev->pm.backend.gpu_cycle_counter_requests && ++ (flags & PM_ENABLE_IRQS)) { ++ /* enable interrupts as the L2 may have to be powered on */ ++ kbase_pm_enable_interrupts(kbdev); ++ kbase_pm_request_l2_caches(kbdev); ++ ++ /* Re-enable the counters if we need to */ ++ spin_lock_irqsave( ++ &kbdev->pm.backend.gpu_cycle_counter_requests_lock, ++ irq_flags); ++ if (kbdev->pm.backend.gpu_cycle_counter_requests) ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), ++ GPU_COMMAND_CYCLE_COUNT_START, NULL); ++ spin_unlock_irqrestore( ++ &kbdev->pm.backend.gpu_cycle_counter_requests_lock, ++ irq_flags); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); ++ kbase_pm_release_l2_caches(kbdev); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); ++ ++ kbase_pm_disable_interrupts(kbdev); ++ } ++ ++ if (flags & PM_ENABLE_IRQS) ++ kbase_pm_enable_interrupts(kbdev); ++ ++exit: ++ /* If GPU is leaving protected mode resume vinstr operation. */ ++ if (kbdev->vinstr_ctx && resume_vinstr) ++ kbase_vinstr_resume(kbdev->vinstr_ctx); ++ ++ return err; ++} ++ ++/** ++ * kbase_pm_request_gpu_cycle_counter_do_request - Request cycle counters ++ * ++ * Increase the count of cycle counter users and turn the cycle counters on if ++ * they were previously off ++ * ++ * This function is designed to be called by ++ * kbase_pm_request_gpu_cycle_counter() or ++ * kbase_pm_request_gpu_cycle_counter_l2_is_on() only ++ * ++ * When this function is called the l2 cache must be on and the l2 cache users ++ * count must have been incremented by a call to ( ++ * kbase_pm_request_l2_caches() or kbase_pm_request_l2_caches_l2_on() ) ++ * ++ * @kbdev: The kbase device structure of the device ++ */ ++static void ++kbase_pm_request_gpu_cycle_counter_do_request(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->pm.backend.gpu_cycle_counter_requests_lock, ++ flags); ++ ++ ++kbdev->pm.backend.gpu_cycle_counter_requests; ++ ++ if (1 == kbdev->pm.backend.gpu_cycle_counter_requests) ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), ++ GPU_COMMAND_CYCLE_COUNT_START, NULL); ++ ++ spin_unlock_irqrestore( ++ &kbdev->pm.backend.gpu_cycle_counter_requests_lock, ++ flags); ++} ++ ++void kbase_pm_request_gpu_cycle_counter(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered); ++ ++ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests < ++ INT_MAX); ++ ++ kbase_pm_request_l2_caches(kbdev); ++ ++ kbase_pm_request_gpu_cycle_counter_do_request(kbdev); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_request_gpu_cycle_counter); ++ ++void kbase_pm_request_gpu_cycle_counter_l2_is_on(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered); ++ ++ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests < ++ INT_MAX); ++ ++ kbase_pm_request_l2_caches_l2_is_on(kbdev); ++ ++ kbase_pm_request_gpu_cycle_counter_do_request(kbdev); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_request_gpu_cycle_counter_l2_is_on); ++ ++void kbase_pm_release_gpu_cycle_counter_nolock(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ spin_lock_irqsave(&kbdev->pm.backend.gpu_cycle_counter_requests_lock, ++ flags); ++ ++ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests > 0); ++ ++ --kbdev->pm.backend.gpu_cycle_counter_requests; ++ ++ if (0 == kbdev->pm.backend.gpu_cycle_counter_requests) ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), ++ GPU_COMMAND_CYCLE_COUNT_STOP, NULL); ++ ++ spin_unlock_irqrestore( ++ &kbdev->pm.backend.gpu_cycle_counter_requests_lock, ++ flags); ++ ++ kbase_pm_release_l2_caches(kbdev); ++} ++ ++void kbase_pm_release_gpu_cycle_counter(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ kbase_pm_release_gpu_cycle_counter_nolock(kbdev); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_release_gpu_cycle_counter); +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_internal.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_internal.h +new file mode 100755 +index 000000000000..6804f45ac27b +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_internal.h +@@ -0,0 +1,548 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * Power management API definitions used internally by GPU backend ++ */ ++ ++#ifndef _KBASE_BACKEND_PM_INTERNAL_H_ ++#define _KBASE_BACKEND_PM_INTERNAL_H_ ++ ++#include ++ ++#include "mali_kbase_pm_ca.h" ++#include "mali_kbase_pm_policy.h" ++ ++ ++/** ++ * kbase_pm_dev_idle - The GPU is idle. ++ * ++ * The OS may choose to turn off idle devices ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_dev_idle(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_dev_activate - The GPU is active. ++ * ++ * The OS should avoid opportunistically turning off the GPU while it is active ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_dev_activate(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_get_present_cores - Get details of the cores that are present in ++ * the device. ++ * ++ * This function can be called by the active power policy to return a bitmask of ++ * the cores (of a specified type) present in the GPU device and also a count of ++ * the number of cores. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid ++ * pointer) ++ * @type: The type of core (see the enum kbase_pm_core_type enumeration) ++ * ++ * Return: The bit mask of cores present ++ */ ++u64 kbase_pm_get_present_cores(struct kbase_device *kbdev, ++ enum kbase_pm_core_type type); ++ ++/** ++ * kbase_pm_get_active_cores - Get details of the cores that are currently ++ * active in the device. ++ * ++ * This function can be called by the active power policy to return a bitmask of ++ * the cores (of a specified type) that are actively processing work (i.e. ++ * turned on *and* busy). ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * @type: The type of core (see the enum kbase_pm_core_type enumeration) ++ * ++ * Return: The bit mask of active cores ++ */ ++u64 kbase_pm_get_active_cores(struct kbase_device *kbdev, ++ enum kbase_pm_core_type type); ++ ++/** ++ * kbase_pm_get_trans_cores - Get details of the cores that are currently ++ * transitioning between power states. ++ * ++ * This function can be called by the active power policy to return a bitmask of ++ * the cores (of a specified type) that are currently transitioning between ++ * power states. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * @type: The type of core (see the enum kbase_pm_core_type enumeration) ++ * ++ * Return: The bit mask of transitioning cores ++ */ ++u64 kbase_pm_get_trans_cores(struct kbase_device *kbdev, ++ enum kbase_pm_core_type type); ++ ++/** ++ * kbase_pm_get_ready_cores - Get details of the cores that are currently ++ * powered and ready for jobs. ++ * ++ * This function can be called by the active power policy to return a bitmask of ++ * the cores (of a specified type) that are powered and ready for jobs (they may ++ * or may not be currently executing jobs). ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * @type: The type of core (see the enum kbase_pm_core_type enumeration) ++ * ++ * Return: The bit mask of ready cores ++ */ ++u64 kbase_pm_get_ready_cores(struct kbase_device *kbdev, ++ enum kbase_pm_core_type type); ++ ++/** ++ * kbase_pm_clock_on - Turn the clock for the device on, and enable device ++ * interrupts. ++ * ++ * This function can be used by a power policy to turn the clock for the GPU on. ++ * It should be modified during integration to perform the necessary actions to ++ * ensure that the GPU is fully powered and clocked. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid ++ * pointer) ++ * @is_resume: true if clock on due to resume after suspend, false otherwise ++ */ ++void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume); ++ ++/** ++ * kbase_pm_clock_off - Disable device interrupts, and turn the clock for the ++ * device off. ++ * ++ * This function can be used by a power policy to turn the clock for the GPU ++ * off. It should be modified during integration to perform the necessary ++ * actions to turn the clock off (if this is possible in the integration). ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid ++ * pointer) ++ * @is_suspend: true if clock off due to suspend, false otherwise ++ * ++ * Return: true if clock was turned off, or ++ * false if clock can not be turned off due to pending page/bus fault ++ * workers. Caller must flush MMU workqueues and retry ++ */ ++bool kbase_pm_clock_off(struct kbase_device *kbdev, bool is_suspend); ++ ++/** ++ * kbase_pm_enable_interrupts - Enable interrupts on the device. ++ * ++ * Interrupts are also enabled after a call to kbase_pm_clock_on(). ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_enable_interrupts(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_disable_interrupts - Disable interrupts on the device. ++ * ++ * This prevents delivery of Power Management interrupts to the CPU so that ++ * kbase_pm_check_transitions_nolock() will not be called from the IRQ handler ++ * until kbase_pm_enable_interrupts() or kbase_pm_clock_on() is called. ++ * ++ * Interrupts are also disabled after a call to kbase_pm_clock_off(). ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_disable_interrupts(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_disable_interrupts_nolock - Version of kbase_pm_disable_interrupts() ++ * that does not take the hwaccess_lock ++ * ++ * Caller must hold the hwaccess_lock. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_disable_interrupts_nolock(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_init_hw - Initialize the hardware. ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * @flags: Flags specifying the type of PM init ++ * ++ * This function checks the GPU ID register to ensure that the GPU is supported ++ * by the driver and performs a reset on the device so that it is in a known ++ * state before the device is used. ++ * ++ * Return: 0 if the device is supported and successfully reset. ++ */ ++int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags); ++ ++/** ++ * kbase_pm_reset_done - The GPU has been reset successfully. ++ * ++ * This function must be called by the GPU interrupt handler when the ++ * RESET_COMPLETED bit is set. It signals to the power management initialization ++ * code that the GPU has been successfully reset. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_reset_done(struct kbase_device *kbdev); ++ ++ ++/** ++ * kbase_pm_check_transitions_nolock - Check if there are any power transitions ++ * to make, and if so start them. ++ * ++ * This function will check the desired_xx_state members of ++ * struct kbase_pm_device_data and the actual status of the hardware to see if ++ * any power transitions can be made at this time to make the hardware state ++ * closer to the state desired by the power policy. ++ * ++ * The return value can be used to check whether all the desired cores are ++ * available, and so whether it's worth submitting a job (e.g. from a Power ++ * Management IRQ). ++ * ++ * Note that this still returns true when desired_xx_state has no ++ * cores. That is: of the no cores desired, none were *un*available. In ++ * this case, the caller may still need to try submitting jobs. This is because ++ * the Core Availability Policy might have taken us to an intermediate state ++ * where no cores are powered, before powering on more cores (e.g. for core ++ * rotation) ++ * ++ * The caller must hold kbase_device.pm.power_change_lock ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * Return: non-zero when all desired cores are available. That is, ++ * it's worthwhile for the caller to submit a job. ++ * false otherwise ++ */ ++bool kbase_pm_check_transitions_nolock(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_check_transitions_sync - Synchronous and locking variant of ++ * kbase_pm_check_transitions_nolock() ++ * ++ * On returning, the desired state at the time of the call will have been met. ++ * ++ * There is nothing to stop the core being switched off by calls to ++ * kbase_pm_release_cores() or kbase_pm_unrequest_cores(). Therefore, the ++ * caller must have already made a call to ++ * kbase_pm_request_cores()/kbase_pm_request_cores_sync() previously. ++ * ++ * The usual use-case for this is to ensure cores are 'READY' after performing ++ * a GPU Reset. ++ * ++ * Unlike kbase_pm_check_transitions_nolock(), the caller must not hold ++ * kbase_device.pm.power_change_lock, because this function will take that ++ * lock itself. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_check_transitions_sync(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_update_cores_state_nolock - Variant of kbase_pm_update_cores_state() ++ * where the caller must hold ++ * kbase_device.pm.power_change_lock ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_update_cores_state_nolock(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_update_cores_state - Update the desired state of shader cores from ++ * the Power Policy, and begin any power ++ * transitions. ++ * ++ * This function will update the desired_xx_state members of ++ * struct kbase_pm_device_data by calling into the current Power Policy. It will ++ * then begin power transitions to make the hardware acheive the desired shader ++ * core state. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_update_cores_state(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_cancel_deferred_poweroff - Cancel any pending requests to power off ++ * the GPU and/or shader cores. ++ * ++ * This should be called by any functions which directly power off the GPU. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_cancel_deferred_poweroff(struct kbase_device *kbdev); ++ ++/** ++ * kbasep_pm_init_core_use_bitmaps - Initialise data tracking the required ++ * and used cores. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbasep_pm_init_core_use_bitmaps(struct kbase_device *kbdev); ++ ++/** ++ * kbasep_pm_metrics_init - Initialize the metrics gathering framework. ++ * ++ * This must be called before other metric gathering APIs are called. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * Return: 0 on success, error code on error ++ */ ++int kbasep_pm_metrics_init(struct kbase_device *kbdev); ++ ++/** ++ * kbasep_pm_metrics_term - Terminate the metrics gathering framework. ++ * ++ * This must be called when metric gathering is no longer required. It is an ++ * error to call any metrics gathering function (other than ++ * kbasep_pm_metrics_init()) after calling this function. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbasep_pm_metrics_term(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_report_vsync - Function to be called by the frame buffer driver to ++ * update the vsync metric. ++ * ++ * This function should be called by the frame buffer driver to update whether ++ * the system is hitting the vsync target or not. buffer_updated should be true ++ * if the vsync corresponded with a new frame being displayed, otherwise it ++ * should be false. This function does not need to be called every vsync, but ++ * only when the value of @buffer_updated differs from a previous call. ++ * ++ * @kbdev: The kbase device structure for the device (must be a ++ * valid pointer) ++ * @buffer_updated: True if the buffer has been updated on this VSync, ++ * false otherwise ++ */ ++void kbase_pm_report_vsync(struct kbase_device *kbdev, int buffer_updated); ++ ++/** ++ * kbase_pm_get_dvfs_action - Determine whether the DVFS system should change ++ * the clock speed of the GPU. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * This function should be called regularly by the DVFS system to check whether ++ * the clock speed of the GPU needs updating. ++ */ ++void kbase_pm_get_dvfs_action(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_request_gpu_cycle_counter - Mark that the GPU cycle counter is ++ * needed ++ * ++ * If the caller is the first caller then the GPU cycle counters will be enabled ++ * along with the l2 cache ++ * ++ * The GPU must be powered when calling this function (i.e. ++ * kbase_pm_context_active() must have been called). ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_request_gpu_cycle_counter(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_request_gpu_cycle_counter_l2_is_on - Mark GPU cycle counter is ++ * needed (l2 cache already on) ++ * ++ * This is a version of the above function ++ * (kbase_pm_request_gpu_cycle_counter()) suitable for being called when the ++ * l2 cache is known to be on and assured to be on until the subsequent call of ++ * kbase_pm_release_gpu_cycle_counter() such as when a job is submitted. It does ++ * not sleep and can be called from atomic functions. ++ * ++ * The GPU must be powered when calling this function (i.e. ++ * kbase_pm_context_active() must have been called) and the l2 cache must be ++ * powered on. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_request_gpu_cycle_counter_l2_is_on(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_release_gpu_cycle_counter - Mark that the GPU cycle counter is no ++ * longer in use ++ * ++ * If the caller is the last caller then the GPU cycle counters will be ++ * disabled. A request must have been made before a call to this. ++ * ++ * Caller must not hold the hwaccess_lock, as it will be taken in this function. ++ * If the caller is already holding this lock then ++ * kbase_pm_release_gpu_cycle_counter_nolock() must be used instead. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_release_gpu_cycle_counter(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_release_gpu_cycle_counter_nolock - Version of kbase_pm_release_gpu_cycle_counter() ++ * that does not take hwaccess_lock ++ * ++ * Caller must hold the hwaccess_lock. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_release_gpu_cycle_counter_nolock(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_wait_for_poweroff_complete - Wait for the poweroff workqueue to ++ * complete ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_wait_for_poweroff_complete(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_register_access_enable - Enable access to GPU registers ++ * ++ * Enables access to the GPU registers before power management has powered up ++ * the GPU with kbase_pm_powerup(). ++ * ++ * Access to registers should be done using kbase_os_reg_read()/write() at this ++ * stage, not kbase_reg_read()/write(). ++ * ++ * This results in the power management callbacks provided in the driver ++ * configuration to get called to turn on power and/or clocks to the GPU. See ++ * kbase_pm_callback_conf. ++ * ++ * This should only be used before power management is powered up with ++ * kbase_pm_powerup() ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_register_access_enable(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_register_access_disable - Disable early register access ++ * ++ * Disables access to the GPU registers enabled earlier by a call to ++ * kbase_pm_register_access_enable(). ++ * ++ * This results in the power management callbacks provided in the driver ++ * configuration to get called to turn off power and/or clocks to the GPU. See ++ * kbase_pm_callback_conf ++ * ++ * This should only be used before power management is powered up with ++ * kbase_pm_powerup() ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_register_access_disable(struct kbase_device *kbdev); ++ ++/* NOTE: kbase_pm_is_suspending is in mali_kbase.h, because it is an inline ++ * function */ ++ ++/** ++ * kbase_pm_metrics_is_active - Check if the power management metrics ++ * collection is active. ++ * ++ * Note that this returns if the power management metrics collection was ++ * active at the time of calling, it is possible that after the call the metrics ++ * collection enable may have changed state. ++ * ++ * The caller must handle the consequence that the state may have changed. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * Return: true if metrics collection was active else false. ++ */ ++bool kbase_pm_metrics_is_active(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_do_poweron - Power on the GPU, and any cores that are requested. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid ++ * pointer) ++ * @is_resume: true if power on due to resume after suspend, ++ * false otherwise ++ */ ++void kbase_pm_do_poweron(struct kbase_device *kbdev, bool is_resume); ++ ++/** ++ * kbase_pm_do_poweroff - Power off the GPU, and any cores that have been ++ * requested. ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid ++ * pointer) ++ * @is_suspend: true if power off due to suspend, ++ * false otherwise ++ */ ++void kbase_pm_do_poweroff(struct kbase_device *kbdev, bool is_suspend); ++ ++#if defined(CONFIG_MALI_DEVFREQ) || defined(CONFIG_MALI_MIDGARD_DVFS) ++void kbase_pm_get_dvfs_utilisation(struct kbase_device *kbdev, ++ unsigned long *total, unsigned long *busy); ++void kbase_pm_reset_dvfs_utilisation(struct kbase_device *kbdev); ++#endif /* defined(CONFIG_MALI_DEVFREQ) || defined(CONFIG_MALI_MIDGARD_DVFS) */ ++ ++#ifdef CONFIG_MALI_MIDGARD_DVFS ++ ++/** ++ * kbase_platform_dvfs_event - Report utilisation to DVFS code ++ * ++ * Function provided by platform specific code when DVFS is enabled to allow ++ * the power management metrics system to report utilisation. ++ * ++ * @kbdev: The kbase device structure for the device (must be a ++ * valid pointer) ++ * @utilisation: The current calculated utilisation by the metrics system. ++ * @util_gl_share: The current calculated gl share of utilisation. ++ * @util_cl_share: The current calculated cl share of utilisation per core ++ * group. ++ * Return: Returns 0 on failure and non zero on success. ++ */ ++ ++int kbase_platform_dvfs_event(struct kbase_device *kbdev, u32 utilisation, ++ u32 util_gl_share, u32 util_cl_share[2]); ++#endif ++ ++void kbase_pm_power_changed(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_metrics_update - Inform the metrics system that an atom is either ++ * about to be run or has just completed. ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * @now: Pointer to the timestamp of the change, or NULL to use current time ++ * ++ * Caller must hold hwaccess_lock ++ */ ++void kbase_pm_metrics_update(struct kbase_device *kbdev, ++ ktime_t *now); ++ ++/** ++ * kbase_pm_cache_snoop_enable - Allow CPU snoops on the GPU ++ * If the GPU does not have coherency this is a no-op ++ * @kbdev: Device pointer ++ * ++ * This function should be called after L2 power up. ++ */ ++ ++void kbase_pm_cache_snoop_enable(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_cache_snoop_disable - Prevent CPU snoops on the GPU ++ * If the GPU does not have coherency this is a no-op ++ * @kbdev: Device pointer ++ * ++ * This function should be called before L2 power off. ++ */ ++void kbase_pm_cache_snoop_disable(struct kbase_device *kbdev); ++ ++#endif /* _KBASE_BACKEND_PM_INTERNAL_H_ */ +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_metrics.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_metrics.c +new file mode 100755 +index 000000000000..024248ca7123 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_metrics.c +@@ -0,0 +1,401 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * Metrics for power management ++ */ ++ ++#include ++#include ++#include ++#include ++ ++/* When VSync is being hit aim for utilisation between 70-90% */ ++#define KBASE_PM_VSYNC_MIN_UTILISATION 70 ++#define KBASE_PM_VSYNC_MAX_UTILISATION 90 ++/* Otherwise aim for 10-40% */ ++#define KBASE_PM_NO_VSYNC_MIN_UTILISATION 10 ++#define KBASE_PM_NO_VSYNC_MAX_UTILISATION 40 ++ ++/* Shift used for kbasep_pm_metrics_data.time_busy/idle - units of (1 << 8) ns ++ * This gives a maximum period between samples of 2^(32+8)/100 ns = slightly ++ * under 11s. Exceeding this will cause overflow */ ++#define KBASE_PM_TIME_SHIFT 8 ++ ++/* Maximum time between sampling of utilization data, without resetting the ++ * counters. */ ++#define MALI_UTILIZATION_MAX_PERIOD 100000 /* ns = 100ms */ ++ ++#ifdef CONFIG_MALI_MIDGARD_DVFS ++static enum hrtimer_restart dvfs_callback(struct hrtimer *timer) ++{ ++ unsigned long flags; ++ struct kbasep_pm_metrics_data *metrics; ++ ++ KBASE_DEBUG_ASSERT(timer != NULL); ++ ++ metrics = container_of(timer, struct kbasep_pm_metrics_data, timer); ++ kbase_pm_get_dvfs_action(metrics->kbdev); ++ ++ spin_lock_irqsave(&metrics->lock, flags); ++ ++ if (metrics->timer_active) ++ hrtimer_start(timer, ++ HR_TIMER_DELAY_MSEC(metrics->kbdev->pm.dvfs_period), ++ HRTIMER_MODE_REL); ++ ++ spin_unlock_irqrestore(&metrics->lock, flags); ++ ++ return HRTIMER_NORESTART; ++} ++#endif /* CONFIG_MALI_MIDGARD_DVFS */ ++ ++int kbasep_pm_metrics_init(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ kbdev->pm.backend.metrics.kbdev = kbdev; ++ ++ kbdev->pm.backend.metrics.time_period_start = ktime_get(); ++ kbdev->pm.backend.metrics.time_busy = 0; ++ kbdev->pm.backend.metrics.time_idle = 0; ++ kbdev->pm.backend.metrics.prev_busy = 0; ++ kbdev->pm.backend.metrics.prev_idle = 0; ++ kbdev->pm.backend.metrics.gpu_active = false; ++ kbdev->pm.backend.metrics.active_cl_ctx[0] = 0; ++ kbdev->pm.backend.metrics.active_cl_ctx[1] = 0; ++ kbdev->pm.backend.metrics.active_gl_ctx[0] = 0; ++ kbdev->pm.backend.metrics.active_gl_ctx[1] = 0; ++ kbdev->pm.backend.metrics.busy_cl[0] = 0; ++ kbdev->pm.backend.metrics.busy_cl[1] = 0; ++ kbdev->pm.backend.metrics.busy_gl = 0; ++ ++ spin_lock_init(&kbdev->pm.backend.metrics.lock); ++ ++#ifdef CONFIG_MALI_MIDGARD_DVFS ++ kbdev->pm.backend.metrics.timer_active = true; ++ hrtimer_init(&kbdev->pm.backend.metrics.timer, CLOCK_MONOTONIC, ++ HRTIMER_MODE_REL); ++ kbdev->pm.backend.metrics.timer.function = dvfs_callback; ++ ++ hrtimer_start(&kbdev->pm.backend.metrics.timer, ++ HR_TIMER_DELAY_MSEC(kbdev->pm.dvfs_period), ++ HRTIMER_MODE_REL); ++#endif /* CONFIG_MALI_MIDGARD_DVFS */ ++ ++ return 0; ++} ++ ++KBASE_EXPORT_TEST_API(kbasep_pm_metrics_init); ++ ++void kbasep_pm_metrics_term(struct kbase_device *kbdev) ++{ ++#ifdef CONFIG_MALI_MIDGARD_DVFS ++ unsigned long flags; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); ++ kbdev->pm.backend.metrics.timer_active = false; ++ spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); ++ ++ hrtimer_cancel(&kbdev->pm.backend.metrics.timer); ++#endif /* CONFIG_MALI_MIDGARD_DVFS */ ++} ++ ++KBASE_EXPORT_TEST_API(kbasep_pm_metrics_term); ++ ++/* caller needs to hold kbdev->pm.backend.metrics.lock before calling this ++ * function ++ */ ++static void kbase_pm_get_dvfs_utilisation_calc(struct kbase_device *kbdev, ++ ktime_t now) ++{ ++ ktime_t diff; ++ ++ lockdep_assert_held(&kbdev->pm.backend.metrics.lock); ++ ++ diff = ktime_sub(now, kbdev->pm.backend.metrics.time_period_start); ++ if (ktime_to_ns(diff) < 0) ++ return; ++ ++ if (kbdev->pm.backend.metrics.gpu_active) { ++ u32 ns_time = (u32) (ktime_to_ns(diff) >> KBASE_PM_TIME_SHIFT); ++ ++ kbdev->pm.backend.metrics.time_busy += ns_time; ++ if (kbdev->pm.backend.metrics.active_cl_ctx[0]) ++ kbdev->pm.backend.metrics.busy_cl[0] += ns_time; ++ if (kbdev->pm.backend.metrics.active_cl_ctx[1]) ++ kbdev->pm.backend.metrics.busy_cl[1] += ns_time; ++ if (kbdev->pm.backend.metrics.active_gl_ctx[0]) ++ kbdev->pm.backend.metrics.busy_gl += ns_time; ++ if (kbdev->pm.backend.metrics.active_gl_ctx[1]) ++ kbdev->pm.backend.metrics.busy_gl += ns_time; ++ } else { ++ kbdev->pm.backend.metrics.time_idle += (u32) (ktime_to_ns(diff) ++ >> KBASE_PM_TIME_SHIFT); ++ } ++ ++ kbdev->pm.backend.metrics.time_period_start = now; ++} ++ ++#if defined(CONFIG_MALI_DEVFREQ) || defined(CONFIG_MALI_MIDGARD_DVFS) ++/* Caller needs to hold kbdev->pm.backend.metrics.lock before calling this ++ * function. ++ */ ++static void kbase_pm_reset_dvfs_utilisation_unlocked(struct kbase_device *kbdev, ++ ktime_t now) ++{ ++ /* Store previous value */ ++ kbdev->pm.backend.metrics.prev_idle = ++ kbdev->pm.backend.metrics.time_idle; ++ kbdev->pm.backend.metrics.prev_busy = ++ kbdev->pm.backend.metrics.time_busy; ++ ++ /* Reset current values */ ++ kbdev->pm.backend.metrics.time_period_start = now; ++ kbdev->pm.backend.metrics.time_idle = 0; ++ kbdev->pm.backend.metrics.time_busy = 0; ++ kbdev->pm.backend.metrics.busy_cl[0] = 0; ++ kbdev->pm.backend.metrics.busy_cl[1] = 0; ++ kbdev->pm.backend.metrics.busy_gl = 0; ++} ++ ++void kbase_pm_reset_dvfs_utilisation(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); ++ kbase_pm_reset_dvfs_utilisation_unlocked(kbdev, ktime_get()); ++ spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); ++} ++ ++void kbase_pm_get_dvfs_utilisation(struct kbase_device *kbdev, ++ unsigned long *total_out, unsigned long *busy_out) ++{ ++ ktime_t now = ktime_get(); ++ unsigned long flags, busy, total; ++ ++ spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); ++ kbase_pm_get_dvfs_utilisation_calc(kbdev, now); ++ ++ busy = kbdev->pm.backend.metrics.time_busy; ++ total = busy + kbdev->pm.backend.metrics.time_idle; ++ ++ /* Reset stats if older than MALI_UTILIZATION_MAX_PERIOD (default ++ * 100ms) */ ++ if (total >= MALI_UTILIZATION_MAX_PERIOD) { ++ kbase_pm_reset_dvfs_utilisation_unlocked(kbdev, now); ++ } else if (total < (MALI_UTILIZATION_MAX_PERIOD / 2)) { ++ total += kbdev->pm.backend.metrics.prev_idle + ++ kbdev->pm.backend.metrics.prev_busy; ++ busy += kbdev->pm.backend.metrics.prev_busy; ++ } ++ ++ *total_out = total; ++ *busy_out = busy; ++ spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); ++} ++#endif ++ ++#ifdef CONFIG_MALI_MIDGARD_DVFS ++ ++/* caller needs to hold kbdev->pm.backend.metrics.lock before calling this ++ * function ++ */ ++int kbase_pm_get_dvfs_utilisation_old(struct kbase_device *kbdev, ++ int *util_gl_share, ++ int util_cl_share[2], ++ ktime_t now) ++{ ++ int utilisation; ++ int busy; ++ ++ kbase_pm_get_dvfs_utilisation_calc(kbdev, now); ++ ++ if (kbdev->pm.backend.metrics.time_idle + ++ kbdev->pm.backend.metrics.time_busy == 0) { ++ /* No data - so we return NOP */ ++ utilisation = -1; ++ if (util_gl_share) ++ *util_gl_share = -1; ++ if (util_cl_share) { ++ util_cl_share[0] = -1; ++ util_cl_share[1] = -1; ++ } ++ goto out; ++ } ++ ++ utilisation = (100 * kbdev->pm.backend.metrics.time_busy) / ++ (kbdev->pm.backend.metrics.time_idle + ++ kbdev->pm.backend.metrics.time_busy); ++ ++ busy = kbdev->pm.backend.metrics.busy_gl + ++ kbdev->pm.backend.metrics.busy_cl[0] + ++ kbdev->pm.backend.metrics.busy_cl[1]; ++ ++ if (busy != 0) { ++ if (util_gl_share) ++ *util_gl_share = ++ (100 * kbdev->pm.backend.metrics.busy_gl) / ++ busy; ++ if (util_cl_share) { ++ util_cl_share[0] = ++ (100 * kbdev->pm.backend.metrics.busy_cl[0]) / ++ busy; ++ util_cl_share[1] = ++ (100 * kbdev->pm.backend.metrics.busy_cl[1]) / ++ busy; ++ } ++ } else { ++ if (util_gl_share) ++ *util_gl_share = -1; ++ if (util_cl_share) { ++ util_cl_share[0] = -1; ++ util_cl_share[1] = -1; ++ } ++ } ++ ++out: ++ return utilisation; ++} ++ ++void kbase_pm_get_dvfs_action(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ int utilisation, util_gl_share; ++ int util_cl_share[2]; ++ ktime_t now; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); ++ ++ now = ktime_get(); ++ ++ utilisation = kbase_pm_get_dvfs_utilisation_old(kbdev, &util_gl_share, ++ util_cl_share, now); ++ ++ if (utilisation < 0 || util_gl_share < 0 || util_cl_share[0] < 0 || ++ util_cl_share[1] < 0) { ++ utilisation = 0; ++ util_gl_share = 0; ++ util_cl_share[0] = 0; ++ util_cl_share[1] = 0; ++ goto out; ++ } ++ ++out: ++#ifdef CONFIG_MALI_MIDGARD_DVFS ++ kbase_platform_dvfs_event(kbdev, utilisation, util_gl_share, ++ util_cl_share); ++#endif /*CONFIG_MALI_MIDGARD_DVFS */ ++ ++ kbase_pm_reset_dvfs_utilisation_unlocked(kbdev, now); ++ ++ spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); ++} ++ ++bool kbase_pm_metrics_is_active(struct kbase_device *kbdev) ++{ ++ bool isactive; ++ unsigned long flags; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); ++ isactive = kbdev->pm.backend.metrics.timer_active; ++ spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); ++ ++ return isactive; ++} ++KBASE_EXPORT_TEST_API(kbase_pm_metrics_is_active); ++ ++#endif /* CONFIG_MALI_MIDGARD_DVFS */ ++ ++/** ++ * kbase_pm_metrics_active_calc - Update PM active counts based on currently ++ * running atoms ++ * @kbdev: Device pointer ++ * ++ * The caller must hold kbdev->pm.backend.metrics.lock ++ */ ++static void kbase_pm_metrics_active_calc(struct kbase_device *kbdev) ++{ ++ int js; ++ ++ lockdep_assert_held(&kbdev->pm.backend.metrics.lock); ++ ++ kbdev->pm.backend.metrics.active_gl_ctx[0] = 0; ++ kbdev->pm.backend.metrics.active_gl_ctx[1] = 0; ++ kbdev->pm.backend.metrics.active_cl_ctx[0] = 0; ++ kbdev->pm.backend.metrics.active_cl_ctx[1] = 0; ++ kbdev->pm.backend.metrics.gpu_active = false; ++ ++ for (js = 0; js < BASE_JM_MAX_NR_SLOTS; js++) { ++ struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, 0); ++ ++ /* Head atom may have just completed, so if it isn't running ++ * then try the next atom */ ++ if (katom && katom->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) ++ katom = kbase_gpu_inspect(kbdev, js, 1); ++ ++ if (katom && katom->gpu_rb_state == ++ KBASE_ATOM_GPU_RB_SUBMITTED) { ++ if (katom->core_req & BASE_JD_REQ_ONLY_COMPUTE) { ++ int device_nr = (katom->core_req & ++ BASE_JD_REQ_SPECIFIC_COHERENT_GROUP) ++ ? katom->device_nr : 0; ++ if (!WARN_ON(device_nr >= 2)) ++ kbdev->pm.backend.metrics. ++ active_cl_ctx[device_nr] = 1; ++ } else { ++ /* Slot 2 should not be running non-compute ++ * atoms */ ++ if (!WARN_ON(js >= 2)) ++ kbdev->pm.backend.metrics. ++ active_gl_ctx[js] = 1; ++ } ++ kbdev->pm.backend.metrics.gpu_active = true; ++ } ++ } ++} ++ ++/* called when job is submitted to or removed from a GPU slot */ ++void kbase_pm_metrics_update(struct kbase_device *kbdev, ktime_t *timestamp) ++{ ++ unsigned long flags; ++ ktime_t now; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); ++ ++ if (!timestamp) { ++ now = ktime_get(); ++ timestamp = &now; ++ } ++ ++ /* Track how long CL and/or GL jobs have been busy for */ ++ kbase_pm_get_dvfs_utilisation_calc(kbdev, *timestamp); ++ ++ kbase_pm_metrics_active_calc(kbdev); ++ ++ spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); ++} +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_policy.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_policy.c +new file mode 100755 +index 000000000000..075f020c66e6 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_policy.c +@@ -0,0 +1,973 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* ++ * Power policy API implementations ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++ ++static const struct kbase_pm_policy *const policy_list[] = { ++#ifdef CONFIG_MALI_NO_MALI ++ &kbase_pm_always_on_policy_ops, ++ &kbase_pm_demand_policy_ops, ++ &kbase_pm_coarse_demand_policy_ops, ++#if !MALI_CUSTOMER_RELEASE ++ &kbase_pm_demand_always_powered_policy_ops, ++ &kbase_pm_fast_start_policy_ops, ++#endif ++#else /* CONFIG_MALI_NO_MALI */ ++#if !PLATFORM_POWER_DOWN_ONLY ++ &kbase_pm_demand_policy_ops, ++#endif /* !PLATFORM_POWER_DOWN_ONLY */ ++ &kbase_pm_coarse_demand_policy_ops, ++ &kbase_pm_always_on_policy_ops, ++#if !MALI_CUSTOMER_RELEASE ++#if !PLATFORM_POWER_DOWN_ONLY ++ &kbase_pm_demand_always_powered_policy_ops, ++ &kbase_pm_fast_start_policy_ops, ++#endif /* !PLATFORM_POWER_DOWN_ONLY */ ++#endif ++#endif /* CONFIG_MALI_NO_MALI */ ++}; ++ ++/* The number of policies available in the system. ++ * This is derived from the number of functions listed in policy_get_functions. ++ */ ++#define POLICY_COUNT (sizeof(policy_list)/sizeof(*policy_list)) ++ ++ ++/* Function IDs for looking up Timeline Trace codes in ++ * kbase_pm_change_state_trace_code */ ++enum kbase_pm_func_id { ++ KBASE_PM_FUNC_ID_REQUEST_CORES_START, ++ KBASE_PM_FUNC_ID_REQUEST_CORES_END, ++ KBASE_PM_FUNC_ID_RELEASE_CORES_START, ++ KBASE_PM_FUNC_ID_RELEASE_CORES_END, ++ /* Note: kbase_pm_unrequest_cores() is on the slow path, and we neither ++ * expect to hit it nor tend to hit it very much anyway. We can detect ++ * whether we need more instrumentation by a difference between ++ * PM_CHECKTRANS events and PM_SEND/HANDLE_EVENT. */ ++ ++ /* Must be the last */ ++ KBASE_PM_FUNC_ID_COUNT ++}; ++ ++ ++/* State changes during request/unrequest/release-ing cores */ ++enum { ++ KBASE_PM_CHANGE_STATE_SHADER = (1u << 0), ++ KBASE_PM_CHANGE_STATE_TILER = (1u << 1), ++ ++ /* These two must be last */ ++ KBASE_PM_CHANGE_STATE_MASK = (KBASE_PM_CHANGE_STATE_TILER | ++ KBASE_PM_CHANGE_STATE_SHADER), ++ KBASE_PM_CHANGE_STATE_COUNT = KBASE_PM_CHANGE_STATE_MASK + 1 ++}; ++typedef u32 kbase_pm_change_state; ++ ++ ++#ifdef CONFIG_MALI_TRACE_TIMELINE ++/* Timeline Trace code lookups for each function */ ++static u32 kbase_pm_change_state_trace_code[KBASE_PM_FUNC_ID_COUNT] ++ [KBASE_PM_CHANGE_STATE_COUNT] = { ++ /* kbase_pm_request_cores */ ++ [KBASE_PM_FUNC_ID_REQUEST_CORES_START][0] = 0, ++ [KBASE_PM_FUNC_ID_REQUEST_CORES_START][KBASE_PM_CHANGE_STATE_SHADER] = ++ SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_START, ++ [KBASE_PM_FUNC_ID_REQUEST_CORES_START][KBASE_PM_CHANGE_STATE_TILER] = ++ SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_TILER_START, ++ [KBASE_PM_FUNC_ID_REQUEST_CORES_START][KBASE_PM_CHANGE_STATE_SHADER | ++ KBASE_PM_CHANGE_STATE_TILER] = ++ SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_TILER_START, ++ ++ [KBASE_PM_FUNC_ID_REQUEST_CORES_END][0] = 0, ++ [KBASE_PM_FUNC_ID_REQUEST_CORES_END][KBASE_PM_CHANGE_STATE_SHADER] = ++ SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_END, ++ [KBASE_PM_FUNC_ID_REQUEST_CORES_END][KBASE_PM_CHANGE_STATE_TILER] = ++ SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_TILER_END, ++ [KBASE_PM_FUNC_ID_REQUEST_CORES_END][KBASE_PM_CHANGE_STATE_SHADER | ++ KBASE_PM_CHANGE_STATE_TILER] = ++ SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_TILER_END, ++ ++ /* kbase_pm_release_cores */ ++ [KBASE_PM_FUNC_ID_RELEASE_CORES_START][0] = 0, ++ [KBASE_PM_FUNC_ID_RELEASE_CORES_START][KBASE_PM_CHANGE_STATE_SHADER] = ++ SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_START, ++ [KBASE_PM_FUNC_ID_RELEASE_CORES_START][KBASE_PM_CHANGE_STATE_TILER] = ++ SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_TILER_START, ++ [KBASE_PM_FUNC_ID_RELEASE_CORES_START][KBASE_PM_CHANGE_STATE_SHADER | ++ KBASE_PM_CHANGE_STATE_TILER] = ++ SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_TILER_START, ++ ++ [KBASE_PM_FUNC_ID_RELEASE_CORES_END][0] = 0, ++ [KBASE_PM_FUNC_ID_RELEASE_CORES_END][KBASE_PM_CHANGE_STATE_SHADER] = ++ SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_END, ++ [KBASE_PM_FUNC_ID_RELEASE_CORES_END][KBASE_PM_CHANGE_STATE_TILER] = ++ SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_TILER_END, ++ [KBASE_PM_FUNC_ID_RELEASE_CORES_END][KBASE_PM_CHANGE_STATE_SHADER | ++ KBASE_PM_CHANGE_STATE_TILER] = ++ SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_TILER_END ++}; ++ ++static inline void kbase_timeline_pm_cores_func(struct kbase_device *kbdev, ++ enum kbase_pm_func_id func_id, ++ kbase_pm_change_state state) ++{ ++ int trace_code; ++ ++ KBASE_DEBUG_ASSERT(func_id >= 0 && func_id < KBASE_PM_FUNC_ID_COUNT); ++ KBASE_DEBUG_ASSERT(state != 0 && (state & KBASE_PM_CHANGE_STATE_MASK) == ++ state); ++ ++ trace_code = kbase_pm_change_state_trace_code[func_id][state]; ++ KBASE_TIMELINE_PM_CHECKTRANS(kbdev, trace_code); ++} ++ ++#else /* CONFIG_MALI_TRACE_TIMELINE */ ++static inline void kbase_timeline_pm_cores_func(struct kbase_device *kbdev, ++ enum kbase_pm_func_id func_id, kbase_pm_change_state state) ++{ ++} ++ ++#endif /* CONFIG_MALI_TRACE_TIMELINE */ ++ ++/** ++ * kbasep_pm_do_poweroff_cores - Process a poweroff request and power down any ++ * requested shader cores ++ * @kbdev: Device pointer ++ */ ++static void kbasep_pm_do_poweroff_cores(struct kbase_device *kbdev) ++{ ++ u64 prev_shader_state = kbdev->pm.backend.desired_shader_state; ++ u64 prev_tiler_state = kbdev->pm.backend.desired_tiler_state; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ kbdev->pm.backend.desired_shader_state &= ++ ~kbdev->pm.backend.shader_poweroff_pending; ++ kbdev->pm.backend.desired_tiler_state &= ++ ~kbdev->pm.backend.tiler_poweroff_pending; ++ ++ kbdev->pm.backend.shader_poweroff_pending = 0; ++ kbdev->pm.backend.tiler_poweroff_pending = 0; ++ ++ if (prev_shader_state != kbdev->pm.backend.desired_shader_state || ++ prev_tiler_state != ++ kbdev->pm.backend.desired_tiler_state || ++ kbdev->pm.backend.ca_in_transition) { ++ bool cores_are_available; ++ ++ KBASE_TIMELINE_PM_CHECKTRANS(kbdev, ++ SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_DEFERRED_START); ++ cores_are_available = kbase_pm_check_transitions_nolock(kbdev); ++ KBASE_TIMELINE_PM_CHECKTRANS(kbdev, ++ SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_DEFERRED_END); ++ ++ /* Don't need 'cores_are_available', ++ * because we don't return anything */ ++ CSTD_UNUSED(cores_are_available); ++ } ++} ++ ++static enum hrtimer_restart ++kbasep_pm_do_gpu_poweroff_callback(struct hrtimer *timer) ++{ ++ struct kbase_device *kbdev; ++ unsigned long flags; ++ ++ kbdev = container_of(timer, struct kbase_device, ++ pm.backend.gpu_poweroff_timer); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ /* It is safe for this call to do nothing if the work item is already ++ * queued. The worker function will read the must up-to-date state of ++ * kbdev->pm.backend.gpu_poweroff_pending under lock. ++ * ++ * If a state change occurs while the worker function is processing, ++ * this call will succeed as a work item can be requeued once it has ++ * started processing. ++ */ ++ if (kbdev->pm.backend.gpu_poweroff_pending) ++ queue_work(kbdev->pm.backend.gpu_poweroff_wq, ++ &kbdev->pm.backend.gpu_poweroff_work); ++ ++ if (kbdev->pm.backend.shader_poweroff_pending || ++ kbdev->pm.backend.tiler_poweroff_pending) { ++ kbdev->pm.backend.shader_poweroff_pending_time--; ++ ++ KBASE_DEBUG_ASSERT( ++ kbdev->pm.backend.shader_poweroff_pending_time ++ >= 0); ++ ++ if (!kbdev->pm.backend.shader_poweroff_pending_time) ++ kbasep_pm_do_poweroff_cores(kbdev); ++ } ++ ++ if (kbdev->pm.backend.poweroff_timer_needed) { ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ hrtimer_add_expires(timer, kbdev->pm.gpu_poweroff_time); ++ ++ return HRTIMER_RESTART; ++ } ++ ++ kbdev->pm.backend.poweroff_timer_running = false; ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ return HRTIMER_NORESTART; ++} ++ ++static void kbasep_pm_do_gpu_poweroff_wq(struct work_struct *data) ++{ ++ unsigned long flags; ++ struct kbase_device *kbdev; ++ bool do_poweroff = false; ++ ++ kbdev = container_of(data, struct kbase_device, ++ pm.backend.gpu_poweroff_work); ++ ++ mutex_lock(&kbdev->pm.lock); ++ ++ if (kbdev->pm.backend.gpu_poweroff_pending == 0) { ++ mutex_unlock(&kbdev->pm.lock); ++ return; ++ } ++ ++ kbdev->pm.backend.gpu_poweroff_pending--; ++ ++ if (kbdev->pm.backend.gpu_poweroff_pending > 0) { ++ mutex_unlock(&kbdev->pm.lock); ++ return; ++ } ++ ++ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_poweroff_pending == 0); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ /* Only power off the GPU if a request is still pending */ ++ if (!kbdev->pm.backend.pm_current_policy->get_core_active(kbdev)) ++ do_poweroff = true; ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ if (do_poweroff) { ++ kbdev->pm.backend.poweroff_timer_needed = false; ++ hrtimer_cancel(&kbdev->pm.backend.gpu_poweroff_timer); ++ kbdev->pm.backend.poweroff_timer_running = false; ++ ++ /* Power off the GPU */ ++ kbase_pm_do_poweroff(kbdev, false); ++ } ++ ++ mutex_unlock(&kbdev->pm.lock); ++} ++ ++int kbase_pm_policy_init(struct kbase_device *kbdev) ++{ ++ struct workqueue_struct *wq; ++ ++ wq = alloc_workqueue("kbase_pm_do_poweroff", ++ WQ_HIGHPRI | WQ_UNBOUND, 1); ++ if (!wq) ++ return -ENOMEM; ++ ++ kbdev->pm.backend.gpu_poweroff_wq = wq; ++ INIT_WORK(&kbdev->pm.backend.gpu_poweroff_work, ++ kbasep_pm_do_gpu_poweroff_wq); ++ hrtimer_init(&kbdev->pm.backend.gpu_poweroff_timer, ++ CLOCK_MONOTONIC, HRTIMER_MODE_REL); ++ kbdev->pm.backend.gpu_poweroff_timer.function = ++ kbasep_pm_do_gpu_poweroff_callback; ++ kbdev->pm.backend.pm_current_policy = policy_list[0]; ++ kbdev->pm.backend.pm_current_policy->init(kbdev); ++ kbdev->pm.gpu_poweroff_time = ++ HR_TIMER_DELAY_NSEC(DEFAULT_PM_GPU_POWEROFF_TICK_NS); ++ kbdev->pm.poweroff_shader_ticks = DEFAULT_PM_POWEROFF_TICK_SHADER; ++ kbdev->pm.poweroff_gpu_ticks = DEFAULT_PM_POWEROFF_TICK_GPU; ++ ++ return 0; ++} ++ ++void kbase_pm_policy_term(struct kbase_device *kbdev) ++{ ++ kbdev->pm.backend.pm_current_policy->term(kbdev); ++ destroy_workqueue(kbdev->pm.backend.gpu_poweroff_wq); ++} ++ ++void kbase_pm_cancel_deferred_poweroff(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ ++ lockdep_assert_held(&kbdev->pm.lock); ++ ++ kbdev->pm.backend.poweroff_timer_needed = false; ++ hrtimer_cancel(&kbdev->pm.backend.gpu_poweroff_timer); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbdev->pm.backend.poweroff_timer_running = false; ++ ++ /* If wq is already running but is held off by pm.lock, make sure it has ++ * no effect */ ++ kbdev->pm.backend.gpu_poweroff_pending = 0; ++ ++ kbdev->pm.backend.shader_poweroff_pending = 0; ++ kbdev->pm.backend.tiler_poweroff_pending = 0; ++ kbdev->pm.backend.shader_poweroff_pending_time = 0; ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++} ++ ++void kbase_pm_update_active(struct kbase_device *kbdev) ++{ ++ struct kbase_pm_device_data *pm = &kbdev->pm; ++ struct kbase_pm_backend_data *backend = &pm->backend; ++ unsigned long flags; ++ bool active; ++ ++ lockdep_assert_held(&pm->lock); ++ ++ /* pm_current_policy will never be NULL while pm.lock is held */ ++ KBASE_DEBUG_ASSERT(backend->pm_current_policy); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ active = backend->pm_current_policy->get_core_active(kbdev); ++ ++ if (active) { ++ if (backend->gpu_poweroff_pending) { ++ /* Cancel any pending power off request */ ++ backend->gpu_poweroff_pending = 0; ++ ++ /* If a request was pending then the GPU was still ++ * powered, so no need to continue */ ++ if (!kbdev->poweroff_pending) { ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, ++ flags); ++ return; ++ } ++ } ++ ++ if (!backend->poweroff_timer_running && !backend->gpu_powered && ++ (pm->poweroff_gpu_ticks || ++ pm->poweroff_shader_ticks)) { ++ backend->poweroff_timer_needed = true; ++ backend->poweroff_timer_running = true; ++ hrtimer_start(&backend->gpu_poweroff_timer, ++ pm->gpu_poweroff_time, ++ HRTIMER_MODE_REL); ++ } ++ ++ /* Power on the GPU and any cores requested by the policy */ ++ if (pm->backend.poweroff_wait_in_progress) { ++ pm->backend.poweron_required = true; ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ } else { ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ kbase_pm_do_poweron(kbdev, false); ++ } ++ } else { ++ /* It is an error for the power policy to power off the GPU ++ * when there are contexts active */ ++ KBASE_DEBUG_ASSERT(pm->active_count == 0); ++ ++ if (backend->shader_poweroff_pending || ++ backend->tiler_poweroff_pending) { ++ backend->shader_poweroff_pending = 0; ++ backend->tiler_poweroff_pending = 0; ++ backend->shader_poweroff_pending_time = 0; ++ } ++ ++ /* Request power off */ ++ if (pm->backend.gpu_powered) { ++ if (pm->poweroff_gpu_ticks) { ++ backend->gpu_poweroff_pending = ++ pm->poweroff_gpu_ticks; ++ backend->poweroff_timer_needed = true; ++ if (!backend->poweroff_timer_running) { ++ /* Start timer if not running (eg if ++ * power policy has been changed from ++ * always_on to something else). This ++ * will ensure the GPU is actually ++ * powered off */ ++ backend->poweroff_timer_running ++ = true; ++ hrtimer_start( ++ &backend->gpu_poweroff_timer, ++ pm->gpu_poweroff_time, ++ HRTIMER_MODE_REL); ++ } ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, ++ flags); ++ } else { ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, ++ flags); ++ ++ /* Power off the GPU immediately */ ++ kbase_pm_do_poweroff(kbdev, false); ++ } ++ } else { ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ } ++ } ++} ++ ++void kbase_pm_update_cores_state_nolock(struct kbase_device *kbdev) ++{ ++ u64 desired_bitmap; ++ u64 desired_tiler_bitmap; ++ bool cores_are_available; ++ bool do_poweroff = false; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (kbdev->pm.backend.pm_current_policy == NULL) ++ return; ++ if (kbdev->pm.backend.poweroff_wait_in_progress) ++ return; ++ ++ if (kbdev->protected_mode_transition && !kbdev->shader_needed_bitmap && ++ !kbdev->shader_inuse_bitmap && !kbdev->tiler_needed_cnt ++ && !kbdev->tiler_inuse_cnt) { ++ /* We are trying to change in/out of protected mode - force all ++ * cores off so that the L2 powers down */ ++ desired_bitmap = 0; ++ desired_tiler_bitmap = 0; ++ } else { ++ desired_bitmap = ++ kbdev->pm.backend.pm_current_policy->get_core_mask(kbdev); ++ desired_bitmap &= kbase_pm_ca_get_core_mask(kbdev); ++ ++ if (kbdev->tiler_needed_cnt > 0 || kbdev->tiler_inuse_cnt > 0) ++ desired_tiler_bitmap = 1; ++ else ++ desired_tiler_bitmap = 0; ++ ++ if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_XAFFINITY)) { ++ /* Unless XAFFINITY is supported, enable core 0 if tiler ++ * required, regardless of core availability */ ++ if (kbdev->tiler_needed_cnt > 0 || ++ kbdev->tiler_inuse_cnt > 0) ++ desired_bitmap |= 1; ++ } ++ } ++ ++ if (kbdev->pm.backend.desired_shader_state != desired_bitmap) ++ KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_DESIRED, NULL, NULL, 0u, ++ (u32)desired_bitmap); ++ /* Are any cores being powered on? */ ++ if (~kbdev->pm.backend.desired_shader_state & desired_bitmap || ++ ~kbdev->pm.backend.desired_tiler_state & desired_tiler_bitmap || ++ kbdev->pm.backend.ca_in_transition) { ++ /* Check if we are powering off any cores before updating shader ++ * state */ ++ if (kbdev->pm.backend.desired_shader_state & ~desired_bitmap || ++ kbdev->pm.backend.desired_tiler_state & ++ ~desired_tiler_bitmap) { ++ /* Start timer to power off cores */ ++ kbdev->pm.backend.shader_poweroff_pending |= ++ (kbdev->pm.backend.desired_shader_state & ++ ~desired_bitmap); ++ kbdev->pm.backend.tiler_poweroff_pending |= ++ (kbdev->pm.backend.desired_tiler_state & ++ ~desired_tiler_bitmap); ++ ++ if (kbdev->pm.poweroff_shader_ticks && ++ !kbdev->protected_mode_transition) ++ kbdev->pm.backend.shader_poweroff_pending_time = ++ kbdev->pm.poweroff_shader_ticks; ++ else ++ do_poweroff = true; ++ } ++ ++ kbdev->pm.backend.desired_shader_state = desired_bitmap; ++ kbdev->pm.backend.desired_tiler_state = desired_tiler_bitmap; ++ ++ /* If any cores are being powered on, transition immediately */ ++ cores_are_available = kbase_pm_check_transitions_nolock(kbdev); ++ } else if (kbdev->pm.backend.desired_shader_state & ~desired_bitmap || ++ kbdev->pm.backend.desired_tiler_state & ++ ~desired_tiler_bitmap) { ++ /* Start timer to power off cores */ ++ kbdev->pm.backend.shader_poweroff_pending |= ++ (kbdev->pm.backend.desired_shader_state & ++ ~desired_bitmap); ++ kbdev->pm.backend.tiler_poweroff_pending |= ++ (kbdev->pm.backend.desired_tiler_state & ++ ~desired_tiler_bitmap); ++ if (kbdev->pm.poweroff_shader_ticks && ++ !kbdev->protected_mode_transition) ++ kbdev->pm.backend.shader_poweroff_pending_time = ++ kbdev->pm.poweroff_shader_ticks; ++ else ++ kbasep_pm_do_poweroff_cores(kbdev); ++ } else if (kbdev->pm.active_count == 0 && desired_bitmap != 0 && ++ desired_tiler_bitmap != 0 && ++ kbdev->pm.backend.poweroff_timer_needed) { ++ /* If power policy is keeping cores on despite there being no ++ * active contexts then disable poweroff timer as it isn't ++ * required. ++ * Only reset poweroff_timer_needed if we're not in the middle ++ * of the power off callback */ ++ kbdev->pm.backend.poweroff_timer_needed = false; ++ } ++ ++ /* Ensure timer does not power off wanted cores and make sure to power ++ * off unwanted cores */ ++ if (kbdev->pm.backend.shader_poweroff_pending || ++ kbdev->pm.backend.tiler_poweroff_pending) { ++ kbdev->pm.backend.shader_poweroff_pending &= ++ ~(kbdev->pm.backend.desired_shader_state & ++ desired_bitmap); ++ kbdev->pm.backend.tiler_poweroff_pending &= ++ ~(kbdev->pm.backend.desired_tiler_state & ++ desired_tiler_bitmap); ++ ++ if (!kbdev->pm.backend.shader_poweroff_pending && ++ !kbdev->pm.backend.tiler_poweroff_pending) ++ kbdev->pm.backend.shader_poweroff_pending_time = 0; ++ } ++ ++ /* Shader poweroff is deferred to the end of the function, to eliminate ++ * issues caused by the core availability policy recursing into this ++ * function */ ++ if (do_poweroff) ++ kbasep_pm_do_poweroff_cores(kbdev); ++ ++ /* Don't need 'cores_are_available', because we don't return anything */ ++ CSTD_UNUSED(cores_are_available); ++} ++ ++void kbase_pm_update_cores_state(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ kbase_pm_update_cores_state_nolock(kbdev); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++} ++ ++int kbase_pm_list_policies(const struct kbase_pm_policy * const **list) ++{ ++ if (!list) ++ return POLICY_COUNT; ++ ++ *list = policy_list; ++ ++ return POLICY_COUNT; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_list_policies); ++ ++const struct kbase_pm_policy *kbase_pm_get_policy(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ return kbdev->pm.backend.pm_current_policy; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_get_policy); ++ ++void kbase_pm_set_policy(struct kbase_device *kbdev, ++ const struct kbase_pm_policy *new_policy) ++{ ++ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; ++ const struct kbase_pm_policy *old_policy; ++ unsigned long flags; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(new_policy != NULL); ++ ++ KBASE_TRACE_ADD(kbdev, PM_SET_POLICY, NULL, NULL, 0u, new_policy->id); ++ ++ /* During a policy change we pretend the GPU is active */ ++ /* A suspend won't happen here, because we're in a syscall from a ++ * userspace thread */ ++ kbase_pm_context_active(kbdev); ++ ++ mutex_lock(&js_devdata->runpool_mutex); ++ mutex_lock(&kbdev->pm.lock); ++ ++ /* Remove the policy to prevent IRQ handlers from working on it */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ old_policy = kbdev->pm.backend.pm_current_policy; ++ kbdev->pm.backend.pm_current_policy = NULL; ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ KBASE_TRACE_ADD(kbdev, PM_CURRENT_POLICY_TERM, NULL, NULL, 0u, ++ old_policy->id); ++ if (old_policy->term) ++ old_policy->term(kbdev); ++ ++ KBASE_TRACE_ADD(kbdev, PM_CURRENT_POLICY_INIT, NULL, NULL, 0u, ++ new_policy->id); ++ if (new_policy->init) ++ new_policy->init(kbdev); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbdev->pm.backend.pm_current_policy = new_policy; ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ /* If any core power state changes were previously attempted, but ++ * couldn't be made because the policy was changing (current_policy was ++ * NULL), then re-try them here. */ ++ kbase_pm_update_active(kbdev); ++ kbase_pm_update_cores_state(kbdev); ++ ++ mutex_unlock(&kbdev->pm.lock); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ ++ /* Now the policy change is finished, we release our fake context active ++ * reference */ ++ kbase_pm_context_idle(kbdev); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_set_policy); ++ ++/* Check whether a state change has finished, and trace it as completed */ ++static void ++kbase_pm_trace_check_and_finish_state_change(struct kbase_device *kbdev) ++{ ++ if ((kbdev->shader_available_bitmap & ++ kbdev->pm.backend.desired_shader_state) ++ == kbdev->pm.backend.desired_shader_state && ++ (kbdev->tiler_available_bitmap & ++ kbdev->pm.backend.desired_tiler_state) ++ == kbdev->pm.backend.desired_tiler_state) ++ kbase_timeline_pm_check_handle_event(kbdev, ++ KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED); ++} ++ ++void kbase_pm_request_cores(struct kbase_device *kbdev, ++ bool tiler_required, u64 shader_cores) ++{ ++ u64 cores; ++ ++ kbase_pm_change_state change_gpu_state = 0u; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ cores = shader_cores; ++ while (cores) { ++ int bitnum = fls64(cores) - 1; ++ u64 bit = 1ULL << bitnum; ++ ++ /* It should be almost impossible for this to overflow. It would ++ * require 2^32 atoms to request a particular core, which would ++ * require 2^24 contexts to submit. This would require an amount ++ * of memory that is impossible on a 32-bit system and extremely ++ * unlikely on a 64-bit system. */ ++ int cnt = ++kbdev->shader_needed_cnt[bitnum]; ++ ++ if (1 == cnt) { ++ kbdev->shader_needed_bitmap |= bit; ++ change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER; ++ } ++ ++ cores &= ~bit; ++ } ++ ++ if (tiler_required) { ++ int cnt = ++kbdev->tiler_needed_cnt; ++ ++ if (1 == cnt) ++ change_gpu_state |= KBASE_PM_CHANGE_STATE_TILER; ++ ++ KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt != 0); ++ } ++ ++ if (change_gpu_state) { ++ KBASE_TRACE_ADD(kbdev, PM_REQUEST_CHANGE_SHADER_NEEDED, NULL, ++ NULL, 0u, (u32) kbdev->shader_needed_bitmap); ++ ++ kbase_timeline_pm_cores_func(kbdev, ++ KBASE_PM_FUNC_ID_REQUEST_CORES_START, ++ change_gpu_state); ++ kbase_pm_update_cores_state_nolock(kbdev); ++ kbase_timeline_pm_cores_func(kbdev, ++ KBASE_PM_FUNC_ID_REQUEST_CORES_END, ++ change_gpu_state); ++ } ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_request_cores); ++ ++void kbase_pm_unrequest_cores(struct kbase_device *kbdev, ++ bool tiler_required, u64 shader_cores) ++{ ++ kbase_pm_change_state change_gpu_state = 0u; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ while (shader_cores) { ++ int bitnum = fls64(shader_cores) - 1; ++ u64 bit = 1ULL << bitnum; ++ int cnt; ++ ++ KBASE_DEBUG_ASSERT(kbdev->shader_needed_cnt[bitnum] > 0); ++ ++ cnt = --kbdev->shader_needed_cnt[bitnum]; ++ ++ if (0 == cnt) { ++ kbdev->shader_needed_bitmap &= ~bit; ++ ++ change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER; ++ } ++ ++ shader_cores &= ~bit; ++ } ++ ++ if (tiler_required) { ++ int cnt; ++ ++ KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt > 0); ++ ++ cnt = --kbdev->tiler_needed_cnt; ++ ++ if (0 == cnt) ++ change_gpu_state |= KBASE_PM_CHANGE_STATE_TILER; ++ } ++ ++ if (change_gpu_state) { ++ KBASE_TRACE_ADD(kbdev, PM_UNREQUEST_CHANGE_SHADER_NEEDED, NULL, ++ NULL, 0u, (u32) kbdev->shader_needed_bitmap); ++ ++ kbase_pm_update_cores_state_nolock(kbdev); ++ ++ /* Trace that any state change effectively completes immediately ++ * - no-one will wait on the state change */ ++ kbase_pm_trace_check_and_finish_state_change(kbdev); ++ } ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_unrequest_cores); ++ ++enum kbase_pm_cores_ready ++kbase_pm_register_inuse_cores(struct kbase_device *kbdev, ++ bool tiler_required, u64 shader_cores) ++{ ++ u64 prev_shader_needed; /* Just for tracing */ ++ u64 prev_shader_inuse; /* Just for tracing */ ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ prev_shader_needed = kbdev->shader_needed_bitmap; ++ prev_shader_inuse = kbdev->shader_inuse_bitmap; ++ ++ /* If desired_shader_state does not contain the requested cores, then ++ * power management is not attempting to powering those cores (most ++ * likely due to core availability policy) and a new job affinity must ++ * be chosen */ ++ if ((kbdev->pm.backend.desired_shader_state & shader_cores) != ++ shader_cores) { ++ return (kbdev->pm.backend.poweroff_wait_in_progress || ++ kbdev->pm.backend.pm_current_policy == NULL) ? ++ KBASE_CORES_NOT_READY : KBASE_NEW_AFFINITY; ++ } ++ ++ if ((kbdev->shader_available_bitmap & shader_cores) != shader_cores || ++ (tiler_required && !kbdev->tiler_available_bitmap)) { ++ /* Trace ongoing core transition */ ++ kbase_timeline_pm_l2_transition_start(kbdev); ++ return KBASE_CORES_NOT_READY; ++ } ++ ++ /* If we started to trace a state change, then trace it has being ++ * finished by now, at the very latest */ ++ kbase_pm_trace_check_and_finish_state_change(kbdev); ++ /* Trace core transition done */ ++ kbase_timeline_pm_l2_transition_done(kbdev); ++ ++ while (shader_cores) { ++ int bitnum = fls64(shader_cores) - 1; ++ u64 bit = 1ULL << bitnum; ++ int cnt; ++ ++ KBASE_DEBUG_ASSERT(kbdev->shader_needed_cnt[bitnum] > 0); ++ ++ cnt = --kbdev->shader_needed_cnt[bitnum]; ++ ++ if (0 == cnt) ++ kbdev->shader_needed_bitmap &= ~bit; ++ ++ /* shader_inuse_cnt should not overflow because there can only ++ * be a very limited number of jobs on the h/w at one time */ ++ ++ kbdev->shader_inuse_cnt[bitnum]++; ++ kbdev->shader_inuse_bitmap |= bit; ++ ++ shader_cores &= ~bit; ++ } ++ ++ if (tiler_required) { ++ KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt > 0); ++ ++ --kbdev->tiler_needed_cnt; ++ ++ kbdev->tiler_inuse_cnt++; ++ ++ KBASE_DEBUG_ASSERT(kbdev->tiler_inuse_cnt != 0); ++ } ++ ++ if (prev_shader_needed != kbdev->shader_needed_bitmap) ++ KBASE_TRACE_ADD(kbdev, PM_REGISTER_CHANGE_SHADER_NEEDED, NULL, ++ NULL, 0u, (u32) kbdev->shader_needed_bitmap); ++ ++ if (prev_shader_inuse != kbdev->shader_inuse_bitmap) ++ KBASE_TRACE_ADD(kbdev, PM_REGISTER_CHANGE_SHADER_INUSE, NULL, ++ NULL, 0u, (u32) kbdev->shader_inuse_bitmap); ++ ++ return KBASE_CORES_READY; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_register_inuse_cores); ++ ++void kbase_pm_release_cores(struct kbase_device *kbdev, ++ bool tiler_required, u64 shader_cores) ++{ ++ kbase_pm_change_state change_gpu_state = 0u; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ while (shader_cores) { ++ int bitnum = fls64(shader_cores) - 1; ++ u64 bit = 1ULL << bitnum; ++ int cnt; ++ ++ KBASE_DEBUG_ASSERT(kbdev->shader_inuse_cnt[bitnum] > 0); ++ ++ cnt = --kbdev->shader_inuse_cnt[bitnum]; ++ ++ if (0 == cnt) { ++ kbdev->shader_inuse_bitmap &= ~bit; ++ change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER; ++ } ++ ++ shader_cores &= ~bit; ++ } ++ ++ if (tiler_required) { ++ int cnt; ++ ++ KBASE_DEBUG_ASSERT(kbdev->tiler_inuse_cnt > 0); ++ ++ cnt = --kbdev->tiler_inuse_cnt; ++ ++ if (0 == cnt) ++ change_gpu_state |= KBASE_PM_CHANGE_STATE_TILER; ++ } ++ ++ if (change_gpu_state) { ++ KBASE_TRACE_ADD(kbdev, PM_RELEASE_CHANGE_SHADER_INUSE, NULL, ++ NULL, 0u, (u32) kbdev->shader_inuse_bitmap); ++ ++ kbase_timeline_pm_cores_func(kbdev, ++ KBASE_PM_FUNC_ID_RELEASE_CORES_START, ++ change_gpu_state); ++ kbase_pm_update_cores_state_nolock(kbdev); ++ kbase_timeline_pm_cores_func(kbdev, ++ KBASE_PM_FUNC_ID_RELEASE_CORES_END, ++ change_gpu_state); ++ ++ /* Trace that any state change completed immediately */ ++ kbase_pm_trace_check_and_finish_state_change(kbdev); ++ } ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_release_cores); ++ ++void kbase_pm_request_cores_sync(struct kbase_device *kbdev, ++ bool tiler_required, ++ u64 shader_cores) ++{ ++ unsigned long flags; ++ ++ kbase_pm_wait_for_poweroff_complete(kbdev); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_pm_request_cores(kbdev, tiler_required, shader_cores); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ kbase_pm_check_transitions_sync(kbdev); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_request_cores_sync); ++ ++void kbase_pm_request_l2_caches(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ u32 prior_l2_users_count; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ prior_l2_users_count = kbdev->l2_users_count++; ++ ++ KBASE_DEBUG_ASSERT(kbdev->l2_users_count != 0); ++ ++ /* if the GPU is reset while the l2 is on, l2 will be off but ++ * prior_l2_users_count will be > 0. l2_available_bitmap will have been ++ * set to 0 though by kbase_pm_init_hw */ ++ if (!prior_l2_users_count || !kbdev->l2_available_bitmap) ++ kbase_pm_check_transitions_nolock(kbdev); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ wait_event(kbdev->pm.backend.l2_powered_wait, ++ kbdev->pm.backend.l2_powered == 1); ++ ++ /* Trace that any state change completed immediately */ ++ kbase_pm_trace_check_and_finish_state_change(kbdev); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_request_l2_caches); ++ ++void kbase_pm_request_l2_caches_l2_is_on(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ kbdev->l2_users_count++; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_request_l2_caches_l2_is_on); ++ ++void kbase_pm_release_l2_caches(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ KBASE_DEBUG_ASSERT(kbdev->l2_users_count > 0); ++ ++ --kbdev->l2_users_count; ++ ++ if (!kbdev->l2_users_count) { ++ kbase_pm_check_transitions_nolock(kbdev); ++ /* Trace that any state change completed immediately */ ++ kbase_pm_trace_check_and_finish_state_change(kbdev); ++ } ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_release_l2_caches); +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_policy.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_policy.h +new file mode 100755 +index 000000000000..611a90e66e65 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_policy.h +@@ -0,0 +1,227 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* ++ * Power policy API definitions ++ */ ++ ++#ifndef _KBASE_PM_POLICY_H_ ++#define _KBASE_PM_POLICY_H_ ++ ++/** ++ * kbase_pm_policy_init - Initialize power policy framework ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * Must be called before calling any other policy function ++ * ++ * Return: 0 if the power policy framework was successfully ++ * initialized, -errno otherwise. ++ */ ++int kbase_pm_policy_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_policy_term - Terminate power policy framework ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_policy_term(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_update_active - Update the active power state of the GPU ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * Calls into the current power policy ++ */ ++void kbase_pm_update_active(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_update_cores - Update the desired core state of the GPU ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * Calls into the current power policy ++ */ ++void kbase_pm_update_cores(struct kbase_device *kbdev); ++ ++ ++enum kbase_pm_cores_ready { ++ KBASE_CORES_NOT_READY = 0, ++ KBASE_NEW_AFFINITY = 1, ++ KBASE_CORES_READY = 2 ++}; ++ ++ ++/** ++ * kbase_pm_request_cores_sync - Synchronous variant of kbase_pm_request_cores() ++ * ++ * @kbdev: The kbase device structure for the device ++ * @tiler_required: true if the tiler is required, false otherwise ++ * @shader_cores: A bitmask of shader cores which are necessary for the job ++ * ++ * When this function returns, the @shader_cores will be in the READY state. ++ * ++ * This is safe variant of kbase_pm_check_transitions_sync(): it handles the ++ * work of ensuring the requested cores will remain powered until a matching ++ * call to kbase_pm_unrequest_cores()/kbase_pm_release_cores() (as appropriate) ++ * is made. ++ */ ++void kbase_pm_request_cores_sync(struct kbase_device *kbdev, ++ bool tiler_required, u64 shader_cores); ++ ++/** ++ * kbase_pm_request_cores - Mark one or more cores as being required ++ * for jobs to be submitted ++ * ++ * @kbdev: The kbase device structure for the device ++ * @tiler_required: true if the tiler is required, false otherwise ++ * @shader_cores: A bitmask of shader cores which are necessary for the job ++ * ++ * This function is called by the job scheduler to mark one or more cores as ++ * being required to submit jobs that are ready to run. ++ * ++ * The cores requested are reference counted and a subsequent call to ++ * kbase_pm_register_inuse_cores() or kbase_pm_unrequest_cores() should be ++ * made to dereference the cores as being 'needed'. ++ * ++ * The active power policy will meet or exceed the requirements of the ++ * requested cores in the system. Any core transitions needed will be begun ++ * immediately, but they might not complete/the cores might not be available ++ * until a Power Management IRQ. ++ * ++ * Return: 0 if the cores were successfully requested, or -errno otherwise. ++ */ ++void kbase_pm_request_cores(struct kbase_device *kbdev, ++ bool tiler_required, u64 shader_cores); ++ ++/** ++ * kbase_pm_unrequest_cores - Unmark one or more cores as being required for ++ * jobs to be submitted. ++ * ++ * @kbdev: The kbase device structure for the device ++ * @tiler_required: true if the tiler is required, false otherwise ++ * @shader_cores: A bitmask of shader cores (as given to ++ * kbase_pm_request_cores() ) ++ * ++ * This function undoes the effect of kbase_pm_request_cores(). It should be ++ * used when a job is not going to be submitted to the hardware (e.g. the job is ++ * cancelled before it is enqueued). ++ * ++ * The active power policy will meet or exceed the requirements of the ++ * requested cores in the system. Any core transitions needed will be begun ++ * immediately, but they might not complete until a Power Management IRQ. ++ * ++ * The policy may use this as an indication that it can power down cores. ++ */ ++void kbase_pm_unrequest_cores(struct kbase_device *kbdev, ++ bool tiler_required, u64 shader_cores); ++ ++/** ++ * kbase_pm_register_inuse_cores - Register a set of cores as in use by a job ++ * ++ * @kbdev: The kbase device structure for the device ++ * @tiler_required: true if the tiler is required, false otherwise ++ * @shader_cores: A bitmask of shader cores (as given to ++ * kbase_pm_request_cores() ) ++ * ++ * This function should be called after kbase_pm_request_cores() when the job ++ * is about to be submitted to the hardware. It will check that the necessary ++ * cores are available and if so update the 'needed' and 'inuse' bitmasks to ++ * reflect that the job is now committed to being run. ++ * ++ * If the necessary cores are not currently available then the function will ++ * return %KBASE_CORES_NOT_READY and have no effect. ++ * ++ * Return: %KBASE_CORES_NOT_READY if the cores are not immediately ready, ++ * ++ * %KBASE_NEW_AFFINITY if the affinity requested is not allowed, ++ * ++ * %KBASE_CORES_READY if the cores requested are already available ++ */ ++enum kbase_pm_cores_ready kbase_pm_register_inuse_cores( ++ struct kbase_device *kbdev, ++ bool tiler_required, ++ u64 shader_cores); ++ ++/** ++ * kbase_pm_release_cores - Release cores after a job has run ++ * ++ * @kbdev: The kbase device structure for the device ++ * @tiler_required: true if the tiler is required, false otherwise ++ * @shader_cores: A bitmask of shader cores (as given to ++ * kbase_pm_register_inuse_cores() ) ++ * ++ * This function should be called when a job has finished running on the ++ * hardware. A call to kbase_pm_register_inuse_cores() must have previously ++ * occurred. The reference counts of the specified cores will be decremented ++ * which may cause the bitmask of 'inuse' cores to be reduced. The power policy ++ * may then turn off any cores which are no longer 'inuse'. ++ */ ++void kbase_pm_release_cores(struct kbase_device *kbdev, ++ bool tiler_required, u64 shader_cores); ++ ++/** ++ * kbase_pm_request_l2_caches - Request l2 caches ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * Request the use of l2 caches for all core groups, power up, wait and prevent ++ * the power manager from powering down the l2 caches. ++ * ++ * This tells the power management that the caches should be powered up, and ++ * they should remain powered, irrespective of the usage of shader cores. This ++ * does not return until the l2 caches are powered up. ++ * ++ * The caller must call kbase_pm_release_l2_caches() when they are finished ++ * to allow normal power management of the l2 caches to resume. ++ * ++ * This should only be used when power management is active. ++ */ ++void kbase_pm_request_l2_caches(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_request_l2_caches_l2_is_on - Request l2 caches but don't power on ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * Increment the count of l2 users but do not attempt to power on the l2 ++ * ++ * It is the callers responsibility to ensure that the l2 is already powered up ++ * and to eventually call kbase_pm_release_l2_caches() ++ */ ++void kbase_pm_request_l2_caches_l2_is_on(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_request_l2_caches - Release l2 caches ++ * ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * ++ * Release the use of l2 caches for all core groups and allow the power manager ++ * to power them down when necessary. ++ * ++ * This tells the power management that the caches can be powered down if ++ * necessary, with respect to the usage of shader cores. ++ * ++ * The caller must have called kbase_pm_request_l2_caches() prior to a call ++ * to this. ++ * ++ * This should only be used when power management is active. ++ */ ++void kbase_pm_release_l2_caches(struct kbase_device *kbdev); ++ ++#endif /* _KBASE_PM_POLICY_H_ */ +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_time.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_time.c +new file mode 100755 +index 000000000000..d08c628dd433 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_time.c +@@ -0,0 +1,103 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include ++#include ++#include ++#include ++ ++void kbase_backend_get_gpu_time(struct kbase_device *kbdev, u64 *cycle_counter, ++ u64 *system_time, struct timespec64 *ts) ++{ ++ u32 hi1, hi2; ++ ++ kbase_pm_request_gpu_cycle_counter(kbdev); ++ ++ /* Read hi, lo, hi to ensure that overflow from lo to hi is handled ++ * correctly */ ++ do { ++ hi1 = kbase_reg_read(kbdev, GPU_CONTROL_REG(CYCLE_COUNT_HI), ++ NULL); ++ *cycle_counter = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(CYCLE_COUNT_LO), NULL); ++ hi2 = kbase_reg_read(kbdev, GPU_CONTROL_REG(CYCLE_COUNT_HI), ++ NULL); ++ *cycle_counter |= (((u64) hi1) << 32); ++ } while (hi1 != hi2); ++ ++ /* Read hi, lo, hi to ensure that overflow from lo to hi is handled ++ * correctly */ ++ do { ++ hi1 = kbase_reg_read(kbdev, GPU_CONTROL_REG(TIMESTAMP_HI), ++ NULL); ++ *system_time = kbase_reg_read(kbdev, ++ GPU_CONTROL_REG(TIMESTAMP_LO), NULL); ++ hi2 = kbase_reg_read(kbdev, GPU_CONTROL_REG(TIMESTAMP_HI), ++ NULL); ++ *system_time |= (((u64) hi1) << 32); ++ } while (hi1 != hi2); ++ ++ /* Record the CPU's idea of current time */ ++ ktime_get_raw_ts64(ts); ++ ++ kbase_pm_release_gpu_cycle_counter(kbdev); ++} ++ ++/** ++ * kbase_wait_write_flush - Wait for GPU write flush ++ * @kctx: Context pointer ++ * ++ * Wait 1000 GPU clock cycles. This delay is known to give the GPU time to flush ++ * its write buffer. ++ * ++ * Only in use for BASE_HW_ISSUE_6367 ++ * ++ * Note : If GPU resets occur then the counters are reset to zero, the delay may ++ * not be as expected. ++ */ ++#ifndef CONFIG_MALI_NO_MALI ++void kbase_wait_write_flush(struct kbase_context *kctx) ++{ ++ u32 base_count = 0; ++ ++ /* ++ * The caller must be holding onto the kctx or the call is from ++ * userspace. ++ */ ++ kbase_pm_context_active(kctx->kbdev); ++ kbase_pm_request_gpu_cycle_counter(kctx->kbdev); ++ ++ while (true) { ++ u32 new_count; ++ ++ new_count = kbase_reg_read(kctx->kbdev, ++ GPU_CONTROL_REG(CYCLE_COUNT_LO), NULL); ++ /* First time around, just store the count. */ ++ if (base_count == 0) { ++ base_count = new_count; ++ continue; ++ } ++ ++ /* No need to handle wrapping, unsigned maths works for this. */ ++ if ((new_count - base_count) > 1000) ++ break; ++ } ++ ++ kbase_pm_release_gpu_cycle_counter(kctx->kbdev); ++ kbase_pm_context_idle(kctx->kbdev); ++} ++#endif /* CONFIG_MALI_NO_MALI */ +diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_time.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_time.h +new file mode 100755 +index 000000000000..433aa4b9cb5e +--- /dev/null ++++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_time.h +@@ -0,0 +1,52 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KBASE_BACKEND_TIME_H_ ++#define _KBASE_BACKEND_TIME_H_ ++ ++/** ++ * kbase_backend_get_gpu_time() - Get current GPU time ++ * @kbdev: Device pointer ++ * @cycle_counter: Pointer to u64 to store cycle counter in ++ * @system_time: Pointer to u64 to store system time in ++ * @ts: Pointer to struct timespec64 to store current monotonic ++ * time in ++ */ ++void kbase_backend_get_gpu_time(struct kbase_device *kbdev, u64 *cycle_counter, ++ u64 *system_time, struct timespec64 *ts); ++ ++/** ++ * kbase_wait_write_flush() - Wait for GPU write flush ++ * @kctx: Context pointer ++ * ++ * Wait 1000 GPU clock cycles. This delay is known to give the GPU time to flush ++ * its write buffer. ++ * ++ * If GPU resets occur then the counters are reset to zero, the delay may not be ++ * as expected. ++ * ++ * This function is only in use for BASE_HW_ISSUE_6367 ++ */ ++#ifdef CONFIG_MALI_NO_MALI ++static inline void kbase_wait_write_flush(struct kbase_context *kctx) ++{ ++} ++#else ++void kbase_wait_write_flush(struct kbase_context *kctx); ++#endif ++ ++#endif /* _KBASE_BACKEND_TIME_H_ */ +diff --git a/drivers/gpu/arm/midgard/docs/Doxyfile b/drivers/gpu/arm/midgard/docs/Doxyfile +new file mode 100755 +index 000000000000..35ff2f1ce4a0 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/docs/Doxyfile +@@ -0,0 +1,126 @@ ++# ++# (C) COPYRIGHT 2011-2013, 2015 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++ ++############################################################################## ++ ++# This file contains per-module Doxygen configuration. Please do not add ++# extra settings to this file without consulting all stakeholders, as they ++# may cause override project-wide settings. ++# ++# Additionally, when defining aliases, macros, sections etc, use the module ++# name as a prefix e.g. gles_my_alias. ++ ++############################################################################## ++ ++@INCLUDE = ../../bldsys/Doxyfile_common ++ ++# The INPUT tag can be used to specify the files and/or directories that contain ++# documented source files. You may enter file names like "myfile.cpp" or ++# directories like "/usr/src/myproject". Separate the files or directories ++# with spaces. ++ ++INPUT += ../../kernel/drivers/gpu/arm/midgard/ ++ ++############################################################################## ++# Everything below here is optional, and in most cases not required ++############################################################################## ++ ++# This tag can be used to specify a number of aliases that acts ++# as commands in the documentation. An alias has the form "name=value". ++# For example adding "sideeffect=\par Side Effects:\n" will allow you to ++# put the command \sideeffect (or @sideeffect) in the documentation, which ++# will result in a user-defined paragraph with heading "Side Effects:". ++# You can put \n's in the value part of an alias to insert newlines. ++ ++ALIASES += ++ ++# The ENABLED_SECTIONS tag can be used to enable conditional ++# documentation sections, marked by \if sectionname ... \endif. ++ ++ENABLED_SECTIONS += ++ ++# If the value of the INPUT tag contains directories, you can use the ++# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp ++# and *.h) to filter out the source-files in the directories. If left ++# blank the following patterns are tested: ++# *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx ++# *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.py *.f90 ++ ++FILE_PATTERNS += ++ ++# The EXCLUDE tag can be used to specify files and/or directories that should ++# excluded from the INPUT source files. This way you can easily exclude a ++# subdirectory from a directory tree whose root is specified with the INPUT tag. ++EXCLUDE += ../../kernel/drivers/gpu/arm/midgard/platform ../../kernel/drivers/gpu/arm/midgard/platform_dummy ../../kernel/drivers/gpu/arm/midgard/scripts ../../kernel/drivers/gpu/arm/midgard/tests ../../kernel/drivers/gpu/arm/midgard/Makefile ../../kernel/drivers/gpu/arm/midgard/Makefile.kbase ../../kernel/drivers/gpu/arm/midgard/Kbuild ../../kernel/drivers/gpu/arm/midgard/Kconfig ../../kernel/drivers/gpu/arm/midgard/sconscript ../../kernel/drivers/gpu/arm/midgard/docs ../../kernel/drivers/gpu/arm/midgard/pm_test_script.sh ../../kernel/drivers/gpu/arm/midgard/mali_uk.h ../../kernel/drivers/gpu/arm/midgard/Makefile ++ ++ ++# If the value of the INPUT tag contains directories, you can use the ++# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude ++# certain files from those directories. Note that the wildcards are matched ++# against the file with absolute path, so to exclude all test directories ++# for example use the pattern */test/* ++ ++EXCLUDE_PATTERNS += ++ ++# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names ++# (namespaces, classes, functions, etc.) that should be excluded from the ++# output. The symbol name can be a fully qualified name, a word, or if the ++# wildcard * is used, a substring. Examples: ANamespace, AClass, ++# AClass::ANamespace, ANamespace::*Test ++ ++EXCLUDE_SYMBOLS += ++ ++# The EXAMPLE_PATH tag can be used to specify one or more files or ++# directories that contain example code fragments that are included (see ++# the \include command). ++ ++EXAMPLE_PATH += ++ ++# The IMAGE_PATH tag can be used to specify one or more files or ++# directories that contain image that are included in the documentation (see ++# the \image command). ++ ++IMAGE_PATH += ++ ++# The INCLUDE_PATH tag can be used to specify one or more directories that ++# contain include files that are not input files but should be processed by ++# the preprocessor. ++ ++INCLUDE_PATH += ++ ++# The PREDEFINED tag can be used to specify one or more macro names that ++# are defined before the preprocessor is started (similar to the -D option of ++# gcc). The argument of the tag is a list of macros of the form: name ++# or name=definition (no spaces). If the definition and the = are ++# omitted =1 is assumed. To prevent a macro definition from being ++# undefined via #undef or recursively expanded use the := operator ++# instead of the = operator. ++ ++PREDEFINED += ++ ++# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then ++# this tag can be used to specify a list of macro names that should be expanded. ++# The macro definition that is found in the sources will be used. ++# Use the PREDEFINED tag if you want to use a different macro definition. ++ ++EXPAND_AS_DEFINED += ++ ++# The DOTFILE_DIRS tag can be used to specify one or more directories that ++# contain dot files that are included in the documentation (see the ++# \dotfile command). ++ ++DOTFILE_DIRS += ../../kernel/drivers/gpu/arm/midgard/docs ++ +diff --git a/drivers/gpu/arm/midgard/docs/policy_operation_diagram.dot b/drivers/gpu/arm/midgard/docs/policy_operation_diagram.dot +new file mode 100755 +index 000000000000..7ae05c2f8ded +--- /dev/null ++++ b/drivers/gpu/arm/midgard/docs/policy_operation_diagram.dot +@@ -0,0 +1,112 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++digraph policy_objects_diagram { ++ rankdir=LR; ++ size="12,8"; ++ compound=true; ++ ++ node [ shape = box ]; ++ ++ subgraph cluster_policy_queues { ++ low_queue [ shape=record label = "LowP | {ctx_lo | ... | ctx_i | ... | ctx_hi}" ]; ++ queues_middle_sep [ label="" shape=plaintext width=0 height=0 ]; ++ ++ rt_queue [ shape=record label = "RT | {ctx_lo | ... | ctx_j | ... | ctx_hi}" ]; ++ ++ label = "Policy's Queue(s)"; ++ } ++ ++ call_enqueue [ shape=plaintext label="enqueue_ctx()" ]; ++ ++ { ++ rank=same; ++ ordering=out; ++ call_dequeue [ shape=plaintext label="dequeue_head_ctx()\n+ runpool_add_ctx()" ]; ++ call_ctxfinish [ shape=plaintext label="runpool_remove_ctx()" ]; ++ ++ call_ctxdone [ shape=plaintext label="don't requeue;\n/* ctx has no more jobs */" ]; ++ } ++ ++ subgraph cluster_runpool { ++ ++ as0 [ width=2 height = 0.25 label="AS0: Job_1, ..., Job_n" ]; ++ as1 [ width=2 height = 0.25 label="AS1: Job_1, ..., Job_m" ]; ++ as2 [ width=2 height = 0.25 label="AS2: Job_1, ..., Job_p" ]; ++ as3 [ width=2 height = 0.25 label="AS3: Job_1, ..., Job_q" ]; ++ ++ label = "Policy's Run Pool"; ++ } ++ ++ { ++ rank=same; ++ call_jdequeue [ shape=plaintext label="dequeue_job()" ]; ++ sstop_dotfixup [ shape=plaintext label="" width=0 height=0 ]; ++ } ++ ++ { ++ rank=same; ++ ordering=out; ++ sstop [ shape=ellipse label="SS-Timer expires" ] ++ jobslots [ shape=record label="Jobslots: | <0>js[0] | <1>js[1] | <2>js[2]" ]; ++ ++ irq [ label="IRQ" shape=ellipse ]; ++ ++ job_finish [ shape=plaintext label="don't requeue;\n/* job done */" ]; ++ } ++ ++ hstop [ shape=ellipse label="HS-Timer expires" ] ++ ++ /* ++ * Edges ++ */ ++ ++ call_enqueue -> queues_middle_sep [ lhead=cluster_policy_queues ]; ++ ++ low_queue:qr -> call_dequeue:w; ++ rt_queue:qr -> call_dequeue:w; ++ ++ call_dequeue -> as1 [lhead=cluster_runpool]; ++ ++ as1->call_jdequeue [ltail=cluster_runpool]; ++ call_jdequeue->jobslots:0; ++ call_jdequeue->sstop_dotfixup [ arrowhead=none]; ++ sstop_dotfixup->sstop [label="Spawn SS-Timer"]; ++ sstop->jobslots [label="SoftStop"]; ++ sstop->hstop [label="Spawn HS-Timer"]; ++ hstop->jobslots:ne [label="HardStop"]; ++ ++ ++ as3->call_ctxfinish:ne [ ltail=cluster_runpool ]; ++ call_ctxfinish:sw->rt_queue:qm [ lhead=cluster_policy_queues label="enqueue_ctx()\n/* ctx still has jobs */" ]; ++ ++ call_ctxfinish->call_ctxdone [constraint=false]; ++ ++ call_ctxdone->call_enqueue [weight=0.1 labeldistance=20.0 labelangle=0.0 taillabel="Job submitted to the ctx" style=dotted constraint=false]; ++ ++ ++ { ++ jobslots->irq [constraint=false]; ++ ++ irq->job_finish [constraint=false]; ++ } ++ ++ irq->as2 [lhead=cluster_runpool label="requeue_job()\n/* timeslice expired */" ]; ++ ++} +diff --git a/drivers/gpu/arm/midgard/docs/policy_overview.dot b/drivers/gpu/arm/midgard/docs/policy_overview.dot +new file mode 100755 +index 000000000000..159b993b7d61 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/docs/policy_overview.dot +@@ -0,0 +1,63 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++digraph policy_objects_diagram { ++ rankdir=LR ++ size="6,6" ++ compound=true; ++ ++ node [ shape = box ]; ++ ++ call_enqueue [ shape=plaintext label="enqueue ctx" ]; ++ ++ ++ policy_queue [ label="Policy's Queue" ]; ++ ++ { ++ rank=same; ++ runpool [ label="Policy's Run Pool" ]; ++ ++ ctx_finish [ label="ctx finished" ]; ++ } ++ ++ { ++ rank=same; ++ jobslots [ shape=record label="Jobslots: | <0>js[0] | <1>js[1] | <2>js[2]" ]; ++ ++ job_finish [ label="Job finished" ]; ++ } ++ ++ ++ ++ /* ++ * Edges ++ */ ++ ++ call_enqueue -> policy_queue; ++ ++ policy_queue->runpool [label="dequeue ctx" weight=0.1]; ++ runpool->policy_queue [label="requeue ctx" weight=0.1]; ++ ++ runpool->ctx_finish [ style=dotted ]; ++ ++ runpool->jobslots [label="dequeue job" weight=0.1]; ++ jobslots->runpool [label="requeue job" weight=0.1]; ++ ++ jobslots->job_finish [ style=dotted ]; ++} +diff --git a/drivers/gpu/arm/midgard/ipa/Kbuild b/drivers/gpu/arm/midgard/ipa/Kbuild +new file mode 100755 +index 000000000000..602b15f5225c +--- /dev/null ++++ b/drivers/gpu/arm/midgard/ipa/Kbuild +@@ -0,0 +1,24 @@ ++# ++# (C) COPYRIGHT 2016-2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++midgard_kbase-y += \ ++ ipa/mali_kbase_ipa_simple.o \ ++ ipa/mali_kbase_ipa.o ++ ++midgard_kbase-$(CONFIG_DEBUG_FS) += ipa/mali_kbase_ipa_debugfs.o ++ ++ifneq ($(wildcard $(src)/ipa/mali_kbase_ipa_tmix.c),) ++ midgard_kbase-y += ipa/mali_kbase_ipa_tmix.o ++endif +diff --git a/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa.c b/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa.c +new file mode 100755 +index 000000000000..01bdbb4e8eb1 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa.c +@@ -0,0 +1,585 @@ ++/* ++ * ++ * (C) COPYRIGHT 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++#include ++#include ++#include ++#include "mali_kbase.h" ++#include "mali_kbase_ipa.h" ++#include "mali_kbase_ipa_debugfs.h" ++ ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0)) ++#include ++#else ++#include ++#define dev_pm_opp_find_freq_exact opp_find_freq_exact ++#define dev_pm_opp_get_voltage opp_get_voltage ++#define dev_pm_opp opp ++#endif ++#include ++ ++#define KBASE_IPA_FALLBACK_MODEL_NAME "mali-simple-power-model" ++ ++static struct kbase_ipa_model_ops *kbase_ipa_all_model_ops[] = { ++ &kbase_simple_ipa_model_ops, ++}; ++ ++int kbase_ipa_model_recalculate(struct kbase_ipa_model *model) ++{ ++ int err = 0; ++ ++ lockdep_assert_held(&model->kbdev->ipa.lock); ++ ++ if (model->ops->recalculate) { ++ err = model->ops->recalculate(model); ++ if (err) { ++ dev_err(model->kbdev->dev, ++ "recalculation of power model %s returned error %d\n", ++ model->ops->name, err); ++ } ++ } ++ ++ return err; ++} ++ ++static struct kbase_ipa_model_ops *kbase_ipa_model_ops_find(struct kbase_device *kbdev, ++ const char *name) ++{ ++ int i; ++ ++ for (i = 0; i < ARRAY_SIZE(kbase_ipa_all_model_ops); ++i) { ++ struct kbase_ipa_model_ops *ops = kbase_ipa_all_model_ops[i]; ++ ++ if (!strcmp(ops->name, name)) ++ return ops; ++ } ++ ++ dev_err(kbdev->dev, "power model \'%s\' not found\n", name); ++ ++ return NULL; ++} ++ ++void kbase_ipa_model_use_fallback_locked(struct kbase_device *kbdev) ++{ ++ atomic_set(&kbdev->ipa_use_configured_model, false); ++} ++ ++void kbase_ipa_model_use_configured_locked(struct kbase_device *kbdev) ++{ ++ atomic_set(&kbdev->ipa_use_configured_model, true); ++} ++ ++const char *kbase_ipa_model_name_from_id(u32 gpu_id) ++{ ++ const u32 prod_id = (gpu_id & GPU_ID_VERSION_PRODUCT_ID) >> ++ GPU_ID_VERSION_PRODUCT_ID_SHIFT; ++ ++ if (GPU_ID_IS_NEW_FORMAT(prod_id)) { ++ switch (GPU_ID2_MODEL_MATCH_VALUE(prod_id)) { ++ case GPU_ID2_PRODUCT_TMIX: ++ return KBASE_IPA_FALLBACK_MODEL_NAME; ++ default: ++ return KBASE_IPA_FALLBACK_MODEL_NAME; ++ } ++ } ++ ++ return KBASE_IPA_FALLBACK_MODEL_NAME; ++} ++ ++static struct device_node *get_model_dt_node(struct kbase_ipa_model *model) ++{ ++ struct device_node *model_dt_node; ++ char compat_string[64]; ++ ++ snprintf(compat_string, sizeof(compat_string), "arm,%s", ++ model->ops->name); ++ ++ model_dt_node = of_find_compatible_node(model->kbdev->dev->of_node, ++ NULL, compat_string); ++ if (!model_dt_node && !model->missing_dt_node_warning) { ++ dev_warn(model->kbdev->dev, ++ "Couldn't find power_model DT node matching \'%s\'\n", ++ compat_string); ++ model->missing_dt_node_warning = true; ++ } ++ ++ return model_dt_node; ++} ++ ++int kbase_ipa_model_add_param_s32(struct kbase_ipa_model *model, ++ const char *name, s32 *addr, ++ size_t num_elems, bool dt_required) ++{ ++ int err, i; ++ struct device_node *model_dt_node = get_model_dt_node(model); ++ char *origin; ++ ++ err = of_property_read_u32_array(model_dt_node, name, addr, num_elems); ++ ++ if (err && dt_required) { ++ memset(addr, 0, sizeof(s32) * num_elems); ++ dev_warn(model->kbdev->dev, ++ "Error %d, no DT entry: %s.%s = %zu*[0]\n", ++ err, model->ops->name, name, num_elems); ++ origin = "zero"; ++ } else if (err && !dt_required) { ++ origin = "default"; ++ } else /* !err */ { ++ origin = "DT"; ++ } ++ ++ /* Create a unique debugfs entry for each element */ ++ for (i = 0; i < num_elems; ++i) { ++ char elem_name[32]; ++ ++ if (num_elems == 1) ++ snprintf(elem_name, sizeof(elem_name), "%s", name); ++ else ++ snprintf(elem_name, sizeof(elem_name), "%s.%d", ++ name, i); ++ ++ dev_dbg(model->kbdev->dev, "%s.%s = %d (%s)\n", ++ model->ops->name, elem_name, addr[i], origin); ++ ++ err = kbase_ipa_model_param_add(model, elem_name, ++ &addr[i], sizeof(s32), ++ PARAM_TYPE_S32); ++ if (err) ++ goto exit; ++ } ++exit: ++ return err; ++} ++ ++int kbase_ipa_model_add_param_string(struct kbase_ipa_model *model, ++ const char *name, char *addr, ++ size_t size, bool dt_required) ++{ ++ int err; ++ struct device_node *model_dt_node = get_model_dt_node(model); ++ const char *string_prop_value; ++ char *origin; ++ ++ err = of_property_read_string(model_dt_node, name, ++ &string_prop_value); ++ if (err && dt_required) { ++ strncpy(addr, "", size - 1); ++ dev_warn(model->kbdev->dev, ++ "Error %d, no DT entry: %s.%s = \'%s\'\n", ++ err, model->ops->name, name, addr); ++ err = 0; ++ origin = "zero"; ++ } else if (err && !dt_required) { ++ origin = "default"; ++ } else /* !err */ { ++ strncpy(addr, string_prop_value, size - 1); ++ origin = "DT"; ++ } ++ ++ addr[size - 1] = '\0'; ++ ++ dev_dbg(model->kbdev->dev, "%s.%s = \'%s\' (%s)\n", ++ model->ops->name, name, string_prop_value, origin); ++ ++ err = kbase_ipa_model_param_add(model, name, addr, size, ++ PARAM_TYPE_STRING); ++ ++ return err; ++} ++ ++void kbase_ipa_term_model(struct kbase_ipa_model *model) ++{ ++ if (!model) ++ return; ++ ++ lockdep_assert_held(&model->kbdev->ipa.lock); ++ ++ if (model->ops->term) ++ model->ops->term(model); ++ ++ kbase_ipa_model_param_free_all(model); ++ ++ kfree(model); ++} ++KBASE_EXPORT_TEST_API(kbase_ipa_term_model); ++ ++struct kbase_ipa_model *kbase_ipa_init_model(struct kbase_device *kbdev, ++ struct kbase_ipa_model_ops *ops) ++{ ++ struct kbase_ipa_model *model; ++ int err; ++ ++ lockdep_assert_held(&kbdev->ipa.lock); ++ ++ if (!ops || !ops->name) ++ return NULL; ++ ++ model = kzalloc(sizeof(struct kbase_ipa_model), GFP_KERNEL); ++ if (!model) ++ return NULL; ++ ++ model->kbdev = kbdev; ++ model->ops = ops; ++ INIT_LIST_HEAD(&model->params); ++ ++ err = model->ops->init(model); ++ if (err) { ++ dev_err(kbdev->dev, ++ "init of power model \'%s\' returned error %d\n", ++ ops->name, err); ++ goto term_model; ++ } ++ ++ err = kbase_ipa_model_recalculate(model); ++ if (err) ++ goto term_model; ++ ++ return model; ++ ++term_model: ++ kbase_ipa_term_model(model); ++ return NULL; ++} ++KBASE_EXPORT_TEST_API(kbase_ipa_init_model); ++ ++static void kbase_ipa_term_locked(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->ipa.lock); ++ ++ /* Clean up the models */ ++ if (kbdev->ipa.configured_model != kbdev->ipa.fallback_model) ++ kbase_ipa_term_model(kbdev->ipa.configured_model); ++ kbase_ipa_term_model(kbdev->ipa.fallback_model); ++ ++ kbdev->ipa.configured_model = NULL; ++ kbdev->ipa.fallback_model = NULL; ++} ++ ++int kbase_ipa_init(struct kbase_device *kbdev) ++{ ++ ++ const char *model_name; ++ struct kbase_ipa_model_ops *ops; ++ struct kbase_ipa_model *default_model = NULL; ++ int err; ++ ++ mutex_init(&kbdev->ipa.lock); ++ /* ++ * Lock during init to avoid warnings from lockdep_assert_held (there ++ * shouldn't be any concurrent access yet). ++ */ ++ mutex_lock(&kbdev->ipa.lock); ++ ++ /* The simple IPA model must *always* be present.*/ ++ ops = kbase_ipa_model_ops_find(kbdev, KBASE_IPA_FALLBACK_MODEL_NAME); ++ ++ if (!ops->do_utilization_scaling_in_framework) { ++ dev_err(kbdev->dev, ++ "Fallback IPA model %s should not account for utilization\n", ++ ops->name); ++ err = -EINVAL; ++ goto end; ++ } ++ ++ default_model = kbase_ipa_init_model(kbdev, ops); ++ if (!default_model) { ++ err = -EINVAL; ++ goto end; ++ } ++ ++ kbdev->ipa.fallback_model = default_model; ++ err = of_property_read_string(kbdev->dev->of_node, ++ "ipa-model", ++ &model_name); ++ if (err) { ++ /* Attempt to load a match from GPU-ID */ ++ u32 gpu_id; ++ ++ gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; ++ model_name = kbase_ipa_model_name_from_id(gpu_id); ++ dev_dbg(kbdev->dev, ++ "Inferring model from GPU ID 0x%x: \'%s\'\n", ++ gpu_id, model_name); ++ } else { ++ dev_dbg(kbdev->dev, ++ "Using ipa-model parameter from DT: \'%s\'\n", ++ model_name); ++ } ++ ++ if (strcmp(KBASE_IPA_FALLBACK_MODEL_NAME, model_name) != 0) { ++ ops = kbase_ipa_model_ops_find(kbdev, model_name); ++ kbdev->ipa.configured_model = kbase_ipa_init_model(kbdev, ops); ++ if (!kbdev->ipa.configured_model) { ++ err = -EINVAL; ++ goto end; ++ } ++ } else { ++ kbdev->ipa.configured_model = default_model; ++ err = 0; ++ } ++ ++ kbase_ipa_model_use_configured_locked(kbdev); ++ ++end: ++ if (err) ++ kbase_ipa_term_locked(kbdev); ++ else ++ dev_info(kbdev->dev, ++ "Using configured power model %s, and fallback %s\n", ++ kbdev->ipa.configured_model->ops->name, ++ kbdev->ipa.fallback_model->ops->name); ++ ++ mutex_unlock(&kbdev->ipa.lock); ++ return err; ++} ++KBASE_EXPORT_TEST_API(kbase_ipa_init); ++ ++void kbase_ipa_term(struct kbase_device *kbdev) ++{ ++ mutex_lock(&kbdev->ipa.lock); ++ kbase_ipa_term_locked(kbdev); ++ mutex_unlock(&kbdev->ipa.lock); ++} ++KBASE_EXPORT_TEST_API(kbase_ipa_term); ++ ++/** ++ * kbase_scale_dynamic_power() - Scale a dynamic power coefficient to an OPP ++ * @c: Dynamic model coefficient, in pW/(Hz V^2). Should be in range ++ * 0 < c < 2^26 to prevent overflow. ++ * @freq: Frequency, in Hz. Range: 2^23 < freq < 2^30 (~8MHz to ~1GHz) ++ * @voltage: Voltage, in mV. Range: 2^9 < voltage < 2^13 (~0.5V to ~8V) ++ * ++ * Keep a record of the approximate range of each value at every stage of the ++ * calculation, to ensure we don't overflow. This makes heavy use of the ++ * approximations 1000 = 2^10 and 1000000 = 2^20, but does the actual ++ * calculations in decimal for increased accuracy. ++ * ++ * Return: Power consumption, in mW. Range: 0 < p < 2^13 (0W to ~8W) ++ */ ++static u32 kbase_scale_dynamic_power(const u32 c, const u32 freq, ++ const u32 voltage) ++{ ++ /* Range: 2^8 < v2 < 2^16 m(V^2) */ ++ const u32 v2 = (voltage * voltage) / 1000; ++ ++ /* Range: 2^3 < f_MHz < 2^10 MHz */ ++ const u32 f_MHz = freq / 1000000; ++ ++ /* Range: 2^11 < v2f_big < 2^26 kHz V^2 */ ++ const u32 v2f_big = v2 * f_MHz; ++ ++ /* Range: 2^1 < v2f < 2^16 MHz V^2 */ ++ const u32 v2f = v2f_big / 1000; ++ ++ /* Range (working backwards from next line): 0 < v2fc < 2^23 uW. ++ * Must be < 2^42 to avoid overflowing the return value. */ ++ const u64 v2fc = (u64) c * (u64) v2f; ++ u32 remainder; ++ ++ /* Range: 0 < v2fc / 1000 < 2^13 mW */ ++ // static inline u64 div_u64_rem(u64 dividend, u32 divisor, u32 *remainder) ++ return div_u64_rem(v2fc, 1000, &remainder); ++} ++ ++/** ++ * kbase_scale_static_power() - Scale a static power coefficient to an OPP ++ * @c: Static model coefficient, in uW/V^3. Should be in range ++ * 0 < c < 2^32 to prevent overflow. ++ * @voltage: Voltage, in mV. Range: 2^9 < voltage < 2^13 (~0.5V to ~8V) ++ * ++ * Return: Power consumption, in mW. Range: 0 < p < 2^13 (0W to ~8W) ++ */ ++u32 kbase_scale_static_power(const u32 c, const u32 voltage) ++{ ++ /* Range: 2^8 < v2 < 2^16 m(V^2) */ ++ const u32 v2 = (voltage * voltage) / 1000; ++ ++ /* Range: 2^17 < v3_big < 2^29 m(V^2) mV */ ++ const u32 v3_big = v2 * voltage; ++ ++ /* Range: 2^7 < v3 < 2^19 m(V^3) */ ++ const u32 v3 = v3_big / 1000; ++ ++ /* ++ * Range (working backwards from next line): 0 < v3c_big < 2^33 nW. ++ * The result should be < 2^52 to avoid overflowing the return value. ++ */ ++ const u64 v3c_big = (u64) c * (u64) v3; ++ u32 remainder; ++ ++ /* Range: 0 < v3c_big / 1000000 < 2^13 mW */ ++ // return v3c_big / 1000000; ++ return div_u64_rem(v3c_big, 1000000, &remainder); ++} ++ ++static struct kbase_ipa_model *get_current_model(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->ipa.lock); ++ ++ if (atomic_read(&kbdev->ipa_use_configured_model)) ++ return kbdev->ipa.configured_model; ++ else ++ return kbdev->ipa.fallback_model; ++} ++ ++static u32 get_static_power_locked(struct kbase_device *kbdev, ++ struct kbase_ipa_model *model, ++ unsigned long voltage) ++{ ++ u32 power = 0; ++ int err; ++ u32 power_coeff; ++ ++ lockdep_assert_held(&model->kbdev->ipa.lock); ++ ++ if (!model->ops->get_static_coeff) ++ model = kbdev->ipa.fallback_model; ++ ++ if (model->ops->get_static_coeff) { ++ err = model->ops->get_static_coeff(model, &power_coeff); ++ if (!err) ++ power = kbase_scale_static_power(power_coeff, ++ (u32) voltage); ++ } ++ ++ return power; ++} ++ ++#ifdef CONFIG_MALI_PWRSOFT_765 ++static unsigned long kbase_get_static_power(struct devfreq *df, ++ unsigned long voltage) ++#else ++static unsigned long kbase_get_static_power(unsigned long voltage) ++#endif ++{ ++ struct kbase_ipa_model *model; ++ u32 power = 0; ++#ifdef CONFIG_MALI_PWRSOFT_765 ++ struct kbase_device *kbdev = dev_get_drvdata(&df->dev); ++#else ++ struct kbase_device *kbdev = kbase_find_device(-1); ++#endif ++ ++ mutex_lock(&kbdev->ipa.lock); ++ ++ model = get_current_model(kbdev); ++ power = get_static_power_locked(kbdev, model, voltage); ++ ++ mutex_unlock(&kbdev->ipa.lock); ++ ++#ifndef CONFIG_MALI_PWRSOFT_765 ++ kbase_release_device(kbdev); ++#endif ++ ++ return power; ++} ++ ++#ifdef CONFIG_MALI_PWRSOFT_765 ++static unsigned long kbase_get_dynamic_power(struct devfreq *df, ++ unsigned long freq, ++ unsigned long voltage) ++#else ++static unsigned long kbase_get_dynamic_power(unsigned long freq, ++ unsigned long voltage) ++#endif ++{ ++ struct kbase_ipa_model *model; ++ u32 power_coeff = 0, power = 0; ++ int err = 0; ++#ifdef CONFIG_MALI_PWRSOFT_765 ++ struct kbase_device *kbdev = dev_get_drvdata(&df->dev); ++#else ++ struct kbase_device *kbdev = kbase_find_device(-1); ++#endif ++ ++ mutex_lock(&kbdev->ipa.lock); ++ ++ model = kbdev->ipa.fallback_model; ++ ++ err = model->ops->get_dynamic_coeff(model, &power_coeff, freq); ++ ++ if (!err) ++ power = kbase_scale_dynamic_power(power_coeff, freq, voltage); ++ else ++ dev_err_ratelimited(kbdev->dev, ++ "Model %s returned error code %d\n", ++ model->ops->name, err); ++ ++ mutex_unlock(&kbdev->ipa.lock); ++ ++#ifndef CONFIG_MALI_PWRSOFT_765 ++ kbase_release_device(kbdev); ++#endif ++ ++ return power; ++} ++ ++int kbase_get_real_power(struct devfreq *df, u32 *power, ++ unsigned long freq, ++ unsigned long voltage) ++{ ++ struct kbase_ipa_model *model; ++ u32 power_coeff = 0; ++ int err = 0; ++ struct kbase_device *kbdev = dev_get_drvdata(&df->dev); ++ ++ mutex_lock(&kbdev->ipa.lock); ++ ++ model = get_current_model(kbdev); ++ ++ err = model->ops->get_dynamic_coeff(model, &power_coeff, freq); ++ ++ /* If we switch to protected model between get_current_model() and ++ * get_dynamic_coeff(), counter reading could fail. If that happens ++ * (unlikely, but possible), revert to the fallback model. */ ++ if (err && model != kbdev->ipa.fallback_model) { ++ model = kbdev->ipa.fallback_model; ++ err = model->ops->get_dynamic_coeff(model, &power_coeff, freq); ++ } ++ ++ if (err) ++ goto exit_unlock; ++ ++ *power = kbase_scale_dynamic_power(power_coeff, freq, voltage); ++ ++ if (model->ops->do_utilization_scaling_in_framework) { ++ struct devfreq_dev_status *status = &df->last_status; ++ unsigned long total_time = max(status->total_time, 1ul); ++ u64 busy_time = min(status->busy_time, total_time); ++ u32 remainder; ++ ++ // *power = ((u64) *power * (u64) busy_time) / total_time; ++ *power = div_u64_rem(((u64) *power * (u64) busy_time), total_time, &remainder); ++ } ++ ++ *power += get_static_power_locked(kbdev, model, voltage); ++ ++exit_unlock: ++ mutex_unlock(&kbdev->ipa.lock); ++ ++ return err; ++} ++KBASE_EXPORT_TEST_API(kbase_get_real_power); ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) ++struct devfreq_cooling_ops kbase_ipa_power_model_ops = { ++#else ++struct devfreq_cooling_power kbase_ipa_power_model_ops = { ++#endif ++ .get_static_power = &kbase_get_static_power, ++ .get_dynamic_power = &kbase_get_dynamic_power, ++}; ++KBASE_EXPORT_TEST_API(kbase_ipa_power_model_ops); +diff --git a/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa.h b/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa.h +new file mode 100755 +index 000000000000..b2d3db149579 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa.h +@@ -0,0 +1,148 @@ ++/* ++ * ++ * (C) COPYRIGHT 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KBASE_IPA_H_ ++#define _KBASE_IPA_H_ ++ ++#if defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL) ++ ++struct devfreq; ++ ++struct kbase_ipa_model { ++ struct list_head link; ++ struct kbase_device *kbdev; ++ void *model_data; ++ struct kbase_ipa_model_ops *ops; ++ struct list_head params; ++ bool missing_dt_node_warning; ++}; ++ ++/** ++ * kbase_ipa_model_add_param_s32 - Add an integer model parameter ++ * @model: pointer to IPA model ++ * @name: name of corresponding debugfs entry ++ * @addr: address where the value is stored ++ * @num_elems: number of elements (1 if not an array) ++ * @dt_required: if false, a corresponding devicetree entry is not required, ++ * and the current value will be used. If true, a warning is ++ * output and the data is zeroed ++ * ++ * Return: 0 on success, or an error code ++ */ ++int kbase_ipa_model_add_param_s32(struct kbase_ipa_model *model, ++ const char *name, s32 *addr, ++ size_t num_elems, bool dt_required); ++ ++/** ++ * kbase_ipa_model_add_param_string - Add a string model parameter ++ * @model: pointer to IPA model ++ * @name: name of corresponding debugfs entry ++ * @addr: address where the value is stored ++ * @size: size, in bytes, of the value storage (so the maximum string ++ * length is size - 1) ++ * @dt_required: if false, a corresponding devicetree entry is not required, ++ * and the current value will be used. If true, a warning is ++ * output and the data is zeroed ++ * ++ * Return: 0 on success, or an error code ++ */ ++int kbase_ipa_model_add_param_string(struct kbase_ipa_model *model, ++ const char *name, char *addr, ++ size_t size, bool dt_required); ++ ++struct kbase_ipa_model_ops { ++ char *name; ++ /* The init, recalculate and term ops on the default model are always ++ * called. However, all the other models are only invoked if the model ++ * is selected in the device tree. Otherwise they are never ++ * initialized. Additional resources can be acquired by models in ++ * init(), however they must be terminated in the term(). ++ */ ++ int (*init)(struct kbase_ipa_model *model); ++ /* Called immediately after init(), or when a parameter is changed, so ++ * that any coefficients derived from model parameters can be ++ * recalculated. */ ++ int (*recalculate)(struct kbase_ipa_model *model); ++ void (*term)(struct kbase_ipa_model *model); ++ /* ++ * get_dynamic_coeff() - calculate dynamic power coefficient ++ * @model: pointer to model ++ * @coeffp: pointer to return value location ++ * @current_freq: frequency the GPU has been running at for the ++ * previous sampling period. ++ * ++ * Calculate a dynamic power coefficient, with units pW/(Hz V^2), which ++ * is then scaled by the IPA framework according to the current OPP's ++ * frequency and voltage. ++ * ++ * Return: 0 on success, or an error code. ++ */ ++ int (*get_dynamic_coeff)(struct kbase_ipa_model *model, u32 *coeffp, ++ u32 current_freq); ++ /* ++ * get_static_coeff() - calculate static power coefficient ++ * @model: pointer to model ++ * @coeffp: pointer to return value location ++ * ++ * Calculate a static power coefficient, with units uW/(V^3), which is ++ * scaled by the IPA framework according to the current OPP's voltage. ++ * ++ * Return: 0 on success, or an error code. ++ */ ++ int (*get_static_coeff)(struct kbase_ipa_model *model, u32 *coeffp); ++ /* If false, the model's get_dynamic_coeff() method accounts for how ++ * long the GPU was active over the sample period. If true, the ++ * framework will scale the calculated power according to the ++ * utilization stats recorded by devfreq in get_real_power(). */ ++ bool do_utilization_scaling_in_framework; ++}; ++ ++/* Models can be registered only in the platform's platform_init_func call */ ++int kbase_ipa_model_ops_register(struct kbase_device *kbdev, ++ struct kbase_ipa_model_ops *new_model_ops); ++struct kbase_ipa_model *kbase_ipa_get_model(struct kbase_device *kbdev, ++ const char *name); ++ ++int kbase_ipa_init(struct kbase_device *kbdev); ++void kbase_ipa_term(struct kbase_device *kbdev); ++void kbase_ipa_model_use_fallback_locked(struct kbase_device *kbdev); ++void kbase_ipa_model_use_configured_locked(struct kbase_device *kbdev); ++int kbase_ipa_model_recalculate(struct kbase_ipa_model *model); ++struct kbase_ipa_model *kbase_ipa_init_model(struct kbase_device *kbdev, ++ struct kbase_ipa_model_ops *ops); ++void kbase_ipa_term_model(struct kbase_ipa_model *model); ++ ++extern struct kbase_ipa_model_ops kbase_simple_ipa_model_ops; ++ ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) ++extern struct devfreq_cooling_ops kbase_ipa_power_model_ops; ++#else ++extern struct devfreq_cooling_power kbase_ipa_power_model_ops; ++#endif ++ ++#else /* !(defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL)) */ ++ ++static inline void kbase_ipa_model_use_fallback_locked(struct kbase_device *kbdev) ++{ } ++ ++static inline void kbase_ipa_model_use_configured_locked(struct kbase_device *kbdev) ++{ } ++ ++#endif /* (defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL)) */ ++ ++#endif +diff --git a/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_debugfs.c b/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_debugfs.c +new file mode 100755 +index 000000000000..eafc14009ddc +--- /dev/null ++++ b/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_debugfs.c +@@ -0,0 +1,219 @@ ++/* ++ * ++ * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include ++#include ++#include ++ ++#include "mali_kbase.h" ++#include "mali_kbase_ipa.h" ++#include "mali_kbase_ipa_debugfs.h" ++ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0)) ++#define DEFINE_DEBUGFS_ATTRIBUTE DEFINE_SIMPLE_ATTRIBUTE ++#endif ++ ++struct kbase_ipa_model_param { ++ char *name; ++ union { ++ void *voidp; ++ s32 *s32p; ++ char *str; ++ } addr; ++ size_t size; ++ enum kbase_ipa_model_param_type type; ++ struct kbase_ipa_model *model; ++ struct list_head link; ++}; ++ ++static int param_int_get(void *data, u64 *val) ++{ ++ struct kbase_ipa_model_param *param = data; ++ ++ mutex_lock(¶m->model->kbdev->ipa.lock); ++ *(s64 *) val = *param->addr.s32p; ++ mutex_unlock(¶m->model->kbdev->ipa.lock); ++ ++ return 0; ++} ++ ++static int param_int_set(void *data, u64 val) ++{ ++ struct kbase_ipa_model_param *param = data; ++ struct kbase_ipa_model *model = param->model; ++ s64 sval = (s64) val; ++ int err = 0; ++ ++ if (sval < S32_MIN || sval > S32_MAX) ++ return -ERANGE; ++ ++ mutex_lock(¶m->model->kbdev->ipa.lock); ++ *param->addr.s32p = val; ++ err = kbase_ipa_model_recalculate(model); ++ mutex_unlock(¶m->model->kbdev->ipa.lock); ++ ++ return err; ++} ++ ++DEFINE_DEBUGFS_ATTRIBUTE(fops_s32, param_int_get, param_int_set, "%lld\n"); ++ ++static ssize_t param_string_get(struct file *file, char __user *user_buf, ++ size_t count, loff_t *ppos) ++{ ++ struct kbase_ipa_model_param *param = file->private_data; ++ ssize_t ret; ++ size_t len; ++ ++ mutex_lock(¶m->model->kbdev->ipa.lock); ++ len = strnlen(param->addr.str, param->size - 1) + 1; ++ ret = simple_read_from_buffer(user_buf, count, ppos, ++ param->addr.str, len); ++ mutex_unlock(¶m->model->kbdev->ipa.lock); ++ ++ return ret; ++} ++ ++static ssize_t param_string_set(struct file *file, const char __user *user_buf, ++ size_t count, loff_t *ppos) ++{ ++ struct kbase_ipa_model_param *param = file->private_data; ++ struct kbase_ipa_model *model = param->model; ++ ssize_t ret = count; ++ size_t buf_size; ++ int err; ++ ++ mutex_lock(&model->kbdev->ipa.lock); ++ ++ if (count > param->size) { ++ ret = -EINVAL; ++ goto end; ++ } ++ ++ buf_size = min(param->size - 1, count); ++ if (copy_from_user(param->addr.str, user_buf, buf_size)) { ++ ret = -EFAULT; ++ goto end; ++ } ++ ++ param->addr.str[buf_size] = '\0'; ++ ++ err = kbase_ipa_model_recalculate(model); ++ if (err < 0) ++ ret = err; ++ ++end: ++ mutex_unlock(&model->kbdev->ipa.lock); ++ ++ return ret; ++} ++ ++static const struct file_operations fops_string = { ++ .read = param_string_get, ++ .write = param_string_set, ++ .open = simple_open, ++ .llseek = default_llseek, ++}; ++ ++int kbase_ipa_model_param_add(struct kbase_ipa_model *model, const char *name, ++ void *addr, size_t size, ++ enum kbase_ipa_model_param_type type) ++{ ++ struct kbase_ipa_model_param *param; ++ ++ param = kzalloc(sizeof(*param), GFP_KERNEL); ++ ++ if (!param) ++ return -ENOMEM; ++ ++ /* 'name' is stack-allocated for array elements, so copy it into ++ * heap-allocated storage */ ++ param->name = kstrdup(name, GFP_KERNEL); ++ param->addr.voidp = addr; ++ param->size = size; ++ param->type = type; ++ param->model = model; ++ ++ list_add(¶m->link, &model->params); ++ ++ return 0; ++} ++ ++void kbase_ipa_model_param_free_all(struct kbase_ipa_model *model) ++{ ++ struct kbase_ipa_model_param *param_p, *param_n; ++ ++ list_for_each_entry_safe(param_p, param_n, &model->params, link) { ++ list_del(¶m_p->link); ++ kfree(param_p->name); ++ kfree(param_p); ++ } ++} ++ ++static void kbase_ipa_model_debugfs_init(struct kbase_ipa_model *model) ++{ ++ struct list_head *it; ++ struct dentry *dir; ++ ++ lockdep_assert_held(&model->kbdev->ipa.lock); ++ ++ dir = debugfs_create_dir(model->ops->name, ++ model->kbdev->mali_debugfs_directory); ++ ++ if (!dir) { ++ dev_err(model->kbdev->dev, ++ "Couldn't create mali debugfs %s directory", ++ model->ops->name); ++ return; ++ } ++ ++ list_for_each(it, &model->params) { ++ struct kbase_ipa_model_param *param = ++ list_entry(it, ++ struct kbase_ipa_model_param, ++ link); ++ const struct file_operations *fops = NULL; ++ ++ switch (param->type) { ++ case PARAM_TYPE_S32: ++ fops = &fops_s32; ++ break; ++ case PARAM_TYPE_STRING: ++ fops = &fops_string; ++ break; ++ } ++ ++ if (unlikely(!fops)) { ++ dev_err(model->kbdev->dev, ++ "Type not set for %s parameter %s\n", ++ model->ops->name, param->name); ++ } else { ++ debugfs_create_file(param->name, S_IRUGO | S_IWUSR, ++ dir, param, fops); ++ } ++ } ++} ++ ++void kbase_ipa_debugfs_init(struct kbase_device *kbdev) ++{ ++ mutex_lock(&kbdev->ipa.lock); ++ ++ if (kbdev->ipa.configured_model != kbdev->ipa.fallback_model) ++ kbase_ipa_model_debugfs_init(kbdev->ipa.configured_model); ++ kbase_ipa_model_debugfs_init(kbdev->ipa.fallback_model); ++ ++ mutex_unlock(&kbdev->ipa.lock); ++} +diff --git a/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_debugfs.h b/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_debugfs.h +new file mode 100755 +index 000000000000..ec06e2096f94 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_debugfs.h +@@ -0,0 +1,49 @@ ++/* ++ * ++ * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KBASE_IPA_DEBUGFS_H_ ++#define _KBASE_IPA_DEBUGFS_H_ ++ ++enum kbase_ipa_model_param_type { ++ PARAM_TYPE_S32 = 1, ++ PARAM_TYPE_STRING, ++}; ++ ++#ifdef CONFIG_DEBUG_FS ++ ++void kbase_ipa_debugfs_init(struct kbase_device *kbdev); ++int kbase_ipa_model_param_add(struct kbase_ipa_model *model, const char *name, ++ void *addr, size_t size, ++ enum kbase_ipa_model_param_type type); ++void kbase_ipa_model_param_free_all(struct kbase_ipa_model *model); ++ ++#else /* CONFIG_DEBUG_FS */ ++ ++static inline int kbase_ipa_model_param_add(struct kbase_ipa_model *model, ++ const char *name, void *addr, ++ size_t size, ++ enum kbase_ipa_model_param_type type) ++{ ++ return 0; ++} ++ ++static inline void kbase_ipa_model_param_free_all(struct kbase_ipa_model *model) ++{ } ++ ++#endif /* CONFIG_DEBUG_FS */ ++ ++#endif /* _KBASE_IPA_DEBUGFS_H_ */ +diff --git a/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_simple.c b/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_simple.c +new file mode 100755 +index 000000000000..da0a4d4a0e7e +--- /dev/null ++++ b/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_simple.c +@@ -0,0 +1,222 @@ ++/* ++ * ++ * (C) COPYRIGHT 2016-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include ++#ifdef CONFIG_DEVFREQ_THERMAL ++#include ++#endif ++#include ++#include ++ ++#include "mali_kbase.h" ++#include "mali_kbase_defs.h" ++ ++/* ++ * This model is primarily designed for the Juno platform. It may not be ++ * suitable for other platforms. The additional resources in this model ++ * should preferably be minimal, as this model is rarely used when a dynamic ++ * model is available. ++ */ ++ ++/** ++ * struct kbase_ipa_model_simple_data - IPA context per device ++ * @dynamic_coefficient: dynamic coefficient of the model ++ * @static_coefficient: static coefficient of the model ++ * @ts: Thermal scaling coefficients of the model ++ * @tz_name: Thermal zone name ++ * @gpu_tz: thermal zone device ++ */ ++ ++struct kbase_ipa_model_simple_data { ++ u32 dynamic_coefficient; ++ u32 static_coefficient; ++ s32 ts[4]; ++ char tz_name[16]; ++ struct thermal_zone_device *gpu_tz; ++}; ++#define FALLBACK_STATIC_TEMPERATURE 55000 ++ ++/** ++ * calculate_temp_scaling_factor() - Calculate temperature scaling coefficient ++ * @ts: Signed coefficients, in order t^0 to t^3, with units Deg^-N ++ * @t: Temperature, in mDeg C. Range: -2^17 < t < 2^17 ++ * ++ * Scale the temperature according to a cubic polynomial whose coefficients are ++ * provided in the device tree. The result is used to scale the static power ++ * coefficient, where 1000000 means no change. ++ * ++ * Return: Temperature scaling factor. Approx range 0 < ret < 10,000,000. ++ */ ++static u32 calculate_temp_scaling_factor(s32 ts[4], s64 t) ++{ ++ /* Range: -2^24 < t2 < 2^24 m(Deg^2) */ ++ u32 remainder; ++ // static inline s64 div_s64_rem(s64 dividend, s32 divisor, s32 *remainder) ++ const s64 t2 = div_s64_rem((t * t), 1000, &remainder); ++ ++ /* Range: -2^31 < t3 < 2^31 m(Deg^3) */ ++ const s64 t3 = div_s64_rem((t * t2), 1000, &remainder); ++ ++ /* ++ * Sum the parts. t^[1-3] are in m(Deg^N), but the coefficients are in ++ * Deg^-N, so we need to multiply the last coefficient by 1000. ++ * Range: -2^63 < res_big < 2^63 ++ */ ++ const s64 res_big = ts[3] * t3 /* +/- 2^62 */ ++ + ts[2] * t2 /* +/- 2^55 */ ++ + ts[1] * t /* +/- 2^48 */ ++ + ts[0] * 1000; /* +/- 2^41 */ ++ ++ /* Range: -2^60 < res_unclamped < 2^60 */ ++ s64 res_unclamped = div_s64_rem(res_big, 1000, &remainder); ++ ++ /* Clamp to range of 0x to 10x the static power */ ++ return clamp(res_unclamped, (s64) 0, (s64) 10000000); ++} ++ ++static int model_static_coeff(struct kbase_ipa_model *model, u32 *coeffp) ++{ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) ++ unsigned long temp; ++#else ++ int temp; ++#endif ++ u32 temp_scaling_factor; ++ struct kbase_ipa_model_simple_data *model_data = ++ (struct kbase_ipa_model_simple_data *) model->model_data; ++ struct thermal_zone_device *gpu_tz = model_data->gpu_tz; ++ u64 coeffp_big; ++ ++ if (gpu_tz) { ++ int ret; ++ ++ ret = gpu_tz->ops->get_temp(gpu_tz, &temp); ++ if (ret) { ++ pr_warn_ratelimited("Error reading temperature for gpu thermal zone: %d\n", ++ ret); ++ temp = FALLBACK_STATIC_TEMPERATURE; ++ } ++ } else { ++ temp = FALLBACK_STATIC_TEMPERATURE; ++ } ++ ++ temp_scaling_factor = calculate_temp_scaling_factor(model_data->ts, ++ temp); ++ coeffp_big = (u64)model_data->static_coefficient * temp_scaling_factor; ++ *coeffp = div_u64(coeffp_big, 1000000); ++ ++ return 0; ++} ++ ++static int model_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp, ++ u32 current_freq) ++{ ++ struct kbase_ipa_model_simple_data *model_data = ++ (struct kbase_ipa_model_simple_data *) model->model_data; ++ ++ *coeffp = model_data->dynamic_coefficient; ++ ++ return 0; ++} ++ ++static int add_params(struct kbase_ipa_model *model) ++{ ++ int err = 0; ++ struct kbase_ipa_model_simple_data *model_data = ++ (struct kbase_ipa_model_simple_data *)model->model_data; ++ ++ err = kbase_ipa_model_add_param_s32(model, "static-coefficient", ++ &model_data->static_coefficient, ++ 1, true); ++ if (err) ++ goto end; ++ ++ err = kbase_ipa_model_add_param_s32(model, "dynamic-coefficient", ++ &model_data->dynamic_coefficient, ++ 1, true); ++ if (err) ++ goto end; ++ ++ err = kbase_ipa_model_add_param_s32(model, "ts", ++ model_data->ts, 4, true); ++ if (err) ++ goto end; ++ ++ err = kbase_ipa_model_add_param_string(model, "thermal-zone", ++ model_data->tz_name, ++ sizeof(model_data->tz_name), true); ++ ++end: ++ return err; ++} ++ ++static int kbase_simple_power_model_init(struct kbase_ipa_model *model) ++{ ++ int err; ++ struct kbase_ipa_model_simple_data *model_data; ++ ++ model_data = kzalloc(sizeof(struct kbase_ipa_model_simple_data), ++ GFP_KERNEL); ++ if (!model_data) ++ return -ENOMEM; ++ ++ model->model_data = (void *) model_data; ++ ++ err = add_params(model); ++ ++ return err; ++} ++ ++static int kbase_simple_power_model_recalculate(struct kbase_ipa_model *model) ++{ ++ struct kbase_ipa_model_simple_data *model_data = ++ (struct kbase_ipa_model_simple_data *)model->model_data; ++ ++ if (!strnlen(model_data->tz_name, sizeof(model_data->tz_name))) { ++ model_data->gpu_tz = NULL; ++ } else { ++ model_data->gpu_tz = thermal_zone_get_zone_by_name(model_data->tz_name); ++ ++ if (IS_ERR(model_data->gpu_tz)) { ++ pr_warn_ratelimited("Error %ld getting thermal zone \'%s\', not yet ready?\n", ++ PTR_ERR(model_data->gpu_tz), ++ model_data->tz_name); ++ model_data->gpu_tz = NULL; ++ return -EPROBE_DEFER; ++ } ++ } ++ ++ return 0; ++} ++ ++static void kbase_simple_power_model_term(struct kbase_ipa_model *model) ++{ ++ struct kbase_ipa_model_simple_data *model_data = ++ (struct kbase_ipa_model_simple_data *)model->model_data; ++ ++ kfree(model_data); ++} ++ ++struct kbase_ipa_model_ops kbase_simple_ipa_model_ops = { ++ .name = "mali-simple-power-model", ++ .init = &kbase_simple_power_model_init, ++ .recalculate = &kbase_simple_power_model_recalculate, ++ .term = &kbase_simple_power_model_term, ++ .get_dynamic_coeff = &model_dynamic_coeff, ++ .get_static_coeff = &model_static_coeff, ++ .do_utilization_scaling_in_framework = true, ++}; +diff --git a/drivers/gpu/arm/midgard/mali_base_hwconfig_features.h b/drivers/gpu/arm/midgard/mali_base_hwconfig_features.h +new file mode 100755 +index 000000000000..6be0a334f99f +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_base_hwconfig_features.h +@@ -0,0 +1,311 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* AUTOMATICALLY GENERATED FILE. If you want to amend the issues/features, ++ * please update base/tools/hwconfig_generator/hwc_{issues,features}.py ++ * For more information see base/tools/hwconfig_generator/README ++ */ ++ ++#ifndef _BASE_HWCONFIG_FEATURES_H_ ++#define _BASE_HWCONFIG_FEATURES_H_ ++ ++enum base_hw_feature { ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, ++ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, ++ BASE_HW_FEATURE_33BIT_VA, ++ BASE_HW_FEATURE_XAFFINITY, ++ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, ++ BASE_HW_FEATURE_MRT, ++ BASE_HW_FEATURE_BRNDOUT_CC, ++ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, ++ BASE_HW_FEATURE_LD_ST_TILEBUFFER, ++ BASE_HW_FEATURE_MSAA_16X, ++ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, ++ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, ++ BASE_HW_FEATURE_OPTIMIZED_COVERAGE_MASK, ++ BASE_HW_FEATURE_T7XX_PAIRING_RULES, ++ BASE_HW_FEATURE_LD_ST_LEA_TEX, ++ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, ++ BASE_HW_FEATURE_WORKGROUP_ROUND_MULTIPLE_OF_4, ++ BASE_HW_FEATURE_IMAGES_IN_FRAGMENT_SHADERS, ++ BASE_HW_FEATURE_TEST4_DATUM_MODE, ++ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, ++ BASE_HW_FEATURE_BRNDOUT_KILL, ++ BASE_HW_FEATURE_WARPING, ++ BASE_HW_FEATURE_V4, ++ BASE_HW_FEATURE_FLUSH_REDUCTION, ++ BASE_HW_FEATURE_PROTECTED_MODE, ++ BASE_HW_FEATURE_COHERENCY_REG, ++ BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, ++ BASE_HW_FEATURE_AARCH64_MMU, ++ BASE_HW_FEATURE_END ++}; ++ ++static const enum base_hw_feature base_hw_features_generic[] = { ++ BASE_HW_FEATURE_END ++}; ++ ++static const enum base_hw_feature base_hw_features_t60x[] = { ++ BASE_HW_FEATURE_LD_ST_LEA_TEX, ++ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, ++ BASE_HW_FEATURE_V4, ++ BASE_HW_FEATURE_END ++}; ++ ++static const enum base_hw_feature base_hw_features_t62x[] = { ++ BASE_HW_FEATURE_LD_ST_LEA_TEX, ++ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, ++ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, ++ BASE_HW_FEATURE_V4, ++ BASE_HW_FEATURE_END ++}; ++ ++static const enum base_hw_feature base_hw_features_t72x[] = { ++ BASE_HW_FEATURE_33BIT_VA, ++ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, ++ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, ++ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, ++ BASE_HW_FEATURE_OPTIMIZED_COVERAGE_MASK, ++ BASE_HW_FEATURE_T7XX_PAIRING_RULES, ++ BASE_HW_FEATURE_WORKGROUP_ROUND_MULTIPLE_OF_4, ++ BASE_HW_FEATURE_WARPING, ++ BASE_HW_FEATURE_V4, ++ BASE_HW_FEATURE_END ++}; ++ ++static const enum base_hw_feature base_hw_features_t76x[] = { ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, ++ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, ++ BASE_HW_FEATURE_XAFFINITY, ++ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, ++ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, ++ BASE_HW_FEATURE_BRNDOUT_CC, ++ BASE_HW_FEATURE_LD_ST_LEA_TEX, ++ BASE_HW_FEATURE_LD_ST_TILEBUFFER, ++ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, ++ BASE_HW_FEATURE_MRT, ++ BASE_HW_FEATURE_MSAA_16X, ++ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, ++ BASE_HW_FEATURE_T7XX_PAIRING_RULES, ++ BASE_HW_FEATURE_TEST4_DATUM_MODE, ++ BASE_HW_FEATURE_END ++}; ++ ++static const enum base_hw_feature base_hw_features_tFxx[] = { ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, ++ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, ++ BASE_HW_FEATURE_XAFFINITY, ++ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, ++ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, ++ BASE_HW_FEATURE_BRNDOUT_CC, ++ BASE_HW_FEATURE_BRNDOUT_KILL, ++ BASE_HW_FEATURE_LD_ST_LEA_TEX, ++ BASE_HW_FEATURE_LD_ST_TILEBUFFER, ++ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, ++ BASE_HW_FEATURE_MRT, ++ BASE_HW_FEATURE_MSAA_16X, ++ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, ++ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, ++ BASE_HW_FEATURE_T7XX_PAIRING_RULES, ++ BASE_HW_FEATURE_TEST4_DATUM_MODE, ++ BASE_HW_FEATURE_END ++}; ++ ++static const enum base_hw_feature base_hw_features_t83x[] = { ++ BASE_HW_FEATURE_33BIT_VA, ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, ++ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, ++ BASE_HW_FEATURE_XAFFINITY, ++ BASE_HW_FEATURE_WARPING, ++ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, ++ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, ++ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, ++ BASE_HW_FEATURE_BRNDOUT_CC, ++ BASE_HW_FEATURE_BRNDOUT_KILL, ++ BASE_HW_FEATURE_LD_ST_LEA_TEX, ++ BASE_HW_FEATURE_LD_ST_TILEBUFFER, ++ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, ++ BASE_HW_FEATURE_MRT, ++ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, ++ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, ++ BASE_HW_FEATURE_T7XX_PAIRING_RULES, ++ BASE_HW_FEATURE_TEST4_DATUM_MODE, ++ BASE_HW_FEATURE_END ++}; ++ ++static const enum base_hw_feature base_hw_features_t82x[] = { ++ BASE_HW_FEATURE_33BIT_VA, ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, ++ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, ++ BASE_HW_FEATURE_XAFFINITY, ++ BASE_HW_FEATURE_WARPING, ++ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, ++ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, ++ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, ++ BASE_HW_FEATURE_BRNDOUT_CC, ++ BASE_HW_FEATURE_BRNDOUT_KILL, ++ BASE_HW_FEATURE_LD_ST_LEA_TEX, ++ BASE_HW_FEATURE_LD_ST_TILEBUFFER, ++ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, ++ BASE_HW_FEATURE_MRT, ++ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, ++ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, ++ BASE_HW_FEATURE_T7XX_PAIRING_RULES, ++ BASE_HW_FEATURE_TEST4_DATUM_MODE, ++ BASE_HW_FEATURE_END ++}; ++ ++static const enum base_hw_feature base_hw_features_tMIx[] = { ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, ++ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, ++ BASE_HW_FEATURE_XAFFINITY, ++ BASE_HW_FEATURE_WARPING, ++ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, ++ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, ++ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, ++ BASE_HW_FEATURE_BRNDOUT_CC, ++ BASE_HW_FEATURE_BRNDOUT_KILL, ++ BASE_HW_FEATURE_LD_ST_LEA_TEX, ++ BASE_HW_FEATURE_LD_ST_TILEBUFFER, ++ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, ++ BASE_HW_FEATURE_MRT, ++ BASE_HW_FEATURE_MSAA_16X, ++ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, ++ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, ++ BASE_HW_FEATURE_T7XX_PAIRING_RULES, ++ BASE_HW_FEATURE_TEST4_DATUM_MODE, ++ BASE_HW_FEATURE_FLUSH_REDUCTION, ++ BASE_HW_FEATURE_PROTECTED_MODE, ++ BASE_HW_FEATURE_COHERENCY_REG, ++ BASE_HW_FEATURE_END ++}; ++ ++static const enum base_hw_feature base_hw_features_tHEx[] = { ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, ++ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, ++ BASE_HW_FEATURE_XAFFINITY, ++ BASE_HW_FEATURE_WARPING, ++ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, ++ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, ++ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, ++ BASE_HW_FEATURE_BRNDOUT_CC, ++ BASE_HW_FEATURE_BRNDOUT_KILL, ++ BASE_HW_FEATURE_LD_ST_LEA_TEX, ++ BASE_HW_FEATURE_LD_ST_TILEBUFFER, ++ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, ++ BASE_HW_FEATURE_MRT, ++ BASE_HW_FEATURE_MSAA_16X, ++ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, ++ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, ++ BASE_HW_FEATURE_T7XX_PAIRING_RULES, ++ BASE_HW_FEATURE_TEST4_DATUM_MODE, ++ BASE_HW_FEATURE_FLUSH_REDUCTION, ++ BASE_HW_FEATURE_PROTECTED_MODE, ++ BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, ++ BASE_HW_FEATURE_COHERENCY_REG, ++ BASE_HW_FEATURE_END ++}; ++ ++static const enum base_hw_feature base_hw_features_tSIx[] = { ++ BASE_HW_FEATURE_33BIT_VA, ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, ++ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, ++ BASE_HW_FEATURE_XAFFINITY, ++ BASE_HW_FEATURE_WARPING, ++ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, ++ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, ++ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, ++ BASE_HW_FEATURE_BRNDOUT_CC, ++ BASE_HW_FEATURE_BRNDOUT_KILL, ++ BASE_HW_FEATURE_LD_ST_LEA_TEX, ++ BASE_HW_FEATURE_LD_ST_TILEBUFFER, ++ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, ++ BASE_HW_FEATURE_MRT, ++ BASE_HW_FEATURE_MSAA_16X, ++ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, ++ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, ++ BASE_HW_FEATURE_T7XX_PAIRING_RULES, ++ BASE_HW_FEATURE_TEST4_DATUM_MODE, ++ BASE_HW_FEATURE_FLUSH_REDUCTION, ++ BASE_HW_FEATURE_PROTECTED_MODE, ++ BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, ++ BASE_HW_FEATURE_COHERENCY_REG, ++ BASE_HW_FEATURE_END ++}; ++ ++ ++#ifdef MALI_INCLUDE_TKAX ++static const enum base_hw_feature base_hw_features_tKAx[] = { ++ BASE_HW_FEATURE_33BIT_VA, ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, ++ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, ++ BASE_HW_FEATURE_XAFFINITY, ++ BASE_HW_FEATURE_WARPING, ++ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, ++ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, ++ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, ++ BASE_HW_FEATURE_BRNDOUT_CC, ++ BASE_HW_FEATURE_BRNDOUT_KILL, ++ BASE_HW_FEATURE_LD_ST_LEA_TEX, ++ BASE_HW_FEATURE_LD_ST_TILEBUFFER, ++ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, ++ BASE_HW_FEATURE_MRT, ++ BASE_HW_FEATURE_MSAA_16X, ++ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, ++ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, ++ BASE_HW_FEATURE_T7XX_PAIRING_RULES, ++ BASE_HW_FEATURE_TEST4_DATUM_MODE, ++ BASE_HW_FEATURE_FLUSH_REDUCTION, ++ BASE_HW_FEATURE_PROTECTED_MODE, ++ BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, ++ BASE_HW_FEATURE_COHERENCY_REG, ++ BASE_HW_FEATURE_END ++}; ++ ++#endif /* MALI_INCLUDE_TKAX */ ++ ++#ifdef MALI_INCLUDE_TTRX ++static const enum base_hw_feature base_hw_features_tTRx[] = { ++ BASE_HW_FEATURE_33BIT_VA, ++ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, ++ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, ++ BASE_HW_FEATURE_XAFFINITY, ++ BASE_HW_FEATURE_WARPING, ++ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, ++ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, ++ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, ++ BASE_HW_FEATURE_BRNDOUT_CC, ++ BASE_HW_FEATURE_BRNDOUT_KILL, ++ BASE_HW_FEATURE_LD_ST_LEA_TEX, ++ BASE_HW_FEATURE_LD_ST_TILEBUFFER, ++ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, ++ BASE_HW_FEATURE_MRT, ++ BASE_HW_FEATURE_MSAA_16X, ++ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, ++ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, ++ BASE_HW_FEATURE_T7XX_PAIRING_RULES, ++ BASE_HW_FEATURE_TEST4_DATUM_MODE, ++ BASE_HW_FEATURE_FLUSH_REDUCTION, ++ BASE_HW_FEATURE_PROTECTED_MODE, ++ BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, ++ BASE_HW_FEATURE_COHERENCY_REG, ++ BASE_HW_FEATURE_END ++}; ++ ++#endif /* MALI_INCLUDE_TTRX */ ++ ++#endif /* _BASE_HWCONFIG_FEATURES_H_ */ +diff --git a/drivers/gpu/arm/midgard/mali_base_hwconfig_issues.h b/drivers/gpu/arm/midgard/mali_base_hwconfig_issues.h +new file mode 100755 +index 000000000000..6d7e5c57e6a4 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_base_hwconfig_issues.h +@@ -0,0 +1,1098 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* AUTOMATICALLY GENERATED FILE. If you want to amend the issues/features, ++ * please update base/tools/hwconfig_generator/hwc_{issues,features}.py ++ * For more information see base/tools/hwconfig_generator/README ++ */ ++ ++#ifndef _BASE_HWCONFIG_ISSUES_H_ ++#define _BASE_HWCONFIG_ISSUES_H_ ++ ++enum base_hw_issue { ++ BASE_HW_ISSUE_5736, ++ BASE_HW_ISSUE_6367, ++ BASE_HW_ISSUE_6398, ++ BASE_HW_ISSUE_6402, ++ BASE_HW_ISSUE_6787, ++ BASE_HW_ISSUE_7027, ++ BASE_HW_ISSUE_7144, ++ BASE_HW_ISSUE_7304, ++ BASE_HW_ISSUE_8073, ++ BASE_HW_ISSUE_8186, ++ BASE_HW_ISSUE_8215, ++ BASE_HW_ISSUE_8245, ++ BASE_HW_ISSUE_8250, ++ BASE_HW_ISSUE_8260, ++ BASE_HW_ISSUE_8280, ++ BASE_HW_ISSUE_8316, ++ BASE_HW_ISSUE_8381, ++ BASE_HW_ISSUE_8394, ++ BASE_HW_ISSUE_8401, ++ BASE_HW_ISSUE_8408, ++ BASE_HW_ISSUE_8443, ++ BASE_HW_ISSUE_8456, ++ BASE_HW_ISSUE_8564, ++ BASE_HW_ISSUE_8634, ++ BASE_HW_ISSUE_8778, ++ BASE_HW_ISSUE_8791, ++ BASE_HW_ISSUE_8833, ++ BASE_HW_ISSUE_8879, ++ BASE_HW_ISSUE_8896, ++ BASE_HW_ISSUE_8975, ++ BASE_HW_ISSUE_8986, ++ BASE_HW_ISSUE_8987, ++ BASE_HW_ISSUE_9010, ++ BASE_HW_ISSUE_9418, ++ BASE_HW_ISSUE_9423, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_9510, ++ BASE_HW_ISSUE_9566, ++ BASE_HW_ISSUE_9630, ++ BASE_HW_ISSUE_10127, ++ BASE_HW_ISSUE_10327, ++ BASE_HW_ISSUE_10410, ++ BASE_HW_ISSUE_10471, ++ BASE_HW_ISSUE_10472, ++ BASE_HW_ISSUE_10487, ++ BASE_HW_ISSUE_10607, ++ BASE_HW_ISSUE_10632, ++ BASE_HW_ISSUE_10649, ++ BASE_HW_ISSUE_10676, ++ BASE_HW_ISSUE_10682, ++ BASE_HW_ISSUE_10684, ++ BASE_HW_ISSUE_10797, ++ BASE_HW_ISSUE_10817, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10931, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_10959, ++ BASE_HW_ISSUE_10969, ++ BASE_HW_ISSUE_10984, ++ BASE_HW_ISSUE_10995, ++ BASE_HW_ISSUE_11012, ++ BASE_HW_ISSUE_11020, ++ BASE_HW_ISSUE_11024, ++ BASE_HW_ISSUE_11035, ++ BASE_HW_ISSUE_11042, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T720_1386, ++ BASE_HW_ISSUE_T76X_26, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3542, ++ BASE_HW_ISSUE_T76X_3556, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3953, ++ BASE_HW_ISSUE_T76X_3960, ++ BASE_HW_ISSUE_T76X_3964, ++ BASE_HW_ISSUE_T76X_3966, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_7940, ++ BASE_HW_ISSUE_TMIX_8042, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TMIX_8138, ++ BASE_HW_ISSUE_TMIX_8206, ++ BASE_HW_ISSUE_TMIX_8343, ++ BASE_HW_ISSUE_TMIX_8463, ++ BASE_HW_ISSUE_TMIX_8456, ++ GPUCORE_1619, ++ BASE_HW_ISSUE_TSIX_1116, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_generic[] = { ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t60x_r0p0_15dev0[] = { ++ BASE_HW_ISSUE_6367, ++ BASE_HW_ISSUE_6398, ++ BASE_HW_ISSUE_6402, ++ BASE_HW_ISSUE_6787, ++ BASE_HW_ISSUE_7027, ++ BASE_HW_ISSUE_7144, ++ BASE_HW_ISSUE_7304, ++ BASE_HW_ISSUE_8073, ++ BASE_HW_ISSUE_8186, ++ BASE_HW_ISSUE_8215, ++ BASE_HW_ISSUE_8245, ++ BASE_HW_ISSUE_8250, ++ BASE_HW_ISSUE_8260, ++ BASE_HW_ISSUE_8280, ++ BASE_HW_ISSUE_8316, ++ BASE_HW_ISSUE_8381, ++ BASE_HW_ISSUE_8394, ++ BASE_HW_ISSUE_8401, ++ BASE_HW_ISSUE_8408, ++ BASE_HW_ISSUE_8443, ++ BASE_HW_ISSUE_8456, ++ BASE_HW_ISSUE_8564, ++ BASE_HW_ISSUE_8634, ++ BASE_HW_ISSUE_8778, ++ BASE_HW_ISSUE_8791, ++ BASE_HW_ISSUE_8833, ++ BASE_HW_ISSUE_8896, ++ BASE_HW_ISSUE_8975, ++ BASE_HW_ISSUE_8986, ++ BASE_HW_ISSUE_8987, ++ BASE_HW_ISSUE_9010, ++ BASE_HW_ISSUE_9418, ++ BASE_HW_ISSUE_9423, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_9510, ++ BASE_HW_ISSUE_9566, ++ BASE_HW_ISSUE_9630, ++ BASE_HW_ISSUE_10410, ++ BASE_HW_ISSUE_10471, ++ BASE_HW_ISSUE_10472, ++ BASE_HW_ISSUE_10487, ++ BASE_HW_ISSUE_10607, ++ BASE_HW_ISSUE_10632, ++ BASE_HW_ISSUE_10649, ++ BASE_HW_ISSUE_10676, ++ BASE_HW_ISSUE_10682, ++ BASE_HW_ISSUE_10684, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10931, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_10969, ++ BASE_HW_ISSUE_10984, ++ BASE_HW_ISSUE_10995, ++ BASE_HW_ISSUE_11012, ++ BASE_HW_ISSUE_11020, ++ BASE_HW_ISSUE_11035, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_3964, ++ GPUCORE_1619, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t60x_r0p0_eac[] = { ++ BASE_HW_ISSUE_6367, ++ BASE_HW_ISSUE_6402, ++ BASE_HW_ISSUE_6787, ++ BASE_HW_ISSUE_7027, ++ BASE_HW_ISSUE_7304, ++ BASE_HW_ISSUE_8408, ++ BASE_HW_ISSUE_8564, ++ BASE_HW_ISSUE_8778, ++ BASE_HW_ISSUE_8975, ++ BASE_HW_ISSUE_9010, ++ BASE_HW_ISSUE_9418, ++ BASE_HW_ISSUE_9423, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_9510, ++ BASE_HW_ISSUE_10410, ++ BASE_HW_ISSUE_10471, ++ BASE_HW_ISSUE_10472, ++ BASE_HW_ISSUE_10487, ++ BASE_HW_ISSUE_10607, ++ BASE_HW_ISSUE_10632, ++ BASE_HW_ISSUE_10649, ++ BASE_HW_ISSUE_10676, ++ BASE_HW_ISSUE_10682, ++ BASE_HW_ISSUE_10684, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10931, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_10969, ++ BASE_HW_ISSUE_11012, ++ BASE_HW_ISSUE_11020, ++ BASE_HW_ISSUE_11035, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_3964, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t60x_r0p1[] = { ++ BASE_HW_ISSUE_6367, ++ BASE_HW_ISSUE_6402, ++ BASE_HW_ISSUE_6787, ++ BASE_HW_ISSUE_7027, ++ BASE_HW_ISSUE_7304, ++ BASE_HW_ISSUE_8408, ++ BASE_HW_ISSUE_8564, ++ BASE_HW_ISSUE_8778, ++ BASE_HW_ISSUE_8975, ++ BASE_HW_ISSUE_9010, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_9510, ++ BASE_HW_ISSUE_10410, ++ BASE_HW_ISSUE_10471, ++ BASE_HW_ISSUE_10472, ++ BASE_HW_ISSUE_10487, ++ BASE_HW_ISSUE_10607, ++ BASE_HW_ISSUE_10632, ++ BASE_HW_ISSUE_10649, ++ BASE_HW_ISSUE_10676, ++ BASE_HW_ISSUE_10682, ++ BASE_HW_ISSUE_10684, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10931, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_11012, ++ BASE_HW_ISSUE_11020, ++ BASE_HW_ISSUE_11035, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3964, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t62x_r0p1[] = { ++ BASE_HW_ISSUE_6402, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10127, ++ BASE_HW_ISSUE_10327, ++ BASE_HW_ISSUE_10410, ++ BASE_HW_ISSUE_10471, ++ BASE_HW_ISSUE_10472, ++ BASE_HW_ISSUE_10487, ++ BASE_HW_ISSUE_10607, ++ BASE_HW_ISSUE_10632, ++ BASE_HW_ISSUE_10649, ++ BASE_HW_ISSUE_10676, ++ BASE_HW_ISSUE_10682, ++ BASE_HW_ISSUE_10684, ++ BASE_HW_ISSUE_10817, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10931, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_10959, ++ BASE_HW_ISSUE_11012, ++ BASE_HW_ISSUE_11020, ++ BASE_HW_ISSUE_11024, ++ BASE_HW_ISSUE_11035, ++ BASE_HW_ISSUE_11042, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t62x_r1p0[] = { ++ BASE_HW_ISSUE_6402, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10471, ++ BASE_HW_ISSUE_10472, ++ BASE_HW_ISSUE_10649, ++ BASE_HW_ISSUE_10684, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10931, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_10959, ++ BASE_HW_ISSUE_11012, ++ BASE_HW_ISSUE_11020, ++ BASE_HW_ISSUE_11024, ++ BASE_HW_ISSUE_11042, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3964, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t62x_r1p1[] = { ++ BASE_HW_ISSUE_6402, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10471, ++ BASE_HW_ISSUE_10472, ++ BASE_HW_ISSUE_10649, ++ BASE_HW_ISSUE_10684, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10931, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_10959, ++ BASE_HW_ISSUE_11012, ++ BASE_HW_ISSUE_11042, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t76x_r0p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_11020, ++ BASE_HW_ISSUE_11024, ++ BASE_HW_ISSUE_11042, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T76X_26, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3542, ++ BASE_HW_ISSUE_T76X_3556, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3953, ++ BASE_HW_ISSUE_T76X_3960, ++ BASE_HW_ISSUE_T76X_3964, ++ BASE_HW_ISSUE_T76X_3966, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t76x_r0p1[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_11020, ++ BASE_HW_ISSUE_11024, ++ BASE_HW_ISSUE_11042, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T76X_26, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3542, ++ BASE_HW_ISSUE_T76X_3556, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3953, ++ BASE_HW_ISSUE_T76X_3960, ++ BASE_HW_ISSUE_T76X_3964, ++ BASE_HW_ISSUE_T76X_3966, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t76x_r0p1_50rel0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_11042, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T76X_26, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3542, ++ BASE_HW_ISSUE_T76X_3556, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3953, ++ BASE_HW_ISSUE_T76X_3960, ++ BASE_HW_ISSUE_T76X_3964, ++ BASE_HW_ISSUE_T76X_3966, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t76x_r0p2[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_11020, ++ BASE_HW_ISSUE_11024, ++ BASE_HW_ISSUE_11042, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T76X_26, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3542, ++ BASE_HW_ISSUE_T76X_3556, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3953, ++ BASE_HW_ISSUE_T76X_3960, ++ BASE_HW_ISSUE_T76X_3964, ++ BASE_HW_ISSUE_T76X_3966, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t76x_r0p3[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_11042, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T76X_26, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3542, ++ BASE_HW_ISSUE_T76X_3556, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3953, ++ BASE_HW_ISSUE_T76X_3960, ++ BASE_HW_ISSUE_T76X_3964, ++ BASE_HW_ISSUE_T76X_3966, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t76x_r1p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_11042, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3953, ++ BASE_HW_ISSUE_T76X_3960, ++ BASE_HW_ISSUE_T76X_3964, ++ BASE_HW_ISSUE_T76X_3966, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t72x_r0p0[] = { ++ BASE_HW_ISSUE_6402, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10471, ++ BASE_HW_ISSUE_10649, ++ BASE_HW_ISSUE_10684, ++ BASE_HW_ISSUE_10797, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_11042, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3964, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t72x_r1p0[] = { ++ BASE_HW_ISSUE_6402, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10471, ++ BASE_HW_ISSUE_10649, ++ BASE_HW_ISSUE_10684, ++ BASE_HW_ISSUE_10797, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_11042, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T720_1386, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3964, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t72x_r1p1[] = { ++ BASE_HW_ISSUE_6402, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10471, ++ BASE_HW_ISSUE_10649, ++ BASE_HW_ISSUE_10684, ++ BASE_HW_ISSUE_10797, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_11042, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T720_1386, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3964, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_model_t72x[] = { ++ BASE_HW_ISSUE_5736, ++ BASE_HW_ISSUE_6402, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10471, ++ BASE_HW_ISSUE_10649, ++ BASE_HW_ISSUE_10797, ++ BASE_HW_ISSUE_11042, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3964, ++ GPUCORE_1619, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_model_t76x[] = { ++ BASE_HW_ISSUE_5736, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_11020, ++ BASE_HW_ISSUE_11024, ++ BASE_HW_ISSUE_11042, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3964, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ GPUCORE_1619, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_model_t60x[] = { ++ BASE_HW_ISSUE_5736, ++ BASE_HW_ISSUE_6402, ++ BASE_HW_ISSUE_8778, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10472, ++ BASE_HW_ISSUE_10649, ++ BASE_HW_ISSUE_10931, ++ BASE_HW_ISSUE_11012, ++ BASE_HW_ISSUE_11020, ++ BASE_HW_ISSUE_11024, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3964, ++ GPUCORE_1619, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_model_t62x[] = { ++ BASE_HW_ISSUE_5736, ++ BASE_HW_ISSUE_6402, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10472, ++ BASE_HW_ISSUE_10649, ++ BASE_HW_ISSUE_10931, ++ BASE_HW_ISSUE_11012, ++ BASE_HW_ISSUE_11020, ++ BASE_HW_ISSUE_11024, ++ BASE_HW_ISSUE_11042, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3964, ++ GPUCORE_1619, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tFRx_r0p1[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3953, ++ BASE_HW_ISSUE_T76X_3960, ++ BASE_HW_ISSUE_T76X_3964, ++ BASE_HW_ISSUE_T76X_3966, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tFRx_r0p2[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3953, ++ BASE_HW_ISSUE_T76X_3964, ++ BASE_HW_ISSUE_T76X_3966, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tFRx_r1p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3953, ++ BASE_HW_ISSUE_T76X_3966, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tFRx_r2p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3953, ++ BASE_HW_ISSUE_T76X_3966, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_model_tFRx[] = { ++ BASE_HW_ISSUE_5736, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3964, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ GPUCORE_1619, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t86x_r0p2[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3953, ++ BASE_HW_ISSUE_T76X_3964, ++ BASE_HW_ISSUE_T76X_3966, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t86x_r1p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3953, ++ BASE_HW_ISSUE_T76X_3966, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t86x_r2p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3953, ++ BASE_HW_ISSUE_T76X_3966, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_model_t86x[] = { ++ BASE_HW_ISSUE_5736, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ GPUCORE_1619, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t83x_r0p1[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T720_1386, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3953, ++ BASE_HW_ISSUE_T76X_3960, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t83x_r1p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T720_1386, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3953, ++ BASE_HW_ISSUE_T76X_3960, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_model_t83x[] = { ++ BASE_HW_ISSUE_5736, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3964, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ GPUCORE_1619, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t82x_r0p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T720_1386, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3953, ++ BASE_HW_ISSUE_T76X_3960, ++ BASE_HW_ISSUE_T76X_3964, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t82x_r0p1[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T720_1386, ++ BASE_HW_ISSUE_T76X_1909, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3953, ++ BASE_HW_ISSUE_T76X_3960, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_t82x_r1p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10821, ++ BASE_HW_ISSUE_10883, ++ BASE_HW_ISSUE_10946, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T720_1386, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3953, ++ BASE_HW_ISSUE_T76X_3960, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_model_t82x[] = { ++ BASE_HW_ISSUE_5736, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_11051, ++ BASE_HW_ISSUE_T76X_1963, ++ BASE_HW_ISSUE_T76X_3086, ++ BASE_HW_ISSUE_T76X_3700, ++ BASE_HW_ISSUE_T76X_3793, ++ BASE_HW_ISSUE_T76X_3979, ++ BASE_HW_ISSUE_TMIX_7891, ++ GPUCORE_1619, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tMIx_r0p0_05dev0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10682, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_T76X_3953, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8042, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TMIX_8138, ++ BASE_HW_ISSUE_TMIX_8206, ++ BASE_HW_ISSUE_TMIX_8343, ++ BASE_HW_ISSUE_TMIX_8463, ++ BASE_HW_ISSUE_TMIX_8456, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tMIx_r0p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10682, ++ BASE_HW_ISSUE_11054, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_7940, ++ BASE_HW_ISSUE_TMIX_8042, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TMIX_8138, ++ BASE_HW_ISSUE_TMIX_8206, ++ BASE_HW_ISSUE_TMIX_8343, ++ BASE_HW_ISSUE_TMIX_8463, ++ BASE_HW_ISSUE_TMIX_8456, ++ BASE_HW_ISSUE_TMIX_8438, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_model_tMIx[] = { ++ BASE_HW_ISSUE_5736, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_7940, ++ BASE_HW_ISSUE_TMIX_8042, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TMIX_8138, ++ BASE_HW_ISSUE_TMIX_8206, ++ BASE_HW_ISSUE_TMIX_8343, ++ BASE_HW_ISSUE_TMIX_8456, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tHEx_r0p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10682, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8042, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tHEx_r0p1[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_10682, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8042, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_model_tHEx[] = { ++ BASE_HW_ISSUE_5736, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TMIX_7891, ++ BASE_HW_ISSUE_TMIX_8042, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tSIx_r0p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TSIX_1116, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tSIx_r0p1[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TSIX_1116, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_tSIx_r1p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TSIX_1116, ++ BASE_HW_ISSUE_END ++}; ++ ++static const enum base_hw_issue base_hw_issues_model_tSIx[] = { ++ BASE_HW_ISSUE_5736, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TSIX_1116, ++ BASE_HW_ISSUE_END ++}; ++ ++ ++ ++#ifdef MALI_INCLUDE_TKAX ++static const enum base_hw_issue base_hw_issues_tKAx_r0p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TSIX_1116, ++ BASE_HW_ISSUE_END ++}; ++ ++#endif /* MALI_INCLUDE_TKAX */ ++ ++#ifdef MALI_INCLUDE_TKAX ++static const enum base_hw_issue base_hw_issues_model_tKAx[] = { ++ BASE_HW_ISSUE_5736, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TSIX_1116, ++ BASE_HW_ISSUE_END ++}; ++ ++#endif /* MALI_INCLUDE_TKAX */ ++ ++#ifdef MALI_INCLUDE_TTRX ++static const enum base_hw_issue base_hw_issues_tTRx_r0p0[] = { ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TSIX_1116, ++ BASE_HW_ISSUE_END ++}; ++ ++#endif /* MALI_INCLUDE_TTRX */ ++ ++#ifdef MALI_INCLUDE_TTRX ++static const enum base_hw_issue base_hw_issues_model_tTRx[] = { ++ BASE_HW_ISSUE_5736, ++ BASE_HW_ISSUE_9435, ++ BASE_HW_ISSUE_TMIX_8133, ++ BASE_HW_ISSUE_TSIX_1116, ++ BASE_HW_ISSUE_END ++}; ++ ++#endif /* MALI_INCLUDE_TTRX */ ++ ++#endif /* _BASE_HWCONFIG_ISSUES_H_ */ +diff --git a/drivers/gpu/arm/midgard/mali_base_kernel.h b/drivers/gpu/arm/midgard/mali_base_kernel.h +new file mode 100755 +index 000000000000..ea5e473caef6 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_base_kernel.h +@@ -0,0 +1,1858 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/** ++ * @file ++ * Base structures shared with the kernel. ++ */ ++ ++#ifndef _BASE_KERNEL_H_ ++#define _BASE_KERNEL_H_ ++ ++#ifndef __user ++#define __user ++#endif ++ ++/* Support UK6 IOCTLS */ ++#define BASE_LEGACY_UK6_SUPPORT 1 ++ ++/* Support UK7 IOCTLS */ ++/* NB: To support UK6 we also need to support UK7 */ ++#define BASE_LEGACY_UK7_SUPPORT 1 ++ ++/* Support UK8 IOCTLS */ ++#define BASE_LEGACY_UK8_SUPPORT 1 ++ ++/* Support UK9 IOCTLS */ ++#define BASE_LEGACY_UK9_SUPPORT 1 ++ ++/* Support UK10_2 IOCTLS */ ++#define BASE_LEGACY_UK10_2_SUPPORT 1 ++ ++/* Support UK10_4 IOCTLS */ ++#define BASE_LEGACY_UK10_4_SUPPORT 1 ++ ++typedef struct base_mem_handle { ++ struct { ++ u64 handle; ++ } basep; ++} base_mem_handle; ++ ++#include "mali_base_mem_priv.h" ++#include "mali_kbase_profiling_gator_api.h" ++#include "mali_midg_coherency.h" ++#include "mali_kbase_gpu_id.h" ++ ++/* ++ * Dependency stuff, keep it private for now. May want to expose it if ++ * we decide to make the number of semaphores a configurable ++ * option. ++ */ ++#define BASE_JD_ATOM_COUNT 512 ++ ++#define BASEP_JD_SEM_PER_WORD_LOG2 5 ++#define BASEP_JD_SEM_PER_WORD (1 << BASEP_JD_SEM_PER_WORD_LOG2) ++#define BASEP_JD_SEM_WORD_NR(x) ((x) >> BASEP_JD_SEM_PER_WORD_LOG2) ++#define BASEP_JD_SEM_MASK_IN_WORD(x) (1 << ((x) & (BASEP_JD_SEM_PER_WORD - 1))) ++#define BASEP_JD_SEM_ARRAY_SIZE BASEP_JD_SEM_WORD_NR(BASE_JD_ATOM_COUNT) ++ ++/* Set/reset values for a software event */ ++#define BASE_JD_SOFT_EVENT_SET ((unsigned char)1) ++#define BASE_JD_SOFT_EVENT_RESET ((unsigned char)0) ++ ++#define BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS 3 ++ ++#define BASE_MAX_COHERENT_GROUPS 16 ++ ++#if defined CDBG_ASSERT ++#define LOCAL_ASSERT CDBG_ASSERT ++#elif defined KBASE_DEBUG_ASSERT ++#define LOCAL_ASSERT KBASE_DEBUG_ASSERT ++#else ++#error assert macro not defined! ++#endif ++ ++#if defined PAGE_MASK ++#define LOCAL_PAGE_LSB ~PAGE_MASK ++#else ++#include ++ ++#if defined OSU_CONFIG_CPU_PAGE_SIZE_LOG2 ++#define LOCAL_PAGE_LSB ((1ul << OSU_CONFIG_CPU_PAGE_SIZE_LOG2) - 1) ++#else ++#error Failed to find page size ++#endif ++#endif ++ ++/** 32/64-bit neutral way to represent pointers */ ++typedef union kbase_pointer { ++ void __user *value; /**< client should store their pointers here */ ++ u32 compat_value; /**< 64-bit kernels should fetch value here when handling 32-bit clients */ ++ u64 sizer; /**< Force 64-bit storage for all clients regardless */ ++} kbase_pointer; ++ ++/** ++ * @addtogroup base_user_api User-side Base APIs ++ * @{ ++ */ ++ ++/** ++ * @addtogroup base_user_api_memory User-side Base Memory APIs ++ * @{ ++ */ ++ ++/** ++ * typedef base_mem_alloc_flags - Memory allocation, access/hint flags. ++ * ++ * A combination of MEM_PROT/MEM_HINT flags must be passed to each allocator ++ * in order to determine the best cache policy. Some combinations are ++ * of course invalid (e.g. MEM_PROT_CPU_WR | MEM_HINT_CPU_RD), ++ * which defines a write-only region on the CPU side, which is ++ * heavily read by the CPU... ++ * Other flags are only meaningful to a particular allocator. ++ * More flags can be added to this list, as long as they don't clash ++ * (see BASE_MEM_FLAGS_NR_BITS for the number of the first free bit). ++ */ ++typedef u32 base_mem_alloc_flags; ++ ++/* Memory allocation, access/hint flags. ++ * ++ * See base_mem_alloc_flags. ++ */ ++ ++/* IN */ ++/* Read access CPU side ++ */ ++#define BASE_MEM_PROT_CPU_RD ((base_mem_alloc_flags)1 << 0) ++ ++/* Write access CPU side ++ */ ++#define BASE_MEM_PROT_CPU_WR ((base_mem_alloc_flags)1 << 1) ++ ++/* Read access GPU side ++ */ ++#define BASE_MEM_PROT_GPU_RD ((base_mem_alloc_flags)1 << 2) ++ ++/* Write access GPU side ++ */ ++#define BASE_MEM_PROT_GPU_WR ((base_mem_alloc_flags)1 << 3) ++ ++/* Execute allowed on the GPU side ++ */ ++#define BASE_MEM_PROT_GPU_EX ((base_mem_alloc_flags)1 << 4) ++ ++ /* BASE_MEM_HINT flags have been removed, but their values are reserved ++ * for backwards compatibility with older user-space drivers. The values ++ * can be re-used once support for r5p0 user-space drivers is removed, ++ * presumably in r7p0. ++ * ++ * RESERVED: (1U << 5) ++ * RESERVED: (1U << 6) ++ * RESERVED: (1U << 7) ++ * RESERVED: (1U << 8) ++ */ ++ ++/* Grow backing store on GPU Page Fault ++ */ ++#define BASE_MEM_GROW_ON_GPF ((base_mem_alloc_flags)1 << 9) ++ ++/* Page coherence Outer shareable, if available ++ */ ++#define BASE_MEM_COHERENT_SYSTEM ((base_mem_alloc_flags)1 << 10) ++ ++/* Page coherence Inner shareable ++ */ ++#define BASE_MEM_COHERENT_LOCAL ((base_mem_alloc_flags)1 << 11) ++ ++/* Should be cached on the CPU ++ */ ++#define BASE_MEM_CACHED_CPU ((base_mem_alloc_flags)1 << 12) ++ ++/* IN/OUT */ ++/* Must have same VA on both the GPU and the CPU ++ */ ++#define BASE_MEM_SAME_VA ((base_mem_alloc_flags)1 << 13) ++ ++/* OUT */ ++/* Must call mmap to acquire a GPU address for the alloc ++ */ ++#define BASE_MEM_NEED_MMAP ((base_mem_alloc_flags)1 << 14) ++ ++/* IN */ ++/* Page coherence Outer shareable, required. ++ */ ++#define BASE_MEM_COHERENT_SYSTEM_REQUIRED ((base_mem_alloc_flags)1 << 15) ++ ++/* Secure memory ++ */ ++#define BASE_MEM_SECURE ((base_mem_alloc_flags)1 << 16) ++ ++/* Not needed physical memory ++ */ ++#define BASE_MEM_DONT_NEED ((base_mem_alloc_flags)1 << 17) ++ ++/* Must use shared CPU/GPU zone (SAME_VA zone) but doesn't require the ++ * addresses to be the same ++ */ ++#define BASE_MEM_IMPORT_SHARED ((base_mem_alloc_flags)1 << 18) ++ ++/* Number of bits used as flags for base memory management ++ * ++ * Must be kept in sync with the base_mem_alloc_flags flags ++ */ ++#define BASE_MEM_FLAGS_NR_BITS 19 ++ ++/* A mask for all output bits, excluding IN/OUT bits. ++ */ ++#define BASE_MEM_FLAGS_OUTPUT_MASK BASE_MEM_NEED_MMAP ++ ++/* A mask for all input bits, including IN/OUT bits. ++ */ ++#define BASE_MEM_FLAGS_INPUT_MASK \ ++ (((1 << BASE_MEM_FLAGS_NR_BITS) - 1) & ~BASE_MEM_FLAGS_OUTPUT_MASK) ++ ++/* A mask for all the flags which are modifiable via the base_mem_set_flags ++ * interface. ++ */ ++#define BASE_MEM_FLAGS_MODIFIABLE \ ++ (BASE_MEM_DONT_NEED | BASE_MEM_COHERENT_SYSTEM | \ ++ BASE_MEM_COHERENT_LOCAL) ++ ++/** ++ * enum base_mem_import_type - Memory types supported by @a base_mem_import ++ * ++ * @BASE_MEM_IMPORT_TYPE_INVALID: Invalid type ++ * @BASE_MEM_IMPORT_TYPE_UMP: UMP import. Handle type is ump_secure_id. ++ * @BASE_MEM_IMPORT_TYPE_UMM: UMM import. Handle type is a file descriptor (int) ++ * @BASE_MEM_IMPORT_TYPE_USER_BUFFER: User buffer import. Handle is a ++ * base_mem_import_user_buffer ++ * ++ * Each type defines what the supported handle type is. ++ * ++ * If any new type is added here ARM must be contacted ++ * to allocate a numeric value for it. ++ * Do not just add a new type without synchronizing with ARM ++ * as future releases from ARM might include other new types ++ * which could clash with your custom types. ++ */ ++typedef enum base_mem_import_type { ++ BASE_MEM_IMPORT_TYPE_INVALID = 0, ++ BASE_MEM_IMPORT_TYPE_UMP = 1, ++ BASE_MEM_IMPORT_TYPE_UMM = 2, ++ BASE_MEM_IMPORT_TYPE_USER_BUFFER = 3 ++} base_mem_import_type; ++ ++/** ++ * struct base_mem_import_user_buffer - Handle of an imported user buffer ++ * ++ * @ptr: kbase_pointer to imported user buffer ++ * @length: length of imported user buffer in bytes ++ * ++ * This structure is used to represent a handle of an imported user buffer. ++ */ ++ ++struct base_mem_import_user_buffer { ++ kbase_pointer ptr; ++ u64 length; ++}; ++ ++/** ++ * @brief Invalid memory handle. ++ * ++ * Return value from functions returning @ref base_mem_handle on error. ++ * ++ * @warning @ref base_mem_handle_new_invalid must be used instead of this macro ++ * in C++ code or other situations where compound literals cannot be used. ++ */ ++#define BASE_MEM_INVALID_HANDLE ((base_mem_handle) { {BASEP_MEM_INVALID_HANDLE} }) ++ ++/** ++ * @brief Special write-alloc memory handle. ++ * ++ * A special handle is used to represent a region where a special page is mapped ++ * with a write-alloc cache setup, typically used when the write result of the ++ * GPU isn't needed, but the GPU must write anyway. ++ * ++ * @warning @ref base_mem_handle_new_write_alloc must be used instead of this macro ++ * in C++ code or other situations where compound literals cannot be used. ++ */ ++#define BASE_MEM_WRITE_ALLOC_PAGES_HANDLE ((base_mem_handle) { {BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE} }) ++ ++#define BASEP_MEM_INVALID_HANDLE (0ull << 12) ++#define BASE_MEM_MMU_DUMP_HANDLE (1ull << 12) ++#define BASE_MEM_TRACE_BUFFER_HANDLE (2ull << 12) ++#define BASE_MEM_MAP_TRACKING_HANDLE (3ull << 12) ++#define BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE (4ull << 12) ++/* reserved handles ..-64< for future special handles */ ++#define BASE_MEM_COOKIE_BASE (64ul << 12) ++#define BASE_MEM_FIRST_FREE_ADDRESS ((BITS_PER_LONG << 12) + \ ++ BASE_MEM_COOKIE_BASE) ++ ++/* Mask to detect 4GB boundary alignment */ ++#define BASE_MEM_MASK_4GB 0xfffff000UL ++ ++ ++/* Bit mask of cookies used for for memory allocation setup */ ++#define KBASE_COOKIE_MASK ~1UL /* bit 0 is reserved */ ++ ++ ++/** ++ * @brief Result codes of changing the size of the backing store allocated to a tmem region ++ */ ++typedef enum base_backing_threshold_status { ++ BASE_BACKING_THRESHOLD_OK = 0, /**< Resize successful */ ++ BASE_BACKING_THRESHOLD_ERROR_OOM = -2, /**< Increase failed due to an out-of-memory condition */ ++ BASE_BACKING_THRESHOLD_ERROR_INVALID_ARGUMENTS = -4 /**< Invalid arguments (not tmem, illegal size request, etc.) */ ++} base_backing_threshold_status; ++ ++/** ++ * @addtogroup base_user_api_memory_defered User-side Base Defered Memory Coherency APIs ++ * @{ ++ */ ++ ++/** ++ * @brief a basic memory operation (sync-set). ++ * ++ * The content of this structure is private, and should only be used ++ * by the accessors. ++ */ ++typedef struct base_syncset { ++ struct basep_syncset basep_sset; ++} base_syncset; ++ ++/** @} end group base_user_api_memory_defered */ ++ ++/** ++ * Handle to represent imported memory object. ++ * Simple opague handle to imported memory, can't be used ++ * with anything but base_external_resource_init to bind to an atom. ++ */ ++typedef struct base_import_handle { ++ struct { ++ u64 handle; ++ } basep; ++} base_import_handle; ++ ++/** @} end group base_user_api_memory */ ++ ++/** ++ * @addtogroup base_user_api_job_dispatch User-side Base Job Dispatcher APIs ++ * @{ ++ */ ++ ++typedef int platform_fence_type; ++#define INVALID_PLATFORM_FENCE ((platform_fence_type)-1) ++ ++/** ++ * Base stream handle. ++ * ++ * References an underlying base stream object. ++ */ ++typedef struct base_stream { ++ struct { ++ int fd; ++ } basep; ++} base_stream; ++ ++/** ++ * Base fence handle. ++ * ++ * References an underlying base fence object. ++ */ ++typedef struct base_fence { ++ struct { ++ int fd; ++ int stream_fd; ++ } basep; ++} base_fence; ++ ++/** ++ * @brief Per-job data ++ * ++ * This structure is used to store per-job data, and is completely unused ++ * by the Base driver. It can be used to store things such as callback ++ * function pointer, data to handle job completion. It is guaranteed to be ++ * untouched by the Base driver. ++ */ ++typedef struct base_jd_udata { ++ u64 blob[2]; /**< per-job data array */ ++} base_jd_udata; ++ ++/** ++ * @brief Memory aliasing info ++ * ++ * Describes a memory handle to be aliased. ++ * A subset of the handle can be chosen for aliasing, given an offset and a ++ * length. ++ * A special handle BASE_MEM_WRITE_ALLOC_PAGES_HANDLE is used to represent a ++ * region where a special page is mapped with a write-alloc cache setup, ++ * typically used when the write result of the GPU isn't needed, but the GPU ++ * must write anyway. ++ * ++ * Offset and length are specified in pages. ++ * Offset must be within the size of the handle. ++ * Offset+length must not overrun the size of the handle. ++ * ++ * @handle Handle to alias, can be BASE_MEM_WRITE_ALLOC_PAGES_HANDLE ++ * @offset Offset within the handle to start aliasing from, in pages. ++ * Not used with BASE_MEM_WRITE_ALLOC_PAGES_HANDLE. ++ * @length Length to alias, in pages. For BASE_MEM_WRITE_ALLOC_PAGES_HANDLE ++ * specifies the number of times the special page is needed. ++ */ ++struct base_mem_aliasing_info { ++ base_mem_handle handle; ++ u64 offset; ++ u64 length; ++}; ++ ++/** ++ * struct base_jit_alloc_info - Structure which describes a JIT allocation ++ * request. ++ * @gpu_alloc_addr: The GPU virtual address to write the JIT ++ * allocated GPU virtual address to. ++ * @va_pages: The minimum number of virtual pages required. ++ * @commit_pages: The minimum number of physical pages which ++ * should back the allocation. ++ * @extent: Granularity of physical pages to grow the ++ * allocation by during a fault. ++ * @id: Unique ID provided by the caller, this is used ++ * to pair allocation and free requests. ++ * Zero is not a valid value. ++ */ ++struct base_jit_alloc_info { ++ u64 gpu_alloc_addr; ++ u64 va_pages; ++ u64 commit_pages; ++ u64 extent; ++ u8 id; ++}; ++ ++/** ++ * @brief Job dependency type. ++ * ++ * A flags field will be inserted into the atom structure to specify whether a dependency is a data or ++ * ordering dependency (by putting it before/after 'core_req' in the structure it should be possible to add without ++ * changing the structure size). ++ * When the flag is set for a particular dependency to signal that it is an ordering only dependency then ++ * errors will not be propagated. ++ */ ++typedef u8 base_jd_dep_type; ++ ++ ++#define BASE_JD_DEP_TYPE_INVALID (0) /**< Invalid dependency */ ++#define BASE_JD_DEP_TYPE_DATA (1U << 0) /**< Data dependency */ ++#define BASE_JD_DEP_TYPE_ORDER (1U << 1) /**< Order dependency */ ++ ++/** ++ * @brief Job chain hardware requirements. ++ * ++ * A job chain must specify what GPU features it needs to allow the ++ * driver to schedule the job correctly. By not specifying the ++ * correct settings can/will cause an early job termination. Multiple ++ * values can be ORed together to specify multiple requirements. ++ * Special case is ::BASE_JD_REQ_DEP, which is used to express complex ++ * dependencies, and that doesn't execute anything on the hardware. ++ */ ++typedef u32 base_jd_core_req; ++ ++/* Requirements that come from the HW */ ++ ++/** ++ * No requirement, dependency only ++ */ ++#define BASE_JD_REQ_DEP ((base_jd_core_req)0) ++ ++/** ++ * Requires fragment shaders ++ */ ++#define BASE_JD_REQ_FS ((base_jd_core_req)1 << 0) ++ ++/** ++ * Requires compute shaders ++ * This covers any of the following Midgard Job types: ++ * - Vertex Shader Job ++ * - Geometry Shader Job ++ * - An actual Compute Shader Job ++ * ++ * Compare this with @ref BASE_JD_REQ_ONLY_COMPUTE, which specifies that the ++ * job is specifically just the "Compute Shader" job type, and not the "Vertex ++ * Shader" nor the "Geometry Shader" job type. ++ */ ++#define BASE_JD_REQ_CS ((base_jd_core_req)1 << 1) ++#define BASE_JD_REQ_T ((base_jd_core_req)1 << 2) /**< Requires tiling */ ++#define BASE_JD_REQ_CF ((base_jd_core_req)1 << 3) /**< Requires cache flushes */ ++#define BASE_JD_REQ_V ((base_jd_core_req)1 << 4) /**< Requires value writeback */ ++ ++/* SW-only requirements - the HW does not expose these as part of the job slot capabilities */ ++ ++/* Requires fragment job with AFBC encoding */ ++#define BASE_JD_REQ_FS_AFBC ((base_jd_core_req)1 << 13) ++ ++/** ++ * SW-only requirement: coalesce completion events. ++ * If this bit is set then completion of this atom will not cause an event to ++ * be sent to userspace, whether successful or not; completion events will be ++ * deferred until an atom completes which does not have this bit set. ++ * ++ * This bit may not be used in combination with BASE_JD_REQ_EXTERNAL_RESOURCES. ++ */ ++#define BASE_JD_REQ_EVENT_COALESCE ((base_jd_core_req)1 << 5) ++ ++/** ++ * SW Only requirement: the job chain requires a coherent core group. We don't ++ * mind which coherent core group is used. ++ */ ++#define BASE_JD_REQ_COHERENT_GROUP ((base_jd_core_req)1 << 6) ++ ++/** ++ * SW Only requirement: The performance counters should be enabled only when ++ * they are needed, to reduce power consumption. ++ */ ++ ++#define BASE_JD_REQ_PERMON ((base_jd_core_req)1 << 7) ++ ++/** ++ * SW Only requirement: External resources are referenced by this atom. ++ * When external resources are referenced no syncsets can be bundled with the atom ++ * but should instead be part of a NULL jobs inserted into the dependency tree. ++ * The first pre_dep object must be configured for the external resouces to use, ++ * the second pre_dep object can be used to create other dependencies. ++ * ++ * This bit may not be used in combination with BASE_JD_REQ_EVENT_COALESCE. ++ */ ++#define BASE_JD_REQ_EXTERNAL_RESOURCES ((base_jd_core_req)1 << 8) ++ ++/** ++ * SW Only requirement: Software defined job. Jobs with this bit set will not be submitted ++ * to the hardware but will cause some action to happen within the driver ++ */ ++#define BASE_JD_REQ_SOFT_JOB ((base_jd_core_req)1 << 9) ++ ++#define BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME (BASE_JD_REQ_SOFT_JOB | 0x1) ++#define BASE_JD_REQ_SOFT_FENCE_TRIGGER (BASE_JD_REQ_SOFT_JOB | 0x2) ++#define BASE_JD_REQ_SOFT_FENCE_WAIT (BASE_JD_REQ_SOFT_JOB | 0x3) ++ ++/** ++ * SW Only requirement : Replay job. ++ * ++ * If the preceding job fails, the replay job will cause the jobs specified in ++ * the list of base_jd_replay_payload pointed to by the jc pointer to be ++ * replayed. ++ * ++ * A replay job will only cause jobs to be replayed up to BASEP_JD_REPLAY_LIMIT ++ * times. If a job fails more than BASEP_JD_REPLAY_LIMIT times then the replay ++ * job is failed, as well as any following dependencies. ++ * ++ * The replayed jobs will require a number of atom IDs. If there are not enough ++ * free atom IDs then the replay job will fail. ++ * ++ * If the preceding job does not fail, then the replay job is returned as ++ * completed. ++ * ++ * The replayed jobs will never be returned to userspace. The preceding failed ++ * job will be returned to userspace as failed; the status of this job should ++ * be ignored. Completion should be determined by the status of the replay soft ++ * job. ++ * ++ * In order for the jobs to be replayed, the job headers will have to be ++ * modified. The Status field will be reset to NOT_STARTED. If the Job Type ++ * field indicates a Vertex Shader Job then it will be changed to Null Job. ++ * ++ * The replayed jobs have the following assumptions : ++ * ++ * - No external resources. Any required external resources will be held by the ++ * replay atom. ++ * - Pre-dependencies are created based on job order. ++ * - Atom numbers are automatically assigned. ++ * - device_nr is set to 0. This is not relevant as ++ * BASE_JD_REQ_SPECIFIC_COHERENT_GROUP should not be set. ++ * - Priority is inherited from the replay job. ++ */ ++#define BASE_JD_REQ_SOFT_REPLAY (BASE_JD_REQ_SOFT_JOB | 0x4) ++/** ++ * SW only requirement: event wait/trigger job. ++ * ++ * - BASE_JD_REQ_SOFT_EVENT_WAIT: this job will block until the event is set. ++ * - BASE_JD_REQ_SOFT_EVENT_SET: this job sets the event, thus unblocks the ++ * other waiting jobs. It completes immediately. ++ * - BASE_JD_REQ_SOFT_EVENT_RESET: this job resets the event, making it ++ * possible for other jobs to wait upon. It completes immediately. ++ */ ++#define BASE_JD_REQ_SOFT_EVENT_WAIT (BASE_JD_REQ_SOFT_JOB | 0x5) ++#define BASE_JD_REQ_SOFT_EVENT_SET (BASE_JD_REQ_SOFT_JOB | 0x6) ++#define BASE_JD_REQ_SOFT_EVENT_RESET (BASE_JD_REQ_SOFT_JOB | 0x7) ++ ++#define BASE_JD_REQ_SOFT_DEBUG_COPY (BASE_JD_REQ_SOFT_JOB | 0x8) ++ ++/** ++ * SW only requirement: Just In Time allocation ++ * ++ * This job requests a JIT allocation based on the request in the ++ * @base_jit_alloc_info structure which is passed via the jc element of ++ * the atom. ++ * ++ * It should be noted that the id entry in @base_jit_alloc_info must not ++ * be reused until it has been released via @BASE_JD_REQ_SOFT_JIT_FREE. ++ * ++ * Should this soft job fail it is expected that a @BASE_JD_REQ_SOFT_JIT_FREE ++ * soft job to free the JIT allocation is still made. ++ * ++ * The job will complete immediately. ++ */ ++#define BASE_JD_REQ_SOFT_JIT_ALLOC (BASE_JD_REQ_SOFT_JOB | 0x9) ++/** ++ * SW only requirement: Just In Time free ++ * ++ * This job requests a JIT allocation created by @BASE_JD_REQ_SOFT_JIT_ALLOC ++ * to be freed. The ID of the JIT allocation is passed via the jc element of ++ * the atom. ++ * ++ * The job will complete immediately. ++ */ ++#define BASE_JD_REQ_SOFT_JIT_FREE (BASE_JD_REQ_SOFT_JOB | 0xa) ++ ++/** ++ * SW only requirement: Map external resource ++ * ++ * This job requests external resource(s) are mapped once the dependencies ++ * of the job have been satisfied. The list of external resources are ++ * passed via the jc element of the atom which is a pointer to a ++ * @base_external_resource_list. ++ */ ++#define BASE_JD_REQ_SOFT_EXT_RES_MAP (BASE_JD_REQ_SOFT_JOB | 0xb) ++/** ++ * SW only requirement: Unmap external resource ++ * ++ * This job requests external resource(s) are unmapped once the dependencies ++ * of the job has been satisfied. The list of external resources are ++ * passed via the jc element of the atom which is a pointer to a ++ * @base_external_resource_list. ++ */ ++#define BASE_JD_REQ_SOFT_EXT_RES_UNMAP (BASE_JD_REQ_SOFT_JOB | 0xc) ++ ++/** ++ * HW Requirement: Requires Compute shaders (but not Vertex or Geometry Shaders) ++ * ++ * This indicates that the Job Chain contains Midgard Jobs of the 'Compute Shaders' type. ++ * ++ * In contrast to @ref BASE_JD_REQ_CS, this does \b not indicate that the Job ++ * Chain contains 'Geometry Shader' or 'Vertex Shader' jobs. ++ */ ++#define BASE_JD_REQ_ONLY_COMPUTE ((base_jd_core_req)1 << 10) ++ ++/** ++ * HW Requirement: Use the base_jd_atom::device_nr field to specify a ++ * particular core group ++ * ++ * If both @ref BASE_JD_REQ_COHERENT_GROUP and this flag are set, this flag takes priority ++ * ++ * This is only guaranteed to work for @ref BASE_JD_REQ_ONLY_COMPUTE atoms. ++ * ++ * If the core availability policy is keeping the required core group turned off, then ++ * the job will fail with a @ref BASE_JD_EVENT_PM_EVENT error code. ++ */ ++#define BASE_JD_REQ_SPECIFIC_COHERENT_GROUP ((base_jd_core_req)1 << 11) ++ ++/** ++ * SW Flag: If this bit is set then the successful completion of this atom ++ * will not cause an event to be sent to userspace ++ */ ++#define BASE_JD_REQ_EVENT_ONLY_ON_FAILURE ((base_jd_core_req)1 << 12) ++ ++/** ++ * SW Flag: If this bit is set then completion of this atom will not cause an ++ * event to be sent to userspace, whether successful or not. ++ */ ++#define BASEP_JD_REQ_EVENT_NEVER ((base_jd_core_req)1 << 14) ++ ++/** ++ * SW Flag: Skip GPU cache clean and invalidation before starting a GPU job. ++ * ++ * If this bit is set then the GPU's cache will not be cleaned and invalidated ++ * until a GPU job starts which does not have this bit set or a job completes ++ * which does not have the @ref BASE_JD_REQ_SKIP_CACHE_END bit set. Do not use if ++ * the CPU may have written to memory addressed by the job since the last job ++ * without this bit set was submitted. ++ */ ++#define BASE_JD_REQ_SKIP_CACHE_START ((base_jd_core_req)1 << 15) ++ ++/** ++ * SW Flag: Skip GPU cache clean and invalidation after a GPU job completes. ++ * ++ * If this bit is set then the GPU's cache will not be cleaned and invalidated ++ * until a GPU job completes which does not have this bit set or a job starts ++ * which does not have the @ref BASE_JD_REQ_SKIP_CACHE_START bti set. Do not use if ++ * the CPU may read from or partially overwrite memory addressed by the job ++ * before the next job without this bit set completes. ++ */ ++#define BASE_JD_REQ_SKIP_CACHE_END ((base_jd_core_req)1 << 16) ++ ++/** ++ * These requirement bits are currently unused in base_jd_core_req ++ */ ++#define BASEP_JD_REQ_RESERVED \ ++ (~(BASE_JD_REQ_ATOM_TYPE | BASE_JD_REQ_EXTERNAL_RESOURCES | \ ++ BASE_JD_REQ_EVENT_ONLY_ON_FAILURE | BASEP_JD_REQ_EVENT_NEVER | \ ++ BASE_JD_REQ_EVENT_COALESCE | \ ++ BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_SPECIFIC_COHERENT_GROUP | \ ++ BASE_JD_REQ_FS_AFBC | BASE_JD_REQ_PERMON | \ ++ BASE_JD_REQ_SKIP_CACHE_START | BASE_JD_REQ_SKIP_CACHE_END)) ++ ++/** ++ * Mask of all bits in base_jd_core_req that control the type of the atom. ++ * ++ * This allows dependency only atoms to have flags set ++ */ ++#define BASE_JD_REQ_ATOM_TYPE \ ++ (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T | BASE_JD_REQ_CF | \ ++ BASE_JD_REQ_V | BASE_JD_REQ_SOFT_JOB | BASE_JD_REQ_ONLY_COMPUTE) ++ ++/** ++ * Mask of all bits in base_jd_core_req that control the type of a soft job. ++ */ ++#define BASE_JD_REQ_SOFT_JOB_TYPE (BASE_JD_REQ_SOFT_JOB | 0x1f) ++ ++/* ++ * Returns non-zero value if core requirements passed define a soft job or ++ * a dependency only job. ++ */ ++#define BASE_JD_REQ_SOFT_JOB_OR_DEP(core_req) \ ++ ((core_req & BASE_JD_REQ_SOFT_JOB) || \ ++ (core_req & BASE_JD_REQ_ATOM_TYPE) == BASE_JD_REQ_DEP) ++ ++/** ++ * @brief States to model state machine processed by kbasep_js_job_check_ref_cores(), which ++ * handles retaining cores for power management and affinity management. ++ * ++ * The state @ref KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY prevents an attack ++ * where lots of atoms could be submitted before powerup, and each has an ++ * affinity chosen that causes other atoms to have an affinity ++ * violation. Whilst the affinity was not causing violations at the time it ++ * was chosen, it could cause violations thereafter. For example, 1000 jobs ++ * could have had their affinity chosen during the powerup time, so any of ++ * those 1000 jobs could cause an affinity violation later on. ++ * ++ * The attack would otherwise occur because other atoms/contexts have to wait for: ++ * -# the currently running atoms (which are causing the violation) to ++ * finish ++ * -# and, the atoms that had their affinity chosen during powerup to ++ * finish. These are run preferentially because they don't cause a ++ * violation, but instead continue to cause the violation in others. ++ * -# or, the attacker is scheduled out (which might not happen for just 2 ++ * contexts) ++ * ++ * By re-choosing the affinity (which is designed to avoid violations at the ++ * time it's chosen), we break condition (2) of the wait, which minimizes the ++ * problem to just waiting for current jobs to finish (which can be bounded if ++ * the Job Scheduling Policy has a timer). ++ */ ++enum kbase_atom_coreref_state { ++ /** Starting state: No affinity chosen, and cores must be requested. kbase_jd_atom::affinity==0 */ ++ KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED, ++ /** Cores requested, but waiting for them to be powered. Requested cores given by kbase_jd_atom::affinity */ ++ KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES, ++ /** Cores given by kbase_jd_atom::affinity are powered, but affinity might be out-of-date, so must recheck */ ++ KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY, ++ /** Cores given by kbase_jd_atom::affinity are powered, and affinity is up-to-date, but must check for violations */ ++ KBASE_ATOM_COREREF_STATE_CHECK_AFFINITY_VIOLATIONS, ++ /** Cores are powered, kbase_jd_atom::affinity up-to-date, no affinity violations: atom can be submitted to HW */ ++ KBASE_ATOM_COREREF_STATE_READY ++}; ++ ++/* ++ * Base Atom priority ++ * ++ * Only certain priority levels are actually implemented, as specified by the ++ * BASE_JD_PRIO_<...> definitions below. It is undefined to use a priority ++ * level that is not one of those defined below. ++ * ++ * Priority levels only affect scheduling between atoms of the same type within ++ * a base context, and only after the atoms have had dependencies resolved. ++ * Fragment atoms does not affect non-frament atoms with lower priorities, and ++ * the other way around. For example, a low priority atom that has had its ++ * dependencies resolved might run before a higher priority atom that has not ++ * had its dependencies resolved. ++ * ++ * The scheduling between base contexts/processes and between atoms from ++ * different base contexts/processes is unaffected by atom priority. ++ * ++ * The atoms are scheduled as follows with respect to their priorities: ++ * - Let atoms 'X' and 'Y' be for the same job slot who have dependencies ++ * resolved, and atom 'X' has a higher priority than atom 'Y' ++ * - If atom 'Y' is currently running on the HW, then it is interrupted to ++ * allow atom 'X' to run soon after ++ * - If instead neither atom 'Y' nor atom 'X' are running, then when choosing ++ * the next atom to run, atom 'X' will always be chosen instead of atom 'Y' ++ * - Any two atoms that have the same priority could run in any order with ++ * respect to each other. That is, there is no ordering constraint between ++ * atoms of the same priority. ++ */ ++typedef u8 base_jd_prio; ++ ++/* Medium atom priority. This is a priority higher than BASE_JD_PRIO_LOW */ ++#define BASE_JD_PRIO_MEDIUM ((base_jd_prio)0) ++/* High atom priority. This is a priority higher than BASE_JD_PRIO_MEDIUM and ++ * BASE_JD_PRIO_LOW */ ++#define BASE_JD_PRIO_HIGH ((base_jd_prio)1) ++/* Low atom priority. */ ++#define BASE_JD_PRIO_LOW ((base_jd_prio)2) ++ ++/* Count of the number of priority levels. This itself is not a valid ++ * base_jd_prio setting */ ++#define BASE_JD_NR_PRIO_LEVELS 3 ++ ++enum kbase_jd_atom_state { ++ /** Atom is not used */ ++ KBASE_JD_ATOM_STATE_UNUSED, ++ /** Atom is queued in JD */ ++ KBASE_JD_ATOM_STATE_QUEUED, ++ /** Atom has been given to JS (is runnable/running) */ ++ KBASE_JD_ATOM_STATE_IN_JS, ++ /** Atom has been completed, but not yet handed back to job dispatcher ++ * for dependency resolution */ ++ KBASE_JD_ATOM_STATE_HW_COMPLETED, ++ /** Atom has been completed, but not yet handed back to userspace */ ++ KBASE_JD_ATOM_STATE_COMPLETED ++}; ++ ++typedef u16 base_atom_id; /**< Type big enough to store an atom number in */ ++ ++struct base_dependency { ++ base_atom_id atom_id; /**< An atom number */ ++ base_jd_dep_type dependency_type; /**< Dependency type */ ++}; ++ ++/* This structure has changed since UK 10.2 for which base_jd_core_req was a u16 value. ++ * In order to keep the size of the structure same, padding field has been adjusted ++ * accordingly and core_req field of a u32 type (to which UK 10.3 base_jd_core_req defines) ++ * is added at the end of the structure. Place in the structure previously occupied by u16 core_req ++ * is kept but renamed to compat_core_req and as such it can be used in ioctl call for job submission ++ * as long as UK 10.2 legacy is supported. Once when this support ends, this field can be left ++ * for possible future use. */ ++typedef struct base_jd_atom_v2 { ++ u64 jc; /**< job-chain GPU address */ ++ struct base_jd_udata udata; /**< user data */ ++ kbase_pointer extres_list; /**< list of external resources */ ++ u16 nr_extres; /**< nr of external resources */ ++ u16 compat_core_req; /**< core requirements which correspond to the legacy support for UK 10.2 */ ++ struct base_dependency pre_dep[2]; /**< pre-dependencies, one need to use SETTER function to assign this field, ++ this is done in order to reduce possibility of improper assigment of a dependency field */ ++ base_atom_id atom_number; /**< unique number to identify the atom */ ++ base_jd_prio prio; /**< Atom priority. Refer to @ref base_jd_prio for more details */ ++ u8 device_nr; /**< coregroup when BASE_JD_REQ_SPECIFIC_COHERENT_GROUP specified */ ++ u8 padding[1]; ++ base_jd_core_req core_req; /**< core requirements */ ++} base_jd_atom_v2; ++ ++#ifdef BASE_LEGACY_UK6_SUPPORT ++struct base_jd_atom_v2_uk6 { ++ u64 jc; /**< job-chain GPU address */ ++ struct base_jd_udata udata; /**< user data */ ++ kbase_pointer extres_list; /**< list of external resources */ ++ u16 nr_extres; /**< nr of external resources */ ++ u16 core_req; /**< core requirements */ ++ base_atom_id pre_dep[2]; /**< pre-dependencies */ ++ base_atom_id atom_number; /**< unique number to identify the atom */ ++ base_jd_prio prio; /**< priority - smaller is higher priority */ ++ u8 device_nr; /**< coregroup when BASE_JD_REQ_SPECIFIC_COHERENT_GROUP specified */ ++ u8 padding[7]; ++}; ++#endif /* BASE_LEGACY_UK6_SUPPORT */ ++ ++typedef enum base_external_resource_access { ++ BASE_EXT_RES_ACCESS_SHARED, ++ BASE_EXT_RES_ACCESS_EXCLUSIVE ++} base_external_resource_access; ++ ++typedef struct base_external_resource { ++ u64 ext_resource; ++} base_external_resource; ++ ++ ++/** ++ * The maximum number of external resources which can be mapped/unmapped ++ * in a single request. ++ */ ++#define BASE_EXT_RES_COUNT_MAX 10 ++ ++/** ++ * struct base_external_resource_list - Structure which describes a list of ++ * external resources. ++ * @count: The number of resources. ++ * @ext_res: Array of external resources which is ++ * sized at allocation time. ++ */ ++struct base_external_resource_list { ++ u64 count; ++ struct base_external_resource ext_res[1]; ++}; ++ ++struct base_jd_debug_copy_buffer { ++ u64 address; ++ u64 size; ++ struct base_external_resource extres; ++}; ++ ++/** ++ * @brief Setter for a dependency structure ++ * ++ * @param[in] dep The kbase jd atom dependency to be initialized. ++ * @param id The atom_id to be assigned. ++ * @param dep_type The dep_type to be assigned. ++ * ++ */ ++static inline void base_jd_atom_dep_set(struct base_dependency *dep, ++ base_atom_id id, base_jd_dep_type dep_type) ++{ ++ LOCAL_ASSERT(dep != NULL); ++ ++ /* ++ * make sure we don't set not allowed combinations ++ * of atom_id/dependency_type. ++ */ ++ LOCAL_ASSERT((id == 0 && dep_type == BASE_JD_DEP_TYPE_INVALID) || ++ (id > 0 && dep_type != BASE_JD_DEP_TYPE_INVALID)); ++ ++ dep->atom_id = id; ++ dep->dependency_type = dep_type; ++} ++ ++/** ++ * @brief Make a copy of a dependency structure ++ * ++ * @param[in,out] dep The kbase jd atom dependency to be written. ++ * @param[in] from The dependency to make a copy from. ++ * ++ */ ++static inline void base_jd_atom_dep_copy(struct base_dependency *dep, ++ const struct base_dependency *from) ++{ ++ LOCAL_ASSERT(dep != NULL); ++ ++ base_jd_atom_dep_set(dep, from->atom_id, from->dependency_type); ++} ++ ++/** ++ * @brief Soft-atom fence trigger setup. ++ * ++ * Sets up an atom to be a SW-only atom signaling a fence ++ * when it reaches the run state. ++ * ++ * Using the existing base dependency system the fence can ++ * be set to trigger when a GPU job has finished. ++ * ++ * The base fence object must not be terminated until the atom ++ * has been submitted to @a base_jd_submit and @a base_jd_submit has returned. ++ * ++ * @a fence must be a valid fence set up with @a base_fence_init. ++ * Calling this function with a uninitialized fence results in undefined behavior. ++ * ++ * @param[out] atom A pre-allocated atom to configure as a fence trigger SW atom ++ * @param[in] fence The base fence object to trigger. ++ */ ++static inline void base_jd_fence_trigger_setup_v2(struct base_jd_atom_v2 *atom, struct base_fence *fence) ++{ ++ LOCAL_ASSERT(atom); ++ LOCAL_ASSERT(fence); ++ LOCAL_ASSERT(fence->basep.fd == INVALID_PLATFORM_FENCE); ++ LOCAL_ASSERT(fence->basep.stream_fd >= 0); ++ atom->jc = (uintptr_t) fence; ++ atom->core_req = BASE_JD_REQ_SOFT_FENCE_TRIGGER; ++} ++ ++/** ++ * @brief Soft-atom fence wait setup. ++ * ++ * Sets up an atom to be a SW-only atom waiting on a fence. ++ * When the fence becomes triggered the atom becomes runnable ++ * and completes immediately. ++ * ++ * Using the existing base dependency system the fence can ++ * be set to block a GPU job until it has been triggered. ++ * ++ * The base fence object must not be terminated until the atom ++ * has been submitted to @a base_jd_submit and @a base_jd_submit has returned. ++ * ++ * @a fence must be a valid fence set up with @a base_fence_init or @a base_fence_import. ++ * Calling this function with a uninitialized fence results in undefined behavior. ++ * ++ * @param[out] atom A pre-allocated atom to configure as a fence wait SW atom ++ * @param[in] fence The base fence object to wait on ++ */ ++static inline void base_jd_fence_wait_setup_v2(struct base_jd_atom_v2 *atom, struct base_fence *fence) ++{ ++ LOCAL_ASSERT(atom); ++ LOCAL_ASSERT(fence); ++ LOCAL_ASSERT(fence->basep.fd >= 0); ++ atom->jc = (uintptr_t) fence; ++ atom->core_req = BASE_JD_REQ_SOFT_FENCE_WAIT; ++} ++ ++/** ++ * @brief External resource info initialization. ++ * ++ * Sets up an external resource object to reference ++ * a memory allocation and the type of access requested. ++ * ++ * @param[in] res The resource object to initialize ++ * @param handle The handle to the imported memory object, must be ++ * obtained by calling @ref base_mem_as_import_handle(). ++ * @param access The type of access requested ++ */ ++static inline void base_external_resource_init(struct base_external_resource *res, struct base_import_handle handle, base_external_resource_access access) ++{ ++ u64 address; ++ ++ address = handle.basep.handle; ++ ++ LOCAL_ASSERT(res != NULL); ++ LOCAL_ASSERT(0 == (address & LOCAL_PAGE_LSB)); ++ LOCAL_ASSERT(access == BASE_EXT_RES_ACCESS_SHARED || access == BASE_EXT_RES_ACCESS_EXCLUSIVE); ++ ++ res->ext_resource = address | (access & LOCAL_PAGE_LSB); ++} ++ ++/** ++ * @brief Job chain event code bits ++ * Defines the bits used to create ::base_jd_event_code ++ */ ++enum { ++ BASE_JD_SW_EVENT_KERNEL = (1u << 15), /**< Kernel side event */ ++ BASE_JD_SW_EVENT = (1u << 14), /**< SW defined event */ ++ BASE_JD_SW_EVENT_SUCCESS = (1u << 13), /**< Event idicates success (SW events only) */ ++ BASE_JD_SW_EVENT_JOB = (0u << 11), /**< Job related event */ ++ BASE_JD_SW_EVENT_BAG = (1u << 11), /**< Bag related event */ ++ BASE_JD_SW_EVENT_INFO = (2u << 11), /**< Misc/info event */ ++ BASE_JD_SW_EVENT_RESERVED = (3u << 11), /**< Reserved event type */ ++ BASE_JD_SW_EVENT_TYPE_MASK = (3u << 11) /**< Mask to extract the type from an event code */ ++}; ++ ++/** ++ * @brief Job chain event codes ++ * ++ * HW and low-level SW events are represented by event codes. ++ * The status of jobs which succeeded are also represented by ++ * an event code (see ::BASE_JD_EVENT_DONE). ++ * Events are usually reported as part of a ::base_jd_event. ++ * ++ * The event codes are encoded in the following way: ++ * @li 10:0 - subtype ++ * @li 12:11 - type ++ * @li 13 - SW success (only valid if the SW bit is set) ++ * @li 14 - SW event (HW event if not set) ++ * @li 15 - Kernel event (should never be seen in userspace) ++ * ++ * Events are split up into ranges as follows: ++ * - BASE_JD_EVENT_RANGE_\_START ++ * - BASE_JD_EVENT_RANGE_\_END ++ * ++ * \a code is in \'s range when: ++ * - BASE_JD_EVENT_RANGE_\_START <= code < BASE_JD_EVENT_RANGE_\_END ++ * ++ * Ranges can be asserted for adjacency by testing that the END of the previous ++ * is equal to the START of the next. This is useful for optimizing some tests ++ * for range. ++ * ++ * A limitation is that the last member of this enum must explicitly be handled ++ * (with an assert-unreachable statement) in switch statements that use ++ * variables of this type. Otherwise, the compiler warns that we have not ++ * handled that enum value. ++ */ ++typedef enum base_jd_event_code { ++ /* HW defined exceptions */ ++ ++ /** Start of HW Non-fault status codes ++ * ++ * @note Obscurely, BASE_JD_EVENT_TERMINATED indicates a real fault, ++ * because the job was hard-stopped ++ */ ++ BASE_JD_EVENT_RANGE_HW_NONFAULT_START = 0, ++ ++ /* non-fatal exceptions */ ++ BASE_JD_EVENT_NOT_STARTED = 0x00, /**< Can't be seen by userspace, treated as 'previous job done' */ ++ BASE_JD_EVENT_DONE = 0x01, ++ BASE_JD_EVENT_STOPPED = 0x03, /**< Can't be seen by userspace, becomes TERMINATED, DONE or JOB_CANCELLED */ ++ BASE_JD_EVENT_TERMINATED = 0x04, /**< This is actually a fault status code - the job was hard stopped */ ++ BASE_JD_EVENT_ACTIVE = 0x08, /**< Can't be seen by userspace, jobs only returned on complete/fail/cancel */ ++ ++ /** End of HW Non-fault status codes ++ * ++ * @note Obscurely, BASE_JD_EVENT_TERMINATED indicates a real fault, ++ * because the job was hard-stopped ++ */ ++ BASE_JD_EVENT_RANGE_HW_NONFAULT_END = 0x40, ++ ++ /** Start of HW fault and SW Error status codes */ ++ BASE_JD_EVENT_RANGE_HW_FAULT_OR_SW_ERROR_START = 0x40, ++ ++ /* job exceptions */ ++ BASE_JD_EVENT_JOB_CONFIG_FAULT = 0x40, ++ BASE_JD_EVENT_JOB_POWER_FAULT = 0x41, ++ BASE_JD_EVENT_JOB_READ_FAULT = 0x42, ++ BASE_JD_EVENT_JOB_WRITE_FAULT = 0x43, ++ BASE_JD_EVENT_JOB_AFFINITY_FAULT = 0x44, ++ BASE_JD_EVENT_JOB_BUS_FAULT = 0x48, ++ BASE_JD_EVENT_INSTR_INVALID_PC = 0x50, ++ BASE_JD_EVENT_INSTR_INVALID_ENC = 0x51, ++ BASE_JD_EVENT_INSTR_TYPE_MISMATCH = 0x52, ++ BASE_JD_EVENT_INSTR_OPERAND_FAULT = 0x53, ++ BASE_JD_EVENT_INSTR_TLS_FAULT = 0x54, ++ BASE_JD_EVENT_INSTR_BARRIER_FAULT = 0x55, ++ BASE_JD_EVENT_INSTR_ALIGN_FAULT = 0x56, ++ BASE_JD_EVENT_DATA_INVALID_FAULT = 0x58, ++ BASE_JD_EVENT_TILE_RANGE_FAULT = 0x59, ++ BASE_JD_EVENT_STATE_FAULT = 0x5A, ++ BASE_JD_EVENT_OUT_OF_MEMORY = 0x60, ++ BASE_JD_EVENT_UNKNOWN = 0x7F, ++ ++ /* GPU exceptions */ ++ BASE_JD_EVENT_DELAYED_BUS_FAULT = 0x80, ++ BASE_JD_EVENT_SHAREABILITY_FAULT = 0x88, ++ ++ /* MMU exceptions */ ++ BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL1 = 0xC1, ++ BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL2 = 0xC2, ++ BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL3 = 0xC3, ++ BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL4 = 0xC4, ++ BASE_JD_EVENT_PERMISSION_FAULT = 0xC8, ++ BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL1 = 0xD1, ++ BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL2 = 0xD2, ++ BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL3 = 0xD3, ++ BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL4 = 0xD4, ++ BASE_JD_EVENT_ACCESS_FLAG = 0xD8, ++ ++ /* SW defined exceptions */ ++ BASE_JD_EVENT_MEM_GROWTH_FAILED = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x000, ++ BASE_JD_EVENT_TIMED_OUT = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x001, ++ BASE_JD_EVENT_JOB_CANCELLED = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x002, ++ BASE_JD_EVENT_JOB_INVALID = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x003, ++ BASE_JD_EVENT_PM_EVENT = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x004, ++ BASE_JD_EVENT_FORCE_REPLAY = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x005, ++ ++ BASE_JD_EVENT_BAG_INVALID = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_BAG | 0x003, ++ ++ /** End of HW fault and SW Error status codes */ ++ BASE_JD_EVENT_RANGE_HW_FAULT_OR_SW_ERROR_END = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_RESERVED | 0x3FF, ++ ++ /** Start of SW Success status codes */ ++ BASE_JD_EVENT_RANGE_SW_SUCCESS_START = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_SUCCESS | 0x000, ++ ++ BASE_JD_EVENT_PROGRESS_REPORT = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_SUCCESS | BASE_JD_SW_EVENT_JOB | 0x000, ++ BASE_JD_EVENT_BAG_DONE = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_SUCCESS | BASE_JD_SW_EVENT_BAG | 0x000, ++ BASE_JD_EVENT_DRV_TERMINATED = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_SUCCESS | BASE_JD_SW_EVENT_INFO | 0x000, ++ ++ /** End of SW Success status codes */ ++ BASE_JD_EVENT_RANGE_SW_SUCCESS_END = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_SUCCESS | BASE_JD_SW_EVENT_RESERVED | 0x3FF, ++ ++ /** Start of Kernel-only status codes. Such codes are never returned to user-space */ ++ BASE_JD_EVENT_RANGE_KERNEL_ONLY_START = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_KERNEL | 0x000, ++ BASE_JD_EVENT_REMOVED_FROM_NEXT = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_KERNEL | BASE_JD_SW_EVENT_JOB | 0x000, ++ ++ /** End of Kernel-only status codes. */ ++ BASE_JD_EVENT_RANGE_KERNEL_ONLY_END = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_KERNEL | BASE_JD_SW_EVENT_RESERVED | 0x3FF ++} base_jd_event_code; ++ ++/** ++ * @brief Event reporting structure ++ * ++ * This structure is used by the kernel driver to report information ++ * about GPU events. The can either be HW-specific events or low-level ++ * SW events, such as job-chain completion. ++ * ++ * The event code contains an event type field which can be extracted ++ * by ANDing with ::BASE_JD_SW_EVENT_TYPE_MASK. ++ * ++ * Based on the event type base_jd_event::data holds: ++ * @li ::BASE_JD_SW_EVENT_JOB : the offset in the ring-buffer for the completed ++ * job-chain ++ * @li ::BASE_JD_SW_EVENT_BAG : The address of the ::base_jd_bag that has ++ * been completed (ie all contained job-chains have been completed). ++ * @li ::BASE_JD_SW_EVENT_INFO : base_jd_event::data not used ++ */ ++typedef struct base_jd_event_v2 { ++ base_jd_event_code event_code; /**< event code */ ++ base_atom_id atom_number; /**< the atom number that has completed */ ++ struct base_jd_udata udata; /**< user data */ ++} base_jd_event_v2; ++ ++/** ++ * Padding required to ensure that the @ref struct base_dump_cpu_gpu_counters structure fills ++ * a full cache line. ++ */ ++ ++#define BASE_CPU_GPU_CACHE_LINE_PADDING (36) ++ ++ ++/** ++ * @brief Structure for BASE_JD_REQ_SOFT_DUMP_CPU_GPU_COUNTERS jobs. ++ * ++ * This structure is stored into the memory pointed to by the @c jc field of @ref base_jd_atom. ++ * ++ * This structure must be padded to ensure that it will occupy whole cache lines. This is to avoid ++ * cases where access to pages containing the structure is shared between cached and un-cached ++ * memory regions, which would cause memory corruption. Here we set the structure size to be 64 bytes ++ * which is the cache line for ARM A15 processors. ++ */ ++ ++typedef struct base_dump_cpu_gpu_counters { ++ u64 system_time; ++ u64 cycle_counter; ++ u64 sec; ++ u32 usec; ++ u8 padding[BASE_CPU_GPU_CACHE_LINE_PADDING]; ++} base_dump_cpu_gpu_counters; ++ ++ ++ ++/** @} end group base_user_api_job_dispatch */ ++ ++#define GPU_MAX_JOB_SLOTS 16 ++ ++/** ++ * @page page_base_user_api_gpuprops User-side Base GPU Property Query API ++ * ++ * The User-side Base GPU Property Query API encapsulates two ++ * sub-modules: ++ * ++ * - @ref base_user_api_gpuprops_dyn "Dynamic GPU Properties" ++ * - @ref base_plat_config_gpuprops "Base Platform Config GPU Properties" ++ * ++ * There is a related third module outside of Base, which is owned by the MIDG ++ * module: ++ * - @ref gpu_props_static "Midgard Compile-time GPU Properties" ++ * ++ * Base only deals with properties that vary between different Midgard ++ * implementations - the Dynamic GPU properties and the Platform Config ++ * properties. ++ * ++ * For properties that are constant for the Midgard Architecture, refer to the ++ * MIDG module. However, we will discuss their relevance here just to ++ * provide background information. ++ * ++ * @section sec_base_user_api_gpuprops_about About the GPU Properties in Base and MIDG modules ++ * ++ * The compile-time properties (Platform Config, Midgard Compile-time ++ * properties) are exposed as pre-processor macros. ++ * ++ * Complementing the compile-time properties are the Dynamic GPU ++ * Properties, which act as a conduit for the Midgard Configuration ++ * Discovery. ++ * ++ * In general, the dynamic properties are present to verify that the platform ++ * has been configured correctly with the right set of Platform Config ++ * Compile-time Properties. ++ * ++ * As a consistent guide across the entire DDK, the choice for dynamic or ++ * compile-time should consider the following, in order: ++ * -# Can the code be written so that it doesn't need to know the ++ * implementation limits at all? ++ * -# If you need the limits, get the information from the Dynamic Property ++ * lookup. This should be done once as you fetch the context, and then cached ++ * as part of the context data structure, so it's cheap to access. ++ * -# If there's a clear and arguable inefficiency in using Dynamic Properties, ++ * then use a Compile-Time Property (Platform Config, or Midgard Compile-time ++ * property). Examples of where this might be sensible follow: ++ * - Part of a critical inner-loop ++ * - Frequent re-use throughout the driver, causing significant extra load ++ * instructions or control flow that would be worthwhile optimizing out. ++ * ++ * We cannot provide an exhaustive set of examples, neither can we provide a ++ * rule for every possible situation. Use common sense, and think about: what ++ * the rest of the driver will be doing; how the compiler might represent the ++ * value if it is a compile-time constant; whether an OEM shipping multiple ++ * devices would benefit much more from a single DDK binary, instead of ++ * insignificant micro-optimizations. ++ * ++ * @section sec_base_user_api_gpuprops_dyn Dynamic GPU Properties ++ * ++ * Dynamic GPU properties are presented in two sets: ++ * -# the commonly used properties in @ref base_gpu_props, which have been ++ * unpacked from GPU register bitfields. ++ * -# The full set of raw, unprocessed properties in @ref gpu_raw_gpu_props ++ * (also a member of @ref base_gpu_props). All of these are presented in ++ * the packed form, as presented by the GPU registers themselves. ++ * ++ * @usecase The raw properties in @ref gpu_raw_gpu_props are necessary to ++ * allow a user of the Mali Tools (e.g. PAT) to determine "Why is this device ++ * behaving differently?". In this case, all information about the ++ * configuration is potentially useful, but it does not need to be processed ++ * by the driver. Instead, the raw registers can be processed by the Mali ++ * Tools software on the host PC. ++ * ++ * The properties returned extend the Midgard Configuration Discovery ++ * registers. For example, GPU clock speed is not specified in the Midgard ++ * Architecture, but is necessary for OpenCL's clGetDeviceInfo() function. ++ * ++ * The GPU properties are obtained by a call to ++ * _mali_base_get_gpu_props(). This simply returns a pointer to a const ++ * base_gpu_props structure. It is constant for the life of a base ++ * context. Multiple calls to _mali_base_get_gpu_props() to a base context ++ * return the same pointer to a constant structure. This avoids cache pollution ++ * of the common data. ++ * ++ * This pointer must not be freed, because it does not point to the start of a ++ * region allocated by the memory allocator; instead, just close the @ref ++ * base_context. ++ * ++ * ++ * @section sec_base_user_api_gpuprops_config Platform Config Compile-time Properties ++ * ++ * The Platform Config File sets up gpu properties that are specific to a ++ * certain platform. Properties that are 'Implementation Defined' in the ++ * Midgard Architecture spec are placed here. ++ * ++ * @note Reference configurations are provided for Midgard Implementations, such as ++ * the Mali-T600 family. The customer need not repeat this information, and can select one of ++ * these reference configurations. For example, VA_BITS, PA_BITS and the ++ * maximum number of samples per pixel might vary between Midgard Implementations, but ++ * \b not for platforms using the Mali-T604. This information is placed in ++ * the reference configuration files. ++ * ++ * The System Integrator creates the following structure: ++ * - platform_XYZ ++ * - platform_XYZ/plat ++ * - platform_XYZ/plat/plat_config.h ++ * ++ * They then edit plat_config.h, using the example plat_config.h files as a ++ * guide. ++ * ++ * At the very least, the customer must set @ref CONFIG_GPU_CORE_TYPE, and will ++ * receive a helpful \#error message if they do not do this correctly. This ++ * selects the Reference Configuration for the Midgard Implementation. The rationale ++ * behind this decision (against asking the customer to write \#include ++ * in their plat_config.h) is as follows: ++ * - This mechanism 'looks' like a regular config file (such as Linux's ++ * .config) ++ * - It is difficult to get wrong in a way that will produce strange build ++ * errors: ++ * - They need not know where the mali_t600.h, other_midg_gpu.h etc. files are stored - and ++ * so they won't accidentally pick another file with 'mali_t600' in its name ++ * - When the build doesn't work, the System Integrator may think the DDK is ++ * doesn't work, and attempt to fix it themselves: ++ * - For the @ref CONFIG_GPU_CORE_TYPE mechanism, the only way to get past the ++ * error is to set @ref CONFIG_GPU_CORE_TYPE, and this is what the \#error tells ++ * you. ++ * - For a \#include mechanism, checks must still be made elsewhere, which the ++ * System Integrator may try working around by setting \#defines (such as ++ * VA_BITS) themselves in their plat_config.h. In the worst case, they may ++ * set the prevention-mechanism \#define of ++ * "A_CORRECT_MIDGARD_CORE_WAS_CHOSEN". ++ * - In this case, they would believe they are on the right track, because ++ * the build progresses with their fix, but with errors elsewhere. ++ * ++ * However, there is nothing to prevent the customer using \#include to organize ++ * their own configurations files hierarchically. ++ * ++ * The mechanism for the header file processing is as follows: ++ * ++ * @dot ++ digraph plat_config_mechanism { ++ rankdir=BT ++ size="6,6" ++ ++ "mali_base.h"; ++ "gpu/mali_gpu.h"; ++ ++ node [ shape=box ]; ++ { ++ rank = same; ordering = out; ++ ++ "gpu/mali_gpu_props.h"; ++ "base/midg_gpus/mali_t600.h"; ++ "base/midg_gpus/other_midg_gpu.h"; ++ } ++ { rank = same; "plat/plat_config.h"; } ++ { ++ rank = same; ++ "gpu/mali_gpu.h" [ shape=box ]; ++ gpu_chooser [ label="" style="invisible" width=0 height=0 fixedsize=true ]; ++ select_gpu [ label="Mali-T600 | Other\n(select_gpu.h)" shape=polygon,sides=4,distortion=0.25 width=3.3 height=0.99 fixedsize=true ] ; ++ } ++ node [ shape=box ]; ++ { rank = same; "plat/plat_config.h"; } ++ { rank = same; "mali_base.h"; } ++ ++ "mali_base.h" -> "gpu/mali_gpu.h" -> "gpu/mali_gpu_props.h"; ++ "mali_base.h" -> "plat/plat_config.h" ; ++ "mali_base.h" -> select_gpu ; ++ ++ "plat/plat_config.h" -> gpu_chooser [style="dotted,bold" dir=none weight=4] ; ++ gpu_chooser -> select_gpu [style="dotted,bold"] ; ++ ++ select_gpu -> "base/midg_gpus/mali_t600.h" ; ++ select_gpu -> "base/midg_gpus/other_midg_gpu.h" ; ++ } ++ @enddot ++ * ++ * ++ * @section sec_base_user_api_gpuprops_kernel Kernel Operation ++ * ++ * During Base Context Create time, user-side makes a single kernel call: ++ * - A call to fill user memory with GPU information structures ++ * ++ * The kernel-side will fill the provided the entire processed @ref base_gpu_props ++ * structure, because this information is required in both ++ * user and kernel side; it does not make sense to decode it twice. ++ * ++ * Coherency groups must be derived from the bitmasks, but this can be done ++ * kernel side, and just once at kernel startup: Coherency groups must already ++ * be known kernel-side, to support chains that specify a 'Only Coherent Group' ++ * SW requirement, or 'Only Coherent Group with Tiler' SW requirement. ++ * ++ * @section sec_base_user_api_gpuprops_cocalc Coherency Group calculation ++ * Creation of the coherent group data is done at device-driver startup, and so ++ * is one-time. This will most likely involve a loop with CLZ, shifting, and ++ * bit clearing on the L2_PRESENT mask, depending on whether the ++ * system is L2 Coherent. The number of shader cores is done by a ++ * population count, since faulty cores may be disabled during production, ++ * producing a non-contiguous mask. ++ * ++ * The memory requirements for this algorithm can be determined either by a u64 ++ * population count on the L2_PRESENT mask (a LUT helper already is ++ * required for the above), or simple assumption that there can be no more than ++ * 16 coherent groups, since core groups are typically 4 cores. ++ */ ++ ++/** ++ * @addtogroup base_user_api_gpuprops User-side Base GPU Property Query APIs ++ * @{ ++ */ ++ ++/** ++ * @addtogroup base_user_api_gpuprops_dyn Dynamic HW Properties ++ * @{ ++ */ ++ ++#define BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS 3 ++ ++#define BASE_MAX_COHERENT_GROUPS 16 ++ ++struct mali_base_gpu_core_props { ++ /** ++ * Product specific value. ++ */ ++ u32 product_id; ++ ++ /** ++ * Status of the GPU release. ++ * No defined values, but starts at 0 and increases by one for each ++ * release status (alpha, beta, EAC, etc.). ++ * 4 bit values (0-15). ++ */ ++ u16 version_status; ++ ++ /** ++ * Minor release number of the GPU. "P" part of an "RnPn" release number. ++ * 8 bit values (0-255). ++ */ ++ u16 minor_revision; ++ ++ /** ++ * Major release number of the GPU. "R" part of an "RnPn" release number. ++ * 4 bit values (0-15). ++ */ ++ u16 major_revision; ++ ++ u16 padding; ++ ++ /** ++ * This property is deprecated since it has not contained the real current ++ * value of GPU clock speed. It is kept here only for backwards compatibility. ++ * For the new ioctl interface, it is ignored and is treated as a padding ++ * to keep the structure of the same size and retain the placement of its ++ * members. ++ */ ++ u32 gpu_speed_mhz; ++ ++ /** ++ * @usecase GPU clock max/min speed is required for computing best/worst case ++ * in tasks as job scheduling ant irq_throttling. (It is not specified in the ++ * Midgard Architecture). ++ * Also, GPU clock max speed is used for OpenCL's clGetDeviceInfo() function. ++ */ ++ u32 gpu_freq_khz_max; ++ u32 gpu_freq_khz_min; ++ ++ /** ++ * Size of the shader program counter, in bits. ++ */ ++ u32 log2_program_counter_size; ++ ++ /** ++ * TEXTURE_FEATURES_x registers, as exposed by the GPU. This is a ++ * bitpattern where a set bit indicates that the format is supported. ++ * ++ * Before using a texture format, it is recommended that the corresponding ++ * bit be checked. ++ */ ++ u32 texture_features[BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS]; ++ ++ /** ++ * Theoretical maximum memory available to the GPU. It is unlikely that a ++ * client will be able to allocate all of this memory for their own ++ * purposes, but this at least provides an upper bound on the memory ++ * available to the GPU. ++ * ++ * This is required for OpenCL's clGetDeviceInfo() call when ++ * CL_DEVICE_GLOBAL_MEM_SIZE is requested, for OpenCL GPU devices. The ++ * client will not be expecting to allocate anywhere near this value. ++ */ ++ u64 gpu_available_memory_size; ++}; ++ ++/** ++ * ++ * More information is possible - but associativity and bus width are not ++ * required by upper-level apis. ++ */ ++struct mali_base_gpu_l2_cache_props { ++ u8 log2_line_size; ++ u8 log2_cache_size; ++ u8 num_l2_slices; /* Number of L2C slices. 1 or higher */ ++ u8 padding[5]; ++}; ++ ++struct mali_base_gpu_tiler_props { ++ u32 bin_size_bytes; /* Max is 4*2^15 */ ++ u32 max_active_levels; /* Max is 2^15 */ ++}; ++ ++/** ++ * GPU threading system details. ++ */ ++struct mali_base_gpu_thread_props { ++ u32 max_threads; /* Max. number of threads per core */ ++ u32 max_workgroup_size; /* Max. number of threads per workgroup */ ++ u32 max_barrier_size; /* Max. number of threads that can synchronize on a simple barrier */ ++ u16 max_registers; /* Total size [1..65535] of the register file available per core. */ ++ u8 max_task_queue; /* Max. tasks [1..255] which may be sent to a core before it becomes blocked. */ ++ u8 max_thread_group_split; /* Max. allowed value [1..15] of the Thread Group Split field. */ ++ u8 impl_tech; /* 0 = Not specified, 1 = Silicon, 2 = FPGA, 3 = SW Model/Emulation */ ++ u8 padding[7]; ++}; ++ ++/** ++ * @brief descriptor for a coherent group ++ * ++ * \c core_mask exposes all cores in that coherent group, and \c num_cores ++ * provides a cached population-count for that mask. ++ * ++ * @note Whilst all cores are exposed in the mask, not all may be available to ++ * the application, depending on the Kernel Power policy. ++ * ++ * @note if u64s must be 8-byte aligned, then this structure has 32-bits of wastage. ++ */ ++struct mali_base_gpu_coherent_group { ++ u64 core_mask; /**< Core restriction mask required for the group */ ++ u16 num_cores; /**< Number of cores in the group */ ++ u16 padding[3]; ++}; ++ ++/** ++ * @brief Coherency group information ++ * ++ * Note that the sizes of the members could be reduced. However, the \c group ++ * member might be 8-byte aligned to ensure the u64 core_mask is 8-byte ++ * aligned, thus leading to wastage if the other members sizes were reduced. ++ * ++ * The groups are sorted by core mask. The core masks are non-repeating and do ++ * not intersect. ++ */ ++struct mali_base_gpu_coherent_group_info { ++ u32 num_groups; ++ ++ /** ++ * Number of core groups (coherent or not) in the GPU. Equivalent to the number of L2 Caches. ++ * ++ * The GPU Counter dumping writes 2048 bytes per core group, regardless of ++ * whether the core groups are coherent or not. Hence this member is needed ++ * to calculate how much memory is required for dumping. ++ * ++ * @note Do not use it to work out how many valid elements are in the ++ * group[] member. Use num_groups instead. ++ */ ++ u32 num_core_groups; ++ ++ /** ++ * Coherency features of the memory, accessed by @ref gpu_mem_features ++ * methods ++ */ ++ u32 coherency; ++ ++ u32 padding; ++ ++ /** ++ * Descriptors of coherent groups ++ */ ++ struct mali_base_gpu_coherent_group group[BASE_MAX_COHERENT_GROUPS]; ++}; ++ ++/** ++ * A complete description of the GPU's Hardware Configuration Discovery ++ * registers. ++ * ++ * The information is presented inefficiently for access. For frequent access, ++ * the values should be better expressed in an unpacked form in the ++ * base_gpu_props structure. ++ * ++ * @usecase The raw properties in @ref gpu_raw_gpu_props are necessary to ++ * allow a user of the Mali Tools (e.g. PAT) to determine "Why is this device ++ * behaving differently?". In this case, all information about the ++ * configuration is potentially useful, but it does not need to be processed ++ * by the driver. Instead, the raw registers can be processed by the Mali ++ * Tools software on the host PC. ++ * ++ */ ++struct gpu_raw_gpu_props { ++ u64 shader_present; ++ u64 tiler_present; ++ u64 l2_present; ++ u64 stack_present; ++ ++ u32 l2_features; ++ u32 suspend_size; /* API 8.2+ */ ++ u32 mem_features; ++ u32 mmu_features; ++ ++ u32 as_present; ++ ++ u32 js_present; ++ u32 js_features[GPU_MAX_JOB_SLOTS]; ++ u32 tiler_features; ++ u32 texture_features[3]; ++ ++ u32 gpu_id; ++ ++ u32 thread_max_threads; ++ u32 thread_max_workgroup_size; ++ u32 thread_max_barrier_size; ++ u32 thread_features; ++ ++ /* ++ * Note: This is the _selected_ coherency mode rather than the ++ * available modes as exposed in the coherency_features register. ++ */ ++ u32 coherency_mode; ++}; ++ ++/** ++ * Return structure for _mali_base_get_gpu_props(). ++ * ++ * NOTE: the raw_props member in this data structure contains the register ++ * values from which the value of the other members are derived. The derived ++ * members exist to allow for efficient access and/or shielding the details ++ * of the layout of the registers. ++ * ++ */ ++typedef struct mali_base_gpu_props { ++ struct mali_base_gpu_core_props core_props; ++ struct mali_base_gpu_l2_cache_props l2_props; ++ u64 unused_1; /* keep for backwards compatibility */ ++ struct mali_base_gpu_tiler_props tiler_props; ++ struct mali_base_gpu_thread_props thread_props; ++ ++ /** This member is large, likely to be 128 bytes */ ++ struct gpu_raw_gpu_props raw_props; ++ ++ /** This must be last member of the structure */ ++ struct mali_base_gpu_coherent_group_info coherency_info; ++} base_gpu_props; ++ ++/** @} end group base_user_api_gpuprops_dyn */ ++ ++/** @} end group base_user_api_gpuprops */ ++ ++/** ++ * @addtogroup base_user_api_core User-side Base core APIs ++ * @{ ++ */ ++ ++/** ++ * \enum base_context_create_flags ++ * ++ * Flags to pass to ::base_context_init. ++ * Flags can be ORed together to enable multiple things. ++ * ++ * These share the same space as BASEP_CONTEXT_FLAG_*, and so must ++ * not collide with them. ++ */ ++enum base_context_create_flags { ++ /** No flags set */ ++ BASE_CONTEXT_CREATE_FLAG_NONE = 0, ++ ++ /** Base context is embedded in a cctx object (flag used for CINSTR software counter macros) */ ++ BASE_CONTEXT_CCTX_EMBEDDED = (1u << 0), ++ ++ /** Base context is a 'System Monitor' context for Hardware counters. ++ * ++ * One important side effect of this is that job submission is disabled. */ ++ BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED = (1u << 1) ++}; ++ ++/** ++ * Bitpattern describing the ::base_context_create_flags that can be passed to base_context_init() ++ */ ++#define BASE_CONTEXT_CREATE_ALLOWED_FLAGS \ ++ (((u32)BASE_CONTEXT_CCTX_EMBEDDED) | \ ++ ((u32)BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED)) ++ ++/** ++ * Bitpattern describing the ::base_context_create_flags that can be passed to the kernel ++ */ ++#define BASE_CONTEXT_CREATE_KERNEL_FLAGS \ ++ ((u32)BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED) ++ ++/* ++ * Private flags used on the base context ++ * ++ * These start at bit 31, and run down to zero. ++ * ++ * They share the same space as @ref base_context_create_flags, and so must ++ * not collide with them. ++ */ ++/** Private flag tracking whether job descriptor dumping is disabled */ ++#define BASEP_CONTEXT_FLAG_JOB_DUMP_DISABLED ((u32)(1 << 31)) ++ ++/** @} end group base_user_api_core */ ++ ++/** @} end group base_user_api */ ++ ++/** ++ * @addtogroup base_plat_config_gpuprops Base Platform Config GPU Properties ++ * @{ ++ * ++ * C Pre-processor macros are exposed here to do with Platform ++ * Config. ++ * ++ * These include: ++ * - GPU Properties that are constant on a particular Midgard Family ++ * Implementation e.g. Maximum samples per pixel on Mali-T600. ++ * - General platform config for the GPU, such as the GPU major and minor ++ * revison. ++ */ ++ ++/** @} end group base_plat_config_gpuprops */ ++ ++/** ++ * @addtogroup base_api Base APIs ++ * @{ ++ */ ++ ++/** ++ * @brief The payload for a replay job. This must be in GPU memory. ++ */ ++typedef struct base_jd_replay_payload { ++ /** ++ * Pointer to the first entry in the base_jd_replay_jc list. These ++ * will be replayed in @b reverse order (so that extra ones can be added ++ * to the head in future soft jobs without affecting this soft job) ++ */ ++ u64 tiler_jc_list; ++ ++ /** ++ * Pointer to the fragment job chain. ++ */ ++ u64 fragment_jc; ++ ++ /** ++ * Pointer to the tiler heap free FBD field to be modified. ++ */ ++ u64 tiler_heap_free; ++ ++ /** ++ * Hierarchy mask for the replayed fragment jobs. May be zero. ++ */ ++ u16 fragment_hierarchy_mask; ++ ++ /** ++ * Hierarchy mask for the replayed tiler jobs. May be zero. ++ */ ++ u16 tiler_hierarchy_mask; ++ ++ /** ++ * Default weight to be used for hierarchy levels not in the original ++ * mask. ++ */ ++ u32 hierarchy_default_weight; ++ ++ /** ++ * Core requirements for the tiler job chain ++ */ ++ base_jd_core_req tiler_core_req; ++ ++ /** ++ * Core requirements for the fragment job chain ++ */ ++ base_jd_core_req fragment_core_req; ++} base_jd_replay_payload; ++ ++#ifdef BASE_LEGACY_UK10_2_SUPPORT ++typedef struct base_jd_replay_payload_uk10_2 { ++ u64 tiler_jc_list; ++ u64 fragment_jc; ++ u64 tiler_heap_free; ++ u16 fragment_hierarchy_mask; ++ u16 tiler_hierarchy_mask; ++ u32 hierarchy_default_weight; ++ u16 tiler_core_req; ++ u16 fragment_core_req; ++ u8 padding[4]; ++} base_jd_replay_payload_uk10_2; ++#endif /* BASE_LEGACY_UK10_2_SUPPORT */ ++ ++/** ++ * @brief An entry in the linked list of job chains to be replayed. This must ++ * be in GPU memory. ++ */ ++typedef struct base_jd_replay_jc { ++ /** ++ * Pointer to next entry in the list. A setting of NULL indicates the ++ * end of the list. ++ */ ++ u64 next; ++ ++ /** ++ * Pointer to the job chain. ++ */ ++ u64 jc; ++ ++} base_jd_replay_jc; ++ ++/* Maximum number of jobs allowed in a fragment chain in the payload of a ++ * replay job */ ++#define BASE_JD_REPLAY_F_CHAIN_JOB_LIMIT 256 ++ ++/** @} end group base_api */ ++ ++typedef struct base_profiling_controls { ++ u32 profiling_controls[FBDUMP_CONTROL_MAX]; ++} base_profiling_controls; ++ ++/* Enable additional tracepoints for latency measurements (TL_ATOM_READY, ++ * TL_ATOM_DONE, TL_ATOM_PRIO_CHANGE, TL_ATOM_EVENT_POST) */ ++#define BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS (1 << 0) ++ ++/* Indicate that job dumping is enabled. This could affect certain timers ++ * to account for the performance impact. */ ++#define BASE_TLSTREAM_JOB_DUMPING_ENABLED (1 << 1) ++ ++#define BASE_TLSTREAM_FLAGS_MASK (BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS | \ ++ BASE_TLSTREAM_JOB_DUMPING_ENABLED) ++ ++#endif /* _BASE_KERNEL_H_ */ +diff --git a/drivers/gpu/arm/midgard/mali_base_mem_priv.h b/drivers/gpu/arm/midgard/mali_base_mem_priv.h +new file mode 100755 +index 000000000000..4a98a72cc37a +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_base_mem_priv.h +@@ -0,0 +1,52 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2014 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#ifndef _BASE_MEM_PRIV_H_ ++#define _BASE_MEM_PRIV_H_ ++ ++#define BASE_SYNCSET_OP_MSYNC (1U << 0) ++#define BASE_SYNCSET_OP_CSYNC (1U << 1) ++ ++/* ++ * This structure describe a basic memory coherency operation. ++ * It can either be: ++ * @li a sync from CPU to Memory: ++ * - type = ::BASE_SYNCSET_OP_MSYNC ++ * - mem_handle = a handle to the memory object on which the operation ++ * is taking place ++ * - user_addr = the address of the range to be synced ++ * - size = the amount of data to be synced, in bytes ++ * - offset is ignored. ++ * @li a sync from Memory to CPU: ++ * - type = ::BASE_SYNCSET_OP_CSYNC ++ * - mem_handle = a handle to the memory object on which the operation ++ * is taking place ++ * - user_addr = the address of the range to be synced ++ * - size = the amount of data to be synced, in bytes. ++ * - offset is ignored. ++ */ ++struct basep_syncset { ++ base_mem_handle mem_handle; ++ u64 user_addr; ++ u64 size; ++ u8 type; ++ u8 padding[7]; ++}; ++ ++#endif +diff --git a/drivers/gpu/arm/midgard/mali_base_vendor_specific_func.h b/drivers/gpu/arm/midgard/mali_base_vendor_specific_func.h +new file mode 100755 +index 000000000000..be454a216a39 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_base_vendor_specific_func.h +@@ -0,0 +1,24 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010, 2012-2013, 2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++#ifndef _BASE_VENDOR_SPEC_FUNC_H_ ++#define _BASE_VENDOR_SPEC_FUNC_H_ ++ ++int kbase_get_vendor_specific_cpu_clock_speed(u32 * const); ++ ++#endif /*_BASE_VENDOR_SPEC_FUNC_H_*/ +diff --git a/drivers/gpu/arm/midgard/mali_kbase.h b/drivers/gpu/arm/midgard/mali_kbase.h +new file mode 100755 +index 000000000000..0d9bf23dc685 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase.h +@@ -0,0 +1,612 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#ifndef _KBASE_H_ ++#define _KBASE_H_ ++ ++#include ++ ++#include ++ ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "mali_base_kernel.h" ++#include ++#include ++ ++/* ++ * Include mali_kbase_defs.h first as this provides types needed by other local ++ * header files. ++ */ ++#include "mali_kbase_defs.h" ++ ++#include "mali_kbase_context.h" ++#include "mali_kbase_strings.h" ++#include "mali_kbase_mem_lowlevel.h" ++#include "mali_kbase_trace_timeline.h" ++#include "mali_kbase_js.h" ++#include "mali_kbase_mem.h" ++#include "mali_kbase_utility.h" ++#include "mali_kbase_gpu_memory_debugfs.h" ++#include "mali_kbase_mem_profile_debugfs.h" ++#include "mali_kbase_debug_job_fault.h" ++#include "mali_kbase_jd_debugfs.h" ++#include "mali_kbase_gpuprops.h" ++#include "mali_kbase_jm.h" ++#include "mali_kbase_vinstr.h" ++ ++#include "ipa/mali_kbase_ipa.h" ++ ++#ifdef CONFIG_GPU_TRACEPOINTS ++#include ++#endif ++/** ++ * @page page_base_kernel_main Kernel-side Base (KBase) APIs ++ */ ++ ++/** ++ * @defgroup base_kbase_api Kernel-side Base (KBase) APIs ++ */ ++ ++struct kbase_device *kbase_device_alloc(void); ++/* ++* note: configuration attributes member of kbdev needs to have ++* been setup before calling kbase_device_init ++*/ ++ ++/* ++* API to acquire device list semaphore and return pointer ++* to the device list head ++*/ ++const struct list_head *kbase_dev_list_get(void); ++/* API to release the device list semaphore */ ++void kbase_dev_list_put(const struct list_head *dev_list); ++ ++int kbase_device_init(struct kbase_device * const kbdev); ++void kbase_device_term(struct kbase_device *kbdev); ++void kbase_device_free(struct kbase_device *kbdev); ++int kbase_device_has_feature(struct kbase_device *kbdev, u32 feature); ++ ++/* Needed for gator integration and for reporting vsync information */ ++struct kbase_device *kbase_find_device(int minor); ++void kbase_release_device(struct kbase_device *kbdev); ++ ++void kbase_set_profiling_control(struct kbase_device *kbdev, u32 control, u32 value); ++ ++struct kbase_context * ++kbase_create_context(struct kbase_device *kbdev, bool is_compat); ++void kbase_destroy_context(struct kbase_context *kctx); ++ ++int kbase_jd_init(struct kbase_context *kctx); ++void kbase_jd_exit(struct kbase_context *kctx); ++ ++/** ++ * kbase_jd_submit - Submit atoms to the job dispatcher ++ * ++ * @kctx: The kbase context to submit to ++ * @user_addr: The address in user space of the struct base_jd_atom_v2 array ++ * @nr_atoms: The number of atoms in the array ++ * @stride: sizeof(struct base_jd_atom_v2) ++ * @uk6_atom: true if the atoms are legacy atoms (struct base_jd_atom_v2_uk6) ++ * ++ * Return: 0 on success or error code ++ */ ++int kbase_jd_submit(struct kbase_context *kctx, ++ void __user *user_addr, u32 nr_atoms, u32 stride, ++ bool uk6_atom); ++ ++/** ++ * kbase_jd_done_worker - Handle a job completion ++ * @data: a &struct work_struct ++ * ++ * This function requeues the job from the runpool (if it was soft-stopped or ++ * removed from NEXT registers). ++ * ++ * Removes it from the system if it finished/failed/was cancelled. ++ * ++ * Resolves dependencies to add dependent jobs to the context, potentially ++ * starting them if necessary (which may add more references to the context) ++ * ++ * Releases the reference to the context from the no-longer-running job. ++ * ++ * Handles retrying submission outside of IRQ context if it failed from within ++ * IRQ context. ++ */ ++void kbase_jd_done_worker(struct work_struct *data); ++ ++void kbase_jd_done(struct kbase_jd_atom *katom, int slot_nr, ktime_t *end_timestamp, ++ kbasep_js_atom_done_code done_code); ++void kbase_jd_cancel(struct kbase_device *kbdev, struct kbase_jd_atom *katom); ++void kbase_jd_zap_context(struct kbase_context *kctx); ++bool jd_done_nolock(struct kbase_jd_atom *katom, ++ struct list_head *completed_jobs_ctx); ++void kbase_jd_free_external_resources(struct kbase_jd_atom *katom); ++bool jd_submit_atom(struct kbase_context *kctx, ++ const struct base_jd_atom_v2 *user_atom, ++ struct kbase_jd_atom *katom); ++void kbase_jd_dep_clear_locked(struct kbase_jd_atom *katom); ++ ++void kbase_job_done(struct kbase_device *kbdev, u32 done); ++ ++/** ++ * kbase_job_slot_ctx_priority_check_locked(): - Check for lower priority atoms ++ * and soft stop them ++ * @kctx: Pointer to context to check. ++ * @katom: Pointer to priority atom. ++ * ++ * Atoms from @kctx on the same job slot as @katom, which have lower priority ++ * than @katom will be soft stopped and put back in the queue, so that atoms ++ * with higher priority can run. ++ * ++ * The hwaccess_lock must be held when calling this function. ++ */ ++void kbase_job_slot_ctx_priority_check_locked(struct kbase_context *kctx, ++ struct kbase_jd_atom *katom); ++ ++void kbase_job_slot_softstop(struct kbase_device *kbdev, int js, ++ struct kbase_jd_atom *target_katom); ++void kbase_job_slot_softstop_swflags(struct kbase_device *kbdev, int js, ++ struct kbase_jd_atom *target_katom, u32 sw_flags); ++void kbase_job_slot_hardstop(struct kbase_context *kctx, int js, ++ struct kbase_jd_atom *target_katom); ++void kbase_job_check_enter_disjoint(struct kbase_device *kbdev, u32 action, ++ base_jd_core_req core_reqs, struct kbase_jd_atom *target_katom); ++void kbase_job_check_leave_disjoint(struct kbase_device *kbdev, ++ struct kbase_jd_atom *target_katom); ++ ++void kbase_event_post(struct kbase_context *ctx, struct kbase_jd_atom *event); ++int kbase_event_dequeue(struct kbase_context *ctx, struct base_jd_event_v2 *uevent); ++int kbase_event_pending(struct kbase_context *ctx); ++int kbase_event_init(struct kbase_context *kctx); ++void kbase_event_close(struct kbase_context *kctx); ++void kbase_event_cleanup(struct kbase_context *kctx); ++void kbase_event_wakeup(struct kbase_context *kctx); ++ ++int kbase_process_soft_job(struct kbase_jd_atom *katom); ++int kbase_prepare_soft_job(struct kbase_jd_atom *katom); ++void kbase_finish_soft_job(struct kbase_jd_atom *katom); ++void kbase_cancel_soft_job(struct kbase_jd_atom *katom); ++void kbase_resume_suspended_soft_jobs(struct kbase_device *kbdev); ++void kbasep_remove_waiting_soft_job(struct kbase_jd_atom *katom); ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++void kbase_soft_event_wait_callback(struct kbase_jd_atom *katom); ++#endif ++int kbase_soft_event_update(struct kbase_context *kctx, ++ u64 event, ++ unsigned char new_status); ++ ++bool kbase_replay_process(struct kbase_jd_atom *katom); ++ ++void kbasep_soft_job_timeout_worker(struct timer_list *t); ++void kbasep_complete_triggered_soft_events(struct kbase_context *kctx, u64 evt); ++ ++/* api used internally for register access. Contains validation and tracing */ ++void kbase_device_trace_register_access(struct kbase_context *kctx, enum kbase_reg_access_type type, u16 reg_offset, u32 reg_value); ++int kbase_device_trace_buffer_install( ++ struct kbase_context *kctx, u32 *tb, size_t size); ++void kbase_device_trace_buffer_uninstall(struct kbase_context *kctx); ++ ++/* api to be ported per OS, only need to do the raw register access */ ++void kbase_os_reg_write(struct kbase_device *kbdev, u16 offset, u32 value); ++u32 kbase_os_reg_read(struct kbase_device *kbdev, u16 offset); ++ ++void kbasep_as_do_poke(struct work_struct *work); ++ ++/** Returns the name associated with a Mali exception code ++ * ++ * This function is called from the interrupt handler when a GPU fault occurs. ++ * It reports the details of the fault using KBASE_DEBUG_PRINT_WARN. ++ * ++ * @param[in] kbdev The kbase device that the GPU fault occurred from. ++ * @param[in] exception_code exception code ++ * @return name associated with the exception code ++ */ ++const char *kbase_exception_name(struct kbase_device *kbdev, ++ u32 exception_code); ++ ++/** ++ * Check whether a system suspend is in progress, or has already been suspended ++ * ++ * The caller should ensure that either kbdev->pm.active_count_lock is held, or ++ * a dmb was executed recently (to ensure the value is most ++ * up-to-date). However, without a lock the value could change afterwards. ++ * ++ * @return false if a suspend is not in progress ++ * @return !=false otherwise ++ */ ++static inline bool kbase_pm_is_suspending(struct kbase_device *kbdev) ++{ ++ return kbdev->pm.suspending; ++} ++ ++/** ++ * Return the atom's ID, as was originally supplied by userspace in ++ * base_jd_atom_v2::atom_number ++ */ ++static inline int kbase_jd_atom_id(struct kbase_context *kctx, struct kbase_jd_atom *katom) ++{ ++ int result; ++ ++ KBASE_DEBUG_ASSERT(kctx); ++ KBASE_DEBUG_ASSERT(katom); ++ KBASE_DEBUG_ASSERT(katom->kctx == kctx); ++ ++ result = katom - &kctx->jctx.atoms[0]; ++ KBASE_DEBUG_ASSERT(result >= 0 && result <= BASE_JD_ATOM_COUNT); ++ return result; ++} ++ ++/** ++ * kbase_jd_atom_from_id - Return the atom structure for the given atom ID ++ * @kctx: Context pointer ++ * @id: ID of atom to retrieve ++ * ++ * Return: Pointer to struct kbase_jd_atom associated with the supplied ID ++ */ ++static inline struct kbase_jd_atom *kbase_jd_atom_from_id( ++ struct kbase_context *kctx, int id) ++{ ++ return &kctx->jctx.atoms[id]; ++} ++ ++/** ++ * Initialize the disjoint state ++ * ++ * The disjoint event count and state are both set to zero. ++ * ++ * Disjoint functions usage: ++ * ++ * The disjoint event count should be incremented whenever a disjoint event occurs. ++ * ++ * There are several cases which are regarded as disjoint behavior. Rather than just increment ++ * the counter during disjoint events we also increment the counter when jobs may be affected ++ * by what the GPU is currently doing. To facilitate this we have the concept of disjoint state. ++ * ++ * Disjoint state is entered during GPU reset and for the entire time that an atom is replaying ++ * (as part of the replay workaround). Increasing the disjoint state also increases the count of ++ * disjoint events. ++ * ++ * The disjoint state is then used to increase the count of disjoint events during job submission ++ * and job completion. Any atom submitted or completed while the disjoint state is greater than ++ * zero is regarded as a disjoint event. ++ * ++ * The disjoint event counter is also incremented immediately whenever a job is soft stopped ++ * and during context creation. ++ * ++ * @param kbdev The kbase device ++ */ ++void kbase_disjoint_init(struct kbase_device *kbdev); ++ ++/** ++ * Increase the count of disjoint events ++ * called when a disjoint event has happened ++ * ++ * @param kbdev The kbase device ++ */ ++void kbase_disjoint_event(struct kbase_device *kbdev); ++ ++/** ++ * Increase the count of disjoint events only if the GPU is in a disjoint state ++ * ++ * This should be called when something happens which could be disjoint if the GPU ++ * is in a disjoint state. The state refcount keeps track of this. ++ * ++ * @param kbdev The kbase device ++ */ ++void kbase_disjoint_event_potential(struct kbase_device *kbdev); ++ ++/** ++ * Returns the count of disjoint events ++ * ++ * @param kbdev The kbase device ++ * @return the count of disjoint events ++ */ ++u32 kbase_disjoint_event_get(struct kbase_device *kbdev); ++ ++/** ++ * Increment the refcount state indicating that the GPU is in a disjoint state. ++ * ++ * Also Increment the disjoint event count (calls @ref kbase_disjoint_event) ++ * eventually after the disjoint state has completed @ref kbase_disjoint_state_down ++ * should be called ++ * ++ * @param kbdev The kbase device ++ */ ++void kbase_disjoint_state_up(struct kbase_device *kbdev); ++ ++/** ++ * Decrement the refcount state ++ * ++ * Also Increment the disjoint event count (calls @ref kbase_disjoint_event) ++ * ++ * Called after @ref kbase_disjoint_state_up once the disjoint state is over ++ * ++ * @param kbdev The kbase device ++ */ ++void kbase_disjoint_state_down(struct kbase_device *kbdev); ++ ++/** ++ * If a job is soft stopped and the number of contexts is >= this value ++ * it is reported as a disjoint event ++ */ ++#define KBASE_DISJOINT_STATE_INTERLEAVED_CONTEXT_COUNT_THRESHOLD 2 ++ ++#if !defined(UINT64_MAX) ++ #define UINT64_MAX ((uint64_t)0xFFFFFFFFFFFFFFFFULL) ++#endif ++ ++#if KBASE_TRACE_ENABLE ++void kbasep_trace_debugfs_init(struct kbase_device *kbdev); ++ ++#ifndef CONFIG_MALI_SYSTEM_TRACE ++/** Add trace values about a job-slot ++ * ++ * @note Any functions called through this macro will still be evaluated in ++ * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any ++ * functions called to get the parameters supplied to this macro must: ++ * - be static or static inline ++ * - must just return 0 and have no other statements present in the body. ++ */ ++#define KBASE_TRACE_ADD_SLOT(kbdev, code, ctx, katom, gpu_addr, jobslot) \ ++ kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \ ++ KBASE_TRACE_FLAG_JOBSLOT, 0, jobslot, 0) ++ ++/** Add trace values about a job-slot, with info ++ * ++ * @note Any functions called through this macro will still be evaluated in ++ * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any ++ * functions called to get the parameters supplied to this macro must: ++ * - be static or static inline ++ * - must just return 0 and have no other statements present in the body. ++ */ ++#define KBASE_TRACE_ADD_SLOT_INFO(kbdev, code, ctx, katom, gpu_addr, jobslot, info_val) \ ++ kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \ ++ KBASE_TRACE_FLAG_JOBSLOT, 0, jobslot, info_val) ++ ++/** Add trace values about a ctx refcount ++ * ++ * @note Any functions called through this macro will still be evaluated in ++ * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any ++ * functions called to get the parameters supplied to this macro must: ++ * - be static or static inline ++ * - must just return 0 and have no other statements present in the body. ++ */ ++#define KBASE_TRACE_ADD_REFCOUNT(kbdev, code, ctx, katom, gpu_addr, refcount) \ ++ kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \ ++ KBASE_TRACE_FLAG_REFCOUNT, refcount, 0, 0) ++/** Add trace values about a ctx refcount, and info ++ * ++ * @note Any functions called through this macro will still be evaluated in ++ * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any ++ * functions called to get the parameters supplied to this macro must: ++ * - be static or static inline ++ * - must just return 0 and have no other statements present in the body. ++ */ ++#define KBASE_TRACE_ADD_REFCOUNT_INFO(kbdev, code, ctx, katom, gpu_addr, refcount, info_val) \ ++ kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \ ++ KBASE_TRACE_FLAG_REFCOUNT, refcount, 0, info_val) ++ ++/** Add trace values (no slot or refcount) ++ * ++ * @note Any functions called through this macro will still be evaluated in ++ * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any ++ * functions called to get the parameters supplied to this macro must: ++ * - be static or static inline ++ * - must just return 0 and have no other statements present in the body. ++ */ ++#define KBASE_TRACE_ADD(kbdev, code, ctx, katom, gpu_addr, info_val) \ ++ kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \ ++ 0, 0, 0, info_val) ++ ++/** Clear the trace */ ++#define KBASE_TRACE_CLEAR(kbdev) \ ++ kbasep_trace_clear(kbdev) ++ ++/** Dump the slot trace */ ++#define KBASE_TRACE_DUMP(kbdev) \ ++ kbasep_trace_dump(kbdev) ++ ++/** PRIVATE - do not use directly. Use KBASE_TRACE_ADD() instead */ ++void kbasep_trace_add(struct kbase_device *kbdev, enum kbase_trace_code code, void *ctx, struct kbase_jd_atom *katom, u64 gpu_addr, u8 flags, int refcount, int jobslot, unsigned long info_val); ++/** PRIVATE - do not use directly. Use KBASE_TRACE_CLEAR() instead */ ++void kbasep_trace_clear(struct kbase_device *kbdev); ++#else /* #ifndef CONFIG_MALI_SYSTEM_TRACE */ ++/* Dispatch kbase trace events as system trace events */ ++#include ++#define KBASE_TRACE_ADD_SLOT(kbdev, code, ctx, katom, gpu_addr, jobslot)\ ++ trace_mali_##code(jobslot, 0) ++ ++#define KBASE_TRACE_ADD_SLOT_INFO(kbdev, code, ctx, katom, gpu_addr, jobslot, info_val)\ ++ trace_mali_##code(jobslot, info_val) ++ ++#define KBASE_TRACE_ADD_REFCOUNT(kbdev, code, ctx, katom, gpu_addr, refcount)\ ++ trace_mali_##code(refcount, 0) ++ ++#define KBASE_TRACE_ADD_REFCOUNT_INFO(kbdev, code, ctx, katom, gpu_addr, refcount, info_val)\ ++ trace_mali_##code(refcount, info_val) ++ ++#define KBASE_TRACE_ADD(kbdev, code, ctx, katom, gpu_addr, info_val)\ ++ trace_mali_##code(gpu_addr, info_val) ++ ++#define KBASE_TRACE_CLEAR(kbdev)\ ++ do {\ ++ CSTD_UNUSED(kbdev);\ ++ CSTD_NOP(0);\ ++ } while (0) ++#define KBASE_TRACE_DUMP(kbdev)\ ++ do {\ ++ CSTD_UNUSED(kbdev);\ ++ CSTD_NOP(0);\ ++ } while (0) ++ ++#endif /* #ifndef CONFIG_MALI_SYSTEM_TRACE */ ++#else ++#define KBASE_TRACE_ADD_SLOT(kbdev, code, ctx, katom, gpu_addr, jobslot)\ ++ do {\ ++ CSTD_UNUSED(kbdev);\ ++ CSTD_NOP(code);\ ++ CSTD_UNUSED(ctx);\ ++ CSTD_UNUSED(katom);\ ++ CSTD_UNUSED(gpu_addr);\ ++ CSTD_UNUSED(jobslot);\ ++ } while (0) ++ ++#define KBASE_TRACE_ADD_SLOT_INFO(kbdev, code, ctx, katom, gpu_addr, jobslot, info_val)\ ++ do {\ ++ CSTD_UNUSED(kbdev);\ ++ CSTD_NOP(code);\ ++ CSTD_UNUSED(ctx);\ ++ CSTD_UNUSED(katom);\ ++ CSTD_UNUSED(gpu_addr);\ ++ CSTD_UNUSED(jobslot);\ ++ CSTD_UNUSED(info_val);\ ++ CSTD_NOP(0);\ ++ } while (0) ++ ++#define KBASE_TRACE_ADD_REFCOUNT(kbdev, code, ctx, katom, gpu_addr, refcount)\ ++ do {\ ++ CSTD_UNUSED(kbdev);\ ++ CSTD_NOP(code);\ ++ CSTD_UNUSED(ctx);\ ++ CSTD_UNUSED(katom);\ ++ CSTD_UNUSED(gpu_addr);\ ++ CSTD_UNUSED(refcount);\ ++ CSTD_NOP(0);\ ++ } while (0) ++ ++#define KBASE_TRACE_ADD_REFCOUNT_INFO(kbdev, code, ctx, katom, gpu_addr, refcount, info_val)\ ++ do {\ ++ CSTD_UNUSED(kbdev);\ ++ CSTD_NOP(code);\ ++ CSTD_UNUSED(ctx);\ ++ CSTD_UNUSED(katom);\ ++ CSTD_UNUSED(gpu_addr);\ ++ CSTD_UNUSED(info_val);\ ++ CSTD_NOP(0);\ ++ } while (0) ++ ++#define KBASE_TRACE_ADD(kbdev, code, subcode, ctx, katom, val)\ ++ do {\ ++ CSTD_UNUSED(kbdev);\ ++ CSTD_NOP(code);\ ++ CSTD_UNUSED(subcode);\ ++ CSTD_UNUSED(ctx);\ ++ CSTD_UNUSED(katom);\ ++ CSTD_UNUSED(val);\ ++ CSTD_NOP(0);\ ++ } while (0) ++ ++#define KBASE_TRACE_CLEAR(kbdev)\ ++ do {\ ++ CSTD_UNUSED(kbdev);\ ++ CSTD_NOP(0);\ ++ } while (0) ++#define KBASE_TRACE_DUMP(kbdev)\ ++ do {\ ++ CSTD_UNUSED(kbdev);\ ++ CSTD_NOP(0);\ ++ } while (0) ++#endif /* KBASE_TRACE_ENABLE */ ++/** PRIVATE - do not use directly. Use KBASE_TRACE_DUMP() instead */ ++void kbasep_trace_dump(struct kbase_device *kbdev); ++ ++#ifdef CONFIG_MALI_DEBUG ++/** ++ * kbase_set_driver_inactive - Force driver to go inactive ++ * @kbdev: Device pointer ++ * @inactive: true if driver should go inactive, false otherwise ++ * ++ * Forcing the driver inactive will cause all future IOCTLs to wait until the ++ * driver is made active again. This is intended solely for the use of tests ++ * which require that no jobs are running while the test executes. ++ */ ++void kbase_set_driver_inactive(struct kbase_device *kbdev, bool inactive); ++#endif /* CONFIG_MALI_DEBUG */ ++ ++ ++#if defined(CONFIG_DEBUG_FS) && !defined(CONFIG_MALI_NO_MALI) ++ ++/* kbase_io_history_init - initialize data struct for register access history ++ * ++ * @kbdev The register history to initialize ++ * @n The number of register accesses that the buffer could hold ++ * ++ * @return 0 if successfully initialized, failure otherwise ++ */ ++int kbase_io_history_init(struct kbase_io_history *h, u16 n); ++ ++/* kbase_io_history_term - uninit all resources for the register access history ++ * ++ * @h The register history to terminate ++ */ ++void kbase_io_history_term(struct kbase_io_history *h); ++ ++/* kbase_io_history_dump - print the register history to the kernel ring buffer ++ * ++ * @kbdev Pointer to kbase_device containing the register history to dump ++ */ ++void kbase_io_history_dump(struct kbase_device *kbdev); ++ ++/** ++ * kbase_io_history_resize - resize the register access history buffer. ++ * ++ * @h: Pointer to a valid register history to resize ++ * @new_size: Number of accesses the buffer could hold ++ * ++ * A successful resize will clear all recent register accesses. ++ * If resizing fails for any reason (e.g., could not allocate memory, invalid ++ * buffer size) then the original buffer will be kept intact. ++ * ++ * @return 0 if the buffer was resized, failure otherwise ++ */ ++int kbase_io_history_resize(struct kbase_io_history *h, u16 new_size); ++ ++#else /* CONFIG_DEBUG_FS */ ++ ++#define kbase_io_history_init(...) ((int)0) ++ ++#define kbase_io_history_term CSTD_NOP ++ ++#define kbase_io_history_dump CSTD_NOP ++ ++#define kbase_io_history_resize CSTD_NOP ++ ++#endif /* CONFIG_DEBUG_FS */ ++ ++ ++#endif ++ ++ ++ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_10969_workaround.c b/drivers/gpu/arm/midgard/mali_kbase_10969_workaround.c +new file mode 100755 +index 000000000000..fde0f8ff8582 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_10969_workaround.c +@@ -0,0 +1,209 @@ ++/* ++ * ++ * (C) COPYRIGHT 2013-2015, 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++#include ++#include ++#include ++ ++/* This function is used to solve an HW issue with single iterator GPUs. ++ * If a fragment job is soft-stopped on the edge of its bounding box, can happen that the ++ * restart index is out of bounds and the rerun causes a tile range fault. If this happens ++ * we try to clamp the restart index to a correct value and rerun the job. ++ */ ++/* Mask of X and Y coordinates for the coordinates words in the descriptors*/ ++#define X_COORDINATE_MASK 0x00000FFF ++#define Y_COORDINATE_MASK 0x0FFF0000 ++/* Max number of words needed from the fragment shader job descriptor */ ++#define JOB_HEADER_SIZE_IN_WORDS 10 ++#define JOB_HEADER_SIZE (JOB_HEADER_SIZE_IN_WORDS*sizeof(u32)) ++ ++/* Word 0: Status Word */ ++#define JOB_DESC_STATUS_WORD 0 ++/* Word 1: Restart Index */ ++#define JOB_DESC_RESTART_INDEX_WORD 1 ++/* Word 2: Fault address low word */ ++#define JOB_DESC_FAULT_ADDR_LOW_WORD 2 ++/* Word 8: Minimum Tile Coordinates */ ++#define FRAG_JOB_DESC_MIN_TILE_COORD_WORD 8 ++/* Word 9: Maximum Tile Coordinates */ ++#define FRAG_JOB_DESC_MAX_TILE_COORD_WORD 9 ++ ++int kbasep_10969_workaround_clamp_coordinates(struct kbase_jd_atom *katom) ++{ ++ struct device *dev = katom->kctx->kbdev->dev; ++ u32 clamped = 0; ++ struct kbase_va_region *region; ++ phys_addr_t *page_array; ++ u64 page_index; ++ u32 offset = katom->jc & (~PAGE_MASK); ++ u32 *page_1 = NULL; ++ u32 *page_2 = NULL; ++ u32 job_header[JOB_HEADER_SIZE_IN_WORDS]; ++ void *dst = job_header; ++ u32 minX, minY, maxX, maxY; ++ u32 restartX, restartY; ++ struct page *p; ++ u32 copy_size; ++ ++ dev_warn(dev, "Called TILE_RANGE_FAULT workaround clamping function.\n"); ++ if (!(katom->core_req & BASE_JD_REQ_FS)) ++ return 0; ++ ++ kbase_gpu_vm_lock(katom->kctx); ++ region = kbase_region_tracker_find_region_enclosing_address(katom->kctx, ++ katom->jc); ++ if (!region || (region->flags & KBASE_REG_FREE)) ++ goto out_unlock; ++ ++ page_array = kbase_get_cpu_phy_pages(region); ++ if (!page_array) ++ goto out_unlock; ++ ++ page_index = (katom->jc >> PAGE_SHIFT) - region->start_pfn; ++ ++ p = pfn_to_page(PFN_DOWN(page_array[page_index])); ++ ++ /* we need the first 10 words of the fragment shader job descriptor. ++ * We need to check that the offset + 10 words is less that the page ++ * size otherwise we need to load the next page. ++ * page_size_overflow will be equal to 0 in case the whole descriptor ++ * is within the page > 0 otherwise. ++ */ ++ copy_size = MIN(PAGE_SIZE - offset, JOB_HEADER_SIZE); ++ ++ page_1 = kmap_atomic(p); ++ ++ /* page_1 is a u32 pointer, offset is expressed in bytes */ ++ page_1 += offset>>2; ++ ++ kbase_sync_single_for_cpu(katom->kctx->kbdev, ++ kbase_dma_addr(p) + offset, ++ copy_size, DMA_BIDIRECTIONAL); ++ ++ memcpy(dst, page_1, copy_size); ++ ++ /* The data needed overflows page the dimension, ++ * need to map the subsequent page */ ++ if (copy_size < JOB_HEADER_SIZE) { ++ p = pfn_to_page(PFN_DOWN(page_array[page_index + 1])); ++ page_2 = kmap_atomic(p); ++ ++ kbase_sync_single_for_cpu(katom->kctx->kbdev, ++ kbase_dma_addr(p), ++ JOB_HEADER_SIZE - copy_size, DMA_BIDIRECTIONAL); ++ ++ memcpy(dst + copy_size, page_2, JOB_HEADER_SIZE - copy_size); ++ } ++ ++ /* We managed to correctly map one or two pages (in case of overflow) */ ++ /* Get Bounding Box data and restart index from fault address low word */ ++ minX = job_header[FRAG_JOB_DESC_MIN_TILE_COORD_WORD] & X_COORDINATE_MASK; ++ minY = job_header[FRAG_JOB_DESC_MIN_TILE_COORD_WORD] & Y_COORDINATE_MASK; ++ maxX = job_header[FRAG_JOB_DESC_MAX_TILE_COORD_WORD] & X_COORDINATE_MASK; ++ maxY = job_header[FRAG_JOB_DESC_MAX_TILE_COORD_WORD] & Y_COORDINATE_MASK; ++ restartX = job_header[JOB_DESC_FAULT_ADDR_LOW_WORD] & X_COORDINATE_MASK; ++ restartY = job_header[JOB_DESC_FAULT_ADDR_LOW_WORD] & Y_COORDINATE_MASK; ++ ++ dev_warn(dev, "Before Clamping:\n" ++ "Jobstatus: %08x\n" ++ "restartIdx: %08x\n" ++ "Fault_addr_low: %08x\n" ++ "minCoordsX: %08x minCoordsY: %08x\n" ++ "maxCoordsX: %08x maxCoordsY: %08x\n", ++ job_header[JOB_DESC_STATUS_WORD], ++ job_header[JOB_DESC_RESTART_INDEX_WORD], ++ job_header[JOB_DESC_FAULT_ADDR_LOW_WORD], ++ minX, minY, ++ maxX, maxY); ++ ++ /* Set the restart index to the one which generated the fault*/ ++ job_header[JOB_DESC_RESTART_INDEX_WORD] = ++ job_header[JOB_DESC_FAULT_ADDR_LOW_WORD]; ++ ++ if (restartX < minX) { ++ job_header[JOB_DESC_RESTART_INDEX_WORD] = (minX) | restartY; ++ dev_warn(dev, ++ "Clamping restart X index to minimum. %08x clamped to %08x\n", ++ restartX, minX); ++ clamped = 1; ++ } ++ if (restartY < minY) { ++ job_header[JOB_DESC_RESTART_INDEX_WORD] = (minY) | restartX; ++ dev_warn(dev, ++ "Clamping restart Y index to minimum. %08x clamped to %08x\n", ++ restartY, minY); ++ clamped = 1; ++ } ++ if (restartX > maxX) { ++ job_header[JOB_DESC_RESTART_INDEX_WORD] = (maxX) | restartY; ++ dev_warn(dev, ++ "Clamping restart X index to maximum. %08x clamped to %08x\n", ++ restartX, maxX); ++ clamped = 1; ++ } ++ if (restartY > maxY) { ++ job_header[JOB_DESC_RESTART_INDEX_WORD] = (maxY) | restartX; ++ dev_warn(dev, ++ "Clamping restart Y index to maximum. %08x clamped to %08x\n", ++ restartY, maxY); ++ clamped = 1; ++ } ++ ++ if (clamped) { ++ /* Reset the fault address low word ++ * and set the job status to STOPPED */ ++ job_header[JOB_DESC_FAULT_ADDR_LOW_WORD] = 0x0; ++ job_header[JOB_DESC_STATUS_WORD] = BASE_JD_EVENT_STOPPED; ++ dev_warn(dev, "After Clamping:\n" ++ "Jobstatus: %08x\n" ++ "restartIdx: %08x\n" ++ "Fault_addr_low: %08x\n" ++ "minCoordsX: %08x minCoordsY: %08x\n" ++ "maxCoordsX: %08x maxCoordsY: %08x\n", ++ job_header[JOB_DESC_STATUS_WORD], ++ job_header[JOB_DESC_RESTART_INDEX_WORD], ++ job_header[JOB_DESC_FAULT_ADDR_LOW_WORD], ++ minX, minY, ++ maxX, maxY); ++ ++ /* Flush CPU cache to update memory for future GPU reads*/ ++ memcpy(page_1, dst, copy_size); ++ p = pfn_to_page(PFN_DOWN(page_array[page_index])); ++ ++ kbase_sync_single_for_device(katom->kctx->kbdev, ++ kbase_dma_addr(p) + offset, ++ copy_size, DMA_TO_DEVICE); ++ ++ if (copy_size < JOB_HEADER_SIZE) { ++ memcpy(page_2, dst + copy_size, ++ JOB_HEADER_SIZE - copy_size); ++ p = pfn_to_page(PFN_DOWN(page_array[page_index + 1])); ++ ++ kbase_sync_single_for_device(katom->kctx->kbdev, ++ kbase_dma_addr(p), ++ JOB_HEADER_SIZE - copy_size, ++ DMA_TO_DEVICE); ++ } ++ } ++ if (copy_size < JOB_HEADER_SIZE) ++ kunmap_atomic(page_2); ++ ++ kunmap_atomic(page_1); ++ ++out_unlock: ++ kbase_gpu_vm_unlock(katom->kctx); ++ return clamped; ++} +diff --git a/drivers/gpu/arm/midgard/mali_kbase_10969_workaround.h b/drivers/gpu/arm/midgard/mali_kbase_10969_workaround.h +new file mode 100755 +index 000000000000..099a29861672 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_10969_workaround.h +@@ -0,0 +1,23 @@ ++/* ++ * ++ * (C) COPYRIGHT 2013-2014 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KBASE_10969_WORKAROUND_ ++#define _KBASE_10969_WORKAROUND_ ++ ++int kbasep_10969_workaround_clamp_coordinates(struct kbase_jd_atom *katom); ++ ++#endif /* _KBASE_10969_WORKAROUND_ */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_as_fault_debugfs.c b/drivers/gpu/arm/midgard/mali_kbase_as_fault_debugfs.c +new file mode 100755 +index 000000000000..f910fe970feb +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_as_fault_debugfs.c +@@ -0,0 +1,102 @@ ++/* ++ * ++ * (C) COPYRIGHT 2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include ++ ++#include ++#include ++ ++#ifdef CONFIG_DEBUG_FS ++#ifdef CONFIG_MALI_DEBUG ++ ++static int kbase_as_fault_read(struct seq_file *sfile, void *data) ++{ ++ uintptr_t as_no = (uintptr_t) sfile->private; ++ ++ struct list_head *entry; ++ const struct list_head *kbdev_list; ++ struct kbase_device *kbdev = NULL; ++ ++ kbdev_list = kbase_dev_list_get(); ++ ++ list_for_each(entry, kbdev_list) { ++ kbdev = list_entry(entry, struct kbase_device, entry); ++ ++ if(kbdev->debugfs_as_read_bitmap & (1ULL << as_no)) { ++ ++ /* don't show this one again until another fault occors */ ++ kbdev->debugfs_as_read_bitmap &= ~(1ULL << as_no); ++ ++ /* output the last page fault addr */ ++ seq_printf(sfile, "%llu\n", (u64) kbdev->as[as_no].fault_addr); ++ } ++ ++ } ++ ++ kbase_dev_list_put(kbdev_list); ++ ++ return 0; ++} ++ ++static int kbase_as_fault_debugfs_open(struct inode *in, struct file *file) ++{ ++ return single_open(file, kbase_as_fault_read , in->i_private); ++} ++ ++static const struct file_operations as_fault_fops = { ++ .open = kbase_as_fault_debugfs_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = single_release, ++}; ++ ++#endif /* CONFIG_MALI_DEBUG */ ++#endif /* CONFIG_DEBUG_FS */ ++ ++/* ++ * Initialize debugfs entry for each address space ++ */ ++void kbase_as_fault_debugfs_init(struct kbase_device *kbdev) ++{ ++#ifdef CONFIG_DEBUG_FS ++#ifdef CONFIG_MALI_DEBUG ++ uint i; ++ char as_name[64]; ++ struct dentry *debugfs_directory; ++ ++ kbdev->debugfs_as_read_bitmap = 0ULL; ++ ++ KBASE_DEBUG_ASSERT(kbdev->nr_hw_address_spaces); ++ KBASE_DEBUG_ASSERT(sizeof(kbdev->as[0].fault_addr) == sizeof(u64)); ++ ++ debugfs_directory = debugfs_create_dir("address_spaces", ++ kbdev->mali_debugfs_directory); ++ ++ if(debugfs_directory) { ++ for(i = 0; i < kbdev->nr_hw_address_spaces; i++) { ++ snprintf(as_name, ARRAY_SIZE(as_name), "as%u", i); ++ debugfs_create_file(as_name, S_IRUGO, ++ debugfs_directory, (void*) ((uintptr_t) i), &as_fault_fops); ++ } ++ } ++ else ++ dev_warn(kbdev->dev, "unable to create address_spaces debugfs directory"); ++ ++#endif /* CONFIG_MALI_DEBUG */ ++#endif /* CONFIG_DEBUG_FS */ ++ return; ++} +diff --git a/drivers/gpu/arm/midgard/mali_kbase_as_fault_debugfs.h b/drivers/gpu/arm/midgard/mali_kbase_as_fault_debugfs.h +new file mode 100755 +index 000000000000..3ed2248897fc +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_as_fault_debugfs.h +@@ -0,0 +1,45 @@ ++/* ++ * ++ * (C) COPYRIGHT 2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KBASE_AS_FAULT_DEBUG_FS_H ++#define _KBASE_AS_FAULT_DEBUG_FS_H ++ ++/** ++ * kbase_as_fault_debugfs_init() - Add debugfs files for reporting page faults ++ * ++ * @kbdev: Pointer to kbase_device ++ */ ++void kbase_as_fault_debugfs_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_as_fault_debugfs_new() - make the last fault available on debugfs ++ * ++ * @kbdev: Pointer to kbase_device ++ * @as_no: The address space the fault occurred on ++ */ ++static inline void ++kbase_as_fault_debugfs_new(struct kbase_device *kbdev, int as_no) ++{ ++#ifdef CONFIG_DEBUG_FS ++#ifdef CONFIG_MALI_DEBUG ++ kbdev->debugfs_as_read_bitmap |= (1ULL << as_no); ++#endif /* CONFIG_DEBUG_FS */ ++#endif /* CONFIG_MALI_DEBUG */ ++ return; ++} ++ ++#endif /*_KBASE_AS_FAULT_DEBUG_FS_H*/ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_cache_policy.c b/drivers/gpu/arm/midgard/mali_kbase_cache_policy.c +new file mode 100755 +index 000000000000..c67b3e97f1af +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_cache_policy.c +@@ -0,0 +1,64 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * Cache Policy API. ++ */ ++ ++#include "mali_kbase_cache_policy.h" ++ ++/* ++ * The output flags should be a combination of the following values: ++ * KBASE_REG_CPU_CACHED: CPU cache should be enabled. ++ */ ++u32 kbase_cache_enabled(u32 flags, u32 nr_pages) ++{ ++ u32 cache_flags = 0; ++ ++ CSTD_UNUSED(nr_pages); ++ ++ if (flags & BASE_MEM_CACHED_CPU) ++ cache_flags |= KBASE_REG_CPU_CACHED; ++ ++ return cache_flags; ++} ++ ++ ++void kbase_sync_single_for_device(struct kbase_device *kbdev, dma_addr_t handle, ++ size_t size, enum dma_data_direction dir) ++{ ++/* Check if kernel is using coherency with GPU */ ++#ifdef CONFIG_MALI_COH_KERN ++ if (kbdev->system_coherency == COHERENCY_ACE) ++ return; ++#endif /* CONFIG_MALI_COH_KERN */ ++ dma_sync_single_for_device(kbdev->dev, handle, size, dir); ++} ++ ++ ++void kbase_sync_single_for_cpu(struct kbase_device *kbdev, dma_addr_t handle, ++ size_t size, enum dma_data_direction dir) ++{ ++/* Check if kernel is using coherency with GPU */ ++#ifdef CONFIG_MALI_COH_KERN ++ if (kbdev->system_coherency == COHERENCY_ACE) ++ return; ++#endif /* CONFIG_MALI_COH_KERN */ ++ dma_sync_single_for_cpu(kbdev->dev, handle, size, dir); ++} +diff --git a/drivers/gpu/arm/midgard/mali_kbase_cache_policy.h b/drivers/gpu/arm/midgard/mali_kbase_cache_policy.h +new file mode 100755 +index 000000000000..0c18bdb357b0 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_cache_policy.h +@@ -0,0 +1,45 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2013, 2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * Cache Policy API. ++ */ ++ ++#ifndef _KBASE_CACHE_POLICY_H_ ++#define _KBASE_CACHE_POLICY_H_ ++ ++#include "mali_kbase.h" ++#include "mali_base_kernel.h" ++ ++/** ++ * kbase_cache_enabled - Choose the cache policy for a specific region ++ * @flags: flags describing attributes of the region ++ * @nr_pages: total number of pages (backed or not) for the region ++ * ++ * Tells whether the CPU and GPU caches should be enabled or not for a specific ++ * region. ++ * This function can be modified to customize the cache policy depending on the ++ * flags and size of the region. ++ * ++ * Return: a combination of %KBASE_REG_CPU_CACHED and %KBASE_REG_GPU_CACHED ++ * depending on the cache policy ++ */ ++u32 kbase_cache_enabled(u32 flags, u32 nr_pages); ++ ++#endif /* _KBASE_CACHE_POLICY_H_ */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_config.c b/drivers/gpu/arm/midgard/mali_kbase_config.c +new file mode 100755 +index 000000000000..fb615ae02ead +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_config.c +@@ -0,0 +1,51 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#include ++#include ++#include ++ ++int kbasep_platform_device_init(struct kbase_device *kbdev) ++{ ++ struct kbase_platform_funcs_conf *platform_funcs_p; ++ ++ platform_funcs_p = (struct kbase_platform_funcs_conf *)PLATFORM_FUNCS; ++ if (platform_funcs_p && platform_funcs_p->platform_init_func) ++ return platform_funcs_p->platform_init_func(kbdev); ++ ++ return 0; ++} ++ ++void kbasep_platform_device_term(struct kbase_device *kbdev) ++{ ++ struct kbase_platform_funcs_conf *platform_funcs_p; ++ ++ platform_funcs_p = (struct kbase_platform_funcs_conf *)PLATFORM_FUNCS; ++ if (platform_funcs_p && platform_funcs_p->platform_term_func) ++ platform_funcs_p->platform_term_func(kbdev); ++} ++ ++int kbase_cpuprops_get_default_clock_speed(u32 * const clock_speed) ++{ ++ KBASE_DEBUG_ASSERT(NULL != clock_speed); ++ ++ *clock_speed = 100; ++ return 0; ++} ++ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_config.h b/drivers/gpu/arm/midgard/mali_kbase_config.h +new file mode 100755 +index 000000000000..356d52bcd774 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_config.h +@@ -0,0 +1,345 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/** ++ * @file mali_kbase_config.h ++ * Configuration API and Attributes for KBase ++ */ ++ ++#ifndef _KBASE_CONFIG_H_ ++#define _KBASE_CONFIG_H_ ++ ++#include ++ ++#include ++#include ++ ++/** ++ * @addtogroup base_api ++ * @{ ++ */ ++ ++/** ++ * @addtogroup base_kbase_api ++ * @{ ++ */ ++ ++/** ++ * @addtogroup kbase_config Configuration API and Attributes ++ * @{ ++ */ ++ ++#include ++ ++/* Forward declaration of struct kbase_device */ ++struct kbase_device; ++ ++/** ++ * kbase_platform_funcs_conf - Specifies platform init/term function pointers ++ * ++ * Specifies the functions pointers for platform specific initialization and ++ * termination. By default no functions are required. No additional platform ++ * specific control is necessary. ++ */ ++struct kbase_platform_funcs_conf { ++ /** ++ * platform_init_func - platform specific init function pointer ++ * @kbdev - kbase_device pointer ++ * ++ * Returns 0 on success, negative error code otherwise. ++ * ++ * Function pointer for platform specific initialization or NULL if no ++ * initialization function is required. At the point this the GPU is ++ * not active and its power and clocks are in unknown (platform specific ++ * state) as kbase doesn't yet have control of power and clocks. ++ * ++ * The platform specific private pointer kbase_device::platform_context ++ * can be accessed (and possibly initialized) in here. ++ */ ++ int (*platform_init_func)(struct kbase_device *kbdev); ++ /** ++ * platform_term_func - platform specific termination function pointer ++ * @kbdev - kbase_device pointer ++ * ++ * Function pointer for platform specific termination or NULL if no ++ * termination function is required. At the point this the GPU will be ++ * idle but still powered and clocked. ++ * ++ * The platform specific private pointer kbase_device::platform_context ++ * can be accessed (and possibly terminated) in here. ++ */ ++ void (*platform_term_func)(struct kbase_device *kbdev); ++}; ++ ++/* ++ * @brief Specifies the callbacks for power management ++ * ++ * By default no callbacks will be made and the GPU must not be powered off. ++ */ ++struct kbase_pm_callback_conf { ++ /** Callback for when the GPU is idle and the power to it can be switched off. ++ * ++ * The system integrator can decide whether to either do nothing, just switch off ++ * the clocks to the GPU, or to completely power down the GPU. ++ * The platform specific private pointer kbase_device::platform_context can be accessed and modified in here. It is the ++ * platform \em callbacks responsibility to initialize and terminate this pointer if used (see @ref kbase_platform_funcs_conf). ++ */ ++ void (*power_off_callback)(struct kbase_device *kbdev); ++ ++ /** Callback for when the GPU is about to become active and power must be supplied. ++ * ++ * This function must not return until the GPU is powered and clocked sufficiently for register access to ++ * succeed. The return value specifies whether the GPU was powered down since the call to power_off_callback. ++ * If the GPU state has been lost then this function must return 1, otherwise it should return 0. ++ * The platform specific private pointer kbase_device::platform_context can be accessed and modified in here. It is the ++ * platform \em callbacks responsibility to initialize and terminate this pointer if used (see @ref kbase_platform_funcs_conf). ++ * ++ * The return value of the first call to this function is ignored. ++ * ++ * @return 1 if the GPU state may have been lost, 0 otherwise. ++ */ ++ int (*power_on_callback)(struct kbase_device *kbdev); ++ ++ /** Callback for when the system is requesting a suspend and GPU power ++ * must be switched off. ++ * ++ * Note that if this callback is present, then this may be called ++ * without a preceding call to power_off_callback. Therefore this ++ * callback must be able to take any action that might otherwise happen ++ * in power_off_callback. ++ * ++ * The platform specific private pointer kbase_device::platform_context ++ * can be accessed and modified in here. It is the platform \em ++ * callbacks responsibility to initialize and terminate this pointer if ++ * used (see @ref kbase_platform_funcs_conf). ++ */ ++ void (*power_suspend_callback)(struct kbase_device *kbdev); ++ ++ /** Callback for when the system is resuming from a suspend and GPU ++ * power must be switched on. ++ * ++ * Note that if this callback is present, then this may be called ++ * without a following call to power_on_callback. Therefore this ++ * callback must be able to take any action that might otherwise happen ++ * in power_on_callback. ++ * ++ * The platform specific private pointer kbase_device::platform_context ++ * can be accessed and modified in here. It is the platform \em ++ * callbacks responsibility to initialize and terminate this pointer if ++ * used (see @ref kbase_platform_funcs_conf). ++ */ ++ void (*power_resume_callback)(struct kbase_device *kbdev); ++ ++ /** Callback for handling runtime power management initialization. ++ * ++ * The runtime power management callbacks @ref power_runtime_off_callback and @ref power_runtime_on_callback ++ * will become active from calls made to the OS from within this function. ++ * The runtime calls can be triggered by calls from @ref power_off_callback and @ref power_on_callback. ++ * Note: for linux the kernel must have CONFIG_PM_RUNTIME enabled to use this feature. ++ * ++ * @return 0 on success, else int error code. ++ */ ++ int (*power_runtime_init_callback)(struct kbase_device *kbdev); ++ ++ /** Callback for handling runtime power management termination. ++ * ++ * The runtime power management callbacks @ref power_runtime_off_callback and @ref power_runtime_on_callback ++ * should no longer be called by the OS on completion of this function. ++ * Note: for linux the kernel must have CONFIG_PM_RUNTIME enabled to use this feature. ++ */ ++ void (*power_runtime_term_callback)(struct kbase_device *kbdev); ++ ++ /** Callback for runtime power-off power management callback ++ * ++ * For linux this callback will be called by the kernel runtime_suspend callback. ++ * Note: for linux the kernel must have CONFIG_PM_RUNTIME enabled to use this feature. ++ * ++ * @return 0 on success, else OS error code. ++ */ ++ void (*power_runtime_off_callback)(struct kbase_device *kbdev); ++ ++ /** Callback for runtime power-on power management callback ++ * ++ * For linux this callback will be called by the kernel runtime_resume callback. ++ * Note: for linux the kernel must have CONFIG_PM_RUNTIME enabled to use this feature. ++ */ ++ int (*power_runtime_on_callback)(struct kbase_device *kbdev); ++ ++ /* ++ * Optional callback for checking if GPU can be suspended when idle ++ * ++ * This callback will be called by the runtime power management core ++ * when the reference count goes to 0 to provide notification that the ++ * GPU now seems idle. ++ * ++ * If this callback finds that the GPU can't be powered off, or handles ++ * suspend by powering off directly or queueing up a power off, a ++ * non-zero value must be returned to prevent the runtime PM core from ++ * also triggering a suspend. ++ * ++ * Returning 0 will cause the runtime PM core to conduct a regular ++ * autosuspend. ++ * ++ * This callback is optional and if not provided regular autosuspend ++ * will be triggered. ++ * ++ * Note: The Linux kernel must have CONFIG_PM_RUNTIME enabled to use ++ * this feature. ++ * ++ * Return 0 if GPU can be suspended, positive value if it can not be ++ * suspeneded by runtime PM, else OS error code ++ */ ++ int (*power_runtime_idle_callback)(struct kbase_device *kbdev); ++}; ++ ++/** ++ * kbase_cpuprops_get_default_clock_speed - default for CPU_SPEED_FUNC ++ * @clock_speed - see kbase_cpu_clk_speed_func for details on the parameters ++ * ++ * Returns 0 on success, negative error code otherwise. ++ * ++ * Default implementation of CPU_SPEED_FUNC. This function sets clock_speed ++ * to 100, so will be an underestimate for any real system. ++ */ ++int kbase_cpuprops_get_default_clock_speed(u32 * const clock_speed); ++ ++/** ++ * kbase_cpu_clk_speed_func - Type of the function pointer for CPU_SPEED_FUNC ++ * @param clock_speed - pointer to store the current CPU clock speed in MHz ++ * ++ * Returns 0 on success, otherwise negative error code. ++ * ++ * This is mainly used to implement OpenCL's clGetDeviceInfo(). ++ */ ++typedef int (*kbase_cpu_clk_speed_func) (u32 *clock_speed); ++ ++/** ++ * kbase_gpu_clk_speed_func - Type of the function pointer for GPU_SPEED_FUNC ++ * @param clock_speed - pointer to store the current GPU clock speed in MHz ++ * ++ * Returns 0 on success, otherwise negative error code. ++ * When an error is returned the caller assumes maximum GPU speed stored in ++ * gpu_freq_khz_max. ++ * ++ * If the system timer is not available then this function is required ++ * for the OpenCL queue profiling to return correct timing information. ++ * ++ */ ++typedef int (*kbase_gpu_clk_speed_func) (u32 *clock_speed); ++ ++#ifdef CONFIG_OF ++struct kbase_platform_config { ++}; ++#else ++ ++/* ++ * @brief Specifies start and end of I/O memory region. ++ */ ++struct kbase_io_memory_region { ++ u64 start; ++ u64 end; ++}; ++ ++/* ++ * @brief Specifies I/O related resources like IRQs and memory region for I/O operations. ++ */ ++struct kbase_io_resources { ++ u32 job_irq_number; ++ u32 mmu_irq_number; ++ u32 gpu_irq_number; ++ struct kbase_io_memory_region io_memory_region; ++}; ++ ++struct kbase_platform_config { ++ const struct kbase_io_resources *io_resources; ++}; ++ ++#endif /* CONFIG_OF */ ++ ++/** ++ * @brief Gets the pointer to platform config. ++ * ++ * @return Pointer to the platform config ++ */ ++struct kbase_platform_config *kbase_get_platform_config(void); ++ ++/** ++ * kbasep_platform_device_init: - Platform specific call to initialize hardware ++ * @kbdev: kbase device pointer ++ * ++ * Function calls a platform defined routine if specified in the configuration ++ * attributes. The routine can initialize any hardware and context state that ++ * is required for the GPU block to function. ++ * ++ * Return: 0 if no errors have been found in the config. ++ * Negative error code otherwise. ++ */ ++int kbasep_platform_device_init(struct kbase_device *kbdev); ++ ++/** ++ * kbasep_platform_device_term - Platform specific call to terminate hardware ++ * @kbdev: Kbase device pointer ++ * ++ * Function calls a platform defined routine if specified in the configuration ++ * attributes. The routine can destroy any platform specific context state and ++ * shut down any hardware functionality that are outside of the Power Management ++ * callbacks. ++ * ++ */ ++void kbasep_platform_device_term(struct kbase_device *kbdev); ++ ++ ++/** ++ * kbase_platform_early_init - Early initialisation of the platform code ++ * ++ * This function will be called when the module is loaded to perform any ++ * early initialisation required by the platform code. Such as reading ++ * platform specific device tree entries for the GPU. ++ * ++ * Return: 0 for success, any other fail causes module initialisation to fail ++ */ ++int kbase_platform_early_init(void); ++ ++#ifndef CONFIG_OF ++#ifdef CONFIG_MALI_PLATFORM_FAKE ++/** ++ * kbase_platform_fake_register - Register a platform device for the GPU ++ * ++ * This can be used to register a platform device on systems where device tree ++ * is not enabled and the platform initialisation code in the kernel doesn't ++ * create the GPU device. Where possible device tree should be used instead. ++ * ++ * Return: 0 for success, any other fail causes module initialisation to fail ++ */ ++int kbase_platform_fake_register(void); ++ ++/** ++ * kbase_platform_fake_unregister - Unregister a fake platform device ++ * ++ * Unregister the platform device created with kbase_platform_fake_register() ++ */ ++void kbase_platform_fake_unregister(void); ++#endif ++#endif ++ ++ /** @} *//* end group kbase_config */ ++ /** @} *//* end group base_kbase_api */ ++ /** @} *//* end group base_api */ ++ ++#endif /* _KBASE_CONFIG_H_ */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_config_defaults.h b/drivers/gpu/arm/midgard/mali_kbase_config_defaults.h +new file mode 100755 +index 000000000000..1cf44b3500cf +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_config_defaults.h +@@ -0,0 +1,227 @@ ++/* ++ * ++ * (C) COPYRIGHT 2013-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/** ++ * @file mali_kbase_config_defaults.h ++ * ++ * Default values for configuration settings ++ * ++ */ ++ ++#ifndef _KBASE_CONFIG_DEFAULTS_H_ ++#define _KBASE_CONFIG_DEFAULTS_H_ ++ ++/* Include mandatory definitions per platform */ ++#include ++ ++/** ++* Boolean indicating whether the driver is configured to be secure at ++* a potential loss of performance. ++* ++* This currently affects only r0p0-15dev0 HW and earlier. ++* ++* On r0p0-15dev0 HW and earlier, there are tradeoffs between security and ++* performance: ++* ++* - When this is set to true, the driver remains fully secure, ++* but potentially loses performance compared with setting this to ++* false. ++* - When set to false, the driver is open to certain security ++* attacks. ++* ++* From r0p0-00rel0 and onwards, there is no security loss by setting ++* this to false, and no performance loss by setting it to ++* true. ++*/ ++#define DEFAULT_SECURE_BUT_LOSS_OF_PERFORMANCE false ++ ++enum { ++ /** ++ * Use unrestricted Address ID width on the AXI bus. ++ */ ++ KBASE_AID_32 = 0x0, ++ ++ /** ++ * Restrict GPU to a half of maximum Address ID count. ++ * This will reduce performance, but reduce bus load due to GPU. ++ */ ++ KBASE_AID_16 = 0x3, ++ ++ /** ++ * Restrict GPU to a quarter of maximum Address ID count. ++ * This will reduce performance, but reduce bus load due to GPU. ++ */ ++ KBASE_AID_8 = 0x2, ++ ++ /** ++ * Restrict GPU to an eighth of maximum Address ID count. ++ * This will reduce performance, but reduce bus load due to GPU. ++ */ ++ KBASE_AID_4 = 0x1 ++}; ++ ++/** ++ * Default setting for read Address ID limiting on AXI bus. ++ * ++ * Attached value: u32 register value ++ * KBASE_AID_32 - use the full 32 IDs (5 ID bits) ++ * KBASE_AID_16 - use 16 IDs (4 ID bits) ++ * KBASE_AID_8 - use 8 IDs (3 ID bits) ++ * KBASE_AID_4 - use 4 IDs (2 ID bits) ++ * Default value: KBASE_AID_32 (no limit). Note hardware implementation ++ * may limit to a lower value. ++ */ ++#define DEFAULT_ARID_LIMIT KBASE_AID_32 ++ ++/** ++ * Default setting for write Address ID limiting on AXI. ++ * ++ * Attached value: u32 register value ++ * KBASE_AID_32 - use the full 32 IDs (5 ID bits) ++ * KBASE_AID_16 - use 16 IDs (4 ID bits) ++ * KBASE_AID_8 - use 8 IDs (3 ID bits) ++ * KBASE_AID_4 - use 4 IDs (2 ID bits) ++ * Default value: KBASE_AID_32 (no limit). Note hardware implementation ++ * may limit to a lower value. ++ */ ++#define DEFAULT_AWID_LIMIT KBASE_AID_32 ++ ++/** ++ * Default UMP device mapping. A UMP_DEVICE__SHIFT value which ++ * defines which UMP device this GPU should be mapped to. ++ */ ++#define DEFAULT_UMP_GPU_DEVICE_SHIFT UMP_DEVICE_Z_SHIFT ++ ++/* ++ * Default period for DVFS sampling ++ */ ++// #define DEFAULT_PM_DVFS_PERIOD 100 /* 100ms */ ++#define DEFAULT_PM_DVFS_PERIOD 20 /* 20 ms */ ++ ++/* ++ * Power Management poweroff tick granuality. This is in nanoseconds to ++ * allow HR timer support. ++ * ++ * On each scheduling tick, the power manager core may decide to: ++ * -# Power off one or more shader cores ++ * -# Power off the entire GPU ++ */ ++#define DEFAULT_PM_GPU_POWEROFF_TICK_NS (400000) /* 400us */ ++ ++/* ++ * Power Manager number of ticks before shader cores are powered off ++ */ ++#define DEFAULT_PM_POWEROFF_TICK_SHADER (2) /* 400-800us */ ++ ++/* ++ * Power Manager number of ticks before GPU is powered off ++ */ ++#define DEFAULT_PM_POWEROFF_TICK_GPU (2) /* 400-800us */ ++ ++/* ++ * Default scheduling tick granuality ++ */ ++#define DEFAULT_JS_SCHEDULING_PERIOD_NS (100000000u) /* 100ms */ ++ ++/* ++ * Default minimum number of scheduling ticks before jobs are soft-stopped. ++ * ++ * This defines the time-slice for a job (which may be different from that of a ++ * context) ++ */ ++#define DEFAULT_JS_SOFT_STOP_TICKS (1) /* 100ms-200ms */ ++ ++/* ++ * Default minimum number of scheduling ticks before CL jobs are soft-stopped. ++ */ ++#define DEFAULT_JS_SOFT_STOP_TICKS_CL (1) /* 100ms-200ms */ ++ ++/* ++ * Default minimum number of scheduling ticks before jobs are hard-stopped ++ */ ++#define DEFAULT_JS_HARD_STOP_TICKS_SS (50) /* 5s */ ++#define DEFAULT_JS_HARD_STOP_TICKS_SS_8408 (300) /* 30s */ ++ ++/* ++ * Default minimum number of scheduling ticks before CL jobs are hard-stopped. ++ */ ++#define DEFAULT_JS_HARD_STOP_TICKS_CL (50) /* 5s */ ++ ++/* ++ * Default minimum number of scheduling ticks before jobs are hard-stopped ++ * during dumping ++ */ ++#define DEFAULT_JS_HARD_STOP_TICKS_DUMPING (15000) /* 1500s */ ++ ++/* ++ * Default timeout for some software jobs, after which the software event wait ++ * jobs will be cancelled. ++ */ ++#define DEFAULT_JS_SOFT_JOB_TIMEOUT (3000) /* 3s */ ++ ++/* ++ * Default minimum number of scheduling ticks before the GPU is reset to clear a ++ * "stuck" job ++ */ ++#define DEFAULT_JS_RESET_TICKS_SS (55) /* 5.5s */ ++#define DEFAULT_JS_RESET_TICKS_SS_8408 (450) /* 45s */ ++ ++/* ++ * Default minimum number of scheduling ticks before the GPU is reset to clear a ++ * "stuck" CL job. ++ */ ++#define DEFAULT_JS_RESET_TICKS_CL (55) /* 5.5s */ ++ ++/* ++ * Default minimum number of scheduling ticks before the GPU is reset to clear a ++ * "stuck" job during dumping. ++ */ ++#define DEFAULT_JS_RESET_TICKS_DUMPING (15020) /* 1502s */ ++ ++/* ++ * Default number of milliseconds given for other jobs on the GPU to be ++ * soft-stopped when the GPU needs to be reset. ++ */ ++#define DEFAULT_RESET_TIMEOUT_MS (3000) /* 3s */ ++ ++/* ++ * Default timeslice that a context is scheduled in for, in nanoseconds. ++ * ++ * When a context has used up this amount of time across its jobs, it is ++ * scheduled out to let another run. ++ * ++ * @note the resolution is nanoseconds (ns) here, because that's the format ++ * often used by the OS. ++ */ ++#define DEFAULT_JS_CTX_TIMESLICE_NS (50000000) /* 50ms */ ++ ++/* ++ * Perform GPU power down using only platform specific code, skipping DDK power ++ * management. ++ * ++ * If this is non-zero then kbase will avoid powering down shader cores, the ++ * tiler, and the L2 cache, instead just powering down the entire GPU through ++ * platform specific code. This may be required for certain platform ++ * integrations. ++ * ++ * Note that as this prevents kbase from powering down shader cores, this limits ++ * the available power policies to coarse_demand and always_on. ++ */ ++#define PLATFORM_POWER_DOWN_ONLY (1) ++ ++#endif /* _KBASE_CONFIG_DEFAULTS_H_ */ ++ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_context.c b/drivers/gpu/arm/midgard/mali_kbase_context.c +new file mode 100755 +index 000000000000..6338a7e221ed +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_context.c +@@ -0,0 +1,342 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * Base kernel context APIs ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++ ++/** ++ * kbase_create_context() - Create a kernel base context. ++ * @kbdev: Kbase device ++ * @is_compat: Force creation of a 32-bit context ++ * ++ * Allocate and init a kernel base context. ++ * ++ * Return: new kbase context ++ */ ++struct kbase_context * ++kbase_create_context(struct kbase_device *kbdev, bool is_compat) ++{ ++ struct kbase_context *kctx; ++ int err; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ /* zero-inited as lot of code assume it's zero'ed out on create */ ++ kctx = vzalloc(sizeof(*kctx)); ++ ++ if (!kctx) ++ goto out; ++ ++ /* creating a context is considered a disjoint event */ ++ kbase_disjoint_event(kbdev); ++ ++ kctx->kbdev = kbdev; ++ kctx->as_nr = KBASEP_AS_NR_INVALID; ++ atomic_set(&kctx->refcount, 0); ++ if (is_compat) ++ kbase_ctx_flag_set(kctx, KCTX_COMPAT); ++#ifdef CONFIG_MALI_TRACE_TIMELINE ++ kctx->timeline.owner_tgid = task_tgid_nr(current); ++#endif ++ atomic_set(&kctx->setup_complete, 0); ++ atomic_set(&kctx->setup_in_progress, 0); ++ spin_lock_init(&kctx->mm_update_lock); ++ kctx->process_mm = NULL; ++ atomic_set(&kctx->nonmapped_pages, 0); ++ kctx->slots_pullable = 0; ++ kctx->tgid = current->tgid; ++ kctx->pid = current->pid; ++ ++ err = kbase_mem_pool_init(&kctx->mem_pool, ++ kbdev->mem_pool_max_size_default, ++ kctx->kbdev, &kbdev->mem_pool); ++ if (err) ++ goto free_kctx; ++ ++ err = kbase_mem_evictable_init(kctx); ++ if (err) ++ goto free_pool; ++ ++ atomic_set(&kctx->used_pages, 0); ++ ++ err = kbase_jd_init(kctx); ++ if (err) ++ goto deinit_evictable; ++ ++ err = kbasep_js_kctx_init(kctx); ++ if (err) ++ goto free_jd; /* safe to call kbasep_js_kctx_term in this case */ ++ ++ err = kbase_event_init(kctx); ++ if (err) ++ goto free_jd; ++ ++ atomic_set(&kctx->drain_pending, 0); ++ ++ mutex_init(&kctx->reg_lock); ++ ++ INIT_LIST_HEAD(&kctx->waiting_soft_jobs); ++ spin_lock_init(&kctx->waiting_soft_jobs_lock); ++#ifdef CONFIG_KDS ++ INIT_LIST_HEAD(&kctx->waiting_kds_resource); ++#endif ++ err = kbase_dma_fence_init(kctx); ++ if (err) ++ goto free_event; ++ ++ err = kbase_mmu_init(kctx); ++ if (err) ++ goto term_dma_fence; ++ ++ do { ++ err = kbase_mem_pool_grow(&kctx->mem_pool, ++ MIDGARD_MMU_BOTTOMLEVEL); ++ if (err) ++ goto pgd_no_mem; ++ ++ mutex_lock(&kctx->mmu_lock); ++ kctx->pgd = kbase_mmu_alloc_pgd(kctx); ++ mutex_unlock(&kctx->mmu_lock); ++ } while (!kctx->pgd); ++ ++ kctx->aliasing_sink_page = kbase_mem_alloc_page(kctx->kbdev); ++ if (!kctx->aliasing_sink_page) ++ goto no_sink_page; ++ ++ init_waitqueue_head(&kctx->event_queue); ++ ++ kctx->cookies = KBASE_COOKIE_MASK; ++ ++ /* Make sure page 0 is not used... */ ++ err = kbase_region_tracker_init(kctx); ++ if (err) ++ goto no_region_tracker; ++ ++ err = kbase_sticky_resource_init(kctx); ++ if (err) ++ goto no_sticky; ++ ++ err = kbase_jit_init(kctx); ++ if (err) ++ goto no_jit; ++#ifdef CONFIG_GPU_TRACEPOINTS ++ atomic_set(&kctx->jctx.work_id, 0); ++#endif ++#ifdef CONFIG_MALI_TRACE_TIMELINE ++ atomic_set(&kctx->timeline.jd_atoms_in_flight, 0); ++#endif ++ ++ kctx->id = atomic_add_return(1, &(kbdev->ctx_num)) - 1; ++ ++ mutex_init(&kctx->vinstr_cli_lock); ++ ++ timer_setup(&kctx->soft_job_timeout, ++ kbasep_soft_job_timeout_worker, ++ 0); ++ ++ return kctx; ++ ++no_jit: ++ kbase_gpu_vm_lock(kctx); ++ kbase_sticky_resource_term(kctx); ++ kbase_gpu_vm_unlock(kctx); ++no_sticky: ++ kbase_region_tracker_term(kctx); ++no_region_tracker: ++ kbase_mem_pool_free(&kctx->mem_pool, kctx->aliasing_sink_page, false); ++no_sink_page: ++ /* VM lock needed for the call to kbase_mmu_free_pgd */ ++ kbase_gpu_vm_lock(kctx); ++ kbase_mmu_free_pgd(kctx); ++ kbase_gpu_vm_unlock(kctx); ++pgd_no_mem: ++ kbase_mmu_term(kctx); ++term_dma_fence: ++ kbase_dma_fence_term(kctx); ++free_event: ++ kbase_event_cleanup(kctx); ++free_jd: ++ /* Safe to call this one even when didn't initialize (assuming kctx was sufficiently zeroed) */ ++ kbasep_js_kctx_term(kctx); ++ kbase_jd_exit(kctx); ++deinit_evictable: ++ kbase_mem_evictable_deinit(kctx); ++free_pool: ++ kbase_mem_pool_term(&kctx->mem_pool); ++free_kctx: ++ vfree(kctx); ++out: ++ return NULL; ++} ++KBASE_EXPORT_SYMBOL(kbase_create_context); ++ ++static void kbase_reg_pending_dtor(struct kbase_va_region *reg) ++{ ++ dev_dbg(reg->kctx->kbdev->dev, "Freeing pending unmapped region\n"); ++ kbase_mem_phy_alloc_put(reg->cpu_alloc); ++ kbase_mem_phy_alloc_put(reg->gpu_alloc); ++ kfree(reg); ++} ++ ++/** ++ * kbase_destroy_context - Destroy a kernel base context. ++ * @kctx: Context to destroy ++ * ++ * Calls kbase_destroy_os_context() to free OS specific structures. ++ * Will release all outstanding regions. ++ */ ++void kbase_destroy_context(struct kbase_context *kctx) ++{ ++ struct kbase_device *kbdev; ++ int pages; ++ unsigned long pending_regions_to_clean; ++ unsigned long flags; ++ ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ ++ kbdev = kctx->kbdev; ++ KBASE_DEBUG_ASSERT(NULL != kbdev); ++ ++ KBASE_TRACE_ADD(kbdev, CORE_CTX_DESTROY, kctx, NULL, 0u, 0u); ++ ++ /* Ensure the core is powered up for the destroy process */ ++ /* A suspend won't happen here, because we're in a syscall from a userspace ++ * thread. */ ++ kbase_pm_context_active(kbdev); ++ ++ kbase_jd_zap_context(kctx); ++ ++#ifdef CONFIG_DEBUG_FS ++ /* Removing the rest of the debugfs entries here as we want to keep the ++ * atom debugfs interface alive until all atoms have completed. This ++ * is useful for debugging hung contexts. */ ++ debugfs_remove_recursive(kctx->kctx_dentry); ++#endif ++ ++ kbase_event_cleanup(kctx); ++ ++ /* ++ * JIT must be terminated before the code below as it must be called ++ * without the region lock being held. ++ * The code above ensures no new JIT allocations can be made by ++ * by the time we get to this point of context tear down. ++ */ ++ kbase_jit_term(kctx); ++ ++ kbase_gpu_vm_lock(kctx); ++ ++ kbase_sticky_resource_term(kctx); ++ ++ /* MMU is disabled as part of scheduling out the context */ ++ kbase_mmu_free_pgd(kctx); ++ ++ /* drop the aliasing sink page now that it can't be mapped anymore */ ++ kbase_mem_pool_free(&kctx->mem_pool, kctx->aliasing_sink_page, false); ++ ++ /* free pending region setups */ ++ pending_regions_to_clean = (~kctx->cookies) & KBASE_COOKIE_MASK; ++ while (pending_regions_to_clean) { ++ unsigned int cookie = __ffs(pending_regions_to_clean); ++ ++ BUG_ON(!kctx->pending_regions[cookie]); ++ ++ kbase_reg_pending_dtor(kctx->pending_regions[cookie]); ++ ++ kctx->pending_regions[cookie] = NULL; ++ pending_regions_to_clean &= ~(1UL << cookie); ++ } ++ ++ kbase_region_tracker_term(kctx); ++ kbase_gpu_vm_unlock(kctx); ++ ++ /* Safe to call this one even when didn't initialize (assuming kctx was sufficiently zeroed) */ ++ kbasep_js_kctx_term(kctx); ++ ++ kbase_jd_exit(kctx); ++ ++ kbase_pm_context_idle(kbdev); ++ ++ kbase_dma_fence_term(kctx); ++ ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, flags); ++ kbase_ctx_sched_remove_ctx(kctx); ++ spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, flags); ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ ++ kbase_mmu_term(kctx); ++ ++ pages = atomic_read(&kctx->used_pages); ++ if (pages != 0) ++ dev_warn(kbdev->dev, "%s: %d pages in use!\n", __func__, pages); ++ ++ kbase_mem_evictable_deinit(kctx); ++ kbase_mem_pool_term(&kctx->mem_pool); ++ WARN_ON(atomic_read(&kctx->nonmapped_pages) != 0); ++ ++ vfree(kctx); ++} ++KBASE_EXPORT_SYMBOL(kbase_destroy_context); ++ ++/** ++ * kbase_context_set_create_flags - Set creation flags on a context ++ * @kctx: Kbase context ++ * @flags: Flags to set ++ * ++ * Return: 0 on success ++ */ ++int kbase_context_set_create_flags(struct kbase_context *kctx, u32 flags) ++{ ++ int err = 0; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ unsigned long irq_flags; ++ ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ /* Validate flags */ ++ if (flags != (flags & BASE_CONTEXT_CREATE_KERNEL_FLAGS)) { ++ err = -EINVAL; ++ goto out; ++ } ++ ++ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); ++ spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, irq_flags); ++ ++ /* Translate the flags */ ++ if ((flags & BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED) == 0) ++ kbase_ctx_flag_clear(kctx, KCTX_SUBMIT_DISABLED); ++ ++ /* Latch the initial attributes into the Job Scheduler */ ++ kbasep_js_ctx_attr_set_initial_attrs(kctx->kbdev, kctx); ++ ++ spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, irq_flags); ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ out: ++ return err; ++} ++KBASE_EXPORT_SYMBOL(kbase_context_set_create_flags); +diff --git a/drivers/gpu/arm/midgard/mali_kbase_context.h b/drivers/gpu/arm/midgard/mali_kbase_context.h +new file mode 100755 +index 000000000000..a3f5bb0ce0da +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_context.h +@@ -0,0 +1,90 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KBASE_CONTEXT_H_ ++#define _KBASE_CONTEXT_H_ ++ ++#include ++ ++ ++int kbase_context_set_create_flags(struct kbase_context *kctx, u32 flags); ++ ++/** ++ * kbase_ctx_flag - Check if @flag is set on @kctx ++ * @kctx: Pointer to kbase context to check ++ * @flag: Flag to check ++ * ++ * Return: true if @flag is set on @kctx, false if not. ++ */ ++static inline bool kbase_ctx_flag(struct kbase_context *kctx, ++ enum kbase_context_flags flag) ++{ ++ return atomic_read(&kctx->flags) & flag; ++} ++ ++/** ++ * kbase_ctx_flag_clear - Clear @flag on @kctx ++ * @kctx: Pointer to kbase context ++ * @flag: Flag to clear ++ * ++ * Clear the @flag on @kctx. This is done atomically, so other flags being ++ * cleared or set at the same time will be safe. ++ * ++ * Some flags have locking requirements, check the documentation for the ++ * respective flags. ++ */ ++static inline void kbase_ctx_flag_clear(struct kbase_context *kctx, ++ enum kbase_context_flags flag) ++{ ++#if KERNEL_VERSION(4, 3, 0) > LINUX_VERSION_CODE ++ /* ++ * Earlier kernel versions doesn't have atomic_andnot() or ++ * atomic_and(). atomic_clear_mask() was only available on some ++ * architectures and removed on arm in v3.13 on arm and arm64. ++ * ++ * Use a compare-exchange loop to clear the flag on pre 4.3 kernels, ++ * when atomic_andnot() becomes available. ++ */ ++ int old, new; ++ ++ do { ++ old = atomic_read(&kctx->flags); ++ new = old & ~flag; ++ ++ } while (atomic_cmpxchg(&kctx->flags, old, new) != old); ++#else ++ atomic_andnot(flag, &kctx->flags); ++#endif ++} ++ ++/** ++ * kbase_ctx_flag_set - Set @flag on @kctx ++ * @kctx: Pointer to kbase context ++ * @flag: Flag to clear ++ * ++ * Set the @flag on @kctx. This is done atomically, so other flags being ++ * cleared or set at the same time will be safe. ++ * ++ * Some flags have locking requirements, check the documentation for the ++ * respective flags. ++ */ ++static inline void kbase_ctx_flag_set(struct kbase_context *kctx, ++ enum kbase_context_flags flag) ++{ ++ atomic_or(flag, &kctx->flags); ++} ++#endif /* _KBASE_CONTEXT_H_ */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_core_linux.c b/drivers/gpu/arm/midgard/mali_kbase_core_linux.c +new file mode 100755 +index 000000000000..1425dcc0718f +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_core_linux.c +@@ -0,0 +1,4990 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++#define ENABLE_DEBUG_LOG ++#include "platform/rk/custom_log.h" ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#ifdef CONFIG_MALI_DEVFREQ ++#include ++#include ++#ifdef CONFIG_DEVFREQ_THERMAL ++#include ++#endif /* CONFIG_DEVFREQ_THERMAL */ ++#endif /* CONFIG_MALI_DEVFREQ */ ++#ifdef CONFIG_MALI_NO_MALI ++#include "mali_kbase_model_linux.h" ++#endif /* CONFIG_MALI_NO_MALI */ ++#include "mali_kbase_mem_profile_debugfs_buf_size.h" ++#include "mali_kbase_debug_mem_view.h" ++#include "mali_kbase_mem.h" ++#include "mali_kbase_mem_pool_debugfs.h" ++#if !MALI_CUSTOMER_RELEASE ++#include "mali_kbase_regs_dump_debugfs.h" ++#endif /* !MALI_CUSTOMER_RELEASE */ ++#include "mali_kbase_regs_history_debugfs.h" ++#include ++#include ++#include ++#include ++#include "mali_kbase_ioctl.h" ++ ++#ifdef CONFIG_KDS ++#include ++#include ++#include ++#endif /* CONFIG_KDS */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include /* is_compat_task */ ++#include ++#include ++#ifdef CONFIG_MALI_PLATFORM_DEVICETREE ++#include ++#endif /* CONFIG_MALI_PLATFORM_DEVICETREE */ ++#include ++#include ++#ifdef CONFIG_MALI_PLATFORM_FAKE ++#include ++#endif /*CONFIG_MALI_PLATFORM_FAKE */ ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++#include ++#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ ++#include ++#include ++ ++#include ++ ++ ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0)) ++#include ++#else ++#include ++#endif ++ ++#include ++ ++#include ++ ++/* GPU IRQ Tags */ ++#define JOB_IRQ_TAG 0 ++#define MMU_IRQ_TAG 1 ++#define GPU_IRQ_TAG 2 ++ ++#if MALI_UNIT_TEST ++static struct kbase_exported_test_data shared_kernel_test_data; ++EXPORT_SYMBOL(shared_kernel_test_data); ++#endif /* MALI_UNIT_TEST */ ++ ++/** rk_ext : version of rk_ext on mali_ko, aka. rk_ko_ver. */ ++#define ROCKCHIP_VERSION (13) ++ ++static int kbase_dev_nr; ++ ++static DEFINE_MUTEX(kbase_dev_list_lock); ++static LIST_HEAD(kbase_dev_list); ++ ++#define KERNEL_SIDE_DDK_VERSION_STRING "K:" MALI_RELEASE_NAME "(GPL)" ++static inline void __compile_time_asserts(void) ++{ ++ CSTD_COMPILE_TIME_ASSERT(sizeof(KERNEL_SIDE_DDK_VERSION_STRING) <= KBASE_GET_VERSION_BUFFER_SIZE); ++} ++ ++static int kbase_api_handshake(struct kbase_context *kctx, ++ struct kbase_ioctl_version_check *version) ++{ ++ switch (version->major) { ++#ifdef BASE_LEGACY_UK6_SUPPORT ++ case 6: ++ /* We are backwards compatible with version 6, ++ * so pretend to be the old version */ ++ version->major = 6; ++ version->minor = 1; ++ break; ++#endif /* BASE_LEGACY_UK6_SUPPORT */ ++#ifdef BASE_LEGACY_UK7_SUPPORT ++ case 7: ++ /* We are backwards compatible with version 7, ++ * so pretend to be the old version */ ++ version->major = 7; ++ version->minor = 1; ++ break; ++#endif /* BASE_LEGACY_UK7_SUPPORT */ ++#ifdef BASE_LEGACY_UK8_SUPPORT ++ case 8: ++ /* We are backwards compatible with version 8, ++ * so pretend to be the old version */ ++ version->major = 8; ++ version->minor = 4; ++ break; ++#endif /* BASE_LEGACY_UK8_SUPPORT */ ++#ifdef BASE_LEGACY_UK9_SUPPORT ++ case 9: ++ /* We are backwards compatible with version 9, ++ * so pretend to be the old version */ ++ version->major = 9; ++ version->minor = 0; ++ break; ++#endif /* BASE_LEGACY_UK8_SUPPORT */ ++ case BASE_UK_VERSION_MAJOR: ++ /* set minor to be the lowest common */ ++ version->minor = min_t(int, BASE_UK_VERSION_MINOR, ++ (int)version->minor); ++ break; ++ default: ++ /* We return our actual version regardless if it ++ * matches the version returned by userspace - ++ * userspace can bail if it can't handle this ++ * version */ ++ version->major = BASE_UK_VERSION_MAJOR; ++ version->minor = BASE_UK_VERSION_MINOR; ++ break; ++ } ++ ++ /* save the proposed version number for later use */ ++ kctx->api_version = KBASE_API_VERSION(version->major, version->minor); ++ ++ return 0; ++} ++ ++/** ++ * enum mali_error - Mali error codes shared with userspace ++ * ++ * This is subset of those common Mali errors that can be returned to userspace. ++ * Values of matching user and kernel space enumerators MUST be the same. ++ * MALI_ERROR_NONE is guaranteed to be 0. ++ * ++ * @MALI_ERROR_NONE: Success ++ * @MALI_ERROR_OUT_OF_GPU_MEMORY: Not used in the kernel driver ++ * @MALI_ERROR_OUT_OF_MEMORY: Memory allocation failure ++ * @MALI_ERROR_FUNCTION_FAILED: Generic error code ++ */ ++enum mali_error { ++ MALI_ERROR_NONE = 0, ++ MALI_ERROR_OUT_OF_GPU_MEMORY, ++ MALI_ERROR_OUT_OF_MEMORY, ++ MALI_ERROR_FUNCTION_FAILED, ++}; ++ ++enum { ++ inited_mem = (1u << 0), ++ inited_js = (1u << 1), ++ inited_pm_runtime_init = (1u << 2), ++#ifdef CONFIG_MALI_DEVFREQ ++ inited_devfreq = (1u << 3), ++#endif /* CONFIG_MALI_DEVFREQ */ ++ inited_tlstream = (1u << 4), ++ inited_backend_early = (1u << 5), ++ inited_backend_late = (1u << 6), ++ inited_device = (1u << 7), ++ inited_vinstr = (1u << 8), ++ ++ inited_job_fault = (1u << 10), ++ inited_sysfs_group = (1u << 11), ++ inited_misc_register = (1u << 12), ++ inited_get_device = (1u << 13), ++ inited_dev_list = (1u << 14), ++ inited_debugfs = (1u << 15), ++ inited_gpu_device = (1u << 16), ++ inited_registers_map = (1u << 17), ++ inited_io_history = (1u << 18), ++ inited_power_control = (1u << 19), ++ inited_buslogger = (1u << 20), ++ inited_protected = (1u << 21), ++ inited_ctx_sched = (1u << 22) ++}; ++ ++ ++#ifdef CONFIG_MALI_DEBUG ++#define INACTIVE_WAIT_MS (5000) ++ ++void kbase_set_driver_inactive(struct kbase_device *kbdev, bool inactive) ++{ ++ kbdev->driver_inactive = inactive; ++ wake_up(&kbdev->driver_inactive_wait); ++ ++ /* Wait for any running IOCTLs to complete */ ++ if (inactive) ++ msleep(INACTIVE_WAIT_MS); ++} ++KBASE_EXPORT_TEST_API(kbase_set_driver_inactive); ++#endif /* CONFIG_MALI_DEBUG */ ++ ++/** ++ * kbase_legacy_dispatch - UKK dispatch function ++ * ++ * This is the dispatch function for the legacy UKK ioctl interface. No new ++ * ioctls should be added to this function, see kbase_ioctl instead. ++ * ++ * @kctx: The kernel context structure ++ * @args: Pointer to the data structure passed from/to user space ++ * @args_size: Size of the data structure ++ */ ++static int kbase_legacy_dispatch(struct kbase_context *kctx, ++ void * const args, u32 args_size) ++{ ++ struct kbase_device *kbdev; ++ union uk_header *ukh = args; ++ u32 id; ++ int ret = 0; ++ ++ KBASE_DEBUG_ASSERT(ukh != NULL); ++ ++ kbdev = kctx->kbdev; ++ id = ukh->id; ++ ukh->ret = MALI_ERROR_NONE; /* Be optimistic */ ++ ++#ifdef CONFIG_MALI_DEBUG ++ wait_event(kbdev->driver_inactive_wait, ++ kbdev->driver_inactive == false); ++#endif /* CONFIG_MALI_DEBUG */ ++ ++ if (UKP_FUNC_ID_CHECK_VERSION == id) { ++ struct uku_version_check_args *version_check; ++ struct kbase_ioctl_version_check version; ++ ++ if (args_size != sizeof(struct uku_version_check_args)) { ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ return 0; ++ } ++ version_check = (struct uku_version_check_args *)args; ++ version.minor = version_check->minor; ++ version.major = version_check->major; ++ ++ kbase_api_handshake(kctx, &version); ++ ++ version_check->minor = version.minor; ++ version_check->major = version.major; ++ ukh->ret = MALI_ERROR_NONE; ++ return 0; ++ } ++ ++ /* block calls until version handshake */ ++ if (kctx->api_version == 0) ++ return -EINVAL; ++ ++ if (!atomic_read(&kctx->setup_complete)) { ++ struct kbase_uk_set_flags *kbase_set_flags; ++ ++ /* setup pending, try to signal that we'll do the setup, ++ * if setup was already in progress, err this call ++ */ ++ if (atomic_cmpxchg(&kctx->setup_in_progress, 0, 1) != 0) ++ return -EINVAL; ++ ++ /* if unexpected call, will stay stuck in setup mode ++ * (is it the only call we accept?) ++ */ ++ if (id != KBASE_FUNC_SET_FLAGS) ++ return -EINVAL; ++ ++ kbase_set_flags = (struct kbase_uk_set_flags *)args; ++ ++ /* if not matching the expected call, stay in setup mode */ ++ if (sizeof(*kbase_set_flags) != args_size) ++ goto bad_size; ++ ++ /* if bad flags, will stay stuck in setup mode */ ++ if (kbase_context_set_create_flags(kctx, ++ kbase_set_flags->create_flags) != 0) ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ ++ atomic_set(&kctx->setup_complete, 1); ++ return 0; ++ } ++ ++ /* setup complete, perform normal operation */ ++ switch (id) { ++ case KBASE_FUNC_MEM_JIT_INIT: ++ { ++ struct kbase_uk_mem_jit_init *jit_init = args; ++ ++ if (sizeof(*jit_init) != args_size) ++ goto bad_size; ++ ++ if (kbase_region_tracker_init_jit(kctx, ++ jit_init->va_pages)) ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ break; ++ } ++ case KBASE_FUNC_MEM_ALLOC: ++ { ++ struct kbase_uk_mem_alloc *mem = args; ++ struct kbase_va_region *reg; ++ ++ if (sizeof(*mem) != args_size) ++ goto bad_size; ++ ++#if defined(CONFIG_64BIT) ++ if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { ++ /* force SAME_VA if a 64-bit client */ ++ mem->flags |= BASE_MEM_SAME_VA; ++ } ++#endif ++ ++ reg = kbase_mem_alloc(kctx, mem->va_pages, ++ mem->commit_pages, mem->extent, ++ &mem->flags, &mem->gpu_va); ++ mem->va_alignment = 0; ++ ++ if (!reg) ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ break; ++ } ++ case KBASE_FUNC_MEM_IMPORT: { ++ struct kbase_uk_mem_import *mem_import = args; ++ void __user *phandle; ++ ++ if (sizeof(*mem_import) != args_size) ++ goto bad_size; ++#ifdef CONFIG_COMPAT ++ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) ++ phandle = compat_ptr(mem_import->phandle.compat_value); ++ else ++#endif ++ phandle = mem_import->phandle.value; ++ ++ if (mem_import->type == BASE_MEM_IMPORT_TYPE_INVALID) { ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ break; ++ } ++ ++ if (kbase_mem_import(kctx, ++ (enum base_mem_import_type) ++ mem_import->type, ++ phandle, ++ 0, ++ &mem_import->gpu_va, ++ &mem_import->va_pages, ++ &mem_import->flags)) { ++ mem_import->type = BASE_MEM_IMPORT_TYPE_INVALID; ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ } ++ break; ++ } ++ case KBASE_FUNC_MEM_ALIAS: { ++ struct kbase_uk_mem_alias *alias = args; ++ struct base_mem_aliasing_info __user *user_ai; ++ struct base_mem_aliasing_info *ai; ++ ++ if (sizeof(*alias) != args_size) ++ goto bad_size; ++ ++ if (alias->nents > 2048) { ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ break; ++ } ++ if (!alias->nents) { ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ break; ++ } ++ ++#ifdef CONFIG_COMPAT ++ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) ++ user_ai = compat_ptr(alias->ai.compat_value); ++ else ++#endif ++ user_ai = alias->ai.value; ++ ++ ai = vmalloc(sizeof(*ai) * alias->nents); ++ ++ if (!ai) { ++ ukh->ret = MALI_ERROR_OUT_OF_MEMORY; ++ break; ++ } ++ ++ if (copy_from_user(ai, user_ai, ++ sizeof(*ai) * alias->nents)) { ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ goto copy_failed; ++ } ++ ++ alias->gpu_va = kbase_mem_alias(kctx, &alias->flags, ++ alias->stride, ++ alias->nents, ai, ++ &alias->va_pages); ++ if (!alias->gpu_va) { ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ goto no_alias; ++ } ++no_alias: ++copy_failed: ++ vfree(ai); ++ break; ++ } ++ case KBASE_FUNC_MEM_COMMIT: ++ { ++ struct kbase_uk_mem_commit *commit = args; ++ int ret; ++ ++ if (sizeof(*commit) != args_size) ++ goto bad_size; ++ ++ ret = kbase_mem_commit(kctx, commit->gpu_addr, ++ commit->pages); ++ ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ commit->result_subcode = ++ BASE_BACKING_THRESHOLD_ERROR_INVALID_ARGUMENTS; ++ ++ if (ret == 0) { ++ ukh->ret = MALI_ERROR_NONE; ++ commit->result_subcode = ++ BASE_BACKING_THRESHOLD_OK; ++ } else if (ret == -ENOMEM) { ++ commit->result_subcode = ++ BASE_BACKING_THRESHOLD_ERROR_OOM; ++ } ++ ++ break; ++ } ++ ++ case KBASE_FUNC_MEM_QUERY: ++ { ++ struct kbase_uk_mem_query *query = args; ++ ++ if (sizeof(*query) != args_size) ++ goto bad_size; ++ ++ if (kbase_mem_query(kctx, query->gpu_addr, ++ query->query, &query->value) != 0) ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ else ++ ukh->ret = MALI_ERROR_NONE; ++ break; ++ } ++ break; ++ ++ case KBASE_FUNC_MEM_FLAGS_CHANGE: ++ { ++ struct kbase_uk_mem_flags_change *fc = args; ++ ++ if (sizeof(*fc) != args_size) ++ goto bad_size; ++ ++ if (kbase_mem_flags_change(kctx, fc->gpu_va, ++ fc->flags, fc->mask) != 0) ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ ++ break; ++ } ++ case KBASE_FUNC_MEM_FREE: ++ { ++ struct kbase_uk_mem_free *mem = args; ++ ++ if (sizeof(*mem) != args_size) ++ goto bad_size; ++ ++ if (kbase_mem_free(kctx, mem->gpu_addr) != 0) ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ break; ++ } ++ ++ case KBASE_FUNC_JOB_SUBMIT: ++ { ++ struct kbase_uk_job_submit *job = args; ++ void __user *user_addr = NULL; ++ ++ if (sizeof(*job) != args_size) ++ goto bad_size; ++ ++#ifdef CONFIG_COMPAT ++ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) ++ user_addr = compat_ptr(job->addr.compat_value); ++ else ++#endif ++ user_addr = job->addr.value; ++ ++ if (kbase_jd_submit(kctx, user_addr, job->nr_atoms, ++ job->stride, false) != 0) ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ break; ++ } ++ ++#ifdef BASE_LEGACY_UK6_SUPPORT ++ case KBASE_FUNC_JOB_SUBMIT_UK6: ++ { ++ struct kbase_uk_job_submit *job = args; ++ void __user *user_addr = NULL; ++ ++ if (sizeof(*job) != args_size) ++ goto bad_size; ++ ++#ifdef CONFIG_COMPAT ++ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) ++ user_addr = compat_ptr(job->addr.compat_value); ++ else ++#endif ++ user_addr = job->addr.value; ++ ++ if (kbase_jd_submit(kctx, user_addr, job->nr_atoms, ++ job->stride, true) != 0) ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ break; ++ } ++#endif ++ ++ case KBASE_FUNC_SYNC: ++ { ++ struct kbase_uk_sync_now *sn = args; ++ ++ if (sizeof(*sn) != args_size) ++ goto bad_size; ++ ++#ifndef CONFIG_MALI_COH_USER ++ if (kbase_sync_now(kctx, &sn->sset.basep_sset) != 0) ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++#endif ++ break; ++ } ++ ++ case KBASE_FUNC_DISJOINT_QUERY: ++ { ++ struct kbase_uk_disjoint_query *dquery = args; ++ ++ if (sizeof(*dquery) != args_size) ++ goto bad_size; ++ ++ /* Get the disjointness counter value. */ ++ dquery->counter = kbase_disjoint_event_get(kctx->kbdev); ++ break; ++ } ++ ++ case KBASE_FUNC_POST_TERM: ++ { ++ kbase_event_close(kctx); ++ break; ++ } ++ ++ case KBASE_FUNC_HWCNT_SETUP: ++ { ++ struct kbase_uk_hwcnt_setup *setup = args; ++ ++ if (sizeof(*setup) != args_size) ++ goto bad_size; ++ ++ mutex_lock(&kctx->vinstr_cli_lock); ++ if (kbase_vinstr_legacy_hwc_setup(kbdev->vinstr_ctx, ++ &kctx->vinstr_cli, setup) != 0) ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ mutex_unlock(&kctx->vinstr_cli_lock); ++ break; ++ } ++ ++ case KBASE_FUNC_HWCNT_DUMP: ++ { ++ /* args ignored */ ++ mutex_lock(&kctx->vinstr_cli_lock); ++ if (kbase_vinstr_hwc_dump(kctx->vinstr_cli, ++ BASE_HWCNT_READER_EVENT_MANUAL) != 0) ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ mutex_unlock(&kctx->vinstr_cli_lock); ++ break; ++ } ++ ++ case KBASE_FUNC_HWCNT_CLEAR: ++ { ++ /* args ignored */ ++ mutex_lock(&kctx->vinstr_cli_lock); ++ if (kbase_vinstr_hwc_clear(kctx->vinstr_cli) != 0) ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ mutex_unlock(&kctx->vinstr_cli_lock); ++ break; ++ } ++ ++ case KBASE_FUNC_HWCNT_READER_SETUP: ++ { ++ struct kbase_uk_hwcnt_reader_setup *setup = args; ++ ++ if (sizeof(*setup) != args_size) ++ goto bad_size; ++ ++ mutex_lock(&kctx->vinstr_cli_lock); ++ if (kbase_vinstr_hwcnt_reader_setup(kbdev->vinstr_ctx, ++ setup) != 0) ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ mutex_unlock(&kctx->vinstr_cli_lock); ++ break; ++ } ++ ++ case KBASE_FUNC_GPU_PROPS_REG_DUMP: ++ { ++ struct kbase_uk_gpuprops *setup = args; ++ ++ if (sizeof(*setup) != args_size) ++ goto bad_size; ++ ++ if (kbase_gpuprops_uk_get_props(kctx, setup) != 0) ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ break; ++ } ++ case KBASE_FUNC_FIND_CPU_OFFSET: ++ { ++ struct kbase_uk_find_cpu_offset *find = args; ++ ++ if (sizeof(*find) != args_size) ++ goto bad_size; ++ ++ if (find->gpu_addr & ~PAGE_MASK) { ++ dev_warn(kbdev->dev, ++ "kbase_legacy_dispatch case KBASE_FUNC_FIND_CPU_OFFSET: find->gpu_addr: passed parameter is invalid"); ++ goto out_bad; ++ } ++ ++ if (find->size > SIZE_MAX || find->cpu_addr > ULONG_MAX) { ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ } else { ++ int err; ++ ++ err = kbasep_find_enclosing_cpu_mapping_offset( ++ kctx, ++ find->cpu_addr, ++ find->size, ++ &find->offset); ++ ++ if (err) ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ } ++ break; ++ } ++ case KBASE_FUNC_GET_VERSION: ++ { ++ struct kbase_uk_get_ddk_version *get_version = (struct kbase_uk_get_ddk_version *)args; ++ ++ if (sizeof(*get_version) != args_size) ++ goto bad_size; ++ ++ /* version buffer size check is made in compile time assert */ ++ memcpy(get_version->version_buffer, ++ KERNEL_SIDE_DDK_VERSION_STRING, ++ sizeof(KERNEL_SIDE_DDK_VERSION_STRING)); ++ get_version->version_string_size = ++ sizeof(KERNEL_SIDE_DDK_VERSION_STRING); ++ get_version->rk_version = ROCKCHIP_VERSION; ++ break; ++ } ++ ++ case KBASE_FUNC_STREAM_CREATE: ++ { ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ struct kbase_uk_stream_create *screate = (struct kbase_uk_stream_create *)args; ++ ++ if (sizeof(*screate) != args_size) ++ goto bad_size; ++ ++ if (strnlen(screate->name, sizeof(screate->name)) >= sizeof(screate->name)) { ++ /* not NULL terminated */ ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ break; ++ } ++ ++ if (kbase_sync_fence_stream_create(screate->name, ++ &screate->fd) != 0) ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ else ++ ukh->ret = MALI_ERROR_NONE; ++#else /* CONFIG_SYNC || CONFIG_SYNC_FILE */ ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ ++ break; ++ } ++ case KBASE_FUNC_FENCE_VALIDATE: ++ { ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ struct kbase_uk_fence_validate *fence_validate = (struct kbase_uk_fence_validate *)args; ++ ++ if (sizeof(*fence_validate) != args_size) ++ goto bad_size; ++ ++ if (kbase_sync_fence_validate(fence_validate->fd) != 0) ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ else ++ ukh->ret = MALI_ERROR_NONE; ++#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ ++ break; ++ } ++ ++ case KBASE_FUNC_SET_TEST_DATA: ++ { ++#if MALI_UNIT_TEST ++ struct kbase_uk_set_test_data *set_data = args; ++ ++ shared_kernel_test_data = set_data->test_data; ++ shared_kernel_test_data.kctx.value = (void __user *)kctx; ++ shared_kernel_test_data.mm.value = (void __user *)current->mm; ++ ukh->ret = MALI_ERROR_NONE; ++#endif /* MALI_UNIT_TEST */ ++ break; ++ } ++ ++ case KBASE_FUNC_INJECT_ERROR: ++ { ++#ifdef CONFIG_MALI_ERROR_INJECT ++ unsigned long flags; ++ struct kbase_error_params params = ((struct kbase_uk_error_params *)args)->params; ++ ++ /*mutex lock */ ++ spin_lock_irqsave(&kbdev->reg_op_lock, flags); ++ if (job_atom_inject_error(¶ms) != 0) ++ ukh->ret = MALI_ERROR_OUT_OF_MEMORY; ++ else ++ ukh->ret = MALI_ERROR_NONE; ++ spin_unlock_irqrestore(&kbdev->reg_op_lock, flags); ++ /*mutex unlock */ ++#endif /* CONFIG_MALI_ERROR_INJECT */ ++ break; ++ } ++ ++ case KBASE_FUNC_MODEL_CONTROL: ++ { ++#ifdef CONFIG_MALI_NO_MALI ++ unsigned long flags; ++ struct kbase_model_control_params params = ++ ((struct kbase_uk_model_control_params *)args)->params; ++ ++ /*mutex lock */ ++ spin_lock_irqsave(&kbdev->reg_op_lock, flags); ++ if (gpu_model_control(kbdev->model, ¶ms) != 0) ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ else ++ ukh->ret = MALI_ERROR_NONE; ++ spin_unlock_irqrestore(&kbdev->reg_op_lock, flags); ++ /*mutex unlock */ ++#endif /* CONFIG_MALI_NO_MALI */ ++ break; ++ } ++ ++#ifdef BASE_LEGACY_UK8_SUPPORT ++ case KBASE_FUNC_KEEP_GPU_POWERED: ++ { ++ dev_warn(kbdev->dev, "kbase_legacy_dispatch case KBASE_FUNC_KEEP_GPU_POWERED: function is deprecated and disabled\n"); ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ break; ++ } ++#endif /* BASE_LEGACY_UK8_SUPPORT */ ++ ++ case KBASE_FUNC_GET_PROFILING_CONTROLS: ++ { ++ struct kbase_uk_profiling_controls *controls = ++ (struct kbase_uk_profiling_controls *)args; ++ u32 i; ++ ++ if (sizeof(*controls) != args_size) ++ goto bad_size; ++ ++ for (i = FBDUMP_CONTROL_MIN; i < FBDUMP_CONTROL_MAX; i++) ++ controls->profiling_controls[i] = ++ kbdev->kbase_profiling_controls[i]; ++ ++ break; ++ } ++ ++ /* used only for testing purposes; these controls are to be set by gator through gator API */ ++ case KBASE_FUNC_SET_PROFILING_CONTROLS: ++ { ++ struct kbase_uk_profiling_controls *controls = ++ (struct kbase_uk_profiling_controls *)args; ++ u32 i; ++ ++ if (sizeof(*controls) != args_size) ++ goto bad_size; ++ ++ for (i = FBDUMP_CONTROL_MIN; i < FBDUMP_CONTROL_MAX; i++) ++ _mali_profiling_control(i, controls->profiling_controls[i]); ++ ++ break; ++ } ++ ++ case KBASE_FUNC_DEBUGFS_MEM_PROFILE_ADD: ++ { ++ struct kbase_uk_debugfs_mem_profile_add *add_data = ++ (struct kbase_uk_debugfs_mem_profile_add *)args; ++ char *buf; ++ char __user *user_buf; ++ ++ if (sizeof(*add_data) != args_size) ++ goto bad_size; ++ ++ if (add_data->len > KBASE_MEM_PROFILE_MAX_BUF_SIZE) { ++ dev_err(kbdev->dev, "buffer too big\n"); ++ goto out_bad; ++ } ++ ++#ifdef CONFIG_COMPAT ++ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) ++ user_buf = ++ compat_ptr(add_data->buf.compat_value); ++ else ++#endif ++ user_buf = add_data->buf.value; ++ ++ buf = kmalloc(add_data->len, GFP_KERNEL); ++ if (ZERO_OR_NULL_PTR(buf)) ++ goto out_bad; ++ ++ if (0 != copy_from_user(buf, user_buf, add_data->len)) { ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ kfree(buf); ++ goto out_bad; ++ } ++ ++ if (kbasep_mem_profile_debugfs_insert(kctx, buf, ++ add_data->len)) { ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ goto out_bad; ++ } ++ ++ break; ++ } ++ ++#ifdef CONFIG_MALI_NO_MALI ++ case KBASE_FUNC_SET_PRFCNT_VALUES: ++ { ++ ++ struct kbase_uk_prfcnt_values *params = ++ ((struct kbase_uk_prfcnt_values *)args); ++ gpu_model_set_dummy_prfcnt_sample(params->data, ++ params->size); ++ ++ break; ++ } ++#endif /* CONFIG_MALI_NO_MALI */ ++#ifdef BASE_LEGACY_UK10_4_SUPPORT ++ case KBASE_FUNC_TLSTREAM_ACQUIRE_V10_4: ++ { ++ struct kbase_uk_tlstream_acquire_v10_4 *tlstream_acquire ++ = args; ++ int ret; ++ ++ if (sizeof(*tlstream_acquire) != args_size) ++ goto bad_size; ++ ++ ret = kbase_tlstream_acquire( ++ kctx, 0); ++ if (ret < 0) ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ else ++ tlstream_acquire->fd = ret; ++ break; ++ } ++#endif /* BASE_LEGACY_UK10_4_SUPPORT */ ++ case KBASE_FUNC_TLSTREAM_ACQUIRE: ++ { ++ struct kbase_uk_tlstream_acquire *tlstream_acquire = ++ args; ++ int ret; ++ ++ if (sizeof(*tlstream_acquire) != args_size) ++ goto bad_size; ++ ++ if (tlstream_acquire->flags & ~BASE_TLSTREAM_FLAGS_MASK) ++ goto out_bad; ++ ++ ret = kbase_tlstream_acquire( ++ kctx, tlstream_acquire->flags); ++ if (ret < 0) ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ else ++ tlstream_acquire->fd = ret; ++ break; ++ } ++ case KBASE_FUNC_TLSTREAM_FLUSH: ++ { ++ struct kbase_uk_tlstream_flush *tlstream_flush = ++ args; ++ ++ if (sizeof(*tlstream_flush) != args_size) ++ goto bad_size; ++ ++ kbase_tlstream_flush_streams(); ++ break; ++ } ++#if MALI_UNIT_TEST ++ case KBASE_FUNC_TLSTREAM_TEST: ++ { ++ struct kbase_uk_tlstream_test *tlstream_test = args; ++ ++ if (sizeof(*tlstream_test) != args_size) ++ goto bad_size; ++ ++ kbase_tlstream_test( ++ tlstream_test->tpw_count, ++ tlstream_test->msg_delay, ++ tlstream_test->msg_count, ++ tlstream_test->aux_msg); ++ break; ++ } ++ case KBASE_FUNC_TLSTREAM_STATS: ++ { ++ struct kbase_uk_tlstream_stats *tlstream_stats = args; ++ ++ if (sizeof(*tlstream_stats) != args_size) ++ goto bad_size; ++ ++ kbase_tlstream_stats( ++ &tlstream_stats->bytes_collected, ++ &tlstream_stats->bytes_generated); ++ break; ++ } ++#endif /* MALI_UNIT_TEST */ ++ ++ case KBASE_FUNC_GET_CONTEXT_ID: ++ { ++ struct kbase_uk_context_id *info = args; ++ ++ info->id = kctx->id; ++ break; ++ } ++ ++ case KBASE_FUNC_SOFT_EVENT_UPDATE: ++ { ++ struct kbase_uk_soft_event_update *update = args; ++ ++ if (sizeof(*update) != args_size) ++ goto bad_size; ++ ++ if (((update->new_status != BASE_JD_SOFT_EVENT_SET) && ++ (update->new_status != BASE_JD_SOFT_EVENT_RESET)) || ++ (update->flags != 0)) ++ goto out_bad; ++ ++ if (kbase_soft_event_update(kctx, update->evt, ++ update->new_status)) ++ ukh->ret = MALI_ERROR_FUNCTION_FAILED; ++ ++ break; ++ } ++ ++ default: ++ dev_err(kbdev->dev, "unknown ioctl %u\n", id); ++ goto out_bad; ++ } ++ ++ return ret; ++ ++bad_size: ++ dev_err(kbdev->dev, "Wrong syscall size (%d) for %08x\n", args_size, id); ++out_bad: ++ return -EINVAL; ++} ++ ++static struct kbase_device *to_kbase_device(struct device *dev) ++{ ++ return dev_get_drvdata(dev); ++} ++ ++static int assign_irqs(struct platform_device *pdev) ++{ ++ struct kbase_device *kbdev = to_kbase_device(&pdev->dev); ++ int i; ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ /* 3 IRQ resources */ ++ for (i = 0; i < 3; i++) { ++ struct resource *irq_res; ++ int irqtag; ++ ++ irq_res = platform_get_resource(pdev, IORESOURCE_IRQ, i); ++ if (!irq_res) { ++ dev_err(kbdev->dev, "No IRQ resource at index %d\n", i); ++ return -ENOENT; ++ } ++ ++#ifdef CONFIG_OF ++ if (!strncmp(irq_res->name, "JOB", 4)) { ++ irqtag = JOB_IRQ_TAG; ++ } else if (!strncmp(irq_res->name, "MMU", 4)) { ++ irqtag = MMU_IRQ_TAG; ++ } else if (!strncmp(irq_res->name, "GPU", 4)) { ++ irqtag = GPU_IRQ_TAG; ++ } else { ++ dev_err(&pdev->dev, "Invalid irq res name: '%s'\n", ++ irq_res->name); ++ return -EINVAL; ++ } ++#else ++ irqtag = i; ++#endif /* CONFIG_OF */ ++ kbdev->irqs[irqtag].irq = irq_res->start; ++ kbdev->irqs[irqtag].flags = irq_res->flags & IRQF_TRIGGER_MASK; ++ } ++ ++ return 0; ++} ++ ++/* ++ * API to acquire device list mutex and ++ * return pointer to the device list head ++ */ ++const struct list_head *kbase_dev_list_get(void) ++{ ++ mutex_lock(&kbase_dev_list_lock); ++ return &kbase_dev_list; ++} ++KBASE_EXPORT_TEST_API(kbase_dev_list_get); ++ ++/* API to release the device list mutex */ ++void kbase_dev_list_put(const struct list_head *dev_list) ++{ ++ mutex_unlock(&kbase_dev_list_lock); ++} ++KBASE_EXPORT_TEST_API(kbase_dev_list_put); ++ ++/* Find a particular kbase device (as specified by minor number), or find the "first" device if -1 is specified */ ++struct kbase_device *kbase_find_device(int minor) ++{ ++ struct kbase_device *kbdev = NULL; ++ struct list_head *entry; ++ const struct list_head *dev_list = kbase_dev_list_get(); ++ ++ list_for_each(entry, dev_list) { ++ struct kbase_device *tmp; ++ ++ tmp = list_entry(entry, struct kbase_device, entry); ++ if (tmp->mdev.minor == minor || minor == -1) { ++ kbdev = tmp; ++ get_device(kbdev->dev); ++ break; ++ } ++ } ++ kbase_dev_list_put(dev_list); ++ ++ return kbdev; ++} ++EXPORT_SYMBOL(kbase_find_device); ++ ++void kbase_release_device(struct kbase_device *kbdev) ++{ ++ put_device(kbdev->dev); ++} ++EXPORT_SYMBOL(kbase_release_device); ++ ++#if KERNEL_VERSION(4, 4, 0) > LINUX_VERSION_CODE ++/* ++ * Older versions, before v4.6, of the kernel doesn't have ++ * kstrtobool_from_user(), except longterm 4.4.y which had it added in 4.4.28 ++ */ ++static int kstrtobool_from_user(const char __user *s, size_t count, bool *res) ++{ ++ char buf[32]; ++ ++ count = min(sizeof(buf), count); ++ ++ if (copy_from_user(buf, s, count)) ++ return -EFAULT; ++ buf[count] = '\0'; ++ ++ return strtobool(buf, res); ++} ++#endif ++ ++static ssize_t write_ctx_infinite_cache(struct file *f, const char __user *ubuf, size_t size, loff_t *off) ++{ ++ struct kbase_context *kctx = f->private_data; ++ int err; ++ bool value; ++ ++ err = kstrtobool_from_user(ubuf, size, &value); ++ if (err) ++ return err; ++ ++ if (value) ++ kbase_ctx_flag_set(kctx, KCTX_INFINITE_CACHE); ++ else ++ kbase_ctx_flag_clear(kctx, KCTX_INFINITE_CACHE); ++ ++ return size; ++} ++ ++static ssize_t read_ctx_infinite_cache(struct file *f, char __user *ubuf, size_t size, loff_t *off) ++{ ++ struct kbase_context *kctx = f->private_data; ++ char buf[32]; ++ int count; ++ bool value; ++ ++ value = kbase_ctx_flag(kctx, KCTX_INFINITE_CACHE); ++ ++ count = scnprintf(buf, sizeof(buf), "%s\n", value ? "Y" : "N"); ++ ++ return simple_read_from_buffer(ubuf, size, off, buf, count); ++} ++ ++static const struct file_operations kbase_infinite_cache_fops = { ++ .open = simple_open, ++ .write = write_ctx_infinite_cache, ++ .read = read_ctx_infinite_cache, ++}; ++ ++static int kbase_open(struct inode *inode, struct file *filp) ++{ ++ struct kbase_device *kbdev = NULL; ++ struct kbase_context *kctx; ++ int ret = 0; ++#ifdef CONFIG_DEBUG_FS ++ char kctx_name[64]; ++#endif ++ ++ kbdev = kbase_find_device(iminor(inode)); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ kctx = kbase_create_context(kbdev, is_compat_task()); ++ if (!kctx) { ++ ret = -ENOMEM; ++ goto out; ++ } ++ ++ init_waitqueue_head(&kctx->event_queue); ++ filp->f_mode |= FMODE_UNSIGNED_OFFSET; ++ filp->private_data = kctx; ++ kctx->filp = filp; ++ ++ if (kbdev->infinite_cache_active_default) ++ kbase_ctx_flag_set(kctx, KCTX_INFINITE_CACHE); ++ ++#ifdef CONFIG_DEBUG_FS ++ snprintf(kctx_name, 64, "%d_%d", kctx->tgid, kctx->id); ++ ++ kctx->kctx_dentry = debugfs_create_dir(kctx_name, ++ kbdev->debugfs_ctx_directory); ++ ++ if (IS_ERR_OR_NULL(kctx->kctx_dentry)) { ++ ret = -ENOMEM; ++ goto out; ++ } ++ ++#ifdef CONFIG_MALI_COH_USER ++ /* if cache is completely coherent at hardware level, then remove the ++ * infinite cache control support from debugfs. ++ */ ++#else ++ debugfs_create_file("infinite_cache", 0644, kctx->kctx_dentry, ++ kctx, &kbase_infinite_cache_fops); ++#endif /* CONFIG_MALI_COH_USER */ ++ ++ mutex_init(&kctx->mem_profile_lock); ++ ++ kbasep_jd_debugfs_ctx_init(kctx); ++ kbase_debug_mem_view_init(filp); ++ ++ kbase_debug_job_fault_context_init(kctx); ++ ++ kbase_mem_pool_debugfs_init(kctx->kctx_dentry, &kctx->mem_pool); ++ ++ kbase_jit_debugfs_init(kctx); ++#endif /* CONFIG_DEBUG_FS */ ++ ++ dev_dbg(kbdev->dev, "created base context\n"); ++ ++ { ++ struct kbasep_kctx_list_element *element; ++ ++ element = kzalloc(sizeof(*element), GFP_KERNEL); ++ if (element) { ++ mutex_lock(&kbdev->kctx_list_lock); ++ element->kctx = kctx; ++ list_add(&element->link, &kbdev->kctx_list); ++ KBASE_TLSTREAM_TL_NEW_CTX( ++ element->kctx, ++ (u32)(element->kctx->id), ++ (u32)(element->kctx->tgid)); ++ mutex_unlock(&kbdev->kctx_list_lock); ++ } else { ++ /* we don't treat this as a fail - just warn about it */ ++ dev_warn(kbdev->dev, "couldn't add kctx to kctx_list\n"); ++ } ++ } ++ return 0; ++ ++ out: ++ kbase_release_device(kbdev); ++ return ret; ++} ++ ++static int kbase_release(struct inode *inode, struct file *filp) ++{ ++ struct kbase_context *kctx = filp->private_data; ++ struct kbase_device *kbdev = kctx->kbdev; ++ struct kbasep_kctx_list_element *element, *tmp; ++ bool found_element = false; ++ ++ KBASE_TLSTREAM_TL_DEL_CTX(kctx); ++ ++#ifdef CONFIG_DEBUG_FS ++ kbasep_mem_profile_debugfs_remove(kctx); ++ kbase_debug_job_fault_context_term(kctx); ++#endif ++ ++ mutex_lock(&kbdev->kctx_list_lock); ++ list_for_each_entry_safe(element, tmp, &kbdev->kctx_list, link) { ++ if (element->kctx == kctx) { ++ list_del(&element->link); ++ kfree(element); ++ found_element = true; ++ } ++ } ++ mutex_unlock(&kbdev->kctx_list_lock); ++ if (!found_element) ++ dev_warn(kbdev->dev, "kctx not in kctx_list\n"); ++ ++ filp->private_data = NULL; ++ ++ mutex_lock(&kctx->vinstr_cli_lock); ++ /* If this client was performing hwcnt dumping and did not explicitly ++ * detach itself, remove it from the vinstr core now */ ++ if (kctx->vinstr_cli) { ++ struct kbase_uk_hwcnt_setup setup; ++ ++ setup.dump_buffer = 0llu; ++ kbase_vinstr_legacy_hwc_setup( ++ kbdev->vinstr_ctx, &kctx->vinstr_cli, &setup); ++ } ++ mutex_unlock(&kctx->vinstr_cli_lock); ++ ++ kbase_destroy_context(kctx); ++ ++ dev_dbg(kbdev->dev, "deleted base context\n"); ++ kbase_release_device(kbdev); ++ return 0; ++} ++ ++#define CALL_MAX_SIZE 536 ++ ++static long kbase_legacy_ioctl(struct file *filp, unsigned int cmd, ++ unsigned long arg) ++{ ++ u64 msg[(CALL_MAX_SIZE + 7) >> 3] = { 0xdeadbeefdeadbeefull }; /* alignment fixup */ ++ u32 size = _IOC_SIZE(cmd); ++ struct kbase_context *kctx = filp->private_data; ++ ++ if (size > CALL_MAX_SIZE) ++ return -ENOTTY; ++ ++ if (0 != copy_from_user(&msg, (void __user *)arg, size)) { ++ dev_err(kctx->kbdev->dev, "failed to copy ioctl argument into kernel space\n"); ++ return -EFAULT; ++ } ++ ++ if (kbase_legacy_dispatch(kctx, &msg, size) != 0) ++ return -EFAULT; ++ ++ if (0 != copy_to_user((void __user *)arg, &msg, size)) { ++ dev_err(kctx->kbdev->dev, "failed to copy results of UK call back to user space\n"); ++ return -EFAULT; ++ } ++ return 0; ++} ++ ++static int kbase_api_set_flags(struct kbase_context *kctx, ++ struct kbase_ioctl_set_flags *flags) ++{ ++ int err; ++ ++ /* setup pending, try to signal that we'll do the setup, ++ * if setup was already in progress, err this call ++ */ ++ if (atomic_cmpxchg(&kctx->setup_in_progress, 0, 1) != 0) ++ return -EINVAL; ++ ++ err = kbase_context_set_create_flags(kctx, flags->create_flags); ++ /* if bad flags, will stay stuck in setup mode */ ++ if (err) ++ return err; ++ ++ atomic_set(&kctx->setup_complete, 1); ++ return 0; ++} ++ ++static int kbase_api_job_submit(struct kbase_context *kctx, ++ struct kbase_ioctl_job_submit *submit) ++{ ++ void __user *user_addr = NULL; ++ ++#ifdef CONFIG_COMPAT ++ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) ++ user_addr = compat_ptr(submit->addr.compat_value); ++ else ++#endif ++ user_addr = submit->addr.value; ++ ++ return kbase_jd_submit(kctx, user_addr, submit->nr_atoms, ++ submit->stride, false); ++} ++ ++static int kbase_api_get_gpuprops(struct kbase_context *kctx, ++ struct kbase_ioctl_get_gpuprops *get_props) ++{ ++ struct kbase_gpu_props *kprops = &kctx->kbdev->gpu_props; ++ int err; ++ ++ if (get_props->flags != 0) { ++ dev_err(kctx->kbdev->dev, "Unsupported flags to get_gpuprops"); ++ return -EINVAL; ++ } ++ ++ if (get_props->size == 0) ++ return kprops->prop_buffer_size; ++ if (get_props->size < kprops->prop_buffer_size) ++ return -EINVAL; ++ ++ err = copy_to_user(get_props->buffer.value, kprops->prop_buffer, ++ kprops->prop_buffer_size); ++ if (err) ++ return err; ++ return kprops->prop_buffer_size; ++} ++ ++static int kbase_api_post_term(struct kbase_context *kctx) ++{ ++ kbase_event_close(kctx); ++ return 0; ++} ++ ++static int kbase_api_mem_alloc(struct kbase_context *kctx, ++ union kbase_ioctl_mem_alloc *alloc) ++{ ++ struct kbase_va_region *reg; ++ u64 flags = alloc->in.flags; ++ u64 gpu_va; ++ ++#if defined(CONFIG_64BIT) ++ if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { ++ /* force SAME_VA if a 64-bit client */ ++ flags |= BASE_MEM_SAME_VA; ++ } ++#endif ++ ++ reg = kbase_mem_alloc(kctx, alloc->in.va_pages, ++ alloc->in.commit_pages, ++ alloc->in.extent, ++ &flags, &gpu_va); ++ ++ if (!reg) ++ return -ENOMEM; ++ ++ alloc->out.flags = flags; ++ alloc->out.gpu_va = gpu_va; ++ ++ return 0; ++} ++ ++static int kbase_api_mem_query(struct kbase_context *kctx, ++ union kbase_ioctl_mem_query *query) ++{ ++ return kbase_mem_query(kctx, query->in.gpu_addr, ++ query->in.query, &query->out.value); ++} ++ ++static int kbase_api_mem_free(struct kbase_context *kctx, ++ struct kbase_ioctl_mem_free *free) ++{ ++ return kbase_mem_free(kctx, free->gpu_addr); ++} ++ ++static int kbase_api_hwcnt_reader_setup(struct kbase_context *kctx, ++ struct kbase_ioctl_hwcnt_reader_setup *setup) ++{ ++ int ret; ++ struct kbase_uk_hwcnt_reader_setup args = { ++ .buffer_count = setup->buffer_count, ++ .jm_bm = setup->jm_bm, ++ .shader_bm = setup->shader_bm, ++ .tiler_bm = setup->tiler_bm, ++ .mmu_l2_bm = setup->mmu_l2_bm ++ }; ++ ++ mutex_lock(&kctx->vinstr_cli_lock); ++ ret = kbase_vinstr_hwcnt_reader_setup(kctx->kbdev->vinstr_ctx, &args); ++ mutex_unlock(&kctx->vinstr_cli_lock); ++ ++ if (ret) ++ return ret; ++ return args.fd; ++} ++ ++static int kbase_api_hwcnt_enable(struct kbase_context *kctx, ++ struct kbase_ioctl_hwcnt_enable *enable) ++{ ++ int ret; ++ struct kbase_uk_hwcnt_setup args = { ++ .dump_buffer = enable->dump_buffer, ++ .jm_bm = enable->jm_bm, ++ .shader_bm = enable->shader_bm, ++ .tiler_bm = enable->tiler_bm, ++ .mmu_l2_bm = enable->mmu_l2_bm ++ }; ++ ++ mutex_lock(&kctx->vinstr_cli_lock); ++ ret = kbase_vinstr_legacy_hwc_setup(kctx->kbdev->vinstr_ctx, ++ &kctx->vinstr_cli, &args); ++ mutex_unlock(&kctx->vinstr_cli_lock); ++ ++ return ret; ++} ++ ++static int kbase_api_hwcnt_dump(struct kbase_context *kctx) ++{ ++ int ret; ++ ++ mutex_lock(&kctx->vinstr_cli_lock); ++ ret = kbase_vinstr_hwc_dump(kctx->vinstr_cli, ++ BASE_HWCNT_READER_EVENT_MANUAL); ++ mutex_unlock(&kctx->vinstr_cli_lock); ++ ++ return ret; ++} ++ ++static int kbase_api_hwcnt_clear(struct kbase_context *kctx) ++{ ++ int ret; ++ ++ mutex_lock(&kctx->vinstr_cli_lock); ++ ret = kbase_vinstr_hwc_clear(kctx->vinstr_cli); ++ mutex_unlock(&kctx->vinstr_cli_lock); ++ ++ return ret; ++} ++ ++static int kbase_api_disjoint_query(struct kbase_context *kctx, ++ struct kbase_ioctl_disjoint_query *query) ++{ ++ query->counter = kbase_disjoint_event_get(kctx->kbdev); ++ ++ return 0; ++} ++ ++static int kbase_api_get_ddk_version(struct kbase_context *kctx, ++ struct kbase_ioctl_get_ddk_version *version) ++{ ++ int ret; ++ int len = sizeof(KERNEL_SIDE_DDK_VERSION_STRING); ++ ++ if (version->version_buffer.value == NULL) ++ return len; ++ ++ if (version->size < len) ++ return -EOVERFLOW; ++ ++ ret = copy_to_user(version->version_buffer.value, ++ KERNEL_SIDE_DDK_VERSION_STRING, ++ sizeof(KERNEL_SIDE_DDK_VERSION_STRING)); ++ ++ if (ret) ++ return ret; ++ ++ return len; ++} ++ ++static int kbase_api_mem_jit_init(struct kbase_context *kctx, ++ struct kbase_ioctl_mem_jit_init *jit_init) ++{ ++ return kbase_region_tracker_init_jit(kctx, jit_init->va_pages); ++} ++ ++static int kbase_api_mem_sync(struct kbase_context *kctx, ++ struct kbase_ioctl_mem_sync *sync) ++{ ++#ifdef CONFIG_MALI_COH_USER ++ return 0; ++#endif ++ struct basep_syncset sset = { ++ .mem_handle.basep.handle = sync->handle, ++ .user_addr = sync->user_addr, ++ .size = sync->size, ++ .type = sync->type ++ }; ++ ++ return kbase_sync_now(kctx, &sset); ++} ++ ++static int kbase_api_mem_find_cpu_offset(struct kbase_context *kctx, ++ union kbase_ioctl_mem_find_cpu_offset *find) ++{ ++ return kbasep_find_enclosing_cpu_mapping_offset( ++ kctx, ++ find->in.cpu_addr, ++ find->in.size, ++ &find->out.offset); ++} ++ ++static int kbase_api_get_context_id(struct kbase_context *kctx, ++ struct kbase_ioctl_get_context_id *info) ++{ ++ info->id = kctx->id; ++ ++ return 0; ++} ++ ++static int kbase_api_tlstream_acquire(struct kbase_context *kctx, ++ struct kbase_ioctl_tlstream_acquire *acquire) ++{ ++ return kbase_tlstream_acquire(kctx, acquire->flags); ++} ++ ++static int kbase_api_tlstream_flush(struct kbase_context *kctx) ++{ ++ kbase_tlstream_flush_streams(); ++ ++ return 0; ++} ++ ++static int kbase_api_mem_commit(struct kbase_context *kctx, ++ struct kbase_ioctl_mem_commit *commit) ++{ ++ return kbase_mem_commit(kctx, commit->gpu_addr, commit->pages); ++} ++ ++static int kbase_api_mem_alias(struct kbase_context *kctx, ++ union kbase_ioctl_mem_alias *alias) ++{ ++ struct base_mem_aliasing_info *ai; ++ void __user *user_addr = NULL; ++ u64 flags; ++ int err; ++ ++ if (alias->in.nents == 0 || alias->in.nents > 2048) ++ return -EINVAL; ++ ++ ai = vmalloc(sizeof(*ai) * alias->in.nents); ++ if (!ai) ++ return -ENOMEM; ++ ++#ifdef CONFIG_COMPAT ++ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) ++ user_addr = ++ compat_ptr(alias->in.aliasing_info.compat_value); ++ else ++#endif ++ user_addr = alias->in.aliasing_info.value; ++ ++ err = copy_from_user(ai, user_addr, sizeof(*ai) * alias->in.nents); ++ if (err) { ++ vfree(ai); ++ return err; ++ } ++ ++ flags = alias->in.flags; ++ ++ alias->out.gpu_va = kbase_mem_alias(kctx, &flags, ++ alias->in.stride, alias->in.nents, ++ ai, &alias->out.va_pages); ++ ++ alias->out.flags = flags; ++ ++ vfree(ai); ++ ++ if (alias->out.gpu_va == 0) ++ return -ENOMEM; ++ ++ return 0; ++} ++ ++static int kbase_api_mem_import(struct kbase_context *kctx, ++ union kbase_ioctl_mem_import *import) ++{ ++ int ret; ++ u64 flags = import->in.flags; ++ void __user *phandle; ++ ++#ifdef CONFIG_COMPAT ++ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) ++ phandle = compat_ptr(import->in.phandle.compat_value); ++ else ++#endif ++ phandle = import->in.phandle.value; ++ ++ ret = kbase_mem_import(kctx, ++ import->in.type, ++ phandle, ++ import->in.padding, ++ &import->out.gpu_va, ++ &import->out.va_pages, ++ &flags); ++ ++ import->out.flags = flags; ++ ++ return ret; ++} ++ ++static int kbase_api_mem_flags_change(struct kbase_context *kctx, ++ struct kbase_ioctl_mem_flags_change *change) ++{ ++ return kbase_mem_flags_change(kctx, change->gpu_va, ++ change->flags, change->mask); ++} ++ ++static int kbase_api_stream_create(struct kbase_context *kctx, ++ struct kbase_ioctl_stream_create *stream) ++{ ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ int fd, ret; ++ ++ /* Name must be NULL-terminated and padded with NULLs, so check last ++ * character is NULL ++ */ ++ if (stream->name[sizeof(stream->name)-1] != 0) ++ return -EINVAL; ++ ++ ret = kbase_sync_fence_stream_create(stream->name, &fd); ++ ++ if (ret) ++ return ret; ++ return fd; ++#else ++ return -ENOENT; ++#endif ++} ++ ++static int kbase_api_fence_validate(struct kbase_context *kctx, ++ struct kbase_ioctl_fence_validate *validate) ++{ ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ return kbase_sync_fence_validate(validate->fd); ++#else ++ return -ENOENT; ++#endif ++} ++ ++static int kbase_api_get_profiling_controls(struct kbase_context *kctx, ++ struct kbase_ioctl_get_profiling_controls *controls) ++{ ++ if (controls->count > FBDUMP_CONTROL_MAX) ++ return -EINVAL; ++ ++ return copy_to_user(controls->buffer.value, ++ &kctx->kbdev->kbase_profiling_controls[ ++ FBDUMP_CONTROL_MIN], ++ controls->count * sizeof(u32)); ++} ++ ++static int kbase_api_mem_profile_add(struct kbase_context *kctx, ++ struct kbase_ioctl_mem_profile_add *data) ++{ ++ char __user *user_buf; ++ char *buf; ++ int err; ++ ++ if (data->len > KBASE_MEM_PROFILE_MAX_BUF_SIZE) { ++ dev_err(kctx->kbdev->dev, "mem_profile_add: buffer too big\n"); ++ return -EINVAL; ++ } ++ ++ buf = kmalloc(data->len, GFP_KERNEL); ++ if (ZERO_OR_NULL_PTR(buf)) ++ return -ENOMEM; ++ ++#ifdef CONFIG_COMPAT ++ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) ++ user_buf = compat_ptr(data->buffer.compat_value); ++ else ++#endif ++ user_buf = data->buffer.value; ++ ++ err = copy_from_user(buf, user_buf, data->len); ++ if (err) { ++ kfree(buf); ++ return err; ++ } ++ ++ return kbasep_mem_profile_debugfs_insert(kctx, buf, data->len); ++} ++ ++static int kbase_api_soft_event_update(struct kbase_context *kctx, ++ struct kbase_ioctl_soft_event_update *update) ++{ ++ if (update->flags != 0) ++ return -EINVAL; ++ ++ return kbase_soft_event_update(kctx, update->event, update->new_status); ++} ++ ++#if MALI_UNIT_TEST ++static int kbase_api_tlstream_test(struct kbase_context *kctx, ++ struct kbase_ioctl_tlstream_test *test) ++{ ++ kbase_tlstream_test( ++ test->tpw_count, ++ test->msg_delay, ++ test->msg_count, ++ test->aux_msg); ++ ++ return 0; ++} ++ ++static int kbase_api_tlstream_stats(struct kbase_context *kctx, ++ struct kbase_ioctl_tlstream_stats *stats) ++{ ++ kbase_tlstream_stats( ++ &stats->bytes_collected, ++ &stats->bytes_generated); ++ ++ return 0; ++} ++#endif /* MALI_UNIT_TEST */ ++ ++#define KBASE_HANDLE_IOCTL(cmd, function) \ ++ case cmd: \ ++ do { \ ++ BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_NONE); \ ++ return function(kctx); \ ++ } while (0) ++ ++#define KBASE_HANDLE_IOCTL_IN(cmd, function, type) \ ++ case cmd: \ ++ do { \ ++ type param; \ ++ int err; \ ++ BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_WRITE); \ ++ BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ ++ err = copy_from_user(¶m, uarg, sizeof(param)); \ ++ if (err) \ ++ return -EFAULT; \ ++ return function(kctx, ¶m); \ ++ } while (0) ++ ++#define KBASE_HANDLE_IOCTL_OUT(cmd, function, type) \ ++ case cmd: \ ++ do { \ ++ type param; \ ++ int ret, err; \ ++ BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_READ); \ ++ BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ ++ ret = function(kctx, ¶m); \ ++ err = copy_to_user(uarg, ¶m, sizeof(param)); \ ++ if (err) \ ++ return -EFAULT; \ ++ return ret; \ ++ } while (0) ++ ++#define KBASE_HANDLE_IOCTL_INOUT(cmd, function, type) \ ++ case cmd: \ ++ do { \ ++ type param; \ ++ int ret, err; \ ++ BUILD_BUG_ON(_IOC_DIR(cmd) != (_IOC_WRITE|_IOC_READ)); \ ++ BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ ++ err = copy_from_user(¶m, uarg, sizeof(param)); \ ++ if (err) \ ++ return -EFAULT; \ ++ ret = function(kctx, ¶m); \ ++ err = copy_to_user(uarg, ¶m, sizeof(param)); \ ++ if (err) \ ++ return -EFAULT; \ ++ return ret; \ ++ } while (0) ++ ++static long kbase_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) ++{ ++ struct kbase_context *kctx = filp->private_data; ++ struct kbase_device *kbdev = kctx->kbdev; ++ void __user *uarg = (void __user *)arg; ++ ++ /* The UK ioctl values overflow the cmd field causing the type to be ++ * incremented ++ */ ++ if (_IOC_TYPE(cmd) == LINUX_UK_BASE_MAGIC+2) ++ return kbase_legacy_ioctl(filp, cmd, arg); ++ ++ /* The UK version check IOCTL doesn't overflow the cmd field, so is ++ * handled separately here ++ */ ++ if (cmd == _IOC(_IOC_READ|_IOC_WRITE, LINUX_UK_BASE_MAGIC, ++ UKP_FUNC_ID_CHECK_VERSION, ++ sizeof(struct uku_version_check_args))) ++ return kbase_legacy_ioctl(filp, cmd, arg); ++ ++ /* Only these ioctls are available until setup is complete */ ++ switch (cmd) { ++ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_VERSION_CHECK, ++ kbase_api_handshake, ++ struct kbase_ioctl_version_check); ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_SET_FLAGS, ++ kbase_api_set_flags, ++ struct kbase_ioctl_set_flags); ++ } ++ ++ /* Block call until version handshake and setup is complete */ ++ if (kctx->api_version == 0 || !atomic_read(&kctx->setup_complete)) ++ return -EINVAL; ++ ++ /* Normal ioctls */ ++ switch (cmd) { ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_JOB_SUBMIT, ++ kbase_api_job_submit, ++ struct kbase_ioctl_job_submit); ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_GET_GPUPROPS, ++ kbase_api_get_gpuprops, ++ struct kbase_ioctl_get_gpuprops); ++ KBASE_HANDLE_IOCTL(KBASE_IOCTL_POST_TERM, ++ kbase_api_post_term); ++ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_ALLOC, ++ kbase_api_mem_alloc, ++ union kbase_ioctl_mem_alloc); ++ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_QUERY, ++ kbase_api_mem_query, ++ union kbase_ioctl_mem_query); ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_FREE, ++ kbase_api_mem_free, ++ struct kbase_ioctl_mem_free); ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_HWCNT_READER_SETUP, ++ kbase_api_hwcnt_reader_setup, ++ struct kbase_ioctl_hwcnt_reader_setup); ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_HWCNT_ENABLE, ++ kbase_api_hwcnt_enable, ++ struct kbase_ioctl_hwcnt_enable); ++ KBASE_HANDLE_IOCTL(KBASE_IOCTL_HWCNT_DUMP, ++ kbase_api_hwcnt_dump); ++ KBASE_HANDLE_IOCTL(KBASE_IOCTL_HWCNT_CLEAR, ++ kbase_api_hwcnt_clear); ++ KBASE_HANDLE_IOCTL_OUT(KBASE_IOCTL_DISJOINT_QUERY, ++ kbase_api_disjoint_query, ++ struct kbase_ioctl_disjoint_query); ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_GET_DDK_VERSION, ++ kbase_api_get_ddk_version, ++ struct kbase_ioctl_get_ddk_version); ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_JIT_INIT, ++ kbase_api_mem_jit_init, ++ struct kbase_ioctl_mem_jit_init); ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_SYNC, ++ kbase_api_mem_sync, ++ struct kbase_ioctl_mem_sync); ++ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_FIND_CPU_OFFSET, ++ kbase_api_mem_find_cpu_offset, ++ union kbase_ioctl_mem_find_cpu_offset); ++ KBASE_HANDLE_IOCTL_OUT(KBASE_IOCTL_GET_CONTEXT_ID, ++ kbase_api_get_context_id, ++ struct kbase_ioctl_get_context_id); ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_TLSTREAM_ACQUIRE, ++ kbase_api_tlstream_acquire, ++ struct kbase_ioctl_tlstream_acquire); ++ KBASE_HANDLE_IOCTL(KBASE_IOCTL_TLSTREAM_FLUSH, ++ kbase_api_tlstream_flush); ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_COMMIT, ++ kbase_api_mem_commit, ++ struct kbase_ioctl_mem_commit); ++ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_ALIAS, ++ kbase_api_mem_alias, ++ union kbase_ioctl_mem_alias); ++ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_IMPORT, ++ kbase_api_mem_import, ++ union kbase_ioctl_mem_import); ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_FLAGS_CHANGE, ++ kbase_api_mem_flags_change, ++ struct kbase_ioctl_mem_flags_change); ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_STREAM_CREATE, ++ kbase_api_stream_create, ++ struct kbase_ioctl_stream_create); ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_FENCE_VALIDATE, ++ kbase_api_fence_validate, ++ struct kbase_ioctl_fence_validate); ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_GET_PROFILING_CONTROLS, ++ kbase_api_get_profiling_controls, ++ struct kbase_ioctl_get_profiling_controls); ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_PROFILE_ADD, ++ kbase_api_mem_profile_add, ++ struct kbase_ioctl_mem_profile_add); ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_SOFT_EVENT_UPDATE, ++ kbase_api_soft_event_update, ++ struct kbase_ioctl_soft_event_update); ++ ++#if MALI_UNIT_TEST ++ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_TLSTREAM_TEST, ++ kbase_api_tlstream_test, ++ struct kbase_ioctl_tlstream_test); ++ KBASE_HANDLE_IOCTL_OUT(KBASE_IOCTL_TLSTREAM_STATS, ++ kbase_api_tlstream_stats, ++ struct kbase_ioctl_tlstream_stats); ++#endif ++ } ++ ++ dev_warn(kbdev->dev, "Unknown ioctl 0x%x nr:%d", cmd, _IOC_NR(cmd)); ++ ++ return -ENOIOCTLCMD; ++} ++ ++static ssize_t kbase_read(struct file *filp, char __user *buf, size_t count, loff_t *f_pos) ++{ ++ struct kbase_context *kctx = filp->private_data; ++ struct base_jd_event_v2 uevent; ++ int out_count = 0; ++ ++ if (count < sizeof(uevent)) ++ return -ENOBUFS; ++ ++ do { ++ while (kbase_event_dequeue(kctx, &uevent)) { ++ if (out_count > 0) ++ goto out; ++ ++ if (filp->f_flags & O_NONBLOCK) ++ return -EAGAIN; ++ ++ if (wait_event_interruptible(kctx->event_queue, ++ kbase_event_pending(kctx)) != 0) ++ return -ERESTARTSYS; ++ } ++ if (uevent.event_code == BASE_JD_EVENT_DRV_TERMINATED) { ++ if (out_count == 0) ++ return -EPIPE; ++ goto out; ++ } ++ ++ if (copy_to_user(buf, &uevent, sizeof(uevent)) != 0) ++ return -EFAULT; ++ ++ buf += sizeof(uevent); ++ out_count++; ++ count -= sizeof(uevent); ++ } while (count >= sizeof(uevent)); ++ ++ out: ++ return out_count * sizeof(uevent); ++} ++ ++static unsigned int kbase_poll(struct file *filp, poll_table *wait) ++{ ++ struct kbase_context *kctx = filp->private_data; ++ ++ poll_wait(filp, &kctx->event_queue, wait); ++ if (kbase_event_pending(kctx)) ++ return POLLIN | POLLRDNORM; ++ ++ return 0; ++} ++ ++void kbase_event_wakeup(struct kbase_context *kctx) ++{ ++ KBASE_DEBUG_ASSERT(kctx); ++ ++ wake_up_interruptible(&kctx->event_queue); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_event_wakeup); ++ ++static int kbase_check_flags(int flags) ++{ ++ /* Enforce that the driver keeps the O_CLOEXEC flag so that execve() always ++ * closes the file descriptor in a child process. ++ */ ++ if (0 == (flags & O_CLOEXEC)) ++ return -EINVAL; ++ ++ return 0; ++} ++ ++ ++/** ++ * align_and_check - Align the specified pointer to the provided alignment and ++ * check that it is still in range. ++ * @gap_end: Highest possible start address for allocation (end of gap in ++ * address space) ++ * @gap_start: Start address of current memory area / gap in address space ++ * @info: vm_unmapped_area_info structure passed to caller, containing ++ * alignment, length and limits for the allocation ++ * @is_shader_code: True if the allocation is for shader code (which has ++ * additional alignment requirements) ++ * ++ * Return: true if gap_end is now aligned correctly and is still in range, ++ * false otherwise ++ */ ++static bool align_and_check(unsigned long *gap_end, unsigned long gap_start, ++ struct vm_unmapped_area_info *info, bool is_shader_code) ++{ ++ /* Compute highest gap address at the desired alignment */ ++ (*gap_end) -= info->length; ++ (*gap_end) -= (*gap_end - info->align_offset) & info->align_mask; ++ ++ if (is_shader_code) { ++ /* Check for 4GB boundary */ ++ if (0 == (*gap_end & BASE_MEM_MASK_4GB)) ++ (*gap_end) -= (info->align_offset ? info->align_offset : ++ info->length); ++ if (0 == ((*gap_end + info->length) & BASE_MEM_MASK_4GB)) ++ (*gap_end) -= (info->align_offset ? info->align_offset : ++ info->length); ++ ++ if (!(*gap_end & BASE_MEM_MASK_4GB) || !((*gap_end + ++ info->length) & BASE_MEM_MASK_4GB)) ++ return false; ++ } ++ ++ ++ if ((*gap_end < info->low_limit) || (*gap_end < gap_start)) ++ return false; ++ ++ ++ return true; ++} ++ ++/* The following function is taken from the kernel and just ++ * renamed. As it's not exported to modules we must copy-paste it here. ++ */ ++ ++static unsigned long kbase_unmapped_area_topdown(struct vm_unmapped_area_info ++ *info, bool is_shader_code) ++{ ++ struct mm_struct *mm = current->mm; ++ struct vm_area_struct *vma; ++ unsigned long length, low_limit, high_limit, gap_start, gap_end; ++ ++ /* Adjust search length to account for worst case alignment overhead */ ++ length = info->length + info->align_mask; ++ if (length < info->length) ++ return -ENOMEM; ++ ++ /* ++ * Adjust search limits by the desired length. ++ * See implementation comment at top of unmapped_area(). ++ */ ++ gap_end = info->high_limit; ++ if (gap_end < length) ++ return -ENOMEM; ++ high_limit = gap_end - length; ++ ++ if (info->low_limit > high_limit) ++ return -ENOMEM; ++ low_limit = info->low_limit + length; ++ ++ /* Check highest gap, which does not precede any rbtree node */ ++ gap_start = mm->highest_vm_end; ++ if (gap_start <= high_limit) { ++ if (align_and_check(&gap_end, gap_start, info, is_shader_code)) ++ return gap_end; ++ } ++ ++ /* Check if rbtree root looks promising */ ++ if (RB_EMPTY_ROOT(&mm->mm_rb)) ++ return -ENOMEM; ++ vma = rb_entry(mm->mm_rb.rb_node, struct vm_area_struct, vm_rb); ++ if (vma->rb_subtree_gap < length) ++ return -ENOMEM; ++ ++ while (true) { ++ /* Visit right subtree if it looks promising */ ++ gap_start = vma->vm_prev ? vma->vm_prev->vm_end : 0; ++ if (gap_start <= high_limit && vma->vm_rb.rb_right) { ++ struct vm_area_struct *right = ++ rb_entry(vma->vm_rb.rb_right, ++ struct vm_area_struct, vm_rb); ++ if (right->rb_subtree_gap >= length) { ++ vma = right; ++ continue; ++ } ++ } ++ ++check_current: ++ /* Check if current node has a suitable gap */ ++ gap_end = vma->vm_start; ++ if (gap_end < low_limit) ++ return -ENOMEM; ++ if (gap_start <= high_limit && gap_end - gap_start >= length) { ++ /* We found a suitable gap. Clip it with the original ++ * high_limit. */ ++ if (gap_end > info->high_limit) ++ gap_end = info->high_limit; ++ ++ if (align_and_check(&gap_end, gap_start, info, ++ is_shader_code)) ++ return gap_end; ++ } ++ ++ /* Visit left subtree if it looks promising */ ++ if (vma->vm_rb.rb_left) { ++ struct vm_area_struct *left = ++ rb_entry(vma->vm_rb.rb_left, ++ struct vm_area_struct, vm_rb); ++ if (left->rb_subtree_gap >= length) { ++ vma = left; ++ continue; ++ } ++ } ++ ++ /* Go back up the rbtree to find next candidate node */ ++ while (true) { ++ struct rb_node *prev = &vma->vm_rb; ++ if (!rb_parent(prev)) ++ return -ENOMEM; ++ vma = rb_entry(rb_parent(prev), ++ struct vm_area_struct, vm_rb); ++ if (prev == vma->vm_rb.rb_right) { ++ gap_start = vma->vm_prev ? ++ vma->vm_prev->vm_end : 0; ++ goto check_current; ++ } ++ } ++ } ++ ++ return -ENOMEM; ++} ++ ++static unsigned long kbase_get_unmapped_area(struct file *filp, ++ const unsigned long addr, const unsigned long len, ++ const unsigned long pgoff, const unsigned long flags) ++{ ++ /* based on get_unmapped_area, but simplified slightly due to that some ++ * values are known in advance */ ++ struct kbase_context *kctx = filp->private_data; ++ struct mm_struct *mm = current->mm; ++ struct vm_unmapped_area_info info; ++ unsigned long align_offset = 0; ++ unsigned long align_mask = 0; ++ unsigned long high_limit = mm->mmap_base; ++ unsigned long low_limit = PAGE_SIZE; ++ int cpu_va_bits = BITS_PER_LONG; ++ int gpu_pc_bits = ++ kctx->kbdev->gpu_props.props.core_props.log2_program_counter_size; ++ bool is_shader_code = false; ++ unsigned long ret; ++ ++ /* err on fixed address */ ++ if ((flags & MAP_FIXED) || addr) ++ return -EINVAL; ++ ++#ifdef CONFIG_64BIT ++ /* too big? */ ++ if (len > TASK_SIZE - SZ_2M) ++ return -ENOMEM; ++ ++ if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { ++ ++ if (kbase_hw_has_feature(kctx->kbdev, ++ BASE_HW_FEATURE_33BIT_VA)) { ++ high_limit = kctx->same_va_end << PAGE_SHIFT; ++ } else { ++ high_limit = min_t(unsigned long, mm->mmap_base, ++ (kctx->same_va_end << PAGE_SHIFT)); ++ if (len >= SZ_2M) { ++ align_offset = SZ_2M; ++ align_mask = SZ_2M - 1; ++ } ++ } ++ ++ low_limit = SZ_2M; ++ } else { ++ cpu_va_bits = 32; ++ } ++#endif /* CONFIG_64BIT */ ++ if ((PFN_DOWN(BASE_MEM_COOKIE_BASE) <= pgoff) && ++ (PFN_DOWN(BASE_MEM_FIRST_FREE_ADDRESS) > pgoff)) { ++ int cookie = pgoff - PFN_DOWN(BASE_MEM_COOKIE_BASE); ++ ++ if (!kctx->pending_regions[cookie]) ++ return -EINVAL; ++ ++ if (!(kctx->pending_regions[cookie]->flags & ++ KBASE_REG_GPU_NX)) { ++ if (cpu_va_bits > gpu_pc_bits) { ++ align_offset = 1ULL << gpu_pc_bits; ++ align_mask = align_offset - 1; ++ is_shader_code = true; ++ } ++ } ++#ifndef CONFIG_64BIT ++ } else { ++ return current->mm->get_unmapped_area(filp, addr, len, pgoff, ++ flags); ++#endif ++ } ++ ++ info.flags = 0; ++ info.length = len; ++ info.low_limit = low_limit; ++ info.high_limit = high_limit; ++ info.align_offset = align_offset; ++ info.align_mask = align_mask; ++ ++ ret = kbase_unmapped_area_topdown(&info, is_shader_code); ++ ++ if (IS_ERR_VALUE(ret) && high_limit == mm->mmap_base && ++ high_limit < (kctx->same_va_end << PAGE_SHIFT)) { ++ /* Retry above mmap_base */ ++ info.low_limit = mm->mmap_base; ++ info.high_limit = min_t(u64, TASK_SIZE, ++ (kctx->same_va_end << PAGE_SHIFT)); ++ ++ ret = kbase_unmapped_area_topdown(&info, is_shader_code); ++ } ++ ++ return ret; ++} ++ ++static const struct file_operations kbase_fops = { ++ .owner = THIS_MODULE, ++ .open = kbase_open, ++ .release = kbase_release, ++ .read = kbase_read, ++ .poll = kbase_poll, ++ .unlocked_ioctl = kbase_ioctl, ++ .compat_ioctl = kbase_ioctl, ++ .mmap = kbase_mmap, ++ .check_flags = kbase_check_flags, ++ .get_unmapped_area = kbase_get_unmapped_area, ++}; ++ ++#ifndef CONFIG_MALI_NO_MALI ++void kbase_os_reg_write(struct kbase_device *kbdev, u16 offset, u32 value) ++{ ++ writel(value, kbdev->reg + offset); ++} ++ ++u32 kbase_os_reg_read(struct kbase_device *kbdev, u16 offset) ++{ ++ return readl(kbdev->reg + offset); ++} ++#endif /* !CONFIG_MALI_NO_MALI */ ++ ++/** ++ * show_policy - Show callback for the power_policy sysfs file. ++ * ++ * This function is called to get the contents of the power_policy sysfs ++ * file. This is a list of the available policies with the currently active one ++ * surrounded by square brackets. ++ * ++ * @dev: The device this sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The output buffer for the sysfs file contents ++ * ++ * Return: The number of bytes output to @buf. ++ */ ++static ssize_t show_policy(struct device *dev, struct device_attribute *attr, char *const buf) ++{ ++ struct kbase_device *kbdev; ++ const struct kbase_pm_policy *current_policy; ++ const struct kbase_pm_policy *const *policy_list; ++ int policy_count; ++ int i; ++ ssize_t ret = 0; ++ ++ kbdev = to_kbase_device(dev); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ current_policy = kbase_pm_get_policy(kbdev); ++ ++ policy_count = kbase_pm_list_policies(&policy_list); ++ ++ for (i = 0; i < policy_count && ret < PAGE_SIZE; i++) { ++ if (policy_list[i] == current_policy) ++ ret += scnprintf(buf + ret, PAGE_SIZE - ret, "[%s] ", policy_list[i]->name); ++ else ++ ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s ", policy_list[i]->name); ++ } ++ ++ if (ret < PAGE_SIZE - 1) { ++ ret += scnprintf(buf + ret, PAGE_SIZE - ret, "\n"); ++ } else { ++ buf[PAGE_SIZE - 2] = '\n'; ++ buf[PAGE_SIZE - 1] = '\0'; ++ ret = PAGE_SIZE - 1; ++ } ++ ++ return ret; ++} ++ ++/** ++ * set_policy - Store callback for the power_policy sysfs file. ++ * ++ * This function is called when the power_policy sysfs file is written to. ++ * It matches the requested policy against the available policies and if a ++ * matching policy is found calls kbase_pm_set_policy() to change the ++ * policy. ++ * ++ * @dev: The device with sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The value written to the sysfs file ++ * @count: The number of bytes written to the sysfs file ++ * ++ * Return: @count if the function succeeded. An error code on failure. ++ */ ++static ssize_t set_policy(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) ++{ ++ struct kbase_device *kbdev; ++ const struct kbase_pm_policy *new_policy = NULL; ++ const struct kbase_pm_policy *const *policy_list; ++ int policy_count; ++ int i; ++ ++ kbdev = to_kbase_device(dev); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ policy_count = kbase_pm_list_policies(&policy_list); ++ ++ for (i = 0; i < policy_count; i++) { ++ if (sysfs_streq(policy_list[i]->name, buf)) { ++ new_policy = policy_list[i]; ++ break; ++ } ++ } ++ ++ if (!new_policy) { ++ dev_err(dev, "power_policy: policy not found\n"); ++ return -EINVAL; ++ } ++ ++ kbase_pm_set_policy(kbdev, new_policy); ++ ++ return count; ++} ++ ++/* ++ * The sysfs file power_policy. ++ * ++ * This is used for obtaining information about the available policies, ++ * determining which policy is currently active, and changing the active ++ * policy. ++ */ ++static DEVICE_ATTR(power_policy, S_IRUGO | S_IWUSR, show_policy, set_policy); ++ ++/** ++ * show_ca_policy - Show callback for the core_availability_policy sysfs file. ++ * ++ * This function is called to get the contents of the core_availability_policy ++ * sysfs file. This is a list of the available policies with the currently ++ * active one surrounded by square brackets. ++ * ++ * @dev: The device this sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The output buffer for the sysfs file contents ++ * ++ * Return: The number of bytes output to @buf. ++ */ ++static ssize_t show_ca_policy(struct device *dev, struct device_attribute *attr, char * const buf) ++{ ++ struct kbase_device *kbdev; ++ const struct kbase_pm_ca_policy *current_policy; ++ const struct kbase_pm_ca_policy *const *policy_list; ++ int policy_count; ++ int i; ++ ssize_t ret = 0; ++ ++ kbdev = to_kbase_device(dev); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ current_policy = kbase_pm_ca_get_policy(kbdev); ++ ++ policy_count = kbase_pm_ca_list_policies(&policy_list); ++ ++ for (i = 0; i < policy_count && ret < PAGE_SIZE; i++) { ++ if (policy_list[i] == current_policy) ++ ret += scnprintf(buf + ret, PAGE_SIZE - ret, "[%s] ", policy_list[i]->name); ++ else ++ ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s ", policy_list[i]->name); ++ } ++ ++ if (ret < PAGE_SIZE - 1) { ++ ret += scnprintf(buf + ret, PAGE_SIZE - ret, "\n"); ++ } else { ++ buf[PAGE_SIZE - 2] = '\n'; ++ buf[PAGE_SIZE - 1] = '\0'; ++ ret = PAGE_SIZE - 1; ++ } ++ ++ return ret; ++} ++ ++/** ++ * set_ca_policy - Store callback for the core_availability_policy sysfs file. ++ * ++ * This function is called when the core_availability_policy sysfs file is ++ * written to. It matches the requested policy against the available policies ++ * and if a matching policy is found calls kbase_pm_set_policy() to change ++ * the policy. ++ * ++ * @dev: The device with sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The value written to the sysfs file ++ * @count: The number of bytes written to the sysfs file ++ * ++ * Return: @count if the function succeeded. An error code on failure. ++ */ ++static ssize_t set_ca_policy(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) ++{ ++ struct kbase_device *kbdev; ++ const struct kbase_pm_ca_policy *new_policy = NULL; ++ const struct kbase_pm_ca_policy *const *policy_list; ++ int policy_count; ++ int i; ++ ++ kbdev = to_kbase_device(dev); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ policy_count = kbase_pm_ca_list_policies(&policy_list); ++ ++ for (i = 0; i < policy_count; i++) { ++ if (sysfs_streq(policy_list[i]->name, buf)) { ++ new_policy = policy_list[i]; ++ break; ++ } ++ } ++ ++ if (!new_policy) { ++ dev_err(dev, "core_availability_policy: policy not found\n"); ++ return -EINVAL; ++ } ++ ++ kbase_pm_ca_set_policy(kbdev, new_policy); ++ ++ return count; ++} ++ ++/* ++ * The sysfs file core_availability_policy ++ * ++ * This is used for obtaining information about the available policies, ++ * determining which policy is currently active, and changing the active ++ * policy. ++ */ ++static DEVICE_ATTR(core_availability_policy, S_IRUGO | S_IWUSR, show_ca_policy, set_ca_policy); ++ ++/* ++ * show_core_mask - Show callback for the core_mask sysfs file. ++ * ++ * This function is called to get the contents of the core_mask sysfs file. ++ * ++ * @dev: The device this sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The output buffer for the sysfs file contents ++ * ++ * Return: The number of bytes output to @buf. ++ */ ++static ssize_t show_core_mask(struct device *dev, struct device_attribute *attr, char * const buf) ++{ ++ struct kbase_device *kbdev; ++ ssize_t ret = 0; ++ ++ kbdev = to_kbase_device(dev); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ ret += scnprintf(buf + ret, PAGE_SIZE - ret, ++ "Current core mask (JS0) : 0x%llX\n", ++ kbdev->pm.debug_core_mask[0]); ++ ret += scnprintf(buf + ret, PAGE_SIZE - ret, ++ "Current core mask (JS1) : 0x%llX\n", ++ kbdev->pm.debug_core_mask[1]); ++ ret += scnprintf(buf + ret, PAGE_SIZE - ret, ++ "Current core mask (JS2) : 0x%llX\n", ++ kbdev->pm.debug_core_mask[2]); ++ ret += scnprintf(buf + ret, PAGE_SIZE - ret, ++ "Available core mask : 0x%llX\n", ++ kbdev->gpu_props.props.raw_props.shader_present); ++ ++ return ret; ++} ++ ++/** ++ * set_core_mask - Store callback for the core_mask sysfs file. ++ * ++ * This function is called when the core_mask sysfs file is written to. ++ * ++ * @dev: The device with sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The value written to the sysfs file ++ * @count: The number of bytes written to the sysfs file ++ * ++ * Return: @count if the function succeeded. An error code on failure. ++ */ ++static ssize_t set_core_mask(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) ++{ ++ struct kbase_device *kbdev; ++ u64 new_core_mask[3]; ++ int items; ++ ++ kbdev = to_kbase_device(dev); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ items = sscanf(buf, "%llx %llx %llx", ++ &new_core_mask[0], &new_core_mask[1], ++ &new_core_mask[2]); ++ ++ if (items == 1) ++ new_core_mask[1] = new_core_mask[2] = new_core_mask[0]; ++ ++ if (items == 1 || items == 3) { ++ u64 shader_present = ++ kbdev->gpu_props.props.raw_props.shader_present; ++ u64 group0_core_mask = ++ kbdev->gpu_props.props.coherency_info.group[0]. ++ core_mask; ++ ++ if ((new_core_mask[0] & shader_present) != new_core_mask[0] || ++ !(new_core_mask[0] & group0_core_mask) || ++ (new_core_mask[1] & shader_present) != ++ new_core_mask[1] || ++ !(new_core_mask[1] & group0_core_mask) || ++ (new_core_mask[2] & shader_present) != ++ new_core_mask[2] || ++ !(new_core_mask[2] & group0_core_mask)) { ++ dev_err(dev, "power_policy: invalid core specification\n"); ++ return -EINVAL; ++ } ++ ++ if (kbdev->pm.debug_core_mask[0] != new_core_mask[0] || ++ kbdev->pm.debug_core_mask[1] != ++ new_core_mask[1] || ++ kbdev->pm.debug_core_mask[2] != ++ new_core_mask[2]) { ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ kbase_pm_set_debug_core_mask(kbdev, new_core_mask[0], ++ new_core_mask[1], new_core_mask[2]); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ } ++ ++ return count; ++ } ++ ++ dev_err(kbdev->dev, "Couldn't process set_core_mask write operation.\n" ++ "Use format \n" ++ "or \n"); ++ return -EINVAL; ++} ++ ++/* ++ * The sysfs file core_mask. ++ * ++ * This is used to restrict shader core availability for debugging purposes. ++ * Reading it will show the current core mask and the mask of cores available. ++ * Writing to it will set the current core mask. ++ */ ++static DEVICE_ATTR(core_mask, S_IRUGO | S_IWUSR, show_core_mask, set_core_mask); ++ ++/** ++ * set_soft_job_timeout - Store callback for the soft_job_timeout sysfs ++ * file. ++ * ++ * @dev: The device this sysfs file is for. ++ * @attr: The attributes of the sysfs file. ++ * @buf: The value written to the sysfs file. ++ * @count: The number of bytes written to the sysfs file. ++ * ++ * This allows setting the timeout for software jobs. Waiting soft event wait ++ * jobs will be cancelled after this period expires, while soft fence wait jobs ++ * will print debug information if the fence debug feature is enabled. ++ * ++ * This is expressed in milliseconds. ++ * ++ * Return: count if the function succeeded. An error code on failure. ++ */ ++static ssize_t set_soft_job_timeout(struct device *dev, ++ struct device_attribute *attr, ++ const char *buf, size_t count) ++{ ++ struct kbase_device *kbdev; ++ int soft_job_timeout_ms; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ if ((kstrtoint(buf, 0, &soft_job_timeout_ms) != 0) || ++ (soft_job_timeout_ms <= 0)) ++ return -EINVAL; ++ ++ atomic_set(&kbdev->js_data.soft_job_timeout_ms, ++ soft_job_timeout_ms); ++ ++ return count; ++} ++ ++/** ++ * show_soft_job_timeout - Show callback for the soft_job_timeout sysfs ++ * file. ++ * ++ * This will return the timeout for the software jobs. ++ * ++ * @dev: The device this sysfs file is for. ++ * @attr: The attributes of the sysfs file. ++ * @buf: The output buffer for the sysfs file contents. ++ * ++ * Return: The number of bytes output to buf. ++ */ ++static ssize_t show_soft_job_timeout(struct device *dev, ++ struct device_attribute *attr, ++ char * const buf) ++{ ++ struct kbase_device *kbdev; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ return scnprintf(buf, PAGE_SIZE, "%i\n", ++ atomic_read(&kbdev->js_data.soft_job_timeout_ms)); ++} ++ ++static DEVICE_ATTR(soft_job_timeout, S_IRUGO | S_IWUSR, ++ show_soft_job_timeout, set_soft_job_timeout); ++ ++static u32 timeout_ms_to_ticks(struct kbase_device *kbdev, long timeout_ms, ++ int default_ticks, u32 old_ticks) ++{ ++ if (timeout_ms > 0) { ++ u64 ticks = timeout_ms * 1000000ULL; ++ do_div(ticks, kbdev->js_data.scheduling_period_ns); ++ if (!ticks) ++ return 1; ++ return ticks; ++ } else if (timeout_ms < 0) { ++ return default_ticks; ++ } else { ++ return old_ticks; ++ } ++} ++ ++/** ++ * set_js_timeouts - Store callback for the js_timeouts sysfs file. ++ * ++ * This function is called to get the contents of the js_timeouts sysfs ++ * file. This file contains five values separated by whitespace. The values ++ * are basically the same as %JS_SOFT_STOP_TICKS, %JS_HARD_STOP_TICKS_SS, ++ * %JS_HARD_STOP_TICKS_DUMPING, %JS_RESET_TICKS_SS, %JS_RESET_TICKS_DUMPING ++ * configuration values (in that order), with the difference that the js_timeout ++ * values are expressed in MILLISECONDS. ++ * ++ * The js_timeouts sysfile file allows the current values in ++ * use by the job scheduler to get override. Note that a value needs to ++ * be other than 0 for it to override the current job scheduler value. ++ * ++ * @dev: The device with sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The value written to the sysfs file ++ * @count: The number of bytes written to the sysfs file ++ * ++ * Return: @count if the function succeeded. An error code on failure. ++ */ ++static ssize_t set_js_timeouts(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) ++{ ++ struct kbase_device *kbdev; ++ int items; ++ long js_soft_stop_ms; ++ long js_soft_stop_ms_cl; ++ long js_hard_stop_ms_ss; ++ long js_hard_stop_ms_cl; ++ long js_hard_stop_ms_dumping; ++ long js_reset_ms_ss; ++ long js_reset_ms_cl; ++ long js_reset_ms_dumping; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ items = sscanf(buf, "%ld %ld %ld %ld %ld %ld %ld %ld", ++ &js_soft_stop_ms, &js_soft_stop_ms_cl, ++ &js_hard_stop_ms_ss, &js_hard_stop_ms_cl, ++ &js_hard_stop_ms_dumping, &js_reset_ms_ss, ++ &js_reset_ms_cl, &js_reset_ms_dumping); ++ ++ if (items == 8) { ++ struct kbasep_js_device_data *js_data = &kbdev->js_data; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++#define UPDATE_TIMEOUT(ticks_name, ms_name, default) do {\ ++ js_data->ticks_name = timeout_ms_to_ticks(kbdev, ms_name, \ ++ default, js_data->ticks_name); \ ++ dev_dbg(kbdev->dev, "Overriding " #ticks_name \ ++ " with %lu ticks (%lu ms)\n", \ ++ (unsigned long)js_data->ticks_name, \ ++ ms_name); \ ++ } while (0) ++ ++ UPDATE_TIMEOUT(soft_stop_ticks, js_soft_stop_ms, ++ DEFAULT_JS_SOFT_STOP_TICKS); ++ UPDATE_TIMEOUT(soft_stop_ticks_cl, js_soft_stop_ms_cl, ++ DEFAULT_JS_SOFT_STOP_TICKS_CL); ++ UPDATE_TIMEOUT(hard_stop_ticks_ss, js_hard_stop_ms_ss, ++ kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408) ? ++ DEFAULT_JS_HARD_STOP_TICKS_SS_8408 : ++ DEFAULT_JS_HARD_STOP_TICKS_SS); ++ UPDATE_TIMEOUT(hard_stop_ticks_cl, js_hard_stop_ms_cl, ++ DEFAULT_JS_HARD_STOP_TICKS_CL); ++ UPDATE_TIMEOUT(hard_stop_ticks_dumping, ++ js_hard_stop_ms_dumping, ++ DEFAULT_JS_HARD_STOP_TICKS_DUMPING); ++ UPDATE_TIMEOUT(gpu_reset_ticks_ss, js_reset_ms_ss, ++ kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408) ? ++ DEFAULT_JS_RESET_TICKS_SS_8408 : ++ DEFAULT_JS_RESET_TICKS_SS); ++ UPDATE_TIMEOUT(gpu_reset_ticks_cl, js_reset_ms_cl, ++ DEFAULT_JS_RESET_TICKS_CL); ++ UPDATE_TIMEOUT(gpu_reset_ticks_dumping, js_reset_ms_dumping, ++ DEFAULT_JS_RESET_TICKS_DUMPING); ++ ++ kbase_js_set_timeouts(kbdev); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ return count; ++ } ++ ++ dev_err(kbdev->dev, "Couldn't process js_timeouts write operation.\n" ++ "Use format \n" ++ "Write 0 for no change, -1 to restore default timeout\n"); ++ return -EINVAL; ++} ++ ++static unsigned long get_js_timeout_in_ms( ++ u32 scheduling_period_ns, ++ u32 ticks) ++{ ++ u64 ms = (u64)ticks * scheduling_period_ns; ++ ++ do_div(ms, 1000000UL); ++ return ms; ++} ++ ++/** ++ * show_js_timeouts - Show callback for the js_timeouts sysfs file. ++ * ++ * This function is called to get the contents of the js_timeouts sysfs ++ * file. It returns the last set values written to the js_timeouts sysfs file. ++ * If the file didn't get written yet, the values will be current setting in ++ * use. ++ * @dev: The device this sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The output buffer for the sysfs file contents ++ * ++ * Return: The number of bytes output to @buf. ++ */ ++static ssize_t show_js_timeouts(struct device *dev, struct device_attribute *attr, char * const buf) ++{ ++ struct kbase_device *kbdev; ++ ssize_t ret; ++ unsigned long js_soft_stop_ms; ++ unsigned long js_soft_stop_ms_cl; ++ unsigned long js_hard_stop_ms_ss; ++ unsigned long js_hard_stop_ms_cl; ++ unsigned long js_hard_stop_ms_dumping; ++ unsigned long js_reset_ms_ss; ++ unsigned long js_reset_ms_cl; ++ unsigned long js_reset_ms_dumping; ++ u32 scheduling_period_ns; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ scheduling_period_ns = kbdev->js_data.scheduling_period_ns; ++ ++#define GET_TIMEOUT(name) get_js_timeout_in_ms(\ ++ scheduling_period_ns, \ ++ kbdev->js_data.name) ++ ++ js_soft_stop_ms = GET_TIMEOUT(soft_stop_ticks); ++ js_soft_stop_ms_cl = GET_TIMEOUT(soft_stop_ticks_cl); ++ js_hard_stop_ms_ss = GET_TIMEOUT(hard_stop_ticks_ss); ++ js_hard_stop_ms_cl = GET_TIMEOUT(hard_stop_ticks_cl); ++ js_hard_stop_ms_dumping = GET_TIMEOUT(hard_stop_ticks_dumping); ++ js_reset_ms_ss = GET_TIMEOUT(gpu_reset_ticks_ss); ++ js_reset_ms_cl = GET_TIMEOUT(gpu_reset_ticks_cl); ++ js_reset_ms_dumping = GET_TIMEOUT(gpu_reset_ticks_dumping); ++ ++#undef GET_TIMEOUT ++ ++ ret = scnprintf(buf, PAGE_SIZE, "%lu %lu %lu %lu %lu %lu %lu %lu\n", ++ js_soft_stop_ms, js_soft_stop_ms_cl, ++ js_hard_stop_ms_ss, js_hard_stop_ms_cl, ++ js_hard_stop_ms_dumping, js_reset_ms_ss, ++ js_reset_ms_cl, js_reset_ms_dumping); ++ ++ if (ret >= PAGE_SIZE) { ++ buf[PAGE_SIZE - 2] = '\n'; ++ buf[PAGE_SIZE - 1] = '\0'; ++ ret = PAGE_SIZE - 1; ++ } ++ ++ return ret; ++} ++ ++/* ++ * The sysfs file js_timeouts. ++ * ++ * This is used to override the current job scheduler values for ++ * JS_STOP_STOP_TICKS_SS ++ * JS_STOP_STOP_TICKS_CL ++ * JS_HARD_STOP_TICKS_SS ++ * JS_HARD_STOP_TICKS_CL ++ * JS_HARD_STOP_TICKS_DUMPING ++ * JS_RESET_TICKS_SS ++ * JS_RESET_TICKS_CL ++ * JS_RESET_TICKS_DUMPING. ++ */ ++static DEVICE_ATTR(js_timeouts, S_IRUGO | S_IWUSR, show_js_timeouts, set_js_timeouts); ++ ++static u32 get_new_js_timeout( ++ u32 old_period, ++ u32 old_ticks, ++ u32 new_scheduling_period_ns) ++{ ++ u64 ticks = (u64)old_period * (u64)old_ticks; ++ do_div(ticks, new_scheduling_period_ns); ++ return ticks?ticks:1; ++} ++ ++/** ++ * set_js_scheduling_period - Store callback for the js_scheduling_period sysfs ++ * file ++ * @dev: The device the sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The value written to the sysfs file ++ * @count: The number of bytes written to the sysfs file ++ * ++ * This function is called when the js_scheduling_period sysfs file is written ++ * to. It checks the data written, and if valid updates the js_scheduling_period ++ * value ++ * ++ * Return: @count if the function succeeded. An error code on failure. ++ */ ++static ssize_t set_js_scheduling_period(struct device *dev, ++ struct device_attribute *attr, const char *buf, size_t count) ++{ ++ struct kbase_device *kbdev; ++ int ret; ++ unsigned int js_scheduling_period; ++ u32 new_scheduling_period_ns; ++ u32 old_period; ++ struct kbasep_js_device_data *js_data; ++ unsigned long flags; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ js_data = &kbdev->js_data; ++ ++ ret = kstrtouint(buf, 0, &js_scheduling_period); ++ if (ret || !js_scheduling_period) { ++ dev_err(kbdev->dev, "Couldn't process js_scheduling_period write operation.\n" ++ "Use format \n"); ++ return -EINVAL; ++ } ++ ++ new_scheduling_period_ns = js_scheduling_period * 1000000; ++ ++ /* Update scheduling timeouts */ ++ mutex_lock(&js_data->runpool_mutex); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ /* If no contexts have been scheduled since js_timeouts was last written ++ * to, the new timeouts might not have been latched yet. So check if an ++ * update is pending and use the new values if necessary. */ ++ ++ /* Use previous 'new' scheduling period as a base if present. */ ++ old_period = js_data->scheduling_period_ns; ++ ++#define SET_TIMEOUT(name) \ ++ (js_data->name = get_new_js_timeout(\ ++ old_period, \ ++ kbdev->js_data.name, \ ++ new_scheduling_period_ns)) ++ ++ SET_TIMEOUT(soft_stop_ticks); ++ SET_TIMEOUT(soft_stop_ticks_cl); ++ SET_TIMEOUT(hard_stop_ticks_ss); ++ SET_TIMEOUT(hard_stop_ticks_cl); ++ SET_TIMEOUT(hard_stop_ticks_dumping); ++ SET_TIMEOUT(gpu_reset_ticks_ss); ++ SET_TIMEOUT(gpu_reset_ticks_cl); ++ SET_TIMEOUT(gpu_reset_ticks_dumping); ++ ++#undef SET_TIMEOUT ++ ++ js_data->scheduling_period_ns = new_scheduling_period_ns; ++ ++ kbase_js_set_timeouts(kbdev); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&js_data->runpool_mutex); ++ ++ dev_dbg(kbdev->dev, "JS scheduling period: %dms\n", ++ js_scheduling_period); ++ ++ return count; ++} ++ ++/** ++ * show_js_scheduling_period - Show callback for the js_scheduling_period sysfs ++ * entry. ++ * @dev: The device this sysfs file is for. ++ * @attr: The attributes of the sysfs file. ++ * @buf: The output buffer to receive the GPU information. ++ * ++ * This function is called to get the current period used for the JS scheduling ++ * period. ++ * ++ * Return: The number of bytes output to @buf. ++ */ ++static ssize_t show_js_scheduling_period(struct device *dev, ++ struct device_attribute *attr, char * const buf) ++{ ++ struct kbase_device *kbdev; ++ u32 period; ++ ssize_t ret; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ period = kbdev->js_data.scheduling_period_ns; ++ ++ ret = scnprintf(buf, PAGE_SIZE, "%d\n", ++ period / 1000000); ++ ++ return ret; ++} ++ ++static DEVICE_ATTR(js_scheduling_period, S_IRUGO | S_IWUSR, ++ show_js_scheduling_period, set_js_scheduling_period); ++ ++#if !MALI_CUSTOMER_RELEASE ++/** ++ * set_force_replay - Store callback for the force_replay sysfs file. ++ * ++ * @dev: The device with sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The value written to the sysfs file ++ * @count: The number of bytes written to the sysfs file ++ * ++ * Return: @count if the function succeeded. An error code on failure. ++ */ ++static ssize_t set_force_replay(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) ++{ ++ struct kbase_device *kbdev; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ if (!strncmp("limit=", buf, MIN(6, count))) { ++ int force_replay_limit; ++ int items = sscanf(buf, "limit=%u", &force_replay_limit); ++ ++ if (items == 1) { ++ kbdev->force_replay_random = false; ++ kbdev->force_replay_limit = force_replay_limit; ++ kbdev->force_replay_count = 0; ++ ++ return count; ++ } ++ } else if (!strncmp("random_limit", buf, MIN(12, count))) { ++ kbdev->force_replay_random = true; ++ kbdev->force_replay_count = 0; ++ ++ return count; ++ } else if (!strncmp("norandom_limit", buf, MIN(14, count))) { ++ kbdev->force_replay_random = false; ++ kbdev->force_replay_limit = KBASEP_FORCE_REPLAY_DISABLED; ++ kbdev->force_replay_count = 0; ++ ++ return count; ++ } else if (!strncmp("core_req=", buf, MIN(9, count))) { ++ unsigned int core_req; ++ int items = sscanf(buf, "core_req=%x", &core_req); ++ ++ if (items == 1) { ++ kbdev->force_replay_core_req = (base_jd_core_req)core_req; ++ ++ return count; ++ } ++ } ++ dev_err(kbdev->dev, "Couldn't process force_replay write operation.\nPossible settings: limit=, random_limit, norandom_limit, core_req=\n"); ++ return -EINVAL; ++} ++ ++/** ++ * show_force_replay - Show callback for the force_replay sysfs file. ++ * ++ * This function is called to get the contents of the force_replay sysfs ++ * file. It returns the last set value written to the force_replay sysfs file. ++ * If the file didn't get written yet, the values will be 0. ++ * ++ * @dev: The device this sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The output buffer for the sysfs file contents ++ * ++ * Return: The number of bytes output to @buf. ++ */ ++static ssize_t show_force_replay(struct device *dev, ++ struct device_attribute *attr, char * const buf) ++{ ++ struct kbase_device *kbdev; ++ ssize_t ret; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ if (kbdev->force_replay_random) ++ ret = scnprintf(buf, PAGE_SIZE, ++ "limit=0\nrandom_limit\ncore_req=%x\n", ++ kbdev->force_replay_core_req); ++ else ++ ret = scnprintf(buf, PAGE_SIZE, ++ "limit=%u\nnorandom_limit\ncore_req=%x\n", ++ kbdev->force_replay_limit, ++ kbdev->force_replay_core_req); ++ ++ if (ret >= PAGE_SIZE) { ++ buf[PAGE_SIZE - 2] = '\n'; ++ buf[PAGE_SIZE - 1] = '\0'; ++ ret = PAGE_SIZE - 1; ++ } ++ ++ return ret; ++} ++ ++/* ++ * The sysfs file force_replay. ++ */ ++static DEVICE_ATTR(force_replay, S_IRUGO | S_IWUSR, show_force_replay, ++ set_force_replay); ++#endif /* !MALI_CUSTOMER_RELEASE */ ++ ++#ifdef CONFIG_MALI_DEBUG ++static ssize_t set_js_softstop_always(struct device *dev, ++ struct device_attribute *attr, const char *buf, size_t count) ++{ ++ struct kbase_device *kbdev; ++ int ret; ++ int softstop_always; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ ret = kstrtoint(buf, 0, &softstop_always); ++ if (ret || ((softstop_always != 0) && (softstop_always != 1))) { ++ dev_err(kbdev->dev, "Couldn't process js_softstop_always write operation.\n" ++ "Use format \n"); ++ return -EINVAL; ++ } ++ ++ kbdev->js_data.softstop_always = (bool) softstop_always; ++ dev_dbg(kbdev->dev, "Support for softstop on a single context: %s\n", ++ (kbdev->js_data.softstop_always) ? ++ "Enabled" : "Disabled"); ++ return count; ++} ++ ++static ssize_t show_js_softstop_always(struct device *dev, ++ struct device_attribute *attr, char * const buf) ++{ ++ struct kbase_device *kbdev; ++ ssize_t ret; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ ret = scnprintf(buf, PAGE_SIZE, "%d\n", kbdev->js_data.softstop_always); ++ ++ if (ret >= PAGE_SIZE) { ++ buf[PAGE_SIZE - 2] = '\n'; ++ buf[PAGE_SIZE - 1] = '\0'; ++ ret = PAGE_SIZE - 1; ++ } ++ ++ return ret; ++} ++ ++/* ++ * By default, soft-stops are disabled when only a single context is present. ++ * The ability to enable soft-stop when only a single context is present can be ++ * used for debug and unit-testing purposes. ++ * (see CL t6xx_stress_1 unit-test as an example whereby this feature is used.) ++ */ ++static DEVICE_ATTR(js_softstop_always, S_IRUGO | S_IWUSR, show_js_softstop_always, set_js_softstop_always); ++#endif /* CONFIG_MALI_DEBUG */ ++ ++#ifdef CONFIG_MALI_DEBUG ++typedef void (kbasep_debug_command_func) (struct kbase_device *); ++ ++enum kbasep_debug_command_code { ++ KBASEP_DEBUG_COMMAND_DUMPTRACE, ++ ++ /* This must be the last enum */ ++ KBASEP_DEBUG_COMMAND_COUNT ++}; ++ ++struct kbasep_debug_command { ++ char *str; ++ kbasep_debug_command_func *func; ++}; ++ ++/* Debug commands supported by the driver */ ++static const struct kbasep_debug_command debug_commands[] = { ++ { ++ .str = "dumptrace", ++ .func = &kbasep_trace_dump, ++ } ++}; ++ ++/** ++ * show_debug - Show callback for the debug_command sysfs file. ++ * ++ * This function is called to get the contents of the debug_command sysfs ++ * file. This is a list of the available debug commands, separated by newlines. ++ * ++ * @dev: The device this sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The output buffer for the sysfs file contents ++ * ++ * Return: The number of bytes output to @buf. ++ */ ++static ssize_t show_debug(struct device *dev, struct device_attribute *attr, char * const buf) ++{ ++ struct kbase_device *kbdev; ++ int i; ++ ssize_t ret = 0; ++ ++ kbdev = to_kbase_device(dev); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ for (i = 0; i < KBASEP_DEBUG_COMMAND_COUNT && ret < PAGE_SIZE; i++) ++ ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s\n", debug_commands[i].str); ++ ++ if (ret >= PAGE_SIZE) { ++ buf[PAGE_SIZE - 2] = '\n'; ++ buf[PAGE_SIZE - 1] = '\0'; ++ ret = PAGE_SIZE - 1; ++ } ++ ++ return ret; ++} ++ ++/** ++ * issue_debug - Store callback for the debug_command sysfs file. ++ * ++ * This function is called when the debug_command sysfs file is written to. ++ * It matches the requested command against the available commands, and if ++ * a matching command is found calls the associated function from ++ * @debug_commands to issue the command. ++ * ++ * @dev: The device with sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The value written to the sysfs file ++ * @count: The number of bytes written to the sysfs file ++ * ++ * Return: @count if the function succeeded. An error code on failure. ++ */ ++static ssize_t issue_debug(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) ++{ ++ struct kbase_device *kbdev; ++ int i; ++ ++ kbdev = to_kbase_device(dev); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ for (i = 0; i < KBASEP_DEBUG_COMMAND_COUNT; i++) { ++ if (sysfs_streq(debug_commands[i].str, buf)) { ++ debug_commands[i].func(kbdev); ++ return count; ++ } ++ } ++ ++ /* Debug Command not found */ ++ dev_err(dev, "debug_command: command not known\n"); ++ return -EINVAL; ++} ++ ++/* The sysfs file debug_command. ++ * ++ * This is used to issue general debug commands to the device driver. ++ * Reading it will produce a list of debug commands, separated by newlines. ++ * Writing to it with one of those commands will issue said command. ++ */ ++static DEVICE_ATTR(debug_command, S_IRUGO | S_IWUSR, show_debug, issue_debug); ++#endif /* CONFIG_MALI_DEBUG */ ++ ++/** ++ * kbase_show_gpuinfo - Show callback for the gpuinfo sysfs entry. ++ * @dev: The device this sysfs file is for. ++ * @attr: The attributes of the sysfs file. ++ * @buf: The output buffer to receive the GPU information. ++ * ++ * This function is called to get a description of the present Mali ++ * GPU via the gpuinfo sysfs entry. This includes the GPU family, the ++ * number of cores, the hardware version and the raw product id. For ++ * example ++ * ++ * Mali-T60x MP4 r0p0 0x6956 ++ * ++ * Return: The number of bytes output to @buf. ++ */ ++static ssize_t kbase_show_gpuinfo(struct device *dev, ++ struct device_attribute *attr, char *buf) ++{ ++ static const struct gpu_product_id_name { ++ unsigned id; ++ char *name; ++ } gpu_product_id_names[] = { ++ { .id = GPU_ID_PI_T60X, .name = "Mali-T60x" }, ++ { .id = GPU_ID_PI_T62X, .name = "Mali-T62x" }, ++ { .id = GPU_ID_PI_T72X, .name = "Mali-T72x" }, ++ { .id = GPU_ID_PI_T76X, .name = "Mali-T76x" }, ++ { .id = GPU_ID_PI_T82X, .name = "Mali-T82x" }, ++ { .id = GPU_ID_PI_T83X, .name = "Mali-T83x" }, ++ { .id = GPU_ID_PI_T86X, .name = "Mali-T86x" }, ++ { .id = GPU_ID_PI_TFRX, .name = "Mali-T88x" }, ++ { .id = GPU_ID2_PRODUCT_TMIX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, ++ .name = "Mali-G71" }, ++ { .id = GPU_ID2_PRODUCT_THEX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, ++ .name = "Mali-THEx" }, ++ { .id = GPU_ID2_PRODUCT_TSIX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, ++ .name = "Mali-G51" }, ++ }; ++ const char *product_name = "(Unknown Mali GPU)"; ++ struct kbase_device *kbdev; ++ u32 gpu_id; ++ unsigned product_id, product_id_mask; ++ unsigned i; ++ bool is_new_format; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; ++ product_id = gpu_id >> GPU_ID_VERSION_PRODUCT_ID_SHIFT; ++ is_new_format = GPU_ID_IS_NEW_FORMAT(product_id); ++ product_id_mask = ++ (is_new_format ? ++ GPU_ID2_PRODUCT_MODEL : ++ GPU_ID_VERSION_PRODUCT_ID) >> ++ GPU_ID_VERSION_PRODUCT_ID_SHIFT; ++ ++ for (i = 0; i < ARRAY_SIZE(gpu_product_id_names); ++i) { ++ const struct gpu_product_id_name *p = &gpu_product_id_names[i]; ++ ++ if ((GPU_ID_IS_NEW_FORMAT(p->id) == is_new_format) && ++ (p->id & product_id_mask) == ++ (product_id & product_id_mask)) { ++ product_name = p->name; ++ break; ++ } ++ } ++ ++ return scnprintf(buf, PAGE_SIZE, "%s %d cores r%dp%d 0x%04X\n", ++ product_name, kbdev->gpu_props.num_cores, ++ (gpu_id & GPU_ID_VERSION_MAJOR) >> GPU_ID_VERSION_MAJOR_SHIFT, ++ (gpu_id & GPU_ID_VERSION_MINOR) >> GPU_ID_VERSION_MINOR_SHIFT, ++ product_id); ++} ++static DEVICE_ATTR(gpuinfo, S_IRUGO, kbase_show_gpuinfo, NULL); ++ ++/** ++ * set_dvfs_period - Store callback for the dvfs_period sysfs file. ++ * @dev: The device with sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The value written to the sysfs file ++ * @count: The number of bytes written to the sysfs file ++ * ++ * This function is called when the dvfs_period sysfs file is written to. It ++ * checks the data written, and if valid updates the DVFS period variable, ++ * ++ * Return: @count if the function succeeded. An error code on failure. ++ */ ++static ssize_t set_dvfs_period(struct device *dev, ++ struct device_attribute *attr, const char *buf, size_t count) ++{ ++ struct kbase_device *kbdev; ++ int ret; ++ int dvfs_period; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ ret = kstrtoint(buf, 0, &dvfs_period); ++ if (ret || dvfs_period <= 0) { ++ dev_err(kbdev->dev, "Couldn't process dvfs_period write operation.\n" ++ "Use format \n"); ++ return -EINVAL; ++ } ++ ++ kbdev->pm.dvfs_period = dvfs_period; ++ dev_dbg(kbdev->dev, "DVFS period: %dms\n", dvfs_period); ++ ++ return count; ++} ++ ++/** ++ * show_dvfs_period - Show callback for the dvfs_period sysfs entry. ++ * @dev: The device this sysfs file is for. ++ * @attr: The attributes of the sysfs file. ++ * @buf: The output buffer to receive the GPU information. ++ * ++ * This function is called to get the current period used for the DVFS sample ++ * timer. ++ * ++ * Return: The number of bytes output to @buf. ++ */ ++static ssize_t show_dvfs_period(struct device *dev, ++ struct device_attribute *attr, char * const buf) ++{ ++ struct kbase_device *kbdev; ++ ssize_t ret; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ ret = scnprintf(buf, PAGE_SIZE, "%d\n", kbdev->pm.dvfs_period); ++ ++ return ret; ++} ++ ++static DEVICE_ATTR(dvfs_period, S_IRUGO | S_IWUSR, show_dvfs_period, ++ set_dvfs_period); ++ ++/** ++ * set_pm_poweroff - Store callback for the pm_poweroff sysfs file. ++ * @dev: The device with sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The value written to the sysfs file ++ * @count: The number of bytes written to the sysfs file ++ * ++ * This function is called when the pm_poweroff sysfs file is written to. ++ * ++ * This file contains three values separated by whitespace. The values ++ * are gpu_poweroff_time (the period of the poweroff timer, in ns), ++ * poweroff_shader_ticks (the number of poweroff timer ticks before an idle ++ * shader is powered off), and poweroff_gpu_ticks (the number of poweroff timer ++ * ticks before the GPU is powered off), in that order. ++ * ++ * Return: @count if the function succeeded. An error code on failure. ++ */ ++static ssize_t set_pm_poweroff(struct device *dev, ++ struct device_attribute *attr, const char *buf, size_t count) ++{ ++ struct kbase_device *kbdev; ++ int items; ++ s64 gpu_poweroff_time; ++ int poweroff_shader_ticks, poweroff_gpu_ticks; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ items = sscanf(buf, "%llu %u %u", &gpu_poweroff_time, ++ &poweroff_shader_ticks, ++ &poweroff_gpu_ticks); ++ if (items != 3) { ++ dev_err(kbdev->dev, "Couldn't process pm_poweroff write operation.\n" ++ "Use format \n"); ++ return -EINVAL; ++ } ++ ++ kbdev->pm.gpu_poweroff_time = HR_TIMER_DELAY_NSEC(gpu_poweroff_time); ++ kbdev->pm.poweroff_shader_ticks = poweroff_shader_ticks; ++ kbdev->pm.poweroff_gpu_ticks = poweroff_gpu_ticks; ++ ++ return count; ++} ++ ++/** ++ * show_pm_poweroff - Show callback for the pm_poweroff sysfs entry. ++ * @dev: The device this sysfs file is for. ++ * @attr: The attributes of the sysfs file. ++ * @buf: The output buffer to receive the GPU information. ++ * ++ * This function is called to get the current period used for the DVFS sample ++ * timer. ++ * ++ * Return: The number of bytes output to @buf. ++ */ ++static ssize_t show_pm_poweroff(struct device *dev, ++ struct device_attribute *attr, char * const buf) ++{ ++ struct kbase_device *kbdev; ++ ssize_t ret; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ ret = scnprintf(buf, PAGE_SIZE, "%llu %u %u\n", ++ ktime_to_ns(kbdev->pm.gpu_poweroff_time), ++ kbdev->pm.poweroff_shader_ticks, ++ kbdev->pm.poweroff_gpu_ticks); ++ ++ return ret; ++} ++ ++static DEVICE_ATTR(pm_poweroff, S_IRUGO | S_IWUSR, show_pm_poweroff, ++ set_pm_poweroff); ++ ++/** ++ * set_reset_timeout - Store callback for the reset_timeout sysfs file. ++ * @dev: The device with sysfs file is for ++ * @attr: The attributes of the sysfs file ++ * @buf: The value written to the sysfs file ++ * @count: The number of bytes written to the sysfs file ++ * ++ * This function is called when the reset_timeout sysfs file is written to. It ++ * checks the data written, and if valid updates the reset timeout. ++ * ++ * Return: @count if the function succeeded. An error code on failure. ++ */ ++static ssize_t set_reset_timeout(struct device *dev, ++ struct device_attribute *attr, const char *buf, size_t count) ++{ ++ struct kbase_device *kbdev; ++ int ret; ++ int reset_timeout; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ ret = kstrtoint(buf, 0, &reset_timeout); ++ if (ret || reset_timeout <= 0) { ++ dev_err(kbdev->dev, "Couldn't process reset_timeout write operation.\n" ++ "Use format \n"); ++ return -EINVAL; ++ } ++ ++ kbdev->reset_timeout_ms = reset_timeout; ++ dev_dbg(kbdev->dev, "Reset timeout: %dms\n", reset_timeout); ++ ++ return count; ++} ++ ++/** ++ * show_reset_timeout - Show callback for the reset_timeout sysfs entry. ++ * @dev: The device this sysfs file is for. ++ * @attr: The attributes of the sysfs file. ++ * @buf: The output buffer to receive the GPU information. ++ * ++ * This function is called to get the current reset timeout. ++ * ++ * Return: The number of bytes output to @buf. ++ */ ++static ssize_t show_reset_timeout(struct device *dev, ++ struct device_attribute *attr, char * const buf) ++{ ++ struct kbase_device *kbdev; ++ ssize_t ret; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ ret = scnprintf(buf, PAGE_SIZE, "%d\n", kbdev->reset_timeout_ms); ++ ++ return ret; ++} ++ ++static DEVICE_ATTR(reset_timeout, S_IRUGO | S_IWUSR, show_reset_timeout, ++ set_reset_timeout); ++ ++ ++ ++static ssize_t show_mem_pool_size(struct device *dev, ++ struct device_attribute *attr, char * const buf) ++{ ++ struct kbase_device *kbdev; ++ ssize_t ret; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ ret = scnprintf(buf, PAGE_SIZE, "%zu\n", ++ kbase_mem_pool_size(&kbdev->mem_pool)); ++ ++ return ret; ++} ++ ++static ssize_t set_mem_pool_size(struct device *dev, ++ struct device_attribute *attr, const char *buf, size_t count) ++{ ++ struct kbase_device *kbdev; ++ size_t new_size; ++ int err; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ err = kstrtoul(buf, 0, (unsigned long *)&new_size); ++ if (err) ++ return err; ++ ++ kbase_mem_pool_trim(&kbdev->mem_pool, new_size); ++ ++ return count; ++} ++ ++static DEVICE_ATTR(mem_pool_size, S_IRUGO | S_IWUSR, show_mem_pool_size, ++ set_mem_pool_size); ++ ++static ssize_t show_mem_pool_max_size(struct device *dev, ++ struct device_attribute *attr, char * const buf) ++{ ++ struct kbase_device *kbdev; ++ ssize_t ret; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ ret = scnprintf(buf, PAGE_SIZE, "%zu\n", ++ kbase_mem_pool_max_size(&kbdev->mem_pool)); ++ ++ return ret; ++} ++ ++static ssize_t set_mem_pool_max_size(struct device *dev, ++ struct device_attribute *attr, const char *buf, size_t count) ++{ ++ struct kbase_device *kbdev; ++ size_t new_max_size; ++ int err; ++ ++ kbdev = to_kbase_device(dev); ++ if (!kbdev) ++ return -ENODEV; ++ ++ err = kstrtoul(buf, 0, (unsigned long *)&new_max_size); ++ if (err) ++ return -EINVAL; ++ ++ kbase_mem_pool_set_max_size(&kbdev->mem_pool, new_max_size); ++ ++ return count; ++} ++ ++static DEVICE_ATTR(mem_pool_max_size, S_IRUGO | S_IWUSR, show_mem_pool_max_size, ++ set_mem_pool_max_size); ++ ++#ifdef CONFIG_DEBUG_FS ++ ++/* Number of entries in serialize_jobs_settings[] */ ++#define NR_SERIALIZE_JOBS_SETTINGS 5 ++/* Maximum string length in serialize_jobs_settings[].name */ ++#define MAX_SERIALIZE_JOBS_NAME_LEN 16 ++ ++static struct ++{ ++ char *name; ++ u8 setting; ++} serialize_jobs_settings[NR_SERIALIZE_JOBS_SETTINGS] = { ++ {"none", 0}, ++ {"intra-slot", KBASE_SERIALIZE_INTRA_SLOT}, ++ {"inter-slot", KBASE_SERIALIZE_INTER_SLOT}, ++ {"full", KBASE_SERIALIZE_INTRA_SLOT | KBASE_SERIALIZE_INTER_SLOT}, ++ {"full-reset", KBASE_SERIALIZE_INTRA_SLOT | KBASE_SERIALIZE_INTER_SLOT | ++ KBASE_SERIALIZE_RESET} ++}; ++ ++/** ++ * kbasep_serialize_jobs_seq_show - Show callback for the serialize_jobs debugfs ++ * file ++ * @sfile: seq_file pointer ++ * @data: Private callback data ++ * ++ * This function is called to get the contents of the serialize_jobs debugfs ++ * file. This is a list of the available settings with the currently active one ++ * surrounded by square brackets. ++ * ++ * Return: 0 on success, or an error code on error ++ */ ++static int kbasep_serialize_jobs_seq_show(struct seq_file *sfile, void *data) ++{ ++ struct kbase_device *kbdev = sfile->private; ++ int i; ++ ++ CSTD_UNUSED(data); ++ ++ for (i = 0; i < NR_SERIALIZE_JOBS_SETTINGS; i++) { ++ if (kbdev->serialize_jobs == serialize_jobs_settings[i].setting) ++ seq_printf(sfile, "[%s] ", ++ serialize_jobs_settings[i].name); ++ else ++ seq_printf(sfile, "%s ", ++ serialize_jobs_settings[i].name); ++ } ++ ++ seq_puts(sfile, "\n"); ++ ++ return 0; ++} ++ ++/** ++ * kbasep_serialize_jobs_debugfs_write - Store callback for the serialize_jobs ++ * debugfs file. ++ * @file: File pointer ++ * @ubuf: User buffer containing data to store ++ * @count: Number of bytes in user buffer ++ * @ppos: File position ++ * ++ * This function is called when the serialize_jobs debugfs file is written to. ++ * It matches the requested setting against the available settings and if a ++ * matching setting is found updates kbdev->serialize_jobs. ++ * ++ * Return: @count if the function succeeded. An error code on failure. ++ */ ++static ssize_t kbasep_serialize_jobs_debugfs_write(struct file *file, ++ const char __user *ubuf, size_t count, loff_t *ppos) ++{ ++ struct seq_file *s = file->private_data; ++ struct kbase_device *kbdev = s->private; ++ char buf[MAX_SERIALIZE_JOBS_NAME_LEN]; ++ int i; ++ bool valid = false; ++ ++ CSTD_UNUSED(ppos); ++ ++ count = min_t(size_t, sizeof(buf) - 1, count); ++ if (copy_from_user(buf, ubuf, count)) ++ return -EFAULT; ++ ++ buf[count] = 0; ++ ++ for (i = 0; i < NR_SERIALIZE_JOBS_SETTINGS; i++) { ++ if (sysfs_streq(serialize_jobs_settings[i].name, buf)) { ++ kbdev->serialize_jobs = ++ serialize_jobs_settings[i].setting; ++ valid = true; ++ break; ++ } ++ } ++ ++ if (!valid) { ++ dev_err(kbdev->dev, "serialize_jobs: invalid setting\n"); ++ return -EINVAL; ++ } ++ ++ return count; ++} ++ ++/** ++ * kbasep_serialize_jobs_debugfs_open - Open callback for the serialize_jobs ++ * debugfs file ++ * @in: inode pointer ++ * @file: file pointer ++ * ++ * Return: Zero on success, error code on failure ++ */ ++static int kbasep_serialize_jobs_debugfs_open(struct inode *in, ++ struct file *file) ++{ ++ return single_open(file, kbasep_serialize_jobs_seq_show, in->i_private); ++} ++ ++static const struct file_operations kbasep_serialize_jobs_debugfs_fops = { ++ .open = kbasep_serialize_jobs_debugfs_open, ++ .read = seq_read, ++ .write = kbasep_serialize_jobs_debugfs_write, ++ .llseek = seq_lseek, ++ .release = single_release, ++}; ++ ++#endif /* CONFIG_DEBUG_FS */ ++ ++static int kbasep_protected_mode_init(struct kbase_device *kbdev) ++{ ++#ifdef CONFIG_OF ++ struct device_node *protected_node; ++ struct platform_device *pdev; ++ struct protected_mode_device *protected_dev; ++#endif ++ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_MODE)) { ++ /* Use native protected ops */ ++ kbdev->protected_dev = kzalloc(sizeof(*kbdev->protected_dev), ++ GFP_KERNEL); ++ if (!kbdev->protected_dev) ++ return -ENOMEM; ++ kbdev->protected_dev->data = kbdev; ++ kbdev->protected_ops = &kbase_native_protected_ops; ++ kbdev->protected_mode_support = true; ++ return 0; ++ } ++ ++ kbdev->protected_mode_support = false; ++ ++#ifdef CONFIG_OF ++ protected_node = of_parse_phandle(kbdev->dev->of_node, ++ "protected-mode-switcher", 0); ++ ++ if (!protected_node) ++ protected_node = of_parse_phandle(kbdev->dev->of_node, ++ "secure-mode-switcher", 0); ++ ++ if (!protected_node) { ++ /* If protected_node cannot be looked up then we assume ++ * protected mode is not supported on this platform. */ ++ dev_info(kbdev->dev, "Protected mode not available\n"); ++ return 0; ++ } ++ ++ pdev = of_find_device_by_node(protected_node); ++ if (!pdev) ++ return -EINVAL; ++ ++ protected_dev = platform_get_drvdata(pdev); ++ if (!protected_dev) ++ return -EPROBE_DEFER; ++ ++ kbdev->protected_ops = &protected_dev->ops; ++ kbdev->protected_dev = protected_dev; ++ ++ if (kbdev->protected_ops) { ++ int err; ++ ++ /* Make sure protected mode is disabled on startup */ ++ mutex_lock(&kbdev->pm.lock); ++ err = kbdev->protected_ops->protected_mode_disable( ++ kbdev->protected_dev); ++ mutex_unlock(&kbdev->pm.lock); ++ ++ /* protected_mode_disable() returns -EINVAL if not supported */ ++ kbdev->protected_mode_support = (err != -EINVAL); ++ } ++#endif ++ return 0; ++} ++ ++static void kbasep_protected_mode_term(struct kbase_device *kbdev) ++{ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_MODE)) ++ kfree(kbdev->protected_dev); ++} ++ ++#ifdef CONFIG_MALI_NO_MALI ++static int kbase_common_reg_map(struct kbase_device *kbdev) ++{ ++ return 0; ++} ++static void kbase_common_reg_unmap(struct kbase_device * const kbdev) ++{ ++} ++#else /* CONFIG_MALI_NO_MALI */ ++static int kbase_common_reg_map(struct kbase_device *kbdev) ++{ ++ int err = -ENOMEM; ++ ++ if (!request_mem_region(kbdev->reg_start, kbdev->reg_size, dev_name(kbdev->dev))) { ++ dev_err(kbdev->dev, "Register window unavailable\n"); ++ err = -EIO; ++ goto out_region; ++ } ++ ++ kbdev->reg = ioremap(kbdev->reg_start, kbdev->reg_size); ++ if (!kbdev->reg) { ++ dev_err(kbdev->dev, "Can't remap register window\n"); ++ err = -EINVAL; ++ goto out_ioremap; ++ } ++ ++ return 0; ++ ++ out_ioremap: ++ release_mem_region(kbdev->reg_start, kbdev->reg_size); ++ out_region: ++ return err; ++} ++ ++static void kbase_common_reg_unmap(struct kbase_device * const kbdev) ++{ ++ if (kbdev->reg) { ++ iounmap(kbdev->reg); ++ release_mem_region(kbdev->reg_start, kbdev->reg_size); ++ kbdev->reg = NULL; ++ kbdev->reg_start = 0; ++ kbdev->reg_size = 0; ++ } ++} ++#endif /* CONFIG_MALI_NO_MALI */ ++ ++static int registers_map(struct kbase_device * const kbdev) ++{ ++ ++ /* the first memory resource is the physical address of the GPU ++ * registers */ ++ struct platform_device *pdev = to_platform_device(kbdev->dev); ++ struct resource *reg_res; ++ int err; ++ ++ reg_res = platform_get_resource(pdev, IORESOURCE_MEM, 0); ++ if (!reg_res) { ++ dev_err(kbdev->dev, "Invalid register resource\n"); ++ return -ENOENT; ++ } ++ ++ kbdev->reg_start = reg_res->start; ++ kbdev->reg_size = resource_size(reg_res); ++ ++ err = kbase_common_reg_map(kbdev); ++ if (err) { ++ dev_err(kbdev->dev, "Failed to map registers\n"); ++ return err; ++ } ++ ++ return 0; ++} ++ ++static void registers_unmap(struct kbase_device *kbdev) ++{ ++ kbase_common_reg_unmap(kbdev); ++} ++ ++static int power_control_init(struct platform_device *pdev) ++{ ++ struct kbase_device *kbdev = to_kbase_device(&pdev->dev); ++ int err = 0; ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0)) && defined(CONFIG_OF) \ ++ && defined(CONFIG_REGULATOR) ++ kbdev->regulator = regulator_get_optional(kbdev->dev, "mali"); ++ if (IS_ERR_OR_NULL(kbdev->regulator)) { ++ err = PTR_ERR(kbdev->regulator); ++ kbdev->regulator = NULL; ++ if (err == -EPROBE_DEFER) { ++ dev_err(&pdev->dev, "Failed to get regulator\n"); ++ return err; ++ } ++ dev_info(kbdev->dev, ++ "Continuing without Mali regulator control\n"); ++ /* Allow probe to continue without regulator */ ++ } ++#endif /* LINUX_VERSION_CODE >= 3, 12, 0 */ ++ ++ kbdev->clock = clk_get(kbdev->dev, "clk_mali"); ++ if (IS_ERR_OR_NULL(kbdev->clock)) { ++ err = PTR_ERR(kbdev->clock); ++ kbdev->clock = NULL; ++ if (err == -EPROBE_DEFER) { ++ dev_err(&pdev->dev, "Failed to get clock\n"); ++ goto fail; ++ } ++ dev_info(kbdev->dev, "Continuing without Mali clock control\n"); ++ /* Allow probe to continue without clock. */ ++ } else { ++ err = clk_prepare(kbdev->clock); ++ if (err) { ++ dev_err(kbdev->dev, ++ "Failed to prepare and enable clock (%d)\n", ++ err); ++ goto fail; ++ } ++ } ++ ++ err = kbase_platform_rk_init_opp_table(kbdev); ++ if (err) ++ dev_err(kbdev->dev, "Failed to init_opp_table (%d)\n", err); ++ ++ return 0; ++ ++fail: ++ ++if (kbdev->clock != NULL) { ++ clk_put(kbdev->clock); ++ kbdev->clock = NULL; ++} ++ ++#ifdef CONFIG_REGULATOR ++ if (NULL != kbdev->regulator) { ++ regulator_put(kbdev->regulator); ++ kbdev->regulator = NULL; ++ } ++#endif ++ ++ return err; ++} ++ ++static void power_control_term(struct kbase_device *kbdev) ++{ ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0)) || \ ++ defined(LSK_OPPV2_BACKPORT) ++ dev_pm_opp_of_remove_table(kbdev->dev); ++#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0)) ++ of_free_opp_table(kbdev->dev); ++#endif ++ ++ if (kbdev->clock) { ++ clk_unprepare(kbdev->clock); ++ clk_put(kbdev->clock); ++ kbdev->clock = NULL; ++ } ++ ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0)) && defined(CONFIG_OF) \ ++ && defined(CONFIG_REGULATOR) ++ if (kbdev->regulator) { ++ regulator_put(kbdev->regulator); ++ kbdev->regulator = NULL; ++ } ++#endif /* LINUX_VERSION_CODE >= 3, 12, 0 */ ++} ++ ++#ifdef CONFIG_DEBUG_FS ++ ++#if KBASE_GPU_RESET_EN ++#include ++ ++static void trigger_quirks_reload(struct kbase_device *kbdev) ++{ ++ kbase_pm_context_active(kbdev); ++ if (kbase_prepare_to_reset_gpu(kbdev)) ++ kbase_reset_gpu(kbdev); ++ kbase_pm_context_idle(kbdev); ++} ++ ++#define MAKE_QUIRK_ACCESSORS(type) \ ++static int type##_quirks_set(void *data, u64 val) \ ++{ \ ++ struct kbase_device *kbdev; \ ++ kbdev = (struct kbase_device *)data; \ ++ kbdev->hw_quirks_##type = (u32)val; \ ++ trigger_quirks_reload(kbdev); \ ++ return 0;\ ++} \ ++\ ++static int type##_quirks_get(void *data, u64 *val) \ ++{ \ ++ struct kbase_device *kbdev;\ ++ kbdev = (struct kbase_device *)data;\ ++ *val = kbdev->hw_quirks_##type;\ ++ return 0;\ ++} \ ++DEFINE_SIMPLE_ATTRIBUTE(fops_##type##_quirks, type##_quirks_get,\ ++ type##_quirks_set, "%llu\n") ++ ++MAKE_QUIRK_ACCESSORS(sc); ++MAKE_QUIRK_ACCESSORS(tiler); ++MAKE_QUIRK_ACCESSORS(mmu); ++MAKE_QUIRK_ACCESSORS(jm); ++ ++#endif /* KBASE_GPU_RESET_EN */ ++ ++/** ++ * debugfs_protected_debug_mode_read - "protected_debug_mode" debugfs read ++ * @file: File object to read is for ++ * @buf: User buffer to populate with data ++ * @len: Length of user buffer ++ * @ppos: Offset within file object ++ * ++ * Retrieves the current status of protected debug mode ++ * (0 = disabled, 1 = enabled) ++ * ++ * Return: Number of bytes added to user buffer ++ */ ++static ssize_t debugfs_protected_debug_mode_read(struct file *file, ++ char __user *buf, size_t len, loff_t *ppos) ++{ ++ struct kbase_device *kbdev = (struct kbase_device *)file->private_data; ++ u32 gpu_status; ++ ssize_t ret_val; ++ ++ kbase_pm_context_active(kbdev); ++ gpu_status = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_STATUS), NULL); ++ kbase_pm_context_idle(kbdev); ++ ++ if (gpu_status & GPU_DBGEN) ++ ret_val = simple_read_from_buffer(buf, len, ppos, "1\n", 2); ++ else ++ ret_val = simple_read_from_buffer(buf, len, ppos, "0\n", 2); ++ ++ return ret_val; ++} ++ ++/* ++ * struct fops_protected_debug_mode - "protected_debug_mode" debugfs fops ++ * ++ * Contains the file operations for the "protected_debug_mode" debugfs file ++ */ ++static const struct file_operations fops_protected_debug_mode = { ++ .open = simple_open, ++ .read = debugfs_protected_debug_mode_read, ++ .llseek = default_llseek, ++}; ++ ++static int kbase_device_debugfs_init(struct kbase_device *kbdev) ++{ ++ struct dentry *debugfs_ctx_defaults_directory; ++ int err; ++ ++ kbdev->mali_debugfs_directory = debugfs_create_dir(kbdev->devname, ++ NULL); ++ if (!kbdev->mali_debugfs_directory) { ++ dev_err(kbdev->dev, "Couldn't create mali debugfs directory\n"); ++ err = -ENOMEM; ++ goto out; ++ } ++ ++ kbdev->debugfs_ctx_directory = debugfs_create_dir("ctx", ++ kbdev->mali_debugfs_directory); ++ if (!kbdev->debugfs_ctx_directory) { ++ dev_err(kbdev->dev, "Couldn't create mali debugfs ctx directory\n"); ++ err = -ENOMEM; ++ goto out; ++ } ++ ++ debugfs_ctx_defaults_directory = debugfs_create_dir("defaults", ++ kbdev->debugfs_ctx_directory); ++ if (!debugfs_ctx_defaults_directory) { ++ dev_err(kbdev->dev, "Couldn't create mali debugfs ctx defaults directory\n"); ++ err = -ENOMEM; ++ goto out; ++ } ++ ++#if !MALI_CUSTOMER_RELEASE ++ kbasep_regs_dump_debugfs_init(kbdev); ++#endif /* !MALI_CUSTOMER_RELEASE */ ++ kbasep_regs_history_debugfs_init(kbdev); ++ ++ kbase_debug_job_fault_debugfs_init(kbdev); ++ kbasep_gpu_memory_debugfs_init(kbdev); ++ kbase_as_fault_debugfs_init(kbdev); ++#if KBASE_GPU_RESET_EN ++ /* fops_* variables created by invocations of macro ++ * MAKE_QUIRK_ACCESSORS() above. */ ++ debugfs_create_file("quirks_sc", 0644, ++ kbdev->mali_debugfs_directory, kbdev, ++ &fops_sc_quirks); ++ debugfs_create_file("quirks_tiler", 0644, ++ kbdev->mali_debugfs_directory, kbdev, ++ &fops_tiler_quirks); ++ debugfs_create_file("quirks_mmu", 0644, ++ kbdev->mali_debugfs_directory, kbdev, ++ &fops_mmu_quirks); ++ debugfs_create_file("quirks_jm", 0644, ++ kbdev->mali_debugfs_directory, kbdev, ++ &fops_jm_quirks); ++#endif /* KBASE_GPU_RESET_EN */ ++ ++#ifndef CONFIG_MALI_COH_USER ++ debugfs_create_bool("infinite_cache", 0644, ++ debugfs_ctx_defaults_directory, ++ (bool*)&(kbdev->infinite_cache_active_default)); ++#endif /* CONFIG_MALI_COH_USER */ ++ ++ debugfs_create_size_t("mem_pool_max_size", 0644, ++ debugfs_ctx_defaults_directory, ++ &kbdev->mem_pool_max_size_default); ++ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE)) { ++ debugfs_create_file("protected_debug_mode", S_IRUGO, ++ kbdev->mali_debugfs_directory, kbdev, ++ &fops_protected_debug_mode); ++ } ++ ++#if KBASE_TRACE_ENABLE ++ kbasep_trace_debugfs_init(kbdev); ++#endif /* KBASE_TRACE_ENABLE */ ++ ++#ifdef CONFIG_MALI_TRACE_TIMELINE ++ kbasep_trace_timeline_debugfs_init(kbdev); ++#endif /* CONFIG_MALI_TRACE_TIMELINE */ ++ ++#ifdef CONFIG_MALI_DEVFREQ ++#ifdef CONFIG_DEVFREQ_THERMAL ++ if (kbdev->inited_subsys & inited_devfreq) ++ kbase_ipa_debugfs_init(kbdev); ++#endif /* CONFIG_DEVFREQ_THERMAL */ ++#endif /* CONFIG_MALI_DEVFREQ */ ++ ++#ifdef CONFIG_DEBUG_FS ++ debugfs_create_file("serialize_jobs", S_IRUGO | S_IWUSR, ++ kbdev->mali_debugfs_directory, kbdev, ++ &kbasep_serialize_jobs_debugfs_fops); ++#endif /* CONFIG_DEBUG_FS */ ++ ++ return 0; ++ ++out: ++ debugfs_remove_recursive(kbdev->mali_debugfs_directory); ++ return err; ++} ++ ++static void kbase_device_debugfs_term(struct kbase_device *kbdev) ++{ ++ debugfs_remove_recursive(kbdev->mali_debugfs_directory); ++} ++ ++#else /* CONFIG_DEBUG_FS */ ++static inline int kbase_device_debugfs_init(struct kbase_device *kbdev) ++{ ++ return 0; ++} ++ ++static inline void kbase_device_debugfs_term(struct kbase_device *kbdev) { } ++#endif /* CONFIG_DEBUG_FS */ ++ ++static void kbase_device_coherency_init(struct kbase_device *kbdev, ++ unsigned prod_id) ++{ ++#ifdef CONFIG_OF ++ u32 supported_coherency_bitmap = ++ kbdev->gpu_props.props.raw_props.coherency_mode; ++ const void *coherency_override_dts; ++ u32 override_coherency; ++ ++ /* Only for tMIx : ++ * (COHERENCY_ACE_LITE | COHERENCY_ACE) was incorrectly ++ * documented for tMIx so force correct value here. ++ */ ++ if (GPU_ID_IS_NEW_FORMAT(prod_id) && ++ (GPU_ID2_MODEL_MATCH_VALUE(prod_id) == ++ GPU_ID2_PRODUCT_TMIX)) ++ if (supported_coherency_bitmap == ++ COHERENCY_FEATURE_BIT(COHERENCY_ACE)) ++ supported_coherency_bitmap |= ++ COHERENCY_FEATURE_BIT(COHERENCY_ACE_LITE); ++ ++#endif /* CONFIG_OF */ ++ ++ kbdev->system_coherency = COHERENCY_NONE; ++ ++ /* device tree may override the coherency */ ++#ifdef CONFIG_OF ++ coherency_override_dts = of_get_property(kbdev->dev->of_node, ++ "system-coherency", ++ NULL); ++ if (coherency_override_dts) { ++ ++ override_coherency = be32_to_cpup(coherency_override_dts); ++ ++ if ((override_coherency <= COHERENCY_NONE) && ++ (supported_coherency_bitmap & ++ COHERENCY_FEATURE_BIT(override_coherency))) { ++ ++ kbdev->system_coherency = override_coherency; ++ ++ dev_info(kbdev->dev, ++ "Using coherency mode %u set from dtb", ++ override_coherency); ++ } else ++ dev_warn(kbdev->dev, ++ "Ignoring unsupported coherency mode %u set from dtb", ++ override_coherency); ++ } ++ ++#endif /* CONFIG_OF */ ++ ++ kbdev->gpu_props.props.raw_props.coherency_mode = ++ kbdev->system_coherency; ++} ++ ++#ifdef CONFIG_MALI_FPGA_BUS_LOGGER ++ ++/* Callback used by the kbase bus logger client, to initiate a GPU reset ++ * when the bus log is restarted. GPU reset is used as reference point ++ * in HW bus log analyses. ++ */ ++static void kbase_logging_started_cb(void *data) ++{ ++ struct kbase_device *kbdev = (struct kbase_device *)data; ++ ++ if (kbase_prepare_to_reset_gpu(kbdev)) ++ kbase_reset_gpu(kbdev); ++ dev_info(kbdev->dev, "KBASE - Bus logger restarted\n"); ++} ++#endif ++ ++static struct attribute *kbase_attrs[] = { ++#ifdef CONFIG_MALI_DEBUG ++ &dev_attr_debug_command.attr, ++ &dev_attr_js_softstop_always.attr, ++#endif ++#if !MALI_CUSTOMER_RELEASE ++ &dev_attr_force_replay.attr, ++#endif ++ &dev_attr_js_timeouts.attr, ++ &dev_attr_soft_job_timeout.attr, ++ &dev_attr_gpuinfo.attr, ++ &dev_attr_dvfs_period.attr, ++ &dev_attr_pm_poweroff.attr, ++ &dev_attr_reset_timeout.attr, ++ &dev_attr_js_scheduling_period.attr, ++ &dev_attr_power_policy.attr, ++ &dev_attr_core_availability_policy.attr, ++ &dev_attr_core_mask.attr, ++ &dev_attr_mem_pool_size.attr, ++ &dev_attr_mem_pool_max_size.attr, ++ NULL ++}; ++ ++static const struct attribute_group kbase_attr_group = { ++ .attrs = kbase_attrs, ++}; ++ ++static int kbase_platform_device_remove(struct platform_device *pdev) ++{ ++ struct kbase_device *kbdev = to_kbase_device(&pdev->dev); ++ const struct list_head *dev_list; ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ kfree(kbdev->gpu_props.prop_buffer); ++ ++#ifdef CONFIG_MALI_FPGA_BUS_LOGGER ++ if (kbdev->inited_subsys & inited_buslogger) { ++ bl_core_client_unregister(kbdev->buslogger); ++ kbdev->inited_subsys &= ~inited_buslogger; ++ } ++#endif ++ ++ ++ if (kbdev->inited_subsys & inited_dev_list) { ++ dev_list = kbase_dev_list_get(); ++ list_del(&kbdev->entry); ++ kbase_dev_list_put(dev_list); ++ kbdev->inited_subsys &= ~inited_dev_list; ++ } ++ ++ if (kbdev->inited_subsys & inited_misc_register) { ++ misc_deregister(&kbdev->mdev); ++ kbdev->inited_subsys &= ~inited_misc_register; ++ } ++ ++ if (kbdev->inited_subsys & inited_sysfs_group) { ++ sysfs_remove_group(&kbdev->dev->kobj, &kbase_attr_group); ++ kbdev->inited_subsys &= ~inited_sysfs_group; ++ } ++ ++ if (kbdev->inited_subsys & inited_get_device) { ++ put_device(kbdev->dev); ++ kbdev->inited_subsys &= ~inited_get_device; ++ } ++ ++ if (kbdev->inited_subsys & inited_debugfs) { ++ kbase_device_debugfs_term(kbdev); ++ kbdev->inited_subsys &= ~inited_debugfs; ++ } ++ ++ if (kbdev->inited_subsys & inited_job_fault) { ++ kbase_debug_job_fault_dev_term(kbdev); ++ kbdev->inited_subsys &= ~inited_job_fault; ++ } ++ if (kbdev->inited_subsys & inited_vinstr) { ++ kbase_vinstr_term(kbdev->vinstr_ctx); ++ kbdev->inited_subsys &= ~inited_vinstr; ++ } ++ ++#ifdef CONFIG_MALI_DEVFREQ ++ if (kbdev->inited_subsys & inited_devfreq) { ++ kbase_devfreq_term(kbdev); ++ kbdev->inited_subsys &= ~inited_devfreq; ++ } ++#endif ++ ++ if (kbdev->inited_subsys & inited_backend_late) { ++ kbase_backend_late_term(kbdev); ++ kbdev->inited_subsys &= ~inited_backend_late; ++ } ++ ++ if (kbdev->inited_subsys & inited_tlstream) { ++ kbase_tlstream_term(); ++ kbdev->inited_subsys &= ~inited_tlstream; ++ } ++ ++ /* Bring job and mem sys to a halt before we continue termination */ ++ ++ if (kbdev->inited_subsys & inited_js) ++ kbasep_js_devdata_halt(kbdev); ++ ++ if (kbdev->inited_subsys & inited_mem) ++ kbase_mem_halt(kbdev); ++ ++ if (kbdev->inited_subsys & inited_protected) { ++ kbasep_protected_mode_term(kbdev); ++ kbdev->inited_subsys &= ~inited_protected; ++ } ++ ++ if (kbdev->inited_subsys & inited_js) { ++ kbasep_js_devdata_term(kbdev); ++ kbdev->inited_subsys &= ~inited_js; ++ } ++ ++ if (kbdev->inited_subsys & inited_mem) { ++ kbase_mem_term(kbdev); ++ kbdev->inited_subsys &= ~inited_mem; ++ } ++ ++ if (kbdev->inited_subsys & inited_pm_runtime_init) { ++ kbdev->pm.callback_power_runtime_term(kbdev); ++ kbdev->inited_subsys &= ~inited_pm_runtime_init; ++ } ++ ++ if (kbdev->inited_subsys & inited_ctx_sched) { ++ kbase_ctx_sched_term(kbdev); ++ kbdev->inited_subsys &= ~inited_ctx_sched; ++ } ++ ++ if (kbdev->inited_subsys & inited_device) { ++ kbase_device_term(kbdev); ++ kbdev->inited_subsys &= ~inited_device; ++ } ++ ++ if (kbdev->inited_subsys & inited_backend_early) { ++ kbase_backend_early_term(kbdev); ++ kbdev->inited_subsys &= ~inited_backend_early; ++ } ++ ++ if (kbdev->inited_subsys & inited_io_history) { ++ kbase_io_history_term(&kbdev->io_history); ++ kbdev->inited_subsys &= ~inited_io_history; ++ } ++ ++ if (kbdev->inited_subsys & inited_power_control) { ++ power_control_term(kbdev); ++ kbdev->inited_subsys &= ~inited_power_control; ++ } ++ ++ if (kbdev->inited_subsys & inited_registers_map) { ++ registers_unmap(kbdev); ++ kbdev->inited_subsys &= ~inited_registers_map; ++ } ++ ++#ifdef CONFIG_MALI_NO_MALI ++ if (kbdev->inited_subsys & inited_gpu_device) { ++ gpu_device_destroy(kbdev); ++ kbdev->inited_subsys &= ~inited_gpu_device; ++ } ++#endif /* CONFIG_MALI_NO_MALI */ ++ ++ if (kbdev->inited_subsys != 0) ++ dev_err(kbdev->dev, "Missing sub system termination\n"); ++ ++ kbase_device_free(kbdev); ++ ++ return 0; ++} ++ ++extern void kbase_platform_rk_shutdown(struct kbase_device *kbdev); ++static void kbase_platform_device_shutdown(struct platform_device *pdev) ++{ ++ struct kbase_device *kbdev = to_kbase_device(&pdev->dev); ++ ++ kbase_platform_rk_shutdown(kbdev); ++} ++ ++/* Number of register accesses for the buffer that we allocate during ++ * initialization time. The buffer size can be changed later via debugfs. */ ++#define KBASEP_DEFAULT_REGISTER_HISTORY_SIZE ((u16)512) ++ ++static int kbase_platform_device_probe(struct platform_device *pdev) ++{ ++ struct kbase_device *kbdev; ++ struct mali_base_gpu_core_props *core_props; ++ u32 gpu_id; ++ unsigned prod_id; ++ const struct list_head *dev_list; ++ int err = 0; ++ ++#ifdef CONFIG_OF ++ err = kbase_platform_early_init(); ++ if (err) { ++ dev_err(&pdev->dev, "Early platform initialization failed\n"); ++ kbase_platform_device_remove(pdev); ++ return err; ++ } ++#endif ++ kbdev = kbase_device_alloc(); ++ if (!kbdev) { ++ dev_err(&pdev->dev, "Allocate device failed\n"); ++ kbase_platform_device_remove(pdev); ++ return -ENOMEM; ++ } ++ ++ kbdev->dev = &pdev->dev; ++ dev_set_drvdata(kbdev->dev, kbdev); ++ ++#ifdef CONFIG_MALI_NO_MALI ++ err = gpu_device_create(kbdev); ++ if (err) { ++ dev_err(&pdev->dev, "Dummy model initialization failed\n"); ++ kbase_platform_device_remove(pdev); ++ return err; ++ } ++ kbdev->inited_subsys |= inited_gpu_device; ++#endif /* CONFIG_MALI_NO_MALI */ ++ ++ err = assign_irqs(pdev); ++ if (err) { ++ dev_err(&pdev->dev, "IRQ search failed\n"); ++ kbase_platform_device_remove(pdev); ++ return err; ++ } ++ ++ err = registers_map(kbdev); ++ if (err) { ++ dev_err(&pdev->dev, "Register map failed\n"); ++ kbase_platform_device_remove(pdev); ++ return err; ++ } ++ kbdev->inited_subsys |= inited_registers_map; ++ ++ err = power_control_init(pdev); ++ if (err) { ++ dev_err(&pdev->dev, "Power control initialization failed\n"); ++ kbase_platform_device_remove(pdev); ++ return err; ++ } ++ kbdev->inited_subsys |= inited_power_control; ++ ++ err = kbase_io_history_init(&kbdev->io_history, ++ KBASEP_DEFAULT_REGISTER_HISTORY_SIZE); ++ if (err) { ++ dev_err(&pdev->dev, "Register access history initialization failed\n"); ++ kbase_platform_device_remove(pdev); ++ return -ENOMEM; ++ } ++ kbdev->inited_subsys |= inited_io_history; ++ ++ err = kbase_backend_early_init(kbdev); ++ if (err) { ++ dev_err(kbdev->dev, "Early backend initialization failed\n"); ++ kbase_platform_device_remove(pdev); ++ return err; ++ } ++ kbdev->inited_subsys |= inited_backend_early; ++ ++ scnprintf(kbdev->devname, DEVNAME_SIZE, "%s%d", kbase_drv_name, ++ kbase_dev_nr); ++ ++ kbase_disjoint_init(kbdev); ++ ++ /* obtain min/max configured gpu frequencies */ ++ core_props = &(kbdev->gpu_props.props.core_props); ++ core_props->gpu_freq_khz_min = GPU_FREQ_KHZ_MIN; ++ core_props->gpu_freq_khz_max = GPU_FREQ_KHZ_MAX; ++ ++ err = kbase_device_init(kbdev); ++ if (err) { ++ dev_err(kbdev->dev, "Device initialization failed (%d)\n", err); ++ kbase_platform_device_remove(pdev); ++ return err; ++ } ++ kbdev->inited_subsys |= inited_device; ++ ++ err = kbase_ctx_sched_init(kbdev); ++ if (err) { ++ dev_err(kbdev->dev, "Context scheduler initialization failed (%d)\n", ++ err); ++ kbase_platform_device_remove(pdev); ++ return err; ++ } ++ kbdev->inited_subsys |= inited_ctx_sched; ++ ++ if (kbdev->pm.callback_power_runtime_init) { ++ err = kbdev->pm.callback_power_runtime_init(kbdev); ++ if (err) { ++ dev_err(kbdev->dev, ++ "Runtime PM initialization failed\n"); ++ kbase_platform_device_remove(pdev); ++ return err; ++ } ++ kbdev->inited_subsys |= inited_pm_runtime_init; ++ } ++ ++ err = kbase_mem_init(kbdev); ++ if (err) { ++ dev_err(kbdev->dev, "Memory subsystem initialization failed\n"); ++ kbase_platform_device_remove(pdev); ++ return err; ++ } ++ kbdev->inited_subsys |= inited_mem; ++ ++ gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; ++ gpu_id &= GPU_ID_VERSION_PRODUCT_ID; ++ prod_id = gpu_id >> GPU_ID_VERSION_PRODUCT_ID_SHIFT; ++ ++ kbase_device_coherency_init(kbdev, prod_id); ++ ++ err = kbasep_protected_mode_init(kbdev); ++ if (err) { ++ dev_err(kbdev->dev, "Protected mode subsystem initialization failed\n"); ++ kbase_platform_device_remove(pdev); ++ return err; ++ } ++ kbdev->inited_subsys |= inited_protected; ++ ++ dev_list = kbase_dev_list_get(); ++ list_add(&kbdev->entry, &kbase_dev_list); ++ kbase_dev_list_put(dev_list); ++ kbdev->inited_subsys |= inited_dev_list; ++ ++ err = kbasep_js_devdata_init(kbdev); ++ if (err) { ++ dev_err(kbdev->dev, "Job JS devdata initialization failed\n"); ++ kbase_platform_device_remove(pdev); ++ return err; ++ } ++ kbdev->inited_subsys |= inited_js; ++ ++ err = kbase_tlstream_init(); ++ if (err) { ++ dev_err(kbdev->dev, "Timeline stream initialization failed\n"); ++ kbase_platform_device_remove(pdev); ++ return err; ++ } ++ kbdev->inited_subsys |= inited_tlstream; ++ ++ err = kbase_backend_late_init(kbdev); ++ if (err) { ++ dev_err(kbdev->dev, "Late backend initialization failed\n"); ++ kbase_platform_device_remove(pdev); ++ return err; ++ } ++ kbdev->inited_subsys |= inited_backend_late; ++ ++#ifdef CONFIG_MALI_DEVFREQ ++ err = kbase_devfreq_init(kbdev); ++ if (!err) ++ kbdev->inited_subsys |= inited_devfreq; ++ else ++ dev_err(kbdev->dev, "Continuing without devfreq\n"); ++#endif /* CONFIG_MALI_DEVFREQ */ ++ ++ kbdev->vinstr_ctx = kbase_vinstr_init(kbdev); ++ if (!kbdev->vinstr_ctx) { ++ dev_err(kbdev->dev, ++ "Virtual instrumentation initialization failed\n"); ++ kbase_platform_device_remove(pdev); ++ return -EINVAL; ++ } ++ kbdev->inited_subsys |= inited_vinstr; ++ ++ err = kbase_debug_job_fault_dev_init(kbdev); ++ if (err) { ++ dev_err(kbdev->dev, "Job fault debug initialization failed\n"); ++ kbase_platform_device_remove(pdev); ++ return err; ++ } ++ kbdev->inited_subsys |= inited_job_fault; ++ ++ err = kbase_device_debugfs_init(kbdev); ++ if (err) { ++ dev_err(kbdev->dev, "DebugFS initialization failed"); ++ kbase_platform_device_remove(pdev); ++ return err; ++ } ++ kbdev->inited_subsys |= inited_debugfs; ++ ++ /* initialize the kctx list */ ++ mutex_init(&kbdev->kctx_list_lock); ++ INIT_LIST_HEAD(&kbdev->kctx_list); ++ ++ kbdev->mdev.minor = MISC_DYNAMIC_MINOR; ++ kbdev->mdev.name = kbdev->devname; ++ kbdev->mdev.fops = &kbase_fops; ++ kbdev->mdev.parent = get_device(kbdev->dev); ++ kbdev->inited_subsys |= inited_get_device; ++ ++ /* This needs to happen before registering the device with misc_register(), ++ * otherwise it causes a race condition between registering the device and a ++ * uevent event being generated for userspace, causing udev rules to run ++ * which might expect certain sysfs attributes present. As a result of the ++ * race condition we avoid, some Mali sysfs entries may have appeared to ++ * udev to not exist. ++ ++ * For more information, see ++ * https://www.kernel.org/doc/Documentation/driver-model/device.txt, the ++ * paragraph that starts with "Word of warning", currently the second-last ++ * paragraph. ++ */ ++ err = sysfs_create_group(&kbdev->dev->kobj, &kbase_attr_group); ++ if (err) { ++ dev_err(&pdev->dev, "SysFS group creation failed\n"); ++ kbase_platform_device_remove(pdev); ++ return err; ++ } ++ kbdev->inited_subsys |= inited_sysfs_group; ++ ++ err = misc_register(&kbdev->mdev); ++ if (err) { ++ dev_err(kbdev->dev, "Misc device registration failed for %s\n", ++ kbdev->devname); ++ kbase_platform_device_remove(pdev); ++ return err; ++ } ++ kbdev->inited_subsys |= inited_misc_register; ++ ++ ++#ifdef CONFIG_MALI_FPGA_BUS_LOGGER ++ err = bl_core_client_register(kbdev->devname, ++ kbase_logging_started_cb, ++ kbdev, &kbdev->buslogger, ++ THIS_MODULE, NULL); ++ if (err == 0) { ++ kbdev->inited_subsys |= inited_buslogger; ++ bl_core_set_threshold(kbdev->buslogger, 1024*1024*1024); ++ } else { ++ dev_warn(kbdev->dev, "Bus log client registration failed\n"); ++ err = 0; ++ } ++#endif ++ ++ err = kbase_gpuprops_populate_user_buffer(kbdev); ++ if (err) { ++ dev_err(&pdev->dev, "GPU property population failed"); ++ kbase_platform_device_remove(pdev); ++ return err; ++ } ++ ++ dev_info(kbdev->dev, ++ "Probed as %s\n", dev_name(kbdev->mdev.this_device)); ++ ++ kbase_dev_nr++; ++ ++ return err; ++} ++ ++#undef KBASEP_DEFAULT_REGISTER_HISTORY_SIZE ++ ++/** ++ * kbase_device_suspend - Suspend callback from the OS. ++ * ++ * This is called by Linux when the device should suspend. ++ * ++ * @dev: The device to suspend ++ * ++ * Return: A standard Linux error code ++ */ ++static int kbase_device_suspend(struct device *dev) ++{ ++ struct kbase_device *kbdev = to_kbase_device(dev); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++#if defined(CONFIG_MALI_DEVFREQ) && \ ++ (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) ++ if (kbdev->inited_subsys & inited_devfreq) ++ devfreq_suspend_device(kbdev->devfreq); ++#endif ++ ++ kbase_pm_suspend(kbdev); ++ return 0; ++} ++ ++/** ++ * kbase_device_resume - Resume callback from the OS. ++ * ++ * This is called by Linux when the device should resume from suspension. ++ * ++ * @dev: The device to resume ++ * ++ * Return: A standard Linux error code ++ */ ++static int kbase_device_resume(struct device *dev) ++{ ++ struct kbase_device *kbdev = to_kbase_device(dev); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ kbase_pm_resume(kbdev); ++ ++#if defined(CONFIG_MALI_DEVFREQ) && \ ++ (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) ++ if (kbdev->inited_subsys & inited_devfreq) ++ devfreq_resume_device(kbdev->devfreq); ++#endif ++ return 0; ++} ++ ++/** ++ * kbase_device_runtime_suspend - Runtime suspend callback from the OS. ++ * ++ * This is called by Linux when the device should prepare for a condition in ++ * which it will not be able to communicate with the CPU(s) and RAM due to ++ * power management. ++ * ++ * @dev: The device to suspend ++ * ++ * Return: A standard Linux error code ++ */ ++#ifdef KBASE_PM_RUNTIME ++static int kbase_device_runtime_suspend(struct device *dev) ++{ ++ struct kbase_device *kbdev = to_kbase_device(dev); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++#if defined(CONFIG_MALI_DEVFREQ) && \ ++ (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) ++ if (kbdev->inited_subsys & inited_devfreq) ++ devfreq_suspend_device(kbdev->devfreq); ++#endif ++ ++ if (kbdev->pm.backend.callback_power_runtime_off) { ++ kbdev->pm.backend.callback_power_runtime_off(kbdev); ++ dev_dbg(dev, "runtime suspend\n"); ++ } ++ return 0; ++} ++#endif /* KBASE_PM_RUNTIME */ ++ ++/** ++ * kbase_device_runtime_resume - Runtime resume callback from the OS. ++ * ++ * This is called by Linux when the device should go into a fully active state. ++ * ++ * @dev: The device to suspend ++ * ++ * Return: A standard Linux error code ++ */ ++ ++#ifdef KBASE_PM_RUNTIME ++static int kbase_device_runtime_resume(struct device *dev) ++{ ++ int ret = 0; ++ struct kbase_device *kbdev = to_kbase_device(dev); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ if (kbdev->pm.backend.callback_power_runtime_on) { ++ ret = kbdev->pm.backend.callback_power_runtime_on(kbdev); ++ dev_dbg(dev, "runtime resume\n"); ++ } ++ ++#if defined(CONFIG_MALI_DEVFREQ) && \ ++ (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) ++ if (kbdev->inited_subsys & inited_devfreq) ++ devfreq_resume_device(kbdev->devfreq); ++#endif ++ ++ return ret; ++} ++#endif /* KBASE_PM_RUNTIME */ ++ ++ ++#ifdef KBASE_PM_RUNTIME ++/** ++ * kbase_device_runtime_idle - Runtime idle callback from the OS. ++ * @dev: The device to suspend ++ * ++ * This is called by Linux when the device appears to be inactive and it might ++ * be placed into a low power state. ++ * ++ * Return: 0 if device can be suspended, non-zero to avoid runtime autosuspend, ++ * otherwise a standard Linux error code ++ */ ++static int kbase_device_runtime_idle(struct device *dev) ++{ ++ struct kbase_device *kbdev = to_kbase_device(dev); ++ ++ if (!kbdev) ++ return -ENODEV; ++ ++ /* Use platform specific implementation if it exists. */ ++ if (kbdev->pm.backend.callback_power_runtime_idle) ++ return kbdev->pm.backend.callback_power_runtime_idle(kbdev); ++ ++ return 0; ++} ++#endif /* KBASE_PM_RUNTIME */ ++ ++/* The power management operations for the platform driver. ++ */ ++static const struct dev_pm_ops kbase_pm_ops = { ++ .suspend = kbase_device_suspend, ++ .resume = kbase_device_resume, ++#ifdef KBASE_PM_RUNTIME ++ .runtime_suspend = kbase_device_runtime_suspend, ++ .runtime_resume = kbase_device_runtime_resume, ++ .runtime_idle = kbase_device_runtime_idle, ++#endif /* KBASE_PM_RUNTIME */ ++}; ++ ++#ifdef CONFIG_OF ++static const struct of_device_id kbase_dt_ids[] = { ++ { .compatible = "arm,malit7xx" }, ++ { .compatible = "arm,mali-midgard" }, ++ { /* sentinel */ } ++}; ++MODULE_DEVICE_TABLE(of, kbase_dt_ids); ++#endif ++ ++static struct platform_driver kbase_platform_driver = { ++ .probe = kbase_platform_device_probe, ++ .remove = kbase_platform_device_remove, ++ .shutdown = kbase_platform_device_shutdown, ++ .driver = { ++ .name = "midgard", ++ .owner = THIS_MODULE, ++ .pm = &kbase_pm_ops, ++ .of_match_table = of_match_ptr(kbase_dt_ids), ++ }, ++}; ++ ++/* ++ * The driver will not provide a shortcut to create the Mali platform device ++ * anymore when using Device Tree. ++ */ ++#ifdef CONFIG_OF ++module_platform_driver(kbase_platform_driver); ++#else ++ ++static int __init rockchip_gpu_init_driver(void) ++{ ++ return platform_driver_register(&kbase_platform_driver); ++} ++late_initcall(rockchip_gpu_init_driver); ++ ++static int __init kbase_driver_init(void) ++{ ++ int ret; ++ ++ ret = kbase_platform_early_init(); ++ if (ret) ++ return ret; ++ ++#ifdef CONFIG_MALI_PLATFORM_FAKE ++ ret = kbase_platform_fake_register(); ++ if (ret) ++ return ret; ++#endif ++ ret = platform_driver_register(&kbase_platform_driver); ++#ifdef CONFIG_MALI_PLATFORM_FAKE ++ if (ret) ++ kbase_platform_fake_unregister(); ++#endif ++ return ret; ++} ++ ++static void __exit kbase_driver_exit(void) ++{ ++ platform_driver_unregister(&kbase_platform_driver); ++#ifdef CONFIG_MALI_PLATFORM_FAKE ++ kbase_platform_fake_unregister(); ++#endif ++} ++ ++module_init(kbase_driver_init); ++module_exit(kbase_driver_exit); ++ ++#endif /* CONFIG_OF */ ++ ++MODULE_LICENSE("GPL"); ++MODULE_VERSION(MALI_RELEASE_NAME " (UK version " \ ++ __stringify(BASE_UK_VERSION_MAJOR) "." \ ++ __stringify(BASE_UK_VERSION_MINOR) ")"); ++ ++#if defined(CONFIG_MALI_GATOR_SUPPORT) || defined(CONFIG_MALI_SYSTEM_TRACE) ++#define CREATE_TRACE_POINTS ++#endif ++ ++#ifdef CONFIG_MALI_GATOR_SUPPORT ++/* Create the trace points (otherwise we just get code to call a tracepoint) */ ++#include "mali_linux_trace.h" ++ ++EXPORT_TRACEPOINT_SYMBOL_GPL(mali_job_slots_event); ++EXPORT_TRACEPOINT_SYMBOL_GPL(mali_pm_status); ++EXPORT_TRACEPOINT_SYMBOL_GPL(mali_pm_power_on); ++EXPORT_TRACEPOINT_SYMBOL_GPL(mali_pm_power_off); ++EXPORT_TRACEPOINT_SYMBOL_GPL(mali_page_fault_insert_pages); ++EXPORT_TRACEPOINT_SYMBOL_GPL(mali_mmu_as_in_use); ++EXPORT_TRACEPOINT_SYMBOL_GPL(mali_mmu_as_released); ++EXPORT_TRACEPOINT_SYMBOL_GPL(mali_total_alloc_pages_change); ++ ++void kbase_trace_mali_pm_status(u32 event, u64 value) ++{ ++ trace_mali_pm_status(event, value); ++} ++ ++void kbase_trace_mali_pm_power_off(u32 event, u64 value) ++{ ++ trace_mali_pm_power_off(event, value); ++} ++ ++void kbase_trace_mali_pm_power_on(u32 event, u64 value) ++{ ++ trace_mali_pm_power_on(event, value); ++} ++ ++void kbase_trace_mali_job_slots_event(u32 event, const struct kbase_context *kctx, u8 atom_id) ++{ ++ trace_mali_job_slots_event(event, (kctx != NULL ? kctx->tgid : 0), (kctx != NULL ? kctx->pid : 0), atom_id); ++} ++ ++void kbase_trace_mali_page_fault_insert_pages(int event, u32 value) ++{ ++ trace_mali_page_fault_insert_pages(event, value); ++} ++ ++void kbase_trace_mali_mmu_as_in_use(int event) ++{ ++ trace_mali_mmu_as_in_use(event); ++} ++ ++void kbase_trace_mali_mmu_as_released(int event) ++{ ++ trace_mali_mmu_as_released(event); ++} ++ ++void kbase_trace_mali_total_alloc_pages_change(long long int event) ++{ ++ trace_mali_total_alloc_pages_change(event); ++} ++#endif /* CONFIG_MALI_GATOR_SUPPORT */ ++#ifdef CONFIG_MALI_SYSTEM_TRACE ++#include "mali_linux_kbase_trace.h" ++#endif +diff --git a/drivers/gpu/arm/midgard/mali_kbase_ctx_sched.c b/drivers/gpu/arm/midgard/mali_kbase_ctx_sched.c +new file mode 100755 +index 000000000000..ce004841403f +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_ctx_sched.c +@@ -0,0 +1,208 @@ ++/* ++ * ++ * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include ++#include ++ ++#include "mali_kbase_ctx_sched.h" ++ ++int kbase_ctx_sched_init(struct kbase_device *kbdev) ++{ ++ int as_present = (1U << kbdev->nr_hw_address_spaces) - 1; ++ ++ /* These two must be recalculated if nr_hw_address_spaces changes ++ * (e.g. for HW workarounds) */ ++ kbdev->nr_user_address_spaces = kbdev->nr_hw_address_spaces; ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) { ++ bool use_workaround; ++ ++ use_workaround = DEFAULT_SECURE_BUT_LOSS_OF_PERFORMANCE; ++ if (use_workaround) { ++ dev_dbg(kbdev->dev, "GPU has HW ISSUE 8987, and driver configured for security workaround: 1 address space only"); ++ kbdev->nr_user_address_spaces = 1; ++ } ++ } ++ ++ kbdev->as_free = as_present; /* All ASs initially free */ ++ ++ memset(kbdev->as_to_kctx, 0, sizeof(kbdev->as_to_kctx)); ++ ++ return 0; ++} ++ ++void kbase_ctx_sched_term(struct kbase_device *kbdev) ++{ ++ s8 i; ++ ++ /* Sanity checks */ ++ for (i = 0; i != kbdev->nr_hw_address_spaces; ++i) { ++ WARN_ON(kbdev->as_to_kctx[i] != NULL); ++ WARN_ON(!(kbdev->as_free & (1u << i))); ++ } ++} ++ ++/* kbasep_ctx_sched_find_as_for_ctx - Find a free address space ++ * ++ * @kbdev: The context for which to find a free address space ++ * ++ * Return: A valid AS if successful, otherwise KBASEP_AS_NR_INVALID ++ * ++ * This function returns an address space available for use. It would prefer ++ * returning an AS that has been previously assigned to the context to ++ * avoid having to reprogram the MMU. ++ */ ++static int kbasep_ctx_sched_find_as_for_ctx(struct kbase_context *kctx) ++{ ++ struct kbase_device *const kbdev = kctx->kbdev; ++ int free_as; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ /* First check if the previously assigned AS is available */ ++ if ((kctx->as_nr != KBASEP_AS_NR_INVALID) && ++ (kbdev->as_free & (1u << kctx->as_nr))) ++ return kctx->as_nr; ++ ++ /* The previously assigned AS was taken, we'll be returning any free ++ * AS at this point. ++ */ ++ free_as = ffs(kbdev->as_free) - 1; ++ if (free_as >= 0 && free_as < kbdev->nr_hw_address_spaces) ++ return free_as; ++ ++ return KBASEP_AS_NR_INVALID; ++} ++ ++int kbase_ctx_sched_retain_ctx(struct kbase_context *kctx) ++{ ++ struct kbase_device *const kbdev = kctx->kbdev; ++ ++ lockdep_assert_held(&kbdev->mmu_hw_mutex); ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ WARN_ON(!kbdev->pm.backend.gpu_powered); ++ ++ if (atomic_inc_return(&kctx->refcount) == 1) { ++ int const free_as = kbasep_ctx_sched_find_as_for_ctx(kctx); ++ ++ if (free_as != KBASEP_AS_NR_INVALID) { ++ kbdev->as_free &= ~(1u << free_as); ++ /* Only program the MMU if the context has not been ++ * assigned the same address space before. ++ */ ++ if (free_as != kctx->as_nr) { ++ struct kbase_context *const prev_kctx = ++ kbdev->as_to_kctx[free_as]; ++ ++ if (prev_kctx) { ++ WARN_ON(atomic_read(&prev_kctx->refcount) != 0); ++ kbase_mmu_disable(prev_kctx); ++ prev_kctx->as_nr = KBASEP_AS_NR_INVALID; ++ } ++ ++ kctx->as_nr = free_as; ++ kbdev->as_to_kctx[free_as] = kctx; ++ kbase_mmu_update(kctx); ++ } ++ } else { ++ atomic_dec(&kctx->refcount); ++ ++ /* Failed to find an available address space, we must ++ * be returning an error at this point. ++ */ ++ WARN_ON(kctx->as_nr != KBASEP_AS_NR_INVALID); ++ } ++ } ++ ++ return kctx->as_nr; ++} ++ ++int kbase_ctx_sched_retain_ctx_refcount(struct kbase_context *kctx) ++{ ++ struct kbase_device *const kbdev = kctx->kbdev; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ WARN_ON(atomic_read(&kctx->refcount) == 0); ++ if (atomic_read(&kctx->refcount) == 0) ++ return -1; ++ ++ WARN_ON(kctx->as_nr == KBASEP_AS_NR_INVALID); ++ WARN_ON(kbdev->as_to_kctx[kctx->as_nr] != kctx); ++ ++ atomic_inc(&kctx->refcount); ++ ++ return 0; ++} ++ ++void kbase_ctx_sched_release_ctx(struct kbase_context *kctx) ++{ ++ struct kbase_device *const kbdev = kctx->kbdev; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (atomic_dec_return(&kctx->refcount) == 0) ++ kbdev->as_free |= (1u << kctx->as_nr); ++} ++ ++void kbase_ctx_sched_remove_ctx(struct kbase_context *kctx) ++{ ++ struct kbase_device *const kbdev = kctx->kbdev; ++ ++ lockdep_assert_held(&kbdev->mmu_hw_mutex); ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ WARN_ON(atomic_read(&kctx->refcount) != 0); ++ ++ if (kctx->as_nr != KBASEP_AS_NR_INVALID) { ++ if (kbdev->pm.backend.gpu_powered) ++ kbase_mmu_disable(kctx); ++ ++ kbdev->as_to_kctx[kctx->as_nr] = NULL; ++ kctx->as_nr = KBASEP_AS_NR_INVALID; ++ } ++} ++ ++void kbase_ctx_sched_restore_all_as(struct kbase_device *kbdev) ++{ ++ s8 i; ++ ++ lockdep_assert_held(&kbdev->mmu_hw_mutex); ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ WARN_ON(!kbdev->pm.backend.gpu_powered); ++ ++ for (i = 0; i != kbdev->nr_hw_address_spaces; ++i) { ++ struct kbase_context *kctx; ++ ++ kctx = kbdev->as_to_kctx[i]; ++ if (kctx) { ++ if (atomic_read(&kctx->refcount)) { ++ WARN_ON(kctx->as_nr != i); ++ ++ kbase_mmu_update(kctx); ++ } else { ++ /* This context might have been assigned an ++ * AS before, clear it. ++ */ ++ kbdev->as_to_kctx[kctx->as_nr] = NULL; ++ kctx->as_nr = KBASEP_AS_NR_INVALID; ++ } ++ } else { ++ kbase_mmu_disable_as(kbdev, i); ++ } ++ } ++} +diff --git a/drivers/gpu/arm/midgard/mali_kbase_ctx_sched.h b/drivers/gpu/arm/midgard/mali_kbase_ctx_sched.h +new file mode 100755 +index 000000000000..47474fecc2a9 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_ctx_sched.h +@@ -0,0 +1,134 @@ ++/* ++ * ++ * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KBASE_CTX_SCHED_H_ ++#define _KBASE_CTX_SCHED_H_ ++ ++#include ++ ++/* The Context Scheduler manages address space assignment and reference ++ * counting to kbase_context. The interface has been designed to minimise ++ * interactions between the Job Scheduler and Power Management/MMU to support ++ * both the existing Job Scheduler and Command Stream Frontend interface. ++ * ++ * The initial implementation of the Context Scheduler does not schedule ++ * contexts. Instead it relies on the Job Scheduler/CSF to make decisions of ++ * when to schedule/evict contexts if address spaces are starved. In the ++ * future, once an interface between the CS and JS/CSF have been devised to ++ * provide enough information about how each context is consuming GPU resources, ++ * those decisions can be made in the CS itself, thereby reducing duplicated ++ * code. ++ */ ++ ++/* base_ctx_sched_init - Initialise the context scheduler ++ * ++ * @kbdev: The device for which the context scheduler needs to be ++ * initialised ++ * ++ * Return: 0 for success, otherwise failure ++ * ++ * This must be called during device initilisation. The number of hardware ++ * address spaces must already be established before calling this function. ++ */ ++int kbase_ctx_sched_init(struct kbase_device *kbdev); ++ ++/* base_ctx_sched_term - Terminate the context scheduler ++ * ++ * @kbdev: The device for which the context scheduler needs to be ++ * terminated ++ * ++ * This must be called during device termination after all contexts have been ++ * destroyed. ++ */ ++void kbase_ctx_sched_term(struct kbase_device *kbdev); ++ ++/* kbase_ctx_sched_retain_ctx - Retain a reference to the @ref kbase_context ++ * ++ * @kctx: The context to which to retain a reference ++ * ++ * Return: The address space that the context has been assigned to or ++ * KBASEP_AS_NR_INVALID if no address space was available. ++ * ++ * This function should be called whenever an address space should be assigned ++ * to a context and programmed onto the MMU. It should typically be called ++ * when jobs are ready to be submitted to the GPU. ++ * ++ * It can be called as many times as necessary. The address space will be ++ * assigned to the context for as long as there is a reference to said context. ++ * ++ * The kbase_device::mmu_hw_mutex and kbase_device::hwaccess_lock locks must be ++ * held whilst calling this function. ++ */ ++int kbase_ctx_sched_retain_ctx(struct kbase_context *kctx); ++ ++/* kbase_ctx_sched_retain_ctx_refcount ++ * ++ * @kctx: The context to which to retain a reference ++ * ++ * This function only retains a reference to the context. It must be called ++ * only when the context already has a reference. ++ * ++ * This is typically called inside an atomic session where we know the context ++ * is already scheduled in but want to take an extra reference to ensure that ++ * it doesn't get descheduled. ++ * ++ * The kbase_device::hwaccess_lock must be held whilst calling this function ++ * @return ++ * è‹¥æˆåŠŸ, 返回 0; ++ * è‹¥ *kctx 状æ€å¼‚常, 返回 -1. ++ */ ++int kbase_ctx_sched_retain_ctx_refcount(struct kbase_context *kctx); ++ ++/* kbase_ctx_sched_release_ctx - Release a reference to the @ref kbase_context ++ * ++ * @kctx: The context from which to release a reference ++ * ++ * This function should be called whenever an address space could be unassigned ++ * from a context. When there are no more references to said context, the ++ * address space previously assigned to this context shall be reassigned to ++ * other contexts as needed. ++ * ++ * The kbase_device::hwaccess_lock must be held whilst calling this function ++ */ ++void kbase_ctx_sched_release_ctx(struct kbase_context *kctx); ++ ++/* kbase_ctx_sched_remove_ctx - Unassign previously assigned address space ++ * ++ * @kctx: The context to be removed ++ * ++ * This function should be called when a context is being destroyed. The ++ * context must no longer have any reference. If it has been assigned an ++ * address space before then the AS will be unprogrammed. ++ * ++ * The kbase_device::mmu_hw_mutex and kbase_device::hwaccess_lock locks must be ++ * held whilst calling this function. ++ */ ++void kbase_ctx_sched_remove_ctx(struct kbase_context *kctx); ++ ++/* kbase_ctx_sched_restore_all_as - Reprogram all address spaces ++ * ++ * @kbdev: The device for which address spaces to be reprogrammed ++ * ++ * This function shall reprogram all address spaces previously assigned to ++ * contexts. It can be used after the GPU is reset. ++ * ++ * The kbase_device::mmu_hw_mutex and kbase_device::hwaccess_lock locks must be ++ * held whilst calling this function. ++ */ ++void kbase_ctx_sched_restore_all_as(struct kbase_device *kbdev); ++ ++#endif /* _KBASE_CTX_SCHED_H_ */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_debug.c b/drivers/gpu/arm/midgard/mali_kbase_debug.c +new file mode 100755 +index 000000000000..fb57ac2e31ad +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_debug.c +@@ -0,0 +1,39 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2014 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#include ++ ++static struct kbasep_debug_assert_cb kbasep_debug_assert_registered_cb = { ++ NULL, ++ NULL ++}; ++ ++void kbase_debug_assert_register_hook(kbase_debug_assert_hook *func, void *param) ++{ ++ kbasep_debug_assert_registered_cb.func = func; ++ kbasep_debug_assert_registered_cb.param = param; ++} ++ ++void kbasep_debug_assert_call_hook(void) ++{ ++ if (kbasep_debug_assert_registered_cb.func != NULL) ++ kbasep_debug_assert_registered_cb.func(kbasep_debug_assert_registered_cb.param); ++} ++KBASE_EXPORT_SYMBOL(kbasep_debug_assert_call_hook); ++ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_debug.h b/drivers/gpu/arm/midgard/mali_kbase_debug.h +new file mode 100755 +index 000000000000..5fff2892bb55 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_debug.h +@@ -0,0 +1,164 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#ifndef _KBASE_DEBUG_H ++#define _KBASE_DEBUG_H ++ ++#include ++ ++/** @brief If equals to 0, a trace containing the file, line, and function will be displayed before each message. */ ++#define KBASE_DEBUG_SKIP_TRACE 0 ++ ++/** @brief If different from 0, the trace will only contain the file and line. */ ++#define KBASE_DEBUG_SKIP_FUNCTION_NAME 0 ++ ++/** @brief Disable the asserts tests if set to 1. Default is to disable the asserts in release. */ ++#ifndef KBASE_DEBUG_DISABLE_ASSERTS ++#ifdef CONFIG_MALI_DEBUG ++#define KBASE_DEBUG_DISABLE_ASSERTS 0 ++#else ++#define KBASE_DEBUG_DISABLE_ASSERTS 1 ++#endif ++#endif /* KBASE_DEBUG_DISABLE_ASSERTS */ ++ ++/** Function type that is called on an KBASE_DEBUG_ASSERT() or KBASE_DEBUG_ASSERT_MSG() */ ++typedef void (kbase_debug_assert_hook) (void *); ++ ++struct kbasep_debug_assert_cb { ++ kbase_debug_assert_hook *func; ++ void *param; ++}; ++ ++/** ++ * @def KBASEP_DEBUG_PRINT_TRACE ++ * @brief Private macro containing the format of the trace to display before every message ++ * @sa KBASE_DEBUG_SKIP_TRACE, KBASE_DEBUG_SKIP_FUNCTION_NAME ++ */ ++#if !KBASE_DEBUG_SKIP_TRACE ++#define KBASEP_DEBUG_PRINT_TRACE \ ++ "In file: " __FILE__ " line: " CSTD_STR2(__LINE__) ++#if !KBASE_DEBUG_SKIP_FUNCTION_NAME ++#define KBASEP_DEBUG_PRINT_FUNCTION __func__ ++#else ++#define KBASEP_DEBUG_PRINT_FUNCTION "" ++#endif ++#else ++#define KBASEP_DEBUG_PRINT_TRACE "" ++#endif ++ ++/** ++ * @def KBASEP_DEBUG_ASSERT_OUT(trace, function, ...) ++ * @brief (Private) system printing function associated to the @see KBASE_DEBUG_ASSERT_MSG event. ++ * @param trace location in the code from where the message is printed ++ * @param function function from where the message is printed ++ * @param ... Format string followed by format arguments. ++ * @note function parameter cannot be concatenated with other strings ++ */ ++/* Select the correct system output function*/ ++#ifdef CONFIG_MALI_DEBUG ++#define KBASEP_DEBUG_ASSERT_OUT(trace, function, ...)\ ++ do { \ ++ pr_err("Mali: %s function:%s ", trace, function);\ ++ pr_err(__VA_ARGS__);\ ++ pr_err("\n");\ ++ } while (false) ++#else ++#define KBASEP_DEBUG_ASSERT_OUT(trace, function, ...) CSTD_NOP() ++#endif ++ ++#ifdef CONFIG_MALI_DEBUG ++#define KBASE_CALL_ASSERT_HOOK() kbasep_debug_assert_call_hook() ++#else ++#define KBASE_CALL_ASSERT_HOOK() CSTD_NOP() ++#endif ++ ++/** ++ * @def KBASE_DEBUG_ASSERT(expr) ++ * @brief Calls @see KBASE_PRINT_ASSERT and prints the expression @a expr if @a expr is false ++ * ++ * @note This macro does nothing if the flag @see KBASE_DEBUG_DISABLE_ASSERTS is set to 1 ++ * ++ * @param expr Boolean expression ++ */ ++#define KBASE_DEBUG_ASSERT(expr) \ ++ KBASE_DEBUG_ASSERT_MSG(expr, #expr) ++ ++#if KBASE_DEBUG_DISABLE_ASSERTS ++#define KBASE_DEBUG_ASSERT_MSG(expr, ...) CSTD_NOP() ++#else ++ /** ++ * @def KBASE_DEBUG_ASSERT_MSG(expr, ...) ++ * @brief Calls @see KBASEP_DEBUG_ASSERT_OUT and prints the given message if @a expr is false ++ * ++ * @note This macro does nothing if the flag @see KBASE_DEBUG_DISABLE_ASSERTS is set to 1 ++ * ++ * @param expr Boolean expression ++ * @param ... Message to display when @a expr is false, as a format string followed by format arguments. ++ */ ++#define KBASE_DEBUG_ASSERT_MSG(expr, ...) \ ++ do { \ ++ if (!(expr)) { \ ++ KBASEP_DEBUG_ASSERT_OUT(KBASEP_DEBUG_PRINT_TRACE, KBASEP_DEBUG_PRINT_FUNCTION, __VA_ARGS__);\ ++ KBASE_CALL_ASSERT_HOOK();\ ++ BUG();\ ++ } \ ++ } while (false) ++#endif /* KBASE_DEBUG_DISABLE_ASSERTS */ ++ ++/** ++ * @def KBASE_DEBUG_CODE( X ) ++ * @brief Executes the code inside the macro only in debug mode ++ * ++ * @param X Code to compile only in debug mode. ++ */ ++#ifdef CONFIG_MALI_DEBUG ++#define KBASE_DEBUG_CODE(X) X ++#else ++#define KBASE_DEBUG_CODE(X) CSTD_NOP() ++#endif /* CONFIG_MALI_DEBUG */ ++ ++/** @} */ ++ ++/** ++ * @brief Register a function to call on ASSERT ++ * ++ * Such functions will \b only be called during Debug mode, and for debugging ++ * features \b only. Do not rely on them to be called in general use. ++ * ++ * To disable the hook, supply NULL to \a func. ++ * ++ * @note This function is not thread-safe, and should only be used to ++ * register/deregister once in the module's lifetime. ++ * ++ * @param[in] func the function to call when an assert is triggered. ++ * @param[in] param the parameter to pass to \a func when calling it ++ */ ++void kbase_debug_assert_register_hook(kbase_debug_assert_hook *func, void *param); ++ ++/** ++ * @brief Call a debug assert hook previously registered with kbase_debug_assert_register_hook() ++ * ++ * @note This function is not thread-safe with respect to multiple threads ++ * registering functions and parameters with ++ * kbase_debug_assert_register_hook(). Otherwise, thread safety is the ++ * responsibility of the registered hook. ++ */ ++void kbasep_debug_assert_call_hook(void); ++ ++#endif /* _KBASE_DEBUG_H */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_debug_job_fault.c b/drivers/gpu/arm/midgard/mali_kbase_debug_job_fault.c +new file mode 100755 +index 000000000000..f29430ddf8f9 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_debug_job_fault.c +@@ -0,0 +1,499 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include ++#include ++#include ++ ++#ifdef CONFIG_DEBUG_FS ++ ++static bool kbase_is_job_fault_event_pending(struct kbase_device *kbdev) ++{ ++ struct list_head *event_list = &kbdev->job_fault_event_list; ++ unsigned long flags; ++ bool ret; ++ ++ spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); ++ ret = !list_empty(event_list); ++ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); ++ ++ return ret; ++} ++ ++static bool kbase_ctx_has_no_event_pending(struct kbase_context *kctx) ++{ ++ struct kbase_device *kbdev = kctx->kbdev; ++ struct list_head *event_list = &kctx->kbdev->job_fault_event_list; ++ struct base_job_fault_event *event; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); ++ if (list_empty(event_list)) { ++ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); ++ return true; ++ } ++ list_for_each_entry(event, event_list, head) { ++ if (event->katom->kctx == kctx) { ++ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, ++ flags); ++ return false; ++ } ++ } ++ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); ++ return true; ++} ++ ++/* wait until the fault happen and copy the event */ ++static int kbase_job_fault_event_wait(struct kbase_device *kbdev, ++ struct base_job_fault_event *event) ++{ ++ struct list_head *event_list = &kbdev->job_fault_event_list; ++ struct base_job_fault_event *event_in; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); ++ if (list_empty(event_list)) { ++ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); ++ if (wait_event_interruptible(kbdev->job_fault_wq, ++ kbase_is_job_fault_event_pending(kbdev))) ++ return -ERESTARTSYS; ++ spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); ++ } ++ ++ event_in = list_entry(event_list->next, ++ struct base_job_fault_event, head); ++ event->event_code = event_in->event_code; ++ event->katom = event_in->katom; ++ ++ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); ++ ++ return 0; ++ ++} ++ ++/* remove the event from the queue */ ++static struct base_job_fault_event *kbase_job_fault_event_dequeue( ++ struct kbase_device *kbdev, struct list_head *event_list) ++{ ++ struct base_job_fault_event *event; ++ ++ event = list_entry(event_list->next, ++ struct base_job_fault_event, head); ++ list_del(event_list->next); ++ ++ return event; ++ ++} ++ ++/* Remove all the following atoms after the failed atom in the same context ++ * Call the postponed bottom half of job done. ++ * Then, this context could be rescheduled. ++ */ ++static void kbase_job_fault_resume_event_cleanup(struct kbase_context *kctx) ++{ ++ struct list_head *event_list = &kctx->job_fault_resume_event_list; ++ ++ while (!list_empty(event_list)) { ++ struct base_job_fault_event *event; ++ ++ event = kbase_job_fault_event_dequeue(kctx->kbdev, ++ &kctx->job_fault_resume_event_list); ++ kbase_jd_done_worker(&event->katom->work); ++ } ++ ++} ++ ++/* Remove all the failed atoms that belong to different contexts ++ * Resume all the contexts that were suspend due to failed job ++ */ ++static void kbase_job_fault_event_cleanup(struct kbase_device *kbdev) ++{ ++ struct list_head *event_list = &kbdev->job_fault_event_list; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); ++ while (!list_empty(event_list)) { ++ kbase_job_fault_event_dequeue(kbdev, event_list); ++ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); ++ wake_up(&kbdev->job_fault_resume_wq); ++ spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); ++ } ++ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); ++} ++ ++static void kbase_job_fault_resume_worker(struct work_struct *data) ++{ ++ struct base_job_fault_event *event = container_of(data, ++ struct base_job_fault_event, job_fault_work); ++ struct kbase_context *kctx; ++ struct kbase_jd_atom *katom; ++ ++ katom = event->katom; ++ kctx = katom->kctx; ++ ++ dev_info(kctx->kbdev->dev, "Job dumping wait\n"); ++ ++ /* When it was waked up, it need to check if queue is empty or the ++ * failed atom belongs to different context. If yes, wake up. Both ++ * of them mean the failed job has been dumped. Please note, it ++ * should never happen that the job_fault_event_list has the two ++ * atoms belong to the same context. ++ */ ++ wait_event(kctx->kbdev->job_fault_resume_wq, ++ kbase_ctx_has_no_event_pending(kctx)); ++ ++ atomic_set(&kctx->job_fault_count, 0); ++ kbase_jd_done_worker(&katom->work); ++ ++ /* In case the following atoms were scheduled during failed job dump ++ * the job_done_worker was held. We need to rerun it after the dump ++ * was finished ++ */ ++ kbase_job_fault_resume_event_cleanup(kctx); ++ ++ dev_info(kctx->kbdev->dev, "Job dumping finish, resume scheduler\n"); ++} ++ ++static struct base_job_fault_event *kbase_job_fault_event_queue( ++ struct list_head *event_list, ++ struct kbase_jd_atom *atom, ++ u32 completion_code) ++{ ++ struct base_job_fault_event *event; ++ ++ event = &atom->fault_event; ++ ++ event->katom = atom; ++ event->event_code = completion_code; ++ ++ list_add_tail(&event->head, event_list); ++ ++ return event; ++ ++} ++ ++static void kbase_job_fault_event_post(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom, u32 completion_code) ++{ ++ struct base_job_fault_event *event; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); ++ event = kbase_job_fault_event_queue(&kbdev->job_fault_event_list, ++ katom, completion_code); ++ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); ++ ++ wake_up_interruptible(&kbdev->job_fault_wq); ++ ++ INIT_WORK(&event->job_fault_work, kbase_job_fault_resume_worker); ++ queue_work(kbdev->job_fault_resume_workq, &event->job_fault_work); ++ ++ dev_info(katom->kctx->kbdev->dev, "Job fault happen, start dump: %d_%d", ++ katom->kctx->tgid, katom->kctx->id); ++ ++} ++ ++/* ++ * This function will process the job fault ++ * Get the register copy ++ * Send the failed job dump event ++ * Create a Wait queue to wait until the job dump finish ++ */ ++ ++bool kbase_debug_job_fault_process(struct kbase_jd_atom *katom, ++ u32 completion_code) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ ++ /* Check if dumping is in the process ++ * only one atom of each context can be dumped at the same time ++ * If the atom belongs to different context, it can be dumped ++ */ ++ if (atomic_read(&kctx->job_fault_count) > 0) { ++ kbase_job_fault_event_queue( ++ &kctx->job_fault_resume_event_list, ++ katom, completion_code); ++ dev_info(kctx->kbdev->dev, "queue:%d\n", ++ kbase_jd_atom_id(kctx, katom)); ++ return true; ++ } ++ ++ if (kctx->kbdev->job_fault_debug == true) { ++ ++ if (completion_code != BASE_JD_EVENT_DONE) { ++ ++ if (kbase_job_fault_get_reg_snapshot(kctx) == false) { ++ dev_warn(kctx->kbdev->dev, "get reg dump failed\n"); ++ return false; ++ } ++ ++ kbase_job_fault_event_post(kctx->kbdev, katom, ++ completion_code); ++ atomic_inc(&kctx->job_fault_count); ++ dev_info(kctx->kbdev->dev, "post:%d\n", ++ kbase_jd_atom_id(kctx, katom)); ++ return true; ++ ++ } ++ } ++ return false; ++ ++} ++ ++static int debug_job_fault_show(struct seq_file *m, void *v) ++{ ++ struct kbase_device *kbdev = m->private; ++ struct base_job_fault_event *event = (struct base_job_fault_event *)v; ++ struct kbase_context *kctx = event->katom->kctx; ++ int i; ++ ++ dev_info(kbdev->dev, "debug job fault seq show:%d_%d, %d", ++ kctx->tgid, kctx->id, event->reg_offset); ++ ++ if (kctx->reg_dump == NULL) { ++ dev_warn(kbdev->dev, "reg dump is NULL"); ++ return -1; ++ } ++ ++ if (kctx->reg_dump[event->reg_offset] == ++ REGISTER_DUMP_TERMINATION_FLAG) { ++ /* Return the error here to stop the read. And the ++ * following next() will not be called. The stop can ++ * get the real event resource and release it ++ */ ++ return -1; ++ } ++ ++ if (event->reg_offset == 0) ++ seq_printf(m, "%d_%d\n", kctx->tgid, kctx->id); ++ ++ for (i = 0; i < 50; i++) { ++ if (kctx->reg_dump[event->reg_offset] == ++ REGISTER_DUMP_TERMINATION_FLAG) { ++ break; ++ } ++ seq_printf(m, "%08x: %08x\n", ++ kctx->reg_dump[event->reg_offset], ++ kctx->reg_dump[1+event->reg_offset]); ++ event->reg_offset += 2; ++ ++ } ++ ++ ++ return 0; ++} ++static void *debug_job_fault_next(struct seq_file *m, void *v, loff_t *pos) ++{ ++ struct kbase_device *kbdev = m->private; ++ struct base_job_fault_event *event = (struct base_job_fault_event *)v; ++ ++ dev_info(kbdev->dev, "debug job fault seq next:%d, %d", ++ event->reg_offset, (int)*pos); ++ ++ return event; ++} ++ ++static void *debug_job_fault_start(struct seq_file *m, loff_t *pos) ++{ ++ struct kbase_device *kbdev = m->private; ++ struct base_job_fault_event *event; ++ ++ dev_info(kbdev->dev, "fault job seq start:%d", (int)*pos); ++ ++ /* The condition is trick here. It needs make sure the ++ * fault hasn't happened and the dumping hasn't been started, ++ * or the dumping has finished ++ */ ++ if (*pos == 0) { ++ event = kmalloc(sizeof(*event), GFP_KERNEL); ++ if (!event) ++ return NULL; ++ event->reg_offset = 0; ++ if (kbase_job_fault_event_wait(kbdev, event)) { ++ kfree(event); ++ return NULL; ++ } ++ ++ /* The cache flush workaround is called in bottom half of ++ * job done but we delayed it. Now we should clean cache ++ * earlier. Then the GPU memory dump should be correct. ++ */ ++ kbase_backend_cacheclean(kbdev, event->katom); ++ } else ++ return NULL; ++ ++ return event; ++} ++ ++static void debug_job_fault_stop(struct seq_file *m, void *v) ++{ ++ struct kbase_device *kbdev = m->private; ++ ++ /* here we wake up the kbase_jd_done_worker after stop, it needs ++ * get the memory dump before the register dump in debug daemon, ++ * otherwise, the memory dump may be incorrect. ++ */ ++ ++ if (v != NULL) { ++ kfree(v); ++ dev_info(kbdev->dev, "debug job fault seq stop stage 1"); ++ ++ } else { ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); ++ if (!list_empty(&kbdev->job_fault_event_list)) { ++ kbase_job_fault_event_dequeue(kbdev, ++ &kbdev->job_fault_event_list); ++ wake_up(&kbdev->job_fault_resume_wq); ++ } ++ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); ++ dev_info(kbdev->dev, "debug job fault seq stop stage 2"); ++ } ++ ++} ++ ++static const struct seq_operations ops = { ++ .start = debug_job_fault_start, ++ .next = debug_job_fault_next, ++ .stop = debug_job_fault_stop, ++ .show = debug_job_fault_show, ++}; ++ ++static int debug_job_fault_open(struct inode *in, struct file *file) ++{ ++ struct kbase_device *kbdev = in->i_private; ++ ++ seq_open(file, &ops); ++ ++ ((struct seq_file *)file->private_data)->private = kbdev; ++ dev_info(kbdev->dev, "debug job fault seq open"); ++ ++ kbdev->job_fault_debug = true; ++ ++ return 0; ++ ++} ++ ++static int debug_job_fault_release(struct inode *in, struct file *file) ++{ ++ struct kbase_device *kbdev = in->i_private; ++ ++ seq_release(in, file); ++ ++ kbdev->job_fault_debug = false; ++ ++ /* Clean the unprocessed job fault. After that, all the suspended ++ * contexts could be rescheduled. ++ */ ++ kbase_job_fault_event_cleanup(kbdev); ++ ++ dev_info(kbdev->dev, "debug job fault seq close"); ++ ++ return 0; ++} ++ ++static const struct file_operations kbasep_debug_job_fault_fops = { ++ .open = debug_job_fault_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = debug_job_fault_release, ++}; ++ ++/* ++ * Initialize debugfs entry for job fault dump ++ */ ++void kbase_debug_job_fault_debugfs_init(struct kbase_device *kbdev) ++{ ++ debugfs_create_file("job_fault", S_IRUGO, ++ kbdev->mali_debugfs_directory, kbdev, ++ &kbasep_debug_job_fault_fops); ++} ++ ++ ++int kbase_debug_job_fault_dev_init(struct kbase_device *kbdev) ++{ ++ ++ INIT_LIST_HEAD(&kbdev->job_fault_event_list); ++ ++ init_waitqueue_head(&(kbdev->job_fault_wq)); ++ init_waitqueue_head(&(kbdev->job_fault_resume_wq)); ++ spin_lock_init(&kbdev->job_fault_event_lock); ++ ++ kbdev->job_fault_resume_workq = alloc_workqueue( ++ "kbase_job_fault_resume_work_queue", WQ_MEM_RECLAIM, 1); ++ if (!kbdev->job_fault_resume_workq) ++ return -ENOMEM; ++ ++ kbdev->job_fault_debug = false; ++ ++ return 0; ++} ++ ++/* ++ * Release the relevant resource per device ++ */ ++void kbase_debug_job_fault_dev_term(struct kbase_device *kbdev) ++{ ++ destroy_workqueue(kbdev->job_fault_resume_workq); ++} ++ ++ ++/* ++ * Initialize the relevant data structure per context ++ */ ++void kbase_debug_job_fault_context_init(struct kbase_context *kctx) ++{ ++ ++ /* We need allocate double size register range ++ * Because this memory will keep the register address and value ++ */ ++ kctx->reg_dump = vmalloc(0x4000 * 2); ++ if (kctx->reg_dump == NULL) ++ return; ++ ++ if (kbase_debug_job_fault_reg_snapshot_init(kctx, 0x4000) == false) { ++ vfree(kctx->reg_dump); ++ kctx->reg_dump = NULL; ++ } ++ INIT_LIST_HEAD(&kctx->job_fault_resume_event_list); ++ atomic_set(&kctx->job_fault_count, 0); ++ ++} ++ ++/* ++ * release the relevant resource per context ++ */ ++void kbase_debug_job_fault_context_term(struct kbase_context *kctx) ++{ ++ vfree(kctx->reg_dump); ++} ++ ++#else /* CONFIG_DEBUG_FS */ ++ ++int kbase_debug_job_fault_dev_init(struct kbase_device *kbdev) ++{ ++ kbdev->job_fault_debug = false; ++ ++ return 0; ++} ++ ++void kbase_debug_job_fault_dev_term(struct kbase_device *kbdev) ++{ ++} ++ ++#endif /* CONFIG_DEBUG_FS */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_debug_job_fault.h b/drivers/gpu/arm/midgard/mali_kbase_debug_job_fault.h +new file mode 100755 +index 000000000000..a2bf8983c37c +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_debug_job_fault.h +@@ -0,0 +1,96 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KBASE_DEBUG_JOB_FAULT_H ++#define _KBASE_DEBUG_JOB_FAULT_H ++ ++#include ++#include ++ ++#define REGISTER_DUMP_TERMINATION_FLAG 0xFFFFFFFF ++ ++/** ++ * kbase_debug_job_fault_dev_init - Create the fault event wait queue ++ * per device and initialize the required lists. ++ * @kbdev: Device pointer ++ * ++ * Return: Zero on success or a negative error code. ++ */ ++int kbase_debug_job_fault_dev_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_debug_job_fault_debugfs_init - Initialize job fault debug sysfs ++ * @kbdev: Device pointer ++ */ ++void kbase_debug_job_fault_debugfs_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_debug_job_fault_dev_term - Clean up resources created in ++ * kbase_debug_job_fault_dev_init. ++ * @kbdev: Device pointer ++ */ ++void kbase_debug_job_fault_dev_term(struct kbase_device *kbdev); ++ ++/** ++ * kbase_debug_job_fault_context_init - Initialize the relevant ++ * data structure per context ++ * @kctx: KBase context pointer ++ */ ++void kbase_debug_job_fault_context_init(struct kbase_context *kctx); ++ ++/** ++ * kbase_debug_job_fault_context_term - Release the relevant ++ * resource per context ++ * @kctx: KBase context pointer ++ */ ++void kbase_debug_job_fault_context_term(struct kbase_context *kctx); ++ ++/** ++ * kbase_debug_job_fault_process - Process the failed job. ++ * It will send a event and wake up the job fault waiting queue ++ * Then create a work queue to wait for job dump finish ++ * This function should be called in the interrupt handler and before ++ * jd_done that make sure the jd_done_worker will be delayed until the ++ * job dump finish ++ * @katom: The failed atom pointer ++ * @completion_code: the job status ++ * @return true if dump is going on ++ */ ++bool kbase_debug_job_fault_process(struct kbase_jd_atom *katom, ++ u32 completion_code); ++ ++ ++/** ++ * kbase_debug_job_fault_reg_snapshot_init - Set the interested registers ++ * address during the job fault process, the relevant registers will ++ * be saved when a job fault happen ++ * @kctx: KBase context pointer ++ * @reg_range: Maximum register address space ++ * @return true if initializing successfully ++ */ ++bool kbase_debug_job_fault_reg_snapshot_init(struct kbase_context *kctx, ++ int reg_range); ++ ++/** ++ * kbase_job_fault_get_reg_snapshot - Read the interested registers for ++ * failed job dump ++ * @kctx: KBase context pointer ++ * @return true if getting registers successfully ++ */ ++bool kbase_job_fault_get_reg_snapshot(struct kbase_context *kctx); ++ ++#endif /*_KBASE_DEBUG_JOB_FAULT_H*/ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_debug_mem_view.c b/drivers/gpu/arm/midgard/mali_kbase_debug_mem_view.c +new file mode 100755 +index 000000000000..6f2cbdf571cb +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_debug_mem_view.c +@@ -0,0 +1,306 @@ ++/* ++ * ++ * (C) COPYRIGHT 2013-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* ++ * Debugfs interface to dump the memory visible to the GPU ++ */ ++ ++#include "mali_kbase_debug_mem_view.h" ++#include "mali_kbase.h" ++ ++#include ++#include ++ ++#ifdef CONFIG_DEBUG_FS ++ ++struct debug_mem_mapping { ++ struct list_head node; ++ ++ struct kbase_mem_phy_alloc *alloc; ++ unsigned long flags; ++ ++ u64 start_pfn; ++ size_t nr_pages; ++}; ++ ++struct debug_mem_data { ++ struct list_head mapping_list; ++ struct kbase_context *kctx; ++}; ++ ++struct debug_mem_seq_off { ++ struct list_head *lh; ++ size_t offset; ++}; ++ ++static void *debug_mem_start(struct seq_file *m, loff_t *_pos) ++{ ++ struct debug_mem_data *mem_data = m->private; ++ struct debug_mem_seq_off *data; ++ struct debug_mem_mapping *map; ++ loff_t pos = *_pos; ++ ++ list_for_each_entry(map, &mem_data->mapping_list, node) { ++ if (pos >= map->nr_pages) { ++ pos -= map->nr_pages; ++ } else { ++ data = kmalloc(sizeof(*data), GFP_KERNEL); ++ if (!data) ++ return NULL; ++ data->lh = &map->node; ++ data->offset = pos; ++ return data; ++ } ++ } ++ ++ /* Beyond the end */ ++ return NULL; ++} ++ ++static void debug_mem_stop(struct seq_file *m, void *v) ++{ ++ kfree(v); ++} ++ ++static void *debug_mem_next(struct seq_file *m, void *v, loff_t *pos) ++{ ++ struct debug_mem_data *mem_data = m->private; ++ struct debug_mem_seq_off *data = v; ++ struct debug_mem_mapping *map; ++ ++ map = list_entry(data->lh, struct debug_mem_mapping, node); ++ ++ if (data->offset < map->nr_pages - 1) { ++ data->offset++; ++ ++*pos; ++ return data; ++ } ++ ++ if (list_is_last(data->lh, &mem_data->mapping_list)) { ++ kfree(data); ++ return NULL; ++ } ++ ++ data->lh = data->lh->next; ++ data->offset = 0; ++ ++*pos; ++ ++ return data; ++} ++ ++static int debug_mem_show(struct seq_file *m, void *v) ++{ ++ struct debug_mem_data *mem_data = m->private; ++ struct debug_mem_seq_off *data = v; ++ struct debug_mem_mapping *map; ++ int i, j; ++ struct page *page; ++ uint32_t *mapping; ++ pgprot_t prot = PAGE_KERNEL; ++ ++ map = list_entry(data->lh, struct debug_mem_mapping, node); ++ ++ kbase_gpu_vm_lock(mem_data->kctx); ++ ++ if (data->offset >= map->alloc->nents) { ++ seq_printf(m, "%016llx: Unbacked page\n\n", (map->start_pfn + ++ data->offset) << PAGE_SHIFT); ++ goto out; ++ } ++ ++ if (!(map->flags & KBASE_REG_CPU_CACHED)) ++ prot = pgprot_writecombine(prot); ++ ++ page = pfn_to_page(PFN_DOWN(map->alloc->pages[data->offset])); ++ mapping = vmap(&page, 1, VM_MAP, prot); ++ if (!mapping) ++ goto out; ++ ++ for (i = 0; i < PAGE_SIZE; i += 4*sizeof(*mapping)) { ++ seq_printf(m, "%016llx:", i + ((map->start_pfn + ++ data->offset) << PAGE_SHIFT)); ++ ++ for (j = 0; j < 4*sizeof(*mapping); j += sizeof(*mapping)) ++ seq_printf(m, " %08x", mapping[(i+j)/sizeof(*mapping)]); ++ seq_putc(m, '\n'); ++ } ++ ++ vunmap(mapping); ++ ++ seq_putc(m, '\n'); ++ ++out: ++ kbase_gpu_vm_unlock(mem_data->kctx); ++ return 0; ++} ++ ++static const struct seq_operations ops = { ++ .start = debug_mem_start, ++ .next = debug_mem_next, ++ .stop = debug_mem_stop, ++ .show = debug_mem_show, ++}; ++ ++static int debug_mem_zone_open(struct rb_root *rbtree, ++ struct debug_mem_data *mem_data) ++{ ++ int ret = 0; ++ struct rb_node *p; ++ struct kbase_va_region *reg; ++ struct debug_mem_mapping *mapping; ++ ++ for (p = rb_first(rbtree); p; p = rb_next(p)) { ++ reg = rb_entry(p, struct kbase_va_region, rblink); ++ ++ if (reg->gpu_alloc == NULL) ++ /* Empty region - ignore */ ++ continue; ++ ++ mapping = kmalloc(sizeof(*mapping), GFP_KERNEL); ++ if (!mapping) { ++ ret = -ENOMEM; ++ goto out; ++ } ++ ++ mapping->alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); ++ mapping->start_pfn = reg->start_pfn; ++ mapping->nr_pages = reg->nr_pages; ++ mapping->flags = reg->flags; ++ list_add_tail(&mapping->node, &mem_data->mapping_list); ++ } ++ ++out: ++ return ret; ++} ++ ++static int debug_mem_open(struct inode *i, struct file *file) ++{ ++ struct file *kctx_file = i->i_private; ++ struct kbase_context *kctx = kctx_file->private_data; ++ struct debug_mem_data *mem_data; ++ int ret; ++ ++ ret = seq_open(file, &ops); ++ if (ret) ++ return ret; ++ ++ mem_data = kmalloc(sizeof(*mem_data), GFP_KERNEL); ++ if (!mem_data) { ++ ret = -ENOMEM; ++ goto out; ++ } ++ ++ mem_data->kctx = kctx; ++ ++ INIT_LIST_HEAD(&mem_data->mapping_list); ++ ++ get_file(kctx_file); ++ ++ kbase_gpu_vm_lock(kctx); ++ ++ ret = debug_mem_zone_open(&kctx->reg_rbtree_same, mem_data); ++ if (0 != ret) { ++ kbase_gpu_vm_unlock(kctx); ++ goto out; ++ } ++ ++ ret = debug_mem_zone_open(&kctx->reg_rbtree_exec, mem_data); ++ if (0 != ret) { ++ kbase_gpu_vm_unlock(kctx); ++ goto out; ++ } ++ ++ ret = debug_mem_zone_open(&kctx->reg_rbtree_custom, mem_data); ++ if (0 != ret) { ++ kbase_gpu_vm_unlock(kctx); ++ goto out; ++ } ++ ++ kbase_gpu_vm_unlock(kctx); ++ ++ ((struct seq_file *)file->private_data)->private = mem_data; ++ ++ return 0; ++ ++out: ++ if (mem_data) { ++ while (!list_empty(&mem_data->mapping_list)) { ++ struct debug_mem_mapping *mapping; ++ ++ mapping = list_first_entry(&mem_data->mapping_list, ++ struct debug_mem_mapping, node); ++ kbase_mem_phy_alloc_put(mapping->alloc); ++ list_del(&mapping->node); ++ kfree(mapping); ++ } ++ fput(kctx_file); ++ kfree(mem_data); ++ } ++ seq_release(i, file); ++ return ret; ++} ++ ++static int debug_mem_release(struct inode *inode, struct file *file) ++{ ++ struct file *kctx_file = inode->i_private; ++ struct seq_file *sfile = file->private_data; ++ struct debug_mem_data *mem_data = sfile->private; ++ struct debug_mem_mapping *mapping; ++ ++ seq_release(inode, file); ++ ++ while (!list_empty(&mem_data->mapping_list)) { ++ mapping = list_first_entry(&mem_data->mapping_list, ++ struct debug_mem_mapping, node); ++ kbase_mem_phy_alloc_put(mapping->alloc); ++ list_del(&mapping->node); ++ kfree(mapping); ++ } ++ ++ kfree(mem_data); ++ ++ fput(kctx_file); ++ ++ return 0; ++} ++ ++static const struct file_operations kbase_debug_mem_view_fops = { ++ .open = debug_mem_open, ++ .release = debug_mem_release, ++ .read = seq_read, ++ .llseek = seq_lseek ++}; ++ ++/** ++ * kbase_debug_mem_view_init - Initialise the mem_view sysfs file ++ * @kctx_file: The /dev/mali0 file instance for the context ++ * ++ * This function creates a "mem_view" file which can be used to get a view of ++ * the context's memory as the GPU sees it (i.e. using the GPU's page tables). ++ * ++ * The file is cleaned up by a call to debugfs_remove_recursive() deleting the ++ * parent directory. ++ */ ++void kbase_debug_mem_view_init(struct file *kctx_file) ++{ ++ struct kbase_context *kctx = kctx_file->private_data; ++ ++ debugfs_create_file("mem_view", S_IRUGO, kctx->kctx_dentry, kctx_file, ++ &kbase_debug_mem_view_fops); ++} ++ ++#endif +diff --git a/drivers/gpu/arm/midgard/mali_kbase_debug_mem_view.h b/drivers/gpu/arm/midgard/mali_kbase_debug_mem_view.h +new file mode 100755 +index 000000000000..20ab51a776c6 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_debug_mem_view.h +@@ -0,0 +1,25 @@ ++/* ++ * ++ * (C) COPYRIGHT 2013-2014 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KBASE_DEBUG_MEM_VIEW_H ++#define _KBASE_DEBUG_MEM_VIEW_H ++ ++#include ++ ++void kbase_debug_mem_view_init(struct file *kctx_file); ++ ++#endif +diff --git a/drivers/gpu/arm/midgard/mali_kbase_defs.h b/drivers/gpu/arm/midgard/mali_kbase_defs.h +new file mode 100755 +index 000000000000..b35cade71391 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_defs.h +@@ -0,0 +1,1600 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/** ++ * @file mali_kbase_defs.h ++ * ++ * Defintions (types, defines, etcs) common to Kbase. They are placed here to ++ * allow the hierarchy of header files to work. ++ */ ++ ++#ifndef _KBASE_DEFS_H_ ++#define _KBASE_DEFS_H_ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#include ++#include ++#include ++ ++#ifdef CONFIG_MALI_FPGA_BUS_LOGGER ++#include ++#endif ++ ++ ++#ifdef CONFIG_KDS ++#include ++#endif /* CONFIG_KDS */ ++ ++#if defined(CONFIG_SYNC) ++#include ++#else ++#include "mali_kbase_fence_defs.h" ++#endif ++ ++#ifdef CONFIG_DEBUG_FS ++#include ++#endif /* CONFIG_DEBUG_FS */ ++ ++#ifdef CONFIG_MALI_DEVFREQ ++#include ++#endif /* CONFIG_MALI_DEVFREQ */ ++ ++#include ++#include ++ ++#if defined(CONFIG_PM) ++#define KBASE_PM_RUNTIME 1 ++#endif ++ ++/** Enable SW tracing when set */ ++#ifdef CONFIG_MALI_MIDGARD_ENABLE_TRACE ++#define KBASE_TRACE_ENABLE 1 ++#endif ++ ++#ifndef KBASE_TRACE_ENABLE ++#ifdef CONFIG_MALI_DEBUG ++#define KBASE_TRACE_ENABLE 1 ++#else ++#define KBASE_TRACE_ENABLE 0 ++#endif /* CONFIG_MALI_DEBUG */ ++#endif /* KBASE_TRACE_ENABLE */ ++ ++/** Dump Job slot trace on error (only active if KBASE_TRACE_ENABLE != 0) */ ++#define KBASE_TRACE_DUMP_ON_JOB_SLOT_ERROR 1 ++ ++/** ++ * Number of milliseconds before resetting the GPU when a job cannot be "zapped" from the hardware. ++ * Note that the time is actually ZAP_TIMEOUT+SOFT_STOP_RESET_TIMEOUT between the context zap starting and the GPU ++ * actually being reset to give other contexts time for their jobs to be soft-stopped and removed from the hardware ++ * before resetting. ++ */ ++#define ZAP_TIMEOUT 1000 ++ ++/** Number of milliseconds before we time out on a GPU soft/hard reset */ ++#define RESET_TIMEOUT 500 ++ ++/** ++ * Prevent soft-stops from occuring in scheduling situations ++ * ++ * This is not due to HW issues, but when scheduling is desired to be more predictable. ++ * ++ * Therefore, soft stop may still be disabled due to HW issues. ++ * ++ * @note Soft stop will still be used for non-scheduling purposes e.g. when terminating a context. ++ * ++ * @note if not in use, define this value to 0 instead of \#undef'ing it ++ */ ++#define KBASE_DISABLE_SCHEDULING_SOFT_STOPS 0 ++ ++/** ++ * Prevent hard-stops from occuring in scheduling situations ++ * ++ * This is not due to HW issues, but when scheduling is desired to be more predictable. ++ * ++ * @note Hard stop will still be used for non-scheduling purposes e.g. when terminating a context. ++ * ++ * @note if not in use, define this value to 0 instead of \#undef'ing it ++ */ ++#define KBASE_DISABLE_SCHEDULING_HARD_STOPS 0 ++ ++/** ++ * The maximum number of Job Slots to support in the Hardware. ++ * ++ * You can optimize this down if your target devices will only ever support a ++ * small number of job slots. ++ */ ++#define BASE_JM_MAX_NR_SLOTS 3 ++ ++/** ++ * The maximum number of Address Spaces to support in the Hardware. ++ * ++ * You can optimize this down if your target devices will only ever support a ++ * small number of Address Spaces ++ */ ++#define BASE_MAX_NR_AS 16 ++ ++/* mmu */ ++#define MIDGARD_MMU_VA_BITS 48 ++ ++#if MIDGARD_MMU_VA_BITS > 39 ++#define MIDGARD_MMU_TOPLEVEL 0 ++#else ++#define MIDGARD_MMU_TOPLEVEL 1 ++#endif ++ ++#define MIDGARD_MMU_BOTTOMLEVEL 3 ++ ++#define GROWABLE_FLAGS_REQUIRED (KBASE_REG_PF_GROW | KBASE_REG_GPU_WR) ++ ++/** setting in kbase_context::as_nr that indicates it's invalid */ ++#define KBASEP_AS_NR_INVALID (-1) ++ ++#define KBASE_LOCK_REGION_MAX_SIZE (63) ++#define KBASE_LOCK_REGION_MIN_SIZE (11) ++ ++#define KBASE_TRACE_SIZE_LOG2 8 /* 256 entries */ ++#define KBASE_TRACE_SIZE (1 << KBASE_TRACE_SIZE_LOG2) ++#define KBASE_TRACE_MASK ((1 << KBASE_TRACE_SIZE_LOG2)-1) ++ ++#include "mali_kbase_js_defs.h" ++#include "mali_kbase_hwaccess_defs.h" ++ ++#define KBASEP_FORCE_REPLAY_DISABLED 0 ++ ++/* Maximum force replay limit when randomization is enabled */ ++#define KBASEP_FORCE_REPLAY_RANDOM_LIMIT 16 ++ ++/** Atom has been previously soft-stoppped */ ++#define KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED (1<<1) ++/** Atom has been previously retried to execute */ ++#define KBASE_KATOM_FLAGS_RERUN (1<<2) ++#define KBASE_KATOM_FLAGS_JOBCHAIN (1<<3) ++/** Atom has been previously hard-stopped. */ ++#define KBASE_KATOM_FLAG_BEEN_HARD_STOPPED (1<<4) ++/** Atom has caused us to enter disjoint state */ ++#define KBASE_KATOM_FLAG_IN_DISJOINT (1<<5) ++/* Atom blocked on cross-slot dependency */ ++#define KBASE_KATOM_FLAG_X_DEP_BLOCKED (1<<7) ++/* Atom has fail dependency on cross-slot dependency */ ++#define KBASE_KATOM_FLAG_FAIL_BLOCKER (1<<8) ++/* Atom is currently in the list of atoms blocked on cross-slot dependencies */ ++#define KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST (1<<9) ++/* Atom is currently holding a context reference */ ++#define KBASE_KATOM_FLAG_HOLDING_CTX_REF (1<<10) ++/* Atom requires GPU to be in protected mode */ ++#define KBASE_KATOM_FLAG_PROTECTED (1<<11) ++/* Atom has been stored in runnable_tree */ ++#define KBASE_KATOM_FLAG_JSCTX_IN_TREE (1<<12) ++ ++/* SW related flags about types of JS_COMMAND action ++ * NOTE: These must be masked off by JS_COMMAND_MASK */ ++ ++/** This command causes a disjoint event */ ++#define JS_COMMAND_SW_CAUSES_DISJOINT 0x100 ++ ++/** Bitmask of all SW related flags */ ++#define JS_COMMAND_SW_BITS (JS_COMMAND_SW_CAUSES_DISJOINT) ++ ++#if (JS_COMMAND_SW_BITS & JS_COMMAND_MASK) ++#error JS_COMMAND_SW_BITS not masked off by JS_COMMAND_MASK. Must update JS_COMMAND_SW_<..> bitmasks ++#endif ++ ++/** Soft-stop command that causes a Disjoint event. This of course isn't ++ * entirely masked off by JS_COMMAND_MASK */ ++#define JS_COMMAND_SOFT_STOP_WITH_SW_DISJOINT \ ++ (JS_COMMAND_SW_CAUSES_DISJOINT | JS_COMMAND_SOFT_STOP) ++ ++#define KBASEP_ATOM_ID_INVALID BASE_JD_ATOM_COUNT ++ ++/* Serialize atoms within a slot (ie only one atom per job slot) */ ++#define KBASE_SERIALIZE_INTRA_SLOT (1 << 0) ++/* Serialize atoms between slots (ie only one job slot running at any time) */ ++#define KBASE_SERIALIZE_INTER_SLOT (1 << 1) ++/* Reset the GPU after each atom completion */ ++#define KBASE_SERIALIZE_RESET (1 << 2) ++ ++#ifdef CONFIG_DEBUG_FS ++struct base_job_fault_event { ++ ++ u32 event_code; ++ struct kbase_jd_atom *katom; ++ struct work_struct job_fault_work; ++ struct list_head head; ++ int reg_offset; ++}; ++ ++#endif ++ ++struct kbase_jd_atom_dependency { ++ struct kbase_jd_atom *atom; ++ u8 dep_type; ++}; ++ ++/** ++ * struct kbase_io_access - holds information about 1 register access ++ * ++ * @addr: first bit indicates r/w (r=0, w=1) ++ * @value: value written or read ++ */ ++struct kbase_io_access { ++ uintptr_t addr; ++ u32 value; ++}; ++ ++/** ++ * struct kbase_io_history - keeps track of all recent register accesses ++ * ++ * @enabled: true if register accesses are recorded, false otherwise ++ * @lock: spinlock protecting kbase_io_access array ++ * @count: number of registers read/written ++ * @size: number of elements in kbase_io_access array ++ * @buf: array of kbase_io_access ++ */ ++struct kbase_io_history { ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0)) ++ bool enabled; ++#else ++ u32 enabled; ++#endif ++ ++ spinlock_t lock; ++ size_t count; ++ u16 size; ++ struct kbase_io_access *buf; ++}; ++ ++/** ++ * @brief The function retrieves a read-only reference to the atom field from ++ * the kbase_jd_atom_dependency structure ++ * ++ * @param[in] dep kbase jd atom dependency. ++ * ++ * @return readonly reference to dependent ATOM. ++ */ ++static inline const struct kbase_jd_atom * kbase_jd_katom_dep_atom(const struct kbase_jd_atom_dependency *dep) ++{ ++ LOCAL_ASSERT(dep != NULL); ++ ++ return (const struct kbase_jd_atom *)(dep->atom); ++} ++ ++/** ++ * @brief The function retrieves a read-only reference to the dependency type field from ++ * the kbase_jd_atom_dependency structure ++ * ++ * @param[in] dep kbase jd atom dependency. ++ * ++ * @return A dependency type value. ++ */ ++static inline u8 kbase_jd_katom_dep_type(const struct kbase_jd_atom_dependency *dep) ++{ ++ LOCAL_ASSERT(dep != NULL); ++ ++ return dep->dep_type; ++} ++ ++/** ++ * @brief Setter macro for dep_atom array entry in kbase_jd_atom ++ * ++ * @param[in] dep The kbase jd atom dependency. ++ * @param[in] a The ATOM to be set as a dependency. ++ * @param type The ATOM dependency type to be set. ++ * ++ */ ++static inline void kbase_jd_katom_dep_set(const struct kbase_jd_atom_dependency *const_dep, ++ struct kbase_jd_atom *a, u8 type) ++{ ++ struct kbase_jd_atom_dependency *dep; ++ ++ LOCAL_ASSERT(const_dep != NULL); ++ ++ dep = (struct kbase_jd_atom_dependency *)const_dep; ++ ++ dep->atom = a; ++ dep->dep_type = type; ++} ++ ++/** ++ * @brief Setter macro for dep_atom array entry in kbase_jd_atom ++ * ++ * @param[in] dep The kbase jd atom dependency to be cleared. ++ * ++ */ ++static inline void kbase_jd_katom_dep_clear(const struct kbase_jd_atom_dependency *const_dep) ++{ ++ struct kbase_jd_atom_dependency *dep; ++ ++ LOCAL_ASSERT(const_dep != NULL); ++ ++ dep = (struct kbase_jd_atom_dependency *)const_dep; ++ ++ dep->atom = NULL; ++ dep->dep_type = BASE_JD_DEP_TYPE_INVALID; ++} ++ ++enum kbase_atom_gpu_rb_state { ++ /* Atom is not currently present in slot ringbuffer */ ++ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB, ++ /* Atom is in slot ringbuffer but is blocked on a previous atom */ ++ KBASE_ATOM_GPU_RB_WAITING_BLOCKED, ++ /* Atom is in slot ringbuffer but is waiting for a previous protected ++ * mode transition to complete */ ++ KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV, ++ /* Atom is in slot ringbuffer but is waiting for proected mode ++ * transition */ ++ KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION, ++ /* Atom is in slot ringbuffer but is waiting for cores to become ++ * available */ ++ KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE, ++ /* Atom is in slot ringbuffer but is blocked on affinity */ ++ KBASE_ATOM_GPU_RB_WAITING_AFFINITY, ++ /* Atom is in slot ringbuffer and ready to run */ ++ KBASE_ATOM_GPU_RB_READY, ++ /* Atom is in slot ringbuffer and has been submitted to the GPU */ ++ KBASE_ATOM_GPU_RB_SUBMITTED, ++ /* Atom must be returned to JS as soon as it reaches the head of the ++ * ringbuffer due to a previous failure */ ++ KBASE_ATOM_GPU_RB_RETURN_TO_JS = -1 ++}; ++ ++enum kbase_atom_enter_protected_state { ++ /* ++ * Starting state: ++ * Check if a transition into protected mode is required. ++ * ++ * NOTE: The integer value of this must ++ * match KBASE_ATOM_EXIT_PROTECTED_CHECK. ++ */ ++ KBASE_ATOM_ENTER_PROTECTED_CHECK = 0, ++ /* Wait for vinstr to suspend. */ ++ KBASE_ATOM_ENTER_PROTECTED_VINSTR, ++ /* Wait for the L2 to become idle in preparation for ++ * the coherency change. */ ++ KBASE_ATOM_ENTER_PROTECTED_IDLE_L2, ++ /* End state; ++ * Prepare coherency change. */ ++ KBASE_ATOM_ENTER_PROTECTED_FINISHED, ++}; ++ ++enum kbase_atom_exit_protected_state { ++ /* ++ * Starting state: ++ * Check if a transition out of protected mode is required. ++ * ++ * NOTE: The integer value of this must ++ * match KBASE_ATOM_ENTER_PROTECTED_CHECK. ++ */ ++ KBASE_ATOM_EXIT_PROTECTED_CHECK = 0, ++ /* Wait for the L2 to become idle in preparation ++ * for the reset. */ ++ KBASE_ATOM_EXIT_PROTECTED_IDLE_L2, ++ /* Issue the protected reset. */ ++ KBASE_ATOM_EXIT_PROTECTED_RESET, ++ /* End state; ++ * Wait for the reset to complete. */ ++ KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT, ++}; ++ ++struct kbase_ext_res { ++ u64 gpu_address; ++ struct kbase_mem_phy_alloc *alloc; ++}; ++ ++struct kbase_jd_atom { ++ struct work_struct work; ++ ktime_t start_timestamp; ++ ++ struct base_jd_udata udata; ++ struct kbase_context *kctx; ++ ++ struct list_head dep_head[2]; ++ struct list_head dep_item[2]; ++ const struct kbase_jd_atom_dependency dep[2]; ++ /* List head used during job dispatch job_done processing - as ++ * dependencies may not be entirely resolved at this point, we need to ++ * use a separate list head. */ ++ struct list_head jd_item; ++ /* true if atom's jd_item is currently on a list. Prevents atom being ++ * processed twice. */ ++ bool in_jd_list; ++ ++ u16 nr_extres; ++ struct kbase_ext_res *extres; ++ ++ u32 device_nr; ++ u64 affinity; ++ u64 jc; ++ enum kbase_atom_coreref_state coreref_state; ++#ifdef CONFIG_KDS ++ struct list_head node; ++ struct kds_resource_set *kds_rset; ++ bool kds_dep_satisfied; ++#endif /* CONFIG_KDS */ ++#if defined(CONFIG_SYNC) ++ /* Stores either an input or output fence, depending on soft-job type */ ++ struct sync_fence *fence; ++ struct sync_fence_waiter sync_waiter; ++#endif /* CONFIG_SYNC */ ++#if defined(CONFIG_MALI_DMA_FENCE) || defined(CONFIG_SYNC_FILE) ++ struct { ++ /* Use the functions/API defined in mali_kbase_fence.h to ++ * when working with this sub struct */ ++#if defined(CONFIG_SYNC_FILE) ++ /* Input fence */ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ struct fence *fence_in; ++#else ++ struct dma_fence *fence_in; ++#endif ++#endif ++ /* This points to the dma-buf output fence for this atom. If ++ * this is NULL then there is no fence for this atom and the ++ * following fields related to dma_fence may have invalid data. ++ * ++ * The context and seqno fields contain the details for this ++ * fence. ++ * ++ * This fence is signaled when the katom is completed, ++ * regardless of the event_code of the katom (signal also on ++ * failure). ++ */ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ struct fence *fence; ++#else ++ struct dma_fence *fence; ++#endif ++ /* The dma-buf fence context number for this atom. A unique ++ * context number is allocated to each katom in the context on ++ * context creation. ++ */ ++ unsigned int context; ++ /* The dma-buf fence sequence number for this atom. This is ++ * increased every time this katom uses dma-buf fence. ++ */ ++ atomic_t seqno; ++ /* This contains a list of all callbacks set up to wait on ++ * other fences. This atom must be held back from JS until all ++ * these callbacks have been called and dep_count have reached ++ * 0. The initial value of dep_count must be equal to the ++ * number of callbacks on this list. ++ * ++ * This list is protected by jctx.lock. Callbacks are added to ++ * this list when the atom is built and the wait are set up. ++ * All the callbacks then stay on the list until all callbacks ++ * have been called and the atom is queued, or cancelled, and ++ * then all callbacks are taken off the list and freed. ++ */ ++ struct list_head callbacks; ++ /* Atomic counter of number of outstandind dma-buf fence ++ * dependencies for this atom. When dep_count reaches 0 the ++ * atom may be queued. ++ * ++ * The special value "-1" may only be set after the count ++ * reaches 0, while holding jctx.lock. This indicates that the ++ * atom has been handled, either queued in JS or cancelled. ++ * ++ * If anyone but the dma-fence worker sets this to -1 they must ++ * ensure that any potentially queued worker must have ++ * completed before allowing the atom to be marked as unused. ++ * This can be done by flushing the fence work queue: ++ * kctx->dma_fence.wq. ++ */ ++ atomic_t dep_count; ++ } dma_fence; ++#endif /* CONFIG_MALI_DMA_FENCE || CONFIG_SYNC_FILE*/ ++ ++ /* Note: refer to kbasep_js_atom_retained_state, which will take a copy of some of the following members */ ++ enum base_jd_event_code event_code; ++ base_jd_core_req core_req; /**< core requirements */ ++ /** Job Slot to retry submitting to if submission from IRQ handler failed ++ * ++ * NOTE: see if this can be unified into the another member e.g. the event */ ++ int retry_submit_on_slot; ++ ++ u32 ticks; ++ /* JS atom priority with respect to other atoms on its kctx. */ ++ int sched_priority; ++ ++ int poking; /* BASE_HW_ISSUE_8316 */ ++ ++ wait_queue_head_t completed; ++ enum kbase_jd_atom_state status; ++#ifdef CONFIG_GPU_TRACEPOINTS ++ int work_id; ++#endif ++ /* Assigned after atom is completed. Used to check whether PRLAM-10676 workaround should be applied */ ++ int slot_nr; ++ ++ u32 atom_flags; ++ ++ /* Number of times this atom has been retried. Used by replay soft job. ++ */ ++ int retry_count; ++ ++ enum kbase_atom_gpu_rb_state gpu_rb_state; ++ ++ u64 need_cache_flush_cores_retained; ++ ++ atomic_t blocked; ++ ++ /* Pointer to atom that this atom has same-slot dependency on */ ++ struct kbase_jd_atom *pre_dep; ++ /* Pointer to atom that has same-slot dependency on this atom */ ++ struct kbase_jd_atom *post_dep; ++ ++ /* Pointer to atom that this atom has cross-slot dependency on */ ++ struct kbase_jd_atom *x_pre_dep; ++ /* Pointer to atom that has cross-slot dependency on this atom */ ++ struct kbase_jd_atom *x_post_dep; ++ ++ /* The GPU's flush count recorded at the time of submission, used for ++ * the cache flush optimisation */ ++ u32 flush_id; ++ ++ struct kbase_jd_atom_backend backend; ++#ifdef CONFIG_DEBUG_FS ++ struct base_job_fault_event fault_event; ++#endif ++ ++ /* List head used for three different purposes: ++ * 1. Overflow list for JS ring buffers. If an atom is ready to run, ++ * but there is no room in the JS ring buffer, then the atom is put ++ * on the ring buffer's overflow list using this list node. ++ * 2. List of waiting soft jobs. ++ */ ++ struct list_head queue; ++ ++ /* Used to keep track of all JIT free/alloc jobs in submission order ++ */ ++ struct list_head jit_node; ++ bool jit_blocked; ++ ++ /* If non-zero, this indicates that the atom will fail with the set ++ * event_code when the atom is processed. */ ++ enum base_jd_event_code will_fail_event_code; ++ ++ /* Atoms will only ever be transitioning into, or out of ++ * protected mode so we do not need two separate fields. ++ */ ++ union { ++ enum kbase_atom_enter_protected_state enter; ++ enum kbase_atom_exit_protected_state exit; ++ } protected_state; ++ ++ struct rb_node runnable_tree_node; ++ ++ /* 'Age' of atom relative to other atoms in the context. */ ++ u32 age; ++}; ++ ++static inline bool kbase_jd_katom_is_protected(const struct kbase_jd_atom *katom) ++{ ++ return (bool)(katom->atom_flags & KBASE_KATOM_FLAG_PROTECTED); ++} ++ ++/* ++ * Theory of operations: ++ * ++ * Atom objects are statically allocated within the context structure. ++ * ++ * Each atom is the head of two lists, one for the "left" set of dependencies, one for the "right" set. ++ */ ++ ++#define KBASE_JD_DEP_QUEUE_SIZE 256 ++ ++struct kbase_jd_context { ++ struct mutex lock; ++ struct kbasep_js_kctx_info sched_info; ++ struct kbase_jd_atom atoms[BASE_JD_ATOM_COUNT]; ++ ++ /** Tracks all job-dispatch jobs. This includes those not tracked by ++ * the scheduler: 'not ready to run' and 'dependency-only' jobs. */ ++ u32 job_nr; ++ ++ /** Waitq that reflects whether there are no jobs (including SW-only ++ * dependency jobs). This is set when no jobs are present on the ctx, ++ * and clear when there are jobs. ++ * ++ * @note: Job Dispatcher knows about more jobs than the Job Scheduler: ++ * the Job Scheduler is unaware of jobs that are blocked on dependencies, ++ * and SW-only dependency jobs. ++ * ++ * This waitq can be waited upon to find out when the context jobs are all ++ * done/cancelled (including those that might've been blocked on ++ * dependencies) - and so, whether it can be terminated. However, it should ++ * only be terminated once it is not present in the run-pool (see ++ * kbasep_js_kctx_info::ctx::is_scheduled). ++ * ++ * Since the waitq is only set under kbase_jd_context::lock, ++ * the waiter should also briefly obtain and drop kbase_jd_context::lock to ++ * guarentee that the setter has completed its work on the kbase_context ++ * ++ * This must be updated atomically with: ++ * - kbase_jd_context::job_nr */ ++ wait_queue_head_t zero_jobs_wait; ++ ++ /** Job Done workqueue. */ ++ struct workqueue_struct *job_done_wq; ++ ++ spinlock_t tb_lock; ++ u32 *tb; ++ size_t tb_wrap_offset; ++ ++#ifdef CONFIG_KDS ++ struct kds_callback kds_cb; ++#endif /* CONFIG_KDS */ ++#ifdef CONFIG_GPU_TRACEPOINTS ++ atomic_t work_id; ++#endif ++}; ++ ++struct kbase_device_info { ++ u32 features; ++}; ++ ++/** Poking state for BASE_HW_ISSUE_8316 */ ++enum { ++ KBASE_AS_POKE_STATE_IN_FLIGHT = 1<<0, ++ KBASE_AS_POKE_STATE_KILLING_POKE = 1<<1 ++}; ++ ++/** Poking state for BASE_HW_ISSUE_8316 */ ++typedef u32 kbase_as_poke_state; ++ ++struct kbase_mmu_setup { ++ u64 transtab; ++ u64 memattr; ++ u64 transcfg; ++}; ++ ++/** ++ * Important: Our code makes assumptions that a struct kbase_as structure is always at ++ * kbase_device->as[number]. This is used to recover the containing ++ * struct kbase_device from a struct kbase_as structure. ++ * ++ * Therefore, struct kbase_as structures must not be allocated anywhere else. ++ */ ++struct kbase_as { ++ int number; ++ ++ struct workqueue_struct *pf_wq; ++ struct work_struct work_pagefault; ++ struct work_struct work_busfault; ++ enum kbase_mmu_fault_type fault_type; ++ bool protected_mode; ++ u32 fault_status; ++ u64 fault_addr; ++ u64 fault_extra_addr; ++ ++ struct kbase_mmu_setup current_setup; ++ ++ /* BASE_HW_ISSUE_8316 */ ++ struct workqueue_struct *poke_wq; ++ struct work_struct poke_work; ++ /** Protected by hwaccess_lock */ ++ int poke_refcount; ++ /** Protected by hwaccess_lock */ ++ kbase_as_poke_state poke_state; ++ struct hrtimer poke_timer; ++}; ++ ++static inline int kbase_as_has_bus_fault(struct kbase_as *as) ++{ ++ return as->fault_type == KBASE_MMU_FAULT_TYPE_BUS; ++} ++ ++static inline int kbase_as_has_page_fault(struct kbase_as *as) ++{ ++ return as->fault_type == KBASE_MMU_FAULT_TYPE_PAGE; ++} ++ ++struct kbasep_mem_device { ++ atomic_t used_pages; /* Tracks usage of OS shared memory. Updated ++ when OS memory is allocated/freed. */ ++ ++}; ++ ++#define KBASE_TRACE_CODE(X) KBASE_TRACE_CODE_ ## X ++ ++enum kbase_trace_code { ++ /* IMPORTANT: USE OF SPECIAL #INCLUDE OF NON-STANDARD HEADER FILE ++ * THIS MUST BE USED AT THE START OF THE ENUM */ ++#define KBASE_TRACE_CODE_MAKE_CODE(X) KBASE_TRACE_CODE(X) ++#include "mali_kbase_trace_defs.h" ++#undef KBASE_TRACE_CODE_MAKE_CODE ++ /* Comma on its own, to extend the list */ ++ , ++ /* Must be the last in the enum */ ++ KBASE_TRACE_CODE_COUNT ++}; ++ ++#define KBASE_TRACE_FLAG_REFCOUNT (((u8)1) << 0) ++#define KBASE_TRACE_FLAG_JOBSLOT (((u8)1) << 1) ++ ++struct kbase_trace { ++ struct timespec64 timestamp; ++ u32 thread_id; ++ u32 cpu; ++ void *ctx; ++ bool katom; ++ int atom_number; ++ u64 atom_udata[2]; ++ u64 gpu_addr; ++ unsigned long info_val; ++ u8 code; ++ u8 jobslot; ++ u8 refcount; ++ u8 flags; ++}; ++ ++/** Event IDs for the power management framework. ++ * ++ * Any of these events might be missed, so they should not be relied upon to ++ * find the precise state of the GPU at a particular time in the ++ * trace. Overall, we should get a high percentage of these events for ++ * statisical purposes, and so a few missing should not be a problem */ ++enum kbase_timeline_pm_event { ++ /* helper for tests */ ++ KBASEP_TIMELINE_PM_EVENT_FIRST, ++ ++ /** Event reserved for backwards compatibility with 'init' events */ ++ KBASE_TIMELINE_PM_EVENT_RESERVED_0 = KBASEP_TIMELINE_PM_EVENT_FIRST, ++ ++ /** The power state of the device has changed. ++ * ++ * Specifically, the device has reached a desired or available state. ++ */ ++ KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED, ++ ++ /** The GPU is becoming active. ++ * ++ * This event is sent when the first context is about to use the GPU. ++ */ ++ KBASE_TIMELINE_PM_EVENT_GPU_ACTIVE, ++ ++ /** The GPU is becoming idle. ++ * ++ * This event is sent when the last context has finished using the GPU. ++ */ ++ KBASE_TIMELINE_PM_EVENT_GPU_IDLE, ++ ++ /** Event reserved for backwards compatibility with 'policy_change' ++ * events */ ++ KBASE_TIMELINE_PM_EVENT_RESERVED_4, ++ ++ /** Event reserved for backwards compatibility with 'system_suspend' ++ * events */ ++ KBASE_TIMELINE_PM_EVENT_RESERVED_5, ++ ++ /** Event reserved for backwards compatibility with 'system_resume' ++ * events */ ++ KBASE_TIMELINE_PM_EVENT_RESERVED_6, ++ ++ /** The job scheduler is requesting to power up/down cores. ++ * ++ * This event is sent when: ++ * - powered down cores are needed to complete a job ++ * - powered up cores are not needed anymore ++ */ ++ KBASE_TIMELINE_PM_EVENT_CHANGE_GPU_STATE, ++ ++ KBASEP_TIMELINE_PM_EVENT_LAST = KBASE_TIMELINE_PM_EVENT_CHANGE_GPU_STATE, ++}; ++ ++#ifdef CONFIG_MALI_TRACE_TIMELINE ++struct kbase_trace_kctx_timeline { ++ atomic_t jd_atoms_in_flight; ++ u32 owner_tgid; ++}; ++ ++struct kbase_trace_kbdev_timeline { ++ /* Note: strictly speaking, not needed, because it's in sync with ++ * kbase_device::jm_slots[]::submitted_nr ++ * ++ * But it's kept as an example of how to add global timeline tracking ++ * information ++ * ++ * The caller must hold hwaccess_lock when accessing this */ ++ u8 slot_atoms_submitted[BASE_JM_MAX_NR_SLOTS]; ++ ++ /* Last UID for each PM event */ ++ atomic_t pm_event_uid[KBASEP_TIMELINE_PM_EVENT_LAST+1]; ++ /* Counter for generating PM event UIDs */ ++ atomic_t pm_event_uid_counter; ++ /* ++ * L2 transition state - true indicates that the transition is ongoing ++ * Expected to be protected by hwaccess_lock */ ++ bool l2_transitioning; ++}; ++#endif /* CONFIG_MALI_TRACE_TIMELINE */ ++ ++ ++struct kbasep_kctx_list_element { ++ struct list_head link; ++ struct kbase_context *kctx; ++}; ++ ++/** ++ * Data stored per device for power management. ++ * ++ * This structure contains data for the power management framework. There is one ++ * instance of this structure per device in the system. ++ */ ++struct kbase_pm_device_data { ++ /** ++ * The lock protecting Power Management structures accessed outside of ++ * IRQ. ++ * ++ * This lock must also be held whenever the GPU is being powered on or ++ * off. ++ */ ++ struct mutex lock; ++ ++ /** The reference count of active contexts on this device. */ ++ int active_count; ++ /** Flag indicating suspending/suspended */ ++ bool suspending; ++ /* Wait queue set when active_count == 0 */ ++ wait_queue_head_t zero_active_count_wait; ++ ++ /** ++ * Bit masks identifying the available shader cores that are specified ++ * via sysfs. One mask per job slot. ++ */ ++ u64 debug_core_mask[BASE_JM_MAX_NR_SLOTS]; ++ u64 debug_core_mask_all; ++ ++ /** ++ * Callback for initializing the runtime power management. ++ * ++ * @param kbdev The kbase device ++ * ++ * @return 0 on success, else error code ++ */ ++ int (*callback_power_runtime_init)(struct kbase_device *kbdev); ++ ++ /** ++ * Callback for terminating the runtime power management. ++ * ++ * @param kbdev The kbase device ++ */ ++ void (*callback_power_runtime_term)(struct kbase_device *kbdev); ++ ++ /* Time in milliseconds between each dvfs sample */ ++ u32 dvfs_period; ++ ++ /* Period of GPU poweroff timer */ ++ ktime_t gpu_poweroff_time; ++ ++ /* Number of ticks of GPU poweroff timer before shader is powered off */ ++ int poweroff_shader_ticks; ++ ++ /* Number of ticks of GPU poweroff timer before GPU is powered off */ ++ int poweroff_gpu_ticks; ++ ++ struct kbase_pm_backend_data backend; ++}; ++ ++/** ++ * struct kbase_mem_pool - Page based memory pool for kctx/kbdev ++ * @kbdev: Kbase device where memory is used ++ * @cur_size: Number of free pages currently in the pool (may exceed @max_size ++ * in some corner cases) ++ * @max_size: Maximum number of free pages in the pool ++ * @pool_lock: Lock protecting the pool - must be held when modifying @cur_size ++ * and @page_list ++ * @page_list: List of free pages in the pool ++ * @reclaim: Shrinker for kernel reclaim of free pages ++ * @next_pool: Pointer to next pool where pages can be allocated when this pool ++ * is empty. Pages will spill over to the next pool when this pool ++ * is full. Can be NULL if there is no next pool. ++ */ ++struct kbase_mem_pool { ++ struct kbase_device *kbdev; ++ size_t cur_size; ++ size_t max_size; ++ spinlock_t pool_lock; ++ struct list_head page_list; ++ struct shrinker reclaim; ++ ++ struct kbase_mem_pool *next_pool; ++}; ++ ++/** ++ * struct kbase_devfreq_opp - Lookup table for converting between nominal OPP ++ * frequency, and real frequency and core mask ++ * @opp_freq: Nominal OPP frequency ++ * @real_freq: Real GPU frequency ++ * @core_mask: Shader core mask ++ */ ++struct kbase_devfreq_opp { ++ u64 opp_freq; ++ u64 real_freq; ++ u64 core_mask; ++}; ++ ++#define DEVNAME_SIZE 16 ++ ++struct kbase_device { ++ s8 slot_submit_count_irq[BASE_JM_MAX_NR_SLOTS]; ++ ++ u32 hw_quirks_sc; ++ u32 hw_quirks_tiler; ++ u32 hw_quirks_mmu; ++ u32 hw_quirks_jm; ++ ++ struct list_head entry; ++ struct device *dev; ++ unsigned int kbase_group_error; ++ struct miscdevice mdev; ++ u64 reg_start; ++ size_t reg_size; ++ void __iomem *reg; ++ ++ struct { ++ int irq; ++ int flags; ++ } irqs[3]; ++ ++ struct clk *clock; ++#ifdef CONFIG_REGULATOR ++ struct regulator *regulator; ++#endif ++ char devname[DEVNAME_SIZE]; ++ ++#ifdef CONFIG_MALI_NO_MALI ++ void *model; ++ struct kmem_cache *irq_slab; ++ struct workqueue_struct *irq_workq; ++ atomic_t serving_job_irq; ++ atomic_t serving_gpu_irq; ++ atomic_t serving_mmu_irq; ++ spinlock_t reg_op_lock; ++#endif /* CONFIG_MALI_NO_MALI */ ++ ++ struct kbase_pm_device_data pm; ++ struct kbasep_js_device_data js_data; ++ struct kbase_mem_pool mem_pool; ++ struct kbasep_mem_device memdev; ++ struct kbase_mmu_mode const *mmu_mode; ++ ++ struct kbase_as as[BASE_MAX_NR_AS]; ++ /* The below variables (as_free and as_to_kctx) are managed by the ++ * Context Scheduler. The kbasep_js_device_data::runpool_irq::lock must ++ * be held whilst accessing these. ++ */ ++ u16 as_free; /* Bitpattern of free Address Spaces */ ++ /* Mapping from active Address Spaces to kbase_context */ ++ struct kbase_context *as_to_kctx[BASE_MAX_NR_AS]; ++ ++ ++ spinlock_t mmu_mask_change; ++ ++ struct kbase_gpu_props gpu_props; ++ ++ /** List of SW workarounds for HW issues */ ++ unsigned long hw_issues_mask[(BASE_HW_ISSUE_END + BITS_PER_LONG - 1) / BITS_PER_LONG]; ++ /** List of features available */ ++ unsigned long hw_features_mask[(BASE_HW_FEATURE_END + BITS_PER_LONG - 1) / BITS_PER_LONG]; ++ ++ /* Bitmaps of cores that are currently in use (running jobs). ++ * These should be kept up to date by the job scheduler. ++ * ++ * pm.power_change_lock should be held when accessing these members. ++ * ++ * kbase_pm_check_transitions_nolock() should be called when bits are ++ * cleared to update the power management system and allow transitions to ++ * occur. */ ++ u64 shader_inuse_bitmap; ++ ++ /* Refcount for cores in use */ ++ u32 shader_inuse_cnt[64]; ++ ++ /* Bitmaps of cores the JS needs for jobs ready to run */ ++ u64 shader_needed_bitmap; ++ ++ /* Refcount for cores needed */ ++ u32 shader_needed_cnt[64]; ++ ++ u32 tiler_inuse_cnt; ++ ++ u32 tiler_needed_cnt; ++ ++ /* struct for keeping track of the disjoint information ++ * ++ * The state is > 0 if the GPU is in a disjoint state. Otherwise 0 ++ * The count is the number of disjoint events that have occurred on the GPU ++ */ ++ struct { ++ atomic_t count; ++ atomic_t state; ++ } disjoint_event; ++ ++ /* Refcount for tracking users of the l2 cache, e.g. when using hardware counter instrumentation. */ ++ u32 l2_users_count; ++ ++ /* Bitmaps of cores that are currently available (powered up and the power policy is happy for jobs to be ++ * submitted to these cores. These are updated by the power management code. The job scheduler should avoid ++ * submitting new jobs to any cores that are not marked as available. ++ * ++ * pm.power_change_lock should be held when accessing these members. ++ */ ++ u64 shader_available_bitmap; ++ u64 tiler_available_bitmap; ++ u64 l2_available_bitmap; ++ u64 stack_available_bitmap; ++ ++ u64 shader_ready_bitmap; ++ u64 shader_transitioning_bitmap; ++ ++ s8 nr_hw_address_spaces; /**< Number of address spaces in the GPU (constant after driver initialisation) */ ++ s8 nr_user_address_spaces; /**< Number of address spaces available to user contexts */ ++ ++ /* Structure used for instrumentation and HW counters dumping */ ++ struct kbase_hwcnt { ++ /* The lock should be used when accessing any of the following members */ ++ spinlock_t lock; ++ ++ struct kbase_context *kctx; ++ u64 addr; ++ ++ struct kbase_instr_backend backend; ++ } hwcnt; ++ ++ struct kbase_vinstr_context *vinstr_ctx; ++ ++#if KBASE_TRACE_ENABLE ++ spinlock_t trace_lock; ++ u16 trace_first_out; ++ u16 trace_next_in; ++ struct kbase_trace *trace_rbuf; ++#endif ++ ++ u32 reset_timeout_ms; ++ ++ struct mutex cacheclean_lock; ++ ++ /* Platform specific private data to be accessed by mali_kbase_config_xxx.c only */ ++ void *platform_context; ++ ++ /* List of kbase_contexts created */ ++ struct list_head kctx_list; ++ struct mutex kctx_list_lock; ++ ++#ifdef CONFIG_MALI_DEVFREQ ++ struct devfreq_dev_profile devfreq_profile; ++ struct devfreq *devfreq; ++ unsigned long current_freq; ++ unsigned long current_nominal_freq; ++ unsigned long current_voltage; ++ u64 current_core_mask; ++ struct kbase_devfreq_opp *opp_table; ++ int num_opps; ++ struct monitor_dev_info *mdev_info; ++#ifdef CONFIG_DEVFREQ_THERMAL ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) ++ struct devfreq_cooling_device *devfreq_cooling; ++#else ++ struct thermal_cooling_device *devfreq_cooling; ++#endif ++ /* Current IPA model - true for configured model, false for fallback */ ++ atomic_t ipa_use_configured_model; ++ struct { ++ /* Access to this struct must be with ipa.lock held */ ++ struct mutex lock; ++ struct kbase_ipa_model *configured_model; ++ struct kbase_ipa_model *fallback_model; ++ } ipa; ++#endif /* CONFIG_DEVFREQ_THERMAL */ ++#endif /* CONFIG_MALI_DEVFREQ */ ++ ++ ++#ifdef CONFIG_MALI_TRACE_TIMELINE ++ struct kbase_trace_kbdev_timeline timeline; ++#endif ++ ++ /* ++ * Control for enabling job dump on failure, set when control debugfs ++ * is opened. ++ */ ++ bool job_fault_debug; ++ ++#ifdef CONFIG_DEBUG_FS ++ /* directory for debugfs entries */ ++ struct dentry *mali_debugfs_directory; ++ /* Root directory for per context entry */ ++ struct dentry *debugfs_ctx_directory; ++ ++#ifdef CONFIG_MALI_DEBUG ++ /* bit for each as, set if there is new data to report */ ++ u64 debugfs_as_read_bitmap; ++#endif /* CONFIG_MALI_DEBUG */ ++ ++ /* failed job dump, used for separate debug process */ ++ wait_queue_head_t job_fault_wq; ++ wait_queue_head_t job_fault_resume_wq; ++ struct workqueue_struct *job_fault_resume_workq; ++ struct list_head job_fault_event_list; ++ spinlock_t job_fault_event_lock; ++ struct kbase_context *kctx_fault; ++ ++#if !MALI_CUSTOMER_RELEASE ++ /* Per-device data for register dumping interface */ ++ struct { ++ u16 reg_offset; /* Offset of a GPU_CONTROL register to be ++ dumped upon request */ ++ } regs_dump_debugfs_data; ++#endif /* !MALI_CUSTOMER_RELEASE */ ++#endif /* CONFIG_DEBUG_FS */ ++ ++ /* fbdump profiling controls set by gator */ ++ u32 kbase_profiling_controls[FBDUMP_CONTROL_MAX]; ++ ++ ++#if MALI_CUSTOMER_RELEASE == 0 ++ /* Number of jobs that are run before a job is forced to fail and ++ * replay. May be KBASEP_FORCE_REPLAY_DISABLED, to disable forced ++ * failures. */ ++ int force_replay_limit; ++ /* Count of jobs between forced failures. Incremented on each job. A ++ * job is forced to fail once this is greater than or equal to ++ * force_replay_limit. */ ++ int force_replay_count; ++ /* Core requirement for jobs to be failed and replayed. May be zero. */ ++ base_jd_core_req force_replay_core_req; ++ /* true if force_replay_limit should be randomized. The random ++ * value will be in the range of 1 - KBASEP_FORCE_REPLAY_RANDOM_LIMIT. ++ */ ++ bool force_replay_random; ++#endif ++ ++ /* Total number of created contexts */ ++ atomic_t ctx_num; ++ ++#ifdef CONFIG_DEBUG_FS ++ /* Holds the most recent register accesses */ ++ struct kbase_io_history io_history; ++#endif /* CONFIG_DEBUG_FS */ ++ ++ struct kbase_hwaccess_data hwaccess; ++ ++ /* Count of page/bus faults waiting for workqueues to process */ ++ atomic_t faults_pending; ++ ++ /* true if GPU is powered off or power off operation is in progress */ ++ bool poweroff_pending; ++ ++ ++ /* defaults for new context created for this device */ ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0)) ++ bool infinite_cache_active_default; ++#else ++ u32 infinite_cache_active_default; ++#endif ++ size_t mem_pool_max_size_default; ++ ++ /* current gpu coherency mode */ ++ u32 current_gpu_coherency_mode; ++ /* system coherency mode */ ++ u32 system_coherency; ++ /* Flag to track when cci snoops have been enabled on the interface */ ++ bool cci_snoop_enabled; ++ ++ /* SMC function IDs to call into Trusted firmware to enable/disable ++ * cache snooping. Value of 0 indicates that they are not used ++ */ ++ u32 snoop_enable_smc; ++ u32 snoop_disable_smc; ++ ++ /* Protected mode operations */ ++ struct protected_mode_ops *protected_ops; ++ ++ /* Protected device attached to this kbase device */ ++ struct protected_mode_device *protected_dev; ++ ++ /* ++ * true when GPU is put into protected mode ++ */ ++ bool protected_mode; ++ ++ /* ++ * true when GPU is transitioning into or out of protected mode ++ */ ++ bool protected_mode_transition; ++ ++ /* ++ * true if protected mode is supported ++ */ ++ bool protected_mode_support; ++ ++ ++#ifdef CONFIG_MALI_DEBUG ++ wait_queue_head_t driver_inactive_wait; ++ bool driver_inactive; ++#endif /* CONFIG_MALI_DEBUG */ ++ ++#ifdef CONFIG_MALI_FPGA_BUS_LOGGER ++ /* ++ * Bus logger integration. ++ */ ++ struct bus_logger_client *buslogger; ++#endif ++ /* Boolean indicating if an IRQ flush during reset is in progress. */ ++ bool irq_reset_flush; ++ ++ /* list of inited sub systems. Used during terminate/error recovery */ ++ u32 inited_subsys; ++ ++ spinlock_t hwaccess_lock; ++ ++ /* Protects access to MMU operations */ ++ struct mutex mmu_hw_mutex; ++ ++ /* Current serialization mode. See KBASE_SERIALIZE_* for details */ ++ u8 serialize_jobs; ++}; ++ ++/** ++ * struct jsctx_queue - JS context atom queue ++ * @runnable_tree: Root of RB-tree containing currently runnable atoms on this ++ * job slot. ++ * @x_dep_head: Head item of the linked list of atoms blocked on cross-slot ++ * dependencies. Atoms on this list will be moved to the ++ * runnable_tree when the blocking atom completes. ++ * ++ * hwaccess_lock must be held when accessing this structure. ++ */ ++struct jsctx_queue { ++ struct rb_root runnable_tree; ++ struct list_head x_dep_head; ++}; ++ ++ ++#define KBASE_API_VERSION(major, minor) ((((major) & 0xFFF) << 20) | \ ++ (((minor) & 0xFFF) << 8) | \ ++ ((0 & 0xFF) << 0)) ++ ++/** ++ * enum kbase_context_flags - Flags for kbase contexts ++ * ++ * @KCTX_COMPAT: Set when the context process is a compat process, 32-bit ++ * process on a 64-bit kernel. ++ * ++ * @KCTX_RUNNABLE_REF: Set when context is counted in ++ * kbdev->js_data.nr_contexts_runnable. Must hold queue_mutex when accessing. ++ * ++ * @KCTX_ACTIVE: Set when the context is active. ++ * ++ * @KCTX_PULLED: Set when last kick() caused atoms to be pulled from this ++ * context. ++ * ++ * @KCTX_MEM_PROFILE_INITIALIZED: Set when the context's memory profile has been ++ * initialized. ++ * ++ * @KCTX_INFINITE_CACHE: Set when infinite cache is to be enabled for new ++ * allocations. Existing allocations will not change. ++ * ++ * @KCTX_SUBMIT_DISABLED: Set to prevent context from submitting any jobs. ++ * ++ * @KCTX_PRIVILEGED:Set if the context uses an address space and should be kept ++ * scheduled in. ++ * ++ * @KCTX_SCHEDULED: Set when the context is scheduled on the Run Pool. ++ * This is only ever updated whilst the jsctx_mutex is held. ++ * ++ * @KCTX_DYING: Set when the context process is in the process of being evicted. ++ * ++ * @KCTX_NO_IMPLICIT_SYNC: Set when explicit Android fences are in use on this ++ * context, to disable use of implicit dma-buf fences. This is used to avoid ++ * potential synchronization deadlocks. ++ * ++ * All members need to be separate bits. This enum is intended for use in a ++ * bitmask where multiple values get OR-ed together. ++ */ ++enum kbase_context_flags { ++ KCTX_COMPAT = 1U << 0, ++ KCTX_RUNNABLE_REF = 1U << 1, ++ KCTX_ACTIVE = 1U << 2, ++ KCTX_PULLED = 1U << 3, ++ KCTX_MEM_PROFILE_INITIALIZED = 1U << 4, ++ KCTX_INFINITE_CACHE = 1U << 5, ++ KCTX_SUBMIT_DISABLED = 1U << 6, ++ KCTX_PRIVILEGED = 1U << 7, ++ KCTX_SCHEDULED = 1U << 8, ++ KCTX_DYING = 1U << 9, ++ KCTX_NO_IMPLICIT_SYNC = 1U << 10, ++}; ++ ++struct kbase_context { ++ struct file *filp; ++ struct kbase_device *kbdev; ++ int id; /* System wide unique id */ ++ unsigned long api_version; ++ phys_addr_t pgd; ++ struct list_head event_list; ++ struct list_head event_coalesce_list; ++ struct mutex event_mutex; ++ atomic_t event_closed; ++ struct workqueue_struct *event_workq; ++ atomic_t event_count; ++ int event_coalesce_count; ++ ++ atomic_t flags; ++ ++ atomic_t setup_complete; ++ atomic_t setup_in_progress; ++ ++ u64 *mmu_teardown_pages; ++ ++ struct page *aliasing_sink_page; ++ ++ struct mutex mmu_lock; ++ struct mutex reg_lock; /* To be converted to a rwlock? */ ++ struct rb_root reg_rbtree_same; /* RB tree of GPU (live) regions, ++ * SAME_VA zone */ ++ struct rb_root reg_rbtree_exec; /* RB tree of GPU (live) regions, ++ * EXEC zone */ ++ struct rb_root reg_rbtree_custom; /* RB tree of GPU (live) regions, ++ * CUSTOM_VA zone */ ++ ++ unsigned long cookies; ++ struct kbase_va_region *pending_regions[BITS_PER_LONG]; ++ ++ wait_queue_head_t event_queue; ++ pid_t tgid; ++ pid_t pid; ++ ++ struct kbase_jd_context jctx; ++ atomic_t used_pages; ++ atomic_t nonmapped_pages; ++ ++ struct kbase_mem_pool mem_pool; ++ ++ struct shrinker reclaim; ++ struct list_head evict_list; ++ ++ struct list_head waiting_soft_jobs; ++ spinlock_t waiting_soft_jobs_lock; ++#ifdef CONFIG_KDS ++ struct list_head waiting_kds_resource; ++#endif ++#ifdef CONFIG_MALI_DMA_FENCE ++ struct { ++ struct list_head waiting_resource; ++ struct workqueue_struct *wq; ++ } dma_fence; ++#endif /* CONFIG_MALI_DMA_FENCE */ ++ /** This is effectively part of the Run Pool, because it only has a valid ++ * setting (!=KBASEP_AS_NR_INVALID) whilst the context is scheduled in ++ * ++ * The hwaccess_lock must be held whilst accessing this. ++ * ++ * If the context relating to this as_nr is required, you must use ++ * kbasep_js_runpool_retain_ctx() to ensure that the context doesn't disappear ++ * whilst you're using it. Alternatively, just hold the hwaccess_lock ++ * to ensure the context doesn't disappear (but this has restrictions on what other locks ++ * you can take whilst doing this) */ ++ int as_nr; ++ ++ /* Keeps track of the number of users of this context. A user can be a ++ * job that is available for execution, instrumentation needing to 'pin' ++ * a context for counter collection, etc. If the refcount reaches 0 then ++ * this context is considered inactive and the previously programmed ++ * AS might be cleared at any point. ++ */ ++ atomic_t refcount; ++ ++ /* NOTE: ++ * ++ * Flags are in jctx.sched_info.ctx.flags ++ * Mutable flags *must* be accessed under jctx.sched_info.ctx.jsctx_mutex ++ * ++ * All other flags must be added there */ ++ spinlock_t mm_update_lock; ++ struct mm_struct *process_mm; ++ /* End of the SAME_VA zone */ ++ u64 same_va_end; ++ ++#ifdef CONFIG_MALI_TRACE_TIMELINE ++ struct kbase_trace_kctx_timeline timeline; ++#endif ++#ifdef CONFIG_DEBUG_FS ++ /* Content of mem_profile file */ ++ char *mem_profile_data; ++ /* Size of @c mem_profile_data */ ++ size_t mem_profile_size; ++ /* Mutex guarding memory profile state */ ++ struct mutex mem_profile_lock; ++ /* Memory profile directory under debugfs */ ++ struct dentry *kctx_dentry; ++ ++ /* for job fault debug */ ++ unsigned int *reg_dump; ++ atomic_t job_fault_count; ++ /* This list will keep the following atoms during the dump ++ * in the same context ++ */ ++ struct list_head job_fault_resume_event_list; ++ ++#endif /* CONFIG_DEBUG_FS */ ++ ++ struct jsctx_queue jsctx_queue ++ [KBASE_JS_ATOM_SCHED_PRIO_COUNT][BASE_JM_MAX_NR_SLOTS]; ++ ++ /* Number of atoms currently pulled from this context */ ++ atomic_t atoms_pulled; ++ /* Number of atoms currently pulled from this context, per slot */ ++ atomic_t atoms_pulled_slot[BASE_JM_MAX_NR_SLOTS]; ++ /* Number of atoms currently pulled from this context, per slot and ++ * priority. Hold hwaccess_lock when accessing */ ++ int atoms_pulled_slot_pri[BASE_JM_MAX_NR_SLOTS][ ++ KBASE_JS_ATOM_SCHED_PRIO_COUNT]; ++ ++ /* true if slot is blocked on the given priority. This will be set on a ++ * soft-stop */ ++ bool blocked_js[BASE_JM_MAX_NR_SLOTS][KBASE_JS_ATOM_SCHED_PRIO_COUNT]; ++ ++ /* Bitmask of slots that can be pulled from */ ++ u32 slots_pullable; ++ ++ /* Backend specific data */ ++ struct kbase_context_backend backend; ++ ++ /* Work structure used for deferred ASID assignment */ ++ struct work_struct work; ++ ++ /* Only one userspace vinstr client per kbase context */ ++ struct kbase_vinstr_client *vinstr_cli; ++ struct mutex vinstr_cli_lock; ++ ++ /* List of completed jobs waiting for events to be posted */ ++ struct list_head completed_jobs; ++ /* Number of work items currently pending on job_done_wq */ ++ atomic_t work_count; ++ ++ /* Waiting soft-jobs will fail when this timer expires */ ++ struct timer_list soft_job_timeout; ++ ++ /* JIT allocation management */ ++ struct kbase_va_region *jit_alloc[256]; ++ struct list_head jit_active_head; ++ struct list_head jit_pool_head; ++ struct list_head jit_destroy_head; ++ struct mutex jit_evict_lock; ++ struct work_struct jit_work; ++ ++ /* A list of the JIT soft-jobs in submission order ++ * (protected by kbase_jd_context.lock) ++ */ ++ struct list_head jit_atoms_head; ++ /* A list of pending JIT alloc soft-jobs (using the 'queue' list_head) ++ * (protected by kbase_jd_context.lock) ++ */ ++ struct list_head jit_pending_alloc; ++ ++ /* External sticky resource management */ ++ struct list_head ext_res_meta_head; ++ ++ /* Used to record that a drain was requested from atomic context */ ++ atomic_t drain_pending; ++ ++ /* Current age count, used to determine age for newly submitted atoms */ ++ u32 age_count; ++}; ++ ++/** ++ * struct kbase_ctx_ext_res_meta - Structure which binds an external resource ++ * to a @kbase_context. ++ * @ext_res_node: List head for adding the metadata to a ++ * @kbase_context. ++ * @alloc: The physical memory allocation structure ++ * which is mapped. ++ * @gpu_addr: The GPU virtual address the resource is ++ * mapped to. ++ * ++ * External resources can be mapped into multiple contexts as well as the same ++ * context multiple times. ++ * As kbase_va_region itself isn't refcounted we can't attach our extra ++ * information to it as it could be removed under our feet leaving external ++ * resources pinned. ++ * This metadata structure binds a single external resource to a single ++ * context, ensuring that per context mapping is tracked separately so it can ++ * be overridden when needed and abuses by the application (freeing the resource ++ * multiple times) don't effect the refcount of the physical allocation. ++ */ ++struct kbase_ctx_ext_res_meta { ++ struct list_head ext_res_node; ++ struct kbase_mem_phy_alloc *alloc; ++ u64 gpu_addr; ++}; ++ ++enum kbase_reg_access_type { ++ REG_READ, ++ REG_WRITE ++}; ++ ++enum kbase_share_attr_bits { ++ /* (1ULL << 8) bit is reserved */ ++ SHARE_BOTH_BITS = (2ULL << 8), /* inner and outer shareable coherency */ ++ SHARE_INNER_BITS = (3ULL << 8) /* inner shareable coherency */ ++}; ++ ++/** ++ * kbase_device_is_cpu_coherent - Returns if the device is CPU coherent. ++ * @kbdev: kbase device ++ * ++ * Return: true if the device access are coherent, false if not. ++ */ ++static inline bool kbase_device_is_cpu_coherent(struct kbase_device *kbdev) ++{ ++ if ((kbdev->system_coherency == COHERENCY_ACE_LITE) || ++ (kbdev->system_coherency == COHERENCY_ACE)) ++ return true; ++ ++ return false; ++} ++ ++/* Conversion helpers for setting up high resolution timers */ ++#define HR_TIMER_DELAY_MSEC(x) (ns_to_ktime(((u64)(x))*1000000U)) ++#define HR_TIMER_DELAY_NSEC(x) (ns_to_ktime(x)) ++ ++/* Maximum number of loops polling the GPU for a cache flush before we assume it must have completed */ ++#define KBASE_CLEAN_CACHE_MAX_LOOPS 100000 ++/* Maximum number of loops polling the GPU for an AS command to complete before we assume the GPU has hung */ ++#define KBASE_AS_INACTIVE_MAX_LOOPS 100000 ++ ++/* Maximum number of times a job can be replayed */ ++#define BASEP_JD_REPLAY_LIMIT 15 ++ ++/* JobDescriptorHeader - taken from the architecture specifications, the layout ++ * is currently identical for all GPU archs. */ ++struct job_descriptor_header { ++ u32 exception_status; ++ u32 first_incomplete_task; ++ u64 fault_pointer; ++ u8 job_descriptor_size : 1; ++ u8 job_type : 7; ++ u8 job_barrier : 1; ++ u8 _reserved_01 : 1; ++ u8 _reserved_1 : 1; ++ u8 _reserved_02 : 1; ++ u8 _reserved_03 : 1; ++ u8 _reserved_2 : 1; ++ u8 _reserved_04 : 1; ++ u8 _reserved_05 : 1; ++ u16 job_index; ++ u16 job_dependency_index_1; ++ u16 job_dependency_index_2; ++ union { ++ u64 _64; ++ u32 _32; ++ } next_job; ++}; ++ ++#endif /* _KBASE_DEFS_H_ */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_device.c b/drivers/gpu/arm/midgard/mali_kbase_device.c +new file mode 100755 +index 000000000000..b0eb67da8644 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_device.c +@@ -0,0 +1,674 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * Base kernel device APIs ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#include ++#include ++#include ++#include ++ ++#include ++ ++/* NOTE: Magic - 0x45435254 (TRCE in ASCII). ++ * Supports tracing feature provided in the base module. ++ * Please keep it in sync with the value of base module. ++ */ ++#define TRACE_BUFFER_HEADER_SPECIAL 0x45435254 ++ ++#if KBASE_TRACE_ENABLE ++static const char *kbasep_trace_code_string[] = { ++ /* IMPORTANT: USE OF SPECIAL #INCLUDE OF NON-STANDARD HEADER FILE ++ * THIS MUST BE USED AT THE START OF THE ARRAY */ ++#define KBASE_TRACE_CODE_MAKE_CODE(X) # X ++#include "mali_kbase_trace_defs.h" ++#undef KBASE_TRACE_CODE_MAKE_CODE ++}; ++#endif ++ ++#define DEBUG_MESSAGE_SIZE 256 ++ ++static int kbasep_trace_init(struct kbase_device *kbdev); ++static void kbasep_trace_term(struct kbase_device *kbdev); ++static void kbasep_trace_hook_wrapper(void *param); ++ ++struct kbase_device *kbase_device_alloc(void) ++{ ++ return kzalloc(sizeof(struct kbase_device), GFP_KERNEL); ++} ++ ++static int kbase_device_as_init(struct kbase_device *kbdev, int i) ++{ ++ const char format[] = "mali_mmu%d"; ++ char name[sizeof(format)]; ++ const char poke_format[] = "mali_mmu%d_poker"; ++ char poke_name[sizeof(poke_format)]; ++ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316)) ++ snprintf(poke_name, sizeof(poke_name), poke_format, i); ++ ++ snprintf(name, sizeof(name), format, i); ++ ++ kbdev->as[i].number = i; ++ kbdev->as[i].fault_addr = 0ULL; ++ ++ kbdev->as[i].pf_wq = alloc_workqueue(name, 0, 1); ++ if (!kbdev->as[i].pf_wq) ++ return -EINVAL; ++ ++ INIT_WORK(&kbdev->as[i].work_pagefault, page_fault_worker); ++ INIT_WORK(&kbdev->as[i].work_busfault, bus_fault_worker); ++ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316)) { ++ struct hrtimer *poke_timer = &kbdev->as[i].poke_timer; ++ struct work_struct *poke_work = &kbdev->as[i].poke_work; ++ ++ kbdev->as[i].poke_wq = alloc_workqueue(poke_name, 0, 1); ++ if (!kbdev->as[i].poke_wq) { ++ destroy_workqueue(kbdev->as[i].pf_wq); ++ return -EINVAL; ++ } ++ KBASE_DEBUG_ASSERT(!object_is_on_stack(poke_work)); ++ INIT_WORK(poke_work, kbasep_as_do_poke); ++ ++ hrtimer_init(poke_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); ++ ++ poke_timer->function = kbasep_as_poke_timer_callback; ++ ++ kbdev->as[i].poke_refcount = 0; ++ kbdev->as[i].poke_state = 0u; ++ } ++ ++ return 0; ++} ++ ++static void kbase_device_as_term(struct kbase_device *kbdev, int i) ++{ ++ destroy_workqueue(kbdev->as[i].pf_wq); ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316)) ++ destroy_workqueue(kbdev->as[i].poke_wq); ++} ++ ++static int kbase_device_all_as_init(struct kbase_device *kbdev) ++{ ++ int i, err; ++ ++ for (i = 0; i < kbdev->nr_hw_address_spaces; i++) { ++ err = kbase_device_as_init(kbdev, i); ++ if (err) ++ goto free_workqs; ++ } ++ ++ return 0; ++ ++free_workqs: ++ for (; i > 0; i--) ++ kbase_device_as_term(kbdev, i); ++ ++ return err; ++} ++ ++static void kbase_device_all_as_term(struct kbase_device *kbdev) ++{ ++ int i; ++ ++ for (i = 0; i < kbdev->nr_hw_address_spaces; i++) ++ kbase_device_as_term(kbdev, i); ++} ++ ++int kbase_device_init(struct kbase_device * const kbdev) ++{ ++ int i, err; ++#ifdef CONFIG_ARM64 ++ struct device_node *np = NULL; ++#endif /* CONFIG_ARM64 */ ++ ++ spin_lock_init(&kbdev->mmu_mask_change); ++ mutex_init(&kbdev->mmu_hw_mutex); ++#ifdef CONFIG_ARM64 ++ kbdev->cci_snoop_enabled = false; ++ np = kbdev->dev->of_node; ++ if (np != NULL) { ++ if (of_property_read_u32(np, "snoop_enable_smc", ++ &kbdev->snoop_enable_smc)) ++ kbdev->snoop_enable_smc = 0; ++ if (of_property_read_u32(np, "snoop_disable_smc", ++ &kbdev->snoop_disable_smc)) ++ kbdev->snoop_disable_smc = 0; ++ /* Either both or none of the calls should be provided. */ ++ if (!((kbdev->snoop_disable_smc == 0 ++ && kbdev->snoop_enable_smc == 0) ++ || (kbdev->snoop_disable_smc != 0 ++ && kbdev->snoop_enable_smc != 0))) { ++ WARN_ON(1); ++ err = -EINVAL; ++ goto fail; ++ } ++ } ++#endif /* CONFIG_ARM64 */ ++ /* Get the list of workarounds for issues on the current HW ++ * (identified by the GPU_ID register) ++ */ ++ err = kbase_hw_set_issues_mask(kbdev); ++ if (err) ++ goto fail; ++ ++ /* Set the list of features available on the current HW ++ * (identified by the GPU_ID register) ++ */ ++ kbase_hw_set_features_mask(kbdev); ++ ++ kbase_gpuprops_set_features(kbdev); ++ ++ /* On Linux 4.0+, dma coherency is determined from device tree */ ++#if defined(CONFIG_ARM64) && LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0) ++ set_dma_ops(kbdev->dev, &noncoherent_swiotlb_dma_ops); ++#endif ++ ++ /* Workaround a pre-3.13 Linux issue, where dma_mask is NULL when our ++ * device structure was created by device-tree ++ */ ++ if (!kbdev->dev->dma_mask) ++ kbdev->dev->dma_mask = &kbdev->dev->coherent_dma_mask; ++ ++ err = dma_set_mask(kbdev->dev, ++ DMA_BIT_MASK(kbdev->gpu_props.mmu.pa_bits)); ++ if (err) ++ goto dma_set_mask_failed; ++ ++ err = dma_set_coherent_mask(kbdev->dev, ++ DMA_BIT_MASK(kbdev->gpu_props.mmu.pa_bits)); ++ if (err) ++ goto dma_set_mask_failed; ++ ++ kbdev->nr_hw_address_spaces = kbdev->gpu_props.num_address_spaces; ++ ++ err = kbase_device_all_as_init(kbdev); ++ if (err) ++ goto as_init_failed; ++ ++ spin_lock_init(&kbdev->hwcnt.lock); ++ ++ err = kbasep_trace_init(kbdev); ++ if (err) ++ goto term_as; ++ ++ mutex_init(&kbdev->cacheclean_lock); ++ ++#ifdef CONFIG_MALI_TRACE_TIMELINE ++ for (i = 0; i < BASE_JM_MAX_NR_SLOTS; ++i) ++ kbdev->timeline.slot_atoms_submitted[i] = 0; ++ ++ for (i = 0; i <= KBASEP_TIMELINE_PM_EVENT_LAST; ++i) ++ atomic_set(&kbdev->timeline.pm_event_uid[i], 0); ++#endif /* CONFIG_MALI_TRACE_TIMELINE */ ++ ++ /* fbdump profiling controls set to 0 - fbdump not enabled until changed by gator */ ++ for (i = 0; i < FBDUMP_CONTROL_MAX; i++) ++ kbdev->kbase_profiling_controls[i] = 0; ++ ++ kbase_debug_assert_register_hook(&kbasep_trace_hook_wrapper, kbdev); ++ ++ atomic_set(&kbdev->ctx_num, 0); ++ ++ err = kbase_instr_backend_init(kbdev); ++ if (err) ++ goto term_trace; ++ ++ kbdev->pm.dvfs_period = DEFAULT_PM_DVFS_PERIOD; ++ ++ kbdev->reset_timeout_ms = DEFAULT_RESET_TIMEOUT_MS; ++ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) ++ kbdev->mmu_mode = kbase_mmu_mode_get_aarch64(); ++ else ++ kbdev->mmu_mode = kbase_mmu_mode_get_lpae(); ++ ++#ifdef CONFIG_MALI_DEBUG ++ init_waitqueue_head(&kbdev->driver_inactive_wait); ++#endif /* CONFIG_MALI_DEBUG */ ++ ++ return 0; ++term_trace: ++ kbasep_trace_term(kbdev); ++term_as: ++ kbase_device_all_as_term(kbdev); ++as_init_failed: ++dma_set_mask_failed: ++fail: ++ return err; ++} ++ ++void kbase_device_term(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev); ++ ++#if KBASE_TRACE_ENABLE ++ kbase_debug_assert_register_hook(NULL, NULL); ++#endif ++ ++ kbase_instr_backend_term(kbdev); ++ ++ kbasep_trace_term(kbdev); ++ ++ kbase_device_all_as_term(kbdev); ++} ++ ++void kbase_device_free(struct kbase_device *kbdev) ++{ ++ kfree(kbdev); ++} ++ ++int kbase_device_trace_buffer_install( ++ struct kbase_context *kctx, u32 *tb, size_t size) ++{ ++ unsigned long flags; ++ ++ KBASE_DEBUG_ASSERT(kctx); ++ KBASE_DEBUG_ASSERT(tb); ++ ++ /* Interface uses 16-bit value to track last accessed entry. Each entry ++ * is composed of two 32-bit words. ++ * This limits the size that can be handled without an overflow. */ ++ if (0xFFFF * (2 * sizeof(u32)) < size) ++ return -EINVAL; ++ ++ /* set up the header */ ++ /* magic number in the first 4 bytes */ ++ tb[0] = TRACE_BUFFER_HEADER_SPECIAL; ++ /* Store (write offset = 0, wrap counter = 0, transaction active = no) ++ * write offset 0 means never written. ++ * Offsets 1 to (wrap_offset - 1) used to store values when trace started ++ */ ++ tb[1] = 0; ++ ++ /* install trace buffer */ ++ spin_lock_irqsave(&kctx->jctx.tb_lock, flags); ++ kctx->jctx.tb_wrap_offset = size / 8; ++ kctx->jctx.tb = tb; ++ spin_unlock_irqrestore(&kctx->jctx.tb_lock, flags); ++ ++ return 0; ++} ++ ++void kbase_device_trace_buffer_uninstall(struct kbase_context *kctx) ++{ ++ unsigned long flags; ++ ++ KBASE_DEBUG_ASSERT(kctx); ++ spin_lock_irqsave(&kctx->jctx.tb_lock, flags); ++ kctx->jctx.tb = NULL; ++ kctx->jctx.tb_wrap_offset = 0; ++ spin_unlock_irqrestore(&kctx->jctx.tb_lock, flags); ++} ++ ++void kbase_device_trace_register_access(struct kbase_context *kctx, enum kbase_reg_access_type type, u16 reg_offset, u32 reg_value) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kctx->jctx.tb_lock, flags); ++ if (kctx->jctx.tb) { ++ u16 wrap_count; ++ u16 write_offset; ++ u32 *tb = kctx->jctx.tb; ++ u32 header_word; ++ ++ header_word = tb[1]; ++ KBASE_DEBUG_ASSERT(0 == (header_word & 0x1)); ++ ++ wrap_count = (header_word >> 1) & 0x7FFF; ++ write_offset = (header_word >> 16) & 0xFFFF; ++ ++ /* mark as transaction in progress */ ++ tb[1] |= 0x1; ++ mb(); ++ ++ /* calculate new offset */ ++ write_offset++; ++ if (write_offset == kctx->jctx.tb_wrap_offset) { ++ /* wrap */ ++ write_offset = 1; ++ wrap_count++; ++ wrap_count &= 0x7FFF; /* 15bit wrap counter */ ++ } ++ ++ /* store the trace entry at the selected offset */ ++ tb[write_offset * 2 + 0] = (reg_offset & ~0x3) | ((type == REG_WRITE) ? 0x1 : 0x0); ++ tb[write_offset * 2 + 1] = reg_value; ++ mb(); ++ ++ /* new header word */ ++ header_word = (write_offset << 16) | (wrap_count << 1) | 0x0; /* transaction complete */ ++ tb[1] = header_word; ++ } ++ spin_unlock_irqrestore(&kctx->jctx.tb_lock, flags); ++} ++ ++/* ++ * Device trace functions ++ */ ++#if KBASE_TRACE_ENABLE ++ ++static int kbasep_trace_init(struct kbase_device *kbdev) ++{ ++ struct kbase_trace *rbuf; ++ ++ rbuf = kmalloc_array(KBASE_TRACE_SIZE, sizeof(*rbuf), GFP_KERNEL); ++ ++ if (!rbuf) ++ return -EINVAL; ++ ++ kbdev->trace_rbuf = rbuf; ++ spin_lock_init(&kbdev->trace_lock); ++ return 0; ++} ++ ++static void kbasep_trace_term(struct kbase_device *kbdev) ++{ ++ kfree(kbdev->trace_rbuf); ++} ++ ++static void kbasep_trace_format_msg(struct kbase_trace *trace_msg, char *buffer, int len) ++{ ++ s32 written = 0; ++ ++ /* Initial part of message */ ++ written += MAX(snprintf(buffer + written, MAX(len - written, 0), "%d.%.6d,%d,%d,%s,%p,", (int)trace_msg->timestamp.tv_sec, (int)(trace_msg->timestamp.tv_nsec / 1000), trace_msg->thread_id, trace_msg->cpu, kbasep_trace_code_string[trace_msg->code], trace_msg->ctx), 0); ++ ++ if (trace_msg->katom) ++ written += MAX(snprintf(buffer + written, MAX(len - written, 0), "atom %d (ud: 0x%llx 0x%llx)", trace_msg->atom_number, trace_msg->atom_udata[0], trace_msg->atom_udata[1]), 0); ++ ++ written += MAX(snprintf(buffer + written, MAX(len - written, 0), ",%.8llx,", trace_msg->gpu_addr), 0); ++ ++ /* NOTE: Could add function callbacks to handle different message types */ ++ /* Jobslot present */ ++ if (trace_msg->flags & KBASE_TRACE_FLAG_JOBSLOT) ++ written += MAX(snprintf(buffer + written, MAX(len - written, 0), "%d", trace_msg->jobslot), 0); ++ ++ written += MAX(snprintf(buffer + written, MAX(len - written, 0), ","), 0); ++ ++ /* Refcount present */ ++ if (trace_msg->flags & KBASE_TRACE_FLAG_REFCOUNT) ++ written += MAX(snprintf(buffer + written, MAX(len - written, 0), "%d", trace_msg->refcount), 0); ++ ++ written += MAX(snprintf(buffer + written, MAX(len - written, 0), ","), 0); ++ ++ /* Rest of message */ ++ written += MAX(snprintf(buffer + written, MAX(len - written, 0), "0x%.8lx", trace_msg->info_val), 0); ++} ++ ++static void kbasep_trace_dump_msg(struct kbase_device *kbdev, struct kbase_trace *trace_msg) ++{ ++ char buffer[DEBUG_MESSAGE_SIZE]; ++ ++ kbasep_trace_format_msg(trace_msg, buffer, DEBUG_MESSAGE_SIZE); ++ dev_dbg(kbdev->dev, "%s", buffer); ++} ++ ++void kbasep_trace_add(struct kbase_device *kbdev, enum kbase_trace_code code, void *ctx, struct kbase_jd_atom *katom, u64 gpu_addr, u8 flags, int refcount, int jobslot, unsigned long info_val) ++{ ++ unsigned long irqflags; ++ struct kbase_trace *trace_msg; ++ ++ spin_lock_irqsave(&kbdev->trace_lock, irqflags); ++ ++ trace_msg = &kbdev->trace_rbuf[kbdev->trace_next_in]; ++ ++ /* Fill the message */ ++ trace_msg->thread_id = task_pid_nr(current); ++ trace_msg->cpu = task_cpu(current); ++ ++ ktime_get_real_ts64(&trace_msg->timestamp); ++ ++ trace_msg->code = code; ++ trace_msg->ctx = ctx; ++ ++ if (NULL == katom) { ++ trace_msg->katom = false; ++ } else { ++ trace_msg->katom = true; ++ trace_msg->atom_number = kbase_jd_atom_id(katom->kctx, katom); ++ trace_msg->atom_udata[0] = katom->udata.blob[0]; ++ trace_msg->atom_udata[1] = katom->udata.blob[1]; ++ } ++ ++ trace_msg->gpu_addr = gpu_addr; ++ trace_msg->jobslot = jobslot; ++ trace_msg->refcount = MIN((unsigned int)refcount, 0xFF); ++ trace_msg->info_val = info_val; ++ trace_msg->flags = flags; ++ ++ /* Update the ringbuffer indices */ ++ kbdev->trace_next_in = (kbdev->trace_next_in + 1) & KBASE_TRACE_MASK; ++ if (kbdev->trace_next_in == kbdev->trace_first_out) ++ kbdev->trace_first_out = (kbdev->trace_first_out + 1) & KBASE_TRACE_MASK; ++ ++ /* Done */ ++ ++ spin_unlock_irqrestore(&kbdev->trace_lock, irqflags); ++} ++ ++void kbasep_trace_clear(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->trace_lock, flags); ++ kbdev->trace_first_out = kbdev->trace_next_in; ++ spin_unlock_irqrestore(&kbdev->trace_lock, flags); ++} ++ ++void kbasep_trace_dump(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ u32 start; ++ u32 end; ++ ++ dev_dbg(kbdev->dev, "Dumping trace:\nsecs,nthread,cpu,code,ctx,katom,gpu_addr,jobslot,refcount,info_val"); ++ spin_lock_irqsave(&kbdev->trace_lock, flags); ++ start = kbdev->trace_first_out; ++ end = kbdev->trace_next_in; ++ ++ while (start != end) { ++ struct kbase_trace *trace_msg = &kbdev->trace_rbuf[start]; ++ ++ kbasep_trace_dump_msg(kbdev, trace_msg); ++ ++ start = (start + 1) & KBASE_TRACE_MASK; ++ } ++ dev_dbg(kbdev->dev, "TRACE_END"); ++ ++ spin_unlock_irqrestore(&kbdev->trace_lock, flags); ++ ++ KBASE_TRACE_CLEAR(kbdev); ++} ++ ++static void kbasep_trace_hook_wrapper(void *param) ++{ ++ struct kbase_device *kbdev = (struct kbase_device *)param; ++ ++ kbasep_trace_dump(kbdev); ++} ++ ++#ifdef CONFIG_DEBUG_FS ++struct trace_seq_state { ++ struct kbase_trace trace_buf[KBASE_TRACE_SIZE]; ++ u32 start; ++ u32 end; ++}; ++ ++static void *kbasep_trace_seq_start(struct seq_file *s, loff_t *pos) ++{ ++ struct trace_seq_state *state = s->private; ++ int i; ++ ++ if (*pos > KBASE_TRACE_SIZE) ++ return NULL; ++ i = state->start + *pos; ++ if ((state->end >= state->start && i >= state->end) || ++ i >= state->end + KBASE_TRACE_SIZE) ++ return NULL; ++ ++ i &= KBASE_TRACE_MASK; ++ ++ return &state->trace_buf[i]; ++} ++ ++static void kbasep_trace_seq_stop(struct seq_file *s, void *data) ++{ ++} ++ ++static void *kbasep_trace_seq_next(struct seq_file *s, void *data, loff_t *pos) ++{ ++ struct trace_seq_state *state = s->private; ++ int i; ++ ++ (*pos)++; ++ ++ i = (state->start + *pos) & KBASE_TRACE_MASK; ++ if (i == state->end) ++ return NULL; ++ ++ return &state->trace_buf[i]; ++} ++ ++static int kbasep_trace_seq_show(struct seq_file *s, void *data) ++{ ++ struct kbase_trace *trace_msg = data; ++ char buffer[DEBUG_MESSAGE_SIZE]; ++ ++ kbasep_trace_format_msg(trace_msg, buffer, DEBUG_MESSAGE_SIZE); ++ seq_printf(s, "%s\n", buffer); ++ return 0; ++} ++ ++static const struct seq_operations kbasep_trace_seq_ops = { ++ .start = kbasep_trace_seq_start, ++ .next = kbasep_trace_seq_next, ++ .stop = kbasep_trace_seq_stop, ++ .show = kbasep_trace_seq_show, ++}; ++ ++static int kbasep_trace_debugfs_open(struct inode *inode, struct file *file) ++{ ++ struct kbase_device *kbdev = inode->i_private; ++ unsigned long flags; ++ ++ struct trace_seq_state *state; ++ ++ state = __seq_open_private(file, &kbasep_trace_seq_ops, sizeof(*state)); ++ if (!state) ++ return -ENOMEM; ++ ++ spin_lock_irqsave(&kbdev->trace_lock, flags); ++ state->start = kbdev->trace_first_out; ++ state->end = kbdev->trace_next_in; ++ memcpy(state->trace_buf, kbdev->trace_rbuf, sizeof(state->trace_buf)); ++ spin_unlock_irqrestore(&kbdev->trace_lock, flags); ++ ++ return 0; ++} ++ ++static const struct file_operations kbasep_trace_debugfs_fops = { ++ .open = kbasep_trace_debugfs_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = seq_release_private, ++}; ++ ++void kbasep_trace_debugfs_init(struct kbase_device *kbdev) ++{ ++ debugfs_create_file("mali_trace", S_IRUGO, ++ kbdev->mali_debugfs_directory, kbdev, ++ &kbasep_trace_debugfs_fops); ++} ++ ++#else ++void kbasep_trace_debugfs_init(struct kbase_device *kbdev) ++{ ++} ++#endif /* CONFIG_DEBUG_FS */ ++ ++#else /* KBASE_TRACE_ENABLE */ ++static int kbasep_trace_init(struct kbase_device *kbdev) ++{ ++ CSTD_UNUSED(kbdev); ++ return 0; ++} ++ ++static void kbasep_trace_term(struct kbase_device *kbdev) ++{ ++ CSTD_UNUSED(kbdev); ++} ++ ++static void kbasep_trace_hook_wrapper(void *param) ++{ ++ CSTD_UNUSED(param); ++} ++ ++void kbasep_trace_dump(struct kbase_device *kbdev) ++{ ++ CSTD_UNUSED(kbdev); ++} ++#endif /* KBASE_TRACE_ENABLE */ ++ ++void kbase_set_profiling_control(struct kbase_device *kbdev, u32 control, u32 value) ++{ ++ switch (control) { ++ case FBDUMP_CONTROL_ENABLE: ++ /* fall through */ ++ case FBDUMP_CONTROL_RATE: ++ /* fall through */ ++ case SW_COUNTER_ENABLE: ++ /* fall through */ ++ case FBDUMP_CONTROL_RESIZE_FACTOR: ++ kbdev->kbase_profiling_controls[control] = value; ++ break; ++ default: ++ dev_err(kbdev->dev, "Profiling control %d not found\n", control); ++ break; ++ } ++} ++ ++/* ++ * Called by gator to control the production of ++ * profiling information at runtime ++ * */ ++ ++void _mali_profiling_control(u32 action, u32 value) ++{ ++ struct kbase_device *kbdev = NULL; ++ ++ /* find the first i.e. call with -1 */ ++ kbdev = kbase_find_device(-1); ++ ++ if (NULL != kbdev) ++ kbase_set_profiling_control(kbdev, action, value); ++} ++KBASE_EXPORT_SYMBOL(_mali_profiling_control); ++ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_disjoint_events.c b/drivers/gpu/arm/midgard/mali_kbase_disjoint_events.c +new file mode 100755 +index 000000000000..f70bcccf4050 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_disjoint_events.c +@@ -0,0 +1,76 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* ++ * Base kernel disjoint events helper functions ++ */ ++ ++#include ++ ++void kbase_disjoint_init(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ atomic_set(&kbdev->disjoint_event.count, 0); ++ atomic_set(&kbdev->disjoint_event.state, 0); ++} ++ ++/* increment the disjoint event count */ ++void kbase_disjoint_event(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ atomic_inc(&kbdev->disjoint_event.count); ++} ++ ++/* increment the state and the event counter */ ++void kbase_disjoint_state_up(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ atomic_inc(&kbdev->disjoint_event.state); ++ ++ kbase_disjoint_event(kbdev); ++} ++ ++/* decrement the state */ ++void kbase_disjoint_state_down(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(atomic_read(&kbdev->disjoint_event.state) > 0); ++ ++ kbase_disjoint_event(kbdev); ++ ++ atomic_dec(&kbdev->disjoint_event.state); ++} ++ ++/* increments the count only if the state is > 0 */ ++void kbase_disjoint_event_potential(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ if (atomic_read(&kbdev->disjoint_event.state)) ++ kbase_disjoint_event(kbdev); ++} ++ ++u32 kbase_disjoint_event_get(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ return atomic_read(&kbdev->disjoint_event.count); ++} ++KBASE_EXPORT_TEST_API(kbase_disjoint_event_get); +diff --git a/drivers/gpu/arm/midgard/mali_kbase_dma_fence.c b/drivers/gpu/arm/midgard/mali_kbase_dma_fence.c +new file mode 100755 +index 000000000000..9197743c81d4 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_dma_fence.c +@@ -0,0 +1,449 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++/* Include mali_kbase_dma_fence.h before checking for CONFIG_MALI_DMA_FENCE as ++ * it will be set there. ++ */ ++#include "mali_kbase_dma_fence.h" ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++ ++static void ++kbase_dma_fence_work(struct work_struct *pwork); ++ ++static void ++kbase_dma_fence_waiters_add(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ ++ list_add_tail(&katom->queue, &kctx->dma_fence.waiting_resource); ++} ++ ++static void ++kbase_dma_fence_waiters_remove(struct kbase_jd_atom *katom) ++{ ++ list_del(&katom->queue); ++} ++ ++static int ++kbase_dma_fence_lock_reservations(struct kbase_dma_fence_resv_info *info, ++ struct ww_acquire_ctx *ctx) ++{ ++ struct reservation_object *content_res = NULL; ++ unsigned int content_res_idx = 0; ++ unsigned int r; ++ int err = 0; ++ ++ ww_acquire_init(ctx, &reservation_ww_class); ++ ++retry: ++ for (r = 0; r < info->dma_fence_resv_count; r++) { ++ if (info->resv_objs[r] == content_res) { ++ content_res = NULL; ++ continue; ++ } ++ ++ err = ww_mutex_lock(&info->resv_objs[r]->lock, ctx); ++ if (err) ++ goto error; ++ } ++ ++ ww_acquire_done(ctx); ++ return err; ++ ++error: ++ content_res_idx = r; ++ ++ /* Unlock the locked one ones */ ++ while (r--) ++ ww_mutex_unlock(&info->resv_objs[r]->lock); ++ ++ if (content_res) ++ ww_mutex_unlock(&content_res->lock); ++ ++ /* If we deadlock try with lock_slow and retry */ ++ if (err == -EDEADLK) { ++ content_res = info->resv_objs[content_res_idx]; ++ ww_mutex_lock_slow(&content_res->lock, ctx); ++ goto retry; ++ } ++ ++ /* If we are here the function failed */ ++ ww_acquire_fini(ctx); ++ return err; ++} ++ ++static void ++kbase_dma_fence_unlock_reservations(struct kbase_dma_fence_resv_info *info, ++ struct ww_acquire_ctx *ctx) ++{ ++ unsigned int r; ++ ++ for (r = 0; r < info->dma_fence_resv_count; r++) ++ ww_mutex_unlock(&info->resv_objs[r]->lock); ++ ww_acquire_fini(ctx); ++} ++ ++/** ++ * kbase_dma_fence_queue_work() - Queue work to handle @katom ++ * @katom: Pointer to atom for which to queue work ++ * ++ * Queue kbase_dma_fence_work() for @katom to clean up the fence callbacks and ++ * submit the atom. ++ */ ++static void ++kbase_dma_fence_queue_work(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ bool ret; ++ ++ INIT_WORK(&katom->work, kbase_dma_fence_work); ++ ret = queue_work(kctx->dma_fence.wq, &katom->work); ++ /* Warn if work was already queued, that should not happen. */ ++ WARN_ON(!ret); ++} ++ ++/** ++ * kbase_dma_fence_cancel_atom() - Cancels waiting on an atom ++ * @katom: Katom to cancel ++ * ++ * Locking: katom->dma_fence.callbacks list assumes jctx.lock is held. ++ */ ++static void ++kbase_dma_fence_cancel_atom(struct kbase_jd_atom *katom) ++{ ++ lockdep_assert_held(&katom->kctx->jctx.lock); ++ ++ /* Cancel callbacks and clean up. */ ++ kbase_fence_free_callbacks(katom); ++ ++ /* Mark the atom as handled in case all fences signaled just before ++ * canceling the callbacks and the worker was queued. ++ */ ++ kbase_fence_dep_count_set(katom, -1); ++ ++ /* Prevent job_done_nolock from being called twice on an atom when ++ * there is a race between job completion and cancellation. ++ */ ++ ++ if (katom->status == KBASE_JD_ATOM_STATE_QUEUED) { ++ /* Wait was cancelled - zap the atom */ ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ if (jd_done_nolock(katom, NULL)) ++ kbase_js_sched_all(katom->kctx->kbdev); ++ } ++} ++ ++/** ++ * kbase_dma_fence_work() - Worker thread called when a fence is signaled ++ * @pwork: work_struct containing a pointer to a katom ++ * ++ * This function will clean and mark all dependencies as satisfied ++ */ ++static void ++kbase_dma_fence_work(struct work_struct *pwork) ++{ ++ struct kbase_jd_atom *katom; ++ struct kbase_jd_context *ctx; ++ ++ katom = container_of(pwork, struct kbase_jd_atom, work); ++ ctx = &katom->kctx->jctx; ++ ++ mutex_lock(&ctx->lock); ++ if (kbase_fence_dep_count_read(katom) != 0) ++ goto out; ++ ++ kbase_fence_dep_count_set(katom, -1); ++ ++ /* Remove atom from list of dma-fence waiting atoms. */ ++ kbase_dma_fence_waiters_remove(katom); ++ /* Cleanup callbacks. */ ++ kbase_fence_free_callbacks(katom); ++ /* ++ * Queue atom on GPU, unless it has already completed due to a failing ++ * dependency. Run jd_done_nolock() on the katom if it is completed. ++ */ ++ if (unlikely(katom->status == KBASE_JD_ATOM_STATE_COMPLETED)) ++ jd_done_nolock(katom, NULL); ++ else ++ kbase_jd_dep_clear_locked(katom); ++ ++out: ++ mutex_unlock(&ctx->lock); ++} ++ ++static void ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++kbase_dma_fence_cb(struct fence *fence, struct fence_cb *cb) ++#else ++kbase_dma_fence_cb(struct dma_fence *fence, struct dma_fence_cb *cb) ++#endif ++{ ++ struct kbase_fence_cb *kcb = container_of(cb, ++ struct kbase_fence_cb, ++ fence_cb); ++ struct kbase_jd_atom *katom = kcb->katom; ++ ++ /* If the atom is zapped dep_count will be forced to a negative number ++ * preventing this callback from ever scheduling work. Which in turn ++ * would reschedule the atom. ++ */ ++ ++ if (kbase_fence_dep_count_dec_and_test(katom)) ++ kbase_dma_fence_queue_work(katom); ++} ++ ++static int ++kbase_dma_fence_add_reservation_callback(struct kbase_jd_atom *katom, ++ struct reservation_object *resv, ++ bool exclusive) ++{ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ struct fence *excl_fence = NULL; ++ struct fence **shared_fences = NULL; ++#else ++ struct dma_fence *excl_fence = NULL; ++ struct dma_fence **shared_fences = NULL; ++#endif ++ unsigned int shared_count = 0; ++ int err, i; ++ ++ err = reservation_object_get_fences_rcu(resv, ++ &excl_fence, ++ &shared_count, ++ &shared_fences); ++ if (err) ++ return err; ++ ++ if (excl_fence) { ++ err = kbase_fence_add_callback(katom, ++ excl_fence, ++ kbase_dma_fence_cb); ++ ++ /* Release our reference, taken by reservation_object_get_fences_rcu(), ++ * to the fence. We have set up our callback (if that was possible), ++ * and it's the fence's owner is responsible for singling the fence ++ * before allowing it to disappear. ++ */ ++ dma_fence_put(excl_fence); ++ ++ if (err) ++ goto out; ++ } ++ ++ if (exclusive) { ++ for (i = 0; i < shared_count; i++) { ++ err = kbase_fence_add_callback(katom, ++ shared_fences[i], ++ kbase_dma_fence_cb); ++ if (err) ++ goto out; ++ } ++ } ++ ++ /* Release all our references to the shared fences, taken by ++ * reservation_object_get_fences_rcu(). We have set up our callback (if ++ * that was possible), and it's the fence's owner is responsible for ++ * signaling the fence before allowing it to disappear. ++ */ ++out: ++ for (i = 0; i < shared_count; i++) ++ dma_fence_put(shared_fences[i]); ++ kfree(shared_fences); ++ ++ if (err) { ++ /* ++ * On error, cancel and clean up all callbacks that was set up ++ * before the error. ++ */ ++ kbase_fence_free_callbacks(katom); ++ } ++ ++ return err; ++} ++ ++void kbase_dma_fence_add_reservation(struct reservation_object *resv, ++ struct kbase_dma_fence_resv_info *info, ++ bool exclusive) ++{ ++ unsigned int i; ++ ++ for (i = 0; i < info->dma_fence_resv_count; i++) { ++ /* Duplicate resource, ignore */ ++ if (info->resv_objs[i] == resv) ++ return; ++ } ++ ++ info->resv_objs[info->dma_fence_resv_count] = resv; ++ if (exclusive) ++ set_bit(info->dma_fence_resv_count, ++ info->dma_fence_excl_bitmap); ++ (info->dma_fence_resv_count)++; ++} ++ ++int kbase_dma_fence_wait(struct kbase_jd_atom *katom, ++ struct kbase_dma_fence_resv_info *info) ++{ ++ int err, i; ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ struct fence *fence; ++#else ++ struct dma_fence *fence; ++#endif ++ struct ww_acquire_ctx ww_ctx; ++ ++ lockdep_assert_held(&katom->kctx->jctx.lock); ++ ++ fence = kbase_fence_out_new(katom); ++ if (!fence) { ++ err = -ENOMEM; ++ dev_err(katom->kctx->kbdev->dev, ++ "Error %d creating fence.\n", err); ++ return err; ++ } ++ ++ kbase_fence_dep_count_set(katom, 1); ++ ++ err = kbase_dma_fence_lock_reservations(info, &ww_ctx); ++ if (err) { ++ dev_err(katom->kctx->kbdev->dev, ++ "Error %d locking reservations.\n", err); ++ kbase_fence_dep_count_set(katom, -1); ++ kbase_fence_out_remove(katom); ++ return err; ++ } ++ ++ for (i = 0; i < info->dma_fence_resv_count; i++) { ++ struct reservation_object *obj = info->resv_objs[i]; ++ ++ if (!test_bit(i, info->dma_fence_excl_bitmap)) { ++ err = reservation_object_reserve_shared(obj); ++ if (err) { ++ dev_err(katom->kctx->kbdev->dev, ++ "Error %d reserving space for shared fence.\n", err); ++ goto end; ++ } ++ ++ err = kbase_dma_fence_add_reservation_callback(katom, obj, false); ++ if (err) { ++ dev_err(katom->kctx->kbdev->dev, ++ "Error %d adding reservation to callback.\n", err); ++ goto end; ++ } ++ ++ reservation_object_add_shared_fence(obj, fence); ++ } else { ++ err = kbase_dma_fence_add_reservation_callback(katom, obj, true); ++ if (err) { ++ dev_err(katom->kctx->kbdev->dev, ++ "Error %d adding reservation to callback.\n", err); ++ goto end; ++ } ++ ++ reservation_object_add_excl_fence(obj, fence); ++ } ++ } ++ ++end: ++ kbase_dma_fence_unlock_reservations(info, &ww_ctx); ++ ++ if (likely(!err)) { ++ /* Test if the callbacks are already triggered */ ++ if (kbase_fence_dep_count_dec_and_test(katom)) { ++ kbase_fence_dep_count_set(katom, -1); ++ kbase_fence_free_callbacks(katom); ++ } else { ++ /* Add katom to the list of dma-buf fence waiting atoms ++ * only if it is still waiting. ++ */ ++ kbase_dma_fence_waiters_add(katom); ++ } ++ } else { ++ /* There was an error, cancel callbacks, set dep_count to -1 to ++ * indicate that the atom has been handled (the caller will ++ * kill it for us), signal the fence, free callbacks and the ++ * fence. ++ */ ++ kbase_fence_free_callbacks(katom); ++ kbase_fence_dep_count_set(katom, -1); ++ kbase_dma_fence_signal(katom); ++ } ++ ++ return err; ++} ++ ++void kbase_dma_fence_cancel_all_atoms(struct kbase_context *kctx) ++{ ++ struct list_head *list = &kctx->dma_fence.waiting_resource; ++ ++ while (!list_empty(list)) { ++ struct kbase_jd_atom *katom; ++ ++ katom = list_first_entry(list, struct kbase_jd_atom, queue); ++ kbase_dma_fence_waiters_remove(katom); ++ kbase_dma_fence_cancel_atom(katom); ++ } ++} ++ ++void kbase_dma_fence_cancel_callbacks(struct kbase_jd_atom *katom) ++{ ++ /* Cancel callbacks and clean up. */ ++ if (kbase_fence_free_callbacks(katom)) ++ kbase_dma_fence_queue_work(katom); ++} ++ ++void kbase_dma_fence_signal(struct kbase_jd_atom *katom) ++{ ++ if (!katom->dma_fence.fence) ++ return; ++ ++ /* Signal the atom's fence. */ ++ dma_fence_signal(katom->dma_fence.fence); ++ ++ kbase_fence_out_remove(katom); ++ ++ kbase_fence_free_callbacks(katom); ++} ++ ++void kbase_dma_fence_term(struct kbase_context *kctx) ++{ ++ destroy_workqueue(kctx->dma_fence.wq); ++ kctx->dma_fence.wq = NULL; ++} ++ ++int kbase_dma_fence_init(struct kbase_context *kctx) ++{ ++ INIT_LIST_HEAD(&kctx->dma_fence.waiting_resource); ++ ++ kctx->dma_fence.wq = alloc_workqueue("mali-fence-%d", ++ WQ_UNBOUND, 1, kctx->pid); ++ if (!kctx->dma_fence.wq) ++ return -ENOMEM; ++ ++ return 0; ++} +diff --git a/drivers/gpu/arm/midgard/mali_kbase_dma_fence.h b/drivers/gpu/arm/midgard/mali_kbase_dma_fence.h +new file mode 100755 +index 000000000000..c9ab40350422 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_dma_fence.h +@@ -0,0 +1,131 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KBASE_DMA_FENCE_H_ ++#define _KBASE_DMA_FENCE_H_ ++ ++#ifdef CONFIG_MALI_DMA_FENCE ++ ++#include ++#include ++#include ++ ++ ++/* Forward declaration from mali_kbase_defs.h */ ++struct kbase_jd_atom; ++struct kbase_context; ++ ++/** ++ * struct kbase_dma_fence_resv_info - Structure with list of reservation objects ++ * @resv_objs: Array of reservation objects to attach the ++ * new fence to. ++ * @dma_fence_resv_count: Number of reservation objects in the array. ++ * @dma_fence_excl_bitmap: Specifies which resv_obj are exclusive. ++ * ++ * This is used by some functions to pass around a collection of data about ++ * reservation objects. ++ */ ++struct kbase_dma_fence_resv_info { ++ struct reservation_object **resv_objs; ++ unsigned int dma_fence_resv_count; ++ unsigned long *dma_fence_excl_bitmap; ++}; ++ ++/** ++ * kbase_dma_fence_add_reservation() - Adds a resv to the array of resv_objs ++ * @resv: Reservation object to add to the array. ++ * @info: Pointer to struct with current reservation info ++ * @exclusive: Boolean indicating if exclusive access is needed ++ * ++ * The function adds a new reservation_object to an existing array of ++ * reservation_objects. At the same time keeps track of which objects require ++ * exclusive access in dma_fence_excl_bitmap. ++ */ ++void kbase_dma_fence_add_reservation(struct reservation_object *resv, ++ struct kbase_dma_fence_resv_info *info, ++ bool exclusive); ++ ++/** ++ * kbase_dma_fence_wait() - Creates a new fence and attaches it to the resv_objs ++ * @katom: Katom with the external dependency. ++ * @info: Pointer to struct with current reservation info ++ * ++ * Return: An error code or 0 if succeeds ++ */ ++int kbase_dma_fence_wait(struct kbase_jd_atom *katom, ++ struct kbase_dma_fence_resv_info *info); ++ ++/** ++ * kbase_dma_fence_cancel_ctx() - Cancel all dma-fences blocked atoms on kctx ++ * @kctx: Pointer to kbase context ++ * ++ * This function will cancel and clean up all katoms on @kctx that is waiting ++ * on dma-buf fences. ++ * ++ * Locking: jctx.lock needs to be held when calling this function. ++ */ ++void kbase_dma_fence_cancel_all_atoms(struct kbase_context *kctx); ++ ++/** ++ * kbase_dma_fence_cancel_callbacks() - Cancel only callbacks on katom ++ * @katom: Pointer to katom whose callbacks are to be canceled ++ * ++ * This function cancels all dma-buf fence callbacks on @katom, but does not ++ * cancel the katom itself. ++ * ++ * The caller is responsible for ensuring that jd_done_nolock is called on ++ * @katom. ++ * ++ * Locking: jctx.lock must be held when calling this function. ++ */ ++void kbase_dma_fence_cancel_callbacks(struct kbase_jd_atom *katom); ++ ++/** ++ * kbase_dma_fence_signal() - Signal katom's fence and clean up after wait ++ * @katom: Pointer to katom to signal and clean up ++ * ++ * This function will signal the @katom's fence, if it has one, and clean up ++ * the callback data from the katom's wait on earlier fences. ++ * ++ * Locking: jctx.lock must be held while calling this function. ++ */ ++void kbase_dma_fence_signal(struct kbase_jd_atom *katom); ++ ++/** ++ * kbase_dma_fence_term() - Terminate Mali dma-fence context ++ * @kctx: kbase context to terminate ++ */ ++void kbase_dma_fence_term(struct kbase_context *kctx); ++ ++/** ++ * kbase_dma_fence_init() - Initialize Mali dma-fence context ++ * @kctx: kbase context to initialize ++ */ ++int kbase_dma_fence_init(struct kbase_context *kctx); ++ ++ ++#else /* CONFIG_MALI_DMA_FENCE */ ++/* Dummy functions for when dma-buf fence isn't enabled. */ ++ ++static inline int kbase_dma_fence_init(struct kbase_context *kctx) ++{ ++ return 0; ++} ++ ++static inline void kbase_dma_fence_term(struct kbase_context *kctx) {} ++#endif /* CONFIG_MALI_DMA_FENCE */ ++#endif +diff --git a/drivers/gpu/arm/midgard/mali_kbase_event.c b/drivers/gpu/arm/midgard/mali_kbase_event.c +new file mode 100755 +index 000000000000..188148645f37 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_event.c +@@ -0,0 +1,259 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#include ++#include ++#include ++ ++static struct base_jd_udata kbase_event_process(struct kbase_context *kctx, struct kbase_jd_atom *katom) ++{ ++ struct base_jd_udata data; ++ ++ lockdep_assert_held(&kctx->jctx.lock); ++ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ KBASE_DEBUG_ASSERT(katom != NULL); ++ KBASE_DEBUG_ASSERT(katom->status == KBASE_JD_ATOM_STATE_COMPLETED); ++ ++ data = katom->udata; ++ ++ KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, atomic_sub_return(1, &kctx->timeline.jd_atoms_in_flight)); ++ ++ KBASE_TLSTREAM_TL_NRET_ATOM_CTX(katom, kctx); ++ KBASE_TLSTREAM_TL_DEL_ATOM(katom); ++ ++ katom->status = KBASE_JD_ATOM_STATE_UNUSED; ++ ++ wake_up(&katom->completed); ++ ++ return data; ++} ++ ++int kbase_event_pending(struct kbase_context *ctx) ++{ ++ KBASE_DEBUG_ASSERT(ctx); ++ ++ return (atomic_read(&ctx->event_count) != 0) || ++ (atomic_read(&ctx->event_closed) != 0); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_event_pending); ++ ++int kbase_event_dequeue(struct kbase_context *ctx, struct base_jd_event_v2 *uevent) ++{ ++ struct kbase_jd_atom *atom; ++ ++ KBASE_DEBUG_ASSERT(ctx); ++ ++ mutex_lock(&ctx->event_mutex); ++ ++ if (list_empty(&ctx->event_list)) { ++ if (!atomic_read(&ctx->event_closed)) { ++ mutex_unlock(&ctx->event_mutex); ++ return -1; ++ } ++ ++ /* generate the BASE_JD_EVENT_DRV_TERMINATED message on the fly */ ++ mutex_unlock(&ctx->event_mutex); ++ uevent->event_code = BASE_JD_EVENT_DRV_TERMINATED; ++ memset(&uevent->udata, 0, sizeof(uevent->udata)); ++ dev_dbg(ctx->kbdev->dev, ++ "event system closed, returning BASE_JD_EVENT_DRV_TERMINATED(0x%X)\n", ++ BASE_JD_EVENT_DRV_TERMINATED); ++ return 0; ++ } ++ ++ /* normal event processing */ ++ atomic_dec(&ctx->event_count); ++ atom = list_entry(ctx->event_list.next, struct kbase_jd_atom, dep_item[0]); ++ list_del(ctx->event_list.next); ++ ++ mutex_unlock(&ctx->event_mutex); ++ ++ dev_dbg(ctx->kbdev->dev, "event dequeuing %p\n", (void *)atom); ++ uevent->event_code = atom->event_code; ++ uevent->atom_number = (atom - ctx->jctx.atoms); ++ ++ if (atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) ++ kbase_jd_free_external_resources(atom); ++ ++ mutex_lock(&ctx->jctx.lock); ++ uevent->udata = kbase_event_process(ctx, atom); ++ mutex_unlock(&ctx->jctx.lock); ++ ++ return 0; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_event_dequeue); ++ ++/** ++ * kbase_event_process_noreport_worker - Worker for processing atoms that do not ++ * return an event but do have external ++ * resources ++ * @data: Work structure ++ */ ++static void kbase_event_process_noreport_worker(struct work_struct *data) ++{ ++ struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, ++ work); ++ struct kbase_context *kctx = katom->kctx; ++ ++ if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) ++ kbase_jd_free_external_resources(katom); ++ ++ mutex_lock(&kctx->jctx.lock); ++ kbase_event_process(kctx, katom); ++ mutex_unlock(&kctx->jctx.lock); ++} ++ ++/** ++ * kbase_event_process_noreport - Process atoms that do not return an event ++ * @kctx: Context pointer ++ * @katom: Atom to be processed ++ * ++ * Atoms that do not have external resources will be processed immediately. ++ * Atoms that do have external resources will be processed on a workqueue, in ++ * order to avoid locking issues. ++ */ ++static void kbase_event_process_noreport(struct kbase_context *kctx, ++ struct kbase_jd_atom *katom) ++{ ++ if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) { ++ INIT_WORK(&katom->work, kbase_event_process_noreport_worker); ++ queue_work(kctx->event_workq, &katom->work); ++ } else { ++ kbase_event_process(kctx, katom); ++ } ++} ++ ++/** ++ * kbase_event_coalesce - Move pending events to the main event list ++ * @kctx: Context pointer ++ * ++ * kctx->event_list and kctx->event_coalesce_count must be protected ++ * by a lock unless this is the last thread using them ++ * (and we're about to terminate the lock). ++ * ++ * Return: The number of pending events moved to the main event list ++ */ ++static int kbase_event_coalesce(struct kbase_context *kctx) ++{ ++ const int event_count = kctx->event_coalesce_count; ++ ++ /* Join the list of pending events onto the tail of the main list ++ and reset it */ ++ list_splice_tail_init(&kctx->event_coalesce_list, &kctx->event_list); ++ kctx->event_coalesce_count = 0; ++ ++ /* Return the number of events moved */ ++ return event_count; ++} ++ ++void kbase_event_post(struct kbase_context *ctx, struct kbase_jd_atom *atom) ++{ ++ if (atom->core_req & BASE_JD_REQ_EVENT_ONLY_ON_FAILURE) { ++ if (atom->event_code == BASE_JD_EVENT_DONE) { ++ /* Don't report the event */ ++ kbase_event_process_noreport(ctx, atom); ++ return; ++ } ++ } ++ ++ if (atom->core_req & BASEP_JD_REQ_EVENT_NEVER) { ++ /* Don't report the event */ ++ kbase_event_process_noreport(ctx, atom); ++ return; ++ } ++ KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(atom, TL_ATOM_STATE_POSTED); ++ if (atom->core_req & BASE_JD_REQ_EVENT_COALESCE) { ++ /* Don't report the event until other event(s) have completed */ ++ mutex_lock(&ctx->event_mutex); ++ list_add_tail(&atom->dep_item[0], &ctx->event_coalesce_list); ++ ++ctx->event_coalesce_count; ++ mutex_unlock(&ctx->event_mutex); ++ } else { ++ /* Report the event and any pending events now */ ++ int event_count = 1; ++ ++ mutex_lock(&ctx->event_mutex); ++ event_count += kbase_event_coalesce(ctx); ++ list_add_tail(&atom->dep_item[0], &ctx->event_list); ++ atomic_add(event_count, &ctx->event_count); ++ mutex_unlock(&ctx->event_mutex); ++ ++ kbase_event_wakeup(ctx); ++ } ++} ++KBASE_EXPORT_TEST_API(kbase_event_post); ++ ++void kbase_event_close(struct kbase_context *kctx) ++{ ++ mutex_lock(&kctx->event_mutex); ++ atomic_set(&kctx->event_closed, true); ++ mutex_unlock(&kctx->event_mutex); ++ kbase_event_wakeup(kctx); ++} ++ ++int kbase_event_init(struct kbase_context *kctx) ++{ ++ KBASE_DEBUG_ASSERT(kctx); ++ ++ INIT_LIST_HEAD(&kctx->event_list); ++ INIT_LIST_HEAD(&kctx->event_coalesce_list); ++ mutex_init(&kctx->event_mutex); ++ atomic_set(&kctx->event_count, 0); ++ kctx->event_coalesce_count = 0; ++ atomic_set(&kctx->event_closed, false); ++ kctx->event_workq = alloc_workqueue("kbase_event", WQ_MEM_RECLAIM, 1); ++ ++ if (NULL == kctx->event_workq) ++ return -EINVAL; ++ ++ return 0; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_event_init); ++ ++void kbase_event_cleanup(struct kbase_context *kctx) ++{ ++ int event_count; ++ ++ KBASE_DEBUG_ASSERT(kctx); ++ KBASE_DEBUG_ASSERT(kctx->event_workq); ++ ++ flush_workqueue(kctx->event_workq); ++ destroy_workqueue(kctx->event_workq); ++ ++ /* We use kbase_event_dequeue to remove the remaining events as that ++ * deals with all the cleanup needed for the atoms. ++ * ++ * Note: use of kctx->event_list without a lock is safe because this must be the last ++ * thread using it (because we're about to terminate the lock) ++ */ ++ event_count = kbase_event_coalesce(kctx); ++ atomic_add(event_count, &kctx->event_count); ++ ++ while (!list_empty(&kctx->event_list)) { ++ struct base_jd_event_v2 event; ++ ++ kbase_event_dequeue(kctx, &event); ++ } ++} ++ ++KBASE_EXPORT_TEST_API(kbase_event_cleanup); +diff --git a/drivers/gpu/arm/midgard/mali_kbase_fence.c b/drivers/gpu/arm/midgard/mali_kbase_fence.c +new file mode 100755 +index 000000000000..3bcfb38c31c2 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_fence.c +@@ -0,0 +1,200 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++/* Spin lock protecting all Mali fences as fence->lock. */ ++static DEFINE_SPINLOCK(kbase_fence_lock); ++ ++static const char * ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++kbase_fence_get_driver_name(struct fence *fence) ++#else ++kbase_fence_get_driver_name(struct dma_fence *fence) ++#endif ++{ ++ return kbase_drv_name; ++} ++ ++static const char * ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++kbase_fence_get_timeline_name(struct fence *fence) ++#else ++kbase_fence_get_timeline_name(struct dma_fence *fence) ++#endif ++{ ++ return kbase_timeline_name; ++} ++ ++static bool ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++kbase_fence_enable_signaling(struct fence *fence) ++#else ++kbase_fence_enable_signaling(struct dma_fence *fence) ++#endif ++{ ++ return true; ++} ++ ++static void ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++kbase_fence_fence_value_str(struct fence *fence, char *str, int size) ++#else ++kbase_fence_fence_value_str(struct dma_fence *fence, char *str, int size) ++#endif ++{ ++#if (KERNEL_VERSION(5, 1, 0) > LINUX_VERSION_CODE) ++ snprintf(str, size, "%u", fence->seqno); ++#else ++ snprintf(str, size, "%llu", fence->seqno); ++#endif ++} ++ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++const struct fence_ops kbase_fence_ops = { ++ .wait = fence_default_wait, ++#else ++const struct dma_fence_ops kbase_fence_ops = { ++ .wait = dma_fence_default_wait, ++#endif ++ .get_driver_name = kbase_fence_get_driver_name, ++ .get_timeline_name = kbase_fence_get_timeline_name, ++ .enable_signaling = kbase_fence_enable_signaling, ++ .fence_value_str = kbase_fence_fence_value_str ++}; ++ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++struct fence * ++kbase_fence_out_new(struct kbase_jd_atom *katom) ++#else ++struct dma_fence * ++kbase_fence_out_new(struct kbase_jd_atom *katom) ++#endif ++{ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ struct fence *fence; ++#else ++ struct dma_fence *fence; ++#endif ++ ++ WARN_ON(katom->dma_fence.fence); ++ ++ fence = kzalloc(sizeof(*fence), GFP_KERNEL); ++ if (!fence) ++ return NULL; ++ ++ dma_fence_init(fence, ++ &kbase_fence_ops, ++ &kbase_fence_lock, ++ katom->dma_fence.context, ++ atomic_inc_return(&katom->dma_fence.seqno)); ++ ++ katom->dma_fence.fence = fence; ++ ++ return fence; ++} ++ ++bool ++kbase_fence_free_callbacks(struct kbase_jd_atom *katom) ++{ ++ struct kbase_fence_cb *cb, *tmp; ++ bool res = false; ++ ++ lockdep_assert_held(&katom->kctx->jctx.lock); ++ ++ /* Clean up and free callbacks. */ ++ list_for_each_entry_safe(cb, tmp, &katom->dma_fence.callbacks, node) { ++ bool ret; ++ ++ /* Cancel callbacks that hasn't been called yet. */ ++ ret = dma_fence_remove_callback(cb->fence, &cb->fence_cb); ++ if (ret) { ++ int ret; ++ ++ /* Fence had not signaled, clean up after ++ * canceling. ++ */ ++ ret = atomic_dec_return(&katom->dma_fence.dep_count); ++ ++ if (unlikely(ret == 0)) ++ res = true; ++ } ++ ++ /* ++ * Release the reference taken in ++ * kbase_fence_add_callback(). ++ */ ++ dma_fence_put(cb->fence); ++ list_del(&cb->node); ++ kfree(cb); ++ } ++ ++ return res; ++} ++ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++int ++kbase_fence_add_callback(struct kbase_jd_atom *katom, ++ struct fence *fence, ++ fence_func_t callback) ++#else ++int ++kbase_fence_add_callback(struct kbase_jd_atom *katom, ++ struct dma_fence *fence, ++ dma_fence_func_t callback) ++#endif ++{ ++ int err = 0; ++ struct kbase_fence_cb *kbase_fence_cb; ++ ++ if (!fence) ++ return -EINVAL; ++ ++ kbase_fence_cb = kmalloc(sizeof(*kbase_fence_cb), GFP_KERNEL); ++ if (!kbase_fence_cb) ++ return -ENOMEM; ++ ++ kbase_fence_cb->fence = fence; ++ kbase_fence_cb->katom = katom; ++ INIT_LIST_HEAD(&kbase_fence_cb->node); ++ ++ err = dma_fence_add_callback(fence, &kbase_fence_cb->fence_cb, ++ callback); ++ if (err == -ENOENT) { ++ /* Fence signaled, clear the error and return */ ++ err = 0; ++ kfree(kbase_fence_cb); ++ } else if (err) { ++ kfree(kbase_fence_cb); ++ } else { ++ /* ++ * Get reference to fence that will be kept until callback gets ++ * cleaned up in kbase_fence_free_callbacks(). ++ */ ++ dma_fence_get(fence); ++ atomic_inc(&katom->dma_fence.dep_count); ++ /* Add callback to katom's list of callbacks */ ++ list_add(&kbase_fence_cb->node, &katom->dma_fence.callbacks); ++ } ++ ++ return err; ++} +diff --git a/drivers/gpu/arm/midgard/mali_kbase_fence.h b/drivers/gpu/arm/midgard/mali_kbase_fence.h +new file mode 100755 +index 000000000000..639cc2ef4348 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_fence.h +@@ -0,0 +1,275 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KBASE_FENCE_H_ ++#define _KBASE_FENCE_H_ ++ ++/* ++ * mali_kbase_fence.[hc] has common fence code used by both ++ * - CONFIG_MALI_DMA_FENCE - implicit DMA fences ++ * - CONFIG_SYNC_FILE - explicit fences beginning with 4.9 kernel ++ */ ++ ++#if defined(CONFIG_MALI_DMA_FENCE) || defined(CONFIG_SYNC_FILE) ++ ++#include ++#include "mali_kbase_fence_defs.h" ++#include "mali_kbase.h" ++ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++extern const struct fence_ops kbase_fence_ops; ++#else ++extern const struct dma_fence_ops kbase_fence_ops; ++#endif ++ ++/** ++* struct kbase_fence_cb - Mali dma-fence callback data struct ++* @fence_cb: Callback function ++* @katom: Pointer to katom that is waiting on this callback ++* @fence: Pointer to the fence object on which this callback is waiting ++* @node: List head for linking this callback to the katom ++*/ ++struct kbase_fence_cb { ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ struct fence_cb fence_cb; ++ struct fence *fence; ++#else ++ struct dma_fence_cb fence_cb; ++ struct dma_fence *fence; ++#endif ++ struct kbase_jd_atom *katom; ++ struct list_head node; ++}; ++ ++/** ++ * kbase_fence_out_new() - Creates a new output fence and puts it on the atom ++ * @katom: Atom to create an output fence for ++ * ++ * return: A new fence object on success, NULL on failure. ++ */ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++struct fence *kbase_fence_out_new(struct kbase_jd_atom *katom); ++#else ++struct dma_fence *kbase_fence_out_new(struct kbase_jd_atom *katom); ++#endif ++ ++#if defined(CONFIG_SYNC_FILE) ++/** ++ * kbase_fence_fence_in_set() - Assign input fence to atom ++ * @katom: Atom to assign input fence to ++ * @fence: Input fence to assign to atom ++ * ++ * This function will take ownership of one fence reference! ++ */ ++#define kbase_fence_fence_in_set(katom, fence) \ ++ do { \ ++ WARN_ON((katom)->dma_fence.fence_in); \ ++ (katom)->dma_fence.fence_in = fence; \ ++ } while (0) ++#endif ++ ++/** ++ * kbase_fence_out_remove() - Removes the output fence from atom ++ * @katom: Atom to remove output fence for ++ * ++ * This will also release the reference to this fence which the atom keeps ++ */ ++static inline void kbase_fence_out_remove(struct kbase_jd_atom *katom) ++{ ++ if (katom->dma_fence.fence) { ++ dma_fence_put(katom->dma_fence.fence); ++ katom->dma_fence.fence = NULL; ++ } ++} ++ ++#if defined(CONFIG_SYNC_FILE) ++/** ++ * kbase_fence_out_remove() - Removes the input fence from atom ++ * @katom: Atom to remove input fence for ++ * ++ * This will also release the reference to this fence which the atom keeps ++ */ ++static inline void kbase_fence_in_remove(struct kbase_jd_atom *katom) ++{ ++ if (katom->dma_fence.fence_in) { ++ dma_fence_put(katom->dma_fence.fence_in); ++ katom->dma_fence.fence_in = NULL; ++ } ++} ++#endif ++ ++/** ++ * kbase_fence_out_is_ours() - Check if atom has a valid fence created by us ++ * @katom: Atom to check output fence for ++ * ++ * Return: true if fence exists and is valid, otherwise false ++ */ ++static inline bool kbase_fence_out_is_ours(struct kbase_jd_atom *katom) ++{ ++ return katom->dma_fence.fence && ++ katom->dma_fence.fence->ops == &kbase_fence_ops; ++} ++ ++/** ++ * kbase_fence_out_signal() - Signal output fence of atom ++ * @katom: Atom to signal output fence for ++ * @status: Status to signal with (0 for success, < 0 for error) ++ * ++ * Return: 0 on success, < 0 on error ++ */ ++static inline int kbase_fence_out_signal(struct kbase_jd_atom *katom, ++ int status) ++{ ++ if (status) { ++#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE && \ ++ KERNEL_VERSION(4, 9, 68) <= LINUX_VERSION_CODE) ++ fence_set_error(katom->dma_fence.fence, status); ++#elif (KERNEL_VERSION(4, 11, 0) <= LINUX_VERSION_CODE) ++ dma_fence_set_error(katom->dma_fence.fence, status); ++#else ++ katom->dma_fence.fence->status = status; ++#endif ++ } ++ return dma_fence_signal(katom->dma_fence.fence); ++} ++ ++/** ++ * kbase_fence_add_callback() - Add callback on @fence to block @katom ++ * @katom: Pointer to katom that will be blocked by @fence ++ * @fence: Pointer to fence on which to set up the callback ++ * @callback: Pointer to function to be called when fence is signaled ++ * ++ * Caller needs to hold a reference to @fence when calling this function, and ++ * the caller is responsible for releasing that reference. An additional ++ * reference to @fence will be taken when the callback was successfully set up ++ * and @fence needs to be kept valid until the callback has been called and ++ * cleanup have been done. ++ * ++ * Return: 0 on success: fence was either already signaled, or callback was ++ * set up. Negative error code is returned on error. ++ */ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++int kbase_fence_add_callback(struct kbase_jd_atom *katom, ++ struct fence *fence, ++ fence_func_t callback); ++#else ++int kbase_fence_add_callback(struct kbase_jd_atom *katom, ++ struct dma_fence *fence, ++ dma_fence_func_t callback); ++#endif ++ ++/** ++ * kbase_fence_dep_count_set() - Set dep_count value on atom to specified value ++ * @katom: Atom to set dep_count for ++ * @val: value to set dep_count to ++ * ++ * The dep_count is available to the users of this module so that they can ++ * synchronize completion of the wait with cancellation and adding of more ++ * callbacks. For instance, a user could do the following: ++ * ++ * dep_count set to 1 ++ * callback #1 added, dep_count is increased to 2 ++ * callback #1 happens, dep_count decremented to 1 ++ * since dep_count > 0, no completion is done ++ * callback #2 is added, dep_count is increased to 2 ++ * dep_count decremented to 1 ++ * callback #2 happens, dep_count decremented to 0 ++ * since dep_count now is zero, completion executes ++ * ++ * The dep_count can also be used to make sure that the completion only ++ * executes once. This is typically done by setting dep_count to -1 for the ++ * thread that takes on this responsibility. ++ */ ++static inline void ++kbase_fence_dep_count_set(struct kbase_jd_atom *katom, int val) ++{ ++ atomic_set(&katom->dma_fence.dep_count, val); ++} ++ ++/** ++ * kbase_fence_dep_count_dec_and_test() - Decrements dep_count ++ * @katom: Atom to decrement dep_count for ++ * ++ * See @kbase_fence_dep_count_set for general description about dep_count ++ * ++ * Return: true if value was decremented to zero, otherwise false ++ */ ++static inline bool ++kbase_fence_dep_count_dec_and_test(struct kbase_jd_atom *katom) ++{ ++ return atomic_dec_and_test(&katom->dma_fence.dep_count); ++} ++ ++/** ++ * kbase_fence_dep_count_read() - Returns the current dep_count value ++ * @katom: Pointer to katom ++ * ++ * See @kbase_fence_dep_count_set for general description about dep_count ++ * ++ * Return: The current dep_count value ++ */ ++static inline int kbase_fence_dep_count_read(struct kbase_jd_atom *katom) ++{ ++ return atomic_read(&katom->dma_fence.dep_count); ++} ++ ++/** ++ * kbase_fence_free_callbacks() - Free dma-fence callbacks on a katom ++ * @katom: Pointer to katom ++ * ++ * This function will free all fence callbacks on the katom's list of ++ * callbacks. Callbacks that have not yet been called, because their fence ++ * hasn't yet signaled, will first be removed from the fence. ++ * ++ * Locking: katom->dma_fence.callbacks list assumes jctx.lock is held. ++ * ++ * Return: true if dep_count reached 0, otherwise false. ++ */ ++bool kbase_fence_free_callbacks(struct kbase_jd_atom *katom); ++ ++#if defined(CONFIG_SYNC_FILE) ++/** ++ * kbase_fence_in_get() - Retrieve input fence for atom. ++ * @katom: Atom to get input fence from ++ * ++ * A ref will be taken for the fence, so use @kbase_fence_put() to release it ++ * ++ * Return: The fence, or NULL if there is no input fence for atom ++ */ ++#define kbase_fence_in_get(katom) dma_fence_get((katom)->dma_fence.fence_in) ++#endif ++ ++/** ++ * kbase_fence_out_get() - Retrieve output fence for atom. ++ * @katom: Atom to get output fence from ++ * ++ * A ref will be taken for the fence, so use @kbase_fence_put() to release it ++ * ++ * Return: The fence, or NULL if there is no output fence for atom ++ */ ++#define kbase_fence_out_get(katom) dma_fence_get((katom)->dma_fence.fence) ++ ++/** ++ * kbase_fence_put() - Releases a reference to a fence ++ * @fence: Fence to release reference for. ++ */ ++#define kbase_fence_put(fence) dma_fence_put(fence) ++ ++ ++#endif /* CONFIG_MALI_DMA_FENCE || defined(CONFIG_SYNC_FILE */ ++ ++#endif /* _KBASE_FENCE_H_ */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_fence_defs.h b/drivers/gpu/arm/midgard/mali_kbase_fence_defs.h +new file mode 100755 +index 000000000000..fa2c6dfe999e +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_fence_defs.h +@@ -0,0 +1,51 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KBASE_FENCE_DEFS_H_ ++#define _KBASE_FENCE_DEFS_H_ ++ ++/* ++ * There was a big rename in the 4.10 kernel (fence* -> dma_fence*) ++ * This file hides the compatibility issues with this for the rest the driver ++ */ ++ ++#if defined(CONFIG_MALI_DMA_FENCE) || defined(CONFIG_SYNC_FILE) ++ ++#include ++ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ ++#include ++ ++#define dma_fence_context_alloc(a) fence_context_alloc(a) ++#define dma_fence_init(a, b, c, d, e) fence_init(a, b, c, d, e) ++#define dma_fence_get(a) fence_get(a) ++#define dma_fence_put(a) fence_put(a) ++#define dma_fence_signal(a) fence_signal(a) ++#define dma_fence_is_signaled(a) fence_is_signaled(a) ++#define dma_fence_add_callback(a, b, c) fence_add_callback(a, b, c) ++#define dma_fence_remove_callback(a, b) fence_remove_callback(a, b) ++ ++#else ++ ++#include ++ ++#endif /* < 4.10.0 */ ++ ++#endif /* CONFIG_MALI_DMA_FENCE || CONFIG_SYNC_FILE */ ++ ++#endif /* _KBASE_FENCE_DEFS_H_ */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_gator.h b/drivers/gpu/arm/midgard/mali_kbase_gator.h +new file mode 100755 +index 000000000000..ce65b5562a2b +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_gator.h +@@ -0,0 +1,45 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* NB taken from gator */ ++/* ++ * List of possible actions to be controlled by DS-5 Streamline. ++ * The following numbers are used by gator to control the frame buffer dumping ++ * and s/w counter reporting. We cannot use the enums in mali_uk_types.h because ++ * they are unknown inside gator. ++ */ ++#ifndef _KBASE_GATOR_H_ ++#define _KBASE_GATOR_H_ ++ ++#ifdef CONFIG_MALI_GATOR_SUPPORT ++#define GATOR_MAKE_EVENT(type, number) (((type) << 24) | ((number) << 16)) ++#define GATOR_JOB_SLOT_START 1 ++#define GATOR_JOB_SLOT_STOP 2 ++#define GATOR_JOB_SLOT_SOFT_STOPPED 3 ++ ++void kbase_trace_mali_job_slots_event(u32 event, const struct kbase_context *kctx, u8 atom_id); ++void kbase_trace_mali_pm_status(u32 event, u64 value); ++void kbase_trace_mali_pm_power_off(u32 event, u64 value); ++void kbase_trace_mali_pm_power_on(u32 event, u64 value); ++void kbase_trace_mali_page_fault_insert_pages(int event, u32 value); ++void kbase_trace_mali_mmu_as_in_use(int event); ++void kbase_trace_mali_mmu_as_released(int event); ++void kbase_trace_mali_total_alloc_pages_change(long long int event); ++ ++#endif /* CONFIG_MALI_GATOR_SUPPORT */ ++ ++#endif /* _KBASE_GATOR_H_ */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_gator_api.c b/drivers/gpu/arm/midgard/mali_kbase_gator_api.c +new file mode 100755 +index 000000000000..860e10159fb3 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_gator_api.c +@@ -0,0 +1,334 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include "mali_kbase.h" ++#include "mali_kbase_hw.h" ++#include "mali_kbase_mem_linux.h" ++#include "mali_kbase_gator_api.h" ++#include "mali_kbase_gator_hwcnt_names.h" ++ ++#define MALI_MAX_CORES_PER_GROUP 4 ++#define MALI_MAX_NUM_BLOCKS_PER_GROUP 8 ++#define MALI_COUNTERS_PER_BLOCK 64 ++#define MALI_BYTES_PER_COUNTER 4 ++ ++struct kbase_gator_hwcnt_handles { ++ struct kbase_device *kbdev; ++ struct kbase_vinstr_client *vinstr_cli; ++ void *vinstr_buffer; ++ struct work_struct dump_work; ++ int dump_complete; ++ spinlock_t dump_lock; ++}; ++ ++static void dump_worker(struct work_struct *work); ++ ++const char * const *kbase_gator_hwcnt_init_names(uint32_t *total_counters) ++{ ++ const char * const *hardware_counters; ++ struct kbase_device *kbdev; ++ uint32_t product_id; ++ uint32_t count; ++ ++ if (!total_counters) ++ return NULL; ++ ++ /* Get the first device - it doesn't matter in this case */ ++ kbdev = kbase_find_device(-1); ++ if (!kbdev) ++ return NULL; ++ ++ product_id = kbdev->gpu_props.props.core_props.product_id; ++ ++ if (GPU_ID_IS_NEW_FORMAT(product_id)) { ++ switch (GPU_ID2_MODEL_MATCH_VALUE(product_id)) { ++ case GPU_ID2_PRODUCT_TMIX: ++ hardware_counters = hardware_counters_mali_tMIx; ++ count = ARRAY_SIZE(hardware_counters_mali_tMIx); ++ break; ++ case GPU_ID2_PRODUCT_THEX: ++ hardware_counters = hardware_counters_mali_tHEx; ++ count = ARRAY_SIZE(hardware_counters_mali_tHEx); ++ break; ++ case GPU_ID2_PRODUCT_TSIX: ++ hardware_counters = hardware_counters_mali_tSIx; ++ count = ARRAY_SIZE(hardware_counters_mali_tSIx); ++ break; ++ default: ++ hardware_counters = NULL; ++ count = 0; ++ dev_err(kbdev->dev, "Unrecognized product ID: %u\n", ++ product_id); ++ break; ++ } ++ } else { ++ switch (product_id) { ++ /* If we are using a Mali-T60x device */ ++ case GPU_ID_PI_T60X: ++ hardware_counters = hardware_counters_mali_t60x; ++ count = ARRAY_SIZE(hardware_counters_mali_t60x); ++ break; ++ /* If we are using a Mali-T62x device */ ++ case GPU_ID_PI_T62X: ++ hardware_counters = hardware_counters_mali_t62x; ++ count = ARRAY_SIZE(hardware_counters_mali_t62x); ++ break; ++ /* If we are using a Mali-T72x device */ ++ case GPU_ID_PI_T72X: ++ hardware_counters = hardware_counters_mali_t72x; ++ count = ARRAY_SIZE(hardware_counters_mali_t72x); ++ break; ++ /* If we are using a Mali-T76x device */ ++ case GPU_ID_PI_T76X: ++ hardware_counters = hardware_counters_mali_t76x; ++ count = ARRAY_SIZE(hardware_counters_mali_t76x); ++ break; ++ /* If we are using a Mali-T82x device */ ++ case GPU_ID_PI_T82X: ++ hardware_counters = hardware_counters_mali_t82x; ++ count = ARRAY_SIZE(hardware_counters_mali_t82x); ++ break; ++ /* If we are using a Mali-T83x device */ ++ case GPU_ID_PI_T83X: ++ hardware_counters = hardware_counters_mali_t83x; ++ count = ARRAY_SIZE(hardware_counters_mali_t83x); ++ break; ++ /* If we are using a Mali-T86x device */ ++ case GPU_ID_PI_T86X: ++ hardware_counters = hardware_counters_mali_t86x; ++ count = ARRAY_SIZE(hardware_counters_mali_t86x); ++ break; ++ /* If we are using a Mali-T88x device */ ++ case GPU_ID_PI_TFRX: ++ hardware_counters = hardware_counters_mali_t88x; ++ count = ARRAY_SIZE(hardware_counters_mali_t88x); ++ break; ++ default: ++ hardware_counters = NULL; ++ count = 0; ++ dev_err(kbdev->dev, "Unrecognized product ID: %u\n", ++ product_id); ++ break; ++ } ++ } ++ ++ /* Release the kbdev reference. */ ++ kbase_release_device(kbdev); ++ ++ *total_counters = count; ++ ++ /* If we return a string array take a reference on the module (or fail). */ ++ if (hardware_counters && !try_module_get(THIS_MODULE)) ++ return NULL; ++ ++ return hardware_counters; ++} ++KBASE_EXPORT_SYMBOL(kbase_gator_hwcnt_init_names); ++ ++void kbase_gator_hwcnt_term_names(void) ++{ ++ /* Release the module reference. */ ++ module_put(THIS_MODULE); ++} ++KBASE_EXPORT_SYMBOL(kbase_gator_hwcnt_term_names); ++ ++struct kbase_gator_hwcnt_handles *kbase_gator_hwcnt_init(struct kbase_gator_hwcnt_info *in_out_info) ++{ ++ struct kbase_gator_hwcnt_handles *hand; ++ struct kbase_uk_hwcnt_reader_setup setup; ++ uint32_t dump_size = 0, i = 0; ++ ++ if (!in_out_info) ++ return NULL; ++ ++ hand = kzalloc(sizeof(*hand), GFP_KERNEL); ++ if (!hand) ++ return NULL; ++ ++ INIT_WORK(&hand->dump_work, dump_worker); ++ spin_lock_init(&hand->dump_lock); ++ ++ /* Get the first device */ ++ hand->kbdev = kbase_find_device(-1); ++ if (!hand->kbdev) ++ goto free_hand; ++ ++ dump_size = kbase_vinstr_dump_size(hand->kbdev); ++ hand->vinstr_buffer = kzalloc(dump_size, GFP_KERNEL); ++ if (!hand->vinstr_buffer) ++ goto release_device; ++ in_out_info->kernel_dump_buffer = hand->vinstr_buffer; ++ ++ in_out_info->nr_cores = hand->kbdev->gpu_props.num_cores; ++ in_out_info->nr_core_groups = hand->kbdev->gpu_props.num_core_groups; ++ in_out_info->gpu_id = hand->kbdev->gpu_props.props.core_props.product_id; ++ ++ /* If we are using a v4 device (Mali-T6xx or Mali-T72x) */ ++ if (kbase_hw_has_feature(hand->kbdev, BASE_HW_FEATURE_V4)) { ++ uint32_t cg, j; ++ uint64_t core_mask; ++ ++ /* There are 8 hardware counters blocks per core group */ ++ in_out_info->hwc_layout = kmalloc(sizeof(enum hwc_type) * ++ MALI_MAX_NUM_BLOCKS_PER_GROUP * ++ in_out_info->nr_core_groups, GFP_KERNEL); ++ ++ if (!in_out_info->hwc_layout) ++ goto free_vinstr_buffer; ++ ++ dump_size = in_out_info->nr_core_groups * ++ MALI_MAX_NUM_BLOCKS_PER_GROUP * ++ MALI_COUNTERS_PER_BLOCK * ++ MALI_BYTES_PER_COUNTER; ++ ++ for (cg = 0; cg < in_out_info->nr_core_groups; cg++) { ++ core_mask = hand->kbdev->gpu_props.props.coherency_info.group[cg].core_mask; ++ ++ for (j = 0; j < MALI_MAX_CORES_PER_GROUP; j++) { ++ if (core_mask & (1u << j)) ++ in_out_info->hwc_layout[i++] = SHADER_BLOCK; ++ else ++ in_out_info->hwc_layout[i++] = RESERVED_BLOCK; ++ } ++ ++ in_out_info->hwc_layout[i++] = TILER_BLOCK; ++ in_out_info->hwc_layout[i++] = MMU_L2_BLOCK; ++ ++ in_out_info->hwc_layout[i++] = RESERVED_BLOCK; ++ ++ if (0 == cg) ++ in_out_info->hwc_layout[i++] = JM_BLOCK; ++ else ++ in_out_info->hwc_layout[i++] = RESERVED_BLOCK; ++ } ++ /* If we are using any other device */ ++ } else { ++ uint32_t nr_l2, nr_sc_bits, j; ++ uint64_t core_mask; ++ ++ nr_l2 = hand->kbdev->gpu_props.props.l2_props.num_l2_slices; ++ ++ core_mask = hand->kbdev->gpu_props.props.coherency_info.group[0].core_mask; ++ ++ nr_sc_bits = fls64(core_mask); ++ ++ /* The job manager and tiler sets of counters ++ * are always present */ ++ in_out_info->hwc_layout = kmalloc(sizeof(enum hwc_type) * (2 + nr_sc_bits + nr_l2), GFP_KERNEL); ++ ++ if (!in_out_info->hwc_layout) ++ goto free_vinstr_buffer; ++ ++ dump_size = (2 + nr_sc_bits + nr_l2) * MALI_COUNTERS_PER_BLOCK * MALI_BYTES_PER_COUNTER; ++ ++ in_out_info->hwc_layout[i++] = JM_BLOCK; ++ in_out_info->hwc_layout[i++] = TILER_BLOCK; ++ ++ for (j = 0; j < nr_l2; j++) ++ in_out_info->hwc_layout[i++] = MMU_L2_BLOCK; ++ ++ while (core_mask != 0ull) { ++ if ((core_mask & 1ull) != 0ull) ++ in_out_info->hwc_layout[i++] = SHADER_BLOCK; ++ else ++ in_out_info->hwc_layout[i++] = RESERVED_BLOCK; ++ core_mask >>= 1; ++ } ++ } ++ ++ in_out_info->nr_hwc_blocks = i; ++ in_out_info->size = dump_size; ++ ++ setup.jm_bm = in_out_info->bitmask[0]; ++ setup.tiler_bm = in_out_info->bitmask[1]; ++ setup.shader_bm = in_out_info->bitmask[2]; ++ setup.mmu_l2_bm = in_out_info->bitmask[3]; ++ hand->vinstr_cli = kbase_vinstr_hwcnt_kernel_setup(hand->kbdev->vinstr_ctx, ++ &setup, hand->vinstr_buffer); ++ if (!hand->vinstr_cli) { ++ dev_err(hand->kbdev->dev, "Failed to register gator with vinstr core"); ++ goto free_layout; ++ } ++ ++ return hand; ++ ++free_layout: ++ kfree(in_out_info->hwc_layout); ++ ++free_vinstr_buffer: ++ kfree(hand->vinstr_buffer); ++ ++release_device: ++ kbase_release_device(hand->kbdev); ++ ++free_hand: ++ kfree(hand); ++ return NULL; ++} ++KBASE_EXPORT_SYMBOL(kbase_gator_hwcnt_init); ++ ++void kbase_gator_hwcnt_term(struct kbase_gator_hwcnt_info *in_out_info, struct kbase_gator_hwcnt_handles *opaque_handles) ++{ ++ if (in_out_info) ++ kfree(in_out_info->hwc_layout); ++ ++ if (opaque_handles) { ++ cancel_work_sync(&opaque_handles->dump_work); ++ kbase_vinstr_detach_client(opaque_handles->vinstr_cli); ++ kfree(opaque_handles->vinstr_buffer); ++ kbase_release_device(opaque_handles->kbdev); ++ kfree(opaque_handles); ++ } ++} ++KBASE_EXPORT_SYMBOL(kbase_gator_hwcnt_term); ++ ++static void dump_worker(struct work_struct *work) ++{ ++ struct kbase_gator_hwcnt_handles *hand; ++ ++ hand = container_of(work, struct kbase_gator_hwcnt_handles, dump_work); ++ if (!kbase_vinstr_hwc_dump(hand->vinstr_cli, ++ BASE_HWCNT_READER_EVENT_MANUAL)) { ++ spin_lock_bh(&hand->dump_lock); ++ hand->dump_complete = 1; ++ spin_unlock_bh(&hand->dump_lock); ++ } else { ++ schedule_work(&hand->dump_work); ++ } ++} ++ ++uint32_t kbase_gator_instr_hwcnt_dump_complete( ++ struct kbase_gator_hwcnt_handles *opaque_handles, ++ uint32_t * const success) ++{ ++ ++ if (opaque_handles && success) { ++ *success = opaque_handles->dump_complete; ++ opaque_handles->dump_complete = 0; ++ return *success; ++ } ++ return 0; ++} ++KBASE_EXPORT_SYMBOL(kbase_gator_instr_hwcnt_dump_complete); ++ ++uint32_t kbase_gator_instr_hwcnt_dump_irq(struct kbase_gator_hwcnt_handles *opaque_handles) ++{ ++ if (opaque_handles) ++ schedule_work(&opaque_handles->dump_work); ++ return 0; ++} ++KBASE_EXPORT_SYMBOL(kbase_gator_instr_hwcnt_dump_irq); +diff --git a/drivers/gpu/arm/midgard/mali_kbase_gator_api.h b/drivers/gpu/arm/midgard/mali_kbase_gator_api.h +new file mode 100755 +index 000000000000..ef9ac0f7b633 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_gator_api.h +@@ -0,0 +1,219 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KBASE_GATOR_API_H_ ++#define _KBASE_GATOR_API_H_ ++ ++/** ++ * @brief This file describes the API used by Gator to fetch hardware counters. ++ */ ++ ++/* This define is used by the gator kernel module compile to select which DDK ++ * API calling convention to use. If not defined (legacy DDK) gator assumes ++ * version 1. The version to DDK release mapping is: ++ * Version 1 API: DDK versions r1px, r2px ++ * Version 2 API: DDK versions r3px, r4px ++ * Version 3 API: DDK version r5p0 and newer ++ * ++ * API Usage ++ * ========= ++ * ++ * 1] Call kbase_gator_hwcnt_init_names() to return the list of short counter ++ * names for the GPU present in this device. ++ * ++ * 2] Create a kbase_gator_hwcnt_info structure and set the counter enables for ++ * the counters you want enabled. The enables can all be set for simplicity in ++ * most use cases, but disabling some will let you minimize bandwidth impact. ++ * ++ * 3] Call kbase_gator_hwcnt_init() using the above structure, to create a ++ * counter context. On successful return the DDK will have populated the ++ * structure with a variety of useful information. ++ * ++ * 4] Call kbase_gator_hwcnt_dump_irq() to queue a non-blocking request for a ++ * counter dump. If this returns a non-zero value the request has been queued, ++ * otherwise the driver has been unable to do so (typically because of another ++ * user of the instrumentation exists concurrently). ++ * ++ * 5] Call kbase_gator_hwcnt_dump_complete() to test whether the previously ++ * requested dump has been succesful. If this returns non-zero the counter dump ++ * has resolved, but the value of *success must also be tested as the dump ++ * may have not been successful. If it returns zero the counter dump was ++ * abandoned due to the device being busy (typically because of another ++ * user of the instrumentation exists concurrently). ++ * ++ * 6] Process the counters stored in the buffer pointed to by ... ++ * ++ * kbase_gator_hwcnt_info->kernel_dump_buffer ++ * ++ * In pseudo code you can find all of the counters via this approach: ++ * ++ * ++ * hwcnt_info # pointer to kbase_gator_hwcnt_info structure ++ * hwcnt_name # pointer to name list ++ * ++ * u32 * hwcnt_data = (u32*)hwcnt_info->kernel_dump_buffer ++ * ++ * # Iterate over each 64-counter block in this GPU configuration ++ * for( i = 0; i < hwcnt_info->nr_hwc_blocks; i++) { ++ * hwc_type type = hwcnt_info->hwc_layout[i]; ++ * ++ * # Skip reserved type blocks - they contain no counters at all ++ * if( type == RESERVED_BLOCK ) { ++ * continue; ++ * } ++ * ++ * size_t name_offset = type * 64; ++ * size_t data_offset = i * 64; ++ * ++ * # Iterate over the names of the counters in this block type ++ * for( j = 0; j < 64; j++) { ++ * const char * name = hwcnt_name[name_offset+j]; ++ * ++ * # Skip empty name strings - there is no counter here ++ * if( name[0] == '\0' ) { ++ * continue; ++ * } ++ * ++ * u32 data = hwcnt_data[data_offset+j]; ++ * ++ * printk( "COUNTER: %s DATA: %u\n", name, data ); ++ * } ++ * } ++ * ++ * ++ * Note that in most implementations you typically want to either SUM or ++ * AVERAGE multiple instances of the same counter if, for example, you have ++ * multiple shader cores or multiple L2 caches. The most sensible view for ++ * analysis is to AVERAGE shader core counters, but SUM L2 cache and MMU ++ * counters. ++ * ++ * 7] Goto 4, repeating until you want to stop collecting counters. ++ * ++ * 8] Release the dump resources by calling kbase_gator_hwcnt_term(). ++ * ++ * 9] Release the name table resources by calling ++ * kbase_gator_hwcnt_term_names(). This function must only be called if ++ * init_names() returned a non-NULL value. ++ **/ ++ ++#define MALI_DDK_GATOR_API_VERSION 3 ++ ++enum hwc_type { ++ JM_BLOCK = 0, ++ TILER_BLOCK, ++ SHADER_BLOCK, ++ MMU_L2_BLOCK, ++ RESERVED_BLOCK ++}; ++ ++struct kbase_gator_hwcnt_info { ++ /* Passed from Gator to kbase */ ++ ++ /* the bitmask of enabled hardware counters for each counter block */ ++ uint16_t bitmask[4]; ++ ++ /* Passed from kbase to Gator */ ++ ++ /* ptr to counter dump memory */ ++ void *kernel_dump_buffer; ++ ++ /* size of counter dump memory */ ++ uint32_t size; ++ ++ /* the ID of the Mali device */ ++ uint32_t gpu_id; ++ ++ /* the number of shader cores in the GPU */ ++ uint32_t nr_cores; ++ ++ /* the number of core groups */ ++ uint32_t nr_core_groups; ++ ++ /* the memory layout of the performance counters */ ++ enum hwc_type *hwc_layout; ++ ++ /* the total number of hardware couter blocks */ ++ uint32_t nr_hwc_blocks; ++}; ++ ++/** ++ * @brief Opaque block of Mali data which Gator needs to return to the API later. ++ */ ++struct kbase_gator_hwcnt_handles; ++ ++/** ++ * @brief Initialize the resources Gator needs for performance profiling. ++ * ++ * @param in_out_info A pointer to a structure containing the enabled counters passed from Gator and all the Mali ++ * specific information that will be returned to Gator. On entry Gator must have populated the ++ * 'bitmask' field with the counters it wishes to enable for each class of counter block. ++ * Each entry in the array corresponds to a single counter class based on the "hwc_type" ++ * enumeration, and each bit corresponds to an enable for 4 sequential counters (LSB enables ++ * the first 4 counters in the block, and so on). See the GPU counter array as returned by ++ * kbase_gator_hwcnt_get_names() for the index values of each counter for the curernt GPU. ++ * ++ * @return Pointer to an opaque handle block on success, NULL on error. ++ */ ++extern struct kbase_gator_hwcnt_handles *kbase_gator_hwcnt_init(struct kbase_gator_hwcnt_info *in_out_info); ++ ++/** ++ * @brief Free all resources once Gator has finished using performance counters. ++ * ++ * @param in_out_info A pointer to a structure containing the enabled counters passed from Gator and all the ++ * Mali specific information that will be returned to Gator. ++ * @param opaque_handles A wrapper structure for kbase structures. ++ */ ++extern void kbase_gator_hwcnt_term(struct kbase_gator_hwcnt_info *in_out_info, struct kbase_gator_hwcnt_handles *opaque_handles); ++ ++/** ++ * @brief Poll whether a counter dump is successful. ++ * ++ * @param opaque_handles A wrapper structure for kbase structures. ++ * @param[out] success Non-zero on success, zero on failure. ++ * ++ * @return Zero if the dump is still pending, non-zero if the dump has completed. Note that a ++ * completed dump may not have dumped succesfully, so the caller must test for both ++ * a completed and successful dump before processing counters. ++ */ ++extern uint32_t kbase_gator_instr_hwcnt_dump_complete(struct kbase_gator_hwcnt_handles *opaque_handles, uint32_t * const success); ++ ++/** ++ * @brief Request the generation of a new counter dump. ++ * ++ * @param opaque_handles A wrapper structure for kbase structures. ++ * ++ * @return Zero if the hardware device is busy and cannot handle the request, non-zero otherwise. ++ */ ++extern uint32_t kbase_gator_instr_hwcnt_dump_irq(struct kbase_gator_hwcnt_handles *opaque_handles); ++ ++/** ++ * @brief This function is used to fetch the names table based on the Mali device in use. ++ * ++ * @param[out] total_counters The total number of counters short names in the Mali devices' list. ++ * ++ * @return Pointer to an array of strings of length *total_counters. ++ */ ++extern const char * const *kbase_gator_hwcnt_init_names(uint32_t *total_counters); ++ ++/** ++ * @brief This function is used to terminate the use of the names table. ++ * ++ * This function must only be called if the initial call to kbase_gator_hwcnt_init_names returned a non-NULL value. ++ */ ++extern void kbase_gator_hwcnt_term_names(void); ++ ++#endif +diff --git a/drivers/gpu/arm/midgard/mali_kbase_gator_hwcnt_names.h b/drivers/gpu/arm/midgard/mali_kbase_gator_hwcnt_names.h +new file mode 100755 +index 000000000000..cad19b66200d +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_gator_hwcnt_names.h +@@ -0,0 +1,2170 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KBASE_GATOR_HWCNT_NAMES_H_ ++#define _KBASE_GATOR_HWCNT_NAMES_H_ ++ ++/* ++ * "Short names" for hardware counters used by Streamline. Counters names are ++ * stored in accordance with their memory layout in the binary counter block ++ * emitted by the Mali GPU. Each "master" in the GPU emits a fixed-size block ++ * of 64 counters, and each GPU implements the same set of "masters" although ++ * the counters each master exposes within its block of 64 may vary. ++ * ++ * Counters which are an empty string are simply "holes" in the counter memory ++ * where no counter exists. ++ */ ++ ++static const char * const hardware_counters_mali_t60x[] = { ++ /* Job Manager */ ++ "", ++ "", ++ "", ++ "", ++ "T60x_MESSAGES_SENT", ++ "T60x_MESSAGES_RECEIVED", ++ "T60x_GPU_ACTIVE", ++ "T60x_IRQ_ACTIVE", ++ "T60x_JS0_JOBS", ++ "T60x_JS0_TASKS", ++ "T60x_JS0_ACTIVE", ++ "", ++ "T60x_JS0_WAIT_READ", ++ "T60x_JS0_WAIT_ISSUE", ++ "T60x_JS0_WAIT_DEPEND", ++ "T60x_JS0_WAIT_FINISH", ++ "T60x_JS1_JOBS", ++ "T60x_JS1_TASKS", ++ "T60x_JS1_ACTIVE", ++ "", ++ "T60x_JS1_WAIT_READ", ++ "T60x_JS1_WAIT_ISSUE", ++ "T60x_JS1_WAIT_DEPEND", ++ "T60x_JS1_WAIT_FINISH", ++ "T60x_JS2_JOBS", ++ "T60x_JS2_TASKS", ++ "T60x_JS2_ACTIVE", ++ "", ++ "T60x_JS2_WAIT_READ", ++ "T60x_JS2_WAIT_ISSUE", ++ "T60x_JS2_WAIT_DEPEND", ++ "T60x_JS2_WAIT_FINISH", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ ++ /*Tiler */ ++ "", ++ "", ++ "", ++ "T60x_TI_JOBS_PROCESSED", ++ "T60x_TI_TRIANGLES", ++ "T60x_TI_QUADS", ++ "T60x_TI_POLYGONS", ++ "T60x_TI_POINTS", ++ "T60x_TI_LINES", ++ "T60x_TI_VCACHE_HIT", ++ "T60x_TI_VCACHE_MISS", ++ "T60x_TI_FRONT_FACING", ++ "T60x_TI_BACK_FACING", ++ "T60x_TI_PRIM_VISIBLE", ++ "T60x_TI_PRIM_CULLED", ++ "T60x_TI_PRIM_CLIPPED", ++ "T60x_TI_LEVEL0", ++ "T60x_TI_LEVEL1", ++ "T60x_TI_LEVEL2", ++ "T60x_TI_LEVEL3", ++ "T60x_TI_LEVEL4", ++ "T60x_TI_LEVEL5", ++ "T60x_TI_LEVEL6", ++ "T60x_TI_LEVEL7", ++ "T60x_TI_COMMAND_1", ++ "T60x_TI_COMMAND_2", ++ "T60x_TI_COMMAND_3", ++ "T60x_TI_COMMAND_4", ++ "T60x_TI_COMMAND_4_7", ++ "T60x_TI_COMMAND_8_15", ++ "T60x_TI_COMMAND_16_63", ++ "T60x_TI_COMMAND_64", ++ "T60x_TI_COMPRESS_IN", ++ "T60x_TI_COMPRESS_OUT", ++ "T60x_TI_COMPRESS_FLUSH", ++ "T60x_TI_TIMESTAMPS", ++ "T60x_TI_PCACHE_HIT", ++ "T60x_TI_PCACHE_MISS", ++ "T60x_TI_PCACHE_LINE", ++ "T60x_TI_PCACHE_STALL", ++ "T60x_TI_WRBUF_HIT", ++ "T60x_TI_WRBUF_MISS", ++ "T60x_TI_WRBUF_LINE", ++ "T60x_TI_WRBUF_PARTIAL", ++ "T60x_TI_WRBUF_STALL", ++ "T60x_TI_ACTIVE", ++ "T60x_TI_LOADING_DESC", ++ "T60x_TI_INDEX_WAIT", ++ "T60x_TI_INDEX_RANGE_WAIT", ++ "T60x_TI_VERTEX_WAIT", ++ "T60x_TI_PCACHE_WAIT", ++ "T60x_TI_WRBUF_WAIT", ++ "T60x_TI_BUS_READ", ++ "T60x_TI_BUS_WRITE", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "T60x_TI_UTLB_STALL", ++ "T60x_TI_UTLB_REPLAY_MISS", ++ "T60x_TI_UTLB_REPLAY_FULL", ++ "T60x_TI_UTLB_NEW_MISS", ++ "T60x_TI_UTLB_HIT", ++ ++ /* Shader Core */ ++ "", ++ "", ++ "", ++ "", ++ "T60x_FRAG_ACTIVE", ++ "T60x_FRAG_PRIMITIVES", ++ "T60x_FRAG_PRIMITIVES_DROPPED", ++ "T60x_FRAG_CYCLES_DESC", ++ "T60x_FRAG_CYCLES_PLR", ++ "T60x_FRAG_CYCLES_VERT", ++ "T60x_FRAG_CYCLES_TRISETUP", ++ "T60x_FRAG_CYCLES_RAST", ++ "T60x_FRAG_THREADS", ++ "T60x_FRAG_DUMMY_THREADS", ++ "T60x_FRAG_QUADS_RAST", ++ "T60x_FRAG_QUADS_EZS_TEST", ++ "T60x_FRAG_QUADS_EZS_KILLED", ++ "T60x_FRAG_THREADS_LZS_TEST", ++ "T60x_FRAG_THREADS_LZS_KILLED", ++ "T60x_FRAG_CYCLES_NO_TILE", ++ "T60x_FRAG_NUM_TILES", ++ "T60x_FRAG_TRANS_ELIM", ++ "T60x_COMPUTE_ACTIVE", ++ "T60x_COMPUTE_TASKS", ++ "T60x_COMPUTE_THREADS", ++ "T60x_COMPUTE_CYCLES_DESC", ++ "T60x_TRIPIPE_ACTIVE", ++ "T60x_ARITH_WORDS", ++ "T60x_ARITH_CYCLES_REG", ++ "T60x_ARITH_CYCLES_L0", ++ "T60x_ARITH_FRAG_DEPEND", ++ "T60x_LS_WORDS", ++ "T60x_LS_ISSUES", ++ "T60x_LS_RESTARTS", ++ "T60x_LS_REISSUES_MISS", ++ "T60x_LS_REISSUES_VD", ++ "T60x_LS_REISSUE_ATTRIB_MISS", ++ "T60x_LS_NO_WB", ++ "T60x_TEX_WORDS", ++ "T60x_TEX_BUBBLES", ++ "T60x_TEX_WORDS_L0", ++ "T60x_TEX_WORDS_DESC", ++ "T60x_TEX_ISSUES", ++ "T60x_TEX_RECIRC_FMISS", ++ "T60x_TEX_RECIRC_DESC", ++ "T60x_TEX_RECIRC_MULTI", ++ "T60x_TEX_RECIRC_PMISS", ++ "T60x_TEX_RECIRC_CONF", ++ "T60x_LSC_READ_HITS", ++ "T60x_LSC_READ_MISSES", ++ "T60x_LSC_WRITE_HITS", ++ "T60x_LSC_WRITE_MISSES", ++ "T60x_LSC_ATOMIC_HITS", ++ "T60x_LSC_ATOMIC_MISSES", ++ "T60x_LSC_LINE_FETCHES", ++ "T60x_LSC_DIRTY_LINE", ++ "T60x_LSC_SNOOPS", ++ "T60x_AXI_TLB_STALL", ++ "T60x_AXI_TLB_MISS", ++ "T60x_AXI_TLB_TRANSACTION", ++ "T60x_LS_TLB_MISS", ++ "T60x_LS_TLB_HIT", ++ "T60x_AXI_BEATS_READ", ++ "T60x_AXI_BEATS_WRITTEN", ++ ++ /*L2 and MMU */ ++ "", ++ "", ++ "", ++ "", ++ "T60x_MMU_HIT", ++ "T60x_MMU_NEW_MISS", ++ "T60x_MMU_REPLAY_FULL", ++ "T60x_MMU_REPLAY_MISS", ++ "T60x_MMU_TABLE_WALK", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "T60x_UTLB_HIT", ++ "T60x_UTLB_NEW_MISS", ++ "T60x_UTLB_REPLAY_FULL", ++ "T60x_UTLB_REPLAY_MISS", ++ "T60x_UTLB_STALL", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "T60x_L2_EXT_WRITE_BEATS", ++ "T60x_L2_EXT_READ_BEATS", ++ "T60x_L2_ANY_LOOKUP", ++ "T60x_L2_READ_LOOKUP", ++ "T60x_L2_SREAD_LOOKUP", ++ "T60x_L2_READ_REPLAY", ++ "T60x_L2_READ_SNOOP", ++ "T60x_L2_READ_HIT", ++ "T60x_L2_CLEAN_MISS", ++ "T60x_L2_WRITE_LOOKUP", ++ "T60x_L2_SWRITE_LOOKUP", ++ "T60x_L2_WRITE_REPLAY", ++ "T60x_L2_WRITE_SNOOP", ++ "T60x_L2_WRITE_HIT", ++ "T60x_L2_EXT_READ_FULL", ++ "T60x_L2_EXT_READ_HALF", ++ "T60x_L2_EXT_WRITE_FULL", ++ "T60x_L2_EXT_WRITE_HALF", ++ "T60x_L2_EXT_READ", ++ "T60x_L2_EXT_READ_LINE", ++ "T60x_L2_EXT_WRITE", ++ "T60x_L2_EXT_WRITE_LINE", ++ "T60x_L2_EXT_WRITE_SMALL", ++ "T60x_L2_EXT_BARRIER", ++ "T60x_L2_EXT_AR_STALL", ++ "T60x_L2_EXT_R_BUF_FULL", ++ "T60x_L2_EXT_RD_BUF_FULL", ++ "T60x_L2_EXT_R_RAW", ++ "T60x_L2_EXT_W_STALL", ++ "T60x_L2_EXT_W_BUF_FULL", ++ "T60x_L2_EXT_R_W_HAZARD", ++ "T60x_L2_TAG_HAZARD", ++ "T60x_L2_SNOOP_FULL", ++ "T60x_L2_REPLAY_FULL" ++}; ++static const char * const hardware_counters_mali_t62x[] = { ++ /* Job Manager */ ++ "", ++ "", ++ "", ++ "", ++ "T62x_MESSAGES_SENT", ++ "T62x_MESSAGES_RECEIVED", ++ "T62x_GPU_ACTIVE", ++ "T62x_IRQ_ACTIVE", ++ "T62x_JS0_JOBS", ++ "T62x_JS0_TASKS", ++ "T62x_JS0_ACTIVE", ++ "", ++ "T62x_JS0_WAIT_READ", ++ "T62x_JS0_WAIT_ISSUE", ++ "T62x_JS0_WAIT_DEPEND", ++ "T62x_JS0_WAIT_FINISH", ++ "T62x_JS1_JOBS", ++ "T62x_JS1_TASKS", ++ "T62x_JS1_ACTIVE", ++ "", ++ "T62x_JS1_WAIT_READ", ++ "T62x_JS1_WAIT_ISSUE", ++ "T62x_JS1_WAIT_DEPEND", ++ "T62x_JS1_WAIT_FINISH", ++ "T62x_JS2_JOBS", ++ "T62x_JS2_TASKS", ++ "T62x_JS2_ACTIVE", ++ "", ++ "T62x_JS2_WAIT_READ", ++ "T62x_JS2_WAIT_ISSUE", ++ "T62x_JS2_WAIT_DEPEND", ++ "T62x_JS2_WAIT_FINISH", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ ++ /*Tiler */ ++ "", ++ "", ++ "", ++ "T62x_TI_JOBS_PROCESSED", ++ "T62x_TI_TRIANGLES", ++ "T62x_TI_QUADS", ++ "T62x_TI_POLYGONS", ++ "T62x_TI_POINTS", ++ "T62x_TI_LINES", ++ "T62x_TI_VCACHE_HIT", ++ "T62x_TI_VCACHE_MISS", ++ "T62x_TI_FRONT_FACING", ++ "T62x_TI_BACK_FACING", ++ "T62x_TI_PRIM_VISIBLE", ++ "T62x_TI_PRIM_CULLED", ++ "T62x_TI_PRIM_CLIPPED", ++ "T62x_TI_LEVEL0", ++ "T62x_TI_LEVEL1", ++ "T62x_TI_LEVEL2", ++ "T62x_TI_LEVEL3", ++ "T62x_TI_LEVEL4", ++ "T62x_TI_LEVEL5", ++ "T62x_TI_LEVEL6", ++ "T62x_TI_LEVEL7", ++ "T62x_TI_COMMAND_1", ++ "T62x_TI_COMMAND_2", ++ "T62x_TI_COMMAND_3", ++ "T62x_TI_COMMAND_4", ++ "T62x_TI_COMMAND_5_7", ++ "T62x_TI_COMMAND_8_15", ++ "T62x_TI_COMMAND_16_63", ++ "T62x_TI_COMMAND_64", ++ "T62x_TI_COMPRESS_IN", ++ "T62x_TI_COMPRESS_OUT", ++ "T62x_TI_COMPRESS_FLUSH", ++ "T62x_TI_TIMESTAMPS", ++ "T62x_TI_PCACHE_HIT", ++ "T62x_TI_PCACHE_MISS", ++ "T62x_TI_PCACHE_LINE", ++ "T62x_TI_PCACHE_STALL", ++ "T62x_TI_WRBUF_HIT", ++ "T62x_TI_WRBUF_MISS", ++ "T62x_TI_WRBUF_LINE", ++ "T62x_TI_WRBUF_PARTIAL", ++ "T62x_TI_WRBUF_STALL", ++ "T62x_TI_ACTIVE", ++ "T62x_TI_LOADING_DESC", ++ "T62x_TI_INDEX_WAIT", ++ "T62x_TI_INDEX_RANGE_WAIT", ++ "T62x_TI_VERTEX_WAIT", ++ "T62x_TI_PCACHE_WAIT", ++ "T62x_TI_WRBUF_WAIT", ++ "T62x_TI_BUS_READ", ++ "T62x_TI_BUS_WRITE", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "T62x_TI_UTLB_STALL", ++ "T62x_TI_UTLB_REPLAY_MISS", ++ "T62x_TI_UTLB_REPLAY_FULL", ++ "T62x_TI_UTLB_NEW_MISS", ++ "T62x_TI_UTLB_HIT", ++ ++ /* Shader Core */ ++ "", ++ "", ++ "", ++ "T62x_SHADER_CORE_ACTIVE", ++ "T62x_FRAG_ACTIVE", ++ "T62x_FRAG_PRIMITIVES", ++ "T62x_FRAG_PRIMITIVES_DROPPED", ++ "T62x_FRAG_CYCLES_DESC", ++ "T62x_FRAG_CYCLES_FPKQ_ACTIVE", ++ "T62x_FRAG_CYCLES_VERT", ++ "T62x_FRAG_CYCLES_TRISETUP", ++ "T62x_FRAG_CYCLES_EZS_ACTIVE", ++ "T62x_FRAG_THREADS", ++ "T62x_FRAG_DUMMY_THREADS", ++ "T62x_FRAG_QUADS_RAST", ++ "T62x_FRAG_QUADS_EZS_TEST", ++ "T62x_FRAG_QUADS_EZS_KILLED", ++ "T62x_FRAG_THREADS_LZS_TEST", ++ "T62x_FRAG_THREADS_LZS_KILLED", ++ "T62x_FRAG_CYCLES_NO_TILE", ++ "T62x_FRAG_NUM_TILES", ++ "T62x_FRAG_TRANS_ELIM", ++ "T62x_COMPUTE_ACTIVE", ++ "T62x_COMPUTE_TASKS", ++ "T62x_COMPUTE_THREADS", ++ "T62x_COMPUTE_CYCLES_DESC", ++ "T62x_TRIPIPE_ACTIVE", ++ "T62x_ARITH_WORDS", ++ "T62x_ARITH_CYCLES_REG", ++ "T62x_ARITH_CYCLES_L0", ++ "T62x_ARITH_FRAG_DEPEND", ++ "T62x_LS_WORDS", ++ "T62x_LS_ISSUES", ++ "T62x_LS_RESTARTS", ++ "T62x_LS_REISSUES_MISS", ++ "T62x_LS_REISSUES_VD", ++ "T62x_LS_REISSUE_ATTRIB_MISS", ++ "T62x_LS_NO_WB", ++ "T62x_TEX_WORDS", ++ "T62x_TEX_BUBBLES", ++ "T62x_TEX_WORDS_L0", ++ "T62x_TEX_WORDS_DESC", ++ "T62x_TEX_ISSUES", ++ "T62x_TEX_RECIRC_FMISS", ++ "T62x_TEX_RECIRC_DESC", ++ "T62x_TEX_RECIRC_MULTI", ++ "T62x_TEX_RECIRC_PMISS", ++ "T62x_TEX_RECIRC_CONF", ++ "T62x_LSC_READ_HITS", ++ "T62x_LSC_READ_MISSES", ++ "T62x_LSC_WRITE_HITS", ++ "T62x_LSC_WRITE_MISSES", ++ "T62x_LSC_ATOMIC_HITS", ++ "T62x_LSC_ATOMIC_MISSES", ++ "T62x_LSC_LINE_FETCHES", ++ "T62x_LSC_DIRTY_LINE", ++ "T62x_LSC_SNOOPS", ++ "T62x_AXI_TLB_STALL", ++ "T62x_AXI_TLB_MISS", ++ "T62x_AXI_TLB_TRANSACTION", ++ "T62x_LS_TLB_MISS", ++ "T62x_LS_TLB_HIT", ++ "T62x_AXI_BEATS_READ", ++ "T62x_AXI_BEATS_WRITTEN", ++ ++ /*L2 and MMU */ ++ "", ++ "", ++ "", ++ "", ++ "T62x_MMU_HIT", ++ "T62x_MMU_NEW_MISS", ++ "T62x_MMU_REPLAY_FULL", ++ "T62x_MMU_REPLAY_MISS", ++ "T62x_MMU_TABLE_WALK", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "T62x_UTLB_HIT", ++ "T62x_UTLB_NEW_MISS", ++ "T62x_UTLB_REPLAY_FULL", ++ "T62x_UTLB_REPLAY_MISS", ++ "T62x_UTLB_STALL", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "T62x_L2_EXT_WRITE_BEATS", ++ "T62x_L2_EXT_READ_BEATS", ++ "T62x_L2_ANY_LOOKUP", ++ "T62x_L2_READ_LOOKUP", ++ "T62x_L2_SREAD_LOOKUP", ++ "T62x_L2_READ_REPLAY", ++ "T62x_L2_READ_SNOOP", ++ "T62x_L2_READ_HIT", ++ "T62x_L2_CLEAN_MISS", ++ "T62x_L2_WRITE_LOOKUP", ++ "T62x_L2_SWRITE_LOOKUP", ++ "T62x_L2_WRITE_REPLAY", ++ "T62x_L2_WRITE_SNOOP", ++ "T62x_L2_WRITE_HIT", ++ "T62x_L2_EXT_READ_FULL", ++ "T62x_L2_EXT_READ_HALF", ++ "T62x_L2_EXT_WRITE_FULL", ++ "T62x_L2_EXT_WRITE_HALF", ++ "T62x_L2_EXT_READ", ++ "T62x_L2_EXT_READ_LINE", ++ "T62x_L2_EXT_WRITE", ++ "T62x_L2_EXT_WRITE_LINE", ++ "T62x_L2_EXT_WRITE_SMALL", ++ "T62x_L2_EXT_BARRIER", ++ "T62x_L2_EXT_AR_STALL", ++ "T62x_L2_EXT_R_BUF_FULL", ++ "T62x_L2_EXT_RD_BUF_FULL", ++ "T62x_L2_EXT_R_RAW", ++ "T62x_L2_EXT_W_STALL", ++ "T62x_L2_EXT_W_BUF_FULL", ++ "T62x_L2_EXT_R_W_HAZARD", ++ "T62x_L2_TAG_HAZARD", ++ "T62x_L2_SNOOP_FULL", ++ "T62x_L2_REPLAY_FULL" ++}; ++ ++static const char * const hardware_counters_mali_t72x[] = { ++ /* Job Manager */ ++ "", ++ "", ++ "", ++ "", ++ "T72x_GPU_ACTIVE", ++ "T72x_IRQ_ACTIVE", ++ "T72x_JS0_JOBS", ++ "T72x_JS0_TASKS", ++ "T72x_JS0_ACTIVE", ++ "T72x_JS1_JOBS", ++ "T72x_JS1_TASKS", ++ "T72x_JS1_ACTIVE", ++ "T72x_JS2_JOBS", ++ "T72x_JS2_TASKS", ++ "T72x_JS2_ACTIVE", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ ++ /*Tiler */ ++ "", ++ "", ++ "", ++ "T72x_TI_JOBS_PROCESSED", ++ "T72x_TI_TRIANGLES", ++ "T72x_TI_QUADS", ++ "T72x_TI_POLYGONS", ++ "T72x_TI_POINTS", ++ "T72x_TI_LINES", ++ "T72x_TI_FRONT_FACING", ++ "T72x_TI_BACK_FACING", ++ "T72x_TI_PRIM_VISIBLE", ++ "T72x_TI_PRIM_CULLED", ++ "T72x_TI_PRIM_CLIPPED", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "T72x_TI_ACTIVE", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ ++ /* Shader Core */ ++ "", ++ "", ++ "", ++ "", ++ "T72x_FRAG_ACTIVE", ++ "T72x_FRAG_PRIMITIVES", ++ "T72x_FRAG_PRIMITIVES_DROPPED", ++ "T72x_FRAG_THREADS", ++ "T72x_FRAG_DUMMY_THREADS", ++ "T72x_FRAG_QUADS_RAST", ++ "T72x_FRAG_QUADS_EZS_TEST", ++ "T72x_FRAG_QUADS_EZS_KILLED", ++ "T72x_FRAG_THREADS_LZS_TEST", ++ "T72x_FRAG_THREADS_LZS_KILLED", ++ "T72x_FRAG_CYCLES_NO_TILE", ++ "T72x_FRAG_NUM_TILES", ++ "T72x_FRAG_TRANS_ELIM", ++ "T72x_COMPUTE_ACTIVE", ++ "T72x_COMPUTE_TASKS", ++ "T72x_COMPUTE_THREADS", ++ "T72x_TRIPIPE_ACTIVE", ++ "T72x_ARITH_WORDS", ++ "T72x_ARITH_CYCLES_REG", ++ "T72x_LS_WORDS", ++ "T72x_LS_ISSUES", ++ "T72x_LS_RESTARTS", ++ "T72x_LS_REISSUES_MISS", ++ "T72x_TEX_WORDS", ++ "T72x_TEX_BUBBLES", ++ "T72x_TEX_ISSUES", ++ "T72x_LSC_READ_HITS", ++ "T72x_LSC_READ_MISSES", ++ "T72x_LSC_WRITE_HITS", ++ "T72x_LSC_WRITE_MISSES", ++ "T72x_LSC_ATOMIC_HITS", ++ "T72x_LSC_ATOMIC_MISSES", ++ "T72x_LSC_LINE_FETCHES", ++ "T72x_LSC_DIRTY_LINE", ++ "T72x_LSC_SNOOPS", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ ++ /*L2 and MMU */ ++ "", ++ "", ++ "", ++ "", ++ "T72x_L2_EXT_WRITE_BEAT", ++ "T72x_L2_EXT_READ_BEAT", ++ "T72x_L2_READ_SNOOP", ++ "T72x_L2_READ_HIT", ++ "T72x_L2_WRITE_SNOOP", ++ "T72x_L2_WRITE_HIT", ++ "T72x_L2_EXT_WRITE_SMALL", ++ "T72x_L2_EXT_BARRIER", ++ "T72x_L2_EXT_AR_STALL", ++ "T72x_L2_EXT_W_STALL", ++ "T72x_L2_SNOOP_FULL", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "" ++}; ++ ++static const char * const hardware_counters_mali_t76x[] = { ++ /* Job Manager */ ++ "", ++ "", ++ "", ++ "", ++ "T76x_MESSAGES_SENT", ++ "T76x_MESSAGES_RECEIVED", ++ "T76x_GPU_ACTIVE", ++ "T76x_IRQ_ACTIVE", ++ "T76x_JS0_JOBS", ++ "T76x_JS0_TASKS", ++ "T76x_JS0_ACTIVE", ++ "", ++ "T76x_JS0_WAIT_READ", ++ "T76x_JS0_WAIT_ISSUE", ++ "T76x_JS0_WAIT_DEPEND", ++ "T76x_JS0_WAIT_FINISH", ++ "T76x_JS1_JOBS", ++ "T76x_JS1_TASKS", ++ "T76x_JS1_ACTIVE", ++ "", ++ "T76x_JS1_WAIT_READ", ++ "T76x_JS1_WAIT_ISSUE", ++ "T76x_JS1_WAIT_DEPEND", ++ "T76x_JS1_WAIT_FINISH", ++ "T76x_JS2_JOBS", ++ "T76x_JS2_TASKS", ++ "T76x_JS2_ACTIVE", ++ "", ++ "T76x_JS2_WAIT_READ", ++ "T76x_JS2_WAIT_ISSUE", ++ "T76x_JS2_WAIT_DEPEND", ++ "T76x_JS2_WAIT_FINISH", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ ++ /*Tiler */ ++ "", ++ "", ++ "", ++ "T76x_TI_JOBS_PROCESSED", ++ "T76x_TI_TRIANGLES", ++ "T76x_TI_QUADS", ++ "T76x_TI_POLYGONS", ++ "T76x_TI_POINTS", ++ "T76x_TI_LINES", ++ "T76x_TI_VCACHE_HIT", ++ "T76x_TI_VCACHE_MISS", ++ "T76x_TI_FRONT_FACING", ++ "T76x_TI_BACK_FACING", ++ "T76x_TI_PRIM_VISIBLE", ++ "T76x_TI_PRIM_CULLED", ++ "T76x_TI_PRIM_CLIPPED", ++ "T76x_TI_LEVEL0", ++ "T76x_TI_LEVEL1", ++ "T76x_TI_LEVEL2", ++ "T76x_TI_LEVEL3", ++ "T76x_TI_LEVEL4", ++ "T76x_TI_LEVEL5", ++ "T76x_TI_LEVEL6", ++ "T76x_TI_LEVEL7", ++ "T76x_TI_COMMAND_1", ++ "T76x_TI_COMMAND_2", ++ "T76x_TI_COMMAND_3", ++ "T76x_TI_COMMAND_4", ++ "T76x_TI_COMMAND_5_7", ++ "T76x_TI_COMMAND_8_15", ++ "T76x_TI_COMMAND_16_63", ++ "T76x_TI_COMMAND_64", ++ "T76x_TI_COMPRESS_IN", ++ "T76x_TI_COMPRESS_OUT", ++ "T76x_TI_COMPRESS_FLUSH", ++ "T76x_TI_TIMESTAMPS", ++ "T76x_TI_PCACHE_HIT", ++ "T76x_TI_PCACHE_MISS", ++ "T76x_TI_PCACHE_LINE", ++ "T76x_TI_PCACHE_STALL", ++ "T76x_TI_WRBUF_HIT", ++ "T76x_TI_WRBUF_MISS", ++ "T76x_TI_WRBUF_LINE", ++ "T76x_TI_WRBUF_PARTIAL", ++ "T76x_TI_WRBUF_STALL", ++ "T76x_TI_ACTIVE", ++ "T76x_TI_LOADING_DESC", ++ "T76x_TI_INDEX_WAIT", ++ "T76x_TI_INDEX_RANGE_WAIT", ++ "T76x_TI_VERTEX_WAIT", ++ "T76x_TI_PCACHE_WAIT", ++ "T76x_TI_WRBUF_WAIT", ++ "T76x_TI_BUS_READ", ++ "T76x_TI_BUS_WRITE", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "T76x_TI_UTLB_HIT", ++ "T76x_TI_UTLB_NEW_MISS", ++ "T76x_TI_UTLB_REPLAY_FULL", ++ "T76x_TI_UTLB_REPLAY_MISS", ++ "T76x_TI_UTLB_STALL", ++ ++ /* Shader Core */ ++ "", ++ "", ++ "", ++ "", ++ "T76x_FRAG_ACTIVE", ++ "T76x_FRAG_PRIMITIVES", ++ "T76x_FRAG_PRIMITIVES_DROPPED", ++ "T76x_FRAG_CYCLES_DESC", ++ "T76x_FRAG_CYCLES_FPKQ_ACTIVE", ++ "T76x_FRAG_CYCLES_VERT", ++ "T76x_FRAG_CYCLES_TRISETUP", ++ "T76x_FRAG_CYCLES_EZS_ACTIVE", ++ "T76x_FRAG_THREADS", ++ "T76x_FRAG_DUMMY_THREADS", ++ "T76x_FRAG_QUADS_RAST", ++ "T76x_FRAG_QUADS_EZS_TEST", ++ "T76x_FRAG_QUADS_EZS_KILLED", ++ "T76x_FRAG_THREADS_LZS_TEST", ++ "T76x_FRAG_THREADS_LZS_KILLED", ++ "T76x_FRAG_CYCLES_NO_TILE", ++ "T76x_FRAG_NUM_TILES", ++ "T76x_FRAG_TRANS_ELIM", ++ "T76x_COMPUTE_ACTIVE", ++ "T76x_COMPUTE_TASKS", ++ "T76x_COMPUTE_THREADS", ++ "T76x_COMPUTE_CYCLES_DESC", ++ "T76x_TRIPIPE_ACTIVE", ++ "T76x_ARITH_WORDS", ++ "T76x_ARITH_CYCLES_REG", ++ "T76x_ARITH_CYCLES_L0", ++ "T76x_ARITH_FRAG_DEPEND", ++ "T76x_LS_WORDS", ++ "T76x_LS_ISSUES", ++ "T76x_LS_REISSUE_ATTR", ++ "T76x_LS_REISSUES_VARY", ++ "T76x_LS_VARY_RV_MISS", ++ "T76x_LS_VARY_RV_HIT", ++ "T76x_LS_NO_UNPARK", ++ "T76x_TEX_WORDS", ++ "T76x_TEX_BUBBLES", ++ "T76x_TEX_WORDS_L0", ++ "T76x_TEX_WORDS_DESC", ++ "T76x_TEX_ISSUES", ++ "T76x_TEX_RECIRC_FMISS", ++ "T76x_TEX_RECIRC_DESC", ++ "T76x_TEX_RECIRC_MULTI", ++ "T76x_TEX_RECIRC_PMISS", ++ "T76x_TEX_RECIRC_CONF", ++ "T76x_LSC_READ_HITS", ++ "T76x_LSC_READ_OP", ++ "T76x_LSC_WRITE_HITS", ++ "T76x_LSC_WRITE_OP", ++ "T76x_LSC_ATOMIC_HITS", ++ "T76x_LSC_ATOMIC_OP", ++ "T76x_LSC_LINE_FETCHES", ++ "T76x_LSC_DIRTY_LINE", ++ "T76x_LSC_SNOOPS", ++ "T76x_AXI_TLB_STALL", ++ "T76x_AXI_TLB_MISS", ++ "T76x_AXI_TLB_TRANSACTION", ++ "T76x_LS_TLB_MISS", ++ "T76x_LS_TLB_HIT", ++ "T76x_AXI_BEATS_READ", ++ "T76x_AXI_BEATS_WRITTEN", ++ ++ /*L2 and MMU */ ++ "", ++ "", ++ "", ++ "", ++ "T76x_MMU_HIT", ++ "T76x_MMU_NEW_MISS", ++ "T76x_MMU_REPLAY_FULL", ++ "T76x_MMU_REPLAY_MISS", ++ "T76x_MMU_TABLE_WALK", ++ "T76x_MMU_REQUESTS", ++ "", ++ "", ++ "T76x_UTLB_HIT", ++ "T76x_UTLB_NEW_MISS", ++ "T76x_UTLB_REPLAY_FULL", ++ "T76x_UTLB_REPLAY_MISS", ++ "T76x_UTLB_STALL", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "T76x_L2_EXT_WRITE_BEATS", ++ "T76x_L2_EXT_READ_BEATS", ++ "T76x_L2_ANY_LOOKUP", ++ "T76x_L2_READ_LOOKUP", ++ "T76x_L2_SREAD_LOOKUP", ++ "T76x_L2_READ_REPLAY", ++ "T76x_L2_READ_SNOOP", ++ "T76x_L2_READ_HIT", ++ "T76x_L2_CLEAN_MISS", ++ "T76x_L2_WRITE_LOOKUP", ++ "T76x_L2_SWRITE_LOOKUP", ++ "T76x_L2_WRITE_REPLAY", ++ "T76x_L2_WRITE_SNOOP", ++ "T76x_L2_WRITE_HIT", ++ "T76x_L2_EXT_READ_FULL", ++ "", ++ "T76x_L2_EXT_WRITE_FULL", ++ "T76x_L2_EXT_R_W_HAZARD", ++ "T76x_L2_EXT_READ", ++ "T76x_L2_EXT_READ_LINE", ++ "T76x_L2_EXT_WRITE", ++ "T76x_L2_EXT_WRITE_LINE", ++ "T76x_L2_EXT_WRITE_SMALL", ++ "T76x_L2_EXT_BARRIER", ++ "T76x_L2_EXT_AR_STALL", ++ "T76x_L2_EXT_R_BUF_FULL", ++ "T76x_L2_EXT_RD_BUF_FULL", ++ "T76x_L2_EXT_R_RAW", ++ "T76x_L2_EXT_W_STALL", ++ "T76x_L2_EXT_W_BUF_FULL", ++ "T76x_L2_EXT_R_BUF_FULL", ++ "T76x_L2_TAG_HAZARD", ++ "T76x_L2_SNOOP_FULL", ++ "T76x_L2_REPLAY_FULL" ++}; ++ ++static const char * const hardware_counters_mali_t82x[] = { ++ /* Job Manager */ ++ "", ++ "", ++ "", ++ "", ++ "T82x_MESSAGES_SENT", ++ "T82x_MESSAGES_RECEIVED", ++ "T82x_GPU_ACTIVE", ++ "T82x_IRQ_ACTIVE", ++ "T82x_JS0_JOBS", ++ "T82x_JS0_TASKS", ++ "T82x_JS0_ACTIVE", ++ "", ++ "T82x_JS0_WAIT_READ", ++ "T82x_JS0_WAIT_ISSUE", ++ "T82x_JS0_WAIT_DEPEND", ++ "T82x_JS0_WAIT_FINISH", ++ "T82x_JS1_JOBS", ++ "T82x_JS1_TASKS", ++ "T82x_JS1_ACTIVE", ++ "", ++ "T82x_JS1_WAIT_READ", ++ "T82x_JS1_WAIT_ISSUE", ++ "T82x_JS1_WAIT_DEPEND", ++ "T82x_JS1_WAIT_FINISH", ++ "T82x_JS2_JOBS", ++ "T82x_JS2_TASKS", ++ "T82x_JS2_ACTIVE", ++ "", ++ "T82x_JS2_WAIT_READ", ++ "T82x_JS2_WAIT_ISSUE", ++ "T82x_JS2_WAIT_DEPEND", ++ "T82x_JS2_WAIT_FINISH", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ ++ /*Tiler */ ++ "", ++ "", ++ "", ++ "T82x_TI_JOBS_PROCESSED", ++ "T82x_TI_TRIANGLES", ++ "T82x_TI_QUADS", ++ "T82x_TI_POLYGONS", ++ "T82x_TI_POINTS", ++ "T82x_TI_LINES", ++ "T82x_TI_FRONT_FACING", ++ "T82x_TI_BACK_FACING", ++ "T82x_TI_PRIM_VISIBLE", ++ "T82x_TI_PRIM_CULLED", ++ "T82x_TI_PRIM_CLIPPED", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "T82x_TI_ACTIVE", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ ++ /* Shader Core */ ++ "", ++ "", ++ "", ++ "", ++ "T82x_FRAG_ACTIVE", ++ "T82x_FRAG_PRIMITIVES", ++ "T82x_FRAG_PRIMITIVES_DROPPED", ++ "T82x_FRAG_CYCLES_DESC", ++ "T82x_FRAG_CYCLES_FPKQ_ACTIVE", ++ "T82x_FRAG_CYCLES_VERT", ++ "T82x_FRAG_CYCLES_TRISETUP", ++ "T82x_FRAG_CYCLES_EZS_ACTIVE", ++ "T82x_FRAG_THREADS", ++ "T82x_FRAG_DUMMY_THREADS", ++ "T82x_FRAG_QUADS_RAST", ++ "T82x_FRAG_QUADS_EZS_TEST", ++ "T82x_FRAG_QUADS_EZS_KILLED", ++ "T82x_FRAG_THREADS_LZS_TEST", ++ "T82x_FRAG_THREADS_LZS_KILLED", ++ "T82x_FRAG_CYCLES_NO_TILE", ++ "T82x_FRAG_NUM_TILES", ++ "T82x_FRAG_TRANS_ELIM", ++ "T82x_COMPUTE_ACTIVE", ++ "T82x_COMPUTE_TASKS", ++ "T82x_COMPUTE_THREADS", ++ "T82x_COMPUTE_CYCLES_DESC", ++ "T82x_TRIPIPE_ACTIVE", ++ "T82x_ARITH_WORDS", ++ "T82x_ARITH_CYCLES_REG", ++ "T82x_ARITH_CYCLES_L0", ++ "T82x_ARITH_FRAG_DEPEND", ++ "T82x_LS_WORDS", ++ "T82x_LS_ISSUES", ++ "T82x_LS_REISSUE_ATTR", ++ "T82x_LS_REISSUES_VARY", ++ "T82x_LS_VARY_RV_MISS", ++ "T82x_LS_VARY_RV_HIT", ++ "T82x_LS_NO_UNPARK", ++ "T82x_TEX_WORDS", ++ "T82x_TEX_BUBBLES", ++ "T82x_TEX_WORDS_L0", ++ "T82x_TEX_WORDS_DESC", ++ "T82x_TEX_ISSUES", ++ "T82x_TEX_RECIRC_FMISS", ++ "T82x_TEX_RECIRC_DESC", ++ "T82x_TEX_RECIRC_MULTI", ++ "T82x_TEX_RECIRC_PMISS", ++ "T82x_TEX_RECIRC_CONF", ++ "T82x_LSC_READ_HITS", ++ "T82x_LSC_READ_OP", ++ "T82x_LSC_WRITE_HITS", ++ "T82x_LSC_WRITE_OP", ++ "T82x_LSC_ATOMIC_HITS", ++ "T82x_LSC_ATOMIC_OP", ++ "T82x_LSC_LINE_FETCHES", ++ "T82x_LSC_DIRTY_LINE", ++ "T82x_LSC_SNOOPS", ++ "T82x_AXI_TLB_STALL", ++ "T82x_AXI_TLB_MISS", ++ "T82x_AXI_TLB_TRANSACTION", ++ "T82x_LS_TLB_MISS", ++ "T82x_LS_TLB_HIT", ++ "T82x_AXI_BEATS_READ", ++ "T82x_AXI_BEATS_WRITTEN", ++ ++ /*L2 and MMU */ ++ "", ++ "", ++ "", ++ "", ++ "T82x_MMU_HIT", ++ "T82x_MMU_NEW_MISS", ++ "T82x_MMU_REPLAY_FULL", ++ "T82x_MMU_REPLAY_MISS", ++ "T82x_MMU_TABLE_WALK", ++ "T82x_MMU_REQUESTS", ++ "", ++ "", ++ "T82x_UTLB_HIT", ++ "T82x_UTLB_NEW_MISS", ++ "T82x_UTLB_REPLAY_FULL", ++ "T82x_UTLB_REPLAY_MISS", ++ "T82x_UTLB_STALL", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "T82x_L2_EXT_WRITE_BEATS", ++ "T82x_L2_EXT_READ_BEATS", ++ "T82x_L2_ANY_LOOKUP", ++ "T82x_L2_READ_LOOKUP", ++ "T82x_L2_SREAD_LOOKUP", ++ "T82x_L2_READ_REPLAY", ++ "T82x_L2_READ_SNOOP", ++ "T82x_L2_READ_HIT", ++ "T82x_L2_CLEAN_MISS", ++ "T82x_L2_WRITE_LOOKUP", ++ "T82x_L2_SWRITE_LOOKUP", ++ "T82x_L2_WRITE_REPLAY", ++ "T82x_L2_WRITE_SNOOP", ++ "T82x_L2_WRITE_HIT", ++ "T82x_L2_EXT_READ_FULL", ++ "", ++ "T82x_L2_EXT_WRITE_FULL", ++ "T82x_L2_EXT_R_W_HAZARD", ++ "T82x_L2_EXT_READ", ++ "T82x_L2_EXT_READ_LINE", ++ "T82x_L2_EXT_WRITE", ++ "T82x_L2_EXT_WRITE_LINE", ++ "T82x_L2_EXT_WRITE_SMALL", ++ "T82x_L2_EXT_BARRIER", ++ "T82x_L2_EXT_AR_STALL", ++ "T82x_L2_EXT_R_BUF_FULL", ++ "T82x_L2_EXT_RD_BUF_FULL", ++ "T82x_L2_EXT_R_RAW", ++ "T82x_L2_EXT_W_STALL", ++ "T82x_L2_EXT_W_BUF_FULL", ++ "T82x_L2_EXT_R_BUF_FULL", ++ "T82x_L2_TAG_HAZARD", ++ "T82x_L2_SNOOP_FULL", ++ "T82x_L2_REPLAY_FULL" ++}; ++ ++static const char * const hardware_counters_mali_t83x[] = { ++ /* Job Manager */ ++ "", ++ "", ++ "", ++ "", ++ "T83x_MESSAGES_SENT", ++ "T83x_MESSAGES_RECEIVED", ++ "T83x_GPU_ACTIVE", ++ "T83x_IRQ_ACTIVE", ++ "T83x_JS0_JOBS", ++ "T83x_JS0_TASKS", ++ "T83x_JS0_ACTIVE", ++ "", ++ "T83x_JS0_WAIT_READ", ++ "T83x_JS0_WAIT_ISSUE", ++ "T83x_JS0_WAIT_DEPEND", ++ "T83x_JS0_WAIT_FINISH", ++ "T83x_JS1_JOBS", ++ "T83x_JS1_TASKS", ++ "T83x_JS1_ACTIVE", ++ "", ++ "T83x_JS1_WAIT_READ", ++ "T83x_JS1_WAIT_ISSUE", ++ "T83x_JS1_WAIT_DEPEND", ++ "T83x_JS1_WAIT_FINISH", ++ "T83x_JS2_JOBS", ++ "T83x_JS2_TASKS", ++ "T83x_JS2_ACTIVE", ++ "", ++ "T83x_JS2_WAIT_READ", ++ "T83x_JS2_WAIT_ISSUE", ++ "T83x_JS2_WAIT_DEPEND", ++ "T83x_JS2_WAIT_FINISH", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ ++ /*Tiler */ ++ "", ++ "", ++ "", ++ "T83x_TI_JOBS_PROCESSED", ++ "T83x_TI_TRIANGLES", ++ "T83x_TI_QUADS", ++ "T83x_TI_POLYGONS", ++ "T83x_TI_POINTS", ++ "T83x_TI_LINES", ++ "T83x_TI_FRONT_FACING", ++ "T83x_TI_BACK_FACING", ++ "T83x_TI_PRIM_VISIBLE", ++ "T83x_TI_PRIM_CULLED", ++ "T83x_TI_PRIM_CLIPPED", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "T83x_TI_ACTIVE", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ ++ /* Shader Core */ ++ "", ++ "", ++ "", ++ "", ++ "T83x_FRAG_ACTIVE", ++ "T83x_FRAG_PRIMITIVES", ++ "T83x_FRAG_PRIMITIVES_DROPPED", ++ "T83x_FRAG_CYCLES_DESC", ++ "T83x_FRAG_CYCLES_FPKQ_ACTIVE", ++ "T83x_FRAG_CYCLES_VERT", ++ "T83x_FRAG_CYCLES_TRISETUP", ++ "T83x_FRAG_CYCLES_EZS_ACTIVE", ++ "T83x_FRAG_THREADS", ++ "T83x_FRAG_DUMMY_THREADS", ++ "T83x_FRAG_QUADS_RAST", ++ "T83x_FRAG_QUADS_EZS_TEST", ++ "T83x_FRAG_QUADS_EZS_KILLED", ++ "T83x_FRAG_THREADS_LZS_TEST", ++ "T83x_FRAG_THREADS_LZS_KILLED", ++ "T83x_FRAG_CYCLES_NO_TILE", ++ "T83x_FRAG_NUM_TILES", ++ "T83x_FRAG_TRANS_ELIM", ++ "T83x_COMPUTE_ACTIVE", ++ "T83x_COMPUTE_TASKS", ++ "T83x_COMPUTE_THREADS", ++ "T83x_COMPUTE_CYCLES_DESC", ++ "T83x_TRIPIPE_ACTIVE", ++ "T83x_ARITH_WORDS", ++ "T83x_ARITH_CYCLES_REG", ++ "T83x_ARITH_CYCLES_L0", ++ "T83x_ARITH_FRAG_DEPEND", ++ "T83x_LS_WORDS", ++ "T83x_LS_ISSUES", ++ "T83x_LS_REISSUE_ATTR", ++ "T83x_LS_REISSUES_VARY", ++ "T83x_LS_VARY_RV_MISS", ++ "T83x_LS_VARY_RV_HIT", ++ "T83x_LS_NO_UNPARK", ++ "T83x_TEX_WORDS", ++ "T83x_TEX_BUBBLES", ++ "T83x_TEX_WORDS_L0", ++ "T83x_TEX_WORDS_DESC", ++ "T83x_TEX_ISSUES", ++ "T83x_TEX_RECIRC_FMISS", ++ "T83x_TEX_RECIRC_DESC", ++ "T83x_TEX_RECIRC_MULTI", ++ "T83x_TEX_RECIRC_PMISS", ++ "T83x_TEX_RECIRC_CONF", ++ "T83x_LSC_READ_HITS", ++ "T83x_LSC_READ_OP", ++ "T83x_LSC_WRITE_HITS", ++ "T83x_LSC_WRITE_OP", ++ "T83x_LSC_ATOMIC_HITS", ++ "T83x_LSC_ATOMIC_OP", ++ "T83x_LSC_LINE_FETCHES", ++ "T83x_LSC_DIRTY_LINE", ++ "T83x_LSC_SNOOPS", ++ "T83x_AXI_TLB_STALL", ++ "T83x_AXI_TLB_MISS", ++ "T83x_AXI_TLB_TRANSACTION", ++ "T83x_LS_TLB_MISS", ++ "T83x_LS_TLB_HIT", ++ "T83x_AXI_BEATS_READ", ++ "T83x_AXI_BEATS_WRITTEN", ++ ++ /*L2 and MMU */ ++ "", ++ "", ++ "", ++ "", ++ "T83x_MMU_HIT", ++ "T83x_MMU_NEW_MISS", ++ "T83x_MMU_REPLAY_FULL", ++ "T83x_MMU_REPLAY_MISS", ++ "T83x_MMU_TABLE_WALK", ++ "T83x_MMU_REQUESTS", ++ "", ++ "", ++ "T83x_UTLB_HIT", ++ "T83x_UTLB_NEW_MISS", ++ "T83x_UTLB_REPLAY_FULL", ++ "T83x_UTLB_REPLAY_MISS", ++ "T83x_UTLB_STALL", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "T83x_L2_EXT_WRITE_BEATS", ++ "T83x_L2_EXT_READ_BEATS", ++ "T83x_L2_ANY_LOOKUP", ++ "T83x_L2_READ_LOOKUP", ++ "T83x_L2_SREAD_LOOKUP", ++ "T83x_L2_READ_REPLAY", ++ "T83x_L2_READ_SNOOP", ++ "T83x_L2_READ_HIT", ++ "T83x_L2_CLEAN_MISS", ++ "T83x_L2_WRITE_LOOKUP", ++ "T83x_L2_SWRITE_LOOKUP", ++ "T83x_L2_WRITE_REPLAY", ++ "T83x_L2_WRITE_SNOOP", ++ "T83x_L2_WRITE_HIT", ++ "T83x_L2_EXT_READ_FULL", ++ "", ++ "T83x_L2_EXT_WRITE_FULL", ++ "T83x_L2_EXT_R_W_HAZARD", ++ "T83x_L2_EXT_READ", ++ "T83x_L2_EXT_READ_LINE", ++ "T83x_L2_EXT_WRITE", ++ "T83x_L2_EXT_WRITE_LINE", ++ "T83x_L2_EXT_WRITE_SMALL", ++ "T83x_L2_EXT_BARRIER", ++ "T83x_L2_EXT_AR_STALL", ++ "T83x_L2_EXT_R_BUF_FULL", ++ "T83x_L2_EXT_RD_BUF_FULL", ++ "T83x_L2_EXT_R_RAW", ++ "T83x_L2_EXT_W_STALL", ++ "T83x_L2_EXT_W_BUF_FULL", ++ "T83x_L2_EXT_R_BUF_FULL", ++ "T83x_L2_TAG_HAZARD", ++ "T83x_L2_SNOOP_FULL", ++ "T83x_L2_REPLAY_FULL" ++}; ++ ++static const char * const hardware_counters_mali_t86x[] = { ++ /* Job Manager */ ++ "", ++ "", ++ "", ++ "", ++ "T86x_MESSAGES_SENT", ++ "T86x_MESSAGES_RECEIVED", ++ "T86x_GPU_ACTIVE", ++ "T86x_IRQ_ACTIVE", ++ "T86x_JS0_JOBS", ++ "T86x_JS0_TASKS", ++ "T86x_JS0_ACTIVE", ++ "", ++ "T86x_JS0_WAIT_READ", ++ "T86x_JS0_WAIT_ISSUE", ++ "T86x_JS0_WAIT_DEPEND", ++ "T86x_JS0_WAIT_FINISH", ++ "T86x_JS1_JOBS", ++ "T86x_JS1_TASKS", ++ "T86x_JS1_ACTIVE", ++ "", ++ "T86x_JS1_WAIT_READ", ++ "T86x_JS1_WAIT_ISSUE", ++ "T86x_JS1_WAIT_DEPEND", ++ "T86x_JS1_WAIT_FINISH", ++ "T86x_JS2_JOBS", ++ "T86x_JS2_TASKS", ++ "T86x_JS2_ACTIVE", ++ "", ++ "T86x_JS2_WAIT_READ", ++ "T86x_JS2_WAIT_ISSUE", ++ "T86x_JS2_WAIT_DEPEND", ++ "T86x_JS2_WAIT_FINISH", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ ++ /*Tiler */ ++ "", ++ "", ++ "", ++ "T86x_TI_JOBS_PROCESSED", ++ "T86x_TI_TRIANGLES", ++ "T86x_TI_QUADS", ++ "T86x_TI_POLYGONS", ++ "T86x_TI_POINTS", ++ "T86x_TI_LINES", ++ "T86x_TI_VCACHE_HIT", ++ "T86x_TI_VCACHE_MISS", ++ "T86x_TI_FRONT_FACING", ++ "T86x_TI_BACK_FACING", ++ "T86x_TI_PRIM_VISIBLE", ++ "T86x_TI_PRIM_CULLED", ++ "T86x_TI_PRIM_CLIPPED", ++ "T86x_TI_LEVEL0", ++ "T86x_TI_LEVEL1", ++ "T86x_TI_LEVEL2", ++ "T86x_TI_LEVEL3", ++ "T86x_TI_LEVEL4", ++ "T86x_TI_LEVEL5", ++ "T86x_TI_LEVEL6", ++ "T86x_TI_LEVEL7", ++ "T86x_TI_COMMAND_1", ++ "T86x_TI_COMMAND_2", ++ "T86x_TI_COMMAND_3", ++ "T86x_TI_COMMAND_4", ++ "T86x_TI_COMMAND_5_7", ++ "T86x_TI_COMMAND_8_15", ++ "T86x_TI_COMMAND_16_63", ++ "T86x_TI_COMMAND_64", ++ "T86x_TI_COMPRESS_IN", ++ "T86x_TI_COMPRESS_OUT", ++ "T86x_TI_COMPRESS_FLUSH", ++ "T86x_TI_TIMESTAMPS", ++ "T86x_TI_PCACHE_HIT", ++ "T86x_TI_PCACHE_MISS", ++ "T86x_TI_PCACHE_LINE", ++ "T86x_TI_PCACHE_STALL", ++ "T86x_TI_WRBUF_HIT", ++ "T86x_TI_WRBUF_MISS", ++ "T86x_TI_WRBUF_LINE", ++ "T86x_TI_WRBUF_PARTIAL", ++ "T86x_TI_WRBUF_STALL", ++ "T86x_TI_ACTIVE", ++ "T86x_TI_LOADING_DESC", ++ "T86x_TI_INDEX_WAIT", ++ "T86x_TI_INDEX_RANGE_WAIT", ++ "T86x_TI_VERTEX_WAIT", ++ "T86x_TI_PCACHE_WAIT", ++ "T86x_TI_WRBUF_WAIT", ++ "T86x_TI_BUS_READ", ++ "T86x_TI_BUS_WRITE", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "T86x_TI_UTLB_HIT", ++ "T86x_TI_UTLB_NEW_MISS", ++ "T86x_TI_UTLB_REPLAY_FULL", ++ "T86x_TI_UTLB_REPLAY_MISS", ++ "T86x_TI_UTLB_STALL", ++ ++ /* Shader Core */ ++ "", ++ "", ++ "", ++ "", ++ "T86x_FRAG_ACTIVE", ++ "T86x_FRAG_PRIMITIVES", ++ "T86x_FRAG_PRIMITIVES_DROPPED", ++ "T86x_FRAG_CYCLES_DESC", ++ "T86x_FRAG_CYCLES_FPKQ_ACTIVE", ++ "T86x_FRAG_CYCLES_VERT", ++ "T86x_FRAG_CYCLES_TRISETUP", ++ "T86x_FRAG_CYCLES_EZS_ACTIVE", ++ "T86x_FRAG_THREADS", ++ "T86x_FRAG_DUMMY_THREADS", ++ "T86x_FRAG_QUADS_RAST", ++ "T86x_FRAG_QUADS_EZS_TEST", ++ "T86x_FRAG_QUADS_EZS_KILLED", ++ "T86x_FRAG_THREADS_LZS_TEST", ++ "T86x_FRAG_THREADS_LZS_KILLED", ++ "T86x_FRAG_CYCLES_NO_TILE", ++ "T86x_FRAG_NUM_TILES", ++ "T86x_FRAG_TRANS_ELIM", ++ "T86x_COMPUTE_ACTIVE", ++ "T86x_COMPUTE_TASKS", ++ "T86x_COMPUTE_THREADS", ++ "T86x_COMPUTE_CYCLES_DESC", ++ "T86x_TRIPIPE_ACTIVE", ++ "T86x_ARITH_WORDS", ++ "T86x_ARITH_CYCLES_REG", ++ "T86x_ARITH_CYCLES_L0", ++ "T86x_ARITH_FRAG_DEPEND", ++ "T86x_LS_WORDS", ++ "T86x_LS_ISSUES", ++ "T86x_LS_REISSUE_ATTR", ++ "T86x_LS_REISSUES_VARY", ++ "T86x_LS_VARY_RV_MISS", ++ "T86x_LS_VARY_RV_HIT", ++ "T86x_LS_NO_UNPARK", ++ "T86x_TEX_WORDS", ++ "T86x_TEX_BUBBLES", ++ "T86x_TEX_WORDS_L0", ++ "T86x_TEX_WORDS_DESC", ++ "T86x_TEX_ISSUES", ++ "T86x_TEX_RECIRC_FMISS", ++ "T86x_TEX_RECIRC_DESC", ++ "T86x_TEX_RECIRC_MULTI", ++ "T86x_TEX_RECIRC_PMISS", ++ "T86x_TEX_RECIRC_CONF", ++ "T86x_LSC_READ_HITS", ++ "T86x_LSC_READ_OP", ++ "T86x_LSC_WRITE_HITS", ++ "T86x_LSC_WRITE_OP", ++ "T86x_LSC_ATOMIC_HITS", ++ "T86x_LSC_ATOMIC_OP", ++ "T86x_LSC_LINE_FETCHES", ++ "T86x_LSC_DIRTY_LINE", ++ "T86x_LSC_SNOOPS", ++ "T86x_AXI_TLB_STALL", ++ "T86x_AXI_TLB_MISS", ++ "T86x_AXI_TLB_TRANSACTION", ++ "T86x_LS_TLB_MISS", ++ "T86x_LS_TLB_HIT", ++ "T86x_AXI_BEATS_READ", ++ "T86x_AXI_BEATS_WRITTEN", ++ ++ /*L2 and MMU */ ++ "", ++ "", ++ "", ++ "", ++ "T86x_MMU_HIT", ++ "T86x_MMU_NEW_MISS", ++ "T86x_MMU_REPLAY_FULL", ++ "T86x_MMU_REPLAY_MISS", ++ "T86x_MMU_TABLE_WALK", ++ "T86x_MMU_REQUESTS", ++ "", ++ "", ++ "T86x_UTLB_HIT", ++ "T86x_UTLB_NEW_MISS", ++ "T86x_UTLB_REPLAY_FULL", ++ "T86x_UTLB_REPLAY_MISS", ++ "T86x_UTLB_STALL", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "T86x_L2_EXT_WRITE_BEATS", ++ "T86x_L2_EXT_READ_BEATS", ++ "T86x_L2_ANY_LOOKUP", ++ "T86x_L2_READ_LOOKUP", ++ "T86x_L2_SREAD_LOOKUP", ++ "T86x_L2_READ_REPLAY", ++ "T86x_L2_READ_SNOOP", ++ "T86x_L2_READ_HIT", ++ "T86x_L2_CLEAN_MISS", ++ "T86x_L2_WRITE_LOOKUP", ++ "T86x_L2_SWRITE_LOOKUP", ++ "T86x_L2_WRITE_REPLAY", ++ "T86x_L2_WRITE_SNOOP", ++ "T86x_L2_WRITE_HIT", ++ "T86x_L2_EXT_READ_FULL", ++ "", ++ "T86x_L2_EXT_WRITE_FULL", ++ "T86x_L2_EXT_R_W_HAZARD", ++ "T86x_L2_EXT_READ", ++ "T86x_L2_EXT_READ_LINE", ++ "T86x_L2_EXT_WRITE", ++ "T86x_L2_EXT_WRITE_LINE", ++ "T86x_L2_EXT_WRITE_SMALL", ++ "T86x_L2_EXT_BARRIER", ++ "T86x_L2_EXT_AR_STALL", ++ "T86x_L2_EXT_R_BUF_FULL", ++ "T86x_L2_EXT_RD_BUF_FULL", ++ "T86x_L2_EXT_R_RAW", ++ "T86x_L2_EXT_W_STALL", ++ "T86x_L2_EXT_W_BUF_FULL", ++ "T86x_L2_EXT_R_BUF_FULL", ++ "T86x_L2_TAG_HAZARD", ++ "T86x_L2_SNOOP_FULL", ++ "T86x_L2_REPLAY_FULL" ++}; ++ ++static const char * const hardware_counters_mali_t88x[] = { ++ /* Job Manager */ ++ "", ++ "", ++ "", ++ "", ++ "T88x_MESSAGES_SENT", ++ "T88x_MESSAGES_RECEIVED", ++ "T88x_GPU_ACTIVE", ++ "T88x_IRQ_ACTIVE", ++ "T88x_JS0_JOBS", ++ "T88x_JS0_TASKS", ++ "T88x_JS0_ACTIVE", ++ "", ++ "T88x_JS0_WAIT_READ", ++ "T88x_JS0_WAIT_ISSUE", ++ "T88x_JS0_WAIT_DEPEND", ++ "T88x_JS0_WAIT_FINISH", ++ "T88x_JS1_JOBS", ++ "T88x_JS1_TASKS", ++ "T88x_JS1_ACTIVE", ++ "", ++ "T88x_JS1_WAIT_READ", ++ "T88x_JS1_WAIT_ISSUE", ++ "T88x_JS1_WAIT_DEPEND", ++ "T88x_JS1_WAIT_FINISH", ++ "T88x_JS2_JOBS", ++ "T88x_JS2_TASKS", ++ "T88x_JS2_ACTIVE", ++ "", ++ "T88x_JS2_WAIT_READ", ++ "T88x_JS2_WAIT_ISSUE", ++ "T88x_JS2_WAIT_DEPEND", ++ "T88x_JS2_WAIT_FINISH", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ ++ /*Tiler */ ++ "", ++ "", ++ "", ++ "T88x_TI_JOBS_PROCESSED", ++ "T88x_TI_TRIANGLES", ++ "T88x_TI_QUADS", ++ "T88x_TI_POLYGONS", ++ "T88x_TI_POINTS", ++ "T88x_TI_LINES", ++ "T88x_TI_VCACHE_HIT", ++ "T88x_TI_VCACHE_MISS", ++ "T88x_TI_FRONT_FACING", ++ "T88x_TI_BACK_FACING", ++ "T88x_TI_PRIM_VISIBLE", ++ "T88x_TI_PRIM_CULLED", ++ "T88x_TI_PRIM_CLIPPED", ++ "T88x_TI_LEVEL0", ++ "T88x_TI_LEVEL1", ++ "T88x_TI_LEVEL2", ++ "T88x_TI_LEVEL3", ++ "T88x_TI_LEVEL4", ++ "T88x_TI_LEVEL5", ++ "T88x_TI_LEVEL6", ++ "T88x_TI_LEVEL7", ++ "T88x_TI_COMMAND_1", ++ "T88x_TI_COMMAND_2", ++ "T88x_TI_COMMAND_3", ++ "T88x_TI_COMMAND_4", ++ "T88x_TI_COMMAND_5_7", ++ "T88x_TI_COMMAND_8_15", ++ "T88x_TI_COMMAND_16_63", ++ "T88x_TI_COMMAND_64", ++ "T88x_TI_COMPRESS_IN", ++ "T88x_TI_COMPRESS_OUT", ++ "T88x_TI_COMPRESS_FLUSH", ++ "T88x_TI_TIMESTAMPS", ++ "T88x_TI_PCACHE_HIT", ++ "T88x_TI_PCACHE_MISS", ++ "T88x_TI_PCACHE_LINE", ++ "T88x_TI_PCACHE_STALL", ++ "T88x_TI_WRBUF_HIT", ++ "T88x_TI_WRBUF_MISS", ++ "T88x_TI_WRBUF_LINE", ++ "T88x_TI_WRBUF_PARTIAL", ++ "T88x_TI_WRBUF_STALL", ++ "T88x_TI_ACTIVE", ++ "T88x_TI_LOADING_DESC", ++ "T88x_TI_INDEX_WAIT", ++ "T88x_TI_INDEX_RANGE_WAIT", ++ "T88x_TI_VERTEX_WAIT", ++ "T88x_TI_PCACHE_WAIT", ++ "T88x_TI_WRBUF_WAIT", ++ "T88x_TI_BUS_READ", ++ "T88x_TI_BUS_WRITE", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "T88x_TI_UTLB_HIT", ++ "T88x_TI_UTLB_NEW_MISS", ++ "T88x_TI_UTLB_REPLAY_FULL", ++ "T88x_TI_UTLB_REPLAY_MISS", ++ "T88x_TI_UTLB_STALL", ++ ++ /* Shader Core */ ++ "", ++ "", ++ "", ++ "", ++ "T88x_FRAG_ACTIVE", ++ "T88x_FRAG_PRIMITIVES", ++ "T88x_FRAG_PRIMITIVES_DROPPED", ++ "T88x_FRAG_CYCLES_DESC", ++ "T88x_FRAG_CYCLES_FPKQ_ACTIVE", ++ "T88x_FRAG_CYCLES_VERT", ++ "T88x_FRAG_CYCLES_TRISETUP", ++ "T88x_FRAG_CYCLES_EZS_ACTIVE", ++ "T88x_FRAG_THREADS", ++ "T88x_FRAG_DUMMY_THREADS", ++ "T88x_FRAG_QUADS_RAST", ++ "T88x_FRAG_QUADS_EZS_TEST", ++ "T88x_FRAG_QUADS_EZS_KILLED", ++ "T88x_FRAG_THREADS_LZS_TEST", ++ "T88x_FRAG_THREADS_LZS_KILLED", ++ "T88x_FRAG_CYCLES_NO_TILE", ++ "T88x_FRAG_NUM_TILES", ++ "T88x_FRAG_TRANS_ELIM", ++ "T88x_COMPUTE_ACTIVE", ++ "T88x_COMPUTE_TASKS", ++ "T88x_COMPUTE_THREADS", ++ "T88x_COMPUTE_CYCLES_DESC", ++ "T88x_TRIPIPE_ACTIVE", ++ "T88x_ARITH_WORDS", ++ "T88x_ARITH_CYCLES_REG", ++ "T88x_ARITH_CYCLES_L0", ++ "T88x_ARITH_FRAG_DEPEND", ++ "T88x_LS_WORDS", ++ "T88x_LS_ISSUES", ++ "T88x_LS_REISSUE_ATTR", ++ "T88x_LS_REISSUES_VARY", ++ "T88x_LS_VARY_RV_MISS", ++ "T88x_LS_VARY_RV_HIT", ++ "T88x_LS_NO_UNPARK", ++ "T88x_TEX_WORDS", ++ "T88x_TEX_BUBBLES", ++ "T88x_TEX_WORDS_L0", ++ "T88x_TEX_WORDS_DESC", ++ "T88x_TEX_ISSUES", ++ "T88x_TEX_RECIRC_FMISS", ++ "T88x_TEX_RECIRC_DESC", ++ "T88x_TEX_RECIRC_MULTI", ++ "T88x_TEX_RECIRC_PMISS", ++ "T88x_TEX_RECIRC_CONF", ++ "T88x_LSC_READ_HITS", ++ "T88x_LSC_READ_OP", ++ "T88x_LSC_WRITE_HITS", ++ "T88x_LSC_WRITE_OP", ++ "T88x_LSC_ATOMIC_HITS", ++ "T88x_LSC_ATOMIC_OP", ++ "T88x_LSC_LINE_FETCHES", ++ "T88x_LSC_DIRTY_LINE", ++ "T88x_LSC_SNOOPS", ++ "T88x_AXI_TLB_STALL", ++ "T88x_AXI_TLB_MISS", ++ "T88x_AXI_TLB_TRANSACTION", ++ "T88x_LS_TLB_MISS", ++ "T88x_LS_TLB_HIT", ++ "T88x_AXI_BEATS_READ", ++ "T88x_AXI_BEATS_WRITTEN", ++ ++ /*L2 and MMU */ ++ "", ++ "", ++ "", ++ "", ++ "T88x_MMU_HIT", ++ "T88x_MMU_NEW_MISS", ++ "T88x_MMU_REPLAY_FULL", ++ "T88x_MMU_REPLAY_MISS", ++ "T88x_MMU_TABLE_WALK", ++ "T88x_MMU_REQUESTS", ++ "", ++ "", ++ "T88x_UTLB_HIT", ++ "T88x_UTLB_NEW_MISS", ++ "T88x_UTLB_REPLAY_FULL", ++ "T88x_UTLB_REPLAY_MISS", ++ "T88x_UTLB_STALL", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "T88x_L2_EXT_WRITE_BEATS", ++ "T88x_L2_EXT_READ_BEATS", ++ "T88x_L2_ANY_LOOKUP", ++ "T88x_L2_READ_LOOKUP", ++ "T88x_L2_SREAD_LOOKUP", ++ "T88x_L2_READ_REPLAY", ++ "T88x_L2_READ_SNOOP", ++ "T88x_L2_READ_HIT", ++ "T88x_L2_CLEAN_MISS", ++ "T88x_L2_WRITE_LOOKUP", ++ "T88x_L2_SWRITE_LOOKUP", ++ "T88x_L2_WRITE_REPLAY", ++ "T88x_L2_WRITE_SNOOP", ++ "T88x_L2_WRITE_HIT", ++ "T88x_L2_EXT_READ_FULL", ++ "", ++ "T88x_L2_EXT_WRITE_FULL", ++ "T88x_L2_EXT_R_W_HAZARD", ++ "T88x_L2_EXT_READ", ++ "T88x_L2_EXT_READ_LINE", ++ "T88x_L2_EXT_WRITE", ++ "T88x_L2_EXT_WRITE_LINE", ++ "T88x_L2_EXT_WRITE_SMALL", ++ "T88x_L2_EXT_BARRIER", ++ "T88x_L2_EXT_AR_STALL", ++ "T88x_L2_EXT_R_BUF_FULL", ++ "T88x_L2_EXT_RD_BUF_FULL", ++ "T88x_L2_EXT_R_RAW", ++ "T88x_L2_EXT_W_STALL", ++ "T88x_L2_EXT_W_BUF_FULL", ++ "T88x_L2_EXT_R_BUF_FULL", ++ "T88x_L2_TAG_HAZARD", ++ "T88x_L2_SNOOP_FULL", ++ "T88x_L2_REPLAY_FULL" ++}; ++ ++#include "mali_kbase_gator_hwcnt_names_tmix.h" ++ ++#include "mali_kbase_gator_hwcnt_names_thex.h" ++ ++#include "mali_kbase_gator_hwcnt_names_tsix.h" ++ ++ ++#ifdef MALI_INCLUDE_TKAX ++#include "mali_kbase_gator_hwcnt_names_tkax.h" ++#endif /* MALI_INCLUDE_TKAX */ ++ ++#endif +diff --git a/drivers/gpu/arm/midgard/mali_kbase_gator_hwcnt_names_thex.h b/drivers/gpu/arm/midgard/mali_kbase_gator_hwcnt_names_thex.h +new file mode 100755 +index 000000000000..bcceef4fc9bc +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_gator_hwcnt_names_thex.h +@@ -0,0 +1,291 @@ ++/* ++ * ++ * (C) COPYRIGHT 2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* ++ * This header was autogenerated, it should not be edited. ++ */ ++ ++#ifndef _KBASE_GATOR_HWCNT_NAMES_THEX_H_ ++#define _KBASE_GATOR_HWCNT_NAMES_THEX_H_ ++ ++static const char * const hardware_counters_mali_tHEx[] = { ++ /* Performance counters for the Job Manager */ ++ "", ++ "", ++ "", ++ "", ++ "THEx_MESSAGES_SENT", ++ "THEx_MESSAGES_RECEIVED", ++ "THEx_GPU_ACTIVE", ++ "THEx_IRQ_ACTIVE", ++ "THEx_JS0_JOBS", ++ "THEx_JS0_TASKS", ++ "THEx_JS0_ACTIVE", ++ "", ++ "THEx_JS0_WAIT_READ", ++ "THEx_JS0_WAIT_ISSUE", ++ "THEx_JS0_WAIT_DEPEND", ++ "THEx_JS0_WAIT_FINISH", ++ "THEx_JS1_JOBS", ++ "THEx_JS1_TASKS", ++ "THEx_JS1_ACTIVE", ++ "", ++ "THEx_JS1_WAIT_READ", ++ "THEx_JS1_WAIT_ISSUE", ++ "THEx_JS1_WAIT_DEPEND", ++ "THEx_JS1_WAIT_FINISH", ++ "THEx_JS2_JOBS", ++ "THEx_JS2_TASKS", ++ "THEx_JS2_ACTIVE", ++ "", ++ "THEx_JS2_WAIT_READ", ++ "THEx_JS2_WAIT_ISSUE", ++ "THEx_JS2_WAIT_DEPEND", ++ "THEx_JS2_WAIT_FINISH", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ ++ /* Performance counters for the Tiler */ ++ "", ++ "", ++ "", ++ "", ++ "THEx_TILER_ACTIVE", ++ "THEx_JOBS_PROCESSED", ++ "THEx_TRIANGLES", ++ "THEx_LINES", ++ "THEx_POINTS", ++ "THEx_FRONT_FACING", ++ "THEx_BACK_FACING", ++ "THEx_PRIM_VISIBLE", ++ "THEx_PRIM_CULLED", ++ "THEx_PRIM_CLIPPED", ++ "THEx_PRIM_SAT_CULLED", ++ "", ++ "", ++ "THEx_BUS_READ", ++ "", ++ "THEx_BUS_WRITE", ++ "THEx_LOADING_DESC", ++ "THEx_IDVS_POS_SHAD_REQ", ++ "THEx_IDVS_POS_SHAD_WAIT", ++ "THEx_IDVS_POS_SHAD_STALL", ++ "THEx_IDVS_POS_FIFO_FULL", ++ "THEx_PREFETCH_STALL", ++ "THEx_VCACHE_HIT", ++ "THEx_VCACHE_MISS", ++ "THEx_VCACHE_LINE_WAIT", ++ "THEx_VFETCH_POS_READ_WAIT", ++ "THEx_VFETCH_VERTEX_WAIT", ++ "THEx_VFETCH_STALL", ++ "THEx_PRIMASSY_STALL", ++ "THEx_BBOX_GEN_STALL", ++ "THEx_IDVS_VBU_HIT", ++ "THEx_IDVS_VBU_MISS", ++ "THEx_IDVS_VBU_LINE_DEALLOCATE", ++ "THEx_IDVS_VAR_SHAD_REQ", ++ "THEx_IDVS_VAR_SHAD_STALL", ++ "THEx_BINNER_STALL", ++ "THEx_ITER_STALL", ++ "THEx_COMPRESS_MISS", ++ "THEx_COMPRESS_STALL", ++ "THEx_PCACHE_HIT", ++ "THEx_PCACHE_MISS", ++ "THEx_PCACHE_MISS_STALL", ++ "THEx_PCACHE_EVICT_STALL", ++ "THEx_PMGR_PTR_WR_STALL", ++ "THEx_PMGR_PTR_RD_STALL", ++ "THEx_PMGR_CMD_WR_STALL", ++ "THEx_WRBUF_ACTIVE", ++ "THEx_WRBUF_HIT", ++ "THEx_WRBUF_MISS", ++ "THEx_WRBUF_NO_FREE_LINE_STALL", ++ "THEx_WRBUF_NO_AXI_ID_STALL", ++ "THEx_WRBUF_AXI_STALL", ++ "", ++ "", ++ "", ++ "THEx_UTLB_TRANS", ++ "THEx_UTLB_TRANS_HIT", ++ "THEx_UTLB_TRANS_STALL", ++ "THEx_UTLB_TRANS_MISS_DELAY", ++ "THEx_UTLB_MMU_REQ", ++ ++ /* Performance counters for the Shader Core */ ++ "", ++ "", ++ "", ++ "", ++ "THEx_FRAG_ACTIVE", ++ "THEx_FRAG_PRIMITIVES", ++ "THEx_FRAG_PRIM_RAST", ++ "THEx_FRAG_FPK_ACTIVE", ++ "THEx_FRAG_STARVING", ++ "THEx_FRAG_WARPS", ++ "THEx_FRAG_PARTIAL_WARPS", ++ "THEx_FRAG_QUADS_RAST", ++ "THEx_FRAG_QUADS_EZS_TEST", ++ "THEx_FRAG_QUADS_EZS_UPDATE", ++ "THEx_FRAG_QUADS_EZS_KILL", ++ "THEx_FRAG_LZS_TEST", ++ "THEx_FRAG_LZS_KILL", ++ "", ++ "THEx_FRAG_PTILES", ++ "THEx_FRAG_TRANS_ELIM", ++ "THEx_QUAD_FPK_KILLER", ++ "", ++ "THEx_COMPUTE_ACTIVE", ++ "THEx_COMPUTE_TASKS", ++ "THEx_COMPUTE_WARPS", ++ "THEx_COMPUTE_STARVING", ++ "THEx_EXEC_CORE_ACTIVE", ++ "THEx_EXEC_ACTIVE", ++ "THEx_EXEC_INSTR_COUNT", ++ "THEx_EXEC_INSTR_DIVERGED", ++ "THEx_EXEC_INSTR_STARVING", ++ "THEx_ARITH_INSTR_SINGLE_FMA", ++ "THEx_ARITH_INSTR_DOUBLE", ++ "THEx_ARITH_INSTR_MSG", ++ "THEx_ARITH_INSTR_MSG_ONLY", ++ "THEx_TEX_INSTR", ++ "THEx_TEX_INSTR_MIPMAP", ++ "THEx_TEX_INSTR_COMPRESSED", ++ "THEx_TEX_INSTR_3D", ++ "THEx_TEX_INSTR_TRILINEAR", ++ "THEx_TEX_COORD_ISSUE", ++ "THEx_TEX_COORD_STALL", ++ "THEx_TEX_STARVE_CACHE", ++ "THEx_TEX_STARVE_FILTER", ++ "THEx_LS_MEM_READ_FULL", ++ "THEx_LS_MEM_READ_SHORT", ++ "THEx_LS_MEM_WRITE_FULL", ++ "THEx_LS_MEM_WRITE_SHORT", ++ "THEx_LS_MEM_ATOMIC", ++ "THEx_VARY_INSTR", ++ "THEx_VARY_SLOT_32", ++ "THEx_VARY_SLOT_16", ++ "THEx_ATTR_INSTR", ++ "THEx_ARITH_INSTR_FP_MUL", ++ "THEx_BEATS_RD_FTC", ++ "THEx_BEATS_RD_FTC_EXT", ++ "THEx_BEATS_RD_LSC", ++ "THEx_BEATS_RD_LSC_EXT", ++ "THEx_BEATS_RD_TEX", ++ "THEx_BEATS_RD_TEX_EXT", ++ "THEx_BEATS_RD_OTHER", ++ "THEx_BEATS_WR_LSC", ++ "THEx_BEATS_WR_TIB", ++ "", ++ ++ /* Performance counters for the Memory System */ ++ "", ++ "", ++ "", ++ "", ++ "THEx_MMU_REQUESTS", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "THEx_L2_RD_MSG_IN", ++ "THEx_L2_RD_MSG_IN_STALL", ++ "THEx_L2_WR_MSG_IN", ++ "THEx_L2_WR_MSG_IN_STALL", ++ "THEx_L2_SNP_MSG_IN", ++ "THEx_L2_SNP_MSG_IN_STALL", ++ "THEx_L2_RD_MSG_OUT", ++ "THEx_L2_RD_MSG_OUT_STALL", ++ "THEx_L2_WR_MSG_OUT", ++ "THEx_L2_ANY_LOOKUP", ++ "THEx_L2_READ_LOOKUP", ++ "THEx_L2_WRITE_LOOKUP", ++ "THEx_L2_EXT_SNOOP_LOOKUP", ++ "THEx_L2_EXT_READ", ++ "THEx_L2_EXT_READ_NOSNP", ++ "THEx_L2_EXT_READ_UNIQUE", ++ "THEx_L2_EXT_READ_BEATS", ++ "THEx_L2_EXT_AR_STALL", ++ "THEx_L2_EXT_AR_CNT_Q1", ++ "THEx_L2_EXT_AR_CNT_Q2", ++ "THEx_L2_EXT_AR_CNT_Q3", ++ "THEx_L2_EXT_RRESP_0_127", ++ "THEx_L2_EXT_RRESP_128_191", ++ "THEx_L2_EXT_RRESP_192_255", ++ "THEx_L2_EXT_RRESP_256_319", ++ "THEx_L2_EXT_RRESP_320_383", ++ "THEx_L2_EXT_WRITE", ++ "THEx_L2_EXT_WRITE_NOSNP_FULL", ++ "THEx_L2_EXT_WRITE_NOSNP_PTL", ++ "THEx_L2_EXT_WRITE_SNP_FULL", ++ "THEx_L2_EXT_WRITE_SNP_PTL", ++ "THEx_L2_EXT_WRITE_BEATS", ++ "THEx_L2_EXT_W_STALL", ++ "THEx_L2_EXT_AW_CNT_Q1", ++ "THEx_L2_EXT_AW_CNT_Q2", ++ "THEx_L2_EXT_AW_CNT_Q3", ++ "THEx_L2_EXT_SNOOP", ++ "THEx_L2_EXT_SNOOP_STALL", ++ "THEx_L2_EXT_SNOOP_RESP_CLEAN", ++ "THEx_L2_EXT_SNOOP_RESP_DATA", ++ "THEx_L2_EXT_SNOOP_INTERNAL", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++}; ++ ++#endif /* _KBASE_GATOR_HWCNT_NAMES_THEX_H_ */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_gator_hwcnt_names_tmix.h b/drivers/gpu/arm/midgard/mali_kbase_gator_hwcnt_names_tmix.h +new file mode 100755 +index 000000000000..5ea06770fdb2 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_gator_hwcnt_names_tmix.h +@@ -0,0 +1,291 @@ ++/* ++ * ++ * (C) COPYRIGHT 2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* ++ * This header was autogenerated, it should not be edited. ++ */ ++ ++#ifndef _KBASE_GATOR_HWCNT_NAMES_TMIX_H_ ++#define _KBASE_GATOR_HWCNT_NAMES_TMIX_H_ ++ ++static const char * const hardware_counters_mali_tMIx[] = { ++ /* Performance counters for the Job Manager */ ++ "", ++ "", ++ "", ++ "", ++ "TMIx_MESSAGES_SENT", ++ "TMIx_MESSAGES_RECEIVED", ++ "TMIx_GPU_ACTIVE", ++ "TMIx_IRQ_ACTIVE", ++ "TMIx_JS0_JOBS", ++ "TMIx_JS0_TASKS", ++ "TMIx_JS0_ACTIVE", ++ "", ++ "TMIx_JS0_WAIT_READ", ++ "TMIx_JS0_WAIT_ISSUE", ++ "TMIx_JS0_WAIT_DEPEND", ++ "TMIx_JS0_WAIT_FINISH", ++ "TMIx_JS1_JOBS", ++ "TMIx_JS1_TASKS", ++ "TMIx_JS1_ACTIVE", ++ "", ++ "TMIx_JS1_WAIT_READ", ++ "TMIx_JS1_WAIT_ISSUE", ++ "TMIx_JS1_WAIT_DEPEND", ++ "TMIx_JS1_WAIT_FINISH", ++ "TMIx_JS2_JOBS", ++ "TMIx_JS2_TASKS", ++ "TMIx_JS2_ACTIVE", ++ "", ++ "TMIx_JS2_WAIT_READ", ++ "TMIx_JS2_WAIT_ISSUE", ++ "TMIx_JS2_WAIT_DEPEND", ++ "TMIx_JS2_WAIT_FINISH", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ ++ /* Performance counters for the Tiler */ ++ "", ++ "", ++ "", ++ "", ++ "TMIx_TILER_ACTIVE", ++ "TMIx_JOBS_PROCESSED", ++ "TMIx_TRIANGLES", ++ "TMIx_LINES", ++ "TMIx_POINTS", ++ "TMIx_FRONT_FACING", ++ "TMIx_BACK_FACING", ++ "TMIx_PRIM_VISIBLE", ++ "TMIx_PRIM_CULLED", ++ "TMIx_PRIM_CLIPPED", ++ "TMIx_PRIM_SAT_CULLED", ++ "", ++ "", ++ "TMIx_BUS_READ", ++ "", ++ "TMIx_BUS_WRITE", ++ "TMIx_LOADING_DESC", ++ "TMIx_IDVS_POS_SHAD_REQ", ++ "TMIx_IDVS_POS_SHAD_WAIT", ++ "TMIx_IDVS_POS_SHAD_STALL", ++ "TMIx_IDVS_POS_FIFO_FULL", ++ "TMIx_PREFETCH_STALL", ++ "TMIx_VCACHE_HIT", ++ "TMIx_VCACHE_MISS", ++ "TMIx_VCACHE_LINE_WAIT", ++ "TMIx_VFETCH_POS_READ_WAIT", ++ "TMIx_VFETCH_VERTEX_WAIT", ++ "TMIx_VFETCH_STALL", ++ "TMIx_PRIMASSY_STALL", ++ "TMIx_BBOX_GEN_STALL", ++ "TMIx_IDVS_VBU_HIT", ++ "TMIx_IDVS_VBU_MISS", ++ "TMIx_IDVS_VBU_LINE_DEALLOCATE", ++ "TMIx_IDVS_VAR_SHAD_REQ", ++ "TMIx_IDVS_VAR_SHAD_STALL", ++ "TMIx_BINNER_STALL", ++ "TMIx_ITER_STALL", ++ "TMIx_COMPRESS_MISS", ++ "TMIx_COMPRESS_STALL", ++ "TMIx_PCACHE_HIT", ++ "TMIx_PCACHE_MISS", ++ "TMIx_PCACHE_MISS_STALL", ++ "TMIx_PCACHE_EVICT_STALL", ++ "TMIx_PMGR_PTR_WR_STALL", ++ "TMIx_PMGR_PTR_RD_STALL", ++ "TMIx_PMGR_CMD_WR_STALL", ++ "TMIx_WRBUF_ACTIVE", ++ "TMIx_WRBUF_HIT", ++ "TMIx_WRBUF_MISS", ++ "TMIx_WRBUF_NO_FREE_LINE_STALL", ++ "TMIx_WRBUF_NO_AXI_ID_STALL", ++ "TMIx_WRBUF_AXI_STALL", ++ "", ++ "", ++ "", ++ "TMIx_UTLB_TRANS", ++ "TMIx_UTLB_TRANS_HIT", ++ "TMIx_UTLB_TRANS_STALL", ++ "TMIx_UTLB_TRANS_MISS_DELAY", ++ "TMIx_UTLB_MMU_REQ", ++ ++ /* Performance counters for the Shader Core */ ++ "", ++ "", ++ "", ++ "", ++ "TMIx_FRAG_ACTIVE", ++ "TMIx_FRAG_PRIMITIVES", ++ "TMIx_FRAG_PRIM_RAST", ++ "TMIx_FRAG_FPK_ACTIVE", ++ "TMIx_FRAG_STARVING", ++ "TMIx_FRAG_WARPS", ++ "TMIx_FRAG_PARTIAL_WARPS", ++ "TMIx_FRAG_QUADS_RAST", ++ "TMIx_FRAG_QUADS_EZS_TEST", ++ "TMIx_FRAG_QUADS_EZS_UPDATE", ++ "TMIx_FRAG_QUADS_EZS_KILL", ++ "TMIx_FRAG_LZS_TEST", ++ "TMIx_FRAG_LZS_KILL", ++ "", ++ "TMIx_FRAG_PTILES", ++ "TMIx_FRAG_TRANS_ELIM", ++ "TMIx_QUAD_FPK_KILLER", ++ "", ++ "TMIx_COMPUTE_ACTIVE", ++ "TMIx_COMPUTE_TASKS", ++ "TMIx_COMPUTE_WARPS", ++ "TMIx_COMPUTE_STARVING", ++ "TMIx_EXEC_CORE_ACTIVE", ++ "TMIx_EXEC_ACTIVE", ++ "TMIx_EXEC_INSTR_COUNT", ++ "TMIx_EXEC_INSTR_DIVERGED", ++ "TMIx_EXEC_INSTR_STARVING", ++ "TMIx_ARITH_INSTR_SINGLE_FMA", ++ "TMIx_ARITH_INSTR_DOUBLE", ++ "TMIx_ARITH_INSTR_MSG", ++ "TMIx_ARITH_INSTR_MSG_ONLY", ++ "TMIx_TEX_INSTR", ++ "TMIx_TEX_INSTR_MIPMAP", ++ "TMIx_TEX_INSTR_COMPRESSED", ++ "TMIx_TEX_INSTR_3D", ++ "TMIx_TEX_INSTR_TRILINEAR", ++ "TMIx_TEX_COORD_ISSUE", ++ "TMIx_TEX_COORD_STALL", ++ "TMIx_TEX_STARVE_CACHE", ++ "TMIx_TEX_STARVE_FILTER", ++ "TMIx_LS_MEM_READ_FULL", ++ "TMIx_LS_MEM_READ_SHORT", ++ "TMIx_LS_MEM_WRITE_FULL", ++ "TMIx_LS_MEM_WRITE_SHORT", ++ "TMIx_LS_MEM_ATOMIC", ++ "TMIx_VARY_INSTR", ++ "TMIx_VARY_SLOT_32", ++ "TMIx_VARY_SLOT_16", ++ "TMIx_ATTR_INSTR", ++ "TMIx_ARITH_INSTR_FP_MUL", ++ "TMIx_BEATS_RD_FTC", ++ "TMIx_BEATS_RD_FTC_EXT", ++ "TMIx_BEATS_RD_LSC", ++ "TMIx_BEATS_RD_LSC_EXT", ++ "TMIx_BEATS_RD_TEX", ++ "TMIx_BEATS_RD_TEX_EXT", ++ "TMIx_BEATS_RD_OTHER", ++ "TMIx_BEATS_WR_LSC", ++ "TMIx_BEATS_WR_TIB", ++ "", ++ ++ /* Performance counters for the Memory System */ ++ "", ++ "", ++ "", ++ "", ++ "TMIx_MMU_REQUESTS", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "TMIx_L2_RD_MSG_IN", ++ "TMIx_L2_RD_MSG_IN_STALL", ++ "TMIx_L2_WR_MSG_IN", ++ "TMIx_L2_WR_MSG_IN_STALL", ++ "TMIx_L2_SNP_MSG_IN", ++ "TMIx_L2_SNP_MSG_IN_STALL", ++ "TMIx_L2_RD_MSG_OUT", ++ "TMIx_L2_RD_MSG_OUT_STALL", ++ "TMIx_L2_WR_MSG_OUT", ++ "TMIx_L2_ANY_LOOKUP", ++ "TMIx_L2_READ_LOOKUP", ++ "TMIx_L2_WRITE_LOOKUP", ++ "TMIx_L2_EXT_SNOOP_LOOKUP", ++ "TMIx_L2_EXT_READ", ++ "TMIx_L2_EXT_READ_NOSNP", ++ "TMIx_L2_EXT_READ_UNIQUE", ++ "TMIx_L2_EXT_READ_BEATS", ++ "TMIx_L2_EXT_AR_STALL", ++ "TMIx_L2_EXT_AR_CNT_Q1", ++ "TMIx_L2_EXT_AR_CNT_Q2", ++ "TMIx_L2_EXT_AR_CNT_Q3", ++ "TMIx_L2_EXT_RRESP_0_127", ++ "TMIx_L2_EXT_RRESP_128_191", ++ "TMIx_L2_EXT_RRESP_192_255", ++ "TMIx_L2_EXT_RRESP_256_319", ++ "TMIx_L2_EXT_RRESP_320_383", ++ "TMIx_L2_EXT_WRITE", ++ "TMIx_L2_EXT_WRITE_NOSNP_FULL", ++ "TMIx_L2_EXT_WRITE_NOSNP_PTL", ++ "TMIx_L2_EXT_WRITE_SNP_FULL", ++ "TMIx_L2_EXT_WRITE_SNP_PTL", ++ "TMIx_L2_EXT_WRITE_BEATS", ++ "TMIx_L2_EXT_W_STALL", ++ "TMIx_L2_EXT_AW_CNT_Q1", ++ "TMIx_L2_EXT_AW_CNT_Q2", ++ "TMIx_L2_EXT_AW_CNT_Q3", ++ "TMIx_L2_EXT_SNOOP", ++ "TMIx_L2_EXT_SNOOP_STALL", ++ "TMIx_L2_EXT_SNOOP_RESP_CLEAN", ++ "TMIx_L2_EXT_SNOOP_RESP_DATA", ++ "TMIx_L2_EXT_SNOOP_INTERNAL", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++}; ++ ++#endif /* _KBASE_GATOR_HWCNT_NAMES_TMIX_H_ */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_gator_hwcnt_names_tsix.h b/drivers/gpu/arm/midgard/mali_kbase_gator_hwcnt_names_tsix.h +new file mode 100755 +index 000000000000..be09c4556735 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_gator_hwcnt_names_tsix.h +@@ -0,0 +1,291 @@ ++/* ++ * ++ * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* ++ * This header was autogenerated, it should not be edited. ++ */ ++ ++#ifndef _KBASE_GATOR_HWCNT_NAMES_TSIX_H_ ++#define _KBASE_GATOR_HWCNT_NAMES_TSIX_H_ ++ ++static const char * const hardware_counters_mali_tSIx[] = { ++ /* Performance counters for the Job Manager */ ++ "", ++ "", ++ "", ++ "", ++ "TSIx_MESSAGES_SENT", ++ "TSIx_MESSAGES_RECEIVED", ++ "TSIx_GPU_ACTIVE", ++ "TSIx_IRQ_ACTIVE", ++ "TSIx_JS0_JOBS", ++ "TSIx_JS0_TASKS", ++ "TSIx_JS0_ACTIVE", ++ "", ++ "TSIx_JS0_WAIT_READ", ++ "TSIx_JS0_WAIT_ISSUE", ++ "TSIx_JS0_WAIT_DEPEND", ++ "TSIx_JS0_WAIT_FINISH", ++ "TSIx_JS1_JOBS", ++ "TSIx_JS1_TASKS", ++ "TSIx_JS1_ACTIVE", ++ "", ++ "TSIx_JS1_WAIT_READ", ++ "TSIx_JS1_WAIT_ISSUE", ++ "TSIx_JS1_WAIT_DEPEND", ++ "TSIx_JS1_WAIT_FINISH", ++ "TSIx_JS2_JOBS", ++ "TSIx_JS2_TASKS", ++ "TSIx_JS2_ACTIVE", ++ "", ++ "TSIx_JS2_WAIT_READ", ++ "TSIx_JS2_WAIT_ISSUE", ++ "TSIx_JS2_WAIT_DEPEND", ++ "TSIx_JS2_WAIT_FINISH", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ ++ /* Performance counters for the Tiler */ ++ "", ++ "", ++ "", ++ "", ++ "TSIx_TILER_ACTIVE", ++ "TSIx_JOBS_PROCESSED", ++ "TSIx_TRIANGLES", ++ "TSIx_LINES", ++ "TSIx_POINTS", ++ "TSIx_FRONT_FACING", ++ "TSIx_BACK_FACING", ++ "TSIx_PRIM_VISIBLE", ++ "TSIx_PRIM_CULLED", ++ "TSIx_PRIM_CLIPPED", ++ "TSIx_PRIM_SAT_CULLED", ++ "", ++ "", ++ "TSIx_BUS_READ", ++ "", ++ "TSIx_BUS_WRITE", ++ "TSIx_LOADING_DESC", ++ "TSIx_IDVS_POS_SHAD_REQ", ++ "TSIx_IDVS_POS_SHAD_WAIT", ++ "TSIx_IDVS_POS_SHAD_STALL", ++ "TSIx_IDVS_POS_FIFO_FULL", ++ "TSIx_PREFETCH_STALL", ++ "TSIx_VCACHE_HIT", ++ "TSIx_VCACHE_MISS", ++ "TSIx_VCACHE_LINE_WAIT", ++ "TSIx_VFETCH_POS_READ_WAIT", ++ "TSIx_VFETCH_VERTEX_WAIT", ++ "TSIx_VFETCH_STALL", ++ "TSIx_PRIMASSY_STALL", ++ "TSIx_BBOX_GEN_STALL", ++ "TSIx_IDVS_VBU_HIT", ++ "TSIx_IDVS_VBU_MISS", ++ "TSIx_IDVS_VBU_LINE_DEALLOCATE", ++ "TSIx_IDVS_VAR_SHAD_REQ", ++ "TSIx_IDVS_VAR_SHAD_STALL", ++ "TSIx_BINNER_STALL", ++ "TSIx_ITER_STALL", ++ "TSIx_COMPRESS_MISS", ++ "TSIx_COMPRESS_STALL", ++ "TSIx_PCACHE_HIT", ++ "TSIx_PCACHE_MISS", ++ "TSIx_PCACHE_MISS_STALL", ++ "TSIx_PCACHE_EVICT_STALL", ++ "TSIx_PMGR_PTR_WR_STALL", ++ "TSIx_PMGR_PTR_RD_STALL", ++ "TSIx_PMGR_CMD_WR_STALL", ++ "TSIx_WRBUF_ACTIVE", ++ "TSIx_WRBUF_HIT", ++ "TSIx_WRBUF_MISS", ++ "TSIx_WRBUF_NO_FREE_LINE_STALL", ++ "TSIx_WRBUF_NO_AXI_ID_STALL", ++ "TSIx_WRBUF_AXI_STALL", ++ "", ++ "", ++ "", ++ "TSIx_UTLB_TRANS", ++ "TSIx_UTLB_TRANS_HIT", ++ "TSIx_UTLB_TRANS_STALL", ++ "TSIx_UTLB_TRANS_MISS_DELAY", ++ "TSIx_UTLB_MMU_REQ", ++ ++ /* Performance counters for the Shader Core */ ++ "", ++ "", ++ "", ++ "", ++ "TSIx_FRAG_ACTIVE", ++ "TSIx_FRAG_PRIMITIVES", ++ "TSIx_FRAG_PRIM_RAST", ++ "TSIx_FRAG_FPK_ACTIVE", ++ "TSIx_FRAG_STARVING", ++ "TSIx_FRAG_WARPS", ++ "TSIx_FRAG_PARTIAL_WARPS", ++ "TSIx_FRAG_QUADS_RAST", ++ "TSIx_FRAG_QUADS_EZS_TEST", ++ "TSIx_FRAG_QUADS_EZS_UPDATE", ++ "TSIx_FRAG_QUADS_EZS_KILL", ++ "TSIx_FRAG_LZS_TEST", ++ "TSIx_FRAG_LZS_KILL", ++ "", ++ "TSIx_FRAG_PTILES", ++ "TSIx_FRAG_TRANS_ELIM", ++ "TSIx_QUAD_FPK_KILLER", ++ "", ++ "TSIx_COMPUTE_ACTIVE", ++ "TSIx_COMPUTE_TASKS", ++ "TSIx_COMPUTE_WARPS", ++ "TSIx_COMPUTE_STARVING", ++ "TSIx_EXEC_CORE_ACTIVE", ++ "TSIx_EXEC_ACTIVE", ++ "TSIx_EXEC_INSTR_COUNT", ++ "TSIx_EXEC_INSTR_DIVERGED", ++ "TSIx_EXEC_INSTR_STARVING", ++ "TSIx_ARITH_INSTR_SINGLE_FMA", ++ "TSIx_ARITH_INSTR_DOUBLE", ++ "TSIx_ARITH_INSTR_MSG", ++ "TSIx_ARITH_INSTR_MSG_ONLY", ++ "TSIx_TEX_MSGI_NUM_QUADS", ++ "TSIx_TEX_DFCH_NUM_PASSES", ++ "TSIx_TEX_DFCH_NUM_PASSES_MISS", ++ "TSIx_TEX_DFCH_NUM_PASSES_MIP_MAP", ++ "TSIx_TEX_TIDX_NUM_SPLIT_MIP_MAP", ++ "TSIx_TEX_TFCH_NUM_LINES_FETCHED", ++ "TSIx_TEX_TFCH_NUM_LINES_FETCHED_BLOCK_COMPRESSED", ++ "TSIx_TEX_TFCH_NUM_OPERATIONS", ++ "TSIx_TEX_FILT_NUM_OPERATIONS", ++ "TSIx_LS_MEM_READ_FULL", ++ "TSIx_LS_MEM_READ_SHORT", ++ "TSIx_LS_MEM_WRITE_FULL", ++ "TSIx_LS_MEM_WRITE_SHORT", ++ "TSIx_LS_MEM_ATOMIC", ++ "TSIx_VARY_INSTR", ++ "TSIx_VARY_SLOT_32", ++ "TSIx_VARY_SLOT_16", ++ "TSIx_ATTR_INSTR", ++ "TSIx_ARITH_INSTR_FP_MUL", ++ "TSIx_BEATS_RD_FTC", ++ "TSIx_BEATS_RD_FTC_EXT", ++ "TSIx_BEATS_RD_LSC", ++ "TSIx_BEATS_RD_LSC_EXT", ++ "TSIx_BEATS_RD_TEX", ++ "TSIx_BEATS_RD_TEX_EXT", ++ "TSIx_BEATS_RD_OTHER", ++ "TSIx_BEATS_WR_LSC", ++ "TSIx_BEATS_WR_TIB", ++ "", ++ ++ /* Performance counters for the Memory System */ ++ "", ++ "", ++ "", ++ "", ++ "TSIx_MMU_REQUESTS", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "TSIx_L2_RD_MSG_IN", ++ "TSIx_L2_RD_MSG_IN_STALL", ++ "TSIx_L2_WR_MSG_IN", ++ "TSIx_L2_WR_MSG_IN_STALL", ++ "TSIx_L2_SNP_MSG_IN", ++ "TSIx_L2_SNP_MSG_IN_STALL", ++ "TSIx_L2_RD_MSG_OUT", ++ "TSIx_L2_RD_MSG_OUT_STALL", ++ "TSIx_L2_WR_MSG_OUT", ++ "TSIx_L2_ANY_LOOKUP", ++ "TSIx_L2_READ_LOOKUP", ++ "TSIx_L2_WRITE_LOOKUP", ++ "TSIx_L2_EXT_SNOOP_LOOKUP", ++ "TSIx_L2_EXT_READ", ++ "TSIx_L2_EXT_READ_NOSNP", ++ "TSIx_L2_EXT_READ_UNIQUE", ++ "TSIx_L2_EXT_READ_BEATS", ++ "TSIx_L2_EXT_AR_STALL", ++ "TSIx_L2_EXT_AR_CNT_Q1", ++ "TSIx_L2_EXT_AR_CNT_Q2", ++ "TSIx_L2_EXT_AR_CNT_Q3", ++ "TSIx_L2_EXT_RRESP_0_127", ++ "TSIx_L2_EXT_RRESP_128_191", ++ "TSIx_L2_EXT_RRESP_192_255", ++ "TSIx_L2_EXT_RRESP_256_319", ++ "TSIx_L2_EXT_RRESP_320_383", ++ "TSIx_L2_EXT_WRITE", ++ "TSIx_L2_EXT_WRITE_NOSNP_FULL", ++ "TSIx_L2_EXT_WRITE_NOSNP_PTL", ++ "TSIx_L2_EXT_WRITE_SNP_FULL", ++ "TSIx_L2_EXT_WRITE_SNP_PTL", ++ "TSIx_L2_EXT_WRITE_BEATS", ++ "TSIx_L2_EXT_W_STALL", ++ "TSIx_L2_EXT_AW_CNT_Q1", ++ "TSIx_L2_EXT_AW_CNT_Q2", ++ "TSIx_L2_EXT_AW_CNT_Q3", ++ "TSIx_L2_EXT_SNOOP", ++ "TSIx_L2_EXT_SNOOP_STALL", ++ "TSIx_L2_EXT_SNOOP_RESP_CLEAN", ++ "TSIx_L2_EXT_SNOOP_RESP_DATA", ++ "TSIx_L2_EXT_SNOOP_INTERNAL", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++}; ++ ++#endif /* _KBASE_GATOR_HWCNT_NAMES_TSIX_H_ */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_gpu_id.h b/drivers/gpu/arm/midgard/mali_kbase_gpu_id.h +new file mode 100755 +index 000000000000..42f0111c474f +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_gpu_id.h +@@ -0,0 +1,123 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++#ifndef _KBASE_GPU_ID_H_ ++#define _KBASE_GPU_ID_H_ ++ ++/* GPU_ID register */ ++#define GPU_ID_VERSION_STATUS_SHIFT 0 ++#define GPU_ID_VERSION_MINOR_SHIFT 4 ++#define GPU_ID_VERSION_MAJOR_SHIFT 12 ++#define GPU_ID_VERSION_PRODUCT_ID_SHIFT 16 ++#define GPU_ID_VERSION_STATUS (0xF << GPU_ID_VERSION_STATUS_SHIFT) ++#define GPU_ID_VERSION_MINOR (0xFF << GPU_ID_VERSION_MINOR_SHIFT) ++#define GPU_ID_VERSION_MAJOR (0xF << GPU_ID_VERSION_MAJOR_SHIFT) ++#define GPU_ID_VERSION_PRODUCT_ID (0xFFFF << GPU_ID_VERSION_PRODUCT_ID_SHIFT) ++ ++/* Values for GPU_ID_VERSION_PRODUCT_ID bitfield */ ++#define GPU_ID_PI_T60X 0x6956 ++#define GPU_ID_PI_T62X 0x0620 ++#define GPU_ID_PI_T76X 0x0750 ++#define GPU_ID_PI_T72X 0x0720 ++#define GPU_ID_PI_TFRX 0x0880 ++#define GPU_ID_PI_T86X 0x0860 ++#define GPU_ID_PI_T82X 0x0820 ++#define GPU_ID_PI_T83X 0x0830 ++ ++/* New GPU ID format when PRODUCT_ID is >= 0x1000 (and not 0x6956) */ ++#define GPU_ID_PI_NEW_FORMAT_START 0x1000 ++#define GPU_ID_IS_NEW_FORMAT(product_id) ((product_id) != GPU_ID_PI_T60X && \ ++ (product_id) >= \ ++ GPU_ID_PI_NEW_FORMAT_START) ++ ++#define GPU_ID2_VERSION_STATUS_SHIFT 0 ++#define GPU_ID2_VERSION_MINOR_SHIFT 4 ++#define GPU_ID2_VERSION_MAJOR_SHIFT 12 ++#define GPU_ID2_PRODUCT_MAJOR_SHIFT 16 ++#define GPU_ID2_ARCH_REV_SHIFT 20 ++#define GPU_ID2_ARCH_MINOR_SHIFT 24 ++#define GPU_ID2_ARCH_MAJOR_SHIFT 28 ++#define GPU_ID2_VERSION_STATUS (0xF << GPU_ID2_VERSION_STATUS_SHIFT) ++#define GPU_ID2_VERSION_MINOR (0xFF << GPU_ID2_VERSION_MINOR_SHIFT) ++#define GPU_ID2_VERSION_MAJOR (0xF << GPU_ID2_VERSION_MAJOR_SHIFT) ++#define GPU_ID2_PRODUCT_MAJOR (0xF << GPU_ID2_PRODUCT_MAJOR_SHIFT) ++#define GPU_ID2_ARCH_REV (0xF << GPU_ID2_ARCH_REV_SHIFT) ++#define GPU_ID2_ARCH_MINOR (0xF << GPU_ID2_ARCH_MINOR_SHIFT) ++#define GPU_ID2_ARCH_MAJOR (0xF << GPU_ID2_ARCH_MAJOR_SHIFT) ++#define GPU_ID2_PRODUCT_MODEL (GPU_ID2_ARCH_MAJOR | GPU_ID2_PRODUCT_MAJOR) ++#define GPU_ID2_VERSION (GPU_ID2_VERSION_MAJOR | \ ++ GPU_ID2_VERSION_MINOR | \ ++ GPU_ID2_VERSION_STATUS) ++ ++/* Helper macro to create a partial GPU_ID (new format) that defines ++ a product ignoring its version. */ ++#define GPU_ID2_PRODUCT_MAKE(arch_major, arch_minor, arch_rev, product_major) \ ++ (((arch_major) << GPU_ID2_ARCH_MAJOR_SHIFT) | \ ++ ((arch_minor) << GPU_ID2_ARCH_MINOR_SHIFT) | \ ++ ((arch_rev) << GPU_ID2_ARCH_REV_SHIFT) | \ ++ ((product_major) << GPU_ID2_PRODUCT_MAJOR_SHIFT)) ++ ++/* Helper macro to create a partial GPU_ID (new format) that specifies the ++ revision (major, minor, status) of a product */ ++#define GPU_ID2_VERSION_MAKE(version_major, version_minor, version_status) \ ++ (((version_major) << GPU_ID2_VERSION_MAJOR_SHIFT) | \ ++ ((version_minor) << GPU_ID2_VERSION_MINOR_SHIFT) | \ ++ ((version_status) << GPU_ID2_VERSION_STATUS_SHIFT)) ++ ++/* Helper macro to create a complete GPU_ID (new format) */ ++#define GPU_ID2_MAKE(arch_major, arch_minor, arch_rev, product_major, \ ++ version_major, version_minor, version_status) \ ++ (GPU_ID2_PRODUCT_MAKE(arch_major, arch_minor, arch_rev, \ ++ product_major) | \ ++ GPU_ID2_VERSION_MAKE(version_major, version_minor, \ ++ version_status)) ++ ++/* Helper macro to create a partial GPU_ID (new format) that identifies ++ a particular GPU model by its arch_major and product_major. */ ++#define GPU_ID2_MODEL_MAKE(arch_major, product_major) \ ++ (((arch_major) << GPU_ID2_ARCH_MAJOR_SHIFT) | \ ++ ((product_major) << GPU_ID2_PRODUCT_MAJOR_SHIFT)) ++ ++/* Strip off the non-relevant bits from a product_id value and make it suitable ++ for comparison against the GPU_ID2_PRODUCT_xxx values which identify a GPU ++ model. */ ++#define GPU_ID2_MODEL_MATCH_VALUE(product_id) \ ++ (((product_id) << GPU_ID2_PRODUCT_MAJOR_SHIFT) & \ ++ GPU_ID2_PRODUCT_MODEL) ++ ++#define GPU_ID2_PRODUCT_TMIX GPU_ID2_MODEL_MAKE(6u, 0) ++#define GPU_ID2_PRODUCT_THEX GPU_ID2_MODEL_MAKE(6u, 1) ++#define GPU_ID2_PRODUCT_TSIX GPU_ID2_MODEL_MAKE(7u, 0) ++#ifdef MALI_INCLUDE_TKAX ++#define GPU_ID2_PRODUCT_TKAX GPU_ID2_MODEL_MAKE(9u, 0) ++#endif /* MALI_INCLUDE_TKAX */ ++#ifdef MALI_INCLUDE_TTRX ++#define GPU_ID2_PRODUCT_TTRX GPU_ID2_MODEL_MAKE(10u, 0) ++#endif /* MALI_INCLUDE_TTRX */ ++ ++/* Values for GPU_ID_VERSION_STATUS field for PRODUCT_ID GPU_ID_PI_T60X */ ++#define GPU_ID_S_15DEV0 0x1 ++#define GPU_ID_S_EAC 0x2 ++ ++/* Helper macro to create a GPU_ID assuming valid values for id, major, ++ minor, status */ ++#define GPU_ID_MAKE(id, major, minor, status) \ ++ (((id) << GPU_ID_VERSION_PRODUCT_ID_SHIFT) | \ ++ ((major) << GPU_ID_VERSION_MAJOR_SHIFT) | \ ++ ((minor) << GPU_ID_VERSION_MINOR_SHIFT) | \ ++ ((status) << GPU_ID_VERSION_STATUS_SHIFT)) ++ ++#endif /* _KBASE_GPU_ID_H_ */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_gpu_memory_debugfs.c b/drivers/gpu/arm/midgard/mali_kbase_gpu_memory_debugfs.c +new file mode 100755 +index 000000000000..6df0a1cb1264 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_gpu_memory_debugfs.c +@@ -0,0 +1,97 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include ++ ++#ifdef CONFIG_DEBUG_FS ++/** Show callback for the @c gpu_memory debugfs file. ++ * ++ * This function is called to get the contents of the @c gpu_memory debugfs ++ * file. This is a report of current gpu memory usage. ++ * ++ * @param sfile The debugfs entry ++ * @param data Data associated with the entry ++ * ++ * @return 0 if successfully prints data in debugfs entry file ++ * -1 if it encountered an error ++ */ ++ ++static int kbasep_gpu_memory_seq_show(struct seq_file *sfile, void *data) ++{ ++ struct list_head *entry; ++ const struct list_head *kbdev_list; ++ ++ kbdev_list = kbase_dev_list_get(); ++ list_for_each(entry, kbdev_list) { ++ struct kbase_device *kbdev = NULL; ++ struct kbasep_kctx_list_element *element; ++ ++ kbdev = list_entry(entry, struct kbase_device, entry); ++ /* output the total memory usage and cap for this device */ ++ seq_printf(sfile, "%-16s %10u\n", ++ kbdev->devname, ++ atomic_read(&(kbdev->memdev.used_pages))); ++ mutex_lock(&kbdev->kctx_list_lock); ++ list_for_each_entry(element, &kbdev->kctx_list, link) { ++ /* output the memory usage and cap for each kctx ++ * opened on this device */ ++ seq_printf(sfile, " %s-0x%p %10u\n", ++ "kctx", ++ element->kctx, ++ atomic_read(&(element->kctx->used_pages))); ++ } ++ mutex_unlock(&kbdev->kctx_list_lock); ++ } ++ kbase_dev_list_put(kbdev_list); ++ return 0; ++} ++ ++/* ++ * File operations related to debugfs entry for gpu_memory ++ */ ++static int kbasep_gpu_memory_debugfs_open(struct inode *in, struct file *file) ++{ ++ return single_open(file, kbasep_gpu_memory_seq_show , NULL); ++} ++ ++static const struct file_operations kbasep_gpu_memory_debugfs_fops = { ++ .open = kbasep_gpu_memory_debugfs_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = single_release, ++}; ++ ++/* ++ * Initialize debugfs entry for gpu_memory ++ */ ++void kbasep_gpu_memory_debugfs_init(struct kbase_device *kbdev) ++{ ++ debugfs_create_file("gpu_memory", S_IRUGO, ++ kbdev->mali_debugfs_directory, NULL, ++ &kbasep_gpu_memory_debugfs_fops); ++ return; ++} ++ ++#else ++/* ++ * Stub functions for when debugfs is disabled ++ */ ++void kbasep_gpu_memory_debugfs_init(struct kbase_device *kbdev) ++{ ++ return; ++} ++#endif +diff --git a/drivers/gpu/arm/midgard/mali_kbase_gpu_memory_debugfs.h b/drivers/gpu/arm/midgard/mali_kbase_gpu_memory_debugfs.h +new file mode 100755 +index 000000000000..7045693eb910 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_gpu_memory_debugfs.h +@@ -0,0 +1,37 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2014, 2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/** ++ * @file mali_kbase_gpu_memory_debugfs.h ++ * Header file for gpu_memory entry in debugfs ++ * ++ */ ++ ++#ifndef _KBASE_GPU_MEMORY_DEBUGFS_H ++#define _KBASE_GPU_MEMORY_DEBUGFS_H ++ ++#include ++#include ++ ++/** ++ * @brief Initialize gpu_memory debugfs entry ++ */ ++void kbasep_gpu_memory_debugfs_init(struct kbase_device *kbdev); ++ ++#endif /*_KBASE_GPU_MEMORY_DEBUGFS_H*/ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_gpuprops.c b/drivers/gpu/arm/midgard/mali_kbase_gpuprops.c +new file mode 100755 +index 000000000000..a947a2e03a2f +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_gpuprops.c +@@ -0,0 +1,510 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * Base kernel property query APIs ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include "mali_kbase_ioctl.h" ++#include ++ ++/** ++ * KBASE_UBFX32 - Extracts bits from a 32-bit bitfield. ++ * @value: The value from which to extract bits. ++ * @offset: The first bit to extract (0 being the LSB). ++ * @size: The number of bits to extract. ++ * ++ * Context: @offset + @size <= 32. ++ * ++ * Return: Bits [@offset, @offset + @size) from @value. ++ */ ++/* from mali_cdsb.h */ ++#define KBASE_UBFX32(value, offset, size) \ ++ (((u32)(value) >> (u32)(offset)) & (u32)((1ULL << (u32)(size)) - 1)) ++ ++int kbase_gpuprops_uk_get_props(struct kbase_context *kctx, struct kbase_uk_gpuprops * const kbase_props) ++{ ++ kbase_gpu_clk_speed_func get_gpu_speed_mhz; ++ u32 gpu_speed_mhz; ++ int rc = 1; ++ ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ KBASE_DEBUG_ASSERT(NULL != kbase_props); ++ ++ /* Current GPU speed is requested from the system integrator via the GPU_SPEED_FUNC function. ++ * If that function fails, or the function is not provided by the system integrator, we report the maximum ++ * GPU speed as specified by GPU_FREQ_KHZ_MAX. ++ */ ++ get_gpu_speed_mhz = (kbase_gpu_clk_speed_func) GPU_SPEED_FUNC; ++ if (get_gpu_speed_mhz != NULL) { ++ rc = get_gpu_speed_mhz(&gpu_speed_mhz); ++#ifdef CONFIG_MALI_DEBUG ++ /* Issue a warning message when the reported GPU speed falls outside the min/max range */ ++ if (rc == 0) { ++ u32 gpu_speed_khz = gpu_speed_mhz * 1000; ++ ++ if (gpu_speed_khz < kctx->kbdev->gpu_props.props.core_props.gpu_freq_khz_min || ++ gpu_speed_khz > kctx->kbdev->gpu_props.props.core_props.gpu_freq_khz_max) ++ dev_warn(kctx->kbdev->dev, "GPU Speed is outside of min/max range (got %lu Khz, min %lu Khz, max %lu Khz)\n", ++ (unsigned long)gpu_speed_khz, ++ (unsigned long)kctx->kbdev->gpu_props.props.core_props.gpu_freq_khz_min, ++ (unsigned long)kctx->kbdev->gpu_props.props.core_props.gpu_freq_khz_max); ++ } ++#endif /* CONFIG_MALI_DEBUG */ ++ } ++ if (kctx->kbdev->clock) { ++ gpu_speed_mhz = clk_get_rate(kctx->kbdev->clock) / 1000000; ++ rc = 0; ++ } ++ if (rc != 0) ++ gpu_speed_mhz = kctx->kbdev->gpu_props.props.core_props.gpu_freq_khz_max / 1000; ++ ++ kctx->kbdev->gpu_props.props.core_props.gpu_speed_mhz = gpu_speed_mhz; ++ ++ memcpy(&kbase_props->props, &kctx->kbdev->gpu_props.props, sizeof(kbase_props->props)); ++ ++ /* Before API 8.2 they expect L3 cache info here, which was always 0 */ ++ if (kctx->api_version < KBASE_API_VERSION(8, 2)) ++ kbase_props->props.raw_props.suspend_size = 0; ++ ++ return 0; ++} ++ ++static void kbase_gpuprops_construct_coherent_groups(base_gpu_props * const props) ++{ ++ struct mali_base_gpu_coherent_group *current_group; ++ u64 group_present; ++ u64 group_mask; ++ u64 first_set, first_set_prev; ++ u32 num_groups = 0; ++ ++ KBASE_DEBUG_ASSERT(NULL != props); ++ ++ props->coherency_info.coherency = props->raw_props.mem_features; ++ props->coherency_info.num_core_groups = hweight64(props->raw_props.l2_present); ++ ++ if (props->coherency_info.coherency & GROUPS_L2_COHERENT) { ++ /* Group is l2 coherent */ ++ group_present = props->raw_props.l2_present; ++ } else { ++ /* Group is l1 coherent */ ++ group_present = props->raw_props.shader_present; ++ } ++ ++ /* ++ * The coherent group mask can be computed from the l2 present ++ * register. ++ * ++ * For the coherent group n: ++ * group_mask[n] = (first_set[n] - 1) & ~(first_set[n-1] - 1) ++ * where first_set is group_present with only its nth set-bit kept ++ * (i.e. the position from where a new group starts). ++ * ++ * For instance if the groups are l2 coherent and l2_present=0x0..01111: ++ * The first mask is: ++ * group_mask[1] = (first_set[1] - 1) & ~(first_set[0] - 1) ++ * = (0x0..010 - 1) & ~(0x0..01 - 1) ++ * = 0x0..00f ++ * The second mask is: ++ * group_mask[2] = (first_set[2] - 1) & ~(first_set[1] - 1) ++ * = (0x0..100 - 1) & ~(0x0..010 - 1) ++ * = 0x0..0f0 ++ * And so on until all the bits from group_present have been cleared ++ * (i.e. there is no group left). ++ */ ++ ++ current_group = props->coherency_info.group; ++ first_set = group_present & ~(group_present - 1); ++ ++ while (group_present != 0 && num_groups < BASE_MAX_COHERENT_GROUPS) { ++ group_present -= first_set; /* Clear the current group bit */ ++ first_set_prev = first_set; ++ ++ first_set = group_present & ~(group_present - 1); ++ group_mask = (first_set - 1) & ~(first_set_prev - 1); ++ ++ /* Populate the coherent_group structure for each group */ ++ current_group->core_mask = group_mask & props->raw_props.shader_present; ++ current_group->num_cores = hweight64(current_group->core_mask); ++ ++ num_groups++; ++ current_group++; ++ } ++ ++ if (group_present != 0) ++ pr_warn("Too many coherent groups (keeping only %d groups).\n", BASE_MAX_COHERENT_GROUPS); ++ ++ props->coherency_info.num_groups = num_groups; ++} ++ ++/** ++ * kbase_gpuprops_get_props - Get the GPU configuration ++ * @gpu_props: The &base_gpu_props structure ++ * @kbdev: The &struct kbase_device structure for the device ++ * ++ * Fill the &base_gpu_props structure with values from the GPU configuration ++ * registers. Only the raw properties are filled in this function ++ */ ++static void kbase_gpuprops_get_props(base_gpu_props * const gpu_props, struct kbase_device *kbdev) ++{ ++ struct kbase_gpuprops_regdump regdump; ++ int i; ++ ++ KBASE_DEBUG_ASSERT(NULL != kbdev); ++ KBASE_DEBUG_ASSERT(NULL != gpu_props); ++ ++ /* Dump relevant registers */ ++ kbase_backend_gpuprops_get(kbdev, ®dump); ++ ++ gpu_props->raw_props.gpu_id = regdump.gpu_id; ++ gpu_props->raw_props.tiler_features = regdump.tiler_features; ++ gpu_props->raw_props.mem_features = regdump.mem_features; ++ gpu_props->raw_props.mmu_features = regdump.mmu_features; ++ gpu_props->raw_props.l2_features = regdump.l2_features; ++ gpu_props->raw_props.suspend_size = regdump.suspend_size; ++ ++ gpu_props->raw_props.as_present = regdump.as_present; ++ gpu_props->raw_props.js_present = regdump.js_present; ++ gpu_props->raw_props.shader_present = ++ ((u64) regdump.shader_present_hi << 32) + ++ regdump.shader_present_lo; ++ gpu_props->raw_props.tiler_present = ++ ((u64) regdump.tiler_present_hi << 32) + ++ regdump.tiler_present_lo; ++ gpu_props->raw_props.l2_present = ++ ((u64) regdump.l2_present_hi << 32) + ++ regdump.l2_present_lo; ++#ifdef CONFIG_MALI_CORESTACK ++ gpu_props->raw_props.stack_present = ++ ((u64) regdump.stack_present_hi << 32) + ++ regdump.stack_present_lo; ++#else /* CONFIG_MALI_CORESTACK */ ++ gpu_props->raw_props.stack_present = 0; ++#endif /* CONFIG_MALI_CORESTACK */ ++ ++ for (i = 0; i < GPU_MAX_JOB_SLOTS; i++) ++ gpu_props->raw_props.js_features[i] = regdump.js_features[i]; ++ ++ for (i = 0; i < BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS; i++) ++ gpu_props->raw_props.texture_features[i] = regdump.texture_features[i]; ++ ++ gpu_props->raw_props.thread_max_barrier_size = regdump.thread_max_barrier_size; ++ gpu_props->raw_props.thread_max_threads = regdump.thread_max_threads; ++ gpu_props->raw_props.thread_max_workgroup_size = regdump.thread_max_workgroup_size; ++ gpu_props->raw_props.thread_features = regdump.thread_features; ++} ++ ++void kbase_gpuprops_update_core_props_gpu_id(base_gpu_props * const gpu_props) ++{ ++ gpu_props->core_props.version_status = KBASE_UBFX32(gpu_props->raw_props.gpu_id, 0U, 4); ++ gpu_props->core_props.minor_revision = KBASE_UBFX32(gpu_props->raw_props.gpu_id, 4U, 8); ++ gpu_props->core_props.major_revision = KBASE_UBFX32(gpu_props->raw_props.gpu_id, 12U, 4); ++ gpu_props->core_props.product_id = KBASE_UBFX32(gpu_props->raw_props.gpu_id, 16U, 16); ++} ++ ++/** ++ * kbase_gpuprops_calculate_props - Calculate the derived properties ++ * @gpu_props: The &base_gpu_props structure ++ * @kbdev: The &struct kbase_device structure for the device ++ * ++ * Fill the &base_gpu_props structure with values derived from the GPU ++ * configuration registers ++ */ ++static void kbase_gpuprops_calculate_props(base_gpu_props * const gpu_props, struct kbase_device *kbdev) ++{ ++ int i; ++ ++ /* Populate the base_gpu_props structure */ ++ kbase_gpuprops_update_core_props_gpu_id(gpu_props); ++ gpu_props->core_props.log2_program_counter_size = KBASE_GPU_PC_SIZE_LOG2; ++ gpu_props->core_props.gpu_available_memory_size = totalram_pages() << PAGE_SHIFT; ++ ++ for (i = 0; i < BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS; i++) ++ gpu_props->core_props.texture_features[i] = gpu_props->raw_props.texture_features[i]; ++ ++ gpu_props->l2_props.log2_line_size = KBASE_UBFX32(gpu_props->raw_props.l2_features, 0U, 8); ++ gpu_props->l2_props.log2_cache_size = KBASE_UBFX32(gpu_props->raw_props.l2_features, 16U, 8); ++ ++ /* Field with number of l2 slices is added to MEM_FEATURES register ++ * since t76x. Below code assumes that for older GPU reserved bits will ++ * be read as zero. */ ++ gpu_props->l2_props.num_l2_slices = ++ KBASE_UBFX32(gpu_props->raw_props.mem_features, 8U, 4) + 1; ++ ++ gpu_props->tiler_props.bin_size_bytes = 1 << KBASE_UBFX32(gpu_props->raw_props.tiler_features, 0U, 6); ++ gpu_props->tiler_props.max_active_levels = KBASE_UBFX32(gpu_props->raw_props.tiler_features, 8U, 4); ++ ++ if (gpu_props->raw_props.thread_max_threads == 0) ++ gpu_props->thread_props.max_threads = THREAD_MT_DEFAULT; ++ else ++ gpu_props->thread_props.max_threads = gpu_props->raw_props.thread_max_threads; ++ ++ if (gpu_props->raw_props.thread_max_workgroup_size == 0) ++ gpu_props->thread_props.max_workgroup_size = THREAD_MWS_DEFAULT; ++ else ++ gpu_props->thread_props.max_workgroup_size = gpu_props->raw_props.thread_max_workgroup_size; ++ ++ if (gpu_props->raw_props.thread_max_barrier_size == 0) ++ gpu_props->thread_props.max_barrier_size = THREAD_MBS_DEFAULT; ++ else ++ gpu_props->thread_props.max_barrier_size = gpu_props->raw_props.thread_max_barrier_size; ++ ++ gpu_props->thread_props.max_registers = KBASE_UBFX32(gpu_props->raw_props.thread_features, 0U, 16); ++ gpu_props->thread_props.max_task_queue = KBASE_UBFX32(gpu_props->raw_props.thread_features, 16U, 8); ++ gpu_props->thread_props.max_thread_group_split = KBASE_UBFX32(gpu_props->raw_props.thread_features, 24U, 6); ++ gpu_props->thread_props.impl_tech = KBASE_UBFX32(gpu_props->raw_props.thread_features, 30U, 2); ++ ++ /* If values are not specified, then use defaults */ ++ if (gpu_props->thread_props.max_registers == 0) { ++ gpu_props->thread_props.max_registers = THREAD_MR_DEFAULT; ++ gpu_props->thread_props.max_task_queue = THREAD_MTQ_DEFAULT; ++ gpu_props->thread_props.max_thread_group_split = THREAD_MTGS_DEFAULT; ++ } ++ /* Initialize the coherent_group structure for each group */ ++ kbase_gpuprops_construct_coherent_groups(gpu_props); ++} ++ ++void kbase_gpuprops_set(struct kbase_device *kbdev) ++{ ++ struct kbase_gpu_props *gpu_props; ++ struct gpu_raw_gpu_props *raw; ++ ++ KBASE_DEBUG_ASSERT(NULL != kbdev); ++ gpu_props = &kbdev->gpu_props; ++ raw = &gpu_props->props.raw_props; ++ ++ /* Initialize the base_gpu_props structure from the hardware */ ++ kbase_gpuprops_get_props(&gpu_props->props, kbdev); ++ ++ /* Populate the derived properties */ ++ kbase_gpuprops_calculate_props(&gpu_props->props, kbdev); ++ ++ /* Populate kbase-only fields */ ++ gpu_props->l2_props.associativity = KBASE_UBFX32(raw->l2_features, 8U, 8); ++ gpu_props->l2_props.external_bus_width = KBASE_UBFX32(raw->l2_features, 24U, 8); ++ ++ gpu_props->mem.core_group = KBASE_UBFX32(raw->mem_features, 0U, 1); ++ ++ gpu_props->mmu.va_bits = KBASE_UBFX32(raw->mmu_features, 0U, 8); ++ gpu_props->mmu.pa_bits = KBASE_UBFX32(raw->mmu_features, 8U, 8); ++ ++ gpu_props->num_cores = hweight64(raw->shader_present); ++ gpu_props->num_core_groups = hweight64(raw->l2_present); ++ gpu_props->num_address_spaces = hweight32(raw->as_present); ++ gpu_props->num_job_slots = hweight32(raw->js_present); ++} ++ ++void kbase_gpuprops_set_features(struct kbase_device *kbdev) ++{ ++ base_gpu_props *gpu_props; ++ struct kbase_gpuprops_regdump regdump; ++ ++ gpu_props = &kbdev->gpu_props.props; ++ ++ /* Dump relevant registers */ ++ kbase_backend_gpuprops_get_features(kbdev, ®dump); ++ ++ /* ++ * Copy the raw value from the register, later this will get turned ++ * into the selected coherency mode. ++ * Additionally, add non-coherent mode, as this is always supported. ++ */ ++ gpu_props->raw_props.coherency_mode = regdump.coherency_features | ++ COHERENCY_FEATURE_BIT(COHERENCY_NONE); ++} ++ ++static struct { ++ u32 type; ++ size_t offset; ++ int size; ++} gpu_property_mapping[] = { ++#define PROP(name, member) \ ++ {KBASE_GPUPROP_ ## name, offsetof(struct mali_base_gpu_props, member), \ ++ sizeof(((struct mali_base_gpu_props *)0)->member)} ++ PROP(PRODUCT_ID, core_props.product_id), ++ PROP(VERSION_STATUS, core_props.version_status), ++ PROP(MINOR_REVISION, core_props.minor_revision), ++ PROP(MAJOR_REVISION, core_props.major_revision), ++ PROP(GPU_SPEED_MHZ, core_props.gpu_speed_mhz), ++ PROP(GPU_FREQ_KHZ_MAX, core_props.gpu_freq_khz_max), ++ PROP(GPU_FREQ_KHZ_MIN, core_props.gpu_freq_khz_min), ++ PROP(LOG2_PROGRAM_COUNTER_SIZE, core_props.log2_program_counter_size), ++ PROP(TEXTURE_FEATURES_0, core_props.texture_features[0]), ++ PROP(TEXTURE_FEATURES_1, core_props.texture_features[1]), ++ PROP(TEXTURE_FEATURES_2, core_props.texture_features[2]), ++ PROP(GPU_AVAILABLE_MEMORY_SIZE, core_props.gpu_available_memory_size), ++ ++ PROP(L2_LOG2_LINE_SIZE, l2_props.log2_line_size), ++ PROP(L2_LOG2_CACHE_SIZE, l2_props.log2_cache_size), ++ PROP(L2_NUM_L2_SLICES, l2_props.num_l2_slices), ++ ++ PROP(TILER_BIN_SIZE_BYTES, tiler_props.bin_size_bytes), ++ PROP(TILER_MAX_ACTIVE_LEVELS, tiler_props.max_active_levels), ++ ++ PROP(MAX_THREADS, thread_props.max_threads), ++ PROP(MAX_WORKGROUP_SIZE, thread_props.max_workgroup_size), ++ PROP(MAX_BARRIER_SIZE, thread_props.max_barrier_size), ++ PROP(MAX_REGISTERS, thread_props.max_registers), ++ PROP(MAX_TASK_QUEUE, thread_props.max_task_queue), ++ PROP(MAX_THREAD_GROUP_SPLIT, thread_props.max_thread_group_split), ++ PROP(IMPL_TECH, thread_props.impl_tech), ++ ++ PROP(RAW_SHADER_PRESENT, raw_props.shader_present), ++ PROP(RAW_TILER_PRESENT, raw_props.tiler_present), ++ PROP(RAW_L2_PRESENT, raw_props.l2_present), ++ PROP(RAW_STACK_PRESENT, raw_props.stack_present), ++ PROP(RAW_L2_FEATURES, raw_props.l2_features), ++ PROP(RAW_SUSPEND_SIZE, raw_props.suspend_size), ++ PROP(RAW_MEM_FEATURES, raw_props.mem_features), ++ PROP(RAW_MMU_FEATURES, raw_props.mmu_features), ++ PROP(RAW_AS_PRESENT, raw_props.as_present), ++ PROP(RAW_JS_PRESENT, raw_props.js_present), ++ PROP(RAW_JS_FEATURES_0, raw_props.js_features[0]), ++ PROP(RAW_JS_FEATURES_1, raw_props.js_features[1]), ++ PROP(RAW_JS_FEATURES_2, raw_props.js_features[2]), ++ PROP(RAW_JS_FEATURES_3, raw_props.js_features[3]), ++ PROP(RAW_JS_FEATURES_4, raw_props.js_features[4]), ++ PROP(RAW_JS_FEATURES_5, raw_props.js_features[5]), ++ PROP(RAW_JS_FEATURES_6, raw_props.js_features[6]), ++ PROP(RAW_JS_FEATURES_7, raw_props.js_features[7]), ++ PROP(RAW_JS_FEATURES_8, raw_props.js_features[8]), ++ PROP(RAW_JS_FEATURES_9, raw_props.js_features[9]), ++ PROP(RAW_JS_FEATURES_10, raw_props.js_features[10]), ++ PROP(RAW_JS_FEATURES_11, raw_props.js_features[11]), ++ PROP(RAW_JS_FEATURES_12, raw_props.js_features[12]), ++ PROP(RAW_JS_FEATURES_13, raw_props.js_features[13]), ++ PROP(RAW_JS_FEATURES_14, raw_props.js_features[14]), ++ PROP(RAW_JS_FEATURES_15, raw_props.js_features[15]), ++ PROP(RAW_TILER_FEATURES, raw_props.tiler_features), ++ PROP(RAW_TEXTURE_FEATURES_0, raw_props.texture_features[0]), ++ PROP(RAW_TEXTURE_FEATURES_1, raw_props.texture_features[1]), ++ PROP(RAW_TEXTURE_FEATURES_2, raw_props.texture_features[2]), ++ PROP(RAW_GPU_ID, raw_props.gpu_id), ++ PROP(RAW_THREAD_MAX_THREADS, raw_props.thread_max_threads), ++ PROP(RAW_THREAD_MAX_WORKGROUP_SIZE, ++ raw_props.thread_max_workgroup_size), ++ PROP(RAW_THREAD_MAX_BARRIER_SIZE, raw_props.thread_max_barrier_size), ++ PROP(RAW_THREAD_FEATURES, raw_props.thread_features), ++ PROP(RAW_COHERENCY_MODE, raw_props.coherency_mode), ++ ++ PROP(COHERENCY_NUM_GROUPS, coherency_info.num_groups), ++ PROP(COHERENCY_NUM_CORE_GROUPS, coherency_info.num_core_groups), ++ PROP(COHERENCY_COHERENCY, coherency_info.coherency), ++ PROP(COHERENCY_GROUP_0, coherency_info.group[0].core_mask), ++ PROP(COHERENCY_GROUP_1, coherency_info.group[1].core_mask), ++ PROP(COHERENCY_GROUP_2, coherency_info.group[2].core_mask), ++ PROP(COHERENCY_GROUP_3, coherency_info.group[3].core_mask), ++ PROP(COHERENCY_GROUP_4, coherency_info.group[4].core_mask), ++ PROP(COHERENCY_GROUP_5, coherency_info.group[5].core_mask), ++ PROP(COHERENCY_GROUP_6, coherency_info.group[6].core_mask), ++ PROP(COHERENCY_GROUP_7, coherency_info.group[7].core_mask), ++ PROP(COHERENCY_GROUP_8, coherency_info.group[8].core_mask), ++ PROP(COHERENCY_GROUP_9, coherency_info.group[9].core_mask), ++ PROP(COHERENCY_GROUP_10, coherency_info.group[10].core_mask), ++ PROP(COHERENCY_GROUP_11, coherency_info.group[11].core_mask), ++ PROP(COHERENCY_GROUP_12, coherency_info.group[12].core_mask), ++ PROP(COHERENCY_GROUP_13, coherency_info.group[13].core_mask), ++ PROP(COHERENCY_GROUP_14, coherency_info.group[14].core_mask), ++ PROP(COHERENCY_GROUP_15, coherency_info.group[15].core_mask), ++ ++#undef PROP ++}; ++ ++int kbase_gpuprops_populate_user_buffer(struct kbase_device *kbdev) ++{ ++ struct kbase_gpu_props *kprops = &kbdev->gpu_props; ++ struct mali_base_gpu_props *props = &kprops->props; ++ u32 count = ARRAY_SIZE(gpu_property_mapping); ++ u32 i; ++ u32 size = 0; ++ u8 *p; ++ ++ for (i = 0; i < count; i++) { ++ /* 4 bytes for the ID, and the size of the property */ ++ size += 4 + gpu_property_mapping[i].size; ++ } ++ ++ kprops->prop_buffer_size = size; ++ kprops->prop_buffer = kmalloc(size, GFP_KERNEL); ++ ++ if (!kprops->prop_buffer) { ++ kprops->prop_buffer_size = 0; ++ return -ENOMEM; ++ } ++ ++ p = kprops->prop_buffer; ++ ++#define WRITE_U8(v) (*p++ = (v) & 0xFF) ++#define WRITE_U16(v) do { WRITE_U8(v); WRITE_U8((v) >> 8); } while (0) ++#define WRITE_U32(v) do { WRITE_U16(v); WRITE_U16((v) >> 16); } while (0) ++#define WRITE_U64(v) do { WRITE_U32(v); WRITE_U32((v) >> 32); } while (0) ++ ++ for (i = 0; i < count; i++) { ++ u32 type = gpu_property_mapping[i].type; ++ u8 type_size; ++ void *field = ((u8 *)props) + gpu_property_mapping[i].offset; ++ ++ switch (gpu_property_mapping[i].size) { ++ case 1: ++ type_size = KBASE_GPUPROP_VALUE_SIZE_U8; ++ break; ++ case 2: ++ type_size = KBASE_GPUPROP_VALUE_SIZE_U16; ++ break; ++ case 4: ++ type_size = KBASE_GPUPROP_VALUE_SIZE_U32; ++ break; ++ case 8: ++ type_size = KBASE_GPUPROP_VALUE_SIZE_U64; ++ break; ++ default: ++ dev_err(kbdev->dev, ++ "Invalid gpu_property_mapping type=%d size=%d", ++ type, gpu_property_mapping[i].size); ++ return -EINVAL; ++ } ++ ++ WRITE_U32((type<<2) | type_size); ++ ++ switch (type_size) { ++ case KBASE_GPUPROP_VALUE_SIZE_U8: ++ WRITE_U8(*((u8 *)field)); ++ break; ++ case KBASE_GPUPROP_VALUE_SIZE_U16: ++ WRITE_U16(*((u16 *)field)); ++ break; ++ case KBASE_GPUPROP_VALUE_SIZE_U32: ++ WRITE_U32(*((u32 *)field)); ++ break; ++ case KBASE_GPUPROP_VALUE_SIZE_U64: ++ WRITE_U64(*((u64 *)field)); ++ break; ++ default: /* Cannot be reached */ ++ WARN_ON(1); ++ return -EINVAL; ++ } ++ } ++ ++ return 0; ++} +diff --git a/drivers/gpu/arm/midgard/mali_kbase_gpuprops.h b/drivers/gpu/arm/midgard/mali_kbase_gpuprops.h +new file mode 100755 +index 000000000000..57b3eaf9cd53 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_gpuprops.h +@@ -0,0 +1,84 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2015,2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/** ++ * @file mali_kbase_gpuprops.h ++ * Base kernel property query APIs ++ */ ++ ++#ifndef _KBASE_GPUPROPS_H_ ++#define _KBASE_GPUPROPS_H_ ++ ++#include "mali_kbase_gpuprops_types.h" ++ ++/* Forward definition - see mali_kbase.h */ ++struct kbase_device; ++ ++/** ++ * @brief Set up Kbase GPU properties. ++ * ++ * Set up Kbase GPU properties with information from the GPU registers ++ * ++ * @param kbdev The struct kbase_device structure for the device ++ */ ++void kbase_gpuprops_set(struct kbase_device *kbdev); ++ ++/** ++ * kbase_gpuprops_set_features - Set up Kbase GPU properties ++ * @kbdev: Device pointer ++ * ++ * This function sets up GPU properties that are dependent on the hardware ++ * features bitmask. This function must be preceeded by a call to ++ * kbase_hw_set_features_mask(). ++ */ ++void kbase_gpuprops_set_features(struct kbase_device *kbdev); ++ ++/** ++ * @brief Provide GPU properties to userside through UKU call. ++ * ++ * Fill the struct kbase_uk_gpuprops with values from GPU configuration registers. ++ * ++ * @param kctx The struct kbase_context structure ++ * @param kbase_props A copy of the struct kbase_uk_gpuprops structure from userspace ++ * ++ * @return 0 on success. Any other value indicates failure. ++ */ ++int kbase_gpuprops_uk_get_props(struct kbase_context *kctx, struct kbase_uk_gpuprops * const kbase_props); ++ ++/** ++ * kbase_gpuprops_populate_user_buffer - Populate the GPU properties buffer ++ * @kbdev: The kbase device ++ * ++ * Fills kbdev->gpu_props->prop_buffer with the GPU properties for user ++ * space to read. ++ */ ++int kbase_gpuprops_populate_user_buffer(struct kbase_device *kbdev); ++ ++/** ++ * kbase_gpuprops_update_core_props_gpu_id - break down gpu id value ++ * @gpu_props: the &base_gpu_props structure ++ * ++ * Break down gpu_id value stored in base_gpu_props::raw_props.gpu_id into ++ * separate fields (version_status, minor_revision, major_revision, product_id) ++ * stored in base_gpu_props::core_props. ++ */ ++void kbase_gpuprops_update_core_props_gpu_id(base_gpu_props * const gpu_props); ++ ++ ++#endif /* _KBASE_GPUPROPS_H_ */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_gpuprops_types.h b/drivers/gpu/arm/midgard/mali_kbase_gpuprops_types.h +new file mode 100755 +index 000000000000..10794fc27318 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_gpuprops_types.h +@@ -0,0 +1,92 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/** ++ * @file mali_kbase_gpuprops_types.h ++ * Base kernel property query APIs ++ */ ++ ++#ifndef _KBASE_GPUPROPS_TYPES_H_ ++#define _KBASE_GPUPROPS_TYPES_H_ ++ ++#include "mali_base_kernel.h" ++ ++#define KBASE_GPU_SPEED_MHZ 123 ++#define KBASE_GPU_PC_SIZE_LOG2 24U ++ ++struct kbase_gpuprops_regdump { ++ u32 gpu_id; ++ u32 l2_features; ++ u32 suspend_size; /* API 8.2+ */ ++ u32 tiler_features; ++ u32 mem_features; ++ u32 mmu_features; ++ u32 as_present; ++ u32 js_present; ++ u32 thread_max_threads; ++ u32 thread_max_workgroup_size; ++ u32 thread_max_barrier_size; ++ u32 thread_features; ++ u32 texture_features[BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS]; ++ u32 js_features[GPU_MAX_JOB_SLOTS]; ++ u32 shader_present_lo; ++ u32 shader_present_hi; ++ u32 tiler_present_lo; ++ u32 tiler_present_hi; ++ u32 l2_present_lo; ++ u32 l2_present_hi; ++ u32 stack_present_lo; ++ u32 stack_present_hi; ++ u32 coherency_features; ++}; ++ ++struct kbase_gpu_cache_props { ++ u8 associativity; ++ u8 external_bus_width; ++}; ++ ++struct kbase_gpu_mem_props { ++ u8 core_group; ++}; ++ ++struct kbase_gpu_mmu_props { ++ u8 va_bits; ++ u8 pa_bits; ++}; ++ ++struct kbase_gpu_props { ++ /* kernel-only properties */ ++ u8 num_cores; ++ u8 num_core_groups; ++ u8 num_address_spaces; ++ u8 num_job_slots; ++ ++ struct kbase_gpu_cache_props l2_props; ++ ++ struct kbase_gpu_mem_props mem; ++ struct kbase_gpu_mmu_props mmu; ++ ++ /* Properties shared with userspace */ ++ base_gpu_props props; ++ ++ u32 prop_buffer_size; ++ void *prop_buffer; ++}; ++ ++#endif /* _KBASE_GPUPROPS_TYPES_H_ */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_hw.c b/drivers/gpu/arm/midgard/mali_kbase_hw.c +new file mode 100755 +index 000000000000..9a390d233939 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_hw.c +@@ -0,0 +1,453 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * Run-time work-arounds helpers ++ */ ++ ++#include ++#include ++#include ++#include "mali_kbase.h" ++#include "mali_kbase_hw.h" ++ ++void kbase_hw_set_features_mask(struct kbase_device *kbdev) ++{ ++ const enum base_hw_feature *features; ++ u32 gpu_id; ++ u32 product_id; ++ ++ gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; ++ product_id = gpu_id & GPU_ID_VERSION_PRODUCT_ID; ++ product_id >>= GPU_ID_VERSION_PRODUCT_ID_SHIFT; ++ ++ if (GPU_ID_IS_NEW_FORMAT(product_id)) { ++ switch (gpu_id & GPU_ID2_PRODUCT_MODEL) { ++ case GPU_ID2_PRODUCT_TMIX: ++ features = base_hw_features_tMIx; ++ break; ++ case GPU_ID2_PRODUCT_THEX: ++ features = base_hw_features_tHEx; ++ break; ++ case GPU_ID2_PRODUCT_TSIX: ++ features = base_hw_features_tSIx; ++ break; ++#ifdef MALI_INCLUDE_TKAX ++ case GPU_ID2_PRODUCT_TKAX: ++ features = base_hw_features_tKAx; ++ break; ++#endif /* MALI_INCLUDE_TKAX */ ++#ifdef MALI_INCLUDE_TTRX ++ case GPU_ID2_PRODUCT_TTRX: ++ features = base_hw_features_tTRx; ++ break; ++#endif /* MALI_INCLUDE_TTRX */ ++ default: ++ features = base_hw_features_generic; ++ break; ++ } ++ } else { ++ switch (product_id) { ++ case GPU_ID_PI_TFRX: ++ /* FALLTHROUGH */ ++ case GPU_ID_PI_T86X: ++ features = base_hw_features_tFxx; ++ break; ++ case GPU_ID_PI_T83X: ++ features = base_hw_features_t83x; ++ break; ++ case GPU_ID_PI_T82X: ++ features = base_hw_features_t82x; ++ break; ++ case GPU_ID_PI_T76X: ++ features = base_hw_features_t76x; ++ break; ++ case GPU_ID_PI_T72X: ++ features = base_hw_features_t72x; ++ break; ++ case GPU_ID_PI_T62X: ++ features = base_hw_features_t62x; ++ break; ++ case GPU_ID_PI_T60X: ++ features = base_hw_features_t60x; ++ break; ++ default: ++ features = base_hw_features_generic; ++ break; ++ } ++ } ++ ++ for (; *features != BASE_HW_FEATURE_END; features++) ++ set_bit(*features, &kbdev->hw_features_mask[0]); ++} ++ ++/** ++ * kbase_hw_get_issues_for_new_id - Get the hardware issues for a new GPU ID ++ * @kbdev: Device pointer ++ * ++ * Return: pointer to an array of hardware issues, terminated by ++ * BASE_HW_ISSUE_END. ++ * ++ * This function can only be used on new-format GPU IDs, i.e. those for which ++ * GPU_ID_IS_NEW_FORMAT evaluates as true. The GPU ID is read from the @kbdev. ++ * ++ * In debugging versions of the driver, unknown versions of a known GPU will ++ * be treated as the most recent known version not later than the actual ++ * version. In such circumstances, the GPU ID in @kbdev will also be replaced ++ * with the most recent known version. ++ * ++ * Note: The GPU configuration must have been read by kbase_gpuprops_get_props() ++ * before calling this function. ++ */ ++static const enum base_hw_issue *kbase_hw_get_issues_for_new_id( ++ struct kbase_device *kbdev) ++{ ++ const enum base_hw_issue *issues = NULL; ++ ++ struct base_hw_product { ++ u32 product_model; ++ struct { ++ u32 version; ++ const enum base_hw_issue *issues; ++ } map[7]; ++ }; ++ ++ static const struct base_hw_product base_hw_products[] = { ++ {GPU_ID2_PRODUCT_TMIX, ++ {{GPU_ID2_VERSION_MAKE(0, 0, 1), ++ base_hw_issues_tMIx_r0p0_05dev0}, ++ {GPU_ID2_VERSION_MAKE(0, 0, 2), base_hw_issues_tMIx_r0p0}, ++ {U32_MAX /* sentinel value */, NULL} } }, ++ ++ {GPU_ID2_PRODUCT_THEX, ++ {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tHEx_r0p0}, ++ {GPU_ID2_VERSION_MAKE(0, 0, 1), base_hw_issues_tHEx_r0p0}, ++ {GPU_ID2_VERSION_MAKE(0, 1, 0), base_hw_issues_tHEx_r0p1}, ++ {U32_MAX, NULL} } }, ++ ++ {GPU_ID2_PRODUCT_TSIX, ++ {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tSIx_r0p0}, ++ {GPU_ID2_VERSION_MAKE(0, 0, 1), base_hw_issues_tSIx_r0p0}, ++ {GPU_ID2_VERSION_MAKE(0, 1, 0), base_hw_issues_tSIx_r0p1}, ++ {GPU_ID2_VERSION_MAKE(0, 1, 1), base_hw_issues_tSIx_r0p1}, ++ {GPU_ID2_VERSION_MAKE(1, 0, 0), base_hw_issues_tSIx_r1p0}, ++ {GPU_ID2_VERSION_MAKE(1, 0, 1), base_hw_issues_tSIx_r1p0}, ++ {U32_MAX, NULL} } }, ++ ++ ++#ifdef MALI_INCLUDE_TKAX ++ {GPU_ID2_PRODUCT_TKAX, ++ {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tKAx_r0p0}, ++ {U32_MAX, NULL} } }, ++#endif /* MALI_INCLUDE_TKAX */ ++ ++#ifdef MALI_INCLUDE_TTRX ++ {GPU_ID2_PRODUCT_TTRX, ++ {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tTRx_r0p0}, ++ {U32_MAX, NULL} } }, ++#endif /* MALI_INCLUDE_TTRX */ ++ }; ++ ++ u32 gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; ++ const u32 product_model = gpu_id & GPU_ID2_PRODUCT_MODEL; ++ const struct base_hw_product *product = NULL; ++ size_t p; ++ ++ /* Stop when we reach the end of the products array. */ ++ for (p = 0; p < ARRAY_SIZE(base_hw_products); ++p) { ++ if (product_model == base_hw_products[p].product_model) { ++ product = &base_hw_products[p]; ++ break; ++ } ++ } ++ ++ if (product != NULL) { ++ /* Found a matching product. */ ++ const u32 version = gpu_id & GPU_ID2_VERSION; ++ u32 fallback_version = 0; ++ const enum base_hw_issue *fallback_issues = NULL; ++ size_t v; ++ ++ /* Stop when we reach the end of the map. */ ++ for (v = 0; product->map[v].version != U32_MAX; ++v) { ++ ++ if (version == product->map[v].version) { ++ /* Exact match so stop. */ ++ issues = product->map[v].issues; ++ break; ++ } ++ ++ /* Check whether this is a candidate for most recent ++ known version not later than the actual ++ version. */ ++ if ((version > product->map[v].version) && ++ (product->map[v].version >= fallback_version)) { ++ fallback_version = product->map[v].version; ++ fallback_issues = product->map[v].issues; ++ } ++ } ++ ++ if ((issues == NULL) && (fallback_issues != NULL)) { ++ /* Fall back to the issue set of the most recent known ++ version not later than the actual version. */ ++ issues = fallback_issues; ++ ++ dev_info(kbdev->dev, ++ "r%dp%d status %d is unknown; treating as r%dp%d status %d", ++ (gpu_id & GPU_ID2_VERSION_MAJOR) >> ++ GPU_ID2_VERSION_MAJOR_SHIFT, ++ (gpu_id & GPU_ID2_VERSION_MINOR) >> ++ GPU_ID2_VERSION_MINOR_SHIFT, ++ (gpu_id & GPU_ID2_VERSION_STATUS) >> ++ GPU_ID2_VERSION_STATUS_SHIFT, ++ (fallback_version & GPU_ID2_VERSION_MAJOR) >> ++ GPU_ID2_VERSION_MAJOR_SHIFT, ++ (fallback_version & GPU_ID2_VERSION_MINOR) >> ++ GPU_ID2_VERSION_MINOR_SHIFT, ++ (fallback_version & GPU_ID2_VERSION_STATUS) >> ++ GPU_ID2_VERSION_STATUS_SHIFT); ++ ++ gpu_id &= ~GPU_ID2_VERSION; ++ gpu_id |= fallback_version; ++ kbdev->gpu_props.props.raw_props.gpu_id = gpu_id; ++ ++ kbase_gpuprops_update_core_props_gpu_id(&kbdev->gpu_props.props); ++ } ++ } ++ return issues; ++} ++ ++int kbase_hw_set_issues_mask(struct kbase_device *kbdev) ++{ ++ const enum base_hw_issue *issues; ++ u32 gpu_id; ++ u32 product_id; ++ u32 impl_tech; ++ ++ gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; ++ product_id = gpu_id & GPU_ID_VERSION_PRODUCT_ID; ++ product_id >>= GPU_ID_VERSION_PRODUCT_ID_SHIFT; ++ impl_tech = kbdev->gpu_props.props.thread_props.impl_tech; ++ ++ if (impl_tech != IMPLEMENTATION_MODEL) { ++ if (GPU_ID_IS_NEW_FORMAT(product_id)) { ++ issues = kbase_hw_get_issues_for_new_id(kbdev); ++ if (issues == NULL) { ++ dev_err(kbdev->dev, ++ "Unknown GPU ID %x", gpu_id); ++ return -EINVAL; ++ } ++ ++ /* The GPU ID might have been replaced with the last ++ known version of the same GPU. */ ++ gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; ++ ++ } else { ++ switch (gpu_id) { ++ case GPU_ID_MAKE(GPU_ID_PI_T60X, 0, 0, GPU_ID_S_15DEV0): ++ issues = base_hw_issues_t60x_r0p0_15dev0; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_T60X, 0, 0, GPU_ID_S_EAC): ++ issues = base_hw_issues_t60x_r0p0_eac; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_T60X, 0, 1, 0): ++ issues = base_hw_issues_t60x_r0p1; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_T62X, 0, 1, 0): ++ issues = base_hw_issues_t62x_r0p1; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_T62X, 1, 0, 0): ++ case GPU_ID_MAKE(GPU_ID_PI_T62X, 1, 0, 1): ++ issues = base_hw_issues_t62x_r1p0; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_T62X, 1, 1, 0): ++ issues = base_hw_issues_t62x_r1p1; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_T76X, 0, 0, 1): ++ issues = base_hw_issues_t76x_r0p0; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_T76X, 0, 1, 1): ++ issues = base_hw_issues_t76x_r0p1; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_T76X, 0, 1, 9): ++ issues = base_hw_issues_t76x_r0p1_50rel0; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_T76X, 0, 2, 1): ++ issues = base_hw_issues_t76x_r0p2; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_T76X, 0, 3, 1): ++ issues = base_hw_issues_t76x_r0p3; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_T76X, 1, 0, 0): ++ issues = base_hw_issues_t76x_r1p0; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_T72X, 0, 0, 0): ++ case GPU_ID_MAKE(GPU_ID_PI_T72X, 0, 0, 1): ++ case GPU_ID_MAKE(GPU_ID_PI_T72X, 0, 0, 2): ++ issues = base_hw_issues_t72x_r0p0; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_T72X, 1, 0, 0): ++ issues = base_hw_issues_t72x_r1p0; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_T72X, 1, 1, 0): ++ issues = base_hw_issues_t72x_r1p1; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_TFRX, 0, 1, 2): ++ issues = base_hw_issues_tFRx_r0p1; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_TFRX, 0, 2, 0): ++ issues = base_hw_issues_tFRx_r0p2; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_TFRX, 1, 0, 0): ++ case GPU_ID_MAKE(GPU_ID_PI_TFRX, 1, 0, 8): ++ issues = base_hw_issues_tFRx_r1p0; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_TFRX, 2, 0, 0): ++ issues = base_hw_issues_tFRx_r2p0; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_T86X, 0, 2, 0): ++ issues = base_hw_issues_t86x_r0p2; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_T86X, 1, 0, 0): ++ case GPU_ID_MAKE(GPU_ID_PI_T86X, 1, 0, 8): ++ issues = base_hw_issues_t86x_r1p0; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_T86X, 2, 0, 0): ++ issues = base_hw_issues_t86x_r2p0; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_T83X, 0, 1, 0): ++ issues = base_hw_issues_t83x_r0p1; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_T83X, 1, 0, 0): ++ case GPU_ID_MAKE(GPU_ID_PI_T83X, 1, 0, 8): ++ issues = base_hw_issues_t83x_r1p0; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_T82X, 0, 0, 0): ++ issues = base_hw_issues_t82x_r0p0; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_T82X, 0, 1, 0): ++ issues = base_hw_issues_t82x_r0p1; ++ break; ++ case GPU_ID_MAKE(GPU_ID_PI_T82X, 1, 0, 0): ++ case GPU_ID_MAKE(GPU_ID_PI_T82X, 1, 0, 8): ++ issues = base_hw_issues_t82x_r1p0; ++ break; ++ default: ++ dev_err(kbdev->dev, ++ "Unknown GPU ID %x", gpu_id); ++ return -EINVAL; ++ } ++ } ++ } else { ++ /* Software model */ ++ if (GPU_ID_IS_NEW_FORMAT(product_id)) { ++ switch (gpu_id & GPU_ID2_PRODUCT_MODEL) { ++ case GPU_ID2_PRODUCT_TMIX: ++ issues = base_hw_issues_model_tMIx; ++ break; ++ case GPU_ID2_PRODUCT_THEX: ++ issues = base_hw_issues_model_tHEx; ++ break; ++ case GPU_ID2_PRODUCT_TSIX: ++ issues = base_hw_issues_model_tSIx; ++ break; ++#ifdef MALI_INCLUDE_TKAX ++ case GPU_ID2_PRODUCT_TKAX: ++ issues = base_hw_issues_model_tKAx; ++ break; ++#endif /* MALI_INCLUDE_TKAX */ ++#ifdef MALI_INCLUDE_TTRX ++ case GPU_ID2_PRODUCT_TTRX: ++ issues = base_hw_issues_model_tTRx; ++ break; ++#endif /* MALI_INCLUDE_TTRX */ ++ default: ++ dev_err(kbdev->dev, ++ "Unknown GPU ID %x", gpu_id); ++ return -EINVAL; ++ } ++ } else { ++ switch (product_id) { ++ case GPU_ID_PI_T60X: ++ issues = base_hw_issues_model_t60x; ++ break; ++ case GPU_ID_PI_T62X: ++ issues = base_hw_issues_model_t62x; ++ break; ++ case GPU_ID_PI_T72X: ++ issues = base_hw_issues_model_t72x; ++ break; ++ case GPU_ID_PI_T76X: ++ issues = base_hw_issues_model_t76x; ++ break; ++ case GPU_ID_PI_TFRX: ++ issues = base_hw_issues_model_tFRx; ++ break; ++ case GPU_ID_PI_T86X: ++ issues = base_hw_issues_model_t86x; ++ break; ++ case GPU_ID_PI_T83X: ++ issues = base_hw_issues_model_t83x; ++ break; ++ case GPU_ID_PI_T82X: ++ issues = base_hw_issues_model_t82x; ++ break; ++ default: ++ dev_err(kbdev->dev, "Unknown GPU ID %x", ++ gpu_id); ++ return -EINVAL; ++ } ++ } ++ } ++ ++ if (GPU_ID_IS_NEW_FORMAT(product_id)) { ++ dev_info(kbdev->dev, ++ "GPU identified as 0x%x arch %d.%d.%d r%dp%d status %d", ++ (gpu_id & GPU_ID2_PRODUCT_MAJOR) >> ++ GPU_ID2_PRODUCT_MAJOR_SHIFT, ++ (gpu_id & GPU_ID2_ARCH_MAJOR) >> ++ GPU_ID2_ARCH_MAJOR_SHIFT, ++ (gpu_id & GPU_ID2_ARCH_MINOR) >> ++ GPU_ID2_ARCH_MINOR_SHIFT, ++ (gpu_id & GPU_ID2_ARCH_REV) >> ++ GPU_ID2_ARCH_REV_SHIFT, ++ (gpu_id & GPU_ID2_VERSION_MAJOR) >> ++ GPU_ID2_VERSION_MAJOR_SHIFT, ++ (gpu_id & GPU_ID2_VERSION_MINOR) >> ++ GPU_ID2_VERSION_MINOR_SHIFT, ++ (gpu_id & GPU_ID2_VERSION_STATUS) >> ++ GPU_ID2_VERSION_STATUS_SHIFT); ++ } else { ++ dev_info(kbdev->dev, ++ "GPU identified as 0x%04x r%dp%d status %d", ++ (gpu_id & GPU_ID_VERSION_PRODUCT_ID) >> ++ GPU_ID_VERSION_PRODUCT_ID_SHIFT, ++ (gpu_id & GPU_ID_VERSION_MAJOR) >> ++ GPU_ID_VERSION_MAJOR_SHIFT, ++ (gpu_id & GPU_ID_VERSION_MINOR) >> ++ GPU_ID_VERSION_MINOR_SHIFT, ++ (gpu_id & GPU_ID_VERSION_STATUS) >> ++ GPU_ID_VERSION_STATUS_SHIFT); ++ } ++ ++ for (; *issues != BASE_HW_ISSUE_END; issues++) ++ set_bit(*issues, &kbdev->hw_issues_mask[0]); ++ ++ return 0; ++} +diff --git a/drivers/gpu/arm/midgard/mali_kbase_hw.h b/drivers/gpu/arm/midgard/mali_kbase_hw.h +new file mode 100755 +index 000000000000..754250ce968d +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_hw.h +@@ -0,0 +1,65 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/** ++ * @file ++ * Run-time work-arounds helpers ++ */ ++ ++#ifndef _KBASE_HW_H_ ++#define _KBASE_HW_H_ ++ ++#include "mali_kbase_defs.h" ++ ++/** ++ * @brief Tell whether a work-around should be enabled ++ */ ++#define kbase_hw_has_issue(kbdev, issue)\ ++ test_bit(issue, &(kbdev)->hw_issues_mask[0]) ++ ++/** ++ * @brief Tell whether a feature is supported ++ */ ++#define kbase_hw_has_feature(kbdev, feature)\ ++ test_bit(feature, &(kbdev)->hw_features_mask[0]) ++ ++/** ++ * kbase_hw_set_issues_mask - Set the hardware issues mask based on the GPU ID ++ * @kbdev: Device pointer ++ * ++ * Return: 0 if the GPU ID was recognized, otherwise -EINVAL. ++ * ++ * The GPU ID is read from the @kbdev. ++ * ++ * In debugging versions of the driver, unknown versions of a known GPU with a ++ * new-format ID will be treated as the most recent known version not later ++ * than the actual version. In such circumstances, the GPU ID in @kbdev will ++ * also be replaced with the most recent known version. ++ * ++ * Note: The GPU configuration must have been read by ++ * kbase_gpuprops_get_props() before calling this function. ++ */ ++int kbase_hw_set_issues_mask(struct kbase_device *kbdev); ++ ++/** ++ * @brief Set the features mask depending on the GPU ID ++ */ ++void kbase_hw_set_features_mask(struct kbase_device *kbdev); ++ ++#endif /* _KBASE_HW_H_ */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_hwaccess_backend.h b/drivers/gpu/arm/midgard/mali_kbase_hwaccess_backend.h +new file mode 100755 +index 000000000000..b09be99e6b4e +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_hwaccess_backend.h +@@ -0,0 +1,54 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++/* ++ * HW access backend common APIs ++ */ ++ ++#ifndef _KBASE_HWACCESS_BACKEND_H_ ++#define _KBASE_HWACCESS_BACKEND_H_ ++ ++/** ++ * kbase_backend_early_init - Perform any backend-specific initialization. ++ * @kbdev: Device pointer ++ * ++ * Return: 0 on success, or an error code on failure. ++ */ ++int kbase_backend_early_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_backend_late_init - Perform any backend-specific initialization. ++ * @kbdev: Device pointer ++ * ++ * Return: 0 on success, or an error code on failure. ++ */ ++int kbase_backend_late_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_backend_early_term - Perform any backend-specific termination. ++ * @kbdev: Device pointer ++ */ ++void kbase_backend_early_term(struct kbase_device *kbdev); ++ ++/** ++ * kbase_backend_late_term - Perform any backend-specific termination. ++ * @kbdev: Device pointer ++ */ ++void kbase_backend_late_term(struct kbase_device *kbdev); ++ ++#endif /* _KBASE_HWACCESS_BACKEND_H_ */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_hwaccess_defs.h b/drivers/gpu/arm/midgard/mali_kbase_hwaccess_defs.h +new file mode 100755 +index 000000000000..0acf297192fd +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_hwaccess_defs.h +@@ -0,0 +1,36 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014, 2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++/** ++ * @file mali_kbase_hwaccess_gpu_defs.h ++ * HW access common definitions ++ */ ++ ++#ifndef _KBASE_HWACCESS_DEFS_H_ ++#define _KBASE_HWACCESS_DEFS_H_ ++ ++#include ++ ++/* The hwaccess_lock (a spinlock) must be held when accessing this structure */ ++struct kbase_hwaccess_data { ++ struct kbase_context *active_kctx; ++ ++ struct kbase_backend_data backend; ++}; ++ ++#endif /* _KBASE_HWACCESS_DEFS_H_ */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_hwaccess_gpuprops.h b/drivers/gpu/arm/midgard/mali_kbase_hwaccess_gpuprops.h +new file mode 100755 +index 000000000000..cf8a8131c22e +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_hwaccess_gpuprops.h +@@ -0,0 +1,47 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++/** ++ * Base kernel property query backend APIs ++ */ ++ ++#ifndef _KBASE_HWACCESS_GPUPROPS_H_ ++#define _KBASE_HWACCESS_GPUPROPS_H_ ++ ++/** ++ * kbase_backend_gpuprops_get() - Fill @regdump with GPU properties read from ++ * GPU ++ * @kbdev: Device pointer ++ * @regdump: Pointer to struct kbase_gpuprops_regdump structure ++ */ ++void kbase_backend_gpuprops_get(struct kbase_device *kbdev, ++ struct kbase_gpuprops_regdump *regdump); ++ ++/** ++ * kbase_backend_gpuprops_get - Fill @regdump with GPU properties read from GPU ++ * @kbdev: Device pointer ++ * @regdump: Pointer to struct kbase_gpuprops_regdump structure ++ * ++ * This function reads GPU properties that are dependent on the hardware ++ * features bitmask ++ */ ++void kbase_backend_gpuprops_get_features(struct kbase_device *kbdev, ++ struct kbase_gpuprops_regdump *regdump); ++ ++ ++#endif /* _KBASE_HWACCESS_GPUPROPS_H_ */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_hwaccess_instr.h b/drivers/gpu/arm/midgard/mali_kbase_hwaccess_instr.h +new file mode 100755 +index 000000000000..5de2b7535bb4 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_hwaccess_instr.h +@@ -0,0 +1,116 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ++ * HW Access instrumentation common APIs ++ */ ++ ++#ifndef _KBASE_HWACCESS_INSTR_H_ ++#define _KBASE_HWACCESS_INSTR_H_ ++ ++#include ++ ++/** ++ * kbase_instr_hwcnt_enable_internal - Enable HW counters collection ++ * @kbdev: Kbase device ++ * @kctx: Kbase context ++ * @setup: HW counter setup parameters ++ * ++ * Context: might sleep, waiting for reset to complete ++ * ++ * Return: 0 on success ++ */ ++int kbase_instr_hwcnt_enable_internal(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ struct kbase_uk_hwcnt_setup *setup); ++ ++/** ++ * kbase_instr_hwcnt_disable_internal - Disable HW counters collection ++ * @kctx: Kbase context ++ * ++ * Context: might sleep, waiting for an ongoing dump to complete ++ * ++ * Return: 0 on success ++ */ ++int kbase_instr_hwcnt_disable_internal(struct kbase_context *kctx); ++ ++/** ++ * kbase_instr_hwcnt_request_dump() - Request HW counter dump from GPU ++ * @kctx: Kbase context ++ * ++ * Caller must either wait for kbase_instr_hwcnt_dump_complete() to return true, ++ * of call kbase_instr_hwcnt_wait_for_dump(). ++ * ++ * Return: 0 on success ++ */ ++int kbase_instr_hwcnt_request_dump(struct kbase_context *kctx); ++ ++/** ++ * kbase_instr_hwcnt_wait_for_dump() - Wait until pending HW counter dump has ++ * completed. ++ * @kctx: Kbase context ++ * ++ * Context: will sleep, waiting for dump to complete ++ * ++ * Return: 0 on success ++ */ ++int kbase_instr_hwcnt_wait_for_dump(struct kbase_context *kctx); ++ ++/** ++ * kbase_instr_hwcnt_dump_complete - Tell whether the HW counters dump has ++ * completed ++ * @kctx: Kbase context ++ * @success: Set to true if successful ++ * ++ * Context: does not sleep. ++ * ++ * Return: true if the dump is complete ++ */ ++bool kbase_instr_hwcnt_dump_complete(struct kbase_context *kctx, ++ bool * const success); ++ ++/** ++ * kbase_instr_hwcnt_clear() - Clear HW counters ++ * @kctx: Kbase context ++ * ++ * Context: might sleep, waiting for reset to complete ++ * ++ * Return: 0 on success ++ */ ++int kbase_instr_hwcnt_clear(struct kbase_context *kctx); ++ ++/** ++ * kbase_instr_backend_init() - Initialise the instrumentation backend ++ * @kbdev: Kbase device ++ * ++ * This function should be called during driver initialization. ++ * ++ * Return: 0 on success ++ */ ++int kbase_instr_backend_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_instr_backend_init() - Terminate the instrumentation backend ++ * @kbdev: Kbase device ++ * ++ * This function should be called during driver termination. ++ */ ++void kbase_instr_backend_term(struct kbase_device *kbdev); ++ ++#endif /* _KBASE_HWACCESS_INSTR_H_ */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_hwaccess_jm.h b/drivers/gpu/arm/midgard/mali_kbase_hwaccess_jm.h +new file mode 100755 +index 000000000000..750fda2cd81d +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_hwaccess_jm.h +@@ -0,0 +1,381 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++/* ++ * HW access job manager common APIs ++ */ ++ ++#ifndef _KBASE_HWACCESS_JM_H_ ++#define _KBASE_HWACCESS_JM_H_ ++ ++/** ++ * kbase_backend_run_atom() - Run an atom on the GPU ++ * @kbdev: Device pointer ++ * @atom: Atom to run ++ * ++ * Caller must hold the HW access lock ++ */ ++void kbase_backend_run_atom(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom); ++ ++/** ++ * kbase_backend_slot_update - Update state based on slot ringbuffers ++ * ++ * @kbdev: Device pointer ++ * ++ * Inspect the jobs in the slot ringbuffers and update state. ++ * ++ * This will cause jobs to be submitted to hardware if they are unblocked ++ */ ++void kbase_backend_slot_update(struct kbase_device *kbdev); ++ ++/** ++ * kbase_backend_find_and_release_free_address_space() - Release a free AS ++ * @kbdev: Device pointer ++ * @kctx: Context pointer ++ * ++ * This function can evict an idle context from the runpool, freeing up the ++ * address space it was using. ++ * ++ * The address space is marked as in use. The caller must either assign a ++ * context using kbase_gpu_use_ctx(), or release it using ++ * kbase_ctx_sched_release() ++ * ++ * Return: Number of free address space, or KBASEP_AS_NR_INVALID if none ++ * available ++ */ ++int kbase_backend_find_and_release_free_address_space( ++ struct kbase_device *kbdev, struct kbase_context *kctx); ++ ++/** ++ * kbase_backend_use_ctx() - Activate a currently unscheduled context, using the ++ * provided address space. ++ * @kbdev: Device pointer ++ * @kctx: Context pointer. May be NULL ++ * @as_nr: Free address space to use ++ * ++ * kbase_gpu_next_job() will pull atoms from the active context. ++ * ++ * Return: true if successful, false if ASID not assigned. ++ */ ++bool kbase_backend_use_ctx(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ int as_nr); ++ ++/** ++ * kbase_backend_use_ctx_sched() - Activate a context. ++ * @kbdev: Device pointer ++ * @kctx: Context pointer ++ * ++ * kbase_gpu_next_job() will pull atoms from the active context. ++ * ++ * The context must already be scheduled and assigned to an address space. If ++ * the context is not scheduled, then kbase_gpu_use_ctx() should be used ++ * instead. ++ * ++ * Caller must hold hwaccess_lock ++ * ++ * Return: true if context is now active, false otherwise (ie if context does ++ * not have an address space assigned) ++ */ ++bool kbase_backend_use_ctx_sched(struct kbase_device *kbdev, ++ struct kbase_context *kctx); ++ ++/** ++ * kbase_backend_release_ctx_irq - Release a context from the GPU. This will ++ * de-assign the assigned address space. ++ * @kbdev: Device pointer ++ * @kctx: Context pointer ++ * ++ * Caller must hold kbase_device->mmu_hw_mutex and hwaccess_lock ++ */ ++void kbase_backend_release_ctx_irq(struct kbase_device *kbdev, ++ struct kbase_context *kctx); ++ ++/** ++ * kbase_backend_release_ctx_noirq - Release a context from the GPU. This will ++ * de-assign the assigned address space. ++ * @kbdev: Device pointer ++ * @kctx: Context pointer ++ * ++ * Caller must hold kbase_device->mmu_hw_mutex ++ * ++ * This function must perform any operations that could not be performed in IRQ ++ * context by kbase_backend_release_ctx_irq(). ++ */ ++void kbase_backend_release_ctx_noirq(struct kbase_device *kbdev, ++ struct kbase_context *kctx); ++ ++/** ++ * kbase_backend_cacheclean - Perform a cache clean if the given atom requires ++ * one ++ * @kbdev: Device pointer ++ * @katom: Pointer to the failed atom ++ * ++ * On some GPUs, the GPU cache must be cleaned following a failed atom. This ++ * function performs a clean if it is required by @katom. ++ */ ++void kbase_backend_cacheclean(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom); ++ ++ ++/** ++ * kbase_backend_complete_wq() - Perform backend-specific actions required on ++ * completing an atom. ++ * @kbdev: Device pointer ++ * @katom: Pointer to the atom to complete ++ * ++ * This function should only be called from kbase_jd_done_worker() or ++ * js_return_worker(). ++ * ++ * Return: true if atom has completed, false if atom should be re-submitted ++ */ ++void kbase_backend_complete_wq(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom); ++ ++/** ++ * kbase_backend_complete_wq_post_sched - Perform backend-specific actions ++ * required on completing an atom, after ++ * any scheduling has taken place. ++ * @kbdev: Device pointer ++ * @core_req: Core requirements of atom ++ * @affinity: Affinity of atom ++ * @coreref_state: Coreref state of atom ++ * ++ * This function should only be called from kbase_jd_done_worker() or ++ * js_return_worker(). ++ */ ++void kbase_backend_complete_wq_post_sched(struct kbase_device *kbdev, ++ base_jd_core_req core_req, u64 affinity, ++ enum kbase_atom_coreref_state coreref_state); ++ ++/** ++ * kbase_backend_reset() - The GPU is being reset. Cancel all jobs on the GPU ++ * and remove any others from the ringbuffers. ++ * @kbdev: Device pointer ++ * @end_timestamp: Timestamp of reset ++ */ ++void kbase_backend_reset(struct kbase_device *kbdev, ktime_t *end_timestamp); ++ ++/** ++ * kbase_backend_inspect_head() - Return the atom currently at the head of slot ++ * @js ++ * @kbdev: Device pointer ++ * @js: Job slot to inspect ++ * ++ * Return : Atom currently at the head of slot @js, or NULL ++ */ ++struct kbase_jd_atom *kbase_backend_inspect_head(struct kbase_device *kbdev, ++ int js); ++ ++/** ++ * kbase_backend_inspect_tail - Return the atom currently at the tail of slot ++ * @js ++ * @kbdev: Device pointer ++ * @js: Job slot to inspect ++ * ++ * Return : Atom currently at the head of slot @js, or NULL ++ */ ++struct kbase_jd_atom *kbase_backend_inspect_tail(struct kbase_device *kbdev, ++ int js); ++ ++/** ++ * kbase_backend_nr_atoms_on_slot() - Return the number of atoms currently on a ++ * slot. ++ * @kbdev: Device pointer ++ * @js: Job slot to inspect ++ * ++ * Return : Number of atoms currently on slot ++ */ ++int kbase_backend_nr_atoms_on_slot(struct kbase_device *kbdev, int js); ++ ++/** ++ * kbase_backend_nr_atoms_submitted() - Return the number of atoms on a slot ++ * that are currently on the GPU. ++ * @kbdev: Device pointer ++ * @js: Job slot to inspect ++ * ++ * Return : Number of atoms currently on slot @js that are currently on the GPU. ++ */ ++int kbase_backend_nr_atoms_submitted(struct kbase_device *kbdev, int js); ++ ++/** ++ * kbase_backend_ctx_count_changed() - Number of contexts ready to submit jobs ++ * has changed. ++ * @kbdev: Device pointer ++ * ++ * Perform any required backend-specific actions (eg starting/stopping ++ * scheduling timers). ++ */ ++void kbase_backend_ctx_count_changed(struct kbase_device *kbdev); ++ ++/** ++ * kbase_backend_timeouts_changed() - Job Scheduler timeouts have changed. ++ * @kbdev: Device pointer ++ * ++ * Perform any required backend-specific actions (eg updating timeouts of ++ * currently running atoms). ++ */ ++void kbase_backend_timeouts_changed(struct kbase_device *kbdev); ++ ++/** ++ * kbase_backend_slot_free() - Return the number of jobs that can be currently ++ * submitted to slot @js. ++ * @kbdev: Device pointer ++ * @js: Job slot to inspect ++ * ++ * Return : Number of jobs that can be submitted. ++ */ ++int kbase_backend_slot_free(struct kbase_device *kbdev, int js); ++ ++/** ++ * kbase_job_check_enter_disjoint - potentially leave disjoint state ++ * @kbdev: kbase device ++ * @target_katom: atom which is finishing ++ * ++ * Work out whether to leave disjoint state when finishing an atom that was ++ * originated by kbase_job_check_enter_disjoint(). ++ */ ++void kbase_job_check_leave_disjoint(struct kbase_device *kbdev, ++ struct kbase_jd_atom *target_katom); ++ ++/** ++ * kbase_backend_jm_kill_jobs_from_kctx - Kill all jobs that are currently ++ * running from a context ++ * @kctx: Context pointer ++ * ++ * This is used in response to a page fault to remove all jobs from the faulting ++ * context from the hardware. ++ */ ++void kbase_backend_jm_kill_jobs_from_kctx(struct kbase_context *kctx); ++ ++/** ++ * kbase_jm_wait_for_zero_jobs - Wait for context to have zero jobs running, and ++ * to be descheduled. ++ * @kctx: Context pointer ++ * ++ * This should be called following kbase_js_zap_context(), to ensure the context ++ * can be safely destroyed. ++ */ ++void kbase_jm_wait_for_zero_jobs(struct kbase_context *kctx); ++ ++/** ++ * kbase_backend_get_current_flush_id - Return the current flush ID ++ * ++ * @kbdev: Device pointer ++ * ++ * Return: the current flush ID to be recorded for each job chain ++ */ ++u32 kbase_backend_get_current_flush_id(struct kbase_device *kbdev); ++ ++#if KBASE_GPU_RESET_EN ++/** ++ * kbase_prepare_to_reset_gpu - Prepare for resetting the GPU. ++ * @kbdev: Device pointer ++ * ++ * This function just soft-stops all the slots to ensure that as many jobs as ++ * possible are saved. ++ * ++ * Return: a boolean which should be interpreted as follows: ++ * - true - Prepared for reset, kbase_reset_gpu should be called. ++ * - false - Another thread is performing a reset, kbase_reset_gpu should ++ * not be called. ++ */ ++bool kbase_prepare_to_reset_gpu(struct kbase_device *kbdev); ++ ++/** ++ * kbase_reset_gpu - Reset the GPU ++ * @kbdev: Device pointer ++ * ++ * This function should be called after kbase_prepare_to_reset_gpu if it returns ++ * true. It should never be called without a corresponding call to ++ * kbase_prepare_to_reset_gpu. ++ * ++ * After this function is called (or not called if kbase_prepare_to_reset_gpu ++ * returned false), the caller should wait for kbdev->reset_waitq to be ++ * signalled to know when the reset has completed. ++ */ ++void kbase_reset_gpu(struct kbase_device *kbdev); ++ ++/** ++ * kbase_prepare_to_reset_gpu_locked - Prepare for resetting the GPU. ++ * @kbdev: Device pointer ++ * ++ * This function just soft-stops all the slots to ensure that as many jobs as ++ * possible are saved. ++ * ++ * Return: a boolean which should be interpreted as follows: ++ * - true - Prepared for reset, kbase_reset_gpu should be called. ++ * - false - Another thread is performing a reset, kbase_reset_gpu should ++ * not be called. ++ */ ++bool kbase_prepare_to_reset_gpu_locked(struct kbase_device *kbdev); ++ ++/** ++ * kbase_reset_gpu_locked - Reset the GPU ++ * @kbdev: Device pointer ++ * ++ * This function should be called after kbase_prepare_to_reset_gpu if it ++ * returns true. It should never be called without a corresponding call to ++ * kbase_prepare_to_reset_gpu. ++ * ++ * After this function is called (or not called if kbase_prepare_to_reset_gpu ++ * returned false), the caller should wait for kbdev->reset_waitq to be ++ * signalled to know when the reset has completed. ++ */ ++void kbase_reset_gpu_locked(struct kbase_device *kbdev); ++ ++/** ++ * kbase_reset_gpu_silent - Reset the GPU silently ++ * @kbdev: Device pointer ++ * ++ * Reset the GPU without trying to cancel jobs and don't emit messages into ++ * the kernel log while doing the reset. ++ * ++ * This function should be used in cases where we are doing a controlled reset ++ * of the GPU as part of normal processing (e.g. exiting protected mode) where ++ * the driver will have ensured the scheduler has been idled and all other ++ * users of the GPU (e.g. instrumentation) have been suspended. ++ */ ++void kbase_reset_gpu_silent(struct kbase_device *kbdev); ++ ++/** ++ * kbase_reset_gpu_active - Reports if the GPU is being reset ++ * @kbdev: Device pointer ++ * ++ * Return: True if the GPU is in the process of being reset. ++ */ ++bool kbase_reset_gpu_active(struct kbase_device *kbdev); ++#endif ++ ++/** ++ * kbase_job_slot_hardstop - Hard-stop the specified job slot ++ * @kctx: The kbase context that contains the job(s) that should ++ * be hard-stopped ++ * @js: The job slot to hard-stop ++ * @target_katom: The job that should be hard-stopped (or NULL for all ++ * jobs from the context) ++ * Context: ++ * The job slot lock must be held when calling this function. ++ */ ++void kbase_job_slot_hardstop(struct kbase_context *kctx, int js, ++ struct kbase_jd_atom *target_katom); ++ ++extern struct protected_mode_ops kbase_native_protected_ops; ++ ++#endif /* _KBASE_HWACCESS_JM_H_ */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_hwaccess_pm.h b/drivers/gpu/arm/midgard/mali_kbase_hwaccess_pm.h +new file mode 100755 +index 000000000000..71c7d495c40a +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_hwaccess_pm.h +@@ -0,0 +1,209 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++/** ++ * @file mali_kbase_hwaccess_pm.h ++ * HW access power manager common APIs ++ */ ++ ++#ifndef _KBASE_HWACCESS_PM_H_ ++#define _KBASE_HWACCESS_PM_H_ ++ ++#include ++#include ++ ++#include ++ ++/* Forward definition - see mali_kbase.h */ ++struct kbase_device; ++ ++/* Functions common to all HW access backends */ ++ ++/** ++ * Initialize the power management framework. ++ * ++ * Must be called before any other power management function ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid ++ * pointer) ++ * ++ * @return 0 if the power management framework was successfully ++ * initialized. ++ */ ++int kbase_hwaccess_pm_init(struct kbase_device *kbdev); ++ ++/** ++ * Terminate the power management framework. ++ * ++ * No power management functions may be called after this (except ++ * @ref kbase_pm_init) ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid ++ * pointer) ++ */ ++void kbase_hwaccess_pm_term(struct kbase_device *kbdev); ++ ++/** ++ * kbase_hwaccess_pm_powerup - Power up the GPU. ++ * @kbdev: The kbase device structure for the device (must be a valid pointer) ++ * @flags: Flags to pass on to kbase_pm_init_hw ++ * ++ * Power up GPU after all modules have been initialized and interrupt handlers ++ * installed. ++ * ++ * Return: 0 if powerup was successful. ++ */ ++int kbase_hwaccess_pm_powerup(struct kbase_device *kbdev, ++ unsigned int flags); ++ ++/** ++ * Halt the power management framework. ++ * ++ * Should ensure that no new interrupts are generated, but allow any currently ++ * running interrupt handlers to complete successfully. The GPU is forced off by ++ * the time this function returns, regardless of whether or not the active power ++ * policy asks for the GPU to be powered off. ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid ++ * pointer) ++ */ ++void kbase_hwaccess_pm_halt(struct kbase_device *kbdev); ++ ++/** ++ * Perform any backend-specific actions to suspend the GPU ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid ++ * pointer) ++ */ ++void kbase_hwaccess_pm_suspend(struct kbase_device *kbdev); ++ ++/** ++ * Perform any backend-specific actions to resume the GPU from a suspend ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid ++ * pointer) ++ */ ++void kbase_hwaccess_pm_resume(struct kbase_device *kbdev); ++ ++/** ++ * Perform any required actions for activating the GPU. Called when the first ++ * context goes active. ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid ++ * pointer) ++ */ ++void kbase_hwaccess_pm_gpu_active(struct kbase_device *kbdev); ++ ++/** ++ * Perform any required actions for idling the GPU. Called when the last ++ * context goes idle. ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid ++ * pointer) ++ */ ++void kbase_hwaccess_pm_gpu_idle(struct kbase_device *kbdev); ++ ++ ++/** ++ * Set the debug core mask. ++ * ++ * This determines which cores the power manager is allowed to use. ++ * ++ * @param kbdev The kbase device structure for the device (must be a ++ * valid pointer) ++ * @param new_core_mask_js0 The core mask to use for job slot 0 ++ * @param new_core_mask_js0 The core mask to use for job slot 1 ++ * @param new_core_mask_js0 The core mask to use for job slot 2 ++ */ ++void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev, ++ u64 new_core_mask_js0, u64 new_core_mask_js1, ++ u64 new_core_mask_js2); ++ ++ ++/** ++ * Get the current policy. ++ * ++ * Returns the policy that is currently active. ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid ++ * pointer) ++ * ++ * @return The current policy ++ */ ++const struct kbase_pm_ca_policy ++*kbase_pm_ca_get_policy(struct kbase_device *kbdev); ++ ++/** ++ * Change the policy to the one specified. ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid ++ * pointer) ++ * @param policy The policy to change to (valid pointer returned from ++ * @ref kbase_pm_ca_list_policies) ++ */ ++void kbase_pm_ca_set_policy(struct kbase_device *kbdev, ++ const struct kbase_pm_ca_policy *policy); ++ ++/** ++ * Retrieve a static list of the available policies. ++ * ++ * @param[out] policies An array pointer to take the list of policies. This may ++ * be NULL. The contents of this array must not be ++ * modified. ++ * ++ * @return The number of policies ++ */ ++int ++kbase_pm_ca_list_policies(const struct kbase_pm_ca_policy * const **policies); ++ ++ ++/** ++ * Get the current policy. ++ * ++ * Returns the policy that is currently active. ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid ++ * pointer) ++ * ++ * @return The current policy ++ */ ++const struct kbase_pm_policy *kbase_pm_get_policy(struct kbase_device *kbdev); ++ ++/** ++ * Change the policy to the one specified. ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid ++ * pointer) ++ * @param policy The policy to change to (valid pointer returned from ++ * @ref kbase_pm_list_policies) ++ */ ++void kbase_pm_set_policy(struct kbase_device *kbdev, ++ const struct kbase_pm_policy *policy); ++ ++/** ++ * Retrieve a static list of the available policies. ++ * ++ * @param[out] policies An array pointer to take the list of policies. This may ++ * be NULL. The contents of this array must not be ++ * modified. ++ * ++ * @return The number of policies ++ */ ++int kbase_pm_list_policies(const struct kbase_pm_policy * const **policies); ++ ++#endif /* _KBASE_HWACCESS_PM_H_ */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_hwaccess_time.h b/drivers/gpu/arm/midgard/mali_kbase_hwaccess_time.h +new file mode 100755 +index 000000000000..10b65798e6cf +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_hwaccess_time.h +@@ -0,0 +1,53 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++/** ++ * ++ */ ++ ++#ifndef _KBASE_BACKEND_TIME_H_ ++#define _KBASE_BACKEND_TIME_H_ ++ ++/** ++ * kbase_backend_get_gpu_time() - Get current GPU time ++ * @kbdev: Device pointer ++ * @cycle_counter: Pointer to u64 to store cycle counter in ++ * @system_time: Pointer to u64 to store system time in ++ * @ts: Pointer to struct timespec64 to store current monotonic ++ * time in ++ */ ++void kbase_backend_get_gpu_time(struct kbase_device *kbdev, u64 *cycle_counter, ++ u64 *system_time, struct timespec64 *ts); ++ ++/** ++ * kbase_wait_write_flush() - Wait for GPU write flush ++ * @kctx: Context pointer ++ * ++ * Wait 1000 GPU clock cycles. This delay is known to give the GPU time to flush ++ * its write buffer. ++ * ++ * If GPU resets occur then the counters are reset to zero, the delay may not be ++ * as expected. ++ * ++ * This function is only in use for BASE_HW_ISSUE_6367 ++ */ ++#ifndef CONFIG_MALI_NO_MALI ++void kbase_wait_write_flush(struct kbase_context *kctx); ++#endif ++ ++#endif /* _KBASE_BACKEND_TIME_H_ */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_hwcnt_reader.h b/drivers/gpu/arm/midgard/mali_kbase_hwcnt_reader.h +new file mode 100755 +index 000000000000..cf7bf1b35dc5 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_hwcnt_reader.h +@@ -0,0 +1,66 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KBASE_HWCNT_READER_H_ ++#define _KBASE_HWCNT_READER_H_ ++ ++/* The ids of ioctl commands. */ ++#define KBASE_HWCNT_READER 0xBE ++#define KBASE_HWCNT_READER_GET_HWVER _IOR(KBASE_HWCNT_READER, 0x00, u32) ++#define KBASE_HWCNT_READER_GET_BUFFER_SIZE _IOR(KBASE_HWCNT_READER, 0x01, u32) ++#define KBASE_HWCNT_READER_DUMP _IOW(KBASE_HWCNT_READER, 0x10, u32) ++#define KBASE_HWCNT_READER_CLEAR _IOW(KBASE_HWCNT_READER, 0x11, u32) ++#define KBASE_HWCNT_READER_GET_BUFFER _IOR(KBASE_HWCNT_READER, 0x20,\ ++ struct kbase_hwcnt_reader_metadata) ++#define KBASE_HWCNT_READER_PUT_BUFFER _IOW(KBASE_HWCNT_READER, 0x21,\ ++ struct kbase_hwcnt_reader_metadata) ++#define KBASE_HWCNT_READER_SET_INTERVAL _IOW(KBASE_HWCNT_READER, 0x30, u32) ++#define KBASE_HWCNT_READER_ENABLE_EVENT _IOW(KBASE_HWCNT_READER, 0x40, u32) ++#define KBASE_HWCNT_READER_DISABLE_EVENT _IOW(KBASE_HWCNT_READER, 0x41, u32) ++#define KBASE_HWCNT_READER_GET_API_VERSION _IOW(KBASE_HWCNT_READER, 0xFF, u32) ++ ++/** ++ * struct kbase_hwcnt_reader_metadata - hwcnt reader sample buffer metadata ++ * @timestamp: time when sample was collected ++ * @event_id: id of an event that triggered sample collection ++ * @buffer_idx: position in sampling area where sample buffer was stored ++ */ ++struct kbase_hwcnt_reader_metadata { ++ u64 timestamp; ++ u32 event_id; ++ u32 buffer_idx; ++}; ++ ++/** ++ * enum base_hwcnt_reader_event - hwcnt dumping events ++ * @BASE_HWCNT_READER_EVENT_MANUAL: manual request for dump ++ * @BASE_HWCNT_READER_EVENT_PERIODIC: periodic dump ++ * @BASE_HWCNT_READER_EVENT_PREJOB: prejob dump request ++ * @BASE_HWCNT_READER_EVENT_POSTJOB: postjob dump request ++ * @BASE_HWCNT_READER_EVENT_COUNT: number of supported events ++ */ ++enum base_hwcnt_reader_event { ++ BASE_HWCNT_READER_EVENT_MANUAL, ++ BASE_HWCNT_READER_EVENT_PERIODIC, ++ BASE_HWCNT_READER_EVENT_PREJOB, ++ BASE_HWCNT_READER_EVENT_POSTJOB, ++ ++ BASE_HWCNT_READER_EVENT_COUNT ++}; ++ ++#endif /* _KBASE_HWCNT_READER_H_ */ ++ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_ioctl.h b/drivers/gpu/arm/midgard/mali_kbase_ioctl.h +new file mode 100755 +index 000000000000..dcbed9c774d6 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_ioctl.h +@@ -0,0 +1,656 @@ ++/* ++ * ++ * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KBASE_IOCTL_H_ ++#define _KBASE_IOCTL_H_ ++ ++#ifdef __cpluscplus ++extern "C" { ++#endif ++ ++#include ++ ++#define KBASE_IOCTL_TYPE 0x80 ++ ++#ifdef ANDROID ++/* Android's definition of ioctl is incorrect, specifying the type argument as ++ * 'int'. This creates a warning when using _IOWR (as the top bit is set). Work ++ * round this by redefining _IOC to include a case to 'int'. ++ */ ++#undef _IOC ++#define _IOC(dir, type, nr, size) \ ++ ((int)(((dir) << _IOC_DIRSHIFT) | ((type) << _IOC_TYPESHIFT) | \ ++ ((nr) << _IOC_NRSHIFT) | ((size) << _IOC_SIZESHIFT))) ++#endif ++ ++/** ++ * struct kbase_ioctl_version_check - Check version compatibility with kernel ++ * ++ * @major: Major version number ++ * @minor: Minor version number ++ */ ++struct kbase_ioctl_version_check { ++ __u16 major; ++ __u16 minor; ++}; ++ ++#define KBASE_IOCTL_VERSION_CHECK \ ++ _IOWR(KBASE_IOCTL_TYPE, 0, struct kbase_ioctl_version_check) ++ ++/** ++ * struct kbase_ioctl_set_flags - Set kernel context creation flags ++ * ++ * @create_flags: Flags - see base_context_create_flags ++ */ ++struct kbase_ioctl_set_flags { ++ __u32 create_flags; ++}; ++ ++#define KBASE_IOCTL_SET_FLAGS \ ++ _IOW(KBASE_IOCTL_TYPE, 1, struct kbase_ioctl_set_flags) ++ ++/** ++ * struct kbase_ioctl_job_submit - Submit jobs/atoms to the kernel ++ * ++ * @addr: Memory address of an array of struct base_jd_atom_v2 ++ * @nr_atoms: Number of entries in the array ++ * @stride: sizeof(struct base_jd_atom_v2) ++ */ ++struct kbase_ioctl_job_submit { ++ union kbase_pointer addr; ++ __u32 nr_atoms; ++ __u32 stride; ++}; ++ ++#define KBASE_IOCTL_JOB_SUBMIT \ ++ _IOW(KBASE_IOCTL_TYPE, 2, struct kbase_ioctl_job_submit) ++ ++/** ++ * struct kbase_ioctl_get_gpuprops - Read GPU properties from the kernel ++ * ++ * @buffer: Pointer to the buffer to store properties into ++ * @size: Size of the buffer ++ * @flags: Flags - must be zero for now ++ * ++ * The ioctl will return the number of bytes stored into @buffer or an error ++ * on failure (e.g. @size is too small). If @size is specified as 0 then no ++ * data will be written but the return value will be the number of bytes needed ++ * for all the properties. ++ * ++ * @flags may be used in the future to request a different format for the ++ * buffer. With @flags == 0 the following format is used. ++ * ++ * The buffer will be filled with pairs of values, a u32 key identifying the ++ * property followed by the value. The size of the value is identified using ++ * the bottom bits of the key. The value then immediately followed the key and ++ * is tightly packed (there is no padding). All keys and values are ++ * little-endian. ++ * ++ * 00 = u8 ++ * 01 = u16 ++ * 10 = u32 ++ * 11 = u64 ++ */ ++struct kbase_ioctl_get_gpuprops { ++ union kbase_pointer buffer; ++ __u32 size; ++ __u32 flags; ++}; ++ ++#define KBASE_IOCTL_GET_GPUPROPS \ ++ _IOW(KBASE_IOCTL_TYPE, 3, struct kbase_ioctl_get_gpuprops) ++ ++#define KBASE_IOCTL_POST_TERM \ ++ _IO(KBASE_IOCTL_TYPE, 4) ++ ++/** ++ * union kbase_ioctl_mem_alloc - Allocate memory on the GPU ++ * ++ * @va_pages: The number of pages of virtual address space to reserve ++ * @commit_pages: The number of physical pages to allocate ++ * @extent: The number of extra pages to allocate on each GPU fault which grows ++ * the region ++ * @flags: Flags ++ * @gpu_va: The GPU virtual address which is allocated ++ * ++ * @in: Input parameters ++ * @out: Output parameters ++ */ ++union kbase_ioctl_mem_alloc { ++ struct { ++ __u64 va_pages; ++ __u64 commit_pages; ++ __u64 extent; ++ __u64 flags; ++ } in; ++ struct { ++ __u64 flags; ++ __u64 gpu_va; ++ } out; ++}; ++ ++#define KBASE_IOCTL_MEM_ALLOC \ ++ _IOWR(KBASE_IOCTL_TYPE, 5, union kbase_ioctl_mem_alloc) ++ ++/** ++ * struct kbase_ioctl_mem_query - Query properties of a GPU memory region ++ * @gpu_addr: A GPU address contained within the region ++ * @query: The type of query ++ * @value: The result of the query ++ * ++ * Use a %KBASE_MEM_QUERY_xxx flag as input for @query. ++ * ++ * @in: Input parameters ++ * @out: Output parameters ++ */ ++union kbase_ioctl_mem_query { ++ struct { ++ __u64 gpu_addr; ++ __u64 query; ++ } in; ++ struct { ++ __u64 value; ++ } out; ++}; ++ ++#define KBASE_IOCTL_MEM_QUERY \ ++ _IOWR(KBASE_IOCTL_TYPE, 6, union kbase_ioctl_mem_query) ++ ++#define KBASE_MEM_QUERY_COMMIT_SIZE 1 ++#define KBASE_MEM_QUERY_VA_SIZE 2 ++#define KBASE_MEM_QUERY_FLAGS 3 ++ ++/** ++ * struct kbase_ioctl_mem_free - Free a memory region ++ * @gpu_addr: Handle to the region to free ++ */ ++struct kbase_ioctl_mem_free { ++ __u64 gpu_addr; ++}; ++ ++#define KBASE_IOCTL_MEM_FREE \ ++ _IOW(KBASE_IOCTL_TYPE, 7, struct kbase_ioctl_mem_free) ++ ++/** ++ * struct kbase_ioctl_hwcnt_reader_setup - Setup HWC dumper/reader ++ * @buffer_count: requested number of dumping buffers ++ * @jm_bm: counters selection bitmask (JM) ++ * @shader_bm: counters selection bitmask (Shader) ++ * @tiler_bm: counters selection bitmask (Tiler) ++ * @mmu_l2_bm: counters selection bitmask (MMU_L2) ++ * ++ * A fd is returned from the ioctl if successful, or a negative value on error ++ */ ++struct kbase_ioctl_hwcnt_reader_setup { ++ __u32 buffer_count; ++ __u32 jm_bm; ++ __u32 shader_bm; ++ __u32 tiler_bm; ++ __u32 mmu_l2_bm; ++}; ++ ++#define KBASE_IOCTL_HWCNT_READER_SETUP \ ++ _IOW(KBASE_IOCTL_TYPE, 8, struct kbase_ioctl_hwcnt_reader_setup) ++ ++/** ++ * struct kbase_ioctl_hwcnt_enable - Enable hardware counter collection ++ * @dump_buffer: GPU address to write counters to ++ * @jm_bm: counters selection bitmask (JM) ++ * @shader_bm: counters selection bitmask (Shader) ++ * @tiler_bm: counters selection bitmask (Tiler) ++ * @mmu_l2_bm: counters selection bitmask (MMU_L2) ++ */ ++struct kbase_ioctl_hwcnt_enable { ++ __u64 dump_buffer; ++ __u32 jm_bm; ++ __u32 shader_bm; ++ __u32 tiler_bm; ++ __u32 mmu_l2_bm; ++}; ++ ++#define KBASE_IOCTL_HWCNT_ENABLE \ ++ _IOW(KBASE_IOCTL_TYPE, 9, struct kbase_ioctl_hwcnt_enable) ++ ++#define KBASE_IOCTL_HWCNT_DUMP \ ++ _IO(KBASE_IOCTL_TYPE, 10) ++ ++#define KBASE_IOCTL_HWCNT_CLEAR \ ++ _IO(KBASE_IOCTL_TYPE, 11) ++ ++/** ++ * struct kbase_ioctl_disjoint_query - Query the disjoint counter ++ * @counter: A counter of disjoint events in the kernel ++ */ ++struct kbase_ioctl_disjoint_query { ++ __u32 counter; ++}; ++ ++#define KBASE_IOCTL_DISJOINT_QUERY \ ++ _IOR(KBASE_IOCTL_TYPE, 12, struct kbase_ioctl_disjoint_query) ++ ++/** ++ * struct kbase_ioctl_get_ddk_version - Query the kernel version ++ * @version_buffer: Buffer to receive the kernel version string ++ * @size: Size of the buffer ++ * ++ * The ioctl will return the number of bytes written into version_buffer ++ * (which includes a NULL byte) or a negative error code ++ */ ++struct kbase_ioctl_get_ddk_version { ++ union kbase_pointer version_buffer; ++ __u32 size; ++}; ++ ++#define KBASE_IOCTL_GET_DDK_VERSION \ ++ _IOW(KBASE_IOCTL_TYPE, 13, struct kbase_ioctl_get_ddk_version) ++ ++/** ++ * struct kbase_ioctl_mem_jit_init - Initialise the JIT memory allocator ++ * ++ * @va_pages: Number of VA pages to reserve for JIT ++ * ++ * Note that depending on the VA size of the application and GPU, the value ++ * specified in @va_pages may be ignored. ++ */ ++struct kbase_ioctl_mem_jit_init { ++ __u64 va_pages; ++}; ++ ++#define KBASE_IOCTL_MEM_JIT_INIT \ ++ _IOW(KBASE_IOCTL_TYPE, 14, struct kbase_ioctl_mem_jit_init) ++ ++/** ++ * struct kbase_ioctl_mem_sync - Perform cache maintenance on memory ++ * ++ * @handle: GPU memory handle (GPU VA) ++ * @user_addr: The address where it is mapped in user space ++ * @size: The number of bytes to synchronise ++ * @type: The direction to synchronise: 0 is sync to memory (clean), ++ * 1 is sync from memory (invalidate). Use the BASE_SYNCSET_OP_xxx constants. ++ * @padding: Padding to round up to a multiple of 8 bytes, must be zero ++ */ ++struct kbase_ioctl_mem_sync { ++ __u64 handle; ++ __u64 user_addr; ++ __u64 size; ++ __u8 type; ++ __u8 padding[7]; ++}; ++ ++#define KBASE_IOCTL_MEM_SYNC \ ++ _IOW(KBASE_IOCTL_TYPE, 15, struct kbase_ioctl_mem_sync) ++ ++/** ++ * union kbase_ioctl_mem_find_cpu_offset - Find the offset of a CPU pointer ++ * ++ * @gpu_addr: The GPU address of the memory region ++ * @cpu_addr: The CPU address to locate ++ * @size: A size in bytes to validate is contained within the region ++ * @offset: The offset from the start of the memory region to @cpu_addr ++ * ++ * @in: Input parameters ++ * @out: Output parameters ++ */ ++union kbase_ioctl_mem_find_cpu_offset { ++ struct { ++ __u64 gpu_addr; ++ __u64 cpu_addr; ++ __u64 size; ++ } in; ++ struct { ++ __u64 offset; ++ } out; ++}; ++ ++#define KBASE_IOCTL_MEM_FIND_CPU_OFFSET \ ++ _IOWR(KBASE_IOCTL_TYPE, 16, union kbase_ioctl_mem_find_cpu_offset) ++ ++/** ++ * struct kbase_ioctl_get_context_id - Get the kernel context ID ++ * ++ * @id: The kernel context ID ++ */ ++struct kbase_ioctl_get_context_id { ++ int id; /* This should really be __u32, but see GPUCORE-10048 */ ++}; ++ ++#define KBASE_IOCTL_GET_CONTEXT_ID \ ++ _IOR(KBASE_IOCTL_TYPE, 17, struct kbase_ioctl_get_context_id) ++ ++/** ++ * struct kbase_ioctl_tlstream_acquire - Acquire a tlstream fd ++ * ++ * @flags: Flags ++ * ++ * The ioctl returns a file descriptor when successful ++ */ ++struct kbase_ioctl_tlstream_acquire { ++ __u32 flags; ++}; ++ ++#define KBASE_IOCTL_TLSTREAM_ACQUIRE \ ++ _IOW(KBASE_IOCTL_TYPE, 18, struct kbase_ioctl_tlstream_acquire) ++ ++#define KBASE_IOCTL_TLSTREAM_FLUSH \ ++ _IO(KBASE_IOCTL_TYPE, 19) ++ ++/** ++ * struct kbase_ioctl_mem_commit - Change the amount of memory backing a region ++ * ++ * @gpu_addr: The memory region to modify ++ * @pages: The number of physical pages that should be present ++ * ++ * The ioctl may return on the following error codes or 0 for success: ++ * -ENOMEM: Out of memory ++ * -EINVAL: Invalid arguments ++ */ ++struct kbase_ioctl_mem_commit { ++ __u64 gpu_addr; ++ __u64 pages; ++}; ++ ++#define KBASE_IOCTL_MEM_COMMIT \ ++ _IOW(KBASE_IOCTL_TYPE, 20, struct kbase_ioctl_mem_commit) ++ ++/** ++ * union kbase_ioctl_mem_alias - Create an alias of memory regions ++ * @flags: Flags, see BASE_MEM_xxx ++ * @stride: Bytes between start of each memory region ++ * @nents: The number of regions to pack together into the alias ++ * @aliasing_info: Pointer to an array of struct base_mem_aliasing_info ++ * @gpu_va: Address of the new alias ++ * @va_pages: Size of the new alias ++ * ++ * @in: Input parameters ++ * @out: Output parameters ++ */ ++union kbase_ioctl_mem_alias { ++ struct { ++ __u64 flags; ++ __u64 stride; ++ __u64 nents; ++ union kbase_pointer aliasing_info; ++ } in; ++ struct { ++ __u64 flags; ++ __u64 gpu_va; ++ __u64 va_pages; ++ } out; ++}; ++ ++#define KBASE_IOCTL_MEM_ALIAS \ ++ _IOWR(KBASE_IOCTL_TYPE, 21, union kbase_ioctl_mem_alias) ++ ++/** ++ * union kbase_ioctl_mem_import - Import memory for use by the GPU ++ * @flags: Flags, see BASE_MEM_xxx ++ * @phandle: Handle to the external memory ++ * @type: Type of external memory, see base_mem_import_type ++ * @padding: Amount of extra VA pages to append to the imported buffer ++ * @gpu_va: Address of the new alias ++ * @va_pages: Size of the new alias ++ * ++ * @in: Input parameters ++ * @out: Output parameters ++ */ ++union kbase_ioctl_mem_import { ++ struct { ++ __u64 flags; ++ union kbase_pointer phandle; ++ __u32 type; ++ __u32 padding; ++ } in; ++ struct { ++ __u64 flags; ++ __u64 gpu_va; ++ __u64 va_pages; ++ } out; ++}; ++ ++#define KBASE_IOCTL_MEM_IMPORT \ ++ _IOWR(KBASE_IOCTL_TYPE, 22, union kbase_ioctl_mem_import) ++ ++/** ++ * struct kbase_ioctl_mem_flags_change - Change the flags for a memory region ++ * @gpu_va: The GPU region to modify ++ * @flags: The new flags to set ++ * @mask: Mask of the flags to modify ++ */ ++struct kbase_ioctl_mem_flags_change { ++ __u64 gpu_va; ++ __u64 flags; ++ __u64 mask; ++}; ++ ++#define KBASE_IOCTL_MEM_FLAGS_CHANGE \ ++ _IOW(KBASE_IOCTL_TYPE, 23, struct kbase_ioctl_mem_flags_change) ++ ++/** ++ * struct kbase_ioctl_stream_create - Create a synchronisation stream ++ * @name: A name to identify this stream. Must be NULL-terminated. ++ * ++ * Note that this is also called a "timeline", but is named stream to avoid ++ * confusion with other uses of the word. ++ * ++ * Unused bytes in @name (after the first NULL byte) must be also be NULL bytes. ++ * ++ * The ioctl returns a file descriptor. ++ */ ++struct kbase_ioctl_stream_create { ++ char name[32]; ++}; ++ ++#define KBASE_IOCTL_STREAM_CREATE \ ++ _IOW(KBASE_IOCTL_TYPE, 24, struct kbase_ioctl_stream_create) ++ ++/** ++ * struct kbase_ioctl_fence_validate - Validate a fd refers to a fence ++ * @fd: The file descriptor to validate ++ */ ++struct kbase_ioctl_fence_validate { ++ int fd; ++}; ++ ++#define KBASE_IOCTL_FENCE_VALIDATE \ ++ _IOW(KBASE_IOCTL_TYPE, 25, struct kbase_ioctl_fence_validate) ++ ++/** ++ * struct kbase_ioctl_get_profiling_controls - Get the profiling controls ++ * @count: The size of @buffer in u32 words ++ * @buffer: The buffer to receive the profiling controls ++ */ ++struct kbase_ioctl_get_profiling_controls { ++ union kbase_pointer buffer; ++ __u32 count; ++}; ++ ++#define KBASE_IOCTL_GET_PROFILING_CONTROLS \ ++ _IOW(KBASE_IOCTL_TYPE, 26, struct kbase_ioctl_get_profiling_controls) ++ ++/** ++ * struct kbase_ioctl_mem_profile_add - Provide profiling information to kernel ++ * @buffer: Pointer to the information ++ * @len: Length ++ * @padding: Padding ++ * ++ * The data provided is accessible through a debugfs file ++ */ ++struct kbase_ioctl_mem_profile_add { ++ union kbase_pointer buffer; ++ __u32 len; ++ __u32 padding; ++}; ++ ++#define KBASE_IOCTL_MEM_PROFILE_ADD \ ++ _IOW(KBASE_IOCTL_TYPE, 27, struct kbase_ioctl_mem_profile_add) ++ ++/** ++ * struct kbase_ioctl_soft_event_update - Update the status of a soft-event ++ * @event: GPU address of the event which has been updated ++ * @new_status: The new status to set ++ * @flags: Flags for future expansion ++ */ ++struct kbase_ioctl_soft_event_update { ++ __u64 event; ++ __u32 new_status; ++ __u32 flags; ++}; ++ ++#define KBASE_IOCTL_SOFT_EVENT_UPDATE \ ++ _IOW(KBASE_IOCTL_TYPE, 28, struct kbase_ioctl_soft_event_update) ++ ++/*************** ++ * test ioctls * ++ ***************/ ++#if MALI_UNIT_TEST ++/* These ioctls are purely for test purposes and are not used in the production ++ * driver, they therefore may change without notice ++ */ ++ ++#define KBASE_IOCTL_TEST_TYPE (KBASE_IOCTL_TYPE + 1) ++ ++/** ++ * struct kbase_ioctl_tlstream_test - Start a timeline stream test ++ * ++ * @tpw_count: number of trace point writers in each context ++ * @msg_delay: time delay between tracepoints from one writer in milliseconds ++ * @msg_count: number of trace points written by one writer ++ * @aux_msg: if non-zero aux messages will be included ++ */ ++struct kbase_ioctl_tlstream_test { ++ __u32 tpw_count; ++ __u32 msg_delay; ++ __u32 msg_count; ++ __u32 aux_msg; ++}; ++ ++#define KBASE_IOCTL_TLSTREAM_TEST \ ++ _IOW(KBASE_IOCTL_TEST_TYPE, 1, struct kbase_ioctl_tlstream_test) ++ ++/** ++ * struct kbase_ioctl_tlstream_stats - Read tlstream stats for test purposes ++ * @bytes_collected: number of bytes read by user ++ * @bytes_generated: number of bytes generated by tracepoints ++ */ ++struct kbase_ioctl_tlstream_stats { ++ __u32 bytes_collected; ++ __u32 bytes_generated; ++}; ++ ++#define KBASE_IOCTL_TLSTREAM_STATS \ ++ _IOR(KBASE_IOCTL_TEST_TYPE, 2, struct kbase_ioctl_tlstream_stats) ++ ++#endif ++ ++/********************************** ++ * Definitions for GPU properties * ++ **********************************/ ++#define KBASE_GPUPROP_VALUE_SIZE_U8 (0x0) ++#define KBASE_GPUPROP_VALUE_SIZE_U16 (0x1) ++#define KBASE_GPUPROP_VALUE_SIZE_U32 (0x2) ++#define KBASE_GPUPROP_VALUE_SIZE_U64 (0x3) ++ ++#define KBASE_GPUPROP_PRODUCT_ID 1 ++#define KBASE_GPUPROP_VERSION_STATUS 2 ++#define KBASE_GPUPROP_MINOR_REVISION 3 ++#define KBASE_GPUPROP_MAJOR_REVISION 4 ++#define KBASE_GPUPROP_GPU_SPEED_MHZ 5 ++#define KBASE_GPUPROP_GPU_FREQ_KHZ_MAX 6 ++#define KBASE_GPUPROP_GPU_FREQ_KHZ_MIN 7 ++#define KBASE_GPUPROP_LOG2_PROGRAM_COUNTER_SIZE 8 ++#define KBASE_GPUPROP_TEXTURE_FEATURES_0 9 ++#define KBASE_GPUPROP_TEXTURE_FEATURES_1 10 ++#define KBASE_GPUPROP_TEXTURE_FEATURES_2 11 ++#define KBASE_GPUPROP_GPU_AVAILABLE_MEMORY_SIZE 12 ++ ++#define KBASE_GPUPROP_L2_LOG2_LINE_SIZE 13 ++#define KBASE_GPUPROP_L2_LOG2_CACHE_SIZE 14 ++#define KBASE_GPUPROP_L2_NUM_L2_SLICES 15 ++ ++#define KBASE_GPUPROP_TILER_BIN_SIZE_BYTES 16 ++#define KBASE_GPUPROP_TILER_MAX_ACTIVE_LEVELS 17 ++ ++#define KBASE_GPUPROP_MAX_THREADS 18 ++#define KBASE_GPUPROP_MAX_WORKGROUP_SIZE 19 ++#define KBASE_GPUPROP_MAX_BARRIER_SIZE 20 ++#define KBASE_GPUPROP_MAX_REGISTERS 21 ++#define KBASE_GPUPROP_MAX_TASK_QUEUE 22 ++#define KBASE_GPUPROP_MAX_THREAD_GROUP_SPLIT 23 ++#define KBASE_GPUPROP_IMPL_TECH 24 ++ ++#define KBASE_GPUPROP_RAW_SHADER_PRESENT 25 ++#define KBASE_GPUPROP_RAW_TILER_PRESENT 26 ++#define KBASE_GPUPROP_RAW_L2_PRESENT 27 ++#define KBASE_GPUPROP_RAW_STACK_PRESENT 28 ++#define KBASE_GPUPROP_RAW_L2_FEATURES 29 ++#define KBASE_GPUPROP_RAW_SUSPEND_SIZE 30 ++#define KBASE_GPUPROP_RAW_MEM_FEATURES 31 ++#define KBASE_GPUPROP_RAW_MMU_FEATURES 32 ++#define KBASE_GPUPROP_RAW_AS_PRESENT 33 ++#define KBASE_GPUPROP_RAW_JS_PRESENT 34 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_0 35 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_1 36 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_2 37 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_3 38 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_4 39 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_5 40 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_6 41 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_7 42 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_8 43 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_9 44 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_10 45 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_11 46 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_12 47 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_13 48 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_14 49 ++#define KBASE_GPUPROP_RAW_JS_FEATURES_15 50 ++#define KBASE_GPUPROP_RAW_TILER_FEATURES 51 ++#define KBASE_GPUPROP_RAW_TEXTURE_FEATURES_0 52 ++#define KBASE_GPUPROP_RAW_TEXTURE_FEATURES_1 53 ++#define KBASE_GPUPROP_RAW_TEXTURE_FEATURES_2 54 ++#define KBASE_GPUPROP_RAW_GPU_ID 55 ++#define KBASE_GPUPROP_RAW_THREAD_MAX_THREADS 56 ++#define KBASE_GPUPROP_RAW_THREAD_MAX_WORKGROUP_SIZE 57 ++#define KBASE_GPUPROP_RAW_THREAD_MAX_BARRIER_SIZE 58 ++#define KBASE_GPUPROP_RAW_THREAD_FEATURES 59 ++#define KBASE_GPUPROP_RAW_COHERENCY_MODE 60 ++ ++#define KBASE_GPUPROP_COHERENCY_NUM_GROUPS 61 ++#define KBASE_GPUPROP_COHERENCY_NUM_CORE_GROUPS 62 ++#define KBASE_GPUPROP_COHERENCY_COHERENCY 63 ++#define KBASE_GPUPROP_COHERENCY_GROUP_0 64 ++#define KBASE_GPUPROP_COHERENCY_GROUP_1 65 ++#define KBASE_GPUPROP_COHERENCY_GROUP_2 66 ++#define KBASE_GPUPROP_COHERENCY_GROUP_3 67 ++#define KBASE_GPUPROP_COHERENCY_GROUP_4 68 ++#define KBASE_GPUPROP_COHERENCY_GROUP_5 69 ++#define KBASE_GPUPROP_COHERENCY_GROUP_6 70 ++#define KBASE_GPUPROP_COHERENCY_GROUP_7 71 ++#define KBASE_GPUPROP_COHERENCY_GROUP_8 72 ++#define KBASE_GPUPROP_COHERENCY_GROUP_9 73 ++#define KBASE_GPUPROP_COHERENCY_GROUP_10 74 ++#define KBASE_GPUPROP_COHERENCY_GROUP_11 75 ++#define KBASE_GPUPROP_COHERENCY_GROUP_12 76 ++#define KBASE_GPUPROP_COHERENCY_GROUP_13 77 ++#define KBASE_GPUPROP_COHERENCY_GROUP_14 78 ++#define KBASE_GPUPROP_COHERENCY_GROUP_15 79 ++ ++#ifdef __cpluscplus ++} ++#endif ++ ++#endif +diff --git a/drivers/gpu/arm/midgard/mali_kbase_jd.c b/drivers/gpu/arm/midgard/mali_kbase_jd.c +new file mode 100755 +index 000000000000..d9d8658d31dc +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_jd.c +@@ -0,0 +1,1903 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#if defined(CONFIG_DMA_SHARED_BUFFER) ++#include ++#endif /* defined(CONFIG_DMA_SHARED_BUFFER) */ ++#ifdef CONFIG_COMPAT ++#include ++#endif ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#include ++#include ++ ++#include "mali_kbase_dma_fence.h" ++ ++#define beenthere(kctx, f, a...) dev_dbg(kctx->kbdev->dev, "%s:" f, __func__, ##a) ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 8, 0) ++/* random32 was renamed to prandom_u32 in 3.8 */ ++#define prandom_u32 random32 ++#endif ++ ++/* Return whether katom will run on the GPU or not. Currently only soft jobs and ++ * dependency-only atoms do not run on the GPU */ ++#define IS_GPU_ATOM(katom) (!((katom->core_req & BASE_JD_REQ_SOFT_JOB) || \ ++ ((katom->core_req & BASE_JD_REQ_ATOM_TYPE) == \ ++ BASE_JD_REQ_DEP))) ++/* ++ * This is the kernel side of the API. Only entry points are: ++ * - kbase_jd_submit(): Called from userspace to submit a single bag ++ * - kbase_jd_done(): Called from interrupt context to track the ++ * completion of a job. ++ * Callouts: ++ * - to the job manager (enqueue a job) ++ * - to the event subsystem (signals the completion/failure of bag/job-chains). ++ */ ++ ++static void __user * ++get_compat_pointer(struct kbase_context *kctx, const union kbase_pointer *p) ++{ ++#ifdef CONFIG_COMPAT ++ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) ++ return compat_ptr(p->compat_value); ++#endif ++ return p->value; ++} ++ ++/* Runs an atom, either by handing to the JS or by immediately running it in the case of soft-jobs ++ * ++ * Returns whether the JS needs a reschedule. ++ * ++ * Note that the caller must also check the atom status and ++ * if it is KBASE_JD_ATOM_STATE_COMPLETED must call jd_done_nolock ++ */ ++static int jd_run_atom(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ ++ KBASE_DEBUG_ASSERT(katom->status != KBASE_JD_ATOM_STATE_UNUSED); ++ ++ if ((katom->core_req & BASE_JD_REQ_ATOM_TYPE) == BASE_JD_REQ_DEP) { ++ /* Dependency only atom */ ++ katom->status = KBASE_JD_ATOM_STATE_COMPLETED; ++ return 0; ++ } else if (katom->core_req & BASE_JD_REQ_SOFT_JOB) { ++ /* Soft-job */ ++ if (katom->will_fail_event_code) { ++ katom->status = KBASE_JD_ATOM_STATE_COMPLETED; ++ return 0; ++ } ++ if ((katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) ++ == BASE_JD_REQ_SOFT_REPLAY) { ++ if (!kbase_replay_process(katom)) ++ katom->status = KBASE_JD_ATOM_STATE_COMPLETED; ++ } else if (kbase_process_soft_job(katom) == 0) { ++ kbase_finish_soft_job(katom); ++ katom->status = KBASE_JD_ATOM_STATE_COMPLETED; ++ } ++ return 0; ++ } ++ ++ katom->status = KBASE_JD_ATOM_STATE_IN_JS; ++ /* Queue an action about whether we should try scheduling a context */ ++ return kbasep_js_add_job(kctx, katom); ++} ++ ++#if defined(CONFIG_KDS) || defined(CONFIG_MALI_DMA_FENCE) ++void kbase_jd_dep_clear_locked(struct kbase_jd_atom *katom) ++{ ++ struct kbase_device *kbdev; ++ ++ KBASE_DEBUG_ASSERT(katom); ++ kbdev = katom->kctx->kbdev; ++ KBASE_DEBUG_ASSERT(kbdev); ++ ++ /* Check whether the atom's other dependencies were already met. If ++ * katom is a GPU atom then the job scheduler may be able to represent ++ * the dependencies, hence we may attempt to submit it before they are ++ * met. Other atoms must have had both dependencies resolved. ++ */ ++ if (IS_GPU_ATOM(katom) || ++ (!kbase_jd_katom_dep_atom(&katom->dep[0]) && ++ !kbase_jd_katom_dep_atom(&katom->dep[1]))) { ++ /* katom dep complete, attempt to run it */ ++ bool resched = false; ++ ++ resched = jd_run_atom(katom); ++ ++ if (katom->status == KBASE_JD_ATOM_STATE_COMPLETED) { ++ /* The atom has already finished */ ++ resched |= jd_done_nolock(katom, NULL); ++ } ++ ++ if (resched) ++ kbase_js_sched_all(kbdev); ++ } ++} ++#endif ++ ++#ifdef CONFIG_KDS ++ ++/* Add the katom to the kds waiting list. ++ * Atoms must be added to the waiting list after a successful call to kds_async_waitall. ++ * The caller must hold the kbase_jd_context.lock */ ++ ++static void kbase_jd_kds_waiters_add(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx; ++ ++ KBASE_DEBUG_ASSERT(katom); ++ ++ kctx = katom->kctx; ++ ++ list_add_tail(&katom->node, &kctx->waiting_kds_resource); ++} ++ ++/* Remove the katom from the kds waiting list. ++ * Atoms must be removed from the waiting list before a call to kds_resource_set_release_sync. ++ * The supplied katom must first have been added to the list with a call to kbase_jd_kds_waiters_add. ++ * The caller must hold the kbase_jd_context.lock */ ++ ++static void kbase_jd_kds_waiters_remove(struct kbase_jd_atom *katom) ++{ ++ KBASE_DEBUG_ASSERT(katom); ++ list_del(&katom->node); ++} ++ ++static void kds_dep_clear(void *callback_parameter, void *callback_extra_parameter) ++{ ++ struct kbase_jd_atom *katom; ++ struct kbase_jd_context *ctx; ++ ++ katom = (struct kbase_jd_atom *)callback_parameter; ++ KBASE_DEBUG_ASSERT(katom); ++ ++ ctx = &katom->kctx->jctx; ++ ++ /* If KDS resource has already been satisfied (e.g. due to zapping) ++ * do nothing. ++ */ ++ mutex_lock(&ctx->lock); ++ if (!katom->kds_dep_satisfied) { ++ katom->kds_dep_satisfied = true; ++ kbase_jd_dep_clear_locked(katom); ++ } ++ mutex_unlock(&ctx->lock); ++} ++ ++static void kbase_cancel_kds_wait_job(struct kbase_jd_atom *katom) ++{ ++ KBASE_DEBUG_ASSERT(katom); ++ ++ /* Prevent job_done_nolock from being called twice on an atom when ++ * there is a race between job completion and cancellation */ ++ ++ if (katom->status == KBASE_JD_ATOM_STATE_QUEUED) { ++ /* Wait was cancelled - zap the atom */ ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ if (jd_done_nolock(katom, NULL)) ++ kbase_js_sched_all(katom->kctx->kbdev); ++ } ++} ++#endif /* CONFIG_KDS */ ++ ++void kbase_jd_free_external_resources(struct kbase_jd_atom *katom) ++{ ++#ifdef CONFIG_KDS ++ if (katom->kds_rset) { ++ struct kbase_jd_context *jctx = &katom->kctx->jctx; ++ ++ /* ++ * As the atom is no longer waiting, remove it from ++ * the waiting list. ++ */ ++ ++ mutex_lock(&jctx->lock); ++ kbase_jd_kds_waiters_remove(katom); ++ mutex_unlock(&jctx->lock); ++ ++ /* Release the kds resource or cancel if zapping */ ++ kds_resource_set_release_sync(&katom->kds_rset); ++ } ++#endif /* CONFIG_KDS */ ++ ++#ifdef CONFIG_MALI_DMA_FENCE ++ /* Flush dma-fence workqueue to ensure that any callbacks that may have ++ * been queued are done before continuing. ++ * Any successfully completed atom would have had all it's callbacks ++ * completed before the atom was run, so only flush for failed atoms. ++ */ ++ if (katom->event_code != BASE_JD_EVENT_DONE) ++ flush_workqueue(katom->kctx->dma_fence.wq); ++#endif /* CONFIG_MALI_DMA_FENCE */ ++} ++ ++static void kbase_jd_post_external_resources(struct kbase_jd_atom *katom) ++{ ++ KBASE_DEBUG_ASSERT(katom); ++ KBASE_DEBUG_ASSERT(katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES); ++ ++#ifdef CONFIG_KDS ++ /* Prevent the KDS resource from triggering the atom in case of zapping */ ++ if (katom->kds_rset) ++ katom->kds_dep_satisfied = true; ++#endif /* CONFIG_KDS */ ++ ++#ifdef CONFIG_MALI_DMA_FENCE ++ kbase_dma_fence_signal(katom); ++#endif /* CONFIG_MALI_DMA_FENCE */ ++ ++ kbase_gpu_vm_lock(katom->kctx); ++ /* only roll back if extres is non-NULL */ ++ if (katom->extres) { ++ u32 res_no; ++ ++ res_no = katom->nr_extres; ++ while (res_no-- > 0) { ++ struct kbase_mem_phy_alloc *alloc = katom->extres[res_no].alloc; ++ struct kbase_va_region *reg; ++ ++ reg = kbase_region_tracker_find_region_base_address( ++ katom->kctx, ++ katom->extres[res_no].gpu_address); ++ kbase_unmap_external_resource(katom->kctx, reg, alloc); ++ } ++ kfree(katom->extres); ++ katom->extres = NULL; ++ } ++ kbase_gpu_vm_unlock(katom->kctx); ++} ++ ++/* ++ * Set up external resources needed by this job. ++ * ++ * jctx.lock must be held when this is called. ++ */ ++ ++static int kbase_jd_pre_external_resources(struct kbase_jd_atom *katom, const struct base_jd_atom_v2 *user_atom) ++{ ++ int err_ret_val = -EINVAL; ++ u32 res_no; ++#ifdef CONFIG_KDS ++ u32 kds_res_count = 0; ++ struct kds_resource **kds_resources = NULL; ++ unsigned long *kds_access_bitmap = NULL; ++#endif /* CONFIG_KDS */ ++#ifdef CONFIG_MALI_DMA_FENCE ++ struct kbase_dma_fence_resv_info info = { ++ .dma_fence_resv_count = 0, ++ }; ++#ifdef CONFIG_SYNC ++ /* ++ * When both dma-buf fence and Android native sync is enabled, we ++ * disable dma-buf fence for contexts that are using Android native ++ * fences. ++ */ ++ const bool implicit_sync = !kbase_ctx_flag(katom->kctx, ++ KCTX_NO_IMPLICIT_SYNC); ++#else /* CONFIG_SYNC */ ++ const bool implicit_sync = true; ++#endif /* CONFIG_SYNC */ ++#endif /* CONFIG_MALI_DMA_FENCE */ ++ struct base_external_resource *input_extres; ++ ++ KBASE_DEBUG_ASSERT(katom); ++ KBASE_DEBUG_ASSERT(katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES); ++ ++ /* no resources encoded, early out */ ++ if (!katom->nr_extres) ++ return -EINVAL; ++ ++ katom->extres = kmalloc_array(katom->nr_extres, sizeof(*katom->extres), GFP_KERNEL); ++ if (NULL == katom->extres) { ++ err_ret_val = -ENOMEM; ++ goto early_err_out; ++ } ++ ++ /* copy user buffer to the end of our real buffer. ++ * Make sure the struct sizes haven't changed in a way ++ * we don't support */ ++ BUILD_BUG_ON(sizeof(*input_extres) > sizeof(*katom->extres)); ++ input_extres = (struct base_external_resource *) ++ (((unsigned char *)katom->extres) + ++ (sizeof(*katom->extres) - sizeof(*input_extres)) * ++ katom->nr_extres); ++ ++ if (copy_from_user(input_extres, ++ get_compat_pointer(katom->kctx, &user_atom->extres_list), ++ sizeof(*input_extres) * katom->nr_extres) != 0) { ++ err_ret_val = -EINVAL; ++ goto early_err_out; ++ } ++#ifdef CONFIG_KDS ++ /* assume we have to wait for all */ ++ KBASE_DEBUG_ASSERT(0 != katom->nr_extres); ++ kds_resources = kmalloc_array(katom->nr_extres, sizeof(struct kds_resource *), GFP_KERNEL); ++ ++ if (!kds_resources) { ++ err_ret_val = -ENOMEM; ++ goto early_err_out; ++ } ++ ++ KBASE_DEBUG_ASSERT(0 != katom->nr_extres); ++ kds_access_bitmap = kcalloc(BITS_TO_LONGS(katom->nr_extres), ++ sizeof(unsigned long), ++ GFP_KERNEL); ++ if (!kds_access_bitmap) { ++ err_ret_val = -ENOMEM; ++ goto early_err_out; ++ } ++#endif /* CONFIG_KDS */ ++ ++#ifdef CONFIG_MALI_DMA_FENCE ++ if (implicit_sync) { ++ info.resv_objs = kmalloc_array(katom->nr_extres, ++ sizeof(struct reservation_object *), ++ GFP_KERNEL); ++ if (!info.resv_objs) { ++ err_ret_val = -ENOMEM; ++ goto early_err_out; ++ } ++ ++ info.dma_fence_excl_bitmap = ++ kcalloc(BITS_TO_LONGS(katom->nr_extres), ++ sizeof(unsigned long), GFP_KERNEL); ++ if (!info.dma_fence_excl_bitmap) { ++ err_ret_val = -ENOMEM; ++ goto early_err_out; ++ } ++ } ++#endif /* CONFIG_MALI_DMA_FENCE */ ++ ++ /* Take the processes mmap lock */ ++ down_read(¤t->mm->mmap_lock); ++ ++ /* need to keep the GPU VM locked while we set up UMM buffers */ ++ kbase_gpu_vm_lock(katom->kctx); ++ for (res_no = 0; res_no < katom->nr_extres; res_no++) { ++ struct base_external_resource *res; ++ struct kbase_va_region *reg; ++ struct kbase_mem_phy_alloc *alloc; ++ bool exclusive; ++ ++ res = &input_extres[res_no]; ++ exclusive = (res->ext_resource & BASE_EXT_RES_ACCESS_EXCLUSIVE) ++ ? true : false; ++ reg = kbase_region_tracker_find_region_enclosing_address( ++ katom->kctx, ++ res->ext_resource & ~BASE_EXT_RES_ACCESS_EXCLUSIVE); ++ /* did we find a matching region object? */ ++ if (NULL == reg || (reg->flags & KBASE_REG_FREE)) { ++ /* roll back */ ++ goto failed_loop; ++ } ++ ++ if (!(katom->core_req & BASE_JD_REQ_SOFT_JOB) && ++ (reg->flags & KBASE_REG_SECURE)) { ++ katom->atom_flags |= KBASE_KATOM_FLAG_PROTECTED; ++ } ++ ++ alloc = kbase_map_external_resource(katom->kctx, reg, ++ current->mm ++#ifdef CONFIG_KDS ++ , &kds_res_count, kds_resources, ++ kds_access_bitmap, exclusive ++#endif ++ ); ++ if (!alloc) { ++ err_ret_val = -EINVAL; ++ goto failed_loop; ++ } ++ ++#ifdef CONFIG_MALI_DMA_FENCE ++ if (implicit_sync && ++ reg->gpu_alloc->type == KBASE_MEM_TYPE_IMPORTED_UMM) { ++ struct reservation_object *resv; ++ ++ resv = reg->gpu_alloc->imported.umm.dma_buf->resv; ++ if (resv) ++ kbase_dma_fence_add_reservation(resv, &info, ++ exclusive); ++ } ++#endif /* CONFIG_MALI_DMA_FENCE */ ++ ++ /* finish with updating out array with the data we found */ ++ /* NOTE: It is important that this is the last thing we do (or ++ * at least not before the first write) as we overwrite elements ++ * as we loop and could be overwriting ourself, so no writes ++ * until the last read for an element. ++ * */ ++ katom->extres[res_no].gpu_address = reg->start_pfn << PAGE_SHIFT; /* save the start_pfn (as an address, not pfn) to use fast lookup later */ ++ katom->extres[res_no].alloc = alloc; ++ } ++ /* successfully parsed the extres array */ ++ /* drop the vm lock before we call into kds */ ++ kbase_gpu_vm_unlock(katom->kctx); ++ ++ /* Release the processes mmap lock */ ++ up_read(¤t->mm->mmap_lock); ++ ++#ifdef CONFIG_KDS ++ if (kds_res_count) { ++ int wait_failed; ++ ++ /* We have resources to wait for with kds */ ++ katom->kds_dep_satisfied = false; ++ ++ wait_failed = kds_async_waitall(&katom->kds_rset, ++ &katom->kctx->jctx.kds_cb, katom, NULL, ++ kds_res_count, kds_access_bitmap, ++ kds_resources); ++ ++ if (wait_failed) ++ goto failed_kds_setup; ++ else ++ kbase_jd_kds_waiters_add(katom); ++ } else { ++ /* Nothing to wait for, so kds dep met */ ++ katom->kds_dep_satisfied = true; ++ } ++ kfree(kds_resources); ++ kfree(kds_access_bitmap); ++#endif /* CONFIG_KDS */ ++ ++#ifdef CONFIG_MALI_DMA_FENCE ++ if (implicit_sync) { ++ if (info.dma_fence_resv_count) { ++ int ret; ++ ++ ret = kbase_dma_fence_wait(katom, &info); ++ if (ret < 0) ++ goto failed_dma_fence_setup; ++ } ++ ++ kfree(info.resv_objs); ++ kfree(info.dma_fence_excl_bitmap); ++ } ++#endif /* CONFIG_MALI_DMA_FENCE */ ++ ++ /* all done OK */ ++ return 0; ++ ++/* error handling section */ ++ ++#ifdef CONFIG_MALI_DMA_FENCE ++failed_dma_fence_setup: ++#ifdef CONFIG_KDS ++ /* If we are here, dma_fence setup failed but KDS didn't. ++ * Revert KDS setup if any. ++ */ ++ if (kds_res_count) { ++ mutex_unlock(&katom->kctx->jctx.lock); ++ kds_resource_set_release_sync(&katom->kds_rset); ++ mutex_lock(&katom->kctx->jctx.lock); ++ ++ kbase_jd_kds_waiters_remove(katom); ++ katom->kds_dep_satisfied = true; ++ } ++#endif /* CONFIG_KDS */ ++#endif /* CONFIG_MALI_DMA_FENCE */ ++#ifdef CONFIG_KDS ++failed_kds_setup: ++#endif ++#if defined(CONFIG_KDS) || defined(CONFIG_MALI_DMA_FENCE) ++ /* Lock the processes mmap lock */ ++ down_read(¤t->mm->mmap_lock); ++ ++ /* lock before we unmap */ ++ kbase_gpu_vm_lock(katom->kctx); ++#endif ++ ++ failed_loop: ++ /* undo the loop work */ ++ while (res_no-- > 0) { ++ struct kbase_mem_phy_alloc *alloc = katom->extres[res_no].alloc; ++ ++ kbase_unmap_external_resource(katom->kctx, NULL, alloc); ++ } ++ kbase_gpu_vm_unlock(katom->kctx); ++ ++ /* Release the processes mmap lock */ ++ up_read(¤t->mm->mmap_lock); ++ ++ early_err_out: ++ kfree(katom->extres); ++ katom->extres = NULL; ++#ifdef CONFIG_KDS ++ kfree(kds_resources); ++ kfree(kds_access_bitmap); ++#endif /* CONFIG_KDS */ ++#ifdef CONFIG_MALI_DMA_FENCE ++ if (implicit_sync) { ++ kfree(info.resv_objs); ++ kfree(info.dma_fence_excl_bitmap); ++ } ++#endif ++ return err_ret_val; ++} ++ ++static inline void jd_resolve_dep(struct list_head *out_list, ++ struct kbase_jd_atom *katom, ++ u8 d, bool ctx_is_dying) ++{ ++ u8 other_d = !d; ++ ++ while (!list_empty(&katom->dep_head[d])) { ++ struct kbase_jd_atom *dep_atom; ++ struct kbase_jd_atom *other_dep_atom; ++ u8 dep_type; ++ ++ dep_atom = list_entry(katom->dep_head[d].next, ++ struct kbase_jd_atom, dep_item[d]); ++ list_del(katom->dep_head[d].next); ++ ++ dep_type = kbase_jd_katom_dep_type(&dep_atom->dep[d]); ++ kbase_jd_katom_dep_clear(&dep_atom->dep[d]); ++ ++ if (katom->event_code != BASE_JD_EVENT_DONE && ++ (dep_type != BASE_JD_DEP_TYPE_ORDER)) { ++#ifdef CONFIG_KDS ++ if (!dep_atom->kds_dep_satisfied) { ++ /* Just set kds_dep_satisfied to true. If the callback happens after this then it will early out and ++ * do nothing. If the callback doesn't happen then kbase_jd_post_external_resources will clean up ++ */ ++ dep_atom->kds_dep_satisfied = true; ++ } ++#endif ++ ++#ifdef CONFIG_MALI_DMA_FENCE ++ kbase_dma_fence_cancel_callbacks(dep_atom); ++#endif ++ ++ dep_atom->event_code = katom->event_code; ++ KBASE_DEBUG_ASSERT(dep_atom->status != ++ KBASE_JD_ATOM_STATE_UNUSED); ++ ++ if ((dep_atom->core_req & BASE_JD_REQ_SOFT_REPLAY) ++ != BASE_JD_REQ_SOFT_REPLAY) { ++ dep_atom->will_fail_event_code = ++ dep_atom->event_code; ++ } else { ++ dep_atom->status = ++ KBASE_JD_ATOM_STATE_COMPLETED; ++ } ++ } ++ other_dep_atom = (struct kbase_jd_atom *) ++ kbase_jd_katom_dep_atom(&dep_atom->dep[other_d]); ++ ++ if (!dep_atom->in_jd_list && (!other_dep_atom || ++ (IS_GPU_ATOM(dep_atom) && !ctx_is_dying && ++ !dep_atom->will_fail_event_code && ++ !other_dep_atom->will_fail_event_code))) { ++ bool dep_satisfied = true; ++#ifdef CONFIG_MALI_DMA_FENCE ++ int dep_count; ++ ++ dep_count = kbase_fence_dep_count_read(dep_atom); ++ if (likely(dep_count == -1)) { ++ dep_satisfied = true; ++ } else { ++ /* ++ * There are either still active callbacks, or ++ * all fences for this @dep_atom has signaled, ++ * but the worker that will queue the atom has ++ * not yet run. ++ * ++ * Wait for the fences to signal and the fence ++ * worker to run and handle @dep_atom. If ++ * @dep_atom was completed due to error on ++ * @katom, then the fence worker will pick up ++ * the complete status and error code set on ++ * @dep_atom above. ++ */ ++ dep_satisfied = false; ++ } ++#endif /* CONFIG_MALI_DMA_FENCE */ ++ ++#ifdef CONFIG_KDS ++ dep_satisfied = dep_satisfied && dep_atom->kds_dep_satisfied; ++#endif ++ ++ if (dep_satisfied) { ++ dep_atom->in_jd_list = true; ++ list_add_tail(&dep_atom->jd_item, out_list); ++ } ++ } ++ } ++} ++ ++KBASE_EXPORT_TEST_API(jd_resolve_dep); ++ ++#if MALI_CUSTOMER_RELEASE == 0 ++static void jd_force_failure(struct kbase_device *kbdev, struct kbase_jd_atom *katom) ++{ ++ kbdev->force_replay_count++; ++ ++ if (kbdev->force_replay_count >= kbdev->force_replay_limit) { ++ kbdev->force_replay_count = 0; ++ katom->event_code = BASE_JD_EVENT_FORCE_REPLAY; ++ ++ if (kbdev->force_replay_random) ++ kbdev->force_replay_limit = ++ (prandom_u32() % KBASEP_FORCE_REPLAY_RANDOM_LIMIT) + 1; ++ ++ dev_info(kbdev->dev, "force_replay : promoting to error\n"); ++ } ++} ++ ++/** Test to see if atom should be forced to fail. ++ * ++ * This function will check if an atom has a replay job as a dependent. If so ++ * then it will be considered for forced failure. */ ++static void jd_check_force_failure(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ struct kbase_device *kbdev = kctx->kbdev; ++ int i; ++ ++ if ((kbdev->force_replay_limit == KBASEP_FORCE_REPLAY_DISABLED) || ++ (katom->core_req & BASEP_JD_REQ_EVENT_NEVER)) ++ return; ++ ++ for (i = 1; i < BASE_JD_ATOM_COUNT; i++) { ++ if (kbase_jd_katom_dep_atom(&kctx->jctx.atoms[i].dep[0]) == katom || ++ kbase_jd_katom_dep_atom(&kctx->jctx.atoms[i].dep[1]) == katom) { ++ struct kbase_jd_atom *dep_atom = &kctx->jctx.atoms[i]; ++ ++ if ((dep_atom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) == ++ BASE_JD_REQ_SOFT_REPLAY && ++ (dep_atom->core_req & kbdev->force_replay_core_req) ++ == kbdev->force_replay_core_req) { ++ jd_force_failure(kbdev, katom); ++ return; ++ } ++ } ++ } ++} ++#endif ++ ++/** ++ * is_dep_valid - Validate that a dependency is valid for early dependency ++ * submission ++ * @katom: Dependency atom to validate ++ * ++ * A dependency is valid if any of the following are true : ++ * - It does not exist (a non-existent dependency does not block submission) ++ * - It is in the job scheduler ++ * - It has completed, does not have a failure event code, and has not been ++ * marked to fail in the future ++ * ++ * Return: true if valid, false otherwise ++ */ ++static bool is_dep_valid(struct kbase_jd_atom *katom) ++{ ++ /* If there's no dependency then this is 'valid' from the perspective of ++ * early dependency submission */ ++ if (!katom) ++ return true; ++ ++ /* Dependency must have reached the job scheduler */ ++ if (katom->status < KBASE_JD_ATOM_STATE_IN_JS) ++ return false; ++ ++ /* If dependency has completed and has failed or will fail then it is ++ * not valid */ ++ if (katom->status >= KBASE_JD_ATOM_STATE_HW_COMPLETED && ++ (katom->event_code != BASE_JD_EVENT_DONE || ++ katom->will_fail_event_code)) ++ return false; ++ ++ return true; ++} ++ ++static void jd_try_submitting_deps(struct list_head *out_list, ++ struct kbase_jd_atom *node) ++{ ++ int i; ++ ++ for (i = 0; i < 2; i++) { ++ struct list_head *pos; ++ ++ list_for_each(pos, &node->dep_head[i]) { ++ struct kbase_jd_atom *dep_atom = list_entry(pos, ++ struct kbase_jd_atom, dep_item[i]); ++ ++ if (IS_GPU_ATOM(dep_atom) && !dep_atom->in_jd_list) { ++ /*Check if atom deps look sane*/ ++ bool dep0_valid = is_dep_valid( ++ dep_atom->dep[0].atom); ++ bool dep1_valid = is_dep_valid( ++ dep_atom->dep[1].atom); ++ bool dep_satisfied = true; ++#ifdef CONFIG_MALI_DMA_FENCE ++ int dep_count; ++ ++ dep_count = kbase_fence_dep_count_read( ++ dep_atom); ++ if (likely(dep_count == -1)) { ++ dep_satisfied = true; ++ } else { ++ /* ++ * There are either still active callbacks, or ++ * all fences for this @dep_atom has signaled, ++ * but the worker that will queue the atom has ++ * not yet run. ++ * ++ * Wait for the fences to signal and the fence ++ * worker to run and handle @dep_atom. If ++ * @dep_atom was completed due to error on ++ * @katom, then the fence worker will pick up ++ * the complete status and error code set on ++ * @dep_atom above. ++ */ ++ dep_satisfied = false; ++ } ++#endif /* CONFIG_MALI_DMA_FENCE */ ++#ifdef CONFIG_KDS ++ dep_satisfied = dep_satisfied && ++ dep_atom->kds_dep_satisfied; ++#endif ++ ++ if (dep0_valid && dep1_valid && dep_satisfied) { ++ dep_atom->in_jd_list = true; ++ list_add(&dep_atom->jd_item, out_list); ++ } ++ } ++ } ++ } ++} ++ ++/* ++ * Perform the necessary handling of an atom that has finished running ++ * on the GPU. ++ * ++ * Note that if this is a soft-job that has had kbase_prepare_soft_job called on it then the caller ++ * is responsible for calling kbase_finish_soft_job *before* calling this function. ++ * ++ * The caller must hold the kbase_jd_context.lock. ++ */ ++bool jd_done_nolock(struct kbase_jd_atom *katom, ++ struct list_head *completed_jobs_ctx) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ struct kbase_device *kbdev = kctx->kbdev; ++ struct list_head completed_jobs; ++ struct list_head runnable_jobs; ++ bool need_to_try_schedule_context = false; ++ int i; ++ ++ INIT_LIST_HEAD(&completed_jobs); ++ INIT_LIST_HEAD(&runnable_jobs); ++ ++ KBASE_DEBUG_ASSERT(katom->status != KBASE_JD_ATOM_STATE_UNUSED); ++ ++#if MALI_CUSTOMER_RELEASE == 0 ++ jd_check_force_failure(katom); ++#endif ++ ++ /* This is needed in case an atom is failed due to being invalid, this ++ * can happen *before* the jobs that the atom depends on have completed */ ++ for (i = 0; i < 2; i++) { ++ if (kbase_jd_katom_dep_atom(&katom->dep[i])) { ++ list_del(&katom->dep_item[i]); ++ kbase_jd_katom_dep_clear(&katom->dep[i]); ++ } ++ } ++ ++ /* With PRLAM-10817 or PRLAM-10959 the last tile of a fragment job being soft-stopped can fail with ++ * BASE_JD_EVENT_TILE_RANGE_FAULT. ++ * ++ * So here if the fragment job failed with TILE_RANGE_FAULT and it has been soft-stopped, then we promote the ++ * error code to BASE_JD_EVENT_DONE ++ */ ++ ++ if ((kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10817) || kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10959)) && ++ katom->event_code == BASE_JD_EVENT_TILE_RANGE_FAULT) { ++ if ((katom->core_req & BASE_JD_REQ_FS) && (katom->atom_flags & KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED)) { ++ /* Promote the failure to job done */ ++ katom->event_code = BASE_JD_EVENT_DONE; ++ katom->atom_flags = katom->atom_flags & (~KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED); ++ } ++ } ++ ++ katom->status = KBASE_JD_ATOM_STATE_COMPLETED; ++ list_add_tail(&katom->jd_item, &completed_jobs); ++ ++ while (!list_empty(&completed_jobs)) { ++ katom = list_entry(completed_jobs.prev, struct kbase_jd_atom, jd_item); ++ list_del(completed_jobs.prev); ++ KBASE_DEBUG_ASSERT(katom->status == KBASE_JD_ATOM_STATE_COMPLETED); ++ ++ for (i = 0; i < 2; i++) ++ jd_resolve_dep(&runnable_jobs, katom, i, ++ kbase_ctx_flag(kctx, KCTX_DYING)); ++ ++ if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) ++ kbase_jd_post_external_resources(katom); ++ ++ while (!list_empty(&runnable_jobs)) { ++ struct kbase_jd_atom *node; ++ ++ node = list_entry(runnable_jobs.next, ++ struct kbase_jd_atom, jd_item); ++ list_del(runnable_jobs.next); ++ node->in_jd_list = false; ++ ++ KBASE_DEBUG_ASSERT(node->status != KBASE_JD_ATOM_STATE_UNUSED); ++ ++ if (node->status != KBASE_JD_ATOM_STATE_COMPLETED && ++ !kbase_ctx_flag(kctx, KCTX_DYING)) { ++ need_to_try_schedule_context |= jd_run_atom(node); ++ } else { ++ node->event_code = katom->event_code; ++ ++ if ((node->core_req & ++ BASE_JD_REQ_SOFT_JOB_TYPE) == ++ BASE_JD_REQ_SOFT_REPLAY) { ++ if (kbase_replay_process(node)) ++ /* Don't complete this atom */ ++ continue; ++ } else if (node->core_req & ++ BASE_JD_REQ_SOFT_JOB) { ++ /* If this is a fence wait soft job ++ * then remove it from the list of sync ++ * waiters. ++ */ ++ if (BASE_JD_REQ_SOFT_FENCE_WAIT == node->core_req) ++ kbasep_remove_waiting_soft_job(node); ++ ++ kbase_finish_soft_job(node); ++ } ++ node->status = KBASE_JD_ATOM_STATE_COMPLETED; ++ } ++ ++ if (node->status == KBASE_JD_ATOM_STATE_COMPLETED) { ++ list_add_tail(&node->jd_item, &completed_jobs); ++ } else if (node->status == KBASE_JD_ATOM_STATE_IN_JS && ++ !node->will_fail_event_code) { ++ /* Node successfully submitted, try submitting ++ * dependencies as they may now be representable ++ * in JS */ ++ jd_try_submitting_deps(&runnable_jobs, node); ++ } ++ } ++ ++ /* Register a completed job as a disjoint event when the GPU ++ * is in a disjoint state (ie. being reset or replaying jobs). ++ */ ++ kbase_disjoint_event_potential(kctx->kbdev); ++ if (completed_jobs_ctx) ++ list_add_tail(&katom->jd_item, completed_jobs_ctx); ++ else ++ kbase_event_post(kctx, katom); ++ ++ /* Decrement and check the TOTAL number of jobs. This includes ++ * those not tracked by the scheduler: 'not ready to run' and ++ * 'dependency-only' jobs. */ ++ if (--kctx->jctx.job_nr == 0) ++ wake_up(&kctx->jctx.zero_jobs_wait); /* All events are safely queued now, and we can signal any waiter ++ * that we've got no more jobs (so we can be safely terminated) */ ++ } ++ ++ return need_to_try_schedule_context; ++} ++ ++KBASE_EXPORT_TEST_API(jd_done_nolock); ++ ++#ifdef CONFIG_GPU_TRACEPOINTS ++enum { ++ CORE_REQ_DEP_ONLY, ++ CORE_REQ_SOFT, ++ CORE_REQ_COMPUTE, ++ CORE_REQ_FRAGMENT, ++ CORE_REQ_VERTEX, ++ CORE_REQ_TILER, ++ CORE_REQ_FRAGMENT_VERTEX, ++ CORE_REQ_FRAGMENT_VERTEX_TILER, ++ CORE_REQ_FRAGMENT_TILER, ++ CORE_REQ_VERTEX_TILER, ++ CORE_REQ_UNKNOWN ++}; ++static const char * const core_req_strings[] = { ++ "Dependency Only Job", ++ "Soft Job", ++ "Compute Shader Job", ++ "Fragment Shader Job", ++ "Vertex/Geometry Shader Job", ++ "Tiler Job", ++ "Fragment Shader + Vertex/Geometry Shader Job", ++ "Fragment Shader + Vertex/Geometry Shader Job + Tiler Job", ++ "Fragment Shader + Tiler Job", ++ "Vertex/Geometry Shader Job + Tiler Job", ++ "Unknown Job" ++}; ++static const char *kbasep_map_core_reqs_to_string(base_jd_core_req core_req) ++{ ++ if (core_req & BASE_JD_REQ_SOFT_JOB) ++ return core_req_strings[CORE_REQ_SOFT]; ++ if (core_req & BASE_JD_REQ_ONLY_COMPUTE) ++ return core_req_strings[CORE_REQ_COMPUTE]; ++ switch (core_req & (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T)) { ++ case BASE_JD_REQ_DEP: ++ return core_req_strings[CORE_REQ_DEP_ONLY]; ++ case BASE_JD_REQ_FS: ++ return core_req_strings[CORE_REQ_FRAGMENT]; ++ case BASE_JD_REQ_CS: ++ return core_req_strings[CORE_REQ_VERTEX]; ++ case BASE_JD_REQ_T: ++ return core_req_strings[CORE_REQ_TILER]; ++ case (BASE_JD_REQ_FS | BASE_JD_REQ_CS): ++ return core_req_strings[CORE_REQ_FRAGMENT_VERTEX]; ++ case (BASE_JD_REQ_FS | BASE_JD_REQ_T): ++ return core_req_strings[CORE_REQ_FRAGMENT_TILER]; ++ case (BASE_JD_REQ_CS | BASE_JD_REQ_T): ++ return core_req_strings[CORE_REQ_VERTEX_TILER]; ++ case (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T): ++ return core_req_strings[CORE_REQ_FRAGMENT_VERTEX_TILER]; ++ } ++ return core_req_strings[CORE_REQ_UNKNOWN]; ++} ++#endif ++ ++bool jd_submit_atom(struct kbase_context *kctx, const struct base_jd_atom_v2 *user_atom, struct kbase_jd_atom *katom) ++{ ++ struct kbase_jd_context *jctx = &kctx->jctx; ++ int queued = 0; ++ int i; ++ int sched_prio; ++ bool ret; ++ bool will_fail = false; ++ ++ /* Update the TOTAL number of jobs. This includes those not tracked by ++ * the scheduler: 'not ready to run' and 'dependency-only' jobs. */ ++ jctx->job_nr++; ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) ++ katom->start_timestamp.tv64 = 0; ++#else ++ katom->start_timestamp = 0; ++#endif ++ katom->udata = user_atom->udata; ++ katom->kctx = kctx; ++ katom->nr_extres = user_atom->nr_extres; ++ katom->extres = NULL; ++ katom->device_nr = user_atom->device_nr; ++ katom->affinity = 0; ++ katom->jc = user_atom->jc; ++ katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED; ++ katom->core_req = user_atom->core_req; ++ katom->atom_flags = 0; ++ katom->retry_count = 0; ++ katom->need_cache_flush_cores_retained = 0; ++ katom->pre_dep = NULL; ++ katom->post_dep = NULL; ++ katom->x_pre_dep = NULL; ++ katom->x_post_dep = NULL; ++ katom->will_fail_event_code = BASE_JD_EVENT_NOT_STARTED; ++ ++ /* Implicitly sets katom->protected_state.enter as well. */ ++ katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK; ++ ++ katom->age = kctx->age_count++; ++ ++ INIT_LIST_HEAD(&katom->jd_item); ++#ifdef CONFIG_KDS ++ /* Start by assuming that the KDS dependencies are satisfied, ++ * kbase_jd_pre_external_resources will correct this if there are dependencies */ ++ katom->kds_dep_satisfied = true; ++ katom->kds_rset = NULL; ++#endif /* CONFIG_KDS */ ++#ifdef CONFIG_MALI_DMA_FENCE ++ kbase_fence_dep_count_set(katom, -1); ++#endif ++ ++ /* Don't do anything if there is a mess up with dependencies. ++ This is done in a separate cycle to check both the dependencies at ones, otherwise ++ it will be extra complexity to deal with 1st dependency ( just added to the list ) ++ if only the 2nd one has invalid config. ++ */ ++ for (i = 0; i < 2; i++) { ++ int dep_atom_number = user_atom->pre_dep[i].atom_id; ++ base_jd_dep_type dep_atom_type = user_atom->pre_dep[i].dependency_type; ++ ++ if (dep_atom_number) { ++ if (dep_atom_type != BASE_JD_DEP_TYPE_ORDER && ++ dep_atom_type != BASE_JD_DEP_TYPE_DATA) { ++ katom->event_code = BASE_JD_EVENT_JOB_CONFIG_FAULT; ++ katom->status = KBASE_JD_ATOM_STATE_COMPLETED; ++ ++ /* Wrong dependency setup. Atom will be sent ++ * back to user space. Do not record any ++ * dependencies. */ ++ KBASE_TLSTREAM_TL_NEW_ATOM( ++ katom, ++ kbase_jd_atom_id(kctx, katom)); ++ KBASE_TLSTREAM_TL_RET_ATOM_CTX( ++ katom, kctx); ++ KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(katom, ++ TL_ATOM_STATE_IDLE); ++ ++ ret = jd_done_nolock(katom, NULL); ++ goto out; ++ } ++ } ++ } ++ ++ /* Add dependencies */ ++ for (i = 0; i < 2; i++) { ++ int dep_atom_number = user_atom->pre_dep[i].atom_id; ++ base_jd_dep_type dep_atom_type; ++ struct kbase_jd_atom *dep_atom = &jctx->atoms[dep_atom_number]; ++ ++ dep_atom_type = user_atom->pre_dep[i].dependency_type; ++ kbase_jd_katom_dep_clear(&katom->dep[i]); ++ ++ if (!dep_atom_number) ++ continue; ++ ++ if (dep_atom->status == KBASE_JD_ATOM_STATE_UNUSED || ++ dep_atom->status == KBASE_JD_ATOM_STATE_COMPLETED) { ++ ++ if (dep_atom->event_code == BASE_JD_EVENT_DONE) ++ continue; ++ /* don't stop this atom if it has an order dependency ++ * only to the failed one, try to submit it through ++ * the normal path ++ */ ++ if (dep_atom_type == BASE_JD_DEP_TYPE_ORDER && ++ dep_atom->event_code > BASE_JD_EVENT_ACTIVE) { ++ continue; ++ } ++ ++ /* Atom has completed, propagate the error code if any */ ++ katom->event_code = dep_atom->event_code; ++ katom->status = KBASE_JD_ATOM_STATE_QUEUED; ++ ++ /* This atom is going through soft replay or ++ * will be sent back to user space. Do not record any ++ * dependencies. */ ++ KBASE_TLSTREAM_TL_NEW_ATOM( ++ katom, ++ kbase_jd_atom_id(kctx, katom)); ++ KBASE_TLSTREAM_TL_RET_ATOM_CTX(katom, kctx); ++ KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(katom, ++ TL_ATOM_STATE_IDLE); ++ ++ if ((katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) ++ == BASE_JD_REQ_SOFT_REPLAY) { ++ if (kbase_replay_process(katom)) { ++ ret = false; ++ goto out; ++ } ++ } ++ will_fail = true; ++ ++ } else { ++ /* Atom is in progress, add this atom to the list */ ++ list_add_tail(&katom->dep_item[i], &dep_atom->dep_head[i]); ++ kbase_jd_katom_dep_set(&katom->dep[i], dep_atom, dep_atom_type); ++ queued = 1; ++ } ++ } ++ ++ if (will_fail) { ++ if (!queued) { ++ ret = jd_done_nolock(katom, NULL); ++ ++ goto out; ++ } else { ++ katom->will_fail_event_code = katom->event_code; ++ ret = false; ++ ++ goto out; ++ } ++ } else { ++ /* These must occur after the above loop to ensure that an atom ++ * that depends on a previous atom with the same number behaves ++ * as expected */ ++ katom->event_code = BASE_JD_EVENT_DONE; ++ katom->status = KBASE_JD_ATOM_STATE_QUEUED; ++ } ++ ++ /* For invalid priority, be most lenient and choose the default */ ++ sched_prio = kbasep_js_atom_prio_to_sched_prio(user_atom->prio); ++ if (sched_prio == KBASE_JS_ATOM_SCHED_PRIO_INVALID) ++ sched_prio = KBASE_JS_ATOM_SCHED_PRIO_DEFAULT; ++ katom->sched_priority = sched_prio; ++ ++ /* Create a new atom recording all dependencies it was set up with. */ ++ KBASE_TLSTREAM_TL_NEW_ATOM( ++ katom, ++ kbase_jd_atom_id(kctx, katom)); ++ KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(katom, TL_ATOM_STATE_IDLE); ++ KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY(katom, katom->sched_priority); ++ KBASE_TLSTREAM_TL_RET_ATOM_CTX(katom, kctx); ++ for (i = 0; i < 2; i++) ++ if (BASE_JD_DEP_TYPE_INVALID != kbase_jd_katom_dep_type( ++ &katom->dep[i])) { ++ KBASE_TLSTREAM_TL_DEP_ATOM_ATOM( ++ (void *)kbase_jd_katom_dep_atom( ++ &katom->dep[i]), ++ (void *)katom); ++ } else if (BASE_JD_DEP_TYPE_INVALID != ++ user_atom->pre_dep[i].dependency_type) { ++ /* Resolved dependency. */ ++ int dep_atom_number = ++ user_atom->pre_dep[i].atom_id; ++ struct kbase_jd_atom *dep_atom = ++ &jctx->atoms[dep_atom_number]; ++ ++ KBASE_TLSTREAM_TL_RDEP_ATOM_ATOM( ++ (void *)dep_atom, ++ (void *)katom); ++ } ++ ++ /* Reject atoms with job chain = NULL, as these cause issues with soft-stop */ ++ if (!katom->jc && (katom->core_req & BASE_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_DEP) { ++ dev_warn(kctx->kbdev->dev, "Rejecting atom with jc = NULL"); ++ katom->event_code = BASE_JD_EVENT_JOB_INVALID; ++ ret = jd_done_nolock(katom, NULL); ++ goto out; ++ } ++ ++ /* Reject atoms with an invalid device_nr */ ++ if ((katom->core_req & BASE_JD_REQ_SPECIFIC_COHERENT_GROUP) && ++ (katom->device_nr >= kctx->kbdev->gpu_props.num_core_groups)) { ++ dev_warn(kctx->kbdev->dev, ++ "Rejecting atom with invalid device_nr %d", ++ katom->device_nr); ++ katom->event_code = BASE_JD_EVENT_JOB_INVALID; ++ ret = jd_done_nolock(katom, NULL); ++ goto out; ++ } ++ ++ /* Reject atoms with invalid core requirements */ ++ if ((katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) && ++ (katom->core_req & BASE_JD_REQ_EVENT_COALESCE)) { ++ dev_warn(kctx->kbdev->dev, ++ "Rejecting atom with invalid core requirements"); ++ katom->event_code = BASE_JD_EVENT_JOB_INVALID; ++ katom->core_req &= ~BASE_JD_REQ_EVENT_COALESCE; ++ ret = jd_done_nolock(katom, NULL); ++ goto out; ++ } ++ ++ if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) { ++ /* handle what we need to do to access the external resources */ ++ if (kbase_jd_pre_external_resources(katom, user_atom) != 0) { ++ /* setup failed (no access, bad resource, unknown resource types, etc.) */ ++ katom->event_code = BASE_JD_EVENT_JOB_INVALID; ++ ret = jd_done_nolock(katom, NULL); ++ goto out; ++ } ++ } ++ ++ /* Validate the atom. Function will return error if the atom is ++ * malformed. ++ * ++ * Soft-jobs never enter the job scheduler but have their own initialize method. ++ * ++ * If either fail then we immediately complete the atom with an error. ++ */ ++ if ((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0) { ++ if (!kbase_js_is_atom_valid(kctx->kbdev, katom)) { ++ katom->event_code = BASE_JD_EVENT_JOB_INVALID; ++ ret = jd_done_nolock(katom, NULL); ++ goto out; ++ } ++ } else { ++ /* Soft-job */ ++ if (kbase_prepare_soft_job(katom) != 0) { ++ katom->event_code = BASE_JD_EVENT_JOB_INVALID; ++ ret = jd_done_nolock(katom, NULL); ++ goto out; ++ } ++ } ++ ++#ifdef CONFIG_GPU_TRACEPOINTS ++ katom->work_id = atomic_inc_return(&jctx->work_id); ++ trace_gpu_job_enqueue((u32)kctx->id, katom->work_id, ++ kbasep_map_core_reqs_to_string(katom->core_req)); ++#endif ++ ++ if (queued && !IS_GPU_ATOM(katom)) { ++ ret = false; ++ goto out; ++ } ++#ifdef CONFIG_KDS ++ if (!katom->kds_dep_satisfied) { ++ /* Queue atom due to KDS dependency */ ++ ret = false; ++ goto out; ++ } ++#endif /* CONFIG_KDS */ ++ ++ ++#ifdef CONFIG_MALI_DMA_FENCE ++ if (kbase_fence_dep_count_read(katom) != -1) { ++ ret = false; ++ goto out; ++ } ++#endif /* CONFIG_MALI_DMA_FENCE */ ++ ++ if ((katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) ++ == BASE_JD_REQ_SOFT_REPLAY) { ++ if (kbase_replay_process(katom)) ++ ret = false; ++ else ++ ret = jd_done_nolock(katom, NULL); ++ ++ goto out; ++ } else if (katom->core_req & BASE_JD_REQ_SOFT_JOB) { ++ if (kbase_process_soft_job(katom) == 0) { ++ kbase_finish_soft_job(katom); ++ ret = jd_done_nolock(katom, NULL); ++ goto out; ++ } ++ ++ ret = false; ++ } else if ((katom->core_req & BASE_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_DEP) { ++ katom->status = KBASE_JD_ATOM_STATE_IN_JS; ++ ret = kbasep_js_add_job(kctx, katom); ++ /* If job was cancelled then resolve immediately */ ++ if (katom->event_code == BASE_JD_EVENT_JOB_CANCELLED) ++ ret = jd_done_nolock(katom, NULL); ++ } else { ++ /* This is a pure dependency. Resolve it immediately */ ++ ret = jd_done_nolock(katom, NULL); ++ } ++ ++ out: ++ return ret; ++} ++ ++int kbase_jd_submit(struct kbase_context *kctx, ++ void __user *user_addr, u32 nr_atoms, u32 stride, ++ bool uk6_atom) ++{ ++ struct kbase_jd_context *jctx = &kctx->jctx; ++ int err = 0; ++ int i; ++ bool need_to_try_schedule_context = false; ++ struct kbase_device *kbdev; ++ u32 latest_flush; ++ ++ /* ++ * kbase_jd_submit isn't expected to fail and so all errors with the ++ * jobs are reported by immediately failing them (through event system) ++ */ ++ kbdev = kctx->kbdev; ++ ++ beenthere(kctx, "%s", "Enter"); ++ ++ if (kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) { ++ dev_err(kbdev->dev, "Attempt to submit to a context that has SUBMIT_DISABLED set on it"); ++ return -EINVAL; ++ } ++ ++ if (stride != sizeof(base_jd_atom_v2)) { ++ dev_err(kbdev->dev, "Stride passed to job_submit doesn't match kernel"); ++ return -EINVAL; ++ } ++ ++ KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, atomic_add_return(nr_atoms, ++ &kctx->timeline.jd_atoms_in_flight)); ++ ++ /* All atoms submitted in this call have the same flush ID */ ++ latest_flush = kbase_backend_get_current_flush_id(kbdev); ++ ++ for (i = 0; i < nr_atoms; i++) { ++ struct base_jd_atom_v2 user_atom; ++ struct kbase_jd_atom *katom; ++ ++#ifdef BASE_LEGACY_UK6_SUPPORT ++ BUILD_BUG_ON(sizeof(struct base_jd_atom_v2_uk6) != ++ sizeof(base_jd_atom_v2)); ++ ++ if (uk6_atom) { ++ struct base_jd_atom_v2_uk6 user_atom_v6; ++ base_jd_dep_type dep_types[2] = {BASE_JD_DEP_TYPE_DATA, BASE_JD_DEP_TYPE_DATA}; ++ ++ if (copy_from_user(&user_atom_v6, user_addr, ++ sizeof(user_atom_v6))) { ++ err = -EINVAL; ++ KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, ++ atomic_sub_return( ++ nr_atoms - i, ++ &kctx->timeline.jd_atoms_in_flight)); ++ break; ++ } ++ /* Convert from UK6 atom format to UK7 format */ ++ user_atom.jc = user_atom_v6.jc; ++ user_atom.udata = user_atom_v6.udata; ++ user_atom.extres_list = user_atom_v6.extres_list; ++ user_atom.nr_extres = user_atom_v6.nr_extres; ++ user_atom.core_req = (u32)(user_atom_v6.core_req & 0x7fff); ++ ++ /* atom number 0 is used for no dependency atoms */ ++ if (!user_atom_v6.pre_dep[0]) ++ dep_types[0] = BASE_JD_DEP_TYPE_INVALID; ++ ++ base_jd_atom_dep_set(&user_atom.pre_dep[0], ++ user_atom_v6.pre_dep[0], ++ dep_types[0]); ++ ++ /* atom number 0 is used for no dependency atoms */ ++ if (!user_atom_v6.pre_dep[1]) ++ dep_types[1] = BASE_JD_DEP_TYPE_INVALID; ++ ++ base_jd_atom_dep_set(&user_atom.pre_dep[1], ++ user_atom_v6.pre_dep[1], ++ dep_types[1]); ++ ++ user_atom.atom_number = user_atom_v6.atom_number; ++ user_atom.prio = user_atom_v6.prio; ++ user_atom.device_nr = user_atom_v6.device_nr; ++ } else { ++#endif /* BASE_LEGACY_UK6_SUPPORT */ ++ if (copy_from_user(&user_atom, user_addr, ++ sizeof(user_atom)) != 0) { ++ err = -EINVAL; ++ KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, ++ atomic_sub_return(nr_atoms - i, ++ &kctx->timeline.jd_atoms_in_flight)); ++ break; ++ } ++#ifdef BASE_LEGACY_UK6_SUPPORT ++ } ++#endif ++ ++#ifdef BASE_LEGACY_UK10_2_SUPPORT ++ if (KBASE_API_VERSION(10, 3) > kctx->api_version) ++ user_atom.core_req = (u32)(user_atom.compat_core_req ++ & 0x7fff); ++#endif /* BASE_LEGACY_UK10_2_SUPPORT */ ++ ++ user_addr = (void __user *)((uintptr_t) user_addr + stride); ++ ++ mutex_lock(&jctx->lock); ++#ifndef compiletime_assert ++#define compiletime_assert_defined ++#define compiletime_assert(x, msg) do { switch (0) { case 0: case (x):; } } \ ++while (false) ++#endif ++ compiletime_assert((1 << (8*sizeof(user_atom.atom_number))) >= ++ BASE_JD_ATOM_COUNT, ++ "BASE_JD_ATOM_COUNT and base_atom_id type out of sync"); ++ compiletime_assert(sizeof(user_atom.pre_dep[0].atom_id) == ++ sizeof(user_atom.atom_number), ++ "BASE_JD_ATOM_COUNT and base_atom_id type out of sync"); ++#ifdef compiletime_assert_defined ++#undef compiletime_assert ++#undef compiletime_assert_defined ++#endif ++ if (user_atom.atom_number >= BASE_JD_ATOM_COUNT) { ++ err = -EINVAL; ++ break; ++ } ++ user_atom.atom_number = ++ array_index_nospec(user_atom.atom_number, ++ BASE_JD_ATOM_COUNT); ++ katom = &jctx->atoms[user_atom.atom_number]; ++ ++ /* Record the flush ID for the cache flush optimisation */ ++ katom->flush_id = latest_flush; ++ ++ while (katom->status != KBASE_JD_ATOM_STATE_UNUSED) { ++ /* Atom number is already in use, wait for the atom to ++ * complete ++ */ ++ mutex_unlock(&jctx->lock); ++ ++ /* This thread will wait for the atom to complete. Due ++ * to thread scheduling we are not sure that the other ++ * thread that owns the atom will also schedule the ++ * context, so we force the scheduler to be active and ++ * hence eventually schedule this context at some point ++ * later. ++ */ ++ kbase_js_sched_all(kbdev); ++ ++ if (wait_event_killable(katom->completed, ++ katom->status == ++ KBASE_JD_ATOM_STATE_UNUSED) != 0) { ++ /* We're being killed so the result code ++ * doesn't really matter ++ */ ++ return 0; ++ } ++ mutex_lock(&jctx->lock); ++ } ++ ++ need_to_try_schedule_context |= ++ jd_submit_atom(kctx, &user_atom, katom); ++ ++ /* Register a completed job as a disjoint event when the GPU is in a disjoint state ++ * (ie. being reset or replaying jobs). ++ */ ++ kbase_disjoint_event_potential(kbdev); ++ ++ mutex_unlock(&jctx->lock); ++ } ++ ++ if (need_to_try_schedule_context) ++ kbase_js_sched_all(kbdev); ++ ++ return err; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_jd_submit); ++ ++void kbase_jd_done_worker(struct work_struct *data) ++{ ++ struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, work); ++ struct kbase_jd_context *jctx; ++ struct kbase_context *kctx; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ struct kbase_device *kbdev; ++ struct kbasep_js_device_data *js_devdata; ++ u64 cache_jc = katom->jc; ++ struct kbasep_js_atom_retained_state katom_retained_state; ++ bool context_idle; ++ base_jd_core_req core_req = katom->core_req; ++ u64 affinity = katom->affinity; ++ enum kbase_atom_coreref_state coreref_state = katom->coreref_state; ++ ++ /* Soft jobs should never reach this function */ ++ KBASE_DEBUG_ASSERT((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0); ++ ++ kctx = katom->kctx; ++ jctx = &kctx->jctx; ++ kbdev = kctx->kbdev; ++ js_kctx_info = &kctx->jctx.sched_info; ++ js_devdata = &kbdev->js_data; ++ ++ KBASE_TRACE_ADD(kbdev, JD_DONE_WORKER, kctx, katom, katom->jc, 0); ++ ++ kbase_backend_complete_wq(kbdev, katom); ++ ++ /* ++ * Begin transaction on JD context and JS context ++ */ ++ mutex_lock(&jctx->lock); ++ KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(katom, TL_ATOM_STATE_DONE); ++ mutex_lock(&js_devdata->queue_mutex); ++ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); ++ ++ /* This worker only gets called on contexts that are scheduled *in*. This is ++ * because it only happens in response to an IRQ from a job that was ++ * running. ++ */ ++ KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ ++ if (katom->event_code == BASE_JD_EVENT_STOPPED) { ++ /* Atom has been promoted to stopped */ ++ unsigned long flags; ++ ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ mutex_unlock(&js_devdata->queue_mutex); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ katom->status = KBASE_JD_ATOM_STATE_IN_JS; ++ kbase_js_unpull(kctx, katom); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&jctx->lock); ++ ++ return; ++ } ++ ++ if (katom->event_code != BASE_JD_EVENT_DONE) ++ dev_err(kbdev->dev, ++ "t6xx: GPU fault 0x%02lx from job slot %d\n", ++ (unsigned long)katom->event_code, ++ katom->slot_nr); ++ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316)) ++ kbase_as_poking_timer_release_atom(kbdev, kctx, katom); ++ ++ /* Retain state before the katom disappears */ ++ kbasep_js_atom_retained_state_copy(&katom_retained_state, katom); ++ ++ context_idle = kbase_js_complete_atom_wq(kctx, katom); ++ ++ KBASE_DEBUG_ASSERT(kbasep_js_has_atom_finished(&katom_retained_state)); ++ ++ kbasep_js_remove_job(kbdev, kctx, katom); ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ mutex_unlock(&js_devdata->queue_mutex); ++ katom->atom_flags &= ~KBASE_KATOM_FLAG_HOLDING_CTX_REF; ++ /* jd_done_nolock() requires the jsctx_mutex lock to be dropped */ ++ jd_done_nolock(katom, &kctx->completed_jobs); ++ ++ /* katom may have been freed now, do not use! */ ++ ++ if (context_idle) { ++ unsigned long flags; ++ ++ context_idle = false; ++ mutex_lock(&js_devdata->queue_mutex); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ /* If kbase_sched() has scheduled this context back in then ++ * KCTX_ACTIVE will have been set after we marked it as ++ * inactive, and another pm reference will have been taken, so ++ * drop our reference. But do not call kbase_jm_idle_ctx(), as ++ * the context is active and fast-starting is allowed. ++ * ++ * If an atom has been fast-started then kctx->atoms_pulled will ++ * be non-zero but KCTX_ACTIVE will still be false (as the ++ * previous pm reference has been inherited). Do NOT drop our ++ * reference, as it has been re-used, and leave the context as ++ * active. ++ * ++ * If no new atoms have been started then KCTX_ACTIVE will still ++ * be false and atoms_pulled will be zero, so drop the reference ++ * and call kbase_jm_idle_ctx(). ++ * ++ * As the checks are done under both the queue_mutex and ++ * hwaccess_lock is should be impossible for this to race ++ * with the scheduler code. ++ */ ++ if (kbase_ctx_flag(kctx, KCTX_ACTIVE) || ++ !atomic_read(&kctx->atoms_pulled)) { ++ /* Calling kbase_jm_idle_ctx() here will ensure that ++ * atoms are not fast-started when we drop the ++ * hwaccess_lock. This is not performed if ++ * KCTX_ACTIVE is set as in that case another pm ++ * reference has been taken and a fast-start would be ++ * valid. ++ */ ++ if (!kbase_ctx_flag(kctx, KCTX_ACTIVE)) ++ kbase_jm_idle_ctx(kbdev, kctx); ++ context_idle = true; ++ } else { ++ kbase_ctx_flag_set(kctx, KCTX_ACTIVE); ++ } ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&js_devdata->queue_mutex); ++ } ++ ++ /* ++ * Transaction complete ++ */ ++ mutex_unlock(&jctx->lock); ++ ++ /* Job is now no longer running, so can now safely release the context ++ * reference, and handle any actions that were logged against the atom's retained state */ ++ ++ kbasep_js_runpool_release_ctx_and_katom_retained_state(kbdev, kctx, &katom_retained_state); ++ ++ kbase_js_sched_all(kbdev); ++ ++ if (!atomic_dec_return(&kctx->work_count)) { ++ /* If worker now idle then post all events that jd_done_nolock() ++ * has queued */ ++ mutex_lock(&jctx->lock); ++ while (!list_empty(&kctx->completed_jobs)) { ++ struct kbase_jd_atom *atom = list_entry( ++ kctx->completed_jobs.next, ++ struct kbase_jd_atom, jd_item); ++ list_del(kctx->completed_jobs.next); ++ ++ kbase_event_post(kctx, atom); ++ } ++ mutex_unlock(&jctx->lock); ++ } ++ ++ kbase_backend_complete_wq_post_sched(kbdev, core_req, affinity, ++ coreref_state); ++ ++ if (context_idle) ++ kbase_pm_context_idle(kbdev); ++ ++ KBASE_TRACE_ADD(kbdev, JD_DONE_WORKER_END, kctx, NULL, cache_jc, 0); ++} ++ ++/** ++ * jd_cancel_worker - Work queue job cancel function. ++ * @data: a &struct work_struct ++ * ++ * Only called as part of 'Zapping' a context (which occurs on termination). ++ * Operates serially with the kbase_jd_done_worker() on the work queue. ++ * ++ * This can only be called on contexts that aren't scheduled. ++ * ++ * We don't need to release most of the resources that would occur on ++ * kbase_jd_done() or kbase_jd_done_worker(), because the atoms here must not be ++ * running (by virtue of only being called on contexts that aren't ++ * scheduled). ++ */ ++static void jd_cancel_worker(struct work_struct *data) ++{ ++ struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, work); ++ struct kbase_jd_context *jctx; ++ struct kbase_context *kctx; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ bool need_to_try_schedule_context; ++ bool attr_state_changed; ++ struct kbase_device *kbdev; ++ ++ /* Soft jobs should never reach this function */ ++ KBASE_DEBUG_ASSERT((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0); ++ ++ kctx = katom->kctx; ++ kbdev = kctx->kbdev; ++ jctx = &kctx->jctx; ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ KBASE_TRACE_ADD(kbdev, JD_CANCEL_WORKER, kctx, katom, katom->jc, 0); ++ ++ /* This only gets called on contexts that are scheduled out. Hence, we must ++ * make sure we don't de-ref the number of running jobs (there aren't ++ * any), nor must we try to schedule out the context (it's already ++ * scheduled out). ++ */ ++ KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ ++ /* Scheduler: Remove the job from the system */ ++ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); ++ attr_state_changed = kbasep_js_remove_cancelled_job(kbdev, kctx, katom); ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ ++ mutex_lock(&jctx->lock); ++ ++ need_to_try_schedule_context = jd_done_nolock(katom, NULL); ++ /* Because we're zapping, we're not adding any more jobs to this ctx, so no need to ++ * schedule the context. There's also no need for the jsctx_mutex to have been taken ++ * around this too. */ ++ KBASE_DEBUG_ASSERT(!need_to_try_schedule_context); ++ ++ /* katom may have been freed now, do not use! */ ++ mutex_unlock(&jctx->lock); ++ ++ if (attr_state_changed) ++ kbase_js_sched_all(kbdev); ++} ++ ++/** ++ * kbase_jd_done - Complete a job that has been removed from the Hardware ++ * @katom: atom which has been completed ++ * @slot_nr: slot the atom was on ++ * @end_timestamp: completion time ++ * @done_code: completion code ++ * ++ * This must be used whenever a job has been removed from the Hardware, e.g.: ++ * An IRQ indicates that the job finished (for both error and 'done' codes), or ++ * the job was evicted from the JS_HEAD_NEXT registers during a Soft/Hard stop. ++ * ++ * Some work is carried out immediately, and the rest is deferred onto a ++ * workqueue ++ * ++ * Context: ++ * This can be called safely from atomic context. ++ * The caller must hold kbdev->hwaccess_lock ++ */ ++void kbase_jd_done(struct kbase_jd_atom *katom, int slot_nr, ++ ktime_t *end_timestamp, kbasep_js_atom_done_code done_code) ++{ ++ struct kbase_context *kctx; ++ struct kbase_device *kbdev; ++ ++ KBASE_DEBUG_ASSERT(katom); ++ kctx = katom->kctx; ++ KBASE_DEBUG_ASSERT(kctx); ++ kbdev = kctx->kbdev; ++ KBASE_DEBUG_ASSERT(kbdev); ++ ++ if (done_code & KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT) ++ katom->event_code = BASE_JD_EVENT_REMOVED_FROM_NEXT; ++ ++ KBASE_TRACE_ADD(kbdev, JD_DONE, kctx, katom, katom->jc, 0); ++ ++ kbase_job_check_leave_disjoint(kbdev, katom); ++ ++ katom->slot_nr = slot_nr; ++ ++ atomic_inc(&kctx->work_count); ++ ++#ifdef CONFIG_DEBUG_FS ++ /* a failed job happened and is waiting for dumping*/ ++ if (!katom->will_fail_event_code && ++ kbase_debug_job_fault_process(katom, katom->event_code)) ++ return; ++#endif ++ ++ WARN_ON(work_pending(&katom->work)); ++ KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&katom->work)); ++ INIT_WORK(&katom->work, kbase_jd_done_worker); ++ queue_work(kctx->jctx.job_done_wq, &katom->work); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_jd_done); ++ ++void kbase_jd_cancel(struct kbase_device *kbdev, struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx; ++ ++ KBASE_DEBUG_ASSERT(NULL != kbdev); ++ KBASE_DEBUG_ASSERT(NULL != katom); ++ kctx = katom->kctx; ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ ++ KBASE_TRACE_ADD(kbdev, JD_CANCEL, kctx, katom, katom->jc, 0); ++ ++ /* This should only be done from a context that is not scheduled */ ++ KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ ++ WARN_ON(work_pending(&katom->work)); ++ ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ ++ KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&katom->work)); ++ INIT_WORK(&katom->work, jd_cancel_worker); ++ queue_work(kctx->jctx.job_done_wq, &katom->work); ++} ++ ++ ++void kbase_jd_zap_context(struct kbase_context *kctx) ++{ ++ struct kbase_jd_atom *katom; ++ struct list_head *entry, *tmp; ++ struct kbase_device *kbdev; ++ ++ KBASE_DEBUG_ASSERT(kctx); ++ ++ kbdev = kctx->kbdev; ++ ++ KBASE_TRACE_ADD(kbdev, JD_ZAP_CONTEXT, kctx, NULL, 0u, 0u); ++ ++ kbase_js_zap_context(kctx); ++ ++ mutex_lock(&kctx->jctx.lock); ++ ++ /* ++ * While holding the struct kbase_jd_context lock clean up jobs which are known to kbase but are ++ * queued outside the job scheduler. ++ */ ++ ++ del_timer_sync(&kctx->soft_job_timeout); ++ list_for_each_safe(entry, tmp, &kctx->waiting_soft_jobs) { ++ katom = list_entry(entry, struct kbase_jd_atom, queue); ++ kbase_cancel_soft_job(katom); ++ } ++ ++ ++#ifdef CONFIG_KDS ++ ++ /* For each job waiting on a kds resource, cancel the wait and force the job to ++ * complete early, this is done so that we don't leave jobs outstanding waiting ++ * on kds resources which may never be released when contexts are zapped, resulting ++ * in a hang. ++ * ++ * Note that we can safely iterate over the list as the struct kbase_jd_context lock is held, ++ * this prevents items being removed when calling job_done_nolock in kbase_cancel_kds_wait_job. ++ */ ++ ++ list_for_each(entry, &kctx->waiting_kds_resource) { ++ katom = list_entry(entry, struct kbase_jd_atom, node); ++ ++ kbase_cancel_kds_wait_job(katom); ++ } ++#endif ++ ++#ifdef CONFIG_MALI_DMA_FENCE ++ kbase_dma_fence_cancel_all_atoms(kctx); ++#endif ++ ++ mutex_unlock(&kctx->jctx.lock); ++ ++#ifdef CONFIG_MALI_DMA_FENCE ++ /* Flush dma-fence workqueue to ensure that any callbacks that may have ++ * been queued are done before continuing. ++ */ ++ flush_workqueue(kctx->dma_fence.wq); ++#endif ++ ++ kbase_jm_wait_for_zero_jobs(kctx); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_jd_zap_context); ++ ++int kbase_jd_init(struct kbase_context *kctx) ++{ ++ int i; ++ int mali_err = 0; ++#ifdef CONFIG_KDS ++ int err; ++#endif /* CONFIG_KDS */ ++ ++ KBASE_DEBUG_ASSERT(kctx); ++ ++ kctx->jctx.job_done_wq = alloc_workqueue("mali_jd", ++ WQ_HIGHPRI | WQ_UNBOUND, 1); ++ if (NULL == kctx->jctx.job_done_wq) { ++ mali_err = -ENOMEM; ++ goto out1; ++ } ++ ++ for (i = 0; i < BASE_JD_ATOM_COUNT; i++) { ++ init_waitqueue_head(&kctx->jctx.atoms[i].completed); ++ ++ INIT_LIST_HEAD(&kctx->jctx.atoms[i].dep_head[0]); ++ INIT_LIST_HEAD(&kctx->jctx.atoms[i].dep_head[1]); ++ ++ /* Catch userspace attempting to use an atom which doesn't exist as a pre-dependency */ ++ kctx->jctx.atoms[i].event_code = BASE_JD_EVENT_JOB_INVALID; ++ kctx->jctx.atoms[i].status = KBASE_JD_ATOM_STATE_UNUSED; ++ ++#if defined(CONFIG_MALI_DMA_FENCE) || defined(CONFIG_SYNC_FILE) ++ kctx->jctx.atoms[i].dma_fence.context = ++ dma_fence_context_alloc(1); ++ atomic_set(&kctx->jctx.atoms[i].dma_fence.seqno, 0); ++ INIT_LIST_HEAD(&kctx->jctx.atoms[i].dma_fence.callbacks); ++#endif ++ } ++ ++ mutex_init(&kctx->jctx.lock); ++ ++ init_waitqueue_head(&kctx->jctx.zero_jobs_wait); ++ ++ spin_lock_init(&kctx->jctx.tb_lock); ++ ++#ifdef CONFIG_KDS ++ err = kds_callback_init(&kctx->jctx.kds_cb, 0, kds_dep_clear); ++ if (0 != err) { ++ mali_err = -EINVAL; ++ goto out2; ++ } ++#endif /* CONFIG_KDS */ ++ ++ kctx->jctx.job_nr = 0; ++ INIT_LIST_HEAD(&kctx->completed_jobs); ++ atomic_set(&kctx->work_count, 0); ++ ++ return 0; ++ ++#ifdef CONFIG_KDS ++ out2: ++ destroy_workqueue(kctx->jctx.job_done_wq); ++#endif /* CONFIG_KDS */ ++ out1: ++ return mali_err; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_jd_init); ++ ++void kbase_jd_exit(struct kbase_context *kctx) ++{ ++ KBASE_DEBUG_ASSERT(kctx); ++ ++#ifdef CONFIG_KDS ++ kds_callback_term(&kctx->jctx.kds_cb); ++#endif /* CONFIG_KDS */ ++ /* Work queue is emptied by this */ ++ destroy_workqueue(kctx->jctx.job_done_wq); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_jd_exit); +diff --git a/drivers/gpu/arm/midgard/mali_kbase_jd_debugfs.c b/drivers/gpu/arm/midgard/mali_kbase_jd_debugfs.c +new file mode 100755 +index 000000000000..44643abf85aa +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_jd_debugfs.c +@@ -0,0 +1,233 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifdef CONFIG_DEBUG_FS ++ ++#include ++#include ++#include ++#include ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++#include ++#endif ++ ++struct kbase_jd_debugfs_depinfo { ++ u8 id; ++ char type; ++}; ++ ++static void kbase_jd_debugfs_fence_info(struct kbase_jd_atom *atom, ++ struct seq_file *sfile) ++{ ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ struct kbase_sync_fence_info info; ++ int res; ++ ++ switch (atom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { ++ case BASE_JD_REQ_SOFT_FENCE_TRIGGER: ++ res = kbase_sync_fence_out_info_get(atom, &info); ++ if (res == 0) ++ seq_printf(sfile, "Sa([%p]%d) ", ++ info.fence, info.status); ++ break; ++ case BASE_JD_REQ_SOFT_FENCE_WAIT: ++ res = kbase_sync_fence_in_info_get(atom, &info); ++ if (res == 0) ++ seq_printf(sfile, "Wa([%p]%d) ", ++ info.fence, info.status); ++ break; ++ default: ++ break; ++ } ++#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ ++ ++#ifdef CONFIG_MALI_DMA_FENCE ++ if (atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) { ++ struct kbase_fence_cb *cb; ++ ++ if (atom->dma_fence.fence) { ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ struct fence *fence = atom->dma_fence.fence; ++#else ++ struct dma_fence *fence = atom->dma_fence.fence; ++#endif ++ ++ seq_printf(sfile, ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0)) ++ "Sd(%u#%u: %s) ", ++#else ++ "Sd(%llu#%u: %s) ", ++#endif ++ fence->context, ++ fence->seqno, ++ dma_fence_is_signaled(fence) ? ++ "signaled" : "active"); ++ } ++ ++ list_for_each_entry(cb, &atom->dma_fence.callbacks, ++ node) { ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ struct fence *fence = cb->fence; ++#else ++ struct dma_fence *fence = cb->fence; ++#endif ++ ++ seq_printf(sfile, ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0)) ++ "Wd(%u#%u: %s) ", ++#else ++ "Wd(%llu#%u: %s) ", ++#endif ++ fence->context, ++ fence->seqno, ++ dma_fence_is_signaled(fence) ? ++ "signaled" : "active"); ++ } ++ } ++#endif /* CONFIG_MALI_DMA_FENCE */ ++ ++} ++ ++static void kbasep_jd_debugfs_atom_deps( ++ struct kbase_jd_debugfs_depinfo *deps, ++ struct kbase_jd_atom *atom) ++{ ++ struct kbase_context *kctx = atom->kctx; ++ int i; ++ ++ for (i = 0; i < 2; i++) { ++ deps[i].id = (unsigned)(atom->dep[i].atom ? ++ kbase_jd_atom_id(kctx, atom->dep[i].atom) : 0); ++ ++ switch (atom->dep[i].dep_type) { ++ case BASE_JD_DEP_TYPE_INVALID: ++ deps[i].type = ' '; ++ break; ++ case BASE_JD_DEP_TYPE_DATA: ++ deps[i].type = 'D'; ++ break; ++ case BASE_JD_DEP_TYPE_ORDER: ++ deps[i].type = '>'; ++ break; ++ default: ++ deps[i].type = '?'; ++ break; ++ } ++ } ++} ++/** ++ * kbasep_jd_debugfs_atoms_show - Show callback for the JD atoms debugfs file. ++ * @sfile: The debugfs entry ++ * @data: Data associated with the entry ++ * ++ * This function is called to get the contents of the JD atoms debugfs file. ++ * This is a report of all atoms managed by kbase_jd_context.atoms ++ * ++ * Return: 0 if successfully prints data in debugfs entry file, failure ++ * otherwise ++ */ ++static int kbasep_jd_debugfs_atoms_show(struct seq_file *sfile, void *data) ++{ ++ struct kbase_context *kctx = sfile->private; ++ struct kbase_jd_atom *atoms; ++ unsigned long irq_flags; ++ int i; ++ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ ++ /* Print version */ ++ seq_printf(sfile, "v%u\n", MALI_JD_DEBUGFS_VERSION); ++ ++ /* Print U/K API version */ ++ seq_printf(sfile, "ukv%u.%u\n", BASE_UK_VERSION_MAJOR, ++ BASE_UK_VERSION_MINOR); ++ ++ /* Print table heading */ ++ seq_puts(sfile, " ID, Core req, St, CR, Predeps, Start time, Additional info...\n"); ++ ++ atoms = kctx->jctx.atoms; ++ /* General atom states */ ++ mutex_lock(&kctx->jctx.lock); ++ /* JS-related states */ ++ spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, irq_flags); ++ for (i = 0; i != BASE_JD_ATOM_COUNT; ++i) { ++ struct kbase_jd_atom *atom = &atoms[i]; ++ s64 start_timestamp = 0; ++ struct kbase_jd_debugfs_depinfo deps[2]; ++ ++ if (atom->status == KBASE_JD_ATOM_STATE_UNUSED) ++ continue; ++ ++ /* start_timestamp is cleared as soon as the atom leaves UNUSED state ++ * and set before a job is submitted to the h/w, a non-zero value means ++ * it is valid */ ++ if (ktime_to_ns(atom->start_timestamp)) ++ start_timestamp = ktime_to_ns( ++ ktime_sub(ktime_get(), atom->start_timestamp)); ++ ++ kbasep_jd_debugfs_atom_deps(deps, atom); ++ ++ seq_printf(sfile, ++ "%3u, %8x, %2u, %2u, %c%3u %c%3u, %20lld, ", ++ i, atom->core_req, atom->status, ++ atom->coreref_state, ++ deps[0].type, deps[0].id, ++ deps[1].type, deps[1].id, ++ start_timestamp); ++ ++ ++ kbase_jd_debugfs_fence_info(atom, sfile); ++ ++ seq_puts(sfile, "\n"); ++ } ++ spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, irq_flags); ++ mutex_unlock(&kctx->jctx.lock); ++ ++ return 0; ++} ++ ++ ++/** ++ * kbasep_jd_debugfs_atoms_open - open operation for atom debugfs file ++ * @in: &struct inode pointer ++ * @file: &struct file pointer ++ * ++ * Return: file descriptor ++ */ ++static int kbasep_jd_debugfs_atoms_open(struct inode *in, struct file *file) ++{ ++ return single_open(file, kbasep_jd_debugfs_atoms_show, in->i_private); ++} ++ ++static const struct file_operations kbasep_jd_debugfs_atoms_fops = { ++ .open = kbasep_jd_debugfs_atoms_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = single_release, ++}; ++ ++void kbasep_jd_debugfs_ctx_init(struct kbase_context *kctx) ++{ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ ++ /* Expose all atoms */ ++ debugfs_create_file("atoms", S_IRUGO, kctx->kctx_dentry, kctx, ++ &kbasep_jd_debugfs_atoms_fops); ++ ++} ++ ++#endif /* CONFIG_DEBUG_FS */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_jd_debugfs.h b/drivers/gpu/arm/midgard/mali_kbase_jd_debugfs.h +new file mode 100755 +index 000000000000..0935f1db7296 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_jd_debugfs.h +@@ -0,0 +1,39 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/** ++ * @file mali_kbase_jd_debugfs.h ++ * Header file for job dispatcher-related entries in debugfs ++ */ ++ ++#ifndef _KBASE_JD_DEBUGFS_H ++#define _KBASE_JD_DEBUGFS_H ++ ++#include ++ ++#include ++ ++#define MALI_JD_DEBUGFS_VERSION 2 ++ ++/** ++ * kbasep_jd_debugfs_ctx_init() - Add debugfs entries for JD system ++ * ++ * @kctx Pointer to kbase_context ++ */ ++void kbasep_jd_debugfs_ctx_init(struct kbase_context *kctx); ++ ++#endif /*_KBASE_JD_DEBUGFS_H*/ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_jm.c b/drivers/gpu/arm/midgard/mali_kbase_jm.c +new file mode 100755 +index 000000000000..0c5c6a6f78cb +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_jm.c +@@ -0,0 +1,131 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++/* ++ * HW access job manager common APIs ++ */ ++ ++#include ++#include "mali_kbase_hwaccess_jm.h" ++#include "mali_kbase_jm.h" ++ ++/** ++ * kbase_jm_next_job() - Attempt to run the next @nr_jobs_to_submit jobs on slot ++ * @js on the active context. ++ * @kbdev: Device pointer ++ * @js: Job slot to run on ++ * @nr_jobs_to_submit: Number of jobs to attempt to submit ++ * ++ * Return: true if slot can still be submitted on, false if slot is now full. ++ */ ++static bool kbase_jm_next_job(struct kbase_device *kbdev, int js, ++ int nr_jobs_to_submit) ++{ ++ struct kbase_context *kctx; ++ int i; ++ ++ kctx = kbdev->hwaccess.active_kctx; ++ ++ if (!kctx) ++ return true; ++ ++ for (i = 0; i < nr_jobs_to_submit; i++) { ++ struct kbase_jd_atom *katom = kbase_js_pull(kctx, js); ++ ++ if (!katom) ++ return true; /* Context has no jobs on this slot */ ++ ++ kbase_backend_run_atom(kbdev, katom); ++ } ++ ++ return false; /* Slot ringbuffer should now be full */ ++} ++ ++u32 kbase_jm_kick(struct kbase_device *kbdev, u32 js_mask) ++{ ++ u32 ret_mask = 0; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ while (js_mask) { ++ int js = ffs(js_mask) - 1; ++ int nr_jobs_to_submit = kbase_backend_slot_free(kbdev, js); ++ ++ if (kbase_jm_next_job(kbdev, js, nr_jobs_to_submit)) ++ ret_mask |= (1 << js); ++ ++ js_mask &= ~(1 << js); ++ } ++ ++ return ret_mask; ++} ++ ++void kbase_jm_try_kick(struct kbase_device *kbdev, u32 js_mask) ++{ ++ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (!down_trylock(&js_devdata->schedule_sem)) { ++ kbase_jm_kick(kbdev, js_mask); ++ up(&js_devdata->schedule_sem); ++ } ++} ++ ++void kbase_jm_try_kick_all(struct kbase_device *kbdev) ++{ ++ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (!down_trylock(&js_devdata->schedule_sem)) { ++ kbase_jm_kick_all(kbdev); ++ up(&js_devdata->schedule_sem); ++ } ++} ++ ++void kbase_jm_idle_ctx(struct kbase_device *kbdev, struct kbase_context *kctx) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (kbdev->hwaccess.active_kctx == kctx) ++ kbdev->hwaccess.active_kctx = NULL; ++} ++ ++struct kbase_jd_atom *kbase_jm_return_atom_to_js(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (katom->event_code != BASE_JD_EVENT_STOPPED && ++ katom->event_code != BASE_JD_EVENT_REMOVED_FROM_NEXT) { ++ return kbase_js_complete_atom(katom, NULL); ++ } else { ++ kbase_js_unpull(katom->kctx, katom); ++ return NULL; ++ } ++} ++ ++struct kbase_jd_atom *kbase_jm_complete(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom, ktime_t *end_timestamp) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ return kbase_js_complete_atom(katom, end_timestamp); ++} ++ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_jm.h b/drivers/gpu/arm/midgard/mali_kbase_jm.h +new file mode 100755 +index 000000000000..a74ee24c8058 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_jm.h +@@ -0,0 +1,110 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014, 2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++/* ++ * Job manager common APIs ++ */ ++ ++#ifndef _KBASE_JM_H_ ++#define _KBASE_JM_H_ ++ ++/** ++ * kbase_jm_kick() - Indicate that there are jobs ready to run. ++ * @kbdev: Device pointer ++ * @js_mask: Mask of the job slots that can be pulled from. ++ * ++ * Caller must hold the hwaccess_lock and schedule_sem semaphore ++ * ++ * Return: Mask of the job slots that can still be submitted to. ++ */ ++u32 kbase_jm_kick(struct kbase_device *kbdev, u32 js_mask); ++ ++/** ++ * kbase_jm_kick_all() - Indicate that there are jobs ready to run on all job ++ * slots. ++ * @kbdev: Device pointer ++ * ++ * Caller must hold the hwaccess_lock and schedule_sem semaphore ++ * ++ * Return: Mask of the job slots that can still be submitted to. ++ */ ++static inline u32 kbase_jm_kick_all(struct kbase_device *kbdev) ++{ ++ return kbase_jm_kick(kbdev, (1 << kbdev->gpu_props.num_job_slots) - 1); ++} ++ ++/** ++ * kbase_jm_try_kick - Attempt to call kbase_jm_kick ++ * @kbdev: Device pointer ++ * @js_mask: Mask of the job slots that can be pulled from ++ * Context: Caller must hold hwaccess_lock ++ * ++ * If schedule_sem can be immediately obtained then this function will call ++ * kbase_jm_kick() otherwise it will do nothing. ++ */ ++void kbase_jm_try_kick(struct kbase_device *kbdev, u32 js_mask); ++ ++/** ++ * kbase_jm_try_kick_all() - Attempt to call kbase_jm_kick_all ++ * @kbdev: Device pointer ++ * Context: Caller must hold hwaccess_lock ++ * ++ * If schedule_sem can be immediately obtained then this function will call ++ * kbase_jm_kick_all() otherwise it will do nothing. ++ */ ++void kbase_jm_try_kick_all(struct kbase_device *kbdev); ++ ++/** ++ * kbase_jm_idle_ctx() - Mark a context as idle. ++ * @kbdev: Device pointer ++ * @kctx: Context to mark as idle ++ * ++ * No more atoms will be pulled from this context until it is marked as active ++ * by kbase_js_use_ctx(). ++ * ++ * The context should have no atoms currently pulled from it ++ * (kctx->atoms_pulled == 0). ++ * ++ * Caller must hold the hwaccess_lock ++ */ ++void kbase_jm_idle_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); ++ ++/** ++ * kbase_jm_return_atom_to_js() - Return an atom to the job scheduler that has ++ * been soft-stopped or will fail due to a ++ * dependency ++ * @kbdev: Device pointer ++ * @katom: Atom that has been stopped or will be failed ++ * ++ * Return: Atom that has now been unblocked and can now be run, or NULL if none ++ */ ++struct kbase_jd_atom *kbase_jm_return_atom_to_js(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom); ++ ++/** ++ * kbase_jm_complete() - Complete an atom ++ * @kbdev: Device pointer ++ * @katom: Atom that has completed ++ * @end_timestamp: Timestamp of atom completion ++ * ++ * Return: Atom that has now been unblocked and can now be run, or NULL if none ++ */ ++struct kbase_jd_atom *kbase_jm_complete(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom, ktime_t *end_timestamp); ++ ++#endif /* _KBASE_JM_H_ */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_js.c b/drivers/gpu/arm/midgard/mali_kbase_js.c +new file mode 100755 +index 000000000000..10a1d5909bbe +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_js.c +@@ -0,0 +1,2834 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++/* #define ENABLE_DEBUG_LOG */ ++#include "./platform/rk/custom_log.h" ++ ++/* ++ * Job Scheduler Implementation ++ */ ++#include ++#include ++#if defined(CONFIG_MALI_GATOR_SUPPORT) ++#include ++#endif ++#include ++#include ++#include ++ ++#include ++#include ++ ++#include "mali_kbase_jm.h" ++#include "mali_kbase_hwaccess_jm.h" ++ ++/* ++ * Private types ++ */ ++ ++/* Bitpattern indicating the result of releasing a context */ ++enum { ++ /* The context was descheduled - caller should try scheduling in a new ++ * one to keep the runpool full */ ++ KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED = (1u << 0), ++ /* Ctx attributes were changed - caller should try scheduling all ++ * contexts */ ++ KBASEP_JS_RELEASE_RESULT_SCHED_ALL = (1u << 1) ++}; ++ ++typedef u32 kbasep_js_release_result; ++ ++const int kbasep_js_atom_priority_to_relative[BASE_JD_NR_PRIO_LEVELS] = { ++ KBASE_JS_ATOM_SCHED_PRIO_MED, /* BASE_JD_PRIO_MEDIUM */ ++ KBASE_JS_ATOM_SCHED_PRIO_HIGH, /* BASE_JD_PRIO_HIGH */ ++ KBASE_JS_ATOM_SCHED_PRIO_LOW /* BASE_JD_PRIO_LOW */ ++}; ++ ++const base_jd_prio ++kbasep_js_relative_priority_to_atom[KBASE_JS_ATOM_SCHED_PRIO_COUNT] = { ++ BASE_JD_PRIO_HIGH, /* KBASE_JS_ATOM_SCHED_PRIO_HIGH */ ++ BASE_JD_PRIO_MEDIUM, /* KBASE_JS_ATOM_SCHED_PRIO_MED */ ++ BASE_JD_PRIO_LOW /* KBASE_JS_ATOM_SCHED_PRIO_LOW */ ++}; ++ ++ ++/* ++ * Private function prototypes ++ */ ++static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal( ++ struct kbase_device *kbdev, struct kbase_context *kctx, ++ struct kbasep_js_atom_retained_state *katom_retained_state); ++ ++static int kbase_js_get_slot(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom); ++ ++static void kbase_js_foreach_ctx_job(struct kbase_context *kctx, ++ kbasep_js_ctx_job_cb callback); ++ ++/* Helper for trace subcodes */ ++#if KBASE_TRACE_ENABLE ++static int kbasep_js_trace_get_refcnt(struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++ return atomic_read(&kctx->refcount); ++} ++#else /* KBASE_TRACE_ENABLE */ ++static int kbasep_js_trace_get_refcnt(struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++ CSTD_UNUSED(kbdev); ++ CSTD_UNUSED(kctx); ++ return 0; ++} ++#endif /* KBASE_TRACE_ENABLE */ ++ ++/* ++ * Private functions ++ */ ++ ++/** ++ * core_reqs_from_jsn_features - Convert JSn_FEATURES to core requirements ++ * @features: JSn_FEATURE register value ++ * ++ * Given a JSn_FEATURE register value returns the core requirements that match ++ * ++ * Return: Core requirement bit mask ++ */ ++static base_jd_core_req core_reqs_from_jsn_features(u16 features) ++{ ++ base_jd_core_req core_req = 0u; ++ ++ if ((features & JS_FEATURE_SET_VALUE_JOB) != 0) ++ core_req |= BASE_JD_REQ_V; ++ ++ if ((features & JS_FEATURE_CACHE_FLUSH_JOB) != 0) ++ core_req |= BASE_JD_REQ_CF; ++ ++ if ((features & JS_FEATURE_COMPUTE_JOB) != 0) ++ core_req |= BASE_JD_REQ_CS; ++ ++ if ((features & JS_FEATURE_TILER_JOB) != 0) ++ core_req |= BASE_JD_REQ_T; ++ ++ if ((features & JS_FEATURE_FRAGMENT_JOB) != 0) ++ core_req |= BASE_JD_REQ_FS; ++ ++ return core_req; ++} ++ ++static void kbase_js_sync_timers(struct kbase_device *kbdev) ++{ ++ mutex_lock(&kbdev->js_data.runpool_mutex); ++ kbase_backend_ctx_count_changed(kbdev); ++ mutex_unlock(&kbdev->js_data.runpool_mutex); ++} ++ ++/* Hold the mmu_hw_mutex and hwaccess_lock for this */ ++bool kbasep_js_runpool_retain_ctx_nolock(struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ bool result = false; ++ int as_nr; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ js_devdata = &kbdev->js_data; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ as_nr = kctx->as_nr; ++ if (atomic_read(&kctx->refcount) > 0) { ++ KBASE_DEBUG_ASSERT(as_nr >= 0); ++ ++ kbase_ctx_sched_retain_ctx_refcount(kctx); ++ KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_RETAIN_CTX_NOLOCK, kctx, ++ NULL, 0u, atomic_read(&kctx->refcount)); ++ result = true; ++ } ++ ++ return result; ++} ++ ++/** ++ * jsctx_rb_none_to_pull_prio(): - Check if there are no pullable atoms ++ * @kctx: Pointer to kbase context with ring buffer. ++ * @js: Job slot id to check. ++ * @prio: Priority to check. ++ * ++ * Return true if there are no atoms to pull. There may be running atoms in the ++ * ring buffer even if there are no atoms to pull. It is also possible for the ++ * ring buffer to be full (with running atoms) when this functions returns ++ * true. ++ * ++ * Return: true if there are no atoms to pull, false otherwise. ++ */ ++static inline bool ++jsctx_rb_none_to_pull_prio(struct kbase_context *kctx, int js, int prio) ++{ ++ struct jsctx_queue *rb = &kctx->jsctx_queue[prio][js]; ++ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ ++ return RB_EMPTY_ROOT(&rb->runnable_tree); ++} ++ ++/** ++ * jsctx_rb_none_to_pull(): - Check if all priority ring buffers have no ++ * pullable atoms ++ * @kctx: Pointer to kbase context with ring buffer. ++ * @js: Job slot id to check. ++ * ++ * Caller must hold hwaccess_lock ++ * ++ * Return: true if the ring buffers for all priorities have no pullable atoms, ++ * false otherwise. ++ */ ++static inline bool ++jsctx_rb_none_to_pull(struct kbase_context *kctx, int js) ++{ ++ int prio; ++ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ ++ for (prio = 0; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) { ++ if (!jsctx_rb_none_to_pull_prio(kctx, js, prio)) ++ return false; ++ } ++ ++ return true; ++} ++ ++/** ++ * jsctx_queue_foreach_prio(): - Execute callback for each entry in the queue. ++ * @kctx: Pointer to kbase context with the queue. ++ * @js: Job slot id to iterate. ++ * @prio: Priority id to iterate. ++ * @callback: Function pointer to callback. ++ * ++ * Iterate over a queue and invoke @callback for each entry in the queue, and ++ * remove the entry from the queue. ++ * ++ * If entries are added to the queue while this is running those entries may, or ++ * may not be covered. To ensure that all entries in the buffer have been ++ * enumerated when this function returns jsctx->lock must be held when calling ++ * this function. ++ * ++ * The HW access lock must always be held when calling this function. ++ */ ++static void ++jsctx_queue_foreach_prio(struct kbase_context *kctx, int js, int prio, ++ kbasep_js_ctx_job_cb callback) ++{ ++ struct jsctx_queue *queue = &kctx->jsctx_queue[prio][js]; ++ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ ++ while (!RB_EMPTY_ROOT(&queue->runnable_tree)) { ++ struct rb_node *node = rb_first(&queue->runnable_tree); ++ struct kbase_jd_atom *entry = rb_entry(node, ++ struct kbase_jd_atom, runnable_tree_node); ++ ++ rb_erase(node, &queue->runnable_tree); ++ callback(kctx->kbdev, entry); ++ } ++ ++ while (!list_empty(&queue->x_dep_head)) { ++ struct kbase_jd_atom *entry = list_entry(queue->x_dep_head.next, ++ struct kbase_jd_atom, queue); ++ ++ list_del(queue->x_dep_head.next); ++ ++ callback(kctx->kbdev, entry); ++ } ++} ++ ++/** ++ * jsctx_queue_foreach(): - Execute callback for each entry in every queue ++ * @kctx: Pointer to kbase context with queue. ++ * @js: Job slot id to iterate. ++ * @callback: Function pointer to callback. ++ * ++ * Iterate over all the different priorities, and for each call ++ * jsctx_queue_foreach_prio() to iterate over the queue and invoke @callback ++ * for each entry, and remove the entry from the queue. ++ */ ++static inline void ++jsctx_queue_foreach(struct kbase_context *kctx, int js, ++ kbasep_js_ctx_job_cb callback) ++{ ++ int prio; ++ ++ for (prio = 0; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) ++ jsctx_queue_foreach_prio(kctx, js, prio, callback); ++} ++ ++/** ++ * jsctx_rb_peek_prio(): - Check buffer and get next atom ++ * @kctx: Pointer to kbase context with ring buffer. ++ * @js: Job slot id to check. ++ * @prio: Priority id to check. ++ * ++ * Check the ring buffer for the specified @js and @prio and return a pointer to ++ * the next atom, unless the ring buffer is empty. ++ * ++ * Return: Pointer to next atom in buffer, or NULL if there is no atom. ++ */ ++static inline struct kbase_jd_atom * ++jsctx_rb_peek_prio(struct kbase_context *kctx, int js, int prio) ++{ ++ struct jsctx_queue *rb = &kctx->jsctx_queue[prio][js]; ++ struct rb_node *node; ++ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ ++ node = rb_first(&rb->runnable_tree); ++ if (!node) ++ return NULL; ++ ++ return rb_entry(node, struct kbase_jd_atom, runnable_tree_node); ++} ++ ++/** ++ * jsctx_rb_peek(): - Check all priority buffers and get next atom ++ * @kctx: Pointer to kbase context with ring buffer. ++ * @js: Job slot id to check. ++ * ++ * Check the ring buffers for all priorities, starting from ++ * KBASE_JS_ATOM_SCHED_PRIO_HIGH, for the specified @js and @prio and return a ++ * pointer to the next atom, unless all the priority's ring buffers are empty. ++ * ++ * Caller must hold the hwaccess_lock. ++ * ++ * Return: Pointer to next atom in buffer, or NULL if there is no atom. ++ */ ++static inline struct kbase_jd_atom * ++jsctx_rb_peek(struct kbase_context *kctx, int js) ++{ ++ int prio; ++ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ ++ for (prio = 0; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) { ++ struct kbase_jd_atom *katom; ++ ++ katom = jsctx_rb_peek_prio(kctx, js, prio); ++ if (katom) ++ return katom; ++ } ++ ++ return NULL; ++} ++ ++/** ++ * jsctx_rb_pull(): - Mark atom in list as running ++ * @kctx: Pointer to kbase context with ring buffer. ++ * @katom: Pointer to katom to pull. ++ * ++ * Mark an atom previously obtained from jsctx_rb_peek() as running. ++ * ++ * @katom must currently be at the head of the ring buffer. ++ */ ++static inline void ++jsctx_rb_pull(struct kbase_context *kctx, struct kbase_jd_atom *katom) ++{ ++ int prio = katom->sched_priority; ++ int js = katom->slot_nr; ++ struct jsctx_queue *rb = &kctx->jsctx_queue[prio][js]; ++ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ ++ /* Atoms must be pulled in the correct order. */ ++ WARN_ON(katom != jsctx_rb_peek_prio(kctx, js, prio)); ++ ++ rb_erase(&katom->runnable_tree_node, &rb->runnable_tree); ++} ++ ++#define LESS_THAN_WRAP(a, b) ((s32)(a - b) < 0) ++ ++static void ++jsctx_tree_add(struct kbase_context *kctx, struct kbase_jd_atom *katom) ++{ ++ int prio = katom->sched_priority; ++ int js = katom->slot_nr; ++ struct jsctx_queue *queue = &kctx->jsctx_queue[prio][js]; ++ struct rb_node **new = &(queue->runnable_tree.rb_node), *parent = NULL; ++ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ ++ while (*new) { ++ struct kbase_jd_atom *entry = container_of(*new, ++ struct kbase_jd_atom, runnable_tree_node); ++ ++ parent = *new; ++ if (LESS_THAN_WRAP(katom->age, entry->age)) ++ new = &((*new)->rb_left); ++ else ++ new = &((*new)->rb_right); ++ } ++ ++ /* Add new node and rebalance tree. */ ++ rb_link_node(&katom->runnable_tree_node, parent, new); ++ rb_insert_color(&katom->runnable_tree_node, &queue->runnable_tree); ++} ++ ++/** ++ * jsctx_rb_unpull(): - Undo marking of atom in list as running ++ * @kctx: Pointer to kbase context with ring buffer. ++ * @katom: Pointer to katom to unpull. ++ * ++ * Undo jsctx_rb_pull() and put @katom back in the queue. ++ * ++ * jsctx_rb_unpull() must be called on atoms in the same order the atoms were ++ * pulled. ++ */ ++static inline void ++jsctx_rb_unpull(struct kbase_context *kctx, struct kbase_jd_atom *katom) ++{ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ ++ jsctx_tree_add(kctx, katom); ++} ++ ++static bool kbase_js_ctx_pullable(struct kbase_context *kctx, ++ int js, ++ bool is_scheduled); ++static bool kbase_js_ctx_list_add_pullable_nolock(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ int js); ++static bool kbase_js_ctx_list_add_unpullable_nolock(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ int js); ++ ++/* ++ * Functions private to KBase ('Protected' functions) ++ */ ++int kbasep_js_devdata_init(struct kbase_device * const kbdev) ++{ ++ struct kbasep_js_device_data *jsdd; ++ int i; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ jsdd = &kbdev->js_data; ++ ++#ifdef CONFIG_MALI_DEBUG ++ /* Soft-stop will be disabled on a single context by default unless ++ * softstop_always is set */ ++ jsdd->softstop_always = false; ++#endif /* CONFIG_MALI_DEBUG */ ++ jsdd->nr_all_contexts_running = 0; ++ jsdd->nr_user_contexts_running = 0; ++ jsdd->nr_contexts_pullable = 0; ++ atomic_set(&jsdd->nr_contexts_runnable, 0); ++ /* No ctx allowed to submit */ ++ jsdd->runpool_irq.submit_allowed = 0u; ++ memset(jsdd->runpool_irq.ctx_attr_ref_count, 0, ++ sizeof(jsdd->runpool_irq.ctx_attr_ref_count)); ++ memset(jsdd->runpool_irq.slot_affinities, 0, ++ sizeof(jsdd->runpool_irq.slot_affinities)); ++ memset(jsdd->runpool_irq.slot_affinity_refcount, 0, ++ sizeof(jsdd->runpool_irq.slot_affinity_refcount)); ++ INIT_LIST_HEAD(&jsdd->suspended_soft_jobs_list); ++ ++ /* Config attributes */ ++ jsdd->scheduling_period_ns = DEFAULT_JS_SCHEDULING_PERIOD_NS; ++ jsdd->soft_stop_ticks = DEFAULT_JS_SOFT_STOP_TICKS; ++ jsdd->soft_stop_ticks_cl = DEFAULT_JS_SOFT_STOP_TICKS_CL; ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408)) ++ jsdd->hard_stop_ticks_ss = DEFAULT_JS_HARD_STOP_TICKS_SS_8408; ++ else ++ jsdd->hard_stop_ticks_ss = DEFAULT_JS_HARD_STOP_TICKS_SS; ++ jsdd->hard_stop_ticks_cl = DEFAULT_JS_HARD_STOP_TICKS_CL; ++ jsdd->hard_stop_ticks_dumping = DEFAULT_JS_HARD_STOP_TICKS_DUMPING; ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408)) ++ jsdd->gpu_reset_ticks_ss = DEFAULT_JS_RESET_TICKS_SS_8408; ++ else ++ jsdd->gpu_reset_ticks_ss = DEFAULT_JS_RESET_TICKS_SS; ++ jsdd->gpu_reset_ticks_cl = DEFAULT_JS_RESET_TICKS_CL; ++ jsdd->gpu_reset_ticks_dumping = DEFAULT_JS_RESET_TICKS_DUMPING; ++ jsdd->ctx_timeslice_ns = DEFAULT_JS_CTX_TIMESLICE_NS; ++ atomic_set(&jsdd->soft_job_timeout_ms, DEFAULT_JS_SOFT_JOB_TIMEOUT); ++ ++ dev_dbg(kbdev->dev, "JS Config Attribs: "); ++ dev_dbg(kbdev->dev, "\tscheduling_period_ns:%u", ++ jsdd->scheduling_period_ns); ++ dev_dbg(kbdev->dev, "\tsoft_stop_ticks:%u", ++ jsdd->soft_stop_ticks); ++ dev_dbg(kbdev->dev, "\tsoft_stop_ticks_cl:%u", ++ jsdd->soft_stop_ticks_cl); ++ dev_dbg(kbdev->dev, "\thard_stop_ticks_ss:%u", ++ jsdd->hard_stop_ticks_ss); ++ dev_dbg(kbdev->dev, "\thard_stop_ticks_cl:%u", ++ jsdd->hard_stop_ticks_cl); ++ dev_dbg(kbdev->dev, "\thard_stop_ticks_dumping:%u", ++ jsdd->hard_stop_ticks_dumping); ++ dev_dbg(kbdev->dev, "\tgpu_reset_ticks_ss:%u", ++ jsdd->gpu_reset_ticks_ss); ++ dev_dbg(kbdev->dev, "\tgpu_reset_ticks_cl:%u", ++ jsdd->gpu_reset_ticks_cl); ++ dev_dbg(kbdev->dev, "\tgpu_reset_ticks_dumping:%u", ++ jsdd->gpu_reset_ticks_dumping); ++ dev_dbg(kbdev->dev, "\tctx_timeslice_ns:%u", ++ jsdd->ctx_timeslice_ns); ++ dev_dbg(kbdev->dev, "\tsoft_job_timeout:%i", ++ atomic_read(&jsdd->soft_job_timeout_ms)); ++ ++ if (!(jsdd->soft_stop_ticks < jsdd->hard_stop_ticks_ss && ++ jsdd->hard_stop_ticks_ss < jsdd->gpu_reset_ticks_ss && ++ jsdd->soft_stop_ticks < jsdd->hard_stop_ticks_dumping && ++ jsdd->hard_stop_ticks_dumping < ++ jsdd->gpu_reset_ticks_dumping)) { ++ dev_err(kbdev->dev, "Job scheduler timeouts invalid; soft/hard/reset tick counts should be in increasing order\n"); ++ return -EINVAL; ++ } ++ ++#if KBASE_DISABLE_SCHEDULING_SOFT_STOPS ++ dev_dbg(kbdev->dev, "Job Scheduling Soft-stops disabled, ignoring value for soft_stop_ticks==%u at %uns per tick. Other soft-stops may still occur.", ++ jsdd->soft_stop_ticks, ++ jsdd->scheduling_period_ns); ++#endif ++#if KBASE_DISABLE_SCHEDULING_HARD_STOPS ++ dev_dbg(kbdev->dev, "Job Scheduling Hard-stops disabled, ignoring values for hard_stop_ticks_ss==%d and hard_stop_ticks_dumping==%u at %uns per tick. Other hard-stops may still occur.", ++ jsdd->hard_stop_ticks_ss, ++ jsdd->hard_stop_ticks_dumping, ++ jsdd->scheduling_period_ns); ++#endif ++#if KBASE_DISABLE_SCHEDULING_SOFT_STOPS && KBASE_DISABLE_SCHEDULING_HARD_STOPS ++ dev_dbg(kbdev->dev, "Note: The JS tick timer (if coded) will still be run, but do nothing."); ++#endif ++ ++ for (i = 0; i < kbdev->gpu_props.num_job_slots; ++i) ++ jsdd->js_reqs[i] = core_reqs_from_jsn_features( ++ kbdev->gpu_props.props.raw_props.js_features[i]); ++ ++ /* On error, we could continue on: providing none of the below resources ++ * rely on the ones above */ ++ ++ mutex_init(&jsdd->runpool_mutex); ++ mutex_init(&jsdd->queue_mutex); ++ spin_lock_init(&kbdev->hwaccess_lock); ++ sema_init(&jsdd->schedule_sem, 1); ++ ++ for (i = 0; i < kbdev->gpu_props.num_job_slots; ++i) { ++ INIT_LIST_HEAD(&jsdd->ctx_list_pullable[i]); ++ INIT_LIST_HEAD(&jsdd->ctx_list_unpullable[i]); ++ } ++ ++ return 0; ++} ++ ++void kbasep_js_devdata_halt(struct kbase_device *kbdev) ++{ ++ CSTD_UNUSED(kbdev); ++} ++ ++void kbasep_js_devdata_term(struct kbase_device *kbdev) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ s8 zero_ctx_attr_ref_count[KBASEP_JS_CTX_ATTR_COUNT] = { 0, }; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ js_devdata = &kbdev->js_data; ++ ++ /* The caller must de-register all contexts before calling this ++ */ ++ KBASE_DEBUG_ASSERT(js_devdata->nr_all_contexts_running == 0); ++ KBASE_DEBUG_ASSERT(memcmp( ++ js_devdata->runpool_irq.ctx_attr_ref_count, ++ zero_ctx_attr_ref_count, ++ sizeof(zero_ctx_attr_ref_count)) == 0); ++ CSTD_UNUSED(zero_ctx_attr_ref_count); ++} ++ ++int kbasep_js_kctx_init(struct kbase_context * const kctx) ++{ ++ struct kbase_device *kbdev; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ int i, j; ++ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ ++ kbdev = kctx->kbdev; ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ for (i = 0; i < BASE_JM_MAX_NR_SLOTS; ++i) ++ INIT_LIST_HEAD(&kctx->jctx.sched_info.ctx.ctx_list_entry[i]); ++ ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ js_kctx_info->ctx.nr_jobs = 0; ++ kbase_ctx_flag_clear(kctx, KCTX_SCHEDULED); ++ kbase_ctx_flag_clear(kctx, KCTX_DYING); ++ memset(js_kctx_info->ctx.ctx_attr_ref_count, 0, ++ sizeof(js_kctx_info->ctx.ctx_attr_ref_count)); ++ ++ /* Initially, the context is disabled from submission until the create ++ * flags are set */ ++ kbase_ctx_flag_set(kctx, KCTX_SUBMIT_DISABLED); ++ ++ /* On error, we could continue on: providing none of the below resources ++ * rely on the ones above */ ++ mutex_init(&js_kctx_info->ctx.jsctx_mutex); ++ ++ init_waitqueue_head(&js_kctx_info->ctx.is_scheduled_wait); ++ ++ for (i = 0; i < KBASE_JS_ATOM_SCHED_PRIO_COUNT; i++) { ++ for (j = 0; j < BASE_JM_MAX_NR_SLOTS; j++) { ++ INIT_LIST_HEAD(&kctx->jsctx_queue[i][j].x_dep_head); ++ kctx->jsctx_queue[i][j].runnable_tree = RB_ROOT; ++ } ++ } ++ ++ return 0; ++} ++ ++void kbasep_js_kctx_term(struct kbase_context *kctx) ++{ ++ struct kbase_device *kbdev; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ int js; ++ bool update_ctx_count = false; ++ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ ++ kbdev = kctx->kbdev; ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ /* The caller must de-register all jobs before calling this */ ++ KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ KBASE_DEBUG_ASSERT(js_kctx_info->ctx.nr_jobs == 0); ++ ++ mutex_lock(&kbdev->js_data.queue_mutex); ++ mutex_lock(&kctx->jctx.sched_info.ctx.jsctx_mutex); ++ ++ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) ++ list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); ++ ++ if (kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)) { ++ WARN_ON(atomic_read(&kbdev->js_data.nr_contexts_runnable) <= 0); ++ atomic_dec(&kbdev->js_data.nr_contexts_runnable); ++ update_ctx_count = true; ++ kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); ++ } ++ ++ mutex_unlock(&kctx->jctx.sched_info.ctx.jsctx_mutex); ++ mutex_unlock(&kbdev->js_data.queue_mutex); ++ ++ if (update_ctx_count) { ++ mutex_lock(&kbdev->js_data.runpool_mutex); ++ kbase_backend_ctx_count_changed(kbdev); ++ mutex_unlock(&kbdev->js_data.runpool_mutex); ++ } ++} ++ ++/** ++ * kbase_js_ctx_list_add_pullable_nolock - Variant of ++ * kbase_jd_ctx_list_add_pullable() ++ * where the caller must hold ++ * hwaccess_lock ++ * @kbdev: Device pointer ++ * @kctx: Context to add to queue ++ * @js: Job slot to use ++ * ++ * Caller must hold hwaccess_lock ++ * ++ * Return: true if caller should call kbase_backend_ctx_count_changed() ++ */ ++static bool kbase_js_ctx_list_add_pullable_nolock(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ int js) ++{ ++ bool ret = false; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (!list_empty(&kctx->jctx.sched_info.ctx.ctx_list_entry[js])) ++ list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); ++ ++ list_add_tail(&kctx->jctx.sched_info.ctx.ctx_list_entry[js], ++ &kbdev->js_data.ctx_list_pullable[js]); ++ ++ if (!kctx->slots_pullable) { ++ kbdev->js_data.nr_contexts_pullable++; ++ ret = true; ++ if (!atomic_read(&kctx->atoms_pulled)) { ++ WARN_ON(kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); ++ kbase_ctx_flag_set(kctx, KCTX_RUNNABLE_REF); ++ atomic_inc(&kbdev->js_data.nr_contexts_runnable); ++ } ++ } ++ kctx->slots_pullable |= (1 << js); ++ ++ return ret; ++} ++ ++/** ++ * kbase_js_ctx_list_add_pullable_head_nolock - Variant of ++ * kbase_js_ctx_list_add_pullable_head() ++ * where the caller must hold ++ * hwaccess_lock ++ * @kbdev: Device pointer ++ * @kctx: Context to add to queue ++ * @js: Job slot to use ++ * ++ * Caller must hold hwaccess_lock ++ * ++ * Return: true if caller should call kbase_backend_ctx_count_changed() ++ */ ++static bool kbase_js_ctx_list_add_pullable_head_nolock( ++ struct kbase_device *kbdev, struct kbase_context *kctx, int js) ++{ ++ bool ret = false; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (!list_empty(&kctx->jctx.sched_info.ctx.ctx_list_entry[js])) ++ list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); ++ ++ list_add(&kctx->jctx.sched_info.ctx.ctx_list_entry[js], ++ &kbdev->js_data.ctx_list_pullable[js]); ++ ++ if (!kctx->slots_pullable) { ++ kbdev->js_data.nr_contexts_pullable++; ++ ret = true; ++ if (!atomic_read(&kctx->atoms_pulled)) { ++ WARN_ON(kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); ++ kbase_ctx_flag_set(kctx, KCTX_RUNNABLE_REF); ++ atomic_inc(&kbdev->js_data.nr_contexts_runnable); ++ } ++ } ++ kctx->slots_pullable |= (1 << js); ++ ++ return ret; ++} ++ ++/** ++ * kbase_js_ctx_list_add_pullable_head - Add context to the head of the ++ * per-slot pullable context queue ++ * @kbdev: Device pointer ++ * @kctx: Context to add to queue ++ * @js: Job slot to use ++ * ++ * If the context is on either the pullable or unpullable queues, then it is ++ * removed before being added to the head. ++ * ++ * This function should be used when a context has been scheduled, but no jobs ++ * can currently be pulled from it. ++ * ++ * Return: true if caller should call kbase_backend_ctx_count_changed() ++ */ ++static bool kbase_js_ctx_list_add_pullable_head(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ int js) ++{ ++ bool ret; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ret = kbase_js_ctx_list_add_pullable_head_nolock(kbdev, kctx, js); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ return ret; ++} ++ ++/** ++ * kbase_js_ctx_list_add_unpullable_nolock - Add context to the tail of the ++ * per-slot unpullable context queue ++ * @kbdev: Device pointer ++ * @kctx: Context to add to queue ++ * @js: Job slot to use ++ * ++ * The context must already be on the per-slot pullable queue. It will be ++ * removed from the pullable queue before being added to the unpullable queue. ++ * ++ * This function should be used when a context has been pulled from, and there ++ * are no jobs remaining on the specified slot. ++ * ++ * Caller must hold hwaccess_lock ++ * ++ * Return: true if caller should call kbase_backend_ctx_count_changed() ++ */ ++static bool kbase_js_ctx_list_add_unpullable_nolock(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ int js) ++{ ++ bool ret = false; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ list_move_tail(&kctx->jctx.sched_info.ctx.ctx_list_entry[js], ++ &kbdev->js_data.ctx_list_unpullable[js]); ++ ++ if (kctx->slots_pullable == (1 << js)) { ++ kbdev->js_data.nr_contexts_pullable--; ++ ret = true; ++ if (!atomic_read(&kctx->atoms_pulled)) { ++ WARN_ON(!kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); ++ kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); ++ atomic_dec(&kbdev->js_data.nr_contexts_runnable); ++ } ++ } ++ kctx->slots_pullable &= ~(1 << js); ++ ++ return ret; ++} ++ ++/** ++ * kbase_js_ctx_list_remove_nolock - Remove context from the per-slot pullable ++ * or unpullable context queues ++ * @kbdev: Device pointer ++ * @kctx: Context to remove from queue ++ * @js: Job slot to use ++ * ++ * The context must already be on one of the queues. ++ * ++ * This function should be used when a context has no jobs on the GPU, and no ++ * jobs remaining for the specified slot. ++ * ++ * Caller must hold hwaccess_lock ++ * ++ * Return: true if caller should call kbase_backend_ctx_count_changed() ++ */ ++static bool kbase_js_ctx_list_remove_nolock(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ int js) ++{ ++ bool ret = false; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ WARN_ON(list_empty(&kctx->jctx.sched_info.ctx.ctx_list_entry[js])); ++ ++ list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); ++ ++ if (kctx->slots_pullable == (1 << js)) { ++ kbdev->js_data.nr_contexts_pullable--; ++ ret = true; ++ if (!atomic_read(&kctx->atoms_pulled)) { ++ WARN_ON(!kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); ++ kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); ++ atomic_dec(&kbdev->js_data.nr_contexts_runnable); ++ } ++ } ++ kctx->slots_pullable &= ~(1 << js); ++ ++ return ret; ++} ++ ++/** ++ * kbase_js_ctx_list_pop_head_nolock - Variant of kbase_js_ctx_list_pop_head() ++ * where the caller must hold ++ * hwaccess_lock ++ * @kbdev: Device pointer ++ * @js: Job slot to use ++ * ++ * Caller must hold hwaccess_lock ++ * ++ * Return: Context to use for specified slot. ++ * NULL if no contexts present for specified slot ++ */ ++static struct kbase_context *kbase_js_ctx_list_pop_head_nolock( ++ struct kbase_device *kbdev, ++ int js) ++{ ++ struct kbase_context *kctx; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (list_empty(&kbdev->js_data.ctx_list_pullable[js])) ++ return NULL; ++ ++ kctx = list_entry(kbdev->js_data.ctx_list_pullable[js].next, ++ struct kbase_context, ++ jctx.sched_info.ctx.ctx_list_entry[js]); ++ ++ list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); ++ ++ return kctx; ++} ++ ++/** ++ * kbase_js_ctx_list_pop_head - Pop the head context off the per-slot pullable ++ * queue. ++ * @kbdev: Device pointer ++ * @js: Job slot to use ++ * ++ * Return: Context to use for specified slot. ++ * NULL if no contexts present for specified slot ++ */ ++static struct kbase_context *kbase_js_ctx_list_pop_head( ++ struct kbase_device *kbdev, int js) ++{ ++ struct kbase_context *kctx; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kctx = kbase_js_ctx_list_pop_head_nolock(kbdev, js); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ return kctx; ++} ++ ++/** ++ * kbase_js_ctx_pullable - Return if a context can be pulled from on the ++ * specified slot ++ * @kctx: Context pointer ++ * @js: Job slot to use ++ * @is_scheduled: true if the context is currently scheduled ++ * ++ * Caller must hold hwaccess_lock ++ * ++ * Return: true if context can be pulled from on specified slot ++ * false otherwise ++ */ ++static bool kbase_js_ctx_pullable(struct kbase_context *kctx, int js, ++ bool is_scheduled) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ struct kbase_jd_atom *katom; ++ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ ++ js_devdata = &kctx->kbdev->js_data; ++ ++ if (is_scheduled) { ++ if (!kbasep_js_is_submit_allowed(js_devdata, kctx)) ++ return false; ++ } ++ katom = jsctx_rb_peek(kctx, js); ++ if (!katom) ++ return false; /* No pullable atoms */ ++ if (kctx->blocked_js[js][katom->sched_priority]) ++ return false; ++ if (atomic_read(&katom->blocked)) ++ return false; /* next atom blocked */ ++ if (katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED) { ++ if (katom->x_pre_dep->gpu_rb_state == ++ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB || ++ katom->x_pre_dep->will_fail_event_code) ++ return false; ++ if ((katom->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER) && ++ kbase_backend_nr_atoms_on_slot(kctx->kbdev, js)) ++ return false; ++ } ++ ++ return true; ++} ++ ++static bool kbase_js_dep_validate(struct kbase_context *kctx, ++ struct kbase_jd_atom *katom) ++{ ++ struct kbase_device *kbdev = kctx->kbdev; ++ bool ret = true; ++ bool has_dep = false, has_x_dep = false; ++ int js = kbase_js_get_slot(kbdev, katom); ++ int prio = katom->sched_priority; ++ int i; ++ ++ for (i = 0; i < 2; i++) { ++ struct kbase_jd_atom *dep_atom = katom->dep[i].atom; ++ ++ if (dep_atom) { ++ int dep_js = kbase_js_get_slot(kbdev, dep_atom); ++ int dep_prio = dep_atom->sched_priority; ++ ++ /* Dependent atom must already have been submitted */ ++ if (!(dep_atom->atom_flags & ++ KBASE_KATOM_FLAG_JSCTX_IN_TREE)) { ++ ret = false; ++ break; ++ } ++ ++ /* Dependencies with different priorities can't ++ be represented in the ringbuffer */ ++ if (prio != dep_prio) { ++ ret = false; ++ break; ++ } ++ ++ if (js == dep_js) { ++ /* Only one same-slot dependency can be ++ * represented in the ringbuffer */ ++ if (has_dep) { ++ ret = false; ++ break; ++ } ++ /* Each dependee atom can only have one ++ * same-slot dependency */ ++ if (dep_atom->post_dep) { ++ ret = false; ++ break; ++ } ++ has_dep = true; ++ } else { ++ /* Only one cross-slot dependency can be ++ * represented in the ringbuffer */ ++ if (has_x_dep) { ++ ret = false; ++ break; ++ } ++ /* Each dependee atom can only have one ++ * cross-slot dependency */ ++ if (dep_atom->x_post_dep) { ++ ret = false; ++ break; ++ } ++ /* The dependee atom can not already be in the ++ * HW access ringbuffer */ ++ if (dep_atom->gpu_rb_state != ++ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { ++ ret = false; ++ break; ++ } ++ /* The dependee atom can not already have ++ * completed */ ++ if (dep_atom->status != ++ KBASE_JD_ATOM_STATE_IN_JS) { ++ ret = false; ++ break; ++ } ++ /* Cross-slot dependencies must not violate ++ * PRLAM-8987 affinity restrictions */ ++ if (kbase_hw_has_issue(kbdev, ++ BASE_HW_ISSUE_8987) && ++ (js == 2 || dep_js == 2)) { ++ ret = false; ++ break; ++ } ++ has_x_dep = true; ++ } ++ ++ /* Dependency can be represented in ringbuffers */ ++ } ++ } ++ ++ /* If dependencies can be represented by ringbuffer then clear them from ++ * atom structure */ ++ if (ret) { ++ for (i = 0; i < 2; i++) { ++ struct kbase_jd_atom *dep_atom = katom->dep[i].atom; ++ ++ if (dep_atom) { ++ int dep_js = kbase_js_get_slot(kbdev, dep_atom); ++ ++ if ((js != dep_js) && ++ (dep_atom->status != ++ KBASE_JD_ATOM_STATE_COMPLETED) ++ && (dep_atom->status != ++ KBASE_JD_ATOM_STATE_HW_COMPLETED) ++ && (dep_atom->status != ++ KBASE_JD_ATOM_STATE_UNUSED)) { ++ ++ katom->atom_flags |= ++ KBASE_KATOM_FLAG_X_DEP_BLOCKED; ++ katom->x_pre_dep = dep_atom; ++ dep_atom->x_post_dep = katom; ++ if (kbase_jd_katom_dep_type( ++ &katom->dep[i]) == ++ BASE_JD_DEP_TYPE_DATA) ++ katom->atom_flags |= ++ KBASE_KATOM_FLAG_FAIL_BLOCKER; ++ } ++ if ((kbase_jd_katom_dep_type(&katom->dep[i]) ++ == BASE_JD_DEP_TYPE_DATA) && ++ (js == dep_js)) { ++ katom->pre_dep = dep_atom; ++ dep_atom->post_dep = katom; ++ } ++ ++ list_del(&katom->dep_item[i]); ++ kbase_jd_katom_dep_clear(&katom->dep[i]); ++ } ++ } ++ } ++ ++ return ret; ++} ++ ++bool kbasep_js_add_job(struct kbase_context *kctx, ++ struct kbase_jd_atom *atom) ++{ ++ unsigned long flags; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ struct kbase_device *kbdev; ++ struct kbasep_js_device_data *js_devdata; ++ ++ bool enqueue_required = false; ++ bool timer_sync = false; ++ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ KBASE_DEBUG_ASSERT(atom != NULL); ++ lockdep_assert_held(&kctx->jctx.lock); ++ ++ kbdev = kctx->kbdev; ++ js_devdata = &kbdev->js_data; ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ mutex_lock(&js_devdata->queue_mutex); ++ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); ++ ++ /* ++ * Begin Runpool transaction ++ */ ++ mutex_lock(&js_devdata->runpool_mutex); ++ ++ /* Refcount ctx.nr_jobs */ ++ KBASE_DEBUG_ASSERT(js_kctx_info->ctx.nr_jobs < U32_MAX); ++ ++(js_kctx_info->ctx.nr_jobs); ++ ++ /* Setup any scheduling information */ ++ kbasep_js_clear_job_retry_submit(atom); ++ ++ /* Lock for state available during IRQ */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ if (!kbase_js_dep_validate(kctx, atom)) { ++ /* Dependencies could not be represented */ ++ --(js_kctx_info->ctx.nr_jobs); ++ ++ /* Setting atom status back to queued as it still has unresolved ++ * dependencies */ ++ atom->status = KBASE_JD_ATOM_STATE_QUEUED; ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ ++ goto out_unlock; ++ } ++ ++ KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(atom, TL_ATOM_STATE_READY); ++ KBASE_TIMELINE_ATOM_READY(kctx, kbase_jd_atom_id(kctx, atom)); ++ ++ enqueue_required = kbase_js_dep_resolved_submit(kctx, atom); ++ ++ KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_ADD_JOB, kctx, atom, atom->jc, ++ kbasep_js_trace_get_refcnt(kbdev, kctx)); ++ ++ /* Context Attribute Refcounting */ ++ kbasep_js_ctx_attr_ctx_retain_atom(kbdev, kctx, atom); ++ ++ if (enqueue_required) { ++ if (kbase_js_ctx_pullable(kctx, atom->slot_nr, false)) ++ timer_sync = kbase_js_ctx_list_add_pullable_nolock( ++ kbdev, kctx, atom->slot_nr); ++ else ++ timer_sync = kbase_js_ctx_list_add_unpullable_nolock( ++ kbdev, kctx, atom->slot_nr); ++ } ++ /* If this context is active and the atom is the first on its slot, ++ * kick the job manager to attempt to fast-start the atom */ ++ if (enqueue_required && kctx == kbdev->hwaccess.active_kctx) ++ kbase_jm_try_kick(kbdev, 1 << atom->slot_nr); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ if (timer_sync) ++ kbase_backend_ctx_count_changed(kbdev); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ /* End runpool transaction */ ++ ++ if (!kbase_ctx_flag(kctx, KCTX_SCHEDULED)) { ++ if (kbase_ctx_flag(kctx, KCTX_DYING)) { ++ /* A job got added while/after kbase_job_zap_context() ++ * was called on a non-scheduled context (e.g. KDS ++ * dependency resolved). Kill that job by killing the ++ * context. */ ++ kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, ++ false); ++ } else if (js_kctx_info->ctx.nr_jobs == 1) { ++ /* Handle Refcount going from 0 to 1: schedule the ++ * context on the Queue */ ++ KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ dev_dbg(kbdev->dev, "JS: Enqueue Context %p", kctx); ++ ++ /* Queue was updated - caller must try to ++ * schedule the head context */ ++ WARN_ON(!enqueue_required); ++ } ++ } ++out_unlock: ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ ++ mutex_unlock(&js_devdata->queue_mutex); ++ ++ return enqueue_required; ++} ++ ++void kbasep_js_remove_job(struct kbase_device *kbdev, ++ struct kbase_context *kctx, struct kbase_jd_atom *atom) ++{ ++ struct kbasep_js_kctx_info *js_kctx_info; ++ struct kbasep_js_device_data *js_devdata; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ KBASE_DEBUG_ASSERT(atom != NULL); ++ ++ js_devdata = &kbdev->js_data; ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_REMOVE_JOB, kctx, atom, atom->jc, ++ kbasep_js_trace_get_refcnt(kbdev, kctx)); ++ ++ /* De-refcount ctx.nr_jobs */ ++ KBASE_DEBUG_ASSERT(js_kctx_info->ctx.nr_jobs > 0); ++ --(js_kctx_info->ctx.nr_jobs); ++} ++ ++bool kbasep_js_remove_cancelled_job(struct kbase_device *kbdev, ++ struct kbase_context *kctx, struct kbase_jd_atom *katom) ++{ ++ unsigned long flags; ++ struct kbasep_js_atom_retained_state katom_retained_state; ++ struct kbasep_js_device_data *js_devdata; ++ bool attr_state_changed; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ KBASE_DEBUG_ASSERT(katom != NULL); ++ ++ js_devdata = &kbdev->js_data; ++ ++ kbasep_js_atom_retained_state_copy(&katom_retained_state, katom); ++ kbasep_js_remove_job(kbdev, kctx, katom); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ /* The atom has 'finished' (will not be re-run), so no need to call ++ * kbasep_js_has_atom_finished(). ++ * ++ * This is because it returns false for soft-stopped atoms, but we ++ * want to override that, because we're cancelling an atom regardless of ++ * whether it was soft-stopped or not */ ++ attr_state_changed = kbasep_js_ctx_attr_ctx_release_atom(kbdev, kctx, ++ &katom_retained_state); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ return attr_state_changed; ++} ++ ++bool kbasep_js_runpool_retain_ctx(struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++ unsigned long flags; ++ struct kbasep_js_device_data *js_devdata; ++ bool result; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ js_devdata = &kbdev->js_data; ++ ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ result = kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ ++ return result; ++} ++ ++struct kbase_context *kbasep_js_runpool_lookup_ctx(struct kbase_device *kbdev, ++ int as_nr) ++{ ++ int ret = 0; ++ unsigned long flags; ++ struct kbasep_js_device_data *js_devdata; ++ struct kbase_context *found_kctx = NULL; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(0 <= as_nr && as_nr < BASE_MAX_NR_AS); ++ js_devdata = &kbdev->js_data; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ found_kctx = kbdev->as_to_kctx[as_nr]; ++ ++ if (found_kctx != NULL) { ++ ret = kbase_ctx_sched_retain_ctx_refcount(found_kctx); ++ if (ret != 0) { ++ E("fail to retain ctx_refcount, ret : %d.", ret); ++ found_kctx = NULL; ++ } ++ } ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ return found_kctx; ++} ++ ++/** ++ * kbasep_js_release_result - Try running more jobs after releasing a context ++ * and/or atom ++ * ++ * @kbdev: The kbase_device to operate on ++ * @kctx: The kbase_context to operate on ++ * @katom_retained_state: Retained state from the atom ++ * @runpool_ctx_attr_change: True if the runpool context attributes have changed ++ * ++ * This collates a set of actions that must happen whilst hwaccess_lock is held. ++ * ++ * This includes running more jobs when: ++ * - The previously released kctx caused a ctx attribute change, ++ * - The released atom caused a ctx attribute change, ++ * - Slots were previously blocked due to affinity restrictions, ++ * - Submission during IRQ handling failed. ++ * ++ * Return: %KBASEP_JS_RELEASE_RESULT_SCHED_ALL if context attributes were ++ * changed. The caller should try scheduling all contexts ++ */ ++static kbasep_js_release_result kbasep_js_run_jobs_after_ctx_and_atom_release( ++ struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ struct kbasep_js_atom_retained_state *katom_retained_state, ++ bool runpool_ctx_attr_change) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ kbasep_js_release_result result = 0; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ KBASE_DEBUG_ASSERT(katom_retained_state != NULL); ++ js_devdata = &kbdev->js_data; ++ ++ lockdep_assert_held(&kctx->jctx.sched_info.ctx.jsctx_mutex); ++ lockdep_assert_held(&js_devdata->runpool_mutex); ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (js_devdata->nr_user_contexts_running != 0) { ++ bool retry_submit = false; ++ int retry_jobslot = 0; ++ ++ if (katom_retained_state) ++ retry_submit = kbasep_js_get_atom_retry_submit_slot( ++ katom_retained_state, &retry_jobslot); ++ ++ if (runpool_ctx_attr_change || retry_submit) { ++ /* A change in runpool ctx attributes might mean we can ++ * run more jobs than before */ ++ result = KBASEP_JS_RELEASE_RESULT_SCHED_ALL; ++ ++ KBASE_TRACE_ADD_SLOT(kbdev, JD_DONE_TRY_RUN_NEXT_JOB, ++ kctx, NULL, 0u, retry_jobslot); ++ } ++ } ++ return result; ++} ++ ++/* ++ * Internal function to release the reference on a ctx and an atom's "retained ++ * state", only taking the runpool and as transaction mutexes ++ * ++ * This also starts more jobs running in the case of an ctx-attribute state ++ * change ++ * ++ * This does none of the followup actions for scheduling: ++ * - It does not schedule in a new context ++ * - It does not requeue or handle dying contexts ++ * ++ * For those tasks, just call kbasep_js_runpool_release_ctx() instead ++ * ++ * Requires: ++ * - Context is scheduled in, and kctx->as_nr matches kctx_as_nr ++ * - Context has a non-zero refcount ++ * - Caller holds js_kctx_info->ctx.jsctx_mutex ++ * - Caller holds js_devdata->runpool_mutex ++ */ ++static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal( ++ struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ struct kbasep_js_atom_retained_state *katom_retained_state) ++{ ++ unsigned long flags; ++ struct kbasep_js_device_data *js_devdata; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ ++ kbasep_js_release_result release_result = 0u; ++ bool runpool_ctx_attr_change = false; ++ int kctx_as_nr; ++ struct kbase_as *current_as; ++ int new_ref_count; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ js_kctx_info = &kctx->jctx.sched_info; ++ js_devdata = &kbdev->js_data; ++ ++ /* Ensure context really is scheduled in */ ++ KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ ++ kctx_as_nr = kctx->as_nr; ++ KBASE_DEBUG_ASSERT(kctx_as_nr != KBASEP_AS_NR_INVALID); ++ KBASE_DEBUG_ASSERT(atomic_read(&kctx->refcount) > 0); ++ ++ /* ++ * Transaction begins on AS and runpool_irq ++ * ++ * Assert about out calling contract ++ */ ++ current_as = &kbdev->as[kctx_as_nr]; ++ mutex_lock(&kbdev->pm.lock); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ KBASE_DEBUG_ASSERT(kctx_as_nr == kctx->as_nr); ++ KBASE_DEBUG_ASSERT(atomic_read(&kctx->refcount) > 0); ++ ++ /* Update refcount */ ++ kbase_ctx_sched_release_ctx(kctx); ++ new_ref_count = atomic_read(&kctx->refcount); ++ ++ /* Release the atom if it finished (i.e. wasn't soft-stopped) */ ++ if (kbasep_js_has_atom_finished(katom_retained_state)) ++ runpool_ctx_attr_change |= kbasep_js_ctx_attr_ctx_release_atom( ++ kbdev, kctx, katom_retained_state); ++ ++ KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_RELEASE_CTX, kctx, NULL, 0u, ++ new_ref_count); ++ ++ if (new_ref_count == 2 && kbase_ctx_flag(kctx, KCTX_PRIVILEGED) && ++ !kbase_pm_is_suspending(kbdev)) { ++ /* Context is kept scheduled into an address space even when ++ * there are no jobs, in this case we have to handle the ++ * situation where all jobs have been evicted from the GPU and ++ * submission is disabled. ++ * ++ * At this point we re-enable submission to allow further jobs ++ * to be executed ++ */ ++ kbasep_js_set_submit_allowed(js_devdata, kctx); ++ } ++ ++ /* Make a set of checks to see if the context should be scheduled out. ++ * Note that there'll always be at least 1 reference to the context ++ * which was previously acquired by kbasep_js_schedule_ctx(). */ ++ if (new_ref_count == 1 && ++ (!kbasep_js_is_submit_allowed(js_devdata, kctx) || ++ kbdev->pm.suspending)) { ++ int num_slots = kbdev->gpu_props.num_job_slots; ++ int slot; ++ ++ /* Last reference, and we've been told to remove this context ++ * from the Run Pool */ ++ dev_dbg(kbdev->dev, "JS: RunPool Remove Context %p because refcount=%d, jobs=%d, allowed=%d", ++ kctx, new_ref_count, js_kctx_info->ctx.nr_jobs, ++ kbasep_js_is_submit_allowed(js_devdata, kctx)); ++ ++#if defined(CONFIG_MALI_GATOR_SUPPORT) ++ kbase_trace_mali_mmu_as_released(kctx->as_nr); ++#endif ++ KBASE_TLSTREAM_TL_NRET_AS_CTX(&kbdev->as[kctx->as_nr], kctx); ++ ++ kbase_backend_release_ctx_irq(kbdev, kctx); ++ ++ if (kbdev->hwaccess.active_kctx == kctx) ++ kbdev->hwaccess.active_kctx = NULL; ++ ++ /* Ctx Attribute handling ++ * ++ * Releasing atoms attributes must either happen before this, or ++ * after the KCTX_SHEDULED flag is changed, otherwise we ++ * double-decount the attributes ++ */ ++ runpool_ctx_attr_change |= ++ kbasep_js_ctx_attr_runpool_release_ctx(kbdev, kctx); ++ ++ /* Releasing the context and katom retained state can allow ++ * more jobs to run */ ++ release_result |= ++ kbasep_js_run_jobs_after_ctx_and_atom_release(kbdev, ++ kctx, katom_retained_state, ++ runpool_ctx_attr_change); ++ ++ /* ++ * Transaction ends on AS and runpool_irq: ++ * ++ * By this point, the AS-related data is now clear and ready ++ * for re-use. ++ * ++ * Since releases only occur once for each previous successful ++ * retain, and no more retains are allowed on this context, no ++ * other thread will be operating in this ++ * code whilst we are ++ */ ++ ++ /* Recalculate pullable status for all slots */ ++ for (slot = 0; slot < num_slots; slot++) { ++ if (kbase_js_ctx_pullable(kctx, slot, false)) ++ kbase_js_ctx_list_add_pullable_nolock(kbdev, ++ kctx, slot); ++ } ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ kbase_backend_release_ctx_noirq(kbdev, kctx); ++ ++ mutex_unlock(&kbdev->pm.lock); ++ ++ /* Note: Don't reuse kctx_as_nr now */ ++ ++ /* Synchronize with any timers */ ++ kbase_backend_ctx_count_changed(kbdev); ++ ++ /* update book-keeping info */ ++ kbase_ctx_flag_clear(kctx, KCTX_SCHEDULED); ++ /* Signal any waiter that the context is not scheduled, so is ++ * safe for termination - once the jsctx_mutex is also dropped, ++ * and jobs have finished. */ ++ wake_up(&js_kctx_info->ctx.is_scheduled_wait); ++ ++ /* Queue an action to occur after we've dropped the lock */ ++ release_result |= KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED | ++ KBASEP_JS_RELEASE_RESULT_SCHED_ALL; ++ } else { ++ kbasep_js_run_jobs_after_ctx_and_atom_release(kbdev, kctx, ++ katom_retained_state, runpool_ctx_attr_change); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&kbdev->pm.lock); ++ } ++ ++ return release_result; ++} ++ ++void kbasep_js_runpool_release_ctx_nolock(struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++ struct kbasep_js_atom_retained_state katom_retained_state; ++ ++ /* Setup a dummy katom_retained_state */ ++ kbasep_js_atom_retained_state_init_invalid(&katom_retained_state); ++ ++ kbasep_js_runpool_release_ctx_internal(kbdev, kctx, ++ &katom_retained_state); ++} ++ ++void kbasep_js_runpool_requeue_or_kill_ctx(struct kbase_device *kbdev, ++ struct kbase_context *kctx, bool has_pm_ref) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ js_devdata = &kbdev->js_data; ++ ++ /* This is called if and only if you've you've detached the context from ++ * the Runpool Queue, and not added it back to the Runpool ++ */ ++ KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ ++ if (kbase_ctx_flag(kctx, KCTX_DYING)) { ++ /* Dying: don't requeue, but kill all jobs on the context. This ++ * happens asynchronously */ ++ dev_dbg(kbdev->dev, ++ "JS: ** Killing Context %p on RunPool Remove **", kctx); ++ kbase_js_foreach_ctx_job(kctx, &kbase_jd_cancel); ++ } ++} ++ ++void kbasep_js_runpool_release_ctx_and_katom_retained_state( ++ struct kbase_device *kbdev, struct kbase_context *kctx, ++ struct kbasep_js_atom_retained_state *katom_retained_state) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ kbasep_js_release_result release_result; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ js_kctx_info = &kctx->jctx.sched_info; ++ js_devdata = &kbdev->js_data; ++ ++ mutex_lock(&js_devdata->queue_mutex); ++ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); ++ mutex_lock(&js_devdata->runpool_mutex); ++ ++ release_result = kbasep_js_runpool_release_ctx_internal(kbdev, kctx, ++ katom_retained_state); ++ ++ /* Drop the runpool mutex to allow requeing kctx */ ++ mutex_unlock(&js_devdata->runpool_mutex); ++ ++ if ((release_result & KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED) != 0u) ++ kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, true); ++ ++ /* Drop the jsctx_mutex to allow scheduling in a new context */ ++ ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ mutex_unlock(&js_devdata->queue_mutex); ++ ++ if (release_result & KBASEP_JS_RELEASE_RESULT_SCHED_ALL) ++ kbase_js_sched_all(kbdev); ++} ++ ++void kbasep_js_runpool_release_ctx(struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++ struct kbasep_js_atom_retained_state katom_retained_state; ++ ++ kbasep_js_atom_retained_state_init_invalid(&katom_retained_state); ++ ++ kbasep_js_runpool_release_ctx_and_katom_retained_state(kbdev, kctx, ++ &katom_retained_state); ++} ++ ++/* Variant of kbasep_js_runpool_release_ctx() that doesn't call into ++ * kbase_js_sched_all() */ ++static void kbasep_js_runpool_release_ctx_no_schedule( ++ struct kbase_device *kbdev, struct kbase_context *kctx) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ kbasep_js_release_result release_result; ++ struct kbasep_js_atom_retained_state katom_retained_state_struct; ++ struct kbasep_js_atom_retained_state *katom_retained_state = ++ &katom_retained_state_struct; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ js_kctx_info = &kctx->jctx.sched_info; ++ js_devdata = &kbdev->js_data; ++ kbasep_js_atom_retained_state_init_invalid(katom_retained_state); ++ ++ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); ++ mutex_lock(&js_devdata->runpool_mutex); ++ ++ release_result = kbasep_js_runpool_release_ctx_internal(kbdev, kctx, ++ katom_retained_state); ++ ++ /* Drop the runpool mutex to allow requeing kctx */ ++ mutex_unlock(&js_devdata->runpool_mutex); ++ if ((release_result & KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED) != 0u) ++ kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, true); ++ ++ /* Drop the jsctx_mutex to allow scheduling in a new context */ ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ ++ /* NOTE: could return release_result if the caller would like to know ++ * whether it should schedule a new context, but currently no callers do ++ */ ++} ++ ++void kbase_js_set_timeouts(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ kbase_backend_timeouts_changed(kbdev); ++} ++ ++static bool kbasep_js_schedule_ctx(struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ struct kbase_as *new_address_space = NULL; ++ unsigned long flags; ++ bool kctx_suspended = false; ++ int as_nr; ++ ++ js_devdata = &kbdev->js_data; ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ /* Pick available address space for this context */ ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ as_nr = kbase_ctx_sched_retain_ctx(kctx); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ if (as_nr == KBASEP_AS_NR_INVALID) { ++ as_nr = kbase_backend_find_and_release_free_address_space( ++ kbdev, kctx); ++ if (as_nr != KBASEP_AS_NR_INVALID) { ++ /* Attempt to retain the context again, this should ++ * succeed */ ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ as_nr = kbase_ctx_sched_retain_ctx(kctx); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ ++ WARN_ON(as_nr == KBASEP_AS_NR_INVALID); ++ } ++ } ++ if (as_nr == KBASEP_AS_NR_INVALID) ++ return false; /* No address spaces currently available */ ++ ++ new_address_space = &kbdev->as[as_nr]; ++ ++ /* ++ * Atomic transaction on the Context and Run Pool begins ++ */ ++ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); ++ mutex_lock(&js_devdata->runpool_mutex); ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ /* Check to see if context is dying due to kbase_job_zap_context() */ ++ if (kbase_ctx_flag(kctx, KCTX_DYING)) { ++ /* Roll back the transaction so far and return */ ++ kbase_ctx_sched_release_ctx(kctx); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ ++ return false; ++ } ++ ++ KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_TRY_SCHEDULE_HEAD_CTX, kctx, NULL, ++ 0u, ++ kbasep_js_trace_get_refcnt(kbdev, kctx)); ++ ++ kbase_ctx_flag_set(kctx, KCTX_SCHEDULED); ++ ++ /* Assign context to previously chosen address space */ ++ if (!kbase_backend_use_ctx(kbdev, kctx, as_nr)) { ++ /* Roll back the transaction so far and return */ ++ kbase_ctx_sched_release_ctx(kctx); ++ kbase_ctx_flag_clear(kctx, KCTX_SCHEDULED); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ ++ return false; ++ } ++ ++ kbdev->hwaccess.active_kctx = kctx; ++ ++#if defined(CONFIG_MALI_GATOR_SUPPORT) ++ kbase_trace_mali_mmu_as_in_use(kctx->as_nr); ++#endif ++ KBASE_TLSTREAM_TL_RET_AS_CTX(&kbdev->as[kctx->as_nr], kctx); ++ ++ /* Cause any future waiter-on-termination to wait until the context is ++ * descheduled */ ++ wake_up(&js_kctx_info->ctx.is_scheduled_wait); ++ ++ /* Re-check for suspending: a suspend could've occurred, and all the ++ * contexts could've been removed from the runpool before we took this ++ * lock. In this case, we don't want to allow this context to run jobs, ++ * we just want it out immediately. ++ * ++ * The DMB required to read the suspend flag was issued recently as part ++ * of the hwaccess_lock locking. If a suspend occurs *after* that lock ++ * was taken (i.e. this condition doesn't execute), then the ++ * kbasep_js_suspend() code will cleanup this context instead (by virtue ++ * of it being called strictly after the suspend flag is set, and will ++ * wait for this lock to drop) */ ++ if (kbase_pm_is_suspending(kbdev)) { ++ /* Cause it to leave at some later point */ ++ bool retained; ++ ++ retained = kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx); ++ KBASE_DEBUG_ASSERT(retained); ++ ++ kbasep_js_clear_submit_allowed(js_devdata, kctx); ++ kctx_suspended = true; ++ } ++ ++ /* Transaction complete */ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ ++ /* Synchronize with any timers */ ++ kbase_backend_ctx_count_changed(kbdev); ++ ++ mutex_unlock(&js_devdata->runpool_mutex); ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ /* Note: after this point, the context could potentially get scheduled ++ * out immediately */ ++ ++ if (kctx_suspended) { ++ /* Finishing forcing out the context due to a suspend. Use a ++ * variant of kbasep_js_runpool_release_ctx() that doesn't ++ * schedule a new context, to prevent a risk of recursion back ++ * into this function */ ++ kbasep_js_runpool_release_ctx_no_schedule(kbdev, kctx); ++ return false; ++ } ++ return true; ++} ++ ++static bool kbase_js_use_ctx(struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ if (kbase_ctx_flag(kctx, KCTX_SCHEDULED) && ++ kbase_backend_use_ctx_sched(kbdev, kctx)) { ++ /* Context already has ASID - mark as active */ ++ kbdev->hwaccess.active_kctx = kctx; ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ return true; /* Context already scheduled */ ++ } ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ return kbasep_js_schedule_ctx(kbdev, kctx); ++} ++ ++void kbasep_js_schedule_privileged_ctx(struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++ struct kbasep_js_kctx_info *js_kctx_info; ++ struct kbasep_js_device_data *js_devdata; ++ bool is_scheduled; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ ++ js_devdata = &kbdev->js_data; ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ /* This must never be attempted whilst suspending - i.e. it should only ++ * happen in response to a syscall from a user-space thread */ ++ BUG_ON(kbase_pm_is_suspending(kbdev)); ++ ++ mutex_lock(&js_devdata->queue_mutex); ++ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); ++ ++ /* Mark the context as privileged */ ++ kbase_ctx_flag_set(kctx, KCTX_PRIVILEGED); ++ ++ is_scheduled = kbase_ctx_flag(kctx, KCTX_SCHEDULED); ++ if (!is_scheduled) { ++ /* Add the context to the pullable list */ ++ if (kbase_js_ctx_list_add_pullable_head(kbdev, kctx, 0)) ++ kbase_js_sync_timers(kbdev); ++ ++ /* Fast-starting requires the jsctx_mutex to be dropped, ++ * because it works on multiple ctxs */ ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ mutex_unlock(&js_devdata->queue_mutex); ++ ++ /* Try to schedule the context in */ ++ kbase_js_sched_all(kbdev); ++ ++ /* Wait for the context to be scheduled in */ ++ wait_event(kctx->jctx.sched_info.ctx.is_scheduled_wait, ++ kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ } else { ++ /* Already scheduled in - We need to retain it to keep the ++ * corresponding address space */ ++ kbasep_js_runpool_retain_ctx(kbdev, kctx); ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ mutex_unlock(&js_devdata->queue_mutex); ++ } ++} ++KBASE_EXPORT_TEST_API(kbasep_js_schedule_privileged_ctx); ++ ++void kbasep_js_release_privileged_ctx(struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++ struct kbasep_js_kctx_info *js_kctx_info; ++ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ /* We don't need to use the address space anymore */ ++ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); ++ kbase_ctx_flag_clear(kctx, KCTX_PRIVILEGED); ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ ++ /* Release the context - it will be scheduled out */ ++ kbasep_js_runpool_release_ctx(kbdev, kctx); ++ ++ kbase_js_sched_all(kbdev); ++} ++KBASE_EXPORT_TEST_API(kbasep_js_release_privileged_ctx); ++ ++void kbasep_js_suspend(struct kbase_device *kbdev) ++{ ++ unsigned long flags; ++ struct kbasep_js_device_data *js_devdata; ++ int i; ++ u16 retained = 0u; ++ int nr_privileged_ctx = 0; ++ ++ KBASE_DEBUG_ASSERT(kbdev); ++ KBASE_DEBUG_ASSERT(kbase_pm_is_suspending(kbdev)); ++ js_devdata = &kbdev->js_data; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ /* Prevent all contexts from submitting */ ++ js_devdata->runpool_irq.submit_allowed = 0; ++ ++ /* Retain each of the contexts, so we can cause it to leave even if it ++ * had no refcount to begin with */ ++ for (i = BASE_MAX_NR_AS - 1; i >= 0; --i) { ++ struct kbase_context *kctx = kbdev->as_to_kctx[i]; ++ ++ retained = retained << 1; ++ ++ if (kctx) { ++ kbase_ctx_sched_retain_ctx_refcount(kctx); ++ retained |= 1u; ++ /* We can only cope with up to 1 privileged context - ++ * the instrumented context. It'll be suspended by ++ * disabling instrumentation */ ++ if (kbase_ctx_flag(kctx, KCTX_PRIVILEGED)) { ++ ++nr_privileged_ctx; ++ WARN_ON(nr_privileged_ctx != 1); ++ } ++ } ++ } ++ CSTD_UNUSED(nr_privileged_ctx); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ /* De-ref the previous retain to ensure each context gets pulled out ++ * sometime later. */ ++ for (i = 0; ++ i < BASE_MAX_NR_AS; ++ ++i, retained = retained >> 1) { ++ struct kbase_context *kctx = kbdev->as_to_kctx[i]; ++ ++ if (retained & 1u) ++ kbasep_js_runpool_release_ctx(kbdev, kctx); ++ } ++ ++ /* Caller must wait for all Power Manager active references to be ++ * dropped */ ++} ++ ++void kbasep_js_resume(struct kbase_device *kbdev) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ int js; ++ ++ KBASE_DEBUG_ASSERT(kbdev); ++ js_devdata = &kbdev->js_data; ++ KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev)); ++ ++ mutex_lock(&js_devdata->queue_mutex); ++ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { ++ struct kbase_context *kctx, *n; ++ ++ list_for_each_entry_safe(kctx, n, ++ &kbdev->js_data.ctx_list_unpullable[js], ++ jctx.sched_info.ctx.ctx_list_entry[js]) { ++ struct kbasep_js_kctx_info *js_kctx_info; ++ unsigned long flags; ++ bool timer_sync = false; ++ ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); ++ mutex_lock(&js_devdata->runpool_mutex); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ if (!kbase_ctx_flag(kctx, KCTX_SCHEDULED) && ++ kbase_js_ctx_pullable(kctx, js, false)) ++ timer_sync = ++ kbase_js_ctx_list_add_pullable_nolock( ++ kbdev, kctx, js); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ if (timer_sync) ++ kbase_backend_ctx_count_changed(kbdev); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ } ++ } ++ mutex_unlock(&js_devdata->queue_mutex); ++ ++ /* Restart atom processing */ ++ kbase_js_sched_all(kbdev); ++ ++ /* JS Resume complete */ ++} ++ ++bool kbase_js_is_atom_valid(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom) ++{ ++ if ((katom->core_req & BASE_JD_REQ_FS) && ++ (katom->core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE | ++ BASE_JD_REQ_T))) ++ return false; ++ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987) && ++ (katom->core_req & BASE_JD_REQ_ONLY_COMPUTE) && ++ (katom->core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_T))) ++ return false; ++ ++ return true; ++} ++ ++static int kbase_js_get_slot(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom) ++{ ++ if (katom->core_req & BASE_JD_REQ_FS) ++ return 0; ++ ++ if (katom->core_req & BASE_JD_REQ_ONLY_COMPUTE) { ++ if (katom->device_nr == 1 && ++ kbdev->gpu_props.num_core_groups == 2) ++ return 2; ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) ++ return 2; ++ } ++ ++ return 1; ++} ++ ++bool kbase_js_dep_resolved_submit(struct kbase_context *kctx, ++ struct kbase_jd_atom *katom) ++{ ++ bool enqueue_required; ++ ++ katom->slot_nr = kbase_js_get_slot(kctx->kbdev, katom); ++ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ lockdep_assert_held(&kctx->jctx.lock); ++ ++ /* If slot will transition from unpullable to pullable then add to ++ * pullable list */ ++ if (jsctx_rb_none_to_pull(kctx, katom->slot_nr)) { ++ enqueue_required = true; ++ } else { ++ enqueue_required = false; ++ } ++ if ((katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED) || ++ (katom->pre_dep && (katom->pre_dep->atom_flags & ++ KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST))) { ++ int prio = katom->sched_priority; ++ int js = katom->slot_nr; ++ struct jsctx_queue *queue = &kctx->jsctx_queue[prio][js]; ++ ++ list_add_tail(&katom->queue, &queue->x_dep_head); ++ katom->atom_flags |= KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST; ++ enqueue_required = false; ++ } else { ++ /* Check if there are lower priority jobs to soft stop */ ++ kbase_job_slot_ctx_priority_check_locked(kctx, katom); ++ ++ /* Add atom to ring buffer. */ ++ jsctx_tree_add(kctx, katom); ++ katom->atom_flags |= KBASE_KATOM_FLAG_JSCTX_IN_TREE; ++ } ++ ++ return enqueue_required; ++} ++ ++/** ++ * kbase_js_move_to_tree - Move atom (and any dependent atoms) to the ++ * runnable_tree, ready for execution ++ * @katom: Atom to submit ++ * ++ * It is assumed that @katom does not have KBASE_KATOM_FLAG_X_DEP_BLOCKED set, ++ * but is still present in the x_dep list. If @katom has a same-slot dependent ++ * atom then that atom (and any dependents) will also be moved. ++ */ ++static void kbase_js_move_to_tree(struct kbase_jd_atom *katom) ++{ ++ lockdep_assert_held(&katom->kctx->kbdev->hwaccess_lock); ++ ++ while (katom) { ++ WARN_ON(!(katom->atom_flags & ++ KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST)); ++ ++ if (!(katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED)) { ++ list_del(&katom->queue); ++ katom->atom_flags &= ++ ~KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST; ++ jsctx_tree_add(katom->kctx, katom); ++ katom->atom_flags |= KBASE_KATOM_FLAG_JSCTX_IN_TREE; ++ } else { ++ break; ++ } ++ ++ katom = katom->post_dep; ++ } ++} ++ ++ ++/** ++ * kbase_js_evict_deps - Evict dependencies of a failed atom. ++ * @kctx: Context pointer ++ * @katom: Pointer to the atom that has failed. ++ * @js: The job slot the katom was run on. ++ * @prio: Priority of the katom. ++ * ++ * Remove all post dependencies of an atom from the context ringbuffers. ++ * ++ * The original atom's event_code will be propogated to all dependent atoms. ++ * ++ * Context: Caller must hold the HW access lock ++ */ ++static void kbase_js_evict_deps(struct kbase_context *kctx, ++ struct kbase_jd_atom *katom, int js, int prio) ++{ ++ struct kbase_jd_atom *x_dep = katom->x_post_dep; ++ struct kbase_jd_atom *next_katom = katom->post_dep; ++ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ ++ if (next_katom) { ++ KBASE_DEBUG_ASSERT(next_katom->status != ++ KBASE_JD_ATOM_STATE_HW_COMPLETED); ++ next_katom->will_fail_event_code = katom->event_code; ++ ++ } ++ ++ /* Has cross slot depenency. */ ++ if (x_dep && (x_dep->atom_flags & (KBASE_KATOM_FLAG_JSCTX_IN_TREE | ++ KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST))) { ++ /* Remove dependency.*/ ++ x_dep->atom_flags &= ~KBASE_KATOM_FLAG_X_DEP_BLOCKED; ++ ++ /* Fail if it had a data dependency. */ ++ if (x_dep->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER) { ++ x_dep->will_fail_event_code = katom->event_code; ++ } ++ if (x_dep->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST) ++ kbase_js_move_to_tree(x_dep); ++ } ++} ++ ++struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, int js) ++{ ++ struct kbase_jd_atom *katom; ++ struct kbasep_js_device_data *js_devdata; ++ struct kbase_device *kbdev; ++ int pulled; ++ ++ KBASE_DEBUG_ASSERT(kctx); ++ ++ kbdev = kctx->kbdev; ++ ++ js_devdata = &kbdev->js_data; ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (!kbasep_js_is_submit_allowed(js_devdata, kctx)) ++ return NULL; ++ if (kbase_pm_is_suspending(kbdev)) ++ return NULL; ++ ++ katom = jsctx_rb_peek(kctx, js); ++ if (!katom) ++ return NULL; ++ if (kctx->blocked_js[js][katom->sched_priority]) ++ return NULL; ++ if (atomic_read(&katom->blocked)) ++ return NULL; ++ ++ /* Due to ordering restrictions when unpulling atoms on failure, we do ++ * not allow multiple runs of fail-dep atoms from the same context to be ++ * present on the same slot */ ++ if (katom->pre_dep && atomic_read(&kctx->atoms_pulled_slot[js])) { ++ struct kbase_jd_atom *prev_atom = ++ kbase_backend_inspect_tail(kbdev, js); ++ ++ if (prev_atom && prev_atom->kctx != kctx) ++ return NULL; ++ } ++ ++ if (katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED) { ++ if (katom->x_pre_dep->gpu_rb_state == ++ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB || ++ katom->x_pre_dep->will_fail_event_code) ++ return NULL; ++ if ((katom->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER) && ++ kbase_backend_nr_atoms_on_slot(kbdev, js)) ++ return NULL; ++ } ++ ++ kbase_ctx_flag_set(kctx, KCTX_PULLED); ++ ++ pulled = atomic_inc_return(&kctx->atoms_pulled); ++ if (pulled == 1 && !kctx->slots_pullable) { ++ WARN_ON(kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); ++ kbase_ctx_flag_set(kctx, KCTX_RUNNABLE_REF); ++ atomic_inc(&kbdev->js_data.nr_contexts_runnable); ++ } ++ atomic_inc(&kctx->atoms_pulled_slot[katom->slot_nr]); ++ kctx->atoms_pulled_slot_pri[katom->slot_nr][katom->sched_priority]++; ++ jsctx_rb_pull(kctx, katom); ++ ++ kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx); ++ ++ katom->atom_flags |= KBASE_KATOM_FLAG_HOLDING_CTX_REF; ++ ++ katom->ticks = 0; ++ ++ return katom; ++} ++ ++ ++static void js_return_worker(struct work_struct *data) ++{ ++ struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, ++ work); ++ struct kbase_context *kctx = katom->kctx; ++ struct kbase_device *kbdev = kctx->kbdev; ++ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; ++ struct kbasep_js_kctx_info *js_kctx_info = &kctx->jctx.sched_info; ++ struct kbasep_js_atom_retained_state retained_state; ++ int js = katom->slot_nr; ++ int prio = katom->sched_priority; ++ bool timer_sync = false; ++ bool context_idle = false; ++ unsigned long flags; ++ base_jd_core_req core_req = katom->core_req; ++ u64 affinity = katom->affinity; ++ enum kbase_atom_coreref_state coreref_state = katom->coreref_state; ++ ++ KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_EX(katom); ++ ++ kbase_backend_complete_wq(kbdev, katom); ++ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316)) ++ kbase_as_poking_timer_release_atom(kbdev, kctx, katom); ++ ++ kbasep_js_atom_retained_state_copy(&retained_state, katom); ++ ++ mutex_lock(&js_devdata->queue_mutex); ++ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); ++ ++ atomic_dec(&kctx->atoms_pulled); ++ atomic_dec(&kctx->atoms_pulled_slot[js]); ++ ++ atomic_dec(&katom->blocked); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ kctx->atoms_pulled_slot_pri[js][katom->sched_priority]--; ++ ++ if (!atomic_read(&kctx->atoms_pulled_slot[js]) && ++ jsctx_rb_none_to_pull(kctx, js)) ++ timer_sync |= kbase_js_ctx_list_remove_nolock(kbdev, kctx, js); ++ ++ /* If this slot has been blocked due to soft-stopped atoms, and all ++ * atoms have now been processed, then unblock the slot */ ++ if (!kctx->atoms_pulled_slot_pri[js][prio] && ++ kctx->blocked_js[js][prio]) { ++ kctx->blocked_js[js][prio] = false; ++ ++ /* Only mark the slot as pullable if the context is not idle - ++ * that case is handled below */ ++ if (atomic_read(&kctx->atoms_pulled) && ++ kbase_js_ctx_pullable(kctx, js, true)) ++ timer_sync |= kbase_js_ctx_list_add_pullable_nolock( ++ kbdev, kctx, js); ++ } ++ ++ if (!atomic_read(&kctx->atoms_pulled)) { ++ if (!kctx->slots_pullable) { ++ WARN_ON(!kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); ++ kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); ++ atomic_dec(&kbdev->js_data.nr_contexts_runnable); ++ timer_sync = true; ++ } ++ ++ if (kctx->as_nr != KBASEP_AS_NR_INVALID && ++ !kbase_ctx_flag(kctx, KCTX_DYING)) { ++ int num_slots = kbdev->gpu_props.num_job_slots; ++ int slot; ++ ++ if (!kbasep_js_is_submit_allowed(js_devdata, kctx)) ++ kbasep_js_set_submit_allowed(js_devdata, kctx); ++ ++ for (slot = 0; slot < num_slots; slot++) { ++ if (kbase_js_ctx_pullable(kctx, slot, true)) ++ timer_sync |= ++ kbase_js_ctx_list_add_pullable_nolock( ++ kbdev, kctx, slot); ++ } ++ } ++ ++ kbase_jm_idle_ctx(kbdev, kctx); ++ ++ context_idle = true; ++ } ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ if (context_idle) { ++ WARN_ON(!kbase_ctx_flag(kctx, KCTX_ACTIVE)); ++ kbase_ctx_flag_clear(kctx, KCTX_ACTIVE); ++ kbase_pm_context_idle(kbdev); ++ } ++ ++ if (timer_sync) ++ kbase_js_sync_timers(kbdev); ++ ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ mutex_unlock(&js_devdata->queue_mutex); ++ ++ katom->atom_flags &= ~KBASE_KATOM_FLAG_HOLDING_CTX_REF; ++ kbasep_js_runpool_release_ctx_and_katom_retained_state(kbdev, kctx, ++ &retained_state); ++ ++ kbase_js_sched_all(kbdev); ++ ++ kbase_backend_complete_wq_post_sched(kbdev, core_req, affinity, ++ coreref_state); ++} ++ ++void kbase_js_unpull(struct kbase_context *kctx, struct kbase_jd_atom *katom) ++{ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ ++ jsctx_rb_unpull(kctx, katom); ++ ++ WARN_ON(work_pending(&katom->work)); ++ ++ /* Block re-submission until workqueue has run */ ++ atomic_inc(&katom->blocked); ++ ++ kbase_job_check_leave_disjoint(kctx->kbdev, katom); ++ ++ KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&katom->work)); ++ INIT_WORK(&katom->work, js_return_worker); ++ queue_work(kctx->jctx.job_done_wq, &katom->work); ++} ++ ++bool kbase_js_complete_atom_wq(struct kbase_context *kctx, ++ struct kbase_jd_atom *katom) ++{ ++ struct kbasep_js_kctx_info *js_kctx_info; ++ struct kbasep_js_device_data *js_devdata; ++ struct kbase_device *kbdev; ++ unsigned long flags; ++ bool timer_sync = false; ++ int atom_slot; ++ bool context_idle = false; ++ int prio = katom->sched_priority; ++ ++ kbdev = kctx->kbdev; ++ atom_slot = katom->slot_nr; ++ ++ js_kctx_info = &kctx->jctx.sched_info; ++ js_devdata = &kbdev->js_data; ++ ++ lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); ++ ++ mutex_lock(&js_devdata->runpool_mutex); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ if (katom->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_TREE) { ++ context_idle = !atomic_dec_return(&kctx->atoms_pulled); ++ atomic_dec(&kctx->atoms_pulled_slot[atom_slot]); ++ kctx->atoms_pulled_slot_pri[atom_slot][prio]--; ++ ++ if (!atomic_read(&kctx->atoms_pulled) && ++ !kctx->slots_pullable) { ++ WARN_ON(!kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); ++ kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); ++ atomic_dec(&kbdev->js_data.nr_contexts_runnable); ++ timer_sync = true; ++ } ++ ++ /* If this slot has been blocked due to soft-stopped atoms, and ++ * all atoms have now been processed, then unblock the slot */ ++ if (!kctx->atoms_pulled_slot_pri[atom_slot][prio] ++ && kctx->blocked_js[atom_slot][prio]) { ++ kctx->blocked_js[atom_slot][prio] = false; ++ if (kbase_js_ctx_pullable(kctx, atom_slot, true)) ++ timer_sync |= ++ kbase_js_ctx_list_add_pullable_nolock( ++ kbdev, kctx, atom_slot); ++ } ++ } ++ WARN_ON(!(katom->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_TREE)); ++ ++ if (!atomic_read(&kctx->atoms_pulled_slot[atom_slot]) && ++ jsctx_rb_none_to_pull(kctx, atom_slot)) { ++ if (!list_empty( ++ &kctx->jctx.sched_info.ctx.ctx_list_entry[atom_slot])) ++ timer_sync |= kbase_js_ctx_list_remove_nolock( ++ kctx->kbdev, kctx, atom_slot); ++ } ++ ++ /* ++ * If submission is disabled on this context (most likely due to an ++ * atom failure) and there are now no atoms left in the system then ++ * re-enable submission so that context can be scheduled again. ++ */ ++ if (!kbasep_js_is_submit_allowed(js_devdata, kctx) && ++ !atomic_read(&kctx->atoms_pulled) && ++ !kbase_ctx_flag(kctx, KCTX_DYING)) { ++ int js; ++ ++ kbasep_js_set_submit_allowed(js_devdata, kctx); ++ ++ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { ++ if (kbase_js_ctx_pullable(kctx, js, true)) ++ timer_sync |= ++ kbase_js_ctx_list_add_pullable_nolock( ++ kbdev, kctx, js); ++ } ++ } else if (katom->x_post_dep && ++ kbasep_js_is_submit_allowed(js_devdata, kctx)) { ++ int js; ++ ++ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { ++ if (kbase_js_ctx_pullable(kctx, js, true)) ++ timer_sync |= ++ kbase_js_ctx_list_add_pullable_nolock( ++ kbdev, kctx, js); ++ } ++ } ++ ++ /* Mark context as inactive. The pm reference will be dropped later in ++ * jd_done_worker(). ++ */ ++ if (context_idle) ++ kbase_ctx_flag_clear(kctx, KCTX_ACTIVE); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ if (timer_sync) ++ kbase_backend_ctx_count_changed(kbdev); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ ++ return context_idle; ++} ++ ++struct kbase_jd_atom *kbase_js_complete_atom(struct kbase_jd_atom *katom, ++ ktime_t *end_timestamp) ++{ ++ u64 microseconds_spent = 0; ++ struct kbase_device *kbdev; ++ struct kbase_context *kctx = katom->kctx; ++ struct kbase_jd_atom *x_dep = katom->x_post_dep; ++ ++ kbdev = kctx->kbdev; ++ ++ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ ++ if (katom->will_fail_event_code) ++ katom->event_code = katom->will_fail_event_code; ++ ++ katom->status = KBASE_JD_ATOM_STATE_HW_COMPLETED; ++ ++ if (katom->event_code != BASE_JD_EVENT_DONE) { ++ kbase_js_evict_deps(kctx, katom, katom->slot_nr, ++ katom->sched_priority); ++ } ++ ++#if defined(CONFIG_MALI_GATOR_SUPPORT) ++ kbase_trace_mali_job_slots_event(GATOR_MAKE_EVENT(GATOR_JOB_SLOT_STOP, ++ katom->slot_nr), NULL, 0); ++#endif ++ ++ /* Calculate the job's time used */ ++ if (end_timestamp != NULL) { ++ /* Only calculating it for jobs that really run on the HW (e.g. ++ * removed from next jobs never actually ran, so really did take ++ * zero time) */ ++ ktime_t tick_diff = ktime_sub(*end_timestamp, ++ katom->start_timestamp); ++ ++ microseconds_spent = ktime_to_ns(tick_diff); ++ ++ do_div(microseconds_spent, 1000); ++ ++ /* Round up time spent to the minimum timer resolution */ ++ if (microseconds_spent < KBASEP_JS_TICK_RESOLUTION_US) ++ microseconds_spent = KBASEP_JS_TICK_RESOLUTION_US; ++ } ++ ++ ++ kbase_jd_done(katom, katom->slot_nr, end_timestamp, 0); ++ ++ /* Unblock cross dependency if present */ ++ if (x_dep && (katom->event_code == BASE_JD_EVENT_DONE || ++ !(x_dep->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER)) && ++ (x_dep->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED)) { ++ bool was_pullable = kbase_js_ctx_pullable(kctx, x_dep->slot_nr, ++ false); ++ x_dep->atom_flags &= ~KBASE_KATOM_FLAG_X_DEP_BLOCKED; ++ kbase_js_move_to_tree(x_dep); ++ if (!was_pullable && kbase_js_ctx_pullable(kctx, x_dep->slot_nr, ++ false)) ++ kbase_js_ctx_list_add_pullable_nolock(kbdev, kctx, ++ x_dep->slot_nr); ++ ++ if (x_dep->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_TREE) ++ return x_dep; ++ } ++ ++ return NULL; ++} ++ ++void kbase_js_sched(struct kbase_device *kbdev, int js_mask) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ struct kbase_context *last_active; ++ bool timer_sync = false; ++ bool ctx_waiting = false; ++ ++ js_devdata = &kbdev->js_data; ++ ++ down(&js_devdata->schedule_sem); ++ mutex_lock(&js_devdata->queue_mutex); ++ ++ last_active = kbdev->hwaccess.active_kctx; ++ ++ while (js_mask) { ++ int js; ++ ++ js = ffs(js_mask) - 1; ++ ++ while (1) { ++ struct kbase_context *kctx; ++ unsigned long flags; ++ bool context_idle = false; ++ ++ kctx = kbase_js_ctx_list_pop_head(kbdev, js); ++ ++ if (!kctx) { ++ js_mask &= ~(1 << js); ++ break; /* No contexts on pullable list */ ++ } ++ ++ if (!kbase_ctx_flag(kctx, KCTX_ACTIVE)) { ++ context_idle = true; ++ ++ if (kbase_pm_context_active_handle_suspend( ++ kbdev, ++ KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE)) { ++ /* Suspend pending - return context to ++ * queue and stop scheduling */ ++ mutex_lock( ++ &kctx->jctx.sched_info.ctx.jsctx_mutex); ++ if (kbase_js_ctx_list_add_pullable_head( ++ kctx->kbdev, kctx, js)) ++ kbase_js_sync_timers(kbdev); ++ mutex_unlock( ++ &kctx->jctx.sched_info.ctx.jsctx_mutex); ++ mutex_unlock(&js_devdata->queue_mutex); ++ up(&js_devdata->schedule_sem); ++ return; ++ } ++ kbase_ctx_flag_set(kctx, KCTX_ACTIVE); ++ } ++ ++ if (!kbase_js_use_ctx(kbdev, kctx)) { ++ mutex_lock( ++ &kctx->jctx.sched_info.ctx.jsctx_mutex); ++ /* Context can not be used at this time */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ if (kbase_js_ctx_pullable(kctx, js, false) ++ || kbase_ctx_flag(kctx, KCTX_PRIVILEGED)) ++ timer_sync |= ++ kbase_js_ctx_list_add_pullable_head_nolock( ++ kctx->kbdev, kctx, js); ++ else ++ timer_sync |= ++ kbase_js_ctx_list_add_unpullable_nolock( ++ kctx->kbdev, kctx, js); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, ++ flags); ++ mutex_unlock( ++ &kctx->jctx.sched_info.ctx.jsctx_mutex); ++ if (context_idle) { ++ WARN_ON(!kbase_ctx_flag(kctx, KCTX_ACTIVE)); ++ kbase_ctx_flag_clear(kctx, KCTX_ACTIVE); ++ kbase_pm_context_idle(kbdev); ++ } ++ ++ /* No more jobs can be submitted on this slot */ ++ js_mask &= ~(1 << js); ++ break; ++ } ++ mutex_lock(&kctx->jctx.sched_info.ctx.jsctx_mutex); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ kbase_ctx_flag_clear(kctx, KCTX_PULLED); ++ ++ if (!kbase_jm_kick(kbdev, 1 << js)) ++ /* No more jobs can be submitted on this slot */ ++ js_mask &= ~(1 << js); ++ ++ if (!kbase_ctx_flag(kctx, KCTX_PULLED)) { ++ bool pullable = kbase_js_ctx_pullable(kctx, js, ++ true); ++ ++ /* Failed to pull jobs - push to head of list. ++ * Unless this context is already 'active', in ++ * which case it's effectively already scheduled ++ * so push it to the back of the list. */ ++ if (pullable && kctx == last_active) ++ timer_sync |= ++ kbase_js_ctx_list_add_pullable_nolock( ++ kctx->kbdev, ++ kctx, js); ++ else if (pullable) ++ timer_sync |= ++ kbase_js_ctx_list_add_pullable_head_nolock( ++ kctx->kbdev, ++ kctx, js); ++ else ++ timer_sync |= ++ kbase_js_ctx_list_add_unpullable_nolock( ++ kctx->kbdev, ++ kctx, js); ++ ++ /* If this context is not the active context, ++ * but the active context is pullable on this ++ * slot, then we need to remove the active ++ * marker to prevent it from submitting atoms in ++ * the IRQ handler, which would prevent this ++ * context from making progress. */ ++ if (last_active && kctx != last_active && ++ kbase_js_ctx_pullable( ++ last_active, js, true)) ++ ctx_waiting = true; ++ ++ if (context_idle) { ++ kbase_jm_idle_ctx(kbdev, kctx); ++ spin_unlock_irqrestore( ++ &kbdev->hwaccess_lock, ++ flags); ++ WARN_ON(!kbase_ctx_flag(kctx, KCTX_ACTIVE)); ++ kbase_ctx_flag_clear(kctx, KCTX_ACTIVE); ++ kbase_pm_context_idle(kbdev); ++ } else { ++ spin_unlock_irqrestore( ++ &kbdev->hwaccess_lock, ++ flags); ++ } ++ mutex_unlock( ++ &kctx->jctx.sched_info.ctx.jsctx_mutex); ++ ++ js_mask &= ~(1 << js); ++ break; /* Could not run atoms on this slot */ ++ } ++ ++ /* Push to back of list */ ++ if (kbase_js_ctx_pullable(kctx, js, true)) ++ timer_sync |= ++ kbase_js_ctx_list_add_pullable_nolock( ++ kctx->kbdev, kctx, js); ++ else ++ timer_sync |= ++ kbase_js_ctx_list_add_unpullable_nolock( ++ kctx->kbdev, kctx, js); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&kctx->jctx.sched_info.ctx.jsctx_mutex); ++ } ++ } ++ ++ if (timer_sync) ++ kbase_js_sync_timers(kbdev); ++ ++ if (kbdev->hwaccess.active_kctx == last_active && ctx_waiting) ++ kbdev->hwaccess.active_kctx = NULL; ++ ++ mutex_unlock(&js_devdata->queue_mutex); ++ up(&js_devdata->schedule_sem); ++} ++ ++void kbase_js_zap_context(struct kbase_context *kctx) ++{ ++ struct kbase_device *kbdev = kctx->kbdev; ++ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; ++ struct kbasep_js_kctx_info *js_kctx_info = &kctx->jctx.sched_info; ++ int js; ++ ++ /* ++ * Critical assumption: No more submission is possible outside of the ++ * workqueue. This is because the OS *must* prevent U/K calls (IOCTLs) ++ * whilst the struct kbase_context is terminating. ++ */ ++ ++ /* First, atomically do the following: ++ * - mark the context as dying ++ * - try to evict it from the queue */ ++ mutex_lock(&kctx->jctx.lock); ++ mutex_lock(&js_devdata->queue_mutex); ++ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); ++ kbase_ctx_flag_set(kctx, KCTX_DYING); ++ ++ dev_dbg(kbdev->dev, "Zap: Try Evict Ctx %p", kctx); ++ ++ /* ++ * At this point we know: ++ * - If eviction succeeded, it was in the queue, but now no ++ * longer is ++ * - We must cancel the jobs here. No Power Manager active reference to ++ * release. ++ * - This happens asynchronously - kbase_jd_zap_context() will wait for ++ * those jobs to be killed. ++ * - If eviction failed, then it wasn't in the queue. It is one ++ * of the following: ++ * - a. it didn't have any jobs, and so is not in the Queue or ++ * the Run Pool (not scheduled) ++ * - Hence, no more work required to cancel jobs. No Power Manager ++ * active reference to release. ++ * - b. it was in the middle of a scheduling transaction (and thus must ++ * have at least 1 job). This can happen from a syscall or a ++ * kernel thread. We still hold the jsctx_mutex, and so the thread ++ * must be waiting inside kbasep_js_try_schedule_head_ctx(), ++ * before checking whether the runpool is full. That thread will ++ * continue after we drop the mutex, and will notice the context ++ * is dying. It will rollback the transaction, killing all jobs at ++ * the same time. kbase_jd_zap_context() will wait for those jobs ++ * to be killed. ++ * - Hence, no more work required to cancel jobs, or to release the ++ * Power Manager active reference. ++ * - c. it is scheduled, and may or may not be running jobs ++ * - We must cause it to leave the runpool by stopping it from ++ * submitting any more jobs. When it finally does leave, ++ * kbasep_js_runpool_requeue_or_kill_ctx() will kill all remaining jobs ++ * (because it is dying), release the Power Manager active reference, ++ * and will not requeue the context in the queue. ++ * kbase_jd_zap_context() will wait for those jobs to be killed. ++ * - Hence, work required just to make it leave the runpool. Cancelling ++ * jobs and releasing the Power manager active reference will be ++ * handled when it leaves the runpool. ++ */ ++ if (!kbase_ctx_flag(kctx, KCTX_SCHEDULED)) { ++ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { ++ if (!list_empty( ++ &kctx->jctx.sched_info.ctx.ctx_list_entry[js])) ++ list_del_init( ++ &kctx->jctx.sched_info.ctx.ctx_list_entry[js]); ++ } ++ ++ /* The following events require us to kill off remaining jobs ++ * and update PM book-keeping: ++ * - we evicted it correctly (it must have jobs to be in the ++ * Queue) ++ * ++ * These events need no action, but take this path anyway: ++ * - Case a: it didn't have any jobs, and was never in the Queue ++ * - Case b: scheduling transaction will be partially rolled- ++ * back (this already cancels the jobs) ++ */ ++ ++ KBASE_TRACE_ADD(kbdev, JM_ZAP_NON_SCHEDULED, kctx, NULL, 0u, ++ kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ ++ dev_dbg(kbdev->dev, "Zap: Ctx %p scheduled=0", kctx); ++ ++ /* Only cancel jobs when we evicted from the ++ * queue. No Power Manager active reference was held. ++ * ++ * Having is_dying set ensures that this kills, and ++ * doesn't requeue */ ++ kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, false); ++ ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ mutex_unlock(&js_devdata->queue_mutex); ++ mutex_unlock(&kctx->jctx.lock); ++ } else { ++ unsigned long flags; ++ bool was_retained; ++ ++ /* Case c: didn't evict, but it is scheduled - it's in the Run ++ * Pool */ ++ KBASE_TRACE_ADD(kbdev, JM_ZAP_SCHEDULED, kctx, NULL, 0u, ++ kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ dev_dbg(kbdev->dev, "Zap: Ctx %p is in RunPool", kctx); ++ ++ /* Disable the ctx from submitting any more jobs */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ kbasep_js_clear_submit_allowed(js_devdata, kctx); ++ ++ /* Retain and (later) release the context whilst it is is now ++ * disallowed from submitting jobs - ensures that someone ++ * somewhere will be removing the context later on */ ++ was_retained = kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx); ++ ++ /* Since it's scheduled and we have the jsctx_mutex, it must be ++ * retained successfully */ ++ KBASE_DEBUG_ASSERT(was_retained); ++ ++ dev_dbg(kbdev->dev, "Zap: Ctx %p Kill Any Running jobs", kctx); ++ ++ /* Cancel any remaining running jobs for this kctx - if any. ++ * Submit is disallowed which takes effect immediately, so no ++ * more new jobs will appear after we do this. */ ++ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) ++ kbase_job_slot_hardstop(kctx, js, NULL); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); ++ mutex_unlock(&js_devdata->queue_mutex); ++ mutex_unlock(&kctx->jctx.lock); ++ ++ dev_dbg(kbdev->dev, "Zap: Ctx %p Release (may or may not schedule out immediately)", ++ kctx); ++ ++ kbasep_js_runpool_release_ctx(kbdev, kctx); ++ } ++ ++ KBASE_TRACE_ADD(kbdev, JM_ZAP_DONE, kctx, NULL, 0u, 0u); ++ ++ /* After this, you must wait on both the ++ * kbase_jd_context::zero_jobs_wait and the ++ * kbasep_js_kctx_info::ctx::is_scheduled_waitq - to wait for the jobs ++ * to be destroyed, and the context to be de-scheduled (if it was on the ++ * runpool). ++ * ++ * kbase_jd_zap_context() will do this. */ ++} ++ ++static inline int trace_get_refcnt(struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++ return atomic_read(&kctx->refcount); ++} ++ ++/** ++ * kbase_js_foreach_ctx_job(): - Call a function on all jobs in context ++ * @kctx: Pointer to context. ++ * @callback: Pointer to function to call for each job. ++ * ++ * Call a function on all jobs belonging to a non-queued, non-running ++ * context, and detach the jobs from the context as it goes. ++ * ++ * Due to the locks that might be held at the time of the call, the callback ++ * may need to defer work on a workqueue to complete its actions (e.g. when ++ * cancelling jobs) ++ * ++ * Atoms will be removed from the queue, so this must only be called when ++ * cancelling jobs (which occurs as part of context destruction). ++ * ++ * The locking conditions on the caller are as follows: ++ * - it will be holding kbasep_js_kctx_info::ctx::jsctx_mutex. ++ */ ++static void kbase_js_foreach_ctx_job(struct kbase_context *kctx, ++ kbasep_js_ctx_job_cb callback) ++{ ++ struct kbase_device *kbdev; ++ struct kbasep_js_device_data *js_devdata; ++ unsigned long flags; ++ u32 js; ++ ++ kbdev = kctx->kbdev; ++ js_devdata = &kbdev->js_data; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_POLICY_FOREACH_CTX_JOBS, kctx, NULL, ++ 0u, trace_get_refcnt(kbdev, kctx)); ++ ++ /* Invoke callback on jobs on each slot in turn */ ++ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) ++ jsctx_queue_foreach(kctx, js, callback); ++ ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++} +diff --git a/drivers/gpu/arm/midgard/mali_kbase_js.h b/drivers/gpu/arm/midgard/mali_kbase_js.h +new file mode 100755 +index 000000000000..ddada8e468a1 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_js.h +@@ -0,0 +1,925 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/** ++ * @file mali_kbase_js.h ++ * Job Scheduler APIs. ++ */ ++ ++#ifndef _KBASE_JS_H_ ++#define _KBASE_JS_H_ ++ ++#include "mali_kbase_js_defs.h" ++#include "mali_kbase_context.h" ++#include "mali_kbase_defs.h" ++#include "mali_kbase_debug.h" ++ ++#include "mali_kbase_js_ctx_attr.h" ++ ++/** ++ * @addtogroup base_api ++ * @{ ++ */ ++ ++/** ++ * @addtogroup base_kbase_api ++ * @{ ++ */ ++ ++/** ++ * @addtogroup kbase_js Job Scheduler Internal APIs ++ * @{ ++ * ++ * These APIs are Internal to KBase. ++ */ ++ ++/** ++ * @brief Initialize the Job Scheduler ++ * ++ * The struct kbasep_js_device_data sub-structure of \a kbdev must be zero ++ * initialized before passing to the kbasep_js_devdata_init() function. This is ++ * to give efficient error path code. ++ */ ++int kbasep_js_devdata_init(struct kbase_device * const kbdev); ++ ++/** ++ * @brief Halt the Job Scheduler. ++ * ++ * It is safe to call this on \a kbdev even if it the kbasep_js_device_data ++ * sub-structure was never initialized/failed initialization, to give efficient ++ * error-path code. ++ * ++ * For this to work, the struct kbasep_js_device_data sub-structure of \a kbdev must ++ * be zero initialized before passing to the kbasep_js_devdata_init() ++ * function. This is to give efficient error path code. ++ * ++ * It is a Programming Error to call this whilst there are still kbase_context ++ * structures registered with this scheduler. ++ * ++ */ ++void kbasep_js_devdata_halt(struct kbase_device *kbdev); ++ ++/** ++ * @brief Terminate the Job Scheduler ++ * ++ * It is safe to call this on \a kbdev even if it the kbasep_js_device_data ++ * sub-structure was never initialized/failed initialization, to give efficient ++ * error-path code. ++ * ++ * For this to work, the struct kbasep_js_device_data sub-structure of \a kbdev must ++ * be zero initialized before passing to the kbasep_js_devdata_init() ++ * function. This is to give efficient error path code. ++ * ++ * It is a Programming Error to call this whilst there are still kbase_context ++ * structures registered with this scheduler. ++ */ ++void kbasep_js_devdata_term(struct kbase_device *kbdev); ++ ++/** ++ * @brief Initialize the Scheduling Component of a struct kbase_context on the Job Scheduler. ++ * ++ * This effectively registers a struct kbase_context with a Job Scheduler. ++ * ++ * It does not register any jobs owned by the struct kbase_context with the scheduler. ++ * Those must be separately registered by kbasep_js_add_job(). ++ * ++ * The struct kbase_context must be zero intitialized before passing to the ++ * kbase_js_init() function. This is to give efficient error path code. ++ */ ++int kbasep_js_kctx_init(struct kbase_context * const kctx); ++ ++/** ++ * @brief Terminate the Scheduling Component of a struct kbase_context on the Job Scheduler ++ * ++ * This effectively de-registers a struct kbase_context from its Job Scheduler ++ * ++ * It is safe to call this on a struct kbase_context that has never had or failed ++ * initialization of its jctx.sched_info member, to give efficient error-path ++ * code. ++ * ++ * For this to work, the struct kbase_context must be zero intitialized before passing ++ * to the kbase_js_init() function. ++ * ++ * It is a Programming Error to call this whilst there are still jobs ++ * registered with this context. ++ */ ++void kbasep_js_kctx_term(struct kbase_context *kctx); ++ ++/** ++ * @brief Add a job chain to the Job Scheduler, and take necessary actions to ++ * schedule the context/run the job. ++ * ++ * This atomically does the following: ++ * - Update the numbers of jobs information ++ * - Add the job to the run pool if necessary (part of init_job) ++ * ++ * Once this is done, then an appropriate action is taken: ++ * - If the ctx is scheduled, it attempts to start the next job (which might be ++ * this added job) ++ * - Otherwise, and if this is the first job on the context, it enqueues it on ++ * the Policy Queue ++ * ++ * The Policy's Queue can be updated by this in the following ways: ++ * - In the above case that this is the first job on the context ++ * - If the context is high priority and the context is not scheduled, then it ++ * could cause the Policy to schedule out a low-priority context, allowing ++ * this context to be scheduled in. ++ * ++ * If the context is already scheduled on the RunPool, then adding a job to it ++ * is guarenteed not to update the Policy Queue. And so, the caller is ++ * guarenteed to not need to try scheduling a context from the Run Pool - it ++ * can safely assert that the result is false. ++ * ++ * It is a programming error to have more than U32_MAX jobs in flight at a time. ++ * ++ * The following locking conditions are made on the caller: ++ * - it must \em not hold kbasep_js_kctx_info::ctx::jsctx_mutex. ++ * - it must \em not hold hwaccess_lock (as this will be obtained internally) ++ * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this will be ++ * obtained internally) ++ * - it must \em not hold kbasep_jd_device_data::queue_mutex (again, it's used internally). ++ * ++ * @return true indicates that the Policy Queue was updated, and so the ++ * caller will need to try scheduling a context onto the Run Pool. ++ * @return false indicates that no updates were made to the Policy Queue, ++ * so no further action is required from the caller. This is \b always returned ++ * when the context is currently scheduled. ++ */ ++bool kbasep_js_add_job(struct kbase_context *kctx, struct kbase_jd_atom *atom); ++ ++/** ++ * @brief Remove a job chain from the Job Scheduler, except for its 'retained state'. ++ * ++ * Completely removing a job requires several calls: ++ * - kbasep_js_copy_atom_retained_state(), to capture the 'retained state' of ++ * the atom ++ * - kbasep_js_remove_job(), to partially remove the atom from the Job Scheduler ++ * - kbasep_js_runpool_release_ctx_and_katom_retained_state(), to release the ++ * remaining state held as part of the job having been run. ++ * ++ * In the common case of atoms completing normally, this set of actions is more optimal for spinlock purposes than having kbasep_js_remove_job() handle all of the actions. ++ * ++ * In the case of cancelling atoms, it is easier to call kbasep_js_remove_cancelled_job(), which handles all the necessary actions. ++ * ++ * It is a programming error to call this when: ++ * - \a atom is not a job belonging to kctx. ++ * - \a atom has already been removed from the Job Scheduler. ++ * - \a atom is still in the runpool ++ * ++ * Do not use this for removing jobs being killed by kbase_jd_cancel() - use ++ * kbasep_js_remove_cancelled_job() instead. ++ * ++ * The following locking conditions are made on the caller: ++ * - it must hold kbasep_js_kctx_info::ctx::jsctx_mutex. ++ * ++ */ ++void kbasep_js_remove_job(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *atom); ++ ++/** ++ * @brief Completely remove a job chain from the Job Scheduler, in the case ++ * where the job chain was cancelled. ++ * ++ * This is a variant of kbasep_js_remove_job() that takes care of removing all ++ * of the retained state too. This is generally useful for cancelled atoms, ++ * which need not be handled in an optimal way. ++ * ++ * It is a programming error to call this when: ++ * - \a atom is not a job belonging to kctx. ++ * - \a atom has already been removed from the Job Scheduler. ++ * - \a atom is still in the runpool: ++ * - it is not being killed with kbasep_jd_cancel() ++ * ++ * The following locking conditions are made on the caller: ++ * - it must hold kbasep_js_kctx_info::ctx::jsctx_mutex. ++ * - it must \em not hold the hwaccess_lock, (as this will be obtained ++ * internally) ++ * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this could be ++ * obtained internally) ++ * ++ * @return true indicates that ctx attributes have changed and the caller ++ * should call kbase_js_sched_all() to try to run more jobs ++ * @return false otherwise ++ */ ++bool kbasep_js_remove_cancelled_job(struct kbase_device *kbdev, ++ struct kbase_context *kctx, ++ struct kbase_jd_atom *katom); ++ ++/** ++ * @brief Refcount a context as being busy, preventing it from being scheduled ++ * out. ++ * ++ * @note This function can safely be called from IRQ context. ++ * ++ * The following locking conditions are made on the caller: ++ * - it must \em not hold mmu_hw_mutex and hwaccess_lock, because they will be ++ * used internally. ++ * ++ * @return value != false if the retain succeeded, and the context will not be scheduled out. ++ * @return false if the retain failed (because the context is being/has been scheduled out). ++ */ ++bool kbasep_js_runpool_retain_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); ++ ++/** ++ * @brief Refcount a context as being busy, preventing it from being scheduled ++ * out. ++ * ++ * @note This function can safely be called from IRQ context. ++ * ++ * The following locks must be held by the caller: ++ * - mmu_hw_mutex, hwaccess_lock ++ * ++ * @return value != false if the retain succeeded, and the context will not be scheduled out. ++ * @return false if the retain failed (because the context is being/has been scheduled out). ++ */ ++bool kbasep_js_runpool_retain_ctx_nolock(struct kbase_device *kbdev, struct kbase_context *kctx); ++ ++/** ++ * @brief Lookup a context in the Run Pool based upon its current address space ++ * and ensure that is stays scheduled in. ++ * ++ * The context is refcounted as being busy to prevent it from scheduling ++ * out. It must be released with kbasep_js_runpool_release_ctx() when it is no ++ * longer required to stay scheduled in. ++ * ++ * @note This function can safely be called from IRQ context. ++ * ++ * The following locking conditions are made on the caller: ++ * - it must \em not hold the hwaccess_lock, because it will be used internally. ++ * If the hwaccess_lock is already held, then the caller should use ++ * kbasep_js_runpool_lookup_ctx_nolock() instead. ++ * ++ * @return a valid struct kbase_context on success, which has been refcounted as being busy. ++ * @return NULL on failure, indicating that no context was found in \a as_nr ++ */ ++struct kbase_context *kbasep_js_runpool_lookup_ctx(struct kbase_device *kbdev, int as_nr); ++ ++/** ++ * @brief Handling the requeuing/killing of a context that was evicted from the ++ * policy queue or runpool. ++ * ++ * This should be used whenever handing off a context that has been evicted ++ * from the policy queue or the runpool: ++ * - If the context is not dying and has jobs, it gets re-added to the policy ++ * queue ++ * - Otherwise, it is not added ++ * ++ * In addition, if the context is dying the jobs are killed asynchronously. ++ * ++ * In all cases, the Power Manager active reference is released ++ * (kbase_pm_context_idle()) whenever the has_pm_ref parameter is true. \a ++ * has_pm_ref must be set to false whenever the context was not previously in ++ * the runpool and does not hold a Power Manager active refcount. Note that ++ * contexts in a rollback of kbasep_js_try_schedule_head_ctx() might have an ++ * active refcount even though they weren't in the runpool. ++ * ++ * The following locking conditions are made on the caller: ++ * - it must hold kbasep_js_kctx_info::ctx::jsctx_mutex. ++ * - it must \em not hold kbasep_jd_device_data::queue_mutex (as this will be ++ * obtained internally) ++ */ ++void kbasep_js_runpool_requeue_or_kill_ctx(struct kbase_device *kbdev, struct kbase_context *kctx, bool has_pm_ref); ++ ++/** ++ * @brief Release a refcount of a context being busy, allowing it to be ++ * scheduled out. ++ * ++ * When the refcount reaches zero and the context \em might be scheduled out ++ * (depending on whether the Scheudling Policy has deemed it so, or if it has run ++ * out of jobs). ++ * ++ * If the context does get scheduled out, then The following actions will be ++ * taken as part of deschduling a context: ++ * - For the context being descheduled: ++ * - If the context is in the processing of dying (all the jobs are being ++ * removed from it), then descheduling also kills off any jobs remaining in the ++ * context. ++ * - If the context is not dying, and any jobs remain after descheduling the ++ * context then it is re-enqueued to the Policy's Queue. ++ * - Otherwise, the context is still known to the scheduler, but remains absent ++ * from the Policy Queue until a job is next added to it. ++ * - In all descheduling cases, the Power Manager active reference (obtained ++ * during kbasep_js_try_schedule_head_ctx()) is released (kbase_pm_context_idle()). ++ * ++ * Whilst the context is being descheduled, this also handles actions that ++ * cause more atoms to be run: ++ * - Attempt submitting atoms when the Context Attributes on the Runpool have ++ * changed. This is because the context being scheduled out could mean that ++ * there are more opportunities to run atoms. ++ * - Attempt submitting to a slot that was previously blocked due to affinity ++ * restrictions. This is usually only necessary when releasing a context ++ * happens as part of completing a previous job, but is harmless nonetheless. ++ * - Attempt scheduling in a new context (if one is available), and if necessary, ++ * running a job from that new context. ++ * ++ * Unlike retaining a context in the runpool, this function \b cannot be called ++ * from IRQ context. ++ * ++ * It is a programming error to call this on a \a kctx that is not currently ++ * scheduled, or that already has a zero refcount. ++ * ++ * The following locking conditions are made on the caller: ++ * - it must \em not hold the hwaccess_lock, because it will be used internally. ++ * - it must \em not hold kbasep_js_kctx_info::ctx::jsctx_mutex. ++ * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this will be ++ * obtained internally) ++ * - it must \em not hold the kbase_device::mmu_hw_mutex (as this will be ++ * obtained internally) ++ * - it must \em not hold kbasep_jd_device_data::queue_mutex (as this will be ++ * obtained internally) ++ * ++ */ ++void kbasep_js_runpool_release_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); ++ ++/** ++ * @brief Variant of kbasep_js_runpool_release_ctx() that handles additional ++ * actions from completing an atom. ++ * ++ * This is usually called as part of completing an atom and releasing the ++ * refcount on the context held by the atom. ++ * ++ * Therefore, the extra actions carried out are part of handling actions queued ++ * on a completed atom, namely: ++ * - Releasing the atom's context attributes ++ * - Retrying the submission on a particular slot, because we couldn't submit ++ * on that slot from an IRQ handler. ++ * ++ * The locking conditions of this function are the same as those for ++ * kbasep_js_runpool_release_ctx() ++ */ ++void kbasep_js_runpool_release_ctx_and_katom_retained_state(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbasep_js_atom_retained_state *katom_retained_state); ++ ++/** ++ * @brief Variant of kbase_js_runpool_release_ctx() that assumes that ++ * kbasep_js_device_data::runpool_mutex and ++ * kbasep_js_kctx_info::ctx::jsctx_mutex are held by the caller, and does not ++ * attempt to schedule new contexts. ++ */ ++void kbasep_js_runpool_release_ctx_nolock(struct kbase_device *kbdev, ++ struct kbase_context *kctx); ++ ++/** ++ * @brief Schedule in a privileged context ++ * ++ * This schedules a context in regardless of the context priority. ++ * If the runpool is full, a context will be forced out of the runpool and the function will wait ++ * for the new context to be scheduled in. ++ * The context will be kept scheduled in (and the corresponding address space reserved) until ++ * kbasep_js_release_privileged_ctx is called). ++ * ++ * The following locking conditions are made on the caller: ++ * - it must \em not hold the hwaccess_lock, because it will be used internally. ++ * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this will be ++ * obtained internally) ++ * - it must \em not hold the kbase_device::mmu_hw_mutex (as this will be ++ * obtained internally) ++ * - it must \em not hold kbasep_jd_device_data::queue_mutex (again, it's used internally). ++ * - it must \em not hold kbasep_js_kctx_info::ctx::jsctx_mutex, because it will ++ * be used internally. ++ * ++ */ ++void kbasep_js_schedule_privileged_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); ++ ++/** ++ * @brief Release a privileged context, allowing it to be scheduled out. ++ * ++ * See kbasep_js_runpool_release_ctx for potential side effects. ++ * ++ * The following locking conditions are made on the caller: ++ * - it must \em not hold the hwaccess_lock, because it will be used internally. ++ * - it must \em not hold kbasep_js_kctx_info::ctx::jsctx_mutex. ++ * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this will be ++ * obtained internally) ++ * - it must \em not hold the kbase_device::mmu_hw_mutex (as this will be ++ * obtained internally) ++ * ++ */ ++void kbasep_js_release_privileged_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); ++ ++/** ++ * @brief Try to submit the next job on each slot ++ * ++ * The following locks may be used: ++ * - kbasep_js_device_data::runpool_mutex ++ * - hwaccess_lock ++ */ ++void kbase_js_try_run_jobs(struct kbase_device *kbdev); ++ ++/** ++ * @brief Suspend the job scheduler during a Power Management Suspend event. ++ * ++ * Causes all contexts to be removed from the runpool, and prevents any ++ * contexts from (re)entering the runpool. ++ * ++ * This does not handle suspending the one privileged context: the caller must ++ * instead do this by by suspending the GPU HW Counter Instrumentation. ++ * ++ * This will eventually cause all Power Management active references held by ++ * contexts on the runpool to be released, without running any more atoms. ++ * ++ * The caller must then wait for all Power Mangement active refcount to become ++ * zero before completing the suspend. ++ * ++ * The emptying mechanism may take some time to complete, since it can wait for ++ * jobs to complete naturally instead of forcing them to end quickly. However, ++ * this is bounded by the Job Scheduler's Job Timeouts. Hence, this ++ * function is guaranteed to complete in a finite time. ++ */ ++void kbasep_js_suspend(struct kbase_device *kbdev); ++ ++/** ++ * @brief Resume the Job Scheduler after a Power Management Resume event. ++ * ++ * This restores the actions from kbasep_js_suspend(): ++ * - Schedules contexts back into the runpool ++ * - Resumes running atoms on the GPU ++ */ ++void kbasep_js_resume(struct kbase_device *kbdev); ++ ++/** ++ * @brief Submit an atom to the job scheduler. ++ * ++ * The atom is enqueued on the context's ringbuffer. The caller must have ++ * ensured that all dependencies can be represented in the ringbuffer. ++ * ++ * Caller must hold jctx->lock ++ * ++ * @param[in] kctx Context pointer ++ * @param[in] atom Pointer to the atom to submit ++ * ++ * @return Whether the context requires to be enqueued. */ ++bool kbase_js_dep_resolved_submit(struct kbase_context *kctx, ++ struct kbase_jd_atom *katom); ++ ++/** ++ * jsctx_ll_flush_to_rb() - Pushes atoms from the linked list to ringbuffer. ++ * @kctx: Context Pointer ++ * @prio: Priority (specifies the queue together with js). ++ * @js: Job slot (specifies the queue together with prio). ++ * ++ * Pushes all possible atoms from the linked list to the ringbuffer. ++ * Number of atoms are limited to free space in the ringbuffer and ++ * number of available atoms in the linked list. ++ * ++ */ ++void jsctx_ll_flush_to_rb(struct kbase_context *kctx, int prio, int js); ++/** ++ * @brief Pull an atom from a context in the job scheduler for execution. ++ * ++ * The atom will not be removed from the ringbuffer at this stage. ++ * ++ * The HW access lock must be held when calling this function. ++ * ++ * @param[in] kctx Context to pull from ++ * @param[in] js Job slot to pull from ++ * @return Pointer to an atom, or NULL if there are no atoms for this ++ * slot that can be currently run. ++ */ ++struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, int js); ++ ++/** ++ * @brief Return an atom to the job scheduler ringbuffer. ++ * ++ * An atom is 'unpulled' if execution is stopped but intended to be returned to ++ * later. The most common reason for this is that the atom has been ++ * soft-stopped. ++ * ++ * Note that if multiple atoms are to be 'unpulled', they must be returned in ++ * the reverse order to which they were originally pulled. It is a programming ++ * error to return atoms in any other order. ++ * ++ * The HW access lock must be held when calling this function. ++ * ++ * @param[in] kctx Context pointer ++ * @param[in] atom Pointer to the atom to unpull ++ */ ++void kbase_js_unpull(struct kbase_context *kctx, struct kbase_jd_atom *katom); ++ ++/** ++ * @brief Complete an atom from jd_done_worker(), removing it from the job ++ * scheduler ringbuffer. ++ * ++ * If the atom failed then all dependee atoms marked for failure propagation ++ * will also fail. ++ * ++ * @param[in] kctx Context pointer ++ * @param[in] katom Pointer to the atom to complete ++ * @return true if the context is now idle (no jobs pulled) ++ * false otherwise ++ */ ++bool kbase_js_complete_atom_wq(struct kbase_context *kctx, ++ struct kbase_jd_atom *katom); ++ ++/** ++ * @brief Complete an atom. ++ * ++ * Most of the work required to complete an atom will be performed by ++ * jd_done_worker(). ++ * ++ * The HW access lock must be held when calling this function. ++ * ++ * @param[in] katom Pointer to the atom to complete ++ * @param[in] end_timestamp The time that the atom completed (may be NULL) ++ * ++ * Return: Atom that has now been unblocked and can now be run, or NULL if none ++ */ ++struct kbase_jd_atom *kbase_js_complete_atom(struct kbase_jd_atom *katom, ++ ktime_t *end_timestamp); ++ ++/** ++ * @brief Submit atoms from all available contexts. ++ * ++ * This will attempt to submit as many jobs as possible to the provided job ++ * slots. It will exit when either all job slots are full, or all contexts have ++ * been used. ++ * ++ * @param[in] kbdev Device pointer ++ * @param[in] js_mask Mask of job slots to submit to ++ */ ++void kbase_js_sched(struct kbase_device *kbdev, int js_mask); ++ ++/** ++ * kbase_jd_zap_context - Attempt to deschedule a context that is being ++ * destroyed ++ * @kctx: Context pointer ++ * ++ * This will attempt to remove a context from any internal job scheduler queues ++ * and perform any other actions to ensure a context will not be submitted ++ * from. ++ * ++ * If the context is currently scheduled, then the caller must wait for all ++ * pending jobs to complete before taking any further action. ++ */ ++void kbase_js_zap_context(struct kbase_context *kctx); ++ ++/** ++ * @brief Validate an atom ++ * ++ * This will determine whether the atom can be scheduled onto the GPU. Atoms ++ * with invalid combinations of core requirements will be rejected. ++ * ++ * @param[in] kbdev Device pointer ++ * @param[in] katom Atom to validate ++ * @return true if atom is valid ++ * false otherwise ++ */ ++bool kbase_js_is_atom_valid(struct kbase_device *kbdev, ++ struct kbase_jd_atom *katom); ++ ++/** ++ * kbase_js_set_timeouts - update all JS timeouts with user specified data ++ * @kbdev: Device pointer ++ * ++ * Timeouts are specified through the 'js_timeouts' sysfs file. If a timeout is ++ * set to a positive number then that becomes the new value used, if a timeout ++ * is negative then the default is set. ++ */ ++void kbase_js_set_timeouts(struct kbase_device *kbdev); ++ ++/* ++ * Helpers follow ++ */ ++ ++/** ++ * @brief Check that a context is allowed to submit jobs on this policy ++ * ++ * The purpose of this abstraction is to hide the underlying data size, and wrap up ++ * the long repeated line of code. ++ * ++ * As with any bool, never test the return value with true. ++ * ++ * The caller must hold hwaccess_lock. ++ */ ++static inline bool kbasep_js_is_submit_allowed(struct kbasep_js_device_data *js_devdata, struct kbase_context *kctx) ++{ ++ u16 test_bit; ++ ++ /* Ensure context really is scheduled in */ ++ KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); ++ KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ ++ test_bit = (u16) (1u << kctx->as_nr); ++ ++ return (bool) (js_devdata->runpool_irq.submit_allowed & test_bit); ++} ++ ++/** ++ * @brief Allow a context to submit jobs on this policy ++ * ++ * The purpose of this abstraction is to hide the underlying data size, and wrap up ++ * the long repeated line of code. ++ * ++ * The caller must hold hwaccess_lock. ++ */ ++static inline void kbasep_js_set_submit_allowed(struct kbasep_js_device_data *js_devdata, struct kbase_context *kctx) ++{ ++ u16 set_bit; ++ ++ /* Ensure context really is scheduled in */ ++ KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); ++ KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ ++ set_bit = (u16) (1u << kctx->as_nr); ++ ++ dev_dbg(kctx->kbdev->dev, "JS: Setting Submit Allowed on %p (as=%d)", kctx, kctx->as_nr); ++ ++ js_devdata->runpool_irq.submit_allowed |= set_bit; ++} ++ ++/** ++ * @brief Prevent a context from submitting more jobs on this policy ++ * ++ * The purpose of this abstraction is to hide the underlying data size, and wrap up ++ * the long repeated line of code. ++ * ++ * The caller must hold hwaccess_lock. ++ */ ++static inline void kbasep_js_clear_submit_allowed(struct kbasep_js_device_data *js_devdata, struct kbase_context *kctx) ++{ ++ u16 clear_bit; ++ u16 clear_mask; ++ ++ /* Ensure context really is scheduled in */ ++ KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); ++ KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ ++ clear_bit = (u16) (1u << kctx->as_nr); ++ clear_mask = ~clear_bit; ++ ++ dev_dbg(kctx->kbdev->dev, "JS: Clearing Submit Allowed on %p (as=%d)", kctx, kctx->as_nr); ++ ++ js_devdata->runpool_irq.submit_allowed &= clear_mask; ++} ++ ++/** ++ * @brief Manage the 'retry_submit_on_slot' part of a kbase_jd_atom ++ */ ++static inline void kbasep_js_clear_job_retry_submit(struct kbase_jd_atom *atom) ++{ ++ atom->retry_submit_on_slot = KBASEP_JS_RETRY_SUBMIT_SLOT_INVALID; ++} ++ ++/** ++ * Mark a slot as requiring resubmission by carrying that information on a ++ * completing atom. ++ * ++ * @note This can ASSERT in debug builds if the submit slot has been set to ++ * something other than the current value for @a js. This is because you might ++ * be unintentionally stopping more jobs being submitted on the old submit ++ * slot, and that might cause a scheduling-hang. ++ * ++ * @note If you can guarantee that the atoms for the original slot will be ++ * submitted on some other slot, then call kbasep_js_clear_job_retry_submit() ++ * first to silence the ASSERT. ++ */ ++static inline void kbasep_js_set_job_retry_submit_slot(struct kbase_jd_atom *atom, int js) ++{ ++ KBASE_DEBUG_ASSERT(0 <= js && js <= BASE_JM_MAX_NR_SLOTS); ++ KBASE_DEBUG_ASSERT((atom->retry_submit_on_slot == ++ KBASEP_JS_RETRY_SUBMIT_SLOT_INVALID) ++ || (atom->retry_submit_on_slot == js)); ++ ++ atom->retry_submit_on_slot = js; ++} ++ ++/** ++ * Create an initial 'invalid' atom retained state, that requires no ++ * atom-related work to be done on releasing with ++ * kbasep_js_runpool_release_ctx_and_katom_retained_state() ++ */ ++static inline void kbasep_js_atom_retained_state_init_invalid(struct kbasep_js_atom_retained_state *retained_state) ++{ ++ retained_state->event_code = BASE_JD_EVENT_NOT_STARTED; ++ retained_state->core_req = KBASEP_JS_ATOM_RETAINED_STATE_CORE_REQ_INVALID; ++ retained_state->retry_submit_on_slot = KBASEP_JS_RETRY_SUBMIT_SLOT_INVALID; ++} ++ ++/** ++ * Copy atom state that can be made available after jd_done_nolock() is called ++ * on that atom. ++ */ ++static inline void kbasep_js_atom_retained_state_copy(struct kbasep_js_atom_retained_state *retained_state, const struct kbase_jd_atom *katom) ++{ ++ retained_state->event_code = katom->event_code; ++ retained_state->core_req = katom->core_req; ++ retained_state->retry_submit_on_slot = katom->retry_submit_on_slot; ++ retained_state->sched_priority = katom->sched_priority; ++ retained_state->device_nr = katom->device_nr; ++} ++ ++/** ++ * @brief Determine whether an atom has finished (given its retained state), ++ * and so should be given back to userspace/removed from the system. ++ * ++ * Reasons for an atom not finishing include: ++ * - Being soft-stopped (and so, the atom should be resubmitted sometime later) ++ * ++ * @param[in] katom_retained_state the retained state of the atom to check ++ * @return false if the atom has not finished ++ * @return !=false if the atom has finished ++ */ ++static inline bool kbasep_js_has_atom_finished(const struct kbasep_js_atom_retained_state *katom_retained_state) ++{ ++ return (bool) (katom_retained_state->event_code != BASE_JD_EVENT_STOPPED && katom_retained_state->event_code != BASE_JD_EVENT_REMOVED_FROM_NEXT); ++} ++ ++/** ++ * @brief Determine whether a struct kbasep_js_atom_retained_state is valid ++ * ++ * An invalid struct kbasep_js_atom_retained_state is allowed, and indicates that the ++ * code should just ignore it. ++ * ++ * @param[in] katom_retained_state the atom's retained state to check ++ * @return false if the retained state is invalid, and can be ignored ++ * @return !=false if the retained state is valid ++ */ ++static inline bool kbasep_js_atom_retained_state_is_valid(const struct kbasep_js_atom_retained_state *katom_retained_state) ++{ ++ return (bool) (katom_retained_state->core_req != KBASEP_JS_ATOM_RETAINED_STATE_CORE_REQ_INVALID); ++} ++ ++static inline bool kbasep_js_get_atom_retry_submit_slot(const struct kbasep_js_atom_retained_state *katom_retained_state, int *res) ++{ ++ int js = katom_retained_state->retry_submit_on_slot; ++ ++ *res = js; ++ return (bool) (js >= 0); ++} ++ ++/** ++ * @brief Variant of kbasep_js_runpool_lookup_ctx() that can be used when the ++ * context is guaranteed to be already previously retained. ++ * ++ * It is a programming error to supply the \a as_nr of a context that has not ++ * been previously retained/has a busy refcount of zero. The only exception is ++ * when there is no ctx in \a as_nr (NULL returned). ++ * ++ * The following locking conditions are made on the caller: ++ * - it must \em not hold the hwaccess_lock, because it will be used internally. ++ * ++ * @return a valid struct kbase_context on success, with a refcount that is guaranteed ++ * to be non-zero and unmodified by this function. ++ * @return NULL on failure, indicating that no context was found in \a as_nr ++ */ ++static inline struct kbase_context *kbasep_js_runpool_lookup_ctx_noretain(struct kbase_device *kbdev, int as_nr) ++{ ++ struct kbase_context *found_kctx; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(0 <= as_nr && as_nr < BASE_MAX_NR_AS); ++ ++ found_kctx = kbdev->as_to_kctx[as_nr]; ++ KBASE_DEBUG_ASSERT(found_kctx == NULL || ++ atomic_read(&found_kctx->refcount) > 0); ++ ++ return found_kctx; ++} ++ ++/* ++ * The following locking conditions are made on the caller: ++ * - The caller must hold the kbasep_js_kctx_info::ctx::jsctx_mutex. ++ * - The caller must hold the kbasep_js_device_data::runpool_mutex ++ */ ++static inline void kbase_js_runpool_inc_context_count( ++ struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ ++ js_devdata = &kbdev->js_data; ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); ++ lockdep_assert_held(&js_devdata->runpool_mutex); ++ ++ /* Track total contexts */ ++ KBASE_DEBUG_ASSERT(js_devdata->nr_all_contexts_running < S8_MAX); ++ ++(js_devdata->nr_all_contexts_running); ++ ++ if (!kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) { ++ /* Track contexts that can submit jobs */ ++ KBASE_DEBUG_ASSERT(js_devdata->nr_user_contexts_running < ++ S8_MAX); ++ ++(js_devdata->nr_user_contexts_running); ++ } ++} ++ ++/* ++ * The following locking conditions are made on the caller: ++ * - The caller must hold the kbasep_js_kctx_info::ctx::jsctx_mutex. ++ * - The caller must hold the kbasep_js_device_data::runpool_mutex ++ */ ++static inline void kbase_js_runpool_dec_context_count( ++ struct kbase_device *kbdev, ++ struct kbase_context *kctx) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ ++ js_devdata = &kbdev->js_data; ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); ++ lockdep_assert_held(&js_devdata->runpool_mutex); ++ ++ /* Track total contexts */ ++ --(js_devdata->nr_all_contexts_running); ++ KBASE_DEBUG_ASSERT(js_devdata->nr_all_contexts_running >= 0); ++ ++ if (!kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) { ++ /* Track contexts that can submit jobs */ ++ --(js_devdata->nr_user_contexts_running); ++ KBASE_DEBUG_ASSERT(js_devdata->nr_user_contexts_running >= 0); ++ } ++} ++ ++ ++/** ++ * @brief Submit atoms from all available contexts to all job slots. ++ * ++ * This will attempt to submit as many jobs as possible. It will exit when ++ * either all job slots are full, or all contexts have been used. ++ * ++ * @param[in] kbdev Device pointer ++ */ ++static inline void kbase_js_sched_all(struct kbase_device *kbdev) ++{ ++ kbase_js_sched(kbdev, (1 << kbdev->gpu_props.num_job_slots) - 1); ++} ++ ++extern const int ++kbasep_js_atom_priority_to_relative[BASE_JD_NR_PRIO_LEVELS]; ++ ++extern const base_jd_prio ++kbasep_js_relative_priority_to_atom[KBASE_JS_ATOM_SCHED_PRIO_COUNT]; ++ ++/** ++ * kbasep_js_atom_prio_to_sched_prio(): - Convert atom priority (base_jd_prio) ++ * to relative ordering ++ * @atom_prio: Priority ID to translate. ++ * ++ * Atom priority values for @ref base_jd_prio cannot be compared directly to ++ * find out which are higher or lower. ++ * ++ * This function will convert base_jd_prio values for successively lower ++ * priorities into a monotonically increasing sequence. That is, the lower the ++ * base_jd_prio priority, the higher the value produced by this function. This ++ * is in accordance with how the rest of the kernel treates priority. ++ * ++ * The mapping is 1:1 and the size of the valid input range is the same as the ++ * size of the valid output range, i.e. ++ * KBASE_JS_ATOM_SCHED_PRIO_COUNT == BASE_JD_NR_PRIO_LEVELS ++ * ++ * Note This must be kept in sync with BASE_JD_PRIO_<...> definitions ++ * ++ * Return: On success: a value in the inclusive range ++ * 0..KBASE_JS_ATOM_SCHED_PRIO_COUNT-1. On failure: ++ * KBASE_JS_ATOM_SCHED_PRIO_INVALID ++ */ ++static inline int kbasep_js_atom_prio_to_sched_prio(base_jd_prio atom_prio) ++{ ++ if (atom_prio >= BASE_JD_NR_PRIO_LEVELS) ++ return KBASE_JS_ATOM_SCHED_PRIO_INVALID; ++ ++ return kbasep_js_atom_priority_to_relative[atom_prio]; ++} ++ ++static inline base_jd_prio kbasep_js_sched_prio_to_atom_prio(int sched_prio) ++{ ++ unsigned int prio_idx; ++ ++ KBASE_DEBUG_ASSERT(0 <= sched_prio ++ && sched_prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT); ++ ++ prio_idx = (unsigned int)sched_prio; ++ ++ return kbasep_js_relative_priority_to_atom[prio_idx]; ++} ++ ++ /** @} *//* end group kbase_js */ ++ /** @} *//* end group base_kbase_api */ ++ /** @} *//* end group base_api */ ++ ++#endif /* _KBASE_JS_H_ */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_js_ctx_attr.c b/drivers/gpu/arm/midgard/mali_kbase_js_ctx_attr.c +new file mode 100755 +index 000000000000..321506ada835 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_js_ctx_attr.c +@@ -0,0 +1,301 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++#include ++#include ++ ++/* ++ * Private functions follow ++ */ ++ ++/** ++ * @brief Check whether a ctx has a certain attribute, and if so, retain that ++ * attribute on the runpool. ++ * ++ * Requires: ++ * - jsctx mutex ++ * - runpool_irq spinlock ++ * - ctx is scheduled on the runpool ++ * ++ * @return true indicates a change in ctx attributes state of the runpool. ++ * In this state, the scheduler might be able to submit more jobs than ++ * previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock() ++ * or similar is called sometime later. ++ * @return false indicates no change in ctx attributes state of the runpool. ++ */ ++static bool kbasep_js_ctx_attr_runpool_retain_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ bool runpool_state_changed = false; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); ++ js_devdata = &kbdev->js_data; ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ ++ if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, attribute) != false) { ++ KBASE_DEBUG_ASSERT(js_devdata->runpool_irq.ctx_attr_ref_count[attribute] < S8_MAX); ++ ++(js_devdata->runpool_irq.ctx_attr_ref_count[attribute]); ++ ++ if (js_devdata->runpool_irq.ctx_attr_ref_count[attribute] == 1) { ++ /* First refcount indicates a state change */ ++ runpool_state_changed = true; ++ KBASE_TRACE_ADD(kbdev, JS_CTX_ATTR_NOW_ON_RUNPOOL, kctx, NULL, 0u, attribute); ++ } ++ } ++ ++ return runpool_state_changed; ++} ++ ++/** ++ * @brief Check whether a ctx has a certain attribute, and if so, release that ++ * attribute on the runpool. ++ * ++ * Requires: ++ * - jsctx mutex ++ * - runpool_irq spinlock ++ * - ctx is scheduled on the runpool ++ * ++ * @return true indicates a change in ctx attributes state of the runpool. ++ * In this state, the scheduler might be able to submit more jobs than ++ * previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock() ++ * or similar is called sometime later. ++ * @return false indicates no change in ctx attributes state of the runpool. ++ */ ++static bool kbasep_js_ctx_attr_runpool_release_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ struct kbasep_js_kctx_info *js_kctx_info; ++ bool runpool_state_changed = false; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); ++ js_devdata = &kbdev->js_data; ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); ++ ++ if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, attribute) != false) { ++ KBASE_DEBUG_ASSERT(js_devdata->runpool_irq.ctx_attr_ref_count[attribute] > 0); ++ --(js_devdata->runpool_irq.ctx_attr_ref_count[attribute]); ++ ++ if (js_devdata->runpool_irq.ctx_attr_ref_count[attribute] == 0) { ++ /* Last de-refcount indicates a state change */ ++ runpool_state_changed = true; ++ KBASE_TRACE_ADD(kbdev, JS_CTX_ATTR_NOW_OFF_RUNPOOL, kctx, NULL, 0u, attribute); ++ } ++ } ++ ++ return runpool_state_changed; ++} ++ ++/** ++ * @brief Retain a certain attribute on a ctx, also retaining it on the runpool ++ * if the context is scheduled. ++ * ++ * Requires: ++ * - jsctx mutex ++ * - If the context is scheduled, then runpool_irq spinlock must also be held ++ * ++ * @return true indicates a change in ctx attributes state of the runpool. ++ * This may allow the scheduler to submit more jobs than previously. ++ * @return false indicates no change in ctx attributes state of the runpool. ++ */ ++static bool kbasep_js_ctx_attr_ctx_retain_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute) ++{ ++ struct kbasep_js_kctx_info *js_kctx_info; ++ bool runpool_state_changed = false; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); ++ KBASE_DEBUG_ASSERT(js_kctx_info->ctx.ctx_attr_ref_count[attribute] < U32_MAX); ++ ++ ++(js_kctx_info->ctx.ctx_attr_ref_count[attribute]); ++ ++ if (kbase_ctx_flag(kctx, KCTX_SCHEDULED) && js_kctx_info->ctx.ctx_attr_ref_count[attribute] == 1) { ++ /* Only ref-count the attribute on the runpool for the first time this contexts sees this attribute */ ++ KBASE_TRACE_ADD(kbdev, JS_CTX_ATTR_NOW_ON_CTX, kctx, NULL, 0u, attribute); ++ runpool_state_changed = kbasep_js_ctx_attr_runpool_retain_attr(kbdev, kctx, attribute); ++ } ++ ++ return runpool_state_changed; ++} ++ ++/* ++ * @brief Release a certain attribute on a ctx, also releasing it from the runpool ++ * if the context is scheduled. ++ * ++ * Requires: ++ * - jsctx mutex ++ * - If the context is scheduled, then runpool_irq spinlock must also be held ++ * ++ * @return true indicates a change in ctx attributes state of the runpool. ++ * This may allow the scheduler to submit more jobs than previously. ++ * @return false indicates no change in ctx attributes state of the runpool. ++ */ ++static bool kbasep_js_ctx_attr_ctx_release_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute) ++{ ++ struct kbasep_js_kctx_info *js_kctx_info; ++ bool runpool_state_changed = false; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); ++ KBASE_DEBUG_ASSERT(js_kctx_info->ctx.ctx_attr_ref_count[attribute] > 0); ++ ++ if (kbase_ctx_flag(kctx, KCTX_SCHEDULED) && js_kctx_info->ctx.ctx_attr_ref_count[attribute] == 1) { ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ /* Only de-ref-count the attribute on the runpool when this is the last ctx-reference to it */ ++ runpool_state_changed = kbasep_js_ctx_attr_runpool_release_attr(kbdev, kctx, attribute); ++ KBASE_TRACE_ADD(kbdev, JS_CTX_ATTR_NOW_OFF_CTX, kctx, NULL, 0u, attribute); ++ } ++ ++ /* De-ref must happen afterwards, because kbasep_js_ctx_attr_runpool_release() needs to check it too */ ++ --(js_kctx_info->ctx.ctx_attr_ref_count[attribute]); ++ ++ return runpool_state_changed; ++} ++ ++/* ++ * More commonly used public functions ++ */ ++ ++void kbasep_js_ctx_attr_set_initial_attrs(struct kbase_device *kbdev, struct kbase_context *kctx) ++{ ++ bool runpool_state_changed = false; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ ++ if (kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) { ++ /* This context never submits, so don't track any scheduling attributes */ ++ return; ++ } ++ ++ /* Transfer attributes held in the context flags for contexts that have submit enabled */ ++ ++ /* ... More attributes can be added here ... */ ++ ++ /* The context should not have been scheduled yet, so ASSERT if this caused ++ * runpool state changes (note that other threads *can't* affect the value ++ * of runpool_state_changed, due to how it's calculated) */ ++ KBASE_DEBUG_ASSERT(runpool_state_changed == false); ++ CSTD_UNUSED(runpool_state_changed); ++} ++ ++void kbasep_js_ctx_attr_runpool_retain_ctx(struct kbase_device *kbdev, struct kbase_context *kctx) ++{ ++ bool runpool_state_changed; ++ int i; ++ ++ /* Retain any existing attributes */ ++ for (i = 0; i < KBASEP_JS_CTX_ATTR_COUNT; ++i) { ++ if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, (enum kbasep_js_ctx_attr) i) != false) { ++ /* The context is being scheduled in, so update the runpool with the new attributes */ ++ runpool_state_changed = kbasep_js_ctx_attr_runpool_retain_attr(kbdev, kctx, (enum kbasep_js_ctx_attr) i); ++ ++ /* We don't need to know about state changed, because retaining a ++ * context occurs on scheduling it, and that itself will also try ++ * to run new atoms */ ++ CSTD_UNUSED(runpool_state_changed); ++ } ++ } ++} ++ ++bool kbasep_js_ctx_attr_runpool_release_ctx(struct kbase_device *kbdev, struct kbase_context *kctx) ++{ ++ bool runpool_state_changed = false; ++ int i; ++ ++ /* Release any existing attributes */ ++ for (i = 0; i < KBASEP_JS_CTX_ATTR_COUNT; ++i) { ++ if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, (enum kbasep_js_ctx_attr) i) != false) { ++ /* The context is being scheduled out, so update the runpool on the removed attributes */ ++ runpool_state_changed |= kbasep_js_ctx_attr_runpool_release_attr(kbdev, kctx, (enum kbasep_js_ctx_attr) i); ++ } ++ } ++ ++ return runpool_state_changed; ++} ++ ++void kbasep_js_ctx_attr_ctx_retain_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom) ++{ ++ bool runpool_state_changed = false; ++ base_jd_core_req core_req; ++ ++ KBASE_DEBUG_ASSERT(katom); ++ core_req = katom->core_req; ++ ++ if (core_req & BASE_JD_REQ_ONLY_COMPUTE) ++ runpool_state_changed |= kbasep_js_ctx_attr_ctx_retain_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE); ++ else ++ runpool_state_changed |= kbasep_js_ctx_attr_ctx_retain_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_NON_COMPUTE); ++ ++ if ((core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE | BASE_JD_REQ_T)) != 0 && (core_req & (BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_SPECIFIC_COHERENT_GROUP)) == 0) { ++ /* Atom that can run on slot1 or slot2, and can use all cores */ ++ runpool_state_changed |= kbasep_js_ctx_attr_ctx_retain_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES); ++ } ++ ++ /* We don't need to know about state changed, because retaining an ++ * atom occurs on adding it, and that itself will also try to run ++ * new atoms */ ++ CSTD_UNUSED(runpool_state_changed); ++} ++ ++bool kbasep_js_ctx_attr_ctx_release_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbasep_js_atom_retained_state *katom_retained_state) ++{ ++ bool runpool_state_changed = false; ++ base_jd_core_req core_req; ++ ++ KBASE_DEBUG_ASSERT(katom_retained_state); ++ core_req = katom_retained_state->core_req; ++ ++ /* No-op for invalid atoms */ ++ if (kbasep_js_atom_retained_state_is_valid(katom_retained_state) == false) ++ return false; ++ ++ if (core_req & BASE_JD_REQ_ONLY_COMPUTE) ++ runpool_state_changed |= kbasep_js_ctx_attr_ctx_release_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE); ++ else ++ runpool_state_changed |= kbasep_js_ctx_attr_ctx_release_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_NON_COMPUTE); ++ ++ if ((core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE | BASE_JD_REQ_T)) != 0 && (core_req & (BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_SPECIFIC_COHERENT_GROUP)) == 0) { ++ /* Atom that can run on slot1 or slot2, and can use all cores */ ++ runpool_state_changed |= kbasep_js_ctx_attr_ctx_release_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES); ++ } ++ ++ return runpool_state_changed; ++} +diff --git a/drivers/gpu/arm/midgard/mali_kbase_js_ctx_attr.h b/drivers/gpu/arm/midgard/mali_kbase_js_ctx_attr.h +new file mode 100755 +index 000000000000..ce9183326a57 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_js_ctx_attr.h +@@ -0,0 +1,158 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/** ++ * @file mali_kbase_js_ctx_attr.h ++ * Job Scheduler Context Attribute APIs ++ */ ++ ++#ifndef _KBASE_JS_CTX_ATTR_H_ ++#define _KBASE_JS_CTX_ATTR_H_ ++ ++/** ++ * @addtogroup base_api ++ * @{ ++ */ ++ ++/** ++ * @addtogroup base_kbase_api ++ * @{ ++ */ ++ ++/** ++ * @addtogroup kbase_js ++ * @{ ++ */ ++ ++/** ++ * Set the initial attributes of a context (when context create flags are set) ++ * ++ * Requires: ++ * - Hold the jsctx_mutex ++ */ ++void kbasep_js_ctx_attr_set_initial_attrs(struct kbase_device *kbdev, struct kbase_context *kctx); ++ ++/** ++ * Retain all attributes of a context ++ * ++ * This occurs on scheduling in the context on the runpool (but after ++ * is_scheduled is set) ++ * ++ * Requires: ++ * - jsctx mutex ++ * - runpool_irq spinlock ++ * - ctx->is_scheduled is true ++ */ ++void kbasep_js_ctx_attr_runpool_retain_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); ++ ++/** ++ * Release all attributes of a context ++ * ++ * This occurs on scheduling out the context from the runpool (but before ++ * is_scheduled is cleared) ++ * ++ * Requires: ++ * - jsctx mutex ++ * - runpool_irq spinlock ++ * - ctx->is_scheduled is true ++ * ++ * @return true indicates a change in ctx attributes state of the runpool. ++ * In this state, the scheduler might be able to submit more jobs than ++ * previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock() ++ * or similar is called sometime later. ++ * @return false indicates no change in ctx attributes state of the runpool. ++ */ ++bool kbasep_js_ctx_attr_runpool_release_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); ++ ++/** ++ * Retain all attributes of an atom ++ * ++ * This occurs on adding an atom to a context ++ * ++ * Requires: ++ * - jsctx mutex ++ * - If the context is scheduled, then runpool_irq spinlock must also be held ++ */ ++void kbasep_js_ctx_attr_ctx_retain_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom); ++ ++/** ++ * Release all attributes of an atom, given its retained state. ++ * ++ * This occurs after (permanently) removing an atom from a context ++ * ++ * Requires: ++ * - jsctx mutex ++ * - If the context is scheduled, then runpool_irq spinlock must also be held ++ * ++ * This is a no-op when \a katom_retained_state is invalid. ++ * ++ * @return true indicates a change in ctx attributes state of the runpool. ++ * In this state, the scheduler might be able to submit more jobs than ++ * previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock() ++ * or similar is called sometime later. ++ * @return false indicates no change in ctx attributes state of the runpool. ++ */ ++bool kbasep_js_ctx_attr_ctx_release_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbasep_js_atom_retained_state *katom_retained_state); ++ ++/** ++ * Requires: ++ * - runpool_irq spinlock ++ */ ++static inline s8 kbasep_js_ctx_attr_count_on_runpool(struct kbase_device *kbdev, enum kbasep_js_ctx_attr attribute) ++{ ++ struct kbasep_js_device_data *js_devdata; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); ++ js_devdata = &kbdev->js_data; ++ ++ return js_devdata->runpool_irq.ctx_attr_ref_count[attribute]; ++} ++ ++/** ++ * Requires: ++ * - runpool_irq spinlock ++ */ ++static inline bool kbasep_js_ctx_attr_is_attr_on_runpool(struct kbase_device *kbdev, enum kbasep_js_ctx_attr attribute) ++{ ++ /* In general, attributes are 'on' when they have a non-zero refcount (note: the refcount will never be < 0) */ ++ return (bool) kbasep_js_ctx_attr_count_on_runpool(kbdev, attribute); ++} ++ ++/** ++ * Requires: ++ * - jsctx mutex ++ */ ++static inline bool kbasep_js_ctx_attr_is_attr_on_ctx(struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute) ++{ ++ struct kbasep_js_kctx_info *js_kctx_info; ++ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); ++ js_kctx_info = &kctx->jctx.sched_info; ++ ++ /* In general, attributes are 'on' when they have a refcount (which should never be < 0) */ ++ return (bool) (js_kctx_info->ctx.ctx_attr_ref_count[attribute]); ++} ++ ++ /** @} *//* end group kbase_js */ ++ /** @} *//* end group base_kbase_api */ ++ /** @} *//* end group base_api */ ++ ++#endif /* _KBASE_JS_DEFS_H_ */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_js_defs.h b/drivers/gpu/arm/midgard/mali_kbase_js_defs.h +new file mode 100755 +index 000000000000..ba8b6441549b +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_js_defs.h +@@ -0,0 +1,386 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/** ++ * @file mali_kbase_js.h ++ * Job Scheduler Type Definitions ++ */ ++ ++#ifndef _KBASE_JS_DEFS_H_ ++#define _KBASE_JS_DEFS_H_ ++ ++/** ++ * @addtogroup base_api ++ * @{ ++ */ ++ ++/** ++ * @addtogroup base_kbase_api ++ * @{ ++ */ ++ ++/** ++ * @addtogroup kbase_js ++ * @{ ++ */ ++/* Forward decls */ ++struct kbase_device; ++struct kbase_jd_atom; ++ ++ ++typedef u32 kbase_context_flags; ++ ++struct kbasep_atom_req { ++ base_jd_core_req core_req; ++ kbase_context_flags ctx_req; ++ u32 device_nr; ++}; ++ ++/** Callback function run on all of a context's jobs registered with the Job ++ * Scheduler */ ++typedef void (*kbasep_js_ctx_job_cb)(struct kbase_device *kbdev, struct kbase_jd_atom *katom); ++ ++/** ++ * @brief Maximum number of jobs that can be submitted to a job slot whilst ++ * inside the IRQ handler. ++ * ++ * This is important because GPU NULL jobs can complete whilst the IRQ handler ++ * is running. Otherwise, it potentially allows an unlimited number of GPU NULL ++ * jobs to be submitted inside the IRQ handler, which increases IRQ latency. ++ */ ++#define KBASE_JS_MAX_JOB_SUBMIT_PER_SLOT_PER_IRQ 2 ++ ++/** ++ * @brief Context attributes ++ * ++ * Each context attribute can be thought of as a boolean value that caches some ++ * state information about either the runpool, or the context: ++ * - In the case of the runpool, it is a cache of "Do any contexts owned by ++ * the runpool have attribute X?" ++ * - In the case of a context, it is a cache of "Do any atoms owned by the ++ * context have attribute X?" ++ * ++ * The boolean value of the context attributes often affect scheduling ++ * decisions, such as affinities to use and job slots to use. ++ * ++ * To accomodate changes of state in the context, each attribute is refcounted ++ * in the context, and in the runpool for all running contexts. Specifically: ++ * - The runpool holds a refcount of how many contexts in the runpool have this ++ * attribute. ++ * - The context holds a refcount of how many atoms have this attribute. ++ */ ++enum kbasep_js_ctx_attr { ++ /** Attribute indicating a context that contains Compute jobs. That is, ++ * the context has jobs of type @ref BASE_JD_REQ_ONLY_COMPUTE ++ * ++ * @note A context can be both 'Compute' and 'Non Compute' if it contains ++ * both types of jobs. ++ */ ++ KBASEP_JS_CTX_ATTR_COMPUTE, ++ ++ /** Attribute indicating a context that contains Non-Compute jobs. That is, ++ * the context has some jobs that are \b not of type @ref ++ * BASE_JD_REQ_ONLY_COMPUTE. ++ * ++ * @note A context can be both 'Compute' and 'Non Compute' if it contains ++ * both types of jobs. ++ */ ++ KBASEP_JS_CTX_ATTR_NON_COMPUTE, ++ ++ /** Attribute indicating that a context contains compute-job atoms that ++ * aren't restricted to a coherent group, and can run on all cores. ++ * ++ * Specifically, this is when the atom's \a core_req satisfy: ++ * - (\a core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE | BASE_JD_REQ_T) // uses slot 1 or slot 2 ++ * - && !(\a core_req & BASE_JD_REQ_COHERENT_GROUP) // not restricted to coherent groups ++ * ++ * Such atoms could be blocked from running if one of the coherent groups ++ * is being used by another job slot, so tracking this context attribute ++ * allows us to prevent such situations. ++ * ++ * @note This doesn't take into account the 1-coregroup case, where all ++ * compute atoms would effectively be able to run on 'all cores', but ++ * contexts will still not always get marked with this attribute. Instead, ++ * it is the caller's responsibility to take into account the number of ++ * coregroups when interpreting this attribute. ++ * ++ * @note Whilst Tiler atoms are normally combined with ++ * BASE_JD_REQ_COHERENT_GROUP, it is possible to send such atoms without ++ * BASE_JD_REQ_COHERENT_GROUP set. This is an unlikely case, but it's easy ++ * enough to handle anyway. ++ */ ++ KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES, ++ ++ /** Must be the last in the enum */ ++ KBASEP_JS_CTX_ATTR_COUNT ++}; ++ ++enum { ++ /** Bit indicating that new atom should be started because this atom completed */ ++ KBASE_JS_ATOM_DONE_START_NEW_ATOMS = (1u << 0), ++ /** Bit indicating that the atom was evicted from the JS_NEXT registers */ ++ KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT = (1u << 1) ++}; ++ ++/** Combination of KBASE_JS_ATOM_DONE_<...> bits */ ++typedef u32 kbasep_js_atom_done_code; ++ ++/** ++ * @brief KBase Device Data Job Scheduler sub-structure ++ * ++ * This encapsulates the current context of the Job Scheduler on a particular ++ * device. This context is global to the device, and is not tied to any ++ * particular struct kbase_context running on the device. ++ * ++ * nr_contexts_running and as_free are optimized for packing together (by making ++ * them smaller types than u32). The operations on them should rarely involve ++ * masking. The use of signed types for arithmetic indicates to the compiler that ++ * the value will not rollover (which would be undefined behavior), and so under ++ * the Total License model, it is free to make optimizations based on that (i.e. ++ * to remove masking). ++ */ ++struct kbasep_js_device_data { ++ /* Sub-structure to collect together Job Scheduling data used in IRQ ++ * context. The hwaccess_lock must be held when accessing. */ ++ struct runpool_irq { ++ /** Bitvector indicating whether a currently scheduled context is allowed to submit jobs. ++ * When bit 'N' is set in this, it indicates whether the context bound to address space ++ * 'N' is allowed to submit jobs. ++ */ ++ u16 submit_allowed; ++ ++ /** Context Attributes: ++ * Each is large enough to hold a refcount of the number of contexts ++ * that can fit into the runpool. This is currently BASE_MAX_NR_AS ++ * ++ * Note that when BASE_MAX_NR_AS==16 we need 5 bits (not 4) to store ++ * the refcount. Hence, it's not worthwhile reducing this to ++ * bit-manipulation on u32s to save space (where in contrast, 4 bit ++ * sub-fields would be easy to do and would save space). ++ * ++ * Whilst this must not become negative, the sign bit is used for: ++ * - error detection in debug builds ++ * - Optimization: it is undefined for a signed int to overflow, and so ++ * the compiler can optimize for that never happening (thus, no masking ++ * is required on updating the variable) */ ++ s8 ctx_attr_ref_count[KBASEP_JS_CTX_ATTR_COUNT]; ++ ++ /* ++ * Affinity management and tracking ++ */ ++ /** Bitvector to aid affinity checking. Element 'n' bit 'i' indicates ++ * that slot 'n' is using core i (i.e. slot_affinity_refcount[n][i] > 0) */ ++ u64 slot_affinities[BASE_JM_MAX_NR_SLOTS]; ++ /** Refcount for each core owned by each slot. Used to generate the ++ * slot_affinities array of bitvectors ++ * ++ * The value of the refcount will not exceed BASE_JM_SUBMIT_SLOTS, ++ * because it is refcounted only when a job is definitely about to be ++ * submitted to a slot, and is de-refcounted immediately after a job ++ * finishes */ ++ s8 slot_affinity_refcount[BASE_JM_MAX_NR_SLOTS][64]; ++ } runpool_irq; ++ ++ /** ++ * Run Pool mutex, for managing contexts within the runpool. ++ * Unless otherwise specified, you must hold this lock whilst accessing any ++ * members that follow ++ * ++ * In addition, this is used to access: ++ * - the kbasep_js_kctx_info::runpool substructure ++ */ ++ struct mutex runpool_mutex; ++ ++ /** ++ * Queue Lock, used to access the Policy's queue of contexts independently ++ * of the Run Pool. ++ * ++ * Of course, you don't need the Run Pool lock to access this. ++ */ ++ struct mutex queue_mutex; ++ ++ /** ++ * Scheduling semaphore. This must be held when calling ++ * kbase_jm_kick() ++ */ ++ struct semaphore schedule_sem; ++ ++ /** ++ * List of contexts that can currently be pulled from ++ */ ++ struct list_head ctx_list_pullable[BASE_JM_MAX_NR_SLOTS]; ++ /** ++ * List of contexts that can not currently be pulled from, but have ++ * jobs currently running. ++ */ ++ struct list_head ctx_list_unpullable[BASE_JM_MAX_NR_SLOTS]; ++ ++ /** Number of currently scheduled user contexts (excluding ones that are not submitting jobs) */ ++ s8 nr_user_contexts_running; ++ /** Number of currently scheduled contexts (including ones that are not submitting jobs) */ ++ s8 nr_all_contexts_running; ++ ++ /** Core Requirements to match up with base_js_atom's core_req memeber ++ * @note This is a write-once member, and so no locking is required to read */ ++ base_jd_core_req js_reqs[BASE_JM_MAX_NR_SLOTS]; ++ ++ u32 scheduling_period_ns; /*< Value for JS_SCHEDULING_PERIOD_NS */ ++ u32 soft_stop_ticks; /*< Value for JS_SOFT_STOP_TICKS */ ++ u32 soft_stop_ticks_cl; /*< Value for JS_SOFT_STOP_TICKS_CL */ ++ u32 hard_stop_ticks_ss; /*< Value for JS_HARD_STOP_TICKS_SS */ ++ u32 hard_stop_ticks_cl; /*< Value for JS_HARD_STOP_TICKS_CL */ ++ u32 hard_stop_ticks_dumping; /*< Value for JS_HARD_STOP_TICKS_DUMPING */ ++ u32 gpu_reset_ticks_ss; /*< Value for JS_RESET_TICKS_SS */ ++ u32 gpu_reset_ticks_cl; /*< Value for JS_RESET_TICKS_CL */ ++ u32 gpu_reset_ticks_dumping; /*< Value for JS_RESET_TICKS_DUMPING */ ++ u32 ctx_timeslice_ns; /**< Value for JS_CTX_TIMESLICE_NS */ ++ ++ /**< Value for JS_SOFT_JOB_TIMEOUT */ ++ atomic_t soft_job_timeout_ms; ++ ++ /** List of suspended soft jobs */ ++ struct list_head suspended_soft_jobs_list; ++ ++#ifdef CONFIG_MALI_DEBUG ++ /* Support soft-stop on a single context */ ++ bool softstop_always; ++#endif /* CONFIG_MALI_DEBUG */ ++ ++ /** The initalized-flag is placed at the end, to avoid cache-pollution (we should ++ * only be using this during init/term paths). ++ * @note This is a write-once member, and so no locking is required to read */ ++ int init_status; ++ ++ /* Number of contexts that can currently be pulled from */ ++ u32 nr_contexts_pullable; ++ ++ /* Number of contexts that can either be pulled from or are currently ++ * running */ ++ atomic_t nr_contexts_runnable; ++}; ++ ++/** ++ * @brief KBase Context Job Scheduling information structure ++ * ++ * This is a substructure in the struct kbase_context that encapsulates all the ++ * scheduling information. ++ */ ++struct kbasep_js_kctx_info { ++ ++ /** ++ * Job Scheduler Context information sub-structure. These members are ++ * accessed regardless of whether the context is: ++ * - In the Policy's Run Pool ++ * - In the Policy's Queue ++ * - Not queued nor in the Run Pool. ++ * ++ * You must obtain the jsctx_mutex before accessing any other members of ++ * this substructure. ++ * ++ * You may not access any of these members from IRQ context. ++ */ ++ struct kbase_jsctx { ++ struct mutex jsctx_mutex; /**< Job Scheduler Context lock */ ++ ++ /** Number of jobs ready to run - does \em not include the jobs waiting in ++ * the dispatcher, and dependency-only jobs. See kbase_jd_context::job_nr ++ * for such jobs*/ ++ u32 nr_jobs; ++ ++ /** Context Attributes: ++ * Each is large enough to hold a refcount of the number of atoms on ++ * the context. **/ ++ u32 ctx_attr_ref_count[KBASEP_JS_CTX_ATTR_COUNT]; ++ ++ /** ++ * Wait queue to wait for KCTX_SHEDULED flag state changes. ++ * */ ++ wait_queue_head_t is_scheduled_wait; ++ ++ /** Link implementing JS queues. Context can be present on one ++ * list per job slot ++ */ ++ struct list_head ctx_list_entry[BASE_JM_MAX_NR_SLOTS]; ++ } ctx; ++ ++ /* The initalized-flag is placed at the end, to avoid cache-pollution (we should ++ * only be using this during init/term paths) */ ++ int init_status; ++}; ++ ++/** Subset of atom state that can be available after jd_done_nolock() is called ++ * on that atom. A copy must be taken via kbasep_js_atom_retained_state_copy(), ++ * because the original atom could disappear. */ ++struct kbasep_js_atom_retained_state { ++ /** Event code - to determine whether the atom has finished */ ++ enum base_jd_event_code event_code; ++ /** core requirements */ ++ base_jd_core_req core_req; ++ /* priority */ ++ int sched_priority; ++ /** Job Slot to retry submitting to if submission from IRQ handler failed */ ++ int retry_submit_on_slot; ++ /* Core group atom was executed on */ ++ u32 device_nr; ++ ++}; ++ ++/** ++ * Value signifying 'no retry on a slot required' for: ++ * - kbase_js_atom_retained_state::retry_submit_on_slot ++ * - kbase_jd_atom::retry_submit_on_slot ++ */ ++#define KBASEP_JS_RETRY_SUBMIT_SLOT_INVALID (-1) ++ ++/** ++ * base_jd_core_req value signifying 'invalid' for a kbase_jd_atom_retained_state. ++ * ++ * @see kbase_atom_retained_state_is_valid() ++ */ ++#define KBASEP_JS_ATOM_RETAINED_STATE_CORE_REQ_INVALID BASE_JD_REQ_DEP ++ ++/** ++ * @brief The JS timer resolution, in microseconds ++ * ++ * Any non-zero difference in time will be at least this size. ++ */ ++#define KBASEP_JS_TICK_RESOLUTION_US 1 ++ ++/* ++ * Internal atom priority defines for kbase_jd_atom::sched_prio ++ */ ++enum { ++ KBASE_JS_ATOM_SCHED_PRIO_HIGH = 0, ++ KBASE_JS_ATOM_SCHED_PRIO_MED, ++ KBASE_JS_ATOM_SCHED_PRIO_LOW, ++ KBASE_JS_ATOM_SCHED_PRIO_COUNT, ++}; ++ ++/* Invalid priority for kbase_jd_atom::sched_prio */ ++#define KBASE_JS_ATOM_SCHED_PRIO_INVALID -1 ++ ++/* Default priority in the case of contexts with no atoms, or being lenient ++ * about invalid priorities from userspace */ ++#define KBASE_JS_ATOM_SCHED_PRIO_DEFAULT KBASE_JS_ATOM_SCHED_PRIO_MED ++ ++ /** @} *//* end group kbase_js */ ++ /** @} *//* end group base_kbase_api */ ++ /** @} *//* end group base_api */ ++ ++#endif /* _KBASE_JS_DEFS_H_ */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_linux.h b/drivers/gpu/arm/midgard/mali_kbase_linux.h +new file mode 100755 +index 000000000000..6d1e61fd41e0 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_linux.h +@@ -0,0 +1,43 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2014 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/** ++ * @file mali_kbase_linux.h ++ * Base kernel APIs, Linux implementation. ++ */ ++ ++#ifndef _KBASE_LINUX_H_ ++#define _KBASE_LINUX_H_ ++ ++/* All things that are needed for the Linux port. */ ++#include ++#include ++#include ++#include ++#include ++ ++#if (defined(MALI_KERNEL_TEST_API) && (1 == MALI_KERNEL_TEST_API)) ++ #define KBASE_EXPORT_TEST_API(func) EXPORT_SYMBOL(func) ++#else ++ #define KBASE_EXPORT_TEST_API(func) ++#endif ++ ++#define KBASE_EXPORT_SYMBOL(func) EXPORT_SYMBOL(func) ++ ++#endif /* _KBASE_LINUX_H_ */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_mem.c b/drivers/gpu/arm/midgard/mali_kbase_mem.c +new file mode 100755 +index 000000000000..8cbb5f6765b8 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_mem.c +@@ -0,0 +1,2657 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/** ++ * @file mali_kbase_mem.c ++ * Base kernel memory APIs ++ */ ++#ifdef CONFIG_DMA_SHARED_BUFFER ++#include ++#endif /* CONFIG_DMA_SHARED_BUFFER */ ++#ifdef CONFIG_UMP ++#include ++#endif /* CONFIG_UMP */ ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++/* This function finds out which RB tree the given GPU VA region belongs to ++ * based on the region zone */ ++static struct rb_root *kbase_reg_flags_to_rbtree(struct kbase_context *kctx, ++ struct kbase_va_region *reg) ++{ ++ struct rb_root *rbtree = NULL; ++ ++ switch (reg->flags & KBASE_REG_ZONE_MASK) { ++ case KBASE_REG_ZONE_CUSTOM_VA: ++ rbtree = &kctx->reg_rbtree_custom; ++ break; ++ case KBASE_REG_ZONE_EXEC: ++ rbtree = &kctx->reg_rbtree_exec; ++ break; ++ case KBASE_REG_ZONE_SAME_VA: ++ rbtree = &kctx->reg_rbtree_same; ++ /* fall through */ ++ default: ++ rbtree = &kctx->reg_rbtree_same; ++ break; ++ } ++ ++ return rbtree; ++} ++ ++/* This function finds out which RB tree the given pfn from the GPU VA belongs ++ * to based on the memory zone the pfn refers to */ ++static struct rb_root *kbase_gpu_va_to_rbtree(struct kbase_context *kctx, ++ u64 gpu_pfn) ++{ ++ struct rb_root *rbtree = NULL; ++ ++#ifdef CONFIG_64BIT ++ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) { ++#endif /* CONFIG_64BIT */ ++ if (gpu_pfn >= KBASE_REG_ZONE_CUSTOM_VA_BASE) ++ rbtree = &kctx->reg_rbtree_custom; ++ else if (gpu_pfn >= KBASE_REG_ZONE_EXEC_BASE) ++ rbtree = &kctx->reg_rbtree_exec; ++ else ++ rbtree = &kctx->reg_rbtree_same; ++#ifdef CONFIG_64BIT ++ } else { ++ if (gpu_pfn >= kctx->same_va_end) ++ rbtree = &kctx->reg_rbtree_custom; ++ else ++ rbtree = &kctx->reg_rbtree_same; ++ } ++#endif /* CONFIG_64BIT */ ++ ++ return rbtree; ++} ++ ++/* This function inserts a region into the tree. */ ++static void kbase_region_tracker_insert(struct kbase_context *kctx, ++ struct kbase_va_region *new_reg) ++{ ++ u64 start_pfn = new_reg->start_pfn; ++ struct rb_node **link = NULL; ++ struct rb_node *parent = NULL; ++ struct rb_root *rbtree = NULL; ++ ++ rbtree = kbase_reg_flags_to_rbtree(kctx, new_reg); ++ ++ link = &(rbtree->rb_node); ++ /* Find the right place in the tree using tree search */ ++ while (*link) { ++ struct kbase_va_region *old_reg; ++ ++ parent = *link; ++ old_reg = rb_entry(parent, struct kbase_va_region, rblink); ++ ++ /* RBTree requires no duplicate entries. */ ++ KBASE_DEBUG_ASSERT(old_reg->start_pfn != start_pfn); ++ ++ if (old_reg->start_pfn > start_pfn) ++ link = &(*link)->rb_left; ++ else ++ link = &(*link)->rb_right; ++ } ++ ++ /* Put the new node there, and rebalance tree */ ++ rb_link_node(&(new_reg->rblink), parent, link); ++ ++ rb_insert_color(&(new_reg->rblink), rbtree); ++} ++ ++/* Find allocated region enclosing free range. */ ++static struct kbase_va_region *kbase_region_tracker_find_region_enclosing_range_free( ++ struct kbase_context *kctx, u64 start_pfn, size_t nr_pages) ++{ ++ struct rb_node *rbnode = NULL; ++ struct kbase_va_region *reg = NULL; ++ struct rb_root *rbtree = NULL; ++ ++ u64 end_pfn = start_pfn + nr_pages; ++ ++ rbtree = kbase_gpu_va_to_rbtree(kctx, start_pfn); ++ ++ rbnode = rbtree->rb_node; ++ ++ while (rbnode) { ++ u64 tmp_start_pfn, tmp_end_pfn; ++ ++ reg = rb_entry(rbnode, struct kbase_va_region, rblink); ++ tmp_start_pfn = reg->start_pfn; ++ tmp_end_pfn = reg->start_pfn + reg->nr_pages; ++ ++ /* If start is lower than this, go left. */ ++ if (start_pfn < tmp_start_pfn) ++ rbnode = rbnode->rb_left; ++ /* If end is higher than this, then go right. */ ++ else if (end_pfn > tmp_end_pfn) ++ rbnode = rbnode->rb_right; ++ else /* Enclosing */ ++ return reg; ++ } ++ ++ return NULL; ++} ++ ++/* Find region enclosing given address. */ ++struct kbase_va_region *kbase_region_tracker_find_region_enclosing_address(struct kbase_context *kctx, u64 gpu_addr) ++{ ++ struct rb_node *rbnode; ++ struct kbase_va_region *reg; ++ u64 gpu_pfn = gpu_addr >> PAGE_SHIFT; ++ struct rb_root *rbtree = NULL; ++ ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ rbtree = kbase_gpu_va_to_rbtree(kctx, gpu_pfn); ++ ++ rbnode = rbtree->rb_node; ++ ++ while (rbnode) { ++ u64 tmp_start_pfn, tmp_end_pfn; ++ ++ reg = rb_entry(rbnode, struct kbase_va_region, rblink); ++ tmp_start_pfn = reg->start_pfn; ++ tmp_end_pfn = reg->start_pfn + reg->nr_pages; ++ ++ /* If start is lower than this, go left. */ ++ if (gpu_pfn < tmp_start_pfn) ++ rbnode = rbnode->rb_left; ++ /* If end is higher than this, then go right. */ ++ else if (gpu_pfn >= tmp_end_pfn) ++ rbnode = rbnode->rb_right; ++ else /* Enclosing */ ++ return reg; ++ } ++ ++ return NULL; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_region_tracker_find_region_enclosing_address); ++ ++/* Find region with given base address */ ++struct kbase_va_region *kbase_region_tracker_find_region_base_address(struct kbase_context *kctx, u64 gpu_addr) ++{ ++ u64 gpu_pfn = gpu_addr >> PAGE_SHIFT; ++ struct rb_node *rbnode = NULL; ++ struct kbase_va_region *reg = NULL; ++ struct rb_root *rbtree = NULL; ++ ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ rbtree = kbase_gpu_va_to_rbtree(kctx, gpu_pfn); ++ ++ rbnode = rbtree->rb_node; ++ ++ while (rbnode) { ++ reg = rb_entry(rbnode, struct kbase_va_region, rblink); ++ if (reg->start_pfn > gpu_pfn) ++ rbnode = rbnode->rb_left; ++ else if (reg->start_pfn < gpu_pfn) ++ rbnode = rbnode->rb_right; ++ else ++ return reg; ++ ++ } ++ ++ return NULL; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_region_tracker_find_region_base_address); ++ ++/* Find region meeting given requirements */ ++static struct kbase_va_region *kbase_region_tracker_find_region_meeting_reqs(struct kbase_context *kctx, struct kbase_va_region *reg_reqs, size_t nr_pages, size_t align) ++{ ++ struct rb_node *rbnode = NULL; ++ struct kbase_va_region *reg = NULL; ++ struct rb_root *rbtree = NULL; ++ ++ /* Note that this search is a linear search, as we do not have a target ++ address in mind, so does not benefit from the rbtree search */ ++ ++ rbtree = kbase_reg_flags_to_rbtree(kctx, reg_reqs); ++ ++ rbnode = rb_first(rbtree); ++ ++ while (rbnode) { ++ reg = rb_entry(rbnode, struct kbase_va_region, rblink); ++ if ((reg->nr_pages >= nr_pages) && ++ (reg->flags & KBASE_REG_FREE)) { ++ /* Check alignment */ ++ u64 start_pfn = (reg->start_pfn + align - 1) & ~(align - 1); ++ ++ if ((start_pfn >= reg->start_pfn) && ++ (start_pfn <= (reg->start_pfn + reg->nr_pages - 1)) && ++ ((start_pfn + nr_pages - 1) <= (reg->start_pfn + reg->nr_pages - 1))) ++ return reg; ++ } ++ rbnode = rb_next(rbnode); ++ } ++ ++ return NULL; ++} ++ ++/** ++ * @brief Remove a region object from the global list. ++ * ++ * The region reg is removed, possibly by merging with other free and ++ * compatible adjacent regions. It must be called with the context ++ * region lock held. The associated memory is not released (see ++ * kbase_free_alloced_region). Internal use only. ++ */ ++static int kbase_remove_va_region(struct kbase_context *kctx, struct kbase_va_region *reg) ++{ ++ struct rb_node *rbprev; ++ struct kbase_va_region *prev = NULL; ++ struct rb_node *rbnext; ++ struct kbase_va_region *next = NULL; ++ struct rb_root *reg_rbtree = NULL; ++ ++ int merged_front = 0; ++ int merged_back = 0; ++ int err = 0; ++ ++ reg_rbtree = kbase_reg_flags_to_rbtree(kctx, reg); ++ ++ /* Try to merge with the previous block first */ ++ rbprev = rb_prev(&(reg->rblink)); ++ if (rbprev) { ++ prev = rb_entry(rbprev, struct kbase_va_region, rblink); ++ if (prev->flags & KBASE_REG_FREE) { ++ /* We're compatible with the previous VMA, ++ * merge with it */ ++ WARN_ON((prev->flags & KBASE_REG_ZONE_MASK) != ++ (reg->flags & KBASE_REG_ZONE_MASK)); ++ prev->nr_pages += reg->nr_pages; ++ rb_erase(&(reg->rblink), reg_rbtree); ++ reg = prev; ++ merged_front = 1; ++ } ++ } ++ ++ /* Try to merge with the next block second */ ++ /* Note we do the lookup here as the tree may have been rebalanced. */ ++ rbnext = rb_next(&(reg->rblink)); ++ if (rbnext) { ++ /* We're compatible with the next VMA, merge with it */ ++ next = rb_entry(rbnext, struct kbase_va_region, rblink); ++ if (next->flags & KBASE_REG_FREE) { ++ WARN_ON((next->flags & KBASE_REG_ZONE_MASK) != ++ (reg->flags & KBASE_REG_ZONE_MASK)); ++ next->start_pfn = reg->start_pfn; ++ next->nr_pages += reg->nr_pages; ++ rb_erase(&(reg->rblink), reg_rbtree); ++ merged_back = 1; ++ if (merged_front) { ++ /* We already merged with prev, free it */ ++ kbase_free_alloced_region(reg); ++ } ++ } ++ } ++ ++ /* If we failed to merge then we need to add a new block */ ++ if (!(merged_front || merged_back)) { ++ /* ++ * We didn't merge anything. Add a new free ++ * placeholder and remove the original one. ++ */ ++ struct kbase_va_region *free_reg; ++ ++ free_reg = kbase_alloc_free_region(kctx, reg->start_pfn, reg->nr_pages, reg->flags & KBASE_REG_ZONE_MASK); ++ if (!free_reg) { ++ err = -ENOMEM; ++ goto out; ++ } ++ rb_replace_node(&(reg->rblink), &(free_reg->rblink), reg_rbtree); ++ } ++ ++ out: ++ return err; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_remove_va_region); ++ ++/** ++ * @brief Insert a VA region to the list, replacing the current at_reg. ++ */ ++static int kbase_insert_va_region_nolock(struct kbase_context *kctx, struct kbase_va_region *new_reg, struct kbase_va_region *at_reg, u64 start_pfn, size_t nr_pages) ++{ ++ struct rb_root *reg_rbtree = NULL; ++ int err = 0; ++ ++ reg_rbtree = kbase_reg_flags_to_rbtree(kctx, at_reg); ++ ++ /* Must be a free region */ ++ KBASE_DEBUG_ASSERT((at_reg->flags & KBASE_REG_FREE) != 0); ++ /* start_pfn should be contained within at_reg */ ++ KBASE_DEBUG_ASSERT((start_pfn >= at_reg->start_pfn) && (start_pfn < at_reg->start_pfn + at_reg->nr_pages)); ++ /* at least nr_pages from start_pfn should be contained within at_reg */ ++ KBASE_DEBUG_ASSERT(start_pfn + nr_pages <= at_reg->start_pfn + at_reg->nr_pages); ++ ++ new_reg->start_pfn = start_pfn; ++ new_reg->nr_pages = nr_pages; ++ ++ /* Regions are a whole use, so swap and delete old one. */ ++ if (at_reg->start_pfn == start_pfn && at_reg->nr_pages == nr_pages) { ++ rb_replace_node(&(at_reg->rblink), &(new_reg->rblink), ++ reg_rbtree); ++ kbase_free_alloced_region(at_reg); ++ } ++ /* New region replaces the start of the old one, so insert before. */ ++ else if (at_reg->start_pfn == start_pfn) { ++ at_reg->start_pfn += nr_pages; ++ KBASE_DEBUG_ASSERT(at_reg->nr_pages >= nr_pages); ++ at_reg->nr_pages -= nr_pages; ++ ++ kbase_region_tracker_insert(kctx, new_reg); ++ } ++ /* New region replaces the end of the old one, so insert after. */ ++ else if ((at_reg->start_pfn + at_reg->nr_pages) == (start_pfn + nr_pages)) { ++ at_reg->nr_pages -= nr_pages; ++ ++ kbase_region_tracker_insert(kctx, new_reg); ++ } ++ /* New region splits the old one, so insert and create new */ ++ else { ++ struct kbase_va_region *new_front_reg; ++ ++ new_front_reg = kbase_alloc_free_region(kctx, ++ at_reg->start_pfn, ++ start_pfn - at_reg->start_pfn, ++ at_reg->flags & KBASE_REG_ZONE_MASK); ++ ++ if (new_front_reg) { ++ at_reg->nr_pages -= nr_pages + new_front_reg->nr_pages; ++ at_reg->start_pfn = start_pfn + nr_pages; ++ ++ kbase_region_tracker_insert(kctx, new_front_reg); ++ kbase_region_tracker_insert(kctx, new_reg); ++ } else { ++ err = -ENOMEM; ++ } ++ } ++ ++ return err; ++} ++ ++/** ++ * @brief Add a VA region to the list. ++ */ ++int kbase_add_va_region(struct kbase_context *kctx, ++ struct kbase_va_region *reg, u64 addr, ++ size_t nr_pages, size_t align) ++{ ++ struct kbase_va_region *tmp; ++ u64 gpu_pfn = addr >> PAGE_SHIFT; ++ int err = 0; ++ ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ KBASE_DEBUG_ASSERT(NULL != reg); ++ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ if (!align) ++ align = 1; ++ ++ /* must be a power of 2 */ ++ KBASE_DEBUG_ASSERT((align & (align - 1)) == 0); ++ KBASE_DEBUG_ASSERT(nr_pages > 0); ++ ++ /* Path 1: Map a specific address. Find the enclosing region, which *must* be free. */ ++ if (gpu_pfn) { ++ struct device *dev = kctx->kbdev->dev; ++ ++ KBASE_DEBUG_ASSERT(!(gpu_pfn & (align - 1))); ++ ++ tmp = kbase_region_tracker_find_region_enclosing_range_free(kctx, gpu_pfn, nr_pages); ++ if (!tmp) { ++ dev_warn(dev, "Enclosing region not found: 0x%08llx gpu_pfn, %zu nr_pages", gpu_pfn, nr_pages); ++ err = -ENOMEM; ++ goto exit; ++ } ++ if (!(tmp->flags & KBASE_REG_FREE)) { ++ dev_warn(dev, "Zone mismatch: %lu != %lu", tmp->flags & KBASE_REG_ZONE_MASK, reg->flags & KBASE_REG_ZONE_MASK); ++ dev_warn(dev, "!(tmp->flags & KBASE_REG_FREE): tmp->start_pfn=0x%llx tmp->flags=0x%lx tmp->nr_pages=0x%zx gpu_pfn=0x%llx nr_pages=0x%zx\n", tmp->start_pfn, tmp->flags, tmp->nr_pages, gpu_pfn, nr_pages); ++ dev_warn(dev, "in function %s (%p, %p, 0x%llx, 0x%zx, 0x%zx)\n", __func__, kctx, reg, addr, nr_pages, align); ++ err = -ENOMEM; ++ goto exit; ++ } ++ ++ err = kbase_insert_va_region_nolock(kctx, reg, tmp, gpu_pfn, nr_pages); ++ if (err) { ++ dev_warn(dev, "Failed to insert va region"); ++ err = -ENOMEM; ++ goto exit; ++ } ++ ++ goto exit; ++ } ++ ++ /* Path 2: Map any free address which meets the requirements. */ ++ { ++ u64 start_pfn; ++ ++ /* ++ * Depending on the zone the allocation request is for ++ * we might need to retry it. ++ */ ++ do { ++ tmp = kbase_region_tracker_find_region_meeting_reqs( ++ kctx, reg, nr_pages, align); ++ if (tmp) { ++ start_pfn = (tmp->start_pfn + align - 1) & ++ ~(align - 1); ++ err = kbase_insert_va_region_nolock(kctx, reg, ++ tmp, start_pfn, nr_pages); ++ break; ++ } ++ ++ /* ++ * If the allocation is not from the same zone as JIT ++ * then don't retry, we're out of VA and there is ++ * nothing which can be done about it. ++ */ ++ if ((reg->flags & KBASE_REG_ZONE_MASK) != ++ KBASE_REG_ZONE_CUSTOM_VA) ++ break; ++ } while (kbase_jit_evict(kctx)); ++ ++ if (!tmp) ++ err = -ENOMEM; ++ } ++ ++ exit: ++ return err; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_add_va_region); ++ ++/** ++ * @brief Initialize the internal region tracker data structure. ++ */ ++static void kbase_region_tracker_ds_init(struct kbase_context *kctx, ++ struct kbase_va_region *same_va_reg, ++ struct kbase_va_region *exec_reg, ++ struct kbase_va_region *custom_va_reg) ++{ ++ kctx->reg_rbtree_same = RB_ROOT; ++ kbase_region_tracker_insert(kctx, same_va_reg); ++ ++ /* Although exec and custom_va_reg don't always exist, ++ * initialize unconditionally because of the mem_view debugfs ++ * implementation which relies on these being empty */ ++ kctx->reg_rbtree_exec = RB_ROOT; ++ kctx->reg_rbtree_custom = RB_ROOT; ++ ++ if (exec_reg) ++ kbase_region_tracker_insert(kctx, exec_reg); ++ if (custom_va_reg) ++ kbase_region_tracker_insert(kctx, custom_va_reg); ++} ++ ++static void kbase_region_tracker_erase_rbtree(struct rb_root *rbtree) ++{ ++ struct rb_node *rbnode; ++ struct kbase_va_region *reg; ++ ++ do { ++ rbnode = rb_first(rbtree); ++ if (rbnode) { ++ rb_erase(rbnode, rbtree); ++ reg = rb_entry(rbnode, struct kbase_va_region, rblink); ++ kbase_free_alloced_region(reg); ++ } ++ } while (rbnode); ++} ++ ++void kbase_region_tracker_term(struct kbase_context *kctx) ++{ ++ kbase_region_tracker_erase_rbtree(&kctx->reg_rbtree_same); ++ kbase_region_tracker_erase_rbtree(&kctx->reg_rbtree_exec); ++ kbase_region_tracker_erase_rbtree(&kctx->reg_rbtree_custom); ++} ++ ++/** ++ * Initialize the region tracker data structure. ++ */ ++int kbase_region_tracker_init(struct kbase_context *kctx) ++{ ++ struct kbase_va_region *same_va_reg; ++ struct kbase_va_region *exec_reg = NULL; ++ struct kbase_va_region *custom_va_reg = NULL; ++ size_t same_va_bits = sizeof(void *) * BITS_PER_BYTE; ++ u64 custom_va_size = KBASE_REG_ZONE_CUSTOM_VA_SIZE; ++ u64 gpu_va_limit = (1ULL << kctx->kbdev->gpu_props.mmu.va_bits) >> PAGE_SHIFT; ++ u64 same_va_pages; ++ int err; ++ ++ /* Take the lock as kbase_free_alloced_region requires it */ ++ kbase_gpu_vm_lock(kctx); ++ ++#if defined(CONFIG_ARM64) ++ same_va_bits = VA_BITS; ++#elif defined(CONFIG_X86_64) ++ same_va_bits = 47; ++#elif defined(CONFIG_64BIT) ++#error Unsupported 64-bit architecture ++#endif ++ ++#ifdef CONFIG_64BIT ++ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) ++ same_va_bits = 32; ++ else if (kbase_hw_has_feature(kctx->kbdev, BASE_HW_FEATURE_33BIT_VA)) ++ same_va_bits = 33; ++#endif ++ ++ if (kctx->kbdev->gpu_props.mmu.va_bits < same_va_bits) { ++ err = -EINVAL; ++ goto fail_unlock; ++ } ++ ++ same_va_pages = (1ULL << (same_va_bits - PAGE_SHIFT)) - 1; ++ /* all have SAME_VA */ ++ same_va_reg = kbase_alloc_free_region(kctx, 1, ++ same_va_pages, ++ KBASE_REG_ZONE_SAME_VA); ++ ++ if (!same_va_reg) { ++ err = -ENOMEM; ++ goto fail_unlock; ++ } ++ ++#ifdef CONFIG_64BIT ++ /* 32-bit clients have exec and custom VA zones */ ++ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) { ++#endif ++ if (gpu_va_limit <= KBASE_REG_ZONE_CUSTOM_VA_BASE) { ++ err = -EINVAL; ++ goto fail_free_same_va; ++ } ++ /* If the current size of TMEM is out of range of the ++ * virtual address space addressable by the MMU then ++ * we should shrink it to fit ++ */ ++ if ((KBASE_REG_ZONE_CUSTOM_VA_BASE + KBASE_REG_ZONE_CUSTOM_VA_SIZE) >= gpu_va_limit) ++ custom_va_size = gpu_va_limit - KBASE_REG_ZONE_CUSTOM_VA_BASE; ++ ++ exec_reg = kbase_alloc_free_region(kctx, ++ KBASE_REG_ZONE_EXEC_BASE, ++ KBASE_REG_ZONE_EXEC_SIZE, ++ KBASE_REG_ZONE_EXEC); ++ ++ if (!exec_reg) { ++ err = -ENOMEM; ++ goto fail_free_same_va; ++ } ++ ++ custom_va_reg = kbase_alloc_free_region(kctx, ++ KBASE_REG_ZONE_CUSTOM_VA_BASE, ++ custom_va_size, KBASE_REG_ZONE_CUSTOM_VA); ++ ++ if (!custom_va_reg) { ++ err = -ENOMEM; ++ goto fail_free_exec; ++ } ++#ifdef CONFIG_64BIT ++ } ++#endif ++ ++ kbase_region_tracker_ds_init(kctx, same_va_reg, exec_reg, custom_va_reg); ++ ++ kctx->same_va_end = same_va_pages + 1; ++ ++ kbase_gpu_vm_unlock(kctx); ++ return 0; ++ ++fail_free_exec: ++ kbase_free_alloced_region(exec_reg); ++fail_free_same_va: ++ kbase_free_alloced_region(same_va_reg); ++fail_unlock: ++ kbase_gpu_vm_unlock(kctx); ++ return err; ++} ++ ++int kbase_region_tracker_init_jit(struct kbase_context *kctx, u64 jit_va_pages) ++{ ++#ifdef CONFIG_64BIT ++ struct kbase_va_region *same_va; ++ struct kbase_va_region *custom_va_reg; ++ u64 same_va_bits; ++ u64 total_va_size; ++ int err; ++ ++ /* ++ * Nothing to do for 32-bit clients, JIT uses the existing ++ * custom VA zone. ++ */ ++ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) ++ return 0; ++ ++#if defined(CONFIG_ARM64) ++ same_va_bits = VA_BITS; ++#elif defined(CONFIG_X86_64) ++ same_va_bits = 47; ++#elif defined(CONFIG_64BIT) ++#error Unsupported 64-bit architecture ++#endif ++ ++ if (kbase_hw_has_feature(kctx->kbdev, BASE_HW_FEATURE_33BIT_VA)) ++ same_va_bits = 33; ++ ++ total_va_size = (1ULL << (same_va_bits - PAGE_SHIFT)) - 1; ++ ++ kbase_gpu_vm_lock(kctx); ++ ++ /* ++ * Modify the same VA free region after creation. Be careful to ensure ++ * that allocations haven't been made as they could cause an overlap ++ * to happen with existing same VA allocations and the custom VA zone. ++ */ ++ same_va = kbase_region_tracker_find_region_base_address(kctx, ++ PAGE_SIZE); ++ if (!same_va) { ++ err = -ENOMEM; ++ goto fail_unlock; ++ } ++ ++ /* The region flag or region size has changed since creation so bail. */ ++ if ((!(same_va->flags & KBASE_REG_FREE)) || ++ (same_va->nr_pages != total_va_size)) { ++ err = -ENOMEM; ++ goto fail_unlock; ++ } ++ ++ if (same_va->nr_pages < jit_va_pages || ++ kctx->same_va_end < jit_va_pages) { ++ err = -ENOMEM; ++ goto fail_unlock; ++ } ++ ++ /* It's safe to adjust the same VA zone now */ ++ same_va->nr_pages -= jit_va_pages; ++ kctx->same_va_end -= jit_va_pages; ++ ++ /* ++ * Create a custom VA zone at the end of the VA for allocations which ++ * JIT can use so it doesn't have to allocate VA from the kernel. ++ */ ++ custom_va_reg = kbase_alloc_free_region(kctx, ++ kctx->same_va_end, ++ jit_va_pages, ++ KBASE_REG_ZONE_CUSTOM_VA); ++ ++ if (!custom_va_reg) { ++ /* ++ * The context will be destroyed if we fail here so no point ++ * reverting the change we made to same_va. ++ */ ++ err = -ENOMEM; ++ goto fail_unlock; ++ } ++ ++ kbase_region_tracker_insert(kctx, custom_va_reg); ++ ++ kbase_gpu_vm_unlock(kctx); ++ return 0; ++ ++fail_unlock: ++ kbase_gpu_vm_unlock(kctx); ++ return err; ++#else ++ return 0; ++#endif ++} ++ ++int kbase_mem_init(struct kbase_device *kbdev) ++{ ++ struct kbasep_mem_device *memdev; ++ ++ KBASE_DEBUG_ASSERT(kbdev); ++ ++ memdev = &kbdev->memdev; ++ kbdev->mem_pool_max_size_default = KBASE_MEM_POOL_MAX_SIZE_KCTX; ++ ++ /* Initialize memory usage */ ++ atomic_set(&memdev->used_pages, 0); ++ ++ return kbase_mem_pool_init(&kbdev->mem_pool, ++ KBASE_MEM_POOL_MAX_SIZE_KBDEV, kbdev, NULL); ++} ++ ++void kbase_mem_halt(struct kbase_device *kbdev) ++{ ++ CSTD_UNUSED(kbdev); ++} ++ ++void kbase_mem_term(struct kbase_device *kbdev) ++{ ++ struct kbasep_mem_device *memdev; ++ int pages; ++ ++ KBASE_DEBUG_ASSERT(kbdev); ++ ++ memdev = &kbdev->memdev; ++ ++ pages = atomic_read(&memdev->used_pages); ++ if (pages != 0) ++ dev_warn(kbdev->dev, "%s: %d pages in use!\n", __func__, pages); ++ ++ kbase_mem_pool_term(&kbdev->mem_pool); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_mem_term); ++ ++ ++ ++ ++/** ++ * @brief Allocate a free region object. ++ * ++ * The allocated object is not part of any list yet, and is flagged as ++ * KBASE_REG_FREE. No mapping is allocated yet. ++ * ++ * zone is KBASE_REG_ZONE_CUSTOM_VA, KBASE_REG_ZONE_SAME_VA, or KBASE_REG_ZONE_EXEC ++ * ++ */ ++struct kbase_va_region *kbase_alloc_free_region(struct kbase_context *kctx, u64 start_pfn, size_t nr_pages, int zone) ++{ ++ struct kbase_va_region *new_reg; ++ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ ++ /* zone argument should only contain zone related region flags */ ++ KBASE_DEBUG_ASSERT((zone & ~KBASE_REG_ZONE_MASK) == 0); ++ KBASE_DEBUG_ASSERT(nr_pages > 0); ++ /* 64-bit address range is the max */ ++ KBASE_DEBUG_ASSERT(start_pfn + nr_pages <= (U64_MAX / PAGE_SIZE)); ++ ++ new_reg = kzalloc(sizeof(*new_reg), GFP_KERNEL); ++ ++ if (!new_reg) ++ return NULL; ++ ++ new_reg->cpu_alloc = NULL; /* no alloc bound yet */ ++ new_reg->gpu_alloc = NULL; /* no alloc bound yet */ ++ new_reg->kctx = kctx; ++ new_reg->flags = zone | KBASE_REG_FREE; ++ ++ new_reg->flags |= KBASE_REG_GROWABLE; ++ ++ new_reg->start_pfn = start_pfn; ++ new_reg->nr_pages = nr_pages; ++ ++ return new_reg; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_alloc_free_region); ++ ++/** ++ * @brief Free a region object. ++ * ++ * The described region must be freed of any mapping. ++ * ++ * If the region is not flagged as KBASE_REG_FREE, the region's ++ * alloc object will be released. ++ * It is a bug if no alloc object exists for non-free regions. ++ * ++ */ ++void kbase_free_alloced_region(struct kbase_va_region *reg) ++{ ++ if (!(reg->flags & KBASE_REG_FREE)) { ++ /* ++ * The physical allocation should have been removed from the ++ * eviction list before this function is called. However, in the ++ * case of abnormal process termination or the app leaking the ++ * memory kbase_mem_free_region is not called so it can still be ++ * on the list at termination time of the region tracker. ++ */ ++ if (!list_empty(®->gpu_alloc->evict_node)) { ++ /* ++ * Unlink the physical allocation before unmaking it ++ * evictable so that the allocation isn't grown back to ++ * its last backed size as we're going to unmap it ++ * anyway. ++ */ ++ reg->cpu_alloc->reg = NULL; ++ if (reg->cpu_alloc != reg->gpu_alloc) ++ reg->gpu_alloc->reg = NULL; ++ ++ /* ++ * If a region has been made evictable then we must ++ * unmake it before trying to free it. ++ * If the memory hasn't been reclaimed it will be ++ * unmapped and freed below, if it has been reclaimed ++ * then the operations below are no-ops. ++ */ ++ if (reg->flags & KBASE_REG_DONT_NEED) { ++ KBASE_DEBUG_ASSERT(reg->cpu_alloc->type == ++ KBASE_MEM_TYPE_NATIVE); ++ kbase_mem_evictable_unmake(reg->gpu_alloc); ++ } ++ } ++ ++ /* ++ * Remove the region from the sticky resource metadata ++ * list should it be there. ++ */ ++ kbase_sticky_resource_release(reg->kctx, NULL, ++ reg->start_pfn << PAGE_SHIFT); ++ ++ kbase_mem_phy_alloc_put(reg->cpu_alloc); ++ kbase_mem_phy_alloc_put(reg->gpu_alloc); ++ /* To detect use-after-free in debug builds */ ++ KBASE_DEBUG_CODE(reg->flags |= KBASE_REG_FREE); ++ } ++ kfree(reg); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_free_alloced_region); ++ ++int kbase_gpu_mmap(struct kbase_context *kctx, struct kbase_va_region *reg, u64 addr, size_t nr_pages, size_t align) ++{ ++ int err; ++ size_t i = 0; ++ unsigned long attr; ++ unsigned long mask = ~KBASE_REG_MEMATTR_MASK; ++ ++ if ((kctx->kbdev->system_coherency == COHERENCY_ACE) && ++ (reg->flags & KBASE_REG_SHARE_BOTH)) ++ attr = KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_OUTER_WA); ++ else ++ attr = KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_WRITE_ALLOC); ++ ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ KBASE_DEBUG_ASSERT(NULL != reg); ++ ++ err = kbase_add_va_region(kctx, reg, addr, nr_pages, align); ++ if (err) ++ return err; ++ ++ if (reg->gpu_alloc->type == KBASE_MEM_TYPE_ALIAS) { ++ u64 stride; ++ struct kbase_mem_phy_alloc *alloc; ++ ++ alloc = reg->gpu_alloc; ++ stride = alloc->imported.alias.stride; ++ KBASE_DEBUG_ASSERT(alloc->imported.alias.aliased); ++ for (i = 0; i < alloc->imported.alias.nents; i++) { ++ if (alloc->imported.alias.aliased[i].alloc) { ++ err = kbase_mmu_insert_pages(kctx, ++ reg->start_pfn + (i * stride), ++ alloc->imported.alias.aliased[i].alloc->pages + alloc->imported.alias.aliased[i].offset, ++ alloc->imported.alias.aliased[i].length, ++ reg->flags); ++ if (err) ++ goto bad_insert; ++ ++ kbase_mem_phy_alloc_gpu_mapped(alloc->imported.alias.aliased[i].alloc); ++ } else { ++ err = kbase_mmu_insert_single_page(kctx, ++ reg->start_pfn + i * stride, ++ page_to_phys(kctx->aliasing_sink_page), ++ alloc->imported.alias.aliased[i].length, ++ (reg->flags & mask) | attr); ++ ++ if (err) ++ goto bad_insert; ++ } ++ } ++ } else { ++ err = kbase_mmu_insert_pages(kctx, reg->start_pfn, ++ kbase_get_gpu_phy_pages(reg), ++ kbase_reg_current_backed_size(reg), ++ reg->flags); ++ if (err) ++ goto bad_insert; ++ kbase_mem_phy_alloc_gpu_mapped(reg->gpu_alloc); ++ } ++ ++ return err; ++ ++bad_insert: ++ if (reg->gpu_alloc->type == KBASE_MEM_TYPE_ALIAS) { ++ u64 stride; ++ ++ stride = reg->gpu_alloc->imported.alias.stride; ++ KBASE_DEBUG_ASSERT(reg->gpu_alloc->imported.alias.aliased); ++ while (i--) ++ if (reg->gpu_alloc->imported.alias.aliased[i].alloc) { ++ kbase_mmu_teardown_pages(kctx, reg->start_pfn + (i * stride), reg->gpu_alloc->imported.alias.aliased[i].length); ++ kbase_mem_phy_alloc_gpu_unmapped(reg->gpu_alloc->imported.alias.aliased[i].alloc); ++ } ++ } ++ ++ kbase_remove_va_region(kctx, reg); ++ ++ return err; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_gpu_mmap); ++ ++static void kbase_jd_user_buf_unmap(struct kbase_context *kctx, ++ struct kbase_mem_phy_alloc *alloc, bool writeable); ++ ++int kbase_gpu_munmap(struct kbase_context *kctx, struct kbase_va_region *reg) ++{ ++ int err; ++ ++ if (reg->start_pfn == 0) ++ return 0; ++ ++ if (reg->gpu_alloc && reg->gpu_alloc->type == KBASE_MEM_TYPE_ALIAS) { ++ size_t i; ++ ++ err = kbase_mmu_teardown_pages(kctx, reg->start_pfn, reg->nr_pages); ++ KBASE_DEBUG_ASSERT(reg->gpu_alloc->imported.alias.aliased); ++ for (i = 0; i < reg->gpu_alloc->imported.alias.nents; i++) ++ if (reg->gpu_alloc->imported.alias.aliased[i].alloc) ++ kbase_mem_phy_alloc_gpu_unmapped(reg->gpu_alloc->imported.alias.aliased[i].alloc); ++ } else { ++ err = kbase_mmu_teardown_pages(kctx, reg->start_pfn, kbase_reg_current_backed_size(reg)); ++ kbase_mem_phy_alloc_gpu_unmapped(reg->gpu_alloc); ++ } ++ ++ if (reg->gpu_alloc && reg->gpu_alloc->type == ++ KBASE_MEM_TYPE_IMPORTED_USER_BUF) { ++ struct kbase_alloc_import_user_buf *user_buf = ++ ®->gpu_alloc->imported.user_buf; ++ ++ if (user_buf->current_mapping_usage_count & PINNED_ON_IMPORT) { ++ user_buf->current_mapping_usage_count &= ++ ~PINNED_ON_IMPORT; ++ ++ kbase_jd_user_buf_unmap(kctx, reg->gpu_alloc, ++ (reg->flags & KBASE_REG_GPU_WR)); ++ } ++ } ++ ++ if (err) ++ return err; ++ ++ err = kbase_remove_va_region(kctx, reg); ++ return err; ++} ++ ++static struct kbase_cpu_mapping *kbasep_find_enclosing_cpu_mapping( ++ struct kbase_context *kctx, ++ unsigned long uaddr, size_t size, u64 *offset) ++{ ++ struct vm_area_struct *vma; ++ struct kbase_cpu_mapping *map; ++ unsigned long vm_pgoff_in_region; ++ unsigned long vm_off_in_region; ++ unsigned long map_start; ++ size_t map_size; ++ ++ lockdep_assert_held(¤t->mm->mmap_lock); ++ ++ if ((uintptr_t) uaddr + size < (uintptr_t) uaddr) /* overflow check */ ++ return NULL; ++ ++ vma = find_vma_intersection(current->mm, uaddr, uaddr+size); ++ ++ if (!vma || vma->vm_start > uaddr) ++ return NULL; ++ if (vma->vm_ops != &kbase_vm_ops) ++ /* Not ours! */ ++ return NULL; ++ ++ map = vma->vm_private_data; ++ ++ if (map->kctx != kctx) ++ /* Not from this context! */ ++ return NULL; ++ ++ vm_pgoff_in_region = vma->vm_pgoff - map->region->start_pfn; ++ vm_off_in_region = vm_pgoff_in_region << PAGE_SHIFT; ++ map_start = vma->vm_start - vm_off_in_region; ++ map_size = map->region->nr_pages << PAGE_SHIFT; ++ ++ if ((uaddr + size) > (map_start + map_size)) ++ /* Not within the CPU mapping */ ++ return NULL; ++ ++ *offset = (uaddr - vma->vm_start) + vm_off_in_region; ++ ++ return map; ++} ++ ++int kbasep_find_enclosing_cpu_mapping_offset( ++ struct kbase_context *kctx, ++ unsigned long uaddr, size_t size, u64 *offset) ++{ ++ struct kbase_cpu_mapping *map; ++ ++ kbase_os_mem_map_lock(kctx); ++ ++ map = kbasep_find_enclosing_cpu_mapping(kctx, uaddr, size, offset); ++ ++ kbase_os_mem_map_unlock(kctx); ++ ++ if (!map) ++ return -EINVAL; ++ ++ return 0; ++} ++ ++KBASE_EXPORT_TEST_API(kbasep_find_enclosing_cpu_mapping_offset); ++ ++void kbase_sync_single(struct kbase_context *kctx, ++ phys_addr_t cpu_pa, phys_addr_t gpu_pa, ++ off_t offset, size_t size, enum kbase_sync_type sync_fn) ++{ ++ struct page *cpu_page; ++ ++ cpu_page = pfn_to_page(PFN_DOWN(cpu_pa)); ++ ++ if (likely(cpu_pa == gpu_pa)) { ++ dma_addr_t dma_addr; ++ ++ BUG_ON(!cpu_page); ++ BUG_ON(offset + size > PAGE_SIZE); ++ ++ dma_addr = kbase_dma_addr(cpu_page) + offset; ++ if (sync_fn == KBASE_SYNC_TO_CPU) ++ dma_sync_single_for_cpu(kctx->kbdev->dev, dma_addr, ++ size, DMA_BIDIRECTIONAL); ++ else if (sync_fn == KBASE_SYNC_TO_DEVICE) ++ dma_sync_single_for_device(kctx->kbdev->dev, dma_addr, ++ size, DMA_BIDIRECTIONAL); ++ } else { ++ void *src = NULL; ++ void *dst = NULL; ++ struct page *gpu_page; ++ ++ if (WARN(!gpu_pa, "No GPU PA found for infinite cache op")) ++ return; ++ ++ gpu_page = pfn_to_page(PFN_DOWN(gpu_pa)); ++ ++ if (sync_fn == KBASE_SYNC_TO_DEVICE) { ++ src = ((unsigned char *)kmap(cpu_page)) + offset; ++ dst = ((unsigned char *)kmap(gpu_page)) + offset; ++ } else if (sync_fn == KBASE_SYNC_TO_CPU) { ++ dma_sync_single_for_cpu(kctx->kbdev->dev, ++ kbase_dma_addr(gpu_page) + offset, ++ size, DMA_BIDIRECTIONAL); ++ src = ((unsigned char *)kmap(gpu_page)) + offset; ++ dst = ((unsigned char *)kmap(cpu_page)) + offset; ++ } ++ memcpy(dst, src, size); ++ kunmap(gpu_page); ++ kunmap(cpu_page); ++ if (sync_fn == KBASE_SYNC_TO_DEVICE) ++ dma_sync_single_for_device(kctx->kbdev->dev, ++ kbase_dma_addr(gpu_page) + offset, ++ size, DMA_BIDIRECTIONAL); ++ } ++} ++ ++static int kbase_do_syncset(struct kbase_context *kctx, ++ struct basep_syncset *sset, enum kbase_sync_type sync_fn) ++{ ++ int err = 0; ++ struct kbase_va_region *reg; ++ struct kbase_cpu_mapping *map; ++ unsigned long start; ++ size_t size; ++ phys_addr_t *cpu_pa; ++ phys_addr_t *gpu_pa; ++ u64 page_off, page_count; ++ u64 i; ++ u64 offset; ++ ++ kbase_os_mem_map_lock(kctx); ++ kbase_gpu_vm_lock(kctx); ++ ++ /* find the region where the virtual address is contained */ ++ reg = kbase_region_tracker_find_region_enclosing_address(kctx, ++ sset->mem_handle.basep.handle); ++ if (!reg) { ++ dev_warn(kctx->kbdev->dev, "Can't find region at VA 0x%016llX", ++ sset->mem_handle.basep.handle); ++ err = -EINVAL; ++ goto out_unlock; ++ } ++ ++ if (!(reg->flags & KBASE_REG_CPU_CACHED)) ++ goto out_unlock; ++ ++ start = (uintptr_t)sset->user_addr; ++ size = (size_t)sset->size; ++ ++ map = kbasep_find_enclosing_cpu_mapping(kctx, start, size, &offset); ++ if (!map) { ++ dev_warn(kctx->kbdev->dev, "Can't find CPU mapping 0x%016lX for VA 0x%016llX", ++ start, sset->mem_handle.basep.handle); ++ err = -EINVAL; ++ goto out_unlock; ++ } ++ ++ page_off = offset >> PAGE_SHIFT; ++ offset &= ~PAGE_MASK; ++ page_count = (size + offset + (PAGE_SIZE - 1)) >> PAGE_SHIFT; ++ cpu_pa = kbase_get_cpu_phy_pages(reg); ++ gpu_pa = kbase_get_gpu_phy_pages(reg); ++ ++ if (page_off > reg->nr_pages || ++ page_off + page_count > reg->nr_pages) { ++ /* Sync overflows the region */ ++ err = -EINVAL; ++ goto out_unlock; ++ } ++ ++ /* Sync first page */ ++ if (cpu_pa[page_off]) { ++ size_t sz = MIN(((size_t) PAGE_SIZE - offset), size); ++ ++ kbase_sync_single(kctx, cpu_pa[page_off], gpu_pa[page_off], ++ offset, sz, sync_fn); ++ } ++ ++ /* Sync middle pages (if any) */ ++ for (i = 1; page_count > 2 && i < page_count - 1; i++) { ++ /* we grow upwards, so bail on first non-present page */ ++ if (!cpu_pa[page_off + i]) ++ break; ++ ++ kbase_sync_single(kctx, cpu_pa[page_off + i], ++ gpu_pa[page_off + i], 0, PAGE_SIZE, sync_fn); ++ } ++ ++ /* Sync last page (if any) */ ++ if (page_count > 1 && cpu_pa[page_off + page_count - 1]) { ++ size_t sz = ((start + size - 1) & ~PAGE_MASK) + 1; ++ ++ kbase_sync_single(kctx, cpu_pa[page_off + page_count - 1], ++ gpu_pa[page_off + page_count - 1], 0, sz, ++ sync_fn); ++ } ++ ++out_unlock: ++ kbase_gpu_vm_unlock(kctx); ++ kbase_os_mem_map_unlock(kctx); ++ return err; ++} ++ ++int kbase_sync_now(struct kbase_context *kctx, struct basep_syncset *sset) ++{ ++ int err = -EINVAL; ++ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ KBASE_DEBUG_ASSERT(sset != NULL); ++ ++ if (sset->mem_handle.basep.handle & ~PAGE_MASK) { ++ dev_warn(kctx->kbdev->dev, ++ "mem_handle: passed parameter is invalid"); ++ return -EINVAL; ++ } ++ ++ switch (sset->type) { ++ case BASE_SYNCSET_OP_MSYNC: ++ err = kbase_do_syncset(kctx, sset, KBASE_SYNC_TO_DEVICE); ++ break; ++ ++ case BASE_SYNCSET_OP_CSYNC: ++ err = kbase_do_syncset(kctx, sset, KBASE_SYNC_TO_CPU); ++ break; ++ ++ default: ++ dev_warn(kctx->kbdev->dev, "Unknown msync op %d\n", sset->type); ++ break; ++ } ++ ++ return err; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_sync_now); ++ ++/* vm lock must be held */ ++int kbase_mem_free_region(struct kbase_context *kctx, struct kbase_va_region *reg) ++{ ++ int err; ++ ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ KBASE_DEBUG_ASSERT(NULL != reg); ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ /* ++ * Unlink the physical allocation before unmaking it evictable so ++ * that the allocation isn't grown back to its last backed size ++ * as we're going to unmap it anyway. ++ */ ++ reg->cpu_alloc->reg = NULL; ++ if (reg->cpu_alloc != reg->gpu_alloc) ++ reg->gpu_alloc->reg = NULL; ++ ++ /* ++ * If a region has been made evictable then we must unmake it ++ * before trying to free it. ++ * If the memory hasn't been reclaimed it will be unmapped and freed ++ * below, if it has been reclaimed then the operations below are no-ops. ++ */ ++ if (reg->flags & KBASE_REG_DONT_NEED) { ++ KBASE_DEBUG_ASSERT(reg->cpu_alloc->type == ++ KBASE_MEM_TYPE_NATIVE); ++ kbase_mem_evictable_unmake(reg->gpu_alloc); ++ } ++ ++ err = kbase_gpu_munmap(kctx, reg); ++ if (err) { ++ dev_warn(reg->kctx->kbdev->dev, "Could not unmap from the GPU...\n"); ++ goto out; ++ } ++ ++ /* This will also free the physical pages */ ++ kbase_free_alloced_region(reg); ++ ++ out: ++ return err; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_mem_free_region); ++ ++/** ++ * @brief Free the region from the GPU and unregister it. ++ * ++ * This function implements the free operation on a memory segment. ++ * It will loudly fail if called with outstanding mappings. ++ */ ++int kbase_mem_free(struct kbase_context *kctx, u64 gpu_addr) ++{ ++ int err = 0; ++ struct kbase_va_region *reg; ++ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ ++ if ((gpu_addr & ~PAGE_MASK) && (gpu_addr >= PAGE_SIZE)) { ++ dev_warn(kctx->kbdev->dev, "kbase_mem_free: gpu_addr parameter is invalid"); ++ return -EINVAL; ++ } ++ ++ if (0 == gpu_addr) { ++ dev_warn(kctx->kbdev->dev, "gpu_addr 0 is reserved for the ringbuffer and it's an error to try to free it using kbase_mem_free\n"); ++ return -EINVAL; ++ } ++ kbase_gpu_vm_lock(kctx); ++ ++ if (gpu_addr >= BASE_MEM_COOKIE_BASE && ++ gpu_addr < BASE_MEM_FIRST_FREE_ADDRESS) { ++ int cookie = PFN_DOWN(gpu_addr - BASE_MEM_COOKIE_BASE); ++ ++ reg = kctx->pending_regions[cookie]; ++ if (!reg) { ++ err = -EINVAL; ++ goto out_unlock; ++ } ++ ++ /* ask to unlink the cookie as we'll free it */ ++ ++ kctx->pending_regions[cookie] = NULL; ++ kctx->cookies |= (1UL << cookie); ++ ++ kbase_free_alloced_region(reg); ++ } else { ++ /* A real GPU va */ ++ /* Validate the region */ ++ reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr); ++ if (!reg || (reg->flags & KBASE_REG_FREE)) { ++ dev_warn(kctx->kbdev->dev, "kbase_mem_free called with nonexistent gpu_addr 0x%llX", ++ gpu_addr); ++ err = -EINVAL; ++ goto out_unlock; ++ } ++ ++ if ((reg->flags & KBASE_REG_ZONE_MASK) == KBASE_REG_ZONE_SAME_VA) { ++ /* SAME_VA must be freed through munmap */ ++ dev_warn(kctx->kbdev->dev, "%s called on SAME_VA memory 0x%llX", __func__, ++ gpu_addr); ++ err = -EINVAL; ++ goto out_unlock; ++ } ++ err = kbase_mem_free_region(kctx, reg); ++ } ++ ++ out_unlock: ++ kbase_gpu_vm_unlock(kctx); ++ return err; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_mem_free); ++ ++int kbase_update_region_flags(struct kbase_context *kctx, ++ struct kbase_va_region *reg, unsigned long flags) ++{ ++ KBASE_DEBUG_ASSERT(NULL != reg); ++ KBASE_DEBUG_ASSERT((flags & ~((1ul << BASE_MEM_FLAGS_NR_BITS) - 1)) == 0); ++ ++ reg->flags |= kbase_cache_enabled(flags, reg->nr_pages); ++ /* all memory is now growable */ ++ reg->flags |= KBASE_REG_GROWABLE; ++ ++ if (flags & BASE_MEM_GROW_ON_GPF) ++ reg->flags |= KBASE_REG_PF_GROW; ++ ++ if (flags & BASE_MEM_PROT_CPU_WR) ++ reg->flags |= KBASE_REG_CPU_WR; ++ ++ if (flags & BASE_MEM_PROT_CPU_RD) ++ reg->flags |= KBASE_REG_CPU_RD; ++ ++ if (flags & BASE_MEM_PROT_GPU_WR) ++ reg->flags |= KBASE_REG_GPU_WR; ++ ++ if (flags & BASE_MEM_PROT_GPU_RD) ++ reg->flags |= KBASE_REG_GPU_RD; ++ ++ if (0 == (flags & BASE_MEM_PROT_GPU_EX)) ++ reg->flags |= KBASE_REG_GPU_NX; ++ ++ if (!kbase_device_is_cpu_coherent(kctx->kbdev)) { ++ if (flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) ++ return -EINVAL; ++ } else if (flags & (BASE_MEM_COHERENT_SYSTEM | ++ BASE_MEM_COHERENT_SYSTEM_REQUIRED)) { ++ reg->flags |= KBASE_REG_SHARE_BOTH; ++ } ++ ++ if (!(reg->flags & KBASE_REG_SHARE_BOTH) && ++ flags & BASE_MEM_COHERENT_LOCAL) { ++ reg->flags |= KBASE_REG_SHARE_IN; ++ } ++ ++ /* Set up default MEMATTR usage */ ++ if (kctx->kbdev->system_coherency == COHERENCY_ACE && ++ (reg->flags & KBASE_REG_SHARE_BOTH)) { ++ reg->flags |= ++ KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_DEFAULT_ACE); ++ } else { ++ reg->flags |= ++ KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_DEFAULT); ++ } ++ ++ return 0; ++} ++ ++int kbase_alloc_phy_pages_helper( ++ struct kbase_mem_phy_alloc *alloc, ++ size_t nr_pages_requested) ++{ ++ int new_page_count __maybe_unused; ++ size_t old_page_count = alloc->nents; ++ ++ KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_NATIVE); ++ KBASE_DEBUG_ASSERT(alloc->imported.kctx); ++ ++ if (nr_pages_requested == 0) ++ goto done; /*nothing to do*/ ++ ++ new_page_count = kbase_atomic_add_pages( ++ nr_pages_requested, &alloc->imported.kctx->used_pages); ++ kbase_atomic_add_pages(nr_pages_requested, &alloc->imported.kctx->kbdev->memdev.used_pages); ++ ++ /* Increase mm counters before we allocate pages so that this ++ * allocation is visible to the OOM killer */ ++ kbase_process_page_usage_inc(alloc->imported.kctx, nr_pages_requested); ++ ++ if (kbase_mem_pool_alloc_pages(&alloc->imported.kctx->mem_pool, ++ nr_pages_requested, alloc->pages + old_page_count) != 0) ++ goto no_alloc; ++ ++ KBASE_TLSTREAM_AUX_PAGESALLOC( ++ (u32)alloc->imported.kctx->id, ++ (u64)new_page_count); ++ ++ alloc->nents += nr_pages_requested; ++done: ++ return 0; ++ ++no_alloc: ++ kbase_process_page_usage_dec(alloc->imported.kctx, nr_pages_requested); ++ kbase_atomic_sub_pages(nr_pages_requested, &alloc->imported.kctx->used_pages); ++ kbase_atomic_sub_pages(nr_pages_requested, &alloc->imported.kctx->kbdev->memdev.used_pages); ++ ++ return -ENOMEM; ++} ++ ++int kbase_free_phy_pages_helper( ++ struct kbase_mem_phy_alloc *alloc, ++ size_t nr_pages_to_free) ++{ ++ struct kbase_context *kctx = alloc->imported.kctx; ++ bool syncback; ++ bool reclaimed = (alloc->evicted != 0); ++ phys_addr_t *start_free; ++ int new_page_count __maybe_unused; ++ ++ KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_NATIVE); ++ KBASE_DEBUG_ASSERT(alloc->imported.kctx); ++ KBASE_DEBUG_ASSERT(alloc->nents >= nr_pages_to_free); ++ ++ /* early out if nothing to do */ ++ if (0 == nr_pages_to_free) ++ return 0; ++ ++ start_free = alloc->pages + alloc->nents - nr_pages_to_free; ++ ++ syncback = alloc->properties & KBASE_MEM_PHY_ALLOC_ACCESSED_CACHED; ++ ++ kbase_mem_pool_free_pages(&kctx->mem_pool, ++ nr_pages_to_free, ++ start_free, ++ syncback, ++ reclaimed); ++ ++ alloc->nents -= nr_pages_to_free; ++ ++ /* ++ * If the allocation was not evicted (i.e. evicted == 0) then ++ * the page accounting needs to be done. ++ */ ++ if (!reclaimed) { ++ kbase_process_page_usage_dec(kctx, nr_pages_to_free); ++ new_page_count = kbase_atomic_sub_pages(nr_pages_to_free, ++ &kctx->used_pages); ++ kbase_atomic_sub_pages(nr_pages_to_free, ++ &kctx->kbdev->memdev.used_pages); ++ ++ KBASE_TLSTREAM_AUX_PAGESALLOC( ++ (u32)kctx->id, ++ (u64)new_page_count); ++ } ++ ++ return 0; ++} ++ ++void kbase_mem_kref_free(struct kref *kref) ++{ ++ struct kbase_mem_phy_alloc *alloc; ++ ++ alloc = container_of(kref, struct kbase_mem_phy_alloc, kref); ++ ++ switch (alloc->type) { ++ case KBASE_MEM_TYPE_NATIVE: { ++ WARN_ON(!alloc->imported.kctx); ++ /* ++ * The physical allocation must have been removed from the ++ * eviction list before trying to free it. ++ */ ++ WARN_ON(!list_empty(&alloc->evict_node)); ++ kbase_free_phy_pages_helper(alloc, alloc->nents); ++ break; ++ } ++ case KBASE_MEM_TYPE_ALIAS: { ++ /* just call put on the underlying phy allocs */ ++ size_t i; ++ struct kbase_aliased *aliased; ++ ++ aliased = alloc->imported.alias.aliased; ++ if (aliased) { ++ for (i = 0; i < alloc->imported.alias.nents; i++) ++ if (aliased[i].alloc) ++ kbase_mem_phy_alloc_put(aliased[i].alloc); ++ vfree(aliased); ++ } ++ break; ++ } ++ case KBASE_MEM_TYPE_RAW: ++ /* raw pages, external cleanup */ ++ break; ++ #ifdef CONFIG_UMP ++ case KBASE_MEM_TYPE_IMPORTED_UMP: ++ ump_dd_release(alloc->imported.ump_handle); ++ break; ++#endif ++#ifdef CONFIG_DMA_SHARED_BUFFER ++ case KBASE_MEM_TYPE_IMPORTED_UMM: ++ dma_buf_detach(alloc->imported.umm.dma_buf, ++ alloc->imported.umm.dma_attachment); ++ dma_buf_put(alloc->imported.umm.dma_buf); ++ break; ++#endif ++ case KBASE_MEM_TYPE_IMPORTED_USER_BUF: ++ if (alloc->imported.user_buf.mm) ++ mmdrop(alloc->imported.user_buf.mm); ++ kfree(alloc->imported.user_buf.pages); ++ break; ++ case KBASE_MEM_TYPE_TB:{ ++ void *tb; ++ ++ tb = alloc->imported.kctx->jctx.tb; ++ kbase_device_trace_buffer_uninstall(alloc->imported.kctx); ++ vfree(tb); ++ break; ++ } ++ default: ++ WARN(1, "Unexecpted free of type %d\n", alloc->type); ++ break; ++ } ++ ++ /* Free based on allocation type */ ++ if (alloc->properties & KBASE_MEM_PHY_ALLOC_LARGE) ++ vfree(alloc); ++ else ++ kfree(alloc); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_mem_kref_free); ++ ++int kbase_alloc_phy_pages(struct kbase_va_region *reg, size_t vsize, size_t size) ++{ ++ KBASE_DEBUG_ASSERT(NULL != reg); ++ KBASE_DEBUG_ASSERT(vsize > 0); ++ ++ /* validate user provided arguments */ ++ if (size > vsize || vsize > reg->nr_pages) ++ goto out_term; ++ ++ /* Prevent vsize*sizeof from wrapping around. ++ * For instance, if vsize is 2**29+1, we'll allocate 1 byte and the alloc won't fail. ++ */ ++ if ((size_t) vsize > ((size_t) -1 / sizeof(*reg->cpu_alloc->pages))) ++ goto out_term; ++ ++ KBASE_DEBUG_ASSERT(0 != vsize); ++ ++ if (kbase_alloc_phy_pages_helper(reg->cpu_alloc, size) != 0) ++ goto out_term; ++ ++ reg->cpu_alloc->reg = reg; ++ if (reg->cpu_alloc != reg->gpu_alloc) { ++ if (kbase_alloc_phy_pages_helper(reg->gpu_alloc, size) != 0) ++ goto out_rollback; ++ reg->gpu_alloc->reg = reg; ++ } ++ ++ return 0; ++ ++out_rollback: ++ kbase_free_phy_pages_helper(reg->cpu_alloc, size); ++out_term: ++ return -1; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_alloc_phy_pages); ++ ++bool kbase_check_alloc_flags(unsigned long flags) ++{ ++ /* Only known input flags should be set. */ ++ if (flags & ~BASE_MEM_FLAGS_INPUT_MASK) ++ return false; ++ ++ /* At least one flag should be set */ ++ if (flags == 0) ++ return false; ++ ++ /* Either the GPU or CPU must be reading from the allocated memory */ ++ if ((flags & (BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_GPU_RD)) == 0) ++ return false; ++ ++ /* Either the GPU or CPU must be writing to the allocated memory */ ++ if ((flags & (BASE_MEM_PROT_CPU_WR | BASE_MEM_PROT_GPU_WR)) == 0) ++ return false; ++ ++ /* GPU cannot be writing to GPU executable memory and cannot grow the memory on page fault. */ ++ if ((flags & BASE_MEM_PROT_GPU_EX) && (flags & (BASE_MEM_PROT_GPU_WR | BASE_MEM_GROW_ON_GPF))) ++ return false; ++ ++ /* GPU should have at least read or write access otherwise there is no ++ reason for allocating. */ ++ if ((flags & (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR)) == 0) ++ return false; ++ ++ /* BASE_MEM_IMPORT_SHARED is only valid for imported memory */ ++ if ((flags & BASE_MEM_IMPORT_SHARED) == BASE_MEM_IMPORT_SHARED) ++ return false; ++ ++ return true; ++} ++ ++bool kbase_check_import_flags(unsigned long flags) ++{ ++ /* Only known input flags should be set. */ ++ if (flags & ~BASE_MEM_FLAGS_INPUT_MASK) ++ return false; ++ ++ /* At least one flag should be set */ ++ if (flags == 0) ++ return false; ++ ++ /* Imported memory cannot be GPU executable */ ++ if (flags & BASE_MEM_PROT_GPU_EX) ++ return false; ++ ++ /* Imported memory cannot grow on page fault */ ++ if (flags & BASE_MEM_GROW_ON_GPF) ++ return false; ++ ++ /* GPU should have at least read or write access otherwise there is no ++ reason for importing. */ ++ if ((flags & (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR)) == 0) ++ return false; ++ ++ /* Secure memory cannot be read by the CPU */ ++ if ((flags & BASE_MEM_SECURE) && (flags & BASE_MEM_PROT_CPU_RD)) ++ return false; ++ ++ return true; ++} ++ ++/** ++ * @brief Acquire the per-context region list lock ++ */ ++void kbase_gpu_vm_lock(struct kbase_context *kctx) ++{ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ mutex_lock(&kctx->reg_lock); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_gpu_vm_lock); ++ ++/** ++ * @brief Release the per-context region list lock ++ */ ++void kbase_gpu_vm_unlock(struct kbase_context *kctx) ++{ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ mutex_unlock(&kctx->reg_lock); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_gpu_vm_unlock); ++ ++#ifdef CONFIG_DEBUG_FS ++struct kbase_jit_debugfs_data { ++ int (*func)(struct kbase_jit_debugfs_data *); ++ struct mutex lock; ++ struct kbase_context *kctx; ++ u64 active_value; ++ u64 pool_value; ++ u64 destroy_value; ++ char buffer[50]; ++}; ++ ++static int kbase_jit_debugfs_common_open(struct inode *inode, ++ struct file *file, int (*func)(struct kbase_jit_debugfs_data *)) ++{ ++ struct kbase_jit_debugfs_data *data; ++ ++ data = kzalloc(sizeof(*data), GFP_KERNEL); ++ if (!data) ++ return -ENOMEM; ++ ++ data->func = func; ++ mutex_init(&data->lock); ++ data->kctx = (struct kbase_context *) inode->i_private; ++ ++ file->private_data = data; ++ ++ return nonseekable_open(inode, file); ++} ++ ++static ssize_t kbase_jit_debugfs_common_read(struct file *file, ++ char __user *buf, size_t len, loff_t *ppos) ++{ ++ struct kbase_jit_debugfs_data *data; ++ size_t size; ++ int ret; ++ ++ data = (struct kbase_jit_debugfs_data *) file->private_data; ++ mutex_lock(&data->lock); ++ ++ if (*ppos) { ++ size = strnlen(data->buffer, sizeof(data->buffer)); ++ } else { ++ if (!data->func) { ++ ret = -EACCES; ++ goto out_unlock; ++ } ++ ++ if (data->func(data)) { ++ ret = -EACCES; ++ goto out_unlock; ++ } ++ ++ size = scnprintf(data->buffer, sizeof(data->buffer), ++ "%llu,%llu,%llu", data->active_value, ++ data->pool_value, data->destroy_value); ++ } ++ ++ ret = simple_read_from_buffer(buf, len, ppos, data->buffer, size); ++ ++out_unlock: ++ mutex_unlock(&data->lock); ++ return ret; ++} ++ ++static int kbase_jit_debugfs_common_release(struct inode *inode, ++ struct file *file) ++{ ++ kfree(file->private_data); ++ return 0; ++} ++ ++#define KBASE_JIT_DEBUGFS_DECLARE(__fops, __func) \ ++static int __fops ## _open(struct inode *inode, struct file *file) \ ++{ \ ++ return kbase_jit_debugfs_common_open(inode, file, __func); \ ++} \ ++static const struct file_operations __fops = { \ ++ .owner = THIS_MODULE, \ ++ .open = __fops ## _open, \ ++ .release = kbase_jit_debugfs_common_release, \ ++ .read = kbase_jit_debugfs_common_read, \ ++ .write = NULL, \ ++ .llseek = generic_file_llseek, \ ++} ++ ++static int kbase_jit_debugfs_count_get(struct kbase_jit_debugfs_data *data) ++{ ++ struct kbase_context *kctx = data->kctx; ++ struct list_head *tmp; ++ ++ mutex_lock(&kctx->jit_evict_lock); ++ list_for_each(tmp, &kctx->jit_active_head) { ++ data->active_value++; ++ } ++ ++ list_for_each(tmp, &kctx->jit_pool_head) { ++ data->pool_value++; ++ } ++ ++ list_for_each(tmp, &kctx->jit_destroy_head) { ++ data->destroy_value++; ++ } ++ mutex_unlock(&kctx->jit_evict_lock); ++ ++ return 0; ++} ++KBASE_JIT_DEBUGFS_DECLARE(kbase_jit_debugfs_count_fops, ++ kbase_jit_debugfs_count_get); ++ ++static int kbase_jit_debugfs_vm_get(struct kbase_jit_debugfs_data *data) ++{ ++ struct kbase_context *kctx = data->kctx; ++ struct kbase_va_region *reg; ++ ++ mutex_lock(&kctx->jit_evict_lock); ++ list_for_each_entry(reg, &kctx->jit_active_head, jit_node) { ++ data->active_value += reg->nr_pages; ++ } ++ ++ list_for_each_entry(reg, &kctx->jit_pool_head, jit_node) { ++ data->pool_value += reg->nr_pages; ++ } ++ ++ list_for_each_entry(reg, &kctx->jit_destroy_head, jit_node) { ++ data->destroy_value += reg->nr_pages; ++ } ++ mutex_unlock(&kctx->jit_evict_lock); ++ ++ return 0; ++} ++KBASE_JIT_DEBUGFS_DECLARE(kbase_jit_debugfs_vm_fops, ++ kbase_jit_debugfs_vm_get); ++ ++static int kbase_jit_debugfs_phys_get(struct kbase_jit_debugfs_data *data) ++{ ++ struct kbase_context *kctx = data->kctx; ++ struct kbase_va_region *reg; ++ ++ mutex_lock(&kctx->jit_evict_lock); ++ list_for_each_entry(reg, &kctx->jit_active_head, jit_node) { ++ data->active_value += reg->gpu_alloc->nents; ++ } ++ ++ list_for_each_entry(reg, &kctx->jit_pool_head, jit_node) { ++ data->pool_value += reg->gpu_alloc->nents; ++ } ++ ++ list_for_each_entry(reg, &kctx->jit_destroy_head, jit_node) { ++ data->destroy_value += reg->gpu_alloc->nents; ++ } ++ mutex_unlock(&kctx->jit_evict_lock); ++ ++ return 0; ++} ++KBASE_JIT_DEBUGFS_DECLARE(kbase_jit_debugfs_phys_fops, ++ kbase_jit_debugfs_phys_get); ++ ++void kbase_jit_debugfs_init(struct kbase_context *kctx) ++{ ++ /* Debugfs entry for getting the number of JIT allocations. */ ++ debugfs_create_file("mem_jit_count", S_IRUGO, kctx->kctx_dentry, ++ kctx, &kbase_jit_debugfs_count_fops); ++ ++ /* ++ * Debugfs entry for getting the total number of virtual pages ++ * used by JIT allocations. ++ */ ++ debugfs_create_file("mem_jit_vm", S_IRUGO, kctx->kctx_dentry, ++ kctx, &kbase_jit_debugfs_vm_fops); ++ ++ /* ++ * Debugfs entry for getting the number of physical pages used ++ * by JIT allocations. ++ */ ++ debugfs_create_file("mem_jit_phys", S_IRUGO, kctx->kctx_dentry, ++ kctx, &kbase_jit_debugfs_phys_fops); ++} ++#endif /* CONFIG_DEBUG_FS */ ++ ++/** ++ * kbase_jit_destroy_worker - Deferred worker which frees JIT allocations ++ * @work: Work item ++ * ++ * This function does the work of freeing JIT allocations whose physical ++ * backing has been released. ++ */ ++static void kbase_jit_destroy_worker(struct work_struct *work) ++{ ++ struct kbase_context *kctx; ++ struct kbase_va_region *reg; ++ ++ kctx = container_of(work, struct kbase_context, jit_work); ++ do { ++ mutex_lock(&kctx->jit_evict_lock); ++ if (list_empty(&kctx->jit_destroy_head)) { ++ mutex_unlock(&kctx->jit_evict_lock); ++ break; ++ } ++ ++ reg = list_first_entry(&kctx->jit_destroy_head, ++ struct kbase_va_region, jit_node); ++ ++ list_del(®->jit_node); ++ mutex_unlock(&kctx->jit_evict_lock); ++ ++ kbase_gpu_vm_lock(kctx); ++ kbase_mem_free_region(kctx, reg); ++ kbase_gpu_vm_unlock(kctx); ++ } while (1); ++} ++ ++int kbase_jit_init(struct kbase_context *kctx) ++{ ++ INIT_LIST_HEAD(&kctx->jit_active_head); ++ INIT_LIST_HEAD(&kctx->jit_pool_head); ++ INIT_LIST_HEAD(&kctx->jit_destroy_head); ++ INIT_WORK(&kctx->jit_work, kbase_jit_destroy_worker); ++ ++ INIT_LIST_HEAD(&kctx->jit_pending_alloc); ++ INIT_LIST_HEAD(&kctx->jit_atoms_head); ++ ++ return 0; ++} ++ ++struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx, ++ struct base_jit_alloc_info *info) ++{ ++ struct kbase_va_region *reg = NULL; ++ struct kbase_va_region *walker; ++ struct kbase_va_region *temp; ++ size_t current_diff = SIZE_MAX; ++ ++ int ret; ++ ++ mutex_lock(&kctx->jit_evict_lock); ++ /* ++ * Scan the pool for an existing allocation which meets our ++ * requirements and remove it. ++ */ ++ list_for_each_entry_safe(walker, temp, &kctx->jit_pool_head, jit_node) { ++ ++ if (walker->nr_pages >= info->va_pages) { ++ size_t min_size, max_size, diff; ++ ++ /* ++ * The JIT allocations VA requirements have been ++ * meet, it's suitable but other allocations ++ * might be a better fit. ++ */ ++ min_size = min_t(size_t, walker->gpu_alloc->nents, ++ info->commit_pages); ++ max_size = max_t(size_t, walker->gpu_alloc->nents, ++ info->commit_pages); ++ diff = max_size - min_size; ++ ++ if (current_diff > diff) { ++ current_diff = diff; ++ reg = walker; ++ } ++ ++ /* The allocation is an exact match, stop looking */ ++ if (current_diff == 0) ++ break; ++ } ++ } ++ ++ if (reg) { ++ /* ++ * Remove the found region from the pool and add it to the ++ * active list. ++ */ ++ list_move(®->jit_node, &kctx->jit_active_head); ++ ++ /* ++ * Remove the allocation from the eviction list as it's no ++ * longer eligible for eviction. This must be done before ++ * dropping the jit_evict_lock ++ */ ++ list_del_init(®->gpu_alloc->evict_node); ++ mutex_unlock(&kctx->jit_evict_lock); ++ ++ kbase_gpu_vm_lock(kctx); ++ ++ /* Make the physical backing no longer reclaimable */ ++ if (!kbase_mem_evictable_unmake(reg->gpu_alloc)) ++ goto update_failed; ++ ++ /* Grow the backing if required */ ++ if (reg->gpu_alloc->nents < info->commit_pages) { ++ size_t delta; ++ size_t old_size = reg->gpu_alloc->nents; ++ ++ /* Allocate some more pages */ ++ delta = info->commit_pages - reg->gpu_alloc->nents; ++ if (kbase_alloc_phy_pages_helper(reg->gpu_alloc, delta) ++ != 0) ++ goto update_failed; ++ ++ if (reg->cpu_alloc != reg->gpu_alloc) { ++ if (kbase_alloc_phy_pages_helper( ++ reg->cpu_alloc, delta) != 0) { ++ kbase_free_phy_pages_helper( ++ reg->gpu_alloc, delta); ++ goto update_failed; ++ } ++ } ++ ++ ret = kbase_mem_grow_gpu_mapping(kctx, reg, ++ info->commit_pages, old_size); ++ /* ++ * The grow failed so put the allocation back in the ++ * pool and return failure. ++ */ ++ if (ret) ++ goto update_failed; ++ } ++ kbase_gpu_vm_unlock(kctx); ++ } else { ++ /* No suitable JIT allocation was found so create a new one */ ++ u64 flags = BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_GPU_RD | ++ BASE_MEM_PROT_GPU_WR | BASE_MEM_GROW_ON_GPF | ++ BASE_MEM_COHERENT_LOCAL; ++ u64 gpu_addr; ++ ++ mutex_unlock(&kctx->jit_evict_lock); ++ ++ reg = kbase_mem_alloc(kctx, info->va_pages, info->commit_pages, ++ info->extent, &flags, &gpu_addr); ++ if (!reg) ++ goto out_unlocked; ++ ++ mutex_lock(&kctx->jit_evict_lock); ++ list_add(®->jit_node, &kctx->jit_active_head); ++ mutex_unlock(&kctx->jit_evict_lock); ++ } ++ ++ return reg; ++ ++update_failed: ++ /* ++ * An update to an allocation from the pool failed, chances ++ * are slim a new allocation would fair any better so return ++ * the allocation to the pool and return the function with failure. ++ */ ++ kbase_gpu_vm_unlock(kctx); ++ mutex_lock(&kctx->jit_evict_lock); ++ list_move(®->jit_node, &kctx->jit_pool_head); ++ mutex_unlock(&kctx->jit_evict_lock); ++out_unlocked: ++ return NULL; ++} ++ ++void kbase_jit_free(struct kbase_context *kctx, struct kbase_va_region *reg) ++{ ++ /* The physical backing of memory in the pool is always reclaimable */ ++ kbase_gpu_vm_lock(kctx); ++ kbase_mem_evictable_make(reg->gpu_alloc); ++ kbase_gpu_vm_unlock(kctx); ++ ++ mutex_lock(&kctx->jit_evict_lock); ++ list_move(®->jit_node, &kctx->jit_pool_head); ++ mutex_unlock(&kctx->jit_evict_lock); ++} ++ ++void kbase_jit_backing_lost(struct kbase_va_region *reg) ++{ ++ struct kbase_context *kctx = reg->kctx; ++ ++ lockdep_assert_held(&kctx->jit_evict_lock); ++ ++ /* ++ * JIT allocations will always be on a list, if the region ++ * is not on a list then it's not a JIT allocation. ++ */ ++ if (list_empty(®->jit_node)) ++ return; ++ ++ /* ++ * Freeing the allocation requires locks we might not be able ++ * to take now, so move the allocation to the free list and kick ++ * the worker which will do the freeing. ++ */ ++ list_move(®->jit_node, &kctx->jit_destroy_head); ++ ++ schedule_work(&kctx->jit_work); ++} ++ ++bool kbase_jit_evict(struct kbase_context *kctx) ++{ ++ struct kbase_va_region *reg = NULL; ++ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ /* Free the oldest allocation from the pool */ ++ mutex_lock(&kctx->jit_evict_lock); ++ if (!list_empty(&kctx->jit_pool_head)) { ++ reg = list_entry(kctx->jit_pool_head.prev, ++ struct kbase_va_region, jit_node); ++ list_del(®->jit_node); ++ } ++ mutex_unlock(&kctx->jit_evict_lock); ++ ++ if (reg) ++ kbase_mem_free_region(kctx, reg); ++ ++ return (reg != NULL); ++} ++ ++void kbase_jit_term(struct kbase_context *kctx) ++{ ++ struct kbase_va_region *walker; ++ ++ /* Free all allocations for this context */ ++ ++ /* ++ * Flush the freeing of allocations whose backing has been freed ++ * (i.e. everything in jit_destroy_head). ++ */ ++ cancel_work_sync(&kctx->jit_work); ++ ++ kbase_gpu_vm_lock(kctx); ++ mutex_lock(&kctx->jit_evict_lock); ++ /* Free all allocations from the pool */ ++ while (!list_empty(&kctx->jit_pool_head)) { ++ walker = list_first_entry(&kctx->jit_pool_head, ++ struct kbase_va_region, jit_node); ++ list_del(&walker->jit_node); ++ mutex_unlock(&kctx->jit_evict_lock); ++ kbase_mem_free_region(kctx, walker); ++ mutex_lock(&kctx->jit_evict_lock); ++ } ++ ++ /* Free all allocations from active list */ ++ while (!list_empty(&kctx->jit_active_head)) { ++ walker = list_first_entry(&kctx->jit_active_head, ++ struct kbase_va_region, jit_node); ++ list_del(&walker->jit_node); ++ mutex_unlock(&kctx->jit_evict_lock); ++ kbase_mem_free_region(kctx, walker); ++ mutex_lock(&kctx->jit_evict_lock); ++ } ++ mutex_unlock(&kctx->jit_evict_lock); ++ kbase_gpu_vm_unlock(kctx); ++} ++ ++static int kbase_jd_user_buf_map(struct kbase_context *kctx, ++ struct kbase_va_region *reg) ++{ ++ long pinned_pages; ++ struct kbase_mem_phy_alloc *alloc; ++ struct page **pages; ++ phys_addr_t *pa; ++ long i; ++ int err = -ENOMEM; ++ unsigned long address; ++ struct mm_struct *mm; ++ struct device *dev; ++ unsigned long offset; ++ unsigned long local_size; ++ ++ alloc = reg->gpu_alloc; ++ pa = kbase_get_gpu_phy_pages(reg); ++ address = alloc->imported.user_buf.address; ++ mm = alloc->imported.user_buf.mm; ++ ++ KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_IMPORTED_USER_BUF); ++ ++ pages = alloc->imported.user_buf.pages; ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) ++ pinned_pages = get_user_pages(NULL, mm, ++ address, ++ alloc->imported.user_buf.nr_pages, ++ reg->flags & KBASE_REG_GPU_WR, ++ 0, pages, NULL); ++#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) ++ pinned_pages = get_user_pages_remote(NULL, mm, ++ address, ++ alloc->imported.user_buf.nr_pages, ++ reg->flags & KBASE_REG_GPU_WR, ++ 0, pages, NULL); ++#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) ++ pinned_pages = get_user_pages_remote(NULL, mm, ++ address, ++ alloc->imported.user_buf.nr_pages, ++ reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0, ++ pages, NULL); ++#elif LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0) ++ pinned_pages = get_user_pages_remote(NULL, mm, ++ address, ++ alloc->imported.user_buf.nr_pages, ++ reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0, ++ pages, NULL, NULL); ++#else ++ pinned_pages = get_user_pages_remote(mm, ++ address, ++ alloc->imported.user_buf.nr_pages, ++ reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0, ++ pages, NULL, NULL); ++#endif ++ ++ if (pinned_pages <= 0) ++ return pinned_pages; ++ ++ if (pinned_pages != alloc->imported.user_buf.nr_pages) { ++ for (i = 0; i < pinned_pages; i++) ++ put_page(pages[i]); ++ return -ENOMEM; ++ } ++ ++ dev = kctx->kbdev->dev; ++ offset = address & ~PAGE_MASK; ++ local_size = alloc->imported.user_buf.size; ++ ++ for (i = 0; i < pinned_pages; i++) { ++ dma_addr_t dma_addr; ++ unsigned long min; ++ ++ min = MIN(PAGE_SIZE - offset, local_size); ++ dma_addr = dma_map_page(dev, pages[i], ++ offset, min, ++ DMA_BIDIRECTIONAL); ++ if (dma_mapping_error(dev, dma_addr)) ++ goto unwind; ++ ++ alloc->imported.user_buf.dma_addrs[i] = dma_addr; ++ pa[i] = page_to_phys(pages[i]); ++ ++ local_size -= min; ++ offset = 0; ++ } ++ ++ alloc->nents = pinned_pages; ++ ++ err = kbase_mmu_insert_pages(kctx, reg->start_pfn, pa, ++ kbase_reg_current_backed_size(reg), ++ reg->flags); ++ if (err == 0) ++ return 0; ++ ++ alloc->nents = 0; ++ /* fall down */ ++unwind: ++ while (i--) { ++ dma_unmap_page(kctx->kbdev->dev, ++ alloc->imported.user_buf.dma_addrs[i], ++ PAGE_SIZE, DMA_BIDIRECTIONAL); ++ put_page(pages[i]); ++ pages[i] = NULL; ++ } ++ ++ return err; ++} ++ ++static void kbase_jd_user_buf_unmap(struct kbase_context *kctx, ++ struct kbase_mem_phy_alloc *alloc, bool writeable) ++{ ++ long i; ++ struct page **pages; ++ unsigned long size = alloc->imported.user_buf.size; ++ ++ KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_IMPORTED_USER_BUF); ++ pages = alloc->imported.user_buf.pages; ++ for (i = 0; i < alloc->imported.user_buf.nr_pages; i++) { ++ unsigned long local_size; ++ dma_addr_t dma_addr = alloc->imported.user_buf.dma_addrs[i]; ++ ++ local_size = MIN(size, PAGE_SIZE - (dma_addr & ~PAGE_MASK)); ++ dma_unmap_page(kctx->kbdev->dev, dma_addr, local_size, ++ DMA_BIDIRECTIONAL); ++ if (writeable) ++ set_page_dirty_lock(pages[i]); ++ put_page(pages[i]); ++ pages[i] = NULL; ++ ++ size -= local_size; ++ } ++ alloc->nents = 0; ++} ++ ++ ++/* to replace sg_dma_len. */ ++#define MALI_SG_DMA_LEN(sg) ((sg)->length) ++ ++#ifdef CONFIG_DMA_SHARED_BUFFER ++static int kbase_jd_umm_map(struct kbase_context *kctx, ++ struct kbase_va_region *reg) ++{ ++ struct sg_table *sgt; ++ struct scatterlist *s; ++ int i; ++ phys_addr_t *pa; ++ int err; ++ size_t count = 0; ++ struct kbase_mem_phy_alloc *alloc; ++ ++ alloc = reg->gpu_alloc; ++ ++ KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_IMPORTED_UMM); ++ KBASE_DEBUG_ASSERT(NULL == alloc->imported.umm.sgt); ++ sgt = dma_buf_map_attachment(alloc->imported.umm.dma_attachment, ++ DMA_BIDIRECTIONAL); ++ ++ if (IS_ERR_OR_NULL(sgt)) ++ return -EINVAL; ++ ++ /* save for later */ ++ alloc->imported.umm.sgt = sgt; ++ ++ pa = kbase_get_gpu_phy_pages(reg); ++ KBASE_DEBUG_ASSERT(pa); ++ ++ for_each_sg(sgt->sgl, s, sgt->nents, i) { ++ int j; ++ size_t pages = PFN_UP(MALI_SG_DMA_LEN(s)); ++ ++ WARN_ONCE(MALI_SG_DMA_LEN(s) & (PAGE_SIZE-1), ++ "MALI_SG_DMA_LEN(s)=%u is not a multiple of PAGE_SIZE\n", ++ MALI_SG_DMA_LEN(s)); ++ ++ WARN_ONCE(sg_dma_address(s) & (PAGE_SIZE-1), ++ "sg_dma_address(s)=%llx is not aligned to PAGE_SIZE\n", ++ (unsigned long long) sg_dma_address(s)); ++ ++ for (j = 0; (j < pages) && (count < reg->nr_pages); j++, ++ count++) ++ *pa++ = sg_dma_address(s) + (j << PAGE_SHIFT); ++ WARN_ONCE(j < pages, ++ "sg list from dma_buf_map_attachment > dma_buf->size=%zu\n", ++ alloc->imported.umm.dma_buf->size); ++ } ++ ++ if (!(reg->flags & KBASE_REG_IMPORT_PAD) && ++ WARN_ONCE(count < reg->nr_pages, ++ "sg list from dma_buf_map_attachment < dma_buf->size=%zu\n", ++ alloc->imported.umm.dma_buf->size)) { ++ err = -EINVAL; ++ goto err_unmap_attachment; ++ } ++ ++ /* Update nents as we now have pages to map */ ++ alloc->nents = reg->nr_pages; ++ ++ err = kbase_mmu_insert_pages(kctx, reg->start_pfn, ++ kbase_get_gpu_phy_pages(reg), ++ count, ++ reg->flags | KBASE_REG_GPU_WR | KBASE_REG_GPU_RD); ++ if (err) ++ goto err_unmap_attachment; ++ ++ if (reg->flags & KBASE_REG_IMPORT_PAD) { ++ err = kbase_mmu_insert_single_page(kctx, ++ reg->start_pfn + count, ++ page_to_phys(kctx->aliasing_sink_page), ++ reg->nr_pages - count, ++ (reg->flags | KBASE_REG_GPU_RD) & ++ ~KBASE_REG_GPU_WR); ++ if (err) ++ goto err_teardown_orig_pages; ++ } ++ ++ return 0; ++ ++err_teardown_orig_pages: ++ kbase_mmu_teardown_pages(kctx, reg->start_pfn, count); ++err_unmap_attachment: ++ dma_buf_unmap_attachment(alloc->imported.umm.dma_attachment, ++ alloc->imported.umm.sgt, DMA_BIDIRECTIONAL); ++ alloc->imported.umm.sgt = NULL; ++ ++ return err; ++} ++ ++static void kbase_jd_umm_unmap(struct kbase_context *kctx, ++ struct kbase_mem_phy_alloc *alloc) ++{ ++ KBASE_DEBUG_ASSERT(kctx); ++ KBASE_DEBUG_ASSERT(alloc); ++ KBASE_DEBUG_ASSERT(alloc->imported.umm.dma_attachment); ++ KBASE_DEBUG_ASSERT(alloc->imported.umm.sgt); ++ dma_buf_unmap_attachment(alloc->imported.umm.dma_attachment, ++ alloc->imported.umm.sgt, DMA_BIDIRECTIONAL); ++ alloc->imported.umm.sgt = NULL; ++ alloc->nents = 0; ++} ++#endif /* CONFIG_DMA_SHARED_BUFFER */ ++ ++#if (defined(CONFIG_KDS) && defined(CONFIG_UMP)) \ ++ || defined(CONFIG_DMA_SHARED_BUFFER_USES_KDS) ++static void add_kds_resource(struct kds_resource *kds_res, ++ struct kds_resource **kds_resources, u32 *kds_res_count, ++ unsigned long *kds_access_bitmap, bool exclusive) ++{ ++ u32 i; ++ ++ for (i = 0; i < *kds_res_count; i++) { ++ /* Duplicate resource, ignore */ ++ if (kds_resources[i] == kds_res) ++ return; ++ } ++ ++ kds_resources[*kds_res_count] = kds_res; ++ if (exclusive) ++ set_bit(*kds_res_count, kds_access_bitmap); ++ (*kds_res_count)++; ++} ++#endif ++ ++struct kbase_mem_phy_alloc *kbase_map_external_resource( ++ struct kbase_context *kctx, struct kbase_va_region *reg, ++ struct mm_struct *locked_mm ++#ifdef CONFIG_KDS ++ , u32 *kds_res_count, struct kds_resource **kds_resources, ++ unsigned long *kds_access_bitmap, bool exclusive ++#endif ++ ) ++{ ++ int err; ++ ++ /* decide what needs to happen for this resource */ ++ switch (reg->gpu_alloc->type) { ++ case KBASE_MEM_TYPE_IMPORTED_USER_BUF: { ++ if (reg->gpu_alloc->imported.user_buf.mm != locked_mm) ++ goto exit; ++ ++ reg->gpu_alloc->imported.user_buf.current_mapping_usage_count++; ++ if (1 == reg->gpu_alloc->imported.user_buf.current_mapping_usage_count) { ++ err = kbase_jd_user_buf_map(kctx, reg); ++ if (err) { ++ reg->gpu_alloc->imported.user_buf.current_mapping_usage_count--; ++ goto exit; ++ } ++ } ++ } ++ break; ++ case KBASE_MEM_TYPE_IMPORTED_UMP: { ++#if defined(CONFIG_KDS) && defined(CONFIG_UMP) ++ if (kds_res_count) { ++ struct kds_resource *kds_res; ++ ++ kds_res = ump_dd_kds_resource_get( ++ reg->gpu_alloc->imported.ump_handle); ++ if (kds_res) ++ add_kds_resource(kds_res, kds_resources, ++ kds_res_count, ++ kds_access_bitmap, exclusive); ++ } ++#endif /*defined(CONFIG_KDS) && defined(CONFIG_UMP) */ ++ break; ++ } ++#ifdef CONFIG_DMA_SHARED_BUFFER ++ case KBASE_MEM_TYPE_IMPORTED_UMM: { ++#ifdef CONFIG_DMA_SHARED_BUFFER_USES_KDS ++ if (kds_res_count) { ++ struct kds_resource *kds_res; ++ ++ kds_res = get_dma_buf_kds_resource( ++ reg->gpu_alloc->imported.umm.dma_buf); ++ if (kds_res) ++ add_kds_resource(kds_res, kds_resources, ++ kds_res_count, ++ kds_access_bitmap, exclusive); ++ } ++#endif ++ reg->gpu_alloc->imported.umm.current_mapping_usage_count++; ++ if (1 == reg->gpu_alloc->imported.umm.current_mapping_usage_count) { ++ err = kbase_jd_umm_map(kctx, reg); ++ if (err) { ++ reg->gpu_alloc->imported.umm.current_mapping_usage_count--; ++ goto exit; ++ } ++ } ++ break; ++ } ++#endif ++ default: ++ goto exit; ++ } ++ ++ return kbase_mem_phy_alloc_get(reg->gpu_alloc); ++exit: ++ return NULL; ++} ++ ++void kbase_unmap_external_resource(struct kbase_context *kctx, ++ struct kbase_va_region *reg, struct kbase_mem_phy_alloc *alloc) ++{ ++ switch (alloc->type) { ++#ifdef CONFIG_DMA_SHARED_BUFFER ++ case KBASE_MEM_TYPE_IMPORTED_UMM: { ++ alloc->imported.umm.current_mapping_usage_count--; ++ ++ if (0 == alloc->imported.umm.current_mapping_usage_count) { ++ if (reg && reg->gpu_alloc == alloc) { ++ int err; ++ ++ err = kbase_mmu_teardown_pages( ++ kctx, ++ reg->start_pfn, ++ alloc->nents); ++ WARN_ON(err); ++ } ++ ++ kbase_jd_umm_unmap(kctx, alloc); ++ } ++ } ++ break; ++#endif /* CONFIG_DMA_SHARED_BUFFER */ ++ case KBASE_MEM_TYPE_IMPORTED_USER_BUF: { ++ alloc->imported.user_buf.current_mapping_usage_count--; ++ ++ if (0 == alloc->imported.user_buf.current_mapping_usage_count) { ++ bool writeable = true; ++ ++ if (reg && reg->gpu_alloc == alloc) ++ kbase_mmu_teardown_pages( ++ kctx, ++ reg->start_pfn, ++ kbase_reg_current_backed_size(reg)); ++ ++ if (reg && ((reg->flags & KBASE_REG_GPU_WR) == 0)) ++ writeable = false; ++ ++ kbase_jd_user_buf_unmap(kctx, alloc, writeable); ++ } ++ } ++ break; ++ default: ++ break; ++ } ++ kbase_mem_phy_alloc_put(alloc); ++} ++ ++struct kbase_ctx_ext_res_meta *kbase_sticky_resource_acquire( ++ struct kbase_context *kctx, u64 gpu_addr) ++{ ++ struct kbase_ctx_ext_res_meta *meta = NULL; ++ struct kbase_ctx_ext_res_meta *walker; ++ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ /* ++ * Walk the per context external resource metadata list for the ++ * metadata which matches the region which is being acquired. ++ */ ++ list_for_each_entry(walker, &kctx->ext_res_meta_head, ext_res_node) { ++ if (walker->gpu_addr == gpu_addr) { ++ meta = walker; ++ break; ++ } ++ } ++ ++ /* No metadata exists so create one. */ ++ if (!meta) { ++ struct kbase_va_region *reg; ++ ++ /* Find the region */ ++ reg = kbase_region_tracker_find_region_enclosing_address( ++ kctx, gpu_addr); ++ if (NULL == reg || (reg->flags & KBASE_REG_FREE)) ++ goto failed; ++ ++ /* Allocate the metadata object */ ++ meta = kzalloc(sizeof(*meta), GFP_KERNEL); ++ if (!meta) ++ goto failed; ++ ++ /* ++ * Fill in the metadata object and acquire a reference ++ * for the physical resource. ++ */ ++ meta->alloc = kbase_map_external_resource(kctx, reg, NULL ++#ifdef CONFIG_KDS ++ , NULL, NULL, ++ NULL, false ++#endif ++ ); ++ ++ if (!meta->alloc) ++ goto fail_map; ++ ++ meta->gpu_addr = reg->start_pfn << PAGE_SHIFT; ++ ++ list_add(&meta->ext_res_node, &kctx->ext_res_meta_head); ++ } ++ ++ return meta; ++ ++fail_map: ++ kfree(meta); ++failed: ++ return NULL; ++} ++ ++bool kbase_sticky_resource_release(struct kbase_context *kctx, ++ struct kbase_ctx_ext_res_meta *meta, u64 gpu_addr) ++{ ++ struct kbase_ctx_ext_res_meta *walker; ++ struct kbase_va_region *reg; ++ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ /* Search of the metadata if one isn't provided. */ ++ if (!meta) { ++ /* ++ * Walk the per context external resource metadata list for the ++ * metadata which matches the region which is being released. ++ */ ++ list_for_each_entry(walker, &kctx->ext_res_meta_head, ++ ext_res_node) { ++ if (walker->gpu_addr == gpu_addr) { ++ meta = walker; ++ break; ++ } ++ } ++ } ++ ++ /* No metadata so just return. */ ++ if (!meta) ++ return false; ++ ++ /* Drop the physical memory reference and free the metadata. */ ++ reg = kbase_region_tracker_find_region_enclosing_address( ++ kctx, ++ meta->gpu_addr); ++ ++ kbase_unmap_external_resource(kctx, reg, meta->alloc); ++ list_del(&meta->ext_res_node); ++ kfree(meta); ++ ++ return true; ++} ++ ++int kbase_sticky_resource_init(struct kbase_context *kctx) ++{ ++ INIT_LIST_HEAD(&kctx->ext_res_meta_head); ++ ++ return 0; ++} ++ ++void kbase_sticky_resource_term(struct kbase_context *kctx) ++{ ++ struct kbase_ctx_ext_res_meta *walker; ++ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ /* ++ * Free any sticky resources which haven't been unmapped. ++ * ++ * Note: ++ * We don't care about refcounts at this point as no future ++ * references to the meta data will be made. ++ * Region termination would find these if we didn't free them ++ * here, but it's more efficient if we do the clean up here. ++ */ ++ while (!list_empty(&kctx->ext_res_meta_head)) { ++ walker = list_first_entry(&kctx->ext_res_meta_head, ++ struct kbase_ctx_ext_res_meta, ext_res_node); ++ ++ kbase_sticky_resource_release(kctx, walker, 0); ++ } ++} +diff --git a/drivers/gpu/arm/midgard/mali_kbase_mem.h b/drivers/gpu/arm/midgard/mali_kbase_mem.h +new file mode 100755 +index 000000000000..3f3eaa3fda98 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_mem.h +@@ -0,0 +1,1068 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/** ++ * @file mali_kbase_mem.h ++ * Base kernel memory APIs ++ */ ++ ++#ifndef _KBASE_MEM_H_ ++#define _KBASE_MEM_H_ ++ ++#ifndef _KBASE_H_ ++#error "Don't include this file directly, use mali_kbase.h instead" ++#endif ++ ++#include ++#ifdef CONFIG_KDS ++#include ++#endif /* CONFIG_KDS */ ++#ifdef CONFIG_UMP ++#include ++#endif /* CONFIG_UMP */ ++#include "mali_base_kernel.h" ++#include ++#include "mali_kbase_pm.h" ++#include "mali_kbase_defs.h" ++#if defined(CONFIG_MALI_GATOR_SUPPORT) ++#include "mali_kbase_gator.h" ++#endif ++/* Required for kbase_mem_evictable_unmake */ ++#include "mali_kbase_mem_linux.h" ++ ++/* Part of the workaround for uTLB invalid pages is to ensure we grow/shrink tmem by 4 pages at a time */ ++#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_8316 (2) /* round to 4 pages */ ++ ++/* Part of the workaround for PRLAM-9630 requires us to grow/shrink memory by 8 pages. ++The MMU reads in 8 page table entries from memory at a time, if we have more than one page fault within the same 8 pages and ++page tables are updated accordingly, the MMU does not re-read the page table entries from memory for the subsequent page table ++updates and generates duplicate page faults as the page table information used by the MMU is not valid. */ ++#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_9630 (3) /* round to 8 pages */ ++ ++#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2 (0) /* round to 1 page */ ++ ++/* This must always be a power of 2 */ ++#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES (1u << KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2) ++#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_HW_ISSUE_8316 (1u << KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_8316) ++#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_HW_ISSUE_9630 (1u << KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_9630) ++/** ++ * A CPU mapping ++ */ ++struct kbase_cpu_mapping { ++ struct list_head mappings_list; ++ struct kbase_mem_phy_alloc *alloc; ++ struct kbase_context *kctx; ++ struct kbase_va_region *region; ++ int count; ++ int free_on_close; ++}; ++ ++enum kbase_memory_type { ++ KBASE_MEM_TYPE_NATIVE, ++ KBASE_MEM_TYPE_IMPORTED_UMP, ++ KBASE_MEM_TYPE_IMPORTED_UMM, ++ KBASE_MEM_TYPE_IMPORTED_USER_BUF, ++ KBASE_MEM_TYPE_ALIAS, ++ KBASE_MEM_TYPE_TB, ++ KBASE_MEM_TYPE_RAW ++}; ++ ++/* internal structure, mirroring base_mem_aliasing_info, ++ * but with alloc instead of a gpu va (handle) */ ++struct kbase_aliased { ++ struct kbase_mem_phy_alloc *alloc; /* NULL for special, non-NULL for native */ ++ u64 offset; /* in pages */ ++ u64 length; /* in pages */ ++}; ++ ++/** ++ * @brief Physical pages tracking object properties ++ */ ++#define KBASE_MEM_PHY_ALLOC_ACCESSED_CACHED (1ul << 0) ++#define KBASE_MEM_PHY_ALLOC_LARGE (1ul << 1) ++ ++/* physical pages tracking object. ++ * Set up to track N pages. ++ * N not stored here, the creator holds that info. ++ * This object only tracks how many elements are actually valid (present). ++ * Changing of nents or *pages should only happen if the kbase_mem_phy_alloc is not ++ * shared with another region or client. CPU mappings are OK to exist when changing, as ++ * long as the tracked mappings objects are updated as part of the change. ++ */ ++struct kbase_mem_phy_alloc { ++ struct kref kref; /* number of users of this alloc */ ++ atomic_t gpu_mappings; ++ size_t nents; /* 0..N */ ++ phys_addr_t *pages; /* N elements, only 0..nents are valid */ ++ ++ /* kbase_cpu_mappings */ ++ struct list_head mappings; ++ ++ /* Node used to store this allocation on the eviction list */ ++ struct list_head evict_node; ++ /* Physical backing size when the pages where evicted */ ++ size_t evicted; ++ /* ++ * Back reference to the region structure which created this ++ * allocation, or NULL if it has been freed. ++ */ ++ struct kbase_va_region *reg; ++ ++ /* type of buffer */ ++ enum kbase_memory_type type; ++ ++ unsigned long properties; ++ ++ /* member in union valid based on @a type */ ++ union { ++#ifdef CONFIG_UMP ++ ump_dd_handle ump_handle; ++#endif /* CONFIG_UMP */ ++#if defined(CONFIG_DMA_SHARED_BUFFER) ++ struct { ++ struct dma_buf *dma_buf; ++ struct dma_buf_attachment *dma_attachment; ++ unsigned int current_mapping_usage_count; ++ struct sg_table *sgt; ++ } umm; ++#endif /* defined(CONFIG_DMA_SHARED_BUFFER) */ ++ struct { ++ u64 stride; ++ size_t nents; ++ struct kbase_aliased *aliased; ++ } alias; ++ /* Used by type = (KBASE_MEM_TYPE_NATIVE, KBASE_MEM_TYPE_TB) */ ++ struct kbase_context *kctx; ++ struct kbase_alloc_import_user_buf { ++ unsigned long address; ++ unsigned long size; ++ unsigned long nr_pages; ++ struct page **pages; ++ /* top bit (1<<31) of current_mapping_usage_count ++ * specifies that this import was pinned on import ++ * See PINNED_ON_IMPORT ++ */ ++ u32 current_mapping_usage_count; ++ struct mm_struct *mm; ++ dma_addr_t *dma_addrs; ++ } user_buf; ++ } imported; ++}; ++ ++/* The top bit of kbase_alloc_import_user_buf::current_mapping_usage_count is ++ * used to signify that a buffer was pinned when it was imported. Since the ++ * reference count is limited by the number of atoms that can be submitted at ++ * once there should be no danger of overflowing into this bit. ++ * Stealing the top bit also has the benefit that ++ * current_mapping_usage_count != 0 if and only if the buffer is mapped. ++ */ ++#define PINNED_ON_IMPORT (1<<31) ++ ++static inline void kbase_mem_phy_alloc_gpu_mapped(struct kbase_mem_phy_alloc *alloc) ++{ ++ KBASE_DEBUG_ASSERT(alloc); ++ /* we only track mappings of NATIVE buffers */ ++ if (alloc->type == KBASE_MEM_TYPE_NATIVE) ++ atomic_inc(&alloc->gpu_mappings); ++} ++ ++static inline void kbase_mem_phy_alloc_gpu_unmapped(struct kbase_mem_phy_alloc *alloc) ++{ ++ KBASE_DEBUG_ASSERT(alloc); ++ /* we only track mappings of NATIVE buffers */ ++ if (alloc->type == KBASE_MEM_TYPE_NATIVE) ++ if (0 > atomic_dec_return(&alloc->gpu_mappings)) { ++ pr_err("Mismatched %s:\n", __func__); ++ dump_stack(); ++ } ++} ++ ++void kbase_mem_kref_free(struct kref *kref); ++ ++int kbase_mem_init(struct kbase_device *kbdev); ++void kbase_mem_halt(struct kbase_device *kbdev); ++void kbase_mem_term(struct kbase_device *kbdev); ++ ++static inline struct kbase_mem_phy_alloc *kbase_mem_phy_alloc_get(struct kbase_mem_phy_alloc *alloc) ++{ ++ kref_get(&alloc->kref); ++ return alloc; ++} ++ ++static inline struct kbase_mem_phy_alloc *kbase_mem_phy_alloc_put(struct kbase_mem_phy_alloc *alloc) ++{ ++ kref_put(&alloc->kref, kbase_mem_kref_free); ++ return NULL; ++} ++ ++/** ++ * A GPU memory region, and attributes for CPU mappings. ++ */ ++struct kbase_va_region { ++ struct rb_node rblink; ++ struct list_head link; ++ ++ struct kbase_context *kctx; /* Backlink to base context */ ++ ++ u64 start_pfn; /* The PFN in GPU space */ ++ size_t nr_pages; ++ ++/* Free region */ ++#define KBASE_REG_FREE (1ul << 0) ++/* CPU write access */ ++#define KBASE_REG_CPU_WR (1ul << 1) ++/* GPU write access */ ++#define KBASE_REG_GPU_WR (1ul << 2) ++/* No eXecute flag */ ++#define KBASE_REG_GPU_NX (1ul << 3) ++/* Is CPU cached? */ ++#define KBASE_REG_CPU_CACHED (1ul << 4) ++/* Is GPU cached? */ ++#define KBASE_REG_GPU_CACHED (1ul << 5) ++ ++#define KBASE_REG_GROWABLE (1ul << 6) ++/* Can grow on pf? */ ++#define KBASE_REG_PF_GROW (1ul << 7) ++ ++/* VA managed by us */ ++#define KBASE_REG_CUSTOM_VA (1ul << 8) ++ ++/* inner shareable coherency */ ++#define KBASE_REG_SHARE_IN (1ul << 9) ++/* inner & outer shareable coherency */ ++#define KBASE_REG_SHARE_BOTH (1ul << 10) ++ ++/* Space for 4 different zones */ ++#define KBASE_REG_ZONE_MASK (3ul << 11) ++#define KBASE_REG_ZONE(x) (((x) & 3) << 11) ++ ++/* GPU read access */ ++#define KBASE_REG_GPU_RD (1ul<<13) ++/* CPU read access */ ++#define KBASE_REG_CPU_RD (1ul<<14) ++ ++/* Index of chosen MEMATTR for this region (0..7) */ ++#define KBASE_REG_MEMATTR_MASK (7ul << 16) ++#define KBASE_REG_MEMATTR_INDEX(x) (((x) & 7) << 16) ++#define KBASE_REG_MEMATTR_VALUE(x) (((x) & KBASE_REG_MEMATTR_MASK) >> 16) ++ ++#define KBASE_REG_SECURE (1ul << 19) ++ ++#define KBASE_REG_DONT_NEED (1ul << 20) ++ ++/* Imported buffer is padded? */ ++#define KBASE_REG_IMPORT_PAD (1ul << 21) ++ ++#define KBASE_REG_ZONE_SAME_VA KBASE_REG_ZONE(0) ++ ++/* only used with 32-bit clients */ ++/* ++ * On a 32bit platform, custom VA should be wired from (4GB + shader region) ++ * to the VA limit of the GPU. Unfortunately, the Linux mmap() interface ++ * limits us to 2^32 pages (2^44 bytes, see mmap64 man page for reference). ++ * So we put the default limit to the maximum possible on Linux and shrink ++ * it down, if required by the GPU, during initialization. ++ */ ++ ++/* ++ * Dedicated 16MB region for shader code: ++ * VA range 0x101000000-0x102000000 ++ */ ++#define KBASE_REG_ZONE_EXEC KBASE_REG_ZONE(1) ++#define KBASE_REG_ZONE_EXEC_BASE (0x101000000ULL >> PAGE_SHIFT) ++#define KBASE_REG_ZONE_EXEC_SIZE ((16ULL * 1024 * 1024) >> PAGE_SHIFT) ++ ++#define KBASE_REG_ZONE_CUSTOM_VA KBASE_REG_ZONE(2) ++#define KBASE_REG_ZONE_CUSTOM_VA_BASE (KBASE_REG_ZONE_EXEC_BASE + KBASE_REG_ZONE_EXEC_SIZE) /* Starting after KBASE_REG_ZONE_EXEC */ ++#define KBASE_REG_ZONE_CUSTOM_VA_SIZE (((1ULL << 44) >> PAGE_SHIFT) - KBASE_REG_ZONE_CUSTOM_VA_BASE) ++/* end 32-bit clients only */ ++ ++ unsigned long flags; ++ ++ size_t extent; /* nr of pages alloc'd on PF */ ++ ++ struct kbase_mem_phy_alloc *cpu_alloc; /* the one alloc object we mmap to the CPU when mapping this region */ ++ struct kbase_mem_phy_alloc *gpu_alloc; /* the one alloc object we mmap to the GPU when mapping this region */ ++ ++ /* non-NULL if this memory object is a kds_resource */ ++ struct kds_resource *kds_res; ++ ++ /* List head used to store the region in the JIT allocation pool */ ++ struct list_head jit_node; ++}; ++ ++/* Common functions */ ++static inline phys_addr_t *kbase_get_cpu_phy_pages(struct kbase_va_region *reg) ++{ ++ KBASE_DEBUG_ASSERT(reg); ++ KBASE_DEBUG_ASSERT(reg->cpu_alloc); ++ KBASE_DEBUG_ASSERT(reg->gpu_alloc); ++ KBASE_DEBUG_ASSERT(reg->cpu_alloc->nents == reg->gpu_alloc->nents); ++ ++ return reg->cpu_alloc->pages; ++} ++ ++static inline phys_addr_t *kbase_get_gpu_phy_pages(struct kbase_va_region *reg) ++{ ++ KBASE_DEBUG_ASSERT(reg); ++ KBASE_DEBUG_ASSERT(reg->cpu_alloc); ++ KBASE_DEBUG_ASSERT(reg->gpu_alloc); ++ KBASE_DEBUG_ASSERT(reg->cpu_alloc->nents == reg->gpu_alloc->nents); ++ ++ return reg->gpu_alloc->pages; ++} ++ ++static inline size_t kbase_reg_current_backed_size(struct kbase_va_region *reg) ++{ ++ KBASE_DEBUG_ASSERT(reg); ++ /* if no alloc object the backed size naturally is 0 */ ++ if (!reg->cpu_alloc) ++ return 0; ++ ++ KBASE_DEBUG_ASSERT(reg->cpu_alloc); ++ KBASE_DEBUG_ASSERT(reg->gpu_alloc); ++ KBASE_DEBUG_ASSERT(reg->cpu_alloc->nents == reg->gpu_alloc->nents); ++ ++ return reg->cpu_alloc->nents; ++} ++ ++#define KBASE_MEM_PHY_ALLOC_LARGE_THRESHOLD ((size_t)(4*1024)) /* size above which vmalloc is used over kmalloc */ ++ ++static inline struct kbase_mem_phy_alloc *kbase_alloc_create(size_t nr_pages, enum kbase_memory_type type) ++{ ++ struct kbase_mem_phy_alloc *alloc; ++ size_t alloc_size = sizeof(*alloc) + sizeof(*alloc->pages) * nr_pages; ++ size_t per_page_size = sizeof(*alloc->pages); ++ ++ /* Imported pages may have page private data already in use */ ++ if (type == KBASE_MEM_TYPE_IMPORTED_USER_BUF) { ++ alloc_size += nr_pages * ++ sizeof(*alloc->imported.user_buf.dma_addrs); ++ per_page_size += sizeof(*alloc->imported.user_buf.dma_addrs); ++ } ++ ++ /* ++ * Prevent nr_pages*per_page_size + sizeof(*alloc) from ++ * wrapping around. ++ */ ++ if (nr_pages > ((((size_t) -1) - sizeof(*alloc)) ++ / per_page_size)) ++ return ERR_PTR(-ENOMEM); ++ ++ /* Allocate based on the size to reduce internal fragmentation of vmem */ ++ if (alloc_size > KBASE_MEM_PHY_ALLOC_LARGE_THRESHOLD) ++ alloc = vzalloc(alloc_size); ++ else ++ alloc = kzalloc(alloc_size, GFP_KERNEL); ++ ++ if (!alloc) ++ return ERR_PTR(-ENOMEM); ++ ++ /* Store allocation method */ ++ if (alloc_size > KBASE_MEM_PHY_ALLOC_LARGE_THRESHOLD) ++ alloc->properties |= KBASE_MEM_PHY_ALLOC_LARGE; ++ ++ kref_init(&alloc->kref); ++ atomic_set(&alloc->gpu_mappings, 0); ++ alloc->nents = 0; ++ alloc->pages = (void *)(alloc + 1); ++ INIT_LIST_HEAD(&alloc->mappings); ++ alloc->type = type; ++ ++ if (type == KBASE_MEM_TYPE_IMPORTED_USER_BUF) ++ alloc->imported.user_buf.dma_addrs = ++ (void *) (alloc->pages + nr_pages); ++ ++ return alloc; ++} ++ ++static inline int kbase_reg_prepare_native(struct kbase_va_region *reg, ++ struct kbase_context *kctx) ++{ ++ KBASE_DEBUG_ASSERT(reg); ++ KBASE_DEBUG_ASSERT(!reg->cpu_alloc); ++ KBASE_DEBUG_ASSERT(!reg->gpu_alloc); ++ KBASE_DEBUG_ASSERT(reg->flags & KBASE_REG_FREE); ++ ++ reg->cpu_alloc = kbase_alloc_create(reg->nr_pages, ++ KBASE_MEM_TYPE_NATIVE); ++ if (IS_ERR(reg->cpu_alloc)) ++ return PTR_ERR(reg->cpu_alloc); ++ else if (!reg->cpu_alloc) ++ return -ENOMEM; ++ reg->cpu_alloc->imported.kctx = kctx; ++ INIT_LIST_HEAD(®->cpu_alloc->evict_node); ++ if (kbase_ctx_flag(kctx, KCTX_INFINITE_CACHE) ++ && (reg->flags & KBASE_REG_CPU_CACHED)) { ++ reg->gpu_alloc = kbase_alloc_create(reg->nr_pages, ++ KBASE_MEM_TYPE_NATIVE); ++ reg->gpu_alloc->imported.kctx = kctx; ++ INIT_LIST_HEAD(®->gpu_alloc->evict_node); ++ } else { ++ reg->gpu_alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc); ++ } ++ ++ INIT_LIST_HEAD(®->jit_node); ++ reg->flags &= ~KBASE_REG_FREE; ++ return 0; ++} ++ ++static inline int kbase_atomic_add_pages(int num_pages, atomic_t *used_pages) ++{ ++ int new_val = atomic_add_return(num_pages, used_pages); ++#if defined(CONFIG_MALI_GATOR_SUPPORT) ++ kbase_trace_mali_total_alloc_pages_change((long long int)new_val); ++#endif ++ return new_val; ++} ++ ++static inline int kbase_atomic_sub_pages(int num_pages, atomic_t *used_pages) ++{ ++ int new_val = atomic_sub_return(num_pages, used_pages); ++#if defined(CONFIG_MALI_GATOR_SUPPORT) ++ kbase_trace_mali_total_alloc_pages_change((long long int)new_val); ++#endif ++ return new_val; ++} ++ ++/* ++ * Max size for kbdev memory pool (in pages) ++ */ ++#define KBASE_MEM_POOL_MAX_SIZE_KBDEV (SZ_64M >> PAGE_SHIFT) ++ ++/* ++ * Max size for kctx memory pool (in pages) ++ */ ++#define KBASE_MEM_POOL_MAX_SIZE_KCTX (SZ_64M >> PAGE_SHIFT) ++ ++/** ++ * kbase_mem_pool_init - Create a memory pool for a kbase device ++ * @pool: Memory pool to initialize ++ * @max_size: Maximum number of free pages the pool can hold ++ * @kbdev: Kbase device where memory is used ++ * @next_pool: Pointer to the next pool or NULL. ++ * ++ * Allocations from @pool are in whole pages. Each @pool has a free list where ++ * pages can be quickly allocated from. The free list is initially empty and ++ * filled whenever pages are freed back to the pool. The number of free pages ++ * in the pool will in general not exceed @max_size, but the pool may in ++ * certain corner cases grow above @max_size. ++ * ++ * If @next_pool is not NULL, we will allocate from @next_pool before going to ++ * the kernel allocator. Similarily pages can spill over to @next_pool when ++ * @pool is full. Pages are zeroed before they spill over to another pool, to ++ * prevent leaking information between applications. ++ * ++ * A shrinker is registered so that Linux mm can reclaim pages from the pool as ++ * needed. ++ * ++ * Return: 0 on success, negative -errno on error ++ */ ++int kbase_mem_pool_init(struct kbase_mem_pool *pool, ++ size_t max_size, ++ struct kbase_device *kbdev, ++ struct kbase_mem_pool *next_pool); ++ ++/** ++ * kbase_mem_pool_term - Destroy a memory pool ++ * @pool: Memory pool to destroy ++ * ++ * Pages in the pool will spill over to @next_pool (if available) or freed to ++ * the kernel. ++ */ ++void kbase_mem_pool_term(struct kbase_mem_pool *pool); ++ ++/** ++ * kbase_mem_pool_alloc - Allocate a page from memory pool ++ * @pool: Memory pool to allocate from ++ * ++ * Allocations from the pool are made as follows: ++ * 1. If there are free pages in the pool, allocate a page from @pool. ++ * 2. Otherwise, if @next_pool is not NULL and has free pages, allocate a page ++ * from @next_pool. ++ * 3. Return NULL if no memory in the pool ++ * ++ * Return: Pointer to allocated page, or NULL if allocation failed. ++ */ ++struct page *kbase_mem_pool_alloc(struct kbase_mem_pool *pool); ++ ++/** ++ * kbase_mem_pool_free - Free a page to memory pool ++ * @pool: Memory pool where page should be freed ++ * @page: Page to free to the pool ++ * @dirty: Whether some of the page may be dirty in the cache. ++ * ++ * Pages are freed to the pool as follows: ++ * 1. If @pool is not full, add @page to @pool. ++ * 2. Otherwise, if @next_pool is not NULL and not full, add @page to ++ * @next_pool. ++ * 3. Finally, free @page to the kernel. ++ */ ++void kbase_mem_pool_free(struct kbase_mem_pool *pool, struct page *page, ++ bool dirty); ++ ++/** ++ * kbase_mem_pool_alloc_pages - Allocate pages from memory pool ++ * @pool: Memory pool to allocate from ++ * @nr_pages: Number of pages to allocate ++ * @pages: Pointer to array where the physical address of the allocated ++ * pages will be stored. ++ * ++ * Like kbase_mem_pool_alloc() but optimized for allocating many pages. ++ * ++ * Return: 0 on success, negative -errno on error ++ */ ++int kbase_mem_pool_alloc_pages(struct kbase_mem_pool *pool, size_t nr_pages, ++ phys_addr_t *pages); ++ ++/** ++ * kbase_mem_pool_free_pages - Free pages to memory pool ++ * @pool: Memory pool where pages should be freed ++ * @nr_pages: Number of pages to free ++ * @pages: Pointer to array holding the physical addresses of the pages to ++ * free. ++ * @dirty: Whether any pages may be dirty in the cache. ++ * @reclaimed: Whether the pages where reclaimable and thus should bypass ++ * the pool and go straight to the kernel. ++ * ++ * Like kbase_mem_pool_free() but optimized for freeing many pages. ++ */ ++void kbase_mem_pool_free_pages(struct kbase_mem_pool *pool, size_t nr_pages, ++ phys_addr_t *pages, bool dirty, bool reclaimed); ++ ++/** ++ * kbase_mem_pool_size - Get number of free pages in memory pool ++ * @pool: Memory pool to inspect ++ * ++ * Note: the size of the pool may in certain corner cases exceed @max_size! ++ * ++ * Return: Number of free pages in the pool ++ */ ++static inline size_t kbase_mem_pool_size(struct kbase_mem_pool *pool) ++{ ++ return READ_ONCE(pool->cur_size); ++} ++ ++/** ++ * kbase_mem_pool_max_size - Get maximum number of free pages in memory pool ++ * @pool: Memory pool to inspect ++ * ++ * Return: Maximum number of free pages in the pool ++ */ ++static inline size_t kbase_mem_pool_max_size(struct kbase_mem_pool *pool) ++{ ++ return pool->max_size; ++} ++ ++ ++/** ++ * kbase_mem_pool_set_max_size - Set maximum number of free pages in memory pool ++ * @pool: Memory pool to inspect ++ * @max_size: Maximum number of free pages the pool can hold ++ * ++ * If @max_size is reduced, the pool will be shrunk to adhere to the new limit. ++ * For details see kbase_mem_pool_shrink(). ++ */ ++void kbase_mem_pool_set_max_size(struct kbase_mem_pool *pool, size_t max_size); ++ ++/** ++ * kbase_mem_pool_grow - Grow the pool ++ * @pool: Memory pool to grow ++ * @nr_to_grow: Number of pages to add to the pool ++ * ++ * Adds @nr_to_grow pages to the pool. Note that this may cause the pool to ++ * become larger than the maximum size specified. ++ * ++ * Returns: 0 on success, -ENOMEM if unable to allocate sufficent pages ++ */ ++int kbase_mem_pool_grow(struct kbase_mem_pool *pool, size_t nr_to_grow); ++ ++/** ++ * kbase_mem_pool_trim - Grow or shrink the pool to a new size ++ * @pool: Memory pool to trim ++ * @new_size: New number of pages in the pool ++ * ++ * If @new_size > @cur_size, fill the pool with new pages from the kernel, but ++ * not above the max_size for the pool. ++ * If @new_size < @cur_size, shrink the pool by freeing pages to the kernel. ++ */ ++void kbase_mem_pool_trim(struct kbase_mem_pool *pool, size_t new_size); ++ ++/* ++ * kbase_mem_alloc_page - Allocate a new page for a device ++ * @kbdev: The kbase device ++ * ++ * Most uses should use kbase_mem_pool_alloc to allocate a page. However that ++ * function can fail in the event the pool is empty. ++ * ++ * Return: A new page or NULL if no memory ++ */ ++struct page *kbase_mem_alloc_page(struct kbase_device *kbdev); ++ ++int kbase_region_tracker_init(struct kbase_context *kctx); ++int kbase_region_tracker_init_jit(struct kbase_context *kctx, u64 jit_va_pages); ++void kbase_region_tracker_term(struct kbase_context *kctx); ++ ++struct kbase_va_region *kbase_region_tracker_find_region_enclosing_address(struct kbase_context *kctx, u64 gpu_addr); ++ ++/** ++ * @brief Check that a pointer is actually a valid region. ++ * ++ * Must be called with context lock held. ++ */ ++struct kbase_va_region *kbase_region_tracker_find_region_base_address(struct kbase_context *kctx, u64 gpu_addr); ++ ++struct kbase_va_region *kbase_alloc_free_region(struct kbase_context *kctx, u64 start_pfn, size_t nr_pages, int zone); ++void kbase_free_alloced_region(struct kbase_va_region *reg); ++int kbase_add_va_region(struct kbase_context *kctx, struct kbase_va_region *reg, u64 addr, size_t nr_pages, size_t align); ++ ++bool kbase_check_alloc_flags(unsigned long flags); ++bool kbase_check_import_flags(unsigned long flags); ++ ++/** ++ * kbase_update_region_flags - Convert user space flags to kernel region flags ++ * ++ * @kctx: kbase context ++ * @reg: The region to update the flags on ++ * @flags: The flags passed from user space ++ * ++ * The user space flag BASE_MEM_COHERENT_SYSTEM_REQUIRED will be rejected and ++ * this function will fail if the system does not support system coherency. ++ * ++ * Return: 0 if successful, -EINVAL if the flags are not supported ++ */ ++int kbase_update_region_flags(struct kbase_context *kctx, ++ struct kbase_va_region *reg, unsigned long flags); ++ ++void kbase_gpu_vm_lock(struct kbase_context *kctx); ++void kbase_gpu_vm_unlock(struct kbase_context *kctx); ++ ++int kbase_alloc_phy_pages(struct kbase_va_region *reg, size_t vsize, size_t size); ++ ++int kbase_mmu_init(struct kbase_context *kctx); ++void kbase_mmu_term(struct kbase_context *kctx); ++ ++phys_addr_t kbase_mmu_alloc_pgd(struct kbase_context *kctx); ++void kbase_mmu_free_pgd(struct kbase_context *kctx); ++int kbase_mmu_insert_pages_no_flush(struct kbase_context *kctx, u64 vpfn, ++ phys_addr_t *phys, size_t nr, ++ unsigned long flags); ++int kbase_mmu_insert_pages(struct kbase_context *kctx, u64 vpfn, ++ phys_addr_t *phys, size_t nr, ++ unsigned long flags); ++int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 vpfn, ++ phys_addr_t phys, size_t nr, ++ unsigned long flags); ++ ++int kbase_mmu_teardown_pages(struct kbase_context *kctx, u64 vpfn, size_t nr); ++int kbase_mmu_update_pages(struct kbase_context *kctx, u64 vpfn, phys_addr_t *phys, size_t nr, unsigned long flags); ++ ++/** ++ * @brief Register region and map it on the GPU. ++ * ++ * Call kbase_add_va_region() and map the region on the GPU. ++ */ ++int kbase_gpu_mmap(struct kbase_context *kctx, struct kbase_va_region *reg, u64 addr, size_t nr_pages, size_t align); ++ ++/** ++ * @brief Remove the region from the GPU and unregister it. ++ * ++ * Must be called with context lock held. ++ */ ++int kbase_gpu_munmap(struct kbase_context *kctx, struct kbase_va_region *reg); ++ ++/** ++ * The caller has the following locking conditions: ++ * - It must hold kbase_device->mmu_hw_mutex ++ * - It must hold the hwaccess_lock ++ */ ++void kbase_mmu_update(struct kbase_context *kctx); ++ ++/** ++ * kbase_mmu_disable() - Disable the MMU for a previously active kbase context. ++ * @kctx: Kbase context ++ * ++ * Disable and perform the required cache maintenance to remove the all ++ * data from provided kbase context from the GPU caches. ++ * ++ * The caller has the following locking conditions: ++ * - It must hold kbase_device->mmu_hw_mutex ++ * - It must hold the hwaccess_lock ++ */ ++void kbase_mmu_disable(struct kbase_context *kctx); ++ ++/** ++ * kbase_mmu_disable_as() - Set the MMU to unmapped mode for the specified ++ * address space. ++ * @kbdev: Kbase device ++ * @as_nr: The address space number to set to unmapped. ++ * ++ * This function must only be called during reset/power-up and it used to ++ * ensure the registers are in a known state. ++ * ++ * The caller must hold kbdev->mmu_hw_mutex. ++ */ ++void kbase_mmu_disable_as(struct kbase_device *kbdev, int as_nr); ++ ++void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat); ++ ++/** Dump the MMU tables to a buffer ++ * ++ * This function allocates a buffer (of @c nr_pages pages) to hold a dump of the MMU tables and fills it. If the ++ * buffer is too small then the return value will be NULL. ++ * ++ * The GPU vm lock must be held when calling this function. ++ * ++ * The buffer returned should be freed with @ref vfree when it is no longer required. ++ * ++ * @param[in] kctx The kbase context to dump ++ * @param[in] nr_pages The number of pages to allocate for the buffer. ++ * ++ * @return The address of the buffer containing the MMU dump or NULL on error (including if the @c nr_pages is too ++ * small) ++ */ ++void *kbase_mmu_dump(struct kbase_context *kctx, int nr_pages); ++ ++/** ++ * kbase_sync_now - Perform cache maintenance on a memory region ++ * ++ * @kctx: The kbase context of the region ++ * @sset: A syncset structure describing the region and direction of the ++ * synchronisation required ++ * ++ * Return: 0 on success or error code ++ */ ++int kbase_sync_now(struct kbase_context *kctx, struct basep_syncset *sset); ++void kbase_sync_single(struct kbase_context *kctx, phys_addr_t cpu_pa, ++ phys_addr_t gpu_pa, off_t offset, size_t size, ++ enum kbase_sync_type sync_fn); ++void kbase_pre_job_sync(struct kbase_context *kctx, struct base_syncset *syncsets, size_t nr); ++void kbase_post_job_sync(struct kbase_context *kctx, struct base_syncset *syncsets, size_t nr); ++ ++/* OS specific functions */ ++int kbase_mem_free(struct kbase_context *kctx, u64 gpu_addr); ++int kbase_mem_free_region(struct kbase_context *kctx, struct kbase_va_region *reg); ++void kbase_os_mem_map_lock(struct kbase_context *kctx); ++void kbase_os_mem_map_unlock(struct kbase_context *kctx); ++ ++/** ++ * @brief Update the memory allocation counters for the current process ++ * ++ * OS specific call to updates the current memory allocation counters for the current process with ++ * the supplied delta. ++ * ++ * @param[in] kctx The kbase context ++ * @param[in] pages The desired delta to apply to the memory usage counters. ++ */ ++ ++void kbasep_os_process_page_usage_update(struct kbase_context *kctx, int pages); ++ ++/** ++ * @brief Add to the memory allocation counters for the current process ++ * ++ * OS specific call to add to the current memory allocation counters for the current process by ++ * the supplied amount. ++ * ++ * @param[in] kctx The kernel base context used for the allocation. ++ * @param[in] pages The desired delta to apply to the memory usage counters. ++ */ ++ ++static inline void kbase_process_page_usage_inc(struct kbase_context *kctx, int pages) ++{ ++ kbasep_os_process_page_usage_update(kctx, pages); ++} ++ ++/** ++ * @brief Subtract from the memory allocation counters for the current process ++ * ++ * OS specific call to subtract from the current memory allocation counters for the current process by ++ * the supplied amount. ++ * ++ * @param[in] kctx The kernel base context used for the allocation. ++ * @param[in] pages The desired delta to apply to the memory usage counters. ++ */ ++ ++static inline void kbase_process_page_usage_dec(struct kbase_context *kctx, int pages) ++{ ++ kbasep_os_process_page_usage_update(kctx, 0 - pages); ++} ++ ++/** ++ * kbasep_find_enclosing_cpu_mapping_offset() - Find the offset of the CPU ++ * mapping of a memory allocation containing a given address range ++ * ++ * Searches for a CPU mapping of any part of any region that fully encloses the ++ * CPU virtual address range specified by @uaddr and @size. Returns a failure ++ * indication if only part of the address range lies within a CPU mapping. ++ * ++ * @kctx: The kernel base context used for the allocation. ++ * @uaddr: Start of the CPU virtual address range. ++ * @size: Size of the CPU virtual address range (in bytes). ++ * @offset: The offset from the start of the allocation to the specified CPU ++ * virtual address. ++ * ++ * Return: 0 if offset was obtained successfully. Error code otherwise. ++ */ ++int kbasep_find_enclosing_cpu_mapping_offset( ++ struct kbase_context *kctx, ++ unsigned long uaddr, size_t size, u64 *offset); ++ ++enum hrtimer_restart kbasep_as_poke_timer_callback(struct hrtimer *timer); ++void kbase_as_poking_timer_retain_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom); ++void kbase_as_poking_timer_release_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom); ++ ++/** ++* @brief Allocates physical pages. ++* ++* Allocates \a nr_pages_requested and updates the alloc object. ++* ++* @param[in] alloc allocation object to add pages to ++* @param[in] nr_pages_requested number of physical pages to allocate ++* ++* @return 0 if all pages have been successfully allocated. Error code otherwise ++*/ ++int kbase_alloc_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, size_t nr_pages_requested); ++ ++/** ++* @brief Free physical pages. ++* ++* Frees \a nr_pages and updates the alloc object. ++* ++* @param[in] alloc allocation object to free pages from ++* @param[in] nr_pages_to_free number of physical pages to free ++*/ ++int kbase_free_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, size_t nr_pages_to_free); ++ ++static inline void kbase_set_dma_addr(struct page *p, dma_addr_t dma_addr) ++{ ++ SetPagePrivate(p); ++ if (sizeof(dma_addr_t) > sizeof(p->private)) { ++ /* on 32-bit ARM with LPAE dma_addr_t becomes larger, but the ++ * private field stays the same. So we have to be clever and ++ * use the fact that we only store DMA addresses of whole pages, ++ * so the low bits should be zero */ ++ KBASE_DEBUG_ASSERT(!(dma_addr & (PAGE_SIZE - 1))); ++ set_page_private(p, dma_addr >> PAGE_SHIFT); ++ } else { ++ set_page_private(p, dma_addr); ++ } ++} ++ ++static inline dma_addr_t kbase_dma_addr(struct page *p) ++{ ++ if (sizeof(dma_addr_t) > sizeof(p->private)) ++ return ((dma_addr_t)page_private(p)) << PAGE_SHIFT; ++ ++ return (dma_addr_t)page_private(p); ++} ++ ++static inline void kbase_clear_dma_addr(struct page *p) ++{ ++ ClearPagePrivate(p); ++} ++ ++/** ++* @brief Process a bus or page fault. ++* ++* This function will process a fault on a specific address space ++* ++* @param[in] kbdev The @ref kbase_device the fault happened on ++* @param[in] kctx The @ref kbase_context for the faulting address space if ++* one was found. ++* @param[in] as The address space that has the fault ++*/ ++void kbase_mmu_interrupt_process(struct kbase_device *kbdev, ++ struct kbase_context *kctx, struct kbase_as *as); ++ ++/** ++ * @brief Process a page fault. ++ * ++ * @param[in] data work_struct passed by queue_work() ++ */ ++void page_fault_worker(struct work_struct *data); ++ ++/** ++ * @brief Process a bus fault. ++ * ++ * @param[in] data work_struct passed by queue_work() ++ */ ++void bus_fault_worker(struct work_struct *data); ++ ++/** ++ * @brief Flush MMU workqueues. ++ * ++ * This function will cause any outstanding page or bus faults to be processed. ++ * It should be called prior to powering off the GPU. ++ * ++ * @param[in] kbdev Device pointer ++ */ ++void kbase_flush_mmu_wqs(struct kbase_device *kbdev); ++ ++/** ++ * kbase_sync_single_for_device - update physical memory and give GPU ownership ++ * @kbdev: Device pointer ++ * @handle: DMA address of region ++ * @size: Size of region to sync ++ * @dir: DMA data direction ++ */ ++ ++void kbase_sync_single_for_device(struct kbase_device *kbdev, dma_addr_t handle, ++ size_t size, enum dma_data_direction dir); ++ ++/** ++ * kbase_sync_single_for_cpu - update physical memory and give CPU ownership ++ * @kbdev: Device pointer ++ * @handle: DMA address of region ++ * @size: Size of region to sync ++ * @dir: DMA data direction ++ */ ++ ++void kbase_sync_single_for_cpu(struct kbase_device *kbdev, dma_addr_t handle, ++ size_t size, enum dma_data_direction dir); ++ ++#ifdef CONFIG_DEBUG_FS ++/** ++ * kbase_jit_debugfs_init - Add per context debugfs entry for JIT. ++ * @kctx: kbase context ++ */ ++void kbase_jit_debugfs_init(struct kbase_context *kctx); ++#endif /* CONFIG_DEBUG_FS */ ++ ++/** ++ * kbase_jit_init - Initialize the JIT memory pool management ++ * @kctx: kbase context ++ * ++ * Returns zero on success or negative error number on failure. ++ */ ++int kbase_jit_init(struct kbase_context *kctx); ++ ++/** ++ * kbase_jit_allocate - Allocate JIT memory ++ * @kctx: kbase context ++ * @info: JIT allocation information ++ * ++ * Return: JIT allocation on success or NULL on failure. ++ */ ++struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx, ++ struct base_jit_alloc_info *info); ++ ++/** ++ * kbase_jit_free - Free a JIT allocation ++ * @kctx: kbase context ++ * @reg: JIT allocation ++ * ++ * Frees a JIT allocation and places it into the free pool for later reuse. ++ */ ++void kbase_jit_free(struct kbase_context *kctx, struct kbase_va_region *reg); ++ ++/** ++ * kbase_jit_backing_lost - Inform JIT that an allocation has lost backing ++ * @reg: JIT allocation ++ */ ++void kbase_jit_backing_lost(struct kbase_va_region *reg); ++ ++/** ++ * kbase_jit_evict - Evict a JIT allocation from the pool ++ * @kctx: kbase context ++ * ++ * Evict the least recently used JIT allocation from the pool. This can be ++ * required if normal VA allocations are failing due to VA exhaustion. ++ * ++ * Return: True if a JIT allocation was freed, false otherwise. ++ */ ++bool kbase_jit_evict(struct kbase_context *kctx); ++ ++/** ++ * kbase_jit_term - Terminate the JIT memory pool management ++ * @kctx: kbase context ++ */ ++void kbase_jit_term(struct kbase_context *kctx); ++ ++/** ++ * kbase_map_external_resource - Map an external resource to the GPU. ++ * @kctx: kbase context. ++ * @reg: The region to map. ++ * @locked_mm: The mm_struct which has been locked for this operation. ++ * @kds_res_count: The number of KDS resources. ++ * @kds_resources: Array of KDS resources. ++ * @kds_access_bitmap: Access bitmap for KDS. ++ * @exclusive: If the KDS resource requires exclusive access. ++ * ++ * Return: The physical allocation which backs the region on success or NULL ++ * on failure. ++ */ ++struct kbase_mem_phy_alloc *kbase_map_external_resource( ++ struct kbase_context *kctx, struct kbase_va_region *reg, ++ struct mm_struct *locked_mm ++#ifdef CONFIG_KDS ++ , u32 *kds_res_count, struct kds_resource **kds_resources, ++ unsigned long *kds_access_bitmap, bool exclusive ++#endif ++ ); ++ ++/** ++ * kbase_unmap_external_resource - Unmap an external resource from the GPU. ++ * @kctx: kbase context. ++ * @reg: The region to unmap or NULL if it has already been released. ++ * @alloc: The physical allocation being unmapped. ++ */ ++void kbase_unmap_external_resource(struct kbase_context *kctx, ++ struct kbase_va_region *reg, struct kbase_mem_phy_alloc *alloc); ++ ++/** ++ * kbase_sticky_resource_init - Initialize sticky resource management. ++ * @kctx: kbase context ++ * ++ * Returns zero on success or negative error number on failure. ++ */ ++int kbase_sticky_resource_init(struct kbase_context *kctx); ++ ++/** ++ * kbase_sticky_resource_acquire - Acquire a reference on a sticky resource. ++ * @kctx: kbase context. ++ * @gpu_addr: The GPU address of the external resource. ++ * ++ * Return: The metadata object which represents the binding between the ++ * external resource and the kbase context on success or NULL on failure. ++ */ ++struct kbase_ctx_ext_res_meta *kbase_sticky_resource_acquire( ++ struct kbase_context *kctx, u64 gpu_addr); ++ ++/** ++ * kbase_sticky_resource_release - Release a reference on a sticky resource. ++ * @kctx: kbase context. ++ * @meta: Binding metadata. ++ * @gpu_addr: GPU address of the external resource. ++ * ++ * If meta is NULL then gpu_addr will be used to scan the metadata list and ++ * find the matching metadata (if any), otherwise the provided meta will be ++ * used and gpu_addr will be ignored. ++ * ++ * Return: True if the release found the metadata and the reference was dropped. ++ */ ++bool kbase_sticky_resource_release(struct kbase_context *kctx, ++ struct kbase_ctx_ext_res_meta *meta, u64 gpu_addr); ++ ++/** ++ * kbase_sticky_resource_term - Terminate sticky resource management. ++ * @kctx: kbase context ++ */ ++void kbase_sticky_resource_term(struct kbase_context *kctx); ++ ++#endif /* _KBASE_MEM_H_ */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_mem_linux.c b/drivers/gpu/arm/midgard/mali_kbase_mem_linux.c +new file mode 100755 +index 000000000000..e20315e67242 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_mem_linux.c +@@ -0,0 +1,2578 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/** ++ * @file mali_kbase_mem_linux.c ++ * Base kernel memory APIs, Linux implementation. ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)) && \ ++ (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0)) ++#include ++#endif /* LINUX_VERSION_CODE >= 3.5.0 && < 4.8.0 */ ++#ifdef CONFIG_DMA_SHARED_BUFFER ++#include ++#endif /* defined(CONFIG_DMA_SHARED_BUFFER) */ ++#include ++#include ++ ++#include ++#include ++#include ++#include ++#include ++ ++static int kbase_tracking_page_setup(struct kbase_context *kctx, struct vm_area_struct *vma); ++ ++/** ++ * kbase_mem_shrink_cpu_mapping - Shrink the CPU mapping(s) of an allocation ++ * @kctx: Context the region belongs to ++ * @reg: The GPU region ++ * @new_pages: The number of pages after the shrink ++ * @old_pages: The number of pages before the shrink ++ * ++ * Shrink (or completely remove) all CPU mappings which reference the shrunk ++ * part of the allocation. ++ * ++ * Note: Caller must be holding the processes mmap_lock lock. ++ */ ++static void kbase_mem_shrink_cpu_mapping(struct kbase_context *kctx, ++ struct kbase_va_region *reg, ++ u64 new_pages, u64 old_pages); ++ ++/** ++ * kbase_mem_shrink_gpu_mapping - Shrink the GPU mapping of an allocation ++ * @kctx: Context the region belongs to ++ * @reg: The GPU region or NULL if there isn't one ++ * @new_pages: The number of pages after the shrink ++ * @old_pages: The number of pages before the shrink ++ * ++ * Return: 0 on success, negative -errno on error ++ * ++ * Unmap the shrunk pages from the GPU mapping. Note that the size of the region ++ * itself is unmodified as we still need to reserve the VA, only the page tables ++ * will be modified by this function. ++ */ ++static int kbase_mem_shrink_gpu_mapping(struct kbase_context *kctx, ++ struct kbase_va_region *reg, ++ u64 new_pages, u64 old_pages); ++ ++struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, ++ u64 va_pages, u64 commit_pages, u64 extent, u64 *flags, ++ u64 *gpu_va) ++{ ++ int zone; ++ int gpu_pc_bits; ++ int cpu_va_bits; ++ struct kbase_va_region *reg; ++ struct device *dev; ++ ++ KBASE_DEBUG_ASSERT(kctx); ++ KBASE_DEBUG_ASSERT(flags); ++ KBASE_DEBUG_ASSERT(gpu_va); ++ ++ dev = kctx->kbdev->dev; ++ *gpu_va = 0; /* return 0 on failure */ ++ ++ gpu_pc_bits = kctx->kbdev->gpu_props.props.core_props.log2_program_counter_size; ++ cpu_va_bits = BITS_PER_LONG; ++ ++ if (0 == va_pages) { ++ dev_warn(dev, "kbase_mem_alloc called with 0 va_pages!"); ++ goto bad_size; ++ } ++ ++ if (va_pages > (U64_MAX / PAGE_SIZE)) ++ /* 64-bit address range is the max */ ++ goto bad_size; ++ ++#if defined(CONFIG_64BIT) ++ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) ++ cpu_va_bits = 32; ++#endif ++ ++ if (!kbase_check_alloc_flags(*flags)) { ++ dev_warn(dev, ++ "kbase_mem_alloc called with bad flags (%llx)", ++ (unsigned long long)*flags); ++ goto bad_flags; ++ } ++ ++ if ((*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0 && ++ !kbase_device_is_cpu_coherent(kctx->kbdev)) { ++ dev_warn(dev, "kbase_mem_alloc call required coherent mem when unavailable"); ++ goto bad_flags; ++ } ++ if ((*flags & BASE_MEM_COHERENT_SYSTEM) != 0 && ++ !kbase_device_is_cpu_coherent(kctx->kbdev)) { ++ /* Remove COHERENT_SYSTEM flag if coherent mem is unavailable */ ++ *flags &= ~BASE_MEM_COHERENT_SYSTEM; ++ } ++ ++ /* Limit GPU executable allocs to GPU PC size */ ++ if ((*flags & BASE_MEM_PROT_GPU_EX) && ++ (va_pages > (1ULL << gpu_pc_bits >> PAGE_SHIFT))) ++ goto bad_ex_size; ++ ++ /* find out which VA zone to use */ ++ if (*flags & BASE_MEM_SAME_VA) ++ zone = KBASE_REG_ZONE_SAME_VA; ++ else if (*flags & BASE_MEM_PROT_GPU_EX) ++ zone = KBASE_REG_ZONE_EXEC; ++ else ++ zone = KBASE_REG_ZONE_CUSTOM_VA; ++ ++ reg = kbase_alloc_free_region(kctx, 0, va_pages, zone); ++ if (!reg) { ++ dev_err(dev, "Failed to allocate free region"); ++ goto no_region; ++ } ++ ++ if (kbase_update_region_flags(kctx, reg, *flags) != 0) ++ goto invalid_flags; ++ ++ if (kbase_reg_prepare_native(reg, kctx) != 0) { ++ dev_err(dev, "Failed to prepare region"); ++ goto prepare_failed; ++ } ++ ++ if (*flags & BASE_MEM_GROW_ON_GPF) ++ reg->extent = extent; ++ else ++ reg->extent = 0; ++ ++ if (kbase_alloc_phy_pages(reg, va_pages, commit_pages) != 0) { ++ dev_warn(dev, "Failed to allocate %lld pages (va_pages=%lld)", ++ (unsigned long long)commit_pages, ++ (unsigned long long)va_pages); ++ goto no_mem; ++ } ++ ++ kbase_gpu_vm_lock(kctx); ++ ++ /* mmap needed to setup VA? */ ++ if (*flags & BASE_MEM_SAME_VA) { ++ unsigned long prot = PROT_NONE; ++ unsigned long va_size = va_pages << PAGE_SHIFT; ++ unsigned long va_map = va_size; ++ unsigned long cookie, cookie_nr; ++ unsigned long cpu_addr; ++ ++ /* Bind to a cookie */ ++ if (!kctx->cookies) { ++ dev_err(dev, "No cookies available for allocation!"); ++ kbase_gpu_vm_unlock(kctx); ++ goto no_cookie; ++ } ++ /* return a cookie */ ++ cookie_nr = __ffs(kctx->cookies); ++ kctx->cookies &= ~(1UL << cookie_nr); ++ BUG_ON(kctx->pending_regions[cookie_nr]); ++ kctx->pending_regions[cookie_nr] = reg; ++ ++ kbase_gpu_vm_unlock(kctx); ++ ++ /* relocate to correct base */ ++ cookie = cookie_nr + PFN_DOWN(BASE_MEM_COOKIE_BASE); ++ cookie <<= PAGE_SHIFT; ++ ++ /* ++ * 10.1-10.4 UKU userland relies on the kernel to call mmap. ++ * For all other versions we can just return the cookie ++ */ ++ if (kctx->api_version < KBASE_API_VERSION(10, 1) || ++ kctx->api_version > KBASE_API_VERSION(10, 4)) { ++ *gpu_va = (u64) cookie; ++ return reg; ++ } ++ if (*flags & BASE_MEM_PROT_CPU_RD) ++ prot |= PROT_READ; ++ if (*flags & BASE_MEM_PROT_CPU_WR) ++ prot |= PROT_WRITE; ++ ++ cpu_addr = vm_mmap(kctx->filp, 0, va_map, prot, ++ MAP_SHARED, cookie); ++ ++ if (IS_ERR_VALUE(cpu_addr)) { ++ kbase_gpu_vm_lock(kctx); ++ kctx->pending_regions[cookie_nr] = NULL; ++ kctx->cookies |= (1UL << cookie_nr); ++ kbase_gpu_vm_unlock(kctx); ++ goto no_mmap; ++ } ++ ++ *gpu_va = (u64) cpu_addr; ++ } else /* we control the VA */ { ++ if (kbase_gpu_mmap(kctx, reg, 0, va_pages, 1) != 0) { ++ dev_warn(dev, "Failed to map memory on GPU"); ++ kbase_gpu_vm_unlock(kctx); ++ goto no_mmap; ++ } ++ /* return real GPU VA */ ++ *gpu_va = reg->start_pfn << PAGE_SHIFT; ++ ++ kbase_gpu_vm_unlock(kctx); ++ } ++ ++ return reg; ++ ++no_mmap: ++no_cookie: ++no_mem: ++ kbase_mem_phy_alloc_put(reg->cpu_alloc); ++ kbase_mem_phy_alloc_put(reg->gpu_alloc); ++invalid_flags: ++prepare_failed: ++ kfree(reg); ++no_region: ++bad_ex_size: ++bad_flags: ++bad_size: ++ return NULL; ++} ++KBASE_EXPORT_TEST_API(kbase_mem_alloc); ++ ++int kbase_mem_query(struct kbase_context *kctx, u64 gpu_addr, int query, u64 * const out) ++{ ++ struct kbase_va_region *reg; ++ int ret = -EINVAL; ++ ++ KBASE_DEBUG_ASSERT(kctx); ++ KBASE_DEBUG_ASSERT(out); ++ ++ if (gpu_addr & ~PAGE_MASK) { ++ dev_warn(kctx->kbdev->dev, "mem_query: gpu_addr: passed parameter is invalid"); ++ return -EINVAL; ++ } ++ ++ kbase_gpu_vm_lock(kctx); ++ ++ /* Validate the region */ ++ reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr); ++ if (!reg || (reg->flags & KBASE_REG_FREE)) ++ goto out_unlock; ++ ++ switch (query) { ++ case KBASE_MEM_QUERY_COMMIT_SIZE: ++ if (reg->cpu_alloc->type != KBASE_MEM_TYPE_ALIAS) { ++ *out = kbase_reg_current_backed_size(reg); ++ } else { ++ size_t i; ++ struct kbase_aliased *aliased; ++ *out = 0; ++ aliased = reg->cpu_alloc->imported.alias.aliased; ++ for (i = 0; i < reg->cpu_alloc->imported.alias.nents; i++) ++ *out += aliased[i].length; ++ } ++ break; ++ case KBASE_MEM_QUERY_VA_SIZE: ++ *out = reg->nr_pages; ++ break; ++ case KBASE_MEM_QUERY_FLAGS: ++ { ++ *out = 0; ++ if (KBASE_REG_CPU_WR & reg->flags) ++ *out |= BASE_MEM_PROT_CPU_WR; ++ if (KBASE_REG_CPU_RD & reg->flags) ++ *out |= BASE_MEM_PROT_CPU_RD; ++ if (KBASE_REG_CPU_CACHED & reg->flags) ++ *out |= BASE_MEM_CACHED_CPU; ++ if (KBASE_REG_GPU_WR & reg->flags) ++ *out |= BASE_MEM_PROT_GPU_WR; ++ if (KBASE_REG_GPU_RD & reg->flags) ++ *out |= BASE_MEM_PROT_GPU_RD; ++ if (!(KBASE_REG_GPU_NX & reg->flags)) ++ *out |= BASE_MEM_PROT_GPU_EX; ++ if (KBASE_REG_SHARE_BOTH & reg->flags) ++ *out |= BASE_MEM_COHERENT_SYSTEM; ++ if (KBASE_REG_SHARE_IN & reg->flags) ++ *out |= BASE_MEM_COHERENT_LOCAL; ++ break; ++ } ++ default: ++ *out = 0; ++ goto out_unlock; ++ } ++ ++ ret = 0; ++ ++out_unlock: ++ kbase_gpu_vm_unlock(kctx); ++ return ret; ++} ++ ++/** ++ * kbase_mem_evictable_reclaim_count_objects - Count number of pages in the ++ * Ephemeral memory eviction list. ++ * @s: Shrinker ++ * @sc: Shrinker control ++ * ++ * Return: Number of pages which can be freed. ++ */ ++static ++unsigned long kbase_mem_evictable_reclaim_count_objects(struct shrinker *s, ++ struct shrink_control *sc) ++{ ++ struct kbase_context *kctx; ++ struct kbase_mem_phy_alloc *alloc; ++ unsigned long pages = 0; ++ ++ kctx = container_of(s, struct kbase_context, reclaim); ++ ++ mutex_lock(&kctx->jit_evict_lock); ++ ++ list_for_each_entry(alloc, &kctx->evict_list, evict_node) ++ pages += alloc->nents; ++ ++ mutex_unlock(&kctx->jit_evict_lock); ++ return pages; ++} ++ ++/** ++ * kbase_mem_evictable_reclaim_scan_objects - Scan the Ephemeral memory eviction ++ * list for pages and try to reclaim them. ++ * @s: Shrinker ++ * @sc: Shrinker control ++ * ++ * Return: Number of pages freed (can be less then requested) or -1 if the ++ * shrinker failed to free pages in its pool. ++ * ++ * Note: ++ * This function accesses region structures without taking the region lock, ++ * this is required as the OOM killer can call the shrinker after the region ++ * lock has already been held. ++ * This is safe as we can guarantee that a region on the eviction list will ++ * not be freed (kbase_mem_free_region removes the allocation from the list ++ * before destroying it), or modified by other parts of the driver. ++ * The eviction list itself is guarded by the eviction lock and the MMU updates ++ * are protected by their own lock. ++ */ ++static ++unsigned long kbase_mem_evictable_reclaim_scan_objects(struct shrinker *s, ++ struct shrink_control *sc) ++{ ++ struct kbase_context *kctx; ++ struct kbase_mem_phy_alloc *alloc; ++ struct kbase_mem_phy_alloc *tmp; ++ unsigned long freed = 0; ++ ++ kctx = container_of(s, struct kbase_context, reclaim); ++ mutex_lock(&kctx->jit_evict_lock); ++ ++ list_for_each_entry_safe(alloc, tmp, &kctx->evict_list, evict_node) { ++ int err; ++ ++ err = kbase_mem_shrink_gpu_mapping(kctx, alloc->reg, ++ 0, alloc->nents); ++ if (err != 0) { ++ /* ++ * Failed to remove GPU mapping, tell the shrinker ++ * to stop trying to shrink our slab even though we ++ * have pages in it. ++ */ ++ freed = -1; ++ goto out_unlock; ++ } ++ ++ /* ++ * Update alloc->evicted before freeing the backing so the ++ * helper can determine that it needs to bypass the accounting ++ * and memory pool. ++ */ ++ alloc->evicted = alloc->nents; ++ ++ kbase_free_phy_pages_helper(alloc, alloc->evicted); ++ freed += alloc->evicted; ++ list_del_init(&alloc->evict_node); ++ ++ /* ++ * Inform the JIT allocator this region has lost backing ++ * as it might need to free the allocation. ++ */ ++ kbase_jit_backing_lost(alloc->reg); ++ ++ /* Enough pages have been freed so stop now */ ++ if (freed > sc->nr_to_scan) ++ break; ++ } ++out_unlock: ++ mutex_unlock(&kctx->jit_evict_lock); ++ ++ return freed; ++} ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) ++static int kbase_mem_evictable_reclaim_shrink(struct shrinker *s, ++ struct shrink_control *sc) ++{ ++ if (sc->nr_to_scan == 0) ++ return kbase_mem_evictable_reclaim_count_objects(s, sc); ++ ++ return kbase_mem_evictable_reclaim_scan_objects(s, sc); ++} ++#endif ++ ++int kbase_mem_evictable_init(struct kbase_context *kctx) ++{ ++ INIT_LIST_HEAD(&kctx->evict_list); ++ mutex_init(&kctx->jit_evict_lock); ++ ++ /* Register shrinker */ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) ++ kctx->reclaim.shrink = kbase_mem_evictable_reclaim_shrink; ++#else ++ kctx->reclaim.count_objects = kbase_mem_evictable_reclaim_count_objects; ++ kctx->reclaim.scan_objects = kbase_mem_evictable_reclaim_scan_objects; ++#endif ++ kctx->reclaim.seeks = DEFAULT_SEEKS; ++ /* Kernel versions prior to 3.1 : ++ * struct shrinker does not define batch */ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 1, 0) ++ kctx->reclaim.batch = 0; ++#endif ++ register_shrinker(&kctx->reclaim); ++ return 0; ++} ++ ++void kbase_mem_evictable_deinit(struct kbase_context *kctx) ++{ ++ unregister_shrinker(&kctx->reclaim); ++} ++ ++/** ++ * kbase_mem_evictable_mark_reclaim - Mark the pages as reclaimable. ++ * @alloc: The physical allocation ++ */ ++static void kbase_mem_evictable_mark_reclaim(struct kbase_mem_phy_alloc *alloc) ++{ ++ struct kbase_context *kctx = alloc->imported.kctx; ++ int __maybe_unused new_page_count; ++ ++ kbase_process_page_usage_dec(kctx, alloc->nents); ++ new_page_count = kbase_atomic_sub_pages(alloc->nents, ++ &kctx->used_pages); ++ kbase_atomic_sub_pages(alloc->nents, &kctx->kbdev->memdev.used_pages); ++ ++ KBASE_TLSTREAM_AUX_PAGESALLOC( ++ (u32)kctx->id, ++ (u64)new_page_count); ++} ++ ++/** ++ * kbase_mem_evictable_unmark_reclaim - Mark the pages as no longer reclaimable. ++ * @alloc: The physical allocation ++ */ ++static ++void kbase_mem_evictable_unmark_reclaim(struct kbase_mem_phy_alloc *alloc) ++{ ++ struct kbase_context *kctx = alloc->imported.kctx; ++ int __maybe_unused new_page_count; ++ ++ new_page_count = kbase_atomic_add_pages(alloc->nents, ++ &kctx->used_pages); ++ kbase_atomic_add_pages(alloc->nents, &kctx->kbdev->memdev.used_pages); ++ ++ /* Increase mm counters so that the allocation is accounted for ++ * against the process and thus is visible to the OOM killer. ++ */ ++ kbase_process_page_usage_inc(kctx, alloc->nents); ++ ++ KBASE_TLSTREAM_AUX_PAGESALLOC( ++ (u32)kctx->id, ++ (u64)new_page_count); ++} ++ ++int kbase_mem_evictable_make(struct kbase_mem_phy_alloc *gpu_alloc) ++{ ++ struct kbase_context *kctx = gpu_alloc->imported.kctx; ++ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ /* This alloction can't already be on a list. */ ++ WARN_ON(!list_empty(&gpu_alloc->evict_node)); ++ ++ kbase_mem_shrink_cpu_mapping(kctx, gpu_alloc->reg, ++ 0, gpu_alloc->nents); ++ ++ /* ++ * Add the allocation to the eviction list, after this point the shrink ++ * can reclaim it. ++ */ ++ mutex_lock(&kctx->jit_evict_lock); ++ list_add(&gpu_alloc->evict_node, &kctx->evict_list); ++ mutex_unlock(&kctx->jit_evict_lock); ++ kbase_mem_evictable_mark_reclaim(gpu_alloc); ++ ++ gpu_alloc->reg->flags |= KBASE_REG_DONT_NEED; ++ return 0; ++} ++ ++bool kbase_mem_evictable_unmake(struct kbase_mem_phy_alloc *gpu_alloc) ++{ ++ struct kbase_context *kctx = gpu_alloc->imported.kctx; ++ int err = 0; ++ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ /* ++ * First remove the allocation from the eviction list as it's no ++ * longer eligible for eviction. ++ */ ++ list_del_init(&gpu_alloc->evict_node); ++ ++ if (gpu_alloc->evicted == 0) { ++ /* ++ * The backing is still present, update the VM stats as it's ++ * in use again. ++ */ ++ kbase_mem_evictable_unmark_reclaim(gpu_alloc); ++ } else { ++ /* If the region is still alive ... */ ++ if (gpu_alloc->reg) { ++ /* ... allocate replacement backing ... */ ++ err = kbase_alloc_phy_pages_helper(gpu_alloc, ++ gpu_alloc->evicted); ++ ++ /* ++ * ... and grow the mapping back to its ++ * pre-eviction size. ++ */ ++ if (!err) ++ err = kbase_mem_grow_gpu_mapping(kctx, ++ gpu_alloc->reg, ++ gpu_alloc->evicted, 0); ++ ++ gpu_alloc->evicted = 0; ++ } ++ } ++ ++ /* If the region is still alive remove the DONT_NEED attribute. */ ++ if (gpu_alloc->reg) ++ gpu_alloc->reg->flags &= ~KBASE_REG_DONT_NEED; ++ ++ return (err == 0); ++} ++ ++int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, unsigned int flags, unsigned int mask) ++{ ++ struct kbase_va_region *reg; ++ int ret = -EINVAL; ++ unsigned int real_flags = 0; ++ unsigned int prev_flags = 0; ++ bool prev_needed, new_needed; ++ ++ KBASE_DEBUG_ASSERT(kctx); ++ ++ if (!gpu_addr) ++ return -EINVAL; ++ ++ if ((gpu_addr & ~PAGE_MASK) && (gpu_addr >= PAGE_SIZE)) ++ return -EINVAL; ++ ++ /* nuke other bits */ ++ flags &= mask; ++ ++ /* check for only supported flags */ ++ if (flags & ~(BASE_MEM_FLAGS_MODIFIABLE)) ++ goto out; ++ ++ /* mask covers bits we don't support? */ ++ if (mask & ~(BASE_MEM_FLAGS_MODIFIABLE)) ++ goto out; ++ ++ /* convert flags */ ++ if (BASE_MEM_COHERENT_SYSTEM & flags) ++ real_flags |= KBASE_REG_SHARE_BOTH; ++ else if (BASE_MEM_COHERENT_LOCAL & flags) ++ real_flags |= KBASE_REG_SHARE_IN; ++ ++ /* now we can lock down the context, and find the region */ ++ down_write(¤t->mm->mmap_lock); ++ kbase_gpu_vm_lock(kctx); ++ ++ /* Validate the region */ ++ reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr); ++ if (!reg || (reg->flags & KBASE_REG_FREE)) ++ goto out_unlock; ++ ++ /* Is the region being transitioning between not needed and needed? */ ++ prev_needed = (KBASE_REG_DONT_NEED & reg->flags) == KBASE_REG_DONT_NEED; ++ new_needed = (BASE_MEM_DONT_NEED & flags) == BASE_MEM_DONT_NEED; ++ if (prev_needed != new_needed) { ++ /* Aliased allocations can't be made ephemeral */ ++ if (atomic_read(®->cpu_alloc->gpu_mappings) > 1) ++ goto out_unlock; ++ ++ if (new_needed) { ++ /* Only native allocations can be marked not needed */ ++ if (reg->cpu_alloc->type != KBASE_MEM_TYPE_NATIVE) { ++ ret = -EINVAL; ++ goto out_unlock; ++ } ++ ret = kbase_mem_evictable_make(reg->gpu_alloc); ++ if (ret) ++ goto out_unlock; ++ } else { ++ kbase_mem_evictable_unmake(reg->gpu_alloc); ++ } ++ } ++ ++ /* limit to imported memory */ ++ if ((reg->gpu_alloc->type != KBASE_MEM_TYPE_IMPORTED_UMP) && ++ (reg->gpu_alloc->type != KBASE_MEM_TYPE_IMPORTED_UMM)) ++ goto out_unlock; ++ ++ /* no change? */ ++ if (real_flags == (reg->flags & (KBASE_REG_SHARE_IN | KBASE_REG_SHARE_BOTH))) { ++ ret = 0; ++ goto out_unlock; ++ } ++ ++ /* save for roll back */ ++ prev_flags = reg->flags; ++ reg->flags &= ~(KBASE_REG_SHARE_IN | KBASE_REG_SHARE_BOTH); ++ reg->flags |= real_flags; ++ ++ /* Currently supporting only imported memory */ ++ switch (reg->gpu_alloc->type) { ++#ifdef CONFIG_UMP ++ case KBASE_MEM_TYPE_IMPORTED_UMP: ++ ret = kbase_mmu_update_pages(kctx, reg->start_pfn, kbase_get_cpu_phy_pages(reg), reg->gpu_alloc->nents, reg->flags); ++ break; ++#endif ++#ifdef CONFIG_DMA_SHARED_BUFFER ++ case KBASE_MEM_TYPE_IMPORTED_UMM: ++ /* Future use will use the new flags, existing mapping will NOT be updated ++ * as memory should not be in use by the GPU when updating the flags. ++ */ ++ ret = 0; ++ WARN_ON(reg->gpu_alloc->imported.umm.current_mapping_usage_count); ++ break; ++#endif ++ default: ++ break; ++ } ++ ++ /* roll back on error, i.e. not UMP */ ++ if (ret) ++ reg->flags = prev_flags; ++ ++out_unlock: ++ kbase_gpu_vm_unlock(kctx); ++ up_write(¤t->mm->mmap_lock); ++out: ++ return ret; ++} ++ ++#define KBASE_MEM_IMPORT_HAVE_PAGES (1UL << BASE_MEM_FLAGS_NR_BITS) ++ ++#ifdef CONFIG_UMP ++static struct kbase_va_region *kbase_mem_from_ump(struct kbase_context *kctx, ump_secure_id id, u64 *va_pages, u64 *flags) ++{ ++ struct kbase_va_region *reg; ++ ump_dd_handle umph; ++ u64 block_count; ++ const ump_dd_physical_block_64 *block_array; ++ u64 i, j; ++ int page = 0; ++ ump_alloc_flags ump_flags; ++ ump_alloc_flags cpu_flags; ++ ump_alloc_flags gpu_flags; ++ ++ if (*flags & BASE_MEM_SECURE) ++ goto bad_flags; ++ ++ umph = ump_dd_from_secure_id(id); ++ if (UMP_DD_INVALID_MEMORY_HANDLE == umph) ++ goto bad_id; ++ ++ ump_flags = ump_dd_allocation_flags_get(umph); ++ cpu_flags = (ump_flags >> UMP_DEVICE_CPU_SHIFT) & UMP_DEVICE_MASK; ++ gpu_flags = (ump_flags >> DEFAULT_UMP_GPU_DEVICE_SHIFT) & ++ UMP_DEVICE_MASK; ++ ++ *va_pages = ump_dd_size_get_64(umph); ++ *va_pages >>= PAGE_SHIFT; ++ ++ if (!*va_pages) ++ goto bad_size; ++ ++ if (*va_pages > (U64_MAX / PAGE_SIZE)) ++ /* 64-bit address range is the max */ ++ goto bad_size; ++ ++ if (*flags & BASE_MEM_SAME_VA) ++ reg = kbase_alloc_free_region(kctx, 0, *va_pages, KBASE_REG_ZONE_SAME_VA); ++ else ++ reg = kbase_alloc_free_region(kctx, 0, *va_pages, KBASE_REG_ZONE_CUSTOM_VA); ++ ++ if (!reg) ++ goto no_region; ++ ++ /* we've got pages to map now, and support SAME_VA */ ++ *flags |= KBASE_MEM_IMPORT_HAVE_PAGES; ++ ++ reg->gpu_alloc = kbase_alloc_create(*va_pages, KBASE_MEM_TYPE_IMPORTED_UMP); ++ if (IS_ERR_OR_NULL(reg->gpu_alloc)) ++ goto no_alloc_obj; ++ ++ reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); ++ ++ reg->gpu_alloc->imported.ump_handle = umph; ++ ++ reg->flags &= ~KBASE_REG_FREE; ++ reg->flags |= KBASE_REG_GPU_NX; /* UMP is always No eXecute */ ++ reg->flags &= ~KBASE_REG_GROWABLE; /* UMP cannot be grown */ ++ ++ /* Override import flags based on UMP flags */ ++ *flags &= ~(BASE_MEM_CACHED_CPU); ++ *flags &= ~(BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_CPU_WR); ++ *flags &= ~(BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR); ++ ++ if ((cpu_flags & (UMP_HINT_DEVICE_RD | UMP_HINT_DEVICE_WR)) == ++ (UMP_HINT_DEVICE_RD | UMP_HINT_DEVICE_WR)) { ++ reg->flags |= KBASE_REG_CPU_CACHED; ++ *flags |= BASE_MEM_CACHED_CPU; ++ } ++ ++ if (cpu_flags & UMP_PROT_CPU_WR) { ++ reg->flags |= KBASE_REG_CPU_WR; ++ *flags |= BASE_MEM_PROT_CPU_WR; ++ } ++ ++ if (cpu_flags & UMP_PROT_CPU_RD) { ++ reg->flags |= KBASE_REG_CPU_RD; ++ *flags |= BASE_MEM_PROT_CPU_RD; ++ } ++ ++ if ((gpu_flags & (UMP_HINT_DEVICE_RD | UMP_HINT_DEVICE_WR)) == ++ (UMP_HINT_DEVICE_RD | UMP_HINT_DEVICE_WR)) ++ reg->flags |= KBASE_REG_GPU_CACHED; ++ ++ if (gpu_flags & UMP_PROT_DEVICE_WR) { ++ reg->flags |= KBASE_REG_GPU_WR; ++ *flags |= BASE_MEM_PROT_GPU_WR; ++ } ++ ++ if (gpu_flags & UMP_PROT_DEVICE_RD) { ++ reg->flags |= KBASE_REG_GPU_RD; ++ *flags |= BASE_MEM_PROT_GPU_RD; ++ } ++ ++ /* ump phys block query */ ++ ump_dd_phys_blocks_get_64(umph, &block_count, &block_array); ++ ++ for (i = 0; i < block_count; i++) { ++ for (j = 0; j < (block_array[i].size >> PAGE_SHIFT); j++) { ++ reg->gpu_alloc->pages[page] = block_array[i].addr + (j << PAGE_SHIFT); ++ page++; ++ } ++ } ++ reg->gpu_alloc->nents = *va_pages; ++ reg->extent = 0; ++ ++ return reg; ++ ++no_alloc_obj: ++ kfree(reg); ++no_region: ++bad_size: ++ ump_dd_release(umph); ++bad_id: ++bad_flags: ++ return NULL; ++} ++#endif /* CONFIG_UMP */ ++ ++#ifdef CONFIG_DMA_SHARED_BUFFER ++static struct kbase_va_region *kbase_mem_from_umm(struct kbase_context *kctx, ++ int fd, u64 *va_pages, u64 *flags, u32 padding) ++{ ++ struct kbase_va_region *reg; ++ struct dma_buf *dma_buf; ++ struct dma_buf_attachment *dma_attachment; ++ bool shared_zone = false; ++ ++ dma_buf = dma_buf_get(fd); ++ if (IS_ERR_OR_NULL(dma_buf)) ++ goto no_buf; ++ ++ dma_attachment = dma_buf_attach(dma_buf, kctx->kbdev->dev); ++ if (!dma_attachment) ++ goto no_attachment; ++ ++ *va_pages = (PAGE_ALIGN(dma_buf->size) >> PAGE_SHIFT) + padding; ++ if (!*va_pages) ++ goto bad_size; ++ ++ if (*va_pages > (U64_MAX / PAGE_SIZE)) ++ /* 64-bit address range is the max */ ++ goto bad_size; ++ ++ /* ignore SAME_VA */ ++ *flags &= ~BASE_MEM_SAME_VA; ++ ++ if (*flags & BASE_MEM_IMPORT_SHARED) ++ shared_zone = true; ++ ++#ifdef CONFIG_64BIT ++ if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { ++ /* ++ * 64-bit tasks require us to reserve VA on the CPU that we use ++ * on the GPU. ++ */ ++ shared_zone = true; ++ } ++#endif ++ ++ if (shared_zone) { ++ *flags |= BASE_MEM_NEED_MMAP; ++ reg = kbase_alloc_free_region(kctx, 0, *va_pages, KBASE_REG_ZONE_SAME_VA); ++ } else { ++ reg = kbase_alloc_free_region(kctx, 0, *va_pages, KBASE_REG_ZONE_CUSTOM_VA); ++ } ++ ++ if (!reg) ++ goto no_region; ++ ++ reg->gpu_alloc = kbase_alloc_create(*va_pages, KBASE_MEM_TYPE_IMPORTED_UMM); ++ if (IS_ERR_OR_NULL(reg->gpu_alloc)) ++ goto no_alloc_obj; ++ ++ reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); ++ ++ /* No pages to map yet */ ++ reg->gpu_alloc->nents = 0; ++ ++ if (kbase_update_region_flags(kctx, reg, *flags) != 0) ++ goto invalid_flags; ++ ++ reg->flags &= ~KBASE_REG_FREE; ++ reg->flags |= KBASE_REG_GPU_NX; /* UMM is always No eXecute */ ++ reg->flags &= ~KBASE_REG_GROWABLE; /* UMM cannot be grown */ ++ reg->flags |= KBASE_REG_GPU_CACHED; ++ ++ if (*flags & BASE_MEM_SECURE) ++ reg->flags |= KBASE_REG_SECURE; ++ ++ if (padding) ++ reg->flags |= KBASE_REG_IMPORT_PAD; ++ ++ reg->gpu_alloc->type = KBASE_MEM_TYPE_IMPORTED_UMM; ++ reg->gpu_alloc->imported.umm.sgt = NULL; ++ reg->gpu_alloc->imported.umm.dma_buf = dma_buf; ++ reg->gpu_alloc->imported.umm.dma_attachment = dma_attachment; ++ reg->gpu_alloc->imported.umm.current_mapping_usage_count = 0; ++ reg->extent = 0; ++ ++ return reg; ++ ++invalid_flags: ++ kbase_mem_phy_alloc_put(reg->gpu_alloc); ++no_alloc_obj: ++ kfree(reg); ++no_region: ++bad_size: ++ dma_buf_detach(dma_buf, dma_attachment); ++no_attachment: ++ dma_buf_put(dma_buf); ++no_buf: ++ return NULL; ++} ++#endif /* CONFIG_DMA_SHARED_BUFFER */ ++ ++static u32 kbase_get_cache_line_alignment(struct kbase_context *kctx) ++{ ++ u32 cpu_cache_line_size = cache_line_size(); ++ u32 gpu_cache_line_size = ++ (1UL << kctx->kbdev->gpu_props.props.l2_props.log2_line_size); ++ ++ return ((cpu_cache_line_size > gpu_cache_line_size) ? ++ cpu_cache_line_size : ++ gpu_cache_line_size); ++} ++ ++static struct kbase_va_region *kbase_mem_from_user_buffer( ++ struct kbase_context *kctx, unsigned long address, ++ unsigned long size, u64 *va_pages, u64 *flags) ++{ ++ long i; ++ struct kbase_va_region *reg; ++ long faulted_pages; ++ int zone = KBASE_REG_ZONE_CUSTOM_VA; ++ bool shared_zone = false; ++ u32 cache_line_alignment = kbase_get_cache_line_alignment(kctx); ++ struct kbase_alloc_import_user_buf *user_buf; ++ struct page **pages = NULL; ++ ++ if ((address & (cache_line_alignment - 1)) != 0 || ++ (size & (cache_line_alignment - 1)) != 0) { ++ /* Coherency must be enabled to handle partial cache lines */ ++ if (*flags & (BASE_MEM_COHERENT_SYSTEM | ++ BASE_MEM_COHERENT_SYSTEM_REQUIRED)) { ++ /* Force coherent system required flag, import will ++ * then fail if coherency isn't available ++ */ ++ *flags |= BASE_MEM_COHERENT_SYSTEM_REQUIRED; ++ } else { ++ dev_warn(kctx->kbdev->dev, ++ "User buffer is not cache line aligned and no coherency enabled\n"); ++ goto bad_size; ++ } ++ } ++ ++ *va_pages = (PAGE_ALIGN(address + size) >> PAGE_SHIFT) - ++ PFN_DOWN(address); ++ if (!*va_pages) ++ goto bad_size; ++ ++ if (*va_pages > (UINT64_MAX / PAGE_SIZE)) ++ /* 64-bit address range is the max */ ++ goto bad_size; ++ ++ /* SAME_VA generally not supported with imported memory (no known use cases) */ ++ *flags &= ~BASE_MEM_SAME_VA; ++ ++ if (*flags & BASE_MEM_IMPORT_SHARED) ++ shared_zone = true; ++ ++#ifdef CONFIG_64BIT ++ if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { ++ /* ++ * 64-bit tasks require us to reserve VA on the CPU that we use ++ * on the GPU. ++ */ ++ shared_zone = true; ++ } ++#endif ++ ++ if (shared_zone) { ++ *flags |= BASE_MEM_NEED_MMAP; ++ zone = KBASE_REG_ZONE_SAME_VA; ++ } ++ ++ reg = kbase_alloc_free_region(kctx, 0, *va_pages, zone); ++ ++ if (!reg) ++ goto no_region; ++ ++ reg->gpu_alloc = kbase_alloc_create(*va_pages, ++ KBASE_MEM_TYPE_IMPORTED_USER_BUF); ++ if (IS_ERR_OR_NULL(reg->gpu_alloc)) ++ goto no_alloc_obj; ++ ++ reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); ++ ++ if (kbase_update_region_flags(kctx, reg, *flags) != 0) ++ goto invalid_flags; ++ ++ reg->flags &= ~KBASE_REG_FREE; ++ reg->flags |= KBASE_REG_GPU_NX; /* User-buffers are always No eXecute */ ++ reg->flags &= ~KBASE_REG_GROWABLE; /* Cannot be grown */ ++ reg->flags &= ~KBASE_REG_CPU_CACHED; ++ ++ user_buf = ®->gpu_alloc->imported.user_buf; ++ ++ user_buf->size = size; ++ user_buf->address = address; ++ user_buf->nr_pages = *va_pages; ++ user_buf->mm = current->mm; ++ user_buf->pages = kmalloc_array(*va_pages, sizeof(struct page *), ++ GFP_KERNEL); ++ ++ if (!user_buf->pages) ++ goto no_page_array; ++ ++ /* If the region is coherent with the CPU then the memory is imported ++ * and mapped onto the GPU immediately. ++ * Otherwise get_user_pages is called as a sanity check, but with ++ * NULL as the pages argument which will fault the pages, but not ++ * pin them. The memory will then be pinned only around the jobs that ++ * specify the region as an external resource. ++ */ ++ if (reg->flags & KBASE_REG_SHARE_BOTH) { ++ pages = user_buf->pages; ++ *flags |= KBASE_MEM_IMPORT_HAVE_PAGES; ++ } ++ ++ down_read(¤t->mm->mmap_lock); ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) ++ faulted_pages = get_user_pages(current, current->mm, address, *va_pages, ++ reg->flags & KBASE_REG_GPU_WR, 0, pages, NULL); ++#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) ++ faulted_pages = get_user_pages(address, *va_pages, ++ reg->flags & KBASE_REG_GPU_WR, 0, pages, NULL); ++#else ++ faulted_pages = get_user_pages(address, *va_pages, ++ reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0, ++ pages, NULL); ++#endif ++ ++ up_read(¤t->mm->mmap_lock); ++ ++ if (faulted_pages != *va_pages) ++ goto fault_mismatch; ++ ++ atomic_inc(¤t->mm->mm_count); ++ ++ reg->gpu_alloc->nents = 0; ++ reg->extent = 0; ++ ++ if (pages) { ++ struct device *dev = kctx->kbdev->dev; ++ unsigned long local_size = user_buf->size; ++ unsigned long offset = user_buf->address & ~PAGE_MASK; ++ phys_addr_t *pa = kbase_get_gpu_phy_pages(reg); ++ ++ /* Top bit signifies that this was pinned on import */ ++ user_buf->current_mapping_usage_count |= PINNED_ON_IMPORT; ++ ++ for (i = 0; i < faulted_pages; i++) { ++ dma_addr_t dma_addr; ++ unsigned long min; ++ ++ min = MIN(PAGE_SIZE - offset, local_size); ++ dma_addr = dma_map_page(dev, pages[i], ++ offset, min, ++ DMA_BIDIRECTIONAL); ++ if (dma_mapping_error(dev, dma_addr)) ++ goto unwind_dma_map; ++ ++ user_buf->dma_addrs[i] = dma_addr; ++ pa[i] = page_to_phys(pages[i]); ++ ++ local_size -= min; ++ offset = 0; ++ } ++ ++ reg->gpu_alloc->nents = faulted_pages; ++ } ++ ++ return reg; ++ ++unwind_dma_map: ++ while (i--) { ++ dma_unmap_page(kctx->kbdev->dev, ++ user_buf->dma_addrs[i], ++ PAGE_SIZE, DMA_BIDIRECTIONAL); ++ } ++fault_mismatch: ++ if (pages) { ++ for (i = 0; i < faulted_pages; i++) ++ put_page(pages[i]); ++ } ++ kfree(user_buf->pages); ++no_page_array: ++invalid_flags: ++ kbase_mem_phy_alloc_put(reg->cpu_alloc); ++ kbase_mem_phy_alloc_put(reg->gpu_alloc); ++no_alloc_obj: ++ kfree(reg); ++no_region: ++bad_size: ++ return NULL; ++ ++} ++ ++ ++u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, ++ u64 nents, struct base_mem_aliasing_info *ai, ++ u64 *num_pages) ++{ ++ struct kbase_va_region *reg; ++ u64 gpu_va; ++ size_t i; ++ bool coherent; ++ ++ KBASE_DEBUG_ASSERT(kctx); ++ KBASE_DEBUG_ASSERT(flags); ++ KBASE_DEBUG_ASSERT(ai); ++ KBASE_DEBUG_ASSERT(num_pages); ++ ++ /* mask to only allowed flags */ ++ *flags &= (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR | ++ BASE_MEM_COHERENT_SYSTEM | BASE_MEM_COHERENT_LOCAL | ++ BASE_MEM_COHERENT_SYSTEM_REQUIRED); ++ ++ if (!(*flags & (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR))) { ++ dev_warn(kctx->kbdev->dev, ++ "kbase_mem_alias called with bad flags (%llx)", ++ (unsigned long long)*flags); ++ goto bad_flags; ++ } ++ coherent = (*flags & BASE_MEM_COHERENT_SYSTEM) != 0 || ++ (*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0; ++ ++ if (!stride) ++ goto bad_stride; ++ ++ if (!nents) ++ goto bad_nents; ++ ++ if ((nents * stride) > (U64_MAX / PAGE_SIZE)) ++ /* 64-bit address range is the max */ ++ goto bad_size; ++ ++ /* calculate the number of pages this alias will cover */ ++ *num_pages = nents * stride; ++ ++#ifdef CONFIG_64BIT ++ if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { ++ /* 64-bit tasks must MMAP anyway, but not expose this address to ++ * clients */ ++ *flags |= BASE_MEM_NEED_MMAP; ++ reg = kbase_alloc_free_region(kctx, 0, *num_pages, ++ KBASE_REG_ZONE_SAME_VA); ++ } else { ++#else ++ if (1) { ++#endif ++ reg = kbase_alloc_free_region(kctx, 0, *num_pages, ++ KBASE_REG_ZONE_CUSTOM_VA); ++ } ++ ++ if (!reg) ++ goto no_reg; ++ ++ /* zero-sized page array, as we don't need one/can support one */ ++ reg->gpu_alloc = kbase_alloc_create(0, KBASE_MEM_TYPE_ALIAS); ++ if (IS_ERR_OR_NULL(reg->gpu_alloc)) ++ goto no_alloc_obj; ++ ++ reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); ++ ++ if (kbase_update_region_flags(kctx, reg, *flags) != 0) ++ goto invalid_flags; ++ ++ reg->gpu_alloc->imported.alias.nents = nents; ++ reg->gpu_alloc->imported.alias.stride = stride; ++ reg->gpu_alloc->imported.alias.aliased = vzalloc(sizeof(*reg->gpu_alloc->imported.alias.aliased) * nents); ++ if (!reg->gpu_alloc->imported.alias.aliased) ++ goto no_aliased_array; ++ ++ kbase_gpu_vm_lock(kctx); ++ ++ /* validate and add src handles */ ++ for (i = 0; i < nents; i++) { ++ if (ai[i].handle.basep.handle < BASE_MEM_FIRST_FREE_ADDRESS) { ++ if (ai[i].handle.basep.handle != ++ BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE) ++ goto bad_handle; /* unsupported magic handle */ ++ if (!ai[i].length) ++ goto bad_handle; /* must be > 0 */ ++ if (ai[i].length > stride) ++ goto bad_handle; /* can't be larger than the ++ stride */ ++ reg->gpu_alloc->imported.alias.aliased[i].length = ai[i].length; ++ } else { ++ struct kbase_va_region *aliasing_reg; ++ struct kbase_mem_phy_alloc *alloc; ++ ++ aliasing_reg = kbase_region_tracker_find_region_base_address( ++ kctx, ++ (ai[i].handle.basep.handle >> PAGE_SHIFT) << PAGE_SHIFT); ++ ++ /* validate found region */ ++ if (!aliasing_reg) ++ goto bad_handle; /* Not found */ ++ if (aliasing_reg->flags & KBASE_REG_FREE) ++ goto bad_handle; /* Free region */ ++ if (aliasing_reg->flags & KBASE_REG_DONT_NEED) ++ goto bad_handle; /* Ephemeral region */ ++ if (!aliasing_reg->gpu_alloc) ++ goto bad_handle; /* No alloc */ ++ if (aliasing_reg->gpu_alloc->type != KBASE_MEM_TYPE_NATIVE) ++ goto bad_handle; /* Not a native alloc */ ++ if (coherent != ((aliasing_reg->flags & KBASE_REG_SHARE_BOTH) != 0)) ++ goto bad_handle; ++ /* Non-coherent memory cannot alias ++ coherent memory, and vice versa.*/ ++ ++ /* check size against stride */ ++ if (!ai[i].length) ++ goto bad_handle; /* must be > 0 */ ++ if (ai[i].length > stride) ++ goto bad_handle; /* can't be larger than the ++ stride */ ++ ++ alloc = aliasing_reg->gpu_alloc; ++ ++ /* check against the alloc's size */ ++ if (ai[i].offset > alloc->nents) ++ goto bad_handle; /* beyond end */ ++ if (ai[i].offset + ai[i].length > alloc->nents) ++ goto bad_handle; /* beyond end */ ++ ++ reg->gpu_alloc->imported.alias.aliased[i].alloc = kbase_mem_phy_alloc_get(alloc); ++ reg->gpu_alloc->imported.alias.aliased[i].length = ai[i].length; ++ reg->gpu_alloc->imported.alias.aliased[i].offset = ai[i].offset; ++ } ++ } ++ ++#ifdef CONFIG_64BIT ++ if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { ++ /* Bind to a cookie */ ++ if (!kctx->cookies) { ++ dev_err(kctx->kbdev->dev, "No cookies available for allocation!"); ++ goto no_cookie; ++ } ++ /* return a cookie */ ++ gpu_va = __ffs(kctx->cookies); ++ kctx->cookies &= ~(1UL << gpu_va); ++ BUG_ON(kctx->pending_regions[gpu_va]); ++ kctx->pending_regions[gpu_va] = reg; ++ ++ /* relocate to correct base */ ++ gpu_va += PFN_DOWN(BASE_MEM_COOKIE_BASE); ++ gpu_va <<= PAGE_SHIFT; ++ } else /* we control the VA */ { ++#else ++ if (1) { ++#endif ++ if (kbase_gpu_mmap(kctx, reg, 0, *num_pages, 1) != 0) { ++ dev_warn(kctx->kbdev->dev, "Failed to map memory on GPU"); ++ goto no_mmap; ++ } ++ /* return real GPU VA */ ++ gpu_va = reg->start_pfn << PAGE_SHIFT; ++ } ++ ++ reg->flags &= ~KBASE_REG_FREE; ++ reg->flags &= ~KBASE_REG_GROWABLE; ++ ++ kbase_gpu_vm_unlock(kctx); ++ ++ return gpu_va; ++ ++#ifdef CONFIG_64BIT ++no_cookie: ++#endif ++no_mmap: ++bad_handle: ++ kbase_gpu_vm_unlock(kctx); ++no_aliased_array: ++invalid_flags: ++ kbase_mem_phy_alloc_put(reg->cpu_alloc); ++ kbase_mem_phy_alloc_put(reg->gpu_alloc); ++no_alloc_obj: ++ kfree(reg); ++no_reg: ++bad_size: ++bad_nents: ++bad_stride: ++bad_flags: ++ return 0; ++} ++ ++int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type, ++ void __user *phandle, u32 padding, u64 *gpu_va, u64 *va_pages, ++ u64 *flags) ++{ ++ struct kbase_va_region *reg; ++ ++ KBASE_DEBUG_ASSERT(kctx); ++ KBASE_DEBUG_ASSERT(gpu_va); ++ KBASE_DEBUG_ASSERT(va_pages); ++ KBASE_DEBUG_ASSERT(flags); ++ ++#ifdef CONFIG_64BIT ++ if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) ++ *flags |= BASE_MEM_SAME_VA; ++#endif ++ ++ if (!kbase_check_import_flags(*flags)) { ++ dev_warn(kctx->kbdev->dev, ++ "kbase_mem_import called with bad flags (%llx)", ++ (unsigned long long)*flags); ++ goto bad_flags; ++ } ++ ++ if ((*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0 && ++ !kbase_device_is_cpu_coherent(kctx->kbdev)) { ++ dev_warn(kctx->kbdev->dev, ++ "kbase_mem_import call required coherent mem when unavailable"); ++ goto bad_flags; ++ } ++ if ((*flags & BASE_MEM_COHERENT_SYSTEM) != 0 && ++ !kbase_device_is_cpu_coherent(kctx->kbdev)) { ++ /* Remove COHERENT_SYSTEM flag if coherent mem is unavailable */ ++ *flags &= ~BASE_MEM_COHERENT_SYSTEM; ++ } ++ ++ if ((padding != 0) && (type != BASE_MEM_IMPORT_TYPE_UMM)) { ++ dev_warn(kctx->kbdev->dev, ++ "padding is only supported for UMM"); ++ goto bad_flags; ++ } ++ ++ switch (type) { ++#ifdef CONFIG_UMP ++ case BASE_MEM_IMPORT_TYPE_UMP: { ++ ump_secure_id id; ++ ++ if (get_user(id, (ump_secure_id __user *)phandle)) ++ reg = NULL; ++ else ++ reg = kbase_mem_from_ump(kctx, id, va_pages, flags); ++ } ++ break; ++#endif /* CONFIG_UMP */ ++#ifdef CONFIG_DMA_SHARED_BUFFER ++ case BASE_MEM_IMPORT_TYPE_UMM: { ++ int fd; ++ ++ if (get_user(fd, (int __user *)phandle)) ++ reg = NULL; ++ else ++ reg = kbase_mem_from_umm(kctx, fd, va_pages, flags, ++ padding); ++ } ++ break; ++#endif /* CONFIG_DMA_SHARED_BUFFER */ ++ case BASE_MEM_IMPORT_TYPE_USER_BUFFER: { ++ struct base_mem_import_user_buffer user_buffer; ++ void __user *uptr; ++ ++ if (copy_from_user(&user_buffer, phandle, ++ sizeof(user_buffer))) { ++ reg = NULL; ++ } else { ++#ifdef CONFIG_COMPAT ++ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) ++ uptr = compat_ptr(user_buffer.ptr.compat_value); ++ else ++#endif ++ uptr = user_buffer.ptr.value; ++ ++ reg = kbase_mem_from_user_buffer(kctx, ++ (unsigned long)uptr, user_buffer.length, ++ va_pages, flags); ++ } ++ break; ++ } ++ default: { ++ reg = NULL; ++ break; ++ } ++ } ++ ++ if (!reg) ++ goto no_reg; ++ ++ kbase_gpu_vm_lock(kctx); ++ ++ /* mmap needed to setup VA? */ ++ if (*flags & (BASE_MEM_SAME_VA | BASE_MEM_NEED_MMAP)) { ++ /* Bind to a cookie */ ++ if (!kctx->cookies) ++ goto no_cookie; ++ /* return a cookie */ ++ *gpu_va = __ffs(kctx->cookies); ++ kctx->cookies &= ~(1UL << *gpu_va); ++ BUG_ON(kctx->pending_regions[*gpu_va]); ++ kctx->pending_regions[*gpu_va] = reg; ++ ++ /* relocate to correct base */ ++ *gpu_va += PFN_DOWN(BASE_MEM_COOKIE_BASE); ++ *gpu_va <<= PAGE_SHIFT; ++ ++ } else if (*flags & KBASE_MEM_IMPORT_HAVE_PAGES) { ++ /* we control the VA, mmap now to the GPU */ ++ if (kbase_gpu_mmap(kctx, reg, 0, *va_pages, 1) != 0) ++ goto no_gpu_va; ++ /* return real GPU VA */ ++ *gpu_va = reg->start_pfn << PAGE_SHIFT; ++ } else { ++ /* we control the VA, but nothing to mmap yet */ ++ if (kbase_add_va_region(kctx, reg, 0, *va_pages, 1) != 0) ++ goto no_gpu_va; ++ /* return real GPU VA */ ++ *gpu_va = reg->start_pfn << PAGE_SHIFT; ++ } ++ ++ /* clear out private flags */ ++ *flags &= ((1UL << BASE_MEM_FLAGS_NR_BITS) - 1); ++ ++ kbase_gpu_vm_unlock(kctx); ++ ++ return 0; ++ ++no_gpu_va: ++no_cookie: ++ kbase_gpu_vm_unlock(kctx); ++ kbase_mem_phy_alloc_put(reg->cpu_alloc); ++ kbase_mem_phy_alloc_put(reg->gpu_alloc); ++ kfree(reg); ++no_reg: ++bad_flags: ++ *gpu_va = 0; ++ *va_pages = 0; ++ *flags = 0; ++ return -ENOMEM; ++} ++ ++int kbase_mem_grow_gpu_mapping(struct kbase_context *kctx, ++ struct kbase_va_region *reg, ++ u64 new_pages, u64 old_pages) ++{ ++ phys_addr_t *phy_pages; ++ u64 delta = new_pages - old_pages; ++ int ret = 0; ++ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ /* Map the new pages into the GPU */ ++ phy_pages = kbase_get_gpu_phy_pages(reg); ++ ret = kbase_mmu_insert_pages(kctx, reg->start_pfn + old_pages, ++ phy_pages + old_pages, delta, reg->flags); ++ ++ return ret; ++} ++ ++static void kbase_mem_shrink_cpu_mapping(struct kbase_context *kctx, ++ struct kbase_va_region *reg, ++ u64 new_pages, u64 old_pages) ++{ ++ u64 gpu_va_start = reg->start_pfn; ++ ++ if (new_pages == old_pages) ++ /* Nothing to do */ ++ return; ++ ++ unmap_mapping_range(kctx->filp->f_inode->i_mapping, ++ (gpu_va_start + new_pages)<start_pfn + new_pages, delta); ++ ++ return ret; ++} ++ ++int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages) ++{ ++ u64 old_pages; ++ u64 delta; ++ int res = -EINVAL; ++ struct kbase_va_region *reg; ++ bool read_locked = false; ++ ++ KBASE_DEBUG_ASSERT(kctx); ++ KBASE_DEBUG_ASSERT(gpu_addr != 0); ++ ++ if (gpu_addr & ~PAGE_MASK) { ++ dev_warn(kctx->kbdev->dev, "kbase:mem_commit: gpu_addr: passed parameter is invalid"); ++ return -EINVAL; ++ } ++ ++ down_write(¤t->mm->mmap_lock); ++ kbase_gpu_vm_lock(kctx); ++ ++ /* Validate the region */ ++ reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr); ++ if (!reg || (reg->flags & KBASE_REG_FREE)) ++ goto out_unlock; ++ ++ KBASE_DEBUG_ASSERT(reg->cpu_alloc); ++ KBASE_DEBUG_ASSERT(reg->gpu_alloc); ++ ++ if (reg->gpu_alloc->type != KBASE_MEM_TYPE_NATIVE) ++ goto out_unlock; ++ ++ if (0 == (reg->flags & KBASE_REG_GROWABLE)) ++ goto out_unlock; ++ ++ /* Would overflow the VA region */ ++ if (new_pages > reg->nr_pages) ++ goto out_unlock; ++ ++ /* can't be mapped more than once on the GPU */ ++ if (atomic_read(®->gpu_alloc->gpu_mappings) > 1) ++ goto out_unlock; ++ /* can't grow regions which are ephemeral */ ++ if (reg->flags & KBASE_REG_DONT_NEED) ++ goto out_unlock; ++ ++ if (new_pages == reg->gpu_alloc->nents) { ++ /* no change */ ++ res = 0; ++ goto out_unlock; ++ } ++ ++ old_pages = kbase_reg_current_backed_size(reg); ++ if (new_pages > old_pages) { ++ delta = new_pages - old_pages; ++ ++ /* ++ * No update to the mm so downgrade the writer lock to a read ++ * lock so other readers aren't blocked after this point. ++ */ ++ downgrade_write(¤t->mm->mmap_lock); ++ read_locked = true; ++ ++ /* Allocate some more pages */ ++ if (kbase_alloc_phy_pages_helper(reg->cpu_alloc, delta) != 0) { ++ res = -ENOMEM; ++ goto out_unlock; ++ } ++ if (reg->cpu_alloc != reg->gpu_alloc) { ++ if (kbase_alloc_phy_pages_helper( ++ reg->gpu_alloc, delta) != 0) { ++ res = -ENOMEM; ++ kbase_free_phy_pages_helper(reg->cpu_alloc, ++ delta); ++ goto out_unlock; ++ } ++ } ++ ++ /* No update required for CPU mappings, that's done on fault. */ ++ ++ /* Update GPU mapping. */ ++ res = kbase_mem_grow_gpu_mapping(kctx, reg, ++ new_pages, old_pages); ++ ++ /* On error free the new pages */ ++ if (res) { ++ kbase_free_phy_pages_helper(reg->cpu_alloc, delta); ++ if (reg->cpu_alloc != reg->gpu_alloc) ++ kbase_free_phy_pages_helper(reg->gpu_alloc, ++ delta); ++ res = -ENOMEM; ++ goto out_unlock; ++ } ++ } else { ++ delta = old_pages - new_pages; ++ ++ /* Update all CPU mapping(s) */ ++ kbase_mem_shrink_cpu_mapping(kctx, reg, ++ new_pages, old_pages); ++ ++ /* Update the GPU mapping */ ++ res = kbase_mem_shrink_gpu_mapping(kctx, reg, ++ new_pages, old_pages); ++ if (res) { ++ res = -ENOMEM; ++ goto out_unlock; ++ } ++ ++ kbase_free_phy_pages_helper(reg->cpu_alloc, delta); ++ if (reg->cpu_alloc != reg->gpu_alloc) ++ kbase_free_phy_pages_helper(reg->gpu_alloc, delta); ++ } ++ ++out_unlock: ++ kbase_gpu_vm_unlock(kctx); ++ if (read_locked) ++ up_read(¤t->mm->mmap_lock); ++ else ++ up_write(¤t->mm->mmap_lock); ++ ++ return res; ++} ++ ++static void kbase_cpu_vm_open(struct vm_area_struct *vma) ++{ ++ struct kbase_cpu_mapping *map = vma->vm_private_data; ++ ++ KBASE_DEBUG_ASSERT(map); ++ KBASE_DEBUG_ASSERT(map->count > 0); ++ /* non-atomic as we're under Linux' mm lock */ ++ map->count++; ++} ++ ++static void kbase_cpu_vm_close(struct vm_area_struct *vma) ++{ ++ struct kbase_cpu_mapping *map = vma->vm_private_data; ++ ++ KBASE_DEBUG_ASSERT(map); ++ KBASE_DEBUG_ASSERT(map->count > 0); ++ ++ /* non-atomic as we're under Linux' mm lock */ ++ if (--map->count) ++ return; ++ ++ KBASE_DEBUG_ASSERT(map->kctx); ++ KBASE_DEBUG_ASSERT(map->alloc); ++ ++ kbase_gpu_vm_lock(map->kctx); ++ ++ if (map->free_on_close) { ++ KBASE_DEBUG_ASSERT((map->region->flags & KBASE_REG_ZONE_MASK) == ++ KBASE_REG_ZONE_SAME_VA); ++ /* Avoid freeing memory on the process death which results in ++ * GPU Page Fault. Memory will be freed in kbase_destroy_context ++ */ ++ if (!(current->flags & PF_EXITING)) ++ kbase_mem_free_region(map->kctx, map->region); ++ } ++ ++ list_del(&map->mappings_list); ++ ++ kbase_gpu_vm_unlock(map->kctx); ++ ++ kbase_mem_phy_alloc_put(map->alloc); ++ kfree(map); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_cpu_vm_close); ++ ++ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0)) ++static vm_fault_t kbase_cpu_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) ++{ ++#else ++static vm_fault_t kbase_cpu_vm_fault(struct vm_fault *vmf) ++{ ++ struct vm_area_struct *vma = vmf->vma; ++#endif ++ struct kbase_cpu_mapping *map = vma->vm_private_data; ++ pgoff_t rel_pgoff; ++ size_t i; ++ pgoff_t addr; ++ vm_fault_t ret = VM_FAULT_SIGBUS; ++ ++ KBASE_DEBUG_ASSERT(map); ++ KBASE_DEBUG_ASSERT(map->count > 0); ++ KBASE_DEBUG_ASSERT(map->kctx); ++ KBASE_DEBUG_ASSERT(map->alloc); ++ ++ rel_pgoff = vmf->pgoff - map->region->start_pfn; ++ ++ kbase_gpu_vm_lock(map->kctx); ++ if (rel_pgoff >= map->alloc->nents) ++ goto locked_bad_fault; ++ ++ /* Fault on access to DONT_NEED regions */ ++ if (map->alloc->reg && (map->alloc->reg->flags & KBASE_REG_DONT_NEED)) ++ goto locked_bad_fault; ++ ++ /* insert all valid pages from the fault location */ ++ i = rel_pgoff; ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ addr = (pgoff_t)((uintptr_t)vmf->virtual_address >> PAGE_SHIFT); ++#else ++ addr = (pgoff_t)(vmf->address >> PAGE_SHIFT); ++#endif ++ while (i < map->alloc->nents && (addr < vma->vm_end >> PAGE_SHIFT)) { ++ ret = vmf_insert_pfn(vma, addr << PAGE_SHIFT, ++ PFN_DOWN(map->alloc->pages[i])); ++ if (ret != VM_FAULT_NOPAGE) ++ goto locked_bad_fault; ++ ++ i++; addr++; ++ } ++ ++ kbase_gpu_vm_unlock(map->kctx); ++ /* we resolved it, nothing for VM to do */ ++ return VM_FAULT_NOPAGE; ++ ++locked_bad_fault: ++ kbase_gpu_vm_unlock(map->kctx); ++ return ret; ++} ++ ++const struct vm_operations_struct kbase_vm_ops = { ++ .open = kbase_cpu_vm_open, ++ .close = kbase_cpu_vm_close, ++ .fault = kbase_cpu_vm_fault ++}; ++ ++static int kbase_cpu_mmap(struct kbase_va_region *reg, struct vm_area_struct *vma, void *kaddr, size_t nr_pages, unsigned long aligned_offset, int free_on_close) ++{ ++ struct kbase_cpu_mapping *map; ++ phys_addr_t *page_array; ++ int err = 0; ++ int i; ++ ++ map = kzalloc(sizeof(*map), GFP_KERNEL); ++ ++ if (!map) { ++ WARN_ON(1); ++ err = -ENOMEM; ++ goto out; ++ } ++ ++ /* ++ * VM_DONTCOPY - don't make this mapping available in fork'ed processes ++ * VM_DONTEXPAND - disable mremap on this region ++ * VM_IO - disables paging ++ * VM_DONTDUMP - Don't include in core dumps (3.7 only) ++ * VM_MIXEDMAP - Support mixing struct page*s and raw pfns. ++ * This is needed to support using the dedicated and ++ * the OS based memory backends together. ++ */ ++ /* ++ * This will need updating to propagate coherency flags ++ * See MIDBASE-1057 ++ */ ++ ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0)) ++ vma->vm_flags |= VM_DONTCOPY | VM_DONTDUMP | VM_DONTEXPAND | VM_IO; ++#else ++ vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_RESERVED | VM_IO; ++#endif ++ vma->vm_ops = &kbase_vm_ops; ++ vma->vm_private_data = map; ++ ++ page_array = kbase_get_cpu_phy_pages(reg); ++ ++ if (!(reg->flags & KBASE_REG_CPU_CACHED) && ++ (reg->flags & (KBASE_REG_CPU_WR|KBASE_REG_CPU_RD))) { ++ /* We can't map vmalloc'd memory uncached. ++ * Other memory will have been returned from ++ * kbase_mem_pool which would be ++ * suitable for mapping uncached. ++ */ ++ BUG_ON(kaddr); ++ vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); ++ } ++ ++ if (!kaddr) { ++ unsigned long addr = vma->vm_start + aligned_offset; ++ u64 start_off = vma->vm_pgoff - reg->start_pfn + ++ (aligned_offset>>PAGE_SHIFT); ++ ++ vma->vm_flags |= VM_PFNMAP; ++ for (i = 0; i < nr_pages; i++) { ++ unsigned long pfn = PFN_DOWN(page_array[i + start_off]); ++ vm_fault_t ret; ++ ++ ret = vmf_insert_pfn(vma, addr, pfn); ++ if (WARN_ON(ret != VM_FAULT_NOPAGE)) { ++ if (ret == VM_FAULT_OOM) ++ err = -ENOMEM; ++ else ++ err = -EFAULT; ++ break; ++ } ++ ++ addr += PAGE_SIZE; ++ } ++ } else { ++ WARN_ON(aligned_offset); ++ /* MIXEDMAP so we can vfree the kaddr early and not track it after map time */ ++ vma->vm_flags |= VM_MIXEDMAP; ++ /* vmalloc remaping is easy... */ ++ err = remap_vmalloc_range(vma, kaddr, 0); ++ WARN_ON(err); ++ } ++ ++ if (err) { ++ kfree(map); ++ goto out; ++ } ++ ++ map->region = reg; ++ map->free_on_close = free_on_close; ++ map->kctx = reg->kctx; ++ map->alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc); ++ map->count = 1; /* start with one ref */ ++ ++ if (reg->flags & KBASE_REG_CPU_CACHED) ++ map->alloc->properties |= KBASE_MEM_PHY_ALLOC_ACCESSED_CACHED; ++ ++ list_add(&map->mappings_list, &map->alloc->mappings); ++ ++ out: ++ return err; ++} ++ ++static int kbase_trace_buffer_mmap(struct kbase_context *kctx, struct vm_area_struct *vma, struct kbase_va_region **const reg, void **const kaddr) ++{ ++ struct kbase_va_region *new_reg; ++ u32 nr_pages; ++ size_t size; ++ int err = 0; ++ u32 *tb; ++ int owns_tb = 1; ++ ++ dev_dbg(kctx->kbdev->dev, "in %s\n", __func__); ++ size = (vma->vm_end - vma->vm_start); ++ nr_pages = size >> PAGE_SHIFT; ++ ++ if (!kctx->jctx.tb) { ++ KBASE_DEBUG_ASSERT(0 != size); ++ tb = vmalloc_user(size); ++ ++ if (NULL == tb) { ++ err = -ENOMEM; ++ goto out; ++ } ++ ++ err = kbase_device_trace_buffer_install(kctx, tb, size); ++ if (err) { ++ vfree(tb); ++ goto out; ++ } ++ } else { ++ err = -EINVAL; ++ goto out; ++ } ++ ++ *kaddr = kctx->jctx.tb; ++ ++ new_reg = kbase_alloc_free_region(kctx, 0, nr_pages, KBASE_REG_ZONE_SAME_VA); ++ if (!new_reg) { ++ err = -ENOMEM; ++ WARN_ON(1); ++ goto out_no_region; ++ } ++ ++ new_reg->cpu_alloc = kbase_alloc_create(0, KBASE_MEM_TYPE_TB); ++ if (IS_ERR_OR_NULL(new_reg->cpu_alloc)) { ++ err = -ENOMEM; ++ new_reg->cpu_alloc = NULL; ++ WARN_ON(1); ++ goto out_no_alloc; ++ } ++ ++ new_reg->gpu_alloc = kbase_mem_phy_alloc_get(new_reg->cpu_alloc); ++ ++ new_reg->cpu_alloc->imported.kctx = kctx; ++ new_reg->flags &= ~KBASE_REG_FREE; ++ new_reg->flags |= KBASE_REG_CPU_CACHED; ++ ++ /* alloc now owns the tb */ ++ owns_tb = 0; ++ ++ if (kbase_add_va_region(kctx, new_reg, vma->vm_start, nr_pages, 1) != 0) { ++ err = -ENOMEM; ++ WARN_ON(1); ++ goto out_no_va_region; ++ } ++ ++ *reg = new_reg; ++ ++ /* map read only, noexec */ ++ vma->vm_flags &= ~(VM_WRITE | VM_MAYWRITE | VM_EXEC | VM_MAYEXEC); ++ /* the rest of the flags is added by the cpu_mmap handler */ ++ ++ dev_dbg(kctx->kbdev->dev, "%s done\n", __func__); ++ return 0; ++ ++out_no_va_region: ++out_no_alloc: ++ kbase_free_alloced_region(new_reg); ++out_no_region: ++ if (owns_tb) { ++ kbase_device_trace_buffer_uninstall(kctx); ++ vfree(tb); ++ } ++out: ++ return err; ++} ++ ++static int kbase_mmu_dump_mmap(struct kbase_context *kctx, struct vm_area_struct *vma, struct kbase_va_region **const reg, void **const kmap_addr) ++{ ++ struct kbase_va_region *new_reg; ++ void *kaddr; ++ u32 nr_pages; ++ size_t size; ++ int err = 0; ++ ++ dev_dbg(kctx->kbdev->dev, "in kbase_mmu_dump_mmap\n"); ++ size = (vma->vm_end - vma->vm_start); ++ nr_pages = size >> PAGE_SHIFT; ++ ++ kaddr = kbase_mmu_dump(kctx, nr_pages); ++ ++ if (!kaddr) { ++ err = -ENOMEM; ++ goto out; ++ } ++ ++ new_reg = kbase_alloc_free_region(kctx, 0, nr_pages, KBASE_REG_ZONE_SAME_VA); ++ if (!new_reg) { ++ err = -ENOMEM; ++ WARN_ON(1); ++ goto out; ++ } ++ ++ new_reg->cpu_alloc = kbase_alloc_create(0, KBASE_MEM_TYPE_RAW); ++ if (IS_ERR_OR_NULL(new_reg->cpu_alloc)) { ++ err = -ENOMEM; ++ new_reg->cpu_alloc = NULL; ++ WARN_ON(1); ++ goto out_no_alloc; ++ } ++ ++ new_reg->gpu_alloc = kbase_mem_phy_alloc_get(new_reg->cpu_alloc); ++ ++ new_reg->flags &= ~KBASE_REG_FREE; ++ new_reg->flags |= KBASE_REG_CPU_CACHED; ++ if (kbase_add_va_region(kctx, new_reg, vma->vm_start, nr_pages, 1) != 0) { ++ err = -ENOMEM; ++ WARN_ON(1); ++ goto out_va_region; ++ } ++ ++ *kmap_addr = kaddr; ++ *reg = new_reg; ++ ++ dev_dbg(kctx->kbdev->dev, "kbase_mmu_dump_mmap done\n"); ++ return 0; ++ ++out_no_alloc: ++out_va_region: ++ kbase_free_alloced_region(new_reg); ++out: ++ return err; ++} ++ ++ ++void kbase_os_mem_map_lock(struct kbase_context *kctx) ++{ ++ struct mm_struct *mm = current->mm; ++ (void)kctx; ++ down_read(&mm->mmap_lock); ++} ++ ++void kbase_os_mem_map_unlock(struct kbase_context *kctx) ++{ ++ struct mm_struct *mm = current->mm; ++ (void)kctx; ++ up_read(&mm->mmap_lock); ++} ++ ++static int kbasep_reg_mmap(struct kbase_context *kctx, ++ struct vm_area_struct *vma, ++ struct kbase_va_region **regm, ++ size_t *nr_pages, size_t *aligned_offset) ++ ++{ ++ int cookie = vma->vm_pgoff - PFN_DOWN(BASE_MEM_COOKIE_BASE); ++ struct kbase_va_region *reg; ++ int err = 0; ++ ++ *aligned_offset = 0; ++ ++ dev_dbg(kctx->kbdev->dev, "in kbasep_reg_mmap\n"); ++ ++ /* SAME_VA stuff, fetch the right region */ ++ reg = kctx->pending_regions[cookie]; ++ if (!reg) { ++ err = -ENOMEM; ++ goto out; ++ } ++ ++ if ((reg->flags & KBASE_REG_GPU_NX) && (reg->nr_pages != *nr_pages)) { ++ /* incorrect mmap size */ ++ /* leave the cookie for a potential later ++ * mapping, or to be reclaimed later when the ++ * context is freed */ ++ err = -ENOMEM; ++ goto out; ++ } ++ ++ if ((vma->vm_flags & VM_READ && !(reg->flags & KBASE_REG_CPU_RD)) || ++ (vma->vm_flags & VM_WRITE && !(reg->flags & KBASE_REG_CPU_WR))) { ++ /* VM flags inconsistent with region flags */ ++ err = -EPERM; ++ dev_err(kctx->kbdev->dev, "%s:%d inconsistent VM flags\n", ++ __FILE__, __LINE__); ++ goto out; ++ } ++ ++ /* adjust down nr_pages to what we have physically */ ++ *nr_pages = kbase_reg_current_backed_size(reg); ++ ++ if (kbase_gpu_mmap(kctx, reg, vma->vm_start + *aligned_offset, ++ reg->nr_pages, 1) != 0) { ++ dev_err(kctx->kbdev->dev, "%s:%d\n", __FILE__, __LINE__); ++ /* Unable to map in GPU space. */ ++ WARN_ON(1); ++ err = -ENOMEM; ++ goto out; ++ } ++ /* no need for the cookie anymore */ ++ kctx->pending_regions[cookie] = NULL; ++ kctx->cookies |= (1UL << cookie); ++ ++ /* ++ * Overwrite the offset with the region start_pfn, so we effectively ++ * map from offset 0 in the region. However subtract the aligned ++ * offset so that when user space trims the mapping the beginning of ++ * the trimmed VMA has the correct vm_pgoff; ++ */ ++ vma->vm_pgoff = reg->start_pfn - ((*aligned_offset)>>PAGE_SHIFT); ++out: ++ *regm = reg; ++ dev_dbg(kctx->kbdev->dev, "kbasep_reg_mmap done\n"); ++ ++ return err; ++} ++ ++int kbase_mmap(struct file *file, struct vm_area_struct *vma) ++{ ++ struct kbase_context *kctx = file->private_data; ++ struct kbase_va_region *reg = NULL; ++ void *kaddr = NULL; ++ size_t nr_pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; ++ int err = 0; ++ int free_on_close = 0; ++ struct device *dev = kctx->kbdev->dev; ++ size_t aligned_offset = 0; ++ ++ dev_dbg(dev, "kbase_mmap\n"); ++ ++ /* strip away corresponding VM_MAY% flags to the VM_% flags requested */ ++ vma->vm_flags &= ~((vma->vm_flags & (VM_READ | VM_WRITE)) << 4); ++ ++ if (0 == nr_pages) { ++ err = -EINVAL; ++ goto out; ++ } ++ ++ if (!(vma->vm_flags & VM_SHARED)) { ++ err = -EINVAL; ++ goto out; ++ } ++ ++ kbase_gpu_vm_lock(kctx); ++ ++ if (vma->vm_pgoff == PFN_DOWN(BASE_MEM_MAP_TRACKING_HANDLE)) { ++ /* The non-mapped tracking helper page */ ++ err = kbase_tracking_page_setup(kctx, vma); ++ goto out_unlock; ++ } ++ ++ /* if not the MTP, verify that the MTP has been mapped */ ++ rcu_read_lock(); ++ /* catches both when the special page isn't present or ++ * when we've forked */ ++ if (rcu_dereference(kctx->process_mm) != current->mm) { ++ err = -EINVAL; ++ rcu_read_unlock(); ++ goto out_unlock; ++ } ++ rcu_read_unlock(); ++ ++ switch (vma->vm_pgoff) { ++ case PFN_DOWN(BASEP_MEM_INVALID_HANDLE): ++ case PFN_DOWN(BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE): ++ /* Illegal handle for direct map */ ++ err = -EINVAL; ++ goto out_unlock; ++ case PFN_DOWN(BASE_MEM_TRACE_BUFFER_HANDLE): ++ err = kbase_trace_buffer_mmap(kctx, vma, ®, &kaddr); ++ if (0 != err) ++ goto out_unlock; ++ dev_dbg(dev, "kbase_trace_buffer_mmap ok\n"); ++ /* free the region on munmap */ ++ free_on_close = 1; ++ break; ++ case PFN_DOWN(BASE_MEM_MMU_DUMP_HANDLE): ++ /* MMU dump */ ++ err = kbase_mmu_dump_mmap(kctx, vma, ®, &kaddr); ++ if (0 != err) ++ goto out_unlock; ++ /* free the region on munmap */ ++ free_on_close = 1; ++ break; ++ case PFN_DOWN(BASE_MEM_COOKIE_BASE) ... ++ PFN_DOWN(BASE_MEM_FIRST_FREE_ADDRESS) - 1: { ++ err = kbasep_reg_mmap(kctx, vma, ®, &nr_pages, ++ &aligned_offset); ++ if (0 != err) ++ goto out_unlock; ++ /* free the region on munmap */ ++ free_on_close = 1; ++ break; ++ } ++ default: { ++ reg = kbase_region_tracker_find_region_enclosing_address(kctx, ++ (u64)vma->vm_pgoff << PAGE_SHIFT); ++ ++ if (reg && !(reg->flags & KBASE_REG_FREE)) { ++ /* will this mapping overflow the size of the region? */ ++ if (nr_pages > (reg->nr_pages - ++ (vma->vm_pgoff - reg->start_pfn))) { ++ err = -ENOMEM; ++ goto out_unlock; ++ } ++ ++ if ((vma->vm_flags & VM_READ && ++ !(reg->flags & KBASE_REG_CPU_RD)) || ++ (vma->vm_flags & VM_WRITE && ++ !(reg->flags & KBASE_REG_CPU_WR))) { ++ /* VM flags inconsistent with region flags */ ++ err = -EPERM; ++ dev_err(dev, "%s:%d inconsistent VM flags\n", ++ __FILE__, __LINE__); ++ goto out_unlock; ++ } ++ ++#ifdef CONFIG_DMA_SHARED_BUFFER ++ if (KBASE_MEM_TYPE_IMPORTED_UMM == ++ reg->cpu_alloc->type) { ++ err = dma_buf_mmap( ++ reg->cpu_alloc->imported.umm.dma_buf, ++ vma, vma->vm_pgoff - reg->start_pfn); ++ goto out_unlock; ++ } ++#endif /* CONFIG_DMA_SHARED_BUFFER */ ++ ++ /* limit what we map to the amount currently backed */ ++ if (reg->cpu_alloc->nents < (vma->vm_pgoff - reg->start_pfn + nr_pages)) { ++ if ((vma->vm_pgoff - reg->start_pfn) >= reg->cpu_alloc->nents) ++ nr_pages = 0; ++ else ++ nr_pages = reg->cpu_alloc->nents - (vma->vm_pgoff - reg->start_pfn); ++ } ++ } else { ++ err = -ENOMEM; ++ goto out_unlock; ++ } ++ } /* default */ ++ } /* switch */ ++ ++ err = kbase_cpu_mmap(reg, vma, kaddr, nr_pages, aligned_offset, free_on_close); ++ ++ if (vma->vm_pgoff == PFN_DOWN(BASE_MEM_MMU_DUMP_HANDLE)) { ++ /* MMU dump - userspace should now have a reference on ++ * the pages, so we can now free the kernel mapping */ ++ vfree(kaddr); ++ } ++ ++out_unlock: ++ kbase_gpu_vm_unlock(kctx); ++out: ++ if (err) ++ dev_err(dev, "mmap failed %d\n", err); ++ ++ return err; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_mmap); ++ ++void *kbase_vmap_prot(struct kbase_context *kctx, u64 gpu_addr, size_t size, ++ unsigned long prot_request, struct kbase_vmap_struct *map) ++{ ++ struct kbase_va_region *reg; ++ unsigned long page_index; ++ unsigned int offset = gpu_addr & ~PAGE_MASK; ++ size_t page_count = PFN_UP(offset + size); ++ phys_addr_t *page_array; ++ struct page **pages; ++ void *cpu_addr = NULL; ++ pgprot_t prot; ++ size_t i; ++ bool sync_needed; ++ ++ if (!size || !map) ++ return NULL; ++ ++ /* check if page_count calculation will wrap */ ++ if (size > ((size_t)-1 / PAGE_SIZE)) ++ return NULL; ++ ++ kbase_gpu_vm_lock(kctx); ++ ++ reg = kbase_region_tracker_find_region_enclosing_address(kctx, gpu_addr); ++ if (!reg || (reg->flags & KBASE_REG_FREE)) ++ goto out_unlock; ++ ++ page_index = (gpu_addr >> PAGE_SHIFT) - reg->start_pfn; ++ ++ /* check if page_index + page_count will wrap */ ++ if (-1UL - page_count < page_index) ++ goto out_unlock; ++ ++ if (page_index + page_count > kbase_reg_current_backed_size(reg)) ++ goto out_unlock; ++ ++ if (reg->flags & KBASE_REG_DONT_NEED) ++ goto out_unlock; ++ ++ /* check access permissions can be satisfied ++ * Intended only for checking KBASE_REG_{CPU,GPU}_{RD,WR} */ ++ if ((reg->flags & prot_request) != prot_request) ++ goto out_unlock; ++ ++ page_array = kbase_get_cpu_phy_pages(reg); ++ if (!page_array) ++ goto out_unlock; ++ ++ pages = kmalloc_array(page_count, sizeof(struct page *), GFP_KERNEL); ++ if (!pages) ++ goto out_unlock; ++ ++ for (i = 0; i < page_count; i++) ++ pages[i] = pfn_to_page(PFN_DOWN(page_array[page_index + i])); ++ ++ prot = PAGE_KERNEL; ++ if (!(reg->flags & KBASE_REG_CPU_CACHED)) { ++ /* Map uncached */ ++ prot = pgprot_writecombine(prot); ++ } ++ /* Note: enforcing a RO prot_request onto prot is not done, since: ++ * - CPU-arch-specific integration required ++ * - kbase_vmap() requires no access checks to be made/enforced */ ++ ++ cpu_addr = vmap(pages, page_count, VM_MAP, prot); ++ ++ kfree(pages); ++ ++ if (!cpu_addr) ++ goto out_unlock; ++ ++ map->gpu_addr = gpu_addr; ++ map->cpu_alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc); ++ map->cpu_pages = &kbase_get_cpu_phy_pages(reg)[page_index]; ++ map->gpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); ++ map->gpu_pages = &kbase_get_gpu_phy_pages(reg)[page_index]; ++ map->addr = (void *)((uintptr_t)cpu_addr + offset); ++ map->size = size; ++ map->is_cached = (reg->flags & KBASE_REG_CPU_CACHED) != 0; ++ sync_needed = map->is_cached; ++ ++#ifdef CONFIG_MALI_COH_KERN ++ /* kernel can use coherent memory if supported */ ++ if (kctx->kbdev->system_coherency == COHERENCY_ACE) ++ sync_needed = false; ++#endif ++ ++ if (sync_needed) { ++ /* Sync first page */ ++ size_t sz = MIN(((size_t) PAGE_SIZE - offset), size); ++ phys_addr_t cpu_pa = map->cpu_pages[0]; ++ phys_addr_t gpu_pa = map->gpu_pages[0]; ++ ++ kbase_sync_single(kctx, cpu_pa, gpu_pa, offset, sz, ++ KBASE_SYNC_TO_CPU); ++ ++ /* Sync middle pages (if any) */ ++ for (i = 1; page_count > 2 && i < page_count - 1; i++) { ++ cpu_pa = map->cpu_pages[i]; ++ gpu_pa = map->gpu_pages[i]; ++ kbase_sync_single(kctx, cpu_pa, gpu_pa, 0, PAGE_SIZE, ++ KBASE_SYNC_TO_CPU); ++ } ++ ++ /* Sync last page (if any) */ ++ if (page_count > 1) { ++ cpu_pa = map->cpu_pages[page_count - 1]; ++ gpu_pa = map->gpu_pages[page_count - 1]; ++ sz = ((offset + size - 1) & ~PAGE_MASK) + 1; ++ kbase_sync_single(kctx, cpu_pa, gpu_pa, 0, sz, ++ KBASE_SYNC_TO_CPU); ++ } ++ } ++ kbase_gpu_vm_unlock(kctx); ++ ++ return map->addr; ++ ++out_unlock: ++ kbase_gpu_vm_unlock(kctx); ++ return NULL; ++} ++ ++void *kbase_vmap(struct kbase_context *kctx, u64 gpu_addr, size_t size, ++ struct kbase_vmap_struct *map) ++{ ++ /* 0 is specified for prot_request to indicate no access checks should ++ * be made. ++ * ++ * As mentioned in kbase_vmap_prot() this means that a kernel-side ++ * CPU-RO mapping is not enforced to allow this to work */ ++ return kbase_vmap_prot(kctx, gpu_addr, size, 0u, map); ++} ++KBASE_EXPORT_TEST_API(kbase_vmap); ++ ++void kbase_vunmap(struct kbase_context *kctx, struct kbase_vmap_struct *map) ++{ ++ void *addr = (void *)((uintptr_t)map->addr & PAGE_MASK); ++ bool sync_needed = map->is_cached; ++ vunmap(addr); ++#ifdef CONFIG_MALI_COH_KERN ++ /* kernel can use coherent memory if supported */ ++ if (kctx->kbdev->system_coherency == COHERENCY_ACE) ++ sync_needed = false; ++#endif ++ if (sync_needed) { ++ off_t offset = (uintptr_t)map->addr & ~PAGE_MASK; ++ size_t size = map->size; ++ size_t page_count = PFN_UP(offset + size); ++ size_t i; ++ ++ /* Sync first page */ ++ size_t sz = MIN(((size_t) PAGE_SIZE - offset), size); ++ phys_addr_t cpu_pa = map->cpu_pages[0]; ++ phys_addr_t gpu_pa = map->gpu_pages[0]; ++ ++ kbase_sync_single(kctx, cpu_pa, gpu_pa, offset, sz, ++ KBASE_SYNC_TO_DEVICE); ++ ++ /* Sync middle pages (if any) */ ++ for (i = 1; page_count > 2 && i < page_count - 1; i++) { ++ cpu_pa = map->cpu_pages[i]; ++ gpu_pa = map->gpu_pages[i]; ++ kbase_sync_single(kctx, cpu_pa, gpu_pa, 0, PAGE_SIZE, ++ KBASE_SYNC_TO_DEVICE); ++ } ++ ++ /* Sync last page (if any) */ ++ if (page_count > 1) { ++ cpu_pa = map->cpu_pages[page_count - 1]; ++ gpu_pa = map->gpu_pages[page_count - 1]; ++ sz = ((offset + size - 1) & ~PAGE_MASK) + 1; ++ kbase_sync_single(kctx, cpu_pa, gpu_pa, 0, sz, ++ KBASE_SYNC_TO_DEVICE); ++ } ++ } ++ map->gpu_addr = 0; ++ map->cpu_alloc = kbase_mem_phy_alloc_put(map->cpu_alloc); ++ map->gpu_alloc = kbase_mem_phy_alloc_put(map->gpu_alloc); ++ map->cpu_pages = NULL; ++ map->gpu_pages = NULL; ++ map->addr = NULL; ++ map->size = 0; ++ map->is_cached = false; ++} ++KBASE_EXPORT_TEST_API(kbase_vunmap); ++ ++void kbasep_os_process_page_usage_update(struct kbase_context *kctx, int pages) ++{ ++ struct mm_struct *mm; ++ ++ rcu_read_lock(); ++ mm = rcu_dereference(kctx->process_mm); ++ if (mm) { ++ atomic_add(pages, &kctx->nonmapped_pages); ++#ifdef SPLIT_RSS_COUNTING ++ add_mm_counter(mm, MM_FILEPAGES, pages); ++#else ++ spin_lock(&mm->page_table_lock); ++ add_mm_counter(mm, MM_FILEPAGES, pages); ++ spin_unlock(&mm->page_table_lock); ++#endif ++ } ++ rcu_read_unlock(); ++} ++ ++static void kbasep_os_process_page_usage_drain(struct kbase_context *kctx) ++{ ++ int pages; ++ struct mm_struct *mm; ++ ++ spin_lock(&kctx->mm_update_lock); ++ mm = rcu_dereference_protected(kctx->process_mm, lockdep_is_held(&kctx->mm_update_lock)); ++ if (!mm) { ++ spin_unlock(&kctx->mm_update_lock); ++ return; ++ } ++ ++ rcu_assign_pointer(kctx->process_mm, NULL); ++ spin_unlock(&kctx->mm_update_lock); ++ synchronize_rcu(); ++ ++ pages = atomic_xchg(&kctx->nonmapped_pages, 0); ++#ifdef SPLIT_RSS_COUNTING ++ add_mm_counter(mm, MM_FILEPAGES, -pages); ++#else ++ spin_lock(&mm->page_table_lock); ++ add_mm_counter(mm, MM_FILEPAGES, -pages); ++ spin_unlock(&mm->page_table_lock); ++#endif ++} ++ ++static void kbase_special_vm_close(struct vm_area_struct *vma) ++{ ++ struct kbase_context *kctx; ++ ++ kctx = vma->vm_private_data; ++ kbasep_os_process_page_usage_drain(kctx); ++} ++ ++static const struct vm_operations_struct kbase_vm_special_ops = { ++ .close = kbase_special_vm_close, ++}; ++ ++static int kbase_tracking_page_setup(struct kbase_context *kctx, struct vm_area_struct *vma) ++{ ++ /* check that this is the only tracking page */ ++ spin_lock(&kctx->mm_update_lock); ++ if (rcu_dereference_protected(kctx->process_mm, lockdep_is_held(&kctx->mm_update_lock))) { ++ spin_unlock(&kctx->mm_update_lock); ++ return -EFAULT; ++ } ++ ++ rcu_assign_pointer(kctx->process_mm, current->mm); ++ ++ spin_unlock(&kctx->mm_update_lock); ++ ++ /* no real access */ ++ vma->vm_flags &= ~(VM_READ | VM_MAYREAD | VM_WRITE | VM_MAYWRITE | VM_EXEC | VM_MAYEXEC); ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0)) ++ vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP | VM_IO; ++#else ++ vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_RESERVED | VM_IO; ++#endif ++ vma->vm_ops = &kbase_vm_special_ops; ++ vma->vm_private_data = kctx; ++ ++ return 0; ++} ++void *kbase_va_alloc(struct kbase_context *kctx, u32 size, struct kbase_hwc_dma_mapping *handle) ++{ ++ int i; ++ int res; ++ void *va; ++ dma_addr_t dma_pa; ++ struct kbase_va_region *reg; ++ phys_addr_t *page_array; ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0)) ++ unsigned long attrs = DMA_ATTR_WRITE_COMBINE; ++#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)) ++ DEFINE_DMA_ATTRS(attrs); ++#endif ++ ++ u32 pages = ((size - 1) >> PAGE_SHIFT) + 1; ++ u32 flags = BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_CPU_WR | ++ BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR; ++ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ KBASE_DEBUG_ASSERT(0 != size); ++ KBASE_DEBUG_ASSERT(0 != pages); ++ ++ if (size == 0) ++ goto err; ++ ++ /* All the alloc calls return zeroed memory */ ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0)) ++ va = dma_alloc_attrs(kctx->kbdev->dev, size, &dma_pa, GFP_KERNEL, ++ attrs); ++#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)) ++ dma_set_attr(DMA_ATTR_WRITE_COMBINE, &attrs); ++ va = dma_alloc_attrs(kctx->kbdev->dev, size, &dma_pa, GFP_KERNEL, ++ &attrs); ++#else ++ va = dma_alloc_writecombine(kctx->kbdev->dev, size, &dma_pa, GFP_KERNEL); ++#endif ++ if (!va) ++ goto err; ++ ++ /* Store the state so we can free it later. */ ++ handle->cpu_va = va; ++ handle->dma_pa = dma_pa; ++ handle->size = size; ++ ++ ++ reg = kbase_alloc_free_region(kctx, 0, pages, KBASE_REG_ZONE_SAME_VA); ++ if (!reg) ++ goto no_reg; ++ ++ reg->flags &= ~KBASE_REG_FREE; ++ if (kbase_update_region_flags(kctx, reg, flags) != 0) ++ goto invalid_flags; ++ ++ reg->cpu_alloc = kbase_alloc_create(pages, KBASE_MEM_TYPE_RAW); ++ if (IS_ERR_OR_NULL(reg->cpu_alloc)) ++ goto no_alloc; ++ ++ reg->gpu_alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc); ++ ++ page_array = kbase_get_cpu_phy_pages(reg); ++ ++ for (i = 0; i < pages; i++) ++ page_array[i] = dma_pa + (i << PAGE_SHIFT); ++ ++ reg->cpu_alloc->nents = pages; ++ ++ kbase_gpu_vm_lock(kctx); ++ res = kbase_gpu_mmap(kctx, reg, (uintptr_t) va, pages, 1); ++ kbase_gpu_vm_unlock(kctx); ++ if (res) ++ goto no_mmap; ++ ++ return va; ++ ++no_mmap: ++ kbase_mem_phy_alloc_put(reg->cpu_alloc); ++ kbase_mem_phy_alloc_put(reg->gpu_alloc); ++no_alloc: ++invalid_flags: ++ kfree(reg); ++no_reg: ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0)) ++ dma_free_attrs(kctx->kbdev->dev, size, va, dma_pa, attrs); ++#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)) ++ dma_free_attrs(kctx->kbdev->dev, size, va, dma_pa, &attrs); ++#else ++ dma_free_writecombine(kctx->kbdev->dev, size, va, dma_pa); ++#endif ++err: ++ return NULL; ++} ++KBASE_EXPORT_SYMBOL(kbase_va_alloc); ++ ++void kbase_va_free(struct kbase_context *kctx, struct kbase_hwc_dma_mapping *handle) ++{ ++ struct kbase_va_region *reg; ++ int err; ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)) && \ ++ (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0)) ++ DEFINE_DMA_ATTRS(attrs); ++#endif ++ ++ KBASE_DEBUG_ASSERT(kctx != NULL); ++ KBASE_DEBUG_ASSERT(handle->cpu_va != NULL); ++ ++ kbase_gpu_vm_lock(kctx); ++ reg = kbase_region_tracker_find_region_base_address(kctx, (uintptr_t)handle->cpu_va); ++ KBASE_DEBUG_ASSERT(reg); ++ err = kbase_gpu_munmap(kctx, reg); ++ kbase_gpu_vm_unlock(kctx); ++ KBASE_DEBUG_ASSERT(!err); ++ ++ kbase_mem_phy_alloc_put(reg->cpu_alloc); ++ kbase_mem_phy_alloc_put(reg->gpu_alloc); ++ kfree(reg); ++ ++#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0)) ++ dma_free_attrs(kctx->kbdev->dev, handle->size, ++ handle->cpu_va, handle->dma_pa, DMA_ATTR_WRITE_COMBINE); ++#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)) ++ dma_set_attr(DMA_ATTR_WRITE_COMBINE, &attrs); ++ dma_free_attrs(kctx->kbdev->dev, handle->size, ++ handle->cpu_va, handle->dma_pa, &attrs); ++#else ++ dma_free_writecombine(kctx->kbdev->dev, handle->size, ++ handle->cpu_va, handle->dma_pa); ++#endif ++} ++KBASE_EXPORT_SYMBOL(kbase_va_free); ++ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_mem_linux.h b/drivers/gpu/arm/midgard/mali_kbase_mem_linux.h +new file mode 100755 +index 000000000000..33b3554f9d82 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_mem_linux.h +@@ -0,0 +1,231 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010, 2012-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/** ++ * @file mali_kbase_mem_linux.h ++ * Base kernel memory APIs, Linux implementation. ++ */ ++ ++#ifndef _KBASE_MEM_LINUX_H_ ++#define _KBASE_MEM_LINUX_H_ ++ ++/** A HWC dump mapping */ ++struct kbase_hwc_dma_mapping { ++ void *cpu_va; ++ dma_addr_t dma_pa; ++ size_t size; ++}; ++ ++struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, ++ u64 va_pages, u64 commit_pages, u64 extent, u64 *flags, ++ u64 *gpu_va); ++int kbase_mem_query(struct kbase_context *kctx, u64 gpu_addr, int query, u64 *const pages); ++int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type, ++ void __user *phandle, u32 padding, u64 *gpu_va, u64 *va_pages, ++ u64 *flags); ++u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, u64 nents, struct base_mem_aliasing_info *ai, u64 *num_pages); ++int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, unsigned int flags, unsigned int mask); ++ ++/** ++ * kbase_mem_commit - Change the physical backing size of a region ++ * ++ * @kctx: The kernel context ++ * @gpu_addr: Handle to the memory region ++ * @new_pages: Number of physical pages to back the region with ++ * ++ * Return: 0 on success or error code ++ */ ++int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages); ++ ++int kbase_mmap(struct file *file, struct vm_area_struct *vma); ++ ++/** ++ * kbase_mem_evictable_init - Initialize the Ephemeral memory the eviction ++ * mechanism. ++ * @kctx: The kbase context to initialize. ++ * ++ * Return: Zero on success or -errno on failure. ++ */ ++int kbase_mem_evictable_init(struct kbase_context *kctx); ++ ++/** ++ * kbase_mem_evictable_deinit - De-initialize the Ephemeral memory eviction ++ * mechanism. ++ * @kctx: The kbase context to de-initialize. ++ */ ++void kbase_mem_evictable_deinit(struct kbase_context *kctx); ++ ++/** ++ * kbase_mem_grow_gpu_mapping - Grow the GPU mapping of an allocation ++ * @kctx: Context the region belongs to ++ * @reg: The GPU region ++ * @new_pages: The number of pages after the grow ++ * @old_pages: The number of pages before the grow ++ * ++ * Return: 0 on success, -errno on error. ++ * ++ * Expand the GPU mapping to encompass the new psychical pages which have ++ * been added to the allocation. ++ * ++ * Note: Caller must be holding the region lock. ++ */ ++int kbase_mem_grow_gpu_mapping(struct kbase_context *kctx, ++ struct kbase_va_region *reg, ++ u64 new_pages, u64 old_pages); ++ ++/** ++ * kbase_mem_evictable_make - Make a physical allocation eligible for eviction ++ * @gpu_alloc: The physical allocation to make evictable ++ * ++ * Return: 0 on success, -errno on error. ++ * ++ * Take the provided region and make all the physical pages within it ++ * reclaimable by the kernel, updating the per-process VM stats as well. ++ * Remove any CPU mappings (as these can't be removed in the shrinker callback ++ * as mmap_lock might already be taken) but leave the GPU mapping intact as ++ * and until the shrinker reclaims the allocation. ++ * ++ * Note: Must be called with the region lock of the containing context. ++ */ ++int kbase_mem_evictable_make(struct kbase_mem_phy_alloc *gpu_alloc); ++ ++/** ++ * kbase_mem_evictable_unmake - Remove a physical allocations eligibility for ++ * eviction. ++ * @alloc: The physical allocation to remove eviction eligibility from. ++ * ++ * Return: True if the allocation had its backing restored and false if ++ * it hasn't. ++ * ++ * Make the physical pages in the region no longer reclaimable and update the ++ * per-process stats, if the shrinker has already evicted the memory then ++ * re-allocate it if the region is still alive. ++ * ++ * Note: Must be called with the region lock of the containing context. ++ */ ++bool kbase_mem_evictable_unmake(struct kbase_mem_phy_alloc *alloc); ++ ++struct kbase_vmap_struct { ++ u64 gpu_addr; ++ struct kbase_mem_phy_alloc *cpu_alloc; ++ struct kbase_mem_phy_alloc *gpu_alloc; ++ phys_addr_t *cpu_pages; ++ phys_addr_t *gpu_pages; ++ void *addr; ++ size_t size; ++ bool is_cached; ++}; ++ ++ ++/** ++ * kbase_vmap_prot - Map a GPU VA range into the kernel safely, only if the ++ * requested access permissions are supported ++ * @kctx: Context the VA range belongs to ++ * @gpu_addr: Start address of VA range ++ * @size: Size of VA range ++ * @prot_request: Flags indicating how the caller will then access the memory ++ * @map: Structure to be given to kbase_vunmap() on freeing ++ * ++ * Return: Kernel-accessible CPU pointer to the VA range, or NULL on error ++ * ++ * Map a GPU VA Range into the kernel. The VA range must be contained within a ++ * GPU memory region. Appropriate CPU cache-flushing operations are made as ++ * required, dependent on the CPU mapping for the memory region. ++ * ++ * This is safer than using kmap() on the pages directly, ++ * because the pages here are refcounted to prevent freeing (and hence reuse ++ * elsewhere in the system) until an kbase_vunmap() ++ * ++ * The flags in @prot_request should use KBASE_REG_{CPU,GPU}_{RD,WR}, to check ++ * whether the region should allow the intended access, and return an error if ++ * disallowed. This is essential for security of imported memory, particularly ++ * a user buf from SHM mapped into the process as RO. In that case, write ++ * access must be checked if the intention is for kernel to write to the ++ * memory. ++ * ++ * The checks are also there to help catch access errors on memory where ++ * security is not a concern: imported memory that is always RW, and memory ++ * that was allocated and owned by the process attached to @kctx. In this case, ++ * it helps to identify memory that was was mapped with the wrong access type. ++ * ++ * Note: KBASE_REG_GPU_{RD,WR} flags are currently supported for legacy cases ++ * where either the security of memory is solely dependent on those flags, or ++ * when userspace code was expecting only the GPU to access the memory (e.g. HW ++ * workarounds). ++ * ++ */ ++void *kbase_vmap_prot(struct kbase_context *kctx, u64 gpu_addr, size_t size, ++ unsigned long prot_request, struct kbase_vmap_struct *map); ++ ++/** ++ * kbase_vmap - Map a GPU VA range into the kernel safely ++ * @kctx: Context the VA range belongs to ++ * @gpu_addr: Start address of VA range ++ * @size: Size of VA range ++ * @map: Structure to be given to kbase_vunmap() on freeing ++ * ++ * Return: Kernel-accessible CPU pointer to the VA range, or NULL on error ++ * ++ * Map a GPU VA Range into the kernel. The VA range must be contained within a ++ * GPU memory region. Appropriate CPU cache-flushing operations are made as ++ * required, dependent on the CPU mapping for the memory region. ++ * ++ * This is safer than using kmap() on the pages directly, ++ * because the pages here are refcounted to prevent freeing (and hence reuse ++ * elsewhere in the system) until an kbase_vunmap() ++ * ++ * kbase_vmap_prot() should be used in preference, since kbase_vmap() makes no ++ * checks to ensure the security of e.g. imported user bufs from RO SHM. ++ */ ++void *kbase_vmap(struct kbase_context *kctx, u64 gpu_addr, size_t size, ++ struct kbase_vmap_struct *map); ++ ++/** ++ * kbase_vunmap - Unmap a GPU VA range from the kernel ++ * @kctx: Context the VA range belongs to ++ * @map: Structure describing the mapping from the corresponding kbase_vmap() ++ * call ++ * ++ * Unmaps a GPU VA range from the kernel, given its @map structure obtained ++ * from kbase_vmap(). Appropriate CPU cache-flushing operations are made as ++ * required, dependent on the CPU mapping for the memory region. ++ * ++ * The reference taken on pages during kbase_vmap() is released. ++ */ ++void kbase_vunmap(struct kbase_context *kctx, struct kbase_vmap_struct *map); ++ ++/** @brief Allocate memory from kernel space and map it onto the GPU ++ * ++ * @param kctx The context used for the allocation/mapping ++ * @param size The size of the allocation in bytes ++ * @param handle An opaque structure used to contain the state needed to free the memory ++ * @return the VA for kernel space and GPU MMU ++ */ ++void *kbase_va_alloc(struct kbase_context *kctx, u32 size, struct kbase_hwc_dma_mapping *handle); ++ ++/** @brief Free/unmap memory allocated by kbase_va_alloc ++ * ++ * @param kctx The context used for the allocation/mapping ++ * @param handle An opaque structure returned by the kbase_va_alloc function. ++ */ ++void kbase_va_free(struct kbase_context *kctx, struct kbase_hwc_dma_mapping *handle); ++ ++extern const struct vm_operations_struct kbase_vm_ops; ++ ++#endif /* _KBASE_MEM_LINUX_H_ */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_mem_lowlevel.h b/drivers/gpu/arm/midgard/mali_kbase_mem_lowlevel.h +new file mode 100755 +index 000000000000..9725fd3f05df +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_mem_lowlevel.h +@@ -0,0 +1,45 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2014, 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#ifndef _KBASE_MEM_LOWLEVEL_H ++#define _KBASE_MEM_LOWLEVEL_H ++ ++#ifndef _KBASE_H_ ++#error "Don't include this file directly, use mali_kbase.h instead" ++#endif ++ ++#include ++ ++/** ++ * @brief Flags for kbase_phy_allocator_pages_alloc ++ */ ++#define KBASE_PHY_PAGES_FLAG_DEFAULT (0) /** Default allocation flag */ ++#define KBASE_PHY_PAGES_FLAG_CLEAR (1 << 0) /** Clear the pages after allocation */ ++#define KBASE_PHY_PAGES_FLAG_POISON (1 << 1) /** Fill the memory with a poison value */ ++ ++#define KBASE_PHY_PAGES_SUPPORTED_FLAGS (KBASE_PHY_PAGES_FLAG_DEFAULT|KBASE_PHY_PAGES_FLAG_CLEAR|KBASE_PHY_PAGES_FLAG_POISON) ++ ++#define KBASE_PHY_PAGES_POISON_VALUE 0xFD /** Value to fill the memory with when KBASE_PHY_PAGES_FLAG_POISON is set */ ++ ++enum kbase_sync_type { ++ KBASE_SYNC_TO_CPU, ++ KBASE_SYNC_TO_DEVICE ++}; ++ ++#endif /* _KBASE_LOWLEVEL_H */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_mem_pool.c b/drivers/gpu/arm/midgard/mali_kbase_mem_pool.c +new file mode 100755 +index 000000000000..a8269940a037 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_mem_pool.c +@@ -0,0 +1,569 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#define pool_dbg(pool, format, ...) \ ++ dev_dbg(pool->kbdev->dev, "%s-pool [%zu/%zu]: " format, \ ++ (pool->next_pool) ? "kctx" : "kbdev", \ ++ kbase_mem_pool_size(pool), \ ++ kbase_mem_pool_max_size(pool), \ ++ ##__VA_ARGS__) ++ ++#define NOT_DIRTY false ++#define NOT_RECLAIMED false ++ ++static inline void kbase_mem_pool_lock(struct kbase_mem_pool *pool) ++{ ++ spin_lock(&pool->pool_lock); ++} ++ ++static inline void kbase_mem_pool_unlock(struct kbase_mem_pool *pool) ++{ ++ spin_unlock(&pool->pool_lock); ++} ++ ++static size_t kbase_mem_pool_capacity(struct kbase_mem_pool *pool) ++{ ++ ssize_t max_size = kbase_mem_pool_max_size(pool); ++ ssize_t cur_size = kbase_mem_pool_size(pool); ++ ++ return max(max_size - cur_size, (ssize_t)0); ++} ++ ++static bool kbase_mem_pool_is_full(struct kbase_mem_pool *pool) ++{ ++ return kbase_mem_pool_size(pool) >= kbase_mem_pool_max_size(pool); ++} ++ ++static bool kbase_mem_pool_is_empty(struct kbase_mem_pool *pool) ++{ ++ return kbase_mem_pool_size(pool) == 0; ++} ++ ++static void kbase_mem_pool_add_locked(struct kbase_mem_pool *pool, ++ struct page *p) ++{ ++ lockdep_assert_held(&pool->pool_lock); ++ ++ list_add(&p->lru, &pool->page_list); ++ pool->cur_size++; ++ ++ pool_dbg(pool, "added page\n"); ++} ++ ++static void kbase_mem_pool_add(struct kbase_mem_pool *pool, struct page *p) ++{ ++ kbase_mem_pool_lock(pool); ++ kbase_mem_pool_add_locked(pool, p); ++ kbase_mem_pool_unlock(pool); ++} ++ ++static void kbase_mem_pool_add_list_locked(struct kbase_mem_pool *pool, ++ struct list_head *page_list, size_t nr_pages) ++{ ++ lockdep_assert_held(&pool->pool_lock); ++ ++ list_splice(page_list, &pool->page_list); ++ pool->cur_size += nr_pages; ++ ++ pool_dbg(pool, "added %zu pages\n", nr_pages); ++} ++ ++static void kbase_mem_pool_add_list(struct kbase_mem_pool *pool, ++ struct list_head *page_list, size_t nr_pages) ++{ ++ kbase_mem_pool_lock(pool); ++ kbase_mem_pool_add_list_locked(pool, page_list, nr_pages); ++ kbase_mem_pool_unlock(pool); ++} ++ ++static struct page *kbase_mem_pool_remove_locked(struct kbase_mem_pool *pool) ++{ ++ struct page *p; ++ ++ lockdep_assert_held(&pool->pool_lock); ++ ++ if (kbase_mem_pool_is_empty(pool)) ++ return NULL; ++ ++ p = list_first_entry(&pool->page_list, struct page, lru); ++ list_del_init(&p->lru); ++ pool->cur_size--; ++ ++ pool_dbg(pool, "removed page\n"); ++ ++ return p; ++} ++ ++static struct page *kbase_mem_pool_remove(struct kbase_mem_pool *pool) ++{ ++ struct page *p; ++ ++ kbase_mem_pool_lock(pool); ++ p = kbase_mem_pool_remove_locked(pool); ++ kbase_mem_pool_unlock(pool); ++ ++ return p; ++} ++ ++static void kbase_mem_pool_sync_page(struct kbase_mem_pool *pool, ++ struct page *p) ++{ ++ struct device *dev = pool->kbdev->dev; ++ ++ dma_sync_single_for_device(dev, kbase_dma_addr(p), ++ PAGE_SIZE, DMA_BIDIRECTIONAL); ++} ++ ++static void kbase_mem_pool_zero_page(struct kbase_mem_pool *pool, ++ struct page *p) ++{ ++ clear_highpage(p); ++ kbase_mem_pool_sync_page(pool, p); ++} ++ ++static void kbase_mem_pool_spill(struct kbase_mem_pool *next_pool, ++ struct page *p) ++{ ++ /* Zero page before spilling */ ++ kbase_mem_pool_zero_page(next_pool, p); ++ ++ kbase_mem_pool_add(next_pool, p); ++} ++ ++struct page *kbase_mem_alloc_page(struct kbase_device *kbdev) ++{ ++ struct page *p; ++ gfp_t gfp; ++ struct device *dev = kbdev->dev; ++ dma_addr_t dma_addr; ++ ++#if defined(CONFIG_ARM) && !defined(CONFIG_HAVE_DMA_ATTRS) && \ ++ LINUX_VERSION_CODE < KERNEL_VERSION(3, 5, 0) ++ /* DMA cache sync fails for HIGHMEM before 3.5 on ARM */ ++ gfp = GFP_USER | __GFP_ZERO; ++#else ++ gfp = GFP_HIGHUSER | __GFP_ZERO; ++#endif ++ ++ if (current->flags & PF_KTHREAD) { ++ /* Don't trigger OOM killer from kernel threads, e.g. when ++ * growing memory on GPU page fault */ ++ gfp |= __GFP_NORETRY; ++ } ++ ++ p = alloc_page(gfp); ++ if (!p) ++ return NULL; ++ ++ dma_addr = dma_map_page(dev, p, 0, PAGE_SIZE, DMA_BIDIRECTIONAL); ++ if (dma_mapping_error(dev, dma_addr)) { ++ __free_page(p); ++ return NULL; ++ } ++ ++ WARN_ON(dma_addr != page_to_phys(p)); ++ ++ kbase_set_dma_addr(p, dma_addr); ++ ++ return p; ++} ++ ++static void kbase_mem_pool_free_page(struct kbase_mem_pool *pool, ++ struct page *p) ++{ ++ struct device *dev = pool->kbdev->dev; ++ dma_addr_t dma_addr = kbase_dma_addr(p); ++ ++ dma_unmap_page(dev, dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL); ++ kbase_clear_dma_addr(p); ++ __free_page(p); ++ ++ pool_dbg(pool, "freed page to kernel\n"); ++} ++ ++static size_t kbase_mem_pool_shrink_locked(struct kbase_mem_pool *pool, ++ size_t nr_to_shrink) ++{ ++ struct page *p; ++ size_t i; ++ ++ lockdep_assert_held(&pool->pool_lock); ++ ++ for (i = 0; i < nr_to_shrink && !kbase_mem_pool_is_empty(pool); i++) { ++ p = kbase_mem_pool_remove_locked(pool); ++ kbase_mem_pool_free_page(pool, p); ++ } ++ ++ return i; ++} ++ ++static size_t kbase_mem_pool_shrink(struct kbase_mem_pool *pool, ++ size_t nr_to_shrink) ++{ ++ size_t nr_freed; ++ ++ kbase_mem_pool_lock(pool); ++ nr_freed = kbase_mem_pool_shrink_locked(pool, nr_to_shrink); ++ kbase_mem_pool_unlock(pool); ++ ++ return nr_freed; ++} ++ ++int kbase_mem_pool_grow(struct kbase_mem_pool *pool, ++ size_t nr_to_grow) ++{ ++ struct page *p; ++ size_t i; ++ ++ for (i = 0; i < nr_to_grow; i++) { ++ p = kbase_mem_alloc_page(pool->kbdev); ++ if (!p) ++ return -ENOMEM; ++ kbase_mem_pool_add(pool, p); ++ } ++ ++ return 0; ++} ++ ++void kbase_mem_pool_trim(struct kbase_mem_pool *pool, size_t new_size) ++{ ++ size_t cur_size; ++ ++ cur_size = kbase_mem_pool_size(pool); ++ ++ if (new_size > pool->max_size) ++ new_size = pool->max_size; ++ ++ if (new_size < cur_size) ++ kbase_mem_pool_shrink(pool, cur_size - new_size); ++ else if (new_size > cur_size) ++ kbase_mem_pool_grow(pool, new_size - cur_size); ++} ++ ++void kbase_mem_pool_set_max_size(struct kbase_mem_pool *pool, size_t max_size) ++{ ++ size_t cur_size; ++ size_t nr_to_shrink; ++ ++ kbase_mem_pool_lock(pool); ++ ++ pool->max_size = max_size; ++ ++ cur_size = kbase_mem_pool_size(pool); ++ if (max_size < cur_size) { ++ nr_to_shrink = cur_size - max_size; ++ kbase_mem_pool_shrink_locked(pool, nr_to_shrink); ++ } ++ ++ kbase_mem_pool_unlock(pool); ++} ++ ++ ++static unsigned long kbase_mem_pool_reclaim_count_objects(struct shrinker *s, ++ struct shrink_control *sc) ++{ ++ struct kbase_mem_pool *pool; ++ ++ pool = container_of(s, struct kbase_mem_pool, reclaim); ++ pool_dbg(pool, "reclaim count: %zu\n", kbase_mem_pool_size(pool)); ++ return kbase_mem_pool_size(pool); ++} ++ ++static unsigned long kbase_mem_pool_reclaim_scan_objects(struct shrinker *s, ++ struct shrink_control *sc) ++{ ++ struct kbase_mem_pool *pool; ++ unsigned long freed; ++ ++ pool = container_of(s, struct kbase_mem_pool, reclaim); ++ ++ pool_dbg(pool, "reclaim scan %ld:\n", sc->nr_to_scan); ++ ++ freed = kbase_mem_pool_shrink(pool, sc->nr_to_scan); ++ ++ pool_dbg(pool, "reclaim freed %ld pages\n", freed); ++ ++ return freed; ++} ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) ++static int kbase_mem_pool_reclaim_shrink(struct shrinker *s, ++ struct shrink_control *sc) ++{ ++ if (sc->nr_to_scan == 0) ++ return kbase_mem_pool_reclaim_count_objects(s, sc); ++ ++ return kbase_mem_pool_reclaim_scan_objects(s, sc); ++} ++#endif ++ ++int kbase_mem_pool_init(struct kbase_mem_pool *pool, ++ size_t max_size, ++ struct kbase_device *kbdev, ++ struct kbase_mem_pool *next_pool) ++{ ++ pool->cur_size = 0; ++ pool->max_size = max_size; ++ pool->kbdev = kbdev; ++ pool->next_pool = next_pool; ++ ++ spin_lock_init(&pool->pool_lock); ++ INIT_LIST_HEAD(&pool->page_list); ++ ++ /* Register shrinker */ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) ++ pool->reclaim.shrink = kbase_mem_pool_reclaim_shrink; ++#else ++ pool->reclaim.count_objects = kbase_mem_pool_reclaim_count_objects; ++ pool->reclaim.scan_objects = kbase_mem_pool_reclaim_scan_objects; ++#endif ++ pool->reclaim.seeks = DEFAULT_SEEKS; ++ /* Kernel versions prior to 3.1 : ++ * struct shrinker does not define batch */ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 1, 0) ++ pool->reclaim.batch = 0; ++#endif ++ register_shrinker(&pool->reclaim); ++ ++ pool_dbg(pool, "initialized\n"); ++ ++ return 0; ++} ++ ++void kbase_mem_pool_term(struct kbase_mem_pool *pool) ++{ ++ struct kbase_mem_pool *next_pool = pool->next_pool; ++ struct page *p; ++ size_t nr_to_spill = 0; ++ LIST_HEAD(spill_list); ++ int i; ++ ++ pool_dbg(pool, "terminate()\n"); ++ ++ unregister_shrinker(&pool->reclaim); ++ ++ kbase_mem_pool_lock(pool); ++ pool->max_size = 0; ++ ++ if (next_pool && !kbase_mem_pool_is_full(next_pool)) { ++ /* Spill to next pool (may overspill) */ ++ nr_to_spill = kbase_mem_pool_capacity(next_pool); ++ nr_to_spill = min(kbase_mem_pool_size(pool), nr_to_spill); ++ ++ /* Zero pages first without holding the next_pool lock */ ++ for (i = 0; i < nr_to_spill; i++) { ++ p = kbase_mem_pool_remove_locked(pool); ++ kbase_mem_pool_zero_page(pool, p); ++ list_add(&p->lru, &spill_list); ++ } ++ } ++ ++ while (!kbase_mem_pool_is_empty(pool)) { ++ /* Free remaining pages to kernel */ ++ p = kbase_mem_pool_remove_locked(pool); ++ kbase_mem_pool_free_page(pool, p); ++ } ++ ++ kbase_mem_pool_unlock(pool); ++ ++ if (next_pool && nr_to_spill) { ++ /* Add new page list to next_pool */ ++ kbase_mem_pool_add_list(next_pool, &spill_list, nr_to_spill); ++ ++ pool_dbg(pool, "terminate() spilled %zu pages\n", nr_to_spill); ++ } ++ ++ pool_dbg(pool, "terminated\n"); ++} ++ ++struct page *kbase_mem_pool_alloc(struct kbase_mem_pool *pool) ++{ ++ struct page *p; ++ ++ do { ++ pool_dbg(pool, "alloc()\n"); ++ p = kbase_mem_pool_remove(pool); ++ ++ if (p) ++ return p; ++ ++ pool = pool->next_pool; ++ } while (pool); ++ ++ return NULL; ++} ++ ++void kbase_mem_pool_free(struct kbase_mem_pool *pool, struct page *p, ++ bool dirty) ++{ ++ struct kbase_mem_pool *next_pool = pool->next_pool; ++ ++ pool_dbg(pool, "free()\n"); ++ ++ if (!kbase_mem_pool_is_full(pool)) { ++ /* Add to our own pool */ ++ if (dirty) ++ kbase_mem_pool_sync_page(pool, p); ++ ++ kbase_mem_pool_add(pool, p); ++ } else if (next_pool && !kbase_mem_pool_is_full(next_pool)) { ++ /* Spill to next pool */ ++ kbase_mem_pool_spill(next_pool, p); ++ } else { ++ /* Free page */ ++ kbase_mem_pool_free_page(pool, p); ++ } ++} ++ ++int kbase_mem_pool_alloc_pages(struct kbase_mem_pool *pool, size_t nr_pages, ++ phys_addr_t *pages) ++{ ++ struct page *p; ++ size_t nr_from_pool; ++ size_t i; ++ int err = -ENOMEM; ++ ++ pool_dbg(pool, "alloc_pages(%zu):\n", nr_pages); ++ ++ /* Get pages from this pool */ ++ kbase_mem_pool_lock(pool); ++ nr_from_pool = min(nr_pages, kbase_mem_pool_size(pool)); ++ for (i = 0; i < nr_from_pool; i++) { ++ p = kbase_mem_pool_remove_locked(pool); ++ pages[i] = page_to_phys(p); ++ } ++ kbase_mem_pool_unlock(pool); ++ ++ if (i != nr_pages && pool->next_pool) { ++ /* Allocate via next pool */ ++ err = kbase_mem_pool_alloc_pages(pool->next_pool, ++ nr_pages - i, pages + i); ++ ++ if (err) ++ goto err_rollback; ++ ++ i += nr_pages - i; ++ } ++ ++ /* Get any remaining pages from kernel */ ++ for (; i < nr_pages; i++) { ++ p = kbase_mem_alloc_page(pool->kbdev); ++ if (!p) ++ goto err_rollback; ++ pages[i] = page_to_phys(p); ++ } ++ ++ pool_dbg(pool, "alloc_pages(%zu) done\n", nr_pages); ++ ++ return 0; ++ ++err_rollback: ++ kbase_mem_pool_free_pages(pool, i, pages, NOT_DIRTY, NOT_RECLAIMED); ++ return err; ++} ++ ++static void kbase_mem_pool_add_array(struct kbase_mem_pool *pool, ++ size_t nr_pages, phys_addr_t *pages, bool zero, bool sync) ++{ ++ struct page *p; ++ size_t nr_to_pool = 0; ++ LIST_HEAD(new_page_list); ++ size_t i; ++ ++ if (!nr_pages) ++ return; ++ ++ pool_dbg(pool, "add_array(%zu, zero=%d, sync=%d):\n", ++ nr_pages, zero, sync); ++ ++ /* Zero/sync pages first without holding the pool lock */ ++ for (i = 0; i < nr_pages; i++) { ++ if (unlikely(!pages[i])) ++ continue; ++ ++ p = phys_to_page(pages[i]); ++ ++ if (zero) ++ kbase_mem_pool_zero_page(pool, p); ++ else if (sync) ++ kbase_mem_pool_sync_page(pool, p); ++ ++ list_add(&p->lru, &new_page_list); ++ nr_to_pool++; ++ pages[i] = 0; ++ } ++ ++ /* Add new page list to pool */ ++ kbase_mem_pool_add_list(pool, &new_page_list, nr_to_pool); ++ ++ pool_dbg(pool, "add_array(%zu) added %zu pages\n", ++ nr_pages, nr_to_pool); ++} ++ ++void kbase_mem_pool_free_pages(struct kbase_mem_pool *pool, size_t nr_pages, ++ phys_addr_t *pages, bool dirty, bool reclaimed) ++{ ++ struct kbase_mem_pool *next_pool = pool->next_pool; ++ struct page *p; ++ size_t nr_to_pool; ++ LIST_HEAD(to_pool_list); ++ size_t i = 0; ++ ++ pool_dbg(pool, "free_pages(%zu):\n", nr_pages); ++ ++ if (!reclaimed) { ++ /* Add to this pool */ ++ nr_to_pool = kbase_mem_pool_capacity(pool); ++ nr_to_pool = min(nr_pages, nr_to_pool); ++ ++ kbase_mem_pool_add_array(pool, nr_to_pool, pages, false, dirty); ++ ++ i += nr_to_pool; ++ ++ if (i != nr_pages && next_pool) { ++ /* Spill to next pool (may overspill) */ ++ nr_to_pool = kbase_mem_pool_capacity(next_pool); ++ nr_to_pool = min(nr_pages - i, nr_to_pool); ++ ++ kbase_mem_pool_add_array(next_pool, nr_to_pool, ++ pages + i, true, dirty); ++ i += nr_to_pool; ++ } ++ } ++ ++ /* Free any remaining pages to kernel */ ++ for (; i < nr_pages; i++) { ++ if (unlikely(!pages[i])) ++ continue; ++ ++ p = phys_to_page(pages[i]); ++ ++ kbase_mem_pool_free_page(pool, p); ++ pages[i] = 0; ++ } ++ ++ pool_dbg(pool, "free_pages(%zu) done\n", nr_pages); ++} +diff --git a/drivers/gpu/arm/midgard/mali_kbase_mem_pool_debugfs.c b/drivers/gpu/arm/midgard/mali_kbase_mem_pool_debugfs.c +new file mode 100755 +index 000000000000..585fba036c9e +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_mem_pool_debugfs.c +@@ -0,0 +1,81 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include ++#include ++ ++#include ++ ++#ifdef CONFIG_DEBUG_FS ++ ++static int kbase_mem_pool_debugfs_size_get(void *data, u64 *val) ++{ ++ struct kbase_mem_pool *pool = (struct kbase_mem_pool *)data; ++ ++ *val = kbase_mem_pool_size(pool); ++ ++ return 0; ++} ++ ++static int kbase_mem_pool_debugfs_size_set(void *data, u64 val) ++{ ++ struct kbase_mem_pool *pool = (struct kbase_mem_pool *)data; ++ ++ kbase_mem_pool_trim(pool, val); ++ ++ return 0; ++} ++ ++DEFINE_SIMPLE_ATTRIBUTE(kbase_mem_pool_debugfs_size_fops, ++ kbase_mem_pool_debugfs_size_get, ++ kbase_mem_pool_debugfs_size_set, ++ "%llu\n"); ++ ++static int kbase_mem_pool_debugfs_max_size_get(void *data, u64 *val) ++{ ++ struct kbase_mem_pool *pool = (struct kbase_mem_pool *)data; ++ ++ *val = kbase_mem_pool_max_size(pool); ++ ++ return 0; ++} ++ ++static int kbase_mem_pool_debugfs_max_size_set(void *data, u64 val) ++{ ++ struct kbase_mem_pool *pool = (struct kbase_mem_pool *)data; ++ ++ kbase_mem_pool_set_max_size(pool, val); ++ ++ return 0; ++} ++ ++DEFINE_SIMPLE_ATTRIBUTE(kbase_mem_pool_debugfs_max_size_fops, ++ kbase_mem_pool_debugfs_max_size_get, ++ kbase_mem_pool_debugfs_max_size_set, ++ "%llu\n"); ++ ++void kbase_mem_pool_debugfs_init(struct dentry *parent, ++ struct kbase_mem_pool *pool) ++{ ++ debugfs_create_file("mem_pool_size", S_IRUGO | S_IWUSR, parent, ++ pool, &kbase_mem_pool_debugfs_size_fops); ++ ++ debugfs_create_file("mem_pool_max_size", S_IRUGO | S_IWUSR, parent, ++ pool, &kbase_mem_pool_debugfs_max_size_fops); ++} ++ ++#endif /* CONFIG_DEBUG_FS */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_mem_pool_debugfs.h b/drivers/gpu/arm/midgard/mali_kbase_mem_pool_debugfs.h +new file mode 100755 +index 000000000000..1442854e8956 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_mem_pool_debugfs.h +@@ -0,0 +1,36 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KBASE_MEM_POOL_DEBUGFS_H ++#define _KBASE_MEM_POOL_DEBUGFS_H ++ ++#include ++ ++/** ++ * kbase_mem_pool_debugfs_init - add debugfs knobs for @pool ++ * @parent: Parent debugfs dentry ++ * @pool: Memory pool to control ++ * ++ * Adds two debugfs files under @parent: ++ * - mem_pool_size: get/set the current size of @pool ++ * - mem_pool_max_size: get/set the max size of @pool ++ */ ++void kbase_mem_pool_debugfs_init(struct dentry *parent, ++ struct kbase_mem_pool *pool); ++ ++#endif /*_KBASE_MEM_POOL_DEBUGFS_H*/ ++ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_mem_profile_debugfs.c b/drivers/gpu/arm/midgard/mali_kbase_mem_profile_debugfs.c +new file mode 100755 +index 000000000000..d58fd8d62fde +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_mem_profile_debugfs.c +@@ -0,0 +1,121 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include ++ ++#ifdef CONFIG_DEBUG_FS ++ ++/** Show callback for the @c mem_profile debugfs file. ++ * ++ * This function is called to get the contents of the @c mem_profile debugfs ++ * file. This is a report of current memory usage and distribution in userspace. ++ * ++ * @param sfile The debugfs entry ++ * @param data Data associated with the entry ++ * ++ * @return 0 if it successfully prints data in debugfs entry file, non-zero otherwise ++ */ ++static int kbasep_mem_profile_seq_show(struct seq_file *sfile, void *data) ++{ ++ struct kbase_context *kctx = sfile->private; ++ ++ mutex_lock(&kctx->mem_profile_lock); ++ ++ seq_write(sfile, kctx->mem_profile_data, kctx->mem_profile_size); ++ ++ seq_putc(sfile, '\n'); ++ ++ mutex_unlock(&kctx->mem_profile_lock); ++ ++ return 0; ++} ++ ++/* ++ * File operations related to debugfs entry for mem_profile ++ */ ++static int kbasep_mem_profile_debugfs_open(struct inode *in, struct file *file) ++{ ++ return single_open(file, kbasep_mem_profile_seq_show, in->i_private); ++} ++ ++static const struct file_operations kbasep_mem_profile_debugfs_fops = { ++ .open = kbasep_mem_profile_debugfs_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = single_release, ++}; ++ ++int kbasep_mem_profile_debugfs_insert(struct kbase_context *kctx, char *data, ++ size_t size) ++{ ++ int err = 0; ++ ++ mutex_lock(&kctx->mem_profile_lock); ++ ++ dev_dbg(kctx->kbdev->dev, "initialised: %d", ++ kbase_ctx_flag(kctx, KCTX_MEM_PROFILE_INITIALIZED)); ++ ++ if (!kbase_ctx_flag(kctx, KCTX_MEM_PROFILE_INITIALIZED)) { ++ if (!debugfs_create_file("mem_profile", S_IRUGO, ++ kctx->kctx_dentry, kctx, ++ &kbasep_mem_profile_debugfs_fops)) { ++ err = -EAGAIN; ++ } else { ++ kbase_ctx_flag_set(kctx, ++ KCTX_MEM_PROFILE_INITIALIZED); ++ } ++ } ++ ++ if (kbase_ctx_flag(kctx, KCTX_MEM_PROFILE_INITIALIZED)) { ++ kfree(kctx->mem_profile_data); ++ kctx->mem_profile_data = data; ++ kctx->mem_profile_size = size; ++ } else { ++ kfree(data); ++ } ++ ++ dev_dbg(kctx->kbdev->dev, "returning: %d, initialised: %d", ++ err, kbase_ctx_flag(kctx, KCTX_MEM_PROFILE_INITIALIZED)); ++ ++ mutex_unlock(&kctx->mem_profile_lock); ++ ++ return err; ++} ++ ++void kbasep_mem_profile_debugfs_remove(struct kbase_context *kctx) ++{ ++ mutex_lock(&kctx->mem_profile_lock); ++ ++ dev_dbg(kctx->kbdev->dev, "initialised: %d", ++ kbase_ctx_flag(kctx, KCTX_MEM_PROFILE_INITIALIZED)); ++ ++ kfree(kctx->mem_profile_data); ++ kctx->mem_profile_data = NULL; ++ kctx->mem_profile_size = 0; ++ ++ mutex_unlock(&kctx->mem_profile_lock); ++} ++ ++#else /* CONFIG_DEBUG_FS */ ++ ++int kbasep_mem_profile_debugfs_insert(struct kbase_context *kctx, char *data, ++ size_t size) ++{ ++ kfree(data); ++ return 0; ++} ++#endif /* CONFIG_DEBUG_FS */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_mem_profile_debugfs.h b/drivers/gpu/arm/midgard/mali_kbase_mem_profile_debugfs.h +new file mode 100755 +index 000000000000..a1dc2e0b165b +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_mem_profile_debugfs.h +@@ -0,0 +1,59 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/** ++ * @file mali_kbase_mem_profile_debugfs.h ++ * Header file for mem profiles entries in debugfs ++ * ++ */ ++ ++#ifndef _KBASE_MEM_PROFILE_DEBUGFS_H ++#define _KBASE_MEM_PROFILE_DEBUGFS_H ++ ++#include ++#include ++ ++/** ++ * @brief Remove entry from Mali memory profile debugfs ++ */ ++void kbasep_mem_profile_debugfs_remove(struct kbase_context *kctx); ++ ++/** ++ * @brief Insert @p data to the debugfs file so it can be read by userspace ++ * ++ * The function takes ownership of @p data and frees it later when new data ++ * is inserted. ++ * ++ * If the debugfs entry corresponding to the @p kctx doesn't exist, ++ * an attempt will be made to create it. ++ * ++ * @param kctx The context whose debugfs file @p data should be inserted to ++ * @param data A NULL-terminated string to be inserted to the debugfs file, ++ * without the trailing new line character ++ * @param size The length of the @p data string ++ * @return 0 if @p data inserted correctly ++ * -EAGAIN in case of error ++ * @post @ref mem_profile_initialized will be set to @c true ++ * the first time this function succeeds. ++ */ ++int kbasep_mem_profile_debugfs_insert(struct kbase_context *kctx, char *data, ++ size_t size); ++ ++#endif /*_KBASE_MEM_PROFILE_DEBUGFS_H*/ ++ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_mem_profile_debugfs_buf_size.h b/drivers/gpu/arm/midgard/mali_kbase_mem_profile_debugfs_buf_size.h +new file mode 100755 +index 000000000000..82f0702974c2 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_mem_profile_debugfs_buf_size.h +@@ -0,0 +1,33 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/** ++ * @file mali_kbase_mem_profile_debugfs_buf_size.h ++ * Header file for the size of the buffer to accumulate the histogram report text in ++ */ ++ ++#ifndef _KBASE_MEM_PROFILE_DEBUGFS_BUF_SIZE_H_ ++#define _KBASE_MEM_PROFILE_DEBUGFS_BUF_SIZE_H_ ++ ++/** ++ * The size of the buffer to accumulate the histogram report text in ++ * @see @ref CCTXP_HIST_BUF_SIZE_MAX_LENGTH_REPORT ++ */ ++#define KBASE_MEM_PROFILE_MAX_BUF_SIZE ((size_t) (64 + ((80 + (56 * 64)) * 15) + 56)) ++ ++#endif /*_KBASE_MEM_PROFILE_DEBUGFS_BUF_SIZE_H_*/ ++ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_mmu.c b/drivers/gpu/arm/midgard/mali_kbase_mmu.c +new file mode 100755 +index 000000000000..26144850a588 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_mmu.c +@@ -0,0 +1,2088 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/** ++ * @file mali_kbase_mmu.c ++ * Base kernel MMU management. ++ */ ++ ++/* #define DEBUG 1 */ ++#include ++#include ++#include ++#include ++#if defined(CONFIG_MALI_GATOR_SUPPORT) ++#include ++#endif ++#include ++#include ++#include ++ ++#define beenthere(kctx, f, a...) dev_dbg(kctx->kbdev->dev, "%s:" f, __func__, ##a) ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#define KBASE_MMU_PAGE_ENTRIES 512 ++ ++/** ++ * kbase_mmu_flush_invalidate() - Flush and invalidate the GPU caches. ++ * @kctx: The KBase context. ++ * @vpfn: The virtual page frame number to start the flush on. ++ * @nr: The number of pages to flush. ++ * @sync: Set if the operation should be synchronous or not. ++ * ++ * Issue a cache flush + invalidate to the GPU caches and invalidate the TLBs. ++ * ++ * If sync is not set then transactions still in flight when the flush is issued ++ * may use the old page tables and the data they write will not be written out ++ * to memory, this function returns after the flush has been issued but ++ * before all accesses which might effect the flushed region have completed. ++ * ++ * If sync is set then accesses in the flushed region will be drained ++ * before data is flush and invalidated through L1, L2 and into memory, ++ * after which point this function will return. ++ */ ++static void kbase_mmu_flush_invalidate(struct kbase_context *kctx, ++ u64 vpfn, size_t nr, bool sync); ++ ++/** ++ * kbase_mmu_sync_pgd - sync page directory to memory ++ * @kbdev: Device pointer. ++ * @handle: Address of DMA region. ++ * @size: Size of the region to sync. ++ * ++ * This should be called after each page directory update. ++ */ ++ ++static void kbase_mmu_sync_pgd(struct kbase_device *kbdev, ++ dma_addr_t handle, size_t size) ++{ ++ /* If page table is not coherent then ensure the gpu can read ++ * the pages from memory ++ */ ++ if (kbdev->system_coherency != COHERENCY_ACE) ++ dma_sync_single_for_device(kbdev->dev, handle, size, ++ DMA_TO_DEVICE); ++} ++ ++/* ++ * Definitions: ++ * - PGD: Page Directory. ++ * - PTE: Page Table Entry. A 64bit value pointing to the next ++ * level of translation ++ * - ATE: Address Transation Entry. A 64bit value pointing to ++ * a 4kB physical page. ++ */ ++ ++static void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, ++ struct kbase_as *as, const char *reason_str); ++ ++ ++static size_t make_multiple(size_t minimum, size_t multiple) ++{ ++ size_t remainder = minimum % multiple; ++ ++ if (remainder == 0) ++ return minimum; ++ ++ return minimum + multiple - remainder; ++} ++ ++void page_fault_worker(struct work_struct *data) ++{ ++ u64 fault_pfn; ++ u32 fault_status; ++ size_t new_pages; ++ size_t fault_rel_pfn; ++ struct kbase_as *faulting_as; ++ int as_no; ++ struct kbase_context *kctx; ++ struct kbase_device *kbdev; ++ struct kbase_va_region *region; ++ int err; ++ bool grown = false; ++ ++ faulting_as = container_of(data, struct kbase_as, work_pagefault); ++ fault_pfn = faulting_as->fault_addr >> PAGE_SHIFT; ++ as_no = faulting_as->number; ++ ++ kbdev = container_of(faulting_as, struct kbase_device, as[as_no]); ++ ++ /* Grab the context that was already refcounted in kbase_mmu_interrupt(). ++ * Therefore, it cannot be scheduled out of this AS until we explicitly release it ++ */ ++ kctx = kbasep_js_runpool_lookup_ctx_noretain(kbdev, as_no); ++ if (WARN_ON(!kctx)) { ++ atomic_dec(&kbdev->faults_pending); ++ return; ++ } ++ ++ KBASE_DEBUG_ASSERT(kctx->kbdev == kbdev); ++ ++ if (unlikely(faulting_as->protected_mode)) ++ { ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Protected mode fault"); ++ kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx, ++ KBASE_MMU_FAULT_TYPE_PAGE); ++ ++ goto fault_done; ++ } ++ ++ fault_status = faulting_as->fault_status; ++ switch (fault_status & AS_FAULTSTATUS_EXCEPTION_CODE_MASK) { ++ ++ case AS_FAULTSTATUS_EXCEPTION_CODE_TRANSLATION_FAULT: ++ /* need to check against the region to handle this one */ ++ break; ++ ++ case AS_FAULTSTATUS_EXCEPTION_CODE_PERMISSION_FAULT: ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Permission failure"); ++ goto fault_done; ++ ++ case AS_FAULTSTATUS_EXCEPTION_CODE_TRANSTAB_BUS_FAULT: ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Translation table bus fault"); ++ goto fault_done; ++ ++ case AS_FAULTSTATUS_EXCEPTION_CODE_ACCESS_FLAG: ++ /* nothing to do, but we don't expect this fault currently */ ++ dev_warn(kbdev->dev, "Access flag unexpectedly set"); ++ goto fault_done; ++ ++ case AS_FAULTSTATUS_EXCEPTION_CODE_ADDRESS_SIZE_FAULT: ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Address size fault"); ++ else ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Unknown fault code"); ++ goto fault_done; ++ ++ case AS_FAULTSTATUS_EXCEPTION_CODE_MEMORY_ATTRIBUTES_FAULT: ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Memory attributes fault"); ++ else ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Unknown fault code"); ++ goto fault_done; ++ ++ default: ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Unknown fault code"); ++ goto fault_done; ++ } ++ ++ /* so we have a translation fault, let's see if it is for growable ++ * memory */ ++ kbase_gpu_vm_lock(kctx); ++ ++ region = kbase_region_tracker_find_region_enclosing_address(kctx, ++ faulting_as->fault_addr); ++ if (!region || region->flags & KBASE_REG_FREE) { ++ kbase_gpu_vm_unlock(kctx); ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Memory is not mapped on the GPU"); ++ goto fault_done; ++ } ++ ++ if (region->gpu_alloc->type == KBASE_MEM_TYPE_IMPORTED_UMM) { ++ kbase_gpu_vm_unlock(kctx); ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "DMA-BUF is not mapped on the GPU"); ++ goto fault_done; ++ } ++ ++ if ((region->flags & GROWABLE_FLAGS_REQUIRED) ++ != GROWABLE_FLAGS_REQUIRED) { ++ kbase_gpu_vm_unlock(kctx); ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Memory is not growable"); ++ goto fault_done; ++ } ++ ++ if ((region->flags & KBASE_REG_DONT_NEED)) { ++ kbase_gpu_vm_unlock(kctx); ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Don't need memory can't be grown"); ++ goto fault_done; ++ } ++ ++ /* find the size we need to grow it by */ ++ /* we know the result fit in a size_t due to kbase_region_tracker_find_region_enclosing_address ++ * validating the fault_adress to be within a size_t from the start_pfn */ ++ fault_rel_pfn = fault_pfn - region->start_pfn; ++ ++ if (fault_rel_pfn < kbase_reg_current_backed_size(region)) { ++ dev_dbg(kbdev->dev, "Page fault @ 0x%llx in allocated region 0x%llx-0x%llx of growable TMEM: Ignoring", ++ faulting_as->fault_addr, region->start_pfn, ++ region->start_pfn + ++ kbase_reg_current_backed_size(region)); ++ ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ ++ kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx, ++ KBASE_MMU_FAULT_TYPE_PAGE); ++ /* [1] in case another page fault occurred while we were ++ * handling the (duplicate) page fault we need to ensure we ++ * don't loose the other page fault as result of us clearing ++ * the MMU IRQ. Therefore, after we clear the MMU IRQ we send ++ * an UNLOCK command that will retry any stalled memory ++ * transaction (which should cause the other page fault to be ++ * raised again). ++ */ ++ kbase_mmu_hw_do_operation(kbdev, faulting_as, NULL, 0, 0, ++ AS_COMMAND_UNLOCK, 1); ++ ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ ++ kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx, ++ KBASE_MMU_FAULT_TYPE_PAGE); ++ kbase_gpu_vm_unlock(kctx); ++ ++ goto fault_done; ++ } ++ ++ new_pages = make_multiple(fault_rel_pfn - ++ kbase_reg_current_backed_size(region) + 1, ++ region->extent); ++ ++ /* cap to max vsize */ ++ if (new_pages + kbase_reg_current_backed_size(region) > ++ region->nr_pages) ++ new_pages = region->nr_pages - ++ kbase_reg_current_backed_size(region); ++ ++ if (0 == new_pages) { ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ ++ /* Duplicate of a fault we've already handled, nothing to do */ ++ kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx, ++ KBASE_MMU_FAULT_TYPE_PAGE); ++ /* See comment [1] about UNLOCK usage */ ++ kbase_mmu_hw_do_operation(kbdev, faulting_as, NULL, 0, 0, ++ AS_COMMAND_UNLOCK, 1); ++ ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ ++ kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx, ++ KBASE_MMU_FAULT_TYPE_PAGE); ++ kbase_gpu_vm_unlock(kctx); ++ goto fault_done; ++ } ++ ++ if (kbase_alloc_phy_pages_helper(region->gpu_alloc, new_pages) == 0) { ++ if (region->gpu_alloc != region->cpu_alloc) { ++ if (kbase_alloc_phy_pages_helper( ++ region->cpu_alloc, new_pages) == 0) { ++ grown = true; ++ } else { ++ kbase_free_phy_pages_helper(region->gpu_alloc, ++ new_pages); ++ } ++ } else { ++ grown = true; ++ } ++ } ++ ++ ++ if (grown) { ++ u64 pfn_offset; ++ u32 op; ++ ++ /* alloc success */ ++ KBASE_DEBUG_ASSERT(kbase_reg_current_backed_size(region) <= region->nr_pages); ++ ++ /* set up the new pages */ ++ pfn_offset = kbase_reg_current_backed_size(region) - new_pages; ++ /* ++ * Note: ++ * Issuing an MMU operation will unlock the MMU and cause the ++ * translation to be replayed. If the page insertion fails then ++ * rather then trying to continue the context should be killed ++ * so the no_flush version of insert_pages is used which allows ++ * us to unlock the MMU as we see fit. ++ */ ++ err = kbase_mmu_insert_pages_no_flush(kctx, ++ region->start_pfn + pfn_offset, ++ &kbase_get_gpu_phy_pages(region)[pfn_offset], ++ new_pages, region->flags); ++ if (err) { ++ kbase_free_phy_pages_helper(region->gpu_alloc, new_pages); ++ if (region->gpu_alloc != region->cpu_alloc) ++ kbase_free_phy_pages_helper(region->cpu_alloc, ++ new_pages); ++ kbase_gpu_vm_unlock(kctx); ++ /* The locked VA region will be unlocked and the cache invalidated in here */ ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Page table update failure"); ++ goto fault_done; ++ } ++#if defined(CONFIG_MALI_GATOR_SUPPORT) ++ kbase_trace_mali_page_fault_insert_pages(as_no, new_pages); ++#endif ++ KBASE_TLSTREAM_AUX_PAGEFAULT(kctx->id, (u64)new_pages); ++ ++ /* AS transaction begin */ ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ ++ /* flush L2 and unlock the VA (resumes the MMU) */ ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_6367)) ++ op = AS_COMMAND_FLUSH; ++ else ++ op = AS_COMMAND_FLUSH_PT; ++ ++ /* clear MMU interrupt - this needs to be done after updating ++ * the page tables but before issuing a FLUSH command. The ++ * FLUSH cmd has a side effect that it restarts stalled memory ++ * transactions in other address spaces which may cause ++ * another fault to occur. If we didn't clear the interrupt at ++ * this stage a new IRQ might not be raised when the GPU finds ++ * a MMU IRQ is already pending. ++ */ ++ kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx, ++ KBASE_MMU_FAULT_TYPE_PAGE); ++ ++ kbase_mmu_hw_do_operation(kbdev, faulting_as, kctx, ++ faulting_as->fault_addr >> PAGE_SHIFT, ++ new_pages, ++ op, 1); ++ ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ /* AS transaction end */ ++ ++ /* reenable this in the mask */ ++ kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx, ++ KBASE_MMU_FAULT_TYPE_PAGE); ++ kbase_gpu_vm_unlock(kctx); ++ } else { ++ /* failed to extend, handle as a normal PF */ ++ kbase_gpu_vm_unlock(kctx); ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Page allocation failure"); ++ } ++ ++fault_done: ++ /* ++ * By this point, the fault was handled in some way, ++ * so release the ctx refcount ++ */ ++ kbasep_js_runpool_release_ctx(kbdev, kctx); ++ ++ atomic_dec(&kbdev->faults_pending); ++} ++ ++phys_addr_t kbase_mmu_alloc_pgd(struct kbase_context *kctx) ++{ ++ u64 *page; ++ int i; ++ struct page *p; ++ int new_page_count __maybe_unused; ++ ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ new_page_count = kbase_atomic_add_pages(1, &kctx->used_pages); ++ kbase_atomic_add_pages(1, &kctx->kbdev->memdev.used_pages); ++ ++ p = kbase_mem_pool_alloc(&kctx->mem_pool); ++ if (!p) ++ goto sub_pages; ++ ++ KBASE_TLSTREAM_AUX_PAGESALLOC( ++ (u32)kctx->id, ++ (u64)new_page_count); ++ ++ page = kmap(p); ++ if (NULL == page) ++ goto alloc_free; ++ ++ kbase_process_page_usage_inc(kctx, 1); ++ ++ for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++) ++ kctx->kbdev->mmu_mode->entry_invalidate(&page[i]); ++ ++ kbase_mmu_sync_pgd(kctx->kbdev, kbase_dma_addr(p), PAGE_SIZE); ++ ++ kunmap(p); ++ return page_to_phys(p); ++ ++alloc_free: ++ kbase_mem_pool_free(&kctx->mem_pool, p, false); ++sub_pages: ++ kbase_atomic_sub_pages(1, &kctx->used_pages); ++ kbase_atomic_sub_pages(1, &kctx->kbdev->memdev.used_pages); ++ ++ return 0; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_mmu_alloc_pgd); ++ ++/* Given PGD PFN for level N, return PGD PFN for level N+1, allocating the ++ * new table from the pool if needed and possible ++ */ ++static int mmu_get_next_pgd(struct kbase_context *kctx, ++ phys_addr_t *pgd, u64 vpfn, int level) ++{ ++ u64 *page; ++ phys_addr_t target_pgd; ++ struct page *p; ++ ++ KBASE_DEBUG_ASSERT(*pgd); ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ ++ lockdep_assert_held(&kctx->mmu_lock); ++ ++ /* ++ * Architecture spec defines level-0 as being the top-most. ++ * This is a bit unfortunate here, but we keep the same convention. ++ */ ++ vpfn >>= (3 - level) * 9; ++ vpfn &= 0x1FF; ++ ++ p = pfn_to_page(PFN_DOWN(*pgd)); ++ page = kmap(p); ++ if (NULL == page) { ++ dev_warn(kctx->kbdev->dev, "mmu_get_next_pgd: kmap failure\n"); ++ return -EINVAL; ++ } ++ ++ target_pgd = kctx->kbdev->mmu_mode->pte_to_phy_addr(page[vpfn]); ++ ++ if (!target_pgd) { ++ target_pgd = kbase_mmu_alloc_pgd(kctx); ++ if (!target_pgd) { ++ dev_dbg(kctx->kbdev->dev, "mmu_get_next_pgd: kbase_mmu_alloc_pgd failure\n"); ++ kunmap(p); ++ return -ENOMEM; ++ } ++ ++ kctx->kbdev->mmu_mode->entry_set_pte(&page[vpfn], target_pgd); ++ ++ kbase_mmu_sync_pgd(kctx->kbdev, kbase_dma_addr(p), PAGE_SIZE); ++ /* Rely on the caller to update the address space flags. */ ++ } ++ ++ kunmap(p); ++ *pgd = target_pgd; ++ ++ return 0; ++} ++ ++static int mmu_get_bottom_pgd(struct kbase_context *kctx, ++ u64 vpfn, phys_addr_t *out_pgd) ++{ ++ phys_addr_t pgd; ++ int l; ++ ++ lockdep_assert_held(&kctx->mmu_lock); ++ ++ pgd = kctx->pgd; ++ for (l = MIDGARD_MMU_TOPLEVEL; l < MIDGARD_MMU_BOTTOMLEVEL; l++) { ++ int err = mmu_get_next_pgd(kctx, &pgd, vpfn, l); ++ /* Handle failure condition */ ++ if (err) { ++ dev_dbg(kctx->kbdev->dev, "mmu_get_bottom_pgd: mmu_get_next_pgd failure\n"); ++ return err; ++ } ++ } ++ ++ *out_pgd = pgd; ++ ++ return 0; ++} ++ ++static phys_addr_t mmu_insert_pages_recover_get_next_pgd(struct kbase_context *kctx, phys_addr_t pgd, u64 vpfn, int level) ++{ ++ u64 *page; ++ phys_addr_t target_pgd; ++ ++ KBASE_DEBUG_ASSERT(pgd); ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ ++ lockdep_assert_held(&kctx->mmu_lock); ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ /* ++ * Architecture spec defines level-0 as being the top-most. ++ * This is a bit unfortunate here, but we keep the same convention. ++ */ ++ vpfn >>= (3 - level) * 9; ++ vpfn &= 0x1FF; ++ ++ page = kmap_atomic(pfn_to_page(PFN_DOWN(pgd))); ++ /* kmap_atomic should NEVER fail */ ++ KBASE_DEBUG_ASSERT(NULL != page); ++ ++ target_pgd = kctx->kbdev->mmu_mode->pte_to_phy_addr(page[vpfn]); ++ /* As we are recovering from what has already been set up, we should have a target_pgd */ ++ KBASE_DEBUG_ASSERT(0 != target_pgd); ++ kunmap_atomic(page); ++ return target_pgd; ++} ++ ++static phys_addr_t mmu_insert_pages_recover_get_bottom_pgd(struct kbase_context *kctx, u64 vpfn) ++{ ++ phys_addr_t pgd; ++ int l; ++ ++ lockdep_assert_held(&kctx->mmu_lock); ++ ++ pgd = kctx->pgd; ++ ++ for (l = MIDGARD_MMU_TOPLEVEL; l < MIDGARD_MMU_BOTTOMLEVEL; l++) { ++ pgd = mmu_insert_pages_recover_get_next_pgd(kctx, pgd, vpfn, l); ++ /* Should never fail */ ++ KBASE_DEBUG_ASSERT(0 != pgd); ++ } ++ ++ return pgd; ++} ++ ++static void mmu_insert_pages_failure_recovery(struct kbase_context *kctx, u64 vpfn, ++ size_t nr) ++{ ++ phys_addr_t pgd; ++ u64 *pgd_page; ++ struct kbase_mmu_mode const *mmu_mode; ++ ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ KBASE_DEBUG_ASSERT(0 != vpfn); ++ /* 64-bit address range is the max */ ++ KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE)); ++ ++ lockdep_assert_held(&kctx->mmu_lock); ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ mmu_mode = kctx->kbdev->mmu_mode; ++ ++ while (nr) { ++ unsigned int i; ++ unsigned int index = vpfn & 0x1FF; ++ unsigned int count = KBASE_MMU_PAGE_ENTRIES - index; ++ struct page *p; ++ ++ if (count > nr) ++ count = nr; ++ ++ pgd = mmu_insert_pages_recover_get_bottom_pgd(kctx, vpfn); ++ KBASE_DEBUG_ASSERT(0 != pgd); ++ ++ p = pfn_to_page(PFN_DOWN(pgd)); ++ ++ pgd_page = kmap_atomic(p); ++ KBASE_DEBUG_ASSERT(NULL != pgd_page); ++ ++ /* Invalidate the entries we added */ ++ for (i = 0; i < count; i++) ++ mmu_mode->entry_invalidate(&pgd_page[index + i]); ++ ++ vpfn += count; ++ nr -= count; ++ ++ kbase_mmu_sync_pgd(kctx->kbdev, kbase_dma_addr(p), PAGE_SIZE); ++ ++ kunmap_atomic(pgd_page); ++ } ++} ++ ++/* ++ * Map the single page 'phys' 'nr' of times, starting at GPU PFN 'vpfn' ++ */ ++int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 vpfn, ++ phys_addr_t phys, size_t nr, ++ unsigned long flags) ++{ ++ phys_addr_t pgd; ++ u64 *pgd_page; ++ /* In case the insert_single_page only partially completes we need to be ++ * able to recover */ ++ bool recover_required = false; ++ u64 recover_vpfn = vpfn; ++ size_t recover_count = 0; ++ size_t remain = nr; ++ int err; ++ ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ KBASE_DEBUG_ASSERT(0 != vpfn); ++ /* 64-bit address range is the max */ ++ KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE)); ++ ++ /* Early out if there is nothing to do */ ++ if (nr == 0) ++ return 0; ++ ++ mutex_lock(&kctx->mmu_lock); ++ ++ while (remain) { ++ unsigned int i; ++ unsigned int index = vpfn & 0x1FF; ++ unsigned int count = KBASE_MMU_PAGE_ENTRIES - index; ++ struct page *p; ++ ++ if (count > remain) ++ count = remain; ++ ++ /* ++ * Repeatedly calling mmu_get_bottom_pte() is clearly ++ * suboptimal. We don't have to re-parse the whole tree ++ * each time (just cache the l0-l2 sequence). ++ * On the other hand, it's only a gain when we map more than ++ * 256 pages at once (on average). Do we really care? ++ */ ++ do { ++ err = mmu_get_bottom_pgd(kctx, vpfn, &pgd); ++ if (err != -ENOMEM) ++ break; ++ /* Fill the memory pool with enough pages for ++ * the page walk to succeed ++ */ ++ mutex_unlock(&kctx->mmu_lock); ++ err = kbase_mem_pool_grow(&kctx->mem_pool, ++ MIDGARD_MMU_BOTTOMLEVEL); ++ mutex_lock(&kctx->mmu_lock); ++ } while (!err); ++ if (err) { ++ dev_warn(kctx->kbdev->dev, "kbase_mmu_insert_pages: mmu_get_bottom_pgd failure\n"); ++ if (recover_required) { ++ /* Invalidate the pages we have partially ++ * completed */ ++ mmu_insert_pages_failure_recovery(kctx, ++ recover_vpfn, ++ recover_count); ++ } ++ goto fail_unlock; ++ } ++ ++ p = pfn_to_page(PFN_DOWN(pgd)); ++ pgd_page = kmap(p); ++ if (!pgd_page) { ++ dev_warn(kctx->kbdev->dev, "kbase_mmu_insert_pages: kmap failure\n"); ++ if (recover_required) { ++ /* Invalidate the pages we have partially ++ * completed */ ++ mmu_insert_pages_failure_recovery(kctx, ++ recover_vpfn, ++ recover_count); ++ } ++ err = -ENOMEM; ++ goto fail_unlock; ++ } ++ ++ for (i = 0; i < count; i++) { ++ unsigned int ofs = index + i; ++ ++ KBASE_DEBUG_ASSERT(0 == (pgd_page[ofs] & 1UL)); ++ kctx->kbdev->mmu_mode->entry_set_ate(&pgd_page[ofs], ++ phys, flags); ++ } ++ ++ vpfn += count; ++ remain -= count; ++ ++ kbase_mmu_sync_pgd(kctx->kbdev, ++ kbase_dma_addr(p) + (index * sizeof(u64)), ++ count * sizeof(u64)); ++ ++ kunmap(p); ++ /* We have started modifying the page table. ++ * If further pages need inserting and fail we need to undo what ++ * has already taken place */ ++ recover_required = true; ++ recover_count += count; ++ } ++ mutex_unlock(&kctx->mmu_lock); ++ kbase_mmu_flush_invalidate(kctx, vpfn, nr, false); ++ return 0; ++ ++fail_unlock: ++ mutex_unlock(&kctx->mmu_lock); ++ kbase_mmu_flush_invalidate(kctx, vpfn, nr, false); ++ return err; ++} ++ ++int kbase_mmu_insert_pages_no_flush(struct kbase_context *kctx, u64 vpfn, ++ phys_addr_t *phys, size_t nr, ++ unsigned long flags) ++{ ++ phys_addr_t pgd; ++ u64 *pgd_page; ++ /* In case the insert_pages only partially completes we need to be able ++ * to recover */ ++ bool recover_required = false; ++ u64 recover_vpfn = vpfn; ++ size_t recover_count = 0; ++ size_t remain = nr; ++ int err; ++ ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ KBASE_DEBUG_ASSERT(0 != vpfn); ++ /* 64-bit address range is the max */ ++ KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE)); ++ ++ /* Early out if there is nothing to do */ ++ if (nr == 0) ++ return 0; ++ ++ mutex_lock(&kctx->mmu_lock); ++ ++ while (remain) { ++ unsigned int i; ++ unsigned int index = vpfn & 0x1FF; ++ unsigned int count = KBASE_MMU_PAGE_ENTRIES - index; ++ struct page *p; ++ ++ if (count > remain) ++ count = remain; ++ ++ /* ++ * Repeatedly calling mmu_get_bottom_pte() is clearly ++ * suboptimal. We don't have to re-parse the whole tree ++ * each time (just cache the l0-l2 sequence). ++ * On the other hand, it's only a gain when we map more than ++ * 256 pages at once (on average). Do we really care? ++ */ ++ do { ++ err = mmu_get_bottom_pgd(kctx, vpfn, &pgd); ++ if (err != -ENOMEM) ++ break; ++ /* Fill the memory pool with enough pages for ++ * the page walk to succeed ++ */ ++ mutex_unlock(&kctx->mmu_lock); ++ err = kbase_mem_pool_grow(&kctx->mem_pool, ++ MIDGARD_MMU_BOTTOMLEVEL); ++ mutex_lock(&kctx->mmu_lock); ++ } while (!err); ++ if (err) { ++ dev_warn(kctx->kbdev->dev, "kbase_mmu_insert_pages: mmu_get_bottom_pgd failure\n"); ++ if (recover_required) { ++ /* Invalidate the pages we have partially ++ * completed */ ++ mmu_insert_pages_failure_recovery(kctx, ++ recover_vpfn, ++ recover_count); ++ } ++ goto fail_unlock; ++ } ++ ++ p = pfn_to_page(PFN_DOWN(pgd)); ++ pgd_page = kmap(p); ++ if (!pgd_page) { ++ dev_warn(kctx->kbdev->dev, "kbase_mmu_insert_pages: kmap failure\n"); ++ if (recover_required) { ++ /* Invalidate the pages we have partially ++ * completed */ ++ mmu_insert_pages_failure_recovery(kctx, ++ recover_vpfn, ++ recover_count); ++ } ++ err = -ENOMEM; ++ goto fail_unlock; ++ } ++ ++ for (i = 0; i < count; i++) { ++ unsigned int ofs = index + i; ++ ++ KBASE_DEBUG_ASSERT(0 == (pgd_page[ofs] & 1UL)); ++ kctx->kbdev->mmu_mode->entry_set_ate(&pgd_page[ofs], ++ phys[i], flags); ++ } ++ ++ phys += count; ++ vpfn += count; ++ remain -= count; ++ ++ kbase_mmu_sync_pgd(kctx->kbdev, ++ kbase_dma_addr(p) + (index * sizeof(u64)), ++ count * sizeof(u64)); ++ ++ kunmap(p); ++ /* We have started modifying the page table. If further pages ++ * need inserting and fail we need to undo what has already ++ * taken place */ ++ recover_required = true; ++ recover_count += count; ++ } ++ ++ mutex_unlock(&kctx->mmu_lock); ++ return 0; ++ ++fail_unlock: ++ mutex_unlock(&kctx->mmu_lock); ++ return err; ++} ++ ++/* ++ * Map 'nr' pages pointed to by 'phys' at GPU PFN 'vpfn' ++ */ ++int kbase_mmu_insert_pages(struct kbase_context *kctx, u64 vpfn, ++ phys_addr_t *phys, size_t nr, ++ unsigned long flags) ++{ ++ int err; ++ ++ err = kbase_mmu_insert_pages_no_flush(kctx, vpfn, phys, nr, flags); ++ kbase_mmu_flush_invalidate(kctx, vpfn, nr, false); ++ return err; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_mmu_insert_pages); ++ ++/** ++ * kbase_mmu_flush_invalidate_noretain() - Flush and invalidate the GPU caches ++ * without retaining the kbase context. ++ * @kctx: The KBase context. ++ * @vpfn: The virtual page frame number to start the flush on. ++ * @nr: The number of pages to flush. ++ * @sync: Set if the operation should be synchronous or not. ++ * ++ * As per kbase_mmu_flush_invalidate but doesn't retain the kctx or do any ++ * other locking. ++ */ ++static void kbase_mmu_flush_invalidate_noretain(struct kbase_context *kctx, ++ u64 vpfn, size_t nr, bool sync) ++{ ++ struct kbase_device *kbdev = kctx->kbdev; ++ int err; ++ u32 op; ++ ++ /* Early out if there is nothing to do */ ++ if (nr == 0) ++ return; ++ ++ if (sync) ++ op = AS_COMMAND_FLUSH_MEM; ++ else ++ op = AS_COMMAND_FLUSH_PT; ++ ++ err = kbase_mmu_hw_do_operation(kbdev, ++ &kbdev->as[kctx->as_nr], ++ kctx, vpfn, nr, op, 0); ++#if KBASE_GPU_RESET_EN ++ if (err) { ++ /* Flush failed to complete, assume the ++ * GPU has hung and perform a reset to ++ * recover */ ++ dev_err(kbdev->dev, "Flush for GPU page table update did not complete. Issuing GPU soft-reset to recover\n"); ++ ++ if (kbase_prepare_to_reset_gpu_locked(kbdev)) ++ kbase_reset_gpu_locked(kbdev); ++ } ++#endif /* KBASE_GPU_RESET_EN */ ++ ++#ifndef CONFIG_MALI_NO_MALI ++ /* ++ * As this function could be called in interrupt context the sync ++ * request can't block. Instead log the request and the next flush ++ * request will pick it up. ++ */ ++ if ((!err) && sync && ++ kbase_hw_has_issue(kctx->kbdev, BASE_HW_ISSUE_6367)) ++ atomic_set(&kctx->drain_pending, 1); ++#endif /* !CONFIG_MALI_NO_MALI */ ++} ++ ++static void kbase_mmu_flush_invalidate(struct kbase_context *kctx, ++ u64 vpfn, size_t nr, bool sync) ++{ ++ struct kbase_device *kbdev; ++ bool ctx_is_in_runpool; ++#ifndef CONFIG_MALI_NO_MALI ++ bool drain_pending = false; ++ ++ if (atomic_xchg(&kctx->drain_pending, 0)) ++ drain_pending = true; ++#endif /* !CONFIG_MALI_NO_MALI */ ++ ++ /* Early out if there is nothing to do */ ++ if (nr == 0) ++ return; ++ ++ kbdev = kctx->kbdev; ++ mutex_lock(&kbdev->js_data.queue_mutex); ++ ctx_is_in_runpool = kbasep_js_runpool_retain_ctx(kbdev, kctx); ++ mutex_unlock(&kbdev->js_data.queue_mutex); ++ ++ if (ctx_is_in_runpool) { ++ KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); ++ ++ if (!kbase_pm_context_active_handle_suspend(kbdev, ++ KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) { ++ int err; ++ u32 op; ++ ++ /* AS transaction begin */ ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ ++ if (sync) ++ op = AS_COMMAND_FLUSH_MEM; ++ else ++ op = AS_COMMAND_FLUSH_PT; ++ ++ err = kbase_mmu_hw_do_operation(kbdev, ++ &kbdev->as[kctx->as_nr], ++ kctx, vpfn, nr, op, 0); ++ ++#if KBASE_GPU_RESET_EN ++ if (err) { ++ /* Flush failed to complete, assume the ++ * GPU has hung and perform a reset to ++ * recover */ ++ dev_err(kbdev->dev, "Flush for GPU page table update did not complete. Issueing GPU soft-reset to recover\n"); ++ ++ if (kbase_prepare_to_reset_gpu(kbdev)) ++ kbase_reset_gpu(kbdev); ++ } ++#endif /* KBASE_GPU_RESET_EN */ ++ ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ /* AS transaction end */ ++ ++#ifndef CONFIG_MALI_NO_MALI ++ /* ++ * The transaction lock must be dropped before here ++ * as kbase_wait_write_flush could take it if ++ * the GPU was powered down (static analysis doesn't ++ * know this can't happen). ++ */ ++ drain_pending |= (!err) && sync && ++ kbase_hw_has_issue(kctx->kbdev, ++ BASE_HW_ISSUE_6367); ++ if (drain_pending) { ++ /* Wait for GPU to flush write buffer */ ++ kbase_wait_write_flush(kctx); ++ } ++#endif /* !CONFIG_MALI_NO_MALI */ ++ ++ kbase_pm_context_idle(kbdev); ++ } ++ kbasep_js_runpool_release_ctx(kbdev, kctx); ++ } ++} ++ ++void kbase_mmu_update(struct kbase_context *kctx) ++{ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ lockdep_assert_held(&kctx->kbdev->mmu_hw_mutex); ++ /* ASSERT that the context has a valid as_nr, which is only the case ++ * when it's scheduled in. ++ * ++ * as_nr won't change because the caller has the hwaccess_lock */ ++ KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); ++ ++ kctx->kbdev->mmu_mode->update(kctx); ++} ++KBASE_EXPORT_TEST_API(kbase_mmu_update); ++ ++void kbase_mmu_disable_as(struct kbase_device *kbdev, int as_nr) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ lockdep_assert_held(&kbdev->mmu_hw_mutex); ++ ++ kbdev->mmu_mode->disable_as(kbdev, as_nr); ++} ++ ++void kbase_mmu_disable(struct kbase_context *kctx) ++{ ++ /* ASSERT that the context has a valid as_nr, which is only the case ++ * when it's scheduled in. ++ * ++ * as_nr won't change because the caller has the hwaccess_lock */ ++ KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); ++ ++ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); ++ ++ /* ++ * The address space is being disabled, drain all knowledge of it out ++ * from the caches as pages and page tables might be freed after this. ++ * ++ * The job scheduler code will already be holding the locks and context ++ * so just do the flush. ++ */ ++ kbase_mmu_flush_invalidate_noretain(kctx, 0, ~0, true); ++ ++ kctx->kbdev->mmu_mode->disable_as(kctx->kbdev, kctx->as_nr); ++} ++KBASE_EXPORT_TEST_API(kbase_mmu_disable); ++ ++/* ++ * We actually only discard the ATE, and not the page table ++ * pages. There is a potential DoS here, as we'll leak memory by ++ * having PTEs that are potentially unused. Will require physical ++ * page accounting, so MMU pages are part of the process allocation. ++ * ++ * IMPORTANT: This uses kbasep_js_runpool_release_ctx() when the context is ++ * currently scheduled into the runpool, and so potentially uses a lot of locks. ++ * These locks must be taken in the correct order with respect to others ++ * already held by the caller. Refer to kbasep_js_runpool_release_ctx() for more ++ * information. ++ */ ++int kbase_mmu_teardown_pages(struct kbase_context *kctx, u64 vpfn, size_t nr) ++{ ++ phys_addr_t pgd; ++ u64 *pgd_page; ++ struct kbase_device *kbdev; ++ size_t requested_nr = nr; ++ struct kbase_mmu_mode const *mmu_mode; ++ int err; ++ ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ beenthere(kctx, "kctx %p vpfn %lx nr %zd", (void *)kctx, (unsigned long)vpfn, nr); ++ ++ if (0 == nr) { ++ /* early out if nothing to do */ ++ return 0; ++ } ++ ++ mutex_lock(&kctx->mmu_lock); ++ ++ kbdev = kctx->kbdev; ++ mmu_mode = kbdev->mmu_mode; ++ ++ while (nr) { ++ unsigned int i; ++ unsigned int index = vpfn & 0x1FF; ++ unsigned int count = KBASE_MMU_PAGE_ENTRIES - index; ++ struct page *p; ++ ++ if (count > nr) ++ count = nr; ++ ++ err = mmu_get_bottom_pgd(kctx, vpfn, &pgd); ++ if (err) { ++ dev_warn(kbdev->dev, "kbase_mmu_teardown_pages: mmu_get_bottom_pgd failure\n"); ++ err = -EINVAL; ++ goto fail_unlock; ++ } ++ ++ p = pfn_to_page(PFN_DOWN(pgd)); ++ pgd_page = kmap(p); ++ if (!pgd_page) { ++ dev_warn(kbdev->dev, "kbase_mmu_teardown_pages: kmap failure\n"); ++ err = -ENOMEM; ++ goto fail_unlock; ++ } ++ ++ for (i = 0; i < count; i++) ++ mmu_mode->entry_invalidate(&pgd_page[index + i]); ++ ++ vpfn += count; ++ nr -= count; ++ ++ kbase_mmu_sync_pgd(kctx->kbdev, ++ kbase_dma_addr(p) + (index * sizeof(u64)), ++ count * sizeof(u64)); ++ ++ kunmap(p); ++ } ++ ++ mutex_unlock(&kctx->mmu_lock); ++ kbase_mmu_flush_invalidate(kctx, vpfn, requested_nr, true); ++ return 0; ++ ++fail_unlock: ++ mutex_unlock(&kctx->mmu_lock); ++ kbase_mmu_flush_invalidate(kctx, vpfn, requested_nr, true); ++ return err; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_mmu_teardown_pages); ++ ++/** ++ * Update the entries for specified number of pages pointed to by 'phys' at GPU PFN 'vpfn'. ++ * This call is being triggered as a response to the changes of the mem attributes ++ * ++ * @pre : The caller is responsible for validating the memory attributes ++ * ++ * IMPORTANT: This uses kbasep_js_runpool_release_ctx() when the context is ++ * currently scheduled into the runpool, and so potentially uses a lot of locks. ++ * These locks must be taken in the correct order with respect to others ++ * already held by the caller. Refer to kbasep_js_runpool_release_ctx() for more ++ * information. ++ */ ++int kbase_mmu_update_pages(struct kbase_context *kctx, u64 vpfn, phys_addr_t *phys, size_t nr, unsigned long flags) ++{ ++ phys_addr_t pgd; ++ u64 *pgd_page; ++ size_t requested_nr = nr; ++ struct kbase_mmu_mode const *mmu_mode; ++ int err; ++ ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ KBASE_DEBUG_ASSERT(0 != vpfn); ++ KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE)); ++ ++ /* Early out if there is nothing to do */ ++ if (nr == 0) ++ return 0; ++ ++ mutex_lock(&kctx->mmu_lock); ++ ++ mmu_mode = kctx->kbdev->mmu_mode; ++ ++ dev_warn(kctx->kbdev->dev, "kbase_mmu_update_pages(): updating page share flags on GPU PFN 0x%llx from phys %p, %zu pages", ++ vpfn, phys, nr); ++ ++ while (nr) { ++ unsigned int i; ++ unsigned int index = vpfn & 0x1FF; ++ size_t count = KBASE_MMU_PAGE_ENTRIES - index; ++ struct page *p; ++ ++ if (count > nr) ++ count = nr; ++ ++ do { ++ err = mmu_get_bottom_pgd(kctx, vpfn, &pgd); ++ if (err != -ENOMEM) ++ break; ++ /* Fill the memory pool with enough pages for ++ * the page walk to succeed ++ */ ++ mutex_unlock(&kctx->mmu_lock); ++ err = kbase_mem_pool_grow(&kctx->mem_pool, ++ MIDGARD_MMU_BOTTOMLEVEL); ++ mutex_lock(&kctx->mmu_lock); ++ } while (!err); ++ if (err) { ++ dev_warn(kctx->kbdev->dev, "mmu_get_bottom_pgd failure\n"); ++ goto fail_unlock; ++ } ++ ++ p = pfn_to_page(PFN_DOWN(pgd)); ++ pgd_page = kmap(p); ++ if (!pgd_page) { ++ dev_warn(kctx->kbdev->dev, "kmap failure\n"); ++ err = -ENOMEM; ++ goto fail_unlock; ++ } ++ ++ for (i = 0; i < count; i++) ++ mmu_mode->entry_set_ate(&pgd_page[index + i], phys[i], ++ flags); ++ ++ phys += count; ++ vpfn += count; ++ nr -= count; ++ ++ kbase_mmu_sync_pgd(kctx->kbdev, ++ kbase_dma_addr(p) + (index * sizeof(u64)), ++ count * sizeof(u64)); ++ ++ kunmap(pfn_to_page(PFN_DOWN(pgd))); ++ } ++ ++ mutex_unlock(&kctx->mmu_lock); ++ kbase_mmu_flush_invalidate(kctx, vpfn, requested_nr, true); ++ return 0; ++ ++fail_unlock: ++ mutex_unlock(&kctx->mmu_lock); ++ kbase_mmu_flush_invalidate(kctx, vpfn, requested_nr, true); ++ return err; ++} ++ ++/* This is a debug feature only */ ++static void mmu_check_unused(struct kbase_context *kctx, phys_addr_t pgd) ++{ ++ u64 *page; ++ int i; ++ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ page = kmap_atomic(pfn_to_page(PFN_DOWN(pgd))); ++ /* kmap_atomic should NEVER fail. */ ++ KBASE_DEBUG_ASSERT(NULL != page); ++ ++ for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++) { ++ if (kctx->kbdev->mmu_mode->ate_is_valid(page[i])) ++ beenthere(kctx, "live pte %016lx", (unsigned long)page[i]); ++ } ++ kunmap_atomic(page); ++} ++ ++static void mmu_teardown_level(struct kbase_context *kctx, phys_addr_t pgd, int level, int zap, u64 *pgd_page_buffer) ++{ ++ phys_addr_t target_pgd; ++ u64 *pgd_page; ++ int i; ++ struct kbase_mmu_mode const *mmu_mode; ++ ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ lockdep_assert_held(&kctx->mmu_lock); ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ pgd_page = kmap_atomic(pfn_to_page(PFN_DOWN(pgd))); ++ /* kmap_atomic should NEVER fail. */ ++ KBASE_DEBUG_ASSERT(NULL != pgd_page); ++ /* Copy the page to our preallocated buffer so that we can minimize kmap_atomic usage */ ++ memcpy(pgd_page_buffer, pgd_page, PAGE_SIZE); ++ kunmap_atomic(pgd_page); ++ pgd_page = pgd_page_buffer; ++ ++ mmu_mode = kctx->kbdev->mmu_mode; ++ ++ for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++) { ++ target_pgd = mmu_mode->pte_to_phy_addr(pgd_page[i]); ++ ++ if (target_pgd) { ++ if (level < (MIDGARD_MMU_BOTTOMLEVEL - 1)) { ++ mmu_teardown_level(kctx, target_pgd, level + 1, zap, pgd_page_buffer + (PAGE_SIZE / sizeof(u64))); ++ } else { ++ /* ++ * So target_pte is a level-3 page. ++ * As a leaf, it is safe to free it. ++ * Unless we have live pages attached to it! ++ */ ++ mmu_check_unused(kctx, target_pgd); ++ } ++ ++ beenthere(kctx, "pte %lx level %d", (unsigned long)target_pgd, level + 1); ++ if (zap) { ++ struct page *p = phys_to_page(target_pgd); ++ ++ kbase_mem_pool_free(&kctx->mem_pool, p, true); ++ kbase_process_page_usage_dec(kctx, 1); ++ kbase_atomic_sub_pages(1, &kctx->used_pages); ++ kbase_atomic_sub_pages(1, &kctx->kbdev->memdev.used_pages); ++ } ++ } ++ } ++} ++ ++int kbase_mmu_init(struct kbase_context *kctx) ++{ ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ KBASE_DEBUG_ASSERT(NULL == kctx->mmu_teardown_pages); ++ ++ mutex_init(&kctx->mmu_lock); ++ ++ /* Preallocate MMU depth of four pages for mmu_teardown_level to use */ ++ kctx->mmu_teardown_pages = kmalloc(PAGE_SIZE * 4, GFP_KERNEL); ++ ++ if (NULL == kctx->mmu_teardown_pages) ++ return -ENOMEM; ++ ++ return 0; ++} ++ ++void kbase_mmu_term(struct kbase_context *kctx) ++{ ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ KBASE_DEBUG_ASSERT(NULL != kctx->mmu_teardown_pages); ++ ++ kfree(kctx->mmu_teardown_pages); ++ kctx->mmu_teardown_pages = NULL; ++} ++ ++void kbase_mmu_free_pgd(struct kbase_context *kctx) ++{ ++ int new_page_count __maybe_unused; ++ ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ KBASE_DEBUG_ASSERT(NULL != kctx->mmu_teardown_pages); ++ ++ mutex_lock(&kctx->mmu_lock); ++ mmu_teardown_level(kctx, kctx->pgd, MIDGARD_MMU_TOPLEVEL, 1, kctx->mmu_teardown_pages); ++ mutex_unlock(&kctx->mmu_lock); ++ ++ beenthere(kctx, "pgd %lx", (unsigned long)kctx->pgd); ++ kbase_mem_pool_free(&kctx->mem_pool, phys_to_page(kctx->pgd), true); ++ kbase_process_page_usage_dec(kctx, 1); ++ new_page_count = kbase_atomic_sub_pages(1, &kctx->used_pages); ++ kbase_atomic_sub_pages(1, &kctx->kbdev->memdev.used_pages); ++ ++ KBASE_TLSTREAM_AUX_PAGESALLOC( ++ (u32)kctx->id, ++ (u64)new_page_count); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_mmu_free_pgd); ++ ++static size_t kbasep_mmu_dump_level(struct kbase_context *kctx, phys_addr_t pgd, int level, char ** const buffer, size_t *size_left) ++{ ++ phys_addr_t target_pgd; ++ u64 *pgd_page; ++ int i; ++ size_t size = KBASE_MMU_PAGE_ENTRIES * sizeof(u64) + sizeof(u64); ++ size_t dump_size; ++ struct kbase_mmu_mode const *mmu_mode; ++ ++ KBASE_DEBUG_ASSERT(NULL != kctx); ++ lockdep_assert_held(&kctx->mmu_lock); ++ ++ mmu_mode = kctx->kbdev->mmu_mode; ++ ++ pgd_page = kmap(pfn_to_page(PFN_DOWN(pgd))); ++ if (!pgd_page) { ++ dev_warn(kctx->kbdev->dev, "kbasep_mmu_dump_level: kmap failure\n"); ++ return 0; ++ } ++ ++ if (*size_left >= size) { ++ /* A modified physical address that contains the page table level */ ++ u64 m_pgd = pgd | level; ++ ++ /* Put the modified physical address in the output buffer */ ++ memcpy(*buffer, &m_pgd, sizeof(m_pgd)); ++ *buffer += sizeof(m_pgd); ++ ++ /* Followed by the page table itself */ ++ memcpy(*buffer, pgd_page, sizeof(u64) * KBASE_MMU_PAGE_ENTRIES); ++ *buffer += sizeof(u64) * KBASE_MMU_PAGE_ENTRIES; ++ ++ *size_left -= size; ++ } ++ ++ if (level < MIDGARD_MMU_BOTTOMLEVEL) { ++ for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++) { ++ if (mmu_mode->pte_is_valid(pgd_page[i])) { ++ target_pgd = mmu_mode->pte_to_phy_addr( ++ pgd_page[i]); ++ ++ dump_size = kbasep_mmu_dump_level(kctx, ++ target_pgd, level + 1, ++ buffer, size_left); ++ if (!dump_size) { ++ kunmap(pfn_to_page(PFN_DOWN(pgd))); ++ return 0; ++ } ++ size += dump_size; ++ } ++ } ++ } ++ ++ kunmap(pfn_to_page(PFN_DOWN(pgd))); ++ ++ return size; ++} ++ ++void *kbase_mmu_dump(struct kbase_context *kctx, int nr_pages) ++{ ++ void *kaddr; ++ size_t size_left; ++ ++ KBASE_DEBUG_ASSERT(kctx); ++ ++ if (0 == nr_pages) { ++ /* can't dump in a 0 sized buffer, early out */ ++ return NULL; ++ } ++ ++ size_left = nr_pages * PAGE_SIZE; ++ ++ KBASE_DEBUG_ASSERT(0 != size_left); ++ kaddr = vmalloc_user(size_left); ++ ++ mutex_lock(&kctx->mmu_lock); ++ ++ if (kaddr) { ++ u64 end_marker = 0xFFULL; ++ char *buffer; ++ char *mmu_dump_buffer; ++ u64 config[3]; ++ size_t size; ++ ++ buffer = (char *)kaddr; ++ mmu_dump_buffer = buffer; ++ ++ if (kctx->api_version >= KBASE_API_VERSION(8, 4)) { ++ struct kbase_mmu_setup as_setup; ++ ++ kctx->kbdev->mmu_mode->get_as_setup(kctx, &as_setup); ++ config[0] = as_setup.transtab; ++ config[1] = as_setup.memattr; ++ config[2] = as_setup.transcfg; ++ memcpy(buffer, &config, sizeof(config)); ++ mmu_dump_buffer += sizeof(config); ++ size_left -= sizeof(config); ++ } ++ ++ ++ ++ size = kbasep_mmu_dump_level(kctx, ++ kctx->pgd, ++ MIDGARD_MMU_TOPLEVEL, ++ &mmu_dump_buffer, ++ &size_left); ++ ++ if (!size) ++ goto fail_free; ++ ++ /* Add on the size for the end marker */ ++ size += sizeof(u64); ++ /* Add on the size for the config */ ++ if (kctx->api_version >= KBASE_API_VERSION(8, 4)) ++ size += sizeof(config); ++ ++ ++ if (size > nr_pages * PAGE_SIZE || size_left < sizeof(u64)) { ++ /* The buffer isn't big enough - free the memory and return failure */ ++ goto fail_free; ++ } ++ ++ /* Add the end marker */ ++ memcpy(mmu_dump_buffer, &end_marker, sizeof(u64)); ++ } ++ ++ mutex_unlock(&kctx->mmu_lock); ++ return kaddr; ++ ++fail_free: ++ vfree(kaddr); ++ mutex_unlock(&kctx->mmu_lock); ++ return NULL; ++} ++KBASE_EXPORT_TEST_API(kbase_mmu_dump); ++ ++void bus_fault_worker(struct work_struct *data) ++{ ++ struct kbase_as *faulting_as; ++ int as_no; ++ struct kbase_context *kctx; ++ struct kbase_device *kbdev; ++#if KBASE_GPU_RESET_EN ++ bool reset_status = false; ++#endif /* KBASE_GPU_RESET_EN */ ++ ++ faulting_as = container_of(data, struct kbase_as, work_busfault); ++ ++ as_no = faulting_as->number; ++ ++ kbdev = container_of(faulting_as, struct kbase_device, as[as_no]); ++ ++ /* Grab the context that was already refcounted in kbase_mmu_interrupt(). ++ * Therefore, it cannot be scheduled out of this AS until we explicitly release it ++ */ ++ kctx = kbasep_js_runpool_lookup_ctx_noretain(kbdev, as_no); ++ if (WARN_ON(!kctx)) { ++ atomic_dec(&kbdev->faults_pending); ++ return; ++ } ++ ++ if (unlikely(faulting_as->protected_mode)) ++ { ++ kbase_mmu_report_fault_and_kill(kctx, faulting_as, ++ "Permission failure"); ++ kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx, ++ KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); ++ kbasep_js_runpool_release_ctx(kbdev, kctx); ++ atomic_dec(&kbdev->faults_pending); ++ return; ++ ++ } ++ ++#if KBASE_GPU_RESET_EN ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245)) { ++ /* Due to H/W issue 8245 we need to reset the GPU after using UNMAPPED mode. ++ * We start the reset before switching to UNMAPPED to ensure that unrelated jobs ++ * are evicted from the GPU before the switch. ++ */ ++ dev_err(kbdev->dev, "GPU bus error occurred. For this GPU version we now soft-reset as part of bus error recovery\n"); ++ reset_status = kbase_prepare_to_reset_gpu(kbdev); ++ } ++#endif /* KBASE_GPU_RESET_EN */ ++ /* NOTE: If GPU already powered off for suspend, we don't need to switch to unmapped */ ++ if (!kbase_pm_context_active_handle_suspend(kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) { ++ unsigned long flags; ++ ++ /* switch to UNMAPPED mode, will abort all jobs and stop any hw counter dumping */ ++ /* AS transaction begin */ ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ ++ /* Set the MMU into unmapped mode */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_mmu_disable(kctx); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ /* AS transaction end */ ++ ++ kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx, ++ KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); ++ kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx, ++ KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); ++ ++ kbase_pm_context_idle(kbdev); ++ } ++ ++#if KBASE_GPU_RESET_EN ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245) && reset_status) ++ kbase_reset_gpu(kbdev); ++#endif /* KBASE_GPU_RESET_EN */ ++ ++ kbasep_js_runpool_release_ctx(kbdev, kctx); ++ ++ atomic_dec(&kbdev->faults_pending); ++} ++ ++const char *kbase_exception_name(struct kbase_device *kbdev, u32 exception_code) ++{ ++ const char *e; ++ ++ switch (exception_code) { ++ /* Non-Fault Status code */ ++ case 0x00: ++ e = "NOT_STARTED/IDLE/OK"; ++ break; ++ case 0x01: ++ e = "DONE"; ++ break; ++ case 0x02: ++ e = "INTERRUPTED"; ++ break; ++ case 0x03: ++ e = "STOPPED"; ++ break; ++ case 0x04: ++ e = "TERMINATED"; ++ break; ++ case 0x08: ++ e = "ACTIVE"; ++ break; ++ /* Job exceptions */ ++ case 0x40: ++ e = "JOB_CONFIG_FAULT"; ++ break; ++ case 0x41: ++ e = "JOB_POWER_FAULT"; ++ break; ++ case 0x42: ++ e = "JOB_READ_FAULT"; ++ break; ++ case 0x43: ++ e = "JOB_WRITE_FAULT"; ++ break; ++ case 0x44: ++ e = "JOB_AFFINITY_FAULT"; ++ break; ++ case 0x48: ++ e = "JOB_BUS_FAULT"; ++ break; ++ case 0x50: ++ e = "INSTR_INVALID_PC"; ++ break; ++ case 0x51: ++ e = "INSTR_INVALID_ENC"; ++ break; ++ case 0x52: ++ e = "INSTR_TYPE_MISMATCH"; ++ break; ++ case 0x53: ++ e = "INSTR_OPERAND_FAULT"; ++ break; ++ case 0x54: ++ e = "INSTR_TLS_FAULT"; ++ break; ++ case 0x55: ++ e = "INSTR_BARRIER_FAULT"; ++ break; ++ case 0x56: ++ e = "INSTR_ALIGN_FAULT"; ++ break; ++ case 0x58: ++ e = "DATA_INVALID_FAULT"; ++ break; ++ case 0x59: ++ e = "TILE_RANGE_FAULT"; ++ break; ++ case 0x5A: ++ e = "ADDR_RANGE_FAULT"; ++ break; ++ case 0x60: ++ e = "OUT_OF_MEMORY"; ++ break; ++ /* GPU exceptions */ ++ case 0x80: ++ e = "DELAYED_BUS_FAULT"; ++ break; ++ case 0x88: ++ e = "SHAREABILITY_FAULT"; ++ break; ++ /* MMU exceptions */ ++ case 0xC0: ++ case 0xC1: ++ case 0xC2: ++ case 0xC3: ++ case 0xC4: ++ case 0xC5: ++ case 0xC6: ++ case 0xC7: ++ e = "TRANSLATION_FAULT"; ++ break; ++ case 0xC8: ++ e = "PERMISSION_FAULT"; ++ break; ++ case 0xC9: ++ case 0xCA: ++ case 0xCB: ++ case 0xCC: ++ case 0xCD: ++ case 0xCE: ++ case 0xCF: ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) ++ e = "PERMISSION_FAULT"; ++ else ++ e = "UNKNOWN"; ++ break; ++ case 0xD0: ++ case 0xD1: ++ case 0xD2: ++ case 0xD3: ++ case 0xD4: ++ case 0xD5: ++ case 0xD6: ++ case 0xD7: ++ e = "TRANSTAB_BUS_FAULT"; ++ break; ++ case 0xD8: ++ e = "ACCESS_FLAG"; ++ break; ++ case 0xD9: ++ case 0xDA: ++ case 0xDB: ++ case 0xDC: ++ case 0xDD: ++ case 0xDE: ++ case 0xDF: ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) ++ e = "ACCESS_FLAG"; ++ else ++ e = "UNKNOWN"; ++ break; ++ case 0xE0: ++ case 0xE1: ++ case 0xE2: ++ case 0xE3: ++ case 0xE4: ++ case 0xE5: ++ case 0xE6: ++ case 0xE7: ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) ++ e = "ADDRESS_SIZE_FAULT"; ++ else ++ e = "UNKNOWN"; ++ break; ++ case 0xE8: ++ case 0xE9: ++ case 0xEA: ++ case 0xEB: ++ case 0xEC: ++ case 0xED: ++ case 0xEE: ++ case 0xEF: ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) ++ e = "MEMORY_ATTRIBUTES_FAULT"; ++ else ++ e = "UNKNOWN"; ++ break; ++ default: ++ e = "UNKNOWN"; ++ break; ++ }; ++ ++ return e; ++} ++ ++static const char *access_type_name(struct kbase_device *kbdev, ++ u32 fault_status) ++{ ++ switch (fault_status & AS_FAULTSTATUS_ACCESS_TYPE_MASK) { ++ case AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC: ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) ++ return "ATOMIC"; ++ else ++ return "UNKNOWN"; ++ case AS_FAULTSTATUS_ACCESS_TYPE_READ: ++ return "READ"; ++ case AS_FAULTSTATUS_ACCESS_TYPE_WRITE: ++ return "WRITE"; ++ case AS_FAULTSTATUS_ACCESS_TYPE_EX: ++ return "EXECUTE"; ++ default: ++ WARN_ON(1); ++ return NULL; ++ } ++} ++ ++/** ++ * The caller must ensure it's retained the ctx to prevent it from being scheduled out whilst it's being worked on. ++ */ ++static void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, ++ struct kbase_as *as, const char *reason_str) ++{ ++ unsigned long flags; ++ int exception_type; ++ int access_type; ++ int source_id; ++ int as_no; ++ struct kbase_device *kbdev; ++ struct kbasep_js_device_data *js_devdata; ++ ++#if KBASE_GPU_RESET_EN ++ bool reset_status = false; ++#endif ++ ++ as_no = as->number; ++ kbdev = kctx->kbdev; ++ js_devdata = &kbdev->js_data; ++ ++ /* ASSERT that the context won't leave the runpool */ ++ KBASE_DEBUG_ASSERT(atomic_read(&kctx->refcount) > 0); ++ ++ /* decode the fault status */ ++ exception_type = as->fault_status & 0xFF; ++ access_type = (as->fault_status >> 8) & 0x3; ++ source_id = (as->fault_status >> 16); ++ ++ /* terminal fault, print info about the fault */ ++ dev_err(kbdev->dev, ++ "Unhandled Page fault in AS%d at VA 0x%016llX\n" ++ "Reason: %s\n" ++ "raw fault status: 0x%X\n" ++ "decoded fault status: %s\n" ++ "exception type 0x%X: %s\n" ++ "access type 0x%X: %s\n" ++ "source id 0x%X\n" ++ "pid: %d\n", ++ as_no, as->fault_addr, ++ reason_str, ++ as->fault_status, ++ (as->fault_status & (1 << 10) ? "DECODER FAULT" : "SLAVE FAULT"), ++ exception_type, kbase_exception_name(kbdev, exception_type), ++ access_type, access_type_name(kbdev, as->fault_status), ++ source_id, ++ kctx->pid); ++ ++ /* hardware counters dump fault handling */ ++ if ((kbdev->hwcnt.kctx) && (kbdev->hwcnt.kctx->as_nr == as_no) && ++ (kbdev->hwcnt.backend.state == ++ KBASE_INSTR_STATE_DUMPING)) { ++ unsigned int num_core_groups = kbdev->gpu_props.num_core_groups; ++ ++ if ((as->fault_addr >= kbdev->hwcnt.addr) && ++ (as->fault_addr < (kbdev->hwcnt.addr + ++ (num_core_groups * 2048)))) ++ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_FAULT; ++ } ++ ++ /* Stop the kctx from submitting more jobs and cause it to be scheduled ++ * out/rescheduled - this will occur on releasing the context's refcount */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbasep_js_clear_submit_allowed(js_devdata, kctx); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ /* Kill any running jobs from the context. Submit is disallowed, so no more jobs from this ++ * context can appear in the job slots from this point on */ ++ kbase_backend_jm_kill_jobs_from_kctx(kctx); ++ /* AS transaction begin */ ++ mutex_lock(&kbdev->mmu_hw_mutex); ++#if KBASE_GPU_RESET_EN ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245)) { ++ /* Due to H/W issue 8245 we need to reset the GPU after using UNMAPPED mode. ++ * We start the reset before switching to UNMAPPED to ensure that unrelated jobs ++ * are evicted from the GPU before the switch. ++ */ ++ dev_err(kbdev->dev, "Unhandled page fault. For this GPU version we now soft-reset the GPU as part of page fault recovery."); ++ reset_status = kbase_prepare_to_reset_gpu(kbdev); ++ } ++#endif /* KBASE_GPU_RESET_EN */ ++ /* switch to UNMAPPED mode, will abort all jobs and stop any hw counter dumping */ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_mmu_disable(kctx); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ /* AS transaction end */ ++ /* Clear down the fault */ ++ kbase_mmu_hw_clear_fault(kbdev, as, kctx, ++ KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); ++ kbase_mmu_hw_enable_fault(kbdev, as, kctx, ++ KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); ++ ++#if KBASE_GPU_RESET_EN ++ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245) && reset_status) ++ kbase_reset_gpu(kbdev); ++#endif /* KBASE_GPU_RESET_EN */ ++} ++ ++void kbasep_as_do_poke(struct work_struct *work) ++{ ++ struct kbase_as *as; ++ struct kbase_device *kbdev; ++ struct kbase_context *kctx; ++ unsigned long flags; ++ ++ KBASE_DEBUG_ASSERT(work); ++ as = container_of(work, struct kbase_as, poke_work); ++ kbdev = container_of(as, struct kbase_device, as[as->number]); ++ KBASE_DEBUG_ASSERT(as->poke_state & KBASE_AS_POKE_STATE_IN_FLIGHT); ++ ++ /* GPU power will already be active by virtue of the caller holding a JS ++ * reference on the address space, and will not release it until this worker ++ * has finished */ ++ ++ /* Further to the comment above, we know that while this function is running ++ * the AS will not be released as before the atom is released this workqueue ++ * is flushed (in kbase_as_poking_timer_release_atom) ++ */ ++ kctx = kbasep_js_runpool_lookup_ctx_noretain(kbdev, as->number); ++ ++ /* AS transaction begin */ ++ mutex_lock(&kbdev->mmu_hw_mutex); ++ /* Force a uTLB invalidate */ ++ kbase_mmu_hw_do_operation(kbdev, as, kctx, 0, 0, ++ AS_COMMAND_UNLOCK, 0); ++ mutex_unlock(&kbdev->mmu_hw_mutex); ++ /* AS transaction end */ ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ if (as->poke_refcount && ++ !(as->poke_state & KBASE_AS_POKE_STATE_KILLING_POKE)) { ++ /* Only queue up the timer if we need it, and we're not trying to kill it */ ++ hrtimer_start(&as->poke_timer, HR_TIMER_DELAY_MSEC(5), HRTIMER_MODE_REL); ++ } ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++} ++ ++enum hrtimer_restart kbasep_as_poke_timer_callback(struct hrtimer *timer) ++{ ++ struct kbase_as *as; ++ int queue_work_ret; ++ ++ KBASE_DEBUG_ASSERT(NULL != timer); ++ as = container_of(timer, struct kbase_as, poke_timer); ++ KBASE_DEBUG_ASSERT(as->poke_state & KBASE_AS_POKE_STATE_IN_FLIGHT); ++ ++ queue_work_ret = queue_work(as->poke_wq, &as->poke_work); ++ KBASE_DEBUG_ASSERT(queue_work_ret); ++ return HRTIMER_NORESTART; ++} ++ ++/** ++ * Retain the poking timer on an atom's context (if the atom hasn't already ++ * done so), and start the timer (if it's not already started). ++ * ++ * This must only be called on a context that's scheduled in, and an atom ++ * that's running on the GPU. ++ * ++ * The caller must hold hwaccess_lock ++ * ++ * This can be called safely from atomic context ++ */ ++void kbase_as_poking_timer_retain_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom) ++{ ++ struct kbase_as *as; ++ ++ KBASE_DEBUG_ASSERT(kbdev); ++ KBASE_DEBUG_ASSERT(kctx); ++ KBASE_DEBUG_ASSERT(katom); ++ KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (katom->poking) ++ return; ++ ++ katom->poking = 1; ++ ++ /* It's safe to work on the as/as_nr without an explicit reference, ++ * because the caller holds the hwaccess_lock, and the atom itself ++ * was also running and had already taken a reference */ ++ as = &kbdev->as[kctx->as_nr]; ++ ++ if (++(as->poke_refcount) == 1) { ++ /* First refcount for poke needed: check if not already in flight */ ++ if (!as->poke_state) { ++ /* need to start poking */ ++ as->poke_state |= KBASE_AS_POKE_STATE_IN_FLIGHT; ++ queue_work(as->poke_wq, &as->poke_work); ++ } ++ } ++} ++ ++/** ++ * If an atom holds a poking timer, release it and wait for it to finish ++ * ++ * This must only be called on a context that's scheduled in, and an atom ++ * that still has a JS reference on the context ++ * ++ * This must \b not be called from atomic context, since it can sleep. ++ */ ++void kbase_as_poking_timer_release_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom) ++{ ++ struct kbase_as *as; ++ unsigned long flags; ++ ++ KBASE_DEBUG_ASSERT(kbdev); ++ KBASE_DEBUG_ASSERT(kctx); ++ KBASE_DEBUG_ASSERT(katom); ++ KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); ++ ++ if (!katom->poking) ++ return; ++ ++ as = &kbdev->as[kctx->as_nr]; ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ KBASE_DEBUG_ASSERT(as->poke_refcount > 0); ++ KBASE_DEBUG_ASSERT(as->poke_state & KBASE_AS_POKE_STATE_IN_FLIGHT); ++ ++ if (--(as->poke_refcount) == 0) { ++ as->poke_state |= KBASE_AS_POKE_STATE_KILLING_POKE; ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ hrtimer_cancel(&as->poke_timer); ++ flush_workqueue(as->poke_wq); ++ ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ ++ /* Re-check whether it's still needed */ ++ if (as->poke_refcount) { ++ int queue_work_ret; ++ /* Poking still needed: ++ * - Another retain will not be starting the timer or queueing work, ++ * because it's still marked as in-flight ++ * - The hrtimer has finished, and has not started a new timer or ++ * queued work because it's been marked as killing ++ * ++ * So whatever happens now, just queue the work again */ ++ as->poke_state &= ~((kbase_as_poke_state)KBASE_AS_POKE_STATE_KILLING_POKE); ++ queue_work_ret = queue_work(as->poke_wq, &as->poke_work); ++ KBASE_DEBUG_ASSERT(queue_work_ret); ++ } else { ++ /* It isn't - so mark it as not in flight, and not killing */ ++ as->poke_state = 0u; ++ ++ /* The poke associated with the atom has now finished. If this is ++ * also the last atom on the context, then we can guarentee no more ++ * pokes (and thus no more poking register accesses) will occur on ++ * the context until new atoms are run */ ++ } ++ } ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ ++ katom->poking = 0; ++} ++ ++void kbase_mmu_interrupt_process(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_as *as) ++{ ++ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; ++ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (!kctx) { ++ dev_warn(kbdev->dev, "%s in AS%d at 0x%016llx with no context present! Suprious IRQ or SW Design Error?\n", ++ kbase_as_has_bus_fault(as) ? "Bus error" : "Page fault", ++ as->number, as->fault_addr); ++ ++ /* Since no ctx was found, the MMU must be disabled. */ ++ WARN_ON(as->current_setup.transtab); ++ ++ if (kbase_as_has_bus_fault(as)) { ++ kbase_mmu_hw_clear_fault(kbdev, as, kctx, ++ KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); ++ kbase_mmu_hw_enable_fault(kbdev, as, kctx, ++ KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); ++ } else if (kbase_as_has_page_fault(as)) { ++ kbase_mmu_hw_clear_fault(kbdev, as, kctx, ++ KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); ++ kbase_mmu_hw_enable_fault(kbdev, as, kctx, ++ KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); ++ } ++ ++#if KBASE_GPU_RESET_EN ++ if (kbase_as_has_bus_fault(as) && ++ kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245)) { ++ bool reset_status; ++ /* ++ * Reset the GPU, like in bus_fault_worker, in case an ++ * earlier error hasn't been properly cleared by this ++ * point. ++ */ ++ dev_err(kbdev->dev, "GPU bus error occurred. For this GPU version we now soft-reset as part of bus error recovery\n"); ++ reset_status = kbase_prepare_to_reset_gpu_locked(kbdev); ++ if (reset_status) ++ kbase_reset_gpu_locked(kbdev); ++ } ++#endif /* KBASE_GPU_RESET_EN */ ++ ++ return; ++ } ++ ++ if (kbase_as_has_bus_fault(as)) { ++ /* ++ * hw counters dumping in progress, signal the ++ * other thread that it failed ++ */ ++ if ((kbdev->hwcnt.kctx == kctx) && ++ (kbdev->hwcnt.backend.state == ++ KBASE_INSTR_STATE_DUMPING)) ++ kbdev->hwcnt.backend.state = ++ KBASE_INSTR_STATE_FAULT; ++ ++ /* ++ * Stop the kctx from submitting more jobs and cause it ++ * to be scheduled out/rescheduled when all references ++ * to it are released ++ */ ++ kbasep_js_clear_submit_allowed(js_devdata, kctx); ++ ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) ++ dev_warn(kbdev->dev, ++ "Bus error in AS%d at VA=0x%016llx, IPA=0x%016llx\n", ++ as->number, as->fault_addr, ++ as->fault_extra_addr); ++ else ++ dev_warn(kbdev->dev, "Bus error in AS%d at 0x%016llx\n", ++ as->number, as->fault_addr); ++ ++ /* ++ * We need to switch to UNMAPPED mode - but we do this in a ++ * worker so that we can sleep ++ */ ++ kbdev->kbase_group_error++; ++ KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&as->work_busfault)); ++ WARN_ON(work_pending(&as->work_busfault)); ++ queue_work(as->pf_wq, &as->work_busfault); ++ atomic_inc(&kbdev->faults_pending); ++ } else { ++ kbdev->kbase_group_error++; ++ KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&as->work_pagefault)); ++ WARN_ON(work_pending(&as->work_pagefault)); ++ queue_work(as->pf_wq, &as->work_pagefault); ++ atomic_inc(&kbdev->faults_pending); ++ } ++} ++ ++void kbase_flush_mmu_wqs(struct kbase_device *kbdev) ++{ ++ int i; ++ ++ for (i = 0; i < kbdev->nr_hw_address_spaces; i++) { ++ struct kbase_as *as = &kbdev->as[i]; ++ ++ flush_workqueue(as->pf_wq); ++ } ++} +diff --git a/drivers/gpu/arm/midgard/mali_kbase_mmu_hw.h b/drivers/gpu/arm/midgard/mali_kbase_mmu_hw.h +new file mode 100755 +index 000000000000..986e959e9a0c +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_mmu_hw.h +@@ -0,0 +1,123 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/** ++ * @file ++ * Interface file for accessing MMU hardware functionality ++ */ ++ ++/** ++ * @page mali_kbase_mmu_hw_page MMU hardware interface ++ * ++ * @section mali_kbase_mmu_hw_intro_sec Introduction ++ * This module provides an abstraction for accessing the functionality provided ++ * by the midgard MMU and thus allows all MMU HW access to be contained within ++ * one common place and allows for different backends (implementations) to ++ * be provided. ++ */ ++ ++#ifndef _MALI_KBASE_MMU_HW_H_ ++#define _MALI_KBASE_MMU_HW_H_ ++ ++/* Forward declarations */ ++struct kbase_device; ++struct kbase_as; ++struct kbase_context; ++ ++/** ++ * @addtogroup base_kbase_api ++ * @{ ++ */ ++ ++/** ++ * @addtogroup mali_kbase_mmu_hw MMU access APIs ++ * @{ ++ */ ++ ++/** @brief MMU fault type descriptor. ++ */ ++enum kbase_mmu_fault_type { ++ KBASE_MMU_FAULT_TYPE_UNKNOWN = 0, ++ KBASE_MMU_FAULT_TYPE_PAGE, ++ KBASE_MMU_FAULT_TYPE_BUS, ++ KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED, ++ KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED ++}; ++ ++/** @brief Configure an address space for use. ++ * ++ * Configure the MMU using the address space details setup in the ++ * @ref kbase_context structure. ++ * ++ * @param[in] kbdev kbase device to configure. ++ * @param[in] as address space to configure. ++ * @param[in] kctx kbase context to configure. ++ */ ++void kbase_mmu_hw_configure(struct kbase_device *kbdev, ++ struct kbase_as *as, struct kbase_context *kctx); ++ ++/** @brief Issue an operation to the MMU. ++ * ++ * Issue an operation (MMU invalidate, MMU flush, etc) on the address space that ++ * is associated with the provided @ref kbase_context over the specified range ++ * ++ * @param[in] kbdev kbase device to issue the MMU operation on. ++ * @param[in] as address space to issue the MMU operation on. ++ * @param[in] kctx kbase context to issue the MMU operation on. ++ * @param[in] vpfn MMU Virtual Page Frame Number to start the ++ * operation on. ++ * @param[in] nr Number of pages to work on. ++ * @param[in] type Operation type (written to ASn_COMMAND). ++ * @param[in] handling_irq Is this operation being called during the handling ++ * of an interrupt? ++ * ++ * @return Zero if the operation was successful, non-zero otherwise. ++ */ ++int kbase_mmu_hw_do_operation(struct kbase_device *kbdev, struct kbase_as *as, ++ struct kbase_context *kctx, u64 vpfn, u32 nr, u32 type, ++ unsigned int handling_irq); ++ ++/** @brief Clear a fault that has been previously reported by the MMU. ++ * ++ * Clear a bus error or page fault that has been reported by the MMU. ++ * ++ * @param[in] kbdev kbase device to clear the fault from. ++ * @param[in] as address space to clear the fault from. ++ * @param[in] kctx kbase context to clear the fault from or NULL. ++ * @param[in] type The type of fault that needs to be cleared. ++ */ ++void kbase_mmu_hw_clear_fault(struct kbase_device *kbdev, struct kbase_as *as, ++ struct kbase_context *kctx, enum kbase_mmu_fault_type type); ++ ++/** @brief Enable fault that has been previously reported by the MMU. ++ * ++ * After a page fault or bus error has been reported by the MMU these ++ * will be disabled. After these are handled this function needs to be ++ * called to enable the page fault or bus error fault again. ++ * ++ * @param[in] kbdev kbase device to again enable the fault from. ++ * @param[in] as address space to again enable the fault from. ++ * @param[in] kctx kbase context to again enable the fault from. ++ * @param[in] type The type of fault that needs to be enabled again. ++ */ ++void kbase_mmu_hw_enable_fault(struct kbase_device *kbdev, struct kbase_as *as, ++ struct kbase_context *kctx, enum kbase_mmu_fault_type type); ++ ++/** @} *//* end group mali_kbase_mmu_hw */ ++/** @} *//* end group base_kbase_api */ ++ ++#endif /* _MALI_KBASE_MMU_HW_H_ */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_mmu_mode.h b/drivers/gpu/arm/midgard/mali_kbase_mmu_mode.h +new file mode 100755 +index 000000000000..b487c00426ae +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_mmu_mode.h +@@ -0,0 +1,47 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2015, 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#ifndef _MALI_KBASE_MMU_MODE_ ++#define _MALI_KBASE_MMU_MODE_ ++ ++#include ++ ++/* Forward declarations */ ++struct kbase_context; ++struct kbase_device; ++struct kbase_as; ++struct kbase_mmu_setup; ++ ++struct kbase_mmu_mode { ++ void (*update)(struct kbase_context *kctx); ++ void (*get_as_setup)(struct kbase_context *kctx, ++ struct kbase_mmu_setup * const setup); ++ void (*disable_as)(struct kbase_device *kbdev, int as_nr); ++ phys_addr_t (*pte_to_phy_addr)(u64 entry); ++ int (*ate_is_valid)(u64 ate); ++ int (*pte_is_valid)(u64 pte); ++ void (*entry_set_ate)(u64 *entry, phys_addr_t phy, unsigned long flags); ++ void (*entry_set_pte)(u64 *entry, phys_addr_t phy); ++ void (*entry_invalidate)(u64 *entry); ++}; ++ ++struct kbase_mmu_mode const *kbase_mmu_mode_get_lpae(void); ++struct kbase_mmu_mode const *kbase_mmu_mode_get_aarch64(void); ++ ++#endif /* _MALI_KBASE_MMU_MODE_ */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_mmu_mode_aarch64.c b/drivers/gpu/arm/midgard/mali_kbase_mmu_mode_aarch64.c +new file mode 100755 +index 000000000000..60df171164ff +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_mmu_mode_aarch64.c +@@ -0,0 +1,200 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2014, 2016, 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#include "mali_kbase_mmu_mode.h" ++ ++#include "mali_kbase.h" ++#include "mali_midg_regmap.h" ++ ++#define ENTRY_TYPE_MASK 3ULL ++/* For valid ATEs bit 1 = (level == 3) ? 1 : 0. ++ * The MMU is only ever configured by the driver so that ATEs ++ * are at level 3, so bit 1 should always be set ++ */ ++#define ENTRY_IS_ATE 3ULL ++#define ENTRY_IS_INVAL 2ULL ++#define ENTRY_IS_PTE 3ULL ++ ++#define ENTRY_ATTR_BITS (7ULL << 2) /* bits 4:2 */ ++#define ENTRY_ACCESS_RW (1ULL << 6) /* bits 6:7 */ ++#define ENTRY_ACCESS_RO (3ULL << 6) ++#define ENTRY_SHARE_BITS (3ULL << 8) /* bits 9:8 */ ++#define ENTRY_ACCESS_BIT (1ULL << 10) ++#define ENTRY_NX_BIT (1ULL << 54) ++ ++/* Helper Function to perform assignment of page table entries, to ++ * ensure the use of strd, which is required on LPAE systems. ++ */ ++static inline void page_table_entry_set(u64 *pte, u64 phy) ++{ ++#ifdef CONFIG_64BIT ++ *pte = phy; ++#elif defined(CONFIG_ARM) ++ /* ++ * In order to prevent the compiler keeping cached copies of ++ * memory, we have to explicitly say that we have updated memory. ++ * ++ * Note: We could manually move the data ourselves into R0 and ++ * R1 by specifying register variables that are explicitly ++ * given registers assignments, the down side of this is that ++ * we have to assume cpu endianness. To avoid this we can use ++ * the ldrd to read the data from memory into R0 and R1 which ++ * will respect the cpu endianness, we then use strd to make ++ * the 64 bit assignment to the page table entry. ++ */ ++ asm volatile("ldrd r0, r1, [%[ptemp]]\n\t" ++ "strd r0, r1, [%[pte]]\n\t" ++ : "=m" (*pte) ++ : [ptemp] "r" (&phy), [pte] "r" (pte), "m" (phy) ++ : "r0", "r1"); ++#else ++#error "64-bit atomic write must be implemented for your architecture" ++#endif ++} ++ ++static void mmu_get_as_setup(struct kbase_context *kctx, ++ struct kbase_mmu_setup * const setup) ++{ ++ /* Set up the required caching policies at the correct indices ++ * in the memattr register. ++ */ ++ setup->memattr = ++ (AS_MEMATTR_IMPL_DEF_CACHE_POLICY << ++ (AS_MEMATTR_INDEX_IMPL_DEF_CACHE_POLICY * 8)) | ++ (AS_MEMATTR_FORCE_TO_CACHE_ALL << ++ (AS_MEMATTR_INDEX_FORCE_TO_CACHE_ALL * 8)) | ++ (AS_MEMATTR_WRITE_ALLOC << ++ (AS_MEMATTR_INDEX_WRITE_ALLOC * 8)) | ++ (AS_MEMATTR_AARCH64_OUTER_IMPL_DEF << ++ (AS_MEMATTR_INDEX_OUTER_IMPL_DEF * 8)) | ++ (AS_MEMATTR_AARCH64_OUTER_WA << ++ (AS_MEMATTR_INDEX_OUTER_WA * 8)); ++ ++ setup->transtab = (u64)kctx->pgd & AS_TRANSTAB_BASE_MASK; ++ setup->transcfg = AS_TRANSCFG_ADRMODE_AARCH64_4K; ++} ++ ++static void mmu_update(struct kbase_context *kctx) ++{ ++ struct kbase_device * const kbdev = kctx->kbdev; ++ struct kbase_as * const as = &kbdev->as[kctx->as_nr]; ++ struct kbase_mmu_setup * const current_setup = &as->current_setup; ++ ++ mmu_get_as_setup(kctx, current_setup); ++ ++ /* Apply the address space setting */ ++ kbase_mmu_hw_configure(kbdev, as, kctx); ++} ++ ++static void mmu_disable_as(struct kbase_device *kbdev, int as_nr) ++{ ++ struct kbase_as * const as = &kbdev->as[as_nr]; ++ struct kbase_mmu_setup * const current_setup = &as->current_setup; ++ ++ current_setup->transtab = 0ULL; ++ current_setup->transcfg = AS_TRANSCFG_ADRMODE_UNMAPPED; ++ ++ /* Apply the address space setting */ ++ kbase_mmu_hw_configure(kbdev, as, NULL); ++} ++ ++static phys_addr_t pte_to_phy_addr(u64 entry) ++{ ++ if (!(entry & 1)) ++ return 0; ++ ++ return entry & ~0xFFF; ++} ++ ++static int ate_is_valid(u64 ate) ++{ ++ return ((ate & ENTRY_TYPE_MASK) == ENTRY_IS_ATE); ++} ++ ++static int pte_is_valid(u64 pte) ++{ ++ return ((pte & ENTRY_TYPE_MASK) == ENTRY_IS_PTE); ++} ++ ++/* ++ * Map KBASE_REG flags to MMU flags ++ */ ++static u64 get_mmu_flags(unsigned long flags) ++{ ++ u64 mmu_flags; ++ ++ /* store mem_attr index as 4:2 (macro called ensures 3 bits already) */ ++ mmu_flags = KBASE_REG_MEMATTR_VALUE(flags) << 2; ++ ++ /* Set access flags - note that AArch64 stage 1 does not support ++ * write-only access, so we use read/write instead ++ */ ++ if (flags & KBASE_REG_GPU_WR) ++ mmu_flags |= ENTRY_ACCESS_RW; ++ else if (flags & KBASE_REG_GPU_RD) ++ mmu_flags |= ENTRY_ACCESS_RO; ++ ++ /* nx if requested */ ++ mmu_flags |= (flags & KBASE_REG_GPU_NX) ? ENTRY_NX_BIT : 0; ++ ++ if (flags & KBASE_REG_SHARE_BOTH) { ++ /* inner and outer shareable */ ++ mmu_flags |= SHARE_BOTH_BITS; ++ } else if (flags & KBASE_REG_SHARE_IN) { ++ /* inner shareable coherency */ ++ mmu_flags |= SHARE_INNER_BITS; ++ } ++ ++ return mmu_flags; ++} ++ ++static void entry_set_ate(u64 *entry, phys_addr_t phy, unsigned long flags) ++{ ++ page_table_entry_set(entry, (phy & ~0xFFF) | ++ get_mmu_flags(flags) | ++ ENTRY_ACCESS_BIT | ENTRY_IS_ATE); ++} ++ ++static void entry_set_pte(u64 *entry, phys_addr_t phy) ++{ ++ page_table_entry_set(entry, (phy & ~0xFFF) | ++ ENTRY_ACCESS_BIT | ENTRY_IS_PTE); ++} ++ ++static void entry_invalidate(u64 *entry) ++{ ++ page_table_entry_set(entry, ENTRY_IS_INVAL); ++} ++ ++static struct kbase_mmu_mode const aarch64_mode = { ++ .update = mmu_update, ++ .get_as_setup = mmu_get_as_setup, ++ .disable_as = mmu_disable_as, ++ .pte_to_phy_addr = pte_to_phy_addr, ++ .ate_is_valid = ate_is_valid, ++ .pte_is_valid = pte_is_valid, ++ .entry_set_ate = entry_set_ate, ++ .entry_set_pte = entry_set_pte, ++ .entry_invalidate = entry_invalidate ++}; ++ ++struct kbase_mmu_mode const *kbase_mmu_mode_get_aarch64(void) ++{ ++ return &aarch64_mode; ++} +diff --git a/drivers/gpu/arm/midgard/mali_kbase_mmu_mode_lpae.c b/drivers/gpu/arm/midgard/mali_kbase_mmu_mode_lpae.c +new file mode 100755 +index 000000000000..53fbbc73af91 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_mmu_mode_lpae.c +@@ -0,0 +1,198 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#include "mali_kbase_mmu_mode.h" ++ ++#include "mali_kbase.h" ++#include "mali_midg_regmap.h" ++ ++#define ENTRY_TYPE_MASK 3ULL ++#define ENTRY_IS_ATE 1ULL ++#define ENTRY_IS_INVAL 2ULL ++#define ENTRY_IS_PTE 3ULL ++ ++#define ENTRY_ATTR_BITS (7ULL << 2) /* bits 4:2 */ ++#define ENTRY_RD_BIT (1ULL << 6) ++#define ENTRY_WR_BIT (1ULL << 7) ++#define ENTRY_SHARE_BITS (3ULL << 8) /* bits 9:8 */ ++#define ENTRY_ACCESS_BIT (1ULL << 10) ++#define ENTRY_NX_BIT (1ULL << 54) ++ ++#define ENTRY_FLAGS_MASK (ENTRY_ATTR_BITS | ENTRY_RD_BIT | ENTRY_WR_BIT | \ ++ ENTRY_SHARE_BITS | ENTRY_ACCESS_BIT | ENTRY_NX_BIT) ++ ++/* Helper Function to perform assignment of page table entries, to ++ * ensure the use of strd, which is required on LPAE systems. ++ */ ++static inline void page_table_entry_set(u64 *pte, u64 phy) ++{ ++#ifdef CONFIG_64BIT ++ *pte = phy; ++#elif defined(CONFIG_ARM) ++ /* ++ * In order to prevent the compiler keeping cached copies of ++ * memory, we have to explicitly say that we have updated ++ * memory. ++ * ++ * Note: We could manually move the data ourselves into R0 and ++ * R1 by specifying register variables that are explicitly ++ * given registers assignments, the down side of this is that ++ * we have to assume cpu endianness. To avoid this we can use ++ * the ldrd to read the data from memory into R0 and R1 which ++ * will respect the cpu endianness, we then use strd to make ++ * the 64 bit assignment to the page table entry. ++ */ ++ asm volatile("ldrd r0, r1, [%[ptemp]]\n\t" ++ "strd r0, r1, [%[pte]]\n\t" ++ : "=m" (*pte) ++ : [ptemp] "r" (&phy), [pte] "r" (pte), "m" (phy) ++ : "r0", "r1"); ++#else ++#error "64-bit atomic write must be implemented for your architecture" ++#endif ++} ++ ++static void mmu_get_as_setup(struct kbase_context *kctx, ++ struct kbase_mmu_setup * const setup) ++{ ++ /* Set up the required caching policies at the correct indices ++ * in the memattr register. */ ++ setup->memattr = ++ (AS_MEMATTR_LPAE_IMPL_DEF_CACHE_POLICY << ++ (AS_MEMATTR_INDEX_IMPL_DEF_CACHE_POLICY * 8)) | ++ (AS_MEMATTR_LPAE_FORCE_TO_CACHE_ALL << ++ (AS_MEMATTR_INDEX_FORCE_TO_CACHE_ALL * 8)) | ++ (AS_MEMATTR_LPAE_WRITE_ALLOC << ++ (AS_MEMATTR_INDEX_WRITE_ALLOC * 8)) | ++ (AS_MEMATTR_LPAE_OUTER_IMPL_DEF << ++ (AS_MEMATTR_INDEX_OUTER_IMPL_DEF * 8)) | ++ (AS_MEMATTR_LPAE_OUTER_WA << ++ (AS_MEMATTR_INDEX_OUTER_WA * 8)) | ++ 0; /* The other indices are unused for now */ ++ ++ setup->transtab = ((u64)kctx->pgd & ++ ((0xFFFFFFFFULL << 32) | AS_TRANSTAB_LPAE_ADDR_SPACE_MASK)) | ++ AS_TRANSTAB_LPAE_ADRMODE_TABLE | ++ AS_TRANSTAB_LPAE_READ_INNER; ++ ++ setup->transcfg = 0; ++} ++ ++static void mmu_update(struct kbase_context *kctx) ++{ ++ struct kbase_device * const kbdev = kctx->kbdev; ++ struct kbase_as * const as = &kbdev->as[kctx->as_nr]; ++ struct kbase_mmu_setup * const current_setup = &as->current_setup; ++ ++ mmu_get_as_setup(kctx, current_setup); ++ ++ /* Apply the address space setting */ ++ kbase_mmu_hw_configure(kbdev, as, kctx); ++} ++ ++static void mmu_disable_as(struct kbase_device *kbdev, int as_nr) ++{ ++ struct kbase_as * const as = &kbdev->as[as_nr]; ++ struct kbase_mmu_setup * const current_setup = &as->current_setup; ++ ++ current_setup->transtab = AS_TRANSTAB_LPAE_ADRMODE_UNMAPPED; ++ ++ /* Apply the address space setting */ ++ kbase_mmu_hw_configure(kbdev, as, NULL); ++} ++ ++static phys_addr_t pte_to_phy_addr(u64 entry) ++{ ++ if (!(entry & 1)) ++ return 0; ++ ++ return entry & ~0xFFF; ++} ++ ++static int ate_is_valid(u64 ate) ++{ ++ return ((ate & ENTRY_TYPE_MASK) == ENTRY_IS_ATE); ++} ++ ++static int pte_is_valid(u64 pte) ++{ ++ return ((pte & ENTRY_TYPE_MASK) == ENTRY_IS_PTE); ++} ++ ++/* ++ * Map KBASE_REG flags to MMU flags ++ */ ++static u64 get_mmu_flags(unsigned long flags) ++{ ++ u64 mmu_flags; ++ ++ /* store mem_attr index as 4:2 (macro called ensures 3 bits already) */ ++ mmu_flags = KBASE_REG_MEMATTR_VALUE(flags) << 2; ++ ++ /* write perm if requested */ ++ mmu_flags |= (flags & KBASE_REG_GPU_WR) ? ENTRY_WR_BIT : 0; ++ /* read perm if requested */ ++ mmu_flags |= (flags & KBASE_REG_GPU_RD) ? ENTRY_RD_BIT : 0; ++ /* nx if requested */ ++ mmu_flags |= (flags & KBASE_REG_GPU_NX) ? ENTRY_NX_BIT : 0; ++ ++ if (flags & KBASE_REG_SHARE_BOTH) { ++ /* inner and outer shareable */ ++ mmu_flags |= SHARE_BOTH_BITS; ++ } else if (flags & KBASE_REG_SHARE_IN) { ++ /* inner shareable coherency */ ++ mmu_flags |= SHARE_INNER_BITS; ++ } ++ ++ return mmu_flags; ++} ++ ++static void entry_set_ate(u64 *entry, phys_addr_t phy, unsigned long flags) ++{ ++ page_table_entry_set(entry, (phy & ~0xFFF) | ++ get_mmu_flags(flags) | ++ ENTRY_IS_ATE); ++} ++ ++static void entry_set_pte(u64 *entry, phys_addr_t phy) ++{ ++ page_table_entry_set(entry, (phy & ~0xFFF) | ENTRY_IS_PTE); ++} ++ ++static void entry_invalidate(u64 *entry) ++{ ++ page_table_entry_set(entry, ENTRY_IS_INVAL); ++} ++ ++static struct kbase_mmu_mode const lpae_mode = { ++ .update = mmu_update, ++ .get_as_setup = mmu_get_as_setup, ++ .disable_as = mmu_disable_as, ++ .pte_to_phy_addr = pte_to_phy_addr, ++ .ate_is_valid = ate_is_valid, ++ .pte_is_valid = pte_is_valid, ++ .entry_set_ate = entry_set_ate, ++ .entry_set_pte = entry_set_pte, ++ .entry_invalidate = entry_invalidate ++}; ++ ++struct kbase_mmu_mode const *kbase_mmu_mode_get_lpae(void) ++{ ++ return &lpae_mode; ++} +diff --git a/drivers/gpu/arm/midgard/mali_kbase_platform_fake.c b/drivers/gpu/arm/midgard/mali_kbase_platform_fake.c +new file mode 100755 +index 000000000000..1a44957fe44a +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_platform_fake.c +@@ -0,0 +1,124 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2014, 2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifdef CONFIG_MALI_PLATFORM_FAKE ++ ++#include ++#include ++#include ++#include ++#include ++ ++ ++/* ++ * This file is included only for type definitions and functions belonging to ++ * specific platform folders. Do not add dependencies with symbols that are ++ * defined somewhere else. ++ */ ++#include ++ ++#define PLATFORM_CONFIG_RESOURCE_COUNT 4 ++#define PLATFORM_CONFIG_IRQ_RES_COUNT 3 ++ ++static struct platform_device *mali_device; ++ ++#ifndef CONFIG_OF ++/** ++ * @brief Convert data in struct kbase_io_resources struct to Linux-specific resources ++ * ++ * Function converts data in struct kbase_io_resources struct to an array of Linux resource structures. Note that function ++ * assumes that size of linux_resource array is at least PLATFORM_CONFIG_RESOURCE_COUNT. ++ * Resources are put in fixed order: I/O memory region, job IRQ, MMU IRQ, GPU IRQ. ++ * ++ * @param[in] io_resource Input IO resource data ++ * @param[out] linux_resources Pointer to output array of Linux resource structures ++ */ ++static void kbasep_config_parse_io_resources(const struct kbase_io_resources *io_resources, struct resource *const linux_resources) ++{ ++ if (!io_resources || !linux_resources) { ++ pr_err("%s: couldn't find proper resources\n", __func__); ++ return; ++ } ++ ++ memset(linux_resources, 0, PLATFORM_CONFIG_RESOURCE_COUNT * sizeof(struct resource)); ++ ++ linux_resources[0].start = io_resources->io_memory_region.start; ++ linux_resources[0].end = io_resources->io_memory_region.end; ++ linux_resources[0].flags = IORESOURCE_MEM; ++ ++ linux_resources[1].start = io_resources->job_irq_number; ++ linux_resources[1].end = io_resources->job_irq_number; ++ linux_resources[1].flags = IORESOURCE_IRQ | IORESOURCE_IRQ_HIGHLEVEL; ++ ++ linux_resources[2].start = io_resources->mmu_irq_number; ++ linux_resources[2].end = io_resources->mmu_irq_number; ++ linux_resources[2].flags = IORESOURCE_IRQ | IORESOURCE_IRQ_HIGHLEVEL; ++ ++ linux_resources[3].start = io_resources->gpu_irq_number; ++ linux_resources[3].end = io_resources->gpu_irq_number; ++ linux_resources[3].flags = IORESOURCE_IRQ | IORESOURCE_IRQ_HIGHLEVEL; ++} ++#endif /* CONFIG_OF */ ++ ++int kbase_platform_fake_register(void) ++{ ++ struct kbase_platform_config *config; ++#ifndef CONFIG_OF ++ struct resource resources[PLATFORM_CONFIG_RESOURCE_COUNT]; ++#endif ++ int err; ++ ++ config = kbase_get_platform_config(); /* declared in midgard/mali_kbase_config.h but defined in platform folder */ ++ if (config == NULL) { ++ pr_err("%s: couldn't get platform config\n", __func__); ++ return -ENODEV; ++ } ++ ++ mali_device = platform_device_alloc("mali", 0); ++ if (mali_device == NULL) ++ return -ENOMEM; ++ ++#ifndef CONFIG_OF ++ kbasep_config_parse_io_resources(config->io_resources, resources); ++ err = platform_device_add_resources(mali_device, resources, PLATFORM_CONFIG_RESOURCE_COUNT); ++ if (err) { ++ platform_device_put(mali_device); ++ mali_device = NULL; ++ return err; ++ } ++#endif /* CONFIG_OF */ ++ ++ err = platform_device_add(mali_device); ++ if (err) { ++ platform_device_unregister(mali_device); ++ mali_device = NULL; ++ return err; ++ } ++ ++ return 0; ++} ++EXPORT_SYMBOL(kbase_platform_fake_register); ++ ++void kbase_platform_fake_unregister(void) ++{ ++ if (mali_device) ++ platform_device_unregister(mali_device); ++} ++EXPORT_SYMBOL(kbase_platform_fake_unregister); ++ ++#endif /* CONFIG_MALI_PLATFORM_FAKE */ ++ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_pm.c b/drivers/gpu/arm/midgard/mali_kbase_pm.c +new file mode 100755 +index 000000000000..97d543464c28 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_pm.c +@@ -0,0 +1,205 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/** ++ * @file mali_kbase_pm.c ++ * Base kernel power management APIs ++ */ ++ ++#include ++#include ++#include ++ ++#include ++ ++int kbase_pm_powerup(struct kbase_device *kbdev, unsigned int flags) ++{ ++ return kbase_hwaccess_pm_powerup(kbdev, flags); ++} ++ ++void kbase_pm_halt(struct kbase_device *kbdev) ++{ ++ kbase_hwaccess_pm_halt(kbdev); ++} ++ ++void kbase_pm_context_active(struct kbase_device *kbdev) ++{ ++ (void)kbase_pm_context_active_handle_suspend(kbdev, KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE); ++} ++ ++int kbase_pm_context_active_handle_suspend(struct kbase_device *kbdev, enum kbase_pm_suspend_handler suspend_handler) ++{ ++ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; ++ int c; ++ int old_count; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ /* Trace timeline information about how long it took to handle the decision ++ * to powerup. Sometimes the event might be missed due to reading the count ++ * outside of mutex, but this is necessary to get the trace timing ++ * correct. */ ++ old_count = kbdev->pm.active_count; ++ if (old_count == 0) ++ kbase_timeline_pm_send_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_ACTIVE); ++ ++ mutex_lock(&js_devdata->runpool_mutex); ++ mutex_lock(&kbdev->pm.lock); ++ if (kbase_pm_is_suspending(kbdev)) { ++ switch (suspend_handler) { ++ case KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE: ++ if (kbdev->pm.active_count != 0) ++ break; ++ /* FALLTHROUGH */ ++ case KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE: ++ mutex_unlock(&kbdev->pm.lock); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ if (old_count == 0) ++ kbase_timeline_pm_handle_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_ACTIVE); ++ return 1; ++ ++ case KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE: ++ /* FALLTHROUGH */ ++ default: ++ KBASE_DEBUG_ASSERT_MSG(false, "unreachable"); ++ break; ++ } ++ } ++ c = ++kbdev->pm.active_count; ++ KBASE_TIMELINE_CONTEXT_ACTIVE(kbdev, c); ++ KBASE_TRACE_ADD_REFCOUNT(kbdev, PM_CONTEXT_ACTIVE, NULL, NULL, 0u, c); ++ ++ /* Trace the event being handled */ ++ if (old_count == 0) ++ kbase_timeline_pm_handle_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_ACTIVE); ++ ++ if (c == 1) ++ /* First context active: Power on the GPU and any cores requested by ++ * the policy */ ++ kbase_hwaccess_pm_gpu_active(kbdev); ++ ++ mutex_unlock(&kbdev->pm.lock); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ ++ return 0; ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_context_active); ++ ++void kbase_pm_context_idle(struct kbase_device *kbdev) ++{ ++ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; ++ int c; ++ int old_count; ++ ++ KBASE_DEBUG_ASSERT(kbdev != NULL); ++ ++ /* Trace timeline information about how long it took to handle the decision ++ * to powerdown. Sometimes the event might be missed due to reading the ++ * count outside of mutex, but this is necessary to get the trace timing ++ * correct. */ ++ old_count = kbdev->pm.active_count; ++ if (old_count == 0) ++ kbase_timeline_pm_send_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_IDLE); ++ ++ mutex_lock(&js_devdata->runpool_mutex); ++ mutex_lock(&kbdev->pm.lock); ++ ++ c = --kbdev->pm.active_count; ++ KBASE_TIMELINE_CONTEXT_ACTIVE(kbdev, c); ++ KBASE_TRACE_ADD_REFCOUNT(kbdev, PM_CONTEXT_IDLE, NULL, NULL, 0u, c); ++ ++ KBASE_DEBUG_ASSERT(c >= 0); ++ ++ /* Trace the event being handled */ ++ if (old_count == 0) ++ kbase_timeline_pm_handle_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_IDLE); ++ ++ if (c == 0) { ++ /* Last context has gone idle */ ++ kbase_hwaccess_pm_gpu_idle(kbdev); ++ ++ /* Wake up anyone waiting for this to become 0 (e.g. suspend). The ++ * waiters must synchronize with us by locking the pm.lock after ++ * waiting */ ++ wake_up(&kbdev->pm.zero_active_count_wait); ++ } ++ ++ mutex_unlock(&kbdev->pm.lock); ++ mutex_unlock(&js_devdata->runpool_mutex); ++} ++ ++KBASE_EXPORT_TEST_API(kbase_pm_context_idle); ++ ++void kbase_pm_suspend(struct kbase_device *kbdev) ++{ ++ KBASE_DEBUG_ASSERT(kbdev); ++ ++ /* Suspend vinstr. ++ * This call will block until vinstr is suspended. */ ++ kbase_vinstr_suspend(kbdev->vinstr_ctx); ++ ++ mutex_lock(&kbdev->pm.lock); ++ KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev)); ++ kbdev->pm.suspending = true; ++ mutex_unlock(&kbdev->pm.lock); ++ ++ /* From now on, the active count will drop towards zero. Sometimes, it'll ++ * go up briefly before going down again. However, once it reaches zero it ++ * will stay there - guaranteeing that we've idled all pm references */ ++ ++ /* Suspend job scheduler and associated components, so that it releases all ++ * the PM active count references */ ++ kbasep_js_suspend(kbdev); ++ ++ /* Wait for the active count to reach zero. This is not the same as ++ * waiting for a power down, since not all policies power down when this ++ * reaches zero. */ ++ wait_event(kbdev->pm.zero_active_count_wait, kbdev->pm.active_count == 0); ++ ++ /* NOTE: We synchronize with anything that was just finishing a ++ * kbase_pm_context_idle() call by locking the pm.lock below */ ++ ++ kbase_hwaccess_pm_suspend(kbdev); ++} ++ ++void kbase_pm_resume(struct kbase_device *kbdev) ++{ ++ /* MUST happen before any pm_context_active calls occur */ ++ kbase_hwaccess_pm_resume(kbdev); ++ ++ /* Initial active call, to power on the GPU/cores if needed */ ++ kbase_pm_context_active(kbdev); ++ ++ /* Resume any blocked atoms (which may cause contexts to be scheduled in ++ * and dependent atoms to run) */ ++ kbase_resume_suspended_soft_jobs(kbdev); ++ ++ /* Resume the Job Scheduler and associated components, and start running ++ * atoms */ ++ kbasep_js_resume(kbdev); ++ ++ /* Matching idle call, to power off the GPU/cores if we didn't actually ++ * need it and the policy doesn't want it on */ ++ kbase_pm_context_idle(kbdev); ++ ++ /* Resume vinstr operation */ ++ kbase_vinstr_resume(kbdev->vinstr_ctx); ++} ++ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_pm.h b/drivers/gpu/arm/midgard/mali_kbase_pm.h +new file mode 100755 +index 000000000000..37fa2479df74 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_pm.h +@@ -0,0 +1,171 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/** ++ * @file mali_kbase_pm.h ++ * Power management API definitions ++ */ ++ ++#ifndef _KBASE_PM_H_ ++#define _KBASE_PM_H_ ++ ++#include "mali_kbase_hwaccess_pm.h" ++ ++#define PM_ENABLE_IRQS 0x01 ++#define PM_HW_ISSUES_DETECT 0x02 ++ ++ ++/** Initialize the power management framework. ++ * ++ * Must be called before any other power management function ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid pointer) ++ * ++ * @return 0 if the power management framework was successfully initialized. ++ */ ++int kbase_pm_init(struct kbase_device *kbdev); ++ ++/** Power up GPU after all modules have been initialized and interrupt handlers installed. ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid pointer) ++ * ++ * @param flags Flags to pass on to kbase_pm_init_hw ++ * ++ * @return 0 if powerup was successful. ++ */ ++int kbase_pm_powerup(struct kbase_device *kbdev, unsigned int flags); ++ ++/** ++ * Halt the power management framework. ++ * Should ensure that no new interrupts are generated, ++ * but allow any currently running interrupt handlers to complete successfully. ++ * The GPU is forced off by the time this function returns, regardless of ++ * whether or not the active power policy asks for the GPU to be powered off. ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_halt(struct kbase_device *kbdev); ++ ++/** Terminate the power management framework. ++ * ++ * No power management functions may be called after this ++ * (except @ref kbase_pm_init) ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_term(struct kbase_device *kbdev); ++ ++/** Increment the count of active contexts. ++ * ++ * This function should be called when a context is about to submit a job. It informs the active power policy that the ++ * GPU is going to be in use shortly and the policy is expected to start turning on the GPU. ++ * ++ * This function will block until the GPU is available. ++ * ++ * This function ASSERTS if a suspend is occuring/has occurred whilst this is ++ * in use. Use kbase_pm_contect_active_unless_suspending() instead. ++ * ++ * @note a Suspend is only visible to Kernel threads; user-space threads in a ++ * syscall cannot witness a suspend, because they are frozen before the suspend ++ * begins. ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_context_active(struct kbase_device *kbdev); ++ ++ ++/** Handler codes for doing kbase_pm_context_active_handle_suspend() */ ++enum kbase_pm_suspend_handler { ++ /** A suspend is not expected/not possible - this is the same as ++ * kbase_pm_context_active() */ ++ KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE, ++ /** If we're suspending, fail and don't increase the active count */ ++ KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE, ++ /** If we're suspending, succeed and allow the active count to increase iff ++ * it didn't go from 0->1 (i.e., we didn't re-activate the GPU). ++ * ++ * This should only be used when there is a bounded time on the activation ++ * (e.g. guarantee it's going to be idled very soon after) */ ++ KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE ++}; ++ ++/** Suspend 'safe' variant of kbase_pm_context_active() ++ * ++ * If a suspend is in progress, this allows for various different ways of ++ * handling the suspend. Refer to @ref enum kbase_pm_suspend_handler for details. ++ * ++ * We returns a status code indicating whether we're allowed to keep the GPU ++ * active during the suspend, depending on the handler code. If the status code ++ * indicates a failure, the caller must abort whatever operation it was ++ * attempting, and potentially queue it up for after the OS has resumed. ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid pointer) ++ * @param suspend_handler The handler code for how to handle a suspend that might occur ++ * @return zero Indicates success ++ * @return non-zero Indicates failure due to the system being suspending/suspended. ++ */ ++int kbase_pm_context_active_handle_suspend(struct kbase_device *kbdev, enum kbase_pm_suspend_handler suspend_handler); ++ ++/** Decrement the reference count of active contexts. ++ * ++ * This function should be called when a context becomes idle. After this call the GPU may be turned off by the power ++ * policy so the calling code should ensure that it does not access the GPU's registers. ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_context_idle(struct kbase_device *kbdev); ++ ++/** ++ * Suspend the GPU and prevent any further register accesses to it from Kernel ++ * threads. ++ * ++ * This is called in response to an OS suspend event, and calls into the various ++ * kbase components to complete the suspend. ++ * ++ * @note the mechanisms used here rely on all user-space threads being frozen ++ * by the OS before we suspend. Otherwise, an IOCTL could occur that powers up ++ * the GPU e.g. via atom submission. ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_suspend(struct kbase_device *kbdev); ++ ++/** ++ * Resume the GPU, allow register accesses to it, and resume running atoms on ++ * the GPU. ++ * ++ * This is called in response to an OS resume event, and calls into the various ++ * kbase components to complete the resume. ++ * ++ * @param kbdev The kbase device structure for the device (must be a valid pointer) ++ */ ++void kbase_pm_resume(struct kbase_device *kbdev); ++ ++/** ++ * kbase_pm_vsync_callback - vsync callback ++ * ++ * @buffer_updated: 1 if a new frame was displayed, 0 otherwise ++ * @data: Pointer to the kbase device as returned by kbase_find_device() ++ * ++ * Callback function used to notify the power management code that a vsync has ++ * occurred on the display. ++ */ ++void kbase_pm_vsync_callback(int buffer_updated, void *data); ++ ++#endif /* _KBASE_PM_H_ */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_profiling_gator_api.h b/drivers/gpu/arm/midgard/mali_kbase_profiling_gator_api.h +new file mode 100755 +index 000000000000..7fb674eded37 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_profiling_gator_api.h +@@ -0,0 +1,40 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010, 2013 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/** ++ * @file mali_kbase_profiling_gator_api.h ++ * Model interface ++ */ ++ ++#ifndef _KBASE_PROFILING_GATOR_API_H_ ++#define _KBASE_PROFILING_GATOR_API_H_ ++ ++/* ++ * List of possible actions to be controlled by Streamline. ++ * The following numbers are used by gator to control ++ * the frame buffer dumping and s/w counter reporting. ++ */ ++#define FBDUMP_CONTROL_ENABLE (1) ++#define FBDUMP_CONTROL_RATE (2) ++#define SW_COUNTER_ENABLE (3) ++#define FBDUMP_CONTROL_RESIZE_FACTOR (4) ++#define FBDUMP_CONTROL_MAX (5) ++#define FBDUMP_CONTROL_MIN FBDUMP_CONTROL_ENABLE ++ ++void _mali_profiling_control(u32 action, u32 value); ++ ++#endif /* _KBASE_PROFILING_GATOR_API */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_regs_history_debugfs.c b/drivers/gpu/arm/midgard/mali_kbase_regs_history_debugfs.c +new file mode 100755 +index 000000000000..c970650069cd +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_regs_history_debugfs.c +@@ -0,0 +1,130 @@ ++/* ++ * ++ * (C) COPYRIGHT 2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include "mali_kbase.h" ++ ++#include "mali_kbase_regs_history_debugfs.h" ++ ++#if defined(CONFIG_DEBUG_FS) && !defined(CONFIG_MALI_NO_MALI) ++ ++#include ++ ++ ++static int regs_history_size_get(void *data, u64 *val) ++{ ++ struct kbase_io_history *const h = data; ++ ++ *val = h->size; ++ ++ return 0; ++} ++ ++static int regs_history_size_set(void *data, u64 val) ++{ ++ struct kbase_io_history *const h = data; ++ ++ return kbase_io_history_resize(h, (u16)val); ++} ++ ++ ++DEFINE_SIMPLE_ATTRIBUTE(regs_history_size_fops, ++ regs_history_size_get, ++ regs_history_size_set, ++ "%llu\n"); ++ ++ ++/** ++ * regs_history_show - show callback for the register access history file. ++ * ++ * @sfile: The debugfs entry ++ * @data: Data associated with the entry ++ * ++ * This function is called to dump all recent accesses to the GPU registers. ++ * ++ * @return 0 if successfully prints data in debugfs entry file, failure ++ * otherwise ++ */ ++static int regs_history_show(struct seq_file *sfile, void *data) ++{ ++ struct kbase_io_history *const h = sfile->private; ++ u16 i; ++ size_t iters; ++ unsigned long flags; ++ ++ if (!h->enabled) { ++ seq_puts(sfile, "The register access history is disabled\n"); ++ goto out; ++ } ++ ++ spin_lock_irqsave(&h->lock, flags); ++ ++ iters = (h->size > h->count) ? h->count : h->size; ++ seq_printf(sfile, "Last %zu register accesses of %zu total:\n", iters, ++ h->count); ++ for (i = 0; i < iters; ++i) { ++ struct kbase_io_access *io = ++ &h->buf[(h->count - iters + i) % h->size]; ++ char const access = (io->addr & 1) ? 'w' : 'r'; ++ ++ seq_printf(sfile, "%6i: %c: reg 0x%p val %08x\n", i, access, ++ (void *)(io->addr & ~0x1), io->value); ++ } ++ ++ spin_unlock_irqrestore(&h->lock, flags); ++ ++out: ++ return 0; ++} ++ ++ ++/** ++ * regs_history_open - open operation for regs_history debugfs file ++ * ++ * @in: &struct inode pointer ++ * @file: &struct file pointer ++ * ++ * @return file descriptor ++ */ ++static int regs_history_open(struct inode *in, struct file *file) ++{ ++ return single_open(file, ®s_history_show, in->i_private); ++} ++ ++ ++static const struct file_operations regs_history_fops = { ++ .open = ®s_history_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = single_release, ++}; ++ ++ ++void kbasep_regs_history_debugfs_init(struct kbase_device *kbdev) ++{ ++ debugfs_create_bool("regs_history_enabled", S_IRUGO | S_IWUSR, ++ kbdev->mali_debugfs_directory, ++ &kbdev->io_history.enabled); ++ debugfs_create_file("regs_history_size", S_IRUGO | S_IWUSR, ++ kbdev->mali_debugfs_directory, ++ &kbdev->io_history, ®s_history_size_fops); ++ debugfs_create_file("regs_history", S_IRUGO, ++ kbdev->mali_debugfs_directory, &kbdev->io_history, ++ ®s_history_fops); ++} ++ ++ ++#endif /* CONFIG_DEBUG_FS */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_regs_history_debugfs.h b/drivers/gpu/arm/midgard/mali_kbase_regs_history_debugfs.h +new file mode 100755 +index 000000000000..f10837002330 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_regs_history_debugfs.h +@@ -0,0 +1,50 @@ ++/* ++ * ++ * (C) COPYRIGHT 2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/** ++ * Header file for register access history support via debugfs ++ * ++ * This interface is made available via /sys/kernel/debug/mali#/regs_history*. ++ * ++ * Usage: ++ * - regs_history_enabled: whether recording of register accesses is enabled. ++ * Write 'y' to enable, 'n' to disable. ++ * - regs_history_size: size of the register history buffer, must be > 0 ++ * - regs_history: return the information about last accesses to the registers. ++ */ ++ ++#ifndef _KBASE_REGS_HISTORY_DEBUGFS_H ++#define _KBASE_REGS_HISTORY_DEBUGFS_H ++ ++struct kbase_device; ++ ++#if defined(CONFIG_DEBUG_FS) && !defined(CONFIG_MALI_NO_MALI) ++ ++/** ++ * kbasep_regs_history_debugfs_init - add debugfs entries for register history ++ * ++ * @kbdev: Pointer to kbase_device containing the register history ++ */ ++void kbasep_regs_history_debugfs_init(struct kbase_device *kbdev); ++ ++#else /* CONFIG_DEBUG_FS */ ++ ++#define kbasep_regs_history_debugfs_init CSTD_NOP ++ ++#endif /* CONFIG_DEBUG_FS */ ++ ++#endif /*_KBASE_REGS_HISTORY_DEBUGFS_H*/ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_replay.c b/drivers/gpu/arm/midgard/mali_kbase_replay.c +new file mode 100755 +index 000000000000..84aa3316e435 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_replay.c +@@ -0,0 +1,1166 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/** ++ * @file mali_kbase_replay.c ++ * Replay soft job handlers ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++ ++#define JOB_NOT_STARTED 0 ++#define JOB_TYPE_NULL (1) ++#define JOB_TYPE_VERTEX (5) ++#define JOB_TYPE_TILER (7) ++#define JOB_TYPE_FUSED (8) ++#define JOB_TYPE_FRAGMENT (9) ++ ++#define JOB_HEADER_32_FBD_OFFSET (31*4) ++#define JOB_HEADER_64_FBD_OFFSET (44*4) ++ ++#define FBD_POINTER_MASK (~0x3f) ++ ++#define SFBD_TILER_OFFSET (48*4) ++ ++#define MFBD_TILER_OFFSET (14*4) ++ ++#define FBD_HIERARCHY_WEIGHTS 8 ++#define FBD_HIERARCHY_MASK_MASK 0x1fff ++ ++#define FBD_TYPE 1 ++ ++#define HIERARCHY_WEIGHTS 13 ++ ++#define JOB_HEADER_ID_MAX 0xffff ++ ++#define JOB_SOURCE_ID(status) (((status) >> 16) & 0xFFFF) ++#define JOB_POLYGON_LIST (0x03) ++ ++struct fragment_job { ++ struct job_descriptor_header header; ++ ++ u32 x[2]; ++ union { ++ u64 _64; ++ u32 _32; ++ } fragment_fbd; ++}; ++ ++static void dump_job_head(struct kbase_context *kctx, char *head_str, ++ struct job_descriptor_header *job) ++{ ++#ifdef CONFIG_MALI_DEBUG ++ dev_dbg(kctx->kbdev->dev, "%s\n", head_str); ++ dev_dbg(kctx->kbdev->dev, ++ "addr = %p\n" ++ "exception_status = %x (Source ID: 0x%x Access: 0x%x Exception: 0x%x)\n" ++ "first_incomplete_task = %x\n" ++ "fault_pointer = %llx\n" ++ "job_descriptor_size = %x\n" ++ "job_type = %x\n" ++ "job_barrier = %x\n" ++ "_reserved_01 = %x\n" ++ "_reserved_02 = %x\n" ++ "_reserved_03 = %x\n" ++ "_reserved_04/05 = %x,%x\n" ++ "job_index = %x\n" ++ "dependencies = %x,%x\n", ++ job, job->exception_status, ++ JOB_SOURCE_ID(job->exception_status), ++ (job->exception_status >> 8) & 0x3, ++ job->exception_status & 0xFF, ++ job->first_incomplete_task, ++ job->fault_pointer, job->job_descriptor_size, ++ job->job_type, job->job_barrier, job->_reserved_01, ++ job->_reserved_02, job->_reserved_03, ++ job->_reserved_04, job->_reserved_05, ++ job->job_index, ++ job->job_dependency_index_1, ++ job->job_dependency_index_2); ++ ++ if (job->job_descriptor_size) ++ dev_dbg(kctx->kbdev->dev, "next = %llx\n", ++ job->next_job._64); ++ else ++ dev_dbg(kctx->kbdev->dev, "next = %x\n", ++ job->next_job._32); ++#endif ++} ++ ++static int kbasep_replay_reset_sfbd(struct kbase_context *kctx, ++ u64 fbd_address, u64 tiler_heap_free, ++ u16 hierarchy_mask, u32 default_weight) ++{ ++ struct { ++ u32 padding_1[1]; ++ u32 flags; ++ u64 padding_2[2]; ++ u64 heap_free_address; ++ u32 padding[8]; ++ u32 weights[FBD_HIERARCHY_WEIGHTS]; ++ } *fbd_tiler; ++ struct kbase_vmap_struct map; ++ ++ dev_dbg(kctx->kbdev->dev, "fbd_address: %llx\n", fbd_address); ++ ++ fbd_tiler = kbase_vmap(kctx, fbd_address + SFBD_TILER_OFFSET, ++ sizeof(*fbd_tiler), &map); ++ if (!fbd_tiler) { ++ dev_err(kctx->kbdev->dev, "kbasep_replay_reset_fbd: failed to map fbd\n"); ++ return -EINVAL; ++ } ++ ++#ifdef CONFIG_MALI_DEBUG ++ dev_dbg(kctx->kbdev->dev, ++ "FBD tiler:\n" ++ "flags = %x\n" ++ "heap_free_address = %llx\n", ++ fbd_tiler->flags, fbd_tiler->heap_free_address); ++#endif ++ if (hierarchy_mask) { ++ u32 weights[HIERARCHY_WEIGHTS]; ++ u16 old_hierarchy_mask = fbd_tiler->flags & ++ FBD_HIERARCHY_MASK_MASK; ++ int i, j = 0; ++ ++ for (i = 0; i < HIERARCHY_WEIGHTS; i++) { ++ if (old_hierarchy_mask & (1 << i)) { ++ KBASE_DEBUG_ASSERT(j < FBD_HIERARCHY_WEIGHTS); ++ weights[i] = fbd_tiler->weights[j++]; ++ } else { ++ weights[i] = default_weight; ++ } ++ } ++ ++ ++ dev_dbg(kctx->kbdev->dev, "Old hierarchy mask=%x New hierarchy mask=%x\n", ++ old_hierarchy_mask, hierarchy_mask); ++ ++ for (i = 0; i < HIERARCHY_WEIGHTS; i++) ++ dev_dbg(kctx->kbdev->dev, " Hierarchy weight %02d: %08x\n", ++ i, weights[i]); ++ ++ j = 0; ++ ++ for (i = 0; i < HIERARCHY_WEIGHTS; i++) { ++ if (hierarchy_mask & (1 << i)) { ++ KBASE_DEBUG_ASSERT(j < FBD_HIERARCHY_WEIGHTS); ++ ++ dev_dbg(kctx->kbdev->dev, " Writing hierarchy level %02d (%08x) to %d\n", ++ i, weights[i], j); ++ ++ fbd_tiler->weights[j++] = weights[i]; ++ } ++ } ++ ++ for (; j < FBD_HIERARCHY_WEIGHTS; j++) ++ fbd_tiler->weights[j] = 0; ++ ++ fbd_tiler->flags = hierarchy_mask | (1 << 16); ++ } ++ ++ fbd_tiler->heap_free_address = tiler_heap_free; ++ ++ dev_dbg(kctx->kbdev->dev, "heap_free_address=%llx flags=%x\n", ++ fbd_tiler->heap_free_address, fbd_tiler->flags); ++ ++ kbase_vunmap(kctx, &map); ++ ++ return 0; ++} ++ ++static int kbasep_replay_reset_mfbd(struct kbase_context *kctx, ++ u64 fbd_address, u64 tiler_heap_free, ++ u16 hierarchy_mask, u32 default_weight) ++{ ++ struct kbase_vmap_struct map; ++ struct { ++ u32 padding_0; ++ u32 flags; ++ u64 padding_1[2]; ++ u64 heap_free_address; ++ u64 padding_2; ++ u32 weights[FBD_HIERARCHY_WEIGHTS]; ++ } *fbd_tiler; ++ ++ dev_dbg(kctx->kbdev->dev, "fbd_address: %llx\n", fbd_address); ++ ++ fbd_tiler = kbase_vmap(kctx, fbd_address + MFBD_TILER_OFFSET, ++ sizeof(*fbd_tiler), &map); ++ if (!fbd_tiler) { ++ dev_err(kctx->kbdev->dev, ++ "kbasep_replay_reset_fbd: failed to map fbd\n"); ++ return -EINVAL; ++ } ++ ++#ifdef CONFIG_MALI_DEBUG ++ dev_dbg(kctx->kbdev->dev, "FBD tiler:\n" ++ "flags = %x\n" ++ "heap_free_address = %llx\n", ++ fbd_tiler->flags, ++ fbd_tiler->heap_free_address); ++#endif ++ if (hierarchy_mask) { ++ u32 weights[HIERARCHY_WEIGHTS]; ++ u16 old_hierarchy_mask = (fbd_tiler->flags) & ++ FBD_HIERARCHY_MASK_MASK; ++ int i, j = 0; ++ ++ for (i = 0; i < HIERARCHY_WEIGHTS; i++) { ++ if (old_hierarchy_mask & (1 << i)) { ++ KBASE_DEBUG_ASSERT(j < FBD_HIERARCHY_WEIGHTS); ++ weights[i] = fbd_tiler->weights[j++]; ++ } else { ++ weights[i] = default_weight; ++ } ++ } ++ ++ ++ dev_dbg(kctx->kbdev->dev, "Old hierarchy mask=%x New hierarchy mask=%x\n", ++ old_hierarchy_mask, hierarchy_mask); ++ ++ for (i = 0; i < HIERARCHY_WEIGHTS; i++) ++ dev_dbg(kctx->kbdev->dev, " Hierarchy weight %02d: %08x\n", ++ i, weights[i]); ++ ++ j = 0; ++ ++ for (i = 0; i < HIERARCHY_WEIGHTS; i++) { ++ if (hierarchy_mask & (1 << i)) { ++ KBASE_DEBUG_ASSERT(j < FBD_HIERARCHY_WEIGHTS); ++ ++ dev_dbg(kctx->kbdev->dev, ++ " Writing hierarchy level %02d (%08x) to %d\n", ++ i, weights[i], j); ++ ++ fbd_tiler->weights[j++] = weights[i]; ++ } ++ } ++ ++ for (; j < FBD_HIERARCHY_WEIGHTS; j++) ++ fbd_tiler->weights[j] = 0; ++ ++ fbd_tiler->flags = hierarchy_mask | (1 << 16); ++ } ++ ++ fbd_tiler->heap_free_address = tiler_heap_free; ++ ++ kbase_vunmap(kctx, &map); ++ ++ return 0; ++} ++ ++/** ++ * @brief Reset the status of an FBD pointed to by a tiler job ++ * ++ * This performs two functions : ++ * - Set the hierarchy mask ++ * - Reset the tiler free heap address ++ * ++ * @param[in] kctx Context pointer ++ * @param[in] job_header Address of job header to reset. ++ * @param[in] tiler_heap_free The value to reset Tiler Heap Free to ++ * @param[in] hierarchy_mask The hierarchy mask to use ++ * @param[in] default_weight Default hierarchy weight to write when no other ++ * weight is given in the FBD ++ * @param[in] job_64 true if this job is using 64-bit ++ * descriptors ++ * ++ * @return 0 on success, error code on failure ++ */ ++static int kbasep_replay_reset_tiler_job(struct kbase_context *kctx, ++ u64 job_header, u64 tiler_heap_free, ++ u16 hierarchy_mask, u32 default_weight, bool job_64) ++{ ++ struct kbase_vmap_struct map; ++ u64 fbd_address; ++ ++ if (job_64) { ++ u64 *job_ext; ++ ++ job_ext = kbase_vmap(kctx, ++ job_header + JOB_HEADER_64_FBD_OFFSET, ++ sizeof(*job_ext), &map); ++ ++ if (!job_ext) { ++ dev_err(kctx->kbdev->dev, "kbasep_replay_reset_tiler_job: failed to map jc\n"); ++ return -EINVAL; ++ } ++ ++ fbd_address = *job_ext; ++ ++ kbase_vunmap(kctx, &map); ++ } else { ++ u32 *job_ext; ++ ++ job_ext = kbase_vmap(kctx, ++ job_header + JOB_HEADER_32_FBD_OFFSET, ++ sizeof(*job_ext), &map); ++ ++ if (!job_ext) { ++ dev_err(kctx->kbdev->dev, "kbasep_replay_reset_tiler_job: failed to map jc\n"); ++ return -EINVAL; ++ } ++ ++ fbd_address = *job_ext; ++ ++ kbase_vunmap(kctx, &map); ++ } ++ ++ if (fbd_address & FBD_TYPE) { ++ return kbasep_replay_reset_mfbd(kctx, ++ fbd_address & FBD_POINTER_MASK, ++ tiler_heap_free, ++ hierarchy_mask, ++ default_weight); ++ } else { ++ return kbasep_replay_reset_sfbd(kctx, ++ fbd_address & FBD_POINTER_MASK, ++ tiler_heap_free, ++ hierarchy_mask, ++ default_weight); ++ } ++} ++ ++/** ++ * @brief Reset the status of a job ++ * ++ * This performs the following functions : ++ * ++ * - Reset the Job Status field of each job to NOT_STARTED. ++ * - Set the Job Type field of any Vertex Jobs to Null Job. ++ * - For any jobs using an FBD, set the Tiler Heap Free field to the value of ++ * the tiler_heap_free parameter, and set the hierarchy level mask to the ++ * hier_mask parameter. ++ * - Offset HW dependencies by the hw_job_id_offset parameter ++ * - Set the Perform Job Barrier flag if this job is the first in the chain ++ * - Read the address of the next job header ++ * ++ * @param[in] kctx Context pointer ++ * @param[in,out] job_header Address of job header to reset. Set to address ++ * of next job header on exit. ++ * @param[in] prev_jc Previous job chain to link to, if this job is ++ * the last in the chain. ++ * @param[in] hw_job_id_offset Offset for HW job IDs ++ * @param[in] tiler_heap_free The value to reset Tiler Heap Free to ++ * @param[in] hierarchy_mask The hierarchy mask to use ++ * @param[in] default_weight Default hierarchy weight to write when no other ++ * weight is given in the FBD ++ * @param[in] first_in_chain true if this job is the first in the chain ++ * @param[in] fragment_chain true if this job is in the fragment chain ++ * ++ * @return 0 on success, error code on failure ++ */ ++static int kbasep_replay_reset_job(struct kbase_context *kctx, ++ u64 *job_header, u64 prev_jc, ++ u64 tiler_heap_free, u16 hierarchy_mask, ++ u32 default_weight, u16 hw_job_id_offset, ++ bool first_in_chain, bool fragment_chain) ++{ ++ struct fragment_job *frag_job; ++ struct job_descriptor_header *job; ++ u64 new_job_header; ++ struct kbase_vmap_struct map; ++ ++ frag_job = kbase_vmap(kctx, *job_header, sizeof(*frag_job), &map); ++ if (!frag_job) { ++ dev_err(kctx->kbdev->dev, ++ "kbasep_replay_parse_jc: failed to map jc\n"); ++ return -EINVAL; ++ } ++ job = &frag_job->header; ++ ++ dump_job_head(kctx, "Job header:", job); ++ ++ if (job->exception_status == JOB_NOT_STARTED && !fragment_chain) { ++ dev_err(kctx->kbdev->dev, "Job already not started\n"); ++ goto out_unmap; ++ } ++ job->exception_status = JOB_NOT_STARTED; ++ ++ if (job->job_type == JOB_TYPE_VERTEX) ++ job->job_type = JOB_TYPE_NULL; ++ ++ if (job->job_type == JOB_TYPE_FUSED) { ++ dev_err(kctx->kbdev->dev, "Fused jobs can not be replayed\n"); ++ goto out_unmap; ++ } ++ ++ if (first_in_chain) ++ job->job_barrier = 1; ++ ++ if ((job->job_dependency_index_1 + hw_job_id_offset) > ++ JOB_HEADER_ID_MAX || ++ (job->job_dependency_index_2 + hw_job_id_offset) > ++ JOB_HEADER_ID_MAX || ++ (job->job_index + hw_job_id_offset) > JOB_HEADER_ID_MAX) { ++ dev_err(kctx->kbdev->dev, ++ "Job indicies/dependencies out of valid range\n"); ++ goto out_unmap; ++ } ++ ++ if (job->job_dependency_index_1) ++ job->job_dependency_index_1 += hw_job_id_offset; ++ if (job->job_dependency_index_2) ++ job->job_dependency_index_2 += hw_job_id_offset; ++ ++ job->job_index += hw_job_id_offset; ++ ++ if (job->job_descriptor_size) { ++ new_job_header = job->next_job._64; ++ if (!job->next_job._64) ++ job->next_job._64 = prev_jc; ++ } else { ++ new_job_header = job->next_job._32; ++ if (!job->next_job._32) ++ job->next_job._32 = prev_jc; ++ } ++ dump_job_head(kctx, "Updated to:", job); ++ ++ if (job->job_type == JOB_TYPE_TILER) { ++ bool job_64 = job->job_descriptor_size != 0; ++ ++ if (kbasep_replay_reset_tiler_job(kctx, *job_header, ++ tiler_heap_free, hierarchy_mask, ++ default_weight, job_64) != 0) ++ goto out_unmap; ++ ++ } else if (job->job_type == JOB_TYPE_FRAGMENT) { ++ u64 fbd_address; ++ ++ if (job->job_descriptor_size) ++ fbd_address = frag_job->fragment_fbd._64; ++ else ++ fbd_address = (u64)frag_job->fragment_fbd._32; ++ ++ if (fbd_address & FBD_TYPE) { ++ if (kbasep_replay_reset_mfbd(kctx, ++ fbd_address & FBD_POINTER_MASK, ++ tiler_heap_free, ++ hierarchy_mask, ++ default_weight) != 0) ++ goto out_unmap; ++ } else { ++ if (kbasep_replay_reset_sfbd(kctx, ++ fbd_address & FBD_POINTER_MASK, ++ tiler_heap_free, ++ hierarchy_mask, ++ default_weight) != 0) ++ goto out_unmap; ++ } ++ } ++ ++ kbase_vunmap(kctx, &map); ++ ++ *job_header = new_job_header; ++ ++ return 0; ++ ++out_unmap: ++ kbase_vunmap(kctx, &map); ++ return -EINVAL; ++} ++ ++/** ++ * @brief Find the highest job ID in a job chain ++ * ++ * @param[in] kctx Context pointer ++ * @param[in] jc Job chain start address ++ * @param[out] hw_job_id Highest job ID in chain ++ * ++ * @return 0 on success, error code on failure ++ */ ++static int kbasep_replay_find_hw_job_id(struct kbase_context *kctx, ++ u64 jc, u16 *hw_job_id) ++{ ++ while (jc) { ++ struct job_descriptor_header *job; ++ struct kbase_vmap_struct map; ++ ++ dev_dbg(kctx->kbdev->dev, ++ "kbasep_replay_find_hw_job_id: parsing jc=%llx\n", jc); ++ ++ job = kbase_vmap(kctx, jc, sizeof(*job), &map); ++ if (!job) { ++ dev_err(kctx->kbdev->dev, "failed to map jc\n"); ++ ++ return -EINVAL; ++ } ++ ++ if (job->job_index > *hw_job_id) ++ *hw_job_id = job->job_index; ++ ++ if (job->job_descriptor_size) ++ jc = job->next_job._64; ++ else ++ jc = job->next_job._32; ++ ++ kbase_vunmap(kctx, &map); ++ } ++ ++ return 0; ++} ++ ++/** ++ * @brief Reset the status of a number of jobs ++ * ++ * This function walks the provided job chain, and calls ++ * kbasep_replay_reset_job for each job. It also links the job chain to the ++ * provided previous job chain. ++ * ++ * The function will fail if any of the jobs passed already have status of ++ * NOT_STARTED. ++ * ++ * @param[in] kctx Context pointer ++ * @param[in] jc Job chain to be processed ++ * @param[in] prev_jc Job chain to be added to. May be NULL ++ * @param[in] tiler_heap_free The value to reset Tiler Heap Free to ++ * @param[in] hierarchy_mask The hierarchy mask to use ++ * @param[in] default_weight Default hierarchy weight to write when no other ++ * weight is given in the FBD ++ * @param[in] hw_job_id_offset Offset for HW job IDs ++ * @param[in] fragment_chain true if this chain is the fragment chain ++ * ++ * @return 0 on success, error code otherwise ++ */ ++static int kbasep_replay_parse_jc(struct kbase_context *kctx, ++ u64 jc, u64 prev_jc, ++ u64 tiler_heap_free, u16 hierarchy_mask, ++ u32 default_weight, u16 hw_job_id_offset, ++ bool fragment_chain) ++{ ++ bool first_in_chain = true; ++ int nr_jobs = 0; ++ ++ dev_dbg(kctx->kbdev->dev, "kbasep_replay_parse_jc: jc=%llx hw_job_id=%x\n", ++ jc, hw_job_id_offset); ++ ++ while (jc) { ++ dev_dbg(kctx->kbdev->dev, "kbasep_replay_parse_jc: parsing jc=%llx\n", jc); ++ ++ if (kbasep_replay_reset_job(kctx, &jc, prev_jc, ++ tiler_heap_free, hierarchy_mask, ++ default_weight, hw_job_id_offset, ++ first_in_chain, fragment_chain) != 0) ++ return -EINVAL; ++ ++ first_in_chain = false; ++ ++ nr_jobs++; ++ if (fragment_chain && ++ nr_jobs >= BASE_JD_REPLAY_F_CHAIN_JOB_LIMIT) { ++ dev_err(kctx->kbdev->dev, ++ "Exceeded maximum number of jobs in fragment chain\n"); ++ return -EINVAL; ++ } ++ } ++ ++ return 0; ++} ++ ++/** ++ * @brief Reset the status of a replay job, and set up dependencies ++ * ++ * This performs the actions to allow the replay job to be re-run following ++ * completion of the passed dependency. ++ * ++ * @param[in] katom The atom to be reset ++ * @param[in] dep_atom The dependency to be attached to the atom ++ */ ++static void kbasep_replay_reset_softjob(struct kbase_jd_atom *katom, ++ struct kbase_jd_atom *dep_atom) ++{ ++ katom->status = KBASE_JD_ATOM_STATE_QUEUED; ++ kbase_jd_katom_dep_set(&katom->dep[0], dep_atom, BASE_JD_DEP_TYPE_DATA); ++ list_add_tail(&katom->dep_item[0], &dep_atom->dep_head[0]); ++} ++ ++/** ++ * @brief Allocate an unused katom ++ * ++ * This will search the provided context for an unused katom, and will mark it ++ * as KBASE_JD_ATOM_STATE_QUEUED. ++ * ++ * If no atoms are available then the function will fail. ++ * ++ * @param[in] kctx Context pointer ++ * @return An atom ID, or -1 on failure ++ */ ++static int kbasep_allocate_katom(struct kbase_context *kctx) ++{ ++ struct kbase_jd_context *jctx = &kctx->jctx; ++ int i; ++ ++ for (i = BASE_JD_ATOM_COUNT-1; i > 0; i--) { ++ if (jctx->atoms[i].status == KBASE_JD_ATOM_STATE_UNUSED) { ++ jctx->atoms[i].status = KBASE_JD_ATOM_STATE_QUEUED; ++ dev_dbg(kctx->kbdev->dev, ++ "kbasep_allocate_katom: Allocated atom %d\n", ++ i); ++ return i; ++ } ++ } ++ ++ return -1; ++} ++ ++/** ++ * @brief Release a katom ++ * ++ * This will mark the provided atom as available, and remove any dependencies. ++ * ++ * For use on error path. ++ * ++ * @param[in] kctx Context pointer ++ * @param[in] atom_id ID of atom to release ++ */ ++static void kbasep_release_katom(struct kbase_context *kctx, int atom_id) ++{ ++ struct kbase_jd_context *jctx = &kctx->jctx; ++ ++ dev_dbg(kctx->kbdev->dev, "kbasep_release_katom: Released atom %d\n", ++ atom_id); ++ ++ while (!list_empty(&jctx->atoms[atom_id].dep_head[0])) ++ list_del(jctx->atoms[atom_id].dep_head[0].next); ++ ++ while (!list_empty(&jctx->atoms[atom_id].dep_head[1])) ++ list_del(jctx->atoms[atom_id].dep_head[1].next); ++ ++ jctx->atoms[atom_id].status = KBASE_JD_ATOM_STATE_UNUSED; ++} ++ ++static void kbasep_replay_create_atom(struct kbase_context *kctx, ++ struct base_jd_atom_v2 *atom, ++ int atom_nr, ++ base_jd_prio prio) ++{ ++ atom->nr_extres = 0; ++ atom->extres_list.value = NULL; ++ atom->device_nr = 0; ++ atom->prio = prio; ++ atom->atom_number = atom_nr; ++ ++ base_jd_atom_dep_set(&atom->pre_dep[0], 0 , BASE_JD_DEP_TYPE_INVALID); ++ base_jd_atom_dep_set(&atom->pre_dep[1], 0 , BASE_JD_DEP_TYPE_INVALID); ++ ++ atom->udata.blob[0] = 0; ++ atom->udata.blob[1] = 0; ++} ++ ++/** ++ * @brief Create two atoms for the purpose of replaying jobs ++ * ++ * Two atoms are allocated and created. The jc pointer is not set at this ++ * stage. The second atom has a dependency on the first. The remaining fields ++ * are set up as follows : ++ * ++ * - No external resources. Any required external resources will be held by the ++ * replay atom. ++ * - device_nr is set to 0. This is not relevant as ++ * BASE_JD_REQ_SPECIFIC_COHERENT_GROUP should not be set. ++ * - Priority is inherited from the replay job. ++ * ++ * @param[out] t_atom Atom to use for tiler jobs ++ * @param[out] f_atom Atom to use for fragment jobs ++ * @param[in] prio Priority of new atom (inherited from replay soft ++ * job) ++ * @return 0 on success, error code on failure ++ */ ++static int kbasep_replay_create_atoms(struct kbase_context *kctx, ++ struct base_jd_atom_v2 *t_atom, ++ struct base_jd_atom_v2 *f_atom, ++ base_jd_prio prio) ++{ ++ int t_atom_nr, f_atom_nr; ++ ++ t_atom_nr = kbasep_allocate_katom(kctx); ++ if (t_atom_nr < 0) { ++ dev_err(kctx->kbdev->dev, "Failed to allocate katom\n"); ++ return -EINVAL; ++ } ++ ++ f_atom_nr = kbasep_allocate_katom(kctx); ++ if (f_atom_nr < 0) { ++ dev_err(kctx->kbdev->dev, "Failed to allocate katom\n"); ++ kbasep_release_katom(kctx, t_atom_nr); ++ return -EINVAL; ++ } ++ ++ kbasep_replay_create_atom(kctx, t_atom, t_atom_nr, prio); ++ kbasep_replay_create_atom(kctx, f_atom, f_atom_nr, prio); ++ ++ base_jd_atom_dep_set(&f_atom->pre_dep[0], t_atom_nr , BASE_JD_DEP_TYPE_DATA); ++ ++ return 0; ++} ++ ++#ifdef CONFIG_MALI_DEBUG ++static void payload_dump(struct kbase_context *kctx, base_jd_replay_payload *payload) ++{ ++ u64 next; ++ ++ dev_dbg(kctx->kbdev->dev, "Tiler jc list :\n"); ++ next = payload->tiler_jc_list; ++ ++ while (next) { ++ struct kbase_vmap_struct map; ++ base_jd_replay_jc *jc_struct; ++ ++ jc_struct = kbase_vmap(kctx, next, sizeof(*jc_struct), &map); ++ ++ if (!jc_struct) ++ return; ++ ++ dev_dbg(kctx->kbdev->dev, "* jc_struct=%p jc=%llx next=%llx\n", ++ jc_struct, jc_struct->jc, jc_struct->next); ++ ++ next = jc_struct->next; ++ ++ kbase_vunmap(kctx, &map); ++ } ++} ++#endif ++ ++/** ++ * @brief Parse a base_jd_replay_payload provided by userspace ++ * ++ * This will read the payload from userspace, and parse the job chains. ++ * ++ * @param[in] kctx Context pointer ++ * @param[in] replay_atom Replay soft job atom ++ * @param[in] t_atom Atom to use for tiler jobs ++ * @param[in] f_atom Atom to use for fragment jobs ++ * @return 0 on success, error code on failure ++ */ ++static int kbasep_replay_parse_payload(struct kbase_context *kctx, ++ struct kbase_jd_atom *replay_atom, ++ struct base_jd_atom_v2 *t_atom, ++ struct base_jd_atom_v2 *f_atom) ++{ ++ base_jd_replay_payload *payload = NULL; ++ u64 next; ++ u64 prev_jc = 0; ++ u16 hw_job_id_offset = 0; ++ int ret = -EINVAL; ++ struct kbase_vmap_struct map; ++ ++ dev_dbg(kctx->kbdev->dev, "kbasep_replay_parse_payload: replay_atom->jc = %llx sizeof(payload) = %zu\n", ++ replay_atom->jc, sizeof(payload)); ++ ++ payload = kbase_vmap(kctx, replay_atom->jc, sizeof(*payload), &map); ++ if (!payload) { ++ dev_err(kctx->kbdev->dev, "kbasep_replay_parse_payload: failed to map payload into kernel space\n"); ++ return -EINVAL; ++ } ++ ++#ifdef BASE_LEGACY_UK10_2_SUPPORT ++ if (KBASE_API_VERSION(10, 3) > replay_atom->kctx->api_version) { ++ base_jd_replay_payload_uk10_2 *payload_uk10_2; ++ u16 tiler_core_req; ++ u16 fragment_core_req; ++ ++ payload_uk10_2 = (base_jd_replay_payload_uk10_2 *) payload; ++ memcpy(&tiler_core_req, &payload_uk10_2->tiler_core_req, ++ sizeof(tiler_core_req)); ++ memcpy(&fragment_core_req, &payload_uk10_2->fragment_core_req, ++ sizeof(fragment_core_req)); ++ payload->tiler_core_req = (u32)(tiler_core_req & 0x7fff); ++ payload->fragment_core_req = (u32)(fragment_core_req & 0x7fff); ++ } ++#endif /* BASE_LEGACY_UK10_2_SUPPORT */ ++ ++#ifdef CONFIG_MALI_DEBUG ++ dev_dbg(kctx->kbdev->dev, "kbasep_replay_parse_payload: payload=%p\n", payload); ++ dev_dbg(kctx->kbdev->dev, "Payload structure:\n" ++ "tiler_jc_list = %llx\n" ++ "fragment_jc = %llx\n" ++ "tiler_heap_free = %llx\n" ++ "fragment_hierarchy_mask = %x\n" ++ "tiler_hierarchy_mask = %x\n" ++ "hierarchy_default_weight = %x\n" ++ "tiler_core_req = %x\n" ++ "fragment_core_req = %x\n", ++ payload->tiler_jc_list, ++ payload->fragment_jc, ++ payload->tiler_heap_free, ++ payload->fragment_hierarchy_mask, ++ payload->tiler_hierarchy_mask, ++ payload->hierarchy_default_weight, ++ payload->tiler_core_req, ++ payload->fragment_core_req); ++ payload_dump(kctx, payload); ++#endif ++ t_atom->core_req = payload->tiler_core_req | BASEP_JD_REQ_EVENT_NEVER; ++ f_atom->core_req = payload->fragment_core_req | BASEP_JD_REQ_EVENT_NEVER; ++ ++ /* Sanity check core requirements*/ ++ if ((t_atom->core_req & BASE_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_T || ++ (f_atom->core_req & BASE_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_FS || ++ t_atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES || ++ f_atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) { ++ ++ int t_atom_type = t_atom->core_req & BASE_JD_REQ_ATOM_TYPE & ~BASE_JD_REQ_COHERENT_GROUP; ++ int f_atom_type = f_atom->core_req & BASE_JD_REQ_ATOM_TYPE & ~BASE_JD_REQ_COHERENT_GROUP & ~BASE_JD_REQ_FS_AFBC; ++ int t_has_ex_res = t_atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES; ++ int f_has_ex_res = f_atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES; ++ ++ if (t_atom_type != BASE_JD_REQ_T) { ++ dev_err(kctx->kbdev->dev, "Invalid core requirement: Tiler atom not a tiler job. Was: 0x%x\n Expected: 0x%x", ++ t_atom_type, BASE_JD_REQ_T); ++ } ++ if (f_atom_type != BASE_JD_REQ_FS) { ++ dev_err(kctx->kbdev->dev, "Invalid core requirement: Fragment shader atom not a fragment shader. Was 0x%x Expected: 0x%x\n", ++ f_atom_type, BASE_JD_REQ_FS); ++ } ++ if (t_has_ex_res) { ++ dev_err(kctx->kbdev->dev, "Invalid core requirement: Tiler atom has external resources.\n"); ++ } ++ if (f_has_ex_res) { ++ dev_err(kctx->kbdev->dev, "Invalid core requirement: Fragment shader atom has external resources.\n"); ++ } ++ ++ goto out; ++ } ++ ++ /* Process tiler job chains */ ++ next = payload->tiler_jc_list; ++ if (!next) { ++ dev_err(kctx->kbdev->dev, "Invalid tiler JC list\n"); ++ goto out; ++ } ++ ++ while (next) { ++ base_jd_replay_jc *jc_struct; ++ struct kbase_vmap_struct jc_map; ++ u64 jc; ++ ++ jc_struct = kbase_vmap(kctx, next, sizeof(*jc_struct), &jc_map); ++ ++ if (!jc_struct) { ++ dev_err(kctx->kbdev->dev, "Failed to map jc struct\n"); ++ goto out; ++ } ++ ++ jc = jc_struct->jc; ++ next = jc_struct->next; ++ if (next) ++ jc_struct->jc = 0; ++ ++ kbase_vunmap(kctx, &jc_map); ++ ++ if (jc) { ++ u16 max_hw_job_id = 0; ++ ++ if (kbasep_replay_find_hw_job_id(kctx, jc, ++ &max_hw_job_id) != 0) ++ goto out; ++ ++ if (kbasep_replay_parse_jc(kctx, jc, prev_jc, ++ payload->tiler_heap_free, ++ payload->tiler_hierarchy_mask, ++ payload->hierarchy_default_weight, ++ hw_job_id_offset, false) != 0) { ++ goto out; ++ } ++ ++ hw_job_id_offset += max_hw_job_id; ++ ++ prev_jc = jc; ++ } ++ } ++ t_atom->jc = prev_jc; ++ ++ /* Process fragment job chain */ ++ f_atom->jc = payload->fragment_jc; ++ if (kbasep_replay_parse_jc(kctx, payload->fragment_jc, 0, ++ payload->tiler_heap_free, ++ payload->fragment_hierarchy_mask, ++ payload->hierarchy_default_weight, 0, ++ true) != 0) { ++ goto out; ++ } ++ ++ if (!t_atom->jc || !f_atom->jc) { ++ dev_err(kctx->kbdev->dev, "Invalid payload\n"); ++ goto out; ++ } ++ ++ dev_dbg(kctx->kbdev->dev, "t_atom->jc=%llx f_atom->jc=%llx\n", ++ t_atom->jc, f_atom->jc); ++ ret = 0; ++ ++out: ++ kbase_vunmap(kctx, &map); ++ ++ return ret; ++} ++ ++static void kbase_replay_process_worker(struct work_struct *data) ++{ ++ struct kbase_jd_atom *katom; ++ struct kbase_context *kctx; ++ struct kbase_jd_context *jctx; ++ bool need_to_try_schedule_context = false; ++ ++ struct base_jd_atom_v2 t_atom, f_atom; ++ struct kbase_jd_atom *t_katom, *f_katom; ++ base_jd_prio atom_prio; ++ ++ katom = container_of(data, struct kbase_jd_atom, work); ++ kctx = katom->kctx; ++ jctx = &kctx->jctx; ++ ++ mutex_lock(&jctx->lock); ++ ++ atom_prio = kbasep_js_sched_prio_to_atom_prio(katom->sched_priority); ++ ++ if (kbasep_replay_create_atoms( ++ kctx, &t_atom, &f_atom, atom_prio) != 0) { ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ goto out; ++ } ++ ++ t_katom = &jctx->atoms[t_atom.atom_number]; ++ f_katom = &jctx->atoms[f_atom.atom_number]; ++ ++ if (kbasep_replay_parse_payload(kctx, katom, &t_atom, &f_atom) != 0) { ++ kbasep_release_katom(kctx, t_atom.atom_number); ++ kbasep_release_katom(kctx, f_atom.atom_number); ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ goto out; ++ } ++ ++ kbasep_replay_reset_softjob(katom, f_katom); ++ ++ need_to_try_schedule_context |= jd_submit_atom(kctx, &t_atom, t_katom); ++ if (t_katom->event_code == BASE_JD_EVENT_JOB_INVALID) { ++ dev_err(kctx->kbdev->dev, "Replay failed to submit atom\n"); ++ kbasep_release_katom(kctx, f_atom.atom_number); ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ goto out; ++ } ++ need_to_try_schedule_context |= jd_submit_atom(kctx, &f_atom, f_katom); ++ if (f_katom->event_code == BASE_JD_EVENT_JOB_INVALID) { ++ dev_err(kctx->kbdev->dev, "Replay failed to submit atom\n"); ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ goto out; ++ } ++ ++ katom->event_code = BASE_JD_EVENT_DONE; ++ ++out: ++ if (katom->event_code != BASE_JD_EVENT_DONE) { ++ kbase_disjoint_state_down(kctx->kbdev); ++ ++ need_to_try_schedule_context |= jd_done_nolock(katom, NULL); ++ } ++ ++ if (need_to_try_schedule_context) ++ kbase_js_sched_all(kctx->kbdev); ++ ++ mutex_unlock(&jctx->lock); ++} ++ ++/** ++ * @brief Check job replay fault ++ * ++ * This will read the job payload, checks fault type and source, then decides ++ * whether replay is required. ++ * ++ * @param[in] katom The atom to be processed ++ * @return true (success) if replay required or false on failure. ++ */ ++static bool kbase_replay_fault_check(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ struct device *dev = kctx->kbdev->dev; ++ base_jd_replay_payload *payload; ++ u64 job_header; ++ u64 job_loop_detect; ++ struct job_descriptor_header *job; ++ struct kbase_vmap_struct job_map; ++ struct kbase_vmap_struct map; ++ bool err = false; ++ ++ /* Replay job if fault is of type BASE_JD_EVENT_JOB_WRITE_FAULT or ++ * if force_replay is enabled. ++ */ ++ if (BASE_JD_EVENT_TERMINATED == katom->event_code) { ++ return false; ++ } else if (BASE_JD_EVENT_JOB_WRITE_FAULT == katom->event_code) { ++ return true; ++ } else if (BASE_JD_EVENT_FORCE_REPLAY == katom->event_code) { ++ katom->event_code = BASE_JD_EVENT_DATA_INVALID_FAULT; ++ return true; ++ } else if (BASE_JD_EVENT_DATA_INVALID_FAULT != katom->event_code) { ++ /* No replay for faults of type other than ++ * BASE_JD_EVENT_DATA_INVALID_FAULT. ++ */ ++ return false; ++ } ++ ++ /* Job fault is BASE_JD_EVENT_DATA_INVALID_FAULT, now scan fragment jc ++ * to find out whether the source of exception is POLYGON_LIST. Replay ++ * is required if the source of fault is POLYGON_LIST. ++ */ ++ payload = kbase_vmap(kctx, katom->jc, sizeof(*payload), &map); ++ if (!payload) { ++ dev_err(dev, "kbase_replay_fault_check: failed to map payload.\n"); ++ return false; ++ } ++ ++#ifdef CONFIG_MALI_DEBUG ++ dev_dbg(dev, "kbase_replay_fault_check: payload=%p\n", payload); ++ dev_dbg(dev, "\nPayload structure:\n" ++ "fragment_jc = 0x%llx\n" ++ "fragment_hierarchy_mask = 0x%x\n" ++ "fragment_core_req = 0x%x\n", ++ payload->fragment_jc, ++ payload->fragment_hierarchy_mask, ++ payload->fragment_core_req); ++#endif ++ /* Process fragment job chain */ ++ job_header = (u64) payload->fragment_jc; ++ job_loop_detect = job_header; ++ while (job_header) { ++ job = kbase_vmap(kctx, job_header, sizeof(*job), &job_map); ++ if (!job) { ++ dev_err(dev, "failed to map jc\n"); ++ /* unmap payload*/ ++ kbase_vunmap(kctx, &map); ++ return false; ++ } ++ ++ ++ dump_job_head(kctx, "\njob_head structure:\n", job); ++ ++ /* Replay only when the polygon list reader caused the ++ * DATA_INVALID_FAULT */ ++ if ((BASE_JD_EVENT_DATA_INVALID_FAULT == katom->event_code) && ++ (JOB_POLYGON_LIST == JOB_SOURCE_ID(job->exception_status))) { ++ err = true; ++ kbase_vunmap(kctx, &job_map); ++ break; ++ } ++ ++ /* Move on to next fragment job in the list */ ++ if (job->job_descriptor_size) ++ job_header = job->next_job._64; ++ else ++ job_header = job->next_job._32; ++ ++ kbase_vunmap(kctx, &job_map); ++ ++ /* Job chain loop detected */ ++ if (job_header == job_loop_detect) ++ break; ++ } ++ ++ /* unmap payload*/ ++ kbase_vunmap(kctx, &map); ++ ++ return err; ++} ++ ++ ++/** ++ * @brief Process a replay job ++ * ++ * Called from kbase_process_soft_job. ++ * ++ * On exit, if the job has completed, katom->event_code will have been updated. ++ * If the job has not completed, and is replaying jobs, then the atom status ++ * will have been reset to KBASE_JD_ATOM_STATE_QUEUED. ++ * ++ * @param[in] katom The atom to be processed ++ * @return false if the atom has completed ++ * true if the atom is replaying jobs ++ */ ++bool kbase_replay_process(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ struct kbase_device *kbdev = kctx->kbdev; ++ ++ /* Don't replay this atom if these issues are not present in the ++ * hardware */ ++ if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_11020) && ++ !kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_11024)) { ++ dev_dbg(kbdev->dev, "Hardware does not need replay workaround"); ++ ++ /* Signal failure to userspace */ ++ katom->event_code = BASE_JD_EVENT_JOB_INVALID; ++ ++ return false; ++ } ++ ++ if (katom->event_code == BASE_JD_EVENT_DONE) { ++ dev_dbg(kbdev->dev, "Previous job succeeded - not replaying\n"); ++ ++ if (katom->retry_count) ++ kbase_disjoint_state_down(kbdev); ++ ++ return false; ++ } ++ ++ if (kbase_ctx_flag(kctx, KCTX_DYING)) { ++ dev_dbg(kbdev->dev, "Not replaying; context is dying\n"); ++ ++ if (katom->retry_count) ++ kbase_disjoint_state_down(kbdev); ++ ++ return false; ++ } ++ ++ /* Check job exception type and source before replaying. */ ++ if (!kbase_replay_fault_check(katom)) { ++ dev_dbg(kbdev->dev, ++ "Replay cancelled on event %x\n", katom->event_code); ++ /* katom->event_code is already set to the failure code of the ++ * previous job. ++ */ ++ return false; ++ } ++ ++ dev_warn(kbdev->dev, "Replaying jobs retry=%d\n", ++ katom->retry_count); ++ ++ katom->retry_count++; ++ ++ if (katom->retry_count > BASEP_JD_REPLAY_LIMIT) { ++ dev_err(kbdev->dev, "Replay exceeded limit - failing jobs\n"); ++ ++ kbase_disjoint_state_down(kbdev); ++ ++ /* katom->event_code is already set to the failure code of the ++ previous job */ ++ return false; ++ } ++ ++ /* only enter the disjoint state once for the whole time while the replay is ongoing */ ++ if (katom->retry_count == 1) ++ kbase_disjoint_state_up(kbdev); ++ ++ INIT_WORK(&katom->work, kbase_replay_process_worker); ++ queue_work(kctx->event_workq, &katom->work); ++ ++ return true; ++} +diff --git a/drivers/gpu/arm/midgard/mali_kbase_smc.c b/drivers/gpu/arm/midgard/mali_kbase_smc.c +new file mode 100755 +index 000000000000..6c8cf73ae58c +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_smc.c +@@ -0,0 +1,86 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifdef CONFIG_ARM64 ++ ++#include ++#include ++ ++#include ++ ++/* __asmeq is not available on Kernel versions >= 4.20 */ ++#ifndef __asmeq ++/* ++ * This is used to ensure the compiler did actually allocate the register we ++ * asked it for some inline assembly sequences. Apparently we can't trust the ++ * compiler from one version to another so a bit of paranoia won't hurt. This ++ * string is meant to be concatenated with the inline asm string and will ++ * cause compilation to stop on mismatch. (for details, see gcc PR 15089) ++ */ ++#define __asmeq(x, y) ".ifnc " x "," y " ; .err ; .endif\n\t" ++#endif ++ ++static noinline u64 invoke_smc_fid(u64 function_id, ++ u64 arg0, u64 arg1, u64 arg2) ++{ ++ register u64 x0 asm("x0") = function_id; ++ register u64 x1 asm("x1") = arg0; ++ register u64 x2 asm("x2") = arg1; ++ register u64 x3 asm("x3") = arg2; ++ ++ asm volatile( ++ __asmeq("%0", "x0") ++ __asmeq("%1", "x1") ++ __asmeq("%2", "x2") ++ __asmeq("%3", "x3") ++ "smc #0\n" ++ : "+r" (x0) ++ : "r" (x1), "r" (x2), "r" (x3)); ++ ++ return x0; ++} ++ ++u64 kbase_invoke_smc_fid(u32 fid, u64 arg0, u64 arg1, u64 arg2) ++{ ++ /* Is fast call (bit 31 set) */ ++ KBASE_DEBUG_ASSERT(fid & ~SMC_FAST_CALL); ++ /* bits 16-23 must be zero for fast calls */ ++ KBASE_DEBUG_ASSERT((fid & (0xFF << 16)) == 0); ++ ++ return invoke_smc_fid(fid, arg0, arg1, arg2); ++} ++ ++u64 kbase_invoke_smc(u32 oen, u16 function_number, bool smc64, ++ u64 arg0, u64 arg1, u64 arg2) ++{ ++ u32 fid = 0; ++ ++ /* Only the six bits allowed should be used. */ ++ KBASE_DEBUG_ASSERT((oen & ~SMC_OEN_MASK) == 0); ++ ++ fid |= SMC_FAST_CALL; /* Bit 31: Fast call */ ++ if (smc64) ++ fid |= SMC_64; /* Bit 30: 1=SMC64, 0=SMC32 */ ++ fid |= oen; /* Bit 29:24: OEN */ ++ /* Bit 23:16: Must be zero for fast calls */ ++ fid |= (function_number); /* Bit 15:0: function number */ ++ ++ return kbase_invoke_smc_fid(fid, arg0, arg1, arg2); ++} ++ ++#endif /* CONFIG_ARM64 */ ++ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_smc.h b/drivers/gpu/arm/midgard/mali_kbase_smc.h +new file mode 100755 +index 000000000000..9bff3d2e8b4d +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_smc.h +@@ -0,0 +1,67 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#ifndef _KBASE_SMC_H_ ++#define _KBASE_SMC_H_ ++ ++#ifdef CONFIG_ARM64 ++ ++#include ++ ++#define SMC_FAST_CALL (1 << 31) ++#define SMC_64 (1 << 30) ++ ++#define SMC_OEN_OFFSET 24 ++#define SMC_OEN_MASK (0x3F << SMC_OEN_OFFSET) /* 6 bits */ ++#define SMC_OEN_SIP (2 << SMC_OEN_OFFSET) ++#define SMC_OEN_STD (4 << SMC_OEN_OFFSET) ++ ++ ++/** ++ * kbase_invoke_smc_fid - Perform a secure monitor call ++ * @fid: The SMC function to call, see SMC Calling convention. ++ * @arg0: First argument to the SMC. ++ * @arg1: Second argument to the SMC. ++ * @arg2: Third argument to the SMC. ++ * ++ * See SMC Calling Convention for details. ++ * ++ * Return: the return value from the SMC. ++ */ ++u64 kbase_invoke_smc_fid(u32 fid, u64 arg0, u64 arg1, u64 arg2); ++ ++/** ++ * kbase_invoke_smc_fid - Perform a secure monitor call ++ * @oen: Owning Entity number (SIP, STD etc). ++ * @function_number: The function number within the OEN. ++ * @smc64: use SMC64 calling convention instead of SMC32. ++ * @arg0: First argument to the SMC. ++ * @arg1: Second argument to the SMC. ++ * @arg2: Third argument to the SMC. ++ * ++ * See SMC Calling Convention for details. ++ * ++ * Return: the return value from the SMC call. ++ */ ++u64 kbase_invoke_smc(u32 oen, u16 function_number, bool smc64, ++ u64 arg0, u64 arg1, u64 arg2); ++ ++#endif /* CONFIG_ARM64 */ ++ ++#endif /* _KBASE_SMC_H_ */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_softjobs.c b/drivers/gpu/arm/midgard/mali_kbase_softjobs.c +new file mode 100755 +index 000000000000..396953e780a8 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_softjobs.c +@@ -0,0 +1,1549 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#include ++ ++#if defined(CONFIG_DMA_SHARED_BUFFER) ++#include ++#include ++#endif /* defined(CONFIG_DMA_SHARED_BUFFER) */ ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++#include ++#endif ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++/* Mask to check cache alignment of data structures */ ++#define KBASE_CACHE_ALIGNMENT_MASK ((1<kctx; ++ unsigned long lflags; ++ ++ spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); ++ list_add_tail(&katom->queue, &kctx->waiting_soft_jobs); ++ spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); ++} ++ ++void kbasep_remove_waiting_soft_job(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ unsigned long lflags; ++ ++ spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); ++ list_del(&katom->queue); ++ spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); ++} ++ ++static void kbasep_add_waiting_with_timeout(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ ++ /* Record the start time of this atom so we could cancel it at ++ * the right time. ++ */ ++ katom->start_timestamp = ktime_get(); ++ ++ /* Add the atom to the waiting list before the timer is ++ * (re)started to make sure that it gets processed. ++ */ ++ kbasep_add_waiting_soft_job(katom); ++ ++ /* Schedule timeout of this atom after a period if it is not active */ ++ if (!timer_pending(&kctx->soft_job_timeout)) { ++ int timeout_ms = atomic_read( ++ &kctx->kbdev->js_data.soft_job_timeout_ms); ++ mod_timer(&kctx->soft_job_timeout, ++ jiffies + msecs_to_jiffies(timeout_ms)); ++ } ++} ++ ++static int kbasep_read_soft_event_status( ++ struct kbase_context *kctx, u64 evt, unsigned char *status) ++{ ++ unsigned char *mapped_evt; ++ struct kbase_vmap_struct map; ++ ++ mapped_evt = kbase_vmap(kctx, evt, sizeof(*mapped_evt), &map); ++ if (!mapped_evt) ++ return -EFAULT; ++ ++ *status = *mapped_evt; ++ ++ kbase_vunmap(kctx, &map); ++ ++ return 0; ++} ++ ++static int kbasep_write_soft_event_status( ++ struct kbase_context *kctx, u64 evt, unsigned char new_status) ++{ ++ unsigned char *mapped_evt; ++ struct kbase_vmap_struct map; ++ ++ if ((new_status != BASE_JD_SOFT_EVENT_SET) && ++ (new_status != BASE_JD_SOFT_EVENT_RESET)) ++ return -EINVAL; ++ ++ mapped_evt = kbase_vmap(kctx, evt, sizeof(*mapped_evt), &map); ++ if (!mapped_evt) ++ return -EFAULT; ++ ++ *mapped_evt = new_status; ++ ++ kbase_vunmap(kctx, &map); ++ ++ return 0; ++} ++ ++static int kbase_dump_cpu_gpu_time(struct kbase_jd_atom *katom) ++{ ++ struct kbase_vmap_struct map; ++ void *user_result; ++ struct timespec64 ts; ++ struct base_dump_cpu_gpu_counters data; ++ u64 system_time; ++ u64 cycle_counter; ++ u64 jc = katom->jc; ++ struct kbase_context *kctx = katom->kctx; ++ int pm_active_err; ++ ++ memset(&data, 0, sizeof(data)); ++ ++ /* Take the PM active reference as late as possible - otherwise, it could ++ * delay suspend until we process the atom (which may be at the end of a ++ * long chain of dependencies */ ++ pm_active_err = kbase_pm_context_active_handle_suspend(kctx->kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE); ++ if (pm_active_err) { ++ struct kbasep_js_device_data *js_devdata = &kctx->kbdev->js_data; ++ ++ /* We're suspended - queue this on the list of suspended jobs ++ * Use dep_item[1], because dep_item[0] was previously in use ++ * for 'waiting_soft_jobs'. ++ */ ++ mutex_lock(&js_devdata->runpool_mutex); ++ list_add_tail(&katom->dep_item[1], &js_devdata->suspended_soft_jobs_list); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ ++ /* Also adding this to the list of waiting soft job */ ++ kbasep_add_waiting_soft_job(katom); ++ ++ return pm_active_err; ++ } ++ ++ kbase_backend_get_gpu_time(kctx->kbdev, &cycle_counter, &system_time, ++ &ts); ++ ++ kbase_pm_context_idle(kctx->kbdev); ++ ++ data.sec = ts.tv_sec; ++ data.usec = ts.tv_nsec / 1000; ++ data.system_time = system_time; ++ data.cycle_counter = cycle_counter; ++ ++ /* Assume this atom will be cancelled until we know otherwise */ ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ ++ /* GPU_WR access is checked on the range for returning the result to ++ * userspace for the following reasons: ++ * - security, this is currently how imported user bufs are checked. ++ * - userspace ddk guaranteed to assume region was mapped as GPU_WR */ ++ user_result = kbase_vmap_prot(kctx, jc, sizeof(data), KBASE_REG_GPU_WR, &map); ++ if (!user_result) ++ return 0; ++ ++ memcpy(user_result, &data, sizeof(data)); ++ ++ kbase_vunmap(kctx, &map); ++ ++ /* Atom was fine - mark it as done */ ++ katom->event_code = BASE_JD_EVENT_DONE; ++ ++ return 0; ++} ++ ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++/* Called by the explicit fence mechanism when a fence wait has completed */ ++void kbase_soft_event_wait_callback(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ ++ mutex_lock(&kctx->jctx.lock); ++ kbasep_remove_waiting_soft_job(katom); ++ kbase_finish_soft_job(katom); ++ if (jd_done_nolock(katom, NULL)) ++ kbase_js_sched_all(kctx->kbdev); ++ mutex_unlock(&kctx->jctx.lock); ++} ++#endif ++ ++static void kbasep_soft_event_complete_job(struct work_struct *work) ++{ ++ struct kbase_jd_atom *katom = container_of(work, struct kbase_jd_atom, ++ work); ++ struct kbase_context *kctx = katom->kctx; ++ int resched; ++ ++ mutex_lock(&kctx->jctx.lock); ++ resched = jd_done_nolock(katom, NULL); ++ mutex_unlock(&kctx->jctx.lock); ++ ++ if (resched) ++ kbase_js_sched_all(kctx->kbdev); ++} ++ ++void kbasep_complete_triggered_soft_events(struct kbase_context *kctx, u64 evt) ++{ ++ int cancel_timer = 1; ++ struct list_head *entry, *tmp; ++ unsigned long lflags; ++ ++ spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); ++ list_for_each_safe(entry, tmp, &kctx->waiting_soft_jobs) { ++ struct kbase_jd_atom *katom = list_entry( ++ entry, struct kbase_jd_atom, queue); ++ ++ switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { ++ case BASE_JD_REQ_SOFT_EVENT_WAIT: ++ if (katom->jc == evt) { ++ list_del(&katom->queue); ++ ++ katom->event_code = BASE_JD_EVENT_DONE; ++ INIT_WORK(&katom->work, ++ kbasep_soft_event_complete_job); ++ queue_work(kctx->jctx.job_done_wq, ++ &katom->work); ++ } else { ++ /* There are still other waiting jobs, we cannot ++ * cancel the timer yet. ++ */ ++ cancel_timer = 0; ++ } ++ break; ++#ifdef CONFIG_MALI_FENCE_DEBUG ++ case BASE_JD_REQ_SOFT_FENCE_WAIT: ++ /* Keep the timer running if fence debug is enabled and ++ * there are waiting fence jobs. ++ */ ++ cancel_timer = 0; ++ break; ++#endif ++ } ++ } ++ ++ if (cancel_timer) ++ del_timer(&kctx->soft_job_timeout); ++ spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); ++} ++ ++#ifdef CONFIG_MALI_FENCE_DEBUG ++static void kbase_fence_debug_check_atom(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ struct device *dev = kctx->kbdev->dev; ++ int i; ++ ++ for (i = 0; i < 2; i++) { ++ struct kbase_jd_atom *dep; ++ ++ list_for_each_entry(dep, &katom->dep_head[i], dep_item[i]) { ++ if (dep->status == KBASE_JD_ATOM_STATE_UNUSED || ++ dep->status == KBASE_JD_ATOM_STATE_COMPLETED) ++ continue; ++ ++ if ((dep->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) ++ == BASE_JD_REQ_SOFT_FENCE_TRIGGER) { ++ /* Found blocked trigger fence. */ ++ struct kbase_sync_fence_info info; ++ ++ if (!kbase_sync_fence_in_info_get(dep, &info)) { ++ dev_warn(dev, ++ "\tVictim trigger atom %d fence [%p] %s: %s\n", ++ kbase_jd_atom_id(kctx, dep), ++ info.fence, ++ info.name, ++ kbase_sync_status_string(info.status)); ++ } ++ } ++ ++ kbase_fence_debug_check_atom(dep); ++ } ++ } ++} ++ ++static void kbase_fence_debug_wait_timeout(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ struct device *dev = katom->kctx->kbdev->dev; ++ int timeout_ms = atomic_read(&kctx->kbdev->js_data.soft_job_timeout_ms); ++ unsigned long lflags; ++ struct kbase_sync_fence_info info; ++ ++ spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); ++ ++ if (kbase_sync_fence_in_info_get(katom, &info)) { ++ /* Fence must have signaled just after timeout. */ ++ spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); ++ return; ++ } ++ ++ dev_warn(dev, "ctx %d_%d: Atom %d still waiting for fence [%p] after %dms\n", ++ kctx->tgid, kctx->id, ++ kbase_jd_atom_id(kctx, katom), ++ info.fence, timeout_ms); ++ dev_warn(dev, "\tGuilty fence [%p] %s: %s\n", ++ info.fence, info.name, ++ kbase_sync_status_string(info.status)); ++ ++ /* Search for blocked trigger atoms */ ++ kbase_fence_debug_check_atom(katom); ++ ++ spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); ++ ++ kbase_sync_fence_in_dump(katom); ++} ++ ++struct kbase_fence_debug_work { ++ struct kbase_jd_atom *katom; ++ struct work_struct work; ++}; ++ ++static void kbase_fence_debug_wait_timeout_worker(struct work_struct *work) ++{ ++ struct kbase_fence_debug_work *w = container_of(work, ++ struct kbase_fence_debug_work, work); ++ struct kbase_jd_atom *katom = w->katom; ++ struct kbase_context *kctx = katom->kctx; ++ ++ mutex_lock(&kctx->jctx.lock); ++ kbase_fence_debug_wait_timeout(katom); ++ mutex_unlock(&kctx->jctx.lock); ++ ++ kfree(w); ++} ++ ++static void kbase_fence_debug_timeout(struct kbase_jd_atom *katom) ++{ ++ struct kbase_fence_debug_work *work; ++ struct kbase_context *kctx = katom->kctx; ++ ++ /* Enqueue fence debug worker. Use job_done_wq to get ++ * debug print ordered with job completion. ++ */ ++ work = kzalloc(sizeof(struct kbase_fence_debug_work), GFP_ATOMIC); ++ /* Ignore allocation failure. */ ++ if (work) { ++ work->katom = katom; ++ INIT_WORK(&work->work, kbase_fence_debug_wait_timeout_worker); ++ queue_work(kctx->jctx.job_done_wq, &work->work); ++ } ++} ++#endif /* CONFIG_MALI_FENCE_DEBUG */ ++ ++void kbasep_soft_job_timeout_worker(struct timer_list *t) ++{ ++ struct kbase_context *kctx = from_timer(kctx, t, soft_job_timeout); ++ u32 timeout_ms = (u32)atomic_read( ++ &kctx->kbdev->js_data.soft_job_timeout_ms); ++ struct timer_list *timer = &kctx->soft_job_timeout; ++ ktime_t cur_time = ktime_get(); ++ bool restarting = false; ++ unsigned long lflags; ++ struct list_head *entry, *tmp; ++ ++ spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); ++ list_for_each_safe(entry, tmp, &kctx->waiting_soft_jobs) { ++ struct kbase_jd_atom *katom = list_entry(entry, ++ struct kbase_jd_atom, queue); ++ s64 elapsed_time = ktime_to_ms(ktime_sub(cur_time, ++ katom->start_timestamp)); ++ ++ if (elapsed_time < (s64)timeout_ms) { ++ restarting = true; ++ continue; ++ } ++ ++ switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { ++ case BASE_JD_REQ_SOFT_EVENT_WAIT: ++ /* Take it out of the list to ensure that it ++ * will be cancelled in all cases ++ */ ++ list_del(&katom->queue); ++ ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ INIT_WORK(&katom->work, kbasep_soft_event_complete_job); ++ queue_work(kctx->jctx.job_done_wq, &katom->work); ++ break; ++#ifdef CONFIG_MALI_FENCE_DEBUG ++ case BASE_JD_REQ_SOFT_FENCE_WAIT: ++ kbase_fence_debug_timeout(katom); ++ break; ++#endif ++ } ++ } ++ ++ if (restarting) ++ mod_timer(timer, jiffies + msecs_to_jiffies(timeout_ms)); ++ spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); ++} ++ ++static int kbasep_soft_event_wait(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ unsigned char status; ++ ++ /* The status of this soft-job is stored in jc */ ++ if (kbasep_read_soft_event_status(kctx, katom->jc, &status)) { ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ return 0; ++ } ++ ++ if (status == BASE_JD_SOFT_EVENT_SET) ++ return 0; /* Event already set, nothing to do */ ++ ++ kbasep_add_waiting_with_timeout(katom); ++ ++ return 1; ++} ++ ++static void kbasep_soft_event_update_locked(struct kbase_jd_atom *katom, ++ unsigned char new_status) ++{ ++ /* Complete jobs waiting on the same event */ ++ struct kbase_context *kctx = katom->kctx; ++ ++ if (kbasep_write_soft_event_status(kctx, katom->jc, new_status) != 0) { ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ return; ++ } ++ ++ if (new_status == BASE_JD_SOFT_EVENT_SET) ++ kbasep_complete_triggered_soft_events(kctx, katom->jc); ++} ++ ++/** ++ * kbase_soft_event_update() - Update soft event state ++ * @kctx: Pointer to context ++ * @event: Event to update ++ * @new_status: New status value of event ++ * ++ * Update the event, and wake up any atoms waiting for the event. ++ * ++ * Return: 0 on success, a negative error code on failure. ++ */ ++int kbase_soft_event_update(struct kbase_context *kctx, ++ u64 event, ++ unsigned char new_status) ++{ ++ int err = 0; ++ ++ mutex_lock(&kctx->jctx.lock); ++ ++ if (kbasep_write_soft_event_status(kctx, event, new_status)) { ++ err = -ENOENT; ++ goto out; ++ } ++ ++ if (new_status == BASE_JD_SOFT_EVENT_SET) ++ kbasep_complete_triggered_soft_events(kctx, event); ++ ++out: ++ mutex_unlock(&kctx->jctx.lock); ++ ++ return err; ++} ++ ++static void kbasep_soft_event_cancel_job(struct kbase_jd_atom *katom) ++{ ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ if (jd_done_nolock(katom, NULL)) ++ kbase_js_sched_all(katom->kctx->kbdev); ++} ++ ++struct kbase_debug_copy_buffer { ++ size_t size; ++ struct page **pages; ++ int nr_pages; ++ size_t offset; ++ struct kbase_mem_phy_alloc *gpu_alloc; ++ ++ struct page **extres_pages; ++ int nr_extres_pages; ++}; ++ ++static inline void free_user_buffer(struct kbase_debug_copy_buffer *buffer) ++{ ++ struct page **pages = buffer->extres_pages; ++ int nr_pages = buffer->nr_extres_pages; ++ ++ if (pages) { ++ int i; ++ ++ for (i = 0; i < nr_pages; i++) { ++ struct page *pg = pages[i]; ++ ++ if (pg) ++ put_page(pg); ++ } ++ kfree(pages); ++ } ++} ++ ++static void kbase_debug_copy_finish(struct kbase_jd_atom *katom) ++{ ++ struct kbase_debug_copy_buffer *buffers = ++ (struct kbase_debug_copy_buffer *)(uintptr_t)katom->jc; ++ unsigned int i; ++ unsigned int nr = katom->nr_extres; ++ ++ if (!buffers) ++ return; ++ ++ kbase_gpu_vm_lock(katom->kctx); ++ for (i = 0; i < nr; i++) { ++ int p; ++ struct kbase_mem_phy_alloc *gpu_alloc = buffers[i].gpu_alloc; ++ ++ if (!buffers[i].pages) ++ break; ++ for (p = 0; p < buffers[i].nr_pages; p++) { ++ struct page *pg = buffers[i].pages[p]; ++ ++ if (pg) ++ put_page(pg); ++ } ++ kfree(buffers[i].pages); ++ if (gpu_alloc) { ++ switch (gpu_alloc->type) { ++ case KBASE_MEM_TYPE_IMPORTED_USER_BUF: ++ { ++ free_user_buffer(&buffers[i]); ++ break; ++ } ++ default: ++ /* Nothing to be done. */ ++ break; ++ } ++ kbase_mem_phy_alloc_put(gpu_alloc); ++ } ++ } ++ kbase_gpu_vm_unlock(katom->kctx); ++ kfree(buffers); ++ ++ katom->jc = 0; ++} ++ ++static int kbase_debug_copy_prepare(struct kbase_jd_atom *katom) ++{ ++ struct kbase_debug_copy_buffer *buffers; ++ struct base_jd_debug_copy_buffer *user_buffers = NULL; ++ unsigned int i; ++ unsigned int nr = katom->nr_extres; ++ int ret = 0; ++ void __user *user_structs = (void __user *)(uintptr_t)katom->jc; ++ ++ if (!user_structs) ++ return -EINVAL; ++ ++ buffers = kcalloc(nr, sizeof(*buffers), GFP_KERNEL); ++ if (!buffers) { ++ ret = -ENOMEM; ++ katom->jc = 0; ++ goto out_cleanup; ++ } ++ katom->jc = (u64)(uintptr_t)buffers; ++ ++ user_buffers = kmalloc_array(nr, sizeof(*user_buffers), GFP_KERNEL); ++ ++ if (!user_buffers) { ++ ret = -ENOMEM; ++ goto out_cleanup; ++ } ++ ++ ret = copy_from_user(user_buffers, user_structs, ++ sizeof(*user_buffers)*nr); ++ if (ret) ++ goto out_cleanup; ++ ++ for (i = 0; i < nr; i++) { ++ u64 addr = user_buffers[i].address; ++ u64 page_addr = addr & PAGE_MASK; ++ u64 end_page_addr = addr + user_buffers[i].size - 1; ++ u64 last_page_addr = end_page_addr & PAGE_MASK; ++ int nr_pages = (last_page_addr-page_addr)/PAGE_SIZE+1; ++ int pinned_pages; ++ struct kbase_va_region *reg; ++ struct base_external_resource user_extres; ++ ++ if (!addr) ++ continue; ++ ++ buffers[i].nr_pages = nr_pages; ++ buffers[i].offset = addr & ~PAGE_MASK; ++ if (buffers[i].offset >= PAGE_SIZE) { ++ ret = -EINVAL; ++ goto out_cleanup; ++ } ++ buffers[i].size = user_buffers[i].size; ++ ++ buffers[i].pages = kcalloc(nr_pages, sizeof(struct page *), ++ GFP_KERNEL); ++ if (!buffers[i].pages) { ++ ret = -ENOMEM; ++ goto out_cleanup; ++ } ++ ++ pinned_pages = get_user_pages_fast(page_addr, ++ nr_pages, ++ 1, /* Write */ ++ buffers[i].pages); ++ if (pinned_pages < 0) { ++ ret = pinned_pages; ++ goto out_cleanup; ++ } ++ if (pinned_pages != nr_pages) { ++ ret = -EINVAL; ++ goto out_cleanup; ++ } ++ ++ user_extres = user_buffers[i].extres; ++ if (user_extres.ext_resource == 0ULL) { ++ ret = -EINVAL; ++ goto out_cleanup; ++ } ++ ++ kbase_gpu_vm_lock(katom->kctx); ++ reg = kbase_region_tracker_find_region_enclosing_address( ++ katom->kctx, user_extres.ext_resource & ++ ~BASE_EXT_RES_ACCESS_EXCLUSIVE); ++ ++ if (NULL == reg || NULL == reg->gpu_alloc || ++ (reg->flags & KBASE_REG_FREE)) { ++ ret = -EINVAL; ++ goto out_unlock; ++ } ++ ++ buffers[i].gpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); ++ buffers[i].nr_extres_pages = reg->nr_pages; ++ ++ if (reg->nr_pages*PAGE_SIZE != buffers[i].size) ++ dev_warn(katom->kctx->kbdev->dev, "Copy buffer is not of same size as the external resource to copy.\n"); ++ ++ switch (reg->gpu_alloc->type) { ++ case KBASE_MEM_TYPE_IMPORTED_USER_BUF: ++ { ++ struct kbase_mem_phy_alloc *alloc = reg->gpu_alloc; ++ unsigned long nr_pages = ++ alloc->imported.user_buf.nr_pages; ++ ++ if (alloc->imported.user_buf.mm != current->mm) { ++ ret = -EINVAL; ++ goto out_unlock; ++ } ++ buffers[i].extres_pages = kcalloc(nr_pages, ++ sizeof(struct page *), GFP_KERNEL); ++ if (!buffers[i].extres_pages) { ++ ret = -ENOMEM; ++ goto out_unlock; ++ } ++ ++ ret = get_user_pages_fast( ++ alloc->imported.user_buf.address, ++ nr_pages, 0, ++ buffers[i].extres_pages); ++ if (ret != nr_pages) ++ goto out_unlock; ++ ret = 0; ++ break; ++ } ++ case KBASE_MEM_TYPE_IMPORTED_UMP: ++ { ++ dev_warn(katom->kctx->kbdev->dev, ++ "UMP is not supported for debug_copy jobs\n"); ++ ret = -EINVAL; ++ goto out_unlock; ++ } ++ default: ++ /* Nothing to be done. */ ++ break; ++ } ++ kbase_gpu_vm_unlock(katom->kctx); ++ } ++ kfree(user_buffers); ++ ++ return ret; ++ ++out_unlock: ++ kbase_gpu_vm_unlock(katom->kctx); ++ ++out_cleanup: ++ kfree(buffers); ++ kfree(user_buffers); ++ ++ /* Frees allocated memory for kbase_debug_copy_job struct, including ++ * members, and sets jc to 0 */ ++ kbase_debug_copy_finish(katom); ++ return ret; ++} ++ ++static void kbase_mem_copy_from_extres_page(struct kbase_context *kctx, ++ void *extres_page, struct page **pages, unsigned int nr_pages, ++ unsigned int *target_page_nr, size_t offset, size_t *to_copy) ++{ ++ void *target_page = kmap(pages[*target_page_nr]); ++ size_t chunk = PAGE_SIZE-offset; ++ ++ lockdep_assert_held(&kctx->reg_lock); ++ ++ if (!target_page) { ++ *target_page_nr += 1; ++ dev_warn(kctx->kbdev->dev, "kmap failed in debug_copy job."); ++ return; ++ } ++ ++ chunk = min(chunk, *to_copy); ++ ++ memcpy(target_page + offset, extres_page, chunk); ++ *to_copy -= chunk; ++ ++ kunmap(pages[*target_page_nr]); ++ ++ *target_page_nr += 1; ++ if (*target_page_nr >= nr_pages) ++ return; ++ ++ target_page = kmap(pages[*target_page_nr]); ++ if (!target_page) { ++ *target_page_nr += 1; ++ dev_warn(kctx->kbdev->dev, "kmap failed in debug_copy job."); ++ return; ++ } ++ ++ KBASE_DEBUG_ASSERT(target_page); ++ ++ chunk = min(offset, *to_copy); ++ memcpy(target_page, extres_page + PAGE_SIZE-offset, chunk); ++ *to_copy -= chunk; ++ ++ kunmap(pages[*target_page_nr]); ++} ++ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0) ++static void *dma_buf_kmap_page(struct kbase_mem_phy_alloc *gpu_alloc, ++ unsigned long page_num, struct page **page) ++{ ++ struct sg_table *sgt = gpu_alloc->imported.umm.sgt; ++ struct sg_page_iter sg_iter; ++ unsigned long page_index = 0; ++ ++ if (WARN_ON(gpu_alloc->type != KBASE_MEM_TYPE_IMPORTED_UMM)) ++ return NULL; ++ ++ if (!sgt) ++ return NULL; ++ ++ if (WARN_ON(page_num >= gpu_alloc->nents)) ++ return NULL; ++ ++ for_each_sg_page(sgt->sgl, &sg_iter, sgt->nents, 0) { ++ if (page_index == page_num) { ++ *page = sg_page_iter_page(&sg_iter); ++ ++ return kmap(*page); ++ } ++ page_index++; ++ } ++ ++ return NULL; ++} ++#endif ++ ++static int kbase_mem_copy_from_extres(struct kbase_context *kctx, ++ struct kbase_debug_copy_buffer *buf_data) ++{ ++ unsigned int i; ++ unsigned int target_page_nr = 0; ++ struct page **pages = buf_data->pages; ++ u64 offset = buf_data->offset; ++ size_t extres_size = buf_data->nr_extres_pages*PAGE_SIZE; ++ size_t to_copy = min(extres_size, buf_data->size); ++ struct kbase_mem_phy_alloc *gpu_alloc = buf_data->gpu_alloc; ++ int ret = 0; ++ ++ KBASE_DEBUG_ASSERT(pages != NULL); ++ ++ kbase_gpu_vm_lock(kctx); ++ if (!gpu_alloc) { ++ ret = -EINVAL; ++ goto out_unlock; ++ } ++ ++ switch (gpu_alloc->type) { ++ case KBASE_MEM_TYPE_IMPORTED_USER_BUF: ++ { ++ for (i = 0; i < buf_data->nr_extres_pages; i++) { ++ struct page *pg = buf_data->extres_pages[i]; ++ void *extres_page = kmap(pg); ++ ++ if (extres_page) ++ kbase_mem_copy_from_extres_page(kctx, ++ extres_page, pages, ++ buf_data->nr_pages, ++ &target_page_nr, ++ offset, &to_copy); ++ ++ kunmap(pg); ++ if (target_page_nr >= buf_data->nr_pages) ++ break; ++ } ++ break; ++ } ++ break; ++#ifdef CONFIG_DMA_SHARED_BUFFER ++ case KBASE_MEM_TYPE_IMPORTED_UMM: { ++ struct dma_buf *dma_buf = gpu_alloc->imported.umm.dma_buf; ++ ++ KBASE_DEBUG_ASSERT(dma_buf != NULL); ++ KBASE_DEBUG_ASSERT(dma_buf->size == ++ buf_data->nr_extres_pages * PAGE_SIZE); ++ ++ ret = dma_buf_begin_cpu_access(dma_buf, ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) && !defined(CONFIG_CHROMEOS) ++ 0, buf_data->nr_extres_pages*PAGE_SIZE, ++#endif ++ DMA_FROM_DEVICE); ++ if (ret) ++ goto out_unlock; ++ ++ for (i = 0; i < buf_data->nr_extres_pages; i++) { ++ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0) ++ struct page *pg; ++ void *extres_page = dma_buf_kmap_page(gpu_alloc, i, &pg); ++#else ++ void *extres_page = dma_buf_kmap(dma_buf, i); ++#endif ++ ++ if (extres_page) ++ kbase_mem_copy_from_extres_page(kctx, ++ extres_page, pages, ++ buf_data->nr_pages, ++ &target_page_nr, ++ offset, &to_copy); ++ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0) ++ kunmap(pg); ++#else ++ dma_buf_kunmap(dma_buf, i, extres_page); ++#endif ++ if (target_page_nr >= buf_data->nr_pages) ++ break; ++ } ++ dma_buf_end_cpu_access(dma_buf, ++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) && !defined(CONFIG_CHROMEOS) ++ 0, buf_data->nr_extres_pages*PAGE_SIZE, ++#endif ++ DMA_FROM_DEVICE); ++ break; ++ } ++#endif ++ default: ++ ret = -EINVAL; ++ } ++out_unlock: ++ kbase_gpu_vm_unlock(kctx); ++ return ret; ++ ++} ++ ++static int kbase_debug_copy(struct kbase_jd_atom *katom) ++{ ++ struct kbase_debug_copy_buffer *buffers = ++ (struct kbase_debug_copy_buffer *)(uintptr_t)katom->jc; ++ unsigned int i; ++ ++ for (i = 0; i < katom->nr_extres; i++) { ++ int res = kbase_mem_copy_from_extres(katom->kctx, &buffers[i]); ++ ++ if (res) ++ return res; ++ } ++ ++ return 0; ++} ++ ++static int kbase_jit_allocate_prepare(struct kbase_jd_atom *katom) ++{ ++ __user void *data = (__user void *)(uintptr_t) katom->jc; ++ struct base_jit_alloc_info *info; ++ struct kbase_context *kctx = katom->kctx; ++ int ret; ++ ++ /* Fail the job if there is no info structure */ ++ if (!data) { ++ ret = -EINVAL; ++ goto fail; ++ } ++ ++ /* Copy the information for safe access and future storage */ ++ info = kzalloc(sizeof(*info), GFP_KERNEL); ++ if (!info) { ++ ret = -ENOMEM; ++ goto fail; ++ } ++ ++ if (copy_from_user(info, data, sizeof(*info)) != 0) { ++ ret = -EINVAL; ++ goto free_info; ++ } ++ ++ /* If the ID is zero then fail the job */ ++ if (info->id == 0) { ++ ret = -EINVAL; ++ goto free_info; ++ } ++ ++ /* Sanity check that the PA fits within the VA */ ++ if (info->va_pages < info->commit_pages) { ++ ret = -EINVAL; ++ goto free_info; ++ } ++ ++ /* Ensure the GPU address is correctly aligned */ ++ if ((info->gpu_alloc_addr & 0x7) != 0) { ++ ret = -EINVAL; ++ goto free_info; ++ } ++ ++ /* Replace the user pointer with our kernel allocated info structure */ ++ katom->jc = (u64)(uintptr_t) info; ++ katom->jit_blocked = false; ++ ++ lockdep_assert_held(&kctx->jctx.lock); ++ list_add_tail(&katom->jit_node, &kctx->jit_atoms_head); ++ ++ /* ++ * Note: ++ * The provided info->gpu_alloc_addr isn't validated here as ++ * userland can cache allocations which means that even ++ * though the region is valid it doesn't represent the ++ * same thing it used to. ++ * ++ * Complete validation of va_pages, commit_pages and extent ++ * isn't done here as it will be done during the call to ++ * kbase_mem_alloc. ++ */ ++ return 0; ++ ++free_info: ++ kfree(info); ++fail: ++ katom->jc = 0; ++ return ret; ++} ++ ++static u8 kbase_jit_free_get_id(struct kbase_jd_atom *katom) ++{ ++ if (WARN_ON(katom->core_req != BASE_JD_REQ_SOFT_JIT_FREE)) ++ return 0; ++ ++ return (u8) katom->jc; ++} ++ ++static int kbase_jit_allocate_process(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ struct base_jit_alloc_info *info; ++ struct kbase_va_region *reg; ++ struct kbase_vmap_struct mapping; ++ u64 *ptr, new_addr; ++ ++ if (katom->jit_blocked) { ++ list_del(&katom->queue); ++ katom->jit_blocked = false; ++ } ++ ++ info = (struct base_jit_alloc_info *) (uintptr_t) katom->jc; ++ ++ /* The JIT ID is still in use so fail the allocation */ ++ if (kctx->jit_alloc[info->id]) { ++ katom->event_code = BASE_JD_EVENT_MEM_GROWTH_FAILED; ++ return 0; ++ } ++ ++ /* Create a JIT allocation */ ++ reg = kbase_jit_allocate(kctx, info); ++ if (!reg) { ++ struct kbase_jd_atom *jit_atom; ++ bool can_block = false; ++ ++ lockdep_assert_held(&kctx->jctx.lock); ++ ++ jit_atom = list_first_entry(&kctx->jit_atoms_head, ++ struct kbase_jd_atom, jit_node); ++ ++ list_for_each_entry(jit_atom, &kctx->jit_atoms_head, jit_node) { ++ if (jit_atom == katom) ++ break; ++ if (jit_atom->core_req == BASE_JD_REQ_SOFT_JIT_FREE) { ++ u8 free_id = kbase_jit_free_get_id(jit_atom); ++ ++ if (free_id && kctx->jit_alloc[free_id]) { ++ /* A JIT free which is active and ++ * submitted before this atom ++ */ ++ can_block = true; ++ break; ++ } ++ } ++ } ++ ++ if (!can_block) { ++ /* Mark the allocation so we know it's in use even if ++ * the allocation itself fails. ++ */ ++ kctx->jit_alloc[info->id] = ++ (struct kbase_va_region *) -1; ++ ++ katom->event_code = BASE_JD_EVENT_MEM_GROWTH_FAILED; ++ return 0; ++ } ++ ++ /* There are pending frees for an active allocation ++ * so we should wait to see whether they free the memory. ++ * Add to the beginning of the list to ensure that the atom is ++ * processed only once in kbase_jit_free_finish ++ */ ++ list_add(&katom->queue, &kctx->jit_pending_alloc); ++ katom->jit_blocked = true; ++ ++ return 1; ++ } ++ ++ /* ++ * Write the address of the JIT allocation to the user provided ++ * GPU allocation. ++ */ ++ ptr = kbase_vmap(kctx, info->gpu_alloc_addr, sizeof(*ptr), ++ &mapping); ++ if (!ptr) { ++ /* ++ * Leave the allocation "live" as the JIT free jit will be ++ * submitted anyway. ++ */ ++ katom->event_code = BASE_JD_EVENT_JOB_INVALID; ++ return 0; ++ } ++ ++ new_addr = reg->start_pfn << PAGE_SHIFT; ++ *ptr = new_addr; ++ KBASE_TLSTREAM_TL_ATTRIB_ATOM_JIT( ++ katom, info->gpu_alloc_addr, new_addr); ++ kbase_vunmap(kctx, &mapping); ++ ++ katom->event_code = BASE_JD_EVENT_DONE; ++ ++ /* ++ * Bind it to the user provided ID. Do this last so we can check for ++ * the JIT free racing this JIT alloc job. ++ */ ++ kctx->jit_alloc[info->id] = reg; ++ ++ return 0; ++} ++ ++static void kbase_jit_allocate_finish(struct kbase_jd_atom *katom) ++{ ++ struct base_jit_alloc_info *info; ++ ++ lockdep_assert_held(&katom->kctx->jctx.lock); ++ ++ /* Remove atom from jit_atoms_head list */ ++ list_del(&katom->jit_node); ++ ++ if (katom->jit_blocked) { ++ list_del(&katom->queue); ++ katom->jit_blocked = false; ++ } ++ ++ info = (struct base_jit_alloc_info *) (uintptr_t) katom->jc; ++ /* Free the info structure */ ++ kfree(info); ++} ++ ++static int kbase_jit_free_prepare(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ ++ lockdep_assert_held(&kctx->jctx.lock); ++ list_add_tail(&katom->jit_node, &kctx->jit_atoms_head); ++ ++ return 0; ++} ++ ++static void kbase_jit_free_process(struct kbase_jd_atom *katom) ++{ ++ struct kbase_context *kctx = katom->kctx; ++ u8 id = kbase_jit_free_get_id(katom); ++ ++ /* ++ * If the ID is zero or it is not in use yet then fail the job. ++ */ ++ if ((id == 0) || (kctx->jit_alloc[id] == NULL)) { ++ katom->event_code = BASE_JD_EVENT_JOB_INVALID; ++ return; ++ } ++ ++ /* ++ * If the ID is valid but the allocation request failed still succeed ++ * this soft job but don't try and free the allocation. ++ */ ++ if (kctx->jit_alloc[id] != (struct kbase_va_region *) -1) ++ kbase_jit_free(kctx, kctx->jit_alloc[id]); ++ ++ kctx->jit_alloc[id] = NULL; ++} ++ ++static void kbasep_jit_free_finish_worker(struct work_struct *work) ++{ ++ struct kbase_jd_atom *katom = container_of(work, struct kbase_jd_atom, ++ work); ++ struct kbase_context *kctx = katom->kctx; ++ int resched; ++ ++ mutex_lock(&kctx->jctx.lock); ++ kbase_finish_soft_job(katom); ++ resched = jd_done_nolock(katom, NULL); ++ mutex_unlock(&kctx->jctx.lock); ++ ++ if (resched) ++ kbase_js_sched_all(kctx->kbdev); ++} ++ ++static void kbase_jit_free_finish(struct kbase_jd_atom *katom) ++{ ++ struct list_head *i, *tmp; ++ struct kbase_context *kctx = katom->kctx; ++ ++ lockdep_assert_held(&kctx->jctx.lock); ++ /* Remove this atom from the kctx->jit_atoms_head list */ ++ list_del(&katom->jit_node); ++ ++ list_for_each_safe(i, tmp, &kctx->jit_pending_alloc) { ++ struct kbase_jd_atom *pending_atom = list_entry(i, ++ struct kbase_jd_atom, queue); ++ if (kbase_jit_allocate_process(pending_atom) == 0) { ++ /* Atom has completed */ ++ INIT_WORK(&pending_atom->work, ++ kbasep_jit_free_finish_worker); ++ queue_work(kctx->jctx.job_done_wq, &pending_atom->work); ++ } ++ } ++} ++ ++static int kbase_ext_res_prepare(struct kbase_jd_atom *katom) ++{ ++ __user struct base_external_resource_list *user_ext_res; ++ struct base_external_resource_list *ext_res; ++ u64 count = 0; ++ size_t copy_size; ++ int ret; ++ ++ user_ext_res = (__user struct base_external_resource_list *) ++ (uintptr_t) katom->jc; ++ ++ /* Fail the job if there is no info structure */ ++ if (!user_ext_res) { ++ ret = -EINVAL; ++ goto fail; ++ } ++ ++ if (copy_from_user(&count, &user_ext_res->count, sizeof(u64)) != 0) { ++ ret = -EINVAL; ++ goto fail; ++ } ++ ++ /* Is the number of external resources in range? */ ++ if (!count || count > BASE_EXT_RES_COUNT_MAX) { ++ ret = -EINVAL; ++ goto fail; ++ } ++ ++ /* Copy the information for safe access and future storage */ ++ copy_size = sizeof(*ext_res); ++ copy_size += sizeof(struct base_external_resource) * (count - 1); ++ ext_res = kzalloc(copy_size, GFP_KERNEL); ++ if (!ext_res) { ++ ret = -ENOMEM; ++ goto fail; ++ } ++ ++ if (copy_from_user(ext_res, user_ext_res, copy_size) != 0) { ++ ret = -EINVAL; ++ goto free_info; ++ } ++ ++ /* ++ * Overwrite the count with the first value incase it was changed ++ * after the fact. ++ */ ++ ext_res->count = count; ++ ++ /* ++ * Replace the user pointer with our kernel allocated ++ * ext_res structure. ++ */ ++ katom->jc = (u64)(uintptr_t) ext_res; ++ ++ return 0; ++ ++free_info: ++ kfree(ext_res); ++fail: ++ return ret; ++} ++ ++static void kbase_ext_res_process(struct kbase_jd_atom *katom, bool map) ++{ ++ struct base_external_resource_list *ext_res; ++ int i; ++ bool failed = false; ++ ++ ext_res = (struct base_external_resource_list *) (uintptr_t) katom->jc; ++ if (!ext_res) ++ goto failed_jc; ++ ++ kbase_gpu_vm_lock(katom->kctx); ++ ++ for (i = 0; i < ext_res->count; i++) { ++ u64 gpu_addr; ++ ++ gpu_addr = ext_res->ext_res[i].ext_resource & ++ ~BASE_EXT_RES_ACCESS_EXCLUSIVE; ++ if (map) { ++ if (!kbase_sticky_resource_acquire(katom->kctx, ++ gpu_addr)) ++ goto failed_loop; ++ } else ++ if (!kbase_sticky_resource_release(katom->kctx, NULL, ++ gpu_addr)) ++ failed = true; ++ } ++ ++ /* ++ * In the case of unmap we continue unmapping other resources in the ++ * case of failure but will always report failure if _any_ unmap ++ * request fails. ++ */ ++ if (failed) ++ katom->event_code = BASE_JD_EVENT_JOB_INVALID; ++ else ++ katom->event_code = BASE_JD_EVENT_DONE; ++ ++ kbase_gpu_vm_unlock(katom->kctx); ++ ++ return; ++ ++failed_loop: ++ while (--i > 0) { ++ u64 gpu_addr; ++ ++ gpu_addr = ext_res->ext_res[i].ext_resource & ++ ~BASE_EXT_RES_ACCESS_EXCLUSIVE; ++ ++ kbase_sticky_resource_release(katom->kctx, NULL, gpu_addr); ++ } ++ ++ katom->event_code = BASE_JD_EVENT_JOB_INVALID; ++ kbase_gpu_vm_unlock(katom->kctx); ++ ++failed_jc: ++ return; ++} ++ ++static void kbase_ext_res_finish(struct kbase_jd_atom *katom) ++{ ++ struct base_external_resource_list *ext_res; ++ ++ ext_res = (struct base_external_resource_list *) (uintptr_t) katom->jc; ++ /* Free the info structure */ ++ kfree(ext_res); ++} ++ ++int kbase_process_soft_job(struct kbase_jd_atom *katom) ++{ ++ switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { ++ case BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME: ++ return kbase_dump_cpu_gpu_time(katom); ++ ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ case BASE_JD_REQ_SOFT_FENCE_TRIGGER: ++ katom->event_code = kbase_sync_fence_out_trigger(katom, ++ katom->event_code == BASE_JD_EVENT_DONE ? ++ 0 : -EFAULT); ++ break; ++ case BASE_JD_REQ_SOFT_FENCE_WAIT: ++ { ++ int ret = kbase_sync_fence_in_wait(katom); ++ ++ if (ret == 1) { ++#ifdef CONFIG_MALI_FENCE_DEBUG ++ kbasep_add_waiting_with_timeout(katom); ++#else ++ kbasep_add_waiting_soft_job(katom); ++#endif ++ } ++ return ret; ++ } ++#endif ++ ++ case BASE_JD_REQ_SOFT_REPLAY: ++ return kbase_replay_process(katom); ++ case BASE_JD_REQ_SOFT_EVENT_WAIT: ++ return kbasep_soft_event_wait(katom); ++ case BASE_JD_REQ_SOFT_EVENT_SET: ++ kbasep_soft_event_update_locked(katom, BASE_JD_SOFT_EVENT_SET); ++ break; ++ case BASE_JD_REQ_SOFT_EVENT_RESET: ++ kbasep_soft_event_update_locked(katom, BASE_JD_SOFT_EVENT_RESET); ++ break; ++ case BASE_JD_REQ_SOFT_DEBUG_COPY: ++ { ++ int res = kbase_debug_copy(katom); ++ ++ if (res) ++ katom->event_code = BASE_JD_EVENT_JOB_INVALID; ++ break; ++ } ++ case BASE_JD_REQ_SOFT_JIT_ALLOC: ++ return kbase_jit_allocate_process(katom); ++ case BASE_JD_REQ_SOFT_JIT_FREE: ++ kbase_jit_free_process(katom); ++ break; ++ case BASE_JD_REQ_SOFT_EXT_RES_MAP: ++ kbase_ext_res_process(katom, true); ++ break; ++ case BASE_JD_REQ_SOFT_EXT_RES_UNMAP: ++ kbase_ext_res_process(katom, false); ++ break; ++ } ++ ++ /* Atom is complete */ ++ return 0; ++} ++ ++void kbase_cancel_soft_job(struct kbase_jd_atom *katom) ++{ ++ switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ case BASE_JD_REQ_SOFT_FENCE_WAIT: ++ kbase_sync_fence_in_cancel_wait(katom); ++ break; ++#endif ++ case BASE_JD_REQ_SOFT_EVENT_WAIT: ++ kbasep_soft_event_cancel_job(katom); ++ break; ++ default: ++ /* This soft-job doesn't support cancellation! */ ++ KBASE_DEBUG_ASSERT(0); ++ } ++} ++ ++int kbase_prepare_soft_job(struct kbase_jd_atom *katom) ++{ ++ switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { ++ case BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME: ++ { ++ if (0 != (katom->jc & KBASE_CACHE_ALIGNMENT_MASK)) ++ return -EINVAL; ++ } ++ break; ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ case BASE_JD_REQ_SOFT_FENCE_TRIGGER: ++ { ++ struct base_fence fence; ++ int fd; ++ ++ if (0 != copy_from_user(&fence, (__user void *)(uintptr_t) katom->jc, sizeof(fence))) ++ return -EINVAL; ++ ++ fd = kbase_sync_fence_out_create(katom, ++ fence.basep.stream_fd); ++ if (fd < 0) ++ return -EINVAL; ++ ++ fence.basep.fd = fd; ++ if (0 != copy_to_user((__user void *)(uintptr_t) katom->jc, &fence, sizeof(fence))) { ++ kbase_sync_fence_out_remove(katom); ++ kbase_sync_fence_close_fd(fd); ++ fence.basep.fd = -EINVAL; ++ return -EINVAL; ++ } ++ } ++ break; ++ case BASE_JD_REQ_SOFT_FENCE_WAIT: ++ { ++ struct base_fence fence; ++ int ret; ++ ++ if (0 != copy_from_user(&fence, (__user void *)(uintptr_t) katom->jc, sizeof(fence))) ++ return -EINVAL; ++ ++ /* Get a reference to the fence object */ ++ ret = kbase_sync_fence_in_from_fd(katom, ++ fence.basep.fd); ++ if (ret < 0) ++ return ret; ++ ++#ifdef CONFIG_MALI_DMA_FENCE ++ /* ++ * Set KCTX_NO_IMPLICIT_FENCE in the context the first ++ * time a soft fence wait job is observed. This will ++ * prevent the implicit dma-buf fence to conflict with ++ * the Android native sync fences. ++ */ ++ if (!kbase_ctx_flag(katom->kctx, KCTX_NO_IMPLICIT_SYNC)) ++ kbase_ctx_flag_set(katom->kctx, KCTX_NO_IMPLICIT_SYNC); ++#endif /* CONFIG_MALI_DMA_FENCE */ ++ } ++ break; ++#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ ++ case BASE_JD_REQ_SOFT_JIT_ALLOC: ++ return kbase_jit_allocate_prepare(katom); ++ case BASE_JD_REQ_SOFT_REPLAY: ++ break; ++ case BASE_JD_REQ_SOFT_JIT_FREE: ++ return kbase_jit_free_prepare(katom); ++ case BASE_JD_REQ_SOFT_EVENT_WAIT: ++ case BASE_JD_REQ_SOFT_EVENT_SET: ++ case BASE_JD_REQ_SOFT_EVENT_RESET: ++ if (katom->jc == 0) ++ return -EINVAL; ++ break; ++ case BASE_JD_REQ_SOFT_DEBUG_COPY: ++ return kbase_debug_copy_prepare(katom); ++ case BASE_JD_REQ_SOFT_EXT_RES_MAP: ++ return kbase_ext_res_prepare(katom); ++ case BASE_JD_REQ_SOFT_EXT_RES_UNMAP: ++ return kbase_ext_res_prepare(katom); ++ default: ++ /* Unsupported soft-job */ ++ return -EINVAL; ++ } ++ return 0; ++} ++ ++void kbase_finish_soft_job(struct kbase_jd_atom *katom) ++{ ++ switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { ++ case BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME: ++ /* Nothing to do */ ++ break; ++#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) ++ case BASE_JD_REQ_SOFT_FENCE_TRIGGER: ++ /* If fence has not yet been signaled, do it now */ ++ kbase_sync_fence_out_trigger(katom, katom->event_code == ++ BASE_JD_EVENT_DONE ? 0 : -EFAULT); ++ break; ++ case BASE_JD_REQ_SOFT_FENCE_WAIT: ++ /* Release katom's reference to fence object */ ++ kbase_sync_fence_in_remove(katom); ++ break; ++#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ ++ case BASE_JD_REQ_SOFT_DEBUG_COPY: ++ kbase_debug_copy_finish(katom); ++ break; ++ case BASE_JD_REQ_SOFT_JIT_ALLOC: ++ kbase_jit_allocate_finish(katom); ++ break; ++ case BASE_JD_REQ_SOFT_EXT_RES_MAP: ++ kbase_ext_res_finish(katom); ++ break; ++ case BASE_JD_REQ_SOFT_EXT_RES_UNMAP: ++ kbase_ext_res_finish(katom); ++ break; ++ case BASE_JD_REQ_SOFT_JIT_FREE: ++ kbase_jit_free_finish(katom); ++ break; ++ } ++} ++ ++void kbase_resume_suspended_soft_jobs(struct kbase_device *kbdev) ++{ ++ LIST_HEAD(local_suspended_soft_jobs); ++ struct kbase_jd_atom *tmp_iter; ++ struct kbase_jd_atom *katom_iter; ++ struct kbasep_js_device_data *js_devdata; ++ bool resched = false; ++ ++ KBASE_DEBUG_ASSERT(kbdev); ++ ++ js_devdata = &kbdev->js_data; ++ ++ /* Move out the entire list */ ++ mutex_lock(&js_devdata->runpool_mutex); ++ list_splice_init(&js_devdata->suspended_soft_jobs_list, ++ &local_suspended_soft_jobs); ++ mutex_unlock(&js_devdata->runpool_mutex); ++ ++ /* ++ * Each atom must be detached from the list and ran separately - ++ * it could be re-added to the old list, but this is unlikely ++ */ ++ list_for_each_entry_safe(katom_iter, tmp_iter, ++ &local_suspended_soft_jobs, dep_item[1]) { ++ struct kbase_context *kctx = katom_iter->kctx; ++ ++ mutex_lock(&kctx->jctx.lock); ++ ++ /* Remove from the global list */ ++ list_del(&katom_iter->dep_item[1]); ++ /* Remove from the context's list of waiting soft jobs */ ++ kbasep_remove_waiting_soft_job(katom_iter); ++ ++ if (kbase_process_soft_job(katom_iter) == 0) { ++ kbase_finish_soft_job(katom_iter); ++ resched |= jd_done_nolock(katom_iter, NULL); ++ } else { ++ KBASE_DEBUG_ASSERT((katom_iter->core_req & ++ BASE_JD_REQ_SOFT_JOB_TYPE) ++ != BASE_JD_REQ_SOFT_REPLAY); ++ } ++ ++ mutex_unlock(&kctx->jctx.lock); ++ } ++ ++ if (resched) ++ kbase_js_sched_all(kbdev); ++} +diff --git a/drivers/gpu/arm/midgard/mali_kbase_strings.c b/drivers/gpu/arm/midgard/mali_kbase_strings.c +new file mode 100755 +index 000000000000..c98762cec244 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_strings.c +@@ -0,0 +1,23 @@ ++ /* ++ * ++ * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++#include "mali_kbase_strings.h" ++ ++#define KBASE_DRV_NAME "mali" ++#define KBASE_TIMELINE_NAME KBASE_DRV_NAME ".timeline" ++ ++const char kbase_drv_name[] = KBASE_DRV_NAME; ++const char kbase_timeline_name[] = KBASE_TIMELINE_NAME; +diff --git a/drivers/gpu/arm/midgard/mali_kbase_strings.h b/drivers/gpu/arm/midgard/mali_kbase_strings.h +new file mode 100755 +index 000000000000..41b8fdbec6a4 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_strings.h +@@ -0,0 +1,19 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++extern const char kbase_drv_name[]; ++extern const char kbase_timeline_name[]; +diff --git a/drivers/gpu/arm/midgard/mali_kbase_sync.h b/drivers/gpu/arm/midgard/mali_kbase_sync.h +new file mode 100755 +index 000000000000..33b580595563 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_sync.h +@@ -0,0 +1,203 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/** ++ * @file mali_kbase_sync.h ++ * ++ * This file contains our internal "API" for explicit fences. ++ * It hides the implementation details of the actual explicit fence mechanism ++ * used (Android fences or sync file with DMA fences). ++ */ ++ ++#ifndef MALI_KBASE_SYNC_H ++#define MALI_KBASE_SYNC_H ++ ++#include ++#ifdef CONFIG_SYNC ++#include ++#endif ++#ifdef CONFIG_SYNC_FILE ++#include "mali_kbase_fence_defs.h" ++#include ++#endif ++ ++#include "mali_kbase.h" ++ ++/** ++ * struct kbase_sync_fence_info - Information about a fence ++ * @fence: Pointer to fence (type is void*, as underlaying struct can differ) ++ * @name: The name given to this fence when it was created ++ * @status: < 0 means error, 0 means active, 1 means signaled ++ * ++ * Use kbase_sync_fence_in_info_get() or kbase_sync_fence_out_info_get() ++ * to get the information. ++ */ ++struct kbase_sync_fence_info { ++ void *fence; ++ char name[32]; ++ int status; ++}; ++ ++/** ++ * kbase_sync_fence_stream_create() - Create a stream object ++ * @name: Name of stream (only used to ease debugging/visualization) ++ * @out_fd: A file descriptor representing the created stream object ++ * ++ * Can map down to a timeline implementation in some implementations. ++ * Exposed as a file descriptor. ++ * Life-time controlled via the file descriptor: ++ * - dup to add a ref ++ * - close to remove a ref ++ * ++ * return: 0 on success, < 0 on error ++ */ ++int kbase_sync_fence_stream_create(const char *name, int *const out_fd); ++ ++/** ++ * kbase_sync_fence_out_create Create an explicit output fence to specified atom ++ * @katom: Atom to assign the new explicit fence to ++ * @stream_fd: File descriptor for stream object to create fence on ++ * ++ * return: Valid file descriptor to fence or < 0 on error ++ */ ++int kbase_sync_fence_out_create(struct kbase_jd_atom *katom, int stream_fd); ++ ++/** ++ * kbase_sync_fence_in_from_fd() Assigns an existing fence to specified atom ++ * @katom: Atom to assign the existing explicit fence to ++ * @fd: File descriptor to an existing fence ++ * ++ * Assigns an explicit input fence to atom. ++ * This can later be waited for by calling @kbase_sync_fence_in_wait ++ * ++ * return: 0 on success, < 0 on error ++ */ ++int kbase_sync_fence_in_from_fd(struct kbase_jd_atom *katom, int fd); ++ ++/** ++ * kbase_sync_fence_validate() - Validate a fd to be a valid fence ++ * @fd: File descriptor to check ++ * ++ * This function is only usable to catch unintentional user errors early, ++ * it does not stop malicious code changing the fd after this function returns. ++ * ++ * return 0: if fd is for a valid fence, < 0 if invalid ++ */ ++int kbase_sync_fence_validate(int fd); ++ ++/** ++ * kbase_sync_fence_out_trigger - Signal explicit output fence attached on katom ++ * @katom: Atom with an explicit fence to signal ++ * @result: < 0 means signal with error, 0 >= indicates success ++ * ++ * Signal output fence attached on katom and remove the fence from the atom. ++ * ++ * return: The "next" event code for atom, typically JOB_CANCELLED or EVENT_DONE ++ */ ++enum base_jd_event_code ++kbase_sync_fence_out_trigger(struct kbase_jd_atom *katom, int result); ++ ++/** ++ * kbase_sync_fence_in_wait() - Wait for explicit input fence to be signaled ++ * @katom: Atom with explicit fence to wait for ++ * ++ * If the fence is already signaled, then 0 is returned, and the caller must ++ * continue processing of the katom. ++ * ++ * If the fence isn't already signaled, then this kbase_sync framework will ++ * take responsibility to continue the processing once the fence is signaled. ++ * ++ * return: 0 if already signaled, otherwise 1 ++ */ ++int kbase_sync_fence_in_wait(struct kbase_jd_atom *katom); ++ ++/** ++ * kbase_sync_fence_in_cancel_wait() - Cancel explicit input fence waits ++ * @katom: Atom to cancel wait for ++ * ++ * This function is fully responsible for continuing processing of this atom ++ * (remove_waiting_soft_job + finish_soft_job + jd_done + js_sched_all) ++ */ ++void kbase_sync_fence_in_cancel_wait(struct kbase_jd_atom *katom); ++ ++/** ++ * kbase_sync_fence_in_remove() - Remove the input fence from the katom ++ * @katom: Atom to remove explicit input fence for ++ * ++ * This will also release the corresponding reference. ++ */ ++void kbase_sync_fence_in_remove(struct kbase_jd_atom *katom); ++ ++/** ++ * kbase_sync_fence_out_remove() - Remove the output fence from the katom ++ * @katom: Atom to remove explicit output fence for ++ * ++ * This will also release the corresponding reference. ++ */ ++void kbase_sync_fence_out_remove(struct kbase_jd_atom *katom); ++ ++/** ++ * kbase_sync_fence_close_fd() - Close a file descriptor representing a fence ++ * @fd: File descriptor to close ++ */ ++static inline void kbase_sync_fence_close_fd(int fd) ++{ ++ ksys_close(fd); ++} ++ ++/** ++ * kbase_sync_fence_in_info_get() - Retrieves information about input fence ++ * @katom: Atom to get fence information from ++ * @info: Struct to be filled with fence information ++ * ++ * return: 0 on success, < 0 on error ++ */ ++int kbase_sync_fence_in_info_get(struct kbase_jd_atom *katom, ++ struct kbase_sync_fence_info *info); ++ ++/** ++ * kbase_sync_fence_out_info_get() - Retrieves information about output fence ++ * @katom: Atom to get fence information from ++ * @info: Struct to be filled with fence information ++ * ++ * return: 0 on success, < 0 on error ++ */ ++int kbase_sync_fence_out_info_get(struct kbase_jd_atom *katom, ++ struct kbase_sync_fence_info *info); ++ ++/** ++ * kbase_sync_status_string() - Get string matching @status ++ * @status: Value of fence status. ++ * ++ * return: Pointer to string describing @status. ++ */ ++const char *kbase_sync_status_string(int status); ++ ++/* ++ * Internal worker used to continue processing of atom. ++ */ ++void kbase_sync_fence_wait_worker(struct work_struct *data); ++ ++#ifdef CONFIG_MALI_FENCE_DEBUG ++/** ++ * kbase_sync_fence_in_dump() Trigger a debug dump of atoms input fence state ++ * @katom: Atom to trigger fence debug dump for ++ */ ++void kbase_sync_fence_in_dump(struct kbase_jd_atom *katom); ++#endif ++ ++#endif /* MALI_KBASE_SYNC_H */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_sync_android.c b/drivers/gpu/arm/midgard/mali_kbase_sync_android.c +new file mode 100755 +index 000000000000..d7349dcae69a +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_sync_android.c +@@ -0,0 +1,537 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* ++ * Code for supporting explicit Android fences (CONFIG_SYNC) ++ * Known to be good for kernels 4.5 and earlier. ++ * Replaced with CONFIG_SYNC_FILE for 4.9 and later kernels ++ * (see mali_kbase_sync_file.c) ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include "sync.h" ++#include ++#include ++ ++struct mali_sync_timeline { ++ struct sync_timeline timeline; ++ atomic_t counter; ++ atomic_t signaled; ++}; ++ ++struct mali_sync_pt { ++ struct sync_pt pt; ++ int order; ++ int result; ++}; ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) ++/* For backwards compatibility with kernels before 3.17. After 3.17 ++ * sync_pt_parent is included in the kernel. */ ++static inline struct sync_timeline *sync_pt_parent(struct sync_pt *pt) ++{ ++ return pt->parent; ++} ++#endif ++ ++static struct mali_sync_timeline *to_mali_sync_timeline( ++ struct sync_timeline *timeline) ++{ ++ return container_of(timeline, struct mali_sync_timeline, timeline); ++} ++ ++static struct mali_sync_pt *to_mali_sync_pt(struct sync_pt *pt) ++{ ++ return container_of(pt, struct mali_sync_pt, pt); ++} ++ ++static struct sync_pt *timeline_dup(struct sync_pt *pt) ++{ ++ struct mali_sync_pt *mpt = to_mali_sync_pt(pt); ++ struct mali_sync_pt *new_mpt; ++ struct sync_pt *new_pt = sync_pt_create(sync_pt_parent(pt), ++ sizeof(struct mali_sync_pt)); ++ ++ if (!new_pt) ++ return NULL; ++ ++ new_mpt = to_mali_sync_pt(new_pt); ++ new_mpt->order = mpt->order; ++ new_mpt->result = mpt->result; ++ ++ return new_pt; ++} ++ ++static int timeline_has_signaled(struct sync_pt *pt) ++{ ++ struct mali_sync_pt *mpt = to_mali_sync_pt(pt); ++ struct mali_sync_timeline *mtl = to_mali_sync_timeline( ++ sync_pt_parent(pt)); ++ int result = mpt->result; ++ ++ int diff = atomic_read(&mtl->signaled) - mpt->order; ++ ++ if (diff >= 0) ++ return (result < 0) ? result : 1; ++ ++ return 0; ++} ++ ++static int timeline_compare(struct sync_pt *a, struct sync_pt *b) ++{ ++ struct mali_sync_pt *ma = container_of(a, struct mali_sync_pt, pt); ++ struct mali_sync_pt *mb = container_of(b, struct mali_sync_pt, pt); ++ ++ int diff = ma->order - mb->order; ++ ++ if (diff == 0) ++ return 0; ++ ++ return (diff < 0) ? -1 : 1; ++} ++ ++static void timeline_value_str(struct sync_timeline *timeline, char *str, ++ int size) ++{ ++ struct mali_sync_timeline *mtl = to_mali_sync_timeline(timeline); ++ ++ snprintf(str, size, "%d", atomic_read(&mtl->signaled)); ++} ++ ++static void pt_value_str(struct sync_pt *pt, char *str, int size) ++{ ++ struct mali_sync_pt *mpt = to_mali_sync_pt(pt); ++ ++ snprintf(str, size, "%d(%d)", mpt->order, mpt->result); ++} ++ ++static struct sync_timeline_ops mali_timeline_ops = { ++ .driver_name = "Mali", ++ .dup = timeline_dup, ++ .has_signaled = timeline_has_signaled, ++ .compare = timeline_compare, ++ .timeline_value_str = timeline_value_str, ++ .pt_value_str = pt_value_str, ++}; ++ ++/* Allocates a timeline for Mali ++ * ++ * One timeline should be allocated per API context. ++ */ ++static struct sync_timeline *mali_sync_timeline_alloc(const char *name) ++{ ++ struct sync_timeline *tl; ++ struct mali_sync_timeline *mtl; ++ ++ tl = sync_timeline_create(&mali_timeline_ops, ++ sizeof(struct mali_sync_timeline), name); ++ if (!tl) ++ return NULL; ++ ++ /* Set the counter in our private struct */ ++ mtl = to_mali_sync_timeline(tl); ++ atomic_set(&mtl->counter, 0); ++ atomic_set(&mtl->signaled, 0); ++ ++ return tl; ++} ++ ++static int kbase_stream_close(struct inode *inode, struct file *file) ++{ ++ struct sync_timeline *tl; ++ ++ tl = (struct sync_timeline *)file->private_data; ++ sync_timeline_destroy(tl); ++ return 0; ++} ++ ++static const struct file_operations stream_fops = { ++ .owner = THIS_MODULE, ++ .release = kbase_stream_close, ++}; ++ ++int kbase_sync_fence_stream_create(const char *name, int *const out_fd) ++{ ++ struct sync_timeline *tl; ++ ++ if (!out_fd) ++ return -EINVAL; ++ ++ tl = mali_sync_timeline_alloc(name); ++ if (!tl) ++ return -EINVAL; ++ ++ *out_fd = anon_inode_getfd(name, &stream_fops, tl, O_RDONLY|O_CLOEXEC); ++ ++ if (*out_fd < 0) { ++ sync_timeline_destroy(tl); ++ return -EINVAL; ++ } ++ ++ return 0; ++} ++ ++/* Allocates a sync point within the timeline. ++ * ++ * The timeline must be the one allocated by kbase_sync_timeline_alloc ++ * ++ * Sync points must be triggered in *exactly* the same order as they are ++ * allocated. ++ */ ++static struct sync_pt *kbase_sync_pt_alloc(struct sync_timeline *parent) ++{ ++ struct sync_pt *pt = sync_pt_create(parent, ++ sizeof(struct mali_sync_pt)); ++ struct mali_sync_timeline *mtl = to_mali_sync_timeline(parent); ++ struct mali_sync_pt *mpt; ++ ++ if (!pt) ++ return NULL; ++ ++ mpt = to_mali_sync_pt(pt); ++ mpt->order = atomic_inc_return(&mtl->counter); ++ mpt->result = 0; ++ ++ return pt; ++} ++ ++int kbase_sync_fence_out_create(struct kbase_jd_atom *katom, int tl_fd) ++{ ++ struct sync_timeline *tl; ++ struct sync_pt *pt; ++ struct sync_fence *fence; ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 7, 0) ++ struct files_struct *files; ++ struct fdtable *fdt; ++#endif ++ int fd; ++ struct file *tl_file; ++ ++ tl_file = fget(tl_fd); ++ if (tl_file == NULL) ++ return -EBADF; ++ ++ if (tl_file->f_op != &stream_fops) { ++ fd = -EBADF; ++ goto out; ++ } ++ ++ tl = tl_file->private_data; ++ ++ pt = kbase_sync_pt_alloc(tl); ++ if (!pt) { ++ fd = -EFAULT; ++ goto out; ++ } ++ ++ fence = sync_fence_create("mali_fence", pt); ++ if (!fence) { ++ sync_pt_free(pt); ++ fd = -EFAULT; ++ goto out; ++ } ++ ++ /* from here the fence owns the sync_pt */ ++ ++ /* create a fd representing the fence */ ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0) ++ fd = get_unused_fd_flags(O_RDWR | O_CLOEXEC); ++ if (fd < 0) { ++ sync_fence_put(fence); ++ goto out; ++ } ++#else ++ fd = get_unused_fd(); ++ if (fd < 0) { ++ sync_fence_put(fence); ++ goto out; ++ } ++ ++ files = current->files; ++ spin_lock(&files->file_lock); ++ fdt = files_fdtable(files); ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 4, 0) ++ __set_close_on_exec(fd, fdt); ++#else ++ FD_SET(fd, fdt->close_on_exec); ++#endif ++ spin_unlock(&files->file_lock); ++#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0) */ ++ ++ /* bind fence to the new fd */ ++ sync_fence_install(fence, fd); ++ ++ katom->fence = sync_fence_fdget(fd); ++ if (katom->fence == NULL) { ++ /* The only way the fence can be NULL is if userspace closed it ++ * for us, so we don't need to clear it up */ ++ fd = -EINVAL; ++ goto out; ++ } ++ ++out: ++ fput(tl_file); ++ ++ return fd; ++} ++ ++int kbase_sync_fence_in_from_fd(struct kbase_jd_atom *katom, int fd) ++{ ++ katom->fence = sync_fence_fdget(fd); ++ return katom->fence ? 0 : -ENOENT; ++} ++ ++int kbase_sync_fence_validate(int fd) ++{ ++ struct sync_fence *fence; ++ ++ fence = sync_fence_fdget(fd); ++ if (!fence) ++ return -EINVAL; ++ ++ sync_fence_put(fence); ++ return 0; ++} ++ ++/* Returns true if the specified timeline is allocated by Mali */ ++static int kbase_sync_timeline_is_ours(struct sync_timeline *timeline) ++{ ++ return timeline->ops == &mali_timeline_ops; ++} ++ ++/* Signals a particular sync point ++ * ++ * Sync points must be triggered in *exactly* the same order as they are ++ * allocated. ++ * ++ * If they are signaled in the wrong order then a message will be printed in ++ * debug builds and otherwise attempts to signal order sync_pts will be ignored. ++ * ++ * result can be negative to indicate error, any other value is interpreted as ++ * success. ++ */ ++static void kbase_sync_signal_pt(struct sync_pt *pt, int result) ++{ ++ struct mali_sync_pt *mpt = to_mali_sync_pt(pt); ++ struct mali_sync_timeline *mtl = to_mali_sync_timeline( ++ sync_pt_parent(pt)); ++ int signaled; ++ int diff; ++ ++ mpt->result = result; ++ ++ do { ++ signaled = atomic_read(&mtl->signaled); ++ ++ diff = signaled - mpt->order; ++ ++ if (diff > 0) { ++ /* The timeline is already at or ahead of this point. ++ * This should not happen unless userspace has been ++ * signaling fences out of order, so warn but don't ++ * violate the sync_pt API. ++ * The warning is only in debug builds to prevent ++ * a malicious user being able to spam dmesg. ++ */ ++#ifdef CONFIG_MALI_DEBUG ++ pr_err("Fences were triggered in a different order to allocation!"); ++#endif /* CONFIG_MALI_DEBUG */ ++ return; ++ } ++ } while (atomic_cmpxchg(&mtl->signaled, ++ signaled, mpt->order) != signaled); ++} ++ ++enum base_jd_event_code ++kbase_sync_fence_out_trigger(struct kbase_jd_atom *katom, int result) ++{ ++ struct sync_pt *pt; ++ struct sync_timeline *timeline; ++ ++ if (!katom->fence) ++ return BASE_JD_EVENT_JOB_CANCELLED; ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) ++ if (!list_is_singular(&katom->fence->pt_list_head)) { ++#else ++ if (katom->fence->num_fences != 1) { ++#endif ++ /* Not exactly one item in the list - so it didn't (directly) ++ * come from us */ ++ return BASE_JD_EVENT_JOB_CANCELLED; ++ } ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) ++ pt = list_first_entry(&katom->fence->pt_list_head, ++ struct sync_pt, pt_list); ++#else ++ pt = container_of(katom->fence->cbs[0].sync_pt, struct sync_pt, base); ++#endif ++ timeline = sync_pt_parent(pt); ++ ++ if (!kbase_sync_timeline_is_ours(timeline)) { ++ /* Fence has a sync_pt which isn't ours! */ ++ return BASE_JD_EVENT_JOB_CANCELLED; ++ } ++ ++ kbase_sync_signal_pt(pt, result); ++ ++ sync_timeline_signal(timeline); ++ ++ kbase_sync_fence_out_remove(katom); ++ ++ return (result < 0) ? BASE_JD_EVENT_JOB_CANCELLED : BASE_JD_EVENT_DONE; ++} ++ ++static inline int kbase_fence_get_status(struct sync_fence *fence) ++{ ++ if (!fence) ++ return -ENOENT; ++ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) ++ return fence->status; ++#else ++ return atomic_read(&fence->status); ++#endif ++} ++ ++static void kbase_fence_wait_callback(struct sync_fence *fence, ++ struct sync_fence_waiter *waiter) ++{ ++ struct kbase_jd_atom *katom = container_of(waiter, ++ struct kbase_jd_atom, sync_waiter); ++ struct kbase_context *kctx = katom->kctx; ++ ++ /* Propagate the fence status to the atom. ++ * If negative then cancel this atom and its dependencies. ++ */ ++ if (kbase_fence_get_status(fence) < 0) ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ ++ /* To prevent a potential deadlock we schedule the work onto the ++ * job_done_wq workqueue ++ * ++ * The issue is that we may signal the timeline while holding ++ * kctx->jctx.lock and the callbacks are run synchronously from ++ * sync_timeline_signal. So we simply defer the work. ++ */ ++ ++ INIT_WORK(&katom->work, kbase_sync_fence_wait_worker); ++ queue_work(kctx->jctx.job_done_wq, &katom->work); ++} ++ ++int kbase_sync_fence_in_wait(struct kbase_jd_atom *katom) ++{ ++ int ret; ++ ++ sync_fence_waiter_init(&katom->sync_waiter, kbase_fence_wait_callback); ++ ++ ret = sync_fence_wait_async(katom->fence, &katom->sync_waiter); ++ ++ if (ret == 1) { ++ /* Already signaled */ ++ return 0; ++ } ++ ++ if (ret < 0) { ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ /* We should cause the dependent jobs in the bag to be failed, ++ * to do this we schedule the work queue to complete this job */ ++ INIT_WORK(&katom->work, kbase_sync_fence_wait_worker); ++ queue_work(katom->kctx->jctx.job_done_wq, &katom->work); ++ } ++ ++ return 1; ++} ++ ++void kbase_sync_fence_in_cancel_wait(struct kbase_jd_atom *katom) ++{ ++ if (sync_fence_cancel_async(katom->fence, &katom->sync_waiter) != 0) { ++ /* The wait wasn't cancelled - leave the cleanup for ++ * kbase_fence_wait_callback */ ++ return; ++ } ++ ++ /* Wait was cancelled - zap the atoms */ ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ ++ kbasep_remove_waiting_soft_job(katom); ++ kbase_finish_soft_job(katom); ++ ++ if (jd_done_nolock(katom, NULL)) ++ kbase_js_sched_all(katom->kctx->kbdev); ++} ++ ++void kbase_sync_fence_out_remove(struct kbase_jd_atom *katom) ++{ ++ if (katom->fence) { ++ sync_fence_put(katom->fence); ++ katom->fence = NULL; ++ } ++} ++ ++void kbase_sync_fence_in_remove(struct kbase_jd_atom *katom) ++{ ++ if (katom->fence) { ++ sync_fence_put(katom->fence); ++ katom->fence = NULL; ++ } ++} ++ ++int kbase_sync_fence_in_info_get(struct kbase_jd_atom *katom, ++ struct kbase_sync_fence_info *info) ++{ ++ if (!katom->fence) ++ return -ENOENT; ++ ++ info->fence = katom->fence; ++ info->status = kbase_fence_get_status(katom->fence); ++ strlcpy(info->name, katom->fence->name, sizeof(info->name)); ++ ++ return 0; ++} ++ ++int kbase_sync_fence_out_info_get(struct kbase_jd_atom *katom, ++ struct kbase_sync_fence_info *info) ++{ ++ if (!katom->fence) ++ return -ENOENT; ++ ++ info->fence = katom->fence; ++ info->status = kbase_fence_get_status(katom->fence); ++ strlcpy(info->name, katom->fence->name, sizeof(info->name)); ++ ++ return 0; ++} ++ ++#ifdef CONFIG_MALI_FENCE_DEBUG ++void kbase_sync_fence_in_dump(struct kbase_jd_atom *katom) ++{ ++ /* Dump out the full state of all the Android sync fences. ++ * The function sync_dump() isn't exported to modules, so force ++ * sync_fence_wait() to time out to trigger sync_dump(). ++ */ ++ if (katom->fence) ++ sync_fence_wait(katom->fence, 1); ++} ++#endif +diff --git a/drivers/gpu/arm/midgard/mali_kbase_sync_common.c b/drivers/gpu/arm/midgard/mali_kbase_sync_common.c +new file mode 100755 +index 000000000000..457def296684 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_sync_common.c +@@ -0,0 +1,43 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* ++ * @file mali_kbase_sync_common.c ++ * ++ * Common code for our explicit fence functionality ++ */ ++ ++#include ++#include "mali_kbase.h" ++ ++void kbase_sync_fence_wait_worker(struct work_struct *data) ++{ ++ struct kbase_jd_atom *katom; ++ ++ katom = container_of(data, struct kbase_jd_atom, work); ++ kbase_soft_event_wait_callback(katom); ++} ++ ++const char *kbase_sync_status_string(int status) ++{ ++ if (status == 0) ++ return "signaled"; ++ else if (status > 0) ++ return "active"; ++ else ++ return "error"; ++} +diff --git a/drivers/gpu/arm/midgard/mali_kbase_sync_file.c b/drivers/gpu/arm/midgard/mali_kbase_sync_file.c +new file mode 100755 +index 000000000000..60b5d74db33e +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_sync_file.c +@@ -0,0 +1,359 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* ++ * Code for supporting explicit Linux fences (CONFIG_SYNC_FILE) ++ * Introduced in kernel 4.9. ++ * Android explicit fences (CONFIG_SYNC) can be used for older kernels ++ * (see mali_kbase_sync_android.c) ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include "mali_kbase_fence_defs.h" ++#include "mali_kbase_sync.h" ++#include "mali_kbase_fence.h" ++#include "mali_kbase.h" ++ ++static const struct file_operations stream_fops = { ++ .owner = THIS_MODULE ++}; ++ ++int kbase_sync_fence_stream_create(const char *name, int *const out_fd) ++{ ++ if (!out_fd) ++ return -EINVAL; ++ ++ *out_fd = anon_inode_getfd(name, &stream_fops, NULL, ++ O_RDONLY | O_CLOEXEC); ++ if (*out_fd < 0) ++ return -EINVAL; ++ ++ return 0; ++} ++ ++int kbase_sync_fence_out_create(struct kbase_jd_atom *katom, int stream_fd) ++{ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ struct fence *fence; ++#else ++ struct dma_fence *fence; ++#endif ++ struct sync_file *sync_file; ++ int fd; ++ ++ fence = kbase_fence_out_new(katom); ++ if (!fence) ++ return -ENOMEM; ++ ++#if (KERNEL_VERSION(4, 9, 67) >= LINUX_VERSION_CODE) ++ /* Take an extra reference to the fence on behalf of the sync_file. ++ * This is only needed on older kernels where sync_file_create() ++ * does not take its own reference. This was changed in v4.9.68, ++ * where sync_file_create() now takes its own reference. ++ */ ++ dma_fence_get(fence); ++#endif ++ ++ /* create a sync_file fd representing the fence */ ++ sync_file = sync_file_create(fence); ++ if (!sync_file) { ++ dma_fence_put(fence); ++ kbase_fence_out_remove(katom); ++ return -ENOMEM; ++ } ++ ++ fd = get_unused_fd_flags(O_CLOEXEC); ++ if (fd < 0) { ++ fput(sync_file->file); ++ kbase_fence_out_remove(katom); ++ return fd; ++ } ++ ++ fd_install(fd, sync_file->file); ++ ++ return fd; ++} ++ ++int kbase_sync_fence_in_from_fd(struct kbase_jd_atom *katom, int fd) ++{ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ struct fence *fence = sync_file_get_fence(fd); ++#else ++ struct dma_fence *fence = sync_file_get_fence(fd); ++#endif ++ ++ if (!fence) ++ return -ENOENT; ++ ++ kbase_fence_fence_in_set(katom, fence); ++ ++ return 0; ++} ++ ++int kbase_sync_fence_validate(int fd) ++{ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ struct fence *fence = sync_file_get_fence(fd); ++#else ++ struct dma_fence *fence = sync_file_get_fence(fd); ++#endif ++ ++ if (!fence) ++ return -EINVAL; ++ ++ dma_fence_put(fence); ++ ++ return 0; /* valid */ ++} ++ ++enum base_jd_event_code ++kbase_sync_fence_out_trigger(struct kbase_jd_atom *katom, int result) ++{ ++ int res; ++ ++ if (!kbase_fence_out_is_ours(katom)) { ++ /* Not our fence */ ++ return BASE_JD_EVENT_JOB_CANCELLED; ++ } ++ ++ res = kbase_fence_out_signal(katom, result); ++ if (unlikely(res < 0)) { ++ dev_warn(katom->kctx->kbdev->dev, ++ "fence_signal() failed with %d\n", res); ++ } ++ ++ kbase_sync_fence_out_remove(katom); ++ ++ return (result != 0) ? BASE_JD_EVENT_JOB_CANCELLED : BASE_JD_EVENT_DONE; ++} ++ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++static void kbase_fence_wait_callback(struct fence *fence, ++ struct fence_cb *cb) ++#else ++static void kbase_fence_wait_callback(struct dma_fence *fence, ++ struct dma_fence_cb *cb) ++#endif ++{ ++ struct kbase_fence_cb *kcb = container_of(cb, ++ struct kbase_fence_cb, ++ fence_cb); ++ struct kbase_jd_atom *katom = kcb->katom; ++ struct kbase_context *kctx = katom->kctx; ++ ++ /* Cancel atom if fence is erroneous */ ++#if (KERNEL_VERSION(4, 11, 0) <= LINUX_VERSION_CODE || \ ++ (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE && \ ++ KERNEL_VERSION(4, 9, 68) <= LINUX_VERSION_CODE)) ++ if (dma_fence_is_signaled(kcb->fence) && kcb->fence->error) ++#else ++ if (dma_fence_is_signaled(kcb->fence) && kcb->fence->status < 0) ++#endif ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ ++ if (kbase_fence_dep_count_dec_and_test(katom)) { ++ /* We take responsibility of handling this */ ++ kbase_fence_dep_count_set(katom, -1); ++ ++ /* To prevent a potential deadlock we schedule the work onto the ++ * job_done_wq workqueue ++ * ++ * The issue is that we may signal the timeline while holding ++ * kctx->jctx.lock and the callbacks are run synchronously from ++ * sync_timeline_signal. So we simply defer the work. ++ */ ++ INIT_WORK(&katom->work, kbase_sync_fence_wait_worker); ++ queue_work(kctx->jctx.job_done_wq, &katom->work); ++ } ++} ++ ++int kbase_sync_fence_in_wait(struct kbase_jd_atom *katom) ++{ ++ int err; ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ struct fence *fence; ++#else ++ struct dma_fence *fence; ++#endif ++ ++ fence = kbase_fence_in_get(katom); ++ if (!fence) ++ return 0; /* no input fence to wait for, good to go! */ ++ ++ kbase_fence_dep_count_set(katom, 1); ++ ++ err = kbase_fence_add_callback(katom, fence, kbase_fence_wait_callback); ++ ++ kbase_fence_put(fence); ++ ++ if (likely(!err)) { ++ /* Test if the callbacks are already triggered */ ++ if (kbase_fence_dep_count_dec_and_test(katom)) { ++ kbase_fence_free_callbacks(katom); ++ kbase_fence_dep_count_set(katom, -1); ++ return 0; /* Already signaled, good to go right now */ ++ } ++ ++ /* Callback installed, so we just need to wait for it... */ ++ } else { ++ /* Failure */ ++ kbase_fence_free_callbacks(katom); ++ kbase_fence_dep_count_set(katom, -1); ++ ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ ++ /* We should cause the dependent jobs in the bag to be failed, ++ * to do this we schedule the work queue to complete this job */ ++ ++ INIT_WORK(&katom->work, kbase_sync_fence_wait_worker); ++ queue_work(katom->kctx->jctx.job_done_wq, &katom->work); ++ } ++ ++ return 1; /* completion to be done later by callback/worker */ ++} ++ ++void kbase_sync_fence_in_cancel_wait(struct kbase_jd_atom *katom) ++{ ++ if (!kbase_fence_free_callbacks(katom)) { ++ /* The wait wasn't cancelled - ++ * leave the cleanup for kbase_fence_wait_callback */ ++ return; ++ } ++ ++ /* Take responsibility of completion */ ++ kbase_fence_dep_count_set(katom, -1); ++ ++ /* Wait was cancelled - zap the atoms */ ++ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; ++ ++ kbasep_remove_waiting_soft_job(katom); ++ kbase_finish_soft_job(katom); ++ ++ if (jd_done_nolock(katom, NULL)) ++ kbase_js_sched_all(katom->kctx->kbdev); ++} ++ ++void kbase_sync_fence_out_remove(struct kbase_jd_atom *katom) ++{ ++ kbase_fence_out_remove(katom); ++} ++ ++void kbase_sync_fence_in_remove(struct kbase_jd_atom *katom) ++{ ++ kbase_fence_free_callbacks(katom); ++ kbase_fence_in_remove(katom); ++} ++ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++static void kbase_sync_fence_info_get(struct fence *fence, ++ struct kbase_sync_fence_info *info) ++#else ++static void kbase_sync_fence_info_get(struct dma_fence *fence, ++ struct kbase_sync_fence_info *info) ++#endif ++{ ++ info->fence = fence; ++ ++ /* translate into CONFIG_SYNC status: ++ * < 0 : error ++ * 0 : active ++ * 1 : signaled ++ */ ++ if (dma_fence_is_signaled(fence)) { ++#if (KERNEL_VERSION(4, 11, 0) <= LINUX_VERSION_CODE || \ ++ (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE && \ ++ KERNEL_VERSION(4, 9, 68) <= LINUX_VERSION_CODE)) ++ int status = fence->error; ++#else ++ int status = fence->status; ++#endif ++ if (status < 0) ++ info->status = status; /* signaled with error */ ++ else ++ info->status = 1; /* signaled with success */ ++ } else { ++ info->status = 0; /* still active (unsignaled) */ ++ } ++ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0)) ++ scnprintf(info->name, sizeof(info->name), "%u#%u", ++ fence->context, fence->seqno); ++#elif (LINUX_VERSION_CODE < KERNEL_VERSION(5, 1, 0)) ++ scnprintf(info->name, sizeof(info->name), "%llu#%u", ++ fence->context, fence->seqno); ++#else ++ scnprintf(info->name, sizeof(info->name), "%llu#%llu", ++ fence->context, fence->seqno); ++#endif ++} ++ ++int kbase_sync_fence_in_info_get(struct kbase_jd_atom *katom, ++ struct kbase_sync_fence_info *info) ++{ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ struct fence *fence; ++#else ++ struct dma_fence *fence; ++#endif ++ ++ fence = kbase_fence_in_get(katom); ++ if (!fence) ++ return -ENOENT; ++ ++ kbase_sync_fence_info_get(fence, info); ++ ++ kbase_fence_put(fence); ++ ++ return 0; ++} ++ ++int kbase_sync_fence_out_info_get(struct kbase_jd_atom *katom, ++ struct kbase_sync_fence_info *info) ++{ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) ++ struct fence *fence; ++#else ++ struct dma_fence *fence; ++#endif ++ ++ fence = kbase_fence_out_get(katom); ++ if (!fence) ++ return -ENOENT; ++ ++ kbase_sync_fence_info_get(fence, info); ++ ++ kbase_fence_put(fence); ++ ++ return 0; ++} ++ ++ ++#ifdef CONFIG_MALI_FENCE_DEBUG ++void kbase_sync_fence_in_dump(struct kbase_jd_atom *katom) ++{ ++ /* Not implemented */ ++} ++#endif +diff --git a/drivers/gpu/arm/midgard/mali_kbase_tlstream.c b/drivers/gpu/arm/midgard/mali_kbase_tlstream.c +new file mode 100755 +index 000000000000..c8310c45f143 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_tlstream.c +@@ -0,0 +1,2572 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#include ++#include ++ ++/*****************************************************************************/ ++ ++/* The version of swtrace protocol used in timeline stream. */ ++#define SWTRACE_VERSION 3 ++ ++/* The maximum expected length of string in tracepoint descriptor. */ ++#define STRLEN_MAX 64 /* bytes */ ++ ++/* The number of nanoseconds in a second. */ ++#define NSECS_IN_SEC 1000000000ull /* ns */ ++ ++/* The period of autoflush checker execution in milliseconds. */ ++#define AUTOFLUSH_INTERVAL 1000 /* ms */ ++ ++/* The maximum size of a single packet used by timeline. */ ++#define PACKET_SIZE 4096 /* bytes */ ++ ++/* The number of packets used by one timeline stream. */ ++#define PACKET_COUNT 16 ++ ++/* The number of bytes reserved for packet header. ++ * These value must be defined according to MIPE documentation. */ ++#define PACKET_HEADER_SIZE 8 /* bytes */ ++ ++/* The number of bytes reserved for packet sequence number. ++ * These value must be defined according to MIPE documentation. */ ++#define PACKET_NUMBER_SIZE 4 /* bytes */ ++ ++/* Packet header - first word. ++ * These values must be defined according to MIPE documentation. */ ++#define PACKET_STREAMID_POS 0 ++#define PACKET_STREAMID_LEN 8 ++#define PACKET_RSVD1_POS (PACKET_STREAMID_POS + PACKET_STREAMID_LEN) ++#define PACKET_RSVD1_LEN 8 ++#define PACKET_TYPE_POS (PACKET_RSVD1_POS + PACKET_RSVD1_LEN) ++#define PACKET_TYPE_LEN 3 ++#define PACKET_CLASS_POS (PACKET_TYPE_POS + PACKET_TYPE_LEN) ++#define PACKET_CLASS_LEN 7 ++#define PACKET_FAMILY_POS (PACKET_CLASS_POS + PACKET_CLASS_LEN) ++#define PACKET_FAMILY_LEN 6 ++ ++/* Packet header - second word ++ * These values must be defined according to MIPE documentation. */ ++#define PACKET_LENGTH_POS 0 ++#define PACKET_LENGTH_LEN 24 ++#define PACKET_SEQBIT_POS (PACKET_LENGTH_POS + PACKET_LENGTH_LEN) ++#define PACKET_SEQBIT_LEN 1 ++#define PACKET_RSVD2_POS (PACKET_SEQBIT_POS + PACKET_SEQBIT_LEN) ++#define PACKET_RSVD2_LEN 7 ++ ++/* Types of streams generated by timeline. ++ * Order is significant! Header streams must precede respective body streams. */ ++enum tl_stream_type { ++ TL_STREAM_TYPE_OBJ_HEADER, ++ TL_STREAM_TYPE_OBJ_SUMMARY, ++ TL_STREAM_TYPE_OBJ, ++ TL_STREAM_TYPE_AUX_HEADER, ++ TL_STREAM_TYPE_AUX, ++ ++ TL_STREAM_TYPE_COUNT ++}; ++ ++/* Timeline packet family ids. ++ * Values are significant! Check MIPE documentation. */ ++enum tl_packet_family { ++ TL_PACKET_FAMILY_CTRL = 0, /* control packets */ ++ TL_PACKET_FAMILY_TL = 1, /* timeline packets */ ++ ++ TL_PACKET_FAMILY_COUNT ++}; ++ ++/* Packet classes used in timeline streams. ++ * Values are significant! Check MIPE documentation. */ ++enum tl_packet_class { ++ TL_PACKET_CLASS_OBJ = 0, /* timeline objects packet */ ++ TL_PACKET_CLASS_AUX = 1, /* auxiliary events packet */ ++}; ++ ++/* Packet types used in timeline streams. ++ * Values are significant! Check MIPE documentation. */ ++enum tl_packet_type { ++ TL_PACKET_TYPE_HEADER = 0, /* stream's header/directory */ ++ TL_PACKET_TYPE_BODY = 1, /* stream's body */ ++ TL_PACKET_TYPE_SUMMARY = 2, /* stream's summary */ ++}; ++ ++/* Message ids of trace events that are recorded in the timeline stream. */ ++enum tl_msg_id_obj { ++ /* Timeline object events. */ ++ KBASE_TL_NEW_CTX, ++ KBASE_TL_NEW_GPU, ++ KBASE_TL_NEW_LPU, ++ KBASE_TL_NEW_ATOM, ++ KBASE_TL_NEW_AS, ++ KBASE_TL_DEL_CTX, ++ KBASE_TL_DEL_ATOM, ++ KBASE_TL_LIFELINK_LPU_GPU, ++ KBASE_TL_LIFELINK_AS_GPU, ++ KBASE_TL_RET_CTX_LPU, ++ KBASE_TL_RET_ATOM_CTX, ++ KBASE_TL_RET_ATOM_LPU, ++ KBASE_TL_NRET_CTX_LPU, ++ KBASE_TL_NRET_ATOM_CTX, ++ KBASE_TL_NRET_ATOM_LPU, ++ KBASE_TL_RET_AS_CTX, ++ KBASE_TL_NRET_AS_CTX, ++ KBASE_TL_RET_ATOM_AS, ++ KBASE_TL_NRET_ATOM_AS, ++ KBASE_TL_DEP_ATOM_ATOM, ++ KBASE_TL_NDEP_ATOM_ATOM, ++ KBASE_TL_RDEP_ATOM_ATOM, ++ KBASE_TL_ATTRIB_ATOM_CONFIG, ++ KBASE_TL_ATTRIB_ATOM_PRIORITY, ++ KBASE_TL_ATTRIB_ATOM_STATE, ++ KBASE_TL_ATTRIB_ATOM_PRIORITY_CHANGE, ++ KBASE_TL_ATTRIB_ATOM_JIT, ++ KBASE_TL_ATTRIB_AS_CONFIG, ++ KBASE_TL_EVENT_LPU_SOFTSTOP, ++ KBASE_TL_EVENT_ATOM_SOFTSTOP_EX, ++ KBASE_TL_EVENT_ATOM_SOFTSTOP_ISSUE, ++ ++ /* Job dump specific events. */ ++ KBASE_JD_GPU_SOFT_RESET ++}; ++ ++/* Message ids of trace events that are recorded in the auxiliary stream. */ ++enum tl_msg_id_aux { ++ KBASE_AUX_PM_STATE, ++ KBASE_AUX_PAGEFAULT, ++ KBASE_AUX_PAGESALLOC, ++ KBASE_AUX_DEVFREQ_TARGET, ++ KBASE_AUX_PROTECTED_ENTER_START, ++ KBASE_AUX_PROTECTED_ENTER_END, ++ KBASE_AUX_PROTECTED_LEAVE_START, ++ KBASE_AUX_PROTECTED_LEAVE_END ++}; ++ ++/*****************************************************************************/ ++ ++/** ++ * struct tl_stream - timeline stream structure ++ * @lock: message order lock ++ * @buffer: array of buffers ++ * @wbi: write buffer index ++ * @rbi: read buffer index ++ * @numbered: if non-zero stream's packets are sequentially numbered ++ * @autoflush_counter: counter tracking stream's autoflush state ++ * ++ * This structure holds information needed to construct proper packets in the ++ * timeline stream. Each message in sequence must bear timestamp that is greater ++ * to one in previous message in the same stream. For this reason lock is held ++ * throughout the process of message creation. Each stream contains set of ++ * buffers. Each buffer will hold one MIPE packet. In case there is no free ++ * space required to store incoming message the oldest buffer is discarded. ++ * Each packet in timeline body stream has sequence number embedded (this value ++ * must increment monotonically and is used by packets receiver to discover ++ * buffer overflows. ++ * Autoflush counter is set to negative number when there is no data pending ++ * for flush and it is set to zero on every update of the buffer. Autoflush ++ * timer will increment the counter by one on every expiry. In case there will ++ * be no activity on the buffer during two consecutive timer expiries, stream ++ * buffer will be flushed. ++ */ ++struct tl_stream { ++ spinlock_t lock; ++ ++ struct { ++ atomic_t size; /* number of bytes in buffer */ ++ char data[PACKET_SIZE]; /* buffer's data */ ++ } buffer[PACKET_COUNT]; ++ ++ atomic_t wbi; ++ atomic_t rbi; ++ ++ int numbered; ++ atomic_t autoflush_counter; ++}; ++ ++/** ++ * struct tp_desc - tracepoint message descriptor structure ++ * @id: tracepoint ID identifying message in stream ++ * @id_str: human readable version of tracepoint ID ++ * @name: tracepoint description ++ * @arg_types: tracepoint's arguments types declaration ++ * @arg_names: comma separated list of tracepoint's arguments names ++ */ ++struct tp_desc { ++ u32 id; ++ const char *id_str; ++ const char *name; ++ const char *arg_types; ++ const char *arg_names; ++}; ++ ++/*****************************************************************************/ ++ ++/* Configuration of timeline streams generated by kernel. ++ * Kernel emit only streams containing either timeline object events or ++ * auxiliary events. All streams have stream id value of 1 (as opposed to user ++ * space streams that have value of 0). */ ++static const struct { ++ enum tl_packet_family pkt_family; ++ enum tl_packet_class pkt_class; ++ enum tl_packet_type pkt_type; ++ unsigned int stream_id; ++} tl_stream_cfg[TL_STREAM_TYPE_COUNT] = { ++ {TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_OBJ, TL_PACKET_TYPE_HEADER, 1}, ++ {TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_OBJ, TL_PACKET_TYPE_SUMMARY, 1}, ++ {TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_OBJ, TL_PACKET_TYPE_BODY, 1}, ++ {TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_AUX, TL_PACKET_TYPE_HEADER, 1}, ++ {TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_AUX, TL_PACKET_TYPE_BODY, 1} ++}; ++ ++/* The timeline streams generated by kernel. */ ++static struct tl_stream *tl_stream[TL_STREAM_TYPE_COUNT]; ++ ++/* Autoflush timer. */ ++static struct timer_list autoflush_timer; ++ ++/* If non-zero autoflush timer is active. */ ++static atomic_t autoflush_timer_active; ++ ++/* Reader lock. Only one reader is allowed to have access to the timeline ++ * streams at any given time. */ ++static DEFINE_MUTEX(tl_reader_lock); ++ ++/* Timeline stream event queue. */ ++static DECLARE_WAIT_QUEUE_HEAD(tl_event_queue); ++ ++/* The timeline stream file operations functions. */ ++static ssize_t kbasep_tlstream_read( ++ struct file *filp, ++ char __user *buffer, ++ size_t size, ++ loff_t *f_pos); ++static unsigned int kbasep_tlstream_poll(struct file *filp, poll_table *wait); ++static int kbasep_tlstream_release(struct inode *inode, struct file *filp); ++ ++/* The timeline stream file operations structure. */ ++static const struct file_operations kbasep_tlstream_fops = { ++ .release = kbasep_tlstream_release, ++ .read = kbasep_tlstream_read, ++ .poll = kbasep_tlstream_poll, ++}; ++ ++/* Descriptors of timeline messages transmitted in object events stream. */ ++static const struct tp_desc tp_desc_obj[] = { ++ { ++ KBASE_TL_NEW_CTX, ++ __stringify(KBASE_TL_NEW_CTX), ++ "object ctx is created", ++ "@pII", ++ "ctx,ctx_nr,tgid" ++ }, ++ { ++ KBASE_TL_NEW_GPU, ++ __stringify(KBASE_TL_NEW_GPU), ++ "object gpu is created", ++ "@pII", ++ "gpu,gpu_id,core_count" ++ }, ++ { ++ KBASE_TL_NEW_LPU, ++ __stringify(KBASE_TL_NEW_LPU), ++ "object lpu is created", ++ "@pII", ++ "lpu,lpu_nr,lpu_fn" ++ }, ++ { ++ KBASE_TL_NEW_ATOM, ++ __stringify(KBASE_TL_NEW_ATOM), ++ "object atom is created", ++ "@pI", ++ "atom,atom_nr" ++ }, ++ { ++ KBASE_TL_NEW_AS, ++ __stringify(KBASE_TL_NEW_AS), ++ "address space object is created", ++ "@pI", ++ "address_space,as_nr" ++ }, ++ { ++ KBASE_TL_DEL_CTX, ++ __stringify(KBASE_TL_DEL_CTX), ++ "context is destroyed", ++ "@p", ++ "ctx" ++ }, ++ { ++ KBASE_TL_DEL_ATOM, ++ __stringify(KBASE_TL_DEL_ATOM), ++ "atom is destroyed", ++ "@p", ++ "atom" ++ }, ++ { ++ KBASE_TL_LIFELINK_LPU_GPU, ++ __stringify(KBASE_TL_LIFELINK_LPU_GPU), ++ "lpu is deleted with gpu", ++ "@pp", ++ "lpu,gpu" ++ }, ++ { ++ KBASE_TL_LIFELINK_AS_GPU, ++ __stringify(KBASE_TL_LIFELINK_AS_GPU), ++ "address space is deleted with gpu", ++ "@pp", ++ "address_space,gpu" ++ }, ++ { ++ KBASE_TL_RET_CTX_LPU, ++ __stringify(KBASE_TL_RET_CTX_LPU), ++ "context is retained by lpu", ++ "@pp", ++ "ctx,lpu" ++ }, ++ { ++ KBASE_TL_RET_ATOM_CTX, ++ __stringify(KBASE_TL_RET_ATOM_CTX), ++ "atom is retained by context", ++ "@pp", ++ "atom,ctx" ++ }, ++ { ++ KBASE_TL_RET_ATOM_LPU, ++ __stringify(KBASE_TL_RET_ATOM_LPU), ++ "atom is retained by lpu", ++ "@pps", ++ "atom,lpu,attrib_match_list" ++ }, ++ { ++ KBASE_TL_NRET_CTX_LPU, ++ __stringify(KBASE_TL_NRET_CTX_LPU), ++ "context is released by lpu", ++ "@pp", ++ "ctx,lpu" ++ }, ++ { ++ KBASE_TL_NRET_ATOM_CTX, ++ __stringify(KBASE_TL_NRET_ATOM_CTX), ++ "atom is released by context", ++ "@pp", ++ "atom,ctx" ++ }, ++ { ++ KBASE_TL_NRET_ATOM_LPU, ++ __stringify(KBASE_TL_NRET_ATOM_LPU), ++ "atom is released by lpu", ++ "@pp", ++ "atom,lpu" ++ }, ++ { ++ KBASE_TL_RET_AS_CTX, ++ __stringify(KBASE_TL_RET_AS_CTX), ++ "address space is retained by context", ++ "@pp", ++ "address_space,ctx" ++ }, ++ { ++ KBASE_TL_NRET_AS_CTX, ++ __stringify(KBASE_TL_NRET_AS_CTX), ++ "address space is released by context", ++ "@pp", ++ "address_space,ctx" ++ }, ++ { ++ KBASE_TL_RET_ATOM_AS, ++ __stringify(KBASE_TL_RET_ATOM_AS), ++ "atom is retained by address space", ++ "@pp", ++ "atom,address_space" ++ }, ++ { ++ KBASE_TL_NRET_ATOM_AS, ++ __stringify(KBASE_TL_NRET_ATOM_AS), ++ "atom is released by address space", ++ "@pp", ++ "atom,address_space" ++ }, ++ { ++ KBASE_TL_DEP_ATOM_ATOM, ++ __stringify(KBASE_TL_DEP_ATOM_ATOM), ++ "atom2 depends on atom1", ++ "@pp", ++ "atom1,atom2" ++ }, ++ { ++ KBASE_TL_NDEP_ATOM_ATOM, ++ __stringify(KBASE_TL_NDEP_ATOM_ATOM), ++ "atom2 no longer depends on atom1", ++ "@pp", ++ "atom1,atom2" ++ }, ++ { ++ KBASE_TL_RDEP_ATOM_ATOM, ++ __stringify(KBASE_TL_RDEP_ATOM_ATOM), ++ "resolved dependecy of atom2 depending on atom1", ++ "@pp", ++ "atom1,atom2" ++ }, ++ { ++ KBASE_TL_ATTRIB_ATOM_CONFIG, ++ __stringify(KBASE_TL_ATTRIB_ATOM_CONFIG), ++ "atom job slot attributes", ++ "@pLLI", ++ "atom,descriptor,affinity,config" ++ }, ++ { ++ KBASE_TL_ATTRIB_ATOM_PRIORITY, ++ __stringify(KBASE_TL_ATTRIB_ATOM_PRIORITY), ++ "atom priority", ++ "@pI", ++ "atom,prio" ++ }, ++ { ++ KBASE_TL_ATTRIB_ATOM_STATE, ++ __stringify(KBASE_TL_ATTRIB_ATOM_STATE), ++ "atom state", ++ "@pI", ++ "atom,state" ++ }, ++ { ++ KBASE_TL_ATTRIB_ATOM_PRIORITY_CHANGE, ++ __stringify(KBASE_TL_ATTRIB_ATOM_PRIORITY_CHANGE), ++ "atom caused priority change", ++ "@p", ++ "atom" ++ }, ++ { ++ KBASE_TL_ATTRIB_ATOM_JIT, ++ __stringify(KBASE_TL_ATTRIB_ATOM_JIT), ++ "jit done for atom", ++ "@pLL", ++ "atom,edit_addr,new_addr" ++ }, ++ { ++ KBASE_TL_ATTRIB_AS_CONFIG, ++ __stringify(KBASE_TL_ATTRIB_AS_CONFIG), ++ "address space attributes", ++ "@pLLL", ++ "address_space,transtab,memattr,transcfg" ++ }, ++ { ++ KBASE_TL_EVENT_LPU_SOFTSTOP, ++ __stringify(KBASE_TL_EVENT_LPU_SOFTSTOP), ++ "softstop event on given lpu", ++ "@p", ++ "lpu" ++ }, ++ { ++ KBASE_TL_EVENT_ATOM_SOFTSTOP_EX, ++ __stringify(KBASE_TL_EVENT_ATOM_SOFTSTOP_EX), ++ "atom softstopped", ++ "@p", ++ "atom" ++ }, ++ { ++ KBASE_TL_EVENT_ATOM_SOFTSTOP_ISSUE, ++ __stringify(KBASE_TL_EVENT_SOFTSTOP_ISSUE), ++ "atom softstop issued", ++ "@p", ++ "atom" ++ }, ++ { ++ KBASE_JD_GPU_SOFT_RESET, ++ __stringify(KBASE_JD_GPU_SOFT_RESET), ++ "gpu soft reset", ++ "@p", ++ "gpu" ++ }, ++}; ++ ++/* Descriptors of timeline messages transmitted in auxiliary events stream. */ ++static const struct tp_desc tp_desc_aux[] = { ++ { ++ KBASE_AUX_PM_STATE, ++ __stringify(KBASE_AUX_PM_STATE), ++ "PM state", ++ "@IL", ++ "core_type,core_state_bitset" ++ }, ++ { ++ KBASE_AUX_PAGEFAULT, ++ __stringify(KBASE_AUX_PAGEFAULT), ++ "Page fault", ++ "@IL", ++ "ctx_nr,page_cnt_change" ++ }, ++ { ++ KBASE_AUX_PAGESALLOC, ++ __stringify(KBASE_AUX_PAGESALLOC), ++ "Total alloc pages change", ++ "@IL", ++ "ctx_nr,page_cnt" ++ }, ++ { ++ KBASE_AUX_DEVFREQ_TARGET, ++ __stringify(KBASE_AUX_DEVFREQ_TARGET), ++ "New device frequency target", ++ "@L", ++ "target_freq" ++ }, ++ { ++ KBASE_AUX_PROTECTED_ENTER_START, ++ __stringify(KBASE_AUX_PROTECTED_ENTER_START), ++ "enter protected mode start", ++ "@p", ++ "gpu" ++ }, ++ { ++ KBASE_AUX_PROTECTED_ENTER_END, ++ __stringify(KBASE_AUX_PROTECTED_ENTER_END), ++ "enter protected mode end", ++ "@p", ++ "gpu" ++ }, ++ { ++ KBASE_AUX_PROTECTED_LEAVE_START, ++ __stringify(KBASE_AUX_PROTECTED_LEAVE_START), ++ "leave protected mode start", ++ "@p", ++ "gpu" ++ }, ++ { ++ KBASE_AUX_PROTECTED_LEAVE_END, ++ __stringify(KBASE_AUX_PROTECTED_LEAVE_END), ++ "leave protected mode end", ++ "@p", ++ "gpu" ++ } ++}; ++ ++#if MALI_UNIT_TEST ++/* Number of bytes read by user. */ ++static atomic_t tlstream_bytes_collected = {0}; ++ ++/* Number of bytes generated by tracepoint messages. */ ++static atomic_t tlstream_bytes_generated = {0}; ++#endif /* MALI_UNIT_TEST */ ++ ++/*****************************************************************************/ ++ ++/* Indicator of whether the timeline stream file descriptor is used. */ ++atomic_t kbase_tlstream_enabled = {0}; ++ ++/*****************************************************************************/ ++ ++/** ++ * kbasep_tlstream_get_timestamp - return timestamp ++ * ++ * Function returns timestamp value based on raw monotonic timer. Value will ++ * wrap around zero in case of overflow. ++ * Return: timestamp value ++ */ ++static u64 kbasep_tlstream_get_timestamp(void) ++{ ++ struct timespec64 ts; ++ u64 timestamp; ++ ++ ktime_get_raw_ts64(&ts); ++ timestamp = (u64)ts.tv_sec * NSECS_IN_SEC + ts.tv_nsec; ++ return timestamp; ++} ++ ++/** ++ * kbasep_tlstream_write_bytes - write data to message buffer ++ * @buffer: buffer where data will be written ++ * @pos: position in the buffer where to place data ++ * @bytes: pointer to buffer holding data ++ * @len: length of data to be written ++ * ++ * Return: updated position in the buffer ++ */ ++static size_t kbasep_tlstream_write_bytes( ++ char *buffer, ++ size_t pos, ++ const void *bytes, ++ size_t len) ++{ ++ KBASE_DEBUG_ASSERT(buffer); ++ KBASE_DEBUG_ASSERT(bytes); ++ ++ memcpy(&buffer[pos], bytes, len); ++ ++ return pos + len; ++} ++ ++/** ++ * kbasep_tlstream_write_string - write string to message buffer ++ * @buffer: buffer where data will be written ++ * @pos: position in the buffer where to place data ++ * @string: pointer to buffer holding the source string ++ * @max_write_size: number of bytes that can be stored in buffer ++ * ++ * Return: updated position in the buffer ++ */ ++static size_t kbasep_tlstream_write_string( ++ char *buffer, ++ size_t pos, ++ const char *string, ++ size_t max_write_size) ++{ ++ u32 string_len; ++ ++ KBASE_DEBUG_ASSERT(buffer); ++ KBASE_DEBUG_ASSERT(string); ++ /* Timeline string consists of at least string length and nul ++ * terminator. */ ++ KBASE_DEBUG_ASSERT(max_write_size >= sizeof(string_len) + sizeof(char)); ++ max_write_size -= sizeof(string_len); ++ ++ string_len = strlcpy( ++ &buffer[pos + sizeof(string_len)], ++ string, ++ max_write_size); ++ string_len += sizeof(char); ++ ++ /* Make sure that the source string fit into the buffer. */ ++ KBASE_DEBUG_ASSERT(string_len <= max_write_size); ++ ++ /* Update string length. */ ++ memcpy(&buffer[pos], &string_len, sizeof(string_len)); ++ ++ return pos + sizeof(string_len) + string_len; ++} ++ ++/** ++ * kbasep_tlstream_write_timestamp - write timestamp to message buffer ++ * @buffer: buffer where data will be written ++ * @pos: position in the buffer where to place data ++ * ++ * Return: updated position in the buffer ++ */ ++static size_t kbasep_tlstream_write_timestamp(void *buffer, size_t pos) ++{ ++ u64 timestamp = kbasep_tlstream_get_timestamp(); ++ ++ return kbasep_tlstream_write_bytes( ++ buffer, pos, ++ ×tamp, sizeof(timestamp)); ++} ++ ++/** ++ * kbasep_tlstream_put_bits - put bits in a word ++ * @word: pointer to the words being modified ++ * @value: value that shall be written to given position ++ * @bitpos: position where value shall be written (in bits) ++ * @bitlen: length of value (in bits) ++ */ ++static void kbasep_tlstream_put_bits( ++ u32 *word, ++ u32 value, ++ unsigned int bitpos, ++ unsigned int bitlen) ++{ ++ const u32 mask = ((1 << bitlen) - 1) << bitpos; ++ ++ KBASE_DEBUG_ASSERT(word); ++ KBASE_DEBUG_ASSERT((0 != bitlen) && (32 >= bitlen)); ++ KBASE_DEBUG_ASSERT((bitpos + bitlen) <= 32); ++ ++ *word &= ~mask; ++ *word |= ((value << bitpos) & mask); ++} ++ ++/** ++ * kbasep_tlstream_packet_header_setup - setup the packet header ++ * @buffer: pointer to the buffer ++ * @pkt_family: packet's family ++ * @pkt_type: packet's type ++ * @pkt_class: packet's class ++ * @stream_id: stream id ++ * @numbered: non-zero if this stream is numbered ++ * ++ * Function sets up immutable part of packet header in the given buffer. ++ */ ++static void kbasep_tlstream_packet_header_setup( ++ char *buffer, ++ enum tl_packet_family pkt_family, ++ enum tl_packet_class pkt_class, ++ enum tl_packet_type pkt_type, ++ unsigned int stream_id, ++ int numbered) ++{ ++ u32 word0 = 0; ++ u32 word1 = 0; ++ ++ KBASE_DEBUG_ASSERT(buffer); ++ KBASE_DEBUG_ASSERT(pkt_family == TL_PACKET_FAMILY_TL); ++ KBASE_DEBUG_ASSERT( ++ (pkt_type == TL_PACKET_TYPE_HEADER) || ++ (pkt_type == TL_PACKET_TYPE_SUMMARY) || ++ (pkt_type == TL_PACKET_TYPE_BODY)); ++ KBASE_DEBUG_ASSERT( ++ (pkt_class == TL_PACKET_CLASS_OBJ) || ++ (pkt_class == TL_PACKET_CLASS_AUX)); ++ ++ kbasep_tlstream_put_bits( ++ &word0, pkt_family, ++ PACKET_FAMILY_POS, PACKET_FAMILY_LEN); ++ kbasep_tlstream_put_bits( ++ &word0, pkt_class, ++ PACKET_CLASS_POS, PACKET_CLASS_LEN); ++ kbasep_tlstream_put_bits( ++ &word0, pkt_type, ++ PACKET_TYPE_POS, PACKET_TYPE_LEN); ++ kbasep_tlstream_put_bits( ++ &word0, stream_id, ++ PACKET_STREAMID_POS, PACKET_STREAMID_LEN); ++ ++ if (numbered) ++ kbasep_tlstream_put_bits( ++ &word1, 1, ++ PACKET_SEQBIT_POS, PACKET_SEQBIT_LEN); ++ ++ memcpy(&buffer[0], &word0, sizeof(word0)); ++ memcpy(&buffer[sizeof(word0)], &word1, sizeof(word1)); ++} ++ ++/** ++ * kbasep_tlstream_packet_header_update - update the packet header ++ * @buffer: pointer to the buffer ++ * @data_size: amount of data carried in this packet ++ * ++ * Function updates mutable part of packet header in the given buffer. ++ * Note that value of data_size must not including size of the header. ++ */ ++static void kbasep_tlstream_packet_header_update( ++ char *buffer, ++ size_t data_size) ++{ ++ u32 word0; ++ u32 word1; ++ ++ KBASE_DEBUG_ASSERT(buffer); ++ CSTD_UNUSED(word0); ++ ++ memcpy(&word1, &buffer[sizeof(word0)], sizeof(word1)); ++ ++ kbasep_tlstream_put_bits( ++ &word1, data_size, ++ PACKET_LENGTH_POS, PACKET_LENGTH_LEN); ++ ++ memcpy(&buffer[sizeof(word0)], &word1, sizeof(word1)); ++} ++ ++/** ++ * kbasep_tlstream_packet_number_update - update the packet number ++ * @buffer: pointer to the buffer ++ * @counter: value of packet counter for this packet's stream ++ * ++ * Function updates packet number embedded within the packet placed in the ++ * given buffer. ++ */ ++static void kbasep_tlstream_packet_number_update(char *buffer, u32 counter) ++{ ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ memcpy(&buffer[PACKET_HEADER_SIZE], &counter, sizeof(counter)); ++} ++ ++/** ++ * kbasep_timeline_stream_reset - reset stream ++ * @stream: pointer to the stream structure ++ * ++ * Function discards all pending messages and resets packet counters. ++ */ ++static void kbasep_timeline_stream_reset(struct tl_stream *stream) ++{ ++ unsigned int i; ++ ++ for (i = 0; i < PACKET_COUNT; i++) { ++ if (stream->numbered) ++ atomic_set( ++ &stream->buffer[i].size, ++ PACKET_HEADER_SIZE + ++ PACKET_NUMBER_SIZE); ++ else ++ atomic_set(&stream->buffer[i].size, PACKET_HEADER_SIZE); ++ } ++ ++ atomic_set(&stream->wbi, 0); ++ atomic_set(&stream->rbi, 0); ++} ++ ++/** ++ * kbasep_timeline_stream_init - initialize timeline stream ++ * @stream: pointer to the stream structure ++ * @stream_type: stream type ++ */ ++static void kbasep_timeline_stream_init( ++ struct tl_stream *stream, ++ enum tl_stream_type stream_type) ++{ ++ unsigned int i; ++ ++ KBASE_DEBUG_ASSERT(stream); ++ KBASE_DEBUG_ASSERT(TL_STREAM_TYPE_COUNT > stream_type); ++ ++ spin_lock_init(&stream->lock); ++ ++ /* All packets carrying tracepoints shall be numbered. */ ++ if (TL_PACKET_TYPE_BODY == tl_stream_cfg[stream_type].pkt_type) ++ stream->numbered = 1; ++ else ++ stream->numbered = 0; ++ ++ for (i = 0; i < PACKET_COUNT; i++) ++ kbasep_tlstream_packet_header_setup( ++ stream->buffer[i].data, ++ tl_stream_cfg[stream_type].pkt_family, ++ tl_stream_cfg[stream_type].pkt_class, ++ tl_stream_cfg[stream_type].pkt_type, ++ tl_stream_cfg[stream_type].stream_id, ++ stream->numbered); ++ ++ kbasep_timeline_stream_reset(tl_stream[stream_type]); ++} ++ ++/** ++ * kbasep_timeline_stream_term - terminate timeline stream ++ * @stream: pointer to the stream structure ++ */ ++static void kbasep_timeline_stream_term(struct tl_stream *stream) ++{ ++ KBASE_DEBUG_ASSERT(stream); ++} ++ ++/** ++ * kbasep_tlstream_msgbuf_submit - submit packet to the user space ++ * @stream: pointer to the stream structure ++ * @wb_idx_raw: write buffer index ++ * @wb_size: length of data stored in current buffer ++ * ++ * Function updates currently written buffer with packet header. Then write ++ * index is incremented and buffer is handled to user space. Parameters ++ * of new buffer are returned using provided arguments. ++ * ++ * Return: length of data in new buffer ++ * ++ * Warning: User must update the stream structure with returned value. ++ */ ++static size_t kbasep_tlstream_msgbuf_submit( ++ struct tl_stream *stream, ++ unsigned int wb_idx_raw, ++ unsigned int wb_size) ++{ ++ unsigned int rb_idx_raw = atomic_read(&stream->rbi); ++ unsigned int wb_idx = wb_idx_raw % PACKET_COUNT; ++ ++ /* Set stream as flushed. */ ++ atomic_set(&stream->autoflush_counter, -1); ++ ++ kbasep_tlstream_packet_header_update( ++ stream->buffer[wb_idx].data, ++ wb_size - PACKET_HEADER_SIZE); ++ ++ if (stream->numbered) ++ kbasep_tlstream_packet_number_update( ++ stream->buffer[wb_idx].data, ++ wb_idx_raw); ++ ++ /* Increasing write buffer index will expose this packet to the reader. ++ * As stream->lock is not taken on reader side we must make sure memory ++ * is updated correctly before this will happen. */ ++ smp_wmb(); ++ wb_idx_raw++; ++ atomic_set(&stream->wbi, wb_idx_raw); ++ ++ /* Inform user that packets are ready for reading. */ ++ wake_up_interruptible(&tl_event_queue); ++ ++ /* Detect and mark overflow in this stream. */ ++ if (PACKET_COUNT == wb_idx_raw - rb_idx_raw) { ++ /* Reader side depends on this increment to correctly handle ++ * overflows. The value shall be updated only if it was not ++ * modified by the reader. The data holding buffer will not be ++ * updated before stream->lock is released, however size of the ++ * buffer will. Make sure this increment is globally visible ++ * before information about selected write buffer size. */ ++ atomic_cmpxchg(&stream->rbi, rb_idx_raw, rb_idx_raw + 1); ++ } ++ ++ wb_size = PACKET_HEADER_SIZE; ++ if (stream->numbered) ++ wb_size += PACKET_NUMBER_SIZE; ++ ++ return wb_size; ++} ++ ++/** ++ * kbasep_tlstream_msgbuf_acquire - lock selected stream and reserves buffer ++ * @stream_type: type of the stream that shall be locked ++ * @msg_size: message size ++ * @flags: pointer to store flags passed back on stream release ++ * ++ * Function will lock the stream and reserve the number of bytes requested ++ * in msg_size for the user. ++ * ++ * Return: pointer to the buffer where message can be stored ++ * ++ * Warning: Stream must be released with kbasep_tlstream_msgbuf_release(). ++ * Only atomic operations are allowed while stream is locked ++ * (i.e. do not use any operation that may sleep). ++ */ ++static char *kbasep_tlstream_msgbuf_acquire( ++ enum tl_stream_type stream_type, ++ size_t msg_size, ++ unsigned long *flags) __acquires(&stream->lock) ++{ ++ struct tl_stream *stream; ++ unsigned int wb_idx_raw; ++ unsigned int wb_idx; ++ size_t wb_size; ++ ++ KBASE_DEBUG_ASSERT(TL_STREAM_TYPE_COUNT > stream_type); ++ KBASE_DEBUG_ASSERT( ++ PACKET_SIZE - PACKET_HEADER_SIZE - PACKET_NUMBER_SIZE >= ++ msg_size); ++ ++ stream = tl_stream[stream_type]; ++ ++ spin_lock_irqsave(&stream->lock, *flags); ++ ++ wb_idx_raw = atomic_read(&stream->wbi); ++ wb_idx = wb_idx_raw % PACKET_COUNT; ++ wb_size = atomic_read(&stream->buffer[wb_idx].size); ++ ++ /* Select next buffer if data will not fit into current one. */ ++ if (PACKET_SIZE < wb_size + msg_size) { ++ wb_size = kbasep_tlstream_msgbuf_submit( ++ stream, wb_idx_raw, wb_size); ++ wb_idx = (wb_idx_raw + 1) % PACKET_COUNT; ++ } ++ ++ /* Reserve space in selected buffer. */ ++ atomic_set(&stream->buffer[wb_idx].size, wb_size + msg_size); ++ ++#if MALI_UNIT_TEST ++ atomic_add(msg_size, &tlstream_bytes_generated); ++#endif /* MALI_UNIT_TEST */ ++ ++ return &stream->buffer[wb_idx].data[wb_size]; ++} ++ ++/** ++ * kbasep_tlstream_msgbuf_release - unlock selected stream ++ * @stream_type: type of the stream that shall be locked ++ * @flags: value obtained during stream acquire ++ * ++ * Function releases stream that has been previously locked with a call to ++ * kbasep_tlstream_msgbuf_acquire(). ++ */ ++static void kbasep_tlstream_msgbuf_release( ++ enum tl_stream_type stream_type, ++ unsigned long flags) __releases(&stream->lock) ++{ ++ struct tl_stream *stream; ++ ++ KBASE_DEBUG_ASSERT(TL_STREAM_TYPE_COUNT > stream_type); ++ ++ stream = tl_stream[stream_type]; ++ ++ /* Mark stream as containing unflushed data. */ ++ atomic_set(&stream->autoflush_counter, 0); ++ ++ spin_unlock_irqrestore(&stream->lock, flags); ++} ++ ++/*****************************************************************************/ ++ ++/** ++ * kbasep_tlstream_flush_stream - flush stream ++ * @stype: type of stream to be flushed ++ * ++ * Flush pending data in timeline stream. ++ */ ++static void kbasep_tlstream_flush_stream(enum tl_stream_type stype) ++{ ++ struct tl_stream *stream = tl_stream[stype]; ++ unsigned long flags; ++ unsigned int wb_idx_raw; ++ unsigned int wb_idx; ++ size_t wb_size; ++ size_t min_size = PACKET_HEADER_SIZE; ++ ++ if (stream->numbered) ++ min_size += PACKET_NUMBER_SIZE; ++ ++ spin_lock_irqsave(&stream->lock, flags); ++ ++ wb_idx_raw = atomic_read(&stream->wbi); ++ wb_idx = wb_idx_raw % PACKET_COUNT; ++ wb_size = atomic_read(&stream->buffer[wb_idx].size); ++ ++ if (wb_size > min_size) { ++ wb_size = kbasep_tlstream_msgbuf_submit( ++ stream, wb_idx_raw, wb_size); ++ wb_idx = (wb_idx_raw + 1) % PACKET_COUNT; ++ atomic_set(&stream->buffer[wb_idx].size, wb_size); ++ } ++ spin_unlock_irqrestore(&stream->lock, flags); ++} ++ ++/** ++ * kbasep_tlstream_autoflush_timer_callback - autoflush timer callback ++ * @data: unused ++ * ++ * Timer is executed periodically to check if any of the stream contains ++ * buffer ready to be submitted to user space. ++ */ ++static void kbasep_tlstream_autoflush_timer_callback(struct timer_list *t) ++{ ++ enum tl_stream_type stype; ++ int rcode; ++ ++ CSTD_UNUSED(t); ++ ++ for (stype = 0; stype < TL_STREAM_TYPE_COUNT; stype++) { ++ struct tl_stream *stream = tl_stream[stype]; ++ unsigned long flags; ++ unsigned int wb_idx_raw; ++ unsigned int wb_idx; ++ size_t wb_size; ++ size_t min_size = PACKET_HEADER_SIZE; ++ ++ int af_cnt = atomic_read(&stream->autoflush_counter); ++ ++ /* Check if stream contain unflushed data. */ ++ if (0 > af_cnt) ++ continue; ++ ++ /* Check if stream should be flushed now. */ ++ if (af_cnt != atomic_cmpxchg( ++ &stream->autoflush_counter, ++ af_cnt, ++ af_cnt + 1)) ++ continue; ++ if (!af_cnt) ++ continue; ++ ++ /* Autoflush this stream. */ ++ if (stream->numbered) ++ min_size += PACKET_NUMBER_SIZE; ++ ++ spin_lock_irqsave(&stream->lock, flags); ++ ++ wb_idx_raw = atomic_read(&stream->wbi); ++ wb_idx = wb_idx_raw % PACKET_COUNT; ++ wb_size = atomic_read(&stream->buffer[wb_idx].size); ++ ++ if (wb_size > min_size) { ++ wb_size = kbasep_tlstream_msgbuf_submit( ++ stream, wb_idx_raw, wb_size); ++ wb_idx = (wb_idx_raw + 1) % PACKET_COUNT; ++ atomic_set(&stream->buffer[wb_idx].size, ++ wb_size); ++ } ++ spin_unlock_irqrestore(&stream->lock, flags); ++ } ++ ++ if (atomic_read(&autoflush_timer_active)) ++ rcode = mod_timer( ++ &autoflush_timer, ++ jiffies + msecs_to_jiffies(AUTOFLUSH_INTERVAL)); ++ CSTD_UNUSED(rcode); ++} ++ ++/** ++ * kbasep_tlstream_packet_pending - check timeline streams for pending packets ++ * @stype: pointer to variable where stream type will be placed ++ * @rb_idx_raw: pointer to variable where read buffer index will be placed ++ * ++ * Function checks all streams for pending packets. It will stop as soon as ++ * packet ready to be submitted to user space is detected. Variables under ++ * pointers, passed as the parameters to this function will be updated with ++ * values pointing to right stream and buffer. ++ * ++ * Return: non-zero if any of timeline streams has at last one packet ready ++ */ ++static int kbasep_tlstream_packet_pending( ++ enum tl_stream_type *stype, ++ unsigned int *rb_idx_raw) ++{ ++ int pending = 0; ++ ++ KBASE_DEBUG_ASSERT(stype); ++ KBASE_DEBUG_ASSERT(rb_idx_raw); ++ ++ for ( ++ *stype = 0; ++ (*stype < TL_STREAM_TYPE_COUNT) && !pending; ++ (*stype)++) { ++ if (NULL != tl_stream[*stype]) { ++ *rb_idx_raw = atomic_read(&tl_stream[*stype]->rbi); ++ /* Read buffer index may be updated by writer in case of ++ * overflow. Read and write buffer indexes must be ++ * loaded in correct order. */ ++ smp_rmb(); ++ if (atomic_read(&tl_stream[*stype]->wbi) != *rb_idx_raw) ++ pending = 1; ++ } ++ } ++ (*stype)--; ++ ++ return pending; ++} ++ ++/** ++ * kbasep_tlstream_read - copy data from streams to buffer provided by user ++ * @filp: pointer to file structure (unused) ++ * @buffer: pointer to the buffer provided by user ++ * @size: maximum amount of data that can be stored in the buffer ++ * @f_pos: pointer to file offset (unused) ++ * ++ * Return: number of bytes stored in the buffer ++ */ ++static ssize_t kbasep_tlstream_read( ++ struct file *filp, ++ char __user *buffer, ++ size_t size, ++ loff_t *f_pos) ++{ ++ ssize_t copy_len = 0; ++ ++ KBASE_DEBUG_ASSERT(filp); ++ KBASE_DEBUG_ASSERT(f_pos); ++ ++ if (!buffer) ++ return -EINVAL; ++ ++ if ((0 > *f_pos) || (PACKET_SIZE > size)) ++ return -EINVAL; ++ ++ mutex_lock(&tl_reader_lock); ++ ++ while (copy_len < size) { ++ enum tl_stream_type stype; ++ unsigned int rb_idx_raw = 0; ++ unsigned int rb_idx; ++ size_t rb_size; ++ ++ /* If we don't have any data yet, wait for packet to be ++ * submitted. If we already read some packets and there is no ++ * packet pending return back to user. */ ++ if (0 < copy_len) { ++ if (!kbasep_tlstream_packet_pending( ++ &stype, ++ &rb_idx_raw)) ++ break; ++ } else { ++ if (wait_event_interruptible( ++ tl_event_queue, ++ kbasep_tlstream_packet_pending( ++ &stype, ++ &rb_idx_raw))) { ++ copy_len = -ERESTARTSYS; ++ break; ++ } ++ } ++ ++ /* Check if this packet fits into the user buffer. ++ * If so copy its content. */ ++ rb_idx = rb_idx_raw % PACKET_COUNT; ++ rb_size = atomic_read(&tl_stream[stype]->buffer[rb_idx].size); ++ if (rb_size > size - copy_len) ++ break; ++ if (copy_to_user( ++ &buffer[copy_len], ++ tl_stream[stype]->buffer[rb_idx].data, ++ rb_size)) { ++ copy_len = -EFAULT; ++ break; ++ } ++ ++ /* If the rbi still points to the packet we just processed ++ * then there was no overflow so we add the copied size to ++ * copy_len and move rbi on to the next packet ++ */ ++ smp_rmb(); ++ if (atomic_read(&tl_stream[stype]->rbi) == rb_idx_raw) { ++ copy_len += rb_size; ++ atomic_inc(&tl_stream[stype]->rbi); ++ ++#if MALI_UNIT_TEST ++ atomic_add(rb_size, &tlstream_bytes_collected); ++#endif /* MALI_UNIT_TEST */ ++ } ++ } ++ ++ mutex_unlock(&tl_reader_lock); ++ ++ return copy_len; ++} ++ ++/** ++ * kbasep_tlstream_poll - poll timeline stream for packets ++ * @filp: pointer to file structure ++ * @wait: pointer to poll table ++ * Return: POLLIN if data can be read without blocking, otherwise zero ++ */ ++static unsigned int kbasep_tlstream_poll(struct file *filp, poll_table *wait) ++{ ++ enum tl_stream_type stream_type; ++ unsigned int rb_idx; ++ ++ KBASE_DEBUG_ASSERT(filp); ++ KBASE_DEBUG_ASSERT(wait); ++ ++ poll_wait(filp, &tl_event_queue, wait); ++ if (kbasep_tlstream_packet_pending(&stream_type, &rb_idx)) ++ return POLLIN; ++ return 0; ++} ++ ++/** ++ * kbasep_tlstream_release - release timeline stream descriptor ++ * @inode: pointer to inode structure ++ * @filp: pointer to file structure ++ * ++ * Return always return zero ++ */ ++static int kbasep_tlstream_release(struct inode *inode, struct file *filp) ++{ ++ KBASE_DEBUG_ASSERT(inode); ++ KBASE_DEBUG_ASSERT(filp); ++ CSTD_UNUSED(inode); ++ CSTD_UNUSED(filp); ++ ++ /* Stop autoflush timer before releasing access to streams. */ ++ atomic_set(&autoflush_timer_active, 0); ++ del_timer_sync(&autoflush_timer); ++ ++ atomic_set(&kbase_tlstream_enabled, 0); ++ return 0; ++} ++ ++/** ++ * kbasep_tlstream_timeline_header - prepare timeline header stream packet ++ * @stream_type: type of the stream that will carry header data ++ * @tp_desc: pointer to array with tracepoint descriptors ++ * @tp_count: number of descriptors in the given array ++ * ++ * Functions fills in information about tracepoints stored in body stream ++ * associated with this header stream. ++ */ ++static void kbasep_tlstream_timeline_header( ++ enum tl_stream_type stream_type, ++ const struct tp_desc *tp_desc, ++ u32 tp_count) ++{ ++ const u8 tv = SWTRACE_VERSION; /* protocol version */ ++ const u8 ps = sizeof(void *); /* pointer size */ ++ size_t msg_size = sizeof(tv) + sizeof(ps) + sizeof(tp_count); ++ char *buffer; ++ size_t pos = 0; ++ unsigned long flags; ++ unsigned int i; ++ ++ KBASE_DEBUG_ASSERT(TL_STREAM_TYPE_COUNT > stream_type); ++ KBASE_DEBUG_ASSERT(tp_desc); ++ ++ /* Calculate the size of the timeline message. */ ++ for (i = 0; i < tp_count; i++) { ++ msg_size += sizeof(tp_desc[i].id); ++ msg_size += ++ strnlen(tp_desc[i].id_str, STRLEN_MAX) + ++ sizeof(char) + sizeof(u32); ++ msg_size += ++ strnlen(tp_desc[i].name, STRLEN_MAX) + ++ sizeof(char) + sizeof(u32); ++ msg_size += ++ strnlen(tp_desc[i].arg_types, STRLEN_MAX) + ++ sizeof(char) + sizeof(u32); ++ msg_size += ++ strnlen(tp_desc[i].arg_names, STRLEN_MAX) + ++ sizeof(char) + sizeof(u32); ++ } ++ ++ KBASE_DEBUG_ASSERT(PACKET_SIZE - PACKET_HEADER_SIZE >= msg_size); ++ ++ buffer = kbasep_tlstream_msgbuf_acquire(stream_type, msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &tv, sizeof(tv)); ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &ps, sizeof(ps)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &tp_count, sizeof(tp_count)); ++ ++ for (i = 0; i < tp_count; i++) { ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, ++ &tp_desc[i].id, sizeof(tp_desc[i].id)); ++ pos = kbasep_tlstream_write_string( ++ buffer, pos, ++ tp_desc[i].id_str, msg_size - pos); ++ pos = kbasep_tlstream_write_string( ++ buffer, pos, ++ tp_desc[i].name, msg_size - pos); ++ pos = kbasep_tlstream_write_string( ++ buffer, pos, ++ tp_desc[i].arg_types, msg_size - pos); ++ pos = kbasep_tlstream_write_string( ++ buffer, pos, ++ tp_desc[i].arg_names, msg_size - pos); ++ } ++ ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(stream_type, flags); ++ ++ /* We don't expect any more data to be read in this stream. ++ * As header stream must be read before its associated body stream, ++ * make this packet visible to the user straightaway. */ ++ kbasep_tlstream_flush_stream(stream_type); ++} ++ ++/*****************************************************************************/ ++ ++int kbase_tlstream_init(void) ++{ ++ enum tl_stream_type i; ++ ++ /* Prepare stream structures. */ ++ for (i = 0; i < TL_STREAM_TYPE_COUNT; i++) { ++ tl_stream[i] = kmalloc(sizeof(**tl_stream), GFP_KERNEL); ++ if (!tl_stream[i]) ++ break; ++ kbasep_timeline_stream_init(tl_stream[i], i); ++ } ++ if (TL_STREAM_TYPE_COUNT > i) { ++ for (; i > 0; i--) { ++ kbasep_timeline_stream_term(tl_stream[i - 1]); ++ kfree(tl_stream[i - 1]); ++ } ++ return -ENOMEM; ++ } ++ ++ /* Initialize autoflush timer. */ ++ atomic_set(&autoflush_timer_active, 0); ++ timer_setup(&autoflush_timer, ++ kbasep_tlstream_autoflush_timer_callback, ++ 0); ++ ++ return 0; ++} ++ ++void kbase_tlstream_term(void) ++{ ++ enum tl_stream_type i; ++ ++ for (i = 0; i < TL_STREAM_TYPE_COUNT; i++) { ++ kbasep_timeline_stream_term(tl_stream[i]); ++ kfree(tl_stream[i]); ++ } ++} ++ ++static void kbase_create_timeline_objects(struct kbase_context *kctx) ++{ ++ struct kbase_device *kbdev = kctx->kbdev; ++ unsigned int lpu_id; ++ unsigned int as_nr; ++ struct kbasep_kctx_list_element *element; ++ ++ /* Create LPU objects. */ ++ for (lpu_id = 0; lpu_id < kbdev->gpu_props.num_job_slots; lpu_id++) { ++ u32 *lpu = ++ &kbdev->gpu_props.props.raw_props.js_features[lpu_id]; ++ KBASE_TLSTREAM_TL_SUMMARY_NEW_LPU(lpu, lpu_id, *lpu); ++ } ++ ++ /* Create Address Space objects. */ ++ for (as_nr = 0; as_nr < kbdev->nr_hw_address_spaces; as_nr++) ++ KBASE_TLSTREAM_TL_SUMMARY_NEW_AS(&kbdev->as[as_nr], as_nr); ++ ++ /* Create GPU object and make it retain all LPUs and address spaces. */ ++ KBASE_TLSTREAM_TL_SUMMARY_NEW_GPU( ++ kbdev, ++ kbdev->gpu_props.props.raw_props.gpu_id, ++ kbdev->gpu_props.num_cores); ++ ++ for (lpu_id = 0; lpu_id < kbdev->gpu_props.num_job_slots; lpu_id++) { ++ void *lpu = ++ &kbdev->gpu_props.props.raw_props.js_features[lpu_id]; ++ KBASE_TLSTREAM_TL_SUMMARY_LIFELINK_LPU_GPU(lpu, kbdev); ++ } ++ for (as_nr = 0; as_nr < kbdev->nr_hw_address_spaces; as_nr++) ++ KBASE_TLSTREAM_TL_SUMMARY_LIFELINK_AS_GPU( ++ &kbdev->as[as_nr], ++ kbdev); ++ ++ /* Create object for each known context. */ ++ mutex_lock(&kbdev->kctx_list_lock); ++ list_for_each_entry(element, &kbdev->kctx_list, link) { ++ KBASE_TLSTREAM_TL_SUMMARY_NEW_CTX( ++ element->kctx, ++ (u32)(element->kctx->id), ++ (u32)(element->kctx->tgid)); ++ } ++ /* Before releasing the lock, reset body stream buffers. ++ * This will prevent context creation message to be directed to both ++ * summary and body stream. ++ */ ++ kbase_tlstream_reset_body_streams(); ++ mutex_unlock(&kbdev->kctx_list_lock); ++ /* Static object are placed into summary packet that needs to be ++ * transmitted first. Flush all streams to make it available to ++ * user space. ++ */ ++ kbase_tlstream_flush_streams(); ++} ++ ++int kbase_tlstream_acquire(struct kbase_context *kctx, u32 flags) ++{ ++ int ret; ++ u32 tlstream_enabled = TLSTREAM_ENABLED | flags; ++ ++ if (0 == atomic_cmpxchg(&kbase_tlstream_enabled, 0, tlstream_enabled)) { ++ int rcode; ++ ++ ret = anon_inode_getfd( ++ "[mali_tlstream]", ++ &kbasep_tlstream_fops, ++ kctx, ++ O_RDONLY | O_CLOEXEC); ++ if (ret < 0) { ++ atomic_set(&kbase_tlstream_enabled, 0); ++ return ret; ++ } ++ ++ /* Reset and initialize header streams. */ ++ kbasep_timeline_stream_reset( ++ tl_stream[TL_STREAM_TYPE_OBJ_HEADER]); ++ kbasep_timeline_stream_reset( ++ tl_stream[TL_STREAM_TYPE_OBJ_SUMMARY]); ++ kbasep_timeline_stream_reset( ++ tl_stream[TL_STREAM_TYPE_AUX_HEADER]); ++ kbasep_tlstream_timeline_header( ++ TL_STREAM_TYPE_OBJ_HEADER, ++ tp_desc_obj, ++ ARRAY_SIZE(tp_desc_obj)); ++ kbasep_tlstream_timeline_header( ++ TL_STREAM_TYPE_AUX_HEADER, ++ tp_desc_aux, ++ ARRAY_SIZE(tp_desc_aux)); ++ ++ /* Start autoflush timer. */ ++ atomic_set(&autoflush_timer_active, 1); ++ rcode = mod_timer( ++ &autoflush_timer, ++ jiffies + msecs_to_jiffies(AUTOFLUSH_INTERVAL)); ++ CSTD_UNUSED(rcode); ++ ++ /* If job dumping is enabled, readjust the software event's ++ * timeout as the default value of 3 seconds is often ++ * insufficient. */ ++ if (flags & BASE_TLSTREAM_JOB_DUMPING_ENABLED) { ++ dev_info(kctx->kbdev->dev, ++ "Job dumping is enabled, readjusting the software event's timeout\n"); ++ atomic_set(&kctx->kbdev->js_data.soft_job_timeout_ms, ++ 1800000); ++ } ++ ++ /* Summary stream was cleared during acquire. ++ * Create static timeline objects that will be ++ * read by client. ++ */ ++ kbase_create_timeline_objects(kctx); ++ ++ } else { ++ ret = -EBUSY; ++ } ++ ++ return ret; ++} ++ ++void kbase_tlstream_flush_streams(void) ++{ ++ enum tl_stream_type stype; ++ ++ for (stype = 0; stype < TL_STREAM_TYPE_COUNT; stype++) ++ kbasep_tlstream_flush_stream(stype); ++} ++ ++void kbase_tlstream_reset_body_streams(void) ++{ ++ kbasep_timeline_stream_reset( ++ tl_stream[TL_STREAM_TYPE_OBJ]); ++ kbasep_timeline_stream_reset( ++ tl_stream[TL_STREAM_TYPE_AUX]); ++} ++ ++#if MALI_UNIT_TEST ++void kbase_tlstream_stats(u32 *bytes_collected, u32 *bytes_generated) ++{ ++ KBASE_DEBUG_ASSERT(bytes_collected); ++ KBASE_DEBUG_ASSERT(bytes_generated); ++ *bytes_collected = atomic_read(&tlstream_bytes_collected); ++ *bytes_generated = atomic_read(&tlstream_bytes_generated); ++} ++#endif /* MALI_UNIT_TEST */ ++ ++/*****************************************************************************/ ++ ++void __kbase_tlstream_tl_summary_new_ctx(void *context, u32 nr, u32 tgid) ++{ ++ const u32 msg_id = KBASE_TL_NEW_CTX; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(context) + sizeof(nr) + ++ sizeof(tgid); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ_SUMMARY, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &context, sizeof(context)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &nr, sizeof(nr)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &tgid, sizeof(tgid)); ++ ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags); ++} ++ ++void __kbase_tlstream_tl_summary_new_gpu(void *gpu, u32 id, u32 core_count) ++{ ++ const u32 msg_id = KBASE_TL_NEW_GPU; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(gpu) + sizeof(id) + ++ sizeof(core_count); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ_SUMMARY, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &gpu, sizeof(gpu)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &id, sizeof(id)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &core_count, sizeof(core_count)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags); ++} ++ ++void __kbase_tlstream_tl_summary_new_lpu(void *lpu, u32 nr, u32 fn) ++{ ++ const u32 msg_id = KBASE_TL_NEW_LPU; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(lpu) + sizeof(nr) + ++ sizeof(fn); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ_SUMMARY, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &lpu, sizeof(lpu)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &nr, sizeof(nr)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &fn, sizeof(fn)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags); ++} ++ ++void __kbase_tlstream_tl_summary_lifelink_lpu_gpu(void *lpu, void *gpu) ++{ ++ const u32 msg_id = KBASE_TL_LIFELINK_LPU_GPU; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(lpu) + sizeof(gpu); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ_SUMMARY, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &lpu, sizeof(lpu)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &gpu, sizeof(gpu)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags); ++} ++ ++void __kbase_tlstream_tl_summary_new_as(void *as, u32 nr) ++{ ++ const u32 msg_id = KBASE_TL_NEW_AS; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(as) + sizeof(nr); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ_SUMMARY, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &as, sizeof(as)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &nr, sizeof(nr)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags); ++} ++ ++void __kbase_tlstream_tl_summary_lifelink_as_gpu(void *as, void *gpu) ++{ ++ const u32 msg_id = KBASE_TL_LIFELINK_AS_GPU; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(as) + sizeof(gpu); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ_SUMMARY, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &as, sizeof(as)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &gpu, sizeof(gpu)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags); ++} ++ ++/*****************************************************************************/ ++ ++void __kbase_tlstream_tl_new_ctx(void *context, u32 nr, u32 tgid) ++{ ++ const u32 msg_id = KBASE_TL_NEW_CTX; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(context) + sizeof(nr) + ++ sizeof(tgid); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &context, sizeof(context)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &nr, sizeof(nr)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &tgid, sizeof(tgid)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_new_atom(void *atom, u32 nr) ++{ ++ const u32 msg_id = KBASE_TL_NEW_ATOM; ++ const size_t msg_size = sizeof(msg_id) + sizeof(u64) + sizeof(atom) + ++ sizeof(nr); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &atom, sizeof(atom)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &nr, sizeof(nr)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_del_ctx(void *context) ++{ ++ const u32 msg_id = KBASE_TL_DEL_CTX; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(context); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &context, sizeof(context)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_del_atom(void *atom) ++{ ++ const u32 msg_id = KBASE_TL_DEL_ATOM; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(atom); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &atom, sizeof(atom)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_ret_ctx_lpu(void *context, void *lpu) ++{ ++ const u32 msg_id = KBASE_TL_RET_CTX_LPU; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(context) + sizeof(lpu); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &context, sizeof(context)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &lpu, sizeof(lpu)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_ret_atom_ctx(void *atom, void *context) ++{ ++ const u32 msg_id = KBASE_TL_RET_ATOM_CTX; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(context); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &atom, sizeof(atom)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &context, sizeof(context)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_ret_atom_lpu( ++ void *atom, void *lpu, const char *attrib_match_list) ++{ ++ const u32 msg_id = KBASE_TL_RET_ATOM_LPU; ++ const size_t msg_s0 = sizeof(u32) + sizeof(char) + ++ strnlen(attrib_match_list, STRLEN_MAX); ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + ++ sizeof(atom) + sizeof(lpu) + msg_s0; ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &atom, sizeof(atom)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &lpu, sizeof(lpu)); ++ pos = kbasep_tlstream_write_string( ++ buffer, pos, attrib_match_list, msg_s0); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_nret_ctx_lpu(void *context, void *lpu) ++{ ++ const u32 msg_id = KBASE_TL_NRET_CTX_LPU; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(context) + sizeof(lpu); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &context, sizeof(context)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &lpu, sizeof(lpu)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_nret_atom_ctx(void *atom, void *context) ++{ ++ const u32 msg_id = KBASE_TL_NRET_ATOM_CTX; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(context); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &atom, sizeof(atom)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &context, sizeof(context)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_dep_atom_atom(void *atom1, void *atom2) ++{ ++ const u32 msg_id = KBASE_TL_DEP_ATOM_ATOM; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(atom1) + sizeof(atom2); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &atom1, sizeof(atom1)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &atom2, sizeof(atom2)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_ndep_atom_atom(void *atom1, void *atom2) ++{ ++ const u32 msg_id = KBASE_TL_NDEP_ATOM_ATOM; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(atom1) + sizeof(atom2); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &atom1, sizeof(atom1)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &atom2, sizeof(atom2)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_rdep_atom_atom(void *atom1, void *atom2) ++{ ++ const u32 msg_id = KBASE_TL_RDEP_ATOM_ATOM; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(atom1) + sizeof(atom2); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &atom1, sizeof(atom1)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &atom2, sizeof(atom2)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_nret_atom_lpu(void *atom, void *lpu) ++{ ++ const u32 msg_id = KBASE_TL_NRET_ATOM_LPU; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(lpu); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &atom, sizeof(atom)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &lpu, sizeof(lpu)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_ret_as_ctx(void *as, void *ctx) ++{ ++ const u32 msg_id = KBASE_TL_RET_AS_CTX; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(as) + sizeof(ctx); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &as, sizeof(as)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &ctx, sizeof(ctx)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_nret_as_ctx(void *as, void *ctx) ++{ ++ const u32 msg_id = KBASE_TL_NRET_AS_CTX; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(as) + sizeof(ctx); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &as, sizeof(as)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &ctx, sizeof(ctx)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_ret_atom_as(void *atom, void *as) ++{ ++ const u32 msg_id = KBASE_TL_RET_ATOM_AS; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(as); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &atom, sizeof(atom)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &as, sizeof(as)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_nret_atom_as(void *atom, void *as) ++{ ++ const u32 msg_id = KBASE_TL_NRET_ATOM_AS; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(as); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &atom, sizeof(atom)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &as, sizeof(as)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_attrib_atom_config( ++ void *atom, u64 jd, u64 affinity, u32 config) ++{ ++ const u32 msg_id = KBASE_TL_ATTRIB_ATOM_CONFIG; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(atom) + ++ sizeof(jd) + sizeof(affinity) + sizeof(config); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &atom, sizeof(atom)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &jd, sizeof(jd)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &affinity, sizeof(affinity)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &config, sizeof(config)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_attrib_atom_priority(void *atom, u32 prio) ++{ ++ const u32 msg_id = KBASE_TL_ATTRIB_ATOM_PRIORITY; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(prio); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &atom, sizeof(atom)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &prio, sizeof(prio)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_attrib_atom_state(void *atom, u32 state) ++{ ++ const u32 msg_id = KBASE_TL_ATTRIB_ATOM_STATE; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(state); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &atom, sizeof(atom)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &state, sizeof(state)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_attrib_atom_priority_change(void *atom) ++{ ++ const u32 msg_id = KBASE_TL_ATTRIB_ATOM_PRIORITY_CHANGE; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(atom); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &atom, sizeof(atom)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_attrib_atom_jit( ++ void *atom, u64 edit_addr, u64 new_addr) ++{ ++ const u32 msg_id = KBASE_TL_ATTRIB_ATOM_JIT; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(atom) ++ + sizeof(edit_addr) + sizeof(new_addr); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &atom, sizeof(atom)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &edit_addr, sizeof(edit_addr)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &new_addr, sizeof(new_addr)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_attrib_as_config( ++ void *as, u64 transtab, u64 memattr, u64 transcfg) ++{ ++ const u32 msg_id = KBASE_TL_ATTRIB_AS_CONFIG; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(as) + ++ sizeof(transtab) + sizeof(memattr) + sizeof(transcfg); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &as, sizeof(as)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &transtab, sizeof(transtab)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &memattr, sizeof(memattr)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &transcfg, sizeof(transcfg)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_event_lpu_softstop(void *lpu) ++{ ++ const u32 msg_id = KBASE_TL_EVENT_LPU_SOFTSTOP; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(lpu); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &lpu, sizeof(lpu)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_event_atom_softstop_ex(void *atom) ++{ ++ const u32 msg_id = KBASE_TL_EVENT_ATOM_SOFTSTOP_EX; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(atom); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &atom, sizeof(atom)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_tl_event_atom_softstop_issue(void *atom) ++{ ++ const u32 msg_id = KBASE_TL_EVENT_ATOM_SOFTSTOP_ISSUE; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(atom); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &atom, sizeof(atom)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++void __kbase_tlstream_jd_gpu_soft_reset(void *gpu) ++{ ++ const u32 msg_id = KBASE_JD_GPU_SOFT_RESET; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(gpu); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_OBJ, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &gpu, sizeof(gpu)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); ++} ++ ++/*****************************************************************************/ ++ ++void __kbase_tlstream_aux_pm_state(u32 core_type, u64 state) ++{ ++ const u32 msg_id = KBASE_AUX_PM_STATE; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(core_type) + ++ sizeof(state); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_AUX, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &core_type, sizeof(core_type)); ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &state, sizeof(state)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags); ++} ++ ++void __kbase_tlstream_aux_pagefault(u32 ctx_nr, u64 page_count_change) ++{ ++ const u32 msg_id = KBASE_AUX_PAGEFAULT; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(ctx_nr) + ++ sizeof(page_count_change); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_AUX, msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &ctx_nr, sizeof(ctx_nr)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, ++ &page_count_change, sizeof(page_count_change)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags); ++} ++ ++void __kbase_tlstream_aux_pagesalloc(u32 ctx_nr, u64 page_count) ++{ ++ const u32 msg_id = KBASE_AUX_PAGESALLOC; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(ctx_nr) + ++ sizeof(page_count); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_AUX, msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &ctx_nr, sizeof(ctx_nr)); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &page_count, sizeof(page_count)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags); ++} ++ ++void __kbase_tlstream_aux_devfreq_target(u64 target_freq) ++{ ++ const u32 msg_id = KBASE_AUX_DEVFREQ_TARGET; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(target_freq); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_AUX, msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &target_freq, sizeof(target_freq)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags); ++} ++ ++void __kbase_tlstream_aux_protected_enter_start(void *gpu) ++{ ++ const u32 msg_id = KBASE_AUX_PROTECTED_ENTER_START; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(gpu); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_AUX, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &gpu, sizeof(gpu)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags); ++} ++void __kbase_tlstream_aux_protected_enter_end(void *gpu) ++{ ++ const u32 msg_id = KBASE_AUX_PROTECTED_ENTER_END; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(gpu); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_AUX, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &gpu, sizeof(gpu)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags); ++} ++ ++void __kbase_tlstream_aux_protected_leave_start(void *gpu) ++{ ++ const u32 msg_id = KBASE_AUX_PROTECTED_LEAVE_START; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(gpu); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_AUX, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &gpu, sizeof(gpu)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags); ++} ++void __kbase_tlstream_aux_protected_leave_end(void *gpu) ++{ ++ const u32 msg_id = KBASE_AUX_PROTECTED_LEAVE_END; ++ const size_t msg_size = ++ sizeof(msg_id) + sizeof(u64) + sizeof(gpu); ++ unsigned long flags; ++ char *buffer; ++ size_t pos = 0; ++ ++ buffer = kbasep_tlstream_msgbuf_acquire( ++ TL_STREAM_TYPE_AUX, ++ msg_size, &flags); ++ KBASE_DEBUG_ASSERT(buffer); ++ ++ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); ++ pos = kbasep_tlstream_write_timestamp(buffer, pos); ++ pos = kbasep_tlstream_write_bytes( ++ buffer, pos, &gpu, sizeof(gpu)); ++ KBASE_DEBUG_ASSERT(msg_size == pos); ++ ++ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags); ++} +diff --git a/drivers/gpu/arm/midgard/mali_kbase_tlstream.h b/drivers/gpu/arm/midgard/mali_kbase_tlstream.h +new file mode 100755 +index 000000000000..c0a1117d5f25 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_tlstream.h +@@ -0,0 +1,623 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#if !defined(_KBASE_TLSTREAM_H) ++#define _KBASE_TLSTREAM_H ++ ++#include ++ ++/*****************************************************************************/ ++ ++/** ++ * kbase_tlstream_init - initialize timeline infrastructure in kernel ++ * Return: zero on success, negative number on error ++ */ ++int kbase_tlstream_init(void); ++ ++/** ++ * kbase_tlstream_term - terminate timeline infrastructure in kernel ++ * ++ * Timeline need have to been previously enabled with kbase_tlstream_init(). ++ */ ++void kbase_tlstream_term(void); ++ ++/** ++ * kbase_tlstream_acquire - acquire timeline stream file descriptor ++ * @kctx: kernel common context ++ * @flags: timeline stream flags ++ * ++ * This descriptor is meant to be used by userspace timeline to gain access to ++ * kernel timeline stream. This stream is later broadcasted by user space to the ++ * timeline client. ++ * Only one entity can own the descriptor at any given time. Descriptor shall be ++ * closed if unused. If descriptor cannot be obtained (i.e. when it is already ++ * being used) return will be a negative value. ++ * ++ * Return: file descriptor on success, negative number on error ++ */ ++int kbase_tlstream_acquire(struct kbase_context *kctx, u32 flags); ++ ++/** ++ * kbase_tlstream_flush_streams - flush timeline streams. ++ * ++ * Function will flush pending data in all timeline streams. ++ */ ++void kbase_tlstream_flush_streams(void); ++ ++/** ++ * kbase_tlstream_reset_body_streams - reset timeline body streams. ++ * ++ * Function will discard pending data in all timeline body streams. ++ */ ++void kbase_tlstream_reset_body_streams(void); ++ ++#if MALI_UNIT_TEST ++/** ++ * kbase_tlstream_test - start timeline stream data generator ++ * @tpw_count: number of trace point writers in each context ++ * @msg_delay: time delay in milliseconds between trace points written by one ++ * writer ++ * @msg_count: number of trace points written by one writer ++ * @aux_msg: if non-zero aux messages will be included ++ * ++ * This test starts a requested number of asynchronous writers in both IRQ and ++ * thread context. Each writer will generate required number of test ++ * tracepoints (tracepoints with embedded information about writer that ++ * should be verified by user space reader). Tracepoints will be emitted in ++ * all timeline body streams. If aux_msg is non-zero writer will also ++ * generate not testable tracepoints (tracepoints without information about ++ * writer). These tracepoints are used to check correctness of remaining ++ * timeline message generating functions. Writer will wait requested time ++ * between generating another set of messages. This call blocks until all ++ * writers finish. ++ */ ++void kbase_tlstream_test( ++ unsigned int tpw_count, ++ unsigned int msg_delay, ++ unsigned int msg_count, ++ int aux_msg); ++ ++/** ++ * kbase_tlstream_stats - read timeline stream statistics ++ * @bytes_collected: will hold number of bytes read by the user ++ * @bytes_generated: will hold number of bytes generated by trace points ++ */ ++void kbase_tlstream_stats(u32 *bytes_collected, u32 *bytes_generated); ++#endif /* MALI_UNIT_TEST */ ++ ++/*****************************************************************************/ ++ ++#define TL_ATOM_STATE_IDLE 0 ++#define TL_ATOM_STATE_READY 1 ++#define TL_ATOM_STATE_DONE 2 ++#define TL_ATOM_STATE_POSTED 3 ++ ++void __kbase_tlstream_tl_summary_new_ctx(void *context, u32 nr, u32 tgid); ++void __kbase_tlstream_tl_summary_new_gpu(void *gpu, u32 id, u32 core_count); ++void __kbase_tlstream_tl_summary_new_lpu(void *lpu, u32 nr, u32 fn); ++void __kbase_tlstream_tl_summary_lifelink_lpu_gpu(void *lpu, void *gpu); ++void __kbase_tlstream_tl_summary_new_as(void *as, u32 nr); ++void __kbase_tlstream_tl_summary_lifelink_as_gpu(void *as, void *gpu); ++void __kbase_tlstream_tl_new_ctx(void *context, u32 nr, u32 tgid); ++void __kbase_tlstream_tl_new_atom(void *atom, u32 nr); ++void __kbase_tlstream_tl_del_ctx(void *context); ++void __kbase_tlstream_tl_del_atom(void *atom); ++void __kbase_tlstream_tl_ret_ctx_lpu(void *context, void *lpu); ++void __kbase_tlstream_tl_ret_atom_ctx(void *atom, void *context); ++void __kbase_tlstream_tl_ret_atom_lpu( ++ void *atom, void *lpu, const char *attrib_match_list); ++void __kbase_tlstream_tl_nret_ctx_lpu(void *context, void *lpu); ++void __kbase_tlstream_tl_nret_atom_ctx(void *atom, void *context); ++void __kbase_tlstream_tl_nret_atom_lpu(void *atom, void *lpu); ++void __kbase_tlstream_tl_ret_as_ctx(void *as, void *ctx); ++void __kbase_tlstream_tl_nret_as_ctx(void *as, void *ctx); ++void __kbase_tlstream_tl_ret_atom_as(void *atom, void *as); ++void __kbase_tlstream_tl_nret_atom_as(void *atom, void *as); ++void __kbase_tlstream_tl_dep_atom_atom(void *atom1, void *atom2); ++void __kbase_tlstream_tl_ndep_atom_atom(void *atom1, void *atom2); ++void __kbase_tlstream_tl_rdep_atom_atom(void *atom1, void *atom2); ++void __kbase_tlstream_tl_attrib_atom_config( ++ void *atom, u64 jd, u64 affinity, u32 config); ++void __kbase_tlstream_tl_attrib_atom_priority(void *atom, u32 prio); ++void __kbase_tlstream_tl_attrib_atom_state(void *atom, u32 state); ++void __kbase_tlstream_tl_attrib_atom_priority_change(void *atom); ++void __kbase_tlstream_tl_attrib_atom_jit( ++ void *atom, u64 edit_addr, u64 new_addr); ++void __kbase_tlstream_tl_attrib_as_config( ++ void *as, u64 transtab, u64 memattr, u64 transcfg); ++void __kbase_tlstream_tl_event_atom_softstop_ex(void *atom); ++void __kbase_tlstream_tl_event_lpu_softstop(void *lpu); ++void __kbase_tlstream_tl_event_atom_softstop_issue(void *atom); ++void __kbase_tlstream_jd_gpu_soft_reset(void *gpu); ++void __kbase_tlstream_aux_pm_state(u32 core_type, u64 state); ++void __kbase_tlstream_aux_pagefault(u32 ctx_nr, u64 page_count_change); ++void __kbase_tlstream_aux_pagesalloc(u32 ctx_nr, u64 page_count); ++void __kbase_tlstream_aux_devfreq_target(u64 target_freq); ++void __kbase_tlstream_aux_protected_enter_start(void *gpu); ++void __kbase_tlstream_aux_protected_enter_end(void *gpu); ++void __kbase_tlstream_aux_protected_leave_start(void *gpu); ++void __kbase_tlstream_aux_protected_leave_end(void *gpu); ++ ++#define TLSTREAM_ENABLED (1 << 31) ++ ++extern atomic_t kbase_tlstream_enabled; ++ ++#define __TRACE_IF_ENABLED(trace_name, ...) \ ++ do { \ ++ int enabled = atomic_read(&kbase_tlstream_enabled); \ ++ if (enabled & TLSTREAM_ENABLED) \ ++ __kbase_tlstream_##trace_name(__VA_ARGS__); \ ++ } while (0) ++ ++#define __TRACE_IF_ENABLED_LATENCY(trace_name, ...) \ ++ do { \ ++ int enabled = atomic_read(&kbase_tlstream_enabled); \ ++ if (enabled & BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS) \ ++ __kbase_tlstream_##trace_name(__VA_ARGS__); \ ++ } while (0) ++ ++#define __TRACE_IF_ENABLED_JD(trace_name, ...) \ ++ do { \ ++ int enabled = atomic_read(&kbase_tlstream_enabled); \ ++ if (enabled & BASE_TLSTREAM_JOB_DUMPING_ENABLED) \ ++ __kbase_tlstream_##trace_name(__VA_ARGS__); \ ++ } while (0) ++ ++/*****************************************************************************/ ++ ++/** ++ * KBASE_TLSTREAM_TL_SUMMARY_NEW_CTX - create context object in timeline ++ * summary ++ * @context: name of the context object ++ * @nr: context number ++ * @tgid: thread Group Id ++ * ++ * Function emits a timeline message informing about context creation. Context ++ * is created with context number (its attribute), that can be used to link ++ * kbase context with userspace context. ++ * This message is directed to timeline summary stream. ++ */ ++#define KBASE_TLSTREAM_TL_SUMMARY_NEW_CTX(context, nr, tgid) \ ++ __TRACE_IF_ENABLED(tl_summary_new_ctx, context, nr, tgid) ++ ++/** ++ * KBASE_TLSTREAM_TL_SUMMARY_NEW_GPU - create GPU object in timeline summary ++ * @gpu: name of the GPU object ++ * @id: id value of this GPU ++ * @core_count: number of cores this GPU hosts ++ * ++ * Function emits a timeline message informing about GPU creation. GPU is ++ * created with two attributes: id and core count. ++ * This message is directed to timeline summary stream. ++ */ ++#define KBASE_TLSTREAM_TL_SUMMARY_NEW_GPU(gpu, id, core_count) \ ++ __TRACE_IF_ENABLED(tl_summary_new_gpu, gpu, id, core_count) ++ ++/** ++ * KBASE_TLSTREAM_TL_SUMMARY_NEW_LPU - create LPU object in timeline summary ++ * @lpu: name of the Logical Processing Unit object ++ * @nr: sequential number assigned to this LPU ++ * @fn: property describing this LPU's functional abilities ++ * ++ * Function emits a timeline message informing about LPU creation. LPU is ++ * created with two attributes: number linking this LPU with GPU's job slot ++ * and function bearing information about this LPU abilities. ++ * This message is directed to timeline summary stream. ++ */ ++#define KBASE_TLSTREAM_TL_SUMMARY_NEW_LPU(lpu, nr, fn) \ ++ __TRACE_IF_ENABLED(tl_summary_new_lpu, lpu, nr, fn) ++ ++/** ++ * KBASE_TLSTREAM_TL_SUMMARY_LIFELINK_LPU_GPU - lifelink LPU object to GPU ++ * @lpu: name of the Logical Processing Unit object ++ * @gpu: name of the GPU object ++ * ++ * Function emits a timeline message informing that LPU object shall be deleted ++ * along with GPU object. ++ * This message is directed to timeline summary stream. ++ */ ++#define KBASE_TLSTREAM_TL_SUMMARY_LIFELINK_LPU_GPU(lpu, gpu) \ ++ __TRACE_IF_ENABLED(tl_summary_lifelink_lpu_gpu, lpu, gpu) ++ ++/** ++ * KBASE_TLSTREAM_TL_SUMMARY_NEW_AS - create address space object in timeline summary ++ * @as: name of the address space object ++ * @nr: sequential number assigned to this address space ++ * ++ * Function emits a timeline message informing about address space creation. ++ * Address space is created with one attribute: number identifying this ++ * address space. ++ * This message is directed to timeline summary stream. ++ */ ++#define KBASE_TLSTREAM_TL_SUMMARY_NEW_AS(as, nr) \ ++ __TRACE_IF_ENABLED(tl_summary_new_as, as, nr) ++ ++/** ++ * KBASE_TLSTREAM_TL_SUMMARY_LIFELINK_AS_GPU - lifelink address space object to GPU ++ * @as: name of the address space object ++ * @gpu: name of the GPU object ++ * ++ * Function emits a timeline message informing that address space object ++ * shall be deleted along with GPU object. ++ * This message is directed to timeline summary stream. ++ */ ++#define KBASE_TLSTREAM_TL_SUMMARY_LIFELINK_AS_GPU(as, gpu) \ ++ __TRACE_IF_ENABLED(tl_summary_lifelink_as_gpu, as, gpu) ++ ++/** ++ * KBASE_TLSTREAM_TL_NEW_CTX - create context object in timeline ++ * @context: name of the context object ++ * @nr: context number ++ * @tgid: thread Group Id ++ * ++ * Function emits a timeline message informing about context creation. Context ++ * is created with context number (its attribute), that can be used to link ++ * kbase context with userspace context. ++ */ ++#define KBASE_TLSTREAM_TL_NEW_CTX(context, nr, tgid) \ ++ __TRACE_IF_ENABLED(tl_new_ctx, context, nr, tgid) ++ ++/** ++ * KBASE_TLSTREAM_TL_NEW_ATOM - create atom object in timeline ++ * @atom: name of the atom object ++ * @nr: sequential number assigned to this atom ++ * ++ * Function emits a timeline message informing about atom creation. Atom is ++ * created with atom number (its attribute) that links it with actual work ++ * bucket id understood by hardware. ++ */ ++#define KBASE_TLSTREAM_TL_NEW_ATOM(atom, nr) \ ++ __TRACE_IF_ENABLED(tl_new_atom, atom, nr) ++ ++/** ++ * KBASE_TLSTREAM_TL_DEL_CTX - destroy context object in timeline ++ * @context: name of the context object ++ * ++ * Function emits a timeline message informing that context object ceased to ++ * exist. ++ */ ++#define KBASE_TLSTREAM_TL_DEL_CTX(context) \ ++ __TRACE_IF_ENABLED(tl_del_ctx, context) ++ ++/** ++ * KBASE_TLSTREAM_TL_DEL_ATOM - destroy atom object in timeline ++ * @atom: name of the atom object ++ * ++ * Function emits a timeline message informing that atom object ceased to ++ * exist. ++ */ ++#define KBASE_TLSTREAM_TL_DEL_ATOM(atom) \ ++ __TRACE_IF_ENABLED(tl_del_atom, atom) ++ ++/** ++ * KBASE_TLSTREAM_TL_RET_CTX_LPU - retain context by LPU ++ * @context: name of the context object ++ * @lpu: name of the Logical Processing Unit object ++ * ++ * Function emits a timeline message informing that context is being held ++ * by LPU and must not be deleted unless it is released. ++ */ ++#define KBASE_TLSTREAM_TL_RET_CTX_LPU(context, lpu) \ ++ __TRACE_IF_ENABLED(tl_ret_ctx_lpu, context, lpu) ++ ++/** ++ * KBASE_TLSTREAM_TL_RET_ATOM_CTX - retain atom by context ++ * @atom: name of the atom object ++ * @context: name of the context object ++ * ++ * Function emits a timeline message informing that atom object is being held ++ * by context and must not be deleted unless it is released. ++ */ ++#define KBASE_TLSTREAM_TL_RET_ATOM_CTX(atom, context) \ ++ __TRACE_IF_ENABLED(tl_ret_atom_ctx, atom, context) ++ ++/** ++ * KBASE_TLSTREAM_TL_RET_ATOM_LPU - retain atom by LPU ++ * @atom: name of the atom object ++ * @lpu: name of the Logical Processing Unit object ++ * @attrib_match_list: list containing match operator attributes ++ * ++ * Function emits a timeline message informing that atom object is being held ++ * by LPU and must not be deleted unless it is released. ++ */ ++#define KBASE_TLSTREAM_TL_RET_ATOM_LPU(atom, lpu, attrib_match_list) \ ++ __TRACE_IF_ENABLED(tl_ret_atom_lpu, atom, lpu, attrib_match_list) ++ ++/** ++ * KBASE_TLSTREAM_TL_NRET_CTX_LPU - release context by LPU ++ * @context: name of the context object ++ * @lpu: name of the Logical Processing Unit object ++ * ++ * Function emits a timeline message informing that context is being released ++ * by LPU object. ++ */ ++#define KBASE_TLSTREAM_TL_NRET_CTX_LPU(context, lpu) \ ++ __TRACE_IF_ENABLED(tl_nret_ctx_lpu, context, lpu) ++ ++/** ++ * KBASE_TLSTREAM_TL_NRET_ATOM_CTX - release atom by context ++ * @atom: name of the atom object ++ * @context: name of the context object ++ * ++ * Function emits a timeline message informing that atom object is being ++ * released by context. ++ */ ++#define KBASE_TLSTREAM_TL_NRET_ATOM_CTX(atom, context) \ ++ __TRACE_IF_ENABLED(tl_nret_atom_ctx, atom, context) ++ ++/** ++ * KBASE_TLSTREAM_TL_NRET_ATOM_LPU - release atom by LPU ++ * @atom: name of the atom object ++ * @lpu: name of the Logical Processing Unit object ++ * ++ * Function emits a timeline message informing that atom object is being ++ * released by LPU. ++ */ ++#define KBASE_TLSTREAM_TL_NRET_ATOM_LPU(atom, lpu) \ ++ __TRACE_IF_ENABLED(tl_nret_atom_lpu, atom, lpu) ++ ++/** ++ * KBASE_TLSTREAM_TL_RET_AS_CTX - lifelink address space object to context ++ * @as: name of the address space object ++ * @ctx: name of the context object ++ * ++ * Function emits a timeline message informing that address space object ++ * is being held by the context object. ++ */ ++#define KBASE_TLSTREAM_TL_RET_AS_CTX(as, ctx) \ ++ __TRACE_IF_ENABLED(tl_ret_as_ctx, as, ctx) ++ ++/** ++ * KBASE_TLSTREAM_TL_NRET_AS_CTX - release address space by context ++ * @as: name of the address space object ++ * @ctx: name of the context object ++ * ++ * Function emits a timeline message informing that address space object ++ * is being released by atom. ++ */ ++#define KBASE_TLSTREAM_TL_NRET_AS_CTX(as, ctx) \ ++ __TRACE_IF_ENABLED(tl_nret_as_ctx, as, ctx) ++ ++/** ++ * KBASE_TLSTREAM_TL_RET_ATOM_AS - retain atom by address space ++ * @atom: name of the atom object ++ * @as: name of the address space object ++ * ++ * Function emits a timeline message informing that atom object is being held ++ * by address space and must not be deleted unless it is released. ++ */ ++#define KBASE_TLSTREAM_TL_RET_ATOM_AS(atom, as) \ ++ __TRACE_IF_ENABLED(tl_ret_atom_as, atom, as) ++ ++/** ++ * KBASE_TLSTREAM_TL_NRET_ATOM_AS - release atom by address space ++ * @atom: name of the atom object ++ * @as: name of the address space object ++ * ++ * Function emits a timeline message informing that atom object is being ++ * released by address space. ++ */ ++#define KBASE_TLSTREAM_TL_NRET_ATOM_AS(atom, as) \ ++ __TRACE_IF_ENABLED(tl_nret_atom_as, atom, as) ++ ++/** ++ * KBASE_TLSTREAM_TL_DEP_ATOM_ATOM - parent atom depends on child atom ++ * @atom1: name of the child atom object ++ * @atom2: name of the parent atom object that depends on child atom ++ * ++ * Function emits a timeline message informing that parent atom waits for ++ * child atom object to be completed before start its execution. ++ */ ++#define KBASE_TLSTREAM_TL_DEP_ATOM_ATOM(atom1, atom2) \ ++ __TRACE_IF_ENABLED(tl_dep_atom_atom, atom1, atom2) ++ ++/** ++ * KBASE_TLSTREAM_TL_NDEP_ATOM_ATOM - dependency between atoms resolved ++ * @atom1: name of the child atom object ++ * @atom2: name of the parent atom object that depended on child atom ++ * ++ * Function emits a timeline message informing that parent atom execution ++ * dependency on child atom has been resolved. ++ */ ++#define KBASE_TLSTREAM_TL_NDEP_ATOM_ATOM(atom1, atom2) \ ++ __TRACE_IF_ENABLED(tl_ndep_atom_atom, atom1, atom2) ++ ++/** ++ * KBASE_TLSTREAM_TL_RDEP_ATOM_ATOM - information about already resolved dependency between atoms ++ * @atom1: name of the child atom object ++ * @atom2: name of the parent atom object that depended on child atom ++ * ++ * Function emits a timeline message informing that parent atom execution ++ * dependency on child atom has been resolved. ++ */ ++#define KBASE_TLSTREAM_TL_RDEP_ATOM_ATOM(atom1, atom2) \ ++ __TRACE_IF_ENABLED(tl_rdep_atom_atom, atom1, atom2) ++ ++/** ++ * KBASE_TLSTREAM_TL_ATTRIB_ATOM_CONFIG - atom job slot attributes ++ * @atom: name of the atom object ++ * @jd: job descriptor address ++ * @affinity: job affinity ++ * @config: job config ++ * ++ * Function emits a timeline message containing atom attributes. ++ */ ++#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_CONFIG(atom, jd, affinity, config) \ ++ __TRACE_IF_ENABLED(tl_attrib_atom_config, atom, jd, affinity, config) ++ ++/** ++ * KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY - atom priority ++ * @atom: name of the atom object ++ * @prio: atom priority ++ * ++ * Function emits a timeline message containing atom priority. ++ */ ++#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY(atom, prio) \ ++ __TRACE_IF_ENABLED_LATENCY(tl_attrib_atom_priority, atom, prio) ++ ++/** ++ * KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE - atom state ++ * @atom: name of the atom object ++ * @state: atom state ++ * ++ * Function emits a timeline message containing atom state. ++ */ ++#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(atom, state) \ ++ __TRACE_IF_ENABLED_LATENCY(tl_attrib_atom_state, atom, state) ++ ++/** ++ * KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY_CHANGE - atom caused priority change ++ * @atom: name of the atom object ++ * ++ * Function emits a timeline message signalling priority change ++ */ ++#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY_CHANGE(atom) \ ++ __TRACE_IF_ENABLED_LATENCY(tl_attrib_atom_priority_change, atom) ++ ++/** ++ * KBASE_TLSTREAM_TL_ATTRIB_ATOM_JIT - jit happened on atom ++ * @atom: atom identifier ++ * @edit_addr: address edited by jit ++ * @new_addr: address placed into the edited location ++ */ ++#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_JIT(atom, edit_addr, new_addr) \ ++ __TRACE_IF_ENABLED_JD(tl_attrib_atom_jit, atom, edit_addr, new_addr) ++ ++/** ++ * KBASE_TLSTREAM_TL_ATTRIB_AS_CONFIG - address space attributes ++ * @as: assigned address space ++ * @transtab: configuration of the TRANSTAB register ++ * @memattr: configuration of the MEMATTR register ++ * @transcfg: configuration of the TRANSCFG register (or zero if not present) ++ * ++ * Function emits a timeline message containing address space attributes. ++ */ ++#define KBASE_TLSTREAM_TL_ATTRIB_AS_CONFIG(as, transtab, memattr, transcfg) \ ++ __TRACE_IF_ENABLED(tl_attrib_as_config, as, transtab, memattr, transcfg) ++ ++/** ++ * KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_ex ++ * @atom: atom identifier ++ */ ++#define KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_EX(atom) \ ++ __TRACE_IF_ENABLED(tl_event_atom_softstop_ex, atom) ++ ++/** ++ * KBASE_TLSTREAM_TL_EVENT_LPU_softstop ++ * @lpu: name of the LPU object ++ */ ++#define KBASE_TLSTREAM_TL_EVENT_LPU_SOFTSTOP(lpu) \ ++ __TRACE_IF_ENABLED(tl_event_lpu_softstop, lpu) ++ ++/** ++ * KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_issue ++ * @atom: atom identifier ++ */ ++#define KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_ISSUE(atom) \ ++ __TRACE_IF_ENABLED(tl_event_atom_softstop_issue, atom) ++ ++/** ++ * KBASE_TLSTREAM_JD_GPU_SOFT_RESET - The GPU is being soft reset ++ * @gpu: name of the GPU object ++ * ++ * This imperative tracepoint is specific to job dumping. ++ * Function emits a timeline message indicating GPU soft reset. ++ */ ++#define KBASE_TLSTREAM_JD_GPU_SOFT_RESET(gpu) \ ++ __TRACE_IF_ENABLED(jd_gpu_soft_reset, gpu) ++ ++ ++/** ++ * KBASE_TLSTREAM_AUX_PM_STATE - timeline message: power management state ++ * @core_type: core type (shader, tiler, l2 cache, l3 cache) ++ * @state: 64bits bitmask reporting power state of the cores (1-ON, 0-OFF) ++ */ ++#define KBASE_TLSTREAM_AUX_PM_STATE(core_type, state) \ ++ __TRACE_IF_ENABLED(aux_pm_state, core_type, state) ++ ++/** ++ * KBASE_TLSTREAM_AUX_PAGEFAULT - timeline message: MMU page fault event ++ * resulting in new pages being mapped ++ * @ctx_nr: kernel context number ++ * @page_count_change: number of pages to be added ++ */ ++#define KBASE_TLSTREAM_AUX_PAGEFAULT(ctx_nr, page_count_change) \ ++ __TRACE_IF_ENABLED(aux_pagefault, ctx_nr, page_count_change) ++ ++/** ++ * KBASE_TLSTREAM_AUX_PAGESALLOC - timeline message: total number of allocated ++ * pages is changed ++ * @ctx_nr: kernel context number ++ * @page_count: number of pages used by the context ++ */ ++#define KBASE_TLSTREAM_AUX_PAGESALLOC(ctx_nr, page_count) \ ++ __TRACE_IF_ENABLED(aux_pagesalloc, ctx_nr, page_count) ++ ++/** ++ * KBASE_TLSTREAM_AUX_DEVFREQ_TARGET - timeline message: new target DVFS ++ * frequency ++ * @target_freq: new target frequency ++ */ ++#define KBASE_TLSTREAM_AUX_DEVFREQ_TARGET(target_freq) \ ++ __TRACE_IF_ENABLED(aux_devfreq_target, target_freq) ++ ++/** ++ * KBASE_TLSTREAM_AUX_PROTECTED_ENTER_START - The GPU has started transitioning ++ * to protected mode ++ * @gpu: name of the GPU object ++ * ++ * Function emits a timeline message indicating the GPU is starting to ++ * transition to protected mode. ++ */ ++#define KBASE_TLSTREAM_AUX_PROTECTED_ENTER_START(gpu) \ ++ __TRACE_IF_ENABLED_LATENCY(aux_protected_enter_start, gpu) ++ ++/** ++ * KBASE_TLSTREAM_AUX_PROTECTED_ENTER_END - The GPU has finished transitioning ++ * to protected mode ++ * @gpu: name of the GPU object ++ * ++ * Function emits a timeline message indicating the GPU has finished ++ * transitioning to protected mode. ++ */ ++#define KBASE_TLSTREAM_AUX_PROTECTED_ENTER_END(gpu) \ ++ __TRACE_IF_ENABLED_LATENCY(aux_protected_enter_end, gpu) ++ ++/** ++ * KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_START - The GPU has started transitioning ++ * to non-protected mode ++ * @gpu: name of the GPU object ++ * ++ * Function emits a timeline message indicating the GPU is starting to ++ * transition to non-protected mode. ++ */ ++#define KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_START(gpu) \ ++ __TRACE_IF_ENABLED_LATENCY(aux_protected_leave_start, gpu) ++ ++/** ++ * KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_END - The GPU has finished transitioning ++ * to non-protected mode ++ * @gpu: name of the GPU object ++ * ++ * Function emits a timeline message indicating the GPU has finished ++ * transitioning to non-protected mode. ++ */ ++#define KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_END(gpu) \ ++ __TRACE_IF_ENABLED_LATENCY(aux_protected_leave_end, gpu) ++ ++#endif /* _KBASE_TLSTREAM_H */ ++ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_trace_defs.h b/drivers/gpu/arm/midgard/mali_kbase_trace_defs.h +new file mode 100755 +index 000000000000..e2e0544208ce +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_trace_defs.h +@@ -0,0 +1,264 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2014 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ***** IMPORTANT: THIS IS NOT A NORMAL HEADER FILE ***** ++ * ***** DO NOT INCLUDE DIRECTLY ***** ++ * ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** */ ++ ++/* ++ * The purpose of this header file is just to contain a list of trace code idenitifers ++ * ++ * Each identifier is wrapped in a macro, so that its string form and enum form can be created ++ * ++ * Each macro is separated with a comma, to allow insertion into an array initializer or enum definition block. ++ * ++ * This allows automatic creation of an enum and a corresponding array of strings ++ * ++ * Before #including, the includer MUST #define KBASE_TRACE_CODE_MAKE_CODE. ++ * After #including, the includer MUST #under KBASE_TRACE_CODE_MAKE_CODE. ++ * ++ * e.g.: ++ * #define KBASE_TRACE_CODE( X ) KBASE_TRACE_CODE_ ## X ++ * typedef enum ++ * { ++ * #define KBASE_TRACE_CODE_MAKE_CODE( X ) KBASE_TRACE_CODE( X ) ++ * #include "mali_kbase_trace_defs.h" ++ * #undef KBASE_TRACE_CODE_MAKE_CODE ++ * } kbase_trace_code; ++ * ++ * IMPORTANT: THIS FILE MUST NOT BE USED FOR ANY OTHER PURPOSE OTHER THAN THE ABOVE ++ * ++ * ++ * The use of the macro here is: ++ * - KBASE_TRACE_CODE_MAKE_CODE( X ) ++ * ++ * Which produces: ++ * - For an enum, KBASE_TRACE_CODE_X ++ * - For a string, "X" ++ * ++ * ++ * For example: ++ * - KBASE_TRACE_CODE_MAKE_CODE( JM_JOB_COMPLETE ) expands to: ++ * - KBASE_TRACE_CODE_JM_JOB_COMPLETE for the enum ++ * - "JM_JOB_COMPLETE" for the string ++ * - To use it to trace an event, do: ++ * - KBASE_TRACE_ADD( kbdev, JM_JOB_COMPLETE, subcode, kctx, uatom, val ); ++ */ ++ ++#if 0 /* Dummy section to avoid breaking formatting */ ++int dummy_array[] = { ++#endif ++ ++/* ++ * Core events ++ */ ++ /* no info_val, no gpu_addr, no atom */ ++ KBASE_TRACE_CODE_MAKE_CODE(CORE_CTX_DESTROY), ++ /* no info_val, no gpu_addr, no atom */ ++ KBASE_TRACE_CODE_MAKE_CODE(CORE_CTX_HWINSTR_TERM), ++ /* info_val == GPU_IRQ_STATUS register */ ++ KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_IRQ), ++ /* info_val == bits cleared */ ++ KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_IRQ_CLEAR), ++ /* info_val == GPU_IRQ_STATUS register */ ++ KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_IRQ_DONE), ++ KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_SOFT_RESET), ++ KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_HARD_RESET), ++ KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_PRFCNT_CLEAR), ++ /* GPU addr==dump address */ ++ KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_PRFCNT_SAMPLE), ++ KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_CLEAN_INV_CACHES), ++/* ++ * Job Slot management events ++ */ ++ /* info_val==irq rawstat at start */ ++ KBASE_TRACE_CODE_MAKE_CODE(JM_IRQ), ++ /* info_val==jobs processed */ ++ KBASE_TRACE_CODE_MAKE_CODE(JM_IRQ_END), ++/* In the following: ++ * ++ * - ctx is set if a corresponding job found (NULL otherwise, e.g. some soft-stop cases) ++ * - uatom==kernel-side mapped uatom address (for correlation with user-side) ++ */ ++ /* info_val==exit code; gpu_addr==chain gpuaddr */ ++ KBASE_TRACE_CODE_MAKE_CODE(JM_JOB_DONE), ++ /* gpu_addr==JS_HEAD_NEXT written, info_val==lower 32 bits of affinity */ ++ KBASE_TRACE_CODE_MAKE_CODE(JM_SUBMIT), ++ /* gpu_addr is as follows: ++ * - If JS_STATUS active after soft-stop, val==gpu addr written to ++ * JS_HEAD on submit ++ * - otherwise gpu_addr==0 */ ++ KBASE_TRACE_CODE_MAKE_CODE(JM_SOFTSTOP), ++ KBASE_TRACE_CODE_MAKE_CODE(JM_SOFTSTOP_0), ++ KBASE_TRACE_CODE_MAKE_CODE(JM_SOFTSTOP_1), ++ /* gpu_addr==JS_HEAD read */ ++ KBASE_TRACE_CODE_MAKE_CODE(JM_HARDSTOP), ++ /* gpu_addr==JS_HEAD read */ ++ KBASE_TRACE_CODE_MAKE_CODE(JM_HARDSTOP_0), ++ /* gpu_addr==JS_HEAD read */ ++ KBASE_TRACE_CODE_MAKE_CODE(JM_HARDSTOP_1), ++ /* gpu_addr==JS_TAIL read */ ++ KBASE_TRACE_CODE_MAKE_CODE(JM_UPDATE_HEAD), ++/* gpu_addr is as follows: ++ * - If JS_STATUS active before soft-stop, val==JS_HEAD ++ * - otherwise gpu_addr==0 ++ */ ++ /* gpu_addr==JS_HEAD read */ ++ KBASE_TRACE_CODE_MAKE_CODE(JM_CHECK_HEAD), ++ KBASE_TRACE_CODE_MAKE_CODE(JM_FLUSH_WORKQS), ++ KBASE_TRACE_CODE_MAKE_CODE(JM_FLUSH_WORKQS_DONE), ++ /* info_val == is_scheduled */ ++ KBASE_TRACE_CODE_MAKE_CODE(JM_ZAP_NON_SCHEDULED), ++ /* info_val == is_scheduled */ ++ KBASE_TRACE_CODE_MAKE_CODE(JM_ZAP_SCHEDULED), ++ KBASE_TRACE_CODE_MAKE_CODE(JM_ZAP_DONE), ++ /* info_val == nr jobs submitted */ ++ KBASE_TRACE_CODE_MAKE_CODE(JM_SLOT_SOFT_OR_HARD_STOP), ++ /* gpu_addr==JS_HEAD_NEXT last written */ ++ KBASE_TRACE_CODE_MAKE_CODE(JM_SLOT_EVICT), ++ KBASE_TRACE_CODE_MAKE_CODE(JM_SUBMIT_AFTER_RESET), ++ KBASE_TRACE_CODE_MAKE_CODE(JM_BEGIN_RESET_WORKER), ++ KBASE_TRACE_CODE_MAKE_CODE(JM_END_RESET_WORKER), ++/* ++ * Job dispatch events ++ */ ++ /* gpu_addr==value to write into JS_HEAD */ ++ KBASE_TRACE_CODE_MAKE_CODE(JD_DONE), ++ /* gpu_addr==value to write into JS_HEAD */ ++ KBASE_TRACE_CODE_MAKE_CODE(JD_DONE_WORKER), ++ /* gpu_addr==value to write into JS_HEAD */ ++ KBASE_TRACE_CODE_MAKE_CODE(JD_DONE_WORKER_END), ++ /* gpu_addr==value to write into JS_HEAD */ ++ KBASE_TRACE_CODE_MAKE_CODE(JD_DONE_TRY_RUN_NEXT_JOB), ++ /* gpu_addr==0, info_val==0, uatom==0 */ ++ KBASE_TRACE_CODE_MAKE_CODE(JD_ZAP_CONTEXT), ++ /* gpu_addr==value to write into JS_HEAD */ ++ KBASE_TRACE_CODE_MAKE_CODE(JD_CANCEL), ++ /* gpu_addr==value to write into JS_HEAD */ ++ KBASE_TRACE_CODE_MAKE_CODE(JD_CANCEL_WORKER), ++/* ++ * Scheduler Core events ++ */ ++ KBASE_TRACE_CODE_MAKE_CODE(JS_RETAIN_CTX_NOLOCK), ++ /* gpu_addr==value to write into JS_HEAD */ ++ KBASE_TRACE_CODE_MAKE_CODE(JS_ADD_JOB), ++ /* gpu_addr==last value written/would be written to JS_HEAD */ ++ KBASE_TRACE_CODE_MAKE_CODE(JS_REMOVE_JOB), ++ KBASE_TRACE_CODE_MAKE_CODE(JS_RETAIN_CTX), ++ KBASE_TRACE_CODE_MAKE_CODE(JS_RELEASE_CTX), ++ KBASE_TRACE_CODE_MAKE_CODE(JS_TRY_SCHEDULE_HEAD_CTX), ++ /* gpu_addr==value to write into JS_HEAD */ ++ KBASE_TRACE_CODE_MAKE_CODE(JS_JOB_DONE_TRY_RUN_NEXT_JOB), ++ /* gpu_addr==value to write into JS_HEAD */ ++ KBASE_TRACE_CODE_MAKE_CODE(JS_JOB_DONE_RETRY_NEEDED), ++ /* kctx is the one being evicted, info_val == kctx to put in */ ++ KBASE_TRACE_CODE_MAKE_CODE(JS_FAST_START_EVICTS_CTX), ++ KBASE_TRACE_CODE_MAKE_CODE(JS_AFFINITY_SUBMIT_TO_BLOCKED), ++ /* info_val == lower 32 bits of affinity */ ++ KBASE_TRACE_CODE_MAKE_CODE(JS_AFFINITY_CURRENT), ++ /* info_val == lower 32 bits of affinity */ ++ KBASE_TRACE_CODE_MAKE_CODE(JS_CORE_REF_REQUEST_CORES_FAILED), ++ /* info_val == lower 32 bits of affinity */ ++ KBASE_TRACE_CODE_MAKE_CODE(JS_CORE_REF_REGISTER_INUSE_FAILED), ++ /* info_val == lower 32 bits of rechecked affinity */ ++ KBASE_TRACE_CODE_MAKE_CODE(JS_CORE_REF_REQUEST_ON_RECHECK_FAILED), ++ /* info_val == lower 32 bits of rechecked affinity */ ++ KBASE_TRACE_CODE_MAKE_CODE(JS_CORE_REF_REGISTER_ON_RECHECK_FAILED), ++ /* info_val == lower 32 bits of affinity */ ++ KBASE_TRACE_CODE_MAKE_CODE(JS_CORE_REF_AFFINITY_WOULD_VIOLATE), ++ /* info_val == the ctx attribute now on ctx */ ++ KBASE_TRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_ON_CTX), ++ /* info_val == the ctx attribute now on runpool */ ++ KBASE_TRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_ON_RUNPOOL), ++ /* info_val == the ctx attribute now off ctx */ ++ KBASE_TRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_OFF_CTX), ++ /* info_val == the ctx attribute now off runpool */ ++ KBASE_TRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_OFF_RUNPOOL), ++/* ++ * Scheduler Policy events ++ */ ++ KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_INIT_CTX), ++ KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_TERM_CTX), ++ /* info_val == whether it was evicted */ ++ KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_TRY_EVICT_CTX), ++ KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_FOREACH_CTX_JOBS), ++ KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_ENQUEUE_CTX), ++ KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_DEQUEUE_HEAD_CTX), ++ KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_RUNPOOL_ADD_CTX), ++ KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_RUNPOOL_REMOVE_CTX), ++ KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_DEQUEUE_JOB), ++ KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_DEQUEUE_JOB_IRQ), ++ /* gpu_addr==JS_HEAD to write if the job were run */ ++ KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_ENQUEUE_JOB), ++ KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_TIMER_START), ++ KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_TIMER_END), ++/* ++ * Power Management Events ++ */ ++ KBASE_TRACE_CODE_MAKE_CODE(PM_JOB_SUBMIT_AFTER_POWERING_UP), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_JOB_SUBMIT_AFTER_POWERED_UP), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_PWRON), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_PWRON_TILER), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_PWRON_L2), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_PWROFF), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_PWROFF_TILER), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_PWROFF_L2), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_POWERED), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_POWERED_TILER), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_POWERED_L2), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_DESIRED), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_DESIRED_TILER), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_AVAILABLE), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_AVAILABLE_TILER), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_AVAILABLE), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_AVAILABLE_TILER), ++ /* PM_DESIRED_REACHED: gpu_addr == pm.gpu_in_desired_state */ ++ KBASE_TRACE_CODE_MAKE_CODE(PM_DESIRED_REACHED), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_DESIRED_REACHED_TILER), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_REGISTER_CHANGE_SHADER_INUSE), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_REGISTER_CHANGE_TILER_INUSE), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_REGISTER_CHANGE_SHADER_NEEDED), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_REGISTER_CHANGE_TILER_NEEDED), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_RELEASE_CHANGE_SHADER_INUSE), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_RELEASE_CHANGE_TILER_INUSE), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_UNREQUEST_CHANGE_SHADER_NEEDED), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_UNREQUEST_CHANGE_TILER_NEEDED), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_REQUEST_CHANGE_SHADER_NEEDED), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_REQUEST_CHANGE_TILER_NEEDED), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_WAKE_WAITERS), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_CONTEXT_ACTIVE), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_CONTEXT_IDLE), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_GPU_ON), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_GPU_OFF), ++ /* info_val == policy number, or -1 for "Already changing" */ ++ KBASE_TRACE_CODE_MAKE_CODE(PM_SET_POLICY), ++ KBASE_TRACE_CODE_MAKE_CODE(PM_CA_SET_POLICY), ++ /* info_val == policy number */ ++ KBASE_TRACE_CODE_MAKE_CODE(PM_CURRENT_POLICY_INIT), ++ /* info_val == policy number */ ++ KBASE_TRACE_CODE_MAKE_CODE(PM_CURRENT_POLICY_TERM), ++/* Unused code just to make it easier to not have a comma at the end. ++ * All other codes MUST come before this */ ++ KBASE_TRACE_CODE_MAKE_CODE(DUMMY) ++ ++#if 0 /* Dummy section to avoid breaking formatting */ ++}; ++#endif ++ ++/* ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_trace_timeline.c b/drivers/gpu/arm/midgard/mali_kbase_trace_timeline.c +new file mode 100755 +index 000000000000..5830e87f0818 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_trace_timeline.c +@@ -0,0 +1,236 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#include ++#include ++#include ++ ++#define CREATE_TRACE_POINTS ++ ++#ifdef CONFIG_MALI_TRACE_TIMELINE ++#include "mali_timeline.h" ++ ++#include ++#include ++ ++EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_atoms_in_flight); ++EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_atom); ++EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_gpu_slot_active); ++EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_gpu_slot_action); ++EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_gpu_power_active); ++EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_l2_power_active); ++EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_pm_event); ++EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_slot_atom); ++EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_pm_checktrans); ++EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_context_active); ++ ++struct kbase_trace_timeline_desc { ++ char *enum_str; ++ char *desc; ++ char *format; ++ char *format_desc; ++}; ++ ++static struct kbase_trace_timeline_desc kbase_trace_timeline_desc_table[] = { ++ #define KBASE_TIMELINE_TRACE_CODE(enum_val, desc, format, format_desc) { #enum_val, desc, format, format_desc } ++ #include "mali_kbase_trace_timeline_defs.h" ++ #undef KBASE_TIMELINE_TRACE_CODE ++}; ++ ++#define KBASE_NR_TRACE_CODES ARRAY_SIZE(kbase_trace_timeline_desc_table) ++ ++static void *kbasep_trace_timeline_seq_start(struct seq_file *s, loff_t *pos) ++{ ++ if (*pos >= KBASE_NR_TRACE_CODES) ++ return NULL; ++ ++ return &kbase_trace_timeline_desc_table[*pos]; ++} ++ ++static void kbasep_trace_timeline_seq_stop(struct seq_file *s, void *data) ++{ ++} ++ ++static void *kbasep_trace_timeline_seq_next(struct seq_file *s, void *data, loff_t *pos) ++{ ++ (*pos)++; ++ ++ if (*pos == KBASE_NR_TRACE_CODES) ++ return NULL; ++ ++ return &kbase_trace_timeline_desc_table[*pos]; ++} ++ ++static int kbasep_trace_timeline_seq_show(struct seq_file *s, void *data) ++{ ++ struct kbase_trace_timeline_desc *trace_desc = data; ++ ++ seq_printf(s, "%s#%s#%s#%s\n", trace_desc->enum_str, trace_desc->desc, trace_desc->format, trace_desc->format_desc); ++ return 0; ++} ++ ++ ++static const struct seq_operations kbasep_trace_timeline_seq_ops = { ++ .start = kbasep_trace_timeline_seq_start, ++ .next = kbasep_trace_timeline_seq_next, ++ .stop = kbasep_trace_timeline_seq_stop, ++ .show = kbasep_trace_timeline_seq_show, ++}; ++ ++static int kbasep_trace_timeline_debugfs_open(struct inode *inode, struct file *file) ++{ ++ return seq_open(file, &kbasep_trace_timeline_seq_ops); ++} ++ ++static const struct file_operations kbasep_trace_timeline_debugfs_fops = { ++ .open = kbasep_trace_timeline_debugfs_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = seq_release, ++}; ++ ++#ifdef CONFIG_DEBUG_FS ++ ++void kbasep_trace_timeline_debugfs_init(struct kbase_device *kbdev) ++{ ++ debugfs_create_file("mali_timeline_defs", ++ S_IRUGO, kbdev->mali_debugfs_directory, NULL, ++ &kbasep_trace_timeline_debugfs_fops); ++} ++ ++#endif /* CONFIG_DEBUG_FS */ ++ ++void kbase_timeline_job_slot_submit(struct kbase_device *kbdev, struct kbase_context *kctx, ++ struct kbase_jd_atom *katom, int js) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (kbdev->timeline.slot_atoms_submitted[js] > 0) { ++ KBASE_TIMELINE_JOB_START_NEXT(kctx, js, 1); ++ } else { ++ base_atom_id atom_number = kbase_jd_atom_id(kctx, katom); ++ ++ KBASE_TIMELINE_JOB_START_HEAD(kctx, js, 1); ++ KBASE_TIMELINE_JOB_START(kctx, js, atom_number); ++ } ++ ++kbdev->timeline.slot_atoms_submitted[js]; ++ ++ KBASE_TIMELINE_ATOMS_SUBMITTED(kctx, js, kbdev->timeline.slot_atoms_submitted[js]); ++} ++ ++void kbase_timeline_job_slot_done(struct kbase_device *kbdev, struct kbase_context *kctx, ++ struct kbase_jd_atom *katom, int js, ++ kbasep_js_atom_done_code done_code) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ ++ if (done_code & KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT) { ++ KBASE_TIMELINE_JOB_START_NEXT(kctx, js, 0); ++ } else { ++ /* Job finished in JS_HEAD */ ++ base_atom_id atom_number = kbase_jd_atom_id(kctx, katom); ++ ++ KBASE_TIMELINE_JOB_START_HEAD(kctx, js, 0); ++ KBASE_TIMELINE_JOB_STOP(kctx, js, atom_number); ++ ++ /* see if we need to trace the job in JS_NEXT moving to JS_HEAD */ ++ if (kbase_backend_nr_atoms_submitted(kbdev, js)) { ++ struct kbase_jd_atom *next_katom; ++ struct kbase_context *next_kctx; ++ ++ /* Peek the next atom - note that the atom in JS_HEAD will already ++ * have been dequeued */ ++ next_katom = kbase_backend_inspect_head(kbdev, js); ++ WARN_ON(!next_katom); ++ next_kctx = next_katom->kctx; ++ KBASE_TIMELINE_JOB_START_NEXT(next_kctx, js, 0); ++ KBASE_TIMELINE_JOB_START_HEAD(next_kctx, js, 1); ++ KBASE_TIMELINE_JOB_START(next_kctx, js, kbase_jd_atom_id(next_kctx, next_katom)); ++ } ++ } ++ ++ --kbdev->timeline.slot_atoms_submitted[js]; ++ ++ KBASE_TIMELINE_ATOMS_SUBMITTED(kctx, js, kbdev->timeline.slot_atoms_submitted[js]); ++} ++ ++void kbase_timeline_pm_send_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event_sent) ++{ ++ int uid = 0; ++ int old_uid; ++ ++ /* If a producer already exists for the event, try to use their UID (multiple-producers) */ ++ uid = atomic_read(&kbdev->timeline.pm_event_uid[event_sent]); ++ old_uid = uid; ++ ++ /* Get a new non-zero UID if we don't have one yet */ ++ while (!uid) ++ uid = atomic_inc_return(&kbdev->timeline.pm_event_uid_counter); ++ ++ /* Try to use this UID */ ++ if (old_uid != atomic_cmpxchg(&kbdev->timeline.pm_event_uid[event_sent], old_uid, uid)) ++ /* If it changed, raced with another producer: we've lost this UID */ ++ uid = 0; ++ ++ KBASE_TIMELINE_PM_SEND_EVENT(kbdev, event_sent, uid); ++} ++ ++void kbase_timeline_pm_check_handle_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event) ++{ ++ int uid = atomic_read(&kbdev->timeline.pm_event_uid[event]); ++ ++ if (uid != 0) { ++ if (uid != atomic_cmpxchg(&kbdev->timeline.pm_event_uid[event], uid, 0)) ++ /* If it changed, raced with another consumer: we've lost this UID */ ++ uid = 0; ++ ++ KBASE_TIMELINE_PM_HANDLE_EVENT(kbdev, event, uid); ++ } ++} ++ ++void kbase_timeline_pm_handle_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event) ++{ ++ int uid = atomic_read(&kbdev->timeline.pm_event_uid[event]); ++ ++ if (uid != atomic_cmpxchg(&kbdev->timeline.pm_event_uid[event], uid, 0)) ++ /* If it changed, raced with another consumer: we've lost this UID */ ++ uid = 0; ++ ++ KBASE_TIMELINE_PM_HANDLE_EVENT(kbdev, event, uid); ++} ++ ++void kbase_timeline_pm_l2_transition_start(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ /* Simply log the start of the transition */ ++ kbdev->timeline.l2_transitioning = true; ++ KBASE_TIMELINE_POWERING_L2(kbdev); ++} ++ ++void kbase_timeline_pm_l2_transition_done(struct kbase_device *kbdev) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++ /* Simply log the end of the transition */ ++ if (kbdev->timeline.l2_transitioning) { ++ kbdev->timeline.l2_transitioning = false; ++ KBASE_TIMELINE_POWERED_L2(kbdev); ++ } ++} ++ ++#endif /* CONFIG_MALI_TRACE_TIMELINE */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_trace_timeline.h b/drivers/gpu/arm/midgard/mali_kbase_trace_timeline.h +new file mode 100755 +index 000000000000..a04f7c1420e0 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_trace_timeline.h +@@ -0,0 +1,363 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#if !defined(_KBASE_TRACE_TIMELINE_H) ++#define _KBASE_TRACE_TIMELINE_H ++ ++#ifdef CONFIG_MALI_TRACE_TIMELINE ++ ++enum kbase_trace_timeline_code { ++ #define KBASE_TIMELINE_TRACE_CODE(enum_val, desc, format, format_desc) enum_val ++ #include "mali_kbase_trace_timeline_defs.h" ++ #undef KBASE_TIMELINE_TRACE_CODE ++}; ++ ++#ifdef CONFIG_DEBUG_FS ++ ++/** Initialize Timeline DebugFS entries */ ++void kbasep_trace_timeline_debugfs_init(struct kbase_device *kbdev); ++ ++#else /* CONFIG_DEBUG_FS */ ++ ++#define kbasep_trace_timeline_debugfs_init CSTD_NOP ++ ++#endif /* CONFIG_DEBUG_FS */ ++ ++/* mali_timeline.h defines kernel tracepoints used by the KBASE_TIMELINE ++ * functions. ++ * Output is timestamped by either sched_clock() (default), local_clock(), or ++ * cpu_clock(), depending on /sys/kernel/debug/tracing/trace_clock */ ++#include "mali_timeline.h" ++ ++/* Trace number of atoms in flight for kctx (atoms either not completed, or in ++ process of being returned to user */ ++#define KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, count) \ ++ do { \ ++ struct timespec64 ts; \ ++ ktime_get_raw_ts64(&ts); \ ++ trace_mali_timeline_atoms_in_flight(ts.tv_sec, ts.tv_nsec, \ ++ (int)kctx->timeline.owner_tgid, \ ++ count); \ ++ } while (0) ++ ++/* Trace atom_id being Ready to Run */ ++#define KBASE_TIMELINE_ATOM_READY(kctx, atom_id) \ ++ do { \ ++ struct timespec64 ts; \ ++ ktime_get_raw_ts64(&ts); \ ++ trace_mali_timeline_atom(ts.tv_sec, ts.tv_nsec, \ ++ CTX_FLOW_ATOM_READY, \ ++ (int)kctx->timeline.owner_tgid, \ ++ atom_id); \ ++ } while (0) ++ ++/* Trace number of atoms submitted to job slot js ++ * ++ * NOTE: This uses a different tracepoint to the head/next/soft-stop actions, ++ * so that those actions can be filtered out separately from this ++ * ++ * This is because this is more useful, as we can use it to calculate general ++ * utilization easily and accurately */ ++#define KBASE_TIMELINE_ATOMS_SUBMITTED(kctx, js, count) \ ++ do { \ ++ struct timespec64 ts; \ ++ ktime_get_raw_ts64(&ts); \ ++ trace_mali_timeline_gpu_slot_active(ts.tv_sec, ts.tv_nsec, \ ++ SW_SET_GPU_SLOT_ACTIVE, \ ++ (int)kctx->timeline.owner_tgid, \ ++ js, count); \ ++ } while (0) ++ ++ ++/* Trace atoms present in JS_NEXT */ ++#define KBASE_TIMELINE_JOB_START_NEXT(kctx, js, count) \ ++ do { \ ++ struct timespec64 ts; \ ++ ktime_get_raw_ts64(&ts); \ ++ trace_mali_timeline_gpu_slot_action(ts.tv_sec, ts.tv_nsec, \ ++ SW_SET_GPU_SLOT_NEXT, \ ++ (int)kctx->timeline.owner_tgid, \ ++ js, count); \ ++ } while (0) ++ ++/* Trace atoms present in JS_HEAD */ ++#define KBASE_TIMELINE_JOB_START_HEAD(kctx, js, count) \ ++ do { \ ++ struct timespec64 ts; \ ++ ktime_get_raw_ts64(&ts); \ ++ trace_mali_timeline_gpu_slot_action(ts.tv_sec, ts.tv_nsec, \ ++ SW_SET_GPU_SLOT_HEAD, \ ++ (int)kctx->timeline.owner_tgid, \ ++ js, count); \ ++ } while (0) ++ ++/* Trace that a soft stop/evict from next is being attempted on a slot */ ++#define KBASE_TIMELINE_TRY_SOFT_STOP(kctx, js, count) \ ++ do { \ ++ struct timespec64 ts; \ ++ ktime_get_raw_ts64(&ts); \ ++ trace_mali_timeline_gpu_slot_action(ts.tv_sec, ts.tv_nsec, \ ++ SW_SET_GPU_SLOT_STOPPING, \ ++ (kctx) ? (int)kctx->timeline.owner_tgid : 0, \ ++ js, count); \ ++ } while (0) ++ ++ ++ ++/* Trace state of overall GPU power */ ++#define KBASE_TIMELINE_GPU_POWER(kbdev, active) \ ++ do { \ ++ struct timespec64 ts; \ ++ ktime_get_raw_ts64(&ts); \ ++ trace_mali_timeline_gpu_power_active(ts.tv_sec, ts.tv_nsec, \ ++ SW_SET_GPU_POWER_ACTIVE, active); \ ++ } while (0) ++ ++/* Trace state of tiler power */ ++#define KBASE_TIMELINE_POWER_TILER(kbdev, bitmap) \ ++ do { \ ++ struct timespec64 ts; \ ++ ktime_get_raw_ts64(&ts); \ ++ trace_mali_timeline_gpu_power_active(ts.tv_sec, ts.tv_nsec, \ ++ SW_SET_GPU_POWER_TILER_ACTIVE, \ ++ hweight64(bitmap)); \ ++ } while (0) ++ ++/* Trace number of shaders currently powered */ ++#define KBASE_TIMELINE_POWER_SHADER(kbdev, bitmap) \ ++ do { \ ++ struct timespec64 ts; \ ++ ktime_get_raw_ts64(&ts); \ ++ trace_mali_timeline_gpu_power_active(ts.tv_sec, ts.tv_nsec, \ ++ SW_SET_GPU_POWER_SHADER_ACTIVE, \ ++ hweight64(bitmap)); \ ++ } while (0) ++ ++/* Trace state of L2 power */ ++#define KBASE_TIMELINE_POWER_L2(kbdev, bitmap) \ ++ do { \ ++ struct timespec64 ts; \ ++ ktime_get_raw_ts64(&ts); \ ++ trace_mali_timeline_gpu_power_active(ts.tv_sec, ts.tv_nsec, \ ++ SW_SET_GPU_POWER_L2_ACTIVE, \ ++ hweight64(bitmap)); \ ++ } while (0) ++ ++/* Trace state of L2 cache*/ ++#define KBASE_TIMELINE_POWERING_L2(kbdev) \ ++ do { \ ++ struct timespec64 ts; \ ++ ktime_get_raw_ts64(&ts); \ ++ trace_mali_timeline_l2_power_active(ts.tv_sec, ts.tv_nsec, \ ++ SW_FLOW_GPU_POWER_L2_POWERING, \ ++ 1); \ ++ } while (0) ++ ++#define KBASE_TIMELINE_POWERED_L2(kbdev) \ ++ do { \ ++ struct timespec64 ts; \ ++ ktime_get_raw_ts64(&ts); \ ++ trace_mali_timeline_l2_power_active(ts.tv_sec, ts.tv_nsec, \ ++ SW_FLOW_GPU_POWER_L2_ACTIVE, \ ++ 1); \ ++ } while (0) ++ ++/* Trace kbase_pm_send_event message send */ ++#define KBASE_TIMELINE_PM_SEND_EVENT(kbdev, event_type, pm_event_id) \ ++ do { \ ++ struct timespec64 ts; \ ++ ktime_get_raw_ts64(&ts); \ ++ trace_mali_timeline_pm_event(ts.tv_sec, ts.tv_nsec, \ ++ SW_FLOW_PM_SEND_EVENT, \ ++ event_type, pm_event_id); \ ++ } while (0) ++ ++/* Trace kbase_pm_worker message receive */ ++#define KBASE_TIMELINE_PM_HANDLE_EVENT(kbdev, event_type, pm_event_id) \ ++ do { \ ++ struct timespec64 ts; \ ++ ktime_get_raw_ts64(&ts); \ ++ trace_mali_timeline_pm_event(ts.tv_sec, ts.tv_nsec, \ ++ SW_FLOW_PM_HANDLE_EVENT, \ ++ event_type, pm_event_id); \ ++ } while (0) ++ ++ ++/* Trace atom_id starting in JS_HEAD */ ++#define KBASE_TIMELINE_JOB_START(kctx, js, _consumerof_atom_number) \ ++ do { \ ++ struct timespec64 ts; \ ++ ktime_get_raw_ts64(&ts); \ ++ trace_mali_timeline_slot_atom(ts.tv_sec, ts.tv_nsec, \ ++ HW_START_GPU_JOB_CHAIN_SW_APPROX, \ ++ (int)kctx->timeline.owner_tgid, \ ++ js, _consumerof_atom_number); \ ++ } while (0) ++ ++/* Trace atom_id stopping on JS_HEAD */ ++#define KBASE_TIMELINE_JOB_STOP(kctx, js, _producerof_atom_number_completed) \ ++ do { \ ++ struct timespec64 ts; \ ++ ktime_get_raw_ts64(&ts); \ ++ trace_mali_timeline_slot_atom(ts.tv_sec, ts.tv_nsec, \ ++ HW_STOP_GPU_JOB_CHAIN_SW_APPROX, \ ++ (int)kctx->timeline.owner_tgid, \ ++ js, _producerof_atom_number_completed); \ ++ } while (0) ++ ++/** Trace beginning/end of a call to kbase_pm_check_transitions_nolock from a ++ * certin caller */ ++#define KBASE_TIMELINE_PM_CHECKTRANS(kbdev, trace_code) \ ++ do { \ ++ struct timespec64 ts; \ ++ ktime_get_raw_ts64(&ts); \ ++ trace_mali_timeline_pm_checktrans(ts.tv_sec, ts.tv_nsec, \ ++ trace_code, 1); \ ++ } while (0) ++ ++/* Trace number of contexts active */ ++#define KBASE_TIMELINE_CONTEXT_ACTIVE(kbdev, count) \ ++ do { \ ++ struct timespec64 ts; \ ++ ktime_get_raw_ts64(&ts); \ ++ trace_mali_timeline_context_active(ts.tv_sec, ts.tv_nsec, \ ++ count); \ ++ } while (0) ++ ++/* NOTE: kbase_timeline_pm_cores_func() is in mali_kbase_pm_policy.c */ ++ ++/** ++ * Trace that an atom is starting on a job slot ++ * ++ * The caller must be holding hwaccess_lock ++ */ ++void kbase_timeline_job_slot_submit(struct kbase_device *kbdev, struct kbase_context *kctx, ++ struct kbase_jd_atom *katom, int js); ++ ++/** ++ * Trace that an atom has done on a job slot ++ * ++ * 'Done' in this sense can occur either because: ++ * - the atom in JS_HEAD finished ++ * - the atom in JS_NEXT was evicted ++ * ++ * Whether the atom finished or was evicted is passed in @a done_code ++ * ++ * It is assumed that the atom has already been removed from the submit slot, ++ * with either: ++ * - kbasep_jm_dequeue_submit_slot() ++ * - kbasep_jm_dequeue_tail_submit_slot() ++ * ++ * The caller must be holding hwaccess_lock ++ */ ++void kbase_timeline_job_slot_done(struct kbase_device *kbdev, struct kbase_context *kctx, ++ struct kbase_jd_atom *katom, int js, ++ kbasep_js_atom_done_code done_code); ++ ++ ++/** Trace a pm event starting */ ++void kbase_timeline_pm_send_event(struct kbase_device *kbdev, ++ enum kbase_timeline_pm_event event_sent); ++ ++/** Trace a pm event finishing */ ++void kbase_timeline_pm_check_handle_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event); ++ ++/** Check whether a pm event was present, and if so trace finishing it */ ++void kbase_timeline_pm_handle_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event); ++ ++/** Trace L2 power-up start */ ++void kbase_timeline_pm_l2_transition_start(struct kbase_device *kbdev); ++ ++/** Trace L2 power-up done */ ++void kbase_timeline_pm_l2_transition_done(struct kbase_device *kbdev); ++ ++#else ++ ++#define KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, count) CSTD_NOP() ++ ++#define KBASE_TIMELINE_ATOM_READY(kctx, atom_id) CSTD_NOP() ++ ++#define KBASE_TIMELINE_ATOMS_SUBMITTED(kctx, js, count) CSTD_NOP() ++ ++#define KBASE_TIMELINE_JOB_START_NEXT(kctx, js, count) CSTD_NOP() ++ ++#define KBASE_TIMELINE_JOB_START_HEAD(kctx, js, count) CSTD_NOP() ++ ++#define KBASE_TIMELINE_TRY_SOFT_STOP(kctx, js, count) CSTD_NOP() ++ ++#define KBASE_TIMELINE_GPU_POWER(kbdev, active) CSTD_NOP() ++ ++#define KBASE_TIMELINE_POWER_TILER(kbdev, bitmap) CSTD_NOP() ++ ++#define KBASE_TIMELINE_POWER_SHADER(kbdev, bitmap) CSTD_NOP() ++ ++#define KBASE_TIMELINE_POWER_L2(kbdev, active) CSTD_NOP() ++ ++#define KBASE_TIMELINE_POWERING_L2(kbdev) CSTD_NOP() ++ ++#define KBASE_TIMELINE_POWERED_L2(kbdev) CSTD_NOP() ++ ++#define KBASE_TIMELINE_PM_SEND_EVENT(kbdev, event_type, pm_event_id) CSTD_NOP() ++ ++#define KBASE_TIMELINE_PM_HANDLE_EVENT(kbdev, event_type, pm_event_id) CSTD_NOP() ++ ++#define KBASE_TIMELINE_JOB_START(kctx, js, _consumerof_atom_number) CSTD_NOP() ++ ++#define KBASE_TIMELINE_JOB_STOP(kctx, js, _producerof_atom_number_completed) CSTD_NOP() ++ ++#define KBASE_TIMELINE_PM_CHECKTRANS(kbdev, trace_code) CSTD_NOP() ++ ++#define KBASE_TIMELINE_CONTEXT_ACTIVE(kbdev, count) CSTD_NOP() ++ ++static inline void kbase_timeline_job_slot_submit(struct kbase_device *kbdev, struct kbase_context *kctx, ++ struct kbase_jd_atom *katom, int js) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++} ++ ++static inline void kbase_timeline_job_slot_done(struct kbase_device *kbdev, struct kbase_context *kctx, ++ struct kbase_jd_atom *katom, int js, ++ kbasep_js_atom_done_code done_code) ++{ ++ lockdep_assert_held(&kbdev->hwaccess_lock); ++} ++ ++static inline void kbase_timeline_pm_send_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event_sent) ++{ ++} ++ ++static inline void kbase_timeline_pm_check_handle_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event) ++{ ++} ++ ++static inline void kbase_timeline_pm_handle_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event) ++{ ++} ++ ++static inline void kbase_timeline_pm_l2_transition_start(struct kbase_device *kbdev) ++{ ++} ++ ++static inline void kbase_timeline_pm_l2_transition_done(struct kbase_device *kbdev) ++{ ++} ++#endif /* CONFIG_MALI_TRACE_TIMELINE */ ++ ++#endif /* _KBASE_TRACE_TIMELINE_H */ ++ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_trace_timeline_defs.h b/drivers/gpu/arm/midgard/mali_kbase_trace_timeline_defs.h +new file mode 100755 +index 000000000000..156a95a67f4a +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_trace_timeline_defs.h +@@ -0,0 +1,140 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/* ***** IMPORTANT: THIS IS NOT A NORMAL HEADER FILE ***** ++ * ***** DO NOT INCLUDE DIRECTLY ***** ++ * ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** */ ++ ++/* ++ * Conventions on Event Names: ++ * ++ * - The prefix determines something about how the timeline should be ++ * displayed, and is split up into various parts, separated by underscores: ++ * - 'SW' and 'HW' as the first part will be used to determine whether a ++ * timeline is to do with Software or Hardware - effectively, separate ++ * 'channels' for Software and Hardware ++ * - 'START', 'STOP', 'ENTER', 'LEAVE' can be used in the second part, and ++ * signify related pairs of events - these are optional. ++ * - 'FLOW' indicates a generic event, which can use dependencies ++ * - This gives events such as: ++ * - 'SW_ENTER_FOO' ++ * - 'SW_LEAVE_FOO' ++ * - 'SW_FLOW_BAR_1' ++ * - 'SW_FLOW_BAR_2' ++ * - 'HW_START_BAZ' ++ * - 'HW_STOP_BAZ' ++ * - And an unadorned HW event: ++ * - 'HW_BAZ_FROZBOZ' ++ */ ++ ++/* ++ * Conventions on parameter names: ++ * - anything with 'instance' in the name will have a separate timeline based ++ * on that instances. ++ * - underscored-prefixed parameters will by hidden by default on timelines ++ * ++ * Hence: ++ * - Different job slots have their own 'instance', based on the instance value ++ * - Per-context info (e.g. atoms on a context) have their own 'instance' ++ * (i.e. each context should be on a different timeline) ++ * ++ * Note that globally-shared resources can be tagged with a tgid, but we don't ++ * want an instance per context: ++ * - There's no point having separate Job Slot timelines for each context, that ++ * would be confusing - there's only really 3 job slots! ++ * - There's no point having separate Shader-powered timelines for each ++ * context, that would be confusing - all shader cores (whether it be 4, 8, ++ * etc) are shared in the system. ++ */ ++ ++ /* ++ * CTX events ++ */ ++ /* Separate timelines for each context 'instance'*/ ++ KBASE_TIMELINE_TRACE_CODE(CTX_SET_NR_ATOMS_IN_FLIGHT, "CTX: Atoms in flight", "%d,%d", "_instance_tgid,_value_number_of_atoms"), ++ KBASE_TIMELINE_TRACE_CODE(CTX_FLOW_ATOM_READY, "CTX: Atoms Ready to Run", "%d,%d,%d", "_instance_tgid,_consumerof_atom_number,_producerof_atom_number_ready"), ++ ++ /* ++ * SW Events ++ */ ++ /* Separate timelines for each slot 'instance' */ ++ KBASE_TIMELINE_TRACE_CODE(SW_SET_GPU_SLOT_ACTIVE, "SW: GPU slot active", "%d,%d,%d", "_tgid,_instance_slot,_value_number_of_atoms"), ++ KBASE_TIMELINE_TRACE_CODE(SW_SET_GPU_SLOT_NEXT, "SW: GPU atom in NEXT", "%d,%d,%d", "_tgid,_instance_slot,_value_is_an_atom_in_next"), ++ KBASE_TIMELINE_TRACE_CODE(SW_SET_GPU_SLOT_HEAD, "SW: GPU atom in HEAD", "%d,%d,%d", "_tgid,_instance_slot,_value_is_an_atom_in_head"), ++ KBASE_TIMELINE_TRACE_CODE(SW_SET_GPU_SLOT_STOPPING, "SW: Try Soft-Stop on GPU slot", "%d,%d,%d", "_tgid,_instance_slot,_value_is_slot_stopping"), ++ /* Shader and overall power is shared - can't have separate instances of ++ * it, just tagging with the context */ ++ KBASE_TIMELINE_TRACE_CODE(SW_SET_GPU_POWER_ACTIVE, "SW: GPU power active", "%d,%d", "_tgid,_value_is_power_active"), ++ KBASE_TIMELINE_TRACE_CODE(SW_SET_GPU_POWER_TILER_ACTIVE, "SW: GPU tiler powered", "%d,%d", "_tgid,_value_number_of_tilers"), ++ KBASE_TIMELINE_TRACE_CODE(SW_SET_GPU_POWER_SHADER_ACTIVE, "SW: GPU shaders powered", "%d,%d", "_tgid,_value_number_of_shaders"), ++ KBASE_TIMELINE_TRACE_CODE(SW_SET_GPU_POWER_L2_ACTIVE, "SW: GPU L2 powered", "%d,%d", "_tgid,_value_number_of_l2"), ++ ++ /* SW Power event messaging. _event_type is one from the kbase_pm_event enum */ ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_SEND_EVENT, "SW: PM Send Event", "%d,%d,%d", "_tgid,_event_type,_writerof_pm_event_id"), ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_HANDLE_EVENT, "SW: PM Handle Event", "%d,%d,%d", "_tgid,_event_type,_finalconsumerof_pm_event_id"), ++ /* SW L2 power events */ ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_GPU_POWER_L2_POWERING, "SW: GPU L2 powering", "%d,%d", "_tgid,_writerof_l2_transitioning"), ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_GPU_POWER_L2_ACTIVE, "SW: GPU L2 powering done", "%d,%d", "_tgid,_finalconsumerof_l2_transitioning"), ++ ++ KBASE_TIMELINE_TRACE_CODE(SW_SET_CONTEXT_ACTIVE, "SW: Context Active", "%d,%d", "_tgid,_value_active"), ++ ++ /* ++ * BEGIN: Significant SW Functions that call kbase_pm_check_transitions_nolock() ++ */ ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_DO_POWEROFF_START, "SW: PM CheckTrans from kbase_pm_do_poweroff", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_do_poweroff"), ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_DO_POWEROFF_END, "SW: PM CheckTrans from kbase_pm_do_poweroff", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_do_poweroff"), ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_DO_POWERON_START, "SW: PM CheckTrans from kbase_pm_do_poweron", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_do_poweron"), ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_DO_POWERON_END, "SW: PM CheckTrans from kbase_pm_do_poweron", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_do_poweron"), ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_GPU_INTERRUPT_START, "SW: PM CheckTrans from kbase_gpu_interrupt", "%d,%d", "_tgid,_writerof_pm_checktrans_gpu_interrupt"), ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_GPU_INTERRUPT_END, "SW: PM CheckTrans from kbase_gpu_interrupt", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_gpu_interrupt"), ++ ++ /* ++ * Significant Indirect callers of kbase_pm_check_transitions_nolock() ++ */ ++ /* kbase_pm_request_cores */ ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_START, "SW: PM CheckTrans from kbase_pm_request_cores(shader)", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_request_cores_shader"), ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_END, "SW: PM CheckTrans from kbase_pm_request_cores(shader)", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_request_cores_shader"), ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_TILER_START, "SW: PM CheckTrans from kbase_pm_request_cores(tiler)", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_request_cores_tiler"), ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_TILER_END, "SW: PM CheckTrans from kbase_pm_request_cores(tiler)", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_request_cores_tiler"), ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_TILER_START, "SW: PM CheckTrans from kbase_pm_request_cores(shader+tiler)", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_request_cores_shader_tiler"), ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_TILER_END, "SW: PM CheckTrans from kbase_pm_request_cores(shader+tiler)", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_request_cores_shader_tiler"), ++ /* kbase_pm_release_cores */ ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_START, "SW: PM CheckTrans from kbase_pm_release_cores(shader)", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_release_cores_shader"), ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_END, "SW: PM CheckTrans from kbase_pm_release_cores(shader)", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_release_cores_shader"), ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_TILER_START, "SW: PM CheckTrans from kbase_pm_release_cores(tiler)", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_release_cores_tiler"), ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_TILER_END, "SW: PM CheckTrans from kbase_pm_release_cores(tiler)", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_release_cores_tiler"), ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_TILER_START, "SW: PM CheckTrans from kbase_pm_release_cores(shader+tiler)", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_release_cores_shader_tiler"), ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_TILER_END, "SW: PM CheckTrans from kbase_pm_release_cores(shader+tiler)", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_release_cores_shader_tiler"), ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_DEFERRED_START, "SW: PM CheckTrans from kbasep_pm_do_shader_poweroff_callback", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_do_shader_poweroff_callback"), ++ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_DEFERRED_END, "SW: PM CheckTrans from kbasep_pm_do_shader_poweroff_callback", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_do_shader_poweroff_callback"), ++ /* ++ * END: SW Functions that call kbase_pm_check_transitions_nolock() ++ */ ++ ++ /* ++ * HW Events ++ */ ++ KBASE_TIMELINE_TRACE_CODE(HW_MMU_FAULT, ++"HW: MMU Fault", "%d,%d,%d", "_tgid,fault_type,fault_stage,asid"), ++ KBASE_TIMELINE_TRACE_CODE(HW_START_GPU_JOB_CHAIN_SW_APPROX, ++"HW: Job Chain start (SW approximated)", "%d,%d,%d", ++"_tgid,job_slot,_consumerof_atom_number_ready"), ++ KBASE_TIMELINE_TRACE_CODE(HW_STOP_GPU_JOB_CHAIN_SW_APPROX, ++"HW: Job Chain stop (SW approximated)", "%d,%d,%d", ++"_tgid,job_slot,_producerof_atom_number_completed") +diff --git a/drivers/gpu/arm/midgard/mali_kbase_uku.h b/drivers/gpu/arm/midgard/mali_kbase_uku.h +new file mode 100755 +index 000000000000..c22a59324248 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_uku.h +@@ -0,0 +1,545 @@ ++/* ++ * ++ * (C) COPYRIGHT 2008-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#ifndef _KBASE_UKU_H_ ++#define _KBASE_UKU_H_ ++ ++#include "mali_uk.h" ++#include "mali_base_kernel.h" ++ ++/* This file needs to support being included from kernel and userside (which use different defines) */ ++#if defined(CONFIG_MALI_ERROR_INJECT) || MALI_ERROR_INJECT_ON ++#define SUPPORT_MALI_ERROR_INJECT ++#endif /* defined(CONFIG_MALI_ERROR_INJECT) || MALI_ERROR_INJECT_ON */ ++#if defined(CONFIG_MALI_NO_MALI) ++#define SUPPORT_MALI_NO_MALI ++#elif defined(MALI_NO_MALI) ++#if MALI_NO_MALI ++#define SUPPORT_MALI_NO_MALI ++#endif ++#endif ++ ++#if defined(SUPPORT_MALI_NO_MALI) || defined(SUPPORT_MALI_ERROR_INJECT) ++#include "backend/gpu/mali_kbase_model_dummy.h" ++#endif ++ ++#include "mali_kbase_gpuprops_types.h" ++ ++/* ++ * 10.1: ++ * - Do mmap in kernel for SAME_VA memory allocations rather then ++ * calling back into the kernel as a 2nd stage of the allocation request. ++ * ++ * 10.2: ++ * - Add KBASE_FUNC_MEM_JIT_INIT which allows clients to request a custom VA ++ * region for use with JIT (ignored on 32-bit platforms) ++ * ++ * 10.3: ++ * - base_jd_core_req typedef-ed to u32 (instead of to u16) ++ * - two flags added: BASE_JD_REQ_SKIP_CACHE_STAT / _END ++ * ++ * 10.4: ++ * - Removed KBASE_FUNC_EXT_BUFFER_LOCK used only in internal tests ++ * ++ * 10.5: ++ * - Reverted to performing mmap in user space so that tools like valgrind work. ++ * ++ * 10.6: ++ * - Add flags input variable to KBASE_FUNC_TLSTREAM_ACQUIRE ++ */ ++#define BASE_UK_VERSION_MAJOR 10 ++#define BASE_UK_VERSION_MINOR 6 ++ ++#define LINUX_UK_BASE_MAGIC 0x80 ++ ++struct kbase_uk_mem_alloc { ++ union uk_header header; ++ /* IN */ ++ u64 va_pages; ++ u64 commit_pages; ++ u64 extent; ++ /* IN/OUT */ ++ u64 flags; ++ /* OUT */ ++ u64 gpu_va; ++ u16 va_alignment; ++ u8 padding[6]; ++}; ++ ++struct kbase_uk_mem_free { ++ union uk_header header; ++ /* IN */ ++ u64 gpu_addr; ++ /* OUT */ ++}; ++ ++struct kbase_uk_mem_alias { ++ union uk_header header; ++ /* IN/OUT */ ++ u64 flags; ++ /* IN */ ++ u64 stride; ++ u64 nents; ++ union kbase_pointer ai; ++ /* OUT */ ++ u64 gpu_va; ++ u64 va_pages; ++}; ++ ++struct kbase_uk_mem_import { ++ union uk_header header; ++ /* IN */ ++ union kbase_pointer phandle; ++ u32 type; ++ u32 padding; ++ /* IN/OUT */ ++ u64 flags; ++ /* OUT */ ++ u64 gpu_va; ++ u64 va_pages; ++}; ++ ++struct kbase_uk_mem_flags_change { ++ union uk_header header; ++ /* IN */ ++ u64 gpu_va; ++ u64 flags; ++ u64 mask; ++}; ++ ++struct kbase_uk_job_submit { ++ union uk_header header; ++ /* IN */ ++ union kbase_pointer addr; ++ u32 nr_atoms; ++ u32 stride; /* bytes between atoms, i.e. sizeof(base_jd_atom_v2) */ ++ /* OUT */ ++}; ++ ++struct kbase_uk_post_term { ++ union uk_header header; ++}; ++ ++struct kbase_uk_sync_now { ++ union uk_header header; ++ ++ /* IN */ ++ struct base_syncset sset; ++ ++ /* OUT */ ++}; ++ ++struct kbase_uk_hwcnt_setup { ++ union uk_header header; ++ ++ /* IN */ ++ u64 dump_buffer; ++ u32 jm_bm; ++ u32 shader_bm; ++ u32 tiler_bm; ++ u32 unused_1; /* keep for backwards compatibility */ ++ u32 mmu_l2_bm; ++ u32 padding; ++ /* OUT */ ++}; ++ ++/** ++ * struct kbase_uk_hwcnt_reader_setup - User/Kernel space data exchange structure ++ * @header: UK structure header ++ * @buffer_count: requested number of dumping buffers ++ * @jm_bm: counters selection bitmask (JM) ++ * @shader_bm: counters selection bitmask (Shader) ++ * @tiler_bm: counters selection bitmask (Tiler) ++ * @mmu_l2_bm: counters selection bitmask (MMU_L2) ++ * @fd: dumping notification file descriptor ++ * ++ * This structure sets up HWC dumper/reader for this context. ++ * Multiple instances can be created for single context. ++ */ ++struct kbase_uk_hwcnt_reader_setup { ++ union uk_header header; ++ ++ /* IN */ ++ u32 buffer_count; ++ u32 jm_bm; ++ u32 shader_bm; ++ u32 tiler_bm; ++ u32 mmu_l2_bm; ++ ++ /* OUT */ ++ s32 fd; ++}; ++ ++struct kbase_uk_hwcnt_dump { ++ union uk_header header; ++}; ++ ++struct kbase_uk_hwcnt_clear { ++ union uk_header header; ++}; ++ ++struct kbase_uk_fence_validate { ++ union uk_header header; ++ /* IN */ ++ s32 fd; ++ u32 padding; ++ /* OUT */ ++}; ++ ++struct kbase_uk_stream_create { ++ union uk_header header; ++ /* IN */ ++ char name[32]; ++ /* OUT */ ++ s32 fd; ++ u32 padding; ++}; ++ ++struct kbase_uk_gpuprops { ++ union uk_header header; ++ ++ /* IN */ ++ struct mali_base_gpu_props props; ++ /* OUT */ ++}; ++ ++struct kbase_uk_mem_query { ++ union uk_header header; ++ /* IN */ ++ u64 gpu_addr; ++#define KBASE_MEM_QUERY_COMMIT_SIZE 1 ++#define KBASE_MEM_QUERY_VA_SIZE 2 ++#define KBASE_MEM_QUERY_FLAGS 3 ++ u64 query; ++ /* OUT */ ++ u64 value; ++}; ++ ++struct kbase_uk_mem_commit { ++ union uk_header header; ++ /* IN */ ++ u64 gpu_addr; ++ u64 pages; ++ /* OUT */ ++ u32 result_subcode; ++ u32 padding; ++}; ++ ++struct kbase_uk_find_cpu_offset { ++ union uk_header header; ++ /* IN */ ++ u64 gpu_addr; ++ u64 cpu_addr; ++ u64 size; ++ /* OUT */ ++ u64 offset; ++}; ++ ++#define KBASE_GET_VERSION_BUFFER_SIZE 64 ++struct kbase_uk_get_ddk_version { ++ union uk_header header; ++ /* OUT */ ++ char version_buffer[KBASE_GET_VERSION_BUFFER_SIZE]; ++ u32 version_string_size; ++ u32 padding; ++ u32 rk_version; ++}; ++ ++struct kbase_uk_disjoint_query { ++ union uk_header header; ++ /* OUT */ ++ u32 counter; ++ u32 padding; ++}; ++ ++struct kbase_uk_set_flags { ++ union uk_header header; ++ /* IN */ ++ u32 create_flags; ++ u32 padding; ++}; ++ ++#if MALI_UNIT_TEST ++#define TEST_ADDR_COUNT 4 ++#define KBASE_TEST_BUFFER_SIZE 128 ++struct kbase_exported_test_data { ++ u64 test_addr[TEST_ADDR_COUNT]; /**< memory address */ ++ u32 test_addr_pages[TEST_ADDR_COUNT]; /**< memory size in pages */ ++ union kbase_pointer kctx; /**< base context created by process */ ++ union kbase_pointer mm; /**< pointer to process address space */ ++ u8 buffer1[KBASE_TEST_BUFFER_SIZE]; /**< unit test defined parameter */ ++ u8 buffer2[KBASE_TEST_BUFFER_SIZE]; /**< unit test defined parameter */ ++}; ++ ++struct kbase_uk_set_test_data { ++ union uk_header header; ++ /* IN */ ++ struct kbase_exported_test_data test_data; ++}; ++ ++#endif /* MALI_UNIT_TEST */ ++ ++#ifdef SUPPORT_MALI_ERROR_INJECT ++struct kbase_uk_error_params { ++ union uk_header header; ++ /* IN */ ++ struct kbase_error_params params; ++}; ++#endif /* SUPPORT_MALI_ERROR_INJECT */ ++ ++#ifdef SUPPORT_MALI_NO_MALI ++struct kbase_uk_model_control_params { ++ union uk_header header; ++ /* IN */ ++ struct kbase_model_control_params params; ++}; ++#endif /* SUPPORT_MALI_NO_MALI */ ++ ++#ifdef BASE_LEGACY_UK8_SUPPORT ++struct kbase_uk_keep_gpu_powered { ++ union uk_header header; ++ u32 enabled; ++ u32 padding; ++}; ++#endif /* BASE_LEGACY_UK8_SUPPORT */ ++ ++struct kbase_uk_profiling_controls { ++ union uk_header header; ++ u32 profiling_controls[FBDUMP_CONTROL_MAX]; ++}; ++ ++struct kbase_uk_debugfs_mem_profile_add { ++ union uk_header header; ++ u32 len; ++ u32 padding; ++ union kbase_pointer buf; ++}; ++ ++struct kbase_uk_context_id { ++ union uk_header header; ++ /* OUT */ ++ int id; ++}; ++ ++/** ++ * struct kbase_uk_tlstream_acquire - User/Kernel space data exchange structure ++ * @header: UK structure header ++ * @flags: timeline stream flags ++ * @fd: timeline stream file descriptor ++ * ++ * This structure is used when performing a call to acquire kernel side timeline ++ * stream file descriptor. ++ */ ++struct kbase_uk_tlstream_acquire { ++ union uk_header header; ++ /* IN */ ++ u32 flags; ++ /* OUT */ ++ s32 fd; ++}; ++ ++/** ++ * struct kbase_uk_tlstream_acquire_v10_4 - User/Kernel space data exchange ++ * structure ++ * @header: UK structure header ++ * @fd: timeline stream file descriptor ++ * ++ * This structure is used when performing a call to acquire kernel side timeline ++ * stream file descriptor. ++ */ ++struct kbase_uk_tlstream_acquire_v10_4 { ++ union uk_header header; ++ /* IN */ ++ /* OUT */ ++ s32 fd; ++}; ++ ++/** ++ * struct kbase_uk_tlstream_flush - User/Kernel space data exchange structure ++ * @header: UK structure header ++ * ++ * This structure is used when performing a call to flush kernel side ++ * timeline streams. ++ */ ++struct kbase_uk_tlstream_flush { ++ union uk_header header; ++ /* IN */ ++ /* OUT */ ++}; ++ ++#if MALI_UNIT_TEST ++/** ++ * struct kbase_uk_tlstream_test - User/Kernel space data exchange structure ++ * @header: UK structure header ++ * @tpw_count: number of trace point writers in each context ++ * @msg_delay: time delay between tracepoints from one writer in milliseconds ++ * @msg_count: number of trace points written by one writer ++ * @aux_msg: if non-zero aux messages will be included ++ * ++ * This structure is used when performing a call to start timeline stream test ++ * embedded in kernel. ++ */ ++struct kbase_uk_tlstream_test { ++ union uk_header header; ++ /* IN */ ++ u32 tpw_count; ++ u32 msg_delay; ++ u32 msg_count; ++ u32 aux_msg; ++ /* OUT */ ++}; ++ ++/** ++ * struct kbase_uk_tlstream_stats - User/Kernel space data exchange structure ++ * @header: UK structure header ++ * @bytes_collected: number of bytes read by user ++ * @bytes_generated: number of bytes generated by tracepoints ++ * ++ * This structure is used when performing a call to obtain timeline stream ++ * statistics. ++ */ ++struct kbase_uk_tlstream_stats { ++ union uk_header header; /**< UK structure header. */ ++ /* IN */ ++ /* OUT */ ++ u32 bytes_collected; ++ u32 bytes_generated; ++}; ++#endif /* MALI_UNIT_TEST */ ++ ++/** ++ * struct struct kbase_uk_prfcnt_value for the KBASE_FUNC_SET_PRFCNT_VALUES ioctl ++ * @header: UK structure header ++ * @data: Counter samples for the dummy model ++ * @size:............Size of the counter sample data ++ */ ++struct kbase_uk_prfcnt_values { ++ union uk_header header; ++ /* IN */ ++ u32 *data; ++ u32 size; ++}; ++ ++/** ++ * struct kbase_uk_soft_event_update - User/Kernel space data exchange structure ++ * @header: UK structure header ++ * @evt: the GPU address containing the event ++ * @new_status: the new event status, must be either BASE_JD_SOFT_EVENT_SET or ++ * BASE_JD_SOFT_EVENT_RESET ++ * @flags: reserved for future uses, must be set to 0 ++ * ++ * This structure is used to update the status of a software event. If the ++ * event's status is set to BASE_JD_SOFT_EVENT_SET, any job currently waiting ++ * on this event will complete. ++ */ ++struct kbase_uk_soft_event_update { ++ union uk_header header; ++ /* IN */ ++ u64 evt; ++ u32 new_status; ++ u32 flags; ++}; ++ ++/** ++ * struct kbase_uk_mem_jit_init - User/Kernel space data exchange structure ++ * @header: UK structure header ++ * @va_pages: Number of virtual pages required for JIT ++ * ++ * This structure is used when requesting initialization of JIT. ++ */ ++struct kbase_uk_mem_jit_init { ++ union uk_header header; ++ /* IN */ ++ u64 va_pages; ++}; ++ ++enum kbase_uk_function_id { ++ KBASE_FUNC_MEM_ALLOC = (UK_FUNC_ID + 0), ++ KBASE_FUNC_MEM_IMPORT = (UK_FUNC_ID + 1), ++ KBASE_FUNC_MEM_COMMIT = (UK_FUNC_ID + 2), ++ KBASE_FUNC_MEM_QUERY = (UK_FUNC_ID + 3), ++ KBASE_FUNC_MEM_FREE = (UK_FUNC_ID + 4), ++ KBASE_FUNC_MEM_FLAGS_CHANGE = (UK_FUNC_ID + 5), ++ KBASE_FUNC_MEM_ALIAS = (UK_FUNC_ID + 6), ++ ++#ifdef BASE_LEGACY_UK6_SUPPORT ++ KBASE_FUNC_JOB_SUBMIT_UK6 = (UK_FUNC_ID + 7), ++#endif /* BASE_LEGACY_UK6_SUPPORT */ ++ ++ KBASE_FUNC_SYNC = (UK_FUNC_ID + 8), ++ ++ KBASE_FUNC_POST_TERM = (UK_FUNC_ID + 9), ++ ++ KBASE_FUNC_HWCNT_SETUP = (UK_FUNC_ID + 10), ++ KBASE_FUNC_HWCNT_DUMP = (UK_FUNC_ID + 11), ++ KBASE_FUNC_HWCNT_CLEAR = (UK_FUNC_ID + 12), ++ ++ KBASE_FUNC_GPU_PROPS_REG_DUMP = (UK_FUNC_ID + 14), ++ ++ KBASE_FUNC_FIND_CPU_OFFSET = (UK_FUNC_ID + 15), ++ ++ KBASE_FUNC_GET_VERSION = (UK_FUNC_ID + 16), ++ KBASE_FUNC_SET_FLAGS = (UK_FUNC_ID + 18), ++ ++ KBASE_FUNC_SET_TEST_DATA = (UK_FUNC_ID + 19), ++ KBASE_FUNC_INJECT_ERROR = (UK_FUNC_ID + 20), ++ KBASE_FUNC_MODEL_CONTROL = (UK_FUNC_ID + 21), ++ ++#ifdef BASE_LEGACY_UK8_SUPPORT ++ KBASE_FUNC_KEEP_GPU_POWERED = (UK_FUNC_ID + 22), ++#endif /* BASE_LEGACY_UK8_SUPPORT */ ++ ++ KBASE_FUNC_FENCE_VALIDATE = (UK_FUNC_ID + 23), ++ KBASE_FUNC_STREAM_CREATE = (UK_FUNC_ID + 24), ++ KBASE_FUNC_GET_PROFILING_CONTROLS = (UK_FUNC_ID + 25), ++ KBASE_FUNC_SET_PROFILING_CONTROLS = (UK_FUNC_ID + 26), ++ /* to be used only for testing ++ * purposes, otherwise these controls ++ * are set through gator API */ ++ ++ KBASE_FUNC_DEBUGFS_MEM_PROFILE_ADD = (UK_FUNC_ID + 27), ++ KBASE_FUNC_JOB_SUBMIT = (UK_FUNC_ID + 28), ++ KBASE_FUNC_DISJOINT_QUERY = (UK_FUNC_ID + 29), ++ ++ KBASE_FUNC_GET_CONTEXT_ID = (UK_FUNC_ID + 31), ++ ++ KBASE_FUNC_TLSTREAM_ACQUIRE_V10_4 = (UK_FUNC_ID + 32), ++#if MALI_UNIT_TEST ++ KBASE_FUNC_TLSTREAM_TEST = (UK_FUNC_ID + 33), ++ KBASE_FUNC_TLSTREAM_STATS = (UK_FUNC_ID + 34), ++#endif /* MALI_UNIT_TEST */ ++ KBASE_FUNC_TLSTREAM_FLUSH = (UK_FUNC_ID + 35), ++ ++ KBASE_FUNC_HWCNT_READER_SETUP = (UK_FUNC_ID + 36), ++ ++#ifdef SUPPORT_MALI_NO_MALI ++ KBASE_FUNC_SET_PRFCNT_VALUES = (UK_FUNC_ID + 37), ++#endif ++ ++ KBASE_FUNC_SOFT_EVENT_UPDATE = (UK_FUNC_ID + 38), ++ ++ KBASE_FUNC_MEM_JIT_INIT = (UK_FUNC_ID + 39), ++ ++ KBASE_FUNC_TLSTREAM_ACQUIRE = (UK_FUNC_ID + 40), ++ ++ KBASE_FUNC_MAX ++}; ++ ++#endif /* _KBASE_UKU_H_ */ ++ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_utility.c b/drivers/gpu/arm/midgard/mali_kbase_utility.c +new file mode 100755 +index 000000000000..be474ff87401 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_utility.c +@@ -0,0 +1,33 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2013, 2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#include ++ ++bool kbasep_list_member_of(const struct list_head *base, struct list_head *entry) ++{ ++ struct list_head *pos = base->next; ++ ++ while (pos != base) { ++ if (pos == entry) ++ return true; ++ ++ pos = pos->next; ++ } ++ return false; ++} +diff --git a/drivers/gpu/arm/midgard/mali_kbase_utility.h b/drivers/gpu/arm/midgard/mali_kbase_utility.h +new file mode 100755 +index 000000000000..fd7252dab0de +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_utility.h +@@ -0,0 +1,37 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2013, 2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#ifndef _KBASE_UTILITY_H ++#define _KBASE_UTILITY_H ++ ++#ifndef _KBASE_H_ ++#error "Don't include this file directly, use mali_kbase.h instead" ++#endif ++ ++/** Test whether the given list entry is a member of the given list. ++ * ++ * @param base The head of the list to be tested ++ * @param entry The list entry to be tested ++ * ++ * @return true if entry is a member of base ++ * false otherwise ++ */ ++bool kbasep_list_member_of(const struct list_head *base, struct list_head *entry); ++ ++#endif /* _KBASE_UTILITY_H */ +diff --git a/drivers/gpu/arm/midgard/mali_kbase_vinstr.c b/drivers/gpu/arm/midgard/mali_kbase_vinstr.c +new file mode 100755 +index 000000000000..8395568d0efa +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_vinstr.c +@@ -0,0 +1,2070 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++/*****************************************************************************/ ++ ++/* Hwcnt reader API version */ ++#define HWCNT_READER_API 1 ++ ++/* The number of nanoseconds in a second. */ ++#define NSECS_IN_SEC 1000000000ull /* ns */ ++ ++/* The time resolution of dumping service. */ ++#define DUMPING_RESOLUTION 500000ull /* ns */ ++ ++/* The maximal supported number of dumping buffers. */ ++#define MAX_BUFFER_COUNT 32 ++ ++/* Size and number of hw counters blocks. */ ++#define NR_CNT_BLOCKS_PER_GROUP 8 ++#define NR_CNT_PER_BLOCK 64 ++#define NR_BYTES_PER_CNT 4 ++#define NR_BYTES_PER_HDR 16 ++#define PRFCNT_EN_MASK_OFFSET 0x8 ++ ++/*****************************************************************************/ ++ ++enum { ++ SHADER_HWCNT_BM, ++ TILER_HWCNT_BM, ++ MMU_L2_HWCNT_BM, ++ JM_HWCNT_BM ++}; ++ ++enum vinstr_state { ++ VINSTR_IDLE, ++ VINSTR_DUMPING, ++ VINSTR_SUSPENDING, ++ VINSTR_SUSPENDED, ++ VINSTR_RESUMING ++}; ++ ++/** ++ * struct kbase_vinstr_context - vinstr context per device ++ * @lock: protects the entire vinstr context ++ * @kbdev: pointer to kbase device ++ * @kctx: pointer to kbase context ++ * @vmap: vinstr vmap for mapping hwcnt dump buffer ++ * @gpu_va: GPU hwcnt dump buffer address ++ * @cpu_va: the CPU side mapping of the hwcnt dump buffer ++ * @dump_size: size of the dump buffer in bytes ++ * @bitmap: current set of counters monitored, not always in sync ++ * with hardware ++ * @reprogram: when true, reprogram hwcnt block with the new set of ++ * counters ++ * @state: vinstr state ++ * @state_lock: protects information about vinstr state ++ * @suspend_waitq: notification queue to trigger state re-validation ++ * @suspend_cnt: reference counter of vinstr's suspend state ++ * @suspend_work: worker to execute on entering suspended state ++ * @resume_work: worker to execute on leaving suspended state ++ * @nclients: number of attached clients, pending or otherwise ++ * @waiting_clients: head of list of clients being periodically sampled ++ * @idle_clients: head of list of clients being idle ++ * @suspended_clients: head of list of clients being suspended ++ * @thread: periodic sampling thread ++ * @waitq: notification queue of sampling thread ++ * @request_pending: request for action for sampling thread ++ */ ++struct kbase_vinstr_context { ++ struct mutex lock; ++ struct kbase_device *kbdev; ++ struct kbase_context *kctx; ++ ++ struct kbase_vmap_struct vmap; ++ u64 gpu_va; ++ void *cpu_va; ++ size_t dump_size; ++ u32 bitmap[4]; ++ bool reprogram; ++ ++ enum vinstr_state state; ++ struct spinlock state_lock; ++ wait_queue_head_t suspend_waitq; ++ unsigned int suspend_cnt; ++ struct work_struct suspend_work; ++ struct work_struct resume_work; ++ ++ u32 nclients; ++ struct list_head waiting_clients; ++ struct list_head idle_clients; ++ struct list_head suspended_clients; ++ ++ struct task_struct *thread; ++ wait_queue_head_t waitq; ++ atomic_t request_pending; ++}; ++ ++/** ++ * struct kbase_vinstr_client - a vinstr client attached to a vinstr context ++ * @vinstr_ctx: vinstr context client is attached to ++ * @list: node used to attach this client to list in vinstr context ++ * @buffer_count: number of buffers this client is using ++ * @event_mask: events this client reacts to ++ * @dump_size: size of one dump buffer in bytes ++ * @bitmap: bitmap request for JM, TILER, SHADER and MMU counters ++ * @legacy_buffer: userspace hwcnt dump buffer (legacy interface) ++ * @kernel_buffer: kernel hwcnt dump buffer (kernel client interface) ++ * @accum_buffer: temporary accumulation buffer for preserving counters ++ * @dump_time: next time this clients shall request hwcnt dump ++ * @dump_interval: interval between periodic hwcnt dumps ++ * @dump_buffers: kernel hwcnt dump buffers allocated by this client ++ * @dump_buffers_meta: metadata of dump buffers ++ * @meta_idx: index of metadata being accessed by userspace ++ * @read_idx: index of buffer read by userspace ++ * @write_idx: index of buffer being written by dumping service ++ * @waitq: client's notification queue ++ * @pending: when true, client has attached but hwcnt not yet updated ++ */ ++struct kbase_vinstr_client { ++ struct kbase_vinstr_context *vinstr_ctx; ++ struct list_head list; ++ unsigned int buffer_count; ++ u32 event_mask; ++ size_t dump_size; ++ u32 bitmap[4]; ++ void __user *legacy_buffer; ++ void *kernel_buffer; ++ void *accum_buffer; ++ u64 dump_time; ++ u32 dump_interval; ++ char *dump_buffers; ++ struct kbase_hwcnt_reader_metadata *dump_buffers_meta; ++ atomic_t meta_idx; ++ atomic_t read_idx; ++ atomic_t write_idx; ++ wait_queue_head_t waitq; ++ bool pending; ++}; ++ ++/** ++ * struct kbasep_vinstr_wake_up_timer - vinstr service thread wake up timer ++ * @hrtimer: high resolution timer ++ * @vinstr_ctx: vinstr context ++ */ ++struct kbasep_vinstr_wake_up_timer { ++ struct hrtimer hrtimer; ++ struct kbase_vinstr_context *vinstr_ctx; ++}; ++ ++/*****************************************************************************/ ++ ++static int kbasep_vinstr_service_task(void *data); ++ ++static unsigned int kbasep_vinstr_hwcnt_reader_poll( ++ struct file *filp, ++ poll_table *wait); ++static long kbasep_vinstr_hwcnt_reader_ioctl( ++ struct file *filp, ++ unsigned int cmd, ++ unsigned long arg); ++static int kbasep_vinstr_hwcnt_reader_mmap( ++ struct file *filp, ++ struct vm_area_struct *vma); ++static int kbasep_vinstr_hwcnt_reader_release( ++ struct inode *inode, ++ struct file *filp); ++ ++/* The timeline stream file operations structure. */ ++static const struct file_operations vinstr_client_fops = { ++ .poll = kbasep_vinstr_hwcnt_reader_poll, ++ .unlocked_ioctl = kbasep_vinstr_hwcnt_reader_ioctl, ++ .compat_ioctl = kbasep_vinstr_hwcnt_reader_ioctl, ++ .mmap = kbasep_vinstr_hwcnt_reader_mmap, ++ .release = kbasep_vinstr_hwcnt_reader_release, ++}; ++ ++/*****************************************************************************/ ++ ++static int enable_hwcnt(struct kbase_vinstr_context *vinstr_ctx) ++{ ++ struct kbase_context *kctx = vinstr_ctx->kctx; ++ struct kbase_device *kbdev = kctx->kbdev; ++ struct kbase_uk_hwcnt_setup setup; ++ int err; ++ ++ setup.dump_buffer = vinstr_ctx->gpu_va; ++ setup.jm_bm = vinstr_ctx->bitmap[JM_HWCNT_BM]; ++ setup.tiler_bm = vinstr_ctx->bitmap[TILER_HWCNT_BM]; ++ setup.shader_bm = vinstr_ctx->bitmap[SHADER_HWCNT_BM]; ++ setup.mmu_l2_bm = vinstr_ctx->bitmap[MMU_L2_HWCNT_BM]; ++ ++ /* Mark the context as active so the GPU is kept turned on */ ++ /* A suspend won't happen here, because we're in a syscall from a ++ * userspace thread. */ ++ kbase_pm_context_active(kbdev); ++ ++ /* Schedule the context in */ ++ kbasep_js_schedule_privileged_ctx(kbdev, kctx); ++ err = kbase_instr_hwcnt_enable_internal(kbdev, kctx, &setup); ++ if (err) { ++ /* Release the context. This had its own Power Manager Active ++ * reference */ ++ kbasep_js_release_privileged_ctx(kbdev, kctx); ++ ++ /* Also release our Power Manager Active reference */ ++ kbase_pm_context_idle(kbdev); ++ } ++ ++ return err; ++} ++ ++static void disable_hwcnt(struct kbase_vinstr_context *vinstr_ctx) ++{ ++ struct kbase_context *kctx = vinstr_ctx->kctx; ++ struct kbase_device *kbdev = kctx->kbdev; ++ int err; ++ ++ err = kbase_instr_hwcnt_disable_internal(kctx); ++ if (err) { ++ dev_warn(kbdev->dev, "Failed to disable HW counters (ctx:%p)", ++ kctx); ++ return; ++ } ++ ++ /* Release the context. This had its own Power Manager Active reference. */ ++ kbasep_js_release_privileged_ctx(kbdev, kctx); ++ ++ /* Also release our Power Manager Active reference. */ ++ kbase_pm_context_idle(kbdev); ++ ++ dev_dbg(kbdev->dev, "HW counters dumping disabled for context %p", kctx); ++} ++ ++static int reprogram_hwcnt(struct kbase_vinstr_context *vinstr_ctx) ++{ ++ disable_hwcnt(vinstr_ctx); ++ return enable_hwcnt(vinstr_ctx); ++} ++ ++static void hwcnt_bitmap_set(u32 dst[4], u32 src[4]) ++{ ++ dst[JM_HWCNT_BM] = src[JM_HWCNT_BM]; ++ dst[TILER_HWCNT_BM] = src[TILER_HWCNT_BM]; ++ dst[SHADER_HWCNT_BM] = src[SHADER_HWCNT_BM]; ++ dst[MMU_L2_HWCNT_BM] = src[MMU_L2_HWCNT_BM]; ++} ++ ++static void hwcnt_bitmap_union(u32 dst[4], u32 src[4]) ++{ ++ dst[JM_HWCNT_BM] |= src[JM_HWCNT_BM]; ++ dst[TILER_HWCNT_BM] |= src[TILER_HWCNT_BM]; ++ dst[SHADER_HWCNT_BM] |= src[SHADER_HWCNT_BM]; ++ dst[MMU_L2_HWCNT_BM] |= src[MMU_L2_HWCNT_BM]; ++} ++ ++size_t kbase_vinstr_dump_size(struct kbase_device *kbdev) ++{ ++ size_t dump_size; ++ ++#ifndef CONFIG_MALI_NO_MALI ++ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_V4)) { ++ u32 nr_cg; ++ ++ nr_cg = kbdev->gpu_props.num_core_groups; ++ dump_size = nr_cg * NR_CNT_BLOCKS_PER_GROUP * ++ NR_CNT_PER_BLOCK * ++ NR_BYTES_PER_CNT; ++ } else ++#endif /* CONFIG_MALI_NO_MALI */ ++ { ++ /* assume v5 for now */ ++ base_gpu_props *props = &kbdev->gpu_props.props; ++ u32 nr_l2 = props->l2_props.num_l2_slices; ++ u64 core_mask = props->coherency_info.group[0].core_mask; ++ u32 nr_blocks = fls64(core_mask); ++ ++ /* JM and tiler counter blocks are always present */ ++ dump_size = (2 + nr_l2 + nr_blocks) * ++ NR_CNT_PER_BLOCK * ++ NR_BYTES_PER_CNT; ++ } ++ return dump_size; ++} ++KBASE_EXPORT_TEST_API(kbase_vinstr_dump_size); ++ ++static size_t kbasep_vinstr_dump_size_ctx( ++ struct kbase_vinstr_context *vinstr_ctx) ++{ ++ return kbase_vinstr_dump_size(vinstr_ctx->kctx->kbdev); ++} ++ ++static int kbasep_vinstr_map_kernel_dump_buffer( ++ struct kbase_vinstr_context *vinstr_ctx) ++{ ++ struct kbase_va_region *reg; ++ struct kbase_context *kctx = vinstr_ctx->kctx; ++ u64 flags, nr_pages; ++ ++ flags = BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_GPU_WR; ++ vinstr_ctx->dump_size = kbasep_vinstr_dump_size_ctx(vinstr_ctx); ++ nr_pages = PFN_UP(vinstr_ctx->dump_size); ++ ++ reg = kbase_mem_alloc(kctx, nr_pages, nr_pages, 0, &flags, ++ &vinstr_ctx->gpu_va); ++ if (!reg) ++ return -ENOMEM; ++ ++ vinstr_ctx->cpu_va = kbase_vmap( ++ kctx, ++ vinstr_ctx->gpu_va, ++ vinstr_ctx->dump_size, ++ &vinstr_ctx->vmap); ++ if (!vinstr_ctx->cpu_va) { ++ kbase_mem_free(kctx, vinstr_ctx->gpu_va); ++ return -ENOMEM; ++ } ++ ++ return 0; ++} ++ ++static void kbasep_vinstr_unmap_kernel_dump_buffer( ++ struct kbase_vinstr_context *vinstr_ctx) ++{ ++ struct kbase_context *kctx = vinstr_ctx->kctx; ++ ++ kbase_vunmap(kctx, &vinstr_ctx->vmap); ++ kbase_mem_free(kctx, vinstr_ctx->gpu_va); ++} ++ ++/** ++ * kbasep_vinstr_create_kctx - create kernel context for vinstr ++ * @vinstr_ctx: vinstr context ++ * Return: zero on success ++ */ ++static int kbasep_vinstr_create_kctx(struct kbase_vinstr_context *vinstr_ctx) ++{ ++ struct kbase_device *kbdev = vinstr_ctx->kbdev; ++ struct kbasep_kctx_list_element *element; ++ unsigned long flags; ++ bool enable_backend = false; ++ int err; ++ ++ vinstr_ctx->kctx = kbase_create_context(vinstr_ctx->kbdev, true); ++ if (!vinstr_ctx->kctx) ++ return -ENOMEM; ++ ++ /* Map the master kernel dump buffer. The HW dumps the counters ++ * into this memory region. */ ++ err = kbasep_vinstr_map_kernel_dump_buffer(vinstr_ctx); ++ if (err) { ++ kbase_destroy_context(vinstr_ctx->kctx); ++ vinstr_ctx->kctx = NULL; ++ return err; ++ } ++ ++ /* Add kernel context to list of contexts associated with device. */ ++ element = kzalloc(sizeof(*element), GFP_KERNEL); ++ if (element) { ++ element->kctx = vinstr_ctx->kctx; ++ mutex_lock(&kbdev->kctx_list_lock); ++ list_add(&element->link, &kbdev->kctx_list); ++ ++ /* Inform timeline client about new context. ++ * Do this while holding the lock to avoid tracepoint ++ * being created in both body and summary stream. */ ++ KBASE_TLSTREAM_TL_NEW_CTX( ++ vinstr_ctx->kctx, ++ (u32)(vinstr_ctx->kctx->id), ++ (u32)(vinstr_ctx->kctx->tgid)); ++ ++ mutex_unlock(&kbdev->kctx_list_lock); ++ } else { ++ /* Don't treat this as a fail - just warn about it. */ ++ dev_warn(kbdev->dev, ++ "couldn't add kctx to kctx_list\n"); ++ } ++ ++ /* Don't enable hardware counters if vinstr is suspended. ++ * Note that vinstr resume code is run under vinstr context lock, ++ * lower layer will be enabled as needed on resume. */ ++ spin_lock_irqsave(&vinstr_ctx->state_lock, flags); ++ if (VINSTR_IDLE == vinstr_ctx->state) ++ enable_backend = true; ++ spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); ++ if (enable_backend) ++ err = enable_hwcnt(vinstr_ctx); ++ ++ if (err) { ++ kbasep_vinstr_unmap_kernel_dump_buffer(vinstr_ctx); ++ kbase_destroy_context(vinstr_ctx->kctx); ++ if (element) { ++ mutex_lock(&kbdev->kctx_list_lock); ++ list_del(&element->link); ++ kfree(element); ++ mutex_unlock(&kbdev->kctx_list_lock); ++ } ++ KBASE_TLSTREAM_TL_DEL_CTX(vinstr_ctx->kctx); ++ vinstr_ctx->kctx = NULL; ++ return err; ++ } ++ ++ vinstr_ctx->thread = kthread_run( ++ kbasep_vinstr_service_task, ++ vinstr_ctx, ++ "mali_vinstr_service"); ++ if (!vinstr_ctx->thread) { ++ disable_hwcnt(vinstr_ctx); ++ kbasep_vinstr_unmap_kernel_dump_buffer(vinstr_ctx); ++ kbase_destroy_context(vinstr_ctx->kctx); ++ if (element) { ++ mutex_lock(&kbdev->kctx_list_lock); ++ list_del(&element->link); ++ kfree(element); ++ mutex_unlock(&kbdev->kctx_list_lock); ++ } ++ KBASE_TLSTREAM_TL_DEL_CTX(vinstr_ctx->kctx); ++ vinstr_ctx->kctx = NULL; ++ return -EFAULT; ++ } ++ ++ return 0; ++} ++ ++/** ++ * kbasep_vinstr_destroy_kctx - destroy vinstr's kernel context ++ * @vinstr_ctx: vinstr context ++ */ ++static void kbasep_vinstr_destroy_kctx(struct kbase_vinstr_context *vinstr_ctx) ++{ ++ struct kbase_device *kbdev = vinstr_ctx->kbdev; ++ struct kbasep_kctx_list_element *element; ++ struct kbasep_kctx_list_element *tmp; ++ bool found = false; ++ ++ /* Release hw counters dumping resources. */ ++ vinstr_ctx->thread = NULL; ++ disable_hwcnt(vinstr_ctx); ++ kbasep_vinstr_unmap_kernel_dump_buffer(vinstr_ctx); ++ kbase_destroy_context(vinstr_ctx->kctx); ++ ++ /* Remove kernel context from the device's contexts list. */ ++ mutex_lock(&kbdev->kctx_list_lock); ++ list_for_each_entry_safe(element, tmp, &kbdev->kctx_list, link) { ++ if (element->kctx == vinstr_ctx->kctx) { ++ list_del(&element->link); ++ kfree(element); ++ found = true; ++ } ++ } ++ mutex_unlock(&kbdev->kctx_list_lock); ++ ++ if (!found) ++ dev_warn(kbdev->dev, "kctx not in kctx_list\n"); ++ ++ /* Inform timeline client about context destruction. */ ++ KBASE_TLSTREAM_TL_DEL_CTX(vinstr_ctx->kctx); ++ ++ vinstr_ctx->kctx = NULL; ++} ++ ++/** ++ * kbasep_vinstr_attach_client - Attach a client to the vinstr core ++ * @vinstr_ctx: vinstr context ++ * @buffer_count: requested number of dump buffers ++ * @bitmap: bitmaps describing which counters should be enabled ++ * @argp: pointer where notification descriptor shall be stored ++ * @kernel_buffer: pointer to kernel side buffer ++ * ++ * Return: vinstr opaque client handle or NULL on failure ++ */ ++static struct kbase_vinstr_client *kbasep_vinstr_attach_client( ++ struct kbase_vinstr_context *vinstr_ctx, u32 buffer_count, ++ u32 bitmap[4], void *argp, void *kernel_buffer) ++{ ++ struct task_struct *thread = NULL; ++ struct kbase_vinstr_client *cli; ++ ++ KBASE_DEBUG_ASSERT(vinstr_ctx); ++ ++ if (buffer_count > MAX_BUFFER_COUNT ++ || (buffer_count & (buffer_count - 1))) ++ return NULL; ++ ++ cli = kzalloc(sizeof(*cli), GFP_KERNEL); ++ if (!cli) ++ return NULL; ++ ++ cli->vinstr_ctx = vinstr_ctx; ++ cli->buffer_count = buffer_count; ++ cli->event_mask = ++ (1 << BASE_HWCNT_READER_EVENT_MANUAL) | ++ (1 << BASE_HWCNT_READER_EVENT_PERIODIC); ++ cli->pending = true; ++ ++ hwcnt_bitmap_set(cli->bitmap, bitmap); ++ ++ mutex_lock(&vinstr_ctx->lock); ++ ++ hwcnt_bitmap_union(vinstr_ctx->bitmap, cli->bitmap); ++ vinstr_ctx->reprogram = true; ++ ++ /* If this is the first client, create the vinstr kbase ++ * context. This context is permanently resident until the ++ * last client exits. */ ++ if (!vinstr_ctx->nclients) { ++ hwcnt_bitmap_set(vinstr_ctx->bitmap, cli->bitmap); ++ if (kbasep_vinstr_create_kctx(vinstr_ctx) < 0) ++ goto error; ++ ++ vinstr_ctx->reprogram = false; ++ cli->pending = false; ++ } ++ ++ /* The GPU resets the counter block every time there is a request ++ * to dump it. We need a per client kernel buffer for accumulating ++ * the counters. */ ++ cli->dump_size = kbasep_vinstr_dump_size_ctx(vinstr_ctx); ++ cli->accum_buffer = kzalloc(cli->dump_size, GFP_KERNEL); ++ if (!cli->accum_buffer) ++ goto error; ++ ++ /* Prepare buffers. */ ++ if (cli->buffer_count) { ++ int *fd = (int *)argp; ++ size_t tmp; ++ ++ /* Allocate area for buffers metadata storage. */ ++ tmp = sizeof(struct kbase_hwcnt_reader_metadata) * ++ cli->buffer_count; ++ cli->dump_buffers_meta = kmalloc(tmp, GFP_KERNEL); ++ if (!cli->dump_buffers_meta) ++ goto error; ++ ++ /* Allocate required number of dumping buffers. */ ++ cli->dump_buffers = (char *)__get_free_pages( ++ GFP_KERNEL | __GFP_ZERO, ++ get_order(cli->dump_size * cli->buffer_count)); ++ if (!cli->dump_buffers) ++ goto error; ++ ++ /* Create descriptor for user-kernel data exchange. */ ++ *fd = anon_inode_getfd( ++ "[mali_vinstr_desc]", ++ &vinstr_client_fops, ++ cli, ++ O_RDONLY | O_CLOEXEC); ++ if (0 > *fd) ++ goto error; ++ } else if (kernel_buffer) { ++ cli->kernel_buffer = kernel_buffer; ++ } else { ++ cli->legacy_buffer = (void __user *)argp; ++ } ++ ++ atomic_set(&cli->read_idx, 0); ++ atomic_set(&cli->meta_idx, 0); ++ atomic_set(&cli->write_idx, 0); ++ init_waitqueue_head(&cli->waitq); ++ ++ vinstr_ctx->nclients++; ++ list_add(&cli->list, &vinstr_ctx->idle_clients); ++ ++ mutex_unlock(&vinstr_ctx->lock); ++ ++ return cli; ++ ++error: ++ kfree(cli->dump_buffers_meta); ++ if (cli->dump_buffers) ++ free_pages( ++ (unsigned long)cli->dump_buffers, ++ get_order(cli->dump_size * cli->buffer_count)); ++ kfree(cli->accum_buffer); ++ if (!vinstr_ctx->nclients && vinstr_ctx->kctx) { ++ thread = vinstr_ctx->thread; ++ kbasep_vinstr_destroy_kctx(vinstr_ctx); ++ } ++ kfree(cli); ++ ++ mutex_unlock(&vinstr_ctx->lock); ++ ++ /* Thread must be stopped after lock is released. */ ++ if (thread) ++ kthread_stop(thread); ++ ++ return NULL; ++} ++ ++void kbase_vinstr_detach_client(struct kbase_vinstr_client *cli) ++{ ++ struct kbase_vinstr_context *vinstr_ctx; ++ struct kbase_vinstr_client *iter, *tmp; ++ struct task_struct *thread = NULL; ++ u32 zerobitmap[4] = { 0 }; ++ int cli_found = 0; ++ ++ KBASE_DEBUG_ASSERT(cli); ++ vinstr_ctx = cli->vinstr_ctx; ++ KBASE_DEBUG_ASSERT(vinstr_ctx); ++ ++ mutex_lock(&vinstr_ctx->lock); ++ ++ list_for_each_entry_safe(iter, tmp, &vinstr_ctx->idle_clients, list) { ++ if (iter == cli) { ++ vinstr_ctx->reprogram = true; ++ cli_found = 1; ++ list_del(&iter->list); ++ break; ++ } ++ } ++ if (!cli_found) { ++ list_for_each_entry_safe( ++ iter, tmp, &vinstr_ctx->waiting_clients, list) { ++ if (iter == cli) { ++ vinstr_ctx->reprogram = true; ++ cli_found = 1; ++ list_del(&iter->list); ++ break; ++ } ++ } ++ } ++ KBASE_DEBUG_ASSERT(cli_found); ++ ++ kfree(cli->dump_buffers_meta); ++ free_pages( ++ (unsigned long)cli->dump_buffers, ++ get_order(cli->dump_size * cli->buffer_count)); ++ kfree(cli->accum_buffer); ++ kfree(cli); ++ ++ vinstr_ctx->nclients--; ++ if (!vinstr_ctx->nclients) { ++ thread = vinstr_ctx->thread; ++ kbasep_vinstr_destroy_kctx(vinstr_ctx); ++ } ++ ++ /* Rebuild context bitmap now that the client has detached */ ++ hwcnt_bitmap_set(vinstr_ctx->bitmap, zerobitmap); ++ list_for_each_entry(iter, &vinstr_ctx->idle_clients, list) ++ hwcnt_bitmap_union(vinstr_ctx->bitmap, iter->bitmap); ++ list_for_each_entry(iter, &vinstr_ctx->waiting_clients, list) ++ hwcnt_bitmap_union(vinstr_ctx->bitmap, iter->bitmap); ++ ++ mutex_unlock(&vinstr_ctx->lock); ++ ++ /* Thread must be stopped after lock is released. */ ++ if (thread) ++ kthread_stop(thread); ++} ++KBASE_EXPORT_TEST_API(kbase_vinstr_detach_client); ++ ++/* Accumulate counters in the dump buffer */ ++static void accum_dump_buffer(void *dst, void *src, size_t dump_size) ++{ ++ size_t block_size = NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT; ++ u32 *d = dst; ++ u32 *s = src; ++ size_t i, j; ++ ++ for (i = 0; i < dump_size; i += block_size) { ++ /* skip over the header block */ ++ d += NR_BYTES_PER_HDR / sizeof(u32); ++ s += NR_BYTES_PER_HDR / sizeof(u32); ++ for (j = 0; j < (block_size - NR_BYTES_PER_HDR) / sizeof(u32); j++) { ++ /* saturate result if addition would result in wraparound */ ++ if (U32_MAX - *d < *s) ++ *d = U32_MAX; ++ else ++ *d += *s; ++ d++; ++ s++; ++ } ++ } ++} ++ ++/* This is the Midgard v4 patch function. It copies the headers for each ++ * of the defined blocks from the master kernel buffer and then patches up ++ * the performance counter enable mask for each of the blocks to exclude ++ * counters that were not requested by the client. */ ++static void patch_dump_buffer_hdr_v4( ++ struct kbase_vinstr_context *vinstr_ctx, ++ struct kbase_vinstr_client *cli) ++{ ++ u32 *mask; ++ u8 *dst = cli->accum_buffer; ++ u8 *src = vinstr_ctx->cpu_va; ++ u32 nr_cg = vinstr_ctx->kctx->kbdev->gpu_props.num_core_groups; ++ size_t i, group_size, group; ++ enum { ++ SC0_BASE = 0 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT, ++ SC1_BASE = 1 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT, ++ SC2_BASE = 2 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT, ++ SC3_BASE = 3 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT, ++ TILER_BASE = 4 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT, ++ MMU_L2_BASE = 5 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT, ++ JM_BASE = 7 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT ++ }; ++ ++ group_size = NR_CNT_BLOCKS_PER_GROUP * ++ NR_CNT_PER_BLOCK * ++ NR_BYTES_PER_CNT; ++ for (i = 0; i < nr_cg; i++) { ++ group = i * group_size; ++ /* copy shader core headers */ ++ memcpy(&dst[group + SC0_BASE], &src[group + SC0_BASE], ++ NR_BYTES_PER_HDR); ++ memcpy(&dst[group + SC1_BASE], &src[group + SC1_BASE], ++ NR_BYTES_PER_HDR); ++ memcpy(&dst[group + SC2_BASE], &src[group + SC2_BASE], ++ NR_BYTES_PER_HDR); ++ memcpy(&dst[group + SC3_BASE], &src[group + SC3_BASE], ++ NR_BYTES_PER_HDR); ++ ++ /* copy tiler header */ ++ memcpy(&dst[group + TILER_BASE], &src[group + TILER_BASE], ++ NR_BYTES_PER_HDR); ++ ++ /* copy mmu header */ ++ memcpy(&dst[group + MMU_L2_BASE], &src[group + MMU_L2_BASE], ++ NR_BYTES_PER_HDR); ++ ++ /* copy job manager header */ ++ memcpy(&dst[group + JM_BASE], &src[group + JM_BASE], ++ NR_BYTES_PER_HDR); ++ ++ /* patch the shader core enable mask */ ++ mask = (u32 *)&dst[group + SC0_BASE + PRFCNT_EN_MASK_OFFSET]; ++ *mask &= cli->bitmap[SHADER_HWCNT_BM]; ++ mask = (u32 *)&dst[group + SC1_BASE + PRFCNT_EN_MASK_OFFSET]; ++ *mask &= cli->bitmap[SHADER_HWCNT_BM]; ++ mask = (u32 *)&dst[group + SC2_BASE + PRFCNT_EN_MASK_OFFSET]; ++ *mask &= cli->bitmap[SHADER_HWCNT_BM]; ++ mask = (u32 *)&dst[group + SC3_BASE + PRFCNT_EN_MASK_OFFSET]; ++ *mask &= cli->bitmap[SHADER_HWCNT_BM]; ++ ++ /* patch the tiler core enable mask */ ++ mask = (u32 *)&dst[group + TILER_BASE + PRFCNT_EN_MASK_OFFSET]; ++ *mask &= cli->bitmap[TILER_HWCNT_BM]; ++ ++ /* patch the mmu core enable mask */ ++ mask = (u32 *)&dst[group + MMU_L2_BASE + PRFCNT_EN_MASK_OFFSET]; ++ *mask &= cli->bitmap[MMU_L2_HWCNT_BM]; ++ ++ /* patch the job manager enable mask */ ++ mask = (u32 *)&dst[group + JM_BASE + PRFCNT_EN_MASK_OFFSET]; ++ *mask &= cli->bitmap[JM_HWCNT_BM]; ++ } ++} ++ ++/* This is the Midgard v5 patch function. It copies the headers for each ++ * of the defined blocks from the master kernel buffer and then patches up ++ * the performance counter enable mask for each of the blocks to exclude ++ * counters that were not requested by the client. */ ++static void patch_dump_buffer_hdr_v5( ++ struct kbase_vinstr_context *vinstr_ctx, ++ struct kbase_vinstr_client *cli) ++{ ++ struct kbase_device *kbdev = vinstr_ctx->kctx->kbdev; ++ u32 i, nr_l2; ++ u64 core_mask; ++ u32 *mask; ++ u8 *dst = cli->accum_buffer; ++ u8 *src = vinstr_ctx->cpu_va; ++ size_t block_size = NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT; ++ ++ /* copy and patch job manager header */ ++ memcpy(dst, src, NR_BYTES_PER_HDR); ++ mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET]; ++ *mask &= cli->bitmap[JM_HWCNT_BM]; ++ dst += block_size; ++ src += block_size; ++ ++ /* copy and patch tiler header */ ++ memcpy(dst, src, NR_BYTES_PER_HDR); ++ mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET]; ++ *mask &= cli->bitmap[TILER_HWCNT_BM]; ++ dst += block_size; ++ src += block_size; ++ ++ /* copy and patch MMU/L2C headers */ ++ nr_l2 = kbdev->gpu_props.props.l2_props.num_l2_slices; ++ for (i = 0; i < nr_l2; i++) { ++ memcpy(dst, src, NR_BYTES_PER_HDR); ++ mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET]; ++ *mask &= cli->bitmap[MMU_L2_HWCNT_BM]; ++ dst += block_size; ++ src += block_size; ++ } ++ ++ /* copy and patch shader core headers */ ++ core_mask = kbdev->gpu_props.props.coherency_info.group[0].core_mask; ++ while (0ull != core_mask) { ++ memcpy(dst, src, NR_BYTES_PER_HDR); ++ if (0ull != (core_mask & 1ull)) { ++ /* if block is not reserved update header */ ++ mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET]; ++ *mask &= cli->bitmap[SHADER_HWCNT_BM]; ++ } ++ dst += block_size; ++ src += block_size; ++ ++ core_mask >>= 1; ++ } ++} ++ ++/** ++ * accum_clients - accumulate dumped hw counters for all known clients ++ * @vinstr_ctx: vinstr context ++ */ ++static void accum_clients(struct kbase_vinstr_context *vinstr_ctx) ++{ ++ struct kbase_vinstr_client *iter; ++ int v4 = 0; ++ ++#ifndef CONFIG_MALI_NO_MALI ++ v4 = kbase_hw_has_feature(vinstr_ctx->kbdev, BASE_HW_FEATURE_V4); ++#endif ++ ++ list_for_each_entry(iter, &vinstr_ctx->idle_clients, list) { ++ /* Don't bother accumulating clients whose hwcnt requests ++ * have not yet been honoured. */ ++ if (iter->pending) ++ continue; ++ if (v4) ++ patch_dump_buffer_hdr_v4(vinstr_ctx, iter); ++ else ++ patch_dump_buffer_hdr_v5(vinstr_ctx, iter); ++ accum_dump_buffer( ++ iter->accum_buffer, ++ vinstr_ctx->cpu_va, ++ iter->dump_size); ++ } ++ list_for_each_entry(iter, &vinstr_ctx->waiting_clients, list) { ++ /* Don't bother accumulating clients whose hwcnt requests ++ * have not yet been honoured. */ ++ if (iter->pending) ++ continue; ++ if (v4) ++ patch_dump_buffer_hdr_v4(vinstr_ctx, iter); ++ else ++ patch_dump_buffer_hdr_v5(vinstr_ctx, iter); ++ accum_dump_buffer( ++ iter->accum_buffer, ++ vinstr_ctx->cpu_va, ++ iter->dump_size); ++ } ++} ++ ++/*****************************************************************************/ ++ ++/** ++ * kbasep_vinstr_get_timestamp - return timestamp ++ * ++ * Function returns timestamp value based on raw monotonic timer. Value will ++ * wrap around zero in case of overflow. ++ * ++ * Return: timestamp value ++ */ ++static u64 kbasep_vinstr_get_timestamp(void) ++{ ++ struct timespec64 ts; ++ ++ ktime_get_raw_ts64(&ts); ++ return (u64)ts.tv_sec * NSECS_IN_SEC + ts.tv_nsec; ++} ++ ++/** ++ * kbasep_vinstr_add_dump_request - register client's dumping request ++ * @cli: requesting client ++ * @waiting_clients: list of pending dumping requests ++ */ ++static void kbasep_vinstr_add_dump_request( ++ struct kbase_vinstr_client *cli, ++ struct list_head *waiting_clients) ++{ ++ struct kbase_vinstr_client *tmp; ++ ++ if (list_empty(waiting_clients)) { ++ list_add(&cli->list, waiting_clients); ++ return; ++ } ++ list_for_each_entry(tmp, waiting_clients, list) { ++ if (tmp->dump_time > cli->dump_time) { ++ list_add_tail(&cli->list, &tmp->list); ++ return; ++ } ++ } ++ list_add_tail(&cli->list, waiting_clients); ++} ++ ++/** ++ * kbasep_vinstr_collect_and_accumulate - collect hw counters via low level ++ * dump and accumulate them for known ++ * clients ++ * @vinstr_ctx: vinstr context ++ * @timestamp: pointer where collection timestamp will be recorded ++ * ++ * Return: zero on success ++ */ ++static int kbasep_vinstr_collect_and_accumulate( ++ struct kbase_vinstr_context *vinstr_ctx, u64 *timestamp) ++{ ++ unsigned long flags; ++ int rcode; ++ ++#ifdef CONFIG_MALI_NO_MALI ++ /* The dummy model needs the CPU mapping. */ ++ gpu_model_set_dummy_prfcnt_base_cpu(vinstr_ctx->cpu_va); ++#endif ++ ++ spin_lock_irqsave(&vinstr_ctx->state_lock, flags); ++ if (VINSTR_IDLE != vinstr_ctx->state) { ++ spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); ++ return -EAGAIN; ++ } else { ++ vinstr_ctx->state = VINSTR_DUMPING; ++ } ++ spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); ++ ++ /* Request HW counters dump. ++ * Disable preemption to make dump timestamp more accurate. */ ++ preempt_disable(); ++ *timestamp = kbasep_vinstr_get_timestamp(); ++ rcode = kbase_instr_hwcnt_request_dump(vinstr_ctx->kctx); ++ preempt_enable(); ++ ++ if (!rcode) ++ rcode = kbase_instr_hwcnt_wait_for_dump(vinstr_ctx->kctx); ++ WARN_ON(rcode); ++ ++ spin_lock_irqsave(&vinstr_ctx->state_lock, flags); ++ switch (vinstr_ctx->state) ++ { ++ case VINSTR_SUSPENDING: ++ schedule_work(&vinstr_ctx->suspend_work); ++ break; ++ case VINSTR_DUMPING: ++ vinstr_ctx->state = VINSTR_IDLE; ++ wake_up_all(&vinstr_ctx->suspend_waitq); ++ break; ++ default: ++ break; ++ } ++ spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); ++ ++ /* Accumulate values of collected counters. */ ++ if (!rcode) ++ accum_clients(vinstr_ctx); ++ ++ return rcode; ++} ++ ++/** ++ * kbasep_vinstr_fill_dump_buffer - copy accumulated counters to empty kernel ++ * buffer ++ * @cli: requesting client ++ * @timestamp: timestamp when counters were collected ++ * @event_id: id of event that caused triggered counters collection ++ * ++ * Return: zero on success ++ */ ++static int kbasep_vinstr_fill_dump_buffer( ++ struct kbase_vinstr_client *cli, u64 timestamp, ++ enum base_hwcnt_reader_event event_id) ++{ ++ unsigned int write_idx = atomic_read(&cli->write_idx); ++ unsigned int read_idx = atomic_read(&cli->read_idx); ++ ++ struct kbase_hwcnt_reader_metadata *meta; ++ void *buffer; ++ ++ /* Check if there is a place to copy HWC block into. */ ++ if (write_idx - read_idx == cli->buffer_count) ++ return -1; ++ write_idx %= cli->buffer_count; ++ ++ /* Fill in dump buffer and its metadata. */ ++ buffer = &cli->dump_buffers[write_idx * cli->dump_size]; ++ meta = &cli->dump_buffers_meta[write_idx]; ++ meta->timestamp = timestamp; ++ meta->event_id = event_id; ++ meta->buffer_idx = write_idx; ++ memcpy(buffer, cli->accum_buffer, cli->dump_size); ++ return 0; ++} ++ ++/** ++ * kbasep_vinstr_fill_dump_buffer_legacy - copy accumulated counters to buffer ++ * allocated in userspace ++ * @cli: requesting client ++ * ++ * Return: zero on success ++ * ++ * This is part of legacy ioctl interface. ++ */ ++static int kbasep_vinstr_fill_dump_buffer_legacy( ++ struct kbase_vinstr_client *cli) ++{ ++ void __user *buffer = cli->legacy_buffer; ++ int rcode; ++ ++ /* Copy data to user buffer. */ ++ rcode = copy_to_user(buffer, cli->accum_buffer, cli->dump_size); ++ if (rcode) ++ pr_warn("error while copying buffer to user\n"); ++ return rcode; ++} ++ ++/** ++ * kbasep_vinstr_fill_dump_buffer_kernel - copy accumulated counters to buffer ++ * allocated in kernel space ++ * @cli: requesting client ++ * ++ * Return: zero on success ++ * ++ * This is part of the kernel client interface. ++ */ ++static int kbasep_vinstr_fill_dump_buffer_kernel( ++ struct kbase_vinstr_client *cli) ++{ ++ memcpy(cli->kernel_buffer, cli->accum_buffer, cli->dump_size); ++ ++ return 0; ++} ++ ++/** ++ * kbasep_vinstr_reprogram - reprogram hwcnt set collected by inst ++ * @vinstr_ctx: vinstr context ++ */ ++static void kbasep_vinstr_reprogram( ++ struct kbase_vinstr_context *vinstr_ctx) ++{ ++ unsigned long flags; ++ bool suspended = false; ++ ++ /* Don't enable hardware counters if vinstr is suspended. */ ++ spin_lock_irqsave(&vinstr_ctx->state_lock, flags); ++ if (VINSTR_IDLE != vinstr_ctx->state) ++ suspended = true; ++ spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); ++ if (suspended) ++ return; ++ ++ /* Change to suspended state is done while holding vinstr context ++ * lock. Below code will then no re-enable the instrumentation. */ ++ ++ if (vinstr_ctx->reprogram) { ++ struct kbase_vinstr_client *iter; ++ ++ if (!reprogram_hwcnt(vinstr_ctx)) { ++ vinstr_ctx->reprogram = false; ++ list_for_each_entry( ++ iter, ++ &vinstr_ctx->idle_clients, ++ list) ++ iter->pending = false; ++ list_for_each_entry( ++ iter, ++ &vinstr_ctx->waiting_clients, ++ list) ++ iter->pending = false; ++ } ++ } ++} ++ ++/** ++ * kbasep_vinstr_update_client - copy accumulated counters to user readable ++ * buffer and notify the user ++ * @cli: requesting client ++ * @timestamp: timestamp when counters were collected ++ * @event_id: id of event that caused triggered counters collection ++ * ++ * Return: zero on success ++ */ ++static int kbasep_vinstr_update_client( ++ struct kbase_vinstr_client *cli, u64 timestamp, ++ enum base_hwcnt_reader_event event_id) ++{ ++ int rcode = 0; ++ ++ /* Copy collected counters to user readable buffer. */ ++ if (cli->buffer_count) ++ rcode = kbasep_vinstr_fill_dump_buffer( ++ cli, timestamp, event_id); ++ else if (cli->kernel_buffer) ++ rcode = kbasep_vinstr_fill_dump_buffer_kernel(cli); ++ else ++ rcode = kbasep_vinstr_fill_dump_buffer_legacy(cli); ++ ++ if (rcode) ++ goto exit; ++ ++ ++ /* Notify client. Make sure all changes to memory are visible. */ ++ wmb(); ++ atomic_inc(&cli->write_idx); ++ wake_up_interruptible(&cli->waitq); ++ ++ /* Prepare for next request. */ ++ memset(cli->accum_buffer, 0, cli->dump_size); ++ ++exit: ++ return rcode; ++} ++ ++/** ++ * kbasep_vinstr_wake_up_callback - vinstr wake up timer wake up function ++ * ++ * @hrtimer: high resolution timer ++ * ++ * Return: High resolution timer restart enum. ++ */ ++static enum hrtimer_restart kbasep_vinstr_wake_up_callback( ++ struct hrtimer *hrtimer) ++{ ++ struct kbasep_vinstr_wake_up_timer *timer = ++ container_of( ++ hrtimer, ++ struct kbasep_vinstr_wake_up_timer, ++ hrtimer); ++ ++ KBASE_DEBUG_ASSERT(timer); ++ ++ atomic_set(&timer->vinstr_ctx->request_pending, 1); ++ wake_up_all(&timer->vinstr_ctx->waitq); ++ ++ return HRTIMER_NORESTART; ++} ++ ++#ifdef CONFIG_DEBUG_OBJECT_TIMERS ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0)) ++/** ++ * kbase_destroy_hrtimer_on_stack - kernel's destroy_hrtimer_on_stack(), ++ * rewritten ++ * ++ * @timer: high resolution timer ++ * ++ * destroy_hrtimer_on_stack() was exported only for 4.7.0 kernel so for ++ * earlier kernel versions it is not possible to call it explicitly. ++ * Since this function must accompany hrtimer_init_on_stack(), which ++ * has to be used for hrtimer initialization if CONFIG_DEBUG_OBJECT_TIMERS ++ * is defined in order to avoid the warning about object on stack not being ++ * annotated, we rewrite it here to be used for earlier kernel versions. ++ */ ++static void kbase_destroy_hrtimer_on_stack(struct hrtimer *timer) ++{ ++ debug_object_free(timer, &hrtimer_debug_descr); ++} ++#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0) */ ++#endif /* CONFIG_DEBUG_OBJECT_TIMERS */ ++ ++/** ++ * kbasep_vinstr_service_task - HWC dumping service thread ++ * ++ * @data: Pointer to vinstr context structure. ++ * ++ * Return: Always returns zero. ++ */ ++static int kbasep_vinstr_service_task(void *data) ++{ ++ struct kbase_vinstr_context *vinstr_ctx = data; ++ struct kbasep_vinstr_wake_up_timer timer; ++ ++ KBASE_DEBUG_ASSERT(vinstr_ctx); ++ ++ hrtimer_init_on_stack(&timer.hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); ++ ++ timer.hrtimer.function = kbasep_vinstr_wake_up_callback; ++ timer.vinstr_ctx = vinstr_ctx; ++ ++ while (!kthread_should_stop()) { ++ struct kbase_vinstr_client *cli = NULL; ++ struct kbase_vinstr_client *tmp; ++ int rcode; ++ ++ u64 timestamp = kbasep_vinstr_get_timestamp(); ++ u64 dump_time = 0; ++ struct list_head expired_requests; ++ ++ /* Hold lock while performing operations on lists of clients. */ ++ mutex_lock(&vinstr_ctx->lock); ++ ++ /* Closing thread must not interact with client requests. */ ++ if (current == vinstr_ctx->thread) { ++ atomic_set(&vinstr_ctx->request_pending, 0); ++ ++ if (!list_empty(&vinstr_ctx->waiting_clients)) { ++ cli = list_first_entry( ++ &vinstr_ctx->waiting_clients, ++ struct kbase_vinstr_client, ++ list); ++ dump_time = cli->dump_time; ++ } ++ } ++ ++ if (!cli || ((s64)timestamp - (s64)dump_time < 0ll)) { ++ mutex_unlock(&vinstr_ctx->lock); ++ ++ /* Sleep until next dumping event or service request. */ ++ if (cli) { ++ u64 diff = dump_time - timestamp; ++ ++ hrtimer_start( ++ &timer.hrtimer, ++ ns_to_ktime(diff), ++ HRTIMER_MODE_REL); ++ } ++ wait_event( ++ vinstr_ctx->waitq, ++ atomic_read( ++ &vinstr_ctx->request_pending) || ++ kthread_should_stop()); ++ hrtimer_cancel(&timer.hrtimer); ++ continue; ++ } ++ ++ rcode = kbasep_vinstr_collect_and_accumulate(vinstr_ctx, ++ ×tamp); ++ ++ INIT_LIST_HEAD(&expired_requests); ++ ++ /* Find all expired requests. */ ++ list_for_each_entry_safe( ++ cli, ++ tmp, ++ &vinstr_ctx->waiting_clients, ++ list) { ++ s64 tdiff = ++ (s64)(timestamp + DUMPING_RESOLUTION) - ++ (s64)cli->dump_time; ++ if (tdiff >= 0ll) { ++ list_del(&cli->list); ++ list_add(&cli->list, &expired_requests); ++ } else { ++ break; ++ } ++ } ++ ++ /* Fill data for each request found. */ ++ list_for_each_entry_safe(cli, tmp, &expired_requests, list) { ++ /* Ensure that legacy buffer will not be used from ++ * this kthread context. */ ++ BUG_ON(0 == cli->buffer_count); ++ /* Expect only periodically sampled clients. */ ++ BUG_ON(0 == cli->dump_interval); ++ ++ if (!rcode) ++ kbasep_vinstr_update_client( ++ cli, ++ timestamp, ++ BASE_HWCNT_READER_EVENT_PERIODIC); ++ ++ /* Set new dumping time. Drop missed probing times. */ ++ do { ++ cli->dump_time += cli->dump_interval; ++ } while (cli->dump_time < timestamp); ++ ++ list_del(&cli->list); ++ kbasep_vinstr_add_dump_request( ++ cli, ++ &vinstr_ctx->waiting_clients); ++ } ++ ++ /* Reprogram counters set if required. */ ++ kbasep_vinstr_reprogram(vinstr_ctx); ++ ++ mutex_unlock(&vinstr_ctx->lock); ++ } ++ ++#ifdef CONFIG_DEBUG_OBJECTS_TIMERS ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0)) ++ kbase_destroy_hrtimer_on_stack(&timer.hrtimer); ++#else ++ destroy_hrtimer_on_stack(&timer.hrtimer); ++#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0)) */ ++#endif /* CONFIG_DEBUG_OBJECTS_TIMERS */ ++ ++ return 0; ++} ++ ++/*****************************************************************************/ ++ ++/** ++ * kbasep_vinstr_hwcnt_reader_buffer_ready - check if client has ready buffers ++ * @cli: pointer to vinstr client structure ++ * ++ * Return: non-zero if client has at least one dumping buffer filled that was ++ * not notified to user yet ++ */ ++static int kbasep_vinstr_hwcnt_reader_buffer_ready( ++ struct kbase_vinstr_client *cli) ++{ ++ KBASE_DEBUG_ASSERT(cli); ++ return atomic_read(&cli->write_idx) != atomic_read(&cli->meta_idx); ++} ++ ++/** ++ * kbasep_vinstr_hwcnt_reader_ioctl_get_buffer - hwcnt reader's ioctl command ++ * @cli: pointer to vinstr client structure ++ * @buffer: pointer to userspace buffer ++ * @size: size of buffer ++ * ++ * Return: zero on success ++ */ ++static long kbasep_vinstr_hwcnt_reader_ioctl_get_buffer( ++ struct kbase_vinstr_client *cli, void __user *buffer, ++ size_t size) ++{ ++ unsigned int meta_idx = atomic_read(&cli->meta_idx); ++ unsigned int idx = meta_idx % cli->buffer_count; ++ ++ struct kbase_hwcnt_reader_metadata *meta = &cli->dump_buffers_meta[idx]; ++ ++ /* Metadata sanity check. */ ++ KBASE_DEBUG_ASSERT(idx == meta->buffer_idx); ++ ++ if (sizeof(struct kbase_hwcnt_reader_metadata) != size) ++ return -EINVAL; ++ ++ /* Check if there is any buffer available. */ ++ if (atomic_read(&cli->write_idx) == meta_idx) ++ return -EAGAIN; ++ ++ /* Check if previously taken buffer was put back. */ ++ if (atomic_read(&cli->read_idx) != meta_idx) ++ return -EBUSY; ++ ++ /* Copy next available buffer's metadata to user. */ ++ if (copy_to_user(buffer, meta, size)) ++ return -EFAULT; ++ ++ atomic_inc(&cli->meta_idx); ++ ++ return 0; ++} ++ ++/** ++ * kbasep_vinstr_hwcnt_reader_ioctl_put_buffer - hwcnt reader's ioctl command ++ * @cli: pointer to vinstr client structure ++ * @buffer: pointer to userspace buffer ++ * @size: size of buffer ++ * ++ * Return: zero on success ++ */ ++static long kbasep_vinstr_hwcnt_reader_ioctl_put_buffer( ++ struct kbase_vinstr_client *cli, void __user *buffer, ++ size_t size) ++{ ++ unsigned int read_idx = atomic_read(&cli->read_idx); ++ unsigned int idx = read_idx % cli->buffer_count; ++ ++ struct kbase_hwcnt_reader_metadata meta; ++ ++ if (sizeof(struct kbase_hwcnt_reader_metadata) != size) ++ return -EINVAL; ++ ++ /* Check if any buffer was taken. */ ++ if (atomic_read(&cli->meta_idx) == read_idx) ++ return -EPERM; ++ ++ /* Check if correct buffer is put back. */ ++ if (copy_from_user(&meta, buffer, size)) ++ return -EFAULT; ++ if (idx != meta.buffer_idx) ++ return -EINVAL; ++ ++ atomic_inc(&cli->read_idx); ++ ++ return 0; ++} ++ ++/** ++ * kbasep_vinstr_hwcnt_reader_ioctl_set_interval - hwcnt reader's ioctl command ++ * @cli: pointer to vinstr client structure ++ * @interval: periodic dumping interval (disable periodic dumping if zero) ++ * ++ * Return: zero on success ++ */ ++static long kbasep_vinstr_hwcnt_reader_ioctl_set_interval( ++ struct kbase_vinstr_client *cli, u32 interval) ++{ ++ struct kbase_vinstr_context *vinstr_ctx = cli->vinstr_ctx; ++ ++ KBASE_DEBUG_ASSERT(vinstr_ctx); ++ ++ mutex_lock(&vinstr_ctx->lock); ++ ++ list_del(&cli->list); ++ ++ cli->dump_interval = interval; ++ ++ /* If interval is non-zero, enable periodic dumping for this client. */ ++ if (cli->dump_interval) { ++ if (DUMPING_RESOLUTION > cli->dump_interval) ++ cli->dump_interval = DUMPING_RESOLUTION; ++ cli->dump_time = ++ kbasep_vinstr_get_timestamp() + cli->dump_interval; ++ ++ kbasep_vinstr_add_dump_request( ++ cli, &vinstr_ctx->waiting_clients); ++ ++ atomic_set(&vinstr_ctx->request_pending, 1); ++ wake_up_all(&vinstr_ctx->waitq); ++ } else { ++ list_add(&cli->list, &vinstr_ctx->idle_clients); ++ } ++ ++ mutex_unlock(&vinstr_ctx->lock); ++ ++ return 0; ++} ++ ++/** ++ * kbasep_vinstr_hwcnt_reader_event_mask - return event mask for event id ++ * @event_id: id of event ++ * Return: event_mask or zero if event is not supported or maskable ++ */ ++static u32 kbasep_vinstr_hwcnt_reader_event_mask( ++ enum base_hwcnt_reader_event event_id) ++{ ++ u32 event_mask = 0; ++ ++ switch (event_id) { ++ case BASE_HWCNT_READER_EVENT_PREJOB: ++ case BASE_HWCNT_READER_EVENT_POSTJOB: ++ /* These event are maskable. */ ++ event_mask = (1 << event_id); ++ break; ++ ++ case BASE_HWCNT_READER_EVENT_MANUAL: ++ case BASE_HWCNT_READER_EVENT_PERIODIC: ++ /* These event are non-maskable. */ ++ default: ++ /* These event are not supported. */ ++ break; ++ } ++ ++ return event_mask; ++} ++ ++/** ++ * kbasep_vinstr_hwcnt_reader_ioctl_enable_event - hwcnt reader's ioctl command ++ * @cli: pointer to vinstr client structure ++ * @event_id: id of event to enable ++ * ++ * Return: zero on success ++ */ ++static long kbasep_vinstr_hwcnt_reader_ioctl_enable_event( ++ struct kbase_vinstr_client *cli, ++ enum base_hwcnt_reader_event event_id) ++{ ++ struct kbase_vinstr_context *vinstr_ctx = cli->vinstr_ctx; ++ u32 event_mask; ++ ++ KBASE_DEBUG_ASSERT(vinstr_ctx); ++ ++ event_mask = kbasep_vinstr_hwcnt_reader_event_mask(event_id); ++ if (!event_mask) ++ return -EINVAL; ++ ++ mutex_lock(&vinstr_ctx->lock); ++ cli->event_mask |= event_mask; ++ mutex_unlock(&vinstr_ctx->lock); ++ ++ return 0; ++} ++ ++/** ++ * kbasep_vinstr_hwcnt_reader_ioctl_disable_event - hwcnt reader's ioctl command ++ * @cli: pointer to vinstr client structure ++ * @event_id: id of event to disable ++ * ++ * Return: zero on success ++ */ ++static long kbasep_vinstr_hwcnt_reader_ioctl_disable_event( ++ struct kbase_vinstr_client *cli, ++ enum base_hwcnt_reader_event event_id) ++{ ++ struct kbase_vinstr_context *vinstr_ctx = cli->vinstr_ctx; ++ u32 event_mask; ++ ++ KBASE_DEBUG_ASSERT(vinstr_ctx); ++ ++ event_mask = kbasep_vinstr_hwcnt_reader_event_mask(event_id); ++ if (!event_mask) ++ return -EINVAL; ++ ++ mutex_lock(&vinstr_ctx->lock); ++ cli->event_mask &= ~event_mask; ++ mutex_unlock(&vinstr_ctx->lock); ++ ++ return 0; ++} ++ ++/** ++ * kbasep_vinstr_hwcnt_reader_ioctl_get_hwver - hwcnt reader's ioctl command ++ * @cli: pointer to vinstr client structure ++ * @hwver: pointer to user buffer where hw version will be stored ++ * ++ * Return: zero on success ++ */ ++static long kbasep_vinstr_hwcnt_reader_ioctl_get_hwver( ++ struct kbase_vinstr_client *cli, u32 __user *hwver) ++{ ++#ifndef CONFIG_MALI_NO_MALI ++ struct kbase_vinstr_context *vinstr_ctx = cli->vinstr_ctx; ++#endif ++ ++ u32 ver = 5; ++ ++#ifndef CONFIG_MALI_NO_MALI ++ KBASE_DEBUG_ASSERT(vinstr_ctx); ++ if (kbase_hw_has_feature(vinstr_ctx->kbdev, BASE_HW_FEATURE_V4)) ++ ver = 4; ++#endif ++ ++ return put_user(ver, hwver); ++} ++ ++/** ++ * kbasep_vinstr_hwcnt_reader_ioctl - hwcnt reader's ioctl ++ * @filp: pointer to file structure ++ * @cmd: user command ++ * @arg: command's argument ++ * ++ * Return: zero on success ++ */ ++static long kbasep_vinstr_hwcnt_reader_ioctl(struct file *filp, ++ unsigned int cmd, unsigned long arg) ++{ ++ long rcode = 0; ++ struct kbase_vinstr_client *cli; ++ ++ KBASE_DEBUG_ASSERT(filp); ++ ++ cli = filp->private_data; ++ KBASE_DEBUG_ASSERT(cli); ++ ++ if (unlikely(KBASE_HWCNT_READER != _IOC_TYPE(cmd))) ++ return -EINVAL; ++ ++ switch (cmd) { ++ case KBASE_HWCNT_READER_GET_API_VERSION: ++ rcode = put_user(HWCNT_READER_API, (u32 __user *)arg); ++ break; ++ case KBASE_HWCNT_READER_GET_HWVER: ++ rcode = kbasep_vinstr_hwcnt_reader_ioctl_get_hwver( ++ cli, (u32 __user *)arg); ++ break; ++ case KBASE_HWCNT_READER_GET_BUFFER_SIZE: ++ KBASE_DEBUG_ASSERT(cli->vinstr_ctx); ++ rcode = put_user( ++ (u32)cli->vinstr_ctx->dump_size, ++ (u32 __user *)arg); ++ break; ++ case KBASE_HWCNT_READER_DUMP: ++ rcode = kbase_vinstr_hwc_dump( ++ cli, BASE_HWCNT_READER_EVENT_MANUAL); ++ break; ++ case KBASE_HWCNT_READER_CLEAR: ++ rcode = kbase_vinstr_hwc_clear(cli); ++ break; ++ case KBASE_HWCNT_READER_GET_BUFFER: ++ rcode = kbasep_vinstr_hwcnt_reader_ioctl_get_buffer( ++ cli, (void __user *)arg, _IOC_SIZE(cmd)); ++ break; ++ case KBASE_HWCNT_READER_PUT_BUFFER: ++ rcode = kbasep_vinstr_hwcnt_reader_ioctl_put_buffer( ++ cli, (void __user *)arg, _IOC_SIZE(cmd)); ++ break; ++ case KBASE_HWCNT_READER_SET_INTERVAL: ++ rcode = kbasep_vinstr_hwcnt_reader_ioctl_set_interval( ++ cli, (u32)arg); ++ break; ++ case KBASE_HWCNT_READER_ENABLE_EVENT: ++ rcode = kbasep_vinstr_hwcnt_reader_ioctl_enable_event( ++ cli, (enum base_hwcnt_reader_event)arg); ++ break; ++ case KBASE_HWCNT_READER_DISABLE_EVENT: ++ rcode = kbasep_vinstr_hwcnt_reader_ioctl_disable_event( ++ cli, (enum base_hwcnt_reader_event)arg); ++ break; ++ default: ++ rcode = -EINVAL; ++ break; ++ } ++ ++ return rcode; ++} ++ ++/** ++ * kbasep_vinstr_hwcnt_reader_poll - hwcnt reader's poll ++ * @filp: pointer to file structure ++ * @wait: pointer to poll table ++ * Return: POLLIN if data can be read without blocking, otherwise zero ++ */ ++static unsigned int kbasep_vinstr_hwcnt_reader_poll(struct file *filp, ++ poll_table *wait) ++{ ++ struct kbase_vinstr_client *cli; ++ ++ KBASE_DEBUG_ASSERT(filp); ++ KBASE_DEBUG_ASSERT(wait); ++ ++ cli = filp->private_data; ++ KBASE_DEBUG_ASSERT(cli); ++ ++ poll_wait(filp, &cli->waitq, wait); ++ if (kbasep_vinstr_hwcnt_reader_buffer_ready(cli)) ++ return POLLIN; ++ return 0; ++} ++ ++/** ++ * kbasep_vinstr_hwcnt_reader_mmap - hwcnt reader's mmap ++ * @filp: pointer to file structure ++ * @vma: pointer to vma structure ++ * Return: zero on success ++ */ ++static int kbasep_vinstr_hwcnt_reader_mmap(struct file *filp, ++ struct vm_area_struct *vma) ++{ ++ struct kbase_vinstr_client *cli; ++ unsigned long size, addr, pfn, offset; ++ unsigned long vm_size = vma->vm_end - vma->vm_start; ++ ++ KBASE_DEBUG_ASSERT(filp); ++ KBASE_DEBUG_ASSERT(vma); ++ ++ cli = filp->private_data; ++ KBASE_DEBUG_ASSERT(cli); ++ ++ size = cli->buffer_count * cli->dump_size; ++ ++ if (vma->vm_pgoff > (size >> PAGE_SHIFT)) ++ return -EINVAL; ++ ++ offset = vma->vm_pgoff << PAGE_SHIFT; ++ if (vm_size > size - offset) ++ return -EINVAL; ++ ++ addr = __pa((unsigned long)cli->dump_buffers + offset); ++ pfn = addr >> PAGE_SHIFT; ++ ++ return remap_pfn_range( ++ vma, ++ vma->vm_start, ++ pfn, ++ vm_size, ++ vma->vm_page_prot); ++} ++ ++/** ++ * kbasep_vinstr_hwcnt_reader_release - hwcnt reader's release ++ * @inode: pointer to inode structure ++ * @filp: pointer to file structure ++ * Return always return zero ++ */ ++static int kbasep_vinstr_hwcnt_reader_release(struct inode *inode, ++ struct file *filp) ++{ ++ struct kbase_vinstr_client *cli; ++ ++ KBASE_DEBUG_ASSERT(inode); ++ KBASE_DEBUG_ASSERT(filp); ++ ++ cli = filp->private_data; ++ KBASE_DEBUG_ASSERT(cli); ++ ++ kbase_vinstr_detach_client(cli); ++ return 0; ++} ++ ++/*****************************************************************************/ ++ ++/** ++ * kbasep_vinstr_kick_scheduler - trigger scheduler cycle ++ * @kbdev: pointer to kbase device structure ++ */ ++static void kbasep_vinstr_kick_scheduler(struct kbase_device *kbdev) ++{ ++ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; ++ unsigned long flags; ++ ++ down(&js_devdata->schedule_sem); ++ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); ++ kbase_backend_slot_update(kbdev); ++ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); ++ up(&js_devdata->schedule_sem); ++} ++ ++/** ++ * kbasep_vinstr_suspend_worker - worker suspending vinstr module ++ * @data: pointer to work structure ++ */ ++static void kbasep_vinstr_suspend_worker(struct work_struct *data) ++{ ++ struct kbase_vinstr_context *vinstr_ctx; ++ unsigned long flags; ++ ++ vinstr_ctx = container_of(data, struct kbase_vinstr_context, ++ suspend_work); ++ ++ mutex_lock(&vinstr_ctx->lock); ++ ++ if (vinstr_ctx->kctx) ++ disable_hwcnt(vinstr_ctx); ++ ++ spin_lock_irqsave(&vinstr_ctx->state_lock, flags); ++ vinstr_ctx->state = VINSTR_SUSPENDED; ++ wake_up_all(&vinstr_ctx->suspend_waitq); ++ spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); ++ ++ mutex_unlock(&vinstr_ctx->lock); ++ ++ /* Kick GPU scheduler to allow entering protected mode. ++ * This must happen after vinstr was suspended. */ ++ kbasep_vinstr_kick_scheduler(vinstr_ctx->kbdev); ++} ++ ++/** ++ * kbasep_vinstr_suspend_worker - worker resuming vinstr module ++ * @data: pointer to work structure ++ */ ++static void kbasep_vinstr_resume_worker(struct work_struct *data) ++{ ++ struct kbase_vinstr_context *vinstr_ctx; ++ unsigned long flags; ++ ++ vinstr_ctx = container_of(data, struct kbase_vinstr_context, ++ resume_work); ++ ++ mutex_lock(&vinstr_ctx->lock); ++ ++ if (vinstr_ctx->kctx) ++ enable_hwcnt(vinstr_ctx); ++ ++ spin_lock_irqsave(&vinstr_ctx->state_lock, flags); ++ vinstr_ctx->state = VINSTR_IDLE; ++ wake_up_all(&vinstr_ctx->suspend_waitq); ++ spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); ++ ++ mutex_unlock(&vinstr_ctx->lock); ++ ++ /* Kick GPU scheduler to allow entering protected mode. ++ * Note that scheduler state machine might requested re-entry to ++ * protected mode before vinstr was resumed. ++ * This must happen after vinstr was release. */ ++ kbasep_vinstr_kick_scheduler(vinstr_ctx->kbdev); ++} ++ ++/*****************************************************************************/ ++ ++struct kbase_vinstr_context *kbase_vinstr_init(struct kbase_device *kbdev) ++{ ++ struct kbase_vinstr_context *vinstr_ctx; ++ ++ vinstr_ctx = kzalloc(sizeof(*vinstr_ctx), GFP_KERNEL); ++ if (!vinstr_ctx) ++ return NULL; ++ ++ INIT_LIST_HEAD(&vinstr_ctx->idle_clients); ++ INIT_LIST_HEAD(&vinstr_ctx->waiting_clients); ++ mutex_init(&vinstr_ctx->lock); ++ spin_lock_init(&vinstr_ctx->state_lock); ++ vinstr_ctx->kbdev = kbdev; ++ vinstr_ctx->thread = NULL; ++ vinstr_ctx->state = VINSTR_IDLE; ++ vinstr_ctx->suspend_cnt = 0; ++ INIT_WORK(&vinstr_ctx->suspend_work, kbasep_vinstr_suspend_worker); ++ INIT_WORK(&vinstr_ctx->resume_work, kbasep_vinstr_resume_worker); ++ init_waitqueue_head(&vinstr_ctx->suspend_waitq); ++ ++ atomic_set(&vinstr_ctx->request_pending, 0); ++ init_waitqueue_head(&vinstr_ctx->waitq); ++ ++ return vinstr_ctx; ++} ++ ++void kbase_vinstr_term(struct kbase_vinstr_context *vinstr_ctx) ++{ ++ struct kbase_vinstr_client *cli; ++ ++ /* Stop service thread first. */ ++ if (vinstr_ctx->thread) ++ kthread_stop(vinstr_ctx->thread); ++ ++ /* Wait for workers. */ ++ flush_work(&vinstr_ctx->suspend_work); ++ flush_work(&vinstr_ctx->resume_work); ++ ++ while (1) { ++ struct list_head *list = &vinstr_ctx->idle_clients; ++ ++ if (list_empty(list)) { ++ list = &vinstr_ctx->waiting_clients; ++ if (list_empty(list)) ++ break; ++ } ++ ++ cli = list_first_entry(list, struct kbase_vinstr_client, list); ++ list_del(&cli->list); ++ kfree(cli->accum_buffer); ++ kfree(cli); ++ vinstr_ctx->nclients--; ++ } ++ KBASE_DEBUG_ASSERT(!vinstr_ctx->nclients); ++ if (vinstr_ctx->kctx) ++ kbasep_vinstr_destroy_kctx(vinstr_ctx); ++ kfree(vinstr_ctx); ++} ++ ++int kbase_vinstr_hwcnt_reader_setup(struct kbase_vinstr_context *vinstr_ctx, ++ struct kbase_uk_hwcnt_reader_setup *setup) ++{ ++ struct kbase_vinstr_client *cli; ++ u32 bitmap[4]; ++ ++ KBASE_DEBUG_ASSERT(vinstr_ctx); ++ KBASE_DEBUG_ASSERT(setup); ++ KBASE_DEBUG_ASSERT(setup->buffer_count); ++ ++ bitmap[SHADER_HWCNT_BM] = setup->shader_bm; ++ bitmap[TILER_HWCNT_BM] = setup->tiler_bm; ++ bitmap[MMU_L2_HWCNT_BM] = setup->mmu_l2_bm; ++ bitmap[JM_HWCNT_BM] = setup->jm_bm; ++ ++ cli = kbasep_vinstr_attach_client( ++ vinstr_ctx, ++ setup->buffer_count, ++ bitmap, ++ &setup->fd, ++ NULL); ++ ++ if (!cli) ++ return -ENOMEM; ++ ++ return 0; ++} ++ ++int kbase_vinstr_legacy_hwc_setup( ++ struct kbase_vinstr_context *vinstr_ctx, ++ struct kbase_vinstr_client **cli, ++ struct kbase_uk_hwcnt_setup *setup) ++{ ++ KBASE_DEBUG_ASSERT(vinstr_ctx); ++ KBASE_DEBUG_ASSERT(setup); ++ KBASE_DEBUG_ASSERT(cli); ++ ++ if (setup->dump_buffer) { ++ u32 bitmap[4]; ++ ++ bitmap[SHADER_HWCNT_BM] = setup->shader_bm; ++ bitmap[TILER_HWCNT_BM] = setup->tiler_bm; ++ bitmap[MMU_L2_HWCNT_BM] = setup->mmu_l2_bm; ++ bitmap[JM_HWCNT_BM] = setup->jm_bm; ++ ++ if (*cli) ++ return -EBUSY; ++ ++ *cli = kbasep_vinstr_attach_client( ++ vinstr_ctx, ++ 0, ++ bitmap, ++ (void *)(long)setup->dump_buffer, ++ NULL); ++ ++ if (!(*cli)) ++ return -ENOMEM; ++ } else { ++ if (!*cli) ++ return -EINVAL; ++ ++ kbase_vinstr_detach_client(*cli); ++ *cli = NULL; ++ } ++ ++ return 0; ++} ++ ++struct kbase_vinstr_client *kbase_vinstr_hwcnt_kernel_setup( ++ struct kbase_vinstr_context *vinstr_ctx, ++ struct kbase_uk_hwcnt_reader_setup *setup, ++ void *kernel_buffer) ++{ ++ u32 bitmap[4]; ++ ++ if (!vinstr_ctx || !setup || !kernel_buffer) ++ return NULL; ++ ++ bitmap[SHADER_HWCNT_BM] = setup->shader_bm; ++ bitmap[TILER_HWCNT_BM] = setup->tiler_bm; ++ bitmap[MMU_L2_HWCNT_BM] = setup->mmu_l2_bm; ++ bitmap[JM_HWCNT_BM] = setup->jm_bm; ++ ++ return kbasep_vinstr_attach_client( ++ vinstr_ctx, ++ 0, ++ bitmap, ++ NULL, ++ kernel_buffer); ++} ++KBASE_EXPORT_TEST_API(kbase_vinstr_hwcnt_kernel_setup); ++ ++int kbase_vinstr_hwc_dump(struct kbase_vinstr_client *cli, ++ enum base_hwcnt_reader_event event_id) ++{ ++ int rcode = 0; ++ struct kbase_vinstr_context *vinstr_ctx; ++ u64 timestamp; ++ u32 event_mask; ++ ++ if (!cli) ++ return -EINVAL; ++ ++ vinstr_ctx = cli->vinstr_ctx; ++ KBASE_DEBUG_ASSERT(vinstr_ctx); ++ ++ KBASE_DEBUG_ASSERT(event_id < BASE_HWCNT_READER_EVENT_COUNT); ++ event_mask = 1 << event_id; ++ ++ mutex_lock(&vinstr_ctx->lock); ++ ++ if (event_mask & cli->event_mask) { ++ rcode = kbasep_vinstr_collect_and_accumulate( ++ vinstr_ctx, ++ ×tamp); ++ if (rcode) ++ goto exit; ++ ++ rcode = kbasep_vinstr_update_client(cli, timestamp, event_id); ++ if (rcode) ++ goto exit; ++ ++ kbasep_vinstr_reprogram(vinstr_ctx); ++ } ++ ++exit: ++ mutex_unlock(&vinstr_ctx->lock); ++ ++ return rcode; ++} ++KBASE_EXPORT_TEST_API(kbase_vinstr_hwc_dump); ++ ++int kbase_vinstr_hwc_clear(struct kbase_vinstr_client *cli) ++{ ++ struct kbase_vinstr_context *vinstr_ctx; ++ int rcode; ++ u64 unused; ++ ++ if (!cli) ++ return -EINVAL; ++ ++ vinstr_ctx = cli->vinstr_ctx; ++ KBASE_DEBUG_ASSERT(vinstr_ctx); ++ ++ mutex_lock(&vinstr_ctx->lock); ++ ++ rcode = kbasep_vinstr_collect_and_accumulate(vinstr_ctx, &unused); ++ if (rcode) ++ goto exit; ++ rcode = kbase_instr_hwcnt_clear(vinstr_ctx->kctx); ++ if (rcode) ++ goto exit; ++ memset(cli->accum_buffer, 0, cli->dump_size); ++ ++ kbasep_vinstr_reprogram(vinstr_ctx); ++ ++exit: ++ mutex_unlock(&vinstr_ctx->lock); ++ ++ return rcode; ++} ++ ++int kbase_vinstr_try_suspend(struct kbase_vinstr_context *vinstr_ctx) ++{ ++ unsigned long flags; ++ int ret = -EAGAIN; ++ ++ KBASE_DEBUG_ASSERT(vinstr_ctx); ++ ++ spin_lock_irqsave(&vinstr_ctx->state_lock, flags); ++ switch (vinstr_ctx->state) { ++ case VINSTR_SUSPENDED: ++ vinstr_ctx->suspend_cnt++; ++ /* overflow shall not happen */ ++ BUG_ON(0 == vinstr_ctx->suspend_cnt); ++ ret = 0; ++ break; ++ ++ case VINSTR_IDLE: ++ vinstr_ctx->state = VINSTR_SUSPENDING; ++ schedule_work(&vinstr_ctx->suspend_work); ++ break; ++ ++ case VINSTR_DUMPING: ++ vinstr_ctx->state = VINSTR_SUSPENDING; ++ break; ++ ++ case VINSTR_SUSPENDING: ++ /* fall through */ ++ case VINSTR_RESUMING: ++ break; ++ ++ default: ++ BUG(); ++ break; ++ } ++ spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); ++ ++ return ret; ++} ++ ++void kbase_vinstr_suspend(struct kbase_vinstr_context *vinstr_ctx) ++{ ++ wait_event(vinstr_ctx->suspend_waitq, ++ (0 == kbase_vinstr_try_suspend(vinstr_ctx))); ++} ++ ++void kbase_vinstr_resume(struct kbase_vinstr_context *vinstr_ctx) ++{ ++ unsigned long flags; ++ ++ KBASE_DEBUG_ASSERT(vinstr_ctx); ++ ++ spin_lock_irqsave(&vinstr_ctx->state_lock, flags); ++ BUG_ON(VINSTR_SUSPENDING == vinstr_ctx->state); ++ if (VINSTR_SUSPENDED == vinstr_ctx->state) { ++ BUG_ON(0 == vinstr_ctx->suspend_cnt); ++ vinstr_ctx->suspend_cnt--; ++ if (0 == vinstr_ctx->suspend_cnt) { ++ vinstr_ctx->state = VINSTR_RESUMING; ++ schedule_work(&vinstr_ctx->resume_work); ++ } ++ } ++ spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); ++} +diff --git a/drivers/gpu/arm/midgard/mali_kbase_vinstr.h b/drivers/gpu/arm/midgard/mali_kbase_vinstr.h +new file mode 100755 +index 000000000000..6207d25aef06 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_kbase_vinstr.h +@@ -0,0 +1,155 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KBASE_VINSTR_H_ ++#define _KBASE_VINSTR_H_ ++ ++#include ++#include ++ ++/*****************************************************************************/ ++ ++struct kbase_vinstr_context; ++struct kbase_vinstr_client; ++ ++/*****************************************************************************/ ++ ++/** ++ * kbase_vinstr_init() - initialize the vinstr core ++ * @kbdev: kbase device ++ * ++ * Return: pointer to the vinstr context on success or NULL on failure ++ */ ++struct kbase_vinstr_context *kbase_vinstr_init(struct kbase_device *kbdev); ++ ++/** ++ * kbase_vinstr_term() - terminate the vinstr core ++ * @vinstr_ctx: vinstr context ++ */ ++void kbase_vinstr_term(struct kbase_vinstr_context *vinstr_ctx); ++ ++/** ++ * kbase_vinstr_hwcnt_reader_setup - configure hw counters reader ++ * @vinstr_ctx: vinstr context ++ * @setup: reader's configuration ++ * ++ * Return: zero on success ++ */ ++int kbase_vinstr_hwcnt_reader_setup( ++ struct kbase_vinstr_context *vinstr_ctx, ++ struct kbase_uk_hwcnt_reader_setup *setup); ++ ++/** ++ * kbase_vinstr_legacy_hwc_setup - configure hw counters for dumping ++ * @vinstr_ctx: vinstr context ++ * @cli: pointer where to store pointer to new vinstr client structure ++ * @setup: hwc configuration ++ * ++ * Return: zero on success ++ */ ++int kbase_vinstr_legacy_hwc_setup( ++ struct kbase_vinstr_context *vinstr_ctx, ++ struct kbase_vinstr_client **cli, ++ struct kbase_uk_hwcnt_setup *setup); ++ ++/** ++ * kbase_vinstr_hwcnt_kernel_setup - configure hw counters for kernel side ++ * client ++ * @vinstr_ctx: vinstr context ++ * @setup: reader's configuration ++ * @kernel_buffer: pointer to dump buffer ++ * ++ * setup->buffer_count and setup->fd are not used for kernel side clients. ++ * ++ * Return: pointer to client structure, or NULL on failure ++ */ ++struct kbase_vinstr_client *kbase_vinstr_hwcnt_kernel_setup( ++ struct kbase_vinstr_context *vinstr_ctx, ++ struct kbase_uk_hwcnt_reader_setup *setup, ++ void *kernel_buffer); ++ ++/** ++ * kbase_vinstr_hwc_dump - issue counter dump for vinstr client ++ * @cli: pointer to vinstr client ++ * @event_id: id of event that triggered hwcnt dump ++ * ++ * Return: zero on success ++ */ ++int kbase_vinstr_hwc_dump( ++ struct kbase_vinstr_client *cli, ++ enum base_hwcnt_reader_event event_id); ++ ++/** ++ * kbase_vinstr_hwc_clear - performs a reset of the hardware counters for ++ * a given kbase context ++ * @cli: pointer to vinstr client ++ * ++ * Return: zero on success ++ */ ++int kbase_vinstr_hwc_clear(struct kbase_vinstr_client *cli); ++ ++/** ++ * kbase_vinstr_try_suspend - try suspending operation of a given vinstr context ++ * @vinstr_ctx: vinstr context ++ * ++ * Return: 0 on success, or negative if state change is in progress ++ * ++ * Warning: This API call is non-generic. It is meant to be used only by ++ * job scheduler state machine. ++ * ++ * Function initiates vinstr switch to suspended state. Once it was called ++ * vinstr enters suspending state. If function return non-zero value, it ++ * indicates that state switch is not complete and function must be called ++ * again. On state switch vinstr will trigger job scheduler state machine ++ * cycle. ++ */ ++int kbase_vinstr_try_suspend(struct kbase_vinstr_context *vinstr_ctx); ++ ++/** ++ * kbase_vinstr_suspend - suspends operation of a given vinstr context ++ * @vinstr_ctx: vinstr context ++ * ++ * Function initiates vinstr switch to suspended state. Then it blocks until ++ * operation is completed. ++ */ ++void kbase_vinstr_suspend(struct kbase_vinstr_context *vinstr_ctx); ++ ++/** ++ * kbase_vinstr_resume - resumes operation of a given vinstr context ++ * @vinstr_ctx: vinstr context ++ * ++ * Function can be called only if it was preceded by a successful call ++ * to kbase_vinstr_suspend. ++ */ ++void kbase_vinstr_resume(struct kbase_vinstr_context *vinstr_ctx); ++ ++/** ++ * kbase_vinstr_dump_size - Return required size of dump buffer ++ * @kbdev: device pointer ++ * ++ * Return : buffer size in bytes ++ */ ++size_t kbase_vinstr_dump_size(struct kbase_device *kbdev); ++ ++/** ++ * kbase_vinstr_detach_client - Detach a client from the vinstr core ++ * @cli: pointer to vinstr client ++ */ ++void kbase_vinstr_detach_client(struct kbase_vinstr_client *cli); ++ ++#endif /* _KBASE_VINSTR_H_ */ ++ +diff --git a/drivers/gpu/arm/midgard/mali_linux_kbase_trace.h b/drivers/gpu/arm/midgard/mali_linux_kbase_trace.h +new file mode 100755 +index 000000000000..5d6b4021d626 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_linux_kbase_trace.h +@@ -0,0 +1,201 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++#if !defined(_TRACE_MALI_KBASE_H) || defined(TRACE_HEADER_MULTI_READ) ++#define _TRACE_MALI_KBASE_H ++ ++#undef TRACE_SYSTEM ++#define TRACE_SYSTEM mali ++ ++#include ++ ++DECLARE_EVENT_CLASS(mali_slot_template, ++ TP_PROTO(int jobslot, unsigned int info_val), ++ TP_ARGS(jobslot, info_val), ++ TP_STRUCT__entry( ++ __field(unsigned int, jobslot) ++ __field(unsigned int, info_val) ++ ), ++ TP_fast_assign( ++ __entry->jobslot = jobslot; ++ __entry->info_val = info_val; ++ ), ++ TP_printk("jobslot=%u info=%u", __entry->jobslot, __entry->info_val) ++); ++ ++#define DEFINE_MALI_SLOT_EVENT(name) \ ++DEFINE_EVENT(mali_slot_template, mali_##name, \ ++ TP_PROTO(int jobslot, unsigned int info_val), \ ++ TP_ARGS(jobslot, info_val)) ++DEFINE_MALI_SLOT_EVENT(JM_SUBMIT); ++DEFINE_MALI_SLOT_EVENT(JM_JOB_DONE); ++DEFINE_MALI_SLOT_EVENT(JM_UPDATE_HEAD); ++DEFINE_MALI_SLOT_EVENT(JM_CHECK_HEAD); ++DEFINE_MALI_SLOT_EVENT(JM_SOFTSTOP); ++DEFINE_MALI_SLOT_EVENT(JM_SOFTSTOP_0); ++DEFINE_MALI_SLOT_EVENT(JM_SOFTSTOP_1); ++DEFINE_MALI_SLOT_EVENT(JM_HARDSTOP); ++DEFINE_MALI_SLOT_EVENT(JM_HARDSTOP_0); ++DEFINE_MALI_SLOT_EVENT(JM_HARDSTOP_1); ++DEFINE_MALI_SLOT_EVENT(JM_SLOT_SOFT_OR_HARD_STOP); ++DEFINE_MALI_SLOT_EVENT(JM_SLOT_EVICT); ++DEFINE_MALI_SLOT_EVENT(JM_BEGIN_RESET_WORKER); ++DEFINE_MALI_SLOT_EVENT(JM_END_RESET_WORKER); ++DEFINE_MALI_SLOT_EVENT(JS_CORE_REF_REGISTER_ON_RECHECK_FAILED); ++DEFINE_MALI_SLOT_EVENT(JS_AFFINITY_SUBMIT_TO_BLOCKED); ++DEFINE_MALI_SLOT_EVENT(JS_AFFINITY_CURRENT); ++DEFINE_MALI_SLOT_EVENT(JD_DONE_TRY_RUN_NEXT_JOB); ++DEFINE_MALI_SLOT_EVENT(JS_CORE_REF_REQUEST_CORES_FAILED); ++DEFINE_MALI_SLOT_EVENT(JS_CORE_REF_REGISTER_INUSE_FAILED); ++DEFINE_MALI_SLOT_EVENT(JS_CORE_REF_REQUEST_ON_RECHECK_FAILED); ++DEFINE_MALI_SLOT_EVENT(JS_CORE_REF_AFFINITY_WOULD_VIOLATE); ++DEFINE_MALI_SLOT_EVENT(JS_JOB_DONE_TRY_RUN_NEXT_JOB); ++DEFINE_MALI_SLOT_EVENT(JS_JOB_DONE_RETRY_NEEDED); ++DEFINE_MALI_SLOT_EVENT(JS_POLICY_DEQUEUE_JOB); ++DEFINE_MALI_SLOT_EVENT(JS_POLICY_DEQUEUE_JOB_IRQ); ++#undef DEFINE_MALI_SLOT_EVENT ++ ++DECLARE_EVENT_CLASS(mali_refcount_template, ++ TP_PROTO(int refcount, unsigned int info_val), ++ TP_ARGS(refcount, info_val), ++ TP_STRUCT__entry( ++ __field(unsigned int, refcount) ++ __field(unsigned int, info_val) ++ ), ++ TP_fast_assign( ++ __entry->refcount = refcount; ++ __entry->info_val = info_val; ++ ), ++ TP_printk("refcount=%u info=%u", __entry->refcount, __entry->info_val) ++); ++ ++#define DEFINE_MALI_REFCOUNT_EVENT(name) \ ++DEFINE_EVENT(mali_refcount_template, mali_##name, \ ++ TP_PROTO(int refcount, unsigned int info_val), \ ++ TP_ARGS(refcount, info_val)) ++DEFINE_MALI_REFCOUNT_EVENT(JS_RETAIN_CTX_NOLOCK); ++DEFINE_MALI_REFCOUNT_EVENT(JS_ADD_JOB); ++DEFINE_MALI_REFCOUNT_EVENT(JS_REMOVE_JOB); ++DEFINE_MALI_REFCOUNT_EVENT(JS_RETAIN_CTX); ++DEFINE_MALI_REFCOUNT_EVENT(JS_RELEASE_CTX); ++DEFINE_MALI_REFCOUNT_EVENT(JS_TRY_SCHEDULE_HEAD_CTX); ++DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_INIT_CTX); ++DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_TERM_CTX); ++DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_ENQUEUE_CTX); ++DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_DEQUEUE_HEAD_CTX); ++DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_TRY_EVICT_CTX); ++DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_RUNPOOL_ADD_CTX); ++DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_RUNPOOL_REMOVE_CTX); ++DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_FOREACH_CTX_JOBS); ++DEFINE_MALI_REFCOUNT_EVENT(PM_CONTEXT_ACTIVE); ++DEFINE_MALI_REFCOUNT_EVENT(PM_CONTEXT_IDLE); ++#undef DEFINE_MALI_REFCOUNT_EVENT ++ ++DECLARE_EVENT_CLASS(mali_add_template, ++ TP_PROTO(int gpu_addr, unsigned int info_val), ++ TP_ARGS(gpu_addr, info_val), ++ TP_STRUCT__entry( ++ __field(unsigned int, gpu_addr) ++ __field(unsigned int, info_val) ++ ), ++ TP_fast_assign( ++ __entry->gpu_addr = gpu_addr; ++ __entry->info_val = info_val; ++ ), ++ TP_printk("gpu_addr=%u info=%u", __entry->gpu_addr, __entry->info_val) ++); ++ ++#define DEFINE_MALI_ADD_EVENT(name) \ ++DEFINE_EVENT(mali_add_template, mali_##name, \ ++ TP_PROTO(int gpu_addr, unsigned int info_val), \ ++ TP_ARGS(gpu_addr, info_val)) ++DEFINE_MALI_ADD_EVENT(CORE_CTX_DESTROY); ++DEFINE_MALI_ADD_EVENT(CORE_CTX_HWINSTR_TERM); ++DEFINE_MALI_ADD_EVENT(CORE_GPU_IRQ); ++DEFINE_MALI_ADD_EVENT(CORE_GPU_IRQ_CLEAR); ++DEFINE_MALI_ADD_EVENT(CORE_GPU_IRQ_DONE); ++DEFINE_MALI_ADD_EVENT(CORE_GPU_SOFT_RESET); ++DEFINE_MALI_ADD_EVENT(CORE_GPU_HARD_RESET); ++DEFINE_MALI_ADD_EVENT(CORE_GPU_PRFCNT_SAMPLE); ++DEFINE_MALI_ADD_EVENT(CORE_GPU_PRFCNT_CLEAR); ++DEFINE_MALI_ADD_EVENT(CORE_GPU_CLEAN_INV_CACHES); ++DEFINE_MALI_ADD_EVENT(JD_DONE_WORKER); ++DEFINE_MALI_ADD_EVENT(JD_DONE_WORKER_END); ++DEFINE_MALI_ADD_EVENT(JD_CANCEL_WORKER); ++DEFINE_MALI_ADD_EVENT(JD_DONE); ++DEFINE_MALI_ADD_EVENT(JD_CANCEL); ++DEFINE_MALI_ADD_EVENT(JD_ZAP_CONTEXT); ++DEFINE_MALI_ADD_EVENT(JM_IRQ); ++DEFINE_MALI_ADD_EVENT(JM_IRQ_END); ++DEFINE_MALI_ADD_EVENT(JM_FLUSH_WORKQS); ++DEFINE_MALI_ADD_EVENT(JM_FLUSH_WORKQS_DONE); ++DEFINE_MALI_ADD_EVENT(JM_ZAP_NON_SCHEDULED); ++DEFINE_MALI_ADD_EVENT(JM_ZAP_SCHEDULED); ++DEFINE_MALI_ADD_EVENT(JM_ZAP_DONE); ++DEFINE_MALI_ADD_EVENT(JM_SUBMIT_AFTER_RESET); ++DEFINE_MALI_ADD_EVENT(JM_JOB_COMPLETE); ++DEFINE_MALI_ADD_EVENT(JS_FAST_START_EVICTS_CTX); ++DEFINE_MALI_ADD_EVENT(JS_CTX_ATTR_NOW_ON_RUNPOOL); ++DEFINE_MALI_ADD_EVENT(JS_CTX_ATTR_NOW_OFF_RUNPOOL); ++DEFINE_MALI_ADD_EVENT(JS_CTX_ATTR_NOW_ON_CTX); ++DEFINE_MALI_ADD_EVENT(JS_CTX_ATTR_NOW_OFF_CTX); ++DEFINE_MALI_ADD_EVENT(JS_POLICY_TIMER_END); ++DEFINE_MALI_ADD_EVENT(JS_POLICY_TIMER_START); ++DEFINE_MALI_ADD_EVENT(JS_POLICY_ENQUEUE_JOB); ++DEFINE_MALI_ADD_EVENT(PM_CORES_CHANGE_DESIRED); ++DEFINE_MALI_ADD_EVENT(PM_JOB_SUBMIT_AFTER_POWERING_UP); ++DEFINE_MALI_ADD_EVENT(PM_JOB_SUBMIT_AFTER_POWERED_UP); ++DEFINE_MALI_ADD_EVENT(PM_PWRON); ++DEFINE_MALI_ADD_EVENT(PM_PWRON_TILER); ++DEFINE_MALI_ADD_EVENT(PM_PWRON_L2); ++DEFINE_MALI_ADD_EVENT(PM_PWROFF); ++DEFINE_MALI_ADD_EVENT(PM_PWROFF_TILER); ++DEFINE_MALI_ADD_EVENT(PM_PWROFF_L2); ++DEFINE_MALI_ADD_EVENT(PM_CORES_POWERED); ++DEFINE_MALI_ADD_EVENT(PM_CORES_POWERED_TILER); ++DEFINE_MALI_ADD_EVENT(PM_CORES_POWERED_L2); ++DEFINE_MALI_ADD_EVENT(PM_DESIRED_REACHED); ++DEFINE_MALI_ADD_EVENT(PM_DESIRED_REACHED_TILER); ++DEFINE_MALI_ADD_EVENT(PM_UNREQUEST_CHANGE_SHADER_NEEDED); ++DEFINE_MALI_ADD_EVENT(PM_REQUEST_CHANGE_SHADER_NEEDED); ++DEFINE_MALI_ADD_EVENT(PM_REGISTER_CHANGE_SHADER_NEEDED); ++DEFINE_MALI_ADD_EVENT(PM_REGISTER_CHANGE_SHADER_INUSE); ++DEFINE_MALI_ADD_EVENT(PM_RELEASE_CHANGE_SHADER_INUSE); ++DEFINE_MALI_ADD_EVENT(PM_CORES_AVAILABLE); ++DEFINE_MALI_ADD_EVENT(PM_CORES_AVAILABLE_TILER); ++DEFINE_MALI_ADD_EVENT(PM_CORES_CHANGE_AVAILABLE); ++DEFINE_MALI_ADD_EVENT(PM_CORES_CHANGE_AVAILABLE_TILER); ++DEFINE_MALI_ADD_EVENT(PM_GPU_ON); ++DEFINE_MALI_ADD_EVENT(PM_GPU_OFF); ++DEFINE_MALI_ADD_EVENT(PM_SET_POLICY); ++DEFINE_MALI_ADD_EVENT(PM_CURRENT_POLICY_INIT); ++DEFINE_MALI_ADD_EVENT(PM_CURRENT_POLICY_TERM); ++DEFINE_MALI_ADD_EVENT(PM_CA_SET_POLICY); ++DEFINE_MALI_ADD_EVENT(PM_WAKE_WAITERS); ++#undef DEFINE_MALI_ADD_EVENT ++ ++#endif /* _TRACE_MALI_KBASE_H */ ++ ++#undef TRACE_INCLUDE_PATH ++#undef linux ++#define TRACE_INCLUDE_PATH . ++#undef TRACE_INCLUDE_FILE ++#define TRACE_INCLUDE_FILE mali_linux_kbase_trace ++ ++/* This part must be outside protection */ ++#include +diff --git a/drivers/gpu/arm/midgard/mali_linux_trace.h b/drivers/gpu/arm/midgard/mali_linux_trace.h +new file mode 100755 +index 000000000000..2be06a552768 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_linux_trace.h +@@ -0,0 +1,189 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#if !defined(_TRACE_MALI_H) || defined(TRACE_HEADER_MULTI_READ) ++#define _TRACE_MALI_H ++ ++#undef TRACE_SYSTEM ++#define TRACE_SYSTEM mali ++#define TRACE_INCLUDE_FILE mali_linux_trace ++ ++#include ++ ++#define MALI_JOB_SLOTS_EVENT_CHANGED ++ ++/** ++ * mali_job_slots_event - called from mali_kbase_core_linux.c ++ * @event_id: ORed together bitfields representing a type of event, made with the GATOR_MAKE_EVENT() macro. ++ */ ++TRACE_EVENT(mali_job_slots_event, ++ TP_PROTO(unsigned int event_id, unsigned int tgid, unsigned int pid, ++ unsigned char job_id), ++ TP_ARGS(event_id, tgid, pid, job_id), ++ TP_STRUCT__entry( ++ __field(unsigned int, event_id) ++ __field(unsigned int, tgid) ++ __field(unsigned int, pid) ++ __field(unsigned char, job_id) ++ ), ++ TP_fast_assign( ++ __entry->event_id = event_id; ++ __entry->tgid = tgid; ++ __entry->pid = pid; ++ __entry->job_id = job_id; ++ ), ++ TP_printk("event=%u tgid=%u pid=%u job_id=%u", ++ __entry->event_id, __entry->tgid, __entry->pid, __entry->job_id) ++); ++ ++/** ++ * mali_pm_status - Called by mali_kbase_pm_driver.c ++ * @event_id: core type (shader, tiler, l2 cache) ++ * @value: 64bits bitmask reporting either power status of the cores (1-ON, 0-OFF) ++ */ ++TRACE_EVENT(mali_pm_status, ++ TP_PROTO(unsigned int event_id, unsigned long long value), ++ TP_ARGS(event_id, value), ++ TP_STRUCT__entry( ++ __field(unsigned int, event_id) ++ __field(unsigned long long, value) ++ ), ++ TP_fast_assign( ++ __entry->event_id = event_id; ++ __entry->value = value; ++ ), ++ TP_printk("event %u = %llu", __entry->event_id, __entry->value) ++); ++ ++/** ++ * mali_pm_power_on - Called by mali_kbase_pm_driver.c ++ * @event_id: core type (shader, tiler, l2 cache) ++ * @value: 64bits bitmask reporting the cores to power up ++ */ ++TRACE_EVENT(mali_pm_power_on, ++ TP_PROTO(unsigned int event_id, unsigned long long value), ++ TP_ARGS(event_id, value), ++ TP_STRUCT__entry( ++ __field(unsigned int, event_id) ++ __field(unsigned long long, value) ++ ), ++ TP_fast_assign( ++ __entry->event_id = event_id; ++ __entry->value = value; ++ ), ++ TP_printk("event %u = %llu", __entry->event_id, __entry->value) ++); ++ ++/** ++ * mali_pm_power_off - Called by mali_kbase_pm_driver.c ++ * @event_id: core type (shader, tiler, l2 cache) ++ * @value: 64bits bitmask reporting the cores to power down ++ */ ++TRACE_EVENT(mali_pm_power_off, ++ TP_PROTO(unsigned int event_id, unsigned long long value), ++ TP_ARGS(event_id, value), ++ TP_STRUCT__entry( ++ __field(unsigned int, event_id) ++ __field(unsigned long long, value) ++ ), ++ TP_fast_assign( ++ __entry->event_id = event_id; ++ __entry->value = value; ++ ), ++ TP_printk("event %u = %llu", __entry->event_id, __entry->value) ++); ++ ++/** ++ * mali_page_fault_insert_pages - Called by page_fault_worker() ++ * it reports an MMU page fault resulting in new pages being mapped. ++ * @event_id: MMU address space number. ++ * @value: number of newly allocated pages ++ */ ++TRACE_EVENT(mali_page_fault_insert_pages, ++ TP_PROTO(int event_id, unsigned long value), ++ TP_ARGS(event_id, value), ++ TP_STRUCT__entry( ++ __field(int, event_id) ++ __field(unsigned long, value) ++ ), ++ TP_fast_assign( ++ __entry->event_id = event_id; ++ __entry->value = value; ++ ), ++ TP_printk("event %d = %lu", __entry->event_id, __entry->value) ++); ++ ++/** ++ * mali_mmu_as_in_use - Called by assign_and_activate_kctx_addr_space() ++ * it reports that a certain MMU address space is in use now. ++ * @event_id: MMU address space number. ++ */ ++TRACE_EVENT(mali_mmu_as_in_use, ++ TP_PROTO(int event_id), ++ TP_ARGS(event_id), ++ TP_STRUCT__entry( ++ __field(int, event_id) ++ ), ++ TP_fast_assign( ++ __entry->event_id = event_id; ++ ), ++ TP_printk("event=%d", __entry->event_id) ++); ++ ++/** ++ * mali_mmu_as_released - Called by kbasep_js_runpool_release_ctx_internal() ++ * it reports that a certain MMU address space has been released now. ++ * @event_id: MMU address space number. ++ */ ++TRACE_EVENT(mali_mmu_as_released, ++ TP_PROTO(int event_id), ++ TP_ARGS(event_id), ++ TP_STRUCT__entry( ++ __field(int, event_id) ++ ), ++ TP_fast_assign( ++ __entry->event_id = event_id; ++ ), ++ TP_printk("event=%d", __entry->event_id) ++); ++ ++/** ++ * mali_total_alloc_pages_change - Called by kbase_atomic_add_pages() ++ * and by kbase_atomic_sub_pages() ++ * it reports that the total number of allocated pages is changed. ++ * @event_id: number of pages to be added or subtracted (according to the sign). ++ */ ++TRACE_EVENT(mali_total_alloc_pages_change, ++ TP_PROTO(long long int event_id), ++ TP_ARGS(event_id), ++ TP_STRUCT__entry( ++ __field(long long int, event_id) ++ ), ++ TP_fast_assign( ++ __entry->event_id = event_id; ++ ), ++ TP_printk("event=%lld", __entry->event_id) ++); ++ ++#endif /* _TRACE_MALI_H */ ++ ++#undef TRACE_INCLUDE_PATH ++#undef linux ++#define TRACE_INCLUDE_PATH . ++ ++/* This part must be outside protection */ ++#include +diff --git a/drivers/gpu/arm/midgard/mali_malisw.h b/drivers/gpu/arm/midgard/mali_malisw.h +new file mode 100755 +index 000000000000..99452933eab4 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_malisw.h +@@ -0,0 +1,131 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/** ++ * Kernel-wide include for common macros and types. ++ */ ++ ++#ifndef _MALISW_H_ ++#define _MALISW_H_ ++ ++#include ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 0) ++#define U8_MAX ((u8)~0U) ++#define S8_MAX ((s8)(U8_MAX>>1)) ++#define S8_MIN ((s8)(-S8_MAX - 1)) ++#define U16_MAX ((u16)~0U) ++#define S16_MAX ((s16)(U16_MAX>>1)) ++#define S16_MIN ((s16)(-S16_MAX - 1)) ++#define U32_MAX ((u32)~0U) ++#define S32_MAX ((s32)(U32_MAX>>1)) ++#define S32_MIN ((s32)(-S32_MAX - 1)) ++#define U64_MAX ((u64)~0ULL) ++#define S64_MAX ((s64)(U64_MAX>>1)) ++#define S64_MIN ((s64)(-S64_MAX - 1)) ++#endif /* LINUX_VERSION_CODE */ ++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 5, 0) ++#define SIZE_MAX (~(size_t)0) ++#endif /* LINUX_VERSION_CODE */ ++ ++/** ++ * MIN - Return the lesser of two values. ++ * ++ * As a macro it may evaluate its arguments more than once. ++ * Refer to MAX macro for more details ++ */ ++#define MIN(x, y) ((x) < (y) ? (x) : (y)) ++ ++/** ++ * MAX - Return the greater of two values. ++ * ++ * As a macro it may evaluate its arguments more than once. ++ * If called on the same two arguments as MIN it is guaranteed to return ++ * the one that MIN didn't return. This is significant for types where not ++ * all values are comparable e.g. NaNs in floating-point types. But if you want ++ * to retrieve the min and max of two values, consider using a conditional swap ++ * instead. ++ */ ++#define MAX(x, y) ((x) < (y) ? (y) : (x)) ++ ++/** ++ * @hideinitializer ++ * Function-like macro for suppressing unused variable warnings. Where possible ++ * such variables should be removed; this macro is present for cases where we ++ * much support API backwards compatibility. ++ */ ++#define CSTD_UNUSED(x) ((void)(x)) ++ ++/** ++ * @hideinitializer ++ * Function-like macro for use where "no behavior" is desired. This is useful ++ * when compile time macros turn a function-like macro in to a no-op, but ++ * where having no statement is otherwise invalid. ++ */ ++#define CSTD_NOP(...) ((void)#__VA_ARGS__) ++ ++/** ++ * Function-like macro for converting a pointer in to a u64 for storing into ++ * an external data structure. This is commonly used when pairing a 32-bit ++ * CPU with a 64-bit peripheral, such as a Midgard GPU. C's type promotion ++ * is complex and a straight cast does not work reliably as pointers are ++ * often considered as signed. ++ */ ++#define PTR_TO_U64(x) ((uint64_t)((uintptr_t)(x))) ++ ++/** ++ * @hideinitializer ++ * Function-like macro for stringizing a single level macro. ++ * @code ++ * #define MY_MACRO 32 ++ * CSTD_STR1( MY_MACRO ) ++ * > "MY_MACRO" ++ * @endcode ++ */ ++#define CSTD_STR1(x) #x ++ ++/** ++ * @hideinitializer ++ * Function-like macro for stringizing a macro's value. This should not be used ++ * if the macro is defined in a way which may have no value; use the ++ * alternative @c CSTD_STR2N macro should be used instead. ++ * @code ++ * #define MY_MACRO 32 ++ * CSTD_STR2( MY_MACRO ) ++ * > "32" ++ * @endcode ++ */ ++#define CSTD_STR2(x) CSTD_STR1(x) ++ ++/** ++ * Specify an assertion value which is evaluated at compile time. Recommended ++ * usage is specification of a @c static @c INLINE function containing all of ++ * the assertions thus: ++ * ++ * @code ++ * static INLINE [module]_compile_time_assertions( void ) ++ * { ++ * COMPILE_TIME_ASSERT( sizeof(uintptr_t) == sizeof(intptr_t) ); ++ * } ++ * @endcode ++ * ++ * @note Use @c static not @c STATIC. We never want to turn off this @c static ++ * specification for testing purposes. ++ */ ++#define CSTD_COMPILE_TIME_ASSERT(expr) \ ++ do { switch (0) { case 0: case (expr):; } } while (false) ++ ++#endif /* _MALISW_H_ */ +diff --git a/drivers/gpu/arm/midgard/mali_midg_coherency.h b/drivers/gpu/arm/midgard/mali_midg_coherency.h +new file mode 100755 +index 000000000000..a509cbd5f175 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_midg_coherency.h +@@ -0,0 +1,26 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _MIDG_COHERENCY_H_ ++#define _MIDG_COHERENCY_H_ ++ ++#define COHERENCY_ACE_LITE 0 ++#define COHERENCY_ACE 1 ++#define COHERENCY_NONE 31 ++#define COHERENCY_FEATURE_BIT(x) (1 << (x)) ++ ++#endif /* _MIDG_COHERENCY_H_ */ +diff --git a/drivers/gpu/arm/midgard/mali_midg_regmap.h b/drivers/gpu/arm/midgard/mali_midg_regmap.h +new file mode 100755 +index 000000000000..7d7b7bcd3cc3 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_midg_regmap.h +@@ -0,0 +1,611 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _MIDGARD_REGMAP_H_ ++#define _MIDGARD_REGMAP_H_ ++ ++#include "mali_midg_coherency.h" ++#include "mali_kbase_gpu_id.h" ++ ++/* ++ * Begin Register Offsets ++ */ ++ ++#define GPU_CONTROL_BASE 0x0000 ++#define GPU_CONTROL_REG(r) (GPU_CONTROL_BASE + (r)) ++#define GPU_ID 0x000 /* (RO) GPU and revision identifier */ ++#define L2_FEATURES 0x004 /* (RO) Level 2 cache features */ ++#define SUSPEND_SIZE 0x008 /* (RO) Fixed-function suspend buffer ++ size */ ++#define TILER_FEATURES 0x00C /* (RO) Tiler Features */ ++#define MEM_FEATURES 0x010 /* (RO) Memory system features */ ++#define MMU_FEATURES 0x014 /* (RO) MMU features */ ++#define AS_PRESENT 0x018 /* (RO) Address space slots present */ ++#define JS_PRESENT 0x01C /* (RO) Job slots present */ ++#define GPU_IRQ_RAWSTAT 0x020 /* (RW) */ ++#define GPU_IRQ_CLEAR 0x024 /* (WO) */ ++#define GPU_IRQ_MASK 0x028 /* (RW) */ ++#define GPU_IRQ_STATUS 0x02C /* (RO) */ ++ ++/* IRQ flags */ ++#define GPU_FAULT (1 << 0) /* A GPU Fault has occurred */ ++#define MULTIPLE_GPU_FAULTS (1 << 7) /* More than one GPU Fault occurred. */ ++#define RESET_COMPLETED (1 << 8) /* Set when a reset has completed. Intended to use with SOFT_RESET ++ commands which may take time. */ ++#define POWER_CHANGED_SINGLE (1 << 9) /* Set when a single core has finished powering up or down. */ ++#define POWER_CHANGED_ALL (1 << 10) /* Set when all cores have finished powering up or down ++ and the power manager is idle. */ ++ ++#define PRFCNT_SAMPLE_COMPLETED (1 << 16) /* Set when a performance count sample has completed. */ ++#define CLEAN_CACHES_COMPLETED (1 << 17) /* Set when a cache clean operation has completed. */ ++ ++#define GPU_IRQ_REG_ALL (GPU_FAULT | MULTIPLE_GPU_FAULTS | RESET_COMPLETED \ ++ | POWER_CHANGED_ALL | PRFCNT_SAMPLE_COMPLETED) ++ ++#define GPU_COMMAND 0x030 /* (WO) */ ++#define GPU_STATUS 0x034 /* (RO) */ ++#define LATEST_FLUSH 0x038 /* (RO) */ ++ ++#define GROUPS_L2_COHERENT (1 << 0) /* Cores groups are l2 coherent */ ++#define GPU_DBGEN (1 << 8) /* DBGEN wire status */ ++ ++#define GPU_FAULTSTATUS 0x03C /* (RO) GPU exception type and fault status */ ++#define GPU_FAULTADDRESS_LO 0x040 /* (RO) GPU exception fault address, low word */ ++#define GPU_FAULTADDRESS_HI 0x044 /* (RO) GPU exception fault address, high word */ ++ ++#define PWR_KEY 0x050 /* (WO) Power manager key register */ ++#define PWR_OVERRIDE0 0x054 /* (RW) Power manager override settings */ ++#define PWR_OVERRIDE1 0x058 /* (RW) Power manager override settings */ ++ ++#define PRFCNT_BASE_LO 0x060 /* (RW) Performance counter memory region base address, low word */ ++#define PRFCNT_BASE_HI 0x064 /* (RW) Performance counter memory region base address, high word */ ++#define PRFCNT_CONFIG 0x068 /* (RW) Performance counter configuration */ ++#define PRFCNT_JM_EN 0x06C /* (RW) Performance counter enable flags for Job Manager */ ++#define PRFCNT_SHADER_EN 0x070 /* (RW) Performance counter enable flags for shader cores */ ++#define PRFCNT_TILER_EN 0x074 /* (RW) Performance counter enable flags for tiler */ ++#define PRFCNT_MMU_L2_EN 0x07C /* (RW) Performance counter enable flags for MMU/L2 cache */ ++ ++#define CYCLE_COUNT_LO 0x090 /* (RO) Cycle counter, low word */ ++#define CYCLE_COUNT_HI 0x094 /* (RO) Cycle counter, high word */ ++#define TIMESTAMP_LO 0x098 /* (RO) Global time stamp counter, low word */ ++#define TIMESTAMP_HI 0x09C /* (RO) Global time stamp counter, high word */ ++ ++#define THREAD_MAX_THREADS 0x0A0 /* (RO) Maximum number of threads per core */ ++#define THREAD_MAX_WORKGROUP_SIZE 0x0A4 /* (RO) Maximum workgroup size */ ++#define THREAD_MAX_BARRIER_SIZE 0x0A8 /* (RO) Maximum threads waiting at a barrier */ ++#define THREAD_FEATURES 0x0AC /* (RO) Thread features */ ++ ++#define TEXTURE_FEATURES_0 0x0B0 /* (RO) Support flags for indexed texture formats 0..31 */ ++#define TEXTURE_FEATURES_1 0x0B4 /* (RO) Support flags for indexed texture formats 32..63 */ ++#define TEXTURE_FEATURES_2 0x0B8 /* (RO) Support flags for indexed texture formats 64..95 */ ++ ++#define TEXTURE_FEATURES_REG(n) GPU_CONTROL_REG(TEXTURE_FEATURES_0 + ((n) << 2)) ++ ++#define JS0_FEATURES 0x0C0 /* (RO) Features of job slot 0 */ ++#define JS1_FEATURES 0x0C4 /* (RO) Features of job slot 1 */ ++#define JS2_FEATURES 0x0C8 /* (RO) Features of job slot 2 */ ++#define JS3_FEATURES 0x0CC /* (RO) Features of job slot 3 */ ++#define JS4_FEATURES 0x0D0 /* (RO) Features of job slot 4 */ ++#define JS5_FEATURES 0x0D4 /* (RO) Features of job slot 5 */ ++#define JS6_FEATURES 0x0D8 /* (RO) Features of job slot 6 */ ++#define JS7_FEATURES 0x0DC /* (RO) Features of job slot 7 */ ++#define JS8_FEATURES 0x0E0 /* (RO) Features of job slot 8 */ ++#define JS9_FEATURES 0x0E4 /* (RO) Features of job slot 9 */ ++#define JS10_FEATURES 0x0E8 /* (RO) Features of job slot 10 */ ++#define JS11_FEATURES 0x0EC /* (RO) Features of job slot 11 */ ++#define JS12_FEATURES 0x0F0 /* (RO) Features of job slot 12 */ ++#define JS13_FEATURES 0x0F4 /* (RO) Features of job slot 13 */ ++#define JS14_FEATURES 0x0F8 /* (RO) Features of job slot 14 */ ++#define JS15_FEATURES 0x0FC /* (RO) Features of job slot 15 */ ++ ++#define JS_FEATURES_REG(n) GPU_CONTROL_REG(JS0_FEATURES + ((n) << 2)) ++ ++#define SHADER_PRESENT_LO 0x100 /* (RO) Shader core present bitmap, low word */ ++#define SHADER_PRESENT_HI 0x104 /* (RO) Shader core present bitmap, high word */ ++ ++#define TILER_PRESENT_LO 0x110 /* (RO) Tiler core present bitmap, low word */ ++#define TILER_PRESENT_HI 0x114 /* (RO) Tiler core present bitmap, high word */ ++ ++#define L2_PRESENT_LO 0x120 /* (RO) Level 2 cache present bitmap, low word */ ++#define L2_PRESENT_HI 0x124 /* (RO) Level 2 cache present bitmap, high word */ ++ ++#define STACK_PRESENT_LO 0xE00 /* (RO) Core stack present bitmap, low word */ ++#define STACK_PRESENT_HI 0xE04 /* (RO) Core stack present bitmap, high word */ ++ ++ ++#define SHADER_READY_LO 0x140 /* (RO) Shader core ready bitmap, low word */ ++#define SHADER_READY_HI 0x144 /* (RO) Shader core ready bitmap, high word */ ++ ++#define TILER_READY_LO 0x150 /* (RO) Tiler core ready bitmap, low word */ ++#define TILER_READY_HI 0x154 /* (RO) Tiler core ready bitmap, high word */ ++ ++#define L2_READY_LO 0x160 /* (RO) Level 2 cache ready bitmap, low word */ ++#define L2_READY_HI 0x164 /* (RO) Level 2 cache ready bitmap, high word */ ++ ++#define STACK_READY_LO 0xE10 /* (RO) Core stack ready bitmap, low word */ ++#define STACK_READY_HI 0xE14 /* (RO) Core stack ready bitmap, high word */ ++ ++ ++#define SHADER_PWRON_LO 0x180 /* (WO) Shader core power on bitmap, low word */ ++#define SHADER_PWRON_HI 0x184 /* (WO) Shader core power on bitmap, high word */ ++ ++#define TILER_PWRON_LO 0x190 /* (WO) Tiler core power on bitmap, low word */ ++#define TILER_PWRON_HI 0x194 /* (WO) Tiler core power on bitmap, high word */ ++ ++#define L2_PWRON_LO 0x1A0 /* (WO) Level 2 cache power on bitmap, low word */ ++#define L2_PWRON_HI 0x1A4 /* (WO) Level 2 cache power on bitmap, high word */ ++ ++#define STACK_PWRON_LO 0xE20 /* (RO) Core stack power on bitmap, low word */ ++#define STACK_PWRON_HI 0xE24 /* (RO) Core stack power on bitmap, high word */ ++ ++ ++#define SHADER_PWROFF_LO 0x1C0 /* (WO) Shader core power off bitmap, low word */ ++#define SHADER_PWROFF_HI 0x1C4 /* (WO) Shader core power off bitmap, high word */ ++ ++#define TILER_PWROFF_LO 0x1D0 /* (WO) Tiler core power off bitmap, low word */ ++#define TILER_PWROFF_HI 0x1D4 /* (WO) Tiler core power off bitmap, high word */ ++ ++#define L2_PWROFF_LO 0x1E0 /* (WO) Level 2 cache power off bitmap, low word */ ++#define L2_PWROFF_HI 0x1E4 /* (WO) Level 2 cache power off bitmap, high word */ ++ ++#define STACK_PWROFF_LO 0xE30 /* (RO) Core stack power off bitmap, low word */ ++#define STACK_PRWOFF_HI 0xE34 /* (RO) Core stack power off bitmap, high word */ ++ ++ ++#define SHADER_PWRTRANS_LO 0x200 /* (RO) Shader core power transition bitmap, low word */ ++#define SHADER_PWRTRANS_HI 0x204 /* (RO) Shader core power transition bitmap, high word */ ++ ++#define TILER_PWRTRANS_LO 0x210 /* (RO) Tiler core power transition bitmap, low word */ ++#define TILER_PWRTRANS_HI 0x214 /* (RO) Tiler core power transition bitmap, high word */ ++ ++#define L2_PWRTRANS_LO 0x220 /* (RO) Level 2 cache power transition bitmap, low word */ ++#define L2_PWRTRANS_HI 0x224 /* (RO) Level 2 cache power transition bitmap, high word */ ++ ++#define STACK_PWRTRANS_LO 0xE40 /* (RO) Core stack power transition bitmap, low word */ ++#define STACK_PRWTRANS_HI 0xE44 /* (RO) Core stack power transition bitmap, high word */ ++ ++ ++#define SHADER_PWRACTIVE_LO 0x240 /* (RO) Shader core active bitmap, low word */ ++#define SHADER_PWRACTIVE_HI 0x244 /* (RO) Shader core active bitmap, high word */ ++ ++#define TILER_PWRACTIVE_LO 0x250 /* (RO) Tiler core active bitmap, low word */ ++#define TILER_PWRACTIVE_HI 0x254 /* (RO) Tiler core active bitmap, high word */ ++ ++#define L2_PWRACTIVE_LO 0x260 /* (RO) Level 2 cache active bitmap, low word */ ++#define L2_PWRACTIVE_HI 0x264 /* (RO) Level 2 cache active bitmap, high word */ ++ ++#define COHERENCY_FEATURES 0x300 /* (RO) Coherency features present */ ++#define COHERENCY_ENABLE 0x304 /* (RW) Coherency enable */ ++ ++#define JM_CONFIG 0xF00 /* (RW) Job Manager configuration register (Implementation specific register) */ ++#define SHADER_CONFIG 0xF04 /* (RW) Shader core configuration settings (Implementation specific register) */ ++#define TILER_CONFIG 0xF08 /* (RW) Tiler core configuration settings (Implementation specific register) */ ++#define L2_MMU_CONFIG 0xF0C /* (RW) Configuration of the L2 cache and MMU (Implementation specific register) */ ++ ++#define JOB_CONTROL_BASE 0x1000 ++ ++#define JOB_CONTROL_REG(r) (JOB_CONTROL_BASE + (r)) ++ ++#define JOB_IRQ_RAWSTAT 0x000 /* Raw interrupt status register */ ++#define JOB_IRQ_CLEAR 0x004 /* Interrupt clear register */ ++#define JOB_IRQ_MASK 0x008 /* Interrupt mask register */ ++#define JOB_IRQ_STATUS 0x00C /* Interrupt status register */ ++#define JOB_IRQ_JS_STATE 0x010 /* status==active and _next == busy snapshot from last JOB_IRQ_CLEAR */ ++#define JOB_IRQ_THROTTLE 0x014 /* cycles to delay delivering an interrupt externally. The JOB_IRQ_STATUS is NOT affected by this, just the delivery of the interrupt. */ ++ ++#define JOB_SLOT0 0x800 /* Configuration registers for job slot 0 */ ++#define JOB_SLOT1 0x880 /* Configuration registers for job slot 1 */ ++#define JOB_SLOT2 0x900 /* Configuration registers for job slot 2 */ ++#define JOB_SLOT3 0x980 /* Configuration registers for job slot 3 */ ++#define JOB_SLOT4 0xA00 /* Configuration registers for job slot 4 */ ++#define JOB_SLOT5 0xA80 /* Configuration registers for job slot 5 */ ++#define JOB_SLOT6 0xB00 /* Configuration registers for job slot 6 */ ++#define JOB_SLOT7 0xB80 /* Configuration registers for job slot 7 */ ++#define JOB_SLOT8 0xC00 /* Configuration registers for job slot 8 */ ++#define JOB_SLOT9 0xC80 /* Configuration registers for job slot 9 */ ++#define JOB_SLOT10 0xD00 /* Configuration registers for job slot 10 */ ++#define JOB_SLOT11 0xD80 /* Configuration registers for job slot 11 */ ++#define JOB_SLOT12 0xE00 /* Configuration registers for job slot 12 */ ++#define JOB_SLOT13 0xE80 /* Configuration registers for job slot 13 */ ++#define JOB_SLOT14 0xF00 /* Configuration registers for job slot 14 */ ++#define JOB_SLOT15 0xF80 /* Configuration registers for job slot 15 */ ++ ++#define JOB_SLOT_REG(n, r) (JOB_CONTROL_REG(JOB_SLOT0 + ((n) << 7)) + (r)) ++ ++#define JS_HEAD_LO 0x00 /* (RO) Job queue head pointer for job slot n, low word */ ++#define JS_HEAD_HI 0x04 /* (RO) Job queue head pointer for job slot n, high word */ ++#define JS_TAIL_LO 0x08 /* (RO) Job queue tail pointer for job slot n, low word */ ++#define JS_TAIL_HI 0x0C /* (RO) Job queue tail pointer for job slot n, high word */ ++#define JS_AFFINITY_LO 0x10 /* (RO) Core affinity mask for job slot n, low word */ ++#define JS_AFFINITY_HI 0x14 /* (RO) Core affinity mask for job slot n, high word */ ++#define JS_CONFIG 0x18 /* (RO) Configuration settings for job slot n */ ++#define JS_XAFFINITY 0x1C /* (RO) Extended affinity mask for job ++ slot n */ ++ ++#define JS_COMMAND 0x20 /* (WO) Command register for job slot n */ ++#define JS_STATUS 0x24 /* (RO) Status register for job slot n */ ++ ++#define JS_HEAD_NEXT_LO 0x40 /* (RW) Next job queue head pointer for job slot n, low word */ ++#define JS_HEAD_NEXT_HI 0x44 /* (RW) Next job queue head pointer for job slot n, high word */ ++ ++#define JS_AFFINITY_NEXT_LO 0x50 /* (RW) Next core affinity mask for job slot n, low word */ ++#define JS_AFFINITY_NEXT_HI 0x54 /* (RW) Next core affinity mask for job slot n, high word */ ++#define JS_CONFIG_NEXT 0x58 /* (RW) Next configuration settings for job slot n */ ++#define JS_XAFFINITY_NEXT 0x5C /* (RW) Next extended affinity mask for ++ job slot n */ ++ ++#define JS_COMMAND_NEXT 0x60 /* (RW) Next command register for job slot n */ ++ ++#define JS_FLUSH_ID_NEXT 0x70 /* (RW) Next job slot n cache flush ID */ ++ ++#define MEMORY_MANAGEMENT_BASE 0x2000 ++#define MMU_REG(r) (MEMORY_MANAGEMENT_BASE + (r)) ++ ++#define MMU_IRQ_RAWSTAT 0x000 /* (RW) Raw interrupt status register */ ++#define MMU_IRQ_CLEAR 0x004 /* (WO) Interrupt clear register */ ++#define MMU_IRQ_MASK 0x008 /* (RW) Interrupt mask register */ ++#define MMU_IRQ_STATUS 0x00C /* (RO) Interrupt status register */ ++ ++#define MMU_AS0 0x400 /* Configuration registers for address space 0 */ ++#define MMU_AS1 0x440 /* Configuration registers for address space 1 */ ++#define MMU_AS2 0x480 /* Configuration registers for address space 2 */ ++#define MMU_AS3 0x4C0 /* Configuration registers for address space 3 */ ++#define MMU_AS4 0x500 /* Configuration registers for address space 4 */ ++#define MMU_AS5 0x540 /* Configuration registers for address space 5 */ ++#define MMU_AS6 0x580 /* Configuration registers for address space 6 */ ++#define MMU_AS7 0x5C0 /* Configuration registers for address space 7 */ ++#define MMU_AS8 0x600 /* Configuration registers for address space 8 */ ++#define MMU_AS9 0x640 /* Configuration registers for address space 9 */ ++#define MMU_AS10 0x680 /* Configuration registers for address space 10 */ ++#define MMU_AS11 0x6C0 /* Configuration registers for address space 11 */ ++#define MMU_AS12 0x700 /* Configuration registers for address space 12 */ ++#define MMU_AS13 0x740 /* Configuration registers for address space 13 */ ++#define MMU_AS14 0x780 /* Configuration registers for address space 14 */ ++#define MMU_AS15 0x7C0 /* Configuration registers for address space 15 */ ++ ++#define MMU_AS_REG(n, r) (MMU_REG(MMU_AS0 + ((n) << 6)) + (r)) ++ ++#define AS_TRANSTAB_LO 0x00 /* (RW) Translation Table Base Address for address space n, low word */ ++#define AS_TRANSTAB_HI 0x04 /* (RW) Translation Table Base Address for address space n, high word */ ++#define AS_MEMATTR_LO 0x08 /* (RW) Memory attributes for address space n, low word. */ ++#define AS_MEMATTR_HI 0x0C /* (RW) Memory attributes for address space n, high word. */ ++#define AS_LOCKADDR_LO 0x10 /* (RW) Lock region address for address space n, low word */ ++#define AS_LOCKADDR_HI 0x14 /* (RW) Lock region address for address space n, high word */ ++#define AS_COMMAND 0x18 /* (WO) MMU command register for address space n */ ++#define AS_FAULTSTATUS 0x1C /* (RO) MMU fault status register for address space n */ ++#define AS_FAULTADDRESS_LO 0x20 /* (RO) Fault Address for address space n, low word */ ++#define AS_FAULTADDRESS_HI 0x24 /* (RO) Fault Address for address space n, high word */ ++#define AS_STATUS 0x28 /* (RO) Status flags for address space n */ ++ ++ ++/* (RW) Translation table configuration for address space n, low word */ ++#define AS_TRANSCFG_LO 0x30 ++/* (RW) Translation table configuration for address space n, high word */ ++#define AS_TRANSCFG_HI 0x34 ++/* (RO) Secondary fault address for address space n, low word */ ++#define AS_FAULTEXTRA_LO 0x38 ++/* (RO) Secondary fault address for address space n, high word */ ++#define AS_FAULTEXTRA_HI 0x3C ++ ++/* End Register Offsets */ ++ ++/* ++ * MMU_IRQ_RAWSTAT register values. Values are valid also for ++ MMU_IRQ_CLEAR, MMU_IRQ_MASK, MMU_IRQ_STATUS registers. ++ */ ++ ++#define MMU_PAGE_FAULT_FLAGS 16 ++ ++/* Macros returning a bitmask to retrieve page fault or bus error flags from ++ * MMU registers */ ++#define MMU_PAGE_FAULT(n) (1UL << (n)) ++#define MMU_BUS_ERROR(n) (1UL << ((n) + MMU_PAGE_FAULT_FLAGS)) ++ ++/* ++ * Begin LPAE MMU TRANSTAB register values ++ */ ++#define AS_TRANSTAB_LPAE_ADDR_SPACE_MASK 0xfffff000 ++#define AS_TRANSTAB_LPAE_ADRMODE_UNMAPPED (0u << 0) ++#define AS_TRANSTAB_LPAE_ADRMODE_IDENTITY (1u << 1) ++#define AS_TRANSTAB_LPAE_ADRMODE_TABLE (3u << 0) ++#define AS_TRANSTAB_LPAE_READ_INNER (1u << 2) ++#define AS_TRANSTAB_LPAE_SHARE_OUTER (1u << 4) ++ ++#define AS_TRANSTAB_LPAE_ADRMODE_MASK 0x00000003 ++ ++/* ++ * Begin AARCH64 MMU TRANSTAB register values ++ */ ++#define MMU_HW_OUTA_BITS 40 ++#define AS_TRANSTAB_BASE_MASK ((1ULL << MMU_HW_OUTA_BITS) - (1ULL << 4)) ++ ++/* ++ * Begin MMU STATUS register values ++ */ ++#define AS_STATUS_AS_ACTIVE 0x01 ++ ++#define AS_FAULTSTATUS_EXCEPTION_CODE_MASK (0x7<<3) ++#define AS_FAULTSTATUS_EXCEPTION_CODE_TRANSLATION_FAULT (0x0<<3) ++#define AS_FAULTSTATUS_EXCEPTION_CODE_PERMISSION_FAULT (0x1<<3) ++#define AS_FAULTSTATUS_EXCEPTION_CODE_TRANSTAB_BUS_FAULT (0x2<<3) ++#define AS_FAULTSTATUS_EXCEPTION_CODE_ACCESS_FLAG (0x3<<3) ++ ++#define AS_FAULTSTATUS_EXCEPTION_CODE_ADDRESS_SIZE_FAULT (0x4<<3) ++#define AS_FAULTSTATUS_EXCEPTION_CODE_MEMORY_ATTRIBUTES_FAULT (0x5<<3) ++ ++#define AS_FAULTSTATUS_ACCESS_TYPE_MASK (0x3<<8) ++#define AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC (0x0<<8) ++#define AS_FAULTSTATUS_ACCESS_TYPE_EX (0x1<<8) ++#define AS_FAULTSTATUS_ACCESS_TYPE_READ (0x2<<8) ++#define AS_FAULTSTATUS_ACCESS_TYPE_WRITE (0x3<<8) ++ ++/* ++ * Begin MMU TRANSCFG register values ++ */ ++ ++#define AS_TRANSCFG_ADRMODE_LEGACY 0 ++#define AS_TRANSCFG_ADRMODE_UNMAPPED 1 ++#define AS_TRANSCFG_ADRMODE_IDENTITY 2 ++#define AS_TRANSCFG_ADRMODE_AARCH64_4K 6 ++#define AS_TRANSCFG_ADRMODE_AARCH64_64K 8 ++ ++#define AS_TRANSCFG_ADRMODE_MASK 0xF ++ ++ ++/* ++ * Begin TRANSCFG register values ++ */ ++#define AS_TRANSCFG_PTW_MEMATTR_MASK (3 << 24) ++#define AS_TRANSCFG_PTW_MEMATTR_NON_CACHEABLE (1 << 24) ++#define AS_TRANSCFG_PTW_MEMATTR_WRITE_BACK (2 << 24) ++ ++#define AS_TRANSCFG_PTW_SH_MASK ((3 << 28)) ++#define AS_TRANSCFG_PTW_SH_OS (2 << 28) ++#define AS_TRANSCFG_PTW_SH_IS (3 << 28) ++ ++/* ++ * Begin Command Values ++ */ ++ ++/* JS_COMMAND register commands */ ++#define JS_COMMAND_NOP 0x00 /* NOP Operation. Writing this value is ignored */ ++#define JS_COMMAND_START 0x01 /* Start processing a job chain. Writing this value is ignored */ ++#define JS_COMMAND_SOFT_STOP 0x02 /* Gently stop processing a job chain */ ++#define JS_COMMAND_HARD_STOP 0x03 /* Rudely stop processing a job chain */ ++#define JS_COMMAND_SOFT_STOP_0 0x04 /* Execute SOFT_STOP if JOB_CHAIN_FLAG is 0 */ ++#define JS_COMMAND_HARD_STOP_0 0x05 /* Execute HARD_STOP if JOB_CHAIN_FLAG is 0 */ ++#define JS_COMMAND_SOFT_STOP_1 0x06 /* Execute SOFT_STOP if JOB_CHAIN_FLAG is 1 */ ++#define JS_COMMAND_HARD_STOP_1 0x07 /* Execute HARD_STOP if JOB_CHAIN_FLAG is 1 */ ++ ++#define JS_COMMAND_MASK 0x07 /* Mask of bits currently in use by the HW */ ++ ++/* AS_COMMAND register commands */ ++#define AS_COMMAND_NOP 0x00 /* NOP Operation */ ++#define AS_COMMAND_UPDATE 0x01 /* Broadcasts the values in AS_TRANSTAB and ASn_MEMATTR to all MMUs */ ++#define AS_COMMAND_LOCK 0x02 /* Issue a lock region command to all MMUs */ ++#define AS_COMMAND_UNLOCK 0x03 /* Issue a flush region command to all MMUs */ ++#define AS_COMMAND_FLUSH 0x04 /* Flush all L2 caches then issue a flush region command to all MMUs ++ (deprecated - only for use with T60x) */ ++#define AS_COMMAND_FLUSH_PT 0x04 /* Flush all L2 caches then issue a flush region command to all MMUs */ ++#define AS_COMMAND_FLUSH_MEM 0x05 /* Wait for memory accesses to complete, flush all the L1s cache then ++ flush all L2 caches then issue a flush region command to all MMUs */ ++ ++/* Possible values of JS_CONFIG and JS_CONFIG_NEXT registers */ ++#define JS_CONFIG_START_FLUSH_NO_ACTION (0u << 0) ++#define JS_CONFIG_START_FLUSH_CLEAN (1u << 8) ++#define JS_CONFIG_START_FLUSH_CLEAN_INVALIDATE (3u << 8) ++#define JS_CONFIG_START_MMU (1u << 10) ++#define JS_CONFIG_JOB_CHAIN_FLAG (1u << 11) ++#define JS_CONFIG_END_FLUSH_NO_ACTION JS_CONFIG_START_FLUSH_NO_ACTION ++#define JS_CONFIG_END_FLUSH_CLEAN (1u << 12) ++#define JS_CONFIG_END_FLUSH_CLEAN_INVALIDATE (3u << 12) ++#define JS_CONFIG_ENABLE_FLUSH_REDUCTION (1u << 14) ++#define JS_CONFIG_DISABLE_DESCRIPTOR_WR_BK (1u << 15) ++#define JS_CONFIG_THREAD_PRI(n) ((n) << 16) ++ ++/* JS_XAFFINITY register values */ ++#define JS_XAFFINITY_XAFFINITY_ENABLE (1u << 0) ++#define JS_XAFFINITY_TILER_ENABLE (1u << 8) ++#define JS_XAFFINITY_CACHE_ENABLE (1u << 16) ++ ++/* JS_STATUS register values */ ++ ++/* NOTE: Please keep this values in sync with enum base_jd_event_code in mali_base_kernel.h. ++ * The values are separated to avoid dependency of userspace and kernel code. ++ */ ++ ++/* Group of values representing the job status insead a particular fault */ ++#define JS_STATUS_NO_EXCEPTION_BASE 0x00 ++#define JS_STATUS_INTERRUPTED (JS_STATUS_NO_EXCEPTION_BASE + 0x02) /* 0x02 means INTERRUPTED */ ++#define JS_STATUS_STOPPED (JS_STATUS_NO_EXCEPTION_BASE + 0x03) /* 0x03 means STOPPED */ ++#define JS_STATUS_TERMINATED (JS_STATUS_NO_EXCEPTION_BASE + 0x04) /* 0x04 means TERMINATED */ ++ ++/* General fault values */ ++#define JS_STATUS_FAULT_BASE 0x40 ++#define JS_STATUS_CONFIG_FAULT (JS_STATUS_FAULT_BASE) /* 0x40 means CONFIG FAULT */ ++#define JS_STATUS_POWER_FAULT (JS_STATUS_FAULT_BASE + 0x01) /* 0x41 means POWER FAULT */ ++#define JS_STATUS_READ_FAULT (JS_STATUS_FAULT_BASE + 0x02) /* 0x42 means READ FAULT */ ++#define JS_STATUS_WRITE_FAULT (JS_STATUS_FAULT_BASE + 0x03) /* 0x43 means WRITE FAULT */ ++#define JS_STATUS_AFFINITY_FAULT (JS_STATUS_FAULT_BASE + 0x04) /* 0x44 means AFFINITY FAULT */ ++#define JS_STATUS_BUS_FAULT (JS_STATUS_FAULT_BASE + 0x08) /* 0x48 means BUS FAULT */ ++ ++/* Instruction or data faults */ ++#define JS_STATUS_INSTRUCTION_FAULT_BASE 0x50 ++#define JS_STATUS_INSTR_INVALID_PC (JS_STATUS_INSTRUCTION_FAULT_BASE) /* 0x50 means INSTR INVALID PC */ ++#define JS_STATUS_INSTR_INVALID_ENC (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x01) /* 0x51 means INSTR INVALID ENC */ ++#define JS_STATUS_INSTR_TYPE_MISMATCH (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x02) /* 0x52 means INSTR TYPE MISMATCH */ ++#define JS_STATUS_INSTR_OPERAND_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x03) /* 0x53 means INSTR OPERAND FAULT */ ++#define JS_STATUS_INSTR_TLS_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x04) /* 0x54 means INSTR TLS FAULT */ ++#define JS_STATUS_INSTR_BARRIER_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x05) /* 0x55 means INSTR BARRIER FAULT */ ++#define JS_STATUS_INSTR_ALIGN_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x06) /* 0x56 means INSTR ALIGN FAULT */ ++/* NOTE: No fault with 0x57 code defined in spec. */ ++#define JS_STATUS_DATA_INVALID_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x08) /* 0x58 means DATA INVALID FAULT */ ++#define JS_STATUS_TILE_RANGE_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x09) /* 0x59 means TILE RANGE FAULT */ ++#define JS_STATUS_ADDRESS_RANGE_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x0A) /* 0x5A means ADDRESS RANGE FAULT */ ++ ++/* Other faults */ ++#define JS_STATUS_MEMORY_FAULT_BASE 0x60 ++#define JS_STATUS_OUT_OF_MEMORY (JS_STATUS_MEMORY_FAULT_BASE) /* 0x60 means OUT OF MEMORY */ ++#define JS_STATUS_UNKNOWN 0x7F /* 0x7F means UNKNOWN */ ++ ++/* GPU_COMMAND values */ ++#define GPU_COMMAND_NOP 0x00 /* No operation, nothing happens */ ++#define GPU_COMMAND_SOFT_RESET 0x01 /* Stop all external bus interfaces, and then reset the entire GPU. */ ++#define GPU_COMMAND_HARD_RESET 0x02 /* Immediately reset the entire GPU. */ ++#define GPU_COMMAND_PRFCNT_CLEAR 0x03 /* Clear all performance counters, setting them all to zero. */ ++#define GPU_COMMAND_PRFCNT_SAMPLE 0x04 /* Sample all performance counters, writing them out to memory */ ++#define GPU_COMMAND_CYCLE_COUNT_START 0x05 /* Starts the cycle counter, and system timestamp propagation */ ++#define GPU_COMMAND_CYCLE_COUNT_STOP 0x06 /* Stops the cycle counter, and system timestamp propagation */ ++#define GPU_COMMAND_CLEAN_CACHES 0x07 /* Clean all caches */ ++#define GPU_COMMAND_CLEAN_INV_CACHES 0x08 /* Clean and invalidate all caches */ ++#define GPU_COMMAND_SET_PROTECTED_MODE 0x09 /* Places the GPU in protected mode */ ++ ++/* End Command Values */ ++ ++/* GPU_STATUS values */ ++#define GPU_STATUS_PRFCNT_ACTIVE (1 << 2) /* Set if the performance counters are active. */ ++#define GPU_STATUS_PROTECTED_MODE_ACTIVE (1 << 7) /* Set if protected mode is active */ ++ ++/* PRFCNT_CONFIG register values */ ++#define PRFCNT_CONFIG_MODE_SHIFT 0 /* Counter mode position. */ ++#define PRFCNT_CONFIG_AS_SHIFT 4 /* Address space bitmap position. */ ++#define PRFCNT_CONFIG_SETSELECT_SHIFT 8 /* Set select position. */ ++ ++#define PRFCNT_CONFIG_MODE_OFF 0 /* The performance counters are disabled. */ ++#define PRFCNT_CONFIG_MODE_MANUAL 1 /* The performance counters are enabled, but are only written out when a PRFCNT_SAMPLE command is issued using the GPU_COMMAND register. */ ++#define PRFCNT_CONFIG_MODE_TILE 2 /* The performance counters are enabled, and are written out each time a tile finishes rendering. */ ++ ++/* AS_MEMATTR values: */ ++/* Use GPU implementation-defined caching policy. */ ++#define AS_MEMATTR_IMPL_DEF_CACHE_POLICY 0x88ull ++/* The attribute set to force all resources to be cached. */ ++#define AS_MEMATTR_FORCE_TO_CACHE_ALL 0x8Full ++/* Inner write-alloc cache setup, no outer caching */ ++#define AS_MEMATTR_WRITE_ALLOC 0x8Dull ++ ++/* Set to implementation defined, outer caching */ ++#define AS_MEMATTR_AARCH64_OUTER_IMPL_DEF 0x88ull ++/* Set to write back memory, outer caching */ ++#define AS_MEMATTR_AARCH64_OUTER_WA 0x8Dull ++ ++/* Use GPU implementation-defined caching policy. */ ++#define AS_MEMATTR_LPAE_IMPL_DEF_CACHE_POLICY 0x48ull ++/* The attribute set to force all resources to be cached. */ ++#define AS_MEMATTR_LPAE_FORCE_TO_CACHE_ALL 0x4Full ++/* Inner write-alloc cache setup, no outer caching */ ++#define AS_MEMATTR_LPAE_WRITE_ALLOC 0x4Dull ++/* Set to implementation defined, outer caching */ ++#define AS_MEMATTR_LPAE_OUTER_IMPL_DEF 0x88ull ++/* Set to write back memory, outer caching */ ++#define AS_MEMATTR_LPAE_OUTER_WA 0x8Dull ++ ++/* Symbol for default MEMATTR to use */ ++ ++/* Default is - HW implementation defined caching */ ++#define AS_MEMATTR_INDEX_DEFAULT 0 ++#define AS_MEMATTR_INDEX_DEFAULT_ACE 3 ++ ++/* HW implementation defined caching */ ++#define AS_MEMATTR_INDEX_IMPL_DEF_CACHE_POLICY 0 ++/* Force cache on */ ++#define AS_MEMATTR_INDEX_FORCE_TO_CACHE_ALL 1 ++/* Write-alloc */ ++#define AS_MEMATTR_INDEX_WRITE_ALLOC 2 ++/* Outer coherent, inner implementation defined policy */ ++#define AS_MEMATTR_INDEX_OUTER_IMPL_DEF 3 ++/* Outer coherent, write alloc inner */ ++#define AS_MEMATTR_INDEX_OUTER_WA 4 ++ ++/* JS_FEATURES register */ ++ ++#define JS_FEATURE_NULL_JOB (1u << 1) ++#define JS_FEATURE_SET_VALUE_JOB (1u << 2) ++#define JS_FEATURE_CACHE_FLUSH_JOB (1u << 3) ++#define JS_FEATURE_COMPUTE_JOB (1u << 4) ++#define JS_FEATURE_VERTEX_JOB (1u << 5) ++#define JS_FEATURE_GEOMETRY_JOB (1u << 6) ++#define JS_FEATURE_TILER_JOB (1u << 7) ++#define JS_FEATURE_FUSED_JOB (1u << 8) ++#define JS_FEATURE_FRAGMENT_JOB (1u << 9) ++ ++/* End JS_FEATURES register */ ++ ++/* L2_MMU_CONFIG register */ ++#define L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY_SHIFT (23) ++#define L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY (0x1 << L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY_SHIFT) ++#define L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT (24) ++#define L2_MMU_CONFIG_LIMIT_EXTERNAL_READS (0x3 << L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT) ++#define L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_OCTANT (0x1 << L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT) ++#define L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_QUARTER (0x2 << L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT) ++#define L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_HALF (0x3 << L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT) ++ ++#define L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT (26) ++#define L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES (0x3 << L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT) ++#define L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_OCTANT (0x1 << L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT) ++#define L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_QUARTER (0x2 << L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT) ++#define L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_HALF (0x3 << L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT) ++/* End L2_MMU_CONFIG register */ ++ ++/* THREAD_* registers */ ++ ++/* THREAD_FEATURES IMPLEMENTATION_TECHNOLOGY values */ ++#define IMPLEMENTATION_UNSPECIFIED 0 ++#define IMPLEMENTATION_SILICON 1 ++#define IMPLEMENTATION_FPGA 2 ++#define IMPLEMENTATION_MODEL 3 ++ ++/* Default values when registers are not supported by the implemented hardware */ ++#define THREAD_MT_DEFAULT 256 ++#define THREAD_MWS_DEFAULT 256 ++#define THREAD_MBS_DEFAULT 256 ++#define THREAD_MR_DEFAULT 1024 ++#define THREAD_MTQ_DEFAULT 4 ++#define THREAD_MTGS_DEFAULT 10 ++ ++/* End THREAD_* registers */ ++ ++/* SHADER_CONFIG register */ ++ ++#define SC_ALT_COUNTERS (1ul << 3) ++#define SC_OVERRIDE_FWD_PIXEL_KILL (1ul << 4) ++#define SC_SDC_DISABLE_OQ_DISCARD (1ul << 6) ++#define SC_LS_ALLOW_ATTR_TYPES (1ul << 16) ++#define SC_LS_PAUSEBUFFER_DISABLE (1ul << 16) ++#define SC_LS_ATTR_CHECK_DISABLE (1ul << 18) ++#define SC_ENABLE_TEXGRD_FLAGS (1ul << 25) ++/* End SHADER_CONFIG register */ ++ ++/* TILER_CONFIG register */ ++ ++#define TC_CLOCK_GATE_OVERRIDE (1ul << 0) ++ ++/* End TILER_CONFIG register */ ++ ++/* JM_CONFIG register */ ++ ++#define JM_TIMESTAMP_OVERRIDE (1ul << 0) ++#define JM_CLOCK_GATE_OVERRIDE (1ul << 1) ++#define JM_JOB_THROTTLE_ENABLE (1ul << 2) ++#define JM_JOB_THROTTLE_LIMIT_SHIFT (3) ++#define JM_MAX_JOB_THROTTLE_LIMIT (0x3F) ++#define JM_FORCE_COHERENCY_FEATURES_SHIFT (2) ++#define JM_IDVS_GROUP_SIZE_SHIFT (16) ++#define JM_MAX_IDVS_GROUP_SIZE (0x3F) ++/* End JM_CONFIG register */ ++ ++ ++#endif /* _MIDGARD_REGMAP_H_ */ +diff --git a/drivers/gpu/arm/midgard/mali_timeline.h b/drivers/gpu/arm/midgard/mali_timeline.h +new file mode 100755 +index 000000000000..bd5f6614b6bb +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_timeline.h +@@ -0,0 +1,396 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#undef TRACE_SYSTEM ++#define TRACE_SYSTEM mali_timeline ++ ++#if !defined(_MALI_TIMELINE_H) || defined(TRACE_HEADER_MULTI_READ) ++#define _MALI_TIMELINE_H ++ ++#include ++ ++TRACE_EVENT(mali_timeline_atoms_in_flight, ++ ++ TP_PROTO(u64 ts_sec, ++ u32 ts_nsec, ++ int tgid, ++ int count), ++ ++ TP_ARGS(ts_sec, ++ ts_nsec, ++ tgid, ++ count), ++ ++ TP_STRUCT__entry( ++ __field(u64, ts_sec) ++ __field(u32, ts_nsec) ++ __field(int, tgid) ++ __field(int, count) ++ ), ++ ++ TP_fast_assign( ++ __entry->ts_sec = ts_sec; ++ __entry->ts_nsec = ts_nsec; ++ __entry->tgid = tgid; ++ __entry->count = count; ++ ), ++ ++ TP_printk("%i,%i.%.9i,%i,%i", CTX_SET_NR_ATOMS_IN_FLIGHT, ++ (int)__entry->ts_sec, ++ (int)__entry->ts_nsec, ++ __entry->tgid, ++ __entry->count) ++); ++ ++ ++TRACE_EVENT(mali_timeline_atom, ++ ++ TP_PROTO(u64 ts_sec, ++ u32 ts_nsec, ++ int event_type, ++ int tgid, ++ int atom_id), ++ ++ TP_ARGS(ts_sec, ++ ts_nsec, ++ event_type, ++ tgid, ++ atom_id), ++ ++ TP_STRUCT__entry( ++ __field(u64, ts_sec) ++ __field(u32, ts_nsec) ++ __field(int, event_type) ++ __field(int, tgid) ++ __field(int, atom_id) ++ ), ++ ++ TP_fast_assign( ++ __entry->ts_sec = ts_sec; ++ __entry->ts_nsec = ts_nsec; ++ __entry->event_type = event_type; ++ __entry->tgid = tgid; ++ __entry->atom_id = atom_id; ++ ), ++ ++ TP_printk("%i,%i.%.9i,%i,%i,%i", __entry->event_type, ++ (int)__entry->ts_sec, ++ (int)__entry->ts_nsec, ++ __entry->tgid, ++ __entry->atom_id, ++ __entry->atom_id) ++); ++ ++TRACE_EVENT(mali_timeline_gpu_slot_active, ++ ++ TP_PROTO(u64 ts_sec, ++ u32 ts_nsec, ++ int event_type, ++ int tgid, ++ int js, ++ int count), ++ ++ TP_ARGS(ts_sec, ++ ts_nsec, ++ event_type, ++ tgid, ++ js, ++ count), ++ ++ TP_STRUCT__entry( ++ __field(u64, ts_sec) ++ __field(u32, ts_nsec) ++ __field(int, event_type) ++ __field(int, tgid) ++ __field(int, js) ++ __field(int, count) ++ ), ++ ++ TP_fast_assign( ++ __entry->ts_sec = ts_sec; ++ __entry->ts_nsec = ts_nsec; ++ __entry->event_type = event_type; ++ __entry->tgid = tgid; ++ __entry->js = js; ++ __entry->count = count; ++ ), ++ ++ TP_printk("%i,%i.%.9i,%i,%i,%i", __entry->event_type, ++ (int)__entry->ts_sec, ++ (int)__entry->ts_nsec, ++ __entry->tgid, ++ __entry->js, ++ __entry->count) ++); ++ ++TRACE_EVENT(mali_timeline_gpu_slot_action, ++ ++ TP_PROTO(u64 ts_sec, ++ u32 ts_nsec, ++ int event_type, ++ int tgid, ++ int js, ++ int count), ++ ++ TP_ARGS(ts_sec, ++ ts_nsec, ++ event_type, ++ tgid, ++ js, ++ count), ++ ++ TP_STRUCT__entry( ++ __field(u64, ts_sec) ++ __field(u32, ts_nsec) ++ __field(int, event_type) ++ __field(int, tgid) ++ __field(int, js) ++ __field(int, count) ++ ), ++ ++ TP_fast_assign( ++ __entry->ts_sec = ts_sec; ++ __entry->ts_nsec = ts_nsec; ++ __entry->event_type = event_type; ++ __entry->tgid = tgid; ++ __entry->js = js; ++ __entry->count = count; ++ ), ++ ++ TP_printk("%i,%i.%.9i,%i,%i,%i", __entry->event_type, ++ (int)__entry->ts_sec, ++ (int)__entry->ts_nsec, ++ __entry->tgid, ++ __entry->js, ++ __entry->count) ++); ++ ++TRACE_EVENT(mali_timeline_gpu_power_active, ++ ++ TP_PROTO(u64 ts_sec, ++ u32 ts_nsec, ++ int event_type, ++ int active), ++ ++ TP_ARGS(ts_sec, ++ ts_nsec, ++ event_type, ++ active), ++ ++ TP_STRUCT__entry( ++ __field(u64, ts_sec) ++ __field(u32, ts_nsec) ++ __field(int, event_type) ++ __field(int, active) ++ ), ++ ++ TP_fast_assign( ++ __entry->ts_sec = ts_sec; ++ __entry->ts_nsec = ts_nsec; ++ __entry->event_type = event_type; ++ __entry->active = active; ++ ), ++ ++ TP_printk("%i,%i.%.9i,0,%i", __entry->event_type, ++ (int)__entry->ts_sec, ++ (int)__entry->ts_nsec, ++ __entry->active) ++ ++); ++ ++TRACE_EVENT(mali_timeline_l2_power_active, ++ ++ TP_PROTO(u64 ts_sec, ++ u32 ts_nsec, ++ int event_type, ++ int state), ++ ++ TP_ARGS(ts_sec, ++ ts_nsec, ++ event_type, ++ state), ++ ++ TP_STRUCT__entry( ++ __field(u64, ts_sec) ++ __field(u32, ts_nsec) ++ __field(int, event_type) ++ __field(int, state) ++ ), ++ ++ TP_fast_assign( ++ __entry->ts_sec = ts_sec; ++ __entry->ts_nsec = ts_nsec; ++ __entry->event_type = event_type; ++ __entry->state = state; ++ ), ++ ++ TP_printk("%i,%i.%.9i,0,%i", __entry->event_type, ++ (int)__entry->ts_sec, ++ (int)__entry->ts_nsec, ++ __entry->state) ++ ++); ++TRACE_EVENT(mali_timeline_pm_event, ++ ++ TP_PROTO(u64 ts_sec, ++ u32 ts_nsec, ++ int event_type, ++ int pm_event_type, ++ unsigned int pm_event_id), ++ ++ TP_ARGS(ts_sec, ++ ts_nsec, ++ event_type, ++ pm_event_type, ++ pm_event_id), ++ ++ TP_STRUCT__entry( ++ __field(u64, ts_sec) ++ __field(u32, ts_nsec) ++ __field(int, event_type) ++ __field(int, pm_event_type) ++ __field(unsigned int, pm_event_id) ++ ), ++ ++ TP_fast_assign( ++ __entry->ts_sec = ts_sec; ++ __entry->ts_nsec = ts_nsec; ++ __entry->event_type = event_type; ++ __entry->pm_event_type = pm_event_type; ++ __entry->pm_event_id = pm_event_id; ++ ), ++ ++ TP_printk("%i,%i.%.9i,0,%i,%u", __entry->event_type, ++ (int)__entry->ts_sec, ++ (int)__entry->ts_nsec, ++ __entry->pm_event_type, __entry->pm_event_id) ++ ++); ++ ++TRACE_EVENT(mali_timeline_slot_atom, ++ ++ TP_PROTO(u64 ts_sec, ++ u32 ts_nsec, ++ int event_type, ++ int tgid, ++ int js, ++ int atom_id), ++ ++ TP_ARGS(ts_sec, ++ ts_nsec, ++ event_type, ++ tgid, ++ js, ++ atom_id), ++ ++ TP_STRUCT__entry( ++ __field(u64, ts_sec) ++ __field(u32, ts_nsec) ++ __field(int, event_type) ++ __field(int, tgid) ++ __field(int, js) ++ __field(int, atom_id) ++ ), ++ ++ TP_fast_assign( ++ __entry->ts_sec = ts_sec; ++ __entry->ts_nsec = ts_nsec; ++ __entry->event_type = event_type; ++ __entry->tgid = tgid; ++ __entry->js = js; ++ __entry->atom_id = atom_id; ++ ), ++ ++ TP_printk("%i,%i.%.9i,%i,%i,%i", __entry->event_type, ++ (int)__entry->ts_sec, ++ (int)__entry->ts_nsec, ++ __entry->tgid, ++ __entry->js, ++ __entry->atom_id) ++); ++ ++TRACE_EVENT(mali_timeline_pm_checktrans, ++ ++ TP_PROTO(u64 ts_sec, ++ u32 ts_nsec, ++ int trans_code, ++ int trans_id), ++ ++ TP_ARGS(ts_sec, ++ ts_nsec, ++ trans_code, ++ trans_id), ++ ++ TP_STRUCT__entry( ++ __field(u64, ts_sec) ++ __field(u32, ts_nsec) ++ __field(int, trans_code) ++ __field(int, trans_id) ++ ), ++ ++ TP_fast_assign( ++ __entry->ts_sec = ts_sec; ++ __entry->ts_nsec = ts_nsec; ++ __entry->trans_code = trans_code; ++ __entry->trans_id = trans_id; ++ ), ++ ++ TP_printk("%i,%i.%.9i,0,%i", __entry->trans_code, ++ (int)__entry->ts_sec, ++ (int)__entry->ts_nsec, ++ __entry->trans_id) ++ ++); ++ ++TRACE_EVENT(mali_timeline_context_active, ++ ++ TP_PROTO(u64 ts_sec, ++ u32 ts_nsec, ++ int count), ++ ++ TP_ARGS(ts_sec, ++ ts_nsec, ++ count), ++ ++ TP_STRUCT__entry( ++ __field(u64, ts_sec) ++ __field(u32, ts_nsec) ++ __field(int, count) ++ ), ++ ++ TP_fast_assign( ++ __entry->ts_sec = ts_sec; ++ __entry->ts_nsec = ts_nsec; ++ __entry->count = count; ++ ), ++ ++ TP_printk("%i,%i.%.9i,0,%i", SW_SET_CONTEXT_ACTIVE, ++ (int)__entry->ts_sec, ++ (int)__entry->ts_nsec, ++ __entry->count) ++); ++ ++#endif /* _MALI_TIMELINE_H */ ++ ++#undef TRACE_INCLUDE_PATH ++#define TRACE_INCLUDE_PATH . ++ ++/* This part must be outside protection */ ++#include ++ +diff --git a/drivers/gpu/arm/midgard/mali_uk.h b/drivers/gpu/arm/midgard/mali_uk.h +new file mode 100755 +index 000000000000..841d03fb5873 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/mali_uk.h +@@ -0,0 +1,141 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010, 2012-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++/** ++ * @file mali_uk.h ++ * Types and definitions that are common across OSs for both the user ++ * and kernel side of the User-Kernel interface. ++ */ ++ ++#ifndef _UK_H_ ++#define _UK_H_ ++ ++#ifdef __cplusplus ++extern "C" { ++#endif /* __cplusplus */ ++ ++/** ++ * @addtogroup base_api ++ * @{ ++ */ ++ ++/** ++ * @defgroup uk_api User-Kernel Interface API ++ * ++ * The User-Kernel Interface abstracts the communication mechanism between the user and kernel-side code of device ++ * drivers developed as part of the Midgard DDK. Currently that includes the Base driver and the UMP driver. ++ * ++ * It exposes an OS independent API to user-side code (UKU) which routes functions calls to an OS-independent ++ * kernel-side API (UKK) via an OS-specific communication mechanism. ++ * ++ * This API is internal to the Midgard DDK and is not exposed to any applications. ++ * ++ * @{ ++ */ ++ ++/** ++ * These are identifiers for kernel-side drivers implementing a UK interface, aka UKK clients. The ++ * UK module maps this to an OS specific device name, e.g. "gpu_base" -> "GPU0:". Specify this ++ * identifier to select a UKK client to the uku_open() function. ++ * ++ * When a new UKK client driver is created a new identifier needs to be added to the uk_client_id ++ * enumeration and the uku_open() implemenation for the various OS ports need to be updated to ++ * provide a mapping of the identifier to the OS specific device name. ++ * ++ */ ++enum uk_client_id { ++ /** ++ * Value used to identify the Base driver UK client. ++ */ ++ UK_CLIENT_MALI_T600_BASE, ++ ++ /** The number of uk clients supported. This must be the last member of the enum */ ++ UK_CLIENT_COUNT ++}; ++ ++/** ++ * Each function callable through the UK interface has a unique number. ++ * Functions provided by UK clients start from number UK_FUNC_ID. ++ * Numbers below UK_FUNC_ID are used for internal UK functions. ++ */ ++enum uk_func { ++ UKP_FUNC_ID_CHECK_VERSION, /**< UKK Core internal function */ ++ /** ++ * Each UK client numbers the functions they provide starting from ++ * number UK_FUNC_ID. This number is then eventually assigned to the ++ * id field of the union uk_header structure when preparing to make a ++ * UK call. See your UK client for a list of their function numbers. ++ */ ++ UK_FUNC_ID = 512 ++}; ++ ++/** ++ * Arguments for a UK call are stored in a structure. This structure consists ++ * of a fixed size header and a payload. The header carries a 32-bit number ++ * identifying the UK function to be called (see uk_func). When the UKK client ++ * receives this header and executed the requested UK function, it will use ++ * the same header to store the result of the function in the form of a ++ * int return code. The size of this structure is such that the ++ * first member of the payload following the header can be accessed efficiently ++ * on a 32 and 64-bit kernel and the structure has the same size regardless ++ * of a 32 or 64-bit kernel. The uk_kernel_size_type type should be defined ++ * accordingly in the OS specific mali_uk_os.h header file. ++ */ ++union uk_header { ++ /** ++ * 32-bit number identifying the UK function to be called. ++ * Also see uk_func. ++ */ ++ u32 id; ++ /** ++ * The int return code returned by the called UK function. ++ * See the specification of the particular UK function you are ++ * calling for the meaning of the error codes returned. All ++ * UK functions return 0 on success. ++ */ ++ u32 ret; ++ /* ++ * Used to ensure 64-bit alignment of this union. Do not remove. ++ * This field is used for padding and does not need to be initialized. ++ */ ++ u64 sizer; ++}; ++ ++/** ++ * This structure carries a 16-bit major and minor number and is sent along with an internal UK call ++ * used during uku_open to identify the versions of the UK module in use by the user-side and kernel-side. ++ */ ++struct uku_version_check_args { ++ union uk_header header; ++ /**< UK call header */ ++ u16 major; ++ /**< This field carries the user-side major version on input and the kernel-side major version on output */ ++ u16 minor; ++ /**< This field carries the user-side minor version on input and the kernel-side minor version on output. */ ++ u8 padding[4]; ++}; ++ ++/** @} end group uk_api */ ++ ++/** @} *//* end group base_api */ ++ ++#ifdef __cplusplus ++} ++#endif /* __cplusplus */ ++#endif /* _UK_H_ */ +diff --git a/drivers/gpu/arm/midgard/platform/Kconfig b/drivers/gpu/arm/midgard/platform/Kconfig +new file mode 100755 +index 000000000000..8fb4e917c4fa +--- /dev/null ++++ b/drivers/gpu/arm/midgard/platform/Kconfig +@@ -0,0 +1,24 @@ ++# ++# (C) COPYRIGHT 2012 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++ ++ ++# Add your platform specific Kconfig file here ++# ++# "drivers/gpu/arm/midgard/platform/xxx/Kconfig" ++# ++# Where xxx is the platform name is the name set in MALI_PLATFORM_THIRDPARTY_NAME ++# ++ +diff --git a/drivers/gpu/arm/midgard/platform/devicetree/Kbuild b/drivers/gpu/arm/midgard/platform/devicetree/Kbuild +new file mode 100755 +index 000000000000..e888a42fc69a +--- /dev/null ++++ b/drivers/gpu/arm/midgard/platform/devicetree/Kbuild +@@ -0,0 +1,18 @@ ++# ++# (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++mali_kbase-y += \ ++ $(MALI_PLATFORM_THIRDPARTY_DIR)/mali_kbase_config_devicetree.o \ ++ $(MALI_PLATFORM_THIRDPARTY_DIR)/mali_kbase_runtime_pm.o +diff --git a/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_config_devicetree.c b/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_config_devicetree.c +new file mode 100755 +index 000000000000..b2a7c93f12a9 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_config_devicetree.c +@@ -0,0 +1,31 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include ++ ++int kbase_platform_early_init(void) ++{ ++ /* Nothing needed at this stage */ ++ return 0; ++} ++ ++static struct kbase_platform_config dummy_platform_config; ++ ++struct kbase_platform_config *kbase_get_platform_config(void) ++{ ++ return &dummy_platform_config; ++} +diff --git a/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_config_platform.h b/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_config_platform.h +new file mode 100755 +index 000000000000..49e107f98000 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_config_platform.h +@@ -0,0 +1,73 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/** ++ * Maximum frequency GPU will be clocked at. Given in kHz. ++ * This must be specified as there is no default value. ++ * ++ * Attached value: number in kHz ++ * Default value: NA ++ */ ++#define GPU_FREQ_KHZ_MAX (5000) ++/** ++ * Minimum frequency GPU will be clocked at. Given in kHz. ++ * This must be specified as there is no default value. ++ * ++ * Attached value: number in kHz ++ * Default value: NA ++ */ ++#define GPU_FREQ_KHZ_MIN (5000) ++ ++/** ++ * CPU_SPEED_FUNC - A pointer to a function that calculates the CPU clock ++ * ++ * CPU clock speed of the platform is in MHz - see kbase_cpu_clk_speed_func ++ * for the function prototype. ++ * ++ * Attached value: A kbase_cpu_clk_speed_func. ++ * Default Value: NA ++ */ ++#define CPU_SPEED_FUNC (NULL) ++ ++/** ++ * GPU_SPEED_FUNC - A pointer to a function that calculates the GPU clock ++ * ++ * GPU clock speed of the platform in MHz - see kbase_gpu_clk_speed_func ++ * for the function prototype. ++ * ++ * Attached value: A kbase_gpu_clk_speed_func. ++ * Default Value: NA ++ */ ++#define GPU_SPEED_FUNC (NULL) ++ ++/** ++ * Power management configuration ++ * ++ * Attached value: pointer to @ref kbase_pm_callback_conf ++ * Default value: See @ref kbase_pm_callback_conf ++ */ ++#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks) ++ ++/** ++ * Platform specific configuration functions ++ * ++ * Attached value: pointer to @ref kbase_platform_funcs_conf ++ * Default value: See @ref kbase_platform_funcs_conf ++ */ ++#define PLATFORM_FUNCS (NULL) ++ ++extern struct kbase_pm_callback_conf pm_callbacks; +diff --git a/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_runtime_pm.c b/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_runtime_pm.c +new file mode 100755 +index 000000000000..aa4376afd3ba +--- /dev/null ++++ b/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_runtime_pm.c +@@ -0,0 +1,100 @@ ++/* ++ * ++ * (C) COPYRIGHT 2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include ++#include ++#include ++#include ++ ++static int pm_callback_power_on(struct kbase_device *kbdev) ++{ ++ int ret; ++ ++ dev_dbg(kbdev->dev, "pm_callback_power_on %p\n", ++ (void *)kbdev->dev->pm_domain); ++ ++ ret = pm_runtime_get_sync(kbdev->dev); ++ ++ dev_dbg(kbdev->dev, "pm_runtime_get returned %d\n", ret); ++ ++ return 1; ++} ++ ++static void pm_callback_power_off(struct kbase_device *kbdev) ++{ ++ dev_dbg(kbdev->dev, "pm_callback_power_off\n"); ++ ++ pm_runtime_put_autosuspend(kbdev->dev); ++} ++ ++int kbase_device_runtime_init(struct kbase_device *kbdev) ++{ ++ dev_dbg(kbdev->dev, "kbase_device_runtime_init\n"); ++ pm_runtime_enable(kbdev->dev); ++ ++ return 0; ++} ++ ++void kbase_device_runtime_disable(struct kbase_device *kbdev) ++{ ++ dev_dbg(kbdev->dev, "kbase_device_runtime_disable\n"); ++ pm_runtime_disable(kbdev->dev); ++} ++ ++static int pm_callback_runtime_on(struct kbase_device *kbdev) ++{ ++ dev_dbg(kbdev->dev, "pm_callback_runtime_on\n"); ++ ++ return 0; ++} ++ ++static void pm_callback_runtime_off(struct kbase_device *kbdev) ++{ ++ dev_dbg(kbdev->dev, "pm_callback_runtime_off\n"); ++} ++ ++static void pm_callback_resume(struct kbase_device *kbdev) ++{ ++ int ret = pm_callback_runtime_on(kbdev); ++ ++ WARN_ON(ret); ++} ++ ++static void pm_callback_suspend(struct kbase_device *kbdev) ++{ ++ pm_callback_runtime_off(kbdev); ++} ++ ++struct kbase_pm_callback_conf pm_callbacks = { ++ .power_on_callback = pm_callback_power_on, ++ .power_off_callback = pm_callback_power_off, ++ .power_suspend_callback = pm_callback_suspend, ++ .power_resume_callback = pm_callback_resume, ++#ifdef KBASE_PM_RUNTIME ++ .power_runtime_init_callback = kbase_device_runtime_init, ++ .power_runtime_term_callback = kbase_device_runtime_disable, ++ .power_runtime_on_callback = pm_callback_runtime_on, ++ .power_runtime_off_callback = pm_callback_runtime_off, ++#else /* KBASE_PM_RUNTIME */ ++ .power_runtime_init_callback = NULL, ++ .power_runtime_term_callback = NULL, ++ .power_runtime_on_callback = NULL, ++ .power_runtime_off_callback = NULL, ++#endif /* KBASE_PM_RUNTIME */ ++}; ++ ++ +diff --git a/drivers/gpu/arm/midgard/platform/mali_kbase_platform_common.h b/drivers/gpu/arm/midgard/platform/mali_kbase_platform_common.h +new file mode 100755 +index 000000000000..c11085af5f24 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/platform/mali_kbase_platform_common.h +@@ -0,0 +1,28 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2013 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++#include ++ ++ ++/** ++ * @brief Entry point to transfer control to a platform for early initialization ++ * ++ * This function is called early on in the initialization during execution of ++ * @ref kbase_driver_init. ++ * ++ * @return Zero to indicate success non-zero for failure. ++ */ ++int kbase_platform_early_init(void); ++int kbase_platform_rk_init_opp_table(struct kbase_device *kbdev); +diff --git a/drivers/gpu/arm/midgard/platform/mali_kbase_platform_fake.h b/drivers/gpu/arm/midgard/platform/mali_kbase_platform_fake.h +new file mode 100755 +index 000000000000..01f9dfce93cc +--- /dev/null ++++ b/drivers/gpu/arm/midgard/platform/mali_kbase_platform_fake.h +@@ -0,0 +1,38 @@ ++/* ++ * ++ * (C) COPYRIGHT 2010-2014 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifdef CONFIG_MALI_PLATFORM_FAKE ++ ++/** ++ * kbase_platform_fake_register - Entry point for fake platform registration ++ * ++ * This function is called early on in the initialization during execution of ++ * kbase_driver_init. ++ * ++ * Return: 0 to indicate success, non-zero for failure. ++ */ ++int kbase_platform_fake_register(void); ++ ++/** ++ * kbase_platform_fake_unregister - Entry point for fake platform unregistration ++ * ++ * This function is called in the termination during execution of ++ * kbase_driver_exit. ++ */ ++void kbase_platform_fake_unregister(void); ++ ++#endif /* CONFIG_MALI_PLATFORM_FAKE */ +diff --git a/drivers/gpu/arm/midgard/platform/rk/Kbuild b/drivers/gpu/arm/midgard/platform/rk/Kbuild +new file mode 100755 +index 000000000000..db993487e3be +--- /dev/null ++++ b/drivers/gpu/arm/midgard/platform/rk/Kbuild +@@ -0,0 +1,17 @@ ++# ++# (C) COPYRIGHT 2012-2013 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++midgard_kbase-y += \ ++ $(MALI_PLATFORM_THIRDPARTY_DIR)/mali_kbase_config_rk.o ++ +diff --git a/drivers/gpu/arm/midgard/platform/rk/custom_log.h b/drivers/gpu/arm/midgard/platform/rk/custom_log.h +new file mode 100755 +index 000000000000..fe5e1224149e +--- /dev/null ++++ b/drivers/gpu/arm/midgard/platform/rk/custom_log.h +@@ -0,0 +1,209 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* ---------------------------------------------------------------------------- ++ * File: custom_log.h ++ * ++ * Desc: ChenZhen å好的 log 输出的定制实现. ++ * ++ * -------------------------------------------------------------------- ++ * < 习语 å’Œ 缩略语 > : ++ * ++ * -------------------------------------------------------------------- ++ * Usage: ++ * ++ * Note: ++ * ++ * Author: ChenZhen ++ * ++ * ---------------------------------------------------------------------------- ++ * Version: ++ * v1.0 ++ * ---------------------------------------------------------------------------- ++ * Log: ++ ----Fri Nov 19 15:20:28 2010 v1.0 ++ * ++ * ---------------------------------------------------------------------------- ++ */ ++ ++#ifndef __CUSTOM_LOG_H__ ++#define __CUSTOM_LOG_H__ ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++/* ----------------------------------------------------------------------------- ++ * Include Files ++ * ----------------------------------------------------------------------------- ++ */ ++#include ++#include ++ ++/* ----------------------------------------------------------------------------- ++ * Macros Definition ++ * ----------------------------------------------------------------------------- ++ */ ++ ++/** 若下列 macro 有被定义, æ‰ ä½¿èƒ½ log 输出. */ ++/* #define ENABLE_DEBUG_LOG */ ++ ++/*----------------------------------------------------------------------------*/ ++ ++#ifdef ENABLE_VERBOSE_LOG ++/** Verbose log. */ ++#define V(fmt, args...) \ ++ pr_debug("V : [File] : %s; [Line] : %d; [Func] : %s(); " fmt \ ++ "\n", \ ++ __FILE__, \ ++ __LINE__, \ ++ __func__, \ ++ ## args) ++#else ++#define V(...) ((void)0) ++#endif ++ ++#ifdef ENABLE_DEBUG_LOG ++/** Debug log. */ ++#define D(fmt, args...) \ ++ pr_info("D : [File] : %s; [Line] : %d; [Func] : %s(); " fmt \ ++ "\n", \ ++ __FILE__, \ ++ __LINE__, \ ++ __func__, \ ++ ## args) ++#else ++#define D(...) ((void)0) ++#endif ++ ++#define I(fmt, args...) \ ++ pr_info("I : [File] : %s; [Line] : %d; [Func] : %s(); " fmt \ ++ "\n", \ ++ __FILE__, \ ++ __LINE__, \ ++ __func__, \ ++ ## args) ++ ++#define W(fmt, args...) \ ++ pr_warn("W : [File] : %s; [Line] : %d; [Func] : %s(); " \ ++ fmt "\n", \ ++ __FILE__, \ ++ __LINE__, \ ++ __func__, \ ++ ## args) ++ ++#define E(fmt, args...) \ ++ pr_err("E : [File] : %s; [Line] : %d; [Func] : %s(); " fmt \ ++ "\n", \ ++ __FILE__, \ ++ __LINE__, \ ++ __func__, \ ++ ## args) ++ ++/*-------------------------------------------------------*/ ++ ++/** 使用 D(), 以å进制的形å¼æ‰“å°å˜é‡ 'var' çš„ value. */ ++#define D_DEC(var) D(#var " = %d.", var) ++ ++#define E_DEC(var) E(#var " = %d.", var) ++ ++/** 使用 D(), 以å六进制的形å¼æ‰“å°å˜é‡ 'var' çš„ value. */ ++#define D_HEX(var) D(#var " = 0x%x.", var) ++ ++#define E_HEX(var) E(#var " = 0x%x.", var) ++ ++/** ++ * 使用 D(), 以å六进制的形å¼, ++ * 打å°æŒ‡é’ˆç±»åž‹å˜é‡ 'ptr' çš„ value. ++ */ ++#define D_PTR(ptr) D(#ptr " = %p.", ptr) ++ ++#define E_PTR(ptr) E(#ptr " = %p.", ptr) ++ ++/** 使用 D(), æ‰“å° char 字串. */ ++#define D_STR(p_str) \ ++do { \ ++ if (!p_str) { \ ++ D(#p_str " = NULL."); \ ++ else \ ++ D(#p_str " = '%s'.", p_str); \ ++} while (0) ++ ++#define E_STR(p_str) \ ++do { \ ++ if (!p_str) \ ++ E(#p_str " = NULL."); \ ++ else \ ++ E(#p_str " = '%s'.", p_str); \ ++} while (0) ++ ++#ifdef ENABLE_DEBUG_LOG ++/** ++ * log 从 'p_start' 地å€å¼€å§‹çš„ 'len' 个字节的数æ®. ++ */ ++#define D_MEM(p_start, len) \ ++do { \ ++ int i = 0; \ ++ char *p = (char *)(p_start); \ ++ D("dump memory from addr of '" #p_start "', from %p, length %d' : ", \ ++ (p_start), \ ++ (len)); \ ++ pr_debug("\t\t"); \ ++ for (i = 0; i < (len); i++) \ ++ pr_debug("0x%02x, ", p[i]); \ ++ pr_debug("\n"); \ ++} while (0) ++#else ++#define D_MEM(...) ((void)0) ++#endif ++ ++/*-------------------------------------------------------*/ ++ ++/** ++ * 在特定æ¡ä»¶ä¸‹, 判定 error å‘生, ++ * å°†å˜é‡ 'ret_var' 设置 'err_code', ++ * log 输出对应的 Error Caution, ++ * 然åŽè·³è½¬ 'label' 指定的代ç å¤„执行. ++ * @param msg ++ * 纯字串形å¼çš„æ示信æ¯. ++ * @param ret_var ++ * 标识函数执行状æ€æˆ–者结果的å˜é‡, ++ * 将被设置具体的 Error Code. ++ * 通常是 'ret' or 'result'. ++ * @param err_code ++ * 表å¾ç‰¹å®š error 的常数标识, ++ * 通常是 å®çš„å½¢æ€. ++ * @param label ++ * 程åºå°†è¦è·³è½¬åˆ°çš„错误处ç†ä»£ç çš„æ ‡å·, ++ * 通常就是 'EXIT'. ++ * @param args... ++ * 对应 'msg_fmt' 实å‚中, ++ * '%s', '%d', ... 等转æ¢è¯´æ˜Žç¬¦çš„具体å¯å˜é•¿å®žå‚. ++ */ ++#define SET_ERROR_AND_JUMP(msg_fmt, ret_var, err_code, label, args...) \ ++do { \ ++ E("To set '" #ret_var "' to %d('" #err_code "'), because : " msg_fmt, \ ++ (err_code), \ ++ ## args); \ ++ (ret_var) = (err_code); \ ++ goto label; \ ++} while (0) ++ ++/* ----------------------------------------------------------------------------- ++ * Types and Structures Definition ++ * ----------------------------------------------------------------------------- ++ */ ++ ++/* ----------------------------------------------------------------------------- ++ * Global Functions' Prototype ++ * ----------------------------------------------------------------------------- ++ */ ++ ++/* ----------------------------------------------------------------------------- ++ * Inline Functions Implementation ++ * ----------------------------------------------------------------------------- ++ */ ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif /* __CUSTOM_LOG_H__ */ +diff --git a/drivers/gpu/arm/midgard/platform/rk/mali_kbase_config_platform.h b/drivers/gpu/arm/midgard/platform/rk/mali_kbase_config_platform.h +new file mode 100755 +index 000000000000..07c5b6f8a760 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/platform/rk/mali_kbase_config_platform.h +@@ -0,0 +1,88 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ */ ++ ++/** ++ * @file mali_kbase_config_platform.h ++ * 声明 platform_config_of_rk (platform_rk çš„ platform_config). ++ */ ++ ++/** ++ * Maximum frequency GPU will be clocked at. ++ * Given in kHz. ++ * This must be specified as there is no default value. ++ * ++ * Attached value: number in kHz ++ * Default value: NA ++ */ ++#define GPU_FREQ_KHZ_MAX (5000) ++ ++/** ++ * Minimum frequency GPU will be clocked at. ++ * Given in kHz. ++ * This must be specified as there is no default value. ++ * ++ * Attached value: number in kHz ++ * Default value: NA ++ */ ++#define GPU_FREQ_KHZ_MIN (5000) ++ ++/** ++ * CPU_SPEED_FUNC ++ * - A pointer to a function that calculates the CPU clock ++ * ++ * CPU clock speed of the platform is in MHz ++ * - see kbase_cpu_clk_speed_func for the function prototype. ++ * ++ * Attached value: A kbase_cpu_clk_speed_func. ++ * Default Value: NA ++ */ ++#define CPU_SPEED_FUNC (NULL) ++ ++/** ++ * GPU_SPEED_FUNC ++ * - A pointer to a function that calculates the GPU clock ++ * ++ * GPU clock speed of the platform in MHz ++ * - see kbase_gpu_clk_speed_func for the function prototype. ++ * ++ * Attached value: A kbase_gpu_clk_speed_func. ++ * Default Value: NA ++ */ ++#define GPU_SPEED_FUNC (NULL) ++ ++/** ++ * Power management configuration ++ * ++ * Attached value: ++ * pointer to @ref kbase_pm_callback_conf ++ * Default value: ++ * See @ref kbase_pm_callback_conf ++ */ ++#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks) ++extern struct kbase_pm_callback_conf pm_callbacks; ++ ++/** ++ * Platform specific configuration functions ++ * ++ * Attached value: ++ * pointer to @ref kbase_platform_funcs_conf ++ * Default value: ++ * See @ref kbase_platform_funcs_conf ++ */ ++#define PLATFORM_FUNCS (&platform_funcs) ++extern struct kbase_platform_funcs_conf platform_funcs; ++ ++/** ++ * Secure mode switch ++ * ++ * Attached value: pointer to @ref kbase_secure_ops ++ */ ++#define SECURE_CALLBACKS (NULL) ++ +diff --git a/drivers/gpu/arm/midgard/platform/rk/mali_kbase_config_rk.c b/drivers/gpu/arm/midgard/platform/rk/mali_kbase_config_rk.c +new file mode 100755 +index 000000000000..9c71a78f95d7 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/platform/rk/mali_kbase_config_rk.c +@@ -0,0 +1,486 @@ ++/* ++ * ++ * (C) COPYRIGHT ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ */ ++ ++/* #define ENABLE_DEBUG_LOG */ ++#include "custom_log.h" ++ ++#include ++#include ++#include ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "mali_kbase_rk.h" ++ ++/** ++ * @file mali_kbase_config_rk.c ++ * 对 platform_config_of_rk 的具体实现. ++ * ++ * mali_device_driver 包å«ä¸¤éƒ¨åˆ† : ++ * .DP : platform_dependent_part_in_mdd : ++ * ä¾èµ– platform 部分, ++ * æºç åœ¨ /platform// ++ * 在 mali_device_driver 内部, ++ * 记为 platform_dependent_part, ++ * 也被记为 platform_specific_code. ++ * .DP : common_parts_in_mdd : ++ * arm 实现的通用的部分, ++ * æºç åœ¨ / 下. ++ * 在 mali_device_driver 内部, 记为 common_parts. ++ */ ++ ++/*---------------------------------------------------------------------------*/ ++ ++#ifdef CONFIG_REGULATOR ++static int rk_pm_enable_regulator(struct kbase_device *kbdev); ++static void rk_pm_disable_regulator(struct kbase_device *kbdev); ++#else ++static inline int rk_pm_enable_regulator(struct kbase_device *kbdev) ++{ ++ return 0; ++} ++ ++static inline void rk_pm_disable_regulator(struct kbase_device *kbdev) ++{ ++} ++#endif ++ ++static int rk_pm_enable_clk(struct kbase_device *kbdev); ++ ++static void rk_pm_disable_clk(struct kbase_device *kbdev); ++ ++static int kbase_platform_rk_create_sysfs_files(struct device *dev); ++ ++static void kbase_platform_rk_remove_sysfs_files(struct device *dev); ++ ++/*---------------------------------------------------------------------------*/ ++ ++static void rk_pm_power_off_delay_work(struct work_struct *work) ++{ ++ struct rk_context *platform = ++ container_of(to_delayed_work(work), struct rk_context, work); ++ struct kbase_device *kbdev = platform->kbdev; ++ ++ if (!platform->is_powered) { ++ D("mali_dev is already powered off."); ++ return; ++ } ++ ++ if (pm_runtime_enabled(kbdev->dev)) { ++ D("to put_sync_suspend mali_dev."); ++ pm_runtime_put_sync_suspend(kbdev->dev); ++ } ++ ++ rk_pm_disable_regulator(kbdev); ++ ++ platform->is_powered = false; ++ KBASE_TIMELINE_GPU_POWER(kbdev, 0); ++ wake_unlock(&platform->wake_lock); ++} ++ ++static int kbase_platform_rk_init(struct kbase_device *kbdev) ++{ ++ int ret = 0; ++ struct rk_context *platform; ++ ++ platform = kzalloc(sizeof(*platform), GFP_KERNEL); ++ if (!platform) { ++ E("err."); ++ return -ENOMEM; ++ } ++ ++ platform->is_powered = false; ++ platform->kbdev = kbdev; ++ ++ platform->delay_ms = 200; ++ if (of_property_read_u32(kbdev->dev->of_node, "power-off-delay-ms", ++ &platform->delay_ms)) ++ W("power-off-delay-ms not available."); ++ ++ platform->power_off_wq = create_freezable_workqueue("gpu_power_off_wq"); ++ if (!platform->power_off_wq) { ++ E("couldn't create workqueue"); ++ ret = -ENOMEM; ++ goto err_wq; ++ } ++ INIT_DEFERRABLE_WORK(&platform->work, rk_pm_power_off_delay_work); ++ ++ wake_lock_init(&platform->wake_lock, WAKE_LOCK_SUSPEND, "gpu"); ++ ++ platform->utilisation_period = DEFAULT_UTILISATION_PERIOD_IN_MS; ++ ++ ret = kbase_platform_rk_create_sysfs_files(kbdev->dev); ++ if (ret) { ++ E("fail to create sysfs_files. ret = %d.", ret); ++ goto err_sysfs_files; ++ } ++ ++ kbdev->platform_context = (void *)platform; ++ pm_runtime_enable(kbdev->dev); ++ ++ return 0; ++ ++err_sysfs_files: ++ wake_lock_destroy(&platform->wake_lock); ++ destroy_workqueue(platform->power_off_wq); ++err_wq: ++ return ret; ++} ++ ++static void kbase_platform_rk_term(struct kbase_device *kbdev) ++{ ++ struct rk_context *platform = ++ (struct rk_context *)kbdev->platform_context; ++ ++ pm_runtime_disable(kbdev->dev); ++ kbdev->platform_context = NULL; ++ ++ if (platform) { ++ cancel_delayed_work_sync(&platform->work); ++ wake_lock_destroy(&platform->wake_lock); ++ destroy_workqueue(platform->power_off_wq); ++ platform->is_powered = false; ++ platform->kbdev = NULL; ++ kfree(platform); ++ } ++ kbase_platform_rk_remove_sysfs_files(kbdev->dev); ++} ++ ++struct kbase_platform_funcs_conf platform_funcs = { ++ .platform_init_func = &kbase_platform_rk_init, ++ .platform_term_func = &kbase_platform_rk_term, ++}; ++ ++/*---------------------------------------------------------------------------*/ ++ ++static int rk_pm_callback_runtime_on(struct kbase_device *kbdev) ++{ ++ return 0; ++} ++ ++static void rk_pm_callback_runtime_off(struct kbase_device *kbdev) ++{ ++} ++ ++static int rk_pm_callback_power_on(struct kbase_device *kbdev) ++{ ++ int ret = 1; /* Assume GPU has been powered off */ ++ int err = 0; ++ struct rk_context *platform = get_rk_context(kbdev); ++ ++ cancel_delayed_work_sync(&platform->work); ++ ++ err = rk_pm_enable_clk(kbdev); ++ if (err) { ++ E("failed to enable clk: %d", err); ++ return err; ++ } ++ ++ if (platform->is_powered) { ++ D("mali_device is already powered."); ++ return 0; ++ } ++ ++ /* we must enable vdd_gpu before pd_gpu_in_chip. */ ++ err = rk_pm_enable_regulator(kbdev); ++ if (err) { ++ E("fail to enable regulator, err : %d.", err); ++ return err; ++ } ++ ++ /* è‹¥ mali_dev çš„ runtime_pm 是 enabled çš„, 则... */ ++ if (pm_runtime_enabled(kbdev->dev)) { ++ D("to resume mali_dev syncly."); ++ /* 对 pd_in_chip çš„ on æ“作, ++ * 将在 pm_domain çš„ runtime_pm_callbacks 中完æˆ. ++ */ ++ err = pm_runtime_get_sync(kbdev->dev); ++ if (err < 0) { ++ E("failed to runtime resume device: %d.", err); ++ return err; ++ } else if (err == 1) { /* runtime_pm_status is still active */ ++ D("chip has NOT been powered off, no need to re-init."); ++ ret = 0; ++ } ++ } ++ ++ platform->is_powered = true; ++ KBASE_TIMELINE_GPU_POWER(kbdev, 1); ++ wake_lock(&platform->wake_lock); ++ ++ return ret; ++} ++ ++static void rk_pm_callback_power_off(struct kbase_device *kbdev) ++{ ++ struct rk_context *platform = get_rk_context(kbdev); ++ ++ rk_pm_disable_clk(kbdev); ++ queue_delayed_work(platform->power_off_wq, &platform->work, ++ msecs_to_jiffies(platform->delay_ms)); ++} ++ ++int rk_kbase_device_runtime_init(struct kbase_device *kbdev) ++{ ++ return 0; ++} ++ ++void rk_kbase_device_runtime_disable(struct kbase_device *kbdev) ++{ ++} ++ ++struct kbase_pm_callback_conf pm_callbacks = { ++ .power_on_callback = rk_pm_callback_power_on, ++ .power_off_callback = rk_pm_callback_power_off, ++#ifdef CONFIG_PM ++ .power_runtime_init_callback = rk_kbase_device_runtime_init, ++ .power_runtime_term_callback = rk_kbase_device_runtime_disable, ++ .power_runtime_on_callback = rk_pm_callback_runtime_on, ++ .power_runtime_off_callback = rk_pm_callback_runtime_off, ++#else /* CONFIG_PM */ ++ .power_runtime_init_callback = NULL, ++ .power_runtime_term_callback = NULL, ++ .power_runtime_on_callback = NULL, ++ .power_runtime_off_callback = NULL, ++#endif /* CONFIG_PM */ ++}; ++ ++int kbase_platform_early_init(void) ++{ ++ /* Nothing needed at this stage */ ++ return 0; ++} ++ ++/*---------------------------------------------------------------------------*/ ++ ++void kbase_platform_rk_shutdown(struct kbase_device *kbdev) ++{ ++ I("to make vdd_gpu enabled for turning off pd_gpu in pm_framework."); ++ rk_pm_enable_regulator(kbdev); ++} ++ ++/*---------------------------------------------------------------------------*/ ++ ++#ifdef CONFIG_REGULATOR ++static int rk_pm_enable_regulator(struct kbase_device *kbdev) ++{ ++ int ret = 0; ++ ++ if (!kbdev->regulator) { ++ W("no mali regulator control, no need to enable."); ++ goto EXIT; ++ } ++ ++ D("to enable regulator."); ++ ret = regulator_enable(kbdev->regulator); ++ if (ret) { ++ E("fail to enable regulator, ret : %d.", ret); ++ goto EXIT; ++ } ++ ++EXIT: ++ return ret; ++} ++ ++static void rk_pm_disable_regulator(struct kbase_device *kbdev) ++{ ++ if (!(kbdev->regulator)) { ++ W("no mali regulator control, no need to disable."); ++ return; ++ } ++ ++ D("to disable regulator."); ++ regulator_disable(kbdev->regulator); ++} ++#endif ++ ++static int rk_pm_enable_clk(struct kbase_device *kbdev) ++{ ++ int err = 0; ++ ++ if (!(kbdev->clock)) { ++ W("no mali clock control, no need to enable."); ++ } else { ++ D("to enable clk."); ++ err = clk_enable(kbdev->clock); ++ if (err) ++ E("failed to enable clk: %d.", err); ++ } ++ ++ return err; ++} ++ ++static void rk_pm_disable_clk(struct kbase_device *kbdev) ++{ ++ if (!(kbdev->clock)) { ++ W("no mali clock control, no need to disable."); ++ } else { ++ D("to disable clk."); ++ clk_disable(kbdev->clock); ++ } ++} ++ ++/*---------------------------------------------------------------------------*/ ++ ++static ssize_t utilisation_period_show(struct device *dev, ++ struct device_attribute *attr, ++ char *buf) ++{ ++ struct kbase_device *kbdev = dev_get_drvdata(dev); ++ struct rk_context *platform = get_rk_context(kbdev); ++ ssize_t ret = 0; ++ ++ ret += snprintf(buf, PAGE_SIZE, "%u\n", platform->utilisation_period); ++ ++ return ret; ++} ++ ++static ssize_t utilisation_period_store(struct device *dev, ++ struct device_attribute *attr, ++ const char *buf, ++ size_t count) ++{ ++ struct kbase_device *kbdev = dev_get_drvdata(dev); ++ struct rk_context *platform = get_rk_context(kbdev); ++ int ret = 0; ++ ++ ret = kstrtouint(buf, 0, &platform->utilisation_period); ++ if (ret) { ++ E("invalid input period : %s.", buf); ++ return ret; ++ } ++ D("set utilisation_period to '%d'.", platform->utilisation_period); ++ ++ return count; ++} ++ ++static ssize_t utilisation_show(struct device *dev, ++ struct device_attribute *attr, ++ char *buf) ++{ ++ struct kbase_device *kbdev = dev_get_drvdata(dev); ++ struct rk_context *platform = get_rk_context(kbdev); ++ ssize_t ret = 0; ++ unsigned long period_in_us = platform->utilisation_period * 1000; ++ unsigned long total_time; ++ unsigned long busy_time; ++ unsigned long utilisation; ++ ++ kbase_pm_reset_dvfs_utilisation(kbdev); ++ usleep_range(period_in_us, period_in_us + 100); ++ kbase_pm_get_dvfs_utilisation(kbdev, &total_time, &busy_time); ++ /* 'devfreq_dev_profile' instance registered to devfreq ++ * also uses kbase_pm_reset_dvfs_utilisation ++ * and kbase_pm_get_dvfs_utilisation. ++ * it's better to cat this file when DVFS is disabled. ++ */ ++ D("total_time : %lu, busy_time : %lu.", total_time, busy_time); ++ ++ utilisation = busy_time * 100 / total_time; ++ ret += snprintf(buf, PAGE_SIZE, "%ld\n", utilisation); ++ ++ return ret; ++} ++ ++static DEVICE_ATTR_RW(utilisation_period); ++static DEVICE_ATTR_RO(utilisation); ++ ++static int kbase_platform_rk_create_sysfs_files(struct device *dev) ++{ ++ int ret = 0; ++ ++ ret = device_create_file(dev, &dev_attr_utilisation_period); ++ if (ret) { ++ E("fail to create sysfs file 'utilisation_period'."); ++ goto out; ++ } ++ ++ ret = device_create_file(dev, &dev_attr_utilisation); ++ if (ret) { ++ E("fail to create sysfs file 'utilisation'."); ++ goto remove_utilisation_period; ++ } ++ ++ return 0; ++ ++remove_utilisation_period: ++ device_remove_file(dev, &dev_attr_utilisation_period); ++out: ++ return ret; ++} ++ ++static void kbase_platform_rk_remove_sysfs_files(struct device *dev) ++{ ++ device_remove_file(dev, &dev_attr_utilisation_period); ++ device_remove_file(dev, &dev_attr_utilisation); ++} ++ ++static int rk3288_get_soc_info(struct device *dev, struct device_node *np, ++ int *bin, int *process) ++{ ++ int ret = -EINVAL; ++ u8 value = 0; ++ char *name; ++ ++ if (!bin) ++ goto out; ++ ++ if (soc_is_rk3288w()) ++ name = "performance-w"; ++ else ++ name = "performance"; ++ if (of_property_match_string(np, "nvmem-cell-names", name) >= 0) { ++ ret = rockchip_nvmem_cell_read_u8(np, name, &value); ++ if (ret) { ++ dev_err(dev, "Failed to get soc performance value\n"); ++ goto out; ++ } ++ if (value & 0x2) ++ *bin = 3; ++ else if (value & 0x01) ++ *bin = 2; ++ else ++ *bin = 0; ++ } else { ++ dev_err(dev, "Failed to get bin config\n"); ++ } ++ if (*bin >= 0) ++ dev_info(dev, "bin=%d\n", *bin); ++ ++out: ++ return ret; ++} ++ ++static const struct of_device_id rockchip_mali_of_match[] = { ++ { ++ .compatible = "rockchip,rk3288", ++ .data = (void *)&rk3288_get_soc_info, ++ }, ++ { ++ .compatible = "rockchip,rk3288w", ++ .data = (void *)&rk3288_get_soc_info, ++ }, ++ {}, ++}; ++ ++int kbase_platform_rk_init_opp_table(struct kbase_device *kbdev) ++{ ++ return rockchip_init_opp_table(kbdev->dev, rockchip_mali_of_match, ++ "gpu_leakage", "mali"); ++} +diff --git a/drivers/gpu/arm/midgard/platform/rk/mali_kbase_config_rk.c.rej b/drivers/gpu/arm/midgard/platform/rk/mali_kbase_config_rk.c.rej +new file mode 100644 +index 000000000000..a68ec9e6f4d6 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/platform/rk/mali_kbase_config_rk.c.rej +@@ -0,0 +1,20 @@ ++diff a/drivers/gpu/arm/midgard/platform/rk/mali_kbase_config_rk.c b/drivers/gpu/arm/midgard/platform/rk/mali_kbase_config_rk.c (rejected hunks) ++@@ -434,7 +434,8 @@ static void kbase_platform_rk_remove_sysfs_files(struct device *dev) ++ static int rk3288_get_soc_info(struct device *dev, struct device_node *np, ++ int *bin, int *process) ++ { ++- int ret = -EINVAL, value = -EINVAL; +++ int ret = -EINVAL; +++ u8 value = 0; ++ char *name; ++ ++ if (!bin) ++@@ -445,7 +446,7 @@ static int rk3288_get_soc_info(struct device *dev, struct device_node *np, ++ else ++ name = "performance"; ++ if (of_property_match_string(np, "nvmem-cell-names", name) >= 0) { ++- ret = rockchip_get_efuse_value(np, name, &value); +++ ret = rockchip_nvmem_cell_read_u8(np, name, &value); ++ if (ret) { ++ dev_err(dev, "Failed to get soc performance value\n"); ++ goto out; +diff --git a/drivers/gpu/arm/midgard/platform/rk/mali_kbase_rk.h b/drivers/gpu/arm/midgard/platform/rk/mali_kbase_rk.h +new file mode 100755 +index 000000000000..6eab25014d21 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/platform/rk/mali_kbase_rk.h +@@ -0,0 +1,62 @@ ++/* drivers/gpu/t6xx/kbase/src/platform/rk/mali_kbase_platform.h ++ * Rockchip SoC Mali-Midgard platform-dependent codes ++ * ++ * This program is free software; you can redistribute it and/or modify ++ * it under the terms of the GNU General Public License version 2 as ++ * published by the Free Software FoundatIon. ++ */ ++ ++/** ++ * @file mali_kbase_rk.h ++ * ++ * defines work_context type of platform_dependent_part. ++ */ ++ ++#ifndef _MALI_KBASE_RK_H_ ++#define _MALI_KBASE_RK_H_ ++ ++#include ++ ++/*---------------------------------------------------------------------------*/ ++ ++#define DEFAULT_UTILISATION_PERIOD_IN_MS (100) ++ ++/*---------------------------------------------------------------------------*/ ++ ++/* ++ * struct rk_context - work_context of platform_dependent_part_of_rk. ++ */ ++struct rk_context { ++ /* ++ * record the status of common_parts calling 'power_on_callback' ++ * and 'power_off_callback'. ++ */ ++ bool is_powered; ++ ++ struct kbase_device *kbdev; ++ ++ struct workqueue_struct *power_off_wq; ++ /* delayed_work_to_power_off_gpu. */ ++ struct delayed_work work; ++ unsigned int delay_ms; ++ ++ /* ++ * WAKE_LOCK_SUSPEND for ensuring to run ++ * delayed_work_to_power_off_gpu before suspend. ++ */ ++ struct wake_lock wake_lock; ++ ++ /* debug only, the period in ms to count gpu_utilisation. */ ++ unsigned int utilisation_period; ++}; ++ ++/*---------------------------------------------------------------------------*/ ++ ++static inline struct rk_context *get_rk_context( ++ const struct kbase_device *kbdev) ++{ ++ return (struct rk_context *)(kbdev->platform_context); ++} ++ ++#endif /* _MALI_KBASE_RK_H_ */ ++ +diff --git a/drivers/gpu/arm/midgard/platform/vexpress/Kbuild b/drivers/gpu/arm/midgard/platform/vexpress/Kbuild +new file mode 100755 +index 000000000000..1caa293666d3 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/platform/vexpress/Kbuild +@@ -0,0 +1,18 @@ ++# ++# (C) COPYRIGHT 2012-2013, 2016 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++mali_kbase-y += \ ++ $(MALI_PLATFORM_THIRDPARTY_DIR)/mali_kbase_config_vexpress.o \ ++ $(MALI_PLATFORM_THIRDPARTY_DIR)/mali_kbase_cpu_vexpress.o +diff --git a/drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_config_platform.h b/drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_config_platform.h +new file mode 100755 +index 000000000000..02835f129aa3 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_config_platform.h +@@ -0,0 +1,75 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include "mali_kbase_cpu_vexpress.h" ++ ++/** ++ * Maximum frequency GPU will be clocked at. Given in kHz. ++ * This must be specified as there is no default value. ++ * ++ * Attached value: number in kHz ++ * Default value: NA ++ */ ++#define GPU_FREQ_KHZ_MAX kbase_get_platform_max_freq() ++/** ++ * Minimum frequency GPU will be clocked at. Given in kHz. ++ * This must be specified as there is no default value. ++ * ++ * Attached value: number in kHz ++ * Default value: NA ++ */ ++#define GPU_FREQ_KHZ_MIN kbase_get_platform_min_freq() ++ ++/** ++ * CPU_SPEED_FUNC - A pointer to a function that calculates the CPU clock ++ * ++ * CPU clock speed of the platform is in MHz - see kbase_cpu_clk_speed_func ++ * for the function prototype. ++ * ++ * Attached value: A kbase_cpu_clk_speed_func. ++ * Default Value: NA ++ */ ++#define CPU_SPEED_FUNC (&kbase_get_vexpress_cpu_clock_speed) ++ ++/** ++ * GPU_SPEED_FUNC - A pointer to a function that calculates the GPU clock ++ * ++ * GPU clock speed of the platform in MHz - see kbase_gpu_clk_speed_func ++ * for the function prototype. ++ * ++ * Attached value: A kbase_gpu_clk_speed_func. ++ * Default Value: NA ++ */ ++#define GPU_SPEED_FUNC (NULL) ++ ++/** ++ * Power management configuration ++ * ++ * Attached value: pointer to @ref kbase_pm_callback_conf ++ * Default value: See @ref kbase_pm_callback_conf ++ */ ++#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks) ++ ++/** ++ * Platform specific configuration functions ++ * ++ * Attached value: pointer to @ref kbase_platform_funcs_conf ++ * Default value: See @ref kbase_platform_funcs_conf ++ */ ++#define PLATFORM_FUNCS (NULL) ++ ++extern struct kbase_pm_callback_conf pm_callbacks; +diff --git a/drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_config_vexpress.c b/drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_config_vexpress.c +new file mode 100755 +index 000000000000..15ce2bc5eea5 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_config_vexpress.c +@@ -0,0 +1,85 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#include ++#include ++#include ++#include ++#include "mali_kbase_cpu_vexpress.h" ++#include "mali_kbase_config_platform.h" ++ ++#define HARD_RESET_AT_POWER_OFF 0 ++ ++#ifndef CONFIG_OF ++static struct kbase_io_resources io_resources = { ++ .job_irq_number = 68, ++ .mmu_irq_number = 69, ++ .gpu_irq_number = 70, ++ .io_memory_region = { ++ .start = 0xFC010000, ++ .end = 0xFC010000 + (4096 * 4) - 1 ++ } ++}; ++#endif /* CONFIG_OF */ ++ ++static int pm_callback_power_on(struct kbase_device *kbdev) ++{ ++ /* Nothing is needed on VExpress, but we may have destroyed GPU state (if the below HARD_RESET code is active) */ ++ return 1; ++} ++ ++static void pm_callback_power_off(struct kbase_device *kbdev) ++{ ++#if HARD_RESET_AT_POWER_OFF ++ /* Cause a GPU hard reset to test whether we have actually idled the GPU ++ * and that we properly reconfigure the GPU on power up. ++ * Usually this would be dangerous, but if the GPU is working correctly it should ++ * be completely safe as the GPU should not be active at this point. ++ * However this is disabled normally because it will most likely interfere with ++ * bus logging etc. ++ */ ++ KBASE_TRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, NULL, 0u, 0); ++ kbase_os_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_HARD_RESET); ++#endif ++} ++ ++struct kbase_pm_callback_conf pm_callbacks = { ++ .power_on_callback = pm_callback_power_on, ++ .power_off_callback = pm_callback_power_off, ++ .power_suspend_callback = NULL, ++ .power_resume_callback = NULL ++}; ++ ++static struct kbase_platform_config versatile_platform_config = { ++#ifndef CONFIG_OF ++ .io_resources = &io_resources ++#endif ++}; ++ ++struct kbase_platform_config *kbase_get_platform_config(void) ++{ ++ return &versatile_platform_config; ++} ++ ++ ++int kbase_platform_early_init(void) ++{ ++ /* Nothing needed at this stage */ ++ return 0; ++} +diff --git a/drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_cpu_vexpress.c b/drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_cpu_vexpress.c +new file mode 100755 +index 000000000000..4665f98cbbe4 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_cpu_vexpress.c +@@ -0,0 +1,279 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include ++#include ++#include "mali_kbase_cpu_vexpress.h" ++ ++#define HZ_IN_MHZ (1000000) ++ ++#define CORETILE_EXPRESS_A9X4_SCC_START (0x100E2000) ++#define MOTHERBOARD_SYS_CFG_START (0x10000000) ++#define SYS_CFGDATA_OFFSET (0x000000A0) ++#define SYS_CFGCTRL_OFFSET (0x000000A4) ++#define SYS_CFGSTAT_OFFSET (0x000000A8) ++ ++#define SYS_CFGCTRL_START_BIT_VALUE (1 << 31) ++#define READ_REG_BIT_VALUE (0 << 30) ++#define DCC_DEFAULT_BIT_VALUE (0 << 26) ++#define SYS_CFG_OSC_FUNC_BIT_VALUE (1 << 20) ++#define SITE_DEFAULT_BIT_VALUE (1 << 16) ++#define BOARD_STACK_POS_DEFAULT_BIT_VALUE (0 << 12) ++#define DEVICE_DEFAULT_BIT_VALUE (2 << 0) ++#define SYS_CFG_COMPLETE_BIT_VALUE (1 << 0) ++#define SYS_CFG_ERROR_BIT_VALUE (1 << 1) ++ ++#define FEED_REG_BIT_MASK (0x0F) ++#define FCLK_PA_DIVIDE_BIT_SHIFT (0x03) ++#define FCLK_PB_DIVIDE_BIT_SHIFT (0x07) ++#define FCLK_PC_DIVIDE_BIT_SHIFT (0x0B) ++#define AXICLK_PA_DIVIDE_BIT_SHIFT (0x0F) ++#define AXICLK_PB_DIVIDE_BIT_SHIFT (0x13) ++ ++/* the following three values used for reading ++ * HBI value of the LogicTile daughterboard */ ++#define VE_MOTHERBOARD_PERIPHERALS_SMB_CS7 (0x10000000) ++#define VE_SYS_PROC_ID1_OFFSET (0x00000088) ++#define VE_LOGIC_TILE_HBI_MASK (0x00000FFF) ++ ++#define IS_SINGLE_BIT_SET(val, pos) (val&(1<> ++ FCLK_PA_DIVIDE_BIT_SHIFT); ++ /* CFGRW0[10:7] */ ++ pb_divide = ((reg_val & (FEED_REG_BIT_MASK << ++ FCLK_PB_DIVIDE_BIT_SHIFT)) >> ++ FCLK_PB_DIVIDE_BIT_SHIFT); ++ *cpu_clock = osc2_value * (pa_divide + 1) / (pb_divide + 1); ++ } else if (IS_SINGLE_BIT_SET(reg_val, 1)) { ++ /* CFGRW0[1] - CLKOC */ ++ /* CFGRW0[6:3] */ ++ pa_divide = ((reg_val & (FEED_REG_BIT_MASK << ++ FCLK_PA_DIVIDE_BIT_SHIFT)) >> ++ FCLK_PA_DIVIDE_BIT_SHIFT); ++ /* CFGRW0[14:11] */ ++ pc_divide = ((reg_val & (FEED_REG_BIT_MASK << ++ FCLK_PC_DIVIDE_BIT_SHIFT)) >> ++ FCLK_PC_DIVIDE_BIT_SHIFT); ++ *cpu_clock = osc2_value * (pa_divide + 1) / (pc_divide + 1); ++ } else if (IS_SINGLE_BIT_SET(reg_val, 2)) { ++ /* CFGRW0[2] - FACLK */ ++ /* CFGRW0[18:15] */ ++ pa_divide = ((reg_val & (FEED_REG_BIT_MASK << ++ AXICLK_PA_DIVIDE_BIT_SHIFT)) >> ++ AXICLK_PA_DIVIDE_BIT_SHIFT); ++ /* CFGRW0[22:19] */ ++ pb_divide = ((reg_val & (FEED_REG_BIT_MASK << ++ AXICLK_PB_DIVIDE_BIT_SHIFT)) >> ++ AXICLK_PB_DIVIDE_BIT_SHIFT); ++ *cpu_clock = osc2_value * (pa_divide + 1) / (pb_divide + 1); ++ } else { ++ err = -EIO; ++ } ++ ++set_reg_error: ++ongoing_request: ++ raw_spin_unlock(&syscfg_lock); ++ *cpu_clock /= HZ_IN_MHZ; ++ ++ if (!err) ++ cpu_clock_speed = *cpu_clock; ++ ++ iounmap(scc_reg); ++ ++scc_reg_map_failed: ++ iounmap(syscfg_reg); ++ ++syscfg_reg_map_failed: ++ ++ return err; ++} ++ ++/** ++ * kbase_get_platform_logic_tile_type - determines which LogicTile type ++ * is used by Versatile Express ++ * ++ * When platform_config build parameter is specified as vexpress, i.e., ++ * platform_config=vexpress, GPU frequency may vary dependent on the ++ * particular platform. The GPU frequency depends on the LogicTile type. ++ * ++ * This function determines which LogicTile type is used by the platform by ++ * reading the HBI value of the daughterboard which holds the LogicTile: ++ * ++ * 0x217 HBI0217 Virtex-6 ++ * 0x192 HBI0192 Virtex-5 ++ * 0x247 HBI0247 Virtex-7 ++ * ++ * Return: HBI value of the logic tile daughterboard, zero if not accessible ++ */ ++static u32 kbase_get_platform_logic_tile_type(void) ++{ ++ void __iomem *syscfg_reg = NULL; ++ u32 sys_procid1 = 0; ++ ++ syscfg_reg = ioremap(VE_MOTHERBOARD_PERIPHERALS_SMB_CS7 + VE_SYS_PROC_ID1_OFFSET, 4); ++ if (NULL != syscfg_reg) { ++ sys_procid1 = readl(syscfg_reg); ++ iounmap(syscfg_reg); ++ } ++ ++ return sys_procid1 & VE_LOGIC_TILE_HBI_MASK; ++} ++ ++u32 kbase_get_platform_min_freq(void) ++{ ++ u32 ve_logic_tile = kbase_get_platform_logic_tile_type(); ++ ++ switch (ve_logic_tile) { ++ case 0x217: ++ /* Virtex 6, HBI0217 */ ++ return VE_VIRTEX6_GPU_FREQ_MIN; ++ case 0x247: ++ /* Virtex 7, HBI0247 */ ++ return VE_VIRTEX7_GPU_FREQ_MIN; ++ default: ++ /* all other logic tiles, i.e., Virtex 5 HBI0192 ++ * or unsuccessful reading from the platform - ++ * fall back to some default value */ ++ return VE_DEFAULT_GPU_FREQ_MIN; ++ } ++} ++ ++u32 kbase_get_platform_max_freq(void) ++{ ++ u32 ve_logic_tile = kbase_get_platform_logic_tile_type(); ++ ++ switch (ve_logic_tile) { ++ case 0x217: ++ /* Virtex 6, HBI0217 */ ++ return VE_VIRTEX6_GPU_FREQ_MAX; ++ case 0x247: ++ /* Virtex 7, HBI0247 */ ++ return VE_VIRTEX7_GPU_FREQ_MAX; ++ default: ++ /* all other logic tiles, i.e., Virtex 5 HBI0192 ++ * or unsuccessful reading from the platform - ++ * fall back to some default value */ ++ return VE_DEFAULT_GPU_FREQ_MAX; ++ } ++} +diff --git a/drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_cpu_vexpress.h b/drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_cpu_vexpress.h +new file mode 100755 +index 000000000000..da865698133a +--- /dev/null ++++ b/drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_cpu_vexpress.h +@@ -0,0 +1,38 @@ ++/* ++ * ++ * (C) COPYRIGHT 2012-2013, 2015-2016 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#ifndef _KBASE_CPU_VEXPRESS_H_ ++#define _KBASE_CPU_VEXPRESS_H_ ++ ++/** ++ * Versatile Express implementation of @ref kbase_cpu_clk_speed_func. ++ */ ++int kbase_get_vexpress_cpu_clock_speed(u32 *cpu_clock); ++ ++/** ++ * Get the minimum GPU frequency for the attached logic tile ++ */ ++u32 kbase_get_platform_min_freq(void); ++ ++/** ++ * Get the maximum GPU frequency for the attached logic tile ++ */ ++u32 kbase_get_platform_max_freq(void); ++ ++#endif /* _KBASE_CPU_VEXPRESS_H_ */ +diff --git a/drivers/gpu/arm/midgard/platform/vexpress_1xv7_a57/Kbuild b/drivers/gpu/arm/midgard/platform/vexpress_1xv7_a57/Kbuild +new file mode 100755 +index 000000000000..7efe8fa4263b +--- /dev/null ++++ b/drivers/gpu/arm/midgard/platform/vexpress_1xv7_a57/Kbuild +@@ -0,0 +1,16 @@ ++# ++# (C) COPYRIGHT 2013-2014, 2016 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++mali_kbase-y += $(MALI_PLATFORM_THIRDPARTY_DIR)/mali_kbase_config_vexpress.o +diff --git a/drivers/gpu/arm/midgard/platform/vexpress_1xv7_a57/mali_kbase_config_platform.h b/drivers/gpu/arm/midgard/platform/vexpress_1xv7_a57/mali_kbase_config_platform.h +new file mode 100755 +index 000000000000..0efbf3962f98 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/platform/vexpress_1xv7_a57/mali_kbase_config_platform.h +@@ -0,0 +1,73 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/** ++ * Maximum frequency GPU will be clocked at. Given in kHz. ++ * This must be specified as there is no default value. ++ * ++ * Attached value: number in kHz ++ * Default value: NA ++ */ ++#define GPU_FREQ_KHZ_MAX 5000 ++/** ++ * Minimum frequency GPU will be clocked at. Given in kHz. ++ * This must be specified as there is no default value. ++ * ++ * Attached value: number in kHz ++ * Default value: NA ++ */ ++#define GPU_FREQ_KHZ_MIN 5000 ++ ++/** ++ * CPU_SPEED_FUNC - A pointer to a function that calculates the CPU clock ++ * ++ * CPU clock speed of the platform is in MHz - see kbase_cpu_clk_speed_func ++ * for the function prototype. ++ * ++ * Attached value: A kbase_cpu_clk_speed_func. ++ * Default Value: NA ++ */ ++#define CPU_SPEED_FUNC (&kbase_cpuprops_get_default_clock_speed) ++ ++/** ++ * GPU_SPEED_FUNC - A pointer to a function that calculates the GPU clock ++ * ++ * GPU clock speed of the platform in MHz - see kbase_gpu_clk_speed_func ++ * for the function prototype. ++ * ++ * Attached value: A kbase_gpu_clk_speed_func. ++ * Default Value: NA ++ */ ++#define GPU_SPEED_FUNC (NULL) ++ ++/** ++ * Power management configuration ++ * ++ * Attached value: pointer to @ref kbase_pm_callback_conf ++ * Default value: See @ref kbase_pm_callback_conf ++ */ ++#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks) ++ ++/** ++ * Platform specific configuration functions ++ * ++ * Attached value: pointer to @ref kbase_platform_funcs_conf ++ * Default value: See @ref kbase_platform_funcs_conf ++ */ ++#define PLATFORM_FUNCS (NULL) ++ ++extern struct kbase_pm_callback_conf pm_callbacks; +diff --git a/drivers/gpu/arm/midgard/platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c b/drivers/gpu/arm/midgard/platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c +new file mode 100755 +index 000000000000..3ff0930fb4a3 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c +@@ -0,0 +1,79 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2014 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include ++#include ++#include ++#include ++ ++#define HARD_RESET_AT_POWER_OFF 0 ++ ++#ifndef CONFIG_OF ++static struct kbase_io_resources io_resources = { ++ .job_irq_number = 68, ++ .mmu_irq_number = 69, ++ .gpu_irq_number = 70, ++ .io_memory_region = { ++ .start = 0x2f010000, ++ .end = 0x2f010000 + (4096 * 4) - 1} ++}; ++#endif ++ ++static int pm_callback_power_on(struct kbase_device *kbdev) ++{ ++ /* Nothing is needed on VExpress, but we may have destroyed GPU state (if the below HARD_RESET code is active) */ ++ return 1; ++} ++ ++static void pm_callback_power_off(struct kbase_device *kbdev) ++{ ++#if HARD_RESET_AT_POWER_OFF ++ /* Cause a GPU hard reset to test whether we have actually idled the GPU ++ * and that we properly reconfigure the GPU on power up. ++ * Usually this would be dangerous, but if the GPU is working correctly it should ++ * be completely safe as the GPU should not be active at this point. ++ * However this is disabled normally because it will most likely interfere with ++ * bus logging etc. ++ */ ++ KBASE_TRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, NULL, 0u, 0); ++ kbase_os_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_HARD_RESET); ++#endif ++} ++ ++struct kbase_pm_callback_conf pm_callbacks = { ++ .power_on_callback = pm_callback_power_on, ++ .power_off_callback = pm_callback_power_off, ++ .power_suspend_callback = NULL, ++ .power_resume_callback = NULL ++}; ++ ++static struct kbase_platform_config versatile_platform_config = { ++#ifndef CONFIG_OF ++ .io_resources = &io_resources ++#endif ++}; ++ ++struct kbase_platform_config *kbase_get_platform_config(void) ++{ ++ return &versatile_platform_config; ++} ++ ++int kbase_platform_early_init(void) ++{ ++ /* Nothing needed at this stage */ ++ return 0; ++} +diff --git a/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/Kbuild b/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/Kbuild +new file mode 100755 +index 000000000000..1caa293666d3 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/Kbuild +@@ -0,0 +1,18 @@ ++# ++# (C) COPYRIGHT 2012-2013, 2016 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++mali_kbase-y += \ ++ $(MALI_PLATFORM_THIRDPARTY_DIR)/mali_kbase_config_vexpress.o \ ++ $(MALI_PLATFORM_THIRDPARTY_DIR)/mali_kbase_cpu_vexpress.o +diff --git a/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_platform.h b/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_platform.h +new file mode 100755 +index 000000000000..dbdf21e009f9 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_platform.h +@@ -0,0 +1,75 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include "mali_kbase_cpu_vexpress.h" ++ ++/** ++ * Maximum frequency GPU will be clocked at. Given in kHz. ++ * This must be specified as there is no default value. ++ * ++ * Attached value: number in kHz ++ * Default value: NA ++ */ ++#define GPU_FREQ_KHZ_MAX 10000 ++/** ++ * Minimum frequency GPU will be clocked at. Given in kHz. ++ * This must be specified as there is no default value. ++ * ++ * Attached value: number in kHz ++ * Default value: NA ++ */ ++#define GPU_FREQ_KHZ_MIN 10000 ++ ++/** ++ * CPU_SPEED_FUNC - A pointer to a function that calculates the CPU clock ++ * ++ * CPU clock speed of the platform is in MHz - see kbase_cpu_clk_speed_func ++ * for the function prototype. ++ * ++ * Attached value: A kbase_cpu_clk_speed_func. ++ * Default Value: NA ++ */ ++#define CPU_SPEED_FUNC (&kbase_get_vexpress_cpu_clock_speed) ++ ++/** ++ * GPU_SPEED_FUNC - A pointer to a function that calculates the GPU clock ++ * ++ * GPU clock speed of the platform in MHz - see kbase_gpu_clk_speed_func ++ * for the function prototype. ++ * ++ * Attached value: A kbase_gpu_clk_speed_func. ++ * Default Value: NA ++ */ ++#define GPU_SPEED_FUNC (NULL) ++ ++/** ++ * Power management configuration ++ * ++ * Attached value: pointer to @ref kbase_pm_callback_conf ++ * Default value: See @ref kbase_pm_callback_conf ++ */ ++#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks) ++ ++/** ++ * Platform specific configuration functions ++ * ++ * Attached value: pointer to @ref kbase_platform_funcs_conf ++ * Default value: See @ref kbase_platform_funcs_conf ++ */ ++#define PLATFORM_FUNCS (NULL) ++ ++extern struct kbase_pm_callback_conf pm_callbacks; +diff --git a/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_vexpress.c b/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_vexpress.c +new file mode 100755 +index 000000000000..76ffe4a1e59e +--- /dev/null ++++ b/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_vexpress.c +@@ -0,0 +1,83 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2014 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#include ++#include ++#include ++#include ++#include "mali_kbase_cpu_vexpress.h" ++ ++#define HARD_RESET_AT_POWER_OFF 0 ++ ++#ifndef CONFIG_OF ++static struct kbase_io_resources io_resources = { ++ .job_irq_number = 75, ++ .mmu_irq_number = 76, ++ .gpu_irq_number = 77, ++ .io_memory_region = { ++ .start = 0x2F000000, ++ .end = 0x2F000000 + (4096 * 4) - 1} ++}; ++#endif ++ ++static int pm_callback_power_on(struct kbase_device *kbdev) ++{ ++ /* Nothing is needed on VExpress, but we may have destroyed GPU state (if the below HARD_RESET code is active) */ ++ return 1; ++} ++ ++static void pm_callback_power_off(struct kbase_device *kbdev) ++{ ++#if HARD_RESET_AT_POWER_OFF ++ /* Cause a GPU hard reset to test whether we have actually idled the GPU ++ * and that we properly reconfigure the GPU on power up. ++ * Usually this would be dangerous, but if the GPU is working correctly it should ++ * be completely safe as the GPU should not be active at this point. ++ * However this is disabled normally because it will most likely interfere with ++ * bus logging etc. ++ */ ++ KBASE_TRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, NULL, 0u, 0); ++ kbase_os_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_HARD_RESET); ++#endif ++} ++ ++struct kbase_pm_callback_conf pm_callbacks = { ++ .power_on_callback = pm_callback_power_on, ++ .power_off_callback = pm_callback_power_off, ++ .power_suspend_callback = NULL, ++ .power_resume_callback = NULL ++}; ++ ++static struct kbase_platform_config versatile_platform_config = { ++#ifndef CONFIG_OF ++ .io_resources = &io_resources ++#endif ++}; ++ ++struct kbase_platform_config *kbase_get_platform_config(void) ++{ ++ return &versatile_platform_config; ++} ++ ++int kbase_platform_early_init(void) ++{ ++ /* Nothing needed at this stage */ ++ return 0; ++} ++ +diff --git a/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/mali_kbase_cpu_vexpress.c b/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/mali_kbase_cpu_vexpress.c +new file mode 100755 +index 000000000000..816dff49835f +--- /dev/null ++++ b/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/mali_kbase_cpu_vexpress.c +@@ -0,0 +1,71 @@ ++/* ++ * ++ * (C) COPYRIGHT 2011-2013 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++ ++ ++#include ++#include ++#include "mali_kbase_cpu_vexpress.h" ++ ++#define HZ_IN_MHZ (1000000) ++ ++#define CORETILE_EXPRESS_A9X4_SCC_START (0x100E2000) ++#define MOTHERBOARD_SYS_CFG_START (0x10000000) ++#define SYS_CFGDATA_OFFSET (0x000000A0) ++#define SYS_CFGCTRL_OFFSET (0x000000A4) ++#define SYS_CFGSTAT_OFFSET (0x000000A8) ++ ++#define SYS_CFGCTRL_START_BIT_VALUE (1 << 31) ++#define READ_REG_BIT_VALUE (0 << 30) ++#define DCC_DEFAULT_BIT_VALUE (0 << 26) ++#define SYS_CFG_OSC_FUNC_BIT_VALUE (1 << 20) ++#define SITE_DEFAULT_BIT_VALUE (1 << 16) ++#define BOARD_STACK_POS_DEFAULT_BIT_VALUE (0 << 12) ++#define DEVICE_DEFAULT_BIT_VALUE (2 << 0) ++#define SYS_CFG_COMPLETE_BIT_VALUE (1 << 0) ++#define SYS_CFG_ERROR_BIT_VALUE (1 << 1) ++ ++#define FEED_REG_BIT_MASK (0x0F) ++#define FCLK_PA_DIVIDE_BIT_SHIFT (0x03) ++#define FCLK_PB_DIVIDE_BIT_SHIFT (0x07) ++#define FCLK_PC_DIVIDE_BIT_SHIFT (0x0B) ++#define AXICLK_PA_DIVIDE_BIT_SHIFT (0x0F) ++#define AXICLK_PB_DIVIDE_BIT_SHIFT (0x13) ++ ++#define IS_SINGLE_BIT_SET(val, pos) (val&(1< ++ ++/** ++ * @addtogroup uk_api User-Kernel Interface API ++ * @{ ++ */ ++ ++/** ++ * @addtogroup uk_api_kernel UKK (Kernel side) ++ * @{ ++ */ ++ ++/** ++ * Internal OS specific data structure associated with each UKK session. Part ++ * of a ukk_session object. ++ */ ++typedef struct ukkp_session { ++ int dummy; /**< No internal OS specific data at this time */ ++} ukkp_session; ++ ++/** @} end group uk_api_kernel */ ++ ++/** @} end group uk_api */ ++ ++#endif /* _UKK_OS_H__ */ +diff --git a/drivers/gpu/arm/midgard/protected_mode_switcher.h b/drivers/gpu/arm/midgard/protected_mode_switcher.h +new file mode 100755 +index 000000000000..5dc2f3ba8cf6 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/protected_mode_switcher.h +@@ -0,0 +1,64 @@ ++/* ++ * ++ * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _PROTECTED_MODE_SWITCH_H_ ++#define _PROTECTED_MODE_SWITCH_H_ ++ ++struct protected_mode_device; ++ ++/** ++ * struct protected_mode_ops - Callbacks for protected mode switch operations ++ * ++ * @protected_mode_enable: Callback to enable protected mode for device ++ * @protected_mode_disable: Callback to disable protected mode for device ++ */ ++struct protected_mode_ops { ++ /** ++ * protected_mode_enable() - Enable protected mode on device ++ * @dev: The struct device ++ * ++ * Return: 0 on success, non-zero on error ++ */ ++ int (*protected_mode_enable)( ++ struct protected_mode_device *protected_dev); ++ ++ /** ++ * protected_mode_disable() - Disable protected mode on device, and ++ * reset device ++ * @dev: The struct device ++ * ++ * Return: 0 on success, non-zero on error ++ */ ++ int (*protected_mode_disable)( ++ struct protected_mode_device *protected_dev); ++}; ++ ++/** ++ * struct protected_mode_device - Device structure for protected mode devices ++ * ++ * @ops - Callbacks associated with this device ++ * @data - Pointer to device private data ++ * ++ * This structure should be registered with the platform device using ++ * platform_set_drvdata(). ++ */ ++struct protected_mode_device { ++ struct protected_mode_ops ops; ++ void *data; ++}; ++ ++#endif /* _PROTECTED_MODE_SWITCH_H_ */ +diff --git a/drivers/gpu/arm/midgard/rename.h b/drivers/gpu/arm/midgard/rename.h +new file mode 100755 +index 000000000000..c94b67ff2c27 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/rename.h +@@ -0,0 +1,422 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#ifndef _RENAME_H ++#define _RENAME_H ++#define __crc_kbase_create_context midgard___crc_kbase_create_context ++#define __crc_kbase_destroy_context midgard___crc_kbase_destroy_context ++#define __crc_kbase_find_device midgard___crc_kbase_find_device ++#define __crc_kbase_instr_hwcnt_clear midgard___crc_kbase_instr_hwcnt_clear ++#define __crc_kbase_instr_hwcnt_dump_complete midgard___crc_kbase_instr_hwcnt_dump_complete ++#define __crc_kbase_instr_hwcnt_request_dump midgard___crc_kbase_instr_hwcnt_request_dump ++#define __crc_kbase_release_device midgard___crc_kbase_release_device ++#define jd_done_nolock midgard_jd_done_nolock ++#define kbase_add_va_region midgard_kbase_add_va_region ++#define kbase_alloc_free_region midgard_kbase_alloc_free_region ++#define kbase_alloc_phy_pages_helper midgard_kbase_alloc_phy_pages_helper ++#define kbase_alloc_phy_pages midgard_kbase_alloc_phy_pages ++#define kbase_as_fault_debugfs_init midgard_kbase_as_fault_debugfs_init ++#define kbase_backend_complete_wq midgard_kbase_backend_complete_wq ++#define kbase_backend_complete_wq_post_sched midgard_kbase_backend_complete_wq_post_sched ++#define kbase_backend_ctx_count_changed midgard_kbase_backend_ctx_count_changed ++#define kbase_backend_find_and_release_free_address_space midgard_kbase_backend_find_and_release_free_address_space ++#define kbase_backend_get_current_flush_id midgard_kbase_backend_get_current_flush_id ++#define kbase_backend_get_gpu_time midgard_kbase_backend_get_gpu_time ++#define kbase_backend_gpuprops_get_features midgard_kbase_backend_gpuprops_get_features ++#define kbase_backend_gpuprops_get midgard_kbase_backend_gpuprops_get ++#define kbase_backend_inspect_tail midgard_kbase_backend_inspect_tail ++#define kbase_backend_nr_atoms_on_slot midgard_kbase_backend_nr_atoms_on_slot ++#define kbase_backend_nr_atoms_submitted midgard_kbase_backend_nr_atoms_submitted ++#define kbase_backend_release_ctx_irq midgard_kbase_backend_release_ctx_irq ++#define kbase_backend_release_ctx_noirq midgard_kbase_backend_release_ctx_noirq ++#define kbase_backend_reset midgard_kbase_backend_reset ++#define kbase_backend_run_atom midgard_kbase_backend_run_atom ++#define kbase_backend_slot_free midgard_kbase_backend_slot_free ++#define kbase_backend_slot_update midgard_kbase_backend_slot_update ++#define kbase_backend_soft_hard_stop_slot midgard_kbase_backend_soft_hard_stop_slot ++#define kbase_backend_timeouts_changed midgard_kbase_backend_timeouts_changed ++#define kbase_backend_timer_init midgard_kbase_backend_timer_init ++#define kbase_backend_timer_resume midgard_kbase_backend_timer_resume ++#define kbase_backend_timer_suspend midgard_kbase_backend_timer_suspend ++#define kbase_backend_timer_term midgard_kbase_backend_timer_term ++#define kbase_backend_use_ctx midgard_kbase_backend_use_ctx ++#define kbase_backend_use_ctx_sched midgard_kbase_backend_use_ctx_sched ++#define kbase_cache_enabled midgard_kbase_cache_enabled ++#define kbase_cache_set_coherency_mode midgard_kbase_cache_set_coherency_mode ++#define kbase_cancel_soft_job midgard_kbase_cancel_soft_job ++#define kbase_check_alloc_flags midgard_kbase_check_alloc_flags ++#define kbase_check_import_flags midgard_kbase_check_import_flags ++#define kbase_clean_caches_done midgard_kbase_clean_caches_done ++#define kbase_create_context midgard_kbase_create_context ++#define kbase_ctx_sched_init midgard_kbase_ctx_sched_init ++#define kbase_ctx_sched_release_ctx midgard_kbase_ctx_sched_release_ctx ++#define kbase_ctx_sched_remove_ctx midgard_kbase_ctx_sched_remove_ctx ++#define kbase_ctx_sched_restore_all_as midgard_kbase_ctx_sched_restore_all_as ++#define kbase_ctx_sched_retain_ctx midgard_kbase_ctx_sched_retain_ctx ++#define kbase_ctx_sched_retain_ctx_refcount midgard_kbase_ctx_sched_retain_ctx_refcount ++#define kbase_ctx_sched_term midgard_kbase_ctx_sched_term ++#define kbase_debug_assert_register_hook midgard_kbase_debug_assert_register_hook ++#define kbase_debug_job_fault_context_init midgard_kbase_debug_job_fault_context_init ++#define kbase_debug_job_fault_context_term midgard_kbase_debug_job_fault_context_term ++#define kbase_debug_job_fault_debugfs_init midgard_kbase_debug_job_fault_debugfs_init ++#define kbase_debug_job_fault_dev_init midgard_kbase_debug_job_fault_dev_init ++#define kbase_debug_job_fault_dev_term midgard_kbase_debug_job_fault_dev_term ++#define kbase_debug_job_fault_process midgard_kbase_debug_job_fault_process ++#define kbase_debug_job_fault_reg_snapshot_init midgard_kbase_debug_job_fault_reg_snapshot_init ++#define kbase_debug_mem_view_init midgard_kbase_debug_mem_view_init ++#define kbase_destroy_context midgard_kbase_destroy_context ++#define kbase_devfreq_init midgard_kbase_devfreq_init ++#define kbase_devfreq_set_core_mask midgard_kbase_devfreq_set_core_mask ++#define kbase_devfreq_term midgard_kbase_devfreq_term ++#define kbase_device_alloc midgard_kbase_device_alloc ++#define kbase_device_free midgard_kbase_device_free ++#define kbase_device_init midgard_kbase_device_init ++#define kbase_device_term midgard_kbase_device_term ++#define kbase_disjoint_event_get midgard_kbase_disjoint_event_get ++#define kbase_disjoint_event midgard_kbase_disjoint_event ++#define kbase_disjoint_event_potential midgard_kbase_disjoint_event_potential ++#define kbase_disjoint_init midgard_kbase_disjoint_init ++#define kbase_disjoint_state_down midgard_kbase_disjoint_state_down ++#define kbase_disjoint_state_up midgard_kbase_disjoint_state_up ++#define kbase_drv_name midgard_kbase_drv_name ++#define kbase_event_cleanup midgard_kbase_event_cleanup ++#define kbase_event_close midgard_kbase_event_close ++#define kbase_event_dequeue midgard_kbase_event_dequeue ++#define kbase_event_init midgard_kbase_event_init ++#define kbase_event_pending midgard_kbase_event_pending ++#define kbase_event_post midgard_kbase_event_post ++#define kbase_event_wakeup midgard_kbase_event_wakeup ++#define kbase_fence_add_callback midgard_kbase_fence_add_callback ++#define kbase_fence_free_callbacks midgard_kbase_fence_free_callbacks ++#define kbase_fence_ops midgard_kbase_fence_ops ++#define kbase_fence_out_new midgard_kbase_fence_out_new ++#define kbase_find_device midgard_kbase_find_device ++#define kbase_finish_soft_job midgard_kbase_finish_soft_job ++#define kbase_flush_mmu_wqs midgard_kbase_flush_mmu_wqs ++#define kbase_free_alloced_region midgard_kbase_free_alloced_region ++#define kbase_free_phy_pages_helper midgard_kbase_free_phy_pages_helper ++#define kbase_get_real_power midgard_kbase_get_real_power ++#define kbase_gpu_complete_hw midgard_kbase_gpu_complete_hw ++#define kbase_gpu_dump_slots midgard_kbase_gpu_dump_slots ++#define kbase_gpu_inspect midgard_kbase_gpu_inspect ++#define kbase_gpu_interrupt midgard_kbase_gpu_interrupt ++#define kbase_gpu_irq_evict midgard_kbase_gpu_irq_evict ++#define kbase_gpu_mmap midgard_kbase_gpu_mmap ++#define kbase_gpu_munmap midgard_kbase_gpu_munmap ++#define kbase_gpuprops_populate_user_buffer midgard_kbase_gpuprops_populate_user_buffer ++#define kbase_gpuprops_set_features midgard_kbase_gpuprops_set_features ++#define kbase_gpuprops_set midgard_kbase_gpuprops_set ++#define kbase_gpuprops_update_core_props_gpu_id midgard_kbase_gpuprops_update_core_props_gpu_id ++#define kbase_gpu_vm_lock midgard_kbase_gpu_vm_lock ++#define kbase_gpu_vm_unlock midgard_kbase_gpu_vm_unlock ++#define kbase_hwaccess_pm_gpu_active midgard_kbase_hwaccess_pm_gpu_active ++#define kbase_hwaccess_pm_gpu_idle midgard_kbase_hwaccess_pm_gpu_idle ++#define kbase_hwaccess_pm_halt midgard_kbase_hwaccess_pm_halt ++#define kbase_hwaccess_pm_init midgard_kbase_hwaccess_pm_init ++#define kbase_hwaccess_pm_powerup midgard_kbase_hwaccess_pm_powerup ++#define kbase_hwaccess_pm_resume midgard_kbase_hwaccess_pm_resume ++#define kbase_hwaccess_pm_suspend midgard_kbase_hwaccess_pm_suspend ++#define kbase_hwaccess_pm_term midgard_kbase_hwaccess_pm_term ++#define kbase_hw_set_features_mask midgard_kbase_hw_set_features_mask ++#define kbase_hw_set_issues_mask midgard_kbase_hw_set_issues_mask ++#define kbase_install_interrupts midgard_kbase_install_interrupts ++#define kbase_instr_backend_init midgard_kbase_instr_backend_init ++#define kbase_instr_backend_term midgard_kbase_instr_backend_term ++#define kbase_instr_hwcnt_clear midgard_kbase_instr_hwcnt_clear ++#define kbase_instr_hwcnt_disable_internal midgard_kbase_instr_hwcnt_disable_internal ++#define kbase_instr_hwcnt_dump_complete midgard_kbase_instr_hwcnt_dump_complete ++#define kbase_instr_hwcnt_enable_internal midgard_kbase_instr_hwcnt_enable_internal ++#define kbase_instr_hwcnt_request_dump midgard_kbase_instr_hwcnt_request_dump ++#define kbase_instr_hwcnt_sample_done midgard_kbase_instr_hwcnt_sample_done ++#define kbase_instr_hwcnt_wait_for_dump midgard_kbase_instr_hwcnt_wait_for_dump ++#define kbase_invoke_smc_fid midgard_kbase_invoke_smc_fid ++#define kbase_invoke_smc midgard_kbase_invoke_smc ++#define kbase_io_history_dump midgard_kbase_io_history_dump ++#define kbase_io_history_init midgard_kbase_io_history_init ++#define kbase_io_history_term midgard_kbase_io_history_term ++#define kbase_ipa_debugfs_init midgard_kbase_ipa_debugfs_init ++#define kbase_ipa_init midgard_kbase_ipa_init ++#define kbase_ipa_init_model midgard_kbase_ipa_init_model ++#define kbase_ipa_model_add_param_s32 midgard_kbase_ipa_model_add_param_s32 ++#define kbase_ipa_model_add_param_string midgard_kbase_ipa_model_add_param_string ++#define kbase_ipa_model_name_from_id midgard_kbase_ipa_model_name_from_id ++#define kbase_ipa_model_param_add midgard_kbase_ipa_model_param_add ++#define kbase_ipa_model_param_free_all midgard_kbase_ipa_model_param_free_all ++#define kbase_ipa_model_recalculate midgard_kbase_ipa_model_recalculate ++#define kbase_ipa_power_model_ops midgard_kbase_ipa_power_model_ops ++#define kbase_ipa_term midgard_kbase_ipa_term ++#define kbase_ipa_term_model midgard_kbase_ipa_term_model ++#define kbase_jd_cancel midgard_kbase_jd_cancel ++#define kbase_jd_done midgard_kbase_jd_done ++#define kbase_jd_done_worker midgard_kbase_jd_done_worker ++#define kbase_jd_exit midgard_kbase_jd_exit ++#define kbase_jd_free_external_resources midgard_kbase_jd_free_external_resources ++#define kbase_jd_init midgard_kbase_jd_init ++#define kbase_jd_submit midgard_kbase_jd_submit ++#define kbase_jd_zap_context midgard_kbase_jd_zap_context ++#define kbase_jit_allocate midgard_kbase_jit_allocate ++#define kbase_jit_backing_lost midgard_kbase_jit_backing_lost ++#define kbase_jit_debugfs_init midgard_kbase_jit_debugfs_init ++#define kbase_jit_evict midgard_kbase_jit_evict ++#define kbase_jit_free midgard_kbase_jit_free ++#define kbase_jit_init midgard_kbase_jit_init ++#define kbase_jit_term midgard_kbase_jit_term ++#define kbase_jm_complete midgard_kbase_jm_complete ++#define kbase_jm_idle_ctx midgard_kbase_jm_idle_ctx ++#define kbase_jm_kick midgard_kbase_jm_kick ++#define kbase_jm_return_atom_to_js midgard_kbase_jm_return_atom_to_js ++#define kbase_jm_try_kick_all midgard_kbase_jm_try_kick_all ++#define kbase_jm_try_kick midgard_kbase_jm_try_kick ++#define kbase_jm_wait_for_zero_jobs midgard_kbase_jm_wait_for_zero_jobs ++#define kbase_job_check_enter_disjoint midgard_kbase_job_check_enter_disjoint ++#define kbase_job_check_leave_disjoint midgard_kbase_job_check_leave_disjoint ++#define kbase_job_done midgard_kbase_job_done ++#define kbase_job_fault_get_reg_snapshot midgard_kbase_job_fault_get_reg_snapshot ++#define kbase_job_hw_submit midgard_kbase_job_hw_submit ++#define kbase_job_slot_ctx_priority_check_locked midgard_kbase_job_slot_ctx_priority_check_locked ++#define kbase_job_slot_halt midgard_kbase_job_slot_halt ++#define kbase_job_slot_hardstop midgard_kbase_job_slot_hardstop ++#define kbase_job_slot_init midgard_kbase_job_slot_init ++#define kbase_job_slot_softstop midgard_kbase_job_slot_softstop ++#define kbase_job_slot_softstop_swflags midgard_kbase_job_slot_softstop_swflags ++#define kbase_job_slot_term midgard_kbase_job_slot_term ++#define kbase_js_complete_atom midgard_kbase_js_complete_atom ++#define kbase_js_complete_atom_wq midgard_kbase_js_complete_atom_wq ++#define kbase_js_dep_resolved_submit midgard_kbase_js_dep_resolved_submit ++#define kbase_js_is_atom_valid midgard_kbase_js_is_atom_valid ++#define kbase_js_pull midgard_kbase_js_pull ++#define kbase_js_sched midgard_kbase_js_sched ++#define kbase_js_set_timeouts midgard_kbase_js_set_timeouts ++#define kbase_js_unpull midgard_kbase_js_unpull ++#define kbase_js_zap_context midgard_kbase_js_zap_context ++#define kbase_map_external_resource midgard_kbase_map_external_resource ++#define kbase_mem_alias midgard_kbase_mem_alias ++#define kbase_mem_alloc midgard_kbase_mem_alloc ++#define kbase_mem_alloc_page midgard_kbase_mem_alloc_page ++#define kbase_mem_commit midgard_kbase_mem_commit ++#define kbase_mem_evictable_deinit midgard_kbase_mem_evictable_deinit ++#define kbase_mem_evictable_init midgard_kbase_mem_evictable_init ++#define kbase_mem_evictable_make midgard_kbase_mem_evictable_make ++#define kbase_mem_evictable_unmake midgard_kbase_mem_evictable_unmake ++#define kbase_mem_flags_change midgard_kbase_mem_flags_change ++#define kbase_mem_free midgard_kbase_mem_free ++#define kbase_mem_free_region midgard_kbase_mem_free_region ++#define kbase_mem_grow_gpu_mapping midgard_kbase_mem_grow_gpu_mapping ++#define kbase_mem_halt midgard_kbase_mem_halt ++#define kbase_mem_import midgard_kbase_mem_import ++#define kbase_mem_init midgard_kbase_mem_init ++#define kbase_mem_kref_free midgard_kbase_mem_kref_free ++#define kbase_mem_pool_alloc midgard_kbase_mem_pool_alloc ++#define kbase_mem_pool_alloc_pages midgard_kbase_mem_pool_alloc_pages ++#define kbase_mem_pool_debugfs_init midgard_kbase_mem_pool_debugfs_init ++#define kbase_mem_pool_free midgard_kbase_mem_pool_free ++#define kbase_mem_pool_free_pages midgard_kbase_mem_pool_free_pages ++#define kbase_mem_pool_grow midgard_kbase_mem_pool_grow ++#define kbase_mem_pool_init midgard_kbase_mem_pool_init ++#define kbase_mem_pool_set_max_size midgard_kbase_mem_pool_set_max_size ++#define kbase_mem_pool_term midgard_kbase_mem_pool_term ++#define kbase_mem_pool_trim midgard_kbase_mem_pool_trim ++#define kbase_mem_query midgard_kbase_mem_query ++#define kbase_mem_term midgard_kbase_mem_term ++#define kbase_mmu_disable_as midgard_kbase_mmu_disable_as ++#define kbase_mmu_disable midgard_kbase_mmu_disable ++#define kbase_mmu_dump midgard_kbase_mmu_dump ++#define kbase_mmu_hw_clear_fault midgard_kbase_mmu_hw_clear_fault ++#define kbase_mmu_hw_configure midgard_kbase_mmu_hw_configure ++#define kbase_mmu_hw_do_operation midgard_kbase_mmu_hw_do_operation ++#define kbase_mmu_hw_enable_fault midgard_kbase_mmu_hw_enable_fault ++#define kbase_mmu_init midgard_kbase_mmu_init ++#define kbase_mmu_insert_pages midgard_kbase_mmu_insert_pages ++#define kbase_mmu_insert_pages_no_flush midgard_kbase_mmu_insert_pages_no_flush ++#define kbase_mmu_insert_single_page midgard_kbase_mmu_insert_single_page ++#define kbase_mmu_interrupt midgard_kbase_mmu_interrupt ++#define kbase_mmu_mode_get_aarch64 midgard_kbase_mmu_mode_get_aarch64 ++#define kbase_mmu_mode_get_lpae midgard_kbase_mmu_mode_get_lpae ++#define kbase_mmu_teardown_pages midgard_kbase_mmu_teardown_pages ++#define kbase_mmu_term midgard_kbase_mmu_term ++#define kbase_mmu_update midgard_kbase_mmu_update ++#define kbase_mmu_update_pages midgard_kbase_mmu_update_pages ++#define kbase_os_mem_map_lock midgard_kbase_os_mem_map_lock ++#define kbase_os_mem_map_unlock midgard_kbase_os_mem_map_unlock ++#define kbasep_cache_clean_worker midgard_kbasep_cache_clean_worker ++#define kbasep_common_test_interrupt_handlers midgard_kbasep_common_test_interrupt_handlers ++#define kbasep_complete_triggered_soft_events midgard_kbasep_complete_triggered_soft_events ++#define kbasep_debug_assert_call_hook midgard_kbasep_debug_assert_call_hook ++#define kbasep_find_enclosing_cpu_mapping_offset midgard_kbasep_find_enclosing_cpu_mapping_offset ++#define kbasep_gpu_memory_debugfs_init midgard_kbasep_gpu_memory_debugfs_init ++#define kbasep_jd_debugfs_ctx_init midgard_kbasep_jd_debugfs_ctx_init ++#define kbasep_job_slot_soft_or_hard_stop_do_action midgard_kbasep_job_slot_soft_or_hard_stop_do_action ++#define kbasep_js_add_job midgard_kbasep_js_add_job ++#define kbasep_js_atom_priority_to_relative midgard_kbasep_js_atom_priority_to_relative ++#define kbasep_js_ctx_attr_ctx_release_atom midgard_kbasep_js_ctx_attr_ctx_release_atom ++#define kbasep_js_ctx_attr_ctx_retain_atom midgard_kbasep_js_ctx_attr_ctx_retain_atom ++#define kbasep_js_ctx_attr_runpool_release_ctx midgard_kbasep_js_ctx_attr_runpool_release_ctx ++#define kbasep_js_ctx_attr_runpool_retain_ctx midgard_kbasep_js_ctx_attr_runpool_retain_ctx ++#define kbasep_js_devdata_halt midgard_kbasep_js_devdata_halt ++#define kbasep_js_devdata_init midgard_kbasep_js_devdata_init ++#define kbasep_js_devdata_term midgard_kbasep_js_devdata_term ++#define kbasep_js_kctx_init midgard_kbasep_js_kctx_init ++#define kbasep_js_kctx_term midgard_kbasep_js_kctx_term ++#define kbasep_js_relative_priority_to_atom midgard_kbasep_js_relative_priority_to_atom ++#define kbasep_js_release_privileged_ctx midgard_kbasep_js_release_privileged_ctx ++#define kbasep_js_remove_cancelled_job midgard_kbasep_js_remove_cancelled_job ++#define kbasep_js_remove_job midgard_kbasep_js_remove_job ++#define kbasep_js_resume midgard_kbasep_js_resume ++#define kbasep_js_runpool_release_ctx_and_katom_retained_state midgard_kbasep_js_runpool_release_ctx_and_katom_retained_state ++#define kbasep_js_runpool_release_ctx midgard_kbasep_js_runpool_release_ctx ++#define kbasep_js_runpool_release_ctx_nolock midgard_kbasep_js_runpool_release_ctx_nolock ++#define kbasep_js_runpool_requeue_or_kill_ctx midgard_kbasep_js_runpool_requeue_or_kill_ctx ++#define kbasep_js_schedule_privileged_ctx midgard_kbasep_js_schedule_privileged_ctx ++#define kbasep_js_suspend midgard_kbasep_js_suspend ++#define kbase_platform_early_init midgard_kbase_platform_early_init ++#define kbase_platform_rk_init_opp_table midgard_kbase_platform_rk_init_opp_table ++#define kbase_platform_rk_shutdown midgard_kbase_platform_rk_shutdown ++#define kbase_pm_always_on_policy_ops midgard_kbase_pm_always_on_policy_ops ++#define kbase_pm_cache_snoop_disable midgard_kbase_pm_cache_snoop_disable ++#define kbase_pm_cache_snoop_enable midgard_kbase_pm_cache_snoop_enable ++#define kbase_pm_ca_get_core_mask midgard_kbase_pm_ca_get_core_mask ++#define kbase_pm_ca_init midgard_kbase_pm_ca_init ++#define kbase_pm_ca_term midgard_kbase_pm_ca_term ++#define kbase_pm_clock_off midgard_kbase_pm_clock_off ++#define kbase_pm_clock_on midgard_kbase_pm_clock_on ++#define kbase_pm_coarse_demand_policy_ops midgard_kbase_pm_coarse_demand_policy_ops ++#define kbase_pm_context_active_handle_suspend midgard_kbase_pm_context_active_handle_suspend ++#define kbase_pm_context_active midgard_kbase_pm_context_active ++#define kbase_pm_context_idle midgard_kbase_pm_context_idle ++#define kbase_pm_disable_interrupts midgard_kbase_pm_disable_interrupts ++#define kbase_pm_disable_interrupts_nolock midgard_kbase_pm_disable_interrupts_nolock ++#define kbase_pm_do_poweroff midgard_kbase_pm_do_poweroff ++#define kbase_pm_do_poweron midgard_kbase_pm_do_poweron ++#define kbasep_mem_profile_debugfs_insert midgard_kbasep_mem_profile_debugfs_insert ++#define kbasep_mem_profile_debugfs_remove midgard_kbasep_mem_profile_debugfs_remove ++#define kbase_pm_enable_interrupts midgard_kbase_pm_enable_interrupts ++#define kbase_pm_get_active_cores midgard_kbase_pm_get_active_cores ++#define kbase_pm_get_policy midgard_kbase_pm_get_policy ++#define kbase_pm_get_present_cores midgard_kbase_pm_get_present_cores ++#define kbase_pm_get_ready_cores midgard_kbase_pm_get_ready_cores ++#define kbase_pm_get_trans_cores midgard_kbase_pm_get_trans_cores ++#define kbase_pm_halt midgard_kbase_pm_halt ++#define kbase_pm_init_hw midgard_kbase_pm_init_hw ++#define kbase_pm_list_policies midgard_kbase_pm_list_policies ++#define kbase_pm_metrics_update midgard_kbase_pm_metrics_update ++#define kbase_pm_policy_init midgard_kbase_pm_policy_init ++#define kbase_pm_policy_term midgard_kbase_pm_policy_term ++#define kbase_pm_power_changed midgard_kbase_pm_power_changed ++#define kbase_pm_powerup midgard_kbase_pm_powerup ++#define kbase_pm_register_access_disable midgard_kbase_pm_register_access_disable ++#define kbase_pm_register_access_enable midgard_kbase_pm_register_access_enable ++#define kbase_pm_release_gpu_cycle_counter midgard_kbase_pm_release_gpu_cycle_counter ++#define kbase_pm_release_gpu_cycle_counter_nolock midgard_kbase_pm_release_gpu_cycle_counter_nolock ++#define kbase_pm_request_gpu_cycle_counter_l2_is_on midgard_kbase_pm_request_gpu_cycle_counter_l2_is_on ++#define kbase_pm_request_gpu_cycle_counter midgard_kbase_pm_request_gpu_cycle_counter ++#define kbase_pm_reset_done midgard_kbase_pm_reset_done ++#define kbase_pm_resume midgard_kbase_pm_resume ++#define kbase_pm_set_debug_core_mask midgard_kbase_pm_set_debug_core_mask ++#define kbase_pm_set_policy midgard_kbase_pm_set_policy ++#define kbase_pm_suspend midgard_kbase_pm_suspend ++#define kbase_pm_update_active midgard_kbase_pm_update_active ++#define kbase_pm_update_cores_state midgard_kbase_pm_update_cores_state ++#define kbase_pm_update_cores_state_nolock midgard_kbase_pm_update_cores_state_nolock ++#define kbase_pm_wait_for_poweroff_complete midgard_kbase_pm_wait_for_poweroff_complete ++#define kbasep_os_process_page_usage_update midgard_kbasep_os_process_page_usage_update ++#define kbasep_platform_device_init midgard_kbasep_platform_device_init ++#define kbasep_platform_device_term midgard_kbasep_platform_device_term ++#define kbasep_pm_metrics_init midgard_kbasep_pm_metrics_init ++#define kbasep_pm_metrics_term midgard_kbasep_pm_metrics_term ++#define kbasep_regs_history_debugfs_init midgard_kbasep_regs_history_debugfs_init ++#define kbasep_remove_waiting_soft_job midgard_kbasep_remove_waiting_soft_job ++#define kbase_prepare_soft_job midgard_kbase_prepare_soft_job ++#define kbase_prepare_to_reset_gpu_locked midgard_kbase_prepare_to_reset_gpu_locked ++#define kbase_prepare_to_reset_gpu midgard_kbase_prepare_to_reset_gpu ++#define kbase_process_soft_job midgard_kbase_process_soft_job ++#define kbasep_soft_job_timeout_worker midgard_kbasep_soft_job_timeout_worker ++#define kbase_region_tracker_find_region_base_address midgard_kbase_region_tracker_find_region_base_address ++#define kbase_region_tracker_find_region_enclosing_address midgard_kbase_region_tracker_find_region_enclosing_address ++#define kbase_region_tracker_init_jit midgard_kbase_region_tracker_init_jit ++#define kbase_region_tracker_init midgard_kbase_region_tracker_init ++#define kbase_region_tracker_term midgard_kbase_region_tracker_term ++#define kbase_reg_read midgard_kbase_reg_read ++#define kbase_reg_write midgard_kbase_reg_write ++#define kbase_release_device midgard_kbase_release_device ++#define kbase_release_interrupts midgard_kbase_release_interrupts ++#define kbase_reset_gpu_locked midgard_kbase_reset_gpu_locked ++#define kbase_reset_gpu midgard_kbase_reset_gpu ++#define kbase_reset_gpu_silent midgard_kbase_reset_gpu_silent ++#define kbase_resume_suspended_soft_jobs midgard_kbase_resume_suspended_soft_jobs ++#define kbase_scale_static_power midgard_kbase_scale_static_power ++#define kbase_set_custom_irq_handler midgard_kbase_set_custom_irq_handler ++#define kbase_simple_ipa_model_ops midgard_kbase_simple_ipa_model_ops ++#define kbase_soft_event_update midgard_kbase_soft_event_update ++#define kbase_soft_event_wait_callback midgard_kbase_soft_event_wait_callback ++#define kbase_sticky_resource_acquire midgard_kbase_sticky_resource_acquire ++#define kbase_sticky_resource_init midgard_kbase_sticky_resource_init ++#define kbase_sticky_resource_release midgard_kbase_sticky_resource_release ++#define kbase_sticky_resource_term midgard_kbase_sticky_resource_term ++#define kbase_sync_fence_in_cancel_wait midgard_kbase_sync_fence_in_cancel_wait ++#define kbase_sync_fence_in_dump midgard_kbase_sync_fence_in_dump ++#define kbase_sync_fence_in_from_fd midgard_kbase_sync_fence_in_from_fd ++#define kbase_sync_fence_in_info_get midgard_kbase_sync_fence_in_info_get ++#define kbase_sync_fence_in_remove midgard_kbase_sync_fence_in_remove ++#define kbase_sync_fence_in_wait midgard_kbase_sync_fence_in_wait ++#define kbase_sync_fence_out_create midgard_kbase_sync_fence_out_create ++#define kbase_sync_fence_out_info_get midgard_kbase_sync_fence_out_info_get ++#define kbase_sync_fence_out_remove midgard_kbase_sync_fence_out_remove ++#define kbase_sync_fence_out_trigger midgard_kbase_sync_fence_out_trigger ++#define kbase_sync_fence_stream_create midgard_kbase_sync_fence_stream_create ++#define kbase_sync_fence_validate midgard_kbase_sync_fence_validate ++#define kbase_sync_fence_wait_worker midgard_kbase_sync_fence_wait_worker ++#define kbase_synchronize_irqs midgard_kbase_synchronize_irqs ++#define kbase_sync_now midgard_kbase_sync_now ++#define kbase_sync_single_for_cpu midgard_kbase_sync_single_for_cpu ++#define kbase_sync_single_for_device midgard_kbase_sync_single_for_device ++#define kbase_sync_single midgard_kbase_sync_single ++#define kbase_sync_status_string midgard_kbase_sync_status_string ++#define kbase_timeline_name midgard_kbase_timeline_name ++#define __kbase_tlstream_aux_devfreq_target midgard___kbase_tlstream_aux_devfreq_target ++#define __kbase_tlstream_aux_pagefault midgard___kbase_tlstream_aux_pagefault ++#define __kbase_tlstream_aux_pagesalloc midgard___kbase_tlstream_aux_pagesalloc ++#define __kbase_tlstream_aux_pm_state midgard___kbase_tlstream_aux_pm_state ++#define __kbase_tlstream_aux_protected_enter_end midgard___kbase_tlstream_aux_protected_enter_end ++#define __kbase_tlstream_aux_protected_enter_start midgard___kbase_tlstream_aux_protected_enter_start ++#define __kbase_tlstream_aux_protected_leave_end midgard___kbase_tlstream_aux_protected_leave_end ++#define __kbase_tlstream_aux_protected_leave_start midgard___kbase_tlstream_aux_protected_leave_start ++#define kbase_tlstream_init midgard_kbase_tlstream_init ++#define __kbase_tlstream_jd_gpu_soft_reset midgard___kbase_tlstream_jd_gpu_soft_reset ++#define kbase_tlstream_term midgard_kbase_tlstream_term ++#define __kbase_tlstream_tl_attrib_as_config midgard___kbase_tlstream_tl_attrib_as_config ++#define __kbase_tlstream_tl_attrib_atom_config midgard___kbase_tlstream_tl_attrib_atom_config ++#define __kbase_tlstream_tl_attrib_atom_jit midgard___kbase_tlstream_tl_attrib_atom_jit ++#define __kbase_tlstream_tl_attrib_atom_priority midgard___kbase_tlstream_tl_attrib_atom_priority ++#define __kbase_tlstream_tl_attrib_atom_state midgard___kbase_tlstream_tl_attrib_atom_state ++#define __kbase_tlstream_tl_del_atom midgard___kbase_tlstream_tl_del_atom ++#define __kbase_tlstream_tl_del_ctx midgard___kbase_tlstream_tl_del_ctx ++#define __kbase_tlstream_tl_event_atom_softstop_ex midgard___kbase_tlstream_tl_event_atom_softstop_ex ++#define __kbase_tlstream_tl_event_atom_softstop_issue midgard___kbase_tlstream_tl_event_atom_softstop_issue ++#define __kbase_tlstream_tl_event_lpu_softstop midgard___kbase_tlstream_tl_event_lpu_softstop ++#define __kbase_tlstream_tl_new_atom midgard___kbase_tlstream_tl_new_atom ++#define __kbase_tlstream_tl_new_ctx midgard___kbase_tlstream_tl_new_ctx ++#define __kbase_tlstream_tl_nret_as_ctx midgard___kbase_tlstream_tl_nret_as_ctx ++#define __kbase_tlstream_tl_nret_atom_as midgard___kbase_tlstream_tl_nret_atom_as ++#define __kbase_tlstream_tl_nret_atom_ctx midgard___kbase_tlstream_tl_nret_atom_ctx ++#define __kbase_tlstream_tl_nret_atom_lpu midgard___kbase_tlstream_tl_nret_atom_lpu ++#define __kbase_tlstream_tl_nret_ctx_lpu midgard___kbase_tlstream_tl_nret_ctx_lpu ++#define __kbase_tlstream_tl_ret_as_ctx midgard___kbase_tlstream_tl_ret_as_ctx ++#define __kbase_tlstream_tl_ret_atom_as midgard___kbase_tlstream_tl_ret_atom_as ++#define __kbase_tlstream_tl_ret_atom_ctx midgard___kbase_tlstream_tl_ret_atom_ctx ++#define __kbase_tlstream_tl_ret_atom_lpu midgard___kbase_tlstream_tl_ret_atom_lpu ++#define __kbase_tlstream_tl_ret_ctx_lpu midgard___kbase_tlstream_tl_ret_ctx_lpu ++#define kbase_unmap_external_resource midgard_kbase_unmap_external_resource ++#define kbase_update_region_flags midgard_kbase_update_region_flags ++#define kbase_vinstr_hwcnt_reader_setup midgard_kbase_vinstr_hwcnt_reader_setup ++#define kbase_vinstr_init midgard_kbase_vinstr_init ++#define kbase_vinstr_resume midgard_kbase_vinstr_resume ++#define kbase_vinstr_suspend midgard_kbase_vinstr_suspend ++#define kbase_vinstr_term midgard_kbase_vinstr_term ++#define kbase_vmap midgard_kbase_vmap ++#define kbase_vmap_prot midgard_kbase_vmap_prot ++#define kbase_vm_ops midgard_kbase_vm_ops ++#define kbase_vunmap midgard_kbase_vunmap ++#define _mali_profiling_control midgard__mali_profiling_control ++#define platform_funcs midgard_platform_funcs ++#define pm_callbacks midgard_pm_callbacks ++#define rk_kbase_device_runtime_disable midgard_rk_kbase_device_runtime_disable ++#define rk_kbase_device_runtime_init midgard_rk_kbase_device_runtime_init ++#endif +diff --git a/drivers/gpu/arm/midgard/sconscript b/drivers/gpu/arm/midgard/sconscript +new file mode 100755 +index 000000000000..ff23d7aebe6e +--- /dev/null ++++ b/drivers/gpu/arm/midgard/sconscript +@@ -0,0 +1,92 @@ ++# ++# (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++import sys ++Import('env') ++ ++SConscript( 'tests/sconscript' ) ++ ++mock_test = 0 ++ ++# Fake platform is a transient solution for GPL drivers running in kernel that does not provide configuration via platform data. ++# For such kernels fake_platform_device should be set to 1. For kernels providing platform data fake_platform_device should be set to 0. ++if env['platform_config']=='devicetree' or env['platform_config']=='juno_soc': ++ fake_platform_device = 0 ++else: ++ fake_platform_device = 1 ++ ++# Source files required for kbase. ++kbase_src = [ ++ Glob('*.c'), ++ Glob('backend/*/*.c'), ++ Glob('internal/*/*.c'), ++ Glob('ipa/*.c') ++] ++ ++if env['platform_config']=='juno_soc': ++ kbase_src += [Glob('platform/devicetree/*.c')] ++else: ++ kbase_src += [Glob('platform/%s/*.c' % env['platform_config'])] ++ ++if Glob('#kernel/drivers/gpu/arm/midgard/tests/internal/src/mock') and env['unit'] == '1': ++ kbase_src += [Glob('#kernel/drivers/gpu/arm/midgard/tests/internal/src/mock/*.c')] ++ mock_test = 1 ++ ++# we need platform config for GPL version using fake platform ++if fake_platform_device==1: ++ # Check if we are compiling for PBX ++ if env.KernelConfigEnabled("CONFIG_MACH_REALVIEW_PBX") and \ ++ env["platform_config"] in {"vexpress", "vexpress_6xvirtex7_10mhz"}: ++ sys.stderr.write("WARNING: Building for a PBX kernel but with platform_config=vexpress*\n") ++ # if the file platform config file is in the tpip directory then use that, otherwise use the default config directory ++ if Glob('#kernel/drivers/gpu/arm/midgard/config/tpip/*%s.c' % (env['platform_config'])): ++ kbase_src += Glob('#kernel/drivers/gpu/arm/midgard/config/tpip/*%s.c' % (env['platform_config'])) ++ else: ++ kbase_src += Glob('#kernel/drivers/gpu/arm/midgard/config/*%s.c' % (env['platform_config'])) ++ ++make_args = env.kernel_get_config_defines(ret_list = True, ++ fake = fake_platform_device) + [ ++ 'PLATFORM=%s' % env['platform'], ++ 'MALI_ERROR_INJECT_ON=%s' % env['error_inject'], ++ 'MALI_KERNEL_TEST_API=%s' % env['debug'], ++ 'MALI_UNIT_TEST=%s' % env['unit'], ++ 'MALI_RELEASE_NAME=%s' % env['mali_release_name'], ++ 'MALI_MOCK_TEST=%s' % mock_test, ++ 'MALI_CUSTOMER_RELEASE=%s' % env['release'], ++ 'MALI_INSTRUMENTATION_LEVEL=%s' % env['instr'], ++ 'MALI_COVERAGE=%s' % env['coverage'], ++ 'MALI_BUS_LOG=%s' % env['buslog'] ++] ++ ++kbase = env.BuildKernelModule('$STATIC_LIB_PATH/mali_kbase.ko', kbase_src, ++ make_args = make_args) ++ ++# Add a dependency on kds.ko. ++# Only necessary when KDS is not built into the kernel. ++# ++if env['os'] != 'android': ++ if not env.KernelConfigEnabled("CONFIG_KDS"): ++ env.Depends(kbase, '$STATIC_LIB_PATH/kds.ko') ++ ++# need Module.symvers from ump.ko build ++if int(env['ump']) == 1: ++ env.Depends(kbase, '$STATIC_LIB_PATH/ump.ko') ++ ++if 'smc_protected_mode_switcher' in env: ++ env.Depends('$STATIC_LIB_PATH/mali_kbase.ko', '$STATIC_LIB_PATH/smc_protected_mode_switcher.ko') ++ ++env.KernelObjTarget('kbase', kbase) ++ ++env.AppendUnique(BASE=['cutils_linked_list']) +diff --git a/drivers/gpu/arm/midgard/tests/Kbuild b/drivers/gpu/arm/midgard/tests/Kbuild +new file mode 100755 +index 000000000000..b4bed0473439 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/tests/Kbuild +@@ -0,0 +1,17 @@ ++# ++# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++obj-$(CONFIG_MALI_KUTF) += kutf/ ++obj-$(CONFIG_MALI_IRQ_LATENCY) += mali_kutf_irq_test/ +diff --git a/drivers/gpu/arm/midgard/tests/Kconfig b/drivers/gpu/arm/midgard/tests/Kconfig +new file mode 100755 +index 000000000000..da0515c065de +--- /dev/null ++++ b/drivers/gpu/arm/midgard/tests/Kconfig +@@ -0,0 +1,17 @@ ++# ++# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++source "drivers/gpu/arm/midgard/tests/kutf/Kconfig" ++source "drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Kconfig" +diff --git a/drivers/gpu/arm/midgard/tests/include/kutf/kutf_mem.h b/drivers/gpu/arm/midgard/tests/include/kutf/kutf_mem.h +new file mode 100755 +index 000000000000..0d145e42a0ca +--- /dev/null ++++ b/drivers/gpu/arm/midgard/tests/include/kutf/kutf_mem.h +@@ -0,0 +1,65 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KERNEL_UTF_MEM_H_ ++#define _KERNEL_UTF_MEM_H_ ++ ++/* kutf_mem.h ++ * Functions for management of memory pools in the kernel. ++ * ++ * This module implements a memory pool allocator, allowing a test ++ * implementation to allocate linked allocations which can then be freed by a ++ * single free which releases all of the resources held by the entire pool. ++ * ++ * Note that it is not possible to free single resources within the pool once ++ * allocated. ++ */ ++ ++#include ++ ++/** ++ * struct kutf_mempool - the memory pool context management structure ++ * @head: list head on which the allocations in this context are added to ++ * ++ */ ++struct kutf_mempool { ++ struct list_head head; ++}; ++ ++/** ++ * kutf_mempool_init() - Initialize a memory pool. ++ * @pool: Memory pool structure to initialize, provided by the user ++ * ++ * Return: zero on success ++ */ ++int kutf_mempool_init(struct kutf_mempool *pool); ++ ++/** ++ * kutf_mempool_alloc() - Allocate memory from a pool ++ * @pool: Memory pool to allocate from ++ * @size: Size of memory wanted in number of bytes ++ * ++ * Return: Pointer to memory on success, NULL on failure. ++ */ ++void *kutf_mempool_alloc(struct kutf_mempool *pool, size_t size); ++ ++/** ++ * kutf_mempool_destroy() - Destroy a memory pool, freeing all memory within it. ++ * @pool: The memory pool to free ++ */ ++void kutf_mempool_destroy(struct kutf_mempool *pool); ++#endif /* _KERNEL_UTF_MEM_H_ */ +diff --git a/drivers/gpu/arm/midgard/tests/include/kutf/kutf_resultset.h b/drivers/gpu/arm/midgard/tests/include/kutf/kutf_resultset.h +new file mode 100755 +index 000000000000..1cc85f1b7a46 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/tests/include/kutf/kutf_resultset.h +@@ -0,0 +1,121 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KERNEL_UTF_RESULTSET_H_ ++#define _KERNEL_UTF_RESULTSET_H_ ++ ++/* kutf_resultset.h ++ * Functions and structures for handling test results and result sets. ++ * ++ * This section of the kernel UTF contains structures and functions used for the ++ * management of Results and Result Sets. ++ */ ++ ++/** ++ * enum kutf_result_status - Status values for a single Test error. ++ * @KUTF_RESULT_BENCHMARK: Result is a meta-result containing benchmark ++ * results. ++ * @KUTF_RESULT_SKIP: The test was skipped. ++ * @KUTF_RESULT_UNKNOWN: The test has an unknown result. ++ * @KUTF_RESULT_PASS: The test result passed. ++ * @KUTF_RESULT_DEBUG: The test result passed, but raised a debug ++ * message. ++ * @KUTF_RESULT_INFO: The test result passed, but raised ++ * an informative message. ++ * @KUTF_RESULT_WARN: The test result passed, but raised a warning ++ * message. ++ * @KUTF_RESULT_FAIL: The test result failed with a non-fatal error. ++ * @KUTF_RESULT_FATAL: The test result failed with a fatal error. ++ * @KUTF_RESULT_ABORT: The test result failed due to a non-UTF ++ * assertion failure. ++ * @KUTF_RESULT_COUNT: The current number of possible status messages. ++ */ ++enum kutf_result_status { ++ KUTF_RESULT_BENCHMARK = -3, ++ KUTF_RESULT_SKIP = -2, ++ KUTF_RESULT_UNKNOWN = -1, ++ ++ KUTF_RESULT_PASS = 0, ++ KUTF_RESULT_DEBUG = 1, ++ KUTF_RESULT_INFO = 2, ++ KUTF_RESULT_WARN = 3, ++ KUTF_RESULT_FAIL = 4, ++ KUTF_RESULT_FATAL = 5, ++ KUTF_RESULT_ABORT = 6, ++ ++ KUTF_RESULT_COUNT ++}; ++ ++/* The maximum size of a kutf_result_status result when ++ * converted to a string ++ */ ++#define KUTF_ERROR_MAX_NAME_SIZE 21 ++ ++#ifdef __KERNEL__ ++ ++#include ++ ++/** ++ * struct kutf_result - Represents a single test result. ++ * @node: Next result in the list of results. ++ * @status: The status summary (pass / warn / fail / etc). ++ * @message: A more verbose status message. ++ */ ++struct kutf_result { ++ struct list_head node; ++ enum kutf_result_status status; ++ const char *message; ++}; ++ ++/** ++ * kutf_create_result_set() - Create a new result set ++ * to which results can be added. ++ * ++ * Return: The created resultset. ++ */ ++struct kutf_result_set *kutf_create_result_set(void); ++ ++/** ++ * kutf_add_result() - Add a result to the end of an existing resultset. ++ * ++ * @mempool: The memory pool to allocate the result storage from. ++ * @set: The resultset to add the result to. ++ * @status: The result status to add. ++ * @message: The result message to add. ++ */ ++void kutf_add_result(struct kutf_mempool *mempool, struct kutf_result_set *set, ++ enum kutf_result_status status, const char *message); ++ ++/** ++ * kutf_remove_result() - Remove a result from the head of a resultset. ++ * @set: The resultset. ++ * ++ * Return: result or NULL if there are no further results in the resultset. ++ */ ++struct kutf_result *kutf_remove_result( ++ struct kutf_result_set *set); ++ ++/** ++ * kutf_destroy_result_set() - Free a previously created resultset. ++ * ++ * @results: The result set whose resources to free. ++ */ ++void kutf_destroy_result_set(struct kutf_result_set *results); ++ ++#endif /* __KERNEL__ */ ++ ++#endif /* _KERNEL_UTF_RESULTSET_H_ */ +diff --git a/drivers/gpu/arm/midgard/tests/include/kutf/kutf_suite.h b/drivers/gpu/arm/midgard/tests/include/kutf/kutf_suite.h +new file mode 100755 +index 000000000000..754c3adb1cca +--- /dev/null ++++ b/drivers/gpu/arm/midgard/tests/include/kutf/kutf_suite.h +@@ -0,0 +1,508 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KERNEL_UTF_SUITE_H_ ++#define _KERNEL_UTF_SUITE_H_ ++ ++/* kutf_suite.h ++ * Functions for management of test suites. ++ * ++ * This collection of data structures, macros, and functions are used to ++ * create Test Suites, Tests within those Test Suites, and Fixture variants ++ * of each test. ++ */ ++ ++#include ++#include ++ ++/** ++ * Pseudo-flag indicating an absence of any specified test class. Note that ++ * tests should not be annotated with this constant as it is simply a zero ++ * value; tests without a more specific class must be marked with the flag ++ * KUTF_F_TEST_GENERIC. ++ */ ++#define KUTF_F_TEST_NONE ((unsigned int)(0)) ++ ++/** ++ * Class indicating this test is a smoke test. ++ * A given set of smoke tests should be quick to run, enabling rapid turn-around ++ * of "regress-on-commit" test runs. ++ */ ++#define KUTF_F_TEST_SMOKETEST ((unsigned int)(1 << 1)) ++ ++/** ++ * Class indicating this test is a performance test. ++ * These tests typically produce a performance metric, such as "time to run" or ++ * "frames per second", ++ */ ++#define KUTF_F_TEST_PERFORMANCE ((unsigned int)(1 << 2)) ++ ++/** ++ * Class indicating that this test is a deprecated test. ++ * These tests have typically been replaced by an alternative test which is ++ * more efficient, or has better coverage. ++ */ ++#define KUTF_F_TEST_DEPRECATED ((unsigned int)(1 << 3)) ++ ++/** ++ * Class indicating that this test is a known failure. ++ * These tests have typically been run and failed, but marking them as a known ++ * failure means it is easier to triage results. ++ * ++ * It is typically more convenient to triage known failures using the ++ * results database and web UI, as this means there is no need to modify the ++ * test code. ++ */ ++#define KUTF_F_TEST_EXPECTED_FAILURE ((unsigned int)(1 << 4)) ++ ++/** ++ * Class indicating that this test is a generic test, which is not a member of ++ * a more specific test class. Tests which are not created with a specific set ++ * of filter flags by the user are assigned this test class by default. ++ */ ++#define KUTF_F_TEST_GENERIC ((unsigned int)(1 << 5)) ++ ++/** ++ * Class indicating this test is a resource allocation failure test. ++ * A resource allocation failure test will test that an error code is ++ * correctly propagated when an allocation fails. ++ */ ++#define KUTF_F_TEST_RESFAIL ((unsigned int)(1 << 6)) ++ ++/** ++ * Additional flag indicating that this test is an expected failure when ++ * run in resource failure mode. These tests are never run when running ++ * the low resource mode. ++ */ ++#define KUTF_F_TEST_EXPECTED_FAILURE_RF ((unsigned int)(1 << 7)) ++ ++/** ++ * Flag reserved for user-defined filter zero. ++ */ ++#define KUTF_F_TEST_USER_0 ((unsigned int)(1 << 24)) ++ ++/** ++ * Flag reserved for user-defined filter one. ++ */ ++#define KUTF_F_TEST_USER_1 ((unsigned int)(1 << 25)) ++ ++/** ++ * Flag reserved for user-defined filter two. ++ */ ++#define KUTF_F_TEST_USER_2 ((unsigned int)(1 << 26)) ++ ++/** ++ * Flag reserved for user-defined filter three. ++ */ ++#define KUTF_F_TEST_USER_3 ((unsigned int)(1 << 27)) ++ ++/** ++ * Flag reserved for user-defined filter four. ++ */ ++#define KUTF_F_TEST_USER_4 ((unsigned int)(1 << 28)) ++ ++/** ++ * Flag reserved for user-defined filter five. ++ */ ++#define KUTF_F_TEST_USER_5 ((unsigned int)(1 << 29)) ++ ++/** ++ * Flag reserved for user-defined filter six. ++ */ ++#define KUTF_F_TEST_USER_6 ((unsigned int)(1 << 30)) ++ ++/** ++ * Flag reserved for user-defined filter seven. ++ */ ++#define KUTF_F_TEST_USER_7 ((unsigned int)(1 << 31)) ++ ++/** ++ * Pseudo-flag indicating that all test classes should be executed. ++ */ ++#define KUTF_F_TEST_ALL ((unsigned int)(0xFFFFFFFFU)) ++ ++/** ++ * union kutf_callback_data - Union used to store test callback data ++ * @ptr_value: pointer to the location where test callback data ++ * are stored ++ * @u32_value: a number which represents test callback data ++ */ ++union kutf_callback_data { ++ void *ptr_value; ++ u32 u32_value; ++}; ++ ++/** ++ * struct kutf_context - Structure representing a kernel test context ++ * @suite: Convenience pointer to the suite this context ++ * is running ++ * @test_fix: The fixture that is being run in this context ++ * @fixture_pool: The memory pool used for the duration of ++ * the fixture/text context. ++ * @fixture: The user provided fixture structure. ++ * @fixture_index: The index (id) of the current fixture. ++ * @fixture_name: The name of the current fixture (or NULL if unnamed). ++ * @test_data: Any user private data associated with this test ++ * @result_set: All the results logged by this test context ++ * @status: The status of the currently running fixture. ++ * @expected_status: The expected status on exist of the currently ++ * running fixture. ++ */ ++struct kutf_context { ++ struct kutf_suite *suite; ++ struct kutf_test_fixture *test_fix; ++ struct kutf_mempool fixture_pool; ++ void *fixture; ++ unsigned int fixture_index; ++ const char *fixture_name; ++ union kutf_callback_data test_data; ++ struct kutf_result_set *result_set; ++ enum kutf_result_status status; ++ enum kutf_result_status expected_status; ++}; ++ ++/** ++ * struct kutf_suite - Structure representing a kernel test suite ++ * @app: The application this suite belongs to. ++ * @name: The name of this suite. ++ * @suite_data: Any user private data associated with this ++ * suite. ++ * @create_fixture: Function used to create a new fixture instance ++ * @remove_fixture: Function used to destroy a new fixture instance ++ * @fixture_variants: The number of variants (must be at least 1). ++ * @suite_default_flags: Suite global filter flags which are set on ++ * all tests. ++ * @node: List node for suite_list ++ * @dir: The debugfs directory for this suite ++ * @test_list: List head to store all the tests which are ++ * part of this suite ++ */ ++struct kutf_suite { ++ struct kutf_application *app; ++ const char *name; ++ union kutf_callback_data suite_data; ++ void *(*create_fixture)(struct kutf_context *context); ++ void (*remove_fixture)(struct kutf_context *context); ++ unsigned int fixture_variants; ++ unsigned int suite_default_flags; ++ struct list_head node; ++ struct dentry *dir; ++ struct list_head test_list; ++}; ++ ++/* ============================================================================ ++ Application functions ++============================================================================ */ ++ ++/** ++ * kutf_create_application() - Create an in kernel test application. ++ * @name: The name of the test application. ++ * ++ * Return: pointer to the kutf_application on success or NULL ++ * on failure ++ */ ++struct kutf_application *kutf_create_application(const char *name); ++ ++/** ++ * kutf_destroy_application() - Destroy an in kernel test application. ++ * ++ * @app: The test application to destroy. ++ */ ++void kutf_destroy_application(struct kutf_application *app); ++ ++/* ============================================================================ ++ Suite functions ++============================================================================ */ ++ ++/** ++ * kutf_create_suite() - Create a kernel test suite. ++ * @app: The test application to create the suite in. ++ * @name: The name of the suite. ++ * @fixture_count: The number of fixtures to run over the test ++ * functions in this suite ++ * @create_fixture: Callback used to create a fixture. The returned value ++ * is stored in the fixture pointer in the context for ++ * use in the test functions. ++ * @remove_fixture: Callback used to remove a previously created fixture. ++ * ++ * Suite names must be unique. Should two suites with the same name be ++ * registered with the same application then this function will fail, if they ++ * are registered with different applications then the function will not detect ++ * this and the call will succeed. ++ * ++ * Return: pointer to the created kutf_suite on success or NULL ++ * on failure ++ */ ++struct kutf_suite *kutf_create_suite( ++ struct kutf_application *app, ++ const char *name, ++ unsigned int fixture_count, ++ void *(*create_fixture)(struct kutf_context *context), ++ void (*remove_fixture)(struct kutf_context *context)); ++ ++/** ++ * kutf_create_suite_with_filters() - Create a kernel test suite with user ++ * defined default filters. ++ * @app: The test application to create the suite in. ++ * @name: The name of the suite. ++ * @fixture_count: The number of fixtures to run over the test ++ * functions in this suite ++ * @create_fixture: Callback used to create a fixture. The returned value ++ * is stored in the fixture pointer in the context for ++ * use in the test functions. ++ * @remove_fixture: Callback used to remove a previously created fixture. ++ * @filters: Filters to apply to a test if it doesn't provide its own ++ * ++ * Suite names must be unique. Should two suites with the same name be ++ * registered with the same application then this function will fail, if they ++ * are registered with different applications then the function will not detect ++ * this and the call will succeed. ++ * ++ * Return: pointer to the created kutf_suite on success or NULL on failure ++ */ ++struct kutf_suite *kutf_create_suite_with_filters( ++ struct kutf_application *app, ++ const char *name, ++ unsigned int fixture_count, ++ void *(*create_fixture)(struct kutf_context *context), ++ void (*remove_fixture)(struct kutf_context *context), ++ unsigned int filters); ++ ++/** ++ * kutf_create_suite_with_filters_and_data() - Create a kernel test suite with ++ * user defined default filters. ++ * @app: The test application to create the suite in. ++ * @name: The name of the suite. ++ * @fixture_count: The number of fixtures to run over the test ++ * functions in this suite ++ * @create_fixture: Callback used to create a fixture. The returned value ++ * is stored in the fixture pointer in the context for ++ * use in the test functions. ++ * @remove_fixture: Callback used to remove a previously created fixture. ++ * @filters: Filters to apply to a test if it doesn't provide its own ++ * @suite_data: Suite specific callback data, provided during the ++ * running of the test in the kutf_context ++ * ++ * Return: pointer to the created kutf_suite on success or NULL ++ * on failure ++ */ ++struct kutf_suite *kutf_create_suite_with_filters_and_data( ++ struct kutf_application *app, ++ const char *name, ++ unsigned int fixture_count, ++ void *(*create_fixture)(struct kutf_context *context), ++ void (*remove_fixture)(struct kutf_context *context), ++ unsigned int filters, ++ union kutf_callback_data suite_data); ++ ++/** ++ * kutf_add_test() - Add a test to a kernel test suite. ++ * @suite: The suite to add the test to. ++ * @id: The ID of the test. ++ * @name: The name of the test. ++ * @execute: Callback to the test function to run. ++ * ++ * Note: As no filters are provided the test will use the suite filters instead ++ */ ++void kutf_add_test(struct kutf_suite *suite, ++ unsigned int id, ++ const char *name, ++ void (*execute)(struct kutf_context *context)); ++ ++/** ++ * kutf_add_test_with_filters() - Add a test to a kernel test suite with filters ++ * @suite: The suite to add the test to. ++ * @id: The ID of the test. ++ * @name: The name of the test. ++ * @execute: Callback to the test function to run. ++ * @filters: A set of filtering flags, assigning test categories. ++ */ ++void kutf_add_test_with_filters(struct kutf_suite *suite, ++ unsigned int id, ++ const char *name, ++ void (*execute)(struct kutf_context *context), ++ unsigned int filters); ++ ++/** ++ * kutf_add_test_with_filters_and_data() - Add a test to a kernel test suite ++ * with filters. ++ * @suite: The suite to add the test to. ++ * @id: The ID of the test. ++ * @name: The name of the test. ++ * @execute: Callback to the test function to run. ++ * @filters: A set of filtering flags, assigning test categories. ++ * @test_data: Test specific callback data, provoided during the ++ * running of the test in the kutf_context ++ */ ++void kutf_add_test_with_filters_and_data( ++ struct kutf_suite *suite, ++ unsigned int id, ++ const char *name, ++ void (*execute)(struct kutf_context *context), ++ unsigned int filters, ++ union kutf_callback_data test_data); ++ ++/* ============================================================================ ++ Test functions ++============================================================================ */ ++/** ++ * kutf_test_log_result_external() - Log a result which has been created ++ * externally into a in a standard form ++ * recognized by the log parser. ++ * @context: The test context the test is running in ++ * @message: The message for this result ++ * @new_status: The result status of this log message ++ */ ++void kutf_test_log_result_external( ++ struct kutf_context *context, ++ const char *message, ++ enum kutf_result_status new_status); ++ ++/** ++ * kutf_test_expect_abort() - Tell the kernel that you expect the current ++ * fixture to produce an abort. ++ * @context: The test context this test is running in. ++ */ ++void kutf_test_expect_abort(struct kutf_context *context); ++ ++/** ++ * kutf_test_expect_fatal() - Tell the kernel that you expect the current ++ * fixture to produce a fatal error. ++ * @context: The test context this test is running in. ++ */ ++void kutf_test_expect_fatal(struct kutf_context *context); ++ ++/** ++ * kutf_test_expect_fail() - Tell the kernel that you expect the current ++ * fixture to fail. ++ * @context: The test context this test is running in. ++ */ ++void kutf_test_expect_fail(struct kutf_context *context); ++ ++/** ++ * kutf_test_expect_warn() - Tell the kernel that you expect the current ++ * fixture to produce a warning. ++ * @context: The test context this test is running in. ++ */ ++void kutf_test_expect_warn(struct kutf_context *context); ++ ++/** ++ * kutf_test_expect_pass() - Tell the kernel that you expect the current ++ * fixture to pass. ++ * @context: The test context this test is running in. ++ */ ++void kutf_test_expect_pass(struct kutf_context *context); ++ ++/** ++ * kutf_test_skip() - Tell the kernel that the test should be skipped. ++ * @context: The test context this test is running in. ++ */ ++void kutf_test_skip(struct kutf_context *context); ++ ++/** ++ * kutf_test_skip_msg() - Tell the kernel that this test has been skipped, ++ * supplying a reason string. ++ * @context: The test context this test is running in. ++ * @message: A message string containing the reason for the skip. ++ * ++ * Note: The message must not be freed during the lifetime of the test run. ++ * This means it should either be a prebaked string, or if a dynamic string ++ * is required it must be created with kutf_dsprintf which will store ++ * the resultant string in a buffer who's lifetime is the same as the test run. ++ */ ++void kutf_test_skip_msg(struct kutf_context *context, const char *message); ++ ++/** ++ * kutf_test_pass() - Tell the kernel that this test has passed. ++ * @context: The test context this test is running in. ++ * @message: A message string containing the reason for the pass. ++ * ++ * Note: The message must not be freed during the lifetime of the test run. ++ * This means it should either be a pre-baked string, or if a dynamic string ++ * is required it must be created with kutf_dsprintf which will store ++ * the resultant string in a buffer who's lifetime is the same as the test run. ++ */ ++void kutf_test_pass(struct kutf_context *context, char const *message); ++ ++/** ++ * kutf_test_debug() - Send a debug message ++ * @context: The test context this test is running in. ++ * @message: A message string containing the debug information. ++ * ++ * Note: The message must not be freed during the lifetime of the test run. ++ * This means it should either be a pre-baked string, or if a dynamic string ++ * is required it must be created with kutf_dsprintf which will store ++ * the resultant string in a buffer who's lifetime is the same as the test run. ++ */ ++void kutf_test_debug(struct kutf_context *context, char const *message); ++ ++/** ++ * kutf_test_info() - Send an information message ++ * @context: The test context this test is running in. ++ * @message: A message string containing the information message. ++ * ++ * Note: The message must not be freed during the lifetime of the test run. ++ * This means it should either be a pre-baked string, or if a dynamic string ++ * is required it must be created with kutf_dsprintf which will store ++ * the resultant string in a buffer who's lifetime is the same as the test run. ++ */ ++void kutf_test_info(struct kutf_context *context, char const *message); ++ ++/** ++ * kutf_test_warn() - Send a warning message ++ * @context: The test context this test is running in. ++ * @message: A message string containing the warning message. ++ * ++ * Note: The message must not be freed during the lifetime of the test run. ++ * This means it should either be a pre-baked string, or if a dynamic string ++ * is required it must be created with kutf_dsprintf which will store ++ * the resultant string in a buffer who's lifetime is the same as the test run. ++ */ ++void kutf_test_warn(struct kutf_context *context, char const *message); ++ ++/** ++ * kutf_test_fail() - Tell the kernel that a test has failed ++ * @context: The test context this test is running in. ++ * @message: A message string containing the failure message. ++ * ++ * Note: The message must not be freed during the lifetime of the test run. ++ * This means it should either be a pre-baked string, or if a dynamic string ++ * is required it must be created with kutf_dsprintf which will store ++ * the resultant string in a buffer who's lifetime is the same as the test run. ++ */ ++void kutf_test_fail(struct kutf_context *context, char const *message); ++ ++/** ++ * kutf_test_fatal() - Tell the kernel that a test has triggered a fatal error ++ * @context: The test context this test is running in. ++ * @message: A message string containing the fatal error message. ++ * ++ * Note: The message must not be freed during the lifetime of the test run. ++ * This means it should either be a pre-baked string, or if a dynamic string ++ * is required it must be created with kutf_dsprintf which will store ++ * the resultant string in a buffer who's lifetime is the same as the test run. ++ */ ++void kutf_test_fatal(struct kutf_context *context, char const *message); ++ ++/** ++ * kutf_test_abort() - Tell the kernel that a test triggered an abort in the test ++ * ++ * @context: The test context this test is running in. ++ */ ++void kutf_test_abort(struct kutf_context *context); ++ ++#endif /* _KERNEL_UTF_SUITE_H_ */ +diff --git a/drivers/gpu/arm/midgard/tests/include/kutf/kutf_utils.h b/drivers/gpu/arm/midgard/tests/include/kutf/kutf_utils.h +new file mode 100755 +index 000000000000..c458c1f73802 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/tests/include/kutf/kutf_utils.h +@@ -0,0 +1,55 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#ifndef _KERNEL_UTF_UTILS_H_ ++#define _KERNEL_UTF_UTILS_H_ ++ ++/* kutf_utils.h ++ * Utilities for the kernel UTF test infrastructure. ++ * ++ * This collection of library functions are provided for use by kernel UTF ++ * and users of kernel UTF which don't directly fit within the other ++ * code modules. ++ */ ++ ++#include ++ ++/** ++ * Maximum size of the message strings within kernel UTF, messages longer then ++ * this will be truncated. ++ */ ++#define KUTF_MAX_DSPRINTF_LEN 1024 ++ ++/** ++ * kutf_dsprintf() - dynamic sprintf ++ * @pool: memory pool to allocate from ++ * @fmt: The format string describing the string to document. ++ * @... The parameters to feed in to the format string. ++ * ++ * This function implements sprintf which dynamically allocates memory to store ++ * the string. The library will free the memory containing the string when the ++ * result set is cleared or destroyed. ++ * ++ * Note The returned string may be truncated to fit an internal temporary ++ * buffer, which is KUTF_MAX_DSPRINTF_LEN bytes in length. ++ * ++ * Return: Returns pointer to allocated string, or NULL on error. ++ */ ++const char *kutf_dsprintf(struct kutf_mempool *pool, ++ const char *fmt, ...); ++ ++#endif /* _KERNEL_UTF_UTILS_H_ */ +diff --git a/drivers/gpu/arm/midgard/tests/kutf/Kbuild b/drivers/gpu/arm/midgard/tests/kutf/Kbuild +new file mode 100755 +index 000000000000..6b840c2ef7b7 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/tests/kutf/Kbuild +@@ -0,0 +1,20 @@ ++# ++# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++ccflags-y += -I$(src)/../include ++ ++obj-$(CONFIG_MALI_KUTF) += kutf.o ++ ++kutf-y := kutf_mem.o kutf_resultset.o kutf_suite.o kutf_utils.o +diff --git a/drivers/gpu/arm/midgard/tests/kutf/Kconfig b/drivers/gpu/arm/midgard/tests/kutf/Kconfig +new file mode 100755 +index 000000000000..84364716afe3 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/tests/kutf/Kconfig +@@ -0,0 +1,22 @@ ++# ++# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++ ++config MALI_KUTF ++ tristate "Mali Kernel Unit Test Framework" ++ default n ++ help ++ Enables MALI testing framework. To compile it as a module, ++ choose M here - this will generate a single module called kutf. +diff --git a/drivers/gpu/arm/midgard/tests/kutf/Makefile b/drivers/gpu/arm/midgard/tests/kutf/Makefile +new file mode 100755 +index 000000000000..010c92ca39b9 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/tests/kutf/Makefile +@@ -0,0 +1,29 @@ ++# ++# (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++# linux build system bootstrap for out-of-tree module ++ ++# default to building for the host ++ARCH ?= $(shell uname -m) ++ ++ifeq ($(KDIR),) ++$(error Must specify KDIR to point to the kernel to target)) ++endif ++ ++all: ++ $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) $(SCONS_CONFIGS) EXTRA_CFLAGS=-I$(CURDIR)/../include modules ++ ++clean: ++ $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) clean +diff --git a/drivers/gpu/arm/midgard/tests/kutf/kutf_mem.c b/drivers/gpu/arm/midgard/tests/kutf/kutf_mem.c +new file mode 100755 +index 000000000000..5408e57d469a +--- /dev/null ++++ b/drivers/gpu/arm/midgard/tests/kutf/kutf_mem.c +@@ -0,0 +1,94 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* Kernel UTF memory management functions */ ++ ++#include ++#include ++#include ++ ++#include ++ ++ ++/** ++ * struct kutf_alloc_entry - Structure representing an allocation. ++ * @node: List node for use with kutf_mempool. ++ * @data: Data area of the allocation ++ */ ++struct kutf_alloc_entry { ++ struct list_head node; ++ u8 data[0]; ++}; ++ ++int kutf_mempool_init(struct kutf_mempool *pool) ++{ ++ if (!pool) { ++ pr_err("NULL pointer passed to %s\n", __func__); ++ return -1; ++ } ++ ++ INIT_LIST_HEAD(&pool->head); ++ ++ return 0; ++} ++EXPORT_SYMBOL(kutf_mempool_init); ++ ++void kutf_mempool_destroy(struct kutf_mempool *pool) ++{ ++ struct list_head *remove; ++ struct list_head *tmp; ++ ++ if (!pool) { ++ pr_err("NULL pointer passed to %s\n", __func__); ++ return; ++ } ++ ++ list_for_each_safe(remove, tmp, &pool->head) { ++ struct kutf_alloc_entry *remove_alloc; ++ ++ remove_alloc = list_entry(remove, struct kutf_alloc_entry, node); ++ list_del(&remove_alloc->node); ++ kfree(remove_alloc); ++ } ++} ++EXPORT_SYMBOL(kutf_mempool_destroy); ++ ++void *kutf_mempool_alloc(struct kutf_mempool *pool, size_t size) ++{ ++ struct kutf_alloc_entry *ret; ++ ++ if (!pool) { ++ pr_err("NULL pointer passed to %s\n", __func__); ++ goto fail_pool; ++ } ++ ++ ret = kmalloc(sizeof(*ret) + size, GFP_KERNEL); ++ if (!ret) { ++ pr_err("Failed to allocate memory\n"); ++ goto fail_alloc; ++ } ++ ++ INIT_LIST_HEAD(&ret->node); ++ list_add(&ret->node, &pool->head); ++ ++ return &ret->data[0]; ++ ++fail_alloc: ++fail_pool: ++ return NULL; ++} ++EXPORT_SYMBOL(kutf_mempool_alloc); +diff --git a/drivers/gpu/arm/midgard/tests/kutf/kutf_resultset.c b/drivers/gpu/arm/midgard/tests/kutf/kutf_resultset.c +new file mode 100755 +index 000000000000..5bd04969fd55 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/tests/kutf/kutf_resultset.c +@@ -0,0 +1,95 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* Kernel UTF result management functions */ ++ ++#include ++#include ++#include ++ ++#include ++ ++/** ++ * struct kutf_result_set - Represents a set of results. ++ * @results: Pointer to the linked list where the results are stored. ++ */ ++struct kutf_result_set { ++ struct list_head results; ++}; ++ ++struct kutf_result_set *kutf_create_result_set(void) ++{ ++ struct kutf_result_set *set; ++ ++ set = kmalloc(sizeof(*set), GFP_KERNEL); ++ if (!set) { ++ pr_err("Failed to allocate resultset"); ++ goto fail_alloc; ++ } ++ ++ INIT_LIST_HEAD(&set->results); ++ ++ return set; ++ ++fail_alloc: ++ return NULL; ++} ++ ++void kutf_add_result(struct kutf_mempool *mempool, ++ struct kutf_result_set *set, ++ enum kutf_result_status status, ++ const char *message) ++{ ++ /* Create the new result */ ++ struct kutf_result *new_result; ++ ++ BUG_ON(set == NULL); ++ ++ new_result = kutf_mempool_alloc(mempool, sizeof(*new_result)); ++ if (!new_result) { ++ pr_err("Result allocation failed\n"); ++ return; ++ } ++ ++ INIT_LIST_HEAD(&new_result->node); ++ new_result->status = status; ++ new_result->message = message; ++ ++ list_add_tail(&new_result->node, &set->results); ++} ++ ++void kutf_destroy_result_set(struct kutf_result_set *set) ++{ ++ if (!list_empty(&set->results)) ++ pr_err("kutf_destroy_result_set: Unread results from test\n"); ++ ++ kfree(set); ++} ++ ++struct kutf_result *kutf_remove_result(struct kutf_result_set *set) ++{ ++ if (!list_empty(&set->results)) { ++ struct kutf_result *ret; ++ ++ ret = list_first_entry(&set->results, struct kutf_result, node); ++ list_del(&ret->node); ++ return ret; ++ } ++ ++ return NULL; ++} ++ +diff --git a/drivers/gpu/arm/midgard/tests/kutf/kutf_suite.c b/drivers/gpu/arm/midgard/tests/kutf/kutf_suite.c +new file mode 100755 +index 000000000000..a7cfd3be9c46 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/tests/kutf/kutf_suite.c +@@ -0,0 +1,1041 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* Kernel UTF suite, test and fixture management including user to kernel ++ * interaction */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++ ++#include ++#include ++#include ++ ++#if defined(CONFIG_DEBUG_FS) ++ ++/** ++ * struct kutf_application - Structure which represents kutf application ++ * @name: The name of this test application. ++ * @dir: The debugfs directory for this test ++ * @suite_list: List head to store all the suites which are part of this ++ * application ++ */ ++struct kutf_application { ++ const char *name; ++ struct dentry *dir; ++ struct list_head suite_list; ++}; ++ ++/** ++ * struct kutf_test_function - Structure which represents kutf test function ++ * @suite: Back reference to the suite this test function ++ * belongs to ++ * @filters: Filters that apply to this test function ++ * @test_id: Test ID ++ * @execute: Function to run for this test ++ * @test_data: Static data for this test ++ * @node: List node for test_list ++ * @variant_list: List head to store all the variants which can run on ++ * this function ++ * @dir: debugfs directory for this test function ++ */ ++struct kutf_test_function { ++ struct kutf_suite *suite; ++ unsigned int filters; ++ unsigned int test_id; ++ void (*execute)(struct kutf_context *context); ++ union kutf_callback_data test_data; ++ struct list_head node; ++ struct list_head variant_list; ++ struct dentry *dir; ++}; ++ ++/** ++ * struct kutf_test_fixture - Structure which holds information on the kutf ++ * test fixture ++ * @test_func: Test function this fixture belongs to ++ * @fixture_index: Index of this fixture ++ * @node: List node for variant_list ++ * @dir: debugfs directory for this test fixture ++ */ ++struct kutf_test_fixture { ++ struct kutf_test_function *test_func; ++ unsigned int fixture_index; ++ struct list_head node; ++ struct dentry *dir; ++}; ++ ++struct dentry *base_dir; ++ ++/** ++ * struct kutf_convert_table - Structure which keeps test results ++ * @result_name: Status of the test result ++ * @result: Status value for a single test ++ */ ++struct kutf_convert_table { ++ char result_name[50]; ++ enum kutf_result_status result; ++}; ++ ++struct kutf_convert_table kutf_convert[] = { ++#define ADD_UTF_RESULT(_name) \ ++{ \ ++ #_name, \ ++ _name, \ ++}, ++ADD_UTF_RESULT(KUTF_RESULT_BENCHMARK) ++ADD_UTF_RESULT(KUTF_RESULT_SKIP) ++ADD_UTF_RESULT(KUTF_RESULT_UNKNOWN) ++ADD_UTF_RESULT(KUTF_RESULT_PASS) ++ADD_UTF_RESULT(KUTF_RESULT_DEBUG) ++ADD_UTF_RESULT(KUTF_RESULT_INFO) ++ADD_UTF_RESULT(KUTF_RESULT_WARN) ++ADD_UTF_RESULT(KUTF_RESULT_FAIL) ++ADD_UTF_RESULT(KUTF_RESULT_FATAL) ++ADD_UTF_RESULT(KUTF_RESULT_ABORT) ++}; ++ ++#define UTF_CONVERT_SIZE (ARRAY_SIZE(kutf_convert)) ++ ++/** ++ * kutf_create_context() - Create a test context in which a specific fixture ++ * of an application will be run and its results ++ * reported back to the user ++ * @test_fix: Test fixture to be run. ++ * ++ * Return: Returns the created test context on success or NULL on failure ++ */ ++static struct kutf_context *kutf_create_context( ++ struct kutf_test_fixture *test_fix); ++ ++/** ++ * kutf_destroy_context() - Destroy a previously created test context ++ * @context: Test context to destroy ++ */ ++static void kutf_destroy_context(struct kutf_context *context); ++ ++/** ++ * kutf_set_result() - Set the test result against the specified test context ++ * @context: Test context ++ * @status: Result status ++ */ ++static void kutf_set_result(struct kutf_context *context, ++ enum kutf_result_status status); ++ ++/** ++ * kutf_set_expected_result() - Set the expected test result for the specified ++ * test context ++ * @context: Test context ++ * @expected_status: Expected result status ++ */ ++static void kutf_set_expected_result(struct kutf_context *context, ++ enum kutf_result_status expected_status); ++ ++#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 4, 0)) ++/* Pre 3.4.0 kernels don't have the simple_open helper */ ++ ++/** ++ * simple_open() - Helper for file opening which stores the inode private data ++ * into the file private data ++ * @inode: File entry representation ++ * @file: A specific opening of the file ++ * ++ * Return: always 0; if inode private data do not exist, the file will not ++ * be assigned private data ++ */ ++static int simple_open(struct inode *inode, struct file *file) ++{ ++ if (inode->i_private) ++ file->private_data = inode->i_private; ++ return 0; ++} ++#endif ++ ++/** ++ * kutf_result_to_string() - Converts a KUTF result into a string ++ * @result_str: Output result string ++ * @result: Result status to convert ++ * ++ * Return: 1 if test result was successfully converted to string, 0 otherwise ++ */ ++static int kutf_result_to_string(char **result_str, ++ enum kutf_result_status result) ++{ ++ int i; ++ int ret = 0; ++ ++ for (i = 0; i < UTF_CONVERT_SIZE; i++) { ++ if (result == kutf_convert[i].result) { ++ *result_str = kutf_convert[i].result_name; ++ ret = 1; ++ } ++ } ++ return ret; ++} ++ ++/** ++ * kutf_debugfs_const_string_read() - Simple debugfs read callback which ++ * returns a constant string ++ * @file: Opened file to read from ++ * @buf: User buffer to write the data into ++ * @len: Amount of data to read ++ * @ppos: Offset into file to read from ++ * ++ * Return: On success, the number of bytes read and offset @ppos advanced by ++ * this number; on error, negative value ++ */ ++static ssize_t kutf_debugfs_const_string_read(struct file *file, ++ char __user *buf, size_t len, loff_t *ppos) ++{ ++ char *str = file->private_data; ++ ++ return simple_read_from_buffer(buf, len, ppos, str, strlen(str)); ++} ++ ++static const struct file_operations kutf_debugfs_const_string_ops = { ++ .owner = THIS_MODULE, ++ .open = simple_open, ++ .read = kutf_debugfs_const_string_read, ++ .llseek = default_llseek, ++}; ++ ++/** ++ * kutf_add_explicit_result() - Check if an explicit result needs to be added ++ * @context: KUTF test context ++ */ ++static void kutf_add_explicit_result(struct kutf_context *context) ++{ ++ switch (context->expected_status) { ++ case KUTF_RESULT_UNKNOWN: ++ if (context->status == KUTF_RESULT_UNKNOWN) ++ kutf_test_pass(context, "(implicit pass)"); ++ break; ++ ++ case KUTF_RESULT_WARN: ++ if (context->status == KUTF_RESULT_WARN) ++ kutf_test_pass(context, ++ "Pass (expected warn occurred)"); ++ else if (context->status != KUTF_RESULT_SKIP) ++ kutf_test_fail(context, ++ "Fail (expected warn missing)"); ++ break; ++ ++ case KUTF_RESULT_FAIL: ++ if (context->status == KUTF_RESULT_FAIL) ++ kutf_test_pass(context, ++ "Pass (expected fail occurred)"); ++ else if (context->status != KUTF_RESULT_SKIP) { ++ /* Force the expected status so the fail gets logged */ ++ context->expected_status = KUTF_RESULT_PASS; ++ kutf_test_fail(context, ++ "Fail (expected fail missing)"); ++ } ++ break; ++ ++ case KUTF_RESULT_FATAL: ++ if (context->status == KUTF_RESULT_FATAL) ++ kutf_test_pass(context, ++ "Pass (expected fatal occurred)"); ++ else if (context->status != KUTF_RESULT_SKIP) ++ kutf_test_fail(context, ++ "Fail (expected fatal missing)"); ++ break; ++ ++ case KUTF_RESULT_ABORT: ++ if (context->status == KUTF_RESULT_ABORT) ++ kutf_test_pass(context, ++ "Pass (expected abort occurred)"); ++ else if (context->status != KUTF_RESULT_SKIP) ++ kutf_test_fail(context, ++ "Fail (expected abort missing)"); ++ break; ++ default: ++ break; ++ } ++} ++ ++/** ++ * kutf_debugfs_run_open() Debugfs open callback for the "run" entry. ++ * @inode: inode of the opened file ++ * @file: Opened file to read from ++ * ++ * This function retrieves the test fixture data that is associated with the ++ * opened file and works back to get the test, suite and application so ++ * it can then run the test that is associated with the file entry. ++ * ++ * Return: 0 on success ++ */ ++static int kutf_debugfs_run_open(struct inode *inode, struct file *file) ++{ ++ struct kutf_test_fixture *test_fix = inode->i_private; ++ struct kutf_test_function *test_func = test_fix->test_func; ++ struct kutf_suite *suite = test_func->suite; ++ struct kutf_context *test_context; ++ ++ test_context = kutf_create_context(test_fix); ++ if (!test_context) ++ return -ENODEV; ++ ++ file->private_data = test_context; ++ ++ /* ++ * Call the create fixture function if required before the ++ * fixture is run ++ */ ++ if (suite->create_fixture) ++ test_context->fixture = suite->create_fixture(test_context); ++ ++ /* Only run the test if the fixture was created (if required) */ ++ if ((suite->create_fixture && test_context->fixture) || ++ (!suite->create_fixture)) { ++ /* Run this fixture */ ++ test_func->execute(test_context); ++ ++ if (suite->remove_fixture) ++ suite->remove_fixture(test_context); ++ ++ kutf_add_explicit_result(test_context); ++ } ++ return 0; ++} ++ ++/** ++ * kutf_debugfs_run_read() - Debugfs read callback for the "run" entry. ++ * @file: Opened file to read from ++ * @buf: User buffer to write the data into ++ * @len: Amount of data to read ++ * @ppos: Offset into file to read from ++ * ++ * This function emits the results which where logged during the opening of ++ * the file kutf_debugfs_run_open. ++ * Results will be emitted one at a time, once all the results have been read ++ * 0 will be returned to indicate there is no more data. ++ * ++ * Return: Number of bytes read. ++ */ ++static ssize_t kutf_debugfs_run_read(struct file *file, char __user *buf, ++ size_t len, loff_t *ppos) ++{ ++ struct kutf_context *test_context = file->private_data; ++ struct kutf_result *res; ++ unsigned long bytes_not_copied; ++ ssize_t bytes_copied = 0; ++ ++ /* Note: This code assumes a result is read completely */ ++ res = kutf_remove_result(test_context->result_set); ++ if (res) { ++ char *kutf_str_ptr = NULL; ++ unsigned int kutf_str_len = 0; ++ unsigned int message_len = 0; ++ char separator = ':'; ++ char terminator = '\n'; ++ ++ kutf_result_to_string(&kutf_str_ptr, res->status); ++ if (kutf_str_ptr) ++ kutf_str_len = strlen(kutf_str_ptr); ++ ++ if (res->message) ++ message_len = strlen(res->message); ++ ++ if ((kutf_str_len + 1 + message_len + 1) > len) { ++ pr_err("Not enough space in user buffer for a single result"); ++ return 0; ++ } ++ ++ /* First copy the result string */ ++ if (kutf_str_ptr) { ++ bytes_not_copied = copy_to_user(&buf[0], kutf_str_ptr, ++ kutf_str_len); ++ bytes_copied += kutf_str_len - bytes_not_copied; ++ if (bytes_not_copied) ++ goto exit; ++ } ++ ++ /* Then the separator */ ++ bytes_not_copied = copy_to_user(&buf[bytes_copied], ++ &separator, 1); ++ bytes_copied += 1 - bytes_not_copied; ++ if (bytes_not_copied) ++ goto exit; ++ ++ /* Finally Next copy the result string */ ++ if (res->message) { ++ bytes_not_copied = copy_to_user(&buf[bytes_copied], ++ res->message, message_len); ++ bytes_copied += message_len - bytes_not_copied; ++ if (bytes_not_copied) ++ goto exit; ++ } ++ ++ /* Finally the terminator */ ++ bytes_not_copied = copy_to_user(&buf[bytes_copied], ++ &terminator, 1); ++ bytes_copied += 1 - bytes_not_copied; ++ } ++exit: ++ return bytes_copied; ++} ++ ++/** ++ * kutf_debugfs_run_release() - Debugfs release callback for the "run" entry. ++ * @inode: File entry representation ++ * @file: A specific opening of the file ++ * ++ * Release any resources that where created during the opening of the file ++ * ++ * Return: 0 on success ++ */ ++static int kutf_debugfs_run_release(struct inode *inode, struct file *file) ++{ ++ struct kutf_context *test_context = file->private_data; ++ ++ kutf_destroy_context(test_context); ++ return 0; ++} ++ ++static const struct file_operations kutf_debugfs_run_ops = { ++ .owner = THIS_MODULE, ++ .open = kutf_debugfs_run_open, ++ .read = kutf_debugfs_run_read, ++ .release = kutf_debugfs_run_release, ++ .llseek = default_llseek, ++}; ++ ++/** ++ * create_fixture_variant() - Creates a fixture variant for the specified ++ * test function and index and the debugfs entries ++ * that represent it. ++ * @test_func: Test function ++ * @fixture_index: Fixture index ++ * ++ * Return: 0 on success, negative value corresponding to error code in failure ++ */ ++static int create_fixture_variant(struct kutf_test_function *test_func, ++ unsigned int fixture_index) ++{ ++ struct kutf_test_fixture *test_fix; ++ char name[11]; /* Enough to print the MAX_UINT32 + the null terminator */ ++ struct dentry *tmp; ++ int err; ++ ++ test_fix = kmalloc(sizeof(*test_fix), GFP_KERNEL); ++ if (!test_fix) { ++ pr_err("Failed to create debugfs directory when adding fixture\n"); ++ err = -ENOMEM; ++ goto fail_alloc; ++ } ++ ++ test_fix->test_func = test_func; ++ test_fix->fixture_index = fixture_index; ++ ++ snprintf(name, sizeof(name), "%d", fixture_index); ++ test_fix->dir = debugfs_create_dir(name, test_func->dir); ++ if (!test_func->dir) { ++ pr_err("Failed to create debugfs directory when adding fixture\n"); ++ /* Might not be the right error, we don't get it passed back to us */ ++ err = -EEXIST; ++ goto fail_dir; ++ } ++ ++ tmp = debugfs_create_file("type", S_IROTH, test_fix->dir, "fixture\n", ++ &kutf_debugfs_const_string_ops); ++ if (!tmp) { ++ pr_err("Failed to create debugfs file \"type\" when adding fixture\n"); ++ /* Might not be the right error, we don't get it passed back to us */ ++ err = -EEXIST; ++ goto fail_file; ++ } ++ ++ tmp = debugfs_create_file("run", S_IROTH, test_fix->dir, test_fix, ++ &kutf_debugfs_run_ops); ++ if (!tmp) { ++ pr_err("Failed to create debugfs file \"run\" when adding fixture\n"); ++ /* Might not be the right error, we don't get it passed back to us */ ++ err = -EEXIST; ++ goto fail_file; ++ } ++ ++ list_add(&test_fix->node, &test_func->variant_list); ++ return 0; ++ ++fail_file: ++ debugfs_remove_recursive(test_fix->dir); ++fail_dir: ++ kfree(test_fix); ++fail_alloc: ++ return err; ++} ++ ++/** ++ * kutf_remove_test_variant() - Destroy a previously created fixture variant. ++ * @test_fix: Test fixture ++ */ ++static void kutf_remove_test_variant(struct kutf_test_fixture *test_fix) ++{ ++ debugfs_remove_recursive(test_fix->dir); ++ kfree(test_fix); ++} ++ ++void kutf_add_test_with_filters_and_data( ++ struct kutf_suite *suite, ++ unsigned int id, ++ const char *name, ++ void (*execute)(struct kutf_context *context), ++ unsigned int filters, ++ union kutf_callback_data test_data) ++{ ++ struct kutf_test_function *test_func; ++ struct dentry *tmp; ++ unsigned int i; ++ ++ test_func = kmalloc(sizeof(*test_func), GFP_KERNEL); ++ if (!test_func) { ++ pr_err("Failed to allocate memory when adding test %s\n", name); ++ goto fail_alloc; ++ } ++ ++ INIT_LIST_HEAD(&test_func->variant_list); ++ ++ test_func->dir = debugfs_create_dir(name, suite->dir); ++ if (!test_func->dir) { ++ pr_err("Failed to create debugfs directory when adding test %s\n", name); ++ goto fail_dir; ++ } ++ ++ tmp = debugfs_create_file("type", S_IROTH, test_func->dir, "test\n", ++ &kutf_debugfs_const_string_ops); ++ if (!tmp) { ++ pr_err("Failed to create debugfs file \"type\" when adding test %s\n", name); ++ goto fail_file; ++ } ++ ++ test_func->filters = filters; ++ tmp = debugfs_create_x32("filters", S_IROTH, test_func->dir, ++ &test_func->filters); ++ if (!tmp) { ++ pr_err("Failed to create debugfs file \"filters\" when adding test %s\n", name); ++ goto fail_file; ++ } ++ ++ test_func->test_id = id; ++ tmp = debugfs_create_u32("test_id", S_IROTH, test_func->dir, ++ &test_func->test_id); ++ if (!tmp) { ++ pr_err("Failed to create debugfs file \"test_id\" when adding test %s\n", name); ++ goto fail_file; ++ } ++ ++ for (i = 0; i < suite->fixture_variants; i++) { ++ if (create_fixture_variant(test_func, i)) { ++ pr_err("Failed to create fixture %d when adding test %s\n", i, name); ++ goto fail_file; ++ } ++ } ++ ++ test_func->suite = suite; ++ test_func->execute = execute; ++ test_func->test_data = test_data; ++ ++ list_add(&test_func->node, &suite->test_list); ++ return; ++ ++fail_file: ++ debugfs_remove_recursive(test_func->dir); ++fail_dir: ++ kfree(test_func); ++fail_alloc: ++ return; ++} ++EXPORT_SYMBOL(kutf_add_test_with_filters_and_data); ++ ++void kutf_add_test_with_filters( ++ struct kutf_suite *suite, ++ unsigned int id, ++ const char *name, ++ void (*execute)(struct kutf_context *context), ++ unsigned int filters) ++{ ++ union kutf_callback_data data; ++ ++ data.ptr_value = NULL; ++ ++ kutf_add_test_with_filters_and_data(suite, ++ id, ++ name, ++ execute, ++ suite->suite_default_flags, ++ data); ++} ++EXPORT_SYMBOL(kutf_add_test_with_filters); ++ ++void kutf_add_test(struct kutf_suite *suite, ++ unsigned int id, ++ const char *name, ++ void (*execute)(struct kutf_context *context)) ++{ ++ union kutf_callback_data data; ++ ++ data.ptr_value = NULL; ++ ++ kutf_add_test_with_filters_and_data(suite, ++ id, ++ name, ++ execute, ++ suite->suite_default_flags, ++ data); ++} ++EXPORT_SYMBOL(kutf_add_test); ++ ++/** ++ * kutf_remove_test(): Remove a previously added test function. ++ * @test_func: Test function ++ */ ++static void kutf_remove_test(struct kutf_test_function *test_func) ++{ ++ struct list_head *pos; ++ struct list_head *tmp; ++ ++ list_for_each_safe(pos, tmp, &test_func->variant_list) { ++ struct kutf_test_fixture *test_fix; ++ ++ test_fix = list_entry(pos, struct kutf_test_fixture, node); ++ kutf_remove_test_variant(test_fix); ++ } ++ ++ list_del(&test_func->node); ++ debugfs_remove_recursive(test_func->dir); ++ kfree(test_func); ++} ++ ++struct kutf_suite *kutf_create_suite_with_filters_and_data( ++ struct kutf_application *app, ++ const char *name, ++ unsigned int fixture_count, ++ void *(*create_fixture)(struct kutf_context *context), ++ void (*remove_fixture)(struct kutf_context *context), ++ unsigned int filters, ++ union kutf_callback_data suite_data) ++{ ++ struct kutf_suite *suite; ++ struct dentry *tmp; ++ ++ suite = kmalloc(sizeof(*suite), GFP_KERNEL); ++ if (!suite) { ++ pr_err("Failed to allocate memory when creating suite %s\n", name); ++ goto fail_kmalloc; ++ } ++ ++ suite->dir = debugfs_create_dir(name, app->dir); ++ if (!suite->dir) { ++ pr_err("Failed to create debugfs directory when adding test %s\n", name); ++ goto fail_debugfs; ++ } ++ ++ tmp = debugfs_create_file("type", S_IROTH, suite->dir, "suite\n", ++ &kutf_debugfs_const_string_ops); ++ if (!tmp) { ++ pr_err("Failed to create debugfs file \"type\" when adding test %s\n", name); ++ goto fail_file; ++ } ++ ++ INIT_LIST_HEAD(&suite->test_list); ++ suite->app = app; ++ suite->name = name; ++ suite->fixture_variants = fixture_count; ++ suite->create_fixture = create_fixture; ++ suite->remove_fixture = remove_fixture; ++ suite->suite_default_flags = filters; ++ suite->suite_data = suite_data; ++ ++ list_add(&suite->node, &app->suite_list); ++ ++ return suite; ++ ++fail_file: ++ debugfs_remove_recursive(suite->dir); ++fail_debugfs: ++ kfree(suite); ++fail_kmalloc: ++ return NULL; ++} ++EXPORT_SYMBOL(kutf_create_suite_with_filters_and_data); ++ ++struct kutf_suite *kutf_create_suite_with_filters( ++ struct kutf_application *app, ++ const char *name, ++ unsigned int fixture_count, ++ void *(*create_fixture)(struct kutf_context *context), ++ void (*remove_fixture)(struct kutf_context *context), ++ unsigned int filters) ++{ ++ union kutf_callback_data data; ++ ++ data.ptr_value = NULL; ++ return kutf_create_suite_with_filters_and_data(app, ++ name, ++ fixture_count, ++ create_fixture, ++ remove_fixture, ++ filters, ++ data); ++} ++EXPORT_SYMBOL(kutf_create_suite_with_filters); ++ ++struct kutf_suite *kutf_create_suite( ++ struct kutf_application *app, ++ const char *name, ++ unsigned int fixture_count, ++ void *(*create_fixture)(struct kutf_context *context), ++ void (*remove_fixture)(struct kutf_context *context)) ++{ ++ union kutf_callback_data data; ++ ++ data.ptr_value = NULL; ++ return kutf_create_suite_with_filters_and_data(app, ++ name, ++ fixture_count, ++ create_fixture, ++ remove_fixture, ++ KUTF_F_TEST_GENERIC, ++ data); ++} ++EXPORT_SYMBOL(kutf_create_suite); ++ ++/** ++ * kutf_destroy_suite() - Destroy a previously added test suite. ++ * @suite: Test suite ++ */ ++static void kutf_destroy_suite(struct kutf_suite *suite) ++{ ++ struct list_head *pos; ++ struct list_head *tmp; ++ ++ list_for_each_safe(pos, tmp, &suite->test_list) { ++ struct kutf_test_function *test_func; ++ ++ test_func = list_entry(pos, struct kutf_test_function, node); ++ kutf_remove_test(test_func); ++ } ++ ++ list_del(&suite->node); ++ debugfs_remove_recursive(suite->dir); ++ kfree(suite); ++} ++ ++struct kutf_application *kutf_create_application(const char *name) ++{ ++ struct kutf_application *app; ++ struct dentry *tmp; ++ ++ app = kmalloc(sizeof(*app), GFP_KERNEL); ++ if (!app) { ++ pr_err("Failed to create allocate memory when creating application %s\n", name); ++ goto fail_kmalloc; ++ } ++ ++ app->dir = debugfs_create_dir(name, base_dir); ++ if (!app->dir) { ++ pr_err("Failed to create debugfs direcotry when creating application %s\n", name); ++ goto fail_debugfs; ++ } ++ ++ tmp = debugfs_create_file("type", S_IROTH, app->dir, "application\n", ++ &kutf_debugfs_const_string_ops); ++ if (!tmp) { ++ pr_err("Failed to create debugfs file \"type\" when creating application %s\n", name); ++ goto fail_file; ++ } ++ ++ INIT_LIST_HEAD(&app->suite_list); ++ app->name = name; ++ ++ return app; ++ ++fail_file: ++ debugfs_remove_recursive(app->dir); ++fail_debugfs: ++ kfree(app); ++fail_kmalloc: ++ return NULL; ++} ++EXPORT_SYMBOL(kutf_create_application); ++ ++void kutf_destroy_application(struct kutf_application *app) ++{ ++ struct list_head *pos; ++ struct list_head *tmp; ++ ++ list_for_each_safe(pos, tmp, &app->suite_list) { ++ struct kutf_suite *suite; ++ ++ suite = list_entry(pos, struct kutf_suite, node); ++ kutf_destroy_suite(suite); ++ } ++ ++ debugfs_remove_recursive(app->dir); ++ kfree(app); ++} ++EXPORT_SYMBOL(kutf_destroy_application); ++ ++static struct kutf_context *kutf_create_context( ++ struct kutf_test_fixture *test_fix) ++{ ++ struct kutf_context *new_context; ++ ++ new_context = kmalloc(sizeof(*new_context), GFP_KERNEL); ++ if (!new_context) { ++ pr_err("Failed to allocate test context"); ++ goto fail_alloc; ++ } ++ ++ new_context->result_set = kutf_create_result_set(); ++ if (!new_context->result_set) { ++ pr_err("Failed to create resultset"); ++ goto fail_result_set; ++ } ++ ++ new_context->test_fix = test_fix; ++ /* Save the pointer to the suite as the callbacks will require it */ ++ new_context->suite = test_fix->test_func->suite; ++ new_context->status = KUTF_RESULT_UNKNOWN; ++ new_context->expected_status = KUTF_RESULT_UNKNOWN; ++ ++ kutf_mempool_init(&new_context->fixture_pool); ++ new_context->fixture = NULL; ++ new_context->fixture_index = test_fix->fixture_index; ++ new_context->fixture_name = NULL; ++ new_context->test_data = test_fix->test_func->test_data; ++ ++ return new_context; ++ ++fail_result_set: ++ kfree(new_context); ++fail_alloc: ++ return NULL; ++} ++ ++static void kutf_destroy_context(struct kutf_context *context) ++{ ++ kutf_destroy_result_set(context->result_set); ++ kutf_mempool_destroy(&context->fixture_pool); ++ kfree(context); ++} ++ ++static void kutf_set_result(struct kutf_context *context, ++ enum kutf_result_status status) ++{ ++ context->status = status; ++} ++ ++static void kutf_set_expected_result(struct kutf_context *context, ++ enum kutf_result_status expected_status) ++{ ++ context->expected_status = expected_status; ++} ++ ++/** ++ * kutf_test_log_result() - Log a result for the specified test context ++ * @context: Test context ++ * @message: Result string ++ * @new_status: Result status ++ */ ++static void kutf_test_log_result( ++ struct kutf_context *context, ++ const char *message, ++ enum kutf_result_status new_status) ++{ ++ if (context->status < new_status) ++ context->status = new_status; ++ ++ if (context->expected_status != new_status) ++ kutf_add_result(&context->fixture_pool, context->result_set, ++ new_status, message); ++} ++ ++void kutf_test_log_result_external( ++ struct kutf_context *context, ++ const char *message, ++ enum kutf_result_status new_status) ++{ ++ kutf_test_log_result(context, message, new_status); ++} ++EXPORT_SYMBOL(kutf_test_log_result_external); ++ ++void kutf_test_expect_abort(struct kutf_context *context) ++{ ++ kutf_set_expected_result(context, KUTF_RESULT_ABORT); ++} ++EXPORT_SYMBOL(kutf_test_expect_abort); ++ ++void kutf_test_expect_fatal(struct kutf_context *context) ++{ ++ kutf_set_expected_result(context, KUTF_RESULT_FATAL); ++} ++EXPORT_SYMBOL(kutf_test_expect_fatal); ++ ++void kutf_test_expect_fail(struct kutf_context *context) ++{ ++ kutf_set_expected_result(context, KUTF_RESULT_FAIL); ++} ++EXPORT_SYMBOL(kutf_test_expect_fail); ++ ++void kutf_test_expect_warn(struct kutf_context *context) ++{ ++ kutf_set_expected_result(context, KUTF_RESULT_WARN); ++} ++EXPORT_SYMBOL(kutf_test_expect_warn); ++ ++void kutf_test_expect_pass(struct kutf_context *context) ++{ ++ kutf_set_expected_result(context, KUTF_RESULT_PASS); ++} ++EXPORT_SYMBOL(kutf_test_expect_pass); ++ ++void kutf_test_skip(struct kutf_context *context) ++{ ++ kutf_set_result(context, KUTF_RESULT_SKIP); ++ kutf_set_expected_result(context, KUTF_RESULT_UNKNOWN); ++ ++ kutf_test_log_result(context, "Test skipped", KUTF_RESULT_SKIP); ++} ++EXPORT_SYMBOL(kutf_test_skip); ++ ++void kutf_test_skip_msg(struct kutf_context *context, const char *message) ++{ ++ kutf_set_result(context, KUTF_RESULT_SKIP); ++ kutf_set_expected_result(context, KUTF_RESULT_UNKNOWN); ++ ++ kutf_test_log_result(context, kutf_dsprintf(&context->fixture_pool, ++ "Test skipped: %s", message), KUTF_RESULT_SKIP); ++ kutf_test_log_result(context, "!!!Test skipped!!!", KUTF_RESULT_SKIP); ++} ++EXPORT_SYMBOL(kutf_test_skip_msg); ++ ++void kutf_test_debug(struct kutf_context *context, char const *message) ++{ ++ kutf_test_log_result(context, message, KUTF_RESULT_DEBUG); ++} ++EXPORT_SYMBOL(kutf_test_debug); ++ ++void kutf_test_pass(struct kutf_context *context, char const *message) ++{ ++ static const char explicit_message[] = "(explicit pass)"; ++ ++ if (!message) ++ message = explicit_message; ++ ++ kutf_test_log_result(context, message, KUTF_RESULT_PASS); ++} ++EXPORT_SYMBOL(kutf_test_pass); ++ ++void kutf_test_info(struct kutf_context *context, char const *message) ++{ ++ kutf_test_log_result(context, message, KUTF_RESULT_INFO); ++} ++EXPORT_SYMBOL(kutf_test_info); ++ ++void kutf_test_warn(struct kutf_context *context, char const *message) ++{ ++ kutf_test_log_result(context, message, KUTF_RESULT_WARN); ++} ++EXPORT_SYMBOL(kutf_test_warn); ++ ++void kutf_test_fail(struct kutf_context *context, char const *message) ++{ ++ kutf_test_log_result(context, message, KUTF_RESULT_FAIL); ++} ++EXPORT_SYMBOL(kutf_test_fail); ++ ++void kutf_test_fatal(struct kutf_context *context, char const *message) ++{ ++ kutf_test_log_result(context, message, KUTF_RESULT_FATAL); ++} ++EXPORT_SYMBOL(kutf_test_fatal); ++ ++void kutf_test_abort(struct kutf_context *context) ++{ ++ kutf_test_log_result(context, "", KUTF_RESULT_ABORT); ++} ++EXPORT_SYMBOL(kutf_test_abort); ++ ++/** ++ * init_kutf_core() - Module entry point. ++ * ++ * Create the base entry point in debugfs. ++ */ ++static int __init init_kutf_core(void) ++{ ++ int ret; ++ ++ base_dir = debugfs_create_dir("kutf_tests", NULL); ++ if (!base_dir) { ++ ret = -ENODEV; ++ goto exit_dir; ++ } ++ ++ return 0; ++ ++exit_dir: ++ return ret; ++} ++ ++/** ++ * exit_kutf_core() - Module exit point. ++ * ++ * Remove the base entry point in debugfs. ++ */ ++static void __exit exit_kutf_core(void) ++{ ++ debugfs_remove_recursive(base_dir); ++} ++ ++#else /* defined(CONFIG_DEBUG_FS) */ ++ ++/** ++ * init_kutf_core() - Module entry point. ++ * ++ * Stub for when build against a kernel without debugfs support ++ */ ++static int __init init_kutf_core(void) ++{ ++ pr_debug("KUTF requires a kernel with debug fs support"); ++ ++ return -ENODEV; ++} ++ ++/** ++ * exit_kutf_core() - Module exit point. ++ * ++ * Stub for when build against a kernel without debugfs support ++ */ ++static void __exit exit_kutf_core(void) ++{ ++} ++#endif /* defined(CONFIG_DEBUG_FS) */ ++ ++MODULE_LICENSE("GPL"); ++ ++module_init(init_kutf_core); ++module_exit(exit_kutf_core); +diff --git a/drivers/gpu/arm/midgard/tests/kutf/kutf_utils.c b/drivers/gpu/arm/midgard/tests/kutf/kutf_utils.c +new file mode 100755 +index 000000000000..a429a2dbf788 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/tests/kutf/kutf_utils.c +@@ -0,0 +1,71 @@ ++/* ++ * ++ * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++/* Kernel UTF utility functions */ ++ ++#include ++#include ++#include ++#include ++ ++#include ++#include ++ ++static char tmp_buffer[KUTF_MAX_DSPRINTF_LEN]; ++ ++DEFINE_MUTEX(buffer_lock); ++ ++const char *kutf_dsprintf(struct kutf_mempool *pool, ++ const char *fmt, ...) ++{ ++ va_list args; ++ int len; ++ int size; ++ void *buffer; ++ ++ mutex_lock(&buffer_lock); ++ va_start(args, fmt); ++ len = vsnprintf(tmp_buffer, sizeof(tmp_buffer), fmt, args); ++ va_end(args); ++ ++ if (len < 0) { ++ pr_err("kutf_dsprintf: Bad format dsprintf format %s\n", fmt); ++ goto fail_format; ++ } ++ ++ if (len >= sizeof(tmp_buffer)) { ++ pr_warn("kutf_dsprintf: Truncated dsprintf message %s\n", fmt); ++ size = sizeof(tmp_buffer); ++ } else { ++ size = len + 1; ++ } ++ ++ buffer = kutf_mempool_alloc(pool, size); ++ if (!buffer) ++ goto fail_alloc; ++ ++ memcpy(buffer, tmp_buffer, size); ++ mutex_unlock(&buffer_lock); ++ ++ return buffer; ++ ++fail_alloc: ++fail_format: ++ mutex_unlock(&buffer_lock); ++ return NULL; ++} ++EXPORT_SYMBOL(kutf_dsprintf); +diff --git a/drivers/gpu/arm/midgard/tests/kutf/sconscript b/drivers/gpu/arm/midgard/tests/kutf/sconscript +new file mode 100755 +index 000000000000..d7f112448e42 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/tests/kutf/sconscript +@@ -0,0 +1,21 @@ ++# ++# (C) COPYRIGHT 2014-2016, 2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++Import('kutf_env') ++ ++make_args = kutf_env.kernel_get_config_defines(ret_list = True) ++ ++mod = kutf_env.BuildKernelModule('$STATIC_LIB_PATH/kutf.ko', Glob('*.c'), make_args = make_args) ++kutf_env.KernelObjTarget('kutf', mod) +diff --git a/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Kbuild b/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Kbuild +new file mode 100755 +index 000000000000..0cd9cebe9d8b +--- /dev/null ++++ b/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Kbuild +@@ -0,0 +1,20 @@ ++# ++# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++ccflags-y += -I$(src)/../include -I$(src)/../../../ -I$(src)/../../ -I$(src)/../../backend/gpu -I$(srctree)/drivers/staging/android ++ ++obj-$(CONFIG_MALI_IRQ_LATENCY) += mali_kutf_irq_test.o ++ ++mali_kutf_irq_test-y := mali_kutf_irq_test_main.o +diff --git a/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Kconfig b/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Kconfig +new file mode 100755 +index 000000000000..16f68d15c46e +--- /dev/null ++++ b/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Kconfig +@@ -0,0 +1,23 @@ ++# ++# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++config MALI_IRQ_LATENCY ++ tristate "Mali GPU IRQ latency measurement" ++ depends on MALI_MIDGARD && MALI_DEBUG && MALI_KUTF ++ default n ++ help ++ This option will build a test module mali_kutf_irq_test that ++ can determine the latency of the Mali GPU IRQ on your system. ++ Choosing M here will generate a single module called mali_kutf_irq_test. +diff --git a/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Makefile b/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Makefile +new file mode 100755 +index 000000000000..4e948767a4ac +--- /dev/null ++++ b/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Makefile +@@ -0,0 +1,51 @@ ++# ++# (C) COPYRIGHT 2015, 2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++# linux build system bootstrap for out-of-tree module ++ ++# default to building for the host ++ARCH ?= $(shell uname -m) ++ ++ifeq ($(KDIR),) ++$(error Must specify KDIR to point to the kernel to target)) ++endif ++ ++TEST_CCFLAGS := \ ++ -DMALI_DEBUG=$(MALI_DEBUG) \ ++ -DMALI_BACKEND_KERNEL=$(MALI_BACKEND_KERNEL) \ ++ -DMALI_MODEL=$(MALI_MODEL) \ ++ -DMALI_NO_MALI=$(MALI_NO_MALI) \ ++ -DMALI_BASE_QA_LEAK=$(MALI_BASE_QA_LEAK) \ ++ -DMALI_BASE_QA_RESFAIL=$(MALI_BASE_QA_RESFAIL) \ ++ -DMALI_BASE_QA_USE_AFTER_FREE=$(MALI_BASE_QA_USE_AFTER_FREE) \ ++ -DMALI_UNIT_TEST=$(MALI_UNIT_TEST) \ ++ -DMALI_USE_UMP=$(MALI_USE_UMP) \ ++ -DMALI_ERROR_INJECT_ON=$(MALI_ERROR_INJECT_ON) \ ++ -DMALI_CUSTOMER_RELEASE=$(MALI_CUSTOMER_RELEASE) \ ++ $(SCONS_CFLAGS) \ ++ -I$(CURDIR)/../include \ ++ -I$(CURDIR)/../../../../../../include \ ++ -I$(CURDIR)/../../../ \ ++ -I$(CURDIR)/../../ \ ++ -I$(CURDIR)/../../backend/gpu \ ++ -I$(CURDIR)/ \ ++ -I$(srctree)/drivers/staging/android \ ++ -I$(srctree)/include/linux ++ ++all: ++ $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) $(SCONS_CONFIGS) EXTRA_CFLAGS="$(TEST_CCFLAGS)" KBUILD_EXTRA_SYMBOLS="$(CURDIR)/../kutf/Module.symvers $(CURDIR)/../../Module.symvers" modules ++ ++clean: ++ $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) clean +diff --git a/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c b/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c +new file mode 100755 +index 000000000000..e2ff4432bf80 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c +@@ -0,0 +1,257 @@ ++/* ++ * ++ * (C) COPYRIGHT 2016, 2017 ARM Limited. All rights reserved. ++ * ++ * This program is free software and is provided to you under the terms of the ++ * GNU General Public License version 2 as published by the Free Software ++ * Foundation, and any use by you of this program is subject to the terms ++ * of such GNU licence. ++ * ++ * A copy of the licence is included with the program, and can also be obtained ++ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++ * Boston, MA 02110-1301, USA. ++ * ++ */ ++ ++ ++ ++#include ++#include ++#include ++ ++#include "mali_kbase.h" ++#include ++ ++#include ++#include ++ ++/* ++ * This file contains the code which is used for measuring interrupt latency ++ * of the Mali GPU IRQ. In particular, function mali_kutf_irq_latency() is ++ * used with this purpose and it is called within KUTF framework - a kernel ++ * unit test framework. The measured latency provided by this test should ++ * be representative for the latency of the Mali JOB/MMU IRQs as well. ++ */ ++ ++/* KUTF test application pointer for this test */ ++struct kutf_application *irq_app; ++ ++/** ++ * struct kutf_irq_fixture data - test fixture used by the test functions. ++ * @kbdev: kbase device for the GPU. ++ * ++ */ ++struct kutf_irq_fixture_data { ++ struct kbase_device *kbdev; ++}; ++ ++#define SEC_TO_NANO(s) ((s)*1000000000LL) ++ ++/* ID for the GPU IRQ */ ++#define GPU_IRQ_HANDLER 2 ++ ++#define NR_TEST_IRQS 1000000 ++ ++/* IRQ for the test to trigger. Currently MULTIPLE_GPU_FAULTS as we would not ++ * expect to see this in normal use (e.g., when Android is running). */ ++#define TEST_IRQ MULTIPLE_GPU_FAULTS ++ ++#define IRQ_TIMEOUT HZ ++ ++/* Kernel API for setting irq throttle hook callback and irq time in us*/ ++extern int kbase_set_custom_irq_handler(struct kbase_device *kbdev, ++ irq_handler_t custom_handler, ++ int irq_type); ++extern irqreturn_t kbase_gpu_irq_handler(int irq, void *data); ++ ++static DECLARE_WAIT_QUEUE_HEAD(wait); ++static bool triggered; ++static u64 irq_time; ++ ++static void *kbase_untag(void *ptr) ++{ ++ return (void *)(((uintptr_t) ptr) & ~3); ++} ++ ++/** ++ * kbase_gpu_irq_custom_handler - Custom IRQ throttle handler ++ * @irq: IRQ number ++ * @data: Data associated with this IRQ ++ * ++ * Return: state of the IRQ ++ */ ++static irqreturn_t kbase_gpu_irq_custom_handler(int irq, void *data) ++{ ++ struct kbase_device *kbdev = kbase_untag(data); ++ u32 val; ++ ++ val = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_STATUS), NULL); ++ if (val & TEST_IRQ) { ++ struct timespec64 tval; ++ ++ ktime_get_real_ts64(&tval); ++ irq_time = SEC_TO_NANO(tval.tv_sec) + (tval.tv_nsec); ++ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), val, ++ NULL); ++ ++ triggered = true; ++ wake_up(&wait); ++ ++ return IRQ_HANDLED; ++ } ++ ++ /* Trigger main irq handler */ ++ return kbase_gpu_irq_handler(irq, data); ++} ++ ++/** ++ * mali_kutf_irq_default_create_fixture() - Creates the fixture data required ++ * for all the tests in the irq suite. ++ * @context: KUTF context. ++ * ++ * Return: Fixture data created on success or NULL on failure ++ */ ++static void *mali_kutf_irq_default_create_fixture( ++ struct kutf_context *context) ++{ ++ struct kutf_irq_fixture_data *data; ++ ++ data = kutf_mempool_alloc(&context->fixture_pool, ++ sizeof(struct kutf_irq_fixture_data)); ++ ++ if (!data) ++ goto fail; ++ ++ /* Acquire the kbase device */ ++ data->kbdev = kbase_find_device(-1); ++ if (data->kbdev == NULL) { ++ kutf_test_fail(context, "Failed to find kbase device"); ++ goto fail; ++ } ++ ++ return data; ++ ++fail: ++ return NULL; ++} ++ ++/** ++ * mali_kutf_irq_default_remove_fixture() - Destroy fixture data previously ++ * created by mali_kutf_irq_default_create_fixture. ++ * ++ * @context: KUTF context. ++ */ ++static void mali_kutf_irq_default_remove_fixture( ++ struct kutf_context *context) ++{ ++ struct kutf_irq_fixture_data *data = context->fixture; ++ struct kbase_device *kbdev = data->kbdev; ++ ++ kbase_release_device(kbdev); ++} ++ ++/** ++ * mali_kutf_irq_latency() - measure GPU IRQ latency ++ * @context: kutf context within which to perform the test ++ * ++ * The test triggers IRQs manually, and measures the ++ * time between triggering the IRQ and the IRQ handler being executed. ++ * ++ * This is not a traditional test, in that the pass/fail status has little ++ * meaning (other than indicating that the IRQ handler executed at all). Instead ++ * the results are in the latencies provided with the test result. There is no ++ * meaningful pass/fail result that can be obtained here, instead the latencies ++ * are provided for manual analysis only. ++ */ ++static void mali_kutf_irq_latency(struct kutf_context *context) ++{ ++ struct kutf_irq_fixture_data *data = context->fixture; ++ struct kbase_device *kbdev = data->kbdev; ++ u64 min_time = U64_MAX, max_time = 0, average_time = 0; ++ int i; ++ bool test_failed = false; ++ ++ /* Force GPU to be powered */ ++ kbase_pm_context_active(kbdev); ++ ++ kbase_set_custom_irq_handler(kbdev, kbase_gpu_irq_custom_handler, ++ GPU_IRQ_HANDLER); ++ ++ for (i = 0; i < NR_TEST_IRQS; i++) { ++ struct timespec64 tval; ++ u64 start_time; ++ int ret; ++ ++ triggered = false; ++ ktime_get_real_ts64(&tval); ++ start_time = SEC_TO_NANO(tval.tv_sec) + (tval.tv_nsec); ++ ++ /* Trigger fake IRQ */ ++ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), ++ TEST_IRQ, NULL); ++ ++ ret = wait_event_timeout(wait, triggered != false, IRQ_TIMEOUT); ++ ++ if (ret == 0) { ++ kutf_test_fail(context, "Timed out waiting for IRQ\n"); ++ test_failed = true; ++ break; ++ } ++ ++ if ((irq_time - start_time) < min_time) ++ min_time = irq_time - start_time; ++ if ((irq_time - start_time) > max_time) ++ max_time = irq_time - start_time; ++ average_time += irq_time - start_time; ++ ++ udelay(10); ++ } ++ ++ /* Go back to default handler */ ++ kbase_set_custom_irq_handler(kbdev, NULL, GPU_IRQ_HANDLER); ++ ++ kbase_pm_context_idle(kbdev); ++ ++ if (!test_failed) { ++ const char *results; ++ ++ do_div(average_time, NR_TEST_IRQS); ++ results = kutf_dsprintf(&context->fixture_pool, ++ "Min latency = %lldns, Max latency = %lldns, Average latency = %lldns\n", ++ min_time, max_time, average_time); ++ kutf_test_pass(context, results); ++ } ++} ++ ++/** ++ * Module entry point for this test. ++ */ ++int mali_kutf_irq_test_main_init(void) ++{ ++ struct kutf_suite *suite; ++ ++ irq_app = kutf_create_application("irq"); ++ suite = kutf_create_suite(irq_app, "irq_default", ++ 1, mali_kutf_irq_default_create_fixture, ++ mali_kutf_irq_default_remove_fixture); ++ ++ kutf_add_test(suite, 0x0, "irq_latency", ++ mali_kutf_irq_latency); ++ return 0; ++} ++ ++/** ++ * Module exit point for this test. ++ */ ++void mali_kutf_irq_test_main_exit(void) ++{ ++ kutf_destroy_application(irq_app); ++} ++ ++module_init(mali_kutf_irq_test_main_init); ++module_exit(mali_kutf_irq_test_main_exit); ++ ++MODULE_LICENSE("GPL"); ++MODULE_AUTHOR("ARM Ltd."); ++MODULE_VERSION("1.0"); +diff --git a/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/sconscript b/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/sconscript +new file mode 100755 +index 000000000000..ec837f16448d +--- /dev/null ++++ b/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/sconscript +@@ -0,0 +1,30 @@ ++# ++# (C) COPYRIGHT 2015, 2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++import os ++Import('env') ++ ++src = [Glob('#kernel/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/*.c'), Glob('#kernel/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Makefile')] ++ ++if env.GetOption('clean') : ++ env.Execute(Action("make clean", '[CLEAN] mali_kutf_irq_test')) ++ cmd = env.Command('$STATIC_LIB_PATH/mali_kutf_irq_test.ko', src, []) ++ env.KernelObjTarget('mali_kutf_irq_test', cmd) ++else: ++ makeAction=Action("cd ${SOURCE.dir} && make MALI_DEBUG=${debug} MALI_BACKEND_KERNEL=1 MALI_ERROR_INJECT_ON=${error_inject} MALI_MODEL=${mali_model} MALI_NO_MALI=${no_mali} MALI_HW_VERSION=${hwver} MALI_UNIT_TEST=${unit} MALI_USE_UMP=${ump} MALI_CUSTOMER_RELEASE=${release} %s %s && ( ( [ -f mali_kutf_irq_test.ko ] && cp mali_kutf_irq_test.ko $STATIC_LIB_PATH/ ) || touch $STATIC_LIB_PATH/mali_kutf_irq_test.ko)" % (env.base_get_qa_settings(), env.kernel_get_config_defines()), '$MAKECOMSTR') ++ cmd = env.Command('$STATIC_LIB_PATH/mali_kutf_irq_test.ko', src, [makeAction]) ++ env.Depends('$STATIC_LIB_PATH/mali_kutf_irq_test.ko', '$STATIC_LIB_PATH/kutf.ko') ++ env.Depends('$STATIC_LIB_PATH/mali_kutf_irq_test.ko', '$STATIC_LIB_PATH/mali_kbase.ko') ++ env.KernelObjTarget('mali_kutf_irq_test', cmd) +diff --git a/drivers/gpu/arm/midgard/tests/sconscript b/drivers/gpu/arm/midgard/tests/sconscript +new file mode 100755 +index 000000000000..5337e1078e20 +--- /dev/null ++++ b/drivers/gpu/arm/midgard/tests/sconscript +@@ -0,0 +1,37 @@ ++# ++# (C) COPYRIGHT 2010-2011, 2013, 2017 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# A copy of the licence is included with the program, and can also be obtained ++# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, ++# Boston, MA 02110-1301, USA. ++# ++# ++ ++ ++Import ('env') ++ ++kutf_env = env.Clone() ++kutf_env.Append(CPPPATH = '#kernel/drivers/gpu/arm/midgard/tests/include') ++Export('kutf_env') ++ ++if Glob('internal/sconscript'): ++ SConscript('internal/sconscript') ++ ++if kutf_env['debug'] == '1': ++ SConscript('kutf/sconscript') ++ SConscript('mali_kutf_irq_test/sconscript') ++ ++ if Glob('kutf_test/sconscript'): ++ SConscript('kutf_test/sconscript') ++ ++ if Glob('kutf_test_runner/sconscript'): ++ SConscript('kutf_test_runner/sconscript') ++ ++if env['unit'] == '1': ++ SConscript('mali_kutf_ipa_test/sconscript') ++ SConscript('mali_kutf_vinstr_test/sconscript') +diff --git a/drivers/gpu/arm/sconscript b/drivers/gpu/arm/sconscript +new file mode 100755 +index 000000000000..a06092bd5bf0 +--- /dev/null ++++ b/drivers/gpu/arm/sconscript +@@ -0,0 +1,25 @@ ++# ++# (C) COPYRIGHT 2015-2016 ARM Limited. All rights reserved. ++# ++# This program is free software and is provided to you under the terms of the ++# GNU General Public License version 2 as published by the Free Software ++# Foundation, and any use by you of this program is subject to the terms ++# of such GNU licence. ++# ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# GNU General Public License for more details. ++# ++# You should have received a copy of the GNU General Public License ++# along with this program; if not, you can access it online at ++# http://www.gnu.org/licenses/gpl-2.0.html. ++# ++# SPDX-License-Identifier: GPL-2.0 ++# ++# ++ ++import glob ++ ++ ++SConscript('midgard/sconscript') +diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig +index ca868271f4c4..2a9184156172 100644 +--- a/drivers/gpu/drm/Kconfig ++++ b/drivers/gpu/drm/Kconfig +@@ -31,6 +31,10 @@ config DRM_MIPI_DBI + tristate + depends on DRM + ++config DRM_IGNORE_IOTCL_PERMIT ++ bool "Ignore drm ioctl permission" ++ depends on DRM && ANDROID && NO_GKI ++ + config DRM_MIPI_DSI + bool + depends on DRM +diff --git a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c +index aa1bb86293fd..b8d4a5f49924 100644 +--- a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c ++++ b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c +@@ -13,6 +13,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -40,6 +41,20 @@ struct bridge_init { + struct device_node *node; + }; + ++static bool analogix_dp_bandwidth_ok(struct analogix_dp_device *dp, ++ const struct drm_display_mode *mode, ++ unsigned int rate, unsigned int lanes) ++{ ++ u32 max_bw, req_bw, bpp = 24; ++ ++ req_bw = mode->clock * bpp / 8; ++ max_bw = lanes * rate; ++ if (req_bw > max_bw) ++ return false; ++ ++ return true; ++} ++ + static int analogix_dp_init_dp(struct analogix_dp_device *dp) + { + int ret; +@@ -64,6 +79,46 @@ static int analogix_dp_init_dp(struct analogix_dp_device *dp) + return 0; + } + ++static int analogix_dp_panel_prepare(struct analogix_dp_device *dp) ++{ ++ int ret; ++ ++ mutex_lock(&dp->panel_lock); ++ ++ if (dp->panel_is_prepared) ++ goto out; ++ ++ ret = drm_panel_prepare(dp->plat_data->panel); ++ if (ret) ++ goto out; ++ ++ dp->panel_is_prepared = true; ++ ++out: ++ mutex_unlock(&dp->panel_lock); ++ return 0; ++} ++ ++static int analogix_dp_panel_unprepare(struct analogix_dp_device *dp) ++{ ++ int ret; ++ ++ mutex_lock(&dp->panel_lock); ++ ++ if (!dp->panel_is_prepared) ++ goto out; ++ ++ ret = drm_panel_unprepare(dp->plat_data->panel); ++ if (ret) ++ goto out; ++ ++ dp->panel_is_prepared = false; ++ ++out: ++ mutex_unlock(&dp->panel_lock); ++ return 0; ++} ++ + static int analogix_dp_detect_hpd(struct analogix_dp_device *dp) + { + int timeout_loop = 0; +@@ -108,6 +163,9 @@ static bool analogix_dp_detect_sink_psr(struct analogix_dp_device *dp) + unsigned char psr_version; + int ret; + ++ if (!device_property_read_bool(dp->dev, "support-psr")) ++ return 0; ++ + ret = drm_dp_dpcd_readb(&dp->aux, DP_PSR_SUPPORT, &psr_version); + if (ret != 1) { + dev_err(dp->dev, "failed to get PSR version, disable it\n"); +@@ -216,8 +274,24 @@ static int analogix_dp_set_enhanced_mode(struct analogix_dp_device *dp) + if (ret < 0) + return ret; + ++ if (!data) { ++ /* ++ * A setting of 1 indicates that this is an eDP device that ++ * uses only Enhanced Framing, independently of the setting by ++ * the source of ENHANCED_FRAME_EN ++ */ ++ ret = drm_dp_dpcd_readb(&dp->aux, DP_EDP_CONFIGURATION_CAP, ++ &data); ++ if (ret < 0) ++ return ret; ++ ++ data = !!(data & DP_FRAMING_CHANGE_CAP); ++ } ++ + analogix_dp_enable_enhanced_mode(dp, data); + ++ dp->link_train.enhanced_framing = data; ++ + return 0; + } + +@@ -233,32 +307,10 @@ static int analogix_dp_training_pattern_dis(struct analogix_dp_device *dp) + return ret < 0 ? ret : 0; + } + +-static void +-analogix_dp_set_lane_lane_pre_emphasis(struct analogix_dp_device *dp, +- int pre_emphasis, int lane) +-{ +- switch (lane) { +- case 0: +- analogix_dp_set_lane0_pre_emphasis(dp, pre_emphasis); +- break; +- case 1: +- analogix_dp_set_lane1_pre_emphasis(dp, pre_emphasis); +- break; +- +- case 2: +- analogix_dp_set_lane2_pre_emphasis(dp, pre_emphasis); +- break; +- +- case 3: +- analogix_dp_set_lane3_pre_emphasis(dp, pre_emphasis); +- break; +- } +-} +- + static int analogix_dp_link_start(struct analogix_dp_device *dp) + { + u8 buf[4]; +- int lane, lane_count, pll_tries, retval; ++ int lane, lane_count, retval; + + lane_count = dp->link_train.lane_count; + +@@ -278,6 +330,14 @@ static int analogix_dp_link_start(struct analogix_dp_device *dp) + retval = drm_dp_dpcd_write(&dp->aux, DP_LINK_BW_SET, buf, 2); + if (retval < 0) + return retval; ++ ++ /* Spread AMP if required, enable 8b/10b coding */ ++ buf[0] = analogix_dp_ssc_supported(dp) ? DP_SPREAD_AMP_0_5 : 0; ++ buf[1] = DP_SET_ANSI_8B10B; ++ retval = drm_dp_dpcd_write(&dp->aux, DP_DOWNSPREAD_CTRL, buf, 2); ++ if (retval < 0) ++ return retval; ++ + /* set enhanced mode if available */ + retval = analogix_dp_set_enhanced_mode(dp); + if (retval < 0) { +@@ -285,22 +345,12 @@ static int analogix_dp_link_start(struct analogix_dp_device *dp) + return retval; + } + +- /* Set TX pre-emphasis to minimum */ ++ /* Set TX voltage-swing and pre-emphasis to minimum */ + for (lane = 0; lane < lane_count; lane++) +- analogix_dp_set_lane_lane_pre_emphasis(dp, +- PRE_EMPHASIS_LEVEL_0, lane); +- +- /* Wait for PLL lock */ +- pll_tries = 0; +- while (analogix_dp_get_pll_lock_status(dp) == PLL_UNLOCKED) { +- if (pll_tries == DP_TIMEOUT_LOOP_COUNT) { +- dev_err(dp->dev, "Wait for PLL lock timed out\n"); +- return -ETIMEDOUT; +- } +- +- pll_tries++; +- usleep_range(90, 120); +- } ++ dp->link_train.training_lane[lane] = ++ DP_TRAIN_VOLTAGE_SWING_LEVEL_0 | ++ DP_TRAIN_PRE_EMPH_LEVEL_0; ++ analogix_dp_set_lane_link_training(dp); + + /* Set training pattern 1 */ + analogix_dp_set_training_pattern(dp, TRAINING_PTN1); +@@ -383,54 +433,6 @@ static unsigned char analogix_dp_get_adjust_request_pre_emphasis( + return ((link_value >> shift) & 0xc) >> 2; + } + +-static void analogix_dp_set_lane_link_training(struct analogix_dp_device *dp, +- u8 training_lane_set, int lane) +-{ +- switch (lane) { +- case 0: +- analogix_dp_set_lane0_link_training(dp, training_lane_set); +- break; +- case 1: +- analogix_dp_set_lane1_link_training(dp, training_lane_set); +- break; +- +- case 2: +- analogix_dp_set_lane2_link_training(dp, training_lane_set); +- break; +- +- case 3: +- analogix_dp_set_lane3_link_training(dp, training_lane_set); +- break; +- } +-} +- +-static unsigned int +-analogix_dp_get_lane_link_training(struct analogix_dp_device *dp, +- int lane) +-{ +- u32 reg; +- +- switch (lane) { +- case 0: +- reg = analogix_dp_get_lane0_link_training(dp); +- break; +- case 1: +- reg = analogix_dp_get_lane1_link_training(dp); +- break; +- case 2: +- reg = analogix_dp_get_lane2_link_training(dp); +- break; +- case 3: +- reg = analogix_dp_get_lane3_link_training(dp); +- break; +- default: +- WARN_ON(1); +- return 0; +- } +- +- return reg; +-} +- + static void analogix_dp_reduce_link_rate(struct analogix_dp_device *dp) + { + analogix_dp_training_pattern_dis(dp); +@@ -519,25 +521,23 @@ static int analogix_dp_process_clock_recovery(struct analogix_dp_device *dp) + return -EIO; + } + } +- } + +- analogix_dp_get_adjust_training_lane(dp, adjust_request); +- +- for (lane = 0; lane < lane_count; lane++) +- analogix_dp_set_lane_link_training(dp, +- dp->link_train.training_lane[lane], lane); ++ analogix_dp_get_adjust_training_lane(dp, adjust_request); ++ analogix_dp_set_lane_link_training(dp); + +- retval = drm_dp_dpcd_write(&dp->aux, DP_TRAINING_LANE0_SET, +- dp->link_train.training_lane, lane_count); +- if (retval < 0) +- return retval; ++ retval = drm_dp_dpcd_write(&dp->aux, DP_TRAINING_LANE0_SET, ++ dp->link_train.training_lane, ++ lane_count); ++ if (retval < 0) ++ return retval; ++ } + + return 0; + } + + static int analogix_dp_process_equalizer_training(struct analogix_dp_device *dp) + { +- int lane, lane_count, retval; ++ int lane_count, retval; + u32 reg; + u8 link_align, link_status[2], adjust_request[2]; + +@@ -597,9 +597,7 @@ static int analogix_dp_process_equalizer_training(struct analogix_dp_device *dp) + return -EIO; + } + +- for (lane = 0; lane < lane_count; lane++) +- analogix_dp_set_lane_link_training(dp, +- dp->link_train.training_lane[lane], lane); ++ analogix_dp_set_lane_link_training(dp); + + retval = drm_dp_dpcd_write(&dp->aux, DP_TRAINING_LANE0_SET, + dp->link_train.training_lane, lane_count); +@@ -640,8 +638,10 @@ static void analogix_dp_get_max_rx_lane_count(struct analogix_dp_device *dp, + static int analogix_dp_full_link_train(struct analogix_dp_device *dp, + u32 max_lanes, u32 max_rate) + { ++ struct video_info *video = &dp->video_info; + int retval = 0; + bool training_finished = false; ++ u8 dpcd; + + /* + * MACRO_RST must be applied after the PLL_LOCK to avoid +@@ -667,6 +667,16 @@ static int analogix_dp_full_link_train(struct analogix_dp_device *dp, + dp->link_train.lane_count = (u8)LANE_COUNT1; + } + ++ if (!analogix_dp_bandwidth_ok(dp, &video->mode, ++ drm_dp_bw_code_to_link_rate(dp->link_train.link_rate), ++ dp->link_train.lane_count)) { ++ dev_err(dp->dev, "bandwidth overflow\n"); ++ return -EINVAL; ++ } ++ ++ drm_dp_dpcd_readb(&dp->aux, DP_MAX_DOWNSPREAD, &dpcd); ++ dp->link_train.ssc = !!(dpcd & DP_MAX_DOWNSPREAD_0_5); ++ + /* Setup TX lane count & rate */ + if (dp->link_train.lane_count > max_lanes) + dp->link_train.lane_count = max_lanes; +@@ -711,27 +721,15 @@ static int analogix_dp_full_link_train(struct analogix_dp_device *dp, + + static int analogix_dp_fast_link_train(struct analogix_dp_device *dp) + { +- int i, ret; ++ int ret; + u8 link_align, link_status[2]; +- enum pll_status status; + + analogix_dp_reset_macro(dp); + + analogix_dp_set_link_bandwidth(dp, dp->link_train.link_rate); + analogix_dp_set_lane_count(dp, dp->link_train.lane_count); +- +- for (i = 0; i < dp->link_train.lane_count; i++) { +- analogix_dp_set_lane_link_training(dp, +- dp->link_train.training_lane[i], i); +- } +- +- ret = readx_poll_timeout(analogix_dp_get_pll_lock_status, dp, status, +- status != PLL_UNLOCKED, 120, +- 120 * DP_TIMEOUT_LOOP_COUNT); +- if (ret) { +- DRM_DEV_ERROR(dp->dev, "Wait for pll lock failed %d\n", ret); +- return ret; +- } ++ analogix_dp_set_lane_link_training(dp); ++ analogix_dp_enable_enhanced_mode(dp, dp->link_train.enhanced_framing); + + /* source Set training pattern 1 */ + analogix_dp_set_training_pattern(dp, TRAINING_PTN1); +@@ -742,7 +740,6 @@ static int analogix_dp_fast_link_train(struct analogix_dp_device *dp) + /* From DP spec, pattern must be on-screen for a minimum 500us */ + usleep_range(500, 600); + +- /* TODO: enhanced_mode?*/ + analogix_dp_set_training_pattern(dp, DP_NONE); + + /* +@@ -884,25 +881,44 @@ static int analogix_dp_enable_scramble(struct analogix_dp_device *dp, + return ret < 0 ? ret : 0; + } + ++static irqreturn_t analogix_dp_hpd_irq_handler(int irq, void *arg) ++{ ++ struct analogix_dp_device *dp = arg; ++ ++ if (dp->drm_dev) ++ drm_helper_hpd_irq_event(dp->drm_dev); ++ ++ return IRQ_HANDLED; ++} ++ + static irqreturn_t analogix_dp_hardirq(int irq, void *arg) + { + struct analogix_dp_device *dp = arg; +- irqreturn_t ret = IRQ_NONE; + enum dp_irq_type irq_type; ++ int ret; ++ ++ ret = pm_runtime_get_sync(dp->dev); ++ if (ret < 0) ++ return IRQ_NONE; + + irq_type = analogix_dp_get_irq_type(dp); +- if (irq_type != DP_IRQ_TYPE_UNKNOWN) { ++ if (irq_type != DP_IRQ_TYPE_UNKNOWN) + analogix_dp_mute_hpd_interrupt(dp); +- ret = IRQ_WAKE_THREAD; +- } + +- return ret; ++ pm_runtime_put_sync(dp->dev); ++ ++ return IRQ_WAKE_THREAD; + } + + static irqreturn_t analogix_dp_irq_thread(int irq, void *arg) + { + struct analogix_dp_device *dp = arg; + enum dp_irq_type irq_type; ++ int ret; ++ ++ ret = pm_runtime_get_sync(dp->dev); ++ if (ret < 0) ++ return IRQ_NONE; + + irq_type = analogix_dp_get_irq_type(dp); + if (irq_type & DP_IRQ_TYPE_HP_CABLE_IN || +@@ -917,6 +933,8 @@ static irqreturn_t analogix_dp_irq_thread(int irq, void *arg) + analogix_dp_unmute_hpd_interrupt(dp); + } + ++ pm_runtime_put_sync(dp->dev); ++ + return IRQ_HANDLED; + } + +@@ -938,13 +956,12 @@ static int analogix_dp_fast_link_train_detection(struct analogix_dp_device *dp) + + static int analogix_dp_commit(struct analogix_dp_device *dp) + { ++ struct video_info *video = &dp->video_info; + int ret; + +- /* Keep the panel disabled while we configure video */ +- if (dp->plat_data->panel) { +- if (drm_panel_disable(dp->plat_data->panel)) +- DRM_ERROR("failed to disable the panel\n"); +- } ++ if (device_property_read_bool(dp->dev, "panel-self-test")) ++ return drm_dp_dpcd_writeb(&dp->aux, DP_EDP_CONFIGURATION_SET, ++ DP_PANEL_SELF_TEST_ENABLE); + + ret = analogix_dp_train_link(dp); + if (ret) { +@@ -959,21 +976,17 @@ static int analogix_dp_commit(struct analogix_dp_device *dp) + } + + analogix_dp_init_video(dp); ++ analogix_dp_set_video_format(dp); ++ ++ if (video->video_bist_enable) ++ analogix_dp_video_bist_enable(dp); ++ + ret = analogix_dp_config_video(dp); + if (ret) { + dev_err(dp->dev, "unable to config video\n"); + return ret; + } + +- /* Safe to enable the panel now */ +- if (dp->plat_data->panel) { +- ret = drm_panel_enable(dp->plat_data->panel); +- if (ret) { +- DRM_ERROR("failed to enable the panel\n"); +- return ret; +- } +- } +- + /* Check whether panel supports fast training */ + ret = analogix_dp_fast_link_train_detection(dp); + if (ret) +@@ -1058,66 +1071,18 @@ static int analogix_dp_disable_psr(struct analogix_dp_device *dp) + return analogix_dp_send_psr_spd(dp, &psr_vsc, true); + } + +-/* +- * This function is a bit of a catch-all for panel preparation, hopefully +- * simplifying the logic of functions that need to prepare/unprepare the panel +- * below. +- * +- * If @prepare is true, this function will prepare the panel. Conversely, if it +- * is false, the panel will be unprepared. +- * +- * If @is_modeset_prepare is true, the function will disregard the current state +- * of the panel and either prepare/unprepare the panel based on @prepare. Once +- * it finishes, it will update dp->panel_is_modeset to reflect the current state +- * of the panel. +- */ +-static int analogix_dp_prepare_panel(struct analogix_dp_device *dp, +- bool prepare, bool is_modeset_prepare) +-{ +- int ret = 0; +- +- if (!dp->plat_data->panel) +- return 0; +- +- mutex_lock(&dp->panel_lock); +- +- /* +- * Exit early if this is a temporary prepare/unprepare and we're already +- * modeset (since we neither want to prepare twice or unprepare early). +- */ +- if (dp->panel_is_modeset && !is_modeset_prepare) +- goto out; +- +- if (prepare) +- ret = drm_panel_prepare(dp->plat_data->panel); +- else +- ret = drm_panel_unprepare(dp->plat_data->panel); +- +- if (ret) +- goto out; +- +- if (is_modeset_prepare) +- dp->panel_is_modeset = prepare; +- +-out: +- mutex_unlock(&dp->panel_lock); +- return ret; +-} +- + static int analogix_dp_get_modes(struct drm_connector *connector) + { + struct analogix_dp_device *dp = to_dp(connector); + struct edid *edid; +- int ret, num_modes = 0; ++ int num_modes = 0; + +- if (dp->plat_data->panel) { ++ if (dp->plat_data->panel) + num_modes += drm_panel_get_modes(dp->plat_data->panel, connector); +- } else { +- ret = analogix_dp_prepare_panel(dp, true, false); +- if (ret) { +- DRM_ERROR("Failed to prepare panel (%d)\n", ret); +- return 0; +- } ++ ++ if (!num_modes) { ++ if (dp->plat_data->panel) ++ analogix_dp_panel_prepare(dp); + + pm_runtime_get_sync(dp->dev); + edid = drm_get_edid(connector, &dp->aux.ddc); +@@ -1128,10 +1093,6 @@ static int analogix_dp_get_modes(struct drm_connector *connector) + num_modes += drm_add_edid_modes(&dp->connector, edid); + kfree(edid); + } +- +- ret = analogix_dp_prepare_panel(dp, false, false); +- if (ret) +- DRM_ERROR("Failed to unprepare panel (%d)\n", ret); + } + + if (dp->plat_data->get_modes) +@@ -1186,23 +1147,16 @@ analogix_dp_detect(struct drm_connector *connector, bool force) + { + struct analogix_dp_device *dp = to_dp(connector); + enum drm_connector_status status = connector_status_disconnected; +- int ret; + + if (dp->plat_data->panel) +- return connector_status_connected; ++ analogix_dp_panel_prepare(dp); + +- ret = analogix_dp_prepare_panel(dp, true, false); +- if (ret) { +- DRM_ERROR("Failed to prepare panel (%d)\n", ret); +- return connector_status_disconnected; +- } ++ pm_runtime_get_sync(dp->dev); + + if (!analogix_dp_detect_hpd(dp)) + status = connector_status_connected; + +- ret = analogix_dp_prepare_panel(dp, false, false); +- if (ret) +- DRM_ERROR("Failed to unprepare panel (%d)\n", ret); ++ pm_runtime_put(dp->dev); + + return status; + } +@@ -1295,7 +1249,6 @@ analogix_dp_bridge_atomic_pre_enable(struct drm_bridge *bridge, + struct analogix_dp_device *dp = bridge->driver_private; + struct drm_crtc *crtc; + struct drm_crtc_state *old_crtc_state; +- int ret; + + crtc = analogix_dp_get_new_crtc(dp, old_state); + if (!crtc) +@@ -1306,9 +1259,8 @@ analogix_dp_bridge_atomic_pre_enable(struct drm_bridge *bridge, + if (old_crtc_state && old_crtc_state->self_refresh_active) + return; + +- ret = analogix_dp_prepare_panel(dp, true, true); +- if (ret) +- DRM_ERROR("failed to setup the panel ret = %d\n", ret); ++ if (dp->plat_data->panel) ++ analogix_dp_panel_prepare(dp); + } + + static int analogix_dp_set_bridge(struct analogix_dp_device *dp) +@@ -1317,16 +1269,10 @@ static int analogix_dp_set_bridge(struct analogix_dp_device *dp) + + pm_runtime_get_sync(dp->dev); + +- ret = clk_prepare_enable(dp->clock); +- if (ret < 0) { +- DRM_ERROR("Failed to prepare_enable the clock clk [%d]\n", ret); +- goto out_dp_clk_pre; +- } +- + if (dp->plat_data->power_on_start) + dp->plat_data->power_on_start(dp->plat_data); + +- phy_power_on(dp->phy); ++ analogix_dp_phy_power_on(dp); + + ret = analogix_dp_init_dp(dp); + if (ret) +@@ -1344,11 +1290,14 @@ static int analogix_dp_set_bridge(struct analogix_dp_device *dp) + } + + ret = analogix_dp_commit(dp); +- if (ret) { ++ if (ret < 0) { + DRM_ERROR("dp commit error, ret = %d\n", ret); + goto out_dp_init; + } + ++ if (dp->plat_data->panel) ++ drm_panel_enable(dp->plat_data->panel); ++ + if (dp->plat_data->power_on_end) + dp->plat_data->power_on_end(dp->plat_data); + +@@ -1356,11 +1305,9 @@ static int analogix_dp_set_bridge(struct analogix_dp_device *dp) + return 0; + + out_dp_init: +- phy_power_off(dp->phy); ++ analogix_dp_phy_power_off(dp); + if (dp->plat_data->power_off) + dp->plat_data->power_off(dp->plat_data); +- clk_disable_unprepare(dp->clock); +-out_dp_clk_pre: + pm_runtime_put_sync(dp->dev); + + return ret; +@@ -1409,7 +1356,6 @@ analogix_dp_bridge_atomic_enable(struct drm_bridge *bridge, + static void analogix_dp_bridge_disable(struct drm_bridge *bridge) + { + struct analogix_dp_device *dp = bridge->driver_private; +- int ret; + + if (dp->dpms_mode != DRM_MODE_DPMS_ON) + return; +@@ -1426,16 +1372,14 @@ static void analogix_dp_bridge_disable(struct drm_bridge *bridge) + if (dp->plat_data->power_off) + dp->plat_data->power_off(dp->plat_data); + ++ analogix_dp_reset_aux(dp); + analogix_dp_set_analog_power_down(dp, POWER_ALL, 1); +- phy_power_off(dp->phy); +- +- clk_disable_unprepare(dp->clock); ++ analogix_dp_phy_power_off(dp); + + pm_runtime_put_sync(dp->dev); + +- ret = analogix_dp_prepare_panel(dp, false, true); +- if (ret) +- DRM_ERROR("failed to setup the panel ret = %d\n", ret); ++ if (dp->plat_data->panel) ++ analogix_dp_panel_unprepare(dp); + + dp->fast_train_enable = false; + dp->psr_supported = false; +@@ -1500,6 +1444,8 @@ static void analogix_dp_bridge_mode_set(struct drm_bridge *bridge, + struct device_node *dp_node = dp->dev->of_node; + int vic; + ++ drm_mode_copy(&video->mode, mode); ++ + /* Input video interlaces & hsync pol & vsync pol */ + video->interlaced = !!(mode->flags & DRM_MODE_FLAG_INTERLACE); + video->v_sync_polarity = !!(mode->flags & DRM_MODE_FLAG_NVSYNC); +@@ -1567,6 +1513,21 @@ static void analogix_dp_bridge_mode_set(struct drm_bridge *bridge, + video->interlaced = true; + } + ++static enum drm_mode_status ++analogix_dp_bridge_mode_valid(struct drm_bridge *bridge, ++ const struct drm_display_info *info, ++ const struct drm_display_mode *mode) ++{ ++ struct analogix_dp_device *dp = bridge->driver_private; ++ ++ if (!analogix_dp_bandwidth_ok(dp, mode, ++ drm_dp_bw_code_to_link_rate(dp->video_info.max_link_rate), ++ dp->video_info.max_lane_count)) ++ return MODE_BAD; ++ ++ return MODE_OK; ++} ++ + static const struct drm_bridge_funcs analogix_dp_bridge_funcs = { + .atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state, + .atomic_destroy_state = drm_atomic_helper_bridge_destroy_state, +@@ -1577,6 +1538,7 @@ static const struct drm_bridge_funcs analogix_dp_bridge_funcs = { + .atomic_post_disable = analogix_dp_bridge_atomic_post_disable, + .mode_set = analogix_dp_bridge_mode_set, + .attach = analogix_dp_bridge_attach, ++ .mode_valid = analogix_dp_bridge_mode_valid, + }; + + static int analogix_dp_create_bridge(struct drm_device *drm_dev, +@@ -1613,6 +1575,7 @@ static int analogix_dp_dt_parse_pdata(struct analogix_dp_device *dp) + switch (dp->plat_data->dev_type) { + case RK3288_DP: + case RK3399_EDP: ++ case RK3568_EDP: + /* + * Like Rk3288 DisplayPort TRM indicate that "Main link + * containing 4 physical lanes of 2.7/1.62 Gbps/lane". +@@ -1632,6 +1595,9 @@ static int analogix_dp_dt_parse_pdata(struct analogix_dp_device *dp) + break; + } + ++ video_info->video_bist_enable = ++ of_property_read_bool(dp_node, "analogix,video-bist-enable"); ++ + return 0; + } + +@@ -1643,13 +1609,54 @@ static ssize_t analogix_dpaux_transfer(struct drm_dp_aux *aux, + return analogix_dp_transfer(dp, msg); + } + ++int analogix_dp_audio_hw_params(struct analogix_dp_device *dp, ++ struct hdmi_codec_daifmt *daifmt, ++ struct hdmi_codec_params *params) ++{ ++ switch (daifmt->fmt) { ++ case HDMI_SPDIF: ++ analogix_dp_audio_config_spdif(dp); ++ break; ++ case HDMI_I2S: ++ analogix_dp_audio_config_i2s(dp); ++ break; ++ default: ++ DRM_DEV_ERROR(dp->dev, "invalid daifmt %d\n", daifmt->fmt); ++ return -EINVAL; ++ } ++ ++ return 0; ++} ++EXPORT_SYMBOL_GPL(analogix_dp_audio_hw_params); ++ ++void analogix_dp_audio_shutdown(struct analogix_dp_device *dp) ++{ ++ analogix_dp_audio_disable(dp); ++} ++EXPORT_SYMBOL_GPL(analogix_dp_audio_shutdown); ++ ++int analogix_dp_audio_startup(struct analogix_dp_device *dp) ++{ ++ analogix_dp_audio_enable(dp); ++ ++ return 0; ++} ++EXPORT_SYMBOL_GPL(analogix_dp_audio_startup); ++ ++int analogix_dp_audio_get_eld(struct analogix_dp_device *dp, u8 *buf, size_t len) ++{ ++ memcpy(buf, dp->connector.eld, min(sizeof(dp->connector.eld), len)); ++ ++ return 0; ++} ++EXPORT_SYMBOL_GPL(analogix_dp_audio_get_eld); ++ + struct analogix_dp_device * + analogix_dp_probe(struct device *dev, struct analogix_dp_plat_data *plat_data) + { + struct platform_device *pdev = to_platform_device(dev); + struct analogix_dp_device *dp; + struct resource *res; +- unsigned int irq_flags; + int ret; + + if (!plat_data) { +@@ -1665,7 +1672,7 @@ analogix_dp_probe(struct device *dev, struct analogix_dp_plat_data *plat_data) + dp->dpms_mode = DRM_MODE_DPMS_OFF; + + mutex_init(&dp->panel_lock); +- dp->panel_is_modeset = false; ++ dp->panel_is_prepared = false; + + /* + * platform dp driver need containor_of the plat_data to get +@@ -1694,13 +1701,13 @@ analogix_dp_probe(struct device *dev, struct analogix_dp_plat_data *plat_data) + } + } + +- dp->clock = devm_clk_get(&pdev->dev, "dp"); +- if (IS_ERR(dp->clock)) { +- dev_err(&pdev->dev, "failed to get clock\n"); +- return ERR_CAST(dp->clock); ++ ret = devm_clk_bulk_get_all(dev, &dp->clks); ++ if (ret < 0) { ++ dev_err(dev, "failed to get clocks %d\n", ret); ++ return ERR_PTR(ret); + } + +- clk_prepare_enable(dp->clock); ++ dp->nr_clks = ret; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + +@@ -1722,34 +1729,35 @@ analogix_dp_probe(struct device *dev, struct analogix_dp_plat_data *plat_data) + } + + if (dp->hpd_gpiod) { +- /* +- * Set up the hotplug GPIO from the device tree as an interrupt. +- * Simply specifying a different interrupt in the device tree +- * doesn't work since we handle hotplug rather differently when +- * using a GPIO. We also need the actual GPIO specifier so +- * that we can get the current state of the GPIO. +- */ +- dp->irq = gpiod_to_irq(dp->hpd_gpiod); +- irq_flags = IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING; +- } else { +- dp->irq = platform_get_irq(pdev, 0); +- irq_flags = 0; ++ ret = devm_request_threaded_irq(dev, ++ gpiod_to_irq(dp->hpd_gpiod), ++ NULL, ++ analogix_dp_hpd_irq_handler, ++ IRQF_TRIGGER_RISING | ++ IRQF_TRIGGER_FALLING | ++ IRQF_ONESHOT, ++ "analogix-hpd", dp); ++ if (ret) { ++ dev_err(dev, "failed to request hpd IRQ: %d\n", ret); ++ return ERR_PTR(ret); ++ } + } + ++ dp->irq = platform_get_irq(pdev, 0); + if (dp->irq == -ENXIO) { + dev_err(&pdev->dev, "failed to get irq\n"); + return ERR_PTR(-ENODEV); + } + ++ irq_set_status_flags(dp->irq, IRQ_NOAUTOEN); + ret = devm_request_threaded_irq(&pdev->dev, dp->irq, + analogix_dp_hardirq, + analogix_dp_irq_thread, +- irq_flags, "analogix-dp", dp); ++ 0, "analogix-dp", dp); + if (ret) { + dev_err(&pdev->dev, "failed to request irq\n"); + return ERR_PTR(ret); + } +- disable_irq(dp->irq); + + return dp; + } +@@ -1804,45 +1812,22 @@ EXPORT_SYMBOL_GPL(analogix_dp_unbind); + + void analogix_dp_remove(struct analogix_dp_device *dp) + { +- clk_disable_unprepare(dp->clock); + } + EXPORT_SYMBOL_GPL(analogix_dp_remove); + +-#ifdef CONFIG_PM +-int analogix_dp_suspend(struct analogix_dp_device *dp) ++int analogix_dp_runtime_suspend(struct analogix_dp_device *dp) + { +- clk_disable_unprepare(dp->clock); +- +- if (dp->plat_data->panel) { +- if (drm_panel_unprepare(dp->plat_data->panel)) +- DRM_ERROR("failed to turnoff the panel\n"); +- } ++ clk_bulk_disable_unprepare(dp->nr_clks, dp->clks); + + return 0; + } +-EXPORT_SYMBOL_GPL(analogix_dp_suspend); ++EXPORT_SYMBOL_GPL(analogix_dp_runtime_suspend); + +-int analogix_dp_resume(struct analogix_dp_device *dp) ++int analogix_dp_runtime_resume(struct analogix_dp_device *dp) + { +- int ret; +- +- ret = clk_prepare_enable(dp->clock); +- if (ret < 0) { +- DRM_ERROR("Failed to prepare_enable the clock clk [%d]\n", ret); +- return ret; +- } +- +- if (dp->plat_data->panel) { +- if (drm_panel_prepare(dp->plat_data->panel)) { +- DRM_ERROR("failed to setup the panel\n"); +- return -EBUSY; +- } +- } +- +- return 0; ++ return clk_bulk_prepare_enable(dp->nr_clks, dp->clks); + } +-EXPORT_SYMBOL_GPL(analogix_dp_resume); +-#endif ++EXPORT_SYMBOL_GPL(analogix_dp_runtime_resume); + + int analogix_dp_start_crc(struct drm_connector *connector) + { +diff --git a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.h b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.h +index c051502d7fbf..de32a352383d 100644 +--- a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.h ++++ b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.h +@@ -129,6 +129,7 @@ enum dp_irq_type { + + struct video_info { + char *name; ++ struct drm_display_mode mode; + + bool h_sync_polarity; + bool v_sync_polarity; +@@ -141,6 +142,8 @@ struct video_info { + + int max_link_rate; + enum link_lane_count_type max_lane_count; ++ ++ bool video_bist_enable; + }; + + struct link_train { +@@ -150,6 +153,8 @@ struct link_train { + u8 link_rate; + u8 lane_count; + u8 training_lane[4]; ++ bool ssc; ++ bool enhanced_framing; + + enum link_training_state lt_state; + }; +@@ -161,13 +166,15 @@ struct analogix_dp_device { + struct drm_connector connector; + struct drm_bridge *bridge; + struct drm_dp_aux aux; +- struct clk *clock; ++ struct clk_bulk_data *clks; ++ int nr_clks; + unsigned int irq; + void __iomem *reg_base; + + struct video_info video_info; + struct link_train link_train; + struct phy *phy; ++ bool phy_enabled; + int dpms_mode; + struct gpio_desc *hpd_gpiod; + bool force_hpd; +@@ -175,7 +182,7 @@ struct analogix_dp_device { + bool psr_supported; + + struct mutex panel_lock; +- bool panel_is_modeset; ++ bool panel_is_prepared; + + struct analogix_dp_plat_data *plat_data; + }; +@@ -213,26 +220,8 @@ void analogix_dp_enable_enhanced_mode(struct analogix_dp_device *dp, + bool enable); + void analogix_dp_set_training_pattern(struct analogix_dp_device *dp, + enum pattern_set pattern); +-void analogix_dp_set_lane0_pre_emphasis(struct analogix_dp_device *dp, +- u32 level); +-void analogix_dp_set_lane1_pre_emphasis(struct analogix_dp_device *dp, +- u32 level); +-void analogix_dp_set_lane2_pre_emphasis(struct analogix_dp_device *dp, +- u32 level); +-void analogix_dp_set_lane3_pre_emphasis(struct analogix_dp_device *dp, +- u32 level); +-void analogix_dp_set_lane0_link_training(struct analogix_dp_device *dp, +- u32 training_lane); +-void analogix_dp_set_lane1_link_training(struct analogix_dp_device *dp, +- u32 training_lane); +-void analogix_dp_set_lane2_link_training(struct analogix_dp_device *dp, +- u32 training_lane); +-void analogix_dp_set_lane3_link_training(struct analogix_dp_device *dp, +- u32 training_lane); +-u32 analogix_dp_get_lane0_link_training(struct analogix_dp_device *dp); +-u32 analogix_dp_get_lane1_link_training(struct analogix_dp_device *dp); +-u32 analogix_dp_get_lane2_link_training(struct analogix_dp_device *dp); +-u32 analogix_dp_get_lane3_link_training(struct analogix_dp_device *dp); ++void analogix_dp_set_lane_link_training(struct analogix_dp_device *dp); ++u32 analogix_dp_get_lane_link_training(struct analogix_dp_device *dp, u8 lane); + void analogix_dp_reset_macro(struct analogix_dp_device *dp); + void analogix_dp_init_video(struct analogix_dp_device *dp); + +@@ -255,5 +244,14 @@ int analogix_dp_send_psr_spd(struct analogix_dp_device *dp, + struct dp_sdp *vsc, bool blocking); + ssize_t analogix_dp_transfer(struct analogix_dp_device *dp, + struct drm_dp_aux_msg *msg); ++void analogix_dp_set_video_format(struct analogix_dp_device *dp); ++void analogix_dp_video_bist_enable(struct analogix_dp_device *dp); ++bool analogix_dp_ssc_supported(struct analogix_dp_device *dp); ++void analogix_dp_phy_power_on(struct analogix_dp_device *dp); ++void analogix_dp_phy_power_off(struct analogix_dp_device *dp); ++void analogix_dp_audio_config_spdif(struct analogix_dp_device *dp); ++void analogix_dp_audio_config_i2s(struct analogix_dp_device *dp); ++void analogix_dp_audio_enable(struct analogix_dp_device *dp); ++void analogix_dp_audio_disable(struct analogix_dp_device *dp); + + #endif /* _ANALOGIX_DP_CORE_H */ +diff --git a/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c b/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c +index 914c569ab8c1..ca91db220580 100644 +--- a/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c ++++ b/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c +@@ -11,6 +11,7 @@ + #include + #include + #include ++#include + + #include + +@@ -21,20 +22,37 @@ + #define COMMON_INT_MASK_2 0 + #define COMMON_INT_MASK_3 0 + #define COMMON_INT_MASK_4 (HOTPLUG_CHG | HPD_LOST | PLUG) +-#define INT_STA_MASK INT_HPD ++ ++static void analogix_dp_write(struct analogix_dp_device *dp, u32 reg, u32 val) ++{ ++ if (dp->plat_data && is_rockchip(dp->plat_data->dev_type)) { ++ readl(dp->reg_base); ++ writel(val, dp->reg_base + reg); ++ } ++ ++ writel(val, dp->reg_base + reg); ++} ++ ++static u32 analogix_dp_read(struct analogix_dp_device *dp, u32 reg) ++{ ++ if (dp->plat_data && is_rockchip(dp->plat_data->dev_type)) ++ readl(dp->reg_base + reg); ++ ++ return readl(dp->reg_base + reg); ++} + + void analogix_dp_enable_video_mute(struct analogix_dp_device *dp, bool enable) + { + u32 reg; + + if (enable) { +- reg = readl(dp->reg_base + ANALOGIX_DP_VIDEO_CTL_1); ++ reg = analogix_dp_read(dp, ANALOGIX_DP_VIDEO_CTL_1); + reg |= HDCP_VIDEO_MUTE; +- writel(reg, dp->reg_base + ANALOGIX_DP_VIDEO_CTL_1); ++ analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_1, reg); + } else { +- reg = readl(dp->reg_base + ANALOGIX_DP_VIDEO_CTL_1); ++ reg = analogix_dp_read(dp, ANALOGIX_DP_VIDEO_CTL_1); + reg &= ~HDCP_VIDEO_MUTE; +- writel(reg, dp->reg_base + ANALOGIX_DP_VIDEO_CTL_1); ++ analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_1, reg); + } + } + +@@ -42,9 +60,9 @@ void analogix_dp_stop_video(struct analogix_dp_device *dp) + { + u32 reg; + +- reg = readl(dp->reg_base + ANALOGIX_DP_VIDEO_CTL_1); ++ reg = analogix_dp_read(dp, ANALOGIX_DP_VIDEO_CTL_1); + reg &= ~VIDEO_EN; +- writel(reg, dp->reg_base + ANALOGIX_DP_VIDEO_CTL_1); ++ analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_1, reg); + } + + void analogix_dp_lane_swap(struct analogix_dp_device *dp, bool enable) +@@ -58,7 +76,7 @@ void analogix_dp_lane_swap(struct analogix_dp_device *dp, bool enable) + reg = LANE3_MAP_LOGIC_LANE_3 | LANE2_MAP_LOGIC_LANE_2 | + LANE1_MAP_LOGIC_LANE_1 | LANE0_MAP_LOGIC_LANE_0; + +- writel(reg, dp->reg_base + ANALOGIX_DP_LANE_MAP); ++ analogix_dp_write(dp, ANALOGIX_DP_LANE_MAP, reg); + } + + void analogix_dp_init_analog_param(struct analogix_dp_device *dp) +@@ -66,53 +84,53 @@ void analogix_dp_init_analog_param(struct analogix_dp_device *dp) + u32 reg; + + reg = TX_TERMINAL_CTRL_50_OHM; +- writel(reg, dp->reg_base + ANALOGIX_DP_ANALOG_CTL_1); ++ analogix_dp_write(dp, ANALOGIX_DP_ANALOG_CTL_1, reg); + + reg = SEL_24M | TX_DVDD_BIT_1_0625V; +- writel(reg, dp->reg_base + ANALOGIX_DP_ANALOG_CTL_2); ++ analogix_dp_write(dp, ANALOGIX_DP_ANALOG_CTL_2, reg); + + if (dp->plat_data && is_rockchip(dp->plat_data->dev_type)) { + reg = REF_CLK_24M; + if (dp->plat_data->dev_type == RK3288_DP) + reg ^= REF_CLK_MASK; + +- writel(reg, dp->reg_base + ANALOGIX_DP_PLL_REG_1); +- writel(0x95, dp->reg_base + ANALOGIX_DP_PLL_REG_2); +- writel(0x40, dp->reg_base + ANALOGIX_DP_PLL_REG_3); +- writel(0x58, dp->reg_base + ANALOGIX_DP_PLL_REG_4); +- writel(0x22, dp->reg_base + ANALOGIX_DP_PLL_REG_5); ++ analogix_dp_write(dp, ANALOGIX_DP_PLL_REG_1, reg); ++ analogix_dp_write(dp, ANALOGIX_DP_PLL_REG_2, 0x95); ++ analogix_dp_write(dp, ANALOGIX_DP_PLL_REG_3, 0x40); ++ analogix_dp_write(dp, ANALOGIX_DP_PLL_REG_4, 0x58); ++ analogix_dp_write(dp, ANALOGIX_DP_PLL_REG_5, 0x22); + } + + reg = DRIVE_DVDD_BIT_1_0625V | VCO_BIT_600_MICRO; +- writel(reg, dp->reg_base + ANALOGIX_DP_ANALOG_CTL_3); ++ analogix_dp_write(dp, ANALOGIX_DP_ANALOG_CTL_3, reg); + + reg = PD_RING_OSC | AUX_TERMINAL_CTRL_50_OHM | + TX_CUR1_2X | TX_CUR_16_MA; +- writel(reg, dp->reg_base + ANALOGIX_DP_PLL_FILTER_CTL_1); ++ analogix_dp_write(dp, ANALOGIX_DP_PLL_FILTER_CTL_1, reg); + + reg = CH3_AMP_400_MV | CH2_AMP_400_MV | + CH1_AMP_400_MV | CH0_AMP_400_MV; +- writel(reg, dp->reg_base + ANALOGIX_DP_TX_AMP_TUNING_CTL); ++ analogix_dp_write(dp, ANALOGIX_DP_TX_AMP_TUNING_CTL, reg); + } + + void analogix_dp_init_interrupt(struct analogix_dp_device *dp) + { + /* Set interrupt pin assertion polarity as high */ +- writel(INT_POL1 | INT_POL0, dp->reg_base + ANALOGIX_DP_INT_CTL); ++ analogix_dp_write(dp, ANALOGIX_DP_INT_CTL, INT_POL1 | INT_POL0); + + /* Clear pending regisers */ +- writel(0xff, dp->reg_base + ANALOGIX_DP_COMMON_INT_STA_1); +- writel(0x4f, dp->reg_base + ANALOGIX_DP_COMMON_INT_STA_2); +- writel(0xe0, dp->reg_base + ANALOGIX_DP_COMMON_INT_STA_3); +- writel(0xe7, dp->reg_base + ANALOGIX_DP_COMMON_INT_STA_4); +- writel(0x63, dp->reg_base + ANALOGIX_DP_INT_STA); ++ analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_STA_1, 0xff); ++ analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_STA_2, 0x4f); ++ analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_STA_3, 0xe0); ++ analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_STA_4, 0xe7); ++ analogix_dp_write(dp, ANALOGIX_DP_INT_STA, 0x63); + + /* 0:mask,1: unmask */ +- writel(0x00, dp->reg_base + ANALOGIX_DP_COMMON_INT_MASK_1); +- writel(0x00, dp->reg_base + ANALOGIX_DP_COMMON_INT_MASK_2); +- writel(0x00, dp->reg_base + ANALOGIX_DP_COMMON_INT_MASK_3); +- writel(0x00, dp->reg_base + ANALOGIX_DP_COMMON_INT_MASK_4); +- writel(0x00, dp->reg_base + ANALOGIX_DP_INT_STA_MASK); ++ analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_MASK_1, 0x00); ++ analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_MASK_2, 0x00); ++ analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_MASK_3, 0x00); ++ analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_MASK_4, 0x00); ++ analogix_dp_write(dp, ANALOGIX_DP_INT_STA_MASK, 0x00); + } + + void analogix_dp_reset(struct analogix_dp_device *dp) +@@ -130,44 +148,44 @@ void analogix_dp_reset(struct analogix_dp_device *dp) + AUD_FIFO_FUNC_EN_N | AUD_FUNC_EN_N | + HDCP_FUNC_EN_N | SW_FUNC_EN_N; + +- writel(reg, dp->reg_base + ANALOGIX_DP_FUNC_EN_1); ++ analogix_dp_write(dp, ANALOGIX_DP_FUNC_EN_1, reg); + + reg = SSC_FUNC_EN_N | AUX_FUNC_EN_N | + SERDES_FIFO_FUNC_EN_N | + LS_CLK_DOMAIN_FUNC_EN_N; +- writel(reg, dp->reg_base + ANALOGIX_DP_FUNC_EN_2); ++ analogix_dp_write(dp, ANALOGIX_DP_FUNC_EN_2, reg); + + usleep_range(20, 30); + + analogix_dp_lane_swap(dp, 0); + +- writel(0x0, dp->reg_base + ANALOGIX_DP_SYS_CTL_1); +- writel(0x40, dp->reg_base + ANALOGIX_DP_SYS_CTL_2); +- writel(0x0, dp->reg_base + ANALOGIX_DP_SYS_CTL_3); +- writel(0x0, dp->reg_base + ANALOGIX_DP_SYS_CTL_4); ++ analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_1, 0x0); ++ analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_2, 0x40); ++ analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_3, 0x0); ++ analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_4, 0x0); + +- writel(0x0, dp->reg_base + ANALOGIX_DP_PKT_SEND_CTL); +- writel(0x0, dp->reg_base + ANALOGIX_DP_HDCP_CTL); ++ analogix_dp_write(dp, ANALOGIX_DP_PKT_SEND_CTL, 0x0); ++ analogix_dp_write(dp, ANALOGIX_DP_HDCP_CTL, 0x0); + +- writel(0x5e, dp->reg_base + ANALOGIX_DP_HPD_DEGLITCH_L); +- writel(0x1a, dp->reg_base + ANALOGIX_DP_HPD_DEGLITCH_H); ++ analogix_dp_write(dp, ANALOGIX_DP_HPD_DEGLITCH_L, 0x5e); ++ analogix_dp_write(dp, ANALOGIX_DP_HPD_DEGLITCH_H, 0x1a); + +- writel(0x10, dp->reg_base + ANALOGIX_DP_LINK_DEBUG_CTL); ++ analogix_dp_write(dp, ANALOGIX_DP_LINK_DEBUG_CTL, 0x10); + +- writel(0x0, dp->reg_base + ANALOGIX_DP_PHY_TEST); ++ analogix_dp_write(dp, ANALOGIX_DP_PHY_TEST, 0x0); + +- writel(0x0, dp->reg_base + ANALOGIX_DP_VIDEO_FIFO_THRD); +- writel(0x20, dp->reg_base + ANALOGIX_DP_AUDIO_MARGIN); ++ analogix_dp_write(dp, ANALOGIX_DP_VIDEO_FIFO_THRD, 0x0); ++ analogix_dp_write(dp, ANALOGIX_DP_AUDIO_MARGIN, 0x20); + +- writel(0x4, dp->reg_base + ANALOGIX_DP_M_VID_GEN_FILTER_TH); +- writel(0x2, dp->reg_base + ANALOGIX_DP_M_AUD_GEN_FILTER_TH); ++ analogix_dp_write(dp, ANALOGIX_DP_M_VID_GEN_FILTER_TH, 0x4); ++ analogix_dp_write(dp, ANALOGIX_DP_M_AUD_GEN_FILTER_TH, 0x2); + +- writel(0x00000101, dp->reg_base + ANALOGIX_DP_SOC_GENERAL_CTL); ++ analogix_dp_write(dp, ANALOGIX_DP_SOC_GENERAL_CTL, 0x00000101); + } + + void analogix_dp_swreset(struct analogix_dp_device *dp) + { +- writel(RESET_DP_TX, dp->reg_base + ANALOGIX_DP_TX_SW_RESET); ++ analogix_dp_write(dp, ANALOGIX_DP_TX_SW_RESET, RESET_DP_TX); + } + + void analogix_dp_config_interrupt(struct analogix_dp_device *dp) +@@ -176,19 +194,18 @@ void analogix_dp_config_interrupt(struct analogix_dp_device *dp) + + /* 0: mask, 1: unmask */ + reg = COMMON_INT_MASK_1; +- writel(reg, dp->reg_base + ANALOGIX_DP_COMMON_INT_MASK_1); ++ analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_MASK_1, reg); + + reg = COMMON_INT_MASK_2; +- writel(reg, dp->reg_base + ANALOGIX_DP_COMMON_INT_MASK_2); ++ analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_MASK_2, reg); + + reg = COMMON_INT_MASK_3; +- writel(reg, dp->reg_base + ANALOGIX_DP_COMMON_INT_MASK_3); ++ analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_MASK_3, reg); + +- reg = COMMON_INT_MASK_4; +- writel(reg, dp->reg_base + ANALOGIX_DP_COMMON_INT_MASK_4); +- +- reg = INT_STA_MASK; +- writel(reg, dp->reg_base + ANALOGIX_DP_INT_STA_MASK); ++ if (dp->force_hpd || dp->hpd_gpiod) ++ analogix_dp_mute_hpd_interrupt(dp); ++ else ++ analogix_dp_unmute_hpd_interrupt(dp); + } + + void analogix_dp_mute_hpd_interrupt(struct analogix_dp_device *dp) +@@ -196,13 +213,13 @@ void analogix_dp_mute_hpd_interrupt(struct analogix_dp_device *dp) + u32 reg; + + /* 0: mask, 1: unmask */ +- reg = readl(dp->reg_base + ANALOGIX_DP_COMMON_INT_MASK_4); ++ reg = analogix_dp_read(dp, ANALOGIX_DP_COMMON_INT_MASK_4); + reg &= ~COMMON_INT_MASK_4; +- writel(reg, dp->reg_base + ANALOGIX_DP_COMMON_INT_MASK_4); ++ analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_MASK_4, reg); + +- reg = readl(dp->reg_base + ANALOGIX_DP_INT_STA_MASK); +- reg &= ~INT_STA_MASK; +- writel(reg, dp->reg_base + ANALOGIX_DP_INT_STA_MASK); ++ reg = analogix_dp_read(dp, ANALOGIX_DP_INT_STA_MASK); ++ reg &= ~INT_HPD; ++ analogix_dp_write(dp, ANALOGIX_DP_INT_STA_MASK, reg); + } + + void analogix_dp_unmute_hpd_interrupt(struct analogix_dp_device *dp) +@@ -211,17 +228,18 @@ void analogix_dp_unmute_hpd_interrupt(struct analogix_dp_device *dp) + + /* 0: mask, 1: unmask */ + reg = COMMON_INT_MASK_4; +- writel(reg, dp->reg_base + ANALOGIX_DP_COMMON_INT_MASK_4); ++ analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_MASK_4, reg); + +- reg = INT_STA_MASK; +- writel(reg, dp->reg_base + ANALOGIX_DP_INT_STA_MASK); ++ reg = analogix_dp_read(dp, ANALOGIX_DP_INT_STA_MASK); ++ reg |= INT_HPD; ++ analogix_dp_write(dp, ANALOGIX_DP_INT_STA_MASK, reg); + } + + enum pll_status analogix_dp_get_pll_lock_status(struct analogix_dp_device *dp) + { + u32 reg; + +- reg = readl(dp->reg_base + ANALOGIX_DP_DEBUG_CTL); ++ reg = analogix_dp_read(dp, ANALOGIX_DP_DEBUG_CTL); + if (reg & PLL_LOCK) + return PLL_LOCKED; + else +@@ -239,12 +257,12 @@ void analogix_dp_set_pll_power_down(struct analogix_dp_device *dp, bool enable) + mask = RK_PLL_PD; + } + +- reg = readl(dp->reg_base + pd_addr); ++ reg = analogix_dp_read(dp, pd_addr); + if (enable) + reg |= mask; + else + reg &= ~mask; +- writel(reg, dp->reg_base + pd_addr); ++ analogix_dp_write(dp, pd_addr, reg); + } + + void analogix_dp_set_analog_power_down(struct analogix_dp_device *dp, +@@ -265,52 +283,54 @@ void analogix_dp_set_analog_power_down(struct analogix_dp_device *dp, + else + mask = AUX_PD; + +- reg = readl(dp->reg_base + phy_pd_addr); +- if (enable) ++ reg = analogix_dp_read(dp, phy_pd_addr); ++ if (enable) { ++ reg &= ~(DP_INC_BG | DP_EXP_BG); + reg |= mask; +- else ++ } else { + reg &= ~mask; +- writel(reg, dp->reg_base + phy_pd_addr); ++ } ++ analogix_dp_write(dp, phy_pd_addr, reg); + break; + case CH0_BLOCK: + mask = CH0_PD; +- reg = readl(dp->reg_base + phy_pd_addr); ++ reg = analogix_dp_read(dp, phy_pd_addr); + + if (enable) + reg |= mask; + else + reg &= ~mask; +- writel(reg, dp->reg_base + phy_pd_addr); ++ analogix_dp_write(dp, phy_pd_addr, reg); + break; + case CH1_BLOCK: + mask = CH1_PD; +- reg = readl(dp->reg_base + phy_pd_addr); ++ reg = analogix_dp_read(dp, phy_pd_addr); + + if (enable) + reg |= mask; + else + reg &= ~mask; +- writel(reg, dp->reg_base + phy_pd_addr); ++ analogix_dp_write(dp, phy_pd_addr, reg); + break; + case CH2_BLOCK: + mask = CH2_PD; +- reg = readl(dp->reg_base + phy_pd_addr); ++ reg = analogix_dp_read(dp, phy_pd_addr); + + if (enable) + reg |= mask; + else + reg &= ~mask; +- writel(reg, dp->reg_base + phy_pd_addr); ++ analogix_dp_write(dp, phy_pd_addr, reg); + break; + case CH3_BLOCK: + mask = CH3_PD; +- reg = readl(dp->reg_base + phy_pd_addr); ++ reg = analogix_dp_read(dp, phy_pd_addr); + + if (enable) + reg |= mask; + else + reg &= ~mask; +- writel(reg, dp->reg_base + phy_pd_addr); ++ analogix_dp_write(dp, phy_pd_addr, reg); + break; + case ANALOG_TOTAL: + /* +@@ -323,29 +343,29 @@ void analogix_dp_set_analog_power_down(struct analogix_dp_device *dp, + else + mask = DP_PHY_PD; + +- reg = readl(dp->reg_base + phy_pd_addr); ++ reg = analogix_dp_read(dp, phy_pd_addr); + if (enable) + reg |= mask; + else + reg &= ~mask; + +- writel(reg, dp->reg_base + phy_pd_addr); ++ analogix_dp_write(dp, phy_pd_addr, reg); + if (dp->plat_data && is_rockchip(dp->plat_data->dev_type)) + usleep_range(10, 15); + break; + case POWER_ALL: + if (enable) { + reg = DP_ALL_PD; +- writel(reg, dp->reg_base + phy_pd_addr); ++ analogix_dp_write(dp, phy_pd_addr, reg); + } else { + reg = DP_ALL_PD; +- writel(reg, dp->reg_base + phy_pd_addr); ++ analogix_dp_write(dp, phy_pd_addr, reg); + usleep_range(10, 15); + reg &= ~DP_INC_BG; +- writel(reg, dp->reg_base + phy_pd_addr); ++ analogix_dp_write(dp, phy_pd_addr, reg); + usleep_range(10, 15); + +- writel(0x00, dp->reg_base + phy_pd_addr); ++ analogix_dp_write(dp, phy_pd_addr, 0x00); + } + break; + default: +@@ -356,36 +376,24 @@ void analogix_dp_set_analog_power_down(struct analogix_dp_device *dp, + int analogix_dp_init_analog_func(struct analogix_dp_device *dp) + { + u32 reg; +- int timeout_loop = 0; + + analogix_dp_set_analog_power_down(dp, POWER_ALL, 0); + + reg = PLL_LOCK_CHG; +- writel(reg, dp->reg_base + ANALOGIX_DP_COMMON_INT_STA_1); ++ analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_STA_1, reg); + +- reg = readl(dp->reg_base + ANALOGIX_DP_DEBUG_CTL); ++ reg = analogix_dp_read(dp, ANALOGIX_DP_DEBUG_CTL); + reg &= ~(F_PLL_LOCK | PLL_LOCK_CTRL); +- writel(reg, dp->reg_base + ANALOGIX_DP_DEBUG_CTL); ++ analogix_dp_write(dp, ANALOGIX_DP_DEBUG_CTL, reg); + + /* Power up PLL */ +- if (analogix_dp_get_pll_lock_status(dp) == PLL_UNLOCKED) { +- analogix_dp_set_pll_power_down(dp, 0); +- +- while (analogix_dp_get_pll_lock_status(dp) == PLL_UNLOCKED) { +- timeout_loop++; +- if (DP_TIMEOUT_LOOP_COUNT < timeout_loop) { +- dev_err(dp->dev, "failed to get pll lock status\n"); +- return -ETIMEDOUT; +- } +- usleep_range(10, 20); +- } +- } ++ analogix_dp_set_pll_power_down(dp, 0); + + /* Enable Serdes FIFO function and Link symbol clock domain module */ +- reg = readl(dp->reg_base + ANALOGIX_DP_FUNC_EN_2); ++ reg = analogix_dp_read(dp, ANALOGIX_DP_FUNC_EN_2); + reg &= ~(SERDES_FIFO_FUNC_EN_N | LS_CLK_DOMAIN_FUNC_EN_N + | AUX_FUNC_EN_N); +- writel(reg, dp->reg_base + ANALOGIX_DP_FUNC_EN_2); ++ analogix_dp_write(dp, ANALOGIX_DP_FUNC_EN_2, reg); + return 0; + } + +@@ -397,10 +405,10 @@ void analogix_dp_clear_hotplug_interrupts(struct analogix_dp_device *dp) + return; + + reg = HOTPLUG_CHG | HPD_LOST | PLUG; +- writel(reg, dp->reg_base + ANALOGIX_DP_COMMON_INT_STA_4); ++ analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_STA_4, reg); + + reg = INT_HPD; +- writel(reg, dp->reg_base + ANALOGIX_DP_INT_STA); ++ analogix_dp_write(dp, ANALOGIX_DP_INT_STA, reg); + } + + void analogix_dp_init_hpd(struct analogix_dp_device *dp) +@@ -412,45 +420,37 @@ void analogix_dp_init_hpd(struct analogix_dp_device *dp) + + analogix_dp_clear_hotplug_interrupts(dp); + +- reg = readl(dp->reg_base + ANALOGIX_DP_SYS_CTL_3); ++ reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_3); + reg &= ~(F_HPD | HPD_CTRL); +- writel(reg, dp->reg_base + ANALOGIX_DP_SYS_CTL_3); ++ analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_3, reg); + } + + void analogix_dp_force_hpd(struct analogix_dp_device *dp) + { + u32 reg; + +- reg = readl(dp->reg_base + ANALOGIX_DP_SYS_CTL_3); +- reg = (F_HPD | HPD_CTRL); +- writel(reg, dp->reg_base + ANALOGIX_DP_SYS_CTL_3); ++ reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_3); ++ reg |= (F_HPD | HPD_CTRL); ++ analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_3, reg); + } + + enum dp_irq_type analogix_dp_get_irq_type(struct analogix_dp_device *dp) + { + u32 reg; + +- if (dp->hpd_gpiod) { +- reg = gpiod_get_value(dp->hpd_gpiod); +- if (reg) +- return DP_IRQ_TYPE_HP_CABLE_IN; +- else +- return DP_IRQ_TYPE_HP_CABLE_OUT; +- } else { +- /* Parse hotplug interrupt status register */ +- reg = readl(dp->reg_base + ANALOGIX_DP_COMMON_INT_STA_4); ++ /* Parse hotplug interrupt status register */ ++ reg = analogix_dp_read(dp, ANALOGIX_DP_COMMON_INT_STA_4); + +- if (reg & PLUG) +- return DP_IRQ_TYPE_HP_CABLE_IN; ++ if (reg & PLUG) ++ return DP_IRQ_TYPE_HP_CABLE_IN; + +- if (reg & HPD_LOST) +- return DP_IRQ_TYPE_HP_CABLE_OUT; ++ if (reg & HPD_LOST) ++ return DP_IRQ_TYPE_HP_CABLE_OUT; + +- if (reg & HOTPLUG_CHG) +- return DP_IRQ_TYPE_HP_CHANGE; ++ if (reg & HOTPLUG_CHG) ++ return DP_IRQ_TYPE_HP_CHANGE; + +- return DP_IRQ_TYPE_UNKNOWN; +- } ++ return DP_IRQ_TYPE_UNKNOWN; + } + + void analogix_dp_reset_aux(struct analogix_dp_device *dp) +@@ -458,9 +458,9 @@ void analogix_dp_reset_aux(struct analogix_dp_device *dp) + u32 reg; + + /* Disable AUX channel module */ +- reg = readl(dp->reg_base + ANALOGIX_DP_FUNC_EN_2); ++ reg = analogix_dp_read(dp, ANALOGIX_DP_FUNC_EN_2); + reg |= AUX_FUNC_EN_N; +- writel(reg, dp->reg_base + ANALOGIX_DP_FUNC_EN_2); ++ analogix_dp_write(dp, ANALOGIX_DP_FUNC_EN_2, reg); + } + + void analogix_dp_init_aux(struct analogix_dp_device *dp) +@@ -469,7 +469,7 @@ void analogix_dp_init_aux(struct analogix_dp_device *dp) + + /* Clear inerrupts related to AUX channel */ + reg = RPLY_RECEIV | AUX_ERR; +- writel(reg, dp->reg_base + ANALOGIX_DP_INT_STA); ++ analogix_dp_write(dp, ANALOGIX_DP_INT_STA, reg); + + analogix_dp_set_analog_power_down(dp, AUX_BLOCK, true); + usleep_range(10, 11); +@@ -487,16 +487,17 @@ void analogix_dp_init_aux(struct analogix_dp_device *dp) + reg |= AUX_HW_RETRY_COUNT_SEL(0) | + AUX_HW_RETRY_INTERVAL_600_MICROSECONDS; + +- writel(reg, dp->reg_base + ANALOGIX_DP_AUX_HW_RETRY_CTL); ++ analogix_dp_write(dp, ANALOGIX_DP_AUX_HW_RETRY_CTL, reg); + + /* Receive AUX Channel DEFER commands equal to DEFFER_COUNT*64 */ + reg = DEFER_CTRL_EN | DEFER_COUNT(1); +- writel(reg, dp->reg_base + ANALOGIX_DP_AUX_CH_DEFER_CTL); ++ analogix_dp_write(dp, ANALOGIX_DP_AUX_CH_DEFER_CTL, reg); + + /* Enable AUX channel module */ +- reg = readl(dp->reg_base + ANALOGIX_DP_FUNC_EN_2); ++ analogix_dp_enable_sw_function(dp); ++ reg = analogix_dp_read(dp, ANALOGIX_DP_FUNC_EN_2); + reg &= ~AUX_FUNC_EN_N; +- writel(reg, dp->reg_base + ANALOGIX_DP_FUNC_EN_2); ++ analogix_dp_write(dp, ANALOGIX_DP_FUNC_EN_2, reg); + } + + int analogix_dp_get_plug_in_status(struct analogix_dp_device *dp) +@@ -507,7 +508,7 @@ int analogix_dp_get_plug_in_status(struct analogix_dp_device *dp) + if (gpiod_get_value(dp->hpd_gpiod)) + return 0; + } else { +- reg = readl(dp->reg_base + ANALOGIX_DP_SYS_CTL_3); ++ reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_3); + if (reg & HPD_STATUS) + return 0; + } +@@ -519,145 +520,150 @@ void analogix_dp_enable_sw_function(struct analogix_dp_device *dp) + { + u32 reg; + +- reg = readl(dp->reg_base + ANALOGIX_DP_FUNC_EN_1); ++ reg = analogix_dp_read(dp, ANALOGIX_DP_FUNC_EN_1); + reg &= ~SW_FUNC_EN_N; +- writel(reg, dp->reg_base + ANALOGIX_DP_FUNC_EN_1); +-} +- +-int analogix_dp_start_aux_transaction(struct analogix_dp_device *dp) +-{ +- int reg; +- int retval = 0; +- int timeout_loop = 0; +- +- /* Enable AUX CH operation */ +- reg = readl(dp->reg_base + ANALOGIX_DP_AUX_CH_CTL_2); +- reg |= AUX_EN; +- writel(reg, dp->reg_base + ANALOGIX_DP_AUX_CH_CTL_2); +- +- /* Is AUX CH command reply received? */ +- reg = readl(dp->reg_base + ANALOGIX_DP_INT_STA); +- while (!(reg & RPLY_RECEIV)) { +- timeout_loop++; +- if (DP_TIMEOUT_LOOP_COUNT < timeout_loop) { +- dev_err(dp->dev, "AUX CH command reply failed!\n"); +- return -ETIMEDOUT; +- } +- reg = readl(dp->reg_base + ANALOGIX_DP_INT_STA); +- usleep_range(10, 11); +- } +- +- /* Clear interrupt source for AUX CH command reply */ +- writel(RPLY_RECEIV, dp->reg_base + ANALOGIX_DP_INT_STA); +- +- /* Clear interrupt source for AUX CH access error */ +- reg = readl(dp->reg_base + ANALOGIX_DP_INT_STA); +- if (reg & AUX_ERR) { +- writel(AUX_ERR, dp->reg_base + ANALOGIX_DP_INT_STA); +- return -EREMOTEIO; +- } +- +- /* Check AUX CH error access status */ +- reg = readl(dp->reg_base + ANALOGIX_DP_AUX_CH_STA); +- if ((reg & AUX_STATUS_MASK) != 0) { +- dev_err(dp->dev, "AUX CH error happens: %d\n\n", +- reg & AUX_STATUS_MASK); +- return -EREMOTEIO; +- } +- +- return retval; ++ analogix_dp_write(dp, ANALOGIX_DP_FUNC_EN_1, reg); + } + +-int analogix_dp_write_byte_to_dpcd(struct analogix_dp_device *dp, +- unsigned int reg_addr, +- unsigned char data) ++bool analogix_dp_ssc_supported(struct analogix_dp_device *dp) + { +- u32 reg; +- int i; +- int retval; +- +- for (i = 0; i < 3; i++) { +- /* Clear AUX CH data buffer */ +- reg = BUF_CLR; +- writel(reg, dp->reg_base + ANALOGIX_DP_BUFFER_DATA_CTL); +- +- /* Select DPCD device address */ +- reg = AUX_ADDR_7_0(reg_addr); +- writel(reg, dp->reg_base + ANALOGIX_DP_AUX_ADDR_7_0); +- reg = AUX_ADDR_15_8(reg_addr); +- writel(reg, dp->reg_base + ANALOGIX_DP_AUX_ADDR_15_8); +- reg = AUX_ADDR_19_16(reg_addr); +- writel(reg, dp->reg_base + ANALOGIX_DP_AUX_ADDR_19_16); +- +- /* Write data buffer */ +- reg = (unsigned int)data; +- writel(reg, dp->reg_base + ANALOGIX_DP_BUF_DATA_0); +- +- /* +- * Set DisplayPort transaction and write 1 byte +- * If bit 3 is 1, DisplayPort transaction. +- * If Bit 3 is 0, I2C transaction. +- */ +- reg = AUX_TX_COMM_DP_TRANSACTION | AUX_TX_COMM_WRITE; +- writel(reg, dp->reg_base + ANALOGIX_DP_AUX_CH_CTL_1); +- +- /* Start AUX transaction */ +- retval = analogix_dp_start_aux_transaction(dp); +- if (retval == 0) +- break; +- +- dev_dbg(dp->dev, "%s: Aux Transaction fail!\n", __func__); +- } +- +- return retval; ++ /* Check if SSC is supported by both sides */ ++ return dp->plat_data->ssc && dp->link_train.ssc; + } + + void analogix_dp_set_link_bandwidth(struct analogix_dp_device *dp, u32 bwtype) + { +- u32 reg; ++ u32 reg, status; ++ int ret; + + reg = bwtype; + if ((bwtype == DP_LINK_BW_2_7) || (bwtype == DP_LINK_BW_1_62)) +- writel(reg, dp->reg_base + ANALOGIX_DP_LINK_BW_SET); ++ analogix_dp_write(dp, ANALOGIX_DP_LINK_BW_SET, reg); ++ ++ if (dp->phy) { ++ union phy_configure_opts phy_cfg = {0}; ++ ++ phy_cfg.dp.lanes = dp->link_train.lane_count; ++ phy_cfg.dp.link_rate = ++ drm_dp_bw_code_to_link_rate(dp->link_train.link_rate) / 100; ++ phy_cfg.dp.ssc = analogix_dp_ssc_supported(dp); ++ phy_cfg.dp.set_lanes = false; ++ phy_cfg.dp.set_rate = true; ++ phy_cfg.dp.set_voltages = false; ++ ret = phy_configure(dp->phy, &phy_cfg); ++ if (ret && ret != -EOPNOTSUPP) { ++ dev_err(dp->dev, "%s: phy_configure failed: %d\n", ++ __func__, ret); ++ return; ++ } ++ } ++ ++ ret = readx_poll_timeout(analogix_dp_get_pll_lock_status, dp, status, ++ status != PLL_UNLOCKED, 120, ++ 120 * DP_TIMEOUT_LOOP_COUNT); ++ if (ret) { ++ dev_err(dp->dev, "Wait for pll lock failed %d\n", ret); ++ return; ++ } + } + + void analogix_dp_get_link_bandwidth(struct analogix_dp_device *dp, u32 *bwtype) + { + u32 reg; + +- reg = readl(dp->reg_base + ANALOGIX_DP_LINK_BW_SET); ++ reg = analogix_dp_read(dp, ANALOGIX_DP_LINK_BW_SET); + *bwtype = reg; + } + + void analogix_dp_set_lane_count(struct analogix_dp_device *dp, u32 count) + { + u32 reg; ++ int ret; + + reg = count; +- writel(reg, dp->reg_base + ANALOGIX_DP_LANE_COUNT_SET); ++ analogix_dp_write(dp, ANALOGIX_DP_LANE_COUNT_SET, reg); ++ ++ if (dp->phy) { ++ union phy_configure_opts phy_cfg = {0}; ++ ++ phy_cfg.dp.lanes = dp->link_train.lane_count; ++ phy_cfg.dp.set_lanes = true; ++ phy_cfg.dp.set_rate = false; ++ phy_cfg.dp.set_voltages = false; ++ ret = phy_configure(dp->phy, &phy_cfg); ++ if (ret && ret != -EOPNOTSUPP) { ++ dev_err(dp->dev, "%s: phy_configure() failed: %d\n", ++ __func__, ret); ++ return; ++ } ++ } + } + + void analogix_dp_get_lane_count(struct analogix_dp_device *dp, u32 *count) + { + u32 reg; + +- reg = readl(dp->reg_base + ANALOGIX_DP_LANE_COUNT_SET); ++ reg = analogix_dp_read(dp, ANALOGIX_DP_LANE_COUNT_SET); + *count = reg; + } + ++void analogix_dp_set_lane_link_training(struct analogix_dp_device *dp) ++{ ++ u8 lane; ++ int ret; ++ ++ for (lane = 0; lane < dp->link_train.lane_count; lane++) ++ analogix_dp_write(dp, ++ ANALOGIX_DP_LN0_LINK_TRAINING_CTL + 4 * lane, ++ dp->link_train.training_lane[lane]); ++ ++ if (dp->phy) { ++ union phy_configure_opts phy_cfg = {0}; ++ ++ for (lane = 0; lane < dp->link_train.lane_count; lane++) { ++ u8 training_lane = dp->link_train.training_lane[lane]; ++ u8 vs, pe; ++ ++ vs = (training_lane & DP_TRAIN_VOLTAGE_SWING_MASK) >> ++ DP_TRAIN_VOLTAGE_SWING_SHIFT; ++ pe = (training_lane & DP_TRAIN_PRE_EMPHASIS_MASK) >> ++ DP_TRAIN_PRE_EMPHASIS_SHIFT; ++ phy_cfg.dp.voltage[lane] = vs; ++ phy_cfg.dp.pre[lane] = pe; ++ } ++ ++ phy_cfg.dp.lanes = dp->link_train.lane_count; ++ phy_cfg.dp.set_lanes = false; ++ phy_cfg.dp.set_rate = false; ++ phy_cfg.dp.set_voltages = true; ++ ret = phy_configure(dp->phy, &phy_cfg); ++ if (ret && ret != -EOPNOTSUPP) { ++ dev_err(dp->dev, "%s: phy_configure() failed: %d\n", ++ __func__, ret); ++ return; ++ } ++ } ++} ++ ++u32 analogix_dp_get_lane_link_training(struct analogix_dp_device *dp, u8 lane) ++{ ++ return analogix_dp_read(dp, ++ ANALOGIX_DP_LN0_LINK_TRAINING_CTL + 4 * lane); ++} ++ + void analogix_dp_enable_enhanced_mode(struct analogix_dp_device *dp, + bool enable) + { + u32 reg; + + if (enable) { +- reg = readl(dp->reg_base + ANALOGIX_DP_SYS_CTL_4); ++ reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_4); + reg |= ENHANCED; +- writel(reg, dp->reg_base + ANALOGIX_DP_SYS_CTL_4); ++ analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_4, reg); + } else { +- reg = readl(dp->reg_base + ANALOGIX_DP_SYS_CTL_4); ++ reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_4); + reg &= ~ENHANCED; +- writel(reg, dp->reg_base + ANALOGIX_DP_SYS_CTL_4); ++ analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_4, reg); + } + } + +@@ -669,144 +675,44 @@ void analogix_dp_set_training_pattern(struct analogix_dp_device *dp, + switch (pattern) { + case PRBS7: + reg = SCRAMBLING_ENABLE | LINK_QUAL_PATTERN_SET_PRBS7; +- writel(reg, dp->reg_base + ANALOGIX_DP_TRAINING_PTN_SET); ++ analogix_dp_write(dp, ANALOGIX_DP_TRAINING_PTN_SET, reg); + break; + case D10_2: + reg = SCRAMBLING_ENABLE | LINK_QUAL_PATTERN_SET_D10_2; +- writel(reg, dp->reg_base + ANALOGIX_DP_TRAINING_PTN_SET); ++ analogix_dp_write(dp, ANALOGIX_DP_TRAINING_PTN_SET, reg); + break; + case TRAINING_PTN1: + reg = SCRAMBLING_DISABLE | SW_TRAINING_PATTERN_SET_PTN1; +- writel(reg, dp->reg_base + ANALOGIX_DP_TRAINING_PTN_SET); ++ analogix_dp_write(dp, ANALOGIX_DP_TRAINING_PTN_SET, reg); + break; + case TRAINING_PTN2: + reg = SCRAMBLING_DISABLE | SW_TRAINING_PATTERN_SET_PTN2; +- writel(reg, dp->reg_base + ANALOGIX_DP_TRAINING_PTN_SET); ++ analogix_dp_write(dp, ANALOGIX_DP_TRAINING_PTN_SET, reg); + break; + case DP_NONE: + reg = SCRAMBLING_ENABLE | + LINK_QUAL_PATTERN_SET_DISABLE | + SW_TRAINING_PATTERN_SET_NORMAL; +- writel(reg, dp->reg_base + ANALOGIX_DP_TRAINING_PTN_SET); ++ analogix_dp_write(dp, ANALOGIX_DP_TRAINING_PTN_SET, reg); + break; + default: + break; + } + } + +-void analogix_dp_set_lane0_pre_emphasis(struct analogix_dp_device *dp, +- u32 level) +-{ +- u32 reg; +- +- reg = readl(dp->reg_base + ANALOGIX_DP_LN0_LINK_TRAINING_CTL); +- reg &= ~PRE_EMPHASIS_SET_MASK; +- reg |= level << PRE_EMPHASIS_SET_SHIFT; +- writel(reg, dp->reg_base + ANALOGIX_DP_LN0_LINK_TRAINING_CTL); +-} +- +-void analogix_dp_set_lane1_pre_emphasis(struct analogix_dp_device *dp, +- u32 level) +-{ +- u32 reg; +- +- reg = readl(dp->reg_base + ANALOGIX_DP_LN1_LINK_TRAINING_CTL); +- reg &= ~PRE_EMPHASIS_SET_MASK; +- reg |= level << PRE_EMPHASIS_SET_SHIFT; +- writel(reg, dp->reg_base + ANALOGIX_DP_LN1_LINK_TRAINING_CTL); +-} +- +-void analogix_dp_set_lane2_pre_emphasis(struct analogix_dp_device *dp, +- u32 level) +-{ +- u32 reg; +- +- reg = readl(dp->reg_base + ANALOGIX_DP_LN2_LINK_TRAINING_CTL); +- reg &= ~PRE_EMPHASIS_SET_MASK; +- reg |= level << PRE_EMPHASIS_SET_SHIFT; +- writel(reg, dp->reg_base + ANALOGIX_DP_LN2_LINK_TRAINING_CTL); +-} +- +-void analogix_dp_set_lane3_pre_emphasis(struct analogix_dp_device *dp, +- u32 level) +-{ +- u32 reg; +- +- reg = readl(dp->reg_base + ANALOGIX_DP_LN3_LINK_TRAINING_CTL); +- reg &= ~PRE_EMPHASIS_SET_MASK; +- reg |= level << PRE_EMPHASIS_SET_SHIFT; +- writel(reg, dp->reg_base + ANALOGIX_DP_LN3_LINK_TRAINING_CTL); +-} +- +-void analogix_dp_set_lane0_link_training(struct analogix_dp_device *dp, +- u32 training_lane) +-{ +- u32 reg; +- +- reg = training_lane; +- writel(reg, dp->reg_base + ANALOGIX_DP_LN0_LINK_TRAINING_CTL); +-} +- +-void analogix_dp_set_lane1_link_training(struct analogix_dp_device *dp, +- u32 training_lane) +-{ +- u32 reg; +- +- reg = training_lane; +- writel(reg, dp->reg_base + ANALOGIX_DP_LN1_LINK_TRAINING_CTL); +-} +- +-void analogix_dp_set_lane2_link_training(struct analogix_dp_device *dp, +- u32 training_lane) +-{ +- u32 reg; +- +- reg = training_lane; +- writel(reg, dp->reg_base + ANALOGIX_DP_LN2_LINK_TRAINING_CTL); +-} +- +-void analogix_dp_set_lane3_link_training(struct analogix_dp_device *dp, +- u32 training_lane) +-{ +- u32 reg; +- +- reg = training_lane; +- writel(reg, dp->reg_base + ANALOGIX_DP_LN3_LINK_TRAINING_CTL); +-} +- +-u32 analogix_dp_get_lane0_link_training(struct analogix_dp_device *dp) +-{ +- return readl(dp->reg_base + ANALOGIX_DP_LN0_LINK_TRAINING_CTL); +-} +- +-u32 analogix_dp_get_lane1_link_training(struct analogix_dp_device *dp) +-{ +- return readl(dp->reg_base + ANALOGIX_DP_LN1_LINK_TRAINING_CTL); +-} +- +-u32 analogix_dp_get_lane2_link_training(struct analogix_dp_device *dp) +-{ +- return readl(dp->reg_base + ANALOGIX_DP_LN2_LINK_TRAINING_CTL); +-} +- +-u32 analogix_dp_get_lane3_link_training(struct analogix_dp_device *dp) +-{ +- return readl(dp->reg_base + ANALOGIX_DP_LN3_LINK_TRAINING_CTL); +-} +- + void analogix_dp_reset_macro(struct analogix_dp_device *dp) + { + u32 reg; + +- reg = readl(dp->reg_base + ANALOGIX_DP_PHY_TEST); ++ reg = analogix_dp_read(dp, ANALOGIX_DP_PHY_TEST); + reg |= MACRO_RST; +- writel(reg, dp->reg_base + ANALOGIX_DP_PHY_TEST); ++ analogix_dp_write(dp, ANALOGIX_DP_PHY_TEST, reg); + + /* 10 us is the minimum reset time. */ + usleep_range(10, 20); + + reg &= ~MACRO_RST; +- writel(reg, dp->reg_base + ANALOGIX_DP_PHY_TEST); ++ analogix_dp_write(dp, ANALOGIX_DP_PHY_TEST, reg); + } + + void analogix_dp_init_video(struct analogix_dp_device *dp) +@@ -814,19 +720,19 @@ void analogix_dp_init_video(struct analogix_dp_device *dp) + u32 reg; + + reg = VSYNC_DET | VID_FORMAT_CHG | VID_CLK_CHG; +- writel(reg, dp->reg_base + ANALOGIX_DP_COMMON_INT_STA_1); ++ analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_STA_1, reg); + + reg = 0x0; +- writel(reg, dp->reg_base + ANALOGIX_DP_SYS_CTL_1); ++ analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_1, reg); + + reg = CHA_CRI(4) | CHA_CTRL; +- writel(reg, dp->reg_base + ANALOGIX_DP_SYS_CTL_2); ++ analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_2, reg); + + reg = 0x0; +- writel(reg, dp->reg_base + ANALOGIX_DP_SYS_CTL_3); ++ analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_3, reg); + + reg = VID_HRES_TH(2) | VID_VRES_TH(0); +- writel(reg, dp->reg_base + ANALOGIX_DP_VIDEO_CTL_8); ++ analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_8, reg); + } + + void analogix_dp_set_video_color_format(struct analogix_dp_device *dp) +@@ -837,36 +743,36 @@ void analogix_dp_set_video_color_format(struct analogix_dp_device *dp) + reg = (dp->video_info.dynamic_range << IN_D_RANGE_SHIFT) | + (dp->video_info.color_depth << IN_BPC_SHIFT) | + (dp->video_info.color_space << IN_COLOR_F_SHIFT); +- writel(reg, dp->reg_base + ANALOGIX_DP_VIDEO_CTL_2); ++ analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_2, reg); + + /* Set Input Color YCbCr Coefficients to ITU601 or ITU709 */ +- reg = readl(dp->reg_base + ANALOGIX_DP_VIDEO_CTL_3); ++ reg = analogix_dp_read(dp, ANALOGIX_DP_VIDEO_CTL_3); + reg &= ~IN_YC_COEFFI_MASK; + if (dp->video_info.ycbcr_coeff) + reg |= IN_YC_COEFFI_ITU709; + else + reg |= IN_YC_COEFFI_ITU601; +- writel(reg, dp->reg_base + ANALOGIX_DP_VIDEO_CTL_3); ++ analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_3, reg); + } + + int analogix_dp_is_slave_video_stream_clock_on(struct analogix_dp_device *dp) + { + u32 reg; + +- reg = readl(dp->reg_base + ANALOGIX_DP_SYS_CTL_1); +- writel(reg, dp->reg_base + ANALOGIX_DP_SYS_CTL_1); ++ reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_1); ++ analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_1, reg); + +- reg = readl(dp->reg_base + ANALOGIX_DP_SYS_CTL_1); ++ reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_1); + + if (!(reg & DET_STA)) { + dev_dbg(dp->dev, "Input stream clock not detected.\n"); + return -EINVAL; + } + +- reg = readl(dp->reg_base + ANALOGIX_DP_SYS_CTL_2); +- writel(reg, dp->reg_base + ANALOGIX_DP_SYS_CTL_2); ++ reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_2); ++ analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_2, reg); + +- reg = readl(dp->reg_base + ANALOGIX_DP_SYS_CTL_2); ++ reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_2); + dev_dbg(dp->dev, "wait SYS_CTL_2.\n"); + + if (reg & CHA_STA) { +@@ -884,30 +790,30 @@ void analogix_dp_set_video_cr_mn(struct analogix_dp_device *dp, + u32 reg; + + if (type == REGISTER_M) { +- reg = readl(dp->reg_base + ANALOGIX_DP_SYS_CTL_4); ++ reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_4); + reg |= FIX_M_VID; +- writel(reg, dp->reg_base + ANALOGIX_DP_SYS_CTL_4); ++ analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_4, reg); + reg = m_value & 0xff; +- writel(reg, dp->reg_base + ANALOGIX_DP_M_VID_0); ++ analogix_dp_write(dp, ANALOGIX_DP_M_VID_0, reg); + reg = (m_value >> 8) & 0xff; +- writel(reg, dp->reg_base + ANALOGIX_DP_M_VID_1); ++ analogix_dp_write(dp, ANALOGIX_DP_M_VID_1, reg); + reg = (m_value >> 16) & 0xff; +- writel(reg, dp->reg_base + ANALOGIX_DP_M_VID_2); ++ analogix_dp_write(dp, ANALOGIX_DP_M_VID_2, reg); + + reg = n_value & 0xff; +- writel(reg, dp->reg_base + ANALOGIX_DP_N_VID_0); ++ analogix_dp_write(dp, ANALOGIX_DP_N_VID_0, reg); + reg = (n_value >> 8) & 0xff; +- writel(reg, dp->reg_base + ANALOGIX_DP_N_VID_1); ++ analogix_dp_write(dp, ANALOGIX_DP_N_VID_1, reg); + reg = (n_value >> 16) & 0xff; +- writel(reg, dp->reg_base + ANALOGIX_DP_N_VID_2); ++ analogix_dp_write(dp, ANALOGIX_DP_N_VID_2, reg); + } else { +- reg = readl(dp->reg_base + ANALOGIX_DP_SYS_CTL_4); ++ reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_4); + reg &= ~FIX_M_VID; +- writel(reg, dp->reg_base + ANALOGIX_DP_SYS_CTL_4); ++ analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_4, reg); + +- writel(0x00, dp->reg_base + ANALOGIX_DP_N_VID_0); +- writel(0x80, dp->reg_base + ANALOGIX_DP_N_VID_1); +- writel(0x00, dp->reg_base + ANALOGIX_DP_N_VID_2); ++ analogix_dp_write(dp, ANALOGIX_DP_N_VID_0, 0x00); ++ analogix_dp_write(dp, ANALOGIX_DP_N_VID_1, 0x80); ++ analogix_dp_write(dp, ANALOGIX_DP_N_VID_2, 0x00); + } + } + +@@ -916,13 +822,13 @@ void analogix_dp_set_video_timing_mode(struct analogix_dp_device *dp, u32 type) + u32 reg; + + if (type == VIDEO_TIMING_FROM_CAPTURE) { +- reg = readl(dp->reg_base + ANALOGIX_DP_VIDEO_CTL_10); ++ reg = analogix_dp_read(dp, ANALOGIX_DP_VIDEO_CTL_10); + reg &= ~FORMAT_SEL; +- writel(reg, dp->reg_base + ANALOGIX_DP_VIDEO_CTL_10); ++ analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_10, reg); + } else { +- reg = readl(dp->reg_base + ANALOGIX_DP_VIDEO_CTL_10); ++ reg = analogix_dp_read(dp, ANALOGIX_DP_VIDEO_CTL_10); + reg |= FORMAT_SEL; +- writel(reg, dp->reg_base + ANALOGIX_DP_VIDEO_CTL_10); ++ analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_10, reg); + } + } + +@@ -931,15 +837,15 @@ void analogix_dp_enable_video_master(struct analogix_dp_device *dp, bool enable) + u32 reg; + + if (enable) { +- reg = readl(dp->reg_base + ANALOGIX_DP_SOC_GENERAL_CTL); ++ reg = analogix_dp_read(dp, ANALOGIX_DP_SOC_GENERAL_CTL); + reg &= ~VIDEO_MODE_MASK; + reg |= VIDEO_MASTER_MODE_EN | VIDEO_MODE_MASTER_MODE; +- writel(reg, dp->reg_base + ANALOGIX_DP_SOC_GENERAL_CTL); ++ analogix_dp_write(dp, ANALOGIX_DP_SOC_GENERAL_CTL, reg); + } else { +- reg = readl(dp->reg_base + ANALOGIX_DP_SOC_GENERAL_CTL); ++ reg = analogix_dp_read(dp, ANALOGIX_DP_SOC_GENERAL_CTL); + reg &= ~VIDEO_MODE_MASK; + reg |= VIDEO_MODE_SLAVE_MODE; +- writel(reg, dp->reg_base + ANALOGIX_DP_SOC_GENERAL_CTL); ++ analogix_dp_write(dp, ANALOGIX_DP_SOC_GENERAL_CTL, reg); + } + } + +@@ -947,19 +853,19 @@ void analogix_dp_start_video(struct analogix_dp_device *dp) + { + u32 reg; + +- reg = readl(dp->reg_base + ANALOGIX_DP_VIDEO_CTL_1); ++ reg = analogix_dp_read(dp, ANALOGIX_DP_VIDEO_CTL_1); + reg |= VIDEO_EN; +- writel(reg, dp->reg_base + ANALOGIX_DP_VIDEO_CTL_1); ++ analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_1, reg); + } + + int analogix_dp_is_video_stream_on(struct analogix_dp_device *dp) + { + u32 reg; + +- reg = readl(dp->reg_base + ANALOGIX_DP_SYS_CTL_3); +- writel(reg, dp->reg_base + ANALOGIX_DP_SYS_CTL_3); ++ reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_3); ++ analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_3, reg); + +- reg = readl(dp->reg_base + ANALOGIX_DP_SYS_CTL_3); ++ reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_3); + if (!(reg & STRM_VALID)) { + dev_dbg(dp->dev, "Input video stream is not detected.\n"); + return -EINVAL; +@@ -972,55 +878,55 @@ void analogix_dp_config_video_slave_mode(struct analogix_dp_device *dp) + { + u32 reg; + +- reg = readl(dp->reg_base + ANALOGIX_DP_FUNC_EN_1); ++ reg = analogix_dp_read(dp, ANALOGIX_DP_FUNC_EN_1); + if (dp->plat_data && is_rockchip(dp->plat_data->dev_type)) { + reg &= ~(RK_VID_CAP_FUNC_EN_N | RK_VID_FIFO_FUNC_EN_N); + } else { + reg &= ~(MASTER_VID_FUNC_EN_N | SLAVE_VID_FUNC_EN_N); + reg |= MASTER_VID_FUNC_EN_N; + } +- writel(reg, dp->reg_base + ANALOGIX_DP_FUNC_EN_1); ++ analogix_dp_write(dp, ANALOGIX_DP_FUNC_EN_1, reg); + +- reg = readl(dp->reg_base + ANALOGIX_DP_VIDEO_CTL_10); ++ reg = analogix_dp_read(dp, ANALOGIX_DP_VIDEO_CTL_10); + reg &= ~INTERACE_SCAN_CFG; + reg |= (dp->video_info.interlaced << 2); +- writel(reg, dp->reg_base + ANALOGIX_DP_VIDEO_CTL_10); ++ analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_10, reg); + +- reg = readl(dp->reg_base + ANALOGIX_DP_VIDEO_CTL_10); ++ reg = analogix_dp_read(dp, ANALOGIX_DP_VIDEO_CTL_10); + reg &= ~VSYNC_POLARITY_CFG; + reg |= (dp->video_info.v_sync_polarity << 1); +- writel(reg, dp->reg_base + ANALOGIX_DP_VIDEO_CTL_10); ++ analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_10, reg); + +- reg = readl(dp->reg_base + ANALOGIX_DP_VIDEO_CTL_10); ++ reg = analogix_dp_read(dp, ANALOGIX_DP_VIDEO_CTL_10); + reg &= ~HSYNC_POLARITY_CFG; + reg |= (dp->video_info.h_sync_polarity << 0); +- writel(reg, dp->reg_base + ANALOGIX_DP_VIDEO_CTL_10); ++ analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_10, reg); + + reg = AUDIO_MODE_SPDIF_MODE | VIDEO_MODE_SLAVE_MODE; +- writel(reg, dp->reg_base + ANALOGIX_DP_SOC_GENERAL_CTL); ++ analogix_dp_write(dp, ANALOGIX_DP_SOC_GENERAL_CTL, reg); + } + + void analogix_dp_enable_scrambling(struct analogix_dp_device *dp) + { + u32 reg; + +- reg = readl(dp->reg_base + ANALOGIX_DP_TRAINING_PTN_SET); ++ reg = analogix_dp_read(dp, ANALOGIX_DP_TRAINING_PTN_SET); + reg &= ~SCRAMBLING_DISABLE; +- writel(reg, dp->reg_base + ANALOGIX_DP_TRAINING_PTN_SET); ++ analogix_dp_write(dp, ANALOGIX_DP_TRAINING_PTN_SET, reg); + } + + void analogix_dp_disable_scrambling(struct analogix_dp_device *dp) + { + u32 reg; + +- reg = readl(dp->reg_base + ANALOGIX_DP_TRAINING_PTN_SET); ++ reg = analogix_dp_read(dp, ANALOGIX_DP_TRAINING_PTN_SET); + reg |= SCRAMBLING_DISABLE; +- writel(reg, dp->reg_base + ANALOGIX_DP_TRAINING_PTN_SET); ++ analogix_dp_write(dp, ANALOGIX_DP_TRAINING_PTN_SET, reg); + } + + void analogix_dp_enable_psr_crc(struct analogix_dp_device *dp) + { +- writel(PSR_VID_CRC_ENABLE, dp->reg_base + ANALOGIX_DP_CRC_CON); ++ analogix_dp_write(dp, ANALOGIX_DP_CRC_CON, PSR_VID_CRC_ENABLE); + } + + static ssize_t analogix_dp_get_psr_status(struct analogix_dp_device *dp) +@@ -1044,44 +950,44 @@ int analogix_dp_send_psr_spd(struct analogix_dp_device *dp, + ssize_t psr_status; + + /* don't send info frame */ +- val = readl(dp->reg_base + ANALOGIX_DP_PKT_SEND_CTL); ++ val = analogix_dp_read(dp, ANALOGIX_DP_PKT_SEND_CTL); + val &= ~IF_EN; +- writel(val, dp->reg_base + ANALOGIX_DP_PKT_SEND_CTL); ++ analogix_dp_write(dp, ANALOGIX_DP_PKT_SEND_CTL, val); + + /* configure single frame update mode */ +- writel(PSR_FRAME_UP_TYPE_BURST | PSR_CRC_SEL_HARDWARE, +- dp->reg_base + ANALOGIX_DP_PSR_FRAME_UPDATE_CTRL); ++ analogix_dp_write(dp, ANALOGIX_DP_PSR_FRAME_UPDATE_CTRL, ++ PSR_FRAME_UP_TYPE_BURST | PSR_CRC_SEL_HARDWARE); + + /* configure VSC HB0~HB3 */ +- writel(vsc->sdp_header.HB0, dp->reg_base + ANALOGIX_DP_SPD_HB0); +- writel(vsc->sdp_header.HB1, dp->reg_base + ANALOGIX_DP_SPD_HB1); +- writel(vsc->sdp_header.HB2, dp->reg_base + ANALOGIX_DP_SPD_HB2); +- writel(vsc->sdp_header.HB3, dp->reg_base + ANALOGIX_DP_SPD_HB3); ++ analogix_dp_write(dp, ANALOGIX_DP_SPD_HB0, vsc->sdp_header.HB0); ++ analogix_dp_write(dp, ANALOGIX_DP_SPD_HB1, vsc->sdp_header.HB1); ++ analogix_dp_write(dp, ANALOGIX_DP_SPD_HB2, vsc->sdp_header.HB2); ++ analogix_dp_write(dp, ANALOGIX_DP_SPD_HB3, vsc->sdp_header.HB3); + + /* configure reused VSC PB0~PB3, magic number from vendor */ +- writel(0x00, dp->reg_base + ANALOGIX_DP_SPD_PB0); +- writel(0x16, dp->reg_base + ANALOGIX_DP_SPD_PB1); +- writel(0xCE, dp->reg_base + ANALOGIX_DP_SPD_PB2); +- writel(0x5D, dp->reg_base + ANALOGIX_DP_SPD_PB3); ++ analogix_dp_write(dp, ANALOGIX_DP_SPD_PB0, 0x00); ++ analogix_dp_write(dp, ANALOGIX_DP_SPD_PB1, 0x16); ++ analogix_dp_write(dp, ANALOGIX_DP_SPD_PB2, 0xCE); ++ analogix_dp_write(dp, ANALOGIX_DP_SPD_PB3, 0x5D); + + /* configure DB0 / DB1 values */ +- writel(vsc->db[0], dp->reg_base + ANALOGIX_DP_VSC_SHADOW_DB0); +- writel(vsc->db[1], dp->reg_base + ANALOGIX_DP_VSC_SHADOW_DB1); ++ analogix_dp_write(dp, ANALOGIX_DP_VSC_SHADOW_DB0, vsc->db[0]); ++ analogix_dp_write(dp, ANALOGIX_DP_VSC_SHADOW_DB1, vsc->db[1]); + + /* set reuse spd inforframe */ +- val = readl(dp->reg_base + ANALOGIX_DP_VIDEO_CTL_3); ++ val = analogix_dp_read(dp, ANALOGIX_DP_VIDEO_CTL_3); + val |= REUSE_SPD_EN; +- writel(val, dp->reg_base + ANALOGIX_DP_VIDEO_CTL_3); ++ analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_3, val); + + /* mark info frame update */ +- val = readl(dp->reg_base + ANALOGIX_DP_PKT_SEND_CTL); ++ val = analogix_dp_read(dp, ANALOGIX_DP_PKT_SEND_CTL); + val = (val | IF_UP) & ~IF_EN; +- writel(val, dp->reg_base + ANALOGIX_DP_PKT_SEND_CTL); ++ analogix_dp_write(dp, ANALOGIX_DP_PKT_SEND_CTL, val); + + /* send info frame */ +- val = readl(dp->reg_base + ANALOGIX_DP_PKT_SEND_CTL); ++ val = analogix_dp_read(dp, ANALOGIX_DP_PKT_SEND_CTL); + val |= IF_EN; +- writel(val, dp->reg_base + ANALOGIX_DP_PKT_SEND_CTL); ++ analogix_dp_write(dp, ANALOGIX_DP_PKT_SEND_CTL, val); + + if (!blocking) + return 0; +@@ -1098,6 +1004,26 @@ int analogix_dp_send_psr_spd(struct analogix_dp_device *dp, + return 0; + } + ++void analogix_dp_phy_power_on(struct analogix_dp_device *dp) ++{ ++ if (dp->phy_enabled) ++ return; ++ ++ phy_power_on(dp->phy); ++ ++ dp->phy_enabled = true; ++} ++ ++void analogix_dp_phy_power_off(struct analogix_dp_device *dp) ++{ ++ if (!dp->phy_enabled) ++ return; ++ ++ phy_power_off(dp->phy); ++ ++ dp->phy_enabled = false; ++} ++ + ssize_t analogix_dp_transfer(struct analogix_dp_device *dp, + struct drm_dp_aux_msg *msg) + { +@@ -1112,9 +1038,15 @@ ssize_t analogix_dp_transfer(struct analogix_dp_device *dp, + if (WARN_ON(msg->size > 16)) + return -E2BIG; + ++ reg = analogix_dp_read(dp, ANALOGIX_DP_FUNC_EN_2); ++ if (reg & AUX_FUNC_EN_N) { ++ analogix_dp_phy_power_on(dp); ++ analogix_dp_init_aux(dp); ++ } ++ + /* Clear AUX CH data buffer */ + reg = BUF_CLR; +- writel(reg, dp->reg_base + ANALOGIX_DP_BUFFER_DATA_CTL); ++ analogix_dp_write(dp, ANALOGIX_DP_BUFFER_DATA_CTL, reg); + + switch (msg->request & ~DP_AUX_I2C_MOT) { + case DP_AUX_I2C_WRITE: +@@ -1142,21 +1074,21 @@ ssize_t analogix_dp_transfer(struct analogix_dp_device *dp, + } + + reg |= AUX_LENGTH(msg->size); +- writel(reg, dp->reg_base + ANALOGIX_DP_AUX_CH_CTL_1); ++ analogix_dp_write(dp, ANALOGIX_DP_AUX_CH_CTL_1, reg); + + /* Select DPCD device address */ + reg = AUX_ADDR_7_0(msg->address); +- writel(reg, dp->reg_base + ANALOGIX_DP_AUX_ADDR_7_0); ++ analogix_dp_write(dp, ANALOGIX_DP_AUX_ADDR_7_0, reg); + reg = AUX_ADDR_15_8(msg->address); +- writel(reg, dp->reg_base + ANALOGIX_DP_AUX_ADDR_15_8); ++ analogix_dp_write(dp, ANALOGIX_DP_AUX_ADDR_15_8, reg); + reg = AUX_ADDR_19_16(msg->address); +- writel(reg, dp->reg_base + ANALOGIX_DP_AUX_ADDR_19_16); ++ analogix_dp_write(dp, ANALOGIX_DP_AUX_ADDR_19_16, reg); + + if (!(msg->request & DP_AUX_I2C_READ)) { + for (i = 0; i < msg->size; i++) { + reg = buffer[i]; +- writel(reg, dp->reg_base + ANALOGIX_DP_BUF_DATA_0 + +- 4 * i); ++ analogix_dp_write(dp, ANALOGIX_DP_BUF_DATA_0 + 4 * i, ++ reg); + num_transferred++; + } + } +@@ -1168,7 +1100,7 @@ ssize_t analogix_dp_transfer(struct analogix_dp_device *dp, + if (msg->size < 1) + reg |= ADDR_ONLY; + +- writel(reg, dp->reg_base + ANALOGIX_DP_AUX_CH_CTL_2); ++ analogix_dp_write(dp, ANALOGIX_DP_AUX_CH_CTL_2, reg); + + ret = readx_poll_timeout(readl, dp->reg_base + ANALOGIX_DP_AUX_CH_CTL_2, + reg, !(reg & AUX_EN), 25, 500 * 1000); +@@ -1187,13 +1119,13 @@ ssize_t analogix_dp_transfer(struct analogix_dp_device *dp, + } + + /* Clear interrupt source for AUX CH command reply */ +- writel(RPLY_RECEIV, dp->reg_base + ANALOGIX_DP_INT_STA); ++ analogix_dp_write(dp, ANALOGIX_DP_INT_STA, RPLY_RECEIV); + + /* Clear interrupt source for AUX CH access error */ +- reg = readl(dp->reg_base + ANALOGIX_DP_INT_STA); +- status_reg = readl(dp->reg_base + ANALOGIX_DP_AUX_CH_STA); ++ reg = analogix_dp_read(dp, ANALOGIX_DP_INT_STA); ++ status_reg = analogix_dp_read(dp, ANALOGIX_DP_AUX_CH_STA); + if ((reg & AUX_ERR) || (status_reg & AUX_STATUS_MASK)) { +- writel(AUX_ERR, dp->reg_base + ANALOGIX_DP_INT_STA); ++ analogix_dp_write(dp, ANALOGIX_DP_INT_STA, AUX_ERR); + + dev_warn(dp->dev, "AUX CH error happened: %#x (%d)\n", + status_reg & AUX_STATUS_MASK, !!(reg & AUX_ERR)); +@@ -1202,15 +1134,15 @@ ssize_t analogix_dp_transfer(struct analogix_dp_device *dp, + + if (msg->request & DP_AUX_I2C_READ) { + for (i = 0; i < msg->size; i++) { +- reg = readl(dp->reg_base + ANALOGIX_DP_BUF_DATA_0 + +- 4 * i); ++ reg = analogix_dp_read(dp, ANALOGIX_DP_BUF_DATA_0 + ++ 4 * i); + buffer[i] = (unsigned char)reg; + num_transferred++; + } + } + + /* Check if Rx sends defer */ +- reg = readl(dp->reg_base + ANALOGIX_DP_AUX_RX_COMM); ++ reg = analogix_dp_read(dp, ANALOGIX_DP_AUX_RX_COMM); + if (reg == AUX_RX_COMM_AUX_DEFER) + msg->reply = DP_AUX_NATIVE_REPLY_DEFER; + else if (reg == AUX_RX_COMM_I2C_DEFER) +@@ -1222,7 +1154,7 @@ ssize_t analogix_dp_transfer(struct analogix_dp_device *dp, + (msg->request & ~DP_AUX_I2C_MOT) == DP_AUX_NATIVE_READ) + msg->reply = DP_AUX_NATIVE_REPLY_ACK; + +- return num_transferred > 0 ? num_transferred : -EBUSY; ++ return (num_transferred == msg->size) ? num_transferred : -EBUSY; + + aux_error: + /* if aux err happen, reset aux */ +@@ -1230,3 +1162,119 @@ ssize_t analogix_dp_transfer(struct analogix_dp_device *dp, + + return -EREMOTEIO; + } ++ ++void analogix_dp_set_video_format(struct analogix_dp_device *dp) ++{ ++ struct video_info *video = &dp->video_info; ++ const struct drm_display_mode *mode = &video->mode; ++ unsigned int hsw, hfp, hbp, vsw, vfp, vbp; ++ ++ hsw = mode->hsync_end - mode->hsync_start; ++ hfp = mode->hsync_start - mode->hdisplay; ++ hbp = mode->htotal - mode->hsync_end; ++ vsw = mode->vsync_end - mode->vsync_start; ++ vfp = mode->vsync_start - mode->vdisplay; ++ vbp = mode->vtotal - mode->vsync_end; ++ ++ /* Set Video Format Parameters */ ++ analogix_dp_write(dp, ANALOGIX_DP_TOTAL_LINE_CFG_L, ++ TOTAL_LINE_CFG_L(mode->vtotal)); ++ analogix_dp_write(dp, ANALOGIX_DP_TOTAL_LINE_CFG_H, ++ TOTAL_LINE_CFG_H(mode->vtotal >> 8)); ++ analogix_dp_write(dp, ANALOGIX_DP_ACTIVE_LINE_CFG_L, ++ ACTIVE_LINE_CFG_L(mode->vdisplay)); ++ analogix_dp_write(dp, ANALOGIX_DP_ACTIVE_LINE_CFG_H, ++ ACTIVE_LINE_CFG_H(mode->vdisplay >> 8)); ++ analogix_dp_write(dp, ANALOGIX_DP_V_F_PORCH_CFG, ++ V_F_PORCH_CFG(vfp)); ++ analogix_dp_write(dp, ANALOGIX_DP_V_SYNC_WIDTH_CFG, ++ V_SYNC_WIDTH_CFG(vsw)); ++ analogix_dp_write(dp, ANALOGIX_DP_V_B_PORCH_CFG, ++ V_B_PORCH_CFG(vbp)); ++ analogix_dp_write(dp, ANALOGIX_DP_TOTAL_PIXEL_CFG_L, ++ TOTAL_PIXEL_CFG_L(mode->htotal)); ++ analogix_dp_write(dp, ANALOGIX_DP_TOTAL_PIXEL_CFG_H, ++ TOTAL_PIXEL_CFG_H(mode->htotal >> 8)); ++ analogix_dp_write(dp, ANALOGIX_DP_ACTIVE_PIXEL_CFG_L, ++ ACTIVE_PIXEL_CFG_L(mode->hdisplay)); ++ analogix_dp_write(dp, ANALOGIX_DP_ACTIVE_PIXEL_CFG_H, ++ ACTIVE_PIXEL_CFG_H(mode->hdisplay >> 8)); ++ analogix_dp_write(dp, ANALOGIX_DP_H_F_PORCH_CFG_L, ++ H_F_PORCH_CFG_L(hfp)); ++ analogix_dp_write(dp, ANALOGIX_DP_H_F_PORCH_CFG_H, ++ H_F_PORCH_CFG_H(hfp >> 8)); ++ analogix_dp_write(dp, ANALOGIX_DP_H_SYNC_CFG_L, ++ H_SYNC_CFG_L(hsw)); ++ analogix_dp_write(dp, ANALOGIX_DP_H_SYNC_CFG_H, ++ H_SYNC_CFG_H(hsw >> 8)); ++ analogix_dp_write(dp, ANALOGIX_DP_H_B_PORCH_CFG_L, ++ H_B_PORCH_CFG_L(hbp)); ++ analogix_dp_write(dp, ANALOGIX_DP_H_B_PORCH_CFG_H, ++ H_B_PORCH_CFG_H(hbp >> 8)); ++} ++ ++void analogix_dp_video_bist_enable(struct analogix_dp_device *dp) ++{ ++ u32 reg; ++ ++ /* Enable Video BIST */ ++ analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_4, BIST_EN); ++ ++ /* ++ * Note that if BIST_EN is set to 1, F_SEL must be cleared to 0 ++ * although video format information comes from registers set by user. ++ */ ++ reg = analogix_dp_read(dp, ANALOGIX_DP_VIDEO_CTL_10); ++ reg &= ~FORMAT_SEL; ++ analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_10, reg); ++} ++ ++void analogix_dp_audio_config_i2s(struct analogix_dp_device *dp) ++{ ++ u32 reg; ++ ++ reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_4); ++ reg &= ~FIX_M_AUD; ++ analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_4, reg); ++ ++ reg = analogix_dp_read(dp, ANALOGIX_DP_I2S_CTRL); ++ reg |= I2S_EN; ++ analogix_dp_write(dp, ANALOGIX_DP_I2S_CTRL, reg); ++} ++ ++void analogix_dp_audio_config_spdif(struct analogix_dp_device *dp) ++{ ++ u32 reg; ++ ++ reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_4); ++ reg &= ~FIX_M_AUD; ++ analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_4, reg); ++ ++ reg = analogix_dp_read(dp, ANALOGIX_DP_SPDIF_AUDIO_CTL_0); ++ reg |= AUD_SPDIF_EN; ++ analogix_dp_write(dp, ANALOGIX_DP_SPDIF_AUDIO_CTL_0, reg); ++} ++ ++void analogix_dp_audio_enable(struct analogix_dp_device *dp) ++{ ++ u32 reg; ++ ++ reg = analogix_dp_read(dp, ANALOGIX_DP_FUNC_EN_1); ++ reg &= ~(AUD_FIFO_FUNC_EN_N | AUD_FUNC_EN_N); ++ analogix_dp_write(dp, ANALOGIX_DP_FUNC_EN_1, reg); ++ ++ reg = analogix_dp_read(dp, ANALOGIX_DP_AUD_CTL); ++ reg |= MISC_CTRL_RESET | DP_AUDIO_EN; ++ analogix_dp_write(dp, ANALOGIX_DP_AUD_CTL, reg); ++} ++ ++void analogix_dp_audio_disable(struct analogix_dp_device *dp) ++{ ++ u32 reg; ++ ++ analogix_dp_write(dp, ANALOGIX_DP_AUD_CTL, 0); ++ ++ reg = analogix_dp_read(dp, ANALOGIX_DP_FUNC_EN_1); ++ reg |= AUD_FIFO_FUNC_EN_N | AUD_FUNC_EN_N; ++ analogix_dp_write(dp, ANALOGIX_DP_FUNC_EN_1, reg); ++} +diff --git a/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.h b/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.h +index e284ee8da58b..346024aedc8b 100644 +--- a/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.h ++++ b/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.h +@@ -15,9 +15,27 @@ + #define ANALOGIX_DP_VIDEO_CTL_1 0x20 + #define ANALOGIX_DP_VIDEO_CTL_2 0x24 + #define ANALOGIX_DP_VIDEO_CTL_3 0x28 ++#define ANALOGIX_DP_VIDEO_CTL_4 0x2C + + #define ANALOGIX_DP_VIDEO_CTL_8 0x3C + #define ANALOGIX_DP_VIDEO_CTL_10 0x44 ++#define ANALOGIX_DP_TOTAL_LINE_CFG_L 0x48 ++#define ANALOGIX_DP_TOTAL_LINE_CFG_H 0x4C ++#define ANALOGIX_DP_ACTIVE_LINE_CFG_L 0x50 ++#define ANALOGIX_DP_ACTIVE_LINE_CFG_H 0x54 ++#define ANALOGIX_DP_V_F_PORCH_CFG 0x58 ++#define ANALOGIX_DP_V_SYNC_WIDTH_CFG 0x5C ++#define ANALOGIX_DP_V_B_PORCH_CFG 0x60 ++#define ANALOGIX_DP_TOTAL_PIXEL_CFG_L 0x64 ++#define ANALOGIX_DP_TOTAL_PIXEL_CFG_H 0x68 ++#define ANALOGIX_DP_ACTIVE_PIXEL_CFG_L 0x6C ++#define ANALOGIX_DP_ACTIVE_PIXEL_CFG_H 0x70 ++#define ANALOGIX_DP_H_F_PORCH_CFG_L 0x74 ++#define ANALOGIX_DP_H_F_PORCH_CFG_H 0x78 ++#define ANALOGIX_DP_H_SYNC_CFG_L 0x7C ++#define ANALOGIX_DP_H_SYNC_CFG_H 0x80 ++#define ANALOGIX_DP_H_B_PORCH_CFG_L 0x84 ++#define ANALOGIX_DP_H_B_PORCH_CFG_H 0x88 + + #define ANALOGIX_DP_SPDIF_AUDIO_CTL_0 0xD8 + +@@ -70,7 +88,7 @@ + #define ANALOGIX_DP_SYS_CTL_2 0x604 + #define ANALOGIX_DP_SYS_CTL_3 0x608 + #define ANALOGIX_DP_SYS_CTL_4 0x60C +- ++#define ANALOGIX_DP_AUD_CTL 0x618 + #define ANALOGIX_DP_PKT_SEND_CTL 0x640 + #define ANALOGIX_DP_HDCP_CTL 0x648 + +@@ -116,8 +134,9 @@ + #define ANALOGIX_DP_BUF_DATA_0 0x7C0 + + #define ANALOGIX_DP_SOC_GENERAL_CTL 0x800 +- ++#define ANALOGIX_DP_AUD_CHANNEL_CTL 0x834 + #define ANALOGIX_DP_CRC_CON 0x890 ++#define ANALOGIX_DP_I2S_CTRL 0x9C8 + + /* ANALOGIX_DP_TX_SW_RESET */ + #define RESET_DP_TX (0x1 << 0) +@@ -171,6 +190,11 @@ + #define VID_CHK_UPDATE_TYPE_0 (0x0 << 4) + #define REUSE_SPD_EN (0x1 << 3) + ++/* ANALOGIX_DP_VIDEO_CTL_4 */ ++#define BIST_EN (0x1 << 3) ++#define BIST_WIDTH(x) (((x) & 0x1) << 2) ++#define BIST_TYPE(x) (((x) & 0x3) << 0) ++ + /* ANALOGIX_DP_VIDEO_CTL_8 */ + #define VID_HRES_TH(x) (((x) & 0xf) << 4) + #define VID_VRES_TH(x) (((x) & 0xf) << 0) +@@ -181,6 +205,60 @@ + #define VSYNC_POLARITY_CFG (0x1 << 1) + #define HSYNC_POLARITY_CFG (0x1 << 0) + ++/* ANALOGIX_DP_TOTAL_LINE_CFG_L */ ++#define TOTAL_LINE_CFG_L(x) (((x) & 0xff) << 0) ++ ++/* ANALOGIX_DP_TOTAL_LINE_CFG_H */ ++#define TOTAL_LINE_CFG_H(x) (((x) & 0xf) << 0) ++ ++/* ANALOGIX_DP_ACTIVE_LINE_CFG_L */ ++#define ACTIVE_LINE_CFG_L(x) (((x) & 0xff) << 0) ++ ++/* ANALOGIX_DP_ACTIVE_LINE_CFG_H */ ++#define ACTIVE_LINE_CFG_H(x) (((x) & 0xf) << 0) ++ ++/* ANALOGIX_DP_V_F_PORCH_CFG */ ++#define V_F_PORCH_CFG(x) (((x) & 0xff) << 0) ++ ++/* ANALOGIX_DP_V_SYNC_WIDTH_CFG */ ++#define V_SYNC_WIDTH_CFG(x) (((x) & 0xff) << 0) ++ ++/* ANALOGIX_DP_V_B_PORCH_CFG */ ++#define V_B_PORCH_CFG(x) (((x) & 0xff) << 0) ++ ++/* ANALOGIX_DP_TOTAL_PIXEL_CFG_L */ ++#define TOTAL_PIXEL_CFG_L(x) (((x) & 0xff) << 0) ++ ++/* ANALOGIX_DP_TOTAL_PIXEL_CFG_H */ ++#define TOTAL_PIXEL_CFG_H(x) (((x) & 0x3f) << 0) ++ ++/* ANALOGIX_DP_ACTIVE_PIXEL_CFG_L */ ++#define ACTIVE_PIXEL_CFG_L(x) (((x) & 0xff) << 0) ++ ++/* ANALOGIX_DP_ACTIVE_PIXEL_CFG_H */ ++#define ACTIVE_PIXEL_CFG_H(x) (((x) & 0x3f) << 0) ++ ++/* ANALOGIX_DP_H_F_PORCH_CFG_L */ ++#define H_F_PORCH_CFG_L(x) (((x) & 0xff) << 0) ++ ++/* ANALOGIX_DP_H_F_PORCH_CFG_H */ ++#define H_F_PORCH_CFG_H(x) (((x) & 0xf) << 0) ++ ++/* ANALOGIX_DP_H_SYNC_CFG_L */ ++#define H_SYNC_CFG_L(x) (((x) & 0xff) << 0) ++ ++/* ANALOGIX_DP_H_SYNC_CFG_H */ ++#define H_SYNC_CFG_H(x) (((x) & 0xf) << 0) ++ ++/* ANALOGIX_DP_H_B_PORCH_CFG_L */ ++#define H_B_PORCH_CFG_L(x) (((x) & 0xff) << 0) ++ ++/* ANALOGIX_DP_H_B_PORCH_CFG_H */ ++#define H_B_PORCH_CFG_H(x) (((x) & 0xf) << 0) ++ ++/* ANALOGIX_DP_SPDIF_AUDIO_CTL_0 */ ++#define AUD_SPDIF_EN (0x1 << 7) ++ + /* ANALOGIX_DP_PLL_REG_1 */ + #define REF_CLK_24M (0x1 << 0) + #define REF_CLK_27M (0x0 << 0) +@@ -309,6 +387,10 @@ + #define FIX_M_VID (0x1 << 2) + #define M_VID_UPDATE_CTRL (0x3 << 0) + ++/* ANALOGIX_DP_AUD_CTL */ ++#define MISC_CTRL_RESET (0x1 << 4) ++#define DP_AUDIO_EN (0x1 << 0) ++ + /* ANALOGIX_DP_TRAINING_PTN_SET */ + #define SCRAMBLER_TYPE (0x1 << 9) + #define HW_LINK_TRAINING_PATTERN (0x1 << 8) +@@ -406,6 +488,11 @@ + #define VIDEO_MODE_SLAVE_MODE (0x1 << 0) + #define VIDEO_MODE_MASTER_MODE (0x0 << 0) + ++/* ANALOGIX_DP_AUD_CHANNEL_CTL */ ++#define AUD_CHANNEL_COUNT_6 (0x5 << 0) ++#define AUD_CHANNEL_COUNT_4 (0x3 << 0) ++#define AUD_CHANNEL_COUNT_2 (0x1 << 0) ++ + /* ANALOGIX_DP_PKT_SEND_CTL */ + #define IF_UP (0x1 << 4) + #define IF_EN (0x1 << 0) +@@ -414,4 +501,7 @@ + #define PSR_VID_CRC_FLUSH (0x1 << 2) + #define PSR_VID_CRC_ENABLE (0x1 << 0) + ++/* ANALOGIX_DP_I2S_CTRL */ ++#define I2S_EN (0x1 << 4) ++ + #endif /* _ANALOGIX_DP_REG_H */ +diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c b/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c +index 0c79a9ba48bb..fc39edc48877 100644 +--- a/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c ++++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c +@@ -9,6 +9,8 @@ + #include + #include + #include ++#include ++#include + #include + #include + #include +@@ -18,6 +20,7 @@ + #include + #include + #include ++#include + + #include + +@@ -48,6 +51,11 @@ + + #define HDMI14_MAX_TMDSCLK 340000000 + ++static const unsigned int dw_hdmi_cable[] = { ++ EXTCON_DISP_HDMI, ++ EXTCON_NONE, ++}; ++ + enum hdmi_datamap { + RGB444_8B = 0x01, + RGB444_10B = 0x03, +@@ -62,6 +70,61 @@ enum hdmi_datamap { + YCbCr422_12B = 0x12, + }; + ++/* ++ * Unless otherwise noted, entries in this table are 100% optimization. ++ * Values can be obtained from hdmi_compute_n() but that function is ++ * slow so we pre-compute values we expect to see. ++ * ++ * All 32k and 48k values are expected to be the same (due to the way ++ * the math works) for any rate that's an exact kHz. ++ */ ++static const struct dw_hdmi_audio_tmds_n common_tmds_n_table[] = { ++ { .tmds = 25175000, .n_32k = 4096, .n_44k1 = 12854, .n_48k = 6144, }, ++ { .tmds = 25200000, .n_32k = 4096, .n_44k1 = 5656, .n_48k = 6144, }, ++ { .tmds = 27000000, .n_32k = 4096, .n_44k1 = 5488, .n_48k = 6144, }, ++ { .tmds = 28320000, .n_32k = 4096, .n_44k1 = 5586, .n_48k = 6144, }, ++ { .tmds = 30240000, .n_32k = 4096, .n_44k1 = 5642, .n_48k = 6144, }, ++ { .tmds = 31500000, .n_32k = 4096, .n_44k1 = 5600, .n_48k = 6144, }, ++ { .tmds = 32000000, .n_32k = 4096, .n_44k1 = 5733, .n_48k = 6144, }, ++ { .tmds = 33750000, .n_32k = 4096, .n_44k1 = 6272, .n_48k = 6144, }, ++ { .tmds = 36000000, .n_32k = 4096, .n_44k1 = 5684, .n_48k = 6144, }, ++ { .tmds = 40000000, .n_32k = 4096, .n_44k1 = 5733, .n_48k = 6144, }, ++ { .tmds = 49500000, .n_32k = 4096, .n_44k1 = 5488, .n_48k = 6144, }, ++ { .tmds = 50000000, .n_32k = 4096, .n_44k1 = 5292, .n_48k = 6144, }, ++ { .tmds = 54000000, .n_32k = 4096, .n_44k1 = 5684, .n_48k = 6144, }, ++ { .tmds = 65000000, .n_32k = 4096, .n_44k1 = 7056, .n_48k = 6144, }, ++ { .tmds = 68250000, .n_32k = 4096, .n_44k1 = 5376, .n_48k = 6144, }, ++ { .tmds = 71000000, .n_32k = 4096, .n_44k1 = 7056, .n_48k = 6144, }, ++ { .tmds = 72000000, .n_32k = 4096, .n_44k1 = 5635, .n_48k = 6144, }, ++ { .tmds = 73250000, .n_32k = 4096, .n_44k1 = 14112, .n_48k = 6144, }, ++ { .tmds = 74250000, .n_32k = 4096, .n_44k1 = 6272, .n_48k = 6144, }, ++ { .tmds = 75000000, .n_32k = 4096, .n_44k1 = 5880, .n_48k = 6144, }, ++ { .tmds = 78750000, .n_32k = 4096, .n_44k1 = 5600, .n_48k = 6144, }, ++ { .tmds = 78800000, .n_32k = 4096, .n_44k1 = 5292, .n_48k = 6144, }, ++ { .tmds = 79500000, .n_32k = 4096, .n_44k1 = 4704, .n_48k = 6144, }, ++ { .tmds = 83500000, .n_32k = 4096, .n_44k1 = 7056, .n_48k = 6144, }, ++ { .tmds = 85500000, .n_32k = 4096, .n_44k1 = 5488, .n_48k = 6144, }, ++ { .tmds = 88750000, .n_32k = 4096, .n_44k1 = 14112, .n_48k = 6144, }, ++ { .tmds = 97750000, .n_32k = 4096, .n_44k1 = 14112, .n_48k = 6144, }, ++ { .tmds = 101000000, .n_32k = 4096, .n_44k1 = 7056, .n_48k = 6144, }, ++ { .tmds = 106500000, .n_32k = 4096, .n_44k1 = 4704, .n_48k = 6144, }, ++ { .tmds = 108000000, .n_32k = 4096, .n_44k1 = 5684, .n_48k = 6144, }, ++ { .tmds = 115500000, .n_32k = 4096, .n_44k1 = 5712, .n_48k = 6144, }, ++ { .tmds = 119000000, .n_32k = 4096, .n_44k1 = 5544, .n_48k = 6144, }, ++ { .tmds = 135000000, .n_32k = 4096, .n_44k1 = 5488, .n_48k = 6144, }, ++ { .tmds = 146250000, .n_32k = 4096, .n_44k1 = 6272, .n_48k = 6144, }, ++ { .tmds = 148500000, .n_32k = 4096, .n_44k1 = 5488, .n_48k = 6144, }, ++ { .tmds = 154000000, .n_32k = 4096, .n_44k1 = 5544, .n_48k = 6144, }, ++ { .tmds = 162000000, .n_32k = 4096, .n_44k1 = 5684, .n_48k = 6144, }, ++ ++ /* For 297 MHz+ HDMI spec have some other rule for setting N */ ++ { .tmds = 297000000, .n_32k = 3073, .n_44k1 = 4704, .n_48k = 5120, }, ++ { .tmds = 594000000, .n_32k = 3073, .n_44k1 = 9408, .n_48k = 10240, }, ++ ++ /* End of table */ ++ { .tmds = 0, .n_32k = 0, .n_44k1 = 0, .n_48k = 0, }, ++}; ++ + static const u16 csc_coeff_default[3][4] = { + { 0x2000, 0x0000, 0x0000, 0x0000 }, + { 0x0000, 0x2000, 0x0000, 0x0000 }, +@@ -112,6 +175,7 @@ struct hdmi_data_info { + unsigned int enc_out_bus_format; + unsigned int enc_in_encoding; + unsigned int enc_out_encoding; ++ unsigned int quant_range; + unsigned int pix_repet_factor; + unsigned int hdcp_enable; + struct hdmi_vmode video_mode; +@@ -158,6 +222,7 @@ struct dw_hdmi { + const struct dw_hdmi_plat_data *plat_data; + + int vic; ++ int irq; + + u8 edid[HDMI_EDID_LEN]; + +@@ -174,6 +239,10 @@ struct dw_hdmi { + void __iomem *regs; + bool sink_is_hdmi; + bool sink_has_audio; ++ bool hpd_state; ++ ++ struct delayed_work work; ++ struct workqueue_struct *workqueue; + + struct pinctrl *pinctrl; + struct pinctrl_state *default_state; +@@ -195,6 +264,8 @@ struct dw_hdmi { + unsigned int audio_n; + bool audio_enable; + ++ struct extcon_dev *extcon; ++ + unsigned int reg_shift; + struct regmap *regm; + void (*enable_audio)(struct dw_hdmi *hdmi); +@@ -206,6 +277,7 @@ struct dw_hdmi { + hdmi_codec_plugged_cb plugged_cb; + struct device *codec_dev; + enum drm_connector_status last_connector_result; ++ bool initialized; /* hdmi is enabled before bind */ + }; + + #define HDMI_IH_PHY_STAT0_RX_SENSE \ +@@ -263,6 +335,49 @@ static void hdmi_mask_writeb(struct dw_hdmi *hdmi, u8 data, unsigned int reg, + hdmi_modb(hdmi, data << shift, mask, reg); + } + ++static void repo_hpd_event(struct work_struct *p_work) ++{ ++ struct dw_hdmi *hdmi = container_of(p_work, struct dw_hdmi, work.work); ++ enum drm_connector_status status = hdmi->hpd_state ? ++ connector_status_connected : connector_status_disconnected; ++ ++ if (hdmi->bridge.dev) { ++ drm_helper_hpd_irq_event(hdmi->bridge.dev); ++ drm_bridge_hpd_notify(&hdmi->bridge, status); ++ } ++ ++ extcon_set_state_sync(hdmi->extcon, EXTCON_DISP_HDMI, hdmi->hpd_state); ++} ++ ++static bool check_hdmi_irq(struct dw_hdmi *hdmi, int intr_stat, ++ int phy_int_pol) ++{ ++ int msecs; ++ ++ /* To determine whether interrupt type is HPD */ ++ if (!(intr_stat & HDMI_IH_PHY_STAT0_HPD)) ++ return false; ++ ++ if (phy_int_pol & HDMI_PHY_HPD) { ++ dev_dbg(hdmi->dev, "dw hdmi plug in\n"); ++ msecs = 150; ++ hdmi->hpd_state = true; ++ } else { ++ dev_dbg(hdmi->dev, "dw hdmi plug out\n"); ++ msecs = 20; ++ hdmi->hpd_state = false; ++ } ++ mod_delayed_work(hdmi->workqueue, &hdmi->work, msecs_to_jiffies(msecs)); ++ ++ return true; ++} ++ ++static void init_hpd_work(struct dw_hdmi *hdmi) ++{ ++ hdmi->workqueue = create_workqueue("hpd_queue"); ++ INIT_DELAYED_WORK(&hdmi->work, repo_hpd_event); ++} ++ + static void dw_hdmi_i2c_init(struct dw_hdmi *hdmi) + { + hdmi_writeb(hdmi, HDMI_PHY_I2CM_INT_ADDR_DONE_POL, +@@ -290,6 +405,9 @@ static void dw_hdmi_i2c_init(struct dw_hdmi *hdmi) + /* Mute DONE and ERROR interrupts */ + hdmi_writeb(hdmi, HDMI_IH_I2CM_STAT0_ERROR | HDMI_IH_I2CM_STAT0_DONE, + HDMI_IH_MUTE_I2CM_STAT0); ++ ++ /* set SDA high level holding time */ ++ hdmi_writeb(hdmi, 0x48, HDMI_I2CM_SDA_HOLD); + } + + static bool dw_hdmi_i2c_unwedge(struct dw_hdmi *hdmi) +@@ -570,60 +688,117 @@ static void hdmi_set_cts_n(struct dw_hdmi *hdmi, unsigned int cts, + hdmi_writeb(hdmi, n & 0xff, HDMI_AUD_N1); + } + +-static unsigned int hdmi_compute_n(unsigned int freq, unsigned long pixel_clk) ++static int hdmi_match_tmds_n_table(struct dw_hdmi *hdmi, ++ unsigned long pixel_clk, ++ unsigned long freq) + { +- unsigned int n = (128 * freq) / 1000; +- unsigned int mult = 1; ++ const struct dw_hdmi_plat_data *plat_data = hdmi->plat_data; ++ const struct dw_hdmi_audio_tmds_n *tmds_n = NULL; ++ int i; + +- while (freq > 48000) { +- mult *= 2; +- freq /= 2; ++ if (plat_data->tmds_n_table) { ++ for (i = 0; plat_data->tmds_n_table[i].tmds != 0; i++) { ++ if (pixel_clk == plat_data->tmds_n_table[i].tmds) { ++ tmds_n = &plat_data->tmds_n_table[i]; ++ break; ++ } ++ } + } + ++ if (tmds_n == NULL) { ++ for (i = 0; common_tmds_n_table[i].tmds != 0; i++) { ++ if (pixel_clk == common_tmds_n_table[i].tmds) { ++ tmds_n = &common_tmds_n_table[i]; ++ break; ++ } ++ } ++ } ++ ++ if (tmds_n == NULL) ++ return -ENOENT; ++ + switch (freq) { + case 32000: +- if (pixel_clk == 25175000) +- n = 4576; +- else if (pixel_clk == 27027000) +- n = 4096; +- else if (pixel_clk == 74176000 || pixel_clk == 148352000) +- n = 11648; +- else +- n = 4096; +- n *= mult; +- break; +- ++ return tmds_n->n_32k; + case 44100: +- if (pixel_clk == 25175000) +- n = 7007; +- else if (pixel_clk == 74176000) +- n = 17836; +- else if (pixel_clk == 148352000) +- n = 8918; +- else +- n = 6272; +- n *= mult; +- break; +- ++ case 88200: ++ case 176400: ++ return (freq / 44100) * tmds_n->n_44k1; + case 48000: +- if (pixel_clk == 25175000) +- n = 6864; +- else if (pixel_clk == 27027000) +- n = 6144; +- else if (pixel_clk == 74176000) +- n = 11648; +- else if (pixel_clk == 148352000) +- n = 5824; +- else +- n = 6144; +- n *= mult; +- break; +- ++ case 96000: ++ case 192000: ++ return (freq / 48000) * tmds_n->n_48k; + default: +- break; ++ return -ENOENT; ++ } ++} ++ ++static u64 hdmi_audio_math_diff(unsigned int freq, unsigned int n, ++ unsigned int pixel_clk) ++{ ++ u64 final, diff; ++ u64 cts; ++ ++ final = (u64)pixel_clk * n; ++ ++ cts = final; ++ do_div(cts, 128 * freq); ++ ++ diff = final - (u64)cts * (128 * freq); ++ ++ return diff; ++} ++ ++static unsigned int hdmi_compute_n(struct dw_hdmi *hdmi, ++ unsigned long pixel_clk, ++ unsigned long freq) ++{ ++ unsigned int min_n = DIV_ROUND_UP((128 * freq), 1500); ++ unsigned int max_n = (128 * freq) / 300; ++ unsigned int ideal_n = (128 * freq) / 1000; ++ unsigned int best_n_distance = ideal_n; ++ unsigned int best_n = 0; ++ u64 best_diff = U64_MAX; ++ int n; ++ ++ /* If the ideal N could satisfy the audio math, then just take it */ ++ if (hdmi_audio_math_diff(freq, ideal_n, pixel_clk) == 0) ++ return ideal_n; ++ ++ for (n = min_n; n <= max_n; n++) { ++ u64 diff = hdmi_audio_math_diff(freq, n, pixel_clk); ++ ++ if (diff < best_diff || (diff == best_diff && ++ abs(n - ideal_n) < best_n_distance)) { ++ best_n = n; ++ best_diff = diff; ++ best_n_distance = abs(best_n - ideal_n); ++ } ++ ++ /* ++ * The best N already satisfy the audio math, and also be ++ * the closest value to ideal N, so just cut the loop. ++ */ ++ if ((best_diff == 0) && (abs(n - ideal_n) > best_n_distance)) ++ break; + } + +- return n; ++ return best_n; ++} ++ ++static unsigned int hdmi_find_n(struct dw_hdmi *hdmi, unsigned long pixel_clk, ++ unsigned long sample_rate) ++{ ++ int n; ++ ++ n = hdmi_match_tmds_n_table(hdmi, pixel_clk, sample_rate); ++ if (n > 0) ++ return n; ++ ++ dev_warn(hdmi->dev, "Rate %lu missing; compute N dynamically\n", ++ pixel_clk); ++ ++ return hdmi_compute_n(hdmi, pixel_clk, sample_rate); + } + + /* +@@ -654,7 +829,7 @@ static void hdmi_set_clk_regenerator(struct dw_hdmi *hdmi, + u8 config3; + u64 tmp; + +- n = hdmi_compute_n(sample_rate, pixel_clk); ++ n = hdmi_find_n(hdmi, pixel_clk, sample_rate); + + config3 = hdmi_readb(hdmi, HDMI_CONFIG3_ID); + +@@ -1005,6 +1180,15 @@ static bool is_csc_needed(struct dw_hdmi *hdmi) + is_color_space_interpolation(hdmi); + } + ++static bool is_rgb_full_to_limited_needed(struct dw_hdmi *hdmi) ++{ ++ if (hdmi->hdmi_data.quant_range == HDMI_QUANTIZATION_RANGE_LIMITED || ++ (!hdmi->hdmi_data.quant_range && hdmi->hdmi_data.rgb_limited_range)) ++ return true; ++ ++ return false; ++} ++ + static void dw_hdmi_update_csc_coeffs(struct dw_hdmi *hdmi) + { + const u16 (*csc_coeff)[3][4] = &csc_coeff_default; +@@ -1027,7 +1211,7 @@ static void dw_hdmi_update_csc_coeffs(struct dw_hdmi *hdmi) + csc_coeff = &csc_coeff_rgb_in_eitu709; + csc_scale = 0; + } else if (is_input_rgb && is_output_rgb && +- hdmi->hdmi_data.rgb_limited_range) { ++ is_rgb_full_to_limited_needed(hdmi)) { + csc_coeff = &csc_coeff_rgb_full_to_rgb_limited; + } + +@@ -1155,7 +1339,7 @@ static void hdmi_video_packetize(struct dw_hdmi *hdmi) + HDMI_VP_STUFF_PR_STUFFING_MASK, HDMI_VP_STUFF); + + /* Data from pixel repeater block */ +- if (hdmi_data->pix_repet_factor > 1) { ++ if (hdmi_data->pix_repet_factor > 0) { + vp_conf = HDMI_VP_CONF_PR_EN_ENABLE | + HDMI_VP_CONF_BYPASS_SELECT_PIX_REPEATER; + } else { /* data from packetizer block */ +@@ -1269,6 +1453,23 @@ static bool dw_hdmi_support_scdc(struct dw_hdmi *hdmi, + return true; + } + ++static int hdmi_phy_i2c_read(struct dw_hdmi *hdmi, unsigned char addr) ++{ ++ int val; ++ ++ hdmi_writeb(hdmi, 0xFF, HDMI_IH_I2CMPHY_STAT0); ++ hdmi_writeb(hdmi, addr, HDMI_PHY_I2CM_ADDRESS_ADDR); ++ hdmi_writeb(hdmi, 0, HDMI_PHY_I2CM_DATAI_1_ADDR); ++ hdmi_writeb(hdmi, 0, HDMI_PHY_I2CM_DATAI_0_ADDR); ++ hdmi_writeb(hdmi, HDMI_PHY_I2CM_OPERATION_ADDR_READ, ++ HDMI_PHY_I2CM_OPERATION_ADDR); ++ hdmi_phy_wait_i2c_done(hdmi, 1000); ++ val = hdmi_readb(hdmi, HDMI_PHY_I2CM_DATAI_1_ADDR); ++ val = (val & 0xff) << 8; ++ val += hdmi_readb(hdmi, HDMI_PHY_I2CM_DATAI_0_ADDR) & 0xff; ++ return val; ++} ++ + /* + * HDMI2.0 Specifies the following procedure for High TMDS Bit Rates: + * - The Source shall suspend transmission of the TMDS clock and data +@@ -1447,6 +1648,10 @@ static int hdmi_phy_configure_dwc_hdmi_3d_tx(struct dw_hdmi *hdmi, + const struct dw_hdmi_curr_ctrl *curr_ctrl = pdata->cur_ctr; + const struct dw_hdmi_phy_config *phy_config = pdata->phy_config; + ++ if (hdmi_bus_fmt_is_yuv420(hdmi->hdmi_data.enc_out_bus_format) && ++ pdata->mpll_cfg_420) ++ mpll_config = pdata->mpll_cfg_420; ++ + /* TOFIX Will need 420 specific PHY configuration tables */ + + /* PLL/MPLL Cfg - always match on final entry */ +@@ -1642,10 +1847,15 @@ static void hdmi_config_AVI(struct dw_hdmi *hdmi, + drm_hdmi_avi_infoframe_from_display_mode(&frame, connector, mode); + + if (hdmi_bus_fmt_is_rgb(hdmi->hdmi_data.enc_out_bus_format)) { +- drm_hdmi_avi_infoframe_quant_range(&frame, connector, mode, +- hdmi->hdmi_data.rgb_limited_range ? +- HDMI_QUANTIZATION_RANGE_LIMITED : +- HDMI_QUANTIZATION_RANGE_FULL); ++ /* default range */ ++ if (!hdmi->hdmi_data.quant_range) ++ drm_hdmi_avi_infoframe_quant_range(&frame, connector, mode, ++ hdmi->hdmi_data.rgb_limited_range ? ++ HDMI_QUANTIZATION_RANGE_LIMITED : ++ HDMI_QUANTIZATION_RANGE_FULL); ++ else ++ drm_hdmi_avi_infoframe_quant_range(&frame, connector, mode, ++ hdmi->hdmi_data.quant_range); + } else { + frame.quantization_range = HDMI_QUANTIZATION_RANGE_DEFAULT; + frame.ycc_quantization_range = +@@ -1858,8 +2068,10 @@ static void hdmi_av_composer(struct dw_hdmi *hdmi, + int hblank, vblank, h_de_hs, v_de_vs, hsync_len, vsync_len; + unsigned int vdisplay, hdisplay; + +- vmode->mpixelclock = mode->clock * 1000; + ++ vmode->mpixelclock = mode->crtc_clock * 1000; ++ if (hdmi_bus_fmt_is_yuv420(hdmi->hdmi_data.enc_out_bus_format)) ++ vmode->mpixelclock /= 2; + dev_dbg(hdmi->dev, "final pixclk = %d\n", vmode->mpixelclock); + + vmode->mtmdsclock = vmode->mpixelclock; +@@ -2047,6 +2259,12 @@ static void dw_hdmi_enable_video_path(struct dw_hdmi *hdmi) + hdmi->mc_clkdis &= ~HDMI_MC_CLKDIS_TMDSCLK_DISABLE; + hdmi_writeb(hdmi, hdmi->mc_clkdis, HDMI_MC_CLKDIS); + ++ /* Enable pixel repetition path */ ++ if (hdmi->hdmi_data.video_mode.mpixelrepetitioninput) { ++ hdmi->mc_clkdis &= ~HDMI_MC_CLKDIS_PREPCLK_DISABLE; ++ hdmi_writeb(hdmi, hdmi->mc_clkdis, HDMI_MC_CLKDIS); ++ } ++ + /* Enable csc path */ + if (is_csc_needed(hdmi)) { + hdmi->mc_clkdis &= ~HDMI_MC_CLKDIS_CSCCLK_DISABLE; +@@ -2122,6 +2340,7 @@ static int dw_hdmi_setup(struct dw_hdmi *hdmi, + const struct drm_display_mode *mode) + { + int ret; ++ void *data = hdmi->plat_data->phy_data; + + hdmi_disable_overflow_interrupts(hdmi); + +@@ -2133,35 +2352,70 @@ static int dw_hdmi_setup(struct dw_hdmi *hdmi, + dev_dbg(hdmi->dev, "CEA mode used vic=%d\n", hdmi->vic); + } + +- if ((hdmi->vic == 6) || (hdmi->vic == 7) || +- (hdmi->vic == 21) || (hdmi->vic == 22) || +- (hdmi->vic == 2) || (hdmi->vic == 3) || +- (hdmi->vic == 17) || (hdmi->vic == 18)) ++ if (hdmi->plat_data->get_enc_out_encoding) ++ hdmi->hdmi_data.enc_out_encoding = ++ hdmi->plat_data->get_enc_out_encoding(data); ++ else if ((hdmi->vic == 6) || (hdmi->vic == 7) || ++ (hdmi->vic == 21) || (hdmi->vic == 22) || ++ (hdmi->vic == 2) || (hdmi->vic == 3) || ++ (hdmi->vic == 17) || (hdmi->vic == 18)) + hdmi->hdmi_data.enc_out_encoding = V4L2_YCBCR_ENC_601; + else + hdmi->hdmi_data.enc_out_encoding = V4L2_YCBCR_ENC_709; + +- hdmi->hdmi_data.video_mode.mpixelrepetitionoutput = 0; +- hdmi->hdmi_data.video_mode.mpixelrepetitioninput = 0; ++ if (mode->flags & DRM_MODE_FLAG_DBLCLK) { ++ hdmi->hdmi_data.video_mode.mpixelrepetitionoutput = 1; ++ hdmi->hdmi_data.video_mode.mpixelrepetitioninput = 1; ++ } else { ++ hdmi->hdmi_data.video_mode.mpixelrepetitionoutput = 0; ++ hdmi->hdmi_data.video_mode.mpixelrepetitioninput = 0; ++ } ++ /* TOFIX: Get input format from plat data or fallback to RGB888 */ ++ if (hdmi->plat_data->get_input_bus_format) ++ hdmi->hdmi_data.enc_in_bus_format = ++ hdmi->plat_data->get_input_bus_format(data); ++ else if (hdmi->plat_data->input_bus_format) ++ hdmi->hdmi_data.enc_in_bus_format = ++ hdmi->plat_data->input_bus_format; ++ else ++ hdmi->hdmi_data.enc_in_bus_format = ++ MEDIA_BUS_FMT_RGB888_1X24; + +- if (hdmi->hdmi_data.enc_in_bus_format == MEDIA_BUS_FMT_FIXED) +- hdmi->hdmi_data.enc_in_bus_format = MEDIA_BUS_FMT_RGB888_1X24; ++ /* TOFIX: Default to RGB888 output format */ ++ if (hdmi->plat_data->get_output_bus_format) ++ hdmi->hdmi_data.enc_out_bus_format = ++ hdmi->plat_data->get_output_bus_format(data); ++ else ++ hdmi->hdmi_data.enc_out_bus_format = ++ MEDIA_BUS_FMT_RGB888_1X24; + + /* TOFIX: Get input encoding from plat data or fallback to none */ +- if (hdmi->plat_data->input_bus_encoding) ++ if (hdmi->plat_data->get_enc_in_encoding) ++ hdmi->hdmi_data.enc_in_encoding = ++ hdmi->plat_data->get_enc_in_encoding(data); ++ else if (hdmi->plat_data->input_bus_encoding) + hdmi->hdmi_data.enc_in_encoding = + hdmi->plat_data->input_bus_encoding; + else + hdmi->hdmi_data.enc_in_encoding = V4L2_YCBCR_ENC_DEFAULT; + +- if (hdmi->hdmi_data.enc_out_bus_format == MEDIA_BUS_FMT_FIXED) +- hdmi->hdmi_data.enc_out_bus_format = MEDIA_BUS_FMT_RGB888_1X24; ++ ++ if (hdmi->plat_data->get_quant_range) ++ hdmi->hdmi_data.quant_range = ++ hdmi->plat_data->get_quant_range(data); + + hdmi->hdmi_data.rgb_limited_range = hdmi->sink_is_hdmi && + drm_default_rgb_quant_range(mode) == + HDMI_QUANTIZATION_RANGE_LIMITED; + +- hdmi->hdmi_data.pix_repet_factor = 0; ++ /* ++ * According to the dw-hdmi specification 6.4.2 ++ * vp_pr_cd[3:0]: ++ * 0000b: No pixel repetition (pixel sent only once) ++ * 0001b: Pixel sent two times (pixel repeated once) ++ */ ++ hdmi->hdmi_data.pix_repet_factor = ++ (mode->flags & DRM_MODE_FLAG_DBLCLK) ? 1 : 0; + hdmi->hdmi_data.hdcp_enable = 0; + hdmi->hdmi_data.video_mode.mdataenablepolarity = true; + +@@ -2295,6 +2549,10 @@ static void dw_hdmi_update_power(struct dw_hdmi *hdmi) + } + + if (force == DRM_FORCE_OFF) { ++ if (hdmi->initialized) { ++ hdmi->initialized = false; ++ hdmi->disabled = true; ++ } + if (hdmi->bridge_is_on) + dw_hdmi_poweroff(hdmi); + } else { +@@ -2395,6 +2653,15 @@ static int dw_hdmi_connector_get_modes(struct drm_connector *connector) + return ret; + } + ++static struct drm_encoder * ++dw_hdmi_connector_best_encoder(struct drm_connector *connector) ++{ ++ struct dw_hdmi *hdmi = container_of(connector, struct dw_hdmi, ++ connector); ++ ++ return hdmi->bridge.encoder; ++} ++ + static bool hdr_metadata_equal(const struct drm_connector_state *old_state, + const struct drm_connector_state *new_state) + { +@@ -2434,12 +2701,66 @@ static int dw_hdmi_connector_atomic_check(struct drm_connector *connector, + return 0; + } + ++static int ++dw_hdmi_atomic_connector_set_property(struct drm_connector *connector, ++ struct drm_connector_state *state, ++ struct drm_property *property, ++ uint64_t val) ++{ ++ struct dw_hdmi *hdmi = container_of(connector, struct dw_hdmi, ++ connector); ++ const struct dw_hdmi_property_ops *ops = ++ hdmi->plat_data->property_ops; ++ ++ if (ops && ops->set_property) ++ return ops->set_property(connector, state, property, ++ val, hdmi->plat_data->phy_data); ++ else ++ return -EINVAL; ++} ++ ++static int ++dw_hdmi_atomic_connector_get_property(struct drm_connector *connector, ++ const struct drm_connector_state *state, ++ struct drm_property *property, ++ uint64_t *val) ++{ ++ struct dw_hdmi *hdmi = container_of(connector, struct dw_hdmi, ++ connector); ++ const struct dw_hdmi_property_ops *ops = ++ hdmi->plat_data->property_ops; ++ ++ if (ops && ops->get_property) ++ return ops->get_property(connector, state, property, ++ val, hdmi->plat_data->phy_data); ++ else ++ return -EINVAL; ++} ++ ++static int ++dw_hdmi_connector_set_property(struct drm_connector *connector, ++ struct drm_property *property, uint64_t val) ++{ ++ return dw_hdmi_atomic_connector_set_property(connector, NULL, ++ property, val); ++} ++ + static void dw_hdmi_connector_force(struct drm_connector *connector) + { + struct dw_hdmi *hdmi = container_of(connector, struct dw_hdmi, + connector); + + mutex_lock(&hdmi->mutex); ++ ++ if (!hdmi->disabled && hdmi->force != connector->force) { ++ if (connector->force == DRM_FORCE_OFF) ++ extcon_set_state_sync(hdmi->extcon, EXTCON_DISP_HDMI, ++ false); ++ else if (connector->force == DRM_FORCE_ON) ++ extcon_set_state_sync(hdmi->extcon, EXTCON_DISP_HDMI, ++ true); ++ } ++ + hdmi->force = connector->force; + dw_hdmi_update_power(hdmi); + dw_hdmi_update_phy_mask(hdmi); +@@ -2452,15 +2773,81 @@ static const struct drm_connector_funcs dw_hdmi_connector_funcs = { + .destroy = drm_connector_cleanup, + .force = dw_hdmi_connector_force, + .reset = drm_atomic_helper_connector_reset, ++ .set_property = dw_hdmi_connector_set_property, + .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state, + .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, ++ .atomic_set_property = dw_hdmi_atomic_connector_set_property, ++ .atomic_get_property = dw_hdmi_atomic_connector_get_property, + }; + + static const struct drm_connector_helper_funcs dw_hdmi_connector_helper_funcs = { + .get_modes = dw_hdmi_connector_get_modes, ++ .best_encoder = dw_hdmi_connector_best_encoder, + .atomic_check = dw_hdmi_connector_atomic_check, + }; + ++static void dw_hdmi_attach_properties(struct dw_hdmi *hdmi) ++{ ++ unsigned int color = MEDIA_BUS_FMT_RGB888_1X24; ++ int video_mapping, colorspace; ++ enum drm_connector_status connect_status = ++ hdmi->phy.ops->read_hpd(hdmi, hdmi->phy.data); ++ const struct dw_hdmi_property_ops *ops = ++ hdmi->plat_data->property_ops; ++ ++ if (connect_status == connector_status_connected) { ++ video_mapping = (hdmi_readb(hdmi, HDMI_TX_INVID0) & ++ HDMI_TX_INVID0_VIDEO_MAPPING_MASK); ++ colorspace = (hdmi_readb(hdmi, HDMI_FC_AVICONF0) & ++ HDMI_FC_AVICONF0_PIX_FMT_MASK); ++ switch (video_mapping) { ++ case 0x01: ++ color = MEDIA_BUS_FMT_RGB888_1X24; ++ break; ++ case 0x03: ++ color = MEDIA_BUS_FMT_RGB101010_1X30; ++ break; ++ case 0x09: ++ if (colorspace == HDMI_COLORSPACE_YUV420) ++ color = MEDIA_BUS_FMT_UYYVYY8_0_5X24; ++ else ++ color = MEDIA_BUS_FMT_YUV8_1X24; ++ break; ++ case 0x0b: ++ if (colorspace == HDMI_COLORSPACE_YUV420) ++ color = MEDIA_BUS_FMT_UYYVYY10_0_5X30; ++ else ++ color = MEDIA_BUS_FMT_YUV10_1X30; ++ break; ++ case 0x14: ++ color = MEDIA_BUS_FMT_UYVY10_1X20; ++ break; ++ case 0x16: ++ color = MEDIA_BUS_FMT_UYVY8_1X16; ++ break; ++ default: ++ color = MEDIA_BUS_FMT_RGB888_1X24; ++ dev_err(hdmi->dev, "unexpected mapping: 0x%x\n", ++ video_mapping); ++ } ++ } ++ ++ if (ops && ops->attach_properties) ++ return ops->attach_properties(&hdmi->connector, ++ color, hdmi->version, ++ hdmi->plat_data->phy_data); ++} ++ ++static void dw_hdmi_destroy_properties(struct dw_hdmi *hdmi) ++{ ++ const struct dw_hdmi_property_ops *ops = ++ hdmi->plat_data->property_ops; ++ ++ if (ops && ops->destroy_properties) ++ return ops->destroy_properties(&hdmi->connector, ++ hdmi->plat_data->phy_data); ++} ++ + static int dw_hdmi_connector_create(struct dw_hdmi *hdmi) + { + struct drm_connector *connector = &hdmi->connector; +@@ -2497,6 +2884,8 @@ static int dw_hdmi_connector_create(struct dw_hdmi *hdmi) + + drm_connector_attach_encoder(connector, hdmi->bridge.encoder); + ++ dw_hdmi_attach_properties(hdmi); ++ + cec_fill_conn_info_from_drm(&conn_info, connector); + + notifier = cec_notifier_conn_register(hdmi->dev, NULL, &conn_info); +@@ -2812,16 +3201,13 @@ dw_hdmi_bridge_mode_valid(struct drm_bridge *bridge, + const struct drm_display_mode *mode) + { + struct dw_hdmi *hdmi = bridge->driver_private; ++ struct drm_connector *connector = &hdmi->connector; + const struct dw_hdmi_plat_data *pdata = hdmi->plat_data; + enum drm_mode_status mode_status = MODE_OK; + +- /* We don't support double-clocked modes */ +- if (mode->flags & DRM_MODE_FLAG_DBLCLK) +- return MODE_BAD; +- + if (pdata->mode_valid) +- mode_status = pdata->mode_valid(hdmi, pdata->priv_data, info, +- mode); ++ mode_status = pdata->mode_valid(connector, pdata->priv_data, ++ info, mode); + + return mode_status; + } +@@ -3015,20 +3401,7 @@ static irqreturn_t dw_hdmi_irq(int irq, void *dev_id) + } + } + +- if (intr_stat & HDMI_IH_PHY_STAT0_HPD) { +- enum drm_connector_status status = phy_int_pol & HDMI_PHY_HPD +- ? connector_status_connected +- : connector_status_disconnected; +- +- dev_dbg(hdmi->dev, "EVENT=%s\n", +- status == connector_status_connected ? +- "plugin" : "plugout"); +- +- if (hdmi->bridge.dev) { +- drm_helper_hpd_irq_event(hdmi->bridge.dev); +- drm_bridge_hpd_notify(&hdmi->bridge, status); +- } +- } ++ check_hdmi_irq(hdmi, intr_stat, phy_int_pol); + + hdmi_writeb(hdmi, intr_stat, HDMI_IH_PHY_STAT0); + hdmi_writeb(hdmi, ~(HDMI_IH_PHY_STAT0_HPD | HDMI_IH_PHY_STAT0_RX_SENSE), +@@ -3173,6 +3546,158 @@ static void dw_hdmi_init_hw(struct dw_hdmi *hdmi) + hdmi->phy.ops->setup_hpd(hdmi, hdmi->phy.data); + } + ++#include ++#include ++#include ++ ++struct dw_hdmi_reg_table { ++ int reg_base; ++ int reg_end; ++}; ++ ++static const struct dw_hdmi_reg_table hdmi_reg_table[] = { ++ {HDMI_DESIGN_ID, HDMI_CONFIG3_ID}, ++ {HDMI_IH_FC_STAT0, HDMI_IH_MUTE}, ++ {HDMI_TX_INVID0, HDMI_TX_BCBDATA1}, ++ {HDMI_VP_STATUS, HDMI_VP_POL}, ++ {HDMI_FC_INVIDCONF, HDMI_FC_DBGTMDS2}, ++ {HDMI_PHY_CONF0, HDMI_PHY_POL0}, ++ {HDMI_PHY_I2CM_SLAVE_ADDR, HDMI_PHY_I2CM_FS_SCL_LCNT_0_ADDR}, ++ {HDMI_AUD_CONF0, 0x3624}, ++ {HDMI_MC_SFRDIV, HDMI_MC_HEACPHY_RST}, ++ {HDMI_CSC_CFG, HDMI_CSC_COEF_C4_LSB}, ++ {HDMI_A_HDCPCFG0, 0x52bb}, ++ {0x7800, 0x7818}, ++ {0x7900, 0x790e}, ++ {HDMI_CEC_CTRL, HDMI_CEC_WKUPCTRL}, ++ {HDMI_I2CM_SLAVE, 0x7e31}, ++}; ++ ++static int dw_hdmi_ctrl_show(struct seq_file *s, void *v) ++{ ++ struct dw_hdmi *hdmi = s->private; ++ u32 i = 0, j = 0, val = 0; ++ ++ seq_puts(s, "\n>>>hdmi_ctl reg "); ++ for (i = 0; i < 16; i++) ++ seq_printf(s, " %2x", i); ++ seq_puts(s, "\n---------------------------------------------------"); ++ ++ for (i = 0; i < ARRAY_SIZE(hdmi_reg_table); i++) { ++ for (j = hdmi_reg_table[i].reg_base; ++ j <= hdmi_reg_table[i].reg_end; j++) { ++ val = hdmi_readb(hdmi, j); ++ if ((j - hdmi_reg_table[i].reg_base) % 16 == 0) ++ seq_printf(s, "\n>>>hdmi_ctl %04x:", j); ++ seq_printf(s, " %02x", val); ++ } ++ } ++ seq_puts(s, "\n---------------------------------------------------\n"); ++ ++ return 0; ++} ++ ++static int dw_hdmi_ctrl_open(struct inode *inode, struct file *file) ++{ ++ return single_open(file, dw_hdmi_ctrl_show, inode->i_private); ++} ++ ++static ssize_t ++dw_hdmi_ctrl_write(struct file *file, const char __user *buf, ++ size_t count, loff_t *ppos) ++{ ++ struct dw_hdmi *hdmi = ++ ((struct seq_file *)file->private_data)->private; ++ u32 reg, val; ++ char kbuf[25]; ++ ++ if (copy_from_user(kbuf, buf, count)) ++ return -EFAULT; ++ if (sscanf(kbuf, "%x%x", ®, &val) == -1) ++ return -EFAULT; ++ if (reg > HDMI_I2CM_FS_SCL_LCNT_0_ADDR) { ++ dev_err(hdmi->dev, "it is no a hdmi register\n"); ++ return count; ++ } ++ dev_info(hdmi->dev, "/**********hdmi register config******/"); ++ dev_info(hdmi->dev, "\n reg=%x val=%x\n", reg, val); ++ hdmi_writeb(hdmi, val, reg); ++ return count; ++} ++ ++static const struct file_operations dw_hdmi_ctrl_fops = { ++ .owner = THIS_MODULE, ++ .open = dw_hdmi_ctrl_open, ++ .read = seq_read, ++ .write = dw_hdmi_ctrl_write, ++ .llseek = seq_lseek, ++ .release = single_release, ++}; ++ ++static int dw_hdmi_phy_show(struct seq_file *s, void *v) ++{ ++ struct dw_hdmi *hdmi = s->private; ++ u32 i; ++ ++ seq_puts(s, "\n>>>hdmi_phy reg "); ++ for (i = 0; i < 0x28; i++) ++ seq_printf(s, "regs %02x val %04x\n", ++ i, hdmi_phy_i2c_read(hdmi, i)); ++ return 0; ++} ++ ++static int dw_hdmi_phy_open(struct inode *inode, struct file *file) ++{ ++ return single_open(file, dw_hdmi_phy_show, inode->i_private); ++} ++ ++static ssize_t ++dw_hdmi_phy_write(struct file *file, const char __user *buf, ++ size_t count, loff_t *ppos) ++{ ++ struct dw_hdmi *hdmi = ++ ((struct seq_file *)file->private_data)->private; ++ u32 reg, val; ++ char kbuf[25]; ++ ++ if (copy_from_user(kbuf, buf, count)) ++ return -EFAULT; ++ if (sscanf(kbuf, "%x%x", ®, &val) == -1) ++ return -EFAULT; ++ if (reg > 0x28) { ++ dev_err(hdmi->dev, "it is not a hdmi phy register\n"); ++ return count; ++ } ++ dev_info(hdmi->dev, "/*******hdmi phy register config******/"); ++ dev_info(hdmi->dev, "\n reg=%x val=%x\n", reg, val); ++ dw_hdmi_phy_i2c_write(hdmi, val, reg); ++ return count; ++} ++ ++static const struct file_operations dw_hdmi_phy_fops = { ++ .owner = THIS_MODULE, ++ .open = dw_hdmi_phy_open, ++ .read = seq_read, ++ .write = dw_hdmi_phy_write, ++ .llseek = seq_lseek, ++ .release = single_release, ++}; ++ ++static void dw_hdmi_register_debugfs(struct device *dev, struct dw_hdmi *hdmi) ++{ ++ struct dentry *debugfs_dir; ++ ++ debugfs_dir = debugfs_create_dir("dw-hdmi", NULL); ++ if (IS_ERR(debugfs_dir)) { ++ dev_err(dev, "failed to create debugfs dir!\n"); ++ return; ++ } ++ debugfs_create_file("ctrl", 0400, debugfs_dir, ++ hdmi, &dw_hdmi_ctrl_fops); ++ debugfs_create_file("phy", 0400, debugfs_dir, ++ hdmi, &dw_hdmi_phy_fops); ++} ++ + /* ----------------------------------------------------------------------------- + * Probe/remove API, used from platforms based on the DRM bridge API. + */ +@@ -3212,6 +3737,8 @@ struct dw_hdmi *dw_hdmi_probe(struct platform_device *pdev, + mutex_init(&hdmi->cec_notifier_mutex); + spin_lock_init(&hdmi->audio_lock); + ++ dev_set_name(dev, "hdmi"); ++ + ddc_node = of_parse_phandle(np, "ddc-i2c-bus", 0); + if (ddc_node) { + hdmi->ddc = of_get_i2c_adapter_by_node(ddc_node); +@@ -3328,6 +3855,20 @@ struct dw_hdmi *dw_hdmi_probe(struct platform_device *pdev, + prod_id1 & HDMI_PRODUCT_ID1_HDCP ? "with" : "without", + hdmi->phy.name); + ++ hdmi->initialized = false; ++ ret = hdmi_readb(hdmi, HDMI_PHY_STAT0); ++ if ((ret & HDMI_PHY_TX_PHY_LOCK) && (ret & HDMI_PHY_HPD) && ++ hdmi_readb(hdmi, HDMI_FC_EXCTRLDUR)) { ++ hdmi->mc_clkdis = hdmi_readb(hdmi, HDMI_MC_CLKDIS); ++ hdmi->disabled = false; ++ hdmi->bridge_is_on = true; ++ hdmi->phy.enabled = true; ++ hdmi->initialized = true; ++ } else if (ret & HDMI_PHY_TX_PHY_LOCK) { ++ hdmi->phy.ops->disable(hdmi, hdmi->phy.data); ++ } ++ ++ init_hpd_work(hdmi); + dw_hdmi_init_hw(hdmi); + + irq = platform_get_irq(pdev, 0); +@@ -3336,6 +3877,7 @@ struct dw_hdmi *dw_hdmi_probe(struct platform_device *pdev, + goto err_iahb; + } + ++ hdmi->irq = irq; + ret = devm_request_threaded_irq(dev, irq, dw_hdmi_hardirq, + dw_hdmi_irq, IRQF_SHARED, + dev_name(dev), hdmi); +@@ -3434,8 +3976,33 @@ struct dw_hdmi *dw_hdmi_probe(struct platform_device *pdev, + hdmi->cec = platform_device_register_full(&pdevinfo); + } + ++ hdmi->extcon = devm_extcon_dev_allocate(hdmi->dev, dw_hdmi_cable); ++ if (IS_ERR(hdmi->extcon)) { ++ ret = PTR_ERR(hdmi->extcon); ++ dev_err(hdmi->dev, "allocate extcon failed: %d\n", ret); ++ goto err_iahb; ++ } ++ ++ ret = devm_extcon_dev_register(hdmi->dev, hdmi->extcon); ++ if (ret) { ++ dev_err(hdmi->dev, "failed to register extcon: %d\n", ++ ret); ++ goto err_iahb; ++ } ++ ++ ret = extcon_set_property_capability(hdmi->extcon, EXTCON_DISP_HDMI, ++ EXTCON_PROP_DISP_HPD); ++ if (ret) { ++ dev_err(hdmi->dev, ++ "failed to set USB property capability: %d\n", ++ ret); ++ goto err_iahb; ++ } ++ + drm_bridge_add(&hdmi->bridge); + ++ dw_hdmi_register_debugfs(dev, hdmi); ++ + return hdmi; + + err_iahb: +@@ -3463,6 +4030,8 @@ void dw_hdmi_remove(struct dw_hdmi *hdmi) + /* Disable all interrupts */ + hdmi_writeb(hdmi, ~0, HDMI_IH_MUTE_PHY_STAT0); + ++ dw_hdmi_destroy_properties(hdmi); ++ + clk_disable_unprepare(hdmi->iahb_clk); + clk_disable_unprepare(hdmi->isfr_clk); + if (hdmi->cec_clk) +@@ -3480,7 +4049,7 @@ EXPORT_SYMBOL_GPL(dw_hdmi_remove); + */ + struct dw_hdmi *dw_hdmi_bind(struct platform_device *pdev, + struct drm_encoder *encoder, +- const struct dw_hdmi_plat_data *plat_data) ++ struct dw_hdmi_plat_data *plat_data) + { + struct dw_hdmi *hdmi; + int ret; +@@ -3495,6 +4064,7 @@ struct dw_hdmi *dw_hdmi_bind(struct platform_device *pdev, + DRM_ERROR("Failed to initialize bridge with drm\n"); + return ERR_PTR(ret); + } ++ plat_data->connector = &hdmi->connector; + + return hdmi; + } +@@ -3506,9 +4076,59 @@ void dw_hdmi_unbind(struct dw_hdmi *hdmi) + } + EXPORT_SYMBOL_GPL(dw_hdmi_unbind); + ++static void dw_hdmi_reg_initial(struct dw_hdmi *hdmi) ++{ ++ if (hdmi_readb(hdmi, HDMI_IH_MUTE)) { ++ initialize_hdmi_ih_mutes(hdmi); ++ hdmi_writeb(hdmi, HDMI_PHY_I2CM_INT_ADDR_DONE_POL, ++ HDMI_PHY_I2CM_INT_ADDR); ++ ++ hdmi_writeb(hdmi, HDMI_PHY_I2CM_CTLINT_ADDR_NAC_POL | ++ HDMI_PHY_I2CM_CTLINT_ADDR_ARBITRATION_POL, ++ HDMI_PHY_I2CM_CTLINT_ADDR); ++ ++ hdmi_writeb(hdmi, HDMI_PHY_HPD | HDMI_PHY_RX_SENSE, ++ HDMI_PHY_POL0); ++ hdmi_writeb(hdmi, hdmi->phy_mask, HDMI_PHY_MASK0); ++ hdmi_writeb(hdmi, ~(HDMI_IH_PHY_STAT0_HPD | ++ HDMI_IH_PHY_STAT0_RX_SENSE), ++ HDMI_IH_MUTE_PHY_STAT0); ++ } ++} ++ ++void dw_hdmi_suspend(struct dw_hdmi *hdmi) ++{ ++ mutex_lock(&hdmi->mutex); ++ ++ /* ++ * When system shutdown, hdmi should be disabled. ++ * When system suspend, dw_hdmi_bridge_disable will disable hdmi first. ++ * To prevent duplicate operation, we should determine whether hdmi ++ * has been disabled. ++ */ ++ if (!hdmi->disabled) { ++ hdmi->disabled = true; ++ dw_hdmi_update_power(hdmi); ++ dw_hdmi_update_phy_mask(hdmi); ++ } ++ mutex_unlock(&hdmi->mutex); ++ ++ if (hdmi->irq) ++ disable_irq(hdmi->irq); ++ pinctrl_pm_select_sleep_state(hdmi->dev); ++} ++EXPORT_SYMBOL_GPL(dw_hdmi_suspend); ++ + void dw_hdmi_resume(struct dw_hdmi *hdmi) + { +- dw_hdmi_init_hw(hdmi); ++ pinctrl_pm_select_default_state(hdmi->dev); ++ mutex_lock(&hdmi->mutex); ++ dw_hdmi_reg_initial(hdmi); ++ if (hdmi->i2c) ++ dw_hdmi_i2c_init(hdmi); ++ if (hdmi->irq) ++ enable_irq(hdmi->irq); ++ mutex_unlock(&hdmi->mutex); + } + EXPORT_SYMBOL_GPL(dw_hdmi_resume); + +diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi.h b/drivers/gpu/drm/bridge/synopsys/dw-hdmi.h +index 1999db05bc3b..7ca4bd5041ed 100644 +--- a/drivers/gpu/drm/bridge/synopsys/dw-hdmi.h ++++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi.h +@@ -509,6 +509,51 @@ + #define HDMI_A_PRESETUP 0x501A + #define HDMI_A_SRM_BASE 0x5020 + ++/* CEC Engine Registers */ ++#define HDMI_CEC_CTRL 0x7D00 ++#define HDMI_CEC_STAT 0x7D01 ++#define HDMI_CEC_MASK 0x7D02 ++#define HDMI_CEC_POLARITY 0x7D03 ++#define HDMI_CEC_INT 0x7D04 ++#define HDMI_CEC_ADDR_L 0x7D05 ++#define HDMI_CEC_ADDR_H 0x7D06 ++#define HDMI_CEC_TX_CNT 0x7D07 ++#define HDMI_CEC_RX_CNT 0x7D08 ++#define HDMI_CEC_TX_DATA0 0x7D10 ++#define HDMI_CEC_TX_DATA1 0x7D11 ++#define HDMI_CEC_TX_DATA2 0x7D12 ++#define HDMI_CEC_TX_DATA3 0x7D13 ++#define HDMI_CEC_TX_DATA4 0x7D14 ++#define HDMI_CEC_TX_DATA5 0x7D15 ++#define HDMI_CEC_TX_DATA6 0x7D16 ++#define HDMI_CEC_TX_DATA7 0x7D17 ++#define HDMI_CEC_TX_DATA8 0x7D18 ++#define HDMI_CEC_TX_DATA9 0x7D19 ++#define HDMI_CEC_TX_DATA10 0x7D1a ++#define HDMI_CEC_TX_DATA11 0x7D1b ++#define HDMI_CEC_TX_DATA12 0x7D1c ++#define HDMI_CEC_TX_DATA13 0x7D1d ++#define HDMI_CEC_TX_DATA14 0x7D1e ++#define HDMI_CEC_TX_DATA15 0x7D1f ++#define HDMI_CEC_RX_DATA0 0x7D20 ++#define HDMI_CEC_RX_DATA1 0x7D21 ++#define HDMI_CEC_RX_DATA2 0x7D22 ++#define HDMI_CEC_RX_DATA3 0x7D23 ++#define HDMI_CEC_RX_DATA4 0x7D24 ++#define HDMI_CEC_RX_DATA5 0x7D25 ++#define HDMI_CEC_RX_DATA6 0x7D26 ++#define HDMI_CEC_RX_DATA7 0x7D27 ++#define HDMI_CEC_RX_DATA8 0x7D28 ++#define HDMI_CEC_RX_DATA9 0x7D29 ++#define HDMI_CEC_RX_DATA10 0x7D2a ++#define HDMI_CEC_RX_DATA11 0x7D2b ++#define HDMI_CEC_RX_DATA12 0x7D2c ++#define HDMI_CEC_RX_DATA13 0x7D2d ++#define HDMI_CEC_RX_DATA14 0x7D2e ++#define HDMI_CEC_RX_DATA15 0x7D2f ++#define HDMI_CEC_LOCK 0x7D30 ++#define HDMI_CEC_WKUPCTRL 0x7D31 ++ + /* I2C Master Registers (E-DDC) */ + #define HDMI_I2CM_SLAVE 0x7E00 + #define HDMI_I2CM_ADDRESS 0x7E01 +@@ -529,6 +574,7 @@ + #define HDMI_I2CM_FS_SCL_HCNT_0_ADDR 0x7E10 + #define HDMI_I2CM_FS_SCL_LCNT_1_ADDR 0x7E11 + #define HDMI_I2CM_FS_SCL_LCNT_0_ADDR 0x7E12 ++#define HDMI_I2CM_SDA_HOLD 0x7E13 + + enum { + /* PRODUCT_ID0 field values */ +diff --git a/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.c b/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.c +index 6b268f9445b3..d1fe9347244a 100644 +--- a/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.c ++++ b/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.c +@@ -1246,6 +1246,12 @@ void dw_mipi_dsi_unbind(struct dw_mipi_dsi *dsi) + } + EXPORT_SYMBOL_GPL(dw_mipi_dsi_unbind); + ++struct drm_connector *dw_mipi_dsi_get_connector(struct dw_mipi_dsi *dsi) ++{ ++ return drm_panel_bridge_connector(dsi->panel_bridge); ++} ++EXPORT_SYMBOL_GPL(dw_mipi_dsi_get_connector); ++ + MODULE_AUTHOR("Chris Zhong "); + MODULE_AUTHOR("Philippe Cornu "); + MODULE_DESCRIPTION("DW MIPI DSI host controller driver"); +diff --git a/drivers/gpu/drm/drm_ioctl.c b/drivers/gpu/drm/drm_ioctl.c +index 4606cc938b36..385d3ac4151a 100644 +--- a/drivers/gpu/drm/drm_ioctl.c ++++ b/drivers/gpu/drm/drm_ioctl.c +@@ -537,6 +537,7 @@ int drm_version(struct drm_device *dev, void *data, + */ + int drm_ioctl_permit(u32 flags, struct drm_file *file_priv) + { ++#ifndef CONFIG_DRM_IGNORE_IOTCL_PERMIT + /* ROOT_ONLY is only for CAP_SYS_ADMIN */ + if (unlikely((flags & DRM_ROOT_ONLY) && !capable(CAP_SYS_ADMIN))) + return -EACCES; +@@ -555,6 +556,7 @@ int drm_ioctl_permit(u32 flags, struct drm_file *file_priv) + if (unlikely(!(flags & DRM_RENDER_ALLOW) && + drm_is_render_client(file_priv))) + return -EACCES; ++#endif + + return 0; + } +diff --git a/drivers/gpu/drm/drm_prime.c b/drivers/gpu/drm/drm_prime.c +index 9f955f2010c2..ca6d13eb3190 100644 +--- a/drivers/gpu/drm/drm_prime.c ++++ b/drivers/gpu/drm/drm_prime.c +@@ -780,6 +780,28 @@ int drm_gem_dmabuf_mmap(struct dma_buf *dma_buf, struct vm_area_struct *vma) + } + EXPORT_SYMBOL(drm_gem_dmabuf_mmap); + ++/** ++ * drm_gem_dmabuf_get_uuid - dma_buf get_uuid implementation for GEM ++ * @dma_buf: buffer to query ++ * @uuid: uuid outparam ++ * ++ * Queries the buffer's virtio UUID. This can be used as the ++ * &dma_buf_ops.get_uuid callback. Calls into &drm_driver.gem_prime_get_uuid. ++ * ++ * Returns 0 on success or a negative error code on failure. ++ */ ++int drm_gem_dmabuf_get_uuid(struct dma_buf *dma_buf, uuid_t *uuid) ++{ ++ struct drm_gem_object *obj = dma_buf->priv; ++ struct drm_device *dev = obj->dev; ++ ++ if (!dev->driver->gem_prime_get_uuid) ++ return -ENODEV; ++ ++ return dev->driver->gem_prime_get_uuid(obj, uuid); ++} ++EXPORT_SYMBOL(drm_gem_dmabuf_get_uuid); ++ + static const struct dma_buf_ops drm_gem_prime_dmabuf_ops = { + .cache_sgt_mapping = true, + .attach = drm_gem_map_attach, +@@ -790,6 +812,7 @@ static const struct dma_buf_ops drm_gem_prime_dmabuf_ops = { + .mmap = drm_gem_dmabuf_mmap, + .vmap = drm_gem_dmabuf_vmap, + .vunmap = drm_gem_dmabuf_vunmap, ++ .get_uuid = drm_gem_dmabuf_get_uuid, + }; + + /** +diff --git a/drivers/gpu/drm/i915/gt/intel_engine_cs.c b/drivers/gpu/drm/i915/gt/intel_engine_cs.c +index c940ac3aae2f..a19537706ed1 100644 +--- a/drivers/gpu/drm/i915/gt/intel_engine_cs.c ++++ b/drivers/gpu/drm/i915/gt/intel_engine_cs.c +@@ -305,9 +305,8 @@ static int intel_engine_setup(struct intel_gt *gt, enum intel_engine_id id) + engine->i915 = i915; + engine->gt = gt; + engine->uncore = gt->uncore; ++ engine->hw_id = engine->guc_id = info->hw_id; + engine->mmio_base = __engine_mmio_base(i915, info->mmio_bases); +- engine->hw_id = info->hw_id; +- engine->guc_id = MAKE_GUC_ID(info->class, info->instance); + + engine->class = info->class; + engine->instance = info->instance; +diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc.c b/drivers/gpu/drm/i915/gt/uc/intel_guc.c +index 6909da1e1a73..942c7c187adb 100644 +--- a/drivers/gpu/drm/i915/gt/uc/intel_guc.c ++++ b/drivers/gpu/drm/i915/gt/uc/intel_guc.c +@@ -213,6 +213,23 @@ static u32 guc_ctl_feature_flags(struct intel_guc *guc) + return flags; + } + ++static u32 guc_ctl_ctxinfo_flags(struct intel_guc *guc) ++{ ++ u32 flags = 0; ++ ++ if (intel_guc_submission_is_used(guc)) { ++ u32 ctxnum, base; ++ ++ base = intel_guc_ggtt_offset(guc, guc->stage_desc_pool); ++ ctxnum = GUC_MAX_STAGE_DESCRIPTORS / 16; ++ ++ base >>= PAGE_SHIFT; ++ flags |= (base << GUC_CTL_BASE_ADDR_SHIFT) | ++ (ctxnum << GUC_CTL_CTXNUM_IN16_SHIFT); ++ } ++ return flags; ++} ++ + static u32 guc_ctl_log_params_flags(struct intel_guc *guc) + { + u32 offset = intel_guc_ggtt_offset(guc, guc->log.vma) >> PAGE_SHIFT; +@@ -274,6 +291,7 @@ static void guc_init_params(struct intel_guc *guc) + + BUILD_BUG_ON(sizeof(guc->params) != GUC_CTL_MAX_DWORDS * sizeof(u32)); + ++ params[GUC_CTL_CTXINFO] = guc_ctl_ctxinfo_flags(guc); + params[GUC_CTL_LOG_PARAMS] = guc_ctl_log_params_flags(guc); + params[GUC_CTL_FEATURE] = guc_ctl_feature_flags(guc); + params[GUC_CTL_DEBUG] = guc_ctl_debug_flags(guc); +diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c +index 7950d28beb8c..d44061033f23 100644 +--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c ++++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c +@@ -10,52 +10,11 @@ + + /* + * The Additional Data Struct (ADS) has pointers for different buffers used by +- * the GuC. One single gem object contains the ADS struct itself (guc_ads) and +- * all the extra buffers indirectly linked via the ADS struct's entries. +- * +- * Layout of the ADS blob allocated for the GuC: +- * +- * +---------------------------------------+ <== base +- * | guc_ads | +- * +---------------------------------------+ +- * | guc_policies | +- * +---------------------------------------+ +- * | guc_gt_system_info | +- * +---------------------------------------+ +- * | guc_clients_info | +- * +---------------------------------------+ +- * | guc_ct_pool_entry[size] | +- * +---------------------------------------+ +- * | padding | +- * +---------------------------------------+ <== 4K aligned +- * | private data | +- * +---------------------------------------+ +- * | padding | +- * +---------------------------------------+ <== 4K aligned ++ * the GuC. One single gem object contains the ADS struct itself (guc_ads), the ++ * scheduling policies (guc_policies), a structure describing a collection of ++ * register sets (guc_mmio_reg_state) and some extra pages for the GuC to save ++ * its internal state for sleep. + */ +-struct __guc_ads_blob { +- struct guc_ads ads; +- struct guc_policies policies; +- struct guc_gt_system_info system_info; +- struct guc_clients_info clients_info; +- struct guc_ct_pool_entry ct_pool[GUC_CT_POOL_SIZE]; +-} __packed; +- +-static u32 guc_ads_private_data_size(struct intel_guc *guc) +-{ +- return PAGE_ALIGN(guc->fw.private_data_size); +-} +- +-static u32 guc_ads_private_data_offset(struct intel_guc *guc) +-{ +- return PAGE_ALIGN(sizeof(struct __guc_ads_blob)); +-} +- +-static u32 guc_ads_blob_size(struct intel_guc *guc) +-{ +- return guc_ads_private_data_offset(guc) + +- guc_ads_private_data_size(guc); +-} + + static void guc_policy_init(struct guc_policy *policy) + { +@@ -89,37 +48,26 @@ static void guc_ct_pool_entries_init(struct guc_ct_pool_entry *pool, u32 num) + memset(pool, 0, num * sizeof(*pool)); + } + +-static void guc_mapping_table_init(struct intel_gt *gt, +- struct guc_gt_system_info *system_info) +-{ +- unsigned int i, j; +- struct intel_engine_cs *engine; +- enum intel_engine_id id; +- +- /* Table must be set to invalid values for entries not used */ +- for (i = 0; i < GUC_MAX_ENGINE_CLASSES; ++i) +- for (j = 0; j < GUC_MAX_INSTANCES_PER_CLASS; ++j) +- system_info->mapping_table[i][j] = +- GUC_MAX_INSTANCES_PER_CLASS; +- +- for_each_engine(engine, gt, id) { +- u8 guc_class = engine->class; +- +- system_info->mapping_table[guc_class][engine->instance] = +- engine->instance; +- } +-} +- + /* + * The first 80 dwords of the register state context, containing the + * execlists and ppgtt registers. + */ + #define LR_HW_CONTEXT_SIZE (80 * sizeof(u32)) + ++/* The ads obj includes the struct itself and buffers passed to GuC */ ++struct __guc_ads_blob { ++ struct guc_ads ads; ++ struct guc_policies policies; ++ struct guc_mmio_reg_state reg_state; ++ struct guc_gt_system_info system_info; ++ struct guc_clients_info clients_info; ++ struct guc_ct_pool_entry ct_pool[GUC_CT_POOL_SIZE]; ++ u8 reg_state_buffer[GUC_S3_SAVE_SPACE_PAGES * PAGE_SIZE]; ++} __packed; ++ + static void __guc_ads_init(struct intel_guc *guc) + { + struct intel_gt *gt = guc_to_gt(guc); +- struct drm_i915_private *i915 = gt->i915; + struct __guc_ads_blob *blob = guc->ads_blob; + const u32 skipped_size = LRC_PPHWSP_SZ * PAGE_SIZE + LR_HW_CONTEXT_SIZE; + u32 base; +@@ -151,25 +99,13 @@ static void __guc_ads_init(struct intel_guc *guc) + } + + /* System info */ +- blob->system_info.engine_enabled_masks[RENDER_CLASS] = 1; +- blob->system_info.engine_enabled_masks[COPY_ENGINE_CLASS] = 1; +- blob->system_info.engine_enabled_masks[VIDEO_DECODE_CLASS] = VDBOX_MASK(gt); +- blob->system_info.engine_enabled_masks[VIDEO_ENHANCEMENT_CLASS] = VEBOX_MASK(gt); +- +- blob->system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_SLICE_ENABLED] = +- hweight8(gt->info.sseu.slice_mask); +- blob->system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_VDBOX_SFC_SUPPORT_MASK] = +- gt->info.vdbox_sfc_access; +- +- if (INTEL_GEN(i915) >= 12 && !IS_DGFX(i915)) { +- u32 distdbreg = intel_uncore_read(gt->uncore, +- GEN12_DIST_DBS_POPULATED); +- blob->system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_DOORBELL_COUNT_PER_SQIDI] = +- ((distdbreg >> GEN12_DOORBELLS_PER_SQIDI_SHIFT) & +- GEN12_DOORBELLS_PER_SQIDI) + 1; +- } ++ blob->system_info.slice_enabled = hweight8(gt->info.sseu.slice_mask); ++ blob->system_info.rcs_enabled = 1; ++ blob->system_info.bcs_enabled = 1; + +- guc_mapping_table_init(guc_to_gt(guc), &blob->system_info); ++ blob->system_info.vdbox_enable_mask = VDBOX_MASK(gt); ++ blob->system_info.vebox_enable_mask = VEBOX_MASK(gt); ++ blob->system_info.vdbox_sfc_support_mask = gt->info.vdbox_sfc_access; + + base = intel_guc_ggtt_offset(guc, guc->ads_vma); + +@@ -182,12 +118,11 @@ static void __guc_ads_init(struct intel_guc *guc) + + /* ADS */ + blob->ads.scheduler_policies = base + ptr_offset(blob, policies); ++ blob->ads.reg_state_buffer = base + ptr_offset(blob, reg_state_buffer); ++ blob->ads.reg_state_addr = base + ptr_offset(blob, reg_state); + blob->ads.gt_system_info = base + ptr_offset(blob, system_info); + blob->ads.clients_info = base + ptr_offset(blob, clients_info); + +- /* Private Data */ +- blob->ads.private_data = base + guc_ads_private_data_offset(guc); +- + i915_gem_object_flush_map(guc->ads_vma->obj); + } + +@@ -200,15 +135,14 @@ static void __guc_ads_init(struct intel_guc *guc) + */ + int intel_guc_ads_create(struct intel_guc *guc) + { +- u32 size; ++ const u32 size = PAGE_ALIGN(sizeof(struct __guc_ads_blob)); + int ret; + + GEM_BUG_ON(guc->ads_vma); + +- size = guc_ads_blob_size(guc); +- + ret = intel_guc_allocate_and_map_vma(guc, size, &guc->ads_vma, + (void **)&guc->ads_blob); ++ + if (ret) + return ret; + +@@ -222,18 +156,6 @@ void intel_guc_ads_destroy(struct intel_guc *guc) + i915_vma_unpin_and_release(&guc->ads_vma, I915_VMA_RELEASE_MAP); + } + +-static void guc_ads_private_data_reset(struct intel_guc *guc) +-{ +- u32 size; +- +- size = guc_ads_private_data_size(guc); +- if (!size) +- return; +- +- memset((void *)guc->ads_blob + guc_ads_private_data_offset(guc), 0, +- size); +-} +- + /** + * intel_guc_ads_reset() - prepares GuC Additional Data Struct for reuse + * @guc: intel_guc struct +@@ -246,8 +168,5 @@ void intel_guc_ads_reset(struct intel_guc *guc) + { + if (!guc->ads_vma) + return; +- + __guc_ads_init(guc); +- +- guc_ads_private_data_reset(guc); + } +diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h +index 79c560d9c0b6..a6b733c146c9 100644 +--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h ++++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h +@@ -26,8 +26,8 @@ + #define GUC_VIDEO_ENGINE2 4 + #define GUC_MAX_ENGINES_NUM (GUC_VIDEO_ENGINE2 + 1) + +-#define GUC_MAX_ENGINE_CLASSES 16 +-#define GUC_MAX_INSTANCES_PER_CLASS 32 ++#define GUC_MAX_ENGINE_CLASSES 5 ++#define GUC_MAX_INSTANCES_PER_CLASS 16 + + #define GUC_DOORBELL_INVALID 256 + +@@ -62,7 +62,12 @@ + #define GUC_STAGE_DESC_ATTR_PCH BIT(6) + #define GUC_STAGE_DESC_ATTR_TERMINATED BIT(7) + +-#define GUC_CTL_LOG_PARAMS 0 ++/* New GuC control data */ ++#define GUC_CTL_CTXINFO 0 ++#define GUC_CTL_CTXNUM_IN16_SHIFT 0 ++#define GUC_CTL_BASE_ADDR_SHIFT 12 ++ ++#define GUC_CTL_LOG_PARAMS 1 + #define GUC_LOG_VALID (1 << 0) + #define GUC_LOG_NOTIFY_ON_HALF_FULL (1 << 1) + #define GUC_LOG_ALLOC_IN_MEGABYTE (1 << 3) +@@ -74,11 +79,11 @@ + #define GUC_LOG_ISR_MASK (0x7 << GUC_LOG_ISR_SHIFT) + #define GUC_LOG_BUF_ADDR_SHIFT 12 + +-#define GUC_CTL_WA 1 +-#define GUC_CTL_FEATURE 2 ++#define GUC_CTL_WA 2 ++#define GUC_CTL_FEATURE 3 + #define GUC_CTL_DISABLE_SCHEDULER (1 << 14) + +-#define GUC_CTL_DEBUG 3 ++#define GUC_CTL_DEBUG 4 + #define GUC_LOG_VERBOSITY_SHIFT 0 + #define GUC_LOG_VERBOSITY_LOW (0 << GUC_LOG_VERBOSITY_SHIFT) + #define GUC_LOG_VERBOSITY_MED (1 << GUC_LOG_VERBOSITY_SHIFT) +@@ -92,37 +97,12 @@ + #define GUC_LOG_DISABLED (1 << 6) + #define GUC_PROFILE_ENABLED (1 << 7) + +-#define GUC_CTL_ADS 4 ++#define GUC_CTL_ADS 5 + #define GUC_ADS_ADDR_SHIFT 1 + #define GUC_ADS_ADDR_MASK (0xFFFFF << GUC_ADS_ADDR_SHIFT) + + #define GUC_CTL_MAX_DWORDS (SOFT_SCRATCH_COUNT - 2) /* [1..14] */ + +-/* Generic GT SysInfo data types */ +-#define GUC_GENERIC_GT_SYSINFO_SLICE_ENABLED 0 +-#define GUC_GENERIC_GT_SYSINFO_VDBOX_SFC_SUPPORT_MASK 1 +-#define GUC_GENERIC_GT_SYSINFO_DOORBELL_COUNT_PER_SQIDI 2 +-#define GUC_GENERIC_GT_SYSINFO_MAX 16 +- +-/* +- * The class goes in bits [0..2] of the GuC ID, the instance in bits [3..6]. +- * Bit 7 can be used for operations that apply to all engine classes&instances. +- */ +-#define GUC_ENGINE_CLASS_SHIFT 0 +-#define GUC_ENGINE_CLASS_MASK (0x7 << GUC_ENGINE_CLASS_SHIFT) +-#define GUC_ENGINE_INSTANCE_SHIFT 3 +-#define GUC_ENGINE_INSTANCE_MASK (0xf << GUC_ENGINE_INSTANCE_SHIFT) +-#define GUC_ENGINE_ALL_INSTANCES BIT(7) +- +-#define MAKE_GUC_ID(class, instance) \ +- (((class) << GUC_ENGINE_CLASS_SHIFT) | \ +- ((instance) << GUC_ENGINE_INSTANCE_SHIFT)) +- +-#define GUC_ID_TO_ENGINE_CLASS(guc_id) \ +- (((guc_id) & GUC_ENGINE_CLASS_MASK) >> GUC_ENGINE_CLASS_SHIFT) +-#define GUC_ID_TO_ENGINE_INSTANCE(guc_id) \ +- (((guc_id) & GUC_ENGINE_INSTANCE_MASK) >> GUC_ENGINE_INSTANCE_SHIFT) +- + /* Work item for submitting workloads into work queue of GuC. */ + struct guc_wq_item { + u32 header; +@@ -356,6 +336,11 @@ struct guc_policies { + } __packed; + + /* GuC MMIO reg state struct */ ++ ++ ++#define GUC_REGSET_MAX_REGISTERS 64 ++#define GUC_S3_SAVE_SPACE_PAGES 10 ++ + struct guc_mmio_reg { + u32 offset; + u32 value; +@@ -363,18 +348,28 @@ struct guc_mmio_reg { + #define GUC_REGSET_MASKED (1 << 0) + } __packed; + ++struct guc_mmio_regset { ++ struct guc_mmio_reg registers[GUC_REGSET_MAX_REGISTERS]; ++ u32 values_valid; ++ u32 number_of_registers; ++} __packed; ++ + /* GuC register sets */ +-struct guc_mmio_reg_set { +- u32 address; +- u16 count; +- u16 reserved; ++struct guc_mmio_reg_state { ++ struct guc_mmio_regset engine_reg[GUC_MAX_ENGINE_CLASSES][GUC_MAX_INSTANCES_PER_CLASS]; ++ u32 reserved[98]; + } __packed; + + /* HW info */ + struct guc_gt_system_info { +- u8 mapping_table[GUC_MAX_ENGINE_CLASSES][GUC_MAX_INSTANCES_PER_CLASS]; +- u32 engine_enabled_masks[GUC_MAX_ENGINE_CLASSES]; +- u32 generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_MAX]; ++ u32 slice_enabled; ++ u32 rcs_enabled; ++ u32 reserved0; ++ u32 bcs_enabled; ++ u32 vdbox_enable_mask; ++ u32 vdbox_sfc_support_mask; ++ u32 vebox_enable_mask; ++ u32 reserved[9]; + } __packed; + + /* Clients info */ +@@ -395,16 +390,15 @@ struct guc_clients_info { + + /* GuC Additional Data Struct */ + struct guc_ads { +- struct guc_mmio_reg_set reg_state_list[GUC_MAX_ENGINE_CLASSES][GUC_MAX_INSTANCES_PER_CLASS]; +- u32 reserved0; ++ u32 reg_state_addr; ++ u32 reg_state_buffer; + u32 scheduler_policies; + u32 gt_system_info; + u32 clients_info; + u32 control_data; + u32 golden_context_lrca[GUC_MAX_ENGINE_CLASSES]; + u32 eng_state_size[GUC_MAX_ENGINE_CLASSES]; +- u32 private_data; +- u32 reserved[15]; ++ u32 reserved[16]; + } __packed; + + /* GuC logging structures */ +diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_reg.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_reg.h +index b37fc2ffaef2..1949346e714e 100644 +--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_reg.h ++++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_reg.h +@@ -118,11 +118,6 @@ struct guc_doorbell_info { + #define GEN8_DRB_VALID (1<<0) + #define GEN8_DRBREGU(x) _MMIO(0x1000 + (x) * 8 + 4) + +-#define GEN12_DIST_DBS_POPULATED _MMIO(0xd08) +-#define GEN12_DOORBELLS_PER_SQIDI_SHIFT 16 +-#define GEN12_DOORBELLS_PER_SQIDI (0xff) +-#define GEN12_SQIDIS_DOORBELL_EXIST (0xffff) +- + #define DE_GUCRMR _MMIO(0x44054) + + #define GUC_BCS_RCS_IER _MMIO(0xC550) +diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c +index ee4ac3922277..80e8b6c3bc8c 100644 +--- a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c ++++ b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c +@@ -44,19 +44,23 @@ void intel_uc_fw_change_status(struct intel_uc_fw *uc_fw, + * List of required GuC and HuC binaries per-platform. + * Must be ordered based on platform + revid, from newer to older. + * ++ * TGL 35.2 is interface-compatible with 33.0 for previous Gens. The deltas ++ * between 33.0 and 35.2 are only related to new additions to support new Gen12 ++ * features. ++ * + * Note that RKL uses the same firmware as TGL. + */ + #define INTEL_UC_FIRMWARE_DEFS(fw_def, guc_def, huc_def) \ +- fw_def(ROCKETLAKE, 0, guc_def(tgl, 49, 0, 1), huc_def(tgl, 7, 5, 0)) \ +- fw_def(TIGERLAKE, 0, guc_def(tgl, 49, 0, 1), huc_def(tgl, 7, 5, 0)) \ +- fw_def(ELKHARTLAKE, 0, guc_def(ehl, 49, 0, 1), huc_def(ehl, 9, 0, 0)) \ +- fw_def(ICELAKE, 0, guc_def(icl, 49, 0, 1), huc_def(icl, 9, 0, 0)) \ +- fw_def(COMETLAKE, 5, guc_def(cml, 49, 0, 1), huc_def(cml, 4, 0, 0)) \ +- fw_def(COFFEELAKE, 0, guc_def(kbl, 49, 0, 1), huc_def(kbl, 4, 0, 0)) \ +- fw_def(GEMINILAKE, 0, guc_def(glk, 49, 0, 1), huc_def(glk, 4, 0, 0)) \ +- fw_def(KABYLAKE, 0, guc_def(kbl, 49, 0, 1), huc_def(kbl, 4, 0, 0)) \ +- fw_def(BROXTON, 0, guc_def(bxt, 49, 0, 1), huc_def(bxt, 2, 0, 0)) \ +- fw_def(SKYLAKE, 0, guc_def(skl, 49, 0, 1), huc_def(skl, 2, 0, 0)) ++ fw_def(ROCKETLAKE, 0, guc_def(tgl, 35, 2, 0), huc_def(tgl, 7, 5, 0)) \ ++ fw_def(TIGERLAKE, 0, guc_def(tgl, 35, 2, 0), huc_def(tgl, 7, 5, 0)) \ ++ fw_def(ELKHARTLAKE, 0, guc_def(ehl, 33, 0, 4), huc_def(ehl, 9, 0, 0)) \ ++ fw_def(ICELAKE, 0, guc_def(icl, 33, 0, 0), huc_def(icl, 9, 0, 0)) \ ++ fw_def(COMETLAKE, 5, guc_def(cml, 33, 0, 0), huc_def(cml, 4, 0, 0)) \ ++ fw_def(COFFEELAKE, 0, guc_def(kbl, 33, 0, 0), huc_def(kbl, 4, 0, 0)) \ ++ fw_def(GEMINILAKE, 0, guc_def(glk, 33, 0, 0), huc_def(glk, 4, 0, 0)) \ ++ fw_def(KABYLAKE, 0, guc_def(kbl, 33, 0, 0), huc_def(kbl, 4, 0, 0)) \ ++ fw_def(BROXTON, 0, guc_def(bxt, 33, 0, 0), huc_def(bxt, 2, 0, 0)) \ ++ fw_def(SKYLAKE, 0, guc_def(skl, 33, 0, 0), huc_def(skl, 2, 0, 0)) + + #define __MAKE_UC_FW_PATH(prefix_, name_, major_, minor_, patch_) \ + "i915/" \ +@@ -367,9 +371,6 @@ int intel_uc_fw_fetch(struct intel_uc_fw *uc_fw) + } + } + +- if (uc_fw->type == INTEL_UC_FW_TYPE_GUC) +- uc_fw->private_data_size = css->private_data_size; +- + obj = i915_gem_object_create_shmem_from_data(i915, fw->data, fw->size); + if (IS_ERR(obj)) { + err = PTR_ERR(obj); +diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.h b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.h +index 99bb1fe1af66..23d3a423ac0f 100644 +--- a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.h ++++ b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.h +@@ -88,8 +88,6 @@ struct intel_uc_fw { + + u32 rsa_size; + u32 ucode_size; +- +- u32 private_data_size; + }; + + #ifdef CONFIG_DRM_I915_DEBUG_GUC +diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw_abi.h b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw_abi.h +index e41ffc7a7fbc..029214cdedd5 100644 +--- a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw_abi.h ++++ b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw_abi.h +@@ -69,11 +69,7 @@ struct uc_css_header { + #define CSS_SW_VERSION_UC_MAJOR (0xFF << 16) + #define CSS_SW_VERSION_UC_MINOR (0xFF << 8) + #define CSS_SW_VERSION_UC_PATCH (0xFF << 0) +- u32 reserved0[13]; +- union { +- u32 private_data_size; /* only applies to GuC */ +- u32 reserved1; +- }; ++ u32 reserved[14]; + u32 header_info; + } __packed; + static_assert(sizeof(struct uc_css_header) == 128); +diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c +index ac96b6ab44c0..42fc5c813a9b 100644 +--- a/drivers/gpu/drm/nouveau/nouveau_drm.c ++++ b/drivers/gpu/drm/nouveau/nouveau_drm.c +@@ -557,7 +557,6 @@ nouveau_drm_device_init(struct drm_device *dev) + nvkm_dbgopt(nouveau_debug, "DRM"); + + INIT_LIST_HEAD(&drm->clients); +- mutex_init(&drm->clients_lock); + spin_lock_init(&drm->tile.lock); + + /* workaround an odd issue on nvc1 by disabling the device's +@@ -628,7 +627,6 @@ nouveau_drm_device_init(struct drm_device *dev) + static void + nouveau_drm_device_fini(struct drm_device *dev) + { +- struct nouveau_cli *cli, *temp_cli; + struct nouveau_drm *drm = nouveau_drm(dev); + + if (nouveau_pmops_runtime()) { +@@ -653,28 +651,9 @@ nouveau_drm_device_fini(struct drm_device *dev) + nouveau_ttm_fini(drm); + nouveau_vga_fini(drm); + +- /* +- * There may be existing clients from as-yet unclosed files. For now, +- * clean them up here rather than deferring until the file is closed, +- * but this likely not correct if we want to support hot-unplugging +- * properly. +- */ +- mutex_lock(&drm->clients_lock); +- list_for_each_entry_safe(cli, temp_cli, &drm->clients, head) { +- list_del(&cli->head); +- mutex_lock(&cli->mutex); +- if (cli->abi16) +- nouveau_abi16_fini(cli->abi16); +- mutex_unlock(&cli->mutex); +- nouveau_cli_fini(cli); +- kfree(cli); +- } +- mutex_unlock(&drm->clients_lock); +- + nouveau_cli_fini(&drm->client); + nouveau_cli_fini(&drm->master); + nvif_parent_dtor(&drm->parent); +- mutex_destroy(&drm->clients_lock); + kfree(drm); + } + +@@ -813,7 +792,7 @@ nouveau_drm_device_remove(struct drm_device *dev) + struct nvkm_client *client; + struct nvkm_device *device; + +- drm_dev_unplug(dev); ++ drm_dev_unregister(dev); + + dev->irq_enabled = false; + client = nvxx_client(&drm->client.base); +@@ -1107,9 +1086,9 @@ nouveau_drm_open(struct drm_device *dev, struct drm_file *fpriv) + + fpriv->driver_priv = cli; + +- mutex_lock(&drm->clients_lock); ++ mutex_lock(&drm->client.mutex); + list_add(&cli->head, &drm->clients); +- mutex_unlock(&drm->clients_lock); ++ mutex_unlock(&drm->client.mutex); + + done: + if (ret && cli) { +@@ -1127,16 +1106,6 @@ nouveau_drm_postclose(struct drm_device *dev, struct drm_file *fpriv) + { + struct nouveau_cli *cli = nouveau_cli(fpriv); + struct nouveau_drm *drm = nouveau_drm(dev); +- int dev_index; +- +- /* +- * The device is gone, and as it currently stands all clients are +- * cleaned up in the removal codepath. In the future this may change +- * so that we can support hot-unplugging, but for now we immediately +- * return to avoid a double-free situation. +- */ +- if (!drm_dev_enter(dev, &dev_index)) +- return; + + pm_runtime_get_sync(dev->dev); + +@@ -1145,15 +1114,14 @@ nouveau_drm_postclose(struct drm_device *dev, struct drm_file *fpriv) + nouveau_abi16_fini(cli->abi16); + mutex_unlock(&cli->mutex); + +- mutex_lock(&drm->clients_lock); ++ mutex_lock(&drm->client.mutex); + list_del(&cli->head); +- mutex_unlock(&drm->clients_lock); ++ mutex_unlock(&drm->client.mutex); + + nouveau_cli_fini(cli); + kfree(cli); + pm_runtime_mark_last_busy(dev->dev); + pm_runtime_put_autosuspend(dev->dev); +- drm_dev_exit(dev_index); + } + + static const struct drm_ioctl_desc +diff --git a/drivers/gpu/drm/nouveau/nouveau_drv.h b/drivers/gpu/drm/nouveau/nouveau_drv.h +index 8b252dca0fc3..b8025507a9e4 100644 +--- a/drivers/gpu/drm/nouveau/nouveau_drv.h ++++ b/drivers/gpu/drm/nouveau/nouveau_drv.h +@@ -142,11 +142,6 @@ struct nouveau_drm { + + struct list_head clients; + +- /** +- * @clients_lock: Protects access to the @clients list of &struct nouveau_cli. +- */ +- struct mutex clients_lock; +- + u8 old_pm_cap; + + struct { +diff --git a/drivers/gpu/drm/panel/panel-simple.c b/drivers/gpu/drm/panel/panel-simple.c +index 204674fccd64..b1af4a039fb4 100644 +--- a/drivers/gpu/drm/panel/panel-simple.c ++++ b/drivers/gpu/drm/panel/panel-simple.c +@@ -30,6 +30,7 @@ + #include + + #include